/*
 *  debugfs.h - a tiny little debug file system
 *
 *  Copyright (C) 2004 Greg Kroah-Hartman <greg@kroah.com>
 *  Copyright (C) 2004 IBM Inc.
 *
 *	This program is free software; you can redistribute it and/or
 *	modify it under the terms of the GNU General Public License version
 *	2 as published by the Free Software Foundation.
 *
 *  debugfs is for people to use instead of /proc or /sys.
 *  See Documentation/DocBook/filesystems for more details.
 */

#ifndef _DEBUGFS_H_
#define _DEBUGFS_H_

#include <linux/fs.h>

#include <linux/types.h>

struct file_operations;

struct debugfs_blob_wrapper {
	void *data;
	unsigned long size;
};

extern struct dentry *arch_debugfs_dir;

#if defined(CONFIG_DEBUG_FS)

/* declared over in file.c */
extern const struct file_operations debugfs_file_operations;
extern const struct inode_operations debugfs_link_operations;

struct dentry *debugfs_create_file(const char *name, mode_t mode,
				   struct dentry *parent, void *data,
				   const struct file_operations *fops);

struct dentry *debugfs_create_dir(const char *name, struct dentry *parent);

struct dentry *debugfs_create_symlink(const char *name, struct dentry *parent,
				      const char *dest);

void debugfs_remove(struct dentry *dentry);
void debugfs_remove_recursive(struct dentry *dentry);

struct dentry *debugfs_rename(struct dentry *old_dir, struct dentry *old_dentry,
                struct dentry *new_dir, const char *new_name);

struct dentry *debugfs_create_u8(const char *name, mode_t mode,
				 struct dentry *parent, u8 *value);
struct dentry *debugfs_create_u16(const char *name, mode_t mode,
				  struct dentry *parent, u16 *value);
struct dentry *debugfs_create_u32(const char *name, mode_t mode,
				  struct dentry *parent, u32 *value);
struct dentry *debugfs_create_u64(const char *name, mode_t mode,
				  struct dentry *parent, u64 *value);
struct dentry *debugfs_create_x8(const char *name, mode_t mode,
				 struct dentry *parent, u8 *value);
struct dentry *debugfs_create_x16(const char *name, mode_t mode,
				  struct dentry *parent, u16 *value);
struct dentry *debugfs_create_x32(const char *name, mode_t mode,
				  struct dentry *parent, u32 *value);
struct dentry *debugfs_create_x64(const char *name, mode_t mode,
				  struct dentry *parent, u64 *value);
struct dentry *debugfs_create_size_t(const char *name, mode_t mode,
				     struct dentry *parent, size_t *value);
struct dentry *debugfs_create_bool(const char *name, mode_t mode,
				  struct dentry *parent, u32 *value);

struct dentry *debugfs_create_blob(const char *name, mode_t mode,
				  struct dentry *parent,
				  struct debugfs_blob_wrapper *blob);

bool debugfs_initialized(void);

#else

#include <linux/err.h>

/* 
 * We do not return NULL from these functions if CONFIG_DEBUG_FS is not enabled
 * so users have a chance to detect if there was a real error or not.  We don't
 * want to duplicate the design decision mistakes of procfs and devfs again.
 */

static inline struct dentry *debugfs_create_file(const char *name, mode_t mode,
					struct dentry *parent, void *data,
					const struct file_operations *fops)
{
	return ERR_PTR(-ENODEV);
}

static inline struct dentry *debugfs_create_dir(const char *name,
						struct dentry *parent)
{
	return ERR_PTR(-ENODEV);
}

static inline struct dentry *debugfs_create_symlink(const char *name,
						    struct dentry *parent,
						    const char *dest)
{
	return ERR_PTR(-ENODEV);
}

static inline void debugfs_remove(struct dentry *dentry)
{ }

static inline void debugfs_remove_recursive(struct dentry *dentry)
{ }

static inline struct dentry *debugfs_rename(struct dentry *old_dir, struct dentry *old_dentry,
                struct dentry *new_dir, char *new_name)
{
	return ERR_PTR(-ENODEV);
}

static inline struct dentry *debugfs_create_u8(const char *name, mode_t mode,
					       struct dentry *parent,
					       u8 *value)
{
	return ERR_PTR(-ENODEV);
}

static inline struct dentry *debugfs_create_u16(const char *name, mode_t mode,
						struct dentry *parent,
						u16 *value)
{
	return ERR_PTR(-ENODEV);
}

static inline struct dentry *debugfs_create_u32(const char *name, mode_t mode,
						struct dentry *parent,
						u32 *value)
{
	return ERR_PTR(-ENODEV);
}

static inline struct dentry *debugfs_create_u64(const char *name, mode_t mode,
						struct dentry *parent,
						u64 *value)
{
	return ERR_PTR(-ENODEV);
}

static inline struct dentry *debugfs_create_x8(const char *name, mode_t mode,
					       struct dentry *parent,
					       u8 *value)
{
	return ERR_PTR(-ENODEV);
}

static inline struct dentry *debugfs_create_x16(const char *name, mode_t mode,
						struct dentry *parent,
						u16 *value)
{
	return ERR_PTR(-ENODEV);
}

static inline struct dentry *debugfs_create_x32(const char *name, mode_t mode,
						struct dentry *parent,
						u32 *value)
{
	return ERR_PTR(-ENODEV);
}

static inline struct dentry *debugfs_create_size_t(const char *name, mode_t mode,
				     struct dentry *parent,
				     size_t *value)
{
	return ERR_PTR(-ENODEV);
}

static inline struct dentry *debugfs_create_bool(const char *name, mode_t mode,
						 struct dentry *parent,
						 u32 *value)
{
	return ERR_PTR(-ENODEV);
}

static inline struct dentry *debugfs_create_blob(const char *name, mode_t mode,
				  struct dentry *parent,
				  struct debugfs_blob_wrapper *blob)
{
	return ERR_PTR(-ENODEV);
}

static inline bool debugfs_initialized(void)
{
	return false;
}

#endif

#endif
ath'>path: <a href='/cgit/linux-arm.git/diff/?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>root</a>/<a href='/cgit/linux-arm.git/diff/drivers?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers</a>/<a href='/cgit/linux-arm.git/diff/drivers/infiniband?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>infiniband</a></div><div class='content'><div class='cgit-panel'><b>diff options</b><form method='get'><input type='hidden' name='id2' value='84b0d6a7a11ec976da347d50de9d5a556743de16'/><table><tr><td colspan='2'/></tr><tr><td class='label'>context:</td><td class='ctrl'><select name='context' onchange='this.form.submit();'><option value='1'>1</option><option value='2'>2</option><option value='3' selected='selected'>3</option><option value='4'>4</option><option value='5'>5</option><option value='6'>6</option><option value='7'>7</option><option value='8'>8</option><option value='9'>9</option><option value='10'>10</option><option value='15'>15</option><option value='20'>20</option><option value='25'>25</option><option value='30'>30</option><option value='35'>35</option><option value='40'>40</option></select></td></tr><tr><td class='label'>space:</td><td class='ctrl'><select name='ignorews' onchange='this.form.submit();'><option value='0' selected='selected'>include</option><option value='1'>ignore</option></select></td></tr><tr><td class='label'>mode:</td><td class='ctrl'><select name='dt' onchange='this.form.submit();'><option value='0' selected='selected'>unified</option><option value='1'>ssdiff</option><option value='2'>stat only</option></select></td></tr><tr><td/><td class='ctrl'><noscript><input type='submit' value='reload'/></noscript></td></tr></table></form></div><div class='diffstat-header'><a href='/cgit/linux-arm.git/diff/?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>Diffstat</a> (limited to 'drivers/infiniband')</div><table summary='diffstat' class='diffstat'><tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/Kconfig</a></td><td class='right'>64</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/Makefile</a></td><td class='right'>1</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/Makefile</a></td><td class='right'>23</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/addr.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/addr.c</a></td><td class='right'>128</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.6%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 98.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/agent.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/agent.c</a></td><td class='right'>35</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/cache.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cache.c</a></td><td class='right'>504</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.1%;'/><td class='rem' style='width: 2.1%;'/><td class='none' style='width: 94.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/cgroup.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cgroup.c</a></td><td class='right'>10</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/cm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cm.c</a></td><td class='right'>3236</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 16.7%;'/><td class='rem' style='width: 16.4%;'/><td class='none' style='width: 66.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/cm_msgs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cm_msgs.h</a></td><td class='right'>791</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 7.9%;'/><td class='none' style='width: 91.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/cm_trace.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cm_trace.c</a></td><td class='right'>15</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/cm_trace.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cm_trace.h</a></td><td class='right'>414</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 95.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/cma.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cma.c</a></td><td class='right'>2482</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 17.3%;'/><td class='rem' style='width: 8.1%;'/><td class='none' style='width: 74.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/cma_configfs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cma_configfs.c</a></td><td class='right'>67</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/cma_priv.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cma_priv.h</a></td><td class='right'>34</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/cma_trace.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cma_trace.c</a></td><td class='right'>16</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/cma_trace.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cma_trace.h</a></td><td class='right'>361</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 96.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/core_priv.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/core_priv.h</a></td><td class='right'>135</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.8%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 98.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/counters.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/counters.c</a></td><td class='right'>681</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 7.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 93.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/cq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cq.c</a></td><td class='right'>361</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.3%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 96.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/device.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/device.c</a></td><td class='right'>1200</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 9.1%;'/><td class='rem' style='width: 3.2%;'/><td class='none' style='width: 87.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/fmr_pool.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/fmr_pool.c</a></td><td class='right'>507</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 5.2%;'/><td class='none' style='width: 94.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/ib_core_uverbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/ib_core_uverbs.c</a></td><td class='right'>367</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.8%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 96.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/iwcm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/iwcm.c</a></td><td class='right'>143</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.8%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 98.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/iwcm.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/iwcm.h</a></td><td class='right'>2</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/iwpm_msg.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/iwpm_msg.c</a></td><td class='right'>86</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/iwpm_util.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/iwpm_util.c</a></td><td class='right'>105</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.8%;'/><td class='none' style='width: 98.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/iwpm_util.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/iwpm_util.h</a></td><td class='right'>28</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/lag.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/lag.c</a></td><td class='right'>135</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/mad.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/mad.c</a></td><td class='right'>950</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.9%;'/><td class='rem' style='width: 4.8%;'/><td class='none' style='width: 90.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/mad_priv.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/mad_priv.h</a></td><td class='right'>83</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.8%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/mad_rmpp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/mad_rmpp.c</a></td><td class='right'>80</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/mr_pool.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/mr_pool.c</a></td><td class='right'>18</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/multicast.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/multicast.c</a></td><td class='right'>43</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/netlink.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/netlink.c</a></td><td class='right'>161</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.9%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 98.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/nldev.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/nldev.c</a></td><td class='right'>1906</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 17.2%;'/><td class='rem' style='width: 2.3%;'/><td class='none' style='width: 80.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/opa_smi.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/opa_smi.h</a></td><td class='right'>4</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/rdma_core.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/rdma_core.c</a></td><td class='right'>431</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.5%;'/><td class='rem' style='width: 1.9%;'/><td class='none' style='width: 95.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/rdma_core.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/rdma_core.h</a></td><td class='right'>51</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/restrack.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/restrack.c</a></td><td class='right'>261</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.3%;'/><td class='rem' style='width: 1.4%;'/><td class='none' style='width: 97.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/restrack.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/restrack.h</a></td><td class='right'>8</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/roce_gid_mgmt.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/roce_gid_mgmt.c</a></td><td class='right'>106</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 98.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/rw.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/rw.c</a></td><td class='right'>306</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.5%;'/><td class='rem' style='width: 1.6%;'/><td class='none' style='width: 96.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/sa.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/sa.h</a></td><td class='right'>2</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/sa_query.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/sa_query.c</a></td><td class='right'>704</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.0%;'/><td class='rem' style='width: 3.2%;'/><td class='none' style='width: 92.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/security.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/security.c</a></td><td class='right'>50</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/smi.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/smi.c</a></td><td class='right'>12</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/smi.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/smi.h</a></td><td class='right'>4</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/sysfs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/sysfs.c</a></td><td class='right'>1330</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 7.2%;'/><td class='rem' style='width: 6.5%;'/><td class='none' style='width: 86.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/trace.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/trace.c</a></td><td class='right'>12</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/ucaps.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/ucaps.c</a></td><td class='right'>267</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/ucm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/ucm.c</a></td><td class='right'>1350</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 13.8%;'/><td class='none' style='width: 86.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/ucma.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/ucma.c</a></td><td class='right'>953</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 5.7%;'/><td class='rem' style='width: 4.1%;'/><td class='none' style='width: 90.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/ud_header.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/ud_header.c</a></td><td class='right'>85</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.9%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/umem.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/umem.c</a></td><td class='right'>336</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 2.2%;'/><td class='none' style='width: 96.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/umem_dmabuf.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/umem_dmabuf.c</a></td><td class='right'>275</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.8%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/umem_odp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/umem_odp.c</a></td><td class='right'>936</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.1%;'/><td class='rem' style='width: 6.5%;'/><td class='none' style='width: 90.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/user_mad.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/user_mad.c</a></td><td class='right'>235</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.7%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 97.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs.h</a></td><td class='right'>102</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 99.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_cmd.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_cmd.c</a></td><td class='right'>1208</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 6.2%;'/><td class='rem' style='width: 6.1%;'/><td class='none' style='width: 87.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_ioctl.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_ioctl.c</a></td><td class='right'>191</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 98.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_main.c</a></td><td class='right'>585</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.4%;'/><td class='rem' style='width: 3.6%;'/><td class='none' style='width: 94.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_marshall.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_marshall.c</a></td><td class='right'>44</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_std_types.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types.c</a></td><td class='right'>163</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 1.2%;'/><td class='none' style='width: 98.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_std_types_async_fd.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_async_fd.c</a></td><td class='right'>79</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.8%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_std_types_counters.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_counters.c</a></td><td class='right'>24</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_std_types_cq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_cq.c</a></td><td class='right'>150</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 98.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_std_types_device.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_device.c</a></td><td class='right'>290</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.9%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 97.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_std_types_dm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_dm.c</a></td><td class='right'>6</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_std_types_dmah.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_dmah.c</a></td><td class='right'>145</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.5%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_std_types_flow_action.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_flow_action.c</a></td><td class='right'>389</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 4.0%;'/><td class='none' style='width: 96.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_std_types_mr.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_mr.c</a></td><td class='right'>349</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.5%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 96.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_std_types_qp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_qp.c</a></td><td class='right'>380</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 96.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_std_types_srq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_srq.c</a></td><td class='right'>234</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_std_types_wq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_wq.c</a></td><td class='right'>194</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/uverbs_uapi.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_uapi.c</a></td><td class='right'>25</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/core/verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/verbs.c</a></td><td class='right'>1239</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 8.6%;'/><td class='rem' style='width: 4.1%;'/><td class='none' style='width: 87.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/Makefile</a></td><td class='right'>11</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bng_re/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bng_re/Kconfig</a></td><td class='right'>10</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bng_re/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bng_re/Makefile</a></td><td class='right'>8</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bng_re/bng_debugfs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bng_re/bng_debugfs.c</a></td><td class='right'>39</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bng_re/bng_debugfs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bng_re/bng_debugfs.h</a></td><td class='right'>12</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bng_re/bng_dev.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bng_re/bng_dev.c</a></td><td class='right'>534</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 5.5%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 94.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bng_re/bng_fw.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bng_re/bng_fw.c</a></td><td class='right'>767</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 7.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 92.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bng_re/bng_fw.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bng_re/bng_fw.h</a></td><td class='right'>211</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bng_re/bng_re.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bng_re/bng_re.h</a></td><td class='right'>85</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bng_re/bng_res.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bng_re/bng_res.c</a></td><td class='right'>279</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bng_re/bng_res.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bng_re/bng_res.h</a></td><td class='right'>215</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bng_re/bng_sp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bng_re/bng_sp.c</a></td><td class='right'>131</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bng_re/bng_sp.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bng_re/bng_sp.h</a></td><td class='right'>47</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bng_re/bng_tlv.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bng_re/bng_tlv.h</a></td><td class='right'>128</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/Kconfig</a></td><td class='right'>11</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/Makefile</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/bnxt_re.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/bnxt_re.h</a></td><td class='right'>209</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.6%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 97.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/debugfs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/debugfs.c</a></td><td class='right'>524</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 5.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 94.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/debugfs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/debugfs.h</a></td><td class='right'>55</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/hw_counters.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/hw_counters.c</a></td><td class='right'>523</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.6%;'/><td class='rem' style='width: 1.7%;'/><td class='none' style='width: 94.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/hw_counters.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/hw_counters.h</a></td><td class='right'>81</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/ib_verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/ib_verbs.c</a></td><td class='right'>2736</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 20.2%;'/><td class='rem' style='width: 7.8%;'/><td class='none' style='width: 72.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/ib_verbs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/ib_verbs.h</a></td><td class='right'>128</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.1%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 98.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/main.c</a></td><td class='right'>2672</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 18.2%;'/><td class='rem' style='width: 9.2%;'/><td class='none' style='width: 72.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/qplib_fp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_fp.c</a></td><td class='right'>2246</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 13.5%;'/><td class='rem' style='width: 9.5%;'/><td class='none' style='width: 77.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/qplib_fp.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_fp.h</a></td><td class='right'>368</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.5%;'/><td class='rem' style='width: 1.2%;'/><td class='none' style='width: 96.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_rcfw.c</a></td><td class='right'>1263</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 8.5%;'/><td class='rem' style='width: 4.4%;'/><td class='none' style='width: 87.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_rcfw.h</a></td><td class='right'>296</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.5%;'/><td class='rem' style='width: 1.5%;'/><td class='none' style='width: 97.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/qplib_res.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_res.c</a></td><td class='right'>809</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.7%;'/><td class='rem' style='width: 3.6%;'/><td class='none' style='width: 91.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/qplib_res.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_res.h</a></td><td class='right'>470</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.3%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 95.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/qplib_sp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_sp.c</a></td><td class='right'>937</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 6.2%;'/><td class='rem' style='width: 3.3%;'/><td class='none' style='width: 90.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/qplib_sp.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_sp.h</a></td><td class='right'>157</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.4%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 98.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/qplib_tlv.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_tlv.h</a></td><td class='right'>162</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/bnxt_re/roce_hsi.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/roce_hsi.h</a></td><td class='right'>7355</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 46.4%;'/><td class='rem' style='width: 28.9%;'/><td class='none' style='width: 24.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/Kconfig</a></td><td class='right'>18</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/Makefile</a></td><td class='right'>7</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/cxio_hal.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/cxio_hal.c</a></td><td class='right'>1331</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 13.6%;'/><td class='none' style='width: 86.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/cxio_hal.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/cxio_hal.h</a></td><td class='right'>205</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 2.1%;'/><td class='none' style='width: 97.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/cxio_resource.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/cxio_resource.c</a></td><td class='right'>344</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 3.5%;'/><td class='none' style='width: 96.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/cxio_resource.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/cxio_resource.h</a></td><td class='right'>69</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/cxio_wr.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/cxio_wr.h</a></td><td class='right'>802</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 8.2%;'/><td class='none' style='width: 91.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/iwch.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch.c</a></td><td class='right'>282</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 2.9%;'/><td class='none' style='width: 97.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/iwch.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch.h</a></td><td class='right'>155</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 1.6%;'/><td class='none' style='width: 98.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/iwch_cm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch_cm.c</a></td><td class='right'>2258</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 23.1%;'/><td class='none' style='width: 76.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/iwch_cm.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch_cm.h</a></td><td class='right'>233</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 2.4%;'/><td class='none' style='width: 97.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/iwch_cq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch_cq.c</a></td><td class='right'>230</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 2.4%;'/><td class='none' style='width: 97.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/iwch_ev.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch_ev.c</a></td><td class='right'>232</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 2.4%;'/><td class='none' style='width: 97.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/iwch_mem.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch_mem.c</a></td><td class='right'>101</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 1.0%;'/><td class='none' style='width: 99.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/iwch_provider.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch_provider.c</a></td><td class='right'>1402</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 14.4%;'/><td class='none' style='width: 85.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/iwch_provider.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch_provider.h</a></td><td class='right'>347</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 3.6%;'/><td class='none' style='width: 96.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/iwch_qp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch_qp.c</a></td><td class='right'>1082</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 11.1%;'/><td class='none' style='width: 88.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb3/tcb.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/tcb.h</a></td><td class='right'>632</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 6.5%;'/><td class='none' style='width: 93.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb4/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/Kconfig</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb4/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/Makefile</a></td><td class='right'>1</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb4/cm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/cm.c</a></td><td class='right'>126</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 98.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb4/cq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/cq.c</a></td><td class='right'>80</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb4/device.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/device.c</a></td><td class='right'>32</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb4/ev.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/ev.c</a></td><td class='right'>10</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb4/id_table.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/id_table.c</a></td><td class='right'>21</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb4/iw_cxgb4.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/iw_cxgb4.h</a></td><td class='right'>92</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.6%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb4/mem.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/mem.c</a></td><td class='right'>151</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 1.2%;'/><td class='none' style='width: 98.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb4/provider.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/provider.c</a></td><td class='right'>234</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.9%;'/><td class='rem' style='width: 1.5%;'/><td class='none' style='width: 97.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb4/qp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/qp.c</a></td><td class='right'>202</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.1%;'/><td class='rem' style='width: 1.0%;'/><td class='none' style='width: 97.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb4/resource.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/resource.c</a></td><td class='right'>18</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb4/restrack.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/restrack.c</a></td><td class='right'>28</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb4/t4.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/t4.h</a></td><td class='right'>33</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/cxgb4/t4fw_ri_api.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/t4fw_ri_api.h</a></td><td class='right'>34</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/efa/efa.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa.h</a></td><td class='right'>99</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 99.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_admin_cmds_defs.h</a></td><td class='right'>474</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.2%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 95.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/efa/efa_admin_defs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_admin_defs.h</a></td><td class='right'>49</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/efa/efa_com.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_com.c</a></td><td class='right'>567</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.4%;'/><td class='rem' style='width: 2.4%;'/><td class='none' style='width: 94.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/efa/efa_com.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_com.h</a></td><td class='right'>44</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/efa/efa_com_cmd.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_com_cmd.c</a></td><td class='right'>370</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.6%;'/><td class='rem' style='width: 1.1%;'/><td class='none' style='width: 96.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/efa/efa_com_cmd.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_com_cmd.h</a></td><td class='right'>115</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.0%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 98.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/efa/efa_common_defs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_common_defs.h</a></td><td class='right'>13</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/efa/efa_io_defs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_io_defs.h</a></td><td class='right'>391</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 96.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/efa/efa_main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_main.c</a></td><td class='right'>401</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.9%;'/><td class='rem' style='width: 1.2%;'/><td class='none' style='width: 95.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/efa/efa_regs_defs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_regs_defs.h</a></td><td class='right'>30</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/efa/efa_verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_verbs.c</a></td><td class='right'>1473</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 10.2%;'/><td class='rem' style='width: 4.9%;'/><td class='none' style='width: 84.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/erdma/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/erdma/Kconfig</a></td><td class='right'>12</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/erdma/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/erdma/Makefile</a></td><td class='right'>4</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/erdma/erdma.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/erdma/erdma.h</a></td><td class='right'>283</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/erdma/erdma_cm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/erdma/erdma_cm.c</a></td><td class='right'>1431</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 14.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 85.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/erdma/erdma_cm.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/erdma/erdma_cm.h</a></td><td class='right'>167</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/erdma/erdma_cmdq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/erdma/erdma_cmdq.c</a></td><td class='right'>452</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 95.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/erdma/erdma_cq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/erdma/erdma_cq.c</a></td><td class='right'>268</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/erdma/erdma_eq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/erdma/erdma_eq.c</a></td><td class='right'>326</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 96.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/erdma/erdma_hw.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/erdma/erdma_hw.h</a></td><td class='right'>753</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 7.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 92.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/erdma/erdma_main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/erdma/erdma_main.c</a></td><td class='right'>684</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 7.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 93.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/erdma/erdma_qp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/erdma/erdma_qp.c</a></td><td class='right'>757</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 7.8%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 92.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/erdma/erdma_verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/erdma/erdma_verbs.c</a></td><td class='right'>2300</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 23.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 76.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/erdma/erdma_verbs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/erdma/erdma_verbs.h</a></td><td class='right'>491</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 5.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 95.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/Kconfig</a></td><td class='right'>13</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/Makefile</a></td><td class='right'>6</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/affinity.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/affinity.c</a></td><td class='right'>199</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 1.3%;'/><td class='none' style='width: 98.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/affinity.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/affinity.h</a></td><td class='right'>48</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/aspm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/aspm.c</a></td><td class='right'>270</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.8%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/aspm.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/aspm.h</a></td><td class='right'>307</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 3.0%;'/><td class='none' style='width: 96.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/chip.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/chip.c</a></td><td class='right'>800</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 5.1%;'/><td class='rem' style='width: 3.1%;'/><td class='none' style='width: 91.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/chip.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/chip.h</a></td><td class='right'>70</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/chip_registers.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/chip_registers.h</a></td><td class='right'>51</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/common.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/common.h</a></td><td class='right'>115</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 1.1%;'/><td class='none' style='width: 98.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/debugfs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/debugfs.c</a></td><td class='right'>113</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 1.0%;'/><td class='none' style='width: 98.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/debugfs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/debugfs.h</a></td><td class='right'>58</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/device.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/device.c</a></td><td class='right'>121</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 0.9%;'/><td class='none' style='width: 98.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/device.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/device.h</a></td><td class='right'>49</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/driver.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/driver.c</a></td><td class='right'>530</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.2%;'/><td class='rem' style='width: 2.2%;'/><td class='none' style='width: 94.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/efivar.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/efivar.c</a></td><td class='right'>60</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/efivar.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/efivar.h</a></td><td class='right'>45</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/eprom.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/eprom.c</a></td><td class='right'>45</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/eprom.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/eprom.h</a></td><td class='right'>44</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/exp_rcv.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/exp_rcv.c</a></td><td class='right'>58</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/exp_rcv.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/exp_rcv.h</a></td><td class='right'>53</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/fault.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/fault.c</a></td><td class='right'>71</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/fault.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/fault.h</a></td><td class='right'>50</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/file_ops.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/file_ops.c</a></td><td class='right'>237</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 1.2%;'/><td class='none' style='width: 97.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/firmware.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/firmware.c</a></td><td class='right'>70</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/hfi.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/hfi.h</a></td><td class='right'>373</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.6%;'/><td class='rem' style='width: 1.2%;'/><td class='none' style='width: 96.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/init.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/init.c</a></td><td class='right'>326</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.0%;'/><td class='rem' style='width: 2.3%;'/><td class='none' style='width: 96.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/intr.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/intr.c</a></td><td class='right'>75</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.8%;'/><td class='none' style='width: 99.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/iowait.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/iowait.c</a></td><td class='right'>8</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/iowait.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/iowait.h</a></td><td class='right'>55</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/ipoib.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/ipoib.h</a></td><td class='right'>171</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.8%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/ipoib_main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/ipoib_main.c</a></td><td class='right'>250</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/ipoib_rx.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/ipoib_rx.c</a></td><td class='right'>92</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/ipoib_tx.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/ipoib_tx.c</a></td><td class='right'>868</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 8.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 91.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/mad.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/mad.c</a></td><td class='right'>284</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 1.8%;'/><td class='none' style='width: 97.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/mad.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/mad.h</a></td><td class='right'>48</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/mmu_rb.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/mmu_rb.c</a></td><td class='right'>283</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 1.7%;'/><td class='none' style='width: 97.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/mmu_rb.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/mmu_rb.h</a></td><td class='right'>91</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/msix.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/msix.c</a></td><td class='right'>189</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.9%;'/><td class='rem' style='width: 1.0%;'/><td class='none' style='width: 98.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/msix.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/msix.h</a></td><td class='right'>52</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/netdev.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/netdev.h</a></td><td class='right'>105</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/netdev_rx.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/netdev_rx.c</a></td><td class='right'>487</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 5.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 95.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/opa_compat.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/opa_compat.h</a></td><td class='right'>48</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/opfn.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/opfn.c</a></td><td class='right'>4</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/pcie.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/pcie.c</a></td><td class='right'>132</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 1.0%;'/><td class='none' style='width: 98.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/pin_system.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/pin_system.c</a></td><td class='right'>474</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 95.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/pinning.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/pinning.h</a></td><td class='right'>20</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/pio.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/pio.c</a></td><td class='right'>101</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 99.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/pio.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/pio.h</a></td><td class='right'>55</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/pio_copy.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/pio_copy.c</a></td><td class='right'>60</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/platform.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/platform.c</a></td><td class='right'>56</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/platform.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/platform.h</a></td><td class='right'>45</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/qp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/qp.c</a></td><td class='right'>92</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/qp.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/qp.h</a></td><td class='right'>62</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/qsfp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/qsfp.c</a></td><td class='right'>72</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/qsfp.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/qsfp.h</a></td><td class='right'>46</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/rc.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/rc.c</a></td><td class='right'>193</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 1.3%;'/><td class='none' style='width: 98.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/ruc.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/ruc.c</a></td><td class='right'>51</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/sdma.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/sdma.c</a></td><td class='right'>216</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.9%;'/><td class='rem' style='width: 1.3%;'/><td class='none' style='width: 97.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/sdma.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/sdma.h</a></td><td class='right'>130</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 0.9%;'/><td class='none' style='width: 98.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/sdma_txreq.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/sdma_txreq.h</a></td><td class='right'>47</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/sysfs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/sysfs.c</a></td><td class='right'>639</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.3%;'/><td class='rem' style='width: 4.2%;'/><td class='none' style='width: 93.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/tid_rdma.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/tid_rdma.c</a></td><td class='right'>319</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.5%;'/><td class='rem' style='width: 1.8%;'/><td class='none' style='width: 96.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/tid_rdma.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/tid_rdma.h</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/trace.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace.c</a></td><td class='right'>91</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/trace.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace.h</a></td><td class='right'>44</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/trace_ctxts.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_ctxts.h</a></td><td class='right'>58</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/trace_dbg.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_dbg.h</a></td><td class='right'>62</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/trace_ibhdrs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_ibhdrs.h</a></td><td class='right'>47</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/trace_misc.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_misc.h</a></td><td class='right'>49</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/trace_mmu.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_mmu.h</a></td><td class='right'>97</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/trace_rc.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_rc.h</a></td><td class='right'>49</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/trace_rx.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_rx.h</a></td><td class='right'>62</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/trace_tid.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_tid.h</a></td><td class='right'>56</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/trace_tx.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_tx.h</a></td><td class='right'>296</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.5%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 97.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/uc.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/uc.c</a></td><td class='right'>63</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/ud.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/ud.c</a></td><td class='right'>92</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/user_exp_rcv.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/user_exp_rcv.c</a></td><td class='right'>480</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.4%;'/><td class='rem' style='width: 2.5%;'/><td class='none' style='width: 95.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/user_exp_rcv.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/user_exp_rcv.h</a></td><td class='right'>63</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/user_pages.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/user_pages.c</a></td><td class='right'>115</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 98.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/user_sdma.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/user_sdma.c</a></td><td class='right'>446</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 3.8%;'/><td class='none' style='width: 95.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/user_sdma.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/user_sdma.h</a></td><td class='right'>75</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 99.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/verbs.c</a></td><td class='right'>249</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.0%;'/><td class='rem' style='width: 1.6%;'/><td class='none' style='width: 97.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/verbs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/verbs.h</a></td><td class='right'>61</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/verbs_txreq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/verbs_txreq.c</a></td><td class='right'>46</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/verbs_txreq.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/verbs_txreq.h</a></td><td class='right'>54</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/vnic.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/vnic.h</a></td><td class='right'>55</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/vnic_main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/vnic_main.c</a></td><td class='right'>371</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 3.1%;'/><td class='none' style='width: 96.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hfi1/vnic_sdma.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/vnic_sdma.c</a></td><td class='right'>51</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/Kconfig</a></td><td class='right'>32</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/Makefile</a></td><td class='right'>14</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_ah.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_ah.c</a></td><td class='right'>126</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.8%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 98.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_alloc.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_alloc.c</a></td><td class='right'>291</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 1.8%;'/><td class='none' style='width: 97.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_bond.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_bond.c</a></td><td class='right'>1012</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 10.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 89.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_bond.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_bond.h</a></td><td class='right'>95</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_cmd.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_cmd.c</a></td><td class='right'>228</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.1%;'/><td class='rem' style='width: 1.2%;'/><td class='none' style='width: 97.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_cmd.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_cmd.h</a></td><td class='right'>29</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_common.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_common.h</a></td><td class='right'>253</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 2.1%;'/><td class='none' style='width: 97.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_cq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_cq.c</a></td><td class='right'>746</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.1%;'/><td class='rem' style='width: 3.5%;'/><td class='none' style='width: 92.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_db.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_db.c</a></td><td class='right'>40</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_debugfs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_debugfs.c</a></td><td class='right'>111</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_debugfs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_debugfs.h</a></td><td class='right'>33</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_device.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_device.h</a></td><td class='right'>1142</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 6.5%;'/><td class='rem' style='width: 5.2%;'/><td class='none' style='width: 88.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_hem.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_hem.c</a></td><td class='right'>1587</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 9.7%;'/><td class='rem' style='width: 6.5%;'/><td class='none' style='width: 83.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_hem.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_hem.h</a></td><td class='right'>110</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.8%;'/><td class='none' style='width: 98.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_hw_v1.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_hw_v1.c</a></td><td class='right'>4734</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 48.5%;'/><td class='none' style='width: 51.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_hw_v1.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_hw_v1.h</a></td><td class='right'>1099</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 11.3%;'/><td class='none' style='width: 88.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_hw_v2.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_hw_v2.c</a></td><td class='right'>9762</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 55.1%;'/><td class='rem' style='width: 44.9%;'/><td class='none' style='width: 0.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_hw_v2.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_hw_v2.h</a></td><td class='right'>2297</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 10.2%;'/><td class='rem' style='width: 13.4%;'/><td class='none' style='width: 76.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c</a></td><td class='right'>35</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_main.c</a></td><td class='right'>989</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 6.5%;'/><td class='rem' style='width: 3.6%;'/><td class='none' style='width: 89.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_mr.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_mr.c</a></td><td class='right'>1977</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 8.1%;'/><td class='rem' style='width: 12.1%;'/><td class='none' style='width: 79.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_pd.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_pd.c</a></td><td class='right'>153</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.9%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 98.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_qp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_qp.c</a></td><td class='right'>1837</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 11.7%;'/><td class='rem' style='width: 7.1%;'/><td class='none' style='width: 81.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_restrack.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_restrack.c</a></td><td class='right'>242</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.8%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 97.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_srq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_srq.c</a></td><td class='right'>654</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.9%;'/><td class='rem' style='width: 2.8%;'/><td class='none' style='width: 93.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/hns/hns_roce_trace.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_trace.h</a></td><td class='right'>216</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/Kconfig</a></td><td class='right'>8</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/Makefile</a></td><td class='right'>10</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw.h</a></td><td class='right'>602</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 6.2%;'/><td class='none' style='width: 93.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_cm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_cm.c</a></td><td class='right'>4419</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 45.3%;'/><td class='none' style='width: 54.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_cm.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_cm.h</a></td><td class='right'>462</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 4.7%;'/><td class='none' style='width: 95.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_ctrl.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_ctrl.c</a></td><td class='right'>5198</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 53.2%;'/><td class='none' style='width: 46.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_d.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_d.h</a></td><td class='right'>1737</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 17.8%;'/><td class='none' style='width: 82.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_hmc.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_hmc.c</a></td><td class='right'>821</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 8.4%;'/><td class='none' style='width: 91.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_hmc.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_hmc.h</a></td><td class='right'>241</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 2.5%;'/><td class='none' style='width: 97.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_hw.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_hw.c</a></td><td class='right'>852</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 8.7%;'/><td class='none' style='width: 91.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_main.c</a></td><td class='right'>2066</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 21.2%;'/><td class='none' style='width: 78.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_osdep.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_osdep.h</a></td><td class='right'>217</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 2.2%;'/><td class='none' style='width: 97.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_p.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_p.h</a></td><td class='right'>128</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 1.3%;'/><td class='none' style='width: 98.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_pble.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_pble.c</a></td><td class='right'>612</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 6.3%;'/><td class='none' style='width: 93.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_pble.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_pble.h</a></td><td class='right'>131</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 1.3%;'/><td class='none' style='width: 98.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_puda.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_puda.c</a></td><td class='right'>1493</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 15.3%;'/><td class='none' style='width: 84.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_puda.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_puda.h</a></td><td class='right'>188</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 1.9%;'/><td class='none' style='width: 98.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_register.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_register.h</a></td><td class='right'>1030</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 10.6%;'/><td class='none' style='width: 89.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_status.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_status.h</a></td><td class='right'>101</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 1.0%;'/><td class='none' style='width: 99.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_type.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_type.h</a></td><td class='right'>1363</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 14.0%;'/><td class='none' style='width: 86.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_uk.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_uk.c</a></td><td class='right'>1232</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 12.6%;'/><td class='none' style='width: 87.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_user.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_user.h</a></td><td class='right'>430</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 4.4%;'/><td class='none' style='width: 95.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_utils.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_utils.c</a></td><td class='right'>1553</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 15.9%;'/><td class='none' style='width: 84.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_verbs.c</a></td><td class='right'>2807</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 28.8%;'/><td class='none' style='width: 71.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_verbs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_verbs.h</a></td><td class='right'>179</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 1.8%;'/><td class='none' style='width: 98.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_vf.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_vf.c</a></td><td class='right'>85</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.9%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_vf.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_vf.h</a></td><td class='right'>62</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_virtchnl.c</a></td><td class='right'>756</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 7.7%;'/><td class='none' style='width: 92.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/i40iw/i40iw_virtchnl.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_virtchnl.h</a></td><td class='right'>124</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 1.3%;'/><td class='none' style='width: 98.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ionic/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ionic/Kconfig</a></td><td class='right'>15</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ionic/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ionic/Makefile</a></td><td class='right'>9</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ionic/ionic_admin.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ionic/ionic_admin.c</a></td><td class='right'>1229</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 12.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 87.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ionic/ionic_controlpath.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ionic/ionic_controlpath.c</a></td><td class='right'>2679</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 27.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 72.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ionic/ionic_datapath.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ionic/ionic_datapath.c</a></td><td class='right'>1399</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 14.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 85.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ionic/ionic_fw.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ionic/ionic_fw.h</a></td><td class='right'>1029</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 10.5%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 89.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ionic/ionic_hw_stats.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ionic/ionic_hw_stats.c</a></td><td class='right'>484</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 5.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 95.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ionic/ionic_ibdev.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ionic/ionic_ibdev.c</a></td><td class='right'>440</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.5%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 95.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ionic/ionic_ibdev.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ionic/ionic_ibdev.h</a></td><td class='right'>517</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 5.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 94.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ionic/ionic_lif_cfg.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ionic/ionic_lif_cfg.c</a></td><td class='right'>111</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ionic/ionic_lif_cfg.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ionic/ionic_lif_cfg.h</a></td><td class='right'>66</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ionic/ionic_pgtbl.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ionic/ionic_pgtbl.c</a></td><td class='right'>143</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.5%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ionic/ionic_queue.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ionic/ionic_queue.c</a></td><td class='right'>52</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ionic/ionic_queue.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ionic/ionic_queue.h</a></td><td class='right'>234</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ionic/ionic_res.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ionic/ionic_res.h</a></td><td class='right'>154</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/Kconfig</a></td><td class='right'>14</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/Makefile</a></td><td class='right'>31</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/cm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/cm.c</a></td><td class='right'>4434</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 45.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 54.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/cm.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/cm.h</a></td><td class='right'>416</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 95.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/ctrl.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/ctrl.c</a></td><td class='right'>6602</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 67.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 32.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/defs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/defs.h</a></td><td class='right'>1184</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 12.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 87.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/hmc.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/hmc.c</a></td><td class='right'>709</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 7.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 92.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/hmc.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/hmc.h</a></td><td class='right'>186</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/hw.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/hw.c</a></td><td class='right'>2823</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 28.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 71.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/i40iw_hw.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/i40iw_hw.c</a></td><td class='right'>261</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/i40iw_hw.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/i40iw_hw.h</a></td><td class='right'>162</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/i40iw_if.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/i40iw_if.c</a></td><td class='right'>220</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/icrdma_hw.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/icrdma_hw.c</a></td><td class='right'>205</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/icrdma_hw.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/icrdma_hw.h</a></td><td class='right'>73</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/icrdma_if.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/icrdma_if.c</a></td><td class='right'>347</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 96.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/ig3rdma_hw.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/ig3rdma_hw.c</a></td><td class='right'>170</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/ig3rdma_hw.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/ig3rdma_hw.h</a></td><td class='right'>32</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/ig3rdma_if.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/ig3rdma_if.c</a></td><td class='right'>236</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/irdma.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/irdma.h</a></td><td class='right'>174</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.8%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/main.c</a></td><td class='right'>211</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/main.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/main.h</a></td><td class='right'>578</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 5.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 94.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/osdep.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/osdep.h</a></td><td class='right'>74</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.8%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/pble.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/pble.c</a></td><td class='right'>519</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 5.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 94.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/pble.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/pble.h</a></td><td class='right'>132</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/protos.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/protos.h</a></td><td class='right'>97</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/puda.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/puda.c</a></td><td class='right'>1718</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 17.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 82.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/puda.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/puda.h</a></td><td class='right'>182</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/trace.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/trace.c</a></td><td class='right'>112</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/trace.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/trace.h</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/trace_cm.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/trace_cm.h</a></td><td class='right'>460</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 95.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/type.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/type.h</a></td><td class='right'>1674</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 17.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 82.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/uda.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/uda.c</a></td><td class='right'>265</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/uda.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/uda.h</a></td><td class='right'>87</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/uda_d.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/uda_d.h</a></td><td class='right'>127</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/uk.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/uk.c</a></td><td class='right'>1930</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 19.8%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 80.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/user.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/user.h</a></td><td class='right'>676</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 6.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 93.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/utils.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/utils.c</a></td><td class='right'>2508</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 25.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 74.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/verbs.c</a></td><td class='right'>5517</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 56.5%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 43.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/verbs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/verbs.h</a></td><td class='right'>341</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.5%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 96.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/virtchnl.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/virtchnl.c</a></td><td class='right'>618</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 6.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 93.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/virtchnl.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/virtchnl.h</a></td><td class='right'>176</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.8%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/ws.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/ws.c</a></td><td class='right'>406</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 95.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/irdma/ws.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/irdma/ws.h</a></td><td class='right'>41</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mana/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mana/Kconfig</a></td><td class='right'>10</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mana/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mana/Makefile</a></td><td class='right'>4</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mana/ah.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mana/ah.c</a></td><td class='right'>58</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mana/counters.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mana/counters.c</a></td><td class='right'>179</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.8%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mana/counters.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mana/counters.h</a></td><td class='right'>62</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mana/cq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mana/cq.c</a></td><td class='right'>342</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.5%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 96.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mana/device.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mana/device.c</a></td><td class='right'>261</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mana/main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mana/main.c</a></td><td class='right'>1134</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 11.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 88.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mana/mana_ib.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mana/mana_ib.h</a></td><td class='right'>738</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 7.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 92.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mana/mr.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mana/mr.c</a></td><td class='right'>319</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 96.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mana/qp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mana/qp.c</a></td><td class='right'>921</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 9.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 90.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mana/shadow_queue.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mana/shadow_queue.h</a></td><td class='right'>115</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mana/wq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mana/wq.c</a></td><td class='right'>92</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mana/wr.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mana/wr.c</a></td><td class='right'>168</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx4/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/Kconfig</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx4/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/Makefile</a></td><td class='right'>1</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx4/ah.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/ah.c</a></td><td class='right'>16</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx4/alias_GUID.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/alias_GUID.c</a></td><td class='right'>31</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx4/cm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/cm.c</a></td><td class='right'>198</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.7%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 98.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx4/cq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/cq.c</a></td><td class='right'>82</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx4/doorbell.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/doorbell.c</a></td><td class='right'>6</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx4/mad.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/mad.c</a></td><td class='right'>251</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.4%;'/><td class='rem' style='width: 1.2%;'/><td class='none' style='width: 97.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx4/main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/main.c</a></td><td class='right'>707</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.7%;'/><td class='rem' style='width: 3.5%;'/><td class='none' style='width: 92.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx4/mcg.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/mcg.c</a></td><td class='right'>92</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx4/mlx4_ib.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/mlx4_ib.h</a></td><td class='right'>157</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.0%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 98.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx4/mr.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/mr.c</a></td><td class='right'>459</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 4.2%;'/><td class='none' style='width: 95.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx4/qp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/qp.c</a></td><td class='right'>795</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.4%;'/><td class='rem' style='width: 3.7%;'/><td class='none' style='width: 91.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx4/srq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/srq.c</a></td><td class='right'>23</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx4/sysfs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/sysfs.c</a></td><td class='right'>72</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/Kconfig</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/Makefile</a></td><td class='right'>36</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/ah.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/ah.c</a></td><td class='right'>55</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/cmd.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/cmd.c</a></td><td class='right'>463</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.1%;'/><td class='rem' style='width: 3.6%;'/><td class='none' style='width: 95.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/cmd.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/cmd.h</a></td><td class='right'>27</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/cong.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/cong.c</a></td><td class='right'>69</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.6%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/counters.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/counters.c</a></td><td class='right'>1279</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 13.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 86.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/counters.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/counters.h</a></td><td class='right'>19</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/cq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/cq.c</a></td><td class='right'>366</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.3%;'/><td class='rem' style='width: 1.4%;'/><td class='none' style='width: 96.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/data_direct.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/data_direct.c</a></td><td class='right'>227</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/data_direct.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/data_direct.h</a></td><td class='right'>23</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/devx.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/devx.c</a></td><td class='right'>1887</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 16.4%;'/><td class='rem' style='width: 2.9%;'/><td class='none' style='width: 80.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/devx.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/devx.h</a></td><td class='right'>50</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/dm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/dm.c</a></td><td class='right'>612</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 6.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 93.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/dm.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/dm.h</a></td><td class='right'>68</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/dmah.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/dmah.c</a></td><td class='right'>54</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/dmah.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/dmah.h</a></td><td class='right'>23</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/doorbell.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/doorbell.c</a></td><td class='right'>17</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/flow.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/flow.c</a></td><td class='right'>682</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 7.0%;'/><td class='none' style='width: 93.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/fs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/fs.c</a></td><td class='right'>3516</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 36.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 64.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/fs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/fs.h</a></td><td class='right'>36</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/gsi.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/gsi.c</a></td><td class='right'>223</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.9%;'/><td class='rem' style='width: 1.4%;'/><td class='none' style='width: 97.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/ib_rep.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/ib_rep.c</a></td><td class='right'>335</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.7%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 96.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/ib_rep.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/ib_rep.h</a></td><td class='right'>57</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/ib_virt.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/ib_virt.c</a></td><td class='right'>41</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/macsec.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/macsec.c</a></td><td class='right'>364</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 96.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/macsec.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/macsec.h</a></td><td class='right'>29</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/mad.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/mad.c</a></td><td class='right'>373</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.4%;'/><td class='rem' style='width: 1.4%;'/><td class='none' style='width: 96.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/main.c</a></td><td class='right'>5640</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 20.2%;'/><td class='rem' style='width: 37.6%;'/><td class='none' style='width: 42.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/mem.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/mem.c</a></td><td class='right'>231</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 1.9%;'/><td class='none' style='width: 97.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/mlx5_ib.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/mlx5_ib.h</a></td><td class='right'>1185</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 8.0%;'/><td class='rem' style='width: 4.1%;'/><td class='none' style='width: 87.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/mr.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/mr.c</a></td><td class='right'>3203</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 21.6%;'/><td class='rem' style='width: 11.2%;'/><td class='none' style='width: 67.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/odp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/odp.c</a></td><td class='right'>1821</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 10.8%;'/><td class='rem' style='width: 7.9%;'/><td class='none' style='width: 81.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/qos.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/qos.c</a></td><td class='right'>133</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/qp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/qp.c</a></td><td class='right'>5060</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 23.4%;'/><td class='rem' style='width: 28.4%;'/><td class='none' style='width: 48.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/qp.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/qp.h</a></td><td class='right'>60</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/qpc.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/qpc.c</a></td><td class='right'>697</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 7.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 92.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/restrack.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/restrack.c</a></td><td class='right'>217</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/restrack.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/restrack.h</a></td><td class='right'>13</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/srq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/srq.c</a></td><td class='right'>97</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/srq.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/srq.h</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/srq_cmd.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/srq_cmd.c</a></td><td class='right'>225</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.5%;'/><td class='rem' style='width: 0.8%;'/><td class='none' style='width: 97.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/std_types.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/std_types.c</a></td><td class='right'>277</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.8%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/umr.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/umr.c</a></td><td class='right'>1129</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 11.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 88.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/umr.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/umr.h</a></td><td class='right'>113</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/wr.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/wr.c</a></td><td class='right'>1284</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 13.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 86.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mlx5/wr.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/wr.h</a></td><td class='right'>136</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/Kconfig</a></td><td class='right'>5</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/mthca_allocator.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/mthca_allocator.c</a></td><td class='right'>17</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/mthca_av.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/mthca_av.c</a></td><td class='right'>8</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/mthca_catas.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/mthca_catas.c</a></td><td class='right'>4</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/mthca_cmd.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/mthca_cmd.c</a></td><td class='right'>6</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/mthca_cq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/mthca_cq.c</a></td><td class='right'>12</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/mthca_dev.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/mthca_dev.h</a></td><td class='right'>31</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/mthca_eq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/mthca_eq.c</a></td><td class='right'>21</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/mthca_mad.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/mthca_mad.c</a></td><td class='right'>76</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/mthca_main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/mthca_main.c</a></td><td class='right'>23</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/mthca_memfree.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/mthca_memfree.c</a></td><td class='right'>35</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/mthca_memfree.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/mthca_memfree.h</a></td><td class='right'>2</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/mthca_mr.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/mthca_mr.c</a></td><td class='right'>289</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 2.8%;'/><td class='none' style='width: 97.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/mthca_profile.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/mthca_profile.c</a></td><td class='right'>2</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/mthca_provider.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/mthca_provider.c</a></td><td class='right'>373</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 2.6%;'/><td class='none' style='width: 96.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/mthca_provider.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/mthca_provider.h</a></td><td class='right'>50</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/mthca/mthca_qp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mthca/mthca_qp.c</a></td><td class='right'>90</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/nes/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/nes/Kconfig</a></td><td class='right'>15</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/nes/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/nes/Makefile</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/nes/nes.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/nes/nes.c</a></td><td class='right'>1205</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 12.3%;'/><td class='none' style='width: 87.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/nes/nes.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/nes/nes.h</a></td><td class='right'>574</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 5.9%;'/><td class='none' style='width: 94.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/nes/nes_cm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/nes/nes_cm.c</a></td><td class='right'>3992</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 40.9%;'/><td class='none' style='width: 59.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/nes/nes_cm.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/nes/nes_cm.h</a></td><td class='right'>470</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 4.8%;'/><td class='none' style='width: 95.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/nes/nes_context.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/nes/nes_context.h</a></td><td class='right'>193</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 2.0%;'/><td class='none' style='width: 98.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/nes/nes_hw.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/nes/nes_hw.c</a></td><td class='right'>3887</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 39.8%;'/><td class='none' style='width: 60.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/nes/nes_hw.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/nes/nes_hw.h</a></td><td class='right'>1380</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 14.1%;'/><td class='none' style='width: 85.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/nes/nes_mgt.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/nes/nes_mgt.c</a></td><td class='right'>1155</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 11.8%;'/><td class='none' style='width: 88.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/nes/nes_mgt.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/nes/nes_mgt.h</a></td><td class='right'>97</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 1.0%;'/><td class='none' style='width: 99.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/nes/nes_nic.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/nes/nes_nic.c</a></td><td class='right'>1870</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 19.2%;'/><td class='none' style='width: 80.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/nes/nes_utils.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/nes/nes_utils.c</a></td><td class='right'>916</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 9.4%;'/><td class='none' style='width: 90.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/nes/nes_verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/nes/nes_verbs.c</a></td><td class='right'>3759</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 38.5%;'/><td class='none' style='width: 61.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/nes/nes_verbs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/nes/nes_verbs.h</a></td><td class='right'>198</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 2.0%;'/><td class='none' style='width: 98.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ocrdma/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ocrdma/Kconfig</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ocrdma/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ocrdma/Makefile</a></td><td class='right'>1</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ocrdma/ocrdma.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ocrdma/ocrdma.h</a></td><td class='right'>2</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ocrdma/ocrdma_ah.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ocrdma/ocrdma_ah.c</a></td><td class='right'>39</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ocrdma/ocrdma_ah.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ocrdma/ocrdma_ah.h</a></td><td class='right'>15</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ocrdma/ocrdma_hw.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ocrdma/ocrdma_hw.c</a></td><td class='right'>35</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ocrdma/ocrdma_hw.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ocrdma/ocrdma_hw.h</a></td><td class='right'>2</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ocrdma/ocrdma_main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ocrdma/ocrdma_main.c</a></td><td class='right'>79</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ocrdma/ocrdma_sli.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ocrdma/ocrdma_sli.h</a></td><td class='right'>2</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ocrdma/ocrdma_stats.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ocrdma/ocrdma_stats.c</a></td><td class='right'>31</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ocrdma/ocrdma_stats.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ocrdma/ocrdma_stats.h</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ocrdma/ocrdma_verbs.c</a></td><td class='right'>161</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 0.9%;'/><td class='none' style='width: 98.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/ocrdma/ocrdma_verbs.h</a></td><td class='right'>29</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qedr/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qedr/Kconfig</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qedr/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qedr/Makefile</a></td><td class='right'>1</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qedr/main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qedr/main.c</a></td><td class='right'>154</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.0%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 98.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qedr/qedr.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qedr/qedr.h</a></td><td class='right'>113</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.9%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 98.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qedr/qedr_iw_cm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qedr/qedr_iw_cm.c</a></td><td class='right'>169</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 98.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qedr/qedr_roce_cm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qedr/qedr_roce_cm.c</a></td><td class='right'>18</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qedr/qedr_roce_cm.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qedr/qedr_roce_cm.h</a></td><td class='right'>5</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qedr/verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qedr/verbs.c</a></td><td class='right'>1354</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 8.7%;'/><td class='rem' style='width: 5.2%;'/><td class='none' style='width: 86.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qedr/verbs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qedr/verbs.h</a></td><td class='right'>49</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/Kconfig</a></td><td class='right'>16</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/Makefile</a></td><td class='right'>17</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib.h</a></td><td class='right'>1525</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 15.6%;'/><td class='none' style='width: 84.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_6120_regs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_6120_regs.h</a></td><td class='right'>977</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 10.0%;'/><td class='none' style='width: 90.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_7220.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_7220.h</a></td><td class='right'>149</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 1.5%;'/><td class='none' style='width: 98.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_7220_regs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_7220_regs.h</a></td><td class='right'>1496</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 15.3%;'/><td class='none' style='width: 84.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_7322_regs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_7322_regs.h</a></td><td class='right'>3163</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 32.4%;'/><td class='none' style='width: 67.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_common.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_common.h</a></td><td class='right'>805</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 8.2%;'/><td class='none' style='width: 91.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_debugfs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_debugfs.c</a></td><td class='right'>274</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 2.8%;'/><td class='none' style='width: 97.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_debugfs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_debugfs.h</a></td><td class='right'>45</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_diag.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_diag.c</a></td><td class='right'>906</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 9.3%;'/><td class='none' style='width: 90.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_driver.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_driver.c</a></td><td class='right'>803</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 8.2%;'/><td class='none' style='width: 91.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_eeprom.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_eeprom.c</a></td><td class='right'>271</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 2.8%;'/><td class='none' style='width: 97.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_file_ops.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_file_ops.c</a></td><td class='right'>2405</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 24.6%;'/><td class='none' style='width: 75.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_fs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_fs.c</a></td><td class='right'>599</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 6.1%;'/><td class='none' style='width: 93.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_iba6120.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_iba6120.c</a></td><td class='right'>3537</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 36.2%;'/><td class='none' style='width: 63.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_iba7220.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_iba7220.c</a></td><td class='right'>4596</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 47.1%;'/><td class='none' style='width: 52.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_iba7322.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_iba7322.c</a></td><td class='right'>8506</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 87.1%;'/><td class='none' style='width: 12.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_init.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_init.c</a></td><td class='right'>1799</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 18.4%;'/><td class='none' style='width: 81.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_intr.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_intr.c</a></td><td class='right'>240</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 2.5%;'/><td class='none' style='width: 97.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_mad.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_mad.c</a></td><td class='right'>2499</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 25.6%;'/><td class='none' style='width: 74.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_mad.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_mad.h</a></td><td class='right'>300</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 3.1%;'/><td class='none' style='width: 96.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_pcie.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_pcie.c</a></td><td class='right'>613</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 6.3%;'/><td class='none' style='width: 93.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_pio_copy.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_pio_copy.c</a></td><td class='right'>64</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.7%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_qp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_qp.c</a></td><td class='right'>454</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 4.7%;'/><td class='none' style='width: 95.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_qsfp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_qsfp.c</a></td><td class='right'>549</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 5.6%;'/><td class='none' style='width: 94.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_qsfp.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_qsfp.h</a></td><td class='right'>188</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 1.9%;'/><td class='none' style='width: 98.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_rc.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_rc.c</a></td><td class='right'>2153</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 22.1%;'/><td class='none' style='width: 77.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_ruc.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_ruc.c</a></td><td class='right'>314</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 3.2%;'/><td class='none' style='width: 96.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_sd7220.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_sd7220.c</a></td><td class='right'>1446</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 14.8%;'/><td class='none' style='width: 85.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_sdma.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_sdma.c</a></td><td class='right'>999</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 10.2%;'/><td class='none' style='width: 89.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_sysfs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_sysfs.c</a></td><td class='right'>852</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 8.7%;'/><td class='none' style='width: 91.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_twsi.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_twsi.c</a></td><td class='right'>501</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 5.1%;'/><td class='none' style='width: 94.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_tx.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_tx.c</a></td><td class='right'>568</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 5.8%;'/><td class='none' style='width: 94.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_uc.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_uc.c</a></td><td class='right'>521</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 5.3%;'/><td class='none' style='width: 94.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_ud.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_ud.c</a></td><td class='right'>582</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 6.0%;'/><td class='none' style='width: 94.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_user_pages.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_user_pages.c</a></td><td class='right'>143</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 1.5%;'/><td class='none' style='width: 98.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_user_sdma.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_user_sdma.c</a></td><td class='right'>1461</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 15.0%;'/><td class='none' style='width: 85.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_user_sdma.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_user_sdma.h</a></td><td class='right'>52</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_verbs.c</a></td><td class='right'>1708</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 17.5%;'/><td class='none' style='width: 82.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_verbs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_verbs.h</a></td><td class='right'>404</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 4.1%;'/><td class='none' style='width: 95.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_wc_ppc64.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_wc_ppc64.c</a></td><td class='right'>62</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/qib/qib_wc_x86_64.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/qib/qib_wc_x86_64.c</a></td><td class='right'>150</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 1.5%;'/><td class='none' style='width: 98.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/Kconfig</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_abi.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_abi.h</a></td><td class='right'>2</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_debugfs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_debugfs.c</a></td><td class='right'>1</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_fwd.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_fwd.c</a></td><td class='right'>6</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_fwd.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_fwd.h</a></td><td class='right'>2</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_ib.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_ib.h</a></td><td class='right'>6</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_ib_main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_ib_main.c</a></td><td class='right'>167</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.9%;'/><td class='rem' style='width: 0.8%;'/><td class='none' style='width: 98.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c</a></td><td class='right'>35</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_ib_qp_grp.h</a></td><td class='right'>10</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_ib_sysfs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_ib_sysfs.c</a></td><td class='right'>103</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 98.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_ib_verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_ib_verbs.c</a></td><td class='right'>158</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 0.9%;'/><td class='none' style='width: 98.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_ib_verbs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_ib_verbs.h</a></td><td class='right'>21</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_transport.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_transport.c</a></td><td class='right'>1</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_uiom.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_uiom.c</a></td><td class='right'>49</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_uiom.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_uiom.h</a></td><td class='right'>5</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.c</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.h</a></td><td class='right'>4</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/usnic/usnic_vnic.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/usnic/usnic_vnic.c</a></td><td class='right'>1</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/vmw_pvrdma/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/vmw_pvrdma/Kconfig</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/vmw_pvrdma/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/vmw_pvrdma/Makefile</a></td><td class='right'>1</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/vmw_pvrdma/pvrdma.h</a></td><td class='right'>26</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c</a></td><td class='right'>61</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h</a></td><td class='right'>15</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/vmw_pvrdma/pvrdma_doorbell.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/vmw_pvrdma/pvrdma_doorbell.c</a></td><td class='right'>10</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c</a></td><td class='right'>158</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 0.9%;'/><td class='none' style='width: 98.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c</a></td><td class='right'>9</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c</a></td><td class='right'>15</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c</a></td><td class='right'>199</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 0.8%;'/><td class='none' style='width: 98.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c</a></td><td class='right'>13</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c</a></td><td class='right'>62</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h</a></td><td class='right'>68</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/Makefile</a></td><td class='right'>2</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/Kconfig</a></td><td class='right'>7</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/Makefile</a></td><td class='right'>1</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/ah.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/ah.c</a></td><td class='right'>71</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/ah.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/ah.h</a></td><td class='right'>56</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/cq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/cq.c</a></td><td class='right'>306</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.6%;'/><td class='rem' style='width: 1.6%;'/><td class='none' style='width: 96.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/cq.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/cq.h</a></td><td class='right'>55</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/mad.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/mad.c</a></td><td class='right'>56</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/mad.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/mad.h</a></td><td class='right'>52</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/mcast.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/mcast.c</a></td><td class='right'>58</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/mcast.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/mcast.h</a></td><td class='right'>50</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/mmap.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/mmap.c</a></td><td class='right'>49</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/mmap.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/mmap.h</a></td><td class='right'>50</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/mr.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/mr.c</a></td><td class='right'>255</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 2.3%;'/><td class='none' style='width: 97.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/mr.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/mr.h</a></td><td class='right'>68</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/pd.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/pd.c</a></td><td class='right'>47</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/pd.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/pd.h</a></td><td class='right'>52</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/qp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/qp.c</a></td><td class='right'>749</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.4%;'/><td class='rem' style='width: 3.3%;'/><td class='none' style='width: 92.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/qp.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/qp.h</a></td><td class='right'>57</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/rc.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/rc.c</a></td><td class='right'>92</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/srq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/srq.c</a></td><td class='right'>127</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 0.8%;'/><td class='none' style='width: 98.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/srq.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/srq.h</a></td><td class='right'>52</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/trace.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/trace.c</a></td><td class='right'>44</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/trace.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/trace.h</a></td><td class='right'>46</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/trace_cq.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/trace_cq.h</a></td><td class='right'>48</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/trace_mr.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/trace_mr.h</a></td><td class='right'>100</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.6%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/trace_qp.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/trace_qp.h</a></td><td class='right'>48</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/trace_rc.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/trace_rc.h</a></td><td class='right'>46</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/trace_rvt.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/trace_rvt.h</a></td><td class='right'>46</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/trace_tx.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/trace_tx.h</a></td><td class='right'>48</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/vt.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/vt.c</a></td><td class='right'>158</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 1.2%;'/><td class='none' style='width: 98.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rdmavt/vt.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rdmavt/vt.h</a></td><td class='right'>70</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/Kconfig</a></td><td class='right'>10</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/Makefile</a></td><td class='right'>4</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe.c</a></td><td class='right'>278</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.9%;'/><td class='rem' style='width: 1.9%;'/><td class='none' style='width: 97.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe.h</a></td><td class='right'>120</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 98.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_av.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_av.c</a></td><td class='right'>132</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.0%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 98.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_comp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_comp.c</a></td><td class='right'>414</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.4%;'/><td class='rem' style='width: 1.9%;'/><td class='none' style='width: 95.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_cq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_cq.c</a></td><td class='right'>113</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.9%;'/><td class='none' style='width: 98.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_hdr.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_hdr.h</a></td><td class='right'>257</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.4%;'/><td class='rem' style='width: 1.2%;'/><td class='none' style='width: 97.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_hw_counters.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_hw_counters.c</a></td><td class='right'>80</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_hw_counters.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_hw_counters.h</a></td><td class='right'>37</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_icrc.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_icrc.c</a></td><td class='right'>105</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 98.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_loc.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_loc.h</a></td><td class='right'>273</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 1.6%;'/><td class='none' style='width: 97.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_mcast.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_mcast.c</a></td><td class='right'>569</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.5%;'/><td class='rem' style='width: 1.3%;'/><td class='none' style='width: 94.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_mmap.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_mmap.c</a></td><td class='right'>39</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_mr.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_mr.c</a></td><td class='right'>879</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 5.0%;'/><td class='rem' style='width: 4.0%;'/><td class='none' style='width: 91.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_mw.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_mw.c</a></td><td class='right'>338</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.5%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 96.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_net.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_net.c</a></td><td class='right'>357</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.1%;'/><td class='rem' style='width: 1.6%;'/><td class='none' style='width: 96.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_net.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_net.h</a></td><td class='right'>29</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_odp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_odp.c</a></td><td class='right'>577</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 5.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 94.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_opcode.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_opcode.c</a></td><td class='right'>816</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.3%;'/><td class='rem' style='width: 4.1%;'/><td class='none' style='width: 91.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_opcode.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_opcode.h</a></td><td class='right'>58</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_param.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_param.h</a></td><td class='right'>107</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 98.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_pool.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_pool.c</a></td><td class='right'>616</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.7%;'/><td class='rem' style='width: 4.6%;'/><td class='none' style='width: 93.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_pool.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_pool.h</a></td><td class='right'>145</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 1.2%;'/><td class='none' style='width: 98.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_qp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_qp.c</a></td><td class='right'>740</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.2%;'/><td class='rem' style='width: 3.4%;'/><td class='none' style='width: 92.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_queue.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_queue.c</a></td><td class='right'>97</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_queue.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_queue.h</a></td><td class='right'>308</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.1%;'/><td class='rem' style='width: 1.0%;'/><td class='none' style='width: 96.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_recv.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_recv.c</a></td><td class='right'>296</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 1.8%;'/><td class='none' style='width: 97.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_req.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_req.c</a></td><td class='right'>538</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.2%;'/><td class='rem' style='width: 2.3%;'/><td class='none' style='width: 94.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_resp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_resp.c</a></td><td class='right'>1125</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 7.3%;'/><td class='rem' style='width: 4.2%;'/><td class='none' style='width: 88.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_srq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_srq.c</a></td><td class='right'>217</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 1.0%;'/><td class='none' style='width: 97.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='del'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_sysfs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_sysfs.c</a></td><td class='right'>135</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 1.4%;'/><td class='none' style='width: 98.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_task.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_task.c</a></td><td class='right'>334</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.4%;'/><td class='rem' style='width: 1.1%;'/><td class='none' style='width: 96.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_task.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_task.h</a></td><td class='right'>78</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.6%;'/><td class='none' style='width: 99.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_verbs.c</a></td><td class='right'>1484</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 9.3%;'/><td class='rem' style='width: 6.0%;'/><td class='none' style='width: 84.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/rxe/rxe_verbs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/rxe/rxe_verbs.h</a></td><td class='right'>296</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.8%;'/><td class='rem' style='width: 1.3%;'/><td class='none' style='width: 97.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/siw/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/siw/Kconfig</a></td><td class='right'>20</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/siw/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/siw/Makefile</a></td><td class='right'>11</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/siw/iwarp.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/siw/iwarp.h</a></td><td class='right'>367</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.8%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 96.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/siw/siw.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/siw/siw.h</a></td><td class='right'>729</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 7.5%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 92.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/siw/siw_cm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/siw/siw_cm.c</a></td><td class='right'>2009</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 20.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 79.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/siw/siw_cm.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/siw/siw_cm.h</a></td><td class='right'>133</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/siw/siw_cq.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/siw/siw_cq.c</a></td><td class='right'>122</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/siw/siw_main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/siw/siw_main.c</a></td><td class='right'>508</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 5.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 94.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/siw/siw_mem.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/siw/siw_mem.c</a></td><td class='right'>400</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.1%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 95.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/siw/siw_mem.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/siw/siw_mem.h</a></td><td class='right'>69</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.7%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/siw/siw_qp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/siw/siw_qp.c</a></td><td class='right'>1318</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 13.5%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 86.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/siw/siw_qp_rx.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/siw/siw_qp_rx.c</a></td><td class='right'>1455</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 14.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 85.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/siw/siw_qp_tx.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/siw/siw_qp_tx.c</a></td><td class='right'>1306</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 13.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 86.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/siw/siw_verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/siw/siw_verbs.c</a></td><td class='right'>1891</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 19.4%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 80.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/sw/siw/siw_verbs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/sw/siw/siw_verbs.h</a></td><td class='right'>91</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/Makefile</a></td><td class='right'>1</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/ipoib/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/ipoib/Kconfig</a></td><td class='right'>11</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/ipoib/ipoib.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/ipoib/ipoib.h</a></td><td class='right'>43</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/ipoib/ipoib_cm.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/ipoib/ipoib_cm.c</a></td><td class='right'>54</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/ipoib/ipoib_ethtool.c</a></td><td class='right'>35</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/ipoib/ipoib_fs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/ipoib/ipoib_fs.c</a></td><td class='right'>50</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.5%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/ipoib/ipoib_ib.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/ipoib/ipoib_ib.c</a></td><td class='right'>189</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 0.8%;'/><td class='none' style='width: 98.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/ipoib/ipoib_main.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/ipoib/ipoib_main.c</a></td><td class='right'>467</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.2%;'/><td class='rem' style='width: 1.6%;'/><td class='none' style='width: 95.2%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/ipoib/ipoib_multicast.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/ipoib/ipoib_multicast.c</a></td><td class='right'>52</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.4%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/ipoib/ipoib_netlink.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/ipoib/ipoib_netlink.c</a></td><td class='right'>29</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/ipoib/ipoib_verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/ipoib/ipoib_verbs.c</a></td><td class='right'>26</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/ipoib/ipoib_vlan.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/ipoib/ipoib_vlan.c</a></td><td class='right'>39</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/iser/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/iser/Kconfig</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/iser/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/iser/Makefile</a></td><td class='right'>1</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/iser/iscsi_iser.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/iser/iscsi_iser.c</a></td><td class='right'>230</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.0%;'/><td class='rem' style='width: 1.4%;'/><td class='none' style='width: 97.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/iser/iscsi_iser.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/iser/iscsi_iser.h</a></td><td class='right'>245</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 2.0%;'/><td class='none' style='width: 97.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/iser/iser_initiator.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/iser/iser_initiator.c</a></td><td class='right'>223</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.0%;'/><td class='rem' style='width: 1.3%;'/><td class='none' style='width: 97.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/iser/iser_memory.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/iser/iser_memory.c</a></td><td class='right'>428</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.2%;'/><td class='rem' style='width: 3.1%;'/><td class='none' style='width: 95.6%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/iser/iser_verbs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/iser/iser_verbs.c</a></td><td class='right'>643</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.2%;'/><td class='rem' style='width: 4.4%;'/><td class='none' style='width: 93.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/isert/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/isert/Kconfig</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/isert/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/isert/Makefile</a></td><td class='right'>1</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/isert/ib_isert.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/isert/ib_isert.c</a></td><td class='right'>549</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.4%;'/><td class='rem' style='width: 3.2%;'/><td class='none' style='width: 94.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/isert/ib_isert.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/isert/ib_isert.h</a></td><td class='right'>69</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.4%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 99.3%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/opa_vnic/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/opa_vnic/Kconfig</a></td><td class='right'>7</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/opa_vnic/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/opa_vnic/Makefile</a></td><td class='right'>4</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h</a></td><td class='right'>33</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/opa_vnic/opa_vnic_ethtool.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/opa_vnic/opa_vnic_ethtool.c</a></td><td class='right'>12</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/opa_vnic/opa_vnic_internal.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/opa_vnic/opa_vnic_internal.h</a></td><td class='right'>9</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.1%;'/><td class='none' style='width: 99.9%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/opa_vnic/opa_vnic_netdev.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/opa_vnic/opa_vnic_netdev.c</a></td><td class='right'>1</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c</a></td><td class='right'>27</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.1%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema_iface.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/opa_vnic/opa_vnic_vema_iface.c</a></td><td class='right'>2</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/Kconfig</a></td><td class='right'>27</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/Makefile</a></td><td class='right'>21</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/README?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/README</a></td><td class='right'>213</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c</a></td><td class='right'>198</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c</a></td><td class='right'>514</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 5.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 94.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c</a></td><td class='right'>15</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h</a></td><td class='right'>86</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/rtrs-clt.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/rtrs-clt.c</a></td><td class='right'>3207</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 32.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 67.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/rtrs-clt.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/rtrs-clt.h</a></td><td class='right'>252</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 2.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 97.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/rtrs-log.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/rtrs-log.h</a></td><td class='right'>28</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/rtrs-pri.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/rtrs-pri.h</a></td><td class='right'>408</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 95.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/rtrs-srv-stats.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/rtrs-srv-stats.c</a></td><td class='right'>51</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.5%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c</a></td><td class='right'>319</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 3.3%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 96.7%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/rtrs-srv-trace.c</a></td><td class='right'>16</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.2%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.8%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h</a></td><td class='right'>88</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 99.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/rtrs-srv.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/rtrs-srv.c</a></td><td class='right'>2346</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 24.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 76.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/rtrs-srv.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/rtrs-srv.h</a></td><td class='right'>156</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/rtrs.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/rtrs.c</a></td><td class='right'>645</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 6.6%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 93.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='add'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/rtrs/rtrs.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/rtrs/rtrs.h</a></td><td class='right'>188</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 1.9%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 98.1%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/srp/Kbuild?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/srp/Kbuild</a></td><td class='right'>1</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/srp/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/srp/Kconfig</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/srp/ib_srp.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/srp/ib_srp.c</a></td><td class='right'>1039</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.7%;'/><td class='rem' style='width: 5.9%;'/><td class='none' style='width: 89.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/srp/ib_srp.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/srp/ib_srp.h</a></td><td class='right'>57</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.3%;'/><td class='rem' style='width: 0.3%;'/><td class='none' style='width: 99.4%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/srpt/Kconfig?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/srpt/Kconfig</a></td><td class='right'>3</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/srpt/Makefile?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/srpt/Makefile</a></td><td class='right'>1</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.0%;'/><td class='rem' style='width: 0.0%;'/><td class='none' style='width: 100.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/srpt/ib_srpt.c?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/srpt/ib_srpt.c</a></td><td class='right'>683</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 4.1%;'/><td class='rem' style='width: 2.9%;'/><td class='none' style='width: 93.0%;'/></tr></table></td></tr>
<tr><td class='mode'>-rw-r--r--</td><td class='upd'><a href='/cgit/linux-arm.git/diff/drivers/infiniband/ulp/srpt/ib_srpt.h?id2=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/ulp/srpt/ib_srpt.h</a></td><td class='right'>75</td><td class='graph'><table summary='file diffstat' width='100%'><tr><td class='add' style='width: 0.5%;'/><td class='rem' style='width: 0.2%;'/><td class='none' style='width: 99.2%;'/></tr></table></td></tr>
</table><div class='diffstat-summary'>717 files changed, 168443 insertions, 175986 deletions</div><table summary='diff' class='diff'><tr><td><div class='head'>diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig<br/>index cbfbea49f126..794b9778816b 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/Kconfig?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/Kconfig</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/Kconfig?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/Kconfig</a></div><div class='hunk'>@@ -1,3 +1,4 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='ctx'> menuconfig INFINIBAND</div><div class='ctx'> 	tristate "InfiniBand support"</div><div class='ctx'> 	depends on HAS_IOMEM &amp;&amp; HAS_DMA</div><div class='hunk'>@@ -6,7 +7,8 @@ menuconfig INFINIBAND</div><div class='ctx'> 	depends on m || IPV6 != m</div><div class='ctx'> 	depends on !ALPHA</div><div class='ctx'> 	select IRQ_POLL</div><div class='del'>-	---help---</div><div class='add'>+	select DIMLIB</div><div class='add'>+	help</div><div class='ctx'> 	  Core support for InfiniBand (IB).  Make sure to also select</div><div class='ctx'> 	  any protocols you wish to use as well as drivers for your</div><div class='ctx'> 	  InfiniBand hardware.</div><div class='hunk'>@@ -16,7 +18,7 @@ if INFINIBAND</div><div class='ctx'> config INFINIBAND_USER_MAD</div><div class='ctx'> 	tristate "InfiniBand userspace MAD support"</div><div class='ctx'> 	depends on INFINIBAND</div><div class='del'>-	---help---</div><div class='add'>+	help</div><div class='ctx'> 	  Userspace InfiniBand Management Datagram (MAD) support.  This</div><div class='ctx'> 	  is the kernel side of the userspace MAD support, which allows</div><div class='ctx'> 	  userspace processes to send and receive MADs. You will also</div><div class='hunk'>@@ -26,7 +28,7 @@ config INFINIBAND_USER_MAD</div><div class='ctx'> config INFINIBAND_USER_ACCESS</div><div class='ctx'> 	tristate "InfiniBand userspace access (verbs and CM)"</div><div class='ctx'> 	depends on MMU</div><div class='del'>-	---help---</div><div class='add'>+	help</div><div class='ctx'> 	  Userspace InfiniBand access support.  This enables the</div><div class='ctx'> 	  kernel side of userspace verbs and the userspace</div><div class='ctx'> 	  communication manager (CM).  This allows userspace processes</div><div class='hunk'>@@ -35,37 +37,21 @@ config INFINIBAND_USER_ACCESS</div><div class='ctx'> 	  libibverbs, libibcm and a hardware driver library from</div><div class='ctx'> 	  rdma-core &lt;https://github.com/linux-rdma/rdma-core&gt;.</div><div class='ctx'> </div><div class='del'>-config INFINIBAND_USER_ACCESS_UCM</div><div class='del'>-	tristate "Userspace CM (UCM, DEPRECATED)"</div><div class='del'>-	depends on BROKEN || COMPILE_TEST</div><div class='del'>-	depends on INFINIBAND_USER_ACCESS</div><div class='del'>-	help</div><div class='del'>-	  The UCM module has known security flaws, which no one is</div><div class='del'>-	  interested to fix. The user-space part of this code was</div><div class='del'>-	  dropped from the upstream a long time ago.</div><div class='del'>-</div><div class='del'>-	  This option is DEPRECATED and planned to be removed.</div><div class='del'>-</div><div class='del'>-config INFINIBAND_EXP_LEGACY_VERBS_NEW_UAPI</div><div class='del'>-	bool "Allow experimental legacy verbs in new ioctl uAPI  (EXPERIMENTAL)"</div><div class='del'>-	depends on INFINIBAND_USER_ACCESS</div><div class='del'>-	---help---</div><div class='del'>-	  IOCTL based uAPI support for Infiniband is enabled by default for</div><div class='del'>-	  new verbs only. This allows userspace to invoke the IOCTL based uAPI</div><div class='del'>-	  for current legacy verbs too.</div><div class='del'>-</div><div class='ctx'> config INFINIBAND_USER_MEM</div><div class='ctx'> 	bool</div><div class='ctx'> 	depends on INFINIBAND_USER_ACCESS != n</div><div class='ctx'> 	depends on MMU</div><div class='add'>+	select DMA_SHARED_BUFFER</div><div class='ctx'> 	default y</div><div class='ctx'> </div><div class='ctx'> config INFINIBAND_ON_DEMAND_PAGING</div><div class='ctx'> 	bool "InfiniBand on-demand paging support"</div><div class='ctx'> 	depends on INFINIBAND_USER_MEM</div><div class='ctx'> 	select MMU_NOTIFIER</div><div class='add'>+	select INTERVAL_TREE</div><div class='add'>+	select HMM_MIRROR</div><div class='ctx'> 	default y</div><div class='del'>-	---help---</div><div class='add'>+	help</div><div class='ctx'> 	  On demand paging support for the InfiniBand subsystem.</div><div class='ctx'> 	  Together with driver support this allows registration of</div><div class='ctx'> 	  memory regions without pinning their pages, fetching the</div><div class='hunk'>@@ -75,7 +61,7 @@ config INFINIBAND_ADDR_TRANS</div><div class='ctx'> 	bool "RDMA/CM"</div><div class='ctx'> 	depends on INFINIBAND</div><div class='ctx'> 	default y</div><div class='del'>-	---help---</div><div class='add'>+	help</div><div class='ctx'> 	  Support for RDMA communication manager (CM).</div><div class='ctx'> 	  This allows for a generic connection abstraction over RDMA.</div><div class='ctx'> </div><div class='hunk'>@@ -83,30 +69,37 @@ config INFINIBAND_ADDR_TRANS_CONFIGFS</div><div class='ctx'> 	bool</div><div class='ctx'> 	depends on INFINIBAND_ADDR_TRANS &amp;&amp; CONFIGFS_FS &amp;&amp; !(INFINIBAND=y &amp;&amp; CONFIGFS_FS=m)</div><div class='ctx'> 	default y</div><div class='del'>-	---help---</div><div class='add'>+	help</div><div class='ctx'> 	  ConfigFS support for RDMA communication manager (CM).</div><div class='ctx'> 	  This allows the user to config the default GID type that the CM</div><div class='ctx'> 	  uses for each device, when initiaing new connections.</div><div class='ctx'> </div><div class='add'>+config INFINIBAND_VIRT_DMA</div><div class='add'>+	def_bool !HIGHMEM</div><div class='add'>+</div><div class='ctx'> if INFINIBAND_USER_ACCESS || !INFINIBAND_USER_ACCESS</div><div class='del'>-source "drivers/infiniband/hw/mthca/Kconfig"</div><div class='del'>-source "drivers/infiniband/hw/qib/Kconfig"</div><div class='del'>-source "drivers/infiniband/hw/cxgb3/Kconfig"</div><div class='add'>+if !UML</div><div class='add'>+source "drivers/infiniband/hw/bnxt_re/Kconfig"</div><div class='add'>+source "drivers/infiniband/hw/bng_re/Kconfig"</div><div class='ctx'> source "drivers/infiniband/hw/cxgb4/Kconfig"</div><div class='ctx'> source "drivers/infiniband/hw/efa/Kconfig"</div><div class='del'>-source "drivers/infiniband/hw/i40iw/Kconfig"</div><div class='add'>+source "drivers/infiniband/hw/erdma/Kconfig"</div><div class='add'>+source "drivers/infiniband/hw/hfi1/Kconfig"</div><div class='add'>+source "drivers/infiniband/hw/hns/Kconfig"</div><div class='add'>+source "drivers/infiniband/hw/ionic/Kconfig"</div><div class='add'>+source "drivers/infiniband/hw/irdma/Kconfig"</div><div class='add'>+source "drivers/infiniband/hw/mana/Kconfig"</div><div class='ctx'> source "drivers/infiniband/hw/mlx4/Kconfig"</div><div class='ctx'> source "drivers/infiniband/hw/mlx5/Kconfig"</div><div class='del'>-source "drivers/infiniband/hw/nes/Kconfig"</div><div class='add'>+source "drivers/infiniband/hw/mthca/Kconfig"</div><div class='ctx'> source "drivers/infiniband/hw/ocrdma/Kconfig"</div><div class='del'>-source "drivers/infiniband/hw/vmw_pvrdma/Kconfig"</div><div class='del'>-source "drivers/infiniband/hw/usnic/Kconfig"</div><div class='del'>-source "drivers/infiniband/hw/hns/Kconfig"</div><div class='del'>-source "drivers/infiniband/hw/bnxt_re/Kconfig"</div><div class='del'>-source "drivers/infiniband/hw/hfi1/Kconfig"</div><div class='ctx'> source "drivers/infiniband/hw/qedr/Kconfig"</div><div class='add'>+source "drivers/infiniband/hw/usnic/Kconfig"</div><div class='add'>+source "drivers/infiniband/hw/vmw_pvrdma/Kconfig"</div><div class='ctx'> source "drivers/infiniband/sw/rdmavt/Kconfig"</div><div class='add'>+endif # !UML</div><div class='ctx'> source "drivers/infiniband/sw/rxe/Kconfig"</div><div class='add'>+source "drivers/infiniband/sw/siw/Kconfig"</div><div class='ctx'> endif</div><div class='ctx'> </div><div class='ctx'> source "drivers/infiniband/ulp/ipoib/Kconfig"</div><div class='hunk'>@@ -116,6 +109,7 @@ source "drivers/infiniband/ulp/srpt/Kconfig"</div><div class='ctx'> </div><div class='ctx'> source "drivers/infiniband/ulp/iser/Kconfig"</div><div class='ctx'> source "drivers/infiniband/ulp/isert/Kconfig"</div><div class='add'>+source "drivers/infiniband/ulp/rtrs/Kconfig"</div><div class='ctx'> </div><div class='ctx'> source "drivers/infiniband/ulp/opa_vnic/Kconfig"</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/Makefile b/drivers/infiniband/Makefile<br/>index fad0b44c356f..8603cdfcfdcb 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/Makefile?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/Makefile</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/Makefile?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/Makefile</a></div><div class='hunk'>@@ -1,3 +1,4 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='ctx'> obj-$(CONFIG_INFINIBAND)		+= core/</div><div class='ctx'> obj-$(CONFIG_INFINIBAND)		+= hw/</div><div class='ctx'> obj-$(CONFIG_INFINIBAND)		+= ulp/</div><div class='head'>diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile<br/>index 313f2349b518..f483e0c12444 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/Makefile?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/Makefile</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/Makefile?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/Makefile</a></div><div class='hunk'>@@ -6,22 +6,23 @@ obj-$(CONFIG_INFINIBAND) +=		ib_core.o ib_cm.o iw_cm.o \</div><div class='ctx'> 					$(infiniband-y)</div><div class='ctx'> obj-$(CONFIG_INFINIBAND_USER_MAD) +=	ib_umad.o</div><div class='ctx'> obj-$(CONFIG_INFINIBAND_USER_ACCESS) += ib_uverbs.o $(user_access-y)</div><div class='del'>-obj-$(CONFIG_INFINIBAND_USER_ACCESS_UCM) += ib_ucm.o $(user_access-y)</div><div class='ctx'> </div><div class='ctx'> ib_core-y :=			packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \</div><div class='del'>-				device.o fmr_pool.o cache.o netlink.o \</div><div class='add'>+				device.o cache.o netlink.o \</div><div class='ctx'> 				roce_gid_mgmt.o mr_pool.o addr.o sa_query.o \</div><div class='ctx'> 				multicast.o mad.o smi.o agent.o mad_rmpp.o \</div><div class='del'>-				nldev.o restrack.o</div><div class='add'>+				nldev.o restrack.o counters.o ib_core_uverbs.o \</div><div class='add'>+				trace.o lag.o</div><div class='ctx'> </div><div class='ctx'> ib_core-$(CONFIG_SECURITY_INFINIBAND) += security.o</div><div class='ctx'> ib_core-$(CONFIG_CGROUP_RDMA) += cgroup.o</div><div class='ctx'> </div><div class='del'>-ib_cm-y :=			cm.o</div><div class='add'>+ib_cm-y :=			cm.o cm_trace.o</div><div class='ctx'> </div><div class='ctx'> iw_cm-y :=			iwcm.o iwpm_util.o iwpm_msg.o</div><div class='ctx'> </div><div class='del'>-rdma_cm-y :=			cma.o</div><div class='add'>+CFLAGS_cma_trace.o +=		-I$(src)</div><div class='add'>+rdma_cm-y :=			cma.o cma_trace.o</div><div class='ctx'> </div><div class='ctx'> rdma_cm-$(CONFIG_INFINIBAND_ADDR_TRANS_CONFIGFS) += cma_configfs.o</div><div class='ctx'> </div><div class='hunk'>@@ -29,13 +30,17 @@ rdma_ucm-y :=			ucma.o</div><div class='ctx'> </div><div class='ctx'> ib_umad-y :=			user_mad.o</div><div class='ctx'> </div><div class='del'>-ib_ucm-y :=			ucm.o</div><div class='del'>-</div><div class='ctx'> ib_uverbs-y :=			uverbs_main.o uverbs_cmd.o uverbs_marshall.o \</div><div class='ctx'> 				rdma_core.o uverbs_std_types.o uverbs_ioctl.o \</div><div class='ctx'> 				uverbs_std_types_cq.o \</div><div class='add'>+				uverbs_std_types_dmah.o \</div><div class='ctx'> 				uverbs_std_types_flow_action.o uverbs_std_types_dm.o \</div><div class='ctx'> 				uverbs_std_types_mr.o uverbs_std_types_counters.o \</div><div class='del'>-				uverbs_uapi.o uverbs_std_types_device.o</div><div class='del'>-ib_uverbs-$(CONFIG_INFINIBAND_USER_MEM) += umem.o</div><div class='add'>+				uverbs_uapi.o uverbs_std_types_device.o \</div><div class='add'>+				uverbs_std_types_async_fd.o \</div><div class='add'>+				uverbs_std_types_srq.o \</div><div class='add'>+				uverbs_std_types_wq.o \</div><div class='add'>+				uverbs_std_types_qp.o \</div><div class='add'>+				ucaps.o</div><div class='add'>+ib_uverbs-$(CONFIG_INFINIBAND_USER_MEM) += umem.o umem_dmabuf.o</div><div class='ctx'> ib_uverbs-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += umem_odp.o</div><div class='head'>diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c<br/>index 2f7d14159841..61596cda2b65 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/addr.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/addr.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/addr.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/addr.c</a></div><div class='hunk'>@@ -37,7 +37,6 @@</div><div class='ctx'> #include &lt;linux/inetdevice.h&gt;</div><div class='ctx'> #include &lt;linux/slab.h&gt;</div><div class='ctx'> #include &lt;linux/workqueue.h&gt;</div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='ctx'> #include &lt;net/arp.h&gt;</div><div class='ctx'> #include &lt;net/neighbour.h&gt;</div><div class='ctx'> #include &lt;net/route.h&gt;</div><div class='hunk'>@@ -76,7 +75,9 @@ static struct workqueue_struct *addr_wq;</div><div class='ctx'> </div><div class='ctx'> static const struct nla_policy ib_nl_addr_policy[LS_NLA_TYPE_MAX] = {</div><div class='ctx'> 	[LS_NLA_TYPE_DGID] = {.type = NLA_BINARY,</div><div class='del'>-		.len = sizeof(struct rdma_nla_ls_gid)},</div><div class='add'>+		.len = sizeof(struct rdma_nla_ls_gid),</div><div class='add'>+		.validation_type = NLA_VALIDATE_MIN,</div><div class='add'>+		.min = sizeof(struct rdma_nla_ls_gid)},</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static inline bool ib_nl_is_good_ip_resp(const struct nlmsghdr *nlh)</div><div class='hunk'>@@ -139,7 +140,7 @@ int ib_nl_handle_ip_res_resp(struct sk_buff *skb,</div><div class='ctx'> 	if (ib_nl_is_good_ip_resp(nlh))</div><div class='ctx'> 		ib_nl_process_good_ip_rsep(nlh);</div><div class='ctx'> </div><div class='del'>-	return skb-&gt;len;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int ib_nl_ip_send_msg(struct rdma_dev_addr *dev_addr,</div><div class='hunk'>@@ -183,7 +184,7 @@ static int ib_nl_ip_send_msg(struct rdma_dev_addr *dev_addr,</div><div class='ctx'> </div><div class='ctx'> 	/* Repair the nlmsg header length */</div><div class='ctx'> 	nlmsg_end(skb, nlh);</div><div class='del'>-	rdma_nl_multicast(skb, RDMA_NL_GROUP_LS, GFP_KERNEL);</div><div class='add'>+	rdma_nl_multicast(&amp;init_net, skb, RDMA_NL_GROUP_LS, GFP_KERNEL);</div><div class='ctx'> </div><div class='ctx'> 	/* Make the request retry, so when we get the response from userspace</div><div class='ctx'> 	 * we will have something.</div><div class='hunk'>@@ -337,7 +338,7 @@ static int dst_fetch_ha(const struct dst_entry *dst,</div><div class='ctx'> 		neigh_event_send(n, NULL);</div><div class='ctx'> 		ret = -ENODATA;</div><div class='ctx'> 	} else {</div><div class='del'>-		memcpy(dev_addr-&gt;dst_dev_addr, n-&gt;ha, MAX_ADDR_LEN);</div><div class='add'>+		neigh_ha_snapshot(dev_addr-&gt;dst_dev_addr, n, dst-&gt;dev);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	neigh_release(n);</div><div class='hunk'>@@ -347,16 +348,10 @@ static int dst_fetch_ha(const struct dst_entry *dst,</div><div class='ctx'> </div><div class='ctx'> static bool has_gateway(const struct dst_entry *dst, sa_family_t family)</div><div class='ctx'> {</div><div class='del'>-	struct rtable *rt;</div><div class='del'>-	struct rt6_info *rt6;</div><div class='del'>-</div><div class='del'>-	if (family == AF_INET) {</div><div class='del'>-		rt = container_of(dst, struct rtable, dst);</div><div class='del'>-		return rt-&gt;rt_gw_family == AF_INET;</div><div class='del'>-	}</div><div class='add'>+	if (family == AF_INET)</div><div class='add'>+		return dst_rtable(dst)-&gt;rt_uses_gateway;</div><div class='ctx'> </div><div class='del'>-	rt6 = container_of(dst, struct rt6_info, dst);</div><div class='del'>-	return rt6-&gt;rt6i_flags &amp; RTF_GATEWAY;</div><div class='add'>+	return dst_rt6_info(dst)-&gt;rt6i_flags &amp; RTF_GATEWAY;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int fetch_ha(const struct dst_entry *dst, struct rdma_dev_addr *dev_addr,</div><div class='hunk'>@@ -371,6 +366,8 @@ static int fetch_ha(const struct dst_entry *dst, struct rdma_dev_addr *dev_addr,</div><div class='ctx'> 		(const void *)&amp;dst_in6-&gt;sin6_addr;</div><div class='ctx'> 	sa_family_t family = dst_in-&gt;sa_family;</div><div class='ctx'> </div><div class='add'>+	might_sleep();</div><div class='add'>+</div><div class='ctx'> 	/* If we have a gateway in IB mode then it must be an IB network */</div><div class='ctx'> 	if (has_gateway(dst, family) &amp;&amp; dev_addr-&gt;network == RDMA_NETWORK_IB)</div><div class='ctx'> 		return ib_nl_fetch_ha(dev_addr, daddr, seq, family);</div><div class='hunk'>@@ -421,16 +418,15 @@ static int addr6_resolve(struct sockaddr *src_sock,</div><div class='ctx'> 				(const struct sockaddr_in6 *)dst_sock;</div><div class='ctx'> 	struct flowi6 fl6;</div><div class='ctx'> 	struct dst_entry *dst;</div><div class='del'>-	int ret;</div><div class='ctx'> </div><div class='ctx'> 	memset(&amp;fl6, 0, sizeof fl6);</div><div class='ctx'> 	fl6.daddr = dst_in-&gt;sin6_addr;</div><div class='ctx'> 	fl6.saddr = src_in-&gt;sin6_addr;</div><div class='ctx'> 	fl6.flowi6_oif = addr-&gt;bound_dev_if;</div><div class='ctx'> </div><div class='del'>-	ret = ipv6_stub-&gt;ipv6_dst_lookup(addr-&gt;net, NULL, &amp;dst, &amp;fl6);</div><div class='del'>-	if (ret &lt; 0)</div><div class='del'>-		return ret;</div><div class='add'>+	dst = ipv6_stub-&gt;ipv6_dst_lookup_flow(addr-&gt;net, NULL, &amp;fl6, NULL);</div><div class='add'>+	if (IS_ERR(dst))</div><div class='add'>+		return PTR_ERR(dst);</div><div class='ctx'> </div><div class='ctx'> 	if (ipv6_addr_any(&amp;src_in-&gt;sin6_addr))</div><div class='ctx'> 		src_in-&gt;sin6_addr = fl6.saddr;</div><div class='hunk'>@@ -450,63 +446,41 @@ static int addr6_resolve(struct sockaddr *src_sock,</div><div class='ctx'> }</div><div class='ctx'> #endif</div><div class='ctx'> </div><div class='add'>+static bool is_dst_local(const struct dst_entry *dst)</div><div class='add'>+{</div><div class='add'>+	if (dst-&gt;ops-&gt;family == AF_INET)</div><div class='add'>+		return !!(dst_rtable(dst)-&gt;rt_type &amp; RTN_LOCAL);</div><div class='add'>+	else if (dst-&gt;ops-&gt;family == AF_INET6)</div><div class='add'>+		return !!(dst_rt6_info(dst)-&gt;rt6i_flags &amp; RTF_LOCAL);</div><div class='add'>+	else</div><div class='add'>+		return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int addr_resolve_neigh(const struct dst_entry *dst,</div><div class='ctx'> 			      const struct sockaddr *dst_in,</div><div class='ctx'> 			      struct rdma_dev_addr *addr,</div><div class='del'>-			      unsigned int ndev_flags,</div><div class='ctx'> 			      u32 seq)</div><div class='ctx'> {</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	if (ndev_flags &amp; IFF_LOOPBACK) {</div><div class='add'>+	if (is_dst_local(dst)) {</div><div class='add'>+		/* When the destination is local entry, source and destination</div><div class='add'>+		 * are same. Skip the neighbour lookup.</div><div class='add'>+		 */</div><div class='ctx'> 		memcpy(addr-&gt;dst_dev_addr, addr-&gt;src_dev_addr, MAX_ADDR_LEN);</div><div class='del'>-	} else {</div><div class='del'>-		if (!(ndev_flags &amp; IFF_NOARP)) {</div><div class='del'>-			/* If the device doesn't do ARP internally */</div><div class='del'>-			ret = fetch_ha(dst, addr, dst_in, seq);</div><div class='del'>-		}</div><div class='add'>+		return 0;</div><div class='ctx'> 	}</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int copy_src_l2_addr(struct rdma_dev_addr *dev_addr,</div><div class='del'>-			    const struct sockaddr *dst_in,</div><div class='del'>-			    const struct dst_entry *dst,</div><div class='del'>-			    const struct net_device *ndev)</div><div class='del'>-{</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	if (dst-&gt;dev-&gt;flags &amp; IFF_LOOPBACK)</div><div class='del'>-		ret = rdma_translate_ip(dst_in, dev_addr);</div><div class='del'>-	else</div><div class='del'>-		rdma_copy_src_l2_addr(dev_addr, dst-&gt;dev);</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * If there's a gateway and type of device not ARPHRD_INFINIBAND,</div><div class='del'>-	 * we're definitely in RoCE v2 (as RoCE v1 isn't routable) set the</div><div class='del'>-	 * network type accordingly.</div><div class='del'>-	 */</div><div class='del'>-	if (has_gateway(dst, dst_in-&gt;sa_family) &amp;&amp;</div><div class='del'>-	    ndev-&gt;type != ARPHRD_INFINIBAND)</div><div class='del'>-		dev_addr-&gt;network = dst_in-&gt;sa_family == AF_INET ?</div><div class='del'>-						RDMA_NETWORK_IPV4 :</div><div class='del'>-						RDMA_NETWORK_IPV6;</div><div class='del'>-	else</div><div class='del'>-		dev_addr-&gt;network = RDMA_NETWORK_IB;</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='add'>+	return fetch_ha(dst, addr, dst_in, seq);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int rdma_set_src_addr_rcu(struct rdma_dev_addr *dev_addr,</div><div class='del'>-				 unsigned int *ndev_flags,</div><div class='ctx'> 				 const struct sockaddr *dst_in,</div><div class='ctx'> 				 const struct dst_entry *dst)</div><div class='ctx'> {</div><div class='ctx'> 	struct net_device *ndev = READ_ONCE(dst-&gt;dev);</div><div class='ctx'> </div><div class='del'>-	*ndev_flags = ndev-&gt;flags;</div><div class='ctx'> 	/* A physical device must be the RDMA device to use */</div><div class='del'>-	if (ndev-&gt;flags &amp; IFF_LOOPBACK) {</div><div class='add'>+	if (is_dst_local(dst)) {</div><div class='add'>+		int ret;</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * RDMA (IB/RoCE, iWarp) doesn't run on lo interface or</div><div class='ctx'> 		 * loopback IP address. So if route is resolved to loopback</div><div class='hunk'>@@ -516,9 +490,27 @@ static int rdma_set_src_addr_rcu(struct rdma_dev_addr *dev_addr,</div><div class='ctx'> 		ndev = rdma_find_ndev_for_src_ip_rcu(dev_net(ndev), dst_in);</div><div class='ctx'> 		if (IS_ERR(ndev))</div><div class='ctx'> 			return -ENODEV;</div><div class='add'>+		ret = rdma_translate_ip(dst_in, dev_addr);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	} else {</div><div class='add'>+		rdma_copy_src_l2_addr(dev_addr, dst-&gt;dev);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return copy_src_l2_addr(dev_addr, dst_in, dst, ndev);</div><div class='add'>+	/*</div><div class='add'>+	 * If there's a gateway and type of device not ARPHRD_INFINIBAND,</div><div class='add'>+	 * we're definitely in RoCE v2 (as RoCE v1 isn't routable) set the</div><div class='add'>+	 * network type accordingly.</div><div class='add'>+	 */</div><div class='add'>+	if (has_gateway(dst, dst_in-&gt;sa_family) &amp;&amp;</div><div class='add'>+	    ndev-&gt;type != ARPHRD_INFINIBAND)</div><div class='add'>+		dev_addr-&gt;network = dst_in-&gt;sa_family == AF_INET ?</div><div class='add'>+						RDMA_NETWORK_IPV4 :</div><div class='add'>+						RDMA_NETWORK_IPV6;</div><div class='add'>+	else</div><div class='add'>+		dev_addr-&gt;network = RDMA_NETWORK_IB;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int set_addr_netns_by_gid_rcu(struct rdma_dev_addr *addr)</div><div class='hunk'>@@ -555,7 +547,6 @@ static int addr_resolve(struct sockaddr *src_in,</div><div class='ctx'> 			u32 seq)</div><div class='ctx'> {</div><div class='ctx'> 	struct dst_entry *dst = NULL;</div><div class='del'>-	unsigned int ndev_flags = 0;</div><div class='ctx'> 	struct rtable *rt = NULL;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='hunk'>@@ -592,7 +583,7 @@ static int addr_resolve(struct sockaddr *src_in,</div><div class='ctx'> 		rcu_read_unlock();</div><div class='ctx'> 		goto done;</div><div class='ctx'> 	}</div><div class='del'>-	ret = rdma_set_src_addr_rcu(addr, &amp;ndev_flags, dst_in, dst);</div><div class='add'>+	ret = rdma_set_src_addr_rcu(addr, dst_in, dst);</div><div class='ctx'> 	rcu_read_unlock();</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='hunk'>@@ -600,7 +591,7 @@ static int addr_resolve(struct sockaddr *src_in,</div><div class='ctx'> 	 * only if src addr translation didn't fail.</div><div class='ctx'> 	 */</div><div class='ctx'> 	if (!ret &amp;&amp; resolve_neigh)</div><div class='del'>-		ret = addr_resolve_neigh(dst, dst_in, addr, ndev_flags, seq);</div><div class='add'>+		ret = addr_resolve_neigh(dst, dst_in, addr, seq);</div><div class='ctx'> </div><div class='ctx'> 	if (src_in-&gt;sa_family == AF_INET)</div><div class='ctx'> 		ip_rt_put(rt);</div><div class='hunk'>@@ -646,13 +637,12 @@ static void process_one_req(struct work_struct *_work)</div><div class='ctx'> 	req-&gt;callback = NULL;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_bh(&amp;lock);</div><div class='add'>+	/*</div><div class='add'>+	 * Although the work will normally have been canceled by the workqueue,</div><div class='add'>+	 * it can still be requeued as long as it is on the req_list.</div><div class='add'>+	 */</div><div class='add'>+	cancel_delayed_work(&amp;req-&gt;work);</div><div class='ctx'> 	if (!list_empty(&amp;req-&gt;list)) {</div><div class='del'>-		/*</div><div class='del'>-		 * Although the work will normally have been canceled by the</div><div class='del'>-		 * workqueue, it can still be requeued as long as it is on the</div><div class='del'>-		 * req_list.</div><div class='del'>-		 */</div><div class='del'>-		cancel_delayed_work(&amp;req-&gt;work);</div><div class='ctx'> 		list_del_init(&amp;req-&gt;list);</div><div class='ctx'> 		kfree(req);</div><div class='ctx'> 	}</div><div class='hunk'>@@ -728,6 +718,8 @@ int roce_resolve_route_from_path(struct sa_path_rec *rec,</div><div class='ctx'> 	struct rdma_dev_addr dev_addr = {};</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	might_sleep();</div><div class='add'>+</div><div class='ctx'> 	if (rec-&gt;roce.route_resolved)</div><div class='ctx'> 		return 0;</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/core/agent.c b/drivers/infiniband/core/agent.c<br/>index f82b4260de42..25a060a28301 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/agent.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/agent.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/agent.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/agent.c</a></div><div class='hunk'>@@ -59,7 +59,16 @@ __ib_get_agent_port(const struct ib_device *device, int port_num)</div><div class='ctx'> 	struct ib_agent_port_private *entry;</div><div class='ctx'> </div><div class='ctx'> 	list_for_each_entry(entry, &amp;ib_agent_port_list, port_list) {</div><div class='del'>-		if (entry-&gt;agent[1]-&gt;device == device &amp;&amp;</div><div class='add'>+		/* Need to check both agent[0] and agent[1], as an agent port</div><div class='add'>+		 * may only have one of them</div><div class='add'>+		 */</div><div class='add'>+		if (entry-&gt;agent[0] &amp;&amp;</div><div class='add'>+		    entry-&gt;agent[0]-&gt;device == device &amp;&amp;</div><div class='add'>+		    entry-&gt;agent[0]-&gt;port_num == port_num)</div><div class='add'>+			return entry;</div><div class='add'>+</div><div class='add'>+		if (entry-&gt;agent[1] &amp;&amp;</div><div class='add'>+		    entry-&gt;agent[1]-&gt;device == device &amp;&amp;</div><div class='ctx'> 		    entry-&gt;agent[1]-&gt;port_num == port_num)</div><div class='ctx'> 			return entry;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -101,8 +110,7 @@ void agent_send_response(const struct ib_mad_hdr *mad_hdr, const struct ib_grh *</div><div class='ctx'> 	agent = port_priv-&gt;agent[qpn];</div><div class='ctx'> 	ah = ib_create_ah_from_wc(agent-&gt;qp-&gt;pd, wc, grh, port_num);</div><div class='ctx'> 	if (IS_ERR(ah)) {</div><div class='del'>-		dev_err(&amp;device-&gt;dev, "ib_create_ah_from_wc error %ld\n",</div><div class='del'>-			PTR_ERR(ah));</div><div class='add'>+		dev_err(&amp;device-&gt;dev, "ib_create_ah_from_wc error %pe\n", ah);</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -172,14 +180,16 @@ int ib_agent_port_open(struct ib_device *device, int port_num)</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* Obtain send only MAD agent for GSI QP */</div><div class='del'>-	port_priv-&gt;agent[1] = ib_register_mad_agent(device, port_num,</div><div class='del'>-						    IB_QPT_GSI, NULL, 0,</div><div class='del'>-						    &amp;agent_send_handler,</div><div class='del'>-						    NULL, NULL, 0);</div><div class='del'>-	if (IS_ERR(port_priv-&gt;agent[1])) {</div><div class='del'>-		ret = PTR_ERR(port_priv-&gt;agent[1]);</div><div class='del'>-		goto error3;</div><div class='add'>+	if (rdma_cap_ib_cm(device, port_num)) {</div><div class='add'>+		/* Obtain send only MAD agent for GSI QP */</div><div class='add'>+		port_priv-&gt;agent[1] = ib_register_mad_agent(device, port_num,</div><div class='add'>+							    IB_QPT_GSI, NULL, 0,</div><div class='add'>+							    &amp;agent_send_handler,</div><div class='add'>+							    NULL, NULL, 0);</div><div class='add'>+		if (IS_ERR(port_priv-&gt;agent[1])) {</div><div class='add'>+			ret = PTR_ERR(port_priv-&gt;agent[1]);</div><div class='add'>+			goto error3;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;ib_agent_port_list_lock, flags);</div><div class='hunk'>@@ -212,7 +222,8 @@ int ib_agent_port_close(struct ib_device *device, int port_num)</div><div class='ctx'> 	list_del(&amp;port_priv-&gt;port_list);</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;ib_agent_port_list_lock, flags);</div><div class='ctx'> </div><div class='del'>-	ib_unregister_mad_agent(port_priv-&gt;agent[1]);</div><div class='add'>+	if (port_priv-&gt;agent[1])</div><div class='add'>+		ib_unregister_mad_agent(port_priv-&gt;agent[1]);</div><div class='ctx'> 	if (port_priv-&gt;agent[0])</div><div class='ctx'> 		ib_unregister_mad_agent(port_priv-&gt;agent[0]);</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c<br/>index 18e476b3ced0..81cf3c902e81 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cache.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cache.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cache.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/cache.c</a></div><div class='hunk'>@@ -33,7 +33,7 @@</div><div class='ctx'>  * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='add'>+#include &lt;linux/if_vlan.h&gt;</div><div class='ctx'> #include &lt;linux/errno.h&gt;</div><div class='ctx'> #include &lt;linux/slab.h&gt;</div><div class='ctx'> #include &lt;linux/workqueue.h&gt;</div><div class='hunk'>@@ -46,14 +46,13 @@</div><div class='ctx'> </div><div class='ctx'> struct ib_pkey_cache {</div><div class='ctx'> 	int             table_len;</div><div class='del'>-	u16             table[0];</div><div class='add'>+	u16             table[] __counted_by(table_len);</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct ib_update_work {</div><div class='ctx'> 	struct work_struct work;</div><div class='del'>-	struct ib_device  *device;</div><div class='del'>-	u8                 port_num;</div><div class='del'>-	bool		   enforce_security;</div><div class='add'>+	struct ib_event event;</div><div class='add'>+	bool enforce_security;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> union ib_gid zgid;</div><div class='hunk'>@@ -122,7 +121,7 @@ struct ib_gid_table {</div><div class='ctx'> 	u32				default_gid_indices;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static void dispatch_gid_change_event(struct ib_device *ib_dev, u8 port)</div><div class='add'>+static void dispatch_gid_change_event(struct ib_device *ib_dev, u32 port)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_event event;</div><div class='ctx'> </div><div class='hunk'>@@ -130,11 +129,15 @@ static void dispatch_gid_change_event(struct ib_device *ib_dev, u8 port)</div><div class='ctx'> 	event.element.port_num	= port;</div><div class='ctx'> 	event.event		= IB_EVENT_GID_CHANGE;</div><div class='ctx'> </div><div class='del'>-	ib_dispatch_event(&amp;event);</div><div class='add'>+	ib_dispatch_event_clients(&amp;event);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static const char * const gid_type_str[] = {</div><div class='add'>+	/* IB/RoCE v1 value is set for IB_GID_TYPE_IB and IB_GID_TYPE_ROCE for</div><div class='add'>+	 * user space compatibility reasons.</div><div class='add'>+	 */</div><div class='ctx'> 	[IB_GID_TYPE_IB]	= "IB/RoCE v1",</div><div class='add'>+	[IB_GID_TYPE_ROCE]	= "IB/RoCE v1",</div><div class='ctx'> 	[IB_GID_TYPE_ROCE_UDP_ENCAP]	= "RoCE v2",</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -194,7 +197,7 @@ int ib_cache_gid_parse_type_str(const char *buf)</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_cache_gid_parse_type_str);</div><div class='ctx'> </div><div class='del'>-static struct ib_gid_table *rdma_gid_table(struct ib_device *device, u8 port)</div><div class='add'>+static struct ib_gid_table *rdma_gid_table(struct ib_device *device, u32 port)</div><div class='ctx'> {</div><div class='ctx'> 	return device-&gt;port_data[port].cache.gid;</div><div class='ctx'> }</div><div class='hunk'>@@ -234,10 +237,10 @@ static void put_gid_ndev(struct rcu_head *head)</div><div class='ctx'> static void free_gid_entry_locked(struct ib_gid_table_entry *entry)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *device = entry-&gt;attr.device;</div><div class='del'>-	u8 port_num = entry-&gt;attr.port_num;</div><div class='add'>+	u32 port_num = entry-&gt;attr.port_num;</div><div class='ctx'> 	struct ib_gid_table *table = rdma_gid_table(device, port_num);</div><div class='ctx'> </div><div class='del'>-	dev_dbg(&amp;device-&gt;dev, "%s port=%d index=%d gid %pI6\n", __func__,</div><div class='add'>+	dev_dbg(&amp;device-&gt;dev, "%s port=%u index=%u gid %pI6\n", __func__,</div><div class='ctx'> 		port_num, entry-&gt;attr.index, entry-&gt;attr.gid.raw);</div><div class='ctx'> </div><div class='ctx'> 	write_lock_irq(&amp;table-&gt;rwlock);</div><div class='hunk'>@@ -279,7 +282,7 @@ static void free_gid_work(struct work_struct *work)</div><div class='ctx'> 	struct ib_gid_table_entry *entry =</div><div class='ctx'> 		container_of(work, struct ib_gid_table_entry, del_work);</div><div class='ctx'> 	struct ib_device *device = entry-&gt;attr.device;</div><div class='del'>-	u8 port_num = entry-&gt;attr.port_num;</div><div class='add'>+	u32 port_num = entry-&gt;attr.port_num;</div><div class='ctx'> 	struct ib_gid_table *table = rdma_gid_table(device, port_num);</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;table-&gt;lock);</div><div class='hunk'>@@ -320,7 +323,7 @@ static void store_gid_entry(struct ib_gid_table *table,</div><div class='ctx'> {</div><div class='ctx'> 	entry-&gt;state = GID_TABLE_ENTRY_VALID;</div><div class='ctx'> </div><div class='del'>-	dev_dbg(&amp;entry-&gt;attr.device-&gt;dev, "%s port=%d index=%d gid %pI6\n",</div><div class='add'>+	dev_dbg(&amp;entry-&gt;attr.device-&gt;dev, "%s port=%u index=%u gid %pI6\n",</div><div class='ctx'> 		__func__, entry-&gt;attr.port_num, entry-&gt;attr.index,</div><div class='ctx'> 		entry-&gt;attr.gid.raw);</div><div class='ctx'> </div><div class='hunk'>@@ -351,7 +354,7 @@ static int add_roce_gid(struct ib_gid_table_entry *entry)</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (!attr-&gt;ndev) {</div><div class='del'>-		dev_err(&amp;attr-&gt;device-&gt;dev, "%s NULL netdev port=%d index=%d\n",</div><div class='add'>+		dev_err(&amp;attr-&gt;device-&gt;dev, "%s NULL netdev port=%u index=%u\n",</div><div class='ctx'> 			__func__, attr-&gt;port_num, attr-&gt;index);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -359,7 +362,7 @@ static int add_roce_gid(struct ib_gid_table_entry *entry)</div><div class='ctx'> 		ret = attr-&gt;device-&gt;ops.add_gid(attr, &amp;entry-&gt;context);</div><div class='ctx'> 		if (ret) {</div><div class='ctx'> 			dev_err(&amp;attr-&gt;device-&gt;dev,</div><div class='del'>-				"%s GID add failed port=%d index=%d\n",</div><div class='add'>+				"%s GID add failed port=%u index=%u\n",</div><div class='ctx'> 				__func__, attr-&gt;port_num, attr-&gt;index);</div><div class='ctx'> 			return ret;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -376,7 +379,7 @@ static int add_roce_gid(struct ib_gid_table_entry *entry)</div><div class='ctx'>  * @ix:		GID entry index to delete</div><div class='ctx'>  *</div><div class='ctx'>  */</div><div class='del'>-static void del_gid(struct ib_device *ib_dev, u8 port,</div><div class='add'>+static void del_gid(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 		    struct ib_gid_table *table, int ix)</div><div class='ctx'> {</div><div class='ctx'> 	struct roce_gid_ndev_storage *ndev_storage;</div><div class='hunk'>@@ -384,7 +387,7 @@ static void del_gid(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> </div><div class='ctx'> 	lockdep_assert_held(&amp;table-&gt;lock);</div><div class='ctx'> </div><div class='del'>-	dev_dbg(&amp;ib_dev-&gt;dev, "%s port=%d index=%d gid %pI6\n", __func__, port,</div><div class='add'>+	dev_dbg(&amp;ib_dev-&gt;dev, "%s port=%u index=%d gid %pI6\n", __func__, port,</div><div class='ctx'> 		ix, table-&gt;data_vec[ix]-&gt;attr.gid.raw);</div><div class='ctx'> </div><div class='ctx'> 	write_lock_irq(&amp;table-&gt;rwlock);</div><div class='hunk'>@@ -397,6 +400,9 @@ static void del_gid(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> 		table-&gt;data_vec[ix] = NULL;</div><div class='ctx'> 	write_unlock_irq(&amp;table-&gt;rwlock);</div><div class='ctx'> </div><div class='add'>+	if (rdma_cap_roce_gid_table(ib_dev, port))</div><div class='add'>+		ib_dev-&gt;ops.del_gid(&amp;entry-&gt;attr, &amp;entry-&gt;context);</div><div class='add'>+</div><div class='ctx'> 	ndev_storage = entry-&gt;ndev_storage;</div><div class='ctx'> 	if (ndev_storage) {</div><div class='ctx'> 		entry-&gt;ndev_storage = NULL;</div><div class='hunk'>@@ -404,9 +410,6 @@ static void del_gid(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> 		call_rcu(&amp;ndev_storage-&gt;rcu_head, put_gid_ndev);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (rdma_cap_roce_gid_table(ib_dev, port))</div><div class='del'>-		ib_dev-&gt;ops.del_gid(&amp;entry-&gt;attr, &amp;entry-&gt;context);</div><div class='del'>-</div><div class='ctx'> 	put_gid_entry_locked(entry);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -540,7 +543,7 @@ static void make_default_gid(struct  net_device *dev, union ib_gid *gid)</div><div class='ctx'> 	addrconf_ifid_eui48(&amp;gid-&gt;raw[8], dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int __ib_cache_gid_add(struct ib_device *ib_dev, u8 port,</div><div class='add'>+static int __ib_cache_gid_add(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 			      union ib_gid *gid, struct ib_gid_attr *attr,</div><div class='ctx'> 			      unsigned long mask, bool default_gid)</div><div class='ctx'> {</div><div class='hunk'>@@ -579,12 +582,12 @@ static int __ib_cache_gid_add(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> out_unlock:</div><div class='ctx'> 	mutex_unlock(&amp;table-&gt;lock);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		pr_warn("%s: unable to add gid %pI6 error=%d\n",</div><div class='del'>-			__func__, gid-&gt;raw, ret);</div><div class='add'>+		pr_warn_ratelimited("%s: unable to add gid %pI6 error=%d\n",</div><div class='add'>+				    __func__, gid-&gt;raw, ret);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int ib_cache_gid_add(struct ib_device *ib_dev, u8 port,</div><div class='add'>+int ib_cache_gid_add(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 		     union ib_gid *gid, struct ib_gid_attr *attr)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long mask = GID_ATTR_FIND_MASK_GID |</div><div class='hunk'>@@ -595,7 +598,7 @@ int ib_cache_gid_add(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int</div><div class='del'>-_ib_cache_gid_del(struct ib_device *ib_dev, u8 port,</div><div class='add'>+_ib_cache_gid_del(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 		  union ib_gid *gid, struct ib_gid_attr *attr,</div><div class='ctx'> 		  unsigned long mask, bool default_gid)</div><div class='ctx'> {</div><div class='hunk'>@@ -624,7 +627,7 @@ out_unlock:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int ib_cache_gid_del(struct ib_device *ib_dev, u8 port,</div><div class='add'>+int ib_cache_gid_del(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 		     union ib_gid *gid, struct ib_gid_attr *attr)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long mask = GID_ATTR_FIND_MASK_GID	  |</div><div class='hunk'>@@ -635,7 +638,7 @@ int ib_cache_gid_del(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> 	return _ib_cache_gid_del(ib_dev, port, gid, attr, mask, false);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u8 port,</div><div class='add'>+int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 				     struct net_device *ndev)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_gid_table *table;</div><div class='hunk'>@@ -666,11 +669,10 @@ int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u8 port,</div><div class='ctx'>  * rdma_find_gid_by_port - Returns the GID entry attributes when it finds</div><div class='ctx'>  * a valid GID entry for given search parameters. It searches for the specified</div><div class='ctx'>  * GID value in the local software cache.</div><div class='del'>- * @device: The device to query.</div><div class='add'>+ * @ib_dev: The device to query.</div><div class='ctx'>  * @gid: The GID value to search for.</div><div class='ctx'>  * @gid_type: The GID type to search for.</div><div class='del'>- * @port_num: The port number of the device where the GID value should be</div><div class='del'>- *   searched.</div><div class='add'>+ * @port: The port number of the device where the GID value should be searched.</div><div class='ctx'>  * @ndev: In RoCE, the net device of the device. NULL means ignore.</div><div class='ctx'>  *</div><div class='ctx'>  * Returns sgid attributes if the GID is found with valid reference or</div><div class='hunk'>@@ -681,7 +683,7 @@ const struct ib_gid_attr *</div><div class='ctx'> rdma_find_gid_by_port(struct ib_device *ib_dev,</div><div class='ctx'> 		      const union ib_gid *gid,</div><div class='ctx'> 		      enum ib_gid_type gid_type,</div><div class='del'>-		      u8 port, struct net_device *ndev)</div><div class='add'>+		      u32 port, struct net_device *ndev)</div><div class='ctx'> {</div><div class='ctx'> 	int local_index;</div><div class='ctx'> 	struct ib_gid_table *table;</div><div class='hunk'>@@ -716,7 +718,7 @@ EXPORT_SYMBOL(rdma_find_gid_by_port);</div><div class='ctx'> /**</div><div class='ctx'>  * rdma_find_gid_by_filter - Returns the GID table attribute where a</div><div class='ctx'>  * specified GID value occurs</div><div class='del'>- * @device: The device to query.</div><div class='add'>+ * @ib_dev: The device to query.</div><div class='ctx'>  * @gid: The GID value to search for.</div><div class='ctx'>  * @port: The port number of the device where the GID value could be</div><div class='ctx'>  *   searched.</div><div class='hunk'>@@ -725,13 +727,14 @@ EXPORT_SYMBOL(rdma_find_gid_by_port);</div><div class='ctx'>  *   otherwise, we continue searching the GID table. It's guaranteed that</div><div class='ctx'>  *   while filter is executed, ndev field is valid and the structure won't</div><div class='ctx'>  *   change. filter is executed in an atomic context. filter must not be NULL.</div><div class='add'>+ * @context: Private data to pass into the call-back.</div><div class='ctx'>  *</div><div class='ctx'>  * rdma_find_gid_by_filter() searches for the specified GID value</div><div class='ctx'>  * of which the filter function returns true in the port's GID table.</div><div class='ctx'>  *</div><div class='ctx'>  */</div><div class='ctx'> const struct ib_gid_attr *rdma_find_gid_by_filter(</div><div class='del'>-	struct ib_device *ib_dev, const union ib_gid *gid, u8 port,</div><div class='add'>+	struct ib_device *ib_dev, const union ib_gid *gid, u32 port,</div><div class='ctx'> 	bool (*filter)(const union ib_gid *gid, const struct ib_gid_attr *,</div><div class='ctx'> 		       void *),</div><div class='ctx'> 	void *context)</div><div class='hunk'>@@ -791,7 +794,6 @@ err_free_table:</div><div class='ctx'> static void release_gid_table(struct ib_device *device,</div><div class='ctx'> 			      struct ib_gid_table *table)</div><div class='ctx'> {</div><div class='del'>-	bool leak = false;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='ctx'> 	if (!table)</div><div class='hunk'>@@ -800,43 +802,35 @@ static void release_gid_table(struct ib_device *device,</div><div class='ctx'> 	for (i = 0; i &lt; table-&gt;sz; i++) {</div><div class='ctx'> 		if (is_gid_entry_free(table-&gt;data_vec[i]))</div><div class='ctx'> 			continue;</div><div class='del'>-		if (kref_read(&amp;table-&gt;data_vec[i]-&gt;kref) &gt; 1) {</div><div class='del'>-			dev_err(&amp;device-&gt;dev,</div><div class='del'>-				"GID entry ref leak for index %d ref=%d\n", i,</div><div class='del'>-				kref_read(&amp;table-&gt;data_vec[i]-&gt;kref));</div><div class='del'>-			leak = true;</div><div class='del'>-		}</div><div class='add'>+</div><div class='add'>+		WARN_ONCE(true,</div><div class='add'>+			  "GID entry ref leak for dev %s index %d ref=%u\n",</div><div class='add'>+			  dev_name(&amp;device-&gt;dev), i,</div><div class='add'>+			  kref_read(&amp;table-&gt;data_vec[i]-&gt;kref));</div><div class='ctx'> 	}</div><div class='del'>-	if (leak)</div><div class='del'>-		return;</div><div class='ctx'> </div><div class='add'>+	mutex_destroy(&amp;table-&gt;lock);</div><div class='ctx'> 	kfree(table-&gt;data_vec);</div><div class='ctx'> 	kfree(table);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cleanup_gid_table_port(struct ib_device *ib_dev, u8 port,</div><div class='add'>+static void cleanup_gid_table_port(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 				   struct ib_gid_table *table)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='del'>-	bool deleted = false;</div><div class='ctx'> </div><div class='ctx'> 	if (!table)</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;table-&gt;lock);</div><div class='ctx'> 	for (i = 0; i &lt; table-&gt;sz; ++i) {</div><div class='del'>-		if (is_gid_entry_valid(table-&gt;data_vec[i])) {</div><div class='add'>+		if (is_gid_entry_valid(table-&gt;data_vec[i]))</div><div class='ctx'> 			del_gid(ib_dev, port, table, i);</div><div class='del'>-			deleted = true;</div><div class='del'>-		}</div><div class='ctx'> 	}</div><div class='ctx'> 	mutex_unlock(&amp;table-&gt;lock);</div><div class='del'>-</div><div class='del'>-	if (deleted)</div><div class='del'>-		dispatch_gid_change_event(ib_dev, port);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u8 port,</div><div class='add'>+void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 				  struct net_device *ndev,</div><div class='ctx'> 				  unsigned long gid_type_mask,</div><div class='ctx'> 				  enum ib_cache_gid_default_mode mode)</div><div class='hunk'>@@ -869,7 +863,7 @@ void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void gid_table_reserve_default(struct ib_device *ib_dev, u8 port,</div><div class='add'>+static void gid_table_reserve_default(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 				      struct ib_gid_table *table)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned int i;</div><div class='hunk'>@@ -886,7 +880,7 @@ static void gid_table_reserve_default(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> </div><div class='ctx'> static void gid_table_release_one(struct ib_device *ib_dev)</div><div class='ctx'> {</div><div class='del'>-	unsigned int p;</div><div class='add'>+	u32 p;</div><div class='ctx'> </div><div class='ctx'> 	rdma_for_each_port (ib_dev, p) {</div><div class='ctx'> 		release_gid_table(ib_dev, ib_dev-&gt;port_data[p].cache.gid);</div><div class='hunk'>@@ -897,7 +891,7 @@ static void gid_table_release_one(struct ib_device *ib_dev)</div><div class='ctx'> static int _gid_table_setup_one(struct ib_device *ib_dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_gid_table *table;</div><div class='del'>-	unsigned int rdma_port;</div><div class='add'>+	u32 rdma_port;</div><div class='ctx'> </div><div class='ctx'> 	rdma_for_each_port (ib_dev, rdma_port) {</div><div class='ctx'> 		table = alloc_gid_table(</div><div class='hunk'>@@ -917,7 +911,7 @@ rollback_table_setup:</div><div class='ctx'> </div><div class='ctx'> static void gid_table_cleanup_one(struct ib_device *ib_dev)</div><div class='ctx'> {</div><div class='del'>-	unsigned int p;</div><div class='add'>+	u32 p;</div><div class='ctx'> </div><div class='ctx'> 	rdma_for_each_port (ib_dev, p)</div><div class='ctx'> 		cleanup_gid_table_port(ib_dev, p,</div><div class='hunk'>@@ -952,12 +946,12 @@ static int gid_table_setup_one(struct ib_device *ib_dev)</div><div class='ctx'>  * Returns 0 on success or appropriate error code.</div><div class='ctx'>  *</div><div class='ctx'>  */</div><div class='del'>-int rdma_query_gid(struct ib_device *device, u8 port_num,</div><div class='add'>+int rdma_query_gid(struct ib_device *device, u32 port_num,</div><div class='ctx'> 		   int index, union ib_gid *gid)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_gid_table *table;</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	int res = -EINVAL;</div><div class='add'>+	int res;</div><div class='ctx'> </div><div class='ctx'> 	if (!rdma_is_port_valid(device, port_num))</div><div class='ctx'> 		return -EINVAL;</div><div class='hunk'>@@ -965,9 +959,15 @@ int rdma_query_gid(struct ib_device *device, u8 port_num,</div><div class='ctx'> 	table = rdma_gid_table(device, port_num);</div><div class='ctx'> 	read_lock_irqsave(&amp;table-&gt;rwlock, flags);</div><div class='ctx'> </div><div class='del'>-	if (index &lt; 0 || index &gt;= table-&gt;sz ||</div><div class='del'>-	    !is_gid_entry_valid(table-&gt;data_vec[index]))</div><div class='add'>+	if (index &lt; 0 || index &gt;= table-&gt;sz) {</div><div class='add'>+		res = -EINVAL;</div><div class='ctx'> 		goto done;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!is_gid_entry_valid(table-&gt;data_vec[index])) {</div><div class='add'>+		res = -ENOENT;</div><div class='add'>+		goto done;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	memcpy(gid, &amp;table-&gt;data_vec[index]-&gt;attr.gid, sizeof(*gid));</div><div class='ctx'> 	res = 0;</div><div class='hunk'>@@ -979,6 +979,23 @@ done:</div><div class='ctx'> EXPORT_SYMBOL(rdma_query_gid);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='add'>+ * rdma_read_gid_hw_context - Read the HW GID context from GID attribute</div><div class='add'>+ * @attr:		Potinter to the GID attribute</div><div class='add'>+ *</div><div class='add'>+ * rdma_read_gid_hw_context() reads the drivers GID HW context corresponding</div><div class='add'>+ * to the SGID attr. Callers are required to already be holding the reference</div><div class='add'>+ * to an existing GID entry.</div><div class='add'>+ *</div><div class='add'>+ * Returns the HW GID context</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+void *rdma_read_gid_hw_context(const struct ib_gid_attr *attr)</div><div class='add'>+{</div><div class='add'>+	return container_of(attr, struct ib_gid_table_entry, attr)-&gt;context;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_read_gid_hw_context);</div><div class='add'>+</div><div class='add'>+/**</div><div class='ctx'>  * rdma_find_gid - Returns SGID attributes if the matching GID is found.</div><div class='ctx'>  * @device: The device to query.</div><div class='ctx'>  * @gid: The GID value to search for.</div><div class='hunk'>@@ -999,7 +1016,7 @@ const struct ib_gid_attr *rdma_find_gid(struct ib_device *device,</div><div class='ctx'> 	unsigned long mask = GID_ATTR_FIND_MASK_GID |</div><div class='ctx'> 			     GID_ATTR_FIND_MASK_GID_TYPE;</div><div class='ctx'> 	struct ib_gid_attr gid_attr_val = {.ndev = ndev, .gid_type = gid_type};</div><div class='del'>-	unsigned int p;</div><div class='add'>+	u32 p;</div><div class='ctx'> </div><div class='ctx'> 	if (ndev)</div><div class='ctx'> 		mask |= GID_ATTR_FIND_MASK_NETDEV;</div><div class='hunk'>@@ -1028,7 +1045,7 @@ const struct ib_gid_attr *rdma_find_gid(struct ib_device *device,</div><div class='ctx'> EXPORT_SYMBOL(rdma_find_gid);</div><div class='ctx'> </div><div class='ctx'> int ib_get_cached_pkey(struct ib_device *device,</div><div class='del'>-		       u8                port_num,</div><div class='add'>+		       u32               port_num,</div><div class='ctx'> 		       int               index,</div><div class='ctx'> 		       u16              *pkey)</div><div class='ctx'> {</div><div class='hunk'>@@ -1039,42 +1056,34 @@ int ib_get_cached_pkey(struct ib_device *device,</div><div class='ctx'> 	if (!rdma_is_port_valid(device, port_num))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	read_lock_irqsave(&amp;device-&gt;cache.lock, flags);</div><div class='add'>+	read_lock_irqsave(&amp;device-&gt;cache_lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	cache = device-&gt;port_data[port_num].cache.pkey;</div><div class='ctx'> </div><div class='del'>-	if (index &lt; 0 || index &gt;= cache-&gt;table_len)</div><div class='add'>+	if (!cache || index &lt; 0 || index &gt;= cache-&gt;table_len)</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 	else</div><div class='ctx'> 		*pkey = cache-&gt;table[index];</div><div class='ctx'> </div><div class='del'>-	read_unlock_irqrestore(&amp;device-&gt;cache.lock, flags);</div><div class='add'>+	read_unlock_irqrestore(&amp;device-&gt;cache_lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_get_cached_pkey);</div><div class='ctx'> </div><div class='del'>-int ib_get_cached_subnet_prefix(struct ib_device *device,</div><div class='del'>-				u8                port_num,</div><div class='del'>-				u64              *sn_pfx)</div><div class='add'>+void ib_get_cached_subnet_prefix(struct ib_device *device, u32 port_num,</div><div class='add'>+				u64 *sn_pfx)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> </div><div class='del'>-	if (!rdma_is_port_valid(device, port_num))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	read_lock_irqsave(&amp;device-&gt;cache.lock, flags);</div><div class='add'>+	read_lock_irqsave(&amp;device-&gt;cache_lock, flags);</div><div class='ctx'> 	*sn_pfx = device-&gt;port_data[port_num].cache.subnet_prefix;</div><div class='del'>-	read_unlock_irqrestore(&amp;device-&gt;cache.lock, flags);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='add'>+	read_unlock_irqrestore(&amp;device-&gt;cache_lock, flags);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_get_cached_subnet_prefix);</div><div class='ctx'> </div><div class='del'>-int ib_find_cached_pkey(struct ib_device *device,</div><div class='del'>-			u8                port_num,</div><div class='del'>-			u16               pkey,</div><div class='del'>-			u16              *index)</div><div class='add'>+int ib_find_cached_pkey(struct ib_device *device, u32 port_num,</div><div class='add'>+			u16 pkey, u16 *index)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_pkey_cache *cache;</div><div class='ctx'> 	unsigned long flags;</div><div class='hunk'>@@ -1085,9 +1094,13 @@ int ib_find_cached_pkey(struct ib_device *device,</div><div class='ctx'> 	if (!rdma_is_port_valid(device, port_num))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	read_lock_irqsave(&amp;device-&gt;cache.lock, flags);</div><div class='add'>+	read_lock_irqsave(&amp;device-&gt;cache_lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	cache = device-&gt;port_data[port_num].cache.pkey;</div><div class='add'>+	if (!cache) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	*index = -1;</div><div class='ctx'> </div><div class='hunk'>@@ -1097,8 +1110,9 @@ int ib_find_cached_pkey(struct ib_device *device,</div><div class='ctx'> 				*index = i;</div><div class='ctx'> 				ret = 0;</div><div class='ctx'> 				break;</div><div class='del'>-			} else</div><div class='add'>+			} else {</div><div class='ctx'> 				partial_ix = i;</div><div class='add'>+			}</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='ctx'> 	if (ret &amp;&amp; partial_ix &gt;= 0) {</div><div class='hunk'>@@ -1106,47 +1120,14 @@ int ib_find_cached_pkey(struct ib_device *device,</div><div class='ctx'> 		ret = 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	read_unlock_irqrestore(&amp;device-&gt;cache.lock, flags);</div><div class='add'>+err:</div><div class='add'>+	read_unlock_irqrestore(&amp;device-&gt;cache_lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_find_cached_pkey);</div><div class='ctx'> </div><div class='del'>-int ib_find_exact_cached_pkey(struct ib_device *device,</div><div class='del'>-			      u8                port_num,</div><div class='del'>-			      u16               pkey,</div><div class='del'>-			      u16              *index)</div><div class='del'>-{</div><div class='del'>-	struct ib_pkey_cache *cache;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int i;</div><div class='del'>-	int ret = -ENOENT;</div><div class='del'>-</div><div class='del'>-	if (!rdma_is_port_valid(device, port_num))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	read_lock_irqsave(&amp;device-&gt;cache.lock, flags);</div><div class='del'>-</div><div class='del'>-	cache = device-&gt;port_data[port_num].cache.pkey;</div><div class='del'>-</div><div class='del'>-	*index = -1;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; cache-&gt;table_len; ++i)</div><div class='del'>-		if (cache-&gt;table[i] == pkey) {</div><div class='del'>-			*index = i;</div><div class='del'>-			ret = 0;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-	read_unlock_irqrestore(&amp;device-&gt;cache.lock, flags);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_find_exact_cached_pkey);</div><div class='del'>-</div><div class='del'>-int ib_get_cached_lmc(struct ib_device *device,</div><div class='del'>-		      u8                port_num,</div><div class='del'>-		      u8                *lmc)</div><div class='add'>+int ib_get_cached_lmc(struct ib_device *device, u32 port_num, u8 *lmc)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int ret = 0;</div><div class='hunk'>@@ -1154,16 +1135,15 @@ int ib_get_cached_lmc(struct ib_device *device,</div><div class='ctx'> 	if (!rdma_is_port_valid(device, port_num))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	read_lock_irqsave(&amp;device-&gt;cache.lock, flags);</div><div class='add'>+	read_lock_irqsave(&amp;device-&gt;cache_lock, flags);</div><div class='ctx'> 	*lmc = device-&gt;port_data[port_num].cache.lmc;</div><div class='del'>-	read_unlock_irqrestore(&amp;device-&gt;cache.lock, flags);</div><div class='add'>+	read_unlock_irqrestore(&amp;device-&gt;cache_lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_get_cached_lmc);</div><div class='ctx'> </div><div class='del'>-int ib_get_cached_port_state(struct ib_device   *device,</div><div class='del'>-			     u8                  port_num,</div><div class='add'>+int ib_get_cached_port_state(struct ib_device *device, u32 port_num,</div><div class='ctx'> 			     enum ib_port_state *port_state)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long flags;</div><div class='hunk'>@@ -1172,9 +1152,9 @@ int ib_get_cached_port_state(struct ib_device   *device,</div><div class='ctx'> 	if (!rdma_is_port_valid(device, port_num))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	read_lock_irqsave(&amp;device-&gt;cache.lock, flags);</div><div class='add'>+	read_lock_irqsave(&amp;device-&gt;cache_lock, flags);</div><div class='ctx'> 	*port_state = device-&gt;port_data[port_num].cache.port_state;</div><div class='del'>-	read_unlock_irqrestore(&amp;device-&gt;cache.lock, flags);</div><div class='add'>+	read_unlock_irqrestore(&amp;device-&gt;cache_lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -1197,9 +1177,9 @@ EXPORT_SYMBOL(ib_get_cached_port_state);</div><div class='ctx'>  * code.</div><div class='ctx'>  */</div><div class='ctx'> const struct ib_gid_attr *</div><div class='del'>-rdma_get_gid_attr(struct ib_device *device, u8 port_num, int index)</div><div class='add'>+rdma_get_gid_attr(struct ib_device *device, u32 port_num, int index)</div><div class='ctx'> {</div><div class='del'>-	const struct ib_gid_attr *attr = ERR_PTR(-EINVAL);</div><div class='add'>+	const struct ib_gid_attr *attr = ERR_PTR(-ENODATA);</div><div class='ctx'> 	struct ib_gid_table *table;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> </div><div class='hunk'>@@ -1223,6 +1203,63 @@ done:</div><div class='ctx'> EXPORT_SYMBOL(rdma_get_gid_attr);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='add'>+ * rdma_query_gid_table - Reads GID table entries of all the ports of a device up to max_entries.</div><div class='add'>+ * @device: The device to query.</div><div class='add'>+ * @entries: Entries where GID entries are returned.</div><div class='add'>+ * @max_entries: Maximum number of entries that can be returned.</div><div class='add'>+ * Entries array must be allocated to hold max_entries number of entries.</div><div class='add'>+ *</div><div class='add'>+ * Returns number of entries on success or appropriate error code.</div><div class='add'>+ */</div><div class='add'>+ssize_t rdma_query_gid_table(struct ib_device *device,</div><div class='add'>+			     struct ib_uverbs_gid_entry *entries,</div><div class='add'>+			     size_t max_entries)</div><div class='add'>+{</div><div class='add'>+	const struct ib_gid_attr *gid_attr;</div><div class='add'>+	ssize_t num_entries = 0, ret;</div><div class='add'>+	struct ib_gid_table *table;</div><div class='add'>+	u32 port_num, i;</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	rdma_for_each_port(device, port_num) {</div><div class='add'>+		table = rdma_gid_table(device, port_num);</div><div class='add'>+		read_lock_irqsave(&amp;table-&gt;rwlock, flags);</div><div class='add'>+		for (i = 0; i &lt; table-&gt;sz; i++) {</div><div class='add'>+			if (!is_gid_entry_valid(table-&gt;data_vec[i]))</div><div class='add'>+				continue;</div><div class='add'>+			if (num_entries &gt;= max_entries) {</div><div class='add'>+				ret = -EINVAL;</div><div class='add'>+				goto err;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			gid_attr = &amp;table-&gt;data_vec[i]-&gt;attr;</div><div class='add'>+</div><div class='add'>+			memcpy(&amp;entries-&gt;gid, &amp;gid_attr-&gt;gid,</div><div class='add'>+			       sizeof(gid_attr-&gt;gid));</div><div class='add'>+			entries-&gt;gid_index = gid_attr-&gt;index;</div><div class='add'>+			entries-&gt;port_num = gid_attr-&gt;port_num;</div><div class='add'>+			entries-&gt;gid_type = gid_attr-&gt;gid_type;</div><div class='add'>+			ndev = rcu_dereference_protected(</div><div class='add'>+				gid_attr-&gt;ndev,</div><div class='add'>+				lockdep_is_held(&amp;table-&gt;rwlock));</div><div class='add'>+			if (ndev)</div><div class='add'>+				entries-&gt;netdev_ifindex = ndev-&gt;ifindex;</div><div class='add'>+</div><div class='add'>+			num_entries++;</div><div class='add'>+			entries++;</div><div class='add'>+		}</div><div class='add'>+		read_unlock_irqrestore(&amp;table-&gt;rwlock, flags);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return num_entries;</div><div class='add'>+err:</div><div class='add'>+	read_unlock_irqrestore(&amp;table-&gt;rwlock, flags);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_query_gid_table);</div><div class='add'>+</div><div class='add'>+/**</div><div class='ctx'>  * rdma_put_gid_attr - Release reference to the GID attribute</div><div class='ctx'>  * @attr:		Pointer to the GID attribute whose reference</div><div class='ctx'>  *			needs to be released.</div><div class='hunk'>@@ -1278,8 +1315,8 @@ struct net_device *rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr)</div><div class='ctx'> 	struct ib_gid_table_entry *entry =</div><div class='ctx'> 			container_of(attr, struct ib_gid_table_entry, attr);</div><div class='ctx'> 	struct ib_device *device = entry-&gt;attr.device;</div><div class='del'>-	struct net_device *ndev = ERR_PTR(-ENODEV);</div><div class='del'>-	u8 port_num = entry-&gt;attr.port_num;</div><div class='add'>+	struct net_device *ndev = ERR_PTR(-EINVAL);</div><div class='add'>+	u32 port_num = entry-&gt;attr.port_num;</div><div class='ctx'> 	struct ib_gid_table *table;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	bool valid;</div><div class='hunk'>@@ -1290,8 +1327,7 @@ struct net_device *rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr)</div><div class='ctx'> 	valid = is_gid_entry_valid(table-&gt;data_vec[attr-&gt;index]);</div><div class='ctx'> 	if (valid) {</div><div class='ctx'> 		ndev = rcu_dereference(attr-&gt;ndev);</div><div class='del'>-		if (!ndev ||</div><div class='del'>-		    (ndev &amp;&amp; ((READ_ONCE(ndev-&gt;flags) &amp; IFF_UP) == 0)))</div><div class='add'>+		if (!ndev)</div><div class='ctx'> 			ndev = ERR_PTR(-ENODEV);</div><div class='ctx'> 	}</div><div class='ctx'> 	read_unlock_irqrestore(&amp;table-&gt;rwlock, flags);</div><div class='hunk'>@@ -1299,9 +1335,10 @@ struct net_device *rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr)</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_read_gid_attr_ndev_rcu);</div><div class='ctx'> </div><div class='del'>-static int get_lower_dev_vlan(struct net_device *lower_dev, void *data)</div><div class='add'>+static int get_lower_dev_vlan(struct net_device *lower_dev,</div><div class='add'>+			      struct netdev_nested_priv *priv)</div><div class='ctx'> {</div><div class='del'>-	u16 *vlan_id = data;</div><div class='add'>+	u16 *vlan_id = (u16 *)priv-&gt;data;</div><div class='ctx'> </div><div class='ctx'> 	if (is_vlan_dev(lower_dev))</div><div class='ctx'> 		*vlan_id = vlan_dev_vlan_id(lower_dev);</div><div class='hunk'>@@ -1327,6 +1364,9 @@ static int get_lower_dev_vlan(struct net_device *lower_dev, void *data)</div><div class='ctx'> int rdma_read_gid_l2_fields(const struct ib_gid_attr *attr,</div><div class='ctx'> 			    u16 *vlan_id, u8 *smac)</div><div class='ctx'> {</div><div class='add'>+	struct netdev_nested_priv priv = {</div><div class='add'>+		.data = (void *)vlan_id,</div><div class='add'>+	};</div><div class='ctx'> 	struct net_device *ndev;</div><div class='ctx'> </div><div class='ctx'> 	rcu_read_lock();</div><div class='hunk'>@@ -1343,11 +1383,11 @@ int rdma_read_gid_l2_fields(const struct ib_gid_attr *attr,</div><div class='ctx'> 			*vlan_id = vlan_dev_vlan_id(ndev);</div><div class='ctx'> 		} else {</div><div class='ctx'> 			/* If the netdev is upper device and if it's lower</div><div class='del'>-			 * device is vlan device, consider vlan id of the</div><div class='add'>+			 * device is vlan device, consider vlan id of</div><div class='ctx'> 			 * the lower vlan device for this gid entry.</div><div class='ctx'> 			 */</div><div class='ctx'> 			netdev_walk_all_lower_dev_rcu(attr-&gt;ndev,</div><div class='del'>-					get_lower_dev_vlan, vlan_id);</div><div class='add'>+					get_lower_dev_vlan, &amp;priv);</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> 	rcu_read_unlock();</div><div class='hunk'>@@ -1356,7 +1396,7 @@ int rdma_read_gid_l2_fields(const struct ib_gid_attr *attr,</div><div class='ctx'> EXPORT_SYMBOL(rdma_read_gid_l2_fields);</div><div class='ctx'> </div><div class='ctx'> static int config_non_roce_gid_cache(struct ib_device *device,</div><div class='del'>-				     u8 port, int gid_tbl_len)</div><div class='add'>+				     u32 port, struct ib_port_attr *tprops)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_gid_attr gid_attr = {};</div><div class='ctx'> 	struct ib_gid_table *table;</div><div class='hunk'>@@ -1368,7 +1408,7 @@ static int config_non_roce_gid_cache(struct ib_device *device,</div><div class='ctx'> 	table = rdma_gid_table(device, port);</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;table-&gt;lock);</div><div class='del'>-	for (i = 0; i &lt; gid_tbl_len; ++i) {</div><div class='add'>+	for (i = 0; i &lt; tprops-&gt;gid_tbl_len; ++i) {</div><div class='ctx'> 		if (!device-&gt;ops.query_gid)</div><div class='ctx'> 			continue;</div><div class='ctx'> 		ret = device-&gt;ops.query_gid(device, port, i, &amp;gid_attr.gid);</div><div class='hunk'>@@ -1378,7 +1418,20 @@ static int config_non_roce_gid_cache(struct ib_device *device,</div><div class='ctx'> 				 i);</div><div class='ctx'> 			goto err;</div><div class='ctx'> 		}</div><div class='add'>+</div><div class='add'>+		if (rdma_protocol_iwarp(device, port)) {</div><div class='add'>+			struct net_device *ndev;</div><div class='add'>+</div><div class='add'>+			ndev = ib_device_get_netdev(device, port);</div><div class='add'>+			if (!ndev)</div><div class='add'>+				continue;</div><div class='add'>+			RCU_INIT_POINTER(gid_attr.ndev, ndev);</div><div class='add'>+			dev_put(ndev);</div><div class='add'>+		}</div><div class='add'>+</div><div class='ctx'> 		gid_attr.index = i;</div><div class='add'>+		tprops-&gt;subnet_prefix =</div><div class='add'>+			be64_to_cpu(gid_attr.gid.global.subnet_prefix);</div><div class='ctx'> 		add_modify_gid(table, &amp;gid_attr);</div><div class='ctx'> 	}</div><div class='ctx'> err:</div><div class='hunk'>@@ -1386,21 +1439,22 @@ err:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void ib_cache_update(struct ib_device *device,</div><div class='del'>-			    u8                port,</div><div class='del'>-			    bool	      enforce_security)</div><div class='add'>+static int</div><div class='add'>+ib_cache_update(struct ib_device *device, u32 port, bool update_gids,</div><div class='add'>+		bool update_pkeys, bool enforce_security)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr       *tprops = NULL;</div><div class='del'>-	struct ib_pkey_cache      *pkey_cache = NULL, *old_pkey_cache;</div><div class='add'>+	struct ib_pkey_cache      *pkey_cache = NULL;</div><div class='add'>+	struct ib_pkey_cache      *old_pkey_cache = NULL;</div><div class='ctx'> 	int                        i;</div><div class='ctx'> 	int                        ret;</div><div class='ctx'> </div><div class='ctx'> 	if (!rdma_is_port_valid(device, port))</div><div class='del'>-		return;</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	tprops = kmalloc(sizeof *tprops, GFP_KERNEL);</div><div class='ctx'> 	if (!tprops)</div><div class='del'>-		return;</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	ret = ib_query_port(device, port, tprops);</div><div class='ctx'> 	if (ret) {</div><div class='hunk'>@@ -1408,41 +1462,55 @@ static void ib_cache_update(struct ib_device *device,</div><div class='ctx'> 		goto err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!rdma_protocol_roce(device, port)) {</div><div class='add'>+	if (!rdma_protocol_roce(device, port) &amp;&amp; update_gids) {</div><div class='ctx'> 		ret = config_non_roce_gid_cache(device, port,</div><div class='del'>-						tprops-&gt;gid_tbl_len);</div><div class='add'>+						tprops);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	pkey_cache = kmalloc(struct_size(pkey_cache, table,</div><div class='del'>-					 tprops-&gt;pkey_tbl_len),</div><div class='del'>-			     GFP_KERNEL);</div><div class='del'>-	if (!pkey_cache)</div><div class='del'>-		goto err;</div><div class='del'>-</div><div class='del'>-	pkey_cache-&gt;table_len = tprops-&gt;pkey_tbl_len;</div><div class='add'>+	update_pkeys &amp;= !!tprops-&gt;pkey_tbl_len;</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; pkey_cache-&gt;table_len; ++i) {</div><div class='del'>-		ret = ib_query_pkey(device, port, i, pkey_cache-&gt;table + i);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_warn(&amp;device-&gt;dev,</div><div class='del'>-				 "ib_query_pkey failed (%d) for index %d\n",</div><div class='del'>-				 ret, i);</div><div class='add'>+	if (update_pkeys) {</div><div class='add'>+		pkey_cache = kmalloc(struct_size(pkey_cache, table,</div><div class='add'>+						 tprops-&gt;pkey_tbl_len),</div><div class='add'>+				     GFP_KERNEL);</div><div class='add'>+		if (!pkey_cache) {</div><div class='add'>+			ret = -ENOMEM;</div><div class='ctx'> 			goto err;</div><div class='ctx'> 		}</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	write_lock_irq(&amp;device-&gt;cache.lock);</div><div class='add'>+		pkey_cache-&gt;table_len = tprops-&gt;pkey_tbl_len;</div><div class='add'>+</div><div class='add'>+		for (i = 0; i &lt; pkey_cache-&gt;table_len; ++i) {</div><div class='add'>+			ret = ib_query_pkey(device, port, i,</div><div class='add'>+					    pkey_cache-&gt;table + i);</div><div class='add'>+			if (ret) {</div><div class='add'>+				dev_warn(&amp;device-&gt;dev,</div><div class='add'>+					 "ib_query_pkey failed (%d) for index %d\n",</div><div class='add'>+					 ret, i);</div><div class='add'>+				goto err;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	old_pkey_cache = device-&gt;port_data[port].cache.pkey;</div><div class='add'>+	write_lock_irq(&amp;device-&gt;cache_lock);</div><div class='ctx'> </div><div class='del'>-	device-&gt;port_data[port].cache.pkey = pkey_cache;</div><div class='add'>+	if (update_pkeys) {</div><div class='add'>+		old_pkey_cache = device-&gt;port_data[port].cache.pkey;</div><div class='add'>+		device-&gt;port_data[port].cache.pkey = pkey_cache;</div><div class='add'>+	}</div><div class='ctx'> 	device-&gt;port_data[port].cache.lmc = tprops-&gt;lmc;</div><div class='add'>+</div><div class='add'>+	if (device-&gt;port_data[port].cache.port_state != IB_PORT_NOP &amp;&amp;</div><div class='add'>+	    device-&gt;port_data[port].cache.port_state != tprops-&gt;state)</div><div class='add'>+		ibdev_info(device, "Port: %d Link %s\n", port,</div><div class='add'>+			   ib_port_state_to_str(tprops-&gt;state));</div><div class='add'>+</div><div class='ctx'> 	device-&gt;port_data[port].cache.port_state = tprops-&gt;state;</div><div class='ctx'> </div><div class='ctx'> 	device-&gt;port_data[port].cache.subnet_prefix = tprops-&gt;subnet_prefix;</div><div class='del'>-	write_unlock_irq(&amp;device-&gt;cache.lock);</div><div class='add'>+	write_unlock_irq(&amp;device-&gt;cache_lock);</div><div class='ctx'> </div><div class='ctx'> 	if (enforce_security)</div><div class='ctx'> 		ib_security_cache_change(device,</div><div class='hunk'>@@ -1451,74 +1519,110 @@ static void ib_cache_update(struct ib_device *device,</div><div class='ctx'> </div><div class='ctx'> 	kfree(old_pkey_cache);</div><div class='ctx'> 	kfree(tprops);</div><div class='del'>-	return;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='ctx'> err:</div><div class='ctx'> 	kfree(pkey_cache);</div><div class='ctx'> 	kfree(tprops);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ib_cache_event_task(struct work_struct *_work)</div><div class='add'>+{</div><div class='add'>+	struct ib_update_work *work =</div><div class='add'>+		container_of(_work, struct ib_update_work, work);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	/* Before distributing the cache update event, first sync</div><div class='add'>+	 * the cache.</div><div class='add'>+	 */</div><div class='add'>+	ret = ib_cache_update(work-&gt;event.device, work-&gt;event.element.port_num,</div><div class='add'>+			      work-&gt;event.event == IB_EVENT_GID_CHANGE,</div><div class='add'>+			      work-&gt;event.event == IB_EVENT_PKEY_CHANGE,</div><div class='add'>+			      work-&gt;enforce_security);</div><div class='add'>+</div><div class='add'>+	/* GID event is notified already for individual GID entries by</div><div class='add'>+	 * dispatch_gid_change_event(). Hence, notifiy for rest of the</div><div class='add'>+	 * events.</div><div class='add'>+	 */</div><div class='add'>+	if (!ret &amp;&amp; work-&gt;event.event != IB_EVENT_GID_CHANGE)</div><div class='add'>+		ib_dispatch_event_clients(&amp;work-&gt;event);</div><div class='add'>+</div><div class='add'>+	kfree(work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void ib_cache_task(struct work_struct *_work)</div><div class='add'>+static void ib_generic_event_task(struct work_struct *_work)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_update_work *work =</div><div class='ctx'> 		container_of(_work, struct ib_update_work, work);</div><div class='ctx'> </div><div class='del'>-	ib_cache_update(work-&gt;device,</div><div class='del'>-			work-&gt;port_num,</div><div class='del'>-			work-&gt;enforce_security);</div><div class='add'>+	ib_dispatch_event_clients(&amp;work-&gt;event);</div><div class='ctx'> 	kfree(work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void ib_cache_event(struct ib_event_handler *handler,</div><div class='del'>-			   struct ib_event *event)</div><div class='add'>+static bool is_cache_update_event(const struct ib_event *event)</div><div class='add'>+{</div><div class='add'>+	return (event-&gt;event == IB_EVENT_PORT_ERR    ||</div><div class='add'>+		event-&gt;event == IB_EVENT_PORT_ACTIVE ||</div><div class='add'>+		event-&gt;event == IB_EVENT_LID_CHANGE  ||</div><div class='add'>+		event-&gt;event == IB_EVENT_PKEY_CHANGE ||</div><div class='add'>+		event-&gt;event == IB_EVENT_CLIENT_REREGISTER ||</div><div class='add'>+		event-&gt;event == IB_EVENT_GID_CHANGE);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ib_dispatch_event - Dispatch an asynchronous event</div><div class='add'>+ * @event:Event to dispatch</div><div class='add'>+ *</div><div class='add'>+ * Low-level drivers must call ib_dispatch_event() to dispatch the</div><div class='add'>+ * event to all registered event handlers when an asynchronous event</div><div class='add'>+ * occurs.</div><div class='add'>+ */</div><div class='add'>+void ib_dispatch_event(const struct ib_event *event)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_update_work *work;</div><div class='ctx'> </div><div class='del'>-	if (event-&gt;event == IB_EVENT_PORT_ERR    ||</div><div class='del'>-	    event-&gt;event == IB_EVENT_PORT_ACTIVE ||</div><div class='del'>-	    event-&gt;event == IB_EVENT_LID_CHANGE  ||</div><div class='del'>-	    event-&gt;event == IB_EVENT_PKEY_CHANGE ||</div><div class='del'>-	    event-&gt;event == IB_EVENT_CLIENT_REREGISTER ||</div><div class='del'>-	    event-&gt;event == IB_EVENT_GID_CHANGE) {</div><div class='del'>-		work = kmalloc(sizeof *work, GFP_ATOMIC);</div><div class='del'>-		if (work) {</div><div class='del'>-			INIT_WORK(&amp;work-&gt;work, ib_cache_task);</div><div class='del'>-			work-&gt;device   = event-&gt;device;</div><div class='del'>-			work-&gt;port_num = event-&gt;element.port_num;</div><div class='del'>-			if (event-&gt;event == IB_EVENT_PKEY_CHANGE ||</div><div class='del'>-			    event-&gt;event == IB_EVENT_GID_CHANGE)</div><div class='del'>-				work-&gt;enforce_security = true;</div><div class='del'>-			else</div><div class='del'>-				work-&gt;enforce_security = false;</div><div class='del'>-</div><div class='del'>-			queue_work(ib_wq, &amp;work-&gt;work);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='add'>+	work = kzalloc(sizeof(*work), GFP_ATOMIC);</div><div class='add'>+	if (!work)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	if (is_cache_update_event(event))</div><div class='add'>+		INIT_WORK(&amp;work-&gt;work, ib_cache_event_task);</div><div class='add'>+	else</div><div class='add'>+		INIT_WORK(&amp;work-&gt;work, ib_generic_event_task);</div><div class='add'>+</div><div class='add'>+	work-&gt;event = *event;</div><div class='add'>+	if (event-&gt;event == IB_EVENT_PKEY_CHANGE ||</div><div class='add'>+	    event-&gt;event == IB_EVENT_GID_CHANGE)</div><div class='add'>+		work-&gt;enforce_security = true;</div><div class='add'>+</div><div class='add'>+	queue_work(ib_wq, &amp;work-&gt;work);</div><div class='ctx'> }</div><div class='add'>+EXPORT_SYMBOL(ib_dispatch_event);</div><div class='ctx'> </div><div class='ctx'> int ib_cache_setup_one(struct ib_device *device)</div><div class='ctx'> {</div><div class='del'>-	unsigned int p;</div><div class='add'>+	u32 p;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	rwlock_init(&amp;device-&gt;cache.lock);</div><div class='del'>-</div><div class='ctx'> 	err = gid_table_setup_one(device);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='del'>-	rdma_for_each_port (device, p)</div><div class='del'>-		ib_cache_update(device, p, true);</div><div class='add'>+	rdma_for_each_port (device, p) {</div><div class='add'>+		err = ib_cache_update(device, p, true, true, true);</div><div class='add'>+		if (err) {</div><div class='add'>+			gid_table_cleanup_one(device);</div><div class='add'>+			return err;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	INIT_IB_EVENT_HANDLER(&amp;device-&gt;cache.event_handler,</div><div class='del'>-			      device, ib_cache_event);</div><div class='del'>-	ib_register_event_handler(&amp;device-&gt;cache.event_handler);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void ib_cache_release_one(struct ib_device *device)</div><div class='ctx'> {</div><div class='del'>-	unsigned int p;</div><div class='add'>+	u32 p;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * The release function frees all the cache elements.</div><div class='hunk'>@@ -1534,14 +1638,12 @@ void ib_cache_release_one(struct ib_device *device)</div><div class='ctx'> </div><div class='ctx'> void ib_cache_cleanup_one(struct ib_device *device)</div><div class='ctx'> {</div><div class='del'>-	/* The cleanup function unregisters the event handler,</div><div class='del'>-	 * waits for all in-progress workqueue elements and cleans</div><div class='del'>-	 * up the GID cache. This function should be called after</div><div class='del'>-	 * the device was removed from the devices list and all</div><div class='del'>-	 * clients were removed, so the cache exists but is</div><div class='add'>+	/* The cleanup function waits for all in-progress workqueue</div><div class='add'>+	 * elements and cleans up the GID cache. This function should be</div><div class='add'>+	 * called after the device was removed from the devices list and</div><div class='add'>+	 * all clients were removed, so the cache exists but is</div><div class='ctx'> 	 * non-functional and shouldn't be updated anymore.</div><div class='ctx'> 	 */</div><div class='del'>-	ib_unregister_event_handler(&amp;device-&gt;cache.event_handler);</div><div class='ctx'> 	flush_workqueue(ib_wq);</div><div class='ctx'> 	gid_table_cleanup_one(device);</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/core/cgroup.c b/drivers/infiniband/core/cgroup.c<br/>index 388fd04e5f63..1f037fe01450 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cgroup.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cgroup.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cgroup.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/cgroup.c</a></div><div class='hunk'>@@ -1,14 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0-only</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright (C) 2016 Parav Pandit &lt;pandit.parav@gmail.com&gt;</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify it</div><div class='del'>- * under the terms and conditions of the GNU General Public License,</div><div class='del'>- * version 2, as published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope it will be useful, but WITHOUT</div><div class='del'>- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or</div><div class='del'>- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for</div><div class='del'>- * more details.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include "core_priv.h"</div><div class='head'>diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c<br/>index da10e6ccb43c..024df6ee239d 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cm.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cm.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cm.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/cm.c</a></div><div class='hunk'>@@ -1,36 +1,10 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright (c) 2004-2007 Intel Corporation.  All rights reserved.</div><div class='ctx'>  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.</div><div class='ctx'>  * Copyright (c) 2004, 2005 Voltaire Corporation.  All rights reserved.</div><div class='ctx'>  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='add'>+ * Copyright (c) 2019, Mellanox Technologies inc.  All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/completion.h&gt;</div><div class='hunk'>@@ -51,13 +25,18 @@</div><div class='ctx'> </div><div class='ctx'> #include &lt;rdma/ib_cache.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_cm.h&gt;</div><div class='add'>+#include &lt;rdma/ib_sysfs.h&gt;</div><div class='ctx'> #include "cm_msgs.h"</div><div class='ctx'> #include "core_priv.h"</div><div class='add'>+#include "cm_trace.h"</div><div class='ctx'> </div><div class='ctx'> MODULE_AUTHOR("Sean Hefty");</div><div class='ctx'> MODULE_DESCRIPTION("InfiniBand CM");</div><div class='ctx'> MODULE_LICENSE("Dual BSD/GPL");</div><div class='ctx'> </div><div class='add'>+#define CM_DIRECT_RETRY_CTX ((void *) 1UL)</div><div class='add'>+#define CM_MRA_SETTING 24 /* 4.096us * 2^24 = ~68.7 seconds */</div><div class='add'>+</div><div class='ctx'> static const char * const ibcm_rej_reason_strs[] = {</div><div class='ctx'> 	[IB_CM_REJ_NO_QP]			= "no QP",</div><div class='ctx'> 	[IB_CM_REJ_NO_EEC]			= "no EEC",</div><div class='hunk'>@@ -92,6 +71,8 @@ static const char * const ibcm_rej_reason_strs[] = {</div><div class='ctx'> 	[IB_CM_REJ_INVALID_CLASS_VERSION]	= "invalid class version",</div><div class='ctx'> 	[IB_CM_REJ_INVALID_FLOW_LABEL]		= "invalid flow label",</div><div class='ctx'> 	[IB_CM_REJ_INVALID_ALT_FLOW_LABEL]	= "invalid alt flow label",</div><div class='add'>+	[IB_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED] =</div><div class='add'>+		"vendor option is not supported",</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> const char *__attribute_const__ ibcm_reject_msg(int reason)</div><div class='hunk'>@@ -106,8 +87,21 @@ const char *__attribute_const__ ibcm_reject_msg(int reason)</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ibcm_reject_msg);</div><div class='ctx'> </div><div class='del'>-static void cm_add_one(struct ib_device *device);</div><div class='add'>+struct cm_id_private;</div><div class='add'>+struct cm_work;</div><div class='add'>+static int cm_add_one(struct ib_device *device);</div><div class='ctx'> static void cm_remove_one(struct ib_device *device, void *client_data);</div><div class='add'>+static void cm_process_work(struct cm_id_private *cm_id_priv,</div><div class='add'>+			    struct cm_work *work);</div><div class='add'>+static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,</div><div class='add'>+				   struct ib_cm_sidr_rep_param *param);</div><div class='add'>+static void cm_issue_dreq(struct cm_id_private *cm_id_priv);</div><div class='add'>+static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,</div><div class='add'>+			       void *private_data, u8 private_data_len);</div><div class='add'>+static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,</div><div class='add'>+			      enum ib_cm_rej_reason reason, void *ari,</div><div class='add'>+			      u8 ari_length, const void *private_data,</div><div class='add'>+			      u8 private_data_len);</div><div class='ctx'> </div><div class='ctx'> static struct ib_client cm_client = {</div><div class='ctx'> 	.name   = "cm",</div><div class='hunk'>@@ -130,8 +124,6 @@ static struct ib_cm {</div><div class='ctx'> 	__be32 random_id_operand;</div><div class='ctx'> 	struct list_head timewait_list;</div><div class='ctx'> 	struct workqueue_struct *wq;</div><div class='del'>-	/* Sync on cm change port state */</div><div class='del'>-	spinlock_t state_lock;</div><div class='ctx'> } cm;</div><div class='ctx'> </div><div class='ctx'> /* Counter indexes ordered by attribute ID */</div><div class='hunk'>@@ -159,77 +151,34 @@ enum {</div><div class='ctx'> 	CM_COUNTER_GROUPS</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static char const counter_group_names[CM_COUNTER_GROUPS]</div><div class='del'>-				     [sizeof("cm_rx_duplicates")] = {</div><div class='del'>-	"cm_tx_msgs", "cm_tx_retries",</div><div class='del'>-	"cm_rx_msgs", "cm_rx_duplicates"</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct cm_counter_group {</div><div class='del'>-	struct kobject obj;</div><div class='del'>-	atomic_long_t counter[CM_ATTR_COUNT];</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> struct cm_counter_attribute {</div><div class='del'>-	struct attribute attr;</div><div class='del'>-	int index;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define CM_COUNTER_ATTR(_name, _index) \</div><div class='del'>-struct cm_counter_attribute cm_##_name##_counter_attr = { \</div><div class='del'>-	.attr = { .name = __stringify(_name), .mode = 0444 }, \</div><div class='del'>-	.index = _index \</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static CM_COUNTER_ATTR(req, CM_REQ_COUNTER);</div><div class='del'>-static CM_COUNTER_ATTR(mra, CM_MRA_COUNTER);</div><div class='del'>-static CM_COUNTER_ATTR(rej, CM_REJ_COUNTER);</div><div class='del'>-static CM_COUNTER_ATTR(rep, CM_REP_COUNTER);</div><div class='del'>-static CM_COUNTER_ATTR(rtu, CM_RTU_COUNTER);</div><div class='del'>-static CM_COUNTER_ATTR(dreq, CM_DREQ_COUNTER);</div><div class='del'>-static CM_COUNTER_ATTR(drep, CM_DREP_COUNTER);</div><div class='del'>-static CM_COUNTER_ATTR(sidr_req, CM_SIDR_REQ_COUNTER);</div><div class='del'>-static CM_COUNTER_ATTR(sidr_rep, CM_SIDR_REP_COUNTER);</div><div class='del'>-static CM_COUNTER_ATTR(lap, CM_LAP_COUNTER);</div><div class='del'>-static CM_COUNTER_ATTR(apr, CM_APR_COUNTER);</div><div class='del'>-</div><div class='del'>-static struct attribute *cm_counter_default_attrs[] = {</div><div class='del'>-	&amp;cm_req_counter_attr.attr,</div><div class='del'>-	&amp;cm_mra_counter_attr.attr,</div><div class='del'>-	&amp;cm_rej_counter_attr.attr,</div><div class='del'>-	&amp;cm_rep_counter_attr.attr,</div><div class='del'>-	&amp;cm_rtu_counter_attr.attr,</div><div class='del'>-	&amp;cm_dreq_counter_attr.attr,</div><div class='del'>-	&amp;cm_drep_counter_attr.attr,</div><div class='del'>-	&amp;cm_sidr_req_counter_attr.attr,</div><div class='del'>-	&amp;cm_sidr_rep_counter_attr.attr,</div><div class='del'>-	&amp;cm_lap_counter_attr.attr,</div><div class='del'>-	&amp;cm_apr_counter_attr.attr,</div><div class='del'>-	NULL</div><div class='add'>+	struct ib_port_attribute attr;</div><div class='add'>+	unsigned short group;</div><div class='add'>+	unsigned short index;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct cm_port {</div><div class='ctx'> 	struct cm_device *cm_dev;</div><div class='ctx'> 	struct ib_mad_agent *mad_agent;</div><div class='del'>-	struct kobject port_obj;</div><div class='del'>-	u8 port_num;</div><div class='del'>-	struct list_head cm_priv_prim_list;</div><div class='del'>-	struct list_head cm_priv_altr_list;</div><div class='del'>-	struct cm_counter_group counter_group[CM_COUNTER_GROUPS];</div><div class='add'>+	struct ib_mad_agent *rep_agent;</div><div class='add'>+	u32 port_num;</div><div class='add'>+	atomic_long_t counters[CM_COUNTER_GROUPS][CM_ATTR_COUNT];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct cm_device {</div><div class='add'>+	struct kref kref;</div><div class='ctx'> 	struct list_head list;</div><div class='add'>+	rwlock_t mad_agent_lock;</div><div class='ctx'> 	struct ib_device *ib_device;</div><div class='ctx'> 	u8 ack_delay;</div><div class='ctx'> 	int going_down;</div><div class='del'>-	struct cm_port *port[0];</div><div class='add'>+	struct cm_port *port[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct cm_av {</div><div class='ctx'> 	struct cm_port *port;</div><div class='del'>-	union ib_gid dgid;</div><div class='ctx'> 	struct rdma_ah_attr ah_attr;</div><div class='add'>+	u16 dlid_datapath;</div><div class='ctx'> 	u16 pkey_index;</div><div class='ctx'> 	u8 timeout;</div><div class='ctx'> };</div><div class='hunk'>@@ -242,11 +191,11 @@ struct cm_work {</div><div class='ctx'> 	__be32 local_id;			/* Established / timewait */</div><div class='ctx'> 	__be32 remote_id;</div><div class='ctx'> 	struct ib_cm_event cm_event;</div><div class='del'>-	struct sa_path_rec path[0];</div><div class='add'>+	struct sa_path_rec path[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct cm_timewait_info {</div><div class='del'>-	struct cm_work work;			/* Must be first. */</div><div class='add'>+	struct cm_work work;</div><div class='ctx'> 	struct list_head list;</div><div class='ctx'> 	struct rb_node remote_qp_node;</div><div class='ctx'> 	struct rb_node remote_id_node;</div><div class='hunk'>@@ -261,12 +210,15 @@ struct cm_id_private {</div><div class='ctx'> </div><div class='ctx'> 	struct rb_node service_node;</div><div class='ctx'> 	struct rb_node sidr_id_node;</div><div class='add'>+	u32 sidr_slid;</div><div class='ctx'> 	spinlock_t lock;	/* Do not acquire inside cm.lock */</div><div class='ctx'> 	struct completion comp;</div><div class='del'>-	atomic_t refcount;</div><div class='add'>+	refcount_t refcount;</div><div class='ctx'> 	/* Number of clients sharing this ib_cm_id. Only valid for listeners.</div><div class='del'>-	 * Protected by the cm.lock spinlock. */</div><div class='add'>+	 * Protected by the cm.lock spinlock.</div><div class='add'>+	 */</div><div class='ctx'> 	int listen_sharecount;</div><div class='add'>+	struct rcu_head rcu;</div><div class='ctx'> </div><div class='ctx'> 	struct ib_mad_send_buf *msg;</div><div class='ctx'> 	struct cm_timewait_info *timewait_info;</div><div class='hunk'>@@ -286,101 +238,155 @@ struct cm_id_private {</div><div class='ctx'> 	__be16 pkey;</div><div class='ctx'> 	u8 private_data_len;</div><div class='ctx'> 	u8 max_cm_retries;</div><div class='del'>-	u8 peer_to_peer;</div><div class='ctx'> 	u8 responder_resources;</div><div class='ctx'> 	u8 initiator_depth;</div><div class='ctx'> 	u8 retry_count;</div><div class='ctx'> 	u8 rnr_retry_count;</div><div class='del'>-	u8 service_timeout;</div><div class='ctx'> 	u8 target_ack_delay;</div><div class='ctx'> </div><div class='del'>-	struct list_head prim_list;</div><div class='del'>-	struct list_head altr_list;</div><div class='del'>-	/* Indicates that the send port mad is registered and av is set */</div><div class='del'>-	int prim_send_port_not_ready;</div><div class='del'>-	int altr_send_port_not_ready;</div><div class='del'>-</div><div class='ctx'> 	struct list_head work_list;</div><div class='ctx'> 	atomic_t work_count;</div><div class='add'>+</div><div class='add'>+	struct rdma_ucm_ece ece;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static void cm_dev_release(struct kref *kref)</div><div class='add'>+{</div><div class='add'>+	struct cm_device *cm_dev = container_of(kref, struct cm_device, kref);</div><div class='add'>+	u32 i;</div><div class='add'>+</div><div class='add'>+	rdma_for_each_port(cm_dev-&gt;ib_device, i)</div><div class='add'>+		kfree(cm_dev-&gt;port[i - 1]);</div><div class='add'>+</div><div class='add'>+	kfree(cm_dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void cm_device_put(struct cm_device *cm_dev)</div><div class='add'>+{</div><div class='add'>+	kref_put(&amp;cm_dev-&gt;kref, cm_dev_release);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static void cm_work_handler(struct work_struct *work);</div><div class='ctx'> </div><div class='ctx'> static inline void cm_deref_id(struct cm_id_private *cm_id_priv)</div><div class='ctx'> {</div><div class='del'>-	if (atomic_dec_and_test(&amp;cm_id_priv-&gt;refcount))</div><div class='add'>+	if (refcount_dec_and_test(&amp;cm_id_priv-&gt;refcount))</div><div class='ctx'> 		complete(&amp;cm_id_priv-&gt;comp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int cm_alloc_msg(struct cm_id_private *cm_id_priv,</div><div class='del'>-			struct ib_mad_send_buf **msg)</div><div class='add'>+static struct ib_mad_send_buf *</div><div class='add'>+cm_alloc_msg_agent(struct cm_id_private *cm_id_priv, bool rep_agent)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_mad_agent *mad_agent;</div><div class='ctx'> 	struct ib_mad_send_buf *m;</div><div class='ctx'> 	struct ib_ah *ah;</div><div class='del'>-	struct cm_av *av;</div><div class='del'>-	unsigned long flags, flags2;</div><div class='del'>-	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	/* don't let the port to be released till the agent is down */</div><div class='del'>-	spin_lock_irqsave(&amp;cm.state_lock, flags2);</div><div class='del'>-	spin_lock_irqsave(&amp;cm.lock, flags);</div><div class='del'>-	if (!cm_id_priv-&gt;prim_send_port_not_ready)</div><div class='del'>-		av = &amp;cm_id_priv-&gt;av;</div><div class='del'>-	else if (!cm_id_priv-&gt;altr_send_port_not_ready &amp;&amp;</div><div class='del'>-		 (cm_id_priv-&gt;alt_av.port))</div><div class='del'>-		av = &amp;cm_id_priv-&gt;alt_av;</div><div class='del'>-	else {</div><div class='del'>-		pr_info("%s: not valid CM id\n", __func__);</div><div class='del'>-		ret = -ENODEV;</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm.lock, flags);</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm.lock, flags);</div><div class='del'>-	/* Make sure the port haven't released the mad yet */</div><div class='del'>-	mad_agent = cm_id_priv-&gt;av.port-&gt;mad_agent;</div><div class='add'>+	lockdep_assert_held(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+</div><div class='add'>+	if (!cm_id_priv-&gt;av.port)</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	read_lock(&amp;cm_id_priv-&gt;av.port-&gt;cm_dev-&gt;mad_agent_lock);</div><div class='add'>+	mad_agent = rep_agent ? cm_id_priv-&gt;av.port-&gt;rep_agent :</div><div class='add'>+				cm_id_priv-&gt;av.port-&gt;mad_agent;</div><div class='ctx'> 	if (!mad_agent) {</div><div class='del'>-		pr_info("%s: not a valid MAD agent\n", __func__);</div><div class='del'>-		ret = -ENODEV;</div><div class='add'>+		m = ERR_PTR(-EINVAL);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='del'>-	ah = rdma_create_ah(mad_agent-&gt;qp-&gt;pd, &amp;av-&gt;ah_attr, 0);</div><div class='add'>+</div><div class='add'>+	ah = rdma_create_ah(mad_agent-&gt;qp-&gt;pd, &amp;cm_id_priv-&gt;av.ah_attr, 0);</div><div class='ctx'> 	if (IS_ERR(ah)) {</div><div class='del'>-		ret = PTR_ERR(ah);</div><div class='add'>+		m = ERR_CAST(ah);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	m = ib_create_send_mad(mad_agent, cm_id_priv-&gt;id.remote_cm_qpn,</div><div class='del'>-			       av-&gt;pkey_index,</div><div class='add'>+			       cm_id_priv-&gt;av.pkey_index,</div><div class='ctx'> 			       0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,</div><div class='ctx'> 			       GFP_ATOMIC,</div><div class='ctx'> 			       IB_MGMT_BASE_VERSION);</div><div class='ctx'> 	if (IS_ERR(m)) {</div><div class='ctx'> 		rdma_destroy_ah(ah, 0);</div><div class='del'>-		ret = PTR_ERR(m);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* Timeout set by caller if response is expected. */</div><div class='ctx'> 	m-&gt;ah = ah;</div><div class='del'>-	m-&gt;retries = cm_id_priv-&gt;max_cm_retries;</div><div class='del'>-</div><div class='del'>-	atomic_inc(&amp;cm_id_priv-&gt;refcount);</div><div class='del'>-	m-&gt;context[0] = cm_id_priv;</div><div class='del'>-	*msg = m;</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm.state_lock, flags2);</div><div class='del'>-	return ret;</div><div class='add'>+	read_unlock(&amp;cm_id_priv-&gt;av.port-&gt;cm_dev-&gt;mad_agent_lock);</div><div class='add'>+	return m;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct ib_mad_send_buf *cm_alloc_response_msg_no_ah(struct cm_port *port,</div><div class='del'>-							   struct ib_mad_recv_wc *mad_recv_wc)</div><div class='add'>+static struct ib_mad_send_buf *cm_alloc_msg(struct cm_id_private *cm_id_priv)</div><div class='ctx'> {</div><div class='del'>-	return ib_create_send_mad(port-&gt;mad_agent, 1, mad_recv_wc-&gt;wc-&gt;pkey_index,</div><div class='del'>-				  0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,</div><div class='del'>-				  GFP_ATOMIC,</div><div class='del'>-				  IB_MGMT_BASE_VERSION);</div><div class='add'>+	return cm_alloc_msg_agent(cm_id_priv, false);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void cm_free_msg(struct ib_mad_send_buf *msg)</div><div class='add'>+{</div><div class='add'>+	if (msg-&gt;ah)</div><div class='add'>+		rdma_destroy_ah(msg-&gt;ah, 0);</div><div class='add'>+	ib_free_send_mad(msg);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct ib_mad_send_buf *</div><div class='add'>+cm_alloc_priv_msg_rep(struct cm_id_private *cm_id_priv, enum ib_cm_state state,</div><div class='add'>+		      bool rep_agent)</div><div class='add'>+{</div><div class='add'>+	struct ib_mad_send_buf *msg;</div><div class='add'>+</div><div class='add'>+	lockdep_assert_held(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+</div><div class='add'>+	msg = cm_alloc_msg_agent(cm_id_priv, rep_agent);</div><div class='add'>+	if (IS_ERR(msg))</div><div class='add'>+		return msg;</div><div class='add'>+</div><div class='add'>+	cm_id_priv-&gt;msg = msg;</div><div class='add'>+	refcount_inc(&amp;cm_id_priv-&gt;refcount);</div><div class='add'>+	msg-&gt;context[0] = cm_id_priv;</div><div class='add'>+	msg-&gt;context[1] = (void *) (unsigned long) state;</div><div class='add'>+</div><div class='add'>+	msg-&gt;retries = cm_id_priv-&gt;max_cm_retries;</div><div class='add'>+	msg-&gt;timeout_ms = cm_id_priv-&gt;timeout_ms;</div><div class='add'>+</div><div class='add'>+	return msg;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct ib_mad_send_buf *</div><div class='add'>+cm_alloc_priv_msg(struct cm_id_private *cm_id_priv, enum ib_cm_state state)</div><div class='add'>+{</div><div class='add'>+	return cm_alloc_priv_msg_rep(cm_id_priv, state, false);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void cm_free_priv_msg(struct ib_mad_send_buf *msg)</div><div class='add'>+{</div><div class='add'>+	struct cm_id_private *cm_id_priv = msg-&gt;context[0];</div><div class='add'>+</div><div class='add'>+	lockdep_assert_held(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+</div><div class='add'>+	if (!WARN_ON(cm_id_priv-&gt;msg != msg))</div><div class='add'>+		cm_id_priv-&gt;msg = NULL;</div><div class='add'>+</div><div class='add'>+	if (msg-&gt;ah)</div><div class='add'>+		rdma_destroy_ah(msg-&gt;ah, 0);</div><div class='add'>+	cm_deref_id(cm_id_priv);</div><div class='add'>+	ib_free_send_mad(msg);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct ib_mad_send_buf *</div><div class='add'>+cm_alloc_response_msg_no_ah(struct cm_port *port,</div><div class='add'>+			    struct ib_mad_recv_wc *mad_recv_wc,</div><div class='add'>+			    bool direct_retry)</div><div class='add'>+{</div><div class='add'>+	struct ib_mad_send_buf *m;</div><div class='add'>+</div><div class='add'>+	m = ib_create_send_mad(port-&gt;mad_agent, 1, mad_recv_wc-&gt;wc-&gt;pkey_index,</div><div class='add'>+			       0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,</div><div class='add'>+			       GFP_ATOMIC, IB_MGMT_BASE_VERSION);</div><div class='add'>+	if (!IS_ERR(m))</div><div class='add'>+		m-&gt;context[0] = direct_retry ? CM_DIRECT_RETRY_CTX : NULL;</div><div class='add'>+</div><div class='add'>+	return m;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int cm_create_response_msg_ah(struct cm_port *port,</div><div class='hunk'>@@ -398,29 +404,21 @@ static int cm_create_response_msg_ah(struct cm_port *port,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cm_free_msg(struct ib_mad_send_buf *msg)</div><div class='del'>-{</div><div class='del'>-	if (msg-&gt;ah)</div><div class='del'>-		rdma_destroy_ah(msg-&gt;ah, 0);</div><div class='del'>-	if (msg-&gt;context[0])</div><div class='del'>-		cm_deref_id(msg-&gt;context[0]);</div><div class='del'>-	ib_free_send_mad(msg);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int cm_alloc_response_msg(struct cm_port *port,</div><div class='ctx'> 				 struct ib_mad_recv_wc *mad_recv_wc,</div><div class='add'>+				 bool direct_retry,</div><div class='ctx'> 				 struct ib_mad_send_buf **msg)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_mad_send_buf *m;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	m = cm_alloc_response_msg_no_ah(port, mad_recv_wc);</div><div class='add'>+	m = cm_alloc_response_msg_no_ah(port, mad_recv_wc, direct_retry);</div><div class='ctx'> 	if (IS_ERR(m))</div><div class='ctx'> 		return PTR_ERR(m);</div><div class='ctx'> </div><div class='ctx'> 	ret = cm_create_response_msg_ah(port, mad_recv_wc, m);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		cm_free_msg(m);</div><div class='add'>+		ib_free_send_mad(m);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -428,8 +426,7 @@ static int cm_alloc_response_msg(struct cm_port *port,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void * cm_copy_private_data(const void *private_data,</div><div class='del'>-				   u8 private_data_len)</div><div class='add'>+static void *cm_copy_private_data(const void *private_data, u8 private_data_len)</div><div class='ctx'> {</div><div class='ctx'> 	void *data;</div><div class='ctx'> </div><div class='hunk'>@@ -453,62 +450,38 @@ static void cm_set_private_data(struct cm_id_private *cm_id_priv,</div><div class='ctx'> 	cm_id_priv-&gt;private_data_len = private_data_len;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc,</div><div class='del'>-			      struct ib_grh *grh, struct cm_av *av)</div><div class='add'>+static void cm_set_av_port(struct cm_av *av, struct cm_port *port)</div><div class='ctx'> {</div><div class='del'>-	struct rdma_ah_attr new_ah_attr;</div><div class='del'>-	int ret;</div><div class='add'>+	struct cm_port *old_port = av-&gt;port;</div><div class='ctx'> </div><div class='del'>-	av-&gt;port = port;</div><div class='del'>-	av-&gt;pkey_index = wc-&gt;pkey_index;</div><div class='add'>+	if (old_port == port)</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * av-&gt;ah_attr might be initialized based on past wc during incoming</div><div class='del'>-	 * connect request or while sending out connect request. So initialize</div><div class='del'>-	 * a new ah_attr on stack. If initialization fails, old ah_attr is</div><div class='del'>-	 * used for sending any responses. If initialization is successful,</div><div class='del'>-	 * than new ah_attr is used by overwriting old one.</div><div class='del'>-	 */</div><div class='del'>-	ret = ib_init_ah_attr_from_wc(port-&gt;cm_dev-&gt;ib_device,</div><div class='del'>-				      port-&gt;port_num, wc,</div><div class='del'>-				      grh, &amp;new_ah_attr);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='add'>+	av-&gt;port = port;</div><div class='add'>+	if (old_port)</div><div class='add'>+		cm_device_put(old_port-&gt;cm_dev);</div><div class='add'>+	if (port)</div><div class='add'>+		kref_get(&amp;port-&gt;cm_dev-&gt;kref);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	rdma_move_ah_attr(&amp;av-&gt;ah_attr, &amp;new_ah_attr);</div><div class='del'>-	return 0;</div><div class='add'>+static void cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc,</div><div class='add'>+			       struct rdma_ah_attr *ah_attr, struct cm_av *av)</div><div class='add'>+{</div><div class='add'>+	cm_set_av_port(av, port);</div><div class='add'>+	av-&gt;pkey_index = wc-&gt;pkey_index;</div><div class='add'>+	rdma_move_ah_attr(&amp;av-&gt;ah_attr, ah_attr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,</div><div class='ctx'> 				   struct ib_grh *grh, struct cm_av *av)</div><div class='ctx'> {</div><div class='del'>-	av-&gt;port = port;</div><div class='add'>+	cm_set_av_port(av, port);</div><div class='ctx'> 	av-&gt;pkey_index = wc-&gt;pkey_index;</div><div class='ctx'> 	return ib_init_ah_attr_from_wc(port-&gt;cm_dev-&gt;ib_device,</div><div class='ctx'> 				       port-&gt;port_num, wc,</div><div class='ctx'> 				       grh, &amp;av-&gt;ah_attr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int add_cm_id_to_port_list(struct cm_id_private *cm_id_priv,</div><div class='del'>-				  struct cm_av *av,</div><div class='del'>-				  struct cm_port *port)</div><div class='del'>-{</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;cm.lock, flags);</div><div class='del'>-</div><div class='del'>-	if (&amp;cm_id_priv-&gt;av == av)</div><div class='del'>-		list_add_tail(&amp;cm_id_priv-&gt;prim_list, &amp;port-&gt;cm_priv_prim_list);</div><div class='del'>-	else if (&amp;cm_id_priv-&gt;alt_av == av)</div><div class='del'>-		list_add_tail(&amp;cm_id_priv-&gt;altr_list, &amp;port-&gt;cm_priv_altr_list);</div><div class='del'>-	else</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm.lock, flags);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static struct cm_port *</div><div class='ctx'> get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr)</div><div class='ctx'> {</div><div class='hunk'>@@ -552,8 +525,7 @@ get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr)</div><div class='ctx'> </div><div class='ctx'> static int cm_init_av_by_path(struct sa_path_rec *path,</div><div class='ctx'> 			      const struct ib_gid_attr *sgid_attr,</div><div class='del'>-			      struct cm_av *av,</div><div class='del'>-			      struct cm_id_private *cm_id_priv)</div><div class='add'>+			      struct cm_av *av)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_ah_attr new_ah_attr;</div><div class='ctx'> 	struct cm_device *cm_dev;</div><div class='hunk'>@@ -570,7 +542,7 @@ static int cm_init_av_by_path(struct sa_path_rec *path,</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	av-&gt;port = port;</div><div class='add'>+	cm_set_av_port(av, port);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * av-&gt;ah_attr might be initialized based on wc or during</div><div class='hunk'>@@ -587,60 +559,41 @@ static int cm_init_av_by_path(struct sa_path_rec *path,</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='ctx'> 	av-&gt;timeout = path-&gt;packet_life_time + 1;</div><div class='del'>-</div><div class='del'>-	ret = add_cm_id_to_port_list(cm_id_priv, av, port);</div><div class='del'>-	if (ret) {</div><div class='del'>-		rdma_destroy_ah_attr(&amp;new_ah_attr);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='ctx'> 	rdma_move_ah_attr(&amp;av-&gt;ah_attr, &amp;new_ah_attr);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int cm_alloc_id(struct cm_id_private *cm_id_priv)</div><div class='add'>+/* Move av created by cm_init_av_by_path(), so av.dgid is not moved */</div><div class='add'>+static void cm_move_av_from_path(struct cm_av *dest, struct cm_av *src)</div><div class='ctx'> {</div><div class='del'>-	int err;</div><div class='del'>-	u32 id;</div><div class='del'>-</div><div class='del'>-	err = xa_alloc_cyclic_irq(&amp;cm.local_id_table, &amp;id, cm_id_priv,</div><div class='del'>-			xa_limit_32b, &amp;cm.local_id_next, GFP_KERNEL);</div><div class='del'>-</div><div class='del'>-	cm_id_priv-&gt;id.local_id = (__force __be32)id ^ cm.random_id_operand;</div><div class='del'>-	return err;</div><div class='add'>+	cm_set_av_port(dest, src-&gt;port);</div><div class='add'>+	cm_set_av_port(src, NULL);</div><div class='add'>+	dest-&gt;pkey_index = src-&gt;pkey_index;</div><div class='add'>+	rdma_move_ah_attr(&amp;dest-&gt;ah_attr, &amp;src-&gt;ah_attr);</div><div class='add'>+	dest-&gt;timeout = src-&gt;timeout;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static u32 cm_local_id(__be32 local_id)</div><div class='add'>+static void cm_destroy_av(struct cm_av *av)</div><div class='ctx'> {</div><div class='del'>-	return (__force u32) (local_id ^ cm.random_id_operand);</div><div class='add'>+	rdma_destroy_ah_attr(&amp;av-&gt;ah_attr);</div><div class='add'>+	cm_set_av_port(av, NULL);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cm_free_id(__be32 local_id)</div><div class='add'>+static u32 cm_local_id(__be32 local_id)</div><div class='ctx'> {</div><div class='del'>-	xa_erase_irq(&amp;cm.local_id_table, cm_local_id(local_id));</div><div class='add'>+	return (__force u32) (local_id ^ cm.random_id_operand);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct cm_id_private * cm_get_id(__be32 local_id, __be32 remote_id)</div><div class='add'>+static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='ctx'> </div><div class='add'>+	rcu_read_lock();</div><div class='ctx'> 	cm_id_priv = xa_load(&amp;cm.local_id_table, cm_local_id(local_id));</div><div class='del'>-	if (cm_id_priv) {</div><div class='del'>-		if (cm_id_priv-&gt;id.remote_id == remote_id)</div><div class='del'>-			atomic_inc(&amp;cm_id_priv-&gt;refcount);</div><div class='del'>-		else</div><div class='del'>-			cm_id_priv = NULL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return cm_id_priv;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct cm_id_private * cm_acquire_id(__be32 local_id, __be32 remote_id)</div><div class='del'>-{</div><div class='del'>-	struct cm_id_private *cm_id_priv;</div><div class='del'>-</div><div class='del'>-	spin_lock_irq(&amp;cm.lock);</div><div class='del'>-	cm_id_priv = cm_get_id(local_id, remote_id);</div><div class='del'>-	spin_unlock_irq(&amp;cm.lock);</div><div class='add'>+	if (!cm_id_priv || cm_id_priv-&gt;id.remote_id != remote_id ||</div><div class='add'>+	    !refcount_inc_not_zero(&amp;cm_id_priv-&gt;refcount))</div><div class='add'>+		cm_id_priv = NULL;</div><div class='add'>+	rcu_read_unlock();</div><div class='ctx'> </div><div class='ctx'> 	return cm_id_priv;</div><div class='ctx'> }</div><div class='hunk'>@@ -670,22 +623,25 @@ static int be64_gt(__be64 a, __be64 b)</div><div class='ctx'> 	return (__force u64) a &gt; (__force u64) b;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct cm_id_private * cm_insert_listen(struct cm_id_private *cm_id_priv)</div><div class='add'>+/*</div><div class='add'>+ * Inserts a new cm_id_priv into the listen_service_table. Returns cm_id_priv</div><div class='add'>+ * if the new ID was inserted, NULL if it could not be inserted due to a</div><div class='add'>+ * collision, or the existing cm_id_priv ready for shared usage.</div><div class='add'>+ */</div><div class='add'>+static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv,</div><div class='add'>+					      ib_cm_handler shared_handler)</div><div class='ctx'> {</div><div class='ctx'> 	struct rb_node **link = &amp;cm.listen_service_table.rb_node;</div><div class='ctx'> 	struct rb_node *parent = NULL;</div><div class='ctx'> 	struct cm_id_private *cur_cm_id_priv;</div><div class='ctx'> 	__be64 service_id = cm_id_priv-&gt;id.service_id;</div><div class='del'>-	__be64 service_mask = cm_id_priv-&gt;id.service_mask;</div><div class='add'>+	unsigned long flags;</div><div class='ctx'> </div><div class='add'>+	spin_lock_irqsave(&amp;cm.lock, flags);</div><div class='ctx'> 	while (*link) {</div><div class='ctx'> 		parent = *link;</div><div class='ctx'> 		cur_cm_id_priv = rb_entry(parent, struct cm_id_private,</div><div class='ctx'> 					  service_node);</div><div class='del'>-		if ((cur_cm_id_priv-&gt;id.service_mask &amp; service_id) ==</div><div class='del'>-		    (service_mask &amp; cur_cm_id_priv-&gt;id.service_id) &amp;&amp;</div><div class='del'>-		    (cm_id_priv-&gt;id.device == cur_cm_id_priv-&gt;id.device))</div><div class='del'>-			return cur_cm_id_priv;</div><div class='ctx'> </div><div class='ctx'> 		if (cm_id_priv-&gt;id.device &lt; cur_cm_id_priv-&gt;id.device)</div><div class='ctx'> 			link = &amp;(*link)-&gt;rb_left;</div><div class='hunk'>@@ -695,26 +651,38 @@ static struct cm_id_private * cm_insert_listen(struct cm_id_private *cm_id_priv)</div><div class='ctx'> 			link = &amp;(*link)-&gt;rb_left;</div><div class='ctx'> 		else if (be64_gt(service_id, cur_cm_id_priv-&gt;id.service_id))</div><div class='ctx'> 			link = &amp;(*link)-&gt;rb_right;</div><div class='del'>-		else</div><div class='del'>-			link = &amp;(*link)-&gt;rb_right;</div><div class='add'>+		else {</div><div class='add'>+			/*</div><div class='add'>+			 * Sharing an ib_cm_id with different handlers is not</div><div class='add'>+			 * supported</div><div class='add'>+			 */</div><div class='add'>+			if (cur_cm_id_priv-&gt;id.cm_handler != shared_handler ||</div><div class='add'>+			    cur_cm_id_priv-&gt;id.context ||</div><div class='add'>+			    WARN_ON(!cur_cm_id_priv-&gt;id.cm_handler)) {</div><div class='add'>+				spin_unlock_irqrestore(&amp;cm.lock, flags);</div><div class='add'>+				return NULL;</div><div class='add'>+			}</div><div class='add'>+			refcount_inc(&amp;cur_cm_id_priv-&gt;refcount);</div><div class='add'>+			cur_cm_id_priv-&gt;listen_sharecount++;</div><div class='add'>+			spin_unlock_irqrestore(&amp;cm.lock, flags);</div><div class='add'>+			return cur_cm_id_priv;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='add'>+	cm_id_priv-&gt;listen_sharecount++;</div><div class='ctx'> 	rb_link_node(&amp;cm_id_priv-&gt;service_node, parent, link);</div><div class='ctx'> 	rb_insert_color(&amp;cm_id_priv-&gt;service_node, &amp;cm.listen_service_table);</div><div class='del'>-	return NULL;</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm.lock, flags);</div><div class='add'>+	return cm_id_priv;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct cm_id_private * cm_find_listen(struct ib_device *device,</div><div class='del'>-					     __be64 service_id)</div><div class='add'>+static struct cm_id_private *cm_find_listen(struct ib_device *device,</div><div class='add'>+					    __be64 service_id)</div><div class='ctx'> {</div><div class='ctx'> 	struct rb_node *node = cm.listen_service_table.rb_node;</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='ctx'> </div><div class='ctx'> 	while (node) {</div><div class='ctx'> 		cm_id_priv = rb_entry(node, struct cm_id_private, service_node);</div><div class='del'>-		if ((cm_id_priv-&gt;id.service_mask &amp; service_id) ==</div><div class='del'>-		     cm_id_priv-&gt;id.service_id &amp;&amp;</div><div class='del'>-		    (cm_id_priv-&gt;id.device == device))</div><div class='del'>-			return cm_id_priv;</div><div class='ctx'> </div><div class='ctx'> 		if (device &lt; cm_id_priv-&gt;id.device)</div><div class='ctx'> 			node = node-&gt;rb_left;</div><div class='hunk'>@@ -724,14 +692,16 @@ static struct cm_id_private * cm_find_listen(struct ib_device *device,</div><div class='ctx'> 			node = node-&gt;rb_left;</div><div class='ctx'> 		else if (be64_gt(service_id, cm_id_priv-&gt;id.service_id))</div><div class='ctx'> 			node = node-&gt;rb_right;</div><div class='del'>-		else</div><div class='del'>-			node = node-&gt;rb_right;</div><div class='add'>+		else {</div><div class='add'>+			refcount_inc(&amp;cm_id_priv-&gt;refcount);</div><div class='add'>+			return cm_id_priv;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> 	return NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct cm_timewait_info * cm_insert_remote_id(struct cm_timewait_info</div><div class='del'>-						     *timewait_info)</div><div class='add'>+static struct cm_timewait_info *</div><div class='add'>+cm_insert_remote_id(struct cm_timewait_info *timewait_info)</div><div class='ctx'> {</div><div class='ctx'> 	struct rb_node **link = &amp;cm.remote_id_table.rb_node;</div><div class='ctx'> 	struct rb_node *parent = NULL;</div><div class='hunk'>@@ -760,12 +730,14 @@ static struct cm_timewait_info * cm_insert_remote_id(struct cm_timewait_info</div><div class='ctx'> 	return NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct cm_timewait_info * cm_find_remote_id(__be64 remote_ca_guid,</div><div class='del'>-						   __be32 remote_id)</div><div class='add'>+static struct cm_id_private *cm_find_remote_id(__be64 remote_ca_guid,</div><div class='add'>+					       __be32 remote_id)</div><div class='ctx'> {</div><div class='ctx'> 	struct rb_node *node = cm.remote_id_table.rb_node;</div><div class='ctx'> 	struct cm_timewait_info *timewait_info;</div><div class='add'>+	struct cm_id_private *res = NULL;</div><div class='ctx'> </div><div class='add'>+	spin_lock_irq(&amp;cm.lock);</div><div class='ctx'> 	while (node) {</div><div class='ctx'> 		timewait_info = rb_entry(node, struct cm_timewait_info,</div><div class='ctx'> 					 remote_id_node);</div><div class='hunk'>@@ -777,14 +749,18 @@ static struct cm_timewait_info * cm_find_remote_id(__be64 remote_ca_guid,</div><div class='ctx'> 			node = node-&gt;rb_left;</div><div class='ctx'> 		else if (be64_gt(remote_ca_guid, timewait_info-&gt;remote_ca_guid))</div><div class='ctx'> 			node = node-&gt;rb_right;</div><div class='del'>-		else</div><div class='del'>-			return timewait_info;</div><div class='add'>+		else {</div><div class='add'>+			res = cm_acquire_id(timewait_info-&gt;work.local_id,</div><div class='add'>+					     timewait_info-&gt;work.remote_id);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='del'>-	return NULL;</div><div class='add'>+	spin_unlock_irq(&amp;cm.lock);</div><div class='add'>+	return res;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct cm_timewait_info * cm_insert_remote_qpn(struct cm_timewait_info</div><div class='del'>-						      *timewait_info)</div><div class='add'>+static struct cm_timewait_info *</div><div class='add'>+cm_insert_remote_qpn(struct cm_timewait_info *timewait_info)</div><div class='ctx'> {</div><div class='ctx'> 	struct rb_node **link = &amp;cm.remote_qp_table.rb_node;</div><div class='ctx'> 	struct rb_node *parent = NULL;</div><div class='hunk'>@@ -813,13 +789,12 @@ static struct cm_timewait_info * cm_insert_remote_qpn(struct cm_timewait_info</div><div class='ctx'> 	return NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct cm_id_private * cm_insert_remote_sidr(struct cm_id_private</div><div class='del'>-						    *cm_id_priv)</div><div class='add'>+static struct cm_id_private *</div><div class='add'>+cm_insert_remote_sidr(struct cm_id_private *cm_id_priv)</div><div class='ctx'> {</div><div class='ctx'> 	struct rb_node **link = &amp;cm.remote_sidr_table.rb_node;</div><div class='ctx'> 	struct rb_node *parent = NULL;</div><div class='ctx'> 	struct cm_id_private *cur_cm_id_priv;</div><div class='del'>-	union ib_gid *port_gid = &amp;cm_id_priv-&gt;av.dgid;</div><div class='ctx'> 	__be32 remote_id = cm_id_priv-&gt;id.remote_id;</div><div class='ctx'> </div><div class='ctx'> 	while (*link) {</div><div class='hunk'>@@ -831,12 +806,9 @@ static struct cm_id_private * cm_insert_remote_sidr(struct cm_id_private</div><div class='ctx'> 		else if (be32_gt(remote_id, cur_cm_id_priv-&gt;id.remote_id))</div><div class='ctx'> 			link = &amp;(*link)-&gt;rb_right;</div><div class='ctx'> 		else {</div><div class='del'>-			int cmp;</div><div class='del'>-			cmp = memcmp(port_gid, &amp;cur_cm_id_priv-&gt;av.dgid,</div><div class='del'>-				     sizeof *port_gid);</div><div class='del'>-			if (cmp &lt; 0)</div><div class='add'>+			if (cur_cm_id_priv-&gt;sidr_slid &lt; cm_id_priv-&gt;sidr_slid)</div><div class='ctx'> 				link = &amp;(*link)-&gt;rb_left;</div><div class='del'>-			else if (cmp &gt; 0)</div><div class='add'>+			else if (cur_cm_id_priv-&gt;sidr_slid &gt; cm_id_priv-&gt;sidr_slid)</div><div class='ctx'> 				link = &amp;(*link)-&gt;rb_right;</div><div class='ctx'> 			else</div><div class='ctx'> 				return cur_cm_id_priv;</div><div class='hunk'>@@ -847,21 +819,12 @@ static struct cm_id_private * cm_insert_remote_sidr(struct cm_id_private</div><div class='ctx'> 	return NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cm_reject_sidr_req(struct cm_id_private *cm_id_priv,</div><div class='del'>-			       enum ib_cm_sidr_status status)</div><div class='del'>-{</div><div class='del'>-	struct ib_cm_sidr_rep_param param;</div><div class='del'>-</div><div class='del'>-	memset(&amp;param, 0, sizeof param);</div><div class='del'>-	param.status = status;</div><div class='del'>-	ib_send_cm_sidr_rep(&amp;cm_id_priv-&gt;id, &amp;param);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct ib_cm_id *ib_create_cm_id(struct ib_device *device,</div><div class='del'>-				 ib_cm_handler cm_handler,</div><div class='del'>-				 void *context)</div><div class='add'>+static struct cm_id_private *cm_alloc_id_priv(struct ib_device *device,</div><div class='add'>+					      ib_cm_handler cm_handler,</div><div class='add'>+					      void *context)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='add'>+	u32 id;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);</div><div class='hunk'>@@ -873,26 +836,54 @@ struct ib_cm_id *ib_create_cm_id(struct ib_device *device,</div><div class='ctx'> 	cm_id_priv-&gt;id.cm_handler = cm_handler;</div><div class='ctx'> 	cm_id_priv-&gt;id.context = context;</div><div class='ctx'> 	cm_id_priv-&gt;id.remote_cm_qpn = 1;</div><div class='del'>-	ret = cm_alloc_id(cm_id_priv);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto error;</div><div class='ctx'> </div><div class='add'>+	RB_CLEAR_NODE(&amp;cm_id_priv-&gt;service_node);</div><div class='add'>+	RB_CLEAR_NODE(&amp;cm_id_priv-&gt;sidr_id_node);</div><div class='ctx'> 	spin_lock_init(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> 	init_completion(&amp;cm_id_priv-&gt;comp);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;cm_id_priv-&gt;work_list);</div><div class='del'>-	INIT_LIST_HEAD(&amp;cm_id_priv-&gt;prim_list);</div><div class='del'>-	INIT_LIST_HEAD(&amp;cm_id_priv-&gt;altr_list);</div><div class='ctx'> 	atomic_set(&amp;cm_id_priv-&gt;work_count, -1);</div><div class='del'>-	atomic_set(&amp;cm_id_priv-&gt;refcount, 1);</div><div class='del'>-	return &amp;cm_id_priv-&gt;id;</div><div class='add'>+	refcount_set(&amp;cm_id_priv-&gt;refcount, 1);</div><div class='add'>+</div><div class='add'>+	ret = xa_alloc_cyclic(&amp;cm.local_id_table, &amp;id, NULL, xa_limit_32b,</div><div class='add'>+			      &amp;cm.local_id_next, GFP_KERNEL);</div><div class='add'>+	if (ret &lt; 0)</div><div class='add'>+		goto error;</div><div class='add'>+	cm_id_priv-&gt;id.local_id = (__force __be32)id ^ cm.random_id_operand;</div><div class='add'>+</div><div class='add'>+	return cm_id_priv;</div><div class='ctx'> </div><div class='ctx'> error:</div><div class='ctx'> 	kfree(cm_id_priv);</div><div class='del'>-	return ERR_PTR(-ENOMEM);</div><div class='add'>+	return ERR_PTR(ret);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * Make the ID visible to the MAD handlers and other threads that use the</div><div class='add'>+ * xarray.</div><div class='add'>+ */</div><div class='add'>+static void cm_finalize_id(struct cm_id_private *cm_id_priv)</div><div class='add'>+{</div><div class='add'>+	xa_store(&amp;cm.local_id_table, cm_local_id(cm_id_priv-&gt;id.local_id),</div><div class='add'>+		 cm_id_priv, GFP_ATOMIC);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_cm_id *ib_create_cm_id(struct ib_device *device,</div><div class='add'>+				 ib_cm_handler cm_handler,</div><div class='add'>+				 void *context)</div><div class='add'>+{</div><div class='add'>+	struct cm_id_private *cm_id_priv;</div><div class='add'>+</div><div class='add'>+	cm_id_priv = cm_alloc_id_priv(device, cm_handler, context);</div><div class='add'>+	if (IS_ERR(cm_id_priv))</div><div class='add'>+		return ERR_CAST(cm_id_priv);</div><div class='add'>+</div><div class='add'>+	cm_finalize_id(cm_id_priv);</div><div class='add'>+	return &amp;cm_id_priv-&gt;id;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_create_cm_id);</div><div class='ctx'> </div><div class='del'>-static struct cm_work * cm_dequeue_work(struct cm_id_private *cm_id_priv)</div><div class='add'>+static struct cm_work *cm_dequeue_work(struct cm_id_private *cm_id_priv)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_work *work;</div><div class='ctx'> </div><div class='hunk'>@@ -911,6 +902,36 @@ static void cm_free_work(struct cm_work *work)</div><div class='ctx'> 	kfree(work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void cm_queue_work_unlock(struct cm_id_private *cm_id_priv,</div><div class='add'>+				 struct cm_work *work)</div><div class='add'>+	__releases(&amp;cm_id_priv-&gt;lock)</div><div class='add'>+{</div><div class='add'>+	bool immediate;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * To deliver the event to the user callback we have the drop the</div><div class='add'>+	 * spinlock, however, we need to ensure that the user callback is single</div><div class='add'>+	 * threaded and receives events in the temporal order. If there are</div><div class='add'>+	 * already events being processed then thread new events onto a list,</div><div class='add'>+	 * the thread currently processing will pick them up.</div><div class='add'>+	 */</div><div class='add'>+	immediate = atomic_inc_and_test(&amp;cm_id_priv-&gt;work_count);</div><div class='add'>+	if (!immediate) {</div><div class='add'>+		list_add_tail(&amp;work-&gt;list, &amp;cm_id_priv-&gt;work_list);</div><div class='add'>+		/*</div><div class='add'>+		 * This routine always consumes incoming reference. Once queued</div><div class='add'>+		 * to the work_list then a reference is held by the thread</div><div class='add'>+		 * currently running cm_process_work() and this reference is not</div><div class='add'>+		 * needed.</div><div class='add'>+		 */</div><div class='add'>+		cm_deref_id(cm_id_priv);</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+</div><div class='add'>+	if (immediate)</div><div class='add'>+		cm_process_work(cm_id_priv, work);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static inline int cm_convert_to_ms(int iba_time)</div><div class='ctx'> {</div><div class='ctx'> 	/* approximate conversion to ms from 4.096us x 2^iba_time */</div><div class='hunk'>@@ -936,8 +957,10 @@ static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)</div><div class='ctx'> 	return min(31, ack_timeout);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cm_cleanup_timewait(struct cm_timewait_info *timewait_info)</div><div class='add'>+static void cm_remove_remote(struct cm_id_private *cm_id_priv)</div><div class='ctx'> {</div><div class='add'>+	struct cm_timewait_info *timewait_info = cm_id_priv-&gt;timewait_info;</div><div class='add'>+</div><div class='ctx'> 	if (timewait_info-&gt;inserted_remote_id) {</div><div class='ctx'> 		rb_erase(&amp;timewait_info-&gt;remote_id_node, &amp;cm.remote_id_table);</div><div class='ctx'> 		timewait_info-&gt;inserted_remote_id = 0;</div><div class='hunk'>@@ -949,7 +972,7 @@ static void cm_cleanup_timewait(struct cm_timewait_info *timewait_info)</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct cm_timewait_info * cm_create_timewait_info(__be32 local_id)</div><div class='add'>+static struct cm_timewait_info *cm_create_timewait_info(__be32 local_id)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_timewait_info *timewait_info;</div><div class='ctx'> </div><div class='hunk'>@@ -969,12 +992,14 @@ static void cm_enter_timewait(struct cm_id_private *cm_id_priv)</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	struct cm_device *cm_dev;</div><div class='ctx'> </div><div class='add'>+	lockdep_assert_held(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+</div><div class='ctx'> 	cm_dev = ib_get_client_data(cm_id_priv-&gt;id.device, &amp;cm_client);</div><div class='ctx'> 	if (!cm_dev)</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;cm.lock, flags);</div><div class='del'>-	cm_cleanup_timewait(cm_id_priv-&gt;timewait_info);</div><div class='add'>+	cm_remove_remote(cm_id_priv);</div><div class='ctx'> 	list_add_tail(&amp;cm_id_priv-&gt;timewait_info-&gt;list, &amp;cm.timewait_list);</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;cm.lock, flags);</div><div class='ctx'> </div><div class='hunk'>@@ -993,6 +1018,11 @@ static void cm_enter_timewait(struct cm_id_private *cm_id_priv)</div><div class='ctx'> 				   msecs_to_jiffies(wait_time));</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;cm.lock, flags);</div><div class='ctx'> </div><div class='add'>+	/*</div><div class='add'>+	 * The timewait_info is converted into a work and gets freed during</div><div class='add'>+	 * cm_free_work() in cm_timewait_handler().</div><div class='add'>+	 */</div><div class='add'>+	BUILD_BUG_ON(offsetof(struct cm_timewait_info, work) != 0);</div><div class='ctx'> 	cm_id_priv-&gt;timewait_info = NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1000,122 +1030,157 @@ static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> </div><div class='add'>+	lockdep_assert_held(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+</div><div class='ctx'> 	cm_id_priv-&gt;id.state = IB_CM_IDLE;</div><div class='ctx'> 	if (cm_id_priv-&gt;timewait_info) {</div><div class='ctx'> 		spin_lock_irqsave(&amp;cm.lock, flags);</div><div class='del'>-		cm_cleanup_timewait(cm_id_priv-&gt;timewait_info);</div><div class='add'>+		cm_remove_remote(cm_id_priv);</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;cm.lock, flags);</div><div class='ctx'> 		kfree(cm_id_priv-&gt;timewait_info);</div><div class='ctx'> 		cm_id_priv-&gt;timewait_info = NULL;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static noinline void cm_destroy_id_wait_timeout(struct ib_cm_id *cm_id,</div><div class='add'>+						enum ib_cm_state old_state)</div><div class='add'>+{</div><div class='add'>+	struct cm_id_private *cm_id_priv;</div><div class='add'>+</div><div class='add'>+	cm_id_priv = container_of(cm_id, struct cm_id_private, id);</div><div class='add'>+	pr_err_ratelimited("%s: cm_id=%p timed out. state %d -&gt; %d, refcnt=%d\n", __func__,</div><div class='add'>+			   cm_id, old_state, cm_id-&gt;state, refcount_read(&amp;cm_id_priv-&gt;refcount));</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static void cm_destroy_id(struct ib_cm_id *cm_id, int err)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='add'>+	enum ib_cm_state old_state;</div><div class='add'>+	unsigned long timeout;</div><div class='ctx'> 	struct cm_work *work;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);</div><div class='del'>-retest:</div><div class='ctx'> 	spin_lock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+	old_state = cm_id-&gt;state;</div><div class='add'>+retest:</div><div class='ctx'> 	switch (cm_id-&gt;state) {</div><div class='ctx'> 	case IB_CM_LISTEN:</div><div class='del'>-		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-</div><div class='del'>-		spin_lock_irq(&amp;cm.lock);</div><div class='add'>+		spin_lock(&amp;cm.lock);</div><div class='ctx'> 		if (--cm_id_priv-&gt;listen_sharecount &gt; 0) {</div><div class='ctx'> 			/* The id is still shared. */</div><div class='add'>+			WARN_ON(refcount_read(&amp;cm_id_priv-&gt;refcount) == 1);</div><div class='add'>+			spin_unlock(&amp;cm.lock);</div><div class='add'>+			spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> 			cm_deref_id(cm_id_priv);</div><div class='del'>-			spin_unlock_irq(&amp;cm.lock);</div><div class='ctx'> 			return;</div><div class='ctx'> 		}</div><div class='add'>+		cm_id-&gt;state = IB_CM_IDLE;</div><div class='ctx'> 		rb_erase(&amp;cm_id_priv-&gt;service_node, &amp;cm.listen_service_table);</div><div class='del'>-		spin_unlock_irq(&amp;cm.lock);</div><div class='add'>+		RB_CLEAR_NODE(&amp;cm_id_priv-&gt;service_node);</div><div class='add'>+		spin_unlock(&amp;cm.lock);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_SIDR_REQ_SENT:</div><div class='ctx'> 		cm_id-&gt;state = IB_CM_IDLE;</div><div class='del'>-		ib_cancel_mad(cm_id_priv-&gt;av.port-&gt;mad_agent, cm_id_priv-&gt;msg);</div><div class='del'>-		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+		ib_cancel_mad(cm_id_priv-&gt;msg);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_SIDR_REQ_RCVD:</div><div class='del'>-		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-		cm_reject_sidr_req(cm_id_priv, IB_SIDR_REJECT);</div><div class='del'>-		spin_lock_irq(&amp;cm.lock);</div><div class='del'>-		if (!RB_EMPTY_NODE(&amp;cm_id_priv-&gt;sidr_id_node))</div><div class='del'>-			rb_erase(&amp;cm_id_priv-&gt;sidr_id_node,</div><div class='del'>-				 &amp;cm.remote_sidr_table);</div><div class='del'>-		spin_unlock_irq(&amp;cm.lock);</div><div class='add'>+		cm_send_sidr_rep_locked(cm_id_priv,</div><div class='add'>+					&amp;(struct ib_cm_sidr_rep_param){</div><div class='add'>+						.status = IB_SIDR_REJECT });</div><div class='add'>+		/* cm_send_sidr_rep_locked will not move to IDLE if it fails */</div><div class='add'>+		cm_id-&gt;state = IB_CM_IDLE;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_REQ_SENT:</div><div class='ctx'> 	case IB_CM_MRA_REQ_RCVD:</div><div class='del'>-		ib_cancel_mad(cm_id_priv-&gt;av.port-&gt;mad_agent, cm_id_priv-&gt;msg);</div><div class='del'>-		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-		ib_send_cm_rej(cm_id, IB_CM_REJ_TIMEOUT,</div><div class='del'>-			       &amp;cm_id_priv-&gt;id.device-&gt;node_guid,</div><div class='del'>-			       sizeof cm_id_priv-&gt;id.device-&gt;node_guid,</div><div class='del'>-			       NULL, 0);</div><div class='add'>+		ib_cancel_mad(cm_id_priv-&gt;msg);</div><div class='add'>+		cm_send_rej_locked(cm_id_priv, IB_CM_REJ_TIMEOUT,</div><div class='add'>+				   &amp;cm_id_priv-&gt;id.device-&gt;node_guid,</div><div class='add'>+				   sizeof(cm_id_priv-&gt;id.device-&gt;node_guid),</div><div class='add'>+				   NULL, 0);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_REQ_RCVD:</div><div class='ctx'> 		if (err == -ENOMEM) {</div><div class='ctx'> 			/* Do not reject to allow future retries. */</div><div class='ctx'> 			cm_reset_to_idle(cm_id_priv);</div><div class='del'>-			spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> 		} else {</div><div class='del'>-			spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-			ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,</div><div class='del'>-				       NULL, 0, NULL, 0);</div><div class='add'>+			cm_send_rej_locked(cm_id_priv,</div><div class='add'>+					   IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,</div><div class='add'>+					   NULL, 0);</div><div class='ctx'> 		}</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_REP_SENT:</div><div class='ctx'> 	case IB_CM_MRA_REP_RCVD:</div><div class='del'>-		ib_cancel_mad(cm_id_priv-&gt;av.port-&gt;mad_agent, cm_id_priv-&gt;msg);</div><div class='del'>-		/* Fall through */</div><div class='add'>+		ib_cancel_mad(cm_id_priv-&gt;msg);</div><div class='add'>+		cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,</div><div class='add'>+				   0, NULL, 0);</div><div class='add'>+		goto retest;</div><div class='ctx'> 	case IB_CM_MRA_REQ_SENT:</div><div class='ctx'> 	case IB_CM_REP_RCVD:</div><div class='ctx'> 	case IB_CM_MRA_REP_SENT:</div><div class='del'>-		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-		ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,</div><div class='del'>-			       NULL, 0, NULL, 0);</div><div class='add'>+		cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,</div><div class='add'>+				   0, NULL, 0);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_ESTABLISHED:</div><div class='del'>-		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-		if (cm_id_priv-&gt;qp_type == IB_QPT_XRC_TGT)</div><div class='add'>+		if (cm_id_priv-&gt;qp_type == IB_QPT_XRC_TGT) {</div><div class='add'>+			cm_id-&gt;state = IB_CM_IDLE;</div><div class='ctx'> 			break;</div><div class='del'>-		ib_send_cm_dreq(cm_id, NULL, 0);</div><div class='add'>+		}</div><div class='add'>+		cm_issue_dreq(cm_id_priv);</div><div class='add'>+		cm_enter_timewait(cm_id_priv);</div><div class='ctx'> 		goto retest;</div><div class='ctx'> 	case IB_CM_DREQ_SENT:</div><div class='del'>-		ib_cancel_mad(cm_id_priv-&gt;av.port-&gt;mad_agent, cm_id_priv-&gt;msg);</div><div class='add'>+		ib_cancel_mad(cm_id_priv-&gt;msg);</div><div class='ctx'> 		cm_enter_timewait(cm_id_priv);</div><div class='del'>-		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-		break;</div><div class='add'>+		goto retest;</div><div class='ctx'> 	case IB_CM_DREQ_RCVD:</div><div class='del'>-		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-		ib_send_cm_drep(cm_id, NULL, 0);</div><div class='add'>+		cm_send_drep_locked(cm_id_priv, NULL, 0);</div><div class='add'>+		WARN_ON(cm_id-&gt;state != IB_CM_TIMEWAIT);</div><div class='add'>+		goto retest;</div><div class='add'>+	case IB_CM_TIMEWAIT:</div><div class='add'>+		/*</div><div class='add'>+		 * The cm_acquire_id in cm_timewait_handler will stop working</div><div class='add'>+		 * once we do xa_erase below, so just move to idle here for</div><div class='add'>+		 * consistency.</div><div class='add'>+		 */</div><div class='add'>+		cm_id-&gt;state = IB_CM_IDLE;</div><div class='ctx'> 		break;</div><div class='del'>-	default:</div><div class='del'>-		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+	case IB_CM_IDLE:</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='add'>+	WARN_ON(cm_id-&gt;state != IB_CM_IDLE);</div><div class='ctx'> </div><div class='del'>-	spin_lock_irq(&amp;cm.lock);</div><div class='del'>-	if (!list_empty(&amp;cm_id_priv-&gt;altr_list) &amp;&amp;</div><div class='del'>-	    (!cm_id_priv-&gt;altr_send_port_not_ready))</div><div class='del'>-		list_del(&amp;cm_id_priv-&gt;altr_list);</div><div class='del'>-	if (!list_empty(&amp;cm_id_priv-&gt;prim_list) &amp;&amp;</div><div class='del'>-	    (!cm_id_priv-&gt;prim_send_port_not_ready))</div><div class='del'>-		list_del(&amp;cm_id_priv-&gt;prim_list);</div><div class='del'>-	spin_unlock_irq(&amp;cm.lock);</div><div class='add'>+	spin_lock(&amp;cm.lock);</div><div class='add'>+	/* Required for cleanup paths related cm_req_handler() */</div><div class='add'>+	if (cm_id_priv-&gt;timewait_info) {</div><div class='add'>+		cm_remove_remote(cm_id_priv);</div><div class='add'>+		kfree(cm_id_priv-&gt;timewait_info);</div><div class='add'>+		cm_id_priv-&gt;timewait_info = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	WARN_ON(cm_id_priv-&gt;listen_sharecount);</div><div class='add'>+	WARN_ON(!RB_EMPTY_NODE(&amp;cm_id_priv-&gt;service_node));</div><div class='add'>+	if (!RB_EMPTY_NODE(&amp;cm_id_priv-&gt;sidr_id_node))</div><div class='add'>+		rb_erase(&amp;cm_id_priv-&gt;sidr_id_node, &amp;cm.remote_sidr_table);</div><div class='add'>+	spin_unlock(&amp;cm.lock);</div><div class='add'>+	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> </div><div class='del'>-	cm_free_id(cm_id-&gt;local_id);</div><div class='add'>+	xa_erase(&amp;cm.local_id_table, cm_local_id(cm_id-&gt;local_id));</div><div class='ctx'> 	cm_deref_id(cm_id_priv);</div><div class='del'>-	wait_for_completion(&amp;cm_id_priv-&gt;comp);</div><div class='add'>+	timeout = msecs_to_jiffies((cm_id_priv-&gt;max_cm_retries * cm_id_priv-&gt;timeout_ms * 5) / 4);</div><div class='add'>+	do {</div><div class='add'>+		ret = wait_for_completion_timeout(&amp;cm_id_priv-&gt;comp, timeout);</div><div class='add'>+		if (!ret) /* timeout happened */</div><div class='add'>+			cm_destroy_id_wait_timeout(cm_id, old_state);</div><div class='add'>+	} while (!ret);</div><div class='add'>+</div><div class='ctx'> 	while ((work = cm_dequeue_work(cm_id_priv)) != NULL)</div><div class='ctx'> 		cm_free_work(work);</div><div class='ctx'> </div><div class='del'>-	rdma_destroy_ah_attr(&amp;cm_id_priv-&gt;av.ah_attr);</div><div class='del'>-	rdma_destroy_ah_attr(&amp;cm_id_priv-&gt;alt_av.ah_attr);</div><div class='add'>+	cm_destroy_av(&amp;cm_id_priv-&gt;av);</div><div class='add'>+	cm_destroy_av(&amp;cm_id_priv-&gt;alt_av);</div><div class='ctx'> 	kfree(cm_id_priv-&gt;private_data);</div><div class='del'>-	kfree(cm_id_priv);</div><div class='add'>+	kfree_rcu(cm_id_priv, rcu);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void ib_destroy_cm_id(struct ib_cm_id *cm_id)</div><div class='hunk'>@@ -1124,70 +1189,63 @@ void ib_destroy_cm_id(struct ib_cm_id *cm_id)</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_destroy_cm_id);</div><div class='ctx'> </div><div class='add'>+static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id)</div><div class='add'>+{</div><div class='add'>+	if ((service_id &amp; IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &amp;&amp;</div><div class='add'>+	    (service_id != IB_CM_ASSIGN_SERVICE_ID))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (service_id == IB_CM_ASSIGN_SERVICE_ID)</div><div class='add'>+		cm_id_priv-&gt;id.service_id = cpu_to_be64(cm.listen_service_id++);</div><div class='add'>+	else</div><div class='add'>+		cm_id_priv-&gt;id.service_id = service_id;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /**</div><div class='del'>- * __ib_cm_listen - Initiates listening on the specified service ID for</div><div class='add'>+ * ib_cm_listen - Initiates listening on the specified service ID for</div><div class='ctx'>  *   connection and service ID resolution requests.</div><div class='ctx'>  * @cm_id: Connection identifier associated with the listen request.</div><div class='ctx'>  * @service_id: Service identifier matched against incoming connection</div><div class='ctx'>  *   and service ID resolution requests.  The service ID should be specified</div><div class='ctx'>  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will</div><div class='ctx'>  *   assign a service ID to the caller.</div><div class='del'>- * @service_mask: Mask applied to service ID used to listen across a</div><div class='del'>- *   range of service IDs.  If set to 0, the service ID is matched</div><div class='del'>- *   exactly.  This parameter is ignored if %service_id is set to</div><div class='del'>- *   IB_CM_ASSIGN_SERVICE_ID.</div><div class='ctx'>  */</div><div class='del'>-static int __ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id,</div><div class='del'>-			  __be64 service_mask)</div><div class='add'>+int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id)</div><div class='ctx'> {</div><div class='del'>-	struct cm_id_private *cm_id_priv, *cur_cm_id_priv;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	service_mask = service_mask ? service_mask : ~cpu_to_be64(0);</div><div class='del'>-	service_id &amp;= service_mask;</div><div class='del'>-	if ((service_id &amp; IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &amp;&amp;</div><div class='del'>-	    (service_id != IB_CM_ASSIGN_SERVICE_ID))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	cm_id_priv = container_of(cm_id, struct cm_id_private, id);</div><div class='del'>-	if (cm_id-&gt;state != IB_CM_IDLE)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	cm_id-&gt;state = IB_CM_LISTEN;</div><div class='del'>-	++cm_id_priv-&gt;listen_sharecount;</div><div class='add'>+	struct cm_id_private *cm_id_priv =</div><div class='add'>+		container_of(cm_id, struct cm_id_private, id);</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	if (service_id == IB_CM_ASSIGN_SERVICE_ID) {</div><div class='del'>-		cm_id-&gt;service_id = cpu_to_be64(cm.listen_service_id++);</div><div class='del'>-		cm_id-&gt;service_mask = ~cpu_to_be64(0);</div><div class='del'>-	} else {</div><div class='del'>-		cm_id-&gt;service_id = service_id;</div><div class='del'>-		cm_id-&gt;service_mask = service_mask;</div><div class='add'>+	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+	if (cm_id_priv-&gt;id.state != IB_CM_IDLE) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto out;</div><div class='ctx'> 	}</div><div class='del'>-	cur_cm_id_priv = cm_insert_listen(cm_id_priv);</div><div class='ctx'> </div><div class='del'>-	if (cur_cm_id_priv) {</div><div class='del'>-		cm_id-&gt;state = IB_CM_IDLE;</div><div class='del'>-		--cm_id_priv-&gt;listen_sharecount;</div><div class='add'>+	ret = cm_init_listen(cm_id_priv, service_id);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	if (!cm_insert_listen(cm_id_priv, NULL)) {</div><div class='ctx'> 		ret = -EBUSY;</div><div class='add'>+		goto out;</div><div class='ctx'> 	}</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='ctx'> </div><div class='del'>-int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask)</div><div class='del'>-{</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;cm.lock, flags);</div><div class='del'>-	ret = __ib_cm_listen(cm_id, service_id, service_mask);</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm.lock, flags);</div><div class='add'>+	cm_id_priv-&gt;id.state = IB_CM_LISTEN;</div><div class='add'>+	ret = 0;</div><div class='ctx'> </div><div class='add'>+out:</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_cm_listen);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * Create a new listening ib_cm_id and listen on the given service ID.</div><div class='add'>+ * ib_cm_insert_listen - Create a new listening ib_cm_id and listen on</div><div class='add'>+ *			 the given service ID.</div><div class='ctx'>  *</div><div class='ctx'>  * If there's an existing ID listening on that same device and service ID,</div><div class='ctx'>  * return it.</div><div class='hunk'>@@ -1206,59 +1264,57 @@ struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device,</div><div class='ctx'> 				     ib_cm_handler cm_handler,</div><div class='ctx'> 				     __be64 service_id)</div><div class='ctx'> {</div><div class='add'>+	struct cm_id_private *listen_id_priv;</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='del'>-	struct ib_cm_id *cm_id;</div><div class='del'>-	unsigned long flags;</div><div class='ctx'> 	int err = 0;</div><div class='ctx'> </div><div class='ctx'> 	/* Create an ID in advance, since the creation may sleep */</div><div class='del'>-	cm_id = ib_create_cm_id(device, cm_handler, NULL);</div><div class='del'>-	if (IS_ERR(cm_id))</div><div class='del'>-		return cm_id;</div><div class='add'>+	cm_id_priv = cm_alloc_id_priv(device, cm_handler, NULL);</div><div class='add'>+	if (IS_ERR(cm_id_priv))</div><div class='add'>+		return ERR_CAST(cm_id_priv);</div><div class='ctx'> </div><div class='del'>-	spin_lock_irqsave(&amp;cm.lock, flags);</div><div class='add'>+	err = cm_init_listen(cm_id_priv, service_id);</div><div class='add'>+	if (err) {</div><div class='add'>+		ib_destroy_cm_id(&amp;cm_id_priv-&gt;id);</div><div class='add'>+		return ERR_PTR(err);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (service_id == IB_CM_ASSIGN_SERVICE_ID)</div><div class='del'>-		goto new_id;</div><div class='del'>-</div><div class='del'>-	/* Find an existing ID */</div><div class='del'>-	cm_id_priv = cm_find_listen(device, service_id);</div><div class='del'>-	if (cm_id_priv) {</div><div class='del'>-		if (cm_id-&gt;cm_handler != cm_handler || cm_id-&gt;context) {</div><div class='del'>-			/* Sharing an ib_cm_id with different handlers is not</div><div class='del'>-			 * supported */</div><div class='del'>-			spin_unlock_irqrestore(&amp;cm.lock, flags);</div><div class='add'>+	spin_lock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+	listen_id_priv = cm_insert_listen(cm_id_priv, cm_handler);</div><div class='add'>+	if (listen_id_priv != cm_id_priv) {</div><div class='add'>+		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+		ib_destroy_cm_id(&amp;cm_id_priv-&gt;id);</div><div class='add'>+		if (!listen_id_priv)</div><div class='ctx'> 			return ERR_PTR(-EINVAL);</div><div class='del'>-		}</div><div class='del'>-		atomic_inc(&amp;cm_id_priv-&gt;refcount);</div><div class='del'>-		++cm_id_priv-&gt;listen_sharecount;</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm.lock, flags);</div><div class='del'>-</div><div class='del'>-		ib_destroy_cm_id(cm_id);</div><div class='del'>-		cm_id = &amp;cm_id_priv-&gt;id;</div><div class='del'>-		return cm_id;</div><div class='add'>+		return &amp;listen_id_priv-&gt;id;</div><div class='ctx'> 	}</div><div class='add'>+	cm_id_priv-&gt;id.state = IB_CM_LISTEN;</div><div class='add'>+	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> </div><div class='del'>-new_id:</div><div class='del'>-	/* Use newly created ID */</div><div class='del'>-	err = __ib_cm_listen(cm_id, service_id, 0);</div><div class='del'>-</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm.lock, flags);</div><div class='add'>+	/*</div><div class='add'>+	 * A listen ID does not need to be in the xarray since it does not</div><div class='add'>+	 * receive mads, is not placed in the remote_id or remote_qpn rbtree,</div><div class='add'>+	 * and does not enter timewait.</div><div class='add'>+	 */</div><div class='ctx'> </div><div class='del'>-	if (err) {</div><div class='del'>-		ib_destroy_cm_id(cm_id);</div><div class='del'>-		return ERR_PTR(err);</div><div class='del'>-	}</div><div class='del'>-	return cm_id;</div><div class='add'>+	return &amp;cm_id_priv-&gt;id;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_cm_insert_listen);</div><div class='ctx'> </div><div class='ctx'> static __be64 cm_form_tid(struct cm_id_private *cm_id_priv)</div><div class='ctx'> {</div><div class='del'>-	u64 hi_tid, low_tid;</div><div class='add'>+	u64 hi_tid = 0, low_tid;</div><div class='add'>+</div><div class='add'>+	lockdep_assert_held(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> </div><div class='del'>-	hi_tid   = ((u64) cm_id_priv-&gt;av.port-&gt;mad_agent-&gt;hi_tid) &lt;&lt; 32;</div><div class='del'>-	low_tid  = (u64)cm_id_priv-&gt;id.local_id;</div><div class='add'>+	low_tid = (u64)cm_id_priv-&gt;id.local_id;</div><div class='add'>+	if (!cm_id_priv-&gt;av.port)</div><div class='add'>+		return cpu_to_be64(low_tid);</div><div class='add'>+</div><div class='add'>+	read_lock(&amp;cm_id_priv-&gt;av.port-&gt;cm_dev-&gt;mad_agent_lock);</div><div class='add'>+	if (cm_id_priv-&gt;av.port-&gt;mad_agent)</div><div class='add'>+		hi_tid = ((u64)cm_id_priv-&gt;av.port-&gt;mad_agent-&gt;hi_tid) &lt;&lt; 32;</div><div class='add'>+	read_unlock(&amp;cm_id_priv-&gt;av.port-&gt;cm_dev-&gt;mad_agent_lock);</div><div class='ctx'> 	return cpu_to_be64(hi_tid | low_tid);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1273,6 +1329,13 @@ static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,</div><div class='ctx'> 	hdr-&gt;tid	   = tid;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void cm_format_mad_ece_hdr(struct ib_mad_hdr *hdr, __be16 attr_id,</div><div class='add'>+				  __be64 tid, u32 attr_mod)</div><div class='add'>+{</div><div class='add'>+	cm_format_mad_hdr(hdr, attr_id, tid);</div><div class='add'>+	hdr-&gt;attr_mod = cpu_to_be32(attr_mod);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static void cm_format_req(struct cm_req_msg *req_msg,</div><div class='ctx'> 			  struct cm_id_private *cm_id_priv,</div><div class='ctx'> 			  struct ib_cm_req_param *param)</div><div class='hunk'>@@ -1280,62 +1343,88 @@ static void cm_format_req(struct cm_req_msg *req_msg,</div><div class='ctx'> 	struct sa_path_rec *pri_path = param-&gt;primary_path;</div><div class='ctx'> 	struct sa_path_rec *alt_path = param-&gt;alternate_path;</div><div class='ctx'> 	bool pri_ext = false;</div><div class='add'>+	__be16 lid;</div><div class='ctx'> </div><div class='ctx'> 	if (pri_path-&gt;rec_type == SA_PATH_REC_TYPE_OPA)</div><div class='ctx'> 		pri_ext = opa_is_extended_lid(pri_path-&gt;opa.dlid,</div><div class='ctx'> 					      pri_path-&gt;opa.slid);</div><div class='ctx'> </div><div class='del'>-	cm_format_mad_hdr(&amp;req_msg-&gt;hdr, CM_REQ_ATTR_ID,</div><div class='del'>-			  cm_form_tid(cm_id_priv));</div><div class='del'>-</div><div class='del'>-	req_msg-&gt;local_comm_id = cm_id_priv-&gt;id.local_id;</div><div class='del'>-	req_msg-&gt;service_id = param-&gt;service_id;</div><div class='del'>-	req_msg-&gt;local_ca_guid = cm_id_priv-&gt;id.device-&gt;node_guid;</div><div class='del'>-	cm_req_set_local_qpn(req_msg, cpu_to_be32(param-&gt;qp_num));</div><div class='del'>-	cm_req_set_init_depth(req_msg, param-&gt;initiator_depth);</div><div class='del'>-	cm_req_set_remote_resp_timeout(req_msg,</div><div class='del'>-				       param-&gt;remote_cm_response_timeout);</div><div class='add'>+	cm_format_mad_ece_hdr(&amp;req_msg-&gt;hdr, CM_REQ_ATTR_ID,</div><div class='add'>+			      cm_form_tid(cm_id_priv), param-&gt;ece.attr_mod);</div><div class='add'>+</div><div class='add'>+	IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg,</div><div class='add'>+		be32_to_cpu(cm_id_priv-&gt;id.local_id));</div><div class='add'>+	IBA_SET(CM_REQ_SERVICE_ID, req_msg, be64_to_cpu(param-&gt;service_id));</div><div class='add'>+	IBA_SET(CM_REQ_LOCAL_CA_GUID, req_msg,</div><div class='add'>+		be64_to_cpu(cm_id_priv-&gt;id.device-&gt;node_guid));</div><div class='add'>+	IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param-&gt;qp_num);</div><div class='add'>+	IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param-&gt;initiator_depth);</div><div class='add'>+	IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg,</div><div class='add'>+		param-&gt;remote_cm_response_timeout);</div><div class='ctx'> 	cm_req_set_qp_type(req_msg, param-&gt;qp_type);</div><div class='del'>-	cm_req_set_flow_ctrl(req_msg, param-&gt;flow_control);</div><div class='del'>-	cm_req_set_starting_psn(req_msg, cpu_to_be32(param-&gt;starting_psn));</div><div class='del'>-	cm_req_set_local_resp_timeout(req_msg,</div><div class='del'>-				      param-&gt;local_cm_response_timeout);</div><div class='del'>-	req_msg-&gt;pkey = param-&gt;primary_path-&gt;pkey;</div><div class='del'>-	cm_req_set_path_mtu(req_msg, param-&gt;primary_path-&gt;mtu);</div><div class='del'>-	cm_req_set_max_cm_retries(req_msg, param-&gt;max_cm_retries);</div><div class='add'>+	IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param-&gt;flow_control);</div><div class='add'>+	IBA_SET(CM_REQ_STARTING_PSN, req_msg, param-&gt;starting_psn);</div><div class='add'>+	IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg,</div><div class='add'>+		param-&gt;local_cm_response_timeout);</div><div class='add'>+	IBA_SET(CM_REQ_PARTITION_KEY, req_msg,</div><div class='add'>+		be16_to_cpu(param-&gt;primary_path-&gt;pkey));</div><div class='add'>+	IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg,</div><div class='add'>+		param-&gt;primary_path-&gt;mtu);</div><div class='add'>+	IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param-&gt;max_cm_retries);</div><div class='ctx'> </div><div class='ctx'> 	if (param-&gt;qp_type != IB_QPT_XRC_INI) {</div><div class='del'>-		cm_req_set_resp_res(req_msg, param-&gt;responder_resources);</div><div class='del'>-		cm_req_set_retry_count(req_msg, param-&gt;retry_count);</div><div class='del'>-		cm_req_set_rnr_retry_count(req_msg, param-&gt;rnr_retry_count);</div><div class='del'>-		cm_req_set_srq(req_msg, param-&gt;srq);</div><div class='add'>+		IBA_SET(CM_REQ_RESPONDER_RESOURCES, req_msg,</div><div class='add'>+			param-&gt;responder_resources);</div><div class='add'>+		IBA_SET(CM_REQ_RETRY_COUNT, req_msg, param-&gt;retry_count);</div><div class='add'>+		IBA_SET(CM_REQ_RNR_RETRY_COUNT, req_msg,</div><div class='add'>+			param-&gt;rnr_retry_count);</div><div class='add'>+		IBA_SET(CM_REQ_SRQ, req_msg, param-&gt;srq);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	req_msg-&gt;primary_local_gid = pri_path-&gt;sgid;</div><div class='del'>-	req_msg-&gt;primary_remote_gid = pri_path-&gt;dgid;</div><div class='add'>+	*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) =</div><div class='add'>+		pri_path-&gt;sgid;</div><div class='add'>+	*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) =</div><div class='add'>+		pri_path-&gt;dgid;</div><div class='ctx'> 	if (pri_ext) {</div><div class='del'>-		req_msg-&gt;primary_local_gid.global.interface_id</div><div class='del'>-			= OPA_MAKE_ID(be32_to_cpu(pri_path-&gt;opa.slid));</div><div class='del'>-		req_msg-&gt;primary_remote_gid.global.interface_id</div><div class='del'>-			= OPA_MAKE_ID(be32_to_cpu(pri_path-&gt;opa.dlid));</div><div class='add'>+		IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg)</div><div class='add'>+			-&gt;global.interface_id =</div><div class='add'>+			OPA_MAKE_ID(be32_to_cpu(pri_path-&gt;opa.slid));</div><div class='add'>+		IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg)</div><div class='add'>+			-&gt;global.interface_id =</div><div class='add'>+			OPA_MAKE_ID(be32_to_cpu(pri_path-&gt;opa.dlid));</div><div class='ctx'> 	}</div><div class='ctx'> 	if (pri_path-&gt;hop_limit &lt;= 1) {</div><div class='del'>-		req_msg-&gt;primary_local_lid = pri_ext ? 0 :</div><div class='del'>-			htons(ntohl(sa_path_get_slid(pri_path)));</div><div class='del'>-		req_msg-&gt;primary_remote_lid = pri_ext ? 0 :</div><div class='del'>-			htons(ntohl(sa_path_get_dlid(pri_path)));</div><div class='add'>+		IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,</div><div class='add'>+			be16_to_cpu(pri_ext ? 0 :</div><div class='add'>+					      htons(ntohl(sa_path_get_slid(</div><div class='add'>+						      pri_path)))));</div><div class='add'>+		IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,</div><div class='add'>+			be16_to_cpu(pri_ext ? 0 :</div><div class='add'>+					      htons(ntohl(sa_path_get_dlid(</div><div class='add'>+						      pri_path)))));</div><div class='ctx'> 	} else {</div><div class='add'>+</div><div class='add'>+		if (param-&gt;primary_path_inbound) {</div><div class='add'>+			lid = param-&gt;primary_path_inbound-&gt;ib.dlid;</div><div class='add'>+			IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,</div><div class='add'>+				be16_to_cpu(lid));</div><div class='add'>+		} else</div><div class='add'>+			IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,</div><div class='add'>+				be16_to_cpu(IB_LID_PERMISSIVE));</div><div class='add'>+</div><div class='ctx'> 		/* Work-around until there's a way to obtain remote LID info */</div><div class='del'>-		req_msg-&gt;primary_local_lid = IB_LID_PERMISSIVE;</div><div class='del'>-		req_msg-&gt;primary_remote_lid = IB_LID_PERMISSIVE;</div><div class='add'>+		IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,</div><div class='add'>+			be16_to_cpu(IB_LID_PERMISSIVE));</div><div class='ctx'> 	}</div><div class='del'>-	cm_req_set_primary_flow_label(req_msg, pri_path-&gt;flow_label);</div><div class='del'>-	cm_req_set_primary_packet_rate(req_msg, pri_path-&gt;rate);</div><div class='del'>-	req_msg-&gt;primary_traffic_class = pri_path-&gt;traffic_class;</div><div class='del'>-	req_msg-&gt;primary_hop_limit = pri_path-&gt;hop_limit;</div><div class='del'>-	cm_req_set_primary_sl(req_msg, pri_path-&gt;sl);</div><div class='del'>-	cm_req_set_primary_subnet_local(req_msg, (pri_path-&gt;hop_limit &lt;= 1));</div><div class='del'>-	cm_req_set_primary_local_ack_timeout(req_msg,</div><div class='add'>+	IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg,</div><div class='add'>+		be32_to_cpu(pri_path-&gt;flow_label));</div><div class='add'>+	IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path-&gt;rate);</div><div class='add'>+	IBA_SET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg, pri_path-&gt;traffic_class);</div><div class='add'>+	IBA_SET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg, pri_path-&gt;hop_limit);</div><div class='add'>+	IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path-&gt;sl);</div><div class='add'>+	IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg,</div><div class='add'>+		(pri_path-&gt;hop_limit &lt;= 1));</div><div class='add'>+	IBA_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg,</div><div class='ctx'> 		cm_ack_timeout(cm_id_priv-&gt;av.port-&gt;cm_dev-&gt;ack_delay,</div><div class='ctx'> 			       pri_path-&gt;packet_life_time));</div><div class='ctx'> </div><div class='hunk'>@@ -1346,46 +1435,60 @@ static void cm_format_req(struct cm_req_msg *req_msg,</div><div class='ctx'> 			alt_ext = opa_is_extended_lid(alt_path-&gt;opa.dlid,</div><div class='ctx'> 						      alt_path-&gt;opa.slid);</div><div class='ctx'> </div><div class='del'>-		req_msg-&gt;alt_local_gid = alt_path-&gt;sgid;</div><div class='del'>-		req_msg-&gt;alt_remote_gid = alt_path-&gt;dgid;</div><div class='add'>+		*IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg) =</div><div class='add'>+			alt_path-&gt;sgid;</div><div class='add'>+		*IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg) =</div><div class='add'>+			alt_path-&gt;dgid;</div><div class='ctx'> 		if (alt_ext) {</div><div class='del'>-			req_msg-&gt;alt_local_gid.global.interface_id</div><div class='del'>-				= OPA_MAKE_ID(be32_to_cpu(alt_path-&gt;opa.slid));</div><div class='del'>-			req_msg-&gt;alt_remote_gid.global.interface_id</div><div class='del'>-				= OPA_MAKE_ID(be32_to_cpu(alt_path-&gt;opa.dlid));</div><div class='add'>+			IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,</div><div class='add'>+					req_msg)</div><div class='add'>+				-&gt;global.interface_id =</div><div class='add'>+				OPA_MAKE_ID(be32_to_cpu(alt_path-&gt;opa.slid));</div><div class='add'>+			IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID,</div><div class='add'>+					req_msg)</div><div class='add'>+				-&gt;global.interface_id =</div><div class='add'>+				OPA_MAKE_ID(be32_to_cpu(alt_path-&gt;opa.dlid));</div><div class='ctx'> 		}</div><div class='ctx'> 		if (alt_path-&gt;hop_limit &lt;= 1) {</div><div class='del'>-			req_msg-&gt;alt_local_lid = alt_ext ? 0 :</div><div class='del'>-				htons(ntohl(sa_path_get_slid(alt_path)));</div><div class='del'>-			req_msg-&gt;alt_remote_lid = alt_ext ? 0 :</div><div class='del'>-				htons(ntohl(sa_path_get_dlid(alt_path)));</div><div class='add'>+			IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,</div><div class='add'>+				be16_to_cpu(</div><div class='add'>+					alt_ext ? 0 :</div><div class='add'>+						  htons(ntohl(sa_path_get_slid(</div><div class='add'>+							  alt_path)))));</div><div class='add'>+			IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,</div><div class='add'>+				be16_to_cpu(</div><div class='add'>+					alt_ext ? 0 :</div><div class='add'>+						  htons(ntohl(sa_path_get_dlid(</div><div class='add'>+							  alt_path)))));</div><div class='ctx'> 		} else {</div><div class='del'>-			req_msg-&gt;alt_local_lid = IB_LID_PERMISSIVE;</div><div class='del'>-			req_msg-&gt;alt_remote_lid = IB_LID_PERMISSIVE;</div><div class='add'>+			IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,</div><div class='add'>+				be16_to_cpu(IB_LID_PERMISSIVE));</div><div class='add'>+			IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,</div><div class='add'>+				be16_to_cpu(IB_LID_PERMISSIVE));</div><div class='ctx'> 		}</div><div class='del'>-		cm_req_set_alt_flow_label(req_msg,</div><div class='del'>-					  alt_path-&gt;flow_label);</div><div class='del'>-		cm_req_set_alt_packet_rate(req_msg, alt_path-&gt;rate);</div><div class='del'>-		req_msg-&gt;alt_traffic_class = alt_path-&gt;traffic_class;</div><div class='del'>-		req_msg-&gt;alt_hop_limit = alt_path-&gt;hop_limit;</div><div class='del'>-		cm_req_set_alt_sl(req_msg, alt_path-&gt;sl);</div><div class='del'>-		cm_req_set_alt_subnet_local(req_msg, (alt_path-&gt;hop_limit &lt;= 1));</div><div class='del'>-		cm_req_set_alt_local_ack_timeout(req_msg,</div><div class='add'>+		IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg,</div><div class='add'>+			be32_to_cpu(alt_path-&gt;flow_label));</div><div class='add'>+		IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path-&gt;rate);</div><div class='add'>+		IBA_SET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg,</div><div class='add'>+			alt_path-&gt;traffic_class);</div><div class='add'>+		IBA_SET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg,</div><div class='add'>+			alt_path-&gt;hop_limit);</div><div class='add'>+		IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path-&gt;sl);</div><div class='add'>+		IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg,</div><div class='add'>+			(alt_path-&gt;hop_limit &lt;= 1));</div><div class='add'>+		IBA_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg,</div><div class='ctx'> 			cm_ack_timeout(cm_id_priv-&gt;av.port-&gt;cm_dev-&gt;ack_delay,</div><div class='ctx'> 				       alt_path-&gt;packet_life_time));</div><div class='ctx'> 	}</div><div class='add'>+	IBA_SET(CM_REQ_VENDOR_ID, req_msg, param-&gt;ece.vendor_id);</div><div class='ctx'> </div><div class='ctx'> 	if (param-&gt;private_data &amp;&amp; param-&gt;private_data_len)</div><div class='del'>-		memcpy(req_msg-&gt;private_data, param-&gt;private_data,</div><div class='del'>-		       param-&gt;private_data_len);</div><div class='add'>+		IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param-&gt;private_data,</div><div class='add'>+			    param-&gt;private_data_len);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int cm_validate_req_param(struct ib_cm_req_param *param)</div><div class='ctx'> {</div><div class='del'>-	/* peer-to-peer not supported */</div><div class='del'>-	if (param-&gt;peer_to_peer)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='ctx'> 	if (!param-&gt;primary_path)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='hunk'>@@ -1408,7 +1511,9 @@ static int cm_validate_req_param(struct ib_cm_req_param *param)</div><div class='ctx'> int ib_send_cm_req(struct ib_cm_id *cm_id,</div><div class='ctx'> 		   struct ib_cm_req_param *param)</div><div class='ctx'> {</div><div class='add'>+	struct cm_av av = {}, alt_av = {};</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='add'>+	struct ib_mad_send_buf *msg;</div><div class='ctx'> 	struct cm_req_msg *req_msg;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int ret;</div><div class='hunk'>@@ -1420,10 +1525,9 @@ int ib_send_cm_req(struct ib_cm_id *cm_id,</div><div class='ctx'> 	/* Verify that we're not in timewait. */</div><div class='ctx'> 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);</div><div class='ctx'> 	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	if (cm_id-&gt;state != IB_CM_IDLE) {</div><div class='add'>+	if (cm_id-&gt;state != IB_CM_IDLE || WARN_ON(cm_id_priv-&gt;timewait_info)) {</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-		goto out;</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> </div><div class='hunk'>@@ -1431,22 +1535,23 @@ int ib_send_cm_req(struct ib_cm_id *cm_id,</div><div class='ctx'> 							    id.local_id);</div><div class='ctx'> 	if (IS_ERR(cm_id_priv-&gt;timewait_info)) {</div><div class='ctx'> 		ret = PTR_ERR(cm_id_priv-&gt;timewait_info);</div><div class='del'>-		goto out;</div><div class='add'>+		cm_id_priv-&gt;timewait_info = NULL;</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ret = cm_init_av_by_path(param-&gt;primary_path,</div><div class='del'>-				 param-&gt;ppath_sgid_attr, &amp;cm_id_priv-&gt;av,</div><div class='del'>-				 cm_id_priv);</div><div class='add'>+				 param-&gt;ppath_sgid_attr, &amp;av);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto error1;</div><div class='add'>+		return ret;</div><div class='ctx'> 	if (param-&gt;alternate_path) {</div><div class='ctx'> 		ret = cm_init_av_by_path(param-&gt;alternate_path, NULL,</div><div class='del'>-					 &amp;cm_id_priv-&gt;alt_av, cm_id_priv);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto error1;</div><div class='add'>+					 &amp;alt_av);</div><div class='add'>+		if (ret) {</div><div class='add'>+			cm_destroy_av(&amp;av);</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> 	cm_id-&gt;service_id = param-&gt;service_id;</div><div class='del'>-	cm_id-&gt;service_mask = ~cpu_to_be64(0);</div><div class='ctx'> 	cm_id_priv-&gt;timeout_ms = cm_convert_to_ms(</div><div class='ctx'> 				    param-&gt;primary_path-&gt;packet_life_time) * 2 +</div><div class='ctx'> 				 cm_convert_to_ms(</div><div class='hunk'>@@ -1459,33 +1564,42 @@ int ib_send_cm_req(struct ib_cm_id *cm_id,</div><div class='ctx'> 	cm_id_priv-&gt;pkey = param-&gt;primary_path-&gt;pkey;</div><div class='ctx'> 	cm_id_priv-&gt;qp_type = param-&gt;qp_type;</div><div class='ctx'> </div><div class='del'>-	ret = cm_alloc_msg(cm_id_priv, &amp;cm_id_priv-&gt;msg);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto error1;</div><div class='add'>+	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	cm_move_av_from_path(&amp;cm_id_priv-&gt;av, &amp;av);</div><div class='add'>+	if (param-&gt;primary_path_outbound)</div><div class='add'>+		cm_id_priv-&gt;av.dlid_datapath =</div><div class='add'>+			be16_to_cpu(param-&gt;primary_path_outbound-&gt;ib.dlid);</div><div class='add'>+</div><div class='add'>+	if (param-&gt;alternate_path)</div><div class='add'>+		cm_move_av_from_path(&amp;cm_id_priv-&gt;alt_av, &amp;alt_av);</div><div class='ctx'> </div><div class='del'>-	req_msg = (struct cm_req_msg *) cm_id_priv-&gt;msg-&gt;mad;</div><div class='add'>+	msg = cm_alloc_priv_msg(cm_id_priv, IB_CM_REQ_SENT);</div><div class='add'>+	if (IS_ERR(msg)) {</div><div class='add'>+		ret = PTR_ERR(msg);</div><div class='add'>+		goto out_unlock;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	req_msg = (struct cm_req_msg *)msg-&gt;mad;</div><div class='ctx'> 	cm_format_req(req_msg, cm_id_priv, param);</div><div class='ctx'> 	cm_id_priv-&gt;tid = req_msg-&gt;hdr.tid;</div><div class='del'>-	cm_id_priv-&gt;msg-&gt;timeout_ms = cm_id_priv-&gt;timeout_ms;</div><div class='del'>-	cm_id_priv-&gt;msg-&gt;context[1] = (void *) (unsigned long) IB_CM_REQ_SENT;</div><div class='ctx'> </div><div class='del'>-	cm_id_priv-&gt;local_qpn = cm_req_get_local_qpn(req_msg);</div><div class='del'>-	cm_id_priv-&gt;rq_psn = cm_req_get_starting_psn(req_msg);</div><div class='add'>+	cm_id_priv-&gt;local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));</div><div class='add'>+	cm_id_priv-&gt;rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));</div><div class='ctx'> </div><div class='del'>-	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	ret = ib_post_send_mad(cm_id_priv-&gt;msg, NULL);</div><div class='del'>-	if (ret) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-		goto error2;</div><div class='del'>-	}</div><div class='add'>+	trace_icm_send_req(&amp;cm_id_priv-&gt;id);</div><div class='add'>+	ret = ib_post_send_mad(msg, NULL);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out_free;</div><div class='ctx'> 	BUG_ON(cm_id-&gt;state != IB_CM_IDLE);</div><div class='ctx'> 	cm_id-&gt;state = IB_CM_REQ_SENT;</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 	return 0;</div><div class='del'>-</div><div class='del'>-error2:	cm_free_msg(cm_id_priv-&gt;msg);</div><div class='del'>-error1:	kfree(cm_id_priv-&gt;timewait_info);</div><div class='del'>-out:	return ret;</div><div class='add'>+out_free:</div><div class='add'>+	cm_free_priv_msg(msg);</div><div class='add'>+out_unlock:</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_send_cm_req);</div><div class='ctx'> </div><div class='hunk'>@@ -1499,7 +1613,7 @@ static int cm_issue_rej(struct cm_port *port,</div><div class='ctx'> 	struct cm_rej_msg *rej_msg, *rcv_msg;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = cm_alloc_response_msg(port, mad_recv_wc, &amp;msg);</div><div class='add'>+	ret = cm_alloc_response_msg(port, mad_recv_wc, false, &amp;msg);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='hunk'>@@ -1508,16 +1622,21 @@ static int cm_issue_rej(struct cm_port *port,</div><div class='ctx'> 	rej_msg = (struct cm_rej_msg *) msg-&gt;mad;</div><div class='ctx'> </div><div class='ctx'> 	cm_format_mad_hdr(&amp;rej_msg-&gt;hdr, CM_REJ_ATTR_ID, rcv_msg-&gt;hdr.tid);</div><div class='del'>-	rej_msg-&gt;remote_comm_id = rcv_msg-&gt;local_comm_id;</div><div class='del'>-	rej_msg-&gt;local_comm_id = rcv_msg-&gt;remote_comm_id;</div><div class='del'>-	cm_rej_set_msg_rejected(rej_msg, msg_rejected);</div><div class='del'>-	rej_msg-&gt;reason = cpu_to_be16(reason);</div><div class='add'>+	IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,</div><div class='add'>+		IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg));</div><div class='add'>+	IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,</div><div class='add'>+		IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));</div><div class='add'>+	IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected);</div><div class='add'>+	IBA_SET(CM_REJ_REASON, rej_msg, reason);</div><div class='ctx'> </div><div class='ctx'> 	if (ari &amp;&amp; ari_length) {</div><div class='del'>-		cm_rej_set_reject_info_len(rej_msg, ari_length);</div><div class='del'>-		memcpy(rej_msg-&gt;ari, ari, ari_length);</div><div class='add'>+		IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);</div><div class='add'>+		IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	trace_icm_issue_rej(</div><div class='add'>+		IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg),</div><div class='add'>+		IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));</div><div class='ctx'> 	ret = ib_post_send_mad(msg, NULL);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		cm_free_msg(msg);</div><div class='hunk'>@@ -1525,21 +1644,15 @@ static int cm_issue_rej(struct cm_port *port,</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline int cm_is_active_peer(__be64 local_ca_guid, __be64 remote_ca_guid,</div><div class='del'>-				    __be32 local_qpn, __be32 remote_qpn)</div><div class='del'>-{</div><div class='del'>-	return (be64_to_cpu(local_ca_guid) &gt; be64_to_cpu(remote_ca_guid) ||</div><div class='del'>-		((local_ca_guid == remote_ca_guid) &amp;&amp;</div><div class='del'>-		 (be32_to_cpu(local_qpn) &gt; be32_to_cpu(remote_qpn))));</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static bool cm_req_has_alt_path(struct cm_req_msg *req_msg)</div><div class='ctx'> {</div><div class='del'>-	return ((req_msg-&gt;alt_local_lid) ||</div><div class='del'>-		(ib_is_opa_gid(&amp;req_msg-&gt;alt_local_gid)));</div><div class='add'>+	return ((cpu_to_be16(</div><div class='add'>+			IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) ||</div><div class='add'>+		(ib_is_opa_gid(IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,</div><div class='add'>+					       req_msg))));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cm_path_set_rec_type(struct ib_device *ib_device, u8 port_num,</div><div class='add'>+static void cm_path_set_rec_type(struct ib_device *ib_device, u32 port_num,</div><div class='ctx'> 				 struct sa_path_rec *path, union ib_gid *gid)</div><div class='ctx'> {</div><div class='ctx'> 	if (ib_is_opa_gid(gid) &amp;&amp; rdma_cap_opa_ah(ib_device, port_num))</div><div class='hunk'>@@ -1550,20 +1663,23 @@ static void cm_path_set_rec_type(struct ib_device *ib_device, u8 port_num,</div><div class='ctx'> </div><div class='ctx'> static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg,</div><div class='ctx'> 					struct sa_path_rec *primary_path,</div><div class='del'>-					struct sa_path_rec *alt_path)</div><div class='add'>+					struct sa_path_rec *alt_path,</div><div class='add'>+					struct ib_wc *wc)</div><div class='ctx'> {</div><div class='ctx'> 	u32 lid;</div><div class='ctx'> </div><div class='ctx'> 	if (primary_path-&gt;rec_type != SA_PATH_REC_TYPE_OPA) {</div><div class='del'>-		sa_path_set_dlid(primary_path,</div><div class='del'>-				 ntohs(req_msg-&gt;primary_local_lid));</div><div class='add'>+		sa_path_set_dlid(primary_path, wc-&gt;slid);</div><div class='ctx'> 		sa_path_set_slid(primary_path,</div><div class='del'>-				 ntohs(req_msg-&gt;primary_remote_lid));</div><div class='add'>+				 IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,</div><div class='add'>+					 req_msg));</div><div class='ctx'> 	} else {</div><div class='del'>-		lid = opa_get_lid_from_gid(&amp;req_msg-&gt;primary_local_gid);</div><div class='add'>+		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(</div><div class='add'>+			CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg));</div><div class='ctx'> 		sa_path_set_dlid(primary_path, lid);</div><div class='ctx'> </div><div class='del'>-		lid = opa_get_lid_from_gid(&amp;req_msg-&gt;primary_remote_gid);</div><div class='add'>+		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(</div><div class='add'>+			CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg));</div><div class='ctx'> 		sa_path_set_slid(primary_path, lid);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -1571,77 +1687,98 @@ static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg,</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='ctx'> 	if (alt_path-&gt;rec_type != SA_PATH_REC_TYPE_OPA) {</div><div class='del'>-		sa_path_set_dlid(alt_path, ntohs(req_msg-&gt;alt_local_lid));</div><div class='del'>-		sa_path_set_slid(alt_path, ntohs(req_msg-&gt;alt_remote_lid));</div><div class='add'>+		sa_path_set_dlid(alt_path,</div><div class='add'>+				 IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,</div><div class='add'>+					 req_msg));</div><div class='add'>+		sa_path_set_slid(alt_path,</div><div class='add'>+				 IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,</div><div class='add'>+					 req_msg));</div><div class='ctx'> 	} else {</div><div class='del'>-		lid = opa_get_lid_from_gid(&amp;req_msg-&gt;alt_local_gid);</div><div class='add'>+		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(</div><div class='add'>+			CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg));</div><div class='ctx'> 		sa_path_set_dlid(alt_path, lid);</div><div class='ctx'> </div><div class='del'>-		lid = opa_get_lid_from_gid(&amp;req_msg-&gt;alt_remote_gid);</div><div class='add'>+		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(</div><div class='add'>+			CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg));</div><div class='ctx'> 		sa_path_set_slid(alt_path, lid);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void cm_format_paths_from_req(struct cm_req_msg *req_msg,</div><div class='ctx'> 				     struct sa_path_rec *primary_path,</div><div class='del'>-				     struct sa_path_rec *alt_path)</div><div class='add'>+				     struct sa_path_rec *alt_path,</div><div class='add'>+				     struct ib_wc *wc)</div><div class='ctx'> {</div><div class='del'>-	primary_path-&gt;dgid = req_msg-&gt;primary_local_gid;</div><div class='del'>-	primary_path-&gt;sgid = req_msg-&gt;primary_remote_gid;</div><div class='del'>-	primary_path-&gt;flow_label = cm_req_get_primary_flow_label(req_msg);</div><div class='del'>-	primary_path-&gt;hop_limit = req_msg-&gt;primary_hop_limit;</div><div class='del'>-	primary_path-&gt;traffic_class = req_msg-&gt;primary_traffic_class;</div><div class='add'>+	primary_path-&gt;dgid =</div><div class='add'>+		*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg);</div><div class='add'>+	primary_path-&gt;sgid =</div><div class='add'>+		*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg);</div><div class='add'>+	primary_path-&gt;flow_label =</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg));</div><div class='add'>+	primary_path-&gt;hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg);</div><div class='add'>+	primary_path-&gt;traffic_class =</div><div class='add'>+		IBA_GET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg);</div><div class='ctx'> 	primary_path-&gt;reversible = 1;</div><div class='del'>-	primary_path-&gt;pkey = req_msg-&gt;pkey;</div><div class='del'>-	primary_path-&gt;sl = cm_req_get_primary_sl(req_msg);</div><div class='add'>+	primary_path-&gt;pkey =</div><div class='add'>+		cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));</div><div class='add'>+	primary_path-&gt;sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg);</div><div class='ctx'> 	primary_path-&gt;mtu_selector = IB_SA_EQ;</div><div class='del'>-	primary_path-&gt;mtu = cm_req_get_path_mtu(req_msg);</div><div class='add'>+	primary_path-&gt;mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);</div><div class='ctx'> 	primary_path-&gt;rate_selector = IB_SA_EQ;</div><div class='del'>-	primary_path-&gt;rate = cm_req_get_primary_packet_rate(req_msg);</div><div class='add'>+	primary_path-&gt;rate = IBA_GET(CM_REQ_PRIMARY_PACKET_RATE, req_msg);</div><div class='ctx'> 	primary_path-&gt;packet_life_time_selector = IB_SA_EQ;</div><div class='ctx'> 	primary_path-&gt;packet_life_time =</div><div class='del'>-		cm_req_get_primary_local_ack_timeout(req_msg);</div><div class='add'>+		IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg);</div><div class='ctx'> 	primary_path-&gt;packet_life_time -= (primary_path-&gt;packet_life_time &gt; 0);</div><div class='del'>-	primary_path-&gt;service_id = req_msg-&gt;service_id;</div><div class='add'>+	primary_path-&gt;service_id =</div><div class='add'>+		cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));</div><div class='ctx'> 	if (sa_path_is_roce(primary_path))</div><div class='ctx'> 		primary_path-&gt;roce.route_resolved = false;</div><div class='ctx'> </div><div class='ctx'> 	if (cm_req_has_alt_path(req_msg)) {</div><div class='del'>-		alt_path-&gt;dgid = req_msg-&gt;alt_local_gid;</div><div class='del'>-		alt_path-&gt;sgid = req_msg-&gt;alt_remote_gid;</div><div class='del'>-		alt_path-&gt;flow_label = cm_req_get_alt_flow_label(req_msg);</div><div class='del'>-		alt_path-&gt;hop_limit = req_msg-&gt;alt_hop_limit;</div><div class='del'>-		alt_path-&gt;traffic_class = req_msg-&gt;alt_traffic_class;</div><div class='add'>+		alt_path-&gt;dgid = *IBA_GET_MEM_PTR(</div><div class='add'>+			CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg);</div><div class='add'>+		alt_path-&gt;sgid = *IBA_GET_MEM_PTR(</div><div class='add'>+			CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg);</div><div class='add'>+		alt_path-&gt;flow_label = cpu_to_be32(</div><div class='add'>+			IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg));</div><div class='add'>+		alt_path-&gt;hop_limit =</div><div class='add'>+			IBA_GET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg);</div><div class='add'>+		alt_path-&gt;traffic_class =</div><div class='add'>+			IBA_GET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg);</div><div class='ctx'> 		alt_path-&gt;reversible = 1;</div><div class='del'>-		alt_path-&gt;pkey = req_msg-&gt;pkey;</div><div class='del'>-		alt_path-&gt;sl = cm_req_get_alt_sl(req_msg);</div><div class='add'>+		alt_path-&gt;pkey =</div><div class='add'>+			cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));</div><div class='add'>+		alt_path-&gt;sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg);</div><div class='ctx'> 		alt_path-&gt;mtu_selector = IB_SA_EQ;</div><div class='del'>-		alt_path-&gt;mtu = cm_req_get_path_mtu(req_msg);</div><div class='add'>+		alt_path-&gt;mtu =</div><div class='add'>+			IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);</div><div class='ctx'> 		alt_path-&gt;rate_selector = IB_SA_EQ;</div><div class='del'>-		alt_path-&gt;rate = cm_req_get_alt_packet_rate(req_msg);</div><div class='add'>+		alt_path-&gt;rate = IBA_GET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg);</div><div class='ctx'> 		alt_path-&gt;packet_life_time_selector = IB_SA_EQ;</div><div class='ctx'> 		alt_path-&gt;packet_life_time =</div><div class='del'>-			cm_req_get_alt_local_ack_timeout(req_msg);</div><div class='add'>+			IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg);</div><div class='ctx'> 		alt_path-&gt;packet_life_time -= (alt_path-&gt;packet_life_time &gt; 0);</div><div class='del'>-		alt_path-&gt;service_id = req_msg-&gt;service_id;</div><div class='add'>+		alt_path-&gt;service_id =</div><div class='add'>+			cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));</div><div class='ctx'> </div><div class='ctx'> 		if (sa_path_is_roce(alt_path))</div><div class='ctx'> 			alt_path-&gt;roce.route_resolved = false;</div><div class='ctx'> 	}</div><div class='del'>-	cm_format_path_lid_from_req(req_msg, primary_path, alt_path);</div><div class='add'>+	cm_format_path_lid_from_req(req_msg, primary_path, alt_path, wc);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static u16 cm_get_bth_pkey(struct cm_work *work)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *ib_dev = work-&gt;port-&gt;cm_dev-&gt;ib_device;</div><div class='del'>-	u8 port_num = work-&gt;port-&gt;port_num;</div><div class='add'>+	u32 port_num = work-&gt;port-&gt;port_num;</div><div class='ctx'> 	u16 pkey_index = work-&gt;mad_recv_wc-&gt;wc-&gt;pkey_index;</div><div class='ctx'> 	u16 pkey;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	ret = ib_get_cached_pkey(ib_dev, port_num, pkey_index, &amp;pkey);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_warn_ratelimited(&amp;ib_dev-&gt;dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %d, pkey index %d). %d\n",</div><div class='add'>+		dev_warn_ratelimited(&amp;ib_dev-&gt;dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %u, pkey index %u). %d\n",</div><div class='ctx'> 				     port_num, pkey_index, ret);</div><div class='ctx'> 		return 0;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -1650,7 +1787,7 @@ static u16 cm_get_bth_pkey(struct cm_work *work)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * Convert OPA SGID to IB SGID</div><div class='add'>+ * cm_opa_to_ib_sgid - Convert OPA SGID to IB SGID</div><div class='ctx'>  * ULPs (such as IPoIB) do not understand OPA GIDs and will</div><div class='ctx'>  * reject them as the local_gid will not match the sgid. Therefore,</div><div class='ctx'>  * change the pathrec's SGID to an IB SGID.</div><div class='hunk'>@@ -1662,7 +1799,7 @@ static void cm_opa_to_ib_sgid(struct cm_work *work,</div><div class='ctx'> 			      struct sa_path_rec *path)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *dev = work-&gt;port-&gt;cm_dev-&gt;ib_device;</div><div class='del'>-	u8 port_num = work-&gt;port-&gt;port_num;</div><div class='add'>+	u32 port_num = work-&gt;port-&gt;port_num;</div><div class='ctx'> </div><div class='ctx'> 	if (rdma_cap_opa_ah(dev, port_num) &amp;&amp;</div><div class='ctx'> 	    (ib_is_opa_gid(&amp;path-&gt;sgid))) {</div><div class='hunk'>@@ -1698,23 +1835,28 @@ static void cm_format_req_event(struct cm_work *work,</div><div class='ctx'> 	} else {</div><div class='ctx'> 		param-&gt;alternate_path = NULL;</div><div class='ctx'> 	}</div><div class='del'>-	param-&gt;remote_ca_guid = req_msg-&gt;local_ca_guid;</div><div class='del'>-	param-&gt;remote_qkey = be32_to_cpu(req_msg-&gt;local_qkey);</div><div class='del'>-	param-&gt;remote_qpn = be32_to_cpu(cm_req_get_local_qpn(req_msg));</div><div class='add'>+	param-&gt;remote_ca_guid =</div><div class='add'>+		cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));</div><div class='add'>+	param-&gt;remote_qkey = IBA_GET(CM_REQ_LOCAL_Q_KEY, req_msg);</div><div class='add'>+	param-&gt;remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg);</div><div class='ctx'> 	param-&gt;qp_type = cm_req_get_qp_type(req_msg);</div><div class='del'>-	param-&gt;starting_psn = be32_to_cpu(cm_req_get_starting_psn(req_msg));</div><div class='del'>-	param-&gt;responder_resources = cm_req_get_init_depth(req_msg);</div><div class='del'>-	param-&gt;initiator_depth = cm_req_get_resp_res(req_msg);</div><div class='add'>+	param-&gt;starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg);</div><div class='add'>+	param-&gt;responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);</div><div class='add'>+	param-&gt;initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);</div><div class='ctx'> 	param-&gt;local_cm_response_timeout =</div><div class='del'>-					cm_req_get_remote_resp_timeout(req_msg);</div><div class='del'>-	param-&gt;flow_control = cm_req_get_flow_ctrl(req_msg);</div><div class='add'>+		IBA_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg);</div><div class='add'>+	param-&gt;flow_control = IBA_GET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg);</div><div class='ctx'> 	param-&gt;remote_cm_response_timeout =</div><div class='del'>-					cm_req_get_local_resp_timeout(req_msg);</div><div class='del'>-	param-&gt;retry_count = cm_req_get_retry_count(req_msg);</div><div class='del'>-	param-&gt;rnr_retry_count = cm_req_get_rnr_retry_count(req_msg);</div><div class='del'>-	param-&gt;srq = cm_req_get_srq(req_msg);</div><div class='add'>+		IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg);</div><div class='add'>+	param-&gt;retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);</div><div class='add'>+	param-&gt;rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);</div><div class='add'>+	param-&gt;srq = IBA_GET(CM_REQ_SRQ, req_msg);</div><div class='ctx'> 	param-&gt;ppath_sgid_attr = cm_id_priv-&gt;av.ah_attr.grh.sgid_attr;</div><div class='del'>-	work-&gt;cm_event.private_data = &amp;req_msg-&gt;private_data;</div><div class='add'>+	param-&gt;ece.vendor_id = IBA_GET(CM_REQ_VENDOR_ID, req_msg);</div><div class='add'>+	param-&gt;ece.attr_mod = be32_to_cpu(req_msg-&gt;hdr.attr_mod);</div><div class='add'>+</div><div class='add'>+	work-&gt;cm_event.private_data =</div><div class='add'>+		IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void cm_process_work(struct cm_id_private *cm_id_priv,</div><div class='hunk'>@@ -1744,58 +1886,67 @@ static void cm_process_work(struct cm_id_private *cm_id_priv,</div><div class='ctx'> </div><div class='ctx'> static void cm_format_mra(struct cm_mra_msg *mra_msg,</div><div class='ctx'> 			  struct cm_id_private *cm_id_priv,</div><div class='del'>-			  enum cm_msg_response msg_mraed, u8 service_timeout,</div><div class='add'>+			  enum cm_msg_response msg_mraed,</div><div class='ctx'> 			  const void *private_data, u8 private_data_len)</div><div class='ctx'> {</div><div class='ctx'> 	cm_format_mad_hdr(&amp;mra_msg-&gt;hdr, CM_MRA_ATTR_ID, cm_id_priv-&gt;tid);</div><div class='del'>-	cm_mra_set_msg_mraed(mra_msg, msg_mraed);</div><div class='del'>-	mra_msg-&gt;local_comm_id = cm_id_priv-&gt;id.local_id;</div><div class='del'>-	mra_msg-&gt;remote_comm_id = cm_id_priv-&gt;id.remote_id;</div><div class='del'>-	cm_mra_set_service_timeout(mra_msg, service_timeout);</div><div class='add'>+	IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed);</div><div class='add'>+	IBA_SET(CM_MRA_LOCAL_COMM_ID, mra_msg,</div><div class='add'>+		be32_to_cpu(cm_id_priv-&gt;id.local_id));</div><div class='add'>+	IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg,</div><div class='add'>+		be32_to_cpu(cm_id_priv-&gt;id.remote_id));</div><div class='add'>+	IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, CM_MRA_SETTING);</div><div class='ctx'> </div><div class='ctx'> 	if (private_data &amp;&amp; private_data_len)</div><div class='del'>-		memcpy(mra_msg-&gt;private_data, private_data, private_data_len);</div><div class='add'>+		IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data,</div><div class='add'>+			    private_data_len);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void cm_format_rej(struct cm_rej_msg *rej_msg,</div><div class='ctx'> 			  struct cm_id_private *cm_id_priv,</div><div class='del'>-			  enum ib_cm_rej_reason reason,</div><div class='del'>-			  void *ari,</div><div class='del'>-			  u8 ari_length,</div><div class='del'>-			  const void *private_data,</div><div class='del'>-			  u8 private_data_len)</div><div class='add'>+			  enum ib_cm_rej_reason reason, void *ari,</div><div class='add'>+			  u8 ari_length, const void *private_data,</div><div class='add'>+			  u8 private_data_len, enum ib_cm_state state)</div><div class='ctx'> {</div><div class='add'>+	lockdep_assert_held(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+</div><div class='ctx'> 	cm_format_mad_hdr(&amp;rej_msg-&gt;hdr, CM_REJ_ATTR_ID, cm_id_priv-&gt;tid);</div><div class='del'>-	rej_msg-&gt;remote_comm_id = cm_id_priv-&gt;id.remote_id;</div><div class='add'>+	IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,</div><div class='add'>+		be32_to_cpu(cm_id_priv-&gt;id.remote_id));</div><div class='ctx'> </div><div class='del'>-	switch(cm_id_priv-&gt;id.state) {</div><div class='add'>+	switch (state) {</div><div class='ctx'> 	case IB_CM_REQ_RCVD:</div><div class='del'>-		rej_msg-&gt;local_comm_id = 0;</div><div class='del'>-		cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ);</div><div class='add'>+		IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, be32_to_cpu(0));</div><div class='add'>+		IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_MRA_REQ_SENT:</div><div class='del'>-		rej_msg-&gt;local_comm_id = cm_id_priv-&gt;id.local_id;</div><div class='del'>-		cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ);</div><div class='add'>+		IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,</div><div class='add'>+			be32_to_cpu(cm_id_priv-&gt;id.local_id));</div><div class='add'>+		IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_REP_RCVD:</div><div class='ctx'> 	case IB_CM_MRA_REP_SENT:</div><div class='del'>-		rej_msg-&gt;local_comm_id = cm_id_priv-&gt;id.local_id;</div><div class='del'>-		cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REP);</div><div class='add'>+		IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,</div><div class='add'>+			be32_to_cpu(cm_id_priv-&gt;id.local_id));</div><div class='add'>+		IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP);</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='del'>-		rej_msg-&gt;local_comm_id = cm_id_priv-&gt;id.local_id;</div><div class='del'>-		cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_OTHER);</div><div class='add'>+		IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,</div><div class='add'>+			be32_to_cpu(cm_id_priv-&gt;id.local_id));</div><div class='add'>+		IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg,</div><div class='add'>+			CM_MSG_RESPONSE_OTHER);</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	rej_msg-&gt;reason = cpu_to_be16(reason);</div><div class='add'>+	IBA_SET(CM_REJ_REASON, rej_msg, reason);</div><div class='ctx'> 	if (ari &amp;&amp; ari_length) {</div><div class='del'>-		cm_rej_set_reject_info_len(rej_msg, ari_length);</div><div class='del'>-		memcpy(rej_msg-&gt;ari, ari, ari_length);</div><div class='add'>+		IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);</div><div class='add'>+		IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (private_data &amp;&amp; private_data_len)</div><div class='del'>-		memcpy(rej_msg-&gt;private_data, private_data, private_data_len);</div><div class='add'>+		IBA_SET_MEM(CM_REJ_PRIVATE_DATA, rej_msg, private_data,</div><div class='add'>+			    private_data_len);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void cm_dup_req_handler(struct cm_work *work,</div><div class='hunk'>@@ -1804,14 +1955,18 @@ static void cm_dup_req_handler(struct cm_work *work,</div><div class='ctx'> 	struct ib_mad_send_buf *msg = NULL;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	atomic_long_inc(&amp;work-&gt;port-&gt;counter_group[CM_RECV_DUPLICATES].</div><div class='del'>-			counter[CM_REQ_COUNTER]);</div><div class='add'>+	atomic_long_inc(</div><div class='add'>+		&amp;work-&gt;port-&gt;counters[CM_RECV_DUPLICATES][CM_REQ_COUNTER]);</div><div class='ctx'> </div><div class='ctx'> 	/* Quick state check to discard duplicate REQs. */</div><div class='del'>-	if (cm_id_priv-&gt;id.state == IB_CM_REQ_RCVD)</div><div class='add'>+	spin_lock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+	if (cm_id_priv-&gt;id.state == IB_CM_REQ_RCVD) {</div><div class='add'>+		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> 		return;</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> </div><div class='del'>-	ret = cm_alloc_response_msg(work-&gt;port, work-&gt;mad_recv_wc, &amp;msg);</div><div class='add'>+	ret = cm_alloc_response_msg(work-&gt;port, work-&gt;mad_recv_wc, true, &amp;msg);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='hunk'>@@ -1819,19 +1974,21 @@ static void cm_dup_req_handler(struct cm_work *work,</div><div class='ctx'> 	switch (cm_id_priv-&gt;id.state) {</div><div class='ctx'> 	case IB_CM_MRA_REQ_SENT:</div><div class='ctx'> 		cm_format_mra((struct cm_mra_msg *) msg-&gt;mad, cm_id_priv,</div><div class='del'>-			      CM_MSG_RESPONSE_REQ, cm_id_priv-&gt;service_timeout,</div><div class='add'>+			      CM_MSG_RESPONSE_REQ,</div><div class='ctx'> 			      cm_id_priv-&gt;private_data,</div><div class='ctx'> 			      cm_id_priv-&gt;private_data_len);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_TIMEWAIT:</div><div class='del'>-		cm_format_rej((struct cm_rej_msg *) msg-&gt;mad, cm_id_priv,</div><div class='del'>-			      IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0);</div><div class='add'>+		cm_format_rej((struct cm_rej_msg *)msg-&gt;mad, cm_id_priv,</div><div class='add'>+			      IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0,</div><div class='add'>+			      IB_CM_TIMEWAIT);</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		goto unlock;</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> </div><div class='add'>+	trace_icm_send_dup_req(&amp;cm_id_priv-&gt;id);</div><div class='ctx'> 	ret = ib_post_send_mad(msg, NULL);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto free;</div><div class='hunk'>@@ -1841,13 +1998,12 @@ unlock:	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> free:	cm_free_msg(msg);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct cm_id_private * cm_match_req(struct cm_work *work,</div><div class='del'>-					   struct cm_id_private *cm_id_priv)</div><div class='add'>+static struct cm_id_private *cm_match_req(struct cm_work *work,</div><div class='add'>+					  struct cm_id_private *cm_id_priv)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;</div><div class='ctx'> 	struct cm_timewait_info *timewait_info;</div><div class='ctx'> 	struct cm_req_msg *req_msg;</div><div class='del'>-	struct ib_cm_id *cm_id;</div><div class='ctx'> </div><div class='ctx'> 	req_msg = (struct cm_req_msg *)work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='ctx'> </div><div class='hunk'>@@ -1855,7 +2011,7 @@ static struct cm_id_private * cm_match_req(struct cm_work *work,</div><div class='ctx'> 	spin_lock_irq(&amp;cm.lock);</div><div class='ctx'> 	timewait_info = cm_insert_remote_id(cm_id_priv-&gt;timewait_info);</div><div class='ctx'> 	if (timewait_info) {</div><div class='del'>-		cur_cm_id_priv = cm_get_id(timewait_info-&gt;work.local_id,</div><div class='add'>+		cur_cm_id_priv = cm_acquire_id(timewait_info-&gt;work.local_id,</div><div class='ctx'> 					   timewait_info-&gt;work.remote_id);</div><div class='ctx'> 		spin_unlock_irq(&amp;cm.lock);</div><div class='ctx'> 		if (cur_cm_id_priv) {</div><div class='hunk'>@@ -1868,8 +2024,8 @@ static struct cm_id_private * cm_match_req(struct cm_work *work,</div><div class='ctx'> 	/* Check for stale connections. */</div><div class='ctx'> 	timewait_info = cm_insert_remote_qpn(cm_id_priv-&gt;timewait_info);</div><div class='ctx'> 	if (timewait_info) {</div><div class='del'>-		cm_cleanup_timewait(cm_id_priv-&gt;timewait_info);</div><div class='del'>-		cur_cm_id_priv = cm_get_id(timewait_info-&gt;work.local_id,</div><div class='add'>+		cm_remove_remote(cm_id_priv);</div><div class='add'>+		cur_cm_id_priv = cm_acquire_id(timewait_info-&gt;work.local_id,</div><div class='ctx'> 					   timewait_info-&gt;work.remote_id);</div><div class='ctx'> </div><div class='ctx'> 		spin_unlock_irq(&amp;cm.lock);</div><div class='hunk'>@@ -1877,30 +2033,25 @@ static struct cm_id_private * cm_match_req(struct cm_work *work,</div><div class='ctx'> 			     IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,</div><div class='ctx'> 			     NULL, 0);</div><div class='ctx'> 		if (cur_cm_id_priv) {</div><div class='del'>-			cm_id = &amp;cur_cm_id_priv-&gt;id;</div><div class='del'>-			ib_send_cm_dreq(cm_id, NULL, 0);</div><div class='add'>+			ib_send_cm_dreq(&amp;cur_cm_id_priv-&gt;id, NULL, 0);</div><div class='ctx'> 			cm_deref_id(cur_cm_id_priv);</div><div class='ctx'> 		}</div><div class='ctx'> 		return NULL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Find matching listen request. */</div><div class='del'>-	listen_cm_id_priv = cm_find_listen(cm_id_priv-&gt;id.device,</div><div class='del'>-					   req_msg-&gt;service_id);</div><div class='add'>+	listen_cm_id_priv = cm_find_listen(</div><div class='add'>+		cm_id_priv-&gt;id.device,</div><div class='add'>+		cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)));</div><div class='ctx'> 	if (!listen_cm_id_priv) {</div><div class='del'>-		cm_cleanup_timewait(cm_id_priv-&gt;timewait_info);</div><div class='add'>+		cm_remove_remote(cm_id_priv);</div><div class='ctx'> 		spin_unlock_irq(&amp;cm.lock);</div><div class='ctx'> 		cm_issue_rej(work-&gt;port, work-&gt;mad_recv_wc,</div><div class='ctx'> 			     IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,</div><div class='ctx'> 			     NULL, 0);</div><div class='del'>-		goto out;</div><div class='add'>+		return NULL;</div><div class='ctx'> 	}</div><div class='del'>-	atomic_inc(&amp;listen_cm_id_priv-&gt;refcount);</div><div class='del'>-	atomic_inc(&amp;cm_id_priv-&gt;refcount);</div><div class='del'>-	cm_id_priv-&gt;id.state = IB_CM_REQ_RCVD;</div><div class='del'>-	atomic_inc(&amp;cm_id_priv-&gt;work_count);</div><div class='ctx'> 	spin_unlock_irq(&amp;cm.lock);</div><div class='del'>-out:</div><div class='ctx'> 	return listen_cm_id_priv;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1911,30 +2062,37 @@ out:</div><div class='ctx'>  */</div><div class='ctx'> static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)</div><div class='ctx'> {</div><div class='del'>-	if (!cm_req_get_primary_subnet_local(req_msg)) {</div><div class='del'>-		if (req_msg-&gt;primary_local_lid == IB_LID_PERMISSIVE) {</div><div class='del'>-			req_msg-&gt;primary_local_lid = ib_lid_be16(wc-&gt;slid);</div><div class='del'>-			cm_req_set_primary_sl(req_msg, wc-&gt;sl);</div><div class='add'>+	if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) {</div><div class='add'>+		if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID,</div><div class='add'>+					req_msg)) == IB_LID_PERMISSIVE) {</div><div class='add'>+			IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,</div><div class='add'>+				be16_to_cpu(ib_lid_be16(wc-&gt;slid)));</div><div class='add'>+			IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc-&gt;sl);</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		if (req_msg-&gt;primary_remote_lid == IB_LID_PERMISSIVE)</div><div class='del'>-			req_msg-&gt;primary_remote_lid = cpu_to_be16(wc-&gt;dlid_path_bits);</div><div class='add'>+		if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,</div><div class='add'>+					req_msg)) == IB_LID_PERMISSIVE)</div><div class='add'>+			IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,</div><div class='add'>+				wc-&gt;dlid_path_bits);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!cm_req_get_alt_subnet_local(req_msg)) {</div><div class='del'>-		if (req_msg-&gt;alt_local_lid == IB_LID_PERMISSIVE) {</div><div class='del'>-			req_msg-&gt;alt_local_lid = ib_lid_be16(wc-&gt;slid);</div><div class='del'>-			cm_req_set_alt_sl(req_msg, wc-&gt;sl);</div><div class='add'>+	if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) {</div><div class='add'>+		if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,</div><div class='add'>+					req_msg)) == IB_LID_PERMISSIVE) {</div><div class='add'>+			IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,</div><div class='add'>+				be16_to_cpu(ib_lid_be16(wc-&gt;slid)));</div><div class='add'>+			IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc-&gt;sl);</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		if (req_msg-&gt;alt_remote_lid == IB_LID_PERMISSIVE)</div><div class='del'>-			req_msg-&gt;alt_remote_lid = cpu_to_be16(wc-&gt;dlid_path_bits);</div><div class='add'>+		if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,</div><div class='add'>+					req_msg)) == IB_LID_PERMISSIVE)</div><div class='add'>+			IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,</div><div class='add'>+				wc-&gt;dlid_path_bits);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int cm_req_handler(struct cm_work *work)</div><div class='ctx'> {</div><div class='del'>-	struct ib_cm_id *cm_id;</div><div class='ctx'> 	struct cm_id_private *cm_id_priv, *listen_cm_id_priv;</div><div class='ctx'> 	struct cm_req_msg *req_msg;</div><div class='ctx'> 	const struct ib_global_route *grh;</div><div class='hunk'>@@ -1943,12 +2101,32 @@ static int cm_req_handler(struct cm_work *work)</div><div class='ctx'> </div><div class='ctx'> 	req_msg = (struct cm_req_msg *)work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='ctx'> </div><div class='del'>-	cm_id = ib_create_cm_id(work-&gt;port-&gt;cm_dev-&gt;ib_device, NULL, NULL);</div><div class='del'>-	if (IS_ERR(cm_id))</div><div class='del'>-		return PTR_ERR(cm_id);</div><div class='add'>+	cm_id_priv =</div><div class='add'>+		cm_alloc_id_priv(work-&gt;port-&gt;cm_dev-&gt;ib_device, NULL, NULL);</div><div class='add'>+	if (IS_ERR(cm_id_priv))</div><div class='add'>+		return PTR_ERR(cm_id_priv);</div><div class='add'>+</div><div class='add'>+	cm_id_priv-&gt;id.remote_id =</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg));</div><div class='add'>+	cm_id_priv-&gt;id.service_id =</div><div class='add'>+		cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));</div><div class='add'>+	cm_id_priv-&gt;tid = req_msg-&gt;hdr.tid;</div><div class='add'>+	cm_id_priv-&gt;timeout_ms = cm_convert_to_ms(</div><div class='add'>+		IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg));</div><div class='add'>+	cm_id_priv-&gt;max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg);</div><div class='add'>+	cm_id_priv-&gt;remote_qpn =</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));</div><div class='add'>+	cm_id_priv-&gt;initiator_depth =</div><div class='add'>+		IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);</div><div class='add'>+	cm_id_priv-&gt;responder_resources =</div><div class='add'>+		IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);</div><div class='add'>+	cm_id_priv-&gt;path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);</div><div class='add'>+	cm_id_priv-&gt;pkey = cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));</div><div class='add'>+	cm_id_priv-&gt;sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));</div><div class='add'>+	cm_id_priv-&gt;retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);</div><div class='add'>+	cm_id_priv-&gt;rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);</div><div class='add'>+	cm_id_priv-&gt;qp_type = cm_req_get_qp_type(req_msg);</div><div class='ctx'> </div><div class='del'>-	cm_id_priv = container_of(cm_id, struct cm_id_private, id);</div><div class='del'>-	cm_id_priv-&gt;id.remote_id = req_msg-&gt;local_comm_id;</div><div class='ctx'> 	ret = cm_init_av_for_response(work-&gt;port, work-&gt;mad_recv_wc-&gt;wc,</div><div class='ctx'> 				      work-&gt;mad_recv_wc-&gt;recv_buf.grh,</div><div class='ctx'> 				      &amp;cm_id_priv-&gt;av);</div><div class='hunk'>@@ -1958,54 +2136,57 @@ static int cm_req_handler(struct cm_work *work)</div><div class='ctx'> 							    id.local_id);</div><div class='ctx'> 	if (IS_ERR(cm_id_priv-&gt;timewait_info)) {</div><div class='ctx'> 		ret = PTR_ERR(cm_id_priv-&gt;timewait_info);</div><div class='add'>+		cm_id_priv-&gt;timewait_info = NULL;</div><div class='ctx'> 		goto destroy;</div><div class='ctx'> 	}</div><div class='del'>-	cm_id_priv-&gt;timewait_info-&gt;work.remote_id = req_msg-&gt;local_comm_id;</div><div class='del'>-	cm_id_priv-&gt;timewait_info-&gt;remote_ca_guid = req_msg-&gt;local_ca_guid;</div><div class='del'>-	cm_id_priv-&gt;timewait_info-&gt;remote_qpn = cm_req_get_local_qpn(req_msg);</div><div class='add'>+	cm_id_priv-&gt;timewait_info-&gt;work.remote_id = cm_id_priv-&gt;id.remote_id;</div><div class='add'>+	cm_id_priv-&gt;timewait_info-&gt;remote_ca_guid =</div><div class='add'>+		cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));</div><div class='add'>+	cm_id_priv-&gt;timewait_info-&gt;remote_qpn = cm_id_priv-&gt;remote_qpn;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Note that the ID pointer is not in the xarray at this point,</div><div class='add'>+	 * so this set is only visible to the local thread.</div><div class='add'>+	 */</div><div class='add'>+	cm_id_priv-&gt;id.state = IB_CM_REQ_RCVD;</div><div class='ctx'> </div><div class='ctx'> 	listen_cm_id_priv = cm_match_req(work, cm_id_priv);</div><div class='ctx'> 	if (!listen_cm_id_priv) {</div><div class='del'>-		pr_debug("%s: local_id %d, no listen_cm_id_priv\n", __func__,</div><div class='del'>-			 be32_to_cpu(cm_id-&gt;local_id));</div><div class='add'>+		trace_icm_no_listener_err(&amp;cm_id_priv-&gt;id);</div><div class='add'>+		cm_id_priv-&gt;id.state = IB_CM_IDLE;</div><div class='ctx'> 		ret = -EINVAL;</div><div class='del'>-		goto free_timeinfo;</div><div class='add'>+		goto destroy;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	cm_id_priv-&gt;id.cm_handler = listen_cm_id_priv-&gt;id.cm_handler;</div><div class='del'>-	cm_id_priv-&gt;id.context = listen_cm_id_priv-&gt;id.context;</div><div class='del'>-	cm_id_priv-&gt;id.service_id = req_msg-&gt;service_id;</div><div class='del'>-	cm_id_priv-&gt;id.service_mask = ~cpu_to_be64(0);</div><div class='del'>-</div><div class='del'>-	cm_process_routed_req(req_msg, work-&gt;mad_recv_wc-&gt;wc);</div><div class='del'>-</div><div class='ctx'> 	memset(&amp;work-&gt;path[0], 0, sizeof(work-&gt;path[0]));</div><div class='ctx'> 	if (cm_req_has_alt_path(req_msg))</div><div class='ctx'> 		memset(&amp;work-&gt;path[1], 0, sizeof(work-&gt;path[1]));</div><div class='ctx'> 	grh = rdma_ah_read_grh(&amp;cm_id_priv-&gt;av.ah_attr);</div><div class='ctx'> 	gid_attr = grh-&gt;sgid_attr;</div><div class='ctx'> </div><div class='del'>-	if (gid_attr &amp;&amp;</div><div class='del'>-	    rdma_protocol_roce(work-&gt;port-&gt;cm_dev-&gt;ib_device,</div><div class='del'>-			       work-&gt;port-&gt;port_num)) {</div><div class='add'>+	if (cm_id_priv-&gt;av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE) {</div><div class='ctx'> 		work-&gt;path[0].rec_type =</div><div class='ctx'> 			sa_conv_gid_to_pathrec_type(gid_attr-&gt;gid_type);</div><div class='ctx'> 	} else {</div><div class='del'>-		cm_path_set_rec_type(work-&gt;port-&gt;cm_dev-&gt;ib_device,</div><div class='del'>-				     work-&gt;port-&gt;port_num,</div><div class='del'>-				     &amp;work-&gt;path[0],</div><div class='del'>-				     &amp;req_msg-&gt;primary_local_gid);</div><div class='add'>+		cm_process_routed_req(req_msg, work-&gt;mad_recv_wc-&gt;wc);</div><div class='add'>+		cm_path_set_rec_type(</div><div class='add'>+			work-&gt;port-&gt;cm_dev-&gt;ib_device, work-&gt;port-&gt;port_num,</div><div class='add'>+			&amp;work-&gt;path[0],</div><div class='add'>+			IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID,</div><div class='add'>+					req_msg));</div><div class='ctx'> 	}</div><div class='ctx'> 	if (cm_req_has_alt_path(req_msg))</div><div class='ctx'> 		work-&gt;path[1].rec_type = work-&gt;path[0].rec_type;</div><div class='ctx'> 	cm_format_paths_from_req(req_msg, &amp;work-&gt;path[0],</div><div class='del'>-				 &amp;work-&gt;path[1]);</div><div class='add'>+				 &amp;work-&gt;path[1], work-&gt;mad_recv_wc-&gt;wc);</div><div class='ctx'> 	if (cm_id_priv-&gt;av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)</div><div class='ctx'> 		sa_path_set_dmac(&amp;work-&gt;path[0],</div><div class='ctx'> 				 cm_id_priv-&gt;av.ah_attr.roce.dmac);</div><div class='ctx'> 	work-&gt;path[0].hop_limit = grh-&gt;hop_limit;</div><div class='del'>-	ret = cm_init_av_by_path(&amp;work-&gt;path[0], gid_attr, &amp;cm_id_priv-&gt;av,</div><div class='del'>-				 cm_id_priv);</div><div class='add'>+</div><div class='add'>+	/* This destroy call is needed to pair with cm_init_av_for_response */</div><div class='add'>+	cm_destroy_av(&amp;cm_id_priv-&gt;av);</div><div class='add'>+	ret = cm_init_av_by_path(&amp;work-&gt;path[0], gid_attr, &amp;cm_id_priv-&gt;av);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		int err;</div><div class='ctx'> </div><div class='hunk'>@@ -2013,51 +2194,55 @@ static int cm_req_handler(struct cm_work *work)</div><div class='ctx'> 				     work-&gt;port-&gt;port_num, 0,</div><div class='ctx'> 				     &amp;work-&gt;path[0].sgid);</div><div class='ctx'> 		if (err)</div><div class='del'>-			ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,</div><div class='add'>+			ib_send_cm_rej(&amp;cm_id_priv-&gt;id, IB_CM_REJ_INVALID_GID,</div><div class='ctx'> 				       NULL, 0, NULL, 0);</div><div class='ctx'> 		else</div><div class='del'>-			ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,</div><div class='add'>+			ib_send_cm_rej(&amp;cm_id_priv-&gt;id, IB_CM_REJ_INVALID_GID,</div><div class='ctx'> 				       &amp;work-&gt;path[0].sgid,</div><div class='ctx'> 				       sizeof(work-&gt;path[0].sgid),</div><div class='ctx'> 				       NULL, 0);</div><div class='ctx'> 		goto rejected;</div><div class='ctx'> 	}</div><div class='add'>+	if (cm_id_priv-&gt;av.ah_attr.type == RDMA_AH_ATTR_TYPE_IB)</div><div class='add'>+		cm_id_priv-&gt;av.dlid_datapath =</div><div class='add'>+			IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg);</div><div class='add'>+</div><div class='ctx'> 	if (cm_req_has_alt_path(req_msg)) {</div><div class='ctx'> 		ret = cm_init_av_by_path(&amp;work-&gt;path[1], NULL,</div><div class='del'>-					 &amp;cm_id_priv-&gt;alt_av, cm_id_priv);</div><div class='add'>+					 &amp;cm_id_priv-&gt;alt_av);</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_ALT_GID,</div><div class='add'>+			ib_send_cm_rej(&amp;cm_id_priv-&gt;id,</div><div class='add'>+				       IB_CM_REJ_INVALID_ALT_GID,</div><div class='ctx'> 				       &amp;work-&gt;path[0].sgid,</div><div class='ctx'> 				       sizeof(work-&gt;path[0].sgid), NULL, 0);</div><div class='ctx'> 			goto rejected;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='del'>-	cm_id_priv-&gt;tid = req_msg-&gt;hdr.tid;</div><div class='del'>-	cm_id_priv-&gt;timeout_ms = cm_convert_to_ms(</div><div class='del'>-					cm_req_get_local_resp_timeout(req_msg));</div><div class='del'>-	cm_id_priv-&gt;max_cm_retries = cm_req_get_max_cm_retries(req_msg);</div><div class='del'>-	cm_id_priv-&gt;remote_qpn = cm_req_get_local_qpn(req_msg);</div><div class='del'>-	cm_id_priv-&gt;initiator_depth = cm_req_get_resp_res(req_msg);</div><div class='del'>-	cm_id_priv-&gt;responder_resources = cm_req_get_init_depth(req_msg);</div><div class='del'>-	cm_id_priv-&gt;path_mtu = cm_req_get_path_mtu(req_msg);</div><div class='del'>-	cm_id_priv-&gt;pkey = req_msg-&gt;pkey;</div><div class='del'>-	cm_id_priv-&gt;sq_psn = cm_req_get_starting_psn(req_msg);</div><div class='del'>-	cm_id_priv-&gt;retry_count = cm_req_get_retry_count(req_msg);</div><div class='del'>-	cm_id_priv-&gt;rnr_retry_count = cm_req_get_rnr_retry_count(req_msg);</div><div class='del'>-	cm_id_priv-&gt;qp_type = cm_req_get_qp_type(req_msg);</div><div class='ctx'> </div><div class='add'>+	cm_id_priv-&gt;id.cm_handler = listen_cm_id_priv-&gt;id.cm_handler;</div><div class='add'>+	cm_id_priv-&gt;id.context = listen_cm_id_priv-&gt;id.context;</div><div class='ctx'> 	cm_format_req_event(work, cm_id_priv, &amp;listen_cm_id_priv-&gt;id);</div><div class='del'>-	cm_process_work(cm_id_priv, work);</div><div class='add'>+</div><div class='add'>+	/* Now MAD handlers can see the new ID */</div><div class='add'>+	spin_lock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+	cm_finalize_id(cm_id_priv);</div><div class='add'>+</div><div class='add'>+	/* Refcount belongs to the event, pairs with cm_process_work() */</div><div class='add'>+	refcount_inc(&amp;cm_id_priv-&gt;refcount);</div><div class='add'>+	cm_queue_work_unlock(cm_id_priv, work);</div><div class='add'>+	/*</div><div class='add'>+	 * Since this ID was just created and was not made visible to other MAD</div><div class='add'>+	 * handlers until the cm_finalize_id() above we know that the</div><div class='add'>+	 * cm_process_work() will deliver the event and the listen_cm_id</div><div class='add'>+	 * embedded in the event can be derefed here.</div><div class='add'>+	 */</div><div class='ctx'> 	cm_deref_id(listen_cm_id_priv);</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> rejected:</div><div class='del'>-	atomic_dec(&amp;cm_id_priv-&gt;refcount);</div><div class='ctx'> 	cm_deref_id(listen_cm_id_priv);</div><div class='del'>-free_timeinfo:</div><div class='del'>-	kfree(cm_id_priv-&gt;timewait_info);</div><div class='ctx'> destroy:</div><div class='del'>-	ib_destroy_cm_id(cm_id);</div><div class='add'>+	ib_destroy_cm_id(&amp;cm_id_priv-&gt;id);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -2065,30 +2250,41 @@ static void cm_format_rep(struct cm_rep_msg *rep_msg,</div><div class='ctx'> 			  struct cm_id_private *cm_id_priv,</div><div class='ctx'> 			  struct ib_cm_rep_param *param)</div><div class='ctx'> {</div><div class='del'>-	cm_format_mad_hdr(&amp;rep_msg-&gt;hdr, CM_REP_ATTR_ID, cm_id_priv-&gt;tid);</div><div class='del'>-	rep_msg-&gt;local_comm_id = cm_id_priv-&gt;id.local_id;</div><div class='del'>-	rep_msg-&gt;remote_comm_id = cm_id_priv-&gt;id.remote_id;</div><div class='del'>-	cm_rep_set_starting_psn(rep_msg, cpu_to_be32(param-&gt;starting_psn));</div><div class='del'>-	rep_msg-&gt;resp_resources = param-&gt;responder_resources;</div><div class='del'>-	cm_rep_set_target_ack_delay(rep_msg,</div><div class='del'>-				    cm_id_priv-&gt;av.port-&gt;cm_dev-&gt;ack_delay);</div><div class='del'>-	cm_rep_set_failover(rep_msg, param-&gt;failover_accepted);</div><div class='del'>-	cm_rep_set_rnr_retry_count(rep_msg, param-&gt;rnr_retry_count);</div><div class='del'>-	rep_msg-&gt;local_ca_guid = cm_id_priv-&gt;id.device-&gt;node_guid;</div><div class='add'>+	cm_format_mad_ece_hdr(&amp;rep_msg-&gt;hdr, CM_REP_ATTR_ID, cm_id_priv-&gt;tid,</div><div class='add'>+			      param-&gt;ece.attr_mod);</div><div class='add'>+	IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg,</div><div class='add'>+		be32_to_cpu(cm_id_priv-&gt;id.local_id));</div><div class='add'>+	IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg,</div><div class='add'>+		be32_to_cpu(cm_id_priv-&gt;id.remote_id));</div><div class='add'>+	IBA_SET(CM_REP_STARTING_PSN, rep_msg, param-&gt;starting_psn);</div><div class='add'>+	IBA_SET(CM_REP_RESPONDER_RESOURCES, rep_msg,</div><div class='add'>+		param-&gt;responder_resources);</div><div class='add'>+	IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg,</div><div class='add'>+		cm_id_priv-&gt;av.port-&gt;cm_dev-&gt;ack_delay);</div><div class='add'>+	IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param-&gt;failover_accepted);</div><div class='add'>+	IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param-&gt;rnr_retry_count);</div><div class='add'>+	IBA_SET(CM_REP_LOCAL_CA_GUID, rep_msg,</div><div class='add'>+		be64_to_cpu(cm_id_priv-&gt;id.device-&gt;node_guid));</div><div class='ctx'> </div><div class='ctx'> 	if (cm_id_priv-&gt;qp_type != IB_QPT_XRC_TGT) {</div><div class='del'>-		rep_msg-&gt;initiator_depth = param-&gt;initiator_depth;</div><div class='del'>-		cm_rep_set_flow_ctrl(rep_msg, param-&gt;flow_control);</div><div class='del'>-		cm_rep_set_srq(rep_msg, param-&gt;srq);</div><div class='del'>-		cm_rep_set_local_qpn(rep_msg, cpu_to_be32(param-&gt;qp_num));</div><div class='add'>+		IBA_SET(CM_REP_INITIATOR_DEPTH, rep_msg,</div><div class='add'>+			param-&gt;initiator_depth);</div><div class='add'>+		IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg,</div><div class='add'>+			param-&gt;flow_control);</div><div class='add'>+		IBA_SET(CM_REP_SRQ, rep_msg, param-&gt;srq);</div><div class='add'>+		IBA_SET(CM_REP_LOCAL_QPN, rep_msg, param-&gt;qp_num);</div><div class='ctx'> 	} else {</div><div class='del'>-		cm_rep_set_srq(rep_msg, 1);</div><div class='del'>-		cm_rep_set_local_eecn(rep_msg, cpu_to_be32(param-&gt;qp_num));</div><div class='add'>+		IBA_SET(CM_REP_SRQ, rep_msg, 1);</div><div class='add'>+		IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param-&gt;qp_num);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	IBA_SET(CM_REP_VENDOR_ID_L, rep_msg, param-&gt;ece.vendor_id);</div><div class='add'>+	IBA_SET(CM_REP_VENDOR_ID_M, rep_msg, param-&gt;ece.vendor_id &gt;&gt; 8);</div><div class='add'>+	IBA_SET(CM_REP_VENDOR_ID_H, rep_msg, param-&gt;ece.vendor_id &gt;&gt; 16);</div><div class='add'>+</div><div class='ctx'> 	if (param-&gt;private_data &amp;&amp; param-&gt;private_data_len)</div><div class='del'>-		memcpy(rep_msg-&gt;private_data, param-&gt;private_data,</div><div class='del'>-		       param-&gt;private_data_len);</div><div class='add'>+		IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param-&gt;private_data,</div><div class='add'>+			    param-&gt;private_data_len);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int ib_send_cm_rep(struct ib_cm_id *cm_id,</div><div class='hunk'>@@ -2108,36 +2304,40 @@ int ib_send_cm_rep(struct ib_cm_id *cm_id,</div><div class='ctx'> 	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 	if (cm_id-&gt;state != IB_CM_REQ_RCVD &amp;&amp;</div><div class='ctx'> 	    cm_id-&gt;state != IB_CM_MRA_REQ_SENT) {</div><div class='del'>-		pr_debug("%s: local_comm_id %d, cm_id-&gt;state: %d\n", __func__,</div><div class='del'>-			 be32_to_cpu(cm_id_priv-&gt;id.local_id), cm_id-&gt;state);</div><div class='add'>+		trace_icm_send_rep_err(cm_id_priv-&gt;id.local_id, cm_id-&gt;state);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = cm_alloc_msg(cm_id_priv, &amp;msg);</div><div class='del'>-	if (ret)</div><div class='add'>+	msg = cm_alloc_priv_msg_rep(cm_id_priv, IB_CM_REP_SENT, true);</div><div class='add'>+	if (IS_ERR(msg)) {</div><div class='add'>+		ret = PTR_ERR(msg);</div><div class='ctx'> 		goto out;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	rep_msg = (struct cm_rep_msg *) msg-&gt;mad;</div><div class='ctx'> 	cm_format_rep(rep_msg, cm_id_priv, param);</div><div class='del'>-	msg-&gt;timeout_ms = cm_id_priv-&gt;timeout_ms;</div><div class='del'>-	msg-&gt;context[1] = (void *) (unsigned long) IB_CM_REP_SENT;</div><div class='ctx'> </div><div class='add'>+	trace_icm_send_rep(cm_id);</div><div class='ctx'> 	ret = ib_post_send_mad(msg, NULL);</div><div class='del'>-	if (ret) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-		cm_free_msg(msg);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out_free;</div><div class='ctx'> </div><div class='ctx'> 	cm_id-&gt;state = IB_CM_REP_SENT;</div><div class='del'>-	cm_id_priv-&gt;msg = msg;</div><div class='ctx'> 	cm_id_priv-&gt;initiator_depth = param-&gt;initiator_depth;</div><div class='ctx'> 	cm_id_priv-&gt;responder_resources = param-&gt;responder_resources;</div><div class='del'>-	cm_id_priv-&gt;rq_psn = cm_rep_get_starting_psn(rep_msg);</div><div class='add'>+	cm_id_priv-&gt;rq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));</div><div class='add'>+	WARN_ONCE(param-&gt;qp_num &amp; 0xFF000000,</div><div class='add'>+		  "IBTA declares QPN to be 24 bits, but it is 0x%X\n",</div><div class='add'>+		  param-&gt;qp_num);</div><div class='ctx'> 	cm_id_priv-&gt;local_qpn = cpu_to_be32(param-&gt;qp_num &amp; 0xFFFFFF);</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-out:	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+out_free:</div><div class='add'>+	cm_free_priv_msg(msg);</div><div class='add'>+out:</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_send_cm_rep);</div><div class='hunk'>@@ -2148,11 +2348,14 @@ static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,</div><div class='ctx'> 			  u8 private_data_len)</div><div class='ctx'> {</div><div class='ctx'> 	cm_format_mad_hdr(&amp;rtu_msg-&gt;hdr, CM_RTU_ATTR_ID, cm_id_priv-&gt;tid);</div><div class='del'>-	rtu_msg-&gt;local_comm_id = cm_id_priv-&gt;id.local_id;</div><div class='del'>-	rtu_msg-&gt;remote_comm_id = cm_id_priv-&gt;id.remote_id;</div><div class='add'>+	IBA_SET(CM_RTU_LOCAL_COMM_ID, rtu_msg,</div><div class='add'>+		be32_to_cpu(cm_id_priv-&gt;id.local_id));</div><div class='add'>+	IBA_SET(CM_RTU_REMOTE_COMM_ID, rtu_msg,</div><div class='add'>+		be32_to_cpu(cm_id_priv-&gt;id.remote_id));</div><div class='ctx'> </div><div class='ctx'> 	if (private_data &amp;&amp; private_data_len)</div><div class='del'>-		memcpy(rtu_msg-&gt;private_data, private_data, private_data_len);</div><div class='add'>+		IBA_SET_MEM(CM_RTU_PRIVATE_DATA, rtu_msg, private_data,</div><div class='add'>+			    private_data_len);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int ib_send_cm_rtu(struct ib_cm_id *cm_id,</div><div class='hunk'>@@ -2176,19 +2379,21 @@ int ib_send_cm_rtu(struct ib_cm_id *cm_id,</div><div class='ctx'> 	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 	if (cm_id-&gt;state != IB_CM_REP_RCVD &amp;&amp;</div><div class='ctx'> 	    cm_id-&gt;state != IB_CM_MRA_REP_SENT) {</div><div class='del'>-		pr_debug("%s: local_id %d, cm_id-&gt;state %d\n", __func__,</div><div class='del'>-			 be32_to_cpu(cm_id-&gt;local_id), cm_id-&gt;state);</div><div class='add'>+		trace_icm_send_cm_rtu_err(cm_id);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		goto error;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = cm_alloc_msg(cm_id_priv, &amp;msg);</div><div class='del'>-	if (ret)</div><div class='add'>+	msg = cm_alloc_msg(cm_id_priv);</div><div class='add'>+	if (IS_ERR(msg)) {</div><div class='add'>+		ret = PTR_ERR(msg);</div><div class='ctx'> 		goto error;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	cm_format_rtu((struct cm_rtu_msg *) msg-&gt;mad, cm_id_priv,</div><div class='ctx'> 		      private_data, private_data_len);</div><div class='ctx'> </div><div class='add'>+	trace_icm_send_rtu(cm_id);</div><div class='ctx'> 	ret = ib_post_send_mad(msg, NULL);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='hunk'>@@ -2215,18 +2420,25 @@ static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)</div><div class='ctx'> </div><div class='ctx'> 	rep_msg = (struct cm_rep_msg *)work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='ctx'> 	param = &amp;work-&gt;cm_event.param.rep_rcvd;</div><div class='del'>-	param-&gt;remote_ca_guid = rep_msg-&gt;local_ca_guid;</div><div class='del'>-	param-&gt;remote_qkey = be32_to_cpu(rep_msg-&gt;local_qkey);</div><div class='add'>+	param-&gt;remote_ca_guid =</div><div class='add'>+		cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));</div><div class='add'>+	param-&gt;remote_qkey = IBA_GET(CM_REP_LOCAL_Q_KEY, rep_msg);</div><div class='ctx'> 	param-&gt;remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type));</div><div class='del'>-	param-&gt;starting_psn = be32_to_cpu(cm_rep_get_starting_psn(rep_msg));</div><div class='del'>-	param-&gt;responder_resources = rep_msg-&gt;initiator_depth;</div><div class='del'>-	param-&gt;initiator_depth = rep_msg-&gt;resp_resources;</div><div class='del'>-	param-&gt;target_ack_delay = cm_rep_get_target_ack_delay(rep_msg);</div><div class='del'>-	param-&gt;failover_accepted = cm_rep_get_failover(rep_msg);</div><div class='del'>-	param-&gt;flow_control = cm_rep_get_flow_ctrl(rep_msg);</div><div class='del'>-	param-&gt;rnr_retry_count = cm_rep_get_rnr_retry_count(rep_msg);</div><div class='del'>-	param-&gt;srq = cm_rep_get_srq(rep_msg);</div><div class='del'>-	work-&gt;cm_event.private_data = &amp;rep_msg-&gt;private_data;</div><div class='add'>+	param-&gt;starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg);</div><div class='add'>+	param-&gt;responder_resources = IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);</div><div class='add'>+	param-&gt;initiator_depth = IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);</div><div class='add'>+	param-&gt;target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);</div><div class='add'>+	param-&gt;failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg);</div><div class='add'>+	param-&gt;flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg);</div><div class='add'>+	param-&gt;rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);</div><div class='add'>+	param-&gt;srq = IBA_GET(CM_REP_SRQ, rep_msg);</div><div class='add'>+	param-&gt;ece.vendor_id = IBA_GET(CM_REP_VENDOR_ID_H, rep_msg) &lt;&lt; 16;</div><div class='add'>+	param-&gt;ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_M, rep_msg) &lt;&lt; 8;</div><div class='add'>+	param-&gt;ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_L, rep_msg);</div><div class='add'>+	param-&gt;ece.attr_mod = be32_to_cpu(rep_msg-&gt;hdr.attr_mod);</div><div class='add'>+</div><div class='add'>+	work-&gt;cm_event.private_data =</div><div class='add'>+		IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void cm_dup_rep_handler(struct cm_work *work)</div><div class='hunk'>@@ -2237,14 +2449,15 @@ static void cm_dup_rep_handler(struct cm_work *work)</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	rep_msg = (struct cm_rep_msg *) work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='del'>-	cm_id_priv = cm_acquire_id(rep_msg-&gt;remote_comm_id,</div><div class='del'>-				   rep_msg-&gt;local_comm_id);</div><div class='add'>+	cm_id_priv = cm_acquire_id(</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)),</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)));</div><div class='ctx'> 	if (!cm_id_priv)</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='del'>-	atomic_long_inc(&amp;work-&gt;port-&gt;counter_group[CM_RECV_DUPLICATES].</div><div class='del'>-			counter[CM_REP_COUNTER]);</div><div class='del'>-	ret = cm_alloc_response_msg(work-&gt;port, work-&gt;mad_recv_wc, &amp;msg);</div><div class='add'>+	atomic_long_inc(</div><div class='add'>+		&amp;work-&gt;port-&gt;counters[CM_RECV_DUPLICATES][CM_REP_COUNTER]);</div><div class='add'>+	ret = cm_alloc_response_msg(work-&gt;port, work-&gt;mad_recv_wc, true, &amp;msg);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto deref;</div><div class='ctx'> </div><div class='hunk'>@@ -2255,13 +2468,14 @@ static void cm_dup_rep_handler(struct cm_work *work)</div><div class='ctx'> 			      cm_id_priv-&gt;private_data_len);</div><div class='ctx'> 	else if (cm_id_priv-&gt;id.state == IB_CM_MRA_REP_SENT)</div><div class='ctx'> 		cm_format_mra((struct cm_mra_msg *) msg-&gt;mad, cm_id_priv,</div><div class='del'>-			      CM_MSG_RESPONSE_REP, cm_id_priv-&gt;service_timeout,</div><div class='add'>+			      CM_MSG_RESPONSE_REP,</div><div class='ctx'> 			      cm_id_priv-&gt;private_data,</div><div class='ctx'> 			      cm_id_priv-&gt;private_data_len);</div><div class='ctx'> 	else</div><div class='ctx'> 		goto unlock;</div><div class='ctx'> 	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> </div><div class='add'>+	trace_icm_send_dup_rep(&amp;cm_id_priv-&gt;id);</div><div class='ctx'> 	ret = ib_post_send_mad(msg, NULL);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto free;</div><div class='hunk'>@@ -2278,15 +2492,15 @@ static int cm_rep_handler(struct cm_work *work)</div><div class='ctx'> 	struct cm_rep_msg *rep_msg;</div><div class='ctx'> 	int ret;</div><div class='ctx'> 	struct cm_id_private *cur_cm_id_priv;</div><div class='del'>-	struct ib_cm_id *cm_id;</div><div class='ctx'> 	struct cm_timewait_info *timewait_info;</div><div class='ctx'> </div><div class='ctx'> 	rep_msg = (struct cm_rep_msg *)work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='del'>-	cm_id_priv = cm_acquire_id(rep_msg-&gt;remote_comm_id, 0);</div><div class='add'>+	cm_id_priv = cm_acquire_id(</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 0);</div><div class='ctx'> 	if (!cm_id_priv) {</div><div class='ctx'> 		cm_dup_rep_handler(work);</div><div class='del'>-		pr_debug("%s: remote_comm_id %d, no cm_id_priv\n", __func__,</div><div class='del'>-			 be32_to_cpu(rep_msg-&gt;remote_comm_id));</div><div class='add'>+		trace_icm_remote_no_priv_err(</div><div class='add'>+			 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -2298,17 +2512,19 @@ static int cm_rep_handler(struct cm_work *work)</div><div class='ctx'> 	case IB_CM_MRA_REQ_RCVD:</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='del'>-		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='del'>-		pr_debug("%s: cm_id_priv-&gt;id.state: %d, local_comm_id %d, remote_comm_id %d\n",</div><div class='del'>-			 __func__, cm_id_priv-&gt;id.state,</div><div class='del'>-			 be32_to_cpu(rep_msg-&gt;local_comm_id),</div><div class='del'>-			 be32_to_cpu(rep_msg-&gt;remote_comm_id));</div><div class='add'>+		trace_icm_rep_unknown_err(</div><div class='add'>+			IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),</div><div class='add'>+			IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg),</div><div class='add'>+			cm_id_priv-&gt;id.state);</div><div class='add'>+		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> 		goto error;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	cm_id_priv-&gt;timewait_info-&gt;work.remote_id = rep_msg-&gt;local_comm_id;</div><div class='del'>-	cm_id_priv-&gt;timewait_info-&gt;remote_ca_guid = rep_msg-&gt;local_ca_guid;</div><div class='add'>+	cm_id_priv-&gt;timewait_info-&gt;work.remote_id =</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));</div><div class='add'>+	cm_id_priv-&gt;timewait_info-&gt;remote_ca_guid =</div><div class='add'>+		cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));</div><div class='ctx'> 	cm_id_priv-&gt;timewait_info-&gt;remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv-&gt;qp_type);</div><div class='ctx'> </div><div class='ctx'> 	spin_lock(&amp;cm.lock);</div><div class='hunk'>@@ -2317,17 +2533,15 @@ static int cm_rep_handler(struct cm_work *work)</div><div class='ctx'> 		spin_unlock(&amp;cm.lock);</div><div class='ctx'> 		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='del'>-		pr_debug("%s: Failed to insert remote id %d\n", __func__,</div><div class='del'>-			 be32_to_cpu(rep_msg-&gt;remote_comm_id));</div><div class='add'>+		trace_icm_insert_failed_err(</div><div class='add'>+			 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));</div><div class='ctx'> 		goto error;</div><div class='ctx'> 	}</div><div class='ctx'> 	/* Check for a stale connection. */</div><div class='ctx'> 	timewait_info = cm_insert_remote_qpn(cm_id_priv-&gt;timewait_info);</div><div class='ctx'> 	if (timewait_info) {</div><div class='del'>-		rb_erase(&amp;cm_id_priv-&gt;timewait_info-&gt;remote_id_node,</div><div class='del'>-			 &amp;cm.remote_id_table);</div><div class='del'>-		cm_id_priv-&gt;timewait_info-&gt;inserted_remote_id = 0;</div><div class='del'>-		cur_cm_id_priv = cm_get_id(timewait_info-&gt;work.local_id,</div><div class='add'>+		cm_remove_remote(cm_id_priv);</div><div class='add'>+		cur_cm_id_priv = cm_acquire_id(timewait_info-&gt;work.local_id,</div><div class='ctx'> 					   timewait_info-&gt;work.remote_id);</div><div class='ctx'> </div><div class='ctx'> 		spin_unlock(&amp;cm.lock);</div><div class='hunk'>@@ -2336,13 +2550,12 @@ static int cm_rep_handler(struct cm_work *work)</div><div class='ctx'> 			     IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,</div><div class='ctx'> 			     NULL, 0);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='del'>-		pr_debug("%s: Stale connection. local_comm_id %d, remote_comm_id %d\n",</div><div class='del'>-			 __func__, be32_to_cpu(rep_msg-&gt;local_comm_id),</div><div class='del'>-			 be32_to_cpu(rep_msg-&gt;remote_comm_id));</div><div class='add'>+		trace_icm_staleconn_err(</div><div class='add'>+			IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),</div><div class='add'>+			IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));</div><div class='ctx'> </div><div class='ctx'> 		if (cur_cm_id_priv) {</div><div class='del'>-			cm_id = &amp;cur_cm_id_priv-&gt;id;</div><div class='del'>-			ib_send_cm_dreq(cm_id, NULL, 0);</div><div class='add'>+			ib_send_cm_dreq(&amp;cur_cm_id_priv-&gt;id, NULL, 0);</div><div class='ctx'> 			cm_deref_id(cur_cm_id_priv);</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='hunk'>@@ -2351,13 +2564,17 @@ static int cm_rep_handler(struct cm_work *work)</div><div class='ctx'> 	spin_unlock(&amp;cm.lock);</div><div class='ctx'> </div><div class='ctx'> 	cm_id_priv-&gt;id.state = IB_CM_REP_RCVD;</div><div class='del'>-	cm_id_priv-&gt;id.remote_id = rep_msg-&gt;local_comm_id;</div><div class='add'>+	cm_id_priv-&gt;id.remote_id =</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));</div><div class='ctx'> 	cm_id_priv-&gt;remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv-&gt;qp_type);</div><div class='del'>-	cm_id_priv-&gt;initiator_depth = rep_msg-&gt;resp_resources;</div><div class='del'>-	cm_id_priv-&gt;responder_resources = rep_msg-&gt;initiator_depth;</div><div class='del'>-	cm_id_priv-&gt;sq_psn = cm_rep_get_starting_psn(rep_msg);</div><div class='del'>-	cm_id_priv-&gt;rnr_retry_count = cm_rep_get_rnr_retry_count(rep_msg);</div><div class='del'>-	cm_id_priv-&gt;target_ack_delay = cm_rep_get_target_ack_delay(rep_msg);</div><div class='add'>+	cm_id_priv-&gt;initiator_depth =</div><div class='add'>+		IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);</div><div class='add'>+	cm_id_priv-&gt;responder_resources =</div><div class='add'>+		IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);</div><div class='add'>+	cm_id_priv-&gt;sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));</div><div class='add'>+	cm_id_priv-&gt;rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);</div><div class='add'>+	cm_id_priv-&gt;target_ack_delay =</div><div class='add'>+		IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);</div><div class='ctx'> 	cm_id_priv-&gt;av.timeout =</div><div class='ctx'> 			cm_ack_timeout(cm_id_priv-&gt;target_ack_delay,</div><div class='ctx'> 				       cm_id_priv-&gt;av.timeout - 1);</div><div class='hunk'>@@ -2365,18 +2582,8 @@ static int cm_rep_handler(struct cm_work *work)</div><div class='ctx'> 			cm_ack_timeout(cm_id_priv-&gt;target_ack_delay,</div><div class='ctx'> 				       cm_id_priv-&gt;alt_av.timeout - 1);</div><div class='ctx'> </div><div class='del'>-	/* todo: handle peer_to_peer */</div><div class='del'>-</div><div class='del'>-	ib_cancel_mad(cm_id_priv-&gt;av.port-&gt;mad_agent, cm_id_priv-&gt;msg);</div><div class='del'>-	ret = atomic_inc_and_test(&amp;cm_id_priv-&gt;work_count);</div><div class='del'>-	if (!ret)</div><div class='del'>-		list_add_tail(&amp;work-&gt;list, &amp;cm_id_priv-&gt;work_list);</div><div class='del'>-	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-</div><div class='del'>-	if (ret)</div><div class='del'>-		cm_process_work(cm_id_priv, work);</div><div class='del'>-	else</div><div class='del'>-		cm_deref_id(cm_id_priv);</div><div class='add'>+	ib_cancel_mad(cm_id_priv-&gt;msg);</div><div class='add'>+	cm_queue_work_unlock(cm_id_priv, work);</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> error:</div><div class='hunk'>@@ -2387,7 +2594,6 @@ error:</div><div class='ctx'> static int cm_establish_handler(struct cm_work *work)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='del'>-	int ret;</div><div class='ctx'> </div><div class='ctx'> 	/* See comment in cm_establish about lookup. */</div><div class='ctx'> 	cm_id_priv = cm_acquire_id(work-&gt;local_id, work-&gt;remote_id);</div><div class='hunk'>@@ -2400,16 +2606,8 @@ static int cm_establish_handler(struct cm_work *work)</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ib_cancel_mad(cm_id_priv-&gt;av.port-&gt;mad_agent, cm_id_priv-&gt;msg);</div><div class='del'>-	ret = atomic_inc_and_test(&amp;cm_id_priv-&gt;work_count);</div><div class='del'>-	if (!ret)</div><div class='del'>-		list_add_tail(&amp;work-&gt;list, &amp;cm_id_priv-&gt;work_list);</div><div class='del'>-	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-</div><div class='del'>-	if (ret)</div><div class='del'>-		cm_process_work(cm_id_priv, work);</div><div class='del'>-	else</div><div class='del'>-		cm_deref_id(cm_id_priv);</div><div class='add'>+	ib_cancel_mad(cm_id_priv-&gt;msg);</div><div class='add'>+	cm_queue_work_unlock(cm_id_priv, work);</div><div class='ctx'> 	return 0;</div><div class='ctx'> out:</div><div class='ctx'> 	cm_deref_id(cm_id_priv);</div><div class='hunk'>@@ -2420,36 +2618,29 @@ static int cm_rtu_handler(struct cm_work *work)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='ctx'> 	struct cm_rtu_msg *rtu_msg;</div><div class='del'>-	int ret;</div><div class='ctx'> </div><div class='ctx'> 	rtu_msg = (struct cm_rtu_msg *)work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='del'>-	cm_id_priv = cm_acquire_id(rtu_msg-&gt;remote_comm_id,</div><div class='del'>-				   rtu_msg-&gt;local_comm_id);</div><div class='add'>+	cm_id_priv = cm_acquire_id(</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_RTU_REMOTE_COMM_ID, rtu_msg)),</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_RTU_LOCAL_COMM_ID, rtu_msg)));</div><div class='ctx'> 	if (!cm_id_priv)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	work-&gt;cm_event.private_data = &amp;rtu_msg-&gt;private_data;</div><div class='add'>+	work-&gt;cm_event.private_data =</div><div class='add'>+		IBA_GET_MEM_PTR(CM_RTU_PRIVATE_DATA, rtu_msg);</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> 	if (cm_id_priv-&gt;id.state != IB_CM_REP_SENT &amp;&amp;</div><div class='ctx'> 	    cm_id_priv-&gt;id.state != IB_CM_MRA_REP_RCVD) {</div><div class='ctx'> 		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-		atomic_long_inc(&amp;work-&gt;port-&gt;counter_group[CM_RECV_DUPLICATES].</div><div class='del'>-				counter[CM_RTU_COUNTER]);</div><div class='add'>+		atomic_long_inc(&amp;work-&gt;port-&gt;counters[CM_RECV_DUPLICATES]</div><div class='add'>+						     [CM_RTU_COUNTER]);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> 	cm_id_priv-&gt;id.state = IB_CM_ESTABLISHED;</div><div class='ctx'> </div><div class='del'>-	ib_cancel_mad(cm_id_priv-&gt;av.port-&gt;mad_agent, cm_id_priv-&gt;msg);</div><div class='del'>-	ret = atomic_inc_and_test(&amp;cm_id_priv-&gt;work_count);</div><div class='del'>-	if (!ret)</div><div class='del'>-		list_add_tail(&amp;work-&gt;list, &amp;cm_id_priv-&gt;work_list);</div><div class='del'>-	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-</div><div class='del'>-	if (ret)</div><div class='del'>-		cm_process_work(cm_id_priv, work);</div><div class='del'>-	else</div><div class='del'>-		cm_deref_id(cm_id_priv);</div><div class='add'>+	ib_cancel_mad(cm_id_priv-&gt;msg);</div><div class='add'>+	cm_queue_work_unlock(cm_id_priv, work);</div><div class='ctx'> 	return 0;</div><div class='ctx'> out:</div><div class='ctx'> 	cm_deref_id(cm_id_priv);</div><div class='hunk'>@@ -2463,19 +2654,42 @@ static void cm_format_dreq(struct cm_dreq_msg *dreq_msg,</div><div class='ctx'> {</div><div class='ctx'> 	cm_format_mad_hdr(&amp;dreq_msg-&gt;hdr, CM_DREQ_ATTR_ID,</div><div class='ctx'> 			  cm_form_tid(cm_id_priv));</div><div class='del'>-	dreq_msg-&gt;local_comm_id = cm_id_priv-&gt;id.local_id;</div><div class='del'>-	dreq_msg-&gt;remote_comm_id = cm_id_priv-&gt;id.remote_id;</div><div class='del'>-	cm_dreq_set_remote_qpn(dreq_msg, cm_id_priv-&gt;remote_qpn);</div><div class='add'>+	IBA_SET(CM_DREQ_LOCAL_COMM_ID, dreq_msg,</div><div class='add'>+		be32_to_cpu(cm_id_priv-&gt;id.local_id));</div><div class='add'>+	IBA_SET(CM_DREQ_REMOTE_COMM_ID, dreq_msg,</div><div class='add'>+		be32_to_cpu(cm_id_priv-&gt;id.remote_id));</div><div class='add'>+	IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg,</div><div class='add'>+		be32_to_cpu(cm_id_priv-&gt;remote_qpn));</div><div class='ctx'> </div><div class='ctx'> 	if (private_data &amp;&amp; private_data_len)</div><div class='del'>-		memcpy(dreq_msg-&gt;private_data, private_data, private_data_len);</div><div class='add'>+		IBA_SET_MEM(CM_DREQ_PRIVATE_DATA, dreq_msg, private_data,</div><div class='add'>+			    private_data_len);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int ib_send_cm_dreq(struct ib_cm_id *cm_id,</div><div class='del'>-		    const void *private_data,</div><div class='add'>+static void cm_issue_dreq(struct cm_id_private *cm_id_priv)</div><div class='add'>+{</div><div class='add'>+	struct ib_mad_send_buf *msg;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	lockdep_assert_held(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+</div><div class='add'>+	msg = cm_alloc_msg(cm_id_priv);</div><div class='add'>+	if (IS_ERR(msg))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	cm_format_dreq((struct cm_dreq_msg *) msg-&gt;mad, cm_id_priv, NULL, 0);</div><div class='add'>+</div><div class='add'>+	trace_icm_send_dreq(&amp;cm_id_priv-&gt;id);</div><div class='add'>+	ret = ib_post_send_mad(msg, NULL);</div><div class='add'>+	if (ret)</div><div class='add'>+		cm_free_msg(msg);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ib_send_cm_dreq(struct ib_cm_id *cm_id, const void *private_data,</div><div class='ctx'> 		    u8 private_data_len)</div><div class='ctx'> {</div><div class='del'>-	struct cm_id_private *cm_id_priv;</div><div class='add'>+	struct cm_id_private *cm_id_priv =</div><div class='add'>+		container_of(cm_id, struct cm_id_private, id);</div><div class='ctx'> 	struct ib_mad_send_buf *msg;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int ret;</div><div class='hunk'>@@ -2483,41 +2697,38 @@ int ib_send_cm_dreq(struct ib_cm_id *cm_id,</div><div class='ctx'> 	if (private_data &amp;&amp; private_data_len &gt; IB_CM_DREQ_PRIVATE_DATA_SIZE)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	cm_id_priv = container_of(cm_id, struct cm_id_private, id);</div><div class='ctx'> 	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	if (cm_id-&gt;state != IB_CM_ESTABLISHED) {</div><div class='del'>-		pr_debug("%s: local_id %d, cm_id-&gt;state: %d\n", __func__,</div><div class='del'>-			 be32_to_cpu(cm_id-&gt;local_id), cm_id-&gt;state);</div><div class='add'>+	if (cm_id_priv-&gt;id.state != IB_CM_ESTABLISHED) {</div><div class='add'>+		trace_icm_dreq_skipped(&amp;cm_id_priv-&gt;id);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='del'>-		goto out;</div><div class='add'>+		goto unlock;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (cm_id-&gt;lap_state == IB_CM_LAP_SENT ||</div><div class='del'>-	    cm_id-&gt;lap_state == IB_CM_MRA_LAP_RCVD)</div><div class='del'>-		ib_cancel_mad(cm_id_priv-&gt;av.port-&gt;mad_agent, cm_id_priv-&gt;msg);</div><div class='add'>+	if (cm_id_priv-&gt;id.lap_state == IB_CM_LAP_SENT ||</div><div class='add'>+	    cm_id_priv-&gt;id.lap_state == IB_CM_MRA_LAP_RCVD)</div><div class='add'>+		ib_cancel_mad(cm_id_priv-&gt;msg);</div><div class='ctx'> </div><div class='del'>-	ret = cm_alloc_msg(cm_id_priv, &amp;msg);</div><div class='del'>-	if (ret) {</div><div class='add'>+	msg = cm_alloc_priv_msg(cm_id_priv, IB_CM_DREQ_SENT);</div><div class='add'>+	if (IS_ERR(msg)) {</div><div class='ctx'> 		cm_enter_timewait(cm_id_priv);</div><div class='del'>-		goto out;</div><div class='add'>+		ret = PTR_ERR(msg);</div><div class='add'>+		goto unlock;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	cm_format_dreq((struct cm_dreq_msg *) msg-&gt;mad, cm_id_priv,</div><div class='ctx'> 		       private_data, private_data_len);</div><div class='del'>-	msg-&gt;timeout_ms = cm_id_priv-&gt;timeout_ms;</div><div class='del'>-	msg-&gt;context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT;</div><div class='ctx'> </div><div class='add'>+	trace_icm_send_dreq(&amp;cm_id_priv-&gt;id);</div><div class='ctx'> 	ret = ib_post_send_mad(msg, NULL);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		cm_enter_timewait(cm_id_priv);</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-		cm_free_msg(msg);</div><div class='del'>-		return ret;</div><div class='add'>+		cm_free_priv_msg(msg);</div><div class='add'>+		goto unlock;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	cm_id-&gt;state = IB_CM_DREQ_SENT;</div><div class='del'>-	cm_id_priv-&gt;msg = msg;</div><div class='del'>-out:	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+	cm_id_priv-&gt;id.state = IB_CM_DREQ_SENT;</div><div class='add'>+unlock:</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_send_cm_dreq);</div><div class='hunk'>@@ -2528,58 +2739,68 @@ static void cm_format_drep(struct cm_drep_msg *drep_msg,</div><div class='ctx'> 			  u8 private_data_len)</div><div class='ctx'> {</div><div class='ctx'> 	cm_format_mad_hdr(&amp;drep_msg-&gt;hdr, CM_DREP_ATTR_ID, cm_id_priv-&gt;tid);</div><div class='del'>-	drep_msg-&gt;local_comm_id = cm_id_priv-&gt;id.local_id;</div><div class='del'>-	drep_msg-&gt;remote_comm_id = cm_id_priv-&gt;id.remote_id;</div><div class='add'>+	IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,</div><div class='add'>+		be32_to_cpu(cm_id_priv-&gt;id.local_id));</div><div class='add'>+	IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,</div><div class='add'>+		be32_to_cpu(cm_id_priv-&gt;id.remote_id));</div><div class='ctx'> </div><div class='ctx'> 	if (private_data &amp;&amp; private_data_len)</div><div class='del'>-		memcpy(drep_msg-&gt;private_data, private_data, private_data_len);</div><div class='add'>+		IBA_SET_MEM(CM_DREP_PRIVATE_DATA, drep_msg, private_data,</div><div class='add'>+			    private_data_len);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int ib_send_cm_drep(struct ib_cm_id *cm_id,</div><div class='del'>-		    const void *private_data,</div><div class='del'>-		    u8 private_data_len)</div><div class='add'>+static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,</div><div class='add'>+			       void *private_data, u8 private_data_len)</div><div class='ctx'> {</div><div class='del'>-	struct cm_id_private *cm_id_priv;</div><div class='ctx'> 	struct ib_mad_send_buf *msg;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	void *data;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	lockdep_assert_held(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+</div><div class='ctx'> 	if (private_data &amp;&amp; private_data_len &gt; IB_CM_DREP_PRIVATE_DATA_SIZE)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	data = cm_copy_private_data(private_data, private_data_len);</div><div class='del'>-	if (IS_ERR(data))</div><div class='del'>-		return PTR_ERR(data);</div><div class='del'>-</div><div class='del'>-	cm_id_priv = container_of(cm_id, struct cm_id_private, id);</div><div class='del'>-	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	if (cm_id-&gt;state != IB_CM_DREQ_RCVD) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-		kfree(data);</div><div class='del'>-		pr_debug("%s: local_id %d, cm_idcm_id-&gt;state(%d) != IB_CM_DREQ_RCVD\n",</div><div class='del'>-			 __func__, be32_to_cpu(cm_id-&gt;local_id), cm_id-&gt;state);</div><div class='add'>+	if (cm_id_priv-&gt;id.state != IB_CM_DREQ_RCVD) {</div><div class='add'>+		trace_icm_send_drep_err(&amp;cm_id_priv-&gt;id);</div><div class='add'>+		kfree(private_data);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	cm_set_private_data(cm_id_priv, data, private_data_len);</div><div class='add'>+	cm_set_private_data(cm_id_priv, private_data, private_data_len);</div><div class='ctx'> 	cm_enter_timewait(cm_id_priv);</div><div class='ctx'> </div><div class='del'>-	ret = cm_alloc_msg(cm_id_priv, &amp;msg);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto out;</div><div class='add'>+	msg = cm_alloc_msg(cm_id_priv);</div><div class='add'>+	if (IS_ERR(msg))</div><div class='add'>+		return PTR_ERR(msg);</div><div class='ctx'> </div><div class='ctx'> 	cm_format_drep((struct cm_drep_msg *) msg-&gt;mad, cm_id_priv,</div><div class='ctx'> 		       private_data, private_data_len);</div><div class='ctx'> </div><div class='add'>+	trace_icm_send_drep(&amp;cm_id_priv-&gt;id);</div><div class='ctx'> 	ret = ib_post_send_mad(msg, NULL);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 		cm_free_msg(msg);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ib_send_cm_drep(struct ib_cm_id *cm_id, const void *private_data,</div><div class='add'>+		    u8 private_data_len)</div><div class='add'>+{</div><div class='add'>+	struct cm_id_private *cm_id_priv =</div><div class='add'>+		container_of(cm_id, struct cm_id_private, id);</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	void *data;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-out:	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+	data = cm_copy_private_data(private_data, private_data_len);</div><div class='add'>+	if (IS_ERR(data))</div><div class='add'>+		return PTR_ERR(data);</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+	ret = cm_send_drep_locked(cm_id_priv, data, private_data_len);</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_send_cm_drep);</div><div class='hunk'>@@ -2592,7 +2813,7 @@ static int cm_issue_drep(struct cm_port *port,</div><div class='ctx'> 	struct cm_drep_msg *drep_msg;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = cm_alloc_response_msg(port, mad_recv_wc, &amp;msg);</div><div class='add'>+	ret = cm_alloc_response_msg(port, mad_recv_wc, true, &amp;msg);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='hunk'>@@ -2600,9 +2821,14 @@ static int cm_issue_drep(struct cm_port *port,</div><div class='ctx'> 	drep_msg = (struct cm_drep_msg *) msg-&gt;mad;</div><div class='ctx'> </div><div class='ctx'> 	cm_format_mad_hdr(&amp;drep_msg-&gt;hdr, CM_DREP_ATTR_ID, dreq_msg-&gt;hdr.tid);</div><div class='del'>-	drep_msg-&gt;remote_comm_id = dreq_msg-&gt;local_comm_id;</div><div class='del'>-	drep_msg-&gt;local_comm_id = dreq_msg-&gt;remote_comm_id;</div><div class='del'>-</div><div class='add'>+	IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,</div><div class='add'>+		IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg));</div><div class='add'>+	IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,</div><div class='add'>+		IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));</div><div class='add'>+</div><div class='add'>+	trace_icm_issue_drep(</div><div class='add'>+		IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),</div><div class='add'>+		IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));</div><div class='ctx'> 	ret = ib_post_send_mad(msg, NULL);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		cm_free_msg(msg);</div><div class='hunk'>@@ -2615,43 +2841,45 @@ static int cm_dreq_handler(struct cm_work *work)</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='ctx'> 	struct cm_dreq_msg *dreq_msg;</div><div class='ctx'> 	struct ib_mad_send_buf *msg = NULL;</div><div class='del'>-	int ret;</div><div class='ctx'> </div><div class='ctx'> 	dreq_msg = (struct cm_dreq_msg *)work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='del'>-	cm_id_priv = cm_acquire_id(dreq_msg-&gt;remote_comm_id,</div><div class='del'>-				   dreq_msg-&gt;local_comm_id);</div><div class='add'>+	cm_id_priv = cm_acquire_id(</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)),</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg)));</div><div class='ctx'> 	if (!cm_id_priv) {</div><div class='del'>-		atomic_long_inc(&amp;work-&gt;port-&gt;counter_group[CM_RECV_DUPLICATES].</div><div class='del'>-				counter[CM_DREQ_COUNTER]);</div><div class='add'>+		atomic_long_inc(&amp;work-&gt;port-&gt;counters[CM_RECV_DUPLICATES]</div><div class='add'>+						     [CM_DREQ_COUNTER]);</div><div class='ctx'> 		cm_issue_drep(work-&gt;port, work-&gt;mad_recv_wc);</div><div class='del'>-		pr_debug("%s: no cm_id_priv, local_comm_id %d, remote_comm_id %d\n",</div><div class='del'>-			 __func__, be32_to_cpu(dreq_msg-&gt;local_comm_id),</div><div class='del'>-			 be32_to_cpu(dreq_msg-&gt;remote_comm_id));</div><div class='add'>+		trace_icm_no_priv_err(</div><div class='add'>+			IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),</div><div class='add'>+			IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	work-&gt;cm_event.private_data = &amp;dreq_msg-&gt;private_data;</div><div class='add'>+	work-&gt;cm_event.private_data =</div><div class='add'>+		IBA_GET_MEM_PTR(CM_DREQ_PRIVATE_DATA, dreq_msg);</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-	if (cm_id_priv-&gt;local_qpn != cm_dreq_get_remote_qpn(dreq_msg))</div><div class='add'>+	if (cm_id_priv-&gt;local_qpn !=</div><div class='add'>+	    cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg)))</div><div class='ctx'> 		goto unlock;</div><div class='ctx'> </div><div class='ctx'> 	switch (cm_id_priv-&gt;id.state) {</div><div class='ctx'> 	case IB_CM_REP_SENT:</div><div class='ctx'> 	case IB_CM_DREQ_SENT:</div><div class='del'>-		ib_cancel_mad(cm_id_priv-&gt;av.port-&gt;mad_agent, cm_id_priv-&gt;msg);</div><div class='add'>+	case IB_CM_MRA_REP_RCVD:</div><div class='add'>+		ib_cancel_mad(cm_id_priv-&gt;msg);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_ESTABLISHED:</div><div class='ctx'> 		if (cm_id_priv-&gt;id.lap_state == IB_CM_LAP_SENT ||</div><div class='ctx'> 		    cm_id_priv-&gt;id.lap_state == IB_CM_MRA_LAP_RCVD)</div><div class='del'>-			ib_cancel_mad(cm_id_priv-&gt;av.port-&gt;mad_agent, cm_id_priv-&gt;msg);</div><div class='del'>-		break;</div><div class='del'>-	case IB_CM_MRA_REP_RCVD:</div><div class='add'>+			ib_cancel_mad(cm_id_priv-&gt;msg);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_TIMEWAIT:</div><div class='del'>-		atomic_long_inc(&amp;work-&gt;port-&gt;counter_group[CM_RECV_DUPLICATES].</div><div class='del'>-				counter[CM_DREQ_COUNTER]);</div><div class='del'>-		msg = cm_alloc_response_msg_no_ah(work-&gt;port, work-&gt;mad_recv_wc);</div><div class='add'>+		atomic_long_inc(&amp;work-&gt;port-&gt;counters[CM_RECV_DUPLICATES]</div><div class='add'>+						     [CM_DREQ_COUNTER]);</div><div class='add'>+		msg = cm_alloc_response_msg_no_ah(work-&gt;port, work-&gt;mad_recv_wc,</div><div class='add'>+						  true);</div><div class='ctx'> 		if (IS_ERR(msg))</div><div class='ctx'> 			goto unlock;</div><div class='ctx'> </div><div class='hunk'>@@ -2665,26 +2893,16 @@ static int cm_dreq_handler(struct cm_work *work)</div><div class='ctx'> 			cm_free_msg(msg);</div><div class='ctx'> 		goto deref;</div><div class='ctx'> 	case IB_CM_DREQ_RCVD:</div><div class='del'>-		atomic_long_inc(&amp;work-&gt;port-&gt;counter_group[CM_RECV_DUPLICATES].</div><div class='del'>-				counter[CM_DREQ_COUNTER]);</div><div class='add'>+		atomic_long_inc(&amp;work-&gt;port-&gt;counters[CM_RECV_DUPLICATES]</div><div class='add'>+						     [CM_DREQ_COUNTER]);</div><div class='ctx'> 		goto unlock;</div><div class='ctx'> 	default:</div><div class='del'>-		pr_debug("%s: local_id %d, cm_id_priv-&gt;id.state: %d\n",</div><div class='del'>-			 __func__, be32_to_cpu(cm_id_priv-&gt;id.local_id),</div><div class='del'>-			 cm_id_priv-&gt;id.state);</div><div class='add'>+		trace_icm_dreq_unknown_err(&amp;cm_id_priv-&gt;id);</div><div class='ctx'> 		goto unlock;</div><div class='ctx'> 	}</div><div class='ctx'> 	cm_id_priv-&gt;id.state = IB_CM_DREQ_RCVD;</div><div class='ctx'> 	cm_id_priv-&gt;tid = dreq_msg-&gt;hdr.tid;</div><div class='del'>-	ret = atomic_inc_and_test(&amp;cm_id_priv-&gt;work_count);</div><div class='del'>-	if (!ret)</div><div class='del'>-		list_add_tail(&amp;work-&gt;list, &amp;cm_id_priv-&gt;work_list);</div><div class='del'>-	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-</div><div class='del'>-	if (ret)</div><div class='del'>-		cm_process_work(cm_id_priv, work);</div><div class='del'>-	else</div><div class='del'>-		cm_deref_id(cm_id_priv);</div><div class='add'>+	cm_queue_work_unlock(cm_id_priv, work);</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> unlock:	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='hunk'>@@ -2696,15 +2914,16 @@ static int cm_drep_handler(struct cm_work *work)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='ctx'> 	struct cm_drep_msg *drep_msg;</div><div class='del'>-	int ret;</div><div class='ctx'> </div><div class='ctx'> 	drep_msg = (struct cm_drep_msg *)work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='del'>-	cm_id_priv = cm_acquire_id(drep_msg-&gt;remote_comm_id,</div><div class='del'>-				   drep_msg-&gt;local_comm_id);</div><div class='add'>+	cm_id_priv = cm_acquire_id(</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_DREP_REMOTE_COMM_ID, drep_msg)),</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_DREP_LOCAL_COMM_ID, drep_msg)));</div><div class='ctx'> 	if (!cm_id_priv)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	work-&gt;cm_event.private_data = &amp;drep_msg-&gt;private_data;</div><div class='add'>+	work-&gt;cm_event.private_data =</div><div class='add'>+		IBA_GET_MEM_PTR(CM_DREP_PRIVATE_DATA, drep_msg);</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> 	if (cm_id_priv-&gt;id.state != IB_CM_DREQ_SENT &amp;&amp;</div><div class='hunk'>@@ -2714,81 +2933,83 @@ static int cm_drep_handler(struct cm_work *work)</div><div class='ctx'> 	}</div><div class='ctx'> 	cm_enter_timewait(cm_id_priv);</div><div class='ctx'> </div><div class='del'>-	ib_cancel_mad(cm_id_priv-&gt;av.port-&gt;mad_agent, cm_id_priv-&gt;msg);</div><div class='del'>-	ret = atomic_inc_and_test(&amp;cm_id_priv-&gt;work_count);</div><div class='del'>-	if (!ret)</div><div class='del'>-		list_add_tail(&amp;work-&gt;list, &amp;cm_id_priv-&gt;work_list);</div><div class='del'>-	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-</div><div class='del'>-	if (ret)</div><div class='del'>-		cm_process_work(cm_id_priv, work);</div><div class='del'>-	else</div><div class='del'>-		cm_deref_id(cm_id_priv);</div><div class='add'>+	ib_cancel_mad(cm_id_priv-&gt;msg);</div><div class='add'>+	cm_queue_work_unlock(cm_id_priv, work);</div><div class='ctx'> 	return 0;</div><div class='ctx'> out:</div><div class='ctx'> 	cm_deref_id(cm_id_priv);</div><div class='ctx'> 	return -EINVAL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int ib_send_cm_rej(struct ib_cm_id *cm_id,</div><div class='del'>-		   enum ib_cm_rej_reason reason,</div><div class='del'>-		   void *ari,</div><div class='del'>-		   u8 ari_length,</div><div class='del'>-		   const void *private_data,</div><div class='del'>-		   u8 private_data_len)</div><div class='add'>+static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,</div><div class='add'>+			      enum ib_cm_rej_reason reason, void *ari,</div><div class='add'>+			      u8 ari_length, const void *private_data,</div><div class='add'>+			      u8 private_data_len)</div><div class='ctx'> {</div><div class='del'>-	struct cm_id_private *cm_id_priv;</div><div class='add'>+	enum ib_cm_state state = cm_id_priv-&gt;id.state;</div><div class='ctx'> 	struct ib_mad_send_buf *msg;</div><div class='del'>-	unsigned long flags;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	lockdep_assert_held(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+</div><div class='ctx'> 	if ((private_data &amp;&amp; private_data_len &gt; IB_CM_REJ_PRIVATE_DATA_SIZE) ||</div><div class='ctx'> 	    (ari &amp;&amp; ari_length &gt; IB_CM_REJ_ARI_LENGTH))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	cm_id_priv = container_of(cm_id, struct cm_id_private, id);</div><div class='add'>+	trace_icm_send_rej(&amp;cm_id_priv-&gt;id, reason);</div><div class='ctx'> </div><div class='del'>-	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	switch (cm_id-&gt;state) {</div><div class='add'>+	switch (state) {</div><div class='ctx'> 	case IB_CM_REQ_SENT:</div><div class='ctx'> 	case IB_CM_MRA_REQ_RCVD:</div><div class='ctx'> 	case IB_CM_REQ_RCVD:</div><div class='ctx'> 	case IB_CM_MRA_REQ_SENT:</div><div class='ctx'> 	case IB_CM_REP_RCVD:</div><div class='ctx'> 	case IB_CM_MRA_REP_SENT:</div><div class='del'>-		ret = cm_alloc_msg(cm_id_priv, &amp;msg);</div><div class='del'>-		if (!ret)</div><div class='del'>-			cm_format_rej((struct cm_rej_msg *) msg-&gt;mad,</div><div class='del'>-				      cm_id_priv, reason, ari, ari_length,</div><div class='del'>-				      private_data, private_data_len);</div><div class='del'>-</div><div class='ctx'> 		cm_reset_to_idle(cm_id_priv);</div><div class='add'>+		msg = cm_alloc_msg(cm_id_priv);</div><div class='add'>+		if (IS_ERR(msg))</div><div class='add'>+			return PTR_ERR(msg);</div><div class='add'>+		cm_format_rej((struct cm_rej_msg *)msg-&gt;mad, cm_id_priv, reason,</div><div class='add'>+			      ari, ari_length, private_data, private_data_len,</div><div class='add'>+			      state);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_REP_SENT:</div><div class='ctx'> 	case IB_CM_MRA_REP_RCVD:</div><div class='del'>-		ret = cm_alloc_msg(cm_id_priv, &amp;msg);</div><div class='del'>-		if (!ret)</div><div class='del'>-			cm_format_rej((struct cm_rej_msg *) msg-&gt;mad,</div><div class='del'>-				      cm_id_priv, reason, ari, ari_length,</div><div class='del'>-				      private_data, private_data_len);</div><div class='del'>-</div><div class='ctx'> 		cm_enter_timewait(cm_id_priv);</div><div class='add'>+		msg = cm_alloc_msg(cm_id_priv);</div><div class='add'>+		if (IS_ERR(msg))</div><div class='add'>+			return PTR_ERR(msg);</div><div class='add'>+		cm_format_rej((struct cm_rej_msg *)msg-&gt;mad, cm_id_priv, reason,</div><div class='add'>+			      ari, ari_length, private_data, private_data_len,</div><div class='add'>+			      state);</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='del'>-		pr_debug("%s: local_id %d, cm_id-&gt;state: %d\n", __func__,</div><div class='del'>-			 be32_to_cpu(cm_id_priv-&gt;id.local_id), cm_id-&gt;state);</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-		goto out;</div><div class='add'>+		trace_icm_send_unknown_rej_err(&amp;cm_id_priv-&gt;id);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (ret)</div><div class='del'>-		goto out;</div><div class='del'>-</div><div class='ctx'> 	ret = ib_post_send_mad(msg, NULL);</div><div class='del'>-	if (ret)</div><div class='add'>+	if (ret) {</div><div class='ctx'> 		cm_free_msg(msg);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-out:	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+int ib_send_cm_rej(struct ib_cm_id *cm_id, enum ib_cm_rej_reason reason,</div><div class='add'>+		   void *ari, u8 ari_length, const void *private_data,</div><div class='add'>+		   u8 private_data_len)</div><div class='add'>+{</div><div class='add'>+	struct cm_id_private *cm_id_priv =</div><div class='add'>+		container_of(cm_id, struct cm_id_private, id);</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+	ret = cm_send_rej_locked(cm_id_priv, reason, ari, ari_length,</div><div class='add'>+				 private_data, private_data_len);</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_send_cm_rej);</div><div class='hunk'>@@ -2800,41 +3021,33 @@ static void cm_format_rej_event(struct cm_work *work)</div><div class='ctx'> </div><div class='ctx'> 	rej_msg = (struct cm_rej_msg *)work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='ctx'> 	param = &amp;work-&gt;cm_event.param.rej_rcvd;</div><div class='del'>-	param-&gt;ari = rej_msg-&gt;ari;</div><div class='del'>-	param-&gt;ari_length = cm_rej_get_reject_info_len(rej_msg);</div><div class='del'>-	param-&gt;reason = __be16_to_cpu(rej_msg-&gt;reason);</div><div class='del'>-	work-&gt;cm_event.private_data = &amp;rej_msg-&gt;private_data;</div><div class='add'>+	param-&gt;ari = IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg);</div><div class='add'>+	param-&gt;ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg);</div><div class='add'>+	param-&gt;reason = IBA_GET(CM_REJ_REASON, rej_msg);</div><div class='add'>+	work-&gt;cm_event.private_data =</div><div class='add'>+		IBA_GET_MEM_PTR(CM_REJ_PRIVATE_DATA, rej_msg);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)</div><div class='add'>+static struct cm_id_private *cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)</div><div class='ctx'> {</div><div class='del'>-	struct cm_timewait_info *timewait_info;</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='ctx'> 	__be32 remote_id;</div><div class='ctx'> </div><div class='del'>-	remote_id = rej_msg-&gt;local_comm_id;</div><div class='del'>-</div><div class='del'>-	if (__be16_to_cpu(rej_msg-&gt;reason) == IB_CM_REJ_TIMEOUT) {</div><div class='del'>-		spin_lock_irq(&amp;cm.lock);</div><div class='del'>-		timewait_info = cm_find_remote_id( *((__be64 *) rej_msg-&gt;ari),</div><div class='del'>-						  remote_id);</div><div class='del'>-		if (!timewait_info) {</div><div class='del'>-			spin_unlock_irq(&amp;cm.lock);</div><div class='del'>-			return NULL;</div><div class='del'>-		}</div><div class='del'>-		cm_id_priv = xa_load(&amp;cm.local_id_table,</div><div class='del'>-				cm_local_id(timewait_info-&gt;work.local_id));</div><div class='del'>-		if (cm_id_priv) {</div><div class='del'>-			if (cm_id_priv-&gt;id.remote_id == remote_id)</div><div class='del'>-				atomic_inc(&amp;cm_id_priv-&gt;refcount);</div><div class='del'>-			else</div><div class='del'>-				cm_id_priv = NULL;</div><div class='del'>-		}</div><div class='del'>-		spin_unlock_irq(&amp;cm.lock);</div><div class='del'>-	} else if (cm_rej_get_msg_rejected(rej_msg) == CM_MSG_RESPONSE_REQ)</div><div class='del'>-		cm_id_priv = cm_acquire_id(rej_msg-&gt;remote_comm_id, 0);</div><div class='add'>+	remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg));</div><div class='add'>+</div><div class='add'>+	if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) {</div><div class='add'>+		cm_id_priv = cm_find_remote_id(</div><div class='add'>+			*((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)),</div><div class='add'>+			remote_id);</div><div class='add'>+	} else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) ==</div><div class='add'>+		   CM_MSG_RESPONSE_REQ)</div><div class='add'>+		cm_id_priv = cm_acquire_id(</div><div class='add'>+			cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),</div><div class='add'>+			0);</div><div class='ctx'> 	else</div><div class='del'>-		cm_id_priv = cm_acquire_id(rej_msg-&gt;remote_comm_id, remote_id);</div><div class='add'>+		cm_id_priv = cm_acquire_id(</div><div class='add'>+			cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),</div><div class='add'>+			remote_id);</div><div class='ctx'> </div><div class='ctx'> 	return cm_id_priv;</div><div class='ctx'> }</div><div class='hunk'>@@ -2843,7 +3056,6 @@ static int cm_rej_handler(struct cm_work *work)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='ctx'> 	struct cm_rej_msg *rej_msg;</div><div class='del'>-	int ret;</div><div class='ctx'> </div><div class='ctx'> 	rej_msg = (struct cm_rej_msg *)work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='ctx'> 	cm_id_priv = cm_acquire_rejected_id(rej_msg);</div><div class='hunk'>@@ -2858,18 +3070,18 @@ static int cm_rej_handler(struct cm_work *work)</div><div class='ctx'> 	case IB_CM_MRA_REQ_RCVD:</div><div class='ctx'> 	case IB_CM_REP_SENT:</div><div class='ctx'> 	case IB_CM_MRA_REP_RCVD:</div><div class='del'>-		ib_cancel_mad(cm_id_priv-&gt;av.port-&gt;mad_agent, cm_id_priv-&gt;msg);</div><div class='del'>-		/* fall through */</div><div class='add'>+		ib_cancel_mad(cm_id_priv-&gt;msg);</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case IB_CM_REQ_RCVD:</div><div class='ctx'> 	case IB_CM_MRA_REQ_SENT:</div><div class='del'>-		if (__be16_to_cpu(rej_msg-&gt;reason) == IB_CM_REJ_STALE_CONN)</div><div class='add'>+		if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_STALE_CONN)</div><div class='ctx'> 			cm_enter_timewait(cm_id_priv);</div><div class='ctx'> 		else</div><div class='ctx'> 			cm_reset_to_idle(cm_id_priv);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_DREQ_SENT:</div><div class='del'>-		ib_cancel_mad(cm_id_priv-&gt;av.port-&gt;mad_agent, cm_id_priv-&gt;msg);</div><div class='del'>-		/* fall through */</div><div class='add'>+		ib_cancel_mad(cm_id_priv-&gt;msg);</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case IB_CM_REP_RCVD:</div><div class='ctx'> 	case IB_CM_MRA_REP_SENT:</div><div class='ctx'> 		cm_enter_timewait(cm_id_priv);</div><div class='hunk'>@@ -2878,127 +3090,79 @@ static int cm_rej_handler(struct cm_work *work)</div><div class='ctx'> 		if (cm_id_priv-&gt;id.lap_state == IB_CM_LAP_UNINIT ||</div><div class='ctx'> 		    cm_id_priv-&gt;id.lap_state == IB_CM_LAP_SENT) {</div><div class='ctx'> 			if (cm_id_priv-&gt;id.lap_state == IB_CM_LAP_SENT)</div><div class='del'>-				ib_cancel_mad(cm_id_priv-&gt;av.port-&gt;mad_agent,</div><div class='del'>-					      cm_id_priv-&gt;msg);</div><div class='add'>+				ib_cancel_mad(cm_id_priv-&gt;msg);</div><div class='ctx'> 			cm_enter_timewait(cm_id_priv);</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	default:</div><div class='add'>+		trace_icm_rej_unknown_err(&amp;cm_id_priv-&gt;id);</div><div class='ctx'> 		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-		pr_debug("%s: local_id %d, cm_id_priv-&gt;id.state: %d\n",</div><div class='del'>-			 __func__, be32_to_cpu(cm_id_priv-&gt;id.local_id),</div><div class='del'>-			 cm_id_priv-&gt;id.state);</div><div class='del'>-		ret = -EINVAL;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = atomic_inc_and_test(&amp;cm_id_priv-&gt;work_count);</div><div class='del'>-	if (!ret)</div><div class='del'>-		list_add_tail(&amp;work-&gt;list, &amp;cm_id_priv-&gt;work_list);</div><div class='del'>-	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-</div><div class='del'>-	if (ret)</div><div class='del'>-		cm_process_work(cm_id_priv, work);</div><div class='del'>-	else</div><div class='del'>-		cm_deref_id(cm_id_priv);</div><div class='add'>+	cm_queue_work_unlock(cm_id_priv, work);</div><div class='ctx'> 	return 0;</div><div class='ctx'> out:</div><div class='ctx'> 	cm_deref_id(cm_id_priv);</div><div class='ctx'> 	return -EINVAL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int ib_send_cm_mra(struct ib_cm_id *cm_id,</div><div class='del'>-		   u8 service_timeout,</div><div class='del'>-		   const void *private_data,</div><div class='del'>-		   u8 private_data_len)</div><div class='add'>+int ib_prepare_cm_mra(struct ib_cm_id *cm_id)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='del'>-	struct ib_mad_send_buf *msg;</div><div class='ctx'> 	enum ib_cm_state cm_state;</div><div class='ctx'> 	enum ib_cm_lap_state lap_state;</div><div class='del'>-	enum cm_msg_response msg_response;</div><div class='del'>-	void *data;</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if (private_data &amp;&amp; private_data_len &gt; IB_CM_MRA_PRIVATE_DATA_SIZE)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	data = cm_copy_private_data(private_data, private_data_len);</div><div class='del'>-	if (IS_ERR(data))</div><div class='del'>-		return PTR_ERR(data);</div><div class='add'>+	int ret = 0;</div><div class='ctx'> </div><div class='ctx'> 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	switch(cm_id_priv-&gt;id.state) {</div><div class='add'>+	switch (cm_id_priv-&gt;id.state) {</div><div class='ctx'> 	case IB_CM_REQ_RCVD:</div><div class='ctx'> 		cm_state = IB_CM_MRA_REQ_SENT;</div><div class='ctx'> 		lap_state = cm_id-&gt;lap_state;</div><div class='del'>-		msg_response = CM_MSG_RESPONSE_REQ;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_REP_RCVD:</div><div class='ctx'> 		cm_state = IB_CM_MRA_REP_SENT;</div><div class='ctx'> 		lap_state = cm_id-&gt;lap_state;</div><div class='del'>-		msg_response = CM_MSG_RESPONSE_REP;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_ESTABLISHED:</div><div class='ctx'> 		if (cm_id-&gt;lap_state == IB_CM_LAP_RCVD) {</div><div class='ctx'> 			cm_state = cm_id-&gt;state;</div><div class='ctx'> 			lap_state = IB_CM_MRA_LAP_SENT;</div><div class='del'>-			msg_response = CM_MSG_RESPONSE_OTHER;</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	default:</div><div class='del'>-		pr_debug("%s: local_id %d, cm_id_priv-&gt;id.state: %d\n",</div><div class='del'>-			 __func__, be32_to_cpu(cm_id_priv-&gt;id.local_id),</div><div class='del'>-			 cm_id_priv-&gt;id.state);</div><div class='add'>+		trace_icm_prepare_mra_unknown_err(&amp;cm_id_priv-&gt;id);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='del'>-		goto error1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!(service_timeout &amp; IB_CM_MRA_FLAG_DELAY)) {</div><div class='del'>-		ret = cm_alloc_msg(cm_id_priv, &amp;msg);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto error1;</div><div class='del'>-</div><div class='del'>-		cm_format_mra((struct cm_mra_msg *) msg-&gt;mad, cm_id_priv,</div><div class='del'>-			      msg_response, service_timeout,</div><div class='del'>-			      private_data, private_data_len);</div><div class='del'>-		ret = ib_post_send_mad(msg, NULL);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto error2;</div><div class='add'>+		goto error_unlock;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	cm_id-&gt;state = cm_state;</div><div class='ctx'> 	cm_id-&gt;lap_state = lap_state;</div><div class='del'>-	cm_id_priv-&gt;service_timeout = service_timeout;</div><div class='del'>-	cm_set_private_data(cm_id_priv, data, private_data_len);</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	return 0;</div><div class='add'>+	cm_set_private_data(cm_id_priv, NULL, 0);</div><div class='ctx'> </div><div class='del'>-error1:	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	kfree(data);</div><div class='del'>-	return ret;</div><div class='del'>-</div><div class='del'>-error2:	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	kfree(data);</div><div class='del'>-	cm_free_msg(msg);</div><div class='add'>+error_unlock:</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(ib_send_cm_mra);</div><div class='add'>+EXPORT_SYMBOL(ib_prepare_cm_mra);</div><div class='ctx'> </div><div class='del'>-static struct cm_id_private * cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)</div><div class='add'>+static struct cm_id_private *cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)</div><div class='ctx'> {</div><div class='del'>-	switch (cm_mra_get_msg_mraed(mra_msg)) {</div><div class='add'>+	switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) {</div><div class='ctx'> 	case CM_MSG_RESPONSE_REQ:</div><div class='del'>-		return cm_acquire_id(mra_msg-&gt;remote_comm_id, 0);</div><div class='add'>+		return cm_acquire_id(</div><div class='add'>+			cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),</div><div class='add'>+			0);</div><div class='ctx'> 	case CM_MSG_RESPONSE_REP:</div><div class='ctx'> 	case CM_MSG_RESPONSE_OTHER:</div><div class='del'>-		return cm_acquire_id(mra_msg-&gt;remote_comm_id,</div><div class='del'>-				     mra_msg-&gt;local_comm_id);</div><div class='add'>+		return cm_acquire_id(</div><div class='add'>+			cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),</div><div class='add'>+			cpu_to_be32(IBA_GET(CM_MRA_LOCAL_COMM_ID, mra_msg)));</div><div class='ctx'> 	default:</div><div class='ctx'> 		return NULL;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -3008,71 +3172,62 @@ static int cm_mra_handler(struct cm_work *work)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='ctx'> 	struct cm_mra_msg *mra_msg;</div><div class='del'>-	int timeout, ret;</div><div class='add'>+	int timeout;</div><div class='ctx'> </div><div class='ctx'> 	mra_msg = (struct cm_mra_msg *)work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='ctx'> 	cm_id_priv = cm_acquire_mraed_id(mra_msg);</div><div class='ctx'> 	if (!cm_id_priv)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	work-&gt;cm_event.private_data = &amp;mra_msg-&gt;private_data;</div><div class='add'>+	work-&gt;cm_event.private_data =</div><div class='add'>+		IBA_GET_MEM_PTR(CM_MRA_PRIVATE_DATA, mra_msg);</div><div class='ctx'> 	work-&gt;cm_event.param.mra_rcvd.service_timeout =</div><div class='del'>-					cm_mra_get_service_timeout(mra_msg);</div><div class='del'>-	timeout = cm_convert_to_ms(cm_mra_get_service_timeout(mra_msg)) +</div><div class='add'>+		IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg);</div><div class='add'>+	timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) +</div><div class='ctx'> 		  cm_convert_to_ms(cm_id_priv-&gt;av.timeout);</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> 	switch (cm_id_priv-&gt;id.state) {</div><div class='ctx'> 	case IB_CM_REQ_SENT:</div><div class='del'>-		if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REQ ||</div><div class='del'>-		    ib_modify_mad(cm_id_priv-&gt;av.port-&gt;mad_agent,</div><div class='del'>-				  cm_id_priv-&gt;msg, timeout))</div><div class='add'>+		if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=</div><div class='add'>+			    CM_MSG_RESPONSE_REQ ||</div><div class='add'>+		    ib_modify_mad(cm_id_priv-&gt;msg, timeout))</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		cm_id_priv-&gt;id.state = IB_CM_MRA_REQ_RCVD;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_REP_SENT:</div><div class='del'>-		if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REP ||</div><div class='del'>-		    ib_modify_mad(cm_id_priv-&gt;av.port-&gt;mad_agent,</div><div class='del'>-				  cm_id_priv-&gt;msg, timeout))</div><div class='add'>+		if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=</div><div class='add'>+			    CM_MSG_RESPONSE_REP ||</div><div class='add'>+		    ib_modify_mad(cm_id_priv-&gt;msg, timeout))</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		cm_id_priv-&gt;id.state = IB_CM_MRA_REP_RCVD;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_ESTABLISHED:</div><div class='del'>-		if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_OTHER ||</div><div class='add'>+		if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=</div><div class='add'>+			    CM_MSG_RESPONSE_OTHER ||</div><div class='ctx'> 		    cm_id_priv-&gt;id.lap_state != IB_CM_LAP_SENT ||</div><div class='del'>-		    ib_modify_mad(cm_id_priv-&gt;av.port-&gt;mad_agent,</div><div class='del'>-				  cm_id_priv-&gt;msg, timeout)) {</div><div class='add'>+		    ib_modify_mad(cm_id_priv-&gt;msg, timeout)) {</div><div class='ctx'> 			if (cm_id_priv-&gt;id.lap_state == IB_CM_MRA_LAP_RCVD)</div><div class='del'>-				atomic_long_inc(&amp;work-&gt;port-&gt;</div><div class='del'>-						counter_group[CM_RECV_DUPLICATES].</div><div class='del'>-						counter[CM_MRA_COUNTER]);</div><div class='add'>+				atomic_long_inc(</div><div class='add'>+					&amp;work-&gt;port-&gt;counters[CM_RECV_DUPLICATES]</div><div class='add'>+							     [CM_MRA_COUNTER]);</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		}</div><div class='ctx'> 		cm_id_priv-&gt;id.lap_state = IB_CM_MRA_LAP_RCVD;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_MRA_REQ_RCVD:</div><div class='ctx'> 	case IB_CM_MRA_REP_RCVD:</div><div class='del'>-		atomic_long_inc(&amp;work-&gt;port-&gt;counter_group[CM_RECV_DUPLICATES].</div><div class='del'>-				counter[CM_MRA_COUNTER]);</div><div class='del'>-		/* fall through */</div><div class='add'>+		atomic_long_inc(&amp;work-&gt;port-&gt;counters[CM_RECV_DUPLICATES]</div><div class='add'>+						     [CM_MRA_COUNTER]);</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	default:</div><div class='del'>-		pr_debug("%s local_id %d, cm_id_priv-&gt;id.state: %d\n",</div><div class='del'>-			 __func__, be32_to_cpu(cm_id_priv-&gt;id.local_id),</div><div class='del'>-			 cm_id_priv-&gt;id.state);</div><div class='add'>+		trace_icm_mra_unknown_err(&amp;cm_id_priv-&gt;id);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	cm_id_priv-&gt;msg-&gt;context[1] = (void *) (unsigned long)</div><div class='ctx'> 				      cm_id_priv-&gt;id.state;</div><div class='del'>-	ret = atomic_inc_and_test(&amp;cm_id_priv-&gt;work_count);</div><div class='del'>-	if (!ret)</div><div class='del'>-		list_add_tail(&amp;work-&gt;list, &amp;cm_id_priv-&gt;work_list);</div><div class='del'>-	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-</div><div class='del'>-	if (ret)</div><div class='del'>-		cm_process_work(cm_id_priv, work);</div><div class='del'>-	else</div><div class='del'>-		cm_deref_id(cm_id_priv);</div><div class='add'>+	cm_queue_work_unlock(cm_id_priv, work);</div><div class='ctx'> 	return 0;</div><div class='ctx'> out:</div><div class='ctx'> 	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='hunk'>@@ -3080,117 +3235,23 @@ out:</div><div class='ctx'> 	return -EINVAL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cm_format_lap(struct cm_lap_msg *lap_msg,</div><div class='del'>-			  struct cm_id_private *cm_id_priv,</div><div class='del'>-			  struct sa_path_rec *alternate_path,</div><div class='del'>-			  const void *private_data,</div><div class='del'>-			  u8 private_data_len)</div><div class='del'>-{</div><div class='del'>-	bool alt_ext = false;</div><div class='del'>-</div><div class='del'>-	if (alternate_path-&gt;rec_type == SA_PATH_REC_TYPE_OPA)</div><div class='del'>-		alt_ext = opa_is_extended_lid(alternate_path-&gt;opa.dlid,</div><div class='del'>-					      alternate_path-&gt;opa.slid);</div><div class='del'>-	cm_format_mad_hdr(&amp;lap_msg-&gt;hdr, CM_LAP_ATTR_ID,</div><div class='del'>-			  cm_form_tid(cm_id_priv));</div><div class='del'>-	lap_msg-&gt;local_comm_id = cm_id_priv-&gt;id.local_id;</div><div class='del'>-	lap_msg-&gt;remote_comm_id = cm_id_priv-&gt;id.remote_id;</div><div class='del'>-	cm_lap_set_remote_qpn(lap_msg, cm_id_priv-&gt;remote_qpn);</div><div class='del'>-	/* todo: need remote CM response timeout */</div><div class='del'>-	cm_lap_set_remote_resp_timeout(lap_msg, 0x1F);</div><div class='del'>-	lap_msg-&gt;alt_local_lid =</div><div class='del'>-		htons(ntohl(sa_path_get_slid(alternate_path)));</div><div class='del'>-	lap_msg-&gt;alt_remote_lid =</div><div class='del'>-		htons(ntohl(sa_path_get_dlid(alternate_path)));</div><div class='del'>-	lap_msg-&gt;alt_local_gid = alternate_path-&gt;sgid;</div><div class='del'>-	lap_msg-&gt;alt_remote_gid = alternate_path-&gt;dgid;</div><div class='del'>-	if (alt_ext) {</div><div class='del'>-		lap_msg-&gt;alt_local_gid.global.interface_id</div><div class='del'>-			= OPA_MAKE_ID(be32_to_cpu(alternate_path-&gt;opa.slid));</div><div class='del'>-		lap_msg-&gt;alt_remote_gid.global.interface_id</div><div class='del'>-			= OPA_MAKE_ID(be32_to_cpu(alternate_path-&gt;opa.dlid));</div><div class='del'>-	}</div><div class='del'>-	cm_lap_set_flow_label(lap_msg, alternate_path-&gt;flow_label);</div><div class='del'>-	cm_lap_set_traffic_class(lap_msg, alternate_path-&gt;traffic_class);</div><div class='del'>-	lap_msg-&gt;alt_hop_limit = alternate_path-&gt;hop_limit;</div><div class='del'>-	cm_lap_set_packet_rate(lap_msg, alternate_path-&gt;rate);</div><div class='del'>-	cm_lap_set_sl(lap_msg, alternate_path-&gt;sl);</div><div class='del'>-	cm_lap_set_subnet_local(lap_msg, 1); /* local only... */</div><div class='del'>-	cm_lap_set_local_ack_timeout(lap_msg,</div><div class='del'>-		cm_ack_timeout(cm_id_priv-&gt;av.port-&gt;cm_dev-&gt;ack_delay,</div><div class='del'>-			       alternate_path-&gt;packet_life_time));</div><div class='del'>-</div><div class='del'>-	if (private_data &amp;&amp; private_data_len)</div><div class='del'>-		memcpy(lap_msg-&gt;private_data, private_data, private_data_len);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int ib_send_cm_lap(struct ib_cm_id *cm_id,</div><div class='del'>-		   struct sa_path_rec *alternate_path,</div><div class='del'>-		   const void *private_data,</div><div class='del'>-		   u8 private_data_len)</div><div class='del'>-{</div><div class='del'>-	struct cm_id_private *cm_id_priv;</div><div class='del'>-	struct ib_mad_send_buf *msg;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if (private_data &amp;&amp; private_data_len &gt; IB_CM_LAP_PRIVATE_DATA_SIZE)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	cm_id_priv = container_of(cm_id, struct cm_id_private, id);</div><div class='del'>-	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	if (cm_id-&gt;state != IB_CM_ESTABLISHED ||</div><div class='del'>-	    (cm_id-&gt;lap_state != IB_CM_LAP_UNINIT &amp;&amp;</div><div class='del'>-	     cm_id-&gt;lap_state != IB_CM_LAP_IDLE)) {</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = cm_init_av_by_path(alternate_path, NULL, &amp;cm_id_priv-&gt;alt_av,</div><div class='del'>-				 cm_id_priv);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto out;</div><div class='del'>-	cm_id_priv-&gt;alt_av.timeout =</div><div class='del'>-			cm_ack_timeout(cm_id_priv-&gt;target_ack_delay,</div><div class='del'>-				       cm_id_priv-&gt;alt_av.timeout - 1);</div><div class='del'>-</div><div class='del'>-	ret = cm_alloc_msg(cm_id_priv, &amp;msg);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto out;</div><div class='del'>-</div><div class='del'>-	cm_format_lap((struct cm_lap_msg *) msg-&gt;mad, cm_id_priv,</div><div class='del'>-		      alternate_path, private_data, private_data_len);</div><div class='del'>-	msg-&gt;timeout_ms = cm_id_priv-&gt;timeout_ms;</div><div class='del'>-	msg-&gt;context[1] = (void *) (unsigned long) IB_CM_ESTABLISHED;</div><div class='del'>-</div><div class='del'>-	ret = ib_post_send_mad(msg, NULL);</div><div class='del'>-	if (ret) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-		cm_free_msg(msg);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cm_id-&gt;lap_state = IB_CM_LAP_SENT;</div><div class='del'>-	cm_id_priv-&gt;msg = msg;</div><div class='del'>-</div><div class='del'>-out:	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_send_cm_lap);</div><div class='del'>-</div><div class='ctx'> static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg,</div><div class='ctx'> 					struct sa_path_rec *path)</div><div class='ctx'> {</div><div class='ctx'> 	u32 lid;</div><div class='ctx'> </div><div class='ctx'> 	if (path-&gt;rec_type != SA_PATH_REC_TYPE_OPA) {</div><div class='del'>-		sa_path_set_dlid(path, ntohs(lap_msg-&gt;alt_local_lid));</div><div class='del'>-		sa_path_set_slid(path, ntohs(lap_msg-&gt;alt_remote_lid));</div><div class='add'>+		sa_path_set_dlid(path, IBA_GET(CM_LAP_ALTERNATE_LOCAL_PORT_LID,</div><div class='add'>+					       lap_msg));</div><div class='add'>+		sa_path_set_slid(path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID,</div><div class='add'>+					       lap_msg));</div><div class='ctx'> 	} else {</div><div class='del'>-		lid = opa_get_lid_from_gid(&amp;lap_msg-&gt;alt_local_gid);</div><div class='add'>+		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(</div><div class='add'>+			CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg));</div><div class='ctx'> 		sa_path_set_dlid(path, lid);</div><div class='ctx'> </div><div class='del'>-		lid = opa_get_lid_from_gid(&amp;lap_msg-&gt;alt_remote_gid);</div><div class='add'>+		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(</div><div class='add'>+			CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg));</div><div class='ctx'> 		sa_path_set_slid(path, lid);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='hunk'>@@ -3199,20 +3260,23 @@ static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,</div><div class='ctx'> 				    struct sa_path_rec *path,</div><div class='ctx'> 				    struct cm_lap_msg *lap_msg)</div><div class='ctx'> {</div><div class='del'>-	path-&gt;dgid = lap_msg-&gt;alt_local_gid;</div><div class='del'>-	path-&gt;sgid = lap_msg-&gt;alt_remote_gid;</div><div class='del'>-	path-&gt;flow_label = cm_lap_get_flow_label(lap_msg);</div><div class='del'>-	path-&gt;hop_limit = lap_msg-&gt;alt_hop_limit;</div><div class='del'>-	path-&gt;traffic_class = cm_lap_get_traffic_class(lap_msg);</div><div class='add'>+	path-&gt;dgid = *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg);</div><div class='add'>+	path-&gt;sgid =</div><div class='add'>+		*IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg);</div><div class='add'>+	path-&gt;flow_label =</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg));</div><div class='add'>+	path-&gt;hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg);</div><div class='add'>+	path-&gt;traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg);</div><div class='ctx'> 	path-&gt;reversible = 1;</div><div class='ctx'> 	path-&gt;pkey = cm_id_priv-&gt;pkey;</div><div class='del'>-	path-&gt;sl = cm_lap_get_sl(lap_msg);</div><div class='add'>+	path-&gt;sl = IBA_GET(CM_LAP_ALTERNATE_SL, lap_msg);</div><div class='ctx'> 	path-&gt;mtu_selector = IB_SA_EQ;</div><div class='ctx'> 	path-&gt;mtu = cm_id_priv-&gt;path_mtu;</div><div class='ctx'> 	path-&gt;rate_selector = IB_SA_EQ;</div><div class='del'>-	path-&gt;rate = cm_lap_get_packet_rate(lap_msg);</div><div class='add'>+	path-&gt;rate = IBA_GET(CM_LAP_ALTERNATE_PACKET_RATE, lap_msg);</div><div class='ctx'> 	path-&gt;packet_life_time_selector = IB_SA_EQ;</div><div class='del'>-	path-&gt;packet_life_time = cm_lap_get_local_ack_timeout(lap_msg);</div><div class='add'>+	path-&gt;packet_life_time =</div><div class='add'>+		IBA_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, lap_msg);</div><div class='ctx'> 	path-&gt;packet_life_time -= (path-&gt;packet_life_time &gt; 0);</div><div class='ctx'> 	cm_format_path_lid_from_lap(lap_msg, path);</div><div class='ctx'> }</div><div class='hunk'>@@ -3223,6 +3287,8 @@ static int cm_lap_handler(struct cm_work *work)</div><div class='ctx'> 	struct cm_lap_msg *lap_msg;</div><div class='ctx'> 	struct ib_cm_lap_event_param *param;</div><div class='ctx'> 	struct ib_mad_send_buf *msg = NULL;</div><div class='add'>+	struct rdma_ah_attr ah_attr;</div><div class='add'>+	struct cm_av alt_av = {};</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	/* Currently Alternate path messages are not supported for</div><div class='hunk'>@@ -3234,22 +3300,42 @@ static int cm_lap_handler(struct cm_work *work)</div><div class='ctx'> </div><div class='ctx'> 	/* todo: verify LAP request and send reject APR if invalid. */</div><div class='ctx'> 	lap_msg = (struct cm_lap_msg *)work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='del'>-	cm_id_priv = cm_acquire_id(lap_msg-&gt;remote_comm_id,</div><div class='del'>-				   lap_msg-&gt;local_comm_id);</div><div class='add'>+	cm_id_priv = cm_acquire_id(</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_LAP_REMOTE_COMM_ID, lap_msg)),</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_LAP_LOCAL_COMM_ID, lap_msg)));</div><div class='ctx'> 	if (!cm_id_priv)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	param = &amp;work-&gt;cm_event.param.lap_rcvd;</div><div class='ctx'> 	memset(&amp;work-&gt;path[0], 0, sizeof(work-&gt;path[1]));</div><div class='ctx'> 	cm_path_set_rec_type(work-&gt;port-&gt;cm_dev-&gt;ib_device,</div><div class='del'>-			     work-&gt;port-&gt;port_num,</div><div class='del'>-			     &amp;work-&gt;path[0],</div><div class='del'>-			     &amp;lap_msg-&gt;alt_local_gid);</div><div class='add'>+			     work-&gt;port-&gt;port_num, &amp;work-&gt;path[0],</div><div class='add'>+			     IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID,</div><div class='add'>+					     lap_msg));</div><div class='ctx'> 	param-&gt;alternate_path = &amp;work-&gt;path[0];</div><div class='ctx'> 	cm_format_path_from_lap(cm_id_priv, param-&gt;alternate_path, lap_msg);</div><div class='del'>-	work-&gt;cm_event.private_data = &amp;lap_msg-&gt;private_data;</div><div class='add'>+	work-&gt;cm_event.private_data =</div><div class='add'>+		IBA_GET_MEM_PTR(CM_LAP_PRIVATE_DATA, lap_msg);</div><div class='add'>+</div><div class='add'>+	ret = ib_init_ah_attr_from_wc(work-&gt;port-&gt;cm_dev-&gt;ib_device,</div><div class='add'>+				      work-&gt;port-&gt;port_num,</div><div class='add'>+				      work-&gt;mad_recv_wc-&gt;wc,</div><div class='add'>+				      work-&gt;mad_recv_wc-&gt;recv_buf.grh,</div><div class='add'>+				      &amp;ah_attr);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto deref;</div><div class='add'>+</div><div class='add'>+	ret = cm_init_av_by_path(param-&gt;alternate_path, NULL, &amp;alt_av);</div><div class='add'>+	if (ret) {</div><div class='add'>+		rdma_destroy_ah_attr(&amp;ah_attr);</div><div class='add'>+		goto deref;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+	cm_init_av_for_lap(work-&gt;port, work-&gt;mad_recv_wc-&gt;wc,</div><div class='add'>+			   &amp;ah_attr, &amp;cm_id_priv-&gt;av);</div><div class='add'>+	cm_move_av_from_path(&amp;cm_id_priv-&gt;alt_av, &amp;alt_av);</div><div class='add'>+</div><div class='ctx'> 	if (cm_id_priv-&gt;id.state != IB_CM_ESTABLISHED)</div><div class='ctx'> 		goto unlock;</div><div class='ctx'> </div><div class='hunk'>@@ -3258,15 +3344,15 @@ static int cm_lap_handler(struct cm_work *work)</div><div class='ctx'> 	case IB_CM_LAP_IDLE:</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_MRA_LAP_SENT:</div><div class='del'>-		atomic_long_inc(&amp;work-&gt;port-&gt;counter_group[CM_RECV_DUPLICATES].</div><div class='del'>-				counter[CM_LAP_COUNTER]);</div><div class='del'>-		msg = cm_alloc_response_msg_no_ah(work-&gt;port, work-&gt;mad_recv_wc);</div><div class='add'>+		atomic_long_inc(&amp;work-&gt;port-&gt;counters[CM_RECV_DUPLICATES]</div><div class='add'>+						     [CM_LAP_COUNTER]);</div><div class='add'>+		msg = cm_alloc_response_msg_no_ah(work-&gt;port, work-&gt;mad_recv_wc,</div><div class='add'>+						  true);</div><div class='ctx'> 		if (IS_ERR(msg))</div><div class='ctx'> 			goto unlock;</div><div class='ctx'> </div><div class='ctx'> 		cm_format_mra((struct cm_mra_msg *) msg-&gt;mad, cm_id_priv,</div><div class='ctx'> 			      CM_MSG_RESPONSE_OTHER,</div><div class='del'>-			      cm_id_priv-&gt;service_timeout,</div><div class='ctx'> 			      cm_id_priv-&gt;private_data,</div><div class='ctx'> 			      cm_id_priv-&gt;private_data_len);</div><div class='ctx'> 		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='hunk'>@@ -3276,35 +3362,16 @@ static int cm_lap_handler(struct cm_work *work)</div><div class='ctx'> 			cm_free_msg(msg);</div><div class='ctx'> 		goto deref;</div><div class='ctx'> 	case IB_CM_LAP_RCVD:</div><div class='del'>-		atomic_long_inc(&amp;work-&gt;port-&gt;counter_group[CM_RECV_DUPLICATES].</div><div class='del'>-				counter[CM_LAP_COUNTER]);</div><div class='add'>+		atomic_long_inc(&amp;work-&gt;port-&gt;counters[CM_RECV_DUPLICATES]</div><div class='add'>+						     [CM_LAP_COUNTER]);</div><div class='ctx'> 		goto unlock;</div><div class='ctx'> 	default:</div><div class='ctx'> 		goto unlock;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = cm_init_av_for_lap(work-&gt;port, work-&gt;mad_recv_wc-&gt;wc,</div><div class='del'>-				 work-&gt;mad_recv_wc-&gt;recv_buf.grh,</div><div class='del'>-				 &amp;cm_id_priv-&gt;av);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto unlock;</div><div class='del'>-</div><div class='del'>-	ret = cm_init_av_by_path(param-&gt;alternate_path, NULL,</div><div class='del'>-				 &amp;cm_id_priv-&gt;alt_av, cm_id_priv);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto unlock;</div><div class='del'>-</div><div class='ctx'> 	cm_id_priv-&gt;id.lap_state = IB_CM_LAP_RCVD;</div><div class='ctx'> 	cm_id_priv-&gt;tid = lap_msg-&gt;hdr.tid;</div><div class='del'>-	ret = atomic_inc_and_test(&amp;cm_id_priv-&gt;work_count);</div><div class='del'>-	if (!ret)</div><div class='del'>-		list_add_tail(&amp;work-&gt;list, &amp;cm_id_priv-&gt;work_list);</div><div class='del'>-	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-</div><div class='del'>-	if (ret)</div><div class='del'>-		cm_process_work(cm_id_priv, work);</div><div class='del'>-	else</div><div class='del'>-		cm_deref_id(cm_id_priv);</div><div class='add'>+	cm_queue_work_unlock(cm_id_priv, work);</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> unlock:	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='hunk'>@@ -3312,77 +3379,10 @@ deref:	cm_deref_id(cm_id_priv);</div><div class='ctx'> 	return -EINVAL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cm_format_apr(struct cm_apr_msg *apr_msg,</div><div class='del'>-			  struct cm_id_private *cm_id_priv,</div><div class='del'>-			  enum ib_cm_apr_status status,</div><div class='del'>-			  void *info,</div><div class='del'>-			  u8 info_length,</div><div class='del'>-			  const void *private_data,</div><div class='del'>-			  u8 private_data_len)</div><div class='del'>-{</div><div class='del'>-	cm_format_mad_hdr(&amp;apr_msg-&gt;hdr, CM_APR_ATTR_ID, cm_id_priv-&gt;tid);</div><div class='del'>-	apr_msg-&gt;local_comm_id = cm_id_priv-&gt;id.local_id;</div><div class='del'>-	apr_msg-&gt;remote_comm_id = cm_id_priv-&gt;id.remote_id;</div><div class='del'>-	apr_msg-&gt;ap_status = (u8) status;</div><div class='del'>-</div><div class='del'>-	if (info &amp;&amp; info_length) {</div><div class='del'>-		apr_msg-&gt;info_length = info_length;</div><div class='del'>-		memcpy(apr_msg-&gt;info, info, info_length);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (private_data &amp;&amp; private_data_len)</div><div class='del'>-		memcpy(apr_msg-&gt;private_data, private_data, private_data_len);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int ib_send_cm_apr(struct ib_cm_id *cm_id,</div><div class='del'>-		   enum ib_cm_apr_status status,</div><div class='del'>-		   void *info,</div><div class='del'>-		   u8 info_length,</div><div class='del'>-		   const void *private_data,</div><div class='del'>-		   u8 private_data_len)</div><div class='del'>-{</div><div class='del'>-	struct cm_id_private *cm_id_priv;</div><div class='del'>-	struct ib_mad_send_buf *msg;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if ((private_data &amp;&amp; private_data_len &gt; IB_CM_APR_PRIVATE_DATA_SIZE) ||</div><div class='del'>-	    (info &amp;&amp; info_length &gt; IB_CM_APR_INFO_LENGTH))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	cm_id_priv = container_of(cm_id, struct cm_id_private, id);</div><div class='del'>-	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	if (cm_id-&gt;state != IB_CM_ESTABLISHED ||</div><div class='del'>-	    (cm_id-&gt;lap_state != IB_CM_LAP_RCVD &amp;&amp;</div><div class='del'>-	     cm_id-&gt;lap_state != IB_CM_MRA_LAP_SENT)) {</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = cm_alloc_msg(cm_id_priv, &amp;msg);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto out;</div><div class='del'>-</div><div class='del'>-	cm_format_apr((struct cm_apr_msg *) msg-&gt;mad, cm_id_priv, status,</div><div class='del'>-		      info, info_length, private_data, private_data_len);</div><div class='del'>-	ret = ib_post_send_mad(msg, NULL);</div><div class='del'>-	if (ret) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-		cm_free_msg(msg);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cm_id-&gt;lap_state = IB_CM_LAP_IDLE;</div><div class='del'>-out:	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_send_cm_apr);</div><div class='del'>-</div><div class='ctx'> static int cm_apr_handler(struct cm_work *work)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='ctx'> 	struct cm_apr_msg *apr_msg;</div><div class='del'>-	int ret;</div><div class='ctx'> </div><div class='ctx'> 	/* Currently Alternate path messages are not supported for</div><div class='ctx'> 	 * RoCE link layer.</div><div class='hunk'>@@ -3392,15 +3392,20 @@ static int cm_apr_handler(struct cm_work *work)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	apr_msg = (struct cm_apr_msg *)work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='del'>-	cm_id_priv = cm_acquire_id(apr_msg-&gt;remote_comm_id,</div><div class='del'>-				   apr_msg-&gt;local_comm_id);</div><div class='add'>+	cm_id_priv = cm_acquire_id(</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_APR_REMOTE_COMM_ID, apr_msg)),</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_APR_LOCAL_COMM_ID, apr_msg)));</div><div class='ctx'> 	if (!cm_id_priv)</div><div class='ctx'> 		return -EINVAL; /* Unmatched reply. */</div><div class='ctx'> </div><div class='del'>-	work-&gt;cm_event.param.apr_rcvd.ap_status = apr_msg-&gt;ap_status;</div><div class='del'>-	work-&gt;cm_event.param.apr_rcvd.apr_info = &amp;apr_msg-&gt;info;</div><div class='del'>-	work-&gt;cm_event.param.apr_rcvd.info_len = apr_msg-&gt;info_length;</div><div class='del'>-	work-&gt;cm_event.private_data = &amp;apr_msg-&gt;private_data;</div><div class='add'>+	work-&gt;cm_event.param.apr_rcvd.ap_status =</div><div class='add'>+		IBA_GET(CM_APR_AR_STATUS, apr_msg);</div><div class='add'>+	work-&gt;cm_event.param.apr_rcvd.apr_info =</div><div class='add'>+		IBA_GET_MEM_PTR(CM_APR_ADDITIONAL_INFORMATION, apr_msg);</div><div class='add'>+	work-&gt;cm_event.param.apr_rcvd.info_len =</div><div class='add'>+		IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg);</div><div class='add'>+	work-&gt;cm_event.private_data =</div><div class='add'>+		IBA_GET_MEM_PTR(CM_APR_PRIVATE_DATA, apr_msg);</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> 	if (cm_id_priv-&gt;id.state != IB_CM_ESTABLISHED ||</div><div class='hunk'>@@ -3410,18 +3415,8 @@ static int cm_apr_handler(struct cm_work *work)</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> 	cm_id_priv-&gt;id.lap_state = IB_CM_LAP_IDLE;</div><div class='del'>-	ib_cancel_mad(cm_id_priv-&gt;av.port-&gt;mad_agent, cm_id_priv-&gt;msg);</div><div class='del'>-	cm_id_priv-&gt;msg = NULL;</div><div class='del'>-</div><div class='del'>-	ret = atomic_inc_and_test(&amp;cm_id_priv-&gt;work_count);</div><div class='del'>-	if (!ret)</div><div class='del'>-		list_add_tail(&amp;work-&gt;list, &amp;cm_id_priv-&gt;work_list);</div><div class='del'>-	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-</div><div class='del'>-	if (ret)</div><div class='del'>-		cm_process_work(cm_id_priv, work);</div><div class='del'>-	else</div><div class='del'>-		cm_deref_id(cm_id_priv);</div><div class='add'>+	ib_cancel_mad(cm_id_priv-&gt;msg);</div><div class='add'>+	cm_queue_work_unlock(cm_id_priv, work);</div><div class='ctx'> 	return 0;</div><div class='ctx'> out:</div><div class='ctx'> 	cm_deref_id(cm_id_priv);</div><div class='hunk'>@@ -3432,9 +3427,8 @@ static int cm_timewait_handler(struct cm_work *work)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_timewait_info *timewait_info;</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='del'>-	int ret;</div><div class='ctx'> </div><div class='del'>-	timewait_info = (struct cm_timewait_info *)work;</div><div class='add'>+	timewait_info = container_of(work, struct cm_timewait_info, work);</div><div class='ctx'> 	spin_lock_irq(&amp;cm.lock);</div><div class='ctx'> 	list_del(&amp;timewait_info-&gt;list);</div><div class='ctx'> 	spin_unlock_irq(&amp;cm.lock);</div><div class='hunk'>@@ -3451,15 +3445,7 @@ static int cm_timewait_handler(struct cm_work *work)</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> 	cm_id_priv-&gt;id.state = IB_CM_IDLE;</div><div class='del'>-	ret = atomic_inc_and_test(&amp;cm_id_priv-&gt;work_count);</div><div class='del'>-	if (!ret)</div><div class='del'>-		list_add_tail(&amp;work-&gt;list, &amp;cm_id_priv-&gt;work_list);</div><div class='del'>-	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-</div><div class='del'>-	if (ret)</div><div class='del'>-		cm_process_work(cm_id_priv, work);</div><div class='del'>-	else</div><div class='del'>-		cm_deref_id(cm_id_priv);</div><div class='add'>+	cm_queue_work_unlock(cm_id_priv, work);</div><div class='ctx'> 	return 0;</div><div class='ctx'> out:</div><div class='ctx'> 	cm_deref_id(cm_id_priv);</div><div class='hunk'>@@ -3472,13 +3458,16 @@ static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg,</div><div class='ctx'> {</div><div class='ctx'> 	cm_format_mad_hdr(&amp;sidr_req_msg-&gt;hdr, CM_SIDR_REQ_ATTR_ID,</div><div class='ctx'> 			  cm_form_tid(cm_id_priv));</div><div class='del'>-	sidr_req_msg-&gt;request_id = cm_id_priv-&gt;id.local_id;</div><div class='del'>-	sidr_req_msg-&gt;pkey = param-&gt;path-&gt;pkey;</div><div class='del'>-	sidr_req_msg-&gt;service_id = param-&gt;service_id;</div><div class='add'>+	IBA_SET(CM_SIDR_REQ_REQUESTID, sidr_req_msg,</div><div class='add'>+		be32_to_cpu(cm_id_priv-&gt;id.local_id));</div><div class='add'>+	IBA_SET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg,</div><div class='add'>+		be16_to_cpu(param-&gt;path-&gt;pkey));</div><div class='add'>+	IBA_SET(CM_SIDR_REQ_SERVICEID, sidr_req_msg,</div><div class='add'>+		be64_to_cpu(param-&gt;service_id));</div><div class='ctx'> </div><div class='ctx'> 	if (param-&gt;private_data &amp;&amp; param-&gt;private_data_len)</div><div class='del'>-		memcpy(sidr_req_msg-&gt;private_data, param-&gt;private_data,</div><div class='del'>-		       param-&gt;private_data_len);</div><div class='add'>+		IBA_SET_MEM(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg,</div><div class='add'>+			    param-&gt;private_data, param-&gt;private_data_len);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,</div><div class='hunk'>@@ -3486,6 +3475,7 @@ int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='ctx'> 	struct ib_mad_send_buf *msg;</div><div class='add'>+	struct cm_av av = {};</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='hunk'>@@ -3494,40 +3484,40 @@ int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);</div><div class='del'>-	ret = cm_init_av_by_path(param-&gt;path, param-&gt;sgid_attr,</div><div class='del'>-				 &amp;cm_id_priv-&gt;av,</div><div class='del'>-				 cm_id_priv);</div><div class='add'>+	ret = cm_init_av_by_path(param-&gt;path, param-&gt;sgid_attr, &amp;av);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto out;</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='add'>+	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+	cm_move_av_from_path(&amp;cm_id_priv-&gt;av, &amp;av);</div><div class='ctx'> 	cm_id-&gt;service_id = param-&gt;service_id;</div><div class='del'>-	cm_id-&gt;service_mask = ~cpu_to_be64(0);</div><div class='ctx'> 	cm_id_priv-&gt;timeout_ms = param-&gt;timeout_ms;</div><div class='ctx'> 	cm_id_priv-&gt;max_cm_retries = param-&gt;max_cm_retries;</div><div class='del'>-	ret = cm_alloc_msg(cm_id_priv, &amp;msg);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto out;</div><div class='del'>-</div><div class='del'>-	cm_format_sidr_req((struct cm_sidr_req_msg *) msg-&gt;mad, cm_id_priv,</div><div class='del'>-			   param);</div><div class='del'>-	msg-&gt;timeout_ms = cm_id_priv-&gt;timeout_ms;</div><div class='del'>-	msg-&gt;context[1] = (void *) (unsigned long) IB_CM_SIDR_REQ_SENT;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	if (cm_id-&gt;state == IB_CM_IDLE)</div><div class='del'>-		ret = ib_post_send_mad(msg, NULL);</div><div class='del'>-	else</div><div class='add'>+	if (cm_id-&gt;state != IB_CM_IDLE) {</div><div class='ctx'> 		ret = -EINVAL;</div><div class='add'>+		goto out_unlock;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (ret) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-		cm_free_msg(msg);</div><div class='del'>-		goto out;</div><div class='add'>+	msg = cm_alloc_priv_msg(cm_id_priv, IB_CM_SIDR_REQ_SENT);</div><div class='add'>+	if (IS_ERR(msg)) {</div><div class='add'>+		ret = PTR_ERR(msg);</div><div class='add'>+		goto out_unlock;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	cm_format_sidr_req((struct cm_sidr_req_msg *)msg-&gt;mad, cm_id_priv,</div><div class='add'>+			   param);</div><div class='add'>+</div><div class='add'>+	trace_icm_send_sidr_req(&amp;cm_id_priv-&gt;id);</div><div class='add'>+	ret = ib_post_send_mad(msg, NULL);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out_free;</div><div class='ctx'> 	cm_id-&gt;state = IB_CM_SIDR_REQ_SENT;</div><div class='del'>-	cm_id_priv-&gt;msg = msg;</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-out:</div><div class='add'>+	return 0;</div><div class='add'>+out_free:</div><div class='add'>+	cm_free_priv_msg(msg);</div><div class='add'>+out_unlock:</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_send_cm_sidr_req);</div><div class='hunk'>@@ -3542,72 +3532,86 @@ static void cm_format_sidr_req_event(struct cm_work *work,</div><div class='ctx'> 	sidr_req_msg = (struct cm_sidr_req_msg *)</div><div class='ctx'> 				work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='ctx'> 	param = &amp;work-&gt;cm_event.param.sidr_req_rcvd;</div><div class='del'>-	param-&gt;pkey = __be16_to_cpu(sidr_req_msg-&gt;pkey);</div><div class='add'>+	param-&gt;pkey = IBA_GET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg);</div><div class='ctx'> 	param-&gt;listen_id = listen_id;</div><div class='del'>-	param-&gt;service_id = sidr_req_msg-&gt;service_id;</div><div class='add'>+	param-&gt;service_id =</div><div class='add'>+		cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));</div><div class='ctx'> 	param-&gt;bth_pkey = cm_get_bth_pkey(work);</div><div class='ctx'> 	param-&gt;port = work-&gt;port-&gt;port_num;</div><div class='ctx'> 	param-&gt;sgid_attr = rx_cm_id-&gt;av.ah_attr.grh.sgid_attr;</div><div class='del'>-	work-&gt;cm_event.private_data = &amp;sidr_req_msg-&gt;private_data;</div><div class='add'>+	work-&gt;cm_event.private_data =</div><div class='add'>+		IBA_GET_MEM_PTR(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int cm_sidr_req_handler(struct cm_work *work)</div><div class='ctx'> {</div><div class='del'>-	struct ib_cm_id *cm_id;</div><div class='del'>-	struct cm_id_private *cm_id_priv, *cur_cm_id_priv;</div><div class='add'>+	struct cm_id_private *cm_id_priv, *listen_cm_id_priv;</div><div class='ctx'> 	struct cm_sidr_req_msg *sidr_req_msg;</div><div class='ctx'> 	struct ib_wc *wc;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	cm_id = ib_create_cm_id(work-&gt;port-&gt;cm_dev-&gt;ib_device, NULL, NULL);</div><div class='del'>-	if (IS_ERR(cm_id))</div><div class='del'>-		return PTR_ERR(cm_id);</div><div class='del'>-	cm_id_priv = container_of(cm_id, struct cm_id_private, id);</div><div class='add'>+	cm_id_priv =</div><div class='add'>+		cm_alloc_id_priv(work-&gt;port-&gt;cm_dev-&gt;ib_device, NULL, NULL);</div><div class='add'>+	if (IS_ERR(cm_id_priv))</div><div class='add'>+		return PTR_ERR(cm_id_priv);</div><div class='ctx'> </div><div class='ctx'> 	/* Record SGID/SLID and request ID for lookup. */</div><div class='ctx'> 	sidr_req_msg = (struct cm_sidr_req_msg *)</div><div class='ctx'> 				work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='add'>+</div><div class='add'>+	cm_id_priv-&gt;id.remote_id =</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg));</div><div class='add'>+	cm_id_priv-&gt;id.service_id =</div><div class='add'>+		cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));</div><div class='add'>+	cm_id_priv-&gt;tid = sidr_req_msg-&gt;hdr.tid;</div><div class='add'>+</div><div class='ctx'> 	wc = work-&gt;mad_recv_wc-&gt;wc;</div><div class='del'>-	cm_id_priv-&gt;av.dgid.global.subnet_prefix = cpu_to_be64(wc-&gt;slid);</div><div class='del'>-	cm_id_priv-&gt;av.dgid.global.interface_id = 0;</div><div class='add'>+	cm_id_priv-&gt;sidr_slid = wc-&gt;slid;</div><div class='ctx'> 	ret = cm_init_av_for_response(work-&gt;port, work-&gt;mad_recv_wc-&gt;wc,</div><div class='ctx'> 				      work-&gt;mad_recv_wc-&gt;recv_buf.grh,</div><div class='ctx'> 				      &amp;cm_id_priv-&gt;av);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	cm_id_priv-&gt;id.remote_id = sidr_req_msg-&gt;request_id;</div><div class='del'>-	cm_id_priv-&gt;tid = sidr_req_msg-&gt;hdr.tid;</div><div class='del'>-	atomic_inc(&amp;cm_id_priv-&gt;work_count);</div><div class='del'>-</div><div class='ctx'> 	spin_lock_irq(&amp;cm.lock);</div><div class='del'>-	cur_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);</div><div class='del'>-	if (cur_cm_id_priv) {</div><div class='add'>+	listen_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);</div><div class='add'>+	if (listen_cm_id_priv) {</div><div class='ctx'> 		spin_unlock_irq(&amp;cm.lock);</div><div class='del'>-		atomic_long_inc(&amp;work-&gt;port-&gt;counter_group[CM_RECV_DUPLICATES].</div><div class='del'>-				counter[CM_SIDR_REQ_COUNTER]);</div><div class='add'>+		atomic_long_inc(&amp;work-&gt;port-&gt;counters[CM_RECV_DUPLICATES]</div><div class='add'>+						     [CM_SIDR_REQ_COUNTER]);</div><div class='ctx'> 		goto out; /* Duplicate message. */</div><div class='ctx'> 	}</div><div class='ctx'> 	cm_id_priv-&gt;id.state = IB_CM_SIDR_REQ_RCVD;</div><div class='del'>-	cur_cm_id_priv = cm_find_listen(cm_id-&gt;device,</div><div class='del'>-					sidr_req_msg-&gt;service_id);</div><div class='del'>-	if (!cur_cm_id_priv) {</div><div class='add'>+	listen_cm_id_priv = cm_find_listen(cm_id_priv-&gt;id.device,</div><div class='add'>+					   cm_id_priv-&gt;id.service_id);</div><div class='add'>+	if (!listen_cm_id_priv) {</div><div class='ctx'> 		spin_unlock_irq(&amp;cm.lock);</div><div class='del'>-		cm_reject_sidr_req(cm_id_priv, IB_SIDR_UNSUPPORTED);</div><div class='add'>+		ib_send_cm_sidr_rep(&amp;cm_id_priv-&gt;id,</div><div class='add'>+				    &amp;(struct ib_cm_sidr_rep_param){</div><div class='add'>+					    .status = IB_SIDR_UNSUPPORTED });</div><div class='ctx'> 		goto out; /* No match. */</div><div class='ctx'> 	}</div><div class='del'>-	atomic_inc(&amp;cur_cm_id_priv-&gt;refcount);</div><div class='del'>-	atomic_inc(&amp;cm_id_priv-&gt;refcount);</div><div class='ctx'> 	spin_unlock_irq(&amp;cm.lock);</div><div class='ctx'> </div><div class='del'>-	cm_id_priv-&gt;id.cm_handler = cur_cm_id_priv-&gt;id.cm_handler;</div><div class='del'>-	cm_id_priv-&gt;id.context = cur_cm_id_priv-&gt;id.context;</div><div class='del'>-	cm_id_priv-&gt;id.service_id = sidr_req_msg-&gt;service_id;</div><div class='del'>-	cm_id_priv-&gt;id.service_mask = ~cpu_to_be64(0);</div><div class='add'>+	cm_id_priv-&gt;id.cm_handler = listen_cm_id_priv-&gt;id.cm_handler;</div><div class='add'>+	cm_id_priv-&gt;id.context = listen_cm_id_priv-&gt;id.context;</div><div class='ctx'> </div><div class='del'>-	cm_format_sidr_req_event(work, cm_id_priv, &amp;cur_cm_id_priv-&gt;id);</div><div class='del'>-	cm_process_work(cm_id_priv, work);</div><div class='del'>-	cm_deref_id(cur_cm_id_priv);</div><div class='add'>+	/*</div><div class='add'>+	 * A SIDR ID does not need to be in the xarray since it does not receive</div><div class='add'>+	 * mads, is not placed in the remote_id or remote_qpn rbtree, and does</div><div class='add'>+	 * not enter timewait.</div><div class='add'>+	 */</div><div class='add'>+</div><div class='add'>+	cm_format_sidr_req_event(work, cm_id_priv, &amp;listen_cm_id_priv-&gt;id);</div><div class='add'>+	ret = cm_id_priv-&gt;id.cm_handler(&amp;cm_id_priv-&gt;id, &amp;work-&gt;cm_event);</div><div class='add'>+	cm_free_work(work);</div><div class='add'>+	/*</div><div class='add'>+	 * A pointer to the listen_cm_id is held in the event, so this deref</div><div class='add'>+	 * must be after the event is delivered above.</div><div class='add'>+	 */</div><div class='add'>+	cm_deref_id(listen_cm_id_priv);</div><div class='add'>+	if (ret)</div><div class='add'>+		cm_destroy_id(&amp;cm_id_priv-&gt;id, ret);</div><div class='ctx'> 	return 0;</div><div class='ctx'> out:</div><div class='ctx'> 	ib_destroy_cm_id(&amp;cm_id_priv-&gt;id);</div><div class='hunk'>@@ -3618,57 +3622,59 @@ static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,</div><div class='ctx'> 			       struct cm_id_private *cm_id_priv,</div><div class='ctx'> 			       struct ib_cm_sidr_rep_param *param)</div><div class='ctx'> {</div><div class='del'>-	cm_format_mad_hdr(&amp;sidr_rep_msg-&gt;hdr, CM_SIDR_REP_ATTR_ID,</div><div class='del'>-			  cm_id_priv-&gt;tid);</div><div class='del'>-	sidr_rep_msg-&gt;request_id = cm_id_priv-&gt;id.remote_id;</div><div class='del'>-	sidr_rep_msg-&gt;status = param-&gt;status;</div><div class='del'>-	cm_sidr_rep_set_qpn(sidr_rep_msg, cpu_to_be32(param-&gt;qp_num));</div><div class='del'>-	sidr_rep_msg-&gt;service_id = cm_id_priv-&gt;id.service_id;</div><div class='del'>-	sidr_rep_msg-&gt;qkey = cpu_to_be32(param-&gt;qkey);</div><div class='add'>+	cm_format_mad_ece_hdr(&amp;sidr_rep_msg-&gt;hdr, CM_SIDR_REP_ATTR_ID,</div><div class='add'>+			      cm_id_priv-&gt;tid, param-&gt;ece.attr_mod);</div><div class='add'>+	IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg,</div><div class='add'>+		be32_to_cpu(cm_id_priv-&gt;id.remote_id));</div><div class='add'>+	IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param-&gt;status);</div><div class='add'>+	IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param-&gt;qp_num);</div><div class='add'>+	IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg,</div><div class='add'>+		be64_to_cpu(cm_id_priv-&gt;id.service_id));</div><div class='add'>+	IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param-&gt;qkey);</div><div class='add'>+	IBA_SET(CM_SIDR_REP_VENDOR_ID_L, sidr_rep_msg,</div><div class='add'>+		param-&gt;ece.vendor_id &amp; 0xFF);</div><div class='add'>+	IBA_SET(CM_SIDR_REP_VENDOR_ID_H, sidr_rep_msg,</div><div class='add'>+		(param-&gt;ece.vendor_id &gt;&gt; 8) &amp; 0xFF);</div><div class='ctx'> </div><div class='ctx'> 	if (param-&gt;info &amp;&amp; param-&gt;info_length)</div><div class='del'>-		memcpy(sidr_rep_msg-&gt;info, param-&gt;info, param-&gt;info_length);</div><div class='add'>+		IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg,</div><div class='add'>+			    param-&gt;info, param-&gt;info_length);</div><div class='ctx'> </div><div class='ctx'> 	if (param-&gt;private_data &amp;&amp; param-&gt;private_data_len)</div><div class='del'>-		memcpy(sidr_rep_msg-&gt;private_data, param-&gt;private_data,</div><div class='del'>-		       param-&gt;private_data_len);</div><div class='add'>+		IBA_SET_MEM(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg,</div><div class='add'>+			    param-&gt;private_data, param-&gt;private_data_len);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,</div><div class='del'>-			struct ib_cm_sidr_rep_param *param)</div><div class='add'>+static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,</div><div class='add'>+				   struct ib_cm_sidr_rep_param *param)</div><div class='ctx'> {</div><div class='del'>-	struct cm_id_private *cm_id_priv;</div><div class='ctx'> 	struct ib_mad_send_buf *msg;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	lockdep_assert_held(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+</div><div class='ctx'> 	if ((param-&gt;info &amp;&amp; param-&gt;info_length &gt; IB_CM_SIDR_REP_INFO_LENGTH) ||</div><div class='ctx'> 	    (param-&gt;private_data &amp;&amp;</div><div class='ctx'> 	     param-&gt;private_data_len &gt; IB_CM_SIDR_REP_PRIVATE_DATA_SIZE))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	cm_id_priv = container_of(cm_id, struct cm_id_private, id);</div><div class='del'>-	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	if (cm_id-&gt;state != IB_CM_SIDR_REQ_RCVD) {</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-		goto error;</div><div class='del'>-	}</div><div class='add'>+	if (cm_id_priv-&gt;id.state != IB_CM_SIDR_REQ_RCVD)</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	ret = cm_alloc_msg(cm_id_priv, &amp;msg);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto error;</div><div class='add'>+	msg = cm_alloc_msg(cm_id_priv);</div><div class='add'>+	if (IS_ERR(msg))</div><div class='add'>+		return PTR_ERR(msg);</div><div class='ctx'> </div><div class='ctx'> 	cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg-&gt;mad, cm_id_priv,</div><div class='ctx'> 			   param);</div><div class='add'>+	trace_icm_send_sidr_rep(&amp;cm_id_priv-&gt;id);</div><div class='ctx'> 	ret = ib_post_send_mad(msg, NULL);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 		cm_free_msg(msg);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='del'>-	cm_id-&gt;state = IB_CM_IDLE;</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-</div><div class='add'>+	cm_id_priv-&gt;id.state = IB_CM_IDLE;</div><div class='ctx'> 	spin_lock_irqsave(&amp;cm.lock, flags);</div><div class='ctx'> 	if (!RB_EMPTY_NODE(&amp;cm_id_priv-&gt;sidr_id_node)) {</div><div class='ctx'> 		rb_erase(&amp;cm_id_priv-&gt;sidr_id_node, &amp;cm.remote_sidr_table);</div><div class='hunk'>@@ -3676,8 +3682,19 @@ int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;cm.lock, flags);</div><div class='ctx'> 	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-error:	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,</div><div class='add'>+			struct ib_cm_sidr_rep_param *param)</div><div class='add'>+{</div><div class='add'>+	struct cm_id_private *cm_id_priv =</div><div class='add'>+		container_of(cm_id, struct cm_id_private, id);</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+	ret = cm_send_sidr_rep_locked(cm_id_priv, param);</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_send_cm_sidr_rep);</div><div class='hunk'>@@ -3691,13 +3708,16 @@ static void cm_format_sidr_rep_event(struct cm_work *work,</div><div class='ctx'> 	sidr_rep_msg = (struct cm_sidr_rep_msg *)</div><div class='ctx'> 				work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='ctx'> 	param = &amp;work-&gt;cm_event.param.sidr_rep_rcvd;</div><div class='del'>-	param-&gt;status = sidr_rep_msg-&gt;status;</div><div class='del'>-	param-&gt;qkey = be32_to_cpu(sidr_rep_msg-&gt;qkey);</div><div class='del'>-	param-&gt;qpn = be32_to_cpu(cm_sidr_rep_get_qpn(sidr_rep_msg));</div><div class='del'>-	param-&gt;info = &amp;sidr_rep_msg-&gt;info;</div><div class='del'>-	param-&gt;info_len = sidr_rep_msg-&gt;info_length;</div><div class='add'>+	param-&gt;status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg);</div><div class='add'>+	param-&gt;qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg);</div><div class='add'>+	param-&gt;qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg);</div><div class='add'>+	param-&gt;info = IBA_GET_MEM_PTR(CM_SIDR_REP_ADDITIONAL_INFORMATION,</div><div class='add'>+				      sidr_rep_msg);</div><div class='add'>+	param-&gt;info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH,</div><div class='add'>+				  sidr_rep_msg);</div><div class='ctx'> 	param-&gt;sgid_attr = cm_id_priv-&gt;av.ah_attr.grh.sgid_attr;</div><div class='del'>-	work-&gt;cm_event.private_data = &amp;sidr_rep_msg-&gt;private_data;</div><div class='add'>+	work-&gt;cm_event.private_data =</div><div class='add'>+		IBA_GET_MEM_PTR(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int cm_sidr_rep_handler(struct cm_work *work)</div><div class='hunk'>@@ -3707,7 +3727,8 @@ static int cm_sidr_rep_handler(struct cm_work *work)</div><div class='ctx'> </div><div class='ctx'> 	sidr_rep_msg = (struct cm_sidr_rep_msg *)</div><div class='ctx'> 				work-&gt;mad_recv_wc-&gt;recv_buf.mad;</div><div class='del'>-	cm_id_priv = cm_acquire_id(sidr_rep_msg-&gt;request_id, 0);</div><div class='add'>+	cm_id_priv = cm_acquire_id(</div><div class='add'>+		cpu_to_be32(IBA_GET(CM_SIDR_REP_REQUESTID, sidr_rep_msg)), 0);</div><div class='ctx'> 	if (!cm_id_priv)</div><div class='ctx'> 		return -EINVAL; /* Unmatched reply. */</div><div class='ctx'> </div><div class='hunk'>@@ -3717,7 +3738,7 @@ static int cm_sidr_rep_handler(struct cm_work *work)</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> 	cm_id_priv-&gt;id.state = IB_CM_IDLE;</div><div class='del'>-	ib_cancel_mad(cm_id_priv-&gt;av.port-&gt;mad_agent, cm_id_priv-&gt;msg);</div><div class='add'>+	ib_cancel_mad(cm_id_priv-&gt;msg);</div><div class='ctx'> 	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> </div><div class='ctx'> 	cm_format_sidr_rep_event(work, cm_id_priv);</div><div class='hunk'>@@ -3728,25 +3749,29 @@ out:</div><div class='ctx'> 	return -EINVAL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cm_process_send_error(struct ib_mad_send_buf *msg,</div><div class='add'>+static void cm_process_send_error(struct cm_id_private *cm_id_priv,</div><div class='add'>+				  struct ib_mad_send_buf *msg,</div><div class='ctx'> 				  enum ib_wc_status wc_status)</div><div class='ctx'> {</div><div class='del'>-	struct cm_id_private *cm_id_priv;</div><div class='del'>-	struct ib_cm_event cm_event;</div><div class='del'>-	enum ib_cm_state state;</div><div class='add'>+	enum ib_cm_state state = (unsigned long) msg-&gt;context[1];</div><div class='add'>+	struct ib_cm_event cm_event = {};</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	memset(&amp;cm_event, 0, sizeof cm_event);</div><div class='del'>-	cm_id_priv = msg-&gt;context[0];</div><div class='del'>-</div><div class='del'>-	/* Discard old sends or ones without a response. */</div><div class='add'>+	/* Discard old sends. */</div><div class='ctx'> 	spin_lock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-	state = (enum ib_cm_state) (unsigned long) msg-&gt;context[1];</div><div class='del'>-	if (msg != cm_id_priv-&gt;msg || state != cm_id_priv-&gt;id.state)</div><div class='del'>-		goto discard;</div><div class='add'>+	if (msg != cm_id_priv-&gt;msg) {</div><div class='add'>+		spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='add'>+		cm_free_msg(msg);</div><div class='add'>+		cm_deref_id(cm_id_priv);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+	cm_free_priv_msg(msg);</div><div class='add'>+</div><div class='add'>+	if (state != cm_id_priv-&gt;id.state || wc_status == IB_WC_SUCCESS ||</div><div class='add'>+	    wc_status == IB_WC_WR_FLUSH_ERR)</div><div class='add'>+		goto out_unlock;</div><div class='ctx'> </div><div class='del'>-	pr_debug_ratelimited("CM: failed sending MAD in state %d. (%s)\n",</div><div class='del'>-			     state, ib_wc_status_msg(wc_status));</div><div class='add'>+	trace_icm_mad_send_err(state, wc_status);</div><div class='ctx'> 	switch (state) {</div><div class='ctx'> 	case IB_CM_REQ_SENT:</div><div class='ctx'> 	case IB_CM_MRA_REQ_RCVD:</div><div class='hunk'>@@ -3767,26 +3792,25 @@ static void cm_process_send_error(struct ib_mad_send_buf *msg,</div><div class='ctx'> 		cm_event.event = IB_CM_SIDR_REQ_ERROR;</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='del'>-		goto discard;</div><div class='add'>+		goto out_unlock;</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='ctx'> 	cm_event.param.send_status = wc_status;</div><div class='ctx'> </div><div class='ctx'> 	/* No other events can occur on the cm_id at this point. */</div><div class='ctx'> 	ret = cm_id_priv-&gt;id.cm_handler(&amp;cm_id_priv-&gt;id, &amp;cm_event);</div><div class='del'>-	cm_free_msg(msg);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		ib_destroy_cm_id(&amp;cm_id_priv-&gt;id);</div><div class='ctx'> 	return;</div><div class='del'>-discard:</div><div class='add'>+out_unlock:</div><div class='ctx'> 	spin_unlock_irq(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-	cm_free_msg(msg);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void cm_send_handler(struct ib_mad_agent *mad_agent,</div><div class='ctx'> 			    struct ib_mad_send_wc *mad_send_wc)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_mad_send_buf *msg = mad_send_wc-&gt;send_buf;</div><div class='add'>+	struct cm_id_private *cm_id_priv;</div><div class='ctx'> 	struct cm_port *port;</div><div class='ctx'> 	u16 attr_index;</div><div class='ctx'> </div><div class='hunk'>@@ -3794,33 +3818,22 @@ static void cm_send_handler(struct ib_mad_agent *mad_agent,</div><div class='ctx'> 	attr_index = be16_to_cpu(((struct ib_mad_hdr *)</div><div class='ctx'> 				  msg-&gt;mad)-&gt;attr_id) - CM_ATTR_ID_OFFSET;</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * If the send was in response to a received message (context[0] is not</div><div class='del'>-	 * set to a cm_id), and is not a REJ, then it is a send that was</div><div class='del'>-	 * manually retried.</div><div class='del'>-	 */</div><div class='del'>-	if (!msg-&gt;context[0] &amp;&amp; (attr_index != CM_REJ_COUNTER))</div><div class='add'>+	if (msg-&gt;context[0] == CM_DIRECT_RETRY_CTX) {</div><div class='ctx'> 		msg-&gt;retries = 1;</div><div class='add'>+		cm_id_priv = NULL;</div><div class='add'>+	} else {</div><div class='add'>+		cm_id_priv = msg-&gt;context[0];</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	atomic_long_add(1 + msg-&gt;retries,</div><div class='del'>-			&amp;port-&gt;counter_group[CM_XMIT].counter[attr_index]);</div><div class='add'>+	atomic_long_add(1 + msg-&gt;retries, &amp;port-&gt;counters[CM_XMIT][attr_index]);</div><div class='ctx'> 	if (msg-&gt;retries)</div><div class='ctx'> 		atomic_long_add(msg-&gt;retries,</div><div class='del'>-				&amp;port-&gt;counter_group[CM_XMIT_RETRIES].</div><div class='del'>-				counter[attr_index]);</div><div class='add'>+				&amp;port-&gt;counters[CM_XMIT_RETRIES][attr_index]);</div><div class='ctx'> </div><div class='del'>-	switch (mad_send_wc-&gt;status) {</div><div class='del'>-	case IB_WC_SUCCESS:</div><div class='del'>-	case IB_WC_WR_FLUSH_ERR:</div><div class='add'>+	if (cm_id_priv)</div><div class='add'>+		cm_process_send_error(cm_id_priv, msg, mad_send_wc-&gt;status);</div><div class='add'>+	else</div><div class='ctx'> 		cm_free_msg(msg);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		if (msg-&gt;context[0] &amp;&amp; msg-&gt;context[1])</div><div class='del'>-			cm_process_send_error(msg, mad_send_wc-&gt;status);</div><div class='del'>-		else</div><div class='del'>-			cm_free_msg(msg);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void cm_work_handler(struct work_struct *_work)</div><div class='hunk'>@@ -3869,7 +3882,7 @@ static void cm_work_handler(struct work_struct *_work)</div><div class='ctx'> 		ret = cm_timewait_handler(work);</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='del'>-		pr_debug("cm_event.event: 0x%x\n", work-&gt;cm_event.event);</div><div class='add'>+		trace_icm_handler_err(work-&gt;cm_event.event);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -3895,8 +3908,7 @@ static int cm_establish(struct ib_cm_id *cm_id)</div><div class='ctx'> </div><div class='ctx'> 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);</div><div class='ctx'> 	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	switch (cm_id-&gt;state)</div><div class='del'>-	{</div><div class='add'>+	switch (cm_id-&gt;state) {</div><div class='ctx'> 	case IB_CM_REP_SENT:</div><div class='ctx'> 	case IB_CM_MRA_REP_RCVD:</div><div class='ctx'> 		cm_id-&gt;state = IB_CM_ESTABLISHED;</div><div class='hunk'>@@ -3905,8 +3917,7 @@ static int cm_establish(struct ib_cm_id *cm_id)</div><div class='ctx'> 		ret = -EISCONN;</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='del'>-		pr_debug("%s: local_id %d, cm_id-&gt;state: %d\n", __func__,</div><div class='del'>-			 be32_to_cpu(cm_id-&gt;local_id), cm_id-&gt;state);</div><div class='add'>+		trace_icm_establish_err(cm_id);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -3946,9 +3957,7 @@ out:</div><div class='ctx'> static int cm_migrate(struct ib_cm_id *cm_id)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_id_private *cm_id_priv;</div><div class='del'>-	struct cm_av tmp_av;</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	int tmp_send_port_not_ready;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='ctx'> 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);</div><div class='hunk'>@@ -3957,14 +3966,7 @@ static int cm_migrate(struct ib_cm_id *cm_id)</div><div class='ctx'> 	    (cm_id-&gt;lap_state == IB_CM_LAP_UNINIT ||</div><div class='ctx'> 	     cm_id-&gt;lap_state == IB_CM_LAP_IDLE)) {</div><div class='ctx'> 		cm_id-&gt;lap_state = IB_CM_LAP_IDLE;</div><div class='del'>-		/* Swap address vector */</div><div class='del'>-		tmp_av = cm_id_priv-&gt;av;</div><div class='ctx'> 		cm_id_priv-&gt;av = cm_id_priv-&gt;alt_av;</div><div class='del'>-		cm_id_priv-&gt;alt_av = tmp_av;</div><div class='del'>-		/* Swap port send ready state */</div><div class='del'>-		tmp_send_port_not_ready = cm_id_priv-&gt;prim_send_port_not_ready;</div><div class='del'>-		cm_id_priv-&gt;prim_send_port_not_ready = cm_id_priv-&gt;altr_send_port_not_ready;</div><div class='del'>-		cm_id_priv-&gt;altr_send_port_not_ready = tmp_send_port_not_ready;</div><div class='ctx'> 	} else</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='hunk'>@@ -4046,8 +4048,7 @@ static void cm_recv_handler(struct ib_mad_agent *mad_agent,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	attr_id = be16_to_cpu(mad_recv_wc-&gt;recv_buf.mad-&gt;mad_hdr.attr_id);</div><div class='del'>-	atomic_long_inc(&amp;port-&gt;counter_group[CM_RECV].</div><div class='del'>-			counter[attr_id - CM_ATTR_ID_OFFSET]);</div><div class='add'>+	atomic_long_inc(&amp;port-&gt;counters[CM_RECV][attr_id - CM_ATTR_ID_OFFSET]);</div><div class='ctx'> </div><div class='ctx'> 	work = kmalloc(struct_size(work, path, paths), GFP_KERNEL);</div><div class='ctx'> 	if (!work) {</div><div class='hunk'>@@ -4095,17 +4096,25 @@ static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,</div><div class='ctx'> 		*qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |</div><div class='ctx'> 				IB_QP_PKEY_INDEX | IB_QP_PORT;</div><div class='ctx'> 		qp_attr-&gt;qp_access_flags = IB_ACCESS_REMOTE_WRITE;</div><div class='del'>-		if (cm_id_priv-&gt;responder_resources)</div><div class='add'>+		if (cm_id_priv-&gt;responder_resources) {</div><div class='add'>+			struct ib_device *ib_dev = cm_id_priv-&gt;id.device;</div><div class='add'>+			u64 support_flush = ib_dev-&gt;attrs.device_cap_flags &amp;</div><div class='add'>+			  (IB_DEVICE_FLUSH_GLOBAL | IB_DEVICE_FLUSH_PERSISTENT);</div><div class='add'>+			u32 flushable = support_flush ?</div><div class='add'>+					(IB_ACCESS_FLUSH_GLOBAL |</div><div class='add'>+					 IB_ACCESS_FLUSH_PERSISTENT) : 0;</div><div class='add'>+</div><div class='ctx'> 			qp_attr-&gt;qp_access_flags |= IB_ACCESS_REMOTE_READ |</div><div class='del'>-						    IB_ACCESS_REMOTE_ATOMIC;</div><div class='add'>+						    IB_ACCESS_REMOTE_ATOMIC |</div><div class='add'>+						    flushable;</div><div class='add'>+		}</div><div class='ctx'> 		qp_attr-&gt;pkey_index = cm_id_priv-&gt;av.pkey_index;</div><div class='del'>-		qp_attr-&gt;port_num = cm_id_priv-&gt;av.port-&gt;port_num;</div><div class='add'>+		if (cm_id_priv-&gt;av.port)</div><div class='add'>+			qp_attr-&gt;port_num = cm_id_priv-&gt;av.port-&gt;port_num;</div><div class='ctx'> 		ret = 0;</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='del'>-		pr_debug("%s: local_id %d, cm_id_priv-&gt;id.state: %d\n",</div><div class='del'>-			 __func__, be32_to_cpu(cm_id_priv-&gt;id.local_id),</div><div class='del'>-			 cm_id_priv-&gt;id.state);</div><div class='add'>+		trace_icm_qp_init_err(&amp;cm_id_priv-&gt;id);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -4132,6 +4141,10 @@ static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,</div><div class='ctx'> 		*qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU |</div><div class='ctx'> 				IB_QP_DEST_QPN | IB_QP_RQ_PSN;</div><div class='ctx'> 		qp_attr-&gt;ah_attr = cm_id_priv-&gt;av.ah_attr;</div><div class='add'>+		if ((qp_attr-&gt;ah_attr.type == RDMA_AH_ATTR_TYPE_IB) &amp;&amp;</div><div class='add'>+		    cm_id_priv-&gt;av.dlid_datapath &amp;&amp;</div><div class='add'>+		    (cm_id_priv-&gt;av.dlid_datapath != 0xffff))</div><div class='add'>+			qp_attr-&gt;ah_attr.ib.dlid = cm_id_priv-&gt;av.dlid_datapath;</div><div class='ctx'> 		qp_attr-&gt;path_mtu = cm_id_priv-&gt;path_mtu;</div><div class='ctx'> 		qp_attr-&gt;dest_qp_num = be32_to_cpu(cm_id_priv-&gt;remote_qpn);</div><div class='ctx'> 		qp_attr-&gt;rq_psn = be32_to_cpu(cm_id_priv-&gt;rq_psn);</div><div class='hunk'>@@ -4143,7 +4156,8 @@ static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,</div><div class='ctx'> 					cm_id_priv-&gt;responder_resources;</div><div class='ctx'> 			qp_attr-&gt;min_rnr_timer = 0;</div><div class='ctx'> 		}</div><div class='del'>-		if (rdma_ah_get_dlid(&amp;cm_id_priv-&gt;alt_av.ah_attr)) {</div><div class='add'>+		if (rdma_ah_get_dlid(&amp;cm_id_priv-&gt;alt_av.ah_attr) &amp;&amp;</div><div class='add'>+		    cm_id_priv-&gt;alt_av.port) {</div><div class='ctx'> 			*qp_attr_mask |= IB_QP_ALT_PATH;</div><div class='ctx'> 			qp_attr-&gt;alt_port_num = cm_id_priv-&gt;alt_av.port-&gt;port_num;</div><div class='ctx'> 			qp_attr-&gt;alt_pkey_index = cm_id_priv-&gt;alt_av.pkey_index;</div><div class='hunk'>@@ -4153,9 +4167,7 @@ static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,</div><div class='ctx'> 		ret = 0;</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='del'>-		pr_debug("%s: local_id %d, cm_id_priv-&gt;id.state: %d\n",</div><div class='del'>-			 __func__, be32_to_cpu(cm_id_priv-&gt;id.local_id),</div><div class='del'>-			 cm_id_priv-&gt;id.state);</div><div class='add'>+		trace_icm_qp_rtr_err(&amp;cm_id_priv-&gt;id);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -4192,7 +4204,7 @@ static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,</div><div class='ctx'> 				qp_attr-&gt;retry_cnt = cm_id_priv-&gt;retry_count;</div><div class='ctx'> 				qp_attr-&gt;rnr_retry = cm_id_priv-&gt;rnr_retry_count;</div><div class='ctx'> 				qp_attr-&gt;max_rd_atomic = cm_id_priv-&gt;initiator_depth;</div><div class='del'>-				/* fall through */</div><div class='add'>+				fallthrough;</div><div class='ctx'> 			case IB_QPT_XRC_TGT:</div><div class='ctx'> 				*qp_attr_mask |= IB_QP_TIMEOUT;</div><div class='ctx'> 				qp_attr-&gt;timeout = cm_id_priv-&gt;av.timeout;</div><div class='hunk'>@@ -4206,7 +4218,9 @@ static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,</div><div class='ctx'> 			}</div><div class='ctx'> 		} else {</div><div class='ctx'> 			*qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE;</div><div class='del'>-			qp_attr-&gt;alt_port_num = cm_id_priv-&gt;alt_av.port-&gt;port_num;</div><div class='add'>+			if (cm_id_priv-&gt;alt_av.port)</div><div class='add'>+				qp_attr-&gt;alt_port_num =</div><div class='add'>+					cm_id_priv-&gt;alt_av.port-&gt;port_num;</div><div class='ctx'> 			qp_attr-&gt;alt_pkey_index = cm_id_priv-&gt;alt_av.pkey_index;</div><div class='ctx'> 			qp_attr-&gt;alt_timeout = cm_id_priv-&gt;alt_av.timeout;</div><div class='ctx'> 			qp_attr-&gt;alt_ah_attr = cm_id_priv-&gt;alt_av.ah_attr;</div><div class='hunk'>@@ -4215,9 +4229,7 @@ static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,</div><div class='ctx'> 		ret = 0;</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='del'>-		pr_debug("%s: local_id %d, cm_id_priv-&gt;id.state: %d\n",</div><div class='del'>-			 __func__, be32_to_cpu(cm_id_priv-&gt;id.local_id),</div><div class='del'>-			 cm_id_priv-&gt;id.state);</div><div class='add'>+		trace_icm_qp_rts_err(&amp;cm_id_priv-&gt;id);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -4251,75 +4263,76 @@ int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_cm_init_qp_attr);</div><div class='ctx'> </div><div class='del'>-static ssize_t cm_show_counter(struct kobject *obj, struct attribute *attr,</div><div class='del'>-			       char *buf)</div><div class='add'>+static ssize_t cm_show_counter(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+			       struct ib_port_attribute *attr, char *buf)</div><div class='ctx'> {</div><div class='del'>-	struct cm_counter_group *group;</div><div class='del'>-	struct cm_counter_attribute *cm_attr;</div><div class='del'>-</div><div class='del'>-	group = container_of(obj, struct cm_counter_group, obj);</div><div class='del'>-	cm_attr = container_of(attr, struct cm_counter_attribute, attr);</div><div class='del'>-</div><div class='del'>-	return sprintf(buf, "%ld\n",</div><div class='del'>-		       atomic_long_read(&amp;group-&gt;counter[cm_attr-&gt;index]));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static const struct sysfs_ops cm_counter_ops = {</div><div class='del'>-	.show = cm_show_counter</div><div class='del'>-};</div><div class='add'>+	struct cm_counter_attribute *cm_attr =</div><div class='add'>+		container_of(attr, struct cm_counter_attribute, attr);</div><div class='add'>+	struct cm_device *cm_dev = ib_get_client_data(ibdev, &amp;cm_client);</div><div class='ctx'> </div><div class='del'>-static struct kobj_type cm_counter_obj_type = {</div><div class='del'>-	.sysfs_ops = &amp;cm_counter_ops,</div><div class='del'>-	.default_attrs = cm_counter_default_attrs</div><div class='del'>-};</div><div class='add'>+	if (WARN_ON(!cm_dev))</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-static char *cm_devnode(struct device *dev, umode_t *mode)</div><div class='del'>-{</div><div class='del'>-	if (mode)</div><div class='del'>-		*mode = 0666;</div><div class='del'>-	return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));</div><div class='add'>+	return sysfs_emit(</div><div class='add'>+		buf, "%ld\n",</div><div class='add'>+		atomic_long_read(</div><div class='add'>+			&amp;cm_dev-&gt;port[port_num - 1]</div><div class='add'>+				 -&gt;counters[cm_attr-&gt;group][cm_attr-&gt;index]));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct class cm_class = {</div><div class='del'>-	.owner   = THIS_MODULE,</div><div class='del'>-	.name    = "infiniband_cm",</div><div class='del'>-	.devnode = cm_devnode,</div><div class='del'>-};</div><div class='del'>-EXPORT_SYMBOL(cm_class);</div><div class='del'>-</div><div class='del'>-static int cm_create_port_fs(struct cm_port *port)</div><div class='del'>-{</div><div class='del'>-	int i, ret;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; CM_COUNTER_GROUPS; i++) {</div><div class='del'>-		ret = ib_port_register_module_stat(port-&gt;cm_dev-&gt;ib_device,</div><div class='del'>-						   port-&gt;port_num,</div><div class='del'>-						   &amp;port-&gt;counter_group[i].obj,</div><div class='del'>-						   &amp;cm_counter_obj_type,</div><div class='del'>-						   counter_group_names[i]);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto error;</div><div class='add'>+#define CM_COUNTER_ATTR(_name, _group, _index)                                 \</div><div class='add'>+	{                                                                      \</div><div class='add'>+		.attr = __ATTR(_name, 0444, cm_show_counter, NULL),            \</div><div class='add'>+		.group = _group, .index = _index                               \</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-error:</div><div class='del'>-	while (i--)</div><div class='del'>-		ib_port_unregister_module_stat(&amp;port-&gt;counter_group[i].obj);</div><div class='del'>-	return ret;</div><div class='del'>-</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void cm_remove_port_fs(struct cm_port *port)</div><div class='del'>-{</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; CM_COUNTER_GROUPS; i++)</div><div class='del'>-		ib_port_unregister_module_stat(&amp;port-&gt;counter_group[i].obj);</div><div class='add'>+#define CM_COUNTER_GROUP(_group, _name)                                        \</div><div class='add'>+	static struct cm_counter_attribute cm_counter_attr_##_group[] = {      \</div><div class='add'>+		CM_COUNTER_ATTR(req, _group, CM_REQ_COUNTER),                  \</div><div class='add'>+		CM_COUNTER_ATTR(mra, _group, CM_MRA_COUNTER),                  \</div><div class='add'>+		CM_COUNTER_ATTR(rej, _group, CM_REJ_COUNTER),                  \</div><div class='add'>+		CM_COUNTER_ATTR(rep, _group, CM_REP_COUNTER),                  \</div><div class='add'>+		CM_COUNTER_ATTR(rtu, _group, CM_RTU_COUNTER),                  \</div><div class='add'>+		CM_COUNTER_ATTR(dreq, _group, CM_DREQ_COUNTER),                \</div><div class='add'>+		CM_COUNTER_ATTR(drep, _group, CM_DREP_COUNTER),                \</div><div class='add'>+		CM_COUNTER_ATTR(sidr_req, _group, CM_SIDR_REQ_COUNTER),        \</div><div class='add'>+		CM_COUNTER_ATTR(sidr_rep, _group, CM_SIDR_REP_COUNTER),        \</div><div class='add'>+		CM_COUNTER_ATTR(lap, _group, CM_LAP_COUNTER),                  \</div><div class='add'>+		CM_COUNTER_ATTR(apr, _group, CM_APR_COUNTER),                  \</div><div class='add'>+	};                                                                     \</div><div class='add'>+	static struct attribute *cm_counter_attrs_##_group[] = {               \</div><div class='add'>+		&amp;cm_counter_attr_##_group[0].attr.attr,                        \</div><div class='add'>+		&amp;cm_counter_attr_##_group[1].attr.attr,                        \</div><div class='add'>+		&amp;cm_counter_attr_##_group[2].attr.attr,                        \</div><div class='add'>+		&amp;cm_counter_attr_##_group[3].attr.attr,                        \</div><div class='add'>+		&amp;cm_counter_attr_##_group[4].attr.attr,                        \</div><div class='add'>+		&amp;cm_counter_attr_##_group[5].attr.attr,                        \</div><div class='add'>+		&amp;cm_counter_attr_##_group[6].attr.attr,                        \</div><div class='add'>+		&amp;cm_counter_attr_##_group[7].attr.attr,                        \</div><div class='add'>+		&amp;cm_counter_attr_##_group[8].attr.attr,                        \</div><div class='add'>+		&amp;cm_counter_attr_##_group[9].attr.attr,                        \</div><div class='add'>+		&amp;cm_counter_attr_##_group[10].attr.attr,                       \</div><div class='add'>+		NULL,                                                          \</div><div class='add'>+	};                                                                     \</div><div class='add'>+	static const struct attribute_group cm_counter_group_##_group = {      \</div><div class='add'>+		.name = _name,                                                 \</div><div class='add'>+		.attrs = cm_counter_attrs_##_group,                            \</div><div class='add'>+	};</div><div class='ctx'> </div><div class='del'>-}</div><div class='add'>+CM_COUNTER_GROUP(CM_XMIT, "cm_tx_msgs")</div><div class='add'>+CM_COUNTER_GROUP(CM_XMIT_RETRIES, "cm_tx_retries")</div><div class='add'>+CM_COUNTER_GROUP(CM_RECV, "cm_rx_msgs")</div><div class='add'>+CM_COUNTER_GROUP(CM_RECV_DUPLICATES, "cm_rx_duplicates")</div><div class='add'>+</div><div class='add'>+static const struct attribute_group *cm_counter_groups[] = {</div><div class='add'>+	&amp;cm_counter_group_CM_XMIT,</div><div class='add'>+	&amp;cm_counter_group_CM_XMIT_RETRIES,</div><div class='add'>+	&amp;cm_counter_group_CM_RECV,</div><div class='add'>+	&amp;cm_counter_group_CM_RECV_DUPLICATES,</div><div class='add'>+	NULL,</div><div class='add'>+};</div><div class='ctx'> </div><div class='del'>-static void cm_add_one(struct ib_device *ib_device)</div><div class='add'>+static int cm_add_one(struct ib_device *ib_device)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_device *cm_dev;</div><div class='ctx'> 	struct cm_port *port;</div><div class='hunk'>@@ -4333,34 +4346,38 @@ static void cm_add_one(struct ib_device *ib_device)</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int ret;</div><div class='ctx'> 	int count = 0;</div><div class='del'>-	u8 i;</div><div class='add'>+	u32 i;</div><div class='ctx'> </div><div class='ctx'> 	cm_dev = kzalloc(struct_size(cm_dev, port, ib_device-&gt;phys_port_cnt),</div><div class='ctx'> 			 GFP_KERNEL);</div><div class='ctx'> 	if (!cm_dev)</div><div class='del'>-		return;</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='add'>+	kref_init(&amp;cm_dev-&gt;kref);</div><div class='add'>+	rwlock_init(&amp;cm_dev-&gt;mad_agent_lock);</div><div class='ctx'> 	cm_dev-&gt;ib_device = ib_device;</div><div class='ctx'> 	cm_dev-&gt;ack_delay = ib_device-&gt;attrs.local_ca_ack_delay;</div><div class='ctx'> 	cm_dev-&gt;going_down = 0;</div><div class='ctx'> </div><div class='add'>+	ib_set_client_data(ib_device, &amp;cm_client, cm_dev);</div><div class='add'>+</div><div class='ctx'> 	set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);</div><div class='del'>-	for (i = 1; i &lt;= ib_device-&gt;phys_port_cnt; i++) {</div><div class='add'>+	rdma_for_each_port (ib_device, i) {</div><div class='ctx'> 		if (!rdma_cap_ib_cm(ib_device, i))</div><div class='ctx'> 			continue;</div><div class='ctx'> </div><div class='ctx'> 		port = kzalloc(sizeof *port, GFP_KERNEL);</div><div class='del'>-		if (!port)</div><div class='add'>+		if (!port) {</div><div class='add'>+			ret = -ENOMEM;</div><div class='ctx'> 			goto error1;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='ctx'> 		cm_dev-&gt;port[i-1] = port;</div><div class='ctx'> 		port-&gt;cm_dev = cm_dev;</div><div class='ctx'> 		port-&gt;port_num = i;</div><div class='ctx'> </div><div class='del'>-		INIT_LIST_HEAD(&amp;port-&gt;cm_priv_prim_list);</div><div class='del'>-		INIT_LIST_HEAD(&amp;port-&gt;cm_priv_altr_list);</div><div class='del'>-</div><div class='del'>-		ret = cm_create_port_fs(port);</div><div class='add'>+		ret = ib_port_register_client_groups(ib_device, i,</div><div class='add'>+						     cm_counter_groups);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto error1;</div><div class='ctx'> </div><div class='hunk'>@@ -4372,30 +4389,47 @@ static void cm_add_one(struct ib_device *ib_device)</div><div class='ctx'> 							cm_recv_handler,</div><div class='ctx'> 							port,</div><div class='ctx'> 							0);</div><div class='del'>-		if (IS_ERR(port-&gt;mad_agent))</div><div class='add'>+		if (IS_ERR(port-&gt;mad_agent)) {</div><div class='add'>+			ret = PTR_ERR(port-&gt;mad_agent);</div><div class='ctx'> 			goto error2;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		port-&gt;rep_agent = ib_register_mad_agent(ib_device, i,</div><div class='add'>+							IB_QPT_GSI,</div><div class='add'>+							NULL,</div><div class='add'>+							0,</div><div class='add'>+							cm_send_handler,</div><div class='add'>+							NULL,</div><div class='add'>+							port,</div><div class='add'>+							0);</div><div class='add'>+		if (IS_ERR(port-&gt;rep_agent)) {</div><div class='add'>+			ret = PTR_ERR(port-&gt;rep_agent);</div><div class='add'>+			goto error3;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='ctx'> 		ret = ib_modify_port(ib_device, i, 0, &amp;port_modify);</div><div class='ctx'> 		if (ret)</div><div class='del'>-			goto error3;</div><div class='add'>+			goto error4;</div><div class='ctx'> </div><div class='ctx'> 		count++;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!count)</div><div class='add'>+	if (!count) {</div><div class='add'>+		ret = -EOPNOTSUPP;</div><div class='ctx'> 		goto free;</div><div class='del'>-</div><div class='del'>-	ib_set_client_data(ib_device, &amp;cm_client, cm_dev);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	write_lock_irqsave(&amp;cm.device_lock, flags);</div><div class='ctx'> 	list_add_tail(&amp;cm_dev-&gt;list, &amp;cm.device_list);</div><div class='ctx'> 	write_unlock_irqrestore(&amp;cm.device_lock, flags);</div><div class='del'>-	return;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='add'>+error4:</div><div class='add'>+	ib_unregister_mad_agent(port-&gt;rep_agent);</div><div class='ctx'> error3:</div><div class='ctx'> 	ib_unregister_mad_agent(port-&gt;mad_agent);</div><div class='ctx'> error2:</div><div class='del'>-	cm_remove_port_fs(port);</div><div class='add'>+	ib_port_unregister_client_groups(ib_device, i, cm_counter_groups);</div><div class='ctx'> error1:</div><div class='ctx'> 	port_modify.set_port_cap_mask = 0;</div><div class='ctx'> 	port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;</div><div class='hunk'>@@ -4405,27 +4439,25 @@ error1:</div><div class='ctx'> </div><div class='ctx'> 		port = cm_dev-&gt;port[i-1];</div><div class='ctx'> 		ib_modify_port(ib_device, port-&gt;port_num, 0, &amp;port_modify);</div><div class='add'>+		ib_unregister_mad_agent(port-&gt;rep_agent);</div><div class='ctx'> 		ib_unregister_mad_agent(port-&gt;mad_agent);</div><div class='del'>-		cm_remove_port_fs(port);</div><div class='add'>+		ib_port_unregister_client_groups(ib_device, i,</div><div class='add'>+						 cm_counter_groups);</div><div class='ctx'> 	}</div><div class='ctx'> free:</div><div class='del'>-	kfree(cm_dev);</div><div class='add'>+	cm_device_put(cm_dev);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void cm_remove_one(struct ib_device *ib_device, void *client_data)</div><div class='ctx'> {</div><div class='ctx'> 	struct cm_device *cm_dev = client_data;</div><div class='ctx'> 	struct cm_port *port;</div><div class='del'>-	struct cm_id_private *cm_id_priv;</div><div class='del'>-	struct ib_mad_agent *cur_mad_agent;</div><div class='ctx'> 	struct ib_port_modify port_modify = {</div><div class='ctx'> 		.clr_port_cap_mask = IB_PORT_CM_SUP</div><div class='ctx'> 	};</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	if (!cm_dev)</div><div class='del'>-		return;</div><div class='add'>+	u32 i;</div><div class='ctx'> </div><div class='ctx'> 	write_lock_irqsave(&amp;cm.device_lock, flags);</div><div class='ctx'> 	list_del(&amp;cm_dev-&gt;list);</div><div class='hunk'>@@ -4435,34 +4467,38 @@ static void cm_remove_one(struct ib_device *ib_device, void *client_data)</div><div class='ctx'> 	cm_dev-&gt;going_down = 1;</div><div class='ctx'> 	spin_unlock_irq(&amp;cm.lock);</div><div class='ctx'> </div><div class='del'>-	for (i = 1; i &lt;= ib_device-&gt;phys_port_cnt; i++) {</div><div class='add'>+	rdma_for_each_port (ib_device, i) {</div><div class='add'>+		struct ib_mad_agent *mad_agent;</div><div class='add'>+		struct ib_mad_agent *rep_agent;</div><div class='add'>+</div><div class='ctx'> 		if (!rdma_cap_ib_cm(ib_device, i))</div><div class='ctx'> 			continue;</div><div class='ctx'> </div><div class='ctx'> 		port = cm_dev-&gt;port[i-1];</div><div class='add'>+		mad_agent = port-&gt;mad_agent;</div><div class='add'>+		rep_agent = port-&gt;rep_agent;</div><div class='ctx'> 		ib_modify_port(ib_device, port-&gt;port_num, 0, &amp;port_modify);</div><div class='del'>-		/* Mark all the cm_id's as not valid */</div><div class='del'>-		spin_lock_irq(&amp;cm.lock);</div><div class='del'>-		list_for_each_entry(cm_id_priv, &amp;port-&gt;cm_priv_altr_list, altr_list)</div><div class='del'>-			cm_id_priv-&gt;altr_send_port_not_ready = 1;</div><div class='del'>-		list_for_each_entry(cm_id_priv, &amp;port-&gt;cm_priv_prim_list, prim_list)</div><div class='del'>-			cm_id_priv-&gt;prim_send_port_not_ready = 1;</div><div class='del'>-		spin_unlock_irq(&amp;cm.lock);</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * We flush the queue here after the going_down set, this</div><div class='ctx'> 		 * verify that no new works will be queued in the recv handler,</div><div class='ctx'> 		 * after that we can call the unregister_mad_agent</div><div class='ctx'> 		 */</div><div class='ctx'> 		flush_workqueue(cm.wq);</div><div class='del'>-		spin_lock_irq(&amp;cm.state_lock);</div><div class='del'>-		cur_mad_agent = port-&gt;mad_agent;</div><div class='add'>+		/*</div><div class='add'>+		 * The above ensures no call paths from the work are running,</div><div class='add'>+		 * the remaining paths all take the mad_agent_lock.</div><div class='add'>+		 */</div><div class='add'>+		write_lock(&amp;cm_dev-&gt;mad_agent_lock);</div><div class='ctx'> 		port-&gt;mad_agent = NULL;</div><div class='del'>-		spin_unlock_irq(&amp;cm.state_lock);</div><div class='del'>-		ib_unregister_mad_agent(cur_mad_agent);</div><div class='del'>-		cm_remove_port_fs(port);</div><div class='add'>+		port-&gt;rep_agent = NULL;</div><div class='add'>+		write_unlock(&amp;cm_dev-&gt;mad_agent_lock);</div><div class='add'>+		ib_unregister_mad_agent(mad_agent);</div><div class='add'>+		ib_unregister_mad_agent(rep_agent);</div><div class='add'>+		ib_port_unregister_client_groups(ib_device, i,</div><div class='add'>+						 cm_counter_groups);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	kfree(cm_dev);</div><div class='add'>+	cm_device_put(cm_dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __init ib_cm_init(void)</div><div class='hunk'>@@ -4472,23 +4508,16 @@ static int __init ib_cm_init(void)</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;cm.device_list);</div><div class='ctx'> 	rwlock_init(&amp;cm.device_lock);</div><div class='ctx'> 	spin_lock_init(&amp;cm.lock);</div><div class='del'>-	spin_lock_init(&amp;cm.state_lock);</div><div class='ctx'> 	cm.listen_service_table = RB_ROOT;</div><div class='ctx'> 	cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);</div><div class='ctx'> 	cm.remote_id_table = RB_ROOT;</div><div class='ctx'> 	cm.remote_qp_table = RB_ROOT;</div><div class='ctx'> 	cm.remote_sidr_table = RB_ROOT;</div><div class='del'>-	xa_init_flags(&amp;cm.local_id_table, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);</div><div class='add'>+	xa_init_flags(&amp;cm.local_id_table, XA_FLAGS_ALLOC);</div><div class='ctx'> 	get_random_bytes(&amp;cm.random_id_operand, sizeof cm.random_id_operand);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;cm.timewait_list);</div><div class='ctx'> </div><div class='del'>-	ret = class_register(&amp;cm_class);</div><div class='del'>-	if (ret) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto error1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cm.wq = alloc_workqueue("ib_cm", 0, 1);</div><div class='add'>+	cm.wq = alloc_workqueue("ib_cm", WQ_PERCPU, 1);</div><div class='ctx'> 	if (!cm.wq) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='ctx'> 		goto error2;</div><div class='hunk'>@@ -4502,8 +4531,6 @@ static int __init ib_cm_init(void)</div><div class='ctx'> error3:</div><div class='ctx'> 	destroy_workqueue(cm.wq);</div><div class='ctx'> error2:</div><div class='del'>-	class_unregister(&amp;cm_class);</div><div class='del'>-error1:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -4524,7 +4551,6 @@ static void __exit ib_cm_cleanup(void)</div><div class='ctx'> 		kfree(timewait_info);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	class_unregister(&amp;cm_class);</div><div class='ctx'> 	WARN_ON(!xa_empty(&amp;cm.local_id_table));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/core/cm_msgs.h b/drivers/infiniband/core/cm_msgs.h<br/>index 3d16d614aff6..8462de7ca26e 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cm_msgs.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cm_msgs.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cm_msgs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/cm_msgs.h</a></div><div class='hunk'>@@ -1,39 +1,14 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright (c) 2004, 2011 Intel Corporation.  All rights reserved.</div><div class='ctx'>  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.</div><div class='ctx'>  * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING the madirectory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use source and binary forms, with or</div><div class='del'>- *     withmodification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retathe above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHWARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS THE</div><div class='del'>- * SOFTWARE.</div><div class='add'>+ * Copyright (c) 2019, Mellanox Technologies inc.  All rights reserved.</div><div class='ctx'>  */</div><div class='del'>-#if !defined(CM_MSGS_H)</div><div class='add'>+#ifndef CM_MSGS_H</div><div class='ctx'> #define CM_MSGS_H</div><div class='ctx'> </div><div class='add'>+#include &lt;rdma/ibta_vol1_c12.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_mad.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_cm.h&gt;</div><div class='ctx'> </div><div class='hunk'>@@ -44,120 +19,14 @@</div><div class='ctx'> </div><div class='ctx'> #define IB_CM_CLASS_VERSION	2 /* IB specification 1.2 */</div><div class='ctx'> </div><div class='del'>-struct cm_req_msg {</div><div class='del'>-	struct ib_mad_hdr hdr;</div><div class='del'>-</div><div class='del'>-	__be32 local_comm_id;</div><div class='del'>-	__be32 rsvd4;</div><div class='del'>-	__be64 service_id;</div><div class='del'>-	__be64 local_ca_guid;</div><div class='del'>-	__be32 rsvd24;</div><div class='del'>-	__be32 local_qkey;</div><div class='del'>-	/* local QPN:24, responder resources:8 */</div><div class='del'>-	__be32 offset32;</div><div class='del'>-	/* local EECN:24, initiator depth:8 */</div><div class='del'>-	__be32 offset36;</div><div class='del'>-	/*</div><div class='del'>-	 * remote EECN:24, remote CM response timeout:5,</div><div class='del'>-	 * transport service type:2, end-to-end flow control:1</div><div class='del'>-	 */</div><div class='del'>-	__be32 offset40;</div><div class='del'>-	/* starting PSN:24, local CM response timeout:5, retry count:3 */</div><div class='del'>-	__be32 offset44;</div><div class='del'>-	__be16 pkey;</div><div class='del'>-	/* path MTU:4, RDC exists:1, RNR retry count:3. */</div><div class='del'>-	u8 offset50;</div><div class='del'>-	/* max CM Retries:4, SRQ:1, extended transport type:3 */</div><div class='del'>-	u8 offset51;</div><div class='del'>-</div><div class='del'>-	__be16 primary_local_lid;</div><div class='del'>-	__be16 primary_remote_lid;</div><div class='del'>-	union ib_gid primary_local_gid;</div><div class='del'>-	union ib_gid primary_remote_gid;</div><div class='del'>-	/* flow label:20, rsvd:6, packet rate:6 */</div><div class='del'>-	__be32 primary_offset88;</div><div class='del'>-	u8 primary_traffic_class;</div><div class='del'>-	u8 primary_hop_limit;</div><div class='del'>-	/* SL:4, subnet local:1, rsvd:3 */</div><div class='del'>-	u8 primary_offset94;</div><div class='del'>-	/* local ACK timeout:5, rsvd:3 */</div><div class='del'>-	u8 primary_offset95;</div><div class='del'>-</div><div class='del'>-	__be16 alt_local_lid;</div><div class='del'>-	__be16 alt_remote_lid;</div><div class='del'>-	union ib_gid alt_local_gid;</div><div class='del'>-	union ib_gid alt_remote_gid;</div><div class='del'>-	/* flow label:20, rsvd:6, packet rate:6 */</div><div class='del'>-	__be32 alt_offset132;</div><div class='del'>-	u8 alt_traffic_class;</div><div class='del'>-	u8 alt_hop_limit;</div><div class='del'>-	/* SL:4, subnet local:1, rsvd:3 */</div><div class='del'>-	u8 alt_offset138;</div><div class='del'>-	/* local ACK timeout:5, rsvd:3 */</div><div class='del'>-	u8 alt_offset139;</div><div class='del'>-</div><div class='del'>-	u32 private_data[IB_CM_REQ_PRIVATE_DATA_SIZE / sizeof(u32)];</div><div class='del'>-</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-static inline __be32 cm_req_get_local_qpn(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return cpu_to_be32(be32_to_cpu(req_msg-&gt;offset32) &gt;&gt; 8);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_local_qpn(struct cm_req_msg *req_msg, __be32 qpn)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;offset32 = cpu_to_be32((be32_to_cpu(qpn) &lt;&lt; 8) |</div><div class='del'>-					 (be32_to_cpu(req_msg-&gt;offset32) &amp;</div><div class='del'>-					  0x000000FF));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_resp_res(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) be32_to_cpu(req_msg-&gt;offset32);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_resp_res(struct cm_req_msg *req_msg, u8 resp_res)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;offset32 = cpu_to_be32(resp_res |</div><div class='del'>-					(be32_to_cpu(req_msg-&gt;offset32) &amp;</div><div class='del'>-					 0xFFFFFF00));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_init_depth(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) be32_to_cpu(req_msg-&gt;offset36);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_init_depth(struct cm_req_msg *req_msg,</div><div class='del'>-					 u8 init_depth)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;offset36 = cpu_to_be32(init_depth |</div><div class='del'>-					(be32_to_cpu(req_msg-&gt;offset36) &amp;</div><div class='del'>-					 0xFFFFFF00));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_remote_resp_timeout(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) ((be32_to_cpu(req_msg-&gt;offset40) &amp; 0xF8) &gt;&gt; 3);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_remote_resp_timeout(struct cm_req_msg *req_msg,</div><div class='del'>-						  u8 resp_timeout)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;offset40 = cpu_to_be32((resp_timeout &lt;&lt; 3) |</div><div class='del'>-					 (be32_to_cpu(req_msg-&gt;offset40) &amp;</div><div class='del'>-					  0xFFFFFF07));</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static inline enum ib_qp_type cm_req_get_qp_type(struct cm_req_msg *req_msg)</div><div class='ctx'> {</div><div class='del'>-	u8 transport_type = (u8) (be32_to_cpu(req_msg-&gt;offset40) &amp; 0x06) &gt;&gt; 1;</div><div class='del'>-	switch(transport_type) {</div><div class='add'>+	u8 transport_type = IBA_GET(CM_REQ_TRANSPORT_SERVICE_TYPE, req_msg);</div><div class='add'>+	switch (transport_type) {</div><div class='ctx'> 	case 0: return IB_QPT_RC;</div><div class='ctx'> 	case 1: return IB_QPT_UC;</div><div class='ctx'> 	case 3:</div><div class='del'>-		switch (req_msg-&gt;offset51 &amp; 0x7) {</div><div class='add'>+		switch (IBA_GET(CM_REQ_EXTENDED_TRANSPORT_TYPE, req_msg)) {</div><div class='ctx'> 		case 1: return IB_QPT_XRC_TGT;</div><div class='ctx'> 		default: return 0;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -168,242 +37,19 @@ static inline enum ib_qp_type cm_req_get_qp_type(struct cm_req_msg *req_msg)</div><div class='ctx'> static inline void cm_req_set_qp_type(struct cm_req_msg *req_msg,</div><div class='ctx'> 				      enum ib_qp_type qp_type)</div><div class='ctx'> {</div><div class='del'>-	switch(qp_type) {</div><div class='add'>+	switch (qp_type) {</div><div class='ctx'> 	case IB_QPT_UC:</div><div class='del'>-		req_msg-&gt;offset40 = cpu_to_be32((be32_to_cpu(</div><div class='del'>-						  req_msg-&gt;offset40) &amp;</div><div class='del'>-						   0xFFFFFFF9) | 0x2);</div><div class='add'>+		IBA_SET(CM_REQ_TRANSPORT_SERVICE_TYPE, req_msg, 1);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_QPT_XRC_INI:</div><div class='del'>-		req_msg-&gt;offset40 = cpu_to_be32((be32_to_cpu(</div><div class='del'>-						 req_msg-&gt;offset40) &amp;</div><div class='del'>-						   0xFFFFFFF9) | 0x6);</div><div class='del'>-		req_msg-&gt;offset51 = (req_msg-&gt;offset51 &amp; 0xF8) | 1;</div><div class='add'>+		IBA_SET(CM_REQ_TRANSPORT_SERVICE_TYPE, req_msg, 3);</div><div class='add'>+		IBA_SET(CM_REQ_EXTENDED_TRANSPORT_TYPE, req_msg, 1);</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='del'>-		req_msg-&gt;offset40 = cpu_to_be32(be32_to_cpu(</div><div class='del'>-						 req_msg-&gt;offset40) &amp;</div><div class='del'>-						  0xFFFFFFF9);</div><div class='add'>+		IBA_SET(CM_REQ_TRANSPORT_SERVICE_TYPE, req_msg, 0);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline u8 cm_req_get_flow_ctrl(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return be32_to_cpu(req_msg-&gt;offset40) &amp; 0x1;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_flow_ctrl(struct cm_req_msg *req_msg,</div><div class='del'>-					u8 flow_ctrl)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;offset40 = cpu_to_be32((flow_ctrl &amp; 0x1) |</div><div class='del'>-					 (be32_to_cpu(req_msg-&gt;offset40) &amp;</div><div class='del'>-					  0xFFFFFFFE));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline __be32 cm_req_get_starting_psn(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return cpu_to_be32(be32_to_cpu(req_msg-&gt;offset44) &gt;&gt; 8);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_starting_psn(struct cm_req_msg *req_msg,</div><div class='del'>-					   __be32 starting_psn)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;offset44 = cpu_to_be32((be32_to_cpu(starting_psn) &lt;&lt; 8) |</div><div class='del'>-			    (be32_to_cpu(req_msg-&gt;offset44) &amp; 0x000000FF));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_local_resp_timeout(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) ((be32_to_cpu(req_msg-&gt;offset44) &amp; 0xF8) &gt;&gt; 3);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_local_resp_timeout(struct cm_req_msg *req_msg,</div><div class='del'>-						 u8 resp_timeout)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;offset44 = cpu_to_be32((resp_timeout &lt;&lt; 3) |</div><div class='del'>-			    (be32_to_cpu(req_msg-&gt;offset44) &amp; 0xFFFFFF07));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_retry_count(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) (be32_to_cpu(req_msg-&gt;offset44) &amp; 0x7);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_retry_count(struct cm_req_msg *req_msg,</div><div class='del'>-					  u8 retry_count)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;offset44 = cpu_to_be32((retry_count &amp; 0x7) |</div><div class='del'>-			    (be32_to_cpu(req_msg-&gt;offset44) &amp; 0xFFFFFFF8));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_path_mtu(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return req_msg-&gt;offset50 &gt;&gt; 4;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_path_mtu(struct cm_req_msg *req_msg, u8 path_mtu)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;offset50 = (u8) ((req_msg-&gt;offset50 &amp; 0xF) | (path_mtu &lt;&lt; 4));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_rnr_retry_count(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return req_msg-&gt;offset50 &amp; 0x7;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_rnr_retry_count(struct cm_req_msg *req_msg,</div><div class='del'>-					      u8 rnr_retry_count)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;offset50 = (u8) ((req_msg-&gt;offset50 &amp; 0xF8) |</div><div class='del'>-				  (rnr_retry_count &amp; 0x7));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_max_cm_retries(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return req_msg-&gt;offset51 &gt;&gt; 4;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_max_cm_retries(struct cm_req_msg *req_msg,</div><div class='del'>-					     u8 retries)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;offset51 = (u8) ((req_msg-&gt;offset51 &amp; 0xF) | (retries &lt;&lt; 4));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_srq(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return (req_msg-&gt;offset51 &amp; 0x8) &gt;&gt; 3;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_srq(struct cm_req_msg *req_msg, u8 srq)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;offset51 = (u8) ((req_msg-&gt;offset51 &amp; 0xF7) |</div><div class='del'>-				  ((srq &amp; 0x1) &lt;&lt; 3));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline __be32 cm_req_get_primary_flow_label(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return cpu_to_be32(be32_to_cpu(req_msg-&gt;primary_offset88) &gt;&gt; 12);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_primary_flow_label(struct cm_req_msg *req_msg,</div><div class='del'>-						 __be32 flow_label)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;primary_offset88 = cpu_to_be32(</div><div class='del'>-				    (be32_to_cpu(req_msg-&gt;primary_offset88) &amp;</div><div class='del'>-				     0x00000FFF) |</div><div class='del'>-				     (be32_to_cpu(flow_label) &lt;&lt; 12));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_primary_packet_rate(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) (be32_to_cpu(req_msg-&gt;primary_offset88) &amp; 0x3F);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_primary_packet_rate(struct cm_req_msg *req_msg,</div><div class='del'>-						  u8 rate)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;primary_offset88 = cpu_to_be32(</div><div class='del'>-				    (be32_to_cpu(req_msg-&gt;primary_offset88) &amp;</div><div class='del'>-				     0xFFFFFFC0) | (rate &amp; 0x3F));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_primary_sl(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) (req_msg-&gt;primary_offset94 &gt;&gt; 4);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_primary_sl(struct cm_req_msg *req_msg, u8 sl)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;primary_offset94 = (u8) ((req_msg-&gt;primary_offset94 &amp; 0x0F) |</div><div class='del'>-					  (sl &lt;&lt; 4));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_primary_subnet_local(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) ((req_msg-&gt;primary_offset94 &amp; 0x08) &gt;&gt; 3);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_primary_subnet_local(struct cm_req_msg *req_msg,</div><div class='del'>-						   u8 subnet_local)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;primary_offset94 = (u8) ((req_msg-&gt;primary_offset94 &amp; 0xF7) |</div><div class='del'>-					  ((subnet_local &amp; 0x1) &lt;&lt; 3));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_primary_local_ack_timeout(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) (req_msg-&gt;primary_offset95 &gt;&gt; 3);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_primary_local_ack_timeout(struct cm_req_msg *req_msg,</div><div class='del'>-							u8 local_ack_timeout)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;primary_offset95 = (u8) ((req_msg-&gt;primary_offset95 &amp; 0x07) |</div><div class='del'>-					  (local_ack_timeout &lt;&lt; 3));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline __be32 cm_req_get_alt_flow_label(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return cpu_to_be32(be32_to_cpu(req_msg-&gt;alt_offset132) &gt;&gt; 12);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_alt_flow_label(struct cm_req_msg *req_msg,</div><div class='del'>-					     __be32 flow_label)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;alt_offset132 = cpu_to_be32(</div><div class='del'>-				 (be32_to_cpu(req_msg-&gt;alt_offset132) &amp;</div><div class='del'>-				  0x00000FFF) |</div><div class='del'>-				  (be32_to_cpu(flow_label) &lt;&lt; 12));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_alt_packet_rate(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) (be32_to_cpu(req_msg-&gt;alt_offset132) &amp; 0x3F);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_alt_packet_rate(struct cm_req_msg *req_msg,</div><div class='del'>-					      u8 rate)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;alt_offset132 = cpu_to_be32(</div><div class='del'>-				 (be32_to_cpu(req_msg-&gt;alt_offset132) &amp;</div><div class='del'>-				  0xFFFFFFC0) | (rate &amp; 0x3F));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_alt_sl(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) (req_msg-&gt;alt_offset138 &gt;&gt; 4);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_alt_sl(struct cm_req_msg *req_msg, u8 sl)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;alt_offset138 = (u8) ((req_msg-&gt;alt_offset138 &amp; 0x0F) |</div><div class='del'>-				       (sl &lt;&lt; 4));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_alt_subnet_local(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) ((req_msg-&gt;alt_offset138 &amp; 0x08) &gt;&gt; 3);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_alt_subnet_local(struct cm_req_msg *req_msg,</div><div class='del'>-					       u8 subnet_local)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;alt_offset138 = (u8) ((req_msg-&gt;alt_offset138 &amp; 0xF7) |</div><div class='del'>-				       ((subnet_local &amp; 0x1) &lt;&lt; 3));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_req_get_alt_local_ack_timeout(struct cm_req_msg *req_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) (req_msg-&gt;alt_offset139 &gt;&gt; 3);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_req_set_alt_local_ack_timeout(struct cm_req_msg *req_msg,</div><div class='del'>-						    u8 local_ack_timeout)</div><div class='del'>-{</div><div class='del'>-	req_msg-&gt;alt_offset139 = (u8) ((req_msg-&gt;alt_offset139 &amp; 0x07) |</div><div class='del'>-				       (local_ack_timeout &lt;&lt; 3));</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> /* Message REJected or MRAed */</div><div class='ctx'> enum cm_msg_response {</div><div class='ctx'> 	CM_MSG_RESPONSE_REQ = 0x0,</div><div class='hunk'>@@ -411,419 +57,12 @@ enum cm_msg_response {</div><div class='ctx'> 	CM_MSG_RESPONSE_OTHER = 0x2</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>- struct cm_mra_msg {</div><div class='del'>-	struct ib_mad_hdr hdr;</div><div class='del'>-</div><div class='del'>-	__be32 local_comm_id;</div><div class='del'>-	__be32 remote_comm_id;</div><div class='del'>-	/* message MRAed:2, rsvd:6 */</div><div class='del'>-	u8 offset8;</div><div class='del'>-	/* service timeout:5, rsvd:3 */</div><div class='del'>-	u8 offset9;</div><div class='del'>-</div><div class='del'>-	u8 private_data[IB_CM_MRA_PRIVATE_DATA_SIZE];</div><div class='del'>-</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-static inline u8 cm_mra_get_msg_mraed(struct cm_mra_msg *mra_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) (mra_msg-&gt;offset8 &gt;&gt; 6);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_mra_set_msg_mraed(struct cm_mra_msg *mra_msg, u8 msg)</div><div class='del'>-{</div><div class='del'>-	mra_msg-&gt;offset8 = (u8) ((mra_msg-&gt;offset8 &amp; 0x3F) | (msg &lt;&lt; 6));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_mra_get_service_timeout(struct cm_mra_msg *mra_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) (mra_msg-&gt;offset9 &gt;&gt; 3);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_mra_set_service_timeout(struct cm_mra_msg *mra_msg,</div><div class='del'>-					      u8 service_timeout)</div><div class='del'>-{</div><div class='del'>-	mra_msg-&gt;offset9 = (u8) ((mra_msg-&gt;offset9 &amp; 0x07) |</div><div class='del'>-				 (service_timeout &lt;&lt; 3));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct cm_rej_msg {</div><div class='del'>-	struct ib_mad_hdr hdr;</div><div class='del'>-</div><div class='del'>-	__be32 local_comm_id;</div><div class='del'>-	__be32 remote_comm_id;</div><div class='del'>-	/* message REJected:2, rsvd:6 */</div><div class='del'>-	u8 offset8;</div><div class='del'>-	/* reject info length:7, rsvd:1. */</div><div class='del'>-	u8 offset9;</div><div class='del'>-	__be16 reason;</div><div class='del'>-	u8 ari[IB_CM_REJ_ARI_LENGTH];</div><div class='del'>-</div><div class='del'>-	u8 private_data[IB_CM_REJ_PRIVATE_DATA_SIZE];</div><div class='del'>-</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-static inline u8 cm_rej_get_msg_rejected(struct cm_rej_msg *rej_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) (rej_msg-&gt;offset8 &gt;&gt; 6);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_rej_set_msg_rejected(struct cm_rej_msg *rej_msg, u8 msg)</div><div class='del'>-{</div><div class='del'>-	rej_msg-&gt;offset8 = (u8) ((rej_msg-&gt;offset8 &amp; 0x3F) | (msg &lt;&lt; 6));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_rej_get_reject_info_len(struct cm_rej_msg *rej_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) (rej_msg-&gt;offset9 &gt;&gt; 1);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_rej_set_reject_info_len(struct cm_rej_msg *rej_msg,</div><div class='del'>-					      u8 len)</div><div class='del'>-{</div><div class='del'>-	rej_msg-&gt;offset9 = (u8) ((rej_msg-&gt;offset9 &amp; 0x1) | (len &lt;&lt; 1));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct cm_rep_msg {</div><div class='del'>-	struct ib_mad_hdr hdr;</div><div class='del'>-</div><div class='del'>-	__be32 local_comm_id;</div><div class='del'>-	__be32 remote_comm_id;</div><div class='del'>-	__be32 local_qkey;</div><div class='del'>-	/* local QPN:24, rsvd:8 */</div><div class='del'>-	__be32 offset12;</div><div class='del'>-	/* local EECN:24, rsvd:8 */</div><div class='del'>-	__be32 offset16;</div><div class='del'>-	/* starting PSN:24 rsvd:8 */</div><div class='del'>-	__be32 offset20;</div><div class='del'>-	u8 resp_resources;</div><div class='del'>-	u8 initiator_depth;</div><div class='del'>-	/* target ACK delay:5, failover accepted:2, end-to-end flow control:1 */</div><div class='del'>-	u8 offset26;</div><div class='del'>-	/* RNR retry count:3, SRQ:1, rsvd:5 */</div><div class='del'>-	u8 offset27;</div><div class='del'>-	__be64 local_ca_guid;</div><div class='del'>-</div><div class='del'>-	u8 private_data[IB_CM_REP_PRIVATE_DATA_SIZE];</div><div class='del'>-</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-static inline __be32 cm_rep_get_local_qpn(struct cm_rep_msg *rep_msg)</div><div class='del'>-{</div><div class='del'>-	return cpu_to_be32(be32_to_cpu(rep_msg-&gt;offset12) &gt;&gt; 8);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_rep_set_local_qpn(struct cm_rep_msg *rep_msg, __be32 qpn)</div><div class='del'>-{</div><div class='del'>-	rep_msg-&gt;offset12 = cpu_to_be32((be32_to_cpu(qpn) &lt;&lt; 8) |</div><div class='del'>-			    (be32_to_cpu(rep_msg-&gt;offset12) &amp; 0x000000FF));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline __be32 cm_rep_get_local_eecn(struct cm_rep_msg *rep_msg)</div><div class='del'>-{</div><div class='del'>-	return cpu_to_be32(be32_to_cpu(rep_msg-&gt;offset16) &gt;&gt; 8);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_rep_set_local_eecn(struct cm_rep_msg *rep_msg, __be32 eecn)</div><div class='del'>-{</div><div class='del'>-	rep_msg-&gt;offset16 = cpu_to_be32((be32_to_cpu(eecn) &lt;&lt; 8) |</div><div class='del'>-			    (be32_to_cpu(rep_msg-&gt;offset16) &amp; 0x000000FF));</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static inline __be32 cm_rep_get_qpn(struct cm_rep_msg *rep_msg, enum ib_qp_type qp_type)</div><div class='ctx'> {</div><div class='ctx'> 	return (qp_type == IB_QPT_XRC_INI) ?</div><div class='del'>-		cm_rep_get_local_eecn(rep_msg) : cm_rep_get_local_qpn(rep_msg);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline __be32 cm_rep_get_starting_psn(struct cm_rep_msg *rep_msg)</div><div class='del'>-{</div><div class='del'>-	return cpu_to_be32(be32_to_cpu(rep_msg-&gt;offset20) &gt;&gt; 8);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_rep_set_starting_psn(struct cm_rep_msg *rep_msg,</div><div class='del'>-					   __be32 starting_psn)</div><div class='del'>-{</div><div class='del'>-	rep_msg-&gt;offset20 = cpu_to_be32((be32_to_cpu(starting_psn) &lt;&lt; 8) |</div><div class='del'>-			    (be32_to_cpu(rep_msg-&gt;offset20) &amp; 0x000000FF));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_rep_get_target_ack_delay(struct cm_rep_msg *rep_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) (rep_msg-&gt;offset26 &gt;&gt; 3);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_rep_set_target_ack_delay(struct cm_rep_msg *rep_msg,</div><div class='del'>-					       u8 target_ack_delay)</div><div class='del'>-{</div><div class='del'>-	rep_msg-&gt;offset26 = (u8) ((rep_msg-&gt;offset26 &amp; 0x07) |</div><div class='del'>-				  (target_ack_delay &lt;&lt; 3));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_rep_get_failover(struct cm_rep_msg *rep_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) ((rep_msg-&gt;offset26 &amp; 0x06) &gt;&gt; 1);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_rep_set_failover(struct cm_rep_msg *rep_msg, u8 failover)</div><div class='del'>-{</div><div class='del'>-	rep_msg-&gt;offset26 = (u8) ((rep_msg-&gt;offset26 &amp; 0xF9) |</div><div class='del'>-				  ((failover &amp; 0x3) &lt;&lt; 1));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_rep_get_flow_ctrl(struct cm_rep_msg *rep_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) (rep_msg-&gt;offset26 &amp; 0x01);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_rep_set_flow_ctrl(struct cm_rep_msg *rep_msg,</div><div class='del'>-					    u8 flow_ctrl)</div><div class='del'>-{</div><div class='del'>-	rep_msg-&gt;offset26 = (u8) ((rep_msg-&gt;offset26 &amp; 0xFE) |</div><div class='del'>-				  (flow_ctrl &amp; 0x1));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_rep_get_rnr_retry_count(struct cm_rep_msg *rep_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) (rep_msg-&gt;offset27 &gt;&gt; 5);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_rep_set_rnr_retry_count(struct cm_rep_msg *rep_msg,</div><div class='del'>-					      u8 rnr_retry_count)</div><div class='del'>-{</div><div class='del'>-	rep_msg-&gt;offset27 = (u8) ((rep_msg-&gt;offset27 &amp; 0x1F) |</div><div class='del'>-				  (rnr_retry_count &lt;&lt; 5));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_rep_get_srq(struct cm_rep_msg *rep_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) ((rep_msg-&gt;offset27 &gt;&gt; 4) &amp; 0x1);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_rep_set_srq(struct cm_rep_msg *rep_msg, u8 srq)</div><div class='del'>-{</div><div class='del'>-	rep_msg-&gt;offset27 = (u8) ((rep_msg-&gt;offset27 &amp; 0xEF) |</div><div class='del'>-				  ((srq &amp; 0x1) &lt;&lt; 4));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct cm_rtu_msg {</div><div class='del'>-	struct ib_mad_hdr hdr;</div><div class='del'>-</div><div class='del'>-	__be32 local_comm_id;</div><div class='del'>-	__be32 remote_comm_id;</div><div class='del'>-</div><div class='del'>-	u8 private_data[IB_CM_RTU_PRIVATE_DATA_SIZE];</div><div class='del'>-</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-struct cm_dreq_msg {</div><div class='del'>-	struct ib_mad_hdr hdr;</div><div class='del'>-</div><div class='del'>-	__be32 local_comm_id;</div><div class='del'>-	__be32 remote_comm_id;</div><div class='del'>-	/* remote QPN/EECN:24, rsvd:8 */</div><div class='del'>-	__be32 offset8;</div><div class='del'>-</div><div class='del'>-	u8 private_data[IB_CM_DREQ_PRIVATE_DATA_SIZE];</div><div class='del'>-</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-static inline __be32 cm_dreq_get_remote_qpn(struct cm_dreq_msg *dreq_msg)</div><div class='del'>-{</div><div class='del'>-	return cpu_to_be32(be32_to_cpu(dreq_msg-&gt;offset8) &gt;&gt; 8);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_dreq_set_remote_qpn(struct cm_dreq_msg *dreq_msg, __be32 qpn)</div><div class='del'>-{</div><div class='del'>-	dreq_msg-&gt;offset8 = cpu_to_be32((be32_to_cpu(qpn) &lt;&lt; 8) |</div><div class='del'>-			    (be32_to_cpu(dreq_msg-&gt;offset8) &amp; 0x000000FF));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct cm_drep_msg {</div><div class='del'>-	struct ib_mad_hdr hdr;</div><div class='del'>-</div><div class='del'>-	__be32 local_comm_id;</div><div class='del'>-	__be32 remote_comm_id;</div><div class='del'>-</div><div class='del'>-	u8 private_data[IB_CM_DREP_PRIVATE_DATA_SIZE];</div><div class='del'>-</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-struct cm_lap_msg {</div><div class='del'>-	struct ib_mad_hdr hdr;</div><div class='del'>-</div><div class='del'>-	__be32 local_comm_id;</div><div class='del'>-	__be32 remote_comm_id;</div><div class='del'>-</div><div class='del'>-	__be32 rsvd8;</div><div class='del'>-	/* remote QPN/EECN:24, remote CM response timeout:5, rsvd:3 */</div><div class='del'>-	__be32 offset12;</div><div class='del'>-	__be32 rsvd16;</div><div class='del'>-</div><div class='del'>-	__be16 alt_local_lid;</div><div class='del'>-	__be16 alt_remote_lid;</div><div class='del'>-	union ib_gid alt_local_gid;</div><div class='del'>-	union ib_gid alt_remote_gid;</div><div class='del'>-	/* flow label:20, rsvd:4, traffic class:8 */</div><div class='del'>-	__be32 offset56;</div><div class='del'>-	u8 alt_hop_limit;</div><div class='del'>-	/* rsvd:2, packet rate:6 */</div><div class='del'>-	u8 offset61;</div><div class='del'>-	/* SL:4, subnet local:1, rsvd:3 */</div><div class='del'>-	u8 offset62;</div><div class='del'>-	/* local ACK timeout:5, rsvd:3 */</div><div class='del'>-	u8 offset63;</div><div class='del'>-</div><div class='del'>-	u8 private_data[IB_CM_LAP_PRIVATE_DATA_SIZE];</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-static inline __be32 cm_lap_get_remote_qpn(struct cm_lap_msg *lap_msg)</div><div class='del'>-{</div><div class='del'>-	return cpu_to_be32(be32_to_cpu(lap_msg-&gt;offset12) &gt;&gt; 8);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_lap_set_remote_qpn(struct cm_lap_msg *lap_msg, __be32 qpn)</div><div class='del'>-{</div><div class='del'>-	lap_msg-&gt;offset12 = cpu_to_be32((be32_to_cpu(qpn) &lt;&lt; 8) |</div><div class='del'>-					 (be32_to_cpu(lap_msg-&gt;offset12) &amp;</div><div class='del'>-					  0x000000FF));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_lap_get_remote_resp_timeout(struct cm_lap_msg *lap_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) ((be32_to_cpu(lap_msg-&gt;offset12) &amp; 0xF8) &gt;&gt; 3);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_lap_set_remote_resp_timeout(struct cm_lap_msg *lap_msg,</div><div class='del'>-						  u8 resp_timeout)</div><div class='del'>-{</div><div class='del'>-	lap_msg-&gt;offset12 = cpu_to_be32((resp_timeout &lt;&lt; 3) |</div><div class='del'>-					 (be32_to_cpu(lap_msg-&gt;offset12) &amp;</div><div class='del'>-					  0xFFFFFF07));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline __be32 cm_lap_get_flow_label(struct cm_lap_msg *lap_msg)</div><div class='del'>-{</div><div class='del'>-	return cpu_to_be32(be32_to_cpu(lap_msg-&gt;offset56) &gt;&gt; 12);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_lap_set_flow_label(struct cm_lap_msg *lap_msg,</div><div class='del'>-					 __be32 flow_label)</div><div class='del'>-{</div><div class='del'>-	lap_msg-&gt;offset56 = cpu_to_be32(</div><div class='del'>-				 (be32_to_cpu(lap_msg-&gt;offset56) &amp; 0x00000FFF) |</div><div class='del'>-				 (be32_to_cpu(flow_label) &lt;&lt; 12));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_lap_get_traffic_class(struct cm_lap_msg *lap_msg)</div><div class='del'>-{</div><div class='del'>-	return (u8) be32_to_cpu(lap_msg-&gt;offset56);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_lap_set_traffic_class(struct cm_lap_msg *lap_msg,</div><div class='del'>-					    u8 traffic_class)</div><div class='del'>-{</div><div class='del'>-	lap_msg-&gt;offset56 = cpu_to_be32(traffic_class |</div><div class='del'>-					 (be32_to_cpu(lap_msg-&gt;offset56) &amp;</div><div class='del'>-					  0xFFFFFF00));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_lap_get_packet_rate(struct cm_lap_msg *lap_msg)</div><div class='del'>-{</div><div class='del'>-	return lap_msg-&gt;offset61 &amp; 0x3F;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_lap_set_packet_rate(struct cm_lap_msg *lap_msg,</div><div class='del'>-					  u8 packet_rate)</div><div class='del'>-{</div><div class='del'>-	lap_msg-&gt;offset61 = (packet_rate &amp; 0x3F) | (lap_msg-&gt;offset61 &amp; 0xC0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_lap_get_sl(struct cm_lap_msg *lap_msg)</div><div class='del'>-{</div><div class='del'>-	return lap_msg-&gt;offset62 &gt;&gt; 4;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_lap_set_sl(struct cm_lap_msg *lap_msg, u8 sl)</div><div class='del'>-{</div><div class='del'>-	lap_msg-&gt;offset62 = (sl &lt;&lt; 4) | (lap_msg-&gt;offset62 &amp; 0x0F);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 cm_lap_get_subnet_local(struct cm_lap_msg *lap_msg)</div><div class='del'>-{</div><div class='del'>-	return (lap_msg-&gt;offset62 &gt;&gt; 3) &amp; 0x1;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_lap_set_subnet_local(struct cm_lap_msg *lap_msg,</div><div class='del'>-					   u8 subnet_local)</div><div class='del'>-{</div><div class='del'>-	lap_msg-&gt;offset62 = ((subnet_local &amp; 0x1) &lt;&lt; 3) |</div><div class='del'>-			     (lap_msg-&gt;offset61 &amp; 0xF7);</div><div class='del'>-}</div><div class='del'>-static inline u8 cm_lap_get_local_ack_timeout(struct cm_lap_msg *lap_msg)</div><div class='del'>-{</div><div class='del'>-	return lap_msg-&gt;offset63 &gt;&gt; 3;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_lap_set_local_ack_timeout(struct cm_lap_msg *lap_msg,</div><div class='del'>-						u8 local_ack_timeout)</div><div class='del'>-{</div><div class='del'>-	lap_msg-&gt;offset63 = (local_ack_timeout &lt;&lt; 3) |</div><div class='del'>-			    (lap_msg-&gt;offset63 &amp; 0x07);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct cm_apr_msg {</div><div class='del'>-	struct ib_mad_hdr hdr;</div><div class='del'>-</div><div class='del'>-	__be32 local_comm_id;</div><div class='del'>-	__be32 remote_comm_id;</div><div class='del'>-</div><div class='del'>-	u8 info_length;</div><div class='del'>-	u8 ap_status;</div><div class='del'>-	__be16 rsvd;</div><div class='del'>-	u8 info[IB_CM_APR_INFO_LENGTH];</div><div class='del'>-</div><div class='del'>-	u8 private_data[IB_CM_APR_PRIVATE_DATA_SIZE];</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-struct cm_sidr_req_msg {</div><div class='del'>-	struct ib_mad_hdr hdr;</div><div class='del'>-</div><div class='del'>-	__be32 request_id;</div><div class='del'>-	__be16 pkey;</div><div class='del'>-	__be16 rsvd;</div><div class='del'>-	__be64 service_id;</div><div class='del'>-</div><div class='del'>-	u32 private_data[IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE / sizeof(u32)];</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-struct cm_sidr_rep_msg {</div><div class='del'>-	struct ib_mad_hdr hdr;</div><div class='del'>-</div><div class='del'>-	__be32 request_id;</div><div class='del'>-	u8 status;</div><div class='del'>-	u8 info_length;</div><div class='del'>-	__be16 rsvd;</div><div class='del'>-	/* QPN:24, rsvd:8 */</div><div class='del'>-	__be32 offset8;</div><div class='del'>-	__be64 service_id;</div><div class='del'>-	__be32 qkey;</div><div class='del'>-	u8 info[IB_CM_SIDR_REP_INFO_LENGTH];</div><div class='del'>-</div><div class='del'>-	u8 private_data[IB_CM_SIDR_REP_PRIVATE_DATA_SIZE];</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-static inline __be32 cm_sidr_rep_get_qpn(struct cm_sidr_rep_msg *sidr_rep_msg)</div><div class='del'>-{</div><div class='del'>-	return cpu_to_be32(be32_to_cpu(sidr_rep_msg-&gt;offset8) &gt;&gt; 8);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cm_sidr_rep_set_qpn(struct cm_sidr_rep_msg *sidr_rep_msg,</div><div class='del'>-				       __be32 qpn)</div><div class='del'>-{</div><div class='del'>-	sidr_rep_msg-&gt;offset8 = cpu_to_be32((be32_to_cpu(qpn) &lt;&lt; 8) |</div><div class='del'>-					(be32_to_cpu(sidr_rep_msg-&gt;offset8) &amp;</div><div class='del'>-					 0x000000FF));</div><div class='add'>+		       cpu_to_be32(IBA_GET(CM_REP_LOCAL_EE_CONTEXT_NUMBER,</div><div class='add'>+					   rep_msg)) :</div><div class='add'>+		       cpu_to_be32(IBA_GET(CM_REP_LOCAL_QPN, rep_msg));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> #endif /* CM_MSGS_H */</div><div class='head'>diff --git a/drivers/infiniband/core/cm_trace.c b/drivers/infiniband/core/cm_trace.c<br/>new file mode 100644<br/>index 000000000000..8f3482f66338<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cm_trace.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/cm_trace.c</a></div><div class='hunk'>@@ -0,0 +1,15 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+/*</div><div class='add'>+ * Trace points for the IB Connection Manager.</div><div class='add'>+ *</div><div class='add'>+ * Author: Chuck Lever &lt;chuck.lever@oracle.com&gt;</div><div class='add'>+ *</div><div class='add'>+ * Copyright (c) 2020, Oracle and/or its affiliates.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/rdma_cm.h&gt;</div><div class='add'>+#include "cma_priv.h"</div><div class='add'>+</div><div class='add'>+#define CREATE_TRACE_POINTS</div><div class='add'>+</div><div class='add'>+#include "cm_trace.h"</div><div class='head'>diff --git a/drivers/infiniband/core/cm_trace.h b/drivers/infiniband/core/cm_trace.h<br/>new file mode 100644<br/>index 000000000000..4a4987da69d4<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cm_trace.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/cm_trace.h</a></div><div class='hunk'>@@ -0,0 +1,414 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0-only */</div><div class='add'>+/*</div><div class='add'>+ * Trace point definitions for the RDMA Connect Manager.</div><div class='add'>+ *</div><div class='add'>+ * Author: Chuck Lever &lt;chuck.lever@oracle.com&gt;</div><div class='add'>+ *</div><div class='add'>+ * Copyright (c) 2020 Oracle and/or its affiliates.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#undef TRACE_SYSTEM</div><div class='add'>+#define TRACE_SYSTEM ib_cma</div><div class='add'>+</div><div class='add'>+#if !defined(_TRACE_IB_CMA_H) || defined(TRACE_HEADER_MULTI_READ)</div><div class='add'>+</div><div class='add'>+#define _TRACE_IB_CMA_H</div><div class='add'>+</div><div class='add'>+#include &lt;linux/tracepoint.h&gt;</div><div class='add'>+#include &lt;rdma/ib_cm.h&gt;</div><div class='add'>+#include &lt;trace/misc/rdma.h&gt;</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * enum ib_cm_state, from include/rdma/ib_cm.h</div><div class='add'>+ */</div><div class='add'>+#define IB_CM_STATE_LIST					\</div><div class='add'>+	ib_cm_state(IDLE)					\</div><div class='add'>+	ib_cm_state(LISTEN)					\</div><div class='add'>+	ib_cm_state(REQ_SENT)					\</div><div class='add'>+	ib_cm_state(REQ_RCVD)					\</div><div class='add'>+	ib_cm_state(MRA_REQ_SENT)				\</div><div class='add'>+	ib_cm_state(MRA_REQ_RCVD)				\</div><div class='add'>+	ib_cm_state(REP_SENT)					\</div><div class='add'>+	ib_cm_state(REP_RCVD)					\</div><div class='add'>+	ib_cm_state(MRA_REP_SENT)				\</div><div class='add'>+	ib_cm_state(MRA_REP_RCVD)				\</div><div class='add'>+	ib_cm_state(ESTABLISHED)				\</div><div class='add'>+	ib_cm_state(DREQ_SENT)					\</div><div class='add'>+	ib_cm_state(DREQ_RCVD)					\</div><div class='add'>+	ib_cm_state(TIMEWAIT)					\</div><div class='add'>+	ib_cm_state(SIDR_REQ_SENT)				\</div><div class='add'>+	ib_cm_state_end(SIDR_REQ_RCVD)</div><div class='add'>+</div><div class='add'>+#undef  ib_cm_state</div><div class='add'>+#undef  ib_cm_state_end</div><div class='add'>+#define ib_cm_state(x)		TRACE_DEFINE_ENUM(IB_CM_##x);</div><div class='add'>+#define ib_cm_state_end(x)	TRACE_DEFINE_ENUM(IB_CM_##x);</div><div class='add'>+</div><div class='add'>+IB_CM_STATE_LIST</div><div class='add'>+</div><div class='add'>+#undef  ib_cm_state</div><div class='add'>+#undef  ib_cm_state_end</div><div class='add'>+#define ib_cm_state(x)		{ IB_CM_##x, #x },</div><div class='add'>+#define ib_cm_state_end(x)	{ IB_CM_##x, #x }</div><div class='add'>+</div><div class='add'>+#define show_ib_cm_state(x) \</div><div class='add'>+		__print_symbolic(x, IB_CM_STATE_LIST)</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * enum ib_cm_lap_state, from include/rdma/ib_cm.h</div><div class='add'>+ */</div><div class='add'>+#define IB_CM_LAP_STATE_LIST					\</div><div class='add'>+	ib_cm_lap_state(LAP_UNINIT)				\</div><div class='add'>+	ib_cm_lap_state(LAP_IDLE)				\</div><div class='add'>+	ib_cm_lap_state(LAP_SENT)				\</div><div class='add'>+	ib_cm_lap_state(LAP_RCVD)				\</div><div class='add'>+	ib_cm_lap_state(MRA_LAP_SENT)				\</div><div class='add'>+	ib_cm_lap_state_end(MRA_LAP_RCVD)</div><div class='add'>+</div><div class='add'>+#undef  ib_cm_lap_state</div><div class='add'>+#undef  ib_cm_lap_state_end</div><div class='add'>+#define ib_cm_lap_state(x)	TRACE_DEFINE_ENUM(IB_CM_##x);</div><div class='add'>+#define ib_cm_lap_state_end(x)	TRACE_DEFINE_ENUM(IB_CM_##x);</div><div class='add'>+</div><div class='add'>+IB_CM_LAP_STATE_LIST</div><div class='add'>+</div><div class='add'>+#undef  ib_cm_lap_state</div><div class='add'>+#undef  ib_cm_lap_state_end</div><div class='add'>+#define ib_cm_lap_state(x)	{ IB_CM_##x, #x },</div><div class='add'>+#define ib_cm_lap_state_end(x)	{ IB_CM_##x, #x }</div><div class='add'>+</div><div class='add'>+#define show_ib_cm_lap_state(x) \</div><div class='add'>+		__print_symbolic(x, IB_CM_LAP_STATE_LIST)</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * enum ib_cm_rej_reason, from include/rdma/ib_cm.h</div><div class='add'>+ */</div><div class='add'>+#define IB_CM_REJ_REASON_LIST					\</div><div class='add'>+	ib_cm_rej_reason(REJ_NO_QP)				\</div><div class='add'>+	ib_cm_rej_reason(REJ_NO_EEC)				\</div><div class='add'>+	ib_cm_rej_reason(REJ_NO_RESOURCES)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_TIMEOUT)				\</div><div class='add'>+	ib_cm_rej_reason(REJ_UNSUPPORTED)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_COMM_ID)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_COMM_INSTANCE)		\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_SERVICE_ID)		\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_TRANSPORT_TYPE)		\</div><div class='add'>+	ib_cm_rej_reason(REJ_STALE_CONN)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_RDC_NOT_EXIST)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_GID)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_LID)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_SL)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_TRAFFIC_CLASS)		\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_HOP_LIMIT)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_PACKET_RATE)		\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_ALT_GID)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_ALT_LID)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_ALT_SL)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_ALT_TRAFFIC_CLASS)		\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_ALT_HOP_LIMIT)		\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_ALT_PACKET_RATE)		\</div><div class='add'>+	ib_cm_rej_reason(REJ_PORT_CM_REDIRECT)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_PORT_REDIRECT)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_MTU)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_INSUFFICIENT_RESP_RESOURCES)	\</div><div class='add'>+	ib_cm_rej_reason(REJ_CONSUMER_DEFINED)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_RNR_RETRY)			\</div><div class='add'>+	ib_cm_rej_reason(REJ_DUPLICATE_LOCAL_COMM_ID)		\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_CLASS_VERSION)		\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_FLOW_LABEL)		\</div><div class='add'>+	ib_cm_rej_reason(REJ_INVALID_ALT_FLOW_LABEL)		\</div><div class='add'>+	ib_cm_rej_reason_end(REJ_VENDOR_OPTION_NOT_SUPPORTED)</div><div class='add'>+</div><div class='add'>+#undef  ib_cm_rej_reason</div><div class='add'>+#undef  ib_cm_rej_reason_end</div><div class='add'>+#define ib_cm_rej_reason(x)	TRACE_DEFINE_ENUM(IB_CM_##x);</div><div class='add'>+#define ib_cm_rej_reason_end(x)	TRACE_DEFINE_ENUM(IB_CM_##x);</div><div class='add'>+</div><div class='add'>+IB_CM_REJ_REASON_LIST</div><div class='add'>+</div><div class='add'>+#undef  ib_cm_rej_reason</div><div class='add'>+#undef  ib_cm_rej_reason_end</div><div class='add'>+#define ib_cm_rej_reason(x)	{ IB_CM_##x, #x },</div><div class='add'>+#define ib_cm_rej_reason_end(x)	{ IB_CM_##x, #x }</div><div class='add'>+</div><div class='add'>+#define show_ib_cm_rej_reason(x) \</div><div class='add'>+		__print_symbolic(x, IB_CM_REJ_REASON_LIST)</div><div class='add'>+</div><div class='add'>+DECLARE_EVENT_CLASS(icm_id_class,</div><div class='add'>+	TP_PROTO(</div><div class='add'>+		const struct ib_cm_id *cm_id</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_ARGS(cm_id),</div><div class='add'>+</div><div class='add'>+	TP_STRUCT__entry(</div><div class='add'>+		__field(const void *, cm_id)	/* for eBPF scripts */</div><div class='add'>+		__field(unsigned int, local_id)</div><div class='add'>+		__field(unsigned int, remote_id)</div><div class='add'>+		__field(unsigned long, state)</div><div class='add'>+		__field(unsigned long, lap_state)</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_fast_assign(</div><div class='add'>+		__entry-&gt;cm_id = cm_id;</div><div class='add'>+		__entry-&gt;local_id = be32_to_cpu(cm_id-&gt;local_id);</div><div class='add'>+		__entry-&gt;remote_id = be32_to_cpu(cm_id-&gt;remote_id);</div><div class='add'>+		__entry-&gt;state = cm_id-&gt;state;</div><div class='add'>+		__entry-&gt;lap_state = cm_id-&gt;lap_state;</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_printk("local_id=%u remote_id=%u state=%s lap_state=%s",</div><div class='add'>+		__entry-&gt;local_id, __entry-&gt;remote_id,</div><div class='add'>+		show_ib_cm_state(__entry-&gt;state),</div><div class='add'>+		show_ib_cm_lap_state(__entry-&gt;lap_state)</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+#define DEFINE_CM_SEND_EVENT(name)					\</div><div class='add'>+		DEFINE_EVENT(icm_id_class,				\</div><div class='add'>+				icm_send_##name,				\</div><div class='add'>+				TP_PROTO(				\</div><div class='add'>+					const struct ib_cm_id *cm_id	\</div><div class='add'>+				),					\</div><div class='add'>+				TP_ARGS(cm_id))</div><div class='add'>+</div><div class='add'>+DEFINE_CM_SEND_EVENT(req);</div><div class='add'>+DEFINE_CM_SEND_EVENT(rep);</div><div class='add'>+DEFINE_CM_SEND_EVENT(dup_req);</div><div class='add'>+DEFINE_CM_SEND_EVENT(dup_rep);</div><div class='add'>+DEFINE_CM_SEND_EVENT(rtu);</div><div class='add'>+DEFINE_CM_SEND_EVENT(mra);</div><div class='add'>+DEFINE_CM_SEND_EVENT(sidr_req);</div><div class='add'>+DEFINE_CM_SEND_EVENT(sidr_rep);</div><div class='add'>+DEFINE_CM_SEND_EVENT(dreq);</div><div class='add'>+DEFINE_CM_SEND_EVENT(drep);</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(icm_send_rej,</div><div class='add'>+	TP_PROTO(</div><div class='add'>+		const struct ib_cm_id *cm_id,</div><div class='add'>+		enum ib_cm_rej_reason reason</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_ARGS(cm_id, reason),</div><div class='add'>+</div><div class='add'>+	TP_STRUCT__entry(</div><div class='add'>+		__field(const void *, cm_id)</div><div class='add'>+		__field(u32, local_id)</div><div class='add'>+		__field(u32, remote_id)</div><div class='add'>+		__field(unsigned long, state)</div><div class='add'>+		__field(unsigned long, reason)</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_fast_assign(</div><div class='add'>+		__entry-&gt;cm_id = cm_id;</div><div class='add'>+		__entry-&gt;local_id = be32_to_cpu(cm_id-&gt;local_id);</div><div class='add'>+		__entry-&gt;remote_id = be32_to_cpu(cm_id-&gt;remote_id);</div><div class='add'>+		__entry-&gt;state = cm_id-&gt;state;</div><div class='add'>+		__entry-&gt;reason = reason;</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_printk("local_id=%u remote_id=%u state=%s reason=%s",</div><div class='add'>+		__entry-&gt;local_id, __entry-&gt;remote_id,</div><div class='add'>+		show_ib_cm_state(__entry-&gt;state),</div><div class='add'>+		show_ib_cm_rej_reason(__entry-&gt;reason)</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+#define DEFINE_CM_ERR_EVENT(name)					\</div><div class='add'>+		DEFINE_EVENT(icm_id_class,				\</div><div class='add'>+				icm_##name##_err,			\</div><div class='add'>+				TP_PROTO(				\</div><div class='add'>+					const struct ib_cm_id *cm_id	\</div><div class='add'>+				),					\</div><div class='add'>+				TP_ARGS(cm_id))</div><div class='add'>+</div><div class='add'>+DEFINE_CM_ERR_EVENT(send_cm_rtu);</div><div class='add'>+DEFINE_CM_ERR_EVENT(establish);</div><div class='add'>+DEFINE_CM_ERR_EVENT(no_listener);</div><div class='add'>+DEFINE_CM_ERR_EVENT(send_drep);</div><div class='add'>+DEFINE_CM_ERR_EVENT(dreq_unknown);</div><div class='add'>+DEFINE_CM_ERR_EVENT(send_unknown_rej);</div><div class='add'>+DEFINE_CM_ERR_EVENT(rej_unknown);</div><div class='add'>+DEFINE_CM_ERR_EVENT(prepare_mra_unknown);</div><div class='add'>+DEFINE_CM_ERR_EVENT(mra_unknown);</div><div class='add'>+DEFINE_CM_ERR_EVENT(qp_init);</div><div class='add'>+DEFINE_CM_ERR_EVENT(qp_rtr);</div><div class='add'>+DEFINE_CM_ERR_EVENT(qp_rts);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(icm_id_class,						\</div><div class='add'>+	icm_dreq_skipped,						\</div><div class='add'>+	TP_PROTO(							\</div><div class='add'>+		const struct ib_cm_id *cm_id				\</div><div class='add'>+	),								\</div><div class='add'>+	TP_ARGS(cm_id)							\</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DECLARE_EVENT_CLASS(icm_local_class,</div><div class='add'>+	TP_PROTO(</div><div class='add'>+		unsigned int local_id,</div><div class='add'>+		unsigned int remote_id</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_ARGS(local_id, remote_id),</div><div class='add'>+</div><div class='add'>+	TP_STRUCT__entry(</div><div class='add'>+		__field(unsigned int, local_id)</div><div class='add'>+		__field(unsigned int, remote_id)</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_fast_assign(</div><div class='add'>+		__entry-&gt;local_id = local_id;</div><div class='add'>+		__entry-&gt;remote_id = remote_id;</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_printk("local_id=%u remote_id=%u",</div><div class='add'>+		__entry-&gt;local_id, __entry-&gt;remote_id</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+#define DEFINE_CM_LOCAL_EVENT(name)					\</div><div class='add'>+		DEFINE_EVENT(icm_local_class,				\</div><div class='add'>+				icm_##name,				\</div><div class='add'>+				TP_PROTO(				\</div><div class='add'>+					unsigned int local_id,			\</div><div class='add'>+					unsigned int remote_id			\</div><div class='add'>+				),					\</div><div class='add'>+				TP_ARGS(local_id, remote_id))</div><div class='add'>+</div><div class='add'>+DEFINE_CM_LOCAL_EVENT(issue_rej);</div><div class='add'>+DEFINE_CM_LOCAL_EVENT(issue_drep);</div><div class='add'>+DEFINE_CM_LOCAL_EVENT(staleconn_err);</div><div class='add'>+DEFINE_CM_LOCAL_EVENT(no_priv_err);</div><div class='add'>+</div><div class='add'>+DECLARE_EVENT_CLASS(icm_remote_class,</div><div class='add'>+	TP_PROTO(</div><div class='add'>+		u32 remote_id</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_ARGS(remote_id),</div><div class='add'>+</div><div class='add'>+	TP_STRUCT__entry(</div><div class='add'>+		__field(u32, remote_id)</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_fast_assign(</div><div class='add'>+		__entry-&gt;remote_id = remote_id;</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_printk("remote_id=%u",</div><div class='add'>+		__entry-&gt;remote_id</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+#define DEFINE_CM_REMOTE_EVENT(name)					\</div><div class='add'>+		DEFINE_EVENT(icm_remote_class,				\</div><div class='add'>+				icm_##name,				\</div><div class='add'>+				TP_PROTO(				\</div><div class='add'>+					u32 remote_id			\</div><div class='add'>+				),					\</div><div class='add'>+				TP_ARGS(remote_id))</div><div class='add'>+</div><div class='add'>+DEFINE_CM_REMOTE_EVENT(remote_no_priv_err);</div><div class='add'>+DEFINE_CM_REMOTE_EVENT(insert_failed_err);</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(icm_send_rep_err,</div><div class='add'>+	TP_PROTO(</div><div class='add'>+		__be32 local_id,</div><div class='add'>+		enum ib_cm_state state</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_ARGS(local_id, state),</div><div class='add'>+</div><div class='add'>+	TP_STRUCT__entry(</div><div class='add'>+		__field(unsigned int, local_id)</div><div class='add'>+		__field(unsigned long, state)</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_fast_assign(</div><div class='add'>+		__entry-&gt;local_id = be32_to_cpu(local_id);</div><div class='add'>+		__entry-&gt;state = state;</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_printk("local_id=%u state=%s",</div><div class='add'>+		__entry-&gt;local_id, show_ib_cm_state(__entry-&gt;state)</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(icm_rep_unknown_err,</div><div class='add'>+	TP_PROTO(</div><div class='add'>+		unsigned int local_id,</div><div class='add'>+		unsigned int remote_id,</div><div class='add'>+		enum ib_cm_state state</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_ARGS(local_id, remote_id, state),</div><div class='add'>+</div><div class='add'>+	TP_STRUCT__entry(</div><div class='add'>+		__field(unsigned int, local_id)</div><div class='add'>+		__field(unsigned int, remote_id)</div><div class='add'>+		__field(unsigned long, state)</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_fast_assign(</div><div class='add'>+		__entry-&gt;local_id = local_id;</div><div class='add'>+		__entry-&gt;remote_id = remote_id;</div><div class='add'>+		__entry-&gt;state = state;</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_printk("local_id=%u remote_id=%u state=%s",</div><div class='add'>+		__entry-&gt;local_id, __entry-&gt;remote_id,</div><div class='add'>+		show_ib_cm_state(__entry-&gt;state)</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(icm_handler_err,</div><div class='add'>+	TP_PROTO(</div><div class='add'>+		enum ib_cm_event_type event</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_ARGS(event),</div><div class='add'>+</div><div class='add'>+	TP_STRUCT__entry(</div><div class='add'>+		__field(unsigned long, event)</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_fast_assign(</div><div class='add'>+		__entry-&gt;event = event;</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_printk("unhandled event=%s",</div><div class='add'>+		rdma_show_ib_cm_event(__entry-&gt;event)</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(icm_mad_send_err,</div><div class='add'>+	TP_PROTO(</div><div class='add'>+		enum ib_cm_state state,</div><div class='add'>+		enum ib_wc_status wc_status</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_ARGS(state, wc_status),</div><div class='add'>+</div><div class='add'>+	TP_STRUCT__entry(</div><div class='add'>+		__field(unsigned long, state)</div><div class='add'>+		__field(unsigned long, wc_status)</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_fast_assign(</div><div class='add'>+		__entry-&gt;state = state;</div><div class='add'>+		__entry-&gt;wc_status = wc_status;</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_printk("state=%s completion status=%s",</div><div class='add'>+		show_ib_cm_state(__entry-&gt;state),</div><div class='add'>+		rdma_show_wc_status(__entry-&gt;wc_status)</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+#endif /* _TRACE_IB_CMA_H */</div><div class='add'>+</div><div class='add'>+#undef TRACE_INCLUDE_PATH</div><div class='add'>+#define TRACE_INCLUDE_PATH ../../drivers/infiniband/core</div><div class='add'>+#define TRACE_INCLUDE_FILE cm_trace</div><div class='add'>+</div><div class='add'>+#include &lt;trace/define_trace.h&gt;</div><div class='head'>diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c<br/>index 19f1730a4f24..95e89f5c147c 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cma.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cma.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cma.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/cma.c</a></div><div class='hunk'>@@ -1,36 +1,9 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright (c) 2005 Voltaire Inc.  All rights reserved.</div><div class='ctx'>  * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.</div><div class='del'>- * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.</div><div class='add'>+ * Copyright (c) 1999-2019, Mellanox Technologies, Inc. All rights reserved.</div><div class='ctx'>  * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/completion.h&gt;</div><div class='hunk'>@@ -38,6 +11,7 @@</div><div class='ctx'> #include &lt;linux/in6.h&gt;</div><div class='ctx'> #include &lt;linux/mutex.h&gt;</div><div class='ctx'> #include &lt;linux/random.h&gt;</div><div class='add'>+#include &lt;linux/rbtree.h&gt;</div><div class='ctx'> #include &lt;linux/igmp.h&gt;</div><div class='ctx'> #include &lt;linux/xarray.h&gt;</div><div class='ctx'> #include &lt;linux/inetdevice.h&gt;</div><div class='hunk'>@@ -47,6 +21,7 @@</div><div class='ctx'> </div><div class='ctx'> #include &lt;net/net_namespace.h&gt;</div><div class='ctx'> #include &lt;net/netns/generic.h&gt;</div><div class='add'>+#include &lt;net/netevent.h&gt;</div><div class='ctx'> #include &lt;net/tcp.h&gt;</div><div class='ctx'> #include &lt;net/ipv6.h&gt;</div><div class='ctx'> #include &lt;net/ip_fib.h&gt;</div><div class='hunk'>@@ -63,16 +38,15 @@</div><div class='ctx'> </div><div class='ctx'> #include "core_priv.h"</div><div class='ctx'> #include "cma_priv.h"</div><div class='add'>+#include "cma_trace.h"</div><div class='ctx'> </div><div class='ctx'> MODULE_AUTHOR("Sean Hefty");</div><div class='ctx'> MODULE_DESCRIPTION("Generic RDMA CM Agent");</div><div class='ctx'> MODULE_LICENSE("Dual BSD/GPL");</div><div class='ctx'> </div><div class='ctx'> #define CMA_CM_RESPONSE_TIMEOUT 20</div><div class='del'>-#define CMA_QUERY_CLASSPORT_INFO_TIMEOUT 3000</div><div class='ctx'> #define CMA_MAX_CM_RETRIES 15</div><div class='del'>-#define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)</div><div class='del'>-#define CMA_IBOE_PACKET_LIFETIME 18</div><div class='add'>+#define CMA_IBOE_PACKET_LIFETIME 16</div><div class='ctx'> #define CMA_PREFERRED_ROCE_GID_TYPE IB_GID_TYPE_ROCE_UDP_ENCAP</div><div class='ctx'> </div><div class='ctx'> static const char * const cma_events[] = {</div><div class='hunk'>@@ -94,6 +68,11 @@ static const char * const cma_events[] = {</div><div class='ctx'> 	[RDMA_CM_EVENT_TIMEWAIT_EXIT]	 = "timewait exit",</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid,</div><div class='add'>+			      enum ib_gid_type gid_type);</div><div class='add'>+</div><div class='add'>+static void cma_netevent_work_handler(struct work_struct *_work);</div><div class='add'>+</div><div class='ctx'> const char *__attribute_const__ rdma_event_msg(enum rdma_cm_event_type event)</div><div class='ctx'> {</div><div class='ctx'> 	size_t index = event;</div><div class='hunk'>@@ -117,7 +96,13 @@ const char *__attribute_const__ rdma_reject_msg(struct rdma_cm_id *id,</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_reject_msg);</div><div class='ctx'> </div><div class='del'>-bool rdma_is_consumer_reject(struct rdma_cm_id *id, int reason)</div><div class='add'>+/**</div><div class='add'>+ * rdma_is_consumer_reject - return true if the consumer rejected the connect</div><div class='add'>+ *                           request.</div><div class='add'>+ * @id: Communication identifier that received the REJECT event.</div><div class='add'>+ * @reason: Value returned in the REJECT event status field.</div><div class='add'>+ */</div><div class='add'>+static bool rdma_is_consumer_reject(struct rdma_cm_id *id, int reason)</div><div class='ctx'> {</div><div class='ctx'> 	if (rdma_ib_or_roce(id-&gt;device, id-&gt;port_num))</div><div class='ctx'> 		return reason == IB_CM_REJ_CONSUMER_DEFINED;</div><div class='hunk'>@@ -128,7 +113,6 @@ bool rdma_is_consumer_reject(struct rdma_cm_id *id, int reason)</div><div class='ctx'> 	WARN_ON_ONCE(1);</div><div class='ctx'> 	return false;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(rdma_is_consumer_reject);</div><div class='ctx'> </div><div class='ctx'> const void *rdma_consumer_reject_data(struct rdma_cm_id *id,</div><div class='ctx'> 				      struct rdma_cm_event *ev, u8 *data_len)</div><div class='hunk'>@@ -161,20 +145,7 @@ struct iw_cm_id *rdma_iw_cm_id(struct rdma_cm_id *id)</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_iw_cm_id);</div><div class='ctx'> </div><div class='del'>-/**</div><div class='del'>- * rdma_res_to_id() - return the rdma_cm_id pointer for this restrack.</div><div class='del'>- * @res: rdma resource tracking entry pointer</div><div class='del'>- */</div><div class='del'>-struct rdma_cm_id *rdma_res_to_id(struct rdma_restrack_entry *res)</div><div class='del'>-{</div><div class='del'>-	struct rdma_id_private *id_priv =</div><div class='del'>-		container_of(res, struct rdma_id_private, res);</div><div class='del'>-</div><div class='del'>-	return &amp;id_priv-&gt;id;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(rdma_res_to_id);</div><div class='del'>-</div><div class='del'>-static void cma_add_one(struct ib_device *device);</div><div class='add'>+static int cma_add_one(struct ib_device *device);</div><div class='ctx'> static void cma_remove_one(struct ib_device *device, void *client_data);</div><div class='ctx'> </div><div class='ctx'> static struct ib_client cma_client = {</div><div class='hunk'>@@ -187,6 +158,9 @@ static struct ib_sa_client sa_client;</div><div class='ctx'> static LIST_HEAD(dev_list);</div><div class='ctx'> static LIST_HEAD(listen_any_list);</div><div class='ctx'> static DEFINE_MUTEX(lock);</div><div class='add'>+static struct rb_root id_table = RB_ROOT;</div><div class='add'>+/* Serialize operations of id_table tree */</div><div class='add'>+static DEFINE_SPINLOCK(id_table_lock);</div><div class='ctx'> static struct workqueue_struct *cma_wq;</div><div class='ctx'> static unsigned int cma_pernet_id;</div><div class='ctx'> </div><div class='hunk'>@@ -221,11 +195,16 @@ struct xarray *cma_pernet_xa(struct net *net, enum rdma_ucm_port_space ps)</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+struct id_table_entry {</div><div class='add'>+	struct list_head id_list;</div><div class='add'>+	struct rb_node rb_node;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct cma_device {</div><div class='ctx'> 	struct list_head	list;</div><div class='ctx'> 	struct ib_device	*device;</div><div class='ctx'> 	struct completion	comp;</div><div class='del'>-	atomic_t		refcount;</div><div class='add'>+	refcount_t refcount;</div><div class='ctx'> 	struct list_head	id_list;</div><div class='ctx'> 	enum ib_gid_type	*default_gid_type;</div><div class='ctx'> 	u8			*default_roce_tos;</div><div class='hunk'>@@ -237,14 +216,6 @@ struct rdma_bind_list {</div><div class='ctx'> 	unsigned short		port;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct class_port_info_context {</div><div class='del'>-	struct ib_class_port_info	*class_port_info;</div><div class='del'>-	struct ib_device		*device;</div><div class='del'>-	struct completion		done;</div><div class='del'>-	struct ib_sa_query		*sa_query;</div><div class='del'>-	u8				port_num;</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> static int cma_ps_alloc(struct net *net, enum rdma_ucm_port_space ps,</div><div class='ctx'> 			struct rdma_bind_list *bind_list, int snum)</div><div class='ctx'> {</div><div class='hunk'>@@ -273,9 +244,15 @@ enum {</div><div class='ctx'> 	CMA_OPTION_AFONLY,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-void cma_ref_dev(struct cma_device *cma_dev)</div><div class='add'>+void cma_dev_get(struct cma_device *cma_dev)</div><div class='ctx'> {</div><div class='del'>-	atomic_inc(&amp;cma_dev-&gt;refcount);</div><div class='add'>+	refcount_inc(&amp;cma_dev-&gt;refcount);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void cma_dev_put(struct cma_device *cma_dev)</div><div class='add'>+{</div><div class='add'>+	if (refcount_dec_and_test(&amp;cma_dev-&gt;refcount))</div><div class='add'>+		complete(&amp;cma_dev-&gt;comp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter	filter,</div><div class='hunk'>@@ -293,13 +270,13 @@ struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter	filter,</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='ctx'> 	if (found_cma_dev)</div><div class='del'>-		cma_ref_dev(found_cma_dev);</div><div class='add'>+		cma_dev_get(found_cma_dev);</div><div class='ctx'> 	mutex_unlock(&amp;lock);</div><div class='ctx'> 	return found_cma_dev;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int cma_get_default_gid_type(struct cma_device *cma_dev,</div><div class='del'>-			     unsigned int port)</div><div class='add'>+			     u32 port)</div><div class='ctx'> {</div><div class='ctx'> 	if (!rdma_is_port_valid(cma_dev-&gt;device, port))</div><div class='ctx'> 		return -EINVAL;</div><div class='hunk'>@@ -308,7 +285,7 @@ int cma_get_default_gid_type(struct cma_device *cma_dev,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int cma_set_default_gid_type(struct cma_device *cma_dev,</div><div class='del'>-			     unsigned int port,</div><div class='add'>+			     u32 port,</div><div class='ctx'> 			     enum ib_gid_type default_gid_type)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long supported_gids;</div><div class='hunk'>@@ -316,6 +293,10 @@ int cma_set_default_gid_type(struct cma_device *cma_dev,</div><div class='ctx'> 	if (!rdma_is_port_valid(cma_dev-&gt;device, port))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='add'>+	if (default_gid_type == IB_GID_TYPE_IB &amp;&amp;</div><div class='add'>+	    rdma_protocol_roce_eth_encap(cma_dev-&gt;device, port))</div><div class='add'>+		default_gid_type = IB_GID_TYPE_ROCE;</div><div class='add'>+</div><div class='ctx'> 	supported_gids = roce_gid_type_mask_support(cma_dev-&gt;device, port);</div><div class='ctx'> </div><div class='ctx'> 	if (!(supported_gids &amp; 1 &lt;&lt; default_gid_type))</div><div class='hunk'>@@ -327,7 +308,7 @@ int cma_set_default_gid_type(struct cma_device *cma_dev,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int cma_get_default_roce_tos(struct cma_device *cma_dev, unsigned int port)</div><div class='add'>+int cma_get_default_roce_tos(struct cma_device *cma_dev, u32 port)</div><div class='ctx'> {</div><div class='ctx'> 	if (!rdma_is_port_valid(cma_dev-&gt;device, port))</div><div class='ctx'> 		return -EINVAL;</div><div class='hunk'>@@ -335,7 +316,7 @@ int cma_get_default_roce_tos(struct cma_device *cma_dev, unsigned int port)</div><div class='ctx'> 	return cma_dev-&gt;default_roce_tos[port - rdma_start_port(cma_dev-&gt;device)];</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int cma_set_default_roce_tos(struct cma_device *cma_dev, unsigned int port,</div><div class='add'>+int cma_set_default_roce_tos(struct cma_device *cma_dev, u32 port,</div><div class='ctx'> 			     u8 default_roce_tos)</div><div class='ctx'> {</div><div class='ctx'> 	if (!rdma_is_port_valid(cma_dev-&gt;device, port))</div><div class='hunk'>@@ -361,12 +342,15 @@ struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev)</div><div class='ctx'> struct cma_multicast {</div><div class='ctx'> 	struct rdma_id_private *id_priv;</div><div class='ctx'> 	union {</div><div class='del'>-		struct ib_sa_multicast *ib;</div><div class='del'>-	} multicast;</div><div class='add'>+		struct ib_sa_multicast *sa_mc;</div><div class='add'>+		struct {</div><div class='add'>+			struct work_struct work;</div><div class='add'>+			struct rdma_cm_event event;</div><div class='add'>+		} iboe_join;</div><div class='add'>+	};</div><div class='ctx'> 	struct list_head	list;</div><div class='ctx'> 	void			*context;</div><div class='ctx'> 	struct sockaddr_storage	addr;</div><div class='del'>-	struct kref		mcref;</div><div class='ctx'> 	u8			join_state;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -378,18 +362,6 @@ struct cma_work {</div><div class='ctx'> 	struct rdma_cm_event	event;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct cma_ndev_work {</div><div class='del'>-	struct work_struct	work;</div><div class='del'>-	struct rdma_id_private	*id;</div><div class='del'>-	struct rdma_cm_event	event;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct iboe_mcast_work {</div><div class='del'>-	struct work_struct	 work;</div><div class='del'>-	struct rdma_id_private	*id;</div><div class='del'>-	struct cma_multicast	*mc;</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> union cma_ip_addr {</div><div class='ctx'> 	struct in6_addr ip6;</div><div class='ctx'> 	struct {</div><div class='hunk'>@@ -419,23 +391,21 @@ struct cma_req_info {</div><div class='ctx'> 	u16 pkey;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp)</div><div class='del'>-{</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;id_priv-&gt;lock, flags);</div><div class='del'>-	ret = (id_priv-&gt;state == comp);</div><div class='del'>-	spin_unlock_irqrestore(&amp;id_priv-&gt;lock, flags);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int cma_comp_exch(struct rdma_id_private *id_priv,</div><div class='ctx'> 			 enum rdma_cm_state comp, enum rdma_cm_state exch)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	/*</div><div class='add'>+	 * The FSM uses a funny double locking where state is protected by both</div><div class='add'>+	 * the handler_mutex and the spinlock. State is not allowed to change</div><div class='add'>+	 * to/from a handler_mutex protected value without also holding</div><div class='add'>+	 * handler_mutex.</div><div class='add'>+	 */</div><div class='add'>+	if (comp == RDMA_CM_CONNECT || exch == RDMA_CM_CONNECT)</div><div class='add'>+		lockdep_assert_held(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+</div><div class='ctx'> 	spin_lock_irqsave(&amp;id_priv-&gt;lock, flags);</div><div class='ctx'> 	if ((ret = (id_priv-&gt;state == comp)))</div><div class='ctx'> 		id_priv-&gt;state = exch;</div><div class='hunk'>@@ -443,27 +413,24 @@ static int cma_comp_exch(struct rdma_id_private *id_priv,</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv,</div><div class='del'>-				   enum rdma_cm_state exch)</div><div class='add'>+static inline u8 cma_get_ip_ver(const struct cma_hdr *hdr)</div><div class='ctx'> {</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	enum rdma_cm_state old;</div><div class='add'>+	return hdr-&gt;ip_version &gt;&gt; 4;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	spin_lock_irqsave(&amp;id_priv-&gt;lock, flags);</div><div class='del'>-	old = id_priv-&gt;state;</div><div class='del'>-	id_priv-&gt;state = exch;</div><div class='del'>-	spin_unlock_irqrestore(&amp;id_priv-&gt;lock, flags);</div><div class='del'>-	return old;</div><div class='add'>+static void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)</div><div class='add'>+{</div><div class='add'>+	hdr-&gt;ip_version = (ip_ver &lt;&lt; 4) | (hdr-&gt;ip_version &amp; 0xF);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline u8 cma_get_ip_ver(const struct cma_hdr *hdr)</div><div class='add'>+static struct sockaddr *cma_src_addr(struct rdma_id_private *id_priv)</div><div class='ctx'> {</div><div class='del'>-	return hdr-&gt;ip_version &gt;&gt; 4;</div><div class='add'>+	return (struct sockaddr *)&amp;id_priv-&gt;id.route.addr.src_addr;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)</div><div class='add'>+static inline struct sockaddr *cma_dst_addr(struct rdma_id_private *id_priv)</div><div class='ctx'> {</div><div class='del'>-	hdr-&gt;ip_version = (ip_ver &lt;&lt; 4) | (hdr-&gt;ip_version &amp; 0xF);</div><div class='add'>+	return (struct sockaddr *)&amp;id_priv-&gt;id.route.addr.dst_addr;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int cma_igmp_send(struct net_device *ndev, union ib_gid *mgid, bool join)</div><div class='hunk'>@@ -486,19 +453,135 @@ static int cma_igmp_send(struct net_device *ndev, union ib_gid *mgid, bool join)</div><div class='ctx'> 	return (in_dev) ? 0 : -ENODEV;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int compare_netdev_and_ip(int ifindex_a, struct sockaddr *sa,</div><div class='add'>+				 struct id_table_entry *entry_b)</div><div class='add'>+{</div><div class='add'>+	struct rdma_id_private *id_priv = list_first_entry(</div><div class='add'>+		&amp;entry_b-&gt;id_list, struct rdma_id_private, id_list_entry);</div><div class='add'>+	int ifindex_b = id_priv-&gt;id.route.addr.dev_addr.bound_dev_if;</div><div class='add'>+	struct sockaddr *sb = cma_dst_addr(id_priv);</div><div class='add'>+</div><div class='add'>+	if (ifindex_a != ifindex_b)</div><div class='add'>+		return (ifindex_a &gt; ifindex_b) ? 1 : -1;</div><div class='add'>+</div><div class='add'>+	if (sa-&gt;sa_family != sb-&gt;sa_family)</div><div class='add'>+		return sa-&gt;sa_family - sb-&gt;sa_family;</div><div class='add'>+</div><div class='add'>+	if (sa-&gt;sa_family == AF_INET &amp;&amp;</div><div class='add'>+	    __builtin_object_size(sa, 0) &gt;= sizeof(struct sockaddr_in)) {</div><div class='add'>+		return memcmp(&amp;((struct sockaddr_in *)sa)-&gt;sin_addr,</div><div class='add'>+			      &amp;((struct sockaddr_in *)sb)-&gt;sin_addr,</div><div class='add'>+			      sizeof(((struct sockaddr_in *)sa)-&gt;sin_addr));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (sa-&gt;sa_family == AF_INET6 &amp;&amp;</div><div class='add'>+	    __builtin_object_size(sa, 0) &gt;= sizeof(struct sockaddr_in6)) {</div><div class='add'>+		return ipv6_addr_cmp(&amp;((struct sockaddr_in6 *)sa)-&gt;sin6_addr,</div><div class='add'>+				     &amp;((struct sockaddr_in6 *)sb)-&gt;sin6_addr);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return -1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int cma_add_id_to_tree(struct rdma_id_private *node_id_priv)</div><div class='add'>+{</div><div class='add'>+	struct rb_node **new, *parent = NULL;</div><div class='add'>+	struct id_table_entry *this, *node;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int result;</div><div class='add'>+</div><div class='add'>+	node = kzalloc(sizeof(*node), GFP_KERNEL);</div><div class='add'>+	if (!node)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;id_table_lock, flags);</div><div class='add'>+	new = &amp;id_table.rb_node;</div><div class='add'>+	while (*new) {</div><div class='add'>+		this = container_of(*new, struct id_table_entry, rb_node);</div><div class='add'>+		result = compare_netdev_and_ip(</div><div class='add'>+			node_id_priv-&gt;id.route.addr.dev_addr.bound_dev_if,</div><div class='add'>+			cma_dst_addr(node_id_priv), this);</div><div class='add'>+</div><div class='add'>+		parent = *new;</div><div class='add'>+		if (result &lt; 0)</div><div class='add'>+			new = &amp;((*new)-&gt;rb_left);</div><div class='add'>+		else if (result &gt; 0)</div><div class='add'>+			new = &amp;((*new)-&gt;rb_right);</div><div class='add'>+		else {</div><div class='add'>+			list_add_tail(&amp;node_id_priv-&gt;id_list_entry,</div><div class='add'>+				      &amp;this-&gt;id_list);</div><div class='add'>+			kfree(node);</div><div class='add'>+			goto unlock;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;node-&gt;id_list);</div><div class='add'>+	list_add_tail(&amp;node_id_priv-&gt;id_list_entry, &amp;node-&gt;id_list);</div><div class='add'>+</div><div class='add'>+	rb_link_node(&amp;node-&gt;rb_node, parent, new);</div><div class='add'>+	rb_insert_color(&amp;node-&gt;rb_node, &amp;id_table);</div><div class='add'>+</div><div class='add'>+unlock:</div><div class='add'>+	spin_unlock_irqrestore(&amp;id_table_lock, flags);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct id_table_entry *</div><div class='add'>+node_from_ndev_ip(struct rb_root *root, int ifindex, struct sockaddr *sa)</div><div class='add'>+{</div><div class='add'>+	struct rb_node *node = root-&gt;rb_node;</div><div class='add'>+	struct id_table_entry *data;</div><div class='add'>+	int result;</div><div class='add'>+</div><div class='add'>+	while (node) {</div><div class='add'>+		data = container_of(node, struct id_table_entry, rb_node);</div><div class='add'>+		result = compare_netdev_and_ip(ifindex, sa, data);</div><div class='add'>+		if (result &lt; 0)</div><div class='add'>+			node = node-&gt;rb_left;</div><div class='add'>+		else if (result &gt; 0)</div><div class='add'>+			node = node-&gt;rb_right;</div><div class='add'>+		else</div><div class='add'>+			return data;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void cma_remove_id_from_tree(struct rdma_id_private *id_priv)</div><div class='add'>+{</div><div class='add'>+	struct id_table_entry *data;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;id_table_lock, flags);</div><div class='add'>+	if (list_empty(&amp;id_priv-&gt;id_list_entry))</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	data = node_from_ndev_ip(&amp;id_table,</div><div class='add'>+				 id_priv-&gt;id.route.addr.dev_addr.bound_dev_if,</div><div class='add'>+				 cma_dst_addr(id_priv));</div><div class='add'>+	if (!data)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	list_del_init(&amp;id_priv-&gt;id_list_entry);</div><div class='add'>+	if (list_empty(&amp;data-&gt;id_list)) {</div><div class='add'>+		rb_erase(&amp;data-&gt;rb_node, &amp;id_table);</div><div class='add'>+		kfree(data);</div><div class='add'>+	}</div><div class='add'>+out:</div><div class='add'>+	spin_unlock_irqrestore(&amp;id_table_lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static void _cma_attach_to_dev(struct rdma_id_private *id_priv,</div><div class='ctx'> 			       struct cma_device *cma_dev)</div><div class='ctx'> {</div><div class='del'>-	cma_ref_dev(cma_dev);</div><div class='add'>+	cma_dev_get(cma_dev);</div><div class='ctx'> 	id_priv-&gt;cma_dev = cma_dev;</div><div class='ctx'> 	id_priv-&gt;id.device = cma_dev-&gt;device;</div><div class='ctx'> 	id_priv-&gt;id.route.addr.dev_addr.transport =</div><div class='ctx'> 		rdma_node_get_transport(cma_dev-&gt;device-&gt;node_type);</div><div class='del'>-	list_add_tail(&amp;id_priv-&gt;list, &amp;cma_dev-&gt;id_list);</div><div class='del'>-	if (id_priv-&gt;res.kern_name)</div><div class='del'>-		rdma_restrack_kadd(&amp;id_priv-&gt;res);</div><div class='del'>-	else</div><div class='del'>-		rdma_restrack_uadd(&amp;id_priv-&gt;res);</div><div class='add'>+	list_add_tail(&amp;id_priv-&gt;device_item, &amp;cma_dev-&gt;id_list);</div><div class='add'>+</div><div class='add'>+	trace_cm_id_attach(id_priv, cma_dev-&gt;device);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void cma_attach_to_dev(struct rdma_id_private *id_priv,</div><div class='hunk'>@@ -510,60 +593,30 @@ static void cma_attach_to_dev(struct rdma_id_private *id_priv,</div><div class='ctx'> 					  rdma_start_port(cma_dev-&gt;device)];</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void cma_deref_dev(struct cma_device *cma_dev)</div><div class='del'>-{</div><div class='del'>-	if (atomic_dec_and_test(&amp;cma_dev-&gt;refcount))</div><div class='del'>-		complete(&amp;cma_dev-&gt;comp);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void release_mc(struct kref *kref)</div><div class='del'>-{</div><div class='del'>-	struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref);</div><div class='del'>-</div><div class='del'>-	kfree(mc-&gt;multicast.ib);</div><div class='del'>-	kfree(mc);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static void cma_release_dev(struct rdma_id_private *id_priv)</div><div class='ctx'> {</div><div class='ctx'> 	mutex_lock(&amp;lock);</div><div class='del'>-	list_del(&amp;id_priv-&gt;list);</div><div class='del'>-	cma_deref_dev(id_priv-&gt;cma_dev);</div><div class='add'>+	list_del_init(&amp;id_priv-&gt;device_item);</div><div class='add'>+	cma_dev_put(id_priv-&gt;cma_dev);</div><div class='ctx'> 	id_priv-&gt;cma_dev = NULL;</div><div class='add'>+	id_priv-&gt;id.device = NULL;</div><div class='add'>+	if (id_priv-&gt;id.route.addr.dev_addr.sgid_attr) {</div><div class='add'>+		rdma_put_gid_attr(id_priv-&gt;id.route.addr.dev_addr.sgid_attr);</div><div class='add'>+		id_priv-&gt;id.route.addr.dev_addr.sgid_attr = NULL;</div><div class='add'>+	}</div><div class='ctx'> 	mutex_unlock(&amp;lock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline struct sockaddr *cma_src_addr(struct rdma_id_private *id_priv)</div><div class='del'>-{</div><div class='del'>-	return (struct sockaddr *) &amp;id_priv-&gt;id.route.addr.src_addr;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline struct sockaddr *cma_dst_addr(struct rdma_id_private *id_priv)</div><div class='del'>-{</div><div class='del'>-	return (struct sockaddr *) &amp;id_priv-&gt;id.route.addr.dst_addr;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static inline unsigned short cma_family(struct rdma_id_private *id_priv)</div><div class='ctx'> {</div><div class='ctx'> 	return id_priv-&gt;id.route.addr.src_addr.ss_family;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int cma_set_qkey(struct rdma_id_private *id_priv, u32 qkey)</div><div class='add'>+static int cma_set_default_qkey(struct rdma_id_private *id_priv)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_sa_mcmember_rec rec;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	if (id_priv-&gt;qkey) {</div><div class='del'>-		if (qkey &amp;&amp; id_priv-&gt;qkey != qkey)</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (qkey) {</div><div class='del'>-		id_priv-&gt;qkey = qkey;</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	switch (id_priv-&gt;id.ps) {</div><div class='ctx'> 	case RDMA_PS_UDP:</div><div class='ctx'> 	case RDMA_PS_IB:</div><div class='hunk'>@@ -583,6 +636,16 @@ static int cma_set_qkey(struct rdma_id_private *id_priv, u32 qkey)</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int cma_set_qkey(struct rdma_id_private *id_priv, u32 qkey)</div><div class='add'>+{</div><div class='add'>+	if (!qkey ||</div><div class='add'>+	    (id_priv-&gt;qkey &amp;&amp; (id_priv-&gt;qkey != qkey)))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	id_priv-&gt;qkey = qkey;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static void cma_translate_ib(struct sockaddr_ib *sib, struct rdma_dev_addr *dev_addr)</div><div class='ctx'> {</div><div class='ctx'> 	dev_addr-&gt;dev_type = ARPHRD_INFINIBAND;</div><div class='hunk'>@@ -605,37 +668,90 @@ static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_a</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static const struct ib_gid_attr *</div><div class='del'>-cma_validate_port(struct ib_device *device, u8 port,</div><div class='add'>+cma_validate_port(struct ib_device *device, u32 port,</div><div class='ctx'> 		  enum ib_gid_type gid_type,</div><div class='ctx'> 		  union ib_gid *gid,</div><div class='ctx'> 		  struct rdma_id_private *id_priv)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_dev_addr *dev_addr = &amp;id_priv-&gt;id.route.addr.dev_addr;</div><div class='add'>+	const struct ib_gid_attr *sgid_attr = ERR_PTR(-ENODEV);</div><div class='ctx'> 	int bound_if_index = dev_addr-&gt;bound_dev_if;</div><div class='del'>-	const struct ib_gid_attr *sgid_attr;</div><div class='ctx'> 	int dev_type = dev_addr-&gt;dev_type;</div><div class='ctx'> 	struct net_device *ndev = NULL;</div><div class='add'>+	struct net_device *pdev = NULL;</div><div class='ctx'> </div><div class='ctx'> 	if (!rdma_dev_access_netns(device, id_priv-&gt;id.route.addr.dev_addr.net))</div><div class='del'>-		return ERR_PTR(-ENODEV);</div><div class='add'>+		goto out;</div><div class='ctx'> </div><div class='ctx'> 	if ((dev_type == ARPHRD_INFINIBAND) &amp;&amp; !rdma_protocol_ib(device, port))</div><div class='del'>-		return ERR_PTR(-ENODEV);</div><div class='add'>+		goto out;</div><div class='ctx'> </div><div class='ctx'> 	if ((dev_type != ARPHRD_INFINIBAND) &amp;&amp; rdma_protocol_ib(device, port))</div><div class='del'>-		return ERR_PTR(-ENODEV);</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * For drivers that do not associate more than one net device with</div><div class='add'>+	 * their gid tables, such as iWARP drivers, it is sufficient to</div><div class='add'>+	 * return the first table entry.</div><div class='add'>+	 *</div><div class='add'>+	 * Other driver classes might be included in the future.</div><div class='add'>+	 */</div><div class='add'>+	if (rdma_protocol_iwarp(device, port)) {</div><div class='add'>+		sgid_attr = rdma_get_gid_attr(device, port, 0);</div><div class='add'>+		if (IS_ERR(sgid_attr))</div><div class='add'>+			goto out;</div><div class='ctx'> </div><div class='del'>-	if (dev_type == ARPHRD_ETHER &amp;&amp; rdma_protocol_roce(device, port)) {</div><div class='del'>-		ndev = dev_get_by_index(dev_addr-&gt;net, bound_if_index);</div><div class='del'>-		if (!ndev)</div><div class='del'>-			return ERR_PTR(-ENODEV);</div><div class='add'>+		rcu_read_lock();</div><div class='add'>+		ndev = rcu_dereference(sgid_attr-&gt;ndev);</div><div class='add'>+		if (ndev-&gt;ifindex != bound_if_index) {</div><div class='add'>+			pdev = dev_get_by_index_rcu(dev_addr-&gt;net, bound_if_index);</div><div class='add'>+			if (pdev) {</div><div class='add'>+				if (is_vlan_dev(pdev)) {</div><div class='add'>+					pdev = vlan_dev_real_dev(pdev);</div><div class='add'>+					if (ndev-&gt;ifindex == pdev-&gt;ifindex)</div><div class='add'>+						bound_if_index = pdev-&gt;ifindex;</div><div class='add'>+				}</div><div class='add'>+				if (is_vlan_dev(ndev)) {</div><div class='add'>+					pdev = vlan_dev_real_dev(ndev);</div><div class='add'>+					if (bound_if_index == pdev-&gt;ifindex)</div><div class='add'>+						bound_if_index = ndev-&gt;ifindex;</div><div class='add'>+				}</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+		if (!net_eq(dev_net(ndev), dev_addr-&gt;net) ||</div><div class='add'>+		    ndev-&gt;ifindex != bound_if_index) {</div><div class='add'>+			rdma_put_gid_attr(sgid_attr);</div><div class='add'>+			sgid_attr = ERR_PTR(-ENODEV);</div><div class='add'>+		}</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * For a RXE device, it should work with TUN device and normal ethernet</div><div class='add'>+	 * devices. Use driver_id to check if a device is a RXE device or not.</div><div class='add'>+	 * ARPHDR_NONE means a TUN device.</div><div class='add'>+	 */</div><div class='add'>+	if (device-&gt;ops.driver_id == RDMA_DRIVER_RXE) {</div><div class='add'>+		if ((dev_type == ARPHRD_NONE || dev_type == ARPHRD_ETHER)</div><div class='add'>+			&amp;&amp; rdma_protocol_roce(device, port)) {</div><div class='add'>+			ndev = dev_get_by_index(dev_addr-&gt;net, bound_if_index);</div><div class='add'>+			if (!ndev)</div><div class='add'>+				goto out;</div><div class='add'>+		}</div><div class='ctx'> 	} else {</div><div class='del'>-		gid_type = IB_GID_TYPE_IB;</div><div class='add'>+		if (dev_type == ARPHRD_ETHER &amp;&amp; rdma_protocol_roce(device, port)) {</div><div class='add'>+			ndev = dev_get_by_index(dev_addr-&gt;net, bound_if_index);</div><div class='add'>+			if (!ndev)</div><div class='add'>+				goto out;</div><div class='add'>+		} else {</div><div class='add'>+			gid_type = IB_GID_TYPE_IB;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	sgid_attr = rdma_find_gid_by_port(device, gid, gid_type, port, ndev);</div><div class='del'>-	if (ndev)</div><div class='del'>-		dev_put(ndev);</div><div class='add'>+	dev_put(ndev);</div><div class='add'>+out:</div><div class='ctx'> 	return sgid_attr;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -663,7 +779,7 @@ static int cma_acquire_dev_by_src_ip(struct rdma_id_private *id_priv)</div><div class='ctx'> 	struct cma_device *cma_dev;</div><div class='ctx'> 	enum ib_gid_type gid_type;</div><div class='ctx'> 	int ret = -ENODEV;</div><div class='del'>-	unsigned int port;</div><div class='add'>+	u32 port;</div><div class='ctx'> </div><div class='ctx'> 	if (dev_addr-&gt;dev_type != ARPHRD_INFINIBAND &amp;&amp;</div><div class='ctx'> 	    id_priv-&gt;id.ps == RDMA_PS_IPOIB)</div><div class='hunk'>@@ -743,6 +859,7 @@ static int cma_ib_acquire_dev(struct rdma_id_private *id_priv,</div><div class='ctx'> 	mutex_lock(&amp;lock);</div><div class='ctx'> 	cma_attach_to_dev(id_priv, listen_id_priv-&gt;cma_dev);</div><div class='ctx'> 	mutex_unlock(&amp;lock);</div><div class='add'>+	rdma_restrack_add(&amp;id_priv-&gt;res);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -755,7 +872,7 @@ static int cma_iw_acquire_dev(struct rdma_id_private *id_priv,</div><div class='ctx'> 	enum ib_gid_type gid_type;</div><div class='ctx'> 	int ret = -ENODEV;</div><div class='ctx'> 	union ib_gid gid;</div><div class='del'>-	u8 port;</div><div class='add'>+	u32 port;</div><div class='ctx'> </div><div class='ctx'> 	if (dev_addr-&gt;dev_type != ARPHRD_INFINIBAND &amp;&amp;</div><div class='ctx'> 	    id_priv-&gt;id.ps == RDMA_PS_IPOIB)</div><div class='hunk'>@@ -779,7 +896,7 @@ static int cma_iw_acquire_dev(struct rdma_id_private *id_priv,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	list_for_each_entry(cma_dev, &amp;dev_list, list) {</div><div class='del'>-		for (port = 1; port &lt;= cma_dev-&gt;device-&gt;phys_port_cnt; ++port) {</div><div class='add'>+		rdma_for_each_port (cma_dev-&gt;device, port) {</div><div class='ctx'> 			if (listen_id_priv-&gt;cma_dev == cma_dev &amp;&amp;</div><div class='ctx'> 			    listen_id_priv-&gt;id.port_num == port)</div><div class='ctx'> 				continue;</div><div class='hunk'>@@ -797,8 +914,10 @@ static int cma_iw_acquire_dev(struct rdma_id_private *id_priv,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='del'>-	if (!ret)</div><div class='add'>+	if (!ret) {</div><div class='ctx'> 		cma_attach_to_dev(id_priv, cma_dev);</div><div class='add'>+		rdma_restrack_add(&amp;id_priv-&gt;res);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	mutex_unlock(&amp;lock);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -812,9 +931,10 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv)</div><div class='ctx'> 	struct cma_device *cma_dev, *cur_dev;</div><div class='ctx'> 	struct sockaddr_ib *addr;</div><div class='ctx'> 	union ib_gid gid, sgid, *dgid;</div><div class='add'>+	unsigned int p;</div><div class='ctx'> 	u16 pkey, index;</div><div class='del'>-	u8 p;</div><div class='ctx'> 	enum ib_port_state port_state;</div><div class='add'>+	int ret;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='ctx'> 	cma_dev = NULL;</div><div class='hunk'>@@ -824,7 +944,7 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv)</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;lock);</div><div class='ctx'> 	list_for_each_entry(cur_dev, &amp;dev_list, list) {</div><div class='del'>-		for (p = 1; p &lt;= cur_dev-&gt;device-&gt;phys_port_cnt; ++p) {</div><div class='add'>+		rdma_for_each_port (cur_dev-&gt;device, p) {</div><div class='ctx'> 			if (!rdma_cap_af_ib(cur_dev-&gt;device, p))</div><div class='ctx'> 				continue;</div><div class='ctx'> </div><div class='hunk'>@@ -833,9 +953,14 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv)</div><div class='ctx'> </div><div class='ctx'> 			if (ib_get_cached_port_state(cur_dev-&gt;device, p, &amp;port_state))</div><div class='ctx'> 				continue;</div><div class='del'>-			for (i = 0; !rdma_query_gid(cur_dev-&gt;device,</div><div class='del'>-						    p, i, &amp;gid);</div><div class='del'>-			     i++) {</div><div class='add'>+</div><div class='add'>+			for (i = 0; i &lt; cur_dev-&gt;device-&gt;port_data[p].immutable.gid_tbl_len;</div><div class='add'>+			     ++i) {</div><div class='add'>+				ret = rdma_query_gid(cur_dev-&gt;device, p, i,</div><div class='add'>+						     &amp;gid);</div><div class='add'>+				if (ret)</div><div class='add'>+					continue;</div><div class='add'>+</div><div class='ctx'> 				if (!memcmp(&amp;gid, dgid, sizeof(gid))) {</div><div class='ctx'> 					cma_dev = cur_dev;</div><div class='ctx'> 					sgid = gid;</div><div class='hunk'>@@ -859,6 +984,7 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv)</div><div class='ctx'> </div><div class='ctx'> found:</div><div class='ctx'> 	cma_attach_to_dev(id_priv, cma_dev);</div><div class='add'>+	rdma_restrack_add(&amp;id_priv-&gt;res);</div><div class='ctx'> 	mutex_unlock(&amp;lock);</div><div class='ctx'> 	addr = (struct sockaddr_ib *)cma_src_addr(id_priv);</div><div class='ctx'> 	memcpy(&amp;addr-&gt;sib_addr, &amp;sgid, sizeof(sgid));</div><div class='hunk'>@@ -866,16 +992,21 @@ found:</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cma_deref_id(struct rdma_id_private *id_priv)</div><div class='add'>+static void cma_id_get(struct rdma_id_private *id_priv)</div><div class='ctx'> {</div><div class='del'>-	if (atomic_dec_and_test(&amp;id_priv-&gt;refcount))</div><div class='add'>+	refcount_inc(&amp;id_priv-&gt;refcount);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void cma_id_put(struct rdma_id_private *id_priv)</div><div class='add'>+{</div><div class='add'>+	if (refcount_dec_and_test(&amp;id_priv-&gt;refcount))</div><div class='ctx'> 		complete(&amp;id_priv-&gt;comp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct rdma_cm_id *__rdma_create_id(struct net *net,</div><div class='del'>-				    rdma_cm_event_handler event_handler,</div><div class='del'>-				    void *context, enum rdma_ucm_port_space ps,</div><div class='del'>-				    enum ib_qp_type qp_type, const char *caller)</div><div class='add'>+static struct rdma_id_private *</div><div class='add'>+__rdma_create_id(struct net *net, rdma_cm_event_handler event_handler,</div><div class='add'>+		 void *context, enum rdma_ucm_port_space ps,</div><div class='add'>+		 enum ib_qp_type qp_type, const struct rdma_id_private *parent)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_id_private *id_priv;</div><div class='ctx'> </div><div class='hunk'>@@ -883,8 +1014,6 @@ struct rdma_cm_id *__rdma_create_id(struct net *net,</div><div class='ctx'> 	if (!id_priv)</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='del'>-	rdma_restrack_set_task(&amp;id_priv-&gt;res, caller);</div><div class='del'>-	id_priv-&gt;res.type = RDMA_RESTRACK_CM_ID;</div><div class='ctx'> 	id_priv-&gt;state = RDMA_CM_IDLE;</div><div class='ctx'> 	id_priv-&gt;id.context = context;</div><div class='ctx'> 	id_priv-&gt;id.event_handler = event_handler;</div><div class='hunk'>@@ -892,21 +1021,61 @@ struct rdma_cm_id *__rdma_create_id(struct net *net,</div><div class='ctx'> 	id_priv-&gt;id.qp_type = qp_type;</div><div class='ctx'> 	id_priv-&gt;tos_set = false;</div><div class='ctx'> 	id_priv-&gt;timeout_set = false;</div><div class='add'>+	id_priv-&gt;min_rnr_timer_set = false;</div><div class='ctx'> 	id_priv-&gt;gid_type = IB_GID_TYPE_IB;</div><div class='ctx'> 	spin_lock_init(&amp;id_priv-&gt;lock);</div><div class='ctx'> 	mutex_init(&amp;id_priv-&gt;qp_mutex);</div><div class='ctx'> 	init_completion(&amp;id_priv-&gt;comp);</div><div class='del'>-	atomic_set(&amp;id_priv-&gt;refcount, 1);</div><div class='add'>+	refcount_set(&amp;id_priv-&gt;refcount, 1);</div><div class='ctx'> 	mutex_init(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+	INIT_LIST_HEAD(&amp;id_priv-&gt;device_item);</div><div class='add'>+	INIT_LIST_HEAD(&amp;id_priv-&gt;id_list_entry);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;id_priv-&gt;listen_list);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;id_priv-&gt;mc_list);</div><div class='ctx'> 	get_random_bytes(&amp;id_priv-&gt;seq_num, sizeof id_priv-&gt;seq_num);</div><div class='ctx'> 	id_priv-&gt;id.route.addr.dev_addr.net = get_net(net);</div><div class='ctx'> 	id_priv-&gt;seq_num &amp;= 0x00ffffff;</div><div class='add'>+	INIT_WORK(&amp;id_priv-&gt;id.net_work, cma_netevent_work_handler);</div><div class='ctx'> </div><div class='del'>-	return &amp;id_priv-&gt;id;</div><div class='add'>+	rdma_restrack_new(&amp;id_priv-&gt;res, RDMA_RESTRACK_CM_ID);</div><div class='add'>+	if (parent)</div><div class='add'>+		rdma_restrack_parent_name(&amp;id_priv-&gt;res, &amp;parent-&gt;res);</div><div class='add'>+</div><div class='add'>+	return id_priv;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(__rdma_create_id);</div><div class='add'>+</div><div class='add'>+struct rdma_cm_id *</div><div class='add'>+__rdma_create_kernel_id(struct net *net, rdma_cm_event_handler event_handler,</div><div class='add'>+			void *context, enum rdma_ucm_port_space ps,</div><div class='add'>+			enum ib_qp_type qp_type, const char *caller)</div><div class='add'>+{</div><div class='add'>+	struct rdma_id_private *ret;</div><div class='add'>+</div><div class='add'>+	ret = __rdma_create_id(net, event_handler, context, ps, qp_type, NULL);</div><div class='add'>+	if (IS_ERR(ret))</div><div class='add'>+		return ERR_CAST(ret);</div><div class='add'>+</div><div class='add'>+	rdma_restrack_set_name(&amp;ret-&gt;res, caller);</div><div class='add'>+	return &amp;ret-&gt;id;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(__rdma_create_kernel_id);</div><div class='add'>+</div><div class='add'>+struct rdma_cm_id *rdma_create_user_id(rdma_cm_event_handler event_handler,</div><div class='add'>+				       void *context,</div><div class='add'>+				       enum rdma_ucm_port_space ps,</div><div class='add'>+				       enum ib_qp_type qp_type)</div><div class='add'>+{</div><div class='add'>+	struct rdma_id_private *ret;</div><div class='add'>+</div><div class='add'>+	ret = __rdma_create_id(current-&gt;nsproxy-&gt;net_ns, event_handler, context,</div><div class='add'>+			       ps, qp_type, NULL);</div><div class='add'>+	if (IS_ERR(ret))</div><div class='add'>+		return ERR_CAST(ret);</div><div class='add'>+</div><div class='add'>+	rdma_restrack_set_name(&amp;ret-&gt;res, NULL);</div><div class='add'>+	return &amp;ret-&gt;id;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_create_user_id);</div><div class='ctx'> </div><div class='ctx'> static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)</div><div class='ctx'> {</div><div class='hunk'>@@ -955,27 +1124,34 @@ int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='del'>-	if (id-&gt;device != pd-&gt;device)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	if (id-&gt;device != pd-&gt;device) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto out_err;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	qp_init_attr-&gt;port_num = id-&gt;port_num;</div><div class='ctx'> 	qp = ib_create_qp(pd, qp_init_attr);</div><div class='del'>-	if (IS_ERR(qp))</div><div class='del'>-		return PTR_ERR(qp);</div><div class='add'>+	if (IS_ERR(qp)) {</div><div class='add'>+		ret = PTR_ERR(qp);</div><div class='add'>+		goto out_err;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	if (id-&gt;qp_type == IB_QPT_UD)</div><div class='ctx'> 		ret = cma_init_ud_qp(id_priv, qp);</div><div class='ctx'> 	else</div><div class='ctx'> 		ret = cma_init_conn_qp(id_priv, qp);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err;</div><div class='add'>+		goto out_destroy;</div><div class='ctx'> </div><div class='ctx'> 	id-&gt;qp = qp;</div><div class='ctx'> 	id_priv-&gt;qp_num = qp-&gt;qp_num;</div><div class='ctx'> 	id_priv-&gt;srq = (qp-&gt;srq != NULL);</div><div class='add'>+	trace_cm_qp_create(id_priv, pd, qp_init_attr, 0);</div><div class='ctx'> 	return 0;</div><div class='del'>-err:</div><div class='add'>+out_destroy:</div><div class='ctx'> 	ib_destroy_qp(qp);</div><div class='add'>+out_err:</div><div class='add'>+	trace_cm_qp_create(id_priv, pd, qp_init_attr, ret);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_create_qp);</div><div class='hunk'>@@ -985,6 +1161,7 @@ void rdma_destroy_qp(struct rdma_cm_id *id)</div><div class='ctx'> 	struct rdma_id_private *id_priv;</div><div class='ctx'> </div><div class='ctx'> 	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='add'>+	trace_cm_qp_destroy(id_priv);</div><div class='ctx'> 	mutex_lock(&amp;id_priv-&gt;qp_mutex);</div><div class='ctx'> 	ib_destroy_qp(id_priv-&gt;id.qp);</div><div class='ctx'> 	id_priv-&gt;id.qp = NULL;</div><div class='hunk'>@@ -1093,7 +1270,7 @@ static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,</div><div class='ctx'> 	*qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;</div><div class='ctx'> </div><div class='ctx'> 	if (id_priv-&gt;id.qp_type == IB_QPT_UD) {</div><div class='del'>-		ret = cma_set_qkey(id_priv, 0);</div><div class='add'>+		ret = cma_set_default_qkey(id_priv);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			return ret;</div><div class='ctx'> </div><div class='hunk'>@@ -1131,12 +1308,16 @@ int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 						 qp_attr_mask);</div><div class='ctx'> 		qp_attr-&gt;port_num = id_priv-&gt;id.port_num;</div><div class='ctx'> 		*qp_attr_mask |= IB_QP_PORT;</div><div class='del'>-	} else</div><div class='add'>+	} else {</div><div class='ctx'> 		ret = -ENOSYS;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	if ((*qp_attr_mask &amp; IB_QP_TIMEOUT) &amp;&amp; id_priv-&gt;timeout_set)</div><div class='ctx'> 		qp_attr-&gt;timeout = id_priv-&gt;timeout;</div><div class='ctx'> </div><div class='add'>+	if ((*qp_attr_mask &amp; IB_QP_MIN_RNR_TIMER) &amp;&amp; id_priv-&gt;min_rnr_timer_set)</div><div class='add'>+		qp_attr-&gt;min_rnr_timer = id_priv-&gt;min_rnr_timer;</div><div class='add'>+</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_init_qp_attr);</div><div class='hunk'>@@ -1423,7 +1604,7 @@ static bool validate_ipv4_net_dev(struct net_device *net_dev,</div><div class='ctx'> 		return false;</div><div class='ctx'> </div><div class='ctx'> 	memset(&amp;fl4, 0, sizeof(fl4));</div><div class='del'>-	fl4.flowi4_iif = net_dev-&gt;ifindex;</div><div class='add'>+	fl4.flowi4_oif = net_dev-&gt;ifindex;</div><div class='ctx'> 	fl4.daddr = daddr;</div><div class='ctx'> 	fl4.saddr = saddr;</div><div class='ctx'> </div><div class='hunk'>@@ -1577,7 +1758,7 @@ static bool cma_match_private_data(struct rdma_id_private *id_priv,</div><div class='ctx'> static bool cma_protocol_roce(const struct rdma_cm_id *id)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *device = id-&gt;device;</div><div class='del'>-	const int port_num = id-&gt;port_num ?: rdma_start_port(device);</div><div class='add'>+	const u32 port_num = id-&gt;port_num ?: rdma_start_port(device);</div><div class='ctx'> </div><div class='ctx'> 	return rdma_protocol_roce(device, port_num);</div><div class='ctx'> }</div><div class='hunk'>@@ -1631,6 +1812,8 @@ static struct rdma_id_private *cma_find_listener(</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_id_private *id_priv, *id_priv_dev;</div><div class='ctx'> </div><div class='add'>+	lockdep_assert_held(&amp;lock);</div><div class='add'>+</div><div class='ctx'> 	if (!bind_list)</div><div class='ctx'> 		return ERR_PTR(-EINVAL);</div><div class='ctx'> </div><div class='hunk'>@@ -1641,7 +1824,7 @@ static struct rdma_id_private *cma_find_listener(</div><div class='ctx'> 				return id_priv;</div><div class='ctx'> 			list_for_each_entry(id_priv_dev,</div><div class='ctx'> 					    &amp;id_priv-&gt;listen_list,</div><div class='del'>-					    listen_list) {</div><div class='add'>+					    listen_item) {</div><div class='ctx'> 				if (id_priv_dev-&gt;id.device == cm_id-&gt;device &amp;&amp;</div><div class='ctx'> 				    cma_match_net_dev(&amp;id_priv_dev-&gt;id,</div><div class='ctx'> 						      net_dev, req))</div><div class='hunk'>@@ -1677,6 +1860,7 @@ cma_ib_id_from_event(struct ib_cm_id *cm_id,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;lock);</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Net namespace might be getting deleted while route lookup,</div><div class='ctx'> 	 * cm_id lookup is in progress. Therefore, perform netdevice</div><div class='hunk'>@@ -1705,8 +1889,8 @@ cma_ib_id_from_event(struct ib_cm_id *cm_id,</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='ctx'> 		if (!validate_net_dev(*net_dev,</div><div class='del'>-				 (struct sockaddr *)&amp;req-&gt;listen_addr_storage,</div><div class='del'>-				 (struct sockaddr *)&amp;req-&gt;src_addr_storage)) {</div><div class='add'>+				 (struct sockaddr *)&amp;req-&gt;src_addr_storage,</div><div class='add'>+				 (struct sockaddr *)&amp;req-&gt;listen_addr_storage)) {</div><div class='ctx'> 			id_priv = ERR_PTR(-EHOSTUNREACH);</div><div class='ctx'> 			goto err;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -1718,6 +1902,7 @@ cma_ib_id_from_event(struct ib_cm_id *cm_id,</div><div class='ctx'> 	id_priv = cma_find_listener(bind_list, cm_id, ib_event, req, *net_dev);</div><div class='ctx'> err:</div><div class='ctx'> 	rcu_read_unlock();</div><div class='add'>+	mutex_unlock(&amp;lock);</div><div class='ctx'> 	if (IS_ERR(id_priv) &amp;&amp; *net_dev) {</div><div class='ctx'> 		dev_put(*net_dev);</div><div class='ctx'> 		*net_dev = NULL;</div><div class='hunk'>@@ -1738,28 +1923,36 @@ static void cma_cancel_route(struct rdma_id_private *id_priv)</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cma_cancel_listens(struct rdma_id_private *id_priv)</div><div class='add'>+static void _cma_cancel_listens(struct rdma_id_private *id_priv)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_id_private *dev_id_priv;</div><div class='ctx'> </div><div class='add'>+	lockdep_assert_held(&amp;lock);</div><div class='add'>+</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Remove from listen_any_list to prevent added devices from spawning</div><div class='ctx'> 	 * additional listen requests.</div><div class='ctx'> 	 */</div><div class='del'>-	mutex_lock(&amp;lock);</div><div class='del'>-	list_del(&amp;id_priv-&gt;list);</div><div class='add'>+	list_del_init(&amp;id_priv-&gt;listen_any_item);</div><div class='ctx'> </div><div class='ctx'> 	while (!list_empty(&amp;id_priv-&gt;listen_list)) {</div><div class='del'>-		dev_id_priv = list_entry(id_priv-&gt;listen_list.next,</div><div class='del'>-					 struct rdma_id_private, listen_list);</div><div class='add'>+		dev_id_priv =</div><div class='add'>+			list_first_entry(&amp;id_priv-&gt;listen_list,</div><div class='add'>+					 struct rdma_id_private, listen_item);</div><div class='ctx'> 		/* sync with device removal to avoid duplicate destruction */</div><div class='del'>-		list_del_init(&amp;dev_id_priv-&gt;list);</div><div class='del'>-		list_del(&amp;dev_id_priv-&gt;listen_list);</div><div class='add'>+		list_del_init(&amp;dev_id_priv-&gt;device_item);</div><div class='add'>+		list_del_init(&amp;dev_id_priv-&gt;listen_item);</div><div class='ctx'> 		mutex_unlock(&amp;lock);</div><div class='ctx'> </div><div class='ctx'> 		rdma_destroy_id(&amp;dev_id_priv-&gt;id);</div><div class='ctx'> 		mutex_lock(&amp;lock);</div><div class='ctx'> 	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void cma_cancel_listens(struct rdma_id_private *id_priv)</div><div class='add'>+{</div><div class='add'>+	mutex_lock(&amp;lock);</div><div class='add'>+	_cma_cancel_listens(id_priv);</div><div class='ctx'> 	mutex_unlock(&amp;lock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1768,6 +1961,14 @@ static void cma_cancel_operation(struct rdma_id_private *id_priv,</div><div class='ctx'> {</div><div class='ctx'> 	switch (state) {</div><div class='ctx'> 	case RDMA_CM_ADDR_QUERY:</div><div class='add'>+		/*</div><div class='add'>+		 * We can avoid doing the rdma_addr_cancel() based on state,</div><div class='add'>+		 * only RDMA_CM_ADDR_QUERY has a work that could still execute.</div><div class='add'>+		 * Notice that the addr_handler work could still be exiting</div><div class='add'>+		 * outside this state, however due to the interaction with the</div><div class='add'>+		 * handler_mutex the work is guaranteed not to touch id_priv</div><div class='add'>+		 * during exit.</div><div class='add'>+		 */</div><div class='ctx'> 		rdma_addr_cancel(&amp;id_priv-&gt;id.route.addr.dev_addr);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case RDMA_CM_ROUTE_QUERY:</div><div class='hunk'>@@ -1799,19 +2000,39 @@ static void cma_release_port(struct rdma_id_private *id_priv)</div><div class='ctx'> 	mutex_unlock(&amp;lock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cma_leave_roce_mc_group(struct rdma_id_private *id_priv,</div><div class='del'>-				    struct cma_multicast *mc)</div><div class='add'>+static void destroy_mc(struct rdma_id_private *id_priv,</div><div class='add'>+		       struct cma_multicast *mc)</div><div class='ctx'> {</div><div class='del'>-	struct rdma_dev_addr *dev_addr = &amp;id_priv-&gt;id.route.addr.dev_addr;</div><div class='del'>-	struct net_device *ndev = NULL;</div><div class='add'>+	bool send_only = mc-&gt;join_state == BIT(SENDONLY_FULLMEMBER_JOIN);</div><div class='ctx'> </div><div class='del'>-	if (dev_addr-&gt;bound_dev_if)</div><div class='del'>-		ndev = dev_get_by_index(dev_addr-&gt;net, dev_addr-&gt;bound_dev_if);</div><div class='del'>-	if (ndev) {</div><div class='del'>-		cma_igmp_send(ndev, &amp;mc-&gt;multicast.ib-&gt;rec.mgid, false);</div><div class='add'>+	if (rdma_cap_ib_mcast(id_priv-&gt;id.device, id_priv-&gt;id.port_num))</div><div class='add'>+		ib_sa_free_multicast(mc-&gt;sa_mc);</div><div class='add'>+</div><div class='add'>+	if (rdma_protocol_roce(id_priv-&gt;id.device, id_priv-&gt;id.port_num)) {</div><div class='add'>+		struct rdma_dev_addr *dev_addr =</div><div class='add'>+			&amp;id_priv-&gt;id.route.addr.dev_addr;</div><div class='add'>+		struct net_device *ndev = NULL;</div><div class='add'>+</div><div class='add'>+		if (dev_addr-&gt;bound_dev_if)</div><div class='add'>+			ndev = dev_get_by_index(dev_addr-&gt;net,</div><div class='add'>+						dev_addr-&gt;bound_dev_if);</div><div class='add'>+		if (ndev &amp;&amp; !send_only) {</div><div class='add'>+			enum ib_gid_type gid_type;</div><div class='add'>+			union ib_gid mgid;</div><div class='add'>+</div><div class='add'>+			gid_type = id_priv-&gt;cma_dev-&gt;default_gid_type</div><div class='add'>+					   [id_priv-&gt;id.port_num -</div><div class='add'>+					    rdma_start_port(</div><div class='add'>+						    id_priv-&gt;cma_dev-&gt;device)];</div><div class='add'>+			cma_iboe_set_mgid((struct sockaddr *)&amp;mc-&gt;addr, &amp;mgid,</div><div class='add'>+					  gid_type);</div><div class='add'>+			cma_igmp_send(ndev, &amp;mgid, false);</div><div class='add'>+		}</div><div class='ctx'> 		dev_put(ndev);</div><div class='add'>+</div><div class='add'>+		cancel_work_sync(&amp;mc-&gt;iboe_join.work);</div><div class='ctx'> 	}</div><div class='del'>-	kref_put(&amp;mc-&gt;mcref, release_mc);</div><div class='add'>+	kfree(mc);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void cma_leave_mc_groups(struct rdma_id_private *id_priv)</div><div class='hunk'>@@ -1819,36 +2040,20 @@ static void cma_leave_mc_groups(struct rdma_id_private *id_priv)</div><div class='ctx'> 	struct cma_multicast *mc;</div><div class='ctx'> </div><div class='ctx'> 	while (!list_empty(&amp;id_priv-&gt;mc_list)) {</div><div class='del'>-		mc = container_of(id_priv-&gt;mc_list.next,</div><div class='del'>-				  struct cma_multicast, list);</div><div class='add'>+		mc = list_first_entry(&amp;id_priv-&gt;mc_list, struct cma_multicast,</div><div class='add'>+				      list);</div><div class='ctx'> 		list_del(&amp;mc-&gt;list);</div><div class='del'>-		if (rdma_cap_ib_mcast(id_priv-&gt;cma_dev-&gt;device,</div><div class='del'>-				      id_priv-&gt;id.port_num)) {</div><div class='del'>-			ib_sa_free_multicast(mc-&gt;multicast.ib);</div><div class='del'>-			kfree(mc);</div><div class='del'>-		} else {</div><div class='del'>-			cma_leave_roce_mc_group(id_priv, mc);</div><div class='del'>-		}</div><div class='add'>+		destroy_mc(id_priv, mc);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void rdma_destroy_id(struct rdma_cm_id *id)</div><div class='add'>+static void _destroy_id(struct rdma_id_private *id_priv,</div><div class='add'>+			enum rdma_cm_state state)</div><div class='ctx'> {</div><div class='del'>-	struct rdma_id_private *id_priv;</div><div class='del'>-	enum rdma_cm_state state;</div><div class='del'>-</div><div class='del'>-	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='del'>-	state = cma_exch(id_priv, RDMA_CM_DESTROYING);</div><div class='ctx'> 	cma_cancel_operation(id_priv, state);</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * Wait for any active callback to finish.  New callbacks will find</div><div class='del'>-	 * the id_priv state set to destroying and abort.</div><div class='del'>-	 */</div><div class='del'>-	mutex_lock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-	mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-</div><div class='ctx'> 	rdma_restrack_del(&amp;id_priv-&gt;res);</div><div class='add'>+	cma_remove_id_from_tree(id_priv);</div><div class='ctx'> 	if (id_priv-&gt;cma_dev) {</div><div class='ctx'> 		if (rdma_cap_ib_cm(id_priv-&gt;id.device, 1)) {</div><div class='ctx'> 			if (id_priv-&gt;cm_id.ib)</div><div class='hunk'>@@ -1862,20 +2067,56 @@ void rdma_destroy_id(struct rdma_cm_id *id)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	cma_release_port(id_priv);</div><div class='del'>-	cma_deref_id(id_priv);</div><div class='add'>+	cma_id_put(id_priv);</div><div class='ctx'> 	wait_for_completion(&amp;id_priv-&gt;comp);</div><div class='ctx'> </div><div class='ctx'> 	if (id_priv-&gt;internal_id)</div><div class='del'>-		cma_deref_id(id_priv-&gt;id.context);</div><div class='add'>+		cma_id_put(id_priv-&gt;id.context);</div><div class='ctx'> </div><div class='ctx'> 	kfree(id_priv-&gt;id.route.path_rec);</div><div class='del'>-</div><div class='del'>-	if (id_priv-&gt;id.route.addr.dev_addr.sgid_attr)</div><div class='del'>-		rdma_put_gid_attr(id_priv-&gt;id.route.addr.dev_addr.sgid_attr);</div><div class='add'>+	kfree(id_priv-&gt;id.route.path_rec_inbound);</div><div class='add'>+	kfree(id_priv-&gt;id.route.path_rec_outbound);</div><div class='add'>+	kfree(id_priv-&gt;id.route.service_recs);</div><div class='ctx'> </div><div class='ctx'> 	put_net(id_priv-&gt;id.route.addr.dev_addr.net);</div><div class='ctx'> 	kfree(id_priv);</div><div class='ctx'> }</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * destroy an ID from within the handler_mutex. This ensures that no other</div><div class='add'>+ * handlers can start running concurrently.</div><div class='add'>+ */</div><div class='add'>+static void destroy_id_handler_unlock(struct rdma_id_private *id_priv)</div><div class='add'>+	__releases(&amp;idprv-&gt;handler_mutex)</div><div class='add'>+{</div><div class='add'>+	enum rdma_cm_state state;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	trace_cm_id_destroy(id_priv);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Setting the state to destroyed under the handler mutex provides a</div><div class='add'>+	 * fence against calling handler callbacks. If this is invoked due to</div><div class='add'>+	 * the failure of a handler callback then it guarentees that no future</div><div class='add'>+	 * handlers will be called.</div><div class='add'>+	 */</div><div class='add'>+	lockdep_assert_held(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+	spin_lock_irqsave(&amp;id_priv-&gt;lock, flags);</div><div class='add'>+	state = id_priv-&gt;state;</div><div class='add'>+	id_priv-&gt;state = RDMA_CM_DESTROYING;</div><div class='add'>+	spin_unlock_irqrestore(&amp;id_priv-&gt;lock, flags);</div><div class='add'>+	mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+	_destroy_id(id_priv, state);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void rdma_destroy_id(struct rdma_cm_id *id)</div><div class='add'>+{</div><div class='add'>+	struct rdma_id_private *id_priv =</div><div class='add'>+		container_of(id, struct rdma_id_private, id);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+	destroy_id_handler_unlock(id_priv);</div><div class='add'>+}</div><div class='ctx'> EXPORT_SYMBOL(rdma_destroy_id);</div><div class='ctx'> </div><div class='ctx'> static int cma_rep_recv(struct rdma_id_private *id_priv)</div><div class='hunk'>@@ -1890,6 +2131,7 @@ static int cma_rep_recv(struct rdma_id_private *id_priv)</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto reject;</div><div class='ctx'> </div><div class='add'>+	trace_cm_send_rtu(id_priv);</div><div class='ctx'> 	ret = ib_send_cm_rtu(id_priv-&gt;cm_id.ib, NULL, 0);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto reject;</div><div class='hunk'>@@ -1898,6 +2140,7 @@ static int cma_rep_recv(struct rdma_id_private *id_priv)</div><div class='ctx'> reject:</div><div class='ctx'> 	pr_debug_ratelimited("RDMA CM: CONNECT_ERROR: failed to handle reply. status %d\n", ret);</div><div class='ctx'> 	cma_modify_qp_err(id_priv);</div><div class='add'>+	trace_cm_send_rej(id_priv);</div><div class='ctx'> 	ib_send_cm_rej(id_priv-&gt;cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,</div><div class='ctx'> 		       NULL, 0, NULL, 0);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -1915,6 +2158,22 @@ static void cma_set_rep_event_data(struct rdma_cm_event *event,</div><div class='ctx'> 	event-&gt;param.conn.rnr_retry_count = rep_data-&gt;rnr_retry_count;</div><div class='ctx'> 	event-&gt;param.conn.srq = rep_data-&gt;srq;</div><div class='ctx'> 	event-&gt;param.conn.qp_num = rep_data-&gt;remote_qpn;</div><div class='add'>+</div><div class='add'>+	event-&gt;ece.vendor_id = rep_data-&gt;ece.vendor_id;</div><div class='add'>+	event-&gt;ece.attr_mod = rep_data-&gt;ece.attr_mod;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int cma_cm_event_handler(struct rdma_id_private *id_priv,</div><div class='add'>+				struct rdma_cm_event *event)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	lockdep_assert_held(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+</div><div class='add'>+	trace_cm_event_handler(id_priv, event);</div><div class='add'>+	ret = id_priv-&gt;id.event_handler(&amp;id_priv-&gt;id, event);</div><div class='add'>+	trace_cm_event_done(id_priv, event, ret);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int cma_ib_handler(struct ib_cm_id *cm_id,</div><div class='hunk'>@@ -1922,13 +2181,15 @@ static int cma_ib_handler(struct ib_cm_id *cm_id,</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_id_private *id_priv = cm_id-&gt;context;</div><div class='ctx'> 	struct rdma_cm_event event = {};</div><div class='del'>-	int ret = 0;</div><div class='add'>+	enum rdma_cm_state state;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+	state = READ_ONCE(id_priv-&gt;state);</div><div class='ctx'> 	if ((ib_event-&gt;event != IB_CM_TIMEWAIT_EXIT &amp;&amp;</div><div class='del'>-	     id_priv-&gt;state != RDMA_CM_CONNECT) ||</div><div class='add'>+	     state != RDMA_CM_CONNECT) ||</div><div class='ctx'> 	    (ib_event-&gt;event == IB_CM_TIMEWAIT_EXIT &amp;&amp;</div><div class='del'>-	     id_priv-&gt;state != RDMA_CM_DISCONNECT))</div><div class='add'>+	     state != RDMA_CM_DISCONNECT))</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='ctx'> 	switch (ib_event-&gt;event) {</div><div class='hunk'>@@ -1938,9 +2199,11 @@ static int cma_ib_handler(struct ib_cm_id *cm_id,</div><div class='ctx'> 		event.status = -ETIMEDOUT;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_REP_RECEIVED:</div><div class='del'>-		if (cma_comp(id_priv, RDMA_CM_CONNECT) &amp;&amp;</div><div class='del'>-		    (id_priv-&gt;id.qp_type != IB_QPT_UD))</div><div class='del'>-			ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);</div><div class='add'>+		if (state == RDMA_CM_CONNECT &amp;&amp;</div><div class='add'>+		    (id_priv-&gt;id.qp_type != IB_QPT_UD)) {</div><div class='add'>+			trace_cm_prepare_mra(id_priv);</div><div class='add'>+			ib_prepare_cm_mra(cm_id);</div><div class='add'>+		}</div><div class='ctx'> 		if (id_priv-&gt;id.qp) {</div><div class='ctx'> 			event.status = cma_rep_recv(id_priv);</div><div class='ctx'> 			event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :</div><div class='hunk'>@@ -1956,7 +2219,8 @@ static int cma_ib_handler(struct ib_cm_id *cm_id,</div><div class='ctx'> 		event.event = RDMA_CM_EVENT_ESTABLISHED;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_CM_DREQ_ERROR:</div><div class='del'>-		event.status = -ETIMEDOUT; /* fall through */</div><div class='add'>+		event.status = -ETIMEDOUT;</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case IB_CM_DREQ_RECEIVED:</div><div class='ctx'> 	case IB_CM_DREP_RECEIVED:</div><div class='ctx'> 		if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT,</div><div class='hunk'>@@ -1985,18 +2249,16 @@ static int cma_ib_handler(struct ib_cm_id *cm_id,</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = id_priv-&gt;id.event_handler(&amp;id_priv-&gt;id, &amp;event);</div><div class='add'>+	ret = cma_cm_event_handler(id_priv, &amp;event);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		/* Destroy the CM ID by returning a non-zero value. */</div><div class='ctx'> 		id_priv-&gt;cm_id.ib = NULL;</div><div class='del'>-		cma_exch(id_priv, RDMA_CM_DESTROYING);</div><div class='del'>-		mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-		rdma_destroy_id(&amp;id_priv-&gt;id);</div><div class='add'>+		destroy_id_handler_unlock(id_priv);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> out:</div><div class='ctx'> 	mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-	return ret;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static struct rdma_id_private *</div><div class='hunk'>@@ -2015,28 +2277,29 @@ cma_ib_new_conn_id(const struct rdma_cm_id *listen_id,</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	listen_id_priv = container_of(listen_id, struct rdma_id_private, id);</div><div class='del'>-	id = __rdma_create_id(listen_id-&gt;route.addr.dev_addr.net,</div><div class='del'>-			    listen_id-&gt;event_handler, listen_id-&gt;context,</div><div class='del'>-			    listen_id-&gt;ps, ib_event-&gt;param.req_rcvd.qp_type,</div><div class='del'>-			    listen_id_priv-&gt;res.kern_name);</div><div class='del'>-	if (IS_ERR(id))</div><div class='add'>+	id_priv = __rdma_create_id(listen_id-&gt;route.addr.dev_addr.net,</div><div class='add'>+				   listen_id-&gt;event_handler, listen_id-&gt;context,</div><div class='add'>+				   listen_id-&gt;ps,</div><div class='add'>+				   ib_event-&gt;param.req_rcvd.qp_type,</div><div class='add'>+				   listen_id_priv);</div><div class='add'>+	if (IS_ERR(id_priv))</div><div class='ctx'> 		return NULL;</div><div class='ctx'> </div><div class='del'>-	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='add'>+	id = &amp;id_priv-&gt;id;</div><div class='ctx'> 	if (cma_save_net_info((struct sockaddr *)&amp;id-&gt;route.addr.src_addr,</div><div class='ctx'> 			      (struct sockaddr *)&amp;id-&gt;route.addr.dst_addr,</div><div class='ctx'> 			      listen_id, ib_event, ss_family, service_id))</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='ctx'> 	rt = &amp;id-&gt;route;</div><div class='del'>-	rt-&gt;num_paths = ib_event-&gt;param.req_rcvd.alternate_path ? 2 : 1;</div><div class='del'>-	rt-&gt;path_rec = kmalloc_array(rt-&gt;num_paths, sizeof(*rt-&gt;path_rec),</div><div class='del'>-				     GFP_KERNEL);</div><div class='add'>+	rt-&gt;num_pri_alt_paths = ib_event-&gt;param.req_rcvd.alternate_path ? 2 : 1;</div><div class='add'>+	rt-&gt;path_rec = kmalloc_array(rt-&gt;num_pri_alt_paths,</div><div class='add'>+				     sizeof(*rt-&gt;path_rec), GFP_KERNEL);</div><div class='ctx'> 	if (!rt-&gt;path_rec)</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='ctx'> 	rt-&gt;path_rec[0] = *path;</div><div class='del'>-	if (rt-&gt;num_paths == 2)</div><div class='add'>+	if (rt-&gt;num_pri_alt_paths == 2)</div><div class='ctx'> 		rt-&gt;path_rec[1] = *ib_event-&gt;param.req_rcvd.alternate_path;</div><div class='ctx'> </div><div class='ctx'> 	if (net_dev) {</div><div class='hunk'>@@ -2076,13 +2339,13 @@ cma_ib_new_udp_id(const struct rdma_cm_id *listen_id,</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	listen_id_priv = container_of(listen_id, struct rdma_id_private, id);</div><div class='del'>-	id = __rdma_create_id(net, listen_id-&gt;event_handler, listen_id-&gt;context,</div><div class='del'>-			      listen_id-&gt;ps, IB_QPT_UD,</div><div class='del'>-			      listen_id_priv-&gt;res.kern_name);</div><div class='del'>-	if (IS_ERR(id))</div><div class='add'>+	id_priv = __rdma_create_id(net, listen_id-&gt;event_handler,</div><div class='add'>+				   listen_id-&gt;context, listen_id-&gt;ps, IB_QPT_UD,</div><div class='add'>+				   listen_id_priv);</div><div class='add'>+	if (IS_ERR(id_priv))</div><div class='ctx'> 		return NULL;</div><div class='ctx'> </div><div class='del'>-	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='add'>+	id = &amp;id_priv-&gt;id;</div><div class='ctx'> 	if (cma_save_net_info((struct sockaddr *)&amp;id-&gt;route.addr.src_addr,</div><div class='ctx'> 			      (struct sockaddr *)&amp;id-&gt;route.addr.dst_addr,</div><div class='ctx'> 			      listen_id, ib_event, ss_family,</div><div class='hunk'>@@ -2120,6 +2383,9 @@ static void cma_set_req_event_data(struct rdma_cm_event *event,</div><div class='ctx'> 	event-&gt;param.conn.rnr_retry_count = req_data-&gt;rnr_retry_count;</div><div class='ctx'> 	event-&gt;param.conn.srq = req_data-&gt;srq;</div><div class='ctx'> 	event-&gt;param.conn.qp_num = req_data-&gt;remote_qpn;</div><div class='add'>+</div><div class='add'>+	event-&gt;ece.vendor_id = req_data-&gt;ece.vendor_id;</div><div class='add'>+	event-&gt;ece.attr_mod = req_data-&gt;ece.attr_mod;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int cma_ib_check_req_qp_type(const struct rdma_cm_id *id,</div><div class='hunk'>@@ -2146,15 +2412,16 @@ static int cma_ib_req_handler(struct ib_cm_id *cm_id,</div><div class='ctx'> 	if (IS_ERR(listen_id))</div><div class='ctx'> 		return PTR_ERR(listen_id);</div><div class='ctx'> </div><div class='add'>+	trace_cm_req_handler(listen_id, ib_event-&gt;event);</div><div class='ctx'> 	if (!cma_ib_check_req_qp_type(&amp;listen_id-&gt;id, ib_event)) {</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		goto net_dev_put;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;listen_id-&gt;handler_mutex);</div><div class='del'>-	if (listen_id-&gt;state != RDMA_CM_LISTEN) {</div><div class='add'>+	if (READ_ONCE(listen_id-&gt;state) != RDMA_CM_LISTEN) {</div><div class='ctx'> 		ret = -ECONNABORTED;</div><div class='del'>-		goto err1;</div><div class='add'>+		goto err_unlock;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	offset = cma_user_data_offset(listen_id);</div><div class='hunk'>@@ -2171,57 +2438,41 @@ static int cma_ib_req_handler(struct ib_cm_id *cm_id,</div><div class='ctx'> 	}</div><div class='ctx'> 	if (!conn_id) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='del'>-		goto err1;</div><div class='add'>+		goto err_unlock;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock_nested(&amp;conn_id-&gt;handler_mutex, SINGLE_DEPTH_NESTING);</div><div class='ctx'> 	ret = cma_ib_acquire_dev(conn_id, listen_id, &amp;req);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err2;</div><div class='add'>+	if (ret) {</div><div class='add'>+		destroy_id_handler_unlock(conn_id);</div><div class='add'>+		goto err_unlock;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	conn_id-&gt;cm_id.ib = cm_id;</div><div class='ctx'> 	cm_id-&gt;context = conn_id;</div><div class='ctx'> 	cm_id-&gt;cm_handler = cma_ib_handler;</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * Protect against the user destroying conn_id from another thread</div><div class='del'>-	 * until we're done accessing it.</div><div class='del'>-	 */</div><div class='del'>-	atomic_inc(&amp;conn_id-&gt;refcount);</div><div class='del'>-	ret = conn_id-&gt;id.event_handler(&amp;conn_id-&gt;id, &amp;event);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err3;</div><div class='del'>-	/*</div><div class='del'>-	 * Acquire mutex to prevent user executing rdma_destroy_id()</div><div class='del'>-	 * while we're accessing the cm_id.</div><div class='del'>-	 */</div><div class='del'>-	mutex_lock(&amp;lock);</div><div class='del'>-	if (cma_comp(conn_id, RDMA_CM_CONNECT) &amp;&amp;</div><div class='del'>-	    (conn_id-&gt;id.qp_type != IB_QPT_UD))</div><div class='del'>-		ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);</div><div class='del'>-	mutex_unlock(&amp;lock);</div><div class='del'>-	mutex_unlock(&amp;conn_id-&gt;handler_mutex);</div><div class='del'>-	mutex_unlock(&amp;listen_id-&gt;handler_mutex);</div><div class='del'>-	cma_deref_id(conn_id);</div><div class='del'>-	if (net_dev)</div><div class='del'>-		dev_put(net_dev);</div><div class='del'>-	return 0;</div><div class='add'>+	ret = cma_cm_event_handler(conn_id, &amp;event);</div><div class='add'>+	if (ret) {</div><div class='add'>+		/* Destroy the CM ID by returning a non-zero value. */</div><div class='add'>+		conn_id-&gt;cm_id.ib = NULL;</div><div class='add'>+		mutex_unlock(&amp;listen_id-&gt;handler_mutex);</div><div class='add'>+		destroy_id_handler_unlock(conn_id);</div><div class='add'>+		goto net_dev_put;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-err3:</div><div class='del'>-	cma_deref_id(conn_id);</div><div class='del'>-	/* Destroy the CM ID by returning a non-zero value. */</div><div class='del'>-	conn_id-&gt;cm_id.ib = NULL;</div><div class='del'>-err2:</div><div class='del'>-	cma_exch(conn_id, RDMA_CM_DESTROYING);</div><div class='add'>+	if (READ_ONCE(conn_id-&gt;state) == RDMA_CM_CONNECT &amp;&amp;</div><div class='add'>+	    conn_id-&gt;id.qp_type != IB_QPT_UD) {</div><div class='add'>+		trace_cm_prepare_mra(cm_id-&gt;context);</div><div class='add'>+		ib_prepare_cm_mra(cm_id);</div><div class='add'>+	}</div><div class='ctx'> 	mutex_unlock(&amp;conn_id-&gt;handler_mutex);</div><div class='del'>-err1:</div><div class='add'>+</div><div class='add'>+err_unlock:</div><div class='ctx'> 	mutex_unlock(&amp;listen_id-&gt;handler_mutex);</div><div class='del'>-	if (conn_id)</div><div class='del'>-		rdma_destroy_id(&amp;conn_id-&gt;id);</div><div class='ctx'> </div><div class='ctx'> net_dev_put:</div><div class='del'>-	if (net_dev)</div><div class='del'>-		dev_put(net_dev);</div><div class='add'>+	dev_put(net_dev);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -2271,7 +2522,7 @@ static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)</div><div class='ctx'> 	struct sockaddr *raddr = (struct sockaddr *)&amp;iw_event-&gt;remote_addr;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-	if (id_priv-&gt;state != RDMA_CM_CONNECT)</div><div class='add'>+	if (READ_ONCE(id_priv-&gt;state) != RDMA_CM_CONNECT)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='ctx'> 	switch (iw_event-&gt;event) {</div><div class='hunk'>@@ -2313,13 +2564,11 @@ static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)</div><div class='ctx'> 	event.status = iw_event-&gt;status;</div><div class='ctx'> 	event.param.conn.private_data = iw_event-&gt;private_data;</div><div class='ctx'> 	event.param.conn.private_data_len = iw_event-&gt;private_data_len;</div><div class='del'>-	ret = id_priv-&gt;id.event_handler(&amp;id_priv-&gt;id, &amp;event);</div><div class='add'>+	ret = cma_cm_event_handler(id_priv, &amp;event);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		/* Destroy the CM ID by returning a non-zero value. */</div><div class='ctx'> 		id_priv-&gt;cm_id.iw = NULL;</div><div class='del'>-		cma_exch(id_priv, RDMA_CM_DESTROYING);</div><div class='del'>-		mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-		rdma_destroy_id(&amp;id_priv-&gt;id);</div><div class='add'>+		destroy_id_handler_unlock(id_priv);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -2331,7 +2580,6 @@ out:</div><div class='ctx'> static int iw_conn_req_handler(struct iw_cm_id *cm_id,</div><div class='ctx'> 			       struct iw_cm_event *iw_event)</div><div class='ctx'> {</div><div class='del'>-	struct rdma_cm_id *new_cm_id;</div><div class='ctx'> 	struct rdma_id_private *listen_id, *conn_id;</div><div class='ctx'> 	struct rdma_cm_event event = {};</div><div class='ctx'> 	int ret = -ECONNABORTED;</div><div class='hunk'>@@ -2347,35 +2595,33 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id,</div><div class='ctx'> 	listen_id = cm_id-&gt;context;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;listen_id-&gt;handler_mutex);</div><div class='del'>-	if (listen_id-&gt;state != RDMA_CM_LISTEN)</div><div class='add'>+	if (READ_ONCE(listen_id-&gt;state) != RDMA_CM_LISTEN)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='ctx'> 	/* Create a new RDMA id for the new IW CM ID */</div><div class='del'>-	new_cm_id = __rdma_create_id(listen_id-&gt;id.route.addr.dev_addr.net,</div><div class='del'>-				     listen_id-&gt;id.event_handler,</div><div class='del'>-				     listen_id-&gt;id.context,</div><div class='del'>-				     RDMA_PS_TCP, IB_QPT_RC,</div><div class='del'>-				     listen_id-&gt;res.kern_name);</div><div class='del'>-	if (IS_ERR(new_cm_id)) {</div><div class='add'>+	conn_id = __rdma_create_id(listen_id-&gt;id.route.addr.dev_addr.net,</div><div class='add'>+				   listen_id-&gt;id.event_handler,</div><div class='add'>+				   listen_id-&gt;id.context, RDMA_PS_TCP,</div><div class='add'>+				   IB_QPT_RC, listen_id);</div><div class='add'>+	if (IS_ERR(conn_id)) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='del'>-	conn_id = container_of(new_cm_id, struct rdma_id_private, id);</div><div class='ctx'> 	mutex_lock_nested(&amp;conn_id-&gt;handler_mutex, SINGLE_DEPTH_NESTING);</div><div class='ctx'> 	conn_id-&gt;state = RDMA_CM_CONNECT;</div><div class='ctx'> </div><div class='ctx'> 	ret = rdma_translate_ip(laddr, &amp;conn_id-&gt;id.route.addr.dev_addr);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		mutex_unlock(&amp;conn_id-&gt;handler_mutex);</div><div class='del'>-		rdma_destroy_id(new_cm_id);</div><div class='del'>-		goto out;</div><div class='add'>+		mutex_unlock(&amp;listen_id-&gt;handler_mutex);</div><div class='add'>+		destroy_id_handler_unlock(conn_id);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ret = cma_iw_acquire_dev(conn_id, listen_id);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		mutex_unlock(&amp;conn_id-&gt;handler_mutex);</div><div class='del'>-		rdma_destroy_id(new_cm_id);</div><div class='del'>-		goto out;</div><div class='add'>+		mutex_unlock(&amp;listen_id-&gt;handler_mutex);</div><div class='add'>+		destroy_id_handler_unlock(conn_id);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	conn_id-&gt;cm_id.iw = cm_id;</div><div class='hunk'>@@ -2385,24 +2631,16 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id,</div><div class='ctx'> 	memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr));</div><div class='ctx'> 	memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr));</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * Protect against the user destroying conn_id from another thread</div><div class='del'>-	 * until we're done accessing it.</div><div class='del'>-	 */</div><div class='del'>-	atomic_inc(&amp;conn_id-&gt;refcount);</div><div class='del'>-	ret = conn_id-&gt;id.event_handler(&amp;conn_id-&gt;id, &amp;event);</div><div class='add'>+	ret = cma_cm_event_handler(conn_id, &amp;event);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		/* User wants to destroy the CM ID */</div><div class='ctx'> 		conn_id-&gt;cm_id.iw = NULL;</div><div class='del'>-		cma_exch(conn_id, RDMA_CM_DESTROYING);</div><div class='del'>-		mutex_unlock(&amp;conn_id-&gt;handler_mutex);</div><div class='del'>-		cma_deref_id(conn_id);</div><div class='del'>-		rdma_destroy_id(&amp;conn_id-&gt;id);</div><div class='del'>-		goto out;</div><div class='add'>+		mutex_unlock(&amp;listen_id-&gt;handler_mutex);</div><div class='add'>+		destroy_id_handler_unlock(conn_id);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	mutex_unlock(&amp;conn_id-&gt;handler_mutex);</div><div class='del'>-	cma_deref_id(conn_id);</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='ctx'> 	mutex_unlock(&amp;listen_id-&gt;handler_mutex);</div><div class='hunk'>@@ -2437,8 +2675,11 @@ static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)</div><div class='ctx'> 	if (IS_ERR(id))</div><div class='ctx'> 		return PTR_ERR(id);</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;id_priv-&gt;qp_mutex);</div><div class='ctx'> 	id-&gt;tos = id_priv-&gt;tos;</div><div class='ctx'> 	id-&gt;tos_set = id_priv-&gt;tos_set;</div><div class='add'>+	mutex_unlock(&amp;id_priv-&gt;qp_mutex);</div><div class='add'>+	id-&gt;afonly = id_priv-&gt;afonly;</div><div class='ctx'> 	id_priv-&gt;cm_id.iw = id;</div><div class='ctx'> </div><div class='ctx'> 	memcpy(&amp;id_priv-&gt;cm_id.iw-&gt;local_addr, cma_src_addr(id_priv),</div><div class='hunk'>@@ -2459,56 +2700,88 @@ static int cma_listen_handler(struct rdma_cm_id *id,</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_id_private *id_priv = id-&gt;context;</div><div class='ctx'> </div><div class='add'>+	/* Listening IDs are always destroyed on removal */</div><div class='add'>+	if (event-&gt;event == RDMA_CM_EVENT_DEVICE_REMOVAL)</div><div class='add'>+		return -1;</div><div class='add'>+</div><div class='ctx'> 	id-&gt;context = id_priv-&gt;id.context;</div><div class='ctx'> 	id-&gt;event_handler = id_priv-&gt;id.event_handler;</div><div class='add'>+	trace_cm_event_handler(id_priv, event);</div><div class='ctx'> 	return id_priv-&gt;id.event_handler(id, event);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cma_listen_on_dev(struct rdma_id_private *id_priv,</div><div class='del'>-			      struct cma_device *cma_dev)</div><div class='add'>+static int cma_listen_on_dev(struct rdma_id_private *id_priv,</div><div class='add'>+			     struct cma_device *cma_dev,</div><div class='add'>+			     struct rdma_id_private **to_destroy)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_id_private *dev_id_priv;</div><div class='del'>-	struct rdma_cm_id *id;</div><div class='ctx'> 	struct net *net = id_priv-&gt;id.route.addr.dev_addr.net;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	if (cma_family(id_priv) == AF_IB &amp;&amp; !rdma_cap_ib_cm(cma_dev-&gt;device, 1))</div><div class='del'>-		return;</div><div class='add'>+	lockdep_assert_held(&amp;lock);</div><div class='ctx'> </div><div class='del'>-	id = __rdma_create_id(net, cma_listen_handler, id_priv, id_priv-&gt;id.ps,</div><div class='del'>-			      id_priv-&gt;id.qp_type, id_priv-&gt;res.kern_name);</div><div class='del'>-	if (IS_ERR(id))</div><div class='del'>-		return;</div><div class='add'>+	*to_destroy = NULL;</div><div class='add'>+	if (cma_family(id_priv) == AF_IB &amp;&amp; !rdma_cap_ib_cm(cma_dev-&gt;device, 1))</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-	dev_id_priv = container_of(id, struct rdma_id_private, id);</div><div class='add'>+	dev_id_priv =</div><div class='add'>+		__rdma_create_id(net, cma_listen_handler, id_priv,</div><div class='add'>+				 id_priv-&gt;id.ps, id_priv-&gt;id.qp_type, id_priv);</div><div class='add'>+	if (IS_ERR(dev_id_priv))</div><div class='add'>+		return PTR_ERR(dev_id_priv);</div><div class='ctx'> </div><div class='ctx'> 	dev_id_priv-&gt;state = RDMA_CM_ADDR_BOUND;</div><div class='ctx'> 	memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv),</div><div class='ctx'> 	       rdma_addr_size(cma_src_addr(id_priv)));</div><div class='ctx'> </div><div class='ctx'> 	_cma_attach_to_dev(dev_id_priv, cma_dev);</div><div class='del'>-	list_add_tail(&amp;dev_id_priv-&gt;listen_list, &amp;id_priv-&gt;listen_list);</div><div class='del'>-	atomic_inc(&amp;id_priv-&gt;refcount);</div><div class='add'>+	rdma_restrack_add(&amp;dev_id_priv-&gt;res);</div><div class='add'>+	cma_id_get(id_priv);</div><div class='ctx'> 	dev_id_priv-&gt;internal_id = 1;</div><div class='ctx'> 	dev_id_priv-&gt;afonly = id_priv-&gt;afonly;</div><div class='add'>+	mutex_lock(&amp;id_priv-&gt;qp_mutex);</div><div class='ctx'> 	dev_id_priv-&gt;tos_set = id_priv-&gt;tos_set;</div><div class='ctx'> 	dev_id_priv-&gt;tos = id_priv-&gt;tos;</div><div class='add'>+	mutex_unlock(&amp;id_priv-&gt;qp_mutex);</div><div class='ctx'> </div><div class='del'>-	ret = rdma_listen(id, id_priv-&gt;backlog);</div><div class='add'>+	ret = rdma_listen(&amp;dev_id_priv-&gt;id, id_priv-&gt;backlog);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		dev_warn(&amp;cma_dev-&gt;device-&gt;dev,</div><div class='del'>-			 "RDMA CMA: cma_listen_on_dev, error %d\n", ret);</div><div class='add'>+		goto err_listen;</div><div class='add'>+	list_add_tail(&amp;dev_id_priv-&gt;listen_item, &amp;id_priv-&gt;listen_list);</div><div class='add'>+	return 0;</div><div class='add'>+err_listen:</div><div class='add'>+	/* Caller must destroy this after releasing lock */</div><div class='add'>+	*to_destroy = dev_id_priv;</div><div class='add'>+	dev_warn(&amp;cma_dev-&gt;device-&gt;dev, "RDMA CMA: %s, error %d\n", __func__, ret);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cma_listen_on_all(struct rdma_id_private *id_priv)</div><div class='add'>+static int cma_listen_on_all(struct rdma_id_private *id_priv)</div><div class='ctx'> {</div><div class='add'>+	struct rdma_id_private *to_destroy;</div><div class='ctx'> 	struct cma_device *cma_dev;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;lock);</div><div class='del'>-	list_add_tail(&amp;id_priv-&gt;list, &amp;listen_any_list);</div><div class='del'>-	list_for_each_entry(cma_dev, &amp;dev_list, list)</div><div class='del'>-		cma_listen_on_dev(id_priv, cma_dev);</div><div class='add'>+	list_add_tail(&amp;id_priv-&gt;listen_any_item, &amp;listen_any_list);</div><div class='add'>+	list_for_each_entry(cma_dev, &amp;dev_list, list) {</div><div class='add'>+		ret = cma_listen_on_dev(id_priv, cma_dev, &amp;to_destroy);</div><div class='add'>+		if (ret) {</div><div class='add'>+			/* Prevent racing with cma_process_remove() */</div><div class='add'>+			if (to_destroy)</div><div class='add'>+				list_del_init(&amp;to_destroy-&gt;device_item);</div><div class='add'>+			goto err_listen;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;lock);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_listen:</div><div class='add'>+	_cma_cancel_listens(id_priv);</div><div class='ctx'> 	mutex_unlock(&amp;lock);</div><div class='add'>+	if (to_destroy)</div><div class='add'>+		rdma_destroy_id(&amp;to_destroy-&gt;id);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void rdma_set_service_type(struct rdma_cm_id *id, int tos)</div><div class='hunk'>@@ -2516,8 +2789,10 @@ void rdma_set_service_type(struct rdma_cm_id *id, int tos)</div><div class='ctx'> 	struct rdma_id_private *id_priv;</div><div class='ctx'> </div><div class='ctx'> 	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='add'>+	mutex_lock(&amp;id_priv-&gt;qp_mutex);</div><div class='ctx'> 	id_priv-&gt;tos = (u8) tos;</div><div class='ctx'> 	id_priv-&gt;tos_set = true;</div><div class='add'>+	mutex_unlock(&amp;id_priv-&gt;qp_mutex);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_set_service_type);</div><div class='ctx'> </div><div class='hunk'>@@ -2530,7 +2805,9 @@ EXPORT_SYMBOL(rdma_set_service_type);</div><div class='ctx'>  * This function should be called before rdma_connect() on active side,</div><div class='ctx'>  * and on passive side before rdma_accept(). It is applicable to primary</div><div class='ctx'>  * path only. The timeout will affect the local side of the QP, it is not</div><div class='del'>- * negotiated with remote side and zero disables the timer.</div><div class='add'>+ * negotiated with remote side and zero disables the timer. In case it is</div><div class='add'>+ * set before rdma_resolve_route, the value will also be used to determine</div><div class='add'>+ * PacketLifeTime for RoCE.</div><div class='ctx'>  *</div><div class='ctx'>  * Return: 0 for success</div><div class='ctx'>  */</div><div class='hunk'>@@ -2538,37 +2815,128 @@ int rdma_set_ack_timeout(struct rdma_cm_id *id, u8 timeout)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_id_private *id_priv;</div><div class='ctx'> </div><div class='del'>-	if (id-&gt;qp_type != IB_QPT_RC)</div><div class='add'>+	if (id-&gt;qp_type != IB_QPT_RC &amp;&amp; id-&gt;qp_type != IB_QPT_XRC_INI)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='add'>+	mutex_lock(&amp;id_priv-&gt;qp_mutex);</div><div class='ctx'> 	id_priv-&gt;timeout = timeout;</div><div class='ctx'> 	id_priv-&gt;timeout_set = true;</div><div class='add'>+	mutex_unlock(&amp;id_priv-&gt;qp_mutex);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_set_ack_timeout);</div><div class='ctx'> </div><div class='add'>+/**</div><div class='add'>+ * rdma_set_min_rnr_timer() - Set the minimum RNR Retry timer of the</div><div class='add'>+ *			      QP associated with a connection identifier.</div><div class='add'>+ * @id: Communication identifier to associated with service type.</div><div class='add'>+ * @min_rnr_timer: 5-bit value encoded as Table 45: "Encoding for RNR NAK</div><div class='add'>+ *		   Timer Field" in the IBTA specification.</div><div class='add'>+ *</div><div class='add'>+ * This function should be called before rdma_connect() on active</div><div class='add'>+ * side, and on passive side before rdma_accept(). The timer value</div><div class='add'>+ * will be associated with the local QP. When it receives a send it is</div><div class='add'>+ * not read to handle, typically if the receive queue is empty, an RNR</div><div class='add'>+ * Retry NAK is returned to the requester with the min_rnr_timer</div><div class='add'>+ * encoded. The requester will then wait at least the time specified</div><div class='add'>+ * in the NAK before retrying. The default is zero, which translates</div><div class='add'>+ * to a minimum RNR Timer value of 655 ms.</div><div class='add'>+ *</div><div class='add'>+ * Return: 0 for success</div><div class='add'>+ */</div><div class='add'>+int rdma_set_min_rnr_timer(struct rdma_cm_id *id, u8 min_rnr_timer)</div><div class='add'>+{</div><div class='add'>+	struct rdma_id_private *id_priv;</div><div class='add'>+</div><div class='add'>+	/* It is a five-bit value */</div><div class='add'>+	if (min_rnr_timer &amp; 0xe0)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (WARN_ON(id-&gt;qp_type != IB_QPT_RC &amp;&amp; id-&gt;qp_type != IB_QPT_XRC_TGT))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='add'>+	mutex_lock(&amp;id_priv-&gt;qp_mutex);</div><div class='add'>+	id_priv-&gt;min_rnr_timer = min_rnr_timer;</div><div class='add'>+	id_priv-&gt;min_rnr_timer_set = true;</div><div class='add'>+	mutex_unlock(&amp;id_priv-&gt;qp_mutex);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_set_min_rnr_timer);</div><div class='add'>+</div><div class='add'>+static int route_set_path_rec_inbound(struct cma_work *work,</div><div class='add'>+				      struct sa_path_rec *path_rec)</div><div class='add'>+{</div><div class='add'>+	struct rdma_route *route = &amp;work-&gt;id-&gt;id.route;</div><div class='add'>+</div><div class='add'>+	if (!route-&gt;path_rec_inbound) {</div><div class='add'>+		route-&gt;path_rec_inbound =</div><div class='add'>+			kzalloc(sizeof(*route-&gt;path_rec_inbound), GFP_KERNEL);</div><div class='add'>+		if (!route-&gt;path_rec_inbound)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*route-&gt;path_rec_inbound = *path_rec;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int route_set_path_rec_outbound(struct cma_work *work,</div><div class='add'>+				       struct sa_path_rec *path_rec)</div><div class='add'>+{</div><div class='add'>+	struct rdma_route *route = &amp;work-&gt;id-&gt;id.route;</div><div class='add'>+</div><div class='add'>+	if (!route-&gt;path_rec_outbound) {</div><div class='add'>+		route-&gt;path_rec_outbound =</div><div class='add'>+			kzalloc(sizeof(*route-&gt;path_rec_outbound), GFP_KERNEL);</div><div class='add'>+		if (!route-&gt;path_rec_outbound)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*route-&gt;path_rec_outbound = *path_rec;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static void cma_query_handler(int status, struct sa_path_rec *path_rec,</div><div class='del'>-			      void *context)</div><div class='add'>+			      unsigned int num_prs, void *context)</div><div class='ctx'> {</div><div class='ctx'> 	struct cma_work *work = context;</div><div class='ctx'> 	struct rdma_route *route;</div><div class='add'>+	int i;</div><div class='ctx'> </div><div class='ctx'> 	route = &amp;work-&gt;id-&gt;id.route;</div><div class='ctx'> </div><div class='del'>-	if (!status) {</div><div class='del'>-		route-&gt;num_paths = 1;</div><div class='del'>-		*route-&gt;path_rec = *path_rec;</div><div class='del'>-	} else {</div><div class='del'>-		work-&gt;old_state = RDMA_CM_ROUTE_QUERY;</div><div class='del'>-		work-&gt;new_state = RDMA_CM_ADDR_RESOLVED;</div><div class='del'>-		work-&gt;event.event = RDMA_CM_EVENT_ROUTE_ERROR;</div><div class='del'>-		work-&gt;event.status = status;</div><div class='del'>-		pr_debug_ratelimited("RDMA CM: ROUTE_ERROR: failed to query path. status %d\n",</div><div class='del'>-				     status);</div><div class='add'>+	if (status)</div><div class='add'>+		goto fail;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; num_prs; i++) {</div><div class='add'>+		if (!path_rec[i].flags || (path_rec[i].flags &amp; IB_PATH_GMP))</div><div class='add'>+			*route-&gt;path_rec = path_rec[i];</div><div class='add'>+		else if (path_rec[i].flags &amp; IB_PATH_INBOUND)</div><div class='add'>+			status = route_set_path_rec_inbound(work, &amp;path_rec[i]);</div><div class='add'>+		else if (path_rec[i].flags &amp; IB_PATH_OUTBOUND)</div><div class='add'>+			status = route_set_path_rec_outbound(work,</div><div class='add'>+							     &amp;path_rec[i]);</div><div class='add'>+		else</div><div class='add'>+			status = -EINVAL;</div><div class='add'>+</div><div class='add'>+		if (status)</div><div class='add'>+			goto fail;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	route-&gt;num_pri_alt_paths = 1;</div><div class='add'>+	queue_work(cma_wq, &amp;work-&gt;work);</div><div class='add'>+	return;</div><div class='add'>+</div><div class='add'>+fail:</div><div class='add'>+	work-&gt;old_state = RDMA_CM_ROUTE_QUERY;</div><div class='add'>+	work-&gt;new_state = RDMA_CM_ADDR_RESOLVED;</div><div class='add'>+	work-&gt;event.event = RDMA_CM_EVENT_ROUTE_ERROR;</div><div class='add'>+	work-&gt;event.status = status;</div><div class='add'>+	pr_debug_ratelimited("RDMA CM: ROUTE_ERROR: failed to query path. status %d\n",</div><div class='add'>+			     status);</div><div class='ctx'> 	queue_work(cma_wq, &amp;work-&gt;work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -2625,49 +2993,54 @@ static int cma_query_ib_route(struct rdma_id_private *id_priv,</div><div class='ctx'> 	return (id_priv-&gt;query_id &lt; 0) ? id_priv-&gt;query_id : 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cma_work_handler(struct work_struct *_work)</div><div class='add'>+static void cma_iboe_join_work_handler(struct work_struct *work)</div><div class='ctx'> {</div><div class='del'>-	struct cma_work *work = container_of(_work, struct cma_work, work);</div><div class='del'>-	struct rdma_id_private *id_priv = work-&gt;id;</div><div class='del'>-	int destroy = 0;</div><div class='add'>+	struct cma_multicast *mc =</div><div class='add'>+		container_of(work, struct cma_multicast, iboe_join.work);</div><div class='add'>+	struct rdma_cm_event *event = &amp;mc-&gt;iboe_join.event;</div><div class='add'>+	struct rdma_id_private *id_priv = mc-&gt;id_priv;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-	if (!cma_comp_exch(id_priv, work-&gt;old_state, work-&gt;new_state))</div><div class='del'>-		goto out;</div><div class='add'>+	if (READ_ONCE(id_priv-&gt;state) == RDMA_CM_DESTROYING ||</div><div class='add'>+	    READ_ONCE(id_priv-&gt;state) == RDMA_CM_DEVICE_REMOVAL)</div><div class='add'>+		goto out_unlock;</div><div class='ctx'> </div><div class='del'>-	if (id_priv-&gt;id.event_handler(&amp;id_priv-&gt;id, &amp;work-&gt;event)) {</div><div class='del'>-		cma_exch(id_priv, RDMA_CM_DESTROYING);</div><div class='del'>-		destroy = 1;</div><div class='del'>-	}</div><div class='del'>-out:</div><div class='add'>+	ret = cma_cm_event_handler(id_priv, event);</div><div class='add'>+	WARN_ON(ret);</div><div class='add'>+</div><div class='add'>+out_unlock:</div><div class='ctx'> 	mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-	cma_deref_id(id_priv);</div><div class='del'>-	if (destroy)</div><div class='del'>-		rdma_destroy_id(&amp;id_priv-&gt;id);</div><div class='del'>-	kfree(work);</div><div class='add'>+	if (event-&gt;event == RDMA_CM_EVENT_MULTICAST_JOIN)</div><div class='add'>+		rdma_destroy_ah_attr(&amp;event-&gt;param.ud.ah_attr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cma_ndev_work_handler(struct work_struct *_work)</div><div class='add'>+static void cma_work_handler(struct work_struct *_work)</div><div class='ctx'> {</div><div class='del'>-	struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);</div><div class='add'>+	struct cma_work *work = container_of(_work, struct cma_work, work);</div><div class='ctx'> 	struct rdma_id_private *id_priv = work-&gt;id;</div><div class='del'>-	int destroy = 0;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-	if (id_priv-&gt;state == RDMA_CM_DESTROYING ||</div><div class='del'>-	    id_priv-&gt;state == RDMA_CM_DEVICE_REMOVAL)</div><div class='del'>-		goto out;</div><div class='add'>+	if (READ_ONCE(id_priv-&gt;state) == RDMA_CM_DESTROYING ||</div><div class='add'>+	    READ_ONCE(id_priv-&gt;state) == RDMA_CM_DEVICE_REMOVAL)</div><div class='add'>+		goto out_unlock;</div><div class='add'>+	if (work-&gt;old_state != 0 || work-&gt;new_state != 0) {</div><div class='add'>+		if (!cma_comp_exch(id_priv, work-&gt;old_state, work-&gt;new_state))</div><div class='add'>+			goto out_unlock;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (id_priv-&gt;id.event_handler(&amp;id_priv-&gt;id, &amp;work-&gt;event)) {</div><div class='del'>-		cma_exch(id_priv, RDMA_CM_DESTROYING);</div><div class='del'>-		destroy = 1;</div><div class='add'>+	if (cma_cm_event_handler(id_priv, &amp;work-&gt;event)) {</div><div class='add'>+		cma_id_put(id_priv);</div><div class='add'>+		destroy_id_handler_unlock(id_priv);</div><div class='add'>+		goto out_free;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-out:</div><div class='add'>+out_unlock:</div><div class='ctx'> 	mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-	cma_deref_id(id_priv);</div><div class='del'>-	if (destroy)</div><div class='del'>-		rdma_destroy_id(&amp;id_priv-&gt;id);</div><div class='add'>+	cma_id_put(id_priv);</div><div class='add'>+out_free:</div><div class='add'>+	if (work-&gt;event.event == RDMA_CM_EVENT_MULTICAST_JOIN)</div><div class='add'>+		rdma_destroy_ah_attr(&amp;work-&gt;event.param.ud.ah_attr);</div><div class='ctx'> 	kfree(work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -2681,14 +3054,19 @@ static void cma_init_resolve_route_work(struct cma_work *work,</div><div class='ctx'> 	work-&gt;event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cma_init_resolve_addr_work(struct cma_work *work,</div><div class='del'>-				       struct rdma_id_private *id_priv)</div><div class='add'>+static void enqueue_resolve_addr_work(struct cma_work *work,</div><div class='add'>+				      struct rdma_id_private *id_priv)</div><div class='ctx'> {</div><div class='add'>+	/* Balances with cma_id_put() in cma_work_handler */</div><div class='add'>+	cma_id_get(id_priv);</div><div class='add'>+</div><div class='ctx'> 	work-&gt;id = id_priv;</div><div class='ctx'> 	INIT_WORK(&amp;work-&gt;work, cma_work_handler);</div><div class='ctx'> 	work-&gt;old_state = RDMA_CM_ADDR_QUERY;</div><div class='ctx'> 	work-&gt;new_state = RDMA_CM_ADDR_RESOLVED;</div><div class='ctx'> 	work-&gt;event.event = RDMA_CM_EVENT_ADDR_RESOLVED;</div><div class='add'>+</div><div class='add'>+	queue_work(cma_wq, &amp;work-&gt;work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int cma_resolve_ib_route(struct rdma_id_private *id_priv,</div><div class='hunk'>@@ -2704,7 +3082,8 @@ static int cma_resolve_ib_route(struct rdma_id_private *id_priv,</div><div class='ctx'> </div><div class='ctx'> 	cma_init_resolve_route_work(work, id_priv);</div><div class='ctx'> </div><div class='del'>-	route-&gt;path_rec = kmalloc(sizeof *route-&gt;path_rec, GFP_KERNEL);</div><div class='add'>+	if (!route-&gt;path_rec)</div><div class='add'>+		route-&gt;path_rec = kmalloc(sizeof *route-&gt;path_rec, GFP_KERNEL);</div><div class='ctx'> 	if (!route-&gt;path_rec) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='ctx'> 		goto err1;</div><div class='hunk'>@@ -2802,7 +3181,7 @@ int rdma_set_ib_path(struct rdma_cm_id *id,</div><div class='ctx'> 		dev_put(ndev);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	id-&gt;route.num_paths = 1;</div><div class='add'>+	id-&gt;route.num_pri_alt_paths = 1;</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> err_free:</div><div class='hunk'>@@ -2827,22 +3206,86 @@ static int cma_resolve_iw_route(struct rdma_id_private *id_priv)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int iboe_tos_to_sl(struct net_device *ndev, int tos)</div><div class='add'>+static int get_vlan_ndev_tc(struct net_device *vlan_ndev, int prio)</div><div class='ctx'> {</div><div class='del'>-	int prio;</div><div class='ctx'> 	struct net_device *dev;</div><div class='ctx'> </div><div class='del'>-	prio = rt_tos2priority(tos);</div><div class='del'>-	dev = is_vlan_dev(ndev) ? vlan_dev_real_dev(ndev) : ndev;</div><div class='add'>+	dev = vlan_dev_real_dev(vlan_ndev);</div><div class='ctx'> 	if (dev-&gt;num_tc)</div><div class='ctx'> 		return netdev_get_prio_tc_map(dev, prio);</div><div class='ctx'> </div><div class='del'>-#if IS_ENABLED(CONFIG_VLAN_8021Q)</div><div class='add'>+	return (vlan_dev_get_egress_qos_mask(vlan_ndev, prio) &amp;</div><div class='add'>+		VLAN_PRIO_MASK) &gt;&gt; VLAN_PRIO_SHIFT;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct iboe_prio_tc_map {</div><div class='add'>+	int input_prio;</div><div class='add'>+	int output_tc;</div><div class='add'>+	bool found;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int get_lower_vlan_dev_tc(struct net_device *dev,</div><div class='add'>+				 struct netdev_nested_priv *priv)</div><div class='add'>+{</div><div class='add'>+	struct iboe_prio_tc_map *map = (struct iboe_prio_tc_map *)priv-&gt;data;</div><div class='add'>+</div><div class='add'>+	if (is_vlan_dev(dev))</div><div class='add'>+		map-&gt;output_tc = get_vlan_ndev_tc(dev, map-&gt;input_prio);</div><div class='add'>+	else if (dev-&gt;num_tc)</div><div class='add'>+		map-&gt;output_tc = netdev_get_prio_tc_map(dev, map-&gt;input_prio);</div><div class='add'>+	else</div><div class='add'>+		map-&gt;output_tc = 0;</div><div class='add'>+	/* We are interested only in first level VLAN device, so always</div><div class='add'>+	 * return 1 to stop iterating over next level devices.</div><div class='add'>+	 */</div><div class='add'>+	map-&gt;found = true;</div><div class='add'>+	return 1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int iboe_tos_to_sl(struct net_device *ndev, int tos)</div><div class='add'>+{</div><div class='add'>+	struct iboe_prio_tc_map prio_tc_map = {};</div><div class='add'>+	int prio = rt_tos2priority(tos);</div><div class='add'>+	struct netdev_nested_priv priv;</div><div class='add'>+</div><div class='add'>+	/* If VLAN device, get it directly from the VLAN netdev */</div><div class='ctx'> 	if (is_vlan_dev(ndev))</div><div class='del'>-		return (vlan_dev_get_egress_qos_mask(ndev, prio) &amp;</div><div class='del'>-			VLAN_PRIO_MASK) &gt;&gt; VLAN_PRIO_SHIFT;</div><div class='del'>-#endif</div><div class='del'>-	return 0;</div><div class='add'>+		return get_vlan_ndev_tc(ndev, prio);</div><div class='add'>+</div><div class='add'>+	prio_tc_map.input_prio = prio;</div><div class='add'>+	priv.data = (void *)&amp;prio_tc_map;</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	netdev_walk_all_lower_dev_rcu(ndev,</div><div class='add'>+				      get_lower_vlan_dev_tc,</div><div class='add'>+				      &amp;priv);</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+	/* If map is found from lower device, use it; Otherwise</div><div class='add'>+	 * continue with the current netdevice to get priority to tc map.</div><div class='add'>+	 */</div><div class='add'>+	if (prio_tc_map.found)</div><div class='add'>+		return prio_tc_map.output_tc;</div><div class='add'>+	else if (ndev-&gt;num_tc)</div><div class='add'>+		return netdev_get_prio_tc_map(ndev, prio);</div><div class='add'>+	else</div><div class='add'>+		return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static __be32 cma_get_roce_udp_flow_label(struct rdma_id_private *id_priv)</div><div class='add'>+{</div><div class='add'>+	struct sockaddr_in6 *addr6;</div><div class='add'>+	u16 dport, sport;</div><div class='add'>+	u32 hash, fl;</div><div class='add'>+</div><div class='add'>+	addr6 = (struct sockaddr_in6 *)cma_src_addr(id_priv);</div><div class='add'>+	fl = be32_to_cpu(addr6-&gt;sin6_flowinfo) &amp; IB_GRH_FLOWLABEL_MASK;</div><div class='add'>+	if ((cma_family(id_priv) != AF_INET6) || !fl) {</div><div class='add'>+		dport = be16_to_cpu(cma_port(cma_dst_addr(id_priv)));</div><div class='add'>+		sport = be16_to_cpu(cma_port(cma_src_addr(id_priv)));</div><div class='add'>+		hash = (u32)sport * 31 + dport;</div><div class='add'>+		fl = hash &amp; IB_GRH_FLOWLABEL_MASK;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return cpu_to_be32(fl);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)</div><div class='hunk'>@@ -2855,8 +3298,11 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)</div><div class='ctx'> </div><div class='ctx'> 	u8 default_roce_tos = id_priv-&gt;cma_dev-&gt;default_roce_tos[id_priv-&gt;id.port_num -</div><div class='ctx'> 					rdma_start_port(id_priv-&gt;cma_dev-&gt;device)];</div><div class='del'>-	u8 tos = id_priv-&gt;tos_set ? id_priv-&gt;tos : default_roce_tos;</div><div class='add'>+	u8 tos;</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;id_priv-&gt;qp_mutex);</div><div class='add'>+	tos = id_priv-&gt;tos_set ? id_priv-&gt;tos : default_roce_tos;</div><div class='add'>+	mutex_unlock(&amp;id_priv-&gt;qp_mutex);</div><div class='ctx'> </div><div class='ctx'> 	work = kzalloc(sizeof *work, GFP_KERNEL);</div><div class='ctx'> 	if (!work)</div><div class='hunk'>@@ -2868,7 +3314,7 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)</div><div class='ctx'> 		goto err1;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	route-&gt;num_paths = 1;</div><div class='add'>+	route-&gt;num_pri_alt_paths = 1;</div><div class='ctx'> </div><div class='ctx'> 	ndev = cma_iboe_set_path_rec_l2_fields(id_priv);</div><div class='ctx'> 	if (!ndev) {</div><div class='hunk'>@@ -2893,15 +3339,33 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)</div><div class='ctx'> 	route-&gt;path_rec-&gt;traffic_class = tos;</div><div class='ctx'> 	route-&gt;path_rec-&gt;mtu = iboe_get_mtu(ndev-&gt;mtu);</div><div class='ctx'> 	route-&gt;path_rec-&gt;rate_selector = IB_SA_EQ;</div><div class='del'>-	route-&gt;path_rec-&gt;rate = iboe_get_rate(ndev);</div><div class='add'>+	route-&gt;path_rec-&gt;rate = IB_RATE_PORT_CURRENT;</div><div class='ctx'> 	dev_put(ndev);</div><div class='ctx'> 	route-&gt;path_rec-&gt;packet_life_time_selector = IB_SA_EQ;</div><div class='del'>-	route-&gt;path_rec-&gt;packet_life_time = CMA_IBOE_PACKET_LIFETIME;</div><div class='add'>+	/* In case ACK timeout is set, use this value to calculate</div><div class='add'>+	 * PacketLifeTime.  As per IBTA 12.7.34,</div><div class='add'>+	 * local ACK timeout = (2 * PacketLifeTime + Local CA’s ACK delay).</div><div class='add'>+	 * Assuming a negligible local ACK delay, we can use</div><div class='add'>+	 * PacketLifeTime = local ACK timeout/2</div><div class='add'>+	 * as a reasonable approximation for RoCE networks.</div><div class='add'>+	 */</div><div class='add'>+	mutex_lock(&amp;id_priv-&gt;qp_mutex);</div><div class='add'>+	if (id_priv-&gt;timeout_set &amp;&amp; id_priv-&gt;timeout)</div><div class='add'>+		route-&gt;path_rec-&gt;packet_life_time = id_priv-&gt;timeout - 1;</div><div class='add'>+	else</div><div class='add'>+		route-&gt;path_rec-&gt;packet_life_time = CMA_IBOE_PACKET_LIFETIME;</div><div class='add'>+	mutex_unlock(&amp;id_priv-&gt;qp_mutex);</div><div class='add'>+</div><div class='ctx'> 	if (!route-&gt;path_rec-&gt;mtu) {</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		goto err2;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (rdma_protocol_roce_udp_encap(id_priv-&gt;id.device,</div><div class='add'>+					 id_priv-&gt;id.port_num))</div><div class='add'>+		route-&gt;path_rec-&gt;flow_label =</div><div class='add'>+			cma_get_roce_udp_flow_label(id_priv);</div><div class='add'>+</div><div class='ctx'> 	cma_init_resolve_route_work(work, id_priv);</div><div class='ctx'> 	queue_work(cma_wq, &amp;work-&gt;work);</div><div class='ctx'> </div><div class='hunk'>@@ -2910,6 +3374,7 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)</div><div class='ctx'> err2:</div><div class='ctx'> 	kfree(route-&gt;path_rec);</div><div class='ctx'> 	route-&gt;path_rec = NULL;</div><div class='add'>+	route-&gt;num_pri_alt_paths = 0;</div><div class='ctx'> err1:</div><div class='ctx'> 	kfree(work);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -2918,17 +3383,28 @@ err1:</div><div class='ctx'> int rdma_resolve_route(struct rdma_cm_id *id, unsigned long timeout_ms)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_id_private *id_priv;</div><div class='add'>+	enum rdma_cm_state state;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	if (!timeout_ms)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='ctx'> 	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='del'>-	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY))</div><div class='add'>+	state = id_priv-&gt;state;</div><div class='add'>+	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,</div><div class='add'>+			   RDMA_CM_ROUTE_QUERY) &amp;&amp;</div><div class='add'>+	    !cma_comp_exch(id_priv, RDMA_CM_ADDRINFO_RESOLVED,</div><div class='add'>+			   RDMA_CM_ROUTE_QUERY))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	atomic_inc(&amp;id_priv-&gt;refcount);</div><div class='add'>+	cma_id_get(id_priv);</div><div class='ctx'> 	if (rdma_cap_ib_sa(id-&gt;device, id-&gt;port_num))</div><div class='ctx'> 		ret = cma_resolve_ib_route(id_priv, timeout_ms);</div><div class='del'>-	else if (rdma_protocol_roce(id-&gt;device, id-&gt;port_num))</div><div class='add'>+	else if (rdma_protocol_roce(id-&gt;device, id-&gt;port_num)) {</div><div class='ctx'> 		ret = cma_resolve_iboe_route(id_priv);</div><div class='add'>+		if (!ret)</div><div class='add'>+			cma_add_id_to_tree(id_priv);</div><div class='add'>+	}</div><div class='ctx'> 	else if (rdma_protocol_iwarp(id-&gt;device, id-&gt;port_num))</div><div class='ctx'> 		ret = cma_resolve_iw_route(id_priv);</div><div class='ctx'> 	else</div><div class='hunk'>@@ -2939,8 +3415,8 @@ int rdma_resolve_route(struct rdma_cm_id *id, unsigned long timeout_ms)</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> err:</div><div class='del'>-	cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED);</div><div class='del'>-	cma_deref_id(id_priv);</div><div class='add'>+	cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, state);</div><div class='add'>+	cma_id_put(id_priv);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_resolve_route);</div><div class='hunk'>@@ -2967,9 +3443,9 @@ static int cma_bind_loopback(struct rdma_id_private *id_priv)</div><div class='ctx'> 	struct cma_device *cma_dev, *cur_dev;</div><div class='ctx'> 	union ib_gid gid;</div><div class='ctx'> 	enum ib_port_state port_state;</div><div class='add'>+	unsigned int p;</div><div class='ctx'> 	u16 pkey;</div><div class='ctx'> 	int ret;</div><div class='del'>-	u8 p;</div><div class='ctx'> </div><div class='ctx'> 	cma_dev = NULL;</div><div class='ctx'> 	mutex_lock(&amp;lock);</div><div class='hunk'>@@ -2981,7 +3457,7 @@ static int cma_bind_loopback(struct rdma_id_private *id_priv)</div><div class='ctx'> 		if (!cma_dev)</div><div class='ctx'> 			cma_dev = cur_dev;</div><div class='ctx'> </div><div class='del'>-		for (p = 1; p &lt;= cur_dev-&gt;device-&gt;phys_port_cnt; ++p) {</div><div class='add'>+		rdma_for_each_port (cur_dev-&gt;device, p) {</div><div class='ctx'> 			if (!ib_get_cached_port_state(cur_dev-&gt;device, p, &amp;port_state) &amp;&amp;</div><div class='ctx'> 			    port_state == IB_PORT_ACTIVE) {</div><div class='ctx'> 				cma_dev = cur_dev;</div><div class='hunk'>@@ -3014,6 +3490,7 @@ port_found:</div><div class='ctx'> 	ib_addr_set_pkey(&amp;id_priv-&gt;id.route.addr.dev_addr, pkey);</div><div class='ctx'> 	id_priv-&gt;id.port_num = p;</div><div class='ctx'> 	cma_attach_to_dev(id_priv, cma_dev);</div><div class='add'>+	rdma_restrack_add(&amp;id_priv-&gt;res);</div><div class='ctx'> 	cma_set_loopback(cma_src_addr(id_priv));</div><div class='ctx'> out:</div><div class='ctx'> 	mutex_unlock(&amp;lock);</div><div class='hunk'>@@ -3046,7 +3523,8 @@ static void addr_handler(int status, struct sockaddr *src_addr,</div><div class='ctx'> 		if (status)</div><div class='ctx'> 			pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to acquire device. status %d\n",</div><div class='ctx'> 					     status);</div><div class='del'>-	} else {</div><div class='add'>+		rdma_restrack_add(&amp;id_priv-&gt;res);</div><div class='add'>+	} else if (status) {</div><div class='ctx'> 		pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to resolve IP. status %d\n", status);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -3061,10 +3539,8 @@ static void addr_handler(int status, struct sockaddr *src_addr,</div><div class='ctx'> 	} else</div><div class='ctx'> 		event.event = RDMA_CM_EVENT_ADDR_RESOLVED;</div><div class='ctx'> </div><div class='del'>-	if (id_priv-&gt;id.event_handler(&amp;id_priv-&gt;id, &amp;event)) {</div><div class='del'>-		cma_exch(id_priv, RDMA_CM_DESTROYING);</div><div class='del'>-		mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-		rdma_destroy_id(&amp;id_priv-&gt;id);</div><div class='add'>+	if (cma_cm_event_handler(id_priv, &amp;event)) {</div><div class='add'>+		destroy_id_handler_unlock(id_priv);</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> out:</div><div class='hunk'>@@ -3090,8 +3566,7 @@ static int cma_resolve_loopback(struct rdma_id_private *id_priv)</div><div class='ctx'> 	rdma_addr_get_sgid(&amp;id_priv-&gt;id.route.addr.dev_addr, &amp;gid);</div><div class='ctx'> 	rdma_addr_set_dgid(&amp;id_priv-&gt;id.route.addr.dev_addr, &amp;gid);</div><div class='ctx'> </div><div class='del'>-	cma_init_resolve_addr_work(work, id_priv);</div><div class='del'>-	queue_work(cma_wq, &amp;work-&gt;work);</div><div class='add'>+	enqueue_resolve_addr_work(work, id_priv);</div><div class='ctx'> 	return 0;</div><div class='ctx'> err:</div><div class='ctx'> 	kfree(work);</div><div class='hunk'>@@ -3116,77 +3591,13 @@ static int cma_resolve_ib_addr(struct rdma_id_private *id_priv)</div><div class='ctx'> 	rdma_addr_set_dgid(&amp;id_priv-&gt;id.route.addr.dev_addr, (union ib_gid *)</div><div class='ctx'> 		&amp;(((struct sockaddr_ib *) &amp;id_priv-&gt;id.route.addr.dst_addr)-&gt;sib_addr));</div><div class='ctx'> </div><div class='del'>-	cma_init_resolve_addr_work(work, id_priv);</div><div class='del'>-	queue_work(cma_wq, &amp;work-&gt;work);</div><div class='add'>+	enqueue_resolve_addr_work(work, id_priv);</div><div class='ctx'> 	return 0;</div><div class='ctx'> err:</div><div class='ctx'> 	kfree(work);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,</div><div class='del'>-			 const struct sockaddr *dst_addr)</div><div class='del'>-{</div><div class='del'>-	if (!src_addr || !src_addr-&gt;sa_family) {</div><div class='del'>-		src_addr = (struct sockaddr *) &amp;id-&gt;route.addr.src_addr;</div><div class='del'>-		src_addr-&gt;sa_family = dst_addr-&gt;sa_family;</div><div class='del'>-		if (IS_ENABLED(CONFIG_IPV6) &amp;&amp;</div><div class='del'>-		    dst_addr-&gt;sa_family == AF_INET6) {</div><div class='del'>-			struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *) src_addr;</div><div class='del'>-			struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *) dst_addr;</div><div class='del'>-			src_addr6-&gt;sin6_scope_id = dst_addr6-&gt;sin6_scope_id;</div><div class='del'>-			if (ipv6_addr_type(&amp;dst_addr6-&gt;sin6_addr) &amp; IPV6_ADDR_LINKLOCAL)</div><div class='del'>-				id-&gt;route.addr.dev_addr.bound_dev_if = dst_addr6-&gt;sin6_scope_id;</div><div class='del'>-		} else if (dst_addr-&gt;sa_family == AF_IB) {</div><div class='del'>-			((struct sockaddr_ib *) src_addr)-&gt;sib_pkey =</div><div class='del'>-				((struct sockaddr_ib *) dst_addr)-&gt;sib_pkey;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	return rdma_bind_addr(id, src_addr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,</div><div class='del'>-		      const struct sockaddr *dst_addr, unsigned long timeout_ms)</div><div class='del'>-{</div><div class='del'>-	struct rdma_id_private *id_priv;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='del'>-	if (id_priv-&gt;state == RDMA_CM_IDLE) {</div><div class='del'>-		ret = cma_bind_addr(id, src_addr, dst_addr);</div><div class='del'>-		if (ret)</div><div class='del'>-			return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (cma_family(id_priv) != dst_addr-&gt;sa_family)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr));</div><div class='del'>-	if (cma_any_addr(dst_addr)) {</div><div class='del'>-		ret = cma_resolve_loopback(id_priv);</div><div class='del'>-	} else {</div><div class='del'>-		if (dst_addr-&gt;sa_family == AF_IB) {</div><div class='del'>-			ret = cma_resolve_ib_addr(id_priv);</div><div class='del'>-		} else {</div><div class='del'>-			ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr,</div><div class='del'>-					      &amp;id-&gt;route.addr.dev_addr,</div><div class='del'>-					      timeout_ms, addr_handler,</div><div class='del'>-					      false, id_priv);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-err:</div><div class='del'>-	cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(rdma_resolve_addr);</div><div class='del'>-</div><div class='ctx'> int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_id_private *id_priv;</div><div class='hunk'>@@ -3195,7 +3606,8 @@ int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)</div><div class='ctx'> </div><div class='ctx'> 	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='ctx'> 	spin_lock_irqsave(&amp;id_priv-&gt;lock, flags);</div><div class='del'>-	if (reuse || id_priv-&gt;state == RDMA_CM_IDLE) {</div><div class='add'>+	if ((reuse &amp;&amp; id_priv-&gt;state != RDMA_CM_LISTEN) ||</div><div class='add'>+	    id_priv-&gt;state == RDMA_CM_IDLE) {</div><div class='ctx'> 		id_priv-&gt;reuseaddr = reuse;</div><div class='ctx'> 		ret = 0;</div><div class='ctx'> 	} else {</div><div class='hunk'>@@ -3234,6 +3646,8 @@ static void cma_bind_port(struct rdma_bind_list *bind_list,</div><div class='ctx'> 	u64 sid, mask;</div><div class='ctx'> 	__be16 port;</div><div class='ctx'> </div><div class='add'>+	lockdep_assert_held(&amp;lock);</div><div class='add'>+</div><div class='ctx'> 	addr = cma_src_addr(id_priv);</div><div class='ctx'> 	port = htons(bind_list-&gt;port);</div><div class='ctx'> </div><div class='hunk'>@@ -3262,6 +3676,8 @@ static int cma_alloc_port(enum rdma_ucm_port_space ps,</div><div class='ctx'> 	struct rdma_bind_list *bind_list;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	lockdep_assert_held(&amp;lock);</div><div class='add'>+</div><div class='ctx'> 	bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);</div><div class='ctx'> 	if (!bind_list)</div><div class='ctx'> 		return -ENOMEM;</div><div class='hunk'>@@ -3288,6 +3704,8 @@ static int cma_port_is_unique(struct rdma_bind_list *bind_list,</div><div class='ctx'> 	struct sockaddr  *saddr = cma_src_addr(id_priv);</div><div class='ctx'> 	__be16 dport = cma_port(daddr);</div><div class='ctx'> </div><div class='add'>+	lockdep_assert_held(&amp;lock);</div><div class='add'>+</div><div class='ctx'> 	hlist_for_each_entry(cur_id, &amp;bind_list-&gt;owners, node) {</div><div class='ctx'> 		struct sockaddr  *cur_daddr = cma_dst_addr(cur_id);</div><div class='ctx'> 		struct sockaddr  *cur_saddr = cma_src_addr(cur_id);</div><div class='hunk'>@@ -3327,9 +3745,11 @@ static int cma_alloc_any_port(enum rdma_ucm_port_space ps,</div><div class='ctx'> 	unsigned int rover;</div><div class='ctx'> 	struct net *net = id_priv-&gt;id.route.addr.dev_addr.net;</div><div class='ctx'> </div><div class='add'>+	lockdep_assert_held(&amp;lock);</div><div class='add'>+</div><div class='ctx'> 	inet_get_local_port_range(net, &amp;low, &amp;high);</div><div class='ctx'> 	remaining = (high - low) + 1;</div><div class='del'>-	rover = prandom_u32() % remaining + low;</div><div class='add'>+	rover = get_random_u32_inclusive(low, remaining + low - 1);</div><div class='ctx'> retry:</div><div class='ctx'> 	if (last_used_port != rover) {</div><div class='ctx'> 		struct rdma_bind_list *bind_list;</div><div class='hunk'>@@ -3374,13 +3794,14 @@ static int cma_check_port(struct rdma_bind_list *bind_list,</div><div class='ctx'> 	struct rdma_id_private *cur_id;</div><div class='ctx'> 	struct sockaddr *addr, *cur_addr;</div><div class='ctx'> </div><div class='add'>+	lockdep_assert_held(&amp;lock);</div><div class='add'>+</div><div class='ctx'> 	addr = cma_src_addr(id_priv);</div><div class='ctx'> 	hlist_for_each_entry(cur_id, &amp;bind_list-&gt;owners, node) {</div><div class='ctx'> 		if (id_priv == cur_id)</div><div class='ctx'> 			continue;</div><div class='ctx'> </div><div class='del'>-		if ((cur_id-&gt;state != RDMA_CM_LISTEN) &amp;&amp; reuseaddr &amp;&amp;</div><div class='del'>-		    cur_id-&gt;reuseaddr)</div><div class='add'>+		if (reuseaddr &amp;&amp; cur_id-&gt;reuseaddr)</div><div class='ctx'> 			continue;</div><div class='ctx'> </div><div class='ctx'> 		cur_addr = cma_src_addr(cur_id);</div><div class='hunk'>@@ -3404,6 +3825,8 @@ static int cma_use_port(enum rdma_ucm_port_space ps,</div><div class='ctx'> 	unsigned short snum;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	lockdep_assert_held(&amp;lock);</div><div class='add'>+</div><div class='ctx'> 	snum = ntohs(cma_port(cma_src_addr(id_priv)));</div><div class='ctx'> 	if (snum &lt; PROT_SOCK &amp;&amp; !capable(CAP_NET_BIND_SERVICE))</div><div class='ctx'> 		return -EACCES;</div><div class='hunk'>@@ -3419,18 +3842,6 @@ static int cma_use_port(enum rdma_ucm_port_space ps,</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int cma_bind_listen(struct rdma_id_private *id_priv)</div><div class='del'>-{</div><div class='del'>-	struct rdma_bind_list *bind_list = id_priv-&gt;bind_list;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;lock);</div><div class='del'>-	if (bind_list-&gt;owners.first-&gt;next)</div><div class='del'>-		ret = cma_check_port(bind_list, id_priv, 0);</div><div class='del'>-	mutex_unlock(&amp;lock);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static enum rdma_ucm_port_space</div><div class='ctx'> cma_select_inet_ps(struct rdma_id_private *id_priv)</div><div class='ctx'> {</div><div class='hunk'>@@ -3524,28 +3935,41 @@ static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,</div><div class='ctx'> </div><div class='ctx'> int rdma_listen(struct rdma_cm_id *id, int backlog)</div><div class='ctx'> {</div><div class='del'>-	struct rdma_id_private *id_priv;</div><div class='add'>+	struct rdma_id_private *id_priv =</div><div class='add'>+		container_of(id, struct rdma_id_private, id);</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='del'>-	if (id_priv-&gt;state == RDMA_CM_IDLE) {</div><div class='del'>-		id-&gt;route.addr.src_addr.ss_family = AF_INET;</div><div class='del'>-		ret = rdma_bind_addr(id, cma_src_addr(id_priv));</div><div class='add'>+	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN)) {</div><div class='add'>+		struct sockaddr_in any_in = {</div><div class='add'>+			.sin_family = AF_INET,</div><div class='add'>+			.sin_addr.s_addr = htonl(INADDR_ANY),</div><div class='add'>+		};</div><div class='add'>+</div><div class='add'>+		/* For a well behaved ULP state will be RDMA_CM_IDLE */</div><div class='add'>+		ret = rdma_bind_addr(id, (struct sockaddr *)&amp;any_in);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			return ret;</div><div class='add'>+		if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND,</div><div class='add'>+					   RDMA_CM_LISTEN)))</div><div class='add'>+			return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='add'>+	/*</div><div class='add'>+	 * Once the ID reaches RDMA_CM_LISTEN it is not allowed to be reusable</div><div class='add'>+	 * any more, and has to be unique in the bind list.</div><div class='add'>+	 */</div><div class='ctx'> 	if (id_priv-&gt;reuseaddr) {</div><div class='del'>-		ret = cma_bind_listen(id_priv);</div><div class='add'>+		mutex_lock(&amp;lock);</div><div class='add'>+		ret = cma_check_port(id_priv-&gt;bind_list, id_priv, 0);</div><div class='add'>+		if (!ret)</div><div class='add'>+			id_priv-&gt;reuseaddr = 0;</div><div class='add'>+		mutex_unlock(&amp;lock);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	id_priv-&gt;backlog = backlog;</div><div class='del'>-	if (id-&gt;device) {</div><div class='add'>+	if (id_priv-&gt;cma_dev) {</div><div class='ctx'> 		if (rdma_cap_ib_cm(id-&gt;device, 1)) {</div><div class='ctx'> 			ret = cma_ib_listen(id_priv);</div><div class='ctx'> 			if (ret)</div><div class='hunk'>@@ -3558,38 +3982,44 @@ int rdma_listen(struct rdma_cm_id *id, int backlog)</div><div class='ctx'> 			ret = -ENOSYS;</div><div class='ctx'> 			goto err;</div><div class='ctx'> 		}</div><div class='del'>-	} else</div><div class='del'>-		cma_listen_on_all(id_priv);</div><div class='add'>+	} else {</div><div class='add'>+		ret = cma_listen_on_all(id_priv);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> err:</div><div class='ctx'> 	id_priv-&gt;backlog = 0;</div><div class='add'>+	/*</div><div class='add'>+	 * All the failure paths that lead here will not allow the req_handler's</div><div class='add'>+	 * to have run.</div><div class='add'>+	 */</div><div class='ctx'> 	cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_listen);</div><div class='ctx'> </div><div class='del'>-int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)</div><div class='add'>+static int rdma_bind_addr_dst(struct rdma_id_private *id_priv,</div><div class='add'>+			      struct sockaddr *addr, const struct sockaddr *daddr)</div><div class='ctx'> {</div><div class='del'>-	struct rdma_id_private *id_priv;</div><div class='add'>+	struct sockaddr *id_daddr;</div><div class='ctx'> 	int ret;</div><div class='del'>-	struct sockaddr  *daddr;</div><div class='ctx'> </div><div class='ctx'> 	if (addr-&gt;sa_family != AF_INET &amp;&amp; addr-&gt;sa_family != AF_INET6 &amp;&amp;</div><div class='ctx'> 	    addr-&gt;sa_family != AF_IB)</div><div class='ctx'> 		return -EAFNOSUPPORT;</div><div class='ctx'> </div><div class='del'>-	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='ctx'> 	if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	ret = cma_check_linklocal(&amp;id-&gt;route.addr.dev_addr, addr);</div><div class='add'>+	ret = cma_check_linklocal(&amp;id_priv-&gt;id.route.addr.dev_addr, addr);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err1;</div><div class='ctx'> </div><div class='ctx'> 	memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr));</div><div class='ctx'> 	if (!cma_any_addr(addr)) {</div><div class='del'>-		ret = cma_translate_addr(addr, &amp;id-&gt;route.addr.dev_addr);</div><div class='add'>+		ret = cma_translate_addr(addr, &amp;id_priv-&gt;id.route.addr.dev_addr);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto err1;</div><div class='ctx'> </div><div class='hunk'>@@ -3609,22 +4039,148 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)</div><div class='ctx'> 		}</div><div class='ctx'> #endif</div><div class='ctx'> 	}</div><div class='del'>-	daddr = cma_dst_addr(id_priv);</div><div class='del'>-	daddr-&gt;sa_family = addr-&gt;sa_family;</div><div class='add'>+	id_daddr = cma_dst_addr(id_priv);</div><div class='add'>+	if (daddr != id_daddr)</div><div class='add'>+		memcpy(id_daddr, daddr, rdma_addr_size(addr));</div><div class='add'>+	id_daddr-&gt;sa_family = addr-&gt;sa_family;</div><div class='ctx'> </div><div class='ctx'> 	ret = cma_get_port(id_priv);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err2;</div><div class='ctx'> </div><div class='add'>+	if (!cma_any_addr(addr))</div><div class='add'>+		rdma_restrack_add(&amp;id_priv-&gt;res);</div><div class='ctx'> 	return 0;</div><div class='ctx'> err2:</div><div class='del'>-	rdma_restrack_del(&amp;id_priv-&gt;res);</div><div class='ctx'> 	if (id_priv-&gt;cma_dev)</div><div class='ctx'> 		cma_release_dev(id_priv);</div><div class='ctx'> err1:</div><div class='ctx'> 	cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='add'>+</div><div class='add'>+static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,</div><div class='add'>+			 const struct sockaddr *dst_addr)</div><div class='add'>+{</div><div class='add'>+	struct rdma_id_private *id_priv =</div><div class='add'>+		container_of(id, struct rdma_id_private, id);</div><div class='add'>+	struct sockaddr_storage zero_sock = {};</div><div class='add'>+</div><div class='add'>+	if (src_addr &amp;&amp; src_addr-&gt;sa_family)</div><div class='add'>+		return rdma_bind_addr_dst(id_priv, src_addr, dst_addr);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * When the src_addr is not specified, automatically supply an any addr</div><div class='add'>+	 */</div><div class='add'>+	zero_sock.ss_family = dst_addr-&gt;sa_family;</div><div class='add'>+	if (IS_ENABLED(CONFIG_IPV6) &amp;&amp; dst_addr-&gt;sa_family == AF_INET6) {</div><div class='add'>+		struct sockaddr_in6 *src_addr6 =</div><div class='add'>+			(struct sockaddr_in6 *)&amp;zero_sock;</div><div class='add'>+		struct sockaddr_in6 *dst_addr6 =</div><div class='add'>+			(struct sockaddr_in6 *)dst_addr;</div><div class='add'>+</div><div class='add'>+		src_addr6-&gt;sin6_scope_id = dst_addr6-&gt;sin6_scope_id;</div><div class='add'>+		if (ipv6_addr_type(&amp;dst_addr6-&gt;sin6_addr) &amp; IPV6_ADDR_LINKLOCAL)</div><div class='add'>+			id-&gt;route.addr.dev_addr.bound_dev_if =</div><div class='add'>+				dst_addr6-&gt;sin6_scope_id;</div><div class='add'>+	} else if (dst_addr-&gt;sa_family == AF_IB) {</div><div class='add'>+		((struct sockaddr_ib *)&amp;zero_sock)-&gt;sib_pkey =</div><div class='add'>+			((struct sockaddr_ib *)dst_addr)-&gt;sib_pkey;</div><div class='add'>+	}</div><div class='add'>+	return rdma_bind_addr_dst(id_priv, (struct sockaddr *)&amp;zero_sock, dst_addr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * If required, resolve the source address for bind and leave the id_priv in</div><div class='add'>+ * state RDMA_CM_ADDR_BOUND. This oddly uses the state to determine the prior</div><div class='add'>+ * calls made by ULP, a previously bound ID will not be re-bound and src_addr is</div><div class='add'>+ * ignored.</div><div class='add'>+ */</div><div class='add'>+static int resolve_prepare_src(struct rdma_id_private *id_priv,</div><div class='add'>+			       struct sockaddr *src_addr,</div><div class='add'>+			       const struct sockaddr *dst_addr)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) {</div><div class='add'>+		/* For a well behaved ULP state will be RDMA_CM_IDLE */</div><div class='add'>+		ret = cma_bind_addr(&amp;id_priv-&gt;id, src_addr, dst_addr);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+		if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND,</div><div class='add'>+					   RDMA_CM_ADDR_QUERY)))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+	} else {</div><div class='add'>+		memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (cma_family(id_priv) != dst_addr-&gt;sa_family) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err_state;</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_state:</div><div class='add'>+	cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,</div><div class='add'>+		      const struct sockaddr *dst_addr, unsigned long timeout_ms)</div><div class='add'>+{</div><div class='add'>+	struct rdma_id_private *id_priv =</div><div class='add'>+		container_of(id, struct rdma_id_private, id);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = resolve_prepare_src(id_priv, src_addr, dst_addr);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (cma_any_addr(dst_addr)) {</div><div class='add'>+		ret = cma_resolve_loopback(id_priv);</div><div class='add'>+	} else {</div><div class='add'>+		if (dst_addr-&gt;sa_family == AF_IB) {</div><div class='add'>+			ret = cma_resolve_ib_addr(id_priv);</div><div class='add'>+		} else {</div><div class='add'>+			/*</div><div class='add'>+			 * The FSM can return back to RDMA_CM_ADDR_BOUND after</div><div class='add'>+			 * rdma_resolve_ip() is called, eg through the error</div><div class='add'>+			 * path in addr_handler(). If this happens the existing</div><div class='add'>+			 * request must be canceled before issuing a new one.</div><div class='add'>+			 * Since canceling a request is a bit slow and this</div><div class='add'>+			 * oddball path is rare, keep track once a request has</div><div class='add'>+			 * been issued. The track turns out to be a permanent</div><div class='add'>+			 * state since this is the only cancel as it is</div><div class='add'>+			 * immediately before rdma_resolve_ip().</div><div class='add'>+			 */</div><div class='add'>+			if (id_priv-&gt;used_resolve_ip)</div><div class='add'>+				rdma_addr_cancel(&amp;id-&gt;route.addr.dev_addr);</div><div class='add'>+			else</div><div class='add'>+				id_priv-&gt;used_resolve_ip = 1;</div><div class='add'>+			ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr,</div><div class='add'>+					      &amp;id-&gt;route.addr.dev_addr,</div><div class='add'>+					      timeout_ms, addr_handler,</div><div class='add'>+					      false, id_priv);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+err:</div><div class='add'>+	cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_resolve_addr);</div><div class='add'>+</div><div class='add'>+int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)</div><div class='add'>+{</div><div class='add'>+	struct rdma_id_private *id_priv =</div><div class='add'>+		container_of(id, struct rdma_id_private, id);</div><div class='add'>+</div><div class='add'>+	return rdma_bind_addr_dst(id_priv, addr, cma_dst_addr(id_priv));</div><div class='add'>+}</div><div class='ctx'> EXPORT_SYMBOL(rdma_bind_addr);</div><div class='ctx'> </div><div class='ctx'> static int cma_format_hdr(void *hdr, struct rdma_id_private *id_priv)</div><div class='hunk'>@@ -3664,10 +4220,10 @@ static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,</div><div class='ctx'> 	struct rdma_cm_event event = {};</div><div class='ctx'> 	const struct ib_cm_sidr_rep_event_param *rep =</div><div class='ctx'> 				&amp;ib_event-&gt;param.sidr_rep_rcvd;</div><div class='del'>-	int ret = 0;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-	if (id_priv-&gt;state != RDMA_CM_CONNECT)</div><div class='add'>+	if (READ_ONCE(id_priv-&gt;state) != RDMA_CM_CONNECT)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='ctx'> 	switch (ib_event-&gt;event) {</div><div class='hunk'>@@ -3708,20 +4264,18 @@ static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = id_priv-&gt;id.event_handler(&amp;id_priv-&gt;id, &amp;event);</div><div class='add'>+	ret = cma_cm_event_handler(id_priv, &amp;event);</div><div class='ctx'> </div><div class='ctx'> 	rdma_destroy_ah_attr(&amp;event.param.ud.ah_attr);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		/* Destroy the CM ID by returning a non-zero value. */</div><div class='ctx'> 		id_priv-&gt;cm_id.ib = NULL;</div><div class='del'>-		cma_exch(id_priv, RDMA_CM_DESTROYING);</div><div class='del'>-		mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-		rdma_destroy_id(&amp;id_priv-&gt;id);</div><div class='add'>+		destroy_id_handler_unlock(id_priv);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> out:</div><div class='ctx'> 	mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-	return ret;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,</div><div class='hunk'>@@ -3735,8 +4289,7 @@ static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,</div><div class='ctx'> </div><div class='ctx'> 	memset(&amp;req, 0, sizeof req);</div><div class='ctx'> 	offset = cma_user_data_offset(id_priv);</div><div class='del'>-	req.private_data_len = offset + conn_param-&gt;private_data_len;</div><div class='del'>-	if (req.private_data_len &lt; conn_param-&gt;private_data_len)</div><div class='add'>+	if (check_add_overflow(offset, conn_param-&gt;private_data_len, &amp;req.private_data_len))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	if (req.private_data_len) {</div><div class='hunk'>@@ -3772,6 +4325,7 @@ static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,</div><div class='ctx'> 	req.timeout_ms = 1 &lt;&lt; (CMA_CM_RESPONSE_TIMEOUT - 8);</div><div class='ctx'> 	req.max_cm_retries = CMA_MAX_CM_RETRIES;</div><div class='ctx'> </div><div class='add'>+	trace_cm_send_sidr_req(id_priv);</div><div class='ctx'> 	ret = ib_send_cm_sidr_req(id_priv-&gt;cm_id.ib, &amp;req);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		ib_destroy_cm_id(id_priv-&gt;cm_id.ib);</div><div class='hunk'>@@ -3794,8 +4348,7 @@ static int cma_connect_ib(struct rdma_id_private *id_priv,</div><div class='ctx'> </div><div class='ctx'> 	memset(&amp;req, 0, sizeof req);</div><div class='ctx'> 	offset = cma_user_data_offset(id_priv);</div><div class='del'>-	req.private_data_len = offset + conn_param-&gt;private_data_len;</div><div class='del'>-	if (req.private_data_len &lt; conn_param-&gt;private_data_len)</div><div class='add'>+	if (check_add_overflow(offset, conn_param-&gt;private_data_len, &amp;req.private_data_len))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	if (req.private_data_len) {</div><div class='hunk'>@@ -3826,7 +4379,9 @@ static int cma_connect_ib(struct rdma_id_private *id_priv,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	req.primary_path = &amp;route-&gt;path_rec[0];</div><div class='del'>-	if (route-&gt;num_paths == 2)</div><div class='add'>+	req.primary_path_inbound = route-&gt;path_rec_inbound;</div><div class='add'>+	req.primary_path_outbound = route-&gt;path_rec_outbound;</div><div class='add'>+	if (route-&gt;num_pri_alt_paths == 2)</div><div class='ctx'> 		req.alternate_path = &amp;route-&gt;path_rec[1];</div><div class='ctx'> </div><div class='ctx'> 	req.ppath_sgid_attr = id_priv-&gt;id.route.addr.dev_addr.sgid_attr;</div><div class='hunk'>@@ -3844,7 +4399,10 @@ static int cma_connect_ib(struct rdma_id_private *id_priv,</div><div class='ctx'> 	req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;</div><div class='ctx'> 	req.max_cm_retries = CMA_MAX_CM_RETRIES;</div><div class='ctx'> 	req.srq = id_priv-&gt;srq ? 1 : 0;</div><div class='add'>+	req.ece.vendor_id = id_priv-&gt;ece.vendor_id;</div><div class='add'>+	req.ece.attr_mod = id_priv-&gt;ece.attr_mod;</div><div class='ctx'> </div><div class='add'>+	trace_cm_send_req(id_priv);</div><div class='ctx'> 	ret = ib_send_cm_req(id_priv-&gt;cm_id.ib, &amp;req);</div><div class='ctx'> out:</div><div class='ctx'> 	if (ret &amp;&amp; !IS_ERR(id)) {</div><div class='hunk'>@@ -3867,8 +4425,11 @@ static int cma_connect_iw(struct rdma_id_private *id_priv,</div><div class='ctx'> 	if (IS_ERR(cm_id))</div><div class='ctx'> 		return PTR_ERR(cm_id);</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;id_priv-&gt;qp_mutex);</div><div class='ctx'> 	cm_id-&gt;tos = id_priv-&gt;tos;</div><div class='ctx'> 	cm_id-&gt;tos_set = id_priv-&gt;tos_set;</div><div class='add'>+	mutex_unlock(&amp;id_priv-&gt;qp_mutex);</div><div class='add'>+</div><div class='ctx'> 	id_priv-&gt;cm_id.iw = cm_id;</div><div class='ctx'> </div><div class='ctx'> 	memcpy(&amp;cm_id-&gt;local_addr, cma_src_addr(id_priv),</div><div class='hunk'>@@ -3899,12 +4460,23 @@ out:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)</div><div class='add'>+/**</div><div class='add'>+ * rdma_connect_locked - Initiate an active connection request.</div><div class='add'>+ * @id: Connection identifier to connect.</div><div class='add'>+ * @conn_param: Connection information used for connected QPs.</div><div class='add'>+ *</div><div class='add'>+ * Same as rdma_connect() but can only be called from the</div><div class='add'>+ * RDMA_CM_EVENT_ROUTE_RESOLVED handler callback.</div><div class='add'>+ */</div><div class='add'>+int rdma_connect_locked(struct rdma_cm_id *id,</div><div class='add'>+			struct rdma_conn_param *conn_param)</div><div class='ctx'> {</div><div class='del'>-	struct rdma_id_private *id_priv;</div><div class='add'>+	struct rdma_id_private *id_priv =</div><div class='add'>+		container_of(id, struct rdma_id_private, id);</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='add'>+	lockdep_assert_held(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+</div><div class='ctx'> 	if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='hunk'>@@ -3918,20 +4490,66 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)</div><div class='ctx'> 			ret = cma_resolve_ib_udp(id_priv, conn_param);</div><div class='ctx'> 		else</div><div class='ctx'> 			ret = cma_connect_ib(id_priv, conn_param);</div><div class='del'>-	} else if (rdma_cap_iw_cm(id-&gt;device, id-&gt;port_num))</div><div class='add'>+	} else if (rdma_cap_iw_cm(id-&gt;device, id-&gt;port_num)) {</div><div class='ctx'> 		ret = cma_connect_iw(id_priv, conn_param);</div><div class='del'>-	else</div><div class='add'>+	} else {</div><div class='ctx'> 		ret = -ENOSYS;</div><div class='add'>+	}</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err;</div><div class='del'>-</div><div class='add'>+		goto err_state;</div><div class='ctx'> 	return 0;</div><div class='del'>-err:</div><div class='add'>+err_state:</div><div class='ctx'> 	cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='add'>+EXPORT_SYMBOL(rdma_connect_locked);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * rdma_connect - Initiate an active connection request.</div><div class='add'>+ * @id: Connection identifier to connect.</div><div class='add'>+ * @conn_param: Connection information used for connected QPs.</div><div class='add'>+ *</div><div class='add'>+ * Users must have resolved a route for the rdma_cm_id to connect with by having</div><div class='add'>+ * called rdma_resolve_route before calling this routine.</div><div class='add'>+ *</div><div class='add'>+ * This call will either connect to a remote QP or obtain remote QP information</div><div class='add'>+ * for unconnected rdma_cm_id's.  The actual operation is based on the</div><div class='add'>+ * rdma_cm_id's port space.</div><div class='add'>+ */</div><div class='add'>+int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)</div><div class='add'>+{</div><div class='add'>+	struct rdma_id_private *id_priv =</div><div class='add'>+		container_of(id, struct rdma_id_private, id);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+	ret = rdma_connect_locked(id, conn_param);</div><div class='add'>+	mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> EXPORT_SYMBOL(rdma_connect);</div><div class='ctx'> </div><div class='add'>+/**</div><div class='add'>+ * rdma_connect_ece - Initiate an active connection request with ECE data.</div><div class='add'>+ * @id: Connection identifier to connect.</div><div class='add'>+ * @conn_param: Connection information used for connected QPs.</div><div class='add'>+ * @ece: ECE parameters</div><div class='add'>+ *</div><div class='add'>+ * See rdma_connect() explanation.</div><div class='add'>+ */</div><div class='add'>+int rdma_connect_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,</div><div class='add'>+		     struct rdma_ucm_ece *ece)</div><div class='add'>+{</div><div class='add'>+	struct rdma_id_private *id_priv =</div><div class='add'>+		container_of(id, struct rdma_id_private, id);</div><div class='add'>+</div><div class='add'>+	id_priv-&gt;ece.vendor_id = ece-&gt;vendor_id;</div><div class='add'>+	id_priv-&gt;ece.attr_mod = ece-&gt;attr_mod;</div><div class='add'>+</div><div class='add'>+	return rdma_connect(id, conn_param);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_connect_ece);</div><div class='add'>+</div><div class='ctx'> static int cma_accept_ib(struct rdma_id_private *id_priv,</div><div class='ctx'> 			 struct rdma_conn_param *conn_param)</div><div class='ctx'> {</div><div class='hunk'>@@ -3957,7 +4575,10 @@ static int cma_accept_ib(struct rdma_id_private *id_priv,</div><div class='ctx'> 	rep.flow_control = conn_param-&gt;flow_control;</div><div class='ctx'> 	rep.rnr_retry_count = min_t(u8, 7, conn_param-&gt;rnr_retry_count);</div><div class='ctx'> 	rep.srq = id_priv-&gt;srq ? 1 : 0;</div><div class='add'>+	rep.ece.vendor_id = id_priv-&gt;ece.vendor_id;</div><div class='add'>+	rep.ece.attr_mod = id_priv-&gt;ece.attr_mod;</div><div class='ctx'> </div><div class='add'>+	trace_cm_send_rep(id_priv);</div><div class='ctx'> 	ret = ib_send_cm_rep(id_priv-&gt;cm_id.ib, &amp;rep);</div><div class='ctx'> out:</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -3980,9 +4601,9 @@ static int cma_accept_iw(struct rdma_id_private *id_priv,</div><div class='ctx'> 	iw_param.ird = conn_param-&gt;responder_resources;</div><div class='ctx'> 	iw_param.private_data = conn_param-&gt;private_data;</div><div class='ctx'> 	iw_param.private_data_len = conn_param-&gt;private_data_len;</div><div class='del'>-	if (id_priv-&gt;id.qp) {</div><div class='add'>+	if (id_priv-&gt;id.qp)</div><div class='ctx'> 		iw_param.qpn = id_priv-&gt;qp_num;</div><div class='del'>-	} else</div><div class='add'>+	else</div><div class='ctx'> 		iw_param.qpn = conn_param-&gt;qp_num;</div><div class='ctx'> </div><div class='ctx'> 	return iw_cm_accept(id_priv-&gt;cm_id.iw, &amp;iw_param);</div><div class='hunk'>@@ -3998,29 +4619,53 @@ static int cma_send_sidr_rep(struct rdma_id_private *id_priv,</div><div class='ctx'> 	memset(&amp;rep, 0, sizeof rep);</div><div class='ctx'> 	rep.status = status;</div><div class='ctx'> 	if (status == IB_SIDR_SUCCESS) {</div><div class='del'>-		ret = cma_set_qkey(id_priv, qkey);</div><div class='add'>+		if (qkey)</div><div class='add'>+			ret = cma_set_qkey(id_priv, qkey);</div><div class='add'>+		else</div><div class='add'>+			ret = cma_set_default_qkey(id_priv);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			return ret;</div><div class='ctx'> 		rep.qp_num = id_priv-&gt;qp_num;</div><div class='ctx'> 		rep.qkey = id_priv-&gt;qkey;</div><div class='add'>+</div><div class='add'>+		rep.ece.vendor_id = id_priv-&gt;ece.vendor_id;</div><div class='add'>+		rep.ece.attr_mod = id_priv-&gt;ece.attr_mod;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='ctx'> 	rep.private_data = private_data;</div><div class='ctx'> 	rep.private_data_len = private_data_len;</div><div class='ctx'> </div><div class='add'>+	trace_cm_send_sidr_rep(id_priv);</div><div class='ctx'> 	return ib_send_cm_sidr_rep(id_priv-&gt;cm_id.ib, &amp;rep);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int __rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,</div><div class='del'>-		  const char *caller)</div><div class='add'>+/**</div><div class='add'>+ * rdma_accept - Called to accept a connection request or response.</div><div class='add'>+ * @id: Connection identifier associated with the request.</div><div class='add'>+ * @conn_param: Information needed to establish the connection.  This must be</div><div class='add'>+ *   provided if accepting a connection request.  If accepting a connection</div><div class='add'>+ *   response, this parameter must be NULL.</div><div class='add'>+ *</div><div class='add'>+ * Typically, this routine is only called by the listener to accept a connection</div><div class='add'>+ * request.  It must also be called on the active side of a connection if the</div><div class='add'>+ * user is performing their own QP transitions.</div><div class='add'>+ *</div><div class='add'>+ * In the case of error, a reject message is sent to the remote side and the</div><div class='add'>+ * state of the qp associated with the id is modified to error, such that any</div><div class='add'>+ * previously posted receive buffers would be flushed.</div><div class='add'>+ *</div><div class='add'>+ * This function is for use by kernel ULPs and must be called from under the</div><div class='add'>+ * handler callback.</div><div class='add'>+ */</div><div class='add'>+int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)</div><div class='ctx'> {</div><div class='del'>-	struct rdma_id_private *id_priv;</div><div class='add'>+	struct rdma_id_private *id_priv =</div><div class='add'>+		container_of(id, struct rdma_id_private, id);</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='del'>-</div><div class='del'>-	rdma_restrack_set_task(&amp;id_priv-&gt;res, caller);</div><div class='add'>+	lockdep_assert_held(&amp;id_priv-&gt;handler_mutex);</div><div class='ctx'> </div><div class='del'>-	if (!cma_comp(id_priv, RDMA_CM_CONNECT))</div><div class='add'>+	if (READ_ONCE(id_priv-&gt;state) != RDMA_CM_CONNECT)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	if (!id-&gt;qp &amp;&amp; conn_param) {</div><div class='hunk'>@@ -4044,21 +4689,52 @@ int __rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,</div><div class='ctx'> 			else</div><div class='ctx'> 				ret = cma_rep_recv(id_priv);</div><div class='ctx'> 		}</div><div class='del'>-	} else if (rdma_cap_iw_cm(id-&gt;device, id-&gt;port_num))</div><div class='add'>+	} else if (rdma_cap_iw_cm(id-&gt;device, id-&gt;port_num)) {</div><div class='ctx'> 		ret = cma_accept_iw(id_priv, conn_param);</div><div class='del'>-	else</div><div class='add'>+	} else {</div><div class='ctx'> 		ret = -ENOSYS;</div><div class='del'>-</div><div class='add'>+	}</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto reject;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> reject:</div><div class='ctx'> 	cma_modify_qp_err(id_priv);</div><div class='del'>-	rdma_reject(id, NULL, 0);</div><div class='add'>+	rdma_reject(id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(__rdma_accept);</div><div class='add'>+EXPORT_SYMBOL(rdma_accept);</div><div class='add'>+</div><div class='add'>+int rdma_accept_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,</div><div class='add'>+		    struct rdma_ucm_ece *ece)</div><div class='add'>+{</div><div class='add'>+	struct rdma_id_private *id_priv =</div><div class='add'>+		container_of(id, struct rdma_id_private, id);</div><div class='add'>+</div><div class='add'>+	id_priv-&gt;ece.vendor_id = ece-&gt;vendor_id;</div><div class='add'>+	id_priv-&gt;ece.attr_mod = ece-&gt;attr_mod;</div><div class='add'>+</div><div class='add'>+	return rdma_accept(id, conn_param);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_accept_ece);</div><div class='add'>+</div><div class='add'>+void rdma_lock_handler(struct rdma_cm_id *id)</div><div class='add'>+{</div><div class='add'>+	struct rdma_id_private *id_priv =</div><div class='add'>+		container_of(id, struct rdma_id_private, id);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_lock_handler);</div><div class='add'>+</div><div class='add'>+void rdma_unlock_handler(struct rdma_cm_id *id)</div><div class='add'>+{</div><div class='add'>+	struct rdma_id_private *id_priv =</div><div class='add'>+		container_of(id, struct rdma_id_private, id);</div><div class='add'>+</div><div class='add'>+	mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_unlock_handler);</div><div class='ctx'> </div><div class='ctx'> int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)</div><div class='ctx'> {</div><div class='hunk'>@@ -4082,7 +4758,7 @@ int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)</div><div class='ctx'> EXPORT_SYMBOL(rdma_notify);</div><div class='ctx'> </div><div class='ctx'> int rdma_reject(struct rdma_cm_id *id, const void *private_data,</div><div class='del'>-		u8 private_data_len)</div><div class='add'>+		u8 private_data_len, u8 reason)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_id_private *id_priv;</div><div class='ctx'> 	int ret;</div><div class='hunk'>@@ -4092,18 +4768,20 @@ int rdma_reject(struct rdma_cm_id *id, const void *private_data,</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	if (rdma_cap_ib_cm(id-&gt;device, id-&gt;port_num)) {</div><div class='del'>-		if (id-&gt;qp_type == IB_QPT_UD)</div><div class='add'>+		if (id-&gt;qp_type == IB_QPT_UD) {</div><div class='ctx'> 			ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 0,</div><div class='ctx'> 						private_data, private_data_len);</div><div class='del'>-		else</div><div class='del'>-			ret = ib_send_cm_rej(id_priv-&gt;cm_id.ib,</div><div class='del'>-					     IB_CM_REJ_CONSUMER_DEFINED, NULL,</div><div class='del'>-					     0, private_data, private_data_len);</div><div class='add'>+		} else {</div><div class='add'>+			trace_cm_send_rej(id_priv);</div><div class='add'>+			ret = ib_send_cm_rej(id_priv-&gt;cm_id.ib, reason, NULL, 0,</div><div class='add'>+					     private_data, private_data_len);</div><div class='add'>+		}</div><div class='ctx'> 	} else if (rdma_cap_iw_cm(id-&gt;device, id-&gt;port_num)) {</div><div class='ctx'> 		ret = iw_cm_reject(id_priv-&gt;cm_id.iw,</div><div class='ctx'> 				   private_data, private_data_len);</div><div class='del'>-	} else</div><div class='add'>+	} else {</div><div class='ctx'> 		ret = -ENOSYS;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -4123,8 +4801,13 @@ int rdma_disconnect(struct rdma_cm_id *id)</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		/* Initiate or respond to a disconnect. */</div><div class='del'>-		if (ib_send_cm_dreq(id_priv-&gt;cm_id.ib, NULL, 0))</div><div class='del'>-			ib_send_cm_drep(id_priv-&gt;cm_id.ib, NULL, 0);</div><div class='add'>+		trace_cm_disconnect(id_priv);</div><div class='add'>+		if (ib_send_cm_dreq(id_priv-&gt;cm_id.ib, NULL, 0)) {</div><div class='add'>+			if (!ib_send_cm_drep(id_priv-&gt;cm_id.ib, NULL, 0))</div><div class='add'>+				trace_cm_sent_drep(id_priv);</div><div class='add'>+		} else {</div><div class='add'>+			trace_cm_sent_dreq(id_priv);</div><div class='add'>+		}</div><div class='ctx'> 	} else if (rdma_cap_iw_cm(id-&gt;device, id-&gt;port_num)) {</div><div class='ctx'> 		ret = iw_cm_disconnect(id_priv-&gt;cm_id.iw, 0);</div><div class='ctx'> 	} else</div><div class='hunk'>@@ -4135,70 +4818,68 @@ out:</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_disconnect);</div><div class='ctx'> </div><div class='add'>+static void cma_make_mc_event(int status, struct rdma_id_private *id_priv,</div><div class='add'>+			      struct ib_sa_multicast *multicast,</div><div class='add'>+			      struct rdma_cm_event *event,</div><div class='add'>+			      struct cma_multicast *mc)</div><div class='add'>+{</div><div class='add'>+	struct rdma_dev_addr *dev_addr;</div><div class='add'>+	enum ib_gid_type gid_type;</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+</div><div class='add'>+	if (status)</div><div class='add'>+		pr_debug_ratelimited("RDMA CM: MULTICAST_ERROR: failed to join multicast. status %d\n",</div><div class='add'>+				     status);</div><div class='add'>+</div><div class='add'>+	event-&gt;status = status;</div><div class='add'>+	event-&gt;param.ud.private_data = mc-&gt;context;</div><div class='add'>+	if (status) {</div><div class='add'>+		event-&gt;event = RDMA_CM_EVENT_MULTICAST_ERROR;</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dev_addr = &amp;id_priv-&gt;id.route.addr.dev_addr;</div><div class='add'>+	ndev = dev_get_by_index(dev_addr-&gt;net, dev_addr-&gt;bound_dev_if);</div><div class='add'>+	gid_type =</div><div class='add'>+		id_priv-&gt;cma_dev</div><div class='add'>+			-&gt;default_gid_type[id_priv-&gt;id.port_num -</div><div class='add'>+					   rdma_start_port(</div><div class='add'>+						   id_priv-&gt;cma_dev-&gt;device)];</div><div class='add'>+</div><div class='add'>+	event-&gt;event = RDMA_CM_EVENT_MULTICAST_JOIN;</div><div class='add'>+	if (ib_init_ah_from_mcmember(id_priv-&gt;id.device, id_priv-&gt;id.port_num,</div><div class='add'>+				     &amp;multicast-&gt;rec, ndev, gid_type,</div><div class='add'>+				     &amp;event-&gt;param.ud.ah_attr)) {</div><div class='add'>+		event-&gt;event = RDMA_CM_EVENT_MULTICAST_ERROR;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	event-&gt;param.ud.qp_num = 0xFFFFFF;</div><div class='add'>+	event-&gt;param.ud.qkey = id_priv-&gt;qkey;</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	dev_put(ndev);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)</div><div class='ctx'> {</div><div class='del'>-	struct rdma_id_private *id_priv;</div><div class='ctx'> 	struct cma_multicast *mc = multicast-&gt;context;</div><div class='add'>+	struct rdma_id_private *id_priv = mc-&gt;id_priv;</div><div class='ctx'> 	struct rdma_cm_event event = {};</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	id_priv = mc-&gt;id_priv;</div><div class='ctx'> 	mutex_lock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-	if (id_priv-&gt;state != RDMA_CM_ADDR_BOUND &amp;&amp;</div><div class='del'>-	    id_priv-&gt;state != RDMA_CM_ADDR_RESOLVED)</div><div class='add'>+	if (READ_ONCE(id_priv-&gt;state) == RDMA_CM_DEVICE_REMOVAL ||</div><div class='add'>+	    READ_ONCE(id_priv-&gt;state) == RDMA_CM_DESTROYING)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	if (!status)</div><div class='del'>-		status = cma_set_qkey(id_priv, be32_to_cpu(multicast-&gt;rec.qkey));</div><div class='del'>-	else</div><div class='del'>-		pr_debug_ratelimited("RDMA CM: MULTICAST_ERROR: failed to join multicast. status %d\n",</div><div class='del'>-				     status);</div><div class='del'>-	mutex_lock(&amp;id_priv-&gt;qp_mutex);</div><div class='del'>-	if (!status &amp;&amp; id_priv-&gt;id.qp) {</div><div class='del'>-		status = ib_attach_mcast(id_priv-&gt;id.qp, &amp;multicast-&gt;rec.mgid,</div><div class='del'>-					 be16_to_cpu(multicast-&gt;rec.mlid));</div><div class='del'>-		if (status)</div><div class='del'>-			pr_debug_ratelimited("RDMA CM: MULTICAST_ERROR: failed to attach QP. status %d\n",</div><div class='del'>-					     status);</div><div class='add'>+	ret = cma_set_qkey(id_priv, be32_to_cpu(multicast-&gt;rec.qkey));</div><div class='add'>+	if (!ret) {</div><div class='add'>+		cma_make_mc_event(status, id_priv, multicast, &amp;event, mc);</div><div class='add'>+		ret = cma_cm_event_handler(id_priv, &amp;event);</div><div class='ctx'> 	}</div><div class='del'>-	mutex_unlock(&amp;id_priv-&gt;qp_mutex);</div><div class='del'>-</div><div class='del'>-	event.status = status;</div><div class='del'>-	event.param.ud.private_data = mc-&gt;context;</div><div class='del'>-	if (!status) {</div><div class='del'>-		struct rdma_dev_addr *dev_addr =</div><div class='del'>-			&amp;id_priv-&gt;id.route.addr.dev_addr;</div><div class='del'>-		struct net_device *ndev =</div><div class='del'>-			dev_get_by_index(dev_addr-&gt;net, dev_addr-&gt;bound_dev_if);</div><div class='del'>-		enum ib_gid_type gid_type =</div><div class='del'>-			id_priv-&gt;cma_dev-&gt;default_gid_type[id_priv-&gt;id.port_num -</div><div class='del'>-			rdma_start_port(id_priv-&gt;cma_dev-&gt;device)];</div><div class='del'>-</div><div class='del'>-		event.event = RDMA_CM_EVENT_MULTICAST_JOIN;</div><div class='del'>-		ret = ib_init_ah_from_mcmember(id_priv-&gt;id.device,</div><div class='del'>-					       id_priv-&gt;id.port_num,</div><div class='del'>-					       &amp;multicast-&gt;rec,</div><div class='del'>-					       ndev, gid_type,</div><div class='del'>-					       &amp;event.param.ud.ah_attr);</div><div class='del'>-		if (ret)</div><div class='del'>-			event.event = RDMA_CM_EVENT_MULTICAST_ERROR;</div><div class='del'>-</div><div class='del'>-		event.param.ud.qp_num = 0xFFFFFF;</div><div class='del'>-		event.param.ud.qkey = be32_to_cpu(multicast-&gt;rec.qkey);</div><div class='del'>-		if (ndev)</div><div class='del'>-			dev_put(ndev);</div><div class='del'>-	} else</div><div class='del'>-		event.event = RDMA_CM_EVENT_MULTICAST_ERROR;</div><div class='del'>-</div><div class='del'>-	ret = id_priv-&gt;id.event_handler(&amp;id_priv-&gt;id, &amp;event);</div><div class='del'>-</div><div class='ctx'> 	rdma_destroy_ah_attr(&amp;event.param.ud.ah_attr);</div><div class='del'>-	if (ret) {</div><div class='del'>-		cma_exch(id_priv, RDMA_CM_DESTROYING);</div><div class='del'>-		mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-		rdma_destroy_id(&amp;id_priv-&gt;id);</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='add'>+	WARN_ON(ret);</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='ctx'> 	mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='hunk'>@@ -4249,9 +4930,11 @@ static int cma_join_ib_multicast(struct rdma_id_private *id_priv,</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	ret = cma_set_qkey(id_priv, 0);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='add'>+	if (!id_priv-&gt;qkey) {</div><div class='add'>+		ret = cma_set_default_qkey(id_priv);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	cma_set_mgid(id_priv, (struct sockaddr *) &amp;mc-&gt;addr, &amp;rec.mgid);</div><div class='ctx'> 	rec.qkey = cpu_to_be32(id_priv-&gt;qkey);</div><div class='hunk'>@@ -4259,17 +4942,6 @@ static int cma_join_ib_multicast(struct rdma_id_private *id_priv,</div><div class='ctx'> 	rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));</div><div class='ctx'> 	rec.join_state = mc-&gt;join_state;</div><div class='ctx'> </div><div class='del'>-	if ((rec.join_state == BIT(SENDONLY_FULLMEMBER_JOIN)) &amp;&amp;</div><div class='del'>-	    (!ib_sa_sendonly_fullmem_support(&amp;sa_client,</div><div class='del'>-					     id_priv-&gt;id.device,</div><div class='del'>-					     id_priv-&gt;id.port_num))) {</div><div class='del'>-		dev_warn(</div><div class='del'>-			&amp;id_priv-&gt;id.device-&gt;dev,</div><div class='del'>-			"RDMA CM: port %u Unable to multicast join: SM doesn't support Send Only Full Member option\n",</div><div class='del'>-			id_priv-&gt;id.port_num);</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |</div><div class='ctx'> 		    IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |</div><div class='ctx'> 		    IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |</div><div class='hunk'>@@ -4283,23 +4955,10 @@ static int cma_join_ib_multicast(struct rdma_id_private *id_priv,</div><div class='ctx'> 			     IB_SA_MCMEMBER_REC_MTU |</div><div class='ctx'> 			     IB_SA_MCMEMBER_REC_HOP_LIMIT;</div><div class='ctx'> </div><div class='del'>-	mc-&gt;multicast.ib = ib_sa_join_multicast(&amp;sa_client, id_priv-&gt;id.device,</div><div class='del'>-						id_priv-&gt;id.port_num, &amp;rec,</div><div class='del'>-						comp_mask, GFP_KERNEL,</div><div class='del'>-						cma_ib_mc_handler, mc);</div><div class='del'>-	return PTR_ERR_OR_ZERO(mc-&gt;multicast.ib);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void iboe_mcast_work_handler(struct work_struct *work)</div><div class='del'>-{</div><div class='del'>-	struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work);</div><div class='del'>-	struct cma_multicast *mc = mw-&gt;mc;</div><div class='del'>-	struct ib_sa_multicast *m = mc-&gt;multicast.ib;</div><div class='del'>-</div><div class='del'>-	mc-&gt;multicast.ib-&gt;context = mc;</div><div class='del'>-	cma_ib_mc_handler(0, m);</div><div class='del'>-	kref_put(&amp;mc-&gt;mcref, release_mc);</div><div class='del'>-	kfree(mw);</div><div class='add'>+	mc-&gt;sa_mc = ib_sa_join_multicast(&amp;sa_client, id_priv-&gt;id.device,</div><div class='add'>+					 id_priv-&gt;id.port_num, &amp;rec, comp_mask,</div><div class='add'>+					 GFP_KERNEL, cma_ib_mc_handler, mc);</div><div class='add'>+	return PTR_ERR_OR_ZERO(mc-&gt;sa_mc);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid,</div><div class='hunk'>@@ -4334,52 +4993,38 @@ static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid,</div><div class='ctx'> static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,</div><div class='ctx'> 				   struct cma_multicast *mc)</div><div class='ctx'> {</div><div class='del'>-	struct iboe_mcast_work *work;</div><div class='ctx'> 	struct rdma_dev_addr *dev_addr = &amp;id_priv-&gt;id.route.addr.dev_addr;</div><div class='ctx'> 	int err = 0;</div><div class='ctx'> 	struct sockaddr *addr = (struct sockaddr *)&amp;mc-&gt;addr;</div><div class='ctx'> 	struct net_device *ndev = NULL;</div><div class='add'>+	struct ib_sa_multicast ib = {};</div><div class='ctx'> 	enum ib_gid_type gid_type;</div><div class='ctx'> 	bool send_only;</div><div class='ctx'> </div><div class='ctx'> 	send_only = mc-&gt;join_state == BIT(SENDONLY_FULLMEMBER_JOIN);</div><div class='ctx'> </div><div class='del'>-	if (cma_zero_addr((struct sockaddr *)&amp;mc-&gt;addr))</div><div class='add'>+	if (cma_zero_addr(addr))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	work = kzalloc(sizeof *work, GFP_KERNEL);</div><div class='del'>-	if (!work)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	mc-&gt;multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL);</div><div class='del'>-	if (!mc-&gt;multicast.ib) {</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto out1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	gid_type = id_priv-&gt;cma_dev-&gt;default_gid_type[id_priv-&gt;id.port_num -</div><div class='ctx'> 		   rdma_start_port(id_priv-&gt;cma_dev-&gt;device)];</div><div class='del'>-	cma_iboe_set_mgid(addr, &amp;mc-&gt;multicast.ib-&gt;rec.mgid, gid_type);</div><div class='del'>-</div><div class='del'>-	mc-&gt;multicast.ib-&gt;rec.pkey = cpu_to_be16(0xffff);</div><div class='del'>-	if (id_priv-&gt;id.ps == RDMA_PS_UDP)</div><div class='del'>-		mc-&gt;multicast.ib-&gt;rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);</div><div class='add'>+	cma_iboe_set_mgid(addr, &amp;ib.rec.mgid, gid_type);</div><div class='ctx'> </div><div class='add'>+	ib.rec.pkey = cpu_to_be16(0xffff);</div><div class='ctx'> 	if (dev_addr-&gt;bound_dev_if)</div><div class='ctx'> 		ndev = dev_get_by_index(dev_addr-&gt;net, dev_addr-&gt;bound_dev_if);</div><div class='del'>-	if (!ndev) {</div><div class='del'>-		err = -ENODEV;</div><div class='del'>-		goto out2;</div><div class='del'>-	}</div><div class='del'>-	mc-&gt;multicast.ib-&gt;rec.rate = iboe_get_rate(ndev);</div><div class='del'>-	mc-&gt;multicast.ib-&gt;rec.hop_limit = 1;</div><div class='del'>-	mc-&gt;multicast.ib-&gt;rec.mtu = iboe_get_mtu(ndev-&gt;mtu);</div><div class='add'>+	if (!ndev)</div><div class='add'>+		return -ENODEV;</div><div class='add'>+</div><div class='add'>+	ib.rec.rate = IB_RATE_PORT_CURRENT;</div><div class='add'>+	ib.rec.hop_limit = 1;</div><div class='add'>+	ib.rec.mtu = iboe_get_mtu(ndev-&gt;mtu);</div><div class='ctx'> </div><div class='ctx'> 	if (addr-&gt;sa_family == AF_INET) {</div><div class='ctx'> 		if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {</div><div class='del'>-			mc-&gt;multicast.ib-&gt;rec.hop_limit = IPV6_DEFAULT_HOPLIMIT;</div><div class='add'>+			ib.rec.hop_limit = IPV6_DEFAULT_HOPLIMIT;</div><div class='ctx'> 			if (!send_only) {</div><div class='del'>-				err = cma_igmp_send(ndev, &amp;mc-&gt;multicast.ib-&gt;rec.mgid,</div><div class='add'>+				err = cma_igmp_send(ndev, &amp;ib.rec.mgid,</div><div class='ctx'> 						    true);</div><div class='ctx'> 			}</div><div class='ctx'> 		}</div><div class='hunk'>@@ -4388,44 +5033,41 @@ static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,</div><div class='ctx'> 			err = -ENOTSUPP;</div><div class='ctx'> 	}</div><div class='ctx'> 	dev_put(ndev);</div><div class='del'>-	if (err || !mc-&gt;multicast.ib-&gt;rec.mtu) {</div><div class='del'>-		if (!err)</div><div class='del'>-			err = -EINVAL;</div><div class='del'>-		goto out2;</div><div class='del'>-	}</div><div class='del'>-	rdma_ip2gid((struct sockaddr *)&amp;id_priv-&gt;id.route.addr.src_addr,</div><div class='del'>-		    &amp;mc-&gt;multicast.ib-&gt;rec.port_gid);</div><div class='del'>-	work-&gt;id = id_priv;</div><div class='del'>-	work-&gt;mc = mc;</div><div class='del'>-	INIT_WORK(&amp;work-&gt;work, iboe_mcast_work_handler);</div><div class='del'>-	kref_get(&amp;mc-&gt;mcref);</div><div class='del'>-	queue_work(cma_wq, &amp;work-&gt;work);</div><div class='add'>+	if (err || !ib.rec.mtu)</div><div class='add'>+		return err ?: -EINVAL;</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	if (!id_priv-&gt;qkey)</div><div class='add'>+		cma_set_default_qkey(id_priv);</div><div class='ctx'> </div><div class='del'>-out2:</div><div class='del'>-	kfree(mc-&gt;multicast.ib);</div><div class='del'>-out1:</div><div class='del'>-	kfree(work);</div><div class='del'>-	return err;</div><div class='add'>+	rdma_ip2gid((struct sockaddr *)&amp;id_priv-&gt;id.route.addr.src_addr,</div><div class='add'>+		    &amp;ib.rec.port_gid);</div><div class='add'>+	INIT_WORK(&amp;mc-&gt;iboe_join.work, cma_iboe_join_work_handler);</div><div class='add'>+	cma_make_mc_event(0, id_priv, &amp;ib, &amp;mc-&gt;iboe_join.event, mc);</div><div class='add'>+	queue_work(cma_wq, &amp;mc-&gt;iboe_join.work);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,</div><div class='ctx'> 			u8 join_state, void *context)</div><div class='ctx'> {</div><div class='del'>-	struct rdma_id_private *id_priv;</div><div class='add'>+	struct rdma_id_private *id_priv =</div><div class='add'>+		container_of(id, struct rdma_id_private, id);</div><div class='ctx'> 	struct cma_multicast *mc;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	if (!id-&gt;device)</div><div class='add'>+	/* Not supported for kernel QPs */</div><div class='add'>+	if (WARN_ON(id-&gt;qp))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='del'>-	if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) &amp;&amp;</div><div class='del'>-	    !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED))</div><div class='add'>+	/* ULP is calling this wrong. */</div><div class='add'>+	if (!id-&gt;device || (READ_ONCE(id_priv-&gt;state) != RDMA_CM_ADDR_BOUND &amp;&amp;</div><div class='add'>+			    READ_ONCE(id_priv-&gt;state) != RDMA_CM_ADDR_RESOLVED))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	mc = kmalloc(sizeof *mc, GFP_KERNEL);</div><div class='add'>+	if (id_priv-&gt;id.qp_type != IB_QPT_UD)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	mc = kzalloc(sizeof(*mc), GFP_KERNEL);</div><div class='ctx'> 	if (!mc)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='hunk'>@@ -4435,7 +5077,6 @@ int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,</div><div class='ctx'> 	mc-&gt;join_state = join_state;</div><div class='ctx'> </div><div class='ctx'> 	if (rdma_protocol_roce(id-&gt;device, id-&gt;port_num)) {</div><div class='del'>-		kref_init(&amp;mc-&gt;mcref);</div><div class='ctx'> 		ret = cma_iboe_join_multicast(id_priv, mc);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto out_err;</div><div class='hunk'>@@ -4467,25 +5108,14 @@ void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)</div><div class='ctx'> 	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='ctx'> 	spin_lock_irq(&amp;id_priv-&gt;lock);</div><div class='ctx'> 	list_for_each_entry(mc, &amp;id_priv-&gt;mc_list, list) {</div><div class='del'>-		if (!memcmp(&amp;mc-&gt;addr, addr, rdma_addr_size(addr))) {</div><div class='del'>-			list_del(&amp;mc-&gt;list);</div><div class='del'>-			spin_unlock_irq(&amp;id_priv-&gt;lock);</div><div class='del'>-</div><div class='del'>-			if (id-&gt;qp)</div><div class='del'>-				ib_detach_mcast(id-&gt;qp,</div><div class='del'>-						&amp;mc-&gt;multicast.ib-&gt;rec.mgid,</div><div class='del'>-						be16_to_cpu(mc-&gt;multicast.ib-&gt;rec.mlid));</div><div class='del'>-</div><div class='del'>-			BUG_ON(id_priv-&gt;cma_dev-&gt;device != id-&gt;device);</div><div class='del'>-</div><div class='del'>-			if (rdma_cap_ib_mcast(id-&gt;device, id-&gt;port_num)) {</div><div class='del'>-				ib_sa_free_multicast(mc-&gt;multicast.ib);</div><div class='del'>-				kfree(mc);</div><div class='del'>-			} else if (rdma_protocol_roce(id-&gt;device, id-&gt;port_num)) {</div><div class='del'>-				cma_leave_roce_mc_group(id_priv, mc);</div><div class='del'>-			}</div><div class='del'>-			return;</div><div class='del'>-		}</div><div class='add'>+		if (memcmp(&amp;mc-&gt;addr, addr, rdma_addr_size(addr)) != 0)</div><div class='add'>+			continue;</div><div class='add'>+		list_del(&amp;mc-&gt;list);</div><div class='add'>+		spin_unlock_irq(&amp;id_priv-&gt;lock);</div><div class='add'>+</div><div class='add'>+		WARN_ON(id_priv-&gt;cma_dev-&gt;device != id-&gt;device);</div><div class='add'>+		destroy_mc(id_priv, mc);</div><div class='add'>+		return;</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock_irq(&amp;id_priv-&gt;lock);</div><div class='ctx'> }</div><div class='hunk'>@@ -4494,7 +5124,7 @@ EXPORT_SYMBOL(rdma_leave_multicast);</div><div class='ctx'> static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_dev_addr *dev_addr;</div><div class='del'>-	struct cma_ndev_work *work;</div><div class='add'>+	struct cma_work *work;</div><div class='ctx'> </div><div class='ctx'> 	dev_addr = &amp;id_priv-&gt;id.route.addr.dev_addr;</div><div class='ctx'> </div><div class='hunk'>@@ -4507,10 +5137,10 @@ static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id</div><div class='ctx'> 		if (!work)</div><div class='ctx'> 			return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-		INIT_WORK(&amp;work-&gt;work, cma_ndev_work_handler);</div><div class='add'>+		INIT_WORK(&amp;work-&gt;work, cma_work_handler);</div><div class='ctx'> 		work-&gt;id = id_priv;</div><div class='ctx'> 		work-&gt;event.event = RDMA_CM_EVENT_ADDR_CHANGE;</div><div class='del'>-		atomic_inc(&amp;id_priv-&gt;refcount);</div><div class='add'>+		cma_id_get(id_priv);</div><div class='ctx'> 		queue_work(cma_wq, &amp;work-&gt;work);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -4533,7 +5163,7 @@ static int cma_netdev_callback(struct notifier_block *self, unsigned long event,</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;lock);</div><div class='ctx'> 	list_for_each_entry(cma_dev, &amp;dev_list, list)</div><div class='del'>-		list_for_each_entry(id_priv, &amp;cma_dev-&gt;id_list, list) {</div><div class='add'>+		list_for_each_entry(id_priv, &amp;cma_dev-&gt;id_list, device_item) {</div><div class='ctx'> 			ret = cma_netdev_change(ndev, id_priv);</div><div class='ctx'> 			if (ret)</div><div class='ctx'> 				goto out;</div><div class='hunk'>@@ -4544,33 +5174,192 @@ out:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void cma_netevent_work_handler(struct work_struct *_work)</div><div class='add'>+{</div><div class='add'>+	struct rdma_id_private *id_priv =</div><div class='add'>+		container_of(_work, struct rdma_id_private, id.net_work);</div><div class='add'>+	struct rdma_cm_event event = {};</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+</div><div class='add'>+	if (READ_ONCE(id_priv-&gt;state) == RDMA_CM_DESTROYING ||</div><div class='add'>+	    READ_ONCE(id_priv-&gt;state) == RDMA_CM_DEVICE_REMOVAL)</div><div class='add'>+		goto out_unlock;</div><div class='add'>+</div><div class='add'>+	event.event = RDMA_CM_EVENT_UNREACHABLE;</div><div class='add'>+	event.status = -ETIMEDOUT;</div><div class='add'>+</div><div class='add'>+	if (cma_cm_event_handler(id_priv, &amp;event)) {</div><div class='add'>+		__acquire(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+		id_priv-&gt;cm_id.ib = NULL;</div><div class='add'>+		cma_id_put(id_priv);</div><div class='add'>+		destroy_id_handler_unlock(id_priv);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out_unlock:</div><div class='add'>+	mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+	cma_id_put(id_priv);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int cma_netevent_callback(struct notifier_block *self,</div><div class='add'>+				 unsigned long event, void *ctx)</div><div class='add'>+{</div><div class='add'>+	struct id_table_entry *ips_node = NULL;</div><div class='add'>+	struct rdma_id_private *current_id;</div><div class='add'>+	struct neighbour *neigh = ctx;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	if (event != NETEVENT_NEIGH_UPDATE)</div><div class='add'>+		return NOTIFY_DONE;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;id_table_lock, flags);</div><div class='add'>+	if (neigh-&gt;tbl-&gt;family == AF_INET6) {</div><div class='add'>+		struct sockaddr_in6 neigh_sock_6;</div><div class='add'>+</div><div class='add'>+		neigh_sock_6.sin6_family = AF_INET6;</div><div class='add'>+		neigh_sock_6.sin6_addr = *(struct in6_addr *)neigh-&gt;primary_key;</div><div class='add'>+		ips_node = node_from_ndev_ip(&amp;id_table, neigh-&gt;dev-&gt;ifindex,</div><div class='add'>+					     (struct sockaddr *)&amp;neigh_sock_6);</div><div class='add'>+	} else if (neigh-&gt;tbl-&gt;family == AF_INET) {</div><div class='add'>+		struct sockaddr_in neigh_sock_4;</div><div class='add'>+</div><div class='add'>+		neigh_sock_4.sin_family = AF_INET;</div><div class='add'>+		neigh_sock_4.sin_addr.s_addr = *(__be32 *)(neigh-&gt;primary_key);</div><div class='add'>+		ips_node = node_from_ndev_ip(&amp;id_table, neigh-&gt;dev-&gt;ifindex,</div><div class='add'>+					     (struct sockaddr *)&amp;neigh_sock_4);</div><div class='add'>+	} else</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	if (!ips_node)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	list_for_each_entry(current_id, &amp;ips_node-&gt;id_list, id_list_entry) {</div><div class='add'>+		if (!memcmp(current_id-&gt;id.route.addr.dev_addr.dst_dev_addr,</div><div class='add'>+			   neigh-&gt;ha, ETH_ALEN))</div><div class='add'>+			continue;</div><div class='add'>+		cma_id_get(current_id);</div><div class='add'>+		if (!queue_work(cma_wq, &amp;current_id-&gt;id.net_work))</div><div class='add'>+			cma_id_put(current_id);</div><div class='add'>+	}</div><div class='add'>+out:</div><div class='add'>+	spin_unlock_irqrestore(&amp;id_table_lock, flags);</div><div class='add'>+	return NOTIFY_DONE;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static struct notifier_block cma_nb = {</div><div class='ctx'> 	.notifier_call = cma_netdev_callback</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static void cma_add_one(struct ib_device *device)</div><div class='add'>+static struct notifier_block cma_netevent_cb = {</div><div class='add'>+	.notifier_call = cma_netevent_callback</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static void cma_send_device_removal_put(struct rdma_id_private *id_priv)</div><div class='ctx'> {</div><div class='add'>+	struct rdma_cm_event event = { .event = RDMA_CM_EVENT_DEVICE_REMOVAL };</div><div class='add'>+	enum rdma_cm_state state;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+	/* Record that we want to remove the device */</div><div class='add'>+	spin_lock_irqsave(&amp;id_priv-&gt;lock, flags);</div><div class='add'>+	state = id_priv-&gt;state;</div><div class='add'>+	if (state == RDMA_CM_DESTROYING || state == RDMA_CM_DEVICE_REMOVAL) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;id_priv-&gt;lock, flags);</div><div class='add'>+		mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+		cma_id_put(id_priv);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+	id_priv-&gt;state = RDMA_CM_DEVICE_REMOVAL;</div><div class='add'>+	spin_unlock_irqrestore(&amp;id_priv-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	if (cma_cm_event_handler(id_priv, &amp;event)) {</div><div class='add'>+		/*</div><div class='add'>+		 * At this point the ULP promises it won't call</div><div class='add'>+		 * rdma_destroy_id() concurrently</div><div class='add'>+		 */</div><div class='add'>+		cma_id_put(id_priv);</div><div class='add'>+		mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+		trace_cm_id_destroy(id_priv);</div><div class='add'>+		_destroy_id(id_priv, state);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * If this races with destroy then the thread that first assigns state</div><div class='add'>+	 * to a destroying does the cancel.</div><div class='add'>+	 */</div><div class='add'>+	cma_cancel_operation(id_priv, state);</div><div class='add'>+	cma_id_put(id_priv);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void cma_process_remove(struct cma_device *cma_dev)</div><div class='add'>+{</div><div class='add'>+	mutex_lock(&amp;lock);</div><div class='add'>+	while (!list_empty(&amp;cma_dev-&gt;id_list)) {</div><div class='add'>+		struct rdma_id_private *id_priv = list_first_entry(</div><div class='add'>+			&amp;cma_dev-&gt;id_list, struct rdma_id_private, device_item);</div><div class='add'>+</div><div class='add'>+		list_del_init(&amp;id_priv-&gt;listen_item);</div><div class='add'>+		list_del_init(&amp;id_priv-&gt;device_item);</div><div class='add'>+		cma_id_get(id_priv);</div><div class='add'>+		mutex_unlock(&amp;lock);</div><div class='add'>+</div><div class='add'>+		cma_send_device_removal_put(id_priv);</div><div class='add'>+</div><div class='add'>+		mutex_lock(&amp;lock);</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;lock);</div><div class='add'>+</div><div class='add'>+	cma_dev_put(cma_dev);</div><div class='add'>+	wait_for_completion(&amp;cma_dev-&gt;comp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool cma_supported(struct ib_device *device)</div><div class='add'>+{</div><div class='add'>+	u32 i;</div><div class='add'>+</div><div class='add'>+	rdma_for_each_port(device, i) {</div><div class='add'>+		if (rdma_cap_ib_cm(device, i) || rdma_cap_iw_cm(device, i))</div><div class='add'>+			return true;</div><div class='add'>+	}</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int cma_add_one(struct ib_device *device)</div><div class='add'>+{</div><div class='add'>+	struct rdma_id_private *to_destroy;</div><div class='ctx'> 	struct cma_device *cma_dev;</div><div class='ctx'> 	struct rdma_id_private *id_priv;</div><div class='del'>-	unsigned int i;</div><div class='ctx'> 	unsigned long supported_gids = 0;</div><div class='add'>+	int ret;</div><div class='add'>+	u32 i;</div><div class='add'>+</div><div class='add'>+	if (!cma_supported(device))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='del'>-	cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);</div><div class='add'>+	cma_dev = kmalloc(sizeof(*cma_dev), GFP_KERNEL);</div><div class='ctx'> 	if (!cma_dev)</div><div class='del'>-		return;</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	cma_dev-&gt;device = device;</div><div class='ctx'> 	cma_dev-&gt;default_gid_type = kcalloc(device-&gt;phys_port_cnt,</div><div class='ctx'> 					    sizeof(*cma_dev-&gt;default_gid_type),</div><div class='ctx'> 					    GFP_KERNEL);</div><div class='del'>-	if (!cma_dev-&gt;default_gid_type)</div><div class='add'>+	if (!cma_dev-&gt;default_gid_type) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='ctx'> 		goto free_cma_dev;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	cma_dev-&gt;default_roce_tos = kcalloc(device-&gt;phys_port_cnt,</div><div class='ctx'> 					    sizeof(*cma_dev-&gt;default_roce_tos),</div><div class='ctx'> 					    GFP_KERNEL);</div><div class='del'>-	if (!cma_dev-&gt;default_roce_tos)</div><div class='add'>+	if (!cma_dev-&gt;default_roce_tos) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='ctx'> 		goto free_gid_type;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	rdma_for_each_port (device, i) {</div><div class='ctx'> 		supported_gids = roce_gid_type_mask_support(device, i);</div><div class='hunk'>@@ -4585,86 +5374,42 @@ static void cma_add_one(struct ib_device *device)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	init_completion(&amp;cma_dev-&gt;comp);</div><div class='del'>-	atomic_set(&amp;cma_dev-&gt;refcount, 1);</div><div class='add'>+	refcount_set(&amp;cma_dev-&gt;refcount, 1);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;cma_dev-&gt;id_list);</div><div class='ctx'> 	ib_set_client_data(device, &amp;cma_client, cma_dev);</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;lock);</div><div class='ctx'> 	list_add_tail(&amp;cma_dev-&gt;list, &amp;dev_list);</div><div class='del'>-	list_for_each_entry(id_priv, &amp;listen_any_list, list)</div><div class='del'>-		cma_listen_on_dev(id_priv, cma_dev);</div><div class='add'>+	list_for_each_entry(id_priv, &amp;listen_any_list, listen_any_item) {</div><div class='add'>+		ret = cma_listen_on_dev(id_priv, cma_dev, &amp;to_destroy);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto free_listen;</div><div class='add'>+	}</div><div class='ctx'> 	mutex_unlock(&amp;lock);</div><div class='ctx'> </div><div class='del'>-	return;</div><div class='add'>+	trace_cm_add_one(device);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+free_listen:</div><div class='add'>+	list_del(&amp;cma_dev-&gt;list);</div><div class='add'>+	mutex_unlock(&amp;lock);</div><div class='ctx'> </div><div class='add'>+	/* cma_process_remove() will delete to_destroy */</div><div class='add'>+	cma_process_remove(cma_dev);</div><div class='add'>+	kfree(cma_dev-&gt;default_roce_tos);</div><div class='ctx'> free_gid_type:</div><div class='ctx'> 	kfree(cma_dev-&gt;default_gid_type);</div><div class='ctx'> </div><div class='ctx'> free_cma_dev:</div><div class='ctx'> 	kfree(cma_dev);</div><div class='del'>-</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int cma_remove_id_dev(struct rdma_id_private *id_priv)</div><div class='del'>-{</div><div class='del'>-	struct rdma_cm_event event = {};</div><div class='del'>-	enum rdma_cm_state state;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	/* Record that we want to remove the device */</div><div class='del'>-	state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL);</div><div class='del'>-	if (state == RDMA_CM_DESTROYING)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	cma_cancel_operation(id_priv, state);</div><div class='del'>-	mutex_lock(&amp;id_priv-&gt;handler_mutex);</div><div class='del'>-</div><div class='del'>-	/* Check for destruction from another callback. */</div><div class='del'>-	if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL))</div><div class='del'>-		goto out;</div><div class='del'>-</div><div class='del'>-	event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;</div><div class='del'>-	ret = id_priv-&gt;id.event_handler(&amp;id_priv-&gt;id, &amp;event);</div><div class='del'>-out:</div><div class='del'>-	mutex_unlock(&amp;id_priv-&gt;handler_mutex);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cma_process_remove(struct cma_device *cma_dev)</div><div class='del'>-{</div><div class='del'>-	struct rdma_id_private *id_priv;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;lock);</div><div class='del'>-	while (!list_empty(&amp;cma_dev-&gt;id_list)) {</div><div class='del'>-		id_priv = list_entry(cma_dev-&gt;id_list.next,</div><div class='del'>-				     struct rdma_id_private, list);</div><div class='del'>-</div><div class='del'>-		list_del(&amp;id_priv-&gt;listen_list);</div><div class='del'>-		list_del_init(&amp;id_priv-&gt;list);</div><div class='del'>-		atomic_inc(&amp;id_priv-&gt;refcount);</div><div class='del'>-		mutex_unlock(&amp;lock);</div><div class='del'>-</div><div class='del'>-		ret = id_priv-&gt;internal_id ? 1 : cma_remove_id_dev(id_priv);</div><div class='del'>-		cma_deref_id(id_priv);</div><div class='del'>-		if (ret)</div><div class='del'>-			rdma_destroy_id(&amp;id_priv-&gt;id);</div><div class='del'>-</div><div class='del'>-		mutex_lock(&amp;lock);</div><div class='del'>-	}</div><div class='del'>-	mutex_unlock(&amp;lock);</div><div class='del'>-</div><div class='del'>-	cma_deref_dev(cma_dev);</div><div class='del'>-	wait_for_completion(&amp;cma_dev-&gt;comp);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static void cma_remove_one(struct ib_device *device, void *client_data)</div><div class='ctx'> {</div><div class='ctx'> 	struct cma_device *cma_dev = client_data;</div><div class='ctx'> </div><div class='del'>-	if (!cma_dev)</div><div class='del'>-		return;</div><div class='add'>+	trace_cm_remove_one(device);</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;lock);</div><div class='ctx'> 	list_del(&amp;cma_dev-&gt;list);</div><div class='hunk'>@@ -4709,6 +5454,19 @@ static int __init cma_init(void)</div><div class='ctx'> {</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	/*</div><div class='add'>+	 * There is a rare lock ordering dependency in cma_netdev_callback()</div><div class='add'>+	 * that only happens when bonding is enabled. Teach lockdep that rtnl</div><div class='add'>+	 * must never be nested under lock so it can find these without having</div><div class='add'>+	 * to test with bonding.</div><div class='add'>+	 */</div><div class='add'>+	if (IS_ENABLED(CONFIG_LOCKDEP)) {</div><div class='add'>+		rtnl_lock();</div><div class='add'>+		mutex_lock(&amp;lock);</div><div class='add'>+		mutex_unlock(&amp;lock);</div><div class='add'>+		rtnl_unlock();</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	cma_wq = alloc_ordered_workqueue("rdma_cm", WQ_MEM_RECLAIM);</div><div class='ctx'> 	if (!cma_wq)</div><div class='ctx'> 		return -ENOMEM;</div><div class='hunk'>@@ -4719,18 +5477,25 @@ static int __init cma_init(void)</div><div class='ctx'> </div><div class='ctx'> 	ib_sa_register_client(&amp;sa_client);</div><div class='ctx'> 	register_netdevice_notifier(&amp;cma_nb);</div><div class='add'>+	register_netevent_notifier(&amp;cma_netevent_cb);</div><div class='ctx'> </div><div class='ctx'> 	ret = ib_register_client(&amp;cma_client);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='del'>-	cma_configfs_init();</div><div class='add'>+	ret = cma_configfs_init();</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_ib;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='add'>+err_ib:</div><div class='add'>+	ib_unregister_client(&amp;cma_client);</div><div class='ctx'> err:</div><div class='add'>+	unregister_netevent_notifier(&amp;cma_netevent_cb);</div><div class='ctx'> 	unregister_netdevice_notifier(&amp;cma_nb);</div><div class='ctx'> 	ib_sa_unregister_client(&amp;sa_client);</div><div class='add'>+	unregister_pernet_subsys(&amp;cma_pernet_operations);</div><div class='ctx'> err_wq:</div><div class='ctx'> 	destroy_workqueue(cma_wq);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -4740,6 +5505,7 @@ static void __exit cma_cleanup(void)</div><div class='ctx'> {</div><div class='ctx'> 	cma_configfs_exit();</div><div class='ctx'> 	ib_unregister_client(&amp;cma_client);</div><div class='add'>+	unregister_netevent_notifier(&amp;cma_netevent_cb);</div><div class='ctx'> 	unregister_netdevice_notifier(&amp;cma_nb);</div><div class='ctx'> 	ib_sa_unregister_client(&amp;sa_client);</div><div class='ctx'> 	unregister_pernet_subsys(&amp;cma_pernet_operations);</div><div class='hunk'>@@ -4748,3 +5514,129 @@ static void __exit cma_cleanup(void)</div><div class='ctx'> </div><div class='ctx'> module_init(cma_init);</div><div class='ctx'> module_exit(cma_cleanup);</div><div class='add'>+</div><div class='add'>+static void cma_query_ib_service_handler(int status,</div><div class='add'>+					 struct sa_service_rec *recs,</div><div class='add'>+					 unsigned int num_recs, void *context)</div><div class='add'>+{</div><div class='add'>+	struct cma_work *work = context;</div><div class='add'>+	struct rdma_id_private *id_priv = work-&gt;id;</div><div class='add'>+	struct sockaddr_ib *addr;</div><div class='add'>+</div><div class='add'>+	if (status)</div><div class='add'>+		goto fail;</div><div class='add'>+</div><div class='add'>+	if (!num_recs) {</div><div class='add'>+		status = -ENOENT;</div><div class='add'>+		goto fail;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (id_priv-&gt;id.route.service_recs) {</div><div class='add'>+		status = -EALREADY;</div><div class='add'>+		goto fail;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	id_priv-&gt;id.route.service_recs =</div><div class='add'>+		kmalloc_array(num_recs, sizeof(*recs), GFP_KERNEL);</div><div class='add'>+	if (!id_priv-&gt;id.route.service_recs) {</div><div class='add'>+		status = -ENOMEM;</div><div class='add'>+		goto fail;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	id_priv-&gt;id.route.num_service_recs = num_recs;</div><div class='add'>+	memcpy(id_priv-&gt;id.route.service_recs, recs, sizeof(*recs) * num_recs);</div><div class='add'>+</div><div class='add'>+	addr = (struct sockaddr_ib *)&amp;id_priv-&gt;id.route.addr.dst_addr;</div><div class='add'>+	addr-&gt;sib_family = AF_IB;</div><div class='add'>+	addr-&gt;sib_addr = *(struct ib_addr *)&amp;recs-&gt;gid;</div><div class='add'>+	addr-&gt;sib_pkey = recs-&gt;pkey;</div><div class='add'>+	addr-&gt;sib_sid = recs-&gt;id;</div><div class='add'>+	rdma_addr_set_dgid(&amp;id_priv-&gt;id.route.addr.dev_addr,</div><div class='add'>+			   (union ib_gid *)&amp;addr-&gt;sib_addr);</div><div class='add'>+	ib_addr_set_pkey(&amp;id_priv-&gt;id.route.addr.dev_addr,</div><div class='add'>+			 ntohs(addr-&gt;sib_pkey));</div><div class='add'>+</div><div class='add'>+	queue_work(cma_wq, &amp;work-&gt;work);</div><div class='add'>+	return;</div><div class='add'>+</div><div class='add'>+fail:</div><div class='add'>+	work-&gt;old_state = RDMA_CM_ADDRINFO_QUERY;</div><div class='add'>+	work-&gt;new_state = RDMA_CM_ADDR_BOUND;</div><div class='add'>+	work-&gt;event.event = RDMA_CM_EVENT_ADDRINFO_ERROR;</div><div class='add'>+	work-&gt;event.status = status;</div><div class='add'>+	pr_debug_ratelimited(</div><div class='add'>+		"RDMA CM: SERVICE_ERROR: failed to query service record. status %d\n",</div><div class='add'>+		status);</div><div class='add'>+	queue_work(cma_wq, &amp;work-&gt;work);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int cma_resolve_ib_service(struct rdma_id_private *id_priv,</div><div class='add'>+				  struct rdma_ucm_ib_service *ibs)</div><div class='add'>+{</div><div class='add'>+	struct sa_service_rec sr = {};</div><div class='add'>+	ib_sa_comp_mask mask = 0;</div><div class='add'>+	struct cma_work *work;</div><div class='add'>+</div><div class='add'>+	work = kzalloc(sizeof(*work), GFP_KERNEL);</div><div class='add'>+	if (!work)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cma_id_get(id_priv);</div><div class='add'>+</div><div class='add'>+	work-&gt;id = id_priv;</div><div class='add'>+	INIT_WORK(&amp;work-&gt;work, cma_work_handler);</div><div class='add'>+	work-&gt;old_state = RDMA_CM_ADDRINFO_QUERY;</div><div class='add'>+	work-&gt;new_state = RDMA_CM_ADDRINFO_RESOLVED;</div><div class='add'>+	work-&gt;event.event = RDMA_CM_EVENT_ADDRINFO_RESOLVED;</div><div class='add'>+</div><div class='add'>+	if (ibs-&gt;flags &amp; RDMA_USER_CM_IB_SERVICE_FLAG_ID) {</div><div class='add'>+		sr.id = cpu_to_be64(ibs-&gt;service_id);</div><div class='add'>+		mask |= IB_SA_SERVICE_REC_SERVICE_ID;</div><div class='add'>+	}</div><div class='add'>+	if (ibs-&gt;flags &amp; RDMA_USER_CM_IB_SERVICE_FLAG_NAME) {</div><div class='add'>+		strscpy(sr.name, ibs-&gt;service_name, sizeof(sr.name));</div><div class='add'>+		mask |= IB_SA_SERVICE_REC_SERVICE_NAME;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	id_priv-&gt;query_id = ib_sa_service_rec_get(&amp;sa_client,</div><div class='add'>+						  id_priv-&gt;id.device,</div><div class='add'>+						  id_priv-&gt;id.port_num,</div><div class='add'>+						  &amp;sr, mask,</div><div class='add'>+						  2000, GFP_KERNEL,</div><div class='add'>+						  cma_query_ib_service_handler,</div><div class='add'>+						  work, &amp;id_priv-&gt;query);</div><div class='add'>+</div><div class='add'>+	if (id_priv-&gt;query_id &lt; 0) {</div><div class='add'>+		cma_id_put(id_priv);</div><div class='add'>+		kfree(work);</div><div class='add'>+		return id_priv-&gt;query_id;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int rdma_resolve_ib_service(struct rdma_cm_id *id,</div><div class='add'>+			    struct rdma_ucm_ib_service *ibs)</div><div class='add'>+{</div><div class='add'>+	struct rdma_id_private *id_priv;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	id_priv = container_of(id, struct rdma_id_private, id);</div><div class='add'>+	if (!id_priv-&gt;cma_dev ||</div><div class='add'>+	    !cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDRINFO_QUERY))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (rdma_cap_ib_sa(id-&gt;device, id-&gt;port_num))</div><div class='add'>+		ret = cma_resolve_ib_service(id_priv, ibs);</div><div class='add'>+	else</div><div class='add'>+		ret = -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+err:</div><div class='add'>+	cma_comp_exch(id_priv, RDMA_CM_ADDRINFO_QUERY, RDMA_CM_ADDR_BOUND);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_resolve_ib_service);</div><div class='head'>diff --git a/drivers/infiniband/core/cma_configfs.c b/drivers/infiniband/core/cma_configfs.c<br/>index 3ec2c415bb70..f2fb2d8a6597 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cma_configfs.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cma_configfs.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cma_configfs.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/cma_configfs.c</a></div><div class='hunk'>@@ -30,7 +30,6 @@</div><div class='ctx'>  * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='ctx'> #include &lt;linux/configfs.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_verbs.h&gt;</div><div class='ctx'> #include &lt;rdma/rdma_cm.h&gt;</div><div class='hunk'>@@ -43,7 +42,7 @@ struct cma_device;</div><div class='ctx'> struct cma_dev_group;</div><div class='ctx'> </div><div class='ctx'> struct cma_dev_port_group {</div><div class='del'>-	unsigned int		port_num;</div><div class='add'>+	u32			port_num;</div><div class='ctx'> 	struct cma_dev_group	*cma_dev_group;</div><div class='ctx'> 	struct config_group	group;</div><div class='ctx'> };</div><div class='hunk'>@@ -94,7 +93,7 @@ static int cma_configfs_params_get(struct config_item *item,</div><div class='ctx'> </div><div class='ctx'> static void cma_configfs_params_put(struct cma_device *cma_dev)</div><div class='ctx'> {</div><div class='del'>-	cma_deref_dev(cma_dev);</div><div class='add'>+	cma_dev_put(cma_dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t default_roce_mode_show(struct config_item *item,</div><div class='hunk'>@@ -115,7 +114,7 @@ static ssize_t default_roce_mode_show(struct config_item *item,</div><div class='ctx'> 	if (gid_type &lt; 0)</div><div class='ctx'> 		return gid_type;</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%s\n", ib_cache_gid_type_str(gid_type));</div><div class='add'>+	return sysfs_emit(buf, "%s\n", ib_cache_gid_type_str(gid_type));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t default_roce_mode_store(struct config_item *item,</div><div class='hunk'>@@ -123,16 +122,19 @@ static ssize_t default_roce_mode_store(struct config_item *item,</div><div class='ctx'> {</div><div class='ctx'> 	struct cma_device *cma_dev;</div><div class='ctx'> 	struct cma_dev_port_group *group;</div><div class='del'>-	int gid_type = ib_cache_gid_parse_type_str(buf);</div><div class='add'>+	int gid_type;</div><div class='ctx'> 	ssize_t ret;</div><div class='ctx'> </div><div class='del'>-	if (gid_type &lt; 0)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='ctx'> 	ret = cma_configfs_params_get(item, &amp;cma_dev, &amp;group);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='add'>+	gid_type = ib_cache_gid_parse_type_str(buf);</div><div class='add'>+	if (gid_type &lt; 0) {</div><div class='add'>+		cma_configfs_params_put(cma_dev);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	ret = cma_set_default_gid_type(cma_dev, group-&gt;port_num, gid_type);</div><div class='ctx'> </div><div class='ctx'> 	cma_configfs_params_put(cma_dev);</div><div class='hunk'>@@ -156,7 +158,7 @@ static ssize_t default_roce_tos_show(struct config_item *item, char *buf)</div><div class='ctx'> 	tos = cma_get_default_roce_tos(cma_dev, group-&gt;port_num);</div><div class='ctx'> 	cma_configfs_params_put(cma_dev);</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%u\n", tos);</div><div class='add'>+	return sysfs_emit(buf, "%u\n", tos);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t default_roce_tos_store(struct config_item *item,</div><div class='hunk'>@@ -197,11 +199,10 @@ static const struct config_item_type cma_port_group_type = {</div><div class='ctx'> static int make_cma_ports(struct cma_dev_group *cma_dev_group,</div><div class='ctx'> 			  struct cma_device *cma_dev)</div><div class='ctx'> {</div><div class='del'>-	struct ib_device *ibdev;</div><div class='del'>-	unsigned int i;</div><div class='del'>-	unsigned int ports_num;</div><div class='ctx'> 	struct cma_dev_port_group *ports;</div><div class='del'>-	int err;</div><div class='add'>+	struct ib_device *ibdev;</div><div class='add'>+	u32 ports_num;</div><div class='add'>+	u32 i;</div><div class='ctx'> </div><div class='ctx'> 	ibdev = cma_get_ib_dev(cma_dev);</div><div class='ctx'> </div><div class='hunk'>@@ -212,13 +213,11 @@ static int make_cma_ports(struct cma_dev_group *cma_dev_group,</div><div class='ctx'> 	ports = kcalloc(ports_num, sizeof(*cma_dev_group-&gt;ports),</div><div class='ctx'> 			GFP_KERNEL);</div><div class='ctx'> </div><div class='del'>-	if (!ports) {</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto free;</div><div class='del'>-	}</div><div class='add'>+	if (!ports)</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; ports_num; i++) {</div><div class='del'>-		char port_str[10];</div><div class='add'>+		char port_str[11];</div><div class='ctx'> </div><div class='ctx'> 		ports[i].port_num = i + 1;</div><div class='ctx'> 		snprintf(port_str, sizeof(port_str), "%u", i + 1);</div><div class='hunk'>@@ -231,12 +230,7 @@ static int make_cma_ports(struct cma_dev_group *cma_dev_group,</div><div class='ctx'> </div><div class='ctx'> 	}</div><div class='ctx'> 	cma_dev_group-&gt;ports = ports;</div><div class='del'>-</div><div class='ctx'> 	return 0;</div><div class='del'>-free:</div><div class='del'>-	kfree(ports);</div><div class='del'>-	cma_dev_group-&gt;ports = NULL;</div><div class='del'>-	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void release_cma_dev(struct config_item  *item)</div><div class='hunk'>@@ -298,7 +292,7 @@ static struct config_group *make_cma_dev(struct config_group *group,</div><div class='ctx'> 		goto fail;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	strlcpy(cma_dev_group-&gt;name, name, sizeof(cma_dev_group-&gt;name));</div><div class='add'>+	strscpy(cma_dev_group-&gt;name, name, sizeof(cma_dev_group-&gt;name));</div><div class='ctx'> </div><div class='ctx'> 	config_group_init_type_name(&amp;cma_dev_group-&gt;ports_group, "ports",</div><div class='ctx'> 				    &amp;cma_ports_group_type);</div><div class='hunk'>@@ -312,18 +306,31 @@ static struct config_group *make_cma_dev(struct config_group *group,</div><div class='ctx'> 	configfs_add_default_group(&amp;cma_dev_group-&gt;ports_group,</div><div class='ctx'> 			&amp;cma_dev_group-&gt;device_group);</div><div class='ctx'> </div><div class='del'>-	cma_deref_dev(cma_dev);</div><div class='add'>+	cma_dev_put(cma_dev);</div><div class='ctx'> 	return &amp;cma_dev_group-&gt;device_group;</div><div class='ctx'> </div><div class='ctx'> fail:</div><div class='ctx'> 	if (cma_dev)</div><div class='del'>-		cma_deref_dev(cma_dev);</div><div class='add'>+		cma_dev_put(cma_dev);</div><div class='ctx'> 	kfree(cma_dev_group);</div><div class='ctx'> 	return ERR_PTR(err);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void drop_cma_dev(struct config_group *cgroup, struct config_item *item)</div><div class='add'>+{</div><div class='add'>+	struct config_group *group =</div><div class='add'>+		container_of(item, struct config_group, cg_item);</div><div class='add'>+	struct cma_dev_group *cma_dev_group =</div><div class='add'>+		container_of(group, struct cma_dev_group, device_group);</div><div class='add'>+</div><div class='add'>+	configfs_remove_default_groups(&amp;cma_dev_group-&gt;ports_group);</div><div class='add'>+	configfs_remove_default_groups(&amp;cma_dev_group-&gt;device_group);</div><div class='add'>+	config_item_put(item);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static struct configfs_group_operations cma_subsys_group_ops = {</div><div class='ctx'> 	.make_group	= make_cma_dev,</div><div class='add'>+	.drop_item	= drop_cma_dev,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static const struct config_item_type cma_subsys_type = {</div><div class='hunk'>@@ -342,12 +349,18 @@ static struct configfs_subsystem cma_subsys = {</div><div class='ctx'> </div><div class='ctx'> int __init cma_configfs_init(void)</div><div class='ctx'> {</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='ctx'> 	config_group_init(&amp;cma_subsys.su_group);</div><div class='ctx'> 	mutex_init(&amp;cma_subsys.su_mutex);</div><div class='del'>-	return configfs_register_subsystem(&amp;cma_subsys);</div><div class='add'>+	ret = configfs_register_subsystem(&amp;cma_subsys);</div><div class='add'>+	if (ret)</div><div class='add'>+		mutex_destroy(&amp;cma_subsys.su_mutex);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void __exit cma_configfs_exit(void)</div><div class='ctx'> {</div><div class='ctx'> 	configfs_unregister_subsystem(&amp;cma_subsys);</div><div class='add'>+	mutex_destroy(&amp;cma_subsys.su_mutex);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/core/cma_priv.h b/drivers/infiniband/core/cma_priv.h<br/>index ca7307277518..c604b601f4d9 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cma_priv.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cma_priv.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cma_priv.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/cma_priv.h</a></div><div class='hunk'>@@ -47,7 +47,9 @@ enum rdma_cm_state {</div><div class='ctx'> 	RDMA_CM_ADDR_BOUND,</div><div class='ctx'> 	RDMA_CM_LISTEN,</div><div class='ctx'> 	RDMA_CM_DEVICE_REMOVAL,</div><div class='del'>-	RDMA_CM_DESTROYING</div><div class='add'>+	RDMA_CM_DESTROYING,</div><div class='add'>+	RDMA_CM_ADDRINFO_QUERY,</div><div class='add'>+	RDMA_CM_ADDRINFO_RESOLVED</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct rdma_id_private {</div><div class='hunk'>@@ -55,8 +57,16 @@ struct rdma_id_private {</div><div class='ctx'> </div><div class='ctx'> 	struct rdma_bind_list	*bind_list;</div><div class='ctx'> 	struct hlist_node	node;</div><div class='del'>-	struct list_head	list; /* listen_any_list or cma_device.list */</div><div class='del'>-	struct list_head	listen_list; /* per device listens */</div><div class='add'>+	union {</div><div class='add'>+		struct list_head device_item; /* On cma_device-&gt;id_list */</div><div class='add'>+		struct list_head listen_any_item; /* On listen_any_list */</div><div class='add'>+	};</div><div class='add'>+	union {</div><div class='add'>+		/* On rdma_id_private-&gt;listen_list */</div><div class='add'>+		struct list_head listen_item;</div><div class='add'>+		struct list_head listen_list;</div><div class='add'>+	};</div><div class='add'>+	struct list_head        id_list_entry;</div><div class='ctx'> 	struct cma_device	*cma_dev;</div><div class='ctx'> 	struct list_head	mc_list;</div><div class='ctx'> </div><div class='hunk'>@@ -66,7 +76,7 @@ struct rdma_id_private {</div><div class='ctx'> 	struct mutex		qp_mutex;</div><div class='ctx'> </div><div class='ctx'> 	struct completion	comp;</div><div class='del'>-	atomic_t		refcount;</div><div class='add'>+	refcount_t refcount;</div><div class='ctx'> 	struct mutex		handler_mutex;</div><div class='ctx'> </div><div class='ctx'> 	int			backlog;</div><div class='hunk'>@@ -86,15 +96,19 @@ struct rdma_id_private {</div><div class='ctx'> 	u8			tos;</div><div class='ctx'> 	u8			tos_set:1;</div><div class='ctx'> 	u8                      timeout_set:1;</div><div class='add'>+	u8			min_rnr_timer_set:1;</div><div class='ctx'> 	u8			reuseaddr;</div><div class='ctx'> 	u8			afonly;</div><div class='ctx'> 	u8			timeout;</div><div class='add'>+	u8			min_rnr_timer;</div><div class='add'>+	u8 used_resolve_ip;</div><div class='ctx'> 	enum ib_gid_type	gid_type;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Internal to RDMA/core, don't use in the drivers</div><div class='ctx'> 	 */</div><div class='ctx'> 	struct rdma_restrack_entry     res;</div><div class='add'>+	struct rdma_ucm_ece ece;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> #if IS_ENABLED(CONFIG_INFINIBAND_ADDR_TRANS_CONFIGFS)</div><div class='hunk'>@@ -111,16 +125,16 @@ static inline void cma_configfs_exit(void)</div><div class='ctx'> }</div><div class='ctx'> #endif</div><div class='ctx'> </div><div class='del'>-void cma_ref_dev(struct cma_device *dev);</div><div class='del'>-void cma_deref_dev(struct cma_device *dev);</div><div class='add'>+void cma_dev_get(struct cma_device *dev);</div><div class='add'>+void cma_dev_put(struct cma_device *dev);</div><div class='ctx'> typedef bool (*cma_device_filter)(struct ib_device *, void *);</div><div class='ctx'> struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter filter,</div><div class='ctx'> 					     void *cookie);</div><div class='del'>-int cma_get_default_gid_type(struct cma_device *dev, unsigned int port);</div><div class='del'>-int cma_set_default_gid_type(struct cma_device *dev, unsigned int port,</div><div class='add'>+int cma_get_default_gid_type(struct cma_device *dev, u32 port);</div><div class='add'>+int cma_set_default_gid_type(struct cma_device *dev, u32 port,</div><div class='ctx'> 			     enum ib_gid_type default_gid_type);</div><div class='del'>-int cma_get_default_roce_tos(struct cma_device *dev, unsigned int port);</div><div class='del'>-int cma_set_default_roce_tos(struct cma_device *dev, unsigned int port,</div><div class='add'>+int cma_get_default_roce_tos(struct cma_device *dev, u32 port);</div><div class='add'>+int cma_set_default_roce_tos(struct cma_device *dev, u32 port,</div><div class='ctx'> 			     u8 default_roce_tos);</div><div class='ctx'> struct ib_device *cma_get_ib_dev(struct cma_device *dev);</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/core/cma_trace.c b/drivers/infiniband/core/cma_trace.c<br/>new file mode 100644<br/>index 000000000000..b314a281e10e<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cma_trace.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/cma_trace.c</a></div><div class='hunk'>@@ -0,0 +1,16 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+/*</div><div class='add'>+ * Trace points for the RDMA Connection Manager.</div><div class='add'>+ *</div><div class='add'>+ * Author: Chuck Lever &lt;chuck.lever@oracle.com&gt;</div><div class='add'>+ *</div><div class='add'>+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#define CREATE_TRACE_POINTS</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/rdma_cm.h&gt;</div><div class='add'>+#include &lt;rdma/ib_cm.h&gt;</div><div class='add'>+#include "cma_priv.h"</div><div class='add'>+</div><div class='add'>+#include "cma_trace.h"</div><div class='head'>diff --git a/drivers/infiniband/core/cma_trace.h b/drivers/infiniband/core/cma_trace.h<br/>new file mode 100644<br/>index 000000000000..3456d5f3aa47<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cma_trace.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/cma_trace.h</a></div><div class='hunk'>@@ -0,0 +1,361 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0-only */</div><div class='add'>+/*</div><div class='add'>+ * Trace point definitions for the RDMA Connect Manager.</div><div class='add'>+ *</div><div class='add'>+ * Author: Chuck Lever &lt;chuck.lever@oracle.com&gt;</div><div class='add'>+ *</div><div class='add'>+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#undef TRACE_SYSTEM</div><div class='add'>+#define TRACE_SYSTEM rdma_cma</div><div class='add'>+</div><div class='add'>+#if !defined(_TRACE_RDMA_CMA_H) || defined(TRACE_HEADER_MULTI_READ)</div><div class='add'>+</div><div class='add'>+#define _TRACE_RDMA_CMA_H</div><div class='add'>+</div><div class='add'>+#include &lt;linux/tracepoint.h&gt;</div><div class='add'>+#include &lt;trace/misc/rdma.h&gt;</div><div class='add'>+</div><div class='add'>+</div><div class='add'>+DECLARE_EVENT_CLASS(cma_fsm_class,</div><div class='add'>+	TP_PROTO(</div><div class='add'>+		const struct rdma_id_private *id_priv</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_ARGS(id_priv),</div><div class='add'>+</div><div class='add'>+	TP_STRUCT__entry(</div><div class='add'>+		__field(u32, cm_id)</div><div class='add'>+		__field(u32, tos)</div><div class='add'>+		__array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))</div><div class='add'>+		__array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_fast_assign(</div><div class='add'>+		__entry-&gt;cm_id = id_priv-&gt;res.id;</div><div class='add'>+		__entry-&gt;tos = id_priv-&gt;tos;</div><div class='add'>+		memcpy(__entry-&gt;srcaddr, &amp;id_priv-&gt;id.route.addr.src_addr,</div><div class='add'>+		       sizeof(struct sockaddr_in6));</div><div class='add'>+		memcpy(__entry-&gt;dstaddr, &amp;id_priv-&gt;id.route.addr.dst_addr,</div><div class='add'>+		       sizeof(struct sockaddr_in6));</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u",</div><div class='add'>+		__entry-&gt;cm_id, __entry-&gt;srcaddr, __entry-&gt;dstaddr, __entry-&gt;tos</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+#define DEFINE_CMA_FSM_EVENT(name)						\</div><div class='add'>+		DEFINE_EVENT(cma_fsm_class, cm_##name,				\</div><div class='add'>+				TP_PROTO(					\</div><div class='add'>+					const struct rdma_id_private *id_priv	\</div><div class='add'>+				),						\</div><div class='add'>+				TP_ARGS(id_priv))</div><div class='add'>+</div><div class='add'>+DEFINE_CMA_FSM_EVENT(send_rtu);</div><div class='add'>+DEFINE_CMA_FSM_EVENT(send_rej);</div><div class='add'>+DEFINE_CMA_FSM_EVENT(prepare_mra);</div><div class='add'>+DEFINE_CMA_FSM_EVENT(send_sidr_req);</div><div class='add'>+DEFINE_CMA_FSM_EVENT(send_sidr_rep);</div><div class='add'>+DEFINE_CMA_FSM_EVENT(disconnect);</div><div class='add'>+DEFINE_CMA_FSM_EVENT(sent_drep);</div><div class='add'>+DEFINE_CMA_FSM_EVENT(sent_dreq);</div><div class='add'>+DEFINE_CMA_FSM_EVENT(id_destroy);</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(cm_id_attach,</div><div class='add'>+	TP_PROTO(</div><div class='add'>+		const struct rdma_id_private *id_priv,</div><div class='add'>+		const struct ib_device *device</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_ARGS(id_priv, device),</div><div class='add'>+</div><div class='add'>+	TP_STRUCT__entry(</div><div class='add'>+		__field(u32, cm_id)</div><div class='add'>+		__array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))</div><div class='add'>+		__array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))</div><div class='add'>+		__string(devname, device-&gt;name)</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_fast_assign(</div><div class='add'>+		__entry-&gt;cm_id = id_priv-&gt;res.id;</div><div class='add'>+		memcpy(__entry-&gt;srcaddr, &amp;id_priv-&gt;id.route.addr.src_addr,</div><div class='add'>+		       sizeof(struct sockaddr_in6));</div><div class='add'>+		memcpy(__entry-&gt;dstaddr, &amp;id_priv-&gt;id.route.addr.dst_addr,</div><div class='add'>+		       sizeof(struct sockaddr_in6));</div><div class='add'>+		__assign_str(devname);</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_printk("cm.id=%u src=%pISpc dst=%pISpc device=%s",</div><div class='add'>+		__entry-&gt;cm_id, __entry-&gt;srcaddr, __entry-&gt;dstaddr,</div><div class='add'>+		__get_str(devname)</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DECLARE_EVENT_CLASS(cma_qp_class,</div><div class='add'>+	TP_PROTO(</div><div class='add'>+		const struct rdma_id_private *id_priv</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_ARGS(id_priv),</div><div class='add'>+</div><div class='add'>+	TP_STRUCT__entry(</div><div class='add'>+		__field(u32, cm_id)</div><div class='add'>+		__field(u32, tos)</div><div class='add'>+		__field(u32, qp_num)</div><div class='add'>+		__array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))</div><div class='add'>+		__array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_fast_assign(</div><div class='add'>+		__entry-&gt;cm_id = id_priv-&gt;res.id;</div><div class='add'>+		__entry-&gt;tos = id_priv-&gt;tos;</div><div class='add'>+		__entry-&gt;qp_num = id_priv-&gt;qp_num;</div><div class='add'>+		memcpy(__entry-&gt;srcaddr, &amp;id_priv-&gt;id.route.addr.src_addr,</div><div class='add'>+		       sizeof(struct sockaddr_in6));</div><div class='add'>+		memcpy(__entry-&gt;dstaddr, &amp;id_priv-&gt;id.route.addr.dst_addr,</div><div class='add'>+		       sizeof(struct sockaddr_in6));</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u qp_num=%u",</div><div class='add'>+		__entry-&gt;cm_id, __entry-&gt;srcaddr, __entry-&gt;dstaddr, __entry-&gt;tos,</div><div class='add'>+		__entry-&gt;qp_num</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+#define DEFINE_CMA_QP_EVENT(name)						\</div><div class='add'>+		DEFINE_EVENT(cma_qp_class, cm_##name,				\</div><div class='add'>+				TP_PROTO(					\</div><div class='add'>+					const struct rdma_id_private *id_priv	\</div><div class='add'>+				),						\</div><div class='add'>+				TP_ARGS(id_priv))</div><div class='add'>+</div><div class='add'>+DEFINE_CMA_QP_EVENT(send_req);</div><div class='add'>+DEFINE_CMA_QP_EVENT(send_rep);</div><div class='add'>+DEFINE_CMA_QP_EVENT(qp_destroy);</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * enum ib_wp_type, from include/rdma/ib_verbs.h</div><div class='add'>+ */</div><div class='add'>+#define IB_QP_TYPE_LIST				\</div><div class='add'>+	ib_qp_type(SMI)				\</div><div class='add'>+	ib_qp_type(GSI)				\</div><div class='add'>+	ib_qp_type(RC)				\</div><div class='add'>+	ib_qp_type(UC)				\</div><div class='add'>+	ib_qp_type(UD)				\</div><div class='add'>+	ib_qp_type(RAW_IPV6)			\</div><div class='add'>+	ib_qp_type(RAW_ETHERTYPE)		\</div><div class='add'>+	ib_qp_type(RAW_PACKET)			\</div><div class='add'>+	ib_qp_type(XRC_INI)			\</div><div class='add'>+	ib_qp_type_end(XRC_TGT)</div><div class='add'>+</div><div class='add'>+#undef ib_qp_type</div><div class='add'>+#undef ib_qp_type_end</div><div class='add'>+</div><div class='add'>+#define ib_qp_type(x)		TRACE_DEFINE_ENUM(IB_QPT_##x);</div><div class='add'>+#define ib_qp_type_end(x)	TRACE_DEFINE_ENUM(IB_QPT_##x);</div><div class='add'>+</div><div class='add'>+IB_QP_TYPE_LIST</div><div class='add'>+</div><div class='add'>+#undef ib_qp_type</div><div class='add'>+#undef ib_qp_type_end</div><div class='add'>+</div><div class='add'>+#define ib_qp_type(x)		{ IB_QPT_##x, #x },</div><div class='add'>+#define ib_qp_type_end(x)	{ IB_QPT_##x, #x }</div><div class='add'>+</div><div class='add'>+#define rdma_show_qp_type(x) \</div><div class='add'>+		__print_symbolic(x, IB_QP_TYPE_LIST)</div><div class='add'>+</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(cm_qp_create,</div><div class='add'>+	TP_PROTO(</div><div class='add'>+		const struct rdma_id_private *id_priv,</div><div class='add'>+		const struct ib_pd *pd,</div><div class='add'>+		const struct ib_qp_init_attr *qp_init_attr,</div><div class='add'>+		int rc</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_ARGS(id_priv, pd, qp_init_attr, rc),</div><div class='add'>+</div><div class='add'>+	TP_STRUCT__entry(</div><div class='add'>+		__field(u32, cm_id)</div><div class='add'>+		__field(u32, pd_id)</div><div class='add'>+		__field(u32, tos)</div><div class='add'>+		__field(u32, qp_num)</div><div class='add'>+		__field(u32, send_wr)</div><div class='add'>+		__field(u32, recv_wr)</div><div class='add'>+		__field(int, rc)</div><div class='add'>+		__field(unsigned long, qp_type)</div><div class='add'>+		__array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))</div><div class='add'>+		__array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_fast_assign(</div><div class='add'>+		__entry-&gt;cm_id = id_priv-&gt;res.id;</div><div class='add'>+		__entry-&gt;pd_id = pd-&gt;res.id;</div><div class='add'>+		__entry-&gt;tos = id_priv-&gt;tos;</div><div class='add'>+		__entry-&gt;send_wr = qp_init_attr-&gt;cap.max_send_wr;</div><div class='add'>+		__entry-&gt;recv_wr = qp_init_attr-&gt;cap.max_recv_wr;</div><div class='add'>+		__entry-&gt;rc = rc;</div><div class='add'>+		if (!rc) {</div><div class='add'>+			__entry-&gt;qp_num = id_priv-&gt;qp_num;</div><div class='add'>+			__entry-&gt;qp_type = id_priv-&gt;id.qp_type;</div><div class='add'>+		} else {</div><div class='add'>+			__entry-&gt;qp_num = 0;</div><div class='add'>+			__entry-&gt;qp_type = 0;</div><div class='add'>+		}</div><div class='add'>+		memcpy(__entry-&gt;srcaddr, &amp;id_priv-&gt;id.route.addr.src_addr,</div><div class='add'>+		       sizeof(struct sockaddr_in6));</div><div class='add'>+		memcpy(__entry-&gt;dstaddr, &amp;id_priv-&gt;id.route.addr.dst_addr,</div><div class='add'>+		       sizeof(struct sockaddr_in6));</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u pd.id=%u qp_type=%s"</div><div class='add'>+		" send_wr=%u recv_wr=%u qp_num=%u rc=%d",</div><div class='add'>+		__entry-&gt;cm_id, __entry-&gt;srcaddr, __entry-&gt;dstaddr,</div><div class='add'>+		__entry-&gt;tos, __entry-&gt;pd_id,</div><div class='add'>+		rdma_show_qp_type(__entry-&gt;qp_type), __entry-&gt;send_wr,</div><div class='add'>+		__entry-&gt;recv_wr, __entry-&gt;qp_num, __entry-&gt;rc</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(cm_req_handler,</div><div class='add'>+	TP_PROTO(</div><div class='add'>+		const struct rdma_id_private *id_priv,</div><div class='add'>+		int event</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_ARGS(id_priv, event),</div><div class='add'>+</div><div class='add'>+	TP_STRUCT__entry(</div><div class='add'>+		__field(u32, cm_id)</div><div class='add'>+		__field(u32, tos)</div><div class='add'>+		__field(unsigned long, event)</div><div class='add'>+		__array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))</div><div class='add'>+		__array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_fast_assign(</div><div class='add'>+		__entry-&gt;cm_id = id_priv-&gt;res.id;</div><div class='add'>+		__entry-&gt;tos = id_priv-&gt;tos;</div><div class='add'>+		__entry-&gt;event = event;</div><div class='add'>+		memcpy(__entry-&gt;srcaddr, &amp;id_priv-&gt;id.route.addr.src_addr,</div><div class='add'>+		       sizeof(struct sockaddr_in6));</div><div class='add'>+		memcpy(__entry-&gt;dstaddr, &amp;id_priv-&gt;id.route.addr.dst_addr,</div><div class='add'>+		       sizeof(struct sockaddr_in6));</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u %s (%lu)",</div><div class='add'>+		__entry-&gt;cm_id, __entry-&gt;srcaddr, __entry-&gt;dstaddr, __entry-&gt;tos,</div><div class='add'>+		rdma_show_ib_cm_event(__entry-&gt;event), __entry-&gt;event</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(cm_event_handler,</div><div class='add'>+	TP_PROTO(</div><div class='add'>+		const struct rdma_id_private *id_priv,</div><div class='add'>+		const struct rdma_cm_event *event</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_ARGS(id_priv, event),</div><div class='add'>+</div><div class='add'>+	TP_STRUCT__entry(</div><div class='add'>+		__field(u32, cm_id)</div><div class='add'>+		__field(u32, tos)</div><div class='add'>+		__field(unsigned long, event)</div><div class='add'>+		__field(int, status)</div><div class='add'>+		__array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))</div><div class='add'>+		__array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_fast_assign(</div><div class='add'>+		__entry-&gt;cm_id = id_priv-&gt;res.id;</div><div class='add'>+		__entry-&gt;tos = id_priv-&gt;tos;</div><div class='add'>+		__entry-&gt;event = event-&gt;event;</div><div class='add'>+		__entry-&gt;status = event-&gt;status;</div><div class='add'>+		memcpy(__entry-&gt;srcaddr, &amp;id_priv-&gt;id.route.addr.src_addr,</div><div class='add'>+		       sizeof(struct sockaddr_in6));</div><div class='add'>+		memcpy(__entry-&gt;dstaddr, &amp;id_priv-&gt;id.route.addr.dst_addr,</div><div class='add'>+		       sizeof(struct sockaddr_in6));</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u %s (%lu/%d)",</div><div class='add'>+		__entry-&gt;cm_id, __entry-&gt;srcaddr, __entry-&gt;dstaddr, __entry-&gt;tos,</div><div class='add'>+		rdma_show_cm_event(__entry-&gt;event), __entry-&gt;event,</div><div class='add'>+		__entry-&gt;status</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(cm_event_done,</div><div class='add'>+	TP_PROTO(</div><div class='add'>+		const struct rdma_id_private *id_priv,</div><div class='add'>+		const struct rdma_cm_event *event,</div><div class='add'>+		int result</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_ARGS(id_priv, event, result),</div><div class='add'>+</div><div class='add'>+	TP_STRUCT__entry(</div><div class='add'>+		__field(u32, cm_id)</div><div class='add'>+		__field(u32, tos)</div><div class='add'>+		__field(unsigned long, event)</div><div class='add'>+		__field(int, result)</div><div class='add'>+		__array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))</div><div class='add'>+		__array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_fast_assign(</div><div class='add'>+		__entry-&gt;cm_id = id_priv-&gt;res.id;</div><div class='add'>+		__entry-&gt;tos = id_priv-&gt;tos;</div><div class='add'>+		__entry-&gt;event = event-&gt;event;</div><div class='add'>+		__entry-&gt;result = result;</div><div class='add'>+		memcpy(__entry-&gt;srcaddr, &amp;id_priv-&gt;id.route.addr.src_addr,</div><div class='add'>+		       sizeof(struct sockaddr_in6));</div><div class='add'>+		memcpy(__entry-&gt;dstaddr, &amp;id_priv-&gt;id.route.addr.dst_addr,</div><div class='add'>+		       sizeof(struct sockaddr_in6));</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u %s consumer returns %d",</div><div class='add'>+		__entry-&gt;cm_id, __entry-&gt;srcaddr, __entry-&gt;dstaddr, __entry-&gt;tos,</div><div class='add'>+		rdma_show_cm_event(__entry-&gt;event), __entry-&gt;result</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DECLARE_EVENT_CLASS(cma_client_class,</div><div class='add'>+	TP_PROTO(</div><div class='add'>+		const struct ib_device *device</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_ARGS(device),</div><div class='add'>+</div><div class='add'>+	TP_STRUCT__entry(</div><div class='add'>+		__string(name, device-&gt;name)</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_fast_assign(</div><div class='add'>+		__assign_str(name);</div><div class='add'>+	),</div><div class='add'>+</div><div class='add'>+	TP_printk("device name=%s",</div><div class='add'>+		__get_str(name)</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+#define DEFINE_CMA_CLIENT_EVENT(name)						\</div><div class='add'>+		DEFINE_EVENT(cma_client_class, cm_##name,			\</div><div class='add'>+				TP_PROTO(					\</div><div class='add'>+					const struct ib_device *device		\</div><div class='add'>+				),						\</div><div class='add'>+				TP_ARGS(device))</div><div class='add'>+</div><div class='add'>+DEFINE_CMA_CLIENT_EVENT(add_one);</div><div class='add'>+DEFINE_CMA_CLIENT_EVENT(remove_one);</div><div class='add'>+</div><div class='add'>+#endif /* _TRACE_RDMA_CMA_H */</div><div class='add'>+</div><div class='add'>+#undef TRACE_INCLUDE_PATH</div><div class='add'>+#define TRACE_INCLUDE_PATH .</div><div class='add'>+#define TRACE_INCLUDE_FILE cma_trace</div><div class='add'>+</div><div class='add'>+#include &lt;trace/define_trace.h&gt;</div><div class='head'>diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h<br/>index ff40a450b5d2..05102769a918 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/core_priv.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/core_priv.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/core_priv.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/core_priv.h</a></div><div class='hunk'>@@ -36,12 +36,15 @@</div><div class='ctx'> #include &lt;linux/list.h&gt;</div><div class='ctx'> #include &lt;linux/spinlock.h&gt;</div><div class='ctx'> #include &lt;linux/cgroup_rdma.h&gt;</div><div class='add'>+#include &lt;net/net_namespace.h&gt;</div><div class='add'>+#include &lt;net/netns/generic.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include &lt;rdma/ib_verbs.h&gt;</div><div class='ctx'> #include &lt;rdma/opa_addr.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_mad.h&gt;</div><div class='ctx'> #include &lt;rdma/restrack.h&gt;</div><div class='ctx'> #include "mad_priv.h"</div><div class='add'>+#include "restrack.h"</div><div class='ctx'> </div><div class='ctx'> /* Total number of ports combined across all struct ib_devices's */</div><div class='ctx'> #define RDMA_MAX_PORTS 8192</div><div class='hunk'>@@ -54,21 +57,38 @@ struct pkey_index_qp_list {</div><div class='ctx'> 	struct list_head    qp_list;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+/**</div><div class='add'>+ * struct rdma_dev_net - rdma net namespace metadata for a net</div><div class='add'>+ * @nl_sock:	Pointer to netlink socket</div><div class='add'>+ * @net:	Pointer to owner net namespace</div><div class='add'>+ * @id:		xarray id to identify the net namespace.</div><div class='add'>+ */</div><div class='add'>+struct rdma_dev_net {</div><div class='add'>+	struct sock *nl_sock;</div><div class='add'>+	possible_net_t net;</div><div class='add'>+	u32 id;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> extern const struct attribute_group ib_dev_attr_group;</div><div class='ctx'> extern bool ib_devices_shared_netns;</div><div class='add'>+extern unsigned int rdma_dev_net_id;</div><div class='add'>+</div><div class='add'>+static inline struct rdma_dev_net *rdma_net_to_dev_net(struct net *net)</div><div class='add'>+{</div><div class='add'>+	return net_generic(net, rdma_dev_net_id);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-int ib_device_register_sysfs(struct ib_device *device);</div><div class='del'>-void ib_device_unregister_sysfs(struct ib_device *device);</div><div class='ctx'> int ib_device_rename(struct ib_device *ibdev, const char *name);</div><div class='add'>+int ib_device_set_dim(struct ib_device *ibdev, u8 use_dim);</div><div class='ctx'> </div><div class='del'>-typedef void (*roce_netdev_callback)(struct ib_device *device, u8 port,</div><div class='add'>+typedef void (*roce_netdev_callback)(struct ib_device *device, u32 port,</div><div class='ctx'> 	      struct net_device *idev, void *cookie);</div><div class='ctx'> </div><div class='del'>-typedef bool (*roce_netdev_filter)(struct ib_device *device, u8 port,</div><div class='add'>+typedef bool (*roce_netdev_filter)(struct ib_device *device, u32 port,</div><div class='ctx'> 				   struct net_device *idev, void *cookie);</div><div class='ctx'> </div><div class='ctx'> struct net_device *ib_device_get_netdev(struct ib_device *ib_dev,</div><div class='del'>-					unsigned int port);</div><div class='add'>+					u32 port);</div><div class='ctx'> </div><div class='ctx'> void ib_enum_roce_netdev(struct ib_device *ib_dev,</div><div class='ctx'> 			 roce_netdev_filter filter,</div><div class='hunk'>@@ -88,6 +108,15 @@ typedef int (*nldev_callback)(struct ib_device *device,</div><div class='ctx'> int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb,</div><div class='ctx'> 		     struct netlink_callback *cb);</div><div class='ctx'> </div><div class='add'>+struct ib_client_nl_info {</div><div class='add'>+	struct sk_buff *nl_msg;</div><div class='add'>+	struct device *cdev;</div><div class='add'>+	u32 port;</div><div class='add'>+	u64 abi;</div><div class='add'>+};</div><div class='add'>+int ib_get_client_nl_info(struct ib_device *ibdev, const char *client_name,</div><div class='add'>+			  struct ib_client_nl_info *res);</div><div class='add'>+</div><div class='ctx'> enum ib_cache_gid_default_mode {</div><div class='ctx'> 	IB_CACHE_GID_DEFAULT_MODE_SET,</div><div class='ctx'> 	IB_CACHE_GID_DEFAULT_MODE_DELETE</div><div class='hunk'>@@ -97,28 +126,29 @@ int ib_cache_gid_parse_type_str(const char *buf);</div><div class='ctx'> </div><div class='ctx'> const char *ib_cache_gid_type_str(enum ib_gid_type gid_type);</div><div class='ctx'> </div><div class='del'>-void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u8 port,</div><div class='add'>+void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 				  struct net_device *ndev,</div><div class='ctx'> 				  unsigned long gid_type_mask,</div><div class='ctx'> 				  enum ib_cache_gid_default_mode mode);</div><div class='ctx'> </div><div class='del'>-int ib_cache_gid_add(struct ib_device *ib_dev, u8 port,</div><div class='add'>+int ib_cache_gid_add(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 		     union ib_gid *gid, struct ib_gid_attr *attr);</div><div class='ctx'> </div><div class='del'>-int ib_cache_gid_del(struct ib_device *ib_dev, u8 port,</div><div class='add'>+int ib_cache_gid_del(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 		     union ib_gid *gid, struct ib_gid_attr *attr);</div><div class='ctx'> </div><div class='del'>-int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u8 port,</div><div class='add'>+int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 				     struct net_device *ndev);</div><div class='ctx'> </div><div class='ctx'> int roce_gid_mgmt_init(void);</div><div class='ctx'> void roce_gid_mgmt_cleanup(void);</div><div class='ctx'> </div><div class='del'>-unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u8 port);</div><div class='add'>+unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u32 port);</div><div class='ctx'> </div><div class='ctx'> int ib_cache_setup_one(struct ib_device *device);</div><div class='ctx'> void ib_cache_cleanup_one(struct ib_device *device);</div><div class='ctx'> void ib_cache_release_one(struct ib_device *device);</div><div class='add'>+void ib_dispatch_event_clients(struct ib_event *event);</div><div class='ctx'> </div><div class='ctx'> #ifdef CONFIG_CGROUP_RDMA</div><div class='ctx'> void ib_device_register_rdmacg(struct ib_device *device);</div><div class='hunk'>@@ -169,7 +199,7 @@ void ib_mad_cleanup(void);</div><div class='ctx'> int ib_sa_init(void);</div><div class='ctx'> void ib_sa_cleanup(void);</div><div class='ctx'> </div><div class='del'>-int rdma_nl_init(void);</div><div class='add'>+void rdma_nl_init(void);</div><div class='ctx'> void rdma_nl_exit(void);</div><div class='ctx'> </div><div class='ctx'> int ib_nl_handle_resolve_resp(struct sk_buff *skb,</div><div class='hunk'>@@ -182,15 +212,15 @@ int ib_nl_handle_ip_res_resp(struct sk_buff *skb,</div><div class='ctx'> 			     struct nlmsghdr *nlh,</div><div class='ctx'> 			     struct netlink_ext_ack *extack);</div><div class='ctx'> </div><div class='del'>-int ib_get_cached_subnet_prefix(struct ib_device *device,</div><div class='del'>-				u8                port_num,</div><div class='del'>-				u64              *sn_pfx);</div><div class='add'>+void ib_get_cached_subnet_prefix(struct ib_device *device,</div><div class='add'>+				u32 port_num,</div><div class='add'>+				u64 *sn_pfx);</div><div class='ctx'> </div><div class='ctx'> #ifdef CONFIG_SECURITY_INFINIBAND</div><div class='ctx'> void ib_security_release_port_pkey_list(struct ib_device *device);</div><div class='ctx'> </div><div class='ctx'> void ib_security_cache_change(struct ib_device *device,</div><div class='del'>-			      u8 port_num,</div><div class='add'>+			      u32 port_num,</div><div class='ctx'> 			      u64 subnet_prefix);</div><div class='ctx'> </div><div class='ctx'> int ib_security_modify_qp(struct ib_qp *qp,</div><div class='hunk'>@@ -215,7 +245,7 @@ static inline void ib_security_release_port_pkey_list(struct ib_device *device)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static inline void ib_security_cache_change(struct ib_device *device,</div><div class='del'>-					    u8 port_num,</div><div class='add'>+					    u32 port_num,</div><div class='ctx'> 					    u64 subnet_prefix)</div><div class='ctx'> {</div><div class='ctx'> }</div><div class='hunk'>@@ -286,46 +316,15 @@ struct ib_device *ib_device_get_by_index(const struct net *net, u32 index);</div><div class='ctx'> void nldev_init(void);</div><div class='ctx'> void nldev_exit(void);</div><div class='ctx'> </div><div class='del'>-static inline struct ib_qp *_ib_create_qp(struct ib_device *dev,</div><div class='del'>-					  struct ib_pd *pd,</div><div class='del'>-					  struct ib_qp_init_attr *attr,</div><div class='del'>-					  struct ib_udata *udata,</div><div class='del'>-					  struct ib_uobject *uobj)</div><div class='del'>-{</div><div class='del'>-	struct ib_qp *qp;</div><div class='del'>-</div><div class='del'>-	if (!dev-&gt;ops.create_qp)</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='del'>-</div><div class='del'>-	qp = dev-&gt;ops.create_qp(pd, attr, udata);</div><div class='del'>-	if (IS_ERR(qp))</div><div class='del'>-		return qp;</div><div class='del'>-</div><div class='del'>-	qp-&gt;device = dev;</div><div class='del'>-	qp-&gt;pd = pd;</div><div class='del'>-	qp-&gt;uobject = uobj;</div><div class='del'>-	qp-&gt;real_qp = qp;</div><div class='del'>-	/*</div><div class='del'>-	 * We don't track XRC QPs for now, because they don't have PD</div><div class='del'>-	 * and more importantly they are created internaly by driver,</div><div class='del'>-	 * see mlx5 create_dev_resources() as an example.</div><div class='del'>-	 */</div><div class='del'>-	if (attr-&gt;qp_type &lt; IB_QPT_XRC_INI) {</div><div class='del'>-		qp-&gt;res.type = RDMA_RESTRACK_QP;</div><div class='del'>-		if (uobj)</div><div class='del'>-			rdma_restrack_uadd(&amp;qp-&gt;res);</div><div class='del'>-		else</div><div class='del'>-			rdma_restrack_kadd(&amp;qp-&gt;res);</div><div class='del'>-	} else</div><div class='del'>-		qp-&gt;res.valid = false;</div><div class='del'>-</div><div class='del'>-	return qp;</div><div class='del'>-}</div><div class='add'>+struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd,</div><div class='add'>+				struct ib_qp_init_attr *attr,</div><div class='add'>+				struct ib_udata *udata,</div><div class='add'>+				struct ib_uqp_object *uobj, const char *caller);</div><div class='add'>+</div><div class='add'>+void ib_qp_usecnt_inc(struct ib_qp *qp);</div><div class='add'>+void ib_qp_usecnt_dec(struct ib_qp *qp);</div><div class='ctx'> </div><div class='ctx'> struct rdma_dev_addr;</div><div class='del'>-int rdma_resolve_ip_route(struct sockaddr *src_addr,</div><div class='del'>-			  const struct sockaddr *dst_addr,</div><div class='del'>-			  struct rdma_dev_addr *addr);</div><div class='ctx'> </div><div class='ctx'> int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid,</div><div class='ctx'> 				 const union ib_gid *dgid,</div><div class='hunk'>@@ -342,14 +341,34 @@ struct net_device *rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr);</div><div class='ctx'> </div><div class='ctx'> void ib_free_port_attrs(struct ib_core_device *coredev);</div><div class='ctx'> int ib_setup_port_attrs(struct ib_core_device *coredev);</div><div class='add'>+struct rdma_hw_stats *ib_get_hw_stats_port(struct ib_device *ibdev, u32 port_num);</div><div class='add'>+void ib_device_release_hw_stats(struct hw_stats_device_data *data);</div><div class='add'>+int ib_setup_device_attrs(struct ib_device *ibdev);</div><div class='ctx'> </div><div class='ctx'> int rdma_compatdev_set(u8 enable);</div><div class='ctx'> </div><div class='del'>-int ib_port_register_module_stat(struct ib_device *device, u8 port_num,</div><div class='del'>-				 struct kobject *kobj, struct kobj_type *ktype,</div><div class='del'>-				 const char *name);</div><div class='del'>-void ib_port_unregister_module_stat(struct kobject *kobj);</div><div class='add'>+int ib_port_register_client_groups(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+				   const struct attribute_group **groups);</div><div class='add'>+void ib_port_unregister_client_groups(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+				     const struct attribute_group **groups);</div><div class='ctx'> </div><div class='ctx'> int ib_device_set_netns_put(struct sk_buff *skb,</div><div class='ctx'> 			    struct ib_device *dev, u32 ns_fd);</div><div class='add'>+</div><div class='add'>+int rdma_nl_net_init(struct rdma_dev_net *rnet);</div><div class='add'>+void rdma_nl_net_exit(struct rdma_dev_net *rnet);</div><div class='add'>+</div><div class='add'>+struct rdma_umap_priv {</div><div class='add'>+	struct vm_area_struct *vma;</div><div class='add'>+	struct list_head list;</div><div class='add'>+	struct rdma_user_mmap_entry *entry;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+void rdma_umap_priv_init(struct rdma_umap_priv *priv,</div><div class='add'>+			 struct vm_area_struct *vma,</div><div class='add'>+			 struct rdma_user_mmap_entry *entry);</div><div class='add'>+</div><div class='add'>+void ib_cq_pool_cleanup(struct ib_device *dev);</div><div class='add'>+</div><div class='add'>+bool rdma_nl_get_privileged_qkey(void);</div><div class='ctx'> #endif /* _CORE_PRIV_H */</div><div class='head'>diff --git a/drivers/infiniband/core/counters.c b/drivers/infiniband/core/counters.c<br/>new file mode 100644<br/>index 000000000000..c3aa6d7fc66b<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/counters.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/counters.c</a></div><div class='hunk'>@@ -0,0 +1,681 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2019 Mellanox Technologies. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+#include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+#include &lt;rdma/rdma_counter.h&gt;</div><div class='add'>+</div><div class='add'>+#include "core_priv.h"</div><div class='add'>+#include "restrack.h"</div><div class='add'>+</div><div class='add'>+#define ALL_AUTO_MODE_MASKS (RDMA_COUNTER_MASK_QP_TYPE | RDMA_COUNTER_MASK_PID)</div><div class='add'>+</div><div class='add'>+static int __counter_set_mode(struct rdma_port_counter *port_counter,</div><div class='add'>+			      enum rdma_nl_counter_mode new_mode,</div><div class='add'>+			      enum rdma_nl_counter_mask new_mask,</div><div class='add'>+			      bool bind_opcnt)</div><div class='add'>+{</div><div class='add'>+	if (new_mode == RDMA_COUNTER_MODE_AUTO) {</div><div class='add'>+		if (new_mask &amp; (~ALL_AUTO_MODE_MASKS))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		if (port_counter-&gt;num_counters)</div><div class='add'>+			return -EBUSY;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	port_counter-&gt;mode.mode = new_mode;</div><div class='add'>+	port_counter-&gt;mode.mask = new_mask;</div><div class='add'>+	port_counter-&gt;mode.bind_opcnt = bind_opcnt;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * rdma_counter_set_auto_mode() - Turn on/off per-port auto mode</div><div class='add'>+ *</div><div class='add'>+ * @dev: Device to operate</div><div class='add'>+ * @port: Port to use</div><div class='add'>+ * @mask: Mask to configure</div><div class='add'>+ * @extack: Message to the user</div><div class='add'>+ *</div><div class='add'>+ * Return 0 on success. If counter mode wasn't changed then it is considered</div><div class='add'>+ * as success as well.</div><div class='add'>+ * Return -EBUSY when changing to auto mode while there are bounded counters.</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+int rdma_counter_set_auto_mode(struct ib_device *dev, u32 port,</div><div class='add'>+			       enum rdma_nl_counter_mask mask,</div><div class='add'>+			       bool bind_opcnt,</div><div class='add'>+			       struct netlink_ext_ack *extack)</div><div class='add'>+{</div><div class='add'>+	struct rdma_port_counter *port_counter;</div><div class='add'>+	enum rdma_nl_counter_mode mode;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	port_counter = &amp;dev-&gt;port_data[port].port_counter;</div><div class='add'>+	if (!port_counter-&gt;hstats)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;port_counter-&gt;lock);</div><div class='add'>+	if (mask)</div><div class='add'>+		mode = RDMA_COUNTER_MODE_AUTO;</div><div class='add'>+	else</div><div class='add'>+		mode = (port_counter-&gt;num_counters) ? RDMA_COUNTER_MODE_MANUAL :</div><div class='add'>+						      RDMA_COUNTER_MODE_NONE;</div><div class='add'>+</div><div class='add'>+	if (port_counter-&gt;mode.mode == mode &amp;&amp;</div><div class='add'>+	    port_counter-&gt;mode.mask == mask &amp;&amp;</div><div class='add'>+	    port_counter-&gt;mode.bind_opcnt == bind_opcnt) {</div><div class='add'>+		ret = 0;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = __counter_set_mode(port_counter, mode, mask, bind_opcnt);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	mutex_unlock(&amp;port_counter-&gt;lock);</div><div class='add'>+	if (ret == -EBUSY)</div><div class='add'>+		NL_SET_ERR_MSG(</div><div class='add'>+			extack,</div><div class='add'>+			"Modifying auto mode is not allowed when there is a bound QP");</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void auto_mode_init_counter(struct rdma_counter *counter,</div><div class='add'>+				   const struct ib_qp *qp,</div><div class='add'>+				   enum rdma_nl_counter_mask new_mask)</div><div class='add'>+{</div><div class='add'>+	struct auto_mode_param *param = &amp;counter-&gt;mode.param;</div><div class='add'>+</div><div class='add'>+	counter-&gt;mode.mode = RDMA_COUNTER_MODE_AUTO;</div><div class='add'>+	counter-&gt;mode.mask = new_mask;</div><div class='add'>+</div><div class='add'>+	if (new_mask &amp; RDMA_COUNTER_MASK_QP_TYPE)</div><div class='add'>+		param-&gt;qp_type = qp-&gt;qp_type;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int __rdma_counter_bind_qp(struct rdma_counter *counter,</div><div class='add'>+				  struct ib_qp *qp, u32 port)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;counter)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!qp-&gt;device-&gt;ops.counter_bind_qp)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;counter-&gt;lock);</div><div class='add'>+	ret = qp-&gt;device-&gt;ops.counter_bind_qp(counter, qp, port);</div><div class='add'>+	mutex_unlock(&amp;counter-&gt;lock);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int rdma_counter_modify(struct ib_device *dev, u32 port,</div><div class='add'>+			unsigned int index, bool enable)</div><div class='add'>+{</div><div class='add'>+	struct rdma_hw_stats *stats;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;ops.modify_hw_stat)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	stats = ib_get_hw_stats_port(dev, port);</div><div class='add'>+	if (!stats || index &gt;= stats-&gt;num_counters ||</div><div class='add'>+	    !(stats-&gt;descs[index].flags &amp; IB_STAT_FLAG_OPTIONAL))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;stats-&gt;lock);</div><div class='add'>+</div><div class='add'>+	if (enable != test_bit(index, stats-&gt;is_disabled))</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	ret = dev-&gt;ops.modify_hw_stat(dev, port, index, enable);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	if (enable)</div><div class='add'>+		clear_bit(index, stats-&gt;is_disabled);</div><div class='add'>+	else</div><div class='add'>+		set_bit(index, stats-&gt;is_disabled);</div><div class='add'>+out:</div><div class='add'>+	mutex_unlock(&amp;stats-&gt;lock);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct rdma_counter *alloc_and_bind(struct ib_device *dev, u32 port,</div><div class='add'>+					   struct ib_qp *qp,</div><div class='add'>+					   enum rdma_nl_counter_mode mode,</div><div class='add'>+					   bool bind_opcnt)</div><div class='add'>+{</div><div class='add'>+	struct rdma_port_counter *port_counter;</div><div class='add'>+	struct rdma_counter *counter;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;ops.counter_dealloc || !dev-&gt;ops.counter_alloc_stats)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	counter = rdma_zalloc_drv_obj(dev, rdma_counter);</div><div class='add'>+	if (!counter)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	counter-&gt;device    = dev;</div><div class='add'>+	counter-&gt;port      = port;</div><div class='add'>+</div><div class='add'>+	dev-&gt;ops.counter_init(counter);</div><div class='add'>+</div><div class='add'>+	rdma_restrack_new(&amp;counter-&gt;res, RDMA_RESTRACK_COUNTER);</div><div class='add'>+	counter-&gt;stats = dev-&gt;ops.counter_alloc_stats(counter);</div><div class='add'>+	if (!counter-&gt;stats)</div><div class='add'>+		goto err_stats;</div><div class='add'>+</div><div class='add'>+	port_counter = &amp;dev-&gt;port_data[port].port_counter;</div><div class='add'>+	mutex_lock(&amp;port_counter-&gt;lock);</div><div class='add'>+	switch (mode) {</div><div class='add'>+	case RDMA_COUNTER_MODE_MANUAL:</div><div class='add'>+		ret = __counter_set_mode(port_counter, RDMA_COUNTER_MODE_MANUAL,</div><div class='add'>+					 0, bind_opcnt);</div><div class='add'>+		if (ret) {</div><div class='add'>+			mutex_unlock(&amp;port_counter-&gt;lock);</div><div class='add'>+			goto err_mode;</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	case RDMA_COUNTER_MODE_AUTO:</div><div class='add'>+		auto_mode_init_counter(counter, qp, port_counter-&gt;mode.mask);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ret = -EOPNOTSUPP;</div><div class='add'>+		mutex_unlock(&amp;port_counter-&gt;lock);</div><div class='add'>+		goto err_mode;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	port_counter-&gt;num_counters++;</div><div class='add'>+	mutex_unlock(&amp;port_counter-&gt;lock);</div><div class='add'>+</div><div class='add'>+	counter-&gt;mode.mode = mode;</div><div class='add'>+	counter-&gt;mode.bind_opcnt = bind_opcnt;</div><div class='add'>+	kref_init(&amp;counter-&gt;kref);</div><div class='add'>+	mutex_init(&amp;counter-&gt;lock);</div><div class='add'>+</div><div class='add'>+	ret = __rdma_counter_bind_qp(counter, qp, port);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_mode;</div><div class='add'>+</div><div class='add'>+	rdma_restrack_parent_name(&amp;counter-&gt;res, &amp;qp-&gt;res);</div><div class='add'>+	rdma_restrack_add(&amp;counter-&gt;res);</div><div class='add'>+	return counter;</div><div class='add'>+</div><div class='add'>+err_mode:</div><div class='add'>+	rdma_free_hw_stats_struct(counter-&gt;stats);</div><div class='add'>+err_stats:</div><div class='add'>+	rdma_restrack_put(&amp;counter-&gt;res);</div><div class='add'>+	kfree(counter);</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void rdma_counter_free(struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	struct rdma_port_counter *port_counter;</div><div class='add'>+</div><div class='add'>+	port_counter = &amp;counter-&gt;device-&gt;port_data[counter-&gt;port].port_counter;</div><div class='add'>+	mutex_lock(&amp;port_counter-&gt;lock);</div><div class='add'>+	port_counter-&gt;num_counters--;</div><div class='add'>+	if (!port_counter-&gt;num_counters &amp;&amp;</div><div class='add'>+	    (port_counter-&gt;mode.mode == RDMA_COUNTER_MODE_MANUAL))</div><div class='add'>+		__counter_set_mode(port_counter, RDMA_COUNTER_MODE_NONE, 0,</div><div class='add'>+				   false);</div><div class='add'>+</div><div class='add'>+	mutex_unlock(&amp;port_counter-&gt;lock);</div><div class='add'>+</div><div class='add'>+	rdma_restrack_del(&amp;counter-&gt;res);</div><div class='add'>+	rdma_free_hw_stats_struct(counter-&gt;stats);</div><div class='add'>+	kfree(counter);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter,</div><div class='add'>+			    enum rdma_nl_counter_mask auto_mask)</div><div class='add'>+{</div><div class='add'>+	struct auto_mode_param *param = &amp;counter-&gt;mode.param;</div><div class='add'>+	bool match = true;</div><div class='add'>+</div><div class='add'>+	if (auto_mask &amp; RDMA_COUNTER_MASK_QP_TYPE)</div><div class='add'>+		match &amp;= (param-&gt;qp_type == qp-&gt;qp_type);</div><div class='add'>+</div><div class='add'>+	if (auto_mask &amp; RDMA_COUNTER_MASK_PID)</div><div class='add'>+		match &amp;= (task_pid_nr(counter-&gt;res.task) ==</div><div class='add'>+			  task_pid_nr(qp-&gt;res.task));</div><div class='add'>+</div><div class='add'>+	return match;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int __rdma_counter_unbind_qp(struct ib_qp *qp, u32 port)</div><div class='add'>+{</div><div class='add'>+	struct rdma_counter *counter = qp-&gt;counter;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!qp-&gt;device-&gt;ops.counter_unbind_qp)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;counter-&gt;lock);</div><div class='add'>+	ret = qp-&gt;device-&gt;ops.counter_unbind_qp(qp, port);</div><div class='add'>+	mutex_unlock(&amp;counter-&gt;lock);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void counter_history_stat_update(struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *dev = counter-&gt;device;</div><div class='add'>+	struct rdma_port_counter *port_counter;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	port_counter = &amp;dev-&gt;port_data[counter-&gt;port].port_counter;</div><div class='add'>+	if (!port_counter-&gt;hstats)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	rdma_counter_query_stats(counter);</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; counter-&gt;stats-&gt;num_counters; i++)</div><div class='add'>+		port_counter-&gt;hstats-&gt;value[i] += counter-&gt;stats-&gt;value[i];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * rdma_get_counter_auto_mode - Find the counter that @qp should be bound</div><div class='add'>+ *     with in auto mode</div><div class='add'>+ *</div><div class='add'>+ * Return: The counter (with ref-count increased) if found</div><div class='add'>+ */</div><div class='add'>+static struct rdma_counter *rdma_get_counter_auto_mode(struct ib_qp *qp,</div><div class='add'>+						       u32 port)</div><div class='add'>+{</div><div class='add'>+	struct rdma_port_counter *port_counter;</div><div class='add'>+	struct rdma_counter *counter = NULL;</div><div class='add'>+	struct ib_device *dev = qp-&gt;device;</div><div class='add'>+	struct rdma_restrack_entry *res;</div><div class='add'>+	struct rdma_restrack_root *rt;</div><div class='add'>+	unsigned long id = 0;</div><div class='add'>+</div><div class='add'>+	port_counter = &amp;dev-&gt;port_data[port].port_counter;</div><div class='add'>+	rt = &amp;dev-&gt;res[RDMA_RESTRACK_COUNTER];</div><div class='add'>+	xa_lock(&amp;rt-&gt;xa);</div><div class='add'>+	xa_for_each(&amp;rt-&gt;xa, id, res) {</div><div class='add'>+		counter = container_of(res, struct rdma_counter, res);</div><div class='add'>+		if ((counter-&gt;device != qp-&gt;device) || (counter-&gt;port != port))</div><div class='add'>+			goto next;</div><div class='add'>+</div><div class='add'>+		if (auto_mode_match(qp, counter, port_counter-&gt;mode.mask))</div><div class='add'>+			break;</div><div class='add'>+next:</div><div class='add'>+		counter = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (counter &amp;&amp; !kref_get_unless_zero(&amp;counter-&gt;kref))</div><div class='add'>+		counter = NULL;</div><div class='add'>+</div><div class='add'>+	xa_unlock(&amp;rt-&gt;xa);</div><div class='add'>+	return counter;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void counter_release(struct kref *kref)</div><div class='add'>+{</div><div class='add'>+	struct rdma_counter *counter;</div><div class='add'>+</div><div class='add'>+	counter = container_of(kref, struct rdma_counter, kref);</div><div class='add'>+	counter_history_stat_update(counter);</div><div class='add'>+	counter-&gt;device-&gt;ops.counter_dealloc(counter);</div><div class='add'>+	rdma_counter_free(counter);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * rdma_counter_bind_qp_auto - Check and bind the QP to a counter base on</div><div class='add'>+ *   the auto-mode rule</div><div class='add'>+ */</div><div class='add'>+int rdma_counter_bind_qp_auto(struct ib_qp *qp, u32 port)</div><div class='add'>+{</div><div class='add'>+	struct rdma_port_counter *port_counter;</div><div class='add'>+	struct ib_device *dev = qp-&gt;device;</div><div class='add'>+	struct rdma_counter *counter;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!rdma_restrack_is_tracked(&amp;qp-&gt;res) || rdma_is_kernel_res(&amp;qp-&gt;res))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (!rdma_is_port_valid(dev, port))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	port_counter = &amp;dev-&gt;port_data[port].port_counter;</div><div class='add'>+	if (port_counter-&gt;mode.mode != RDMA_COUNTER_MODE_AUTO)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	counter = rdma_get_counter_auto_mode(qp, port);</div><div class='add'>+	if (counter) {</div><div class='add'>+		ret = __rdma_counter_bind_qp(counter, qp, port);</div><div class='add'>+		if (ret) {</div><div class='add'>+			kref_put(&amp;counter-&gt;kref, counter_release);</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_AUTO,</div><div class='add'>+					 port_counter-&gt;mode.bind_opcnt);</div><div class='add'>+		if (!counter)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * rdma_counter_unbind_qp - Unbind a qp from a counter</div><div class='add'>+ * @force:</div><div class='add'>+ *   true - Decrease the counter ref-count anyway (e.g., qp destroy)</div><div class='add'>+ */</div><div class='add'>+int rdma_counter_unbind_qp(struct ib_qp *qp, u32 port, bool force)</div><div class='add'>+{</div><div class='add'>+	struct rdma_counter *counter = qp-&gt;counter;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!counter)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ret = __rdma_counter_unbind_qp(qp, port);</div><div class='add'>+	if (ret &amp;&amp; !force)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	kref_put(&amp;counter-&gt;kref, counter_release);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int rdma_counter_query_stats(struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *dev = counter-&gt;device;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;ops.counter_update_stats)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;counter-&gt;lock);</div><div class='add'>+	ret = dev-&gt;ops.counter_update_stats(counter);</div><div class='add'>+	mutex_unlock(&amp;counter-&gt;lock);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u64 get_running_counters_hwstat_sum(struct ib_device *dev,</div><div class='add'>+					   u32 port, u32 index)</div><div class='add'>+{</div><div class='add'>+	struct rdma_restrack_entry *res;</div><div class='add'>+	struct rdma_restrack_root *rt;</div><div class='add'>+	struct rdma_counter *counter;</div><div class='add'>+	unsigned long id = 0;</div><div class='add'>+	u64 sum = 0;</div><div class='add'>+</div><div class='add'>+	rt = &amp;dev-&gt;res[RDMA_RESTRACK_COUNTER];</div><div class='add'>+	xa_lock(&amp;rt-&gt;xa);</div><div class='add'>+	xa_for_each(&amp;rt-&gt;xa, id, res) {</div><div class='add'>+		if (!rdma_restrack_get(res))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		xa_unlock(&amp;rt-&gt;xa);</div><div class='add'>+</div><div class='add'>+		counter = container_of(res, struct rdma_counter, res);</div><div class='add'>+		if ((counter-&gt;device != dev) || (counter-&gt;port != port) ||</div><div class='add'>+		    rdma_counter_query_stats(counter))</div><div class='add'>+			goto next;</div><div class='add'>+</div><div class='add'>+		sum += counter-&gt;stats-&gt;value[index];</div><div class='add'>+</div><div class='add'>+next:</div><div class='add'>+		xa_lock(&amp;rt-&gt;xa);</div><div class='add'>+		rdma_restrack_put(res);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	xa_unlock(&amp;rt-&gt;xa);</div><div class='add'>+	return sum;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * rdma_counter_get_hwstat_value() - Get the sum value of all counters on a</div><div class='add'>+ *   specific port, including the running ones and history data</div><div class='add'>+ */</div><div class='add'>+u64 rdma_counter_get_hwstat_value(struct ib_device *dev, u32 port, u32 index)</div><div class='add'>+{</div><div class='add'>+	struct rdma_port_counter *port_counter;</div><div class='add'>+	u64 sum;</div><div class='add'>+</div><div class='add'>+	port_counter = &amp;dev-&gt;port_data[port].port_counter;</div><div class='add'>+	if (!port_counter-&gt;hstats)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	sum = get_running_counters_hwstat_sum(dev, port, index);</div><div class='add'>+	sum += port_counter-&gt;hstats-&gt;value[index];</div><div class='add'>+</div><div class='add'>+	return sum;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct ib_qp *rdma_counter_get_qp(struct ib_device *dev, u32 qp_num)</div><div class='add'>+{</div><div class='add'>+	struct rdma_restrack_entry *res = NULL;</div><div class='add'>+	struct ib_qp *qp = NULL;</div><div class='add'>+</div><div class='add'>+	res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_QP, qp_num);</div><div class='add'>+	if (IS_ERR(res))</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	qp = container_of(res, struct ib_qp, res);</div><div class='add'>+	if (qp-&gt;qp_type == IB_QPT_RAW_PACKET &amp;&amp; !rdma_dev_has_raw_cap(dev))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	return qp;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	rdma_restrack_put(res);</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct rdma_counter *rdma_get_counter_by_id(struct ib_device *dev,</div><div class='add'>+						   u32 counter_id)</div><div class='add'>+{</div><div class='add'>+	struct rdma_restrack_entry *res;</div><div class='add'>+	struct rdma_counter *counter;</div><div class='add'>+</div><div class='add'>+	res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_COUNTER, counter_id);</div><div class='add'>+	if (IS_ERR(res))</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	counter = container_of(res, struct rdma_counter, res);</div><div class='add'>+	kref_get(&amp;counter-&gt;kref);</div><div class='add'>+	rdma_restrack_put(res);</div><div class='add'>+</div><div class='add'>+	return counter;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * rdma_counter_bind_qpn() - Bind QP @qp_num to counter @counter_id</div><div class='add'>+ */</div><div class='add'>+int rdma_counter_bind_qpn(struct ib_device *dev, u32 port,</div><div class='add'>+			  u32 qp_num, u32 counter_id)</div><div class='add'>+{</div><div class='add'>+	struct rdma_port_counter *port_counter;</div><div class='add'>+	struct rdma_counter *counter;</div><div class='add'>+	struct ib_qp *qp;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	port_counter = &amp;dev-&gt;port_data[port].port_counter;</div><div class='add'>+	if (port_counter-&gt;mode.mode == RDMA_COUNTER_MODE_AUTO)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	qp = rdma_counter_get_qp(dev, qp_num);</div><div class='add'>+	if (!qp)</div><div class='add'>+		return -ENOENT;</div><div class='add'>+</div><div class='add'>+	counter = rdma_get_counter_by_id(dev, counter_id);</div><div class='add'>+	if (!counter) {</div><div class='add'>+		ret = -ENOENT;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (rdma_is_kernel_res(&amp;counter-&gt;res) != rdma_is_kernel_res(&amp;qp-&gt;res)) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err_task;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if ((counter-&gt;device != qp-&gt;device) || (counter-&gt;port != qp-&gt;port)) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err_task;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = __rdma_counter_bind_qp(counter, qp, port);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_task;</div><div class='add'>+</div><div class='add'>+	rdma_restrack_put(&amp;qp-&gt;res);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_task:</div><div class='add'>+	kref_put(&amp;counter-&gt;kref, counter_release);</div><div class='add'>+err:</div><div class='add'>+	rdma_restrack_put(&amp;qp-&gt;res);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * rdma_counter_bind_qpn_alloc() - Alloc a counter and bind QP @qp_num to it</div><div class='add'>+ *   The id of new counter is returned in @counter_id</div><div class='add'>+ */</div><div class='add'>+int rdma_counter_bind_qpn_alloc(struct ib_device *dev, u32 port,</div><div class='add'>+				u32 qp_num, u32 *counter_id)</div><div class='add'>+{</div><div class='add'>+	struct rdma_port_counter *port_counter;</div><div class='add'>+	struct rdma_counter *counter;</div><div class='add'>+	struct ib_qp *qp;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!rdma_is_port_valid(dev, port))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	port_counter = &amp;dev-&gt;port_data[port].port_counter;</div><div class='add'>+	if (!port_counter-&gt;hstats)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (port_counter-&gt;mode.mode == RDMA_COUNTER_MODE_AUTO)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	qp = rdma_counter_get_qp(dev, qp_num);</div><div class='add'>+	if (!qp)</div><div class='add'>+		return -ENOENT;</div><div class='add'>+</div><div class='add'>+	if (rdma_is_port_valid(dev, qp-&gt;port) &amp;&amp; (qp-&gt;port != port)) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_MANUAL, true);</div><div class='add'>+	if (!counter) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (counter_id)</div><div class='add'>+		*counter_id = counter-&gt;id;</div><div class='add'>+</div><div class='add'>+	rdma_restrack_put(&amp;qp-&gt;res);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	rdma_restrack_put(&amp;qp-&gt;res);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * rdma_counter_unbind_qpn() - Unbind QP @qp_num from a counter</div><div class='add'>+ */</div><div class='add'>+int rdma_counter_unbind_qpn(struct ib_device *dev, u32 port,</div><div class='add'>+			    u32 qp_num, u32 counter_id)</div><div class='add'>+{</div><div class='add'>+	struct rdma_port_counter *port_counter;</div><div class='add'>+	struct ib_qp *qp;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!rdma_is_port_valid(dev, port))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	qp = rdma_counter_get_qp(dev, qp_num);</div><div class='add'>+	if (!qp)</div><div class='add'>+		return -ENOENT;</div><div class='add'>+</div><div class='add'>+	if (rdma_is_port_valid(dev, qp-&gt;port) &amp;&amp; (qp-&gt;port != port)) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	port_counter = &amp;dev-&gt;port_data[port].port_counter;</div><div class='add'>+	if (!qp-&gt;counter || qp-&gt;counter-&gt;id != counter_id ||</div><div class='add'>+	    port_counter-&gt;mode.mode != RDMA_COUNTER_MODE_MANUAL) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = rdma_counter_unbind_qp(qp, port, false);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	rdma_restrack_put(&amp;qp-&gt;res);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int rdma_counter_get_mode(struct ib_device *dev, u32 port,</div><div class='add'>+			  enum rdma_nl_counter_mode *mode,</div><div class='add'>+			  enum rdma_nl_counter_mask *mask,</div><div class='add'>+			  bool *opcnt)</div><div class='add'>+{</div><div class='add'>+	struct rdma_port_counter *port_counter;</div><div class='add'>+</div><div class='add'>+	port_counter = &amp;dev-&gt;port_data[port].port_counter;</div><div class='add'>+	*mode = port_counter-&gt;mode.mode;</div><div class='add'>+	*mask = port_counter-&gt;mode.mask;</div><div class='add'>+	*opcnt = port_counter-&gt;mode.bind_opcnt;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void rdma_counter_init(struct ib_device *dev)</div><div class='add'>+{</div><div class='add'>+	struct rdma_port_counter *port_counter;</div><div class='add'>+	u32 port, i;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;port_data)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	rdma_for_each_port(dev, port) {</div><div class='add'>+		port_counter = &amp;dev-&gt;port_data[port].port_counter;</div><div class='add'>+		port_counter-&gt;mode.mode = RDMA_COUNTER_MODE_NONE;</div><div class='add'>+		mutex_init(&amp;port_counter-&gt;lock);</div><div class='add'>+</div><div class='add'>+		if (!dev-&gt;ops.alloc_hw_port_stats)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		port_counter-&gt;hstats = dev-&gt;ops.alloc_hw_port_stats(dev, port);</div><div class='add'>+		if (!port_counter-&gt;hstats)</div><div class='add'>+			goto fail;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return;</div><div class='add'>+</div><div class='add'>+fail:</div><div class='add'>+	for (i = port; i &gt;= rdma_start_port(dev); i--) {</div><div class='add'>+		port_counter = &amp;dev-&gt;port_data[port].port_counter;</div><div class='add'>+		rdma_free_hw_stats_struct(port_counter-&gt;hstats);</div><div class='add'>+		port_counter-&gt;hstats = NULL;</div><div class='add'>+		mutex_destroy(&amp;port_counter-&gt;lock);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void rdma_counter_release(struct ib_device *dev)</div><div class='add'>+{</div><div class='add'>+	struct rdma_port_counter *port_counter;</div><div class='add'>+	u32 port;</div><div class='add'>+</div><div class='add'>+	rdma_for_each_port(dev, port) {</div><div class='add'>+		port_counter = &amp;dev-&gt;port_data[port].port_counter;</div><div class='add'>+		rdma_free_hw_stats_struct(port_counter-&gt;hstats);</div><div class='add'>+		mutex_destroy(&amp;port_counter-&gt;lock);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c<br/>index a4c81992267c..584537c71545 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cq.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/cq.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/cq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/cq.c</a></div><div class='hunk'>@@ -1,20 +1,17 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0-only</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright (c) 2015 HGST, a Western Digital Company.</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify it</div><div class='del'>- * under the terms and conditions of the GNU General Public License,</div><div class='del'>- * version 2, as published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope it will be useful, but WITHOUT</div><div class='del'>- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or</div><div class='del'>- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for</div><div class='del'>- * more details.</div><div class='ctx'>  */</div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='ctx'> #include &lt;linux/err.h&gt;</div><div class='ctx'> #include &lt;linux/slab.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_verbs.h&gt;</div><div class='ctx'> </div><div class='add'>+#include "core_priv.h"</div><div class='add'>+</div><div class='add'>+#include &lt;trace/events/rdma_core.h&gt;</div><div class='add'>+/* Max size for shared CQ, may require tuning */</div><div class='add'>+#define IB_MAX_SHARED_CQ_SZ		4096U</div><div class='add'>+</div><div class='ctx'> /* # of WCs to poll for with a single call to ib_poll_cq */</div><div class='ctx'> #define IB_POLL_BATCH			16</div><div class='ctx'> #define IB_POLL_BATCH_DIRECT		8</div><div class='hunk'>@@ -26,18 +23,86 @@</div><div class='ctx'> #define IB_POLL_FLAGS \</div><div class='ctx'> 	(IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS)</div><div class='ctx'> </div><div class='add'>+static const struct dim_cq_moder</div><div class='add'>+rdma_dim_prof[RDMA_DIM_PARAMS_NUM_PROFILES] = {</div><div class='add'>+	{1,   0, 1,  0},</div><div class='add'>+	{1,   0, 4,  0},</div><div class='add'>+	{2,   0, 4,  0},</div><div class='add'>+	{2,   0, 8,  0},</div><div class='add'>+	{4,   0, 8,  0},</div><div class='add'>+	{16,  0, 8,  0},</div><div class='add'>+	{16,  0, 16, 0},</div><div class='add'>+	{32,  0, 16, 0},</div><div class='add'>+	{32,  0, 32, 0},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static void ib_cq_rdma_dim_work(struct work_struct *w)</div><div class='add'>+{</div><div class='add'>+	struct dim *dim = container_of(w, struct dim, work);</div><div class='add'>+	struct ib_cq *cq = dim-&gt;priv;</div><div class='add'>+</div><div class='add'>+	u16 usec = rdma_dim_prof[dim-&gt;profile_ix].usec;</div><div class='add'>+	u16 comps = rdma_dim_prof[dim-&gt;profile_ix].comps;</div><div class='add'>+</div><div class='add'>+	dim-&gt;state = DIM_START_MEASURE;</div><div class='add'>+</div><div class='add'>+	trace_cq_modify(cq, comps, usec);</div><div class='add'>+	cq-&gt;device-&gt;ops.modify_cq(cq, comps, usec);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void rdma_dim_init(struct ib_cq *cq)</div><div class='add'>+{</div><div class='add'>+	struct dim *dim;</div><div class='add'>+</div><div class='add'>+	if (!cq-&gt;device-&gt;ops.modify_cq || !cq-&gt;device-&gt;use_cq_dim ||</div><div class='add'>+	    cq-&gt;poll_ctx == IB_POLL_DIRECT)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	dim = kzalloc(sizeof(struct dim), GFP_KERNEL);</div><div class='add'>+	if (!dim)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	dim-&gt;state = DIM_START_MEASURE;</div><div class='add'>+	dim-&gt;tune_state = DIM_GOING_RIGHT;</div><div class='add'>+	dim-&gt;profile_ix = RDMA_DIM_START_PROFILE;</div><div class='add'>+	dim-&gt;priv = cq;</div><div class='add'>+	cq-&gt;dim = dim;</div><div class='add'>+</div><div class='add'>+	INIT_WORK(&amp;dim-&gt;work, ib_cq_rdma_dim_work);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void rdma_dim_destroy(struct ib_cq *cq)</div><div class='add'>+{</div><div class='add'>+	if (!cq-&gt;dim)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	cancel_work_sync(&amp;cq-&gt;dim-&gt;work);</div><div class='add'>+	kfree(cq-&gt;dim);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int __poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc)</div><div class='add'>+{</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rc = ib_poll_cq(cq, num_entries, wc);</div><div class='add'>+	trace_cq_poll(cq, num_entries, rc);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int __ib_process_cq(struct ib_cq *cq, int budget, struct ib_wc *wcs,</div><div class='ctx'> 			   int batch)</div><div class='ctx'> {</div><div class='ctx'> 	int i, n, completed = 0;</div><div class='ctx'> </div><div class='add'>+	trace_cq_process(cq);</div><div class='add'>+</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * budget might be (-1) if the caller does not</div><div class='ctx'> 	 * want to bound this call, thus we need unsigned</div><div class='ctx'> 	 * minimum here.</div><div class='ctx'> 	 */</div><div class='del'>-	while ((n = ib_poll_cq(cq, min_t(u32, batch,</div><div class='del'>-					 budget - completed), wcs)) &gt; 0) {</div><div class='add'>+	while ((n = __poll_cq(cq, min_t(u32, batch,</div><div class='add'>+					budget - completed), wcs)) &gt; 0) {</div><div class='ctx'> 		for (i = 0; i &lt; n; i++) {</div><div class='ctx'> 			struct ib_wc *wc = &amp;wcs[i];</div><div class='ctx'> </div><div class='hunk'>@@ -57,7 +122,7 @@ static int __ib_process_cq(struct ib_cq *cq, int budget, struct ib_wc *wcs,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * ib_process_direct_cq - process a CQ in caller context</div><div class='add'>+ * ib_process_cq_direct - process a CQ in caller context</div><div class='ctx'>  * @cq:		CQ to process</div><div class='ctx'>  * @budget:	number of CQEs to poll for</div><div class='ctx'>  *</div><div class='hunk'>@@ -86,20 +151,27 @@ static void ib_cq_completion_direct(struct ib_cq *cq, void *private)</div><div class='ctx'> static int ib_poll_handler(struct irq_poll *iop, int budget)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_cq *cq = container_of(iop, struct ib_cq, iop);</div><div class='add'>+	struct dim *dim = cq-&gt;dim;</div><div class='ctx'> 	int completed;</div><div class='ctx'> </div><div class='ctx'> 	completed = __ib_process_cq(cq, budget, cq-&gt;wc, IB_POLL_BATCH);</div><div class='ctx'> 	if (completed &lt; budget) {</div><div class='ctx'> 		irq_poll_complete(&amp;cq-&gt;iop);</div><div class='del'>-		if (ib_req_notify_cq(cq, IB_POLL_FLAGS) &gt; 0)</div><div class='add'>+		if (ib_req_notify_cq(cq, IB_POLL_FLAGS) &gt; 0) {</div><div class='add'>+			trace_cq_reschedule(cq);</div><div class='ctx'> 			irq_poll_sched(&amp;cq-&gt;iop);</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (dim)</div><div class='add'>+		rdma_dim(dim, completed);</div><div class='add'>+</div><div class='ctx'> 	return completed;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void ib_cq_completion_softirq(struct ib_cq *cq, void *private)</div><div class='ctx'> {</div><div class='add'>+	trace_cq_schedule(cq);</div><div class='ctx'> 	irq_poll_sched(&amp;cq-&gt;iop);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -113,10 +185,13 @@ static void ib_cq_poll_work(struct work_struct *work)</div><div class='ctx'> 	if (completed &gt;= IB_POLL_BUDGET_WORKQUEUE ||</div><div class='ctx'> 	    ib_req_notify_cq(cq, IB_POLL_FLAGS) &gt; 0)</div><div class='ctx'> 		queue_work(cq-&gt;comp_wq, &amp;cq-&gt;work);</div><div class='add'>+	else if (cq-&gt;dim)</div><div class='add'>+		rdma_dim(cq-&gt;dim, completed);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private)</div><div class='ctx'> {</div><div class='add'>+	trace_cq_schedule(cq);</div><div class='ctx'> 	queue_work(cq-&gt;comp_wq, &amp;cq-&gt;work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -128,17 +203,15 @@ static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private)</div><div class='ctx'>  * @comp_vector:	HCA completion vectors for this CQ</div><div class='ctx'>  * @poll_ctx:		context to poll the CQ from.</div><div class='ctx'>  * @caller:		module owner name.</div><div class='del'>- * @udata:		Valid user data or NULL for kernel object</div><div class='ctx'>  *</div><div class='ctx'>  * This is the proper interface to allocate a CQ for in-kernel users. A</div><div class='ctx'>  * CQ allocated with this interface will automatically be polled from the</div><div class='ctx'>  * specified context. The ULP must use wr-&gt;wr_cqe instead of wr-&gt;wr_id</div><div class='ctx'>  * to use this CQ abstraction.</div><div class='ctx'>  */</div><div class='del'>-struct ib_cq *__ib_alloc_cq_user(struct ib_device *dev, void *private,</div><div class='del'>-				 int nr_cqe, int comp_vector,</div><div class='del'>-				 enum ib_poll_context poll_ctx,</div><div class='del'>-				 const char *caller, struct ib_udata *udata)</div><div class='add'>+struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private, int nr_cqe,</div><div class='add'>+			    int comp_vector, enum ib_poll_context poll_ctx,</div><div class='add'>+			    const char *caller)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_cq_init_attr cq_attr = {</div><div class='ctx'> 		.cqe		= nr_cqe,</div><div class='hunk'>@@ -147,24 +220,28 @@ struct ib_cq *__ib_alloc_cq_user(struct ib_device *dev, void *private,</div><div class='ctx'> 	struct ib_cq *cq;</div><div class='ctx'> 	int ret = -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	cq = dev-&gt;ops.create_cq(dev, &amp;cq_attr, NULL);</div><div class='del'>-	if (IS_ERR(cq))</div><div class='del'>-		return cq;</div><div class='add'>+	cq = rdma_zalloc_drv_obj(dev, ib_cq);</div><div class='add'>+	if (!cq)</div><div class='add'>+		return ERR_PTR(ret);</div><div class='ctx'> </div><div class='ctx'> 	cq-&gt;device = dev;</div><div class='del'>-	cq-&gt;uobject = NULL;</div><div class='del'>-	cq-&gt;event_handler = NULL;</div><div class='ctx'> 	cq-&gt;cq_context = private;</div><div class='ctx'> 	cq-&gt;poll_ctx = poll_ctx;</div><div class='ctx'> 	atomic_set(&amp;cq-&gt;usecnt, 0);</div><div class='add'>+	cq-&gt;comp_vector = comp_vector;</div><div class='ctx'> </div><div class='ctx'> 	cq-&gt;wc = kmalloc_array(IB_POLL_BATCH, sizeof(*cq-&gt;wc), GFP_KERNEL);</div><div class='ctx'> 	if (!cq-&gt;wc)</div><div class='del'>-		goto out_destroy_cq;</div><div class='add'>+		goto out_free_cq;</div><div class='add'>+</div><div class='add'>+	rdma_restrack_new(&amp;cq-&gt;res, RDMA_RESTRACK_CQ);</div><div class='add'>+	rdma_restrack_set_name(&amp;cq-&gt;res, caller);</div><div class='ctx'> </div><div class='del'>-	cq-&gt;res.type = RDMA_RESTRACK_CQ;</div><div class='del'>-	rdma_restrack_set_task(&amp;cq-&gt;res, caller);</div><div class='del'>-	rdma_restrack_kadd(&amp;cq-&gt;res);</div><div class='add'>+	ret = dev-&gt;ops.create_cq(cq, &amp;cq_attr, NULL);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out_free_wc;</div><div class='add'>+</div><div class='add'>+	rdma_dim_init(cq);</div><div class='ctx'> </div><div class='ctx'> 	switch (cq-&gt;poll_ctx) {</div><div class='ctx'> 	case IB_POLL_DIRECT:</div><div class='hunk'>@@ -186,31 +263,71 @@ struct ib_cq *__ib_alloc_cq_user(struct ib_device *dev, void *private,</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		ret = -EINVAL;</div><div class='del'>-		goto out_free_wc;</div><div class='add'>+		goto out_destroy_cq;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	rdma_restrack_add(&amp;cq-&gt;res);</div><div class='add'>+	trace_cq_alloc(cq, nr_cqe, comp_vector, poll_ctx);</div><div class='ctx'> 	return cq;</div><div class='ctx'> </div><div class='add'>+out_destroy_cq:</div><div class='add'>+	rdma_dim_destroy(cq);</div><div class='add'>+	cq-&gt;device-&gt;ops.destroy_cq(cq, NULL);</div><div class='ctx'> out_free_wc:</div><div class='add'>+	rdma_restrack_put(&amp;cq-&gt;res);</div><div class='ctx'> 	kfree(cq-&gt;wc);</div><div class='del'>-	rdma_restrack_del(&amp;cq-&gt;res);</div><div class='del'>-out_destroy_cq:</div><div class='del'>-	cq-&gt;device-&gt;ops.destroy_cq(cq, udata);</div><div class='add'>+out_free_cq:</div><div class='add'>+	kfree(cq);</div><div class='add'>+	trace_cq_alloc_error(nr_cqe, comp_vector, poll_ctx, ret);</div><div class='ctx'> 	return ERR_PTR(ret);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(__ib_alloc_cq_user);</div><div class='add'>+EXPORT_SYMBOL(__ib_alloc_cq);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * __ib_alloc_cq_any - allocate a completion queue</div><div class='add'>+ * @dev:		device to allocate the CQ for</div><div class='add'>+ * @private:		driver private data, accessible from cq-&gt;cq_context</div><div class='add'>+ * @nr_cqe:		number of CQEs to allocate</div><div class='add'>+ * @poll_ctx:		context to poll the CQ from</div><div class='add'>+ * @caller:		module owner name</div><div class='add'>+ *</div><div class='add'>+ * Attempt to spread ULP Completion Queues over each device's interrupt</div><div class='add'>+ * vectors. A simple best-effort mechanism is used.</div><div class='add'>+ */</div><div class='add'>+struct ib_cq *__ib_alloc_cq_any(struct ib_device *dev, void *private,</div><div class='add'>+				int nr_cqe, enum ib_poll_context poll_ctx,</div><div class='add'>+				const char *caller)</div><div class='add'>+{</div><div class='add'>+	static atomic_t counter;</div><div class='add'>+	int comp_vector = 0;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;num_comp_vectors &gt; 1)</div><div class='add'>+		comp_vector =</div><div class='add'>+			atomic_inc_return(&amp;counter) %</div><div class='add'>+			min_t(int, dev-&gt;num_comp_vectors, num_online_cpus());</div><div class='add'>+</div><div class='add'>+	return __ib_alloc_cq(dev, private, nr_cqe, comp_vector, poll_ctx,</div><div class='add'>+			     caller);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(__ib_alloc_cq_any);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * ib_free_cq - free a completion queue</div><div class='ctx'>  * @cq:		completion queue to free.</div><div class='del'>- * @udata:	User data or NULL for kernel object</div><div class='ctx'>  */</div><div class='del'>-void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata)</div><div class='add'>+void ib_free_cq(struct ib_cq *cq)</div><div class='ctx'> {</div><div class='del'>-	int ret;</div><div class='add'>+	int ret = 0;</div><div class='ctx'> </div><div class='ctx'> 	if (WARN_ON_ONCE(atomic_read(&amp;cq-&gt;usecnt)))</div><div class='ctx'> 		return;</div><div class='add'>+	if (WARN_ON_ONCE(cq-&gt;cqe_used))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	if (cq-&gt;device-&gt;ops.pre_destroy_cq) {</div><div class='add'>+		ret = cq-&gt;device-&gt;ops.pre_destroy_cq(cq);</div><div class='add'>+		WARN_ONCE(ret, "Disable of kernel CQ shouldn't fail");</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	switch (cq-&gt;poll_ctx) {</div><div class='ctx'> 	case IB_POLL_DIRECT:</div><div class='hunk'>@@ -226,9 +343,173 @@ void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata)</div><div class='ctx'> 		WARN_ON_ONCE(1);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	kfree(cq-&gt;wc);</div><div class='add'>+	rdma_dim_destroy(cq);</div><div class='add'>+	trace_cq_free(cq);</div><div class='add'>+	if (cq-&gt;device-&gt;ops.post_destroy_cq)</div><div class='add'>+		cq-&gt;device-&gt;ops.post_destroy_cq(cq);</div><div class='add'>+	else</div><div class='add'>+		ret = cq-&gt;device-&gt;ops.destroy_cq(cq, NULL);</div><div class='add'>+	WARN_ONCE(ret, "Destroy of kernel CQ shouldn't fail");</div><div class='ctx'> 	rdma_restrack_del(&amp;cq-&gt;res);</div><div class='del'>-	ret = cq-&gt;device-&gt;ops.destroy_cq(cq, udata);</div><div class='del'>-	WARN_ON_ONCE(ret);</div><div class='add'>+	kfree(cq-&gt;wc);</div><div class='add'>+	kfree(cq);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_free_cq);</div><div class='add'>+</div><div class='add'>+void ib_cq_pool_cleanup(struct ib_device *dev)</div><div class='add'>+{</div><div class='add'>+	struct ib_cq *cq, *n;</div><div class='add'>+	unsigned int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ARRAY_SIZE(dev-&gt;cq_pools); i++) {</div><div class='add'>+		list_for_each_entry_safe(cq, n, &amp;dev-&gt;cq_pools[i],</div><div class='add'>+					 pool_entry) {</div><div class='add'>+			WARN_ON(cq-&gt;cqe_used);</div><div class='add'>+			list_del(&amp;cq-&gt;pool_entry);</div><div class='add'>+			cq-&gt;shared = false;</div><div class='add'>+			ib_free_cq(cq);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ib_alloc_cqs(struct ib_device *dev, unsigned int nr_cqes,</div><div class='add'>+			enum ib_poll_context poll_ctx)</div><div class='add'>+{</div><div class='add'>+	LIST_HEAD(tmp_list);</div><div class='add'>+	unsigned int nr_cqs, i;</div><div class='add'>+	struct ib_cq *cq, *n;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (poll_ctx &gt; IB_POLL_LAST_POOL_TYPE) {</div><div class='add'>+		WARN_ON_ONCE(poll_ctx &gt; IB_POLL_LAST_POOL_TYPE);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Allocate at least as many CQEs as requested, and otherwise</div><div class='add'>+	 * a reasonable batch size so that we can share CQs between</div><div class='add'>+	 * multiple users instead of allocating a larger number of CQs.</div><div class='add'>+	 */</div><div class='add'>+	nr_cqes = min_t(unsigned int, dev-&gt;attrs.max_cqe,</div><div class='add'>+			max(nr_cqes, IB_MAX_SHARED_CQ_SZ));</div><div class='add'>+	nr_cqs = min_t(unsigned int, dev-&gt;num_comp_vectors, num_online_cpus());</div><div class='add'>+	for (i = 0; i &lt; nr_cqs; i++) {</div><div class='add'>+		cq = ib_alloc_cq(dev, NULL, nr_cqes, i, poll_ctx);</div><div class='add'>+		if (IS_ERR(cq)) {</div><div class='add'>+			ret = PTR_ERR(cq);</div><div class='add'>+			goto out_free_cqs;</div><div class='add'>+		}</div><div class='add'>+		cq-&gt;shared = true;</div><div class='add'>+		list_add_tail(&amp;cq-&gt;pool_entry, &amp;tmp_list);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_lock_irq(&amp;dev-&gt;cq_pools_lock);</div><div class='add'>+	list_splice(&amp;tmp_list, &amp;dev-&gt;cq_pools[poll_ctx]);</div><div class='add'>+	spin_unlock_irq(&amp;dev-&gt;cq_pools_lock);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+out_free_cqs:</div><div class='add'>+	list_for_each_entry_safe(cq, n, &amp;tmp_list, pool_entry) {</div><div class='add'>+		cq-&gt;shared = false;</div><div class='add'>+		ib_free_cq(cq);</div><div class='add'>+	}</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ib_cq_pool_get() - Find the least used completion queue that matches</div><div class='add'>+ *   a given cpu hint (or least used for wild card affinity) and fits</div><div class='add'>+ *   nr_cqe.</div><div class='add'>+ * @dev: rdma device</div><div class='add'>+ * @nr_cqe: number of needed cqe entries</div><div class='add'>+ * @comp_vector_hint: completion vector hint (-1) for the driver to assign</div><div class='add'>+ *   a comp vector based on internal counter</div><div class='add'>+ * @poll_ctx: cq polling context</div><div class='add'>+ *</div><div class='add'>+ * Finds a cq that satisfies @comp_vector_hint and @nr_cqe requirements and</div><div class='add'>+ * claim entries in it for us.  In case there is no available cq, allocate</div><div class='add'>+ * a new cq with the requirements and add it to the device pool.</div><div class='add'>+ * IB_POLL_DIRECT cannot be used for shared cqs so it is not a valid value</div><div class='add'>+ * for @poll_ctx.</div><div class='add'>+ */</div><div class='add'>+struct ib_cq *ib_cq_pool_get(struct ib_device *dev, unsigned int nr_cqe,</div><div class='add'>+			     int comp_vector_hint,</div><div class='add'>+			     enum ib_poll_context poll_ctx)</div><div class='add'>+{</div><div class='add'>+	static unsigned int default_comp_vector;</div><div class='add'>+	unsigned int vector, num_comp_vectors;</div><div class='add'>+	struct ib_cq *cq, *found = NULL;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (poll_ctx &gt; IB_POLL_LAST_POOL_TYPE) {</div><div class='add'>+		WARN_ON_ONCE(poll_ctx &gt; IB_POLL_LAST_POOL_TYPE);</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	num_comp_vectors =</div><div class='add'>+		min_t(unsigned int, dev-&gt;num_comp_vectors, num_online_cpus());</div><div class='add'>+	/* Project the affinty to the device completion vector range */</div><div class='add'>+	if (comp_vector_hint &lt; 0) {</div><div class='add'>+		comp_vector_hint =</div><div class='add'>+			(READ_ONCE(default_comp_vector) + 1) % num_comp_vectors;</div><div class='add'>+		WRITE_ONCE(default_comp_vector, comp_vector_hint);</div><div class='add'>+	}</div><div class='add'>+	vector = comp_vector_hint % num_comp_vectors;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Find the least used CQ with correct affinity and</div><div class='add'>+	 * enough free CQ entries</div><div class='add'>+	 */</div><div class='add'>+	while (!found) {</div><div class='add'>+		spin_lock_irq(&amp;dev-&gt;cq_pools_lock);</div><div class='add'>+		list_for_each_entry(cq, &amp;dev-&gt;cq_pools[poll_ctx],</div><div class='add'>+				    pool_entry) {</div><div class='add'>+			/*</div><div class='add'>+			 * Check to see if we have found a CQ with the</div><div class='add'>+			 * correct completion vector</div><div class='add'>+			 */</div><div class='add'>+			if (vector != cq-&gt;comp_vector)</div><div class='add'>+				continue;</div><div class='add'>+			if (cq-&gt;cqe_used + nr_cqe &gt; cq-&gt;cqe)</div><div class='add'>+				continue;</div><div class='add'>+			found = cq;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (found) {</div><div class='add'>+			found-&gt;cqe_used += nr_cqe;</div><div class='add'>+			spin_unlock_irq(&amp;dev-&gt;cq_pools_lock);</div><div class='add'>+</div><div class='add'>+			return found;</div><div class='add'>+		}</div><div class='add'>+		spin_unlock_irq(&amp;dev-&gt;cq_pools_lock);</div><div class='add'>+</div><div class='add'>+		/*</div><div class='add'>+		 * Didn't find a match or ran out of CQs in the device</div><div class='add'>+		 * pool, allocate a new array of CQs.</div><div class='add'>+		 */</div><div class='add'>+		ret = ib_alloc_cqs(dev, nr_cqe, poll_ctx);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ERR_PTR(ret);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return found;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_cq_pool_get);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ib_cq_pool_put - Return a CQ taken from a shared pool.</div><div class='add'>+ * @cq: The CQ to return.</div><div class='add'>+ * @nr_cqe: The max number of cqes that the user had requested.</div><div class='add'>+ */</div><div class='add'>+void ib_cq_pool_put(struct ib_cq *cq, unsigned int nr_cqe)</div><div class='add'>+{</div><div class='add'>+	if (WARN_ON_ONCE(nr_cqe &gt; cq-&gt;cqe_used))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	spin_lock_irq(&amp;cq-&gt;device-&gt;cq_pools_lock);</div><div class='add'>+	cq-&gt;cqe_used -= nr_cqe;</div><div class='add'>+	spin_unlock_irq(&amp;cq-&gt;device-&gt;cq_pools_lock);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(ib_free_cq_user);</div><div class='add'>+EXPORT_SYMBOL(ib_cq_pool_put);</div><div class='head'>diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c<br/>index 78dc07c6ac4b..13e8a1714bbd 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/device.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/device.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/device.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/device.c</a></div><div class='hunk'>@@ -39,13 +39,13 @@</div><div class='ctx'> #include &lt;linux/init.h&gt;</div><div class='ctx'> #include &lt;linux/netdevice.h&gt;</div><div class='ctx'> #include &lt;net/net_namespace.h&gt;</div><div class='del'>-#include &lt;net/netns/generic.h&gt;</div><div class='ctx'> #include &lt;linux/security.h&gt;</div><div class='ctx'> #include &lt;linux/notifier.h&gt;</div><div class='ctx'> #include &lt;linux/hashtable.h&gt;</div><div class='ctx'> #include &lt;rdma/rdma_netlink.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_addr.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_cache.h&gt;</div><div class='add'>+#include &lt;rdma/rdma_counter.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "core_priv.h"</div><div class='ctx'> #include "restrack.h"</div><div class='hunk'>@@ -58,6 +58,7 @@ struct workqueue_struct *ib_comp_wq;</div><div class='ctx'> struct workqueue_struct *ib_comp_unbound_wq;</div><div class='ctx'> struct workqueue_struct *ib_wq;</div><div class='ctx'> EXPORT_SYMBOL_GPL(ib_wq);</div><div class='add'>+static struct workqueue_struct *ib_unreg_wq;</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * Each of the three rwsem locks (devices, clients, client_data) protects the</div><div class='hunk'>@@ -93,28 +94,24 @@ static DEFINE_XARRAY_FLAGS(devices, XA_FLAGS_ALLOC);</div><div class='ctx'> static DECLARE_RWSEM(devices_rwsem);</div><div class='ctx'> #define DEVICE_REGISTERED XA_MARK_1</div><div class='ctx'> </div><div class='del'>-static LIST_HEAD(client_list);</div><div class='add'>+static u32 highest_client_id;</div><div class='ctx'> #define CLIENT_REGISTERED XA_MARK_1</div><div class='ctx'> static DEFINE_XARRAY_FLAGS(clients, XA_FLAGS_ALLOC);</div><div class='ctx'> static DECLARE_RWSEM(clients_rwsem);</div><div class='ctx'> </div><div class='add'>+static void ib_client_put(struct ib_client *client)</div><div class='add'>+{</div><div class='add'>+	if (refcount_dec_and_test(&amp;client-&gt;uses))</div><div class='add'>+		complete(&amp;client-&gt;uses_zero);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /*</div><div class='ctx'>  * If client_data is registered then the corresponding client must also still</div><div class='ctx'>  * be registered.</div><div class='ctx'>  */</div><div class='ctx'> #define CLIENT_DATA_REGISTERED XA_MARK_1</div><div class='ctx'> </div><div class='del'>-/**</div><div class='del'>- * struct rdma_dev_net - rdma net namespace metadata for a net</div><div class='del'>- * @net:	Pointer to owner net namespace</div><div class='del'>- * @id:		xarray id to identify the net namespace.</div><div class='del'>- */</div><div class='del'>-struct rdma_dev_net {</div><div class='del'>-	possible_net_t net;</div><div class='del'>-	u32 id;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static unsigned int rdma_dev_net_id;</div><div class='add'>+unsigned int rdma_dev_net_id;</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * A list of net namespaces is maintained in an xarray. This is necessary</div><div class='hunk'>@@ -132,17 +129,14 @@ module_param_named(netns_mode, ib_devices_shared_netns, bool, 0444);</div><div class='ctx'> MODULE_PARM_DESC(netns_mode,</div><div class='ctx'> 		 "Share device among net namespaces; default=1 (shared)");</div><div class='ctx'> /**</div><div class='del'>- * rdma_dev_access_netns() - Return whether a rdma device can be accessed</div><div class='add'>+ * rdma_dev_access_netns() - Return whether an rdma device can be accessed</div><div class='ctx'>  *			     from a specified net namespace or not.</div><div class='del'>- * @device:	Pointer to rdma device which needs to be checked</div><div class='add'>+ * @dev:	Pointer to rdma device which needs to be checked</div><div class='ctx'>  * @net:	Pointer to net namesapce for which access to be checked</div><div class='ctx'>  *</div><div class='del'>- * rdma_dev_access_netns() - Return whether a rdma device can be accessed</div><div class='del'>- *			     from a specified net namespace or not. When</div><div class='del'>- *			     rdma device is in shared mode, it ignores the</div><div class='del'>- *			     net namespace. When rdma device is exclusive</div><div class='del'>- *			     to a net namespace, rdma device net namespace is</div><div class='del'>- *			     checked against the specified one.</div><div class='add'>+ * When the rdma device is in shared mode, it ignores the net namespace.</div><div class='add'>+ * When the rdma device is exclusive to a net namespace, rdma device net</div><div class='add'>+ * namespace is checked against the specified one.</div><div class='ctx'>  */</div><div class='ctx'> bool rdma_dev_access_netns(const struct ib_device *dev, const struct net *net)</div><div class='ctx'> {</div><div class='hunk'>@@ -151,6 +145,33 @@ bool rdma_dev_access_netns(const struct ib_device *dev, const struct net *net)</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_dev_access_netns);</div><div class='ctx'> </div><div class='add'>+/**</div><div class='add'>+ * rdma_dev_has_raw_cap() - Returns whether a specified rdma device has</div><div class='add'>+ *			    CAP_NET_RAW capability or not.</div><div class='add'>+ *</div><div class='add'>+ * @dev:	Pointer to rdma device whose capability to be checked</div><div class='add'>+ *</div><div class='add'>+ * Returns true if a rdma device's owning user namespace has CAP_NET_RAW</div><div class='add'>+ * capability, otherwise false. When rdma subsystem is in legacy shared network,</div><div class='add'>+ * namespace mode, the default net namespace is considered.</div><div class='add'>+ */</div><div class='add'>+bool rdma_dev_has_raw_cap(const struct ib_device *dev)</div><div class='add'>+{</div><div class='add'>+	const struct net *net;</div><div class='add'>+</div><div class='add'>+	/* Network namespace is the resource whose user namespace</div><div class='add'>+	 * to be considered. When in shared mode, there is no reliable</div><div class='add'>+	 * network namespace resource, so consider the default net namespace.</div><div class='add'>+	 */</div><div class='add'>+	if (ib_devices_shared_netns)</div><div class='add'>+		net = &amp;init_net;</div><div class='add'>+	else</div><div class='add'>+		net = read_pnet(&amp;dev-&gt;coredev.rdma_net);</div><div class='add'>+</div><div class='add'>+	return ns_capable(net-&gt;user_ns, CAP_NET_RAW);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_dev_has_raw_cap);</div><div class='add'>+</div><div class='ctx'> /*</div><div class='ctx'>  * xarray has this behavior where it won't iterate over NULL values stored in</div><div class='ctx'>  * allocated arrays.  So we need our own iterator to see all values stored in</div><div class='hunk'>@@ -215,23 +236,6 @@ static void __ibdev_printk(const char *level, const struct ib_device *ibdev,</div><div class='ctx'> 		printk("%s(NULL ib_device): %pV", level, vaf);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void ibdev_printk(const char *level, const struct ib_device *ibdev,</div><div class='del'>-		  const char *format, ...)</div><div class='del'>-{</div><div class='del'>-	struct va_format vaf;</div><div class='del'>-	va_list args;</div><div class='del'>-</div><div class='del'>-	va_start(args, format);</div><div class='del'>-</div><div class='del'>-	vaf.fmt = format;</div><div class='del'>-	vaf.va = &amp;args;</div><div class='del'>-</div><div class='del'>-	__ibdev_printk(level, ibdev, &amp;vaf);</div><div class='del'>-</div><div class='del'>-	va_end(args);</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ibdev_printk);</div><div class='del'>-</div><div class='ctx'> #define define_ibdev_printk_level(func, level)                  \</div><div class='ctx'> void func(const struct ib_device *ibdev, const char *fmt, ...)  \</div><div class='ctx'> {                                                               \</div><div class='hunk'>@@ -270,7 +274,7 @@ struct ib_port_data_rcu {</div><div class='ctx'> 	struct ib_port_data pdata[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static int ib_device_check_mandatory(struct ib_device *device)</div><div class='add'>+static void ib_device_check_mandatory(struct ib_device *device)</div><div class='ctx'> {</div><div class='ctx'> #define IB_MANDATORY_FUNC(x) { offsetof(struct ib_device_ops, x), #x }</div><div class='ctx'> 	static const struct {</div><div class='hunk'>@@ -279,7 +283,6 @@ static int ib_device_check_mandatory(struct ib_device *device)</div><div class='ctx'> 	} mandatory_table[] = {</div><div class='ctx'> 		IB_MANDATORY_FUNC(query_device),</div><div class='ctx'> 		IB_MANDATORY_FUNC(query_port),</div><div class='del'>-		IB_MANDATORY_FUNC(query_pkey),</div><div class='ctx'> 		IB_MANDATORY_FUNC(alloc_pd),</div><div class='ctx'> 		IB_MANDATORY_FUNC(dealloc_pd),</div><div class='ctx'> 		IB_MANDATORY_FUNC(create_qp),</div><div class='hunk'>@@ -292,6 +295,7 @@ static int ib_device_check_mandatory(struct ib_device *device)</div><div class='ctx'> 		IB_MANDATORY_FUNC(poll_cq),</div><div class='ctx'> 		IB_MANDATORY_FUNC(req_notify_cq),</div><div class='ctx'> 		IB_MANDATORY_FUNC(get_dma_mr),</div><div class='add'>+		IB_MANDATORY_FUNC(reg_user_mr),</div><div class='ctx'> 		IB_MANDATORY_FUNC(dereg_mr),</div><div class='ctx'> 		IB_MANDATORY_FUNC(get_port_immutable)</div><div class='ctx'> 	};</div><div class='hunk'>@@ -305,8 +309,6 @@ static int ib_device_check_mandatory(struct ib_device *device)</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -375,7 +377,7 @@ struct ib_device *ib_device_get_by_name(const char *name,</div><div class='ctx'> 	down_read(&amp;devices_rwsem);</div><div class='ctx'> 	device = __ib_device_get_by_name(name);</div><div class='ctx'> 	if (device &amp;&amp; driver_id != RDMA_DRIVER_UNKNOWN &amp;&amp;</div><div class='del'>-	    device-&gt;driver_id != driver_id)</div><div class='add'>+	    device-&gt;ops.driver_id != driver_id)</div><div class='ctx'> 		device = NULL;</div><div class='ctx'> </div><div class='ctx'> 	if (device) {</div><div class='hunk'>@@ -409,27 +411,54 @@ static int rename_compat_devs(struct ib_device *device)</div><div class='ctx'> </div><div class='ctx'> int ib_device_rename(struct ib_device *ibdev, const char *name)</div><div class='ctx'> {</div><div class='add'>+	unsigned long index;</div><div class='add'>+	void *client_data;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	down_write(&amp;devices_rwsem);</div><div class='ctx'> 	if (!strcmp(name, dev_name(&amp;ibdev-&gt;dev))) {</div><div class='del'>-		ret = 0;</div><div class='del'>-		goto out;</div><div class='add'>+		up_write(&amp;devices_rwsem);</div><div class='add'>+		return 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (__ib_device_get_by_name(name)) {</div><div class='del'>-		ret = -EEXIST;</div><div class='del'>-		goto out;</div><div class='add'>+		up_write(&amp;devices_rwsem);</div><div class='add'>+		return -EEXIST;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ret = device_rename(&amp;ibdev-&gt;dev, name);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto out;</div><div class='del'>-	strlcpy(ibdev-&gt;name, name, IB_DEVICE_NAME_MAX);</div><div class='add'>+	if (ret) {</div><div class='add'>+		up_write(&amp;devices_rwsem);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	strscpy(ibdev-&gt;name, name, IB_DEVICE_NAME_MAX);</div><div class='ctx'> 	ret = rename_compat_devs(ibdev);</div><div class='del'>-out:</div><div class='del'>-	up_write(&amp;devices_rwsem);</div><div class='del'>-	return ret;</div><div class='add'>+</div><div class='add'>+	downgrade_write(&amp;devices_rwsem);</div><div class='add'>+	down_read(&amp;ibdev-&gt;client_data_rwsem);</div><div class='add'>+	xan_for_each_marked(&amp;ibdev-&gt;client_data, index, client_data,</div><div class='add'>+			    CLIENT_DATA_REGISTERED) {</div><div class='add'>+		struct ib_client *client = xa_load(&amp;clients, index);</div><div class='add'>+</div><div class='add'>+		if (!client || !client-&gt;rename)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		client-&gt;rename(ibdev, client_data);</div><div class='add'>+	}</div><div class='add'>+	up_read(&amp;ibdev-&gt;client_data_rwsem);</div><div class='add'>+	rdma_nl_notify_event(ibdev, 0, RDMA_RENAME_EVENT);</div><div class='add'>+	up_read(&amp;devices_rwsem);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ib_device_set_dim(struct ib_device *ibdev, u8 use_dim)</div><div class='add'>+{</div><div class='add'>+	if (use_dim &gt; 1)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	ibdev-&gt;use_cq_dim = use_dim;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int alloc_name(struct ib_device *ibdev, const char *name)</div><div class='hunk'>@@ -440,7 +469,7 @@ static int alloc_name(struct ib_device *ibdev, const char *name)</div><div class='ctx'> 	int rc;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	lockdep_assert_held_exclusive(&amp;devices_rwsem);</div><div class='add'>+	lockdep_assert_held_write(&amp;devices_rwsem);</div><div class='ctx'> 	ida_init(&amp;inuse);</div><div class='ctx'> 	xa_for_each (&amp;devices, index, device) {</div><div class='ctx'> 		char buf[IB_DEVICE_NAME_MAX];</div><div class='hunk'>@@ -474,18 +503,27 @@ static void ib_device_release(struct device *device)</div><div class='ctx'> </div><div class='ctx'> 	free_netdevs(dev);</div><div class='ctx'> 	WARN_ON(refcount_read(&amp;dev-&gt;refcount));</div><div class='del'>-	ib_cache_release_one(dev);</div><div class='del'>-	ib_security_release_port_pkey_list(dev);</div><div class='del'>-	xa_destroy(&amp;dev-&gt;compat_devs);</div><div class='del'>-	xa_destroy(&amp;dev-&gt;client_data);</div><div class='del'>-	if (dev-&gt;port_data)</div><div class='add'>+	if (dev-&gt;hw_stats_data)</div><div class='add'>+		ib_device_release_hw_stats(dev-&gt;hw_stats_data);</div><div class='add'>+	if (dev-&gt;port_data) {</div><div class='add'>+		ib_cache_release_one(dev);</div><div class='add'>+		ib_security_release_port_pkey_list(dev);</div><div class='add'>+		rdma_counter_release(dev);</div><div class='ctx'> 		kfree_rcu(container_of(dev-&gt;port_data, struct ib_port_data_rcu,</div><div class='ctx'> 				       pdata[0]),</div><div class='ctx'> 			  rcu_head);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_destroy(&amp;dev-&gt;subdev_lock);</div><div class='add'>+	mutex_destroy(&amp;dev-&gt;unregistration_lock);</div><div class='add'>+	mutex_destroy(&amp;dev-&gt;compat_devs_mutex);</div><div class='add'>+</div><div class='add'>+	xa_destroy(&amp;dev-&gt;compat_devs);</div><div class='add'>+	xa_destroy(&amp;dev-&gt;client_data);</div><div class='ctx'> 	kfree_rcu(dev, rcu_head);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int ib_device_uevent(struct device *device,</div><div class='add'>+static int ib_device_uevent(const struct device *device,</div><div class='ctx'> 			    struct kobj_uevent_env *env)</div><div class='ctx'> {</div><div class='ctx'> 	if (add_uevent_var(env, "NAME=%s", dev_name(device)))</div><div class='hunk'>@@ -498,9 +536,9 @@ static int ib_device_uevent(struct device *device,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static const void *net_namespace(struct device *d)</div><div class='add'>+static const void *net_namespace(const struct device *d)</div><div class='ctx'> {</div><div class='del'>-	struct ib_core_device *coredev =</div><div class='add'>+	const struct ib_core_device *coredev =</div><div class='ctx'> 			container_of(d, struct ib_core_device, dev);</div><div class='ctx'> </div><div class='ctx'> 	return read_pnet(&amp;coredev-&gt;rdma_net);</div><div class='hunk'>@@ -517,6 +555,8 @@ static struct class ib_class = {</div><div class='ctx'> static void rdma_init_coredev(struct ib_core_device *coredev,</div><div class='ctx'> 			      struct ib_device *dev, struct net *net)</div><div class='ctx'> {</div><div class='add'>+	bool is_full_dev = &amp;dev-&gt;coredev == coredev;</div><div class='add'>+</div><div class='ctx'> 	/* This BUILD_BUG_ON is intended to catch layout change</div><div class='ctx'> 	 * of union of ib_core_device and device.</div><div class='ctx'> 	 * dev must be the first element as ib_core and providers</div><div class='hunk'>@@ -528,6 +568,13 @@ static void rdma_init_coredev(struct ib_core_device *coredev,</div><div class='ctx'> </div><div class='ctx'> 	coredev-&gt;dev.class = &amp;ib_class;</div><div class='ctx'> 	coredev-&gt;dev.groups = dev-&gt;groups;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Don't expose hw counters outside of the init namespace.</div><div class='add'>+	 */</div><div class='add'>+	if (!is_full_dev &amp;&amp; dev-&gt;hw_stats_attr_index)</div><div class='add'>+		coredev-&gt;dev.groups[dev-&gt;hw_stats_attr_index] = NULL;</div><div class='add'>+</div><div class='ctx'> 	device_initialize(&amp;coredev-&gt;dev);</div><div class='ctx'> 	coredev-&gt;owner = dev;</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;coredev-&gt;port_list);</div><div class='hunk'>@@ -537,6 +584,8 @@ static void rdma_init_coredev(struct ib_core_device *coredev,</div><div class='ctx'> /**</div><div class='ctx'>  * _ib_alloc_device - allocate an IB device struct</div><div class='ctx'>  * @size:size of structure to allocate</div><div class='add'>+ * @net: network namespace device should be located in, namespace</div><div class='add'>+ *       must stay valid until ib_register_device() is completed.</div><div class='ctx'>  *</div><div class='ctx'>  * Low-level drivers should use ib_alloc_device() to allocate &amp;struct</div><div class='ctx'>  * ib_device.  @size is the size of the structure to be allocated,</div><div class='hunk'>@@ -544,9 +593,10 @@ static void rdma_init_coredev(struct ib_core_device *coredev,</div><div class='ctx'>  * ib_dealloc_device() must be used to free structures allocated with</div><div class='ctx'>  * ib_alloc_device().</div><div class='ctx'>  */</div><div class='del'>-struct ib_device *_ib_alloc_device(size_t size)</div><div class='add'>+struct ib_device *_ib_alloc_device(size_t size, struct net *net)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *device;</div><div class='add'>+	unsigned int i;</div><div class='ctx'> </div><div class='ctx'> 	if (WARN_ON(size &lt; sizeof(struct ib_device)))</div><div class='ctx'> 		return NULL;</div><div class='hunk'>@@ -560,11 +610,19 @@ struct ib_device *_ib_alloc_device(size_t size)</div><div class='ctx'> 		return NULL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	device-&gt;groups[0] = &amp;ib_dev_attr_group;</div><div class='del'>-	rdma_init_coredev(&amp;device-&gt;coredev, device, &amp;init_net);</div><div class='add'>+	/* ib_devices_shared_netns can't change while we have active namespaces</div><div class='add'>+	 * in the system which means either init_net is passed or the user has</div><div class='add'>+	 * no idea what they are doing.</div><div class='add'>+	 *</div><div class='add'>+	 * To avoid breaking backward compatibility, when in shared mode,</div><div class='add'>+	 * force to init the device in the init_net.</div><div class='add'>+	 */</div><div class='add'>+	net = ib_devices_shared_netns ? &amp;init_net : net;</div><div class='add'>+	rdma_init_coredev(&amp;device-&gt;coredev, device, net);</div><div class='ctx'> </div><div class='ctx'> 	INIT_LIST_HEAD(&amp;device-&gt;event_handler_list);</div><div class='del'>-	spin_lock_init(&amp;device-&gt;event_handler_lock);</div><div class='add'>+	spin_lock_init(&amp;device-&gt;qp_open_list_lock);</div><div class='add'>+	init_rwsem(&amp;device-&gt;event_handler_rwsem);</div><div class='ctx'> 	mutex_init(&amp;device-&gt;unregistration_lock);</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * client_data needs to be alloc because we don't want our mark to be</div><div class='hunk'>@@ -577,6 +635,48 @@ struct ib_device *_ib_alloc_device(size_t size)</div><div class='ctx'> 	init_completion(&amp;device-&gt;unreg_completion);</div><div class='ctx'> 	INIT_WORK(&amp;device-&gt;unregistration_work, ib_unregister_work);</div><div class='ctx'> </div><div class='add'>+	spin_lock_init(&amp;device-&gt;cq_pools_lock);</div><div class='add'>+	for (i = 0; i &lt; ARRAY_SIZE(device-&gt;cq_pools); i++)</div><div class='add'>+		INIT_LIST_HEAD(&amp;device-&gt;cq_pools[i]);</div><div class='add'>+</div><div class='add'>+	rwlock_init(&amp;device-&gt;cache_lock);</div><div class='add'>+</div><div class='add'>+	device-&gt;uverbs_cmd_mask =</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_ALLOC_MW) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_ATTACH_MCAST) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_CLOSE_XRCD) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_CREATE_AH) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_CREATE_QP) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_CREATE_SRQ) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_CREATE_XSRQ) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_MW) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_DESTROY_AH) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_DESTROY_CQ) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_DESTROY_SRQ) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_DETACH_MCAST) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_MODIFY_QP) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_MODIFY_SRQ) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_OPEN_QP) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_OPEN_XRCD) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_QUERY_SRQ) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_REG_MR) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_REREG_MR) |</div><div class='add'>+		BIT_ULL(IB_USER_VERBS_CMD_RESIZE_CQ);</div><div class='add'>+</div><div class='add'>+	mutex_init(&amp;device-&gt;subdev_lock);</div><div class='add'>+	INIT_LIST_HEAD(&amp;device-&gt;subdev_list_head);</div><div class='add'>+	INIT_LIST_HEAD(&amp;device-&gt;subdev_list);</div><div class='add'>+</div><div class='ctx'> 	return device;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(_ib_alloc_device);</div><div class='hunk'>@@ -633,6 +733,14 @@ static int add_client_context(struct ib_device *device,</div><div class='ctx'> </div><div class='ctx'> 	down_write(&amp;device-&gt;client_data_rwsem);</div><div class='ctx'> 	/*</div><div class='add'>+	 * So long as the client is registered hold both the client and device</div><div class='add'>+	 * unregistration locks.</div><div class='add'>+	 */</div><div class='add'>+	if (!refcount_inc_not_zero(&amp;client-&gt;uses))</div><div class='add'>+		goto out_unlock;</div><div class='add'>+	refcount_inc(&amp;device-&gt;refcount);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='ctx'> 	 * Another caller to add_client_context got here first and has already</div><div class='ctx'> 	 * completely initialized context.</div><div class='ctx'> 	 */</div><div class='hunk'>@@ -645,8 +753,20 @@ static int add_client_context(struct ib_device *device,</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	downgrade_write(&amp;device-&gt;client_data_rwsem);</div><div class='del'>-	if (client-&gt;add)</div><div class='del'>-		client-&gt;add(device);</div><div class='add'>+	if (client-&gt;add) {</div><div class='add'>+		if (client-&gt;add(device)) {</div><div class='add'>+			/*</div><div class='add'>+			 * If a client fails to add then the error code is</div><div class='add'>+			 * ignored, but we won't call any more ops on this</div><div class='add'>+			 * client.</div><div class='add'>+			 */</div><div class='add'>+			xa_erase(&amp;device-&gt;client_data, client-&gt;client_id);</div><div class='add'>+			up_read(&amp;device-&gt;client_data_rwsem);</div><div class='add'>+			ib_device_put(device);</div><div class='add'>+			ib_client_put(client);</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	/* Readers shall not see a client until add has been completed */</div><div class='ctx'> 	xa_set_mark(&amp;device-&gt;client_data, client-&gt;client_id,</div><div class='hunk'>@@ -655,6 +775,9 @@ static int add_client_context(struct ib_device *device,</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='add'>+	ib_device_put(device);</div><div class='add'>+	ib_client_put(client);</div><div class='add'>+out_unlock:</div><div class='ctx'> 	up_write(&amp;device-&gt;client_data_rwsem);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -674,7 +797,7 @@ static void remove_client_context(struct ib_device *device,</div><div class='ctx'> 	client_data = xa_load(&amp;device-&gt;client_data, client_id);</div><div class='ctx'> 	xa_clear_mark(&amp;device-&gt;client_data, client_id, CLIENT_DATA_REGISTERED);</div><div class='ctx'> 	client = xa_load(&amp;clients, client_id);</div><div class='del'>-	downgrade_write(&amp;device-&gt;client_data_rwsem);</div><div class='add'>+	up_write(&amp;device-&gt;client_data_rwsem);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Notice we cannot be holding any exclusive locks when calling the</div><div class='hunk'>@@ -684,23 +807,19 @@ static void remove_client_context(struct ib_device *device,</div><div class='ctx'> 	 *</div><div class='ctx'> 	 * For this reason clients and drivers should not call the</div><div class='ctx'> 	 * unregistration functions will holdling any locks.</div><div class='del'>-	 *</div><div class='del'>-	 * It tempting to drop the client_data_rwsem too, but this is required</div><div class='del'>-	 * to ensure that unregister_client does not return until all clients</div><div class='del'>-	 * are completely unregistered, which is required to avoid module</div><div class='del'>-	 * unloading races.</div><div class='ctx'> 	 */</div><div class='ctx'> 	if (client-&gt;remove)</div><div class='ctx'> 		client-&gt;remove(device, client_data);</div><div class='ctx'> </div><div class='ctx'> 	xa_erase(&amp;device-&gt;client_data, client_id);</div><div class='del'>-	up_read(&amp;device-&gt;client_data_rwsem);</div><div class='add'>+	ib_device_put(device);</div><div class='add'>+	ib_client_put(client);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int alloc_port_data(struct ib_device *device)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_data_rcu *pdata_rcu;</div><div class='del'>-	unsigned int port;</div><div class='add'>+	u32 port;</div><div class='ctx'> </div><div class='ctx'> 	if (device-&gt;port_data)</div><div class='ctx'> 		return 0;</div><div class='hunk'>@@ -709,6 +828,10 @@ static int alloc_port_data(struct ib_device *device)</div><div class='ctx'> 	if (WARN_ON(!device-&gt;phys_port_cnt))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='add'>+	/* Reserve U32_MAX so the logic to go over all the ports is sane */</div><div class='add'>+	if (WARN_ON(device-&gt;phys_port_cnt == U32_MAX))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * device-&gt;port_data is indexed directly by the port number to make</div><div class='ctx'> 	 * access to this data as efficient as possible.</div><div class='hunk'>@@ -717,7 +840,7 @@ static int alloc_port_data(struct ib_device *device)</div><div class='ctx'> 	 * empty slots at the beginning.</div><div class='ctx'> 	 */</div><div class='ctx'> 	pdata_rcu = kzalloc(struct_size(pdata_rcu, pdata,</div><div class='del'>-					rdma_end_port(device) + 1),</div><div class='add'>+					size_add(rdma_end_port(device), 1)),</div><div class='ctx'> 			    GFP_KERNEL);</div><div class='ctx'> 	if (!pdata_rcu)</div><div class='ctx'> 		return -ENOMEM;</div><div class='hunk'>@@ -740,7 +863,7 @@ static int alloc_port_data(struct ib_device *device)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int verify_immutable(const struct ib_device *dev, u8 port)</div><div class='add'>+static int verify_immutable(const struct ib_device *dev, u32 port)</div><div class='ctx'> {</div><div class='ctx'> 	return WARN_ON(!rdma_cap_ib_mad(dev, port) &amp;&amp;</div><div class='ctx'> 			    rdma_max_mad_size(dev, port) != 0);</div><div class='hunk'>@@ -748,7 +871,7 @@ static int verify_immutable(const struct ib_device *dev, u8 port)</div><div class='ctx'> </div><div class='ctx'> static int setup_port_data(struct ib_device *device)</div><div class='ctx'> {</div><div class='del'>-	unsigned int port;</div><div class='add'>+	u32 port;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	ret = alloc_port_data(device);</div><div class='hunk'>@@ -769,6 +892,20 @@ static int setup_port_data(struct ib_device *device)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+/**</div><div class='add'>+ * ib_port_immutable_read() - Read rdma port's immutable data</div><div class='add'>+ * @dev: IB device</div><div class='add'>+ * @port: port number whose immutable data to read. It starts with index 1 and</div><div class='add'>+ *        valid upto including rdma_end_port().</div><div class='add'>+ */</div><div class='add'>+const struct ib_port_immutable*</div><div class='add'>+ib_port_immutable_read(struct ib_device *dev, unsigned int port)</div><div class='add'>+{</div><div class='add'>+	WARN_ON(!rdma_is_port_valid(dev, port));</div><div class='add'>+	return &amp;dev-&gt;port_data[port].immutable;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_port_immutable_read);</div><div class='add'>+</div><div class='ctx'> void ib_get_device_fw_str(struct ib_device *dev, char *str)</div><div class='ctx'> {</div><div class='ctx'> 	if (dev-&gt;ops.get_dev_fw_str)</div><div class='hunk'>@@ -789,15 +926,8 @@ static void ib_policy_change_task(struct work_struct *work)</div><div class='ctx'> </div><div class='ctx'> 		rdma_for_each_port (dev, i) {</div><div class='ctx'> 			u64 sp;</div><div class='del'>-			int ret = ib_get_cached_subnet_prefix(dev,</div><div class='del'>-							      i,</div><div class='del'>-							      &amp;sp);</div><div class='del'>-</div><div class='del'>-			WARN_ONCE(ret,</div><div class='del'>-				  "ib_get_cached_subnet_prefix err: %d, this should never happen here\n",</div><div class='del'>-				  ret);</div><div class='del'>-			if (!ret)</div><div class='del'>-				ib_security_cache_change(dev, i, sp);</div><div class='add'>+			ib_get_cached_subnet_prefix(dev, i, &amp;sp);</div><div class='add'>+			ib_security_cache_change(dev, i, sp);</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> 	up_read(&amp;devices_rwsem);</div><div class='hunk'>@@ -865,7 +995,9 @@ static int add_one_compat_dev(struct ib_device *device,</div><div class='ctx'> 	cdev-&gt;dev.parent = device-&gt;dev.parent;</div><div class='ctx'> 	rdma_init_coredev(cdev, device, read_pnet(&amp;rnet-&gt;net));</div><div class='ctx'> 	cdev-&gt;dev.release = compatdev_release;</div><div class='del'>-	dev_set_name(&amp;cdev-&gt;dev, "%s", dev_name(&amp;device-&gt;dev));</div><div class='add'>+	ret = dev_set_name(&amp;cdev-&gt;dev, "%s", dev_name(&amp;device-&gt;dev));</div><div class='add'>+	if (ret)</div><div class='add'>+		goto add_err;</div><div class='ctx'> </div><div class='ctx'> 	ret = device_add(&amp;cdev-&gt;dev);</div><div class='ctx'> 	if (ret)</div><div class='hunk'>@@ -1019,7 +1151,7 @@ int rdma_compatdev_set(u8 enable)</div><div class='ctx'> </div><div class='ctx'> static void rdma_dev_exit_net(struct net *net)</div><div class='ctx'> {</div><div class='del'>-	struct rdma_dev_net *rnet = net_generic(net, rdma_dev_net_id);</div><div class='add'>+	struct rdma_dev_net *rnet = rdma_net_to_dev_net(net);</div><div class='ctx'> 	struct ib_device *dev;</div><div class='ctx'> 	unsigned long index;</div><div class='ctx'> 	int ret;</div><div class='hunk'>@@ -1053,25 +1185,32 @@ static void rdma_dev_exit_net(struct net *net)</div><div class='ctx'> 	}</div><div class='ctx'> 	up_read(&amp;devices_rwsem);</div><div class='ctx'> </div><div class='add'>+	rdma_nl_net_exit(rnet);</div><div class='ctx'> 	xa_erase(&amp;rdma_nets, rnet-&gt;id);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static __net_init int rdma_dev_init_net(struct net *net)</div><div class='ctx'> {</div><div class='del'>-	struct rdma_dev_net *rnet = net_generic(net, rdma_dev_net_id);</div><div class='add'>+	struct rdma_dev_net *rnet = rdma_net_to_dev_net(net);</div><div class='ctx'> 	unsigned long index;</div><div class='ctx'> 	struct ib_device *dev;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	write_pnet(&amp;rnet-&gt;net, net);</div><div class='add'>+</div><div class='add'>+	ret = rdma_nl_net_init(rnet);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='ctx'> 	/* No need to create any compat devices in default init_net. */</div><div class='ctx'> 	if (net_eq(net, &amp;init_net))</div><div class='ctx'> 		return 0;</div><div class='ctx'> </div><div class='del'>-	write_pnet(&amp;rnet-&gt;net, net);</div><div class='del'>-</div><div class='ctx'> 	ret = xa_alloc(&amp;rdma_nets, &amp;rnet-&gt;id, rnet, xa_limit_32b, GFP_KERNEL);</div><div class='del'>-	if (ret)</div><div class='add'>+	if (ret) {</div><div class='add'>+		rdma_nl_net_exit(rnet);</div><div class='ctx'> 		return ret;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	down_read(&amp;devices_rwsem);</div><div class='ctx'> 	xa_for_each_marked (&amp;devices, index, dev, DEVICE_REGISTERED) {</div><div class='hunk'>@@ -1114,7 +1253,7 @@ static int assign_name(struct ib_device *device, const char *name)</div><div class='ctx'> 		ret = -ENFILE;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='del'>-	strlcpy(device-&gt;name, dev_name(&amp;device-&gt;dev), IB_DEVICE_NAME_MAX);</div><div class='add'>+	strscpy(device-&gt;name, dev_name(&amp;device-&gt;dev), IB_DEVICE_NAME_MAX);</div><div class='ctx'> </div><div class='ctx'> 	ret = xa_alloc_cyclic(&amp;devices, &amp;device-&gt;index, device, xa_limit_31b,</div><div class='ctx'> 			&amp;last_id, GFP_KERNEL);</div><div class='hunk'>@@ -1126,44 +1265,6 @@ out:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void setup_dma_device(struct ib_device *device)</div><div class='del'>-{</div><div class='del'>-	struct device *parent = device-&gt;dev.parent;</div><div class='del'>-</div><div class='del'>-	WARN_ON_ONCE(device-&gt;dma_device);</div><div class='del'>-	if (device-&gt;dev.dma_ops) {</div><div class='del'>-		/*</div><div class='del'>-		 * The caller provided custom DMA operations. Copy the</div><div class='del'>-		 * DMA-related fields that are used by e.g. dma_alloc_coherent()</div><div class='del'>-		 * into device-&gt;dev.</div><div class='del'>-		 */</div><div class='del'>-		device-&gt;dma_device = &amp;device-&gt;dev;</div><div class='del'>-		if (!device-&gt;dev.dma_mask) {</div><div class='del'>-			if (parent)</div><div class='del'>-				device-&gt;dev.dma_mask = parent-&gt;dma_mask;</div><div class='del'>-			else</div><div class='del'>-				WARN_ON_ONCE(true);</div><div class='del'>-		}</div><div class='del'>-		if (!device-&gt;dev.coherent_dma_mask) {</div><div class='del'>-			if (parent)</div><div class='del'>-				device-&gt;dev.coherent_dma_mask =</div><div class='del'>-					parent-&gt;coherent_dma_mask;</div><div class='del'>-			else</div><div class='del'>-				WARN_ON_ONCE(true);</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		/*</div><div class='del'>-		 * The caller did not provide custom DMA operations. Use the</div><div class='del'>-		 * DMA mapping operations of the parent device.</div><div class='del'>-		 */</div><div class='del'>-		WARN_ON_ONCE(!parent);</div><div class='del'>-		device-&gt;dma_device = parent;</div><div class='del'>-	}</div><div class='del'>-	/* Setup default max segment size for all IB devices */</div><div class='del'>-	dma_set_max_seg_size(device-&gt;dma_device, SZ_2G);</div><div class='del'>-</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> /*</div><div class='ctx'>  * setup_device() allocates memory and sets up data that requires calling the</div><div class='ctx'>  * device ops, this is the only reason these actions are not done during</div><div class='hunk'>@@ -1174,11 +1275,7 @@ static int setup_device(struct ib_device *device)</div><div class='ctx'> 	struct ib_udata uhw = {.outlen = 0, .inlen = 0};</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	setup_dma_device(device);</div><div class='del'>-</div><div class='del'>-	ret = ib_device_check_mandatory(device);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='add'>+	ib_device_check_mandatory(device);</div><div class='ctx'> </div><div class='ctx'> 	ret = setup_port_data(device);</div><div class='ctx'> 	if (ret) {</div><div class='hunk'>@@ -1199,7 +1296,7 @@ static int setup_device(struct ib_device *device)</div><div class='ctx'> </div><div class='ctx'> static void disable_device(struct ib_device *device)</div><div class='ctx'> {</div><div class='del'>-	struct ib_client *client;</div><div class='add'>+	u32 cid;</div><div class='ctx'> </div><div class='ctx'> 	WARN_ON(!refcount_read(&amp;device-&gt;refcount));</div><div class='ctx'> </div><div class='hunk'>@@ -1207,10 +1304,21 @@ static void disable_device(struct ib_device *device)</div><div class='ctx'> 	xa_clear_mark(&amp;devices, device-&gt;index, DEVICE_REGISTERED);</div><div class='ctx'> 	up_write(&amp;devices_rwsem);</div><div class='ctx'> </div><div class='add'>+	/*</div><div class='add'>+	 * Remove clients in LIFO order, see assign_client_id. This could be</div><div class='add'>+	 * more efficient if xarray learns to reverse iterate. Since no new</div><div class='add'>+	 * clients can be added to this ib_device past this point we only need</div><div class='add'>+	 * the maximum possible client_id value here.</div><div class='add'>+	 */</div><div class='ctx'> 	down_read(&amp;clients_rwsem);</div><div class='del'>-	list_for_each_entry_reverse(client, &amp;client_list, list)</div><div class='del'>-		remove_client_context(device, client-&gt;client_id);</div><div class='add'>+	cid = highest_client_id;</div><div class='ctx'> 	up_read(&amp;clients_rwsem);</div><div class='add'>+	while (cid) {</div><div class='add'>+		cid--;</div><div class='add'>+		remove_client_context(device, cid);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ib_cq_pool_cleanup(device);</div><div class='ctx'> </div><div class='ctx'> 	/* Pairs with refcount_set in enable_device */</div><div class='ctx'> 	ib_device_put(device);</div><div class='hunk'>@@ -1269,9 +1377,49 @@ out:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void prevent_dealloc_device(struct ib_device *ib_dev)</div><div class='add'>+{</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ib_device_notify_register(struct ib_device *device)</div><div class='add'>+{</div><div class='add'>+	struct net_device *netdev;</div><div class='add'>+	u32 port;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	down_read(&amp;devices_rwsem);</div><div class='add'>+</div><div class='add'>+	/* Mark for userspace that device is ready */</div><div class='add'>+	kobject_uevent(&amp;device-&gt;dev.kobj, KOBJ_ADD);</div><div class='add'>+</div><div class='add'>+	ret = rdma_nl_notify_event(device, 0, RDMA_REGISTER_EVENT);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	rdma_for_each_port(device, port) {</div><div class='add'>+		netdev = ib_device_get_netdev(device, port);</div><div class='add'>+		if (!netdev)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		ret = rdma_nl_notify_event(device, port,</div><div class='add'>+					   RDMA_NETDEV_ATTACH_EVENT);</div><div class='add'>+		dev_put(netdev);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	up_read(&amp;devices_rwsem);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /**</div><div class='ctx'>  * ib_register_device - Register an IB device with IB core</div><div class='del'>- * @device:Device to register</div><div class='add'>+ * @device: Device to register</div><div class='add'>+ * @name: unique string device name. This may include a '%' which will</div><div class='add'>+ * 	  cause a unique index to be added to the passed device name.</div><div class='add'>+ * @dma_device: pointer to a DMA-capable device. If %NULL, then the IB</div><div class='add'>+ *	        device will be used. In this case the caller should fully</div><div class='add'>+ *		setup the ibdev for DMA. This usually means using dma_virt_ops.</div><div class='ctx'>  *</div><div class='ctx'>  * Low-level drivers use ib_register_device() to register their</div><div class='ctx'>  * devices with the IB core.  All registered clients will receive a</div><div class='hunk'>@@ -1282,7 +1430,8 @@ out:</div><div class='ctx'>  * asynchronously then the device pointer may become freed as soon as this</div><div class='ctx'>  * function returns.</div><div class='ctx'>  */</div><div class='del'>-int ib_register_device(struct ib_device *device, const char *name)</div><div class='add'>+int ib_register_device(struct ib_device *device, const char *name,</div><div class='add'>+		       struct device *dma_device)</div><div class='ctx'> {</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='hunk'>@@ -1290,6 +1439,14 @@ int ib_register_device(struct ib_device *device, const char *name)</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='add'>+	/*</div><div class='add'>+	 * If the caller does not provide a DMA capable device then the IB core</div><div class='add'>+	 * will set up ib_sge and scatterlist structures that stash the kernel</div><div class='add'>+	 * virtual address into the address field.</div><div class='add'>+	 */</div><div class='add'>+	WARN_ON(dma_device &amp;&amp; !dma_device-&gt;dma_parms);</div><div class='add'>+	device-&gt;dma_device = dma_device;</div><div class='add'>+</div><div class='ctx'> 	ret = setup_device(device);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='hunk'>@@ -1301,8 +1458,16 @@ int ib_register_device(struct ib_device *device, const char *name)</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	device-&gt;groups[0] = &amp;ib_dev_attr_group;</div><div class='add'>+	device-&gt;groups[1] = device-&gt;ops.device_group;</div><div class='add'>+	ret = ib_setup_device_attrs(device);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto cache_cleanup;</div><div class='add'>+</div><div class='ctx'> 	ib_device_register_rdmacg(device);</div><div class='ctx'> </div><div class='add'>+	rdma_counter_init(device);</div><div class='add'>+</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Ensure that ADD uevent is not fired because it</div><div class='ctx'> 	 * is too early amd device is not initialized yet.</div><div class='hunk'>@@ -1312,7 +1477,7 @@ int ib_register_device(struct ib_device *device, const char *name)</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto cg_cleanup;</div><div class='ctx'> </div><div class='del'>-	ret = ib_device_register_sysfs(device);</div><div class='add'>+	ret = ib_setup_port_attrs(&amp;device-&gt;coredev);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		dev_warn(&amp;device-&gt;dev,</div><div class='ctx'> 			 "Couldn't register device with driver model\n");</div><div class='hunk'>@@ -1320,9 +1485,6 @@ int ib_register_device(struct ib_device *device, const char *name)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ret = enable_device_and_get(device);</div><div class='del'>-	dev_set_uevent_suppress(&amp;device-&gt;dev, false);</div><div class='del'>-	/* Mark for userspace that device is ready */</div><div class='del'>-	kobject_uevent(&amp;device-&gt;dev.kobj, KOBJ_ADD);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		void (*dealloc_fn)(struct ib_device *);</div><div class='ctx'> </div><div class='hunk'>@@ -1334,16 +1496,21 @@ int ib_register_device(struct ib_device *device, const char *name)</div><div class='ctx'> 		 * possibility for a parallel unregistration along with this</div><div class='ctx'> 		 * error flow. Since we have a refcount here we know any</div><div class='ctx'> 		 * parallel flow is stopped in disable_device and will see the</div><div class='del'>-		 * NULL pointers, causing the responsibility to</div><div class='add'>+		 * special dealloc_driver pointer, causing the responsibility to</div><div class='ctx'> 		 * ib_dealloc_device() to revert back to this thread.</div><div class='ctx'> 		 */</div><div class='ctx'> 		dealloc_fn = device-&gt;ops.dealloc_driver;</div><div class='del'>-		device-&gt;ops.dealloc_driver = NULL;</div><div class='add'>+		device-&gt;ops.dealloc_driver = prevent_dealloc_device;</div><div class='ctx'> 		ib_device_put(device);</div><div class='ctx'> 		__ib_unregister_device(device);</div><div class='ctx'> 		device-&gt;ops.dealloc_driver = dealloc_fn;</div><div class='add'>+		dev_set_uevent_suppress(&amp;device-&gt;dev, false);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='add'>+	dev_set_uevent_suppress(&amp;device-&gt;dev, false);</div><div class='add'>+</div><div class='add'>+	ib_device_notify_register(device);</div><div class='add'>+</div><div class='ctx'> 	ib_device_put(device);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='hunk'>@@ -1353,6 +1520,7 @@ dev_cleanup:</div><div class='ctx'> cg_cleanup:</div><div class='ctx'> 	dev_set_uevent_suppress(&amp;device-&gt;dev, false);</div><div class='ctx'> 	ib_device_unregister_rdmacg(device);</div><div class='add'>+cache_cleanup:</div><div class='ctx'> 	ib_cache_cleanup_one(device);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -1361,9 +1529,21 @@ EXPORT_SYMBOL(ib_register_device);</div><div class='ctx'> /* Callers must hold a get on the device. */</div><div class='ctx'> static void __ib_unregister_device(struct ib_device *ib_dev)</div><div class='ctx'> {</div><div class='add'>+	struct ib_device *sub, *tmp;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;ib_dev-&gt;subdev_lock);</div><div class='add'>+	list_for_each_entry_safe_reverse(sub, tmp,</div><div class='add'>+					 &amp;ib_dev-&gt;subdev_list_head,</div><div class='add'>+					 subdev_list) {</div><div class='add'>+		list_del(&amp;sub-&gt;subdev_list);</div><div class='add'>+		ib_dev-&gt;ops.del_sub_dev(sub);</div><div class='add'>+		ib_device_put(ib_dev);</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;ib_dev-&gt;subdev_lock);</div><div class='add'>+</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * We have a registration lock so that all the calls to unregister are</div><div class='del'>-	 * fully fenced, once any unregister returns the device is truely</div><div class='add'>+	 * fully fenced, once any unregister returns the device is truly</div><div class='ctx'> 	 * unregistered even if multiple callers are unregistering it at the</div><div class='ctx'> 	 * same time. This also interacts with the registration flow and</div><div class='ctx'> 	 * provides sane semantics if register and unregister are racing.</div><div class='hunk'>@@ -1373,11 +1553,12 @@ static void __ib_unregister_device(struct ib_device *ib_dev)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='ctx'> 	disable_device(ib_dev);</div><div class='add'>+	rdma_nl_notify_event(ib_dev, 0, RDMA_UNREGISTER_EVENT);</div><div class='ctx'> </div><div class='ctx'> 	/* Expedite removing unregistered pointers from the hash table */</div><div class='ctx'> 	free_netdevs(ib_dev);</div><div class='ctx'> </div><div class='del'>-	ib_device_unregister_sysfs(ib_dev);</div><div class='add'>+	ib_free_port_attrs(&amp;ib_dev-&gt;coredev);</div><div class='ctx'> 	device_del(&amp;ib_dev-&gt;dev);</div><div class='ctx'> 	ib_device_unregister_rdmacg(ib_dev);</div><div class='ctx'> 	ib_cache_cleanup_one(ib_dev);</div><div class='hunk'>@@ -1386,7 +1567,8 @@ static void __ib_unregister_device(struct ib_device *ib_dev)</div><div class='ctx'> 	 * Drivers using the new flow may not call ib_dealloc_device except</div><div class='ctx'> 	 * in error unwind prior to registration success.</div><div class='ctx'> 	 */</div><div class='del'>-	if (ib_dev-&gt;ops.dealloc_driver) {</div><div class='add'>+	if (ib_dev-&gt;ops.dealloc_driver &amp;&amp;</div><div class='add'>+	    ib_dev-&gt;ops.dealloc_driver != prevent_dealloc_device) {</div><div class='ctx'> 		WARN_ON(kref_read(&amp;ib_dev-&gt;dev.kobj.kref) &lt;= 1);</div><div class='ctx'> 		ib_dealloc_device(ib_dev);</div><div class='ctx'> 	}</div><div class='hunk'>@@ -1396,7 +1578,7 @@ out:</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * ib_unregister_device - Unregister an IB device</div><div class='del'>- * @device: The device to unregister</div><div class='add'>+ * @ib_dev: The device to unregister</div><div class='ctx'>  *</div><div class='ctx'>  * Unregister an IB device.  All clients will receive a remove callback.</div><div class='ctx'>  *</div><div class='hunk'>@@ -1418,7 +1600,7 @@ EXPORT_SYMBOL(ib_unregister_device);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * ib_unregister_device_and_put - Unregister a device while holding a 'get'</div><div class='del'>- * device: The device to unregister</div><div class='add'>+ * @ib_dev: The device to unregister</div><div class='ctx'>  *</div><div class='ctx'>  * This is the same as ib_unregister_device(), except it includes an internal</div><div class='ctx'>  * ib_device_put() that should match a 'get' obtained by the caller.</div><div class='hunk'>@@ -1461,7 +1643,7 @@ void ib_unregister_driver(enum rdma_driver_id driver_id)</div><div class='ctx'> </div><div class='ctx'> 	down_read(&amp;devices_rwsem);</div><div class='ctx'> 	xa_for_each (&amp;devices, index, ib_dev) {</div><div class='del'>-		if (ib_dev-&gt;driver_id != driver_id)</div><div class='add'>+		if (ib_dev-&gt;ops.driver_id != driver_id)</div><div class='ctx'> 			continue;</div><div class='ctx'> </div><div class='ctx'> 		get_device(&amp;ib_dev-&gt;dev);</div><div class='hunk'>@@ -1488,7 +1670,7 @@ static void ib_unregister_work(struct work_struct *work)</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * ib_unregister_device_queued - Unregister a device using a work queue</div><div class='del'>- * device: The device to unregister</div><div class='add'>+ * @ib_dev: The device to unregister</div><div class='ctx'>  *</div><div class='ctx'>  * This schedules an asynchronous unregistration using a WQ for the device. A</div><div class='ctx'>  * driver should use this to avoid holding locks while doing unregistration,</div><div class='hunk'>@@ -1502,7 +1684,7 @@ void ib_unregister_device_queued(struct ib_device *ib_dev)</div><div class='ctx'> 	WARN_ON(!refcount_read(&amp;ib_dev-&gt;refcount));</div><div class='ctx'> 	WARN_ON(!ib_dev-&gt;ops.dealloc_driver);</div><div class='ctx'> 	get_device(&amp;ib_dev-&gt;dev);</div><div class='del'>-	if (!queue_work(system_unbound_wq, &amp;ib_dev-&gt;unregistration_work))</div><div class='add'>+	if (!queue_work(ib_unreg_wq, &amp;ib_dev-&gt;unregistration_work))</div><div class='ctx'> 		put_device(&amp;ib_dev-&gt;dev);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_unregister_device_queued);</div><div class='hunk'>@@ -1594,13 +1776,11 @@ int ib_device_set_netns_put(struct sk_buff *skb,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='del'>-	 * Currently supported only for those providers which support</div><div class='del'>-	 * disassociation and don't do port specific sysfs init. Once a</div><div class='del'>-	 * port_cleanup infrastructure is implemented, this limitation will be</div><div class='del'>-	 * removed.</div><div class='add'>+	 * All the ib_clients, including uverbs, are reset when the namespace is</div><div class='add'>+	 * changed and this cannot be blocked waiting for userspace to do</div><div class='add'>+	 * something, so disassociation is mandatory.</div><div class='ctx'> 	 */</div><div class='del'>-	if (!dev-&gt;ops.disassociate_ucontext || dev-&gt;ops.init_port ||</div><div class='del'>-	    ib_devices_shared_netns) {</div><div class='add'>+	if (!dev-&gt;ops.disassociate_ucontext || ib_devices_shared_netns) {</div><div class='ctx'> 		ret = -EOPNOTSUPP;</div><div class='ctx'> 		goto ns_err;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -1631,32 +1811,30 @@ static int assign_client_id(struct ib_client *client)</div><div class='ctx'> {</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	down_write(&amp;clients_rwsem);</div><div class='add'>+	lockdep_assert_held(&amp;clients_rwsem);</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * The add/remove callbacks must be called in FIFO/LIFO order. To</div><div class='ctx'> 	 * achieve this we assign client_ids so they are sorted in</div><div class='del'>-	 * registration order, and retain a linked list we can reverse iterate</div><div class='del'>-	 * to get the LIFO order. The extra linked list can go away if xarray</div><div class='del'>-	 * learns to reverse iterate.</div><div class='add'>+	 * registration order.</div><div class='ctx'> 	 */</div><div class='del'>-	if (list_empty(&amp;client_list)) {</div><div class='del'>-		client-&gt;client_id = 0;</div><div class='del'>-	} else {</div><div class='del'>-		struct ib_client *last;</div><div class='del'>-</div><div class='del'>-		last = list_last_entry(&amp;client_list, struct ib_client, list);</div><div class='del'>-		client-&gt;client_id = last-&gt;client_id + 1;</div><div class='del'>-	}</div><div class='add'>+	client-&gt;client_id = highest_client_id;</div><div class='ctx'> 	ret = xa_insert(&amp;clients, client-&gt;client_id, client, GFP_KERNEL);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto out;</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='add'>+	highest_client_id++;</div><div class='ctx'> 	xa_set_mark(&amp;clients, client-&gt;client_id, CLIENT_REGISTERED);</div><div class='del'>-	list_add_tail(&amp;client-&gt;list, &amp;client_list);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-out:</div><div class='add'>+static void remove_client_id(struct ib_client *client)</div><div class='add'>+{</div><div class='add'>+	down_write(&amp;clients_rwsem);</div><div class='add'>+	xa_erase(&amp;clients, client-&gt;client_id);</div><div class='add'>+	for (; highest_client_id; highest_client_id--)</div><div class='add'>+		if (xa_load(&amp;clients, highest_client_id - 1))</div><div class='add'>+			break;</div><div class='ctx'> 	up_write(&amp;clients_rwsem);</div><div class='del'>-	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='hunk'>@@ -1676,23 +1854,35 @@ int ib_register_client(struct ib_client *client)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *device;</div><div class='ctx'> 	unsigned long index;</div><div class='add'>+	bool need_unreg = false;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	refcount_set(&amp;client-&gt;uses, 1);</div><div class='add'>+	init_completion(&amp;client-&gt;uses_zero);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * The devices_rwsem is held in write mode to ensure that a racing</div><div class='add'>+	 * ib_register_device() sees a consisent view of clients and devices.</div><div class='add'>+	 */</div><div class='add'>+	down_write(&amp;devices_rwsem);</div><div class='add'>+	down_write(&amp;clients_rwsem);</div><div class='ctx'> 	ret = assign_client_id(client);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		return ret;</div><div class='add'>+		goto out;</div><div class='ctx'> </div><div class='del'>-	down_read(&amp;devices_rwsem);</div><div class='add'>+	need_unreg = true;</div><div class='ctx'> 	xa_for_each_marked (&amp;devices, index, device, DEVICE_REGISTERED) {</div><div class='ctx'> 		ret = add_client_context(device, client);</div><div class='del'>-		if (ret) {</div><div class='del'>-			up_read(&amp;devices_rwsem);</div><div class='del'>-			ib_unregister_client(client);</div><div class='del'>-			return ret;</div><div class='del'>-		}</div><div class='add'>+		if (ret)</div><div class='add'>+			goto out;</div><div class='ctx'> 	}</div><div class='del'>-	up_read(&amp;devices_rwsem);</div><div class='del'>-	return 0;</div><div class='add'>+	ret = 0;</div><div class='add'>+out:</div><div class='add'>+	up_write(&amp;clients_rwsem);</div><div class='add'>+	up_write(&amp;devices_rwsem);</div><div class='add'>+	if (need_unreg &amp;&amp; ret)</div><div class='add'>+		ib_unregister_client(client);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_register_client);</div><div class='ctx'> </div><div class='hunk'>@@ -1713,24 +1903,131 @@ void ib_unregister_client(struct ib_client *client)</div><div class='ctx'> 	unsigned long index;</div><div class='ctx'> </div><div class='ctx'> 	down_write(&amp;clients_rwsem);</div><div class='add'>+	ib_client_put(client);</div><div class='ctx'> 	xa_clear_mark(&amp;clients, client-&gt;client_id, CLIENT_REGISTERED);</div><div class='ctx'> 	up_write(&amp;clients_rwsem);</div><div class='del'>-	/*</div><div class='del'>-	 * Every device still known must be serialized to make sure we are</div><div class='del'>-	 * done with the client callbacks before we return.</div><div class='del'>-	 */</div><div class='del'>-	down_read(&amp;devices_rwsem);</div><div class='del'>-	xa_for_each (&amp;devices, index, device)</div><div class='add'>+</div><div class='add'>+	/* We do not want to have locks while calling client-&gt;remove() */</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	xa_for_each (&amp;devices, index, device) {</div><div class='add'>+		if (!ib_device_try_get(device))</div><div class='add'>+			continue;</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+</div><div class='ctx'> 		remove_client_context(device, client-&gt;client_id);</div><div class='del'>-	up_read(&amp;devices_rwsem);</div><div class='ctx'> </div><div class='del'>-	down_write(&amp;clients_rwsem);</div><div class='del'>-	list_del(&amp;client-&gt;list);</div><div class='del'>-	xa_erase(&amp;clients, client-&gt;client_id);</div><div class='del'>-	up_write(&amp;clients_rwsem);</div><div class='add'>+		ib_device_put(device);</div><div class='add'>+		rcu_read_lock();</div><div class='add'>+	}</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * remove_client_context() is not a fence, it can return even though a</div><div class='add'>+	 * removal is ongoing. Wait until all removals are completed.</div><div class='add'>+	 */</div><div class='add'>+	wait_for_completion(&amp;client-&gt;uses_zero);</div><div class='add'>+	remove_client_id(client);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_unregister_client);</div><div class='ctx'> </div><div class='add'>+static int __ib_get_global_client_nl_info(const char *client_name,</div><div class='add'>+					  struct ib_client_nl_info *res)</div><div class='add'>+{</div><div class='add'>+	struct ib_client *client;</div><div class='add'>+	unsigned long index;</div><div class='add'>+	int ret = -ENOENT;</div><div class='add'>+</div><div class='add'>+	down_read(&amp;clients_rwsem);</div><div class='add'>+	xa_for_each_marked (&amp;clients, index, client, CLIENT_REGISTERED) {</div><div class='add'>+		if (strcmp(client-&gt;name, client_name) != 0)</div><div class='add'>+			continue;</div><div class='add'>+		if (!client-&gt;get_global_nl_info) {</div><div class='add'>+			ret = -EOPNOTSUPP;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		ret = client-&gt;get_global_nl_info(res);</div><div class='add'>+		if (WARN_ON(ret == -ENOENT))</div><div class='add'>+			ret = -EINVAL;</div><div class='add'>+		if (!ret &amp;&amp; res-&gt;cdev)</div><div class='add'>+			get_device(res-&gt;cdev);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+	up_read(&amp;clients_rwsem);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int __ib_get_client_nl_info(struct ib_device *ibdev,</div><div class='add'>+				   const char *client_name,</div><div class='add'>+				   struct ib_client_nl_info *res)</div><div class='add'>+{</div><div class='add'>+	unsigned long index;</div><div class='add'>+	void *client_data;</div><div class='add'>+	int ret = -ENOENT;</div><div class='add'>+</div><div class='add'>+	down_read(&amp;ibdev-&gt;client_data_rwsem);</div><div class='add'>+	xan_for_each_marked (&amp;ibdev-&gt;client_data, index, client_data,</div><div class='add'>+			     CLIENT_DATA_REGISTERED) {</div><div class='add'>+		struct ib_client *client = xa_load(&amp;clients, index);</div><div class='add'>+</div><div class='add'>+		if (!client || strcmp(client-&gt;name, client_name) != 0)</div><div class='add'>+			continue;</div><div class='add'>+		if (!client-&gt;get_nl_info) {</div><div class='add'>+			ret = -EOPNOTSUPP;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		ret = client-&gt;get_nl_info(ibdev, client_data, res);</div><div class='add'>+		if (WARN_ON(ret == -ENOENT))</div><div class='add'>+			ret = -EINVAL;</div><div class='add'>+</div><div class='add'>+		/*</div><div class='add'>+		 * The cdev is guaranteed valid as long as we are inside the</div><div class='add'>+		 * client_data_rwsem as remove_one can't be called. Keep it</div><div class='add'>+		 * valid for the caller.</div><div class='add'>+		 */</div><div class='add'>+		if (!ret &amp;&amp; res-&gt;cdev)</div><div class='add'>+			get_device(res-&gt;cdev);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+	up_read(&amp;ibdev-&gt;client_data_rwsem);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ib_get_client_nl_info - Fetch the nl_info from a client</div><div class='add'>+ * @ibdev: IB device</div><div class='add'>+ * @client_name: Name of the client</div><div class='add'>+ * @res: Result of the query</div><div class='add'>+ */</div><div class='add'>+int ib_get_client_nl_info(struct ib_device *ibdev, const char *client_name,</div><div class='add'>+			  struct ib_client_nl_info *res)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (ibdev)</div><div class='add'>+		ret = __ib_get_client_nl_info(ibdev, client_name, res);</div><div class='add'>+	else</div><div class='add'>+		ret = __ib_get_global_client_nl_info(client_name, res);</div><div class='add'>+#ifdef CONFIG_MODULES</div><div class='add'>+	if (ret == -ENOENT) {</div><div class='add'>+		request_module("rdma-client-%s", client_name);</div><div class='add'>+		if (ibdev)</div><div class='add'>+			ret = __ib_get_client_nl_info(ibdev, client_name, res);</div><div class='add'>+		else</div><div class='add'>+			ret = __ib_get_global_client_nl_info(client_name, res);</div><div class='add'>+	}</div><div class='add'>+#endif</div><div class='add'>+	if (ret) {</div><div class='add'>+		if (ret == -ENOENT)</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (WARN_ON(!res-&gt;cdev))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /**</div><div class='ctx'>  * ib_set_client_data - Set IB client context</div><div class='ctx'>  * @device:Device to set context for</div><div class='hunk'>@@ -1762,17 +2059,15 @@ EXPORT_SYMBOL(ib_set_client_data);</div><div class='ctx'>  *</div><div class='ctx'>  * ib_register_event_handler() registers an event handler that will be</div><div class='ctx'>  * called back when asynchronous IB events occur (as defined in</div><div class='del'>- * chapter 11 of the InfiniBand Architecture Specification).  This</div><div class='del'>- * callback may occur in interrupt context.</div><div class='add'>+ * chapter 11 of the InfiniBand Architecture Specification). This</div><div class='add'>+ * callback occurs in workqueue context.</div><div class='ctx'>  */</div><div class='ctx'> void ib_register_event_handler(struct ib_event_handler *event_handler)</div><div class='ctx'> {</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;event_handler-&gt;device-&gt;event_handler_lock, flags);</div><div class='add'>+	down_write(&amp;event_handler-&gt;device-&gt;event_handler_rwsem);</div><div class='ctx'> 	list_add_tail(&amp;event_handler-&gt;list,</div><div class='ctx'> 		      &amp;event_handler-&gt;device-&gt;event_handler_list);</div><div class='del'>-	spin_unlock_irqrestore(&amp;event_handler-&gt;device-&gt;event_handler_lock, flags);</div><div class='add'>+	up_write(&amp;event_handler-&gt;device-&gt;event_handler_rwsem);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_register_event_handler);</div><div class='ctx'> </div><div class='hunk'>@@ -1785,35 +2080,83 @@ EXPORT_SYMBOL(ib_register_event_handler);</div><div class='ctx'>  */</div><div class='ctx'> void ib_unregister_event_handler(struct ib_event_handler *event_handler)</div><div class='ctx'> {</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;event_handler-&gt;device-&gt;event_handler_lock, flags);</div><div class='add'>+	down_write(&amp;event_handler-&gt;device-&gt;event_handler_rwsem);</div><div class='ctx'> 	list_del(&amp;event_handler-&gt;list);</div><div class='del'>-	spin_unlock_irqrestore(&amp;event_handler-&gt;device-&gt;event_handler_lock, flags);</div><div class='add'>+	up_write(&amp;event_handler-&gt;device-&gt;event_handler_rwsem);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_unregister_event_handler);</div><div class='ctx'> </div><div class='del'>-/**</div><div class='del'>- * ib_dispatch_event - Dispatch an asynchronous event</div><div class='del'>- * @event:Event to dispatch</div><div class='del'>- *</div><div class='del'>- * Low-level drivers must call ib_dispatch_event() to dispatch the</div><div class='del'>- * event to all registered event handlers when an asynchronous event</div><div class='del'>- * occurs.</div><div class='del'>- */</div><div class='del'>-void ib_dispatch_event(struct ib_event *event)</div><div class='add'>+void ib_dispatch_event_clients(struct ib_event *event)</div><div class='ctx'> {</div><div class='del'>-	unsigned long flags;</div><div class='ctx'> 	struct ib_event_handler *handler;</div><div class='ctx'> </div><div class='del'>-	spin_lock_irqsave(&amp;event-&gt;device-&gt;event_handler_lock, flags);</div><div class='add'>+	down_read(&amp;event-&gt;device-&gt;event_handler_rwsem);</div><div class='ctx'> </div><div class='ctx'> 	list_for_each_entry(handler, &amp;event-&gt;device-&gt;event_handler_list, list)</div><div class='ctx'> 		handler-&gt;handler(handler, event);</div><div class='ctx'> </div><div class='del'>-	spin_unlock_irqrestore(&amp;event-&gt;device-&gt;event_handler_lock, flags);</div><div class='add'>+	up_read(&amp;event-&gt;device-&gt;event_handler_rwsem);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int iw_query_port(struct ib_device *device,</div><div class='add'>+			   u32 port_num,</div><div class='add'>+			   struct ib_port_attr *port_attr)</div><div class='add'>+{</div><div class='add'>+	struct in_device *inetdev;</div><div class='add'>+	struct net_device *netdev;</div><div class='add'>+</div><div class='add'>+	memset(port_attr, 0, sizeof(*port_attr));</div><div class='add'>+</div><div class='add'>+	netdev = ib_device_get_netdev(device, port_num);</div><div class='add'>+	if (!netdev)</div><div class='add'>+		return -ENODEV;</div><div class='add'>+</div><div class='add'>+	port_attr-&gt;max_mtu = IB_MTU_4096;</div><div class='add'>+	port_attr-&gt;active_mtu = ib_mtu_int_to_enum(netdev-&gt;mtu);</div><div class='add'>+</div><div class='add'>+	if (!netif_carrier_ok(netdev)) {</div><div class='add'>+		port_attr-&gt;state = IB_PORT_DOWN;</div><div class='add'>+		port_attr-&gt;phys_state = IB_PORT_PHYS_STATE_DISABLED;</div><div class='add'>+	} else {</div><div class='add'>+		rcu_read_lock();</div><div class='add'>+		inetdev = __in_dev_get_rcu(netdev);</div><div class='add'>+</div><div class='add'>+		if (inetdev &amp;&amp; inetdev-&gt;ifa_list) {</div><div class='add'>+			port_attr-&gt;state = IB_PORT_ACTIVE;</div><div class='add'>+			port_attr-&gt;phys_state = IB_PORT_PHYS_STATE_LINK_UP;</div><div class='add'>+		} else {</div><div class='add'>+			port_attr-&gt;state = IB_PORT_INIT;</div><div class='add'>+			port_attr-&gt;phys_state =</div><div class='add'>+				IB_PORT_PHYS_STATE_PORT_CONFIGURATION_TRAINING;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dev_put(netdev);</div><div class='add'>+	return device-&gt;ops.query_port(device, port_num, port_attr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int __ib_query_port(struct ib_device *device,</div><div class='add'>+			   u32 port_num,</div><div class='add'>+			   struct ib_port_attr *port_attr)</div><div class='add'>+{</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	memset(port_attr, 0, sizeof(*port_attr));</div><div class='add'>+</div><div class='add'>+	err = device-&gt;ops.query_port(device, port_num, port_attr);</div><div class='add'>+	if (err || port_attr-&gt;subnet_prefix)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	if (rdma_port_get_link_layer(device, port_num) !=</div><div class='add'>+	    IB_LINK_LAYER_INFINIBAND)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	ib_get_cached_subnet_prefix(device, port_num,</div><div class='add'>+				    &amp;port_attr-&gt;subnet_prefix);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(ib_dispatch_event);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * ib_query_port - Query IB port attributes</div><div class='hunk'>@@ -1825,29 +2168,16 @@ EXPORT_SYMBOL(ib_dispatch_event);</div><div class='ctx'>  * @port_attr pointer.</div><div class='ctx'>  */</div><div class='ctx'> int ib_query_port(struct ib_device *device,</div><div class='del'>-		  u8 port_num,</div><div class='add'>+		  u32 port_num,</div><div class='ctx'> 		  struct ib_port_attr *port_attr)</div><div class='ctx'> {</div><div class='del'>-	union ib_gid gid;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='ctx'> 	if (!rdma_is_port_valid(device, port_num))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	memset(port_attr, 0, sizeof(*port_attr));</div><div class='del'>-	err = device-&gt;ops.query_port(device, port_num, port_attr);</div><div class='del'>-	if (err || port_attr-&gt;subnet_prefix)</div><div class='del'>-		return err;</div><div class='del'>-</div><div class='del'>-	if (rdma_port_get_link_layer(device, port_num) != IB_LINK_LAYER_INFINIBAND)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	err = device-&gt;ops.query_gid(device, port_num, 0, &amp;gid);</div><div class='del'>-	if (err)</div><div class='del'>-		return err;</div><div class='del'>-</div><div class='del'>-	port_attr-&gt;subnet_prefix = be64_to_cpu(gid.global.subnet_prefix);</div><div class='del'>-	return 0;</div><div class='add'>+	if (rdma_protocol_iwarp(device, port_num))</div><div class='add'>+		return iw_query_port(device, port_num, port_attr);</div><div class='add'>+	else</div><div class='add'>+		return __ib_query_port(device, port_num, port_attr);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_query_port);</div><div class='ctx'> </div><div class='hunk'>@@ -1890,13 +2220,17 @@ static void add_ndev_hash(struct ib_port_data *pdata)</div><div class='ctx'>  * NETDEV_UNREGISTER event.</div><div class='ctx'>  */</div><div class='ctx'> int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev,</div><div class='del'>-			 unsigned int port)</div><div class='add'>+			 u32 port)</div><div class='ctx'> {</div><div class='add'>+	enum rdma_nl_notify_event_type etype;</div><div class='ctx'> 	struct net_device *old_ndev;</div><div class='ctx'> 	struct ib_port_data *pdata;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	if (!rdma_is_port_valid(ib_dev, port))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Drivers wish to call this before ib_register_driver, so we have to</div><div class='ctx'> 	 * setup the port data early.</div><div class='hunk'>@@ -1905,9 +2239,6 @@ int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev,</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	if (!rdma_is_port_valid(ib_dev, port))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='ctx'> 	pdata = &amp;ib_dev-&gt;port_data[port];</div><div class='ctx'> 	spin_lock_irqsave(&amp;pdata-&gt;netdev_lock, flags);</div><div class='ctx'> 	old_ndev = rcu_dereference_protected(</div><div class='hunk'>@@ -1917,14 +2248,19 @@ int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev,</div><div class='ctx'> 		return 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (ndev)</div><div class='del'>-		dev_hold(ndev);</div><div class='ctx'> 	rcu_assign_pointer(pdata-&gt;netdev, ndev);</div><div class='add'>+	netdev_put(old_ndev, &amp;pdata-&gt;netdev_tracker);</div><div class='add'>+	netdev_hold(ndev, &amp;pdata-&gt;netdev_tracker, GFP_ATOMIC);</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;pdata-&gt;netdev_lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	add_ndev_hash(pdata);</div><div class='del'>-	if (old_ndev)</div><div class='del'>-		dev_put(old_ndev);</div><div class='add'>+</div><div class='add'>+	/* Make sure that the device is registered before we send events */</div><div class='add'>+	if (xa_load(&amp;devices, ib_dev-&gt;index) != ib_dev)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	etype = ndev ? RDMA_NETDEV_ATTACH_EVENT : RDMA_NETDEV_DETACH_EVENT;</div><div class='add'>+	rdma_nl_notify_event(ib_dev, port, etype);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -1933,7 +2269,10 @@ EXPORT_SYMBOL(ib_device_set_netdev);</div><div class='ctx'> static void free_netdevs(struct ib_device *ib_dev)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	unsigned int port;</div><div class='add'>+	u32 port;</div><div class='add'>+</div><div class='add'>+	if (!ib_dev-&gt;port_data)</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='ctx'> 	rdma_for_each_port (ib_dev, port) {</div><div class='ctx'> 		struct ib_port_data *pdata = &amp;ib_dev-&gt;port_data[port];</div><div class='hunk'>@@ -1954,14 +2293,14 @@ static void free_netdevs(struct ib_device *ib_dev)</div><div class='ctx'> 			 * comparisons after the put</div><div class='ctx'> 			 */</div><div class='ctx'> 			rcu_assign_pointer(pdata-&gt;netdev, NULL);</div><div class='del'>-			dev_put(ndev);</div><div class='add'>+			netdev_put(ndev, &amp;pdata-&gt;netdev_tracker);</div><div class='ctx'> 		}</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;pdata-&gt;netdev_lock, flags);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct net_device *ib_device_get_netdev(struct ib_device *ib_dev,</div><div class='del'>-					unsigned int port)</div><div class='add'>+					u32 port)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_data *pdata;</div><div class='ctx'> 	struct net_device *res;</div><div class='hunk'>@@ -1969,6 +2308,9 @@ struct net_device *ib_device_get_netdev(struct ib_device *ib_dev,</div><div class='ctx'> 	if (!rdma_is_port_valid(ib_dev, port))</div><div class='ctx'> 		return NULL;</div><div class='ctx'> </div><div class='add'>+	if (!ib_dev-&gt;port_data)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='ctx'> 	pdata = &amp;ib_dev-&gt;port_data[port];</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='hunk'>@@ -1981,22 +2323,40 @@ struct net_device *ib_device_get_netdev(struct ib_device *ib_dev,</div><div class='ctx'> 		spin_lock(&amp;pdata-&gt;netdev_lock);</div><div class='ctx'> 		res = rcu_dereference_protected(</div><div class='ctx'> 			pdata-&gt;netdev, lockdep_is_held(&amp;pdata-&gt;netdev_lock));</div><div class='del'>-		if (res)</div><div class='del'>-			dev_hold(res);</div><div class='add'>+		dev_hold(res);</div><div class='ctx'> 		spin_unlock(&amp;pdata-&gt;netdev_lock);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * If we are starting to unregister expedite things by preventing</div><div class='del'>-	 * propagation of an unregistering netdev.</div><div class='del'>-	 */</div><div class='del'>-	if (res &amp;&amp; res-&gt;reg_state != NETREG_REGISTERED) {</div><div class='del'>-		dev_put(res);</div><div class='del'>-		return NULL;</div><div class='add'>+	return res;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_device_get_netdev);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ib_query_netdev_port - Query the port number of a net_device</div><div class='add'>+ * associated with an ibdev</div><div class='add'>+ * @ibdev: IB device</div><div class='add'>+ * @ndev: Network device</div><div class='add'>+ * @port: IB port the net_device is connected to</div><div class='add'>+ */</div><div class='add'>+int ib_query_netdev_port(struct ib_device *ibdev, struct net_device *ndev,</div><div class='add'>+			 u32 *port)</div><div class='add'>+{</div><div class='add'>+	struct net_device *ib_ndev;</div><div class='add'>+	u32 port_num;</div><div class='add'>+</div><div class='add'>+	rdma_for_each_port(ibdev, port_num) {</div><div class='add'>+		ib_ndev = ib_device_get_netdev(ibdev, port_num);</div><div class='add'>+		if (ndev == ib_ndev) {</div><div class='add'>+			*port = port_num;</div><div class='add'>+			dev_put(ib_ndev);</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+		dev_put(ib_ndev);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return res;</div><div class='add'>+	return -ENOENT;</div><div class='ctx'> }</div><div class='add'>+EXPORT_SYMBOL(ib_query_netdev_port);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * ib_device_get_by_netdev - Find an IB device associated with a netdev</div><div class='hunk'>@@ -2018,7 +2378,7 @@ struct ib_device *ib_device_get_by_netdev(struct net_device *ndev,</div><div class='ctx'> 				    (uintptr_t)ndev) {</div><div class='ctx'> 		if (rcu_access_pointer(cur-&gt;netdev) == ndev &amp;&amp;</div><div class='ctx'> 		    (driver_id == RDMA_DRIVER_UNKNOWN ||</div><div class='del'>-		     cur-&gt;ib_dev-&gt;driver_id == driver_id) &amp;&amp;</div><div class='add'>+		     cur-&gt;ib_dev-&gt;ops.driver_id == driver_id) &amp;&amp;</div><div class='ctx'> 		    ib_device_try_get(cur-&gt;ib_dev)) {</div><div class='ctx'> 			res = cur-&gt;ib_dev;</div><div class='ctx'> 			break;</div><div class='hunk'>@@ -2048,7 +2408,7 @@ void ib_enum_roce_netdev(struct ib_device *ib_dev,</div><div class='ctx'> 			 roce_netdev_callback cb,</div><div class='ctx'> 			 void *cookie)</div><div class='ctx'> {</div><div class='del'>-	unsigned int port;</div><div class='add'>+	u32 port;</div><div class='ctx'> </div><div class='ctx'> 	rdma_for_each_port (ib_dev, port)</div><div class='ctx'> 		if (rdma_protocol_roce(ib_dev, port)) {</div><div class='hunk'>@@ -2057,9 +2417,7 @@ void ib_enum_roce_netdev(struct ib_device *ib_dev,</div><div class='ctx'> </div><div class='ctx'> 			if (filter(ib_dev, port, idev, filter_cookie))</div><div class='ctx'> 				cb(ib_dev, port, idev, cookie);</div><div class='del'>-</div><div class='del'>-			if (idev)</div><div class='del'>-				dev_put(idev);</div><div class='add'>+			dev_put(idev);</div><div class='ctx'> 		}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -2088,7 +2446,7 @@ void ib_enum_all_roce_netdevs(roce_netdev_filter filter,</div><div class='ctx'> 	up_read(&amp;devices_rwsem);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/**</div><div class='add'>+/*</div><div class='ctx'>  * ib_enum_all_devs - enumerate all ib_devices</div><div class='ctx'>  * @cb: Callback to call for each found ib_device</div><div class='ctx'>  *</div><div class='hunk'>@@ -2126,11 +2484,14 @@ int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb,</div><div class='ctx'>  * ib_query_pkey() fetches the specified P_Key table entry.</div><div class='ctx'>  */</div><div class='ctx'> int ib_query_pkey(struct ib_device *device,</div><div class='del'>-		  u8 port_num, u16 index, u16 *pkey)</div><div class='add'>+		  u32 port_num, u16 index, u16 *pkey)</div><div class='ctx'> {</div><div class='ctx'> 	if (!rdma_is_port_valid(device, port_num))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='add'>+	if (!device-&gt;ops.query_pkey)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='ctx'> 	return device-&gt;ops.query_pkey(device, port_num, index, pkey);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_query_pkey);</div><div class='hunk'>@@ -2149,7 +2510,7 @@ int ib_modify_device(struct ib_device *device,</div><div class='ctx'> 		     struct ib_device_modify *device_modify)</div><div class='ctx'> {</div><div class='ctx'> 	if (!device-&gt;ops.modify_device)</div><div class='del'>-		return -ENOSYS;</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	return device-&gt;ops.modify_device(device, device_modify_mask,</div><div class='ctx'> 					 device_modify);</div><div class='hunk'>@@ -2168,7 +2529,7 @@ EXPORT_SYMBOL(ib_modify_device);</div><div class='ctx'>  * @port_modify_mask and @port_modify structure.</div><div class='ctx'>  */</div><div class='ctx'> int ib_modify_port(struct ib_device *device,</div><div class='del'>-		   u8 port_num, int port_modify_mask,</div><div class='add'>+		   u32 port_num, int port_modify_mask,</div><div class='ctx'> 		   struct ib_port_modify *port_modify)</div><div class='ctx'> {</div><div class='ctx'> 	int rc;</div><div class='hunk'>@@ -2180,8 +2541,12 @@ int ib_modify_port(struct ib_device *device,</div><div class='ctx'> 		rc = device-&gt;ops.modify_port(device, port_num,</div><div class='ctx'> 					     port_modify_mask,</div><div class='ctx'> 					     port_modify);</div><div class='add'>+	else if (rdma_protocol_roce(device, port_num) &amp;&amp;</div><div class='add'>+		 ((port_modify-&gt;set_port_cap_mask &amp; ~IB_PORT_CM_SUP) == 0 ||</div><div class='add'>+		  (port_modify-&gt;clr_port_cap_mask &amp; ~IB_PORT_CM_SUP) == 0))</div><div class='add'>+		rc = 0;</div><div class='ctx'> 	else</div><div class='del'>-		rc = rdma_protocol_roce(device, port_num) ? 0 : -ENOSYS;</div><div class='add'>+		rc = -EOPNOTSUPP;</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_modify_port);</div><div class='hunk'>@@ -2196,10 +2561,10 @@ EXPORT_SYMBOL(ib_modify_port);</div><div class='ctx'>  *   parameter may be NULL.</div><div class='ctx'>  */</div><div class='ctx'> int ib_find_gid(struct ib_device *device, union ib_gid *gid,</div><div class='del'>-		u8 *port_num, u16 *index)</div><div class='add'>+		u32 *port_num, u16 *index)</div><div class='ctx'> {</div><div class='ctx'> 	union ib_gid tmp_gid;</div><div class='del'>-	unsigned int port;</div><div class='add'>+	u32 port;</div><div class='ctx'> 	int ret, i;</div><div class='ctx'> </div><div class='ctx'> 	rdma_for_each_port (device, port) {</div><div class='hunk'>@@ -2210,7 +2575,8 @@ int ib_find_gid(struct ib_device *device, union ib_gid *gid,</div><div class='ctx'> 		     ++i) {</div><div class='ctx'> 			ret = rdma_query_gid(device, port, i, &amp;tmp_gid);</div><div class='ctx'> 			if (ret)</div><div class='del'>-				return ret;</div><div class='add'>+				continue;</div><div class='add'>+</div><div class='ctx'> 			if (!memcmp(&amp;tmp_gid, gid, sizeof *gid)) {</div><div class='ctx'> 				*port_num = port;</div><div class='ctx'> 				if (index)</div><div class='hunk'>@@ -2233,7 +2599,7 @@ EXPORT_SYMBOL(ib_find_gid);</div><div class='ctx'>  * @index: The index into the PKey table where the PKey was found.</div><div class='ctx'>  */</div><div class='ctx'> int ib_find_pkey(struct ib_device *device,</div><div class='del'>-		 u8 port_num, u16 pkey, u16 *index)</div><div class='add'>+		 u32 port_num, u16 pkey, u16 *index)</div><div class='ctx'> {</div><div class='ctx'> 	int ret, i;</div><div class='ctx'> 	u16 tmp_pkey;</div><div class='hunk'>@@ -2276,7 +2642,7 @@ EXPORT_SYMBOL(ib_find_pkey);</div><div class='ctx'>  *</div><div class='ctx'>  */</div><div class='ctx'> struct net_device *ib_get_net_dev_by_params(struct ib_device *dev,</div><div class='del'>-					    u8 port,</div><div class='add'>+					    u32 port,</div><div class='ctx'> 					    u16 pkey,</div><div class='ctx'> 					    const union ib_gid *gid,</div><div class='ctx'> 					    const struct sockaddr *addr)</div><div class='hunk'>@@ -2323,12 +2689,30 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)</div><div class='ctx'> </div><div class='ctx'> #define SET_OBJ_SIZE(ptr, name) SET_DEVICE_OP(ptr, size_##name)</div><div class='ctx'> </div><div class='add'>+	if (ops-&gt;driver_id != RDMA_DRIVER_UNKNOWN) {</div><div class='add'>+		WARN_ON(dev_ops-&gt;driver_id != RDMA_DRIVER_UNKNOWN &amp;&amp;</div><div class='add'>+			dev_ops-&gt;driver_id != ops-&gt;driver_id);</div><div class='add'>+		dev_ops-&gt;driver_id = ops-&gt;driver_id;</div><div class='add'>+	}</div><div class='add'>+	if (ops-&gt;owner) {</div><div class='add'>+		WARN_ON(dev_ops-&gt;owner &amp;&amp; dev_ops-&gt;owner != ops-&gt;owner);</div><div class='add'>+		dev_ops-&gt;owner = ops-&gt;owner;</div><div class='add'>+	}</div><div class='add'>+	if (ops-&gt;uverbs_abi_ver)</div><div class='add'>+		dev_ops-&gt;uverbs_abi_ver = ops-&gt;uverbs_abi_ver;</div><div class='add'>+</div><div class='add'>+	dev_ops-&gt;uverbs_no_driver_id_binding |=</div><div class='add'>+		ops-&gt;uverbs_no_driver_id_binding;</div><div class='add'>+</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, add_gid);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, add_sub_dev);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, advise_mr);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, alloc_dm);</div><div class='del'>-	SET_DEVICE_OP(dev_ops, alloc_fmr);</div><div class='del'>-	SET_DEVICE_OP(dev_ops, alloc_hw_stats);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, alloc_dmah);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, alloc_hw_device_stats);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, alloc_hw_port_stats);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, alloc_mr);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, alloc_mr_integrity);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, alloc_mw);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, alloc_pd);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, alloc_rdma_netdev);</div><div class='hunk'>@@ -2336,23 +2720,31 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, alloc_xrcd);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, attach_mcast);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, check_mr_status);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, counter_alloc_stats);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, counter_bind_qp);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, counter_dealloc);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, counter_init);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, counter_unbind_qp);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, counter_update_stats);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, create_ah);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, create_counters);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, create_cq);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, create_cq_umem);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, create_flow);</div><div class='del'>-	SET_DEVICE_OP(dev_ops, create_flow_action_esp);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, create_qp);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, create_rwq_ind_table);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, create_srq);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, create_user_ah);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, create_wq);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, dealloc_dm);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, dealloc_dmah);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, dealloc_driver);</div><div class='del'>-	SET_DEVICE_OP(dev_ops, dealloc_fmr);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, dealloc_mw);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, dealloc_pd);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, dealloc_ucontext);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, dealloc_xrcd);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, del_gid);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, del_sub_dev);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, dereg_mr);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, destroy_ah);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, destroy_counters);</div><div class='hunk'>@@ -2363,22 +2755,33 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, destroy_rwq_ind_table);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, destroy_srq);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, destroy_wq);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, device_group);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, detach_mcast);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, disassociate_ucontext);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, drain_rq);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, drain_sq);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, enable_driver);</div><div class='del'>-	SET_DEVICE_OP(dev_ops, fill_res_entry);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, fill_res_cm_id_entry);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, fill_res_cq_entry);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, fill_res_cq_entry_raw);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, fill_res_mr_entry);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, fill_res_mr_entry_raw);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, fill_res_qp_entry);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, fill_res_qp_entry_raw);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, fill_res_srq_entry);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, fill_res_srq_entry_raw);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, fill_stat_mr_entry);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, get_dev_fw_str);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, get_dma_mr);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, get_hw_stats);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, get_link_layer);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, get_netdev);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, get_numa_node);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, get_port_immutable);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, get_vector_affinity);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, get_vf_config);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, get_vf_guid);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, get_vf_stats);</div><div class='del'>-	SET_DEVICE_OP(dev_ops, init_port);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, iw_accept);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, iw_add_ref);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, iw_connect);</div><div class='hunk'>@@ -2388,18 +2791,22 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, iw_reject);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, iw_rem_ref);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, map_mr_sg);</div><div class='del'>-	SET_DEVICE_OP(dev_ops, map_phys_fmr);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, map_mr_sg_pi);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, mmap);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, mmap_free);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, modify_ah);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, modify_cq);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, modify_device);</div><div class='del'>-	SET_DEVICE_OP(dev_ops, modify_flow_action_esp);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, modify_hw_stat);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, modify_port);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, modify_qp);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, modify_srq);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, modify_wq);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, peek_cq);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, pre_destroy_cq);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, poll_cq);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, port_groups);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, post_destroy_cq);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, post_recv);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, post_send);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, post_srq_recv);</div><div class='hunk'>@@ -2411,25 +2818,99 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, query_port);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, query_qp);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, query_srq);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, query_ucontext);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, rdma_netdev_get_params);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, read_counters);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, reg_dm_mr);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, reg_user_mr);</div><div class='del'>-	SET_DEVICE_OP(dev_ops, req_ncomp_notif);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, reg_user_mr_dmabuf);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, req_notify_cq);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, rereg_user_mr);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, resize_cq);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, set_vf_guid);</div><div class='ctx'> 	SET_DEVICE_OP(dev_ops, set_vf_link_state);</div><div class='del'>-	SET_DEVICE_OP(dev_ops, unmap_fmr);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, ufile_hw_cleanup);</div><div class='add'>+	SET_DEVICE_OP(dev_ops, report_port_event);</div><div class='ctx'> </div><div class='ctx'> 	SET_OBJ_SIZE(dev_ops, ib_ah);</div><div class='add'>+	SET_OBJ_SIZE(dev_ops, ib_counters);</div><div class='add'>+	SET_OBJ_SIZE(dev_ops, ib_cq);</div><div class='add'>+	SET_OBJ_SIZE(dev_ops, ib_dmah);</div><div class='add'>+	SET_OBJ_SIZE(dev_ops, ib_mw);</div><div class='ctx'> 	SET_OBJ_SIZE(dev_ops, ib_pd);</div><div class='add'>+	SET_OBJ_SIZE(dev_ops, ib_qp);</div><div class='add'>+	SET_OBJ_SIZE(dev_ops, ib_rwq_ind_table);</div><div class='ctx'> 	SET_OBJ_SIZE(dev_ops, ib_srq);</div><div class='ctx'> 	SET_OBJ_SIZE(dev_ops, ib_ucontext);</div><div class='add'>+	SET_OBJ_SIZE(dev_ops, ib_xrcd);</div><div class='add'>+	SET_OBJ_SIZE(dev_ops, rdma_counter);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_set_device_ops);</div><div class='ctx'> </div><div class='add'>+int ib_add_sub_device(struct ib_device *parent,</div><div class='add'>+		      enum rdma_nl_dev_type type,</div><div class='add'>+		      const char *name)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *sub;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	if (!parent-&gt;ops.add_sub_dev || !parent-&gt;ops.del_sub_dev)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (!ib_device_try_get(parent))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	sub = parent-&gt;ops.add_sub_dev(parent, type, name);</div><div class='add'>+	if (IS_ERR(sub)) {</div><div class='add'>+		ib_device_put(parent);</div><div class='add'>+		return PTR_ERR(sub);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	sub-&gt;type = type;</div><div class='add'>+	sub-&gt;parent = parent;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;parent-&gt;subdev_lock);</div><div class='add'>+	list_add_tail(&amp;parent-&gt;subdev_list_head, &amp;sub-&gt;subdev_list);</div><div class='add'>+	mutex_unlock(&amp;parent-&gt;subdev_lock);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_add_sub_device);</div><div class='add'>+</div><div class='add'>+int ib_del_sub_device_and_put(struct ib_device *sub)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *parent = sub-&gt;parent;</div><div class='add'>+</div><div class='add'>+	if (!parent)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;parent-&gt;subdev_lock);</div><div class='add'>+	list_del(&amp;sub-&gt;subdev_list);</div><div class='add'>+	mutex_unlock(&amp;parent-&gt;subdev_lock);</div><div class='add'>+</div><div class='add'>+	ib_device_put(sub);</div><div class='add'>+	parent-&gt;ops.del_sub_dev(sub);</div><div class='add'>+	ib_device_put(parent);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_del_sub_device_and_put);</div><div class='add'>+</div><div class='add'>+#ifdef CONFIG_INFINIBAND_VIRT_DMA</div><div class='add'>+int ib_dma_virt_map_sg(struct ib_device *dev, struct scatterlist *sg, int nents)</div><div class='add'>+{</div><div class='add'>+	struct scatterlist *s;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for_each_sg(sg, s, nents, i) {</div><div class='add'>+		sg_dma_address(s) = (uintptr_t)sg_virt(s);</div><div class='add'>+		sg_dma_len(s) = s-&gt;length;</div><div class='add'>+	}</div><div class='add'>+	return nents;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_dma_virt_map_sg);</div><div class='add'>+#endif /* CONFIG_INFINIBAND_VIRT_DMA */</div><div class='add'>+</div><div class='ctx'> static const struct rdma_nl_cbs ibnl_ls_cb_table[RDMA_NL_LS_NUM_OPS] = {</div><div class='ctx'> 	[RDMA_NL_LS_OP_RESOLVE] = {</div><div class='ctx'> 		.doit = ib_nl_handle_resolve_resp,</div><div class='hunk'>@@ -2445,29 +2926,121 @@ static const struct rdma_nl_cbs ibnl_ls_cb_table[RDMA_NL_LS_NUM_OPS] = {</div><div class='ctx'> 	},</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+void ib_dispatch_port_state_event(struct ib_device *ibdev, struct net_device *ndev)</div><div class='add'>+{</div><div class='add'>+	enum ib_port_state curr_state;</div><div class='add'>+	struct ib_event ibevent = {};</div><div class='add'>+	u32 port;</div><div class='add'>+</div><div class='add'>+	if (ib_query_netdev_port(ibdev, ndev, &amp;port))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	curr_state = ib_get_curr_port_state(ndev);</div><div class='add'>+</div><div class='add'>+	write_lock_irq(&amp;ibdev-&gt;cache_lock);</div><div class='add'>+	if (ibdev-&gt;port_data[port].cache.last_port_state == curr_state) {</div><div class='add'>+		write_unlock_irq(&amp;ibdev-&gt;cache_lock);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+	ibdev-&gt;port_data[port].cache.last_port_state = curr_state;</div><div class='add'>+	write_unlock_irq(&amp;ibdev-&gt;cache_lock);</div><div class='add'>+</div><div class='add'>+	ibevent.event = (curr_state == IB_PORT_DOWN) ?</div><div class='add'>+					IB_EVENT_PORT_ERR : IB_EVENT_PORT_ACTIVE;</div><div class='add'>+	ibevent.device = ibdev;</div><div class='add'>+	ibevent.element.port_num = port;</div><div class='add'>+	ib_dispatch_event(&amp;ibevent);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_dispatch_port_state_event);</div><div class='add'>+</div><div class='add'>+static void handle_port_event(struct net_device *ndev, unsigned long event)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev;</div><div class='add'>+</div><div class='add'>+	/* Currently, link events in bonding scenarios are still</div><div class='add'>+	 * reported by drivers that support bonding.</div><div class='add'>+	 */</div><div class='add'>+	if (netif_is_lag_master(ndev) || netif_is_lag_port(ndev))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	ibdev = ib_device_get_by_netdev(ndev, RDMA_DRIVER_UNKNOWN);</div><div class='add'>+	if (!ibdev)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	if (ibdev-&gt;ops.report_port_event) {</div><div class='add'>+		ibdev-&gt;ops.report_port_event(ibdev, ndev, event);</div><div class='add'>+		goto put_ibdev;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ib_dispatch_port_state_event(ibdev, ndev);</div><div class='add'>+</div><div class='add'>+put_ibdev:</div><div class='add'>+	ib_device_put(ibdev);</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int ib_netdevice_event(struct notifier_block *this,</div><div class='add'>+			      unsigned long event, void *ptr)</div><div class='add'>+{</div><div class='add'>+	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);</div><div class='add'>+	struct ib_device *ibdev;</div><div class='add'>+	u32 port;</div><div class='add'>+</div><div class='add'>+	switch (event) {</div><div class='add'>+	case NETDEV_CHANGENAME:</div><div class='add'>+		ibdev = ib_device_get_by_netdev(ndev, RDMA_DRIVER_UNKNOWN);</div><div class='add'>+		if (!ibdev)</div><div class='add'>+			return NOTIFY_DONE;</div><div class='add'>+</div><div class='add'>+		if (ib_query_netdev_port(ibdev, ndev, &amp;port)) {</div><div class='add'>+			ib_device_put(ibdev);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		rdma_nl_notify_event(ibdev, port, RDMA_NETDEV_RENAME_EVENT);</div><div class='add'>+		ib_device_put(ibdev);</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	case NETDEV_UP:</div><div class='add'>+	case NETDEV_CHANGE:</div><div class='add'>+	case NETDEV_DOWN:</div><div class='add'>+		handle_port_event(ndev, event);</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return NOTIFY_DONE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct notifier_block nb_netdevice = {</div><div class='add'>+	.notifier_call = ib_netdevice_event,</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> static int __init ib_core_init(void)</div><div class='ctx'> {</div><div class='del'>-	int ret;</div><div class='add'>+	int ret = -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	ib_wq = alloc_workqueue("infiniband", 0, 0);</div><div class='add'>+	ib_wq = alloc_workqueue("infiniband", WQ_PERCPU, 0);</div><div class='ctx'> 	if (!ib_wq)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	ib_comp_wq = alloc_workqueue("ib-comp-wq",</div><div class='del'>-			WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_SYSFS, 0);</div><div class='del'>-	if (!ib_comp_wq) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='add'>+	ib_unreg_wq = alloc_workqueue("ib-unreg-wq", WQ_UNBOUND,</div><div class='add'>+				      WQ_UNBOUND_MAX_ACTIVE);</div><div class='add'>+	if (!ib_unreg_wq)</div><div class='ctx'> 		goto err;</div><div class='del'>-	}</div><div class='add'>+</div><div class='add'>+	ib_comp_wq = alloc_workqueue("ib-comp-wq",</div><div class='add'>+			WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_SYSFS | WQ_PERCPU, 0);</div><div class='add'>+	if (!ib_comp_wq)</div><div class='add'>+		goto err_unbound;</div><div class='ctx'> </div><div class='ctx'> 	ib_comp_unbound_wq =</div><div class='ctx'> 		alloc_workqueue("ib-comp-unb-wq",</div><div class='ctx'> 				WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM |</div><div class='ctx'> 				WQ_SYSFS, WQ_UNBOUND_MAX_ACTIVE);</div><div class='del'>-	if (!ib_comp_unbound_wq) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='add'>+	if (!ib_comp_unbound_wq)</div><div class='ctx'> 		goto err_comp;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='ctx'> 	ret = class_register(&amp;ib_class);</div><div class='ctx'> 	if (ret) {</div><div class='hunk'>@@ -2475,15 +3048,11 @@ static int __init ib_core_init(void)</div><div class='ctx'> 		goto err_comp_unbound;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = rdma_nl_init();</div><div class='del'>-	if (ret) {</div><div class='del'>-		pr_warn("Couldn't init IB netlink interface: err %d\n", ret);</div><div class='del'>-		goto err_sysfs;</div><div class='del'>-	}</div><div class='add'>+	rdma_nl_init();</div><div class='ctx'> </div><div class='ctx'> 	ret = addr_init();</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		pr_warn("Could't init IB address resolution\n");</div><div class='add'>+		pr_warn("Couldn't init IB address resolution\n");</div><div class='ctx'> 		goto err_ibnl;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -2499,7 +3068,7 @@ static int __init ib_core_init(void)</div><div class='ctx'> 		goto err_mad;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = register_lsm_notifier(&amp;ibdev_lsm_nb);</div><div class='add'>+	ret = register_blocking_lsm_notifier(&amp;ibdev_lsm_nb);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		pr_warn("Couldn't register LSM notifier. ret %d\n", ret);</div><div class='ctx'> 		goto err_sa;</div><div class='hunk'>@@ -2513,12 +3082,22 @@ static int __init ib_core_init(void)</div><div class='ctx'> </div><div class='ctx'> 	nldev_init();</div><div class='ctx'> 	rdma_nl_register(RDMA_NL_LS, ibnl_ls_cb_table);</div><div class='del'>-	roce_gid_mgmt_init();</div><div class='add'>+	ret = roce_gid_mgmt_init();</div><div class='add'>+	if (ret) {</div><div class='add'>+		pr_warn("Couldn't init RoCE GID management\n");</div><div class='add'>+		goto err_parent;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	register_netdevice_notifier(&amp;nb_netdevice);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='add'>+err_parent:</div><div class='add'>+	rdma_nl_unregister(RDMA_NL_LS);</div><div class='add'>+	nldev_exit();</div><div class='add'>+	unregister_pernet_device(&amp;rdma_dev_net_ops);</div><div class='ctx'> err_compat:</div><div class='del'>-	unregister_lsm_notifier(&amp;ibdev_lsm_nb);</div><div class='add'>+	unregister_blocking_lsm_notifier(&amp;ibdev_lsm_nb);</div><div class='ctx'> err_sa:</div><div class='ctx'> 	ib_sa_cleanup();</div><div class='ctx'> err_mad:</div><div class='hunk'>@@ -2526,13 +3105,13 @@ err_mad:</div><div class='ctx'> err_addr:</div><div class='ctx'> 	addr_cleanup();</div><div class='ctx'> err_ibnl:</div><div class='del'>-	rdma_nl_exit();</div><div class='del'>-err_sysfs:</div><div class='ctx'> 	class_unregister(&amp;ib_class);</div><div class='ctx'> err_comp_unbound:</div><div class='ctx'> 	destroy_workqueue(ib_comp_unbound_wq);</div><div class='ctx'> err_comp:</div><div class='ctx'> 	destroy_workqueue(ib_comp_wq);</div><div class='add'>+err_unbound:</div><div class='add'>+	destroy_workqueue(ib_unreg_wq);</div><div class='ctx'> err:</div><div class='ctx'> 	destroy_workqueue(ib_wq);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -2540,11 +3119,12 @@ err:</div><div class='ctx'> </div><div class='ctx'> static void __exit ib_core_cleanup(void)</div><div class='ctx'> {</div><div class='add'>+	unregister_netdevice_notifier(&amp;nb_netdevice);</div><div class='ctx'> 	roce_gid_mgmt_cleanup();</div><div class='del'>-	nldev_exit();</div><div class='ctx'> 	rdma_nl_unregister(RDMA_NL_LS);</div><div class='add'>+	nldev_exit();</div><div class='ctx'> 	unregister_pernet_device(&amp;rdma_dev_net_ops);</div><div class='del'>-	unregister_lsm_notifier(&amp;ibdev_lsm_nb);</div><div class='add'>+	unregister_blocking_lsm_notifier(&amp;ibdev_lsm_nb);</div><div class='ctx'> 	ib_sa_cleanup();</div><div class='ctx'> 	ib_mad_cleanup();</div><div class='ctx'> 	addr_cleanup();</div><div class='hunk'>@@ -2554,7 +3134,7 @@ static void __exit ib_core_cleanup(void)</div><div class='ctx'> 	destroy_workqueue(ib_comp_wq);</div><div class='ctx'> 	/* Make sure that any pending umem accounting work is done. */</div><div class='ctx'> 	destroy_workqueue(ib_wq);</div><div class='del'>-	flush_workqueue(system_unbound_wq);</div><div class='add'>+	destroy_workqueue(ib_unreg_wq);</div><div class='ctx'> 	WARN_ON(!xa_empty(&amp;clients));</div><div class='ctx'> 	WARN_ON(!xa_empty(&amp;devices));</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/core/fmr_pool.c b/drivers/infiniband/core/fmr_pool.c<br/>deleted file mode 100644<br/>index 7d841b689a1e..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/fmr_pool.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/fmr_pool.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,507 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2004 Topspin Communications.  All rights reserved.</div><div class='del'>- * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-</div><div class='del'>-#include &lt;linux/errno.h&gt;</div><div class='del'>-#include &lt;linux/spinlock.h&gt;</div><div class='del'>-#include &lt;linux/export.h&gt;</div><div class='del'>-#include &lt;linux/slab.h&gt;</div><div class='del'>-#include &lt;linux/jhash.h&gt;</div><div class='del'>-#include &lt;linux/kthread.h&gt;</div><div class='del'>-</div><div class='del'>-#include &lt;rdma/ib_fmr_pool.h&gt;</div><div class='del'>-</div><div class='del'>-#include "core_priv.h"</div><div class='del'>-</div><div class='del'>-#define PFX "fmr_pool: "</div><div class='del'>-</div><div class='del'>-enum {</div><div class='del'>-	IB_FMR_MAX_REMAPS = 32,</div><div class='del'>-</div><div class='del'>-	IB_FMR_HASH_BITS  = 8,</div><div class='del'>-	IB_FMR_HASH_SIZE  = 1 &lt;&lt; IB_FMR_HASH_BITS,</div><div class='del'>-	IB_FMR_HASH_MASK  = IB_FMR_HASH_SIZE - 1</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * If an FMR is not in use, then the list member will point to either</div><div class='del'>- * its pool's free_list (if the FMR can be mapped again; that is,</div><div class='del'>- * remap_count &lt; pool-&gt;max_remaps) or its pool's dirty_list (if the</div><div class='del'>- * FMR needs to be unmapped before being remapped).  In either of</div><div class='del'>- * these cases it is a bug if the ref_count is not 0.  In other words,</div><div class='del'>- * if ref_count is &gt; 0, then the list member must not be linked into</div><div class='del'>- * either free_list or dirty_list.</div><div class='del'>- *</div><div class='del'>- * The cache_node member is used to link the FMR into a cache bucket</div><div class='del'>- * (if caching is enabled).  This is independent of the reference</div><div class='del'>- * count of the FMR.  When a valid FMR is released, its ref_count is</div><div class='del'>- * decremented, and if ref_count reaches 0, the FMR is placed in</div><div class='del'>- * either free_list or dirty_list as appropriate.  However, it is not</div><div class='del'>- * removed from the cache and may be "revived" if a call to</div><div class='del'>- * ib_fmr_register_physical() occurs before the FMR is remapped.  In</div><div class='del'>- * this case we just increment the ref_count and remove the FMR from</div><div class='del'>- * free_list/dirty_list.</div><div class='del'>- *</div><div class='del'>- * Before we remap an FMR from free_list, we remove it from the cache</div><div class='del'>- * (to prevent another user from obtaining a stale FMR).  When an FMR</div><div class='del'>- * is released, we add it to the tail of the free list, so that our</div><div class='del'>- * cache eviction policy is "least recently used."</div><div class='del'>- *</div><div class='del'>- * All manipulation of ref_count, list and cache_node is protected by</div><div class='del'>- * pool_lock to maintain consistency.</div><div class='del'>- */</div><div class='del'>-</div><div class='del'>-struct ib_fmr_pool {</div><div class='del'>-	spinlock_t                pool_lock;</div><div class='del'>-</div><div class='del'>-	int                       pool_size;</div><div class='del'>-	int                       max_pages;</div><div class='del'>-	int			  max_remaps;</div><div class='del'>-	int                       dirty_watermark;</div><div class='del'>-	int                       dirty_len;</div><div class='del'>-	struct list_head          free_list;</div><div class='del'>-	struct list_head          dirty_list;</div><div class='del'>-	struct hlist_head        *cache_bucket;</div><div class='del'>-</div><div class='del'>-	void                     (*flush_function)(struct ib_fmr_pool *pool,</div><div class='del'>-						   void *              arg);</div><div class='del'>-	void                     *flush_arg;</div><div class='del'>-</div><div class='del'>-	struct kthread_worker	  *worker;</div><div class='del'>-	struct kthread_work	  work;</div><div class='del'>-</div><div class='del'>-	atomic_t                  req_ser;</div><div class='del'>-	atomic_t                  flush_ser;</div><div class='del'>-</div><div class='del'>-	wait_queue_head_t         force_wait;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static inline u32 ib_fmr_hash(u64 first_page)</div><div class='del'>-{</div><div class='del'>-	return jhash_2words((u32) first_page, (u32) (first_page &gt;&gt; 32), 0) &amp;</div><div class='del'>-		(IB_FMR_HASH_SIZE - 1);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* Caller must hold pool_lock */</div><div class='del'>-static inline struct ib_pool_fmr *ib_fmr_cache_lookup(struct ib_fmr_pool *pool,</div><div class='del'>-						      u64 *page_list,</div><div class='del'>-						      int  page_list_len,</div><div class='del'>-						      u64  io_virtual_address)</div><div class='del'>-{</div><div class='del'>-	struct hlist_head *bucket;</div><div class='del'>-	struct ib_pool_fmr *fmr;</div><div class='del'>-</div><div class='del'>-	if (!pool-&gt;cache_bucket)</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	bucket = pool-&gt;cache_bucket + ib_fmr_hash(*page_list);</div><div class='del'>-</div><div class='del'>-	hlist_for_each_entry(fmr, bucket, cache_node)</div><div class='del'>-		if (io_virtual_address == fmr-&gt;io_virtual_address &amp;&amp;</div><div class='del'>-		    page_list_len      == fmr-&gt;page_list_len      &amp;&amp;</div><div class='del'>-		    !memcmp(page_list, fmr-&gt;page_list,</div><div class='del'>-			    page_list_len * sizeof *page_list))</div><div class='del'>-			return fmr;</div><div class='del'>-</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void ib_fmr_batch_release(struct ib_fmr_pool *pool)</div><div class='del'>-{</div><div class='del'>-	int                 ret;</div><div class='del'>-	struct ib_pool_fmr *fmr;</div><div class='del'>-	LIST_HEAD(unmap_list);</div><div class='del'>-	LIST_HEAD(fmr_list);</div><div class='del'>-</div><div class='del'>-	spin_lock_irq(&amp;pool-&gt;pool_lock);</div><div class='del'>-</div><div class='del'>-	list_for_each_entry(fmr, &amp;pool-&gt;dirty_list, list) {</div><div class='del'>-		hlist_del_init(&amp;fmr-&gt;cache_node);</div><div class='del'>-		fmr-&gt;remap_count = 0;</div><div class='del'>-		list_add_tail(&amp;fmr-&gt;fmr-&gt;list, &amp;fmr_list);</div><div class='del'>-</div><div class='del'>-#ifdef DEBUG</div><div class='del'>-		if (fmr-&gt;ref_count !=0) {</div><div class='del'>-			pr_warn(PFX "Unmapping FMR 0x%08x with ref count %d\n",</div><div class='del'>-				fmr, fmr-&gt;ref_count);</div><div class='del'>-		}</div><div class='del'>-#endif</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	list_splice_init(&amp;pool-&gt;dirty_list, &amp;unmap_list);</div><div class='del'>-	pool-&gt;dirty_len = 0;</div><div class='del'>-</div><div class='del'>-	spin_unlock_irq(&amp;pool-&gt;pool_lock);</div><div class='del'>-</div><div class='del'>-	if (list_empty(&amp;unmap_list)) {</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = ib_unmap_fmr(&amp;fmr_list);</div><div class='del'>-	if (ret)</div><div class='del'>-		pr_warn(PFX "ib_unmap_fmr returned %d\n", ret);</div><div class='del'>-</div><div class='del'>-	spin_lock_irq(&amp;pool-&gt;pool_lock);</div><div class='del'>-	list_splice(&amp;unmap_list, &amp;pool-&gt;free_list);</div><div class='del'>-	spin_unlock_irq(&amp;pool-&gt;pool_lock);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void ib_fmr_cleanup_func(struct kthread_work *work)</div><div class='del'>-{</div><div class='del'>-	struct ib_fmr_pool *pool = container_of(work, struct ib_fmr_pool, work);</div><div class='del'>-</div><div class='del'>-	ib_fmr_batch_release(pool);</div><div class='del'>-	atomic_inc(&amp;pool-&gt;flush_ser);</div><div class='del'>-	wake_up_interruptible(&amp;pool-&gt;force_wait);</div><div class='del'>-</div><div class='del'>-	if (pool-&gt;flush_function)</div><div class='del'>-		pool-&gt;flush_function(pool, pool-&gt;flush_arg);</div><div class='del'>-</div><div class='del'>-	if (atomic_read(&amp;pool-&gt;flush_ser) - atomic_read(&amp;pool-&gt;req_ser) &lt; 0)</div><div class='del'>-		kthread_queue_work(pool-&gt;worker, &amp;pool-&gt;work);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * ib_create_fmr_pool - Create an FMR pool</div><div class='del'>- * @pd:Protection domain for FMRs</div><div class='del'>- * @params:FMR pool parameters</div><div class='del'>- *</div><div class='del'>- * Create a pool of FMRs.  Return value is pointer to new pool or</div><div class='del'>- * error code if creation failed.</div><div class='del'>- */</div><div class='del'>-struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd             *pd,</div><div class='del'>-				       struct ib_fmr_pool_param *params)</div><div class='del'>-{</div><div class='del'>-	struct ib_device   *device;</div><div class='del'>-	struct ib_fmr_pool *pool;</div><div class='del'>-	int i;</div><div class='del'>-	int ret;</div><div class='del'>-	int max_remaps;</div><div class='del'>-</div><div class='del'>-	if (!params)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	device = pd-&gt;device;</div><div class='del'>-	if (!device-&gt;ops.alloc_fmr    || !device-&gt;ops.dealloc_fmr  ||</div><div class='del'>-	    !device-&gt;ops.map_phys_fmr || !device-&gt;ops.unmap_fmr) {</div><div class='del'>-		dev_info(&amp;device-&gt;dev, "Device does not support FMRs\n");</div><div class='del'>-		return ERR_PTR(-ENOSYS);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!device-&gt;attrs.max_map_per_fmr)</div><div class='del'>-		max_remaps = IB_FMR_MAX_REMAPS;</div><div class='del'>-	else</div><div class='del'>-		max_remaps = device-&gt;attrs.max_map_per_fmr;</div><div class='del'>-</div><div class='del'>-	pool = kmalloc(sizeof *pool, GFP_KERNEL);</div><div class='del'>-	if (!pool)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	pool-&gt;cache_bucket   = NULL;</div><div class='del'>-	pool-&gt;flush_function = params-&gt;flush_function;</div><div class='del'>-	pool-&gt;flush_arg      = params-&gt;flush_arg;</div><div class='del'>-</div><div class='del'>-	INIT_LIST_HEAD(&amp;pool-&gt;free_list);</div><div class='del'>-	INIT_LIST_HEAD(&amp;pool-&gt;dirty_list);</div><div class='del'>-</div><div class='del'>-	if (params-&gt;cache) {</div><div class='del'>-		pool-&gt;cache_bucket =</div><div class='del'>-			kmalloc_array(IB_FMR_HASH_SIZE,</div><div class='del'>-				      sizeof(*pool-&gt;cache_bucket),</div><div class='del'>-				      GFP_KERNEL);</div><div class='del'>-		if (!pool-&gt;cache_bucket) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto out_free_pool;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		for (i = 0; i &lt; IB_FMR_HASH_SIZE; ++i)</div><div class='del'>-			INIT_HLIST_HEAD(pool-&gt;cache_bucket + i);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	pool-&gt;pool_size       = 0;</div><div class='del'>-	pool-&gt;max_pages       = params-&gt;max_pages_per_fmr;</div><div class='del'>-	pool-&gt;max_remaps      = max_remaps;</div><div class='del'>-	pool-&gt;dirty_watermark = params-&gt;dirty_watermark;</div><div class='del'>-	pool-&gt;dirty_len       = 0;</div><div class='del'>-	spin_lock_init(&amp;pool-&gt;pool_lock);</div><div class='del'>-	atomic_set(&amp;pool-&gt;req_ser,   0);</div><div class='del'>-	atomic_set(&amp;pool-&gt;flush_ser, 0);</div><div class='del'>-	init_waitqueue_head(&amp;pool-&gt;force_wait);</div><div class='del'>-</div><div class='del'>-	pool-&gt;worker =</div><div class='del'>-		kthread_create_worker(0, "ib_fmr(%s)", dev_name(&amp;device-&gt;dev));</div><div class='del'>-	if (IS_ERR(pool-&gt;worker)) {</div><div class='del'>-		pr_warn(PFX "couldn't start cleanup kthread worker\n");</div><div class='del'>-		ret = PTR_ERR(pool-&gt;worker);</div><div class='del'>-		goto out_free_pool;</div><div class='del'>-	}</div><div class='del'>-	kthread_init_work(&amp;pool-&gt;work, ib_fmr_cleanup_func);</div><div class='del'>-</div><div class='del'>-	{</div><div class='del'>-		struct ib_pool_fmr *fmr;</div><div class='del'>-		struct ib_fmr_attr fmr_attr = {</div><div class='del'>-			.max_pages  = params-&gt;max_pages_per_fmr,</div><div class='del'>-			.max_maps   = pool-&gt;max_remaps,</div><div class='del'>-			.page_shift = params-&gt;page_shift</div><div class='del'>-		};</div><div class='del'>-		int bytes_per_fmr = sizeof *fmr;</div><div class='del'>-</div><div class='del'>-		if (pool-&gt;cache_bucket)</div><div class='del'>-			bytes_per_fmr += params-&gt;max_pages_per_fmr * sizeof (u64);</div><div class='del'>-</div><div class='del'>-		for (i = 0; i &lt; params-&gt;pool_size; ++i) {</div><div class='del'>-			fmr = kmalloc(bytes_per_fmr, GFP_KERNEL);</div><div class='del'>-			if (!fmr)</div><div class='del'>-				goto out_fail;</div><div class='del'>-</div><div class='del'>-			fmr-&gt;pool             = pool;</div><div class='del'>-			fmr-&gt;remap_count      = 0;</div><div class='del'>-			fmr-&gt;ref_count        = 0;</div><div class='del'>-			INIT_HLIST_NODE(&amp;fmr-&gt;cache_node);</div><div class='del'>-</div><div class='del'>-			fmr-&gt;fmr = ib_alloc_fmr(pd, params-&gt;access, &amp;fmr_attr);</div><div class='del'>-			if (IS_ERR(fmr-&gt;fmr)) {</div><div class='del'>-				pr_warn(PFX "fmr_create failed for FMR %d\n",</div><div class='del'>-					i);</div><div class='del'>-				kfree(fmr);</div><div class='del'>-				goto out_fail;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			list_add_tail(&amp;fmr-&gt;list, &amp;pool-&gt;free_list);</div><div class='del'>-			++pool-&gt;pool_size;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return pool;</div><div class='del'>-</div><div class='del'>- out_free_pool:</div><div class='del'>-	kfree(pool-&gt;cache_bucket);</div><div class='del'>-	kfree(pool);</div><div class='del'>-</div><div class='del'>-	return ERR_PTR(ret);</div><div class='del'>-</div><div class='del'>- out_fail:</div><div class='del'>-	ib_destroy_fmr_pool(pool);</div><div class='del'>-</div><div class='del'>-	return ERR_PTR(-ENOMEM);</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_create_fmr_pool);</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * ib_destroy_fmr_pool - Free FMR pool</div><div class='del'>- * @pool:FMR pool to free</div><div class='del'>- *</div><div class='del'>- * Destroy an FMR pool and free all associated resources.</div><div class='del'>- */</div><div class='del'>-void ib_destroy_fmr_pool(struct ib_fmr_pool *pool)</div><div class='del'>-{</div><div class='del'>-	struct ib_pool_fmr *fmr;</div><div class='del'>-	struct ib_pool_fmr *tmp;</div><div class='del'>-	LIST_HEAD(fmr_list);</div><div class='del'>-	int                 i;</div><div class='del'>-</div><div class='del'>-	kthread_destroy_worker(pool-&gt;worker);</div><div class='del'>-	ib_fmr_batch_release(pool);</div><div class='del'>-</div><div class='del'>-	i = 0;</div><div class='del'>-	list_for_each_entry_safe(fmr, tmp, &amp;pool-&gt;free_list, list) {</div><div class='del'>-		if (fmr-&gt;remap_count) {</div><div class='del'>-			INIT_LIST_HEAD(&amp;fmr_list);</div><div class='del'>-			list_add_tail(&amp;fmr-&gt;fmr-&gt;list, &amp;fmr_list);</div><div class='del'>-			ib_unmap_fmr(&amp;fmr_list);</div><div class='del'>-		}</div><div class='del'>-		ib_dealloc_fmr(fmr-&gt;fmr);</div><div class='del'>-		list_del(&amp;fmr-&gt;list);</div><div class='del'>-		kfree(fmr);</div><div class='del'>-		++i;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (i &lt; pool-&gt;pool_size)</div><div class='del'>-		pr_warn(PFX "pool still has %d regions registered\n",</div><div class='del'>-			pool-&gt;pool_size - i);</div><div class='del'>-</div><div class='del'>-	kfree(pool-&gt;cache_bucket);</div><div class='del'>-	kfree(pool);</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_destroy_fmr_pool);</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * ib_flush_fmr_pool - Invalidate all unmapped FMRs</div><div class='del'>- * @pool:FMR pool to flush</div><div class='del'>- *</div><div class='del'>- * Ensure that all unmapped FMRs are fully invalidated.</div><div class='del'>- */</div><div class='del'>-int ib_flush_fmr_pool(struct ib_fmr_pool *pool)</div><div class='del'>-{</div><div class='del'>-	int serial;</div><div class='del'>-	struct ib_pool_fmr *fmr, *next;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * The free_list holds FMRs that may have been used</div><div class='del'>-	 * but have not been remapped enough times to be dirty.</div><div class='del'>-	 * Put them on the dirty list now so that the cleanup</div><div class='del'>-	 * thread will reap them too.</div><div class='del'>-	 */</div><div class='del'>-	spin_lock_irq(&amp;pool-&gt;pool_lock);</div><div class='del'>-	list_for_each_entry_safe(fmr, next, &amp;pool-&gt;free_list, list) {</div><div class='del'>-		if (fmr-&gt;remap_count &gt; 0)</div><div class='del'>-			list_move(&amp;fmr-&gt;list, &amp;pool-&gt;dirty_list);</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irq(&amp;pool-&gt;pool_lock);</div><div class='del'>-</div><div class='del'>-	serial = atomic_inc_return(&amp;pool-&gt;req_ser);</div><div class='del'>-	kthread_queue_work(pool-&gt;worker, &amp;pool-&gt;work);</div><div class='del'>-</div><div class='del'>-	if (wait_event_interruptible(pool-&gt;force_wait,</div><div class='del'>-				     atomic_read(&amp;pool-&gt;flush_ser) - serial &gt;= 0))</div><div class='del'>-		return -EINTR;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_flush_fmr_pool);</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * ib_fmr_pool_map_phys - Map an FMR from an FMR pool.</div><div class='del'>- * @pool_handle: FMR pool to allocate FMR from</div><div class='del'>- * @page_list: List of pages to map</div><div class='del'>- * @list_len: Number of pages in @page_list</div><div class='del'>- * @io_virtual_address: I/O virtual address for new FMR</div><div class='del'>- */</div><div class='del'>-struct ib_pool_fmr *ib_fmr_pool_map_phys(struct ib_fmr_pool *pool_handle,</div><div class='del'>-					 u64                *page_list,</div><div class='del'>-					 int                 list_len,</div><div class='del'>-					 u64                 io_virtual_address)</div><div class='del'>-{</div><div class='del'>-	struct ib_fmr_pool *pool = pool_handle;</div><div class='del'>-	struct ib_pool_fmr *fmr;</div><div class='del'>-	unsigned long       flags;</div><div class='del'>-	int                 result;</div><div class='del'>-</div><div class='del'>-	if (list_len &lt; 1 || list_len &gt; pool-&gt;max_pages)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;pool-&gt;pool_lock, flags);</div><div class='del'>-	fmr = ib_fmr_cache_lookup(pool,</div><div class='del'>-				  page_list,</div><div class='del'>-				  list_len,</div><div class='del'>-				  io_virtual_address);</div><div class='del'>-	if (fmr) {</div><div class='del'>-		/* found in cache */</div><div class='del'>-		++fmr-&gt;ref_count;</div><div class='del'>-		if (fmr-&gt;ref_count == 1) {</div><div class='del'>-			list_del(&amp;fmr-&gt;list);</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		spin_unlock_irqrestore(&amp;pool-&gt;pool_lock, flags);</div><div class='del'>-</div><div class='del'>-		return fmr;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (list_empty(&amp;pool-&gt;free_list)) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;pool-&gt;pool_lock, flags);</div><div class='del'>-		return ERR_PTR(-EAGAIN);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	fmr = list_entry(pool-&gt;free_list.next, struct ib_pool_fmr, list);</div><div class='del'>-	list_del(&amp;fmr-&gt;list);</div><div class='del'>-	hlist_del_init(&amp;fmr-&gt;cache_node);</div><div class='del'>-	spin_unlock_irqrestore(&amp;pool-&gt;pool_lock, flags);</div><div class='del'>-</div><div class='del'>-	result = ib_map_phys_fmr(fmr-&gt;fmr, page_list, list_len,</div><div class='del'>-				 io_virtual_address);</div><div class='del'>-</div><div class='del'>-	if (result) {</div><div class='del'>-		spin_lock_irqsave(&amp;pool-&gt;pool_lock, flags);</div><div class='del'>-		list_add(&amp;fmr-&gt;list, &amp;pool-&gt;free_list);</div><div class='del'>-		spin_unlock_irqrestore(&amp;pool-&gt;pool_lock, flags);</div><div class='del'>-</div><div class='del'>-		pr_warn(PFX "fmr_map returns %d\n", result);</div><div class='del'>-</div><div class='del'>-		return ERR_PTR(result);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	++fmr-&gt;remap_count;</div><div class='del'>-	fmr-&gt;ref_count = 1;</div><div class='del'>-</div><div class='del'>-	if (pool-&gt;cache_bucket) {</div><div class='del'>-		fmr-&gt;io_virtual_address = io_virtual_address;</div><div class='del'>-		fmr-&gt;page_list_len      = list_len;</div><div class='del'>-		memcpy(fmr-&gt;page_list, page_list, list_len * sizeof(*page_list));</div><div class='del'>-</div><div class='del'>-		spin_lock_irqsave(&amp;pool-&gt;pool_lock, flags);</div><div class='del'>-		hlist_add_head(&amp;fmr-&gt;cache_node,</div><div class='del'>-			       pool-&gt;cache_bucket + ib_fmr_hash(fmr-&gt;page_list[0]));</div><div class='del'>-		spin_unlock_irqrestore(&amp;pool-&gt;pool_lock, flags);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return fmr;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_fmr_pool_map_phys);</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * ib_fmr_pool_unmap - Unmap FMR</div><div class='del'>- * @fmr:FMR to unmap</div><div class='del'>- *</div><div class='del'>- * Unmap an FMR.  The FMR mapping may remain valid until the FMR is</div><div class='del'>- * reused (or until ib_flush_fmr_pool() is called).</div><div class='del'>- */</div><div class='del'>-void ib_fmr_pool_unmap(struct ib_pool_fmr *fmr)</div><div class='del'>-{</div><div class='del'>-	struct ib_fmr_pool *pool;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	pool = fmr-&gt;pool;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;pool-&gt;pool_lock, flags);</div><div class='del'>-</div><div class='del'>-	--fmr-&gt;ref_count;</div><div class='del'>-	if (!fmr-&gt;ref_count) {</div><div class='del'>-		if (fmr-&gt;remap_count &lt; pool-&gt;max_remaps) {</div><div class='del'>-			list_add_tail(&amp;fmr-&gt;list, &amp;pool-&gt;free_list);</div><div class='del'>-		} else {</div><div class='del'>-			list_add_tail(&amp;fmr-&gt;list, &amp;pool-&gt;dirty_list);</div><div class='del'>-			if (++pool-&gt;dirty_len &gt;= pool-&gt;dirty_watermark) {</div><div class='del'>-				atomic_inc(&amp;pool-&gt;req_ser);</div><div class='del'>-				kthread_queue_work(pool-&gt;worker, &amp;pool-&gt;work);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-#ifdef DEBUG</div><div class='del'>-	if (fmr-&gt;ref_count &lt; 0)</div><div class='del'>-		pr_warn(PFX "FMR %p has ref count %d &lt; 0\n",</div><div class='del'>-			fmr, fmr-&gt;ref_count);</div><div class='del'>-#endif</div><div class='del'>-</div><div class='del'>-	spin_unlock_irqrestore(&amp;pool-&gt;pool_lock, flags);</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_fmr_pool_unmap);</div><div class='head'>diff --git a/drivers/infiniband/core/ib_core_uverbs.c b/drivers/infiniband/core/ib_core_uverbs.c<br/>new file mode 100644<br/>index 000000000000..b51bd7087a88<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/ib_core_uverbs.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/ib_core_uverbs.c</a></div><div class='hunk'>@@ -0,0 +1,367 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2005 Mellanox Technologies. All rights reserved.</div><div class='add'>+ * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='add'>+ * Copyright 2019 Marvell. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+#include &lt;linux/xarray.h&gt;</div><div class='add'>+#include "uverbs.h"</div><div class='add'>+#include "core_priv.h"</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * rdma_umap_priv_init() - Initialize the private data of a vma</div><div class='add'>+ *</div><div class='add'>+ * @priv: The already allocated private data</div><div class='add'>+ * @vma: The vm area struct that needs private data</div><div class='add'>+ * @entry: entry into the mmap_xa that needs to be linked with</div><div class='add'>+ *       this vma</div><div class='add'>+ *</div><div class='add'>+ * Each time we map IO memory into user space this keeps track of the</div><div class='add'>+ * mapping. When the device is hot-unplugged we 'zap' the mmaps in user space</div><div class='add'>+ * to point to the zero page and allow the hot unplug to proceed.</div><div class='add'>+ *</div><div class='add'>+ * This is necessary for cases like PCI physical hot unplug as the actual BAR</div><div class='add'>+ * memory may vanish after this and access to it from userspace could MCE.</div><div class='add'>+ *</div><div class='add'>+ * RDMA drivers supporting disassociation must have their user space designed</div><div class='add'>+ * to cope in some way with their IO pages going to the zero page.</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+void rdma_umap_priv_init(struct rdma_umap_priv *priv,</div><div class='add'>+			 struct vm_area_struct *vma,</div><div class='add'>+			 struct rdma_user_mmap_entry *entry)</div><div class='add'>+{</div><div class='add'>+	struct ib_uverbs_file *ufile = vma-&gt;vm_file-&gt;private_data;</div><div class='add'>+</div><div class='add'>+	priv-&gt;vma = vma;</div><div class='add'>+	if (entry) {</div><div class='add'>+		kref_get(&amp;entry-&gt;ref);</div><div class='add'>+		priv-&gt;entry = entry;</div><div class='add'>+	}</div><div class='add'>+	vma-&gt;vm_private_data = priv;</div><div class='add'>+	/* vm_ops is setup in ib_uverbs_mmap() to avoid module dependencies */</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;ufile-&gt;umap_lock);</div><div class='add'>+	list_add(&amp;priv-&gt;list, &amp;ufile-&gt;umaps);</div><div class='add'>+	mutex_unlock(&amp;ufile-&gt;umap_lock);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_umap_priv_init);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * rdma_user_mmap_io() - Map IO memory into a process</div><div class='add'>+ *</div><div class='add'>+ * @ucontext: associated user context</div><div class='add'>+ * @vma: the vma related to the current mmap call</div><div class='add'>+ * @pfn: pfn to map</div><div class='add'>+ * @size: size to map</div><div class='add'>+ * @prot: pgprot to use in remap call</div><div class='add'>+ * @entry: mmap_entry retrieved from rdma_user_mmap_entry_get(), or NULL</div><div class='add'>+ *         if mmap_entry is not used by the driver</div><div class='add'>+ *</div><div class='add'>+ * This is to be called by drivers as part of their mmap() functions if they</div><div class='add'>+ * wish to send something like PCI-E BAR memory to userspace.</div><div class='add'>+ *</div><div class='add'>+ * Return -EINVAL on wrong flags or size, -EAGAIN on failure to map. 0 on</div><div class='add'>+ * success.</div><div class='add'>+ */</div><div class='add'>+int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma,</div><div class='add'>+		      unsigned long pfn, unsigned long size, pgprot_t prot,</div><div class='add'>+		      struct rdma_user_mmap_entry *entry)</div><div class='add'>+{</div><div class='add'>+	struct ib_uverbs_file *ufile = ucontext-&gt;ufile;</div><div class='add'>+	struct rdma_umap_priv *priv;</div><div class='add'>+</div><div class='add'>+	if (!(vma-&gt;vm_flags &amp; VM_SHARED))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (vma-&gt;vm_end - vma-&gt;vm_start != size)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	/* Driver is using this wrong, must be called by ib_uverbs_mmap */</div><div class='add'>+	if (WARN_ON(!vma-&gt;vm_file ||</div><div class='add'>+		    vma-&gt;vm_file-&gt;private_data != ufile))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	lockdep_assert_held(&amp;ufile-&gt;device-&gt;disassociate_srcu);</div><div class='add'>+</div><div class='add'>+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);</div><div class='add'>+	if (!priv)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	vma-&gt;vm_page_prot = prot;</div><div class='add'>+	if (io_remap_pfn_range(vma, vma-&gt;vm_start, pfn, size, prot)) {</div><div class='add'>+		kfree(priv);</div><div class='add'>+		return -EAGAIN;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rdma_umap_priv_init(priv, vma, entry);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_user_mmap_io);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * rdma_user_mmap_entry_get_pgoff() - Get an entry from the mmap_xa</div><div class='add'>+ *</div><div class='add'>+ * @ucontext: associated user context</div><div class='add'>+ * @pgoff: The mmap offset &gt;&gt; PAGE_SHIFT</div><div class='add'>+ *</div><div class='add'>+ * This function is called when a user tries to mmap with an offset (returned</div><div class='add'>+ * by rdma_user_mmap_get_offset()) it initially received from the driver. The</div><div class='add'>+ * rdma_user_mmap_entry was created by the function</div><div class='add'>+ * rdma_user_mmap_entry_insert().  This function increases the refcnt of the</div><div class='add'>+ * entry so that it won't be deleted from the xarray in the meantime.</div><div class='add'>+ *</div><div class='add'>+ * Return an reference to an entry if exists or NULL if there is no</div><div class='add'>+ * match. rdma_user_mmap_entry_put() must be called to put the reference.</div><div class='add'>+ */</div><div class='add'>+struct rdma_user_mmap_entry *</div><div class='add'>+rdma_user_mmap_entry_get_pgoff(struct ib_ucontext *ucontext,</div><div class='add'>+			       unsigned long pgoff)</div><div class='add'>+{</div><div class='add'>+	struct rdma_user_mmap_entry *entry;</div><div class='add'>+</div><div class='add'>+	if (pgoff &gt; U32_MAX)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	xa_lock(&amp;ucontext-&gt;mmap_xa);</div><div class='add'>+</div><div class='add'>+	entry = xa_load(&amp;ucontext-&gt;mmap_xa, pgoff);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * If refcount is zero, entry is already being deleted, driver_removed</div><div class='add'>+	 * indicates that the no further mmaps are possible and we waiting for</div><div class='add'>+	 * the active VMAs to be closed.</div><div class='add'>+	 */</div><div class='add'>+	if (!entry || entry-&gt;start_pgoff != pgoff || entry-&gt;driver_removed ||</div><div class='add'>+	    !kref_get_unless_zero(&amp;entry-&gt;ref))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	xa_unlock(&amp;ucontext-&gt;mmap_xa);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(ucontext-&gt;device, "mmap: pgoff[%#lx] npages[%#zx] returned\n",</div><div class='add'>+		  pgoff, entry-&gt;npages);</div><div class='add'>+</div><div class='add'>+	return entry;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	xa_unlock(&amp;ucontext-&gt;mmap_xa);</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_user_mmap_entry_get_pgoff);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * rdma_user_mmap_entry_get() - Get an entry from the mmap_xa</div><div class='add'>+ *</div><div class='add'>+ * @ucontext: associated user context</div><div class='add'>+ * @vma: the vma being mmap'd into</div><div class='add'>+ *</div><div class='add'>+ * This function is like rdma_user_mmap_entry_get_pgoff() except that it also</div><div class='add'>+ * checks that the VMA is correct.</div><div class='add'>+ */</div><div class='add'>+struct rdma_user_mmap_entry *</div><div class='add'>+rdma_user_mmap_entry_get(struct ib_ucontext *ucontext,</div><div class='add'>+			 struct vm_area_struct *vma)</div><div class='add'>+{</div><div class='add'>+	struct rdma_user_mmap_entry *entry;</div><div class='add'>+</div><div class='add'>+	if (!(vma-&gt;vm_flags &amp; VM_SHARED))</div><div class='add'>+		return NULL;</div><div class='add'>+	entry = rdma_user_mmap_entry_get_pgoff(ucontext, vma-&gt;vm_pgoff);</div><div class='add'>+	if (!entry)</div><div class='add'>+		return NULL;</div><div class='add'>+	if (entry-&gt;npages * PAGE_SIZE != vma-&gt;vm_end - vma-&gt;vm_start) {</div><div class='add'>+		rdma_user_mmap_entry_put(entry);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+	return entry;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_user_mmap_entry_get);</div><div class='add'>+</div><div class='add'>+static void rdma_user_mmap_entry_free(struct kref *kref)</div><div class='add'>+{</div><div class='add'>+	struct rdma_user_mmap_entry *entry =</div><div class='add'>+		container_of(kref, struct rdma_user_mmap_entry, ref);</div><div class='add'>+	struct ib_ucontext *ucontext = entry-&gt;ucontext;</div><div class='add'>+	unsigned long i;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Erase all entries occupied by this single entry, this is deferred</div><div class='add'>+	 * until all VMA are closed so that the mmap offsets remain unique.</div><div class='add'>+	 */</div><div class='add'>+	xa_lock(&amp;ucontext-&gt;mmap_xa);</div><div class='add'>+	for (i = 0; i &lt; entry-&gt;npages; i++)</div><div class='add'>+		__xa_erase(&amp;ucontext-&gt;mmap_xa, entry-&gt;start_pgoff + i);</div><div class='add'>+	xa_unlock(&amp;ucontext-&gt;mmap_xa);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(ucontext-&gt;device, "mmap: pgoff[%#lx] npages[%#zx] removed\n",</div><div class='add'>+		  entry-&gt;start_pgoff, entry-&gt;npages);</div><div class='add'>+</div><div class='add'>+	if (ucontext-&gt;device-&gt;ops.mmap_free)</div><div class='add'>+		ucontext-&gt;device-&gt;ops.mmap_free(entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * rdma_user_mmap_entry_put() - Drop reference to the mmap entry</div><div class='add'>+ *</div><div class='add'>+ * @entry: an entry in the mmap_xa</div><div class='add'>+ *</div><div class='add'>+ * This function is called when the mapping is closed if it was</div><div class='add'>+ * an io mapping or when the driver is done with the entry for</div><div class='add'>+ * some other reason.</div><div class='add'>+ * Should be called after rdma_user_mmap_entry_get was called</div><div class='add'>+ * and entry is no longer needed. This function will erase the</div><div class='add'>+ * entry and free it if its refcnt reaches zero.</div><div class='add'>+ */</div><div class='add'>+void rdma_user_mmap_entry_put(struct rdma_user_mmap_entry *entry)</div><div class='add'>+{</div><div class='add'>+	kref_put(&amp;entry-&gt;ref, rdma_user_mmap_entry_free);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_user_mmap_entry_put);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * rdma_user_mmap_entry_remove() - Drop reference to entry and</div><div class='add'>+ *				   mark it as unmmapable</div><div class='add'>+ *</div><div class='add'>+ * @entry: the entry to insert into the mmap_xa</div><div class='add'>+ *</div><div class='add'>+ * Drivers can call this to prevent userspace from creating more mappings for</div><div class='add'>+ * entry, however existing mmaps continue to exist and ops-&gt;mmap_free() will</div><div class='add'>+ * not be called until all user mmaps are destroyed.</div><div class='add'>+ */</div><div class='add'>+void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry)</div><div class='add'>+{</div><div class='add'>+	if (!entry)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	xa_lock(&amp;entry-&gt;ucontext-&gt;mmap_xa);</div><div class='add'>+	entry-&gt;driver_removed = true;</div><div class='add'>+	xa_unlock(&amp;entry-&gt;ucontext-&gt;mmap_xa);</div><div class='add'>+	kref_put(&amp;entry-&gt;ref, rdma_user_mmap_entry_free);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_user_mmap_entry_remove);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * rdma_user_mmap_entry_insert_range() - Insert an entry to the mmap_xa</div><div class='add'>+ *					 in a given range.</div><div class='add'>+ *</div><div class='add'>+ * @ucontext: associated user context.</div><div class='add'>+ * @entry: the entry to insert into the mmap_xa</div><div class='add'>+ * @length: length of the address that will be mmapped</div><div class='add'>+ * @min_pgoff: minimum pgoff to be returned</div><div class='add'>+ * @max_pgoff: maximum pgoff to be returned</div><div class='add'>+ *</div><div class='add'>+ * This function should be called by drivers that use the rdma_user_mmap</div><div class='add'>+ * interface for implementing their mmap syscall A database of mmap offsets is</div><div class='add'>+ * handled in the core and helper functions are provided to insert entries</div><div class='add'>+ * into the database and extract entries when the user calls mmap with the</div><div class='add'>+ * given offset. The function allocates a unique page offset in a given range</div><div class='add'>+ * that should be provided to user, the user will use the offset to retrieve</div><div class='add'>+ * information such as address to be mapped and how.</div><div class='add'>+ *</div><div class='add'>+ * Return: 0 on success and -ENOMEM on failure</div><div class='add'>+ */</div><div class='add'>+int rdma_user_mmap_entry_insert_range(struct ib_ucontext *ucontext,</div><div class='add'>+				      struct rdma_user_mmap_entry *entry,</div><div class='add'>+				      size_t length, u32 min_pgoff,</div><div class='add'>+				      u32 max_pgoff)</div><div class='add'>+{</div><div class='add'>+	struct ib_uverbs_file *ufile = ucontext-&gt;ufile;</div><div class='add'>+	XA_STATE(xas, &amp;ucontext-&gt;mmap_xa, min_pgoff);</div><div class='add'>+	u32 xa_first, xa_last, npages;</div><div class='add'>+	int err;</div><div class='add'>+	u32 i;</div><div class='add'>+</div><div class='add'>+	if (!entry)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	kref_init(&amp;entry-&gt;ref);</div><div class='add'>+	entry-&gt;ucontext = ucontext;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * We want the whole allocation to be done without interruption from a</div><div class='add'>+	 * different thread. The allocation requires finding a free range and</div><div class='add'>+	 * storing. During the xa_insert the lock could be released, possibly</div><div class='add'>+	 * allowing another thread to choose the same range.</div><div class='add'>+	 */</div><div class='add'>+	mutex_lock(&amp;ufile-&gt;umap_lock);</div><div class='add'>+</div><div class='add'>+	xa_lock(&amp;ucontext-&gt;mmap_xa);</div><div class='add'>+</div><div class='add'>+	/* We want to find an empty range */</div><div class='add'>+	npages = (u32)DIV_ROUND_UP(length, PAGE_SIZE);</div><div class='add'>+	entry-&gt;npages = npages;</div><div class='add'>+	while (true) {</div><div class='add'>+		/* First find an empty index */</div><div class='add'>+		xas_find_marked(&amp;xas, max_pgoff, XA_FREE_MARK);</div><div class='add'>+		if (xas.xa_node == XAS_RESTART)</div><div class='add'>+			goto err_unlock;</div><div class='add'>+</div><div class='add'>+		xa_first = xas.xa_index;</div><div class='add'>+</div><div class='add'>+		/* Is there enough room to have the range? */</div><div class='add'>+		if (check_add_overflow(xa_first, npages, &amp;xa_last))</div><div class='add'>+			goto err_unlock;</div><div class='add'>+</div><div class='add'>+		/*</div><div class='add'>+		 * Now look for the next present entry. If an entry doesn't</div><div class='add'>+		 * exist, we found an empty range and can proceed.</div><div class='add'>+		 */</div><div class='add'>+		xas_next_entry(&amp;xas, xa_last - 1);</div><div class='add'>+		if (xas.xa_node == XAS_BOUNDS || xas.xa_index &gt;= xa_last)</div><div class='add'>+			break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = xa_first; i &lt; xa_last; i++) {</div><div class='add'>+		err = __xa_insert(&amp;ucontext-&gt;mmap_xa, i, entry, GFP_KERNEL);</div><div class='add'>+		if (err)</div><div class='add'>+			goto err_undo;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Internally the kernel uses a page offset, in libc this is a byte</div><div class='add'>+	 * offset. Drivers should not return pgoff to userspace.</div><div class='add'>+	 */</div><div class='add'>+	entry-&gt;start_pgoff = xa_first;</div><div class='add'>+	xa_unlock(&amp;ucontext-&gt;mmap_xa);</div><div class='add'>+	mutex_unlock(&amp;ufile-&gt;umap_lock);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(ucontext-&gt;device, "mmap: pgoff[%#lx] npages[%#x] inserted\n",</div><div class='add'>+		  entry-&gt;start_pgoff, npages);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_undo:</div><div class='add'>+	for (; i &gt; xa_first; i--)</div><div class='add'>+		__xa_erase(&amp;ucontext-&gt;mmap_xa, i - 1);</div><div class='add'>+</div><div class='add'>+err_unlock:</div><div class='add'>+	xa_unlock(&amp;ucontext-&gt;mmap_xa);</div><div class='add'>+	mutex_unlock(&amp;ufile-&gt;umap_lock);</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_user_mmap_entry_insert_range);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * rdma_user_mmap_entry_insert() - Insert an entry to the mmap_xa.</div><div class='add'>+ *</div><div class='add'>+ * @ucontext: associated user context.</div><div class='add'>+ * @entry: the entry to insert into the mmap_xa</div><div class='add'>+ * @length: length of the address that will be mmapped</div><div class='add'>+ *</div><div class='add'>+ * This function should be called by drivers that use the rdma_user_mmap</div><div class='add'>+ * interface for handling user mmapped addresses. The database is handled in</div><div class='add'>+ * the core and helper functions are provided to insert entries into the</div><div class='add'>+ * database and extract entries when the user calls mmap with the given offset.</div><div class='add'>+ * The function allocates a unique page offset that should be provided to user,</div><div class='add'>+ * the user will use the offset to retrieve information such as address to</div><div class='add'>+ * be mapped and how.</div><div class='add'>+ *</div><div class='add'>+ * Return: 0 on success and -ENOMEM on failure</div><div class='add'>+ */</div><div class='add'>+int rdma_user_mmap_entry_insert(struct ib_ucontext *ucontext,</div><div class='add'>+				struct rdma_user_mmap_entry *entry,</div><div class='add'>+				size_t length)</div><div class='add'>+{</div><div class='add'>+	return rdma_user_mmap_entry_insert_range(ucontext, entry, length, 0,</div><div class='add'>+						 U32_MAX);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_user_mmap_entry_insert);</div><div class='head'>diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c<br/>index 72141c5b7c95..62410578dec3 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/iwcm.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/iwcm.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/iwcm.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/iwcm.c</a></div><div class='hunk'>@@ -109,9 +109,10 @@ static struct ctl_table iwcm_ctl_table[] = {</div><div class='ctx'> 		.data		= &amp;default_backlog,</div><div class='ctx'> 		.maxlen		= sizeof(default_backlog),</div><div class='ctx'> 		.mode		= 0644,</div><div class='del'>-		.proc_handler	= proc_dointvec,</div><div class='add'>+		.proc_handler	= proc_dointvec_minmax,</div><div class='add'>+		.extra1		= SYSCTL_ZERO,</div><div class='add'>+		.extra2		= SYSCTL_INT_MAX,</div><div class='ctx'> 	},</div><div class='del'>-	{ }</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -144,8 +145,8 @@ static struct iwcm_work *get_work(struct iwcm_id_private *cm_id_priv)</div><div class='ctx'> </div><div class='ctx'> 	if (list_empty(&amp;cm_id_priv-&gt;work_free_list))</div><div class='ctx'> 		return NULL;</div><div class='del'>-	work = list_entry(cm_id_priv-&gt;work_free_list.next, struct iwcm_work,</div><div class='del'>-			  free_list);</div><div class='add'>+	work = list_first_entry(&amp;cm_id_priv-&gt;work_free_list, struct iwcm_work,</div><div class='add'>+				free_list);</div><div class='ctx'> 	list_del_init(&amp;work-&gt;free_list);</div><div class='ctx'> 	return work;</div><div class='ctx'> }</div><div class='hunk'>@@ -159,8 +160,10 @@ static void dealloc_work_entries(struct iwcm_id_private *cm_id_priv)</div><div class='ctx'> {</div><div class='ctx'> 	struct list_head *e, *tmp;</div><div class='ctx'> </div><div class='del'>-	list_for_each_safe(e, tmp, &amp;cm_id_priv-&gt;work_free_list)</div><div class='add'>+	list_for_each_safe(e, tmp, &amp;cm_id_priv-&gt;work_free_list) {</div><div class='add'>+		list_del(e);</div><div class='ctx'> 		kfree(list_entry(e, struct iwcm_work, free_list));</div><div class='add'>+	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int alloc_work_entries(struct iwcm_id_private *cm_id_priv, int count)</div><div class='hunk'>@@ -205,25 +208,24 @@ static void free_cm_id(struct iwcm_id_private *cm_id_priv)</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * Release a reference on cm_id. If the last reference is being</div><div class='del'>- * released, free the cm_id and return 1.</div><div class='add'>+ * released, free the cm_id and return 'true'.</div><div class='ctx'>  */</div><div class='del'>-static int iwcm_deref_id(struct iwcm_id_private *cm_id_priv)</div><div class='add'>+static bool iwcm_deref_id(struct iwcm_id_private *cm_id_priv)</div><div class='ctx'> {</div><div class='del'>-	BUG_ON(atomic_read(&amp;cm_id_priv-&gt;refcount)==0);</div><div class='del'>-	if (atomic_dec_and_test(&amp;cm_id_priv-&gt;refcount)) {</div><div class='add'>+	if (refcount_dec_and_test(&amp;cm_id_priv-&gt;refcount)) {</div><div class='ctx'> 		BUG_ON(!list_empty(&amp;cm_id_priv-&gt;work_list));</div><div class='ctx'> 		free_cm_id(cm_id_priv);</div><div class='del'>-		return 1;</div><div class='add'>+		return true;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	return false;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void add_ref(struct iw_cm_id *cm_id)</div><div class='ctx'> {</div><div class='ctx'> 	struct iwcm_id_private *cm_id_priv;</div><div class='ctx'> 	cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);</div><div class='del'>-	atomic_inc(&amp;cm_id_priv-&gt;refcount);</div><div class='add'>+	refcount_inc(&amp;cm_id_priv-&gt;refcount);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void rem_ref(struct iw_cm_id *cm_id)</div><div class='hunk'>@@ -255,7 +257,7 @@ struct iw_cm_id *iw_create_cm_id(struct ib_device *device,</div><div class='ctx'> 	cm_id_priv-&gt;id.add_ref = add_ref;</div><div class='ctx'> 	cm_id_priv-&gt;id.rem_ref = rem_ref;</div><div class='ctx'> 	spin_lock_init(&amp;cm_id_priv-&gt;lock);</div><div class='del'>-	atomic_set(&amp;cm_id_priv-&gt;refcount, 1);</div><div class='add'>+	refcount_set(&amp;cm_id_priv-&gt;refcount, 1);</div><div class='ctx'> 	init_waitqueue_head(&amp;cm_id_priv-&gt;connect_wait);</div><div class='ctx'> 	init_completion(&amp;cm_id_priv-&gt;destroy_comp);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;cm_id_priv-&gt;work_list);</div><div class='hunk'>@@ -366,12 +368,12 @@ EXPORT_SYMBOL(iw_cm_disconnect);</div><div class='ctx'> /*</div><div class='ctx'>  * CM_ID &lt;-- DESTROYING</div><div class='ctx'>  *</div><div class='del'>- * Clean up all resources associated with the connection and release</div><div class='del'>- * the initial reference taken by iw_create_cm_id.</div><div class='add'>+ * Clean up all resources associated with the connection.</div><div class='ctx'>  */</div><div class='ctx'> static void destroy_cm_id(struct iw_cm_id *cm_id)</div><div class='ctx'> {</div><div class='ctx'> 	struct iwcm_id_private *cm_id_priv;</div><div class='add'>+	struct ib_qp *qp;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> </div><div class='ctx'> 	cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);</div><div class='hunk'>@@ -389,6 +391,9 @@ static void destroy_cm_id(struct iw_cm_id *cm_id)</div><div class='ctx'> 	set_bit(IWCM_F_DROP_EVENTS, &amp;cm_id_priv-&gt;flags);</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+	qp = cm_id_priv-&gt;qp;</div><div class='add'>+	cm_id_priv-&gt;qp = NULL;</div><div class='add'>+</div><div class='ctx'> 	switch (cm_id_priv-&gt;state) {</div><div class='ctx'> 	case IW_CM_STATE_LISTEN:</div><div class='ctx'> 		cm_id_priv-&gt;state = IW_CM_STATE_DESTROYING;</div><div class='hunk'>@@ -401,7 +406,7 @@ static void destroy_cm_id(struct iw_cm_id *cm_id)</div><div class='ctx'> 		cm_id_priv-&gt;state = IW_CM_STATE_DESTROYING;</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 		/* Abrupt close of the connection */</div><div class='del'>-		(void)iwcm_modify_qp_err(cm_id_priv-&gt;qp);</div><div class='add'>+		(void)iwcm_modify_qp_err(qp);</div><div class='ctx'> 		spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IW_CM_STATE_IDLE:</div><div class='hunk'>@@ -426,29 +431,30 @@ static void destroy_cm_id(struct iw_cm_id *cm_id)</div><div class='ctx'> 		BUG();</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='del'>-	if (cm_id_priv-&gt;qp) {</div><div class='del'>-		cm_id_priv-&gt;id.device-&gt;ops.iw_rem_ref(cm_id_priv-&gt;qp);</div><div class='del'>-		cm_id_priv-&gt;qp = NULL;</div><div class='del'>-	}</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+	if (qp)</div><div class='add'>+		cm_id_priv-&gt;id.device-&gt;ops.iw_rem_ref(qp);</div><div class='ctx'> </div><div class='ctx'> 	if (cm_id-&gt;mapped) {</div><div class='ctx'> 		iwpm_remove_mapinfo(&amp;cm_id-&gt;local_addr, &amp;cm_id-&gt;m_local_addr);</div><div class='ctx'> 		iwpm_remove_mapping(&amp;cm_id-&gt;local_addr, RDMA_NL_IWCM);</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='del'>-	(void)iwcm_deref_id(cm_id_priv);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='del'>- * This function is only called by the application thread and cannot</div><div class='del'>- * be called by the event thread. The function will wait for all</div><div class='del'>- * references to be released on the cm_id and then kfree the cm_id</div><div class='del'>- * object.</div><div class='add'>+ * Destroy cm_id. If the cm_id still has other references, wait for all</div><div class='add'>+ * references to be released on the cm_id and then release the initial</div><div class='add'>+ * reference taken by iw_create_cm_id.</div><div class='ctx'>  */</div><div class='ctx'> void iw_destroy_cm_id(struct iw_cm_id *cm_id)</div><div class='ctx'> {</div><div class='add'>+	struct iwcm_id_private *cm_id_priv;</div><div class='add'>+</div><div class='add'>+	cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);</div><div class='ctx'> 	destroy_cm_id(cm_id);</div><div class='add'>+	if (refcount_read(&amp;cm_id_priv-&gt;refcount) &gt; 1)</div><div class='add'>+		flush_workqueue(iwcm_wq);</div><div class='add'>+	iwcm_deref_id(cm_id_priv);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(iw_destroy_cm_id);</div><div class='ctx'> </div><div class='hunk'>@@ -671,11 +677,11 @@ int iw_cm_accept(struct iw_cm_id *cm_id,</div><div class='ctx'> 		BUG_ON(cm_id_priv-&gt;state != IW_CM_STATE_CONN_RECV);</div><div class='ctx'> 		cm_id_priv-&gt;state = IW_CM_STATE_IDLE;</div><div class='ctx'> 		spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-		if (cm_id_priv-&gt;qp) {</div><div class='del'>-			cm_id-&gt;device-&gt;ops.iw_rem_ref(qp);</div><div class='del'>-			cm_id_priv-&gt;qp = NULL;</div><div class='del'>-		}</div><div class='add'>+		qp = cm_id_priv-&gt;qp;</div><div class='add'>+		cm_id_priv-&gt;qp = NULL;</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+		if (qp)</div><div class='add'>+			cm_id-&gt;device-&gt;ops.iw_rem_ref(qp);</div><div class='ctx'> 		clear_bit(IWCM_F_CONNECT_WAIT, &amp;cm_id_priv-&gt;flags);</div><div class='ctx'> 		wake_up_all(&amp;cm_id_priv-&gt;connect_wait);</div><div class='ctx'> 	}</div><div class='hunk'>@@ -696,7 +702,7 @@ int iw_cm_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *iw_param)</div><div class='ctx'> 	struct iwcm_id_private *cm_id_priv;</div><div class='ctx'> 	int ret;</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	struct ib_qp *qp;</div><div class='add'>+	struct ib_qp *qp = NULL;</div><div class='ctx'> </div><div class='ctx'> 	cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);</div><div class='ctx'> </div><div class='hunk'>@@ -730,13 +736,13 @@ int iw_cm_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *iw_param)</div><div class='ctx'> 		return 0;	/* success */</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	if (cm_id_priv-&gt;qp) {</div><div class='del'>-		cm_id-&gt;device-&gt;ops.iw_rem_ref(qp);</div><div class='del'>-		cm_id_priv-&gt;qp = NULL;</div><div class='del'>-	}</div><div class='add'>+	qp = cm_id_priv-&gt;qp;</div><div class='add'>+	cm_id_priv-&gt;qp = NULL;</div><div class='ctx'> 	cm_id_priv-&gt;state = IW_CM_STATE_IDLE;</div><div class='ctx'> err:</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+	if (qp)</div><div class='add'>+		cm_id-&gt;device-&gt;ops.iw_rem_ref(qp);</div><div class='ctx'> 	clear_bit(IWCM_F_CONNECT_WAIT, &amp;cm_id_priv-&gt;flags);</div><div class='ctx'> 	wake_up_all(&amp;cm_id_priv-&gt;connect_wait);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -878,6 +884,7 @@ static int cm_conn_est_handler(struct iwcm_id_private *cm_id_priv,</div><div class='ctx'> static int cm_conn_rep_handler(struct iwcm_id_private *cm_id_priv,</div><div class='ctx'> 			       struct iw_cm_event *iw_event)</div><div class='ctx'> {</div><div class='add'>+	struct ib_qp *qp = NULL;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='hunk'>@@ -896,11 +903,13 @@ static int cm_conn_rep_handler(struct iwcm_id_private *cm_id_priv,</div><div class='ctx'> 		cm_id_priv-&gt;state = IW_CM_STATE_ESTABLISHED;</div><div class='ctx'> 	} else {</div><div class='ctx'> 		/* REJECTED or RESET */</div><div class='del'>-		cm_id_priv-&gt;id.device-&gt;ops.iw_rem_ref(cm_id_priv-&gt;qp);</div><div class='add'>+		qp = cm_id_priv-&gt;qp;</div><div class='ctx'> 		cm_id_priv-&gt;qp = NULL;</div><div class='ctx'> 		cm_id_priv-&gt;state = IW_CM_STATE_IDLE;</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+	if (qp)</div><div class='add'>+		cm_id_priv-&gt;id.device-&gt;ops.iw_rem_ref(qp);</div><div class='ctx'> 	ret = cm_id_priv-&gt;id.cm_handler(&amp;cm_id_priv-&gt;id, iw_event);</div><div class='ctx'> </div><div class='ctx'> 	if (iw_event-&gt;private_data_len)</div><div class='hunk'>@@ -942,21 +951,18 @@ static void cm_disconnect_handler(struct iwcm_id_private *cm_id_priv,</div><div class='ctx'> static int cm_close_handler(struct iwcm_id_private *cm_id_priv,</div><div class='ctx'> 				  struct iw_cm_event *iw_event)</div><div class='ctx'> {</div><div class='add'>+	struct ib_qp *qp;</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	int ret = 0;</div><div class='add'>+	int ret = 0, notify_event = 0;</div><div class='ctx'> 	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+	qp = cm_id_priv-&gt;qp;</div><div class='add'>+	cm_id_priv-&gt;qp = NULL;</div><div class='ctx'> </div><div class='del'>-	if (cm_id_priv-&gt;qp) {</div><div class='del'>-		cm_id_priv-&gt;id.device-&gt;ops.iw_rem_ref(cm_id_priv-&gt;qp);</div><div class='del'>-		cm_id_priv-&gt;qp = NULL;</div><div class='del'>-	}</div><div class='ctx'> 	switch (cm_id_priv-&gt;state) {</div><div class='ctx'> 	case IW_CM_STATE_ESTABLISHED:</div><div class='ctx'> 	case IW_CM_STATE_CLOSING:</div><div class='ctx'> 		cm_id_priv-&gt;state = IW_CM_STATE_IDLE;</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-		ret = cm_id_priv-&gt;id.cm_handler(&amp;cm_id_priv-&gt;id, iw_event);</div><div class='del'>-		spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='add'>+		notify_event = 1;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IW_CM_STATE_DESTROYING:</div><div class='ctx'> 		break;</div><div class='hunk'>@@ -965,6 +971,10 @@ static int cm_close_handler(struct iwcm_id_private *cm_id_priv,</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> </div><div class='add'>+	if (qp)</div><div class='add'>+		cm_id_priv-&gt;id.device-&gt;ops.iw_rem_ref(qp);</div><div class='add'>+	if (notify_event)</div><div class='add'>+		ret = cm_id_priv-&gt;id.cm_handler(&amp;cm_id_priv-&gt;id, iw_event);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1011,30 +1021,27 @@ static void cm_work_handler(struct work_struct *_work)</div><div class='ctx'> 	struct iw_cm_event levent;</div><div class='ctx'> 	struct iwcm_id_private *cm_id_priv = work-&gt;cm_id;</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	int empty;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='del'>-	empty = list_empty(&amp;cm_id_priv-&gt;work_list);</div><div class='del'>-	while (!empty) {</div><div class='del'>-		work = list_entry(cm_id_priv-&gt;work_list.next,</div><div class='del'>-				  struct iwcm_work, list);</div><div class='add'>+	while (!list_empty(&amp;cm_id_priv-&gt;work_list)) {</div><div class='add'>+		work = list_first_entry(&amp;cm_id_priv-&gt;work_list,</div><div class='add'>+					struct iwcm_work, list);</div><div class='ctx'> 		list_del_init(&amp;work-&gt;list);</div><div class='del'>-		empty = list_empty(&amp;cm_id_priv-&gt;work_list);</div><div class='ctx'> 		levent = work-&gt;event;</div><div class='ctx'> 		put_work(work);</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> </div><div class='ctx'> 		if (!test_bit(IWCM_F_DROP_EVENTS, &amp;cm_id_priv-&gt;flags)) {</div><div class='ctx'> 			ret = process_event(cm_id_priv, &amp;levent);</div><div class='del'>-			if (ret)</div><div class='add'>+			if (ret) {</div><div class='ctx'> 				destroy_cm_id(&amp;cm_id_priv-&gt;id);</div><div class='add'>+				WARN_ON_ONCE(iwcm_deref_id(cm_id_priv));</div><div class='add'>+			}</div><div class='ctx'> 		} else</div><div class='ctx'> 			pr_debug("dropping event %d\n", levent.event);</div><div class='ctx'> 		if (iwcm_deref_id(cm_id_priv))</div><div class='ctx'> 			return;</div><div class='del'>-		if (empty)</div><div class='del'>-			return;</div><div class='ctx'> 		spin_lock_irqsave(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='hunk'>@@ -1086,12 +1093,9 @@ static int cm_event_handler(struct iw_cm_id *cm_id,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	atomic_inc(&amp;cm_id_priv-&gt;refcount);</div><div class='del'>-	if (list_empty(&amp;cm_id_priv-&gt;work_list)) {</div><div class='del'>-		list_add_tail(&amp;work-&gt;list, &amp;cm_id_priv-&gt;work_list);</div><div class='del'>-		queue_work(iwcm_wq, &amp;work-&gt;work);</div><div class='del'>-	} else</div><div class='del'>-		list_add_tail(&amp;work-&gt;list, &amp;cm_id_priv-&gt;work_list);</div><div class='add'>+	refcount_inc(&amp;cm_id_priv-&gt;refcount);</div><div class='add'>+	list_add_tail(&amp;work-&gt;list, &amp;cm_id_priv-&gt;work_list);</div><div class='add'>+	queue_work(iwcm_wq, &amp;work-&gt;work);</div><div class='ctx'> out:</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;cm_id_priv-&gt;lock, flags);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -1179,29 +1183,34 @@ static int __init iw_cm_init(void)</div><div class='ctx'> </div><div class='ctx'> 	ret = iwpm_init(RDMA_NL_IWCM);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		pr_err("iw_cm: couldn't init iwpm\n");</div><div class='del'>-	else</div><div class='del'>-		rdma_nl_register(RDMA_NL_IWCM, iwcm_nl_cb_table);</div><div class='del'>-	iwcm_wq = alloc_ordered_workqueue("iw_cm_wq", 0);</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	iwcm_wq = alloc_ordered_workqueue("iw_cm_wq", WQ_MEM_RECLAIM);</div><div class='ctx'> 	if (!iwcm_wq)</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+		goto err_alloc;</div><div class='ctx'> </div><div class='ctx'> 	iwcm_ctl_table_hdr = register_net_sysctl(&amp;init_net, "net/iw_cm",</div><div class='ctx'> 						 iwcm_ctl_table);</div><div class='ctx'> 	if (!iwcm_ctl_table_hdr) {</div><div class='ctx'> 		pr_err("iw_cm: couldn't register sysctl paths\n");</div><div class='del'>-		destroy_workqueue(iwcm_wq);</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+		goto err_sysctl;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	rdma_nl_register(RDMA_NL_IWCM, iwcm_nl_cb_table);</div><div class='ctx'> 	return 0;</div><div class='add'>+</div><div class='add'>+err_sysctl:</div><div class='add'>+	destroy_workqueue(iwcm_wq);</div><div class='add'>+err_alloc:</div><div class='add'>+	iwpm_exit(RDMA_NL_IWCM);</div><div class='add'>+	return -ENOMEM;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void __exit iw_cm_cleanup(void)</div><div class='ctx'> {</div><div class='add'>+	rdma_nl_unregister(RDMA_NL_IWCM);</div><div class='ctx'> 	unregister_net_sysctl_table(iwcm_ctl_table_hdr);</div><div class='ctx'> 	destroy_workqueue(iwcm_wq);</div><div class='del'>-	rdma_nl_unregister(RDMA_NL_IWCM);</div><div class='ctx'> 	iwpm_exit(RDMA_NL_IWCM);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/core/iwcm.h b/drivers/infiniband/core/iwcm.h<br/>index 82c2cd1b0a80..bf74639be128 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/iwcm.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/iwcm.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/iwcm.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/iwcm.h</a></div><div class='hunk'>@@ -52,7 +52,7 @@ struct iwcm_id_private {</div><div class='ctx'> 	wait_queue_head_t connect_wait;</div><div class='ctx'> 	struct list_head work_list;</div><div class='ctx'> 	spinlock_t lock;</div><div class='del'>-	atomic_t refcount;</div><div class='add'>+	refcount_t refcount;</div><div class='ctx'> 	struct list_head work_free_list;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/core/iwpm_msg.c b/drivers/infiniband/core/iwpm_msg.c<br/>index 2452b0ddcf0d..3c9a9869212b 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/iwpm_msg.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/iwpm_msg.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/iwpm_msg.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/iwpm_msg.c</a></div><div class='hunk'>@@ -69,10 +69,6 @@ int iwpm_register_pid(struct iwpm_dev_data *pm_msg, u8 nl_client)</div><div class='ctx'> 	const char *err_str = "";</div><div class='ctx'> 	int ret = -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (!iwpm_valid_client(nl_client)) {</div><div class='del'>-		err_str = "Invalid port mapper client";</div><div class='del'>-		goto pid_query_error;</div><div class='del'>-	}</div><div class='ctx'> 	if (iwpm_check_registration(nl_client, IWPM_REG_VALID) ||</div><div class='ctx'> 			iwpm_user_pid == IWPM_PID_UNAVAILABLE)</div><div class='ctx'> 		return 0;</div><div class='hunk'>@@ -112,7 +108,7 @@ int iwpm_register_pid(struct iwpm_dev_data *pm_msg, u8 nl_client)</div><div class='ctx'> 	pr_debug("%s: Multicasting a nlmsg (dev = %s ifname = %s iwpm = %s)\n",</div><div class='ctx'> 		__func__, pm_msg-&gt;dev_name, pm_msg-&gt;if_name, iwpm_ulib_name);</div><div class='ctx'> </div><div class='del'>-	ret = rdma_nl_multicast(skb, RDMA_NL_GROUP_IWPM, GFP_KERNEL);</div><div class='add'>+	ret = rdma_nl_multicast(&amp;init_net, skb, RDMA_NL_GROUP_IWPM, GFP_KERNEL);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		skb = NULL; /* skb is freed in the netlink send-op handling */</div><div class='ctx'> 		iwpm_user_pid = IWPM_PID_UNAVAILABLE;</div><div class='hunk'>@@ -123,9 +119,8 @@ int iwpm_register_pid(struct iwpm_dev_data *pm_msg, u8 nl_client)</div><div class='ctx'> 	ret = iwpm_wait_complete_req(nlmsg_request);</div><div class='ctx'> 	return ret;</div><div class='ctx'> pid_query_error:</div><div class='del'>-	pr_info("%s: %s (client = %d)\n", __func__, err_str, nl_client);</div><div class='del'>-	if (skb)</div><div class='del'>-		dev_kfree_skb(skb);</div><div class='add'>+	pr_info("%s: %s (client = %u)\n", __func__, err_str, nl_client);</div><div class='add'>+	dev_kfree_skb(skb);</div><div class='ctx'> 	if (nlmsg_request)</div><div class='ctx'> 		iwpm_free_nlmsg_request(&amp;nlmsg_request-&gt;kref);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -154,10 +149,6 @@ int iwpm_add_mapping(struct iwpm_sa_data *pm_msg, u8 nl_client)</div><div class='ctx'> 	const char *err_str = "";</div><div class='ctx'> 	int ret = -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (!iwpm_valid_client(nl_client)) {</div><div class='del'>-		err_str = "Invalid port mapper client";</div><div class='del'>-		goto add_mapping_error;</div><div class='del'>-	}</div><div class='ctx'> 	if (!iwpm_valid_pid())</div><div class='ctx'> 		return 0;</div><div class='ctx'> 	if (!iwpm_check_registration(nl_client, IWPM_REG_VALID)) {</div><div class='hunk'>@@ -202,7 +193,7 @@ int iwpm_add_mapping(struct iwpm_sa_data *pm_msg, u8 nl_client)</div><div class='ctx'> 	nlmsg_end(skb, nlh);</div><div class='ctx'> 	nlmsg_request-&gt;req_buffer = pm_msg;</div><div class='ctx'> </div><div class='del'>-	ret = rdma_nl_unicast_wait(skb, iwpm_user_pid);</div><div class='add'>+	ret = rdma_nl_unicast_wait(&amp;init_net, skb, iwpm_user_pid);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		skb = NULL; /* skb is freed in the netlink send-op handling */</div><div class='ctx'> 		iwpm_user_pid = IWPM_PID_UNDEFINED;</div><div class='hunk'>@@ -212,10 +203,9 @@ int iwpm_add_mapping(struct iwpm_sa_data *pm_msg, u8 nl_client)</div><div class='ctx'> 	ret = iwpm_wait_complete_req(nlmsg_request);</div><div class='ctx'> 	return ret;</div><div class='ctx'> add_mapping_error:</div><div class='del'>-	pr_info("%s: %s (client = %d)\n", __func__, err_str, nl_client);</div><div class='add'>+	pr_info("%s: %s (client = %u)\n", __func__, err_str, nl_client);</div><div class='ctx'> add_mapping_error_nowarn:</div><div class='del'>-	if (skb)</div><div class='del'>-		dev_kfree_skb(skb);</div><div class='add'>+	dev_kfree_skb(skb);</div><div class='ctx'> 	if (nlmsg_request)</div><div class='ctx'> 		iwpm_free_nlmsg_request(&amp;nlmsg_request-&gt;kref);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -242,10 +232,6 @@ int iwpm_add_and_query_mapping(struct iwpm_sa_data *pm_msg, u8 nl_client)</div><div class='ctx'> 	const char *err_str = "";</div><div class='ctx'> 	int ret = -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (!iwpm_valid_client(nl_client)) {</div><div class='del'>-		err_str = "Invalid port mapper client";</div><div class='del'>-		goto query_mapping_error;</div><div class='del'>-	}</div><div class='ctx'> 	if (!iwpm_valid_pid())</div><div class='ctx'> 		return 0;</div><div class='ctx'> 	if (!iwpm_check_registration(nl_client, IWPM_REG_VALID)) {</div><div class='hunk'>@@ -297,7 +283,7 @@ int iwpm_add_and_query_mapping(struct iwpm_sa_data *pm_msg, u8 nl_client)</div><div class='ctx'> 	nlmsg_end(skb, nlh);</div><div class='ctx'> 	nlmsg_request-&gt;req_buffer = pm_msg;</div><div class='ctx'> </div><div class='del'>-	ret = rdma_nl_unicast_wait(skb, iwpm_user_pid);</div><div class='add'>+	ret = rdma_nl_unicast_wait(&amp;init_net, skb, iwpm_user_pid);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		skb = NULL; /* skb is freed in the netlink send-op handling */</div><div class='ctx'> 		err_str = "Unable to send a nlmsg";</div><div class='hunk'>@@ -306,10 +292,9 @@ int iwpm_add_and_query_mapping(struct iwpm_sa_data *pm_msg, u8 nl_client)</div><div class='ctx'> 	ret = iwpm_wait_complete_req(nlmsg_request);</div><div class='ctx'> 	return ret;</div><div class='ctx'> query_mapping_error:</div><div class='del'>-	pr_info("%s: %s (client = %d)\n", __func__, err_str, nl_client);</div><div class='add'>+	pr_info("%s: %s (client = %u)\n", __func__, err_str, nl_client);</div><div class='ctx'> query_mapping_error_nowarn:</div><div class='del'>-	if (skb)</div><div class='del'>-		dev_kfree_skb(skb);</div><div class='add'>+	dev_kfree_skb(skb);</div><div class='ctx'> 	if (nlmsg_request)</div><div class='ctx'> 		iwpm_free_nlmsg_request(&amp;nlmsg_request-&gt;kref);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -334,10 +319,6 @@ int iwpm_remove_mapping(struct sockaddr_storage *local_addr, u8 nl_client)</div><div class='ctx'> 	const char *err_str = "";</div><div class='ctx'> 	int ret = -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (!iwpm_valid_client(nl_client)) {</div><div class='del'>-		err_str = "Invalid port mapper client";</div><div class='del'>-		goto remove_mapping_error;</div><div class='del'>-	}</div><div class='ctx'> 	if (!iwpm_valid_pid())</div><div class='ctx'> 		return 0;</div><div class='ctx'> 	if (iwpm_check_registration(nl_client, IWPM_REG_UNDEF)) {</div><div class='hunk'>@@ -364,7 +345,7 @@ int iwpm_remove_mapping(struct sockaddr_storage *local_addr, u8 nl_client)</div><div class='ctx'> </div><div class='ctx'> 	nlmsg_end(skb, nlh);</div><div class='ctx'> </div><div class='del'>-	ret = rdma_nl_unicast_wait(skb, iwpm_user_pid);</div><div class='add'>+	ret = rdma_nl_unicast_wait(&amp;init_net, skb, iwpm_user_pid);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		skb = NULL; /* skb is freed in the netlink send-op handling */</div><div class='ctx'> 		iwpm_user_pid = IWPM_PID_UNDEFINED;</div><div class='hunk'>@@ -375,7 +356,7 @@ int iwpm_remove_mapping(struct sockaddr_storage *local_addr, u8 nl_client)</div><div class='ctx'> 			"remove_mapping: Local sockaddr:");</div><div class='ctx'> 	return 0;</div><div class='ctx'> remove_mapping_error:</div><div class='del'>-	pr_info("%s: %s (client = %d)\n", __func__, err_str, nl_client);</div><div class='add'>+	pr_info("%s: %s (client = %u)\n", __func__, err_str, nl_client);</div><div class='ctx'> 	if (skb)</div><div class='ctx'> 		dev_kfree_skb_any(skb);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -395,7 +376,7 @@ static const struct nla_policy resp_reg_policy[IWPM_NLA_RREG_PID_MAX] = {</div><div class='ctx'> /**</div><div class='ctx'>  * iwpm_register_pid_cb - Process the port mapper response to</div><div class='ctx'>  *                        iwpm_register_pid query</div><div class='del'>- * @skb:</div><div class='add'>+ * @skb: The socket buffer</div><div class='ctx'>  * @cb: Contains the received message (payload and netlink header)</div><div class='ctx'>  *</div><div class='ctx'>  * If successful, the function receives the userspace port mapper pid</div><div class='hunk'>@@ -434,7 +415,7 @@ int iwpm_register_pid_cb(struct sk_buff *skb, struct netlink_callback *cb)</div><div class='ctx'> 			strcmp(iwpm_ulib_name, iwpm_name) ||</div><div class='ctx'> 			iwpm_version &lt; IWPM_UABI_VERSION_MIN) {</div><div class='ctx'> </div><div class='del'>-		pr_info("%s: Incorrect info (dev = %s name = %s version = %d)\n",</div><div class='add'>+		pr_info("%s: Incorrect info (dev = %s name = %s version = %u)\n",</div><div class='ctx'> 				__func__, dev_name, iwpm_name, iwpm_version);</div><div class='ctx'> 		nlmsg_request-&gt;err_code = IWPM_USER_LIB_INFO_ERR;</div><div class='ctx'> 		goto register_pid_response_exit;</div><div class='hunk'>@@ -442,13 +423,12 @@ int iwpm_register_pid_cb(struct sk_buff *skb, struct netlink_callback *cb)</div><div class='ctx'> 	iwpm_user_pid = cb-&gt;nlh-&gt;nlmsg_pid;</div><div class='ctx'> 	iwpm_ulib_version = iwpm_version;</div><div class='ctx'> 	if (iwpm_ulib_version &lt; IWPM_UABI_VERSION)</div><div class='del'>-		pr_warn_once("%s: Down level iwpmd/pid %u.  Continuing...",</div><div class='add'>+		pr_warn_once("%s: Down level iwpmd/pid %d.  Continuing...",</div><div class='ctx'> 			__func__, iwpm_user_pid);</div><div class='ctx'> 	atomic_set(&amp;echo_nlmsg_seq, cb-&gt;nlh-&gt;nlmsg_seq);</div><div class='ctx'> 	pr_debug("%s: iWarp Port Mapper (pid = %d) is available!\n",</div><div class='ctx'> 			__func__, iwpm_user_pid);</div><div class='del'>-	if (iwpm_valid_client(nl_client))</div><div class='del'>-		iwpm_set_registration(nl_client, IWPM_REG_VALID);</div><div class='add'>+	iwpm_set_registration(nl_client, IWPM_REG_VALID);</div><div class='ctx'> register_pid_response_exit:</div><div class='ctx'> 	nlmsg_request-&gt;request_done = 1;</div><div class='ctx'> 	/* always for found nlmsg_request */</div><div class='hunk'>@@ -471,7 +451,7 @@ static const struct nla_policy resp_add_policy[IWPM_NLA_RMANAGE_MAPPING_MAX] = {</div><div class='ctx'> /**</div><div class='ctx'>  * iwpm_add_mapping_cb - Process the port mapper response to</div><div class='ctx'>  *                       iwpm_add_mapping request</div><div class='del'>- * @skb:</div><div class='add'>+ * @skb: The socket buffer</div><div class='ctx'>  * @cb: Contains the received message (payload and netlink header)</div><div class='ctx'>  */</div><div class='ctx'> int iwpm_add_mapping_cb(struct sk_buff *skb, struct netlink_callback *cb)</div><div class='hunk'>@@ -531,7 +511,8 @@ add_mapping_response_exit:</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* netlink attribute policy for the response to add and query mapping request</div><div class='del'>- * and response with remote address info */</div><div class='add'>+ * and response with remote address info</div><div class='add'>+ */</div><div class='ctx'> static const struct nla_policy resp_query_policy[IWPM_NLA_RQUERY_MAPPING_MAX] = {</div><div class='ctx'> 	[IWPM_NLA_RQUERY_MAPPING_SEQ]     = { .type = NLA_U32 },</div><div class='ctx'> 	[IWPM_NLA_RQUERY_LOCAL_ADDR]      = {</div><div class='hunk'>@@ -548,7 +529,7 @@ static const struct nla_policy resp_query_policy[IWPM_NLA_RQUERY_MAPPING_MAX] =</div><div class='ctx'> /**</div><div class='ctx'>  * iwpm_add_and_query_mapping_cb - Process the port mapper response to</div><div class='ctx'>  *                                 iwpm_add_and_query_mapping request</div><div class='del'>- * @skb:</div><div class='add'>+ * @skb: The socket buffer</div><div class='ctx'>  * @cb: Contains the received message (payload and netlink header)</div><div class='ctx'>  */</div><div class='ctx'> int iwpm_add_and_query_mapping_cb(struct sk_buff *skb,</div><div class='hunk'>@@ -630,7 +611,7 @@ query_mapping_response_exit:</div><div class='ctx'> /**</div><div class='ctx'>  * iwpm_remote_info_cb - Process remote connecting peer address info, which</div><div class='ctx'>  *                       the port mapper has received from the connecting peer</div><div class='del'>- * @skb:</div><div class='add'>+ * @skb: The socket buffer</div><div class='ctx'>  * @cb: Contains the received message (payload and netlink header)</div><div class='ctx'>  *</div><div class='ctx'>  * Stores the IPv4/IPv6 address info in a hash table</div><div class='hunk'>@@ -651,11 +632,6 @@ int iwpm_remote_info_cb(struct sk_buff *skb, struct netlink_callback *cb)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='ctx'> 	nl_client = RDMA_NL_GET_CLIENT(cb-&gt;nlh-&gt;nlmsg_type);</div><div class='del'>-	if (!iwpm_valid_client(nl_client)) {</div><div class='del'>-		pr_info("%s: Invalid port mapper client = %d\n",</div><div class='del'>-				__func__, nl_client);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='ctx'> 	atomic_set(&amp;echo_nlmsg_seq, cb-&gt;nlh-&gt;nlmsg_seq);</div><div class='ctx'> </div><div class='ctx'> 	local_sockaddr = (struct sockaddr_storage *)</div><div class='hunk'>@@ -709,7 +685,7 @@ static const struct nla_policy resp_mapinfo_policy[IWPM_NLA_MAPINFO_REQ_MAX] = {</div><div class='ctx'> /**</div><div class='ctx'>  * iwpm_mapping_info_cb - Process a notification that the userspace</div><div class='ctx'>  *                        port mapper daemon is started</div><div class='del'>- * @skb:</div><div class='add'>+ * @skb: The socket buffer</div><div class='ctx'>  * @cb: Contains the received message (payload and netlink header)</div><div class='ctx'>  *</div><div class='ctx'>  * Using the received port mapper pid, send all the local mapping</div><div class='hunk'>@@ -733,22 +709,17 @@ int iwpm_mapping_info_cb(struct sk_buff *skb, struct netlink_callback *cb)</div><div class='ctx'> 	iwpm_version = nla_get_u16(nltb[IWPM_NLA_MAPINFO_ULIB_VER]);</div><div class='ctx'> 	if (strcmp(iwpm_ulib_name, iwpm_name) ||</div><div class='ctx'> 			iwpm_version &lt; IWPM_UABI_VERSION_MIN) {</div><div class='del'>-		pr_info("%s: Invalid port mapper name = %s version = %d\n",</div><div class='add'>+		pr_info("%s: Invalid port mapper name = %s version = %u\n",</div><div class='ctx'> 				__func__, iwpm_name, iwpm_version);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> 	nl_client = RDMA_NL_GET_CLIENT(cb-&gt;nlh-&gt;nlmsg_type);</div><div class='del'>-	if (!iwpm_valid_client(nl_client)) {</div><div class='del'>-		pr_info("%s: Invalid port mapper client = %d\n",</div><div class='del'>-				__func__, nl_client);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='ctx'> 	iwpm_set_registration(nl_client, IWPM_REG_INCOMPL);</div><div class='ctx'> 	atomic_set(&amp;echo_nlmsg_seq, cb-&gt;nlh-&gt;nlmsg_seq);</div><div class='ctx'> 	iwpm_user_pid = cb-&gt;nlh-&gt;nlmsg_pid;</div><div class='ctx'> </div><div class='ctx'> 	if (iwpm_ulib_version &lt; IWPM_UABI_VERSION)</div><div class='del'>-		pr_warn_once("%s: Down level iwpmd/pid %u.  Continuing...",</div><div class='add'>+		pr_warn_once("%s: Down level iwpmd/pid %d.  Continuing...",</div><div class='ctx'> 			__func__, iwpm_user_pid);</div><div class='ctx'> </div><div class='ctx'> 	if (!iwpm_mapinfo_available())</div><div class='hunk'>@@ -769,7 +740,7 @@ static const struct nla_policy ack_mapinfo_policy[IWPM_NLA_MAPINFO_NUM_MAX] = {</div><div class='ctx'> /**</div><div class='ctx'>  * iwpm_ack_mapping_info_cb - Process the port mapper ack for</div><div class='ctx'>  *                            the provided local mapping info records</div><div class='del'>- * @skb:</div><div class='add'>+ * @skb: The socket buffer</div><div class='ctx'>  * @cb: Contains the received message (payload and netlink header)</div><div class='ctx'>  */</div><div class='ctx'> int iwpm_ack_mapping_info_cb(struct sk_buff *skb, struct netlink_callback *cb)</div><div class='hunk'>@@ -799,7 +770,7 @@ static const struct nla_policy map_error_policy[IWPM_NLA_ERR_MAX] = {</div><div class='ctx'> /**</div><div class='ctx'>  * iwpm_mapping_error_cb - Process port mapper notification for error</div><div class='ctx'>  *</div><div class='del'>- * @skb:</div><div class='add'>+ * @skb: The socket buffer</div><div class='ctx'>  * @cb: Contains the received message (payload and netlink header)</div><div class='ctx'>  */</div><div class='ctx'> int iwpm_mapping_error_cb(struct sk_buff *skb, struct netlink_callback *cb)</div><div class='hunk'>@@ -844,7 +815,7 @@ static const struct nla_policy hello_policy[IWPM_NLA_HELLO_MAX] = {</div><div class='ctx'> /**</div><div class='ctx'>  * iwpm_hello_cb - Process a hello message from iwpmd</div><div class='ctx'>  *</div><div class='del'>- * @skb:</div><div class='add'>+ * @skb: The socket buffer</div><div class='ctx'>  * @cb: Contains the received message (payload and netlink header)</div><div class='ctx'>  *</div><div class='ctx'>  * Using the received port mapper pid, send the kernel's abi_version</div><div class='hunk'>@@ -865,11 +836,6 @@ int iwpm_hello_cb(struct sk_buff *skb, struct netlink_callback *cb)</div><div class='ctx'> 	}</div><div class='ctx'> 	abi_version = nla_get_u16(nltb[IWPM_NLA_HELLO_ABI_VERSION]);</div><div class='ctx'> 	nl_client = RDMA_NL_GET_CLIENT(cb-&gt;nlh-&gt;nlmsg_type);</div><div class='del'>-	if (!iwpm_valid_client(nl_client)) {</div><div class='del'>-		pr_info("%s: Invalid port mapper client = %d\n",</div><div class='del'>-				__func__, nl_client);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='ctx'> 	iwpm_set_registration(nl_client, IWPM_REG_INCOMPL);</div><div class='ctx'> 	atomic_set(&amp;echo_nlmsg_seq, cb-&gt;nlh-&gt;nlmsg_seq);</div><div class='ctx'> 	iwpm_ulib_version = min_t(u16, IWPM_UABI_VERSION, abi_version);</div><div class='head'>diff --git a/drivers/infiniband/core/iwpm_util.c b/drivers/infiniband/core/iwpm_util.c<br/>index 41929bb83739..eecb369898f5 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/iwpm_util.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/iwpm_util.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/iwpm_util.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/iwpm_util.c</a></div><div class='hunk'>@@ -48,7 +48,6 @@ static DEFINE_SPINLOCK(iwpm_mapinfo_lock);</div><div class='ctx'> static struct hlist_head *iwpm_reminfo_bucket;</div><div class='ctx'> static DEFINE_SPINLOCK(iwpm_reminfo_lock);</div><div class='ctx'> </div><div class='del'>-static DEFINE_MUTEX(iwpm_admin_lock);</div><div class='ctx'> static struct iwpm_admin_data iwpm_admin;</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='hunk'>@@ -59,35 +58,21 @@ static struct iwpm_admin_data iwpm_admin;</div><div class='ctx'>  */</div><div class='ctx'> int iwpm_init(u8 nl_client)</div><div class='ctx'> {</div><div class='del'>-	int ret = 0;</div><div class='del'>-	mutex_lock(&amp;iwpm_admin_lock);</div><div class='del'>-	if (atomic_read(&amp;iwpm_admin.refcount) == 0) {</div><div class='del'>-		iwpm_hash_bucket = kcalloc(IWPM_MAPINFO_HASH_SIZE,</div><div class='del'>-					   sizeof(struct hlist_head),</div><div class='del'>-					   GFP_KERNEL);</div><div class='del'>-		if (!iwpm_hash_bucket) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto init_exit;</div><div class='del'>-		}</div><div class='del'>-		iwpm_reminfo_bucket = kcalloc(IWPM_REMINFO_HASH_SIZE,</div><div class='del'>-					      sizeof(struct hlist_head),</div><div class='del'>-					      GFP_KERNEL);</div><div class='del'>-		if (!iwpm_reminfo_bucket) {</div><div class='del'>-			kfree(iwpm_hash_bucket);</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto init_exit;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	atomic_inc(&amp;iwpm_admin.refcount);</div><div class='del'>-init_exit:</div><div class='del'>-	mutex_unlock(&amp;iwpm_admin_lock);</div><div class='del'>-	if (!ret) {</div><div class='del'>-		iwpm_set_valid(nl_client, 1);</div><div class='del'>-		iwpm_set_registration(nl_client, IWPM_REG_UNDEF);</div><div class='del'>-		pr_debug("%s: Mapinfo and reminfo tables are created\n",</div><div class='del'>-				__func__);</div><div class='add'>+	iwpm_hash_bucket = kcalloc(IWPM_MAPINFO_HASH_SIZE,</div><div class='add'>+				   sizeof(struct hlist_head), GFP_KERNEL);</div><div class='add'>+	if (!iwpm_hash_bucket)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	iwpm_reminfo_bucket = kcalloc(IWPM_REMINFO_HASH_SIZE,</div><div class='add'>+				      sizeof(struct hlist_head), GFP_KERNEL);</div><div class='add'>+	if (!iwpm_reminfo_bucket) {</div><div class='add'>+		kfree(iwpm_hash_bucket);</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='del'>-	return ret;</div><div class='add'>+</div><div class='add'>+	iwpm_set_registration(nl_client, IWPM_REG_UNDEF);</div><div class='add'>+	pr_debug("%s: Mapinfo and reminfo tables are created\n", __func__);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void free_hash_bucket(void);</div><div class='hunk'>@@ -101,22 +86,9 @@ static void free_reminfo_bucket(void);</div><div class='ctx'>  */</div><div class='ctx'> int iwpm_exit(u8 nl_client)</div><div class='ctx'> {</div><div class='del'>-</div><div class='del'>-	if (!iwpm_valid_client(nl_client))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	mutex_lock(&amp;iwpm_admin_lock);</div><div class='del'>-	if (atomic_read(&amp;iwpm_admin.refcount) == 0) {</div><div class='del'>-		mutex_unlock(&amp;iwpm_admin_lock);</div><div class='del'>-		pr_err("%s Incorrect usage - negative refcount\n", __func__);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-	if (atomic_dec_and_test(&amp;iwpm_admin.refcount)) {</div><div class='del'>-		free_hash_bucket();</div><div class='del'>-		free_reminfo_bucket();</div><div class='del'>-		pr_debug("%s: Resources are destroyed\n", __func__);</div><div class='del'>-	}</div><div class='del'>-	mutex_unlock(&amp;iwpm_admin_lock);</div><div class='del'>-	iwpm_set_valid(nl_client, 0);</div><div class='add'>+	free_hash_bucket();</div><div class='add'>+	free_reminfo_bucket();</div><div class='add'>+	pr_debug("%s: Resources are destroyed\n", __func__);</div><div class='ctx'> 	iwpm_set_registration(nl_client, IWPM_REG_UNDEF);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -127,8 +99,8 @@ static struct hlist_head *get_mapinfo_hash_bucket(struct sockaddr_storage *,</div><div class='ctx'> /**</div><div class='ctx'>  * iwpm_create_mapinfo - Store local and mapped IPv4/IPv6 address</div><div class='ctx'>  *                       info in a hash table</div><div class='del'>- * @local_addr: Local ip/tcp address</div><div class='del'>- * @mapped_addr: Mapped local ip/tcp address</div><div class='add'>+ * @local_sockaddr: Local ip/tcp address</div><div class='add'>+ * @mapped_sockaddr: Mapped local ip/tcp address</div><div class='ctx'>  * @nl_client: The index of the netlink client</div><div class='ctx'>  * @map_flags: IWPM mapping flags</div><div class='ctx'>  */</div><div class='hunk'>@@ -141,8 +113,6 @@ int iwpm_create_mapinfo(struct sockaddr_storage *local_sockaddr,</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int ret = -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (!iwpm_valid_client(nl_client))</div><div class='del'>-		return ret;</div><div class='ctx'> 	map_info = kzalloc(sizeof(struct iwpm_mapping_info), GFP_KERNEL);</div><div class='ctx'> 	if (!map_info)</div><div class='ctx'> 		return -ENOMEM;</div><div class='hunk'>@@ -174,7 +144,7 @@ int iwpm_create_mapinfo(struct sockaddr_storage *local_sockaddr,</div><div class='ctx'> /**</div><div class='ctx'>  * iwpm_remove_mapinfo - Remove local and mapped IPv4/IPv6 address</div><div class='ctx'>  *                       info from the hash table</div><div class='del'>- * @local_addr: Local ip/tcp address</div><div class='add'>+ * @local_sockaddr: Local ip/tcp address</div><div class='ctx'>  * @mapped_local_addr: Mapped local ip/tcp address</div><div class='ctx'>  *</div><div class='ctx'>  * Returns err code if mapping info is not found in the hash table,</div><div class='hunk'>@@ -302,10 +272,6 @@ int iwpm_get_remote_info(struct sockaddr_storage *mapped_loc_addr,</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int ret = -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (!iwpm_valid_client(nl_client)) {</div><div class='del'>-		pr_info("%s: Invalid client = %d\n", __func__, nl_client);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='ctx'> 	spin_lock_irqsave(&amp;iwpm_reminfo_lock, flags);</div><div class='ctx'> 	if (iwpm_reminfo_bucket) {</div><div class='ctx'> 		hash_bucket_head = get_reminfo_hash_bucket(</div><div class='hunk'>@@ -341,7 +307,7 @@ get_remote_info_exit:</div><div class='ctx'> struct iwpm_nlmsg_request *iwpm_get_nlmsg_request(__u32 nlmsg_seq,</div><div class='ctx'> 					u8 nl_client, gfp_t gfp)</div><div class='ctx'> {</div><div class='del'>-	struct iwpm_nlmsg_request *nlmsg_request = NULL;</div><div class='add'>+	struct iwpm_nlmsg_request *nlmsg_request;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> </div><div class='ctx'> 	nlmsg_request = kzalloc(sizeof(struct iwpm_nlmsg_request), gfp);</div><div class='hunk'>@@ -420,16 +386,6 @@ int iwpm_get_nlmsg_seq(void)</div><div class='ctx'> 	return atomic_inc_return(&amp;iwpm_admin.nlmsg_seq);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int iwpm_valid_client(u8 nl_client)</div><div class='del'>-{</div><div class='del'>-	return iwpm_admin.client_list[nl_client];</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void iwpm_set_valid(u8 nl_client, int valid)</div><div class='del'>-{</div><div class='del'>-	iwpm_admin.client_list[nl_client] = valid;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> /* valid client */</div><div class='ctx'> u32 iwpm_get_registration(u8 nl_client)</div><div class='ctx'> {</div><div class='hunk'>@@ -645,18 +601,17 @@ static int send_mapinfo_num(u32 mapping_num, u8 nl_client, int iwpm_pid)</div><div class='ctx'> </div><div class='ctx'> 	nlmsg_end(skb, nlh);</div><div class='ctx'> </div><div class='del'>-	ret = rdma_nl_unicast(skb, iwpm_pid);</div><div class='add'>+	ret = rdma_nl_unicast(&amp;init_net, skb, iwpm_pid);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		skb = NULL;</div><div class='ctx'> 		err_str = "Unable to send a nlmsg";</div><div class='ctx'> 		goto mapinfo_num_error;</div><div class='ctx'> 	}</div><div class='del'>-	pr_debug("%s: Sent mapping number = %d\n", __func__, mapping_num);</div><div class='add'>+	pr_debug("%s: Sent mapping number = %u\n", __func__, mapping_num);</div><div class='ctx'> 	return 0;</div><div class='ctx'> mapinfo_num_error:</div><div class='ctx'> 	pr_info("%s: %s\n", __func__, err_str);</div><div class='del'>-	if (skb)</div><div class='del'>-		dev_kfree_skb(skb);</div><div class='add'>+	dev_kfree_skb(skb);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -674,7 +629,7 @@ static int send_nlmsg_done(struct sk_buff *skb, u8 nl_client, int iwpm_pid)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='ctx'> 	nlh-&gt;nlmsg_type = NLMSG_DONE;</div><div class='del'>-	ret = rdma_nl_unicast(skb, iwpm_pid);</div><div class='add'>+	ret = rdma_nl_unicast(&amp;init_net, skb, iwpm_pid);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		pr_warn("%s Unable to send a nlmsg\n", __func__);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -778,8 +733,7 @@ send_mapping_info_unlock:</div><div class='ctx'> send_mapping_info_exit:</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		pr_warn("%s: %s (ret = %d)\n", __func__, err_str, ret);</div><div class='del'>-		if (skb)</div><div class='del'>-			dev_kfree_skb(skb);</div><div class='add'>+		dev_kfree_skb(skb);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> 	send_nlmsg_done(skb, nl_client, iwpm_pid);</div><div class='hunk'>@@ -808,7 +762,7 @@ int iwpm_send_hello(u8 nl_client, int iwpm_pid, u16 abi_version)</div><div class='ctx'> {</div><div class='ctx'> 	struct sk_buff *skb = NULL;</div><div class='ctx'> 	struct nlmsghdr *nlh;</div><div class='del'>-	const char *err_str = "";</div><div class='add'>+	const char *err_str;</div><div class='ctx'> 	int ret = -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	skb = iwpm_create_nlmsg(RDMA_NL_IWPM_HELLO, &amp;nlh, nl_client);</div><div class='hunk'>@@ -824,7 +778,7 @@ int iwpm_send_hello(u8 nl_client, int iwpm_pid, u16 abi_version)</div><div class='ctx'> 		goto hello_num_error;</div><div class='ctx'> 	nlmsg_end(skb, nlh);</div><div class='ctx'> </div><div class='del'>-	ret = rdma_nl_unicast(skb, iwpm_pid);</div><div class='add'>+	ret = rdma_nl_unicast(&amp;init_net, skb, iwpm_pid);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		skb = NULL;</div><div class='ctx'> 		err_str = "Unable to send a nlmsg";</div><div class='hunk'>@@ -834,7 +788,6 @@ int iwpm_send_hello(u8 nl_client, int iwpm_pid, u16 abi_version)</div><div class='ctx'> 	return 0;</div><div class='ctx'> hello_num_error:</div><div class='ctx'> 	pr_info("%s: %s\n", __func__, err_str);</div><div class='del'>-	if (skb)</div><div class='del'>-		dev_kfree_skb(skb);</div><div class='add'>+	dev_kfree_skb(skb);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/core/iwpm_util.h b/drivers/infiniband/core/iwpm_util.h<br/>index 7e2bcc72f66c..d6fc8402158a 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/iwpm_util.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/iwpm_util.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/iwpm_util.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/iwpm_util.h</a></div><div class='hunk'>@@ -33,7 +33,6 @@</div><div class='ctx'> #ifndef _IWPM_UTIL_H</div><div class='ctx'> #define _IWPM_UTIL_H</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='ctx'> #include &lt;linux/io.h&gt;</div><div class='ctx'> #include &lt;linux/in.h&gt;</div><div class='ctx'> #include &lt;linux/in6.h&gt;</div><div class='hunk'>@@ -90,9 +89,7 @@ struct iwpm_remote_info {</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct iwpm_admin_data {</div><div class='del'>-	atomic_t refcount;</div><div class='ctx'> 	atomic_t nlmsg_seq;</div><div class='del'>-	int      client_list[RDMA_NL_NUM_CLIENTS];</div><div class='ctx'> 	u32      reg_list[RDMA_NL_NUM_CLIENTS];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -141,29 +138,13 @@ int iwpm_wait_complete_req(struct iwpm_nlmsg_request *nlmsg_request);</div><div class='ctx'> int iwpm_get_nlmsg_seq(void);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * iwpm_add_reminfo - Add remote address info of the connecting peer</div><div class='add'>+ * iwpm_add_remote_info - Add remote address info of the connecting peer</div><div class='ctx'>  *                    to the remote info hash table</div><div class='ctx'>  * @reminfo: The remote info to be added</div><div class='ctx'>  */</div><div class='ctx'> void iwpm_add_remote_info(struct iwpm_remote_info *reminfo);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * iwpm_valid_client - Check if the port mapper client is valid</div><div class='del'>- * @nl_client: The index of the netlink client</div><div class='del'>- *</div><div class='del'>- * Valid clients need to call iwpm_init() before using</div><div class='del'>- * the port mapper</div><div class='del'>- */</div><div class='del'>-int iwpm_valid_client(u8 nl_client);</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * iwpm_set_valid - Set the port mapper client to valid or not</div><div class='del'>- * @nl_client: The index of the netlink client</div><div class='del'>- * @valid: 1 if valid or 0 if invalid</div><div class='del'>- */</div><div class='del'>-void iwpm_set_valid(u8 nl_client, int valid);</div><div class='del'>-</div><div class='del'>-/**</div><div class='ctx'>  * iwpm_check_registration - Check if the client registration</div><div class='ctx'>  *			      matches the given one</div><div class='ctx'>  * @nl_client: The index of the netlink client</div><div class='hunk'>@@ -183,7 +164,7 @@ u32 iwpm_check_registration(u8 nl_client, u32 reg);</div><div class='ctx'> void iwpm_set_registration(u8 nl_client, u32 reg);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * iwpm_get_registration</div><div class='add'>+ * iwpm_get_registration - Get the client registration</div><div class='ctx'>  * @nl_client: The index of the netlink client</div><div class='ctx'>  *</div><div class='ctx'>  * Returns the client registration type</div><div class='hunk'>@@ -210,8 +191,10 @@ int iwpm_mapinfo_available(void);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * iwpm_compare_sockaddr - Compare two sockaddr storage structs</div><div class='add'>+ * @a_sockaddr: first sockaddr to compare</div><div class='add'>+ * @b_sockaddr: second sockaddr to compare</div><div class='ctx'>  *</div><div class='del'>- * Returns 0 if they are holding the same ip/tcp address info,</div><div class='add'>+ * Return: 0 if they are holding the same ip/tcp address info,</div><div class='ctx'>  * otherwise returns 1</div><div class='ctx'>  */</div><div class='ctx'> int iwpm_compare_sockaddr(struct sockaddr_storage *a_sockaddr,</div><div class='hunk'>@@ -272,6 +255,7 @@ void iwpm_print_sockaddr(struct sockaddr_storage *sockaddr, char *msg);</div><div class='ctx'>  * iwpm_send_hello - Send hello response to iwpmd</div><div class='ctx'>  *</div><div class='ctx'>  * @nl_client: The index of the netlink client</div><div class='add'>+ * @iwpm_pid: The pid of the user space port mapper</div><div class='ctx'>  * @abi_version: The kernel's abi_version</div><div class='ctx'>  *</div><div class='ctx'>  * Returns 0 on success or a negative error code</div><div class='head'>diff --git a/drivers/infiniband/core/lag.c b/drivers/infiniband/core/lag.c<br/>new file mode 100644<br/>index 000000000000..8fd80adfe833<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/lag.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/lag.c</a></div><div class='hunk'>@@ -0,0 +1,135 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2020 Mellanox Technologies. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+#include &lt;rdma/ib_cache.h&gt;</div><div class='add'>+#include &lt;rdma/lag.h&gt;</div><div class='add'>+</div><div class='add'>+static struct sk_buff *rdma_build_skb(struct net_device *netdev,</div><div class='add'>+				      struct rdma_ah_attr *ah_attr,</div><div class='add'>+				      gfp_t flags)</div><div class='add'>+{</div><div class='add'>+	struct ipv6hdr *ip6h;</div><div class='add'>+	struct sk_buff *skb;</div><div class='add'>+	struct ethhdr *eth;</div><div class='add'>+	struct iphdr *iph;</div><div class='add'>+	struct udphdr *uh;</div><div class='add'>+	u8 smac[ETH_ALEN];</div><div class='add'>+	bool is_ipv4;</div><div class='add'>+	int hdr_len;</div><div class='add'>+</div><div class='add'>+	is_ipv4 = ipv6_addr_v4mapped((struct in6_addr *)ah_attr-&gt;grh.dgid.raw);</div><div class='add'>+	hdr_len = ETH_HLEN + sizeof(struct udphdr) + LL_RESERVED_SPACE(netdev);</div><div class='add'>+	hdr_len += is_ipv4 ? sizeof(struct iphdr) : sizeof(struct ipv6hdr);</div><div class='add'>+</div><div class='add'>+	skb = alloc_skb(hdr_len, flags);</div><div class='add'>+	if (!skb)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	skb-&gt;dev = netdev;</div><div class='add'>+	skb_reserve(skb, hdr_len);</div><div class='add'>+	skb_push(skb, sizeof(struct udphdr));</div><div class='add'>+	skb_reset_transport_header(skb);</div><div class='add'>+	uh = udp_hdr(skb);</div><div class='add'>+	uh-&gt;source =</div><div class='add'>+		htons(rdma_flow_label_to_udp_sport(ah_attr-&gt;grh.flow_label));</div><div class='add'>+	uh-&gt;dest = htons(ROCE_V2_UDP_DPORT);</div><div class='add'>+	uh-&gt;len = htons(sizeof(struct udphdr));</div><div class='add'>+</div><div class='add'>+	if (is_ipv4) {</div><div class='add'>+		skb_push(skb, sizeof(struct iphdr));</div><div class='add'>+		skb_reset_network_header(skb);</div><div class='add'>+		iph = ip_hdr(skb);</div><div class='add'>+		iph-&gt;frag_off = 0;</div><div class='add'>+		iph-&gt;version = 4;</div><div class='add'>+		iph-&gt;protocol = IPPROTO_UDP;</div><div class='add'>+		iph-&gt;ihl = 0x5;</div><div class='add'>+		iph-&gt;tot_len = htons(sizeof(struct udphdr) + sizeof(struct</div><div class='add'>+								    iphdr));</div><div class='add'>+		memcpy(&amp;iph-&gt;saddr, ah_attr-&gt;grh.sgid_attr-&gt;gid.raw + 12,</div><div class='add'>+		       sizeof(struct in_addr));</div><div class='add'>+		memcpy(&amp;iph-&gt;daddr, ah_attr-&gt;grh.dgid.raw + 12,</div><div class='add'>+		       sizeof(struct in_addr));</div><div class='add'>+	} else {</div><div class='add'>+		skb_push(skb, sizeof(struct ipv6hdr));</div><div class='add'>+		skb_reset_network_header(skb);</div><div class='add'>+		ip6h = ipv6_hdr(skb);</div><div class='add'>+		ip6h-&gt;version = 6;</div><div class='add'>+		ip6h-&gt;nexthdr = IPPROTO_UDP;</div><div class='add'>+		memcpy(&amp;ip6h-&gt;flow_lbl, &amp;ah_attr-&gt;grh.flow_label,</div><div class='add'>+		       sizeof(*ip6h-&gt;flow_lbl));</div><div class='add'>+		memcpy(&amp;ip6h-&gt;saddr, ah_attr-&gt;grh.sgid_attr-&gt;gid.raw,</div><div class='add'>+		       sizeof(struct in6_addr));</div><div class='add'>+		memcpy(&amp;ip6h-&gt;daddr, ah_attr-&gt;grh.dgid.raw,</div><div class='add'>+		       sizeof(struct in6_addr));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	skb_push(skb, sizeof(struct ethhdr));</div><div class='add'>+	skb_reset_mac_header(skb);</div><div class='add'>+	eth = eth_hdr(skb);</div><div class='add'>+	skb-&gt;protocol = eth-&gt;h_proto = htons(is_ipv4 ? ETH_P_IP : ETH_P_IPV6);</div><div class='add'>+	rdma_read_gid_l2_fields(ah_attr-&gt;grh.sgid_attr, NULL, smac);</div><div class='add'>+	memcpy(eth-&gt;h_source, smac, ETH_ALEN);</div><div class='add'>+	memcpy(eth-&gt;h_dest, ah_attr-&gt;roce.dmac, ETH_ALEN);</div><div class='add'>+</div><div class='add'>+	return skb;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct net_device *rdma_get_xmit_slave_udp(struct ib_device *device,</div><div class='add'>+						  struct net_device *master,</div><div class='add'>+						  struct rdma_ah_attr *ah_attr,</div><div class='add'>+						  gfp_t flags)</div><div class='add'>+{</div><div class='add'>+	struct net_device *slave;</div><div class='add'>+	struct sk_buff *skb;</div><div class='add'>+</div><div class='add'>+	skb = rdma_build_skb(master, ah_attr, flags);</div><div class='add'>+	if (!skb)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	slave = netdev_get_xmit_slave(master, skb,</div><div class='add'>+				      !!(device-&gt;lag_flags &amp;</div><div class='add'>+					 RDMA_LAG_FLAGS_HASH_ALL_SLAVES));</div><div class='add'>+	dev_hold(slave);</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+	kfree_skb(skb);</div><div class='add'>+	return slave;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void rdma_lag_put_ah_roce_slave(struct net_device *xmit_slave)</div><div class='add'>+{</div><div class='add'>+	dev_put(xmit_slave);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct net_device *rdma_lag_get_ah_roce_slave(struct ib_device *device,</div><div class='add'>+					      struct rdma_ah_attr *ah_attr,</div><div class='add'>+					      gfp_t flags)</div><div class='add'>+{</div><div class='add'>+	struct net_device *slave = NULL;</div><div class='add'>+	struct net_device *master;</div><div class='add'>+</div><div class='add'>+	if (!(ah_attr-&gt;type == RDMA_AH_ATTR_TYPE_ROCE &amp;&amp;</div><div class='add'>+	      ah_attr-&gt;grh.sgid_attr-&gt;gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP &amp;&amp;</div><div class='add'>+	      ah_attr-&gt;grh.flow_label))</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	master = rdma_read_gid_attr_ndev_rcu(ah_attr-&gt;grh.sgid_attr);</div><div class='add'>+	if (IS_ERR(master)) {</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+		return master;</div><div class='add'>+	}</div><div class='add'>+	dev_hold(master);</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+</div><div class='add'>+	if (!netif_is_bond_master(master))</div><div class='add'>+		goto put;</div><div class='add'>+</div><div class='add'>+	slave = rdma_get_xmit_slave_udp(device, master, ah_attr, flags);</div><div class='add'>+put:</div><div class='add'>+	dev_put(master);</div><div class='add'>+	return slave;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c<br/>index cc99479b2c09..8f26bfb69586 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/mad.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/mad.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/mad.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/mad.c</a></div><div class='hunk'>@@ -59,9 +59,6 @@ static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='ctx'> 			  struct ib_mad_qp_info *qp_info,</div><div class='ctx'> 			  struct trace_event_raw_ib_mad_send_template *entry)</div><div class='ctx'> {</div><div class='del'>-	u16 pkey;</div><div class='del'>-	struct ib_device *dev = qp_info-&gt;port_priv-&gt;device;</div><div class='del'>-	u8 pnum = qp_info-&gt;port_priv-&gt;port_num;</div><div class='ctx'> 	struct ib_ud_wr *wr = &amp;mad_send_wr-&gt;send_wr;</div><div class='ctx'> 	struct rdma_ah_attr attr = {};</div><div class='ctx'> </div><div class='hunk'>@@ -69,8 +66,6 @@ static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='ctx'> </div><div class='ctx'> 	/* These are common */</div><div class='ctx'> 	entry-&gt;sl = attr.sl;</div><div class='del'>-	ib_query_pkey(dev, pnum, wr-&gt;pkey_index, &amp;pkey);</div><div class='del'>-	entry-&gt;pkey = pkey;</div><div class='ctx'> 	entry-&gt;rqpn = wr-&gt;remote_qpn;</div><div class='ctx'> 	entry-&gt;rqkey = wr-&gt;remote_qkey;</div><div class='ctx'> 	entry-&gt;dlid = rdma_ah_get_dlid(&amp;attr);</div><div class='hunk'>@@ -85,7 +80,6 @@ MODULE_PARM_DESC(send_queue_size, "Size of send queue in number of work requests</div><div class='ctx'> module_param_named(recv_queue_size, mad_recvq_size, int, 0444);</div><div class='ctx'> MODULE_PARM_DESC(recv_queue_size, "Size of receive queue in number of work requests");</div><div class='ctx'> </div><div class='del'>-/* Client ID 0 is used for snoop-only clients */</div><div class='ctx'> static DEFINE_XARRAY_ALLOC1(ib_mad_clients);</div><div class='ctx'> static u32 ib_mad_client_next;</div><div class='ctx'> static struct list_head ib_mad_port_list;</div><div class='hunk'>@@ -119,7 +113,7 @@ static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc);</div><div class='ctx'>  * Assumes ib_mad_port_list_lock is being held</div><div class='ctx'>  */</div><div class='ctx'> static inline struct ib_mad_port_private *</div><div class='del'>-__ib_get_mad_port(struct ib_device *device, int port_num)</div><div class='add'>+__ib_get_mad_port(struct ib_device *device, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_mad_port_private *entry;</div><div class='ctx'> </div><div class='hunk'>@@ -135,7 +129,7 @@ __ib_get_mad_port(struct ib_device *device, int port_num)</div><div class='ctx'>  * for a device/port</div><div class='ctx'>  */</div><div class='ctx'> static inline struct ib_mad_port_private *</div><div class='del'>-ib_get_mad_port(struct ib_device *device, int port_num)</div><div class='add'>+ib_get_mad_port(struct ib_device *device, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_mad_port_private *entry;</div><div class='ctx'> 	unsigned long flags;</div><div class='hunk'>@@ -156,8 +150,7 @@ static inline u8 convert_mgmt_class(u8 mgmt_class)</div><div class='ctx'> </div><div class='ctx'> static int get_spl_qp_index(enum ib_qp_type qp_type)</div><div class='ctx'> {</div><div class='del'>-	switch (qp_type)</div><div class='del'>-	{</div><div class='add'>+	switch (qp_type) {</div><div class='ctx'> 	case IB_QPT_SMI:</div><div class='ctx'> 		return 0;</div><div class='ctx'> 	case IB_QPT_GSI:</div><div class='hunk'>@@ -217,13 +210,36 @@ int ib_response_mad(const struct ib_mad_hdr *hdr)</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_response_mad);</div><div class='ctx'> </div><div class='add'>+#define SOL_FC_MAX_DEFAULT_FRAC 4</div><div class='add'>+#define SOL_FC_MAX_SA_FRAC 32</div><div class='add'>+</div><div class='add'>+static int get_sol_fc_max_outstanding(struct ib_mad_reg_req *mad_reg_req)</div><div class='add'>+{</div><div class='add'>+	if (!mad_reg_req)</div><div class='add'>+		/* Send only agent */</div><div class='add'>+		return mad_recvq_size / SOL_FC_MAX_DEFAULT_FRAC;</div><div class='add'>+</div><div class='add'>+	switch (mad_reg_req-&gt;mgmt_class) {</div><div class='add'>+	case IB_MGMT_CLASS_CM:</div><div class='add'>+		return mad_recvq_size / SOL_FC_MAX_DEFAULT_FRAC;</div><div class='add'>+	case IB_MGMT_CLASS_SUBN_ADM:</div><div class='add'>+		return mad_recvq_size / SOL_FC_MAX_SA_FRAC;</div><div class='add'>+	case IB_MGMT_CLASS_SUBN_LID_ROUTED:</div><div class='add'>+	case IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE:</div><div class='add'>+		return min(mad_recvq_size, IB_MAD_QP_RECV_SIZE) /</div><div class='add'>+		       SOL_FC_MAX_DEFAULT_FRAC;</div><div class='add'>+	default:</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /*</div><div class='ctx'>  * ib_register_mad_agent - Register to send/receive MADs</div><div class='ctx'>  *</div><div class='ctx'>  * Context: Process context.</div><div class='ctx'>  */</div><div class='ctx'> struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,</div><div class='del'>-					   u8 port_num,</div><div class='add'>+					   u32 port_num,</div><div class='ctx'> 					   enum ib_qp_type qp_type,</div><div class='ctx'> 					   struct ib_mad_reg_req *mad_reg_req,</div><div class='ctx'> 					   u8 rmpp_version,</div><div class='hunk'>@@ -353,7 +369,7 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,</div><div class='ctx'> 	/* Validate device and port */</div><div class='ctx'> 	port_priv = ib_get_mad_port(device, port_num);</div><div class='ctx'> 	if (!port_priv) {</div><div class='del'>-		dev_dbg_ratelimited(&amp;device-&gt;dev, "%s: Invalid port %d\n",</div><div class='add'>+		dev_dbg_ratelimited(&amp;device-&gt;dev, "%s: Invalid port %u\n",</div><div class='ctx'> 				    __func__, port_num);</div><div class='ctx'> 		ret = ERR_PTR(-ENODEV);</div><div class='ctx'> 		goto error1;</div><div class='hunk'>@@ -398,13 +414,17 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,</div><div class='ctx'> 	spin_lock_init(&amp;mad_agent_priv-&gt;lock);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;mad_agent_priv-&gt;send_list);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;mad_agent_priv-&gt;wait_list);</div><div class='del'>-	INIT_LIST_HEAD(&amp;mad_agent_priv-&gt;done_list);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;mad_agent_priv-&gt;rmpp_list);</div><div class='add'>+	INIT_LIST_HEAD(&amp;mad_agent_priv-&gt;backlog_list);</div><div class='ctx'> 	INIT_DELAYED_WORK(&amp;mad_agent_priv-&gt;timed_work, timeout_sends);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;mad_agent_priv-&gt;local_list);</div><div class='ctx'> 	INIT_WORK(&amp;mad_agent_priv-&gt;local_work, local_completions);</div><div class='del'>-	atomic_set(&amp;mad_agent_priv-&gt;refcount, 1);</div><div class='add'>+	refcount_set(&amp;mad_agent_priv-&gt;refcount, 1);</div><div class='ctx'> 	init_completion(&amp;mad_agent_priv-&gt;comp);</div><div class='add'>+	mad_agent_priv-&gt;sol_fc_send_count = 0;</div><div class='add'>+	mad_agent_priv-&gt;sol_fc_wait_count = 0;</div><div class='add'>+	mad_agent_priv-&gt;sol_fc_max =</div><div class='add'>+		recv_handler ? get_sol_fc_max_outstanding(mad_reg_req) : 0;</div><div class='ctx'> </div><div class='ctx'> 	ret2 = ib_mad_agent_security_setup(&amp;mad_agent_priv-&gt;agent, qp_type);</div><div class='ctx'> 	if (ret2) {</div><div class='hunk'>@@ -483,141 +503,12 @@ error1:</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_register_mad_agent);</div><div class='ctx'> </div><div class='del'>-static inline int is_snooping_sends(int mad_snoop_flags)</div><div class='del'>-{</div><div class='del'>-	return (mad_snoop_flags &amp;</div><div class='del'>-		(/*IB_MAD_SNOOP_POSTED_SENDS |</div><div class='del'>-		 IB_MAD_SNOOP_RMPP_SENDS |*/</div><div class='del'>-		 IB_MAD_SNOOP_SEND_COMPLETIONS /*|</div><div class='del'>-		 IB_MAD_SNOOP_RMPP_SEND_COMPLETIONS*/));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline int is_snooping_recvs(int mad_snoop_flags)</div><div class='del'>-{</div><div class='del'>-	return (mad_snoop_flags &amp;</div><div class='del'>-		(IB_MAD_SNOOP_RECVS /*|</div><div class='del'>-		 IB_MAD_SNOOP_RMPP_RECVS*/));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int register_snoop_agent(struct ib_mad_qp_info *qp_info,</div><div class='del'>-				struct ib_mad_snoop_private *mad_snoop_priv)</div><div class='del'>-{</div><div class='del'>-	struct ib_mad_snoop_private **new_snoop_table;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;qp_info-&gt;snoop_lock, flags);</div><div class='del'>-	/* Check for empty slot in array. */</div><div class='del'>-	for (i = 0; i &lt; qp_info-&gt;snoop_table_size; i++)</div><div class='del'>-		if (!qp_info-&gt;snoop_table[i])</div><div class='del'>-			break;</div><div class='del'>-</div><div class='del'>-	if (i == qp_info-&gt;snoop_table_size) {</div><div class='del'>-		/* Grow table. */</div><div class='del'>-		new_snoop_table = krealloc(qp_info-&gt;snoop_table,</div><div class='del'>-					   sizeof mad_snoop_priv *</div><div class='del'>-					   (qp_info-&gt;snoop_table_size + 1),</div><div class='del'>-					   GFP_ATOMIC);</div><div class='del'>-		if (!new_snoop_table) {</div><div class='del'>-			i = -ENOMEM;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		qp_info-&gt;snoop_table = new_snoop_table;</div><div class='del'>-		qp_info-&gt;snoop_table_size++;</div><div class='del'>-	}</div><div class='del'>-	qp_info-&gt;snoop_table[i] = mad_snoop_priv;</div><div class='del'>-	atomic_inc(&amp;qp_info-&gt;snoop_count);</div><div class='del'>-out:</div><div class='del'>-	spin_unlock_irqrestore(&amp;qp_info-&gt;snoop_lock, flags);</div><div class='del'>-	return i;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct ib_mad_agent *ib_register_mad_snoop(struct ib_device *device,</div><div class='del'>-					   u8 port_num,</div><div class='del'>-					   enum ib_qp_type qp_type,</div><div class='del'>-					   int mad_snoop_flags,</div><div class='del'>-					   ib_mad_snoop_handler snoop_handler,</div><div class='del'>-					   ib_mad_recv_handler recv_handler,</div><div class='del'>-					   void *context)</div><div class='del'>-{</div><div class='del'>-	struct ib_mad_port_private *port_priv;</div><div class='del'>-	struct ib_mad_agent *ret;</div><div class='del'>-	struct ib_mad_snoop_private *mad_snoop_priv;</div><div class='del'>-	int qpn;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	/* Validate parameters */</div><div class='del'>-	if ((is_snooping_sends(mad_snoop_flags) &amp;&amp; !snoop_handler) ||</div><div class='del'>-	    (is_snooping_recvs(mad_snoop_flags) &amp;&amp; !recv_handler)) {</div><div class='del'>-		ret = ERR_PTR(-EINVAL);</div><div class='del'>-		goto error1;</div><div class='del'>-	}</div><div class='del'>-	qpn = get_spl_qp_index(qp_type);</div><div class='del'>-	if (qpn == -1) {</div><div class='del'>-		ret = ERR_PTR(-EINVAL);</div><div class='del'>-		goto error1;</div><div class='del'>-	}</div><div class='del'>-	port_priv = ib_get_mad_port(device, port_num);</div><div class='del'>-	if (!port_priv) {</div><div class='del'>-		ret = ERR_PTR(-ENODEV);</div><div class='del'>-		goto error1;</div><div class='del'>-	}</div><div class='del'>-	/* Allocate structures */</div><div class='del'>-	mad_snoop_priv = kzalloc(sizeof *mad_snoop_priv, GFP_KERNEL);</div><div class='del'>-	if (!mad_snoop_priv) {</div><div class='del'>-		ret = ERR_PTR(-ENOMEM);</div><div class='del'>-		goto error1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Now, fill in the various structures */</div><div class='del'>-	mad_snoop_priv-&gt;qp_info = &amp;port_priv-&gt;qp_info[qpn];</div><div class='del'>-	mad_snoop_priv-&gt;agent.device = device;</div><div class='del'>-	mad_snoop_priv-&gt;agent.recv_handler = recv_handler;</div><div class='del'>-	mad_snoop_priv-&gt;agent.snoop_handler = snoop_handler;</div><div class='del'>-	mad_snoop_priv-&gt;agent.context = context;</div><div class='del'>-	mad_snoop_priv-&gt;agent.qp = port_priv-&gt;qp_info[qpn].qp;</div><div class='del'>-	mad_snoop_priv-&gt;agent.port_num = port_num;</div><div class='del'>-	mad_snoop_priv-&gt;mad_snoop_flags = mad_snoop_flags;</div><div class='del'>-	init_completion(&amp;mad_snoop_priv-&gt;comp);</div><div class='del'>-</div><div class='del'>-	err = ib_mad_agent_security_setup(&amp;mad_snoop_priv-&gt;agent, qp_type);</div><div class='del'>-	if (err) {</div><div class='del'>-		ret = ERR_PTR(err);</div><div class='del'>-		goto error2;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mad_snoop_priv-&gt;snoop_index = register_snoop_agent(</div><div class='del'>-						&amp;port_priv-&gt;qp_info[qpn],</div><div class='del'>-						mad_snoop_priv);</div><div class='del'>-	if (mad_snoop_priv-&gt;snoop_index &lt; 0) {</div><div class='del'>-		ret = ERR_PTR(mad_snoop_priv-&gt;snoop_index);</div><div class='del'>-		goto error3;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	atomic_set(&amp;mad_snoop_priv-&gt;refcount, 1);</div><div class='del'>-	return &amp;mad_snoop_priv-&gt;agent;</div><div class='del'>-error3:</div><div class='del'>-	ib_mad_agent_security_cleanup(&amp;mad_snoop_priv-&gt;agent);</div><div class='del'>-error2:</div><div class='del'>-	kfree(mad_snoop_priv);</div><div class='del'>-error1:</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_register_mad_snoop);</div><div class='del'>-</div><div class='ctx'> static inline void deref_mad_agent(struct ib_mad_agent_private *mad_agent_priv)</div><div class='ctx'> {</div><div class='del'>-	if (atomic_dec_and_test(&amp;mad_agent_priv-&gt;refcount))</div><div class='add'>+	if (refcount_dec_and_test(&amp;mad_agent_priv-&gt;refcount))</div><div class='ctx'> 		complete(&amp;mad_agent_priv-&gt;comp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline void deref_snoop_agent(struct ib_mad_snoop_private *mad_snoop_priv)</div><div class='del'>-{</div><div class='del'>-	if (atomic_dec_and_test(&amp;mad_snoop_priv-&gt;refcount))</div><div class='del'>-		complete(&amp;mad_snoop_priv-&gt;comp);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static void unregister_mad_agent(struct ib_mad_agent_private *mad_agent_priv)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_mad_port_private *port_priv;</div><div class='hunk'>@@ -639,10 +530,10 @@ static void unregister_mad_agent(struct ib_mad_agent_private *mad_agent_priv)</div><div class='ctx'> 	xa_erase(&amp;ib_mad_clients, mad_agent_priv-&gt;agent.hi_tid);</div><div class='ctx'> </div><div class='ctx'> 	flush_workqueue(port_priv-&gt;wq);</div><div class='del'>-	ib_cancel_rmpp_recvs(mad_agent_priv);</div><div class='ctx'> </div><div class='ctx'> 	deref_mad_agent(mad_agent_priv);</div><div class='ctx'> 	wait_for_completion(&amp;mad_agent_priv-&gt;comp);</div><div class='add'>+	ib_cancel_rmpp_recvs(mad_agent_priv);</div><div class='ctx'> </div><div class='ctx'> 	ib_mad_agent_security_cleanup(&amp;mad_agent_priv-&gt;agent);</div><div class='ctx'> </div><div class='hunk'>@@ -650,25 +541,6 @@ static void unregister_mad_agent(struct ib_mad_agent_private *mad_agent_priv)</div><div class='ctx'> 	kfree_rcu(mad_agent_priv, rcu);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void unregister_mad_snoop(struct ib_mad_snoop_private *mad_snoop_priv)</div><div class='del'>-{</div><div class='del'>-	struct ib_mad_qp_info *qp_info;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	qp_info = mad_snoop_priv-&gt;qp_info;</div><div class='del'>-	spin_lock_irqsave(&amp;qp_info-&gt;snoop_lock, flags);</div><div class='del'>-	qp_info-&gt;snoop_table[mad_snoop_priv-&gt;snoop_index] = NULL;</div><div class='del'>-	atomic_dec(&amp;qp_info-&gt;snoop_count);</div><div class='del'>-	spin_unlock_irqrestore(&amp;qp_info-&gt;snoop_lock, flags);</div><div class='del'>-</div><div class='del'>-	deref_snoop_agent(mad_snoop_priv);</div><div class='del'>-	wait_for_completion(&amp;mad_snoop_priv-&gt;comp);</div><div class='del'>-</div><div class='del'>-	ib_mad_agent_security_cleanup(&amp;mad_snoop_priv-&gt;agent);</div><div class='del'>-</div><div class='del'>-	kfree(mad_snoop_priv);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> /*</div><div class='ctx'>  * ib_unregister_mad_agent - Unregisters a client from using MAD services</div><div class='ctx'>  *</div><div class='hunk'>@@ -677,20 +549,11 @@ static void unregister_mad_snoop(struct ib_mad_snoop_private *mad_snoop_priv)</div><div class='ctx'> void ib_unregister_mad_agent(struct ib_mad_agent *mad_agent)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_mad_agent_private *mad_agent_priv;</div><div class='del'>-	struct ib_mad_snoop_private *mad_snoop_priv;</div><div class='del'>-</div><div class='del'>-	/* If the TID is zero, the agent can only snoop. */</div><div class='del'>-	if (mad_agent-&gt;hi_tid) {</div><div class='del'>-		mad_agent_priv = container_of(mad_agent,</div><div class='del'>-					      struct ib_mad_agent_private,</div><div class='del'>-					      agent);</div><div class='del'>-		unregister_mad_agent(mad_agent_priv);</div><div class='del'>-	} else {</div><div class='del'>-		mad_snoop_priv = container_of(mad_agent,</div><div class='del'>-					      struct ib_mad_snoop_private,</div><div class='del'>-					      agent);</div><div class='del'>-		unregister_mad_snoop(mad_snoop_priv);</div><div class='del'>-	}</div><div class='add'>+</div><div class='add'>+	mad_agent_priv = container_of(mad_agent,</div><div class='add'>+				      struct ib_mad_agent_private,</div><div class='add'>+				      agent);</div><div class='add'>+	unregister_mad_agent(mad_agent_priv);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_unregister_mad_agent);</div><div class='ctx'> </div><div class='hunk'>@@ -706,59 +569,8 @@ static void dequeue_mad(struct ib_mad_list_head *mad_list)</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;mad_queue-&gt;lock, flags);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void snoop_send(struct ib_mad_qp_info *qp_info,</div><div class='del'>-		       struct ib_mad_send_buf *send_buf,</div><div class='del'>-		       struct ib_mad_send_wc *mad_send_wc,</div><div class='del'>-		       int mad_snoop_flags)</div><div class='del'>-{</div><div class='del'>-	struct ib_mad_snoop_private *mad_snoop_priv;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;qp_info-&gt;snoop_lock, flags);</div><div class='del'>-	for (i = 0; i &lt; qp_info-&gt;snoop_table_size; i++) {</div><div class='del'>-		mad_snoop_priv = qp_info-&gt;snoop_table[i];</div><div class='del'>-		if (!mad_snoop_priv ||</div><div class='del'>-		    !(mad_snoop_priv-&gt;mad_snoop_flags &amp; mad_snoop_flags))</div><div class='del'>-			continue;</div><div class='del'>-</div><div class='del'>-		atomic_inc(&amp;mad_snoop_priv-&gt;refcount);</div><div class='del'>-		spin_unlock_irqrestore(&amp;qp_info-&gt;snoop_lock, flags);</div><div class='del'>-		mad_snoop_priv-&gt;agent.snoop_handler(&amp;mad_snoop_priv-&gt;agent,</div><div class='del'>-						    send_buf, mad_send_wc);</div><div class='del'>-		deref_snoop_agent(mad_snoop_priv);</div><div class='del'>-		spin_lock_irqsave(&amp;qp_info-&gt;snoop_lock, flags);</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;qp_info-&gt;snoop_lock, flags);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void snoop_recv(struct ib_mad_qp_info *qp_info,</div><div class='del'>-		       struct ib_mad_recv_wc *mad_recv_wc,</div><div class='del'>-		       int mad_snoop_flags)</div><div class='del'>-{</div><div class='del'>-	struct ib_mad_snoop_private *mad_snoop_priv;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;qp_info-&gt;snoop_lock, flags);</div><div class='del'>-	for (i = 0; i &lt; qp_info-&gt;snoop_table_size; i++) {</div><div class='del'>-		mad_snoop_priv = qp_info-&gt;snoop_table[i];</div><div class='del'>-		if (!mad_snoop_priv ||</div><div class='del'>-		    !(mad_snoop_priv-&gt;mad_snoop_flags &amp; mad_snoop_flags))</div><div class='del'>-			continue;</div><div class='del'>-</div><div class='del'>-		atomic_inc(&amp;mad_snoop_priv-&gt;refcount);</div><div class='del'>-		spin_unlock_irqrestore(&amp;qp_info-&gt;snoop_lock, flags);</div><div class='del'>-		mad_snoop_priv-&gt;agent.recv_handler(&amp;mad_snoop_priv-&gt;agent, NULL,</div><div class='del'>-						   mad_recv_wc);</div><div class='del'>-		deref_snoop_agent(mad_snoop_priv);</div><div class='del'>-		spin_lock_irqsave(&amp;qp_info-&gt;snoop_lock, flags);</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;qp_info-&gt;snoop_lock, flags);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static void build_smp_wc(struct ib_qp *qp, struct ib_cqe *cqe, u16 slid,</div><div class='del'>-		u16 pkey_index, u8 port_num, struct ib_wc *wc)</div><div class='add'>+		u16 pkey_index, u32 port_num, struct ib_wc *wc)</div><div class='ctx'> {</div><div class='ctx'> 	memset(wc, 0, sizeof *wc);</div><div class='ctx'> 	wc-&gt;wr_cqe = cqe;</div><div class='hunk'>@@ -817,7 +629,7 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,</div><div class='ctx'> 	struct ib_mad_port_private *port_priv;</div><div class='ctx'> 	struct ib_mad_agent_private *recv_mad_agent = NULL;</div><div class='ctx'> 	struct ib_device *device = mad_agent_priv-&gt;agent.device;</div><div class='del'>-	u8 port_num;</div><div class='add'>+	u32 port_num;</div><div class='ctx'> 	struct ib_wc mad_wc;</div><div class='ctx'> 	struct ib_ud_wr *send_wr = &amp;mad_send_wr-&gt;send_wr;</div><div class='ctx'> 	size_t mad_size = port_mad_size(mad_agent_priv-&gt;qp_info-&gt;port_priv);</div><div class='hunk'>@@ -913,11 +725,10 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,</div><div class='ctx'> </div><div class='ctx'> 	/* No GRH for DR SMP */</div><div class='ctx'> 	ret = device-&gt;ops.process_mad(device, 0, port_num, &amp;mad_wc, NULL,</div><div class='del'>-				      (const struct ib_mad_hdr *)smp, mad_size,</div><div class='del'>-				      (struct ib_mad_hdr *)mad_priv-&gt;mad,</div><div class='del'>-				      &amp;mad_size, &amp;out_mad_pkey_index);</div><div class='del'>-	switch (ret)</div><div class='del'>-	{</div><div class='add'>+				      (const struct ib_mad *)smp,</div><div class='add'>+				      (struct ib_mad *)mad_priv-&gt;mad, &amp;mad_size,</div><div class='add'>+				      &amp;out_mad_pkey_index);</div><div class='add'>+	switch (ret) {</div><div class='ctx'> 	case IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY:</div><div class='ctx'> 		if (ib_response_mad((const struct ib_mad_hdr *)mad_priv-&gt;mad) &amp;&amp;</div><div class='ctx'> 		    mad_agent_priv-&gt;agent.recv_handler) {</div><div class='hunk'>@@ -927,7 +738,7 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,</div><div class='ctx'> 			 * Reference MAD agent until receive</div><div class='ctx'> 			 * side of local completion handled</div><div class='ctx'> 			 */</div><div class='del'>-			atomic_inc(&amp;mad_agent_priv-&gt;refcount);</div><div class='add'>+			refcount_inc(&amp;mad_agent_priv-&gt;refcount);</div><div class='ctx'> 		} else</div><div class='ctx'> 			kfree(mad_priv);</div><div class='ctx'> 		break;</div><div class='hunk'>@@ -967,7 +778,7 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,</div><div class='ctx'> 		local-&gt;return_wc_byte_len = mad_size;</div><div class='ctx'> 	}</div><div class='ctx'> 	/* Reference MAD agent until send side of local completion handled */</div><div class='del'>-	atomic_inc(&amp;mad_agent_priv-&gt;refcount);</div><div class='add'>+	refcount_inc(&amp;mad_agent_priv-&gt;refcount);</div><div class='ctx'> 	/* Queue local completion to local list */</div><div class='ctx'> 	spin_lock_irqsave(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='ctx'> 	list_add_tail(&amp;local-&gt;completion_list, &amp;mad_agent_priv-&gt;local_list);</div><div class='hunk'>@@ -1016,7 +827,7 @@ static int alloc_send_rmpp_list(struct ib_mad_send_wr_private *send_wr,</div><div class='ctx'> </div><div class='ctx'> 	/* Allocate data segments. */</div><div class='ctx'> 	for (left = send_buf-&gt;data_len + pad; left &gt; 0; left -= seg_size) {</div><div class='del'>-		seg = kmalloc(sizeof (*seg) + seg_size, gfp_mask);</div><div class='add'>+		seg = kmalloc(sizeof(*seg) + seg_size, gfp_mask);</div><div class='ctx'> 		if (!seg) {</div><div class='ctx'> 			free_send_rmpp_list(send_wr);</div><div class='ctx'> 			return -ENOMEM;</div><div class='hunk'>@@ -1046,12 +857,11 @@ int ib_mad_kernel_rmpp_agent(const struct ib_mad_agent *agent)</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_mad_kernel_rmpp_agent);</div><div class='ctx'> </div><div class='del'>-struct ib_mad_send_buf * ib_create_send_mad(struct ib_mad_agent *mad_agent,</div><div class='del'>-					    u32 remote_qpn, u16 pkey_index,</div><div class='del'>-					    int rmpp_active,</div><div class='del'>-					    int hdr_len, int data_len,</div><div class='del'>-					    gfp_t gfp_mask,</div><div class='del'>-					    u8 base_version)</div><div class='add'>+struct ib_mad_send_buf *ib_create_send_mad(struct ib_mad_agent *mad_agent,</div><div class='add'>+					   u32 remote_qpn, u16 pkey_index,</div><div class='add'>+					   int rmpp_active, int hdr_len,</div><div class='add'>+					   int data_len, gfp_t gfp_mask,</div><div class='add'>+					   u8 base_version)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_mad_agent_private *mad_agent_priv;</div><div class='ctx'> 	struct ib_mad_send_wr_private *mad_send_wr;</div><div class='hunk'>@@ -1125,7 +935,7 @@ struct ib_mad_send_buf * ib_create_send_mad(struct ib_mad_agent *mad_agent,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	mad_send_wr-&gt;send_buf.mad_agent = mad_agent;</div><div class='del'>-	atomic_inc(&amp;mad_agent_priv-&gt;refcount);</div><div class='add'>+	refcount_inc(&amp;mad_agent_priv-&gt;refcount);</div><div class='ctx'> 	return &amp;mad_send_wr-&gt;send_buf;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_create_send_mad);</div><div class='hunk'>@@ -1272,6 +1082,180 @@ int ib_send_mad(struct ib_mad_send_wr_private *mad_send_wr)</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void handle_queued_state(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='add'>+		       struct ib_mad_agent_private *mad_agent_priv)</div><div class='add'>+{</div><div class='add'>+	if (mad_send_wr-&gt;state == IB_MAD_STATE_WAIT_RESP) {</div><div class='add'>+		mad_agent_priv-&gt;sol_fc_wait_count--;</div><div class='add'>+		list_move_tail(&amp;mad_send_wr-&gt;agent_list,</div><div class='add'>+			       &amp;mad_agent_priv-&gt;backlog_list);</div><div class='add'>+	} else {</div><div class='add'>+		expect_mad_state(mad_send_wr, IB_MAD_STATE_INIT);</div><div class='add'>+		list_add_tail(&amp;mad_send_wr-&gt;agent_list,</div><div class='add'>+			      &amp;mad_agent_priv-&gt;backlog_list);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void handle_send_state(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='add'>+		       struct ib_mad_agent_private *mad_agent_priv)</div><div class='add'>+{</div><div class='add'>+	if (mad_send_wr-&gt;state == IB_MAD_STATE_INIT) {</div><div class='add'>+		list_add_tail(&amp;mad_send_wr-&gt;agent_list,</div><div class='add'>+			      &amp;mad_agent_priv-&gt;send_list);</div><div class='add'>+	} else {</div><div class='add'>+		expect_mad_state2(mad_send_wr, IB_MAD_STATE_WAIT_RESP,</div><div class='add'>+				  IB_MAD_STATE_QUEUED);</div><div class='add'>+		list_move_tail(&amp;mad_send_wr-&gt;agent_list,</div><div class='add'>+			       &amp;mad_agent_priv-&gt;send_list);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (mad_send_wr-&gt;is_solicited_fc) {</div><div class='add'>+		if (mad_send_wr-&gt;state == IB_MAD_STATE_WAIT_RESP)</div><div class='add'>+			mad_agent_priv-&gt;sol_fc_wait_count--;</div><div class='add'>+		mad_agent_priv-&gt;sol_fc_send_count++;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void handle_wait_state(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='add'>+		       struct ib_mad_agent_private *mad_agent_priv)</div><div class='add'>+{</div><div class='add'>+	struct ib_mad_send_wr_private *temp_mad_send_wr;</div><div class='add'>+	struct list_head *list_item;</div><div class='add'>+	unsigned long delay;</div><div class='add'>+</div><div class='add'>+	expect_mad_state3(mad_send_wr, IB_MAD_STATE_SEND_START,</div><div class='add'>+			  IB_MAD_STATE_WAIT_RESP, IB_MAD_STATE_CANCELED);</div><div class='add'>+	if (mad_send_wr-&gt;state == IB_MAD_STATE_SEND_START &amp;&amp;</div><div class='add'>+	    mad_send_wr-&gt;is_solicited_fc) {</div><div class='add'>+		mad_agent_priv-&gt;sol_fc_send_count--;</div><div class='add'>+		mad_agent_priv-&gt;sol_fc_wait_count++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	list_del_init(&amp;mad_send_wr-&gt;agent_list);</div><div class='add'>+	delay = mad_send_wr-&gt;timeout;</div><div class='add'>+	mad_send_wr-&gt;timeout += jiffies;</div><div class='add'>+</div><div class='add'>+	if (delay) {</div><div class='add'>+		list_for_each_prev(list_item,</div><div class='add'>+				   &amp;mad_agent_priv-&gt;wait_list) {</div><div class='add'>+			temp_mad_send_wr = list_entry(</div><div class='add'>+				list_item,</div><div class='add'>+				struct ib_mad_send_wr_private,</div><div class='add'>+				agent_list);</div><div class='add'>+			if (time_after(mad_send_wr-&gt;timeout,</div><div class='add'>+				       temp_mad_send_wr-&gt;timeout))</div><div class='add'>+				break;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		list_item = &amp;mad_agent_priv-&gt;wait_list;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	list_add(&amp;mad_send_wr-&gt;agent_list, list_item);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void handle_early_resp_state(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='add'>+			    struct ib_mad_agent_private *mad_agent_priv)</div><div class='add'>+{</div><div class='add'>+	expect_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);</div><div class='add'>+	mad_agent_priv-&gt;sol_fc_send_count -= mad_send_wr-&gt;is_solicited_fc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void handle_canceled_state(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='add'>+			 struct ib_mad_agent_private *mad_agent_priv)</div><div class='add'>+{</div><div class='add'>+	not_expect_mad_state(mad_send_wr, IB_MAD_STATE_DONE);</div><div class='add'>+	if (mad_send_wr-&gt;is_solicited_fc) {</div><div class='add'>+		if (mad_send_wr-&gt;state == IB_MAD_STATE_SEND_START)</div><div class='add'>+			mad_agent_priv-&gt;sol_fc_send_count--;</div><div class='add'>+		else if (mad_send_wr-&gt;state == IB_MAD_STATE_WAIT_RESP)</div><div class='add'>+			mad_agent_priv-&gt;sol_fc_wait_count--;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void handle_done_state(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='add'>+		       struct ib_mad_agent_private *mad_agent_priv)</div><div class='add'>+{</div><div class='add'>+	if (mad_send_wr-&gt;is_solicited_fc) {</div><div class='add'>+		if (mad_send_wr-&gt;state == IB_MAD_STATE_SEND_START)</div><div class='add'>+			mad_agent_priv-&gt;sol_fc_send_count--;</div><div class='add'>+		else if (mad_send_wr-&gt;state == IB_MAD_STATE_WAIT_RESP)</div><div class='add'>+			mad_agent_priv-&gt;sol_fc_wait_count--;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	list_del_init(&amp;mad_send_wr-&gt;agent_list);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void change_mad_state(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='add'>+			     enum ib_mad_state new_state)</div><div class='add'>+{</div><div class='add'>+	struct ib_mad_agent_private *mad_agent_priv =</div><div class='add'>+		mad_send_wr-&gt;mad_agent_priv;</div><div class='add'>+</div><div class='add'>+	switch (new_state) {</div><div class='add'>+	case IB_MAD_STATE_INIT:</div><div class='add'>+		break;</div><div class='add'>+	case IB_MAD_STATE_QUEUED:</div><div class='add'>+		handle_queued_state(mad_send_wr, mad_agent_priv);</div><div class='add'>+		break;</div><div class='add'>+	case IB_MAD_STATE_SEND_START:</div><div class='add'>+		handle_send_state(mad_send_wr, mad_agent_priv);</div><div class='add'>+		break;</div><div class='add'>+	case IB_MAD_STATE_WAIT_RESP:</div><div class='add'>+		handle_wait_state(mad_send_wr, mad_agent_priv);</div><div class='add'>+		if (mad_send_wr-&gt;state == IB_MAD_STATE_CANCELED)</div><div class='add'>+			return;</div><div class='add'>+		break;</div><div class='add'>+	case IB_MAD_STATE_EARLY_RESP:</div><div class='add'>+		handle_early_resp_state(mad_send_wr, mad_agent_priv);</div><div class='add'>+		break;</div><div class='add'>+	case IB_MAD_STATE_CANCELED:</div><div class='add'>+		handle_canceled_state(mad_send_wr, mad_agent_priv);</div><div class='add'>+		break;</div><div class='add'>+	case IB_MAD_STATE_DONE:</div><div class='add'>+		handle_done_state(mad_send_wr, mad_agent_priv);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mad_send_wr-&gt;state = new_state;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool is_solicited_fc_mad(struct ib_mad_send_wr_private *mad_send_wr)</div><div class='add'>+{</div><div class='add'>+	struct ib_rmpp_mad *rmpp_mad;</div><div class='add'>+	u8 mgmt_class;</div><div class='add'>+</div><div class='add'>+	if (!mad_send_wr-&gt;timeout)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	rmpp_mad = mad_send_wr-&gt;send_buf.mad;</div><div class='add'>+	if (mad_send_wr-&gt;mad_agent_priv-&gt;agent.rmpp_version &amp;&amp;</div><div class='add'>+	    (ib_get_rmpp_flags(&amp;rmpp_mad-&gt;rmpp_hdr) &amp; IB_MGMT_RMPP_FLAG_ACTIVE))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	mgmt_class =</div><div class='add'>+		((struct ib_mad_hdr *)mad_send_wr-&gt;send_buf.mad)-&gt;mgmt_class;</div><div class='add'>+	return mgmt_class == IB_MGMT_CLASS_CM ||</div><div class='add'>+	       mgmt_class == IB_MGMT_CLASS_SUBN_ADM ||</div><div class='add'>+	       mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED ||</div><div class='add'>+	       mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool mad_is_for_backlog(struct ib_mad_send_wr_private *mad_send_wr)</div><div class='add'>+{</div><div class='add'>+	struct ib_mad_agent_private *mad_agent_priv =</div><div class='add'>+		mad_send_wr-&gt;mad_agent_priv;</div><div class='add'>+</div><div class='add'>+	if (!mad_send_wr-&gt;is_solicited_fc || !mad_agent_priv-&gt;sol_fc_max)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	if (!list_empty(&amp;mad_agent_priv-&gt;backlog_list))</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	return mad_agent_priv-&gt;sol_fc_send_count +</div><div class='add'>+		       mad_agent_priv-&gt;sol_fc_wait_count &gt;=</div><div class='add'>+	       mad_agent_priv-&gt;sol_fc_max;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /*</div><div class='ctx'>  * ib_post_send_mad - Posts MAD(s) to the send queue of the QP associated</div><div class='ctx'>  *  with the registered client</div><div class='hunk'>@@ -1297,9 +1281,7 @@ int ib_post_send_mad(struct ib_mad_send_buf *send_buf,</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto error;</div><div class='ctx'> </div><div class='del'>-		if (!send_buf-&gt;mad_agent-&gt;send_handler ||</div><div class='del'>-		    (send_buf-&gt;timeout_ms &amp;&amp;</div><div class='del'>-		     !send_buf-&gt;mad_agent-&gt;recv_handler)) {</div><div class='add'>+		if (!send_buf-&gt;mad_agent-&gt;send_handler) {</div><div class='ctx'> 			ret = -EINVAL;</div><div class='ctx'> 			goto error;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -1335,15 +1317,19 @@ int ib_post_send_mad(struct ib_mad_send_buf *send_buf,</div><div class='ctx'> 		mad_send_wr-&gt;max_retries = send_buf-&gt;retries;</div><div class='ctx'> 		mad_send_wr-&gt;retries_left = send_buf-&gt;retries;</div><div class='ctx'> 		send_buf-&gt;retries = 0;</div><div class='del'>-		/* Reference for work request to QP + response */</div><div class='del'>-		mad_send_wr-&gt;refcount = 1 + (mad_send_wr-&gt;timeout &gt; 0);</div><div class='del'>-		mad_send_wr-&gt;status = IB_WC_SUCCESS;</div><div class='add'>+		change_mad_state(mad_send_wr, IB_MAD_STATE_INIT);</div><div class='ctx'> </div><div class='ctx'> 		/* Reference MAD agent until send completes */</div><div class='del'>-		atomic_inc(&amp;mad_agent_priv-&gt;refcount);</div><div class='add'>+		refcount_inc(&amp;mad_agent_priv-&gt;refcount);</div><div class='ctx'> 		spin_lock_irqsave(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='del'>-		list_add_tail(&amp;mad_send_wr-&gt;agent_list,</div><div class='del'>-			      &amp;mad_agent_priv-&gt;send_list);</div><div class='add'>+		mad_send_wr-&gt;is_solicited_fc = is_solicited_fc_mad(mad_send_wr);</div><div class='add'>+		if (mad_is_for_backlog(mad_send_wr)) {</div><div class='add'>+			change_mad_state(mad_send_wr, IB_MAD_STATE_QUEUED);</div><div class='add'>+			spin_unlock_irqrestore(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='ctx'> </div><div class='ctx'> 		if (ib_mad_kernel_rmpp_agent(&amp;mad_agent_priv-&gt;agent)) {</div><div class='hunk'>@@ -1355,9 +1341,9 @@ int ib_post_send_mad(struct ib_mad_send_buf *send_buf,</div><div class='ctx'> 		if (ret &lt; 0) {</div><div class='ctx'> 			/* Fail send request */</div><div class='ctx'> 			spin_lock_irqsave(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='del'>-			list_del(&amp;mad_send_wr-&gt;agent_list);</div><div class='add'>+			change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);</div><div class='ctx'> 			spin_unlock_irqrestore(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='del'>-			atomic_dec(&amp;mad_agent_priv-&gt;refcount);</div><div class='add'>+			deref_mad_agent(mad_agent_priv);</div><div class='ctx'> 			goto error;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='hunk'>@@ -1397,25 +1383,6 @@ void ib_free_recv_mad(struct ib_mad_recv_wc *mad_recv_wc)</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_free_recv_mad);</div><div class='ctx'> </div><div class='del'>-struct ib_mad_agent *ib_redirect_mad_qp(struct ib_qp *qp,</div><div class='del'>-					u8 rmpp_version,</div><div class='del'>-					ib_mad_send_handler send_handler,</div><div class='del'>-					ib_mad_recv_handler recv_handler,</div><div class='del'>-					void *context)</div><div class='del'>-{</div><div class='del'>-	return ERR_PTR(-EINVAL);	/* XXX: for now */</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_redirect_mad_qp);</div><div class='del'>-</div><div class='del'>-int ib_process_mad_wc(struct ib_mad_agent *mad_agent,</div><div class='del'>-		      struct ib_wc *wc)</div><div class='del'>-{</div><div class='del'>-	dev_err(&amp;mad_agent-&gt;device-&gt;dev,</div><div class='del'>-		"ib_process_mad_wc() not implemented yet\n");</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_process_mad_wc);</div><div class='del'>-</div><div class='ctx'> static int method_in_use(struct ib_mad_mgmt_method_table **method,</div><div class='ctx'> 			 struct ib_mad_reg_req *mad_reg_req)</div><div class='ctx'> {</div><div class='hunk'>@@ -1503,11 +1470,9 @@ static void remove_methods_mad_agent(struct ib_mad_mgmt_method_table *method,</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='ctx'> 	/* Remove any methods for this mad agent */</div><div class='del'>-	for (i = 0; i &lt; IB_MGMT_MAX_METHODS; i++) {</div><div class='del'>-		if (method-&gt;agent[i] == agent) {</div><div class='add'>+	for (i = 0; i &lt; IB_MGMT_MAX_METHODS; i++)</div><div class='add'>+		if (method-&gt;agent[i] == agent)</div><div class='ctx'> 			method-&gt;agent[i] = NULL;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req,</div><div class='hunk'>@@ -1682,9 +1647,8 @@ static void remove_mad_reg_req(struct ib_mad_agent_private *agent_priv)</div><div class='ctx'> 	 * Was MAD registration request supplied</div><div class='ctx'> 	 * with original registration ?</div><div class='ctx'> 	 */</div><div class='del'>-	if (!agent_priv-&gt;reg_req) {</div><div class='add'>+	if (!agent_priv-&gt;reg_req)</div><div class='ctx'> 		goto out;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='ctx'> 	port_priv = agent_priv-&gt;qp_info-&gt;port_priv;</div><div class='ctx'> 	mgmt_class = convert_mgmt_class(agent_priv-&gt;reg_req-&gt;mgmt_class);</div><div class='hunk'>@@ -1782,7 +1746,7 @@ find_mad_agent(struct ib_mad_port_private *port_priv,</div><div class='ctx'> 		hi_tid = be64_to_cpu(mad_hdr-&gt;tid) &gt;&gt; 32;</div><div class='ctx'> 		rcu_read_lock();</div><div class='ctx'> 		mad_agent = xa_load(&amp;ib_mad_clients, hi_tid);</div><div class='del'>-		if (mad_agent &amp;&amp; !atomic_inc_not_zero(&amp;mad_agent-&gt;refcount))</div><div class='add'>+		if (mad_agent &amp;&amp; !refcount_inc_not_zero(&amp;mad_agent-&gt;refcount))</div><div class='ctx'> 			mad_agent = NULL;</div><div class='ctx'> 		rcu_read_unlock();</div><div class='ctx'> 	} else {</div><div class='hunk'>@@ -1834,14 +1798,14 @@ find_mad_agent(struct ib_mad_port_private *port_priv,</div><div class='ctx'> 			}</div><div class='ctx'> 		}</div><div class='ctx'> 		if (mad_agent)</div><div class='del'>-			atomic_inc(&amp;mad_agent-&gt;refcount);</div><div class='add'>+			refcount_inc(&amp;mad_agent-&gt;refcount);</div><div class='ctx'> out:</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;port_priv-&gt;reg_lock, flags);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (mad_agent &amp;&amp; !mad_agent-&gt;agent.recv_handler) {</div><div class='ctx'> 		dev_notice(&amp;port_priv-&gt;device-&gt;dev,</div><div class='del'>-			   "No receive handler for client %p on port %d\n",</div><div class='add'>+			   "No receive handler for client %p on port %u\n",</div><div class='ctx'> 			   &amp;mad_agent-&gt;agent, port_priv-&gt;port_num);</div><div class='ctx'> 		deref_mad_agent(mad_agent);</div><div class='ctx'> 		mad_agent = NULL;</div><div class='hunk'>@@ -1860,7 +1824,7 @@ static int validate_mad(const struct ib_mad_hdr *mad_hdr,</div><div class='ctx'> 	/* Make sure MAD base version is understood */</div><div class='ctx'> 	if (mad_hdr-&gt;base_version != IB_MGMT_BASE_VERSION &amp;&amp;</div><div class='ctx'> 	    (!opa || mad_hdr-&gt;base_version != OPA_MGMT_BASE_VERSION)) {</div><div class='del'>-		pr_err("MAD received with unsupported base version %d %s\n",</div><div class='add'>+		pr_err("MAD received with unsupported base version %u %s\n",</div><div class='ctx'> 		       mad_hdr-&gt;base_version, opa ? "(opa)" : "");</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -1905,15 +1869,16 @@ static inline int rcv_has_same_class(const struct ib_mad_send_wr_private *wr,</div><div class='ctx'> 		rwc-&gt;recv_buf.mad-&gt;mad_hdr.mgmt_class;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline int rcv_has_same_gid(const struct ib_mad_agent_private *mad_agent_priv,</div><div class='del'>-				   const struct ib_mad_send_wr_private *wr,</div><div class='del'>-				   const struct ib_mad_recv_wc *rwc )</div><div class='add'>+static inline int</div><div class='add'>+rcv_has_same_gid(const struct ib_mad_agent_private *mad_agent_priv,</div><div class='add'>+		 const struct ib_mad_send_wr_private *wr,</div><div class='add'>+		 const struct ib_mad_recv_wc *rwc)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_ah_attr attr;</div><div class='ctx'> 	u8 send_resp, rcv_resp;</div><div class='ctx'> 	union ib_gid sgid;</div><div class='ctx'> 	struct ib_device *device = mad_agent_priv-&gt;agent.device;</div><div class='del'>-	u8 port_num = mad_agent_priv-&gt;agent.port_num;</div><div class='add'>+	u32 port_num = mad_agent_priv-&gt;agent.port_num;</div><div class='ctx'> 	u8 lmc;</div><div class='ctx'> 	bool has_grh;</div><div class='ctx'> </div><div class='hunk'>@@ -1984,7 +1949,19 @@ ib_find_send_mad(const struct ib_mad_agent_private *mad_agent_priv,</div><div class='ctx'> 		     */</div><div class='ctx'> 		    (is_direct(mad_hdr-&gt;mgmt_class) ||</div><div class='ctx'> 		     rcv_has_same_gid(mad_agent_priv, wr, wc)))</div><div class='del'>-			return (wr-&gt;status == IB_WC_SUCCESS) ? wr : NULL;</div><div class='add'>+			return (wr-&gt;state != IB_MAD_STATE_CANCELED) ? wr : NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	list_for_each_entry(wr, &amp;mad_agent_priv-&gt;backlog_list, agent_list) {</div><div class='add'>+		if ((wr-&gt;tid == mad_hdr-&gt;tid) &amp;&amp;</div><div class='add'>+		    rcv_has_same_class(wr, wc) &amp;&amp;</div><div class='add'>+		    /*</div><div class='add'>+		     * Don't check GID for direct routed MADs.</div><div class='add'>+		     * These might have permissive LIDs.</div><div class='add'>+		     */</div><div class='add'>+		    (is_direct(mad_hdr-&gt;mgmt_class) ||</div><div class='add'>+		     rcv_has_same_gid(mad_agent_priv, wr, wc)))</div><div class='add'>+			return (wr-&gt;state != IB_MAD_STATE_CANCELED) ? wr : NULL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='hunk'>@@ -2003,17 +1980,55 @@ ib_find_send_mad(const struct ib_mad_agent_private *mad_agent_priv,</div><div class='ctx'> 		    (is_direct(mad_hdr-&gt;mgmt_class) ||</div><div class='ctx'> 		     rcv_has_same_gid(mad_agent_priv, wr, wc)))</div><div class='ctx'> 			/* Verify request has not been canceled */</div><div class='del'>-			return (wr-&gt;status == IB_WC_SUCCESS) ? wr : NULL;</div><div class='add'>+			return (wr-&gt;state != IB_MAD_STATE_CANCELED) ? wr : NULL;</div><div class='ctx'> 	}</div><div class='ctx'> 	return NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void</div><div class='add'>+process_backlog_mads(struct ib_mad_agent_private *mad_agent_priv)</div><div class='add'>+{</div><div class='add'>+	struct ib_mad_send_wr_private *mad_send_wr;</div><div class='add'>+	struct ib_mad_send_wc mad_send_wc = {};</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='add'>+	while (!list_empty(&amp;mad_agent_priv-&gt;backlog_list) &amp;&amp;</div><div class='add'>+	       (mad_agent_priv-&gt;sol_fc_send_count +</div><div class='add'>+			mad_agent_priv-&gt;sol_fc_wait_count &lt;</div><div class='add'>+		mad_agent_priv-&gt;sol_fc_max)) {</div><div class='add'>+		mad_send_wr = list_entry(mad_agent_priv-&gt;backlog_list.next,</div><div class='add'>+					 struct ib_mad_send_wr_private,</div><div class='add'>+					 agent_list);</div><div class='add'>+		change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);</div><div class='add'>+		spin_unlock_irqrestore(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='add'>+		ret = ib_send_mad(mad_send_wr);</div><div class='add'>+		if (ret) {</div><div class='add'>+			spin_lock_irqsave(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='add'>+			deref_mad_agent(mad_agent_priv);</div><div class='add'>+			change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);</div><div class='add'>+			spin_unlock_irqrestore(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='add'>+			mad_send_wc.send_buf = &amp;mad_send_wr-&gt;send_buf;</div><div class='add'>+			mad_send_wc.status = IB_WC_LOC_QP_OP_ERR;</div><div class='add'>+			mad_agent_priv-&gt;agent.send_handler(</div><div class='add'>+				&amp;mad_agent_priv-&gt;agent, &amp;mad_send_wc);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		spin_lock_irqsave(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_unlock_irqrestore(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> void ib_mark_mad_done(struct ib_mad_send_wr_private *mad_send_wr)</div><div class='ctx'> {</div><div class='ctx'> 	mad_send_wr-&gt;timeout = 0;</div><div class='del'>-	if (mad_send_wr-&gt;refcount == 1)</div><div class='del'>-		list_move_tail(&amp;mad_send_wr-&gt;agent_list,</div><div class='del'>-			      &amp;mad_send_wr-&gt;mad_agent_priv-&gt;done_list);</div><div class='add'>+	if (mad_send_wr-&gt;state == IB_MAD_STATE_WAIT_RESP ||</div><div class='add'>+	    mad_send_wr-&gt;state == IB_MAD_STATE_QUEUED)</div><div class='add'>+		change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);</div><div class='add'>+	else</div><div class='add'>+		change_mad_state(mad_send_wr, IB_MAD_STATE_EARLY_RESP);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,</div><div class='hunk'>@@ -2022,6 +2037,7 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,</div><div class='ctx'> 	struct ib_mad_send_wr_private *mad_send_wr;</div><div class='ctx'> 	struct ib_mad_send_wc mad_send_wc;</div><div class='ctx'> 	unsigned long flags;</div><div class='add'>+	bool is_mad_done;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	INIT_LIST_HEAD(&amp;mad_recv_wc-&gt;rmpp_list);</div><div class='hunk'>@@ -2059,16 +2075,18 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,</div><div class='ctx'> 				mad_agent_priv-&gt;agent.recv_handler(</div><div class='ctx'> 						&amp;mad_agent_priv-&gt;agent, NULL,</div><div class='ctx'> 						mad_recv_wc);</div><div class='del'>-				atomic_dec(&amp;mad_agent_priv-&gt;refcount);</div><div class='add'>+				deref_mad_agent(mad_agent_priv);</div><div class='ctx'> 			} else {</div><div class='ctx'> 				/* not user rmpp, revert to normal behavior and</div><div class='del'>-				 * drop the mad */</div><div class='add'>+				 * drop the mad</div><div class='add'>+				 */</div><div class='ctx'> 				ib_free_recv_mad(mad_recv_wc);</div><div class='ctx'> 				deref_mad_agent(mad_agent_priv);</div><div class='ctx'> 				return;</div><div class='ctx'> 			}</div><div class='ctx'> 		} else {</div><div class='ctx'> 			ib_mark_mad_done(mad_send_wr);</div><div class='add'>+			is_mad_done = (mad_send_wr-&gt;state == IB_MAD_STATE_DONE);</div><div class='ctx'> 			spin_unlock_irqrestore(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='ctx'> </div><div class='ctx'> 			/* Defined behavior is to complete response before request */</div><div class='hunk'>@@ -2076,26 +2094,27 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,</div><div class='ctx'> 					&amp;mad_agent_priv-&gt;agent,</div><div class='ctx'> 					&amp;mad_send_wr-&gt;send_buf,</div><div class='ctx'> 					mad_recv_wc);</div><div class='del'>-			atomic_dec(&amp;mad_agent_priv-&gt;refcount);</div><div class='add'>+			deref_mad_agent(mad_agent_priv);</div><div class='ctx'> </div><div class='del'>-			mad_send_wc.status = IB_WC_SUCCESS;</div><div class='del'>-			mad_send_wc.vendor_err = 0;</div><div class='del'>-			mad_send_wc.send_buf = &amp;mad_send_wr-&gt;send_buf;</div><div class='del'>-			ib_mad_complete_send_wr(mad_send_wr, &amp;mad_send_wc);</div><div class='add'>+			if (is_mad_done) {</div><div class='add'>+				mad_send_wc.status = IB_WC_SUCCESS;</div><div class='add'>+				mad_send_wc.vendor_err = 0;</div><div class='add'>+				mad_send_wc.send_buf = &amp;mad_send_wr-&gt;send_buf;</div><div class='add'>+				ib_mad_complete_send_wr(mad_send_wr,</div><div class='add'>+							&amp;mad_send_wc);</div><div class='add'>+			}</div><div class='ctx'> 		}</div><div class='ctx'> 	} else {</div><div class='ctx'> 		mad_agent_priv-&gt;agent.recv_handler(&amp;mad_agent_priv-&gt;agent, NULL,</div><div class='ctx'> 						   mad_recv_wc);</div><div class='ctx'> 		deref_mad_agent(mad_agent_priv);</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='del'>-	return;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static enum smi_action handle_ib_smi(const struct ib_mad_port_private *port_priv,</div><div class='ctx'> 				     const struct ib_mad_qp_info *qp_info,</div><div class='ctx'> 				     const struct ib_wc *wc,</div><div class='del'>-				     int port_num,</div><div class='add'>+				     u32 port_num,</div><div class='ctx'> 				     struct ib_mad_private *recv,</div><div class='ctx'> 				     struct ib_mad_private *response)</div><div class='ctx'> {</div><div class='hunk'>@@ -2182,7 +2201,7 @@ static enum smi_action</div><div class='ctx'> handle_opa_smi(struct ib_mad_port_private *port_priv,</div><div class='ctx'> 	       struct ib_mad_qp_info *qp_info,</div><div class='ctx'> 	       struct ib_wc *wc,</div><div class='del'>-	       int port_num,</div><div class='add'>+	       u32 port_num,</div><div class='ctx'> 	       struct ib_mad_private *recv,</div><div class='ctx'> 	       struct ib_mad_private *response)</div><div class='ctx'> {</div><div class='hunk'>@@ -2238,7 +2257,7 @@ static enum smi_action</div><div class='ctx'> handle_smi(struct ib_mad_port_private *port_priv,</div><div class='ctx'> 	   struct ib_mad_qp_info *qp_info,</div><div class='ctx'> 	   struct ib_wc *wc,</div><div class='del'>-	   int port_num,</div><div class='add'>+	   u32 port_num,</div><div class='ctx'> 	   struct ib_mad_private *recv,</div><div class='ctx'> 	   struct ib_mad_private *response,</div><div class='ctx'> 	   bool opa)</div><div class='hunk'>@@ -2262,7 +2281,7 @@ static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc)</div><div class='ctx'> 	struct ib_mad_private_header *mad_priv_hdr;</div><div class='ctx'> 	struct ib_mad_private *recv, *response = NULL;</div><div class='ctx'> 	struct ib_mad_agent_private *mad_agent;</div><div class='del'>-	int port_num;</div><div class='add'>+	u32 port_num;</div><div class='ctx'> 	int ret = IB_MAD_RESULT_SUCCESS;</div><div class='ctx'> 	size_t mad_size;</div><div class='ctx'> 	u16 resp_mad_pkey_index = 0;</div><div class='hunk'>@@ -2308,9 +2327,6 @@ static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc)</div><div class='ctx'> 	recv-&gt;header.recv_wc.recv_buf.mad = (struct ib_mad *)recv-&gt;mad;</div><div class='ctx'> 	recv-&gt;header.recv_wc.recv_buf.grh = &amp;recv-&gt;grh;</div><div class='ctx'> </div><div class='del'>-	if (atomic_read(&amp;qp_info-&gt;snoop_count))</div><div class='del'>-		snoop_recv(qp_info, &amp;recv-&gt;header.recv_wc, IB_MAD_SNOOP_RECVS);</div><div class='del'>-</div><div class='ctx'> 	/* Validate MAD */</div><div class='ctx'> 	if (!validate_mad((const struct ib_mad_hdr *)recv-&gt;mad, qp_info, opa))</div><div class='ctx'> 		goto out;</div><div class='hunk'>@@ -2340,9 +2356,9 @@ static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc)</div><div class='ctx'> 	if (port_priv-&gt;device-&gt;ops.process_mad) {</div><div class='ctx'> 		ret = port_priv-&gt;device-&gt;ops.process_mad(</div><div class='ctx'> 			port_priv-&gt;device, 0, port_priv-&gt;port_num, wc,</div><div class='del'>-			&amp;recv-&gt;grh, (const struct ib_mad_hdr *)recv-&gt;mad,</div><div class='del'>-			recv-&gt;mad_size, (struct ib_mad_hdr *)response-&gt;mad,</div><div class='del'>-			&amp;mad_size, &amp;resp_mad_pkey_index);</div><div class='add'>+			&amp;recv-&gt;grh, (const struct ib_mad *)recv-&gt;mad,</div><div class='add'>+			(struct ib_mad *)response-&gt;mad, &amp;mad_size,</div><div class='add'>+			&amp;resp_mad_pkey_index);</div><div class='ctx'> </div><div class='ctx'> 		if (opa)</div><div class='ctx'> 			wc-&gt;pkey_index = resp_mad_pkey_index;</div><div class='hunk'>@@ -2414,29 +2430,11 @@ static void adjust_timeout(struct ib_mad_agent_private *mad_agent_priv)</div><div class='ctx'> static void wait_for_response(struct ib_mad_send_wr_private *mad_send_wr)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_mad_agent_private *mad_agent_priv;</div><div class='del'>-	struct ib_mad_send_wr_private *temp_mad_send_wr;</div><div class='del'>-	struct list_head *list_item;</div><div class='ctx'> 	unsigned long delay;</div><div class='ctx'> </div><div class='ctx'> 	mad_agent_priv = mad_send_wr-&gt;mad_agent_priv;</div><div class='del'>-	list_del(&amp;mad_send_wr-&gt;agent_list);</div><div class='del'>-</div><div class='ctx'> 	delay = mad_send_wr-&gt;timeout;</div><div class='del'>-	mad_send_wr-&gt;timeout += jiffies;</div><div class='del'>-</div><div class='del'>-	if (delay) {</div><div class='del'>-		list_for_each_prev(list_item, &amp;mad_agent_priv-&gt;wait_list) {</div><div class='del'>-			temp_mad_send_wr = list_entry(list_item,</div><div class='del'>-						struct ib_mad_send_wr_private,</div><div class='del'>-						agent_list);</div><div class='del'>-			if (time_after(mad_send_wr-&gt;timeout,</div><div class='del'>-				       temp_mad_send_wr-&gt;timeout))</div><div class='del'>-				break;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	else</div><div class='del'>-		list_item = &amp;mad_agent_priv-&gt;wait_list;</div><div class='del'>-	list_add(&amp;mad_send_wr-&gt;agent_list, list_item);</div><div class='add'>+	change_mad_state(mad_send_wr, IB_MAD_STATE_WAIT_RESP);</div><div class='ctx'> </div><div class='ctx'> 	/* Reschedule a work item if we have a shorter timeout */</div><div class='ctx'> 	if (mad_agent_priv-&gt;wait_list.next == &amp;mad_send_wr-&gt;agent_list)</div><div class='hunk'>@@ -2470,32 +2468,28 @@ void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='ctx'> 	} else</div><div class='ctx'> 		ret = IB_RMPP_RESULT_UNHANDLED;</div><div class='ctx'> </div><div class='del'>-	if (mad_send_wc-&gt;status != IB_WC_SUCCESS &amp;&amp;</div><div class='del'>-	    mad_send_wr-&gt;status == IB_WC_SUCCESS) {</div><div class='del'>-		mad_send_wr-&gt;status = mad_send_wc-&gt;status;</div><div class='del'>-		mad_send_wr-&gt;refcount -= (mad_send_wr-&gt;timeout &gt; 0);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (--mad_send_wr-&gt;refcount &gt; 0) {</div><div class='del'>-		if (mad_send_wr-&gt;refcount == 1 &amp;&amp; mad_send_wr-&gt;timeout &amp;&amp;</div><div class='del'>-		    mad_send_wr-&gt;status == IB_WC_SUCCESS) {</div><div class='del'>-			wait_for_response(mad_send_wr);</div><div class='del'>-		}</div><div class='add'>+	if (mad_send_wr-&gt;state == IB_MAD_STATE_CANCELED)</div><div class='add'>+		mad_send_wc-&gt;status = IB_WC_WR_FLUSH_ERR;</div><div class='add'>+	else if (mad_send_wr-&gt;state == IB_MAD_STATE_SEND_START &amp;&amp;</div><div class='add'>+		 mad_send_wr-&gt;timeout) {</div><div class='add'>+		wait_for_response(mad_send_wr);</div><div class='ctx'> 		goto done;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Remove send from MAD agent and notify client of completion */</div><div class='del'>-	list_del(&amp;mad_send_wr-&gt;agent_list);</div><div class='add'>+	if (mad_send_wr-&gt;state != IB_MAD_STATE_DONE)</div><div class='add'>+		change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);</div><div class='ctx'> 	adjust_timeout(mad_agent_priv);</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='ctx'> </div><div class='del'>-	if (mad_send_wr-&gt;status != IB_WC_SUCCESS )</div><div class='del'>-		mad_send_wc-&gt;status = mad_send_wr-&gt;status;</div><div class='del'>-	if (ret == IB_RMPP_RESULT_INTERNAL)</div><div class='add'>+	if (ret == IB_RMPP_RESULT_INTERNAL) {</div><div class='ctx'> 		ib_rmpp_send_handler(mad_send_wc);</div><div class='del'>-	else</div><div class='add'>+	} else {</div><div class='add'>+		if (mad_send_wr-&gt;is_solicited_fc)</div><div class='add'>+			process_backlog_mads(mad_agent_priv);</div><div class='ctx'> 		mad_agent_priv-&gt;agent.send_handler(&amp;mad_agent_priv-&gt;agent,</div><div class='ctx'> 						   mad_send_wc);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	/* Release reference on agent taken when sending */</div><div class='ctx'> 	deref_mad_agent(mad_agent_priv);</div><div class='hunk'>@@ -2557,9 +2551,6 @@ retry:</div><div class='ctx'> 	mad_send_wc.send_buf = &amp;mad_send_wr-&gt;send_buf;</div><div class='ctx'> 	mad_send_wc.status = wc-&gt;status;</div><div class='ctx'> 	mad_send_wc.vendor_err = wc-&gt;vendor_err;</div><div class='del'>-	if (atomic_read(&amp;qp_info-&gt;snoop_count))</div><div class='del'>-		snoop_send(qp_info, &amp;mad_send_wr-&gt;send_buf, &amp;mad_send_wc,</div><div class='del'>-			   IB_MAD_SNOOP_SEND_COMPLETIONS);</div><div class='ctx'> 	ib_mad_complete_send_wr(mad_send_wr, &amp;mad_send_wc);</div><div class='ctx'> </div><div class='ctx'> 	if (queued_send_wr) {</div><div class='hunk'>@@ -2640,40 +2631,53 @@ static bool ib_mad_send_error(struct ib_mad_port_private *port_priv,</div><div class='ctx'> 	return true;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void clear_mad_error_list(struct list_head *list,</div><div class='add'>+				 enum ib_wc_status wc_status,</div><div class='add'>+				 struct ib_mad_agent_private *mad_agent_priv)</div><div class='add'>+{</div><div class='add'>+	struct ib_mad_send_wr_private *mad_send_wr, *n;</div><div class='add'>+	struct ib_mad_send_wc mad_send_wc;</div><div class='add'>+</div><div class='add'>+	mad_send_wc.status = wc_status;</div><div class='add'>+	mad_send_wc.vendor_err = 0;</div><div class='add'>+</div><div class='add'>+	list_for_each_entry_safe(mad_send_wr, n, list, agent_list) {</div><div class='add'>+		mad_send_wc.send_buf = &amp;mad_send_wr-&gt;send_buf;</div><div class='add'>+		mad_agent_priv-&gt;agent.send_handler(&amp;mad_agent_priv-&gt;agent,</div><div class='add'>+						   &amp;mad_send_wc);</div><div class='add'>+		deref_mad_agent(mad_agent_priv);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	struct ib_mad_send_wr_private *mad_send_wr, *temp_mad_send_wr;</div><div class='del'>-	struct ib_mad_send_wc mad_send_wc;</div><div class='ctx'> 	struct list_head cancel_list;</div><div class='ctx'> </div><div class='ctx'> 	INIT_LIST_HEAD(&amp;cancel_list);</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='ctx'> 	list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,</div><div class='del'>-				 &amp;mad_agent_priv-&gt;send_list, agent_list) {</div><div class='del'>-		if (mad_send_wr-&gt;status == IB_WC_SUCCESS) {</div><div class='del'>-			mad_send_wr-&gt;status = IB_WC_WR_FLUSH_ERR;</div><div class='del'>-			mad_send_wr-&gt;refcount -= (mad_send_wr-&gt;timeout &gt; 0);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Empty wait list to prevent receives from finding a request */</div><div class='del'>-	list_splice_init(&amp;mad_agent_priv-&gt;wait_list, &amp;cancel_list);</div><div class='del'>-	spin_unlock_irqrestore(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='add'>+				 &amp;mad_agent_priv-&gt;send_list, agent_list)</div><div class='add'>+		change_mad_state(mad_send_wr, IB_MAD_STATE_CANCELED);</div><div class='ctx'> </div><div class='del'>-	/* Report all cancelled requests */</div><div class='del'>-	mad_send_wc.status = IB_WC_WR_FLUSH_ERR;</div><div class='del'>-	mad_send_wc.vendor_err = 0;</div><div class='add'>+	/* Empty wait &amp; backlog list to prevent receives from finding request */</div><div class='add'>+	list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,</div><div class='add'>+				 &amp;mad_agent_priv-&gt;wait_list, agent_list) {</div><div class='add'>+		change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);</div><div class='add'>+		list_add_tail(&amp;mad_send_wr-&gt;agent_list, &amp;cancel_list);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,</div><div class='del'>-				 &amp;cancel_list, agent_list) {</div><div class='del'>-		mad_send_wc.send_buf = &amp;mad_send_wr-&gt;send_buf;</div><div class='del'>-		list_del(&amp;mad_send_wr-&gt;agent_list);</div><div class='del'>-		mad_agent_priv-&gt;agent.send_handler(&amp;mad_agent_priv-&gt;agent,</div><div class='del'>-						   &amp;mad_send_wc);</div><div class='del'>-		atomic_dec(&amp;mad_agent_priv-&gt;refcount);</div><div class='add'>+				 &amp;mad_agent_priv-&gt;backlog_list, agent_list) {</div><div class='add'>+		change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);</div><div class='add'>+		list_add_tail(&amp;mad_send_wr-&gt;agent_list, &amp;cancel_list);</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	spin_unlock_irqrestore(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='add'>+	/* Report all cancelled requests */</div><div class='add'>+	clear_mad_error_list(&amp;cancel_list, IB_WC_WR_FLUSH_ERR, mad_agent_priv);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static struct ib_mad_send_wr_private*</div><div class='hunk'>@@ -2695,31 +2699,40 @@ find_send_wr(struct ib_mad_agent_private *mad_agent_priv,</div><div class='ctx'> 		    &amp;mad_send_wr-&gt;send_buf == send_buf)</div><div class='ctx'> 			return mad_send_wr;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	list_for_each_entry(mad_send_wr, &amp;mad_agent_priv-&gt;backlog_list,</div><div class='add'>+			    agent_list) {</div><div class='add'>+		if (&amp;mad_send_wr-&gt;send_buf == send_buf)</div><div class='add'>+			return mad_send_wr;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	return NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int ib_modify_mad(struct ib_mad_agent *mad_agent,</div><div class='del'>-		  struct ib_mad_send_buf *send_buf, u32 timeout_ms)</div><div class='add'>+int ib_modify_mad(struct ib_mad_send_buf *send_buf, u32 timeout_ms)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_mad_agent_private *mad_agent_priv;</div><div class='ctx'> 	struct ib_mad_send_wr_private *mad_send_wr;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int active;</div><div class='ctx'> </div><div class='del'>-	mad_agent_priv = container_of(mad_agent, struct ib_mad_agent_private,</div><div class='del'>-				      agent);</div><div class='add'>+	if (!send_buf)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	mad_agent_priv = container_of(send_buf-&gt;mad_agent,</div><div class='add'>+				      struct ib_mad_agent_private, agent);</div><div class='ctx'> 	spin_lock_irqsave(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='ctx'> 	mad_send_wr = find_send_wr(mad_agent_priv, send_buf);</div><div class='del'>-	if (!mad_send_wr || mad_send_wr-&gt;status != IB_WC_SUCCESS) {</div><div class='add'>+	if (!mad_send_wr || mad_send_wr-&gt;state == IB_MAD_STATE_CANCELED) {</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	active = (!mad_send_wr-&gt;timeout || mad_send_wr-&gt;refcount &gt; 1);</div><div class='del'>-	if (!timeout_ms) {</div><div class='del'>-		mad_send_wr-&gt;status = IB_WC_WR_FLUSH_ERR;</div><div class='del'>-		mad_send_wr-&gt;refcount -= (mad_send_wr-&gt;timeout &gt; 0);</div><div class='del'>-	}</div><div class='add'>+	active = ((mad_send_wr-&gt;state == IB_MAD_STATE_SEND_START) ||</div><div class='add'>+		  (mad_send_wr-&gt;state == IB_MAD_STATE_EARLY_RESP) ||</div><div class='add'>+		  (mad_send_wr-&gt;state == IB_MAD_STATE_QUEUED &amp;&amp; timeout_ms));</div><div class='add'>+	if (!timeout_ms)</div><div class='add'>+		change_mad_state(mad_send_wr, IB_MAD_STATE_CANCELED);</div><div class='ctx'> </div><div class='ctx'> 	mad_send_wr-&gt;send_buf.timeout_ms = timeout_ms;</div><div class='ctx'> 	if (active)</div><div class='hunk'>@@ -2732,13 +2745,6 @@ int ib_modify_mad(struct ib_mad_agent *mad_agent,</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_modify_mad);</div><div class='ctx'> </div><div class='del'>-void ib_cancel_mad(struct ib_mad_agent *mad_agent,</div><div class='del'>-		   struct ib_mad_send_buf *send_buf)</div><div class='del'>-{</div><div class='del'>-	ib_modify_mad(mad_agent, send_buf, 0);</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_cancel_mad);</div><div class='del'>-</div><div class='ctx'> static void local_completions(struct work_struct *work)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_mad_agent_private *mad_agent_priv;</div><div class='hunk'>@@ -2801,16 +2807,12 @@ static void local_completions(struct work_struct *work)</div><div class='ctx'> 			local-&gt;mad_priv-&gt;header.recv_wc.recv_buf.grh = NULL;</div><div class='ctx'> 			local-&gt;mad_priv-&gt;header.recv_wc.recv_buf.mad =</div><div class='ctx'> 						(struct ib_mad *)local-&gt;mad_priv-&gt;mad;</div><div class='del'>-			if (atomic_read(&amp;recv_mad_agent-&gt;qp_info-&gt;snoop_count))</div><div class='del'>-				snoop_recv(recv_mad_agent-&gt;qp_info,</div><div class='del'>-					  &amp;local-&gt;mad_priv-&gt;header.recv_wc,</div><div class='del'>-					   IB_MAD_SNOOP_RECVS);</div><div class='ctx'> 			recv_mad_agent-&gt;agent.recv_handler(</div><div class='ctx'> 						&amp;recv_mad_agent-&gt;agent,</div><div class='ctx'> 						&amp;local-&gt;mad_send_wr-&gt;send_buf,</div><div class='ctx'> 						&amp;local-&gt;mad_priv-&gt;header.recv_wc);</div><div class='ctx'> 			spin_lock_irqsave(&amp;recv_mad_agent-&gt;lock, flags);</div><div class='del'>-			atomic_dec(&amp;recv_mad_agent-&gt;refcount);</div><div class='add'>+			deref_mad_agent(recv_mad_agent);</div><div class='ctx'> 			spin_unlock_irqrestore(&amp;recv_mad_agent-&gt;lock, flags);</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='hunk'>@@ -2819,15 +2821,11 @@ local_send_completion:</div><div class='ctx'> 		mad_send_wc.status = IB_WC_SUCCESS;</div><div class='ctx'> 		mad_send_wc.vendor_err = 0;</div><div class='ctx'> 		mad_send_wc.send_buf = &amp;local-&gt;mad_send_wr-&gt;send_buf;</div><div class='del'>-		if (atomic_read(&amp;mad_agent_priv-&gt;qp_info-&gt;snoop_count))</div><div class='del'>-			snoop_send(mad_agent_priv-&gt;qp_info,</div><div class='del'>-				   &amp;local-&gt;mad_send_wr-&gt;send_buf,</div><div class='del'>-				   &amp;mad_send_wc, IB_MAD_SNOOP_SEND_COMPLETIONS);</div><div class='ctx'> 		mad_agent_priv-&gt;agent.send_handler(&amp;mad_agent_priv-&gt;agent,</div><div class='ctx'> 						   &amp;mad_send_wc);</div><div class='ctx'> </div><div class='ctx'> 		spin_lock_irqsave(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='del'>-		atomic_dec(&amp;mad_agent_priv-&gt;refcount);</div><div class='add'>+		deref_mad_agent(mad_agent_priv);</div><div class='ctx'> 		if (free_mad)</div><div class='ctx'> 			kfree(local-&gt;mad_priv);</div><div class='ctx'> 		kfree(local);</div><div class='hunk'>@@ -2846,6 +2844,11 @@ static int retry_send(struct ib_mad_send_wr_private *mad_send_wr)</div><div class='ctx'> 	mad_send_wr-&gt;send_buf.retries++;</div><div class='ctx'> </div><div class='ctx'> 	mad_send_wr-&gt;timeout = msecs_to_jiffies(mad_send_wr-&gt;send_buf.timeout_ms);</div><div class='add'>+	if (mad_send_wr-&gt;is_solicited_fc &amp;&amp;</div><div class='add'>+	    !list_empty(&amp;mad_send_wr-&gt;mad_agent_priv-&gt;backlog_list)) {</div><div class='add'>+		change_mad_state(mad_send_wr, IB_MAD_STATE_QUEUED);</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	if (ib_mad_kernel_rmpp_agent(&amp;mad_send_wr-&gt;mad_agent_priv-&gt;agent)) {</div><div class='ctx'> 		ret = ib_retry_rmpp(mad_send_wr);</div><div class='hunk'>@@ -2863,24 +2866,25 @@ static int retry_send(struct ib_mad_send_wr_private *mad_send_wr)</div><div class='ctx'> 	} else</div><div class='ctx'> 		ret = ib_send_mad(mad_send_wr);</div><div class='ctx'> </div><div class='del'>-	if (!ret) {</div><div class='del'>-		mad_send_wr-&gt;refcount++;</div><div class='del'>-		list_add_tail(&amp;mad_send_wr-&gt;agent_list,</div><div class='del'>-			      &amp;mad_send_wr-&gt;mad_agent_priv-&gt;send_list);</div><div class='del'>-	}</div><div class='add'>+	if (!ret)</div><div class='add'>+		change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);</div><div class='add'>+</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void timeout_sends(struct work_struct *work)</div><div class='ctx'> {</div><div class='del'>-	struct ib_mad_agent_private *mad_agent_priv;</div><div class='ctx'> 	struct ib_mad_send_wr_private *mad_send_wr;</div><div class='del'>-	struct ib_mad_send_wc mad_send_wc;</div><div class='add'>+	struct ib_mad_agent_private *mad_agent_priv;</div><div class='add'>+	struct list_head timeout_list;</div><div class='add'>+	struct list_head cancel_list;</div><div class='add'>+	struct list_head *list_item;</div><div class='ctx'> 	unsigned long flags, delay;</div><div class='ctx'> </div><div class='ctx'> 	mad_agent_priv = container_of(work, struct ib_mad_agent_private,</div><div class='ctx'> 				      timed_work.work);</div><div class='del'>-	mad_send_wc.vendor_err = 0;</div><div class='add'>+	INIT_LIST_HEAD(&amp;timeout_list);</div><div class='add'>+	INIT_LIST_HEAD(&amp;cancel_list);</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='ctx'> 	while (!list_empty(&amp;mad_agent_priv-&gt;wait_list)) {</div><div class='hunk'>@@ -2898,25 +2902,22 @@ static void timeout_sends(struct work_struct *work)</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		list_del(&amp;mad_send_wr-&gt;agent_list);</div><div class='del'>-		if (mad_send_wr-&gt;status == IB_WC_SUCCESS &amp;&amp;</div><div class='del'>-		    !retry_send(mad_send_wr))</div><div class='del'>-			continue;</div><div class='del'>-</div><div class='del'>-		spin_unlock_irqrestore(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-		if (mad_send_wr-&gt;status == IB_WC_SUCCESS)</div><div class='del'>-			mad_send_wc.status = IB_WC_RESP_TIMEOUT_ERR;</div><div class='add'>+		if (mad_send_wr-&gt;state == IB_MAD_STATE_CANCELED)</div><div class='add'>+			list_item = &amp;cancel_list;</div><div class='add'>+		else if (retry_send(mad_send_wr))</div><div class='add'>+			list_item = &amp;timeout_list;</div><div class='ctx'> 		else</div><div class='del'>-			mad_send_wc.status = mad_send_wr-&gt;status;</div><div class='del'>-		mad_send_wc.send_buf = &amp;mad_send_wr-&gt;send_buf;</div><div class='del'>-		mad_agent_priv-&gt;agent.send_handler(&amp;mad_agent_priv-&gt;agent,</div><div class='del'>-						   &amp;mad_send_wc);</div><div class='add'>+			continue;</div><div class='ctx'> </div><div class='del'>-		atomic_dec(&amp;mad_agent_priv-&gt;refcount);</div><div class='del'>-		spin_lock_irqsave(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='add'>+		change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);</div><div class='add'>+		list_add_tail(&amp;mad_send_wr-&gt;agent_list, list_item);</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;mad_agent_priv-&gt;lock, flags);</div><div class='add'>+	process_backlog_mads(mad_agent_priv);</div><div class='add'>+	clear_mad_error_list(&amp;timeout_list, IB_WC_RESP_TIMEOUT_ERR,</div><div class='add'>+			     mad_agent_priv);</div><div class='add'>+	clear_mad_error_list(&amp;cancel_list, IB_WC_WR_FLUSH_ERR, mad_agent_priv);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -2926,11 +2927,11 @@ static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info,</div><div class='ctx'> 				    struct ib_mad_private *mad)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	int post, ret;</div><div class='ctx'> 	struct ib_mad_private *mad_priv;</div><div class='ctx'> 	struct ib_sge sg_list;</div><div class='ctx'> 	struct ib_recv_wr recv_wr;</div><div class='ctx'> 	struct ib_mad_queue *recv_queue = &amp;qp_info-&gt;recv_queue;</div><div class='add'>+	int ret = 0;</div><div class='ctx'> </div><div class='ctx'> 	/* Initialize common scatter list fields */</div><div class='ctx'> 	sg_list.lkey = qp_info-&gt;port_priv-&gt;pd-&gt;local_dma_lkey;</div><div class='hunk'>@@ -2940,7 +2941,7 @@ static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info,</div><div class='ctx'> 	recv_wr.sg_list = &amp;sg_list;</div><div class='ctx'> 	recv_wr.num_sge = 1;</div><div class='ctx'> </div><div class='del'>-	do {</div><div class='add'>+	while (true) {</div><div class='ctx'> 		/* Allocate and map receive buffer */</div><div class='ctx'> 		if (mad) {</div><div class='ctx'> 			mad_priv = mad;</div><div class='hunk'>@@ -2948,10 +2949,8 @@ static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info,</div><div class='ctx'> 		} else {</div><div class='ctx'> 			mad_priv = alloc_mad_private(port_mad_size(qp_info-&gt;port_priv),</div><div class='ctx'> 						     GFP_ATOMIC);</div><div class='del'>-			if (!mad_priv) {</div><div class='del'>-				ret = -ENOMEM;</div><div class='del'>-				break;</div><div class='del'>-			}</div><div class='add'>+			if (!mad_priv)</div><div class='add'>+				return -ENOMEM;</div><div class='ctx'> 		}</div><div class='ctx'> 		sg_list.length = mad_priv_dma_size(mad_priv);</div><div class='ctx'> 		sg_list.addr = ib_dma_map_single(qp_info-&gt;port_priv-&gt;device,</div><div class='hunk'>@@ -2961,35 +2960,40 @@ static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info,</div><div class='ctx'> 		if (unlikely(ib_dma_mapping_error(qp_info-&gt;port_priv-&gt;device,</div><div class='ctx'> 						  sg_list.addr))) {</div><div class='ctx'> 			ret = -ENOMEM;</div><div class='del'>-			break;</div><div class='add'>+			goto free_mad_priv;</div><div class='ctx'> 		}</div><div class='ctx'> 		mad_priv-&gt;header.mapping = sg_list.addr;</div><div class='ctx'> 		mad_priv-&gt;header.mad_list.mad_queue = recv_queue;</div><div class='ctx'> 		mad_priv-&gt;header.mad_list.cqe.done = ib_mad_recv_done;</div><div class='ctx'> 		recv_wr.wr_cqe = &amp;mad_priv-&gt;header.mad_list.cqe;</div><div class='del'>-</div><div class='del'>-		/* Post receive WR */</div><div class='ctx'> 		spin_lock_irqsave(&amp;recv_queue-&gt;lock, flags);</div><div class='del'>-		post = (++recv_queue-&gt;count &lt; recv_queue-&gt;max_active);</div><div class='del'>-		list_add_tail(&amp;mad_priv-&gt;header.mad_list.list, &amp;recv_queue-&gt;list);</div><div class='add'>+		if (recv_queue-&gt;count &gt;= recv_queue-&gt;max_active) {</div><div class='add'>+			/* Fully populated the receive queue */</div><div class='add'>+			spin_unlock_irqrestore(&amp;recv_queue-&gt;lock, flags);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		recv_queue-&gt;count++;</div><div class='add'>+		list_add_tail(&amp;mad_priv-&gt;header.mad_list.list,</div><div class='add'>+			      &amp;recv_queue-&gt;list);</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;recv_queue-&gt;lock, flags);</div><div class='add'>+</div><div class='ctx'> 		ret = ib_post_recv(qp_info-&gt;qp, &amp;recv_wr, NULL);</div><div class='ctx'> 		if (ret) {</div><div class='ctx'> 			spin_lock_irqsave(&amp;recv_queue-&gt;lock, flags);</div><div class='ctx'> 			list_del(&amp;mad_priv-&gt;header.mad_list.list);</div><div class='ctx'> 			recv_queue-&gt;count--;</div><div class='ctx'> 			spin_unlock_irqrestore(&amp;recv_queue-&gt;lock, flags);</div><div class='del'>-			ib_dma_unmap_single(qp_info-&gt;port_priv-&gt;device,</div><div class='del'>-					    mad_priv-&gt;header.mapping,</div><div class='del'>-					    mad_priv_dma_size(mad_priv),</div><div class='del'>-					    DMA_FROM_DEVICE);</div><div class='del'>-			kfree(mad_priv);</div><div class='ctx'> 			dev_err(&amp;qp_info-&gt;port_priv-&gt;device-&gt;dev,</div><div class='ctx'> 				"ib_post_recv failed: %d\n", ret);</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='del'>-	} while (post);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='add'>+	ib_dma_unmap_single(qp_info-&gt;port_priv-&gt;device,</div><div class='add'>+			    mad_priv-&gt;header.mapping,</div><div class='add'>+			    mad_priv_dma_size(mad_priv), DMA_FROM_DEVICE);</div><div class='add'>+free_mad_priv:</div><div class='add'>+	kfree(mad_priv);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -3118,7 +3122,7 @@ static void qp_event_handler(struct ib_event *event, void *qp_context)</div><div class='ctx'> </div><div class='ctx'> 	/* It's worse than that! He's dead, Jim! */</div><div class='ctx'> 	dev_err(&amp;qp_info-&gt;port_priv-&gt;device-&gt;dev,</div><div class='del'>-		"Fatal error (%d) on MAD QP (%d)\n",</div><div class='add'>+		"Fatal error (%d) on MAD QP (%u)\n",</div><div class='ctx'> 		event-&gt;event, qp_info-&gt;qp-&gt;qp_num);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -3138,10 +3142,6 @@ static void init_mad_qp(struct ib_mad_port_private *port_priv,</div><div class='ctx'> 	init_mad_queue(qp_info, &amp;qp_info-&gt;send_queue);</div><div class='ctx'> 	init_mad_queue(qp_info, &amp;qp_info-&gt;recv_queue);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;qp_info-&gt;overflow_list);</div><div class='del'>-	spin_lock_init(&amp;qp_info-&gt;snoop_lock);</div><div class='del'>-	qp_info-&gt;snoop_table = NULL;</div><div class='del'>-	qp_info-&gt;snoop_table_size = 0;</div><div class='del'>-	atomic_set(&amp;qp_info-&gt;snoop_count, 0);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int create_mad_qp(struct ib_mad_qp_info *qp_info,</div><div class='hunk'>@@ -3185,7 +3185,6 @@ static void destroy_mad_qp(struct ib_mad_qp_info *qp_info)</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='ctx'> 	ib_destroy_qp(qp_info-&gt;qp);</div><div class='del'>-	kfree(qp_info-&gt;snoop_table);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -3193,12 +3192,11 @@ static void destroy_mad_qp(struct ib_mad_qp_info *qp_info)</div><div class='ctx'>  * Create the QP, PD, MR, and CQ if needed</div><div class='ctx'>  */</div><div class='ctx'> static int ib_mad_port_open(struct ib_device *device,</div><div class='del'>-			    int port_num)</div><div class='add'>+			    u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	int ret, cq_size;</div><div class='ctx'> 	struct ib_mad_port_private *port_priv;</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	char name[sizeof "ib_mad123"];</div><div class='ctx'> 	int has_smi;</div><div class='ctx'> </div><div class='ctx'> 	if (WARN_ON(rdma_max_mad_size(device, port_num) &lt; IB_MGMT_MAD_SIZE))</div><div class='hunk'>@@ -3224,18 +3222,18 @@ static int ib_mad_port_open(struct ib_device *device,</div><div class='ctx'> 	if (has_smi)</div><div class='ctx'> 		cq_size *= 2;</div><div class='ctx'> </div><div class='add'>+	port_priv-&gt;pd = ib_alloc_pd(device, 0);</div><div class='add'>+	if (IS_ERR(port_priv-&gt;pd)) {</div><div class='add'>+		dev_err(&amp;device-&gt;dev, "Couldn't create ib_mad PD\n");</div><div class='add'>+		ret = PTR_ERR(port_priv-&gt;pd);</div><div class='add'>+		goto error3;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	port_priv-&gt;cq = ib_alloc_cq(port_priv-&gt;device, port_priv, cq_size, 0,</div><div class='ctx'> 			IB_POLL_UNBOUND_WORKQUEUE);</div><div class='ctx'> 	if (IS_ERR(port_priv-&gt;cq)) {</div><div class='ctx'> 		dev_err(&amp;device-&gt;dev, "Couldn't create ib_mad CQ\n");</div><div class='ctx'> 		ret = PTR_ERR(port_priv-&gt;cq);</div><div class='del'>-		goto error3;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	port_priv-&gt;pd = ib_alloc_pd(device, 0);</div><div class='del'>-	if (IS_ERR(port_priv-&gt;pd)) {</div><div class='del'>-		dev_err(&amp;device-&gt;dev, "Couldn't create ib_mad PD\n");</div><div class='del'>-		ret = PTR_ERR(port_priv-&gt;pd);</div><div class='ctx'> 		goto error4;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -3244,12 +3242,15 @@ static int ib_mad_port_open(struct ib_device *device,</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto error6;</div><div class='ctx'> 	}</div><div class='del'>-	ret = create_mad_qp(&amp;port_priv-&gt;qp_info[1], IB_QPT_GSI);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto error7;</div><div class='ctx'> </div><div class='del'>-	snprintf(name, sizeof name, "ib_mad%d", port_num);</div><div class='del'>-	port_priv-&gt;wq = alloc_ordered_workqueue(name, WQ_MEM_RECLAIM);</div><div class='add'>+	if (rdma_cap_ib_cm(device, port_num)) {</div><div class='add'>+		ret = create_mad_qp(&amp;port_priv-&gt;qp_info[1], IB_QPT_GSI);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto error7;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	port_priv-&gt;wq = alloc_ordered_workqueue("ib_mad%u", WQ_MEM_RECLAIM,</div><div class='add'>+						port_num);</div><div class='ctx'> 	if (!port_priv-&gt;wq) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='ctx'> 		goto error8;</div><div class='hunk'>@@ -3278,11 +3279,11 @@ error8:</div><div class='ctx'> error7:</div><div class='ctx'> 	destroy_mad_qp(&amp;port_priv-&gt;qp_info[0]);</div><div class='ctx'> error6:</div><div class='del'>-	ib_dealloc_pd(port_priv-&gt;pd);</div><div class='del'>-error4:</div><div class='ctx'> 	ib_free_cq(port_priv-&gt;cq);</div><div class='ctx'> 	cleanup_recv_queue(&amp;port_priv-&gt;qp_info[1]);</div><div class='ctx'> 	cleanup_recv_queue(&amp;port_priv-&gt;qp_info[0]);</div><div class='add'>+error4:</div><div class='add'>+	ib_dealloc_pd(port_priv-&gt;pd);</div><div class='ctx'> error3:</div><div class='ctx'> 	kfree(port_priv);</div><div class='ctx'> </div><div class='hunk'>@@ -3294,7 +3295,7 @@ error3:</div><div class='ctx'>  * If there are no classes using the port, free the port</div><div class='ctx'>  * resources (CQ, MR, PD, QP) and remove the port's info structure</div><div class='ctx'>  */</div><div class='del'>-static int ib_mad_port_close(struct ib_device *device, int port_num)</div><div class='add'>+static int ib_mad_port_close(struct ib_device *device, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_mad_port_private *port_priv;</div><div class='ctx'> 	unsigned long flags;</div><div class='hunk'>@@ -3303,7 +3304,7 @@ static int ib_mad_port_close(struct ib_device *device, int port_num)</div><div class='ctx'> 	port_priv = __ib_get_mad_port(device, port_num);</div><div class='ctx'> 	if (port_priv == NULL) {</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;ib_mad_port_list_lock, flags);</div><div class='del'>-		dev_err(&amp;device-&gt;dev, "Port %d not found\n", port_num);</div><div class='add'>+		dev_err(&amp;device-&gt;dev, "Port %u not found\n", port_num);</div><div class='ctx'> 		return -ENODEV;</div><div class='ctx'> 	}</div><div class='ctx'> 	list_del_init(&amp;port_priv-&gt;port_list);</div><div class='hunk'>@@ -3312,8 +3313,8 @@ static int ib_mad_port_close(struct ib_device *device, int port_num)</div><div class='ctx'> 	destroy_workqueue(port_priv-&gt;wq);</div><div class='ctx'> 	destroy_mad_qp(&amp;port_priv-&gt;qp_info[1]);</div><div class='ctx'> 	destroy_mad_qp(&amp;port_priv-&gt;qp_info[0]);</div><div class='del'>-	ib_dealloc_pd(port_priv-&gt;pd);</div><div class='ctx'> 	ib_free_cq(port_priv-&gt;cq);</div><div class='add'>+	ib_dealloc_pd(port_priv-&gt;pd);</div><div class='ctx'> 	cleanup_recv_queue(&amp;port_priv-&gt;qp_info[1]);</div><div class='ctx'> 	cleanup_recv_queue(&amp;port_priv-&gt;qp_info[0]);</div><div class='ctx'> 	/* XXX: Handle deallocation of MAD registration tables */</div><div class='hunk'>@@ -3323,9 +3324,11 @@ static int ib_mad_port_close(struct ib_device *device, int port_num)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void ib_mad_init_device(struct ib_device *device)</div><div class='add'>+static int ib_mad_init_device(struct ib_device *device)</div><div class='ctx'> {</div><div class='ctx'> 	int start, i;</div><div class='add'>+	unsigned int count = 0;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	start = rdma_start_port(device);</div><div class='ctx'> </div><div class='hunk'>@@ -3333,17 +3336,23 @@ static void ib_mad_init_device(struct ib_device *device)</div><div class='ctx'> 		if (!rdma_cap_ib_mad(device, i))</div><div class='ctx'> 			continue;</div><div class='ctx'> </div><div class='del'>-		if (ib_mad_port_open(device, i)) {</div><div class='add'>+		ret = ib_mad_port_open(device, i);</div><div class='add'>+		if (ret) {</div><div class='ctx'> 			dev_err(&amp;device-&gt;dev, "Couldn't open port %d\n", i);</div><div class='ctx'> 			goto error;</div><div class='ctx'> 		}</div><div class='del'>-		if (ib_agent_port_open(device, i)) {</div><div class='add'>+		ret = ib_agent_port_open(device, i);</div><div class='add'>+		if (ret) {</div><div class='ctx'> 			dev_err(&amp;device-&gt;dev,</div><div class='ctx'> 				"Couldn't open port %d for agents\n", i);</div><div class='ctx'> 			goto error_agent;</div><div class='ctx'> 		}</div><div class='add'>+		count++;</div><div class='ctx'> 	}</div><div class='del'>-	return;</div><div class='add'>+	if (!count)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='ctx'> error_agent:</div><div class='ctx'> 	if (ib_mad_port_close(device, i))</div><div class='hunk'>@@ -3360,6 +3369,7 @@ error:</div><div class='ctx'> 		if (ib_mad_port_close(device, i))</div><div class='ctx'> 			dev_err(&amp;device-&gt;dev, "Couldn't close port %d\n", i);</div><div class='ctx'> 	}</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void ib_mad_remove_device(struct ib_device *device, void *client_data)</div><div class='hunk'>@@ -3372,9 +3382,9 @@ static void ib_mad_remove_device(struct ib_device *device, void *client_data)</div><div class='ctx'> </div><div class='ctx'> 		if (ib_agent_port_close(device, i))</div><div class='ctx'> 			dev_err(&amp;device-&gt;dev,</div><div class='del'>-				"Couldn't close port %d for agents\n", i);</div><div class='add'>+				"Couldn't close port %u for agents\n", i);</div><div class='ctx'> 		if (ib_mad_port_close(device, i))</div><div class='del'>-			dev_err(&amp;device-&gt;dev, "Couldn't close port %d\n", i);</div><div class='add'>+			dev_err(&amp;device-&gt;dev, "Couldn't close port %u\n", i);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/core/mad_priv.h b/drivers/infiniband/core/mad_priv.h<br/>index 956b3a7dfed7..f444357d33f4 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/mad_priv.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/mad_priv.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/mad_priv.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/mad_priv.h</a></div><div class='hunk'>@@ -79,13 +79,13 @@ struct ib_mad_private {</div><div class='ctx'> 	struct ib_mad_private_header header;</div><div class='ctx'> 	size_t mad_size;</div><div class='ctx'> 	struct ib_grh grh;</div><div class='del'>-	u8 mad[0];</div><div class='add'>+	u8 mad[];</div><div class='ctx'> } __packed;</div><div class='ctx'> </div><div class='ctx'> struct ib_rmpp_segment {</div><div class='ctx'> 	struct list_head list;</div><div class='ctx'> 	u32 num;</div><div class='del'>-	u8 data[0];</div><div class='add'>+	u8 data[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct ib_mad_agent_private {</div><div class='hunk'>@@ -95,15 +95,18 @@ struct ib_mad_agent_private {</div><div class='ctx'> </div><div class='ctx'> 	spinlock_t lock;</div><div class='ctx'> 	struct list_head send_list;</div><div class='add'>+	unsigned int sol_fc_send_count;</div><div class='ctx'> 	struct list_head wait_list;</div><div class='del'>-	struct list_head done_list;</div><div class='add'>+	unsigned int sol_fc_wait_count;</div><div class='ctx'> 	struct delayed_work timed_work;</div><div class='ctx'> 	unsigned long timeout;</div><div class='ctx'> 	struct list_head local_list;</div><div class='ctx'> 	struct work_struct local_work;</div><div class='ctx'> 	struct list_head rmpp_list;</div><div class='add'>+	unsigned int sol_fc_max;</div><div class='add'>+	struct list_head backlog_list;</div><div class='ctx'> </div><div class='del'>-	atomic_t refcount;</div><div class='add'>+	refcount_t refcount;</div><div class='ctx'> 	union {</div><div class='ctx'> 		struct completion comp;</div><div class='ctx'> 		struct rcu_head rcu;</div><div class='hunk'>@@ -115,10 +118,35 @@ struct ib_mad_snoop_private {</div><div class='ctx'> 	struct ib_mad_qp_info *qp_info;</div><div class='ctx'> 	int snoop_index;</div><div class='ctx'> 	int mad_snoop_flags;</div><div class='del'>-	atomic_t refcount;</div><div class='ctx'> 	struct completion comp;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+enum ib_mad_state {</div><div class='add'>+	/* MAD is in the making and is not yet in any list */</div><div class='add'>+	IB_MAD_STATE_INIT,</div><div class='add'>+	/* MAD is in backlog list */</div><div class='add'>+	IB_MAD_STATE_QUEUED,</div><div class='add'>+	/*</div><div class='add'>+	 * MAD was sent to the QP and is waiting for completion</div><div class='add'>+	 * notification in send list.</div><div class='add'>+	 */</div><div class='add'>+	IB_MAD_STATE_SEND_START,</div><div class='add'>+	/*</div><div class='add'>+	 * MAD send completed successfully, waiting for a response</div><div class='add'>+	 * in wait list.</div><div class='add'>+	 */</div><div class='add'>+	IB_MAD_STATE_WAIT_RESP,</div><div class='add'>+	/*</div><div class='add'>+	 * Response came early, before send completion notification,</div><div class='add'>+	 * in send list.</div><div class='add'>+	 */</div><div class='add'>+	IB_MAD_STATE_EARLY_RESP,</div><div class='add'>+	/* MAD was canceled while in wait or send list */</div><div class='add'>+	IB_MAD_STATE_CANCELED,</div><div class='add'>+	/* MAD processing completed, MAD in no list */</div><div class='add'>+	IB_MAD_STATE_DONE</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct ib_mad_send_wr_private {</div><div class='ctx'> 	struct ib_mad_list_head mad_list;</div><div class='ctx'> 	struct list_head agent_list;</div><div class='hunk'>@@ -133,8 +161,6 @@ struct ib_mad_send_wr_private {</div><div class='ctx'> 	int max_retries;</div><div class='ctx'> 	int retries_left;</div><div class='ctx'> 	int retry;</div><div class='del'>-	int refcount;</div><div class='del'>-	enum ib_wc_status status;</div><div class='ctx'> </div><div class='ctx'> 	/* RMPP control */</div><div class='ctx'> 	struct list_head rmpp_list;</div><div class='hunk'>@@ -144,8 +170,48 @@ struct ib_mad_send_wr_private {</div><div class='ctx'> 	int seg_num;</div><div class='ctx'> 	int newwin;</div><div class='ctx'> 	int pad;</div><div class='add'>+</div><div class='add'>+	enum ib_mad_state state;</div><div class='add'>+</div><div class='add'>+	/* Solicited MAD flow control */</div><div class='add'>+	bool is_solicited_fc;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static inline void expect_mad_state(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='add'>+				    enum ib_mad_state expected_state)</div><div class='add'>+{</div><div class='add'>+	if (IS_ENABLED(CONFIG_LOCKDEP))</div><div class='add'>+		WARN_ON(mad_send_wr-&gt;state != expected_state);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void expect_mad_state2(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='add'>+				     enum ib_mad_state expected_state1,</div><div class='add'>+				     enum ib_mad_state expected_state2)</div><div class='add'>+{</div><div class='add'>+	if (IS_ENABLED(CONFIG_LOCKDEP))</div><div class='add'>+		WARN_ON(mad_send_wr-&gt;state != expected_state1 &amp;&amp;</div><div class='add'>+			mad_send_wr-&gt;state != expected_state2);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void expect_mad_state3(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='add'>+				     enum ib_mad_state expected_state1,</div><div class='add'>+				     enum ib_mad_state expected_state2,</div><div class='add'>+				     enum ib_mad_state expected_state3)</div><div class='add'>+{</div><div class='add'>+	if (IS_ENABLED(CONFIG_LOCKDEP))</div><div class='add'>+		WARN_ON(mad_send_wr-&gt;state != expected_state1 &amp;&amp;</div><div class='add'>+			mad_send_wr-&gt;state != expected_state2 &amp;&amp;</div><div class='add'>+			mad_send_wr-&gt;state != expected_state3);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void</div><div class='add'>+not_expect_mad_state(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='add'>+		     enum ib_mad_state wrong_state)</div><div class='add'>+{</div><div class='add'>+	if (IS_ENABLED(CONFIG_LOCKDEP))</div><div class='add'>+		WARN_ON(mad_send_wr-&gt;state == wrong_state);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> struct ib_mad_local_private {</div><div class='ctx'> 	struct list_head completion_list;</div><div class='ctx'> 	struct ib_mad_private *mad_priv;</div><div class='hunk'>@@ -223,4 +289,7 @@ void ib_mark_mad_done(struct ib_mad_send_wr_private *mad_send_wr);</div><div class='ctx'> void ib_reset_mad_timeout(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='ctx'> 			  unsigned long timeout_ms);</div><div class='ctx'> </div><div class='add'>+void change_mad_state(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='add'>+		      enum ib_mad_state new_state);</div><div class='add'>+</div><div class='ctx'> #endif	/* __IB_MAD_PRIV_H__ */</div><div class='head'>diff --git a/drivers/infiniband/core/mad_rmpp.c b/drivers/infiniband/core/mad_rmpp.c<br/>index 5ec57abc0849..1c5e0eaf1c94 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/mad_rmpp.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/mad_rmpp.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/mad_rmpp.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/mad_rmpp.c</a></div><div class='hunk'>@@ -40,8 +40,7 @@</div><div class='ctx'> enum rmpp_state {</div><div class='ctx'> 	RMPP_STATE_ACTIVE,</div><div class='ctx'> 	RMPP_STATE_TIMEOUT,</div><div class='del'>-	RMPP_STATE_COMPLETE,</div><div class='del'>-	RMPP_STATE_CANCELING</div><div class='add'>+	RMPP_STATE_COMPLETE</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct mad_rmpp_recv {</div><div class='hunk'>@@ -52,7 +51,7 @@ struct mad_rmpp_recv {</div><div class='ctx'> 	struct completion comp;</div><div class='ctx'> 	enum rmpp_state state;</div><div class='ctx'> 	spinlock_t lock;</div><div class='del'>-	atomic_t refcount;</div><div class='add'>+	refcount_t refcount;</div><div class='ctx'> </div><div class='ctx'> 	struct ib_ah *ah;</div><div class='ctx'> 	struct ib_mad_recv_wc *rmpp_wc;</div><div class='hunk'>@@ -73,7 +72,7 @@ struct mad_rmpp_recv {</div><div class='ctx'> </div><div class='ctx'> static inline void deref_rmpp_recv(struct mad_rmpp_recv *rmpp_recv)</div><div class='ctx'> {</div><div class='del'>-	if (atomic_dec_and_test(&amp;rmpp_recv-&gt;refcount))</div><div class='add'>+	if (refcount_dec_and_test(&amp;rmpp_recv-&gt;refcount))</div><div class='ctx'> 		complete(&amp;rmpp_recv-&gt;comp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -92,22 +91,18 @@ void ib_cancel_rmpp_recvs(struct ib_mad_agent_private *agent)</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;agent-&gt;lock, flags);</div><div class='ctx'> 	list_for_each_entry(rmpp_recv, &amp;agent-&gt;rmpp_list, list) {</div><div class='del'>-		if (rmpp_recv-&gt;state != RMPP_STATE_COMPLETE)</div><div class='del'>-			ib_free_recv_mad(rmpp_recv-&gt;rmpp_wc);</div><div class='del'>-		rmpp_recv-&gt;state = RMPP_STATE_CANCELING;</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;agent-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-	list_for_each_entry(rmpp_recv, &amp;agent-&gt;rmpp_list, list) {</div><div class='ctx'> 		cancel_delayed_work(&amp;rmpp_recv-&gt;timeout_work);</div><div class='ctx'> 		cancel_delayed_work(&amp;rmpp_recv-&gt;cleanup_work);</div><div class='ctx'> 	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;agent-&gt;lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	flush_workqueue(agent-&gt;qp_info-&gt;port_priv-&gt;wq);</div><div class='ctx'> </div><div class='ctx'> 	list_for_each_entry_safe(rmpp_recv, temp_rmpp_recv,</div><div class='ctx'> 				 &amp;agent-&gt;rmpp_list, list) {</div><div class='ctx'> 		list_del(&amp;rmpp_recv-&gt;list);</div><div class='add'>+		if (rmpp_recv-&gt;state != RMPP_STATE_COMPLETE)</div><div class='add'>+			ib_free_recv_mad(rmpp_recv-&gt;rmpp_wc);</div><div class='ctx'> 		destroy_rmpp_recv(rmpp_recv);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='hunk'>@@ -163,7 +158,7 @@ static struct ib_mad_send_buf *alloc_response_msg(struct ib_mad_agent *agent,</div><div class='ctx'> 	ah = ib_create_ah_from_wc(agent-&gt;qp-&gt;pd, recv_wc-&gt;wc,</div><div class='ctx'> 				  recv_wc-&gt;recv_buf.grh, agent-&gt;port_num);</div><div class='ctx'> 	if (IS_ERR(ah))</div><div class='del'>-		return (void *) ah;</div><div class='add'>+		return ERR_CAST(ah);</div><div class='ctx'> </div><div class='ctx'> 	hdr_len = ib_get_mad_data_offset(recv_wc-&gt;recv_buf.mad-&gt;mad_hdr.mgmt_class);</div><div class='ctx'> 	msg = ib_create_send_mad(agent, recv_wc-&gt;wc-&gt;src_qp,</div><div class='hunk'>@@ -272,10 +267,6 @@ static void recv_cleanup_handler(struct work_struct *work)</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;rmpp_recv-&gt;agent-&gt;lock, flags);</div><div class='del'>-	if (rmpp_recv-&gt;state == RMPP_STATE_CANCELING) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;rmpp_recv-&gt;agent-&gt;lock, flags);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='ctx'> 	list_del(&amp;rmpp_recv-&gt;list);</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;rmpp_recv-&gt;agent-&gt;lock, flags);</div><div class='ctx'> 	destroy_rmpp_recv(rmpp_recv);</div><div class='hunk'>@@ -305,7 +296,7 @@ create_rmpp_recv(struct ib_mad_agent_private *agent,</div><div class='ctx'> 	INIT_DELAYED_WORK(&amp;rmpp_recv-&gt;cleanup_work, recv_cleanup_handler);</div><div class='ctx'> 	spin_lock_init(&amp;rmpp_recv-&gt;lock);</div><div class='ctx'> 	rmpp_recv-&gt;state = RMPP_STATE_ACTIVE;</div><div class='del'>-	atomic_set(&amp;rmpp_recv-&gt;refcount, 1);</div><div class='add'>+	refcount_set(&amp;rmpp_recv-&gt;refcount, 1);</div><div class='ctx'> </div><div class='ctx'> 	rmpp_recv-&gt;rmpp_wc = mad_recv_wc;</div><div class='ctx'> 	rmpp_recv-&gt;cur_seg_buf = &amp;mad_recv_wc-&gt;recv_buf;</div><div class='hunk'>@@ -357,7 +348,7 @@ acquire_rmpp_recv(struct ib_mad_agent_private *agent,</div><div class='ctx'> 	spin_lock_irqsave(&amp;agent-&gt;lock, flags);</div><div class='ctx'> 	rmpp_recv = find_rmpp_recv(agent, mad_recv_wc);</div><div class='ctx'> 	if (rmpp_recv)</div><div class='del'>-		atomic_inc(&amp;rmpp_recv-&gt;refcount);</div><div class='add'>+		refcount_inc(&amp;rmpp_recv-&gt;refcount);</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;agent-&gt;lock, flags);</div><div class='ctx'> 	return rmpp_recv;</div><div class='ctx'> }</div><div class='hunk'>@@ -391,8 +382,8 @@ static inline int get_seg_num(struct ib_mad_recv_buf *seg)</div><div class='ctx'> 	return be32_to_cpu(rmpp_mad-&gt;rmpp_hdr.seg_num);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline struct ib_mad_recv_buf * get_next_seg(struct list_head *rmpp_list,</div><div class='del'>-						    struct ib_mad_recv_buf *seg)</div><div class='add'>+static inline struct ib_mad_recv_buf *get_next_seg(struct list_head *rmpp_list,</div><div class='add'>+						   struct ib_mad_recv_buf *seg)</div><div class='ctx'> {</div><div class='ctx'> 	if (seg-&gt;list.next == rmpp_list)</div><div class='ctx'> 		return NULL;</div><div class='hunk'>@@ -405,8 +396,8 @@ static inline int window_size(struct ib_mad_agent_private *agent)</div><div class='ctx'> 	return max(agent-&gt;qp_info-&gt;recv_queue.max_active &gt;&gt; 3, 1);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct ib_mad_recv_buf * find_seg_location(struct list_head *rmpp_list,</div><div class='del'>-						  int seg_num)</div><div class='add'>+static struct ib_mad_recv_buf *find_seg_location(struct list_head *rmpp_list,</div><div class='add'>+						 int seg_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_mad_recv_buf *seg_buf;</div><div class='ctx'> 	int cur_seg_num;</div><div class='hunk'>@@ -458,7 +449,7 @@ static inline int get_mad_len(struct mad_rmpp_recv *rmpp_recv)</div><div class='ctx'> 	return hdr_size + rmpp_recv-&gt;seg_num * data_size - pad;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct ib_mad_recv_wc * complete_rmpp(struct mad_rmpp_recv *rmpp_recv)</div><div class='add'>+static struct ib_mad_recv_wc *complete_rmpp(struct mad_rmpp_recv *rmpp_recv)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_mad_recv_wc *rmpp_wc;</div><div class='ctx'> </div><div class='hunk'>@@ -553,7 +544,7 @@ start_rmpp(struct ib_mad_agent_private *agent,</div><div class='ctx'> 		destroy_rmpp_recv(rmpp_recv);</div><div class='ctx'> 		return continue_rmpp(agent, mad_recv_wc);</div><div class='ctx'> 	}</div><div class='del'>-	atomic_inc(&amp;rmpp_recv-&gt;refcount);</div><div class='add'>+	refcount_inc(&amp;rmpp_recv-&gt;refcount);</div><div class='ctx'> </div><div class='ctx'> 	if (get_last_flag(&amp;mad_recv_wc-&gt;recv_buf)) {</div><div class='ctx'> 		rmpp_recv-&gt;state = RMPP_STATE_COMPLETE;</div><div class='hunk'>@@ -617,16 +608,20 @@ static void abort_send(struct ib_mad_agent_private *agent,</div><div class='ctx'> 		goto out;	/* Unmatched send */</div><div class='ctx'> </div><div class='ctx'> 	if ((mad_send_wr-&gt;last_ack == mad_send_wr-&gt;send_buf.seg_count) ||</div><div class='del'>-	    (!mad_send_wr-&gt;timeout) || (mad_send_wr-&gt;status != IB_WC_SUCCESS))</div><div class='add'>+	    (!mad_send_wr-&gt;timeout) ||</div><div class='add'>+	    (mad_send_wr-&gt;state == IB_MAD_STATE_CANCELED))</div><div class='ctx'> 		goto out;	/* Send is already done */</div><div class='ctx'> </div><div class='ctx'> 	ib_mark_mad_done(mad_send_wr);</div><div class='add'>+	if (mad_send_wr-&gt;state == IB_MAD_STATE_DONE) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;agent-&gt;lock, flags);</div><div class='add'>+		wc.status = IB_WC_REM_ABORT_ERR;</div><div class='add'>+		wc.vendor_err = rmpp_status;</div><div class='add'>+		wc.send_buf = &amp;mad_send_wr-&gt;send_buf;</div><div class='add'>+		ib_mad_complete_send_wr(mad_send_wr, &amp;wc);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;agent-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-	wc.status = IB_WC_REM_ABORT_ERR;</div><div class='del'>-	wc.vendor_err = rmpp_status;</div><div class='del'>-	wc.send_buf = &amp;mad_send_wr-&gt;send_buf;</div><div class='del'>-	ib_mad_complete_send_wr(mad_send_wr, &amp;wc);</div><div class='ctx'> 	return;</div><div class='ctx'> out:</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;agent-&gt;lock, flags);</div><div class='hunk'>@@ -693,7 +688,8 @@ static void process_rmpp_ack(struct ib_mad_agent_private *agent,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if ((mad_send_wr-&gt;last_ack == mad_send_wr-&gt;send_buf.seg_count) ||</div><div class='del'>-	    (!mad_send_wr-&gt;timeout) || (mad_send_wr-&gt;status != IB_WC_SUCCESS))</div><div class='add'>+	    (!mad_send_wr-&gt;timeout) ||</div><div class='add'>+	    (mad_send_wr-&gt;state == IB_MAD_STATE_CANCELED))</div><div class='ctx'> 		goto out;	/* Send is already done */</div><div class='ctx'> </div><div class='ctx'> 	if (seg_num &gt; mad_send_wr-&gt;send_buf.seg_count ||</div><div class='hunk'>@@ -718,21 +714,24 @@ static void process_rmpp_ack(struct ib_mad_agent_private *agent,</div><div class='ctx'> 			struct ib_mad_send_wc wc;</div><div class='ctx'> </div><div class='ctx'> 			ib_mark_mad_done(mad_send_wr);</div><div class='add'>+			if (mad_send_wr-&gt;state == IB_MAD_STATE_DONE) {</div><div class='add'>+				spin_unlock_irqrestore(&amp;agent-&gt;lock, flags);</div><div class='add'>+				wc.status = IB_WC_SUCCESS;</div><div class='add'>+				wc.vendor_err = 0;</div><div class='add'>+				wc.send_buf = &amp;mad_send_wr-&gt;send_buf;</div><div class='add'>+				ib_mad_complete_send_wr(mad_send_wr, &amp;wc);</div><div class='add'>+				return;</div><div class='add'>+			}</div><div class='ctx'> 			spin_unlock_irqrestore(&amp;agent-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-			wc.status = IB_WC_SUCCESS;</div><div class='del'>-			wc.vendor_err = 0;</div><div class='del'>-			wc.send_buf = &amp;mad_send_wr-&gt;send_buf;</div><div class='del'>-			ib_mad_complete_send_wr(mad_send_wr, &amp;wc);</div><div class='ctx'> 			return;</div><div class='ctx'> 		}</div><div class='del'>-		if (mad_send_wr-&gt;refcount == 1)</div><div class='add'>+		if (mad_send_wr-&gt;state == IB_MAD_STATE_WAIT_RESP)</div><div class='ctx'> 			ib_reset_mad_timeout(mad_send_wr,</div><div class='ctx'> 					     mad_send_wr-&gt;send_buf.timeout_ms);</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;agent-&gt;lock, flags);</div><div class='ctx'> 		ack_ds_ack(agent, mad_recv_wc);</div><div class='ctx'> 		return;</div><div class='del'>-	} else if (mad_send_wr-&gt;refcount == 1 &amp;&amp;</div><div class='add'>+	} else if (mad_send_wr-&gt;state == IB_MAD_STATE_WAIT_RESP &amp;&amp;</div><div class='ctx'> 		   mad_send_wr-&gt;seg_num &lt; mad_send_wr-&gt;newwin &amp;&amp;</div><div class='ctx'> 		   mad_send_wr-&gt;seg_num &lt; mad_send_wr-&gt;send_buf.seg_count) {</div><div class='ctx'> 		/* Send failure will just result in a timeout/retry */</div><div class='hunk'>@@ -740,7 +739,7 @@ static void process_rmpp_ack(struct ib_mad_agent_private *agent,</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto out;</div><div class='ctx'> </div><div class='del'>-		mad_send_wr-&gt;refcount++;</div><div class='add'>+		change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);</div><div class='ctx'> 		list_move_tail(&amp;mad_send_wr-&gt;agent_list,</div><div class='ctx'> 			      &amp;mad_send_wr-&gt;mad_agent_priv-&gt;send_list);</div><div class='ctx'> 	}</div><div class='hunk'>@@ -899,7 +898,6 @@ int ib_send_rmpp_mad(struct ib_mad_send_wr_private *mad_send_wr)</div><div class='ctx'> 	mad_send_wr-&gt;newwin = init_newwin(mad_send_wr);</div><div class='ctx'> </div><div class='ctx'> 	/* We need to wait for the final ACK even if there isn't a response */</div><div class='del'>-	mad_send_wr-&gt;refcount += (mad_send_wr-&gt;timeout == 0);</div><div class='ctx'> 	ret = send_next_seg(mad_send_wr);</div><div class='ctx'> 	if (!ret)</div><div class='ctx'> 		return IB_RMPP_RESULT_CONSUMED;</div><div class='hunk'>@@ -921,7 +919,7 @@ int ib_process_rmpp_send_wc(struct ib_mad_send_wr_private *mad_send_wr,</div><div class='ctx'> 		return IB_RMPP_RESULT_INTERNAL;	 /* ACK, STOP, or ABORT */</div><div class='ctx'> </div><div class='ctx'> 	if (mad_send_wc-&gt;status != IB_WC_SUCCESS ||</div><div class='del'>-	    mad_send_wr-&gt;status != IB_WC_SUCCESS)</div><div class='add'>+	    mad_send_wr-&gt;state == IB_MAD_STATE_CANCELED)</div><div class='ctx'> 		return IB_RMPP_RESULT_PROCESSED; /* Canceled or send error */</div><div class='ctx'> </div><div class='ctx'> 	if (!mad_send_wr-&gt;timeout)</div><div class='head'>diff --git a/drivers/infiniband/core/mr_pool.c b/drivers/infiniband/core/mr_pool.c<br/>index 49d478b2ea94..c0e2df128b34 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/mr_pool.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/mr_pool.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/mr_pool.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/mr_pool.c</a></div><div class='hunk'>@@ -1,14 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0-only</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright (c) 2016 HGST, a Western Digital Company.</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify it</div><div class='del'>- * under the terms and conditions of the GNU General Public License,</div><div class='del'>- * version 2, as published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope it will be useful, but WITHOUT</div><div class='del'>- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or</div><div class='del'>- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for</div><div class='del'>- * more details.</div><div class='ctx'>  */</div><div class='ctx'> #include &lt;rdma/ib_verbs.h&gt;</div><div class='ctx'> #include &lt;rdma/mr_pool.h&gt;</div><div class='hunk'>@@ -42,14 +34,18 @@ void ib_mr_pool_put(struct ib_qp *qp, struct list_head *list, struct ib_mr *mr)</div><div class='ctx'> EXPORT_SYMBOL(ib_mr_pool_put);</div><div class='ctx'> </div><div class='ctx'> int ib_mr_pool_init(struct ib_qp *qp, struct list_head *list, int nr,</div><div class='del'>-		enum ib_mr_type type, u32 max_num_sg)</div><div class='add'>+		enum ib_mr_type type, u32 max_num_sg, u32 max_num_meta_sg)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_mr *mr;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int ret, i;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; nr; i++) {</div><div class='del'>-		mr = ib_alloc_mr(qp-&gt;pd, type, max_num_sg);</div><div class='add'>+		if (type == IB_MR_TYPE_INTEGRITY)</div><div class='add'>+			mr = ib_alloc_mr_integrity(qp-&gt;pd, max_num_sg,</div><div class='add'>+						   max_num_meta_sg);</div><div class='add'>+		else</div><div class='add'>+			mr = ib_alloc_mr(qp-&gt;pd, type, max_num_sg);</div><div class='ctx'> 		if (IS_ERR(mr)) {</div><div class='ctx'> 			ret = PTR_ERR(mr);</div><div class='ctx'> 			goto out;</div><div class='head'>diff --git a/drivers/infiniband/core/multicast.c b/drivers/infiniband/core/multicast.c<br/>index cd338ddc4a39..a236532a9026 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/multicast.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/multicast.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/multicast.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/multicast.c</a></div><div class='hunk'>@@ -42,7 +42,7 @@</div><div class='ctx'> #include &lt;rdma/ib_cache.h&gt;</div><div class='ctx'> #include "sa.h"</div><div class='ctx'> </div><div class='del'>-static void mcast_add_one(struct ib_device *device);</div><div class='add'>+static int mcast_add_one(struct ib_device *device);</div><div class='ctx'> static void mcast_remove_one(struct ib_device *device, void *client_data);</div><div class='ctx'> </div><div class='ctx'> static struct ib_client mcast_client = {</div><div class='hunk'>@@ -61,9 +61,9 @@ struct mcast_port {</div><div class='ctx'> 	struct mcast_device	*dev;</div><div class='ctx'> 	spinlock_t		lock;</div><div class='ctx'> 	struct rb_root		table;</div><div class='del'>-	atomic_t		refcount;</div><div class='add'>+	refcount_t		refcount;</div><div class='ctx'> 	struct completion	comp;</div><div class='del'>-	u8			port_num;</div><div class='add'>+	u32			port_num;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct mcast_device {</div><div class='hunk'>@@ -71,7 +71,7 @@ struct mcast_device {</div><div class='ctx'> 	struct ib_event_handler	event_handler;</div><div class='ctx'> 	int			start_port;</div><div class='ctx'> 	int			end_port;</div><div class='del'>-	struct mcast_port	port[0];</div><div class='add'>+	struct mcast_port	port[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum mcast_state {</div><div class='hunk'>@@ -117,7 +117,7 @@ struct mcast_member {</div><div class='ctx'> 	struct mcast_group	*group;</div><div class='ctx'> 	struct list_head	list;</div><div class='ctx'> 	enum mcast_state	state;</div><div class='del'>-	atomic_t		refcount;</div><div class='add'>+	refcount_t		refcount;</div><div class='ctx'> 	struct completion	comp;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -178,7 +178,7 @@ static struct mcast_group *mcast_insert(struct mcast_port *port,</div><div class='ctx'> </div><div class='ctx'> static void deref_port(struct mcast_port *port)</div><div class='ctx'> {</div><div class='del'>-	if (atomic_dec_and_test(&amp;port-&gt;refcount))</div><div class='add'>+	if (refcount_dec_and_test(&amp;port-&gt;refcount))</div><div class='ctx'> 		complete(&amp;port-&gt;comp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -199,7 +199,7 @@ static void release_group(struct mcast_group *group)</div><div class='ctx'> </div><div class='ctx'> static void deref_member(struct mcast_member *member)</div><div class='ctx'> {</div><div class='del'>-	if (atomic_dec_and_test(&amp;member-&gt;refcount))</div><div class='add'>+	if (refcount_dec_and_test(&amp;member-&gt;refcount))</div><div class='ctx'> 		complete(&amp;member-&gt;comp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -401,7 +401,7 @@ static void process_group_error(struct mcast_group *group)</div><div class='ctx'> 	while (!list_empty(&amp;group-&gt;active_list)) {</div><div class='ctx'> 		member = list_entry(group-&gt;active_list.next,</div><div class='ctx'> 				    struct mcast_member, list);</div><div class='del'>-		atomic_inc(&amp;member-&gt;refcount);</div><div class='add'>+		refcount_inc(&amp;member-&gt;refcount);</div><div class='ctx'> 		list_del_init(&amp;member-&gt;list);</div><div class='ctx'> 		adjust_membership(group, member-&gt;multicast.rec.join_state, -1);</div><div class='ctx'> 		member-&gt;state = MCAST_ERROR;</div><div class='hunk'>@@ -445,7 +445,7 @@ retest:</div><div class='ctx'> 				    struct mcast_member, list);</div><div class='ctx'> 		multicast = &amp;member-&gt;multicast;</div><div class='ctx'> 		join_state = multicast-&gt;rec.join_state;</div><div class='del'>-		atomic_inc(&amp;member-&gt;refcount);</div><div class='add'>+		refcount_inc(&amp;member-&gt;refcount);</div><div class='ctx'> </div><div class='ctx'> 		if (join_state == (group-&gt;rec.join_state &amp; join_state)) {</div><div class='ctx'> 			status = cmp_rec(&amp;group-&gt;rec, &amp;multicast-&gt;rec,</div><div class='hunk'>@@ -497,7 +497,7 @@ static void process_join_error(struct mcast_group *group, int status)</div><div class='ctx'> 	member = list_entry(group-&gt;pending_list.next,</div><div class='ctx'> 			    struct mcast_member, list);</div><div class='ctx'> 	if (group-&gt;last_join == member) {</div><div class='del'>-		atomic_inc(&amp;member-&gt;refcount);</div><div class='add'>+		refcount_inc(&amp;member-&gt;refcount);</div><div class='ctx'> 		list_del_init(&amp;member-&gt;list);</div><div class='ctx'> 		spin_unlock_irq(&amp;group-&gt;lock);</div><div class='ctx'> 		ret = member-&gt;multicast.callback(status, &amp;member-&gt;multicast);</div><div class='hunk'>@@ -589,7 +589,7 @@ static struct mcast_group *acquire_group(struct mcast_port *port,</div><div class='ctx'> 		kfree(group);</div><div class='ctx'> 		group = cur_group;</div><div class='ctx'> 	} else</div><div class='del'>-		atomic_inc(&amp;port-&gt;refcount);</div><div class='add'>+		refcount_inc(&amp;port-&gt;refcount);</div><div class='ctx'> found:</div><div class='ctx'> 	atomic_inc(&amp;group-&gt;refcount);</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;port-&gt;lock, flags);</div><div class='hunk'>@@ -605,7 +605,7 @@ found:</div><div class='ctx'>  */</div><div class='ctx'> struct ib_sa_multicast *</div><div class='ctx'> ib_sa_join_multicast(struct ib_sa_client *client,</div><div class='del'>-		     struct ib_device *device, u8 port_num,</div><div class='add'>+		     struct ib_device *device, u32 port_num,</div><div class='ctx'> 		     struct ib_sa_mcmember_rec *rec,</div><div class='ctx'> 		     ib_sa_comp_mask comp_mask, gfp_t gfp_mask,</div><div class='ctx'> 		     int (*callback)(int status,</div><div class='hunk'>@@ -632,7 +632,7 @@ ib_sa_join_multicast(struct ib_sa_client *client,</div><div class='ctx'> 	member-&gt;multicast.callback = callback;</div><div class='ctx'> 	member-&gt;multicast.context = context;</div><div class='ctx'> 	init_completion(&amp;member-&gt;comp);</div><div class='del'>-	atomic_set(&amp;member-&gt;refcount, 1);</div><div class='add'>+	refcount_set(&amp;member-&gt;refcount, 1);</div><div class='ctx'> 	member-&gt;state = MCAST_JOINING;</div><div class='ctx'> </div><div class='ctx'> 	member-&gt;group = acquire_group(&amp;dev-&gt;port[port_num - dev-&gt;start_port],</div><div class='hunk'>@@ -690,7 +690,7 @@ void ib_sa_free_multicast(struct ib_sa_multicast *multicast)</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_sa_free_multicast);</div><div class='ctx'> </div><div class='del'>-int ib_sa_get_mcmember_rec(struct ib_device *device, u8 port_num,</div><div class='add'>+int ib_sa_get_mcmember_rec(struct ib_device *device, u32 port_num,</div><div class='ctx'> 			   union ib_gid *mgid, struct ib_sa_mcmember_rec *rec)</div><div class='ctx'> {</div><div class='ctx'> 	struct mcast_device *dev;</div><div class='hunk'>@@ -721,6 +721,7 @@ EXPORT_SYMBOL(ib_sa_get_mcmember_rec);</div><div class='ctx'>  * member record and gid of the device.</div><div class='ctx'>  * @device:	RDMA device</div><div class='ctx'>  * @port_num:	Port of the rdma device to consider</div><div class='add'>+ * @rec:	Multicast member record to use</div><div class='ctx'>  * @ndev:	Optional netdevice, applicable only for RoCE</div><div class='ctx'>  * @gid_type:	GID type to consider</div><div class='ctx'>  * @ah_attr:	AH attribute to fillup on successful completion</div><div class='hunk'>@@ -731,7 +732,7 @@ EXPORT_SYMBOL(ib_sa_get_mcmember_rec);</div><div class='ctx'>  * success or appropriate error code.</div><div class='ctx'>  *</div><div class='ctx'>  */</div><div class='del'>-int ib_init_ah_from_mcmember(struct ib_device *device, u8 port_num,</div><div class='add'>+int ib_init_ah_from_mcmember(struct ib_device *device, u32 port_num,</div><div class='ctx'> 			     struct ib_sa_mcmember_rec *rec,</div><div class='ctx'> 			     struct net_device *ndev,</div><div class='ctx'> 			     enum ib_gid_type gid_type,</div><div class='hunk'>@@ -815,7 +816,7 @@ static void mcast_event_handler(struct ib_event_handler *handler,</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void mcast_add_one(struct ib_device *device)</div><div class='add'>+static int mcast_add_one(struct ib_device *device)</div><div class='ctx'> {</div><div class='ctx'> 	struct mcast_device *dev;</div><div class='ctx'> 	struct mcast_port *port;</div><div class='hunk'>@@ -825,7 +826,7 @@ static void mcast_add_one(struct ib_device *device)</div><div class='ctx'> 	dev = kmalloc(struct_size(dev, port, device-&gt;phys_port_cnt),</div><div class='ctx'> 		      GFP_KERNEL);</div><div class='ctx'> 	if (!dev)</div><div class='del'>-		return;</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	dev-&gt;start_port = rdma_start_port(device);</div><div class='ctx'> 	dev-&gt;end_port = rdma_end_port(device);</div><div class='hunk'>@@ -839,13 +840,13 @@ static void mcast_add_one(struct ib_device *device)</div><div class='ctx'> 		spin_lock_init(&amp;port-&gt;lock);</div><div class='ctx'> 		port-&gt;table = RB_ROOT;</div><div class='ctx'> 		init_completion(&amp;port-&gt;comp);</div><div class='del'>-		atomic_set(&amp;port-&gt;refcount, 1);</div><div class='add'>+		refcount_set(&amp;port-&gt;refcount, 1);</div><div class='ctx'> 		++count;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (!count) {</div><div class='ctx'> 		kfree(dev);</div><div class='del'>-		return;</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	dev-&gt;device = device;</div><div class='hunk'>@@ -853,6 +854,7 @@ static void mcast_add_one(struct ib_device *device)</div><div class='ctx'> </div><div class='ctx'> 	INIT_IB_EVENT_HANDLER(&amp;dev-&gt;event_handler, device, mcast_event_handler);</div><div class='ctx'> 	ib_register_event_handler(&amp;dev-&gt;event_handler);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void mcast_remove_one(struct ib_device *device, void *client_data)</div><div class='hunk'>@@ -861,9 +863,6 @@ static void mcast_remove_one(struct ib_device *device, void *client_data)</div><div class='ctx'> 	struct mcast_port *port;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	if (!dev)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='ctx'> 	ib_unregister_event_handler(&amp;dev-&gt;event_handler);</div><div class='ctx'> 	flush_workqueue(mcast_wq);</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/core/netlink.c b/drivers/infiniband/core/netlink.c<br/>index eecfc0b377c9..def14c54b648 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/netlink.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/netlink.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/netlink.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/netlink.c</a></div><div class='hunk'>@@ -36,20 +36,25 @@</div><div class='ctx'> #include &lt;linux/export.h&gt;</div><div class='ctx'> #include &lt;net/netlink.h&gt;</div><div class='ctx'> #include &lt;net/net_namespace.h&gt;</div><div class='add'>+#include &lt;net/netns/generic.h&gt;</div><div class='ctx'> #include &lt;net/sock.h&gt;</div><div class='ctx'> #include &lt;rdma/rdma_netlink.h&gt;</div><div class='ctx'> #include &lt;linux/module.h&gt;</div><div class='ctx'> #include "core_priv.h"</div><div class='ctx'> </div><div class='del'>-static DEFINE_MUTEX(rdma_nl_mutex);</div><div class='del'>-static struct sock *nls;</div><div class='ctx'> static struct {</div><div class='del'>-	const struct rdma_nl_cbs   *cb_table;</div><div class='add'>+	const struct rdma_nl_cbs *cb_table;</div><div class='add'>+	/* Synchronizes between ongoing netlink commands and netlink client</div><div class='add'>+	 * unregistration.</div><div class='add'>+	 */</div><div class='add'>+	struct rw_semaphore sem;</div><div class='ctx'> } rdma_nl_types[RDMA_NL_NUM_CLIENTS];</div><div class='ctx'> </div><div class='ctx'> bool rdma_nl_chk_listeners(unsigned int group)</div><div class='ctx'> {</div><div class='del'>-	return netlink_has_listeners(nls, group);</div><div class='add'>+	struct rdma_dev_net *rnet = rdma_net_to_dev_net(&amp;init_net);</div><div class='add'>+</div><div class='add'>+	return netlink_has_listeners(rnet-&gt;nl_sock, group);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_nl_chk_listeners);</div><div class='ctx'> </div><div class='hunk'>@@ -70,65 +75,56 @@ static bool is_nl_msg_valid(unsigned int type, unsigned int op)</div><div class='ctx'> 	if (type &gt;= RDMA_NL_NUM_CLIENTS)</div><div class='ctx'> 		return false;</div><div class='ctx'> </div><div class='del'>-	return (op &lt; max_num_ops[type]) ? true : false;</div><div class='add'>+	return op &lt; max_num_ops[type];</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static bool is_nl_valid(unsigned int type, unsigned int op)</div><div class='add'>+static const struct rdma_nl_cbs *</div><div class='add'>+get_cb_table(const struct sk_buff *skb, unsigned int type, unsigned int op)</div><div class='ctx'> {</div><div class='ctx'> 	const struct rdma_nl_cbs *cb_table;</div><div class='ctx'> </div><div class='del'>-	if (!is_nl_msg_valid(type, op))</div><div class='del'>-		return false;</div><div class='add'>+	/*</div><div class='add'>+	 * Currently only NLDEV client is supporting netlink commands in</div><div class='add'>+	 * non init_net net namespace.</div><div class='add'>+	 */</div><div class='add'>+	if (sock_net(skb-&gt;sk) != &amp;init_net &amp;&amp; type != RDMA_NL_NLDEV)</div><div class='add'>+		return NULL;</div><div class='ctx'> </div><div class='del'>-	if (!rdma_nl_types[type].cb_table) {</div><div class='del'>-		mutex_unlock(&amp;rdma_nl_mutex);</div><div class='del'>-		request_module("rdma-netlink-subsys-%d", type);</div><div class='del'>-		mutex_lock(&amp;rdma_nl_mutex);</div><div class='del'>-	}</div><div class='add'>+	cb_table = READ_ONCE(rdma_nl_types[type].cb_table);</div><div class='add'>+	if (!cb_table) {</div><div class='add'>+		/*</div><div class='add'>+		 * Didn't get valid reference of the table, attempt module</div><div class='add'>+		 * load once.</div><div class='add'>+		 */</div><div class='add'>+		up_read(&amp;rdma_nl_types[type].sem);</div><div class='ctx'> </div><div class='del'>-	cb_table = rdma_nl_types[type].cb_table;</div><div class='add'>+		request_module("rdma-netlink-subsys-%u", type);</div><div class='ctx'> </div><div class='add'>+		down_read(&amp;rdma_nl_types[type].sem);</div><div class='add'>+		cb_table = READ_ONCE(rdma_nl_types[type].cb_table);</div><div class='add'>+	}</div><div class='ctx'> 	if (!cb_table || (!cb_table[op].dump &amp;&amp; !cb_table[op].doit))</div><div class='del'>-		return false;</div><div class='del'>-	return true;</div><div class='add'>+		return NULL;</div><div class='add'>+	return cb_table;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void rdma_nl_register(unsigned int index,</div><div class='ctx'> 		      const struct rdma_nl_cbs cb_table[])</div><div class='ctx'> {</div><div class='del'>-	mutex_lock(&amp;rdma_nl_mutex);</div><div class='del'>-	if (!is_nl_msg_valid(index, 0)) {</div><div class='del'>-		/*</div><div class='del'>-		 * All clients are not interesting in success/failure of</div><div class='del'>-		 * this call. They want to see the print to error log and</div><div class='del'>-		 * continue their initialization. Print warning for them,</div><div class='del'>-		 * because it is programmer's error to be here.</div><div class='del'>-		 */</div><div class='del'>-		mutex_unlock(&amp;rdma_nl_mutex);</div><div class='del'>-		WARN(true,</div><div class='del'>-		     "The not-valid %u index was supplied to RDMA netlink\n",</div><div class='del'>-		     index);</div><div class='add'>+	if (WARN_ON(!is_nl_msg_valid(index, 0)) ||</div><div class='add'>+	    WARN_ON(READ_ONCE(rdma_nl_types[index].cb_table)))</div><div class='ctx'> 		return;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	if (rdma_nl_types[index].cb_table) {</div><div class='del'>-		mutex_unlock(&amp;rdma_nl_mutex);</div><div class='del'>-		WARN(true,</div><div class='del'>-		     "The %u index is already registered in RDMA netlink\n",</div><div class='del'>-		     index);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	rdma_nl_types[index].cb_table = cb_table;</div><div class='del'>-	mutex_unlock(&amp;rdma_nl_mutex);</div><div class='add'>+	/* Pairs with the READ_ONCE in is_nl_valid() */</div><div class='add'>+	smp_store_release(&amp;rdma_nl_types[index].cb_table, cb_table);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_nl_register);</div><div class='ctx'> </div><div class='ctx'> void rdma_nl_unregister(unsigned int index)</div><div class='ctx'> {</div><div class='del'>-	mutex_lock(&amp;rdma_nl_mutex);</div><div class='add'>+	down_write(&amp;rdma_nl_types[index].sem);</div><div class='ctx'> 	rdma_nl_types[index].cb_table = NULL;</div><div class='del'>-	mutex_unlock(&amp;rdma_nl_mutex);</div><div class='add'>+	up_write(&amp;rdma_nl_types[index].sem);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_nl_unregister);</div><div class='ctx'> </div><div class='hunk'>@@ -160,15 +156,21 @@ static int rdma_nl_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	unsigned int index = RDMA_NL_GET_CLIENT(type);</div><div class='ctx'> 	unsigned int op = RDMA_NL_GET_OP(type);</div><div class='ctx'> 	const struct rdma_nl_cbs *cb_table;</div><div class='add'>+	int err = -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (!is_nl_valid(index, op))</div><div class='add'>+	if (!is_nl_msg_valid(index, op))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	cb_table = rdma_nl_types[index].cb_table;</div><div class='add'>+	down_read(&amp;rdma_nl_types[index].sem);</div><div class='add'>+	cb_table = get_cb_table(skb, index, op);</div><div class='add'>+	if (!cb_table)</div><div class='add'>+		goto done;</div><div class='ctx'> </div><div class='ctx'> 	if ((cb_table[op].flags &amp; RDMA_NL_ADMIN_PERM) &amp;&amp;</div><div class='del'>-	    !netlink_capable(skb, CAP_NET_ADMIN))</div><div class='del'>-		return -EPERM;</div><div class='add'>+	    !netlink_capable(skb, CAP_NET_ADMIN)) {</div><div class='add'>+		err = -EPERM;</div><div class='add'>+		goto done;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * LS responses overload the 0x100 (NLM_F_ROOT) flag.  Don't</div><div class='hunk'>@@ -176,8 +178,8 @@ static int rdma_nl_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	 */</div><div class='ctx'> 	if (index == RDMA_NL_LS) {</div><div class='ctx'> 		if (cb_table[op].doit)</div><div class='del'>-			return cb_table[op].doit(skb, nlh, extack);</div><div class='del'>-		return -EINVAL;</div><div class='add'>+			err = cb_table[op].doit(skb, nlh, extack);</div><div class='add'>+		goto done;</div><div class='ctx'> 	}</div><div class='ctx'> 	/* FIXME: Convert IWCM to properly handle doit callbacks */</div><div class='ctx'> 	if ((nlh-&gt;nlmsg_flags &amp; NLM_F_DUMP) || index == RDMA_NL_IWCM) {</div><div class='hunk'>@@ -185,14 +187,15 @@ static int rdma_nl_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 			.dump = cb_table[op].dump,</div><div class='ctx'> 		};</div><div class='ctx'> 		if (c.dump)</div><div class='del'>-			return netlink_dump_start(nls, skb, nlh, &amp;c);</div><div class='del'>-		return -EINVAL;</div><div class='add'>+			err = netlink_dump_start(skb-&gt;sk, skb, nlh, &amp;c);</div><div class='add'>+		goto done;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (cb_table[op].doit)</div><div class='del'>-		return cb_table[op].doit(skb, nlh, extack);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='add'>+		err = cb_table[op].doit(skb, nlh, extack);</div><div class='add'>+done:</div><div class='add'>+	up_read(&amp;rdma_nl_types[index].sem);</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -253,57 +256,77 @@ skip:</div><div class='ctx'> </div><div class='ctx'> static void rdma_nl_rcv(struct sk_buff *skb)</div><div class='ctx'> {</div><div class='del'>-	mutex_lock(&amp;rdma_nl_mutex);</div><div class='ctx'> 	rdma_nl_rcv_skb(skb, &amp;rdma_nl_rcv_msg);</div><div class='del'>-	mutex_unlock(&amp;rdma_nl_mutex);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int rdma_nl_unicast(struct sk_buff *skb, u32 pid)</div><div class='add'>+int rdma_nl_unicast(struct net *net, struct sk_buff *skb, u32 pid)</div><div class='ctx'> {</div><div class='add'>+	struct rdma_dev_net *rnet = rdma_net_to_dev_net(net);</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	err = netlink_unicast(nls, skb, pid, MSG_DONTWAIT);</div><div class='add'>+	err = netlink_unicast(rnet-&gt;nl_sock, skb, pid, MSG_DONTWAIT);</div><div class='ctx'> 	return (err &lt; 0) ? err : 0;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_nl_unicast);</div><div class='ctx'> </div><div class='del'>-int rdma_nl_unicast_wait(struct sk_buff *skb, __u32 pid)</div><div class='add'>+int rdma_nl_unicast_wait(struct net *net, struct sk_buff *skb, __u32 pid)</div><div class='ctx'> {</div><div class='add'>+	struct rdma_dev_net *rnet = rdma_net_to_dev_net(net);</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	err = netlink_unicast(nls, skb, pid, 0);</div><div class='add'>+	err = netlink_unicast(rnet-&gt;nl_sock, skb, pid, 0);</div><div class='ctx'> 	return (err &lt; 0) ? err : 0;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_nl_unicast_wait);</div><div class='ctx'> </div><div class='del'>-int rdma_nl_multicast(struct sk_buff *skb, unsigned int group, gfp_t flags)</div><div class='add'>+int rdma_nl_multicast(struct net *net, struct sk_buff *skb,</div><div class='add'>+		      unsigned int group, gfp_t flags)</div><div class='ctx'> {</div><div class='del'>-	return nlmsg_multicast(nls, skb, 0, group, flags);</div><div class='add'>+	struct rdma_dev_net *rnet = rdma_net_to_dev_net(net);</div><div class='add'>+</div><div class='add'>+	return nlmsg_multicast(rnet-&gt;nl_sock, skb, 0, group, flags);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_nl_multicast);</div><div class='ctx'> </div><div class='del'>-int __init rdma_nl_init(void)</div><div class='add'>+void rdma_nl_init(void)</div><div class='ctx'> {</div><div class='add'>+	int idx;</div><div class='add'>+</div><div class='add'>+	for (idx = 0; idx &lt; RDMA_NL_NUM_CLIENTS; idx++)</div><div class='add'>+		init_rwsem(&amp;rdma_nl_types[idx].sem);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void rdma_nl_exit(void)</div><div class='add'>+{</div><div class='add'>+	int idx;</div><div class='add'>+</div><div class='add'>+	for (idx = 0; idx &lt; RDMA_NL_NUM_CLIENTS; idx++)</div><div class='add'>+		WARN(rdma_nl_types[idx].cb_table,</div><div class='add'>+		     "Netlink client %d wasn't released prior to unloading %s\n",</div><div class='add'>+		     idx, KBUILD_MODNAME);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int rdma_nl_net_init(struct rdma_dev_net *rnet)</div><div class='add'>+{</div><div class='add'>+	struct net *net = read_pnet(&amp;rnet-&gt;net);</div><div class='ctx'> 	struct netlink_kernel_cfg cfg = {</div><div class='ctx'> 		.input	= rdma_nl_rcv,</div><div class='add'>+		.flags = NL_CFG_F_NONROOT_RECV,</div><div class='ctx'> 	};</div><div class='add'>+	struct sock *nls;</div><div class='ctx'> </div><div class='del'>-	nls = netlink_kernel_create(&amp;init_net, NETLINK_RDMA, &amp;cfg);</div><div class='add'>+	nls = netlink_kernel_create(net, NETLINK_RDMA, &amp;cfg);</div><div class='ctx'> 	if (!nls)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	nls-&gt;sk_sndtimeo = 10 * HZ;</div><div class='add'>+	rnet-&gt;nl_sock = nls;</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void rdma_nl_exit(void)</div><div class='add'>+void rdma_nl_net_exit(struct rdma_dev_net *rnet)</div><div class='ctx'> {</div><div class='del'>-	int idx;</div><div class='del'>-</div><div class='del'>-	for (idx = 0; idx &lt; RDMA_NL_NUM_CLIENTS; idx++)</div><div class='del'>-		rdma_nl_unregister(idx);</div><div class='del'>-</div><div class='del'>-	netlink_kernel_release(nls);</div><div class='add'>+	netlink_kernel_release(rnet-&gt;nl_sock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> MODULE_ALIAS_NET_PF_PROTO(PF_NETLINK, NETLINK_RDMA);</div><div class='head'>diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c<br/>index 69188cbbd99b..2220a2dfab24 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/nldev.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/nldev.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/nldev.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/nldev.c</a></div><div class='hunk'>@@ -41,85 +41,137 @@</div><div class='ctx'> #include "core_priv.h"</div><div class='ctx'> #include "cma_priv.h"</div><div class='ctx'> #include "restrack.h"</div><div class='add'>+#include "uverbs.h"</div><div class='ctx'> </div><div class='add'>+/*</div><div class='add'>+ * This determines whether a non-privileged user is allowed to specify a</div><div class='add'>+ * controlled QKEY or not, when true non-privileged user is allowed to specify</div><div class='add'>+ * a controlled QKEY.</div><div class='add'>+ */</div><div class='add'>+static bool privileged_qkey;</div><div class='add'>+</div><div class='add'>+typedef int (*res_fill_func_t)(struct sk_buff*, bool,</div><div class='add'>+			       struct rdma_restrack_entry*, uint32_t);</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * Sort array elements by the netlink attribute name</div><div class='add'>+ */</div><div class='ctx'> static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = {</div><div class='del'>-	[RDMA_NLDEV_ATTR_DEV_INDEX]     = { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_DEV_NAME]	= { .type = NLA_NUL_STRING,</div><div class='del'>-					    .len = IB_DEVICE_NAME_MAX - 1},</div><div class='del'>-	[RDMA_NLDEV_ATTR_PORT_INDEX]	= { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_FW_VERSION]	= { .type = NLA_NUL_STRING,</div><div class='del'>-					    .len = IB_FW_VERSION_NAME_MAX - 1},</div><div class='del'>-	[RDMA_NLDEV_ATTR_NODE_GUID]	= { .type = NLA_U64 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_SYS_IMAGE_GUID] = { .type = NLA_U64 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_SUBNET_PREFIX]	= { .type = NLA_U64 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_LID]		= { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_SM_LID]	= { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_LMC]		= { .type = NLA_U8 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_PORT_STATE]	= { .type = NLA_U8 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_PORT_PHYS_STATE] = { .type = NLA_U8 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_DEV_NODE_TYPE] = { .type = NLA_U8 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_SUMMARY]	= { .type = NLA_NESTED },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY]	= { .type = NLA_NESTED },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME] = { .type = NLA_NUL_STRING,</div><div class='del'>-					     .len = 16 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR] = { .type = NLA_U64 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_QP]		= { .type = NLA_NESTED },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_QP_ENTRY]		= { .type = NLA_NESTED },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_LQPN]		= { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_RQPN]		= { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_RQ_PSN]		= { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_SQ_PSN]		= { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_PATH_MIG_STATE] = { .type = NLA_U8 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_TYPE]		= { .type = NLA_U8 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_STATE]		= { .type = NLA_U8 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_PID]		= { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_KERN_NAME]		= { .type = NLA_NUL_STRING,</div><div class='del'>-						    .len = TASK_COMM_LEN },</div><div class='add'>+	[RDMA_NLDEV_ATTR_CHARDEV]		= { .type = NLA_U64 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_CHARDEV_ABI]		= { .type = NLA_U64 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_CHARDEV_NAME]		= { .type = NLA_NUL_STRING,</div><div class='add'>+					.len = RDMA_NLDEV_ATTR_EMPTY_STRING },</div><div class='add'>+	[RDMA_NLDEV_ATTR_CHARDEV_TYPE]		= { .type = NLA_NUL_STRING,</div><div class='add'>+					.len = RDMA_NLDEV_ATTR_CHARDEV_TYPE_SIZE },</div><div class='add'>+	[RDMA_NLDEV_ATTR_DEV_DIM]               = { .type = NLA_U8 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_DEV_INDEX]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_DEV_NAME]		= { .type = NLA_NUL_STRING,</div><div class='add'>+					.len = IB_DEVICE_NAME_MAX },</div><div class='add'>+	[RDMA_NLDEV_ATTR_DEV_NODE_TYPE]		= { .type = NLA_U8 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_DEV_PROTOCOL]		= { .type = NLA_NUL_STRING,</div><div class='add'>+					.len = RDMA_NLDEV_ATTR_EMPTY_STRING },</div><div class='add'>+	[RDMA_NLDEV_ATTR_DRIVER]		= { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_DRIVER_ENTRY]		= { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE]	= { .type = NLA_U8 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_DRIVER_STRING]		= { .type = NLA_NUL_STRING,</div><div class='add'>+					.len = RDMA_NLDEV_ATTR_EMPTY_STRING },</div><div class='add'>+	[RDMA_NLDEV_ATTR_DRIVER_S32]		= { .type = NLA_S32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_DRIVER_S64]		= { .type = NLA_S64 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_DRIVER_U32]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_DRIVER_U64]		= { .type = NLA_U64 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_FW_VERSION]		= { .type = NLA_NUL_STRING,</div><div class='add'>+					.len = RDMA_NLDEV_ATTR_EMPTY_STRING },</div><div class='add'>+	[RDMA_NLDEV_ATTR_LID]			= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_LINK_TYPE]		= { .type = NLA_NUL_STRING,</div><div class='add'>+					.len = IFNAMSIZ },</div><div class='add'>+	[RDMA_NLDEV_ATTR_LMC]			= { .type = NLA_U8 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_NDEV_INDEX]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_NDEV_NAME]		= { .type = NLA_NUL_STRING,</div><div class='add'>+					.len = IFNAMSIZ },</div><div class='add'>+	[RDMA_NLDEV_ATTR_NODE_GUID]		= { .type = NLA_U64 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_PORT_INDEX]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_PORT_PHYS_STATE]	= { .type = NLA_U8 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_PORT_STATE]		= { .type = NLA_U8 },</div><div class='ctx'> 	[RDMA_NLDEV_ATTR_RES_CM_ID]		= { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_CM_IDN]		= { .type = NLA_U32 },</div><div class='ctx'> 	[RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY]	= { .type = NLA_NESTED },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_PS]		= { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_SRC_ADDR]	= {</div><div class='del'>-			.len = sizeof(struct __kernel_sockaddr_storage) },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_DST_ADDR]	= {</div><div class='del'>-			.len = sizeof(struct __kernel_sockaddr_storage) },</div><div class='ctx'> 	[RDMA_NLDEV_ATTR_RES_CQ]		= { .type = NLA_NESTED },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_CQ_ENTRY]		= { .type = NLA_NESTED },</div><div class='ctx'> 	[RDMA_NLDEV_ATTR_RES_CQE]		= { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_USECNT]		= { .type = NLA_U64 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_POLL_CTX]		= { .type = NLA_U8 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_MR]		= { .type = NLA_NESTED },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_MR_ENTRY]		= { .type = NLA_NESTED },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_RKEY]		= { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_LKEY]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_CQN]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_CQ_ENTRY]		= { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_CTX]		= { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_CTXN]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_CTX_ENTRY]		= { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_DST_ADDR]		= {</div><div class='add'>+			.len = sizeof(struct __kernel_sockaddr_storage) },</div><div class='ctx'> 	[RDMA_NLDEV_ATTR_RES_IOVA]		= { .type = NLA_U64 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_KERN_NAME]		= { .type = NLA_NUL_STRING,</div><div class='add'>+					.len = RDMA_NLDEV_ATTR_EMPTY_STRING },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_LKEY]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY]	= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_LQPN]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_MR]		= { .type = NLA_NESTED },</div><div class='ctx'> 	[RDMA_NLDEV_ATTR_RES_MRLEN]		= { .type = NLA_U64 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_MRN]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_MR_ENTRY]		= { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_PATH_MIG_STATE]	= { .type = NLA_U8 },</div><div class='ctx'> 	[RDMA_NLDEV_ATTR_RES_PD]		= { .type = NLA_NESTED },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_PD_ENTRY]		= { .type = NLA_NESTED },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY]	= { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY] = { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_NDEV_INDEX]		= { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_NDEV_NAME]		= { .type = NLA_NUL_STRING,</div><div class='del'>-						    .len = IFNAMSIZ },</div><div class='del'>-	[RDMA_NLDEV_ATTR_DRIVER]		= { .type = NLA_NESTED },</div><div class='del'>-	[RDMA_NLDEV_ATTR_DRIVER_ENTRY]		= { .type = NLA_NESTED },</div><div class='del'>-	[RDMA_NLDEV_ATTR_DRIVER_STRING]		= { .type = NLA_NUL_STRING,</div><div class='del'>-				    .len = RDMA_NLDEV_ATTR_ENTRY_STRLEN },</div><div class='del'>-	[RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE]	= { .type = NLA_U8 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_DRIVER_S32]		= { .type = NLA_S32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_DRIVER_U32]		= { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_DRIVER_S64]		= { .type = NLA_S64 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_DRIVER_U64]		= { .type = NLA_U64 },</div><div class='ctx'> 	[RDMA_NLDEV_ATTR_RES_PDN]		= { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_CQN]               = { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_MRN]               = { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_CM_IDN]            = { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_RES_CTXN]              = { .type = NLA_U32 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_LINK_TYPE]		= { .type = NLA_NUL_STRING,</div><div class='del'>-				    .len = RDMA_NLDEV_ATTR_ENTRY_STRLEN },</div><div class='del'>-	[RDMA_NLDEV_SYS_ATTR_NETNS_MODE]	= { .type = NLA_U8 },</div><div class='del'>-	[RDMA_NLDEV_ATTR_DEV_PROTOCOL]		= { .type = NLA_NUL_STRING,</div><div class='del'>-				    .len = RDMA_NLDEV_ATTR_ENTRY_STRLEN },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_PD_ENTRY]		= { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_PID]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_POLL_CTX]		= { .type = NLA_U8 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_PS]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_QP]		= { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_QP_ENTRY]		= { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_RAW]		= { .type = NLA_BINARY },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_RKEY]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_RQPN]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_RQ_PSN]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_SQ_PSN]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_SRC_ADDR]		= {</div><div class='add'>+			.len = sizeof(struct __kernel_sockaddr_storage) },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_STATE]		= { .type = NLA_U8 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_SUMMARY]		= { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY]	= { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR]= { .type = NLA_U64 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME]= { .type = NLA_NUL_STRING,</div><div class='add'>+					.len = RDMA_NLDEV_ATTR_EMPTY_STRING },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_TYPE]		= { .type = NLA_U8 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_SUBTYPE]		= { .type = NLA_NUL_STRING,</div><div class='add'>+					.len = RDMA_NLDEV_ATTR_EMPTY_STRING },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY]= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_USECNT]		= { .type = NLA_U64 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_SRQ]		= { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_SRQN]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_RES_SRQ_ENTRY]		= { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_MIN_RANGE]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_MAX_RANGE]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_SM_LID]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_SUBNET_PREFIX]		= { .type = NLA_U64 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK]	= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_STAT_MODE]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_STAT_RES]		= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_STAT_COUNTER]		= { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY]	= { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]       = { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS]       = { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY]  = { .type = NLA_NESTED },</div><div class='add'>+	[RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME] = { .type = NLA_NUL_STRING },</div><div class='add'>+	[RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE] = { .type = NLA_U64 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_SYS_IMAGE_GUID]	= { .type = NLA_U64 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_UVERBS_DRIVER_ID]	= { .type = NLA_U32 },</div><div class='ctx'> 	[RDMA_NLDEV_NET_NS_FD]			= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_SYS_ATTR_NETNS_MODE]	= { .type = NLA_U8 },</div><div class='add'>+	[RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK]	= { .type = NLA_U8 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_STAT_HWCOUNTER_INDEX]	= { .type = NLA_U32 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC] = { .type = NLA_U8 },</div><div class='add'>+	[RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE] = { .type = NLA_U8 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_DRIVER_DETAILS]	= { .type = NLA_U8 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_DEV_TYPE]		= { .type = NLA_U8 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_PARENT_NAME]		= { .type = NLA_NUL_STRING },</div><div class='add'>+	[RDMA_NLDEV_ATTR_NAME_ASSIGN_TYPE]	= { .type = NLA_U8 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_EVENT_TYPE]		= { .type = NLA_U8 },</div><div class='add'>+	[RDMA_NLDEV_ATTR_STAT_OPCOUNTER_ENABLED] = { .type = NLA_U8 },</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static int put_driver_name_print_type(struct sk_buff *msg, const char *name,</div><div class='hunk'>@@ -159,6 +211,19 @@ static int _rdma_nl_put_driver_u64(struct sk_buff *msg, const char *name,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+int rdma_nl_put_driver_string(struct sk_buff *msg, const char *name,</div><div class='add'>+			      const char *str)</div><div class='add'>+{</div><div class='add'>+	if (put_driver_name_print_type(msg, name,</div><div class='add'>+				       RDMA_NLDEV_PRINT_TYPE_UNSPEC))</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+	if (nla_put_string(msg, RDMA_NLDEV_ATTR_DRIVER_STRING, str))</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_nl_put_driver_string);</div><div class='add'>+</div><div class='ctx'> int rdma_nl_put_driver_u32(struct sk_buff *msg, const char *name, u32 value)</div><div class='ctx'> {</div><div class='ctx'> 	return _rdma_nl_put_driver_u32(msg, name, RDMA_NLDEV_PRINT_TYPE_UNSPEC,</div><div class='hunk'>@@ -188,6 +253,12 @@ int rdma_nl_put_driver_u64_hex(struct sk_buff *msg, const char *name, u64 value)</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_nl_put_driver_u64_hex);</div><div class='ctx'> </div><div class='add'>+bool rdma_nl_get_privileged_qkey(void)</div><div class='add'>+{</div><div class='add'>+	return privileged_qkey;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_nl_get_privileged_qkey);</div><div class='add'>+</div><div class='ctx'> static int fill_nldev_handle(struct sk_buff *msg, struct ib_device *device)</div><div class='ctx'> {</div><div class='ctx'> 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_DEV_INDEX, device-&gt;index))</div><div class='hunk'>@@ -203,7 +274,7 @@ static int fill_dev_info(struct sk_buff *msg, struct ib_device *device)</div><div class='ctx'> {</div><div class='ctx'> 	char fw[IB_FW_VERSION_NAME_MAX];</div><div class='ctx'> 	int ret = 0;</div><div class='del'>-	u8 port;</div><div class='add'>+	u32 port;</div><div class='ctx'> </div><div class='ctx'> 	if (fill_nldev_handle(msg, device))</div><div class='ctx'> 		return -EMSGSIZE;</div><div class='hunk'>@@ -232,6 +303,21 @@ static int fill_dev_info(struct sk_buff *msg, struct ib_device *device)</div><div class='ctx'> 		return -EMSGSIZE;</div><div class='ctx'> 	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_NODE_TYPE, device-&gt;node_type))</div><div class='ctx'> 		return -EMSGSIZE;</div><div class='add'>+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_DIM, device-&gt;use_cq_dim))</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	if (device-&gt;type &amp;&amp;</div><div class='add'>+	    nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_TYPE, device-&gt;type))</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	if (device-&gt;parent &amp;&amp;</div><div class='add'>+	    nla_put_string(msg, RDMA_NLDEV_ATTR_PARENT_NAME,</div><div class='add'>+			   dev_name(&amp;device-&gt;parent-&gt;dev)))</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_NAME_ASSIGN_TYPE,</div><div class='add'>+		       device-&gt;name_assign_type))</div><div class='add'>+		return -EMSGSIZE;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Link type is determined on first port and mlx4 device</div><div class='hunk'>@@ -306,8 +392,7 @@ static int fill_port_info(struct sk_buff *msg,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='del'>-	if (netdev)</div><div class='del'>-		dev_put(netdev);</div><div class='add'>+	dev_put(netdev);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -335,7 +420,8 @@ err:</div><div class='ctx'> 	return -EMSGSIZE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int fill_res_info(struct sk_buff *msg, struct ib_device *device)</div><div class='add'>+static int fill_res_info(struct sk_buff *msg, struct ib_device *device,</div><div class='add'>+			 bool show_details)</div><div class='ctx'> {</div><div class='ctx'> 	static const char * const names[RDMA_RESTRACK_MAX] = {</div><div class='ctx'> 		[RDMA_RESTRACK_PD] = "pd",</div><div class='hunk'>@@ -344,6 +430,7 @@ static int fill_res_info(struct sk_buff *msg, struct ib_device *device)</div><div class='ctx'> 		[RDMA_RESTRACK_CM_ID] = "cm_id",</div><div class='ctx'> 		[RDMA_RESTRACK_MR] = "mr",</div><div class='ctx'> 		[RDMA_RESTRACK_CTX] = "ctx",</div><div class='add'>+		[RDMA_RESTRACK_SRQ] = "srq",</div><div class='ctx'> 	};</div><div class='ctx'> </div><div class='ctx'> 	struct nlattr *table_attr;</div><div class='hunk'>@@ -359,8 +446,7 @@ static int fill_res_info(struct sk_buff *msg, struct ib_device *device)</div><div class='ctx'> 	for (i = 0; i &lt; RDMA_RESTRACK_MAX; i++) {</div><div class='ctx'> 		if (!names[i])</div><div class='ctx'> 			continue;</div><div class='del'>-		curr = rdma_restrack_count(device, i,</div><div class='del'>-					   task_active_pid_ns(current));</div><div class='add'>+		curr = rdma_restrack_count(device, i, show_details);</div><div class='ctx'> 		ret = fill_res_info_entry(msg, names[i], curr);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto err;</div><div class='hunk'>@@ -377,35 +463,41 @@ err:</div><div class='ctx'> static int fill_res_name_pid(struct sk_buff *msg,</div><div class='ctx'> 			     struct rdma_restrack_entry *res)</div><div class='ctx'> {</div><div class='add'>+	int err = 0;</div><div class='add'>+</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * For user resources, user is should read /proc/PID/comm to get the</div><div class='ctx'> 	 * name of the task file.</div><div class='ctx'> 	 */</div><div class='ctx'> 	if (rdma_is_kernel_res(res)) {</div><div class='del'>-		if (nla_put_string(msg, RDMA_NLDEV_ATTR_RES_KERN_NAME,</div><div class='del'>-		    res-&gt;kern_name))</div><div class='del'>-			return -EMSGSIZE;</div><div class='add'>+		err = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_KERN_NAME,</div><div class='add'>+				     res-&gt;kern_name);</div><div class='ctx'> 	} else {</div><div class='del'>-		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PID,</div><div class='del'>-		    task_pid_vnr(res-&gt;task)))</div><div class='del'>-			return -EMSGSIZE;</div><div class='add'>+		pid_t pid;</div><div class='add'>+</div><div class='add'>+		pid = task_pid_vnr(res-&gt;task);</div><div class='add'>+		/*</div><div class='add'>+		 * Task is dead and in zombie state.</div><div class='add'>+		 * There is no need to print PID anymore.</div><div class='add'>+		 */</div><div class='add'>+		if (pid)</div><div class='add'>+			/*</div><div class='add'>+			 * This part is racy, task can be killed and PID will</div><div class='add'>+			 * be zero right here but it is ok, next query won't</div><div class='add'>+			 * return PID. We don't promise real-time reflection</div><div class='add'>+			 * of SW objects.</div><div class='add'>+			 */</div><div class='add'>+			err = nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PID, pid);</div><div class='ctx'> 	}</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='ctx'> </div><div class='del'>-static bool fill_res_entry(struct ib_device *dev, struct sk_buff *msg,</div><div class='del'>-			   struct rdma_restrack_entry *res)</div><div class='del'>-{</div><div class='del'>-	if (!dev-&gt;ops.fill_res_entry)</div><div class='del'>-		return false;</div><div class='del'>-	return dev-&gt;ops.fill_res_entry(msg, res);</div><div class='add'>+	return err ? -EMSGSIZE : 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='del'>-			     struct rdma_restrack_entry *res, uint32_t port)</div><div class='add'>+static int fill_res_qp_entry_query(struct sk_buff *msg,</div><div class='add'>+				   struct rdma_restrack_entry *res,</div><div class='add'>+				   struct ib_device *dev,</div><div class='add'>+				   struct ib_qp *qp)</div><div class='ctx'> {</div><div class='del'>-	struct ib_qp *qp = container_of(res, struct ib_qp, res);</div><div class='del'>-	struct ib_device *dev = qp-&gt;device;</div><div class='ctx'> 	struct ib_qp_init_attr qp_init_attr;</div><div class='ctx'> 	struct ib_qp_attr qp_attr;</div><div class='ctx'> 	int ret;</div><div class='hunk'>@@ -414,16 +506,6 @@ static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	if (port &amp;&amp; port != qp_attr.port_num)</div><div class='del'>-		return -EAGAIN;</div><div class='del'>-</div><div class='del'>-	/* In create_qp() port is not set yet */</div><div class='del'>-	if (qp_attr.port_num &amp;&amp;</div><div class='del'>-	    nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, qp_attr.port_num))</div><div class='del'>-		goto err;</div><div class='del'>-</div><div class='del'>-	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qp-&gt;qp_num))</div><div class='del'>-		goto err;</div><div class='ctx'> 	if (qp-&gt;qp_type == IB_QPT_RC || qp-&gt;qp_type == IB_QPT_UC) {</div><div class='ctx'> 		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RQPN,</div><div class='ctx'> 				qp_attr.dest_qp_num))</div><div class='hunk'>@@ -447,19 +529,53 @@ static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='ctx'> 	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_STATE, qp_attr.qp_state))</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='add'>+	if (dev-&gt;ops.fill_res_qp_entry)</div><div class='add'>+		return dev-&gt;ops.fill_res_qp_entry(msg, qp);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:	return -EMSGSIZE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='add'>+			     struct rdma_restrack_entry *res, uint32_t port)</div><div class='add'>+{</div><div class='add'>+	struct ib_qp *qp = container_of(res, struct ib_qp, res);</div><div class='add'>+	struct ib_device *dev = qp-&gt;device;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (port &amp;&amp; port != qp-&gt;port)</div><div class='add'>+		return -EAGAIN;</div><div class='add'>+</div><div class='add'>+	/* In create_qp() port is not set yet */</div><div class='add'>+	if (qp-&gt;port &amp;&amp; nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, qp-&gt;port))</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	ret = nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qp-&gt;qp_num);</div><div class='add'>+	if (ret)</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='ctx'> 	if (!rdma_is_kernel_res(res) &amp;&amp;</div><div class='ctx'> 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, qp-&gt;pd-&gt;res.id))</div><div class='del'>-		goto err;</div><div class='add'>+		return -EMSGSIZE;</div><div class='ctx'> </div><div class='del'>-	if (fill_res_name_pid(msg, res))</div><div class='del'>-		goto err;</div><div class='add'>+	ret = fill_res_name_pid(msg, res);</div><div class='add'>+	if (ret)</div><div class='add'>+		return -EMSGSIZE;</div><div class='ctx'> </div><div class='del'>-	if (fill_res_entry(dev, msg, res))</div><div class='del'>-		goto err;</div><div class='add'>+	return fill_res_qp_entry_query(msg, res, dev, qp);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+static int fill_res_qp_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='add'>+				 struct rdma_restrack_entry *res, uint32_t port)</div><div class='add'>+{</div><div class='add'>+	struct ib_qp *qp = container_of(res, struct ib_qp, res);</div><div class='add'>+	struct ib_device *dev = qp-&gt;device;</div><div class='ctx'> </div><div class='del'>-err:	return -EMSGSIZE;</div><div class='add'>+	if (port &amp;&amp; port != qp-&gt;port)</div><div class='add'>+		return -EAGAIN;</div><div class='add'>+	if (!dev-&gt;ops.fill_res_qp_entry_raw)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	return dev-&gt;ops.fill_res_qp_entry_raw(msg, qp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int fill_res_cm_id_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='hunk'>@@ -471,7 +587,7 @@ static int fill_res_cm_id_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='ctx'> 	struct rdma_cm_id *cm_id = &amp;id_priv-&gt;id;</div><div class='ctx'> </div><div class='ctx'> 	if (port &amp;&amp; port != cm_id-&gt;port_num)</div><div class='del'>-		return 0;</div><div class='add'>+		return -EAGAIN;</div><div class='ctx'> </div><div class='ctx'> 	if (cm_id-&gt;port_num &amp;&amp;</div><div class='ctx'> 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, cm_id-&gt;port_num))</div><div class='hunk'>@@ -507,9 +623,8 @@ static int fill_res_cm_id_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='ctx'> 	if (fill_res_name_pid(msg, res))</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='del'>-	if (fill_res_entry(dev, msg, res))</div><div class='del'>-		goto err;</div><div class='del'>-</div><div class='add'>+	if (dev-&gt;ops.fill_res_cm_id_entry)</div><div class='add'>+		return dev-&gt;ops.fill_res_cm_id_entry(msg, cm_id);</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> err: return -EMSGSIZE;</div><div class='hunk'>@@ -522,32 +637,42 @@ static int fill_res_cq_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='ctx'> 	struct ib_device *dev = cq-&gt;device;</div><div class='ctx'> </div><div class='ctx'> 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQE, cq-&gt;cqe))</div><div class='del'>-		goto err;</div><div class='add'>+		return -EMSGSIZE;</div><div class='ctx'> 	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_USECNT,</div><div class='ctx'> 			      atomic_read(&amp;cq-&gt;usecnt), RDMA_NLDEV_ATTR_PAD))</div><div class='del'>-		goto err;</div><div class='add'>+		return -EMSGSIZE;</div><div class='ctx'> </div><div class='ctx'> 	/* Poll context is only valid for kernel CQs */</div><div class='ctx'> 	if (rdma_is_kernel_res(res) &amp;&amp;</div><div class='ctx'> 	    nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_POLL_CTX, cq-&gt;poll_ctx))</div><div class='del'>-		goto err;</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_DIM, (cq-&gt;dim != NULL)))</div><div class='add'>+		return -EMSGSIZE;</div><div class='ctx'> </div><div class='ctx'> 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN, res-&gt;id))</div><div class='del'>-		goto err;</div><div class='add'>+		return -EMSGSIZE;</div><div class='ctx'> 	if (!rdma_is_kernel_res(res) &amp;&amp;</div><div class='ctx'> 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN,</div><div class='del'>-			cq-&gt;uobject-&gt;context-&gt;res.id))</div><div class='del'>-		goto err;</div><div class='add'>+			cq-&gt;uobject-&gt;uevent.uobject.context-&gt;res.id))</div><div class='add'>+		return -EMSGSIZE;</div><div class='ctx'> </div><div class='ctx'> 	if (fill_res_name_pid(msg, res))</div><div class='del'>-		goto err;</div><div class='add'>+		return -EMSGSIZE;</div><div class='ctx'> </div><div class='del'>-	if (fill_res_entry(dev, msg, res))</div><div class='del'>-		goto err;</div><div class='add'>+	return (dev-&gt;ops.fill_res_cq_entry) ?</div><div class='add'>+		dev-&gt;ops.fill_res_cq_entry(msg, cq) : 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+static int fill_res_cq_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='add'>+				 struct rdma_restrack_entry *res, uint32_t port)</div><div class='add'>+{</div><div class='add'>+	struct ib_cq *cq = container_of(res, struct ib_cq, res);</div><div class='add'>+	struct ib_device *dev = cq-&gt;device;</div><div class='ctx'> </div><div class='del'>-err:	return -EMSGSIZE;</div><div class='add'>+	if (!dev-&gt;ops.fill_res_cq_entry_raw)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	return dev-&gt;ops.fill_res_cq_entry_raw(msg, cq);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='hunk'>@@ -558,38 +683,45 @@ static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='ctx'> </div><div class='ctx'> 	if (has_cap_net_admin) {</div><div class='ctx'> 		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RKEY, mr-&gt;rkey))</div><div class='del'>-			goto err;</div><div class='add'>+			return -EMSGSIZE;</div><div class='ctx'> 		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LKEY, mr-&gt;lkey))</div><div class='del'>-			goto err;</div><div class='add'>+			return -EMSGSIZE;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_MRLEN, mr-&gt;length,</div><div class='ctx'> 			      RDMA_NLDEV_ATTR_PAD))</div><div class='del'>-		goto err;</div><div class='add'>+		return -EMSGSIZE;</div><div class='ctx'> </div><div class='ctx'> 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res-&gt;id))</div><div class='del'>-		goto err;</div><div class='add'>+		return -EMSGSIZE;</div><div class='ctx'> </div><div class='ctx'> 	if (!rdma_is_kernel_res(res) &amp;&amp;</div><div class='ctx'> 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, mr-&gt;pd-&gt;res.id))</div><div class='del'>-		goto err;</div><div class='add'>+		return -EMSGSIZE;</div><div class='ctx'> </div><div class='ctx'> 	if (fill_res_name_pid(msg, res))</div><div class='del'>-		goto err;</div><div class='add'>+		return -EMSGSIZE;</div><div class='ctx'> </div><div class='del'>-	if (fill_res_entry(dev, msg, res))</div><div class='del'>-		goto err;</div><div class='add'>+	return (dev-&gt;ops.fill_res_mr_entry) ?</div><div class='add'>+		       dev-&gt;ops.fill_res_mr_entry(msg, mr) :</div><div class='add'>+		       0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+static int fill_res_mr_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='add'>+				 struct rdma_restrack_entry *res, uint32_t port)</div><div class='add'>+{</div><div class='add'>+	struct ib_mr *mr = container_of(res, struct ib_mr, res);</div><div class='add'>+	struct ib_device *dev = mr-&gt;pd-&gt;device;</div><div class='ctx'> </div><div class='del'>-err:	return -EMSGSIZE;</div><div class='add'>+	if (!dev-&gt;ops.fill_res_mr_entry_raw)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	return dev-&gt;ops.fill_res_mr_entry_raw(msg, mr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int fill_res_pd_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='ctx'> 			     struct rdma_restrack_entry *res, uint32_t port)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_pd *pd = container_of(res, struct ib_pd, res);</div><div class='del'>-	struct ib_device *dev = pd-&gt;device;</div><div class='ctx'> </div><div class='ctx'> 	if (has_cap_net_admin) {</div><div class='ctx'> 		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY,</div><div class='hunk'>@@ -612,15 +744,327 @@ static int fill_res_pd_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='ctx'> 			pd-&gt;uobject-&gt;context-&gt;res.id))</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='add'>+	return fill_res_name_pid(msg, res);</div><div class='add'>+</div><div class='add'>+err:	return -EMSGSIZE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_res_ctx_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='add'>+			      struct rdma_restrack_entry *res, uint32_t port)</div><div class='add'>+{</div><div class='add'>+	struct ib_ucontext *ctx = container_of(res, struct ib_ucontext, res);</div><div class='add'>+</div><div class='add'>+	if (rdma_is_kernel_res(res))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN, ctx-&gt;res.id))</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	return fill_res_name_pid(msg, res);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_res_range_qp_entry(struct sk_buff *msg, uint32_t min_range,</div><div class='add'>+				   uint32_t max_range)</div><div class='add'>+{</div><div class='add'>+	struct nlattr *entry_attr;</div><div class='add'>+</div><div class='add'>+	if (!min_range)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP_ENTRY);</div><div class='add'>+	if (!entry_attr)</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	if (min_range == max_range) {</div><div class='add'>+		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, min_range))</div><div class='add'>+			goto err;</div><div class='add'>+	} else {</div><div class='add'>+		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_MIN_RANGE, min_range))</div><div class='add'>+			goto err;</div><div class='add'>+		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_MAX_RANGE, max_range))</div><div class='add'>+			goto err;</div><div class='add'>+	}</div><div class='add'>+	nla_nest_end(msg, entry_attr);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	nla_nest_cancel(msg, entry_attr);</div><div class='add'>+	return -EMSGSIZE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_res_srq_qps(struct sk_buff *msg, struct ib_srq *srq)</div><div class='add'>+{</div><div class='add'>+	uint32_t min_range = 0, prev = 0;</div><div class='add'>+	struct rdma_restrack_entry *res;</div><div class='add'>+	struct rdma_restrack_root *rt;</div><div class='add'>+	struct nlattr *table_attr;</div><div class='add'>+	struct ib_qp *qp = NULL;</div><div class='add'>+	unsigned long id = 0;</div><div class='add'>+</div><div class='add'>+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP);</div><div class='add'>+	if (!table_attr)</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	rt = &amp;srq-&gt;device-&gt;res[RDMA_RESTRACK_QP];</div><div class='add'>+	xa_lock(&amp;rt-&gt;xa);</div><div class='add'>+	xa_for_each(&amp;rt-&gt;xa, id, res) {</div><div class='add'>+		if (!rdma_restrack_get(res))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		qp = container_of(res, struct ib_qp, res);</div><div class='add'>+		if (!qp-&gt;srq || (qp-&gt;srq-&gt;res.id != srq-&gt;res.id)) {</div><div class='add'>+			rdma_restrack_put(res);</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (qp-&gt;qp_num &lt; prev)</div><div class='add'>+			/* qp_num should be ascending */</div><div class='add'>+			goto err_loop;</div><div class='add'>+</div><div class='add'>+		if (min_range == 0) {</div><div class='add'>+			min_range = qp-&gt;qp_num;</div><div class='add'>+		} else if (qp-&gt;qp_num &gt; (prev + 1)) {</div><div class='add'>+			if (fill_res_range_qp_entry(msg, min_range, prev))</div><div class='add'>+				goto err_loop;</div><div class='add'>+</div><div class='add'>+			min_range = qp-&gt;qp_num;</div><div class='add'>+		}</div><div class='add'>+		prev = qp-&gt;qp_num;</div><div class='add'>+		rdma_restrack_put(res);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	xa_unlock(&amp;rt-&gt;xa);</div><div class='add'>+</div><div class='add'>+	if (fill_res_range_qp_entry(msg, min_range, prev))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	nla_nest_end(msg, table_attr);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_loop:</div><div class='add'>+	rdma_restrack_put(res);</div><div class='add'>+	xa_unlock(&amp;rt-&gt;xa);</div><div class='add'>+err:</div><div class='add'>+	nla_nest_cancel(msg, table_attr);</div><div class='add'>+	return -EMSGSIZE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_res_srq_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='add'>+			      struct rdma_restrack_entry *res, uint32_t port)</div><div class='add'>+{</div><div class='add'>+	struct ib_srq *srq = container_of(res, struct ib_srq, res);</div><div class='add'>+	struct ib_device *dev = srq-&gt;device;</div><div class='add'>+</div><div class='add'>+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_SRQN, srq-&gt;res.id))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, srq-&gt;srq_type))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, srq-&gt;pd-&gt;res.id))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	if (ib_srq_has_cq(srq-&gt;srq_type)) {</div><div class='add'>+		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN,</div><div class='add'>+				srq-&gt;ext.cq-&gt;res.id))</div><div class='add'>+			goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (fill_res_srq_qps(msg, srq))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='ctx'> 	if (fill_res_name_pid(msg, res))</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='del'>-	if (fill_res_entry(dev, msg, res))</div><div class='add'>+	if (dev-&gt;ops.fill_res_srq_entry)</div><div class='add'>+		return dev-&gt;ops.fill_res_srq_entry(msg, srq);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	return -EMSGSIZE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_res_srq_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='add'>+				 struct rdma_restrack_entry *res, uint32_t port)</div><div class='add'>+{</div><div class='add'>+	struct ib_srq *srq = container_of(res, struct ib_srq, res);</div><div class='add'>+	struct ib_device *dev = srq-&gt;device;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;ops.fill_res_srq_entry_raw)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	return dev-&gt;ops.fill_res_srq_entry_raw(msg, srq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_stat_counter_mode(struct sk_buff *msg,</div><div class='add'>+				  struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	struct rdma_counter_mode *m = &amp;counter-&gt;mode;</div><div class='add'>+</div><div class='add'>+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_MODE, m-&gt;mode))</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	if (m-&gt;mode == RDMA_COUNTER_MODE_AUTO) {</div><div class='add'>+		if ((m-&gt;mask &amp; RDMA_COUNTER_MASK_QP_TYPE) &amp;&amp;</div><div class='add'>+		    nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, m-&gt;param.qp_type))</div><div class='add'>+			return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+		if ((m-&gt;mask &amp; RDMA_COUNTER_MASK_PID) &amp;&amp;</div><div class='add'>+		    fill_res_name_pid(msg, &amp;counter-&gt;res))</div><div class='add'>+			return -EMSGSIZE;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_stat_counter_qp_entry(struct sk_buff *msg, u32 qpn)</div><div class='add'>+{</div><div class='add'>+	struct nlattr *entry_attr;</div><div class='add'>+</div><div class='add'>+	entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP_ENTRY);</div><div class='add'>+	if (!entry_attr)</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qpn))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	nla_nest_end(msg, entry_attr);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	nla_nest_cancel(msg, entry_attr);</div><div class='add'>+	return -EMSGSIZE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_stat_counter_qps(struct sk_buff *msg,</div><div class='add'>+				 struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	struct rdma_restrack_entry *res;</div><div class='add'>+	struct rdma_restrack_root *rt;</div><div class='add'>+	struct nlattr *table_attr;</div><div class='add'>+	struct ib_qp *qp = NULL;</div><div class='add'>+	unsigned long id = 0;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP);</div><div class='add'>+	if (!table_attr)</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	rt = &amp;counter-&gt;device-&gt;res[RDMA_RESTRACK_QP];</div><div class='add'>+	xa_lock(&amp;rt-&gt;xa);</div><div class='add'>+	xa_for_each(&amp;rt-&gt;xa, id, res) {</div><div class='add'>+		qp = container_of(res, struct ib_qp, res);</div><div class='add'>+		if (!qp-&gt;counter || (qp-&gt;counter-&gt;id != counter-&gt;id))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		ret = fill_stat_counter_qp_entry(msg, qp-&gt;qp_num);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	xa_unlock(&amp;rt-&gt;xa);</div><div class='add'>+	nla_nest_end(msg, table_attr);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	xa_unlock(&amp;rt-&gt;xa);</div><div class='add'>+	nla_nest_cancel(msg, table_attr);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int rdma_nl_stat_hwcounter_entry(struct sk_buff *msg, const char *name,</div><div class='add'>+				 u64 value)</div><div class='add'>+{</div><div class='add'>+	struct nlattr *entry_attr;</div><div class='add'>+</div><div class='add'>+	entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY);</div><div class='add'>+	if (!entry_attr)</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	if (nla_put_string(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME,</div><div class='add'>+			   name))</div><div class='add'>+		goto err;</div><div class='add'>+	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE,</div><div class='add'>+			      value, RDMA_NLDEV_ATTR_PAD))</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='add'>+	nla_nest_end(msg, entry_attr);</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-err:	return -EMSGSIZE;</div><div class='add'>+err:</div><div class='add'>+	nla_nest_cancel(msg, entry_attr);</div><div class='add'>+	return -EMSGSIZE;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_nl_stat_hwcounter_entry);</div><div class='add'>+</div><div class='add'>+static int fill_stat_mr_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='add'>+			      struct rdma_restrack_entry *res, uint32_t port)</div><div class='add'>+{</div><div class='add'>+	struct ib_mr *mr = container_of(res, struct ib_mr, res);</div><div class='add'>+	struct ib_device *dev = mr-&gt;pd-&gt;device;</div><div class='add'>+</div><div class='add'>+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res-&gt;id))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;ops.fill_stat_mr_entry)</div><div class='add'>+		return dev-&gt;ops.fill_stat_mr_entry(msg, mr);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	return -EMSGSIZE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_stat_counter_hwcounters(struct sk_buff *msg,</div><div class='add'>+					struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	struct rdma_hw_stats *st = counter-&gt;stats;</div><div class='add'>+	struct nlattr *table_attr;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS);</div><div class='add'>+	if (!table_attr)</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;st-&gt;lock);</div><div class='add'>+	for (i = 0; i &lt; st-&gt;num_counters; i++) {</div><div class='add'>+		if (test_bit(i, st-&gt;is_disabled))</div><div class='add'>+			continue;</div><div class='add'>+		if (rdma_nl_stat_hwcounter_entry(msg, st-&gt;descs[i].name,</div><div class='add'>+						 st-&gt;value[i]))</div><div class='add'>+			goto err;</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;st-&gt;lock);</div><div class='add'>+</div><div class='add'>+	nla_nest_end(msg, table_attr);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	mutex_unlock(&amp;st-&gt;lock);</div><div class='add'>+	nla_nest_cancel(msg, table_attr);</div><div class='add'>+	return -EMSGSIZE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_res_counter_entry(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='add'>+				  struct rdma_restrack_entry *res,</div><div class='add'>+				  uint32_t port)</div><div class='add'>+{</div><div class='add'>+	struct rdma_counter *counter =</div><div class='add'>+		container_of(res, struct rdma_counter, res);</div><div class='add'>+</div><div class='add'>+	if (port &amp;&amp; port != counter-&gt;port)</div><div class='add'>+		return -EAGAIN;</div><div class='add'>+</div><div class='add'>+	/* Dump it even query failed */</div><div class='add'>+	rdma_counter_query_stats(counter);</div><div class='add'>+</div><div class='add'>+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, counter-&gt;port) ||</div><div class='add'>+	    nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, counter-&gt;id) ||</div><div class='add'>+	    fill_stat_counter_mode(msg, counter) ||</div><div class='add'>+	    fill_stat_counter_qps(msg, counter) ||</div><div class='add'>+	    fill_stat_counter_hwcounters(msg, counter))</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='hunk'>@@ -632,8 +1076,8 @@ static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	u32 index;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='del'>-				     nldev_policy, extack);</div><div class='add'>+	err = __nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='add'>+			    nldev_policy, NL_VALIDATE_LIBERAL, extack);</div><div class='ctx'> 	if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='hunk'>@@ -652,6 +1096,10 @@ static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh-&gt;nlmsg_seq,</div><div class='ctx'> 			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),</div><div class='ctx'> 			0, 0);</div><div class='add'>+	if (!nlh) {</div><div class='add'>+		err = -EMSGSIZE;</div><div class='add'>+		goto err_free;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	err = fill_dev_info(msg, device);</div><div class='ctx'> 	if (err)</div><div class='hunk'>@@ -660,7 +1108,7 @@ static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	nlmsg_end(msg, nlh);</div><div class='ctx'> </div><div class='ctx'> 	ib_device_put(device);</div><div class='del'>-	return rdma_nl_unicast(msg, NETLINK_CB(skb).portid);</div><div class='add'>+	return rdma_nl_unicast(sock_net(skb-&gt;sk), msg, NETLINK_CB(skb).portid);</div><div class='ctx'> </div><div class='ctx'> err_free:</div><div class='ctx'> 	nlmsg_free(msg);</div><div class='hunk'>@@ -677,8 +1125,8 @@ static int nldev_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	u32 index;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='del'>-				     nldev_policy, extack);</div><div class='add'>+	err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='add'>+			    nldev_policy, extack);</div><div class='ctx'> 	if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='hunk'>@@ -690,8 +1138,12 @@ static int nldev_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	if (tb[RDMA_NLDEV_ATTR_DEV_NAME]) {</div><div class='ctx'> 		char name[IB_DEVICE_NAME_MAX] = {};</div><div class='ctx'> </div><div class='del'>-		nla_strlcpy(name, tb[RDMA_NLDEV_ATTR_DEV_NAME],</div><div class='add'>+		nla_strscpy(name, tb[RDMA_NLDEV_ATTR_DEV_NAME],</div><div class='ctx'> 			    IB_DEVICE_NAME_MAX);</div><div class='add'>+		if (strlen(name) == 0) {</div><div class='add'>+			err = -EINVAL;</div><div class='add'>+			goto done;</div><div class='add'>+		}</div><div class='ctx'> 		err = ib_device_rename(device, name);</div><div class='ctx'> 		goto done;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -704,6 +1156,14 @@ static int nldev_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 		goto put_done;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (tb[RDMA_NLDEV_ATTR_DEV_DIM]) {</div><div class='add'>+		u8 use_dim;</div><div class='add'>+</div><div class='add'>+		use_dim = nla_get_u8(tb[RDMA_NLDEV_ATTR_DEV_DIM]);</div><div class='add'>+		err = ib_device_set_dim(device,  use_dim);</div><div class='add'>+		goto done;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> done:</div><div class='ctx'> 	ib_device_put(device);</div><div class='ctx'> put_done:</div><div class='hunk'>@@ -725,7 +1185,7 @@ static int _nldev_get_dumpit(struct ib_device *device,</div><div class='ctx'> 			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),</div><div class='ctx'> 			0, NLM_F_MULTI);</div><div class='ctx'> </div><div class='del'>-	if (fill_dev_info(skb, device)) {</div><div class='add'>+	if (!nlh || fill_dev_info(skb, device)) {</div><div class='ctx'> 		nlmsg_cancel(skb, nlh);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -757,8 +1217,8 @@ static int nldev_port_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	u32 port;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='del'>-				     nldev_policy, extack);</div><div class='add'>+	err = __nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='add'>+			    nldev_policy, NL_VALIDATE_LIBERAL, extack);</div><div class='ctx'> 	if (err ||</div><div class='ctx'> 	    !tb[RDMA_NLDEV_ATTR_DEV_INDEX] ||</div><div class='ctx'> 	    !tb[RDMA_NLDEV_ATTR_PORT_INDEX])</div><div class='hunk'>@@ -784,6 +1244,10 @@ static int nldev_port_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh-&gt;nlmsg_seq,</div><div class='ctx'> 			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),</div><div class='ctx'> 			0, 0);</div><div class='add'>+	if (!nlh) {</div><div class='add'>+		err = -EMSGSIZE;</div><div class='add'>+		goto err_free;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	err = fill_port_info(msg, device, port, sock_net(skb-&gt;sk));</div><div class='ctx'> 	if (err)</div><div class='hunk'>@@ -792,7 +1256,7 @@ static int nldev_port_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	nlmsg_end(msg, nlh);</div><div class='ctx'> 	ib_device_put(device);</div><div class='ctx'> </div><div class='del'>-	return rdma_nl_unicast(msg, NETLINK_CB(skb).portid);</div><div class='add'>+	return rdma_nl_unicast(sock_net(skb-&gt;sk), msg, NETLINK_CB(skb).portid);</div><div class='ctx'> </div><div class='ctx'> err_free:</div><div class='ctx'> 	nlmsg_free(msg);</div><div class='hunk'>@@ -813,8 +1277,8 @@ static int nldev_port_get_dumpit(struct sk_buff *skb,</div><div class='ctx'> 	int err;</div><div class='ctx'> 	unsigned int p;</div><div class='ctx'> </div><div class='del'>-	err = nlmsg_parse_deprecated(cb-&gt;nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='del'>-				     nldev_policy, NULL);</div><div class='add'>+	err = __nlmsg_parse(cb-&gt;nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='add'>+			    nldev_policy, NL_VALIDATE_LIBERAL, NULL);</div><div class='ctx'> 	if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='hunk'>@@ -845,7 +1309,7 @@ static int nldev_port_get_dumpit(struct sk_buff *skb,</div><div class='ctx'> 						 RDMA_NLDEV_CMD_PORT_GET),</div><div class='ctx'> 				0, NLM_F_MULTI);</div><div class='ctx'> </div><div class='del'>-		if (fill_port_info(skb, device, p, sock_net(skb-&gt;sk))) {</div><div class='add'>+		if (!nlh || fill_port_info(skb, device, p, sock_net(skb-&gt;sk))) {</div><div class='ctx'> 			nlmsg_cancel(skb, nlh);</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -863,13 +1327,14 @@ static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 			      struct netlink_ext_ack *extack)</div><div class='ctx'> {</div><div class='ctx'> 	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];</div><div class='add'>+	bool show_details = false;</div><div class='ctx'> 	struct ib_device *device;</div><div class='ctx'> 	struct sk_buff *msg;</div><div class='ctx'> 	u32 index;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='del'>-				     nldev_policy, extack);</div><div class='add'>+	ret = __nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='add'>+			    nldev_policy, NL_VALIDATE_LIBERAL, extack);</div><div class='ctx'> 	if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='hunk'>@@ -878,6 +1343,9 @@ static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	if (!device)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='add'>+	if (tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS])</div><div class='add'>+		show_details = nla_get_u8(tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]);</div><div class='add'>+</div><div class='ctx'> 	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);</div><div class='ctx'> 	if (!msg) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='hunk'>@@ -887,14 +1355,18 @@ static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh-&gt;nlmsg_seq,</div><div class='ctx'> 			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),</div><div class='ctx'> 			0, 0);</div><div class='add'>+	if (!nlh) {</div><div class='add'>+		ret = -EMSGSIZE;</div><div class='add'>+		goto err_free;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	ret = fill_res_info(msg, device);</div><div class='add'>+	ret = fill_res_info(msg, device, show_details);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err_free;</div><div class='ctx'> </div><div class='ctx'> 	nlmsg_end(msg, nlh);</div><div class='ctx'> 	ib_device_put(device);</div><div class='del'>-	return rdma_nl_unicast(msg, NETLINK_CB(skb).portid);</div><div class='add'>+	return rdma_nl_unicast(sock_net(skb-&gt;sk), msg, NETLINK_CB(skb).portid);</div><div class='ctx'> </div><div class='ctx'> err_free:</div><div class='ctx'> 	nlmsg_free(msg);</div><div class='hunk'>@@ -918,7 +1390,7 @@ static int _nldev_res_get_dumpit(struct ib_device *device,</div><div class='ctx'> 			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),</div><div class='ctx'> 			0, NLM_F_MULTI);</div><div class='ctx'> </div><div class='del'>-	if (fill_res_info(skb, device)) {</div><div class='add'>+	if (!nlh || fill_res_info(skb, device, false)) {</div><div class='ctx'> 		nlmsg_cancel(skb, nlh);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -938,10 +1410,7 @@ static int nldev_res_get_dumpit(struct sk_buff *skb,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct nldev_fill_res_entry {</div><div class='del'>-	int (*fill_res_func)(struct sk_buff *msg, bool has_cap_net_admin,</div><div class='del'>-			     struct rdma_restrack_entry *res, u32 port);</div><div class='ctx'> 	enum rdma_nldev_attr nldev_attr;</div><div class='del'>-	enum rdma_nldev_command nldev_cmd;</div><div class='ctx'> 	u8 flags;</div><div class='ctx'> 	u32 entry;</div><div class='ctx'> 	u32 id;</div><div class='hunk'>@@ -953,59 +1422,58 @@ enum nldev_res_flags {</div><div class='ctx'> </div><div class='ctx'> static const struct nldev_fill_res_entry fill_entries[RDMA_RESTRACK_MAX] = {</div><div class='ctx'> 	[RDMA_RESTRACK_QP] = {</div><div class='del'>-		.fill_res_func = fill_res_qp_entry,</div><div class='del'>-		.nldev_cmd = RDMA_NLDEV_CMD_RES_QP_GET,</div><div class='ctx'> 		.nldev_attr = RDMA_NLDEV_ATTR_RES_QP,</div><div class='ctx'> 		.entry = RDMA_NLDEV_ATTR_RES_QP_ENTRY,</div><div class='ctx'> 		.id = RDMA_NLDEV_ATTR_RES_LQPN,</div><div class='ctx'> 	},</div><div class='ctx'> 	[RDMA_RESTRACK_CM_ID] = {</div><div class='del'>-		.fill_res_func = fill_res_cm_id_entry,</div><div class='del'>-		.nldev_cmd = RDMA_NLDEV_CMD_RES_CM_ID_GET,</div><div class='ctx'> 		.nldev_attr = RDMA_NLDEV_ATTR_RES_CM_ID,</div><div class='ctx'> 		.entry = RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY,</div><div class='ctx'> 		.id = RDMA_NLDEV_ATTR_RES_CM_IDN,</div><div class='ctx'> 	},</div><div class='ctx'> 	[RDMA_RESTRACK_CQ] = {</div><div class='del'>-		.fill_res_func = fill_res_cq_entry,</div><div class='del'>-		.nldev_cmd = RDMA_NLDEV_CMD_RES_CQ_GET,</div><div class='ctx'> 		.nldev_attr = RDMA_NLDEV_ATTR_RES_CQ,</div><div class='ctx'> 		.flags = NLDEV_PER_DEV,</div><div class='ctx'> 		.entry = RDMA_NLDEV_ATTR_RES_CQ_ENTRY,</div><div class='ctx'> 		.id = RDMA_NLDEV_ATTR_RES_CQN,</div><div class='ctx'> 	},</div><div class='ctx'> 	[RDMA_RESTRACK_MR] = {</div><div class='del'>-		.fill_res_func = fill_res_mr_entry,</div><div class='del'>-		.nldev_cmd = RDMA_NLDEV_CMD_RES_MR_GET,</div><div class='ctx'> 		.nldev_attr = RDMA_NLDEV_ATTR_RES_MR,</div><div class='ctx'> 		.flags = NLDEV_PER_DEV,</div><div class='ctx'> 		.entry = RDMA_NLDEV_ATTR_RES_MR_ENTRY,</div><div class='ctx'> 		.id = RDMA_NLDEV_ATTR_RES_MRN,</div><div class='ctx'> 	},</div><div class='ctx'> 	[RDMA_RESTRACK_PD] = {</div><div class='del'>-		.fill_res_func = fill_res_pd_entry,</div><div class='del'>-		.nldev_cmd = RDMA_NLDEV_CMD_RES_PD_GET,</div><div class='ctx'> 		.nldev_attr = RDMA_NLDEV_ATTR_RES_PD,</div><div class='ctx'> 		.flags = NLDEV_PER_DEV,</div><div class='ctx'> 		.entry = RDMA_NLDEV_ATTR_RES_PD_ENTRY,</div><div class='ctx'> 		.id = RDMA_NLDEV_ATTR_RES_PDN,</div><div class='ctx'> 	},</div><div class='del'>-};</div><div class='add'>+	[RDMA_RESTRACK_COUNTER] = {</div><div class='add'>+		.nldev_attr = RDMA_NLDEV_ATTR_STAT_COUNTER,</div><div class='add'>+		.entry = RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY,</div><div class='add'>+		.id = RDMA_NLDEV_ATTR_STAT_COUNTER_ID,</div><div class='add'>+	},</div><div class='add'>+	[RDMA_RESTRACK_CTX] = {</div><div class='add'>+		.nldev_attr = RDMA_NLDEV_ATTR_RES_CTX,</div><div class='add'>+		.flags = NLDEV_PER_DEV,</div><div class='add'>+		.entry = RDMA_NLDEV_ATTR_RES_CTX_ENTRY,</div><div class='add'>+		.id = RDMA_NLDEV_ATTR_RES_CTXN,</div><div class='add'>+	},</div><div class='add'>+	[RDMA_RESTRACK_SRQ] = {</div><div class='add'>+		.nldev_attr = RDMA_NLDEV_ATTR_RES_SRQ,</div><div class='add'>+		.flags = NLDEV_PER_DEV,</div><div class='add'>+		.entry = RDMA_NLDEV_ATTR_RES_SRQ_ENTRY,</div><div class='add'>+		.id = RDMA_NLDEV_ATTR_RES_SRQN,</div><div class='add'>+	},</div><div class='ctx'> </div><div class='del'>-static bool is_visible_in_pid_ns(struct rdma_restrack_entry *res)</div><div class='del'>-{</div><div class='del'>-	/*</div><div class='del'>-	 * 1. Kern resources should be visible in init name space only</div><div class='del'>-	 * 2. Present only resources visible in the current namespace</div><div class='del'>-	 */</div><div class='del'>-	if (rdma_is_kernel_res(res))</div><div class='del'>-		return task_active_pid_ns(current) == &amp;init_pid_ns;</div><div class='del'>-	return task_active_pid_ns(current) == task_active_pid_ns(res-&gt;task);</div><div class='del'>-}</div><div class='add'>+};</div><div class='ctx'> </div><div class='del'>-static int res_get_common_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='del'>-			       struct netlink_ext_ack *extack,</div><div class='del'>-			       enum rdma_restrack_type res_type)</div><div class='add'>+static noinline_for_stack int</div><div class='add'>+res_get_common_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='add'>+		    struct netlink_ext_ack *extack,</div><div class='add'>+		    enum rdma_restrack_type res_type,</div><div class='add'>+		    res_fill_func_t fill_func)</div><div class='ctx'> {</div><div class='ctx'> 	const struct nldev_fill_res_entry *fe = &amp;fill_entries[res_type];</div><div class='ctx'> 	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];</div><div class='hunk'>@@ -1016,8 +1484,8 @@ static int res_get_common_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	struct sk_buff *msg;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='del'>-				     nldev_policy, extack);</div><div class='add'>+	ret = __nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='add'>+			    nldev_policy, NL_VALIDATE_LIBERAL, extack);</div><div class='ctx'> 	if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !fe-&gt;id || !tb[fe-&gt;id])</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='hunk'>@@ -1047,35 +1515,32 @@ static int res_get_common_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 		goto err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!is_visible_in_pid_ns(res)) {</div><div class='del'>-		ret = -ENOENT;</div><div class='del'>-		goto err_get;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);</div><div class='ctx'> 	if (!msg) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='del'>-		goto err;</div><div class='add'>+		goto err_get;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh-&gt;nlmsg_seq,</div><div class='del'>-			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, fe-&gt;nldev_cmd),</div><div class='add'>+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,</div><div class='add'>+					 RDMA_NL_GET_OP(nlh-&gt;nlmsg_type)),</div><div class='ctx'> 			0, 0);</div><div class='ctx'> </div><div class='del'>-	if (fill_nldev_handle(msg, device)) {</div><div class='add'>+	if (!nlh || fill_nldev_handle(msg, device)) {</div><div class='ctx'> 		ret = -EMSGSIZE;</div><div class='ctx'> 		goto err_free;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	has_cap_net_admin = netlink_capable(skb, CAP_NET_ADMIN);</div><div class='del'>-	ret = fe-&gt;fill_res_func(msg, has_cap_net_admin, res, port);</div><div class='del'>-	rdma_restrack_put(res);</div><div class='add'>+</div><div class='add'>+	ret = fill_func(msg, has_cap_net_admin, res, port);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err_free;</div><div class='ctx'> </div><div class='add'>+	rdma_restrack_put(res);</div><div class='ctx'> 	nlmsg_end(msg, nlh);</div><div class='ctx'> 	ib_device_put(device);</div><div class='del'>-	return rdma_nl_unicast(msg, NETLINK_CB(skb).portid);</div><div class='add'>+	return rdma_nl_unicast(sock_net(skb-&gt;sk), msg, NETLINK_CB(skb).portid);</div><div class='ctx'> </div><div class='ctx'> err_free:</div><div class='ctx'> 	nlmsg_free(msg);</div><div class='hunk'>@@ -1088,13 +1553,15 @@ err:</div><div class='ctx'> </div><div class='ctx'> static int res_get_common_dumpit(struct sk_buff *skb,</div><div class='ctx'> 				 struct netlink_callback *cb,</div><div class='del'>-				 enum rdma_restrack_type res_type)</div><div class='add'>+				 enum rdma_restrack_type res_type,</div><div class='add'>+				 res_fill_func_t fill_func)</div><div class='ctx'> {</div><div class='ctx'> 	const struct nldev_fill_res_entry *fe = &amp;fill_entries[res_type];</div><div class='ctx'> 	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];</div><div class='ctx'> 	struct rdma_restrack_entry *res;</div><div class='ctx'> 	struct rdma_restrack_root *rt;</div><div class='ctx'> 	int err, ret = 0, idx = 0;</div><div class='add'>+	bool show_details = false;</div><div class='ctx'> 	struct nlattr *table_attr;</div><div class='ctx'> 	struct nlattr *entry_attr;</div><div class='ctx'> 	struct ib_device *device;</div><div class='hunk'>@@ -1105,8 +1572,8 @@ static int res_get_common_dumpit(struct sk_buff *skb,</div><div class='ctx'> 	u32 index, port = 0;</div><div class='ctx'> 	bool filled = false;</div><div class='ctx'> </div><div class='del'>-	err = nlmsg_parse_deprecated(cb-&gt;nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='del'>-				     nldev_policy, NULL);</div><div class='add'>+	err = __nlmsg_parse(cb-&gt;nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='add'>+			    nldev_policy, NL_VALIDATE_LIBERAL, NULL);</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Right now, we are expecting the device index to get res information,</div><div class='ctx'> 	 * but it is possible to extend this code to return all devices in</div><div class='hunk'>@@ -1123,6 +1590,9 @@ static int res_get_common_dumpit(struct sk_buff *skb,</div><div class='ctx'> 	if (!device)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='add'>+	if (tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS])</div><div class='add'>+		show_details = nla_get_u8(tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]);</div><div class='add'>+</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * If no PORT_INDEX is supplied, we will return all QPs from that device</div><div class='ctx'> 	 */</div><div class='hunk'>@@ -1135,10 +1605,11 @@ static int res_get_common_dumpit(struct sk_buff *skb,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	nlh = nlmsg_put(skb, NETLINK_CB(cb-&gt;skb).portid, cb-&gt;nlh-&gt;nlmsg_seq,</div><div class='del'>-			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, fe-&gt;nldev_cmd),</div><div class='add'>+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,</div><div class='add'>+					 RDMA_NL_GET_OP(cb-&gt;nlh-&gt;nlmsg_type)),</div><div class='ctx'> 			0, NLM_F_MULTI);</div><div class='ctx'> </div><div class='del'>-	if (fill_nldev_handle(skb, device)) {</div><div class='add'>+	if (!nlh || fill_nldev_handle(skb, device)) {</div><div class='ctx'> 		ret = -EMSGSIZE;</div><div class='ctx'> 		goto err;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -1159,8 +1630,8 @@ static int res_get_common_dumpit(struct sk_buff *skb,</div><div class='ctx'> 	 * objects.</div><div class='ctx'> 	 */</div><div class='ctx'> 	xa_for_each(&amp;rt-&gt;xa, id, res) {</div><div class='del'>-		if (!is_visible_in_pid_ns(res))</div><div class='del'>-			continue;</div><div class='add'>+		if (xa_get_mark(&amp;rt-&gt;xa, res-&gt;id, RESTRACK_DD) &amp;&amp; !show_details)</div><div class='add'>+			goto next;</div><div class='ctx'> </div><div class='ctx'> 		if (idx &lt; start || !rdma_restrack_get(res))</div><div class='ctx'> 			goto next;</div><div class='hunk'>@@ -1176,7 +1647,8 @@ static int res_get_common_dumpit(struct sk_buff *skb,</div><div class='ctx'> 			goto msg_full;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		ret = fe-&gt;fill_res_func(skb, has_cap_net_admin, res, port);</div><div class='add'>+		ret = fill_func(skb, has_cap_net_admin, res, port);</div><div class='add'>+</div><div class='ctx'> 		rdma_restrack_put(res);</div><div class='ctx'> </div><div class='ctx'> 		if (ret) {</div><div class='hunk'>@@ -1223,20 +1695,29 @@ err_index:</div><div class='ctx'> 	static int nldev_res_get_##name##_dumpit(struct sk_buff *skb,          \</div><div class='ctx'> 						 struct netlink_callback *cb)  \</div><div class='ctx'> 	{                                                                      \</div><div class='del'>-		return res_get_common_dumpit(skb, cb, type);                   \</div><div class='add'>+		return res_get_common_dumpit(skb, cb, type,                    \</div><div class='add'>+					     fill_res_##name##_entry);         \</div><div class='ctx'> 	}                                                                      \</div><div class='ctx'> 	static int nldev_res_get_##name##_doit(struct sk_buff *skb,            \</div><div class='ctx'> 					       struct nlmsghdr *nlh,           \</div><div class='ctx'> 					       struct netlink_ext_ack *extack) \</div><div class='ctx'> 	{                                                                      \</div><div class='del'>-		return res_get_common_doit(skb, nlh, extack, type);            \</div><div class='add'>+		return res_get_common_doit(skb, nlh, extack, type,             \</div><div class='add'>+					   fill_res_##name##_entry);           \</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> RES_GET_FUNCS(qp, RDMA_RESTRACK_QP);</div><div class='add'>+RES_GET_FUNCS(qp_raw, RDMA_RESTRACK_QP);</div><div class='ctx'> RES_GET_FUNCS(cm_id, RDMA_RESTRACK_CM_ID);</div><div class='ctx'> RES_GET_FUNCS(cq, RDMA_RESTRACK_CQ);</div><div class='add'>+RES_GET_FUNCS(cq_raw, RDMA_RESTRACK_CQ);</div><div class='ctx'> RES_GET_FUNCS(pd, RDMA_RESTRACK_PD);</div><div class='ctx'> RES_GET_FUNCS(mr, RDMA_RESTRACK_MR);</div><div class='add'>+RES_GET_FUNCS(mr_raw, RDMA_RESTRACK_MR);</div><div class='add'>+RES_GET_FUNCS(counter, RDMA_RESTRACK_COUNTER);</div><div class='add'>+RES_GET_FUNCS(ctx, RDMA_RESTRACK_CTX);</div><div class='add'>+RES_GET_FUNCS(srq, RDMA_RESTRACK_SRQ);</div><div class='add'>+RES_GET_FUNCS(srq_raw, RDMA_RESTRACK_SRQ);</div><div class='ctx'> </div><div class='ctx'> static LIST_HEAD(link_ops);</div><div class='ctx'> static DECLARE_RWSEM(link_ops_rwsem);</div><div class='hunk'>@@ -1284,22 +1765,22 @@ static int nldev_newlink(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	char type[IFNAMSIZ];</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='del'>-				     nldev_policy, extack);</div><div class='add'>+	err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='add'>+			    nldev_policy, extack);</div><div class='ctx'> 	if (err || !tb[RDMA_NLDEV_ATTR_DEV_NAME] ||</div><div class='ctx'> 	    !tb[RDMA_NLDEV_ATTR_LINK_TYPE] || !tb[RDMA_NLDEV_ATTR_NDEV_NAME])</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	nla_strlcpy(ibdev_name, tb[RDMA_NLDEV_ATTR_DEV_NAME],</div><div class='add'>+	nla_strscpy(ibdev_name, tb[RDMA_NLDEV_ATTR_DEV_NAME],</div><div class='ctx'> 		    sizeof(ibdev_name));</div><div class='del'>-	if (strchr(ibdev_name, '%'))</div><div class='add'>+	if (strchr(ibdev_name, '%') || strlen(ibdev_name) == 0)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	nla_strlcpy(type, tb[RDMA_NLDEV_ATTR_LINK_TYPE], sizeof(type));</div><div class='del'>-	nla_strlcpy(ndev_name, tb[RDMA_NLDEV_ATTR_NDEV_NAME],</div><div class='add'>+	nla_strscpy(type, tb[RDMA_NLDEV_ATTR_LINK_TYPE], sizeof(type));</div><div class='add'>+	nla_strscpy(ndev_name, tb[RDMA_NLDEV_ATTR_NDEV_NAME],</div><div class='ctx'> 		    sizeof(ndev_name));</div><div class='ctx'> </div><div class='del'>-	ndev = dev_get_by_name(&amp;init_net, ndev_name);</div><div class='add'>+	ndev = dev_get_by_name(sock_net(skb-&gt;sk), ndev_name);</div><div class='ctx'> 	if (!ndev)</div><div class='ctx'> 		return -ENODEV;</div><div class='ctx'> </div><div class='hunk'>@@ -1328,8 +1809,8 @@ static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	u32 index;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='del'>-				     nldev_policy, extack);</div><div class='add'>+	err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='add'>+			    nldev_policy, extack);</div><div class='ctx'> 	if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='hunk'>@@ -1338,7 +1819,7 @@ static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	if (!device)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (!(device-&gt;attrs.device_cap_flags &amp; IB_DEVICE_ALLOW_USER_UNREG)) {</div><div class='add'>+	if (!(device-&gt;attrs.kernel_cap_flags &amp; IBK_ALLOW_USER_UNREG)) {</div><div class='ctx'> 		ib_device_put(device);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -1347,6 +1828,94 @@ static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int nldev_get_chardev(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='add'>+			     struct netlink_ext_ack *extack)</div><div class='add'>+{</div><div class='add'>+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];</div><div class='add'>+	char client_name[RDMA_NLDEV_ATTR_CHARDEV_TYPE_SIZE];</div><div class='add'>+	struct ib_client_nl_info data = {};</div><div class='add'>+	struct ib_device *ibdev = NULL;</div><div class='add'>+	struct sk_buff *msg;</div><div class='add'>+	u32 index;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	err = __nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, nldev_policy,</div><div class='add'>+			    NL_VALIDATE_LIBERAL, extack);</div><div class='add'>+	if (err || !tb[RDMA_NLDEV_ATTR_CHARDEV_TYPE])</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	nla_strscpy(client_name, tb[RDMA_NLDEV_ATTR_CHARDEV_TYPE],</div><div class='add'>+		    sizeof(client_name));</div><div class='add'>+</div><div class='add'>+	if (tb[RDMA_NLDEV_ATTR_DEV_INDEX]) {</div><div class='add'>+		index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);</div><div class='add'>+		ibdev = ib_device_get_by_index(sock_net(skb-&gt;sk), index);</div><div class='add'>+		if (!ibdev)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {</div><div class='add'>+			data.port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);</div><div class='add'>+			if (!rdma_is_port_valid(ibdev, data.port)) {</div><div class='add'>+				err = -EINVAL;</div><div class='add'>+				goto out_put;</div><div class='add'>+			}</div><div class='add'>+		} else {</div><div class='add'>+			data.port = -1;</div><div class='add'>+		}</div><div class='add'>+	} else if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);</div><div class='add'>+	if (!msg) {</div><div class='add'>+		err = -ENOMEM;</div><div class='add'>+		goto out_put;</div><div class='add'>+	}</div><div class='add'>+	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh-&gt;nlmsg_seq,</div><div class='add'>+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,</div><div class='add'>+					 RDMA_NLDEV_CMD_GET_CHARDEV),</div><div class='add'>+			0, 0);</div><div class='add'>+	if (!nlh) {</div><div class='add'>+		err = -EMSGSIZE;</div><div class='add'>+		goto out_nlmsg;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	data.nl_msg = msg;</div><div class='add'>+	err = ib_get_client_nl_info(ibdev, client_name, &amp;data);</div><div class='add'>+	if (err)</div><div class='add'>+		goto out_nlmsg;</div><div class='add'>+</div><div class='add'>+	err = nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CHARDEV,</div><div class='add'>+				huge_encode_dev(data.cdev-&gt;devt),</div><div class='add'>+				RDMA_NLDEV_ATTR_PAD);</div><div class='add'>+	if (err)</div><div class='add'>+		goto out_data;</div><div class='add'>+	err = nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CHARDEV_ABI, data.abi,</div><div class='add'>+				RDMA_NLDEV_ATTR_PAD);</div><div class='add'>+	if (err)</div><div class='add'>+		goto out_data;</div><div class='add'>+	if (nla_put_string(msg, RDMA_NLDEV_ATTR_CHARDEV_NAME,</div><div class='add'>+			   dev_name(data.cdev))) {</div><div class='add'>+		err = -EMSGSIZE;</div><div class='add'>+		goto out_data;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	nlmsg_end(msg, nlh);</div><div class='add'>+	put_device(data.cdev);</div><div class='add'>+	if (ibdev)</div><div class='add'>+		ib_device_put(ibdev);</div><div class='add'>+	return rdma_nl_unicast(sock_net(skb-&gt;sk), msg, NETLINK_CB(skb).portid);</div><div class='add'>+</div><div class='add'>+out_data:</div><div class='add'>+	put_device(data.cdev);</div><div class='add'>+out_nlmsg:</div><div class='add'>+	nlmsg_free(msg);</div><div class='add'>+out_put:</div><div class='add'>+	if (ibdev)</div><div class='add'>+		ib_device_put(ibdev);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int nldev_sys_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 			      struct netlink_ext_ack *extack)</div><div class='ctx'> {</div><div class='hunk'>@@ -1354,8 +1923,8 @@ static int nldev_sys_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 	struct sk_buff *msg;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='del'>-			  nldev_policy, extack);</div><div class='add'>+	err = __nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='add'>+			    nldev_policy, NL_VALIDATE_LIBERAL, extack);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='hunk'>@@ -1367,6 +1936,10 @@ static int nldev_sys_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,</div><div class='ctx'> 					 RDMA_NLDEV_CMD_SYS_GET),</div><div class='ctx'> 			0, 0);</div><div class='add'>+	if (!nlh) {</div><div class='add'>+		nlmsg_free(msg);</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	err = nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_NETNS_MODE,</div><div class='ctx'> 			 (u8)ib_devices_shared_netns);</div><div class='hunk'>@@ -1374,29 +1947,694 @@ static int nldev_sys_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 		nlmsg_free(msg);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	err = nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE,</div><div class='add'>+			 (u8)privileged_qkey);</div><div class='add'>+	if (err) {</div><div class='add'>+		nlmsg_free(msg);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_MONITOR_MODE, 1);</div><div class='add'>+	if (err) {</div><div class='add'>+		nlmsg_free(msg);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+	/*</div><div class='add'>+	 * Copy-on-fork is supported.</div><div class='add'>+	 * See commits:</div><div class='add'>+	 * 70e806e4e645 ("mm: Do early cow for pinned pages during fork() for ptes")</div><div class='add'>+	 * 4eae4efa2c29 ("hugetlb: do early cow when page pinned on src mm")</div><div class='add'>+	 * for more details. Don't backport this without them.</div><div class='add'>+	 *</div><div class='add'>+	 * Return value ignored on purpose, assume copy-on-fork is not</div><div class='add'>+	 * supported in case of failure.</div><div class='add'>+	 */</div><div class='add'>+	nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK, 1);</div><div class='add'>+</div><div class='ctx'> 	nlmsg_end(msg, nlh);</div><div class='del'>-	return rdma_nl_unicast(msg, NETLINK_CB(skb).portid);</div><div class='add'>+	return rdma_nl_unicast(sock_net(skb-&gt;sk), msg, NETLINK_CB(skb).portid);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int nldev_set_sys_set_netns_doit(struct nlattr *tb[])</div><div class='add'>+{</div><div class='add'>+	u8 enable;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	enable = nla_get_u8(tb[RDMA_NLDEV_SYS_ATTR_NETNS_MODE]);</div><div class='add'>+	/* Only 0 and 1 are supported */</div><div class='add'>+	if (enable &gt; 1)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	err = rdma_compatdev_set(enable);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int nldev_set_sys_set_pqkey_doit(struct nlattr *tb[])</div><div class='add'>+{</div><div class='add'>+	u8 enable;</div><div class='add'>+</div><div class='add'>+	enable = nla_get_u8(tb[RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE]);</div><div class='add'>+	/* Only 0 and 1 are supported */</div><div class='add'>+	if (enable &gt; 1)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	privileged_qkey = enable;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int nldev_set_sys_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='ctx'> 				  struct netlink_ext_ack *extack)</div><div class='ctx'> {</div><div class='ctx'> 	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];</div><div class='del'>-	u8 enable;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='ctx'> 			  nldev_policy, extack);</div><div class='del'>-	if (err || !tb[RDMA_NLDEV_SYS_ATTR_NETNS_MODE])</div><div class='add'>+	if (err)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	enable = nla_get_u8(tb[RDMA_NLDEV_SYS_ATTR_NETNS_MODE]);</div><div class='del'>-	/* Only 0 and 1 are supported */</div><div class='del'>-	if (enable &gt; 1)</div><div class='add'>+	if (tb[RDMA_NLDEV_SYS_ATTR_NETNS_MODE])</div><div class='add'>+		return nldev_set_sys_set_netns_doit(tb);</div><div class='add'>+</div><div class='add'>+	if (tb[RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE])</div><div class='add'>+		return nldev_set_sys_set_pqkey_doit(tb);</div><div class='add'>+</div><div class='add'>+	return -EINVAL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+</div><div class='add'>+static int nldev_stat_set_mode_doit(struct sk_buff *msg,</div><div class='add'>+				    struct netlink_ext_ack *extack,</div><div class='add'>+				    struct nlattr *tb[],</div><div class='add'>+				    struct ib_device *device, u32 port)</div><div class='add'>+{</div><div class='add'>+	u32 mode, mask = 0, qpn, cntn = 0;</div><div class='add'>+	bool opcnt = false;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	/* Currently only counter for QP is supported */</div><div class='add'>+	if (!tb[RDMA_NLDEV_ATTR_STAT_RES] ||</div><div class='add'>+	    nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES]) != RDMA_NLDEV_ATTR_RES_QP)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	err = rdma_compatdev_set(enable);</div><div class='del'>-	return err;</div><div class='add'>+	if (tb[RDMA_NLDEV_ATTR_STAT_OPCOUNTER_ENABLED])</div><div class='add'>+		opcnt = !!nla_get_u8(</div><div class='add'>+			tb[RDMA_NLDEV_ATTR_STAT_OPCOUNTER_ENABLED]);</div><div class='add'>+</div><div class='add'>+	mode = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_MODE]);</div><div class='add'>+	if (mode == RDMA_COUNTER_MODE_AUTO) {</div><div class='add'>+		if (tb[RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK])</div><div class='add'>+			mask = nla_get_u32(</div><div class='add'>+				tb[RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK]);</div><div class='add'>+		return rdma_counter_set_auto_mode(device, port, mask, opcnt,</div><div class='add'>+						  extack);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!tb[RDMA_NLDEV_ATTR_RES_LQPN])</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	qpn = nla_get_u32(tb[RDMA_NLDEV_ATTR_RES_LQPN]);</div><div class='add'>+	if (tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]) {</div><div class='add'>+		cntn = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]);</div><div class='add'>+		ret = rdma_counter_bind_qpn(device, port, qpn, cntn);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	} else {</div><div class='add'>+		ret = rdma_counter_bind_qpn_alloc(device, port, qpn, &amp;cntn);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, cntn) ||</div><div class='add'>+	    nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qpn)) {</div><div class='add'>+		ret = -EMSGSIZE;</div><div class='add'>+		goto err_fill;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_fill:</div><div class='add'>+	rdma_counter_unbind_qpn(device, port, qpn, cntn);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int nldev_stat_set_counter_dynamic_doit(struct nlattr *tb[],</div><div class='add'>+					       struct ib_device *device,</div><div class='add'>+					       u32 port)</div><div class='add'>+{</div><div class='add'>+	struct rdma_hw_stats *stats;</div><div class='add'>+	struct nlattr *entry_attr;</div><div class='add'>+	unsigned long *target;</div><div class='add'>+	int rem, i, ret = 0;</div><div class='add'>+	u32 index;</div><div class='add'>+</div><div class='add'>+	stats = ib_get_hw_stats_port(device, port);</div><div class='add'>+	if (!stats)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	target = kcalloc(BITS_TO_LONGS(stats-&gt;num_counters),</div><div class='add'>+			 sizeof(*stats-&gt;is_disabled), GFP_KERNEL);</div><div class='add'>+	if (!target)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	nla_for_each_nested(entry_attr, tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS],</div><div class='add'>+			    rem) {</div><div class='add'>+		index = nla_get_u32(entry_attr);</div><div class='add'>+		if ((index &gt;= stats-&gt;num_counters) ||</div><div class='add'>+		    !(stats-&gt;descs[index].flags &amp; IB_STAT_FLAG_OPTIONAL)) {</div><div class='add'>+			ret = -EINVAL;</div><div class='add'>+			goto out;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		set_bit(index, target);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; stats-&gt;num_counters; i++) {</div><div class='add'>+		if (!(stats-&gt;descs[i].flags &amp; IB_STAT_FLAG_OPTIONAL))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		ret = rdma_counter_modify(device, port, i, test_bit(i, target));</div><div class='add'>+		if (ret)</div><div class='add'>+			goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	kfree(target);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int nldev_stat_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='add'>+			       struct netlink_ext_ack *extack)</div><div class='add'>+{</div><div class='add'>+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];</div><div class='add'>+	struct ib_device *device;</div><div class='add'>+	struct sk_buff *msg;</div><div class='add'>+	u32 index, port;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, nldev_policy,</div><div class='add'>+			  extack);</div><div class='add'>+	if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX] ||</div><div class='add'>+	    !tb[RDMA_NLDEV_ATTR_PORT_INDEX])</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);</div><div class='add'>+	device = ib_device_get_by_index(sock_net(skb-&gt;sk), index);</div><div class='add'>+	if (!device)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);</div><div class='add'>+	if (!rdma_is_port_valid(device, port)) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err_put_device;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!tb[RDMA_NLDEV_ATTR_STAT_MODE] &amp;&amp;</div><div class='add'>+	    !tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS]) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err_put_device;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);</div><div class='add'>+	if (!msg) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err_put_device;</div><div class='add'>+	}</div><div class='add'>+	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh-&gt;nlmsg_seq,</div><div class='add'>+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,</div><div class='add'>+					 RDMA_NLDEV_CMD_STAT_SET),</div><div class='add'>+			0, 0);</div><div class='add'>+	if (!nlh || fill_nldev_handle(msg, device) ||</div><div class='add'>+	    nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port)) {</div><div class='add'>+		ret = -EMSGSIZE;</div><div class='add'>+		goto err_free_msg;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (tb[RDMA_NLDEV_ATTR_STAT_MODE]) {</div><div class='add'>+		ret = nldev_stat_set_mode_doit(msg, extack, tb, device, port);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_free_msg;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS]) {</div><div class='add'>+		ret = nldev_stat_set_counter_dynamic_doit(tb, device, port);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_free_msg;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	nlmsg_end(msg, nlh);</div><div class='add'>+	ib_device_put(device);</div><div class='add'>+	return rdma_nl_unicast(sock_net(skb-&gt;sk), msg, NETLINK_CB(skb).portid);</div><div class='add'>+</div><div class='add'>+err_free_msg:</div><div class='add'>+	nlmsg_free(msg);</div><div class='add'>+err_put_device:</div><div class='add'>+	ib_device_put(device);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int nldev_stat_del_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='add'>+			       struct netlink_ext_ack *extack)</div><div class='add'>+{</div><div class='add'>+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];</div><div class='add'>+	struct ib_device *device;</div><div class='add'>+	struct sk_buff *msg;</div><div class='add'>+	u32 index, port, qpn, cntn;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='add'>+			  nldev_policy, extack);</div><div class='add'>+	if (ret || !tb[RDMA_NLDEV_ATTR_STAT_RES] ||</div><div class='add'>+	    !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_PORT_INDEX] ||</div><div class='add'>+	    !tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID] ||</div><div class='add'>+	    !tb[RDMA_NLDEV_ATTR_RES_LQPN])</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES]) != RDMA_NLDEV_ATTR_RES_QP)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);</div><div class='add'>+	device = ib_device_get_by_index(sock_net(skb-&gt;sk), index);</div><div class='add'>+	if (!device)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);</div><div class='add'>+	if (!rdma_is_port_valid(device, port)) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);</div><div class='add'>+	if (!msg) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh-&gt;nlmsg_seq,</div><div class='add'>+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,</div><div class='add'>+					 RDMA_NLDEV_CMD_STAT_SET),</div><div class='add'>+			0, 0);</div><div class='add'>+	if (!nlh) {</div><div class='add'>+		ret = -EMSGSIZE;</div><div class='add'>+		goto err_fill;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cntn = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]);</div><div class='add'>+	qpn = nla_get_u32(tb[RDMA_NLDEV_ATTR_RES_LQPN]);</div><div class='add'>+	if (fill_nldev_handle(msg, device) ||</div><div class='add'>+	    nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port) ||</div><div class='add'>+	    nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, cntn) ||</div><div class='add'>+	    nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qpn)) {</div><div class='add'>+		ret = -EMSGSIZE;</div><div class='add'>+		goto err_fill;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = rdma_counter_unbind_qpn(device, port, qpn, cntn);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_fill;</div><div class='add'>+</div><div class='add'>+	nlmsg_end(msg, nlh);</div><div class='add'>+	ib_device_put(device);</div><div class='add'>+	return rdma_nl_unicast(sock_net(skb-&gt;sk), msg, NETLINK_CB(skb).portid);</div><div class='add'>+</div><div class='add'>+err_fill:</div><div class='add'>+	nlmsg_free(msg);</div><div class='add'>+err:</div><div class='add'>+	ib_device_put(device);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static noinline_for_stack int</div><div class='add'>+stat_get_doit_default_counter(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='add'>+			      struct netlink_ext_ack *extack,</div><div class='add'>+			      struct nlattr *tb[])</div><div class='add'>+{</div><div class='add'>+	struct rdma_hw_stats *stats;</div><div class='add'>+	struct nlattr *table_attr;</div><div class='add'>+	struct ib_device *device;</div><div class='add'>+	int ret, num_cnts, i;</div><div class='add'>+	struct sk_buff *msg;</div><div class='add'>+	u32 index, port;</div><div class='add'>+	u64 v;</div><div class='add'>+</div><div class='add'>+	if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_PORT_INDEX])</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);</div><div class='add'>+	device = ib_device_get_by_index(sock_net(skb-&gt;sk), index);</div><div class='add'>+	if (!device)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!device-&gt;ops.alloc_hw_port_stats || !device-&gt;ops.get_hw_stats) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);</div><div class='add'>+	stats = ib_get_hw_stats_port(device, port);</div><div class='add'>+	if (!stats) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);</div><div class='add'>+	if (!msg) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh-&gt;nlmsg_seq,</div><div class='add'>+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,</div><div class='add'>+					 RDMA_NLDEV_CMD_STAT_GET),</div><div class='add'>+			0, 0);</div><div class='add'>+</div><div class='add'>+	if (!nlh || fill_nldev_handle(msg, device) ||</div><div class='add'>+	    nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port)) {</div><div class='add'>+		ret = -EMSGSIZE;</div><div class='add'>+		goto err_msg;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;stats-&gt;lock);</div><div class='add'>+</div><div class='add'>+	num_cnts = device-&gt;ops.get_hw_stats(device, stats, port, 0);</div><div class='add'>+	if (num_cnts &lt; 0) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err_stats;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS);</div><div class='add'>+	if (!table_attr) {</div><div class='add'>+		ret = -EMSGSIZE;</div><div class='add'>+		goto err_stats;</div><div class='add'>+	}</div><div class='add'>+	for (i = 0; i &lt; num_cnts; i++) {</div><div class='add'>+		if (test_bit(i, stats-&gt;is_disabled))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		v = stats-&gt;value[i] +</div><div class='add'>+			rdma_counter_get_hwstat_value(device, port, i);</div><div class='add'>+		if (rdma_nl_stat_hwcounter_entry(msg,</div><div class='add'>+						 stats-&gt;descs[i].name, v)) {</div><div class='add'>+			ret = -EMSGSIZE;</div><div class='add'>+			goto err_table;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	nla_nest_end(msg, table_attr);</div><div class='add'>+</div><div class='add'>+	mutex_unlock(&amp;stats-&gt;lock);</div><div class='add'>+	nlmsg_end(msg, nlh);</div><div class='add'>+	ib_device_put(device);</div><div class='add'>+	return rdma_nl_unicast(sock_net(skb-&gt;sk), msg, NETLINK_CB(skb).portid);</div><div class='add'>+</div><div class='add'>+err_table:</div><div class='add'>+	nla_nest_cancel(msg, table_attr);</div><div class='add'>+err_stats:</div><div class='add'>+	mutex_unlock(&amp;stats-&gt;lock);</div><div class='add'>+err_msg:</div><div class='add'>+	nlmsg_free(msg);</div><div class='add'>+err:</div><div class='add'>+	ib_device_put(device);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static noinline_for_stack int</div><div class='add'>+stat_get_doit_qp(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='add'>+		 struct netlink_ext_ack *extack, struct nlattr *tb[])</div><div class='add'>+</div><div class='add'>+{</div><div class='add'>+	static enum rdma_nl_counter_mode mode;</div><div class='add'>+	static enum rdma_nl_counter_mask mask;</div><div class='add'>+	struct ib_device *device;</div><div class='add'>+	struct sk_buff *msg;</div><div class='add'>+	u32 index, port;</div><div class='add'>+	bool opcnt;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID])</div><div class='add'>+		return nldev_res_get_counter_doit(skb, nlh, extack);</div><div class='add'>+</div><div class='add'>+	if (!tb[RDMA_NLDEV_ATTR_STAT_MODE] ||</div><div class='add'>+	    !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_PORT_INDEX])</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);</div><div class='add'>+	device = ib_device_get_by_index(sock_net(skb-&gt;sk), index);</div><div class='add'>+	if (!device)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);</div><div class='add'>+	if (!rdma_is_port_valid(device, port)) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);</div><div class='add'>+	if (!msg) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh-&gt;nlmsg_seq,</div><div class='add'>+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,</div><div class='add'>+					 RDMA_NLDEV_CMD_STAT_GET),</div><div class='add'>+			0, 0);</div><div class='add'>+	if (!nlh) {</div><div class='add'>+		ret = -EMSGSIZE;</div><div class='add'>+		goto err_msg;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = rdma_counter_get_mode(device, port, &amp;mode, &amp;mask, &amp;opcnt);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_msg;</div><div class='add'>+</div><div class='add'>+	if (fill_nldev_handle(msg, device) ||</div><div class='add'>+	    nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port) ||</div><div class='add'>+	    nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_MODE, mode)) {</div><div class='add'>+		ret = -EMSGSIZE;</div><div class='add'>+		goto err_msg;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if ((mode == RDMA_COUNTER_MODE_AUTO) &amp;&amp;</div><div class='add'>+	    nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK, mask)) {</div><div class='add'>+		ret = -EMSGSIZE;</div><div class='add'>+		goto err_msg;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if ((mode == RDMA_COUNTER_MODE_AUTO) &amp;&amp;</div><div class='add'>+	    nla_put_u8(msg, RDMA_NLDEV_ATTR_STAT_OPCOUNTER_ENABLED, opcnt)) {</div><div class='add'>+		ret = -EMSGSIZE;</div><div class='add'>+		goto err_msg;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	nlmsg_end(msg, nlh);</div><div class='add'>+	ib_device_put(device);</div><div class='add'>+	return rdma_nl_unicast(sock_net(skb-&gt;sk), msg, NETLINK_CB(skb).portid);</div><div class='add'>+</div><div class='add'>+err_msg:</div><div class='add'>+	nlmsg_free(msg);</div><div class='add'>+err:</div><div class='add'>+	ib_device_put(device);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int nldev_stat_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='add'>+			       struct netlink_ext_ack *extack)</div><div class='add'>+{</div><div class='add'>+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = __nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='add'>+			    nldev_policy, NL_VALIDATE_LIBERAL, extack);</div><div class='add'>+	if (ret)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!tb[RDMA_NLDEV_ATTR_STAT_RES])</div><div class='add'>+		return stat_get_doit_default_counter(skb, nlh, extack, tb);</div><div class='add'>+</div><div class='add'>+	switch (nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES])) {</div><div class='add'>+	case RDMA_NLDEV_ATTR_RES_QP:</div><div class='add'>+		ret = stat_get_doit_qp(skb, nlh, extack, tb);</div><div class='add'>+		break;</div><div class='add'>+	case RDMA_NLDEV_ATTR_RES_MR:</div><div class='add'>+		ret = res_get_common_doit(skb, nlh, extack, RDMA_RESTRACK_MR,</div><div class='add'>+					  fill_stat_mr_entry);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int nldev_stat_get_dumpit(struct sk_buff *skb,</div><div class='add'>+				 struct netlink_callback *cb)</div><div class='add'>+{</div><div class='add'>+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = __nlmsg_parse(cb-&gt;nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='add'>+			    nldev_policy, NL_VALIDATE_LIBERAL, NULL);</div><div class='add'>+	if (ret || !tb[RDMA_NLDEV_ATTR_STAT_RES])</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	switch (nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES])) {</div><div class='add'>+	case RDMA_NLDEV_ATTR_RES_QP:</div><div class='add'>+		ret = nldev_res_get_counter_dumpit(skb, cb);</div><div class='add'>+		break;</div><div class='add'>+	case RDMA_NLDEV_ATTR_RES_MR:</div><div class='add'>+		ret = res_get_common_dumpit(skb, cb, RDMA_RESTRACK_MR,</div><div class='add'>+					    fill_stat_mr_entry);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int nldev_stat_get_counter_status_doit(struct sk_buff *skb,</div><div class='add'>+					      struct nlmsghdr *nlh,</div><div class='add'>+					      struct netlink_ext_ack *extack)</div><div class='add'>+{</div><div class='add'>+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX], *table, *entry;</div><div class='add'>+	struct rdma_hw_stats *stats;</div><div class='add'>+	struct ib_device *device;</div><div class='add'>+	struct sk_buff *msg;</div><div class='add'>+	u32 devid, port;</div><div class='add'>+	int ret, i;</div><div class='add'>+</div><div class='add'>+	ret = __nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='add'>+			    nldev_policy, NL_VALIDATE_LIBERAL, extack);</div><div class='add'>+	if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX] ||</div><div class='add'>+	    !tb[RDMA_NLDEV_ATTR_PORT_INDEX])</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	devid = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);</div><div class='add'>+	device = ib_device_get_by_index(sock_net(skb-&gt;sk), devid);</div><div class='add'>+	if (!device)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);</div><div class='add'>+	if (!rdma_is_port_valid(device, port)) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	stats = ib_get_hw_stats_port(device, port);</div><div class='add'>+	if (!stats) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);</div><div class='add'>+	if (!msg) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	nlh = nlmsg_put(</div><div class='add'>+		msg, NETLINK_CB(skb).portid, nlh-&gt;nlmsg_seq,</div><div class='add'>+		RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_STAT_GET_STATUS),</div><div class='add'>+		0, 0);</div><div class='add'>+</div><div class='add'>+	ret = -EMSGSIZE;</div><div class='add'>+	if (!nlh || fill_nldev_handle(msg, device) ||</div><div class='add'>+	    nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port))</div><div class='add'>+		goto err_msg;</div><div class='add'>+</div><div class='add'>+	table = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS);</div><div class='add'>+	if (!table)</div><div class='add'>+		goto err_msg;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;stats-&gt;lock);</div><div class='add'>+	for (i = 0; i &lt; stats-&gt;num_counters; i++) {</div><div class='add'>+		entry = nla_nest_start(msg,</div><div class='add'>+				       RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY);</div><div class='add'>+		if (!entry)</div><div class='add'>+			goto err_msg_table;</div><div class='add'>+</div><div class='add'>+		if (nla_put_string(msg,</div><div class='add'>+				   RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME,</div><div class='add'>+				   stats-&gt;descs[i].name) ||</div><div class='add'>+		    nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_INDEX, i))</div><div class='add'>+			goto err_msg_entry;</div><div class='add'>+</div><div class='add'>+		if ((stats-&gt;descs[i].flags &amp; IB_STAT_FLAG_OPTIONAL) &amp;&amp;</div><div class='add'>+		    (nla_put_u8(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC,</div><div class='add'>+				!test_bit(i, stats-&gt;is_disabled))))</div><div class='add'>+			goto err_msg_entry;</div><div class='add'>+</div><div class='add'>+		nla_nest_end(msg, entry);</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;stats-&gt;lock);</div><div class='add'>+</div><div class='add'>+	nla_nest_end(msg, table);</div><div class='add'>+	nlmsg_end(msg, nlh);</div><div class='add'>+	ib_device_put(device);</div><div class='add'>+	return rdma_nl_unicast(sock_net(skb-&gt;sk), msg, NETLINK_CB(skb).portid);</div><div class='add'>+</div><div class='add'>+err_msg_entry:</div><div class='add'>+	nla_nest_cancel(msg, entry);</div><div class='add'>+err_msg_table:</div><div class='add'>+	mutex_unlock(&amp;stats-&gt;lock);</div><div class='add'>+	nla_nest_cancel(msg, table);</div><div class='add'>+err_msg:</div><div class='add'>+	nlmsg_free(msg);</div><div class='add'>+err:</div><div class='add'>+	ib_device_put(device);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int nldev_newdev(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='add'>+			struct netlink_ext_ack *extack)</div><div class='add'>+{</div><div class='add'>+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];</div><div class='add'>+	enum rdma_nl_dev_type type;</div><div class='add'>+	struct ib_device *parent;</div><div class='add'>+	char name[IFNAMSIZ] = {};</div><div class='add'>+	u32 parentid;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='add'>+			  nldev_policy, extack);</div><div class='add'>+	if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX] ||</div><div class='add'>+		!tb[RDMA_NLDEV_ATTR_DEV_NAME] || !tb[RDMA_NLDEV_ATTR_DEV_TYPE])</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	nla_strscpy(name, tb[RDMA_NLDEV_ATTR_DEV_NAME], sizeof(name));</div><div class='add'>+	type = nla_get_u8(tb[RDMA_NLDEV_ATTR_DEV_TYPE]);</div><div class='add'>+	parentid = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);</div><div class='add'>+	parent = ib_device_get_by_index(sock_net(skb-&gt;sk), parentid);</div><div class='add'>+	if (!parent)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ret = ib_add_sub_device(parent, type, name);</div><div class='add'>+	ib_device_put(parent);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int nldev_deldev(struct sk_buff *skb, struct nlmsghdr *nlh,</div><div class='add'>+			struct netlink_ext_ack *extack)</div><div class='add'>+{</div><div class='add'>+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];</div><div class='add'>+	struct ib_device *device;</div><div class='add'>+	u32 devid;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,</div><div class='add'>+			  nldev_policy, extack);</div><div class='add'>+	if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	devid = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);</div><div class='add'>+	device = ib_device_get_by_index(sock_net(skb-&gt;sk), devid);</div><div class='add'>+	if (!device)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	return ib_del_sub_device_and_put(device);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {</div><div class='hunk'>@@ -1404,6 +2642,9 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {</div><div class='ctx'> 		.doit = nldev_get_doit,</div><div class='ctx'> 		.dump = nldev_get_dumpit,</div><div class='ctx'> 	},</div><div class='add'>+	[RDMA_NLDEV_CMD_GET_CHARDEV] = {</div><div class='add'>+		.doit = nldev_get_chardev,</div><div class='add'>+	},</div><div class='ctx'> 	[RDMA_NLDEV_CMD_SET] = {</div><div class='ctx'> 		.doit = nldev_set_doit,</div><div class='ctx'> 		.flags = RDMA_NL_ADMIN_PERM,</div><div class='hunk'>@@ -1444,6 +2685,14 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {</div><div class='ctx'> 		.doit = nldev_res_get_pd_doit,</div><div class='ctx'> 		.dump = nldev_res_get_pd_dumpit,</div><div class='ctx'> 	},</div><div class='add'>+	[RDMA_NLDEV_CMD_RES_CTX_GET] = {</div><div class='add'>+		.doit = nldev_res_get_ctx_doit,</div><div class='add'>+		.dump = nldev_res_get_ctx_dumpit,</div><div class='add'>+	},</div><div class='add'>+	[RDMA_NLDEV_CMD_RES_SRQ_GET] = {</div><div class='add'>+		.doit = nldev_res_get_srq_doit,</div><div class='add'>+		.dump = nldev_res_get_srq_dumpit,</div><div class='add'>+	},</div><div class='ctx'> 	[RDMA_NLDEV_CMD_SYS_GET] = {</div><div class='ctx'> 		.doit = nldev_sys_get_doit,</div><div class='ctx'> 	},</div><div class='hunk'>@@ -1451,14 +2700,219 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {</div><div class='ctx'> 		.doit = nldev_set_sys_set_doit,</div><div class='ctx'> 		.flags = RDMA_NL_ADMIN_PERM,</div><div class='ctx'> 	},</div><div class='add'>+	[RDMA_NLDEV_CMD_STAT_SET] = {</div><div class='add'>+		.doit = nldev_stat_set_doit,</div><div class='add'>+		.flags = RDMA_NL_ADMIN_PERM,</div><div class='add'>+	},</div><div class='add'>+	[RDMA_NLDEV_CMD_STAT_GET] = {</div><div class='add'>+		.doit = nldev_stat_get_doit,</div><div class='add'>+		.dump = nldev_stat_get_dumpit,</div><div class='add'>+	},</div><div class='add'>+	[RDMA_NLDEV_CMD_STAT_DEL] = {</div><div class='add'>+		.doit = nldev_stat_del_doit,</div><div class='add'>+		.flags = RDMA_NL_ADMIN_PERM,</div><div class='add'>+	},</div><div class='add'>+	[RDMA_NLDEV_CMD_RES_QP_GET_RAW] = {</div><div class='add'>+		.doit = nldev_res_get_qp_raw_doit,</div><div class='add'>+		.dump = nldev_res_get_qp_raw_dumpit,</div><div class='add'>+		.flags = RDMA_NL_ADMIN_PERM,</div><div class='add'>+	},</div><div class='add'>+	[RDMA_NLDEV_CMD_RES_CQ_GET_RAW] = {</div><div class='add'>+		.doit = nldev_res_get_cq_raw_doit,</div><div class='add'>+		.dump = nldev_res_get_cq_raw_dumpit,</div><div class='add'>+		.flags = RDMA_NL_ADMIN_PERM,</div><div class='add'>+	},</div><div class='add'>+	[RDMA_NLDEV_CMD_RES_MR_GET_RAW] = {</div><div class='add'>+		.doit = nldev_res_get_mr_raw_doit,</div><div class='add'>+		.dump = nldev_res_get_mr_raw_dumpit,</div><div class='add'>+		.flags = RDMA_NL_ADMIN_PERM,</div><div class='add'>+	},</div><div class='add'>+	[RDMA_NLDEV_CMD_RES_SRQ_GET_RAW] = {</div><div class='add'>+		.doit = nldev_res_get_srq_raw_doit,</div><div class='add'>+		.dump = nldev_res_get_srq_raw_dumpit,</div><div class='add'>+		.flags = RDMA_NL_ADMIN_PERM,</div><div class='add'>+	},</div><div class='add'>+	[RDMA_NLDEV_CMD_STAT_GET_STATUS] = {</div><div class='add'>+		.doit = nldev_stat_get_counter_status_doit,</div><div class='add'>+	},</div><div class='add'>+	[RDMA_NLDEV_CMD_NEWDEV] = {</div><div class='add'>+		.doit = nldev_newdev,</div><div class='add'>+		.flags = RDMA_NL_ADMIN_PERM,</div><div class='add'>+	},</div><div class='add'>+	[RDMA_NLDEV_CMD_DELDEV] = {</div><div class='add'>+		.doit = nldev_deldev,</div><div class='add'>+		.flags = RDMA_NL_ADMIN_PERM,</div><div class='add'>+	},</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static int fill_mon_netdev_rename(struct sk_buff *msg,</div><div class='add'>+				  struct ib_device *device, u32 port,</div><div class='add'>+				  const struct net *net)</div><div class='add'>+{</div><div class='add'>+	struct net_device *netdev = ib_device_get_netdev(device, port);</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	if (!netdev || !net_eq(dev_net(netdev), net))</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	ret = nla_put_u32(msg, RDMA_NLDEV_ATTR_NDEV_INDEX, netdev-&gt;ifindex);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out;</div><div class='add'>+	ret = nla_put_string(msg, RDMA_NLDEV_ATTR_NDEV_NAME, netdev-&gt;name);</div><div class='add'>+out:</div><div class='add'>+	dev_put(netdev);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_mon_netdev_association(struct sk_buff *msg,</div><div class='add'>+				       struct ib_device *device, u32 port,</div><div class='add'>+				       const struct net *net)</div><div class='add'>+{</div><div class='add'>+	struct net_device *netdev = ib_device_get_netdev(device, port);</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	if (netdev &amp;&amp; !net_eq(dev_net(netdev), net))</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	ret = nla_put_u32(msg, RDMA_NLDEV_ATTR_DEV_INDEX, device-&gt;index);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_NAME,</div><div class='add'>+			     dev_name(&amp;device-&gt;dev));</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	ret = nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	if (netdev) {</div><div class='add'>+		ret = nla_put_u32(msg,</div><div class='add'>+				  RDMA_NLDEV_ATTR_NDEV_INDEX, netdev-&gt;ifindex);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto out;</div><div class='add'>+</div><div class='add'>+		ret = nla_put_string(msg,</div><div class='add'>+				     RDMA_NLDEV_ATTR_NDEV_NAME, netdev-&gt;name);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	dev_put(netdev);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void rdma_nl_notify_err_msg(struct ib_device *device, u32 port_num,</div><div class='add'>+				    enum rdma_nl_notify_event_type type)</div><div class='add'>+{</div><div class='add'>+	struct net_device *netdev;</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case RDMA_REGISTER_EVENT:</div><div class='add'>+		dev_warn_ratelimited(&amp;device-&gt;dev,</div><div class='add'>+				     "Failed to send RDMA monitor register device event\n");</div><div class='add'>+		break;</div><div class='add'>+	case RDMA_UNREGISTER_EVENT:</div><div class='add'>+		dev_warn_ratelimited(&amp;device-&gt;dev,</div><div class='add'>+				     "Failed to send RDMA monitor unregister device event\n");</div><div class='add'>+		break;</div><div class='add'>+	case RDMA_NETDEV_ATTACH_EVENT:</div><div class='add'>+		netdev = ib_device_get_netdev(device, port_num);</div><div class='add'>+		dev_warn_ratelimited(&amp;device-&gt;dev,</div><div class='add'>+				     "Failed to send RDMA monitor netdev attach event: port %d netdev %d\n",</div><div class='add'>+				     port_num, netdev-&gt;ifindex);</div><div class='add'>+		dev_put(netdev);</div><div class='add'>+		break;</div><div class='add'>+	case RDMA_NETDEV_DETACH_EVENT:</div><div class='add'>+		dev_warn_ratelimited(&amp;device-&gt;dev,</div><div class='add'>+				     "Failed to send RDMA monitor netdev detach event: port %d\n",</div><div class='add'>+				     port_num);</div><div class='add'>+		break;</div><div class='add'>+	case RDMA_RENAME_EVENT:</div><div class='add'>+		dev_warn_ratelimited(&amp;device-&gt;dev,</div><div class='add'>+				     "Failed to send RDMA monitor rename device event\n");</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	case RDMA_NETDEV_RENAME_EVENT:</div><div class='add'>+		netdev = ib_device_get_netdev(device, port_num);</div><div class='add'>+		dev_warn_ratelimited(&amp;device-&gt;dev,</div><div class='add'>+				     "Failed to send RDMA monitor netdev rename event: port %d netdev %d\n",</div><div class='add'>+				     port_num, netdev-&gt;ifindex);</div><div class='add'>+		dev_put(netdev);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int rdma_nl_notify_event(struct ib_device *device, u32 port_num,</div><div class='add'>+			  enum rdma_nl_notify_event_type type)</div><div class='add'>+{</div><div class='add'>+	struct sk_buff *skb;</div><div class='add'>+	int ret = -EMSGSIZE;</div><div class='add'>+	struct net *net;</div><div class='add'>+	void *nlh;</div><div class='add'>+</div><div class='add'>+	net = read_pnet(&amp;device-&gt;coredev.rdma_net);</div><div class='add'>+	if (!net)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);</div><div class='add'>+	if (!skb)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	nlh = nlmsg_put(skb, 0, 0,</div><div class='add'>+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_MONITOR),</div><div class='add'>+			0, 0);</div><div class='add'>+	if (!nlh)</div><div class='add'>+		goto err_free;</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case RDMA_REGISTER_EVENT:</div><div class='add'>+	case RDMA_UNREGISTER_EVENT:</div><div class='add'>+	case RDMA_RENAME_EVENT:</div><div class='add'>+		ret = fill_nldev_handle(skb, device);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_free;</div><div class='add'>+		break;</div><div class='add'>+	case RDMA_NETDEV_ATTACH_EVENT:</div><div class='add'>+	case RDMA_NETDEV_DETACH_EVENT:</div><div class='add'>+		ret = fill_mon_netdev_association(skb, device, port_num, net);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_free;</div><div class='add'>+		break;</div><div class='add'>+	case RDMA_NETDEV_RENAME_EVENT:</div><div class='add'>+		ret = fill_mon_netdev_rename(skb, device, port_num, net);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_free;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = nla_put_u8(skb, RDMA_NLDEV_ATTR_EVENT_TYPE, type);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_free;</div><div class='add'>+</div><div class='add'>+	nlmsg_end(skb, nlh);</div><div class='add'>+	ret = rdma_nl_multicast(net, skb, RDMA_NL_GROUP_NOTIFY, GFP_KERNEL);</div><div class='add'>+	if (ret &amp;&amp; ret != -ESRCH) {</div><div class='add'>+		skb = NULL; /* skb is freed in the netlink send-op handling */</div><div class='add'>+		goto err_free;</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_free:</div><div class='add'>+	rdma_nl_notify_err_msg(device, port_num, type);</div><div class='add'>+	nlmsg_free(skb);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> void __init nldev_init(void)</div><div class='ctx'> {</div><div class='ctx'> 	rdma_nl_register(RDMA_NL_NLDEV, nldev_cb_table);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void __exit nldev_exit(void)</div><div class='add'>+void nldev_exit(void)</div><div class='ctx'> {</div><div class='ctx'> 	rdma_nl_unregister(RDMA_NL_NLDEV);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/core/opa_smi.h b/drivers/infiniband/core/opa_smi.h<br/>index af4879bdf3d6..64e2822af70f 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/opa_smi.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/opa_smi.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/opa_smi.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/opa_smi.h</a></div><div class='hunk'>@@ -40,11 +40,11 @@</div><div class='ctx'> #include "smi.h"</div><div class='ctx'> </div><div class='ctx'> enum smi_action opa_smi_handle_dr_smp_recv(struct opa_smp *smp, bool is_switch,</div><div class='del'>-				       int port_num, int phys_port_cnt);</div><div class='add'>+				       u32 port_num, int phys_port_cnt);</div><div class='ctx'> int opa_smi_get_fwd_port(struct opa_smp *smp);</div><div class='ctx'> extern enum smi_forward_action opa_smi_check_forward_dr_smp(struct opa_smp *smp);</div><div class='ctx'> extern enum smi_action opa_smi_handle_dr_smp_send(struct opa_smp *smp,</div><div class='del'>-					      bool is_switch, int port_num);</div><div class='add'>+					      bool is_switch, u32 port_num);</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * Return IB_SMI_HANDLE if the SMP should be handled by the local SMA/SM</div><div class='head'>diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c<br/>index ccf4d069c25c..18918f463361 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/rdma_core.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/rdma_core.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/rdma_core.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/rdma_core.c</a></div><div class='hunk'>@@ -42,29 +42,24 @@</div><div class='ctx'> #include "core_priv.h"</div><div class='ctx'> #include "rdma_core.h"</div><div class='ctx'> </div><div class='del'>-void uverbs_uobject_get(struct ib_uobject *uobject)</div><div class='del'>-{</div><div class='del'>-	kref_get(&amp;uobject-&gt;ref);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static void uverbs_uobject_free(struct kref *ref)</div><div class='ctx'> {</div><div class='del'>-	struct ib_uobject *uobj =</div><div class='del'>-		container_of(ref, struct ib_uobject, ref);</div><div class='del'>-</div><div class='del'>-	if (uobj-&gt;uapi_object-&gt;type_class-&gt;needs_kfree_rcu)</div><div class='del'>-		kfree_rcu(uobj, rcu);</div><div class='del'>-	else</div><div class='del'>-		kfree(uobj);</div><div class='add'>+	kfree_rcu(container_of(ref, struct ib_uobject, ref), rcu);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+/*</div><div class='add'>+ * In order to indicate we no longer needs this uobject, uverbs_uobject_put</div><div class='add'>+ * is called. When the reference count is decreased, the uobject is freed.</div><div class='add'>+ * For example, this is used when attaching a completion channel to a CQ.</div><div class='add'>+ */</div><div class='ctx'> void uverbs_uobject_put(struct ib_uobject *uobject)</div><div class='ctx'> {</div><div class='ctx'> 	kref_put(&amp;uobject-&gt;ref, uverbs_uobject_free);</div><div class='ctx'> }</div><div class='add'>+EXPORT_SYMBOL(uverbs_uobject_put);</div><div class='ctx'> </div><div class='del'>-static int uverbs_try_lock_object(struct ib_uobject *uobj,</div><div class='del'>-				  enum rdma_lookup_mode mode)</div><div class='add'>+int uverbs_try_lock_object(struct ib_uobject *uobj,</div><div class='add'>+			   enum rdma_lookup_mode mode)</div><div class='ctx'> {</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * When a shared access is required, we use a positive counter. Each</div><div class='hunk'>@@ -73,7 +68,7 @@ static int uverbs_try_lock_object(struct ib_uobject *uobj,</div><div class='ctx'> 	 * In exclusive access mode, we check that the counter is zero (nobody</div><div class='ctx'> 	 * claimed this object) and we set it to -1. Releasing a shared access</div><div class='ctx'> 	 * lock is done simply by decreasing the counter. As for exclusive</div><div class='del'>-	 * access locks, since only a single one of them is is allowed</div><div class='add'>+	 * access locks, since only a single one of them is allowed</div><div class='ctx'> 	 * concurrently, setting the counter to zero is enough for releasing</div><div class='ctx'> 	 * this lock.</div><div class='ctx'> 	 */</div><div class='hunk'>@@ -89,6 +84,7 @@ static int uverbs_try_lock_object(struct ib_uobject *uobj,</div><div class='ctx'> 	}</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='add'>+EXPORT_SYMBOL(uverbs_try_lock_object);</div><div class='ctx'> </div><div class='ctx'> static void assert_uverbs_usecnt(struct ib_uobject *uobj,</div><div class='ctx'> 				 enum rdma_lookup_mode mode)</div><div class='hunk'>@@ -117,7 +113,7 @@ static void assert_uverbs_usecnt(struct ib_uobject *uobj,</div><div class='ctx'>  * however the type's allocat_commit function cannot have been called and the</div><div class='ctx'>  * uobject cannot be on the uobjects_lists</div><div class='ctx'>  *</div><div class='del'>- * For RDMA_REMOVE_DESTROY the caller shold be holding a kref (eg via</div><div class='add'>+ * For RDMA_REMOVE_DESTROY the caller should be holding a kref (eg via</div><div class='ctx'>  * rdma_lookup_get_uobject) and the object is left in a state where the caller</div><div class='ctx'>  * needs to call rdma_lookup_put_uobject.</div><div class='ctx'>  *</div><div class='hunk'>@@ -135,34 +131,26 @@ static int uverbs_destroy_uobject(struct ib_uobject *uobj,</div><div class='ctx'> 	lockdep_assert_held(&amp;ufile-&gt;hw_destroy_rwsem);</div><div class='ctx'> 	assert_uverbs_usecnt(uobj, UVERBS_LOOKUP_WRITE);</div><div class='ctx'> </div><div class='del'>-	if (uobj-&gt;object) {</div><div class='del'>-		ret = uobj-&gt;uapi_object-&gt;type_class-&gt;destroy_hw(uobj, reason,</div><div class='del'>-								attrs);</div><div class='del'>-		if (ret) {</div><div class='del'>-			if (ib_is_destroy_retryable(ret, reason, uobj))</div><div class='del'>-				return ret;</div><div class='del'>-</div><div class='del'>-			/* Nothing to be done, dangle the memory and move on */</div><div class='del'>-			WARN(true,</div><div class='del'>-			     "ib_uverbs: failed to remove uobject id %d, driver err=%d",</div><div class='del'>-			     uobj-&gt;id, ret);</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		uobj-&gt;object = NULL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	if (reason == RDMA_REMOVE_ABORT) {</div><div class='ctx'> 		WARN_ON(!list_empty(&amp;uobj-&gt;list));</div><div class='ctx'> 		WARN_ON(!uobj-&gt;context);</div><div class='ctx'> 		uobj-&gt;uapi_object-&gt;type_class-&gt;alloc_abort(uobj);</div><div class='add'>+	} else if (uobj-&gt;object) {</div><div class='add'>+		ret = uobj-&gt;uapi_object-&gt;type_class-&gt;destroy_hw(uobj, reason,</div><div class='add'>+								attrs);</div><div class='add'>+		if (ret)</div><div class='add'>+			/* Nothing to be done, wait till ucontext will clean it */</div><div class='add'>+			return ret;</div><div class='add'>+</div><div class='add'>+		uobj-&gt;object = NULL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	uobj-&gt;context = NULL;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='del'>-	 * For DESTROY the usecnt is held write locked, the caller is expected</div><div class='del'>-	 * to put it unlock and put the object when done with it. Only DESTROY</div><div class='del'>-	 * can remove the IDR handle.</div><div class='add'>+	 * For DESTROY the usecnt is not changed, the caller is expected to</div><div class='add'>+	 * manage it via uobj_put_destroy(). Only DESTROY can remove the IDR</div><div class='add'>+	 * handle.</div><div class='ctx'> 	 */</div><div class='ctx'> 	if (reason != RDMA_REMOVE_DESTROY)</div><div class='ctx'> 		atomic_set(&amp;uobj-&gt;usecnt, 0);</div><div class='hunk'>@@ -194,7 +182,7 @@ static int uverbs_destroy_uobject(struct ib_uobject *uobj,</div><div class='ctx'> /*</div><div class='ctx'>  * This calls uverbs_destroy_uobject() using the RDMA_REMOVE_DESTROY</div><div class='ctx'>  * sequence. It should only be used from command callbacks. On success the</div><div class='del'>- * caller must pair this with rdma_lookup_put_uobject(LOOKUP_WRITE). This</div><div class='add'>+ * caller must pair this with uobj_put_destroy(). This</div><div class='ctx'>  * version requires the caller to have already obtained an</div><div class='ctx'>  * LOOKUP_DESTROY uobject kref.</div><div class='ctx'>  */</div><div class='hunk'>@@ -205,6 +193,13 @@ int uobj_destroy(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs)</div><div class='ctx'> </div><div class='ctx'> 	down_read(&amp;ufile-&gt;hw_destroy_rwsem);</div><div class='ctx'> </div><div class='add'>+	/*</div><div class='add'>+	 * Once the uobject is destroyed by RDMA_REMOVE_DESTROY then it is left</div><div class='add'>+	 * write locked as the callers put it back with UVERBS_LOOKUP_DESTROY.</div><div class='add'>+	 * This is because any other concurrent thread can still see the object</div><div class='add'>+	 * in the xarray due to RCU. Leaving it locked ensures nothing else will</div><div class='add'>+	 * touch it.</div><div class='add'>+	 */</div><div class='ctx'> 	ret = uverbs_try_lock_object(uobj, UVERBS_LOOKUP_WRITE);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto out_unlock;</div><div class='hunk'>@@ -223,7 +218,7 @@ out_unlock:</div><div class='ctx'> /*</div><div class='ctx'>  * uobj_get_destroy destroys the HW object and returns a handle to the uobj</div><div class='ctx'>  * with a NULL object pointer. The caller must pair this with</div><div class='del'>- * uverbs_put_destroy.</div><div class='add'>+ * uobj_put_destroy().</div><div class='ctx'>  */</div><div class='ctx'> struct ib_uobject *__uobj_get_destroy(const struct uverbs_api_object *obj,</div><div class='ctx'> 				      u32 id, struct uverbs_attr_bundle *attrs)</div><div class='hunk'>@@ -257,21 +252,25 @@ int __uobj_perform_destroy(const struct uverbs_api_object *obj, u32 id,</div><div class='ctx'> 	uobj = __uobj_get_destroy(obj, id, attrs);</div><div class='ctx'> 	if (IS_ERR(uobj))</div><div class='ctx'> 		return PTR_ERR(uobj);</div><div class='del'>-</div><div class='del'>-	rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_WRITE);</div><div class='add'>+	uobj_put_destroy(uobj);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* alloc_uobj must be undone by uverbs_destroy_uobject() */</div><div class='del'>-static struct ib_uobject *alloc_uobj(struct ib_uverbs_file *ufile,</div><div class='add'>+static struct ib_uobject *alloc_uobj(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 				     const struct uverbs_api_object *obj)</div><div class='ctx'> {</div><div class='add'>+	struct ib_uverbs_file *ufile = attrs-&gt;ufile;</div><div class='ctx'> 	struct ib_uobject *uobj;</div><div class='del'>-	struct ib_ucontext *ucontext;</div><div class='ctx'> </div><div class='del'>-	ucontext = ib_uverbs_get_ucontext_file(ufile);</div><div class='del'>-	if (IS_ERR(ucontext))</div><div class='del'>-		return ERR_CAST(ucontext);</div><div class='add'>+	if (!attrs-&gt;context) {</div><div class='add'>+		struct ib_ucontext *ucontext =</div><div class='add'>+			ib_uverbs_get_ucontext_file(ufile);</div><div class='add'>+</div><div class='add'>+		if (IS_ERR(ucontext))</div><div class='add'>+			return ERR_CAST(ucontext);</div><div class='add'>+		attrs-&gt;context = ucontext;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	uobj = kzalloc(obj-&gt;type_attrs-&gt;obj_size, GFP_KERNEL);</div><div class='ctx'> 	if (!uobj)</div><div class='hunk'>@@ -281,7 +280,7 @@ static struct ib_uobject *alloc_uobj(struct ib_uverbs_file *ufile,</div><div class='ctx'> 	 * The object is added to the list in the commit stage.</div><div class='ctx'> 	 */</div><div class='ctx'> 	uobj-&gt;ufile = ufile;</div><div class='del'>-	uobj-&gt;context = ucontext;</div><div class='add'>+	uobj-&gt;context = attrs-&gt;context;</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;uobj-&gt;list);</div><div class='ctx'> 	uobj-&gt;uapi_object = obj;</div><div class='ctx'> 	/*</div><div class='hunk'>@@ -358,11 +357,11 @@ lookup_get_fd_uobject(const struct uverbs_api_object *obj,</div><div class='ctx'> </div><div class='ctx'> 	uobject = f-&gt;private_data;</div><div class='ctx'> 	/*</div><div class='del'>-	 * fget(id) ensures we are not currently running uverbs_close_fd,</div><div class='del'>-	 * and the caller is expected to ensure that uverbs_close_fd is never</div><div class='del'>-	 * done while a call top lookup is possible.</div><div class='add'>+	 * fget(id) ensures we are not currently running</div><div class='add'>+	 * uverbs_uobject_fd_release(), and the caller is expected to ensure</div><div class='add'>+	 * that release is never done while a call to lookup is possible.</div><div class='ctx'> 	 */</div><div class='del'>-	if (f-&gt;f_op != fd_type-&gt;fops) {</div><div class='add'>+	if (f-&gt;f_op != fd_type-&gt;fops || uobject-&gt;ufile != ufile) {</div><div class='ctx'> 		fput(f);</div><div class='ctx'> 		return ERR_PTR(-EBADF);</div><div class='ctx'> 	}</div><div class='hunk'>@@ -424,12 +423,12 @@ free:</div><div class='ctx'> </div><div class='ctx'> static struct ib_uobject *</div><div class='ctx'> alloc_begin_idr_uobject(const struct uverbs_api_object *obj,</div><div class='del'>-			struct ib_uverbs_file *ufile)</div><div class='add'>+			struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	int ret;</div><div class='ctx'> 	struct ib_uobject *uobj;</div><div class='ctx'> </div><div class='del'>-	uobj = alloc_uobj(ufile, obj);</div><div class='add'>+	uobj = alloc_uobj(attrs, obj);</div><div class='ctx'> 	if (IS_ERR(uobj))</div><div class='ctx'> 		return uobj;</div><div class='ctx'> </div><div class='hunk'>@@ -445,7 +444,7 @@ alloc_begin_idr_uobject(const struct uverbs_api_object *obj,</div><div class='ctx'> 	return uobj;</div><div class='ctx'> </div><div class='ctx'> remove:</div><div class='del'>-	xa_erase(&amp;ufile-&gt;idr, uobj-&gt;id);</div><div class='add'>+	xa_erase(&amp;attrs-&gt;ufile-&gt;idr, uobj-&gt;id);</div><div class='ctx'> uobj_put:</div><div class='ctx'> 	uverbs_uobject_put(uobj);</div><div class='ctx'> 	return ERR_PTR(ret);</div><div class='hunk'>@@ -453,31 +452,54 @@ uobj_put:</div><div class='ctx'> </div><div class='ctx'> static struct ib_uobject *</div><div class='ctx'> alloc_begin_fd_uobject(const struct uverbs_api_object *obj,</div><div class='del'>-		       struct ib_uverbs_file *ufile)</div><div class='add'>+		       struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='add'>+	const struct uverbs_obj_fd_type *fd_type;</div><div class='ctx'> 	int new_fd;</div><div class='del'>-	struct ib_uobject *uobj;</div><div class='add'>+	struct ib_uobject *uobj, *ret;</div><div class='add'>+	struct file *filp;</div><div class='add'>+</div><div class='add'>+	uobj = alloc_uobj(attrs, obj);</div><div class='add'>+	if (IS_ERR(uobj))</div><div class='add'>+		return uobj;</div><div class='add'>+</div><div class='add'>+	fd_type =</div><div class='add'>+		container_of(obj-&gt;type_attrs, struct uverbs_obj_fd_type, type);</div><div class='add'>+	if (WARN_ON(fd_type-&gt;fops-&gt;release != &amp;uverbs_uobject_fd_release &amp;&amp;</div><div class='add'>+		    fd_type-&gt;fops-&gt;release != &amp;uverbs_async_event_release)) {</div><div class='add'>+		ret = ERR_PTR(-EINVAL);</div><div class='add'>+		goto err_fd;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	new_fd = get_unused_fd_flags(O_CLOEXEC);</div><div class='del'>-	if (new_fd &lt; 0)</div><div class='del'>-		return ERR_PTR(new_fd);</div><div class='add'>+	if (new_fd &lt; 0) {</div><div class='add'>+		ret = ERR_PTR(new_fd);</div><div class='add'>+		goto err_fd;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	uobj = alloc_uobj(ufile, obj);</div><div class='del'>-	if (IS_ERR(uobj)) {</div><div class='del'>-		put_unused_fd(new_fd);</div><div class='del'>-		return uobj;</div><div class='add'>+	/* Note that uverbs_uobject_fd_release() is called during abort */</div><div class='add'>+	filp = anon_inode_getfile(fd_type-&gt;name, fd_type-&gt;fops, NULL,</div><div class='add'>+				  fd_type-&gt;flags);</div><div class='add'>+	if (IS_ERR(filp)) {</div><div class='add'>+		ret = ERR_CAST(filp);</div><div class='add'>+		goto err_getfile;</div><div class='ctx'> 	}</div><div class='add'>+	uobj-&gt;object = filp;</div><div class='ctx'> </div><div class='ctx'> 	uobj-&gt;id = new_fd;</div><div class='del'>-	uobj-&gt;ufile = ufile;</div><div class='del'>-</div><div class='ctx'> 	return uobj;</div><div class='add'>+</div><div class='add'>+err_getfile:</div><div class='add'>+	put_unused_fd(new_fd);</div><div class='add'>+err_fd:</div><div class='add'>+	uverbs_uobject_put(uobj);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj,</div><div class='del'>-					    struct ib_uverbs_file *ufile,</div><div class='ctx'> 					    struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='add'>+	struct ib_uverbs_file *ufile = attrs-&gt;ufile;</div><div class='ctx'> 	struct ib_uobject *ret;</div><div class='ctx'> </div><div class='ctx'> 	if (IS_ERR(obj))</div><div class='hunk'>@@ -491,13 +513,11 @@ struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj,</div><div class='ctx'> 	if (!down_read_trylock(&amp;ufile-&gt;hw_destroy_rwsem))</div><div class='ctx'> 		return ERR_PTR(-EIO);</div><div class='ctx'> </div><div class='del'>-	ret = obj-&gt;type_class-&gt;alloc_begin(obj, ufile);</div><div class='add'>+	ret = obj-&gt;type_class-&gt;alloc_begin(obj, attrs);</div><div class='ctx'> 	if (IS_ERR(ret)) {</div><div class='ctx'> 		up_read(&amp;ufile-&gt;hw_destroy_rwsem);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='del'>-	if (attrs)</div><div class='del'>-		attrs-&gt;context = ret-&gt;context;</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -518,12 +538,7 @@ static int __must_check destroy_hw_idr_uobject(struct ib_uobject *uobj,</div><div class='ctx'> 			     struct uverbs_obj_idr_type, type);</div><div class='ctx'> 	int ret = idr_type-&gt;destroy_object(uobj, why, attrs);</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * We can only fail gracefully if the user requested to destroy the</div><div class='del'>-	 * object or when a retry may be called upon an error.</div><div class='del'>-	 * In the rest of the cases, just remove whatever you can.</div><div class='del'>-	 */</div><div class='del'>-	if (ib_is_destroy_retryable(ret, why, uobj))</div><div class='add'>+	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='ctx'> 	if (why == RDMA_REMOVE_ABORT)</div><div class='hunk'>@@ -544,6 +559,9 @@ static void remove_handle_idr_uobject(struct ib_uobject *uobj)</div><div class='ctx'> </div><div class='ctx'> static void alloc_abort_fd_uobject(struct ib_uobject *uobj)</div><div class='ctx'> {</div><div class='add'>+	struct file *filp = uobj-&gt;object;</div><div class='add'>+</div><div class='add'>+	fput(filp);</div><div class='ctx'> 	put_unused_fd(uobj-&gt;id);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -553,11 +571,8 @@ static int __must_check destroy_hw_fd_uobject(struct ib_uobject *uobj,</div><div class='ctx'> {</div><div class='ctx'> 	const struct uverbs_obj_fd_type *fd_type = container_of(</div><div class='ctx'> 		uobj-&gt;uapi_object-&gt;type_attrs, struct uverbs_obj_fd_type, type);</div><div class='del'>-	int ret = fd_type-&gt;context_closed(uobj, why);</div><div class='del'>-</div><div class='del'>-	if (ib_is_destroy_retryable(ret, why, uobj))</div><div class='del'>-		return ret;</div><div class='ctx'> </div><div class='add'>+	fd_type-&gt;destroy_object(uobj, why);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -565,7 +580,7 @@ static void remove_handle_fd_uobject(struct ib_uobject *uobj)</div><div class='ctx'> {</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int alloc_commit_idr_uobject(struct ib_uobject *uobj)</div><div class='add'>+static void alloc_commit_idr_uobject(struct ib_uobject *uobj)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_uverbs_file *ufile = uobj-&gt;ufile;</div><div class='ctx'> 	void *old;</div><div class='hunk'>@@ -579,33 +594,35 @@ static int alloc_commit_idr_uobject(struct ib_uobject *uobj)</div><div class='ctx'> 	 */</div><div class='ctx'> 	old = xa_store(&amp;ufile-&gt;idr, uobj-&gt;id, uobj, GFP_KERNEL);</div><div class='ctx'> 	WARN_ON(old != NULL);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int alloc_commit_fd_uobject(struct ib_uobject *uobj)</div><div class='add'>+static void swap_idr_uobjects(struct ib_uobject *obj_old,</div><div class='add'>+			     struct ib_uobject *obj_new)</div><div class='ctx'> {</div><div class='del'>-	const struct uverbs_obj_fd_type *fd_type = container_of(</div><div class='del'>-		uobj-&gt;uapi_object-&gt;type_attrs, struct uverbs_obj_fd_type, type);</div><div class='del'>-	int fd = uobj-&gt;id;</div><div class='del'>-	struct file *filp;</div><div class='add'>+	struct ib_uverbs_file *ufile = obj_old-&gt;ufile;</div><div class='add'>+	void *old;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='del'>-	 * The kref for uobj is moved into filp-&gt;private data and put in</div><div class='del'>-	 * uverbs_close_fd(). Once alloc_commit() succeeds uverbs_close_fd()</div><div class='del'>-	 * must be guaranteed to be called from the provided fops release</div><div class='del'>-	 * callback.</div><div class='add'>+	 * New must be an object that been allocated but not yet committed, this</div><div class='add'>+	 * moves the pre-committed state to obj_old, new still must be comitted.</div><div class='ctx'> 	 */</div><div class='del'>-	filp = anon_inode_getfile(fd_type-&gt;name,</div><div class='del'>-				  fd_type-&gt;fops,</div><div class='del'>-				  uobj,</div><div class='del'>-				  fd_type-&gt;flags);</div><div class='del'>-	if (IS_ERR(filp))</div><div class='del'>-		return PTR_ERR(filp);</div><div class='add'>+	old = xa_cmpxchg(&amp;ufile-&gt;idr, obj_old-&gt;id, obj_old, XA_ZERO_ENTRY,</div><div class='add'>+			 GFP_KERNEL);</div><div class='add'>+	if (WARN_ON(old != obj_old))</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-	uobj-&gt;object = filp;</div><div class='add'>+	swap(obj_old-&gt;id, obj_new-&gt;id);</div><div class='ctx'> </div><div class='del'>-	/* Matching put will be done in uverbs_close_fd() */</div><div class='add'>+	old = xa_cmpxchg(&amp;ufile-&gt;idr, obj_old-&gt;id, NULL, obj_old, GFP_KERNEL);</div><div class='add'>+	WARN_ON(old != NULL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void alloc_commit_fd_uobject(struct ib_uobject *uobj)</div><div class='add'>+{</div><div class='add'>+	int fd = uobj-&gt;id;</div><div class='add'>+	struct file *filp = uobj-&gt;object;</div><div class='add'>+</div><div class='add'>+	/* Matching put will be done in uverbs_uobject_fd_release() */</div><div class='ctx'> 	kref_get(&amp;uobj-&gt;ufile-&gt;ref);</div><div class='ctx'> </div><div class='ctx'> 	/* This shouldn't be used anymore. Use the file object instead */</div><div class='hunk'>@@ -613,11 +630,10 @@ static int alloc_commit_fd_uobject(struct ib_uobject *uobj)</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * NOTE: Once we install the file we loose ownership of our kref on</div><div class='del'>-	 * uobj. It will be put by uverbs_close_fd()</div><div class='add'>+	 * uobj. It will be put by uverbs_uobject_fd_release()</div><div class='ctx'> 	 */</div><div class='add'>+	filp-&gt;private_data = uobj;</div><div class='ctx'> 	fd_install(fd, filp);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -625,19 +641,10 @@ static int alloc_commit_fd_uobject(struct ib_uobject *uobj)</div><div class='ctx'>  * caller can no longer assume uobj is valid. If this function fails it</div><div class='ctx'>  * destroys the uboject, including the attached HW object.</div><div class='ctx'>  */</div><div class='del'>-int __must_check rdma_alloc_commit_uobject(struct ib_uobject *uobj,</div><div class='del'>-					   struct uverbs_attr_bundle *attrs)</div><div class='add'>+void rdma_alloc_commit_uobject(struct ib_uobject *uobj,</div><div class='add'>+			       struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_uverbs_file *ufile = attrs-&gt;ufile;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	/* alloc_commit consumes the uobj kref */</div><div class='del'>-	ret = uobj-&gt;uapi_object-&gt;type_class-&gt;alloc_commit(uobj);</div><div class='del'>-	if (ret) {</div><div class='del'>-		uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT, attrs);</div><div class='del'>-		up_read(&amp;ufile-&gt;hw_destroy_rwsem);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='ctx'> 	/* kref is held so long as the uobj is on the uobj list. */</div><div class='ctx'> 	uverbs_uobject_get(uobj);</div><div class='hunk'>@@ -648,10 +655,40 @@ int __must_check rdma_alloc_commit_uobject(struct ib_uobject *uobj,</div><div class='ctx'> 	/* matches atomic_set(-1) in alloc_uobj */</div><div class='ctx'> 	atomic_set(&amp;uobj-&gt;usecnt, 0);</div><div class='ctx'> </div><div class='add'>+	/* alloc_commit consumes the uobj kref */</div><div class='add'>+	uobj-&gt;uapi_object-&gt;type_class-&gt;alloc_commit(uobj);</div><div class='add'>+</div><div class='ctx'> 	/* Matches the down_read in rdma_alloc_begin_uobject */</div><div class='ctx'> 	up_read(&amp;ufile-&gt;hw_destroy_rwsem);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+/*</div><div class='add'>+ * new_uobj will be assigned to the handle currently used by to_uobj, and</div><div class='add'>+ * to_uobj will be destroyed.</div><div class='add'>+ *</div><div class='add'>+ * Upon return the caller must do:</div><div class='add'>+ *    rdma_alloc_commit_uobject(new_uobj)</div><div class='add'>+ *    uobj_put_destroy(to_uobj)</div><div class='add'>+ *</div><div class='add'>+ * to_uobj must have a write get but the put mode switches to destroy once</div><div class='add'>+ * this is called.</div><div class='add'>+ */</div><div class='add'>+void rdma_assign_uobject(struct ib_uobject *to_uobj, struct ib_uobject *new_uobj,</div><div class='add'>+			struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	assert_uverbs_usecnt(new_uobj, UVERBS_LOOKUP_WRITE);</div><div class='add'>+</div><div class='add'>+	if (WARN_ON(to_uobj-&gt;uapi_object != new_uobj-&gt;uapi_object ||</div><div class='add'>+		    !to_uobj-&gt;uapi_object-&gt;type_class-&gt;swap_uobjects))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	to_uobj-&gt;uapi_object-&gt;type_class-&gt;swap_uobjects(to_uobj, new_uobj);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * If this fails then the uobject is still completely valid (though with</div><div class='add'>+	 * a new ID) and we leak it until context close.</div><div class='add'>+	 */</div><div class='add'>+	uverbs_destroy_uobject(to_uobj, RDMA_REMOVE_DESTROY, attrs);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -659,11 +696,25 @@ int __must_check rdma_alloc_commit_uobject(struct ib_uobject *uobj,</div><div class='ctx'>  * object and anything else connected to uobj before calling this.</div><div class='ctx'>  */</div><div class='ctx'> void rdma_alloc_abort_uobject(struct ib_uobject *uobj,</div><div class='del'>-			      struct uverbs_attr_bundle *attrs)</div><div class='add'>+			      struct uverbs_attr_bundle *attrs,</div><div class='add'>+			      bool hw_obj_valid)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_uverbs_file *ufile = uobj-&gt;ufile;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (hw_obj_valid) {</div><div class='add'>+		ret = uobj-&gt;uapi_object-&gt;type_class-&gt;destroy_hw(</div><div class='add'>+			uobj, RDMA_REMOVE_ABORT, attrs);</div><div class='add'>+		/*</div><div class='add'>+		 * If the driver couldn't destroy the object then go ahead and</div><div class='add'>+		 * commit it. Leaking objects that can't be destroyed is only</div><div class='add'>+		 * done during FD close after the driver has a few more tries to</div><div class='add'>+		 * destroy it.</div><div class='add'>+		 */</div><div class='add'>+		if (WARN_ON(ret))</div><div class='add'>+			return rdma_alloc_commit_uobject(uobj, attrs);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	uobj-&gt;object = NULL;</div><div class='ctx'> 	uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT, attrs);</div><div class='ctx'> </div><div class='ctx'> 	/* Matches the down_read in rdma_alloc_begin_uobject */</div><div class='hunk'>@@ -681,7 +732,10 @@ static void lookup_put_fd_uobject(struct ib_uobject *uobj,</div><div class='ctx'> 	struct file *filp = uobj-&gt;object;</div><div class='ctx'> </div><div class='ctx'> 	WARN_ON(mode != UVERBS_LOOKUP_READ);</div><div class='del'>-	/* This indirectly calls uverbs_close_fd and free the object */</div><div class='add'>+	/*</div><div class='add'>+	 * This indirectly calls uverbs_uobject_fd_release() and free the</div><div class='add'>+	 * object</div><div class='add'>+	 */</div><div class='ctx'> 	fput(filp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -689,7 +743,6 @@ void rdma_lookup_put_uobject(struct ib_uobject *uobj,</div><div class='ctx'> 			     enum rdma_lookup_mode mode)</div><div class='ctx'> {</div><div class='ctx'> 	assert_uverbs_usecnt(uobj, mode);</div><div class='del'>-	uobj-&gt;uapi_object-&gt;type_class-&gt;lookup_put(uobj, mode);</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * In order to unlock an object, either decrease its usecnt for</div><div class='ctx'> 	 * read access or zero it in case of exclusive access. See</div><div class='hunk'>@@ -706,6 +759,7 @@ void rdma_lookup_put_uobject(struct ib_uobject *uobj,</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	uobj-&gt;uapi_object-&gt;type_class-&gt;lookup_put(uobj, mode);</div><div class='ctx'> 	/* Pairs with the kref obtained by type-&gt;lookup_get */</div><div class='ctx'> 	uverbs_uobject_put(uobj);</div><div class='ctx'> }</div><div class='hunk'>@@ -744,33 +798,33 @@ const struct uverbs_obj_type_class uverbs_idr_class = {</div><div class='ctx'> 	.lookup_put = lookup_put_idr_uobject,</div><div class='ctx'> 	.destroy_hw = destroy_hw_idr_uobject,</div><div class='ctx'> 	.remove_handle = remove_handle_idr_uobject,</div><div class='del'>-	/*</div><div class='del'>-	 * When we destroy an object, we first just lock it for WRITE and</div><div class='del'>-	 * actually DESTROY it in the finalize stage. So, the problematic</div><div class='del'>-	 * scenario is when we just started the finalize stage of the</div><div class='del'>-	 * destruction (nothing was executed yet). Now, the other thread</div><div class='del'>-	 * fetched the object for READ access, but it didn't lock it yet.</div><div class='del'>-	 * The DESTROY thread continues and starts destroying the object.</div><div class='del'>-	 * When the other thread continue - without the RCU, it would</div><div class='del'>-	 * access freed memory. However, the rcu_read_lock delays the free</div><div class='del'>-	 * until the rcu_read_lock of the READ operation quits. Since the</div><div class='del'>-	 * exclusive lock of the object is still taken by the DESTROY flow, the</div><div class='del'>-	 * READ operation will get -EBUSY and it'll just bail out.</div><div class='del'>-	 */</div><div class='del'>-	.needs_kfree_rcu = true,</div><div class='add'>+	.swap_uobjects = swap_idr_uobjects,</div><div class='ctx'> };</div><div class='ctx'> EXPORT_SYMBOL(uverbs_idr_class);</div><div class='ctx'> </div><div class='del'>-void uverbs_close_fd(struct file *f)</div><div class='add'>+/*</div><div class='add'>+ * Users of UVERBS_TYPE_ALLOC_FD should set this function as the struct</div><div class='add'>+ * file_operations release method.</div><div class='add'>+ */</div><div class='add'>+int uverbs_uobject_fd_release(struct inode *inode, struct file *filp)</div><div class='ctx'> {</div><div class='del'>-	struct ib_uobject *uobj = f-&gt;private_data;</div><div class='del'>-	struct ib_uverbs_file *ufile = uobj-&gt;ufile;</div><div class='del'>-	struct uverbs_attr_bundle attrs = {</div><div class='del'>-		.context = uobj-&gt;context,</div><div class='del'>-		.ufile = ufile,</div><div class='del'>-	};</div><div class='add'>+	struct ib_uverbs_file *ufile;</div><div class='add'>+	struct ib_uobject *uobj;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * This can only happen if the fput came from alloc_abort_fd_uobject()</div><div class='add'>+	 */</div><div class='add'>+	if (!filp-&gt;private_data)</div><div class='add'>+		return 0;</div><div class='add'>+	uobj = filp-&gt;private_data;</div><div class='add'>+	ufile = uobj-&gt;ufile;</div><div class='ctx'> </div><div class='ctx'> 	if (down_read_trylock(&amp;ufile-&gt;hw_destroy_rwsem)) {</div><div class='add'>+		struct uverbs_attr_bundle attrs = {</div><div class='add'>+			.context = uobj-&gt;context,</div><div class='add'>+			.ufile = ufile,</div><div class='add'>+		};</div><div class='add'>+</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * lookup_get_fd_uobject holds the kref on the struct file any</div><div class='ctx'> 		 * time a FD uobj is locked, which prevents this release</div><div class='hunk'>@@ -782,13 +836,14 @@ void uverbs_close_fd(struct file *f)</div><div class='ctx'> 		up_read(&amp;ufile-&gt;hw_destroy_rwsem);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* Matches the get in alloc_begin_fd_uobject */</div><div class='add'>+	/* Matches the get in alloc_commit_fd_uobject() */</div><div class='ctx'> 	kref_put(&amp;ufile-&gt;ref, ib_uverbs_release_file);</div><div class='ctx'> </div><div class='ctx'> 	/* Pairs with filp-&gt;private_data in alloc_begin_fd_uobject */</div><div class='ctx'> 	uverbs_uobject_put(uobj);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(uverbs_close_fd);</div><div class='add'>+EXPORT_SYMBOL(uverbs_uobject_fd_release);</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * Drop the ucontext off the ufile and completely disconnect it from the</div><div class='hunk'>@@ -817,6 +872,7 @@ static void ufile_destroy_ucontext(struct ib_uverbs_file *ufile,</div><div class='ctx'> 	rdma_restrack_del(&amp;ucontext-&gt;res);</div><div class='ctx'> </div><div class='ctx'> 	ib_dev-&gt;ops.dealloc_ucontext(ucontext);</div><div class='add'>+	WARN_ON(!xa_empty(&amp;ucontext-&gt;mmap_xa));</div><div class='ctx'> 	kfree(ucontext);</div><div class='ctx'> </div><div class='ctx'> 	ufile-&gt;ucontext = NULL;</div><div class='hunk'>@@ -825,9 +881,14 @@ static void ufile_destroy_ucontext(struct ib_uverbs_file *ufile,</div><div class='ctx'> static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile,</div><div class='ctx'> 				  enum rdma_remove_reason reason)</div><div class='ctx'> {</div><div class='add'>+	struct uverbs_attr_bundle attrs = { .ufile = ufile };</div><div class='add'>+	struct ib_ucontext *ucontext = ufile-&gt;ucontext;</div><div class='add'>+	struct ib_device *ib_dev = ucontext-&gt;device;</div><div class='ctx'> 	struct ib_uobject *obj, *next_obj;</div><div class='ctx'> 	int ret = -EINVAL;</div><div class='del'>-	struct uverbs_attr_bundle attrs = { .ufile = ufile };</div><div class='add'>+</div><div class='add'>+	if (ib_dev-&gt;ops.ufile_hw_cleanup)</div><div class='add'>+		ib_dev-&gt;ops.ufile_hw_cleanup(ufile);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * This shouldn't run while executing other commands on this</div><div class='hunk'>@@ -845,18 +906,23 @@ static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile,</div><div class='ctx'> 		 * racing with a lookup_get.</div><div class='ctx'> 		 */</div><div class='ctx'> 		WARN_ON(uverbs_try_lock_object(obj, UVERBS_LOOKUP_WRITE));</div><div class='add'>+		if (reason == RDMA_REMOVE_DRIVER_FAILURE)</div><div class='add'>+			obj-&gt;object = NULL;</div><div class='ctx'> 		if (!uverbs_destroy_uobject(obj, reason, &amp;attrs))</div><div class='ctx'> 			ret = 0;</div><div class='ctx'> 		else</div><div class='ctx'> 			atomic_set(&amp;obj-&gt;usecnt, 0);</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	if (reason == RDMA_REMOVE_DRIVER_FAILURE) {</div><div class='add'>+		WARN_ON(!list_empty(&amp;ufile-&gt;uobjects));</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='del'>- * Destroy the uncontext and every uobject associated with it. If called with</div><div class='del'>- * reason != RDMA_REMOVE_CLOSE this will not return until the destruction has</div><div class='del'>- * been completed and ufile-&gt;ucontext is NULL.</div><div class='add'>+ * Destroy the ucontext and every uobject associated with it.</div><div class='ctx'>  *</div><div class='ctx'>  * This is internally locked and can be called in parallel from multiple</div><div class='ctx'>  * contexts.</div><div class='hunk'>@@ -864,22 +930,6 @@ static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile,</div><div class='ctx'> void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile,</div><div class='ctx'> 			     enum rdma_remove_reason reason)</div><div class='ctx'> {</div><div class='del'>-	if (reason == RDMA_REMOVE_CLOSE) {</div><div class='del'>-		/*</div><div class='del'>-		 * During destruction we might trigger something that</div><div class='del'>-		 * synchronously calls release on any file descriptor. For</div><div class='del'>-		 * this reason all paths that come from file_operations</div><div class='del'>-		 * release must use try_lock. They can progress knowing that</div><div class='del'>-		 * there is an ongoing uverbs_destroy_ufile_hw that will clean</div><div class='del'>-		 * up the driver resources.</div><div class='del'>-		 */</div><div class='del'>-		if (!mutex_trylock(&amp;ufile-&gt;ucontext_lock))</div><div class='del'>-			return;</div><div class='del'>-</div><div class='del'>-	} else {</div><div class='del'>-		mutex_lock(&amp;ufile-&gt;ucontext_lock);</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	down_write(&amp;ufile-&gt;hw_destroy_rwsem);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='hunk'>@@ -889,26 +939,16 @@ void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile,</div><div class='ctx'> 	if (!ufile-&gt;ucontext)</div><div class='ctx'> 		goto done;</div><div class='ctx'> </div><div class='del'>-	ufile-&gt;ucontext-&gt;closing = true;</div><div class='del'>-	ufile-&gt;ucontext-&gt;cleanup_retryable = true;</div><div class='del'>-	while (!list_empty(&amp;ufile-&gt;uobjects))</div><div class='del'>-		if (__uverbs_cleanup_ufile(ufile, reason)) {</div><div class='del'>-			/*</div><div class='del'>-			 * No entry was cleaned-up successfully during this</div><div class='del'>-			 * iteration</div><div class='del'>-			 */</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-	ufile-&gt;ucontext-&gt;cleanup_retryable = false;</div><div class='del'>-	if (!list_empty(&amp;ufile-&gt;uobjects))</div><div class='del'>-		__uverbs_cleanup_ufile(ufile, reason);</div><div class='add'>+	while (!list_empty(&amp;ufile-&gt;uobjects) &amp;&amp;</div><div class='add'>+	       !__uverbs_cleanup_ufile(ufile, reason)) {</div><div class='add'>+	}</div><div class='ctx'> </div><div class='add'>+	if (WARN_ON(!list_empty(&amp;ufile-&gt;uobjects)))</div><div class='add'>+		__uverbs_cleanup_ufile(ufile, RDMA_REMOVE_DRIVER_FAILURE);</div><div class='ctx'> 	ufile_destroy_ucontext(ufile, reason);</div><div class='ctx'> </div><div class='ctx'> done:</div><div class='ctx'> 	up_write(&amp;ufile-&gt;hw_destroy_rwsem);</div><div class='del'>-	mutex_unlock(&amp;ufile-&gt;ucontext_lock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> const struct uverbs_obj_type_class uverbs_fd_class = {</div><div class='hunk'>@@ -919,7 +959,6 @@ const struct uverbs_obj_type_class uverbs_fd_class = {</div><div class='ctx'> 	.lookup_put = lookup_put_fd_uobject,</div><div class='ctx'> 	.destroy_hw = destroy_hw_fd_uobject,</div><div class='ctx'> 	.remove_handle = remove_handle_fd_uobject,</div><div class='del'>-	.needs_kfree_rcu = false,</div><div class='ctx'> };</div><div class='ctx'> EXPORT_SYMBOL(uverbs_fd_class);</div><div class='ctx'> </div><div class='hunk'>@@ -942,19 +981,17 @@ uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access,</div><div class='ctx'> 		return rdma_lookup_get_uobject(obj, attrs-&gt;ufile, id,</div><div class='ctx'> 					       UVERBS_LOOKUP_WRITE, attrs);</div><div class='ctx'> 	case UVERBS_ACCESS_NEW:</div><div class='del'>-		return rdma_alloc_begin_uobject(obj, attrs-&gt;ufile, attrs);</div><div class='add'>+		return rdma_alloc_begin_uobject(obj, attrs);</div><div class='ctx'> 	default:</div><div class='ctx'> 		WARN_ON(true);</div><div class='ctx'> 		return ERR_PTR(-EOPNOTSUPP);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int uverbs_finalize_object(struct ib_uobject *uobj,</div><div class='del'>-			   enum uverbs_obj_access access, bool commit,</div><div class='del'>-			   struct uverbs_attr_bundle *attrs)</div><div class='add'>+void uverbs_finalize_object(struct ib_uobject *uobj,</div><div class='add'>+			    enum uverbs_obj_access access, bool hw_obj_valid,</div><div class='add'>+			    bool commit, struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * refcounts should be handled at the object level and not at the</div><div class='ctx'> 	 * uobject level. Refcounts of the objects themselves are done in</div><div class='hunk'>@@ -974,14 +1011,40 @@ int uverbs_finalize_object(struct ib_uobject *uobj,</div><div class='ctx'> 		break;</div><div class='ctx'> 	case UVERBS_ACCESS_NEW:</div><div class='ctx'> 		if (commit)</div><div class='del'>-			ret = rdma_alloc_commit_uobject(uobj, attrs);</div><div class='add'>+			rdma_alloc_commit_uobject(uobj, attrs);</div><div class='ctx'> 		else</div><div class='del'>-			rdma_alloc_abort_uobject(uobj, attrs);</div><div class='add'>+			rdma_alloc_abort_uobject(uobj, attrs, hw_obj_valid);</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		WARN_ON(true);</div><div class='del'>-		ret = -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	return ret;</div><div class='add'>+/**</div><div class='add'>+ * rdma_uattrs_has_raw_cap() - Returns whether a rdma device linked to the</div><div class='add'>+ *			       uverbs attributes file has CAP_NET_RAW</div><div class='add'>+ *			       capability or not.</div><div class='add'>+ *</div><div class='add'>+ * @attrs:       Pointer to uverbs attributes</div><div class='add'>+ *</div><div class='add'>+ * Returns true if a rdma device's owning user namespace has CAP_NET_RAW</div><div class='add'>+ * capability, otherwise false.</div><div class='add'>+ */</div><div class='add'>+bool rdma_uattrs_has_raw_cap(const struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uverbs_file *ufile = attrs-&gt;ufile;</div><div class='add'>+	struct ib_ucontext *ucontext;</div><div class='add'>+	bool has_cap = false;</div><div class='add'>+	int srcu_key;</div><div class='add'>+</div><div class='add'>+	srcu_key = srcu_read_lock(&amp;ufile-&gt;device-&gt;disassociate_srcu);</div><div class='add'>+	ucontext = ib_uverbs_get_ucontext_file(ufile);</div><div class='add'>+	if (IS_ERR(ucontext))</div><div class='add'>+		goto out;</div><div class='add'>+	has_cap = rdma_dev_has_raw_cap(ucontext-&gt;device);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	srcu_read_unlock(&amp;ufile-&gt;device-&gt;disassociate_srcu, srcu_key);</div><div class='add'>+	return has_cap;</div><div class='ctx'> }</div><div class='add'>+EXPORT_SYMBOL(rdma_uattrs_has_raw_cap);</div><div class='head'>diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h<br/>index 5445323629b5..a59b087611cb 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/rdma_core.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/rdma_core.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/rdma_core.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/rdma_core.h</a></div><div class='hunk'>@@ -51,29 +51,6 @@ void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile,</div><div class='ctx'> int uobj_destroy(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs);</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='del'>- * uverbs_uobject_get is called in order to increase the reference count on</div><div class='del'>- * an uobject. This is useful when a handler wants to keep the uobject's memory</div><div class='del'>- * alive, regardless if this uobject is still alive in the context's objects</div><div class='del'>- * repository. Objects are put via uverbs_uobject_put.</div><div class='del'>- */</div><div class='del'>-void uverbs_uobject_get(struct ib_uobject *uobject);</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * In order to indicate we no longer needs this uobject, uverbs_uobject_put</div><div class='del'>- * is called. When the reference count is decreased, the uobject is freed.</div><div class='del'>- * For example, this is used when attaching a completion channel to a CQ.</div><div class='del'>- */</div><div class='del'>-void uverbs_uobject_put(struct ib_uobject *uobject);</div><div class='del'>-</div><div class='del'>-/* Indicate this fd is no longer used by this consumer, but its memory isn't</div><div class='del'>- * necessarily released yet. When the last reference is put, we release the</div><div class='del'>- * memory. After this call is executed, calling uverbs_uobject_get isn't</div><div class='del'>- * allowed.</div><div class='del'>- * This must be called from the release file_operations of the file!</div><div class='del'>- */</div><div class='del'>-void uverbs_close_fd(struct file *f);</div><div class='del'>-</div><div class='del'>-/*</div><div class='ctx'>  * Get an ib_uobject that corresponds to the given id from ufile, assuming</div><div class='ctx'>  * the object is from the given type. Lock it to the required access when</div><div class='ctx'>  * applicable.</div><div class='hunk'>@@ -86,30 +63,17 @@ struct ib_uobject *</div><div class='ctx'> uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access,</div><div class='ctx'> 			     s64 id, struct uverbs_attr_bundle *attrs);</div><div class='ctx'> </div><div class='del'>-/*</div><div class='del'>- * Note that certain finalize stages could return a status:</div><div class='del'>- *   (a) alloc_commit could return a failure if the object is committed at the</div><div class='del'>- *       same time when the context is destroyed.</div><div class='del'>- *   (b) remove_commit could fail if the object wasn't destroyed successfully.</div><div class='del'>- * Since multiple objects could be finalized in one transaction, it is very NOT</div><div class='del'>- * recommended to have several finalize actions which have side effects.</div><div class='del'>- * For example, it's NOT recommended to have a certain action which has both</div><div class='del'>- * a commit action and a destroy action or two destroy objects in the same</div><div class='del'>- * action. The rule of thumb is to have one destroy or commit action with</div><div class='del'>- * multiple lookups.</div><div class='del'>- * The first non zero return value of finalize_object is returned from this</div><div class='del'>- * function. For example, this could happen when we couldn't destroy an</div><div class='del'>- * object.</div><div class='del'>- */</div><div class='del'>-int uverbs_finalize_object(struct ib_uobject *uobj,</div><div class='del'>-			   enum uverbs_obj_access access, bool commit,</div><div class='del'>-			   struct uverbs_attr_bundle *attrs);</div><div class='add'>+void uverbs_finalize_object(struct ib_uobject *uobj,</div><div class='add'>+			    enum uverbs_obj_access access, bool hw_obj_valid,</div><div class='add'>+			    bool commit, struct uverbs_attr_bundle *attrs);</div><div class='ctx'> </div><div class='ctx'> int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx);</div><div class='ctx'> </div><div class='ctx'> void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile);</div><div class='ctx'> void release_ufile_idr_uobject(struct ib_uverbs_file *ufile);</div><div class='ctx'> </div><div class='add'>+struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs);</div><div class='add'>+</div><div class='ctx'> /*</div><div class='ctx'>  * This is the runtime description of the uverbs API, used by the syscall</div><div class='ctx'>  * machinery to validate and dispatch calls.</div><div class='hunk'>@@ -187,13 +151,18 @@ void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,</div><div class='ctx'> 			      unsigned int num_attrs);</div><div class='ctx'> void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile);</div><div class='ctx'> </div><div class='add'>+extern const struct uapi_definition uverbs_def_obj_async_fd[];</div><div class='ctx'> extern const struct uapi_definition uverbs_def_obj_counters[];</div><div class='ctx'> extern const struct uapi_definition uverbs_def_obj_cq[];</div><div class='ctx'> extern const struct uapi_definition uverbs_def_obj_device[];</div><div class='ctx'> extern const struct uapi_definition uverbs_def_obj_dm[];</div><div class='add'>+extern const struct uapi_definition uverbs_def_obj_dmah[];</div><div class='ctx'> extern const struct uapi_definition uverbs_def_obj_flow_action[];</div><div class='ctx'> extern const struct uapi_definition uverbs_def_obj_intf[];</div><div class='ctx'> extern const struct uapi_definition uverbs_def_obj_mr[];</div><div class='add'>+extern const struct uapi_definition uverbs_def_obj_qp[];</div><div class='add'>+extern const struct uapi_definition uverbs_def_obj_srq[];</div><div class='add'>+extern const struct uapi_definition uverbs_def_obj_wq[];</div><div class='ctx'> extern const struct uapi_definition uverbs_def_write_intf[];</div><div class='ctx'> </div><div class='ctx'> static inline const struct uverbs_api_write_method *</div><div class='head'>diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c<br/>index 3b5ff2f7b5f8..b097cfcade1c 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/restrack.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/restrack.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/restrack.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/restrack.c</a></div><div class='hunk'>@@ -6,6 +6,7 @@</div><div class='ctx'> #include &lt;rdma/rdma_cm.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_verbs.h&gt;</div><div class='ctx'> #include &lt;rdma/restrack.h&gt;</div><div class='add'>+#include &lt;rdma/rdma_counter.h&gt;</div><div class='ctx'> #include &lt;linux/mutex.h&gt;</div><div class='ctx'> #include &lt;linux/sched/task.h&gt;</div><div class='ctx'> #include &lt;linux/pid_namespace.h&gt;</div><div class='hunk'>@@ -36,20 +37,6 @@ int rdma_restrack_init(struct ib_device *dev)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static const char *type2str(enum rdma_restrack_type type)</div><div class='del'>-{</div><div class='del'>-	static const char * const names[RDMA_RESTRACK_MAX] = {</div><div class='del'>-		[RDMA_RESTRACK_PD] = "PD",</div><div class='del'>-		[RDMA_RESTRACK_CQ] = "CQ",</div><div class='del'>-		[RDMA_RESTRACK_QP] = "QP",</div><div class='del'>-		[RDMA_RESTRACK_CM_ID] = "CM_ID",</div><div class='del'>-		[RDMA_RESTRACK_MR] = "MR",</div><div class='del'>-		[RDMA_RESTRACK_CTX] = "CTX",</div><div class='del'>-	};</div><div class='del'>-</div><div class='del'>-	return names[type];</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> /**</div><div class='ctx'>  * rdma_restrack_clean() - clean resource tracking</div><div class='ctx'>  * @dev:  IB device</div><div class='hunk'>@@ -57,47 +44,14 @@ static const char *type2str(enum rdma_restrack_type type)</div><div class='ctx'> void rdma_restrack_clean(struct ib_device *dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_restrack_root *rt = dev-&gt;res;</div><div class='del'>-	struct rdma_restrack_entry *e;</div><div class='del'>-	char buf[TASK_COMM_LEN];</div><div class='del'>-	bool found = false;</div><div class='del'>-	const char *owner;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0 ; i &lt; RDMA_RESTRACK_MAX; i++) {</div><div class='ctx'> 		struct xarray *xa = &amp;dev-&gt;res[i].xa;</div><div class='ctx'> </div><div class='del'>-		if (!xa_empty(xa)) {</div><div class='del'>-			unsigned long index;</div><div class='del'>-</div><div class='del'>-			if (!found) {</div><div class='del'>-				pr_err("restrack: %s", CUT_HERE);</div><div class='del'>-				dev_err(&amp;dev-&gt;dev, "BUG: RESTRACK detected leak of resources\n");</div><div class='del'>-			}</div><div class='del'>-			xa_for_each(xa, index, e) {</div><div class='del'>-				if (rdma_is_kernel_res(e)) {</div><div class='del'>-					owner = e-&gt;kern_name;</div><div class='del'>-				} else {</div><div class='del'>-					/*</div><div class='del'>-					 * There is no need to call get_task_struct here,</div><div class='del'>-					 * because we can be here only if there are more</div><div class='del'>-					 * get_task_struct() call than put_task_struct().</div><div class='del'>-					 */</div><div class='del'>-					get_task_comm(buf, e-&gt;task);</div><div class='del'>-					owner = buf;</div><div class='del'>-				}</div><div class='del'>-</div><div class='del'>-				pr_err("restrack: %s %s object allocated by %s is not freed\n",</div><div class='del'>-				       rdma_is_kernel_res(e) ? "Kernel" :</div><div class='del'>-							       "User",</div><div class='del'>-				       type2str(e-&gt;type), owner);</div><div class='del'>-			}</div><div class='del'>-			found = true;</div><div class='del'>-		}</div><div class='add'>+		WARN_ON(!xa_empty(xa));</div><div class='ctx'> 		xa_destroy(xa);</div><div class='ctx'> 	}</div><div class='del'>-	if (found)</div><div class='del'>-		pr_err("restrack: %s", CUT_HERE);</div><div class='del'>-</div><div class='ctx'> 	kfree(rt);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -105,10 +59,10 @@ void rdma_restrack_clean(struct ib_device *dev)</div><div class='ctx'>  * rdma_restrack_count() - the current usage of specific object</div><div class='ctx'>  * @dev:  IB device</div><div class='ctx'>  * @type: actual type of object to operate</div><div class='del'>- * @ns:   PID namespace</div><div class='add'>+ * @show_details: count driver specific objects</div><div class='ctx'>  */</div><div class='ctx'> int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type,</div><div class='del'>-			struct pid_namespace *ns)</div><div class='add'>+			bool show_details)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_restrack_root *rt = &amp;dev-&gt;res[type];</div><div class='ctx'> 	struct rdma_restrack_entry *e;</div><div class='hunk'>@@ -117,42 +71,15 @@ int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type,</div><div class='ctx'> </div><div class='ctx'> 	xa_lock(&amp;rt-&gt;xa);</div><div class='ctx'> 	xas_for_each(&amp;xas, e, U32_MAX) {</div><div class='del'>-		if (ns == &amp;init_pid_ns ||</div><div class='del'>-		    (!rdma_is_kernel_res(e) &amp;&amp;</div><div class='del'>-		     ns == task_active_pid_ns(e-&gt;task)))</div><div class='del'>-			cnt++;</div><div class='add'>+		if (xa_get_mark(&amp;rt-&gt;xa, e-&gt;id, RESTRACK_DD) &amp;&amp; !show_details)</div><div class='add'>+			continue;</div><div class='add'>+		cnt++;</div><div class='ctx'> 	}</div><div class='ctx'> 	xa_unlock(&amp;rt-&gt;xa);</div><div class='ctx'> 	return cnt;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_restrack_count);</div><div class='ctx'> </div><div class='del'>-static void set_kern_name(struct rdma_restrack_entry *res)</div><div class='del'>-{</div><div class='del'>-	struct ib_pd *pd;</div><div class='del'>-</div><div class='del'>-	switch (res-&gt;type) {</div><div class='del'>-	case RDMA_RESTRACK_QP:</div><div class='del'>-		pd = container_of(res, struct ib_qp, res)-&gt;pd;</div><div class='del'>-		if (!pd) {</div><div class='del'>-			WARN_ONCE(true, "XRC QPs are not supported\n");</div><div class='del'>-			/* Survive, despite the programmer's error */</div><div class='del'>-			res-&gt;kern_name = " ";</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case RDMA_RESTRACK_MR:</div><div class='del'>-		pd = container_of(res, struct ib_mr, res)-&gt;pd;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		/* Other types set kern_name directly */</div><div class='del'>-		pd = NULL;</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (pd)</div><div class='del'>-		res-&gt;kern_name = pd-&gt;res.kern_name;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static struct ib_device *res_to_dev(struct rdma_restrack_entry *res)</div><div class='ctx'> {</div><div class='ctx'> 	switch (res-&gt;type) {</div><div class='hunk'>@@ -169,85 +96,136 @@ static struct ib_device *res_to_dev(struct rdma_restrack_entry *res)</div><div class='ctx'> 		return container_of(res, struct ib_mr, res)-&gt;device;</div><div class='ctx'> 	case RDMA_RESTRACK_CTX:</div><div class='ctx'> 		return container_of(res, struct ib_ucontext, res)-&gt;device;</div><div class='add'>+	case RDMA_RESTRACK_COUNTER:</div><div class='add'>+		return container_of(res, struct rdma_counter, res)-&gt;device;</div><div class='add'>+	case RDMA_RESTRACK_SRQ:</div><div class='add'>+		return container_of(res, struct ib_srq, res)-&gt;device;</div><div class='add'>+	case RDMA_RESTRACK_DMAH:</div><div class='add'>+		return container_of(res, struct ib_dmah, res)-&gt;device;</div><div class='ctx'> 	default:</div><div class='ctx'> 		WARN_ONCE(true, "Wrong resource tracking type %u\n", res-&gt;type);</div><div class='ctx'> 		return NULL;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void rdma_restrack_set_task(struct rdma_restrack_entry *res,</div><div class='del'>-			    const char *caller)</div><div class='add'>+/**</div><div class='add'>+ * rdma_restrack_attach_task() - attach the task onto this resource,</div><div class='add'>+ * valid for user space restrack entries.</div><div class='add'>+ * @res:  resource entry</div><div class='add'>+ * @task: the task to attach</div><div class='add'>+ */</div><div class='add'>+static void rdma_restrack_attach_task(struct rdma_restrack_entry *res,</div><div class='add'>+				      struct task_struct *task)</div><div class='ctx'> {</div><div class='del'>-	if (caller) {</div><div class='del'>-		res-&gt;kern_name = caller;</div><div class='add'>+	if (WARN_ON_ONCE(!task))</div><div class='ctx'> 		return;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='ctx'> 	if (res-&gt;task)</div><div class='ctx'> 		put_task_struct(res-&gt;task);</div><div class='del'>-	get_task_struct(current);</div><div class='del'>-	res-&gt;task = current;</div><div class='add'>+	get_task_struct(task);</div><div class='add'>+	res-&gt;task = task;</div><div class='add'>+	res-&gt;user = true;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(rdma_restrack_set_task);</div><div class='ctx'> </div><div class='del'>-static void rdma_restrack_add(struct rdma_restrack_entry *res)</div><div class='add'>+/**</div><div class='add'>+ * rdma_restrack_set_name() - set the task for this resource</div><div class='add'>+ * @res:  resource entry</div><div class='add'>+ * @caller: kernel name, the current task will be used if the caller is NULL.</div><div class='add'>+ */</div><div class='add'>+void rdma_restrack_set_name(struct rdma_restrack_entry *res, const char *caller)</div><div class='ctx'> {</div><div class='del'>-	struct ib_device *dev = res_to_dev(res);</div><div class='del'>-	struct rdma_restrack_root *rt;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if (!dev)</div><div class='add'>+	if (caller) {</div><div class='add'>+		res-&gt;kern_name = caller;</div><div class='ctx'> 		return;</div><div class='del'>-</div><div class='del'>-	rt = &amp;dev-&gt;res[res-&gt;type];</div><div class='del'>-</div><div class='del'>-	kref_init(&amp;res-&gt;kref);</div><div class='del'>-	init_completion(&amp;res-&gt;comp);</div><div class='del'>-	if (res-&gt;type != RDMA_RESTRACK_QP)</div><div class='del'>-		ret = xa_alloc_cyclic(&amp;rt-&gt;xa, &amp;res-&gt;id, res, xa_limit_32b,</div><div class='del'>-				&amp;rt-&gt;next_id, GFP_KERNEL);</div><div class='del'>-	else {</div><div class='del'>-		/* Special case to ensure that LQPN points to right QP */</div><div class='del'>-		struct ib_qp *qp = container_of(res, struct ib_qp, res);</div><div class='del'>-</div><div class='del'>-		ret = xa_insert(&amp;rt-&gt;xa, qp-&gt;qp_num, res, GFP_KERNEL);</div><div class='del'>-		res-&gt;id = ret ? 0 : qp-&gt;qp_num;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!ret)</div><div class='del'>-		res-&gt;valid = true;</div><div class='add'>+	rdma_restrack_attach_task(res, current);</div><div class='ctx'> }</div><div class='add'>+EXPORT_SYMBOL(rdma_restrack_set_name);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * rdma_restrack_kadd() - add kernel object to the reource tracking database</div><div class='del'>- * @res:  resource entry</div><div class='add'>+ * rdma_restrack_parent_name() - set the restrack name properties based</div><div class='add'>+ * on parent restrack</div><div class='add'>+ * @dst: destination resource entry</div><div class='add'>+ * @parent: parent resource entry</div><div class='add'>+ */</div><div class='add'>+void rdma_restrack_parent_name(struct rdma_restrack_entry *dst,</div><div class='add'>+			       const struct rdma_restrack_entry *parent)</div><div class='add'>+{</div><div class='add'>+	if (rdma_is_kernel_res(parent))</div><div class='add'>+		dst-&gt;kern_name = parent-&gt;kern_name;</div><div class='add'>+	else</div><div class='add'>+		rdma_restrack_attach_task(dst, parent-&gt;task);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_restrack_parent_name);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * rdma_restrack_new() - Initializes new restrack entry to allow _put() interface</div><div class='add'>+ * to release memory in fully automatic way.</div><div class='add'>+ * @res: Entry to initialize</div><div class='add'>+ * @type: REstrack type</div><div class='ctx'>  */</div><div class='del'>-void rdma_restrack_kadd(struct rdma_restrack_entry *res)</div><div class='add'>+void rdma_restrack_new(struct rdma_restrack_entry *res,</div><div class='add'>+		       enum rdma_restrack_type type)</div><div class='ctx'> {</div><div class='del'>-	res-&gt;task = NULL;</div><div class='del'>-	set_kern_name(res);</div><div class='del'>-	res-&gt;user = false;</div><div class='del'>-	rdma_restrack_add(res);</div><div class='add'>+	kref_init(&amp;res-&gt;kref);</div><div class='add'>+	init_completion(&amp;res-&gt;comp);</div><div class='add'>+	res-&gt;type = type;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(rdma_restrack_kadd);</div><div class='add'>+EXPORT_SYMBOL(rdma_restrack_new);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * rdma_restrack_uadd() - add user object to the reource tracking database</div><div class='add'>+ * rdma_restrack_add() - add object to the resource tracking database</div><div class='ctx'>  * @res:  resource entry</div><div class='ctx'>  */</div><div class='del'>-void rdma_restrack_uadd(struct rdma_restrack_entry *res)</div><div class='add'>+void rdma_restrack_add(struct rdma_restrack_entry *res)</div><div class='ctx'> {</div><div class='del'>-	if (res-&gt;type != RDMA_RESTRACK_CM_ID)</div><div class='del'>-		res-&gt;task = NULL;</div><div class='add'>+	struct ib_device *dev = res_to_dev(res);</div><div class='add'>+	struct rdma_restrack_root *rt;</div><div class='add'>+	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	if (!res-&gt;task)</div><div class='del'>-		rdma_restrack_set_task(res, NULL);</div><div class='del'>-	res-&gt;kern_name = NULL;</div><div class='add'>+	if (!dev)</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-	res-&gt;user = true;</div><div class='del'>-	rdma_restrack_add(res);</div><div class='add'>+	if (res-&gt;no_track)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	rt = &amp;dev-&gt;res[res-&gt;type];</div><div class='add'>+</div><div class='add'>+	if (res-&gt;type == RDMA_RESTRACK_QP) {</div><div class='add'>+		/* Special case to ensure that LQPN points to right QP */</div><div class='add'>+		struct ib_qp *qp = container_of(res, struct ib_qp, res);</div><div class='add'>+</div><div class='add'>+		WARN_ONCE(qp-&gt;qp_num &gt;&gt; 24 || qp-&gt;port &gt;&gt; 8,</div><div class='add'>+			  "QP number 0x%0X and port 0x%0X", qp-&gt;qp_num,</div><div class='add'>+			  qp-&gt;port);</div><div class='add'>+		res-&gt;id = qp-&gt;qp_num;</div><div class='add'>+		if (qp-&gt;qp_type == IB_QPT_SMI || qp-&gt;qp_type == IB_QPT_GSI)</div><div class='add'>+			res-&gt;id |= qp-&gt;port &lt;&lt; 24;</div><div class='add'>+		ret = xa_insert(&amp;rt-&gt;xa, res-&gt;id, res, GFP_KERNEL);</div><div class='add'>+		if (ret)</div><div class='add'>+			res-&gt;id = 0;</div><div class='add'>+</div><div class='add'>+		if (qp-&gt;qp_type &gt;= IB_QPT_DRIVER)</div><div class='add'>+			xa_set_mark(&amp;rt-&gt;xa, res-&gt;id, RESTRACK_DD);</div><div class='add'>+	} else if (res-&gt;type == RDMA_RESTRACK_COUNTER) {</div><div class='add'>+		/* Special case to ensure that cntn points to right counter */</div><div class='add'>+		struct rdma_counter *counter;</div><div class='add'>+</div><div class='add'>+		counter = container_of(res, struct rdma_counter, res);</div><div class='add'>+		ret = xa_insert(&amp;rt-&gt;xa, counter-&gt;id, res, GFP_KERNEL);</div><div class='add'>+		res-&gt;id = ret ? 0 : counter-&gt;id;</div><div class='add'>+	} else {</div><div class='add'>+		ret = xa_alloc_cyclic(&amp;rt-&gt;xa, &amp;res-&gt;id, res, xa_limit_32b,</div><div class='add'>+				      &amp;rt-&gt;next_id, GFP_KERNEL);</div><div class='add'>+		ret = (ret &lt; 0) ? ret : 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	if (!ret)</div><div class='add'>+		res-&gt;valid = true;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(rdma_restrack_uadd);</div><div class='add'>+EXPORT_SYMBOL(rdma_restrack_add);</div><div class='ctx'> </div><div class='ctx'> int __must_check rdma_restrack_get(struct rdma_restrack_entry *res)</div><div class='ctx'> {</div><div class='hunk'>@@ -285,6 +263,10 @@ static void restrack_release(struct kref *kref)</div><div class='ctx'> 	struct rdma_restrack_entry *res;</div><div class='ctx'> </div><div class='ctx'> 	res = container_of(kref, struct rdma_restrack_entry, kref);</div><div class='add'>+	if (res-&gt;task) {</div><div class='add'>+		put_task_struct(res-&gt;task);</div><div class='add'>+		res-&gt;task = NULL;</div><div class='add'>+	}</div><div class='ctx'> 	complete(&amp;res-&gt;comp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -294,13 +276,25 @@ int rdma_restrack_put(struct rdma_restrack_entry *res)</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_restrack_put);</div><div class='ctx'> </div><div class='add'>+/**</div><div class='add'>+ * rdma_restrack_del() - delete object from the resource tracking database</div><div class='add'>+ * @res:  resource entry</div><div class='add'>+ */</div><div class='ctx'> void rdma_restrack_del(struct rdma_restrack_entry *res)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_restrack_entry *old;</div><div class='ctx'> 	struct rdma_restrack_root *rt;</div><div class='ctx'> 	struct ib_device *dev;</div><div class='ctx'> </div><div class='del'>-	if (!res-&gt;valid)</div><div class='add'>+	if (!res-&gt;valid) {</div><div class='add'>+		if (res-&gt;task) {</div><div class='add'>+			put_task_struct(res-&gt;task);</div><div class='add'>+			res-&gt;task = NULL;</div><div class='add'>+		}</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (res-&gt;no_track)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='ctx'> 	dev = res_to_dev(res);</div><div class='hunk'>@@ -311,15 +305,10 @@ void rdma_restrack_del(struct rdma_restrack_entry *res)</div><div class='ctx'> </div><div class='ctx'> 	old = xa_erase(&amp;rt-&gt;xa, res-&gt;id);</div><div class='ctx'> 	WARN_ON(old != res);</div><div class='del'>-	res-&gt;valid = false;</div><div class='ctx'> </div><div class='add'>+out:</div><div class='add'>+	res-&gt;valid = false;</div><div class='ctx'> 	rdma_restrack_put(res);</div><div class='ctx'> 	wait_for_completion(&amp;res-&gt;comp);</div><div class='del'>-</div><div class='del'>-out:</div><div class='del'>-	if (res-&gt;task) {</div><div class='del'>-		put_task_struct(res-&gt;task);</div><div class='del'>-		res-&gt;task = NULL;</div><div class='del'>-	}</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_restrack_del);</div><div class='head'>diff --git a/drivers/infiniband/core/restrack.h b/drivers/infiniband/core/restrack.h<br/>index 09a1fbdf578e..6a04fc41f738 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/restrack.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/restrack.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/restrack.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/restrack.h</a></div><div class='hunk'>@@ -25,4 +25,12 @@ struct rdma_restrack_root {</div><div class='ctx'> </div><div class='ctx'> int rdma_restrack_init(struct ib_device *dev);</div><div class='ctx'> void rdma_restrack_clean(struct ib_device *dev);</div><div class='add'>+void rdma_restrack_add(struct rdma_restrack_entry *res);</div><div class='add'>+void rdma_restrack_del(struct rdma_restrack_entry *res);</div><div class='add'>+void rdma_restrack_new(struct rdma_restrack_entry *res,</div><div class='add'>+		       enum rdma_restrack_type type);</div><div class='add'>+void rdma_restrack_set_name(struct rdma_restrack_entry *res,</div><div class='add'>+			    const char *caller);</div><div class='add'>+void rdma_restrack_parent_name(struct rdma_restrack_entry *dst,</div><div class='add'>+			       const struct rdma_restrack_entry *parent);</div><div class='ctx'> #endif /* _RDMA_CORE_RESTRACK_H_ */</div><div class='head'>diff --git a/drivers/infiniband/core/roce_gid_mgmt.c b/drivers/infiniband/core/roce_gid_mgmt.c<br/>index 558de0b9895c..a9f2c6b1b29e 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/roce_gid_mgmt.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/roce_gid_mgmt.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/roce_gid_mgmt.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/roce_gid_mgmt.c</a></div><div class='hunk'>@@ -70,7 +70,7 @@ struct netdev_event_work {</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static const struct {</div><div class='del'>-	bool (*is_supported)(const struct ib_device *device, u8 port_num);</div><div class='add'>+	bool (*is_supported)(const struct ib_device *device, u32 port_num);</div><div class='ctx'> 	enum ib_gid_type gid_type;</div><div class='ctx'> } PORT_CAP_TO_GID_TYPE[] = {</div><div class='ctx'> 	{rdma_protocol_roce_eth_encap, IB_GID_TYPE_ROCE},</div><div class='hunk'>@@ -79,7 +79,7 @@ static const struct {</div><div class='ctx'> </div><div class='ctx'> #define CAP_TO_GID_TABLE_SIZE	ARRAY_SIZE(PORT_CAP_TO_GID_TYPE)</div><div class='ctx'> </div><div class='del'>-unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u8 port)</div><div class='add'>+unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u32 port)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='ctx'> 	unsigned int ret_flags = 0;</div><div class='hunk'>@@ -96,7 +96,7 @@ unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u8 port)</div><div class='ctx'> EXPORT_SYMBOL(roce_gid_type_mask_support);</div><div class='ctx'> </div><div class='ctx'> static void update_gid(enum gid_op_type gid_op, struct ib_device *ib_dev,</div><div class='del'>-		       u8 port, union ib_gid *gid,</div><div class='add'>+		       u32 port, union ib_gid *gid,</div><div class='ctx'> 		       struct ib_gid_attr *gid_attr)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='hunk'>@@ -144,7 +144,7 @@ static enum bonding_slave_state is_eth_active_slave_of_bonding_rcu(struct net_de</div><div class='ctx'> #define REQUIRED_BOND_STATES		(BONDING_SLAVE_STATE_ACTIVE |	\</div><div class='ctx'> 					 BONDING_SLAVE_STATE_NA)</div><div class='ctx'> static bool</div><div class='del'>-is_eth_port_of_netdev_filter(struct ib_device *ib_dev, u8 port,</div><div class='add'>+is_eth_port_of_netdev_filter(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 			     struct net_device *rdma_ndev, void *cookie)</div><div class='ctx'> {</div><div class='ctx'> 	struct net_device *real_dev;</div><div class='hunk'>@@ -168,7 +168,7 @@ is_eth_port_of_netdev_filter(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static bool</div><div class='del'>-is_eth_port_inactive_slave_filter(struct ib_device *ib_dev, u8 port,</div><div class='add'>+is_eth_port_inactive_slave_filter(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 				  struct net_device *rdma_ndev, void *cookie)</div><div class='ctx'> {</div><div class='ctx'> 	struct net_device *master_dev;</div><div class='hunk'>@@ -186,18 +186,19 @@ is_eth_port_inactive_slave_filter(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> 	return res;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/** is_ndev_for_default_gid_filter - Check if a given netdevice</div><div class='add'>+/**</div><div class='add'>+ * is_ndev_for_default_gid_filter - Check if a given netdevice</div><div class='ctx'>  * can be considered for default GIDs or not.</div><div class='ctx'>  * @ib_dev:		IB device to check</div><div class='ctx'>  * @port:		Port to consider for adding default GID</div><div class='ctx'>  * @rdma_ndev:		rdma netdevice pointer</div><div class='del'>- * @cookie_ndev:	Netdevice to consider to form a default GID</div><div class='add'>+ * @cookie:             Netdevice to consider to form a default GID</div><div class='ctx'>  *</div><div class='ctx'>  * is_ndev_for_default_gid_filter() returns true if a given netdevice can be</div><div class='ctx'>  * considered for deriving default RoCE GID, returns false otherwise.</div><div class='ctx'>  */</div><div class='ctx'> static bool</div><div class='del'>-is_ndev_for_default_gid_filter(struct ib_device *ib_dev, u8 port,</div><div class='add'>+is_ndev_for_default_gid_filter(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 			       struct net_device *rdma_ndev, void *cookie)</div><div class='ctx'> {</div><div class='ctx'> 	struct net_device *cookie_ndev = cookie;</div><div class='hunk'>@@ -223,13 +224,13 @@ is_ndev_for_default_gid_filter(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> 	return res;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static bool pass_all_filter(struct ib_device *ib_dev, u8 port,</div><div class='add'>+static bool pass_all_filter(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 			    struct net_device *rdma_ndev, void *cookie)</div><div class='ctx'> {</div><div class='ctx'> 	return true;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static bool upper_device_filter(struct ib_device *ib_dev, u8 port,</div><div class='add'>+static bool upper_device_filter(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 				struct net_device *rdma_ndev, void *cookie)</div><div class='ctx'> {</div><div class='ctx'> 	bool res;</div><div class='hunk'>@@ -249,7 +250,7 @@ static bool upper_device_filter(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * is_upper_ndev_bond_master_filter - Check if a given netdevice</div><div class='del'>- * is bond master device of netdevice of the the RDMA device of port.</div><div class='add'>+ * is bond master device of netdevice of the RDMA device of port.</div><div class='ctx'>  * @ib_dev:		IB device to check</div><div class='ctx'>  * @port:		Port to consider for adding default GID</div><div class='ctx'>  * @rdma_ndev:		Pointer to rdma netdevice</div><div class='hunk'>@@ -260,7 +261,7 @@ static bool upper_device_filter(struct ib_device *ib_dev, u8 port,</div><div class='ctx'>  * not have been established as slave device yet.</div><div class='ctx'>  */</div><div class='ctx'> static bool</div><div class='del'>-is_upper_ndev_bond_master_filter(struct ib_device *ib_dev, u8 port,</div><div class='add'>+is_upper_ndev_bond_master_filter(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 				 struct net_device *rdma_ndev,</div><div class='ctx'> 				 void *cookie)</div><div class='ctx'> {</div><div class='hunk'>@@ -280,7 +281,7 @@ is_upper_ndev_bond_master_filter(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> </div><div class='ctx'> static void update_gid_ip(enum gid_op_type gid_op,</div><div class='ctx'> 			  struct ib_device *ib_dev,</div><div class='del'>-			  u8 port, struct net_device *ndev,</div><div class='add'>+			  u32 port, struct net_device *ndev,</div><div class='ctx'> 			  struct sockaddr *addr)</div><div class='ctx'> {</div><div class='ctx'> 	union ib_gid gid;</div><div class='hunk'>@@ -294,7 +295,7 @@ static void update_gid_ip(enum gid_op_type gid_op,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void bond_delete_netdev_default_gids(struct ib_device *ib_dev,</div><div class='del'>-					    u8 port,</div><div class='add'>+					    u32 port,</div><div class='ctx'> 					    struct net_device *rdma_ndev,</div><div class='ctx'> 					    struct net_device *event_ndev)</div><div class='ctx'> {</div><div class='hunk'>@@ -328,8 +329,9 @@ static void bond_delete_netdev_default_gids(struct ib_device *ib_dev,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void enum_netdev_ipv4_ips(struct ib_device *ib_dev,</div><div class='del'>-				 u8 port, struct net_device *ndev)</div><div class='add'>+				 u32 port, struct net_device *ndev)</div><div class='ctx'> {</div><div class='add'>+	const struct in_ifaddr *ifa;</div><div class='ctx'> 	struct in_device *in_dev;</div><div class='ctx'> 	struct sin_list {</div><div class='ctx'> 		struct list_head	list;</div><div class='hunk'>@@ -349,7 +351,7 @@ static void enum_netdev_ipv4_ips(struct ib_device *ib_dev,</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	for_ifa(in_dev) {</div><div class='add'>+	in_dev_for_each_ifa_rcu(ifa, in_dev) {</div><div class='ctx'> 		struct sin_list *entry = kzalloc(sizeof(*entry), GFP_ATOMIC);</div><div class='ctx'> </div><div class='ctx'> 		if (!entry)</div><div class='hunk'>@@ -359,7 +361,7 @@ static void enum_netdev_ipv4_ips(struct ib_device *ib_dev,</div><div class='ctx'> 		entry-&gt;ip.sin_addr.s_addr = ifa-&gt;ifa_address;</div><div class='ctx'> 		list_add_tail(&amp;entry-&gt;list, &amp;sin_list);</div><div class='ctx'> 	}</div><div class='del'>-	endfor_ifa(in_dev);</div><div class='add'>+</div><div class='ctx'> 	rcu_read_unlock();</div><div class='ctx'> </div><div class='ctx'> 	list_for_each_entry_safe(sin_iter, sin_temp, &amp;sin_list, list) {</div><div class='hunk'>@@ -371,7 +373,7 @@ static void enum_netdev_ipv4_ips(struct ib_device *ib_dev,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void enum_netdev_ipv6_ips(struct ib_device *ib_dev,</div><div class='del'>-				 u8 port, struct net_device *ndev)</div><div class='add'>+				 u32 port, struct net_device *ndev)</div><div class='ctx'> {</div><div class='ctx'> 	struct inet6_ifaddr *ifp;</div><div class='ctx'> 	struct inet6_dev *in6_dev;</div><div class='hunk'>@@ -416,7 +418,7 @@ static void enum_netdev_ipv6_ips(struct ib_device *ib_dev,</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void _add_netdev_ips(struct ib_device *ib_dev, u8 port,</div><div class='add'>+static void _add_netdev_ips(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 			    struct net_device *ndev)</div><div class='ctx'> {</div><div class='ctx'> 	enum_netdev_ipv4_ips(ib_dev, port, ndev);</div><div class='hunk'>@@ -424,13 +426,13 @@ static void _add_netdev_ips(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> 		enum_netdev_ipv6_ips(ib_dev, port, ndev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void add_netdev_ips(struct ib_device *ib_dev, u8 port,</div><div class='add'>+static void add_netdev_ips(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 			   struct net_device *rdma_ndev, void *cookie)</div><div class='ctx'> {</div><div class='ctx'> 	_add_netdev_ips(ib_dev, port, cookie);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void del_netdev_ips(struct ib_device *ib_dev, u8 port,</div><div class='add'>+static void del_netdev_ips(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 			   struct net_device *rdma_ndev, void *cookie)</div><div class='ctx'> {</div><div class='ctx'> 	ib_cache_gid_del_all_netdev_gids(ib_dev, port, cookie);</div><div class='hunk'>@@ -445,7 +447,7 @@ static void del_netdev_ips(struct ib_device *ib_dev, u8 port,</div><div class='ctx'>  *</div><div class='ctx'>  * del_default_gids() deletes the default GIDs of the event/cookie netdevice.</div><div class='ctx'>  */</div><div class='del'>-static void del_default_gids(struct ib_device *ib_dev, u8 port,</div><div class='add'>+static void del_default_gids(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 			     struct net_device *rdma_ndev, void *cookie)</div><div class='ctx'> {</div><div class='ctx'> 	struct net_device *cookie_ndev = cookie;</div><div class='hunk'>@@ -457,7 +459,7 @@ static void del_default_gids(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> 				     IB_CACHE_GID_DEFAULT_MODE_DELETE);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void add_default_gids(struct ib_device *ib_dev, u8 port,</div><div class='add'>+static void add_default_gids(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 			     struct net_device *rdma_ndev, void *cookie)</div><div class='ctx'> {</div><div class='ctx'> 	struct net_device *event_ndev = cookie;</div><div class='hunk'>@@ -469,7 +471,7 @@ static void add_default_gids(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void enum_all_gids_of_dev_cb(struct ib_device *ib_dev,</div><div class='del'>-				    u8 port,</div><div class='add'>+				    u32 port,</div><div class='ctx'> 				    struct net_device *rdma_ndev,</div><div class='ctx'> 				    void *cookie)</div><div class='ctx'> {</div><div class='hunk'>@@ -504,7 +506,7 @@ static void enum_all_gids_of_dev_cb(struct ib_device *ib_dev,</div><div class='ctx'>  * rdma_roce_rescan_device - Rescan all of the network devices in the system</div><div class='ctx'>  * and add their gids, as needed, to the relevant RoCE devices.</div><div class='ctx'>  *</div><div class='del'>- * @device:         the rdma device</div><div class='add'>+ * @ib_dev:         the rdma device</div><div class='ctx'>  */</div><div class='ctx'> void rdma_roce_rescan_device(struct ib_device *ib_dev)</div><div class='ctx'> {</div><div class='hunk'>@@ -513,8 +515,29 @@ void rdma_roce_rescan_device(struct ib_device *ib_dev)</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_roce_rescan_device);</div><div class='ctx'> </div><div class='add'>+/**</div><div class='add'>+ * rdma_roce_rescan_port - Rescan all of the network devices in the system</div><div class='add'>+ * and add their gids if relevant to the port of the RoCE device.</div><div class='add'>+ *</div><div class='add'>+ * @ib_dev: IB device</div><div class='add'>+ * @port: Port number</div><div class='add'>+ */</div><div class='add'>+void rdma_roce_rescan_port(struct ib_device *ib_dev, u32 port)</div><div class='add'>+{</div><div class='add'>+	struct net_device *ndev = NULL;</div><div class='add'>+</div><div class='add'>+	if (rdma_protocol_roce(ib_dev, port)) {</div><div class='add'>+		ndev = ib_device_get_netdev(ib_dev, port);</div><div class='add'>+		if (!ndev)</div><div class='add'>+			return;</div><div class='add'>+		enum_all_gids_of_dev_cb(ib_dev, port, ndev, ndev);</div><div class='add'>+		dev_put(ndev);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_roce_rescan_port);</div><div class='add'>+</div><div class='ctx'> static void callback_for_addr_gid_device_scan(struct ib_device *device,</div><div class='del'>-					      u8 port,</div><div class='add'>+					      u32 port,</div><div class='ctx'> 					      struct net_device *rdma_ndev,</div><div class='ctx'> 					      void *cookie)</div><div class='ctx'> {</div><div class='hunk'>@@ -530,10 +553,11 @@ struct upper_list {</div><div class='ctx'> 	struct net_device *upper;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static int netdev_upper_walk(struct net_device *upper, void *data)</div><div class='add'>+static int netdev_upper_walk(struct net_device *upper,</div><div class='add'>+			     struct netdev_nested_priv *priv)</div><div class='ctx'> {</div><div class='ctx'> 	struct upper_list *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);</div><div class='del'>-	struct list_head *upper_list = data;</div><div class='add'>+	struct list_head *upper_list = (struct list_head *)priv-&gt;data;</div><div class='ctx'> </div><div class='ctx'> 	if (!entry)</div><div class='ctx'> 		return 0;</div><div class='hunk'>@@ -545,19 +569,21 @@ static int netdev_upper_walk(struct net_device *upper, void *data)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void handle_netdev_upper(struct ib_device *ib_dev, u8 port,</div><div class='add'>+static void handle_netdev_upper(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 				void *cookie,</div><div class='ctx'> 				void (*handle_netdev)(struct ib_device *ib_dev,</div><div class='del'>-						      u8 port,</div><div class='add'>+						      u32 port,</div><div class='ctx'> 						      struct net_device *ndev))</div><div class='ctx'> {</div><div class='ctx'> 	struct net_device *ndev = cookie;</div><div class='add'>+	struct netdev_nested_priv priv;</div><div class='ctx'> 	struct upper_list *upper_iter;</div><div class='ctx'> 	struct upper_list *upper_temp;</div><div class='ctx'> 	LIST_HEAD(upper_list);</div><div class='ctx'> </div><div class='add'>+	priv.data = &amp;upper_list;</div><div class='ctx'> 	rcu_read_lock();</div><div class='del'>-	netdev_walk_all_upper_dev_rcu(ndev, netdev_upper_walk, &amp;upper_list);</div><div class='add'>+	netdev_walk_all_upper_dev_rcu(ndev, netdev_upper_walk, &amp;priv);</div><div class='ctx'> 	rcu_read_unlock();</div><div class='ctx'> </div><div class='ctx'> 	handle_netdev(ib_dev, port, ndev);</div><div class='hunk'>@@ -570,25 +596,26 @@ static void handle_netdev_upper(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void _roce_del_all_netdev_gids(struct ib_device *ib_dev, u8 port,</div><div class='del'>-				      struct net_device *event_ndev)</div><div class='add'>+void roce_del_all_netdev_gids(struct ib_device *ib_dev,</div><div class='add'>+			      u32 port, struct net_device *ndev)</div><div class='ctx'> {</div><div class='del'>-	ib_cache_gid_del_all_netdev_gids(ib_dev, port, event_ndev);</div><div class='add'>+	ib_cache_gid_del_all_netdev_gids(ib_dev, port, ndev);</div><div class='ctx'> }</div><div class='add'>+EXPORT_SYMBOL(roce_del_all_netdev_gids);</div><div class='ctx'> </div><div class='del'>-static void del_netdev_upper_ips(struct ib_device *ib_dev, u8 port,</div><div class='add'>+static void del_netdev_upper_ips(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 				 struct net_device *rdma_ndev, void *cookie)</div><div class='ctx'> {</div><div class='del'>-	handle_netdev_upper(ib_dev, port, cookie, _roce_del_all_netdev_gids);</div><div class='add'>+	handle_netdev_upper(ib_dev, port, cookie, roce_del_all_netdev_gids);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void add_netdev_upper_ips(struct ib_device *ib_dev, u8 port,</div><div class='add'>+static void add_netdev_upper_ips(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 				 struct net_device *rdma_ndev, void *cookie)</div><div class='ctx'> {</div><div class='ctx'> 	handle_netdev_upper(ib_dev, port, cookie, _add_netdev_ips);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void del_netdev_default_ips_join(struct ib_device *ib_dev, u8 port,</div><div class='add'>+static void del_netdev_default_ips_join(struct ib_device *ib_dev, u32 port,</div><div class='ctx'> 					struct net_device *rdma_ndev,</div><div class='ctx'> 					void *cookie)</div><div class='ctx'> {</div><div class='hunk'>@@ -596,8 +623,7 @@ static void del_netdev_default_ips_join(struct ib_device *ib_dev, u8 port,</div><div class='ctx'> </div><div class='ctx'> 	rcu_read_lock();</div><div class='ctx'> 	master_ndev = netdev_master_upper_dev_get_rcu(rdma_ndev);</div><div class='del'>-	if (master_ndev)</div><div class='del'>-		dev_hold(master_ndev);</div><div class='add'>+	dev_hold(master_ndev);</div><div class='ctx'> 	rcu_read_unlock();</div><div class='ctx'> </div><div class='ctx'> 	if (master_ndev) {</div><div class='head'>diff --git a/drivers/infiniband/core/rw.c b/drivers/infiniband/core/rw.c<br/>index 89a5be3a2f97..6354ddf2a274 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/rw.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/rw.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/rw.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/rw.c</a></div><div class='hunk'>@@ -1,15 +1,8 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0-only</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright (c) 2016 HGST, a Western Digital Company.</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify it</div><div class='del'>- * under the terms and conditions of the GNU General Public License,</div><div class='del'>- * version 2, as published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope it will be useful, but WITHOUT</div><div class='del'>- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or</div><div class='del'>- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for</div><div class='del'>- * more details.</div><div class='ctx'>  */</div><div class='add'>+#include &lt;linux/memremap.h&gt;</div><div class='ctx'> #include &lt;linux/moduleparam.h&gt;</div><div class='ctx'> #include &lt;linux/slab.h&gt;</div><div class='ctx'> #include &lt;linux/pci-p2pdma.h&gt;</div><div class='hunk'>@@ -28,14 +21,17 @@ module_param_named(force_mr, rdma_rw_force_mr, bool, 0);</div><div class='ctx'> MODULE_PARM_DESC(force_mr, "Force usage of MRs for RDMA READ/WRITE operations");</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='del'>- * Check if the device might use memory registration.  This is currently only</div><div class='del'>- * true for iWarp devices. In the future we can hopefully fine tune this based</div><div class='del'>- * on HCA driver input.</div><div class='add'>+ * Report whether memory registration should be used. Memory registration must</div><div class='add'>+ * be used for iWarp devices because of iWARP-specific limitations. Memory</div><div class='add'>+ * registration is also enabled if registering memory might yield better</div><div class='add'>+ * performance than using multiple SGE entries, see rdma_rw_io_needs_mr()</div><div class='ctx'>  */</div><div class='del'>-static inline bool rdma_rw_can_use_mr(struct ib_device *dev, u8 port_num)</div><div class='add'>+static inline bool rdma_rw_can_use_mr(struct ib_device *dev, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	if (rdma_protocol_iwarp(dev, port_num))</div><div class='ctx'> 		return true;</div><div class='add'>+	if (dev-&gt;attrs.max_sgl_rd)</div><div class='add'>+		return true;</div><div class='ctx'> 	if (unlikely(rdma_rw_force_mr))</div><div class='ctx'> 		return true;</div><div class='ctx'> 	return false;</div><div class='hunk'>@@ -43,34 +39,61 @@ static inline bool rdma_rw_can_use_mr(struct ib_device *dev, u8 port_num)</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * Check if the device will use memory registration for this RW operation.</div><div class='del'>- * We currently always use memory registrations for iWarp RDMA READs, and</div><div class='del'>- * have a debug option to force usage of MRs.</div><div class='del'>- *</div><div class='del'>- * XXX: In the future we can hopefully fine tune this based on HCA driver</div><div class='del'>- * input.</div><div class='add'>+ * For RDMA READs we must use MRs on iWarp and can optionally use them as an</div><div class='add'>+ * optimization otherwise.  Additionally we have a debug option to force usage</div><div class='add'>+ * of MRs to help testing this code path.</div><div class='ctx'>  */</div><div class='del'>-static inline bool rdma_rw_io_needs_mr(struct ib_device *dev, u8 port_num,</div><div class='add'>+static inline bool rdma_rw_io_needs_mr(struct ib_device *dev, u32 port_num,</div><div class='ctx'> 		enum dma_data_direction dir, int dma_nents)</div><div class='ctx'> {</div><div class='del'>-	if (rdma_protocol_iwarp(dev, port_num) &amp;&amp; dir == DMA_FROM_DEVICE)</div><div class='del'>-		return true;</div><div class='add'>+	if (dir == DMA_FROM_DEVICE) {</div><div class='add'>+		if (rdma_protocol_iwarp(dev, port_num))</div><div class='add'>+			return true;</div><div class='add'>+		if (dev-&gt;attrs.max_sgl_rd &amp;&amp; dma_nents &gt; dev-&gt;attrs.max_sgl_rd)</div><div class='add'>+			return true;</div><div class='add'>+	}</div><div class='ctx'> 	if (unlikely(rdma_rw_force_mr))</div><div class='ctx'> 		return true;</div><div class='ctx'> 	return false;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline u32 rdma_rw_fr_page_list_len(struct ib_device *dev)</div><div class='add'>+static inline u32 rdma_rw_fr_page_list_len(struct ib_device *dev,</div><div class='add'>+					   bool pi_support)</div><div class='ctx'> {</div><div class='add'>+	u32 max_pages;</div><div class='add'>+</div><div class='add'>+	if (pi_support)</div><div class='add'>+		max_pages = dev-&gt;attrs.max_pi_fast_reg_page_list_len;</div><div class='add'>+	else</div><div class='add'>+		max_pages = dev-&gt;attrs.max_fast_reg_page_list_len;</div><div class='add'>+</div><div class='ctx'> 	/* arbitrary limit to avoid allocating gigantic resources */</div><div class='del'>-	return min_t(u32, dev-&gt;attrs.max_fast_reg_page_list_len, 256);</div><div class='add'>+	return min_t(u32, max_pages, 256);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int rdma_rw_inv_key(struct rdma_rw_reg_ctx *reg)</div><div class='add'>+{</div><div class='add'>+	int count = 0;</div><div class='add'>+</div><div class='add'>+	if (reg-&gt;mr-&gt;need_inval) {</div><div class='add'>+		reg-&gt;inv_wr.opcode = IB_WR_LOCAL_INV;</div><div class='add'>+		reg-&gt;inv_wr.ex.invalidate_rkey = reg-&gt;mr-&gt;lkey;</div><div class='add'>+		reg-&gt;inv_wr.next = &amp;reg-&gt;reg_wr.wr;</div><div class='add'>+		count++;</div><div class='add'>+	} else {</div><div class='add'>+		reg-&gt;inv_wr.next = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return count;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* Caller must have zero-initialized *reg. */</div><div class='del'>-static int rdma_rw_init_one_mr(struct ib_qp *qp, u8 port_num,</div><div class='add'>+static int rdma_rw_init_one_mr(struct ib_qp *qp, u32 port_num,</div><div class='ctx'> 		struct rdma_rw_reg_ctx *reg, struct scatterlist *sg,</div><div class='ctx'> 		u32 sg_cnt, u32 offset)</div><div class='ctx'> {</div><div class='del'>-	u32 pages_per_mr = rdma_rw_fr_page_list_len(qp-&gt;pd-&gt;device);</div><div class='add'>+	u32 pages_per_mr = rdma_rw_fr_page_list_len(qp-&gt;pd-&gt;device,</div><div class='add'>+						    qp-&gt;integrity_en);</div><div class='ctx'> 	u32 nents = min(sg_cnt, pages_per_mr);</div><div class='ctx'> 	int count = 0, ret;</div><div class='ctx'> </div><div class='hunk'>@@ -78,14 +101,7 @@ static int rdma_rw_init_one_mr(struct ib_qp *qp, u8 port_num,</div><div class='ctx'> 	if (!reg-&gt;mr)</div><div class='ctx'> 		return -EAGAIN;</div><div class='ctx'> </div><div class='del'>-	if (reg-&gt;mr-&gt;need_inval) {</div><div class='del'>-		reg-&gt;inv_wr.opcode = IB_WR_LOCAL_INV;</div><div class='del'>-		reg-&gt;inv_wr.ex.invalidate_rkey = reg-&gt;mr-&gt;lkey;</div><div class='del'>-		reg-&gt;inv_wr.next = &amp;reg-&gt;reg_wr.wr;</div><div class='del'>-		count++;</div><div class='del'>-	} else {</div><div class='del'>-		reg-&gt;inv_wr.next = NULL;</div><div class='del'>-	}</div><div class='add'>+	count += rdma_rw_inv_key(reg);</div><div class='ctx'> </div><div class='ctx'> 	ret = ib_map_mr_sg(reg-&gt;mr, sg, nents, &amp;offset, PAGE_SIZE);</div><div class='ctx'> 	if (ret &lt; 0 || ret &lt; nents) {</div><div class='hunk'>@@ -106,14 +122,15 @@ static int rdma_rw_init_one_mr(struct ib_qp *qp, u8 port_num,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int rdma_rw_init_mr_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp,</div><div class='del'>-		u8 port_num, struct scatterlist *sg, u32 sg_cnt, u32 offset,</div><div class='add'>+		u32 port_num, struct scatterlist *sg, u32 sg_cnt, u32 offset,</div><div class='ctx'> 		u64 remote_addr, u32 rkey, enum dma_data_direction dir)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_rw_reg_ctx *prev = NULL;</div><div class='del'>-	u32 pages_per_mr = rdma_rw_fr_page_list_len(qp-&gt;pd-&gt;device);</div><div class='add'>+	u32 pages_per_mr = rdma_rw_fr_page_list_len(qp-&gt;pd-&gt;device,</div><div class='add'>+						    qp-&gt;integrity_en);</div><div class='ctx'> 	int i, j, ret = 0, count = 0;</div><div class='ctx'> </div><div class='del'>-	ctx-&gt;nr_ops = (sg_cnt + pages_per_mr - 1) / pages_per_mr;</div><div class='add'>+	ctx-&gt;nr_ops = DIV_ROUND_UP(sg_cnt, pages_per_mr);</div><div class='ctx'> 	ctx-&gt;reg = kcalloc(ctx-&gt;nr_ops, sizeof(*ctx-&gt;reg), GFP_KERNEL);</div><div class='ctx'> 	if (!ctx-&gt;reg) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='hunk'>@@ -272,21 +289,21 @@ static int rdma_rw_init_single_wr(struct rdma_rw_ctx *ctx, struct ib_qp *qp,</div><div class='ctx'>  * Returns the number of WQEs that will be needed on the workqueue if</div><div class='ctx'>  * successful, or a negative error code.</div><div class='ctx'>  */</div><div class='del'>-int rdma_rw_ctx_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num,</div><div class='add'>+int rdma_rw_ctx_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u32 port_num,</div><div class='ctx'> 		struct scatterlist *sg, u32 sg_cnt, u32 sg_offset,</div><div class='ctx'> 		u64 remote_addr, u32 rkey, enum dma_data_direction dir)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *dev = qp-&gt;pd-&gt;device;</div><div class='add'>+	struct sg_table sgt = {</div><div class='add'>+		.sgl = sg,</div><div class='add'>+		.orig_nents = sg_cnt,</div><div class='add'>+	};</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	if (is_pci_p2pdma_page(sg_page(sg)))</div><div class='del'>-		ret = pci_p2pdma_map_sg(dev-&gt;dma_device, sg, sg_cnt, dir);</div><div class='del'>-	else</div><div class='del'>-		ret = ib_dma_map_sg(dev, sg, sg_cnt, dir);</div><div class='del'>-</div><div class='del'>-	if (!ret)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	sg_cnt = ret;</div><div class='add'>+	ret = ib_dma_map_sgtable_attrs(dev, &amp;sgt, dir, 0);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+	sg_cnt = sgt.nents;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Skip to the S/G entry that sg_offset falls into:</div><div class='hunk'>@@ -322,7 +339,7 @@ int rdma_rw_ctx_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num,</div><div class='ctx'> 	return ret;</div><div class='ctx'> </div><div class='ctx'> out_unmap_sg:</div><div class='del'>-	ib_dma_unmap_sg(dev, sg, sg_cnt, dir);</div><div class='add'>+	ib_dma_unmap_sgtable_attrs(dev, &amp;sgt, dir, 0);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_rw_ctx_init);</div><div class='hunk'>@@ -345,96 +362,85 @@ EXPORT_SYMBOL(rdma_rw_ctx_init);</div><div class='ctx'>  * successful, or a negative error code.</div><div class='ctx'>  */</div><div class='ctx'> int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp,</div><div class='del'>-		u8 port_num, struct scatterlist *sg, u32 sg_cnt,</div><div class='add'>+		u32 port_num, struct scatterlist *sg, u32 sg_cnt,</div><div class='ctx'> 		struct scatterlist *prot_sg, u32 prot_sg_cnt,</div><div class='ctx'> 		struct ib_sig_attrs *sig_attrs,</div><div class='ctx'> 		u64 remote_addr, u32 rkey, enum dma_data_direction dir)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *dev = qp-&gt;pd-&gt;device;</div><div class='del'>-	u32 pages_per_mr = rdma_rw_fr_page_list_len(qp-&gt;pd-&gt;device);</div><div class='add'>+	u32 pages_per_mr = rdma_rw_fr_page_list_len(qp-&gt;pd-&gt;device,</div><div class='add'>+						    qp-&gt;integrity_en);</div><div class='add'>+	struct sg_table sgt = {</div><div class='add'>+		.sgl = sg,</div><div class='add'>+		.orig_nents = sg_cnt,</div><div class='add'>+	};</div><div class='add'>+	struct sg_table prot_sgt = {</div><div class='add'>+		.sgl = prot_sg,</div><div class='add'>+		.orig_nents = prot_sg_cnt,</div><div class='add'>+	};</div><div class='ctx'> 	struct ib_rdma_wr *rdma_wr;</div><div class='del'>-	struct ib_send_wr *prev_wr = NULL;</div><div class='ctx'> 	int count = 0, ret;</div><div class='ctx'> </div><div class='ctx'> 	if (sg_cnt &gt; pages_per_mr || prot_sg_cnt &gt; pages_per_mr) {</div><div class='del'>-		pr_err("SG count too large\n");</div><div class='add'>+		pr_err("SG count too large: sg_cnt=%u, prot_sg_cnt=%u, pages_per_mr=%u\n",</div><div class='add'>+		       sg_cnt, prot_sg_cnt, pages_per_mr);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = ib_dma_map_sg(dev, sg, sg_cnt, dir);</div><div class='del'>-	if (!ret)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	sg_cnt = ret;</div><div class='add'>+	ret = ib_dma_map_sgtable_attrs(dev, &amp;sgt, dir, 0);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_dma_map_sg(dev, prot_sg, prot_sg_cnt, dir);</div><div class='del'>-	if (!ret) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto out_unmap_sg;</div><div class='add'>+	if (prot_sg_cnt) {</div><div class='add'>+		ret = ib_dma_map_sgtable_attrs(dev, &amp;prot_sgt, dir, 0);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto out_unmap_sg;</div><div class='ctx'> 	}</div><div class='del'>-	prot_sg_cnt = ret;</div><div class='ctx'> </div><div class='ctx'> 	ctx-&gt;type = RDMA_RW_SIG_MR;</div><div class='ctx'> 	ctx-&gt;nr_ops = 1;</div><div class='del'>-	ctx-&gt;sig = kcalloc(1, sizeof(*ctx-&gt;sig), GFP_KERNEL);</div><div class='del'>-	if (!ctx-&gt;sig) {</div><div class='add'>+	ctx-&gt;reg = kzalloc(sizeof(*ctx-&gt;reg), GFP_KERNEL);</div><div class='add'>+	if (!ctx-&gt;reg) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='ctx'> 		goto out_unmap_prot_sg;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = rdma_rw_init_one_mr(qp, port_num, &amp;ctx-&gt;sig-&gt;data, sg, sg_cnt, 0);</div><div class='del'>-	if (ret &lt; 0)</div><div class='del'>-		goto out_free_ctx;</div><div class='del'>-	count += ret;</div><div class='del'>-	prev_wr = &amp;ctx-&gt;sig-&gt;data.reg_wr.wr;</div><div class='del'>-</div><div class='del'>-	ret = rdma_rw_init_one_mr(qp, port_num, &amp;ctx-&gt;sig-&gt;prot,</div><div class='del'>-				  prot_sg, prot_sg_cnt, 0);</div><div class='del'>-	if (ret &lt; 0)</div><div class='del'>-		goto out_destroy_data_mr;</div><div class='del'>-	count += ret;</div><div class='del'>-</div><div class='del'>-	if (ctx-&gt;sig-&gt;prot.inv_wr.next)</div><div class='del'>-		prev_wr-&gt;next = &amp;ctx-&gt;sig-&gt;prot.inv_wr;</div><div class='del'>-	else</div><div class='del'>-		prev_wr-&gt;next = &amp;ctx-&gt;sig-&gt;prot.reg_wr.wr;</div><div class='del'>-	prev_wr = &amp;ctx-&gt;sig-&gt;prot.reg_wr.wr;</div><div class='del'>-</div><div class='del'>-	ctx-&gt;sig-&gt;sig_mr = ib_mr_pool_get(qp, &amp;qp-&gt;sig_mrs);</div><div class='del'>-	if (!ctx-&gt;sig-&gt;sig_mr) {</div><div class='add'>+	ctx-&gt;reg-&gt;mr = ib_mr_pool_get(qp, &amp;qp-&gt;sig_mrs);</div><div class='add'>+	if (!ctx-&gt;reg-&gt;mr) {</div><div class='ctx'> 		ret = -EAGAIN;</div><div class='del'>-		goto out_destroy_prot_mr;</div><div class='add'>+		goto out_free_ctx;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (ctx-&gt;sig-&gt;sig_mr-&gt;need_inval) {</div><div class='del'>-		memset(&amp;ctx-&gt;sig-&gt;sig_inv_wr, 0, sizeof(ctx-&gt;sig-&gt;sig_inv_wr));</div><div class='add'>+	count += rdma_rw_inv_key(ctx-&gt;reg);</div><div class='ctx'> </div><div class='del'>-		ctx-&gt;sig-&gt;sig_inv_wr.opcode = IB_WR_LOCAL_INV;</div><div class='del'>-		ctx-&gt;sig-&gt;sig_inv_wr.ex.invalidate_rkey = ctx-&gt;sig-&gt;sig_mr-&gt;rkey;</div><div class='add'>+	memcpy(ctx-&gt;reg-&gt;mr-&gt;sig_attrs, sig_attrs, sizeof(struct ib_sig_attrs));</div><div class='ctx'> </div><div class='del'>-		prev_wr-&gt;next = &amp;ctx-&gt;sig-&gt;sig_inv_wr;</div><div class='del'>-		prev_wr = &amp;ctx-&gt;sig-&gt;sig_inv_wr;</div><div class='add'>+	ret = ib_map_mr_sg_pi(ctx-&gt;reg-&gt;mr, sg, sgt.nents, NULL, prot_sg,</div><div class='add'>+			      prot_sgt.nents, NULL, SZ_4K);</div><div class='add'>+	if (unlikely(ret)) {</div><div class='add'>+		pr_err("failed to map PI sg (%u)\n",</div><div class='add'>+		       sgt.nents + prot_sgt.nents);</div><div class='add'>+		goto out_destroy_sig_mr;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ctx-&gt;sig-&gt;sig_wr.wr.opcode = IB_WR_REG_SIG_MR;</div><div class='del'>-	ctx-&gt;sig-&gt;sig_wr.wr.wr_cqe = NULL;</div><div class='del'>-	ctx-&gt;sig-&gt;sig_wr.wr.sg_list = &amp;ctx-&gt;sig-&gt;data.sge;</div><div class='del'>-	ctx-&gt;sig-&gt;sig_wr.wr.num_sge = 1;</div><div class='del'>-	ctx-&gt;sig-&gt;sig_wr.access_flags = IB_ACCESS_LOCAL_WRITE;</div><div class='del'>-	ctx-&gt;sig-&gt;sig_wr.sig_attrs = sig_attrs;</div><div class='del'>-	ctx-&gt;sig-&gt;sig_wr.sig_mr = ctx-&gt;sig-&gt;sig_mr;</div><div class='del'>-	if (prot_sg_cnt)</div><div class='del'>-		ctx-&gt;sig-&gt;sig_wr.prot = &amp;ctx-&gt;sig-&gt;prot.sge;</div><div class='del'>-	prev_wr-&gt;next = &amp;ctx-&gt;sig-&gt;sig_wr.wr;</div><div class='del'>-	prev_wr = &amp;ctx-&gt;sig-&gt;sig_wr.wr;</div><div class='add'>+	ctx-&gt;reg-&gt;reg_wr.wr.opcode = IB_WR_REG_MR_INTEGRITY;</div><div class='add'>+	ctx-&gt;reg-&gt;reg_wr.wr.wr_cqe = NULL;</div><div class='add'>+	ctx-&gt;reg-&gt;reg_wr.wr.num_sge = 0;</div><div class='add'>+	ctx-&gt;reg-&gt;reg_wr.wr.send_flags = 0;</div><div class='add'>+	ctx-&gt;reg-&gt;reg_wr.access = IB_ACCESS_LOCAL_WRITE;</div><div class='add'>+	if (rdma_protocol_iwarp(qp-&gt;device, port_num))</div><div class='add'>+		ctx-&gt;reg-&gt;reg_wr.access |= IB_ACCESS_REMOTE_WRITE;</div><div class='add'>+	ctx-&gt;reg-&gt;reg_wr.mr = ctx-&gt;reg-&gt;mr;</div><div class='add'>+	ctx-&gt;reg-&gt;reg_wr.key = ctx-&gt;reg-&gt;mr-&gt;lkey;</div><div class='ctx'> 	count++;</div><div class='ctx'> </div><div class='del'>-	ctx-&gt;sig-&gt;sig_sge.addr = 0;</div><div class='del'>-	ctx-&gt;sig-&gt;sig_sge.length = ctx-&gt;sig-&gt;data.sge.length;</div><div class='del'>-	if (sig_attrs-&gt;wire.sig_type != IB_SIG_TYPE_NONE)</div><div class='del'>-		ctx-&gt;sig-&gt;sig_sge.length += ctx-&gt;sig-&gt;prot.sge.length;</div><div class='add'>+	ctx-&gt;reg-&gt;sge.addr = ctx-&gt;reg-&gt;mr-&gt;iova;</div><div class='add'>+	ctx-&gt;reg-&gt;sge.length = ctx-&gt;reg-&gt;mr-&gt;length;</div><div class='add'>+	if (sig_attrs-&gt;wire.sig_type == IB_SIG_TYPE_NONE)</div><div class='add'>+		ctx-&gt;reg-&gt;sge.length -= ctx-&gt;reg-&gt;mr-&gt;sig_attrs-&gt;meta_length;</div><div class='ctx'> </div><div class='del'>-	rdma_wr = &amp;ctx-&gt;sig-&gt;data.wr;</div><div class='del'>-	rdma_wr-&gt;wr.sg_list = &amp;ctx-&gt;sig-&gt;sig_sge;</div><div class='add'>+	rdma_wr = &amp;ctx-&gt;reg-&gt;wr;</div><div class='add'>+	rdma_wr-&gt;wr.sg_list = &amp;ctx-&gt;reg-&gt;sge;</div><div class='ctx'> 	rdma_wr-&gt;wr.num_sge = 1;</div><div class='ctx'> 	rdma_wr-&gt;remote_addr = remote_addr;</div><div class='ctx'> 	rdma_wr-&gt;rkey = rkey;</div><div class='hunk'>@@ -442,23 +448,20 @@ int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp,</div><div class='ctx'> 		rdma_wr-&gt;wr.opcode = IB_WR_RDMA_WRITE;</div><div class='ctx'> 	else</div><div class='ctx'> 		rdma_wr-&gt;wr.opcode = IB_WR_RDMA_READ;</div><div class='del'>-	prev_wr-&gt;next = &amp;rdma_wr-&gt;wr;</div><div class='del'>-	prev_wr = &amp;rdma_wr-&gt;wr;</div><div class='add'>+	ctx-&gt;reg-&gt;reg_wr.wr.next = &amp;rdma_wr-&gt;wr;</div><div class='ctx'> 	count++;</div><div class='ctx'> </div><div class='ctx'> 	return count;</div><div class='ctx'> </div><div class='del'>-out_destroy_prot_mr:</div><div class='del'>-	if (prot_sg_cnt)</div><div class='del'>-		ib_mr_pool_put(qp, &amp;qp-&gt;rdma_mrs, ctx-&gt;sig-&gt;prot.mr);</div><div class='del'>-out_destroy_data_mr:</div><div class='del'>-	ib_mr_pool_put(qp, &amp;qp-&gt;rdma_mrs, ctx-&gt;sig-&gt;data.mr);</div><div class='add'>+out_destroy_sig_mr:</div><div class='add'>+	ib_mr_pool_put(qp, &amp;qp-&gt;sig_mrs, ctx-&gt;reg-&gt;mr);</div><div class='ctx'> out_free_ctx:</div><div class='del'>-	kfree(ctx-&gt;sig);</div><div class='add'>+	kfree(ctx-&gt;reg);</div><div class='ctx'> out_unmap_prot_sg:</div><div class='del'>-	ib_dma_unmap_sg(dev, prot_sg, prot_sg_cnt, dir);</div><div class='add'>+	if (prot_sgt.nents)</div><div class='add'>+		ib_dma_unmap_sgtable_attrs(dev, &amp;prot_sgt, dir, 0);</div><div class='ctx'> out_unmap_sg:</div><div class='del'>-	ib_dma_unmap_sg(dev, sg, sg_cnt, dir);</div><div class='add'>+	ib_dma_unmap_sgtable_attrs(dev, &amp;sgt, dir, 0);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_rw_ctx_signature_init);</div><div class='hunk'>@@ -492,28 +495,13 @@ static void rdma_rw_update_lkey(struct rdma_rw_reg_ctx *reg, bool need_inval)</div><div class='ctx'>  * completion notification.</div><div class='ctx'>  */</div><div class='ctx'> struct ib_send_wr *rdma_rw_ctx_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp,</div><div class='del'>-		u8 port_num, struct ib_cqe *cqe, struct ib_send_wr *chain_wr)</div><div class='add'>+		u32 port_num, struct ib_cqe *cqe, struct ib_send_wr *chain_wr)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_send_wr *first_wr, *last_wr;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='ctx'> 	switch (ctx-&gt;type) {</div><div class='ctx'> 	case RDMA_RW_SIG_MR:</div><div class='del'>-		rdma_rw_update_lkey(&amp;ctx-&gt;sig-&gt;data, true);</div><div class='del'>-		if (ctx-&gt;sig-&gt;prot.mr)</div><div class='del'>-			rdma_rw_update_lkey(&amp;ctx-&gt;sig-&gt;prot, true);</div><div class='del'>-	</div><div class='del'>-		ctx-&gt;sig-&gt;sig_mr-&gt;need_inval = true;</div><div class='del'>-		ib_update_fast_reg_key(ctx-&gt;sig-&gt;sig_mr,</div><div class='del'>-			ib_inc_rkey(ctx-&gt;sig-&gt;sig_mr-&gt;lkey));</div><div class='del'>-		ctx-&gt;sig-&gt;sig_sge.lkey = ctx-&gt;sig-&gt;sig_mr-&gt;lkey;</div><div class='del'>-</div><div class='del'>-		if (ctx-&gt;sig-&gt;data.inv_wr.next)</div><div class='del'>-			first_wr = &amp;ctx-&gt;sig-&gt;data.inv_wr;</div><div class='del'>-		else</div><div class='del'>-			first_wr = &amp;ctx-&gt;sig-&gt;data.reg_wr.wr;</div><div class='del'>-		last_wr = &amp;ctx-&gt;sig-&gt;data.wr.wr;</div><div class='del'>-		break;</div><div class='ctx'> 	case RDMA_RW_MR:</div><div class='ctx'> 		for (i = 0; i &lt; ctx-&gt;nr_ops; i++) {</div><div class='ctx'> 			rdma_rw_update_lkey(&amp;ctx-&gt;reg[i],</div><div class='hunk'>@@ -564,7 +552,7 @@ EXPORT_SYMBOL(rdma_rw_ctx_wrs);</div><div class='ctx'>  * is not set @cqe must be set so that the caller gets a completion</div><div class='ctx'>  * notification.</div><div class='ctx'>  */</div><div class='del'>-int rdma_rw_ctx_post(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num,</div><div class='add'>+int rdma_rw_ctx_post(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u32 port_num,</div><div class='ctx'> 		struct ib_cqe *cqe, struct ib_send_wr *chain_wr)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_send_wr *first_wr;</div><div class='hunk'>@@ -583,8 +571,9 @@ EXPORT_SYMBOL(rdma_rw_ctx_post);</div><div class='ctx'>  * @sg_cnt:	number of entries in @sg</div><div class='ctx'>  * @dir:	%DMA_TO_DEVICE for RDMA WRITE, %DMA_FROM_DEVICE for RDMA READ</div><div class='ctx'>  */</div><div class='del'>-void rdma_rw_ctx_destroy(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num,</div><div class='del'>-		struct scatterlist *sg, u32 sg_cnt, enum dma_data_direction dir)</div><div class='add'>+void rdma_rw_ctx_destroy(struct rdma_rw_ctx *ctx, struct ib_qp *qp,</div><div class='add'>+			 u32 port_num, struct scatterlist *sg, u32 sg_cnt,</div><div class='add'>+			 enum dma_data_direction dir)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='hunk'>@@ -605,15 +594,13 @@ void rdma_rw_ctx_destroy(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num,</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* P2PDMA contexts do not need to be unmapped */</div><div class='del'>-	if (!is_pci_p2pdma_page(sg_page(sg)))</div><div class='del'>-		ib_dma_unmap_sg(qp-&gt;pd-&gt;device, sg, sg_cnt, dir);</div><div class='add'>+	ib_dma_unmap_sg(qp-&gt;pd-&gt;device, sg, sg_cnt, dir);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_rw_ctx_destroy);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * rdma_rw_ctx_destroy_signature - release all resources allocated by</div><div class='del'>- *	rdma_rw_ctx_init_signature</div><div class='add'>+ *	rdma_rw_ctx_signature_init</div><div class='ctx'>  * @ctx:	context to release</div><div class='ctx'>  * @qp:		queue pair to operate on</div><div class='ctx'>  * @port_num:	port num to which the connection is bound</div><div class='hunk'>@@ -624,23 +611,19 @@ EXPORT_SYMBOL(rdma_rw_ctx_destroy);</div><div class='ctx'>  * @dir:	%DMA_TO_DEVICE for RDMA WRITE, %DMA_FROM_DEVICE for RDMA READ</div><div class='ctx'>  */</div><div class='ctx'> void rdma_rw_ctx_destroy_signature(struct rdma_rw_ctx *ctx, struct ib_qp *qp,</div><div class='del'>-		u8 port_num, struct scatterlist *sg, u32 sg_cnt,</div><div class='add'>+		u32 port_num, struct scatterlist *sg, u32 sg_cnt,</div><div class='ctx'> 		struct scatterlist *prot_sg, u32 prot_sg_cnt,</div><div class='ctx'> 		enum dma_data_direction dir)</div><div class='ctx'> {</div><div class='ctx'> 	if (WARN_ON_ONCE(ctx-&gt;type != RDMA_RW_SIG_MR))</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='del'>-	ib_mr_pool_put(qp, &amp;qp-&gt;rdma_mrs, ctx-&gt;sig-&gt;data.mr);</div><div class='del'>-	ib_dma_unmap_sg(qp-&gt;pd-&gt;device, sg, sg_cnt, dir);</div><div class='add'>+	ib_mr_pool_put(qp, &amp;qp-&gt;sig_mrs, ctx-&gt;reg-&gt;mr);</div><div class='add'>+	kfree(ctx-&gt;reg);</div><div class='ctx'> </div><div class='del'>-	if (ctx-&gt;sig-&gt;prot.mr) {</div><div class='del'>-		ib_mr_pool_put(qp, &amp;qp-&gt;rdma_mrs, ctx-&gt;sig-&gt;prot.mr);</div><div class='add'>+	if (prot_sg_cnt)</div><div class='ctx'> 		ib_dma_unmap_sg(qp-&gt;pd-&gt;device, prot_sg, prot_sg_cnt, dir);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ib_mr_pool_put(qp, &amp;qp-&gt;sig_mrs, ctx-&gt;sig-&gt;sig_mr);</div><div class='del'>-	kfree(ctx-&gt;sig);</div><div class='add'>+	ib_dma_unmap_sg(qp-&gt;pd-&gt;device, sg, sg_cnt, dir);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_rw_ctx_destroy_signature);</div><div class='ctx'> </div><div class='hunk'>@@ -655,13 +638,13 @@ EXPORT_SYMBOL(rdma_rw_ctx_destroy_signature);</div><div class='ctx'>  * compute max_rdma_ctxts and the size of the transport's Send and</div><div class='ctx'>  * Send Completion Queues.</div><div class='ctx'>  */</div><div class='del'>-unsigned int rdma_rw_mr_factor(struct ib_device *device, u8 port_num,</div><div class='add'>+unsigned int rdma_rw_mr_factor(struct ib_device *device, u32 port_num,</div><div class='ctx'> 			       unsigned int maxpages)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned int mr_pages;</div><div class='ctx'> </div><div class='ctx'> 	if (rdma_rw_can_use_mr(device, port_num))</div><div class='del'>-		mr_pages = rdma_rw_fr_page_list_len(device);</div><div class='add'>+		mr_pages = rdma_rw_fr_page_list_len(device, false);</div><div class='ctx'> 	else</div><div class='ctx'> 		mr_pages = device-&gt;attrs.max_sge_rd;</div><div class='ctx'> 	return DIV_ROUND_UP(maxpages, mr_pages);</div><div class='hunk'>@@ -683,13 +666,12 @@ void rdma_rw_init_qp(struct ib_device *dev, struct ib_qp_init_attr *attr)</div><div class='ctx'> 	factor = 1;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='del'>-	 * If the devices needs MRs to perform RDMA READ or WRITE operations,</div><div class='add'>+	 * If the device needs MRs to perform RDMA READ or WRITE operations,</div><div class='ctx'> 	 * we'll need two additional MRs for the registrations and the</div><div class='ctx'> 	 * invalidation.</div><div class='ctx'> 	 */</div><div class='del'>-	if (attr-&gt;create_flags &amp; IB_QP_CREATE_SIGNATURE_EN)</div><div class='del'>-		factor += 6;	/* (inv + reg) * (data + prot + sig) */</div><div class='del'>-	else if (rdma_rw_can_use_mr(dev, attr-&gt;port_num))</div><div class='add'>+	if (attr-&gt;create_flags &amp; IB_QP_CREATE_INTEGRITY_EN ||</div><div class='add'>+	    rdma_rw_can_use_mr(dev, attr-&gt;port_num))</div><div class='ctx'> 		factor += 2;	/* inv + reg */</div><div class='ctx'> </div><div class='ctx'> 	attr-&gt;cap.max_send_wr += factor * attr-&gt;cap.max_rdma_ctxs;</div><div class='hunk'>@@ -705,22 +687,24 @@ void rdma_rw_init_qp(struct ib_device *dev, struct ib_qp_init_attr *attr)</div><div class='ctx'> int rdma_rw_init_mrs(struct ib_qp *qp, struct ib_qp_init_attr *attr)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *dev = qp-&gt;pd-&gt;device;</div><div class='del'>-	u32 nr_mrs = 0, nr_sig_mrs = 0;</div><div class='add'>+	u32 nr_mrs = 0, nr_sig_mrs = 0, max_num_sg = 0;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	if (attr-&gt;create_flags &amp; IB_QP_CREATE_SIGNATURE_EN) {</div><div class='add'>+	if (attr-&gt;create_flags &amp; IB_QP_CREATE_INTEGRITY_EN) {</div><div class='ctx'> 		nr_sig_mrs = attr-&gt;cap.max_rdma_ctxs;</div><div class='del'>-		nr_mrs = attr-&gt;cap.max_rdma_ctxs * 2;</div><div class='add'>+		nr_mrs = attr-&gt;cap.max_rdma_ctxs;</div><div class='add'>+		max_num_sg = rdma_rw_fr_page_list_len(dev, true);</div><div class='ctx'> 	} else if (rdma_rw_can_use_mr(dev, attr-&gt;port_num)) {</div><div class='ctx'> 		nr_mrs = attr-&gt;cap.max_rdma_ctxs;</div><div class='add'>+		max_num_sg = rdma_rw_fr_page_list_len(dev, false);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (nr_mrs) {</div><div class='ctx'> 		ret = ib_mr_pool_init(qp, &amp;qp-&gt;rdma_mrs, nr_mrs,</div><div class='ctx'> 				IB_MR_TYPE_MEM_REG,</div><div class='del'>-				rdma_rw_fr_page_list_len(dev));</div><div class='add'>+				max_num_sg, 0);</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			pr_err("%s: failed to allocated %d MRs\n",</div><div class='add'>+			pr_err("%s: failed to allocated %u MRs\n",</div><div class='ctx'> 				__func__, nr_mrs);</div><div class='ctx'> 			return ret;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -728,10 +712,10 @@ int rdma_rw_init_mrs(struct ib_qp *qp, struct ib_qp_init_attr *attr)</div><div class='ctx'> </div><div class='ctx'> 	if (nr_sig_mrs) {</div><div class='ctx'> 		ret = ib_mr_pool_init(qp, &amp;qp-&gt;sig_mrs, nr_sig_mrs,</div><div class='del'>-				IB_MR_TYPE_SIGNATURE, 2);</div><div class='add'>+				IB_MR_TYPE_INTEGRITY, max_num_sg, max_num_sg);</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			pr_err("%s: failed to allocated %d SIG MRs\n",</div><div class='del'>-				__func__, nr_mrs);</div><div class='add'>+			pr_err("%s: failed to allocated %u SIG MRs\n",</div><div class='add'>+				__func__, nr_sig_mrs);</div><div class='ctx'> 			goto out_free_rdma_mrs;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='head'>diff --git a/drivers/infiniband/core/sa.h b/drivers/infiniband/core/sa.h<br/>index cbaaaa92fff3..143de37ae598 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/sa.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/sa.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/sa.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/sa.h</a></div><div class='hunk'>@@ -49,7 +49,7 @@ static inline void ib_sa_client_put(struct ib_sa_client *client)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int ib_sa_mcmember_rec_query(struct ib_sa_client *client,</div><div class='del'>-			     struct ib_device *device, u8 port_num, u8 method,</div><div class='add'>+			     struct ib_device *device, u32 port_num, u8 method,</div><div class='ctx'> 			     struct ib_sa_mcmember_rec *rec,</div><div class='ctx'> 			     ib_sa_comp_mask comp_mask,</div><div class='ctx'> 			     unsigned long timeout_ms, gfp_t gfp_mask,</div><div class='head'>diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c<br/>index 7d8071c7e564..c23e9c847314 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/sa_query.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/sa_query.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/sa_query.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/sa_query.c</a></div><div class='hunk'>@@ -32,7 +32,6 @@</div><div class='ctx'>  * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='ctx'> #include &lt;linux/init.h&gt;</div><div class='ctx'> #include &lt;linux/err.h&gt;</div><div class='ctx'> #include &lt;linux/random.h&gt;</div><div class='hunk'>@@ -51,6 +50,7 @@</div><div class='ctx'> #include &lt;rdma/ib_marshall.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_addr.h&gt;</div><div class='ctx'> #include &lt;rdma/opa_addr.h&gt;</div><div class='add'>+#include &lt;rdma/rdma_cm.h&gt;</div><div class='ctx'> #include "sa.h"</div><div class='ctx'> #include "core_priv.h"</div><div class='ctx'> </div><div class='hunk'>@@ -95,17 +95,20 @@ struct ib_sa_port {</div><div class='ctx'> 	struct delayed_work ib_cpi_work;</div><div class='ctx'> 	spinlock_t                   classport_lock; /* protects class port info set */</div><div class='ctx'> 	spinlock_t           ah_lock;</div><div class='del'>-	u8                   port_num;</div><div class='add'>+	u32		     port_num;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct ib_sa_device {</div><div class='ctx'> 	int                     start_port, end_port;</div><div class='ctx'> 	struct ib_event_handler event_handler;</div><div class='del'>-	struct ib_sa_port port[0];</div><div class='add'>+	struct ib_sa_port port[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct ib_sa_query {</div><div class='del'>-	void (*callback)(struct ib_sa_query *, int, struct ib_sa_mad *);</div><div class='add'>+	void (*callback)(struct ib_sa_query *sa_query, int status,</div><div class='add'>+			 struct ib_sa_mad *mad);</div><div class='add'>+	void (*rmpp_callback)(struct ib_sa_query *sa_query, int status,</div><div class='add'>+			      struct ib_mad_recv_wc *mad);</div><div class='ctx'> 	void (*release)(struct ib_sa_query *);</div><div class='ctx'> 	struct ib_sa_client    *client;</div><div class='ctx'> 	struct ib_sa_port      *port;</div><div class='hunk'>@@ -123,14 +126,9 @@ struct ib_sa_query {</div><div class='ctx'> #define IB_SA_CANCEL			0x00000002</div><div class='ctx'> #define IB_SA_QUERY_OPA			0x00000004</div><div class='ctx'> </div><div class='del'>-struct ib_sa_service_query {</div><div class='del'>-	void (*callback)(int, struct ib_sa_service_rec *, void *);</div><div class='del'>-	void *context;</div><div class='del'>-	struct ib_sa_query sa_query;</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> struct ib_sa_path_query {</div><div class='del'>-	void (*callback)(int, struct sa_path_rec *, void *);</div><div class='add'>+	void (*callback)(int status, struct sa_path_rec *rec,</div><div class='add'>+			 unsigned int num_paths, void *context);</div><div class='ctx'> 	void *context;</div><div class='ctx'> 	struct ib_sa_query sa_query;</div><div class='ctx'> 	struct sa_path_rec *conv_pr;</div><div class='hunk'>@@ -154,6 +152,13 @@ struct ib_sa_mcmember_query {</div><div class='ctx'> 	struct ib_sa_query sa_query;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct ib_sa_service_query {</div><div class='add'>+	void (*callback)(int status, struct sa_service_rec *rec,</div><div class='add'>+			 unsigned int num_services, void *context);</div><div class='add'>+	void *context;</div><div class='add'>+	struct ib_sa_query sa_query;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> static LIST_HEAD(ib_nl_request_list);</div><div class='ctx'> static DEFINE_SPINLOCK(ib_nl_request_lock);</div><div class='ctx'> static atomic_t ib_nl_sa_request_seq;</div><div class='hunk'>@@ -174,7 +179,7 @@ static const struct nla_policy ib_nl_policy[LS_NLA_TYPE_MAX] = {</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> </div><div class='del'>-static void ib_sa_add_one(struct ib_device *device);</div><div class='add'>+static int ib_sa_add_one(struct ib_device *device);</div><div class='ctx'> static void ib_sa_remove_one(struct ib_device *device, void *client_data);</div><div class='ctx'> </div><div class='ctx'> static struct ib_client sa_client = {</div><div class='hunk'>@@ -190,7 +195,7 @@ static u32 tid;</div><div class='ctx'> </div><div class='ctx'> #define PATH_REC_FIELD(field) \</div><div class='ctx'> 	.struct_offset_bytes = offsetof(struct sa_path_rec, field),	\</div><div class='del'>-	.struct_size_bytes   = sizeof((struct sa_path_rec *)0)-&gt;field,	\</div><div class='add'>+	.struct_size_bytes   = sizeof_field(struct sa_path_rec, field),	\</div><div class='ctx'> 	.field_name          = "sa_path_rec:" #field</div><div class='ctx'> </div><div class='ctx'> static const struct ib_field path_rec_table[] = {</div><div class='hunk'>@@ -292,7 +297,7 @@ static const struct ib_field path_rec_table[] = {</div><div class='ctx'> 	.struct_offset_bytes = \</div><div class='ctx'> 		offsetof(struct sa_path_rec, field), \</div><div class='ctx'> 	.struct_size_bytes   = \</div><div class='del'>-		sizeof((struct sa_path_rec *)0)-&gt;field,	\</div><div class='add'>+		sizeof_field(struct sa_path_rec, field),	\</div><div class='ctx'> 	.field_name          = "sa_path_rec:" #field</div><div class='ctx'> </div><div class='ctx'> static const struct ib_field opa_path_rec_table[] = {</div><div class='hunk'>@@ -420,7 +425,7 @@ static const struct ib_field opa_path_rec_table[] = {</div><div class='ctx'> </div><div class='ctx'> #define MCMEMBER_REC_FIELD(field) \</div><div class='ctx'> 	.struct_offset_bytes = offsetof(struct ib_sa_mcmember_rec, field),	\</div><div class='del'>-	.struct_size_bytes   = sizeof ((struct ib_sa_mcmember_rec *) 0)-&gt;field,	\</div><div class='add'>+	.struct_size_bytes   = sizeof_field(struct ib_sa_mcmember_rec, field),	\</div><div class='ctx'> 	.field_name          = "sa_mcmember_rec:" #field</div><div class='ctx'> </div><div class='ctx'> static const struct ib_field mcmember_rec_table[] = {</div><div class='hunk'>@@ -502,57 +507,9 @@ static const struct ib_field mcmember_rec_table[] = {</div><div class='ctx'> 	  .size_bits    = 23 },</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define SERVICE_REC_FIELD(field) \</div><div class='del'>-	.struct_offset_bytes = offsetof(struct ib_sa_service_rec, field),	\</div><div class='del'>-	.struct_size_bytes   = sizeof ((struct ib_sa_service_rec *) 0)-&gt;field,	\</div><div class='del'>-	.field_name          = "sa_service_rec:" #field</div><div class='del'>-</div><div class='del'>-static const struct ib_field service_rec_table[] = {</div><div class='del'>-	{ SERVICE_REC_FIELD(id),</div><div class='del'>-	  .offset_words = 0,</div><div class='del'>-	  .offset_bits  = 0,</div><div class='del'>-	  .size_bits    = 64 },</div><div class='del'>-	{ SERVICE_REC_FIELD(gid),</div><div class='del'>-	  .offset_words = 2,</div><div class='del'>-	  .offset_bits  = 0,</div><div class='del'>-	  .size_bits    = 128 },</div><div class='del'>-	{ SERVICE_REC_FIELD(pkey),</div><div class='del'>-	  .offset_words = 6,</div><div class='del'>-	  .offset_bits  = 0,</div><div class='del'>-	  .size_bits    = 16 },</div><div class='del'>-	{ SERVICE_REC_FIELD(lease),</div><div class='del'>-	  .offset_words = 7,</div><div class='del'>-	  .offset_bits  = 0,</div><div class='del'>-	  .size_bits    = 32 },</div><div class='del'>-	{ SERVICE_REC_FIELD(key),</div><div class='del'>-	  .offset_words = 8,</div><div class='del'>-	  .offset_bits  = 0,</div><div class='del'>-	  .size_bits    = 128 },</div><div class='del'>-	{ SERVICE_REC_FIELD(name),</div><div class='del'>-	  .offset_words = 12,</div><div class='del'>-	  .offset_bits  = 0,</div><div class='del'>-	  .size_bits    = 64*8 },</div><div class='del'>-	{ SERVICE_REC_FIELD(data8),</div><div class='del'>-	  .offset_words = 28,</div><div class='del'>-	  .offset_bits  = 0,</div><div class='del'>-	  .size_bits    = 16*8 },</div><div class='del'>-	{ SERVICE_REC_FIELD(data16),</div><div class='del'>-	  .offset_words = 32,</div><div class='del'>-	  .offset_bits  = 0,</div><div class='del'>-	  .size_bits    = 8*16 },</div><div class='del'>-	{ SERVICE_REC_FIELD(data32),</div><div class='del'>-	  .offset_words = 36,</div><div class='del'>-	  .offset_bits  = 0,</div><div class='del'>-	  .size_bits    = 4*32 },</div><div class='del'>-	{ SERVICE_REC_FIELD(data64),</div><div class='del'>-	  .offset_words = 40,</div><div class='del'>-	  .offset_bits  = 0,</div><div class='del'>-	  .size_bits    = 2*64 },</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> #define CLASSPORTINFO_REC_FIELD(field) \</div><div class='ctx'> 	.struct_offset_bytes = offsetof(struct ib_class_port_info, field),	\</div><div class='del'>-	.struct_size_bytes   = sizeof((struct ib_class_port_info *)0)-&gt;field,	\</div><div class='add'>+	.struct_size_bytes   = sizeof_field(struct ib_class_port_info, field),	\</div><div class='ctx'> 	.field_name          = "ib_class_port_info:" #field</div><div class='ctx'> </div><div class='ctx'> static const struct ib_field ib_classport_info_rec_table[] = {</div><div class='hunk'>@@ -630,7 +587,7 @@ static const struct ib_field ib_classport_info_rec_table[] = {</div><div class='ctx'> 	.struct_offset_bytes =\</div><div class='ctx'> 		offsetof(struct opa_class_port_info, field),	\</div><div class='ctx'> 	.struct_size_bytes   = \</div><div class='del'>-		sizeof((struct opa_class_port_info *)0)-&gt;field,	\</div><div class='add'>+		sizeof_field(struct opa_class_port_info, field),	\</div><div class='ctx'> 	.field_name          = "opa_class_port_info:" #field</div><div class='ctx'> </div><div class='ctx'> static const struct ib_field opa_classport_info_rec_table[] = {</div><div class='hunk'>@@ -710,7 +667,7 @@ static const struct ib_field opa_classport_info_rec_table[] = {</div><div class='ctx'> </div><div class='ctx'> #define GUIDINFO_REC_FIELD(field) \</div><div class='ctx'> 	.struct_offset_bytes = offsetof(struct ib_sa_guidinfo_rec, field),	\</div><div class='del'>-	.struct_size_bytes   = sizeof((struct ib_sa_guidinfo_rec *) 0)-&gt;field,	\</div><div class='add'>+	.struct_size_bytes   = sizeof_field(struct ib_sa_guidinfo_rec, field),	\</div><div class='ctx'> 	.field_name          = "sa_guidinfo_rec:" #field</div><div class='ctx'> </div><div class='ctx'> static const struct ib_field guidinfo_rec_table[] = {</div><div class='hunk'>@@ -736,6 +693,60 @@ static const struct ib_field guidinfo_rec_table[] = {</div><div class='ctx'> 	  .size_bits    = 512 },</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+#define SERVICE_REC_FIELD(field) \</div><div class='add'>+	.struct_offset_bytes = offsetof(struct sa_service_rec, field),     \</div><div class='add'>+	.struct_size_bytes   = sizeof_field(struct sa_service_rec, field), \</div><div class='add'>+	.field_name          = "sa_service_rec:" #field</div><div class='add'>+</div><div class='add'>+static const struct ib_field service_rec_table[] = {</div><div class='add'>+	{ SERVICE_REC_FIELD(id),</div><div class='add'>+	  .offset_words = 0,</div><div class='add'>+	  .offset_bits  = 0,</div><div class='add'>+	  .size_bits    = 64 },</div><div class='add'>+	{ SERVICE_REC_FIELD(gid),</div><div class='add'>+	  .offset_words = 2,</div><div class='add'>+	  .offset_bits  = 0,</div><div class='add'>+	  .size_bits    = 128 },</div><div class='add'>+	{ SERVICE_REC_FIELD(pkey),</div><div class='add'>+	  .offset_words = 6,</div><div class='add'>+	  .offset_bits  = 0,</div><div class='add'>+	  .size_bits    = 16 },</div><div class='add'>+	{ RESERVED,</div><div class='add'>+	  .offset_words = 6,</div><div class='add'>+	  .offset_bits  = 16,</div><div class='add'>+	  .size_bits    = 16 },</div><div class='add'>+	{ SERVICE_REC_FIELD(lease),</div><div class='add'>+	  .offset_words = 7,</div><div class='add'>+	  .offset_bits  = 0,</div><div class='add'>+	  .size_bits    = 32 },</div><div class='add'>+	{ SERVICE_REC_FIELD(key),</div><div class='add'>+	  .offset_words = 8,</div><div class='add'>+	  .offset_bits  = 0,</div><div class='add'>+	  .size_bits    = 128 },</div><div class='add'>+	{ SERVICE_REC_FIELD(name),</div><div class='add'>+	  .offset_words = 12,</div><div class='add'>+	  .offset_bits  = 0,</div><div class='add'>+	  .size_bits    = 512 },</div><div class='add'>+	{ SERVICE_REC_FIELD(data_8),</div><div class='add'>+	  .offset_words = 28,</div><div class='add'>+	  .offset_bits  = 0,</div><div class='add'>+	  .size_bits    = 128 },</div><div class='add'>+	{ SERVICE_REC_FIELD(data_16),</div><div class='add'>+	  .offset_words = 32,</div><div class='add'>+	  .offset_bits  = 0,</div><div class='add'>+	  .size_bits    = 128 },</div><div class='add'>+	{ SERVICE_REC_FIELD(data_32),</div><div class='add'>+	  .offset_words = 36,</div><div class='add'>+	  .offset_bits  = 0,</div><div class='add'>+	  .size_bits    = 128 },</div><div class='add'>+	{ SERVICE_REC_FIELD(data_64),</div><div class='add'>+	  .offset_words = 40,</div><div class='add'>+	  .offset_bits  = 0,</div><div class='add'>+	  .size_bits    = 128 },</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define RDMA_PRIMARY_PATH_MAX_REC_NUM 3</div><div class='add'>+</div><div class='ctx'> static inline void ib_sa_disable_local_svc(struct ib_sa_query *query)</div><div class='ctx'> {</div><div class='ctx'> 	query-&gt;flags &amp;= ~IB_SA_ENABLE_LOCAL_SERVICE;</div><div class='hunk'>@@ -760,13 +771,14 @@ static void ib_nl_set_path_rec_attrs(struct sk_buff *skb,</div><div class='ctx'> </div><div class='ctx'> 	/* Construct the family header first */</div><div class='ctx'> 	header = skb_put(skb, NLMSG_ALIGN(sizeof(*header)));</div><div class='del'>-	memcpy(header-&gt;device_name, dev_name(&amp;query-&gt;port-&gt;agent-&gt;device-&gt;dev),</div><div class='del'>-	       LS_DEVICE_NAME_MAX);</div><div class='add'>+	strscpy_pad(header-&gt;device_name,</div><div class='add'>+		    dev_name(&amp;query-&gt;port-&gt;agent-&gt;device-&gt;dev),</div><div class='add'>+		    LS_DEVICE_NAME_MAX);</div><div class='ctx'> 	header-&gt;port_num = query-&gt;port-&gt;port_num;</div><div class='ctx'> </div><div class='ctx'> 	if ((comp_mask &amp; IB_SA_PATH_REC_REVERSIBLE) &amp;&amp;</div><div class='ctx'> 	    sa_rec-&gt;reversible != 0)</div><div class='del'>-		query-&gt;path_use = LS_RESOLVE_PATH_USE_GMP;</div><div class='add'>+		query-&gt;path_use = LS_RESOLVE_PATH_USE_ALL;</div><div class='ctx'> 	else</div><div class='ctx'> 		query-&gt;path_use = LS_RESOLVE_PATH_USE_UNIDIRECTIONAL;</div><div class='ctx'> 	header-&gt;path_use = query-&gt;path_use;</div><div class='hunk'>@@ -829,13 +841,20 @@ static int ib_nl_get_path_rec_attrs_len(ib_sa_comp_mask comp_mask)</div><div class='ctx'> 	return len;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int ib_nl_send_msg(struct ib_sa_query *query, gfp_t gfp_mask)</div><div class='add'>+static int ib_nl_make_request(struct ib_sa_query *query, gfp_t gfp_mask)</div><div class='ctx'> {</div><div class='ctx'> 	struct sk_buff *skb = NULL;</div><div class='ctx'> 	struct nlmsghdr *nlh;</div><div class='ctx'> 	void *data;</div><div class='ctx'> 	struct ib_sa_mad *mad;</div><div class='ctx'> 	int len;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	unsigned long delay;</div><div class='add'>+	gfp_t gfp_flag;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;query-&gt;list);</div><div class='add'>+	query-&gt;seq = (u32)atomic_inc_return(&amp;ib_nl_sa_request_seq);</div><div class='ctx'> </div><div class='ctx'> 	mad = query-&gt;mad_buf-&gt;mad;</div><div class='ctx'> 	len = ib_nl_get_path_rec_attrs_len(mad-&gt;sa_hdr.comp_mask);</div><div class='hunk'>@@ -860,36 +879,25 @@ static int ib_nl_send_msg(struct ib_sa_query *query, gfp_t gfp_mask)</div><div class='ctx'> 	/* Repair the nlmsg header length */</div><div class='ctx'> 	nlmsg_end(skb, nlh);</div><div class='ctx'> </div><div class='del'>-	return rdma_nl_multicast(skb, RDMA_NL_GROUP_LS, gfp_mask);</div><div class='del'>-}</div><div class='add'>+	gfp_flag = ((gfp_mask &amp; GFP_ATOMIC) == GFP_ATOMIC) ? GFP_ATOMIC :</div><div class='add'>+		GFP_NOWAIT;</div><div class='ctx'> </div><div class='del'>-static int ib_nl_make_request(struct ib_sa_query *query, gfp_t gfp_mask)</div><div class='del'>-{</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	unsigned long delay;</div><div class='del'>-	int ret;</div><div class='add'>+	spin_lock_irqsave(&amp;ib_nl_request_lock, flags);</div><div class='add'>+	ret = rdma_nl_multicast(&amp;init_net, skb, RDMA_NL_GROUP_LS, gfp_flag);</div><div class='ctx'> </div><div class='del'>-	INIT_LIST_HEAD(&amp;query-&gt;list);</div><div class='del'>-	query-&gt;seq = (u32)atomic_inc_return(&amp;ib_nl_sa_request_seq);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out;</div><div class='ctx'> </div><div class='del'>-	/* Put the request on the list first.*/</div><div class='del'>-	spin_lock_irqsave(&amp;ib_nl_request_lock, flags);</div><div class='add'>+	/* Put the request on the list.*/</div><div class='ctx'> 	delay = msecs_to_jiffies(sa_local_svc_timeout_ms);</div><div class='ctx'> 	query-&gt;timeout = delay + jiffies;</div><div class='ctx'> 	list_add_tail(&amp;query-&gt;list, &amp;ib_nl_request_list);</div><div class='ctx'> 	/* Start the timeout if this is the only request */</div><div class='ctx'> 	if (ib_nl_request_list.next == &amp;query-&gt;list)</div><div class='ctx'> 		queue_delayed_work(ib_nl_wq, &amp;ib_nl_timed_work, delay);</div><div class='del'>-	spin_unlock_irqrestore(&amp;ib_nl_request_lock, flags);</div><div class='ctx'> </div><div class='del'>-	ret = ib_nl_send_msg(query, gfp_mask);</div><div class='del'>-	if (ret) {</div><div class='del'>-		ret = -EIO;</div><div class='del'>-		/* Remove the request */</div><div class='del'>-		spin_lock_irqsave(&amp;ib_nl_request_lock, flags);</div><div class='del'>-		list_del(&amp;query-&gt;list);</div><div class='del'>-		spin_unlock_irqrestore(&amp;ib_nl_request_lock, flags);</div><div class='del'>-	}</div><div class='add'>+out:</div><div class='add'>+	spin_unlock_irqrestore(&amp;ib_nl_request_lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -923,50 +931,77 @@ static void send_handler(struct ib_mad_agent *agent,</div><div class='ctx'> static void ib_nl_process_good_resolve_rsp(struct ib_sa_query *query,</div><div class='ctx'> 					   const struct nlmsghdr *nlh)</div><div class='ctx'> {</div><div class='add'>+	struct sa_path_rec recs[RDMA_PRIMARY_PATH_MAX_REC_NUM];</div><div class='add'>+	struct ib_sa_path_query *path_query;</div><div class='add'>+	struct ib_path_rec_data *rec_data;</div><div class='ctx'> 	struct ib_mad_send_wc mad_send_wc;</div><div class='del'>-	struct ib_sa_mad *mad = NULL;</div><div class='ctx'> 	const struct nlattr *head, *curr;</div><div class='del'>-	struct ib_path_rec_data  *rec;</div><div class='del'>-	int len, rem;</div><div class='add'>+	struct ib_sa_mad *mad = NULL;</div><div class='add'>+	int len, rem, status = -EIO;</div><div class='add'>+	unsigned int num_prs = 0;</div><div class='ctx'> 	u32 mask = 0;</div><div class='del'>-	int status = -EIO;</div><div class='del'>-</div><div class='del'>-	if (query-&gt;callback) {</div><div class='del'>-		head = (const struct nlattr *) nlmsg_data(nlh);</div><div class='del'>-		len = nlmsg_len(nlh);</div><div class='del'>-		switch (query-&gt;path_use) {</div><div class='del'>-		case LS_RESOLVE_PATH_USE_UNIDIRECTIONAL:</div><div class='del'>-			mask = IB_PATH_PRIMARY | IB_PATH_OUTBOUND;</div><div class='del'>-			break;</div><div class='ctx'> </div><div class='del'>-		case LS_RESOLVE_PATH_USE_ALL:</div><div class='del'>-		case LS_RESOLVE_PATH_USE_GMP:</div><div class='del'>-		default:</div><div class='del'>-			mask = IB_PATH_PRIMARY | IB_PATH_GMP |</div><div class='del'>-				IB_PATH_BIDIRECTIONAL;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		nla_for_each_attr(curr, head, len, rem) {</div><div class='del'>-			if (curr-&gt;nla_type == LS_NLA_TYPE_PATH_RECORD) {</div><div class='del'>-				rec = nla_data(curr);</div><div class='del'>-				/*</div><div class='del'>-				 * Get the first one. In the future, we may</div><div class='del'>-				 * need to get up to 6 pathrecords.</div><div class='del'>-				 */</div><div class='del'>-				if ((rec-&gt;flags &amp; mask) == mask) {</div><div class='del'>-					mad = query-&gt;mad_buf-&gt;mad;</div><div class='del'>-					mad-&gt;mad_hdr.method |=</div><div class='del'>-						IB_MGMT_METHOD_RESP;</div><div class='del'>-					memcpy(mad-&gt;data, rec-&gt;path_rec,</div><div class='del'>-					       sizeof(rec-&gt;path_rec));</div><div class='del'>-					status = 0;</div><div class='del'>-					break;</div><div class='del'>-				}</div><div class='del'>-			}</div><div class='add'>+	if (!query-&gt;callback)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	path_query = container_of(query, struct ib_sa_path_query, sa_query);</div><div class='add'>+	mad = query-&gt;mad_buf-&gt;mad;</div><div class='add'>+</div><div class='add'>+	head = (const struct nlattr *) nlmsg_data(nlh);</div><div class='add'>+	len = nlmsg_len(nlh);</div><div class='add'>+	switch (query-&gt;path_use) {</div><div class='add'>+	case LS_RESOLVE_PATH_USE_UNIDIRECTIONAL:</div><div class='add'>+		mask = IB_PATH_PRIMARY | IB_PATH_OUTBOUND;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	case LS_RESOLVE_PATH_USE_ALL:</div><div class='add'>+		mask = IB_PATH_PRIMARY;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	case LS_RESOLVE_PATH_USE_GMP:</div><div class='add'>+	default:</div><div class='add'>+		mask = IB_PATH_PRIMARY | IB_PATH_GMP |</div><div class='add'>+			IB_PATH_BIDIRECTIONAL;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	nla_for_each_attr(curr, head, len, rem) {</div><div class='add'>+		if (curr-&gt;nla_type != LS_NLA_TYPE_PATH_RECORD)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		rec_data = nla_data(curr);</div><div class='add'>+		if ((rec_data-&gt;flags &amp; mask) != mask)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		if ((query-&gt;flags &amp; IB_SA_QUERY_OPA) ||</div><div class='add'>+		    path_query-&gt;conv_pr) {</div><div class='add'>+			mad-&gt;mad_hdr.method |= IB_MGMT_METHOD_RESP;</div><div class='add'>+			memcpy(mad-&gt;data, rec_data-&gt;path_rec,</div><div class='add'>+			       sizeof(rec_data-&gt;path_rec));</div><div class='add'>+			query-&gt;callback(query, 0, mad);</div><div class='add'>+			goto out;</div><div class='ctx'> 		}</div><div class='del'>-		query-&gt;callback(query, status, mad);</div><div class='add'>+</div><div class='add'>+		status = 0;</div><div class='add'>+		ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table),</div><div class='add'>+			  rec_data-&gt;path_rec, &amp;recs[num_prs]);</div><div class='add'>+		recs[num_prs].flags = rec_data-&gt;flags;</div><div class='add'>+		recs[num_prs].rec_type = SA_PATH_REC_TYPE_IB;</div><div class='add'>+		sa_path_set_dmac_zero(&amp;recs[num_prs]);</div><div class='add'>+</div><div class='add'>+		num_prs++;</div><div class='add'>+		if (num_prs &gt;= RDMA_PRIMARY_PATH_MAX_REC_NUM)</div><div class='add'>+			break;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (!status) {</div><div class='add'>+		mad-&gt;mad_hdr.method |= IB_MGMT_METHOD_RESP;</div><div class='add'>+		path_query-&gt;callback(status, recs, num_prs,</div><div class='add'>+				     path_query-&gt;context);</div><div class='add'>+	} else</div><div class='add'>+		query-&gt;callback(query, status, mad);</div><div class='add'>+</div><div class='add'>+out:</div><div class='ctx'> 	mad_send_wc.send_buf = query-&gt;mad_buf;</div><div class='ctx'> 	mad_send_wc.status = IB_WC_SUCCESS;</div><div class='ctx'> 	send_handler(query-&gt;mad_buf-&gt;mad_agent, &amp;mad_send_wc);</div><div class='hunk'>@@ -1039,6 +1074,8 @@ int ib_nl_handle_set_timeout(struct sk_buff *skb,</div><div class='ctx'> 	if (timeout &gt; IB_SA_LOCAL_SVC_TIMEOUT_MAX)</div><div class='ctx'> 		timeout = IB_SA_LOCAL_SVC_TIMEOUT_MAX;</div><div class='ctx'> </div><div class='add'>+	spin_lock_irqsave(&amp;ib_nl_request_lock, flags);</div><div class='add'>+</div><div class='ctx'> 	delta = timeout - sa_local_svc_timeout_ms;</div><div class='ctx'> 	if (delta &lt; 0)</div><div class='ctx'> 		abs_delta = -delta;</div><div class='hunk'>@@ -1046,7 +1083,6 @@ int ib_nl_handle_set_timeout(struct sk_buff *skb,</div><div class='ctx'> 		abs_delta = delta;</div><div class='ctx'> </div><div class='ctx'> 	if (delta != 0) {</div><div class='del'>-		spin_lock_irqsave(&amp;ib_nl_request_lock, flags);</div><div class='ctx'> 		sa_local_svc_timeout_ms = timeout;</div><div class='ctx'> 		list_for_each_entry(query, &amp;ib_nl_request_list, list) {</div><div class='ctx'> 			if (delta &lt; 0 &amp;&amp; abs_delta &gt; query-&gt;timeout)</div><div class='hunk'>@@ -1064,11 +1100,12 @@ int ib_nl_handle_set_timeout(struct sk_buff *skb,</div><div class='ctx'> 		if (delay)</div><div class='ctx'> 			mod_delayed_work(ib_nl_wq, &amp;ib_nl_timed_work,</div><div class='ctx'> 					 (unsigned long)delay);</div><div class='del'>-		spin_unlock_irqrestore(&amp;ib_nl_request_lock, flags);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	spin_unlock_irqrestore(&amp;ib_nl_request_lock, flags);</div><div class='add'>+</div><div class='ctx'> settimeout_out:</div><div class='del'>-	return skb-&gt;len;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static inline int ib_nl_is_good_resolve_resp(const struct nlmsghdr *nlh)</div><div class='hunk'>@@ -1092,10 +1129,9 @@ int ib_nl_handle_resolve_resp(struct sk_buff *skb,</div><div class='ctx'> 			      struct netlink_ext_ack *extack)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	struct ib_sa_query *query;</div><div class='add'>+	struct ib_sa_query *query = NULL, *iter;</div><div class='ctx'> 	struct ib_mad_send_buf *send_buf;</div><div class='ctx'> 	struct ib_mad_send_wc mad_send_wc;</div><div class='del'>-	int found = 0;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if ((nlh-&gt;nlmsg_flags &amp; NLM_F_REQUEST) ||</div><div class='hunk'>@@ -1103,20 +1139,21 @@ int ib_nl_handle_resolve_resp(struct sk_buff *skb,</div><div class='ctx'> 		return -EPERM;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;ib_nl_request_lock, flags);</div><div class='del'>-	list_for_each_entry(query, &amp;ib_nl_request_list, list) {</div><div class='add'>+	list_for_each_entry(iter, &amp;ib_nl_request_list, list) {</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * If the query is cancelled, let the timeout routine</div><div class='ctx'> 		 * take care of it.</div><div class='ctx'> 		 */</div><div class='del'>-		if (nlh-&gt;nlmsg_seq == query-&gt;seq) {</div><div class='del'>-			found = !ib_sa_query_cancelled(query);</div><div class='del'>-			if (found)</div><div class='del'>-				list_del(&amp;query-&gt;list);</div><div class='add'>+		if (nlh-&gt;nlmsg_seq == iter-&gt;seq) {</div><div class='add'>+			if (!ib_sa_query_cancelled(iter)) {</div><div class='add'>+				list_del(&amp;iter-&gt;list);</div><div class='add'>+				query = iter;</div><div class='add'>+			}</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!found) {</div><div class='add'>+	if (!query) {</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;ib_nl_request_lock, flags);</div><div class='ctx'> 		goto resp_out;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -1139,7 +1176,7 @@ int ib_nl_handle_resolve_resp(struct sk_buff *skb,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> resp_out:</div><div class='del'>-	return skb-&gt;len;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void free_sm_ah(struct kref *kref)</div><div class='hunk'>@@ -1176,7 +1213,6 @@ EXPORT_SYMBOL(ib_sa_unregister_client);</div><div class='ctx'> void ib_sa_cancel_query(int id, struct ib_sa_query *query)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	struct ib_mad_agent *agent;</div><div class='ctx'> 	struct ib_mad_send_buf *mad_buf;</div><div class='ctx'> </div><div class='ctx'> 	xa_lock_irqsave(&amp;queries, flags);</div><div class='hunk'>@@ -1184,7 +1220,6 @@ void ib_sa_cancel_query(int id, struct ib_sa_query *query)</div><div class='ctx'> 		xa_unlock_irqrestore(&amp;queries, flags);</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='del'>-	agent = query-&gt;port-&gt;agent;</div><div class='ctx'> 	mad_buf = query-&gt;mad_buf;</div><div class='ctx'> 	xa_unlock_irqrestore(&amp;queries, flags);</div><div class='ctx'> </div><div class='hunk'>@@ -1194,11 +1229,11 @@ void ib_sa_cancel_query(int id, struct ib_sa_query *query)</div><div class='ctx'> 	 * sent to the MAD layer and has to be cancelled from there.</div><div class='ctx'> 	 */</div><div class='ctx'> 	if (!ib_nl_cancel_request(query))</div><div class='del'>-		ib_cancel_mad(agent, mad_buf);</div><div class='add'>+		ib_cancel_mad(mad_buf);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_sa_cancel_query);</div><div class='ctx'> </div><div class='del'>-static u8 get_src_path_mask(struct ib_device *device, u8 port_num)</div><div class='add'>+static u8 get_src_path_mask(struct ib_device *device, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_sa_device *sa_dev;</div><div class='ctx'> 	struct ib_sa_port   *port;</div><div class='hunk'>@@ -1217,7 +1252,7 @@ static u8 get_src_path_mask(struct ib_device *device, u8 port_num)</div><div class='ctx'> 	return src_path_mask;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int init_ah_attr_grh_fields(struct ib_device *device, u8 port_num,</div><div class='add'>+static int init_ah_attr_grh_fields(struct ib_device *device, u32 port_num,</div><div class='ctx'> 				   struct sa_path_rec *rec,</div><div class='ctx'> 				   struct rdma_ah_attr *ah_attr,</div><div class='ctx'> 				   const struct ib_gid_attr *gid_attr)</div><div class='hunk'>@@ -1246,7 +1281,7 @@ static int init_ah_attr_grh_fields(struct ib_device *device, u8 port_num,</div><div class='ctx'>  * @port_num: Port on the specified device.</div><div class='ctx'>  * @rec: path record entry to use for ah attributes initialization.</div><div class='ctx'>  * @ah_attr: address handle attributes to initialization from path record.</div><div class='del'>- * @sgid_attr: SGID attribute to consider during initialization.</div><div class='add'>+ * @gid_attr: SGID attribute to consider during initialization.</div><div class='ctx'>  *</div><div class='ctx'>  * When ib_init_ah_attr_from_path() returns success,</div><div class='ctx'>  * (a) for IB link layer it optionally contains a reference to SGID attribute</div><div class='hunk'>@@ -1255,7 +1290,7 @@ static int init_ah_attr_grh_fields(struct ib_device *device, u8 port_num,</div><div class='ctx'>  * User must invoke rdma_destroy_ah_attr() to release reference to SGID</div><div class='ctx'>  * attributes which are initialized using ib_init_ah_attr_from_path().</div><div class='ctx'>  */</div><div class='del'>-int ib_init_ah_attr_from_path(struct ib_device *device, u8 port_num,</div><div class='add'>+int ib_init_ah_attr_from_path(struct ib_device *device, u32 port_num,</div><div class='ctx'> 			      struct sa_path_rec *rec,</div><div class='ctx'> 			      struct rdma_ah_attr *ah_attr,</div><div class='ctx'> 			      const struct ib_gid_attr *gid_attr)</div><div class='hunk'>@@ -1364,6 +1399,7 @@ static int send_mad(struct ib_sa_query *query, unsigned long timeout_ms,</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int ret, id;</div><div class='add'>+	const int nmbr_sa_query_retries = 10;</div><div class='ctx'> </div><div class='ctx'> 	xa_lock_irqsave(&amp;queries, flags);</div><div class='ctx'> 	ret = __xa_alloc(&amp;queries, &amp;id, query, xa_limit_32b, gfp_mask);</div><div class='hunk'>@@ -1371,7 +1407,13 @@ static int send_mad(struct ib_sa_query *query, unsigned long timeout_ms,</div><div class='ctx'> 	if (ret &lt; 0)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	query-&gt;mad_buf-&gt;timeout_ms  = timeout_ms;</div><div class='add'>+	query-&gt;mad_buf-&gt;timeout_ms  = timeout_ms / nmbr_sa_query_retries;</div><div class='add'>+	query-&gt;mad_buf-&gt;retries = nmbr_sa_query_retries;</div><div class='add'>+	if (!query-&gt;mad_buf-&gt;timeout_ms) {</div><div class='add'>+		/* Special case, very small timeout_ms */</div><div class='add'>+		query-&gt;mad_buf-&gt;timeout_ms = 1;</div><div class='add'>+		query-&gt;mad_buf-&gt;retries = timeout_ms;</div><div class='add'>+	}</div><div class='ctx'> 	query-&gt;mad_buf-&gt;context[0] = query;</div><div class='ctx'> 	query-&gt;id = id;</div><div class='ctx'> </div><div class='hunk'>@@ -1411,18 +1453,28 @@ void ib_sa_pack_path(struct sa_path_rec *rec, void *attribute)</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_sa_pack_path);</div><div class='ctx'> </div><div class='add'>+void ib_sa_pack_service(struct sa_service_rec *rec, void *attribute)</div><div class='add'>+{</div><div class='add'>+	ib_pack(service_rec_table, ARRAY_SIZE(service_rec_table), rec,</div><div class='add'>+		attribute);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_sa_pack_service);</div><div class='add'>+</div><div class='add'>+void ib_sa_unpack_service(void *attribute, struct sa_service_rec *rec)</div><div class='add'>+{</div><div class='add'>+	ib_unpack(service_rec_table, ARRAY_SIZE(service_rec_table), attribute,</div><div class='add'>+		  rec);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_sa_unpack_service);</div><div class='add'>+</div><div class='ctx'> static bool ib_sa_opa_pathrecord_support(struct ib_sa_client *client,</div><div class='del'>-					 struct ib_device *device,</div><div class='del'>-					 u8 port_num)</div><div class='add'>+					 struct ib_sa_device *sa_dev,</div><div class='add'>+					 u32 port_num)</div><div class='ctx'> {</div><div class='del'>-	struct ib_sa_device *sa_dev = ib_get_client_data(device, &amp;sa_client);</div><div class='ctx'> 	struct ib_sa_port *port;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	bool ret = false;</div><div class='ctx'> </div><div class='del'>-	if (!sa_dev)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='ctx'> 	port = &amp;sa_dev-&gt;port[port_num - sa_dev-&gt;start_port];</div><div class='ctx'> 	spin_lock_irqsave(&amp;port-&gt;classport_lock, flags);</div><div class='ctx'> 	if (!port-&gt;classport_info.valid)</div><div class='hunk'>@@ -1442,24 +1494,24 @@ enum opa_pr_supported {</div><div class='ctx'> 	PR_IB_SUPPORTED</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/**</div><div class='del'>- * Check if current PR query can be an OPA query.</div><div class='del'>- * Retuns PR_NOT_SUPPORTED if a path record query is not</div><div class='add'>+/*</div><div class='add'>+ * opa_pr_query_possible - Check if current PR query can be an OPA query.</div><div class='add'>+ *</div><div class='add'>+ * Returns PR_NOT_SUPPORTED if a path record query is not</div><div class='ctx'>  * possible, PR_OPA_SUPPORTED if an OPA path record query</div><div class='ctx'>  * is possible and PR_IB_SUPPORTED if an IB path record</div><div class='ctx'>  * query is possible.</div><div class='ctx'>  */</div><div class='ctx'> static int opa_pr_query_possible(struct ib_sa_client *client,</div><div class='del'>-				 struct ib_device *device,</div><div class='del'>-				 u8 port_num,</div><div class='del'>-				 struct sa_path_rec *rec)</div><div class='add'>+				 struct ib_sa_device *sa_dev,</div><div class='add'>+				 struct ib_device *device, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr port_attr;</div><div class='ctx'> </div><div class='ctx'> 	if (ib_query_port(device, port_num, &amp;port_attr))</div><div class='ctx'> 		return PR_NOT_SUPPORTED;</div><div class='ctx'> </div><div class='del'>-	if (ib_sa_opa_pathrecord_support(client, device, port_num))</div><div class='add'>+	if (ib_sa_opa_pathrecord_support(client, sa_dev, port_num))</div><div class='ctx'> 		return PR_OPA_SUPPORTED;</div><div class='ctx'> </div><div class='ctx'> 	if (port_attr.lid &gt;= be16_to_cpu(IB_MULTICAST_LID_BASE))</div><div class='hunk'>@@ -1469,40 +1521,101 @@ static int opa_pr_query_possible(struct ib_sa_client *client,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void ib_sa_path_rec_callback(struct ib_sa_query *sa_query,</div><div class='del'>-				    int status,</div><div class='del'>-				    struct ib_sa_mad *mad)</div><div class='add'>+				    int status, struct ib_sa_mad *mad)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_sa_path_query *query =</div><div class='ctx'> 		container_of(sa_query, struct ib_sa_path_query, sa_query);</div><div class='add'>+	struct sa_path_rec rec = {};</div><div class='ctx'> </div><div class='del'>-	if (mad) {</div><div class='del'>-		struct sa_path_rec rec;</div><div class='add'>+	if (!mad) {</div><div class='add'>+		query-&gt;callback(status, NULL, 0, query-&gt;context);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		if (sa_query-&gt;flags &amp; IB_SA_QUERY_OPA) {</div><div class='del'>-			ib_unpack(opa_path_rec_table,</div><div class='del'>-				  ARRAY_SIZE(opa_path_rec_table),</div><div class='del'>-				  mad-&gt;data, &amp;rec);</div><div class='del'>-			rec.rec_type = SA_PATH_REC_TYPE_OPA;</div><div class='del'>-			query-&gt;callback(status, &amp;rec, query-&gt;context);</div><div class='del'>-		} else {</div><div class='del'>-			ib_unpack(path_rec_table,</div><div class='del'>-				  ARRAY_SIZE(path_rec_table),</div><div class='del'>-				  mad-&gt;data, &amp;rec);</div><div class='del'>-			rec.rec_type = SA_PATH_REC_TYPE_IB;</div><div class='del'>-			sa_path_set_dmac_zero(&amp;rec);</div><div class='add'>+	if (sa_query-&gt;flags &amp; IB_SA_QUERY_OPA) {</div><div class='add'>+		ib_unpack(opa_path_rec_table, ARRAY_SIZE(opa_path_rec_table),</div><div class='add'>+			  mad-&gt;data, &amp;rec);</div><div class='add'>+		rec.rec_type = SA_PATH_REC_TYPE_OPA;</div><div class='add'>+		query-&gt;callback(status, &amp;rec, 1, query-&gt;context);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-			if (query-&gt;conv_pr) {</div><div class='del'>-				struct sa_path_rec opa;</div><div class='add'>+	ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table),</div><div class='add'>+		  mad-&gt;data, &amp;rec);</div><div class='add'>+	rec.rec_type = SA_PATH_REC_TYPE_IB;</div><div class='add'>+	sa_path_set_dmac_zero(&amp;rec);</div><div class='ctx'> </div><div class='del'>-				memset(&amp;opa, 0, sizeof(struct sa_path_rec));</div><div class='del'>-				sa_convert_path_ib_to_opa(&amp;opa, &amp;rec);</div><div class='del'>-				query-&gt;callback(status, &amp;opa, query-&gt;context);</div><div class='del'>-			} else {</div><div class='del'>-				query-&gt;callback(status, &amp;rec, query-&gt;context);</div><div class='add'>+	if (query-&gt;conv_pr) {</div><div class='add'>+		struct sa_path_rec opa;</div><div class='add'>+</div><div class='add'>+		memset(&amp;opa, 0, sizeof(struct sa_path_rec));</div><div class='add'>+		sa_convert_path_ib_to_opa(&amp;opa, &amp;rec);</div><div class='add'>+		query-&gt;callback(status, &amp;opa, 1, query-&gt;context);</div><div class='add'>+	} else {</div><div class='add'>+		query-&gt;callback(status, &amp;rec, 1, query-&gt;context);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define IB_SA_DATA_OFFS 56</div><div class='add'>+#define IB_SERVICE_REC_SZ 176</div><div class='add'>+</div><div class='add'>+static void ib_unpack_service_rmpp(struct sa_service_rec *rec,</div><div class='add'>+				   struct ib_mad_recv_wc *mad_wc,</div><div class='add'>+				   int num_services)</div><div class='add'>+{</div><div class='add'>+	unsigned int cp_sz, data_i, data_size, rec_i = 0, buf_i = 0;</div><div class='add'>+	struct ib_mad_recv_buf *mad_buf;</div><div class='add'>+	u8 buf[IB_SERVICE_REC_SZ];</div><div class='add'>+	u8 *data;</div><div class='add'>+</div><div class='add'>+	data_size = sizeof(((struct ib_sa_mad *) mad_buf-&gt;mad)-&gt;data);</div><div class='add'>+</div><div class='add'>+	list_for_each_entry(mad_buf, &amp;mad_wc-&gt;rmpp_list, list) {</div><div class='add'>+		data = ((struct ib_sa_mad *) mad_buf-&gt;mad)-&gt;data;</div><div class='add'>+		data_i = 0;</div><div class='add'>+		while (data_i &lt; data_size &amp;&amp; rec_i &lt; num_services) {</div><div class='add'>+			cp_sz = min(IB_SERVICE_REC_SZ - buf_i,</div><div class='add'>+				    data_size - data_i);</div><div class='add'>+			memcpy(buf + buf_i, data + data_i, cp_sz);</div><div class='add'>+			data_i += cp_sz;</div><div class='add'>+			buf_i += cp_sz;</div><div class='add'>+			if (buf_i == IB_SERVICE_REC_SZ) {</div><div class='add'>+				ib_sa_unpack_service(buf, rec + rec_i);</div><div class='add'>+				buf_i = 0;</div><div class='add'>+				rec_i++;</div><div class='ctx'> 			}</div><div class='ctx'> 		}</div><div class='del'>-	} else</div><div class='del'>-		query-&gt;callback(status, NULL, query-&gt;context);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ib_sa_service_rec_callback(struct ib_sa_query *sa_query, int status,</div><div class='add'>+				       struct ib_mad_recv_wc *mad_wc)</div><div class='add'>+{</div><div class='add'>+	struct ib_sa_service_query *query =</div><div class='add'>+		container_of(sa_query, struct ib_sa_service_query, sa_query);</div><div class='add'>+	struct sa_service_rec *rec;</div><div class='add'>+	int num_services;</div><div class='add'>+</div><div class='add'>+	if (!mad_wc || !mad_wc-&gt;recv_buf.mad) {</div><div class='add'>+		query-&gt;callback(status, NULL, 0, query-&gt;context);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	num_services = (mad_wc-&gt;mad_len - IB_SA_DATA_OFFS) / IB_SERVICE_REC_SZ;</div><div class='add'>+	if (!num_services) {</div><div class='add'>+		query-&gt;callback(-ENODATA, NULL, 0, query-&gt;context);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rec = kmalloc_array(num_services, sizeof(*rec), GFP_KERNEL);</div><div class='add'>+	if (!rec) {</div><div class='add'>+		query-&gt;callback(-ENOMEM, NULL, 0, query-&gt;context);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ib_unpack_service_rmpp(rec, mad_wc, num_services);</div><div class='add'>+	query-&gt;callback(status, rec, num_services, query-&gt;context);</div><div class='add'>+	kfree(rec);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void ib_sa_path_rec_release(struct ib_sa_query *sa_query)</div><div class='hunk'>@@ -1514,6 +1627,14 @@ static void ib_sa_path_rec_release(struct ib_sa_query *sa_query)</div><div class='ctx'> 	kfree(query);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void ib_sa_service_rec_release(struct ib_sa_query *sa_query)</div><div class='add'>+{</div><div class='add'>+	struct ib_sa_service_query *query =</div><div class='add'>+		container_of(sa_query, struct ib_sa_service_query, sa_query);</div><div class='add'>+</div><div class='add'>+	kfree(query);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /**</div><div class='ctx'>  * ib_sa_path_rec_get - Start a Path get query</div><div class='ctx'>  * @client:SA client</div><div class='hunk'>@@ -1540,13 +1661,13 @@ static void ib_sa_path_rec_release(struct ib_sa_query *sa_query)</div><div class='ctx'>  * the query.</div><div class='ctx'>  */</div><div class='ctx'> int ib_sa_path_rec_get(struct ib_sa_client *client,</div><div class='del'>-		       struct ib_device *device, u8 port_num,</div><div class='add'>+		       struct ib_device *device, u32 port_num,</div><div class='ctx'> 		       struct sa_path_rec *rec,</div><div class='ctx'> 		       ib_sa_comp_mask comp_mask,</div><div class='ctx'> 		       unsigned long timeout_ms, gfp_t gfp_mask,</div><div class='ctx'> 		       void (*callback)(int status,</div><div class='ctx'> 					struct sa_path_rec *resp,</div><div class='del'>-					void *context),</div><div class='add'>+					unsigned int num_paths, void *context),</div><div class='ctx'> 		       void *context,</div><div class='ctx'> 		       struct ib_sa_query **sa_query)</div><div class='ctx'> {</div><div class='hunk'>@@ -1574,7 +1695,7 @@ int ib_sa_path_rec_get(struct ib_sa_client *client,</div><div class='ctx'> </div><div class='ctx'> 	query-&gt;sa_query.port     = port;</div><div class='ctx'> 	if (rec-&gt;rec_type == SA_PATH_REC_TYPE_OPA) {</div><div class='del'>-		status = opa_pr_query_possible(client, device, port_num, rec);</div><div class='add'>+		status = opa_pr_query_possible(client, sa_dev, device, port_num);</div><div class='ctx'> 		if (status == PR_NOT_SUPPORTED) {</div><div class='ctx'> 			ret = -EINVAL;</div><div class='ctx'> 			goto err1;</div><div class='hunk'>@@ -1644,89 +1765,61 @@ err1:</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_sa_path_rec_get);</div><div class='ctx'> </div><div class='del'>-static void ib_sa_service_rec_callback(struct ib_sa_query *sa_query,</div><div class='del'>-				    int status,</div><div class='del'>-				    struct ib_sa_mad *mad)</div><div class='del'>-{</div><div class='del'>-	struct ib_sa_service_query *query =</div><div class='del'>-		container_of(sa_query, struct ib_sa_service_query, sa_query);</div><div class='del'>-</div><div class='del'>-	if (mad) {</div><div class='del'>-		struct ib_sa_service_rec rec;</div><div class='del'>-</div><div class='del'>-		ib_unpack(service_rec_table, ARRAY_SIZE(service_rec_table),</div><div class='del'>-			  mad-&gt;data, &amp;rec);</div><div class='del'>-		query-&gt;callback(status, &amp;rec, query-&gt;context);</div><div class='del'>-	} else</div><div class='del'>-		query-&gt;callback(status, NULL, query-&gt;context);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void ib_sa_service_rec_release(struct ib_sa_query *sa_query)</div><div class='del'>-{</div><div class='del'>-	kfree(container_of(sa_query, struct ib_sa_service_query, sa_query));</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> /**</div><div class='del'>- * ib_sa_service_rec_query - Start Service Record operation</div><div class='del'>- * @client:SA client</div><div class='del'>- * @device:device to send request on</div><div class='del'>- * @port_num: port number to send request on</div><div class='del'>- * @method:SA method - should be get, set, or delete</div><div class='del'>- * @rec:Service Record to send in request</div><div class='del'>- * @comp_mask:component mask to send in request</div><div class='del'>- * @timeout_ms:time to wait for response</div><div class='del'>- * @gfp_mask:GFP mask to use for internal allocations</div><div class='del'>- * @callback:function called when request completes, times out or is</div><div class='add'>+ * ib_sa_service_rec_get - Start a Service get query</div><div class='add'>+ * @client: SA client</div><div class='add'>+ * @device: device to send query on</div><div class='add'>+ * @port_num: port number to send query on</div><div class='add'>+ * @rec: Service Record to send in query</div><div class='add'>+ * @comp_mask: component mask to send in query</div><div class='add'>+ * @timeout_ms: time to wait for response</div><div class='add'>+ * @gfp_mask: GFP mask to use for internal allocations</div><div class='add'>+ * @callback: function called when query completes, times out or is</div><div class='ctx'>  * canceled</div><div class='del'>- * @context:opaque user context passed to callback</div><div class='del'>- * @sa_query:request context, used to cancel request</div><div class='add'>+ * @context: opaque user context passed to callback</div><div class='add'>+ * @sa_query: query context, used to cancel query</div><div class='ctx'>  *</div><div class='del'>- * Send a Service Record set/get/delete to the SA to register,</div><div class='del'>- * unregister or query a service record.</div><div class='del'>- * The callback function will be called when the request completes (or</div><div class='add'>+ * Send a Service Record Get query to the SA to look up a path.  The</div><div class='add'>+ * callback function will be called when the query completes (or</div><div class='ctx'>  * fails); status is 0 for a successful response, -EINTR if the query</div><div class='ctx'>  * is canceled, -ETIMEDOUT is the query timed out, or -EIO if an error</div><div class='ctx'>  * occurred sending the query.  The resp parameter of the callback is</div><div class='ctx'>  * only valid if status is 0.</div><div class='ctx'>  *</div><div class='del'>- * If the return value of ib_sa_service_rec_query() is negative, it is an</div><div class='del'>- * error code.  Otherwise it is a request ID that can be used to cancel</div><div class='add'>+ * If the return value of ib_sa_service_rec_get() is negative, it is an</div><div class='add'>+ * error code. Otherwise it is a query ID that can be used to cancel</div><div class='ctx'>  * the query.</div><div class='ctx'>  */</div><div class='del'>-int ib_sa_service_rec_query(struct ib_sa_client *client,</div><div class='del'>-			    struct ib_device *device, u8 port_num, u8 method,</div><div class='del'>-			    struct ib_sa_service_rec *rec,</div><div class='del'>-			    ib_sa_comp_mask comp_mask,</div><div class='del'>-			    unsigned long timeout_ms, gfp_t gfp_mask,</div><div class='del'>-			    void (*callback)(int status,</div><div class='del'>-					     struct ib_sa_service_rec *resp,</div><div class='del'>-					     void *context),</div><div class='del'>-			    void *context,</div><div class='del'>-			    struct ib_sa_query **sa_query)</div><div class='add'>+int ib_sa_service_rec_get(struct ib_sa_client *client,</div><div class='add'>+			  struct ib_device *device, u32 port_num,</div><div class='add'>+			  struct sa_service_rec *rec,</div><div class='add'>+			  ib_sa_comp_mask comp_mask,</div><div class='add'>+			  unsigned long timeout_ms, gfp_t gfp_mask,</div><div class='add'>+			  void (*callback)(int status,</div><div class='add'>+					   struct sa_service_rec *resp,</div><div class='add'>+					   unsigned int num_services,</div><div class='add'>+					   void *context),</div><div class='add'>+			  void *context, struct ib_sa_query **sa_query)</div><div class='ctx'> {</div><div class='del'>-	struct ib_sa_service_query *query;</div><div class='ctx'> 	struct ib_sa_device *sa_dev = ib_get_client_data(device, &amp;sa_client);</div><div class='del'>-	struct ib_sa_port   *port;</div><div class='add'>+	struct ib_sa_service_query *query;</div><div class='ctx'> 	struct ib_mad_agent *agent;</div><div class='add'>+	struct ib_sa_port   *port;</div><div class='ctx'> 	struct ib_sa_mad *mad;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (!sa_dev)</div><div class='ctx'> 		return -ENODEV;</div><div class='ctx'> </div><div class='del'>-	port  = &amp;sa_dev-&gt;port[port_num - sa_dev-&gt;start_port];</div><div class='add'>+	port = &amp;sa_dev-&gt;port[port_num - sa_dev-&gt;start_port];</div><div class='ctx'> 	agent = port-&gt;agent;</div><div class='ctx'> </div><div class='del'>-	if (method != IB_MGMT_METHOD_GET &amp;&amp;</div><div class='del'>-	    method != IB_MGMT_METHOD_SET &amp;&amp;</div><div class='del'>-	    method != IB_SA_METHOD_DELETE)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='ctx'> 	query = kzalloc(sizeof(*query), gfp_mask);</div><div class='ctx'> 	if (!query)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	query-&gt;sa_query.port     = port;</div><div class='add'>+	query-&gt;sa_query.port = port;</div><div class='add'>+</div><div class='ctx'> 	ret = alloc_mad(&amp;query-&gt;sa_query, gfp_mask);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err1;</div><div class='hunk'>@@ -1739,16 +1832,17 @@ int ib_sa_service_rec_query(struct ib_sa_client *client,</div><div class='ctx'> 	mad = query-&gt;sa_query.mad_buf-&gt;mad;</div><div class='ctx'> 	init_mad(&amp;query-&gt;sa_query, agent);</div><div class='ctx'> </div><div class='del'>-	query-&gt;sa_query.callback = callback ? ib_sa_service_rec_callback : NULL;</div><div class='del'>-	query-&gt;sa_query.release  = ib_sa_service_rec_release;</div><div class='del'>-	mad-&gt;mad_hdr.method	 = method;</div><div class='del'>-	mad-&gt;mad_hdr.attr_id	 = cpu_to_be16(IB_SA_ATTR_SERVICE_REC);</div><div class='del'>-	mad-&gt;sa_hdr.comp_mask	 = comp_mask;</div><div class='add'>+	query-&gt;sa_query.rmpp_callback = callback ? ib_sa_service_rec_callback :</div><div class='add'>+		NULL;</div><div class='add'>+	query-&gt;sa_query.release = ib_sa_service_rec_release;</div><div class='add'>+	mad-&gt;mad_hdr.method	= IB_MGMT_METHOD_GET_TABLE;</div><div class='add'>+	mad-&gt;mad_hdr.attr_id	= cpu_to_be16(IB_SA_ATTR_SERVICE_REC);</div><div class='add'>+	mad-&gt;sa_hdr.comp_mask	= comp_mask;</div><div class='ctx'> </div><div class='del'>-	ib_pack(service_rec_table, ARRAY_SIZE(service_rec_table),</div><div class='del'>-		rec, mad-&gt;data);</div><div class='add'>+	ib_sa_pack_service(rec, mad-&gt;data);</div><div class='ctx'> </div><div class='ctx'> 	*sa_query = &amp;query-&gt;sa_query;</div><div class='add'>+	query-&gt;sa_query.mad_buf-&gt;context[1] = rec;</div><div class='ctx'> </div><div class='ctx'> 	ret = send_mad(&amp;query-&gt;sa_query, timeout_ms, gfp_mask);</div><div class='ctx'> 	if (ret &lt; 0)</div><div class='hunk'>@@ -1760,16 +1854,14 @@ err2:</div><div class='ctx'> 	*sa_query = NULL;</div><div class='ctx'> 	ib_sa_client_put(query-&gt;sa_query.client);</div><div class='ctx'> 	free_mad(&amp;query-&gt;sa_query);</div><div class='del'>-</div><div class='ctx'> err1:</div><div class='ctx'> 	kfree(query);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(ib_sa_service_rec_query);</div><div class='add'>+EXPORT_SYMBOL(ib_sa_service_rec_get);</div><div class='ctx'> </div><div class='ctx'> static void ib_sa_mcmember_rec_callback(struct ib_sa_query *sa_query,</div><div class='del'>-					int status,</div><div class='del'>-					struct ib_sa_mad *mad)</div><div class='add'>+					int status, struct ib_sa_mad *mad)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_sa_mcmember_query *query =</div><div class='ctx'> 		container_of(sa_query, struct ib_sa_mcmember_query, sa_query);</div><div class='hunk'>@@ -1790,7 +1882,7 @@ static void ib_sa_mcmember_rec_release(struct ib_sa_query *sa_query)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int ib_sa_mcmember_rec_query(struct ib_sa_client *client,</div><div class='del'>-			     struct ib_device *device, u8 port_num,</div><div class='add'>+			     struct ib_device *device, u32 port_num,</div><div class='ctx'> 			     u8 method,</div><div class='ctx'> 			     struct ib_sa_mcmember_rec *rec,</div><div class='ctx'> 			     ib_sa_comp_mask comp_mask,</div><div class='hunk'>@@ -1860,8 +1952,7 @@ err1:</div><div class='ctx'> </div><div class='ctx'> /* Support GuidInfoRecord */</div><div class='ctx'> static void ib_sa_guidinfo_rec_callback(struct ib_sa_query *sa_query,</div><div class='del'>-					int status,</div><div class='del'>-					struct ib_sa_mad *mad)</div><div class='add'>+					int status, struct ib_sa_mad *mad)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_sa_guidinfo_query *query =</div><div class='ctx'> 		container_of(sa_query, struct ib_sa_guidinfo_query, sa_query);</div><div class='hunk'>@@ -1882,7 +1973,7 @@ static void ib_sa_guidinfo_rec_release(struct ib_sa_query *sa_query)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int ib_sa_guid_info_rec_query(struct ib_sa_client *client,</div><div class='del'>-			      struct ib_device *device, u8 port_num,</div><div class='add'>+			      struct ib_device *device, u32 port_num,</div><div class='ctx'> 			      struct ib_sa_guidinfo_rec *rec,</div><div class='ctx'> 			      ib_sa_comp_mask comp_mask, u8 method,</div><div class='ctx'> 			      unsigned long timeout_ms, gfp_t gfp_mask,</div><div class='hunk'>@@ -1957,30 +2048,6 @@ err1:</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_sa_guid_info_rec_query);</div><div class='ctx'> </div><div class='del'>-bool ib_sa_sendonly_fullmem_support(struct ib_sa_client *client,</div><div class='del'>-				    struct ib_device *device,</div><div class='del'>-				    u8 port_num)</div><div class='del'>-{</div><div class='del'>-	struct ib_sa_device *sa_dev = ib_get_client_data(device, &amp;sa_client);</div><div class='del'>-	struct ib_sa_port *port;</div><div class='del'>-	bool ret = false;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	if (!sa_dev)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	port  = &amp;sa_dev-&gt;port[port_num - sa_dev-&gt;start_port];</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;port-&gt;classport_lock, flags);</div><div class='del'>-	if ((port-&gt;classport_info.valid) &amp;&amp;</div><div class='del'>-	    (port-&gt;classport_info.data.type == RDMA_CLASS_PORT_INFO_IB))</div><div class='del'>-		ret = ib_get_cpi_capmask2(&amp;port-&gt;classport_info.data.ib)</div><div class='del'>-			&amp; IB_SA_CAP_MASK2_SENDONLY_FULL_MEM_SUPPORT;</div><div class='del'>-	spin_unlock_irqrestore(&amp;port-&gt;classport_lock, flags);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_sa_sendonly_fullmem_support);</div><div class='del'>-</div><div class='ctx'> struct ib_classport_info_context {</div><div class='ctx'> 	struct completion	done;</div><div class='ctx'> 	struct ib_sa_query	*sa_query;</div><div class='hunk'>@@ -1994,8 +2061,7 @@ static void ib_classportinfo_cb(void *context)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void ib_sa_classport_info_rec_callback(struct ib_sa_query *sa_query,</div><div class='del'>-					      int status,</div><div class='del'>-					      struct ib_sa_mad *mad)</div><div class='add'>+					      int status, struct ib_sa_mad *mad)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	struct ib_sa_classport_info_query *query =</div><div class='hunk'>@@ -2163,23 +2229,29 @@ static void send_handler(struct ib_mad_agent *agent,</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_sa_query *query = mad_send_wc-&gt;send_buf-&gt;context[0];</div><div class='ctx'> 	unsigned long flags;</div><div class='add'>+	int status = 0;</div><div class='ctx'> </div><div class='del'>-	if (query-&gt;callback)</div><div class='add'>+	if (query-&gt;callback || query-&gt;rmpp_callback) {</div><div class='ctx'> 		switch (mad_send_wc-&gt;status) {</div><div class='ctx'> 		case IB_WC_SUCCESS:</div><div class='ctx'> 			/* No callback -- already got recv */</div><div class='ctx'> 			break;</div><div class='ctx'> 		case IB_WC_RESP_TIMEOUT_ERR:</div><div class='del'>-			query-&gt;callback(query, -ETIMEDOUT, NULL);</div><div class='add'>+			status = -ETIMEDOUT;</div><div class='ctx'> 			break;</div><div class='ctx'> 		case IB_WC_WR_FLUSH_ERR:</div><div class='del'>-			query-&gt;callback(query, -EINTR, NULL);</div><div class='add'>+			status = -EINTR;</div><div class='ctx'> 			break;</div><div class='ctx'> 		default:</div><div class='del'>-			query-&gt;callback(query, -EIO, NULL);</div><div class='add'>+			status = -EIO;</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='add'>+		if (status)</div><div class='add'>+			query-&gt;callback ? query-&gt;callback(query, status, NULL) :</div><div class='add'>+				query-&gt;rmpp_callback(query, status, NULL);</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	xa_lock_irqsave(&amp;queries, flags);</div><div class='ctx'> 	__xa_erase(&amp;queries, query-&gt;id);</div><div class='ctx'> 	xa_unlock_irqrestore(&amp;queries, flags);</div><div class='hunk'>@@ -2195,17 +2267,25 @@ static void recv_handler(struct ib_mad_agent *mad_agent,</div><div class='ctx'> 			 struct ib_mad_recv_wc *mad_recv_wc)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_sa_query *query;</div><div class='add'>+	struct ib_mad *mad;</div><div class='add'>+</div><div class='ctx'> </div><div class='ctx'> 	if (!send_buf)</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='ctx'> 	query = send_buf-&gt;context[0];</div><div class='del'>-	if (query-&gt;callback) {</div><div class='add'>+	mad = mad_recv_wc-&gt;recv_buf.mad;</div><div class='add'>+</div><div class='add'>+	if (query-&gt;rmpp_callback) {</div><div class='add'>+		if (mad_recv_wc-&gt;wc-&gt;status == IB_WC_SUCCESS)</div><div class='add'>+			query-&gt;rmpp_callback(query, mad-&gt;mad_hdr.status ?</div><div class='add'>+					     -EINVAL : 0, mad_recv_wc);</div><div class='add'>+		else</div><div class='add'>+			query-&gt;rmpp_callback(query, -EIO, NULL);</div><div class='add'>+	} else if (query-&gt;callback) {</div><div class='ctx'> 		if (mad_recv_wc-&gt;wc-&gt;status == IB_WC_SUCCESS)</div><div class='del'>-			query-&gt;callback(query,</div><div class='del'>-					mad_recv_wc-&gt;recv_buf.mad-&gt;mad_hdr.status ?</div><div class='del'>-					-EINVAL : 0,</div><div class='del'>-					(struct ib_sa_mad *) mad_recv_wc-&gt;recv_buf.mad);</div><div class='add'>+			query-&gt;callback(query, mad-&gt;mad_hdr.status ?</div><div class='add'>+					-EINVAL : 0, (struct ib_sa_mad *)mad);</div><div class='ctx'> 		else</div><div class='ctx'> 			query-&gt;callback(query, -EIO, NULL);</div><div class='ctx'> 	}</div><div class='hunk'>@@ -2295,7 +2375,7 @@ static void ib_sa_event(struct ib_event_handler *handler,</div><div class='ctx'> 		unsigned long flags;</div><div class='ctx'> 		struct ib_sa_device *sa_dev =</div><div class='ctx'> 			container_of(handler, typeof(*sa_dev), event_handler);</div><div class='del'>-		u8 port_num = event-&gt;element.port_num - sa_dev-&gt;start_port;</div><div class='add'>+		u32 port_num = event-&gt;element.port_num - sa_dev-&gt;start_port;</div><div class='ctx'> 		struct ib_sa_port *port = &amp;sa_dev-&gt;port[port_num];</div><div class='ctx'> </div><div class='ctx'> 		if (!rdma_cap_ib_sa(handler-&gt;device, port-&gt;port_num))</div><div class='hunk'>@@ -2325,18 +2405,21 @@ static void ib_sa_event(struct ib_event_handler *handler,</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void ib_sa_add_one(struct ib_device *device)</div><div class='add'>+static int ib_sa_add_one(struct ib_device *device)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_sa_device *sa_dev;</div><div class='ctx'> 	int s, e, i;</div><div class='ctx'> 	int count = 0;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	s = rdma_start_port(device);</div><div class='ctx'> 	e = rdma_end_port(device);</div><div class='ctx'> </div><div class='del'>-	sa_dev = kzalloc(struct_size(sa_dev, port, e - s + 1), GFP_KERNEL);</div><div class='add'>+	sa_dev = kzalloc(struct_size(sa_dev, port,</div><div class='add'>+				     size_add(size_sub(e, s), 1)),</div><div class='add'>+			 GFP_KERNEL);</div><div class='ctx'> 	if (!sa_dev)</div><div class='del'>-		return;</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	sa_dev-&gt;start_port = s;</div><div class='ctx'> 	sa_dev-&gt;end_port   = e;</div><div class='hunk'>@@ -2354,10 +2437,13 @@ static void ib_sa_add_one(struct ib_device *device)</div><div class='ctx'> </div><div class='ctx'> 		sa_dev-&gt;port[i].agent =</div><div class='ctx'> 			ib_register_mad_agent(device, i + s, IB_QPT_GSI,</div><div class='del'>-					      NULL, 0, send_handler,</div><div class='del'>-					      recv_handler, sa_dev, 0);</div><div class='del'>-		if (IS_ERR(sa_dev-&gt;port[i].agent))</div><div class='add'>+					      NULL, IB_MGMT_RMPP_VERSION,</div><div class='add'>+					      send_handler, recv_handler,</div><div class='add'>+					      sa_dev, 0);</div><div class='add'>+		if (IS_ERR(sa_dev-&gt;port[i].agent)) {</div><div class='add'>+			ret = PTR_ERR(sa_dev-&gt;port[i].agent);</div><div class='ctx'> 			goto err;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='ctx'> 		INIT_WORK(&amp;sa_dev-&gt;port[i].update_task, update_sm_ah);</div><div class='ctx'> 		INIT_DELAYED_WORK(&amp;sa_dev-&gt;port[i].ib_cpi_work,</div><div class='hunk'>@@ -2366,8 +2452,10 @@ static void ib_sa_add_one(struct ib_device *device)</div><div class='ctx'> 		count++;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!count)</div><div class='add'>+	if (!count) {</div><div class='add'>+		ret = -EOPNOTSUPP;</div><div class='ctx'> 		goto free;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	ib_set_client_data(device, &amp;sa_client, sa_dev);</div><div class='ctx'> </div><div class='hunk'>@@ -2386,7 +2474,7 @@ static void ib_sa_add_one(struct ib_device *device)</div><div class='ctx'> 			update_sm_ah(&amp;sa_dev-&gt;port[i].update_task);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='ctx'> err:</div><div class='ctx'> 	while (--i &gt;= 0) {</div><div class='hunk'>@@ -2395,7 +2483,7 @@ err:</div><div class='ctx'> 	}</div><div class='ctx'> free:</div><div class='ctx'> 	kfree(sa_dev);</div><div class='del'>-	return;</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void ib_sa_remove_one(struct ib_device *device, void *client_data)</div><div class='hunk'>@@ -2403,9 +2491,6 @@ static void ib_sa_remove_one(struct ib_device *device, void *client_data)</div><div class='ctx'> 	struct ib_sa_device *sa_dev = client_data;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	if (!sa_dev)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='ctx'> 	ib_unregister_event_handler(&amp;sa_dev-&gt;event_handler);</div><div class='ctx'> 	flush_workqueue(ib_wq);</div><div class='ctx'> </div><div class='hunk'>@@ -2463,7 +2548,6 @@ err1:</div><div class='ctx'> void ib_sa_cleanup(void)</div><div class='ctx'> {</div><div class='ctx'> 	cancel_delayed_work(&amp;ib_nl_timed_work);</div><div class='del'>-	flush_workqueue(ib_nl_wq);</div><div class='ctx'> 	destroy_workqueue(ib_nl_wq);</div><div class='ctx'> 	mcast_cleanup();</div><div class='ctx'> 	ib_unregister_client(&amp;sa_client);</div><div class='head'>diff --git a/drivers/infiniband/core/security.c b/drivers/infiniband/core/security.c<br/>index 1ab423b19f77..3512c2e54efc 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/security.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/security.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/security.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/security.c</a></div><div class='hunk'>@@ -72,7 +72,7 @@ static int get_pkey_and_subnet_prefix(struct ib_port_pkey *pp,</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_get_cached_subnet_prefix(dev, pp-&gt;port_num, subnet_prefix);</div><div class='add'>+	ib_get_cached_subnet_prefix(dev, pp-&gt;port_num, subnet_prefix);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -193,7 +193,7 @@ static void qp_to_error(struct ib_qp_security *sec)</div><div class='ctx'> </div><div class='ctx'> static inline void check_pkey_qps(struct pkey_index_qp_list *pkey,</div><div class='ctx'> 				  struct ib_device *device,</div><div class='del'>-				  u8 port_num,</div><div class='add'>+				  u32 port_num,</div><div class='ctx'> 				  u64 subnet_prefix)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_pkey *pp, *tmp_pp;</div><div class='hunk'>@@ -245,7 +245,7 @@ static int port_pkey_list_insert(struct ib_port_pkey *pp)</div><div class='ctx'> 	struct pkey_index_qp_list *tmp_pkey;</div><div class='ctx'> 	struct pkey_index_qp_list *pkey;</div><div class='ctx'> 	struct ib_device *dev;</div><div class='del'>-	u8 port_num = pp-&gt;port_num;</div><div class='add'>+	u32 port_num = pp-&gt;port_num;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='ctx'> 	if (pp-&gt;state != IB_PORT_PKEY_VALID)</div><div class='hunk'>@@ -339,27 +339,20 @@ static struct ib_ports_pkeys *get_new_pps(const struct ib_qp *qp,</div><div class='ctx'> 	if (!new_pps)</div><div class='ctx'> 		return NULL;</div><div class='ctx'> </div><div class='del'>-	if (qp_attr_mask &amp; (IB_QP_PKEY_INDEX | IB_QP_PORT)) {</div><div class='del'>-		if (!qp_pps) {</div><div class='del'>-			new_pps-&gt;main.port_num = qp_attr-&gt;port_num;</div><div class='del'>-			new_pps-&gt;main.pkey_index = qp_attr-&gt;pkey_index;</div><div class='del'>-		} else {</div><div class='del'>-			new_pps-&gt;main.port_num = (qp_attr_mask &amp; IB_QP_PORT) ?</div><div class='del'>-						  qp_attr-&gt;port_num :</div><div class='del'>-						  qp_pps-&gt;main.port_num;</div><div class='del'>-</div><div class='del'>-			new_pps-&gt;main.pkey_index =</div><div class='del'>-					(qp_attr_mask &amp; IB_QP_PKEY_INDEX) ?</div><div class='del'>-					 qp_attr-&gt;pkey_index :</div><div class='del'>-					 qp_pps-&gt;main.pkey_index;</div><div class='del'>-		}</div><div class='del'>-		new_pps-&gt;main.state = IB_PORT_PKEY_VALID;</div><div class='del'>-	} else if (qp_pps) {</div><div class='add'>+	if (qp_attr_mask &amp; IB_QP_PORT)</div><div class='add'>+		new_pps-&gt;main.port_num = qp_attr-&gt;port_num;</div><div class='add'>+	else if (qp_pps)</div><div class='ctx'> 		new_pps-&gt;main.port_num = qp_pps-&gt;main.port_num;</div><div class='add'>+</div><div class='add'>+	if (qp_attr_mask &amp; IB_QP_PKEY_INDEX)</div><div class='add'>+		new_pps-&gt;main.pkey_index = qp_attr-&gt;pkey_index;</div><div class='add'>+	else if (qp_pps)</div><div class='ctx'> 		new_pps-&gt;main.pkey_index = qp_pps-&gt;main.pkey_index;</div><div class='del'>-		if (qp_pps-&gt;main.state != IB_PORT_PKEY_NOT_VALID)</div><div class='del'>-			new_pps-&gt;main.state = IB_PORT_PKEY_VALID;</div><div class='del'>-	}</div><div class='add'>+</div><div class='add'>+	if (((qp_attr_mask &amp; IB_QP_PKEY_INDEX) &amp;&amp;</div><div class='add'>+	     (qp_attr_mask &amp; IB_QP_PORT)) ||</div><div class='add'>+	    (qp_pps &amp;&amp; qp_pps-&gt;main.state != IB_PORT_PKEY_NOT_VALID))</div><div class='add'>+		new_pps-&gt;main.state = IB_PORT_PKEY_VALID;</div><div class='ctx'> </div><div class='ctx'> 	if (qp_attr_mask &amp; IB_QP_ALT_PATH) {</div><div class='ctx'> 		new_pps-&gt;alt.port_num = qp_attr-&gt;alt_port_num;</div><div class='hunk'>@@ -426,7 +419,7 @@ int ib_create_qp_security(struct ib_qp *qp, struct ib_device *dev)</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	rdma_for_each_port (dev, i) {</div><div class='del'>-		is_ib = rdma_protocol_ib(dev, i++);</div><div class='add'>+		is_ib = rdma_protocol_ib(dev, i);</div><div class='ctx'> 		if (is_ib)</div><div class='ctx'> 			break;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -545,7 +538,7 @@ void ib_destroy_qp_security_end(struct ib_qp_security *sec)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void ib_security_cache_change(struct ib_device *device,</div><div class='del'>-			      u8 port_num,</div><div class='add'>+			      u32 port_num,</div><div class='ctx'> 			      u64 subnet_prefix)</div><div class='ctx'> {</div><div class='ctx'> 	struct pkey_index_qp_list *pkey;</div><div class='hunk'>@@ -593,7 +586,7 @@ int ib_security_modify_qp(struct ib_qp *qp,</div><div class='ctx'> 	WARN_ONCE((qp_attr_mask &amp; IB_QP_PORT &amp;&amp;</div><div class='ctx'> 		   rdma_protocol_ib(real_qp-&gt;device, qp_attr-&gt;port_num) &amp;&amp;</div><div class='ctx'> 		   !real_qp-&gt;qp_sec),</div><div class='del'>-		   "%s: QP security is not initialized for IB QP: %d\n",</div><div class='add'>+		   "%s: QP security is not initialized for IB QP: %u\n",</div><div class='ctx'> 		   __func__, real_qp-&gt;qp_num);</div><div class='ctx'> </div><div class='ctx'> 	/* The port/pkey settings are maintained only for the real QP. Open</div><div class='hunk'>@@ -656,7 +649,7 @@ int ib_security_modify_qp(struct ib_qp *qp,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int ib_security_pkey_access(struct ib_device *dev,</div><div class='del'>-				   u8 port_num,</div><div class='add'>+				   u32 port_num,</div><div class='ctx'> 				   u16 pkey_index,</div><div class='ctx'> 				   void *sec)</div><div class='ctx'> {</div><div class='hunk'>@@ -671,10 +664,7 @@ static int ib_security_pkey_access(struct ib_device *dev,</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_get_cached_subnet_prefix(dev, port_num, &amp;subnet_prefix);</div><div class='del'>-</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='add'>+	ib_get_cached_subnet_prefix(dev, port_num, &amp;subnet_prefix);</div><div class='ctx'> </div><div class='ctx'> 	return security_ib_pkey_access(sec, subnet_prefix, pkey);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/core/smi.c b/drivers/infiniband/core/smi.c<br/>index f19b23817c2b..45f09b75c893 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/smi.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/smi.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/smi.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/smi.c</a></div><div class='hunk'>@@ -41,7 +41,7 @@</div><div class='ctx'> #include "smi.h"</div><div class='ctx'> #include "opa_smi.h"</div><div class='ctx'> </div><div class='del'>-static enum smi_action __smi_handle_dr_smp_send(bool is_switch, int port_num,</div><div class='add'>+static enum smi_action __smi_handle_dr_smp_send(bool is_switch, u32 port_num,</div><div class='ctx'> 						u8 *hop_ptr, u8 hop_cnt,</div><div class='ctx'> 						const u8 *initial_path,</div><div class='ctx'> 						const u8 *return_path,</div><div class='hunk'>@@ -127,7 +127,7 @@ static enum smi_action __smi_handle_dr_smp_send(bool is_switch, int port_num,</div><div class='ctx'>  * Return IB_SMI_DISCARD if the SMP should be discarded</div><div class='ctx'>  */</div><div class='ctx'> enum smi_action smi_handle_dr_smp_send(struct ib_smp *smp,</div><div class='del'>-				       bool is_switch, int port_num)</div><div class='add'>+				       bool is_switch, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	return __smi_handle_dr_smp_send(is_switch, port_num,</div><div class='ctx'> 					&amp;smp-&gt;hop_ptr, smp-&gt;hop_cnt,</div><div class='hunk'>@@ -139,7 +139,7 @@ enum smi_action smi_handle_dr_smp_send(struct ib_smp *smp,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> enum smi_action opa_smi_handle_dr_smp_send(struct opa_smp *smp,</div><div class='del'>-				       bool is_switch, int port_num)</div><div class='add'>+				       bool is_switch, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	return __smi_handle_dr_smp_send(is_switch, port_num,</div><div class='ctx'> 					&amp;smp-&gt;hop_ptr, smp-&gt;hop_cnt,</div><div class='hunk'>@@ -152,7 +152,7 @@ enum smi_action opa_smi_handle_dr_smp_send(struct opa_smp *smp,</div><div class='ctx'> 					OPA_LID_PERMISSIVE);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static enum smi_action __smi_handle_dr_smp_recv(bool is_switch, int port_num,</div><div class='add'>+static enum smi_action __smi_handle_dr_smp_recv(bool is_switch, u32 port_num,</div><div class='ctx'> 						int phys_port_cnt,</div><div class='ctx'> 						u8 *hop_ptr, u8 hop_cnt,</div><div class='ctx'> 						const u8 *initial_path,</div><div class='hunk'>@@ -238,7 +238,7 @@ static enum smi_action __smi_handle_dr_smp_recv(bool is_switch, int port_num,</div><div class='ctx'>  * Return IB_SMI_DISCARD if the SMP should be dropped</div><div class='ctx'>  */</div><div class='ctx'> enum smi_action smi_handle_dr_smp_recv(struct ib_smp *smp, bool is_switch,</div><div class='del'>-				       int port_num, int phys_port_cnt)</div><div class='add'>+				       u32 port_num, int phys_port_cnt)</div><div class='ctx'> {</div><div class='ctx'> 	return __smi_handle_dr_smp_recv(is_switch, port_num, phys_port_cnt,</div><div class='ctx'> 					&amp;smp-&gt;hop_ptr, smp-&gt;hop_cnt,</div><div class='hunk'>@@ -254,7 +254,7 @@ enum smi_action smi_handle_dr_smp_recv(struct ib_smp *smp, bool is_switch,</div><div class='ctx'>  * Return IB_SMI_DISCARD if the SMP should be dropped</div><div class='ctx'>  */</div><div class='ctx'> enum smi_action opa_smi_handle_dr_smp_recv(struct opa_smp *smp, bool is_switch,</div><div class='del'>-					   int port_num, int phys_port_cnt)</div><div class='add'>+					   u32 port_num, int phys_port_cnt)</div><div class='ctx'> {</div><div class='ctx'> 	return __smi_handle_dr_smp_recv(is_switch, port_num, phys_port_cnt,</div><div class='ctx'> 					&amp;smp-&gt;hop_ptr, smp-&gt;hop_cnt,</div><div class='head'>diff --git a/drivers/infiniband/core/smi.h b/drivers/infiniband/core/smi.h<br/>index 91d9b353ab85..e350ed623c45 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/smi.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/smi.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/smi.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/smi.h</a></div><div class='hunk'>@@ -52,11 +52,11 @@ enum smi_forward_action {</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum smi_action smi_handle_dr_smp_recv(struct ib_smp *smp, bool is_switch,</div><div class='del'>-				       int port_num, int phys_port_cnt);</div><div class='add'>+				       u32 port_num, int phys_port_cnt);</div><div class='ctx'> int smi_get_fwd_port(struct ib_smp *smp);</div><div class='ctx'> extern enum smi_forward_action smi_check_forward_dr_smp(struct ib_smp *smp);</div><div class='ctx'> extern enum smi_action smi_handle_dr_smp_send(struct ib_smp *smp,</div><div class='del'>-					      bool is_switch, int port_num);</div><div class='add'>+					      bool is_switch, u32 port_num);</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * Return IB_SMI_HANDLE if the SMP should be handled by the local SMA/SM</div><div class='head'>diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c<br/>index c78d0c9646ae..0ed862b38b44 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/sysfs.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/sysfs.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/sysfs.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/sysfs.c</a></div><div class='hunk'>@@ -43,207 +43,261 @@</div><div class='ctx'> #include &lt;rdma/ib_mad.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_pma.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_cache.h&gt;</div><div class='add'>+#include &lt;rdma/rdma_counter.h&gt;</div><div class='add'>+#include &lt;rdma/ib_sysfs.h&gt;</div><div class='ctx'> </div><div class='del'>-struct ib_port;</div><div class='add'>+struct port_table_attribute {</div><div class='add'>+	struct ib_port_attribute attr;</div><div class='add'>+	char			name[8];</div><div class='add'>+	int			index;</div><div class='add'>+	__be16			attr_id;</div><div class='add'>+};</div><div class='ctx'> </div><div class='ctx'> struct gid_attr_group {</div><div class='del'>-	struct ib_port		*port;</div><div class='del'>-	struct kobject		kobj;</div><div class='del'>-	struct attribute_group	ndev;</div><div class='del'>-	struct attribute_group	type;</div><div class='add'>+	struct ib_port *port;</div><div class='add'>+	struct kobject kobj;</div><div class='add'>+	struct attribute_group groups[2];</div><div class='add'>+	const struct attribute_group *groups_list[3];</div><div class='add'>+	struct port_table_attribute attrs_list[];</div><div class='ctx'> };</div><div class='add'>+</div><div class='ctx'> struct ib_port {</div><div class='del'>-	struct kobject         kobj;</div><div class='del'>-	struct ib_device      *ibdev;</div><div class='add'>+	struct kobject kobj;</div><div class='add'>+	struct ib_device *ibdev;</div><div class='ctx'> 	struct gid_attr_group *gid_attr_group;</div><div class='del'>-	struct attribute_group gid_group;</div><div class='del'>-	struct attribute_group pkey_group;</div><div class='del'>-	struct attribute_group *pma_table;</div><div class='del'>-	struct attribute_group *hw_stats_ag;</div><div class='del'>-	struct rdma_hw_stats   *hw_stats;</div><div class='del'>-	u8                     port_num;</div><div class='add'>+	struct hw_stats_port_data *hw_stats_data;</div><div class='add'>+</div><div class='add'>+	struct attribute_group groups[3];</div><div class='add'>+	const struct attribute_group *groups_list[5];</div><div class='add'>+	u32 port_num;</div><div class='add'>+	struct port_table_attribute attrs_list[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct port_attribute {</div><div class='del'>-	struct attribute attr;</div><div class='del'>-	ssize_t (*show)(struct ib_port *, struct port_attribute *, char *buf);</div><div class='del'>-	ssize_t (*store)(struct ib_port *, struct port_attribute *,</div><div class='add'>+struct hw_stats_device_attribute {</div><div class='add'>+	struct device_attribute attr;</div><div class='add'>+	ssize_t (*show)(struct ib_device *ibdev, struct rdma_hw_stats *stats,</div><div class='add'>+			unsigned int index, unsigned int port_num, char *buf);</div><div class='add'>+	ssize_t (*store)(struct ib_device *ibdev, struct rdma_hw_stats *stats,</div><div class='add'>+			 unsigned int index, unsigned int port_num,</div><div class='ctx'> 			 const char *buf, size_t count);</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define PORT_ATTR(_name, _mode, _show, _store) \</div><div class='del'>-struct port_attribute port_attr_##_name = __ATTR(_name, _mode, _show, _store)</div><div class='del'>-</div><div class='del'>-#define PORT_ATTR_RO(_name) \</div><div class='del'>-struct port_attribute port_attr_##_name = __ATTR_RO(_name)</div><div class='add'>+struct hw_stats_port_attribute {</div><div class='add'>+	struct ib_port_attribute attr;</div><div class='add'>+	ssize_t (*show)(struct ib_device *ibdev, struct rdma_hw_stats *stats,</div><div class='add'>+			unsigned int index, unsigned int port_num, char *buf);</div><div class='add'>+	ssize_t (*store)(struct ib_device *ibdev, struct rdma_hw_stats *stats,</div><div class='add'>+			 unsigned int index, unsigned int port_num,</div><div class='add'>+			 const char *buf, size_t count);</div><div class='add'>+};</div><div class='ctx'> </div><div class='del'>-struct port_table_attribute {</div><div class='del'>-	struct port_attribute	attr;</div><div class='del'>-	char			name[8];</div><div class='del'>-	int			index;</div><div class='del'>-	__be16			attr_id;</div><div class='add'>+struct hw_stats_device_data {</div><div class='add'>+	struct attribute_group group;</div><div class='add'>+	struct rdma_hw_stats *stats;</div><div class='add'>+	struct hw_stats_device_attribute attrs[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct hw_stats_attribute {</div><div class='del'>-	struct attribute	attr;</div><div class='del'>-	ssize_t			(*show)(struct kobject *kobj,</div><div class='del'>-					struct attribute *attr, char *buf);</div><div class='del'>-	ssize_t			(*store)(struct kobject *kobj,</div><div class='del'>-					 struct attribute *attr,</div><div class='del'>-					 const char *buf,</div><div class='del'>-					 size_t count);</div><div class='del'>-	int			index;</div><div class='del'>-	u8			port_num;</div><div class='add'>+struct hw_stats_port_data {</div><div class='add'>+	struct rdma_hw_stats *stats;</div><div class='add'>+	struct hw_stats_port_attribute attrs[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static ssize_t port_attr_show(struct kobject *kobj,</div><div class='ctx'> 			      struct attribute *attr, char *buf)</div><div class='ctx'> {</div><div class='del'>-	struct port_attribute *port_attr =</div><div class='del'>-		container_of(attr, struct port_attribute, attr);</div><div class='add'>+	struct ib_port_attribute *port_attr =</div><div class='add'>+		container_of(attr, struct ib_port_attribute, attr);</div><div class='ctx'> 	struct ib_port *p = container_of(kobj, struct ib_port, kobj);</div><div class='ctx'> </div><div class='ctx'> 	if (!port_attr-&gt;show)</div><div class='ctx'> 		return -EIO;</div><div class='ctx'> </div><div class='del'>-	return port_attr-&gt;show(p, port_attr, buf);</div><div class='add'>+	return port_attr-&gt;show(p-&gt;ibdev, p-&gt;port_num, port_attr, buf);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t port_attr_store(struct kobject *kobj,</div><div class='ctx'> 			       struct attribute *attr,</div><div class='ctx'> 			       const char *buf, size_t count)</div><div class='ctx'> {</div><div class='del'>-	struct port_attribute *port_attr =</div><div class='del'>-		container_of(attr, struct port_attribute, attr);</div><div class='add'>+	struct ib_port_attribute *port_attr =</div><div class='add'>+		container_of(attr, struct ib_port_attribute, attr);</div><div class='ctx'> 	struct ib_port *p = container_of(kobj, struct ib_port, kobj);</div><div class='ctx'> </div><div class='ctx'> 	if (!port_attr-&gt;store)</div><div class='ctx'> 		return -EIO;</div><div class='del'>-	return port_attr-&gt;store(p, port_attr, buf, count);</div><div class='add'>+	return port_attr-&gt;store(p-&gt;ibdev, p-&gt;port_num, port_attr, buf, count);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+struct ib_device *ib_port_sysfs_get_ibdev_kobj(struct kobject *kobj,</div><div class='add'>+					       u32 *port_num)</div><div class='add'>+{</div><div class='add'>+	struct ib_port *port = container_of(kobj, struct ib_port, kobj);</div><div class='add'>+</div><div class='add'>+	*port_num = port-&gt;port_num;</div><div class='add'>+	return port-&gt;ibdev;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_port_sysfs_get_ibdev_kobj);</div><div class='add'>+</div><div class='ctx'> static const struct sysfs_ops port_sysfs_ops = {</div><div class='ctx'> 	.show	= port_attr_show,</div><div class='ctx'> 	.store	= port_attr_store</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static ssize_t hw_stat_device_show(struct device *dev,</div><div class='add'>+				   struct device_attribute *attr, char *buf)</div><div class='add'>+{</div><div class='add'>+	struct hw_stats_device_attribute *stat_attr =</div><div class='add'>+		container_of(attr, struct hw_stats_device_attribute, attr);</div><div class='add'>+	struct ib_device *ibdev = container_of(dev, struct ib_device, dev);</div><div class='add'>+</div><div class='add'>+	return stat_attr-&gt;show(ibdev, ibdev-&gt;hw_stats_data-&gt;stats,</div><div class='add'>+			       stat_attr - ibdev-&gt;hw_stats_data-&gt;attrs, 0, buf);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static ssize_t hw_stat_device_store(struct device *dev,</div><div class='add'>+				    struct device_attribute *attr,</div><div class='add'>+				    const char *buf, size_t count)</div><div class='add'>+{</div><div class='add'>+	struct hw_stats_device_attribute *stat_attr =</div><div class='add'>+		container_of(attr, struct hw_stats_device_attribute, attr);</div><div class='add'>+	struct ib_device *ibdev = container_of(dev, struct ib_device, dev);</div><div class='add'>+</div><div class='add'>+	return stat_attr-&gt;store(ibdev, ibdev-&gt;hw_stats_data-&gt;stats,</div><div class='add'>+				stat_attr - ibdev-&gt;hw_stats_data-&gt;attrs, 0, buf,</div><div class='add'>+				count);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static ssize_t hw_stat_port_show(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+				 struct ib_port_attribute *attr, char *buf)</div><div class='add'>+{</div><div class='add'>+	struct hw_stats_port_attribute *stat_attr =</div><div class='add'>+		container_of(attr, struct hw_stats_port_attribute, attr);</div><div class='add'>+	struct ib_port *port = ibdev-&gt;port_data[port_num].sysfs;</div><div class='add'>+</div><div class='add'>+	return stat_attr-&gt;show(ibdev, port-&gt;hw_stats_data-&gt;stats,</div><div class='add'>+			       stat_attr - port-&gt;hw_stats_data-&gt;attrs,</div><div class='add'>+			       port-&gt;port_num, buf);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static ssize_t hw_stat_port_store(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+				  struct ib_port_attribute *attr,</div><div class='add'>+				  const char *buf, size_t count)</div><div class='add'>+{</div><div class='add'>+	struct hw_stats_port_attribute *stat_attr =</div><div class='add'>+		container_of(attr, struct hw_stats_port_attribute, attr);</div><div class='add'>+	struct ib_port *port = ibdev-&gt;port_data[port_num].sysfs;</div><div class='add'>+</div><div class='add'>+	return stat_attr-&gt;store(ibdev, port-&gt;hw_stats_data-&gt;stats,</div><div class='add'>+				stat_attr - port-&gt;hw_stats_data-&gt;attrs,</div><div class='add'>+				port-&gt;port_num, buf, count);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static ssize_t gid_attr_show(struct kobject *kobj,</div><div class='ctx'> 			     struct attribute *attr, char *buf)</div><div class='ctx'> {</div><div class='del'>-	struct port_attribute *port_attr =</div><div class='del'>-		container_of(attr, struct port_attribute, attr);</div><div class='add'>+	struct ib_port_attribute *port_attr =</div><div class='add'>+		container_of(attr, struct ib_port_attribute, attr);</div><div class='ctx'> 	struct ib_port *p = container_of(kobj, struct gid_attr_group,</div><div class='ctx'> 					 kobj)-&gt;port;</div><div class='ctx'> </div><div class='ctx'> 	if (!port_attr-&gt;show)</div><div class='ctx'> 		return -EIO;</div><div class='ctx'> </div><div class='del'>-	return port_attr-&gt;show(p, port_attr, buf);</div><div class='add'>+	return port_attr-&gt;show(p-&gt;ibdev, p-&gt;port_num, port_attr, buf);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static const struct sysfs_ops gid_attr_sysfs_ops = {</div><div class='ctx'> 	.show = gid_attr_show</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static ssize_t state_show(struct ib_port *p, struct port_attribute *unused,</div><div class='del'>-			  char *buf)</div><div class='add'>+static ssize_t state_show(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+			  struct ib_port_attribute *unused, char *buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr attr;</div><div class='ctx'> 	ssize_t ret;</div><div class='ctx'> </div><div class='del'>-	static const char *state_name[] = {</div><div class='del'>-		[IB_PORT_NOP]		= "NOP",</div><div class='del'>-		[IB_PORT_DOWN]		= "DOWN",</div><div class='del'>-		[IB_PORT_INIT]		= "INIT",</div><div class='del'>-		[IB_PORT_ARMED]		= "ARMED",</div><div class='del'>-		[IB_PORT_ACTIVE]	= "ACTIVE",</div><div class='del'>-		[IB_PORT_ACTIVE_DEFER]	= "ACTIVE_DEFER"</div><div class='del'>-	};</div><div class='del'>-</div><div class='del'>-	ret = ib_query_port(p-&gt;ibdev, p-&gt;port_num, &amp;attr);</div><div class='add'>+	ret = ib_query_port(ibdev, port_num, &amp;attr);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%d: %s\n", attr.state,</div><div class='del'>-		       attr.state &gt;= 0 &amp;&amp; attr.state &lt; ARRAY_SIZE(state_name) ?</div><div class='del'>-		       state_name[attr.state] : "UNKNOWN");</div><div class='add'>+	return sysfs_emit(buf, "%d: %s\n", attr.state,</div><div class='add'>+			  ib_port_state_to_str(attr.state));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t lid_show(struct ib_port *p, struct port_attribute *unused,</div><div class='del'>-			char *buf)</div><div class='add'>+static ssize_t lid_show(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+			struct ib_port_attribute *unused, char *buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr attr;</div><div class='ctx'> 	ssize_t ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_query_port(p-&gt;ibdev, p-&gt;port_num, &amp;attr);</div><div class='add'>+	ret = ib_query_port(ibdev, port_num, &amp;attr);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "0x%x\n", attr.lid);</div><div class='add'>+	return sysfs_emit(buf, "0x%x\n", attr.lid);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t lid_mask_count_show(struct ib_port *p,</div><div class='del'>-				   struct port_attribute *unused,</div><div class='del'>-				   char *buf)</div><div class='add'>+static ssize_t lid_mask_count_show(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+				   struct ib_port_attribute *unused, char *buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr attr;</div><div class='ctx'> 	ssize_t ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_query_port(p-&gt;ibdev, p-&gt;port_num, &amp;attr);</div><div class='add'>+	ret = ib_query_port(ibdev, port_num, &amp;attr);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%d\n", attr.lmc);</div><div class='add'>+	return sysfs_emit(buf, "%u\n", attr.lmc);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t sm_lid_show(struct ib_port *p, struct port_attribute *unused,</div><div class='del'>-			   char *buf)</div><div class='add'>+static ssize_t sm_lid_show(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+			   struct ib_port_attribute *unused, char *buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr attr;</div><div class='ctx'> 	ssize_t ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_query_port(p-&gt;ibdev, p-&gt;port_num, &amp;attr);</div><div class='add'>+	ret = ib_query_port(ibdev, port_num, &amp;attr);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "0x%x\n", attr.sm_lid);</div><div class='add'>+	return sysfs_emit(buf, "0x%x\n", attr.sm_lid);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t sm_sl_show(struct ib_port *p, struct port_attribute *unused,</div><div class='del'>-			  char *buf)</div><div class='add'>+static ssize_t sm_sl_show(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+			  struct ib_port_attribute *unused, char *buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr attr;</div><div class='ctx'> 	ssize_t ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_query_port(p-&gt;ibdev, p-&gt;port_num, &amp;attr);</div><div class='add'>+	ret = ib_query_port(ibdev, port_num, &amp;attr);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%d\n", attr.sm_sl);</div><div class='add'>+	return sysfs_emit(buf, "%u\n", attr.sm_sl);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t cap_mask_show(struct ib_port *p, struct port_attribute *unused,</div><div class='del'>-			     char *buf)</div><div class='add'>+static ssize_t cap_mask_show(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+			     struct ib_port_attribute *unused, char *buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr attr;</div><div class='ctx'> 	ssize_t ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_query_port(p-&gt;ibdev, p-&gt;port_num, &amp;attr);</div><div class='add'>+	ret = ib_query_port(ibdev, port_num, &amp;attr);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "0x%08x\n", attr.port_cap_flags);</div><div class='add'>+	return sysfs_emit(buf, "0x%08x\n", attr.port_cap_flags);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t rate_show(struct ib_port *p, struct port_attribute *unused,</div><div class='del'>-			 char *buf)</div><div class='add'>+static ssize_t rate_show(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+			 struct ib_port_attribute *unused, char *buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr attr;</div><div class='ctx'> 	char *speed = "";</div><div class='ctx'> 	int rate;		/* in deci-Gb/sec */</div><div class='ctx'> 	ssize_t ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_query_port(p-&gt;ibdev, p-&gt;port_num, &amp;attr);</div><div class='add'>+	ret = ib_query_port(ibdev, port_num, &amp;attr);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='hunk'>@@ -272,6 +326,14 @@ static ssize_t rate_show(struct ib_port *p, struct port_attribute *unused,</div><div class='ctx'> 		speed = " HDR";</div><div class='ctx'> 		rate = 500;</div><div class='ctx'> 		break;</div><div class='add'>+	case IB_SPEED_NDR:</div><div class='add'>+		speed = " NDR";</div><div class='add'>+		rate = 1000;</div><div class='add'>+		break;</div><div class='add'>+	case IB_SPEED_XDR:</div><div class='add'>+		speed = " XDR";</div><div class='add'>+		rate = 2000;</div><div class='add'>+		break;</div><div class='ctx'> 	case IB_SPEED_SDR:</div><div class='ctx'> 	default:		/* default to SDR for invalid rates */</div><div class='ctx'> 		speed = " SDR";</div><div class='hunk'>@@ -283,115 +345,136 @@ static ssize_t rate_show(struct ib_port *p, struct port_attribute *unused,</div><div class='ctx'> 	if (rate &lt; 0)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%d%s Gb/sec (%dX%s)\n",</div><div class='del'>-		       rate / 10, rate % 10 ? ".5" : "",</div><div class='del'>-		       ib_width_enum_to_int(attr.active_width), speed);</div><div class='add'>+	return sysfs_emit(buf, "%d%s Gb/sec (%dX%s)\n", rate / 10,</div><div class='add'>+			  rate % 10 ? ".5" : "",</div><div class='add'>+			  ib_width_enum_to_int(attr.active_width), speed);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t phys_state_show(struct ib_port *p, struct port_attribute *unused,</div><div class='del'>-			       char *buf)</div><div class='add'>+static const char *phys_state_to_str(enum ib_port_phys_state phys_state)</div><div class='add'>+{</div><div class='add'>+	static const char *phys_state_str[] = {</div><div class='add'>+		"&lt;unknown&gt;",</div><div class='add'>+		"Sleep",</div><div class='add'>+		"Polling",</div><div class='add'>+		"Disabled",</div><div class='add'>+		"PortConfigurationTraining",</div><div class='add'>+		"LinkUp",</div><div class='add'>+		"LinkErrorRecovery",</div><div class='add'>+		"Phy Test",</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	if (phys_state &lt; ARRAY_SIZE(phys_state_str))</div><div class='add'>+		return phys_state_str[phys_state];</div><div class='add'>+	return "&lt;unknown&gt;";</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static ssize_t phys_state_show(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+			       struct ib_port_attribute *unused, char *buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr attr;</div><div class='ctx'> </div><div class='ctx'> 	ssize_t ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_query_port(p-&gt;ibdev, p-&gt;port_num, &amp;attr);</div><div class='add'>+	ret = ib_query_port(ibdev, port_num, &amp;attr);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	switch (attr.phys_state) {</div><div class='del'>-	case 1:  return sprintf(buf, "1: Sleep\n");</div><div class='del'>-	case 2:  return sprintf(buf, "2: Polling\n");</div><div class='del'>-	case 3:  return sprintf(buf, "3: Disabled\n");</div><div class='del'>-	case 4:  return sprintf(buf, "4: PortConfigurationTraining\n");</div><div class='del'>-	case 5:  return sprintf(buf, "5: LinkUp\n");</div><div class='del'>-	case 6:  return sprintf(buf, "6: LinkErrorRecovery\n");</div><div class='del'>-	case 7:  return sprintf(buf, "7: Phy Test\n");</div><div class='del'>-	default: return sprintf(buf, "%d: &lt;unknown&gt;\n", attr.phys_state);</div><div class='del'>-	}</div><div class='add'>+	return sysfs_emit(buf, "%u: %s\n", attr.phys_state,</div><div class='add'>+			  phys_state_to_str(attr.phys_state));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t link_layer_show(struct ib_port *p, struct port_attribute *unused,</div><div class='del'>-			       char *buf)</div><div class='add'>+static ssize_t link_layer_show(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+			       struct ib_port_attribute *unused, char *buf)</div><div class='ctx'> {</div><div class='del'>-	switch (rdma_port_get_link_layer(p-&gt;ibdev, p-&gt;port_num)) {</div><div class='add'>+	const char *output;</div><div class='add'>+</div><div class='add'>+	switch (rdma_port_get_link_layer(ibdev, port_num)) {</div><div class='ctx'> 	case IB_LINK_LAYER_INFINIBAND:</div><div class='del'>-		return sprintf(buf, "%s\n", "InfiniBand");</div><div class='add'>+		output = "InfiniBand";</div><div class='add'>+		break;</div><div class='ctx'> 	case IB_LINK_LAYER_ETHERNET:</div><div class='del'>-		return sprintf(buf, "%s\n", "Ethernet");</div><div class='add'>+		output = "Ethernet";</div><div class='add'>+		break;</div><div class='ctx'> 	default:</div><div class='del'>-		return sprintf(buf, "%s\n", "Unknown");</div><div class='add'>+		output = "Unknown";</div><div class='add'>+		break;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	return sysfs_emit(buf, "%s\n", output);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static PORT_ATTR_RO(state);</div><div class='del'>-static PORT_ATTR_RO(lid);</div><div class='del'>-static PORT_ATTR_RO(lid_mask_count);</div><div class='del'>-static PORT_ATTR_RO(sm_lid);</div><div class='del'>-static PORT_ATTR_RO(sm_sl);</div><div class='del'>-static PORT_ATTR_RO(cap_mask);</div><div class='del'>-static PORT_ATTR_RO(rate);</div><div class='del'>-static PORT_ATTR_RO(phys_state);</div><div class='del'>-static PORT_ATTR_RO(link_layer);</div><div class='add'>+static IB_PORT_ATTR_RO(state);</div><div class='add'>+static IB_PORT_ATTR_RO(lid);</div><div class='add'>+static IB_PORT_ATTR_RO(lid_mask_count);</div><div class='add'>+static IB_PORT_ATTR_RO(sm_lid);</div><div class='add'>+static IB_PORT_ATTR_RO(sm_sl);</div><div class='add'>+static IB_PORT_ATTR_RO(cap_mask);</div><div class='add'>+static IB_PORT_ATTR_RO(rate);</div><div class='add'>+static IB_PORT_ATTR_RO(phys_state);</div><div class='add'>+static IB_PORT_ATTR_RO(link_layer);</div><div class='ctx'> </div><div class='ctx'> static struct attribute *port_default_attrs[] = {</div><div class='del'>-	&amp;port_attr_state.attr,</div><div class='del'>-	&amp;port_attr_lid.attr,</div><div class='del'>-	&amp;port_attr_lid_mask_count.attr,</div><div class='del'>-	&amp;port_attr_sm_lid.attr,</div><div class='del'>-	&amp;port_attr_sm_sl.attr,</div><div class='del'>-	&amp;port_attr_cap_mask.attr,</div><div class='del'>-	&amp;port_attr_rate.attr,</div><div class='del'>-	&amp;port_attr_phys_state.attr,</div><div class='del'>-	&amp;port_attr_link_layer.attr,</div><div class='add'>+	&amp;ib_port_attr_state.attr,</div><div class='add'>+	&amp;ib_port_attr_lid.attr,</div><div class='add'>+	&amp;ib_port_attr_lid_mask_count.attr,</div><div class='add'>+	&amp;ib_port_attr_sm_lid.attr,</div><div class='add'>+	&amp;ib_port_attr_sm_sl.attr,</div><div class='add'>+	&amp;ib_port_attr_cap_mask.attr,</div><div class='add'>+	&amp;ib_port_attr_rate.attr,</div><div class='add'>+	&amp;ib_port_attr_phys_state.attr,</div><div class='add'>+	&amp;ib_port_attr_link_layer.attr,</div><div class='ctx'> 	NULL</div><div class='ctx'> };</div><div class='add'>+ATTRIBUTE_GROUPS(port_default);</div><div class='ctx'> </div><div class='del'>-static size_t print_ndev(const struct ib_gid_attr *gid_attr, char *buf)</div><div class='add'>+static ssize_t print_ndev(const struct ib_gid_attr *gid_attr, char *buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct net_device *ndev;</div><div class='del'>-	size_t ret = -EINVAL;</div><div class='add'>+	int ret = -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	rcu_read_lock();</div><div class='ctx'> 	ndev = rcu_dereference(gid_attr-&gt;ndev);</div><div class='ctx'> 	if (ndev)</div><div class='del'>-		ret = sprintf(buf, "%s\n", ndev-&gt;name);</div><div class='add'>+		ret = sysfs_emit(buf, "%s\n", ndev-&gt;name);</div><div class='ctx'> 	rcu_read_unlock();</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static size_t print_gid_type(const struct ib_gid_attr *gid_attr, char *buf)</div><div class='add'>+static ssize_t print_gid_type(const struct ib_gid_attr *gid_attr, char *buf)</div><div class='ctx'> {</div><div class='del'>-	return sprintf(buf, "%s\n", ib_cache_gid_type_str(gid_attr-&gt;gid_type));</div><div class='add'>+	return sysfs_emit(buf, "%s\n",</div><div class='add'>+			  ib_cache_gid_type_str(gid_attr-&gt;gid_type));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t _show_port_gid_attr(</div><div class='del'>-	struct ib_port *p, struct port_attribute *attr, char *buf,</div><div class='del'>-	size_t (*print)(const struct ib_gid_attr *gid_attr, char *buf))</div><div class='add'>+	struct ib_device *ibdev, u32 port_num, struct ib_port_attribute *attr,</div><div class='add'>+	char *buf,</div><div class='add'>+	ssize_t (*print)(const struct ib_gid_attr *gid_attr, char *buf))</div><div class='ctx'> {</div><div class='ctx'> 	struct port_table_attribute *tab_attr =</div><div class='ctx'> 		container_of(attr, struct port_table_attribute, attr);</div><div class='ctx'> 	const struct ib_gid_attr *gid_attr;</div><div class='ctx'> 	ssize_t ret;</div><div class='ctx'> </div><div class='del'>-	gid_attr = rdma_get_gid_attr(p-&gt;ibdev, p-&gt;port_num, tab_attr-&gt;index);</div><div class='add'>+	gid_attr = rdma_get_gid_attr(ibdev, port_num, tab_attr-&gt;index);</div><div class='ctx'> 	if (IS_ERR(gid_attr))</div><div class='del'>-		return PTR_ERR(gid_attr);</div><div class='add'>+		/* -EINVAL is returned for user space compatibility reasons. */</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	ret = print(gid_attr, buf);</div><div class='ctx'> 	rdma_put_gid_attr(gid_attr);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t show_port_gid(struct ib_port *p, struct port_attribute *attr,</div><div class='del'>-			     char *buf)</div><div class='add'>+static ssize_t show_port_gid(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+			     struct ib_port_attribute *attr, char *buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct port_table_attribute *tab_attr =</div><div class='ctx'> 		container_of(attr, struct port_table_attribute, attr);</div><div class='ctx'> 	const struct ib_gid_attr *gid_attr;</div><div class='del'>-	ssize_t ret;</div><div class='add'>+	int len;</div><div class='ctx'> </div><div class='del'>-	gid_attr = rdma_get_gid_attr(p-&gt;ibdev, p-&gt;port_num, tab_attr-&gt;index);</div><div class='add'>+	gid_attr = rdma_get_gid_attr(ibdev, port_num, tab_attr-&gt;index);</div><div class='ctx'> 	if (IS_ERR(gid_attr)) {</div><div class='ctx'> 		const union ib_gid zgid = {};</div><div class='ctx'> </div><div class='hunk'>@@ -404,54 +487,56 @@ static ssize_t show_port_gid(struct ib_port *p, struct port_attribute *attr,</div><div class='ctx'> 		 * space throwing such error on fail to read gid, return zero</div><div class='ctx'> 		 * GID as before. This maintains backward compatibility.</div><div class='ctx'> 		 */</div><div class='del'>-		return sprintf(buf, "%pI6\n", zgid.raw);</div><div class='add'>+		return sysfs_emit(buf, "%pI6\n", zgid.raw);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = sprintf(buf, "%pI6\n", gid_attr-&gt;gid.raw);</div><div class='add'>+	len = sysfs_emit(buf, "%pI6\n", gid_attr-&gt;gid.raw);</div><div class='ctx'> 	rdma_put_gid_attr(gid_attr);</div><div class='del'>-	return ret;</div><div class='add'>+	return len;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t show_port_gid_attr_ndev(struct ib_port *p,</div><div class='del'>-				       struct port_attribute *attr, char *buf)</div><div class='add'>+static ssize_t show_port_gid_attr_ndev(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+				       struct ib_port_attribute *attr,</div><div class='add'>+				       char *buf)</div><div class='ctx'> {</div><div class='del'>-	return _show_port_gid_attr(p, attr, buf, print_ndev);</div><div class='add'>+	return _show_port_gid_attr(ibdev, port_num, attr, buf, print_ndev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t show_port_gid_attr_gid_type(struct ib_port *p,</div><div class='del'>-					   struct port_attribute *attr,</div><div class='add'>+static ssize_t show_port_gid_attr_gid_type(struct ib_device *ibdev,</div><div class='add'>+					   u32 port_num,</div><div class='add'>+					   struct ib_port_attribute *attr,</div><div class='ctx'> 					   char *buf)</div><div class='ctx'> {</div><div class='del'>-	return _show_port_gid_attr(p, attr, buf, print_gid_type);</div><div class='add'>+	return _show_port_gid_attr(ibdev, port_num, attr, buf, print_gid_type);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t show_port_pkey(struct ib_port *p, struct port_attribute *attr,</div><div class='del'>-			      char *buf)</div><div class='add'>+static ssize_t show_port_pkey(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+			      struct ib_port_attribute *attr, char *buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct port_table_attribute *tab_attr =</div><div class='ctx'> 		container_of(attr, struct port_table_attribute, attr);</div><div class='ctx'> 	u16 pkey;</div><div class='del'>-	ssize_t ret;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_query_pkey(p-&gt;ibdev, p-&gt;port_num, tab_attr-&gt;index, &amp;pkey);</div><div class='add'>+	ret = ib_query_pkey(ibdev, port_num, tab_attr-&gt;index, &amp;pkey);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "0x%04x\n", pkey);</div><div class='add'>+	return sysfs_emit(buf, "0x%04x\n", pkey);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> #define PORT_PMA_ATTR(_name, _counter, _width, _offset)			\</div><div class='ctx'> struct port_table_attribute port_pma_attr_##_name = {			\</div><div class='ctx'> 	.attr  = __ATTR(_name, S_IRUGO, show_pma_counter, NULL),	\</div><div class='ctx'> 	.index = (_offset) | ((_width) &lt;&lt; 16) | ((_counter) &lt;&lt; 24),	\</div><div class='del'>-	.attr_id = IB_PMA_PORT_COUNTERS ,				\</div><div class='add'>+	.attr_id = IB_PMA_PORT_COUNTERS,				\</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> #define PORT_PMA_ATTR_EXT(_name, _width, _offset)			\</div><div class='ctx'> struct port_table_attribute port_pma_attr_ext_##_name = {		\</div><div class='ctx'> 	.attr  = __ATTR(_name, S_IRUGO, show_pma_counter, NULL),	\</div><div class='ctx'> 	.index = (_offset) | ((_width) &lt;&lt; 16),				\</div><div class='del'>-	.attr_id = IB_PMA_PORT_COUNTERS_EXT ,				\</div><div class='add'>+	.attr_id = IB_PMA_PORT_COUNTERS_EXT,				\</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -470,8 +555,8 @@ static int get_perf_mad(struct ib_device *dev, int port_num, __be16 attr,</div><div class='ctx'> 	if (!dev-&gt;ops.process_mad)</div><div class='ctx'> 		return -ENOSYS;</div><div class='ctx'> </div><div class='del'>-	in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);</div><div class='del'>-	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);</div><div class='add'>+	in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL);</div><div class='add'>+	out_mad = kzalloc(sizeof(*out_mad), GFP_KERNEL);</div><div class='ctx'> 	if (!in_mad || !out_mad) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='ctx'> 		goto out;</div><div class='hunk'>@@ -486,10 +571,8 @@ static int get_perf_mad(struct ib_device *dev, int port_num, __be16 attr,</div><div class='ctx'> 	if (attr != IB_PMA_CLASS_PORT_INFO)</div><div class='ctx'> 		in_mad-&gt;data[41] = port_num;	/* PortSelect field */</div><div class='ctx'> </div><div class='del'>-	if ((dev-&gt;ops.process_mad(dev, IB_MAD_IGNORE_MKEY,</div><div class='del'>-				  port_num, NULL, NULL,</div><div class='del'>-				  (const struct ib_mad_hdr *)in_mad, mad_size,</div><div class='del'>-				  (struct ib_mad_hdr *)out_mad, &amp;mad_size,</div><div class='add'>+	if ((dev-&gt;ops.process_mad(dev, IB_MAD_IGNORE_MKEY, port_num, NULL, NULL,</div><div class='add'>+				  in_mad, out_mad, &amp;mad_size,</div><div class='ctx'> 				  &amp;out_mad_pkey_index) &amp;</div><div class='ctx'> 	     (IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY)) !=</div><div class='ctx'> 	    (IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY)) {</div><div class='hunk'>@@ -504,47 +587,45 @@ out:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t show_pma_counter(struct ib_port *p, struct port_attribute *attr,</div><div class='del'>-				char *buf)</div><div class='add'>+static ssize_t show_pma_counter(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+				struct ib_port_attribute *attr, char *buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct port_table_attribute *tab_attr =</div><div class='ctx'> 		container_of(attr, struct port_table_attribute, attr);</div><div class='ctx'> 	int offset = tab_attr-&gt;index &amp; 0xffff;</div><div class='ctx'> 	int width  = (tab_attr-&gt;index &gt;&gt; 16) &amp; 0xff;</div><div class='del'>-	ssize_t ret;</div><div class='add'>+	int ret;</div><div class='ctx'> 	u8 data[8];</div><div class='add'>+	int len;</div><div class='ctx'> </div><div class='del'>-	ret = get_perf_mad(p-&gt;ibdev, p-&gt;port_num, tab_attr-&gt;attr_id, &amp;data,</div><div class='add'>+	ret = get_perf_mad(ibdev, port_num, tab_attr-&gt;attr_id, &amp;data,</div><div class='ctx'> 			40 + offset / 8, sizeof(data));</div><div class='ctx'> 	if (ret &lt; 0)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='ctx'> 	switch (width) {</div><div class='ctx'> 	case 4:</div><div class='del'>-		ret = sprintf(buf, "%u\n", (*data &gt;&gt;</div><div class='del'>-					    (4 - (offset % 8))) &amp; 0xf);</div><div class='add'>+		len = sysfs_emit(buf, "%d\n",</div><div class='add'>+				 (*data &gt;&gt; (4 - (offset % 8))) &amp; 0xf);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case 8:</div><div class='del'>-		ret = sprintf(buf, "%u\n", *data);</div><div class='add'>+		len = sysfs_emit(buf, "%u\n", *data);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case 16:</div><div class='del'>-		ret = sprintf(buf, "%u\n",</div><div class='del'>-			      be16_to_cpup((__be16 *)data));</div><div class='add'>+		len = sysfs_emit(buf, "%u\n", be16_to_cpup((__be16 *)data));</div><div class='ctx'> 		break;</div><div class='ctx'> 	case 32:</div><div class='del'>-		ret = sprintf(buf, "%u\n",</div><div class='del'>-			      be32_to_cpup((__be32 *)data));</div><div class='add'>+		len = sysfs_emit(buf, "%u\n", be32_to_cpup((__be32 *)data));</div><div class='ctx'> 		break;</div><div class='ctx'> 	case 64:</div><div class='del'>-		ret = sprintf(buf, "%llu\n",</div><div class='del'>-				be64_to_cpup((__be64 *)data));</div><div class='add'>+		len = sysfs_emit(buf, "%llu\n", be64_to_cpup((__be64 *)data));</div><div class='ctx'> 		break;</div><div class='del'>-</div><div class='ctx'> 	default:</div><div class='del'>-		ret = 0;</div><div class='add'>+		len = 0;</div><div class='add'>+		break;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return ret;</div><div class='add'>+	return len;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static PORT_PMA_ATTR(symbol_error		    ,  0, 16,  32);</div><div class='hunk'>@@ -644,72 +725,49 @@ static struct attribute *pma_attrs_noietf[] = {</div><div class='ctx'> 	NULL</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static struct attribute_group pma_group = {</div><div class='add'>+static const struct attribute_group pma_group = {</div><div class='ctx'> 	.name  = "counters",</div><div class='ctx'> 	.attrs  = pma_attrs</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static struct attribute_group pma_group_ext = {</div><div class='add'>+static const struct attribute_group pma_group_ext = {</div><div class='ctx'> 	.name  = "counters",</div><div class='ctx'> 	.attrs  = pma_attrs_ext</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static struct attribute_group pma_group_noietf = {</div><div class='add'>+static const struct attribute_group pma_group_noietf = {</div><div class='ctx'> 	.name  = "counters",</div><div class='ctx'> 	.attrs  = pma_attrs_noietf</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static void ib_port_release(struct kobject *kobj)</div><div class='ctx'> {</div><div class='del'>-	struct ib_port *p = container_of(kobj, struct ib_port, kobj);</div><div class='del'>-	struct attribute *a;</div><div class='add'>+	struct ib_port *port = container_of(kobj, struct ib_port, kobj);</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	if (p-&gt;gid_group.attrs) {</div><div class='del'>-		for (i = 0; (a = p-&gt;gid_group.attrs[i]); ++i)</div><div class='del'>-			kfree(a);</div><div class='del'>-</div><div class='del'>-		kfree(p-&gt;gid_group.attrs);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (p-&gt;pkey_group.attrs) {</div><div class='del'>-		for (i = 0; (a = p-&gt;pkey_group.attrs[i]); ++i)</div><div class='del'>-			kfree(a);</div><div class='del'>-</div><div class='del'>-		kfree(p-&gt;pkey_group.attrs);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	kfree(p);</div><div class='add'>+	for (i = 0; i != ARRAY_SIZE(port-&gt;groups); i++)</div><div class='add'>+		kfree(port-&gt;groups[i].attrs);</div><div class='add'>+	if (port-&gt;hw_stats_data)</div><div class='add'>+		rdma_free_hw_stats_struct(port-&gt;hw_stats_data-&gt;stats);</div><div class='add'>+	kfree(port-&gt;hw_stats_data);</div><div class='add'>+	kvfree(port);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void ib_port_gid_attr_release(struct kobject *kobj)</div><div class='ctx'> {</div><div class='del'>-	struct gid_attr_group *g = container_of(kobj, struct gid_attr_group,</div><div class='del'>-						kobj);</div><div class='del'>-	struct attribute *a;</div><div class='add'>+	struct gid_attr_group *gid_attr_group =</div><div class='add'>+		container_of(kobj, struct gid_attr_group, kobj);</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	if (g-&gt;ndev.attrs) {</div><div class='del'>-		for (i = 0; (a = g-&gt;ndev.attrs[i]); ++i)</div><div class='del'>-			kfree(a);</div><div class='del'>-</div><div class='del'>-		kfree(g-&gt;ndev.attrs);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (g-&gt;type.attrs) {</div><div class='del'>-		for (i = 0; (a = g-&gt;type.attrs[i]); ++i)</div><div class='del'>-			kfree(a);</div><div class='del'>-</div><div class='del'>-		kfree(g-&gt;type.attrs);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	kfree(g);</div><div class='add'>+	for (i = 0; i != ARRAY_SIZE(gid_attr_group-&gt;groups); i++)</div><div class='add'>+		kfree(gid_attr_group-&gt;groups[i].attrs);</div><div class='add'>+	kfree(gid_attr_group);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static struct kobj_type port_type = {</div><div class='ctx'> 	.release       = ib_port_release,</div><div class='ctx'> 	.sysfs_ops     = &amp;port_sysfs_ops,</div><div class='del'>-	.default_attrs = port_default_attrs</div><div class='add'>+	.default_groups = port_default_groups,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static struct kobj_type gid_attr_type = {</div><div class='hunk'>@@ -717,55 +775,12 @@ static struct kobj_type gid_attr_type = {</div><div class='ctx'> 	.release        = ib_port_gid_attr_release</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static struct attribute **</div><div class='del'>-alloc_group_attrs(ssize_t (*show)(struct ib_port *,</div><div class='del'>-				  struct port_attribute *, char *buf),</div><div class='del'>-		  int len)</div><div class='del'>-{</div><div class='del'>-	struct attribute **tab_attr;</div><div class='del'>-	struct port_table_attribute *element;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	tab_attr = kcalloc(1 + len, sizeof(struct attribute *), GFP_KERNEL);</div><div class='del'>-	if (!tab_attr)</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; len; i++) {</div><div class='del'>-		element = kzalloc(sizeof(struct port_table_attribute),</div><div class='del'>-				  GFP_KERNEL);</div><div class='del'>-		if (!element)</div><div class='del'>-			goto err;</div><div class='del'>-</div><div class='del'>-		if (snprintf(element-&gt;name, sizeof(element-&gt;name),</div><div class='del'>-			     "%d", i) &gt;= sizeof(element-&gt;name)) {</div><div class='del'>-			kfree(element);</div><div class='del'>-			goto err;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		element-&gt;attr.attr.name  = element-&gt;name;</div><div class='del'>-		element-&gt;attr.attr.mode  = S_IRUGO;</div><div class='del'>-		element-&gt;attr.show       = show;</div><div class='del'>-		element-&gt;index		 = i;</div><div class='del'>-		sysfs_attr_init(&amp;element-&gt;attr.attr);</div><div class='del'>-</div><div class='del'>-		tab_attr[i] = &amp;element-&gt;attr.attr;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return tab_attr;</div><div class='del'>-</div><div class='del'>-err:</div><div class='del'>-	while (--i &gt;= 0)</div><div class='del'>-		kfree(tab_attr[i]);</div><div class='del'>-	kfree(tab_attr);</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> /*</div><div class='ctx'>  * Figure out which counter table to use depending on</div><div class='ctx'>  * the device capabilities.</div><div class='ctx'>  */</div><div class='del'>-static struct attribute_group *get_counter_table(struct ib_device *dev,</div><div class='del'>-						 int port_num)</div><div class='add'>+static const struct attribute_group *get_counter_table(struct ib_device *dev,</div><div class='add'>+						       int port_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_class_port_info cpi;</div><div class='ctx'> </div><div class='hunk'>@@ -785,7 +800,7 @@ static struct attribute_group *get_counter_table(struct ib_device *dev,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int update_hw_stats(struct ib_device *dev, struct rdma_hw_stats *stats,</div><div class='del'>-			   u8 port_num, int index)</div><div class='add'>+			   u32 port_num, int index)</div><div class='ctx'> {</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='hunk'>@@ -800,74 +815,50 @@ static int update_hw_stats(struct ib_device *dev, struct rdma_hw_stats *stats,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t print_hw_stat(struct rdma_hw_stats *stats, int index, char *buf)</div><div class='add'>+static int print_hw_stat(struct ib_device *dev, int port_num,</div><div class='add'>+			 struct rdma_hw_stats *stats, int index, char *buf)</div><div class='ctx'> {</div><div class='del'>-	return sprintf(buf, "%llu\n", stats-&gt;value[index]);</div><div class='add'>+	u64 v = rdma_counter_get_hwstat_value(dev, port_num, index);</div><div class='add'>+</div><div class='add'>+	return sysfs_emit(buf, "%llu\n", stats-&gt;value[index] + v);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t show_hw_stats(struct kobject *kobj, struct attribute *attr,</div><div class='del'>-			     char *buf)</div><div class='add'>+static ssize_t show_hw_stats(struct ib_device *ibdev,</div><div class='add'>+			     struct rdma_hw_stats *stats, unsigned int index,</div><div class='add'>+			     unsigned int port_num, char *buf)</div><div class='ctx'> {</div><div class='del'>-	struct ib_device *dev;</div><div class='del'>-	struct ib_port *port;</div><div class='del'>-	struct hw_stats_attribute *hsa;</div><div class='del'>-	struct rdma_hw_stats *stats;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	hsa = container_of(attr, struct hw_stats_attribute, attr);</div><div class='del'>-	if (!hsa-&gt;port_num) {</div><div class='del'>-		dev = container_of((struct device *)kobj,</div><div class='del'>-				   struct ib_device, dev);</div><div class='del'>-		stats = dev-&gt;hw_stats;</div><div class='del'>-	} else {</div><div class='del'>-		port = container_of(kobj, struct ib_port, kobj);</div><div class='del'>-		dev = port-&gt;ibdev;</div><div class='del'>-		stats = port-&gt;hw_stats;</div><div class='del'>-	}</div><div class='ctx'> 	mutex_lock(&amp;stats-&gt;lock);</div><div class='del'>-	ret = update_hw_stats(dev, stats, hsa-&gt;port_num, hsa-&gt;index);</div><div class='add'>+	ret = update_hw_stats(ibdev, stats, port_num, index);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto unlock;</div><div class='del'>-	ret = print_hw_stat(stats, hsa-&gt;index, buf);</div><div class='add'>+	ret = print_hw_stat(ibdev, port_num, stats, index, buf);</div><div class='ctx'> unlock:</div><div class='ctx'> 	mutex_unlock(&amp;stats-&gt;lock);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t show_stats_lifespan(struct kobject *kobj,</div><div class='del'>-				   struct attribute *attr,</div><div class='add'>+static ssize_t show_stats_lifespan(struct ib_device *ibdev,</div><div class='add'>+				   struct rdma_hw_stats *stats,</div><div class='add'>+				   unsigned int index, unsigned int port_num,</div><div class='ctx'> 				   char *buf)</div><div class='ctx'> {</div><div class='del'>-	struct hw_stats_attribute *hsa;</div><div class='del'>-	struct rdma_hw_stats *stats;</div><div class='ctx'> 	int msecs;</div><div class='ctx'> </div><div class='del'>-	hsa = container_of(attr, struct hw_stats_attribute, attr);</div><div class='del'>-	if (!hsa-&gt;port_num) {</div><div class='del'>-		struct ib_device *dev = container_of((struct device *)kobj,</div><div class='del'>-						     struct ib_device, dev);</div><div class='del'>-</div><div class='del'>-		stats = dev-&gt;hw_stats;</div><div class='del'>-	} else {</div><div class='del'>-		struct ib_port *p = container_of(kobj, struct ib_port, kobj);</div><div class='del'>-</div><div class='del'>-		stats = p-&gt;hw_stats;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	mutex_lock(&amp;stats-&gt;lock);</div><div class='ctx'> 	msecs = jiffies_to_msecs(stats-&gt;lifespan);</div><div class='ctx'> 	mutex_unlock(&amp;stats-&gt;lock);</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%d\n", msecs);</div><div class='add'>+	return sysfs_emit(buf, "%d\n", msecs);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t set_stats_lifespan(struct kobject *kobj,</div><div class='del'>-				  struct attribute *attr,</div><div class='del'>-				  const char *buf, size_t count)</div><div class='add'>+static ssize_t set_stats_lifespan(struct ib_device *ibdev,</div><div class='add'>+				   struct rdma_hw_stats *stats,</div><div class='add'>+				   unsigned int index, unsigned int port_num,</div><div class='add'>+				   const char *buf, size_t count)</div><div class='ctx'> {</div><div class='del'>-	struct hw_stats_attribute *hsa;</div><div class='del'>-	struct rdma_hw_stats *stats;</div><div class='ctx'> 	int msecs;</div><div class='ctx'> 	int jiffies;</div><div class='ctx'> 	int ret;</div><div class='hunk'>@@ -878,17 +869,6 @@ static ssize_t set_stats_lifespan(struct kobject *kobj,</div><div class='ctx'> 	if (msecs &lt; 0 || msecs &gt; 10000)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	jiffies = msecs_to_jiffies(msecs);</div><div class='del'>-	hsa = container_of(attr, struct hw_stats_attribute, attr);</div><div class='del'>-	if (!hsa-&gt;port_num) {</div><div class='del'>-		struct ib_device *dev = container_of((struct device *)kobj,</div><div class='del'>-						     struct ib_device, dev);</div><div class='del'>-</div><div class='del'>-		stats = dev-&gt;hw_stats;</div><div class='del'>-	} else {</div><div class='del'>-		struct ib_port *p = container_of(kobj, struct ib_port, kobj);</div><div class='del'>-</div><div class='del'>-		stats = p-&gt;hw_stats;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;stats-&gt;lock);</div><div class='ctx'> 	stats-&gt;lifespan = jiffies;</div><div class='hunk'>@@ -897,298 +877,433 @@ static ssize_t set_stats_lifespan(struct kobject *kobj,</div><div class='ctx'> 	return count;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void free_hsag(struct kobject *kobj, struct attribute_group *attr_group)</div><div class='add'>+static struct hw_stats_device_data *</div><div class='add'>+alloc_hw_stats_device(struct ib_device *ibdev)</div><div class='ctx'> {</div><div class='del'>-	struct attribute **attr;</div><div class='del'>-</div><div class='del'>-	sysfs_remove_group(kobj, attr_group);</div><div class='add'>+	struct hw_stats_device_data *data;</div><div class='add'>+	struct rdma_hw_stats *stats;</div><div class='ctx'> </div><div class='del'>-	for (attr = attr_group-&gt;attrs; *attr; attr++)</div><div class='del'>-		kfree(*attr);</div><div class='del'>-	kfree(attr_group);</div><div class='del'>-}</div><div class='add'>+	if (!ibdev-&gt;ops.alloc_hw_device_stats)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+	stats = ibdev-&gt;ops.alloc_hw_device_stats(ibdev);</div><div class='add'>+	if (!stats)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+	if (!stats-&gt;descs || stats-&gt;num_counters &lt;= 0)</div><div class='add'>+		goto err_free_stats;</div><div class='ctx'> </div><div class='del'>-static struct attribute *alloc_hsa(int index, u8 port_num, const char *name)</div><div class='del'>-{</div><div class='del'>-	struct hw_stats_attribute *hsa;</div><div class='add'>+	/*</div><div class='add'>+	 * Two extra attribue elements here, one for the lifespan entry and</div><div class='add'>+	 * one to NULL terminate the list for the sysfs core code</div><div class='add'>+	 */</div><div class='add'>+	data = kzalloc(struct_size(data, attrs, size_add(stats-&gt;num_counters, 1)),</div><div class='add'>+		       GFP_KERNEL);</div><div class='add'>+	if (!data)</div><div class='add'>+		goto err_free_stats;</div><div class='add'>+	data-&gt;group.attrs = kcalloc(stats-&gt;num_counters + 2,</div><div class='add'>+				    sizeof(*data-&gt;group.attrs), GFP_KERNEL);</div><div class='add'>+	if (!data-&gt;group.attrs)</div><div class='add'>+		goto err_free_data;</div><div class='ctx'> </div><div class='del'>-	hsa = kmalloc(sizeof(*hsa), GFP_KERNEL);</div><div class='del'>-	if (!hsa)</div><div class='del'>-		return NULL;</div><div class='add'>+	data-&gt;group.name = "hw_counters";</div><div class='add'>+	data-&gt;stats = stats;</div><div class='add'>+	return data;</div><div class='ctx'> </div><div class='del'>-	hsa-&gt;attr.name = (char *)name;</div><div class='del'>-	hsa-&gt;attr.mode = S_IRUGO;</div><div class='del'>-	hsa-&gt;show = show_hw_stats;</div><div class='del'>-	hsa-&gt;store = NULL;</div><div class='del'>-	hsa-&gt;index = index;</div><div class='del'>-	hsa-&gt;port_num = port_num;</div><div class='add'>+err_free_data:</div><div class='add'>+	kfree(data);</div><div class='add'>+err_free_stats:</div><div class='add'>+	rdma_free_hw_stats_struct(stats);</div><div class='add'>+	return ERR_PTR(-ENOMEM);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	return &amp;hsa-&gt;attr;</div><div class='add'>+void ib_device_release_hw_stats(struct hw_stats_device_data *data)</div><div class='add'>+{</div><div class='add'>+	kfree(data-&gt;group.attrs);</div><div class='add'>+	rdma_free_hw_stats_struct(data-&gt;stats);</div><div class='add'>+	kfree(data);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct attribute *alloc_hsa_lifespan(char *name, u8 port_num)</div><div class='add'>+int ib_setup_device_attrs(struct ib_device *ibdev)</div><div class='ctx'> {</div><div class='del'>-	struct hw_stats_attribute *hsa;</div><div class='add'>+	struct hw_stats_device_attribute *attr;</div><div class='add'>+	struct hw_stats_device_data *data;</div><div class='add'>+	bool opstat_skipped = false;</div><div class='add'>+	int i, ret, pos = 0;</div><div class='add'>+</div><div class='add'>+	data = alloc_hw_stats_device(ibdev);</div><div class='add'>+	if (IS_ERR(data)) {</div><div class='add'>+		if (PTR_ERR(data) == -EOPNOTSUPP)</div><div class='add'>+			return 0;</div><div class='add'>+		return PTR_ERR(data);</div><div class='add'>+	}</div><div class='add'>+	ibdev-&gt;hw_stats_data = data;</div><div class='ctx'> </div><div class='del'>-	hsa = kmalloc(sizeof(*hsa), GFP_KERNEL);</div><div class='del'>-	if (!hsa)</div><div class='del'>-		return NULL;</div><div class='add'>+	ret = ibdev-&gt;ops.get_hw_stats(ibdev, data-&gt;stats, 0,</div><div class='add'>+				      data-&gt;stats-&gt;num_counters);</div><div class='add'>+	if (ret != data-&gt;stats-&gt;num_counters) {</div><div class='add'>+		if (WARN_ON(ret &gt;= 0))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	data-&gt;stats-&gt;timestamp = jiffies;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; data-&gt;stats-&gt;num_counters; i++) {</div><div class='add'>+		if (data-&gt;stats-&gt;descs[i].flags &amp; IB_STAT_FLAG_OPTIONAL) {</div><div class='add'>+			opstat_skipped = true;</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='del'>-	hsa-&gt;attr.name = name;</div><div class='del'>-	hsa-&gt;attr.mode = S_IWUSR | S_IRUGO;</div><div class='del'>-	hsa-&gt;show = show_stats_lifespan;</div><div class='del'>-	hsa-&gt;store = set_stats_lifespan;</div><div class='del'>-	hsa-&gt;index = 0;</div><div class='del'>-	hsa-&gt;port_num = port_num;</div><div class='add'>+		WARN_ON(opstat_skipped);</div><div class='add'>+		attr = &amp;data-&gt;attrs[pos];</div><div class='add'>+		sysfs_attr_init(&amp;attr-&gt;attr.attr);</div><div class='add'>+		attr-&gt;attr.attr.name = data-&gt;stats-&gt;descs[i].name;</div><div class='add'>+		attr-&gt;attr.attr.mode = 0444;</div><div class='add'>+		attr-&gt;attr.show = hw_stat_device_show;</div><div class='add'>+		attr-&gt;show = show_hw_stats;</div><div class='add'>+		data-&gt;group.attrs[pos] = &amp;attr-&gt;attr.attr;</div><div class='add'>+		pos++;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	return &amp;hsa-&gt;attr;</div><div class='add'>+	attr = &amp;data-&gt;attrs[pos];</div><div class='add'>+	sysfs_attr_init(&amp;attr-&gt;attr.attr);</div><div class='add'>+	attr-&gt;attr.attr.name = "lifespan";</div><div class='add'>+	attr-&gt;attr.attr.mode = 0644;</div><div class='add'>+	attr-&gt;attr.show = hw_stat_device_show;</div><div class='add'>+	attr-&gt;show = show_stats_lifespan;</div><div class='add'>+	attr-&gt;attr.store = hw_stat_device_store;</div><div class='add'>+	attr-&gt;store = set_stats_lifespan;</div><div class='add'>+	data-&gt;group.attrs[pos] = &amp;attr-&gt;attr.attr;</div><div class='add'>+	for (i = 0; i != ARRAY_SIZE(ibdev-&gt;groups); i++)</div><div class='add'>+		if (!ibdev-&gt;groups[i]) {</div><div class='add'>+			ibdev-&gt;groups[i] = &amp;data-&gt;group;</div><div class='add'>+			ibdev-&gt;hw_stats_attr_index = i;</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+	WARN(true, "struct ib_device-&gt;groups is too small");</div><div class='add'>+	return -EINVAL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void setup_hw_stats(struct ib_device *device, struct ib_port *port,</div><div class='del'>-			   u8 port_num)</div><div class='add'>+static struct hw_stats_port_data *</div><div class='add'>+alloc_hw_stats_port(struct ib_port *port, struct attribute_group *group)</div><div class='ctx'> {</div><div class='del'>-	struct attribute_group *hsag;</div><div class='add'>+	struct ib_device *ibdev = port-&gt;ibdev;</div><div class='add'>+	struct hw_stats_port_data *data;</div><div class='ctx'> 	struct rdma_hw_stats *stats;</div><div class='del'>-	int i, ret;</div><div class='del'>-</div><div class='del'>-	stats = device-&gt;ops.alloc_hw_stats(device, port_num);</div><div class='ctx'> </div><div class='add'>+	if (!ibdev-&gt;ops.alloc_hw_port_stats)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+	stats = ibdev-&gt;ops.alloc_hw_port_stats(port-&gt;ibdev, port-&gt;port_num);</div><div class='ctx'> 	if (!stats)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	if (!stats-&gt;names || stats-&gt;num_counters &lt;= 0)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+	if (!stats-&gt;descs || stats-&gt;num_counters &lt;= 0)</div><div class='ctx'> 		goto err_free_stats;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Two extra attribue elements here, one for the lifespan entry and</div><div class='ctx'> 	 * one to NULL terminate the list for the sysfs core code</div><div class='ctx'> 	 */</div><div class='del'>-	hsag = kzalloc(sizeof(*hsag) +</div><div class='del'>-		       sizeof(void *) * (stats-&gt;num_counters + 2),</div><div class='add'>+	data = kzalloc(struct_size(data, attrs, size_add(stats-&gt;num_counters, 1)),</div><div class='ctx'> 		       GFP_KERNEL);</div><div class='del'>-	if (!hsag)</div><div class='add'>+	if (!data)</div><div class='ctx'> 		goto err_free_stats;</div><div class='add'>+	group-&gt;attrs = kcalloc(stats-&gt;num_counters + 2,</div><div class='add'>+				    sizeof(*group-&gt;attrs), GFP_KERNEL);</div><div class='add'>+	if (!group-&gt;attrs)</div><div class='add'>+		goto err_free_data;</div><div class='ctx'> </div><div class='del'>-	ret = device-&gt;ops.get_hw_stats(device, stats, port_num,</div><div class='del'>-				       stats-&gt;num_counters);</div><div class='del'>-	if (ret != stats-&gt;num_counters)</div><div class='del'>-		goto err_free_hsag;</div><div class='add'>+	group-&gt;name = "hw_counters";</div><div class='add'>+	data-&gt;stats = stats;</div><div class='add'>+	return data;</div><div class='ctx'> </div><div class='del'>-	stats-&gt;timestamp = jiffies;</div><div class='del'>-</div><div class='del'>-	hsag-&gt;name = "hw_counters";</div><div class='del'>-	hsag-&gt;attrs = (void *)hsag + sizeof(*hsag);</div><div class='add'>+err_free_data:</div><div class='add'>+	kfree(data);</div><div class='add'>+err_free_stats:</div><div class='add'>+	rdma_free_hw_stats_struct(stats);</div><div class='add'>+	return ERR_PTR(-ENOMEM);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; stats-&gt;num_counters; i++) {</div><div class='del'>-		hsag-&gt;attrs[i] = alloc_hsa(i, port_num, stats-&gt;names[i]);</div><div class='del'>-		if (!hsag-&gt;attrs[i])</div><div class='del'>-			goto err;</div><div class='del'>-		sysfs_attr_init(hsag-&gt;attrs[i]);</div><div class='add'>+static int setup_hw_port_stats(struct ib_port *port,</div><div class='add'>+			       struct attribute_group *group)</div><div class='add'>+{</div><div class='add'>+	struct hw_stats_port_attribute *attr;</div><div class='add'>+	struct hw_stats_port_data *data;</div><div class='add'>+	bool opstat_skipped = false;</div><div class='add'>+	int i, ret, pos = 0;</div><div class='add'>+</div><div class='add'>+	data = alloc_hw_stats_port(port, group);</div><div class='add'>+	if (IS_ERR(data))</div><div class='add'>+		return PTR_ERR(data);</div><div class='add'>+</div><div class='add'>+	ret = port-&gt;ibdev-&gt;ops.get_hw_stats(port-&gt;ibdev, data-&gt;stats,</div><div class='add'>+					    port-&gt;port_num,</div><div class='add'>+					    data-&gt;stats-&gt;num_counters);</div><div class='add'>+	if (ret != data-&gt;stats-&gt;num_counters) {</div><div class='add'>+		if (WARN_ON(ret &gt;= 0))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	mutex_init(&amp;stats-&gt;lock);</div><div class='del'>-	/* treat an error here as non-fatal */</div><div class='del'>-	hsag-&gt;attrs[i] = alloc_hsa_lifespan("lifespan", port_num);</div><div class='del'>-	if (hsag-&gt;attrs[i])</div><div class='del'>-		sysfs_attr_init(hsag-&gt;attrs[i]);</div><div class='add'>+	data-&gt;stats-&gt;timestamp = jiffies;</div><div class='ctx'> </div><div class='del'>-	if (port) {</div><div class='del'>-		struct kobject *kobj = &amp;port-&gt;kobj;</div><div class='del'>-		ret = sysfs_create_group(kobj, hsag);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto err;</div><div class='del'>-		port-&gt;hw_stats_ag = hsag;</div><div class='del'>-		port-&gt;hw_stats = stats;</div><div class='del'>-	} else {</div><div class='del'>-		struct kobject *kobj = &amp;device-&gt;dev.kobj;</div><div class='del'>-		ret = sysfs_create_group(kobj, hsag);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto err;</div><div class='del'>-		device-&gt;hw_stats_ag = hsag;</div><div class='del'>-		device-&gt;hw_stats = stats;</div><div class='add'>+	for (i = 0; i &lt; data-&gt;stats-&gt;num_counters; i++) {</div><div class='add'>+		if (data-&gt;stats-&gt;descs[i].flags &amp; IB_STAT_FLAG_OPTIONAL) {</div><div class='add'>+			opstat_skipped = true;</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		WARN_ON(opstat_skipped);</div><div class='add'>+		attr = &amp;data-&gt;attrs[pos];</div><div class='add'>+		sysfs_attr_init(&amp;attr-&gt;attr.attr);</div><div class='add'>+		attr-&gt;attr.attr.name = data-&gt;stats-&gt;descs[i].name;</div><div class='add'>+		attr-&gt;attr.attr.mode = 0444;</div><div class='add'>+		attr-&gt;attr.show = hw_stat_port_show;</div><div class='add'>+		attr-&gt;show = show_hw_stats;</div><div class='add'>+		group-&gt;attrs[pos] = &amp;attr-&gt;attr.attr;</div><div class='add'>+		pos++;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return;</div><div class='add'>+	attr = &amp;data-&gt;attrs[pos];</div><div class='add'>+	sysfs_attr_init(&amp;attr-&gt;attr.attr);</div><div class='add'>+	attr-&gt;attr.attr.name = "lifespan";</div><div class='add'>+	attr-&gt;attr.attr.mode = 0644;</div><div class='add'>+	attr-&gt;attr.show = hw_stat_port_show;</div><div class='add'>+	attr-&gt;show = show_stats_lifespan;</div><div class='add'>+	attr-&gt;attr.store = hw_stat_port_store;</div><div class='add'>+	attr-&gt;store = set_stats_lifespan;</div><div class='add'>+	group-&gt;attrs[pos] = &amp;attr-&gt;attr.attr;</div><div class='add'>+</div><div class='add'>+	port-&gt;hw_stats_data = data;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-err:</div><div class='del'>-	for (; i &gt;= 0; i--)</div><div class='del'>-		kfree(hsag-&gt;attrs[i]);</div><div class='del'>-err_free_hsag:</div><div class='del'>-	kfree(hsag);</div><div class='del'>-err_free_stats:</div><div class='del'>-	kfree(stats);</div><div class='del'>-	return;</div><div class='add'>+struct rdma_hw_stats *ib_get_hw_stats_port(struct ib_device *ibdev,</div><div class='add'>+					   u32 port_num)</div><div class='add'>+{</div><div class='add'>+	if (!ibdev-&gt;port_data || !rdma_is_port_valid(ibdev, port_num) ||</div><div class='add'>+	    !ibdev-&gt;port_data[port_num].sysfs-&gt;hw_stats_data)</div><div class='add'>+		return NULL;</div><div class='add'>+	return ibdev-&gt;port_data[port_num].sysfs-&gt;hw_stats_data-&gt;stats;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int add_port(struct ib_core_device *coredev, int port_num)</div><div class='add'>+static int</div><div class='add'>+alloc_port_table_group(const char *name, struct attribute_group *group,</div><div class='add'>+		       struct port_table_attribute *attrs, size_t num,</div><div class='add'>+		       ssize_t (*show)(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+				       struct ib_port_attribute *, char *buf))</div><div class='ctx'> {</div><div class='del'>-	struct ib_device *device = rdma_device_to_ibdev(&amp;coredev-&gt;dev);</div><div class='del'>-	bool is_full_dev = &amp;device-&gt;coredev == coredev;</div><div class='del'>-	struct ib_port *p;</div><div class='del'>-	struct ib_port_attr attr;</div><div class='add'>+	struct attribute **attr_list;</div><div class='ctx'> 	int i;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	ret = ib_query_port(device, port_num, &amp;attr);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='ctx'> </div><div class='del'>-	p = kzalloc(sizeof *p, GFP_KERNEL);</div><div class='del'>-	if (!p)</div><div class='add'>+	attr_list = kcalloc(num + 1, sizeof(*attr_list), GFP_KERNEL);</div><div class='add'>+	if (!attr_list)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	p-&gt;ibdev      = device;</div><div class='del'>-	p-&gt;port_num   = port_num;</div><div class='add'>+	for (i = 0; i &lt; num; i++) {</div><div class='add'>+		struct port_table_attribute *element = &amp;attrs[i];</div><div class='ctx'> </div><div class='del'>-	ret = kobject_init_and_add(&amp;p-&gt;kobj, &amp;port_type,</div><div class='del'>-				   coredev-&gt;ports_kobj,</div><div class='del'>-				   "%d", port_num);</div><div class='del'>-	if (ret) {</div><div class='del'>-		kfree(p);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='add'>+		if (snprintf(element-&gt;name, sizeof(element-&gt;name), "%d", i) &gt;=</div><div class='add'>+		    sizeof(element-&gt;name))</div><div class='add'>+			goto err;</div><div class='ctx'> </div><div class='del'>-	p-&gt;gid_attr_group = kzalloc(sizeof(*p-&gt;gid_attr_group), GFP_KERNEL);</div><div class='del'>-	if (!p-&gt;gid_attr_group) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err_put;</div><div class='del'>-	}</div><div class='add'>+		sysfs_attr_init(&amp;element-&gt;attr.attr);</div><div class='add'>+		element-&gt;attr.attr.name = element-&gt;name;</div><div class='add'>+		element-&gt;attr.attr.mode = 0444;</div><div class='add'>+		element-&gt;attr.show = show;</div><div class='add'>+		element-&gt;index = i;</div><div class='ctx'> </div><div class='del'>-	p-&gt;gid_attr_group-&gt;port = p;</div><div class='del'>-	ret = kobject_init_and_add(&amp;p-&gt;gid_attr_group-&gt;kobj, &amp;gid_attr_type,</div><div class='del'>-				   &amp;p-&gt;kobj, "gid_attrs");</div><div class='del'>-	if (ret) {</div><div class='del'>-		kfree(p-&gt;gid_attr_group);</div><div class='del'>-		goto err_put;</div><div class='add'>+		attr_list[i] = &amp;element-&gt;attr.attr;</div><div class='ctx'> 	}</div><div class='add'>+	group-&gt;name = name;</div><div class='add'>+	group-&gt;attrs = attr_list;</div><div class='add'>+	return 0;</div><div class='add'>+err:</div><div class='add'>+	kfree(attr_list);</div><div class='add'>+	return -EINVAL;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (device-&gt;ops.process_mad &amp;&amp; is_full_dev) {</div><div class='del'>-		p-&gt;pma_table = get_counter_table(device, port_num);</div><div class='del'>-		ret = sysfs_create_group(&amp;p-&gt;kobj, p-&gt;pma_table);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto err_put_gid_attrs;</div><div class='del'>-	}</div><div class='add'>+/*</div><div class='add'>+ * Create the sysfs:</div><div class='add'>+ *  ibp0s9/ports/XX/gid_attrs/{ndevs,types}/YYY</div><div class='add'>+ * YYY is the gid table index in decimal</div><div class='add'>+ */</div><div class='add'>+static int setup_gid_attrs(struct ib_port *port,</div><div class='add'>+			   const struct ib_port_attr *attr)</div><div class='add'>+{</div><div class='add'>+	struct gid_attr_group *gid_attr_group;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	p-&gt;gid_group.name  = "gids";</div><div class='del'>-	p-&gt;gid_group.attrs = alloc_group_attrs(show_port_gid, attr.gid_tbl_len);</div><div class='del'>-	if (!p-&gt;gid_group.attrs) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err_remove_pma;</div><div class='del'>-	}</div><div class='add'>+	gid_attr_group = kzalloc(struct_size(gid_attr_group, attrs_list,</div><div class='add'>+					     size_mul(attr-&gt;gid_tbl_len, 2)),</div><div class='add'>+				 GFP_KERNEL);</div><div class='add'>+	if (!gid_attr_group)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	gid_attr_group-&gt;port = port;</div><div class='add'>+	kobject_init(&amp;gid_attr_group-&gt;kobj, &amp;gid_attr_type);</div><div class='ctx'> </div><div class='del'>-	ret = sysfs_create_group(&amp;p-&gt;kobj, &amp;p-&gt;gid_group);</div><div class='add'>+	ret = alloc_port_table_group("ndevs", &amp;gid_attr_group-&gt;groups[0],</div><div class='add'>+				     gid_attr_group-&gt;attrs_list,</div><div class='add'>+				     attr-&gt;gid_tbl_len,</div><div class='add'>+				     show_port_gid_attr_ndev);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err_free_gid;</div><div class='add'>+		goto err_put;</div><div class='add'>+	gid_attr_group-&gt;groups_list[0] = &amp;gid_attr_group-&gt;groups[0];</div><div class='ctx'> </div><div class='del'>-	p-&gt;gid_attr_group-&gt;ndev.name = "ndevs";</div><div class='del'>-	p-&gt;gid_attr_group-&gt;ndev.attrs = alloc_group_attrs(show_port_gid_attr_ndev,</div><div class='del'>-							  attr.gid_tbl_len);</div><div class='del'>-	if (!p-&gt;gid_attr_group-&gt;ndev.attrs) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err_remove_gid;</div><div class='del'>-	}</div><div class='add'>+	ret = alloc_port_table_group(</div><div class='add'>+		"types", &amp;gid_attr_group-&gt;groups[1],</div><div class='add'>+		gid_attr_group-&gt;attrs_list + attr-&gt;gid_tbl_len,</div><div class='add'>+		attr-&gt;gid_tbl_len, show_port_gid_attr_gid_type);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_put;</div><div class='add'>+	gid_attr_group-&gt;groups_list[1] = &amp;gid_attr_group-&gt;groups[1];</div><div class='ctx'> </div><div class='del'>-	ret = sysfs_create_group(&amp;p-&gt;gid_attr_group-&gt;kobj,</div><div class='del'>-				 &amp;p-&gt;gid_attr_group-&gt;ndev);</div><div class='add'>+	ret = kobject_add(&amp;gid_attr_group-&gt;kobj, &amp;port-&gt;kobj, "gid_attrs");</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err_free_gid_ndev;</div><div class='add'>+		goto err_put;</div><div class='add'>+	ret = sysfs_create_groups(&amp;gid_attr_group-&gt;kobj,</div><div class='add'>+				  gid_attr_group-&gt;groups_list);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_del;</div><div class='add'>+	port-&gt;gid_attr_group = gid_attr_group;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-	p-&gt;gid_attr_group-&gt;type.name = "types";</div><div class='del'>-	p-&gt;gid_attr_group-&gt;type.attrs = alloc_group_attrs(show_port_gid_attr_gid_type,</div><div class='del'>-							  attr.gid_tbl_len);</div><div class='del'>-	if (!p-&gt;gid_attr_group-&gt;type.attrs) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err_remove_gid_ndev;</div><div class='del'>-	}</div><div class='add'>+err_del:</div><div class='add'>+	kobject_del(&amp;gid_attr_group-&gt;kobj);</div><div class='add'>+err_put:</div><div class='add'>+	kobject_put(&amp;gid_attr_group-&gt;kobj);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	ret = sysfs_create_group(&amp;p-&gt;gid_attr_group-&gt;kobj,</div><div class='del'>-				 &amp;p-&gt;gid_attr_group-&gt;type);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_free_gid_type;</div><div class='add'>+static void destroy_gid_attrs(struct ib_port *port)</div><div class='add'>+{</div><div class='add'>+	struct gid_attr_group *gid_attr_group = port-&gt;gid_attr_group;</div><div class='ctx'> </div><div class='del'>-	p-&gt;pkey_group.name  = "pkeys";</div><div class='del'>-	p-&gt;pkey_group.attrs = alloc_group_attrs(show_port_pkey,</div><div class='del'>-						attr.pkey_tbl_len);</div><div class='del'>-	if (!p-&gt;pkey_group.attrs) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err_remove_gid_type;</div><div class='del'>-	}</div><div class='add'>+	if (!gid_attr_group)</div><div class='add'>+		return;</div><div class='add'>+	sysfs_remove_groups(&amp;gid_attr_group-&gt;kobj, gid_attr_group-&gt;groups_list);</div><div class='add'>+	kobject_del(&amp;gid_attr_group-&gt;kobj);</div><div class='add'>+	kobject_put(&amp;gid_attr_group-&gt;kobj);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * Create the sysfs:</div><div class='add'>+ *  ibp0s9/ports/XX/{gids,pkeys,counters}/YYY</div><div class='add'>+ */</div><div class='add'>+static struct ib_port *setup_port(struct ib_core_device *coredev, int port_num,</div><div class='add'>+				  const struct ib_port_attr *attr)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *device = rdma_device_to_ibdev(&amp;coredev-&gt;dev);</div><div class='add'>+	bool is_full_dev = &amp;device-&gt;coredev == coredev;</div><div class='add'>+	const struct attribute_group **cur_group;</div><div class='add'>+	struct ib_port *p;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = sysfs_create_group(&amp;p-&gt;kobj, &amp;p-&gt;pkey_group);</div><div class='add'>+	p = kvzalloc(struct_size(p, attrs_list,</div><div class='add'>+				size_add(attr-&gt;gid_tbl_len, attr-&gt;pkey_tbl_len)),</div><div class='add'>+		     GFP_KERNEL);</div><div class='add'>+	if (!p)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+	p-&gt;ibdev = device;</div><div class='add'>+	p-&gt;port_num = port_num;</div><div class='add'>+	kobject_init(&amp;p-&gt;kobj, &amp;port_type);</div><div class='add'>+</div><div class='add'>+	if (device-&gt;port_data &amp;&amp; is_full_dev)</div><div class='add'>+		device-&gt;port_data[port_num].sysfs = p;</div><div class='add'>+</div><div class='add'>+	cur_group = p-&gt;groups_list;</div><div class='add'>+	ret = alloc_port_table_group("gids", &amp;p-&gt;groups[0], p-&gt;attrs_list,</div><div class='add'>+				     attr-&gt;gid_tbl_len, show_port_gid);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err_free_pkey;</div><div class='add'>+		goto err_put;</div><div class='add'>+	*cur_group++ = &amp;p-&gt;groups[0];</div><div class='ctx'> </div><div class='del'>-	if (device-&gt;ops.init_port &amp;&amp; is_full_dev) {</div><div class='del'>-		ret = device-&gt;ops.init_port(device, port_num, &amp;p-&gt;kobj);</div><div class='add'>+	if (attr-&gt;pkey_tbl_len) {</div><div class='add'>+		ret = alloc_port_table_group("pkeys", &amp;p-&gt;groups[1],</div><div class='add'>+					     p-&gt;attrs_list + attr-&gt;gid_tbl_len,</div><div class='add'>+					     attr-&gt;pkey_tbl_len, show_port_pkey);</div><div class='ctx'> 		if (ret)</div><div class='del'>-			goto err_remove_pkey;</div><div class='add'>+			goto err_put;</div><div class='add'>+		*cur_group++ = &amp;p-&gt;groups[1];</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * If port == 0, it means hw_counters are per device and not per</div><div class='del'>-	 * port, so holder should be device. Therefore skip per port conunter</div><div class='del'>-	 * initialization.</div><div class='add'>+	 * port, so holder should be device. Therefore skip per port</div><div class='add'>+	 * counter initialization.</div><div class='ctx'> 	 */</div><div class='del'>-	if (device-&gt;ops.alloc_hw_stats &amp;&amp; port_num &amp;&amp; is_full_dev)</div><div class='del'>-		setup_hw_stats(device, p, port_num);</div><div class='del'>-</div><div class='del'>-	list_add_tail(&amp;p-&gt;kobj.entry, &amp;coredev-&gt;port_list);</div><div class='del'>-</div><div class='del'>-	kobject_uevent(&amp;p-&gt;kobj, KOBJ_ADD);</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err_remove_pkey:</div><div class='del'>-	sysfs_remove_group(&amp;p-&gt;kobj, &amp;p-&gt;pkey_group);</div><div class='del'>-</div><div class='del'>-err_free_pkey:</div><div class='del'>-	for (i = 0; i &lt; attr.pkey_tbl_len; ++i)</div><div class='del'>-		kfree(p-&gt;pkey_group.attrs[i]);</div><div class='del'>-</div><div class='del'>-	kfree(p-&gt;pkey_group.attrs);</div><div class='del'>-	p-&gt;pkey_group.attrs = NULL;</div><div class='del'>-</div><div class='del'>-err_remove_gid_type:</div><div class='del'>-	sysfs_remove_group(&amp;p-&gt;gid_attr_group-&gt;kobj,</div><div class='del'>-			   &amp;p-&gt;gid_attr_group-&gt;type);</div><div class='del'>-</div><div class='del'>-err_free_gid_type:</div><div class='del'>-	for (i = 0; i &lt; attr.gid_tbl_len; ++i)</div><div class='del'>-		kfree(p-&gt;gid_attr_group-&gt;type.attrs[i]);</div><div class='add'>+	if (port_num &amp;&amp; is_full_dev) {</div><div class='add'>+		ret = setup_hw_port_stats(p, &amp;p-&gt;groups[2]);</div><div class='add'>+		if (ret &amp;&amp; ret != -EOPNOTSUPP)</div><div class='add'>+			goto err_put;</div><div class='add'>+		if (!ret)</div><div class='add'>+			*cur_group++ = &amp;p-&gt;groups[2];</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	kfree(p-&gt;gid_attr_group-&gt;type.attrs);</div><div class='del'>-	p-&gt;gid_attr_group-&gt;type.attrs = NULL;</div><div class='add'>+	if (device-&gt;ops.process_mad &amp;&amp; is_full_dev)</div><div class='add'>+		*cur_group++ = get_counter_table(device, port_num);</div><div class='ctx'> </div><div class='del'>-err_remove_gid_ndev:</div><div class='del'>-	sysfs_remove_group(&amp;p-&gt;gid_attr_group-&gt;kobj,</div><div class='del'>-			   &amp;p-&gt;gid_attr_group-&gt;ndev);</div><div class='add'>+	ret = kobject_add(&amp;p-&gt;kobj, coredev-&gt;ports_kobj, "%d", port_num);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_put;</div><div class='add'>+	ret = sysfs_create_groups(&amp;p-&gt;kobj, p-&gt;groups_list);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_del;</div><div class='add'>+	if (is_full_dev) {</div><div class='add'>+		ret = sysfs_create_groups(&amp;p-&gt;kobj, device-&gt;ops.port_groups);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_groups;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-err_free_gid_ndev:</div><div class='del'>-	for (i = 0; i &lt; attr.gid_tbl_len; ++i)</div><div class='del'>-		kfree(p-&gt;gid_attr_group-&gt;ndev.attrs[i]);</div><div class='add'>+	list_add_tail(&amp;p-&gt;kobj.entry, &amp;coredev-&gt;port_list);</div><div class='add'>+	return p;</div><div class='ctx'> </div><div class='del'>-	kfree(p-&gt;gid_attr_group-&gt;ndev.attrs);</div><div class='del'>-	p-&gt;gid_attr_group-&gt;ndev.attrs = NULL;</div><div class='add'>+err_groups:</div><div class='add'>+	sysfs_remove_groups(&amp;p-&gt;kobj, p-&gt;groups_list);</div><div class='add'>+err_del:</div><div class='add'>+	kobject_del(&amp;p-&gt;kobj);</div><div class='add'>+err_put:</div><div class='add'>+	if (device-&gt;port_data &amp;&amp; is_full_dev)</div><div class='add'>+		device-&gt;port_data[port_num].sysfs = NULL;</div><div class='add'>+	kobject_put(&amp;p-&gt;kobj);</div><div class='add'>+	return ERR_PTR(ret);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-err_remove_gid:</div><div class='del'>-	sysfs_remove_group(&amp;p-&gt;kobj, &amp;p-&gt;gid_group);</div><div class='add'>+static void destroy_port(struct ib_core_device *coredev, struct ib_port *port)</div><div class='add'>+{</div><div class='add'>+	bool is_full_dev = &amp;port-&gt;ibdev-&gt;coredev == coredev;</div><div class='ctx'> </div><div class='del'>-err_free_gid:</div><div class='del'>-	for (i = 0; i &lt; attr.gid_tbl_len; ++i)</div><div class='del'>-		kfree(p-&gt;gid_group.attrs[i]);</div><div class='add'>+	list_del(&amp;port-&gt;kobj.entry);</div><div class='add'>+	if (is_full_dev)</div><div class='add'>+		sysfs_remove_groups(&amp;port-&gt;kobj, port-&gt;ibdev-&gt;ops.port_groups);</div><div class='ctx'> </div><div class='del'>-	kfree(p-&gt;gid_group.attrs);</div><div class='del'>-	p-&gt;gid_group.attrs = NULL;</div><div class='add'>+	sysfs_remove_groups(&amp;port-&gt;kobj, port-&gt;groups_list);</div><div class='add'>+	kobject_del(&amp;port-&gt;kobj);</div><div class='ctx'> </div><div class='del'>-err_remove_pma:</div><div class='del'>-	if (p-&gt;pma_table)</div><div class='del'>-		sysfs_remove_group(&amp;p-&gt;kobj, p-&gt;pma_table);</div><div class='add'>+	if (port-&gt;ibdev-&gt;port_data &amp;&amp;</div><div class='add'>+	    port-&gt;ibdev-&gt;port_data[port-&gt;port_num].sysfs == port)</div><div class='add'>+		port-&gt;ibdev-&gt;port_data[port-&gt;port_num].sysfs = NULL;</div><div class='ctx'> </div><div class='del'>-err_put_gid_attrs:</div><div class='del'>-	kobject_put(&amp;p-&gt;gid_attr_group-&gt;kobj);</div><div class='add'>+	kobject_put(&amp;port-&gt;kobj);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-err_put:</div><div class='del'>-	kobject_put(&amp;p-&gt;kobj);</div><div class='del'>-	return ret;</div><div class='add'>+static const char *node_type_string(int node_type)</div><div class='add'>+{</div><div class='add'>+	switch (node_type) {</div><div class='add'>+	case RDMA_NODE_IB_CA:</div><div class='add'>+		return "CA";</div><div class='add'>+	case RDMA_NODE_IB_SWITCH:</div><div class='add'>+		return "switch";</div><div class='add'>+	case RDMA_NODE_IB_ROUTER:</div><div class='add'>+		return "router";</div><div class='add'>+	case RDMA_NODE_RNIC:</div><div class='add'>+		return "RNIC";</div><div class='add'>+	case RDMA_NODE_USNIC:</div><div class='add'>+		return "usNIC";</div><div class='add'>+	case RDMA_NODE_USNIC_UDP:</div><div class='add'>+		return "usNIC UDP";</div><div class='add'>+	case RDMA_NODE_UNSPECIFIED:</div><div class='add'>+		return "unspecified";</div><div class='add'>+	}</div><div class='add'>+	return "&lt;unknown&gt;";</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t node_type_show(struct device *device,</div><div class='hunk'>@@ -1196,16 +1311,8 @@ static ssize_t node_type_show(struct device *device,</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *dev = rdma_device_to_ibdev(device);</div><div class='ctx'> </div><div class='del'>-	switch (dev-&gt;node_type) {</div><div class='del'>-	case RDMA_NODE_IB_CA:	  return sprintf(buf, "%d: CA\n", dev-&gt;node_type);</div><div class='del'>-	case RDMA_NODE_RNIC:	  return sprintf(buf, "%d: RNIC\n", dev-&gt;node_type);</div><div class='del'>-	case RDMA_NODE_USNIC:	  return sprintf(buf, "%d: usNIC\n", dev-&gt;node_type);</div><div class='del'>-	case RDMA_NODE_USNIC_UDP: return sprintf(buf, "%d: usNIC UDP\n", dev-&gt;node_type);</div><div class='del'>-	case RDMA_NODE_UNSPECIFIED: return sprintf(buf, "%d: unspecified\n", dev-&gt;node_type);</div><div class='del'>-	case RDMA_NODE_IB_SWITCH: return sprintf(buf, "%d: switch\n", dev-&gt;node_type);</div><div class='del'>-	case RDMA_NODE_IB_ROUTER: return sprintf(buf, "%d: router\n", dev-&gt;node_type);</div><div class='del'>-	default:		  return sprintf(buf, "%d: &lt;unknown&gt;\n", dev-&gt;node_type);</div><div class='del'>-	}</div><div class='add'>+	return sysfs_emit(buf, "%u: %s\n", dev-&gt;node_type,</div><div class='add'>+			  node_type_string(dev-&gt;node_type));</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(node_type);</div><div class='ctx'> </div><div class='hunk'>@@ -1213,12 +1320,13 @@ static ssize_t sys_image_guid_show(struct device *device,</div><div class='ctx'> 				   struct device_attribute *dev_attr, char *buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *dev = rdma_device_to_ibdev(device);</div><div class='add'>+	__be16 *guid = (__be16 *)&amp;dev-&gt;attrs.sys_image_guid;</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%04x:%04x:%04x:%04x\n",</div><div class='del'>-		       be16_to_cpu(((__be16 *) &amp;dev-&gt;attrs.sys_image_guid)[0]),</div><div class='del'>-		       be16_to_cpu(((__be16 *) &amp;dev-&gt;attrs.sys_image_guid)[1]),</div><div class='del'>-		       be16_to_cpu(((__be16 *) &amp;dev-&gt;attrs.sys_image_guid)[2]),</div><div class='del'>-		       be16_to_cpu(((__be16 *) &amp;dev-&gt;attrs.sys_image_guid)[3]));</div><div class='add'>+	return sysfs_emit(buf, "%04x:%04x:%04x:%04x\n",</div><div class='add'>+			  be16_to_cpu(guid[0]),</div><div class='add'>+			  be16_to_cpu(guid[1]),</div><div class='add'>+			  be16_to_cpu(guid[2]),</div><div class='add'>+			  be16_to_cpu(guid[3]));</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(sys_image_guid);</div><div class='ctx'> </div><div class='hunk'>@@ -1226,12 +1334,13 @@ static ssize_t node_guid_show(struct device *device,</div><div class='ctx'> 			      struct device_attribute *attr, char *buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *dev = rdma_device_to_ibdev(device);</div><div class='add'>+	__be16 *node_guid = (__be16 *)&amp;dev-&gt;node_guid;</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%04x:%04x:%04x:%04x\n",</div><div class='del'>-		       be16_to_cpu(((__be16 *) &amp;dev-&gt;node_guid)[0]),</div><div class='del'>-		       be16_to_cpu(((__be16 *) &amp;dev-&gt;node_guid)[1]),</div><div class='del'>-		       be16_to_cpu(((__be16 *) &amp;dev-&gt;node_guid)[2]),</div><div class='del'>-		       be16_to_cpu(((__be16 *) &amp;dev-&gt;node_guid)[3]));</div><div class='add'>+	return sysfs_emit(buf, "%04x:%04x:%04x:%04x\n",</div><div class='add'>+			  be16_to_cpu(node_guid[0]),</div><div class='add'>+			  be16_to_cpu(node_guid[1]),</div><div class='add'>+			  be16_to_cpu(node_guid[2]),</div><div class='add'>+			  be16_to_cpu(node_guid[3]));</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(node_guid);</div><div class='ctx'> </div><div class='hunk'>@@ -1240,7 +1349,7 @@ static ssize_t node_desc_show(struct device *device,</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *dev = rdma_device_to_ibdev(device);</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%.64s\n", dev-&gt;node_desc);</div><div class='add'>+	return sysfs_emit(buf, "%.64s\n", dev-&gt;node_desc);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t node_desc_store(struct device *device,</div><div class='hunk'>@@ -1252,7 +1361,7 @@ static ssize_t node_desc_store(struct device *device,</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (!dev-&gt;ops.modify_device)</div><div class='del'>-		return -EIO;</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	memcpy(desc.node_desc, buf, min_t(int, count, IB_DEVICE_NODE_DESC_MAX));</div><div class='ctx'> 	ret = ib_modify_device(dev, IB_DEVICE_MODIFY_NODE_DESC, &amp;desc);</div><div class='hunk'>@@ -1267,10 +1376,11 @@ static ssize_t fw_ver_show(struct device *device, struct device_attribute *attr,</div><div class='ctx'> 			   char *buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *dev = rdma_device_to_ibdev(device);</div><div class='add'>+	char version[IB_FW_VERSION_NAME_MAX] = {};</div><div class='ctx'> </div><div class='del'>-	ib_get_device_fw_str(dev, buf);</div><div class='del'>-	strlcat(buf, "\n", IB_FW_VERSION_NAME_MAX);</div><div class='del'>-	return strlen(buf);</div><div class='add'>+	ib_get_device_fw_str(dev, version);</div><div class='add'>+</div><div class='add'>+	return sysfs_emit(buf, "%s\n", version);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(fw_ver);</div><div class='ctx'> </div><div class='hunk'>@@ -1294,21 +1404,8 @@ void ib_free_port_attrs(struct ib_core_device *coredev)</div><div class='ctx'> 	list_for_each_entry_safe(p, t, &amp;coredev-&gt;port_list, entry) {</div><div class='ctx'> 		struct ib_port *port = container_of(p, struct ib_port, kobj);</div><div class='ctx'> </div><div class='del'>-		list_del(&amp;p-&gt;entry);</div><div class='del'>-		if (port-&gt;hw_stats_ag)</div><div class='del'>-			free_hsag(&amp;port-&gt;kobj, port-&gt;hw_stats_ag);</div><div class='del'>-		kfree(port-&gt;hw_stats);</div><div class='del'>-</div><div class='del'>-		if (port-&gt;pma_table)</div><div class='del'>-			sysfs_remove_group(p, port-&gt;pma_table);</div><div class='del'>-		sysfs_remove_group(p, &amp;port-&gt;pkey_group);</div><div class='del'>-		sysfs_remove_group(p, &amp;port-&gt;gid_group);</div><div class='del'>-		sysfs_remove_group(&amp;port-&gt;gid_attr_group-&gt;kobj,</div><div class='del'>-				   &amp;port-&gt;gid_attr_group-&gt;ndev);</div><div class='del'>-		sysfs_remove_group(&amp;port-&gt;gid_attr_group-&gt;kobj,</div><div class='del'>-				   &amp;port-&gt;gid_attr_group-&gt;type);</div><div class='del'>-		kobject_put(&amp;port-&gt;gid_attr_group-&gt;kobj);</div><div class='del'>-		kobject_put(p);</div><div class='add'>+		destroy_gid_attrs(port);</div><div class='add'>+		destroy_port(coredev, port);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	kobject_put(coredev-&gt;ports_kobj);</div><div class='hunk'>@@ -1317,7 +1414,7 @@ void ib_free_port_attrs(struct ib_core_device *coredev)</div><div class='ctx'> int ib_setup_port_attrs(struct ib_core_device *coredev)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *device = rdma_device_to_ibdev(&amp;coredev-&gt;dev);</div><div class='del'>-	unsigned int port;</div><div class='add'>+	u32 port_num;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	coredev-&gt;ports_kobj = kobject_create_and_add("ports",</div><div class='hunk'>@@ -1325,12 +1422,24 @@ int ib_setup_port_attrs(struct ib_core_device *coredev)</div><div class='ctx'> 	if (!coredev-&gt;ports_kobj)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	rdma_for_each_port (device, port) {</div><div class='del'>-		ret = add_port(coredev, port);</div><div class='add'>+	rdma_for_each_port (device, port_num) {</div><div class='add'>+		struct ib_port_attr attr;</div><div class='add'>+		struct ib_port *port;</div><div class='add'>+</div><div class='add'>+		ret = ib_query_port(device, port_num, &amp;attr);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto err_put;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='add'>+		port = setup_port(coredev, port_num, &amp;attr);</div><div class='add'>+		if (IS_ERR(port)) {</div><div class='add'>+			ret = PTR_ERR(port);</div><div class='add'>+			goto err_put;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ret = setup_gid_attrs(port, &amp;attr);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_put;</div><div class='add'>+	}</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> err_put:</div><div class='hunk'>@@ -1338,68 +1447,27 @@ err_put:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int ib_device_register_sysfs(struct ib_device *device)</div><div class='del'>-{</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	ret = ib_setup_port_attrs(&amp;device-&gt;coredev);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	if (device-&gt;ops.alloc_hw_stats)</div><div class='del'>-		setup_hw_stats(device, NULL, 0);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void ib_device_unregister_sysfs(struct ib_device *device)</div><div class='del'>-{</div><div class='del'>-	if (device-&gt;hw_stats_ag)</div><div class='del'>-		free_hsag(&amp;device-&gt;dev.kobj, device-&gt;hw_stats_ag);</div><div class='del'>-	kfree(device-&gt;hw_stats);</div><div class='del'>-</div><div class='del'>-	ib_free_port_attrs(&amp;device-&gt;coredev);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> /**</div><div class='del'>- * ib_port_register_module_stat - add module counters under relevant port</div><div class='del'>- *  of IB device.</div><div class='add'>+ * ib_port_register_client_groups - Add an ib_client's attributes to the port</div><div class='ctx'>  *</div><div class='del'>- * @device: IB device to add counters</div><div class='add'>+ * @ibdev: IB device to add counters</div><div class='ctx'>  * @port_num: valid port number</div><div class='del'>- * @kobj: pointer to the kobject to initialize</div><div class='del'>- * @ktype: pointer to the ktype for this kobject.</div><div class='del'>- * @name: the name of the kobject</div><div class='add'>+ * @groups: Group list of attributes</div><div class='add'>+ *</div><div class='add'>+ * Do not use. Only for legacy sysfs compatibility.</div><div class='ctx'>  */</div><div class='del'>-int ib_port_register_module_stat(struct ib_device *device, u8 port_num,</div><div class='del'>-				 struct kobject *kobj, struct kobj_type *ktype,</div><div class='del'>-				 const char *name)</div><div class='add'>+int ib_port_register_client_groups(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+				   const struct attribute_group **groups)</div><div class='ctx'> {</div><div class='del'>-	struct kobject *p, *t;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	list_for_each_entry_safe(p, t, &amp;device-&gt;coredev.port_list, entry) {</div><div class='del'>-		struct ib_port *port = container_of(p, struct ib_port, kobj);</div><div class='del'>-</div><div class='del'>-		if (port-&gt;port_num != port_num)</div><div class='del'>-			continue;</div><div class='del'>-</div><div class='del'>-		ret = kobject_init_and_add(kobj, ktype, &amp;port-&gt;kobj, "%s",</div><div class='del'>-					   name);</div><div class='del'>-		if (ret)</div><div class='del'>-			return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='add'>+	return sysfs_create_groups(&amp;ibdev-&gt;port_data[port_num].sysfs-&gt;kobj,</div><div class='add'>+				   groups);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(ib_port_register_module_stat);</div><div class='add'>+EXPORT_SYMBOL(ib_port_register_client_groups);</div><div class='ctx'> </div><div class='del'>-/**</div><div class='del'>- * ib_port_unregister_module_stat - release module counters</div><div class='del'>- * @kobj: pointer to the kobject to release</div><div class='del'>- */</div><div class='del'>-void ib_port_unregister_module_stat(struct kobject *kobj)</div><div class='add'>+void ib_port_unregister_client_groups(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+				      const struct attribute_group **groups)</div><div class='ctx'> {</div><div class='del'>-	kobject_put(kobj);</div><div class='add'>+	return sysfs_remove_groups(&amp;ibdev-&gt;port_data[port_num].sysfs-&gt;kobj,</div><div class='add'>+				   groups);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(ib_port_unregister_module_stat);</div><div class='add'>+EXPORT_SYMBOL(ib_port_unregister_client_groups);</div><div class='head'>diff --git a/drivers/infiniband/core/trace.c b/drivers/infiniband/core/trace.c<br/>new file mode 100644<br/>index 000000000000..31e7860d35bf<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/trace.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/trace.c</a></div><div class='hunk'>@@ -0,0 +1,12 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+/*</div><div class='add'>+ * Trace points for core RDMA functions.</div><div class='add'>+ *</div><div class='add'>+ * Author: Chuck Lever &lt;chuck.lever@oracle.com&gt;</div><div class='add'>+ *</div><div class='add'>+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#define CREATE_TRACE_POINTS</div><div class='add'>+</div><div class='add'>+#include &lt;trace/events/rdma_core.h&gt;</div><div class='head'>diff --git a/drivers/infiniband/core/ucaps.c b/drivers/infiniband/core/ucaps.c<br/>new file mode 100644<br/>index 000000000000..de5cb8bf0a61<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/ucaps.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/ucaps.c</a></div><div class='hunk'>@@ -0,0 +1,267 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2025, NVIDIA CORPORATION &amp; AFFILIATES. All rights reserved</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/kref.h&gt;</div><div class='add'>+#include &lt;linux/cdev.h&gt;</div><div class='add'>+#include &lt;linux/mutex.h&gt;</div><div class='add'>+#include &lt;linux/file.h&gt;</div><div class='add'>+#include &lt;linux/fs.h&gt;</div><div class='add'>+#include &lt;rdma/ib_ucaps.h&gt;</div><div class='add'>+</div><div class='add'>+#define RDMA_UCAP_FIRST RDMA_UCAP_MLX5_CTRL_LOCAL</div><div class='add'>+</div><div class='add'>+static DEFINE_MUTEX(ucaps_mutex);</div><div class='add'>+static struct ib_ucap *ucaps_list[RDMA_UCAP_MAX];</div><div class='add'>+static bool ucaps_class_is_registered;</div><div class='add'>+static dev_t ucaps_base_dev;</div><div class='add'>+</div><div class='add'>+struct ib_ucap {</div><div class='add'>+	struct cdev cdev;</div><div class='add'>+	struct device dev;</div><div class='add'>+	struct kref ref;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const char *ucap_names[RDMA_UCAP_MAX] = {</div><div class='add'>+	[RDMA_UCAP_MLX5_CTRL_LOCAL] = "mlx5_perm_ctrl_local",</div><div class='add'>+	[RDMA_UCAP_MLX5_CTRL_OTHER_VHCA] = "mlx5_perm_ctrl_other_vhca"</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static char *ucaps_devnode(const struct device *dev, umode_t *mode)</div><div class='add'>+{</div><div class='add'>+	if (mode)</div><div class='add'>+		*mode = 0600;</div><div class='add'>+</div><div class='add'>+	return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct class ucaps_class = {</div><div class='add'>+	.name = "infiniband_ucaps",</div><div class='add'>+	.devnode = ucaps_devnode,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct file_operations ucaps_cdev_fops = {</div><div class='add'>+	.owner = THIS_MODULE,</div><div class='add'>+	.open = simple_open,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ib_cleanup_ucaps - cleanup all API resources and class.</div><div class='add'>+ *</div><div class='add'>+ * This is called once, when removing the ib_uverbs module.</div><div class='add'>+ */</div><div class='add'>+void ib_cleanup_ucaps(void)</div><div class='add'>+{</div><div class='add'>+	mutex_lock(&amp;ucaps_mutex);</div><div class='add'>+	if (!ucaps_class_is_registered) {</div><div class='add'>+		mutex_unlock(&amp;ucaps_mutex);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (int i = RDMA_UCAP_FIRST; i &lt; RDMA_UCAP_MAX; i++)</div><div class='add'>+		WARN_ON(ucaps_list[i]);</div><div class='add'>+</div><div class='add'>+	class_unregister(&amp;ucaps_class);</div><div class='add'>+	ucaps_class_is_registered = false;</div><div class='add'>+	unregister_chrdev_region(ucaps_base_dev, RDMA_UCAP_MAX);</div><div class='add'>+	mutex_unlock(&amp;ucaps_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_ucap_from_devt(dev_t devt, u64 *idx_mask)</div><div class='add'>+{</div><div class='add'>+	for (int type = RDMA_UCAP_FIRST; type &lt; RDMA_UCAP_MAX; type++) {</div><div class='add'>+		if (ucaps_list[type] &amp;&amp; ucaps_list[type]-&gt;dev.devt == devt) {</div><div class='add'>+			*idx_mask |= 1 &lt;&lt; type;</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return -EINVAL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_devt_from_fd(unsigned int fd, dev_t *ret_dev)</div><div class='add'>+{</div><div class='add'>+	struct file *file;</div><div class='add'>+</div><div class='add'>+	file = fget(fd);</div><div class='add'>+	if (!file)</div><div class='add'>+		return -EBADF;</div><div class='add'>+</div><div class='add'>+	*ret_dev = file_inode(file)-&gt;i_rdev;</div><div class='add'>+	fput(file);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ib_ucaps_init - Initialization required before ucap creation.</div><div class='add'>+ *</div><div class='add'>+ * Return: 0 on success, or a negative errno value on failure</div><div class='add'>+ */</div><div class='add'>+static int ib_ucaps_init(void)</div><div class='add'>+{</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	if (ucaps_class_is_registered)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = class_register(&amp;ucaps_class);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = alloc_chrdev_region(&amp;ucaps_base_dev, 0, RDMA_UCAP_MAX,</div><div class='add'>+				  ucaps_class.name);</div><div class='add'>+	if (ret &lt; 0) {</div><div class='add'>+		class_unregister(&amp;ucaps_class);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ucaps_class_is_registered = true;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ucap_dev_release(struct device *device)</div><div class='add'>+{</div><div class='add'>+	struct ib_ucap *ucap = container_of(device, struct ib_ucap, dev);</div><div class='add'>+</div><div class='add'>+	kfree(ucap);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ib_create_ucap - Add a ucap character device</div><div class='add'>+ * @type: UCAP type</div><div class='add'>+ *</div><div class='add'>+ * Creates a ucap character device in the /dev/infiniband directory. By default,</div><div class='add'>+ * the device has root-only read-write access.</div><div class='add'>+ *</div><div class='add'>+ * A driver may call this multiple times with the same UCAP type. A reference</div><div class='add'>+ * count tracks creations and deletions.</div><div class='add'>+ *</div><div class='add'>+ * Return: 0 on success, or a negative errno value on failure</div><div class='add'>+ */</div><div class='add'>+int ib_create_ucap(enum rdma_user_cap type)</div><div class='add'>+{</div><div class='add'>+	struct ib_ucap *ucap;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (type &gt;= RDMA_UCAP_MAX)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;ucaps_mutex);</div><div class='add'>+	ret = ib_ucaps_init();</div><div class='add'>+	if (ret)</div><div class='add'>+		goto unlock;</div><div class='add'>+</div><div class='add'>+	ucap = ucaps_list[type];</div><div class='add'>+	if (ucap) {</div><div class='add'>+		kref_get(&amp;ucap-&gt;ref);</div><div class='add'>+		mutex_unlock(&amp;ucaps_mutex);</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ucap = kzalloc(sizeof(*ucap), GFP_KERNEL);</div><div class='add'>+	if (!ucap) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto unlock;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	device_initialize(&amp;ucap-&gt;dev);</div><div class='add'>+	ucap-&gt;dev.class = &amp;ucaps_class;</div><div class='add'>+	ucap-&gt;dev.devt = MKDEV(MAJOR(ucaps_base_dev), type);</div><div class='add'>+	ucap-&gt;dev.release = ucap_dev_release;</div><div class='add'>+	ret = dev_set_name(&amp;ucap-&gt;dev, "%s", ucap_names[type]);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_device;</div><div class='add'>+</div><div class='add'>+	cdev_init(&amp;ucap-&gt;cdev, &amp;ucaps_cdev_fops);</div><div class='add'>+	ucap-&gt;cdev.owner = THIS_MODULE;</div><div class='add'>+</div><div class='add'>+	ret = cdev_device_add(&amp;ucap-&gt;cdev, &amp;ucap-&gt;dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_device;</div><div class='add'>+</div><div class='add'>+	kref_init(&amp;ucap-&gt;ref);</div><div class='add'>+	ucaps_list[type] = ucap;</div><div class='add'>+	mutex_unlock(&amp;ucaps_mutex);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_device:</div><div class='add'>+	put_device(&amp;ucap-&gt;dev);</div><div class='add'>+unlock:</div><div class='add'>+	mutex_unlock(&amp;ucaps_mutex);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_create_ucap);</div><div class='add'>+</div><div class='add'>+static void ib_release_ucap(struct kref *ref)</div><div class='add'>+{</div><div class='add'>+	struct ib_ucap *ucap = container_of(ref, struct ib_ucap, ref);</div><div class='add'>+	enum rdma_user_cap type;</div><div class='add'>+</div><div class='add'>+	for (type = RDMA_UCAP_FIRST; type &lt; RDMA_UCAP_MAX; type++) {</div><div class='add'>+		if (ucaps_list[type] == ucap)</div><div class='add'>+			break;</div><div class='add'>+	}</div><div class='add'>+	WARN_ON(type == RDMA_UCAP_MAX);</div><div class='add'>+</div><div class='add'>+	ucaps_list[type] = NULL;</div><div class='add'>+	cdev_device_del(&amp;ucap-&gt;cdev, &amp;ucap-&gt;dev);</div><div class='add'>+	put_device(&amp;ucap-&gt;dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ib_remove_ucap - Remove a ucap character device</div><div class='add'>+ * @type: User cap type</div><div class='add'>+ *</div><div class='add'>+ * Removes the ucap character device according to type. The device is completely</div><div class='add'>+ * removed from the filesystem when its reference count reaches 0.</div><div class='add'>+ */</div><div class='add'>+void ib_remove_ucap(enum rdma_user_cap type)</div><div class='add'>+{</div><div class='add'>+	struct ib_ucap *ucap;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;ucaps_mutex);</div><div class='add'>+	ucap = ucaps_list[type];</div><div class='add'>+	if (WARN_ON(!ucap))</div><div class='add'>+		goto end;</div><div class='add'>+</div><div class='add'>+	kref_put(&amp;ucap-&gt;ref, ib_release_ucap);</div><div class='add'>+end:</div><div class='add'>+	mutex_unlock(&amp;ucaps_mutex);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_remove_ucap);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ib_get_ucaps - Get bitmask of ucap types from file descriptors</div><div class='add'>+ * @fds: Array of file descriptors</div><div class='add'>+ * @fd_count: Number of file descriptors in the array</div><div class='add'>+ * @idx_mask: Bitmask to be updated based on the ucaps in the fd list</div><div class='add'>+ *</div><div class='add'>+ * Given an array of file descriptors, this function returns a bitmask of</div><div class='add'>+ * the ucaps where a bit is set if an FD for that ucap type was in the array.</div><div class='add'>+ *</div><div class='add'>+ * Return: 0 on success, or a negative errno value on failure</div><div class='add'>+ */</div><div class='add'>+int ib_get_ucaps(int *fds, int fd_count, uint64_t *idx_mask)</div><div class='add'>+{</div><div class='add'>+	int ret = 0;</div><div class='add'>+	dev_t dev;</div><div class='add'>+</div><div class='add'>+	*idx_mask = 0;</div><div class='add'>+	mutex_lock(&amp;ucaps_mutex);</div><div class='add'>+	for (int i = 0; i &lt; fd_count; i++) {</div><div class='add'>+		ret = get_devt_from_fd(fds[i], &amp;dev);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto end;</div><div class='add'>+</div><div class='add'>+		ret = get_ucap_from_devt(dev, idx_mask);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto end;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+end:</div><div class='add'>+	mutex_unlock(&amp;ucaps_mutex);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/core/ucm.c b/drivers/infiniband/core/ucm.c<br/>deleted file mode 100644<br/>index 8e7da2d41fd8..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/ucm.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/ucm.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,1350 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2005 Topspin Communications.  All rights reserved.</div><div class='del'>- * Copyright (c) 2005 Intel Corporation.  All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *	copyright notice, this list of conditions and the following</div><div class='del'>- *	disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *	copyright notice, this list of conditions and the following</div><div class='del'>- *	disclaimer in the documentation and/or other materials</div><div class='del'>- *	provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-</div><div class='del'>-#include &lt;linux/completion.h&gt;</div><div class='del'>-#include &lt;linux/init.h&gt;</div><div class='del'>-#include &lt;linux/fs.h&gt;</div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='del'>-#include &lt;linux/device.h&gt;</div><div class='del'>-#include &lt;linux/err.h&gt;</div><div class='del'>-#include &lt;linux/poll.h&gt;</div><div class='del'>-#include &lt;linux/sched.h&gt;</div><div class='del'>-#include &lt;linux/file.h&gt;</div><div class='del'>-#include &lt;linux/mount.h&gt;</div><div class='del'>-#include &lt;linux/cdev.h&gt;</div><div class='del'>-#include &lt;linux/xarray.h&gt;</div><div class='del'>-#include &lt;linux/mutex.h&gt;</div><div class='del'>-#include &lt;linux/slab.h&gt;</div><div class='del'>-</div><div class='del'>-#include &lt;linux/nospec.h&gt;</div><div class='del'>-</div><div class='del'>-#include &lt;linux/uaccess.h&gt;</div><div class='del'>-</div><div class='del'>-#include &lt;rdma/ib.h&gt;</div><div class='del'>-#include &lt;rdma/ib_cm.h&gt;</div><div class='del'>-#include &lt;rdma/ib_user_cm.h&gt;</div><div class='del'>-#include &lt;rdma/ib_marshall.h&gt;</div><div class='del'>-</div><div class='del'>-#include "core_priv.h"</div><div class='del'>-</div><div class='del'>-MODULE_AUTHOR("Libor Michalek");</div><div class='del'>-MODULE_DESCRIPTION("InfiniBand userspace Connection Manager access");</div><div class='del'>-MODULE_LICENSE("Dual BSD/GPL");</div><div class='del'>-</div><div class='del'>-struct ib_ucm_device {</div><div class='del'>-	int			devnum;</div><div class='del'>-	struct cdev		cdev;</div><div class='del'>-	struct device		dev;</div><div class='del'>-	struct ib_device	*ib_dev;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct ib_ucm_file {</div><div class='del'>-	struct mutex file_mutex;</div><div class='del'>-	struct file *filp;</div><div class='del'>-	struct ib_ucm_device *device;</div><div class='del'>-</div><div class='del'>-	struct list_head  ctxs;</div><div class='del'>-	struct list_head  events;</div><div class='del'>-	wait_queue_head_t poll_wait;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct ib_ucm_context {</div><div class='del'>-	int                 id;</div><div class='del'>-	struct completion   comp;</div><div class='del'>-	atomic_t            ref;</div><div class='del'>-	int		    events_reported;</div><div class='del'>-</div><div class='del'>-	struct ib_ucm_file *file;</div><div class='del'>-	struct ib_cm_id    *cm_id;</div><div class='del'>-	__u64		   uid;</div><div class='del'>-</div><div class='del'>-	struct list_head    events;    /* list of pending events. */</div><div class='del'>-	struct list_head    file_list; /* member in file ctx list */</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct ib_ucm_event {</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	struct list_head file_list; /* member in file event list */</div><div class='del'>-	struct list_head ctx_list;  /* member in ctx event list */</div><div class='del'>-</div><div class='del'>-	struct ib_cm_id *cm_id;</div><div class='del'>-	struct ib_ucm_event_resp resp;</div><div class='del'>-	void *data;</div><div class='del'>-	void *info;</div><div class='del'>-	int data_len;</div><div class='del'>-	int info_len;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum {</div><div class='del'>-	IB_UCM_MAJOR = 231,</div><div class='del'>-	IB_UCM_BASE_MINOR = 224,</div><div class='del'>-	IB_UCM_MAX_DEVICES = RDMA_MAX_PORTS,</div><div class='del'>-	IB_UCM_NUM_FIXED_MINOR = 32,</div><div class='del'>-	IB_UCM_NUM_DYNAMIC_MINOR = IB_UCM_MAX_DEVICES - IB_UCM_NUM_FIXED_MINOR,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define IB_UCM_BASE_DEV MKDEV(IB_UCM_MAJOR, IB_UCM_BASE_MINOR)</div><div class='del'>-static dev_t dynamic_ucm_dev;</div><div class='del'>-</div><div class='del'>-static void ib_ucm_add_one(struct ib_device *device);</div><div class='del'>-static void ib_ucm_remove_one(struct ib_device *device, void *client_data);</div><div class='del'>-</div><div class='del'>-static struct ib_client ucm_client = {</div><div class='del'>-	.name   = "ucm",</div><div class='del'>-	.add    = ib_ucm_add_one,</div><div class='del'>-	.remove = ib_ucm_remove_one</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static DEFINE_XARRAY_ALLOC(ctx_id_table);</div><div class='del'>-static DECLARE_BITMAP(dev_map, IB_UCM_MAX_DEVICES);</div><div class='del'>-</div><div class='del'>-static struct ib_ucm_context *ib_ucm_ctx_get(struct ib_ucm_file *file, int id)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-</div><div class='del'>-	xa_lock(&amp;ctx_id_table);</div><div class='del'>-	ctx = xa_load(&amp;ctx_id_table, id);</div><div class='del'>-	if (!ctx)</div><div class='del'>-		ctx = ERR_PTR(-ENOENT);</div><div class='del'>-	else if (ctx-&gt;file != file)</div><div class='del'>-		ctx = ERR_PTR(-EINVAL);</div><div class='del'>-	else</div><div class='del'>-		atomic_inc(&amp;ctx-&gt;ref);</div><div class='del'>-	xa_unlock(&amp;ctx_id_table);</div><div class='del'>-</div><div class='del'>-	return ctx;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void ib_ucm_ctx_put(struct ib_ucm_context *ctx)</div><div class='del'>-{</div><div class='del'>-	if (atomic_dec_and_test(&amp;ctx-&gt;ref))</div><div class='del'>-		complete(&amp;ctx-&gt;comp);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline int ib_ucm_new_cm_id(int event)</div><div class='del'>-{</div><div class='del'>-	return event == IB_CM_REQ_RECEIVED || event == IB_CM_SIDR_REQ_RECEIVED;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void ib_ucm_cleanup_events(struct ib_ucm_context *ctx)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_event *uevent;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;ctx-&gt;file-&gt;file_mutex);</div><div class='del'>-	list_del(&amp;ctx-&gt;file_list);</div><div class='del'>-	while (!list_empty(&amp;ctx-&gt;events)) {</div><div class='del'>-</div><div class='del'>-		uevent = list_entry(ctx-&gt;events.next,</div><div class='del'>-				    struct ib_ucm_event, ctx_list);</div><div class='del'>-		list_del(&amp;uevent-&gt;file_list);</div><div class='del'>-		list_del(&amp;uevent-&gt;ctx_list);</div><div class='del'>-		mutex_unlock(&amp;ctx-&gt;file-&gt;file_mutex);</div><div class='del'>-</div><div class='del'>-		/* clear incoming connections. */</div><div class='del'>-		if (ib_ucm_new_cm_id(uevent-&gt;resp.event))</div><div class='del'>-			ib_destroy_cm_id(uevent-&gt;cm_id);</div><div class='del'>-</div><div class='del'>-		kfree(uevent);</div><div class='del'>-		mutex_lock(&amp;ctx-&gt;file-&gt;file_mutex);</div><div class='del'>-	}</div><div class='del'>-	mutex_unlock(&amp;ctx-&gt;file-&gt;file_mutex);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct ib_ucm_context *ib_ucm_ctx_alloc(struct ib_ucm_file *file)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-</div><div class='del'>-	ctx = kzalloc(sizeof *ctx, GFP_KERNEL);</div><div class='del'>-	if (!ctx)</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	atomic_set(&amp;ctx-&gt;ref, 1);</div><div class='del'>-	init_completion(&amp;ctx-&gt;comp);</div><div class='del'>-	ctx-&gt;file = file;</div><div class='del'>-	INIT_LIST_HEAD(&amp;ctx-&gt;events);</div><div class='del'>-</div><div class='del'>-	if (xa_alloc(&amp;ctx_id_table, &amp;ctx-&gt;id, ctx, xa_limit_32b, GFP_KERNEL))</div><div class='del'>-		goto error;</div><div class='del'>-</div><div class='del'>-	list_add_tail(&amp;ctx-&gt;file_list, &amp;file-&gt;ctxs);</div><div class='del'>-	return ctx;</div><div class='del'>-</div><div class='del'>-error:</div><div class='del'>-	kfree(ctx);</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void ib_ucm_event_req_get(struct ib_ucm_req_event_resp *ureq,</div><div class='del'>-				 const struct ib_cm_req_event_param *kreq)</div><div class='del'>-{</div><div class='del'>-	ureq-&gt;remote_ca_guid             = kreq-&gt;remote_ca_guid;</div><div class='del'>-	ureq-&gt;remote_qkey                = kreq-&gt;remote_qkey;</div><div class='del'>-	ureq-&gt;remote_qpn                 = kreq-&gt;remote_qpn;</div><div class='del'>-	ureq-&gt;qp_type                    = kreq-&gt;qp_type;</div><div class='del'>-	ureq-&gt;starting_psn               = kreq-&gt;starting_psn;</div><div class='del'>-	ureq-&gt;responder_resources        = kreq-&gt;responder_resources;</div><div class='del'>-	ureq-&gt;initiator_depth            = kreq-&gt;initiator_depth;</div><div class='del'>-	ureq-&gt;local_cm_response_timeout  = kreq-&gt;local_cm_response_timeout;</div><div class='del'>-	ureq-&gt;flow_control               = kreq-&gt;flow_control;</div><div class='del'>-	ureq-&gt;remote_cm_response_timeout = kreq-&gt;remote_cm_response_timeout;</div><div class='del'>-	ureq-&gt;retry_count                = kreq-&gt;retry_count;</div><div class='del'>-	ureq-&gt;rnr_retry_count            = kreq-&gt;rnr_retry_count;</div><div class='del'>-	ureq-&gt;srq                        = kreq-&gt;srq;</div><div class='del'>-	ureq-&gt;port			 = kreq-&gt;port;</div><div class='del'>-</div><div class='del'>-	ib_copy_path_rec_to_user(&amp;ureq-&gt;primary_path, kreq-&gt;primary_path);</div><div class='del'>-	if (kreq-&gt;alternate_path)</div><div class='del'>-		ib_copy_path_rec_to_user(&amp;ureq-&gt;alternate_path,</div><div class='del'>-					 kreq-&gt;alternate_path);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void ib_ucm_event_rep_get(struct ib_ucm_rep_event_resp *urep,</div><div class='del'>-				 const struct ib_cm_rep_event_param *krep)</div><div class='del'>-{</div><div class='del'>-	urep-&gt;remote_ca_guid      = krep-&gt;remote_ca_guid;</div><div class='del'>-	urep-&gt;remote_qkey         = krep-&gt;remote_qkey;</div><div class='del'>-	urep-&gt;remote_qpn          = krep-&gt;remote_qpn;</div><div class='del'>-	urep-&gt;starting_psn        = krep-&gt;starting_psn;</div><div class='del'>-	urep-&gt;responder_resources = krep-&gt;responder_resources;</div><div class='del'>-	urep-&gt;initiator_depth     = krep-&gt;initiator_depth;</div><div class='del'>-	urep-&gt;target_ack_delay    = krep-&gt;target_ack_delay;</div><div class='del'>-	urep-&gt;failover_accepted   = krep-&gt;failover_accepted;</div><div class='del'>-	urep-&gt;flow_control        = krep-&gt;flow_control;</div><div class='del'>-	urep-&gt;rnr_retry_count     = krep-&gt;rnr_retry_count;</div><div class='del'>-	urep-&gt;srq                 = krep-&gt;srq;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void ib_ucm_event_sidr_rep_get(struct ib_ucm_sidr_rep_event_resp *urep,</div><div class='del'>-				      const struct ib_cm_sidr_rep_event_param *krep)</div><div class='del'>-{</div><div class='del'>-	urep-&gt;status = krep-&gt;status;</div><div class='del'>-	urep-&gt;qkey   = krep-&gt;qkey;</div><div class='del'>-	urep-&gt;qpn    = krep-&gt;qpn;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static int ib_ucm_event_process(const struct ib_cm_event *evt,</div><div class='del'>-				struct ib_ucm_event *uvt)</div><div class='del'>-{</div><div class='del'>-	void *info = NULL;</div><div class='del'>-</div><div class='del'>-	switch (evt-&gt;event) {</div><div class='del'>-	case IB_CM_REQ_RECEIVED:</div><div class='del'>-		ib_ucm_event_req_get(&amp;uvt-&gt;resp.u.req_resp,</div><div class='del'>-				     &amp;evt-&gt;param.req_rcvd);</div><div class='del'>-		uvt-&gt;data_len      = IB_CM_REQ_PRIVATE_DATA_SIZE;</div><div class='del'>-		uvt-&gt;resp.present  = IB_UCM_PRES_PRIMARY;</div><div class='del'>-		uvt-&gt;resp.present |= (evt-&gt;param.req_rcvd.alternate_path ?</div><div class='del'>-				      IB_UCM_PRES_ALTERNATE : 0);</div><div class='del'>-		break;</div><div class='del'>-	case IB_CM_REP_RECEIVED:</div><div class='del'>-		ib_ucm_event_rep_get(&amp;uvt-&gt;resp.u.rep_resp,</div><div class='del'>-				     &amp;evt-&gt;param.rep_rcvd);</div><div class='del'>-		uvt-&gt;data_len = IB_CM_REP_PRIVATE_DATA_SIZE;</div><div class='del'>-		break;</div><div class='del'>-	case IB_CM_RTU_RECEIVED:</div><div class='del'>-		uvt-&gt;data_len = IB_CM_RTU_PRIVATE_DATA_SIZE;</div><div class='del'>-		uvt-&gt;resp.u.send_status = evt-&gt;param.send_status;</div><div class='del'>-		break;</div><div class='del'>-	case IB_CM_DREQ_RECEIVED:</div><div class='del'>-		uvt-&gt;data_len = IB_CM_DREQ_PRIVATE_DATA_SIZE;</div><div class='del'>-		uvt-&gt;resp.u.send_status = evt-&gt;param.send_status;</div><div class='del'>-		break;</div><div class='del'>-	case IB_CM_DREP_RECEIVED:</div><div class='del'>-		uvt-&gt;data_len = IB_CM_DREP_PRIVATE_DATA_SIZE;</div><div class='del'>-		uvt-&gt;resp.u.send_status = evt-&gt;param.send_status;</div><div class='del'>-		break;</div><div class='del'>-	case IB_CM_MRA_RECEIVED:</div><div class='del'>-		uvt-&gt;resp.u.mra_resp.timeout =</div><div class='del'>-					evt-&gt;param.mra_rcvd.service_timeout;</div><div class='del'>-		uvt-&gt;data_len = IB_CM_MRA_PRIVATE_DATA_SIZE;</div><div class='del'>-		break;</div><div class='del'>-	case IB_CM_REJ_RECEIVED:</div><div class='del'>-		uvt-&gt;resp.u.rej_resp.reason = evt-&gt;param.rej_rcvd.reason;</div><div class='del'>-		uvt-&gt;data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;</div><div class='del'>-		uvt-&gt;info_len = evt-&gt;param.rej_rcvd.ari_length;</div><div class='del'>-		info	      = evt-&gt;param.rej_rcvd.ari;</div><div class='del'>-		break;</div><div class='del'>-	case IB_CM_LAP_RECEIVED:</div><div class='del'>-		ib_copy_path_rec_to_user(&amp;uvt-&gt;resp.u.lap_resp.path,</div><div class='del'>-					 evt-&gt;param.lap_rcvd.alternate_path);</div><div class='del'>-		uvt-&gt;data_len = IB_CM_LAP_PRIVATE_DATA_SIZE;</div><div class='del'>-		uvt-&gt;resp.present = IB_UCM_PRES_ALTERNATE;</div><div class='del'>-		break;</div><div class='del'>-	case IB_CM_APR_RECEIVED:</div><div class='del'>-		uvt-&gt;resp.u.apr_resp.status = evt-&gt;param.apr_rcvd.ap_status;</div><div class='del'>-		uvt-&gt;data_len = IB_CM_APR_PRIVATE_DATA_SIZE;</div><div class='del'>-		uvt-&gt;info_len = evt-&gt;param.apr_rcvd.info_len;</div><div class='del'>-		info	      = evt-&gt;param.apr_rcvd.apr_info;</div><div class='del'>-		break;</div><div class='del'>-	case IB_CM_SIDR_REQ_RECEIVED:</div><div class='del'>-		uvt-&gt;resp.u.sidr_req_resp.pkey =</div><div class='del'>-					evt-&gt;param.sidr_req_rcvd.pkey;</div><div class='del'>-		uvt-&gt;resp.u.sidr_req_resp.port =</div><div class='del'>-					evt-&gt;param.sidr_req_rcvd.port;</div><div class='del'>-		uvt-&gt;data_len = IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE;</div><div class='del'>-		break;</div><div class='del'>-	case IB_CM_SIDR_REP_RECEIVED:</div><div class='del'>-		ib_ucm_event_sidr_rep_get(&amp;uvt-&gt;resp.u.sidr_rep_resp,</div><div class='del'>-					  &amp;evt-&gt;param.sidr_rep_rcvd);</div><div class='del'>-		uvt-&gt;data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;</div><div class='del'>-		uvt-&gt;info_len = evt-&gt;param.sidr_rep_rcvd.info_len;</div><div class='del'>-		info	      = evt-&gt;param.sidr_rep_rcvd.info;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		uvt-&gt;resp.u.send_status = evt-&gt;param.send_status;</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (uvt-&gt;data_len) {</div><div class='del'>-		uvt-&gt;data = kmemdup(evt-&gt;private_data, uvt-&gt;data_len, GFP_KERNEL);</div><div class='del'>-		if (!uvt-&gt;data)</div><div class='del'>-			goto err1;</div><div class='del'>-</div><div class='del'>-		uvt-&gt;resp.present |= IB_UCM_PRES_DATA;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (uvt-&gt;info_len) {</div><div class='del'>-		uvt-&gt;info = kmemdup(info, uvt-&gt;info_len, GFP_KERNEL);</div><div class='del'>-		if (!uvt-&gt;info)</div><div class='del'>-			goto err2;</div><div class='del'>-</div><div class='del'>-		uvt-&gt;resp.present |= IB_UCM_PRES_INFO;</div><div class='del'>-	}</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err2:</div><div class='del'>-	kfree(uvt-&gt;data);</div><div class='del'>-err1:</div><div class='del'>-	return -ENOMEM;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int ib_ucm_event_handler(struct ib_cm_id *cm_id,</div><div class='del'>-				const struct ib_cm_event *event)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_event *uevent;</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	int result = 0;</div><div class='del'>-</div><div class='del'>-	ctx = cm_id-&gt;context;</div><div class='del'>-</div><div class='del'>-	uevent = kzalloc(sizeof *uevent, GFP_KERNEL);</div><div class='del'>-	if (!uevent)</div><div class='del'>-		goto err1;</div><div class='del'>-</div><div class='del'>-	uevent-&gt;ctx = ctx;</div><div class='del'>-	uevent-&gt;cm_id = cm_id;</div><div class='del'>-	uevent-&gt;resp.uid = ctx-&gt;uid;</div><div class='del'>-	uevent-&gt;resp.id = ctx-&gt;id;</div><div class='del'>-	uevent-&gt;resp.event = event-&gt;event;</div><div class='del'>-</div><div class='del'>-	result = ib_ucm_event_process(event, uevent);</div><div class='del'>-	if (result)</div><div class='del'>-		goto err2;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;ctx-&gt;file-&gt;file_mutex);</div><div class='del'>-	list_add_tail(&amp;uevent-&gt;file_list, &amp;ctx-&gt;file-&gt;events);</div><div class='del'>-	list_add_tail(&amp;uevent-&gt;ctx_list, &amp;ctx-&gt;events);</div><div class='del'>-	wake_up_interruptible(&amp;ctx-&gt;file-&gt;poll_wait);</div><div class='del'>-	mutex_unlock(&amp;ctx-&gt;file-&gt;file_mutex);</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err2:</div><div class='del'>-	kfree(uevent);</div><div class='del'>-err1:</div><div class='del'>-	/* Destroy new cm_id's */</div><div class='del'>-	return ib_ucm_new_cm_id(event-&gt;event);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_event(struct ib_ucm_file *file,</div><div class='del'>-			    const char __user *inbuf,</div><div class='del'>-			    int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	struct ib_ucm_event_get cmd;</div><div class='del'>-	struct ib_ucm_event *uevent;</div><div class='del'>-	int result = 0;</div><div class='del'>-</div><div class='del'>-	if (out_len &lt; sizeof(struct ib_ucm_event_resp))</div><div class='del'>-		return -ENOSPC;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;file-&gt;file_mutex);</div><div class='del'>-	while (list_empty(&amp;file-&gt;events)) {</div><div class='del'>-		mutex_unlock(&amp;file-&gt;file_mutex);</div><div class='del'>-</div><div class='del'>-		if (file-&gt;filp-&gt;f_flags &amp; O_NONBLOCK)</div><div class='del'>-			return -EAGAIN;</div><div class='del'>-</div><div class='del'>-		if (wait_event_interruptible(file-&gt;poll_wait,</div><div class='del'>-					     !list_empty(&amp;file-&gt;events)))</div><div class='del'>-			return -ERESTARTSYS;</div><div class='del'>-</div><div class='del'>-		mutex_lock(&amp;file-&gt;file_mutex);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	uevent = list_entry(file-&gt;events.next, struct ib_ucm_event, file_list);</div><div class='del'>-</div><div class='del'>-	if (ib_ucm_new_cm_id(uevent-&gt;resp.event)) {</div><div class='del'>-		ctx = ib_ucm_ctx_alloc(file);</div><div class='del'>-		if (!ctx) {</div><div class='del'>-			result = -ENOMEM;</div><div class='del'>-			goto done;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		ctx-&gt;cm_id = uevent-&gt;cm_id;</div><div class='del'>-		ctx-&gt;cm_id-&gt;context = ctx;</div><div class='del'>-		uevent-&gt;resp.id = ctx-&gt;id;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (copy_to_user(u64_to_user_ptr(cmd.response),</div><div class='del'>-			 &amp;uevent-&gt;resp, sizeof(uevent-&gt;resp))) {</div><div class='del'>-		result = -EFAULT;</div><div class='del'>-		goto done;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (uevent-&gt;data) {</div><div class='del'>-		if (cmd.data_len &lt; uevent-&gt;data_len) {</div><div class='del'>-			result = -ENOMEM;</div><div class='del'>-			goto done;</div><div class='del'>-		}</div><div class='del'>-		if (copy_to_user(u64_to_user_ptr(cmd.data),</div><div class='del'>-				 uevent-&gt;data, uevent-&gt;data_len)) {</div><div class='del'>-			result = -EFAULT;</div><div class='del'>-			goto done;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (uevent-&gt;info) {</div><div class='del'>-		if (cmd.info_len &lt; uevent-&gt;info_len) {</div><div class='del'>-			result = -ENOMEM;</div><div class='del'>-			goto done;</div><div class='del'>-		}</div><div class='del'>-		if (copy_to_user(u64_to_user_ptr(cmd.info),</div><div class='del'>-				 uevent-&gt;info, uevent-&gt;info_len)) {</div><div class='del'>-			result = -EFAULT;</div><div class='del'>-			goto done;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	list_del(&amp;uevent-&gt;file_list);</div><div class='del'>-	list_del(&amp;uevent-&gt;ctx_list);</div><div class='del'>-	uevent-&gt;ctx-&gt;events_reported++;</div><div class='del'>-</div><div class='del'>-	kfree(uevent-&gt;data);</div><div class='del'>-	kfree(uevent-&gt;info);</div><div class='del'>-	kfree(uevent);</div><div class='del'>-done:</div><div class='del'>-	mutex_unlock(&amp;file-&gt;file_mutex);</div><div class='del'>-	return result;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_create_id(struct ib_ucm_file *file,</div><div class='del'>-				const char __user *inbuf,</div><div class='del'>-				int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_create_id cmd;</div><div class='del'>-	struct ib_ucm_create_id_resp resp;</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	int result;</div><div class='del'>-</div><div class='del'>-	if (out_len &lt; sizeof(resp))</div><div class='del'>-		return -ENOSPC;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;file-&gt;file_mutex);</div><div class='del'>-	ctx = ib_ucm_ctx_alloc(file);</div><div class='del'>-	mutex_unlock(&amp;file-&gt;file_mutex);</div><div class='del'>-	if (!ctx)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	ctx-&gt;uid = cmd.uid;</div><div class='del'>-	ctx-&gt;cm_id = ib_create_cm_id(file-&gt;device-&gt;ib_dev,</div><div class='del'>-				     ib_ucm_event_handler, ctx);</div><div class='del'>-	if (IS_ERR(ctx-&gt;cm_id)) {</div><div class='del'>-		result = PTR_ERR(ctx-&gt;cm_id);</div><div class='del'>-		goto err1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	resp.id = ctx-&gt;id;</div><div class='del'>-	if (copy_to_user(u64_to_user_ptr(cmd.response),</div><div class='del'>-			 &amp;resp, sizeof(resp))) {</div><div class='del'>-		result = -EFAULT;</div><div class='del'>-		goto err2;</div><div class='del'>-	}</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err2:</div><div class='del'>-	ib_destroy_cm_id(ctx-&gt;cm_id);</div><div class='del'>-err1:</div><div class='del'>-	xa_erase(&amp;ctx_id_table, ctx-&gt;id);</div><div class='del'>-	kfree(ctx);</div><div class='del'>-	return result;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_destroy_id(struct ib_ucm_file *file,</div><div class='del'>-				 const char __user *inbuf,</div><div class='del'>-				 int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_destroy_id cmd;</div><div class='del'>-	struct ib_ucm_destroy_id_resp resp;</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	int result = 0;</div><div class='del'>-</div><div class='del'>-	if (out_len &lt; sizeof(resp))</div><div class='del'>-		return -ENOSPC;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	xa_lock(&amp;ctx_id_table);</div><div class='del'>-	ctx = xa_load(&amp;ctx_id_table, cmd.id);</div><div class='del'>-	if (!ctx)</div><div class='del'>-		ctx = ERR_PTR(-ENOENT);</div><div class='del'>-	else if (ctx-&gt;file != file)</div><div class='del'>-		ctx = ERR_PTR(-EINVAL);</div><div class='del'>-	else</div><div class='del'>-		__xa_erase(&amp;ctx_id_table, ctx-&gt;id);</div><div class='del'>-	xa_unlock(&amp;ctx_id_table);</div><div class='del'>-</div><div class='del'>-	if (IS_ERR(ctx))</div><div class='del'>-		return PTR_ERR(ctx);</div><div class='del'>-</div><div class='del'>-	ib_ucm_ctx_put(ctx);</div><div class='del'>-	wait_for_completion(&amp;ctx-&gt;comp);</div><div class='del'>-</div><div class='del'>-	/* No new events will be generated after destroying the cm_id. */</div><div class='del'>-	ib_destroy_cm_id(ctx-&gt;cm_id);</div><div class='del'>-	/* Cleanup events not yet reported to the user. */</div><div class='del'>-	ib_ucm_cleanup_events(ctx);</div><div class='del'>-</div><div class='del'>-	resp.events_reported = ctx-&gt;events_reported;</div><div class='del'>-	if (copy_to_user(u64_to_user_ptr(cmd.response),</div><div class='del'>-			 &amp;resp, sizeof(resp)))</div><div class='del'>-		result = -EFAULT;</div><div class='del'>-</div><div class='del'>-	kfree(ctx);</div><div class='del'>-	return result;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_attr_id(struct ib_ucm_file *file,</div><div class='del'>-			      const char __user *inbuf,</div><div class='del'>-			      int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_attr_id_resp resp;</div><div class='del'>-	struct ib_ucm_attr_id cmd;</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	int result = 0;</div><div class='del'>-</div><div class='del'>-	if (out_len &lt; sizeof(resp))</div><div class='del'>-		return -ENOSPC;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	ctx = ib_ucm_ctx_get(file, cmd.id);</div><div class='del'>-	if (IS_ERR(ctx))</div><div class='del'>-		return PTR_ERR(ctx);</div><div class='del'>-</div><div class='del'>-	resp.service_id   = ctx-&gt;cm_id-&gt;service_id;</div><div class='del'>-	resp.service_mask = ctx-&gt;cm_id-&gt;service_mask;</div><div class='del'>-	resp.local_id     = ctx-&gt;cm_id-&gt;local_id;</div><div class='del'>-	resp.remote_id    = ctx-&gt;cm_id-&gt;remote_id;</div><div class='del'>-</div><div class='del'>-	if (copy_to_user(u64_to_user_ptr(cmd.response),</div><div class='del'>-			 &amp;resp, sizeof(resp)))</div><div class='del'>-		result = -EFAULT;</div><div class='del'>-</div><div class='del'>-	ib_ucm_ctx_put(ctx);</div><div class='del'>-	return result;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_init_qp_attr(struct ib_ucm_file *file,</div><div class='del'>-				   const char __user *inbuf,</div><div class='del'>-				   int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	struct ib_uverbs_qp_attr resp;</div><div class='del'>-	struct ib_ucm_init_qp_attr cmd;</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	struct ib_qp_attr qp_attr;</div><div class='del'>-	int result = 0;</div><div class='del'>-</div><div class='del'>-	if (out_len &lt; sizeof(resp))</div><div class='del'>-		return -ENOSPC;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	ctx = ib_ucm_ctx_get(file, cmd.id);</div><div class='del'>-	if (IS_ERR(ctx))</div><div class='del'>-		return PTR_ERR(ctx);</div><div class='del'>-</div><div class='del'>-	resp.qp_attr_mask = 0;</div><div class='del'>-	memset(&amp;qp_attr, 0, sizeof qp_attr);</div><div class='del'>-	qp_attr.qp_state = cmd.qp_state;</div><div class='del'>-	result = ib_cm_init_qp_attr(ctx-&gt;cm_id, &amp;qp_attr, &amp;resp.qp_attr_mask);</div><div class='del'>-	if (result)</div><div class='del'>-		goto out;</div><div class='del'>-</div><div class='del'>-	ib_copy_qp_attr_to_user(ctx-&gt;cm_id-&gt;device, &amp;resp, &amp;qp_attr);</div><div class='del'>-</div><div class='del'>-	if (copy_to_user(u64_to_user_ptr(cmd.response),</div><div class='del'>-			 &amp;resp, sizeof(resp)))</div><div class='del'>-		result = -EFAULT;</div><div class='del'>-</div><div class='del'>-out:</div><div class='del'>-	ib_ucm_ctx_put(ctx);</div><div class='del'>-	return result;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int ucm_validate_listen(__be64 service_id, __be64 service_mask)</div><div class='del'>-{</div><div class='del'>-	service_id &amp;= service_mask;</div><div class='del'>-</div><div class='del'>-	if (((service_id &amp; IB_CMA_SERVICE_ID_MASK) == IB_CMA_SERVICE_ID) ||</div><div class='del'>-	    ((service_id &amp; IB_SDP_SERVICE_ID_MASK) == IB_SDP_SERVICE_ID))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_listen(struct ib_ucm_file *file,</div><div class='del'>-			     const char __user *inbuf,</div><div class='del'>-			     int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_listen cmd;</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	int result;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	ctx = ib_ucm_ctx_get(file, cmd.id);</div><div class='del'>-	if (IS_ERR(ctx))</div><div class='del'>-		return PTR_ERR(ctx);</div><div class='del'>-</div><div class='del'>-	result = ucm_validate_listen(cmd.service_id, cmd.service_mask);</div><div class='del'>-	if (result)</div><div class='del'>-		goto out;</div><div class='del'>-</div><div class='del'>-	result = ib_cm_listen(ctx-&gt;cm_id, cmd.service_id, cmd.service_mask);</div><div class='del'>-out:</div><div class='del'>-	ib_ucm_ctx_put(ctx);</div><div class='del'>-	return result;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_notify(struct ib_ucm_file *file,</div><div class='del'>-			     const char __user *inbuf,</div><div class='del'>-			     int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_notify cmd;</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	int result;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	ctx = ib_ucm_ctx_get(file, cmd.id);</div><div class='del'>-	if (IS_ERR(ctx))</div><div class='del'>-		return PTR_ERR(ctx);</div><div class='del'>-</div><div class='del'>-	result = ib_cm_notify(ctx-&gt;cm_id, (enum ib_event_type) cmd.event);</div><div class='del'>-	ib_ucm_ctx_put(ctx);</div><div class='del'>-	return result;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int ib_ucm_alloc_data(const void **dest, u64 src, u32 len)</div><div class='del'>-{</div><div class='del'>-	void *data;</div><div class='del'>-</div><div class='del'>-	*dest = NULL;</div><div class='del'>-</div><div class='del'>-	if (!len)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	data = memdup_user(u64_to_user_ptr(src), len);</div><div class='del'>-	if (IS_ERR(data))</div><div class='del'>-		return PTR_ERR(data);</div><div class='del'>-</div><div class='del'>-	*dest = data;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int ib_ucm_path_get(struct sa_path_rec **path, u64 src)</div><div class='del'>-{</div><div class='del'>-	struct ib_user_path_rec upath;</div><div class='del'>-	struct sa_path_rec  *sa_path;</div><div class='del'>-</div><div class='del'>-	*path = NULL;</div><div class='del'>-</div><div class='del'>-	if (!src)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	sa_path = kmalloc(sizeof(*sa_path), GFP_KERNEL);</div><div class='del'>-	if (!sa_path)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;upath, u64_to_user_ptr(src),</div><div class='del'>-			   sizeof(upath))) {</div><div class='del'>-</div><div class='del'>-		kfree(sa_path);</div><div class='del'>-		return -EFAULT;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ib_copy_path_rec_from_user(sa_path, &amp;upath);</div><div class='del'>-	*path = sa_path;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_send_req(struct ib_ucm_file *file,</div><div class='del'>-			       const char __user *inbuf,</div><div class='del'>-			       int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	struct ib_cm_req_param param;</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	struct ib_ucm_req cmd;</div><div class='del'>-	int result;</div><div class='del'>-</div><div class='del'>-	param.private_data   = NULL;</div><div class='del'>-	param.primary_path   = NULL;</div><div class='del'>-	param.alternate_path = NULL;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	result = ib_ucm_alloc_data(&amp;param.private_data, cmd.data, cmd.len);</div><div class='del'>-	if (result)</div><div class='del'>-		goto done;</div><div class='del'>-</div><div class='del'>-	result = ib_ucm_path_get(&amp;param.primary_path, cmd.primary_path);</div><div class='del'>-	if (result)</div><div class='del'>-		goto done;</div><div class='del'>-</div><div class='del'>-	result = ib_ucm_path_get(&amp;param.alternate_path, cmd.alternate_path);</div><div class='del'>-	if (result)</div><div class='del'>-		goto done;</div><div class='del'>-</div><div class='del'>-	param.private_data_len           = cmd.len;</div><div class='del'>-	param.service_id                 = cmd.sid;</div><div class='del'>-	param.qp_num                     = cmd.qpn;</div><div class='del'>-	param.qp_type                    = cmd.qp_type;</div><div class='del'>-	param.starting_psn               = cmd.psn;</div><div class='del'>-	param.peer_to_peer               = cmd.peer_to_peer;</div><div class='del'>-	param.responder_resources        = cmd.responder_resources;</div><div class='del'>-	param.initiator_depth            = cmd.initiator_depth;</div><div class='del'>-	param.remote_cm_response_timeout = cmd.remote_cm_response_timeout;</div><div class='del'>-	param.flow_control               = cmd.flow_control;</div><div class='del'>-	param.local_cm_response_timeout  = cmd.local_cm_response_timeout;</div><div class='del'>-	param.retry_count                = cmd.retry_count;</div><div class='del'>-	param.rnr_retry_count            = cmd.rnr_retry_count;</div><div class='del'>-	param.max_cm_retries             = cmd.max_cm_retries;</div><div class='del'>-	param.srq                        = cmd.srq;</div><div class='del'>-</div><div class='del'>-	ctx = ib_ucm_ctx_get(file, cmd.id);</div><div class='del'>-	if (!IS_ERR(ctx)) {</div><div class='del'>-		result = ib_send_cm_req(ctx-&gt;cm_id, &amp;param);</div><div class='del'>-		ib_ucm_ctx_put(ctx);</div><div class='del'>-	} else</div><div class='del'>-		result = PTR_ERR(ctx);</div><div class='del'>-</div><div class='del'>-done:</div><div class='del'>-	kfree(param.private_data);</div><div class='del'>-	kfree(param.primary_path);</div><div class='del'>-	kfree(param.alternate_path);</div><div class='del'>-	return result;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_send_rep(struct ib_ucm_file *file,</div><div class='del'>-			       const char __user *inbuf,</div><div class='del'>-			       int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	struct ib_cm_rep_param param;</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	struct ib_ucm_rep cmd;</div><div class='del'>-	int result;</div><div class='del'>-</div><div class='del'>-	param.private_data = NULL;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	result = ib_ucm_alloc_data(&amp;param.private_data, cmd.data, cmd.len);</div><div class='del'>-	if (result)</div><div class='del'>-		return result;</div><div class='del'>-</div><div class='del'>-	param.qp_num              = cmd.qpn;</div><div class='del'>-	param.starting_psn        = cmd.psn;</div><div class='del'>-	param.private_data_len    = cmd.len;</div><div class='del'>-	param.responder_resources = cmd.responder_resources;</div><div class='del'>-	param.initiator_depth     = cmd.initiator_depth;</div><div class='del'>-	param.failover_accepted   = cmd.failover_accepted;</div><div class='del'>-	param.flow_control        = cmd.flow_control;</div><div class='del'>-	param.rnr_retry_count     = cmd.rnr_retry_count;</div><div class='del'>-	param.srq                 = cmd.srq;</div><div class='del'>-</div><div class='del'>-	ctx = ib_ucm_ctx_get(file, cmd.id);</div><div class='del'>-	if (!IS_ERR(ctx)) {</div><div class='del'>-		ctx-&gt;uid = cmd.uid;</div><div class='del'>-		result = ib_send_cm_rep(ctx-&gt;cm_id, &amp;param);</div><div class='del'>-		ib_ucm_ctx_put(ctx);</div><div class='del'>-	} else</div><div class='del'>-		result = PTR_ERR(ctx);</div><div class='del'>-</div><div class='del'>-	kfree(param.private_data);</div><div class='del'>-	return result;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_send_private_data(struct ib_ucm_file *file,</div><div class='del'>-					const char __user *inbuf, int in_len,</div><div class='del'>-					int (*func)(struct ib_cm_id *cm_id,</div><div class='del'>-						    const void *private_data,</div><div class='del'>-						    u8 private_data_len))</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_private_data cmd;</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	const void *private_data = NULL;</div><div class='del'>-	int result;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	result = ib_ucm_alloc_data(&amp;private_data, cmd.data, cmd.len);</div><div class='del'>-	if (result)</div><div class='del'>-		return result;</div><div class='del'>-</div><div class='del'>-	ctx = ib_ucm_ctx_get(file, cmd.id);</div><div class='del'>-	if (!IS_ERR(ctx)) {</div><div class='del'>-		result = func(ctx-&gt;cm_id, private_data, cmd.len);</div><div class='del'>-		ib_ucm_ctx_put(ctx);</div><div class='del'>-	} else</div><div class='del'>-		result = PTR_ERR(ctx);</div><div class='del'>-</div><div class='del'>-	kfree(private_data);</div><div class='del'>-	return result;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_send_rtu(struct ib_ucm_file *file,</div><div class='del'>-			       const char __user *inbuf,</div><div class='del'>-			       int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	return ib_ucm_send_private_data(file, inbuf, in_len, ib_send_cm_rtu);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_send_dreq(struct ib_ucm_file *file,</div><div class='del'>-				const char __user *inbuf,</div><div class='del'>-				int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	return ib_ucm_send_private_data(file, inbuf, in_len, ib_send_cm_dreq);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_send_drep(struct ib_ucm_file *file,</div><div class='del'>-				const char __user *inbuf,</div><div class='del'>-				int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	return ib_ucm_send_private_data(file, inbuf, in_len, ib_send_cm_drep);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_send_info(struct ib_ucm_file *file,</div><div class='del'>-				const char __user *inbuf, int in_len,</div><div class='del'>-				int (*func)(struct ib_cm_id *cm_id,</div><div class='del'>-					    int status,</div><div class='del'>-					    const void *info,</div><div class='del'>-					    u8 info_len,</div><div class='del'>-					    const void *data,</div><div class='del'>-					    u8 data_len))</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	struct ib_ucm_info cmd;</div><div class='del'>-	const void *data = NULL;</div><div class='del'>-	const void *info = NULL;</div><div class='del'>-	int result;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	result = ib_ucm_alloc_data(&amp;data, cmd.data, cmd.data_len);</div><div class='del'>-	if (result)</div><div class='del'>-		goto done;</div><div class='del'>-</div><div class='del'>-	result = ib_ucm_alloc_data(&amp;info, cmd.info, cmd.info_len);</div><div class='del'>-	if (result)</div><div class='del'>-		goto done;</div><div class='del'>-</div><div class='del'>-	ctx = ib_ucm_ctx_get(file, cmd.id);</div><div class='del'>-	if (!IS_ERR(ctx)) {</div><div class='del'>-		result = func(ctx-&gt;cm_id, cmd.status, info, cmd.info_len,</div><div class='del'>-			      data, cmd.data_len);</div><div class='del'>-		ib_ucm_ctx_put(ctx);</div><div class='del'>-	} else</div><div class='del'>-		result = PTR_ERR(ctx);</div><div class='del'>-</div><div class='del'>-done:</div><div class='del'>-	kfree(data);</div><div class='del'>-	kfree(info);</div><div class='del'>-	return result;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_send_rej(struct ib_ucm_file *file,</div><div class='del'>-			       const char __user *inbuf,</div><div class='del'>-			       int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	return ib_ucm_send_info(file, inbuf, in_len, (void *)ib_send_cm_rej);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_send_apr(struct ib_ucm_file *file,</div><div class='del'>-			       const char __user *inbuf,</div><div class='del'>-			       int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	return ib_ucm_send_info(file, inbuf, in_len, (void *)ib_send_cm_apr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_send_mra(struct ib_ucm_file *file,</div><div class='del'>-			       const char __user *inbuf,</div><div class='del'>-			       int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	struct ib_ucm_mra cmd;</div><div class='del'>-	const void *data = NULL;</div><div class='del'>-	int result;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	result = ib_ucm_alloc_data(&amp;data, cmd.data, cmd.len);</div><div class='del'>-	if (result)</div><div class='del'>-		return result;</div><div class='del'>-</div><div class='del'>-	ctx = ib_ucm_ctx_get(file, cmd.id);</div><div class='del'>-	if (!IS_ERR(ctx)) {</div><div class='del'>-		result = ib_send_cm_mra(ctx-&gt;cm_id, cmd.timeout, data, cmd.len);</div><div class='del'>-		ib_ucm_ctx_put(ctx);</div><div class='del'>-	} else</div><div class='del'>-		result = PTR_ERR(ctx);</div><div class='del'>-</div><div class='del'>-	kfree(data);</div><div class='del'>-	return result;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_send_lap(struct ib_ucm_file *file,</div><div class='del'>-			       const char __user *inbuf,</div><div class='del'>-			       int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	struct sa_path_rec *path = NULL;</div><div class='del'>-	struct ib_ucm_lap cmd;</div><div class='del'>-	const void *data = NULL;</div><div class='del'>-	int result;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	result = ib_ucm_alloc_data(&amp;data, cmd.data, cmd.len);</div><div class='del'>-	if (result)</div><div class='del'>-		goto done;</div><div class='del'>-</div><div class='del'>-	result = ib_ucm_path_get(&amp;path, cmd.path);</div><div class='del'>-	if (result)</div><div class='del'>-		goto done;</div><div class='del'>-</div><div class='del'>-	ctx = ib_ucm_ctx_get(file, cmd.id);</div><div class='del'>-	if (!IS_ERR(ctx)) {</div><div class='del'>-		result = ib_send_cm_lap(ctx-&gt;cm_id, path, data, cmd.len);</div><div class='del'>-		ib_ucm_ctx_put(ctx);</div><div class='del'>-	} else</div><div class='del'>-		result = PTR_ERR(ctx);</div><div class='del'>-</div><div class='del'>-done:</div><div class='del'>-	kfree(data);</div><div class='del'>-	kfree(path);</div><div class='del'>-	return result;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_send_sidr_req(struct ib_ucm_file *file,</div><div class='del'>-				    const char __user *inbuf,</div><div class='del'>-				    int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	struct ib_cm_sidr_req_param param = {};</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	struct ib_ucm_sidr_req cmd;</div><div class='del'>-	int result;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	result = ib_ucm_alloc_data(&amp;param.private_data, cmd.data, cmd.len);</div><div class='del'>-	if (result)</div><div class='del'>-		goto done;</div><div class='del'>-</div><div class='del'>-	result = ib_ucm_path_get(&amp;param.path, cmd.path);</div><div class='del'>-	if (result)</div><div class='del'>-		goto done;</div><div class='del'>-</div><div class='del'>-	param.private_data_len = cmd.len;</div><div class='del'>-	param.service_id       = cmd.sid;</div><div class='del'>-	param.timeout_ms       = cmd.timeout;</div><div class='del'>-	param.max_cm_retries   = cmd.max_cm_retries;</div><div class='del'>-</div><div class='del'>-	ctx = ib_ucm_ctx_get(file, cmd.id);</div><div class='del'>-	if (!IS_ERR(ctx)) {</div><div class='del'>-		result = ib_send_cm_sidr_req(ctx-&gt;cm_id, &amp;param);</div><div class='del'>-		ib_ucm_ctx_put(ctx);</div><div class='del'>-	} else</div><div class='del'>-		result = PTR_ERR(ctx);</div><div class='del'>-</div><div class='del'>-done:</div><div class='del'>-	kfree(param.private_data);</div><div class='del'>-	kfree(param.path);</div><div class='del'>-	return result;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_send_sidr_rep(struct ib_ucm_file *file,</div><div class='del'>-				    const char __user *inbuf,</div><div class='del'>-				    int in_len, int out_len)</div><div class='del'>-{</div><div class='del'>-	struct ib_cm_sidr_rep_param param;</div><div class='del'>-	struct ib_ucm_sidr_rep cmd;</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-	int result;</div><div class='del'>-</div><div class='del'>-	param.info = NULL;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	result = ib_ucm_alloc_data(&amp;param.private_data,</div><div class='del'>-				   cmd.data, cmd.data_len);</div><div class='del'>-	if (result)</div><div class='del'>-		goto done;</div><div class='del'>-</div><div class='del'>-	result = ib_ucm_alloc_data(&amp;param.info, cmd.info, cmd.info_len);</div><div class='del'>-	if (result)</div><div class='del'>-		goto done;</div><div class='del'>-</div><div class='del'>-	param.qp_num		= cmd.qpn;</div><div class='del'>-	param.qkey		= cmd.qkey;</div><div class='del'>-	param.status		= cmd.status;</div><div class='del'>-	param.info_length	= cmd.info_len;</div><div class='del'>-	param.private_data_len	= cmd.data_len;</div><div class='del'>-</div><div class='del'>-	ctx = ib_ucm_ctx_get(file, cmd.id);</div><div class='del'>-	if (!IS_ERR(ctx)) {</div><div class='del'>-		result = ib_send_cm_sidr_rep(ctx-&gt;cm_id, &amp;param);</div><div class='del'>-		ib_ucm_ctx_put(ctx);</div><div class='del'>-	} else</div><div class='del'>-		result = PTR_ERR(ctx);</div><div class='del'>-</div><div class='del'>-done:</div><div class='del'>-	kfree(param.private_data);</div><div class='del'>-	kfree(param.info);</div><div class='del'>-	return result;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t (*ucm_cmd_table[])(struct ib_ucm_file *file,</div><div class='del'>-				  const char __user *inbuf,</div><div class='del'>-				  int in_len, int out_len) = {</div><div class='del'>-	[IB_USER_CM_CMD_CREATE_ID]     = ib_ucm_create_id,</div><div class='del'>-	[IB_USER_CM_CMD_DESTROY_ID]    = ib_ucm_destroy_id,</div><div class='del'>-	[IB_USER_CM_CMD_ATTR_ID]       = ib_ucm_attr_id,</div><div class='del'>-	[IB_USER_CM_CMD_LISTEN]        = ib_ucm_listen,</div><div class='del'>-	[IB_USER_CM_CMD_NOTIFY]        = ib_ucm_notify,</div><div class='del'>-	[IB_USER_CM_CMD_SEND_REQ]      = ib_ucm_send_req,</div><div class='del'>-	[IB_USER_CM_CMD_SEND_REP]      = ib_ucm_send_rep,</div><div class='del'>-	[IB_USER_CM_CMD_SEND_RTU]      = ib_ucm_send_rtu,</div><div class='del'>-	[IB_USER_CM_CMD_SEND_DREQ]     = ib_ucm_send_dreq,</div><div class='del'>-	[IB_USER_CM_CMD_SEND_DREP]     = ib_ucm_send_drep,</div><div class='del'>-	[IB_USER_CM_CMD_SEND_REJ]      = ib_ucm_send_rej,</div><div class='del'>-	[IB_USER_CM_CMD_SEND_MRA]      = ib_ucm_send_mra,</div><div class='del'>-	[IB_USER_CM_CMD_SEND_LAP]      = ib_ucm_send_lap,</div><div class='del'>-	[IB_USER_CM_CMD_SEND_APR]      = ib_ucm_send_apr,</div><div class='del'>-	[IB_USER_CM_CMD_SEND_SIDR_REQ] = ib_ucm_send_sidr_req,</div><div class='del'>-	[IB_USER_CM_CMD_SEND_SIDR_REP] = ib_ucm_send_sidr_rep,</div><div class='del'>-	[IB_USER_CM_CMD_EVENT]	       = ib_ucm_event,</div><div class='del'>-	[IB_USER_CM_CMD_INIT_QP_ATTR]  = ib_ucm_init_qp_attr,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static ssize_t ib_ucm_write(struct file *filp, const char __user *buf,</div><div class='del'>-			    size_t len, loff_t *pos)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_file *file = filp-&gt;private_data;</div><div class='del'>-	struct ib_ucm_cmd_hdr hdr;</div><div class='del'>-	ssize_t result;</div><div class='del'>-</div><div class='del'>-	if (!ib_safe_file_access(filp)) {</div><div class='del'>-		pr_err_once("ucm_write: process %d (%s) changed security contexts after opening file descriptor, this is not allowed.\n",</div><div class='del'>-			    task_tgid_vnr(current), current-&gt;comm);</div><div class='del'>-		return -EACCES;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (len &lt; sizeof(hdr))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	if (copy_from_user(&amp;hdr, buf, sizeof(hdr)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	if (hdr.cmd &gt;= ARRAY_SIZE(ucm_cmd_table))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	hdr.cmd = array_index_nospec(hdr.cmd, ARRAY_SIZE(ucm_cmd_table));</div><div class='del'>-</div><div class='del'>-	if (hdr.in + sizeof(hdr) &gt; len)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	result = ucm_cmd_table[hdr.cmd](file, buf + sizeof(hdr),</div><div class='del'>-					hdr.in, hdr.out);</div><div class='del'>-	if (!result)</div><div class='del'>-		result = len;</div><div class='del'>-</div><div class='del'>-	return result;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static __poll_t ib_ucm_poll(struct file *filp,</div><div class='del'>-				struct poll_table_struct *wait)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_file *file = filp-&gt;private_data;</div><div class='del'>-	__poll_t mask = 0;</div><div class='del'>-</div><div class='del'>-	poll_wait(filp, &amp;file-&gt;poll_wait, wait);</div><div class='del'>-</div><div class='del'>-	if (!list_empty(&amp;file-&gt;events))</div><div class='del'>-		mask = EPOLLIN | EPOLLRDNORM;</div><div class='del'>-</div><div class='del'>-	return mask;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * ib_ucm_open() does not need the BKL:</div><div class='del'>- *</div><div class='del'>- *  - no global state is referred to;</div><div class='del'>- *  - there is no ioctl method to race against;</div><div class='del'>- *  - no further module initialization is required for open to work</div><div class='del'>- *    after the device is registered.</div><div class='del'>- */</div><div class='del'>-static int ib_ucm_open(struct inode *inode, struct file *filp)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_file *file;</div><div class='del'>-</div><div class='del'>-	file = kmalloc(sizeof(*file), GFP_KERNEL);</div><div class='del'>-	if (!file)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	INIT_LIST_HEAD(&amp;file-&gt;events);</div><div class='del'>-	INIT_LIST_HEAD(&amp;file-&gt;ctxs);</div><div class='del'>-	init_waitqueue_head(&amp;file-&gt;poll_wait);</div><div class='del'>-</div><div class='del'>-	mutex_init(&amp;file-&gt;file_mutex);</div><div class='del'>-</div><div class='del'>-	filp-&gt;private_data = file;</div><div class='del'>-	file-&gt;filp = filp;</div><div class='del'>-	file-&gt;device = container_of(inode-&gt;i_cdev, struct ib_ucm_device, cdev);</div><div class='del'>-</div><div class='del'>-	return stream_open(inode, filp);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int ib_ucm_close(struct inode *inode, struct file *filp)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_file *file = filp-&gt;private_data;</div><div class='del'>-	struct ib_ucm_context *ctx;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;file-&gt;file_mutex);</div><div class='del'>-	while (!list_empty(&amp;file-&gt;ctxs)) {</div><div class='del'>-		ctx = list_entry(file-&gt;ctxs.next,</div><div class='del'>-				 struct ib_ucm_context, file_list);</div><div class='del'>-		mutex_unlock(&amp;file-&gt;file_mutex);</div><div class='del'>-</div><div class='del'>-		xa_erase(&amp;ctx_id_table, ctx-&gt;id);</div><div class='del'>-		ib_destroy_cm_id(ctx-&gt;cm_id);</div><div class='del'>-		ib_ucm_cleanup_events(ctx);</div><div class='del'>-		kfree(ctx);</div><div class='del'>-</div><div class='del'>-		mutex_lock(&amp;file-&gt;file_mutex);</div><div class='del'>-	}</div><div class='del'>-	mutex_unlock(&amp;file-&gt;file_mutex);</div><div class='del'>-	kfree(file);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void ib_ucm_release_dev(struct device *dev)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_device *ucm_dev;</div><div class='del'>-</div><div class='del'>-	ucm_dev = container_of(dev, struct ib_ucm_device, dev);</div><div class='del'>-	kfree(ucm_dev);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void ib_ucm_free_dev(struct ib_ucm_device *ucm_dev)</div><div class='del'>-{</div><div class='del'>-	clear_bit(ucm_dev-&gt;devnum, dev_map);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static const struct file_operations ucm_fops = {</div><div class='del'>-	.owner	 = THIS_MODULE,</div><div class='del'>-	.open	 = ib_ucm_open,</div><div class='del'>-	.release = ib_ucm_close,</div><div class='del'>-	.write	 = ib_ucm_write,</div><div class='del'>-	.poll    = ib_ucm_poll,</div><div class='del'>-	.llseek	 = no_llseek,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,</div><div class='del'>-			  char *buf)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_device *ucm_dev;</div><div class='del'>-</div><div class='del'>-	ucm_dev = container_of(dev, struct ib_ucm_device, dev);</div><div class='del'>-	return sprintf(buf, "%s\n", ucm_dev-&gt;ib_dev-&gt;name);</div><div class='del'>-}</div><div class='del'>-static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);</div><div class='del'>-</div><div class='del'>-static void ib_ucm_add_one(struct ib_device *device)</div><div class='del'>-{</div><div class='del'>-	int devnum;</div><div class='del'>-	dev_t base;</div><div class='del'>-	struct ib_ucm_device *ucm_dev;</div><div class='del'>-</div><div class='del'>-	if (!device-&gt;ops.alloc_ucontext || !rdma_cap_ib_cm(device, 1))</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	ucm_dev = kzalloc(sizeof *ucm_dev, GFP_KERNEL);</div><div class='del'>-	if (!ucm_dev)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	device_initialize(&amp;ucm_dev-&gt;dev);</div><div class='del'>-	ucm_dev-&gt;ib_dev = device;</div><div class='del'>-	ucm_dev-&gt;dev.release = ib_ucm_release_dev;</div><div class='del'>-</div><div class='del'>-	devnum = find_first_zero_bit(dev_map, IB_UCM_MAX_DEVICES);</div><div class='del'>-	if (devnum &gt;= IB_UCM_MAX_DEVICES)</div><div class='del'>-		goto err;</div><div class='del'>-	ucm_dev-&gt;devnum = devnum;</div><div class='del'>-	set_bit(devnum, dev_map);</div><div class='del'>-	if (devnum &gt;= IB_UCM_NUM_FIXED_MINOR)</div><div class='del'>-		base = dynamic_ucm_dev + devnum - IB_UCM_NUM_FIXED_MINOR;</div><div class='del'>-	else</div><div class='del'>-		base = IB_UCM_BASE_DEV + devnum;</div><div class='del'>-</div><div class='del'>-	cdev_init(&amp;ucm_dev-&gt;cdev, &amp;ucm_fops);</div><div class='del'>-	ucm_dev-&gt;cdev.owner = THIS_MODULE;</div><div class='del'>-	kobject_set_name(&amp;ucm_dev-&gt;cdev.kobj, "ucm%d", ucm_dev-&gt;devnum);</div><div class='del'>-</div><div class='del'>-	ucm_dev-&gt;dev.class = &amp;cm_class;</div><div class='del'>-	ucm_dev-&gt;dev.parent = device-&gt;dev.parent;</div><div class='del'>-	ucm_dev-&gt;dev.devt = base;</div><div class='del'>-</div><div class='del'>-	dev_set_name(&amp;ucm_dev-&gt;dev, "ucm%d", ucm_dev-&gt;devnum);</div><div class='del'>-	if (cdev_device_add(&amp;ucm_dev-&gt;cdev, &amp;ucm_dev-&gt;dev))</div><div class='del'>-		goto err_devnum;</div><div class='del'>-</div><div class='del'>-	if (device_create_file(&amp;ucm_dev-&gt;dev, &amp;dev_attr_ibdev))</div><div class='del'>-		goto err_dev;</div><div class='del'>-</div><div class='del'>-	ib_set_client_data(device, &amp;ucm_client, ucm_dev);</div><div class='del'>-	return;</div><div class='del'>-</div><div class='del'>-err_dev:</div><div class='del'>-	cdev_device_del(&amp;ucm_dev-&gt;cdev, &amp;ucm_dev-&gt;dev);</div><div class='del'>-err_devnum:</div><div class='del'>-	ib_ucm_free_dev(ucm_dev);</div><div class='del'>-err:</div><div class='del'>-	put_device(&amp;ucm_dev-&gt;dev);</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void ib_ucm_remove_one(struct ib_device *device, void *client_data)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucm_device *ucm_dev = client_data;</div><div class='del'>-</div><div class='del'>-	if (!ucm_dev)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	cdev_device_del(&amp;ucm_dev-&gt;cdev, &amp;ucm_dev-&gt;dev);</div><div class='del'>-	ib_ucm_free_dev(ucm_dev);</div><div class='del'>-	put_device(&amp;ucm_dev-&gt;dev);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static CLASS_ATTR_STRING(abi_version, S_IRUGO,</div><div class='del'>-			 __stringify(IB_USER_CM_ABI_VERSION));</div><div class='del'>-</div><div class='del'>-static int __init ib_ucm_init(void)</div><div class='del'>-{</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	ret = register_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_NUM_FIXED_MINOR,</div><div class='del'>-				     "infiniband_cm");</div><div class='del'>-	if (ret) {</div><div class='del'>-		pr_err("ucm: couldn't register device number\n");</div><div class='del'>-		goto error1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = alloc_chrdev_region(&amp;dynamic_ucm_dev, 0, IB_UCM_NUM_DYNAMIC_MINOR,</div><div class='del'>-				  "infiniband_cm");</div><div class='del'>-	if (ret) {</div><div class='del'>-		pr_err("ucm: couldn't register dynamic device number\n");</div><div class='del'>-		goto err_alloc;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = class_create_file(&amp;cm_class, &amp;class_attr_abi_version.attr);</div><div class='del'>-	if (ret) {</div><div class='del'>-		pr_err("ucm: couldn't create abi_version attribute\n");</div><div class='del'>-		goto error2;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = ib_register_client(&amp;ucm_client);</div><div class='del'>-	if (ret) {</div><div class='del'>-		pr_err("ucm: couldn't register client\n");</div><div class='del'>-		goto error3;</div><div class='del'>-	}</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-error3:</div><div class='del'>-	class_remove_file(&amp;cm_class, &amp;class_attr_abi_version.attr);</div><div class='del'>-error2:</div><div class='del'>-	unregister_chrdev_region(dynamic_ucm_dev, IB_UCM_NUM_DYNAMIC_MINOR);</div><div class='del'>-err_alloc:</div><div class='del'>-	unregister_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_NUM_FIXED_MINOR);</div><div class='del'>-error1:</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void __exit ib_ucm_cleanup(void)</div><div class='del'>-{</div><div class='del'>-	ib_unregister_client(&amp;ucm_client);</div><div class='del'>-	class_remove_file(&amp;cm_class, &amp;class_attr_abi_version.attr);</div><div class='del'>-	unregister_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_NUM_FIXED_MINOR);</div><div class='del'>-	unregister_chrdev_region(dynamic_ucm_dev, IB_UCM_NUM_DYNAMIC_MINOR);</div><div class='del'>-	WARN_ON(!xa_empty(&amp;ctx_id_table));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-module_init(ib_ucm_init);</div><div class='del'>-module_exit(ib_ucm_cleanup);</div><div class='head'>diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c<br/>index 140a338a135f..ec3be65a2b88 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/ucma.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/ucma.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/ucma.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/ucma.c</a></div><div class='hunk'>@@ -52,6 +52,9 @@</div><div class='ctx'> #include &lt;rdma/rdma_cm_ib.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_addr.h&gt;</div><div class='ctx'> #include &lt;rdma/ib.h&gt;</div><div class='add'>+#include &lt;rdma/ib_cm.h&gt;</div><div class='add'>+#include &lt;rdma/rdma_netlink.h&gt;</div><div class='add'>+#include "core_priv.h"</div><div class='ctx'> </div><div class='ctx'> MODULE_AUTHOR("Sean Hefty");</div><div class='ctx'> MODULE_DESCRIPTION("RDMA Userspace Connection Manager Access");</div><div class='hunk'>@@ -66,9 +69,10 @@ static struct ctl_table ucma_ctl_table[] = {</div><div class='ctx'> 		.data		= &amp;max_backlog,</div><div class='ctx'> 		.maxlen		= sizeof max_backlog,</div><div class='ctx'> 		.mode		= 0644,</div><div class='del'>-		.proc_handler	= proc_dointvec,</div><div class='add'>+		.proc_handler	= proc_dointvec_minmax,</div><div class='add'>+		.extra1		= SYSCTL_ZERO,</div><div class='add'>+		.extra2		= SYSCTL_INT_MAX,</div><div class='ctx'> 	},</div><div class='del'>-	{ }</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct ucma_file {</div><div class='hunk'>@@ -77,34 +81,28 @@ struct ucma_file {</div><div class='ctx'> 	struct list_head	ctx_list;</div><div class='ctx'> 	struct list_head	event_list;</div><div class='ctx'> 	wait_queue_head_t	poll_wait;</div><div class='del'>-	struct workqueue_struct	*close_wq;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct ucma_context {</div><div class='del'>-	int			id;</div><div class='add'>+	u32			id;</div><div class='ctx'> 	struct completion	comp;</div><div class='del'>-	atomic_t		ref;</div><div class='add'>+	refcount_t		ref;</div><div class='ctx'> 	int			events_reported;</div><div class='del'>-	int			backlog;</div><div class='add'>+	atomic_t		backlog;</div><div class='ctx'> </div><div class='ctx'> 	struct ucma_file	*file;</div><div class='ctx'> 	struct rdma_cm_id	*cm_id;</div><div class='add'>+	struct mutex		mutex;</div><div class='ctx'> 	u64			uid;</div><div class='ctx'> </div><div class='ctx'> 	struct list_head	list;</div><div class='ctx'> 	struct list_head	mc_list;</div><div class='del'>-	/* mark that device is in process of destroying the internal HW</div><div class='del'>-	 * resources, protected by the global mut</div><div class='del'>-	 */</div><div class='del'>-	int			closing;</div><div class='del'>-	/* sync between removal event and id destroy, protected by file mut */</div><div class='del'>-	int			destroying;</div><div class='ctx'> 	struct work_struct	close_work;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct ucma_multicast {</div><div class='ctx'> 	struct ucma_context	*ctx;</div><div class='del'>-	int			id;</div><div class='add'>+	u32			id;</div><div class='ctx'> 	int			events_reported;</div><div class='ctx'> </div><div class='ctx'> 	u64			uid;</div><div class='hunk'>@@ -115,28 +113,27 @@ struct ucma_multicast {</div><div class='ctx'> </div><div class='ctx'> struct ucma_event {</div><div class='ctx'> 	struct ucma_context	*ctx;</div><div class='add'>+	struct ucma_context	*conn_req_ctx;</div><div class='ctx'> 	struct ucma_multicast	*mc;</div><div class='ctx'> 	struct list_head	list;</div><div class='del'>-	struct rdma_cm_id	*cm_id;</div><div class='ctx'> 	struct rdma_ucm_event_resp resp;</div><div class='del'>-	struct work_struct	close_work;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static DEFINE_MUTEX(mut);</div><div class='del'>-static DEFINE_IDR(ctx_idr);</div><div class='del'>-static DEFINE_IDR(multicast_idr);</div><div class='add'>+static DEFINE_XARRAY_ALLOC(ctx_table);</div><div class='add'>+static DEFINE_XARRAY_ALLOC(multicast_table);</div><div class='ctx'> </div><div class='ctx'> static const struct file_operations ucma_fops;</div><div class='add'>+static int ucma_destroy_private_ctx(struct ucma_context *ctx);</div><div class='ctx'> </div><div class='ctx'> static inline struct ucma_context *_ucma_find_context(int id,</div><div class='ctx'> 						      struct ucma_file *file)</div><div class='ctx'> {</div><div class='ctx'> 	struct ucma_context *ctx;</div><div class='ctx'> </div><div class='del'>-	ctx = idr_find(&amp;ctx_idr, id);</div><div class='add'>+	ctx = xa_load(&amp;ctx_table, id);</div><div class='ctx'> 	if (!ctx)</div><div class='ctx'> 		ctx = ERR_PTR(-ENOENT);</div><div class='del'>-	else if (ctx-&gt;file != file || !ctx-&gt;cm_id)</div><div class='add'>+	else if (ctx-&gt;file != file)</div><div class='ctx'> 		ctx = ERR_PTR(-EINVAL);</div><div class='ctx'> 	return ctx;</div><div class='ctx'> }</div><div class='hunk'>@@ -145,21 +142,18 @@ static struct ucma_context *ucma_get_ctx(struct ucma_file *file, int id)</div><div class='ctx'> {</div><div class='ctx'> 	struct ucma_context *ctx;</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;mut);</div><div class='add'>+	xa_lock(&amp;ctx_table);</div><div class='ctx'> 	ctx = _ucma_find_context(id, file);</div><div class='del'>-	if (!IS_ERR(ctx)) {</div><div class='del'>-		if (ctx-&gt;closing)</div><div class='del'>-			ctx = ERR_PTR(-EIO);</div><div class='del'>-		else</div><div class='del'>-			atomic_inc(&amp;ctx-&gt;ref);</div><div class='del'>-	}</div><div class='del'>-	mutex_unlock(&amp;mut);</div><div class='add'>+	if (!IS_ERR(ctx))</div><div class='add'>+		if (!refcount_inc_not_zero(&amp;ctx-&gt;ref))</div><div class='add'>+			ctx = ERR_PTR(-ENXIO);</div><div class='add'>+	xa_unlock(&amp;ctx_table);</div><div class='ctx'> 	return ctx;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void ucma_put_ctx(struct ucma_context *ctx)</div><div class='ctx'> {</div><div class='del'>-	if (atomic_dec_and_test(&amp;ctx-&gt;ref))</div><div class='add'>+	if (refcount_dec_and_test(&amp;ctx-&gt;ref))</div><div class='ctx'> 		complete(&amp;ctx-&gt;comp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -180,26 +174,21 @@ static struct ucma_context *ucma_get_ctx_dev(struct ucma_file *file, int id)</div><div class='ctx'> 	return ctx;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void ucma_close_event_id(struct work_struct *work)</div><div class='del'>-{</div><div class='del'>-	struct ucma_event *uevent_close =  container_of(work, struct ucma_event, close_work);</div><div class='del'>-</div><div class='del'>-	rdma_destroy_id(uevent_close-&gt;cm_id);</div><div class='del'>-	kfree(uevent_close);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static void ucma_close_id(struct work_struct *work)</div><div class='ctx'> {</div><div class='ctx'> 	struct ucma_context *ctx =  container_of(work, struct ucma_context, close_work);</div><div class='ctx'> </div><div class='ctx'> 	/* once all inflight tasks are finished, we close all underlying</div><div class='ctx'> 	 * resources. The context is still alive till its explicit destryoing</div><div class='del'>-	 * by its creator.</div><div class='add'>+	 * by its creator. This puts back the xarray's reference.</div><div class='ctx'> 	 */</div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='ctx'> 	wait_for_completion(&amp;ctx-&gt;comp);</div><div class='ctx'> 	/* No new events will be generated after destroying the id. */</div><div class='ctx'> 	rdma_destroy_id(ctx-&gt;cm_id);</div><div class='add'>+</div><div class='add'>+	/* Reading the cm_id without holding a positive ref is not allowed */</div><div class='add'>+	ctx-&gt;cm_id = NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static struct ucma_context *ucma_alloc_ctx(struct ucma_file *file)</div><div class='hunk'>@@ -211,46 +200,32 @@ static struct ucma_context *ucma_alloc_ctx(struct ucma_file *file)</div><div class='ctx'> 		return NULL;</div><div class='ctx'> </div><div class='ctx'> 	INIT_WORK(&amp;ctx-&gt;close_work, ucma_close_id);</div><div class='del'>-	atomic_set(&amp;ctx-&gt;ref, 1);</div><div class='ctx'> 	init_completion(&amp;ctx-&gt;comp);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;ctx-&gt;mc_list);</div><div class='add'>+	/* So list_del() will work if we don't do ucma_finish_ctx() */</div><div class='add'>+	INIT_LIST_HEAD(&amp;ctx-&gt;list);</div><div class='ctx'> 	ctx-&gt;file = file;</div><div class='add'>+	mutex_init(&amp;ctx-&gt;mutex);</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;mut);</div><div class='del'>-	ctx-&gt;id = idr_alloc(&amp;ctx_idr, ctx, 0, 0, GFP_KERNEL);</div><div class='del'>-	mutex_unlock(&amp;mut);</div><div class='del'>-	if (ctx-&gt;id &lt; 0)</div><div class='del'>-		goto error;</div><div class='del'>-</div><div class='del'>-	list_add_tail(&amp;ctx-&gt;list, &amp;file-&gt;ctx_list);</div><div class='add'>+	if (xa_alloc(&amp;ctx_table, &amp;ctx-&gt;id, NULL, xa_limit_32b, GFP_KERNEL)) {</div><div class='add'>+		kfree(ctx);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='ctx'> 	return ctx;</div><div class='del'>-</div><div class='del'>-error:</div><div class='del'>-	kfree(ctx);</div><div class='del'>-	return NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct ucma_multicast* ucma_alloc_multicast(struct ucma_context *ctx)</div><div class='add'>+static void ucma_set_ctx_cm_id(struct ucma_context *ctx,</div><div class='add'>+			       struct rdma_cm_id *cm_id)</div><div class='ctx'> {</div><div class='del'>-	struct ucma_multicast *mc;</div><div class='del'>-</div><div class='del'>-	mc = kzalloc(sizeof(*mc), GFP_KERNEL);</div><div class='del'>-	if (!mc)</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;mut);</div><div class='del'>-	mc-&gt;id = idr_alloc(&amp;multicast_idr, NULL, 0, 0, GFP_KERNEL);</div><div class='del'>-	mutex_unlock(&amp;mut);</div><div class='del'>-	if (mc-&gt;id &lt; 0)</div><div class='del'>-		goto error;</div><div class='del'>-</div><div class='del'>-	mc-&gt;ctx = ctx;</div><div class='del'>-	list_add_tail(&amp;mc-&gt;list, &amp;ctx-&gt;mc_list);</div><div class='del'>-	return mc;</div><div class='add'>+	refcount_set(&amp;ctx-&gt;ref, 1);</div><div class='add'>+	ctx-&gt;cm_id = cm_id;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-error:</div><div class='del'>-	kfree(mc);</div><div class='del'>-	return NULL;</div><div class='add'>+static void ucma_finish_ctx(struct ucma_context *ctx)</div><div class='add'>+{</div><div class='add'>+	lockdep_assert_held(&amp;ctx-&gt;file-&gt;mut);</div><div class='add'>+	list_add_tail(&amp;ctx-&gt;list, &amp;ctx-&gt;file-&gt;ctx_list);</div><div class='add'>+	xa_store(&amp;ctx_table, ctx-&gt;id, ctx, GFP_KERNEL);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void ucma_copy_conn_event(struct rdma_ucm_conn_param *dst,</div><div class='hunk'>@@ -260,7 +235,7 @@ static void ucma_copy_conn_event(struct rdma_ucm_conn_param *dst,</div><div class='ctx'> 		memcpy(dst-&gt;private_data, src-&gt;private_data,</div><div class='ctx'> 		       src-&gt;private_data_len);</div><div class='ctx'> 	dst-&gt;private_data_len = src-&gt;private_data_len;</div><div class='del'>-	dst-&gt;responder_resources =src-&gt;responder_resources;</div><div class='add'>+	dst-&gt;responder_resources = src-&gt;responder_resources;</div><div class='ctx'> 	dst-&gt;initiator_depth = src-&gt;initiator_depth;</div><div class='ctx'> 	dst-&gt;flow_control = src-&gt;flow_control;</div><div class='ctx'> 	dst-&gt;retry_count = src-&gt;retry_count;</div><div class='hunk'>@@ -282,10 +257,15 @@ static void ucma_copy_ud_event(struct ib_device *device,</div><div class='ctx'> 	dst-&gt;qkey = src-&gt;qkey;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void ucma_set_event_context(struct ucma_context *ctx,</div><div class='del'>-				   struct rdma_cm_event *event,</div><div class='del'>-				   struct ucma_event *uevent)</div><div class='add'>+static struct ucma_event *ucma_create_uevent(struct ucma_context *ctx,</div><div class='add'>+					     struct rdma_cm_event *event)</div><div class='ctx'> {</div><div class='add'>+	struct ucma_event *uevent;</div><div class='add'>+</div><div class='add'>+	uevent = kzalloc(sizeof(*uevent), GFP_KERNEL);</div><div class='add'>+	if (!uevent)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='ctx'> 	uevent-&gt;ctx = ctx;</div><div class='ctx'> 	switch (event-&gt;event) {</div><div class='ctx'> 	case RDMA_CM_EVENT_MULTICAST_JOIN:</div><div class='hunk'>@@ -300,44 +280,60 @@ static void ucma_set_event_context(struct ucma_context *ctx,</div><div class='ctx'> 		uevent-&gt;resp.id = ctx-&gt;id;</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='add'>+	uevent-&gt;resp.event = event-&gt;event;</div><div class='add'>+	uevent-&gt;resp.status = event-&gt;status;</div><div class='add'>+</div><div class='add'>+	if (event-&gt;event == RDMA_CM_EVENT_ADDRINFO_RESOLVED)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	if (ctx-&gt;cm_id-&gt;qp_type == IB_QPT_UD)</div><div class='add'>+		ucma_copy_ud_event(ctx-&gt;cm_id-&gt;device, &amp;uevent-&gt;resp.param.ud,</div><div class='add'>+				   &amp;event-&gt;param.ud);</div><div class='add'>+	else</div><div class='add'>+		ucma_copy_conn_event(&amp;uevent-&gt;resp.param.conn,</div><div class='add'>+				     &amp;event-&gt;param.conn);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	uevent-&gt;resp.ece.vendor_id = event-&gt;ece.vendor_id;</div><div class='add'>+	uevent-&gt;resp.ece.attr_mod = event-&gt;ece.attr_mod;</div><div class='add'>+	return uevent;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/* Called with file-&gt;mut locked for the relevant context. */</div><div class='del'>-static void ucma_removal_event_handler(struct rdma_cm_id *cm_id)</div><div class='add'>+static int ucma_connect_event_handler(struct rdma_cm_id *cm_id,</div><div class='add'>+				      struct rdma_cm_event *event)</div><div class='ctx'> {</div><div class='del'>-	struct ucma_context *ctx = cm_id-&gt;context;</div><div class='del'>-	struct ucma_event *con_req_eve;</div><div class='del'>-	int event_found = 0;</div><div class='add'>+	struct ucma_context *listen_ctx = cm_id-&gt;context;</div><div class='add'>+	struct ucma_context *ctx;</div><div class='add'>+	struct ucma_event *uevent;</div><div class='ctx'> </div><div class='del'>-	if (ctx-&gt;destroying)</div><div class='del'>-		return;</div><div class='add'>+	if (!atomic_add_unless(&amp;listen_ctx-&gt;backlog, -1, 0))</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	ctx = ucma_alloc_ctx(listen_ctx-&gt;file);</div><div class='add'>+	if (!ctx)</div><div class='add'>+		goto err_backlog;</div><div class='add'>+	ucma_set_ctx_cm_id(ctx, cm_id);</div><div class='ctx'> </div><div class='del'>-	/* only if context is pointing to cm_id that it owns it and can be</div><div class='del'>-	 * queued to be closed, otherwise that cm_id is an inflight one that</div><div class='del'>-	 * is part of that context event list pending to be detached and</div><div class='del'>-	 * reattached to its new context as part of ucma_get_event,</div><div class='del'>-	 * handled separately below.</div><div class='del'>-	 */</div><div class='del'>-	if (ctx-&gt;cm_id == cm_id) {</div><div class='del'>-		mutex_lock(&amp;mut);</div><div class='del'>-		ctx-&gt;closing = 1;</div><div class='del'>-		mutex_unlock(&amp;mut);</div><div class='del'>-		queue_work(ctx-&gt;file-&gt;close_wq, &amp;ctx-&gt;close_work);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='add'>+	uevent = ucma_create_uevent(listen_ctx, event);</div><div class='add'>+	if (!uevent)</div><div class='add'>+		goto err_alloc;</div><div class='add'>+	uevent-&gt;conn_req_ctx = ctx;</div><div class='add'>+	uevent-&gt;resp.id = ctx-&gt;id;</div><div class='ctx'> </div><div class='del'>-	list_for_each_entry(con_req_eve, &amp;ctx-&gt;file-&gt;event_list, list) {</div><div class='del'>-		if (con_req_eve-&gt;cm_id == cm_id &amp;&amp;</div><div class='del'>-		    con_req_eve-&gt;resp.event == RDMA_CM_EVENT_CONNECT_REQUEST) {</div><div class='del'>-			list_del(&amp;con_req_eve-&gt;list);</div><div class='del'>-			INIT_WORK(&amp;con_req_eve-&gt;close_work, ucma_close_event_id);</div><div class='del'>-			queue_work(ctx-&gt;file-&gt;close_wq, &amp;con_req_eve-&gt;close_work);</div><div class='del'>-			event_found = 1;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	if (!event_found)</div><div class='del'>-		pr_err("ucma_removal_event_handler: warning: connect request event wasn't found\n");</div><div class='add'>+	ctx-&gt;cm_id-&gt;context = ctx;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;ctx-&gt;file-&gt;mut);</div><div class='add'>+	ucma_finish_ctx(ctx);</div><div class='add'>+	list_add_tail(&amp;uevent-&gt;list, &amp;ctx-&gt;file-&gt;event_list);</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;file-&gt;mut);</div><div class='add'>+	wake_up_interruptible(&amp;ctx-&gt;file-&gt;poll_wait);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_alloc:</div><div class='add'>+	ucma_destroy_private_ctx(ctx);</div><div class='add'>+err_backlog:</div><div class='add'>+	atomic_inc(&amp;listen_ctx-&gt;backlog);</div><div class='add'>+	/* Returning error causes the new ID to be destroyed */</div><div class='add'>+	return -ENOMEM;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int ucma_event_handler(struct rdma_cm_id *cm_id,</div><div class='hunk'>@@ -345,69 +341,49 @@ static int ucma_event_handler(struct rdma_cm_id *cm_id,</div><div class='ctx'> {</div><div class='ctx'> 	struct ucma_event *uevent;</div><div class='ctx'> 	struct ucma_context *ctx = cm_id-&gt;context;</div><div class='del'>-	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	uevent = kzalloc(sizeof(*uevent), GFP_KERNEL);</div><div class='del'>-	if (!uevent)</div><div class='del'>-		return event-&gt;event == RDMA_CM_EVENT_CONNECT_REQUEST;</div><div class='add'>+	if (event-&gt;event == RDMA_CM_EVENT_CONNECT_REQUEST)</div><div class='add'>+		return ucma_connect_event_handler(cm_id, event);</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;ctx-&gt;file-&gt;mut);</div><div class='del'>-	uevent-&gt;cm_id = cm_id;</div><div class='del'>-	ucma_set_event_context(ctx, event, uevent);</div><div class='del'>-	uevent-&gt;resp.event = event-&gt;event;</div><div class='del'>-	uevent-&gt;resp.status = event-&gt;status;</div><div class='del'>-	if (cm_id-&gt;qp_type == IB_QPT_UD)</div><div class='del'>-		ucma_copy_ud_event(cm_id-&gt;device, &amp;uevent-&gt;resp.param.ud,</div><div class='del'>-				   &amp;event-&gt;param.ud);</div><div class='del'>-	else</div><div class='del'>-		ucma_copy_conn_event(&amp;uevent-&gt;resp.param.conn,</div><div class='del'>-				     &amp;event-&gt;param.conn);</div><div class='del'>-</div><div class='del'>-	if (event-&gt;event == RDMA_CM_EVENT_CONNECT_REQUEST) {</div><div class='del'>-		if (!ctx-&gt;backlog) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			kfree(uevent);</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-		ctx-&gt;backlog--;</div><div class='del'>-	} else if (!ctx-&gt;uid || ctx-&gt;cm_id != cm_id) {</div><div class='del'>-		/*</div><div class='del'>-		 * We ignore events for new connections until userspace has set</div><div class='del'>-		 * their context.  This can only happen if an error occurs on a</div><div class='del'>-		 * new connection before the user accepts it.  This is okay,</div><div class='del'>-		 * since the accept will just fail later. However, we do need</div><div class='del'>-		 * to release the underlying HW resources in case of a device</div><div class='del'>-		 * removal event.</div><div class='del'>-		 */</div><div class='del'>-		if (event-&gt;event == RDMA_CM_EVENT_DEVICE_REMOVAL)</div><div class='del'>-			ucma_removal_event_handler(cm_id);</div><div class='del'>-</div><div class='del'>-		kfree(uevent);</div><div class='del'>-		goto out;</div><div class='add'>+	/*</div><div class='add'>+	 * We ignore events for new connections until userspace has set their</div><div class='add'>+	 * context.  This can only happen if an error occurs on a new connection</div><div class='add'>+	 * before the user accepts it.  This is okay, since the accept will just</div><div class='add'>+	 * fail later. However, we do need to release the underlying HW</div><div class='add'>+	 * resources in case of a device removal event.</div><div class='add'>+	 */</div><div class='add'>+	if (ctx-&gt;uid) {</div><div class='add'>+		uevent = ucma_create_uevent(ctx, event);</div><div class='add'>+		if (!uevent)</div><div class='add'>+			return 0;</div><div class='add'>+</div><div class='add'>+		mutex_lock(&amp;ctx-&gt;file-&gt;mut);</div><div class='add'>+		list_add_tail(&amp;uevent-&gt;list, &amp;ctx-&gt;file-&gt;event_list);</div><div class='add'>+		mutex_unlock(&amp;ctx-&gt;file-&gt;mut);</div><div class='add'>+		wake_up_interruptible(&amp;ctx-&gt;file-&gt;poll_wait);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	list_add_tail(&amp;uevent-&gt;list, &amp;ctx-&gt;file-&gt;event_list);</div><div class='del'>-	wake_up_interruptible(&amp;ctx-&gt;file-&gt;poll_wait);</div><div class='del'>-	if (event-&gt;event == RDMA_CM_EVENT_DEVICE_REMOVAL)</div><div class='del'>-		ucma_removal_event_handler(cm_id);</div><div class='del'>-out:</div><div class='del'>-	mutex_unlock(&amp;ctx-&gt;file-&gt;mut);</div><div class='del'>-	return ret;</div><div class='add'>+	if (event-&gt;event == RDMA_CM_EVENT_DEVICE_REMOVAL) {</div><div class='add'>+		xa_lock(&amp;ctx_table);</div><div class='add'>+		if (xa_load(&amp;ctx_table, ctx-&gt;id) == ctx)</div><div class='add'>+			queue_work(system_dfl_wq, &amp;ctx-&gt;close_work);</div><div class='add'>+		xa_unlock(&amp;ctx_table);</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t ucma_get_event(struct ucma_file *file, const char __user *inbuf,</div><div class='ctx'> 			      int in_len, int out_len)</div><div class='ctx'> {</div><div class='del'>-	struct ucma_context *ctx;</div><div class='ctx'> 	struct rdma_ucm_get_event cmd;</div><div class='ctx'> 	struct ucma_event *uevent;</div><div class='del'>-	int ret = 0;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Old 32 bit user space does not send the 4 byte padding in the</div><div class='ctx'> 	 * reserved field. We don't care, allow it to keep working.</div><div class='ctx'> 	 */</div><div class='del'>-	if (out_len &lt; sizeof(uevent-&gt;resp) - sizeof(uevent-&gt;resp.reserved))</div><div class='add'>+	if (out_len &lt; sizeof(uevent-&gt;resp) - sizeof(uevent-&gt;resp.reserved) -</div><div class='add'>+			      sizeof(uevent-&gt;resp.ece))</div><div class='ctx'> 		return -ENOSPC;</div><div class='ctx'> </div><div class='ctx'> 	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='hunk'>@@ -427,35 +403,25 @@ static ssize_t ucma_get_event(struct ucma_file *file, const char __user *inbuf,</div><div class='ctx'> 		mutex_lock(&amp;file-&gt;mut);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	uevent = list_entry(file-&gt;event_list.next, struct ucma_event, list);</div><div class='del'>-</div><div class='del'>-	if (uevent-&gt;resp.event == RDMA_CM_EVENT_CONNECT_REQUEST) {</div><div class='del'>-		ctx = ucma_alloc_ctx(file);</div><div class='del'>-		if (!ctx) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto done;</div><div class='del'>-		}</div><div class='del'>-		uevent-&gt;ctx-&gt;backlog++;</div><div class='del'>-		ctx-&gt;cm_id = uevent-&gt;cm_id;</div><div class='del'>-		ctx-&gt;cm_id-&gt;context = ctx;</div><div class='del'>-		uevent-&gt;resp.id = ctx-&gt;id;</div><div class='del'>-	}</div><div class='add'>+	uevent = list_first_entry(&amp;file-&gt;event_list, struct ucma_event, list);</div><div class='ctx'> </div><div class='ctx'> 	if (copy_to_user(u64_to_user_ptr(cmd.response),</div><div class='ctx'> 			 &amp;uevent-&gt;resp,</div><div class='ctx'> 			 min_t(size_t, out_len, sizeof(uevent-&gt;resp)))) {</div><div class='del'>-		ret = -EFAULT;</div><div class='del'>-		goto done;</div><div class='add'>+		mutex_unlock(&amp;file-&gt;mut);</div><div class='add'>+		return -EFAULT;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	list_del(&amp;uevent-&gt;list);</div><div class='ctx'> 	uevent-&gt;ctx-&gt;events_reported++;</div><div class='ctx'> 	if (uevent-&gt;mc)</div><div class='ctx'> 		uevent-&gt;mc-&gt;events_reported++;</div><div class='del'>-	kfree(uevent);</div><div class='del'>-done:</div><div class='add'>+	if (uevent-&gt;resp.event == RDMA_CM_EVENT_CONNECT_REQUEST)</div><div class='add'>+		atomic_inc(&amp;uevent-&gt;ctx-&gt;backlog);</div><div class='ctx'> 	mutex_unlock(&amp;file-&gt;mut);</div><div class='del'>-	return ret;</div><div class='add'>+</div><div class='add'>+	kfree(uevent);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int ucma_get_qp_type(struct rdma_ucm_create_id *cmd, enum ib_qp_type *qp_type)</div><div class='hunk'>@@ -496,40 +462,32 @@ static ssize_t ucma_create_id(struct ucma_file *file, const char __user *inbuf,</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;file-&gt;mut);</div><div class='ctx'> 	ctx = ucma_alloc_ctx(file);</div><div class='del'>-	mutex_unlock(&amp;file-&gt;mut);</div><div class='ctx'> 	if (!ctx)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	ctx-&gt;uid = cmd.uid;</div><div class='del'>-	cm_id = __rdma_create_id(current-&gt;nsproxy-&gt;net_ns,</div><div class='del'>-				 ucma_event_handler, ctx, cmd.ps, qp_type, NULL);</div><div class='add'>+	cm_id = rdma_create_user_id(ucma_event_handler, ctx, cmd.ps, qp_type);</div><div class='ctx'> 	if (IS_ERR(cm_id)) {</div><div class='ctx'> 		ret = PTR_ERR(cm_id);</div><div class='ctx'> 		goto err1;</div><div class='ctx'> 	}</div><div class='add'>+	ucma_set_ctx_cm_id(ctx, cm_id);</div><div class='ctx'> </div><div class='ctx'> 	resp.id = ctx-&gt;id;</div><div class='ctx'> 	if (copy_to_user(u64_to_user_ptr(cmd.response),</div><div class='ctx'> 			 &amp;resp, sizeof(resp))) {</div><div class='ctx'> 		ret = -EFAULT;</div><div class='del'>-		goto err2;</div><div class='add'>+		goto err1;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ctx-&gt;cm_id = cm_id;</div><div class='add'>+	mutex_lock(&amp;file-&gt;mut);</div><div class='add'>+	ucma_finish_ctx(ctx);</div><div class='add'>+	mutex_unlock(&amp;file-&gt;mut);</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-err2:</div><div class='del'>-	rdma_destroy_id(cm_id);</div><div class='ctx'> err1:</div><div class='del'>-	mutex_lock(&amp;mut);</div><div class='del'>-	idr_remove(&amp;ctx_idr, ctx-&gt;id);</div><div class='del'>-	mutex_unlock(&amp;mut);</div><div class='del'>-	mutex_lock(&amp;file-&gt;mut);</div><div class='del'>-	list_del(&amp;ctx-&gt;list);</div><div class='del'>-	mutex_unlock(&amp;file-&gt;mut);</div><div class='del'>-	kfree(ctx);</div><div class='add'>+	ucma_destroy_private_ctx(ctx);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -537,19 +495,25 @@ static void ucma_cleanup_multicast(struct ucma_context *ctx)</div><div class='ctx'> {</div><div class='ctx'> 	struct ucma_multicast *mc, *tmp;</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;mut);</div><div class='add'>+	xa_lock(&amp;multicast_table);</div><div class='ctx'> 	list_for_each_entry_safe(mc, tmp, &amp;ctx-&gt;mc_list, list) {</div><div class='ctx'> 		list_del(&amp;mc-&gt;list);</div><div class='del'>-		idr_remove(&amp;multicast_idr, mc-&gt;id);</div><div class='add'>+		/*</div><div class='add'>+		 * At this point mc-&gt;ctx-&gt;ref is 0 so the mc cannot leave the</div><div class='add'>+		 * lock on the reader and this is enough serialization</div><div class='add'>+		 */</div><div class='add'>+		__xa_erase(&amp;multicast_table, mc-&gt;id);</div><div class='ctx'> 		kfree(mc);</div><div class='ctx'> 	}</div><div class='del'>-	mutex_unlock(&amp;mut);</div><div class='add'>+	xa_unlock(&amp;multicast_table);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void ucma_cleanup_mc_events(struct ucma_multicast *mc)</div><div class='ctx'> {</div><div class='ctx'> 	struct ucma_event *uevent, *tmp;</div><div class='ctx'> </div><div class='add'>+	rdma_lock_handler(mc-&gt;ctx-&gt;cm_id);</div><div class='add'>+	mutex_lock(&amp;mc-&gt;ctx-&gt;file-&gt;mut);</div><div class='ctx'> 	list_for_each_entry_safe(uevent, tmp, &amp;mc-&gt;ctx-&gt;file-&gt;event_list, list) {</div><div class='ctx'> 		if (uevent-&gt;mc != mc)</div><div class='ctx'> 			continue;</div><div class='hunk'>@@ -557,45 +521,75 @@ static void ucma_cleanup_mc_events(struct ucma_multicast *mc)</div><div class='ctx'> 		list_del(&amp;uevent-&gt;list);</div><div class='ctx'> 		kfree(uevent);</div><div class='ctx'> 	}</div><div class='add'>+	mutex_unlock(&amp;mc-&gt;ctx-&gt;file-&gt;mut);</div><div class='add'>+	rdma_unlock_handler(mc-&gt;ctx-&gt;cm_id);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/*</div><div class='del'>- * ucma_free_ctx is called after the underlying rdma CM-ID is destroyed. At</div><div class='del'>- * this point, no new events will be reported from the hardware. However, we</div><div class='del'>- * still need to cleanup the UCMA context for this ID. Specifically, there</div><div class='del'>- * might be events that have not yet been consumed by the user space software.</div><div class='del'>- * These might include pending connect requests which we have not completed</div><div class='del'>- * processing.  We cannot call rdma_destroy_id while holding the lock of the</div><div class='del'>- * context (file-&gt;mut), as it might cause a deadlock. We therefore extract all</div><div class='del'>- * relevant events from the context pending events list while holding the</div><div class='del'>- * mutex. After that we release them as needed.</div><div class='del'>- */</div><div class='del'>-static int ucma_free_ctx(struct ucma_context *ctx)</div><div class='add'>+static int ucma_cleanup_ctx_events(struct ucma_context *ctx)</div><div class='ctx'> {</div><div class='ctx'> 	int events_reported;</div><div class='ctx'> 	struct ucma_event *uevent, *tmp;</div><div class='ctx'> 	LIST_HEAD(list);</div><div class='ctx'> </div><div class='del'>-</div><div class='del'>-	ucma_cleanup_multicast(ctx);</div><div class='del'>-</div><div class='del'>-	/* Cleanup events not yet reported to the user. */</div><div class='add'>+	/* Cleanup events not yet reported to the user.*/</div><div class='ctx'> 	mutex_lock(&amp;ctx-&gt;file-&gt;mut);</div><div class='ctx'> 	list_for_each_entry_safe(uevent, tmp, &amp;ctx-&gt;file-&gt;event_list, list) {</div><div class='del'>-		if (uevent-&gt;ctx == ctx)</div><div class='add'>+		if (uevent-&gt;ctx != ctx)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		if (uevent-&gt;resp.event == RDMA_CM_EVENT_CONNECT_REQUEST &amp;&amp;</div><div class='add'>+		    xa_cmpxchg(&amp;ctx_table, uevent-&gt;conn_req_ctx-&gt;id,</div><div class='add'>+			       uevent-&gt;conn_req_ctx, XA_ZERO_ENTRY,</div><div class='add'>+			       GFP_KERNEL) == uevent-&gt;conn_req_ctx) {</div><div class='ctx'> 			list_move_tail(&amp;uevent-&gt;list, &amp;list);</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+		list_del(&amp;uevent-&gt;list);</div><div class='add'>+		kfree(uevent);</div><div class='ctx'> 	}</div><div class='ctx'> 	list_del(&amp;ctx-&gt;list);</div><div class='add'>+	events_reported = ctx-&gt;events_reported;</div><div class='ctx'> 	mutex_unlock(&amp;ctx-&gt;file-&gt;mut);</div><div class='ctx'> </div><div class='add'>+	/*</div><div class='add'>+	 * If this was a listening ID then any connections spawned from it that</div><div class='add'>+	 * have not been delivered to userspace are cleaned up too. Must be done</div><div class='add'>+	 * outside any locks.</div><div class='add'>+	 */</div><div class='ctx'> 	list_for_each_entry_safe(uevent, tmp, &amp;list, list) {</div><div class='del'>-		list_del(&amp;uevent-&gt;list);</div><div class='del'>-		if (uevent-&gt;resp.event == RDMA_CM_EVENT_CONNECT_REQUEST)</div><div class='del'>-			rdma_destroy_id(uevent-&gt;cm_id);</div><div class='add'>+		ucma_destroy_private_ctx(uevent-&gt;conn_req_ctx);</div><div class='ctx'> 		kfree(uevent);</div><div class='ctx'> 	}</div><div class='add'>+	return events_reported;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	events_reported = ctx-&gt;events_reported;</div><div class='add'>+/*</div><div class='add'>+ * When this is called the xarray must have a XA_ZERO_ENTRY in the ctx-&gt;id (ie</div><div class='add'>+ * the ctx is not public to the user). This either because:</div><div class='add'>+ *  - ucma_finish_ctx() hasn't been called</div><div class='add'>+ *  - xa_cmpxchg() succeed to remove the entry (only one thread can succeed)</div><div class='add'>+ */</div><div class='add'>+static int ucma_destroy_private_ctx(struct ucma_context *ctx)</div><div class='add'>+{</div><div class='add'>+	int events_reported;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Destroy the underlying cm_id. New work queuing is prevented now by</div><div class='add'>+	 * the removal from the xarray. Once the work is cancled ref will either</div><div class='add'>+	 * be 0 because the work ran to completion and consumed the ref from the</div><div class='add'>+	 * xarray, or it will be positive because we still have the ref from the</div><div class='add'>+	 * xarray. This can also be 0 in cases where cm_id was never set</div><div class='add'>+	 */</div><div class='add'>+	cancel_work_sync(&amp;ctx-&gt;close_work);</div><div class='add'>+	if (refcount_read(&amp;ctx-&gt;ref))</div><div class='add'>+		ucma_close_id(&amp;ctx-&gt;close_work);</div><div class='add'>+</div><div class='add'>+	events_reported = ucma_cleanup_ctx_events(ctx);</div><div class='add'>+	ucma_cleanup_multicast(ctx);</div><div class='add'>+</div><div class='add'>+	WARN_ON(xa_cmpxchg(&amp;ctx_table, ctx-&gt;id, XA_ZERO_ENTRY, NULL,</div><div class='add'>+			   GFP_KERNEL) != NULL);</div><div class='add'>+	mutex_destroy(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	kfree(ctx);</div><div class='ctx'> 	return events_reported;</div><div class='ctx'> }</div><div class='hunk'>@@ -614,33 +608,19 @@ static ssize_t ucma_destroy_id(struct ucma_file *file, const char __user *inbuf,</div><div class='ctx'> 	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='ctx'> 		return -EFAULT;</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;mut);</div><div class='add'>+	xa_lock(&amp;ctx_table);</div><div class='ctx'> 	ctx = _ucma_find_context(cmd.id, file);</div><div class='del'>-	if (!IS_ERR(ctx))</div><div class='del'>-		idr_remove(&amp;ctx_idr, ctx-&gt;id);</div><div class='del'>-	mutex_unlock(&amp;mut);</div><div class='add'>+	if (!IS_ERR(ctx)) {</div><div class='add'>+		if (__xa_cmpxchg(&amp;ctx_table, ctx-&gt;id, ctx, XA_ZERO_ENTRY,</div><div class='add'>+				 GFP_KERNEL) != ctx)</div><div class='add'>+			ctx = ERR_PTR(-ENOENT);</div><div class='add'>+	}</div><div class='add'>+	xa_unlock(&amp;ctx_table);</div><div class='ctx'> </div><div class='ctx'> 	if (IS_ERR(ctx))</div><div class='ctx'> 		return PTR_ERR(ctx);</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;ctx-&gt;file-&gt;mut);</div><div class='del'>-	ctx-&gt;destroying = 1;</div><div class='del'>-	mutex_unlock(&amp;ctx-&gt;file-&gt;mut);</div><div class='del'>-</div><div class='del'>-	flush_workqueue(ctx-&gt;file-&gt;close_wq);</div><div class='del'>-	/* At this point it's guaranteed that there is no inflight</div><div class='del'>-	 * closing task */</div><div class='del'>-	mutex_lock(&amp;mut);</div><div class='del'>-	if (!ctx-&gt;closing) {</div><div class='del'>-		mutex_unlock(&amp;mut);</div><div class='del'>-		ucma_put_ctx(ctx);</div><div class='del'>-		wait_for_completion(&amp;ctx-&gt;comp);</div><div class='del'>-		rdma_destroy_id(ctx-&gt;cm_id);</div><div class='del'>-	} else {</div><div class='del'>-		mutex_unlock(&amp;mut);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	resp.events_reported = ucma_free_ctx(ctx);</div><div class='add'>+	resp.events_reported = ucma_destroy_private_ctx(ctx);</div><div class='ctx'> 	if (copy_to_user(u64_to_user_ptr(cmd.response),</div><div class='ctx'> 			 &amp;resp, sizeof(resp)))</div><div class='ctx'> 		ret = -EFAULT;</div><div class='hunk'>@@ -665,7 +645,10 @@ static ssize_t ucma_bind_ip(struct ucma_file *file, const char __user *inbuf,</div><div class='ctx'> 	if (IS_ERR(ctx))</div><div class='ctx'> 		return PTR_ERR(ctx);</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ret = rdma_bind_addr(ctx-&gt;cm_id, (struct sockaddr *) &amp;cmd.addr);</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='add'>+</div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -688,7 +671,9 @@ static ssize_t ucma_bind(struct ucma_file *file, const char __user *inbuf,</div><div class='ctx'> 	if (IS_ERR(ctx))</div><div class='ctx'> 		return PTR_ERR(ctx);</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ret = rdma_bind_addr(ctx-&gt;cm_id, (struct sockaddr *) &amp;cmd.addr);</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -712,8 +697,10 @@ static ssize_t ucma_resolve_ip(struct ucma_file *file,</div><div class='ctx'> 	if (IS_ERR(ctx))</div><div class='ctx'> 		return PTR_ERR(ctx);</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ret = rdma_resolve_addr(ctx-&gt;cm_id, (struct sockaddr *) &amp;cmd.src_addr,</div><div class='ctx'> 				(struct sockaddr *) &amp;cmd.dst_addr, cmd.timeout_ms);</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -738,8 +725,32 @@ static ssize_t ucma_resolve_addr(struct ucma_file *file,</div><div class='ctx'> 	if (IS_ERR(ctx))</div><div class='ctx'> 		return PTR_ERR(ctx);</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ret = rdma_resolve_addr(ctx-&gt;cm_id, (struct sockaddr *) &amp;cmd.src_addr,</div><div class='ctx'> 				(struct sockaddr *) &amp;cmd.dst_addr, cmd.timeout_ms);</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='add'>+	ucma_put_ctx(ctx);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static ssize_t ucma_resolve_ib_service(struct ucma_file *file,</div><div class='add'>+				       const char __user *inbuf, int in_len,</div><div class='add'>+				       int out_len)</div><div class='add'>+{</div><div class='add'>+	struct rdma_ucm_resolve_ib_service cmd;</div><div class='add'>+	struct ucma_context *ctx;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='add'>+		return -EFAULT;</div><div class='add'>+</div><div class='add'>+	ctx = ucma_get_ctx(file, cmd.id);</div><div class='add'>+	if (IS_ERR(ctx))</div><div class='add'>+		return PTR_ERR(ctx);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;ctx-&gt;mutex);</div><div class='add'>+	ret = rdma_resolve_ib_service(ctx-&gt;cm_id, &amp;cmd.ibs);</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -759,7 +770,9 @@ static ssize_t ucma_resolve_route(struct ucma_file *file,</div><div class='ctx'> 	if (IS_ERR(ctx))</div><div class='ctx'> 		return PTR_ERR(ctx);</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ret = rdma_resolve_route(ctx-&gt;cm_id, cmd.timeout_ms);</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -769,8 +782,8 @@ static void ucma_copy_ib_route(struct rdma_ucm_query_route_resp *resp,</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_dev_addr *dev_addr;</div><div class='ctx'> </div><div class='del'>-	resp-&gt;num_paths = route-&gt;num_paths;</div><div class='del'>-	switch (route-&gt;num_paths) {</div><div class='add'>+	resp-&gt;num_paths = route-&gt;num_pri_alt_paths;</div><div class='add'>+	switch (route-&gt;num_pri_alt_paths) {</div><div class='ctx'> 	case 0:</div><div class='ctx'> 		dev_addr = &amp;route-&gt;addr.dev_addr;</div><div class='ctx'> 		rdma_addr_get_dgid(dev_addr,</div><div class='hunk'>@@ -782,7 +795,7 @@ static void ucma_copy_ib_route(struct rdma_ucm_query_route_resp *resp,</div><div class='ctx'> 	case 2:</div><div class='ctx'> 		ib_copy_path_rec_to_user(&amp;resp-&gt;ib_route[1],</div><div class='ctx'> 					 &amp;route-&gt;path_rec[1]);</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case 1:</div><div class='ctx'> 		ib_copy_path_rec_to_user(&amp;resp-&gt;ib_route[0],</div><div class='ctx'> 					 &amp;route-&gt;path_rec[0]);</div><div class='hunk'>@@ -796,8 +809,8 @@ static void ucma_copy_iboe_route(struct rdma_ucm_query_route_resp *resp,</div><div class='ctx'> 				 struct rdma_route *route)</div><div class='ctx'> {</div><div class='ctx'> </div><div class='del'>-	resp-&gt;num_paths = route-&gt;num_paths;</div><div class='del'>-	switch (route-&gt;num_paths) {</div><div class='add'>+	resp-&gt;num_paths = route-&gt;num_pri_alt_paths;</div><div class='add'>+	switch (route-&gt;num_pri_alt_paths) {</div><div class='ctx'> 	case 0:</div><div class='ctx'> 		rdma_ip2gid((struct sockaddr *)&amp;route-&gt;addr.dst_addr,</div><div class='ctx'> 			    (union ib_gid *)&amp;resp-&gt;ib_route[0].dgid);</div><div class='hunk'>@@ -808,7 +821,7 @@ static void ucma_copy_iboe_route(struct rdma_ucm_query_route_resp *resp,</div><div class='ctx'> 	case 2:</div><div class='ctx'> 		ib_copy_path_rec_to_user(&amp;resp-&gt;ib_route[1],</div><div class='ctx'> 					 &amp;route-&gt;path_rec[1]);</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case 1:</div><div class='ctx'> 		ib_copy_path_rec_to_user(&amp;resp-&gt;ib_route[0],</div><div class='ctx'> 					 &amp;route-&gt;path_rec[0]);</div><div class='hunk'>@@ -838,7 +851,7 @@ static ssize_t ucma_query_route(struct ucma_file *file,</div><div class='ctx'> 	struct sockaddr *addr;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	if (out_len &lt; sizeof(resp))</div><div class='add'>+	if (out_len &lt; offsetof(struct rdma_ucm_query_route_resp, ibdev_index))</div><div class='ctx'> 		return -ENOSPC;</div><div class='ctx'> </div><div class='ctx'> 	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='hunk'>@@ -848,6 +861,7 @@ static ssize_t ucma_query_route(struct ucma_file *file,</div><div class='ctx'> 	if (IS_ERR(ctx))</div><div class='ctx'> 		return PTR_ERR(ctx);</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	memset(&amp;resp, 0, sizeof resp);</div><div class='ctx'> 	addr = (struct sockaddr *) &amp;ctx-&gt;cm_id-&gt;route.addr.src_addr;</div><div class='ctx'> 	memcpy(&amp;resp.src_addr, addr, addr-&gt;sa_family == AF_INET ?</div><div class='hunk'>@@ -861,6 +875,7 @@ static ssize_t ucma_query_route(struct ucma_file *file,</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='ctx'> 	resp.node_guid = (__force __u64) ctx-&gt;cm_id-&gt;device-&gt;node_guid;</div><div class='add'>+	resp.ibdev_index = ctx-&gt;cm_id-&gt;device-&gt;index;</div><div class='ctx'> 	resp.port_num = ctx-&gt;cm_id-&gt;port_num;</div><div class='ctx'> </div><div class='ctx'> 	if (rdma_cap_ib_sa(ctx-&gt;cm_id-&gt;device, ctx-&gt;cm_id-&gt;port_num))</div><div class='hunk'>@@ -871,8 +886,9 @@ static ssize_t ucma_query_route(struct ucma_file *file,</div><div class='ctx'> 		ucma_copy_iw_route(&amp;resp, &amp;ctx-&gt;cm_id-&gt;route);</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='del'>-	if (copy_to_user(u64_to_user_ptr(cmd.response),</div><div class='del'>-			 &amp;resp, sizeof(resp)))</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='add'>+	if (copy_to_user(u64_to_user_ptr(cmd.response), &amp;resp,</div><div class='add'>+			 min_t(size_t, out_len, sizeof(resp))))</div><div class='ctx'> 		ret = -EFAULT;</div><div class='ctx'> </div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='hunk'>@@ -886,6 +902,7 @@ static void ucma_query_device_addr(struct rdma_cm_id *cm_id,</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='ctx'> 	resp-&gt;node_guid = (__force __u64) cm_id-&gt;device-&gt;node_guid;</div><div class='add'>+	resp-&gt;ibdev_index = cm_id-&gt;device-&gt;index;</div><div class='ctx'> 	resp-&gt;port_num = cm_id-&gt;port_num;</div><div class='ctx'> 	resp-&gt;pkey = (__force __u16) cpu_to_be16(</div><div class='ctx'> 		     ib_addr_get_pkey(&amp;cm_id-&gt;route.addr.dev_addr));</div><div class='hunk'>@@ -898,7 +915,7 @@ static ssize_t ucma_query_addr(struct ucma_context *ctx,</div><div class='ctx'> 	struct sockaddr *addr;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	if (out_len &lt; sizeof(resp))</div><div class='add'>+	if (out_len &lt; offsetof(struct rdma_ucm_query_addr_resp, ibdev_index))</div><div class='ctx'> 		return -ENOSPC;</div><div class='ctx'> </div><div class='ctx'> 	memset(&amp;resp, 0, sizeof resp);</div><div class='hunk'>@@ -913,7 +930,7 @@ static ssize_t ucma_query_addr(struct ucma_context *ctx,</div><div class='ctx'> </div><div class='ctx'> 	ucma_query_device_addr(ctx-&gt;cm_id, &amp;resp);</div><div class='ctx'> </div><div class='del'>-	if (copy_to_user(response, &amp;resp, sizeof(resp)))</div><div class='add'>+	if (copy_to_user(response, &amp;resp, min_t(size_t, out_len, sizeof(resp))))</div><div class='ctx'> 		ret = -EFAULT;</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -932,7 +949,7 @@ static ssize_t ucma_query_path(struct ucma_context *ctx,</div><div class='ctx'> 	if (!resp)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	resp-&gt;num_paths = ctx-&gt;cm_id-&gt;route.num_paths;</div><div class='add'>+	resp-&gt;num_paths = ctx-&gt;cm_id-&gt;route.num_pri_alt_paths;</div><div class='ctx'> 	for (i = 0, out_len -= sizeof(*resp);</div><div class='ctx'> 	     i &lt; resp-&gt;num_paths &amp;&amp; out_len &gt; sizeof(struct ib_path_rec_data);</div><div class='ctx'> 	     i++, out_len -= sizeof(struct ib_path_rec_data)) {</div><div class='hunk'>@@ -951,8 +968,7 @@ static ssize_t ucma_query_path(struct ucma_context *ctx,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (copy_to_user(response, resp,</div><div class='del'>-			 sizeof(*resp) + (i * sizeof(struct ib_path_rec_data))))</div><div class='add'>+	if (copy_to_user(response, resp, struct_size(resp, path_data, i)))</div><div class='ctx'> 		ret = -EFAULT;</div><div class='ctx'> </div><div class='ctx'> 	kfree(resp);</div><div class='hunk'>@@ -966,7 +982,7 @@ static ssize_t ucma_query_gid(struct ucma_context *ctx,</div><div class='ctx'> 	struct sockaddr_ib *addr;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	if (out_len &lt; sizeof(resp))</div><div class='add'>+	if (out_len &lt; offsetof(struct rdma_ucm_query_addr_resp, ibdev_index))</div><div class='ctx'> 		return -ENOSPC;</div><div class='ctx'> </div><div class='ctx'> 	memset(&amp;resp, 0, sizeof resp);</div><div class='hunk'>@@ -999,12 +1015,49 @@ static ssize_t ucma_query_gid(struct ucma_context *ctx,</div><div class='ctx'> 						    &amp;ctx-&gt;cm_id-&gt;route.addr.dst_addr);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (copy_to_user(response, &amp;resp, sizeof(resp)))</div><div class='add'>+	if (copy_to_user(response, &amp;resp, min_t(size_t, out_len, sizeof(resp))))</div><div class='ctx'> 		ret = -EFAULT;</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static ssize_t ucma_query_ib_service(struct ucma_context *ctx,</div><div class='add'>+				     void __user *response, int out_len)</div><div class='add'>+{</div><div class='add'>+	struct rdma_ucm_query_ib_service_resp *resp;</div><div class='add'>+	int n, ret = 0;</div><div class='add'>+</div><div class='add'>+	if (out_len &lt; sizeof(struct rdma_ucm_query_ib_service_resp))</div><div class='add'>+		return -ENOSPC;</div><div class='add'>+</div><div class='add'>+	if (!ctx-&gt;cm_id-&gt;route.service_recs)</div><div class='add'>+		return -ENODATA;</div><div class='add'>+</div><div class='add'>+	resp = kzalloc(out_len, GFP_KERNEL);</div><div class='add'>+	if (!resp)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	resp-&gt;num_service_recs = ctx-&gt;cm_id-&gt;route.num_service_recs;</div><div class='add'>+</div><div class='add'>+	n = (out_len - sizeof(struct rdma_ucm_query_ib_service_resp)) /</div><div class='add'>+		sizeof(struct ib_user_service_rec);</div><div class='add'>+</div><div class='add'>+	if (!n)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	if (n &gt; ctx-&gt;cm_id-&gt;route.num_service_recs)</div><div class='add'>+		n = ctx-&gt;cm_id-&gt;route.num_service_recs;</div><div class='add'>+</div><div class='add'>+	memcpy(resp-&gt;recs, ctx-&gt;cm_id-&gt;route.service_recs,</div><div class='add'>+	       sizeof(*resp-&gt;recs) * n);</div><div class='add'>+	if (copy_to_user(response, resp, struct_size(resp, recs, n)))</div><div class='add'>+		ret = -EFAULT;</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	kfree(resp);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static ssize_t ucma_query(struct ucma_file *file,</div><div class='ctx'> 			  const char __user *inbuf,</div><div class='ctx'> 			  int in_len, int out_len)</div><div class='hunk'>@@ -1022,6 +1075,7 @@ static ssize_t ucma_query(struct ucma_file *file,</div><div class='ctx'> 	if (IS_ERR(ctx))</div><div class='ctx'> 		return PTR_ERR(ctx);</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	switch (cmd.option) {</div><div class='ctx'> 	case RDMA_USER_CM_QUERY_ADDR:</div><div class='ctx'> 		ret = ucma_query_addr(ctx, response, out_len);</div><div class='hunk'>@@ -1032,10 +1086,14 @@ static ssize_t ucma_query(struct ucma_file *file,</div><div class='ctx'> 	case RDMA_USER_CM_QUERY_GID:</div><div class='ctx'> 		ret = ucma_query_gid(ctx, response, out_len);</div><div class='ctx'> 		break;</div><div class='add'>+	case RDMA_USER_CM_QUERY_IB_SERVICE:</div><div class='add'>+		ret = ucma_query_ib_service(ctx, response, out_len);</div><div class='add'>+		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		ret = -ENOSYS;</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='ctx'> </div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -1047,25 +1105,30 @@ static void ucma_copy_conn_param(struct rdma_cm_id *id,</div><div class='ctx'> {</div><div class='ctx'> 	dst-&gt;private_data = src-&gt;private_data;</div><div class='ctx'> 	dst-&gt;private_data_len = src-&gt;private_data_len;</div><div class='del'>-	dst-&gt;responder_resources =src-&gt;responder_resources;</div><div class='add'>+	dst-&gt;responder_resources = src-&gt;responder_resources;</div><div class='ctx'> 	dst-&gt;initiator_depth = src-&gt;initiator_depth;</div><div class='ctx'> 	dst-&gt;flow_control = src-&gt;flow_control;</div><div class='ctx'> 	dst-&gt;retry_count = src-&gt;retry_count;</div><div class='ctx'> 	dst-&gt;rnr_retry_count = src-&gt;rnr_retry_count;</div><div class='ctx'> 	dst-&gt;srq = src-&gt;srq;</div><div class='del'>-	dst-&gt;qp_num = src-&gt;qp_num;</div><div class='add'>+	dst-&gt;qp_num = src-&gt;qp_num &amp; 0xFFFFFF;</div><div class='ctx'> 	dst-&gt;qkey = (id-&gt;route.addr.src_addr.ss_family == AF_IB) ? src-&gt;qkey : 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t ucma_connect(struct ucma_file *file, const char __user *inbuf,</div><div class='ctx'> 			    int in_len, int out_len)</div><div class='ctx'> {</div><div class='del'>-	struct rdma_ucm_connect cmd;</div><div class='ctx'> 	struct rdma_conn_param conn_param;</div><div class='add'>+	struct rdma_ucm_ece ece = {};</div><div class='add'>+	struct rdma_ucm_connect cmd;</div><div class='ctx'> 	struct ucma_context *ctx;</div><div class='add'>+	size_t in_size;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='add'>+	if (in_len &lt; offsetofend(typeof(cmd), reserved))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	in_size = min_t(size_t, in_len, sizeof(cmd));</div><div class='add'>+	if (copy_from_user(&amp;cmd, inbuf, in_size))</div><div class='ctx'> 		return -EFAULT;</div><div class='ctx'> </div><div class='ctx'> 	if (!cmd.conn_param.valid)</div><div class='hunk'>@@ -1076,7 +1139,14 @@ static ssize_t ucma_connect(struct ucma_file *file, const char __user *inbuf,</div><div class='ctx'> 		return PTR_ERR(ctx);</div><div class='ctx'> </div><div class='ctx'> 	ucma_copy_conn_param(ctx-&gt;cm_id, &amp;conn_param, &amp;cmd.conn_param);</div><div class='del'>-	ret = rdma_connect(ctx-&gt;cm_id, &amp;conn_param);</div><div class='add'>+	if (offsetofend(typeof(cmd), ece) &lt;= in_size) {</div><div class='add'>+		ece.vendor_id = cmd.ece.vendor_id;</div><div class='add'>+		ece.attr_mod = cmd.ece.attr_mod;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;ctx-&gt;mutex);</div><div class='add'>+	ret = rdma_connect_ece(ctx-&gt;cm_id, &amp;conn_param, &amp;ece);</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -1095,9 +1165,13 @@ static ssize_t ucma_listen(struct ucma_file *file, const char __user *inbuf,</div><div class='ctx'> 	if (IS_ERR(ctx))</div><div class='ctx'> 		return PTR_ERR(ctx);</div><div class='ctx'> </div><div class='del'>-	ctx-&gt;backlog = cmd.backlog &gt; 0 &amp;&amp; cmd.backlog &lt; max_backlog ?</div><div class='del'>-		       cmd.backlog : max_backlog;</div><div class='del'>-	ret = rdma_listen(ctx-&gt;cm_id, ctx-&gt;backlog);</div><div class='add'>+	if (cmd.backlog &lt;= 0 || cmd.backlog &gt; max_backlog)</div><div class='add'>+		cmd.backlog = max_backlog;</div><div class='add'>+	atomic_set(&amp;ctx-&gt;backlog, cmd.backlog);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;ctx-&gt;mutex);</div><div class='add'>+	ret = rdma_listen(ctx-&gt;cm_id, cmd.backlog);</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -1107,26 +1181,44 @@ static ssize_t ucma_accept(struct ucma_file *file, const char __user *inbuf,</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_ucm_accept cmd;</div><div class='ctx'> 	struct rdma_conn_param conn_param;</div><div class='add'>+	struct rdma_ucm_ece ece = {};</div><div class='ctx'> 	struct ucma_context *ctx;</div><div class='add'>+	size_t in_size;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='add'>+	if (in_len &lt; offsetofend(typeof(cmd), reserved))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	in_size = min_t(size_t, in_len, sizeof(cmd));</div><div class='add'>+	if (copy_from_user(&amp;cmd, inbuf, in_size))</div><div class='ctx'> 		return -EFAULT;</div><div class='ctx'> </div><div class='ctx'> 	ctx = ucma_get_ctx_dev(file, cmd.id);</div><div class='ctx'> 	if (IS_ERR(ctx))</div><div class='ctx'> 		return PTR_ERR(ctx);</div><div class='ctx'> </div><div class='add'>+	if (offsetofend(typeof(cmd), ece) &lt;= in_size) {</div><div class='add'>+		ece.vendor_id = cmd.ece.vendor_id;</div><div class='add'>+		ece.attr_mod = cmd.ece.attr_mod;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	if (cmd.conn_param.valid) {</div><div class='ctx'> 		ucma_copy_conn_param(ctx-&gt;cm_id, &amp;conn_param, &amp;cmd.conn_param);</div><div class='del'>-		mutex_lock(&amp;file-&gt;mut);</div><div class='del'>-		ret = __rdma_accept(ctx-&gt;cm_id, &amp;conn_param, NULL);</div><div class='del'>-		if (!ret)</div><div class='add'>+		mutex_lock(&amp;ctx-&gt;mutex);</div><div class='add'>+		rdma_lock_handler(ctx-&gt;cm_id);</div><div class='add'>+		ret = rdma_accept_ece(ctx-&gt;cm_id, &amp;conn_param, &amp;ece);</div><div class='add'>+		if (!ret) {</div><div class='add'>+			/* The uid must be set atomically with the handler */</div><div class='ctx'> 			ctx-&gt;uid = cmd.uid;</div><div class='del'>-		mutex_unlock(&amp;file-&gt;mut);</div><div class='del'>-	} else</div><div class='del'>-		ret = __rdma_accept(ctx-&gt;cm_id, NULL, NULL);</div><div class='del'>-</div><div class='add'>+		}</div><div class='add'>+		rdma_unlock_handler(ctx-&gt;cm_id);</div><div class='add'>+		mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='add'>+	} else {</div><div class='add'>+		mutex_lock(&amp;ctx-&gt;mutex);</div><div class='add'>+		rdma_lock_handler(ctx-&gt;cm_id);</div><div class='add'>+		ret = rdma_accept_ece(ctx-&gt;cm_id, NULL, &amp;ece);</div><div class='add'>+		rdma_unlock_handler(ctx-&gt;cm_id);</div><div class='add'>+		mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='add'>+	}</div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -1141,11 +1233,25 @@ static ssize_t ucma_reject(struct ucma_file *file, const char __user *inbuf,</div><div class='ctx'> 	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='ctx'> 		return -EFAULT;</div><div class='ctx'> </div><div class='add'>+	if (!cmd.reason)</div><div class='add'>+		cmd.reason = IB_CM_REJ_CONSUMER_DEFINED;</div><div class='add'>+</div><div class='add'>+	switch (cmd.reason) {</div><div class='add'>+	case IB_CM_REJ_CONSUMER_DEFINED:</div><div class='add'>+	case IB_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED:</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	ctx = ucma_get_ctx_dev(file, cmd.id);</div><div class='ctx'> 	if (IS_ERR(ctx))</div><div class='ctx'> 		return PTR_ERR(ctx);</div><div class='ctx'> </div><div class='del'>-	ret = rdma_reject(ctx-&gt;cm_id, cmd.private_data, cmd.private_data_len);</div><div class='add'>+	mutex_lock(&amp;ctx-&gt;mutex);</div><div class='add'>+	ret = rdma_reject(ctx-&gt;cm_id, cmd.private_data, cmd.private_data_len,</div><div class='add'>+			  cmd.reason);</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -1164,7 +1270,9 @@ static ssize_t ucma_disconnect(struct ucma_file *file, const char __user *inbuf,</div><div class='ctx'> 	if (IS_ERR(ctx))</div><div class='ctx'> 		return PTR_ERR(ctx);</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ret = rdma_disconnect(ctx-&gt;cm_id);</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -1195,7 +1303,9 @@ static ssize_t ucma_init_qp_attr(struct ucma_file *file,</div><div class='ctx'> 	resp.qp_attr_mask = 0;</div><div class='ctx'> 	memset(&amp;qp_attr, 0, sizeof qp_attr);</div><div class='ctx'> 	qp_attr.qp_state = cmd.qp_state;</div><div class='add'>+	mutex_lock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ret = rdma_init_qp_attr(ctx-&gt;cm_id, &amp;qp_attr, &amp;resp.qp_attr_mask);</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='hunk'>@@ -1281,9 +1391,13 @@ static int ucma_set_ib_path(struct ucma_context *ctx,</div><div class='ctx'> 		struct sa_path_rec opa;</div><div class='ctx'> </div><div class='ctx'> 		sa_convert_path_ib_to_opa(&amp;opa, &amp;sa_path);</div><div class='add'>+		mutex_lock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 		ret = rdma_set_ib_path(ctx-&gt;cm_id, &amp;opa);</div><div class='add'>+		mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	} else {</div><div class='add'>+		mutex_lock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 		ret = rdma_set_ib_path(ctx-&gt;cm_id, &amp;sa_path);</div><div class='add'>+		mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	}</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='hunk'>@@ -1316,7 +1430,9 @@ static int ucma_set_option_level(struct ucma_context *ctx, int level,</div><div class='ctx'> </div><div class='ctx'> 	switch (level) {</div><div class='ctx'> 	case RDMA_OPTION_ID:</div><div class='add'>+		mutex_lock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 		ret = ucma_set_option_id(ctx, optname, optval, optlen);</div><div class='add'>+		mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case RDMA_OPTION_IB:</div><div class='ctx'> 		ret = ucma_set_option_ib(ctx, optname, optval, optlen);</div><div class='hunk'>@@ -1376,8 +1492,10 @@ static ssize_t ucma_notify(struct ucma_file *file, const char __user *inbuf,</div><div class='ctx'> 	if (IS_ERR(ctx))</div><div class='ctx'> 		return PTR_ERR(ctx);</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	if (ctx-&gt;cm_id-&gt;device)</div><div class='ctx'> 		ret = rdma_notify(ctx-&gt;cm_id, (enum ib_event_type)cmd.event);</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='ctx'> </div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -1411,46 +1529,59 @@ static ssize_t ucma_process_join(struct ucma_file *file,</div><div class='ctx'> 	if (IS_ERR(ctx))</div><div class='ctx'> 		return PTR_ERR(ctx);</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;file-&gt;mut);</div><div class='del'>-	mc = ucma_alloc_multicast(ctx);</div><div class='add'>+	mc = kzalloc(sizeof(*mc), GFP_KERNEL);</div><div class='ctx'> 	if (!mc) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='del'>-		goto err1;</div><div class='add'>+		goto err_put_ctx;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	mc-&gt;ctx = ctx;</div><div class='ctx'> 	mc-&gt;join_state = join_state;</div><div class='ctx'> 	mc-&gt;uid = cmd-&gt;uid;</div><div class='ctx'> 	memcpy(&amp;mc-&gt;addr, addr, cmd-&gt;addr_size);</div><div class='add'>+</div><div class='add'>+	xa_lock(&amp;multicast_table);</div><div class='add'>+	if (__xa_alloc(&amp;multicast_table, &amp;mc-&gt;id, NULL, xa_limit_32b,</div><div class='add'>+		     GFP_KERNEL)) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err_free_mc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	list_add_tail(&amp;mc-&gt;list, &amp;ctx-&gt;mc_list);</div><div class='add'>+	xa_unlock(&amp;multicast_table);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ret = rdma_join_multicast(ctx-&gt;cm_id, (struct sockaddr *)&amp;mc-&gt;addr,</div><div class='ctx'> 				  join_state, mc);</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err2;</div><div class='add'>+		goto err_xa_erase;</div><div class='ctx'> </div><div class='ctx'> 	resp.id = mc-&gt;id;</div><div class='ctx'> 	if (copy_to_user(u64_to_user_ptr(cmd-&gt;response),</div><div class='ctx'> 			 &amp;resp, sizeof(resp))) {</div><div class='ctx'> 		ret = -EFAULT;</div><div class='del'>-		goto err3;</div><div class='add'>+		goto err_leave_multicast;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;mut);</div><div class='del'>-	idr_replace(&amp;multicast_idr, mc, mc-&gt;id);</div><div class='del'>-	mutex_unlock(&amp;mut);</div><div class='add'>+	xa_store(&amp;multicast_table, mc-&gt;id, mc, 0);</div><div class='ctx'> </div><div class='del'>-	mutex_unlock(&amp;file-&gt;mut);</div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-err3:</div><div class='add'>+err_leave_multicast:</div><div class='add'>+	mutex_lock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	rdma_leave_multicast(ctx-&gt;cm_id, (struct sockaddr *) &amp;mc-&gt;addr);</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;mutex);</div><div class='ctx'> 	ucma_cleanup_mc_events(mc);</div><div class='del'>-err2:</div><div class='del'>-	mutex_lock(&amp;mut);</div><div class='del'>-	idr_remove(&amp;multicast_idr, mc-&gt;id);</div><div class='del'>-	mutex_unlock(&amp;mut);</div><div class='add'>+err_xa_erase:</div><div class='add'>+	xa_lock(&amp;multicast_table);</div><div class='ctx'> 	list_del(&amp;mc-&gt;list);</div><div class='add'>+	__xa_erase(&amp;multicast_table, mc-&gt;id);</div><div class='add'>+err_free_mc:</div><div class='add'>+	xa_unlock(&amp;multicast_table);</div><div class='ctx'> 	kfree(mc);</div><div class='del'>-err1:</div><div class='del'>-	mutex_unlock(&amp;file-&gt;mut);</div><div class='add'>+err_put_ctx:</div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -1508,28 +1639,30 @@ static ssize_t ucma_leave_multicast(struct ucma_file *file,</div><div class='ctx'> 	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='ctx'> 		return -EFAULT;</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;mut);</div><div class='del'>-	mc = idr_find(&amp;multicast_idr, cmd.id);</div><div class='add'>+	xa_lock(&amp;multicast_table);</div><div class='add'>+	mc = xa_load(&amp;multicast_table, cmd.id);</div><div class='ctx'> 	if (!mc)</div><div class='ctx'> 		mc = ERR_PTR(-ENOENT);</div><div class='del'>-	else if (mc-&gt;ctx-&gt;file != file)</div><div class='add'>+	else if (READ_ONCE(mc-&gt;ctx-&gt;file) != file)</div><div class='ctx'> 		mc = ERR_PTR(-EINVAL);</div><div class='del'>-	else if (!atomic_inc_not_zero(&amp;mc-&gt;ctx-&gt;ref))</div><div class='add'>+	else if (!refcount_inc_not_zero(&amp;mc-&gt;ctx-&gt;ref))</div><div class='ctx'> 		mc = ERR_PTR(-ENXIO);</div><div class='del'>-	else</div><div class='del'>-		idr_remove(&amp;multicast_idr, mc-&gt;id);</div><div class='del'>-	mutex_unlock(&amp;mut);</div><div class='ctx'> </div><div class='ctx'> 	if (IS_ERR(mc)) {</div><div class='add'>+		xa_unlock(&amp;multicast_table);</div><div class='ctx'> 		ret = PTR_ERR(mc);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	list_del(&amp;mc-&gt;list);</div><div class='add'>+	__xa_erase(&amp;multicast_table, mc-&gt;id);</div><div class='add'>+	xa_unlock(&amp;multicast_table);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;mc-&gt;ctx-&gt;mutex);</div><div class='ctx'> 	rdma_leave_multicast(mc-&gt;ctx-&gt;cm_id, (struct sockaddr *) &amp;mc-&gt;addr);</div><div class='del'>-	mutex_lock(&amp;mc-&gt;ctx-&gt;file-&gt;mut);</div><div class='add'>+	mutex_unlock(&amp;mc-&gt;ctx-&gt;mutex);</div><div class='add'>+</div><div class='ctx'> 	ucma_cleanup_mc_events(mc);</div><div class='del'>-	list_del(&amp;mc-&gt;list);</div><div class='del'>-	mutex_unlock(&amp;mc-&gt;ctx-&gt;file-&gt;mut);</div><div class='ctx'> </div><div class='ctx'> 	ucma_put_ctx(mc-&gt;ctx);</div><div class='ctx'> 	resp.events_reported = mc-&gt;events_reported;</div><div class='hunk'>@@ -1542,46 +1675,15 @@ out:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void ucma_lock_files(struct ucma_file *file1, struct ucma_file *file2)</div><div class='del'>-{</div><div class='del'>-	/* Acquire mutex's based on pointer comparison to prevent deadlock. */</div><div class='del'>-	if (file1 &lt; file2) {</div><div class='del'>-		mutex_lock(&amp;file1-&gt;mut);</div><div class='del'>-		mutex_lock_nested(&amp;file2-&gt;mut, SINGLE_DEPTH_NESTING);</div><div class='del'>-	} else {</div><div class='del'>-		mutex_lock(&amp;file2-&gt;mut);</div><div class='del'>-		mutex_lock_nested(&amp;file1-&gt;mut, SINGLE_DEPTH_NESTING);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void ucma_unlock_files(struct ucma_file *file1, struct ucma_file *file2)</div><div class='del'>-{</div><div class='del'>-	if (file1 &lt; file2) {</div><div class='del'>-		mutex_unlock(&amp;file2-&gt;mut);</div><div class='del'>-		mutex_unlock(&amp;file1-&gt;mut);</div><div class='del'>-	} else {</div><div class='del'>-		mutex_unlock(&amp;file1-&gt;mut);</div><div class='del'>-		mutex_unlock(&amp;file2-&gt;mut);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void ucma_move_events(struct ucma_context *ctx, struct ucma_file *file)</div><div class='del'>-{</div><div class='del'>-	struct ucma_event *uevent, *tmp;</div><div class='del'>-</div><div class='del'>-	list_for_each_entry_safe(uevent, tmp, &amp;ctx-&gt;file-&gt;event_list, list)</div><div class='del'>-		if (uevent-&gt;ctx == ctx)</div><div class='del'>-			list_move_tail(&amp;uevent-&gt;list, &amp;file-&gt;event_list);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static ssize_t ucma_migrate_id(struct ucma_file *new_file,</div><div class='ctx'> 			       const char __user *inbuf,</div><div class='ctx'> 			       int in_len, int out_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_ucm_migrate_id cmd;</div><div class='ctx'> 	struct rdma_ucm_migrate_resp resp;</div><div class='add'>+	struct ucma_event *uevent, *tmp;</div><div class='ctx'> 	struct ucma_context *ctx;</div><div class='del'>-	struct fd f;</div><div class='add'>+	LIST_HEAD(event_list);</div><div class='ctx'> 	struct ucma_file *cur_file;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='hunk'>@@ -1589,50 +1691,106 @@ static ssize_t ucma_migrate_id(struct ucma_file *new_file,</div><div class='ctx'> 		return -EFAULT;</div><div class='ctx'> </div><div class='ctx'> 	/* Get current fd to protect against it being closed */</div><div class='del'>-	f = fdget(cmd.fd);</div><div class='del'>-	if (!f.file)</div><div class='add'>+	CLASS(fd, f)(cmd.fd);</div><div class='add'>+	if (fd_empty(f))</div><div class='ctx'> 		return -ENOENT;</div><div class='del'>-	if (f.file-&gt;f_op != &amp;ucma_fops) {</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-		goto file_put;</div><div class='del'>-	}</div><div class='add'>+	if (fd_file(f)-&gt;f_op != &amp;ucma_fops)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	cur_file = fd_file(f)-&gt;private_data;</div><div class='ctx'> </div><div class='ctx'> 	/* Validate current fd and prevent destruction of id. */</div><div class='del'>-	ctx = ucma_get_ctx(f.file-&gt;private_data, cmd.id);</div><div class='del'>-	if (IS_ERR(ctx)) {</div><div class='del'>-		ret = PTR_ERR(ctx);</div><div class='del'>-		goto file_put;</div><div class='del'>-	}</div><div class='add'>+	ctx = ucma_get_ctx(cur_file, cmd.id);</div><div class='add'>+	if (IS_ERR(ctx))</div><div class='add'>+		return PTR_ERR(ctx);</div><div class='ctx'> </div><div class='del'>-	cur_file = ctx-&gt;file;</div><div class='del'>-	if (cur_file == new_file) {</div><div class='del'>-		resp.events_reported = ctx-&gt;events_reported;</div><div class='del'>-		goto response;</div><div class='add'>+	rdma_lock_handler(ctx-&gt;cm_id);</div><div class='add'>+	/*</div><div class='add'>+	 * ctx-&gt;file can only be changed under the handler &amp; xa_lock. xa_load()</div><div class='add'>+	 * must be checked again to ensure the ctx hasn't begun destruction</div><div class='add'>+	 * since the ucma_get_ctx().</div><div class='add'>+	 */</div><div class='add'>+	xa_lock(&amp;ctx_table);</div><div class='add'>+	if (_ucma_find_context(cmd.id, cur_file) != ctx) {</div><div class='add'>+		xa_unlock(&amp;ctx_table);</div><div class='add'>+		ret = -ENOENT;</div><div class='add'>+		goto err_unlock;</div><div class='ctx'> 	}</div><div class='add'>+	ctx-&gt;file = new_file;</div><div class='add'>+	xa_unlock(&amp;ctx_table);</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;cur_file-&gt;mut);</div><div class='add'>+	list_del(&amp;ctx-&gt;list);</div><div class='ctx'> 	/*</div><div class='del'>-	 * Migrate events between fd's, maintaining order, and avoiding new</div><div class='del'>-	 * events being added before existing events.</div><div class='add'>+	 * At this point lock_handler() prevents addition of new uevents for</div><div class='add'>+	 * this ctx.</div><div class='ctx'> 	 */</div><div class='del'>-	ucma_lock_files(cur_file, new_file);</div><div class='del'>-	mutex_lock(&amp;mut);</div><div class='del'>-</div><div class='del'>-	list_move_tail(&amp;ctx-&gt;list, &amp;new_file-&gt;ctx_list);</div><div class='del'>-	ucma_move_events(ctx, new_file);</div><div class='del'>-	ctx-&gt;file = new_file;</div><div class='add'>+	list_for_each_entry_safe(uevent, tmp, &amp;cur_file-&gt;event_list, list)</div><div class='add'>+		if (uevent-&gt;ctx == ctx)</div><div class='add'>+			list_move_tail(&amp;uevent-&gt;list, &amp;event_list);</div><div class='ctx'> 	resp.events_reported = ctx-&gt;events_reported;</div><div class='add'>+	mutex_unlock(&amp;cur_file-&gt;mut);</div><div class='ctx'> </div><div class='del'>-	mutex_unlock(&amp;mut);</div><div class='del'>-	ucma_unlock_files(cur_file, new_file);</div><div class='add'>+	mutex_lock(&amp;new_file-&gt;mut);</div><div class='add'>+	list_add_tail(&amp;ctx-&gt;list, &amp;new_file-&gt;ctx_list);</div><div class='add'>+	list_splice_tail(&amp;event_list, &amp;new_file-&gt;event_list);</div><div class='add'>+	mutex_unlock(&amp;new_file-&gt;mut);</div><div class='ctx'> </div><div class='del'>-response:</div><div class='ctx'> 	if (copy_to_user(u64_to_user_ptr(cmd.response),</div><div class='ctx'> 			 &amp;resp, sizeof(resp)))</div><div class='ctx'> 		ret = -EFAULT;</div><div class='ctx'> </div><div class='add'>+err_unlock:</div><div class='add'>+	rdma_unlock_handler(ctx-&gt;cm_id);</div><div class='add'>+	ucma_put_ctx(ctx);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static ssize_t ucma_write_cm_event(struct ucma_file *file,</div><div class='add'>+				   const char __user *inbuf, int in_len,</div><div class='add'>+				   int out_len)</div><div class='add'>+{</div><div class='add'>+	struct rdma_ucm_write_cm_event cmd;</div><div class='add'>+	struct rdma_cm_event event = {};</div><div class='add'>+	struct ucma_event *uevent;</div><div class='add'>+	struct ucma_context *ctx;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	if (copy_from_user(&amp;cmd, inbuf, sizeof(cmd)))</div><div class='add'>+		return -EFAULT;</div><div class='add'>+</div><div class='add'>+	if ((cmd.event != RDMA_CM_EVENT_USER) &amp;&amp;</div><div class='add'>+	    (cmd.event != RDMA_CM_EVENT_INTERNAL))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ctx = ucma_get_ctx(file, cmd.id);</div><div class='add'>+	if (IS_ERR(ctx))</div><div class='add'>+		return PTR_ERR(ctx);</div><div class='add'>+</div><div class='add'>+	event.event = cmd.event;</div><div class='add'>+	event.status = cmd.status;</div><div class='add'>+	event.param.arg = cmd.param.arg;</div><div class='add'>+</div><div class='add'>+	uevent = kzalloc(sizeof(*uevent), GFP_KERNEL);</div><div class='add'>+	if (!uevent) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	uevent-&gt;ctx = ctx;</div><div class='add'>+	uevent-&gt;resp.uid = ctx-&gt;uid;</div><div class='add'>+	uevent-&gt;resp.id = ctx-&gt;id;</div><div class='add'>+	uevent-&gt;resp.event = event.event;</div><div class='add'>+	uevent-&gt;resp.status = event.status;</div><div class='add'>+	memcpy(uevent-&gt;resp.param.arg32, &amp;event.param.arg,</div><div class='add'>+	       sizeof(event.param.arg));</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;ctx-&gt;file-&gt;mut);</div><div class='add'>+	list_add_tail(&amp;uevent-&gt;list, &amp;ctx-&gt;file-&gt;event_list);</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;file-&gt;mut);</div><div class='add'>+	wake_up_interruptible(&amp;ctx-&gt;file-&gt;poll_wait);</div><div class='add'>+</div><div class='add'>+out:</div><div class='ctx'> 	ucma_put_ctx(ctx);</div><div class='del'>-file_put:</div><div class='del'>-	fdput(f);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1661,7 +1819,9 @@ static ssize_t (*ucma_cmd_table[])(struct ucma_file *file,</div><div class='ctx'> 	[RDMA_USER_CM_CMD_QUERY]	 = ucma_query,</div><div class='ctx'> 	[RDMA_USER_CM_CMD_BIND]		 = ucma_bind,</div><div class='ctx'> 	[RDMA_USER_CM_CMD_RESOLVE_ADDR]	 = ucma_resolve_addr,</div><div class='del'>-	[RDMA_USER_CM_CMD_JOIN_MCAST]	 = ucma_join_multicast</div><div class='add'>+	[RDMA_USER_CM_CMD_JOIN_MCAST]	 = ucma_join_multicast,</div><div class='add'>+	[RDMA_USER_CM_CMD_RESOLVE_IB_SERVICE] = ucma_resolve_ib_service,</div><div class='add'>+	[RDMA_USER_CM_CMD_WRITE_CM_EVENT] = ucma_write_cm_event,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static ssize_t ucma_write(struct file *filp, const char __user *buf,</div><div class='hunk'>@@ -1672,8 +1832,8 @@ static ssize_t ucma_write(struct file *filp, const char __user *buf,</div><div class='ctx'> 	ssize_t ret;</div><div class='ctx'> </div><div class='ctx'> 	if (!ib_safe_file_access(filp)) {</div><div class='del'>-		pr_err_once("ucma_write: process %d (%s) changed security contexts after opening file descriptor, this is not allowed.\n",</div><div class='del'>-			    task_tgid_vnr(current), current-&gt;comm);</div><div class='add'>+		pr_err_once("%s: process %d (%s) changed security contexts after opening file descriptor, this is not allowed.\n",</div><div class='add'>+			    __func__, task_tgid_vnr(current), current-&gt;comm);</div><div class='ctx'> 		return -EACCES;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -1729,13 +1889,6 @@ static int ucma_open(struct inode *inode, struct file *filp)</div><div class='ctx'> 	if (!file)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	file-&gt;close_wq = alloc_ordered_workqueue("ucma_close_id",</div><div class='del'>-						 WQ_MEM_RECLAIM);</div><div class='del'>-	if (!file-&gt;close_wq) {</div><div class='del'>-		kfree(file);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;file-&gt;event_list);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;file-&gt;ctx_list);</div><div class='ctx'> 	init_waitqueue_head(&amp;file-&gt;poll_wait);</div><div class='hunk'>@@ -1750,40 +1903,23 @@ static int ucma_open(struct inode *inode, struct file *filp)</div><div class='ctx'> static int ucma_close(struct inode *inode, struct file *filp)</div><div class='ctx'> {</div><div class='ctx'> 	struct ucma_file *file = filp-&gt;private_data;</div><div class='del'>-	struct ucma_context *ctx, *tmp;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;file-&gt;mut);</div><div class='del'>-	list_for_each_entry_safe(ctx, tmp, &amp;file-&gt;ctx_list, list) {</div><div class='del'>-		ctx-&gt;destroying = 1;</div><div class='del'>-		mutex_unlock(&amp;file-&gt;mut);</div><div class='del'>-</div><div class='del'>-		mutex_lock(&amp;mut);</div><div class='del'>-		idr_remove(&amp;ctx_idr, ctx-&gt;id);</div><div class='del'>-		mutex_unlock(&amp;mut);</div><div class='ctx'> </div><div class='del'>-		flush_workqueue(file-&gt;close_wq);</div><div class='del'>-		/* At that step once ctx was marked as destroying and workqueue</div><div class='del'>-		 * was flushed we are safe from any inflights handlers that</div><div class='del'>-		 * might put other closing task.</div><div class='del'>-		 */</div><div class='del'>-		mutex_lock(&amp;mut);</div><div class='del'>-		if (!ctx-&gt;closing) {</div><div class='del'>-			mutex_unlock(&amp;mut);</div><div class='del'>-			ucma_put_ctx(ctx);</div><div class='del'>-			wait_for_completion(&amp;ctx-&gt;comp);</div><div class='del'>-			/* rdma_destroy_id ensures that no event handlers are</div><div class='del'>-			 * inflight for that id before releasing it.</div><div class='del'>-			 */</div><div class='del'>-			rdma_destroy_id(ctx-&gt;cm_id);</div><div class='del'>-		} else {</div><div class='del'>-			mutex_unlock(&amp;mut);</div><div class='del'>-		}</div><div class='add'>+	/*</div><div class='add'>+	 * All paths that touch ctx_list or ctx_list starting from write() are</div><div class='add'>+	 * prevented by this being a FD release function. The list_add_tail() in</div><div class='add'>+	 * ucma_connect_event_handler() can run concurrently, however it only</div><div class='add'>+	 * adds to the list *after* a listening ID. By only reading the first of</div><div class='add'>+	 * the list, and relying on ucma_destroy_private_ctx() to block</div><div class='add'>+	 * ucma_connect_event_handler(), no additional locking is needed.</div><div class='add'>+	 */</div><div class='add'>+	while (!list_empty(&amp;file-&gt;ctx_list)) {</div><div class='add'>+		struct ucma_context *ctx = list_first_entry(</div><div class='add'>+			&amp;file-&gt;ctx_list, struct ucma_context, list);</div><div class='ctx'> </div><div class='del'>-		ucma_free_ctx(ctx);</div><div class='del'>-		mutex_lock(&amp;file-&gt;mut);</div><div class='add'>+		WARN_ON(xa_cmpxchg(&amp;ctx_table, ctx-&gt;id, ctx, XA_ZERO_ENTRY,</div><div class='add'>+				   GFP_KERNEL) != ctx);</div><div class='add'>+		ucma_destroy_private_ctx(ctx);</div><div class='ctx'> 	}</div><div class='del'>-	mutex_unlock(&amp;file-&gt;mut);</div><div class='del'>-	destroy_workqueue(file-&gt;close_wq);</div><div class='ctx'> 	kfree(file);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -1794,7 +1930,6 @@ static const struct file_operations ucma_fops = {</div><div class='ctx'> 	.release = ucma_close,</div><div class='ctx'> 	.write	 = ucma_write,</div><div class='ctx'> 	.poll    = ucma_poll,</div><div class='del'>-	.llseek	 = no_llseek,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static struct miscdevice ucma_misc = {</div><div class='hunk'>@@ -1805,13 +1940,25 @@ static struct miscdevice ucma_misc = {</div><div class='ctx'> 	.fops		= &amp;ucma_fops,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static ssize_t show_abi_version(struct device *dev,</div><div class='del'>-				struct device_attribute *attr,</div><div class='del'>-				char *buf)</div><div class='add'>+static int ucma_get_global_nl_info(struct ib_client_nl_info *res)</div><div class='ctx'> {</div><div class='del'>-	return sprintf(buf, "%d\n", RDMA_USER_CM_ABI_VERSION);</div><div class='add'>+	res-&gt;abi = RDMA_USER_CM_ABI_VERSION;</div><div class='add'>+	res-&gt;cdev = ucma_misc.this_device;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='del'>-static DEVICE_ATTR(abi_version, S_IRUGO, show_abi_version, NULL);</div><div class='add'>+</div><div class='add'>+static struct ib_client rdma_cma_client = {</div><div class='add'>+	.name = "rdma_cm",</div><div class='add'>+	.get_global_nl_info = ucma_get_global_nl_info,</div><div class='add'>+};</div><div class='add'>+MODULE_ALIAS_RDMA_CLIENT("rdma_cm");</div><div class='add'>+</div><div class='add'>+static ssize_t abi_version_show(struct device *dev,</div><div class='add'>+				struct device_attribute *attr, char *buf)</div><div class='add'>+{</div><div class='add'>+	return sysfs_emit(buf, "%d\n", RDMA_USER_CM_ABI_VERSION);</div><div class='add'>+}</div><div class='add'>+static DEVICE_ATTR_RO(abi_version);</div><div class='ctx'> </div><div class='ctx'> static int __init ucma_init(void)</div><div class='ctx'> {</div><div class='hunk'>@@ -1833,7 +1980,14 @@ static int __init ucma_init(void)</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='ctx'> 		goto err2;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	ret = ib_register_client(&amp;rdma_cma_client);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err3;</div><div class='add'>+</div><div class='ctx'> 	return 0;</div><div class='add'>+err3:</div><div class='add'>+	unregister_net_sysctl_table(ucma_ctl_table_hdr);</div><div class='ctx'> err2:</div><div class='ctx'> 	device_remove_file(ucma_misc.this_device, &amp;dev_attr_abi_version);</div><div class='ctx'> err1:</div><div class='hunk'>@@ -1843,11 +1997,10 @@ err1:</div><div class='ctx'> </div><div class='ctx'> static void __exit ucma_cleanup(void)</div><div class='ctx'> {</div><div class='add'>+	ib_unregister_client(&amp;rdma_cma_client);</div><div class='ctx'> 	unregister_net_sysctl_table(ucma_ctl_table_hdr);</div><div class='ctx'> 	device_remove_file(ucma_misc.this_device, &amp;dev_attr_abi_version);</div><div class='ctx'> 	misc_deregister(&amp;ucma_misc);</div><div class='del'>-	idr_destroy(&amp;ctx_idr);</div><div class='del'>-	idr_destroy(&amp;multicast_idr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> module_init(ucma_init);</div><div class='head'>diff --git a/drivers/infiniband/core/ud_header.c b/drivers/infiniband/core/ud_header.c<br/>index 29a45d2f8898..8d3dfef9ebaa 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/ud_header.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/ud_header.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/ud_header.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/ud_header.c</a></div><div class='hunk'>@@ -41,7 +41,7 @@</div><div class='ctx'> </div><div class='ctx'> #define STRUCT_FIELD(header, field) \</div><div class='ctx'> 	.struct_offset_bytes = offsetof(struct ib_unpacked_ ## header, field),      \</div><div class='del'>-	.struct_size_bytes   = sizeof ((struct ib_unpacked_ ## header *) 0)-&gt;field, \</div><div class='add'>+	.struct_size_bytes   = sizeof_field(struct ib_unpacked_ ## header, field), \</div><div class='ctx'> 	.field_name          = #header ":" #field</div><div class='ctx'> </div><div class='ctx'> static const struct ib_field lrh_table[]  = {</div><div class='hunk'>@@ -462,86 +462,3 @@ int ib_ud_header_pack(struct ib_ud_header *header,</div><div class='ctx'> 	return len;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_ud_header_pack);</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * ib_ud_header_unpack - Unpack UD header struct from wire format</div><div class='del'>- * @header:UD header struct</div><div class='del'>- * @buf:Buffer to pack into</div><div class='del'>- *</div><div class='del'>- * ib_ud_header_pack() unpacks the UD header structure @header from wire</div><div class='del'>- * format in the buffer @buf.</div><div class='del'>- */</div><div class='del'>-int ib_ud_header_unpack(void                *buf,</div><div class='del'>-			struct ib_ud_header *header)</div><div class='del'>-{</div><div class='del'>-	ib_unpack(lrh_table, ARRAY_SIZE(lrh_table),</div><div class='del'>-		  buf, &amp;header-&gt;lrh);</div><div class='del'>-	buf += IB_LRH_BYTES;</div><div class='del'>-</div><div class='del'>-	if (header-&gt;lrh.link_version != 0) {</div><div class='del'>-		pr_warn("Invalid LRH.link_version %d\n",</div><div class='del'>-			header-&gt;lrh.link_version);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	switch (header-&gt;lrh.link_next_header) {</div><div class='del'>-	case IB_LNH_IBA_LOCAL:</div><div class='del'>-		header-&gt;grh_present = 0;</div><div class='del'>-		break;</div><div class='del'>-</div><div class='del'>-	case IB_LNH_IBA_GLOBAL:</div><div class='del'>-		header-&gt;grh_present = 1;</div><div class='del'>-		ib_unpack(grh_table, ARRAY_SIZE(grh_table),</div><div class='del'>-			  buf, &amp;header-&gt;grh);</div><div class='del'>-		buf += IB_GRH_BYTES;</div><div class='del'>-</div><div class='del'>-		if (header-&gt;grh.ip_version != 6) {</div><div class='del'>-			pr_warn("Invalid GRH.ip_version %d\n",</div><div class='del'>-				header-&gt;grh.ip_version);</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-		if (header-&gt;grh.next_header != 0x1b) {</div><div class='del'>-			pr_warn("Invalid GRH.next_header 0x%02x\n",</div><div class='del'>-				header-&gt;grh.next_header);</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-</div><div class='del'>-	default:</div><div class='del'>-		pr_warn("Invalid LRH.link_next_header %d\n",</div><div class='del'>-			header-&gt;lrh.link_next_header);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ib_unpack(bth_table, ARRAY_SIZE(bth_table),</div><div class='del'>-		  buf, &amp;header-&gt;bth);</div><div class='del'>-	buf += IB_BTH_BYTES;</div><div class='del'>-</div><div class='del'>-	switch (header-&gt;bth.opcode) {</div><div class='del'>-	case IB_OPCODE_UD_SEND_ONLY:</div><div class='del'>-		header-&gt;immediate_present = 0;</div><div class='del'>-		break;</div><div class='del'>-	case IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE:</div><div class='del'>-		header-&gt;immediate_present = 1;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		pr_warn("Invalid BTH.opcode 0x%02x\n", header-&gt;bth.opcode);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (header-&gt;bth.transport_header_version != 0) {</div><div class='del'>-		pr_warn("Invalid BTH.transport_header_version %d\n",</div><div class='del'>-			header-&gt;bth.transport_header_version);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ib_unpack(deth_table, ARRAY_SIZE(deth_table),</div><div class='del'>-		  buf, &amp;header-&gt;deth);</div><div class='del'>-	buf += IB_DETH_BYTES;</div><div class='del'>-</div><div class='del'>-	if (header-&gt;immediate_present)</div><div class='del'>-		memcpy(&amp;header-&gt;immediate_data, buf, sizeof header-&gt;immediate_data);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_ud_header_unpack);</div><div class='head'>diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c<br/>index e7ea819fcb11..8137031c2a65 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/umem.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/umem.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/umem.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/umem.c</a></div><div class='hunk'>@@ -2,6 +2,7 @@</div><div class='ctx'>  * Copyright (c) 2005 Topspin Communications.  All rights reserved.</div><div class='ctx'>  * Copyright (c) 2005 Cisco Systems.  All rights reserved.</div><div class='ctx'>  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.</div><div class='add'>+ * Copyright (c) 2020 Intel Corporation. All rights reserved.</div><div class='ctx'>  *</div><div class='ctx'>  * This software is available to you under a choice of one of two</div><div class='ctx'>  * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='hunk'>@@ -39,94 +40,32 @@</div><div class='ctx'> #include &lt;linux/export.h&gt;</div><div class='ctx'> #include &lt;linux/slab.h&gt;</div><div class='ctx'> #include &lt;linux/pagemap.h&gt;</div><div class='add'>+#include &lt;linux/count_zeros.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_umem_odp.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "uverbs.h"</div><div class='ctx'> </div><div class='del'>-static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dirty)</div><div class='del'>-{</div><div class='del'>-	struct sg_page_iter sg_iter;</div><div class='del'>-	struct page *page;</div><div class='del'>-</div><div class='del'>-	if (umem-&gt;nmap &gt; 0)</div><div class='del'>-		ib_dma_unmap_sg(dev, umem-&gt;sg_head.sgl, umem-&gt;sg_nents,</div><div class='del'>-				DMA_BIDIRECTIONAL);</div><div class='del'>-</div><div class='del'>-	for_each_sg_page(umem-&gt;sg_head.sgl, &amp;sg_iter, umem-&gt;sg_nents, 0) {</div><div class='del'>-		page = sg_page_iter_page(&amp;sg_iter);</div><div class='del'>-		if (!PageDirty(page) &amp;&amp; umem-&gt;writable &amp;&amp; dirty)</div><div class='del'>-			set_page_dirty_lock(page);</div><div class='del'>-		put_page(page);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	sg_free_table(&amp;umem-&gt;sg_head);</div><div class='del'>-}</div><div class='add'>+#define RESCHED_LOOP_CNT_THRESHOLD 0x1000</div><div class='ctx'> </div><div class='del'>-/* ib_umem_add_sg_table - Add N contiguous pages to scatter table</div><div class='del'>- *</div><div class='del'>- * sg: current scatterlist entry</div><div class='del'>- * page_list: array of npage struct page pointers</div><div class='del'>- * npages: number of pages in page_list</div><div class='del'>- * max_seg_sz: maximum segment size in bytes</div><div class='del'>- * nents: [out] number of entries in the scatterlist</div><div class='del'>- *</div><div class='del'>- * Return new end of scatterlist</div><div class='del'>- */</div><div class='del'>-static struct scatterlist *ib_umem_add_sg_table(struct scatterlist *sg,</div><div class='del'>-						struct page **page_list,</div><div class='del'>-						unsigned long npages,</div><div class='del'>-						unsigned int max_seg_sz,</div><div class='del'>-						int *nents)</div><div class='add'>+static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dirty)</div><div class='ctx'> {</div><div class='del'>-	unsigned long first_pfn;</div><div class='del'>-	unsigned long i = 0;</div><div class='del'>-	bool update_cur_sg = false;</div><div class='del'>-	bool first = !sg_page(sg);</div><div class='del'>-</div><div class='del'>-	/* Check if new page_list is contiguous with end of previous page_list.</div><div class='del'>-	 * sg-&gt;length here is a multiple of PAGE_SIZE and sg-&gt;offset is 0.</div><div class='del'>-	 */</div><div class='del'>-	if (!first &amp;&amp; (page_to_pfn(sg_page(sg)) + (sg-&gt;length &gt;&gt; PAGE_SHIFT) ==</div><div class='del'>-		       page_to_pfn(page_list[0])))</div><div class='del'>-		update_cur_sg = true;</div><div class='del'>-</div><div class='del'>-	while (i != npages) {</div><div class='del'>-		unsigned long len;</div><div class='del'>-		struct page *first_page = page_list[i];</div><div class='add'>+	bool make_dirty = umem-&gt;writable &amp;&amp; dirty;</div><div class='add'>+	struct scatterlist *sg;</div><div class='add'>+	unsigned int i;</div><div class='ctx'> </div><div class='del'>-		first_pfn = page_to_pfn(first_page);</div><div class='add'>+	if (dirty)</div><div class='add'>+		ib_dma_unmap_sgtable_attrs(dev, &amp;umem-&gt;sgt_append.sgt,</div><div class='add'>+					   DMA_BIDIRECTIONAL, 0);</div><div class='ctx'> </div><div class='del'>-		/* Compute the number of contiguous pages we have starting</div><div class='del'>-		 * at i</div><div class='del'>-		 */</div><div class='del'>-		for (len = 0; i != npages &amp;&amp;</div><div class='del'>-			      first_pfn + len == page_to_pfn(page_list[i]) &amp;&amp;</div><div class='del'>-			      len &lt; (max_seg_sz &gt;&gt; PAGE_SHIFT);</div><div class='del'>-		     len++)</div><div class='del'>-			i++;</div><div class='del'>-</div><div class='del'>-		/* Squash N contiguous pages from page_list into current sge */</div><div class='del'>-		if (update_cur_sg) {</div><div class='del'>-			if ((max_seg_sz - sg-&gt;length) &gt;= (len &lt;&lt; PAGE_SHIFT)) {</div><div class='del'>-				sg_set_page(sg, sg_page(sg),</div><div class='del'>-					    sg-&gt;length + (len &lt;&lt; PAGE_SHIFT),</div><div class='del'>-					    0);</div><div class='del'>-				update_cur_sg = false;</div><div class='del'>-				continue;</div><div class='del'>-			}</div><div class='del'>-			update_cur_sg = false;</div><div class='del'>-		}</div><div class='add'>+	for_each_sgtable_sg(&amp;umem-&gt;sgt_append.sgt, sg, i) {</div><div class='add'>+		unpin_user_page_range_dirty_lock(sg_page(sg),</div><div class='add'>+			DIV_ROUND_UP(sg-&gt;length, PAGE_SIZE), make_dirty);</div><div class='ctx'> </div><div class='del'>-		/* Squash N contiguous pages into next sge or first sge */</div><div class='del'>-		if (!first)</div><div class='del'>-			sg = sg_next(sg);</div><div class='del'>-</div><div class='del'>-		(*nents)++;</div><div class='del'>-		sg_set_page(sg, first_page, len &lt;&lt; PAGE_SHIFT, 0);</div><div class='del'>-		first = false;</div><div class='add'>+		if (i &amp;&amp; !(i % RESCHED_LOOP_CNT_THRESHOLD))</div><div class='add'>+			cond_resched();</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return sg;</div><div class='add'>+	sg_free_append_table(&amp;umem-&gt;sgt_append);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='hunk'>@@ -147,77 +86,94 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,</div><div class='ctx'> 				     unsigned long pgsz_bitmap,</div><div class='ctx'> 				     unsigned long virt)</div><div class='ctx'> {</div><div class='del'>-	struct scatterlist *sg;</div><div class='del'>-	unsigned int best_pg_bit;</div><div class='add'>+	unsigned long curr_len = 0;</div><div class='add'>+	dma_addr_t curr_base = ~0;</div><div class='ctx'> 	unsigned long va, pgoff;</div><div class='add'>+	struct scatterlist *sg;</div><div class='ctx'> 	dma_addr_t mask;</div><div class='add'>+	dma_addr_t end;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	/* At minimum, drivers must support PAGE_SIZE or smaller */</div><div class='del'>-	if (WARN_ON(!(pgsz_bitmap &amp; GENMASK(PAGE_SHIFT, 0))))</div><div class='del'>-		return 0;</div><div class='add'>+	umem-&gt;iova = va = virt;</div><div class='ctx'> </div><div class='del'>-	va = virt;</div><div class='del'>-	/* max page size not to exceed MR length */</div><div class='del'>-	mask = roundup_pow_of_two(umem-&gt;length);</div><div class='add'>+	if (umem-&gt;is_odp) {</div><div class='add'>+		unsigned int page_size = BIT(to_ib_umem_odp(umem)-&gt;page_shift);</div><div class='add'>+</div><div class='add'>+		/* ODP must always be self consistent. */</div><div class='add'>+		if (!(pgsz_bitmap &amp; page_size))</div><div class='add'>+			return 0;</div><div class='add'>+		return page_size;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* The best result is the smallest page size that results in the minimum</div><div class='add'>+	 * number of required pages. Compute the largest page size that could</div><div class='add'>+	 * work based on VA address bits that don't change.</div><div class='add'>+	 */</div><div class='add'>+	mask = pgsz_bitmap &amp;</div><div class='add'>+	       GENMASK(BITS_PER_LONG - 1,</div><div class='add'>+		       bits_per((umem-&gt;length - 1 + virt) ^ virt));</div><div class='ctx'> 	/* offset into first SGL */</div><div class='ctx'> 	pgoff = umem-&gt;address &amp; ~PAGE_MASK;</div><div class='ctx'> </div><div class='del'>-	for_each_sg(umem-&gt;sg_head.sgl, sg, umem-&gt;nmap, i) {</div><div class='del'>-		/* Walk SGL and reduce max page size if VA/PA bits differ</div><div class='del'>-		 * for any address.</div><div class='add'>+	for_each_sgtable_dma_sg(&amp;umem-&gt;sgt_append.sgt, sg, i) {</div><div class='add'>+		/* If the current entry is physically contiguous with the previous</div><div class='add'>+		 * one, no need to take its start addresses into consideration.</div><div class='ctx'> 		 */</div><div class='del'>-		mask |= (sg_dma_address(sg) + pgoff) ^ va;</div><div class='del'>-		if (i &amp;&amp; i != (umem-&gt;nmap - 1))</div><div class='del'>-			/* restrict by length as well for interior SGEs */</div><div class='del'>-			mask |= sg_dma_len(sg);</div><div class='add'>+		if (check_add_overflow(curr_base, curr_len, &amp;end) ||</div><div class='add'>+		    end != sg_dma_address(sg)) {</div><div class='add'>+</div><div class='add'>+			curr_base = sg_dma_address(sg);</div><div class='add'>+			curr_len = 0;</div><div class='add'>+</div><div class='add'>+			/* Reduce max page size if VA/PA bits differ */</div><div class='add'>+			mask |= (curr_base + pgoff) ^ va;</div><div class='add'>+</div><div class='add'>+			/* The alignment of any VA matching a discontinuity point</div><div class='add'>+			* in the physical memory sets the maximum possible page</div><div class='add'>+			* size as this must be a starting point of a new page that</div><div class='add'>+			* needs to be aligned.</div><div class='add'>+			*/</div><div class='add'>+			if (i != 0)</div><div class='add'>+				mask |= va;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		curr_len += sg_dma_len(sg);</div><div class='ctx'> 		va += sg_dma_len(sg) - pgoff;</div><div class='add'>+</div><div class='ctx'> 		pgoff = 0;</div><div class='ctx'> 	}</div><div class='del'>-	best_pg_bit = rdma_find_pg_bit(mask, pgsz_bitmap);</div><div class='ctx'> </div><div class='del'>-	return BIT_ULL(best_pg_bit);</div><div class='add'>+	/* The mask accumulates 1's in each position where the VA and physical</div><div class='add'>+	 * address differ, thus the length of trailing 0 is the largest page</div><div class='add'>+	 * size that can pass the VA through to the physical.</div><div class='add'>+	 */</div><div class='add'>+	if (mask)</div><div class='add'>+		pgsz_bitmap &amp;= GENMASK(count_trailing_zeros(mask), 0);</div><div class='add'>+	return pgsz_bitmap ? rounddown_pow_of_two(pgsz_bitmap) : 0;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_umem_find_best_pgsz);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * ib_umem_get - Pin and DMA map userspace memory.</div><div class='ctx'>  *</div><div class='del'>- * If access flags indicate ODP memory, avoid pinning. Instead, stores</div><div class='del'>- * the mm for future page fault handling in conjunction with MMU notifiers.</div><div class='del'>- *</div><div class='del'>- * @udata: userspace context to pin memory for</div><div class='add'>+ * @device: IB device to connect UMEM</div><div class='ctx'>  * @addr: userspace virtual address to start at</div><div class='ctx'>  * @size: length of region to pin</div><div class='ctx'>  * @access: IB_ACCESS_xxx flags for memory being pinned</div><div class='del'>- * @dmasync: flush in-flight DMA when the memory region is written</div><div class='ctx'>  */</div><div class='del'>-struct ib_umem *ib_umem_get(struct ib_udata *udata, unsigned long addr,</div><div class='del'>-			    size_t size, int access, int dmasync)</div><div class='add'>+struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr,</div><div class='add'>+			    size_t size, int access)</div><div class='ctx'> {</div><div class='del'>-	struct ib_ucontext *context;</div><div class='ctx'> 	struct ib_umem *umem;</div><div class='ctx'> 	struct page **page_list;</div><div class='ctx'> 	unsigned long lock_limit;</div><div class='ctx'> 	unsigned long new_pinned;</div><div class='ctx'> 	unsigned long cur_base;</div><div class='add'>+	unsigned long dma_attr = 0;</div><div class='ctx'> 	struct mm_struct *mm;</div><div class='ctx'> 	unsigned long npages;</div><div class='del'>-	int ret;</div><div class='del'>-	unsigned long dma_attrs = 0;</div><div class='del'>-	struct scatterlist *sg;</div><div class='del'>-	unsigned int gup_flags = FOLL_WRITE;</div><div class='del'>-</div><div class='del'>-	if (!udata)</div><div class='del'>-		return ERR_PTR(-EIO);</div><div class='del'>-</div><div class='del'>-	context = container_of(udata, struct uverbs_attr_bundle, driver_udata)</div><div class='del'>-			  -&gt;context;</div><div class='del'>-	if (!context)</div><div class='del'>-		return ERR_PTR(-EIO);</div><div class='del'>-</div><div class='del'>-	if (dmasync)</div><div class='del'>-		dma_attrs |= DMA_ATTR_WRITE_BARRIER;</div><div class='add'>+	int pinned, ret;</div><div class='add'>+	unsigned int gup_flags = FOLL_LONGTERM;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * If the combination of the addr and size requested for this memory</div><div class='hunk'>@@ -230,37 +186,24 @@ struct ib_umem *ib_umem_get(struct ib_udata *udata, unsigned long addr,</div><div class='ctx'> 	if (!can_do_mlock())</div><div class='ctx'> 		return ERR_PTR(-EPERM);</div><div class='ctx'> </div><div class='del'>-	if (access &amp; IB_ACCESS_ON_DEMAND) {</div><div class='del'>-		umem = kzalloc(sizeof(struct ib_umem_odp), GFP_KERNEL);</div><div class='del'>-		if (!umem)</div><div class='del'>-			return ERR_PTR(-ENOMEM);</div><div class='del'>-		umem-&gt;is_odp = 1;</div><div class='del'>-	} else {</div><div class='del'>-		umem = kzalloc(sizeof(*umem), GFP_KERNEL);</div><div class='del'>-		if (!umem)</div><div class='del'>-			return ERR_PTR(-ENOMEM);</div><div class='del'>-	}</div><div class='add'>+	if (access &amp; IB_ACCESS_ON_DEMAND)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='ctx'> </div><div class='del'>-	umem-&gt;context    = context;</div><div class='add'>+	umem = kzalloc(sizeof(*umem), GFP_KERNEL);</div><div class='add'>+	if (!umem)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+	umem-&gt;ibdev      = device;</div><div class='ctx'> 	umem-&gt;length     = size;</div><div class='ctx'> 	umem-&gt;address    = addr;</div><div class='del'>-	umem-&gt;page_shift = PAGE_SHIFT;</div><div class='add'>+	/*</div><div class='add'>+	 * Drivers should call ib_umem_find_best_pgsz() to set the iova</div><div class='add'>+	 * correctly.</div><div class='add'>+	 */</div><div class='add'>+	umem-&gt;iova = addr;</div><div class='ctx'> 	umem-&gt;writable   = ib_access_writable(access);</div><div class='ctx'> 	umem-&gt;owning_mm = mm = current-&gt;mm;</div><div class='ctx'> 	mmgrab(mm);</div><div class='ctx'> </div><div class='del'>-	if (access &amp; IB_ACCESS_ON_DEMAND) {</div><div class='del'>-		if (WARN_ON_ONCE(!context-&gt;invalidate_range)) {</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto umem_kfree;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		ret = ib_umem_odp_get(to_ib_umem_odp(umem), access);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto umem_kfree;</div><div class='del'>-		return umem;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	page_list = (struct page **) __get_free_page(GFP_KERNEL);</div><div class='ctx'> 	if (!page_list) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='hunk'>@@ -284,56 +227,44 @@ struct ib_umem *ib_umem_get(struct ib_udata *udata, unsigned long addr,</div><div class='ctx'> </div><div class='ctx'> 	cur_base = addr &amp; PAGE_MASK;</div><div class='ctx'> </div><div class='del'>-	ret = sg_alloc_table(&amp;umem-&gt;sg_head, npages, GFP_KERNEL);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto vma;</div><div class='del'>-</div><div class='del'>-	if (!umem-&gt;writable)</div><div class='del'>-		gup_flags |= FOLL_FORCE;</div><div class='del'>-</div><div class='del'>-	sg = umem-&gt;sg_head.sgl;</div><div class='add'>+	if (umem-&gt;writable)</div><div class='add'>+		gup_flags |= FOLL_WRITE;</div><div class='ctx'> </div><div class='ctx'> 	while (npages) {</div><div class='del'>-		down_read(&amp;mm-&gt;mmap_sem);</div><div class='del'>-		ret = get_user_pages(cur_base,</div><div class='del'>-				     min_t(unsigned long, npages,</div><div class='del'>-					   PAGE_SIZE / sizeof (struct page *)),</div><div class='del'>-				     gup_flags | FOLL_LONGTERM,</div><div class='del'>-				     page_list, NULL);</div><div class='del'>-		if (ret &lt; 0) {</div><div class='del'>-			up_read(&amp;mm-&gt;mmap_sem);</div><div class='add'>+		cond_resched();</div><div class='add'>+		pinned = pin_user_pages_fast(cur_base,</div><div class='add'>+					  min_t(unsigned long, npages,</div><div class='add'>+						PAGE_SIZE /</div><div class='add'>+						sizeof(struct page *)),</div><div class='add'>+					  gup_flags, page_list);</div><div class='add'>+		if (pinned &lt; 0) {</div><div class='add'>+			ret = pinned;</div><div class='ctx'> 			goto umem_release;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		cur_base += ret * PAGE_SIZE;</div><div class='del'>-		npages   -= ret;</div><div class='del'>-</div><div class='del'>-		sg = ib_umem_add_sg_table(sg, page_list, ret,</div><div class='del'>-			dma_get_max_seg_size(context-&gt;device-&gt;dma_device),</div><div class='del'>-			&amp;umem-&gt;sg_nents);</div><div class='del'>-</div><div class='del'>-		up_read(&amp;mm-&gt;mmap_sem);</div><div class='add'>+		cur_base += pinned * PAGE_SIZE;</div><div class='add'>+		npages -= pinned;</div><div class='add'>+		ret = sg_alloc_append_table_from_pages(</div><div class='add'>+			&amp;umem-&gt;sgt_append, page_list, pinned, 0,</div><div class='add'>+			pinned &lt;&lt; PAGE_SHIFT, ib_dma_max_seg_size(device),</div><div class='add'>+			npages, GFP_KERNEL);</div><div class='add'>+		if (ret) {</div><div class='add'>+			unpin_user_pages_dirty_lock(page_list, pinned, 0);</div><div class='add'>+			goto umem_release;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	sg_mark_end(sg);</div><div class='del'>-</div><div class='del'>-	umem-&gt;nmap = ib_dma_map_sg_attrs(context-&gt;device,</div><div class='del'>-				  umem-&gt;sg_head.sgl,</div><div class='del'>-				  umem-&gt;sg_nents,</div><div class='del'>-				  DMA_BIDIRECTIONAL,</div><div class='del'>-				  dma_attrs);</div><div class='add'>+	if (access &amp; IB_ACCESS_RELAXED_ORDERING)</div><div class='add'>+		dma_attr |= DMA_ATTR_WEAK_ORDERING;</div><div class='ctx'> </div><div class='del'>-	if (!umem-&gt;nmap) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='add'>+	ret = ib_dma_map_sgtable_attrs(device, &amp;umem-&gt;sgt_append.sgt,</div><div class='add'>+				       DMA_BIDIRECTIONAL, dma_attr);</div><div class='add'>+	if (ret)</div><div class='ctx'> 		goto umem_release;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = 0;</div><div class='ctx'> 	goto out;</div><div class='ctx'> </div><div class='ctx'> umem_release:</div><div class='del'>-	__ib_umem_release(context-&gt;device, umem, 0);</div><div class='del'>-vma:</div><div class='add'>+	__ib_umem_release(device, umem, 0);</div><div class='ctx'> 	atomic64_sub(ib_umem_num_pages(umem), &amp;mm-&gt;pinned_vm);</div><div class='ctx'> out:</div><div class='ctx'> 	free_page((unsigned long) page_list);</div><div class='hunk'>@@ -346,51 +277,27 @@ umem_kfree:</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_umem_get);</div><div class='ctx'> </div><div class='del'>-static void __ib_umem_release_tail(struct ib_umem *umem)</div><div class='del'>-{</div><div class='del'>-	mmdrop(umem-&gt;owning_mm);</div><div class='del'>-	if (umem-&gt;is_odp)</div><div class='del'>-		kfree(to_ib_umem_odp(umem));</div><div class='del'>-	else</div><div class='del'>-		kfree(umem);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> /**</div><div class='ctx'>  * ib_umem_release - release memory pinned with ib_umem_get</div><div class='ctx'>  * @umem: umem struct to release</div><div class='ctx'>  */</div><div class='ctx'> void ib_umem_release(struct ib_umem *umem)</div><div class='ctx'> {</div><div class='del'>-	if (umem-&gt;is_odp) {</div><div class='del'>-		ib_umem_odp_release(to_ib_umem_odp(umem));</div><div class='del'>-		__ib_umem_release_tail(umem);</div><div class='add'>+	if (!umem)</div><div class='ctx'> 		return;</div><div class='del'>-	}</div><div class='add'>+	if (umem-&gt;is_dmabuf)</div><div class='add'>+		return ib_umem_dmabuf_release(to_ib_umem_dmabuf(umem));</div><div class='add'>+	if (umem-&gt;is_odp)</div><div class='add'>+		return ib_umem_odp_release(to_ib_umem_odp(umem));</div><div class='ctx'> </div><div class='del'>-	__ib_umem_release(umem-&gt;context-&gt;device, umem, 1);</div><div class='add'>+	__ib_umem_release(umem-&gt;ibdev, umem, 1);</div><div class='ctx'> </div><div class='ctx'> 	atomic64_sub(ib_umem_num_pages(umem), &amp;umem-&gt;owning_mm-&gt;pinned_vm);</div><div class='del'>-	__ib_umem_release_tail(umem);</div><div class='add'>+	mmdrop(umem-&gt;owning_mm);</div><div class='add'>+	kfree(umem);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_umem_release);</div><div class='ctx'> </div><div class='del'>-int ib_umem_page_count(struct ib_umem *umem)</div><div class='del'>-{</div><div class='del'>-	int i;</div><div class='del'>-	int n;</div><div class='del'>-	struct scatterlist *sg;</div><div class='del'>-</div><div class='del'>-	if (umem-&gt;is_odp)</div><div class='del'>-		return ib_umem_num_pages(umem);</div><div class='del'>-</div><div class='del'>-	n = 0;</div><div class='del'>-	for_each_sg(umem-&gt;sg_head.sgl, sg, umem-&gt;nmap, i)</div><div class='del'>-		n += sg_dma_len(sg) &gt;&gt; umem-&gt;page_shift;</div><div class='del'>-</div><div class='del'>-	return n;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_umem_page_count);</div><div class='del'>-</div><div class='ctx'> /*</div><div class='ctx'>  * Copy from the given ib_umem's pages to the given buffer.</div><div class='ctx'>  *</div><div class='hunk'>@@ -408,12 +315,13 @@ int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (offset &gt; umem-&gt;length || length &gt; umem-&gt;length - offset) {</div><div class='del'>-		pr_err("ib_umem_copy_from not in range. offset: %zd umem length: %zd end: %zd\n",</div><div class='del'>-		       offset, umem-&gt;length, end);</div><div class='add'>+		pr_err("%s not in range. offset: %zd umem length: %zd end: %zd\n",</div><div class='add'>+		       __func__, offset, umem-&gt;length, end);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = sg_pcopy_to_buffer(umem-&gt;sg_head.sgl, umem-&gt;sg_nents, dst, length,</div><div class='add'>+	ret = sg_pcopy_to_buffer(umem-&gt;sgt_append.sgt.sgl,</div><div class='add'>+				 umem-&gt;sgt_append.sgt.orig_nents, dst, length,</div><div class='ctx'> 				 offset + ib_umem_offset(umem));</div><div class='ctx'> </div><div class='ctx'> 	if (ret &lt; 0)</div><div class='head'>diff --git a/drivers/infiniband/core/umem_dmabuf.c b/drivers/infiniband/core/umem_dmabuf.c<br/>new file mode 100644<br/>index 000000000000..0ec2e4120cc9<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/umem_dmabuf.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/umem_dmabuf.c</a></div><div class='hunk'>@@ -0,0 +1,275 @@</div><div class='add'>+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2020 Intel Corporation. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/dma-buf.h&gt;</div><div class='add'>+#include &lt;linux/dma-resv.h&gt;</div><div class='add'>+#include &lt;linux/dma-mapping.h&gt;</div><div class='add'>+#include &lt;linux/module.h&gt;</div><div class='add'>+</div><div class='add'>+#include "uverbs.h"</div><div class='add'>+</div><div class='add'>+MODULE_IMPORT_NS("DMA_BUF");</div><div class='add'>+</div><div class='add'>+int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf)</div><div class='add'>+{</div><div class='add'>+	struct sg_table *sgt;</div><div class='add'>+	struct scatterlist *sg;</div><div class='add'>+	unsigned long start, end, cur = 0;</div><div class='add'>+	unsigned int nmap = 0;</div><div class='add'>+	long ret;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	dma_resv_assert_held(umem_dmabuf-&gt;attach-&gt;dmabuf-&gt;resv);</div><div class='add'>+</div><div class='add'>+	if (umem_dmabuf-&gt;revoked)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (umem_dmabuf-&gt;sgt)</div><div class='add'>+		goto wait_fence;</div><div class='add'>+</div><div class='add'>+	sgt = dma_buf_map_attachment(umem_dmabuf-&gt;attach,</div><div class='add'>+				     DMA_BIDIRECTIONAL);</div><div class='add'>+	if (IS_ERR(sgt))</div><div class='add'>+		return PTR_ERR(sgt);</div><div class='add'>+</div><div class='add'>+	/* modify the sg list in-place to match umem address and length */</div><div class='add'>+</div><div class='add'>+	start = ALIGN_DOWN(umem_dmabuf-&gt;umem.address, PAGE_SIZE);</div><div class='add'>+	end = ALIGN(umem_dmabuf-&gt;umem.address + umem_dmabuf-&gt;umem.length,</div><div class='add'>+		    PAGE_SIZE);</div><div class='add'>+	for_each_sgtable_dma_sg(sgt, sg, i) {</div><div class='add'>+		if (start &lt; cur + sg_dma_len(sg) &amp;&amp; cur &lt; end)</div><div class='add'>+			nmap++;</div><div class='add'>+		if (cur &lt;= start &amp;&amp; start &lt; cur + sg_dma_len(sg)) {</div><div class='add'>+			unsigned long offset = start - cur;</div><div class='add'>+</div><div class='add'>+			umem_dmabuf-&gt;first_sg = sg;</div><div class='add'>+			umem_dmabuf-&gt;first_sg_offset = offset;</div><div class='add'>+			sg_dma_address(sg) += offset;</div><div class='add'>+			sg_dma_len(sg) -= offset;</div><div class='add'>+			cur += offset;</div><div class='add'>+		}</div><div class='add'>+		if (cur &lt; end &amp;&amp; end &lt;= cur + sg_dma_len(sg)) {</div><div class='add'>+			unsigned long trim = cur + sg_dma_len(sg) - end;</div><div class='add'>+</div><div class='add'>+			umem_dmabuf-&gt;last_sg = sg;</div><div class='add'>+			umem_dmabuf-&gt;last_sg_trim = trim;</div><div class='add'>+			sg_dma_len(sg) -= trim;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		cur += sg_dma_len(sg);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	umem_dmabuf-&gt;umem.sgt_append.sgt.sgl = umem_dmabuf-&gt;first_sg;</div><div class='add'>+	umem_dmabuf-&gt;umem.sgt_append.sgt.nents = nmap;</div><div class='add'>+	umem_dmabuf-&gt;sgt = sgt;</div><div class='add'>+</div><div class='add'>+wait_fence:</div><div class='add'>+	/*</div><div class='add'>+	 * Although the sg list is valid now, the content of the pages</div><div class='add'>+	 * may be not up-to-date. Wait for the exporter to finish</div><div class='add'>+	 * the migration.</div><div class='add'>+	 */</div><div class='add'>+	ret = dma_resv_wait_timeout(umem_dmabuf-&gt;attach-&gt;dmabuf-&gt;resv,</div><div class='add'>+				     DMA_RESV_USAGE_KERNEL,</div><div class='add'>+				     false, MAX_SCHEDULE_TIMEOUT);</div><div class='add'>+	if (ret &lt; 0)</div><div class='add'>+		return ret;</div><div class='add'>+	if (ret == 0)</div><div class='add'>+		return -ETIMEDOUT;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_umem_dmabuf_map_pages);</div><div class='add'>+</div><div class='add'>+void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf)</div><div class='add'>+{</div><div class='add'>+	dma_resv_assert_held(umem_dmabuf-&gt;attach-&gt;dmabuf-&gt;resv);</div><div class='add'>+</div><div class='add'>+	if (!umem_dmabuf-&gt;sgt)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	/* retore the original sg list */</div><div class='add'>+	if (umem_dmabuf-&gt;first_sg) {</div><div class='add'>+		sg_dma_address(umem_dmabuf-&gt;first_sg) -=</div><div class='add'>+			umem_dmabuf-&gt;first_sg_offset;</div><div class='add'>+		sg_dma_len(umem_dmabuf-&gt;first_sg) +=</div><div class='add'>+			umem_dmabuf-&gt;first_sg_offset;</div><div class='add'>+		umem_dmabuf-&gt;first_sg = NULL;</div><div class='add'>+		umem_dmabuf-&gt;first_sg_offset = 0;</div><div class='add'>+	}</div><div class='add'>+	if (umem_dmabuf-&gt;last_sg) {</div><div class='add'>+		sg_dma_len(umem_dmabuf-&gt;last_sg) +=</div><div class='add'>+			umem_dmabuf-&gt;last_sg_trim;</div><div class='add'>+		umem_dmabuf-&gt;last_sg = NULL;</div><div class='add'>+		umem_dmabuf-&gt;last_sg_trim = 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dma_buf_unmap_attachment(umem_dmabuf-&gt;attach, umem_dmabuf-&gt;sgt,</div><div class='add'>+				 DMA_BIDIRECTIONAL);</div><div class='add'>+</div><div class='add'>+	umem_dmabuf-&gt;sgt = NULL;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_umem_dmabuf_unmap_pages);</div><div class='add'>+</div><div class='add'>+static struct ib_umem_dmabuf *</div><div class='add'>+ib_umem_dmabuf_get_with_dma_device(struct ib_device *device,</div><div class='add'>+				   struct device *dma_device,</div><div class='add'>+				   unsigned long offset, size_t size,</div><div class='add'>+				   int fd, int access,</div><div class='add'>+				   const struct dma_buf_attach_ops *ops)</div><div class='add'>+{</div><div class='add'>+	struct dma_buf *dmabuf;</div><div class='add'>+	struct ib_umem_dmabuf *umem_dmabuf;</div><div class='add'>+	struct ib_umem *umem;</div><div class='add'>+	unsigned long end;</div><div class='add'>+	struct ib_umem_dmabuf *ret = ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	if (check_add_overflow(offset, (unsigned long)size, &amp;end))</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (unlikely(!ops || !ops-&gt;move_notify))</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	dmabuf = dma_buf_get(fd);</div><div class='add'>+	if (IS_ERR(dmabuf))</div><div class='add'>+		return ERR_CAST(dmabuf);</div><div class='add'>+</div><div class='add'>+	if (dmabuf-&gt;size &lt; end)</div><div class='add'>+		goto out_release_dmabuf;</div><div class='add'>+</div><div class='add'>+	umem_dmabuf = kzalloc(sizeof(*umem_dmabuf), GFP_KERNEL);</div><div class='add'>+	if (!umem_dmabuf) {</div><div class='add'>+		ret = ERR_PTR(-ENOMEM);</div><div class='add'>+		goto out_release_dmabuf;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	umem = &amp;umem_dmabuf-&gt;umem;</div><div class='add'>+	umem-&gt;ibdev = device;</div><div class='add'>+	umem-&gt;length = size;</div><div class='add'>+	umem-&gt;address = offset;</div><div class='add'>+	umem-&gt;writable = ib_access_writable(access);</div><div class='add'>+	umem-&gt;is_dmabuf = 1;</div><div class='add'>+</div><div class='add'>+	if (!ib_umem_num_pages(umem))</div><div class='add'>+		goto out_free_umem;</div><div class='add'>+</div><div class='add'>+	umem_dmabuf-&gt;attach = dma_buf_dynamic_attach(</div><div class='add'>+					dmabuf,</div><div class='add'>+					dma_device,</div><div class='add'>+					ops,</div><div class='add'>+					umem_dmabuf);</div><div class='add'>+	if (IS_ERR(umem_dmabuf-&gt;attach)) {</div><div class='add'>+		ret = ERR_CAST(umem_dmabuf-&gt;attach);</div><div class='add'>+		goto out_free_umem;</div><div class='add'>+	}</div><div class='add'>+	return umem_dmabuf;</div><div class='add'>+</div><div class='add'>+out_free_umem:</div><div class='add'>+	kfree(umem_dmabuf);</div><div class='add'>+</div><div class='add'>+out_release_dmabuf:</div><div class='add'>+	dma_buf_put(dmabuf);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device,</div><div class='add'>+					  unsigned long offset, size_t size,</div><div class='add'>+					  int fd, int access,</div><div class='add'>+					  const struct dma_buf_attach_ops *ops)</div><div class='add'>+{</div><div class='add'>+	return ib_umem_dmabuf_get_with_dma_device(device, device-&gt;dma_device,</div><div class='add'>+						  offset, size, fd, access, ops);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_umem_dmabuf_get);</div><div class='add'>+</div><div class='add'>+static void</div><div class='add'>+ib_umem_dmabuf_unsupported_move_notify(struct dma_buf_attachment *attach)</div><div class='add'>+{</div><div class='add'>+	struct ib_umem_dmabuf *umem_dmabuf = attach-&gt;importer_priv;</div><div class='add'>+</div><div class='add'>+	ibdev_warn_ratelimited(umem_dmabuf-&gt;umem.ibdev,</div><div class='add'>+			       "Invalidate callback should not be called when memory is pinned\n");</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = {</div><div class='add'>+	.allow_peer2peer = true,</div><div class='add'>+	.move_notify = ib_umem_dmabuf_unsupported_move_notify,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ib_umem_dmabuf *</div><div class='add'>+ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device,</div><div class='add'>+					  struct device *dma_device,</div><div class='add'>+					  unsigned long offset, size_t size,</div><div class='add'>+					  int fd, int access)</div><div class='add'>+{</div><div class='add'>+	struct ib_umem_dmabuf *umem_dmabuf;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	umem_dmabuf = ib_umem_dmabuf_get_with_dma_device(device, dma_device, offset,</div><div class='add'>+							 size, fd, access,</div><div class='add'>+							 &amp;ib_umem_dmabuf_attach_pinned_ops);</div><div class='add'>+	if (IS_ERR(umem_dmabuf))</div><div class='add'>+		return umem_dmabuf;</div><div class='add'>+</div><div class='add'>+	dma_resv_lock(umem_dmabuf-&gt;attach-&gt;dmabuf-&gt;resv, NULL);</div><div class='add'>+	err = dma_buf_pin(umem_dmabuf-&gt;attach);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_release;</div><div class='add'>+	umem_dmabuf-&gt;pinned = 1;</div><div class='add'>+</div><div class='add'>+	err = ib_umem_dmabuf_map_pages(umem_dmabuf);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_unpin;</div><div class='add'>+	dma_resv_unlock(umem_dmabuf-&gt;attach-&gt;dmabuf-&gt;resv);</div><div class='add'>+</div><div class='add'>+	return umem_dmabuf;</div><div class='add'>+</div><div class='add'>+err_unpin:</div><div class='add'>+	dma_buf_unpin(umem_dmabuf-&gt;attach);</div><div class='add'>+err_release:</div><div class='add'>+	dma_resv_unlock(umem_dmabuf-&gt;attach-&gt;dmabuf-&gt;resv);</div><div class='add'>+	ib_umem_release(&amp;umem_dmabuf-&gt;umem);</div><div class='add'>+	return ERR_PTR(err);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned_with_dma_device);</div><div class='add'>+</div><div class='add'>+struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device,</div><div class='add'>+						 unsigned long offset,</div><div class='add'>+						 size_t size, int fd,</div><div class='add'>+						 int access)</div><div class='add'>+{</div><div class='add'>+	return ib_umem_dmabuf_get_pinned_with_dma_device(device, device-&gt;dma_device,</div><div class='add'>+							 offset, size, fd, access);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned);</div><div class='add'>+</div><div class='add'>+void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf)</div><div class='add'>+{</div><div class='add'>+	struct dma_buf *dmabuf = umem_dmabuf-&gt;attach-&gt;dmabuf;</div><div class='add'>+</div><div class='add'>+	dma_resv_lock(dmabuf-&gt;resv, NULL);</div><div class='add'>+	if (umem_dmabuf-&gt;revoked)</div><div class='add'>+		goto end;</div><div class='add'>+	ib_umem_dmabuf_unmap_pages(umem_dmabuf);</div><div class='add'>+	if (umem_dmabuf-&gt;pinned) {</div><div class='add'>+		dma_buf_unpin(umem_dmabuf-&gt;attach);</div><div class='add'>+		umem_dmabuf-&gt;pinned = 0;</div><div class='add'>+	}</div><div class='add'>+	umem_dmabuf-&gt;revoked = 1;</div><div class='add'>+end:</div><div class='add'>+	dma_resv_unlock(dmabuf-&gt;resv);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_umem_dmabuf_revoke);</div><div class='add'>+</div><div class='add'>+void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf)</div><div class='add'>+{</div><div class='add'>+	struct dma_buf *dmabuf = umem_dmabuf-&gt;attach-&gt;dmabuf;</div><div class='add'>+</div><div class='add'>+	ib_umem_dmabuf_revoke(umem_dmabuf);</div><div class='add'>+</div><div class='add'>+	dma_buf_detach(dmabuf, umem_dmabuf-&gt;attach);</div><div class='add'>+	dma_buf_put(dmabuf);</div><div class='add'>+	kfree(umem_dmabuf);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c<br/>index f962b5bbfa40..572a91a62a7b 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/umem_odp.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/umem_odp.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/umem_odp.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/umem_odp.c</a></div><div class='hunk'>@@ -39,424 +39,241 @@</div><div class='ctx'> #include &lt;linux/export.h&gt;</div><div class='ctx'> #include &lt;linux/vmalloc.h&gt;</div><div class='ctx'> #include &lt;linux/hugetlb.h&gt;</div><div class='del'>-#include &lt;linux/interval_tree_generic.h&gt;</div><div class='add'>+#include &lt;linux/interval_tree.h&gt;</div><div class='add'>+#include &lt;linux/hmm.h&gt;</div><div class='add'>+#include &lt;linux/hmm-dma.h&gt;</div><div class='ctx'> #include &lt;linux/pagemap.h&gt;</div><div class='ctx'> </div><div class='del'>-#include &lt;rdma/ib_verbs.h&gt;</div><div class='del'>-#include &lt;rdma/ib_umem.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_umem_odp.h&gt;</div><div class='ctx'> </div><div class='del'>-/*</div><div class='del'>- * The ib_umem list keeps track of memory regions for which the HW</div><div class='del'>- * device request to receive notification when the related memory</div><div class='del'>- * mapping is changed.</div><div class='del'>- *</div><div class='del'>- * ib_umem_lock protects the list.</div><div class='del'>- */</div><div class='del'>-</div><div class='del'>-static u64 node_start(struct umem_odp_node *n)</div><div class='del'>-{</div><div class='del'>-	struct ib_umem_odp *umem_odp =</div><div class='del'>-			container_of(n, struct ib_umem_odp, interval_tree);</div><div class='del'>-</div><div class='del'>-	return ib_umem_start(&amp;umem_odp-&gt;umem);</div><div class='del'>-}</div><div class='add'>+#include "uverbs.h"</div><div class='ctx'> </div><div class='del'>-/* Note that the representation of the intervals in the interval tree</div><div class='del'>- * considers the ending point as contained in the interval, while the</div><div class='del'>- * function ib_umem_end returns the first address which is not contained</div><div class='del'>- * in the umem.</div><div class='del'>- */</div><div class='del'>-static u64 node_last(struct umem_odp_node *n)</div><div class='add'>+static void ib_init_umem_implicit_odp(struct ib_umem_odp *umem_odp)</div><div class='ctx'> {</div><div class='del'>-	struct ib_umem_odp *umem_odp =</div><div class='del'>-			container_of(n, struct ib_umem_odp, interval_tree);</div><div class='del'>-</div><div class='del'>-	return ib_umem_end(&amp;umem_odp-&gt;umem) - 1;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-INTERVAL_TREE_DEFINE(struct umem_odp_node, rb, u64, __subtree_last,</div><div class='del'>-		     node_start, node_last, static, rbt_ib_umem)</div><div class='del'>-</div><div class='del'>-static void ib_umem_notifier_start_account(struct ib_umem_odp *umem_odp)</div><div class='del'>-{</div><div class='del'>-	mutex_lock(&amp;umem_odp-&gt;umem_mutex);</div><div class='del'>-	if (umem_odp-&gt;notifiers_count++ == 0)</div><div class='del'>-		/*</div><div class='del'>-		 * Initialize the completion object for waiting on</div><div class='del'>-		 * notifiers. Since notifier_count is zero, no one should be</div><div class='del'>-		 * waiting right now.</div><div class='del'>-		 */</div><div class='del'>-		reinit_completion(&amp;umem_odp-&gt;notifier_completion);</div><div class='del'>-	mutex_unlock(&amp;umem_odp-&gt;umem_mutex);</div><div class='add'>+	umem_odp-&gt;is_implicit_odp = 1;</div><div class='add'>+	umem_odp-&gt;umem.is_odp = 1;</div><div class='add'>+	mutex_init(&amp;umem_odp-&gt;umem_mutex);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void ib_umem_notifier_end_account(struct ib_umem_odp *umem_odp)</div><div class='add'>+static int ib_init_umem_odp(struct ib_umem_odp *umem_odp,</div><div class='add'>+			    const struct mmu_interval_notifier_ops *ops)</div><div class='ctx'> {</div><div class='del'>-	mutex_lock(&amp;umem_odp-&gt;umem_mutex);</div><div class='del'>-	/*</div><div class='del'>-	 * This sequence increase will notify the QP page fault that the page</div><div class='del'>-	 * that is going to be mapped in the spte could have been freed.</div><div class='del'>-	 */</div><div class='del'>-	++umem_odp-&gt;notifiers_seq;</div><div class='del'>-	if (--umem_odp-&gt;notifiers_count == 0)</div><div class='del'>-		complete_all(&amp;umem_odp-&gt;notifier_completion);</div><div class='del'>-	mutex_unlock(&amp;umem_odp-&gt;umem_mutex);</div><div class='del'>-}</div><div class='add'>+	struct ib_device *dev = umem_odp-&gt;umem.ibdev;</div><div class='add'>+	size_t page_size = 1UL &lt;&lt; umem_odp-&gt;page_shift;</div><div class='add'>+	struct hmm_dma_map *map;</div><div class='add'>+	unsigned long start;</div><div class='add'>+	unsigned long end;</div><div class='add'>+	size_t nr_entries;</div><div class='add'>+	int ret = 0;</div><div class='ctx'> </div><div class='del'>-static int ib_umem_notifier_release_trampoline(struct ib_umem_odp *umem_odp,</div><div class='del'>-					       u64 start, u64 end, void *cookie)</div><div class='del'>-{</div><div class='del'>-	struct ib_umem *umem = &amp;umem_odp-&gt;umem;</div><div class='add'>+	umem_odp-&gt;umem.is_odp = 1;</div><div class='add'>+	mutex_init(&amp;umem_odp-&gt;umem_mutex);</div><div class='ctx'> </div><div class='add'>+	start = ALIGN_DOWN(umem_odp-&gt;umem.address, page_size);</div><div class='add'>+	if (check_add_overflow(umem_odp-&gt;umem.address,</div><div class='add'>+			       (unsigned long)umem_odp-&gt;umem.length, &amp;end))</div><div class='add'>+		return -EOVERFLOW;</div><div class='add'>+	end = ALIGN(end, page_size);</div><div class='add'>+	if (unlikely(end &lt; page_size))</div><div class='add'>+		return -EOVERFLOW;</div><div class='ctx'> 	/*</div><div class='del'>-	 * Increase the number of notifiers running, to</div><div class='del'>-	 * prevent any further fault handling on this MR.</div><div class='add'>+	 * The mmu notifier can be called within reclaim contexts and takes the</div><div class='add'>+	 * umem_mutex. This is rare to trigger in testing, teach lockdep about</div><div class='add'>+	 * it.</div><div class='ctx'> 	 */</div><div class='del'>-	ib_umem_notifier_start_account(umem_odp);</div><div class='del'>-	umem_odp-&gt;dying = 1;</div><div class='del'>-	/* Make sure that the fact the umem is dying is out before we release</div><div class='del'>-	 * all pending page faults. */</div><div class='del'>-	smp_wmb();</div><div class='del'>-	complete_all(&amp;umem_odp-&gt;notifier_completion);</div><div class='del'>-	umem-&gt;context-&gt;invalidate_range(umem_odp, ib_umem_start(umem),</div><div class='del'>-					ib_umem_end(umem));</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void ib_umem_notifier_release(struct mmu_notifier *mn,</div><div class='del'>-				     struct mm_struct *mm)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucontext_per_mm *per_mm =</div><div class='del'>-		container_of(mn, struct ib_ucontext_per_mm, mn);</div><div class='del'>-</div><div class='del'>-	down_read(&amp;per_mm-&gt;umem_rwsem);</div><div class='del'>-	if (per_mm-&gt;active)</div><div class='del'>-		rbt_ib_umem_for_each_in_range(</div><div class='del'>-			&amp;per_mm-&gt;umem_tree, 0, ULLONG_MAX,</div><div class='del'>-			ib_umem_notifier_release_trampoline, true, NULL);</div><div class='del'>-	up_read(&amp;per_mm-&gt;umem_rwsem);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int invalidate_range_start_trampoline(struct ib_umem_odp *item,</div><div class='del'>-					     u64 start, u64 end, void *cookie)</div><div class='del'>-{</div><div class='del'>-	ib_umem_notifier_start_account(item);</div><div class='del'>-	item-&gt;umem.context-&gt;invalidate_range(item, start, end);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int ib_umem_notifier_invalidate_range_start(struct mmu_notifier *mn,</div><div class='del'>-				const struct mmu_notifier_range *range)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucontext_per_mm *per_mm =</div><div class='del'>-		container_of(mn, struct ib_ucontext_per_mm, mn);</div><div class='del'>-</div><div class='del'>-	if (mmu_notifier_range_blockable(range))</div><div class='del'>-		down_read(&amp;per_mm-&gt;umem_rwsem);</div><div class='del'>-	else if (!down_read_trylock(&amp;per_mm-&gt;umem_rwsem))</div><div class='del'>-		return -EAGAIN;</div><div class='del'>-</div><div class='del'>-	if (!per_mm-&gt;active) {</div><div class='del'>-		up_read(&amp;per_mm-&gt;umem_rwsem);</div><div class='del'>-		/*</div><div class='del'>-		 * At this point active is permanently set and visible to this</div><div class='del'>-		 * CPU without a lock, that fact is relied on to skip the unlock</div><div class='del'>-		 * in range_end.</div><div class='del'>-		 */</div><div class='del'>-		return 0;</div><div class='add'>+	if (IS_ENABLED(CONFIG_LOCKDEP)) {</div><div class='add'>+		fs_reclaim_acquire(GFP_KERNEL);</div><div class='add'>+		mutex_lock(&amp;umem_odp-&gt;umem_mutex);</div><div class='add'>+		mutex_unlock(&amp;umem_odp-&gt;umem_mutex);</div><div class='add'>+		fs_reclaim_release(GFP_KERNEL);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return rbt_ib_umem_for_each_in_range(&amp;per_mm-&gt;umem_tree, range-&gt;start,</div><div class='del'>-					     range-&gt;end,</div><div class='del'>-					     invalidate_range_start_trampoline,</div><div class='del'>-					     mmu_notifier_range_blockable(range),</div><div class='del'>-					     NULL);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int invalidate_range_end_trampoline(struct ib_umem_odp *item, u64 start,</div><div class='del'>-					   u64 end, void *cookie)</div><div class='del'>-{</div><div class='del'>-	ib_umem_notifier_end_account(item);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void ib_umem_notifier_invalidate_range_end(struct mmu_notifier *mn,</div><div class='del'>-				const struct mmu_notifier_range *range)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucontext_per_mm *per_mm =</div><div class='del'>-		container_of(mn, struct ib_ucontext_per_mm, mn);</div><div class='del'>-</div><div class='del'>-	if (unlikely(!per_mm-&gt;active))</div><div class='del'>-		return;</div><div class='add'>+	nr_entries = (end - start) &gt;&gt; PAGE_SHIFT;</div><div class='add'>+	if (!(nr_entries * PAGE_SIZE / page_size))</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	rbt_ib_umem_for_each_in_range(&amp;per_mm-&gt;umem_tree, range-&gt;start,</div><div class='del'>-				      range-&gt;end,</div><div class='del'>-				      invalidate_range_end_trampoline, true, NULL);</div><div class='del'>-	up_read(&amp;per_mm-&gt;umem_rwsem);</div><div class='del'>-}</div><div class='add'>+	map = &amp;umem_odp-&gt;map;</div><div class='add'>+	if (ib_uses_virt_dma(dev)) {</div><div class='add'>+		map-&gt;pfn_list = kvcalloc(nr_entries, sizeof(*map-&gt;pfn_list),</div><div class='add'>+					 GFP_KERNEL | __GFP_NOWARN);</div><div class='add'>+		if (!map-&gt;pfn_list)</div><div class='add'>+			ret = -ENOMEM;</div><div class='add'>+	} else</div><div class='add'>+		ret = hmm_dma_map_alloc(dev-&gt;dma_device, map,</div><div class='add'>+					(end - start) &gt;&gt; PAGE_SHIFT,</div><div class='add'>+					1 &lt;&lt; umem_odp-&gt;page_shift);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = mmu_interval_notifier_insert(&amp;umem_odp-&gt;notifier,</div><div class='add'>+					   umem_odp-&gt;umem.owning_mm, start,</div><div class='add'>+					   end - start, ops);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out_free_map;</div><div class='ctx'> </div><div class='del'>-static const struct mmu_notifier_ops ib_umem_notifiers = {</div><div class='del'>-	.release                    = ib_umem_notifier_release,</div><div class='del'>-	.invalidate_range_start     = ib_umem_notifier_invalidate_range_start,</div><div class='del'>-	.invalidate_range_end       = ib_umem_notifier_invalidate_range_end,</div><div class='del'>-};</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-static void add_umem_to_per_mm(struct ib_umem_odp *umem_odp)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucontext_per_mm *per_mm = umem_odp-&gt;per_mm;</div><div class='del'>-	struct ib_umem *umem = &amp;umem_odp-&gt;umem;</div><div class='del'>-</div><div class='del'>-	down_write(&amp;per_mm-&gt;umem_rwsem);</div><div class='del'>-	if (likely(ib_umem_start(umem) != ib_umem_end(umem)))</div><div class='del'>-		rbt_ib_umem_insert(&amp;umem_odp-&gt;interval_tree,</div><div class='del'>-				   &amp;per_mm-&gt;umem_tree);</div><div class='del'>-	up_write(&amp;per_mm-&gt;umem_rwsem);</div><div class='add'>+out_free_map:</div><div class='add'>+	if (ib_uses_virt_dma(dev))</div><div class='add'>+		kvfree(map-&gt;pfn_list);</div><div class='add'>+	else</div><div class='add'>+		hmm_dma_map_free(dev-&gt;dma_device, map);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void remove_umem_from_per_mm(struct ib_umem_odp *umem_odp)</div><div class='add'>+/**</div><div class='add'>+ * ib_umem_odp_alloc_implicit - Allocate a parent implicit ODP umem</div><div class='add'>+ *</div><div class='add'>+ * Implicit ODP umems do not have a VA range and do not have any page lists.</div><div class='add'>+ * They exist only to hold the per_mm reference to help the driver create</div><div class='add'>+ * children umems.</div><div class='add'>+ *</div><div class='add'>+ * @device: IB device to create UMEM</div><div class='add'>+ * @access: ib_reg_mr access flags</div><div class='add'>+ */</div><div class='add'>+struct ib_umem_odp *ib_umem_odp_alloc_implicit(struct ib_device *device,</div><div class='add'>+					       int access)</div><div class='ctx'> {</div><div class='del'>-	struct ib_ucontext_per_mm *per_mm = umem_odp-&gt;per_mm;</div><div class='del'>-	struct ib_umem *umem = &amp;umem_odp-&gt;umem;</div><div class='del'>-</div><div class='del'>-	down_write(&amp;per_mm-&gt;umem_rwsem);</div><div class='del'>-	if (likely(ib_umem_start(umem) != ib_umem_end(umem)))</div><div class='del'>-		rbt_ib_umem_remove(&amp;umem_odp-&gt;interval_tree,</div><div class='del'>-				   &amp;per_mm-&gt;umem_tree);</div><div class='del'>-	complete_all(&amp;umem_odp-&gt;notifier_completion);</div><div class='add'>+	struct ib_umem *umem;</div><div class='add'>+	struct ib_umem_odp *umem_odp;</div><div class='ctx'> </div><div class='del'>-	up_write(&amp;per_mm-&gt;umem_rwsem);</div><div class='del'>-}</div><div class='add'>+	if (access &amp; IB_ACCESS_HUGETLB)</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='ctx'> </div><div class='del'>-static struct ib_ucontext_per_mm *alloc_per_mm(struct ib_ucontext *ctx,</div><div class='del'>-					       struct mm_struct *mm)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucontext_per_mm *per_mm;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	per_mm = kzalloc(sizeof(*per_mm), GFP_KERNEL);</div><div class='del'>-	if (!per_mm)</div><div class='add'>+	umem_odp = kzalloc(sizeof(*umem_odp), GFP_KERNEL);</div><div class='add'>+	if (!umem_odp)</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	per_mm-&gt;context = ctx;</div><div class='del'>-	per_mm-&gt;mm = mm;</div><div class='del'>-	per_mm-&gt;umem_tree = RB_ROOT_CACHED;</div><div class='del'>-	init_rwsem(&amp;per_mm-&gt;umem_rwsem);</div><div class='del'>-	per_mm-&gt;active = true;</div><div class='del'>-</div><div class='del'>-	rcu_read_lock();</div><div class='del'>-	per_mm-&gt;tgid = get_task_pid(current-&gt;group_leader, PIDTYPE_PID);</div><div class='del'>-	rcu_read_unlock();</div><div class='del'>-</div><div class='del'>-	WARN_ON(mm != current-&gt;mm);</div><div class='del'>-</div><div class='del'>-	per_mm-&gt;mn.ops = &amp;ib_umem_notifiers;</div><div class='del'>-	ret = mmu_notifier_register(&amp;per_mm-&gt;mn, per_mm-&gt;mm);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(&amp;ctx-&gt;device-&gt;dev,</div><div class='del'>-			"Failed to register mmu_notifier %d\n", ret);</div><div class='del'>-		goto out_pid;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	list_add(&amp;per_mm-&gt;ucontext_list, &amp;ctx-&gt;per_mm_list);</div><div class='del'>-	return per_mm;</div><div class='del'>-</div><div class='del'>-out_pid:</div><div class='del'>-	put_pid(per_mm-&gt;tgid);</div><div class='del'>-	kfree(per_mm);</div><div class='del'>-	return ERR_PTR(ret);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int get_per_mm(struct ib_umem_odp *umem_odp)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucontext *ctx = umem_odp-&gt;umem.context;</div><div class='del'>-	struct ib_ucontext_per_mm *per_mm;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Generally speaking we expect only one or two per_mm in this list,</div><div class='del'>-	 * so no reason to optimize this search today.</div><div class='del'>-	 */</div><div class='del'>-	mutex_lock(&amp;ctx-&gt;per_mm_list_lock);</div><div class='del'>-	list_for_each_entry(per_mm, &amp;ctx-&gt;per_mm_list, ucontext_list) {</div><div class='del'>-		if (per_mm-&gt;mm == umem_odp-&gt;umem.owning_mm)</div><div class='del'>-			goto found;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	per_mm = alloc_per_mm(ctx, umem_odp-&gt;umem.owning_mm);</div><div class='del'>-	if (IS_ERR(per_mm)) {</div><div class='del'>-		mutex_unlock(&amp;ctx-&gt;per_mm_list_lock);</div><div class='del'>-		return PTR_ERR(per_mm);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-found:</div><div class='del'>-	umem_odp-&gt;per_mm = per_mm;</div><div class='del'>-	per_mm-&gt;odp_mrs_count++;</div><div class='del'>-	mutex_unlock(&amp;ctx-&gt;per_mm_list_lock);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='add'>+	umem = &amp;umem_odp-&gt;umem;</div><div class='add'>+	umem-&gt;ibdev = device;</div><div class='add'>+	umem-&gt;writable = ib_access_writable(access);</div><div class='add'>+	umem-&gt;owning_mm = current-&gt;mm;</div><div class='add'>+	umem_odp-&gt;page_shift = PAGE_SHIFT;</div><div class='add'>+</div><div class='add'>+	umem_odp-&gt;tgid = get_task_pid(current-&gt;group_leader, PIDTYPE_PID);</div><div class='add'>+	ib_init_umem_implicit_odp(umem_odp);</div><div class='add'>+	return umem_odp;</div><div class='ctx'> }</div><div class='add'>+EXPORT_SYMBOL(ib_umem_odp_alloc_implicit);</div><div class='ctx'> </div><div class='del'>-static void free_per_mm(struct rcu_head *rcu)</div><div class='del'>-{</div><div class='del'>-	kfree(container_of(rcu, struct ib_ucontext_per_mm, rcu));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void put_per_mm(struct ib_umem_odp *umem_odp)</div><div class='add'>+/**</div><div class='add'>+ * ib_umem_odp_alloc_child - Allocate a child ODP umem under an implicit</div><div class='add'>+ *                           parent ODP umem</div><div class='add'>+ *</div><div class='add'>+ * @root: The parent umem enclosing the child. This must be allocated using</div><div class='add'>+ *        ib_alloc_implicit_odp_umem()</div><div class='add'>+ * @addr: The starting userspace VA</div><div class='add'>+ * @size: The length of the userspace VA</div><div class='add'>+ * @ops: MMU interval ops, currently only @invalidate</div><div class='add'>+ */</div><div class='add'>+struct ib_umem_odp *</div><div class='add'>+ib_umem_odp_alloc_child(struct ib_umem_odp *root, unsigned long addr,</div><div class='add'>+			size_t size,</div><div class='add'>+			const struct mmu_interval_notifier_ops *ops)</div><div class='ctx'> {</div><div class='del'>-	struct ib_ucontext_per_mm *per_mm = umem_odp-&gt;per_mm;</div><div class='del'>-	struct ib_ucontext *ctx = umem_odp-&gt;umem.context;</div><div class='del'>-	bool need_free;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;ctx-&gt;per_mm_list_lock);</div><div class='del'>-	umem_odp-&gt;per_mm = NULL;</div><div class='del'>-	per_mm-&gt;odp_mrs_count--;</div><div class='del'>-	need_free = per_mm-&gt;odp_mrs_count == 0;</div><div class='del'>-	if (need_free)</div><div class='del'>-		list_del(&amp;per_mm-&gt;ucontext_list);</div><div class='del'>-	mutex_unlock(&amp;ctx-&gt;per_mm_list_lock);</div><div class='del'>-</div><div class='del'>-	if (!need_free)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='ctx'> 	/*</div><div class='del'>-	 * NOTE! mmu_notifier_unregister() can happen between a start/end</div><div class='del'>-	 * callback, resulting in an start/end, and thus an unbalanced</div><div class='del'>-	 * lock. This doesn't really matter to us since we are about to kfree</div><div class='del'>-	 * the memory that holds the lock, however LOCKDEP doesn't like this.</div><div class='add'>+	 * Caller must ensure that root cannot be freed during the call to</div><div class='add'>+	 * ib_alloc_odp_umem.</div><div class='ctx'> 	 */</div><div class='del'>-	down_write(&amp;per_mm-&gt;umem_rwsem);</div><div class='del'>-	per_mm-&gt;active = false;</div><div class='del'>-	up_write(&amp;per_mm-&gt;umem_rwsem);</div><div class='del'>-</div><div class='del'>-	WARN_ON(!RB_EMPTY_ROOT(&amp;per_mm-&gt;umem_tree.rb_root));</div><div class='del'>-	mmu_notifier_unregister_no_release(&amp;per_mm-&gt;mn, per_mm-&gt;mm);</div><div class='del'>-	put_pid(per_mm-&gt;tgid);</div><div class='del'>-	mmu_notifier_call_srcu(&amp;per_mm-&gt;rcu, free_per_mm);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct ib_umem_odp *ib_alloc_odp_umem(struct ib_umem_odp *root,</div><div class='del'>-				      unsigned long addr, size_t size)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucontext_per_mm *per_mm = root-&gt;per_mm;</div><div class='del'>-	struct ib_ucontext *ctx = per_mm-&gt;context;</div><div class='ctx'> 	struct ib_umem_odp *odp_data;</div><div class='ctx'> 	struct ib_umem *umem;</div><div class='del'>-	int pages = size &gt;&gt; PAGE_SHIFT;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	if (WARN_ON(!root-&gt;is_implicit_odp))</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='ctx'> 	odp_data = kzalloc(sizeof(*odp_data), GFP_KERNEL);</div><div class='ctx'> 	if (!odp_data)</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='ctx'> 	umem = &amp;odp_data-&gt;umem;</div><div class='del'>-	umem-&gt;context    = ctx;</div><div class='add'>+	umem-&gt;ibdev = root-&gt;umem.ibdev;</div><div class='ctx'> 	umem-&gt;length     = size;</div><div class='ctx'> 	umem-&gt;address    = addr;</div><div class='del'>-	umem-&gt;page_shift = PAGE_SHIFT;</div><div class='ctx'> 	umem-&gt;writable   = root-&gt;umem.writable;</div><div class='del'>-	umem-&gt;is_odp = 1;</div><div class='del'>-	odp_data-&gt;per_mm = per_mm;</div><div class='del'>-	umem-&gt;owning_mm  = per_mm-&gt;mm;</div><div class='del'>-	mmgrab(umem-&gt;owning_mm);</div><div class='del'>-</div><div class='del'>-	mutex_init(&amp;odp_data-&gt;umem_mutex);</div><div class='del'>-	init_completion(&amp;odp_data-&gt;notifier_completion);</div><div class='del'>-</div><div class='del'>-	odp_data-&gt;page_list =</div><div class='del'>-		vzalloc(array_size(pages, sizeof(*odp_data-&gt;page_list)));</div><div class='del'>-	if (!odp_data-&gt;page_list) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto out_odp_data;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	odp_data-&gt;dma_list =</div><div class='del'>-		vzalloc(array_size(pages, sizeof(*odp_data-&gt;dma_list)));</div><div class='del'>-	if (!odp_data-&gt;dma_list) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto out_page_list;</div><div class='del'>-	}</div><div class='add'>+	umem-&gt;owning_mm  = root-&gt;umem.owning_mm;</div><div class='add'>+	odp_data-&gt;page_shift = PAGE_SHIFT;</div><div class='add'>+	odp_data-&gt;notifier.ops = ops;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='del'>-	 * Caller must ensure that the umem_odp that the per_mm came from</div><div class='del'>-	 * cannot be freed during the call to ib_alloc_odp_umem.</div><div class='add'>+	 * A mmget must be held when registering a notifier, the owming_mm only</div><div class='add'>+	 * has a mm_grab at this point.</div><div class='ctx'> 	 */</div><div class='del'>-	mutex_lock(&amp;ctx-&gt;per_mm_list_lock);</div><div class='del'>-	per_mm-&gt;odp_mrs_count++;</div><div class='del'>-	mutex_unlock(&amp;ctx-&gt;per_mm_list_lock);</div><div class='del'>-	add_umem_to_per_mm(odp_data);</div><div class='add'>+	if (!mmget_not_zero(umem-&gt;owning_mm)) {</div><div class='add'>+		ret = -EFAULT;</div><div class='add'>+		goto out_free;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='add'>+	odp_data-&gt;tgid = get_pid(root-&gt;tgid);</div><div class='add'>+	ret = ib_init_umem_odp(odp_data, ops);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out_tgid;</div><div class='add'>+	mmput(umem-&gt;owning_mm);</div><div class='ctx'> 	return odp_data;</div><div class='ctx'> </div><div class='del'>-out_page_list:</div><div class='del'>-	vfree(odp_data-&gt;page_list);</div><div class='del'>-out_odp_data:</div><div class='del'>-	mmdrop(umem-&gt;owning_mm);</div><div class='add'>+out_tgid:</div><div class='add'>+	put_pid(odp_data-&gt;tgid);</div><div class='add'>+	mmput(umem-&gt;owning_mm);</div><div class='add'>+out_free:</div><div class='ctx'> 	kfree(odp_data);</div><div class='ctx'> 	return ERR_PTR(ret);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(ib_alloc_odp_umem);</div><div class='add'>+EXPORT_SYMBOL(ib_umem_odp_alloc_child);</div><div class='ctx'> </div><div class='del'>-int ib_umem_odp_get(struct ib_umem_odp *umem_odp, int access)</div><div class='add'>+/**</div><div class='add'>+ * ib_umem_odp_get - Create a umem_odp for a userspace va</div><div class='add'>+ *</div><div class='add'>+ * @device: IB device struct to get UMEM</div><div class='add'>+ * @addr: userspace virtual address to start at</div><div class='add'>+ * @size: length of region to pin</div><div class='add'>+ * @access: IB_ACCESS_xxx flags for memory being pinned</div><div class='add'>+ * @ops: MMU interval ops, currently only @invalidate</div><div class='add'>+ *</div><div class='add'>+ * The driver should use when the access flags indicate ODP memory. It avoids</div><div class='add'>+ * pinning, instead, stores the mm for future page fault handling in</div><div class='add'>+ * conjunction with MMU notifiers.</div><div class='add'>+ */</div><div class='add'>+struct ib_umem_odp *ib_umem_odp_get(struct ib_device *device,</div><div class='add'>+				    unsigned long addr, size_t size, int access,</div><div class='add'>+				    const struct mmu_interval_notifier_ops *ops)</div><div class='ctx'> {</div><div class='del'>-	struct ib_umem *umem = &amp;umem_odp-&gt;umem;</div><div class='del'>-	/*</div><div class='del'>-	 * NOTE: This must called in a process context where umem-&gt;owning_mm</div><div class='del'>-	 * == current-&gt;mm</div><div class='del'>-	 */</div><div class='del'>-	struct mm_struct *mm = umem-&gt;owning_mm;</div><div class='del'>-	int ret_val;</div><div class='del'>-</div><div class='del'>-	if (access &amp; IB_ACCESS_HUGETLB) {</div><div class='del'>-		struct vm_area_struct *vma;</div><div class='del'>-		struct hstate *h;</div><div class='del'>-</div><div class='del'>-		down_read(&amp;mm-&gt;mmap_sem);</div><div class='del'>-		vma = find_vma(mm, ib_umem_start(umem));</div><div class='del'>-		if (!vma || !is_vm_hugetlb_page(vma)) {</div><div class='del'>-			up_read(&amp;mm-&gt;mmap_sem);</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-		h = hstate_vma(vma);</div><div class='del'>-		umem-&gt;page_shift = huge_page_shift(h);</div><div class='del'>-		up_read(&amp;mm-&gt;mmap_sem);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mutex_init(&amp;umem_odp-&gt;umem_mutex);</div><div class='del'>-</div><div class='del'>-	init_completion(&amp;umem_odp-&gt;notifier_completion);</div><div class='del'>-</div><div class='del'>-	if (ib_umem_num_pages(umem)) {</div><div class='del'>-		umem_odp-&gt;page_list =</div><div class='del'>-			vzalloc(array_size(sizeof(*umem_odp-&gt;page_list),</div><div class='del'>-					   ib_umem_num_pages(umem)));</div><div class='del'>-		if (!umem_odp-&gt;page_list)</div><div class='del'>-			return -ENOMEM;</div><div class='del'>-</div><div class='del'>-		umem_odp-&gt;dma_list =</div><div class='del'>-			vzalloc(array_size(sizeof(*umem_odp-&gt;dma_list),</div><div class='del'>-					   ib_umem_num_pages(umem)));</div><div class='del'>-		if (!umem_odp-&gt;dma_list) {</div><div class='del'>-			ret_val = -ENOMEM;</div><div class='del'>-			goto out_page_list;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='add'>+	struct ib_umem_odp *umem_odp;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	ret_val = get_per_mm(umem_odp);</div><div class='del'>-	if (ret_val)</div><div class='del'>-		goto out_dma_list;</div><div class='del'>-	add_umem_to_per_mm(umem_odp);</div><div class='add'>+	if (WARN_ON_ONCE(!(access &amp; IB_ACCESS_ON_DEMAND)))</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	umem_odp = kzalloc(sizeof(struct ib_umem_odp), GFP_KERNEL);</div><div class='add'>+	if (!umem_odp)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='del'>-out_dma_list:</div><div class='del'>-	vfree(umem_odp-&gt;dma_list);</div><div class='del'>-out_page_list:</div><div class='del'>-	vfree(umem_odp-&gt;page_list);</div><div class='del'>-	return ret_val;</div><div class='add'>+	umem_odp-&gt;umem.ibdev = device;</div><div class='add'>+	umem_odp-&gt;umem.length = size;</div><div class='add'>+	umem_odp-&gt;umem.address = addr;</div><div class='add'>+	umem_odp-&gt;umem.writable = ib_access_writable(access);</div><div class='add'>+	umem_odp-&gt;umem.owning_mm = current-&gt;mm;</div><div class='add'>+	umem_odp-&gt;notifier.ops = ops;</div><div class='add'>+</div><div class='add'>+	umem_odp-&gt;page_shift = PAGE_SHIFT;</div><div class='add'>+#ifdef CONFIG_HUGETLB_PAGE</div><div class='add'>+	if (access &amp; IB_ACCESS_HUGETLB)</div><div class='add'>+		umem_odp-&gt;page_shift = HPAGE_SHIFT;</div><div class='add'>+#endif</div><div class='add'>+</div><div class='add'>+	umem_odp-&gt;tgid = get_task_pid(current-&gt;group_leader, PIDTYPE_PID);</div><div class='add'>+	ret = ib_init_umem_odp(umem_odp, ops);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_put_pid;</div><div class='add'>+	return umem_odp;</div><div class='add'>+</div><div class='add'>+err_put_pid:</div><div class='add'>+	put_pid(umem_odp-&gt;tgid);</div><div class='add'>+	kfree(umem_odp);</div><div class='add'>+	return ERR_PTR(ret);</div><div class='ctx'> }</div><div class='add'>+EXPORT_SYMBOL(ib_umem_odp_get);</div><div class='ctx'> </div><div class='del'>-void ib_umem_odp_release(struct ib_umem_odp *umem_odp)</div><div class='add'>+static void ib_umem_odp_free(struct ib_umem_odp *umem_odp)</div><div class='ctx'> {</div><div class='del'>-	struct ib_umem *umem = &amp;umem_odp-&gt;umem;</div><div class='add'>+	struct ib_device *dev = umem_odp-&gt;umem.ibdev;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Ensure that no more pages are mapped in the umem.</div><div class='hunk'>@@ -464,107 +281,36 @@ void ib_umem_odp_release(struct ib_umem_odp *umem_odp)</div><div class='ctx'> 	 * It is the driver's responsibility to ensure, before calling us,</div><div class='ctx'> 	 * that the hardware will not attempt to access the MR any more.</div><div class='ctx'> 	 */</div><div class='del'>-	ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem),</div><div class='del'>-				    ib_umem_end(umem));</div><div class='del'>-</div><div class='del'>-	remove_umem_from_per_mm(umem_odp);</div><div class='del'>-	put_per_mm(umem_odp);</div><div class='del'>-	vfree(umem_odp-&gt;dma_list);</div><div class='del'>-	vfree(umem_odp-&gt;page_list);</div><div class='add'>+	mutex_lock(&amp;umem_odp-&gt;umem_mutex);</div><div class='add'>+	ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem_odp),</div><div class='add'>+				    ib_umem_end(umem_odp));</div><div class='add'>+	mutex_unlock(&amp;umem_odp-&gt;umem_mutex);</div><div class='add'>+	mmu_interval_notifier_remove(&amp;umem_odp-&gt;notifier);</div><div class='add'>+	if (ib_uses_virt_dma(dev))</div><div class='add'>+		kvfree(umem_odp-&gt;map.pfn_list);</div><div class='add'>+	else</div><div class='add'>+		hmm_dma_map_free(dev-&gt;dma_device, &amp;umem_odp-&gt;map);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/*</div><div class='del'>- * Map for DMA and insert a single page into the on-demand paging page tables.</div><div class='del'>- *</div><div class='del'>- * @umem: the umem to insert the page to.</div><div class='del'>- * @page_index: index in the umem to add the page to.</div><div class='del'>- * @page: the page struct to map and add.</div><div class='del'>- * @access_mask: access permissions needed for this page.</div><div class='del'>- * @current_seq: sequence number for synchronization with invalidations.</div><div class='del'>- *               the sequence number is taken from</div><div class='del'>- *               umem_odp-&gt;notifiers_seq.</div><div class='del'>- *</div><div class='del'>- * The function returns -EFAULT if the DMA mapping operation fails. It returns</div><div class='del'>- * -EAGAIN if a concurrent invalidation prevents us from updating the page.</div><div class='del'>- *</div><div class='del'>- * The page is released via put_page even if the operation failed. For</div><div class='del'>- * on-demand pinning, the page is released whenever it isn't stored in the</div><div class='del'>- * umem.</div><div class='del'>- */</div><div class='del'>-static int ib_umem_odp_map_dma_single_page(</div><div class='del'>-		struct ib_umem_odp *umem_odp,</div><div class='del'>-		int page_index,</div><div class='del'>-		struct page *page,</div><div class='del'>-		u64 access_mask,</div><div class='del'>-		unsigned long current_seq)</div><div class='add'>+void ib_umem_odp_release(struct ib_umem_odp *umem_odp)</div><div class='ctx'> {</div><div class='del'>-	struct ib_umem *umem = &amp;umem_odp-&gt;umem;</div><div class='del'>-	struct ib_device *dev = umem-&gt;context-&gt;device;</div><div class='del'>-	dma_addr_t dma_addr;</div><div class='del'>-	int remove_existing_mapping = 0;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Note: we avoid writing if seq is different from the initial seq, to</div><div class='del'>-	 * handle case of a racing notifier. This check also allows us to bail</div><div class='del'>-	 * early if we have a notifier running in parallel with us.</div><div class='del'>-	 */</div><div class='del'>-	if (ib_umem_mmu_notifier_retry(umem_odp, current_seq)) {</div><div class='del'>-		ret = -EAGAIN;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-	if (!(umem_odp-&gt;dma_list[page_index])) {</div><div class='del'>-		dma_addr = ib_dma_map_page(dev,</div><div class='del'>-					   page,</div><div class='del'>-					   0, BIT(umem-&gt;page_shift),</div><div class='del'>-					   DMA_BIDIRECTIONAL);</div><div class='del'>-		if (ib_dma_mapping_error(dev, dma_addr)) {</div><div class='del'>-			ret = -EFAULT;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-		umem_odp-&gt;dma_list[page_index] = dma_addr | access_mask;</div><div class='del'>-		umem_odp-&gt;page_list[page_index] = page;</div><div class='del'>-		umem_odp-&gt;npages++;</div><div class='del'>-	} else if (umem_odp-&gt;page_list[page_index] == page) {</div><div class='del'>-		umem_odp-&gt;dma_list[page_index] |= access_mask;</div><div class='del'>-	} else {</div><div class='del'>-		pr_err("error: got different pages in IB device and from get_user_pages. IB device page: %p, gup page: %p\n",</div><div class='del'>-		       umem_odp-&gt;page_list[page_index], page);</div><div class='del'>-		/* Better remove the mapping now, to prevent any further</div><div class='del'>-		 * damage. */</div><div class='del'>-		remove_existing_mapping = 1;</div><div class='del'>-	}</div><div class='add'>+	if (!umem_odp-&gt;is_implicit_odp)</div><div class='add'>+		ib_umem_odp_free(umem_odp);</div><div class='ctx'> </div><div class='del'>-out:</div><div class='del'>-	put_page(page);</div><div class='del'>-</div><div class='del'>-	if (remove_existing_mapping) {</div><div class='del'>-		ib_umem_notifier_start_account(umem_odp);</div><div class='del'>-		umem-&gt;context-&gt;invalidate_range(</div><div class='del'>-			umem_odp,</div><div class='del'>-			ib_umem_start(umem) + (page_index &lt;&lt; umem-&gt;page_shift),</div><div class='del'>-			ib_umem_start(umem) +</div><div class='del'>-				((page_index + 1) &lt;&lt; umem-&gt;page_shift));</div><div class='del'>-		ib_umem_notifier_end_account(umem_odp);</div><div class='del'>-		ret = -EAGAIN;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='add'>+	put_pid(umem_odp-&gt;tgid);</div><div class='add'>+	kfree(umem_odp);</div><div class='ctx'> }</div><div class='add'>+EXPORT_SYMBOL(ib_umem_odp_release);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * ib_umem_odp_map_dma_pages - Pin and DMA map userspace memory in an ODP MR.</div><div class='add'>+ * ib_umem_odp_map_dma_and_lock - DMA map userspace memory in an ODP MR and lock it.</div><div class='ctx'>  *</div><div class='del'>- * Pins the range of pages passed in the argument, and maps them to</div><div class='del'>- * DMA addresses. The DMA addresses of the mapped pages is updated in</div><div class='del'>- * umem_odp-&gt;dma_list.</div><div class='add'>+ * Maps the range passed in the argument to DMA addresses.</div><div class='add'>+ * Upon success the ODP MR will be locked to let caller complete its device</div><div class='add'>+ * page table update.</div><div class='ctx'>  *</div><div class='ctx'>  * Returns the number of pages mapped in success, negative error code</div><div class='ctx'>  * for failure.</div><div class='del'>- * An -EAGAIN error code is returned when a concurrent mmu notifier prevents</div><div class='del'>- * the function from completing its task.</div><div class='del'>- * An -ENOENT error code indicates that userspace process is being terminated</div><div class='del'>- * and mm was already destroyed.</div><div class='ctx'>  * @umem_odp: the umem to map and pin</div><div class='ctx'>  * @user_virt: the address from which we need to map.</div><div class='ctx'>  * @bcnt: the minimal number of bytes to pin and map. The mapping might be</div><div class='hunk'>@@ -573,229 +319,151 @@ out:</div><div class='ctx'>  *        the return value.</div><div class='ctx'>  * @access_mask: bit mask of the requested access permissions for the given</div><div class='ctx'>  *               range.</div><div class='del'>- * @current_seq: the MMU notifiers sequance value for synchronization with</div><div class='del'>- *               invalidations. the sequance number is read from</div><div class='del'>- *               umem_odp-&gt;notifiers_seq before calling this function</div><div class='add'>+ * @fault: is faulting required for the given range</div><div class='ctx'>  */</div><div class='del'>-int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,</div><div class='del'>-			      u64 bcnt, u64 access_mask,</div><div class='del'>-			      unsigned long current_seq)</div><div class='add'>+int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt,</div><div class='add'>+				 u64 bcnt, u64 access_mask, bool fault)</div><div class='add'>+			__acquires(&amp;umem_odp-&gt;umem_mutex)</div><div class='ctx'> {</div><div class='del'>-	struct ib_umem *umem = &amp;umem_odp-&gt;umem;</div><div class='ctx'> 	struct task_struct *owning_process  = NULL;</div><div class='ctx'> 	struct mm_struct *owning_mm = umem_odp-&gt;umem.owning_mm;</div><div class='del'>-	struct page       **local_page_list = NULL;</div><div class='del'>-	u64 page_mask, off;</div><div class='del'>-	int j, k, ret = 0, start_idx, npages = 0, page_shift;</div><div class='del'>-	unsigned int flags = 0;</div><div class='del'>-	phys_addr_t p = 0;</div><div class='del'>-</div><div class='del'>-	if (access_mask == 0)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	if (user_virt &lt; ib_umem_start(umem) ||</div><div class='del'>-	    user_virt + bcnt &gt; ib_umem_end(umem))</div><div class='add'>+	int pfn_index, dma_index, ret = 0, start_idx;</div><div class='add'>+	unsigned int page_shift, hmm_order, pfn_start_idx;</div><div class='add'>+	unsigned long num_pfns, current_seq;</div><div class='add'>+	struct hmm_range range = {};</div><div class='add'>+	unsigned long timeout;</div><div class='add'>+</div><div class='add'>+	if (user_virt &lt; ib_umem_start(umem_odp) ||</div><div class='add'>+	    user_virt + bcnt &gt; ib_umem_end(umem_odp))</div><div class='ctx'> 		return -EFAULT;</div><div class='ctx'> </div><div class='del'>-	local_page_list = (struct page **)__get_free_page(GFP_KERNEL);</div><div class='del'>-	if (!local_page_list)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	page_shift = umem-&gt;page_shift;</div><div class='del'>-	page_mask = ~(BIT(page_shift) - 1);</div><div class='del'>-	off = user_virt &amp; (~page_mask);</div><div class='del'>-	user_virt = user_virt &amp; page_mask;</div><div class='del'>-	bcnt += off; /* Charge for the first page offset as well. */</div><div class='add'>+	page_shift = umem_odp-&gt;page_shift;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * owning_process is allowed to be NULL, this means somehow the mm is</div><div class='ctx'> 	 * existing beyond the lifetime of the originating process.. Presumably</div><div class='ctx'> 	 * mmget_not_zero will fail in this case.</div><div class='ctx'> 	 */</div><div class='del'>-	owning_process = get_pid_task(umem_odp-&gt;per_mm-&gt;tgid, PIDTYPE_PID);</div><div class='add'>+	owning_process = get_pid_task(umem_odp-&gt;tgid, PIDTYPE_PID);</div><div class='ctx'> 	if (!owning_process || !mmget_not_zero(owning_mm)) {</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		goto out_put_task;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (access_mask &amp; ODP_WRITE_ALLOWED_BIT)</div><div class='del'>-		flags |= FOLL_WRITE;</div><div class='add'>+	range.notifier = &amp;umem_odp-&gt;notifier;</div><div class='add'>+	range.start = ALIGN_DOWN(user_virt, 1UL &lt;&lt; page_shift);</div><div class='add'>+	range.end = ALIGN(user_virt + bcnt, 1UL &lt;&lt; page_shift);</div><div class='add'>+	pfn_start_idx = (range.start - ib_umem_start(umem_odp)) &gt;&gt; PAGE_SHIFT;</div><div class='add'>+	num_pfns = (range.end - range.start) &gt;&gt; PAGE_SHIFT;</div><div class='add'>+	if (fault) {</div><div class='add'>+		range.default_flags = HMM_PFN_REQ_FAULT;</div><div class='add'>+</div><div class='add'>+		if (access_mask &amp; HMM_PFN_WRITE)</div><div class='add'>+			range.default_flags |= HMM_PFN_REQ_WRITE;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	range.hmm_pfns = &amp;(umem_odp-&gt;map.pfn_list[pfn_start_idx]);</div><div class='add'>+	timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);</div><div class='add'>+</div><div class='add'>+retry:</div><div class='add'>+	current_seq = range.notifier_seq =</div><div class='add'>+		mmu_interval_read_begin(&amp;umem_odp-&gt;notifier);</div><div class='ctx'> </div><div class='del'>-	start_idx = (user_virt - ib_umem_start(umem)) &gt;&gt; page_shift;</div><div class='del'>-	k = start_idx;</div><div class='add'>+	mmap_read_lock(owning_mm);</div><div class='add'>+	ret = hmm_range_fault(&amp;range);</div><div class='add'>+	mmap_read_unlock(owning_mm);</div><div class='add'>+	if (unlikely(ret)) {</div><div class='add'>+		if (ret == -EBUSY &amp;&amp; !time_after(jiffies, timeout))</div><div class='add'>+			goto retry;</div><div class='add'>+		goto out_put_mm;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	while (bcnt &gt; 0) {</div><div class='del'>-		const size_t gup_num_pages = min_t(size_t,</div><div class='del'>-				(bcnt + BIT(page_shift) - 1) &gt;&gt; page_shift,</div><div class='del'>-				PAGE_SIZE / sizeof(struct page *));</div><div class='add'>+	start_idx = (range.start - ib_umem_start(umem_odp)) &gt;&gt; page_shift;</div><div class='add'>+	dma_index = start_idx;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;umem_odp-&gt;umem_mutex);</div><div class='add'>+	if (mmu_interval_read_retry(&amp;umem_odp-&gt;notifier, current_seq)) {</div><div class='add'>+		mutex_unlock(&amp;umem_odp-&gt;umem_mutex);</div><div class='add'>+		goto retry;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (pfn_index = 0; pfn_index &lt; num_pfns;</div><div class='add'>+		pfn_index += 1 &lt;&lt; (page_shift - PAGE_SHIFT), dma_index++) {</div><div class='ctx'> </div><div class='del'>-		down_read(&amp;owning_mm-&gt;mmap_sem);</div><div class='ctx'> 		/*</div><div class='del'>-		 * Note: this might result in redundent page getting. We can</div><div class='del'>-		 * avoid this by checking dma_list to be 0 before calling</div><div class='del'>-		 * get_user_pages. However, this make the code much more</div><div class='del'>-		 * complex (and doesn't gain us much performance in most use</div><div class='del'>-		 * cases).</div><div class='add'>+		 * Since we asked for hmm_range_fault() to populate</div><div class='add'>+		 * pages it shouldn't return an error entry on success.</div><div class='ctx'> 		 */</div><div class='del'>-		npages = get_user_pages_remote(owning_process, owning_mm,</div><div class='del'>-				user_virt, gup_num_pages,</div><div class='del'>-				flags, local_page_list, NULL, NULL);</div><div class='del'>-		up_read(&amp;owning_mm-&gt;mmap_sem);</div><div class='del'>-</div><div class='del'>-		if (npages &lt; 0) {</div><div class='del'>-			if (npages != -EAGAIN)</div><div class='del'>-				pr_warn("fail to get %zu user pages with error %d\n", gup_num_pages, npages);</div><div class='del'>-			else</div><div class='del'>-				pr_debug("fail to get %zu user pages with error %d\n", gup_num_pages, npages);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='add'>+		WARN_ON(fault &amp;&amp; range.hmm_pfns[pfn_index] &amp; HMM_PFN_ERROR);</div><div class='add'>+		WARN_ON(fault &amp;&amp; !(range.hmm_pfns[pfn_index] &amp; HMM_PFN_VALID));</div><div class='add'>+		if (!(range.hmm_pfns[pfn_index] &amp; HMM_PFN_VALID))</div><div class='add'>+			continue;</div><div class='ctx'> </div><div class='del'>-		bcnt -= min_t(size_t, npages &lt;&lt; PAGE_SHIFT, bcnt);</div><div class='del'>-		mutex_lock(&amp;umem_odp-&gt;umem_mutex);</div><div class='del'>-		for (j = 0; j &lt; npages; j++, user_virt += PAGE_SIZE) {</div><div class='del'>-			if (user_virt &amp; ~page_mask) {</div><div class='del'>-				p += PAGE_SIZE;</div><div class='del'>-				if (page_to_phys(local_page_list[j]) != p) {</div><div class='del'>-					ret = -EFAULT;</div><div class='del'>-					break;</div><div class='del'>-				}</div><div class='del'>-				put_page(local_page_list[j]);</div><div class='del'>-				continue;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			ret = ib_umem_odp_map_dma_single_page(</div><div class='del'>-					umem_odp, k, local_page_list[j],</div><div class='del'>-					access_mask, current_seq);</div><div class='del'>-			if (ret &lt; 0) {</div><div class='del'>-				if (ret != -EAGAIN)</div><div class='del'>-					pr_warn("ib_umem_odp_map_dma_single_page failed with error %d\n", ret);</div><div class='del'>-				else</div><div class='del'>-					pr_debug("ib_umem_odp_map_dma_single_page failed with error %d\n", ret);</div><div class='del'>-				break;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			p = page_to_phys(local_page_list[j]);</div><div class='del'>-			k++;</div><div class='del'>-		}</div><div class='del'>-		mutex_unlock(&amp;umem_odp-&gt;umem_mutex);</div><div class='add'>+		if (range.hmm_pfns[pfn_index] &amp; HMM_PFN_DMA_MAPPED)</div><div class='add'>+			continue;</div><div class='ctx'> </div><div class='del'>-		if (ret &lt; 0) {</div><div class='del'>-			/*</div><div class='del'>-			 * Release pages, remembering that the first page</div><div class='del'>-			 * to hit an error was already released by</div><div class='del'>-			 * ib_umem_odp_map_dma_single_page().</div><div class='del'>-			 */</div><div class='del'>-			if (npages - (j + 1) &gt; 0)</div><div class='del'>-				release_pages(&amp;local_page_list[j+1],</div><div class='del'>-					      npages - (j + 1));</div><div class='add'>+		hmm_order = hmm_pfn_to_map_order(range.hmm_pfns[pfn_index]);</div><div class='add'>+		/* If a hugepage was detected and ODP wasn't set for, the umem</div><div class='add'>+		 * page_shift will be used, the opposite case is an error.</div><div class='add'>+		 */</div><div class='add'>+		if (hmm_order + PAGE_SHIFT &lt; page_shift) {</div><div class='add'>+			ret = -EINVAL;</div><div class='add'>+			ibdev_dbg(umem_odp-&gt;umem.ibdev,</div><div class='add'>+				  "%s: un-expected hmm_order %u, page_shift %u\n",</div><div class='add'>+				  __func__, hmm_order, page_shift);</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='add'>+	/* upon success lock should stay on hold for the callee */</div><div class='add'>+	if (!ret)</div><div class='add'>+		ret = dma_index - start_idx;</div><div class='add'>+	else</div><div class='add'>+		mutex_unlock(&amp;umem_odp-&gt;umem_mutex);</div><div class='ctx'> </div><div class='del'>-	if (ret &gt;= 0) {</div><div class='del'>-		if (npages &lt; 0 &amp;&amp; k == start_idx)</div><div class='del'>-			ret = npages;</div><div class='del'>-		else</div><div class='del'>-			ret = k - start_idx;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mmput(owning_mm);</div><div class='add'>+out_put_mm:</div><div class='add'>+	mmput_async(owning_mm);</div><div class='ctx'> out_put_task:</div><div class='ctx'> 	if (owning_process)</div><div class='ctx'> 		put_task_struct(owning_process);</div><div class='del'>-	free_page((unsigned long)local_page_list);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(ib_umem_odp_map_dma_pages);</div><div class='add'>+EXPORT_SYMBOL(ib_umem_odp_map_dma_and_lock);</div><div class='ctx'> </div><div class='ctx'> void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 virt,</div><div class='ctx'> 				 u64 bound)</div><div class='ctx'> {</div><div class='del'>-	struct ib_umem *umem = &amp;umem_odp-&gt;umem;</div><div class='del'>-	int idx;</div><div class='add'>+	struct ib_device *dev = umem_odp-&gt;umem.ibdev;</div><div class='ctx'> 	u64 addr;</div><div class='del'>-	struct ib_device *dev = umem-&gt;context-&gt;device;</div><div class='del'>-</div><div class='del'>-	virt  = max_t(u64, virt,  ib_umem_start(umem));</div><div class='del'>-	bound = min_t(u64, bound, ib_umem_end(umem));</div><div class='del'>-	/* Note that during the run of this function, the</div><div class='del'>-	 * notifiers_count of the MR is &gt; 0, preventing any racing</div><div class='del'>-	 * faults from completion. We might be racing with other</div><div class='del'>-	 * invalidations, so we must make sure we free each page only</div><div class='del'>-	 * once. */</div><div class='del'>-	mutex_lock(&amp;umem_odp-&gt;umem_mutex);</div><div class='del'>-	for (addr = virt; addr &lt; bound; addr += BIT(umem-&gt;page_shift)) {</div><div class='del'>-		idx = (addr - ib_umem_start(umem)) &gt;&gt; umem-&gt;page_shift;</div><div class='del'>-		if (umem_odp-&gt;page_list[idx]) {</div><div class='del'>-			struct page *page = umem_odp-&gt;page_list[idx];</div><div class='del'>-			dma_addr_t dma = umem_odp-&gt;dma_list[idx];</div><div class='del'>-			dma_addr_t dma_addr = dma &amp; ODP_DMA_ADDR_MASK;</div><div class='del'>-</div><div class='del'>-			WARN_ON(!dma_addr);</div><div class='del'>-</div><div class='del'>-			ib_dma_unmap_page(dev, dma_addr, PAGE_SIZE,</div><div class='del'>-					  DMA_BIDIRECTIONAL);</div><div class='del'>-			if (dma &amp; ODP_WRITE_ALLOWED_BIT) {</div><div class='del'>-				struct page *head_page = compound_head(page);</div><div class='del'>-				/*</div><div class='del'>-				 * set_page_dirty prefers being called with</div><div class='del'>-				 * the page lock. However, MMU notifiers are</div><div class='del'>-				 * called sometimes with and sometimes without</div><div class='del'>-				 * the lock. We rely on the umem_mutex instead</div><div class='del'>-				 * to prevent other mmu notifiers from</div><div class='del'>-				 * continuing and allowing the page mapping to</div><div class='del'>-				 * be removed.</div><div class='del'>-				 */</div><div class='del'>-				set_page_dirty(head_page);</div><div class='del'>-			}</div><div class='del'>-			umem_odp-&gt;page_list[idx] = NULL;</div><div class='del'>-			umem_odp-&gt;dma_list[idx] = 0;</div><div class='del'>-			umem_odp-&gt;npages--;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	mutex_unlock(&amp;umem_odp-&gt;umem_mutex);</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_umem_odp_unmap_dma_pages);</div><div class='ctx'> </div><div class='del'>-/* @last is not a part of the interval. See comment for function</div><div class='del'>- * node_last.</div><div class='del'>- */</div><div class='del'>-int rbt_ib_umem_for_each_in_range(struct rb_root_cached *root,</div><div class='del'>-				  u64 start, u64 last,</div><div class='del'>-				  umem_call_back cb,</div><div class='del'>-				  bool blockable,</div><div class='del'>-				  void *cookie)</div><div class='del'>-{</div><div class='del'>-	int ret_val = 0;</div><div class='del'>-	struct umem_odp_node *node, *next;</div><div class='del'>-	struct ib_umem_odp *umem;</div><div class='del'>-</div><div class='del'>-	if (unlikely(start == last))</div><div class='del'>-		return ret_val;</div><div class='del'>-</div><div class='del'>-	for (node = rbt_ib_umem_iter_first(root, start, last - 1);</div><div class='del'>-			node; node = next) {</div><div class='del'>-		/* TODO move the blockable decision up to the callback */</div><div class='del'>-		if (!blockable)</div><div class='del'>-			return -EAGAIN;</div><div class='del'>-		next = rbt_ib_umem_iter_next(node, start, last - 1);</div><div class='del'>-		umem = container_of(node, struct ib_umem_odp, interval_tree);</div><div class='del'>-		ret_val = cb(umem, start, last, cookie) || ret_val;</div><div class='del'>-	}</div><div class='add'>+	lockdep_assert_held(&amp;umem_odp-&gt;umem_mutex);</div><div class='ctx'> </div><div class='del'>-	return ret_val;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(rbt_ib_umem_for_each_in_range);</div><div class='del'>-</div><div class='del'>-struct ib_umem_odp *rbt_ib_umem_lookup(struct rb_root_cached *root,</div><div class='del'>-				       u64 addr, u64 length)</div><div class='del'>-{</div><div class='del'>-	struct umem_odp_node *node;</div><div class='add'>+	virt = max_t(u64, virt, ib_umem_start(umem_odp));</div><div class='add'>+	bound = min_t(u64, bound, ib_umem_end(umem_odp));</div><div class='add'>+	for (addr = virt; addr &lt; bound; addr += BIT(umem_odp-&gt;page_shift)) {</div><div class='add'>+		u64 offset = addr - ib_umem_start(umem_odp);</div><div class='add'>+		size_t idx = offset &gt;&gt; umem_odp-&gt;page_shift;</div><div class='add'>+		unsigned long pfn = umem_odp-&gt;map.pfn_list[idx];</div><div class='ctx'> </div><div class='del'>-	node = rbt_ib_umem_iter_first(root, addr, addr + length - 1);</div><div class='del'>-	if (node)</div><div class='del'>-		return container_of(node, struct ib_umem_odp, interval_tree);</div><div class='del'>-	return NULL;</div><div class='add'>+		if (!hmm_dma_unmap_pfn(dev-&gt;dma_device, &amp;umem_odp-&gt;map, idx))</div><div class='add'>+			goto clear;</div><div class='ctx'> </div><div class='add'>+		if (pfn &amp; HMM_PFN_WRITE) {</div><div class='add'>+			struct page *page = hmm_pfn_to_page(pfn);</div><div class='add'>+			struct page *head_page = compound_head(page);</div><div class='add'>+			/*</div><div class='add'>+			 * set_page_dirty prefers being called with</div><div class='add'>+			 * the page lock. However, MMU notifiers are</div><div class='add'>+			 * called sometimes with and sometimes without</div><div class='add'>+			 * the lock. We rely on the umem_mutex instead</div><div class='add'>+			 * to prevent other mmu notifiers from</div><div class='add'>+			 * continuing and allowing the page mapping to</div><div class='add'>+			 * be removed.</div><div class='add'>+			 */</div><div class='add'>+			set_page_dirty(head_page);</div><div class='add'>+		}</div><div class='add'>+		umem_odp-&gt;npages--;</div><div class='add'>+clear:</div><div class='add'>+		umem_odp-&gt;map.pfn_list[idx] &amp;= ~HMM_PFN_FLAGS;</div><div class='add'>+	}</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(rbt_ib_umem_lookup);</div><div class='add'>+EXPORT_SYMBOL(ib_umem_odp_unmap_dma_pages);</div><div class='head'>diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c<br/>index 671f07ba1fad..fd67fc9fe85a 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/user_mad.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/user_mad.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/user_mad.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/user_mad.c</a></div><div class='hunk'>@@ -49,11 +49,13 @@</div><div class='ctx'> #include &lt;linux/sched.h&gt;</div><div class='ctx'> #include &lt;linux/semaphore.h&gt;</div><div class='ctx'> #include &lt;linux/slab.h&gt;</div><div class='add'>+#include &lt;linux/nospec.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/uaccess.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include &lt;rdma/ib_mad.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_user_mad.h&gt;</div><div class='add'>+#include &lt;rdma/rdma_netlink.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "core_priv.h"</div><div class='ctx'> </div><div class='hunk'>@@ -61,6 +63,8 @@ MODULE_AUTHOR("Roland Dreier");</div><div class='ctx'> MODULE_DESCRIPTION("InfiniBand userspace MAD packet access");</div><div class='ctx'> MODULE_LICENSE("Dual BSD/GPL");</div><div class='ctx'> </div><div class='add'>+#define MAX_UMAD_RECV_LIST_SIZE 200000</div><div class='add'>+</div><div class='ctx'> enum {</div><div class='ctx'> 	IB_UMAD_MAX_PORTS  = RDMA_MAX_PORTS,</div><div class='ctx'> 	IB_UMAD_MAX_AGENTS = 32,</div><div class='hunk'>@@ -99,7 +103,7 @@ struct ib_umad_port {</div><div class='ctx'> 	struct ib_device      *ib_dev;</div><div class='ctx'> 	struct ib_umad_device *umad_dev;</div><div class='ctx'> 	int                    dev_num;</div><div class='del'>-	u8                     port_num;</div><div class='add'>+	u32                     port_num;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct ib_umad_device {</div><div class='hunk'>@@ -111,6 +115,7 @@ struct ib_umad_file {</div><div class='ctx'> 	struct mutex		mutex;</div><div class='ctx'> 	struct ib_umad_port    *port;</div><div class='ctx'> 	struct list_head	recv_list;</div><div class='add'>+	atomic_t		recv_list_size;</div><div class='ctx'> 	struct list_head	send_list;</div><div class='ctx'> 	struct list_head	port_list;</div><div class='ctx'> 	spinlock_t		send_lock;</div><div class='hunk'>@@ -129,6 +134,11 @@ struct ib_umad_packet {</div><div class='ctx'> 	struct ib_user_mad mad;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct ib_rmpp_mad_hdr {</div><div class='add'>+	struct ib_mad_hdr	mad_hdr;</div><div class='add'>+	struct ib_rmpp_hdr      rmpp_hdr;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='ctx'> #define CREATE_TRACE_POINTS</div><div class='ctx'> #include &lt;trace/events/ib_umad.h&gt;</div><div class='ctx'> </div><div class='hunk'>@@ -140,7 +150,7 @@ static dev_t dynamic_issm_dev;</div><div class='ctx'> </div><div class='ctx'> static DEFINE_IDA(umad_ida);</div><div class='ctx'> </div><div class='del'>-static void ib_umad_add_one(struct ib_device *device);</div><div class='add'>+static int ib_umad_add_one(struct ib_device *device);</div><div class='ctx'> static void ib_umad_remove_one(struct ib_device *device, void *client_data);</div><div class='ctx'> </div><div class='ctx'> static void ib_umad_dev_free(struct kref *kref)</div><div class='hunk'>@@ -163,8 +173,8 @@ static void ib_umad_dev_put(struct ib_umad_device *dev)</div><div class='ctx'> </div><div class='ctx'> static int hdr_size(struct ib_umad_file *file)</div><div class='ctx'> {</div><div class='del'>-	return file-&gt;use_pkey_index ? sizeof (struct ib_user_mad_hdr) :</div><div class='del'>-		sizeof (struct ib_user_mad_hdr_old);</div><div class='add'>+	return file-&gt;use_pkey_index ? sizeof(struct ib_user_mad_hdr) :</div><div class='add'>+				      sizeof(struct ib_user_mad_hdr_old);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* caller must hold file-&gt;mutex */</div><div class='hunk'>@@ -173,24 +183,28 @@ static struct ib_mad_agent *__get_agent(struct ib_umad_file *file, int id)</div><div class='ctx'> 	return file-&gt;agents_dead ? NULL : file-&gt;agent[id];</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int queue_packet(struct ib_umad_file *file,</div><div class='del'>-			struct ib_mad_agent *agent,</div><div class='del'>-			struct ib_umad_packet *packet)</div><div class='add'>+static int queue_packet(struct ib_umad_file *file, struct ib_mad_agent *agent,</div><div class='add'>+			struct ib_umad_packet *packet, bool is_recv_mad)</div><div class='ctx'> {</div><div class='ctx'> 	int ret = 1;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;file-&gt;mutex);</div><div class='ctx'> </div><div class='add'>+	if (is_recv_mad &amp;&amp;</div><div class='add'>+	    atomic_read(&amp;file-&gt;recv_list_size) &gt; MAX_UMAD_RECV_LIST_SIZE)</div><div class='add'>+		goto unlock;</div><div class='add'>+</div><div class='ctx'> 	for (packet-&gt;mad.hdr.id = 0;</div><div class='ctx'> 	     packet-&gt;mad.hdr.id &lt; IB_UMAD_MAX_AGENTS;</div><div class='ctx'> 	     packet-&gt;mad.hdr.id++)</div><div class='ctx'> 		if (agent == __get_agent(file, packet-&gt;mad.hdr.id)) {</div><div class='ctx'> 			list_add_tail(&amp;packet-&gt;list, &amp;file-&gt;recv_list);</div><div class='add'>+			atomic_inc(&amp;file-&gt;recv_list_size);</div><div class='ctx'> 			wake_up_interruptible(&amp;file-&gt;recv_wait);</div><div class='ctx'> 			ret = 0;</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='del'>-</div><div class='add'>+unlock:</div><div class='ctx'> 	mutex_unlock(&amp;file-&gt;mutex);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -217,7 +231,7 @@ static void send_handler(struct ib_mad_agent *agent,</div><div class='ctx'> 	if (send_wc-&gt;status == IB_WC_RESP_TIMEOUT_ERR) {</div><div class='ctx'> 		packet-&gt;length = IB_MGMT_MAD_HDR;</div><div class='ctx'> 		packet-&gt;mad.hdr.status = ETIMEDOUT;</div><div class='del'>-		if (!queue_packet(file, agent, packet))</div><div class='add'>+		if (!queue_packet(file, agent, packet, false))</div><div class='ctx'> 			return;</div><div class='ctx'> 	}</div><div class='ctx'> 	kfree(packet);</div><div class='hunk'>@@ -277,7 +291,7 @@ static void recv_handler(struct ib_mad_agent *agent,</div><div class='ctx'> 		rdma_destroy_ah_attr(&amp;ah_attr);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (queue_packet(file, agent, packet))</div><div class='add'>+	if (queue_packet(file, agent, packet, true))</div><div class='ctx'> 		goto err2;</div><div class='ctx'> 	return;</div><div class='ctx'> </div><div class='hunk'>@@ -377,6 +391,11 @@ static ssize_t ib_umad_read(struct file *filp, char __user *buf,</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;file-&gt;mutex);</div><div class='ctx'> </div><div class='add'>+	if (file-&gt;agents_dead) {</div><div class='add'>+		mutex_unlock(&amp;file-&gt;mutex);</div><div class='add'>+		return -EIO;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	while (list_empty(&amp;file-&gt;recv_list)) {</div><div class='ctx'> 		mutex_unlock(&amp;file-&gt;mutex);</div><div class='ctx'> </div><div class='hunk'>@@ -390,8 +409,14 @@ static ssize_t ib_umad_read(struct file *filp, char __user *buf,</div><div class='ctx'> 		mutex_lock(&amp;file-&gt;mutex);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (file-&gt;agents_dead) {</div><div class='add'>+		mutex_unlock(&amp;file-&gt;mutex);</div><div class='add'>+		return -EIO;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	packet = list_entry(file-&gt;recv_list.next, struct ib_umad_packet, list);</div><div class='ctx'> 	list_del(&amp;packet-&gt;list);</div><div class='add'>+	atomic_dec(&amp;file-&gt;recv_list_size);</div><div class='ctx'> </div><div class='ctx'> 	mutex_unlock(&amp;file-&gt;mutex);</div><div class='ctx'> </div><div class='hunk'>@@ -404,6 +429,7 @@ static ssize_t ib_umad_read(struct file *filp, char __user *buf,</div><div class='ctx'> 		/* Requeue packet */</div><div class='ctx'> 		mutex_lock(&amp;file-&gt;mutex);</div><div class='ctx'> 		list_add(&amp;packet-&gt;list, &amp;file-&gt;recv_list);</div><div class='add'>+		atomic_inc(&amp;file-&gt;recv_list_size);</div><div class='ctx'> 		mutex_unlock(&amp;file-&gt;mutex);</div><div class='ctx'> 	} else {</div><div class='ctx'> 		if (packet-&gt;recv_wc)</div><div class='hunk'>@@ -482,11 +508,11 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,</div><div class='ctx'> 			     size_t count, loff_t *pos)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_umad_file *file = filp-&gt;private_data;</div><div class='add'>+	struct ib_rmpp_mad_hdr *rmpp_mad_hdr;</div><div class='ctx'> 	struct ib_umad_packet *packet;</div><div class='ctx'> 	struct ib_mad_agent *agent;</div><div class='ctx'> 	struct rdma_ah_attr ah_attr;</div><div class='ctx'> 	struct ib_ah *ah;</div><div class='del'>-	struct ib_rmpp_mad *rmpp_mad;</div><div class='ctx'> 	__be64 *tid;</div><div class='ctx'> 	int ret, data_len, hdr_len, copy_offset, rmpp_active;</div><div class='ctx'> 	u8 base_version;</div><div class='hunk'>@@ -494,7 +520,7 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,</div><div class='ctx'> 	if (count &lt; hdr_size(file) + IB_MGMT_RMPP_HDR)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	packet = kzalloc(sizeof *packet + IB_MGMT_RMPP_HDR, GFP_KERNEL);</div><div class='add'>+	packet = kzalloc(sizeof(*packet) + IB_MGMT_RMPP_HDR, GFP_KERNEL);</div><div class='ctx'> 	if (!packet)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='hunk'>@@ -522,7 +548,7 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,</div><div class='ctx'> </div><div class='ctx'> 	agent = __get_agent(file, packet-&gt;mad.hdr.id);</div><div class='ctx'> 	if (!agent) {</div><div class='del'>-		ret = -EINVAL;</div><div class='add'>+		ret = -EIO;</div><div class='ctx'> 		goto err_up;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -548,13 +574,13 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,</div><div class='ctx'> 		goto err_up;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	rmpp_mad = (struct ib_rmpp_mad *) packet-&gt;mad.data;</div><div class='del'>-	hdr_len = ib_get_mad_data_offset(rmpp_mad-&gt;mad_hdr.mgmt_class);</div><div class='add'>+	rmpp_mad_hdr = (struct ib_rmpp_mad_hdr *)packet-&gt;mad.data;</div><div class='add'>+	hdr_len = ib_get_mad_data_offset(rmpp_mad_hdr-&gt;mad_hdr.mgmt_class);</div><div class='ctx'> </div><div class='del'>-	if (ib_is_mad_class_rmpp(rmpp_mad-&gt;mad_hdr.mgmt_class)</div><div class='add'>+	if (ib_is_mad_class_rmpp(rmpp_mad_hdr-&gt;mad_hdr.mgmt_class)</div><div class='ctx'> 	    &amp;&amp; ib_mad_kernel_rmpp_agent(agent)) {</div><div class='ctx'> 		copy_offset = IB_MGMT_RMPP_HDR;</div><div class='del'>-		rmpp_active = ib_get_rmpp_flags(&amp;rmpp_mad-&gt;rmpp_hdr) &amp;</div><div class='add'>+		rmpp_active = ib_get_rmpp_flags(&amp;rmpp_mad_hdr-&gt;rmpp_hdr) &amp;</div><div class='ctx'> 						IB_MGMT_RMPP_FLAG_ACTIVE;</div><div class='ctx'> 	} else {</div><div class='ctx'> 		copy_offset = IB_MGMT_MAD_HDR;</div><div class='hunk'>@@ -603,12 +629,12 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,</div><div class='ctx'> 		tid = &amp;((struct ib_mad_hdr *) packet-&gt;msg-&gt;mad)-&gt;tid;</div><div class='ctx'> 		*tid = cpu_to_be64(((u64) agent-&gt;hi_tid) &lt;&lt; 32 |</div><div class='ctx'> 				   (be64_to_cpup(tid) &amp; 0xffffffff));</div><div class='del'>-		rmpp_mad-&gt;mad_hdr.tid = *tid;</div><div class='add'>+		rmpp_mad_hdr-&gt;mad_hdr.tid = *tid;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (!ib_mad_kernel_rmpp_agent(agent)</div><div class='del'>-	   &amp;&amp; ib_is_mad_class_rmpp(rmpp_mad-&gt;mad_hdr.mgmt_class)</div><div class='del'>-	   &amp;&amp; (ib_get_rmpp_flags(&amp;rmpp_mad-&gt;rmpp_hdr) &amp; IB_MGMT_RMPP_FLAG_ACTIVE)) {</div><div class='add'>+	    &amp;&amp; ib_is_mad_class_rmpp(rmpp_mad_hdr-&gt;mad_hdr.mgmt_class)</div><div class='add'>+	    &amp;&amp; (ib_get_rmpp_flags(&amp;rmpp_mad_hdr-&gt;rmpp_hdr) &amp; IB_MGMT_RMPP_FLAG_ACTIVE)) {</div><div class='ctx'> 		spin_lock_irq(&amp;file-&gt;send_lock);</div><div class='ctx'> 		list_add_tail(&amp;packet-&gt;list, &amp;file-&gt;send_list);</div><div class='ctx'> 		spin_unlock_irq(&amp;file-&gt;send_lock);</div><div class='hunk'>@@ -651,10 +677,14 @@ static __poll_t ib_umad_poll(struct file *filp, struct poll_table_struct *wait)</div><div class='ctx'> 	/* we will always be able to post a MAD send */</div><div class='ctx'> 	__poll_t mask = EPOLLOUT | EPOLLWRNORM;</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;file-&gt;mutex);</div><div class='ctx'> 	poll_wait(filp, &amp;file-&gt;recv_wait, wait);</div><div class='ctx'> </div><div class='ctx'> 	if (!list_empty(&amp;file-&gt;recv_list))</div><div class='ctx'> 		mask |= EPOLLIN | EPOLLRDNORM;</div><div class='add'>+	if (file-&gt;agents_dead)</div><div class='add'>+		mask = EPOLLERR;</div><div class='add'>+	mutex_unlock(&amp;file-&gt;mutex);</div><div class='ctx'> </div><div class='ctx'> 	return mask;</div><div class='ctx'> }</div><div class='hunk'>@@ -672,8 +702,7 @@ static int ib_umad_reg_agent(struct ib_umad_file *file, void __user *arg,</div><div class='ctx'> 	mutex_lock(&amp;file-&gt;mutex);</div><div class='ctx'> </div><div class='ctx'> 	if (!file-&gt;port-&gt;ib_dev) {</div><div class='del'>-		dev_notice(&amp;file-&gt;port-&gt;dev,</div><div class='del'>-			   "ib_umad_reg_agent: invalid device\n");</div><div class='add'>+		dev_notice(&amp;file-&gt;port-&gt;dev, "%s: invalid device\n", __func__);</div><div class='ctx'> 		ret = -EPIPE;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -685,7 +714,7 @@ static int ib_umad_reg_agent(struct ib_umad_file *file, void __user *arg,</div><div class='ctx'> </div><div class='ctx'> 	if (ureq.qpn != 0 &amp;&amp; ureq.qpn != 1) {</div><div class='ctx'> 		dev_notice(&amp;file-&gt;port-&gt;dev,</div><div class='del'>-			   "ib_umad_reg_agent: invalid QPN %d specified\n",</div><div class='add'>+			   "%s: invalid QPN %u specified\n", __func__,</div><div class='ctx'> 			   ureq.qpn);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		goto out;</div><div class='hunk'>@@ -695,9 +724,9 @@ static int ib_umad_reg_agent(struct ib_umad_file *file, void __user *arg,</div><div class='ctx'> 		if (!__get_agent(file, agent_id))</div><div class='ctx'> 			goto found;</div><div class='ctx'> </div><div class='del'>-	dev_notice(&amp;file-&gt;port-&gt;dev,</div><div class='del'>-		   "ib_umad_reg_agent: Max Agents (%u) reached\n",</div><div class='add'>+	dev_notice(&amp;file-&gt;port-&gt;dev, "%s: Max Agents (%u) reached\n", __func__,</div><div class='ctx'> 		   IB_UMAD_MAX_AGENTS);</div><div class='add'>+</div><div class='ctx'> 	ret = -ENOMEM;</div><div class='ctx'> 	goto out;</div><div class='ctx'> </div><div class='hunk'>@@ -744,7 +773,7 @@ found:</div><div class='ctx'> 				"process %s did not enable P_Key index support.\n",</div><div class='ctx'> 				current-&gt;comm);</div><div class='ctx'> 			dev_warn(&amp;file-&gt;port-&gt;dev,</div><div class='del'>-				"   Documentation/infiniband/user_mad.txt has info on the new ABI.\n");</div><div class='add'>+				"   Documentation/infiniband/user_mad.rst has info on the new ABI.\n");</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -774,8 +803,7 @@ static int ib_umad_reg_agent2(struct ib_umad_file *file, void __user *arg)</div><div class='ctx'> 	mutex_lock(&amp;file-&gt;mutex);</div><div class='ctx'> </div><div class='ctx'> 	if (!file-&gt;port-&gt;ib_dev) {</div><div class='del'>-		dev_notice(&amp;file-&gt;port-&gt;dev,</div><div class='del'>-			   "ib_umad_reg_agent2: invalid device\n");</div><div class='add'>+		dev_notice(&amp;file-&gt;port-&gt;dev, "%s: invalid device\n", __func__);</div><div class='ctx'> 		ret = -EPIPE;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -786,17 +814,16 @@ static int ib_umad_reg_agent2(struct ib_umad_file *file, void __user *arg)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (ureq.qpn != 0 &amp;&amp; ureq.qpn != 1) {</div><div class='del'>-		dev_notice(&amp;file-&gt;port-&gt;dev,</div><div class='del'>-			   "ib_umad_reg_agent2: invalid QPN %d specified\n",</div><div class='del'>-			   ureq.qpn);</div><div class='add'>+		dev_notice(&amp;file-&gt;port-&gt;dev, "%s: invalid QPN %u specified\n",</div><div class='add'>+			   __func__, ureq.qpn);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (ureq.flags &amp; ~IB_USER_MAD_REG_FLAGS_CAP) {</div><div class='ctx'> 		dev_notice(&amp;file-&gt;port-&gt;dev,</div><div class='del'>-			   "ib_umad_reg_agent2 failed: invalid registration flags specified 0x%x; supported 0x%x\n",</div><div class='del'>-			   ureq.flags, IB_USER_MAD_REG_FLAGS_CAP);</div><div class='add'>+			   "%s failed: invalid registration flags specified 0x%x; supported 0x%x\n",</div><div class='add'>+			   __func__, ureq.flags, IB_USER_MAD_REG_FLAGS_CAP);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 		if (put_user((u32)IB_USER_MAD_REG_FLAGS_CAP,</div><div class='hunk'>@@ -811,8 +838,7 @@ static int ib_umad_reg_agent2(struct ib_umad_file *file, void __user *arg)</div><div class='ctx'> 		if (!__get_agent(file, agent_id))</div><div class='ctx'> 			goto found;</div><div class='ctx'> </div><div class='del'>-	dev_notice(&amp;file-&gt;port-&gt;dev,</div><div class='del'>-		   "ib_umad_reg_agent2: Max Agents (%u) reached\n",</div><div class='add'>+	dev_notice(&amp;file-&gt;port-&gt;dev, "%s: Max Agents (%u) reached\n", __func__,</div><div class='ctx'> 		   IB_UMAD_MAX_AGENTS);</div><div class='ctx'> 	ret = -ENOMEM;</div><div class='ctx'> 	goto out;</div><div class='hunk'>@@ -824,7 +850,7 @@ found:</div><div class='ctx'> 		req.mgmt_class_version = ureq.mgmt_class_version;</div><div class='ctx'> 		if (ureq.oui &amp; 0xff000000) {</div><div class='ctx'> 			dev_notice(&amp;file-&gt;port-&gt;dev,</div><div class='del'>-				   "ib_umad_reg_agent2 failed: oui invalid 0x%08x\n",</div><div class='add'>+				   "%s failed: oui invalid 0x%08x\n", __func__,</div><div class='ctx'> 				   ureq.oui);</div><div class='ctx'> 			ret = -EINVAL;</div><div class='ctx'> 			goto out;</div><div class='hunk'>@@ -883,11 +909,14 @@ static int ib_umad_unreg_agent(struct ib_umad_file *file, u32 __user *arg)</div><div class='ctx'> </div><div class='ctx'> 	if (get_user(id, arg))</div><div class='ctx'> 		return -EFAULT;</div><div class='add'>+	if (id &gt;= IB_UMAD_MAX_AGENTS)</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;file-&gt;port-&gt;file_mutex);</div><div class='ctx'> 	mutex_lock(&amp;file-&gt;mutex);</div><div class='ctx'> </div><div class='del'>-	if (id &gt;= IB_UMAD_MAX_AGENTS || !__get_agent(file, id)) {</div><div class='add'>+	id = array_index_nospec(id, IB_UMAD_MAX_AGENTS);</div><div class='add'>+	if (!__get_agent(file, id)) {</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -1037,7 +1066,7 @@ static int ib_umad_close(struct inode *inode, struct file *filp)</div><div class='ctx'> 				ib_unregister_mad_agent(file-&gt;agent[i]);</div><div class='ctx'> </div><div class='ctx'> 	mutex_unlock(&amp;file-&gt;port-&gt;file_mutex);</div><div class='del'>-</div><div class='add'>+	mutex_destroy(&amp;file-&gt;mutex);</div><div class='ctx'> 	kfree(file);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -1053,7 +1082,6 @@ static const struct file_operations umad_fops = {</div><div class='ctx'> #endif</div><div class='ctx'> 	.open		= ib_umad_open,</div><div class='ctx'> 	.release	= ib_umad_close,</div><div class='del'>-	.llseek		= no_llseek,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static int ib_umad_sm_open(struct inode *inode, struct file *filp)</div><div class='hunk'>@@ -1121,14 +1149,61 @@ static const struct file_operations umad_sm_fops = {</div><div class='ctx'> 	.owner	 = THIS_MODULE,</div><div class='ctx'> 	.open	 = ib_umad_sm_open,</div><div class='ctx'> 	.release = ib_umad_sm_close,</div><div class='del'>-	.llseek	 = no_llseek,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static struct ib_umad_port *get_port(struct ib_device *ibdev,</div><div class='add'>+				     struct ib_umad_device *umad_dev,</div><div class='add'>+				     u32 port)</div><div class='add'>+{</div><div class='add'>+	if (!umad_dev)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+	if (!rdma_is_port_valid(ibdev, port))</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+	if (!rdma_cap_ib_mad(ibdev, port))</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	return &amp;umad_dev-&gt;ports[port - rdma_start_port(ibdev)];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ib_umad_get_nl_info(struct ib_device *ibdev, void *client_data,</div><div class='add'>+			       struct ib_client_nl_info *res)</div><div class='add'>+{</div><div class='add'>+	struct ib_umad_port *port = get_port(ibdev, client_data, res-&gt;port);</div><div class='add'>+</div><div class='add'>+	if (IS_ERR(port))</div><div class='add'>+		return PTR_ERR(port);</div><div class='add'>+</div><div class='add'>+	res-&gt;abi = IB_USER_MAD_ABI_VERSION;</div><div class='add'>+	res-&gt;cdev = &amp;port-&gt;dev;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static struct ib_client umad_client = {</div><div class='ctx'> 	.name   = "umad",</div><div class='ctx'> 	.add    = ib_umad_add_one,</div><div class='del'>-	.remove = ib_umad_remove_one</div><div class='add'>+	.remove = ib_umad_remove_one,</div><div class='add'>+	.get_nl_info = ib_umad_get_nl_info,</div><div class='ctx'> };</div><div class='add'>+MODULE_ALIAS_RDMA_CLIENT("umad");</div><div class='add'>+</div><div class='add'>+static int ib_issm_get_nl_info(struct ib_device *ibdev, void *client_data,</div><div class='add'>+			       struct ib_client_nl_info *res)</div><div class='add'>+{</div><div class='add'>+	struct ib_umad_port *port = get_port(ibdev, client_data, res-&gt;port);</div><div class='add'>+</div><div class='add'>+	if (IS_ERR(port))</div><div class='add'>+		return PTR_ERR(port);</div><div class='add'>+</div><div class='add'>+	res-&gt;abi = IB_USER_MAD_ABI_VERSION;</div><div class='add'>+	res-&gt;cdev = &amp;port-&gt;sm_dev;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct ib_client issm_client = {</div><div class='add'>+	.name = "issm",</div><div class='add'>+	.get_nl_info = ib_issm_get_nl_info,</div><div class='add'>+};</div><div class='add'>+MODULE_ALIAS_RDMA_CLIENT("issm");</div><div class='ctx'> </div><div class='ctx'> static ssize_t ibdev_show(struct device *dev, struct device_attribute *attr,</div><div class='ctx'> 			  char *buf)</div><div class='hunk'>@@ -1138,7 +1213,7 @@ static ssize_t ibdev_show(struct device *dev, struct device_attribute *attr,</div><div class='ctx'> 	if (!port)</div><div class='ctx'> 		return -ENODEV;</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%s\n", dev_name(&amp;port-&gt;ib_dev-&gt;dev));</div><div class='add'>+	return sysfs_emit(buf, "%s\n", dev_name(&amp;port-&gt;ib_dev-&gt;dev));</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(ibdev);</div><div class='ctx'> </div><div class='hunk'>@@ -1150,7 +1225,7 @@ static ssize_t port_show(struct device *dev, struct device_attribute *attr,</div><div class='ctx'> 	if (!port)</div><div class='ctx'> 		return -ENODEV;</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%d\n", port-&gt;port_num);</div><div class='add'>+	return sysfs_emit(buf, "%d\n", port-&gt;port_num);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(port);</div><div class='ctx'> </div><div class='hunk'>@@ -1161,15 +1236,15 @@ static struct attribute *umad_class_dev_attrs[] = {</div><div class='ctx'> };</div><div class='ctx'> ATTRIBUTE_GROUPS(umad_class_dev);</div><div class='ctx'> </div><div class='del'>-static char *umad_devnode(struct device *dev, umode_t *mode)</div><div class='add'>+static char *umad_devnode(const struct device *dev, umode_t *mode)</div><div class='ctx'> {</div><div class='ctx'> 	return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t abi_version_show(struct class *class,</div><div class='del'>-				struct class_attribute *attr, char *buf)</div><div class='add'>+static ssize_t abi_version_show(const struct class *class,</div><div class='add'>+				const struct class_attribute *attr, char *buf)</div><div class='ctx'> {</div><div class='del'>-	return sprintf(buf, "%d\n", IB_USER_MAD_ABI_VERSION);</div><div class='add'>+	return sysfs_emit(buf, "%d\n", IB_USER_MAD_ABI_VERSION);</div><div class='ctx'> }</div><div class='ctx'> static CLASS_ATTR_RO(abi_version);</div><div class='ctx'> </div><div class='hunk'>@@ -1244,15 +1319,17 @@ static int ib_umad_init_port(struct ib_device *device, int port_num,</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err_cdev;</div><div class='ctx'> </div><div class='del'>-	ib_umad_init_port_dev(&amp;port-&gt;sm_dev, port, device);</div><div class='del'>-	port-&gt;sm_dev.devt = base_issm;</div><div class='del'>-	dev_set_name(&amp;port-&gt;sm_dev, "issm%d", port-&gt;dev_num);</div><div class='del'>-	cdev_init(&amp;port-&gt;sm_cdev, &amp;umad_sm_fops);</div><div class='del'>-	port-&gt;sm_cdev.owner = THIS_MODULE;</div><div class='add'>+	if (rdma_cap_ib_smi(device, port_num)) {</div><div class='add'>+		ib_umad_init_port_dev(&amp;port-&gt;sm_dev, port, device);</div><div class='add'>+		port-&gt;sm_dev.devt = base_issm;</div><div class='add'>+		dev_set_name(&amp;port-&gt;sm_dev, "issm%d", port-&gt;dev_num);</div><div class='add'>+		cdev_init(&amp;port-&gt;sm_cdev, &amp;umad_sm_fops);</div><div class='add'>+		port-&gt;sm_cdev.owner = THIS_MODULE;</div><div class='ctx'> </div><div class='del'>-	ret = cdev_device_add(&amp;port-&gt;sm_cdev, &amp;port-&gt;sm_dev);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_dev;</div><div class='add'>+		ret = cdev_device_add(&amp;port-&gt;sm_cdev, &amp;port-&gt;sm_dev);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_dev;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='hunk'>@@ -1268,8 +1345,15 @@ err_cdev:</div><div class='ctx'> static void ib_umad_kill_port(struct ib_umad_port *port)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_umad_file *file;</div><div class='add'>+	bool has_smi = false;</div><div class='ctx'> 	int id;</div><div class='ctx'> </div><div class='add'>+	if (rdma_cap_ib_smi(port-&gt;ib_dev, port-&gt;port_num)) {</div><div class='add'>+		cdev_device_del(&amp;port-&gt;sm_cdev, &amp;port-&gt;sm_dev);</div><div class='add'>+		has_smi = true;</div><div class='add'>+	}</div><div class='add'>+	cdev_device_del(&amp;port-&gt;cdev, &amp;port-&gt;dev);</div><div class='add'>+</div><div class='ctx'> 	mutex_lock(&amp;port-&gt;file_mutex);</div><div class='ctx'> </div><div class='ctx'> 	/* Mark ib_dev NULL and block ioctl or other file ops to progress</div><div class='hunk'>@@ -1280,6 +1364,7 @@ static void ib_umad_kill_port(struct ib_umad_port *port)</div><div class='ctx'> 	list_for_each_entry(file, &amp;port-&gt;file_list, port_list) {</div><div class='ctx'> 		mutex_lock(&amp;file-&gt;mutex);</div><div class='ctx'> 		file-&gt;agents_dead = 1;</div><div class='add'>+		wake_up_interruptible(&amp;file-&gt;recv_wait);</div><div class='ctx'> 		mutex_unlock(&amp;file-&gt;mutex);</div><div class='ctx'> </div><div class='ctx'> 		for (id = 0; id &lt; IB_UMAD_MAX_AGENTS; ++id)</div><div class='hunk'>@@ -1289,46 +1374,51 @@ static void ib_umad_kill_port(struct ib_umad_port *port)</div><div class='ctx'> </div><div class='ctx'> 	mutex_unlock(&amp;port-&gt;file_mutex);</div><div class='ctx'> </div><div class='del'>-	cdev_device_del(&amp;port-&gt;sm_cdev, &amp;port-&gt;sm_dev);</div><div class='del'>-	cdev_device_del(&amp;port-&gt;cdev, &amp;port-&gt;dev);</div><div class='ctx'> 	ida_free(&amp;umad_ida, port-&gt;dev_num);</div><div class='ctx'> </div><div class='ctx'> 	/* balances device_initialize() */</div><div class='del'>-	put_device(&amp;port-&gt;sm_dev);</div><div class='add'>+	if (has_smi)</div><div class='add'>+		put_device(&amp;port-&gt;sm_dev);</div><div class='ctx'> 	put_device(&amp;port-&gt;dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void ib_umad_add_one(struct ib_device *device)</div><div class='add'>+static int ib_umad_add_one(struct ib_device *device)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_umad_device *umad_dev;</div><div class='ctx'> 	int s, e, i;</div><div class='ctx'> 	int count = 0;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	s = rdma_start_port(device);</div><div class='ctx'> 	e = rdma_end_port(device);</div><div class='ctx'> </div><div class='del'>-	umad_dev = kzalloc(struct_size(umad_dev, ports, e - s + 1), GFP_KERNEL);</div><div class='add'>+	umad_dev = kzalloc(struct_size(umad_dev, ports,</div><div class='add'>+				       size_add(size_sub(e, s), 1)),</div><div class='add'>+			   GFP_KERNEL);</div><div class='ctx'> 	if (!umad_dev)</div><div class='del'>-		return;</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	kref_init(&amp;umad_dev-&gt;kref);</div><div class='ctx'> 	for (i = s; i &lt;= e; ++i) {</div><div class='ctx'> 		if (!rdma_cap_ib_mad(device, i))</div><div class='ctx'> 			continue;</div><div class='ctx'> </div><div class='del'>-		if (ib_umad_init_port(device, i, umad_dev,</div><div class='del'>-				      &amp;umad_dev-&gt;ports[i - s]))</div><div class='add'>+		ret = ib_umad_init_port(device, i, umad_dev,</div><div class='add'>+					&amp;umad_dev-&gt;ports[i - s]);</div><div class='add'>+		if (ret)</div><div class='ctx'> 			goto err;</div><div class='ctx'> </div><div class='ctx'> 		count++;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!count)</div><div class='add'>+	if (!count) {</div><div class='add'>+		ret = -EOPNOTSUPP;</div><div class='ctx'> 		goto free;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	ib_set_client_data(device, &amp;umad_client, umad_dev);</div><div class='ctx'> </div><div class='del'>-	return;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='ctx'> err:</div><div class='ctx'> 	while (--i &gt;= s) {</div><div class='hunk'>@@ -1340,6 +1430,7 @@ err:</div><div class='ctx'> free:</div><div class='ctx'> 	/* balances kref_init */</div><div class='ctx'> 	ib_umad_dev_put(umad_dev);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void ib_umad_remove_one(struct ib_device *device, void *client_data)</div><div class='hunk'>@@ -1347,9 +1438,6 @@ static void ib_umad_remove_one(struct ib_device *device, void *client_data)</div><div class='ctx'> 	struct ib_umad_device *umad_dev = client_data;</div><div class='ctx'> 	unsigned int i;</div><div class='ctx'> </div><div class='del'>-	if (!umad_dev)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='ctx'> 	rdma_for_each_port (device, i) {</div><div class='ctx'> 		if (rdma_cap_ib_mad(device, i))</div><div class='ctx'> 			ib_umad_kill_port(</div><div class='hunk'>@@ -1387,13 +1475,17 @@ static int __init ib_umad_init(void)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ret = ib_register_client(&amp;umad_client);</div><div class='del'>-	if (ret) {</div><div class='del'>-		pr_err("couldn't register ib_umad client\n");</div><div class='add'>+	if (ret)</div><div class='ctx'> 		goto out_class;</div><div class='del'>-	}</div><div class='add'>+</div><div class='add'>+	ret = ib_register_client(&amp;issm_client);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out_client;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='add'>+out_client:</div><div class='add'>+	ib_unregister_client(&amp;umad_client);</div><div class='ctx'> out_class:</div><div class='ctx'> 	class_unregister(&amp;umad_class);</div><div class='ctx'> </div><div class='hunk'>@@ -1411,6 +1503,7 @@ out:</div><div class='ctx'> </div><div class='ctx'> static void __exit ib_umad_cleanup(void)</div><div class='ctx'> {</div><div class='add'>+	ib_unregister_client(&amp;issm_client);</div><div class='ctx'> 	ib_unregister_client(&amp;umad_client);</div><div class='ctx'> 	class_unregister(&amp;umad_class);</div><div class='ctx'> 	unregister_chrdev_region(base_umad_dev,</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h<br/>index 1e5aeb39f774..797e2fcc8072 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs.h</a></div><div class='hunk'>@@ -97,8 +97,8 @@ ib_uverbs_init_udata_buf_or_null(struct ib_udata *udata,</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> struct ib_uverbs_device {</div><div class='del'>-	atomic_t				refcount;</div><div class='del'>-	int					num_comp_vectors;</div><div class='add'>+	refcount_t				refcount;</div><div class='add'>+	u32					num_comp_vectors;</div><div class='ctx'> 	struct completion			comp;</div><div class='ctx'> 	struct device				dev;</div><div class='ctx'> 	/* First group for device attributes, NULL terminated array */</div><div class='hunk'>@@ -111,7 +111,6 @@ struct ib_uverbs_device {</div><div class='ctx'> 	struct srcu_struct			disassociate_srcu;</div><div class='ctx'> 	struct mutex				lists_mutex; /* protect lists */</div><div class='ctx'> 	struct list_head			uverbs_file_list;</div><div class='del'>-	struct list_head			uverbs_events_file_list;</div><div class='ctx'> 	struct uverbs_api			*uapi;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -124,10 +123,9 @@ struct ib_uverbs_event_queue {</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct ib_uverbs_async_event_file {</div><div class='add'>+	struct ib_uobject			uobj;</div><div class='ctx'> 	struct ib_uverbs_event_queue		ev_queue;</div><div class='del'>-	struct ib_uverbs_file		       *uverbs_file;</div><div class='del'>-	struct kref				ref;</div><div class='del'>-	struct list_head			list;</div><div class='add'>+	struct ib_event_handler			event_handler;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct ib_uverbs_completion_event_file {</div><div class='hunk'>@@ -135,36 +133,6 @@ struct ib_uverbs_completion_event_file {</div><div class='ctx'> 	struct ib_uverbs_event_queue		ev_queue;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct ib_uverbs_file {</div><div class='del'>-	struct kref				ref;</div><div class='del'>-	struct ib_uverbs_device		       *device;</div><div class='del'>-	struct mutex				ucontext_lock;</div><div class='del'>-	/*</div><div class='del'>-	 * ucontext must be accessed via ib_uverbs_get_ucontext() or with</div><div class='del'>-	 * ucontext_lock held</div><div class='del'>-	 */</div><div class='del'>-	struct ib_ucontext		       *ucontext;</div><div class='del'>-	struct ib_event_handler			event_handler;</div><div class='del'>-	struct ib_uverbs_async_event_file       *async_file;</div><div class='del'>-	struct list_head			list;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * To access the uobjects list hw_destroy_rwsem must be held for write</div><div class='del'>-	 * OR hw_destroy_rwsem held for read AND uobjects_lock held.</div><div class='del'>-	 * hw_destroy_rwsem should be called across any destruction of the HW</div><div class='del'>-	 * object of an associated uobject.</div><div class='del'>-	 */</div><div class='del'>-	struct rw_semaphore	hw_destroy_rwsem;</div><div class='del'>-	spinlock_t		uobjects_lock;</div><div class='del'>-	struct list_head	uobjects;</div><div class='del'>-</div><div class='del'>-	struct mutex umap_lock;</div><div class='del'>-	struct list_head umaps;</div><div class='del'>-	struct page *disassociate_page;</div><div class='del'>-</div><div class='del'>-	struct xarray		idr;</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> struct ib_uverbs_event {</div><div class='ctx'> 	union {</div><div class='ctx'> 		struct ib_uverbs_async_event_desc	async;</div><div class='hunk'>@@ -183,6 +151,8 @@ struct ib_uverbs_mcast_entry {</div><div class='ctx'> </div><div class='ctx'> struct ib_uevent_object {</div><div class='ctx'> 	struct ib_uobject	uobject;</div><div class='add'>+	struct ib_uverbs_async_event_file *event_file;</div><div class='add'>+	/* List member for ib_uverbs_async_event_file list */</div><div class='ctx'> 	struct list_head	event_list;</div><div class='ctx'> 	u32			events_reported;</div><div class='ctx'> };</div><div class='hunk'>@@ -210,34 +180,35 @@ struct ib_uwq_object {</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct ib_ucq_object {</div><div class='del'>-	struct ib_uobject	uobject;</div><div class='add'>+	struct ib_uevent_object uevent;</div><div class='ctx'> 	struct list_head	comp_list;</div><div class='del'>-	struct list_head	async_list;</div><div class='ctx'> 	u32			comp_events_reported;</div><div class='del'>-	u32			async_events_reported;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> extern const struct file_operations uverbs_event_fops;</div><div class='add'>+extern const struct file_operations uverbs_async_event_fops;</div><div class='ctx'> void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue);</div><div class='del'>-struct file *ib_uverbs_alloc_async_event_file(struct ib_uverbs_file *uverbs_file,</div><div class='del'>-					      struct ib_device *ib_dev);</div><div class='del'>-void ib_uverbs_free_async_event_file(struct ib_uverbs_file *uverbs_file);</div><div class='add'>+void ib_uverbs_init_async_event_file(struct ib_uverbs_async_event_file *ev_file);</div><div class='add'>+void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue);</div><div class='ctx'> void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res);</div><div class='add'>+int uverbs_async_event_release(struct inode *inode, struct file *filp);</div><div class='ctx'> </div><div class='del'>-void ib_uverbs_release_ucq(struct ib_uverbs_file *file,</div><div class='del'>-			   struct ib_uverbs_completion_event_file *ev_file,</div><div class='add'>+int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs);</div><div class='add'>+int ib_init_ucontext(struct uverbs_attr_bundle *attrs);</div><div class='add'>+</div><div class='add'>+void ib_uverbs_release_ucq(struct ib_uverbs_completion_event_file *ev_file,</div><div class='ctx'> 			   struct ib_ucq_object *uobj);</div><div class='del'>-void ib_uverbs_release_uevent(struct ib_uverbs_file *file,</div><div class='del'>-			      struct ib_uevent_object *uobj);</div><div class='add'>+void ib_uverbs_release_uevent(struct ib_uevent_object *uobj);</div><div class='ctx'> void ib_uverbs_release_file(struct kref *ref);</div><div class='add'>+void ib_uverbs_async_handler(struct ib_uverbs_async_event_file *async_file,</div><div class='add'>+			     __u64 element, __u64 event,</div><div class='add'>+			     struct list_head *obj_list, u32 *counter);</div><div class='ctx'> </div><div class='ctx'> void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context);</div><div class='ctx'> void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr);</div><div class='ctx'> void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr);</div><div class='ctx'> void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr);</div><div class='ctx'> void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr);</div><div class='del'>-void ib_uverbs_event_handler(struct ib_event_handler *handler,</div><div class='del'>-			     struct ib_event *event);</div><div class='ctx'> int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd,</div><div class='ctx'> 			   enum rdma_remove_reason why,</div><div class='ctx'> 			   struct uverbs_attr_bundle *attrs);</div><div class='hunk'>@@ -276,23 +247,6 @@ int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,</div><div class='ctx'> 					  size_t kern_filter_sz,</div><div class='ctx'> 					  union ib_flow_spec *ib_spec);</div><div class='ctx'> </div><div class='del'>-extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_DEVICE);</div><div class='del'>-extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_PD);</div><div class='del'>-extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_MR);</div><div class='del'>-extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_COMP_CHANNEL);</div><div class='del'>-extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_CQ);</div><div class='del'>-extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_QP);</div><div class='del'>-extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_AH);</div><div class='del'>-extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_MW);</div><div class='del'>-extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_SRQ);</div><div class='del'>-extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_FLOW);</div><div class='del'>-extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_WQ);</div><div class='del'>-extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_RWQ_IND_TBL);</div><div class='del'>-extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_XRCD);</div><div class='del'>-extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_FLOW_ACTION);</div><div class='del'>-extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_DM);</div><div class='del'>-extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_COUNTERS);</div><div class='del'>-</div><div class='ctx'> /*</div><div class='ctx'>  * ib_uverbs_query_port_resp.port_cap_flags started out as just a copy of the</div><div class='ctx'>  * PortInfo CapabilityMask, but was extended with unique bits.</div><div class='hunk'>@@ -314,6 +268,24 @@ static inline u32 make_port_cap_flags(const struct ib_port_attr *attr)</div><div class='ctx'> 	return res;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static inline struct ib_uverbs_async_event_file *</div><div class='add'>+ib_uverbs_get_async_event(struct uverbs_attr_bundle *attrs,</div><div class='add'>+			  u16 id)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *async_ev_file_uobj;</div><div class='add'>+	struct ib_uverbs_async_event_file *async_ev_file;</div><div class='add'>+</div><div class='add'>+	async_ev_file_uobj = uverbs_attr_get_uobject(attrs, id);</div><div class='add'>+	if (IS_ERR(async_ev_file_uobj))</div><div class='add'>+		async_ev_file = READ_ONCE(attrs-&gt;ufile-&gt;default_async_file);</div><div class='add'>+	else</div><div class='add'>+		async_ev_file = container_of(async_ev_file_uobj,</div><div class='add'>+				       struct ib_uverbs_async_event_file,</div><div class='add'>+				       uobj);</div><div class='add'>+	if (async_ev_file)</div><div class='add'>+		uverbs_uobject_get(&amp;async_ev_file-&gt;uobj);</div><div class='add'>+	return async_ev_file;</div><div class='add'>+}</div><div class='ctx'> </div><div class='ctx'> void copy_port_attr_to_resp(struct ib_port_attr *attr,</div><div class='ctx'> 			    struct ib_uverbs_query_port_resp *resp,</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c<br/>index 5a3a1780ceea..ce16404cdfb8 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_cmd.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_cmd.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_cmd.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_cmd.c</a></div><div class='hunk'>@@ -42,6 +42,7 @@</div><div class='ctx'> </div><div class='ctx'> #include &lt;rdma/uverbs_types.h&gt;</div><div class='ctx'> #include &lt;rdma/uverbs_std_types.h&gt;</div><div class='add'>+#include &lt;rdma/ib_ucaps.h&gt;</div><div class='ctx'> #include "rdma_core.h"</div><div class='ctx'> </div><div class='ctx'> #include "uverbs.h"</div><div class='hunk'>@@ -161,7 +162,7 @@ static const void __user *uverbs_request_next_ptr(struct uverbs_req_iter *iter,</div><div class='ctx'> {</div><div class='ctx'> 	const void __user *res = iter-&gt;cur;</div><div class='ctx'> </div><div class='del'>-	if (iter-&gt;cur + len &gt; iter-&gt;end)</div><div class='add'>+	if (len &gt; iter-&gt;end - iter-&gt;cur)</div><div class='ctx'> 		return (void __force __user *)ERR_PTR(-ENOSPC);</div><div class='ctx'> 	iter-&gt;cur += len;</div><div class='ctx'> 	return res;</div><div class='hunk'>@@ -174,6 +175,17 @@ static int uverbs_request_finish(struct uverbs_req_iter *iter)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+/*</div><div class='add'>+ * When calling a destroy function during an error unwind we need to pass in</div><div class='add'>+ * the udata that is sanitized of all user arguments. Ie from the driver</div><div class='add'>+ * perspective it looks like no udata was passed.</div><div class='add'>+ */</div><div class='add'>+struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	attrs-&gt;driver_udata = (struct ib_udata){};</div><div class='add'>+	return &amp;attrs-&gt;driver_udata;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static struct ib_uverbs_completion_event_file *</div><div class='ctx'> _ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='hunk'>@@ -181,7 +193,7 @@ _ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 					       fd, attrs);</div><div class='ctx'> </div><div class='ctx'> 	if (IS_ERR(uobj))</div><div class='del'>-		return (void *)uobj;</div><div class='add'>+		return ERR_CAST(uobj);</div><div class='ctx'> </div><div class='ctx'> 	uverbs_uobject_get(uobj);</div><div class='ctx'> 	uobj_put_read(uobj);</div><div class='hunk'>@@ -192,84 +204,74 @@ _ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs)</div><div class='ctx'> #define ib_uverbs_lookup_comp_file(_fd, _ufile)                                \</div><div class='ctx'> 	_ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile)</div><div class='ctx'> </div><div class='del'>-static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)</div><div class='add'>+int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='del'>-	struct ib_uverbs_file *file = attrs-&gt;ufile;</div><div class='del'>-	struct ib_uverbs_get_context      cmd;</div><div class='del'>-	struct ib_uverbs_get_context_resp resp;</div><div class='del'>-	struct ib_ucontext		 *ucontext;</div><div class='del'>-	struct file			 *filp;</div><div class='del'>-	struct ib_rdmacg_object		 cg_obj;</div><div class='add'>+	struct ib_uverbs_file *ufile = attrs-&gt;ufile;</div><div class='add'>+	struct ib_ucontext *ucontext;</div><div class='ctx'> 	struct ib_device *ib_dev;</div><div class='del'>-	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = uverbs_request(attrs, &amp;cmd, sizeof(cmd));</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='add'>+	ib_dev = srcu_dereference(ufile-&gt;device-&gt;ib_dev,</div><div class='add'>+				  &amp;ufile-&gt;device-&gt;disassociate_srcu);</div><div class='add'>+	if (!ib_dev)</div><div class='add'>+		return -EIO;</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;file-&gt;ucontext_lock);</div><div class='del'>-	ib_dev = srcu_dereference(file-&gt;device-&gt;ib_dev,</div><div class='del'>-				  &amp;file-&gt;device-&gt;disassociate_srcu);</div><div class='del'>-	if (!ib_dev) {</div><div class='del'>-		ret = -EIO;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='add'>+	ucontext = rdma_zalloc_drv_obj(ib_dev, ib_ucontext);</div><div class='add'>+	if (!ucontext)</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='add'>+	ucontext-&gt;device = ib_dev;</div><div class='add'>+	ucontext-&gt;ufile = ufile;</div><div class='add'>+	xa_init_flags(&amp;ucontext-&gt;mmap_xa, XA_FLAGS_ALLOC);</div><div class='add'>+</div><div class='add'>+	rdma_restrack_new(&amp;ucontext-&gt;res, RDMA_RESTRACK_CTX);</div><div class='add'>+	rdma_restrack_set_name(&amp;ucontext-&gt;res, NULL);</div><div class='add'>+	attrs-&gt;context = ucontext;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ib_init_ucontext(struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_ucontext *ucontext = attrs-&gt;context;</div><div class='add'>+	struct ib_uverbs_file *file = attrs-&gt;ufile;</div><div class='add'>+	int *fd_array;</div><div class='add'>+	int fd_count;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!down_read_trylock(&amp;file-&gt;hw_destroy_rwsem))</div><div class='add'>+		return -EIO;</div><div class='add'>+	mutex_lock(&amp;file-&gt;ucontext_lock);</div><div class='ctx'> 	if (file-&gt;ucontext) {</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		goto err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = ib_rdmacg_try_charge(&amp;cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);</div><div class='add'>+	ret = ib_rdmacg_try_charge(&amp;ucontext-&gt;cg_obj, ucontext-&gt;device,</div><div class='add'>+				   RDMACG_RESOURCE_HCA_HANDLE);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='del'>-	ucontext = rdma_zalloc_drv_obj(ib_dev, ib_ucontext);</div><div class='del'>-	if (!ucontext) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err_alloc;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	attrs-&gt;context = ucontext;</div><div class='del'>-</div><div class='del'>-	ucontext-&gt;res.type = RDMA_RESTRACK_CTX;</div><div class='del'>-	ucontext-&gt;device = ib_dev;</div><div class='del'>-	ucontext-&gt;cg_obj = cg_obj;</div><div class='del'>-	/* ufile is required when some objects are released */</div><div class='del'>-	ucontext-&gt;ufile = file;</div><div class='del'>-</div><div class='del'>-	ucontext-&gt;closing = false;</div><div class='del'>-	ucontext-&gt;cleanup_retryable = false;</div><div class='del'>-</div><div class='del'>-	mutex_init(&amp;ucontext-&gt;per_mm_list_lock);</div><div class='del'>-	INIT_LIST_HEAD(&amp;ucontext-&gt;per_mm_list);</div><div class='del'>-</div><div class='del'>-	ret = get_unused_fd_flags(O_CLOEXEC);</div><div class='del'>-	if (ret &lt; 0)</div><div class='del'>-		goto err_free;</div><div class='del'>-	resp.async_fd = ret;</div><div class='add'>+	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_GET_CONTEXT_FD_ARR)) {</div><div class='add'>+		fd_count = uverbs_attr_ptr_get_array_size(attrs,</div><div class='add'>+							  UVERBS_ATTR_GET_CONTEXT_FD_ARR,</div><div class='add'>+							  sizeof(int));</div><div class='add'>+		if (fd_count &lt; 0) {</div><div class='add'>+			ret = fd_count;</div><div class='add'>+			goto err_uncharge;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='del'>-	filp = ib_uverbs_alloc_async_event_file(file, ib_dev);</div><div class='del'>-	if (IS_ERR(filp)) {</div><div class='del'>-		ret = PTR_ERR(filp);</div><div class='del'>-		goto err_fd;</div><div class='add'>+		fd_array = uverbs_attr_get_alloced_ptr(attrs,</div><div class='add'>+						       UVERBS_ATTR_GET_CONTEXT_FD_ARR);</div><div class='add'>+		ret = ib_get_ucaps(fd_array, fd_count, &amp;ucontext-&gt;enabled_caps);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_uncharge;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	resp.num_comp_vectors = file-&gt;device-&gt;num_comp_vectors;</div><div class='del'>-</div><div class='del'>-	ret = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_file;</div><div class='del'>-</div><div class='del'>-	ret = ib_dev-&gt;ops.alloc_ucontext(ucontext, &amp;attrs-&gt;driver_udata);</div><div class='add'>+	ret = ucontext-&gt;device-&gt;ops.alloc_ucontext(ucontext,</div><div class='add'>+						   &amp;attrs-&gt;driver_udata);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err_file;</div><div class='del'>-	if (!(ib_dev-&gt;attrs.device_cap_flags &amp; IB_DEVICE_ON_DEMAND_PAGING))</div><div class='del'>-		ucontext-&gt;invalidate_range = NULL;</div><div class='del'>-</div><div class='del'>-	rdma_restrack_uadd(&amp;ucontext-&gt;res);</div><div class='add'>+		goto err_uncharge;</div><div class='ctx'> </div><div class='del'>-	fd_install(resp.async_fd, filp);</div><div class='add'>+	rdma_restrack_add(&amp;ucontext-&gt;res);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Make sure that ib_uverbs_get_ucontext() sees the pointer update</div><div class='hunk'>@@ -278,24 +280,63 @@ static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	smp_store_release(&amp;file-&gt;ucontext, ucontext);</div><div class='ctx'> </div><div class='ctx'> 	mutex_unlock(&amp;file-&gt;ucontext_lock);</div><div class='del'>-</div><div class='add'>+	up_read(&amp;file-&gt;hw_destroy_rwsem);</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-err_file:</div><div class='del'>-	ib_uverbs_free_async_event_file(file);</div><div class='del'>-	fput(filp);</div><div class='add'>+err_uncharge:</div><div class='add'>+	ib_rdmacg_uncharge(&amp;ucontext-&gt;cg_obj, ucontext-&gt;device,</div><div class='add'>+			   RDMACG_RESOURCE_HCA_HANDLE);</div><div class='add'>+err:</div><div class='add'>+	mutex_unlock(&amp;file-&gt;ucontext_lock);</div><div class='add'>+	up_read(&amp;file-&gt;hw_destroy_rwsem);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-err_fd:</div><div class='del'>-	put_unused_fd(resp.async_fd);</div><div class='add'>+static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uverbs_get_context_resp resp;</div><div class='add'>+	struct ib_uverbs_get_context cmd;</div><div class='add'>+	struct ib_device *ib_dev;</div><div class='add'>+	struct ib_uobject *uobj;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-err_free:</div><div class='del'>-	kfree(ucontext);</div><div class='add'>+	ret = uverbs_request(attrs, &amp;cmd, sizeof(cmd));</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='del'>-err_alloc:</div><div class='del'>-	ib_rdmacg_uncharge(&amp;cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);</div><div class='add'>+	ret = ib_alloc_ucontext(attrs);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='del'>-err:</div><div class='del'>-	mutex_unlock(&amp;file-&gt;ucontext_lock);</div><div class='add'>+	uobj = uobj_alloc(UVERBS_OBJECT_ASYNC_EVENT, attrs, &amp;ib_dev);</div><div class='add'>+	if (IS_ERR(uobj)) {</div><div class='add'>+		ret = PTR_ERR(uobj);</div><div class='add'>+		goto err_ucontext;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	resp = (struct ib_uverbs_get_context_resp){</div><div class='add'>+		.num_comp_vectors = attrs-&gt;ufile-&gt;device-&gt;num_comp_vectors,</div><div class='add'>+		.async_fd = uobj-&gt;id,</div><div class='add'>+	};</div><div class='add'>+	ret = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_uobj;</div><div class='add'>+</div><div class='add'>+	ret = ib_init_ucontext(attrs);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_uobj;</div><div class='add'>+</div><div class='add'>+	ib_uverbs_init_async_event_file(</div><div class='add'>+		container_of(uobj, struct ib_uverbs_async_event_file, uobj));</div><div class='add'>+	rdma_alloc_commit_uobject(uobj, attrs);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_uobj:</div><div class='add'>+	rdma_alloc_abort_uobject(uobj, attrs, false);</div><div class='add'>+err_ucontext:</div><div class='add'>+	rdma_restrack_put(&amp;attrs-&gt;context-&gt;res);</div><div class='add'>+	kfree(attrs-&gt;context);</div><div class='add'>+	attrs-&gt;context = NULL;</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -315,7 +356,7 @@ static void copy_query_dev_fields(struct ib_ucontext *ucontext,</div><div class='ctx'> 	resp-&gt;hw_ver		= attr-&gt;hw_ver;</div><div class='ctx'> 	resp-&gt;max_qp		= attr-&gt;max_qp;</div><div class='ctx'> 	resp-&gt;max_qp_wr		= attr-&gt;max_qp_wr;</div><div class='del'>-	resp-&gt;device_cap_flags	= lower_32_bits(attr-&gt;device_cap_flags);</div><div class='add'>+	resp-&gt;device_cap_flags  = lower_32_bits(attr-&gt;device_cap_flags);</div><div class='ctx'> 	resp-&gt;max_sge		= min(attr-&gt;max_send_sge, attr-&gt;max_recv_sge);</div><div class='ctx'> 	resp-&gt;max_sge_rd	= attr-&gt;max_sge_rd;</div><div class='ctx'> 	resp-&gt;max_cq		= attr-&gt;max_cq;</div><div class='hunk'>@@ -337,14 +378,12 @@ static void copy_query_dev_fields(struct ib_ucontext *ucontext,</div><div class='ctx'> 	resp-&gt;max_mcast_qp_attach	= attr-&gt;max_mcast_qp_attach;</div><div class='ctx'> 	resp-&gt;max_total_mcast_qp_attach	= attr-&gt;max_total_mcast_qp_attach;</div><div class='ctx'> 	resp-&gt;max_ah			= attr-&gt;max_ah;</div><div class='del'>-	resp-&gt;max_fmr			= attr-&gt;max_fmr;</div><div class='del'>-	resp-&gt;max_map_per_fmr		= attr-&gt;max_map_per_fmr;</div><div class='ctx'> 	resp-&gt;max_srq			= attr-&gt;max_srq;</div><div class='ctx'> 	resp-&gt;max_srq_wr		= attr-&gt;max_srq_wr;</div><div class='ctx'> 	resp-&gt;max_srq_sge		= attr-&gt;max_srq_sge;</div><div class='ctx'> 	resp-&gt;max_pkeys			= attr-&gt;max_pkeys;</div><div class='ctx'> 	resp-&gt;local_ca_ack_delay	= attr-&gt;local_ca_ack_delay;</div><div class='del'>-	resp-&gt;phys_port_cnt		= ib_dev-&gt;phys_port_cnt;</div><div class='add'>+	resp-&gt;phys_port_cnt = min_t(u32, ib_dev-&gt;phys_port_cnt, U8_MAX);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int ib_uverbs_query_device(struct uverbs_attr_bundle *attrs)</div><div class='hunk'>@@ -398,8 +437,8 @@ static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> </div><div class='ctx'> static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='add'>+	struct ib_uverbs_alloc_pd_resp resp = {};</div><div class='ctx'> 	struct ib_uverbs_alloc_pd      cmd;</div><div class='del'>-	struct ib_uverbs_alloc_pd_resp resp;</div><div class='ctx'> 	struct ib_uobject             *uobj;</div><div class='ctx'> 	struct ib_pd                  *pd;</div><div class='ctx'> 	int                            ret;</div><div class='hunk'>@@ -421,29 +460,24 @@ static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> </div><div class='ctx'> 	pd-&gt;device  = ib_dev;</div><div class='ctx'> 	pd-&gt;uobject = uobj;</div><div class='del'>-	pd-&gt;__internal_mr = NULL;</div><div class='ctx'> 	atomic_set(&amp;pd-&gt;usecnt, 0);</div><div class='del'>-	pd-&gt;res.type = RDMA_RESTRACK_PD;</div><div class='add'>+</div><div class='add'>+	rdma_restrack_new(&amp;pd-&gt;res, RDMA_RESTRACK_PD);</div><div class='add'>+	rdma_restrack_set_name(&amp;pd-&gt;res, NULL);</div><div class='ctx'> </div><div class='ctx'> 	ret = ib_dev-&gt;ops.alloc_pd(pd, &amp;attrs-&gt;driver_udata);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err_alloc;</div><div class='add'>+	rdma_restrack_add(&amp;pd-&gt;res);</div><div class='ctx'> </div><div class='ctx'> 	uobj-&gt;object = pd;</div><div class='del'>-	memset(&amp;resp, 0, sizeof resp);</div><div class='del'>-	resp.pd_handle = uobj-&gt;id;</div><div class='del'>-	rdma_restrack_uadd(&amp;pd-&gt;res);</div><div class='add'>+	uobj_finalize_uobj_create(uobj, attrs);</div><div class='ctx'> </div><div class='del'>-	ret = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_copy;</div><div class='del'>-</div><div class='del'>-	return uobj_alloc_commit(uobj, attrs);</div><div class='add'>+	resp.pd_handle = uobj-&gt;id;</div><div class='add'>+	return uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='ctx'> </div><div class='del'>-err_copy:</div><div class='del'>-	ib_dealloc_pd_user(pd, &amp;attrs-&gt;driver_udata);</div><div class='del'>-	pd = NULL;</div><div class='ctx'> err_alloc:</div><div class='add'>+	rdma_restrack_put(&amp;pd-&gt;res);</div><div class='ctx'> 	kfree(pd);</div><div class='ctx'> err:</div><div class='ctx'> 	uobj_alloc_abort(uobj, attrs);</div><div class='hunk'>@@ -550,15 +584,15 @@ static void xrcd_table_delete(struct ib_uverbs_device *dev,</div><div class='ctx'> static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_uverbs_device *ibudev = attrs-&gt;ufile-&gt;device;</div><div class='add'>+	struct ib_uverbs_open_xrcd_resp	resp = {};</div><div class='ctx'> 	struct ib_uverbs_open_xrcd	cmd;</div><div class='del'>-	struct ib_uverbs_open_xrcd_resp	resp;</div><div class='ctx'> 	struct ib_uxrcd_object         *obj;</div><div class='ctx'> 	struct ib_xrcd                 *xrcd = NULL;</div><div class='del'>-	struct fd			f = {NULL, 0};</div><div class='ctx'> 	struct inode                   *inode = NULL;</div><div class='del'>-	int				ret = 0;</div><div class='ctx'> 	int				new_xrcd = 0;</div><div class='ctx'> 	struct ib_device *ib_dev;</div><div class='add'>+	struct fd f = EMPTY_FD;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	ret = uverbs_request(attrs, &amp;cmd, sizeof(cmd));</div><div class='ctx'> 	if (ret)</div><div class='hunk'>@@ -569,12 +603,12 @@ static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	if (cmd.fd != -1) {</div><div class='ctx'> 		/* search for file descriptor */</div><div class='ctx'> 		f = fdget(cmd.fd);</div><div class='del'>-		if (!f.file) {</div><div class='add'>+		if (fd_empty(f)) {</div><div class='ctx'> 			ret = -EBADF;</div><div class='ctx'> 			goto err_tree_mutex_unlock;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		inode = file_inode(f.file);</div><div class='add'>+		inode = file_inode(fd_file(f));</div><div class='ctx'> 		xrcd = find_xrcd(ibudev, inode);</div><div class='ctx'> 		if (!xrcd &amp;&amp; !(cmd.oflags &amp; O_CREAT)) {</div><div class='ctx'> 			/* no file descriptor. Need CREATE flag */</div><div class='hunk'>@@ -596,24 +630,16 @@ static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (!xrcd) {</div><div class='del'>-		xrcd = ib_dev-&gt;ops.alloc_xrcd(ib_dev, &amp;attrs-&gt;driver_udata);</div><div class='add'>+		xrcd = ib_alloc_xrcd_user(ib_dev, inode, &amp;attrs-&gt;driver_udata);</div><div class='ctx'> 		if (IS_ERR(xrcd)) {</div><div class='ctx'> 			ret = PTR_ERR(xrcd);</div><div class='ctx'> 			goto err;</div><div class='ctx'> 		}</div><div class='del'>-</div><div class='del'>-		xrcd-&gt;inode   = inode;</div><div class='del'>-		xrcd-&gt;device  = ib_dev;</div><div class='del'>-		atomic_set(&amp;xrcd-&gt;usecnt, 0);</div><div class='del'>-		mutex_init(&amp;xrcd-&gt;tgt_qp_mutex);</div><div class='del'>-		INIT_LIST_HEAD(&amp;xrcd-&gt;tgt_qp_list);</div><div class='ctx'> 		new_xrcd = 1;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	atomic_set(&amp;obj-&gt;refcnt, 0);</div><div class='ctx'> 	obj-&gt;uobject.object = xrcd;</div><div class='del'>-	memset(&amp;resp, 0, sizeof resp);</div><div class='del'>-	resp.xrcd_handle = obj-&gt;uobject.id;</div><div class='ctx'> </div><div class='ctx'> 	if (inode) {</div><div class='ctx'> 		if (new_xrcd) {</div><div class='hunk'>@@ -625,33 +651,22 @@ static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		atomic_inc(&amp;xrcd-&gt;usecnt);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_copy;</div><div class='del'>-</div><div class='del'>-	if (f.file)</div><div class='del'>-		fdput(f);</div><div class='add'>+	fdput(f);</div><div class='ctx'> </div><div class='ctx'> 	mutex_unlock(&amp;ibudev-&gt;xrcd_tree_mutex);</div><div class='add'>+	uobj_finalize_uobj_create(&amp;obj-&gt;uobject, attrs);</div><div class='ctx'> </div><div class='del'>-	return uobj_alloc_commit(&amp;obj-&gt;uobject, attrs);</div><div class='del'>-</div><div class='del'>-err_copy:</div><div class='del'>-	if (inode) {</div><div class='del'>-		if (new_xrcd)</div><div class='del'>-			xrcd_table_delete(ibudev, inode);</div><div class='del'>-		atomic_dec(&amp;xrcd-&gt;usecnt);</div><div class='del'>-	}</div><div class='add'>+	resp.xrcd_handle = obj-&gt;uobject.id;</div><div class='add'>+	return uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='ctx'> </div><div class='ctx'> err_dealloc_xrcd:</div><div class='del'>-	ib_dealloc_xrcd(xrcd, &amp;attrs-&gt;driver_udata);</div><div class='add'>+	ib_dealloc_xrcd_user(xrcd, uverbs_get_cleared_udata(attrs));</div><div class='ctx'> </div><div class='ctx'> err:</div><div class='ctx'> 	uobj_alloc_abort(&amp;obj-&gt;uobject, attrs);</div><div class='ctx'> </div><div class='ctx'> err_tree_mutex_unlock:</div><div class='del'>-	if (f.file)</div><div class='del'>-		fdput(f);</div><div class='add'>+	fdput(f);</div><div class='ctx'> </div><div class='ctx'> 	mutex_unlock(&amp;ibudev-&gt;xrcd_tree_mutex);</div><div class='ctx'> </div><div class='hunk'>@@ -682,9 +697,8 @@ int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd,</div><div class='ctx'> 	if (inode &amp;&amp; !atomic_dec_and_test(&amp;xrcd-&gt;usecnt))</div><div class='ctx'> 		return 0;</div><div class='ctx'> </div><div class='del'>-	ret = ib_dealloc_xrcd(xrcd, &amp;attrs-&gt;driver_udata);</div><div class='del'>-</div><div class='del'>-	if (ib_is_destroy_retryable(ret, why, uobject)) {</div><div class='add'>+	ret = ib_dealloc_xrcd_user(xrcd, &amp;attrs-&gt;driver_udata);</div><div class='add'>+	if (ret) {</div><div class='ctx'> 		atomic_inc(&amp;xrcd-&gt;usecnt);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -692,13 +706,13 @@ int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd,</div><div class='ctx'> 	if (inode)</div><div class='ctx'> 		xrcd_table_delete(dev, inode);</div><div class='ctx'> </div><div class='del'>-	return ret;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='add'>+	struct ib_uverbs_reg_mr_resp resp = {};</div><div class='ctx'> 	struct ib_uverbs_reg_mr      cmd;</div><div class='del'>-	struct ib_uverbs_reg_mr_resp resp;</div><div class='ctx'> 	struct ib_uobject           *uobj;</div><div class='ctx'> 	struct ib_pd                *pd;</div><div class='ctx'> 	struct ib_mr                *mr;</div><div class='hunk'>@@ -712,31 +726,22 @@ static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	if ((cmd.start &amp; ~PAGE_MASK) != (cmd.hca_va &amp; ~PAGE_MASK))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	ret = ib_check_mr_access(cmd.access_flags);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='ctx'> 	uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &amp;ib_dev);</div><div class='ctx'> 	if (IS_ERR(uobj))</div><div class='ctx'> 		return PTR_ERR(uobj);</div><div class='ctx'> </div><div class='del'>-	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);</div><div class='del'>-	if (!pd) {</div><div class='del'>-		ret = -EINVAL;</div><div class='add'>+	ret = ib_check_mr_access(ib_dev, cmd.access_flags);</div><div class='add'>+	if (ret)</div><div class='ctx'> 		goto err_free;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	if (cmd.access_flags &amp; IB_ACCESS_ON_DEMAND) {</div><div class='del'>-		if (!(pd-&gt;device-&gt;attrs.device_cap_flags &amp;</div><div class='del'>-		      IB_DEVICE_ON_DEMAND_PAGING)) {</div><div class='del'>-			pr_debug("ODP support not available\n");</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto err_put;</div><div class='del'>-		}</div><div class='add'>+	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);</div><div class='add'>+	if (IS_ERR(pd)) {</div><div class='add'>+		ret = PTR_ERR(pd);</div><div class='add'>+		goto err_free;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	mr = pd-&gt;device-&gt;ops.reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,</div><div class='del'>-					 cmd.access_flags,</div><div class='add'>+					 cmd.access_flags, NULL,</div><div class='ctx'> 					 &amp;attrs-&gt;driver_udata);</div><div class='ctx'> 	if (IS_ERR(mr)) {</div><div class='ctx'> 		ret = PTR_ERR(mr);</div><div class='hunk'>@@ -745,33 +750,29 @@ static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;device  = pd-&gt;device;</div><div class='ctx'> 	mr-&gt;pd      = pd;</div><div class='add'>+	mr-&gt;type    = IB_MR_TYPE_USER;</div><div class='ctx'> 	mr-&gt;dm	    = NULL;</div><div class='add'>+	mr-&gt;sig_attrs = NULL;</div><div class='ctx'> 	mr-&gt;uobject = uobj;</div><div class='ctx'> 	atomic_inc(&amp;pd-&gt;usecnt);</div><div class='del'>-	mr-&gt;res.type = RDMA_RESTRACK_MR;</div><div class='del'>-	rdma_restrack_uadd(&amp;mr-&gt;res);</div><div class='add'>+	mr-&gt;iova = cmd.hca_va;</div><div class='add'>+	mr-&gt;length = cmd.length;</div><div class='ctx'> </div><div class='del'>-	uobj-&gt;object = mr;</div><div class='del'>-</div><div class='del'>-	memset(&amp;resp, 0, sizeof resp);</div><div class='del'>-	resp.lkey      = mr-&gt;lkey;</div><div class='del'>-	resp.rkey      = mr-&gt;rkey;</div><div class='del'>-	resp.mr_handle = uobj-&gt;id;</div><div class='del'>-</div><div class='del'>-	ret = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_copy;</div><div class='add'>+	rdma_restrack_new(&amp;mr-&gt;res, RDMA_RESTRACK_MR);</div><div class='add'>+	rdma_restrack_set_name(&amp;mr-&gt;res, NULL);</div><div class='add'>+	rdma_restrack_add(&amp;mr-&gt;res);</div><div class='ctx'> </div><div class='add'>+	uobj-&gt;object = mr;</div><div class='ctx'> 	uobj_put_obj_read(pd);</div><div class='add'>+	uobj_finalize_uobj_create(uobj, attrs);</div><div class='ctx'> </div><div class='del'>-	return uobj_alloc_commit(uobj, attrs);</div><div class='del'>-</div><div class='del'>-err_copy:</div><div class='del'>-	ib_dereg_mr_user(mr, &amp;attrs-&gt;driver_udata);</div><div class='add'>+	resp.lkey = mr-&gt;lkey;</div><div class='add'>+	resp.rkey = mr-&gt;rkey;</div><div class='add'>+	resp.mr_handle = uobj-&gt;id;</div><div class='add'>+	return uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='ctx'> </div><div class='ctx'> err_put:</div><div class='ctx'> 	uobj_put_obj_read(pd);</div><div class='del'>-</div><div class='ctx'> err_free:</div><div class='ctx'> 	uobj_alloc_abort(uobj, attrs);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -781,23 +782,28 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_uverbs_rereg_mr      cmd;</div><div class='ctx'> 	struct ib_uverbs_rereg_mr_resp resp;</div><div class='del'>-	struct ib_pd                *pd = NULL;</div><div class='ctx'> 	struct ib_mr                *mr;</div><div class='del'>-	struct ib_pd		    *old_pd;</div><div class='ctx'> 	int                          ret;</div><div class='ctx'> 	struct ib_uobject	    *uobj;</div><div class='add'>+	struct ib_uobject *new_uobj;</div><div class='add'>+	struct ib_device *ib_dev;</div><div class='add'>+	struct ib_pd *orig_pd;</div><div class='add'>+	struct ib_pd *new_pd;</div><div class='add'>+	struct ib_mr *new_mr;</div><div class='ctx'> </div><div class='ctx'> 	ret = uverbs_request(attrs, &amp;cmd, sizeof(cmd));</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	if (cmd.flags &amp; ~IB_MR_REREG_SUPPORTED || !cmd.flags)</div><div class='add'>+	if (!cmd.flags)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='add'>+	if (cmd.flags &amp; ~IB_MR_REREG_SUPPORTED)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='ctx'> 	if ((cmd.flags &amp; IB_MR_REREG_TRANS) &amp;&amp;</div><div class='del'>-	    (!cmd.start || !cmd.hca_va || 0 &gt;= cmd.length ||</div><div class='del'>-	     (cmd.start &amp; ~PAGE_MASK) != (cmd.hca_va &amp; ~PAGE_MASK)))</div><div class='del'>-			return -EINVAL;</div><div class='add'>+	    (cmd.start &amp; ~PAGE_MASK) != (cmd.hca_va &amp; ~PAGE_MASK))</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);</div><div class='ctx'> 	if (IS_ERR(uobj))</div><div class='hunk'>@@ -811,32 +817,72 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (cmd.flags &amp; IB_MR_REREG_ACCESS) {</div><div class='del'>-		ret = ib_check_mr_access(cmd.access_flags);</div><div class='add'>+		ret = ib_check_mr_access(mr-&gt;device, cmd.access_flags);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto put_uobjs;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	orig_pd = mr-&gt;pd;</div><div class='ctx'> 	if (cmd.flags &amp; IB_MR_REREG_PD) {</div><div class='del'>-		pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle,</div><div class='del'>-				       attrs);</div><div class='del'>-		if (!pd) {</div><div class='del'>-			ret = -EINVAL;</div><div class='add'>+		new_pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle,</div><div class='add'>+					   attrs);</div><div class='add'>+		if (IS_ERR(new_pd)) {</div><div class='add'>+			ret = PTR_ERR(new_pd);</div><div class='ctx'> 			goto put_uobjs;</div><div class='ctx'> 		}</div><div class='add'>+	} else {</div><div class='add'>+		new_pd = mr-&gt;pd;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	old_pd = mr-&gt;pd;</div><div class='del'>-	ret = mr-&gt;device-&gt;ops.rereg_user_mr(mr, cmd.flags, cmd.start,</div><div class='del'>-					    cmd.length, cmd.hca_va,</div><div class='del'>-					    cmd.access_flags, pd,</div><div class='del'>-					    &amp;attrs-&gt;driver_udata);</div><div class='del'>-	if (ret)</div><div class='add'>+	/*</div><div class='add'>+	 * The driver might create a new HW object as part of the rereg, we need</div><div class='add'>+	 * to have a uobject ready to hold it.</div><div class='add'>+	 */</div><div class='add'>+	new_uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &amp;ib_dev);</div><div class='add'>+	if (IS_ERR(new_uobj)) {</div><div class='add'>+		ret = PTR_ERR(new_uobj);</div><div class='ctx'> 		goto put_uobj_pd;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (cmd.flags &amp; IB_MR_REREG_PD) {</div><div class='del'>-		atomic_inc(&amp;pd-&gt;usecnt);</div><div class='del'>-		mr-&gt;pd = pd;</div><div class='del'>-		atomic_dec(&amp;old_pd-&gt;usecnt);</div><div class='add'>+	new_mr = ib_dev-&gt;ops.rereg_user_mr(mr, cmd.flags, cmd.start, cmd.length,</div><div class='add'>+					   cmd.hca_va, cmd.access_flags, new_pd,</div><div class='add'>+					   &amp;attrs-&gt;driver_udata);</div><div class='add'>+	if (IS_ERR(new_mr)) {</div><div class='add'>+		ret = PTR_ERR(new_mr);</div><div class='add'>+		goto put_new_uobj;</div><div class='add'>+	}</div><div class='add'>+	if (new_mr) {</div><div class='add'>+		new_mr-&gt;device = new_pd-&gt;device;</div><div class='add'>+		new_mr-&gt;pd = new_pd;</div><div class='add'>+		new_mr-&gt;type = IB_MR_TYPE_USER;</div><div class='add'>+		new_mr-&gt;uobject = uobj;</div><div class='add'>+		atomic_inc(&amp;new_pd-&gt;usecnt);</div><div class='add'>+		new_uobj-&gt;object = new_mr;</div><div class='add'>+</div><div class='add'>+		rdma_restrack_new(&amp;new_mr-&gt;res, RDMA_RESTRACK_MR);</div><div class='add'>+		rdma_restrack_set_name(&amp;new_mr-&gt;res, NULL);</div><div class='add'>+		rdma_restrack_add(&amp;new_mr-&gt;res);</div><div class='add'>+</div><div class='add'>+		/*</div><div class='add'>+		 * The new uobj for the new HW object is put into the same spot</div><div class='add'>+		 * in the IDR and the old uobj &amp; HW object is deleted.</div><div class='add'>+		 */</div><div class='add'>+		rdma_assign_uobject(uobj, new_uobj, attrs);</div><div class='add'>+		rdma_alloc_commit_uobject(new_uobj, attrs);</div><div class='add'>+		uobj_put_destroy(uobj);</div><div class='add'>+		new_uobj = NULL;</div><div class='add'>+		uobj = NULL;</div><div class='add'>+		mr = new_mr;</div><div class='add'>+	} else {</div><div class='add'>+		if (cmd.flags &amp; IB_MR_REREG_PD) {</div><div class='add'>+			atomic_dec(&amp;orig_pd-&gt;usecnt);</div><div class='add'>+			mr-&gt;pd = new_pd;</div><div class='add'>+			atomic_inc(&amp;new_pd-&gt;usecnt);</div><div class='add'>+		}</div><div class='add'>+		if (cmd.flags &amp; IB_MR_REREG_TRANS) {</div><div class='add'>+			mr-&gt;iova = cmd.hca_va;</div><div class='add'>+			mr-&gt;length = cmd.length;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	memset(&amp;resp, 0, sizeof(resp));</div><div class='hunk'>@@ -845,12 +891,16 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> </div><div class='ctx'> 	ret = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='ctx'> </div><div class='add'>+put_new_uobj:</div><div class='add'>+	if (new_uobj)</div><div class='add'>+		uobj_alloc_abort(new_uobj, attrs);</div><div class='ctx'> put_uobj_pd:</div><div class='ctx'> 	if (cmd.flags &amp; IB_MR_REREG_PD)</div><div class='del'>-		uobj_put_obj_read(pd);</div><div class='add'>+		uobj_put_obj_read(new_pd);</div><div class='ctx'> </div><div class='ctx'> put_uobjs:</div><div class='del'>-	uobj_put_write(uobj);</div><div class='add'>+	if (uobj)</div><div class='add'>+		uobj_put_write(uobj);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -870,7 +920,7 @@ static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_uverbs_alloc_mw      cmd;</div><div class='del'>-	struct ib_uverbs_alloc_mw_resp resp;</div><div class='add'>+	struct ib_uverbs_alloc_mw_resp resp = {};</div><div class='ctx'> 	struct ib_uobject             *uobj;</div><div class='ctx'> 	struct ib_pd                  *pd;</div><div class='ctx'> 	struct ib_mw                  *mw;</div><div class='hunk'>@@ -886,8 +936,8 @@ static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		return PTR_ERR(uobj);</div><div class='ctx'> </div><div class='ctx'> 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);</div><div class='del'>-	if (!pd) {</div><div class='del'>-		ret = -EINVAL;</div><div class='add'>+	if (IS_ERR(pd)) {</div><div class='add'>+		ret = PTR_ERR(pd);</div><div class='ctx'> 		goto err_free;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -896,32 +946,33 @@ static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		goto err_put;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	mw = pd-&gt;device-&gt;ops.alloc_mw(pd, cmd.mw_type, &amp;attrs-&gt;driver_udata);</div><div class='del'>-	if (IS_ERR(mw)) {</div><div class='del'>-		ret = PTR_ERR(mw);</div><div class='add'>+	mw = rdma_zalloc_drv_obj(ib_dev, ib_mw);</div><div class='add'>+	if (!mw) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='ctx'> 		goto err_put;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	mw-&gt;device  = pd-&gt;device;</div><div class='del'>-	mw-&gt;pd      = pd;</div><div class='add'>+	mw-&gt;device = ib_dev;</div><div class='add'>+	mw-&gt;pd = pd;</div><div class='ctx'> 	mw-&gt;uobject = uobj;</div><div class='add'>+	mw-&gt;type = cmd.mw_type;</div><div class='add'>+</div><div class='add'>+	ret = pd-&gt;device-&gt;ops.alloc_mw(mw, &amp;attrs-&gt;driver_udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_alloc;</div><div class='add'>+</div><div class='ctx'> 	atomic_inc(&amp;pd-&gt;usecnt);</div><div class='ctx'> </div><div class='ctx'> 	uobj-&gt;object = mw;</div><div class='add'>+	uobj_put_obj_read(pd);</div><div class='add'>+	uobj_finalize_uobj_create(uobj, attrs);</div><div class='ctx'> </div><div class='del'>-	memset(&amp;resp, 0, sizeof(resp));</div><div class='del'>-	resp.rkey      = mw-&gt;rkey;</div><div class='add'>+	resp.rkey = mw-&gt;rkey;</div><div class='ctx'> 	resp.mw_handle = uobj-&gt;id;</div><div class='add'>+	return uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='ctx'> </div><div class='del'>-	ret = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_copy;</div><div class='del'>-</div><div class='del'>-	uobj_put_obj_read(pd);</div><div class='del'>-	return uobj_alloc_commit(uobj, attrs);</div><div class='del'>-</div><div class='del'>-err_copy:</div><div class='del'>-	uverbs_dealloc_mw(mw);</div><div class='add'>+err_alloc:</div><div class='add'>+	kfree(mw);</div><div class='ctx'> err_put:</div><div class='ctx'> 	uobj_put_obj_read(pd);</div><div class='ctx'> err_free:</div><div class='hunk'>@@ -958,39 +1009,33 @@ static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	if (IS_ERR(uobj))</div><div class='ctx'> 		return PTR_ERR(uobj);</div><div class='ctx'> </div><div class='del'>-	resp.fd = uobj-&gt;id;</div><div class='del'>-</div><div class='ctx'> 	ev_file = container_of(uobj, struct ib_uverbs_completion_event_file,</div><div class='ctx'> 			       uobj);</div><div class='ctx'> 	ib_uverbs_init_event_queue(&amp;ev_file-&gt;ev_queue);</div><div class='add'>+	uobj_finalize_uobj_create(uobj, attrs);</div><div class='ctx'> </div><div class='del'>-	ret = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='del'>-	if (ret) {</div><div class='del'>-		uobj_alloc_abort(uobj, attrs);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return uobj_alloc_commit(uobj, attrs);</div><div class='add'>+	resp.fd = uobj-&gt;id;</div><div class='add'>+	return uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs,</div><div class='del'>-				       struct ib_uverbs_ex_create_cq *cmd)</div><div class='add'>+static int create_cq(struct uverbs_attr_bundle *attrs,</div><div class='add'>+		     struct ib_uverbs_ex_create_cq *cmd)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_ucq_object           *obj;</div><div class='ctx'> 	struct ib_uverbs_completion_event_file    *ev_file = NULL;</div><div class='ctx'> 	struct ib_cq                   *cq;</div><div class='ctx'> 	int                             ret;</div><div class='del'>-	struct ib_uverbs_ex_create_cq_resp resp;</div><div class='add'>+	struct ib_uverbs_ex_create_cq_resp resp = {};</div><div class='ctx'> 	struct ib_cq_init_attr attr = {};</div><div class='ctx'> 	struct ib_device *ib_dev;</div><div class='ctx'> </div><div class='ctx'> 	if (cmd-&gt;comp_vector &gt;= attrs-&gt;ufile-&gt;device-&gt;num_comp_vectors)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,</div><div class='ctx'> 						 &amp;ib_dev);</div><div class='ctx'> 	if (IS_ERR(obj))</div><div class='del'>-		return obj;</div><div class='add'>+		return PTR_ERR(obj);</div><div class='ctx'> </div><div class='ctx'> 	if (cmd-&gt;comp_channel &gt;= 0) {</div><div class='ctx'> 		ev_file = ib_uverbs_lookup_comp_file(cmd-&gt;comp_channel, attrs);</div><div class='hunk'>@@ -1000,65 +1045,60 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	obj-&gt;uobject.user_handle = cmd-&gt;user_handle;</div><div class='del'>-	obj-&gt;comp_events_reported  = 0;</div><div class='del'>-	obj-&gt;async_events_reported = 0;</div><div class='add'>+	obj-&gt;uevent.uobject.user_handle = cmd-&gt;user_handle;</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;obj-&gt;comp_list);</div><div class='del'>-	INIT_LIST_HEAD(&amp;obj-&gt;async_list);</div><div class='add'>+	INIT_LIST_HEAD(&amp;obj-&gt;uevent.event_list);</div><div class='ctx'> </div><div class='ctx'> 	attr.cqe = cmd-&gt;cqe;</div><div class='ctx'> 	attr.comp_vector = cmd-&gt;comp_vector;</div><div class='ctx'> 	attr.flags = cmd-&gt;flags;</div><div class='ctx'> </div><div class='del'>-	cq = ib_dev-&gt;ops.create_cq(ib_dev, &amp;attr, &amp;attrs-&gt;driver_udata);</div><div class='del'>-	if (IS_ERR(cq)) {</div><div class='del'>-		ret = PTR_ERR(cq);</div><div class='add'>+	cq = rdma_zalloc_drv_obj(ib_dev, ib_cq);</div><div class='add'>+	if (!cq) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='ctx'> 		goto err_file;</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='ctx'> 	cq-&gt;device        = ib_dev;</div><div class='del'>-	cq-&gt;uobject       = &amp;obj-&gt;uobject;</div><div class='add'>+	cq-&gt;uobject       = obj;</div><div class='ctx'> 	cq-&gt;comp_handler  = ib_uverbs_comp_handler;</div><div class='ctx'> 	cq-&gt;event_handler = ib_uverbs_cq_event_handler;</div><div class='ctx'> 	cq-&gt;cq_context    = ev_file ? &amp;ev_file-&gt;ev_queue : NULL;</div><div class='ctx'> 	atomic_set(&amp;cq-&gt;usecnt, 0);</div><div class='ctx'> </div><div class='del'>-	obj-&gt;uobject.object = cq;</div><div class='del'>-	memset(&amp;resp, 0, sizeof resp);</div><div class='del'>-	resp.base.cq_handle = obj-&gt;uobject.id;</div><div class='del'>-	resp.base.cqe       = cq-&gt;cqe;</div><div class='del'>-	resp.response_length = uverbs_response_length(attrs, sizeof(resp));</div><div class='del'>-</div><div class='del'>-	cq-&gt;res.type = RDMA_RESTRACK_CQ;</div><div class='del'>-	rdma_restrack_uadd(&amp;cq-&gt;res);</div><div class='add'>+	rdma_restrack_new(&amp;cq-&gt;res, RDMA_RESTRACK_CQ);</div><div class='add'>+	rdma_restrack_set_name(&amp;cq-&gt;res, NULL);</div><div class='ctx'> </div><div class='del'>-	ret = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='add'>+	ret = ib_dev-&gt;ops.create_cq(cq, &amp;attr, attrs);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err_cb;</div><div class='add'>+		goto err_free;</div><div class='add'>+	rdma_restrack_add(&amp;cq-&gt;res);</div><div class='ctx'> </div><div class='del'>-	ret = uobj_alloc_commit(&amp;obj-&gt;uobject, attrs);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ERR_PTR(ret);</div><div class='del'>-	return obj;</div><div class='add'>+	obj-&gt;uevent.uobject.object = cq;</div><div class='add'>+	obj-&gt;uevent.event_file = READ_ONCE(attrs-&gt;ufile-&gt;default_async_file);</div><div class='add'>+	if (obj-&gt;uevent.event_file)</div><div class='add'>+		uverbs_uobject_get(&amp;obj-&gt;uevent.event_file-&gt;uobj);</div><div class='add'>+	uobj_finalize_uobj_create(&amp;obj-&gt;uevent.uobject, attrs);</div><div class='ctx'> </div><div class='del'>-err_cb:</div><div class='del'>-	ib_destroy_cq(cq);</div><div class='add'>+	resp.base.cq_handle = obj-&gt;uevent.uobject.id;</div><div class='add'>+	resp.base.cqe = cq-&gt;cqe;</div><div class='add'>+	resp.response_length = uverbs_response_length(attrs, sizeof(resp));</div><div class='add'>+	return uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='ctx'> </div><div class='add'>+err_free:</div><div class='add'>+	rdma_restrack_put(&amp;cq-&gt;res);</div><div class='add'>+	kfree(cq);</div><div class='ctx'> err_file:</div><div class='ctx'> 	if (ev_file)</div><div class='del'>-		ib_uverbs_release_ucq(attrs-&gt;ufile, ev_file, obj);</div><div class='del'>-</div><div class='add'>+		ib_uverbs_release_ucq(ev_file, obj);</div><div class='ctx'> err:</div><div class='del'>-	uobj_alloc_abort(&amp;obj-&gt;uobject, attrs);</div><div class='del'>-</div><div class='del'>-	return ERR_PTR(ret);</div><div class='add'>+	uobj_alloc_abort(&amp;obj-&gt;uevent.uobject, attrs);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_uverbs_create_cq      cmd;</div><div class='ctx'> 	struct ib_uverbs_ex_create_cq	cmd_ex;</div><div class='del'>-	struct ib_ucq_object           *obj;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	ret = uverbs_request(attrs, &amp;cmd, sizeof(cmd));</div><div class='hunk'>@@ -1071,14 +1111,12 @@ static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	cmd_ex.comp_vector = cmd.comp_vector;</div><div class='ctx'> 	cmd_ex.comp_channel = cmd.comp_channel;</div><div class='ctx'> </div><div class='del'>-	obj = create_cq(attrs, &amp;cmd_ex);</div><div class='del'>-	return PTR_ERR_OR_ZERO(obj);</div><div class='add'>+	return create_cq(attrs, &amp;cmd_ex);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_uverbs_ex_create_cq  cmd;</div><div class='del'>-	struct ib_ucq_object           *obj;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	ret = uverbs_request(attrs, &amp;cmd, sizeof(cmd));</div><div class='hunk'>@@ -1091,8 +1129,7 @@ static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	if (cmd.reserved)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	obj = create_cq(attrs, &amp;cmd);</div><div class='del'>-	return PTR_ERR_OR_ZERO(obj);</div><div class='add'>+	return create_cq(attrs, &amp;cmd);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)</div><div class='hunk'>@@ -1100,15 +1137,15 @@ static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	struct ib_uverbs_resize_cq	cmd;</div><div class='ctx'> 	struct ib_uverbs_resize_cq_resp	resp = {};</div><div class='ctx'> 	struct ib_cq			*cq;</div><div class='del'>-	int				ret = -EINVAL;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	ret = uverbs_request(attrs, &amp;cmd, sizeof(cmd));</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='ctx'> 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);</div><div class='del'>-	if (!cq)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	if (IS_ERR(cq))</div><div class='add'>+		return PTR_ERR(cq);</div><div class='ctx'> </div><div class='ctx'> 	ret = cq-&gt;device-&gt;ops.resize_cq(cq, cmd.cqe, &amp;attrs-&gt;driver_udata);</div><div class='ctx'> 	if (ret)</div><div class='hunk'>@@ -1118,7 +1155,8 @@ static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> </div><div class='ctx'> 	ret = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='ctx'> out:</div><div class='del'>-	uobj_put_obj_read(cq);</div><div class='add'>+	rdma_lookup_put_uobject(&amp;cq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -1168,8 +1206,8 @@ static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='ctx'> 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);</div><div class='del'>-	if (!cq)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	if (IS_ERR(cq))</div><div class='add'>+		return PTR_ERR(cq);</div><div class='ctx'> </div><div class='ctx'> 	/* we copy a struct ib_uverbs_poll_cq_resp to user space */</div><div class='ctx'> 	header_ptr = attrs-&gt;ucore.outbuf;</div><div class='hunk'>@@ -1201,7 +1239,8 @@ static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		ret = uverbs_output_written(attrs, UVERBS_ATTR_CORE_OUT);</div><div class='ctx'> </div><div class='ctx'> out_put:</div><div class='del'>-	uobj_put_obj_read(cq);</div><div class='add'>+	rdma_lookup_put_uobject(&amp;cq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1216,14 +1255,14 @@ static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='ctx'> 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);</div><div class='del'>-	if (!cq)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	if (IS_ERR(cq))</div><div class='add'>+		return PTR_ERR(cq);</div><div class='ctx'> </div><div class='ctx'> 	ib_req_notify_cq(cq, cmd.solicited_only ?</div><div class='ctx'> 			 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP);</div><div class='ctx'> </div><div class='del'>-	uobj_put_obj_read(cq);</div><div class='del'>-</div><div class='add'>+	rdma_lookup_put_uobject(&amp;cq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1243,10 +1282,10 @@ static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	if (IS_ERR(uobj))</div><div class='ctx'> 		return PTR_ERR(uobj);</div><div class='ctx'> </div><div class='del'>-	obj = container_of(uobj, struct ib_ucq_object, uobject);</div><div class='add'>+	obj = container_of(uobj, struct ib_ucq_object, uevent.uobject);</div><div class='ctx'> 	memset(&amp;resp, 0, sizeof(resp));</div><div class='ctx'> 	resp.comp_events_reported  = obj-&gt;comp_events_reported;</div><div class='del'>-	resp.async_events_reported = obj-&gt;async_events_reported;</div><div class='add'>+	resp.async_events_reported = obj-&gt;uevent.events_reported;</div><div class='ctx'> </div><div class='ctx'> 	uobj_put_destroy(uobj);</div><div class='ctx'> </div><div class='hunk'>@@ -1265,14 +1304,27 @@ static int create_qp(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 	struct ib_srq			*srq = NULL;</div><div class='ctx'> 	struct ib_qp			*qp;</div><div class='ctx'> 	struct ib_qp_init_attr		attr = {};</div><div class='del'>-	struct ib_uverbs_ex_create_qp_resp resp;</div><div class='add'>+	struct ib_uverbs_ex_create_qp_resp resp = {};</div><div class='ctx'> 	int				ret;</div><div class='ctx'> 	struct ib_rwq_ind_table *ind_tbl = NULL;</div><div class='ctx'> 	bool has_sq = true;</div><div class='ctx'> 	struct ib_device *ib_dev;</div><div class='ctx'> </div><div class='del'>-	if (cmd-&gt;qp_type == IB_QPT_RAW_PACKET &amp;&amp; !capable(CAP_NET_RAW))</div><div class='del'>-		return -EPERM;</div><div class='add'>+	switch (cmd-&gt;qp_type) {</div><div class='add'>+	case IB_QPT_RAW_PACKET:</div><div class='add'>+		if (!rdma_uattrs_has_raw_cap(attrs))</div><div class='add'>+			return -EPERM;</div><div class='add'>+		fallthrough;</div><div class='add'>+	case IB_QPT_RC:</div><div class='add'>+	case IB_QPT_UC:</div><div class='add'>+	case IB_QPT_UD:</div><div class='add'>+	case IB_QPT_XRC_INI:</div><div class='add'>+	case IB_QPT_XRC_TGT:</div><div class='add'>+	case IB_QPT_DRIVER:</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,</div><div class='ctx'> 						 &amp;ib_dev);</div><div class='hunk'>@@ -1286,8 +1338,8 @@ static int create_qp(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 		ind_tbl = uobj_get_obj_read(rwq_ind_table,</div><div class='ctx'> 					    UVERBS_OBJECT_RWQ_IND_TBL,</div><div class='ctx'> 					    cmd-&gt;rwq_ind_tbl_handle, attrs);</div><div class='del'>-		if (!ind_tbl) {</div><div class='del'>-			ret = -EINVAL;</div><div class='add'>+		if (IS_ERR(ind_tbl)) {</div><div class='add'>+			ret = PTR_ERR(ind_tbl);</div><div class='ctx'> 			goto err_put;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='hunk'>@@ -1325,8 +1377,10 @@ static int create_qp(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 			if (cmd-&gt;is_srq) {</div><div class='ctx'> 				srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ,</div><div class='ctx'> 							cmd-&gt;srq_handle, attrs);</div><div class='del'>-				if (!srq || srq-&gt;srq_type == IB_SRQT_XRC) {</div><div class='del'>-					ret = -EINVAL;</div><div class='add'>+				if (IS_ERR(srq) ||</div><div class='add'>+				    srq-&gt;srq_type == IB_SRQT_XRC) {</div><div class='add'>+					ret = IS_ERR(srq) ? PTR_ERR(srq) :</div><div class='add'>+								  -EINVAL;</div><div class='ctx'> 					goto err_put;</div><div class='ctx'> 				}</div><div class='ctx'> 			}</div><div class='hunk'>@@ -1336,23 +1390,29 @@ static int create_qp(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 					rcq = uobj_get_obj_read(</div><div class='ctx'> 						cq, UVERBS_OBJECT_CQ,</div><div class='ctx'> 						cmd-&gt;recv_cq_handle, attrs);</div><div class='del'>-					if (!rcq) {</div><div class='del'>-						ret = -EINVAL;</div><div class='add'>+					if (IS_ERR(rcq)) {</div><div class='add'>+						ret = PTR_ERR(rcq);</div><div class='ctx'> 						goto err_put;</div><div class='ctx'> 					}</div><div class='ctx'> 				}</div><div class='ctx'> 			}</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		if (has_sq)</div><div class='add'>+		if (has_sq) {</div><div class='ctx'> 			scq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,</div><div class='ctx'> 						cmd-&gt;send_cq_handle, attrs);</div><div class='del'>-		if (!ind_tbl)</div><div class='add'>+			if (IS_ERR(scq)) {</div><div class='add'>+				ret = PTR_ERR(scq);</div><div class='add'>+				goto err_put;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (!ind_tbl &amp;&amp; cmd-&gt;qp_type != IB_QPT_XRC_INI)</div><div class='ctx'> 			rcq = rcq ?: scq;</div><div class='ctx'> 		pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd-&gt;pd_handle,</div><div class='ctx'> 				       attrs);</div><div class='del'>-		if (!pd || (!scq &amp;&amp; has_sq)) {</div><div class='del'>-			ret = -EINVAL;</div><div class='add'>+		if (IS_ERR(pd)) {</div><div class='add'>+			ret = PTR_ERR(pd);</div><div class='ctx'> 			goto err_put;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='hunk'>@@ -1360,7 +1420,6 @@ static int create_qp(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	attr.event_handler = ib_uverbs_qp_event_handler;</div><div class='del'>-	attr.qp_context    = attrs-&gt;ufile;</div><div class='ctx'> 	attr.send_cq       = scq;</div><div class='ctx'> 	attr.recv_cq       = rcq;</div><div class='ctx'> 	attr.srq           = srq;</div><div class='hunk'>@@ -1368,7 +1427,6 @@ static int create_qp(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 	attr.sq_sig_type   = cmd-&gt;sq_sig_all ? IB_SIGNAL_ALL_WR :</div><div class='ctx'> 					      IB_SIGNAL_REQ_WR;</div><div class='ctx'> 	attr.qp_type       = cmd-&gt;qp_type;</div><div class='del'>-	attr.create_flags  = 0;</div><div class='ctx'> </div><div class='ctx'> 	attr.cap.max_send_wr     = cmd-&gt;max_send_wr;</div><div class='ctx'> 	attr.cap.max_recv_wr     = cmd-&gt;max_recv_wr;</div><div class='hunk'>@@ -1376,7 +1434,6 @@ static int create_qp(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 	attr.cap.max_recv_sge    = cmd-&gt;max_recv_sge;</div><div class='ctx'> 	attr.cap.max_inline_data = cmd-&gt;max_inline_data;</div><div class='ctx'> </div><div class='del'>-	obj-&gt;uevent.events_reported     = 0;</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;obj-&gt;uevent.event_list);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;obj-&gt;mcast_list);</div><div class='ctx'> </div><div class='hunk'>@@ -1394,7 +1451,7 @@ static int create_qp(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (attr.create_flags &amp; IB_QP_CREATE_SOURCE_QPN) {</div><div class='del'>-		if (!capable(CAP_NET_RAW)) {</div><div class='add'>+		if (!rdma_uattrs_has_raw_cap(attrs)) {</div><div class='ctx'> 			ret = -EPERM;</div><div class='ctx'> 			goto err_put;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -1402,61 +1459,18 @@ static int create_qp(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 		attr.source_qpn = cmd-&gt;source_qpn;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (cmd-&gt;qp_type == IB_QPT_XRC_TGT)</div><div class='del'>-		qp = ib_create_qp(pd, &amp;attr);</div><div class='del'>-	else</div><div class='del'>-		qp = _ib_create_qp(device, pd, &amp;attr, &amp;attrs-&gt;driver_udata,</div><div class='del'>-				   &amp;obj-&gt;uevent.uobject);</div><div class='del'>-</div><div class='add'>+	qp = ib_create_qp_user(device, pd, &amp;attr, &amp;attrs-&gt;driver_udata, obj,</div><div class='add'>+			       KBUILD_MODNAME);</div><div class='ctx'> 	if (IS_ERR(qp)) {</div><div class='ctx'> 		ret = PTR_ERR(qp);</div><div class='ctx'> 		goto err_put;</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='del'>-	if (cmd-&gt;qp_type != IB_QPT_XRC_TGT) {</div><div class='del'>-		ret = ib_create_qp_security(qp, device);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto err_cb;</div><div class='del'>-</div><div class='del'>-		qp-&gt;pd		  = pd;</div><div class='del'>-		qp-&gt;send_cq	  = attr.send_cq;</div><div class='del'>-		qp-&gt;recv_cq	  = attr.recv_cq;</div><div class='del'>-		qp-&gt;srq		  = attr.srq;</div><div class='del'>-		qp-&gt;rwq_ind_tbl	  = ind_tbl;</div><div class='del'>-		qp-&gt;event_handler = attr.event_handler;</div><div class='del'>-		qp-&gt;qp_context	  = attr.qp_context;</div><div class='del'>-		qp-&gt;qp_type	  = attr.qp_type;</div><div class='del'>-		atomic_set(&amp;qp-&gt;usecnt, 0);</div><div class='del'>-		atomic_inc(&amp;pd-&gt;usecnt);</div><div class='del'>-		qp-&gt;port = 0;</div><div class='del'>-		if (attr.send_cq)</div><div class='del'>-			atomic_inc(&amp;attr.send_cq-&gt;usecnt);</div><div class='del'>-		if (attr.recv_cq)</div><div class='del'>-			atomic_inc(&amp;attr.recv_cq-&gt;usecnt);</div><div class='del'>-		if (attr.srq)</div><div class='del'>-			atomic_inc(&amp;attr.srq-&gt;usecnt);</div><div class='del'>-		if (ind_tbl)</div><div class='del'>-			atomic_inc(&amp;ind_tbl-&gt;usecnt);</div><div class='del'>-	} else {</div><div class='del'>-		/* It is done in _ib_create_qp for other QP types */</div><div class='del'>-		qp-&gt;uobject = &amp;obj-&gt;uevent.uobject;</div><div class='del'>-	}</div><div class='add'>+	ib_qp_usecnt_inc(qp);</div><div class='ctx'> </div><div class='ctx'> 	obj-&gt;uevent.uobject.object = qp;</div><div class='del'>-</div><div class='del'>-	memset(&amp;resp, 0, sizeof resp);</div><div class='del'>-	resp.base.qpn             = qp-&gt;qp_num;</div><div class='del'>-	resp.base.qp_handle       = obj-&gt;uevent.uobject.id;</div><div class='del'>-	resp.base.max_recv_sge    = attr.cap.max_recv_sge;</div><div class='del'>-	resp.base.max_send_sge    = attr.cap.max_send_sge;</div><div class='del'>-	resp.base.max_recv_wr     = attr.cap.max_recv_wr;</div><div class='del'>-	resp.base.max_send_wr     = attr.cap.max_send_wr;</div><div class='del'>-	resp.base.max_inline_data = attr.cap.max_inline_data;</div><div class='del'>-	resp.response_length = uverbs_response_length(attrs, sizeof(resp));</div><div class='del'>-</div><div class='del'>-	ret = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_cb;</div><div class='add'>+	obj-&gt;uevent.event_file = READ_ONCE(attrs-&gt;ufile-&gt;default_async_file);</div><div class='add'>+	if (obj-&gt;uevent.event_file)</div><div class='add'>+		uverbs_uobject_get(&amp;obj-&gt;uevent.event_file-&gt;uobj);</div><div class='ctx'> </div><div class='ctx'> 	if (xrcd) {</div><div class='ctx'> 		obj-&gt;uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,</div><div class='hunk'>@@ -1468,30 +1482,43 @@ static int create_qp(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 	if (pd)</div><div class='ctx'> 		uobj_put_obj_read(pd);</div><div class='ctx'> 	if (scq)</div><div class='del'>-		uobj_put_obj_read(scq);</div><div class='add'>+		rdma_lookup_put_uobject(&amp;scq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+					UVERBS_LOOKUP_READ);</div><div class='ctx'> 	if (rcq &amp;&amp; rcq != scq)</div><div class='del'>-		uobj_put_obj_read(rcq);</div><div class='add'>+		rdma_lookup_put_uobject(&amp;rcq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+					UVERBS_LOOKUP_READ);</div><div class='ctx'> 	if (srq)</div><div class='del'>-		uobj_put_obj_read(srq);</div><div class='add'>+		rdma_lookup_put_uobject(&amp;srq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+					UVERBS_LOOKUP_READ);</div><div class='ctx'> 	if (ind_tbl)</div><div class='ctx'> 		uobj_put_obj_read(ind_tbl);</div><div class='add'>+	uobj_finalize_uobj_create(&amp;obj-&gt;uevent.uobject, attrs);</div><div class='ctx'> </div><div class='del'>-	return uobj_alloc_commit(&amp;obj-&gt;uevent.uobject, attrs);</div><div class='del'>-err_cb:</div><div class='del'>-	ib_destroy_qp(qp);</div><div class='add'>+	resp.base.qpn             = qp-&gt;qp_num;</div><div class='add'>+	resp.base.qp_handle       = obj-&gt;uevent.uobject.id;</div><div class='add'>+	resp.base.max_recv_sge    = attr.cap.max_recv_sge;</div><div class='add'>+	resp.base.max_send_sge    = attr.cap.max_send_sge;</div><div class='add'>+	resp.base.max_recv_wr     = attr.cap.max_recv_wr;</div><div class='add'>+	resp.base.max_send_wr     = attr.cap.max_send_wr;</div><div class='add'>+	resp.base.max_inline_data = attr.cap.max_inline_data;</div><div class='add'>+	resp.response_length = uverbs_response_length(attrs, sizeof(resp));</div><div class='add'>+	return uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='ctx'> </div><div class='ctx'> err_put:</div><div class='ctx'> 	if (!IS_ERR(xrcd_uobj))</div><div class='ctx'> 		uobj_put_read(xrcd_uobj);</div><div class='del'>-	if (pd)</div><div class='add'>+	if (!IS_ERR_OR_NULL(pd))</div><div class='ctx'> 		uobj_put_obj_read(pd);</div><div class='del'>-	if (scq)</div><div class='del'>-		uobj_put_obj_read(scq);</div><div class='del'>-	if (rcq &amp;&amp; rcq != scq)</div><div class='del'>-		uobj_put_obj_read(rcq);</div><div class='del'>-	if (srq)</div><div class='del'>-		uobj_put_obj_read(srq);</div><div class='del'>-	if (ind_tbl)</div><div class='add'>+	if (!IS_ERR_OR_NULL(scq))</div><div class='add'>+		rdma_lookup_put_uobject(&amp;scq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+					UVERBS_LOOKUP_READ);</div><div class='add'>+	if (!IS_ERR_OR_NULL(rcq) &amp;&amp; rcq != scq)</div><div class='add'>+		rdma_lookup_put_uobject(&amp;rcq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+					UVERBS_LOOKUP_READ);</div><div class='add'>+	if (!IS_ERR_OR_NULL(srq))</div><div class='add'>+		rdma_lookup_put_uobject(&amp;srq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+					UVERBS_LOOKUP_READ);</div><div class='add'>+	if (!IS_ERR_OR_NULL(ind_tbl))</div><div class='ctx'> 		uobj_put_obj_read(ind_tbl);</div><div class='ctx'> </div><div class='ctx'> 	uobj_alloc_abort(&amp;obj-&gt;uevent.uobject, attrs);</div><div class='hunk'>@@ -1546,14 +1573,14 @@ static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> </div><div class='ctx'> static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='add'>+	struct ib_uverbs_create_qp_resp resp = {};</div><div class='ctx'> 	struct ib_uverbs_open_qp        cmd;</div><div class='del'>-	struct ib_uverbs_create_qp_resp resp;</div><div class='ctx'> 	struct ib_uqp_object           *obj;</div><div class='ctx'> 	struct ib_xrcd		       *xrcd;</div><div class='del'>-	struct ib_uobject	       *uninitialized_var(xrcd_uobj);</div><div class='ctx'> 	struct ib_qp                   *qp;</div><div class='del'>-	struct ib_qp_open_attr          attr;</div><div class='add'>+	struct ib_qp_open_attr          attr = {};</div><div class='ctx'> 	int ret;</div><div class='add'>+	struct ib_uobject *xrcd_uobj;</div><div class='ctx'> 	struct ib_device *ib_dev;</div><div class='ctx'> </div><div class='ctx'> 	ret = uverbs_request(attrs, &amp;cmd, sizeof(cmd));</div><div class='hunk'>@@ -1578,11 +1605,9 @@ static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	attr.event_handler = ib_uverbs_qp_event_handler;</div><div class='del'>-	attr.qp_context    = attrs-&gt;ufile;</div><div class='ctx'> 	attr.qp_num        = cmd.qpn;</div><div class='ctx'> 	attr.qp_type       = cmd.qp_type;</div><div class='ctx'> </div><div class='del'>-	obj-&gt;uevent.events_reported = 0;</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;obj-&gt;uevent.event_list);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;obj-&gt;mcast_list);</div><div class='ctx'> </div><div class='hunk'>@@ -1595,23 +1620,16 @@ static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	obj-&gt;uevent.uobject.object = qp;</div><div class='ctx'> 	obj-&gt;uevent.uobject.user_handle = cmd.user_handle;</div><div class='ctx'> </div><div class='del'>-	memset(&amp;resp, 0, sizeof resp);</div><div class='del'>-	resp.qpn       = qp-&gt;qp_num;</div><div class='del'>-	resp.qp_handle = obj-&gt;uevent.uobject.id;</div><div class='del'>-</div><div class='del'>-	ret = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_destroy;</div><div class='del'>-</div><div class='ctx'> 	obj-&gt;uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);</div><div class='ctx'> 	atomic_inc(&amp;obj-&gt;uxrcd-&gt;refcnt);</div><div class='del'>-	qp-&gt;uobject = &amp;obj-&gt;uevent.uobject;</div><div class='add'>+	qp-&gt;uobject = obj;</div><div class='ctx'> 	uobj_put_read(xrcd_uobj);</div><div class='add'>+	uobj_finalize_uobj_create(&amp;obj-&gt;uevent.uobject, attrs);</div><div class='ctx'> </div><div class='del'>-	return uobj_alloc_commit(&amp;obj-&gt;uevent.uobject, attrs);</div><div class='add'>+	resp.qpn = qp-&gt;qp_num;</div><div class='add'>+	resp.qp_handle = obj-&gt;uevent.uobject.id;</div><div class='add'>+	return uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='ctx'> </div><div class='del'>-err_destroy:</div><div class='del'>-	ib_destroy_qp(qp);</div><div class='ctx'> err_xrcd:</div><div class='ctx'> 	uobj_put_read(xrcd_uobj);</div><div class='ctx'> err_put:</div><div class='hunk'>@@ -1662,14 +1680,15 @@ static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);</div><div class='del'>-	if (!qp) {</div><div class='del'>-		ret = -EINVAL;</div><div class='add'>+	if (IS_ERR(qp)) {</div><div class='add'>+		ret = PTR_ERR(qp);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr);</div><div class='ctx'> </div><div class='del'>-	uobj_put_obj_read(qp);</div><div class='add'>+	rdma_lookup_put_uobject(&amp;qp-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> </div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto out;</div><div class='hunk'>@@ -1767,8 +1786,8 @@ static int modify_qp(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> </div><div class='ctx'> 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd-&gt;base.qp_handle,</div><div class='ctx'> 			       attrs);</div><div class='del'>-	if (!qp) {</div><div class='del'>-		ret = -EINVAL;</div><div class='add'>+	if (IS_ERR(qp)) {</div><div class='add'>+		ret = PTR_ERR(qp);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -1856,8 +1875,15 @@ static int modify_qp(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 		attr-&gt;path_mtu = cmd-&gt;base.path_mtu;</div><div class='ctx'> 	if (cmd-&gt;base.attr_mask &amp; IB_QP_PATH_MIG_STATE)</div><div class='ctx'> 		attr-&gt;path_mig_state = cmd-&gt;base.path_mig_state;</div><div class='del'>-	if (cmd-&gt;base.attr_mask &amp; IB_QP_QKEY)</div><div class='add'>+	if (cmd-&gt;base.attr_mask &amp; IB_QP_QKEY) {</div><div class='add'>+		if (cmd-&gt;base.qkey &amp; IB_QP_SET_QKEY &amp;&amp;</div><div class='add'>+		    !(rdma_nl_get_privileged_qkey() ||</div><div class='add'>+		      rdma_uattrs_has_raw_cap(attrs))) {</div><div class='add'>+			ret = -EPERM;</div><div class='add'>+			goto release_qp;</div><div class='add'>+		}</div><div class='ctx'> 		attr-&gt;qkey = cmd-&gt;base.qkey;</div><div class='add'>+	}</div><div class='ctx'> 	if (cmd-&gt;base.attr_mask &amp; IB_QP_RQ_PSN)</div><div class='ctx'> 		attr-&gt;rq_psn = cmd-&gt;base.rq_psn;</div><div class='ctx'> 	if (cmd-&gt;base.attr_mask &amp; IB_QP_SQ_PSN)</div><div class='hunk'>@@ -1906,7 +1932,8 @@ static int modify_qp(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 				      &amp;attrs-&gt;driver_udata);</div><div class='ctx'> </div><div class='ctx'> release_qp:</div><div class='del'>-	uobj_put_obj_read(qp);</div><div class='add'>+	rdma_lookup_put_uobject(&amp;qp-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> out:</div><div class='ctx'> 	kfree(attr);</div><div class='ctx'> </div><div class='hunk'>@@ -1922,8 +1949,7 @@ static int ib_uverbs_modify_qp(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	if (cmd.base.attr_mask &amp;</div><div class='del'>-	    ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK &lt;&lt; 1) - 1))</div><div class='add'>+	if (cmd.base.attr_mask &amp; ~IB_QP_ATTR_STANDARD_BITS)</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	return modify_qp(attrs, &amp;cmd);</div><div class='hunk'>@@ -1945,10 +1971,7 @@ static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	 * Last bit is reserved for extending the attr_mask by</div><div class='ctx'> 	 * using another field.</div><div class='ctx'> 	 */</div><div class='del'>-	BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 &lt;&lt; 31));</div><div class='del'>-</div><div class='del'>-	if (cmd.base.attr_mask &amp;</div><div class='del'>-	    ~((IB_USER_LAST_QP_ATTR_MASK &lt;&lt; 1) - 1))</div><div class='add'>+	if (cmd.base.attr_mask &amp; ~(IB_QP_ATTR_STANDARD_BITS | IB_QP_RATE_LIMIT))</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	ret = modify_qp(attrs, &amp;cmd);</div><div class='hunk'>@@ -1985,12 +2008,13 @@ static int ib_uverbs_destroy_qp(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> </div><div class='ctx'> static void *alloc_wr(size_t wr_size, __u32 num_sge)</div><div class='ctx'> {</div><div class='del'>-	if (num_sge &gt;= (U32_MAX - ALIGN(wr_size, sizeof (struct ib_sge))) /</div><div class='del'>-		       sizeof (struct ib_sge))</div><div class='add'>+	if (num_sge &gt;= (U32_MAX - ALIGN(wr_size, sizeof(struct ib_sge))) /</div><div class='add'>+			       sizeof(struct ib_sge))</div><div class='ctx'> 		return NULL;</div><div class='ctx'> </div><div class='del'>-	return kmalloc(ALIGN(wr_size, sizeof (struct ib_sge)) +</div><div class='del'>-			 num_sge * sizeof (struct ib_sge), GFP_KERNEL);</div><div class='add'>+	return kmalloc(ALIGN(wr_size, sizeof(struct ib_sge)) +</div><div class='add'>+			       num_sge * sizeof(struct ib_sge),</div><div class='add'>+		       GFP_KERNEL);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)</div><div class='hunk'>@@ -2012,11 +2036,13 @@ static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	ret = uverbs_request_start(attrs, &amp;iter, &amp;cmd, sizeof(cmd));</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='del'>-	wqes = uverbs_request_next_ptr(&amp;iter, cmd.wqe_size * cmd.wr_count);</div><div class='add'>+	wqes = uverbs_request_next_ptr(&amp;iter, size_mul(cmd.wqe_size,</div><div class='add'>+						       cmd.wr_count));</div><div class='ctx'> 	if (IS_ERR(wqes))</div><div class='ctx'> 		return PTR_ERR(wqes);</div><div class='del'>-	sgls = uverbs_request_next_ptr(</div><div class='del'>-		&amp;iter, cmd.sge_count * sizeof(struct ib_uverbs_sge));</div><div class='add'>+	sgls = uverbs_request_next_ptr(&amp;iter,</div><div class='add'>+				       size_mul(cmd.sge_count,</div><div class='add'>+						sizeof(struct ib_uverbs_sge)));</div><div class='ctx'> 	if (IS_ERR(sgls))</div><div class='ctx'> 		return PTR_ERR(sgls);</div><div class='ctx'> 	ret = uverbs_request_finish(&amp;iter);</div><div class='hunk'>@@ -2028,8 +2054,8 @@ static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);</div><div class='del'>-	if (!qp) {</div><div class='del'>-		ret = -EINVAL;</div><div class='add'>+	if (IS_ERR(qp)) {</div><div class='add'>+		ret = PTR_ERR(qp);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -2066,9 +2092,9 @@ static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> </div><div class='ctx'> 			ud-&gt;ah = uobj_get_obj_read(ah, UVERBS_OBJECT_AH,</div><div class='ctx'> 						   user_wr-&gt;wr.ud.ah, attrs);</div><div class='del'>-			if (!ud-&gt;ah) {</div><div class='add'>+			if (IS_ERR(ud-&gt;ah)) {</div><div class='add'>+				ret = PTR_ERR(ud-&gt;ah);</div><div class='ctx'> 				kfree(ud);</div><div class='del'>-				ret = -EINVAL;</div><div class='ctx'> 				goto out_put;</div><div class='ctx'> 			}</div><div class='ctx'> 			ud-&gt;remote_qpn = user_wr-&gt;wr.ud.remote_qpn;</div><div class='hunk'>@@ -2170,7 +2196,8 @@ static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		ret = ret2;</div><div class='ctx'> </div><div class='ctx'> out_put:</div><div class='del'>-	uobj_put_obj_read(qp);</div><div class='add'>+	rdma_lookup_put_uobject(&amp;qp-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> </div><div class='ctx'> 	while (wr) {</div><div class='ctx'> 		if (is_ud &amp;&amp; ud_wr(wr)-&gt;ah)</div><div class='hunk'>@@ -2198,14 +2225,14 @@ ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count,</div><div class='ctx'> 	const struct ib_sge __user *sgls;</div><div class='ctx'> 	const void __user *wqes;</div><div class='ctx'> </div><div class='del'>-	if (wqe_size &lt; sizeof (struct ib_uverbs_recv_wr))</div><div class='add'>+	if (wqe_size &lt; sizeof(struct ib_uverbs_recv_wr))</div><div class='ctx'> 		return ERR_PTR(-EINVAL);</div><div class='ctx'> </div><div class='del'>-	wqes = uverbs_request_next_ptr(iter, wqe_size * wr_count);</div><div class='add'>+	wqes = uverbs_request_next_ptr(iter, size_mul(wqe_size, wr_count));</div><div class='ctx'> 	if (IS_ERR(wqes))</div><div class='ctx'> 		return ERR_CAST(wqes);</div><div class='del'>-	sgls = uverbs_request_next_ptr(</div><div class='del'>-		iter, sge_count * sizeof(struct ib_uverbs_sge));</div><div class='add'>+	sgls = uverbs_request_next_ptr(iter, size_mul(sge_count,</div><div class='add'>+						      sizeof(struct ib_uverbs_sge)));</div><div class='ctx'> 	if (IS_ERR(sgls))</div><div class='ctx'> 		return ERR_CAST(sgls);</div><div class='ctx'> 	ret = uverbs_request_finish(iter);</div><div class='hunk'>@@ -2231,14 +2258,14 @@ ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count,</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='ctx'> 		if (user_wr-&gt;num_sge &gt;=</div><div class='del'>-		    (U32_MAX - ALIGN(sizeof *next, sizeof (struct ib_sge))) /</div><div class='del'>-		    sizeof (struct ib_sge)) {</div><div class='add'>+		    (U32_MAX - ALIGN(sizeof(*next), sizeof(struct ib_sge))) /</div><div class='add'>+			    sizeof(struct ib_sge)) {</div><div class='ctx'> 			ret = -EINVAL;</div><div class='ctx'> 			goto err;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) +</div><div class='del'>-			       user_wr-&gt;num_sge * sizeof (struct ib_sge),</div><div class='add'>+		next = kmalloc(ALIGN(sizeof(*next), sizeof(struct ib_sge)) +</div><div class='add'>+				       user_wr-&gt;num_sge * sizeof(struct ib_sge),</div><div class='ctx'> 			       GFP_KERNEL);</div><div class='ctx'> 		if (!next) {</div><div class='ctx'> 			ret = -ENOMEM;</div><div class='hunk'>@@ -2256,8 +2283,8 @@ ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count,</div><div class='ctx'> 		next-&gt;num_sge    = user_wr-&gt;num_sge;</div><div class='ctx'> </div><div class='ctx'> 		if (next-&gt;num_sge) {</div><div class='del'>-			next-&gt;sg_list = (void *) next +</div><div class='del'>-				ALIGN(sizeof *next, sizeof (struct ib_sge));</div><div class='add'>+			next-&gt;sg_list = (void *)next +</div><div class='add'>+				ALIGN(sizeof(*next), sizeof(struct ib_sge));</div><div class='ctx'> 			if (copy_from_user(next-&gt;sg_list, sgls + sg_ind,</div><div class='ctx'> 					   next-&gt;num_sge *</div><div class='ctx'> 						   sizeof(struct ib_sge))) {</div><div class='hunk'>@@ -2304,15 +2331,16 @@ static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		return PTR_ERR(wr);</div><div class='ctx'> </div><div class='ctx'> 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);</div><div class='del'>-	if (!qp) {</div><div class='del'>-		ret = -EINVAL;</div><div class='add'>+	if (IS_ERR(qp)) {</div><div class='add'>+		ret = PTR_ERR(qp);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	resp.bad_wr = 0;</div><div class='ctx'> 	ret = qp-&gt;device-&gt;ops.post_recv(qp-&gt;real_qp, wr, &amp;bad_wr);</div><div class='ctx'> </div><div class='del'>-	uobj_put_obj_read(qp);</div><div class='add'>+	rdma_lookup_put_uobject(&amp;qp-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		for (next = wr; next; next = next-&gt;next) {</div><div class='ctx'> 			++resp.bad_wr;</div><div class='hunk'>@@ -2354,15 +2382,16 @@ static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		return PTR_ERR(wr);</div><div class='ctx'> </div><div class='ctx'> 	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);</div><div class='del'>-	if (!srq) {</div><div class='del'>-		ret = -EINVAL;</div><div class='add'>+	if (IS_ERR(srq)) {</div><div class='add'>+		ret = PTR_ERR(srq);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	resp.bad_wr = 0;</div><div class='ctx'> 	ret = srq-&gt;device-&gt;ops.post_srq_recv(srq, wr, &amp;bad_wr);</div><div class='ctx'> </div><div class='del'>-	uobj_put_obj_read(srq);</div><div class='add'>+	rdma_lookup_put_uobject(&amp;srq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> </div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		for (next = wr; next; next = next-&gt;next) {</div><div class='hunk'>@@ -2410,8 +2439,8 @@ static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);</div><div class='del'>-	if (!pd) {</div><div class='del'>-		ret = -EINVAL;</div><div class='add'>+	if (IS_ERR(pd)) {</div><div class='add'>+		ret = PTR_ERR(pd);</div><div class='ctx'> 		goto err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -2442,22 +2471,14 @@ static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	ah-&gt;uobject  = uobj;</div><div class='ctx'> 	uobj-&gt;user_handle = cmd.user_handle;</div><div class='ctx'> 	uobj-&gt;object = ah;</div><div class='del'>-</div><div class='del'>-	resp.ah_handle = uobj-&gt;id;</div><div class='del'>-</div><div class='del'>-	ret = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_copy;</div><div class='del'>-</div><div class='ctx'> 	uobj_put_obj_read(pd);</div><div class='del'>-	return uobj_alloc_commit(uobj, attrs);</div><div class='add'>+	uobj_finalize_uobj_create(uobj, attrs);</div><div class='ctx'> </div><div class='del'>-err_copy:</div><div class='del'>-	rdma_destroy_ah(ah, RDMA_DESTROY_AH_SLEEPABLE);</div><div class='add'>+	resp.ah_handle = uobj-&gt;id;</div><div class='add'>+	return uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='ctx'> </div><div class='ctx'> err_put:</div><div class='ctx'> 	uobj_put_obj_read(pd);</div><div class='del'>-</div><div class='ctx'> err:</div><div class='ctx'> 	uobj_alloc_abort(uobj, attrs);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -2488,10 +2509,10 @@ static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='ctx'> 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);</div><div class='del'>-	if (!qp)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	if (IS_ERR(qp))</div><div class='add'>+		return PTR_ERR(qp);</div><div class='ctx'> </div><div class='del'>-	obj = container_of(qp-&gt;uobject, struct ib_uqp_object, uevent.uobject);</div><div class='add'>+	obj = qp-&gt;uobject;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;obj-&gt;mcast_lock);</div><div class='ctx'> 	list_for_each_entry(mcast, &amp;obj-&gt;mcast_list, list)</div><div class='hunk'>@@ -2518,7 +2539,8 @@ static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> </div><div class='ctx'> out_put:</div><div class='ctx'> 	mutex_unlock(&amp;obj-&gt;mcast_lock);</div><div class='del'>-	uobj_put_obj_read(qp);</div><div class='add'>+	rdma_lookup_put_uobject(&amp;qp-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -2529,7 +2551,7 @@ static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	struct ib_uqp_object         *obj;</div><div class='ctx'> 	struct ib_qp                 *qp;</div><div class='ctx'> 	struct ib_uverbs_mcast_entry *mcast;</div><div class='del'>-	int                           ret = -EINVAL;</div><div class='add'>+	int                           ret;</div><div class='ctx'> 	bool                          found = false;</div><div class='ctx'> </div><div class='ctx'> 	ret = uverbs_request(attrs, &amp;cmd, sizeof(cmd));</div><div class='hunk'>@@ -2537,10 +2559,10 @@ static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='ctx'> 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);</div><div class='del'>-	if (!qp)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	if (IS_ERR(qp))</div><div class='add'>+		return PTR_ERR(qp);</div><div class='ctx'> </div><div class='del'>-	obj = container_of(qp-&gt;uobject, struct ib_uqp_object, uevent.uobject);</div><div class='add'>+	obj = qp-&gt;uobject;</div><div class='ctx'> 	mutex_lock(&amp;obj-&gt;mcast_lock);</div><div class='ctx'> </div><div class='ctx'> 	list_for_each_entry(mcast, &amp;obj-&gt;mcast_list, list)</div><div class='hunk'>@@ -2561,7 +2583,8 @@ static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> </div><div class='ctx'> out_put:</div><div class='ctx'> 	mutex_unlock(&amp;obj-&gt;mcast_lock);</div><div class='del'>-	uobj_put_obj_read(qp);</div><div class='add'>+	rdma_lookup_put_uobject(&amp;qp-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -2671,8 +2694,8 @@ static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 							UVERBS_OBJECT_FLOW_ACTION,</div><div class='ctx'> 							kern_spec-&gt;action.handle,</div><div class='ctx'> 							attrs);</div><div class='del'>-		if (!ib_spec-&gt;action.act)</div><div class='del'>-			return -EINVAL;</div><div class='add'>+		if (IS_ERR(ib_spec-&gt;action.act))</div><div class='add'>+			return PTR_ERR(ib_spec-&gt;action.act);</div><div class='ctx'> 		ib_spec-&gt;action.size =</div><div class='ctx'> 			sizeof(struct ib_flow_spec_action_handle);</div><div class='ctx'> 		flow_resources_add(uflow_res,</div><div class='hunk'>@@ -2689,8 +2712,8 @@ static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 					  UVERBS_OBJECT_COUNTERS,</div><div class='ctx'> 					  kern_spec-&gt;flow_count.handle,</div><div class='ctx'> 					  attrs);</div><div class='del'>-		if (!ib_spec-&gt;flow_count.counters)</div><div class='del'>-			return -EINVAL;</div><div class='add'>+		if (IS_ERR(ib_spec-&gt;flow_count.counters))</div><div class='add'>+			return PTR_ERR(ib_spec-&gt;flow_count.counters);</div><div class='ctx'> 		ib_spec-&gt;flow_count.size =</div><div class='ctx'> 				sizeof(struct ib_flow_spec_action_count);</div><div class='ctx'> 		flow_resources_add(uflow_res,</div><div class='hunk'>@@ -2704,12 +2727,6 @@ static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static size_t kern_spec_filter_sz(const struct ib_uverbs_flow_spec_hdr *spec)</div><div class='del'>-{</div><div class='del'>-	/* Returns user space filter size, includes padding */</div><div class='del'>-	return (spec-&gt;size - sizeof(struct ib_uverbs_flow_spec_hdr)) / 2;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static ssize_t spec_filter_size(const void *kern_spec_filter, u16 kern_filter_size,</div><div class='ctx'> 				u16 ib_real_filter_sz)</div><div class='ctx'> {</div><div class='hunk'>@@ -2748,7 +2765,7 @@ int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,</div><div class='ctx'> </div><div class='ctx'> 	switch (ib_spec-&gt;type &amp; ~IB_FLOW_SPEC_INNER) {</div><div class='ctx'> 	case IB_FLOW_SPEC_ETH:</div><div class='del'>-		ib_filter_sz = offsetof(struct ib_flow_eth_filter, real_sz);</div><div class='add'>+		ib_filter_sz = sizeof(struct ib_flow_eth_filter);</div><div class='ctx'> 		actual_filter_sz = spec_filter_size(kern_spec_mask,</div><div class='ctx'> 						    kern_filter_sz,</div><div class='ctx'> 						    ib_filter_sz);</div><div class='hunk'>@@ -2759,7 +2776,7 @@ int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,</div><div class='ctx'> 		memcpy(&amp;ib_spec-&gt;eth.mask, kern_spec_mask, actual_filter_sz);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_FLOW_SPEC_IPV4:</div><div class='del'>-		ib_filter_sz = offsetof(struct ib_flow_ipv4_filter, real_sz);</div><div class='add'>+		ib_filter_sz = sizeof(struct ib_flow_ipv4_filter);</div><div class='ctx'> 		actual_filter_sz = spec_filter_size(kern_spec_mask,</div><div class='ctx'> 						    kern_filter_sz,</div><div class='ctx'> 						    ib_filter_sz);</div><div class='hunk'>@@ -2770,7 +2787,7 @@ int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,</div><div class='ctx'> 		memcpy(&amp;ib_spec-&gt;ipv4.mask, kern_spec_mask, actual_filter_sz);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_FLOW_SPEC_IPV6:</div><div class='del'>-		ib_filter_sz = offsetof(struct ib_flow_ipv6_filter, real_sz);</div><div class='add'>+		ib_filter_sz = sizeof(struct ib_flow_ipv6_filter);</div><div class='ctx'> 		actual_filter_sz = spec_filter_size(kern_spec_mask,</div><div class='ctx'> 						    kern_filter_sz,</div><div class='ctx'> 						    ib_filter_sz);</div><div class='hunk'>@@ -2786,7 +2803,7 @@ int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_FLOW_SPEC_TCP:</div><div class='ctx'> 	case IB_FLOW_SPEC_UDP:</div><div class='del'>-		ib_filter_sz = offsetof(struct ib_flow_tcp_udp_filter, real_sz);</div><div class='add'>+		ib_filter_sz = sizeof(struct ib_flow_tcp_udp_filter);</div><div class='ctx'> 		actual_filter_sz = spec_filter_size(kern_spec_mask,</div><div class='ctx'> 						    kern_filter_sz,</div><div class='ctx'> 						    ib_filter_sz);</div><div class='hunk'>@@ -2797,7 +2814,7 @@ int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,</div><div class='ctx'> 		memcpy(&amp;ib_spec-&gt;tcp_udp.mask, kern_spec_mask, actual_filter_sz);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_FLOW_SPEC_VXLAN_TUNNEL:</div><div class='del'>-		ib_filter_sz = offsetof(struct ib_flow_tunnel_filter, real_sz);</div><div class='add'>+		ib_filter_sz = sizeof(struct ib_flow_tunnel_filter);</div><div class='ctx'> 		actual_filter_sz = spec_filter_size(kern_spec_mask,</div><div class='ctx'> 						    kern_filter_sz,</div><div class='ctx'> 						    ib_filter_sz);</div><div class='hunk'>@@ -2812,7 +2829,7 @@ int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_FLOW_SPEC_ESP:</div><div class='del'>-		ib_filter_sz = offsetof(struct ib_flow_esp_filter, real_sz);</div><div class='add'>+		ib_filter_sz = sizeof(struct ib_flow_esp_filter);</div><div class='ctx'> 		actual_filter_sz = spec_filter_size(kern_spec_mask,</div><div class='ctx'> 						    kern_filter_sz,</div><div class='ctx'> 						    ib_filter_sz);</div><div class='hunk'>@@ -2823,7 +2840,7 @@ int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,</div><div class='ctx'> 		memcpy(&amp;ib_spec-&gt;esp.mask, kern_spec_mask, actual_filter_sz);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_FLOW_SPEC_GRE:</div><div class='del'>-		ib_filter_sz = offsetof(struct ib_flow_gre_filter, real_sz);</div><div class='add'>+		ib_filter_sz = sizeof(struct ib_flow_gre_filter);</div><div class='ctx'> 		actual_filter_sz = spec_filter_size(kern_spec_mask,</div><div class='ctx'> 						    kern_filter_sz,</div><div class='ctx'> 						    ib_filter_sz);</div><div class='hunk'>@@ -2834,7 +2851,7 @@ int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,</div><div class='ctx'> 		memcpy(&amp;ib_spec-&gt;gre.mask, kern_spec_mask, actual_filter_sz);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_FLOW_SPEC_MPLS:</div><div class='del'>-		ib_filter_sz = offsetof(struct ib_flow_mpls_filter, real_sz);</div><div class='add'>+		ib_filter_sz = sizeof(struct ib_flow_mpls_filter);</div><div class='ctx'> 		actual_filter_sz = spec_filter_size(kern_spec_mask,</div><div class='ctx'> 						    kern_filter_sz,</div><div class='ctx'> 						    ib_filter_sz);</div><div class='hunk'>@@ -2853,11 +2870,16 @@ int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,</div><div class='ctx'> static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec *kern_spec,</div><div class='ctx'> 				       union ib_flow_spec *ib_spec)</div><div class='ctx'> {</div><div class='del'>-	ssize_t kern_filter_sz;</div><div class='add'>+	size_t kern_filter_sz;</div><div class='ctx'> 	void *kern_spec_mask;</div><div class='ctx'> 	void *kern_spec_val;</div><div class='ctx'> </div><div class='del'>-	kern_filter_sz = kern_spec_filter_sz(&amp;kern_spec-&gt;hdr);</div><div class='add'>+	if (check_sub_overflow((size_t)kern_spec-&gt;hdr.size,</div><div class='add'>+			       sizeof(struct ib_uverbs_flow_spec_hdr),</div><div class='add'>+			       &amp;kern_filter_sz))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	kern_filter_sz /= 2;</div><div class='ctx'> </div><div class='ctx'> 	kern_spec_val = (void *)kern_spec +</div><div class='ctx'> 		sizeof(struct ib_uverbs_flow_spec_hdr);</div><div class='hunk'>@@ -2909,26 +2931,25 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		return PTR_ERR(obj);</div><div class='ctx'> </div><div class='ctx'> 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);</div><div class='del'>-	if (!pd) {</div><div class='del'>-		err = -EINVAL;</div><div class='add'>+	if (IS_ERR(pd)) {</div><div class='add'>+		err = PTR_ERR(pd);</div><div class='ctx'> 		goto err_uobj;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);</div><div class='del'>-	if (!cq) {</div><div class='del'>-		err = -EINVAL;</div><div class='add'>+	if (IS_ERR(cq)) {</div><div class='add'>+		err = PTR_ERR(cq);</div><div class='ctx'> 		goto err_put_pd;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	wq_init_attr.cq = cq;</div><div class='ctx'> 	wq_init_attr.max_sge = cmd.max_sge;</div><div class='ctx'> 	wq_init_attr.max_wr = cmd.max_wr;</div><div class='del'>-	wq_init_attr.wq_context = attrs-&gt;ufile;</div><div class='ctx'> 	wq_init_attr.wq_type = cmd.wq_type;</div><div class='ctx'> 	wq_init_attr.event_handler = ib_uverbs_wq_event_handler;</div><div class='ctx'> 	wq_init_attr.create_flags = cmd.create_flags;</div><div class='del'>-	obj-&gt;uevent.events_reported = 0;</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;obj-&gt;uevent.event_list);</div><div class='add'>+	obj-&gt;uevent.uobject.user_handle = cmd.user_handle;</div><div class='ctx'> </div><div class='ctx'> 	wq = pd-&gt;device-&gt;ops.create_wq(pd, &amp;wq_init_attr, &amp;attrs-&gt;driver_udata);</div><div class='ctx'> 	if (IS_ERR(wq)) {</div><div class='hunk'>@@ -2936,37 +2957,34 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		goto err_put_cq;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	wq-&gt;uobject = &amp;obj-&gt;uevent.uobject;</div><div class='add'>+	wq-&gt;uobject = obj;</div><div class='ctx'> 	obj-&gt;uevent.uobject.object = wq;</div><div class='ctx'> 	wq-&gt;wq_type = wq_init_attr.wq_type;</div><div class='ctx'> 	wq-&gt;cq = cq;</div><div class='ctx'> 	wq-&gt;pd = pd;</div><div class='ctx'> 	wq-&gt;device = pd-&gt;device;</div><div class='del'>-	wq-&gt;wq_context = wq_init_attr.wq_context;</div><div class='ctx'> 	atomic_set(&amp;wq-&gt;usecnt, 0);</div><div class='ctx'> 	atomic_inc(&amp;pd-&gt;usecnt);</div><div class='ctx'> 	atomic_inc(&amp;cq-&gt;usecnt);</div><div class='del'>-	wq-&gt;uobject = &amp;obj-&gt;uevent.uobject;</div><div class='del'>-	obj-&gt;uevent.uobject.object = wq;</div><div class='add'>+	obj-&gt;uevent.event_file = READ_ONCE(attrs-&gt;ufile-&gt;default_async_file);</div><div class='add'>+	if (obj-&gt;uevent.event_file)</div><div class='add'>+		uverbs_uobject_get(&amp;obj-&gt;uevent.event_file-&gt;uobj);</div><div class='add'>+</div><div class='add'>+	uobj_put_obj_read(pd);</div><div class='add'>+	rdma_lookup_put_uobject(&amp;cq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='add'>+	uobj_finalize_uobj_create(&amp;obj-&gt;uevent.uobject, attrs);</div><div class='ctx'> </div><div class='del'>-	memset(&amp;resp, 0, sizeof(resp));</div><div class='ctx'> 	resp.wq_handle = obj-&gt;uevent.uobject.id;</div><div class='ctx'> 	resp.max_sge = wq_init_attr.max_sge;</div><div class='ctx'> 	resp.max_wr = wq_init_attr.max_wr;</div><div class='ctx'> 	resp.wqn = wq-&gt;wq_num;</div><div class='ctx'> 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));</div><div class='del'>-	err = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='del'>-	if (err)</div><div class='del'>-		goto err_copy;</div><div class='del'>-</div><div class='del'>-	uobj_put_obj_read(pd);</div><div class='del'>-	uobj_put_obj_read(cq);</div><div class='del'>-	return uobj_alloc_commit(&amp;obj-&gt;uevent.uobject, attrs);</div><div class='add'>+	return uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='ctx'> </div><div class='del'>-err_copy:</div><div class='del'>-	ib_destroy_wq(wq, &amp;attrs-&gt;driver_udata);</div><div class='ctx'> err_put_cq:</div><div class='del'>-	uobj_put_obj_read(cq);</div><div class='add'>+	rdma_lookup_put_uobject(&amp;cq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> err_put_pd:</div><div class='ctx'> 	uobj_put_obj_read(pd);</div><div class='ctx'> err_uobj:</div><div class='hunk'>@@ -3021,18 +3039,36 @@ static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);</div><div class='del'>-	if (!wq)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	if (IS_ERR(wq))</div><div class='add'>+		return PTR_ERR(wq);</div><div class='ctx'> </div><div class='del'>-	wq_attr.curr_wq_state = cmd.curr_wq_state;</div><div class='del'>-	wq_attr.wq_state = cmd.wq_state;</div><div class='ctx'> 	if (cmd.attr_mask &amp; IB_WQ_FLAGS) {</div><div class='ctx'> 		wq_attr.flags = cmd.flags;</div><div class='ctx'> 		wq_attr.flags_mask = cmd.flags_mask;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	if (cmd.attr_mask &amp; IB_WQ_CUR_STATE) {</div><div class='add'>+		if (cmd.curr_wq_state &gt; IB_WQS_ERR)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		wq_attr.curr_wq_state = cmd.curr_wq_state;</div><div class='add'>+	} else {</div><div class='add'>+		wq_attr.curr_wq_state = wq-&gt;state;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (cmd.attr_mask &amp; IB_WQ_STATE) {</div><div class='add'>+		if (cmd.wq_state &gt; IB_WQS_ERR)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		wq_attr.wq_state = cmd.wq_state;</div><div class='add'>+	} else {</div><div class='add'>+		wq_attr.wq_state = wq_attr.curr_wq_state;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	ret = wq-&gt;device-&gt;ops.modify_wq(wq, &amp;wq_attr, cmd.attr_mask,</div><div class='ctx'> 					&amp;attrs-&gt;driver_udata);</div><div class='del'>-	uobj_put_obj_read(wq);</div><div class='add'>+	rdma_lookup_put_uobject(&amp;wq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -3040,14 +3076,14 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_uverbs_ex_create_rwq_ind_table cmd;</div><div class='ctx'> 	struct ib_uverbs_ex_create_rwq_ind_table_resp  resp = {};</div><div class='del'>-	struct ib_uobject		  *uobj;</div><div class='add'>+	struct ib_uobject *uobj;</div><div class='ctx'> 	int err;</div><div class='ctx'> 	struct ib_rwq_ind_table_init_attr init_attr = {};</div><div class='ctx'> 	struct ib_rwq_ind_table *rwq_ind_tbl;</div><div class='del'>-	struct ib_wq	**wqs = NULL;</div><div class='add'>+	struct ib_wq **wqs = NULL;</div><div class='ctx'> 	u32 *wqs_handles = NULL;</div><div class='ctx'> 	struct ib_wq	*wq = NULL;</div><div class='del'>-	int i, j, num_read_wqs;</div><div class='add'>+	int i, num_read_wqs;</div><div class='ctx'> 	u32 num_wq_handles;</div><div class='ctx'> 	struct uverbs_req_iter iter;</div><div class='ctx'> 	struct ib_device *ib_dev;</div><div class='hunk'>@@ -3087,12 +3123,13 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 			num_read_wqs++) {</div><div class='ctx'> 		wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ,</div><div class='ctx'> 				       wqs_handles[num_read_wqs], attrs);</div><div class='del'>-		if (!wq) {</div><div class='del'>-			err = -EINVAL;</div><div class='add'>+		if (IS_ERR(wq)) {</div><div class='add'>+			err = PTR_ERR(wq);</div><div class='ctx'> 			goto put_wqs;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='ctx'> 		wqs[num_read_wqs] = wq;</div><div class='add'>+		atomic_inc(&amp;wqs[num_read_wqs]-&gt;usecnt);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &amp;ib_dev);</div><div class='hunk'>@@ -3101,17 +3138,15 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		goto put_wqs;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size;</div><div class='del'>-	init_attr.ind_tbl = wqs;</div><div class='del'>-</div><div class='del'>-	rwq_ind_tbl = ib_dev-&gt;ops.create_rwq_ind_table(ib_dev, &amp;init_attr,</div><div class='del'>-						       &amp;attrs-&gt;driver_udata);</div><div class='del'>-</div><div class='del'>-	if (IS_ERR(rwq_ind_tbl)) {</div><div class='del'>-		err = PTR_ERR(rwq_ind_tbl);</div><div class='add'>+	rwq_ind_tbl = rdma_zalloc_drv_obj(ib_dev, ib_rwq_ind_table);</div><div class='add'>+	if (!rwq_ind_tbl) {</div><div class='add'>+		err = -ENOMEM;</div><div class='ctx'> 		goto err_uobj;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size;</div><div class='add'>+	init_attr.ind_tbl = wqs;</div><div class='add'>+</div><div class='ctx'> 	rwq_ind_tbl-&gt;ind_tbl = wqs;</div><div class='ctx'> 	rwq_ind_tbl-&gt;log_ind_tbl_size = init_attr.log_ind_tbl_size;</div><div class='ctx'> 	rwq_ind_tbl-&gt;uobject = uobj;</div><div class='hunk'>@@ -3119,31 +3154,32 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	rwq_ind_tbl-&gt;device = ib_dev;</div><div class='ctx'> 	atomic_set(&amp;rwq_ind_tbl-&gt;usecnt, 0);</div><div class='ctx'> </div><div class='add'>+	err = ib_dev-&gt;ops.create_rwq_ind_table(rwq_ind_tbl, &amp;init_attr,</div><div class='add'>+					       &amp;attrs-&gt;driver_udata);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_create;</div><div class='add'>+</div><div class='ctx'> 	for (i = 0; i &lt; num_wq_handles; i++)</div><div class='del'>-		atomic_inc(&amp;wqs[i]-&gt;usecnt);</div><div class='add'>+		rdma_lookup_put_uobject(&amp;wqs[i]-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+					UVERBS_LOOKUP_READ);</div><div class='add'>+	kfree(wqs_handles);</div><div class='add'>+	uobj_finalize_uobj_create(uobj, attrs);</div><div class='ctx'> </div><div class='ctx'> 	resp.ind_tbl_handle = uobj-&gt;id;</div><div class='ctx'> 	resp.ind_tbl_num = rwq_ind_tbl-&gt;ind_tbl_num;</div><div class='ctx'> 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));</div><div class='add'>+	return uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='ctx'> </div><div class='del'>-	err = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='del'>-	if (err)</div><div class='del'>-		goto err_copy;</div><div class='del'>-</div><div class='del'>-	kfree(wqs_handles);</div><div class='del'>-</div><div class='del'>-	for (j = 0; j &lt; num_read_wqs; j++)</div><div class='del'>-		uobj_put_obj_read(wqs[j]);</div><div class='del'>-</div><div class='del'>-	return uobj_alloc_commit(uobj, attrs);</div><div class='del'>-</div><div class='del'>-err_copy:</div><div class='del'>-	ib_destroy_rwq_ind_table(rwq_ind_tbl);</div><div class='add'>+err_create:</div><div class='add'>+	kfree(rwq_ind_tbl);</div><div class='ctx'> err_uobj:</div><div class='ctx'> 	uobj_alloc_abort(uobj, attrs);</div><div class='ctx'> put_wqs:</div><div class='del'>-	for (j = 0; j &lt; num_read_wqs; j++)</div><div class='del'>-		uobj_put_obj_read(wqs[j]);</div><div class='add'>+	for (i = 0; i &lt; num_read_wqs; i++) {</div><div class='add'>+		rdma_lookup_put_uobject(&amp;wqs[i]-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+					UVERBS_LOOKUP_READ);</div><div class='add'>+		atomic_dec(&amp;wqs[i]-&gt;usecnt);</div><div class='add'>+	}</div><div class='ctx'> err_free:</div><div class='ctx'> 	kfree(wqs_handles);</div><div class='ctx'> 	kfree(wqs);</div><div class='hunk'>@@ -3169,7 +3205,7 @@ static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_uverbs_create_flow	  cmd;</div><div class='del'>-	struct ib_uverbs_create_flow_resp resp;</div><div class='add'>+	struct ib_uverbs_create_flow_resp resp = {};</div><div class='ctx'> 	struct ib_uobject		  *uobj;</div><div class='ctx'> 	struct ib_flow			  *flow_id;</div><div class='ctx'> 	struct ib_uverbs_flow_attr	  *kern_flow_attr;</div><div class='hunk'>@@ -3190,7 +3226,7 @@ static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	if (cmd.comp_mask)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (!capable(CAP_NET_RAW))</div><div class='add'>+	if (!rdma_uattrs_has_raw_cap(attrs))</div><div class='ctx'> 		return -EPERM;</div><div class='ctx'> </div><div class='ctx'> 	if (cmd.flow_attr.flags &gt;= IB_FLOW_ATTR_FLAGS_RESERVED)</div><div class='hunk'>@@ -3237,12 +3273,17 @@ static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		goto err_free_attr;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);</div><div class='del'>-	if (!qp) {</div><div class='add'>+	if (!rdma_is_port_valid(uobj-&gt;context-&gt;device, cmd.flow_attr.port)) {</div><div class='ctx'> 		err = -EINVAL;</div><div class='ctx'> 		goto err_uobj;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);</div><div class='add'>+	if (IS_ERR(qp)) {</div><div class='add'>+		err = PTR_ERR(qp);</div><div class='add'>+		goto err_uobj;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	if (qp-&gt;qp_type != IB_QPT_UD &amp;&amp; qp-&gt;qp_type != IB_QPT_RAW_PACKET) {</div><div class='ctx'> 		err = -EINVAL;</div><div class='ctx'> 		goto err_put;</div><div class='hunk'>@@ -3286,14 +3327,14 @@ static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		ib_spec += ((union ib_flow_spec *) ib_spec)-&gt;size;</div><div class='ctx'> 	}</div><div class='ctx'> 	if (cmd.flow_attr.size || (i != flow_attr-&gt;num_of_specs)) {</div><div class='del'>-		pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n",</div><div class='add'>+		pr_warn("create flow failed, flow %d: %u bytes left from uverb cmd\n",</div><div class='ctx'> 			i, cmd.flow_attr.size);</div><div class='ctx'> 		err = -EINVAL;</div><div class='ctx'> 		goto err_free;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	flow_id = qp-&gt;device-&gt;ops.create_flow(</div><div class='del'>-		qp, flow_attr, IB_FLOW_DOMAIN_USER, &amp;attrs-&gt;driver_udata);</div><div class='add'>+	flow_id = qp-&gt;device-&gt;ops.create_flow(qp, flow_attr,</div><div class='add'>+					      &amp;attrs-&gt;driver_udata);</div><div class='ctx'> </div><div class='ctx'> 	if (IS_ERR(flow_id)) {</div><div class='ctx'> 		err = PTR_ERR(flow_id);</div><div class='hunk'>@@ -3302,27 +3343,24 @@ static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> </div><div class='ctx'> 	ib_set_flow(uobj, flow_id, qp, qp-&gt;device, uflow_res);</div><div class='ctx'> </div><div class='del'>-	memset(&amp;resp, 0, sizeof(resp));</div><div class='del'>-	resp.flow_handle = uobj-&gt;id;</div><div class='del'>-</div><div class='del'>-	err = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='del'>-	if (err)</div><div class='del'>-		goto err_copy;</div><div class='del'>-</div><div class='del'>-	uobj_put_obj_read(qp);</div><div class='add'>+	rdma_lookup_put_uobject(&amp;qp-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> 	kfree(flow_attr);</div><div class='add'>+</div><div class='ctx'> 	if (cmd.flow_attr.num_of_specs)</div><div class='ctx'> 		kfree(kern_flow_attr);</div><div class='del'>-	return uobj_alloc_commit(uobj, attrs);</div><div class='del'>-err_copy:</div><div class='del'>-	if (!qp-&gt;device-&gt;ops.destroy_flow(flow_id))</div><div class='del'>-		atomic_dec(&amp;qp-&gt;usecnt);</div><div class='add'>+	uobj_finalize_uobj_create(uobj, attrs);</div><div class='add'>+</div><div class='add'>+	resp.flow_handle = uobj-&gt;id;</div><div class='add'>+	return uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='add'>+</div><div class='ctx'> err_free:</div><div class='ctx'> 	ib_uverbs_flow_resources_free(uflow_res);</div><div class='ctx'> err_free_flow_attr:</div><div class='ctx'> 	kfree(flow_attr);</div><div class='ctx'> err_put:</div><div class='del'>-	uobj_put_obj_read(qp);</div><div class='add'>+	rdma_lookup_put_uobject(&amp;qp-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> err_uobj:</div><div class='ctx'> 	uobj_alloc_abort(uobj, attrs);</div><div class='ctx'> err_free_attr:</div><div class='hunk'>@@ -3350,13 +3388,13 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 				struct ib_uverbs_create_xsrq *cmd,</div><div class='ctx'> 				struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	struct ib_uverbs_create_srq_resp resp;</div><div class='add'>+	struct ib_uverbs_create_srq_resp resp = {};</div><div class='ctx'> 	struct ib_usrq_object           *obj;</div><div class='ctx'> 	struct ib_pd                    *pd;</div><div class='ctx'> 	struct ib_srq                   *srq;</div><div class='del'>-	struct ib_uobject               *uninitialized_var(xrcd_uobj);</div><div class='ctx'> 	struct ib_srq_init_attr          attr;</div><div class='ctx'> 	int ret;</div><div class='add'>+	struct ib_uobject *xrcd_uobj;</div><div class='ctx'> 	struct ib_device *ib_dev;</div><div class='ctx'> </div><div class='ctx'> 	obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs,</div><div class='hunk'>@@ -3388,92 +3426,63 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 	if (ib_srq_has_cq(cmd-&gt;srq_type)) {</div><div class='ctx'> 		attr.ext.cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,</div><div class='ctx'> 						cmd-&gt;cq_handle, attrs);</div><div class='del'>-		if (!attr.ext.cq) {</div><div class='del'>-			ret = -EINVAL;</div><div class='add'>+		if (IS_ERR(attr.ext.cq)) {</div><div class='add'>+			ret = PTR_ERR(attr.ext.cq);</div><div class='ctx'> 			goto err_put_xrcd;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd-&gt;pd_handle, attrs);</div><div class='del'>-	if (!pd) {</div><div class='del'>-		ret = -EINVAL;</div><div class='add'>+	if (IS_ERR(pd)) {</div><div class='add'>+		ret = PTR_ERR(pd);</div><div class='ctx'> 		goto err_put_cq;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	attr.event_handler  = ib_uverbs_srq_event_handler;</div><div class='del'>-	attr.srq_context    = attrs-&gt;ufile;</div><div class='ctx'> 	attr.srq_type       = cmd-&gt;srq_type;</div><div class='ctx'> 	attr.attr.max_wr    = cmd-&gt;max_wr;</div><div class='ctx'> 	attr.attr.max_sge   = cmd-&gt;max_sge;</div><div class='ctx'> 	attr.attr.srq_limit = cmd-&gt;srq_limit;</div><div class='ctx'> </div><div class='del'>-	obj-&gt;uevent.events_reported = 0;</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;obj-&gt;uevent.event_list);</div><div class='add'>+	obj-&gt;uevent.uobject.user_handle = cmd-&gt;user_handle;</div><div class='ctx'> </div><div class='del'>-	srq = rdma_zalloc_drv_obj(ib_dev, ib_srq);</div><div class='del'>-	if (!srq) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err_put;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	srq-&gt;device        = pd-&gt;device;</div><div class='del'>-	srq-&gt;pd            = pd;</div><div class='del'>-	srq-&gt;srq_type	   = cmd-&gt;srq_type;</div><div class='del'>-	srq-&gt;uobject       = &amp;obj-&gt;uevent.uobject;</div><div class='del'>-	srq-&gt;event_handler = attr.event_handler;</div><div class='del'>-	srq-&gt;srq_context   = attr.srq_context;</div><div class='del'>-</div><div class='del'>-	ret = pd-&gt;device-&gt;ops.create_srq(srq, &amp;attr, udata);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_free;</div><div class='del'>-</div><div class='del'>-	if (ib_srq_has_cq(cmd-&gt;srq_type)) {</div><div class='del'>-		srq-&gt;ext.cq       = attr.ext.cq;</div><div class='del'>-		atomic_inc(&amp;attr.ext.cq-&gt;usecnt);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (cmd-&gt;srq_type == IB_SRQT_XRC) {</div><div class='del'>-		srq-&gt;ext.xrc.xrcd = attr.ext.xrc.xrcd;</div><div class='del'>-		atomic_inc(&amp;attr.ext.xrc.xrcd-&gt;usecnt);</div><div class='add'>+	srq = ib_create_srq_user(pd, &amp;attr, obj, udata);</div><div class='add'>+	if (IS_ERR(srq)) {</div><div class='add'>+		ret = PTR_ERR(srq);</div><div class='add'>+		goto err_put_pd;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	atomic_inc(&amp;pd-&gt;usecnt);</div><div class='del'>-	atomic_set(&amp;srq-&gt;usecnt, 0);</div><div class='del'>-</div><div class='ctx'> 	obj-&gt;uevent.uobject.object = srq;</div><div class='ctx'> 	obj-&gt;uevent.uobject.user_handle = cmd-&gt;user_handle;</div><div class='add'>+	obj-&gt;uevent.event_file = READ_ONCE(attrs-&gt;ufile-&gt;default_async_file);</div><div class='add'>+	if (obj-&gt;uevent.event_file)</div><div class='add'>+		uverbs_uobject_get(&amp;obj-&gt;uevent.event_file-&gt;uobj);</div><div class='ctx'> </div><div class='del'>-	memset(&amp;resp, 0, sizeof resp);</div><div class='del'>-	resp.srq_handle = obj-&gt;uevent.uobject.id;</div><div class='del'>-	resp.max_wr     = attr.attr.max_wr;</div><div class='del'>-	resp.max_sge    = attr.attr.max_sge;</div><div class='ctx'> 	if (cmd-&gt;srq_type == IB_SRQT_XRC)</div><div class='ctx'> 		resp.srqn = srq-&gt;ext.xrc.srq_num;</div><div class='ctx'> </div><div class='del'>-	ret = uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_copy;</div><div class='del'>-</div><div class='ctx'> 	if (cmd-&gt;srq_type == IB_SRQT_XRC)</div><div class='ctx'> 		uobj_put_read(xrcd_uobj);</div><div class='ctx'> </div><div class='ctx'> 	if (ib_srq_has_cq(cmd-&gt;srq_type))</div><div class='del'>-		uobj_put_obj_read(attr.ext.cq);</div><div class='add'>+		rdma_lookup_put_uobject(&amp;attr.ext.cq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+					UVERBS_LOOKUP_READ);</div><div class='ctx'> </div><div class='ctx'> 	uobj_put_obj_read(pd);</div><div class='del'>-	return uobj_alloc_commit(&amp;obj-&gt;uevent.uobject, attrs);</div><div class='add'>+	uobj_finalize_uobj_create(&amp;obj-&gt;uevent.uobject, attrs);</div><div class='ctx'> </div><div class='del'>-err_copy:</div><div class='del'>-	ib_destroy_srq_user(srq, &amp;attrs-&gt;driver_udata);</div><div class='add'>+	resp.srq_handle = obj-&gt;uevent.uobject.id;</div><div class='add'>+	resp.max_wr = attr.attr.max_wr;</div><div class='add'>+	resp.max_sge = attr.attr.max_sge;</div><div class='add'>+	return uverbs_response(attrs, &amp;resp, sizeof(resp));</div><div class='ctx'> </div><div class='del'>-err_free:</div><div class='del'>-	kfree(srq);</div><div class='del'>-err_put:</div><div class='add'>+err_put_pd:</div><div class='ctx'> 	uobj_put_obj_read(pd);</div><div class='del'>-</div><div class='ctx'> err_put_cq:</div><div class='ctx'> 	if (ib_srq_has_cq(cmd-&gt;srq_type))</div><div class='del'>-		uobj_put_obj_read(attr.ext.cq);</div><div class='add'>+		rdma_lookup_put_uobject(&amp;attr.ext.cq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+					UVERBS_LOOKUP_READ);</div><div class='ctx'> </div><div class='ctx'> err_put_xrcd:</div><div class='ctx'> 	if (cmd-&gt;srq_type == IB_SRQT_XRC) {</div><div class='hunk'>@@ -3532,8 +3541,8 @@ static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='ctx'> 	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);</div><div class='del'>-	if (!srq)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	if (IS_ERR(srq))</div><div class='add'>+		return PTR_ERR(srq);</div><div class='ctx'> </div><div class='ctx'> 	attr.max_wr    = cmd.max_wr;</div><div class='ctx'> 	attr.srq_limit = cmd.srq_limit;</div><div class='hunk'>@@ -3541,7 +3550,8 @@ static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	ret = srq-&gt;device-&gt;ops.modify_srq(srq, &amp;attr, cmd.attr_mask,</div><div class='ctx'> 					  &amp;attrs-&gt;driver_udata);</div><div class='ctx'> </div><div class='del'>-	uobj_put_obj_read(srq);</div><div class='add'>+	rdma_lookup_put_uobject(&amp;srq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -3559,12 +3569,13 @@ static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='ctx'> 	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);</div><div class='del'>-	if (!srq)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	if (IS_ERR(srq))</div><div class='add'>+		return PTR_ERR(srq);</div><div class='ctx'> </div><div class='ctx'> 	ret = ib_query_srq(srq, &amp;attr);</div><div class='ctx'> </div><div class='del'>-	uobj_put_obj_read(srq);</div><div class='add'>+	rdma_lookup_put_uobject(&amp;srq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> </div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='hunk'>@@ -3684,13 +3695,13 @@ static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);</div><div class='del'>-	if (!cq)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	if (IS_ERR(cq))</div><div class='add'>+		return PTR_ERR(cq);</div><div class='ctx'> </div><div class='ctx'> 	ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period);</div><div class='ctx'> </div><div class='del'>-	uobj_put_obj_read(cq);</div><div class='del'>-</div><div class='add'>+	rdma_lookup_put_uobject(&amp;cq-&gt;uobject-&gt;uevent.uobject,</div><div class='add'>+				UVERBS_LOOKUP_READ);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -3703,13 +3714,10 @@ static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'>  * trailing driver_data flex array. In this case the size of the base struct</div><div class='ctx'>  * cannot be changed.</div><div class='ctx'>  */</div><div class='del'>-#define offsetof_after(_struct, _member)                                       \</div><div class='del'>-	(offsetof(_struct, _member) + sizeof(((_struct *)NULL)-&gt;_member))</div><div class='del'>-</div><div class='ctx'> #define UAPI_DEF_WRITE_IO(req, resp)                                           \</div><div class='ctx'> 	.write.has_resp = 1 +                                                  \</div><div class='ctx'> 			  BUILD_BUG_ON_ZERO(offsetof(req, response) != 0) +    \</div><div class='del'>-			  BUILD_BUG_ON_ZERO(sizeof(((req *)0)-&gt;response) !=    \</div><div class='add'>+			  BUILD_BUG_ON_ZERO(sizeof_field(req, response) !=    \</div><div class='ctx'> 					    sizeof(u64)),                      \</div><div class='ctx'> 	.write.req_size = sizeof(req), .write.resp_size = sizeof(resp)</div><div class='ctx'> </div><div class='hunk'>@@ -3736,11 +3744,11 @@ static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)</div><div class='ctx'>  */</div><div class='ctx'> #define UAPI_DEF_WRITE_IO_EX(req, req_last_member, resp, resp_last_member)     \</div><div class='ctx'> 	.write.has_resp = 1,                                                   \</div><div class='del'>-	.write.req_size = offsetof_after(req, req_last_member),                \</div><div class='del'>-	.write.resp_size = offsetof_after(resp, resp_last_member)</div><div class='add'>+	.write.req_size = offsetofend(req, req_last_member),                   \</div><div class='add'>+	.write.resp_size = offsetofend(resp, resp_last_member)</div><div class='ctx'> </div><div class='ctx'> #define UAPI_DEF_WRITE_I_EX(req, req_last_member)                              \</div><div class='del'>-	.write.req_size = offsetof_after(req, req_last_member)</div><div class='add'>+	.write.req_size = offsetofend(req, req_last_member)</div><div class='ctx'> </div><div class='ctx'> const struct uapi_definition uverbs_def_write_intf[] = {</div><div class='ctx'> 	DECLARE_UVERBS_OBJECT(</div><div class='hunk'>@@ -3749,13 +3757,13 @@ const struct uapi_definition uverbs_def_write_intf[] = {</div><div class='ctx'> 				     ib_uverbs_create_ah,</div><div class='ctx'> 				     UAPI_DEF_WRITE_UDATA_IO(</div><div class='ctx'> 					     struct ib_uverbs_create_ah,</div><div class='del'>-					     struct ib_uverbs_create_ah_resp),</div><div class='del'>-				     UAPI_DEF_METHOD_NEEDS_FN(create_ah)),</div><div class='add'>+					     struct ib_uverbs_create_ah_resp)),</div><div class='ctx'> 		DECLARE_UVERBS_WRITE(</div><div class='ctx'> 			IB_USER_VERBS_CMD_DESTROY_AH,</div><div class='ctx'> 			ib_uverbs_destroy_ah,</div><div class='del'>-			UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_ah),</div><div class='del'>-			UAPI_DEF_METHOD_NEEDS_FN(destroy_ah))),</div><div class='add'>+			UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_ah)),</div><div class='add'>+		UAPI_DEF_OBJ_NEEDS_FN(create_user_ah),</div><div class='add'>+		UAPI_DEF_OBJ_NEEDS_FN(destroy_ah)),</div><div class='ctx'> </div><div class='ctx'> 	DECLARE_UVERBS_OBJECT(</div><div class='ctx'> 		UVERBS_OBJECT_COMP_CHANNEL,</div><div class='hunk'>@@ -3809,7 +3817,7 @@ const struct uapi_definition uverbs_def_write_intf[] = {</div><div class='ctx'> 			IB_USER_VERBS_EX_CMD_MODIFY_CQ,</div><div class='ctx'> 			ib_uverbs_ex_modify_cq,</div><div class='ctx'> 			UAPI_DEF_WRITE_I(struct ib_uverbs_ex_modify_cq),</div><div class='del'>-			UAPI_DEF_METHOD_NEEDS_FN(create_cq))),</div><div class='add'>+			UAPI_DEF_METHOD_NEEDS_FN(modify_cq))),</div><div class='ctx'> </div><div class='ctx'> 	DECLARE_UVERBS_OBJECT(</div><div class='ctx'> 		UVERBS_OBJECT_DEVICE,</div><div class='hunk'>@@ -4055,8 +4063,7 @@ const struct uapi_definition uverbs_def_write_intf[] = {</div><div class='ctx'> 		DECLARE_UVERBS_WRITE(</div><div class='ctx'> 			IB_USER_VERBS_CMD_CLOSE_XRCD,</div><div class='ctx'> 			ib_uverbs_close_xrcd,</div><div class='del'>-			UAPI_DEF_WRITE_I(struct ib_uverbs_close_xrcd),</div><div class='del'>-			UAPI_DEF_METHOD_NEEDS_FN(dealloc_xrcd)),</div><div class='add'>+			UAPI_DEF_WRITE_I(struct ib_uverbs_close_xrcd)),</div><div class='ctx'> 		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_QP,</div><div class='ctx'> 				     ib_uverbs_open_qp,</div><div class='ctx'> 				     UAPI_DEF_WRITE_UDATA_IO(</div><div class='hunk'>@@ -4066,8 +4073,9 @@ const struct uapi_definition uverbs_def_write_intf[] = {</div><div class='ctx'> 				     ib_uverbs_open_xrcd,</div><div class='ctx'> 				     UAPI_DEF_WRITE_UDATA_IO(</div><div class='ctx'> 					     struct ib_uverbs_open_xrcd,</div><div class='del'>-					     struct ib_uverbs_open_xrcd_resp),</div><div class='del'>-				     UAPI_DEF_METHOD_NEEDS_FN(alloc_xrcd))),</div><div class='add'>+					     struct ib_uverbs_open_xrcd_resp)),</div><div class='add'>+		UAPI_DEF_OBJ_NEEDS_FN(alloc_xrcd),</div><div class='add'>+		UAPI_DEF_OBJ_NEEDS_FN(dealloc_xrcd)),</div><div class='ctx'> </div><div class='ctx'> 	{},</div><div class='ctx'> };</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_ioctl.c b/drivers/infiniband/core/uverbs_ioctl.c<br/>index 829b0c6944d8..f80da6a67e24 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_ioctl.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_ioctl.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_ioctl.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_ioctl.c</a></div><div class='hunk'>@@ -36,13 +36,15 @@</div><div class='ctx'> #include "uverbs.h"</div><div class='ctx'> </div><div class='ctx'> struct bundle_alloc_head {</div><div class='del'>-	struct bundle_alloc_head *next;</div><div class='add'>+	struct_group_tagged(bundle_alloc_head_hdr, hdr,</div><div class='add'>+		struct bundle_alloc_head *next;</div><div class='add'>+	);</div><div class='ctx'> 	u8 data[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct bundle_priv {</div><div class='ctx'> 	/* Must be first */</div><div class='del'>-	struct bundle_alloc_head alloc_head;</div><div class='add'>+	struct bundle_alloc_head_hdr alloc_head;</div><div class='ctx'> 	struct bundle_alloc_head *allocated_mem;</div><div class='ctx'> 	size_t internal_avail;</div><div class='ctx'> 	size_t internal_used;</div><div class='hunk'>@@ -58,12 +60,13 @@ struct bundle_priv {</div><div class='ctx'> </div><div class='ctx'> 	DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);</div><div class='ctx'> 	DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);</div><div class='add'>+	DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Must be last. bundle ends in a flex array which overlaps</div><div class='ctx'> 	 * internal_buffer.</div><div class='ctx'> 	 */</div><div class='del'>-	struct uverbs_attr_bundle bundle;</div><div class='add'>+	struct uverbs_attr_bundle_hdr bundle;</div><div class='ctx'> 	u64 internal_buffer[32];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -76,9 +79,10 @@ void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,</div><div class='ctx'> 			      unsigned int num_attrs)</div><div class='ctx'> {</div><div class='ctx'> 	struct bundle_priv *pbundle;</div><div class='add'>+	struct uverbs_attr_bundle *bundle;</div><div class='ctx'> 	size_t bundle_size =</div><div class='ctx'> 		offsetof(struct bundle_priv, internal_buffer) +</div><div class='del'>-		sizeof(*pbundle-&gt;bundle.attrs) * method_elm-&gt;key_bitmap_len +</div><div class='add'>+		sizeof(*bundle-&gt;attrs) * method_elm-&gt;key_bitmap_len +</div><div class='ctx'> 		sizeof(*pbundle-&gt;uattrs) * num_attrs;</div><div class='ctx'> </div><div class='ctx'> 	method_elm-&gt;use_stack = bundle_size &lt;= sizeof(*pbundle);</div><div class='hunk'>@@ -90,7 +94,7 @@ void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * uverbs_alloc() - Quickly allocate memory for use with a bundle</div><div class='add'>+ * _uverbs_alloc() - Quickly allocate memory for use with a bundle</div><div class='ctx'>  * @bundle: The bundle</div><div class='ctx'>  * @size: Number of bytes to allocate</div><div class='ctx'>  * @flags: Allocator flags</div><div class='hunk'>@@ -106,7 +110,7 @@ __malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,</div><div class='ctx'> 			     gfp_t flags)</div><div class='ctx'> {</div><div class='ctx'> 	struct bundle_priv *pbundle =</div><div class='del'>-		container_of(bundle, struct bundle_priv, bundle);</div><div class='add'>+		container_of(&amp;bundle-&gt;hdr, struct bundle_priv, bundle);</div><div class='ctx'> 	size_t new_used;</div><div class='ctx'> 	void *res;</div><div class='ctx'> </div><div class='hunk'>@@ -127,7 +131,7 @@ __malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,</div><div class='ctx'> 	res = (void *)pbundle-&gt;internal_buffer + pbundle-&gt;internal_used;</div><div class='ctx'> 	pbundle-&gt;internal_used =</div><div class='ctx'> 		ALIGN(new_used, sizeof(*pbundle-&gt;internal_buffer));</div><div class='del'>-	if (flags &amp; __GFP_ZERO)</div><div class='add'>+	if (want_init_on_alloc(flags))</div><div class='ctx'> 		memset(res, 0, size);</div><div class='ctx'> 	return res;</div><div class='ctx'> }</div><div class='hunk'>@@ -136,7 +140,7 @@ EXPORT_SYMBOL(_uverbs_alloc);</div><div class='ctx'> static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,</div><div class='ctx'> 				   u16 len)</div><div class='ctx'> {</div><div class='del'>-	if (uattr-&gt;len &gt; sizeof(((struct ib_uverbs_attr *)0)-&gt;data))</div><div class='add'>+	if (uattr-&gt;len &gt; sizeof_field(struct ib_uverbs_attr, data))</div><div class='ctx'> 		return ib_is_buffer_cleared(u64_to_user_ptr(uattr-&gt;data) + len,</div><div class='ctx'> 					    uattr-&gt;len - len);</div><div class='ctx'> </div><div class='hunk'>@@ -148,7 +152,7 @@ static int uverbs_set_output(const struct uverbs_attr_bundle *bundle,</div><div class='ctx'> 			     const struct uverbs_attr *attr)</div><div class='ctx'> {</div><div class='ctx'> 	struct bundle_priv *pbundle =</div><div class='del'>-		container_of(bundle, struct bundle_priv, bundle);</div><div class='add'>+		container_of(&amp;bundle-&gt;hdr, struct bundle_priv, bundle);</div><div class='ctx'> 	u16 flags;</div><div class='ctx'> </div><div class='ctx'> 	flags = pbundle-&gt;uattrs[attr-&gt;ptr_attr.uattr_idx].flags |</div><div class='hunk'>@@ -165,6 +169,8 @@ static int uverbs_process_idrs_array(struct bundle_priv *pbundle,</div><div class='ctx'> 				     struct ib_uverbs_attr *uattr,</div><div class='ctx'> 				     u32 attr_bkey)</div><div class='ctx'> {</div><div class='add'>+	struct uverbs_attr_bundle *bundle =</div><div class='add'>+		container_of(&amp;pbundle-&gt;bundle, struct uverbs_attr_bundle, hdr);</div><div class='ctx'> 	const struct uverbs_attr_spec *spec = &amp;attr_uapi-&gt;spec;</div><div class='ctx'> 	size_t array_len;</div><div class='ctx'> 	u32 *idr_vals;</div><div class='hunk'>@@ -183,7 +189,7 @@ static int uverbs_process_idrs_array(struct bundle_priv *pbundle,</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	attr-&gt;uobjects =</div><div class='del'>-		uverbs_alloc(&amp;pbundle-&gt;bundle,</div><div class='add'>+		uverbs_alloc(bundle,</div><div class='ctx'> 			     array_size(array_len, sizeof(*attr-&gt;uobjects)));</div><div class='ctx'> 	if (IS_ERR(attr-&gt;uobjects))</div><div class='ctx'> 		return PTR_ERR(attr-&gt;uobjects);</div><div class='hunk'>@@ -208,7 +214,7 @@ static int uverbs_process_idrs_array(struct bundle_priv *pbundle,</div><div class='ctx'> 	for (i = 0; i != array_len; i++) {</div><div class='ctx'> 		attr-&gt;uobjects[i] = uverbs_get_uobject_from_file(</div><div class='ctx'> 			spec-&gt;u2.objs_arr.obj_type, spec-&gt;u2.objs_arr.access,</div><div class='del'>-			idr_vals[i], &amp;pbundle-&gt;bundle);</div><div class='add'>+			idr_vals[i], bundle);</div><div class='ctx'> 		if (IS_ERR(attr-&gt;uobjects[i])) {</div><div class='ctx'> 			ret = PTR_ERR(attr-&gt;uobjects[i]);</div><div class='ctx'> 			break;</div><div class='hunk'>@@ -220,24 +226,18 @@ static int uverbs_process_idrs_array(struct bundle_priv *pbundle,</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi,</div><div class='del'>-				  struct uverbs_objs_arr_attr *attr,</div><div class='del'>-				  bool commit, struct uverbs_attr_bundle *attrs)</div><div class='add'>+static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi,</div><div class='add'>+				   struct uverbs_objs_arr_attr *attr,</div><div class='add'>+				   bool commit,</div><div class='add'>+				   struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	const struct uverbs_attr_spec *spec = &amp;attr_uapi-&gt;spec;</div><div class='del'>-	int current_ret;</div><div class='del'>-	int ret = 0;</div><div class='ctx'> 	size_t i;</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i != attr-&gt;len; i++) {</div><div class='del'>-		current_ret = uverbs_finalize_object(attr-&gt;uobjects[i],</div><div class='del'>-						     spec-&gt;u2.objs_arr.access,</div><div class='del'>-						     commit, attrs);</div><div class='del'>-		if (!ret)</div><div class='del'>-			ret = current_ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='add'>+	for (i = 0; i != attr-&gt;len; i++)</div><div class='add'>+		uverbs_finalize_object(attr-&gt;uobjects[i],</div><div class='add'>+				       spec-&gt;u2.objs_arr.access, false, commit,</div><div class='add'>+				       attrs);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int uverbs_process_attr(struct bundle_priv *pbundle,</div><div class='hunk'>@@ -245,7 +245,9 @@ static int uverbs_process_attr(struct bundle_priv *pbundle,</div><div class='ctx'> 			       struct ib_uverbs_attr *uattr, u32 attr_bkey)</div><div class='ctx'> {</div><div class='ctx'> 	const struct uverbs_attr_spec *spec = &amp;attr_uapi-&gt;spec;</div><div class='del'>-	struct uverbs_attr *e = &amp;pbundle-&gt;bundle.attrs[attr_bkey];</div><div class='add'>+	struct uverbs_attr_bundle *bundle =</div><div class='add'>+		container_of(&amp;pbundle-&gt;bundle, struct uverbs_attr_bundle, hdr);</div><div class='add'>+	struct uverbs_attr *e = &amp;bundle-&gt;attrs[attr_bkey];</div><div class='ctx'> 	const struct uverbs_attr_spec *val_spec = spec;</div><div class='ctx'> 	struct uverbs_obj_attr *o_attr;</div><div class='ctx'> </div><div class='hunk'>@@ -264,7 +266,7 @@ static int uverbs_process_attr(struct bundle_priv *pbundle,</div><div class='ctx'> 			return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 		e-&gt;ptr_attr.enum_id = uattr-&gt;attr_data.enum_data.elem_id;</div><div class='del'>-	/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case UVERBS_ATTR_TYPE_PTR_IN:</div><div class='ctx'> 		/* Ensure that any data provided by userspace beyond the known</div><div class='ctx'> 		 * struct is zero. Userspace that knows how to use some future</div><div class='hunk'>@@ -276,7 +278,7 @@ static int uverbs_process_attr(struct bundle_priv *pbundle,</div><div class='ctx'> 		    !uverbs_is_attr_cleared(uattr, val_spec-&gt;u.ptr.len))</div><div class='ctx'> 			return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='del'>-	/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case UVERBS_ATTR_TYPE_PTR_OUT:</div><div class='ctx'> 		if (uattr-&gt;len &lt; val_spec-&gt;u.ptr.min_len ||</div><div class='ctx'> 		    (!val_spec-&gt;zero_trailing &amp;&amp;</div><div class='hunk'>@@ -293,7 +295,7 @@ static int uverbs_process_attr(struct bundle_priv *pbundle,</div><div class='ctx'> 		if (val_spec-&gt;alloc_and_copy &amp;&amp; !uverbs_attr_ptr_is_inline(e)) {</div><div class='ctx'> 			void *p;</div><div class='ctx'> </div><div class='del'>-			p = uverbs_alloc(&amp;pbundle-&gt;bundle, uattr-&gt;len);</div><div class='add'>+			p = uverbs_alloc(bundle, uattr-&gt;len);</div><div class='ctx'> 			if (IS_ERR(p))</div><div class='ctx'> 				return PTR_ERR(p);</div><div class='ctx'> </div><div class='hunk'>@@ -326,7 +328,7 @@ static int uverbs_process_attr(struct bundle_priv *pbundle,</div><div class='ctx'> 		 */</div><div class='ctx'> 		o_attr-&gt;uobject = uverbs_get_uobject_from_file(</div><div class='ctx'> 			spec-&gt;u.obj.obj_type, spec-&gt;u.obj.access,</div><div class='del'>-			uattr-&gt;data_s64, &amp;pbundle-&gt;bundle);</div><div class='add'>+			uattr-&gt;data_s64, bundle);</div><div class='ctx'> 		if (IS_ERR(o_attr-&gt;uobject))</div><div class='ctx'> 			return PTR_ERR(o_attr-&gt;uobject);</div><div class='ctx'> 		__set_bit(attr_bkey, pbundle-&gt;uobj_finalize);</div><div class='hunk'>@@ -342,6 +344,14 @@ static int uverbs_process_attr(struct bundle_priv *pbundle,</div><div class='ctx'> </div><div class='ctx'> 		break;</div><div class='ctx'> </div><div class='add'>+	case UVERBS_ATTR_TYPE_RAW_FD:</div><div class='add'>+		if (uattr-&gt;attr_data.reserved || uattr-&gt;len != 0 ||</div><div class='add'>+		    uattr-&gt;data_s64 &lt; INT_MIN || uattr-&gt;data_s64 &gt; INT_MAX)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		/* _uverbs_get_const_signed() is the accessor */</div><div class='add'>+		e-&gt;ptr_attr.data = uattr-&gt;data_s64;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='ctx'> 	case UVERBS_ATTR_TYPE_IDRS_ARRAY:</div><div class='ctx'> 		return uverbs_process_idrs_array(pbundle, attr_uapi,</div><div class='ctx'> 						 &amp;e-&gt;objs_arr_attr, uattr,</div><div class='hunk'>@@ -419,6 +429,8 @@ static int ib_uverbs_run_method(struct bundle_priv *pbundle,</div><div class='ctx'> 				unsigned int num_attrs)</div><div class='ctx'> {</div><div class='ctx'> 	int (*handler)(struct uverbs_attr_bundle *attrs);</div><div class='add'>+	struct uverbs_attr_bundle *bundle =</div><div class='add'>+		container_of(&amp;pbundle-&gt;bundle, struct uverbs_attr_bundle, hdr);</div><div class='ctx'> 	size_t uattrs_size = array_size(sizeof(*pbundle-&gt;uattrs), num_attrs);</div><div class='ctx'> 	unsigned int destroy_bkey = pbundle-&gt;method_elm-&gt;destroy_bkey;</div><div class='ctx'> 	unsigned int i;</div><div class='hunk'>@@ -431,7 +443,7 @@ static int ib_uverbs_run_method(struct bundle_priv *pbundle,</div><div class='ctx'> 	if (!handler)</div><div class='ctx'> 		return -EIO;</div><div class='ctx'> </div><div class='del'>-	pbundle-&gt;uattrs = uverbs_alloc(&amp;pbundle-&gt;bundle, uattrs_size);</div><div class='add'>+	pbundle-&gt;uattrs = uverbs_alloc(bundle, uattrs_size);</div><div class='ctx'> 	if (IS_ERR(pbundle-&gt;uattrs))</div><div class='ctx'> 		return PTR_ERR(pbundle-&gt;uattrs);</div><div class='ctx'> 	if (copy_from_user(pbundle-&gt;uattrs, pbundle-&gt;user_attrs, uattrs_size))</div><div class='hunk'>@@ -450,25 +462,23 @@ static int ib_uverbs_run_method(struct bundle_priv *pbundle,</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	if (pbundle-&gt;method_elm-&gt;has_udata)</div><div class='del'>-		uverbs_fill_udata(&amp;pbundle-&gt;bundle,</div><div class='del'>-				  &amp;pbundle-&gt;bundle.driver_udata,</div><div class='add'>+		uverbs_fill_udata(bundle, &amp;pbundle-&gt;bundle.driver_udata,</div><div class='ctx'> 				  UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT);</div><div class='ctx'> 	else</div><div class='ctx'> 		pbundle-&gt;bundle.driver_udata = (struct ib_udata){};</div><div class='ctx'> </div><div class='ctx'> 	if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) {</div><div class='del'>-		struct uverbs_obj_attr *destroy_attr =</div><div class='del'>-			&amp;pbundle-&gt;bundle.attrs[destroy_bkey].obj_attr;</div><div class='add'>+		struct uverbs_obj_attr *destroy_attr = &amp;bundle-&gt;attrs[destroy_bkey].obj_attr;</div><div class='ctx'> </div><div class='del'>-		ret = uobj_destroy(destroy_attr-&gt;uobject, &amp;pbundle-&gt;bundle);</div><div class='add'>+		ret = uobj_destroy(destroy_attr-&gt;uobject, bundle);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			return ret;</div><div class='ctx'> 		__clear_bit(destroy_bkey, pbundle-&gt;uobj_finalize);</div><div class='ctx'> </div><div class='del'>-		ret = handler(&amp;pbundle-&gt;bundle);</div><div class='add'>+		ret = handler(bundle);</div><div class='ctx'> 		uobj_put_destroy(destroy_attr-&gt;uobject);</div><div class='ctx'> 	} else {</div><div class='del'>-		ret = handler(&amp;pbundle-&gt;bundle);</div><div class='add'>+		ret = handler(bundle);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='hunk'>@@ -478,10 +488,10 @@ static int ib_uverbs_run_method(struct bundle_priv *pbundle,</div><div class='ctx'> 	 */</div><div class='ctx'> 	if (!ret &amp;&amp; pbundle-&gt;method_elm-&gt;has_udata) {</div><div class='ctx'> 		const struct uverbs_attr *attr =</div><div class='del'>-			uverbs_attr_get(&amp;pbundle-&gt;bundle, UVERBS_ATTR_UHW_OUT);</div><div class='add'>+			uverbs_attr_get(bundle, UVERBS_ATTR_UHW_OUT);</div><div class='ctx'> </div><div class='ctx'> 		if (!IS_ERR(attr))</div><div class='del'>-			ret = uverbs_set_output(&amp;pbundle-&gt;bundle, attr);</div><div class='add'>+			ret = uverbs_set_output(bundle, attr);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='hunk'>@@ -495,35 +505,33 @@ static int ib_uverbs_run_method(struct bundle_priv *pbundle,</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int bundle_destroy(struct bundle_priv *pbundle, bool commit)</div><div class='add'>+static void bundle_destroy(struct bundle_priv *pbundle, bool commit)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned int key_bitmap_len = pbundle-&gt;method_elm-&gt;key_bitmap_len;</div><div class='add'>+	struct uverbs_attr_bundle *bundle =</div><div class='add'>+		container_of(&amp;pbundle-&gt;bundle, struct uverbs_attr_bundle, hdr);</div><div class='ctx'> 	struct bundle_alloc_head *memblock;</div><div class='ctx'> 	unsigned int i;</div><div class='del'>-	int ret = 0;</div><div class='ctx'> </div><div class='ctx'> 	/* fast path for simple uobjects */</div><div class='ctx'> 	i = -1;</div><div class='ctx'> 	while ((i = find_next_bit(pbundle-&gt;uobj_finalize, key_bitmap_len,</div><div class='ctx'> 				  i + 1)) &lt; key_bitmap_len) {</div><div class='del'>-		struct uverbs_attr *attr = &amp;pbundle-&gt;bundle.attrs[i];</div><div class='del'>-		int current_ret;</div><div class='add'>+		struct uverbs_attr *attr = &amp;bundle-&gt;attrs[i];</div><div class='ctx'> </div><div class='del'>-		current_ret = uverbs_finalize_object(</div><div class='add'>+		uverbs_finalize_object(</div><div class='ctx'> 			attr-&gt;obj_attr.uobject,</div><div class='del'>-			attr-&gt;obj_attr.attr_elm-&gt;spec.u.obj.access, commit,</div><div class='del'>-			&amp;pbundle-&gt;bundle);</div><div class='del'>-		if (!ret)</div><div class='del'>-			ret = current_ret;</div><div class='add'>+			attr-&gt;obj_attr.attr_elm-&gt;spec.u.obj.access,</div><div class='add'>+			test_bit(i, pbundle-&gt;uobj_hw_obj_valid),</div><div class='add'>+			commit, bundle);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	i = -1;</div><div class='ctx'> 	while ((i = find_next_bit(pbundle-&gt;spec_finalize, key_bitmap_len,</div><div class='ctx'> 				  i + 1)) &lt; key_bitmap_len) {</div><div class='del'>-		struct uverbs_attr *attr = &amp;pbundle-&gt;bundle.attrs[i];</div><div class='add'>+		struct uverbs_attr *attr = &amp;bundle-&gt;attrs[i];</div><div class='ctx'> 		const struct uverbs_api_attr *attr_uapi;</div><div class='ctx'> 		void __rcu **slot;</div><div class='del'>-		int current_ret;</div><div class='ctx'> </div><div class='ctx'> 		slot = uapi_get_attr_for_method(</div><div class='ctx'> 			pbundle,</div><div class='hunk'>@@ -534,11 +542,8 @@ static int bundle_destroy(struct bundle_priv *pbundle, bool commit)</div><div class='ctx'> 		attr_uapi = rcu_dereference_protected(*slot, true);</div><div class='ctx'> </div><div class='ctx'> 		if (attr_uapi-&gt;spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {</div><div class='del'>-			current_ret = uverbs_free_idrs_array(</div><div class='del'>-				attr_uapi, &amp;attr-&gt;objs_arr_attr, commit,</div><div class='del'>-				&amp;pbundle-&gt;bundle);</div><div class='del'>-			if (!ret)</div><div class='del'>-				ret = current_ret;</div><div class='add'>+			uverbs_free_idrs_array(attr_uapi, &amp;attr-&gt;objs_arr_attr,</div><div class='add'>+					       commit, bundle);</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -548,8 +553,6 @@ static int bundle_destroy(struct bundle_priv *pbundle, bool commit)</div><div class='ctx'> 		memblock = memblock-&gt;next;</div><div class='ctx'> 		kvfree(tmp);</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,</div><div class='hunk'>@@ -562,7 +565,6 @@ static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,</div><div class='ctx'> 	struct bundle_priv *pbundle;</div><div class='ctx'> 	struct bundle_priv onstack;</div><div class='ctx'> 	void __rcu **slot;</div><div class='del'>-	int destroy_ret;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (unlikely(hdr-&gt;driver_id != uapi-&gt;driver_id))</div><div class='hunk'>@@ -584,7 +586,8 @@ static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,</div><div class='ctx'> 			method_elm-&gt;bundle_size -</div><div class='ctx'> 			offsetof(struct bundle_priv, internal_buffer);</div><div class='ctx'> 		pbundle-&gt;alloc_head.next = NULL;</div><div class='del'>-		pbundle-&gt;allocated_mem = &amp;pbundle-&gt;alloc_head;</div><div class='add'>+		pbundle-&gt;allocated_mem = container_of(&amp;pbundle-&gt;alloc_head,</div><div class='add'>+						struct bundle_alloc_head, hdr);</div><div class='ctx'> 	} else {</div><div class='ctx'> 		pbundle = &amp;onstack;</div><div class='ctx'> 		pbundle-&gt;internal_avail = sizeof(pbundle-&gt;internal_buffer);</div><div class='hunk'>@@ -602,18 +605,18 @@ static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,</div><div class='ctx'> 	pbundle-&gt;user_attrs = user_attrs;</div><div class='ctx'> </div><div class='ctx'> 	pbundle-&gt;internal_used = ALIGN(pbundle-&gt;method_elm-&gt;key_bitmap_len *</div><div class='del'>-					       sizeof(*pbundle-&gt;bundle.attrs),</div><div class='del'>-				       sizeof(*pbundle-&gt;internal_buffer));</div><div class='add'>+					       sizeof(*container_of(&amp;pbundle-&gt;bundle,</div><div class='add'>+							struct uverbs_attr_bundle, hdr)-&gt;attrs),</div><div class='add'>+					       sizeof(*pbundle-&gt;internal_buffer));</div><div class='ctx'> 	memset(pbundle-&gt;bundle.attr_present, 0,</div><div class='ctx'> 	       sizeof(pbundle-&gt;bundle.attr_present));</div><div class='ctx'> 	memset(pbundle-&gt;uobj_finalize, 0, sizeof(pbundle-&gt;uobj_finalize));</div><div class='ctx'> 	memset(pbundle-&gt;spec_finalize, 0, sizeof(pbundle-&gt;spec_finalize));</div><div class='add'>+	memset(pbundle-&gt;uobj_hw_obj_valid, 0,</div><div class='add'>+	       sizeof(pbundle-&gt;uobj_hw_obj_valid));</div><div class='ctx'> </div><div class='ctx'> 	ret = ib_uverbs_run_method(pbundle, hdr-&gt;num_attrs);</div><div class='del'>-	destroy_ret = bundle_destroy(pbundle, ret == 0);</div><div class='del'>-	if (unlikely(destroy_ret &amp;&amp; !ret))</div><div class='del'>-		return destroy_ret;</div><div class='del'>-</div><div class='add'>+	bundle_destroy(pbundle, ret == 0);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -707,11 +710,13 @@ void uverbs_fill_udata(struct uverbs_attr_bundle *bundle,</div><div class='ctx'> 		       unsigned int attr_out)</div><div class='ctx'> {</div><div class='ctx'> 	struct bundle_priv *pbundle =</div><div class='del'>-		container_of(bundle, struct bundle_priv, bundle);</div><div class='add'>+		container_of(&amp;bundle-&gt;hdr, struct bundle_priv, bundle);</div><div class='add'>+	struct uverbs_attr_bundle *bundle_aux =</div><div class='add'>+		container_of(&amp;pbundle-&gt;bundle, struct uverbs_attr_bundle, hdr);</div><div class='ctx'> 	const struct uverbs_attr *in =</div><div class='del'>-		uverbs_attr_get(&amp;pbundle-&gt;bundle, attr_in);</div><div class='add'>+		uverbs_attr_get(bundle_aux, attr_in);</div><div class='ctx'> 	const struct uverbs_attr *out =</div><div class='del'>-		uverbs_attr_get(&amp;pbundle-&gt;bundle, attr_out);</div><div class='add'>+		uverbs_attr_get(bundle_aux, attr_out);</div><div class='ctx'> </div><div class='ctx'> 	if (!IS_ERR(in)) {</div><div class='ctx'> 		udata-&gt;inlen = in-&gt;ptr_attr.len;</div><div class='hunk'>@@ -767,9 +772,10 @@ int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx)</div><div class='ctx'> 	return uverbs_set_output(bundle, attr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int _uverbs_get_const(s64 *to, const struct uverbs_attr_bundle *attrs_bundle,</div><div class='del'>-		      size_t idx, s64 lower_bound, u64 upper_bound,</div><div class='del'>-		      s64  *def_val)</div><div class='add'>+int _uverbs_get_const_signed(s64 *to,</div><div class='add'>+			     const struct uverbs_attr_bundle *attrs_bundle,</div><div class='add'>+			     size_t idx, s64 lower_bound, u64 upper_bound,</div><div class='add'>+			     s64  *def_val)</div><div class='ctx'> {</div><div class='ctx'> 	const struct uverbs_attr *attr;</div><div class='ctx'> </div><div class='hunk'>@@ -788,13 +794,39 @@ int _uverbs_get_const(s64 *to, const struct uverbs_attr_bundle *attrs_bundle,</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(_uverbs_get_const);</div><div class='add'>+EXPORT_SYMBOL(_uverbs_get_const_signed);</div><div class='add'>+</div><div class='add'>+int _uverbs_get_const_unsigned(u64 *to,</div><div class='add'>+			       const struct uverbs_attr_bundle *attrs_bundle,</div><div class='add'>+			       size_t idx, u64 upper_bound, u64 *def_val)</div><div class='add'>+{</div><div class='add'>+	const struct uverbs_attr *attr;</div><div class='add'>+</div><div class='add'>+	attr = uverbs_attr_get(attrs_bundle, idx);</div><div class='add'>+	if (IS_ERR(attr)) {</div><div class='add'>+		if ((PTR_ERR(attr) != -ENOENT) || !def_val)</div><div class='add'>+			return PTR_ERR(attr);</div><div class='add'>+</div><div class='add'>+		*to = *def_val;</div><div class='add'>+	} else {</div><div class='add'>+		*to = attr-&gt;ptr_attr.data;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (*to &gt; upper_bound)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(_uverbs_get_const_unsigned);</div><div class='ctx'> </div><div class='ctx'> int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,</div><div class='ctx'> 				  size_t idx, const void *from, size_t size)</div><div class='ctx'> {</div><div class='ctx'> 	const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);</div><div class='ctx'> </div><div class='add'>+	if (IS_ERR(attr))</div><div class='add'>+		return PTR_ERR(attr);</div><div class='add'>+</div><div class='ctx'> 	if (size &lt; attr-&gt;ptr_attr.len) {</div><div class='ctx'> 		if (clear_user(u64_to_user_ptr(attr-&gt;ptr_attr.data) + size,</div><div class='ctx'> 			       attr-&gt;ptr_attr.len - size))</div><div class='hunk'>@@ -802,3 +834,16 @@ int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,</div><div class='ctx'> 	}</div><div class='ctx'> 	return uverbs_copy_to(bundle, idx, from, size);</div><div class='ctx'> }</div><div class='add'>+EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero);</div><div class='add'>+</div><div class='add'>+/* Once called an abort will call through to the type's destroy_hw() */</div><div class='add'>+void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,</div><div class='add'>+				 u16 idx)</div><div class='add'>+{</div><div class='add'>+	struct bundle_priv *pbundle =</div><div class='add'>+		container_of(&amp;bundle-&gt;hdr, struct bundle_priv, bundle);</div><div class='add'>+</div><div class='add'>+	__set_bit(uapi_bkey_attr(uapi_key_attr(idx)),</div><div class='add'>+		  pbundle-&gt;uobj_hw_obj_valid);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(uverbs_finalize_uobj_create);</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c<br/>index 84a5e9a6d483..973fe2c7ef53 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_main.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_main.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_main.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_main.c</a></div><div class='hunk'>@@ -51,6 +51,8 @@</div><div class='ctx'> </div><div class='ctx'> #include &lt;rdma/ib.h&gt;</div><div class='ctx'> #include &lt;rdma/uverbs_std_types.h&gt;</div><div class='add'>+#include &lt;rdma/rdma_netlink.h&gt;</div><div class='add'>+#include &lt;rdma/ib_ucaps.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "uverbs.h"</div><div class='ctx'> #include "core_priv.h"</div><div class='hunk'>@@ -71,11 +73,23 @@ enum {</div><div class='ctx'> #define IB_UVERBS_BASE_DEV	MKDEV(IB_UVERBS_MAJOR, IB_UVERBS_BASE_MINOR)</div><div class='ctx'> </div><div class='ctx'> static dev_t dynamic_uverbs_dev;</div><div class='del'>-static struct class *uverbs_class;</div><div class='ctx'> </div><div class='ctx'> static DEFINE_IDA(uverbs_ida);</div><div class='del'>-static void ib_uverbs_add_one(struct ib_device *device);</div><div class='add'>+static int ib_uverbs_add_one(struct ib_device *device);</div><div class='ctx'> static void ib_uverbs_remove_one(struct ib_device *device, void *client_data);</div><div class='add'>+static struct ib_client uverbs_client;</div><div class='add'>+</div><div class='add'>+static char *uverbs_devnode(const struct device *dev, umode_t *mode)</div><div class='add'>+{</div><div class='add'>+	if (mode)</div><div class='add'>+		*mode = 0666;</div><div class='add'>+	return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct class uverbs_class = {</div><div class='add'>+	.name = "infiniband_verbs",</div><div class='add'>+	.devnode = uverbs_devnode,</div><div class='add'>+};</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * Must be called with the ufile-&gt;device-&gt;disassociate_srcu held, and the lock</div><div class='hunk'>@@ -107,8 +121,11 @@ int uverbs_dealloc_mw(struct ib_mw *mw)</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	ret = mw-&gt;device-&gt;ops.dealloc_mw(mw);</div><div class='del'>-	if (!ret)</div><div class='del'>-		atomic_dec(&amp;pd-&gt;usecnt);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	atomic_dec(&amp;pd-&gt;usecnt);</div><div class='add'>+	kfree(mw);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -119,20 +136,13 @@ static void ib_uverbs_release_dev(struct device *device)</div><div class='ctx'> </div><div class='ctx'> 	uverbs_destroy_api(dev-&gt;uapi);</div><div class='ctx'> 	cleanup_srcu_struct(&amp;dev-&gt;disassociate_srcu);</div><div class='add'>+	mutex_destroy(&amp;dev-&gt;lists_mutex);</div><div class='add'>+	mutex_destroy(&amp;dev-&gt;xrcd_tree_mutex);</div><div class='ctx'> 	kfree(dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void ib_uverbs_release_async_event_file(struct kref *ref)</div><div class='del'>-{</div><div class='del'>-	struct ib_uverbs_async_event_file *file =</div><div class='del'>-		container_of(ref, struct ib_uverbs_async_event_file, ref);</div><div class='del'>-</div><div class='del'>-	kfree(file);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void ib_uverbs_release_ucq(struct ib_uverbs_file *file,</div><div class='del'>-			  struct ib_uverbs_completion_event_file *ev_file,</div><div class='del'>-			  struct ib_ucq_object *uobj)</div><div class='add'>+void ib_uverbs_release_ucq(struct ib_uverbs_completion_event_file *ev_file,</div><div class='add'>+			   struct ib_ucq_object *uobj)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_uverbs_event *evt, *tmp;</div><div class='ctx'> </div><div class='hunk'>@@ -147,25 +157,24 @@ void ib_uverbs_release_ucq(struct ib_uverbs_file *file,</div><div class='ctx'> 		uverbs_uobject_put(&amp;ev_file-&gt;uobj);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	spin_lock_irq(&amp;file-&gt;async_file-&gt;ev_queue.lock);</div><div class='del'>-	list_for_each_entry_safe(evt, tmp, &amp;uobj-&gt;async_list, obj_list) {</div><div class='del'>-		list_del(&amp;evt-&gt;list);</div><div class='del'>-		kfree(evt);</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irq(&amp;file-&gt;async_file-&gt;ev_queue.lock);</div><div class='add'>+	ib_uverbs_release_uevent(&amp;uobj-&gt;uevent);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void ib_uverbs_release_uevent(struct ib_uverbs_file *file,</div><div class='del'>-			      struct ib_uevent_object *uobj)</div><div class='add'>+void ib_uverbs_release_uevent(struct ib_uevent_object *uobj)</div><div class='ctx'> {</div><div class='add'>+	struct ib_uverbs_async_event_file *async_file = uobj-&gt;event_file;</div><div class='ctx'> 	struct ib_uverbs_event *evt, *tmp;</div><div class='ctx'> </div><div class='del'>-	spin_lock_irq(&amp;file-&gt;async_file-&gt;ev_queue.lock);</div><div class='add'>+	if (!async_file)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	spin_lock_irq(&amp;async_file-&gt;ev_queue.lock);</div><div class='ctx'> 	list_for_each_entry_safe(evt, tmp, &amp;uobj-&gt;event_list, obj_list) {</div><div class='ctx'> 		list_del(&amp;evt-&gt;list);</div><div class='ctx'> 		kfree(evt);</div><div class='ctx'> 	}</div><div class='del'>-	spin_unlock_irq(&amp;file-&gt;async_file-&gt;ev_queue.lock);</div><div class='add'>+	spin_unlock_irq(&amp;async_file-&gt;ev_queue.lock);</div><div class='add'>+	uverbs_uobject_put(&amp;async_file-&gt;uobj);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void ib_uverbs_detach_umcast(struct ib_qp *qp,</div><div class='hunk'>@@ -198,24 +207,25 @@ void ib_uverbs_release_file(struct kref *ref)</div><div class='ctx'> 	ib_dev = srcu_dereference(file-&gt;device-&gt;ib_dev,</div><div class='ctx'> 				  &amp;file-&gt;device-&gt;disassociate_srcu);</div><div class='ctx'> 	if (ib_dev &amp;&amp; !ib_dev-&gt;ops.disassociate_ucontext)</div><div class='del'>-		module_put(ib_dev-&gt;owner);</div><div class='add'>+		module_put(ib_dev-&gt;ops.owner);</div><div class='ctx'> 	srcu_read_unlock(&amp;file-&gt;device-&gt;disassociate_srcu, srcu_key);</div><div class='ctx'> </div><div class='del'>-	if (atomic_dec_and_test(&amp;file-&gt;device-&gt;refcount))</div><div class='add'>+	if (refcount_dec_and_test(&amp;file-&gt;device-&gt;refcount))</div><div class='ctx'> 		ib_uverbs_comp_dev(file-&gt;device);</div><div class='ctx'> </div><div class='del'>-	if (file-&gt;async_file)</div><div class='del'>-		kref_put(&amp;file-&gt;async_file-&gt;ref,</div><div class='del'>-			 ib_uverbs_release_async_event_file);</div><div class='add'>+	if (file-&gt;default_async_file)</div><div class='add'>+		uverbs_uobject_put(&amp;file-&gt;default_async_file-&gt;uobj);</div><div class='ctx'> 	put_device(&amp;file-&gt;device-&gt;dev);</div><div class='ctx'> </div><div class='ctx'> 	if (file-&gt;disassociate_page)</div><div class='ctx'> 		__free_pages(file-&gt;disassociate_page, 0);</div><div class='add'>+	mutex_destroy(&amp;file-&gt;disassociation_lock);</div><div class='add'>+	mutex_destroy(&amp;file-&gt;umap_lock);</div><div class='add'>+	mutex_destroy(&amp;file-&gt;ucontext_lock);</div><div class='ctx'> 	kfree(file);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t ib_uverbs_event_read(struct ib_uverbs_event_queue *ev_queue,</div><div class='del'>-				    struct ib_uverbs_file *uverbs_file,</div><div class='ctx'> 				    struct file *filp, char __user *buf,</div><div class='ctx'> 				    size_t count, loff_t *pos,</div><div class='ctx'> 				    size_t eventsz)</div><div class='hunk'>@@ -226,25 +236,20 @@ static ssize_t ib_uverbs_event_read(struct ib_uverbs_event_queue *ev_queue,</div><div class='ctx'> 	spin_lock_irq(&amp;ev_queue-&gt;lock);</div><div class='ctx'> </div><div class='ctx'> 	while (list_empty(&amp;ev_queue-&gt;event_list)) {</div><div class='del'>-		spin_unlock_irq(&amp;ev_queue-&gt;lock);</div><div class='add'>+		if (ev_queue-&gt;is_closed) {</div><div class='add'>+			spin_unlock_irq(&amp;ev_queue-&gt;lock);</div><div class='add'>+			return -EIO;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='add'>+		spin_unlock_irq(&amp;ev_queue-&gt;lock);</div><div class='ctx'> 		if (filp-&gt;f_flags &amp; O_NONBLOCK)</div><div class='ctx'> 			return -EAGAIN;</div><div class='ctx'> </div><div class='ctx'> 		if (wait_event_interruptible(ev_queue-&gt;poll_wait,</div><div class='ctx'> 					     (!list_empty(&amp;ev_queue-&gt;event_list) ||</div><div class='del'>-			/* The barriers built into wait_event_interruptible()</div><div class='del'>-			 * and wake_up() guarentee this will see the null set</div><div class='del'>-			 * without using RCU</div><div class='del'>-			 */</div><div class='del'>-					     !uverbs_file-&gt;device-&gt;ib_dev)))</div><div class='add'>+					      ev_queue-&gt;is_closed)))</div><div class='ctx'> 			return -ERESTARTSYS;</div><div class='ctx'> </div><div class='del'>-		/* If device was disassociated and no event exists set an error */</div><div class='del'>-		if (list_empty(&amp;ev_queue-&gt;event_list) &amp;&amp;</div><div class='del'>-		    !uverbs_file-&gt;device-&gt;ib_dev)</div><div class='del'>-			return -EIO;</div><div class='del'>-</div><div class='ctx'> 		spin_lock_irq(&amp;ev_queue-&gt;lock);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -280,8 +285,7 @@ static ssize_t ib_uverbs_async_event_read(struct file *filp, char __user *buf,</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_uverbs_async_event_file *file = filp-&gt;private_data;</div><div class='ctx'> </div><div class='del'>-	return ib_uverbs_event_read(&amp;file-&gt;ev_queue, file-&gt;uverbs_file, filp,</div><div class='del'>-				    buf, count, pos,</div><div class='add'>+	return ib_uverbs_event_read(&amp;file-&gt;ev_queue, filp, buf, count, pos,</div><div class='ctx'> 				    sizeof(struct ib_uverbs_async_event_desc));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -291,9 +295,8 @@ static ssize_t ib_uverbs_comp_event_read(struct file *filp, char __user *buf,</div><div class='ctx'> 	struct ib_uverbs_completion_event_file *comp_ev_file =</div><div class='ctx'> 		filp-&gt;private_data;</div><div class='ctx'> </div><div class='del'>-	return ib_uverbs_event_read(&amp;comp_ev_file-&gt;ev_queue,</div><div class='del'>-				    comp_ev_file-&gt;uobj.ufile, filp,</div><div class='del'>-				    buf, count, pos,</div><div class='add'>+	return ib_uverbs_event_read(&amp;comp_ev_file-&gt;ev_queue, filp, buf, count,</div><div class='add'>+				    pos,</div><div class='ctx'> 				    sizeof(struct ib_uverbs_comp_event_desc));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -308,6 +311,8 @@ static __poll_t ib_uverbs_event_poll(struct ib_uverbs_event_queue *ev_queue,</div><div class='ctx'> 	spin_lock_irq(&amp;ev_queue-&gt;lock);</div><div class='ctx'> 	if (!list_empty(&amp;ev_queue-&gt;event_list))</div><div class='ctx'> 		pollflags = EPOLLIN | EPOLLRDNORM;</div><div class='add'>+	else if (ev_queue-&gt;is_closed)</div><div class='add'>+		pollflags = EPOLLERR;</div><div class='ctx'> 	spin_unlock_irq(&amp;ev_queue-&gt;lock);</div><div class='ctx'> </div><div class='ctx'> 	return pollflags;</div><div class='hunk'>@@ -316,7 +321,9 @@ static __poll_t ib_uverbs_event_poll(struct ib_uverbs_event_queue *ev_queue,</div><div class='ctx'> static __poll_t ib_uverbs_async_event_poll(struct file *filp,</div><div class='ctx'> 					       struct poll_table_struct *wait)</div><div class='ctx'> {</div><div class='del'>-	return ib_uverbs_event_poll(filp-&gt;private_data, filp, wait);</div><div class='add'>+	struct ib_uverbs_async_event_file *file = filp-&gt;private_data;</div><div class='add'>+</div><div class='add'>+	return ib_uverbs_event_poll(&amp;file-&gt;ev_queue, filp, wait);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static __poll_t ib_uverbs_comp_event_poll(struct file *filp,</div><div class='hunk'>@@ -330,9 +337,9 @@ static __poll_t ib_uverbs_comp_event_poll(struct file *filp,</div><div class='ctx'> </div><div class='ctx'> static int ib_uverbs_async_event_fasync(int fd, struct file *filp, int on)</div><div class='ctx'> {</div><div class='del'>-	struct ib_uverbs_event_queue *ev_queue = filp-&gt;private_data;</div><div class='add'>+	struct ib_uverbs_async_event_file *file = filp-&gt;private_data;</div><div class='ctx'> </div><div class='del'>-	return fasync_helper(fd, filp, on, &amp;ev_queue-&gt;async_queue);</div><div class='add'>+	return fasync_helper(fd, filp, on, &amp;file-&gt;ev_queue.async_queue);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int ib_uverbs_comp_event_fasync(int fd, struct file *filp, int on)</div><div class='hunk'>@@ -343,72 +350,20 @@ static int ib_uverbs_comp_event_fasync(int fd, struct file *filp, int on)</div><div class='ctx'> 	return fasync_helper(fd, filp, on, &amp;comp_ev_file-&gt;ev_queue.async_queue);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int ib_uverbs_async_event_close(struct inode *inode, struct file *filp)</div><div class='del'>-{</div><div class='del'>-	struct ib_uverbs_async_event_file *file = filp-&gt;private_data;</div><div class='del'>-	struct ib_uverbs_file *uverbs_file = file-&gt;uverbs_file;</div><div class='del'>-	struct ib_uverbs_event *entry, *tmp;</div><div class='del'>-	int closed_already = 0;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;uverbs_file-&gt;device-&gt;lists_mutex);</div><div class='del'>-	spin_lock_irq(&amp;file-&gt;ev_queue.lock);</div><div class='del'>-	closed_already = file-&gt;ev_queue.is_closed;</div><div class='del'>-	file-&gt;ev_queue.is_closed = 1;</div><div class='del'>-	list_for_each_entry_safe(entry, tmp, &amp;file-&gt;ev_queue.event_list, list) {</div><div class='del'>-		if (entry-&gt;counter)</div><div class='del'>-			list_del(&amp;entry-&gt;obj_list);</div><div class='del'>-		kfree(entry);</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irq(&amp;file-&gt;ev_queue.lock);</div><div class='del'>-	if (!closed_already) {</div><div class='del'>-		list_del(&amp;file-&gt;list);</div><div class='del'>-		ib_unregister_event_handler(&amp;uverbs_file-&gt;event_handler);</div><div class='del'>-	}</div><div class='del'>-	mutex_unlock(&amp;uverbs_file-&gt;device-&gt;lists_mutex);</div><div class='del'>-</div><div class='del'>-	kref_put(&amp;uverbs_file-&gt;ref, ib_uverbs_release_file);</div><div class='del'>-	kref_put(&amp;file-&gt;ref, ib_uverbs_release_async_event_file);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int ib_uverbs_comp_event_close(struct inode *inode, struct file *filp)</div><div class='del'>-{</div><div class='del'>-	struct ib_uobject *uobj = filp-&gt;private_data;</div><div class='del'>-	struct ib_uverbs_completion_event_file *file = container_of(</div><div class='del'>-		uobj, struct ib_uverbs_completion_event_file, uobj);</div><div class='del'>-	struct ib_uverbs_event *entry, *tmp;</div><div class='del'>-</div><div class='del'>-	spin_lock_irq(&amp;file-&gt;ev_queue.lock);</div><div class='del'>-	list_for_each_entry_safe(entry, tmp, &amp;file-&gt;ev_queue.event_list, list) {</div><div class='del'>-		if (entry-&gt;counter)</div><div class='del'>-			list_del(&amp;entry-&gt;obj_list);</div><div class='del'>-		kfree(entry);</div><div class='del'>-	}</div><div class='del'>-	file-&gt;ev_queue.is_closed = 1;</div><div class='del'>-	spin_unlock_irq(&amp;file-&gt;ev_queue.lock);</div><div class='del'>-</div><div class='del'>-	uverbs_close_fd(filp);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> const struct file_operations uverbs_event_fops = {</div><div class='ctx'> 	.owner	 = THIS_MODULE,</div><div class='ctx'> 	.read	 = ib_uverbs_comp_event_read,</div><div class='ctx'> 	.poll    = ib_uverbs_comp_event_poll,</div><div class='del'>-	.release = ib_uverbs_comp_event_close,</div><div class='add'>+	.release = uverbs_uobject_fd_release,</div><div class='ctx'> 	.fasync  = ib_uverbs_comp_event_fasync,</div><div class='del'>-	.llseek	 = no_llseek,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static const struct file_operations uverbs_async_event_fops = {</div><div class='add'>+const struct file_operations uverbs_async_event_fops = {</div><div class='ctx'> 	.owner	 = THIS_MODULE,</div><div class='ctx'> 	.read	 = ib_uverbs_async_event_read,</div><div class='ctx'> 	.poll    = ib_uverbs_async_event_poll,</div><div class='del'>-	.release = ib_uverbs_async_event_close,</div><div class='add'>+	.release = uverbs_async_event_release,</div><div class='ctx'> 	.fasync  = ib_uverbs_async_event_fasync,</div><div class='del'>-	.llseek	 = no_llseek,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context)</div><div class='hunk'>@@ -433,9 +388,9 @@ void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context)</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	uobj = container_of(cq-&gt;uobject, struct ib_ucq_object, uobject);</div><div class='add'>+	uobj = cq-&gt;uobject;</div><div class='ctx'> </div><div class='del'>-	entry-&gt;desc.comp.cq_handle = cq-&gt;uobject-&gt;user_handle;</div><div class='add'>+	entry-&gt;desc.comp.cq_handle = cq-&gt;uobject-&gt;uevent.uobject.user_handle;</div><div class='ctx'> 	entry-&gt;counter		   = &amp;uobj-&gt;comp_events_reported;</div><div class='ctx'> </div><div class='ctx'> 	list_add_tail(&amp;entry-&gt;list, &amp;ev_queue-&gt;event_list);</div><div class='hunk'>@@ -446,102 +401,81 @@ void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context)</div><div class='ctx'> 	kill_fasync(&amp;ev_queue-&gt;async_queue, SIGIO, POLL_IN);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void ib_uverbs_async_handler(struct ib_uverbs_file *file,</div><div class='del'>-				    __u64 element, __u64 event,</div><div class='del'>-				    struct list_head *obj_list,</div><div class='del'>-				    u32 *counter)</div><div class='add'>+void ib_uverbs_async_handler(struct ib_uverbs_async_event_file *async_file,</div><div class='add'>+			     __u64 element, __u64 event,</div><div class='add'>+			     struct list_head *obj_list, u32 *counter)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_uverbs_event *entry;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> </div><div class='del'>-	spin_lock_irqsave(&amp;file-&gt;async_file-&gt;ev_queue.lock, flags);</div><div class='del'>-	if (file-&gt;async_file-&gt;ev_queue.is_closed) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;file-&gt;async_file-&gt;ev_queue.lock, flags);</div><div class='add'>+	if (!async_file)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;async_file-&gt;ev_queue.lock, flags);</div><div class='add'>+	if (async_file-&gt;ev_queue.is_closed) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;async_file-&gt;ev_queue.lock, flags);</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	entry = kmalloc(sizeof(*entry), GFP_ATOMIC);</div><div class='ctx'> 	if (!entry) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;file-&gt;async_file-&gt;ev_queue.lock, flags);</div><div class='add'>+		spin_unlock_irqrestore(&amp;async_file-&gt;ev_queue.lock, flags);</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	entry-&gt;desc.async.element    = element;</div><div class='add'>+	entry-&gt;desc.async.element = element;</div><div class='ctx'> 	entry-&gt;desc.async.event_type = event;</div><div class='del'>-	entry-&gt;desc.async.reserved   = 0;</div><div class='del'>-	entry-&gt;counter               = counter;</div><div class='add'>+	entry-&gt;desc.async.reserved = 0;</div><div class='add'>+	entry-&gt;counter = counter;</div><div class='ctx'> </div><div class='del'>-	list_add_tail(&amp;entry-&gt;list, &amp;file-&gt;async_file-&gt;ev_queue.event_list);</div><div class='add'>+	list_add_tail(&amp;entry-&gt;list, &amp;async_file-&gt;ev_queue.event_list);</div><div class='ctx'> 	if (obj_list)</div><div class='ctx'> 		list_add_tail(&amp;entry-&gt;obj_list, obj_list);</div><div class='del'>-	spin_unlock_irqrestore(&amp;file-&gt;async_file-&gt;ev_queue.lock, flags);</div><div class='add'>+	spin_unlock_irqrestore(&amp;async_file-&gt;ev_queue.lock, flags);</div><div class='ctx'> </div><div class='del'>-	wake_up_interruptible(&amp;file-&gt;async_file-&gt;ev_queue.poll_wait);</div><div class='del'>-	kill_fasync(&amp;file-&gt;async_file-&gt;ev_queue.async_queue, SIGIO, POLL_IN);</div><div class='add'>+	wake_up_interruptible(&amp;async_file-&gt;ev_queue.poll_wait);</div><div class='add'>+	kill_fasync(&amp;async_file-&gt;ev_queue.async_queue, SIGIO, POLL_IN);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr)</div><div class='add'>+static void uverbs_uobj_event(struct ib_uevent_object *eobj,</div><div class='add'>+			      struct ib_event *event)</div><div class='ctx'> {</div><div class='del'>-	struct ib_ucq_object *uobj = container_of(event-&gt;element.cq-&gt;uobject,</div><div class='del'>-						  struct ib_ucq_object, uobject);</div><div class='add'>+	ib_uverbs_async_handler(eobj-&gt;event_file,</div><div class='add'>+				eobj-&gt;uobject.user_handle, event-&gt;event,</div><div class='add'>+				&amp;eobj-&gt;event_list, &amp;eobj-&gt;events_reported);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	ib_uverbs_async_handler(uobj-&gt;uobject.ufile, uobj-&gt;uobject.user_handle,</div><div class='del'>-				event-&gt;event, &amp;uobj-&gt;async_list,</div><div class='del'>-				&amp;uobj-&gt;async_events_reported);</div><div class='add'>+void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr)</div><div class='add'>+{</div><div class='add'>+	uverbs_uobj_event(&amp;event-&gt;element.cq-&gt;uobject-&gt;uevent, event);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr)</div><div class='ctx'> {</div><div class='del'>-	struct ib_uevent_object *uobj;</div><div class='del'>-</div><div class='ctx'> 	/* for XRC target qp's, check that qp is live */</div><div class='ctx'> 	if (!event-&gt;element.qp-&gt;uobject)</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='del'>-	uobj = container_of(event-&gt;element.qp-&gt;uobject,</div><div class='del'>-			    struct ib_uevent_object, uobject);</div><div class='del'>-</div><div class='del'>-	ib_uverbs_async_handler(context_ptr, uobj-&gt;uobject.user_handle,</div><div class='del'>-				event-&gt;event, &amp;uobj-&gt;event_list,</div><div class='del'>-				&amp;uobj-&gt;events_reported);</div><div class='add'>+	uverbs_uobj_event(&amp;event-&gt;element.qp-&gt;uobject-&gt;uevent, event);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr)</div><div class='ctx'> {</div><div class='del'>-	struct ib_uevent_object *uobj = container_of(event-&gt;element.wq-&gt;uobject,</div><div class='del'>-						  struct ib_uevent_object, uobject);</div><div class='del'>-</div><div class='del'>-	ib_uverbs_async_handler(context_ptr, uobj-&gt;uobject.user_handle,</div><div class='del'>-				event-&gt;event, &amp;uobj-&gt;event_list,</div><div class='del'>-				&amp;uobj-&gt;events_reported);</div><div class='add'>+	uverbs_uobj_event(&amp;event-&gt;element.wq-&gt;uobject-&gt;uevent, event);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr)</div><div class='ctx'> {</div><div class='del'>-	struct ib_uevent_object *uobj;</div><div class='del'>-</div><div class='del'>-	uobj = container_of(event-&gt;element.srq-&gt;uobject,</div><div class='del'>-			    struct ib_uevent_object, uobject);</div><div class='del'>-</div><div class='del'>-	ib_uverbs_async_handler(context_ptr, uobj-&gt;uobject.user_handle,</div><div class='del'>-				event-&gt;event, &amp;uobj-&gt;event_list,</div><div class='del'>-				&amp;uobj-&gt;events_reported);</div><div class='add'>+	uverbs_uobj_event(&amp;event-&gt;element.srq-&gt;uobject-&gt;uevent, event);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void ib_uverbs_event_handler(struct ib_event_handler *handler,</div><div class='del'>-			     struct ib_event *event)</div><div class='add'>+static void ib_uverbs_event_handler(struct ib_event_handler *handler,</div><div class='add'>+				    struct ib_event *event)</div><div class='ctx'> {</div><div class='del'>-	struct ib_uverbs_file *file =</div><div class='del'>-		container_of(handler, struct ib_uverbs_file, event_handler);</div><div class='del'>-</div><div class='del'>-	ib_uverbs_async_handler(file, event-&gt;element.port_num, event-&gt;event,</div><div class='del'>-				NULL, NULL);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void ib_uverbs_free_async_event_file(struct ib_uverbs_file *file)</div><div class='del'>-{</div><div class='del'>-	kref_put(&amp;file-&gt;async_file-&gt;ref, ib_uverbs_release_async_event_file);</div><div class='del'>-	file-&gt;async_file = NULL;</div><div class='add'>+	ib_uverbs_async_handler(</div><div class='add'>+		container_of(handler, struct ib_uverbs_async_event_file,</div><div class='add'>+			     event_handler),</div><div class='add'>+		event-&gt;element.port_num, event-&gt;event, NULL, NULL);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue)</div><div class='hunk'>@@ -553,45 +487,26 @@ void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue)</div><div class='ctx'> 	ev_queue-&gt;async_queue = NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct file *ib_uverbs_alloc_async_event_file(struct ib_uverbs_file *uverbs_file,</div><div class='del'>-					      struct ib_device	*ib_dev)</div><div class='add'>+void ib_uverbs_init_async_event_file(</div><div class='add'>+	struct ib_uverbs_async_event_file *async_file)</div><div class='ctx'> {</div><div class='del'>-	struct ib_uverbs_async_event_file *ev_file;</div><div class='del'>-	struct file *filp;</div><div class='del'>-</div><div class='del'>-	ev_file = kzalloc(sizeof(*ev_file), GFP_KERNEL);</div><div class='del'>-	if (!ev_file)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	ib_uverbs_init_event_queue(&amp;ev_file-&gt;ev_queue);</div><div class='del'>-	ev_file-&gt;uverbs_file = uverbs_file;</div><div class='del'>-	kref_get(&amp;ev_file-&gt;uverbs_file-&gt;ref);</div><div class='del'>-	kref_init(&amp;ev_file-&gt;ref);</div><div class='del'>-	filp = anon_inode_getfile("[infinibandevent]", &amp;uverbs_async_event_fops,</div><div class='del'>-				  ev_file, O_RDONLY);</div><div class='del'>-	if (IS_ERR(filp))</div><div class='del'>-		goto err_put_refs;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;uverbs_file-&gt;device-&gt;lists_mutex);</div><div class='del'>-	list_add_tail(&amp;ev_file-&gt;list,</div><div class='del'>-		      &amp;uverbs_file-&gt;device-&gt;uverbs_events_file_list);</div><div class='del'>-	mutex_unlock(&amp;uverbs_file-&gt;device-&gt;lists_mutex);</div><div class='del'>-</div><div class='del'>-	WARN_ON(uverbs_file-&gt;async_file);</div><div class='del'>-	uverbs_file-&gt;async_file = ev_file;</div><div class='del'>-	kref_get(&amp;uverbs_file-&gt;async_file-&gt;ref);</div><div class='del'>-	INIT_IB_EVENT_HANDLER(&amp;uverbs_file-&gt;event_handler,</div><div class='del'>-			      ib_dev,</div><div class='del'>-			      ib_uverbs_event_handler);</div><div class='del'>-	ib_register_event_handler(&amp;uverbs_file-&gt;event_handler);</div><div class='del'>-	/* At that point async file stuff was fully set */</div><div class='add'>+	struct ib_uverbs_file *uverbs_file = async_file-&gt;uobj.ufile;</div><div class='add'>+	struct ib_device *ib_dev = async_file-&gt;uobj.context-&gt;device;</div><div class='ctx'> </div><div class='del'>-	return filp;</div><div class='add'>+	ib_uverbs_init_event_queue(&amp;async_file-&gt;ev_queue);</div><div class='ctx'> </div><div class='del'>-err_put_refs:</div><div class='del'>-	kref_put(&amp;ev_file-&gt;uverbs_file-&gt;ref, ib_uverbs_release_file);</div><div class='del'>-	kref_put(&amp;ev_file-&gt;ref, ib_uverbs_release_async_event_file);</div><div class='del'>-	return filp;</div><div class='add'>+	/* The first async_event_file becomes the default one for the file. */</div><div class='add'>+	mutex_lock(&amp;uverbs_file-&gt;ucontext_lock);</div><div class='add'>+	if (!uverbs_file-&gt;default_async_file) {</div><div class='add'>+		/* Pairs with the put in ib_uverbs_release_file */</div><div class='add'>+		uverbs_uobject_get(&amp;async_file-&gt;uobj);</div><div class='add'>+		smp_store_release(&amp;uverbs_file-&gt;default_async_file, async_file);</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;uverbs_file-&gt;ucontext_lock);</div><div class='add'>+</div><div class='add'>+	INIT_IB_EVENT_HANDLER(&amp;async_file-&gt;event_handler, ib_dev,</div><div class='add'>+			      ib_uverbs_event_handler);</div><div class='add'>+	ib_register_event_handler(&amp;async_file-&gt;event_handler);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t verify_hdr(struct ib_uverbs_cmd_hdr *hdr,</div><div class='hunk'>@@ -632,7 +547,7 @@ static ssize_t verify_hdr(struct ib_uverbs_cmd_hdr *hdr,</div><div class='ctx'> 	if (hdr-&gt;in_words * 4 != count)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (count &lt; method_elm-&gt;req_size + sizeof(hdr)) {</div><div class='add'>+	if (count &lt; method_elm-&gt;req_size + sizeof(*hdr)) {</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * rdma-core v18 and v19 have a bug where they send DESTROY_CQ</div><div class='ctx'> 		 * with a 16 byte write instead of 24. Old kernels didn't</div><div class='hunk'>@@ -699,6 +614,7 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,</div><div class='ctx'> 	memset(bundle.attr_present, 0, sizeof(bundle.attr_present));</div><div class='ctx'> 	bundle.ufile = file;</div><div class='ctx'> 	bundle.context = NULL; /* only valid if bundle has uobject */</div><div class='add'>+	bundle.uobject = NULL;</div><div class='ctx'> 	if (!method_elm-&gt;is_ex) {</div><div class='ctx'> 		size_t in_len = hdr.in_words * 4 - sizeof(hdr);</div><div class='ctx'> 		size_t out_len = hdr.out_words * 4;</div><div class='hunk'>@@ -762,11 +678,16 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ret = method_elm-&gt;handler(&amp;bundle);</div><div class='add'>+	if (bundle.uobject)</div><div class='add'>+		uverbs_finalize_object(bundle.uobject, UVERBS_ACCESS_NEW, true,</div><div class='add'>+				       !ret, &amp;bundle);</div><div class='ctx'> out_unlock:</div><div class='ctx'> 	srcu_read_unlock(&amp;file-&gt;device-&gt;disassociate_srcu, srcu_key);</div><div class='ctx'> 	return (ret) ? : count;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static const struct vm_operations_struct rdma_umap_ops;</div><div class='add'>+</div><div class='ctx'> static int ib_uverbs_mmap(struct file *filp, struct vm_area_struct *vma)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_uverbs_file *file = filp-&gt;private_data;</div><div class='hunk'>@@ -781,45 +702,18 @@ static int ib_uverbs_mmap(struct file *filp, struct vm_area_struct *vma)</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;file-&gt;disassociation_lock);</div><div class='add'>+</div><div class='add'>+	vma-&gt;vm_ops = &amp;rdma_umap_ops;</div><div class='ctx'> 	ret = ucontext-&gt;device-&gt;ops.mmap(ucontext, vma);</div><div class='add'>+</div><div class='add'>+	mutex_unlock(&amp;file-&gt;disassociation_lock);</div><div class='ctx'> out:</div><div class='ctx'> 	srcu_read_unlock(&amp;file-&gt;device-&gt;disassociate_srcu, srcu_key);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='del'>- * Each time we map IO memory into user space this keeps track of the mapping.</div><div class='del'>- * When the device is hot-unplugged we 'zap' the mmaps in user space to point</div><div class='del'>- * to the zero page and allow the hot unplug to proceed.</div><div class='del'>- *</div><div class='del'>- * This is necessary for cases like PCI physical hot unplug as the actual BAR</div><div class='del'>- * memory may vanish after this and access to it from userspace could MCE.</div><div class='del'>- *</div><div class='del'>- * RDMA drivers supporting disassociation must have their user space designed</div><div class='del'>- * to cope in some way with their IO pages going to the zero page.</div><div class='del'>- */</div><div class='del'>-struct rdma_umap_priv {</div><div class='del'>-	struct vm_area_struct *vma;</div><div class='del'>-	struct list_head list;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static const struct vm_operations_struct rdma_umap_ops;</div><div class='del'>-</div><div class='del'>-static void rdma_umap_priv_init(struct rdma_umap_priv *priv,</div><div class='del'>-				struct vm_area_struct *vma)</div><div class='del'>-{</div><div class='del'>-	struct ib_uverbs_file *ufile = vma-&gt;vm_file-&gt;private_data;</div><div class='del'>-</div><div class='del'>-	priv-&gt;vma = vma;</div><div class='del'>-	vma-&gt;vm_private_data = priv;</div><div class='del'>-	vma-&gt;vm_ops = &amp;rdma_umap_ops;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;ufile-&gt;umap_lock);</div><div class='del'>-	list_add(&amp;priv-&gt;list, &amp;ufile-&gt;umaps);</div><div class='del'>-	mutex_unlock(&amp;ufile-&gt;umap_lock);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='ctx'>  * The VMA has been dup'd, initialize the vm_private_data with a new tracking</div><div class='ctx'>  * struct</div><div class='ctx'>  */</div><div class='hunk'>@@ -835,6 +729,8 @@ static void rdma_umap_open(struct vm_area_struct *vma)</div><div class='ctx'> 	/* We are racing with disassociation */</div><div class='ctx'> 	if (!down_read_trylock(&amp;ufile-&gt;hw_destroy_rwsem))</div><div class='ctx'> 		goto out_zap;</div><div class='add'>+	mutex_lock(&amp;ufile-&gt;disassociation_lock);</div><div class='add'>+</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Disassociation already completed, the VMA should already be zapped.</div><div class='ctx'> 	 */</div><div class='hunk'>@@ -844,12 +740,14 @@ static void rdma_umap_open(struct vm_area_struct *vma)</div><div class='ctx'> 	priv = kzalloc(sizeof(*priv), GFP_KERNEL);</div><div class='ctx'> 	if (!priv)</div><div class='ctx'> 		goto out_unlock;</div><div class='del'>-	rdma_umap_priv_init(priv, vma);</div><div class='add'>+	rdma_umap_priv_init(priv, vma, opriv-&gt;entry);</div><div class='ctx'> </div><div class='add'>+	mutex_unlock(&amp;ufile-&gt;disassociation_lock);</div><div class='ctx'> 	up_read(&amp;ufile-&gt;hw_destroy_rwsem);</div><div class='ctx'> 	return;</div><div class='ctx'> </div><div class='ctx'> out_unlock:</div><div class='add'>+	mutex_unlock(&amp;ufile-&gt;disassociation_lock);</div><div class='ctx'> 	up_read(&amp;ufile-&gt;hw_destroy_rwsem);</div><div class='ctx'> out_zap:</div><div class='ctx'> 	/*</div><div class='hunk'>@@ -875,6 +773,9 @@ static void rdma_umap_close(struct vm_area_struct *vma)</div><div class='ctx'> 	 * this point.</div><div class='ctx'> 	 */</div><div class='ctx'> 	mutex_lock(&amp;ufile-&gt;umap_lock);</div><div class='add'>+	if (priv-&gt;entry)</div><div class='add'>+		rdma_user_mmap_entry_put(priv-&gt;entry);</div><div class='add'>+</div><div class='ctx'> 	list_del(&amp;priv-&gt;list);</div><div class='ctx'> 	mutex_unlock(&amp;ufile-&gt;umap_lock);</div><div class='ctx'> 	kfree(priv);</div><div class='hunk'>@@ -926,49 +827,11 @@ static const struct vm_operations_struct rdma_umap_ops = {</div><div class='ctx'> 	.fault = rdma_umap_fault,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/*</div><div class='del'>- * Map IO memory into a process. This is to be called by drivers as part of</div><div class='del'>- * their mmap() functions if they wish to send something like PCI-E BAR memory</div><div class='del'>- * to userspace.</div><div class='del'>- */</div><div class='del'>-int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma,</div><div class='del'>-		      unsigned long pfn, unsigned long size, pgprot_t prot)</div><div class='del'>-{</div><div class='del'>-	struct ib_uverbs_file *ufile = ucontext-&gt;ufile;</div><div class='del'>-	struct rdma_umap_priv *priv;</div><div class='del'>-</div><div class='del'>-	if (!(vma-&gt;vm_flags &amp; VM_SHARED))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	if (vma-&gt;vm_end - vma-&gt;vm_start != size)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	/* Driver is using this wrong, must be called by ib_uverbs_mmap */</div><div class='del'>-	if (WARN_ON(!vma-&gt;vm_file ||</div><div class='del'>-		    vma-&gt;vm_file-&gt;private_data != ufile))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	lockdep_assert_held(&amp;ufile-&gt;device-&gt;disassociate_srcu);</div><div class='del'>-</div><div class='del'>-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);</div><div class='del'>-	if (!priv)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	vma-&gt;vm_page_prot = prot;</div><div class='del'>-	if (io_remap_pfn_range(vma, vma-&gt;vm_start, pfn, size, prot)) {</div><div class='del'>-		kfree(priv);</div><div class='del'>-		return -EAGAIN;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	rdma_umap_priv_init(priv, vma);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(rdma_user_mmap_io);</div><div class='del'>-</div><div class='ctx'> void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_umap_priv *priv, *next_priv;</div><div class='ctx'> </div><div class='del'>-	lockdep_assert_held(&amp;ufile-&gt;hw_destroy_rwsem);</div><div class='add'>+	mutex_lock(&amp;ufile-&gt;disassociation_lock);</div><div class='ctx'> </div><div class='ctx'> 	while (1) {</div><div class='ctx'> 		struct mm_struct *mm = NULL;</div><div class='hunk'>@@ -984,24 +847,28 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile)</div><div class='ctx'> 			ret = mmget_not_zero(mm);</div><div class='ctx'> 			if (!ret) {</div><div class='ctx'> 				list_del_init(&amp;priv-&gt;list);</div><div class='add'>+				if (priv-&gt;entry) {</div><div class='add'>+					rdma_user_mmap_entry_put(priv-&gt;entry);</div><div class='add'>+					priv-&gt;entry = NULL;</div><div class='add'>+				}</div><div class='ctx'> 				mm = NULL;</div><div class='ctx'> 				continue;</div><div class='ctx'> 			}</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='ctx'> 		mutex_unlock(&amp;ufile-&gt;umap_lock);</div><div class='del'>-		if (!mm)</div><div class='add'>+		if (!mm) {</div><div class='add'>+			mutex_unlock(&amp;ufile-&gt;disassociation_lock);</div><div class='ctx'> 			return;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='ctx'> 		/*</div><div class='del'>-		 * The umap_lock is nested under mmap_sem since it used within</div><div class='add'>+		 * The umap_lock is nested under mmap_lock since it used within</div><div class='ctx'> 		 * the vma_ops callbacks, so we have to clean the list one mm</div><div class='ctx'> 		 * at a time to get the lock ordering right. Typically there</div><div class='ctx'> 		 * will only be one mm, so no big deal.</div><div class='ctx'> 		 */</div><div class='del'>-		down_read(&amp;mm-&gt;mmap_sem);</div><div class='del'>-		if (!mmget_still_valid(mm))</div><div class='del'>-			goto skip_mm;</div><div class='add'>+		mmap_read_lock(mm);</div><div class='ctx'> 		mutex_lock(&amp;ufile-&gt;umap_lock);</div><div class='ctx'> 		list_for_each_entry_safe (priv, next_priv, &amp;ufile-&gt;umaps,</div><div class='ctx'> 					  list) {</div><div class='hunk'>@@ -1013,14 +880,42 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile)</div><div class='ctx'> </div><div class='ctx'> 			zap_vma_ptes(vma, vma-&gt;vm_start,</div><div class='ctx'> 				     vma-&gt;vm_end - vma-&gt;vm_start);</div><div class='add'>+</div><div class='add'>+			if (priv-&gt;entry) {</div><div class='add'>+				rdma_user_mmap_entry_put(priv-&gt;entry);</div><div class='add'>+				priv-&gt;entry = NULL;</div><div class='add'>+			}</div><div class='ctx'> 		}</div><div class='ctx'> 		mutex_unlock(&amp;ufile-&gt;umap_lock);</div><div class='del'>-	skip_mm:</div><div class='del'>-		up_read(&amp;mm-&gt;mmap_sem);</div><div class='add'>+		mmap_read_unlock(mm);</div><div class='ctx'> 		mmput(mm);</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	mutex_unlock(&amp;ufile-&gt;disassociation_lock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+/**</div><div class='add'>+ * rdma_user_mmap_disassociate() - Revoke mmaps for a device</div><div class='add'>+ * @device: device to revoke</div><div class='add'>+ *</div><div class='add'>+ * This function should be called by drivers that need to disable mmaps for the</div><div class='add'>+ * device, for instance because it is going to be reset.</div><div class='add'>+ */</div><div class='add'>+void rdma_user_mmap_disassociate(struct ib_device *device)</div><div class='add'>+{</div><div class='add'>+	struct ib_uverbs_device *uverbs_dev =</div><div class='add'>+		ib_get_client_data(device, &amp;uverbs_client);</div><div class='add'>+	struct ib_uverbs_file *ufile;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;uverbs_dev-&gt;lists_mutex);</div><div class='add'>+	list_for_each_entry(ufile, &amp;uverbs_dev-&gt;uverbs_file_list, list) {</div><div class='add'>+		if (ufile-&gt;ucontext)</div><div class='add'>+			uverbs_user_mmap_disassociate(ufile);</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;uverbs_dev-&gt;lists_mutex);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_user_mmap_disassociate);</div><div class='add'>+</div><div class='ctx'> /*</div><div class='ctx'>  * ib_uverbs_open() does not need the BKL:</div><div class='ctx'>  *</div><div class='hunk'>@@ -1041,7 +936,7 @@ static int ib_uverbs_open(struct inode *inode, struct file *filp)</div><div class='ctx'> 	int srcu_key;</div><div class='ctx'> </div><div class='ctx'> 	dev = container_of(inode-&gt;i_cdev, struct ib_uverbs_device, cdev);</div><div class='del'>-	if (!atomic_inc_not_zero(&amp;dev-&gt;refcount))</div><div class='add'>+	if (!refcount_inc_not_zero(&amp;dev-&gt;refcount))</div><div class='ctx'> 		return -ENXIO;</div><div class='ctx'> </div><div class='ctx'> 	get_device(&amp;dev-&gt;dev);</div><div class='hunk'>@@ -1065,7 +960,7 @@ static int ib_uverbs_open(struct inode *inode, struct file *filp)</div><div class='ctx'> 	module_dependent = !(ib_dev-&gt;ops.disassociate_ucontext);</div><div class='ctx'> </div><div class='ctx'> 	if (module_dependent) {</div><div class='del'>-		if (!try_module_get(ib_dev-&gt;owner)) {</div><div class='add'>+		if (!try_module_get(ib_dev-&gt;ops.owner)) {</div><div class='ctx'> 			ret = -ENODEV;</div><div class='ctx'> 			goto err;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -1090,6 +985,8 @@ static int ib_uverbs_open(struct inode *inode, struct file *filp)</div><div class='ctx'> 	mutex_init(&amp;file-&gt;umap_lock);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;file-&gt;umaps);</div><div class='ctx'> </div><div class='add'>+	mutex_init(&amp;file-&gt;disassociation_lock);</div><div class='add'>+</div><div class='ctx'> 	filp-&gt;private_data = file;</div><div class='ctx'> 	list_add_tail(&amp;file-&gt;list, &amp;dev-&gt;uverbs_file_list);</div><div class='ctx'> 	mutex_unlock(&amp;dev-&gt;lists_mutex);</div><div class='hunk'>@@ -1100,12 +997,12 @@ static int ib_uverbs_open(struct inode *inode, struct file *filp)</div><div class='ctx'> 	return stream_open(inode, filp);</div><div class='ctx'> </div><div class='ctx'> err_module:</div><div class='del'>-	module_put(ib_dev-&gt;owner);</div><div class='add'>+	module_put(ib_dev-&gt;ops.owner);</div><div class='ctx'> </div><div class='ctx'> err:</div><div class='ctx'> 	mutex_unlock(&amp;dev-&gt;lists_mutex);</div><div class='ctx'> 	srcu_read_unlock(&amp;dev-&gt;disassociate_srcu, srcu_key);</div><div class='del'>-	if (atomic_dec_and_test(&amp;dev-&gt;refcount))</div><div class='add'>+	if (refcount_dec_and_test(&amp;dev-&gt;refcount))</div><div class='ctx'> 		ib_uverbs_comp_dev(dev);</div><div class='ctx'> </div><div class='ctx'> 	put_device(&amp;dev-&gt;dev);</div><div class='hunk'>@@ -1132,9 +1029,8 @@ static const struct file_operations uverbs_fops = {</div><div class='ctx'> 	.write	 = ib_uverbs_write,</div><div class='ctx'> 	.open	 = ib_uverbs_open,</div><div class='ctx'> 	.release = ib_uverbs_close,</div><div class='del'>-	.llseek	 = no_llseek,</div><div class='ctx'> 	.unlocked_ioctl = ib_uverbs_ioctl,</div><div class='del'>-	.compat_ioctl = ib_uverbs_ioctl,</div><div class='add'>+	.compat_ioctl = compat_ptr_ioctl,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static const struct file_operations uverbs_mmap_fops = {</div><div class='hunk'>@@ -1143,17 +1039,45 @@ static const struct file_operations uverbs_mmap_fops = {</div><div class='ctx'> 	.mmap    = ib_uverbs_mmap,</div><div class='ctx'> 	.open	 = ib_uverbs_open,</div><div class='ctx'> 	.release = ib_uverbs_close,</div><div class='del'>-	.llseek	 = no_llseek,</div><div class='ctx'> 	.unlocked_ioctl = ib_uverbs_ioctl,</div><div class='del'>-	.compat_ioctl = ib_uverbs_ioctl,</div><div class='add'>+	.compat_ioctl = compat_ptr_ioctl,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static int ib_uverbs_get_nl_info(struct ib_device *ibdev, void *client_data,</div><div class='add'>+				 struct ib_client_nl_info *res)</div><div class='add'>+{</div><div class='add'>+	struct ib_uverbs_device *uverbs_dev = client_data;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (res-&gt;port != -1)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	res-&gt;abi = ibdev-&gt;ops.uverbs_abi_ver;</div><div class='add'>+	res-&gt;cdev = &amp;uverbs_dev-&gt;dev;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * To support DRIVER_ID binding in userspace some of the driver need</div><div class='add'>+	 * upgrading to expose their PCI dependent revision information</div><div class='add'>+	 * through get_context instead of relying on modalias matching. When</div><div class='add'>+	 * the drivers are fixed they can drop this flag.</div><div class='add'>+	 */</div><div class='add'>+	if (!ibdev-&gt;ops.uverbs_no_driver_id_binding) {</div><div class='add'>+		ret = nla_put_u32(res-&gt;nl_msg, RDMA_NLDEV_ATTR_UVERBS_DRIVER_ID,</div><div class='add'>+				  ibdev-&gt;ops.driver_id);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static struct ib_client uverbs_client = {</div><div class='ctx'> 	.name   = "uverbs",</div><div class='ctx'> 	.no_kverbs_req = true,</div><div class='ctx'> 	.add    = ib_uverbs_add_one,</div><div class='del'>-	.remove = ib_uverbs_remove_one</div><div class='add'>+	.remove = ib_uverbs_remove_one,</div><div class='add'>+	.get_nl_info = ib_uverbs_get_nl_info,</div><div class='ctx'> };</div><div class='add'>+MODULE_ALIAS_RDMA_CLIENT("uverbs");</div><div class='ctx'> </div><div class='ctx'> static ssize_t ibdev_show(struct device *device, struct device_attribute *attr,</div><div class='ctx'> 			  char *buf)</div><div class='hunk'>@@ -1167,7 +1091,7 @@ static ssize_t ibdev_show(struct device *device, struct device_attribute *attr,</div><div class='ctx'> 	srcu_key = srcu_read_lock(&amp;dev-&gt;disassociate_srcu);</div><div class='ctx'> 	ib_dev = srcu_dereference(dev-&gt;ib_dev, &amp;dev-&gt;disassociate_srcu);</div><div class='ctx'> 	if (ib_dev)</div><div class='del'>-		ret = sprintf(buf, "%s\n", dev_name(&amp;ib_dev-&gt;dev));</div><div class='add'>+		ret = sysfs_emit(buf, "%s\n", dev_name(&amp;ib_dev-&gt;dev));</div><div class='ctx'> 	srcu_read_unlock(&amp;dev-&gt;disassociate_srcu, srcu_key);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -1186,7 +1110,7 @@ static ssize_t abi_version_show(struct device *device,</div><div class='ctx'> 	srcu_key = srcu_read_lock(&amp;dev-&gt;disassociate_srcu);</div><div class='ctx'> 	ib_dev = srcu_dereference(dev-&gt;ib_dev, &amp;dev-&gt;disassociate_srcu);</div><div class='ctx'> 	if (ib_dev)</div><div class='del'>-		ret = sprintf(buf, "%d\n", ib_dev-&gt;uverbs_abi_ver);</div><div class='add'>+		ret = sysfs_emit(buf, "%u\n", ib_dev-&gt;ops.uverbs_abi_ver);</div><div class='ctx'> 	srcu_read_unlock(&amp;dev-&gt;disassociate_srcu, srcu_key);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -1219,53 +1143,56 @@ static int ib_uverbs_create_uapi(struct ib_device *device,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void ib_uverbs_add_one(struct ib_device *device)</div><div class='add'>+static int ib_uverbs_add_one(struct ib_device *device)</div><div class='ctx'> {</div><div class='ctx'> 	int devnum;</div><div class='ctx'> 	dev_t base;</div><div class='ctx'> 	struct ib_uverbs_device *uverbs_dev;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	if (!device-&gt;ops.alloc_ucontext)</div><div class='del'>-		return;</div><div class='add'>+	if (!device-&gt;ops.alloc_ucontext ||</div><div class='add'>+	    device-&gt;type == RDMA_DEVICE_TYPE_SMI)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	uverbs_dev = kzalloc(sizeof(*uverbs_dev), GFP_KERNEL);</div><div class='ctx'> 	if (!uverbs_dev)</div><div class='del'>-		return;</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	ret = init_srcu_struct(&amp;uverbs_dev-&gt;disassociate_srcu);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		kfree(uverbs_dev);</div><div class='del'>-		return;</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	device_initialize(&amp;uverbs_dev-&gt;dev);</div><div class='del'>-	uverbs_dev-&gt;dev.class = uverbs_class;</div><div class='add'>+	uverbs_dev-&gt;dev.class = &amp;uverbs_class;</div><div class='ctx'> 	uverbs_dev-&gt;dev.parent = device-&gt;dev.parent;</div><div class='ctx'> 	uverbs_dev-&gt;dev.release = ib_uverbs_release_dev;</div><div class='ctx'> 	uverbs_dev-&gt;groups[0] = &amp;dev_attr_group;</div><div class='ctx'> 	uverbs_dev-&gt;dev.groups = uverbs_dev-&gt;groups;</div><div class='del'>-	atomic_set(&amp;uverbs_dev-&gt;refcount, 1);</div><div class='add'>+	refcount_set(&amp;uverbs_dev-&gt;refcount, 1);</div><div class='ctx'> 	init_completion(&amp;uverbs_dev-&gt;comp);</div><div class='ctx'> 	uverbs_dev-&gt;xrcd_tree = RB_ROOT;</div><div class='ctx'> 	mutex_init(&amp;uverbs_dev-&gt;xrcd_tree_mutex);</div><div class='ctx'> 	mutex_init(&amp;uverbs_dev-&gt;lists_mutex);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;uverbs_dev-&gt;uverbs_file_list);</div><div class='del'>-	INIT_LIST_HEAD(&amp;uverbs_dev-&gt;uverbs_events_file_list);</div><div class='ctx'> 	rcu_assign_pointer(uverbs_dev-&gt;ib_dev, device);</div><div class='ctx'> 	uverbs_dev-&gt;num_comp_vectors = device-&gt;num_comp_vectors;</div><div class='ctx'> </div><div class='ctx'> 	devnum = ida_alloc_max(&amp;uverbs_ida, IB_UVERBS_MAX_DEVICES - 1,</div><div class='ctx'> 			       GFP_KERNEL);</div><div class='del'>-	if (devnum &lt; 0)</div><div class='add'>+	if (devnum &lt; 0) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='ctx'> 		goto err;</div><div class='add'>+	}</div><div class='ctx'> 	uverbs_dev-&gt;devnum = devnum;</div><div class='ctx'> 	if (devnum &gt;= IB_UVERBS_NUM_FIXED_MINOR)</div><div class='ctx'> 		base = dynamic_uverbs_dev + devnum - IB_UVERBS_NUM_FIXED_MINOR;</div><div class='ctx'> 	else</div><div class='ctx'> 		base = IB_UVERBS_BASE_DEV + devnum;</div><div class='ctx'> </div><div class='del'>-	if (ib_uverbs_create_uapi(device, uverbs_dev))</div><div class='add'>+	ret = ib_uverbs_create_uapi(device, uverbs_dev);</div><div class='add'>+	if (ret)</div><div class='ctx'> 		goto err_uapi;</div><div class='ctx'> </div><div class='ctx'> 	uverbs_dev-&gt;dev.devt = base;</div><div class='hunk'>@@ -1280,30 +1207,25 @@ static void ib_uverbs_add_one(struct ib_device *device)</div><div class='ctx'> 		goto err_uapi;</div><div class='ctx'> </div><div class='ctx'> 	ib_set_client_data(device, &amp;uverbs_client, uverbs_dev);</div><div class='del'>-	return;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='ctx'> err_uapi:</div><div class='ctx'> 	ida_free(&amp;uverbs_ida, devnum);</div><div class='ctx'> err:</div><div class='del'>-	if (atomic_dec_and_test(&amp;uverbs_dev-&gt;refcount))</div><div class='add'>+	if (refcount_dec_and_test(&amp;uverbs_dev-&gt;refcount))</div><div class='ctx'> 		ib_uverbs_comp_dev(uverbs_dev);</div><div class='ctx'> 	wait_for_completion(&amp;uverbs_dev-&gt;comp);</div><div class='ctx'> 	put_device(&amp;uverbs_dev-&gt;dev);</div><div class='del'>-	return;</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev,</div><div class='ctx'> 					struct ib_device *ib_dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_uverbs_file *file;</div><div class='del'>-	struct ib_uverbs_async_event_file *event_file;</div><div class='del'>-	struct ib_event event;</div><div class='ctx'> </div><div class='ctx'> 	/* Pending running commands to terminate */</div><div class='ctx'> 	uverbs_disassociate_api_pre(uverbs_dev);</div><div class='del'>-	event.event = IB_EVENT_DEVICE_FATAL;</div><div class='del'>-	event.element.port_num = 0;</div><div class='del'>-	event.device = ib_dev;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;uverbs_dev-&gt;lists_mutex);</div><div class='ctx'> 	while (!list_empty(&amp;uverbs_dev-&gt;uverbs_file_list)) {</div><div class='hunk'>@@ -1319,31 +1241,11 @@ static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev,</div><div class='ctx'> 		 */</div><div class='ctx'> 		mutex_unlock(&amp;uverbs_dev-&gt;lists_mutex);</div><div class='ctx'> </div><div class='del'>-		ib_uverbs_event_handler(&amp;file-&gt;event_handler, &amp;event);</div><div class='ctx'> 		uverbs_destroy_ufile_hw(file, RDMA_REMOVE_DRIVER_REMOVE);</div><div class='ctx'> 		kref_put(&amp;file-&gt;ref, ib_uverbs_release_file);</div><div class='ctx'> </div><div class='ctx'> 		mutex_lock(&amp;uverbs_dev-&gt;lists_mutex);</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='del'>-	while (!list_empty(&amp;uverbs_dev-&gt;uverbs_events_file_list)) {</div><div class='del'>-		event_file = list_first_entry(&amp;uverbs_dev-&gt;</div><div class='del'>-					      uverbs_events_file_list,</div><div class='del'>-					      struct ib_uverbs_async_event_file,</div><div class='del'>-					      list);</div><div class='del'>-		spin_lock_irq(&amp;event_file-&gt;ev_queue.lock);</div><div class='del'>-		event_file-&gt;ev_queue.is_closed = 1;</div><div class='del'>-		spin_unlock_irq(&amp;event_file-&gt;ev_queue.lock);</div><div class='del'>-</div><div class='del'>-		list_del(&amp;event_file-&gt;list);</div><div class='del'>-		ib_unregister_event_handler(</div><div class='del'>-			&amp;event_file-&gt;uverbs_file-&gt;event_handler);</div><div class='del'>-		event_file-&gt;uverbs_file-&gt;event_handler.device =</div><div class='del'>-			NULL;</div><div class='del'>-</div><div class='del'>-		wake_up_interruptible(&amp;event_file-&gt;ev_queue.poll_wait);</div><div class='del'>-		kill_fasync(&amp;event_file-&gt;ev_queue.async_queue, SIGIO, POLL_IN);</div><div class='del'>-	}</div><div class='ctx'> 	mutex_unlock(&amp;uverbs_dev-&gt;lists_mutex);</div><div class='ctx'> </div><div class='ctx'> 	uverbs_disassociate_api(uverbs_dev-&gt;uapi);</div><div class='hunk'>@@ -1354,9 +1256,6 @@ static void ib_uverbs_remove_one(struct ib_device *device, void *client_data)</div><div class='ctx'> 	struct ib_uverbs_device *uverbs_dev = client_data;</div><div class='ctx'> 	int wait_clients = 1;</div><div class='ctx'> </div><div class='del'>-	if (!uverbs_dev)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='ctx'> 	cdev_device_del(&amp;uverbs_dev-&gt;cdev, &amp;uverbs_dev-&gt;dev);</div><div class='ctx'> 	ida_free(&amp;uverbs_ida, uverbs_dev-&gt;devnum);</div><div class='ctx'> </div><div class='hunk'>@@ -1376,7 +1275,7 @@ static void ib_uverbs_remove_one(struct ib_device *device, void *client_data)</div><div class='ctx'> 		wait_clients = 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (atomic_dec_and_test(&amp;uverbs_dev-&gt;refcount))</div><div class='add'>+	if (refcount_dec_and_test(&amp;uverbs_dev-&gt;refcount))</div><div class='ctx'> 		ib_uverbs_comp_dev(uverbs_dev);</div><div class='ctx'> 	if (wait_clients)</div><div class='ctx'> 		wait_for_completion(&amp;uverbs_dev-&gt;comp);</div><div class='hunk'>@@ -1384,13 +1283,6 @@ static void ib_uverbs_remove_one(struct ib_device *device, void *client_data)</div><div class='ctx'> 	put_device(&amp;uverbs_dev-&gt;dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static char *uverbs_devnode(struct device *dev, umode_t *mode)</div><div class='del'>-{</div><div class='del'>-	if (mode)</div><div class='del'>-		*mode = 0666;</div><div class='del'>-	return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int __init ib_uverbs_init(void)</div><div class='ctx'> {</div><div class='ctx'> 	int ret;</div><div class='hunk'>@@ -1411,16 +1303,13 @@ static int __init ib_uverbs_init(void)</div><div class='ctx'> 		goto out_alloc;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	uverbs_class = class_create(THIS_MODULE, "infiniband_verbs");</div><div class='del'>-	if (IS_ERR(uverbs_class)) {</div><div class='del'>-		ret = PTR_ERR(uverbs_class);</div><div class='add'>+	ret = class_register(&amp;uverbs_class);</div><div class='add'>+	if (ret) {</div><div class='ctx'> 		pr_err("user_verbs: couldn't create class infiniband_verbs\n");</div><div class='ctx'> 		goto out_chrdev;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	uverbs_class-&gt;devnode = uverbs_devnode;</div><div class='del'>-</div><div class='del'>-	ret = class_create_file(uverbs_class, &amp;class_attr_abi_version.attr);</div><div class='add'>+	ret = class_create_file(&amp;uverbs_class, &amp;class_attr_abi_version.attr);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		pr_err("user_verbs: couldn't create abi_version attribute\n");</div><div class='ctx'> 		goto out_class;</div><div class='hunk'>@@ -1435,7 +1324,7 @@ static int __init ib_uverbs_init(void)</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> out_class:</div><div class='del'>-	class_destroy(uverbs_class);</div><div class='add'>+	class_unregister(&amp;uverbs_class);</div><div class='ctx'> </div><div class='ctx'> out_chrdev:</div><div class='ctx'> 	unregister_chrdev_region(dynamic_uverbs_dev,</div><div class='hunk'>@@ -1452,11 +1341,13 @@ out:</div><div class='ctx'> static void __exit ib_uverbs_cleanup(void)</div><div class='ctx'> {</div><div class='ctx'> 	ib_unregister_client(&amp;uverbs_client);</div><div class='del'>-	class_destroy(uverbs_class);</div><div class='add'>+	class_unregister(&amp;uverbs_class);</div><div class='ctx'> 	unregister_chrdev_region(IB_UVERBS_BASE_DEV,</div><div class='ctx'> 				 IB_UVERBS_NUM_FIXED_MINOR);</div><div class='ctx'> 	unregister_chrdev_region(dynamic_uverbs_dev,</div><div class='ctx'> 				 IB_UVERBS_NUM_DYNAMIC_MINOR);</div><div class='add'>+	ib_cleanup_ucaps();</div><div class='add'>+	mmu_notifier_synchronize();</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> module_init(ib_uverbs_init);</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_marshall.c b/drivers/infiniband/core/uverbs_marshall.c<br/>index b8d715c68ca4..e803f609ec87 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_marshall.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_marshall.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_marshall.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_marshall.c</a></div><div class='hunk'>@@ -66,7 +66,7 @@ void ib_copy_ah_attr_to_user(struct ib_device *device,</div><div class='ctx'> 	struct rdma_ah_attr *src = ah_attr;</div><div class='ctx'> 	struct rdma_ah_attr conv_ah;</div><div class='ctx'> </div><div class='del'>-	memset(&amp;dst-&gt;grh.reserved, 0, sizeof(dst-&gt;grh.reserved));</div><div class='add'>+	memset(&amp;dst-&gt;grh, 0, sizeof(dst-&gt;grh));</div><div class='ctx'> </div><div class='ctx'> 	if ((ah_attr-&gt;type == RDMA_AH_ATTR_TYPE_OPA) &amp;&amp;</div><div class='ctx'> 	    (rdma_ah_get_dlid(ah_attr) &gt; be16_to_cpu(IB_LID_PERMISSIVE)) &amp;&amp;</div><div class='hunk'>@@ -171,45 +171,3 @@ void ib_copy_path_rec_to_user(struct ib_user_path_rec *dst,</div><div class='ctx'> 	__ib_copy_path_rec_to_user(dst, src);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_copy_path_rec_to_user);</div><div class='del'>-</div><div class='del'>-void ib_copy_path_rec_from_user(struct sa_path_rec *dst,</div><div class='del'>-				struct ib_user_path_rec *src)</div><div class='del'>-{</div><div class='del'>-	u32 slid, dlid;</div><div class='del'>-</div><div class='del'>-	memset(dst, 0, sizeof(*dst));</div><div class='del'>-	if ((ib_is_opa_gid((union ib_gid *)src-&gt;sgid)) ||</div><div class='del'>-	    (ib_is_opa_gid((union ib_gid *)src-&gt;dgid))) {</div><div class='del'>-		dst-&gt;rec_type = SA_PATH_REC_TYPE_OPA;</div><div class='del'>-		slid = opa_get_lid_from_gid((union ib_gid *)src-&gt;sgid);</div><div class='del'>-		dlid = opa_get_lid_from_gid((union ib_gid *)src-&gt;dgid);</div><div class='del'>-	} else {</div><div class='del'>-		dst-&gt;rec_type = SA_PATH_REC_TYPE_IB;</div><div class='del'>-		slid = ntohs(src-&gt;slid);</div><div class='del'>-		dlid = ntohs(src-&gt;dlid);</div><div class='del'>-	}</div><div class='del'>-	memcpy(dst-&gt;dgid.raw, src-&gt;dgid, sizeof dst-&gt;dgid);</div><div class='del'>-	memcpy(dst-&gt;sgid.raw, src-&gt;sgid, sizeof dst-&gt;sgid);</div><div class='del'>-</div><div class='del'>-	sa_path_set_dlid(dst, dlid);</div><div class='del'>-	sa_path_set_slid(dst, slid);</div><div class='del'>-	sa_path_set_raw_traffic(dst, src-&gt;raw_traffic);</div><div class='del'>-	dst-&gt;flow_label		= src-&gt;flow_label;</div><div class='del'>-	dst-&gt;hop_limit		= src-&gt;hop_limit;</div><div class='del'>-	dst-&gt;traffic_class	= src-&gt;traffic_class;</div><div class='del'>-	dst-&gt;reversible		= src-&gt;reversible;</div><div class='del'>-	dst-&gt;numb_path		= src-&gt;numb_path;</div><div class='del'>-	dst-&gt;pkey		= src-&gt;pkey;</div><div class='del'>-	dst-&gt;sl			= src-&gt;sl;</div><div class='del'>-	dst-&gt;mtu_selector	= src-&gt;mtu_selector;</div><div class='del'>-	dst-&gt;mtu		= src-&gt;mtu;</div><div class='del'>-	dst-&gt;rate_selector	= src-&gt;rate_selector;</div><div class='del'>-	dst-&gt;rate		= src-&gt;rate;</div><div class='del'>-	dst-&gt;packet_life_time	= src-&gt;packet_life_time;</div><div class='del'>-	dst-&gt;preference		= src-&gt;preference;</div><div class='del'>-	dst-&gt;packet_life_time_selector = src-&gt;packet_life_time_selector;</div><div class='del'>-</div><div class='del'>-	/* TODO: No need to set this */</div><div class='del'>-	sa_path_set_dmac_zero(dst);</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_copy_path_rec_from_user);</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_std_types.c b/drivers/infiniband/core/uverbs_std_types.c<br/>index 35b2e2c640cc..13776a66e2e4 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_std_types.c</a></div><div class='hunk'>@@ -75,96 +75,28 @@ static int uverbs_free_mw(struct ib_uobject *uobject,</div><div class='ctx'> 	return uverbs_dealloc_mw((struct ib_mw *)uobject-&gt;object);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int uverbs_free_qp(struct ib_uobject *uobject,</div><div class='del'>-			  enum rdma_remove_reason why,</div><div class='del'>-			  struct uverbs_attr_bundle *attrs)</div><div class='del'>-{</div><div class='del'>-	struct ib_qp *qp = uobject-&gt;object;</div><div class='del'>-	struct ib_uqp_object *uqp =</div><div class='del'>-		container_of(uobject, struct ib_uqp_object, uevent.uobject);</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If this is a user triggered destroy then do not allow destruction</div><div class='del'>-	 * until the user cleans up all the mcast bindings. Unlike in other</div><div class='del'>-	 * places we forcibly clean up the mcast attachments for !DESTROY</div><div class='del'>-	 * because the mcast attaches are not ubojects and will not be</div><div class='del'>-	 * destroyed by anything else during cleanup processing.</div><div class='del'>-	 */</div><div class='del'>-	if (why == RDMA_REMOVE_DESTROY) {</div><div class='del'>-		if (!list_empty(&amp;uqp-&gt;mcast_list))</div><div class='del'>-			return -EBUSY;</div><div class='del'>-	} else if (qp == qp-&gt;real_qp) {</div><div class='del'>-		ib_uverbs_detach_umcast(qp, uqp);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = ib_destroy_qp_user(qp, &amp;attrs-&gt;driver_udata);</div><div class='del'>-	if (ib_is_destroy_retryable(ret, why, uobject))</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	if (uqp-&gt;uxrcd)</div><div class='del'>-		atomic_dec(&amp;uqp-&gt;uxrcd-&gt;refcnt);</div><div class='del'>-</div><div class='del'>-	ib_uverbs_release_uevent(attrs-&gt;ufile, &amp;uqp-&gt;uevent);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int uverbs_free_rwq_ind_tbl(struct ib_uobject *uobject,</div><div class='ctx'> 				   enum rdma_remove_reason why,</div><div class='ctx'> 				   struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_rwq_ind_table *rwq_ind_tbl = uobject-&gt;object;</div><div class='ctx'> 	struct ib_wq **ind_tbl = rwq_ind_tbl-&gt;ind_tbl;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	ret = ib_destroy_rwq_ind_table(rwq_ind_tbl);</div><div class='del'>-	if (ib_is_destroy_retryable(ret, why, uobject))</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	kfree(ind_tbl);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='add'>+	u32 table_size = (1 &lt;&lt; rwq_ind_tbl-&gt;log_ind_tbl_size);</div><div class='add'>+	int ret, i;</div><div class='ctx'> </div><div class='del'>-static int uverbs_free_wq(struct ib_uobject *uobject,</div><div class='del'>-			  enum rdma_remove_reason why,</div><div class='del'>-			  struct uverbs_attr_bundle *attrs)</div><div class='del'>-{</div><div class='del'>-	struct ib_wq *wq = uobject-&gt;object;</div><div class='del'>-	struct ib_uwq_object *uwq =</div><div class='del'>-		container_of(uobject, struct ib_uwq_object, uevent.uobject);</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	ret = ib_destroy_wq(wq, &amp;attrs-&gt;driver_udata);</div><div class='del'>-	if (ib_is_destroy_retryable(ret, why, uobject))</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	ib_uverbs_release_uevent(attrs-&gt;ufile, &amp;uwq-&gt;uevent);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int uverbs_free_srq(struct ib_uobject *uobject,</div><div class='del'>-			   enum rdma_remove_reason why,</div><div class='del'>-			   struct uverbs_attr_bundle *attrs)</div><div class='del'>-{</div><div class='del'>-	struct ib_srq *srq = uobject-&gt;object;</div><div class='del'>-	struct ib_uevent_object *uevent =</div><div class='del'>-		container_of(uobject, struct ib_uevent_object, uobject);</div><div class='del'>-	enum ib_srq_type  srq_type = srq-&gt;srq_type;</div><div class='del'>-	int ret;</div><div class='add'>+	if (atomic_read(&amp;rwq_ind_tbl-&gt;usecnt))</div><div class='add'>+		return -EBUSY;</div><div class='ctx'> </div><div class='del'>-	ret = ib_destroy_srq_user(srq, &amp;attrs-&gt;driver_udata);</div><div class='del'>-	if (ib_is_destroy_retryable(ret, why, uobject))</div><div class='add'>+	ret = rwq_ind_tbl-&gt;device-&gt;ops.destroy_rwq_ind_table(rwq_ind_tbl);</div><div class='add'>+	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	if (srq_type == IB_SRQT_XRC) {</div><div class='del'>-		struct ib_usrq_object *us =</div><div class='del'>-			container_of(uevent, struct ib_usrq_object, uevent);</div><div class='del'>-</div><div class='del'>-		atomic_dec(&amp;us-&gt;uxrcd-&gt;refcnt);</div><div class='del'>-	}</div><div class='add'>+	for (i = 0; i &lt; table_size; i++)</div><div class='add'>+		atomic_dec(&amp;ind_tbl[i]-&gt;usecnt);</div><div class='ctx'> </div><div class='del'>-	ib_uverbs_release_uevent(attrs-&gt;ufile, uevent);</div><div class='del'>-	return ret;</div><div class='add'>+	kfree(rwq_ind_tbl);</div><div class='add'>+	kfree(ind_tbl);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int uverbs_free_xrcd(struct ib_uobject *uobject,</div><div class='hunk'>@@ -176,9 +108,8 @@ static int uverbs_free_xrcd(struct ib_uobject *uobject,</div><div class='ctx'> 		container_of(uobject, struct ib_uxrcd_object, uobject);</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_destroy_usecnt(&amp;uxrcd-&gt;refcnt, why, uobject);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='add'>+	if (atomic_read(&amp;uxrcd-&gt;refcnt))</div><div class='add'>+		return -EBUSY;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;attrs-&gt;ufile-&gt;device-&gt;xrcd_tree_mutex);</div><div class='ctx'> 	ret = ib_uverbs_dealloc_xrcd(uobject, xrcd, why, attrs);</div><div class='hunk'>@@ -192,34 +123,47 @@ static int uverbs_free_pd(struct ib_uobject *uobject,</div><div class='ctx'> 			  struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_pd *pd = uobject-&gt;object;</div><div class='del'>-	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_destroy_usecnt(&amp;pd-&gt;usecnt, why, uobject);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='add'>+	if (atomic_read(&amp;pd-&gt;usecnt))</div><div class='add'>+		return -EBUSY;</div><div class='ctx'> </div><div class='del'>-	ib_dealloc_pd_user(pd, &amp;attrs-&gt;driver_udata);</div><div class='del'>-	return 0;</div><div class='add'>+	return ib_dealloc_pd_user(pd, &amp;attrs-&gt;driver_udata);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int uverbs_hot_unplug_completion_event_file(struct ib_uobject *uobj,</div><div class='del'>-						   enum rdma_remove_reason why)</div><div class='add'>+void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue)</div><div class='ctx'> {</div><div class='del'>-	struct ib_uverbs_completion_event_file *comp_event_file =</div><div class='del'>-		container_of(uobj, struct ib_uverbs_completion_event_file,</div><div class='del'>-			     uobj);</div><div class='del'>-	struct ib_uverbs_event_queue *event_queue = &amp;comp_event_file-&gt;ev_queue;</div><div class='add'>+	struct ib_uverbs_event *entry, *tmp;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irq(&amp;event_queue-&gt;lock);</div><div class='add'>+	/*</div><div class='add'>+	 * The user must ensure that no new items are added to the event_list</div><div class='add'>+	 * once is_closed is set.</div><div class='add'>+	 */</div><div class='ctx'> 	event_queue-&gt;is_closed = 1;</div><div class='ctx'> 	spin_unlock_irq(&amp;event_queue-&gt;lock);</div><div class='add'>+	wake_up_interruptible(&amp;event_queue-&gt;poll_wait);</div><div class='add'>+	kill_fasync(&amp;event_queue-&gt;async_queue, SIGIO, POLL_IN);</div><div class='ctx'> </div><div class='del'>-	if (why == RDMA_REMOVE_DRIVER_REMOVE) {</div><div class='del'>-		wake_up_interruptible(&amp;event_queue-&gt;poll_wait);</div><div class='del'>-		kill_fasync(&amp;event_queue-&gt;async_queue, SIGIO, POLL_IN);</div><div class='add'>+	spin_lock_irq(&amp;event_queue-&gt;lock);</div><div class='add'>+	list_for_each_entry_safe(entry, tmp, &amp;event_queue-&gt;event_list, list) {</div><div class='add'>+		if (entry-&gt;counter)</div><div class='add'>+			list_del(&amp;entry-&gt;obj_list);</div><div class='add'>+		list_del(&amp;entry-&gt;list);</div><div class='add'>+		kfree(entry);</div><div class='ctx'> 	}</div><div class='del'>-	return 0;</div><div class='del'>-};</div><div class='add'>+	spin_unlock_irq(&amp;event_queue-&gt;lock);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void</div><div class='add'>+uverbs_completion_event_file_destroy_uobj(struct ib_uobject *uobj,</div><div class='add'>+					  enum rdma_remove_reason why)</div><div class='add'>+{</div><div class='add'>+	struct ib_uverbs_completion_event_file *file =</div><div class='add'>+		container_of(uobj, struct ib_uverbs_completion_event_file,</div><div class='add'>+			     uobj);</div><div class='add'>+</div><div class='add'>+	ib_uverbs_free_event_queue(&amp;file-&gt;ev_queue);</div><div class='add'>+}</div><div class='ctx'> </div><div class='ctx'> int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='hunk'>@@ -230,15 +174,11 @@ EXPORT_SYMBOL(uverbs_destroy_def_handler);</div><div class='ctx'> DECLARE_UVERBS_NAMED_OBJECT(</div><div class='ctx'> 	UVERBS_OBJECT_COMP_CHANNEL,</div><div class='ctx'> 	UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_completion_event_file),</div><div class='del'>-			     uverbs_hot_unplug_completion_event_file,</div><div class='add'>+			     uverbs_completion_event_file_destroy_uobj,</div><div class='ctx'> 			     &amp;uverbs_event_fops,</div><div class='ctx'> 			     "[infinibandevent]",</div><div class='ctx'> 			     O_RDONLY));</div><div class='ctx'> </div><div class='del'>-DECLARE_UVERBS_NAMED_OBJECT(</div><div class='del'>-	UVERBS_OBJECT_QP,</div><div class='del'>-	UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uqp_object), uverbs_free_qp));</div><div class='del'>-</div><div class='ctx'> DECLARE_UVERBS_NAMED_METHOD_DESTROY(</div><div class='ctx'> 	UVERBS_METHOD_MW_DESTROY,</div><div class='ctx'> 	UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_MW_HANDLE,</div><div class='hunk'>@@ -250,11 +190,6 @@ DECLARE_UVERBS_NAMED_OBJECT(UVERBS_OBJECT_MW,</div><div class='ctx'> 			    UVERBS_TYPE_ALLOC_IDR(uverbs_free_mw),</div><div class='ctx'> 			    &amp;UVERBS_METHOD(UVERBS_METHOD_MW_DESTROY));</div><div class='ctx'> </div><div class='del'>-DECLARE_UVERBS_NAMED_OBJECT(</div><div class='del'>-	UVERBS_OBJECT_SRQ,</div><div class='del'>-	UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_usrq_object),</div><div class='del'>-				 uverbs_free_srq));</div><div class='del'>-</div><div class='ctx'> DECLARE_UVERBS_NAMED_METHOD_DESTROY(</div><div class='ctx'> 	UVERBS_METHOD_AH_DESTROY,</div><div class='ctx'> 	UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_AH_HANDLE,</div><div class='hunk'>@@ -279,10 +214,6 @@ DECLARE_UVERBS_NAMED_OBJECT(</div><div class='ctx'> 				 uverbs_free_flow),</div><div class='ctx'> 			    &amp;UVERBS_METHOD(UVERBS_METHOD_FLOW_DESTROY));</div><div class='ctx'> </div><div class='del'>-DECLARE_UVERBS_NAMED_OBJECT(</div><div class='del'>-	UVERBS_OBJECT_WQ,</div><div class='del'>-	UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uwq_object), uverbs_free_wq));</div><div class='del'>-</div><div class='ctx'> DECLARE_UVERBS_NAMED_METHOD_DESTROY(</div><div class='ctx'> 	UVERBS_METHOD_RWQ_IND_TBL_DESTROY,</div><div class='ctx'> 	UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_RWQ_IND_TBL_HANDLE,</div><div class='hunk'>@@ -323,18 +254,12 @@ const struct uapi_definition uverbs_def_obj_intf[] = {</div><div class='ctx'> 				      UAPI_DEF_OBJ_NEEDS_FN(dealloc_pd)),</div><div class='ctx'> 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_COMP_CHANNEL,</div><div class='ctx'> 				      UAPI_DEF_OBJ_NEEDS_FN(dealloc_pd)),</div><div class='del'>-	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_QP,</div><div class='del'>-				      UAPI_DEF_OBJ_NEEDS_FN(destroy_qp)),</div><div class='ctx'> 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_AH,</div><div class='ctx'> 				      UAPI_DEF_OBJ_NEEDS_FN(destroy_ah)),</div><div class='ctx'> 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_MW,</div><div class='ctx'> 				      UAPI_DEF_OBJ_NEEDS_FN(dealloc_mw)),</div><div class='del'>-	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_SRQ,</div><div class='del'>-				      UAPI_DEF_OBJ_NEEDS_FN(destroy_srq)),</div><div class='ctx'> 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_FLOW,</div><div class='ctx'> 				      UAPI_DEF_OBJ_NEEDS_FN(destroy_flow)),</div><div class='del'>-	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_WQ,</div><div class='del'>-				      UAPI_DEF_OBJ_NEEDS_FN(destroy_wq)),</div><div class='ctx'> 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(</div><div class='ctx'> 		UVERBS_OBJECT_RWQ_IND_TBL,</div><div class='ctx'> 		UAPI_DEF_OBJ_NEEDS_FN(destroy_rwq_ind_table)),</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_std_types_async_fd.c b/drivers/infiniband/core/uverbs_std_types_async_fd.c<br/>new file mode 100644<br/>index 000000000000..cc24cfdf7aee<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_async_fd.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_std_types_async_fd.c</a></div><div class='hunk'>@@ -0,0 +1,79 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2019, Mellanox Technologies inc.  All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/uverbs_std_types.h&gt;</div><div class='add'>+#include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='add'>+#include "rdma_core.h"</div><div class='add'>+#include "uverbs.h"</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(UVERBS_METHOD_ASYNC_EVENT_ALLOC)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *uobj =</div><div class='add'>+		uverbs_attr_get_uobject(attrs, UVERBS_METHOD_ASYNC_EVENT_ALLOC);</div><div class='add'>+</div><div class='add'>+	ib_uverbs_init_async_event_file(</div><div class='add'>+		container_of(uobj, struct ib_uverbs_async_event_file, uobj));</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void uverbs_async_event_destroy_uobj(struct ib_uobject *uobj,</div><div class='add'>+					    enum rdma_remove_reason why)</div><div class='add'>+{</div><div class='add'>+	struct ib_uverbs_async_event_file *event_file =</div><div class='add'>+		container_of(uobj, struct ib_uverbs_async_event_file, uobj);</div><div class='add'>+</div><div class='add'>+	ib_unregister_event_handler(&amp;event_file-&gt;event_handler);</div><div class='add'>+</div><div class='add'>+	if (why == RDMA_REMOVE_DRIVER_REMOVE)</div><div class='add'>+		ib_uverbs_async_handler(event_file, 0, IB_EVENT_DEVICE_FATAL,</div><div class='add'>+					NULL, NULL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int uverbs_async_event_release(struct inode *inode, struct file *filp)</div><div class='add'>+{</div><div class='add'>+	struct ib_uverbs_async_event_file *event_file;</div><div class='add'>+	struct ib_uobject *uobj = filp-&gt;private_data;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!uobj)</div><div class='add'>+		return uverbs_uobject_fd_release(inode, filp);</div><div class='add'>+</div><div class='add'>+	event_file =</div><div class='add'>+		container_of(uobj, struct ib_uverbs_async_event_file, uobj);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * The async event FD has to deliver IB_EVENT_DEVICE_FATAL even after</div><div class='add'>+	 * disassociation, so cleaning the event list must only happen after</div><div class='add'>+	 * release. The user knows it has reached the end of the event stream</div><div class='add'>+	 * when it sees IB_EVENT_DEVICE_FATAL.</div><div class='add'>+	 */</div><div class='add'>+	uverbs_uobject_get(uobj);</div><div class='add'>+	ret = uverbs_uobject_fd_release(inode, filp);</div><div class='add'>+	ib_uverbs_free_event_queue(&amp;event_file-&gt;ev_queue);</div><div class='add'>+	uverbs_uobject_put(uobj);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	UVERBS_METHOD_ASYNC_EVENT_ALLOC,</div><div class='add'>+	UVERBS_ATTR_FD(UVERBS_ATTR_ASYNC_EVENT_ALLOC_FD_HANDLE,</div><div class='add'>+		       UVERBS_OBJECT_ASYNC_EVENT,</div><div class='add'>+		       UVERBS_ACCESS_NEW,</div><div class='add'>+		       UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_OBJECT(</div><div class='add'>+	UVERBS_OBJECT_ASYNC_EVENT,</div><div class='add'>+	UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_async_event_file),</div><div class='add'>+			     uverbs_async_event_destroy_uobj,</div><div class='add'>+			     &amp;uverbs_async_event_fops,</div><div class='add'>+			     "[infinibandevent]",</div><div class='add'>+			     O_RDONLY),</div><div class='add'>+	&amp;UVERBS_METHOD(UVERBS_METHOD_ASYNC_EVENT_ALLOC));</div><div class='add'>+</div><div class='add'>+const struct uapi_definition uverbs_def_obj_async_fd[] = {</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_ASYNC_EVENT),</div><div class='add'>+	{}</div><div class='add'>+};</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_std_types_counters.c b/drivers/infiniband/core/uverbs_std_types_counters.c<br/>index 9f013304e677..381aa5797641 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_counters.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_counters.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_counters.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_std_types_counters.c</a></div><div class='hunk'>@@ -42,11 +42,14 @@ static int uverbs_free_counters(struct ib_uobject *uobject,</div><div class='ctx'> 	struct ib_counters *counters = uobject-&gt;object;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_destroy_usecnt(&amp;counters-&gt;usecnt, why, uobject);</div><div class='add'>+	if (atomic_read(&amp;counters-&gt;usecnt))</div><div class='add'>+		return -EBUSY;</div><div class='add'>+</div><div class='add'>+	ret = counters-&gt;device-&gt;ops.destroy_counters(counters);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='del'>-</div><div class='del'>-	return counters-&gt;device-&gt;ops.destroy_counters(counters);</div><div class='add'>+	kfree(counters);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int UVERBS_HANDLER(UVERBS_METHOD_COUNTERS_CREATE)(</div><div class='hunk'>@@ -66,20 +69,19 @@ static int UVERBS_HANDLER(UVERBS_METHOD_COUNTERS_CREATE)(</div><div class='ctx'> 	if (!ib_dev-&gt;ops.create_counters)</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='del'>-	counters = ib_dev-&gt;ops.create_counters(ib_dev, attrs);</div><div class='del'>-	if (IS_ERR(counters)) {</div><div class='del'>-		ret = PTR_ERR(counters);</div><div class='del'>-		goto err_create_counters;</div><div class='del'>-	}</div><div class='add'>+	counters = rdma_zalloc_drv_obj(ib_dev, ib_counters);</div><div class='add'>+	if (!counters)</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	counters-&gt;device = ib_dev;</div><div class='ctx'> 	counters-&gt;uobject = uobj;</div><div class='ctx'> 	uobj-&gt;object = counters;</div><div class='ctx'> 	atomic_set(&amp;counters-&gt;usecnt, 0);</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	ret = ib_dev-&gt;ops.create_counters(counters, attrs);</div><div class='add'>+	if (ret)</div><div class='add'>+		kfree(counters);</div><div class='ctx'> </div><div class='del'>-err_create_counters:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -105,6 +107,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_COUNTERS_READ)(</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='ctx'> 	uattr = uverbs_attr_get(attrs, UVERBS_ATTR_READ_COUNTERS_BUFF);</div><div class='add'>+	if (IS_ERR(uattr))</div><div class='add'>+		return PTR_ERR(uattr);</div><div class='ctx'> 	read_attr.ncounters = uattr-&gt;ptr_attr.len / sizeof(u64);</div><div class='ctx'> 	read_attr.counters_buff = uverbs_zalloc(</div><div class='ctx'> 		attrs, array_size(read_attr.ncounters, sizeof(u64)));</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c<br/>index db5c46a1bb2d..fab5d914029d 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_cq.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_cq.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_cq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_std_types_cq.c</a></div><div class='hunk'>@@ -33,6 +33,7 @@</div><div class='ctx'> #include &lt;rdma/uverbs_std_types.h&gt;</div><div class='ctx'> #include "rdma_core.h"</div><div class='ctx'> #include "uverbs.h"</div><div class='add'>+#include "restrack.h"</div><div class='ctx'> </div><div class='ctx'> static int uverbs_free_cq(struct ib_uobject *uobject,</div><div class='ctx'> 			  enum rdma_remove_reason why,</div><div class='hunk'>@@ -41,21 +42,20 @@ static int uverbs_free_cq(struct ib_uobject *uobject,</div><div class='ctx'> 	struct ib_cq *cq = uobject-&gt;object;</div><div class='ctx'> 	struct ib_uverbs_event_queue *ev_queue = cq-&gt;cq_context;</div><div class='ctx'> 	struct ib_ucq_object *ucq =</div><div class='del'>-		container_of(uobject, struct ib_ucq_object, uobject);</div><div class='add'>+		container_of(uobject, struct ib_ucq_object, uevent.uobject);</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	ret = ib_destroy_cq_user(cq, &amp;attrs-&gt;driver_udata);</div><div class='del'>-	if (ib_is_destroy_retryable(ret, why, uobject))</div><div class='add'>+	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='ctx'> 	ib_uverbs_release_ucq(</div><div class='del'>-		attrs-&gt;ufile,</div><div class='ctx'> 		ev_queue ? container_of(ev_queue,</div><div class='ctx'> 					struct ib_uverbs_completion_event_file,</div><div class='ctx'> 					ev_queue) :</div><div class='ctx'> 			   NULL,</div><div class='ctx'> 		ucq);</div><div class='del'>-	return ret;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(</div><div class='hunk'>@@ -63,16 +63,22 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_ucq_object *obj = container_of(</div><div class='ctx'> 		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_CREATE_CQ_HANDLE),</div><div class='del'>-		typeof(*obj), uobject);</div><div class='add'>+		typeof(*obj), uevent.uobject);</div><div class='add'>+	struct ib_uverbs_completion_event_file *ev_file = NULL;</div><div class='ctx'> 	struct ib_device *ib_dev = attrs-&gt;context-&gt;device;</div><div class='del'>-	int ret;</div><div class='del'>-	u64 user_handle;</div><div class='add'>+	struct ib_umem_dmabuf *umem_dmabuf;</div><div class='ctx'> 	struct ib_cq_init_attr attr = {};</div><div class='del'>-	struct ib_cq                   *cq;</div><div class='del'>-	struct ib_uverbs_completion_event_file    *ev_file = NULL;</div><div class='ctx'> 	struct ib_uobject *ev_file_uobj;</div><div class='add'>+	struct ib_umem *umem = NULL;</div><div class='add'>+	u64 buffer_length;</div><div class='add'>+	u64 buffer_offset;</div><div class='add'>+	struct ib_cq *cq;</div><div class='add'>+	u64 user_handle;</div><div class='add'>+	u64 buffer_va;</div><div class='add'>+	int buffer_fd;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	if (!ib_dev-&gt;ops.create_cq || !ib_dev-&gt;ops.destroy_cq)</div><div class='add'>+	if ((!ib_dev-&gt;ops.create_cq &amp;&amp; !ib_dev-&gt;ops.create_cq_umem) || !ib_dev-&gt;ops.destroy_cq)</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	ret = uverbs_copy_from(&amp;attr.comp_vector, attrs,</div><div class='hunk'>@@ -101,43 +107,111 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(</div><div class='ctx'> 		uverbs_uobject_get(ev_file_uobj);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	obj-&gt;uevent.event_file = ib_uverbs_get_async_event(</div><div class='add'>+		attrs, UVERBS_ATTR_CREATE_CQ_EVENT_FD);</div><div class='add'>+</div><div class='ctx'> 	if (attr.comp_vector &gt;= attrs-&gt;ufile-&gt;device-&gt;num_comp_vectors) {</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		goto err_event_file;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	obj-&gt;comp_events_reported  = 0;</div><div class='del'>-	obj-&gt;async_events_reported = 0;</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;obj-&gt;comp_list);</div><div class='del'>-	INIT_LIST_HEAD(&amp;obj-&gt;async_list);</div><div class='add'>+	INIT_LIST_HEAD(&amp;obj-&gt;uevent.event_list);</div><div class='add'>+</div><div class='add'>+	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA)) {</div><div class='ctx'> </div><div class='del'>-	cq = ib_dev-&gt;ops.create_cq(ib_dev, &amp;attr, &amp;attrs-&gt;driver_udata);</div><div class='del'>-	if (IS_ERR(cq)) {</div><div class='del'>-		ret = PTR_ERR(cq);</div><div class='add'>+		ret = uverbs_copy_from(&amp;buffer_va, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_event_file;</div><div class='add'>+</div><div class='add'>+		ret = uverbs_copy_from(&amp;buffer_length, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_event_file;</div><div class='add'>+</div><div class='add'>+		if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD) ||</div><div class='add'>+		    uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) ||</div><div class='add'>+		    !ib_dev-&gt;ops.create_cq_umem) {</div><div class='add'>+			ret = -EINVAL;</div><div class='add'>+			goto err_event_file;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		umem = ib_umem_get(ib_dev, buffer_va, buffer_length, IB_ACCESS_LOCAL_WRITE);</div><div class='add'>+		if (IS_ERR(umem)) {</div><div class='add'>+			ret = PTR_ERR(umem);</div><div class='add'>+			goto err_event_file;</div><div class='add'>+		}</div><div class='add'>+	} else if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD)) {</div><div class='add'>+</div><div class='add'>+		ret = uverbs_get_raw_fd(&amp;buffer_fd, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_event_file;</div><div class='add'>+</div><div class='add'>+		ret = uverbs_copy_from(&amp;buffer_offset, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_event_file;</div><div class='add'>+</div><div class='add'>+		ret = uverbs_copy_from(&amp;buffer_length, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_event_file;</div><div class='add'>+</div><div class='add'>+		if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA) ||</div><div class='add'>+		    !ib_dev-&gt;ops.create_cq_umem) {</div><div class='add'>+			ret = -EINVAL;</div><div class='add'>+			goto err_event_file;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		umem_dmabuf = ib_umem_dmabuf_get_pinned(ib_dev, buffer_offset, buffer_length,</div><div class='add'>+							buffer_fd, IB_ACCESS_LOCAL_WRITE);</div><div class='add'>+		if (IS_ERR(umem_dmabuf)) {</div><div class='add'>+			ret = PTR_ERR(umem_dmabuf);</div><div class='add'>+			goto err_event_file;</div><div class='add'>+		}</div><div class='add'>+		umem = &amp;umem_dmabuf-&gt;umem;</div><div class='add'>+	} else if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) ||</div><div class='add'>+		   uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH) ||</div><div class='add'>+		   !ib_dev-&gt;ops.create_cq) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err_event_file;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cq = rdma_zalloc_drv_obj(ib_dev, ib_cq);</div><div class='add'>+	if (!cq) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		ib_umem_release(umem);</div><div class='ctx'> 		goto err_event_file;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	cq-&gt;device        = ib_dev;</div><div class='del'>-	cq-&gt;uobject       = &amp;obj-&gt;uobject;</div><div class='add'>+	cq-&gt;uobject       = obj;</div><div class='ctx'> 	cq-&gt;comp_handler  = ib_uverbs_comp_handler;</div><div class='ctx'> 	cq-&gt;event_handler = ib_uverbs_cq_event_handler;</div><div class='ctx'> 	cq-&gt;cq_context    = ev_file ? &amp;ev_file-&gt;ev_queue : NULL;</div><div class='del'>-	obj-&gt;uobject.object = cq;</div><div class='del'>-	obj-&gt;uobject.user_handle = user_handle;</div><div class='ctx'> 	atomic_set(&amp;cq-&gt;usecnt, 0);</div><div class='del'>-	cq-&gt;res.type = RDMA_RESTRACK_CQ;</div><div class='del'>-	rdma_restrack_uadd(&amp;cq-&gt;res);</div><div class='ctx'> </div><div class='del'>-	ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_CQ_RESP_CQE, &amp;cq-&gt;cqe,</div><div class='del'>-			     sizeof(cq-&gt;cqe));</div><div class='add'>+	rdma_restrack_new(&amp;cq-&gt;res, RDMA_RESTRACK_CQ);</div><div class='add'>+	rdma_restrack_set_name(&amp;cq-&gt;res, NULL);</div><div class='add'>+</div><div class='add'>+	ret = umem ? ib_dev-&gt;ops.create_cq_umem(cq, &amp;attr, umem, attrs) :</div><div class='add'>+		ib_dev-&gt;ops.create_cq(cq, &amp;attr, attrs);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err_cq;</div><div class='add'>+		goto err_free;</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='del'>-err_cq:</div><div class='del'>-	ib_destroy_cq(cq);</div><div class='add'>+	obj-&gt;uevent.uobject.object = cq;</div><div class='add'>+	obj-&gt;uevent.uobject.user_handle = user_handle;</div><div class='add'>+	rdma_restrack_add(&amp;cq-&gt;res);</div><div class='add'>+	uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_CREATE_CQ_HANDLE);</div><div class='ctx'> </div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_CQ_RESP_CQE, &amp;cq-&gt;cqe,</div><div class='add'>+			     sizeof(cq-&gt;cqe));</div><div class='add'>+	return ret;</div><div class='add'>+</div><div class='add'>+err_free:</div><div class='add'>+	ib_umem_release(umem);</div><div class='add'>+	rdma_restrack_put(&amp;cq-&gt;res);</div><div class='add'>+	kfree(cq);</div><div class='ctx'> err_event_file:</div><div class='add'>+	if (obj-&gt;uevent.event_file)</div><div class='add'>+		uverbs_uobject_put(&amp;obj-&gt;uevent.event_file-&gt;uobj);</div><div class='ctx'> 	if (ev_file)</div><div class='ctx'> 		uverbs_uobject_put(ev_file_uobj);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -167,6 +241,21 @@ DECLARE_UVERBS_NAMED_METHOD(</div><div class='ctx'> 	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_CQ_RESP_CQE,</div><div class='ctx'> 			    UVERBS_ATTR_TYPE(u32),</div><div class='ctx'> 			    UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_FD(UVERBS_ATTR_CREATE_CQ_EVENT_FD,</div><div class='add'>+		       UVERBS_OBJECT_ASYNC_EVENT,</div><div class='add'>+		       UVERBS_ACCESS_READ,</div><div class='add'>+		       UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_CQ_BUFFER_VA,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			   UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			   UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_RAW_FD(UVERBS_ATTR_CREATE_CQ_BUFFER_FD,</div><div class='add'>+			   UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			   UA_OPTIONAL),</div><div class='ctx'> 	UVERBS_ATTR_UHW());</div><div class='ctx'> </div><div class='ctx'> static int UVERBS_HANDLER(UVERBS_METHOD_CQ_DESTROY)(</div><div class='hunk'>@@ -175,10 +264,10 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_DESTROY)(</div><div class='ctx'> 	struct ib_uobject *uobj =</div><div class='ctx'> 		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_DESTROY_CQ_HANDLE);</div><div class='ctx'> 	struct ib_ucq_object *obj =</div><div class='del'>-		container_of(uobj, struct ib_ucq_object, uobject);</div><div class='add'>+		container_of(uobj, struct ib_ucq_object, uevent.uobject);</div><div class='ctx'> 	struct ib_uverbs_destroy_cq_resp resp = {</div><div class='ctx'> 		.comp_events_reported = obj-&gt;comp_events_reported,</div><div class='del'>-		.async_events_reported = obj-&gt;async_events_reported</div><div class='add'>+		.async_events_reported = obj-&gt;uevent.events_reported</div><div class='ctx'> 	};</div><div class='ctx'> </div><div class='ctx'> 	return uverbs_copy_to(attrs, UVERBS_ATTR_DESTROY_CQ_RESP, &amp;resp,</div><div class='hunk'>@@ -198,11 +287,8 @@ DECLARE_UVERBS_NAMED_METHOD(</div><div class='ctx'> DECLARE_UVERBS_NAMED_OBJECT(</div><div class='ctx'> 	UVERBS_OBJECT_CQ,</div><div class='ctx'> 	UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_ucq_object), uverbs_free_cq),</div><div class='del'>-</div><div class='del'>-#if IS_ENABLED(CONFIG_INFINIBAND_EXP_LEGACY_VERBS_NEW_UAPI)</div><div class='ctx'> 	&amp;UVERBS_METHOD(UVERBS_METHOD_CQ_CREATE),</div><div class='ctx'> 	&amp;UVERBS_METHOD(UVERBS_METHOD_CQ_DESTROY)</div><div class='del'>-#endif</div><div class='ctx'> );</div><div class='ctx'> </div><div class='ctx'> const struct uapi_definition uverbs_def_obj_cq[] = {</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_std_types_device.c b/drivers/infiniband/core/uverbs_std_types_device.c<br/>index 2a3f2f01028d..c0fd283d9d6c 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_device.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_device.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_device.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_std_types_device.c</a></div><div class='hunk'>@@ -3,11 +3,13 @@</div><div class='ctx'>  * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='add'>+#include &lt;linux/overflow.h&gt;</div><div class='ctx'> #include &lt;rdma/uverbs_std_types.h&gt;</div><div class='ctx'> #include "rdma_core.h"</div><div class='ctx'> #include "uverbs.h"</div><div class='ctx'> #include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='ctx'> #include &lt;rdma/opa_addr.h&gt;</div><div class='add'>+#include &lt;rdma/ib_cache.h&gt;</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * This ioctl method allows calling any defined write or write_ex</div><div class='hunk'>@@ -38,7 +40,12 @@ static int UVERBS_HANDLER(UVERBS_METHOD_INVOKE_WRITE)(</div><div class='ctx'> 	    attrs-&gt;ucore.outlen &lt; method_elm-&gt;resp_size)</div><div class='ctx'> 		return -ENOSPC;</div><div class='ctx'> </div><div class='del'>-	return method_elm-&gt;handler(attrs);</div><div class='add'>+	attrs-&gt;uobject = NULL;</div><div class='add'>+	rc = method_elm-&gt;handler(attrs);</div><div class='add'>+	if (attrs-&gt;uobject)</div><div class='add'>+		uverbs_finalize_object(attrs-&gt;uobject, UVERBS_ACCESS_NEW, true,</div><div class='add'>+				       !rc, attrs);</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> DECLARE_UVERBS_NAMED_METHOD(UVERBS_METHOD_INVOKE_WRITE,</div><div class='hunk'>@@ -110,8 +117,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_INFO_HANDLES)(</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='ctx'> 	uapi_object = uapi_get_object(attrs-&gt;ufile-&gt;device-&gt;uapi, object_id);</div><div class='del'>-	if (!uapi_object)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	if (IS_ERR(uapi_object))</div><div class='add'>+		return PTR_ERR(uapi_object);</div><div class='ctx'> </div><div class='ctx'> 	handles = gather_objects_handle(attrs-&gt;ufile, uapi_object, attrs,</div><div class='ctx'> 					out_len, &amp;total);</div><div class='hunk'>@@ -160,7 +167,8 @@ void copy_port_attr_to_resp(struct ib_port_attr *attr,</div><div class='ctx'> 	resp-&gt;subnet_timeout = attr-&gt;subnet_timeout;</div><div class='ctx'> 	resp-&gt;init_type_reply = attr-&gt;init_type_reply;</div><div class='ctx'> 	resp-&gt;active_width = attr-&gt;active_width;</div><div class='del'>-	resp-&gt;active_speed = attr-&gt;active_speed;</div><div class='add'>+	/* This ABI needs to be extended to provide any speed more than IB_SPEED_NDR */</div><div class='add'>+	resp-&gt;active_speed = min_t(u16, attr-&gt;active_speed, IB_SPEED_NDR);</div><div class='ctx'> 	resp-&gt;phys_state = attr-&gt;phys_state;</div><div class='ctx'> 	resp-&gt;link_layer = rdma_port_get_link_layer(ib_dev, port_num);</div><div class='ctx'> }</div><div class='hunk'>@@ -195,11 +203,253 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_PORT)(</div><div class='ctx'> </div><div class='ctx'> 	copy_port_attr_to_resp(&amp;attr, &amp;resp.legacy_resp, ib_dev, port_num);</div><div class='ctx'> 	resp.port_cap_flags2 = attr.port_cap_flags2;</div><div class='add'>+	resp.active_speed_ex = attr.active_speed;</div><div class='ctx'> </div><div class='ctx'> 	return uverbs_copy_to_struct_or_zero(attrs, UVERBS_ATTR_QUERY_PORT_RESP,</div><div class='ctx'> 					     &amp;resp, sizeof(resp));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int UVERBS_HANDLER(UVERBS_METHOD_GET_CONTEXT)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	u32 num_comp = attrs-&gt;ufile-&gt;device-&gt;num_comp_vectors;</div><div class='add'>+	u64 core_support = IB_UVERBS_CORE_SUPPORT_OPTIONAL_MR_ACCESS;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS,</div><div class='add'>+			     &amp;num_comp, sizeof(num_comp));</div><div class='add'>+	if (IS_UVERBS_COPY_ERR(ret))</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT,</div><div class='add'>+			     &amp;core_support, sizeof(core_support));</div><div class='add'>+	if (IS_UVERBS_COPY_ERR(ret))</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = ib_alloc_ucontext(attrs);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+	ret = ib_init_ucontext(attrs);</div><div class='add'>+	if (ret) {</div><div class='add'>+		kfree(attrs-&gt;context);</div><div class='add'>+		attrs-&gt;context = NULL;</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_CONTEXT)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	u64 core_support = IB_UVERBS_CORE_SUPPORT_OPTIONAL_MR_ACCESS;</div><div class='add'>+	struct ib_ucontext *ucontext;</div><div class='add'>+	struct ib_device *ib_dev;</div><div class='add'>+	u32 num_comp;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ucontext = ib_uverbs_get_ucontext(attrs);</div><div class='add'>+	if (IS_ERR(ucontext))</div><div class='add'>+		return PTR_ERR(ucontext);</div><div class='add'>+	ib_dev = ucontext-&gt;device;</div><div class='add'>+</div><div class='add'>+	if (!ib_dev-&gt;ops.query_ucontext)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	num_comp = attrs-&gt;ufile-&gt;device-&gt;num_comp_vectors;</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_CONTEXT_NUM_COMP_VECTORS,</div><div class='add'>+			     &amp;num_comp, sizeof(num_comp));</div><div class='add'>+	if (IS_UVERBS_COPY_ERR(ret))</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_CONTEXT_CORE_SUPPORT,</div><div class='add'>+			     &amp;core_support, sizeof(core_support));</div><div class='add'>+	if (IS_UVERBS_COPY_ERR(ret))</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	return ucontext-&gt;device-&gt;ops.query_ucontext(ucontext, attrs);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int copy_gid_entries_to_user(struct uverbs_attr_bundle *attrs,</div><div class='add'>+				    struct ib_uverbs_gid_entry *entries,</div><div class='add'>+				    size_t num_entries, size_t user_entry_size)</div><div class='add'>+{</div><div class='add'>+	const struct uverbs_attr *attr;</div><div class='add'>+	void __user *user_entries;</div><div class='add'>+	size_t copy_len;</div><div class='add'>+	int ret;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (user_entry_size == sizeof(*entries)) {</div><div class='add'>+		ret = uverbs_copy_to(attrs,</div><div class='add'>+				     UVERBS_ATTR_QUERY_GID_TABLE_RESP_ENTRIES,</div><div class='add'>+				     entries, sizeof(*entries) * num_entries);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	copy_len = min_t(size_t, user_entry_size, sizeof(*entries));</div><div class='add'>+	attr = uverbs_attr_get(attrs, UVERBS_ATTR_QUERY_GID_TABLE_RESP_ENTRIES);</div><div class='add'>+	if (IS_ERR(attr))</div><div class='add'>+		return PTR_ERR(attr);</div><div class='add'>+</div><div class='add'>+	user_entries = u64_to_user_ptr(attr-&gt;ptr_attr.data);</div><div class='add'>+	for (i = 0; i &lt; num_entries; i++) {</div><div class='add'>+		if (copy_to_user(user_entries, entries, copy_len))</div><div class='add'>+			return -EFAULT;</div><div class='add'>+</div><div class='add'>+		if (user_entry_size &gt; sizeof(*entries)) {</div><div class='add'>+			if (clear_user(user_entries + sizeof(*entries),</div><div class='add'>+				       user_entry_size - sizeof(*entries)))</div><div class='add'>+				return -EFAULT;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		entries++;</div><div class='add'>+		user_entries += user_entry_size;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return uverbs_output_written(attrs,</div><div class='add'>+				     UVERBS_ATTR_QUERY_GID_TABLE_RESP_ENTRIES);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_GID_TABLE)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uverbs_gid_entry *entries;</div><div class='add'>+	struct ib_ucontext *ucontext;</div><div class='add'>+	struct ib_device *ib_dev;</div><div class='add'>+	size_t user_entry_size;</div><div class='add'>+	ssize_t num_entries;</div><div class='add'>+	int max_entries;</div><div class='add'>+	u32 flags;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_get_flags32(&amp;flags, attrs,</div><div class='add'>+				 UVERBS_ATTR_QUERY_GID_TABLE_FLAGS, 0);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_get_const(&amp;user_entry_size, attrs,</div><div class='add'>+			       UVERBS_ATTR_QUERY_GID_TABLE_ENTRY_SIZE);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (!user_entry_size)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	max_entries = uverbs_attr_ptr_get_array_size(</div><div class='add'>+		attrs, UVERBS_ATTR_QUERY_GID_TABLE_RESP_ENTRIES,</div><div class='add'>+		user_entry_size);</div><div class='add'>+	if (max_entries &lt;= 0)</div><div class='add'>+		return max_entries ?: -EINVAL;</div><div class='add'>+</div><div class='add'>+	ucontext = ib_uverbs_get_ucontext(attrs);</div><div class='add'>+	if (IS_ERR(ucontext))</div><div class='add'>+		return PTR_ERR(ucontext);</div><div class='add'>+	ib_dev = ucontext-&gt;device;</div><div class='add'>+</div><div class='add'>+	entries = uverbs_kcalloc(attrs, max_entries, sizeof(*entries));</div><div class='add'>+	if (IS_ERR(entries))</div><div class='add'>+		return PTR_ERR(entries);</div><div class='add'>+</div><div class='add'>+	num_entries = rdma_query_gid_table(ib_dev, entries, max_entries);</div><div class='add'>+	if (num_entries &lt; 0)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ret = copy_gid_entries_to_user(attrs, entries, num_entries,</div><div class='add'>+				       user_entry_size);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs,</div><div class='add'>+			     UVERBS_ATTR_QUERY_GID_TABLE_RESP_NUM_ENTRIES,</div><div class='add'>+			     &amp;num_entries, sizeof(num_entries));</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_GID_ENTRY)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uverbs_gid_entry entry = {};</div><div class='add'>+	const struct ib_gid_attr *gid_attr;</div><div class='add'>+	struct ib_ucontext *ucontext;</div><div class='add'>+	struct ib_device *ib_dev;</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+	u32 gid_index;</div><div class='add'>+	u32 port_num;</div><div class='add'>+	u32 flags;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_get_flags32(&amp;flags, attrs,</div><div class='add'>+				 UVERBS_ATTR_QUERY_GID_ENTRY_FLAGS, 0);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_get_const(&amp;port_num, attrs,</div><div class='add'>+			       UVERBS_ATTR_QUERY_GID_ENTRY_PORT);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_get_const(&amp;gid_index, attrs,</div><div class='add'>+			       UVERBS_ATTR_QUERY_GID_ENTRY_GID_INDEX);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ucontext = ib_uverbs_get_ucontext(attrs);</div><div class='add'>+	if (IS_ERR(ucontext))</div><div class='add'>+		return PTR_ERR(ucontext);</div><div class='add'>+	ib_dev = ucontext-&gt;device;</div><div class='add'>+</div><div class='add'>+	if (!rdma_is_port_valid(ib_dev, port_num))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	gid_attr = rdma_get_gid_attr(ib_dev, port_num, gid_index);</div><div class='add'>+	if (IS_ERR(gid_attr))</div><div class='add'>+		return PTR_ERR(gid_attr);</div><div class='add'>+</div><div class='add'>+	memcpy(&amp;entry.gid, &amp;gid_attr-&gt;gid, sizeof(gid_attr-&gt;gid));</div><div class='add'>+	entry.gid_index = gid_attr-&gt;index;</div><div class='add'>+	entry.port_num = gid_attr-&gt;port_num;</div><div class='add'>+	entry.gid_type = gid_attr-&gt;gid_type;</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	ndev = rdma_read_gid_attr_ndev_rcu(gid_attr);</div><div class='add'>+	if (IS_ERR(ndev)) {</div><div class='add'>+		if (PTR_ERR(ndev) != -ENODEV) {</div><div class='add'>+			ret = PTR_ERR(ndev);</div><div class='add'>+			rcu_read_unlock();</div><div class='add'>+			goto out;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		entry.netdev_ifindex = ndev-&gt;ifindex;</div><div class='add'>+	}</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to_struct_or_zero(</div><div class='add'>+		attrs, UVERBS_ATTR_QUERY_GID_ENTRY_RESP_ENTRY, &amp;entry,</div><div class='add'>+		sizeof(entry));</div><div class='add'>+out:</div><div class='add'>+	rdma_put_gid_attr(gid_attr);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	UVERBS_METHOD_GET_CONTEXT,</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u32), UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u64), UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_GET_CONTEXT_FD_ARR,</div><div class='add'>+			   UVERBS_ATTR_MIN_SIZE(sizeof(int)),</div><div class='add'>+			   UA_OPTIONAL,</div><div class='add'>+			   UA_ALLOC_AND_COPY),</div><div class='add'>+	UVERBS_ATTR_UHW());</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	UVERBS_METHOD_QUERY_CONTEXT,</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_CONTEXT_NUM_COMP_VECTORS,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u32), UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_CONTEXT_CORE_SUPPORT,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u64), UA_OPTIONAL));</div><div class='add'>+</div><div class='ctx'> DECLARE_UVERBS_NAMED_METHOD(</div><div class='ctx'> 	UVERBS_METHOD_INFO_HANDLES,</div><div class='ctx'> 	/* Also includes any device specific object ids */</div><div class='hunk'>@@ -216,13 +466,41 @@ DECLARE_UVERBS_NAMED_METHOD(</div><div class='ctx'> 	UVERBS_ATTR_PTR_OUT(</div><div class='ctx'> 		UVERBS_ATTR_QUERY_PORT_RESP,</div><div class='ctx'> 		UVERBS_ATTR_STRUCT(struct ib_uverbs_query_port_resp_ex,</div><div class='del'>-				   reserved),</div><div class='add'>+				   active_speed_ex),</div><div class='ctx'> 		UA_MANDATORY));</div><div class='ctx'> </div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	UVERBS_METHOD_QUERY_GID_TABLE,</div><div class='add'>+	UVERBS_ATTR_CONST_IN(UVERBS_ATTR_QUERY_GID_TABLE_ENTRY_SIZE, u64,</div><div class='add'>+			     UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_QUERY_GID_TABLE_FLAGS, u32,</div><div class='add'>+			     UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_GID_TABLE_RESP_ENTRIES,</div><div class='add'>+			    UVERBS_ATTR_MIN_SIZE(0), UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_GID_TABLE_RESP_NUM_ENTRIES,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u64), UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	UVERBS_METHOD_QUERY_GID_ENTRY,</div><div class='add'>+	UVERBS_ATTR_CONST_IN(UVERBS_ATTR_QUERY_GID_ENTRY_PORT, u32,</div><div class='add'>+			     UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_CONST_IN(UVERBS_ATTR_QUERY_GID_ENTRY_GID_INDEX, u32,</div><div class='add'>+			     UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_QUERY_GID_ENTRY_FLAGS, u32,</div><div class='add'>+			     UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_GID_ENTRY_RESP_ENTRY,</div><div class='add'>+			    UVERBS_ATTR_STRUCT(struct ib_uverbs_gid_entry,</div><div class='add'>+					       netdev_ifindex),</div><div class='add'>+			    UA_MANDATORY));</div><div class='add'>+</div><div class='ctx'> DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DEVICE,</div><div class='add'>+			      &amp;UVERBS_METHOD(UVERBS_METHOD_GET_CONTEXT),</div><div class='ctx'> 			      &amp;UVERBS_METHOD(UVERBS_METHOD_INVOKE_WRITE),</div><div class='ctx'> 			      &amp;UVERBS_METHOD(UVERBS_METHOD_INFO_HANDLES),</div><div class='del'>-			      &amp;UVERBS_METHOD(UVERBS_METHOD_QUERY_PORT));</div><div class='add'>+			      &amp;UVERBS_METHOD(UVERBS_METHOD_QUERY_PORT),</div><div class='add'>+			      &amp;UVERBS_METHOD(UVERBS_METHOD_QUERY_CONTEXT),</div><div class='add'>+			      &amp;UVERBS_METHOD(UVERBS_METHOD_QUERY_GID_TABLE),</div><div class='add'>+			      &amp;UVERBS_METHOD(UVERBS_METHOD_QUERY_GID_ENTRY));</div><div class='ctx'> </div><div class='ctx'> const struct uapi_definition uverbs_def_obj_device[] = {</div><div class='ctx'> 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DEVICE),</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_std_types_dm.c b/drivers/infiniband/core/uverbs_std_types_dm.c<br/>index d5a1de33c2c9..98c522cf86d6 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_dm.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_dm.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_dm.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_std_types_dm.c</a></div><div class='hunk'>@@ -39,11 +39,9 @@ static int uverbs_free_dm(struct ib_uobject *uobject,</div><div class='ctx'> 			  struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_dm *dm = uobject-&gt;object;</div><div class='del'>-	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_destroy_usecnt(&amp;dm-&gt;usecnt, why, uobject);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='add'>+	if (atomic_read(&amp;dm-&gt;usecnt))</div><div class='add'>+		return -EBUSY;</div><div class='ctx'> </div><div class='ctx'> 	return dm-&gt;device-&gt;ops.dealloc_dm(dm, attrs);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_std_types_dmah.c b/drivers/infiniband/core/uverbs_std_types_dmah.c<br/>new file mode 100644<br/>index 000000000000..453ce656c6f2<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_dmah.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_std_types_dmah.c</a></div><div class='hunk'>@@ -0,0 +1,145 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2025, NVIDIA CORPORATION &amp; AFFILIATES. All rights reserved</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include "rdma_core.h"</div><div class='add'>+#include "uverbs.h"</div><div class='add'>+#include &lt;rdma/uverbs_std_types.h&gt;</div><div class='add'>+#include "restrack.h"</div><div class='add'>+</div><div class='add'>+static int uverbs_free_dmah(struct ib_uobject *uobject,</div><div class='add'>+			    enum rdma_remove_reason why,</div><div class='add'>+			    struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_dmah *dmah = uobject-&gt;object;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (atomic_read(&amp;dmah-&gt;usecnt))</div><div class='add'>+		return -EBUSY;</div><div class='add'>+</div><div class='add'>+	ret = dmah-&gt;device-&gt;ops.dealloc_dmah(dmah, attrs);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	rdma_restrack_del(&amp;dmah-&gt;res);</div><div class='add'>+	kfree(dmah);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(UVERBS_METHOD_DMAH_ALLOC)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *uobj =</div><div class='add'>+		uverbs_attr_get(attrs, UVERBS_ATTR_ALLOC_DMAH_HANDLE)</div><div class='add'>+			-&gt;obj_attr.uobject;</div><div class='add'>+	struct ib_device *ib_dev = attrs-&gt;context-&gt;device;</div><div class='add'>+	struct ib_dmah *dmah;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	dmah = rdma_zalloc_drv_obj(ib_dev, ib_dmah);</div><div class='add'>+	if (!dmah)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_ALLOC_DMAH_CPU_ID)) {</div><div class='add'>+		ret = uverbs_copy_from(&amp;dmah-&gt;cpu_id, attrs,</div><div class='add'>+				       UVERBS_ATTR_ALLOC_DMAH_CPU_ID);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err;</div><div class='add'>+</div><div class='add'>+		if (!cpumask_test_cpu(dmah-&gt;cpu_id, current-&gt;cpus_ptr)) {</div><div class='add'>+			ret = -EPERM;</div><div class='add'>+			goto err;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		dmah-&gt;valid_fields |= BIT(IB_DMAH_CPU_ID_EXISTS);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_ALLOC_DMAH_TPH_MEM_TYPE)) {</div><div class='add'>+		dmah-&gt;mem_type = uverbs_attr_get_enum_id(attrs,</div><div class='add'>+					UVERBS_ATTR_ALLOC_DMAH_TPH_MEM_TYPE);</div><div class='add'>+		dmah-&gt;valid_fields |= BIT(IB_DMAH_MEM_TYPE_EXISTS);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_ALLOC_DMAH_PH)) {</div><div class='add'>+		ret = uverbs_copy_from(&amp;dmah-&gt;ph, attrs,</div><div class='add'>+				       UVERBS_ATTR_ALLOC_DMAH_PH);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err;</div><div class='add'>+</div><div class='add'>+		/* Per PCIe spec 6.2-1.0, only the lowest two bits are applicable */</div><div class='add'>+		if (dmah-&gt;ph &amp; 0xFC) {</div><div class='add'>+			ret = -EINVAL;</div><div class='add'>+			goto err;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		dmah-&gt;valid_fields |= BIT(IB_DMAH_PH_EXISTS);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dmah-&gt;device = ib_dev;</div><div class='add'>+	dmah-&gt;uobject = uobj;</div><div class='add'>+	atomic_set(&amp;dmah-&gt;usecnt, 0);</div><div class='add'>+</div><div class='add'>+	rdma_restrack_new(&amp;dmah-&gt;res, RDMA_RESTRACK_DMAH);</div><div class='add'>+	rdma_restrack_set_name(&amp;dmah-&gt;res, NULL);</div><div class='add'>+</div><div class='add'>+	ret = ib_dev-&gt;ops.alloc_dmah(dmah, attrs);</div><div class='add'>+	if (ret) {</div><div class='add'>+		rdma_restrack_put(&amp;dmah-&gt;res);</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	uobj-&gt;object = dmah;</div><div class='add'>+	rdma_restrack_add(&amp;dmah-&gt;res);</div><div class='add'>+	uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_ALLOC_DMAH_HANDLE);</div><div class='add'>+	return 0;</div><div class='add'>+err:</div><div class='add'>+	kfree(dmah);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct uverbs_attr_spec uverbs_dmah_mem_type[] = {</div><div class='add'>+	[TPH_MEM_TYPE_VM] = {</div><div class='add'>+		.type = UVERBS_ATTR_TYPE_PTR_IN,</div><div class='add'>+		UVERBS_ATTR_NO_DATA(),</div><div class='add'>+	},</div><div class='add'>+	[TPH_MEM_TYPE_PM] = {</div><div class='add'>+		.type = UVERBS_ATTR_TYPE_PTR_IN,</div><div class='add'>+		UVERBS_ATTR_NO_DATA(),</div><div class='add'>+	},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	UVERBS_METHOD_DMAH_ALLOC,</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_ALLOC_DMAH_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_DMAH,</div><div class='add'>+			UVERBS_ACCESS_NEW,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_ALLOC_DMAH_CPU_ID,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_ENUM_IN(UVERBS_ATTR_ALLOC_DMAH_TPH_MEM_TYPE,</div><div class='add'>+			    uverbs_dmah_mem_type,</div><div class='add'>+			    UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_ALLOC_DMAH_PH,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u8),</div><div class='add'>+			   UA_OPTIONAL));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD_DESTROY(</div><div class='add'>+	UVERBS_METHOD_DMAH_FREE,</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_FREE_DMA_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_DMAH,</div><div class='add'>+			UVERBS_ACCESS_DESTROY,</div><div class='add'>+			UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_OBJECT(UVERBS_OBJECT_DMAH,</div><div class='add'>+			    UVERBS_TYPE_ALLOC_IDR(uverbs_free_dmah),</div><div class='add'>+			    &amp;UVERBS_METHOD(UVERBS_METHOD_DMAH_ALLOC),</div><div class='add'>+			    &amp;UVERBS_METHOD(UVERBS_METHOD_DMAH_FREE));</div><div class='add'>+</div><div class='add'>+const struct uapi_definition uverbs_def_obj_dmah[] = {</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DMAH,</div><div class='add'>+				      UAPI_DEF_OBJ_NEEDS_FN(dealloc_dmah),</div><div class='add'>+				      UAPI_DEF_OBJ_NEEDS_FN(alloc_dmah)),</div><div class='add'>+	{}</div><div class='add'>+};</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_std_types_flow_action.c b/drivers/infiniband/core/uverbs_std_types_flow_action.c<br/>index 459cf165b231..0ddcf6da66c4 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_flow_action.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_flow_action.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_flow_action.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_std_types_flow_action.c</a></div><div class='hunk'>@@ -39,394 +39,13 @@ static int uverbs_free_flow_action(struct ib_uobject *uobject,</div><div class='ctx'> 				   struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_flow_action *action = uobject-&gt;object;</div><div class='del'>-	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_destroy_usecnt(&amp;action-&gt;usecnt, why, uobject);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='add'>+	if (atomic_read(&amp;action-&gt;usecnt))</div><div class='add'>+		return -EBUSY;</div><div class='ctx'> </div><div class='ctx'> 	return action-&gt;device-&gt;ops.destroy_flow_action(action);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static u64 esp_flags_uverbs_to_verbs(struct uverbs_attr_bundle *attrs,</div><div class='del'>-				     u32 flags, bool is_modify)</div><div class='del'>-{</div><div class='del'>-	u64 verbs_flags = flags;</div><div class='del'>-</div><div class='del'>-	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_FLOW_ACTION_ESP_ESN))</div><div class='del'>-		verbs_flags |= IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED;</div><div class='del'>-</div><div class='del'>-	if (is_modify &amp;&amp; uverbs_attr_is_valid(attrs,</div><div class='del'>-					      UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS))</div><div class='del'>-		verbs_flags |= IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS;</div><div class='del'>-</div><div class='del'>-	return verbs_flags;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static int validate_flow_action_esp_keymat_aes_gcm(struct ib_flow_action_attrs_esp_keymats *keymat)</div><div class='del'>-{</div><div class='del'>-	struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm =</div><div class='del'>-		&amp;keymat-&gt;keymat.aes_gcm;</div><div class='del'>-</div><div class='del'>-	if (aes_gcm-&gt;iv_algo &gt; IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ)</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-	if (aes_gcm-&gt;key_len != 32 &amp;&amp;</div><div class='del'>-	    aes_gcm-&gt;key_len != 24 &amp;&amp;</div><div class='del'>-	    aes_gcm-&gt;key_len != 16)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	if (aes_gcm-&gt;icv_len != 16 &amp;&amp;</div><div class='del'>-	    aes_gcm-&gt;icv_len != 8 &amp;&amp;</div><div class='del'>-	    aes_gcm-&gt;icv_len != 12)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int (* const flow_action_esp_keymat_validate[])(struct ib_flow_action_attrs_esp_keymats *keymat) = {</div><div class='del'>-	[IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM] = validate_flow_action_esp_keymat_aes_gcm,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static int flow_action_esp_replay_none(struct ib_flow_action_attrs_esp_replays *replay,</div><div class='del'>-				       bool is_modify)</div><div class='del'>-{</div><div class='del'>-	/* This is used in order to modify an esp flow action with an enabled</div><div class='del'>-	 * replay protection to a disabled one. This is only supported via</div><div class='del'>-	 * modify, as in create verb we can simply drop the REPLAY attribute and</div><div class='del'>-	 * achieve the same thing.</div><div class='del'>-	 */</div><div class='del'>-	return is_modify ? 0 : -EINVAL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int flow_action_esp_replay_def_ok(struct ib_flow_action_attrs_esp_replays *replay,</div><div class='del'>-					 bool is_modify)</div><div class='del'>-{</div><div class='del'>-	/* Some replay protections could always be enabled without validating</div><div class='del'>-	 * anything.</div><div class='del'>-	 */</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int (* const flow_action_esp_replay_validate[])(struct ib_flow_action_attrs_esp_replays *replay,</div><div class='del'>-						       bool is_modify) = {</div><div class='del'>-	[IB_UVERBS_FLOW_ACTION_ESP_REPLAY_NONE] = flow_action_esp_replay_none,</div><div class='del'>-	[IB_UVERBS_FLOW_ACTION_ESP_REPLAY_BMP] = flow_action_esp_replay_def_ok,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static int parse_esp_ip(enum ib_flow_spec_type proto,</div><div class='del'>-			const void __user *val_ptr,</div><div class='del'>-			size_t len, union ib_flow_spec *out)</div><div class='del'>-{</div><div class='del'>-	int ret;</div><div class='del'>-	const struct ib_uverbs_flow_ipv4_filter ipv4 = {</div><div class='del'>-		.src_ip = cpu_to_be32(0xffffffffUL),</div><div class='del'>-		.dst_ip = cpu_to_be32(0xffffffffUL),</div><div class='del'>-		.proto = 0xff,</div><div class='del'>-		.tos = 0xff,</div><div class='del'>-		.ttl = 0xff,</div><div class='del'>-		.flags = 0xff,</div><div class='del'>-	};</div><div class='del'>-	const struct ib_uverbs_flow_ipv6_filter ipv6 = {</div><div class='del'>-		.src_ip = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,</div><div class='del'>-			   0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},</div><div class='del'>-		.dst_ip = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,</div><div class='del'>-			   0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},</div><div class='del'>-		.flow_label = cpu_to_be32(0xffffffffUL),</div><div class='del'>-		.next_hdr = 0xff,</div><div class='del'>-		.traffic_class = 0xff,</div><div class='del'>-		.hop_limit = 0xff,</div><div class='del'>-	};</div><div class='del'>-	union {</div><div class='del'>-		struct ib_uverbs_flow_ipv4_filter ipv4;</div><div class='del'>-		struct ib_uverbs_flow_ipv6_filter ipv6;</div><div class='del'>-	} user_val = {};</div><div class='del'>-	const void *user_pmask;</div><div class='del'>-	size_t val_len;</div><div class='del'>-</div><div class='del'>-	/* If the flow IPv4/IPv6 flow specifications are extended, the mask</div><div class='del'>-	 * should be changed as well.</div><div class='del'>-	 */</div><div class='del'>-	BUILD_BUG_ON(offsetof(struct ib_uverbs_flow_ipv4_filter, flags) +</div><div class='del'>-		     sizeof(ipv4.flags) != sizeof(ipv4));</div><div class='del'>-	BUILD_BUG_ON(offsetof(struct ib_uverbs_flow_ipv6_filter, reserved) +</div><div class='del'>-		     sizeof(ipv6.reserved) != sizeof(ipv6));</div><div class='del'>-</div><div class='del'>-	switch (proto) {</div><div class='del'>-	case IB_FLOW_SPEC_IPV4:</div><div class='del'>-		if (len &gt; sizeof(user_val.ipv4) &amp;&amp;</div><div class='del'>-		    !ib_is_buffer_cleared(val_ptr + sizeof(user_val.ipv4),</div><div class='del'>-					  len - sizeof(user_val.ipv4)))</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-		val_len = min_t(size_t, len, sizeof(user_val.ipv4));</div><div class='del'>-		ret = copy_from_user(&amp;user_val.ipv4, val_ptr,</div><div class='del'>-				     val_len);</div><div class='del'>-		if (ret)</div><div class='del'>-			return -EFAULT;</div><div class='del'>-</div><div class='del'>-		user_pmask = &amp;ipv4;</div><div class='del'>-		break;</div><div class='del'>-	case IB_FLOW_SPEC_IPV6:</div><div class='del'>-		if (len &gt; sizeof(user_val.ipv6) &amp;&amp;</div><div class='del'>-		    !ib_is_buffer_cleared(val_ptr + sizeof(user_val.ipv6),</div><div class='del'>-					  len - sizeof(user_val.ipv6)))</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-		val_len = min_t(size_t, len, sizeof(user_val.ipv6));</div><div class='del'>-		ret = copy_from_user(&amp;user_val.ipv6, val_ptr,</div><div class='del'>-				     val_len);</div><div class='del'>-		if (ret)</div><div class='del'>-			return -EFAULT;</div><div class='del'>-</div><div class='del'>-		user_pmask = &amp;ipv6;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ib_uverbs_kern_spec_to_ib_spec_filter(proto, user_pmask,</div><div class='del'>-						     &amp;user_val,</div><div class='del'>-						     val_len, out);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int flow_action_esp_get_encap(struct ib_flow_spec_list *out,</div><div class='del'>-				     struct uverbs_attr_bundle *attrs)</div><div class='del'>-{</div><div class='del'>-	struct ib_uverbs_flow_action_esp_encap uverbs_encap;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	ret = uverbs_copy_from(&amp;uverbs_encap, attrs,</div><div class='del'>-			       UVERBS_ATTR_FLOW_ACTION_ESP_ENCAP);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	/* We currently support only one encap */</div><div class='del'>-	if (uverbs_encap.next_ptr)</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-	if (uverbs_encap.type != IB_FLOW_SPEC_IPV4 &amp;&amp;</div><div class='del'>-	    uverbs_encap.type != IB_FLOW_SPEC_IPV6)</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-	return parse_esp_ip(uverbs_encap.type,</div><div class='del'>-			    u64_to_user_ptr(uverbs_encap.val_ptr),</div><div class='del'>-			    uverbs_encap.len,</div><div class='del'>-			    &amp;out-&gt;spec);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct ib_flow_action_esp_attr {</div><div class='del'>-	struct	ib_flow_action_attrs_esp		hdr;</div><div class='del'>-	struct	ib_flow_action_attrs_esp_keymats	keymat;</div><div class='del'>-	struct	ib_flow_action_attrs_esp_replays	replay;</div><div class='del'>-	/* We currently support only one spec */</div><div class='del'>-	struct	ib_flow_spec_list			encap;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define ESP_LAST_SUPPORTED_FLAG		IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW</div><div class='del'>-static int parse_flow_action_esp(struct ib_device *ib_dev,</div><div class='del'>-				 struct uverbs_attr_bundle *attrs,</div><div class='del'>-				 struct ib_flow_action_esp_attr *esp_attr,</div><div class='del'>-				 bool is_modify)</div><div class='del'>-{</div><div class='del'>-	struct ib_uverbs_flow_action_esp uverbs_esp = {};</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	/* Optional param, if it doesn't exist, we get -ENOENT and skip it */</div><div class='del'>-	ret = uverbs_copy_from(&amp;esp_attr-&gt;hdr.esn, attrs,</div><div class='del'>-			       UVERBS_ATTR_FLOW_ACTION_ESP_ESN);</div><div class='del'>-	if (IS_UVERBS_COPY_ERR(ret))</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	/* This can be called from FLOW_ACTION_ESP_MODIFY where</div><div class='del'>-	 * UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS is optional</div><div class='del'>-	 */</div><div class='del'>-	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS)) {</div><div class='del'>-		ret = uverbs_copy_from_or_zero(&amp;uverbs_esp, attrs,</div><div class='del'>-					       UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS);</div><div class='del'>-		if (ret)</div><div class='del'>-			return ret;</div><div class='del'>-</div><div class='del'>-		if (uverbs_esp.flags &amp; ~((ESP_LAST_SUPPORTED_FLAG &lt;&lt; 1) - 1))</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-		esp_attr-&gt;hdr.spi = uverbs_esp.spi;</div><div class='del'>-		esp_attr-&gt;hdr.seq = uverbs_esp.seq;</div><div class='del'>-		esp_attr-&gt;hdr.tfc_pad = uverbs_esp.tfc_pad;</div><div class='del'>-		esp_attr-&gt;hdr.hard_limit_pkts = uverbs_esp.hard_limit_pkts;</div><div class='del'>-	}</div><div class='del'>-	esp_attr-&gt;hdr.flags = esp_flags_uverbs_to_verbs(attrs, uverbs_esp.flags,</div><div class='del'>-							is_modify);</div><div class='del'>-</div><div class='del'>-	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT)) {</div><div class='del'>-		esp_attr-&gt;keymat.protocol =</div><div class='del'>-			uverbs_attr_get_enum_id(attrs,</div><div class='del'>-						UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT);</div><div class='del'>-		ret = uverbs_copy_from_or_zero(&amp;esp_attr-&gt;keymat.keymat,</div><div class='del'>-					       attrs,</div><div class='del'>-					       UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT);</div><div class='del'>-		if (ret)</div><div class='del'>-			return ret;</div><div class='del'>-</div><div class='del'>-		ret = flow_action_esp_keymat_validate[esp_attr-&gt;keymat.protocol](&amp;esp_attr-&gt;keymat);</div><div class='del'>-		if (ret)</div><div class='del'>-			return ret;</div><div class='del'>-</div><div class='del'>-		esp_attr-&gt;hdr.keymat = &amp;esp_attr-&gt;keymat;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_FLOW_ACTION_ESP_REPLAY)) {</div><div class='del'>-		esp_attr-&gt;replay.protocol =</div><div class='del'>-			uverbs_attr_get_enum_id(attrs,</div><div class='del'>-						UVERBS_ATTR_FLOW_ACTION_ESP_REPLAY);</div><div class='del'>-</div><div class='del'>-		ret = uverbs_copy_from_or_zero(&amp;esp_attr-&gt;replay.replay,</div><div class='del'>-					       attrs,</div><div class='del'>-					       UVERBS_ATTR_FLOW_ACTION_ESP_REPLAY);</div><div class='del'>-		if (ret)</div><div class='del'>-			return ret;</div><div class='del'>-</div><div class='del'>-		ret = flow_action_esp_replay_validate[esp_attr-&gt;replay.protocol](&amp;esp_attr-&gt;replay,</div><div class='del'>-										 is_modify);</div><div class='del'>-		if (ret)</div><div class='del'>-			return ret;</div><div class='del'>-</div><div class='del'>-		esp_attr-&gt;hdr.replay = &amp;esp_attr-&gt;replay;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_FLOW_ACTION_ESP_ENCAP)) {</div><div class='del'>-		ret = flow_action_esp_get_encap(&amp;esp_attr-&gt;encap, attrs);</div><div class='del'>-		if (ret)</div><div class='del'>-			return ret;</div><div class='del'>-</div><div class='del'>-		esp_attr-&gt;hdr.encap = &amp;esp_attr-&gt;encap;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int UVERBS_HANDLER(UVERBS_METHOD_FLOW_ACTION_ESP_CREATE)(</div><div class='del'>-	struct uverbs_attr_bundle *attrs)</div><div class='del'>-{</div><div class='del'>-	struct ib_uobject *uobj = uverbs_attr_get_uobject(</div><div class='del'>-		attrs, UVERBS_ATTR_CREATE_FLOW_ACTION_ESP_HANDLE);</div><div class='del'>-	struct ib_device *ib_dev = attrs-&gt;context-&gt;device;</div><div class='del'>-	int				  ret;</div><div class='del'>-	struct ib_flow_action		  *action;</div><div class='del'>-	struct ib_flow_action_esp_attr	  esp_attr = {};</div><div class='del'>-</div><div class='del'>-	if (!ib_dev-&gt;ops.create_flow_action_esp)</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-	ret = parse_flow_action_esp(ib_dev, attrs, &amp;esp_attr, false);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	/* No need to check as this attribute is marked as MANDATORY */</div><div class='del'>-	action = ib_dev-&gt;ops.create_flow_action_esp(ib_dev, &amp;esp_attr.hdr,</div><div class='del'>-						    attrs);</div><div class='del'>-	if (IS_ERR(action))</div><div class='del'>-		return PTR_ERR(action);</div><div class='del'>-</div><div class='del'>-	uverbs_flow_action_fill_action(action, uobj, ib_dev,</div><div class='del'>-				       IB_FLOW_ACTION_ESP);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int UVERBS_HANDLER(UVERBS_METHOD_FLOW_ACTION_ESP_MODIFY)(</div><div class='del'>-	struct uverbs_attr_bundle *attrs)</div><div class='del'>-{</div><div class='del'>-	struct ib_uobject *uobj = uverbs_attr_get_uobject(</div><div class='del'>-		attrs, UVERBS_ATTR_MODIFY_FLOW_ACTION_ESP_HANDLE);</div><div class='del'>-	struct ib_flow_action *action = uobj-&gt;object;</div><div class='del'>-	int				  ret;</div><div class='del'>-	struct ib_flow_action_esp_attr	  esp_attr = {};</div><div class='del'>-</div><div class='del'>-	if (!action-&gt;device-&gt;ops.modify_flow_action_esp)</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-	ret = parse_flow_action_esp(action-&gt;device, attrs, &amp;esp_attr, true);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	if (action-&gt;type != IB_FLOW_ACTION_ESP)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	return action-&gt;device-&gt;ops.modify_flow_action_esp(action,</div><div class='del'>-							  &amp;esp_attr.hdr,</div><div class='del'>-							  attrs);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static const struct uverbs_attr_spec uverbs_flow_action_esp_keymat[] = {</div><div class='del'>-	[IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM] = {</div><div class='del'>-		.type = UVERBS_ATTR_TYPE_PTR_IN,</div><div class='del'>-		UVERBS_ATTR_STRUCT(</div><div class='del'>-			struct ib_uverbs_flow_action_esp_keymat_aes_gcm,</div><div class='del'>-			aes_key),</div><div class='del'>-	},</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static const struct uverbs_attr_spec uverbs_flow_action_esp_replay[] = {</div><div class='del'>-	[IB_UVERBS_FLOW_ACTION_ESP_REPLAY_NONE] = {</div><div class='del'>-		.type = UVERBS_ATTR_TYPE_PTR_IN,</div><div class='del'>-		UVERBS_ATTR_NO_DATA(),</div><div class='del'>-	},</div><div class='del'>-	[IB_UVERBS_FLOW_ACTION_ESP_REPLAY_BMP] = {</div><div class='del'>-		.type = UVERBS_ATTR_TYPE_PTR_IN,</div><div class='del'>-		UVERBS_ATTR_STRUCT(struct ib_uverbs_flow_action_esp_replay_bmp,</div><div class='del'>-				   size),</div><div class='del'>-	},</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-DECLARE_UVERBS_NAMED_METHOD(</div><div class='del'>-	UVERBS_METHOD_FLOW_ACTION_ESP_CREATE,</div><div class='del'>-	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_FLOW_ACTION_ESP_HANDLE,</div><div class='del'>-			UVERBS_OBJECT_FLOW_ACTION,</div><div class='del'>-			UVERBS_ACCESS_NEW,</div><div class='del'>-			UA_MANDATORY),</div><div class='del'>-	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS,</div><div class='del'>-			   UVERBS_ATTR_STRUCT(struct ib_uverbs_flow_action_esp,</div><div class='del'>-					      hard_limit_pkts),</div><div class='del'>-			   UA_MANDATORY),</div><div class='del'>-	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_FLOW_ACTION_ESP_ESN,</div><div class='del'>-			   UVERBS_ATTR_TYPE(__u32),</div><div class='del'>-			   UA_OPTIONAL),</div><div class='del'>-	UVERBS_ATTR_ENUM_IN(UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT,</div><div class='del'>-			    uverbs_flow_action_esp_keymat,</div><div class='del'>-			    UA_MANDATORY),</div><div class='del'>-	UVERBS_ATTR_ENUM_IN(UVERBS_ATTR_FLOW_ACTION_ESP_REPLAY,</div><div class='del'>-			    uverbs_flow_action_esp_replay,</div><div class='del'>-			    UA_OPTIONAL),</div><div class='del'>-	UVERBS_ATTR_PTR_IN(</div><div class='del'>-		UVERBS_ATTR_FLOW_ACTION_ESP_ENCAP,</div><div class='del'>-		UVERBS_ATTR_TYPE(struct ib_uverbs_flow_action_esp_encap),</div><div class='del'>-		UA_OPTIONAL));</div><div class='del'>-</div><div class='del'>-DECLARE_UVERBS_NAMED_METHOD(</div><div class='del'>-	UVERBS_METHOD_FLOW_ACTION_ESP_MODIFY,</div><div class='del'>-	UVERBS_ATTR_IDR(UVERBS_ATTR_MODIFY_FLOW_ACTION_ESP_HANDLE,</div><div class='del'>-			UVERBS_OBJECT_FLOW_ACTION,</div><div class='del'>-			UVERBS_ACCESS_WRITE,</div><div class='del'>-			UA_MANDATORY),</div><div class='del'>-	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS,</div><div class='del'>-			   UVERBS_ATTR_STRUCT(struct ib_uverbs_flow_action_esp,</div><div class='del'>-					      hard_limit_pkts),</div><div class='del'>-			   UA_OPTIONAL),</div><div class='del'>-	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_FLOW_ACTION_ESP_ESN,</div><div class='del'>-			   UVERBS_ATTR_TYPE(__u32),</div><div class='del'>-			   UA_OPTIONAL),</div><div class='del'>-	UVERBS_ATTR_ENUM_IN(UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT,</div><div class='del'>-			    uverbs_flow_action_esp_keymat,</div><div class='del'>-			    UA_OPTIONAL),</div><div class='del'>-	UVERBS_ATTR_ENUM_IN(UVERBS_ATTR_FLOW_ACTION_ESP_REPLAY,</div><div class='del'>-			    uverbs_flow_action_esp_replay,</div><div class='del'>-			    UA_OPTIONAL),</div><div class='del'>-	UVERBS_ATTR_PTR_IN(</div><div class='del'>-		UVERBS_ATTR_FLOW_ACTION_ESP_ENCAP,</div><div class='del'>-		UVERBS_ATTR_TYPE(struct ib_uverbs_flow_action_esp_encap),</div><div class='del'>-		UA_OPTIONAL));</div><div class='del'>-</div><div class='ctx'> DECLARE_UVERBS_NAMED_METHOD_DESTROY(</div><div class='ctx'> 	UVERBS_METHOD_FLOW_ACTION_DESTROY,</div><div class='ctx'> 	UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_FLOW_ACTION_HANDLE,</div><div class='hunk'>@@ -437,9 +56,7 @@ DECLARE_UVERBS_NAMED_METHOD_DESTROY(</div><div class='ctx'> DECLARE_UVERBS_NAMED_OBJECT(</div><div class='ctx'> 	UVERBS_OBJECT_FLOW_ACTION,</div><div class='ctx'> 	UVERBS_TYPE_ALLOC_IDR(uverbs_free_flow_action),</div><div class='del'>-	&amp;UVERBS_METHOD(UVERBS_METHOD_FLOW_ACTION_ESP_CREATE),</div><div class='del'>-	&amp;UVERBS_METHOD(UVERBS_METHOD_FLOW_ACTION_DESTROY),</div><div class='del'>-	&amp;UVERBS_METHOD(UVERBS_METHOD_FLOW_ACTION_ESP_MODIFY));</div><div class='add'>+	&amp;UVERBS_METHOD(UVERBS_METHOD_FLOW_ACTION_DESTROY));</div><div class='ctx'> </div><div class='ctx'> const struct uapi_definition uverbs_def_obj_flow_action[] = {</div><div class='ctx'> 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_std_types_mr.c b/drivers/infiniband/core/uverbs_std_types_mr.c<br/>index 610d3b9f7654..570b9656801d 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_mr.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_std_types_mr.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_mr.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_std_types_mr.c</a></div><div class='hunk'>@@ -1,5 +1,6 @@</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.</div><div class='add'>+ * Copyright (c) 2020, Intel Corporation.  All rights reserved.</div><div class='ctx'>  *</div><div class='ctx'>  * This software is available to you under a choice of one of two</div><div class='ctx'>  * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='hunk'>@@ -33,6 +34,7 @@</div><div class='ctx'> #include "rdma_core.h"</div><div class='ctx'> #include "uverbs.h"</div><div class='ctx'> #include &lt;rdma/uverbs_std_types.h&gt;</div><div class='add'>+#include "restrack.h"</div><div class='ctx'> </div><div class='ctx'> static int uverbs_free_mr(struct ib_uobject *uobject,</div><div class='ctx'> 			  enum rdma_remove_reason why,</div><div class='hunk'>@@ -69,7 +71,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_ADVISE_MR)(</div><div class='ctx'> </div><div class='ctx'> 	num_sge = uverbs_attr_ptr_get_array_size(</div><div class='ctx'> 		attrs, UVERBS_ATTR_ADVISE_MR_SGE_LIST, sizeof(struct ib_sge));</div><div class='del'>-	if (num_sge &lt; 0)</div><div class='add'>+	if (num_sge &lt;= 0)</div><div class='ctx'> 		return num_sge;</div><div class='ctx'> </div><div class='ctx'> 	sg_list = uverbs_attr_get_alloced_ptr(attrs,</div><div class='hunk'>@@ -114,7 +116,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DM_MR_REG)(</div><div class='ctx'> 	if (!(attr.access_flags &amp; IB_ZERO_BASED))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	ret = ib_check_mr_access(attr.access_flags);</div><div class='add'>+	ret = ib_check_mr_access(ib_dev, attr.access_flags);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='hunk'>@@ -128,28 +130,268 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DM_MR_REG)(</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;device  = pd-&gt;device;</div><div class='ctx'> 	mr-&gt;pd      = pd;</div><div class='add'>+	mr-&gt;type    = IB_MR_TYPE_DM;</div><div class='ctx'> 	mr-&gt;dm      = dm;</div><div class='ctx'> 	mr-&gt;uobject = uobj;</div><div class='ctx'> 	atomic_inc(&amp;pd-&gt;usecnt);</div><div class='ctx'> 	atomic_inc(&amp;dm-&gt;usecnt);</div><div class='ctx'> </div><div class='add'>+	rdma_restrack_new(&amp;mr-&gt;res, RDMA_RESTRACK_MR);</div><div class='add'>+	rdma_restrack_set_name(&amp;mr-&gt;res, NULL);</div><div class='add'>+	rdma_restrack_add(&amp;mr-&gt;res);</div><div class='ctx'> 	uobj-&gt;object = mr;</div><div class='ctx'> </div><div class='add'>+	uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_REG_DM_MR_HANDLE);</div><div class='add'>+</div><div class='ctx'> 	ret = uverbs_copy_to(attrs, UVERBS_ATTR_REG_DM_MR_RESP_LKEY, &amp;mr-&gt;lkey,</div><div class='ctx'> 			     sizeof(mr-&gt;lkey));</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err_dereg;</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='ctx'> 	ret = uverbs_copy_to(attrs, UVERBS_ATTR_REG_DM_MR_RESP_RKEY,</div><div class='ctx'> 			     &amp;mr-&gt;rkey, sizeof(mr-&gt;rkey));</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_MR)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_mr *mr =</div><div class='add'>+		uverbs_attr_get_obj(attrs, UVERBS_ATTR_QUERY_MR_HANDLE);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_MR_RESP_LKEY, &amp;mr-&gt;lkey,</div><div class='add'>+			     sizeof(mr-&gt;lkey));</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_MR_RESP_RKEY,</div><div class='add'>+			     &amp;mr-&gt;rkey, sizeof(mr-&gt;rkey));</div><div class='add'>+</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_MR_RESP_LENGTH,</div><div class='add'>+			     &amp;mr-&gt;length, sizeof(mr-&gt;length));</div><div class='add'>+</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_MR_RESP_IOVA,</div><div class='add'>+			     &amp;mr-&gt;iova, sizeof(mr-&gt;iova));</div><div class='add'>+</div><div class='add'>+	return IS_UVERBS_COPY_ERR(ret) ? ret : 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(UVERBS_METHOD_REG_DMABUF_MR)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *uobj =</div><div class='add'>+		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_REG_DMABUF_MR_HANDLE);</div><div class='add'>+	struct ib_pd *pd =</div><div class='add'>+		uverbs_attr_get_obj(attrs, UVERBS_ATTR_REG_DMABUF_MR_PD_HANDLE);</div><div class='add'>+	struct ib_device *ib_dev = pd-&gt;device;</div><div class='add'>+</div><div class='add'>+	u64 offset, length, iova;</div><div class='add'>+	u32 fd, access_flags;</div><div class='add'>+	struct ib_mr *mr;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!ib_dev-&gt;ops.reg_user_mr_dmabuf)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_from(&amp;offset, attrs,</div><div class='add'>+			       UVERBS_ATTR_REG_DMABUF_MR_OFFSET);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_from(&amp;length, attrs,</div><div class='add'>+			       UVERBS_ATTR_REG_DMABUF_MR_LENGTH);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_from(&amp;iova, attrs,</div><div class='add'>+			       UVERBS_ATTR_REG_DMABUF_MR_IOVA);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if ((offset &amp; ~PAGE_MASK) != (iova &amp; ~PAGE_MASK))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_from(&amp;fd, attrs,</div><div class='add'>+			       UVERBS_ATTR_REG_DMABUF_MR_FD);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_get_flags32(&amp;access_flags, attrs,</div><div class='add'>+				 UVERBS_ATTR_REG_DMABUF_MR_ACCESS_FLAGS,</div><div class='add'>+				 IB_ACCESS_LOCAL_WRITE |</div><div class='add'>+				 IB_ACCESS_REMOTE_READ |</div><div class='add'>+				 IB_ACCESS_REMOTE_WRITE |</div><div class='add'>+				 IB_ACCESS_REMOTE_ATOMIC |</div><div class='add'>+				 IB_ACCESS_RELAXED_ORDERING);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = ib_check_mr_access(ib_dev, access_flags);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	mr = pd-&gt;device-&gt;ops.reg_user_mr_dmabuf(pd, offset, length, iova, fd,</div><div class='add'>+						access_flags, NULL,</div><div class='add'>+						attrs);</div><div class='add'>+	if (IS_ERR(mr))</div><div class='add'>+		return PTR_ERR(mr);</div><div class='add'>+</div><div class='add'>+	mr-&gt;device = pd-&gt;device;</div><div class='add'>+	mr-&gt;pd = pd;</div><div class='add'>+	mr-&gt;type = IB_MR_TYPE_USER;</div><div class='add'>+	mr-&gt;uobject = uobj;</div><div class='add'>+	atomic_inc(&amp;pd-&gt;usecnt);</div><div class='add'>+</div><div class='add'>+	rdma_restrack_new(&amp;mr-&gt;res, RDMA_RESTRACK_MR);</div><div class='add'>+	rdma_restrack_set_name(&amp;mr-&gt;res, NULL);</div><div class='add'>+	rdma_restrack_add(&amp;mr-&gt;res);</div><div class='add'>+	uobj-&gt;object = mr;</div><div class='add'>+</div><div class='add'>+	uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_REG_DMABUF_MR_HANDLE);</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_REG_DMABUF_MR_RESP_LKEY,</div><div class='add'>+			     &amp;mr-&gt;lkey, sizeof(mr-&gt;lkey));</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_REG_DMABUF_MR_RESP_RKEY,</div><div class='add'>+			     &amp;mr-&gt;rkey, sizeof(mr-&gt;rkey));</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(UVERBS_METHOD_REG_MR)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *uobj =</div><div class='add'>+		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_REG_MR_HANDLE);</div><div class='add'>+	struct ib_pd *pd =</div><div class='add'>+		uverbs_attr_get_obj(attrs, UVERBS_ATTR_REG_MR_PD_HANDLE);</div><div class='add'>+	u32 valid_access_flags = IB_ACCESS_SUPPORTED;</div><div class='add'>+	u64 length, iova, fd_offset = 0, addr = 0;</div><div class='add'>+	struct ib_device *ib_dev = pd-&gt;device;</div><div class='add'>+	struct ib_dmah *dmah = NULL;</div><div class='add'>+	bool has_fd_offset = false;</div><div class='add'>+	bool has_addr = false;</div><div class='add'>+	bool has_fd = false;</div><div class='add'>+	u32 access_flags;</div><div class='add'>+	struct ib_mr *mr;</div><div class='add'>+	int fd;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_from(&amp;iova, attrs, UVERBS_ATTR_REG_MR_IOVA);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err_dereg;</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_from(&amp;length, attrs, UVERBS_ATTR_REG_MR_LENGTH);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_REG_MR_ADDR)) {</div><div class='add'>+		ret = uverbs_copy_from(&amp;addr, attrs,</div><div class='add'>+				       UVERBS_ATTR_REG_MR_ADDR);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+		has_addr = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_REG_MR_FD_OFFSET)) {</div><div class='add'>+		ret = uverbs_copy_from(&amp;fd_offset, attrs,</div><div class='add'>+				       UVERBS_ATTR_REG_MR_FD_OFFSET);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+		has_fd_offset = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_REG_MR_FD)) {</div><div class='add'>+		ret = uverbs_get_raw_fd(&amp;fd, attrs,</div><div class='add'>+					UVERBS_ATTR_REG_MR_FD);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+		has_fd = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (has_fd) {</div><div class='add'>+		if (!ib_dev-&gt;ops.reg_user_mr_dmabuf)</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+		/* FD requires offset and can't come with addr */</div><div class='add'>+		if (!has_fd_offset || has_addr)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		if ((fd_offset &amp; ~PAGE_MASK) != (iova &amp; ~PAGE_MASK))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		valid_access_flags = IB_ACCESS_LOCAL_WRITE |</div><div class='add'>+				     IB_ACCESS_REMOTE_READ |</div><div class='add'>+				     IB_ACCESS_REMOTE_WRITE |</div><div class='add'>+				     IB_ACCESS_REMOTE_ATOMIC |</div><div class='add'>+				     IB_ACCESS_RELAXED_ORDERING;</div><div class='add'>+	} else {</div><div class='add'>+		if (!has_addr || has_fd_offset)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		if ((addr &amp; ~PAGE_MASK) != (iova &amp; ~PAGE_MASK))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_REG_MR_DMA_HANDLE)) {</div><div class='add'>+		dmah = uverbs_attr_get_obj(attrs,</div><div class='add'>+					   UVERBS_ATTR_REG_MR_DMA_HANDLE);</div><div class='add'>+		if (IS_ERR(dmah))</div><div class='add'>+			return PTR_ERR(dmah);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = uverbs_get_flags32(&amp;access_flags, attrs,</div><div class='add'>+				 UVERBS_ATTR_REG_MR_ACCESS_FLAGS,</div><div class='add'>+				 valid_access_flags);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = ib_check_mr_access(ib_dev, access_flags);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (has_fd)</div><div class='add'>+		mr = pd-&gt;device-&gt;ops.reg_user_mr_dmabuf(pd, fd_offset, length,</div><div class='add'>+							iova, fd, access_flags,</div><div class='add'>+							dmah, attrs);</div><div class='add'>+	else</div><div class='add'>+		mr = pd-&gt;device-&gt;ops.reg_user_mr(pd, addr, length, iova,</div><div class='add'>+						 access_flags, dmah, NULL);</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	if (IS_ERR(mr))</div><div class='add'>+		return PTR_ERR(mr);</div><div class='add'>+</div><div class='add'>+	mr-&gt;device = pd-&gt;device;</div><div class='add'>+	mr-&gt;pd = pd;</div><div class='add'>+	mr-&gt;type = IB_MR_TYPE_USER;</div><div class='add'>+	mr-&gt;uobject = uobj;</div><div class='add'>+	atomic_inc(&amp;pd-&gt;usecnt);</div><div class='add'>+	if (dmah) {</div><div class='add'>+		mr-&gt;dmah = dmah;</div><div class='add'>+		atomic_inc(&amp;dmah-&gt;usecnt);</div><div class='add'>+	}</div><div class='add'>+	rdma_restrack_new(&amp;mr-&gt;res, RDMA_RESTRACK_MR);</div><div class='add'>+	rdma_restrack_set_name(&amp;mr-&gt;res, NULL);</div><div class='add'>+	rdma_restrack_add(&amp;mr-&gt;res);</div><div class='add'>+	uobj-&gt;object = mr;</div><div class='ctx'> </div><div class='del'>-err_dereg:</div><div class='del'>-	ib_dereg_mr_user(mr, &amp;attrs-&gt;driver_udata);</div><div class='add'>+	uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_REG_MR_HANDLE);</div><div class='ctx'> </div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_REG_MR_RESP_LKEY,</div><div class='add'>+			     &amp;mr-&gt;lkey, sizeof(mr-&gt;lkey));</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_REG_MR_RESP_RKEY,</div><div class='add'>+			     &amp;mr-&gt;rkey, sizeof(mr-&gt;rkey));</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -171,6 +413,25 @@ DECLARE_UVERBS_NAMED_METHOD(</div><div class='ctx'> 			   UA_ALLOC_AND_COPY));</div><div class='ctx'> </div><div class='ctx'> DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	UVERBS_METHOD_QUERY_MR,</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_QUERY_MR_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_MR,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_MR_RESP_RKEY,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			    UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_MR_RESP_LKEY,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			    UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_MR_RESP_LENGTH,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			    UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_MR_RESP_IOVA,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			    UA_OPTIONAL));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='ctx'> 	UVERBS_METHOD_DM_MR_REG,</div><div class='ctx'> 	UVERBS_ATTR_IDR(UVERBS_ATTR_REG_DM_MR_HANDLE,</div><div class='ctx'> 			UVERBS_OBJECT_MR,</div><div class='hunk'>@@ -199,6 +460,75 @@ DECLARE_UVERBS_NAMED_METHOD(</div><div class='ctx'> 			    UVERBS_ATTR_TYPE(u32),</div><div class='ctx'> 			    UA_MANDATORY));</div><div class='ctx'> </div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	UVERBS_METHOD_REG_DMABUF_MR,</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_REG_DMABUF_MR_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_MR,</div><div class='add'>+			UVERBS_ACCESS_NEW,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_REG_DMABUF_MR_PD_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_PD,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_REG_DMABUF_MR_OFFSET,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_REG_DMABUF_MR_LENGTH,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_REG_DMABUF_MR_IOVA,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_REG_DMABUF_MR_FD,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_REG_DMABUF_MR_ACCESS_FLAGS,</div><div class='add'>+			     enum ib_access_flags),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_REG_DMABUF_MR_RESP_LKEY,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			    UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_REG_DMABUF_MR_RESP_RKEY,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			    UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	UVERBS_METHOD_REG_MR,</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_REG_MR_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_MR,</div><div class='add'>+			UVERBS_ACCESS_NEW,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_REG_MR_PD_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_PD,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_REG_MR_DMA_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_DMAH,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_REG_MR_IOVA,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_REG_MR_LENGTH,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_REG_MR_ACCESS_FLAGS,</div><div class='add'>+			     enum ib_access_flags,</div><div class='add'>+			     UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_REG_MR_ADDR,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			   UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_REG_MR_FD_OFFSET,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			   UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_RAW_FD(UVERBS_ATTR_REG_MR_FD,</div><div class='add'>+			   UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_REG_MR_RESP_LKEY,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			    UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_REG_MR_RESP_RKEY,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			    UA_MANDATORY));</div><div class='add'>+</div><div class='ctx'> DECLARE_UVERBS_NAMED_METHOD_DESTROY(</div><div class='ctx'> 	UVERBS_METHOD_MR_DESTROY,</div><div class='ctx'> 	UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_MR_HANDLE,</div><div class='hunk'>@@ -209,9 +539,12 @@ DECLARE_UVERBS_NAMED_METHOD_DESTROY(</div><div class='ctx'> DECLARE_UVERBS_NAMED_OBJECT(</div><div class='ctx'> 	UVERBS_OBJECT_MR,</div><div class='ctx'> 	UVERBS_TYPE_ALLOC_IDR(uverbs_free_mr),</div><div class='add'>+	&amp;UVERBS_METHOD(UVERBS_METHOD_ADVISE_MR),</div><div class='ctx'> 	&amp;UVERBS_METHOD(UVERBS_METHOD_DM_MR_REG),</div><div class='ctx'> 	&amp;UVERBS_METHOD(UVERBS_METHOD_MR_DESTROY),</div><div class='del'>-	&amp;UVERBS_METHOD(UVERBS_METHOD_ADVISE_MR));</div><div class='add'>+	&amp;UVERBS_METHOD(UVERBS_METHOD_QUERY_MR),</div><div class='add'>+	&amp;UVERBS_METHOD(UVERBS_METHOD_REG_DMABUF_MR),</div><div class='add'>+	&amp;UVERBS_METHOD(UVERBS_METHOD_REG_MR));</div><div class='ctx'> </div><div class='ctx'> const struct uapi_definition uverbs_def_obj_mr[] = {</div><div class='ctx'> 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_MR,</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_std_types_qp.c b/drivers/infiniband/core/uverbs_std_types_qp.c<br/>new file mode 100644<br/>index 000000000000..be0730e8509e<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_qp.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_std_types_qp.c</a></div><div class='hunk'>@@ -0,0 +1,380 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2020, Mellanox Technologies inc.  All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/uverbs_std_types.h&gt;</div><div class='add'>+#include "rdma_core.h"</div><div class='add'>+#include "uverbs.h"</div><div class='add'>+#include "core_priv.h"</div><div class='add'>+</div><div class='add'>+static int uverbs_free_qp(struct ib_uobject *uobject,</div><div class='add'>+			  enum rdma_remove_reason why,</div><div class='add'>+			  struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_qp *qp = uobject-&gt;object;</div><div class='add'>+	struct ib_uqp_object *uqp =</div><div class='add'>+		container_of(uobject, struct ib_uqp_object, uevent.uobject);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * If this is a user triggered destroy then do not allow destruction</div><div class='add'>+	 * until the user cleans up all the mcast bindings. Unlike in other</div><div class='add'>+	 * places we forcibly clean up the mcast attachments for !DESTROY</div><div class='add'>+	 * because the mcast attaches are not ubojects and will not be</div><div class='add'>+	 * destroyed by anything else during cleanup processing.</div><div class='add'>+	 */</div><div class='add'>+	if (why == RDMA_REMOVE_DESTROY) {</div><div class='add'>+		if (!list_empty(&amp;uqp-&gt;mcast_list))</div><div class='add'>+			return -EBUSY;</div><div class='add'>+	} else if (qp == qp-&gt;real_qp) {</div><div class='add'>+		ib_uverbs_detach_umcast(qp, uqp);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = ib_destroy_qp_user(qp, &amp;attrs-&gt;driver_udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (uqp-&gt;uxrcd)</div><div class='add'>+		atomic_dec(&amp;uqp-&gt;uxrcd-&gt;refcnt);</div><div class='add'>+</div><div class='add'>+	ib_uverbs_release_uevent(&amp;uqp-&gt;uevent);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int check_creation_flags(enum ib_qp_type qp_type,</div><div class='add'>+				u32 create_flags)</div><div class='add'>+{</div><div class='add'>+	create_flags &amp;= ~IB_UVERBS_QP_CREATE_SQ_SIG_ALL;</div><div class='add'>+</div><div class='add'>+	if (!create_flags || qp_type == IB_QPT_DRIVER)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (qp_type != IB_QPT_RAW_PACKET &amp;&amp; qp_type != IB_QPT_UD)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if ((create_flags &amp; IB_UVERBS_QP_CREATE_SCATTER_FCS ||</div><div class='add'>+	     create_flags &amp; IB_UVERBS_QP_CREATE_CVLAN_STRIPPING) &amp;&amp;</div><div class='add'>+	     qp_type != IB_QPT_RAW_PACKET)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void set_caps(struct ib_qp_init_attr *attr,</div><div class='add'>+		     struct ib_uverbs_qp_cap *cap, bool req)</div><div class='add'>+{</div><div class='add'>+	if (req) {</div><div class='add'>+		attr-&gt;cap.max_send_wr = cap-&gt;max_send_wr;</div><div class='add'>+		attr-&gt;cap.max_recv_wr = cap-&gt;max_recv_wr;</div><div class='add'>+		attr-&gt;cap.max_send_sge = cap-&gt;max_send_sge;</div><div class='add'>+		attr-&gt;cap.max_recv_sge = cap-&gt;max_recv_sge;</div><div class='add'>+		attr-&gt;cap.max_inline_data = cap-&gt;max_inline_data;</div><div class='add'>+	} else {</div><div class='add'>+		cap-&gt;max_send_wr = attr-&gt;cap.max_send_wr;</div><div class='add'>+		cap-&gt;max_recv_wr = attr-&gt;cap.max_recv_wr;</div><div class='add'>+		cap-&gt;max_send_sge = attr-&gt;cap.max_send_sge;</div><div class='add'>+		cap-&gt;max_recv_sge = attr-&gt;cap.max_recv_sge;</div><div class='add'>+		cap-&gt;max_inline_data = attr-&gt;cap.max_inline_data;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(UVERBS_METHOD_QP_CREATE)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uqp_object *obj = container_of(</div><div class='add'>+		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_CREATE_QP_HANDLE),</div><div class='add'>+		typeof(*obj), uevent.uobject);</div><div class='add'>+	struct ib_qp_init_attr attr = {};</div><div class='add'>+	struct ib_uverbs_qp_cap cap = {};</div><div class='add'>+	struct ib_rwq_ind_table *rwq_ind_tbl = NULL;</div><div class='add'>+	struct ib_qp *qp;</div><div class='add'>+	struct ib_pd *pd = NULL;</div><div class='add'>+	struct ib_srq *srq = NULL;</div><div class='add'>+	struct ib_cq *recv_cq = NULL;</div><div class='add'>+	struct ib_cq *send_cq = NULL;</div><div class='add'>+	struct ib_xrcd *xrcd = NULL;</div><div class='add'>+	struct ib_uobject *xrcd_uobj = NULL;</div><div class='add'>+	struct ib_device *device;</div><div class='add'>+	u64 user_handle;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_from_or_zero(&amp;cap, attrs,</div><div class='add'>+			       UVERBS_ATTR_CREATE_QP_CAP);</div><div class='add'>+	if (!ret)</div><div class='add'>+		ret = uverbs_copy_from(&amp;user_handle, attrs,</div><div class='add'>+				       UVERBS_ATTR_CREATE_QP_USER_HANDLE);</div><div class='add'>+	if (!ret)</div><div class='add'>+		ret = uverbs_get_const(&amp;attr.qp_type, attrs,</div><div class='add'>+				       UVERBS_ATTR_CREATE_QP_TYPE);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	switch (attr.qp_type) {</div><div class='add'>+	case IB_QPT_XRC_TGT:</div><div class='add'>+		if (uverbs_attr_is_valid(attrs,</div><div class='add'>+				UVERBS_ATTR_CREATE_QP_RECV_CQ_HANDLE) ||</div><div class='add'>+		    uverbs_attr_is_valid(attrs,</div><div class='add'>+				UVERBS_ATTR_CREATE_QP_SEND_CQ_HANDLE) ||</div><div class='add'>+		    uverbs_attr_is_valid(attrs,</div><div class='add'>+				UVERBS_ATTR_CREATE_QP_PD_HANDLE) ||</div><div class='add'>+		    uverbs_attr_is_valid(attrs,</div><div class='add'>+				UVERBS_ATTR_CREATE_QP_IND_TABLE_HANDLE))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		xrcd_uobj = uverbs_attr_get_uobject(attrs,</div><div class='add'>+					UVERBS_ATTR_CREATE_QP_XRCD_HANDLE);</div><div class='add'>+		if (IS_ERR(xrcd_uobj))</div><div class='add'>+			return PTR_ERR(xrcd_uobj);</div><div class='add'>+</div><div class='add'>+		xrcd = (struct ib_xrcd *)xrcd_uobj-&gt;object;</div><div class='add'>+		if (!xrcd)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		device = xrcd-&gt;device;</div><div class='add'>+		break;</div><div class='add'>+	case IB_UVERBS_QPT_RAW_PACKET:</div><div class='add'>+		if (!rdma_uattrs_has_raw_cap(attrs))</div><div class='add'>+			return -EPERM;</div><div class='add'>+		fallthrough;</div><div class='add'>+	case IB_UVERBS_QPT_RC:</div><div class='add'>+	case IB_UVERBS_QPT_UC:</div><div class='add'>+	case IB_UVERBS_QPT_UD:</div><div class='add'>+	case IB_UVERBS_QPT_XRC_INI:</div><div class='add'>+	case IB_UVERBS_QPT_DRIVER:</div><div class='add'>+		if (uverbs_attr_is_valid(attrs,</div><div class='add'>+					 UVERBS_ATTR_CREATE_QP_XRCD_HANDLE) ||</div><div class='add'>+		   (uverbs_attr_is_valid(attrs,</div><div class='add'>+					 UVERBS_ATTR_CREATE_QP_SRQ_HANDLE) &amp;&amp;</div><div class='add'>+			attr.qp_type == IB_QPT_XRC_INI))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		pd = uverbs_attr_get_obj(attrs,</div><div class='add'>+					 UVERBS_ATTR_CREATE_QP_PD_HANDLE);</div><div class='add'>+		if (IS_ERR(pd))</div><div class='add'>+			return PTR_ERR(pd);</div><div class='add'>+</div><div class='add'>+		rwq_ind_tbl = uverbs_attr_get_obj(attrs,</div><div class='add'>+			UVERBS_ATTR_CREATE_QP_IND_TABLE_HANDLE);</div><div class='add'>+		if (!IS_ERR(rwq_ind_tbl)) {</div><div class='add'>+			if (cap.max_recv_wr || cap.max_recv_sge ||</div><div class='add'>+			    uverbs_attr_is_valid(attrs,</div><div class='add'>+				UVERBS_ATTR_CREATE_QP_RECV_CQ_HANDLE) ||</div><div class='add'>+			    uverbs_attr_is_valid(attrs,</div><div class='add'>+					UVERBS_ATTR_CREATE_QP_SRQ_HANDLE))</div><div class='add'>+				return -EINVAL;</div><div class='add'>+</div><div class='add'>+			/* send_cq is optional */</div><div class='add'>+			if (cap.max_send_wr) {</div><div class='add'>+				send_cq = uverbs_attr_get_obj(attrs,</div><div class='add'>+					UVERBS_ATTR_CREATE_QP_SEND_CQ_HANDLE);</div><div class='add'>+				if (IS_ERR(send_cq))</div><div class='add'>+					return PTR_ERR(send_cq);</div><div class='add'>+			}</div><div class='add'>+			attr.rwq_ind_tbl = rwq_ind_tbl;</div><div class='add'>+		} else {</div><div class='add'>+			send_cq = uverbs_attr_get_obj(attrs,</div><div class='add'>+					UVERBS_ATTR_CREATE_QP_SEND_CQ_HANDLE);</div><div class='add'>+			if (IS_ERR(send_cq))</div><div class='add'>+				return PTR_ERR(send_cq);</div><div class='add'>+</div><div class='add'>+			if (attr.qp_type != IB_QPT_XRC_INI) {</div><div class='add'>+				recv_cq = uverbs_attr_get_obj(attrs,</div><div class='add'>+					UVERBS_ATTR_CREATE_QP_RECV_CQ_HANDLE);</div><div class='add'>+				if (IS_ERR(recv_cq))</div><div class='add'>+					return PTR_ERR(recv_cq);</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		device = pd-&gt;device;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = uverbs_get_flags32(&amp;attr.create_flags, attrs,</div><div class='add'>+			 UVERBS_ATTR_CREATE_QP_FLAGS,</div><div class='add'>+			 IB_UVERBS_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |</div><div class='add'>+			 IB_UVERBS_QP_CREATE_SCATTER_FCS |</div><div class='add'>+			 IB_UVERBS_QP_CREATE_CVLAN_STRIPPING |</div><div class='add'>+			 IB_UVERBS_QP_CREATE_PCI_WRITE_END_PADDING |</div><div class='add'>+			 IB_UVERBS_QP_CREATE_SQ_SIG_ALL);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = check_creation_flags(attr.qp_type, attr.create_flags);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (uverbs_attr_is_valid(attrs,</div><div class='add'>+			UVERBS_ATTR_CREATE_QP_SOURCE_QPN)) {</div><div class='add'>+		ret = uverbs_copy_from(&amp;attr.source_qpn, attrs,</div><div class='add'>+				       UVERBS_ATTR_CREATE_QP_SOURCE_QPN);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+		attr.create_flags |= IB_QP_CREATE_SOURCE_QPN;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	srq = uverbs_attr_get_obj(attrs,</div><div class='add'>+				  UVERBS_ATTR_CREATE_QP_SRQ_HANDLE);</div><div class='add'>+	if (!IS_ERR(srq)) {</div><div class='add'>+		if ((srq-&gt;srq_type == IB_SRQT_XRC &amp;&amp;</div><div class='add'>+			attr.qp_type != IB_QPT_XRC_TGT) ||</div><div class='add'>+		    (srq-&gt;srq_type != IB_SRQT_XRC &amp;&amp;</div><div class='add'>+			attr.qp_type == IB_QPT_XRC_TGT))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		attr.srq = srq;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	obj-&gt;uevent.event_file = ib_uverbs_get_async_event(attrs,</div><div class='add'>+					UVERBS_ATTR_CREATE_QP_EVENT_FD);</div><div class='add'>+	INIT_LIST_HEAD(&amp;obj-&gt;uevent.event_list);</div><div class='add'>+	INIT_LIST_HEAD(&amp;obj-&gt;mcast_list);</div><div class='add'>+	obj-&gt;uevent.uobject.user_handle = user_handle;</div><div class='add'>+	attr.event_handler = ib_uverbs_qp_event_handler;</div><div class='add'>+	attr.send_cq = send_cq;</div><div class='add'>+	attr.recv_cq = recv_cq;</div><div class='add'>+	attr.xrcd = xrcd;</div><div class='add'>+	if (attr.create_flags &amp; IB_UVERBS_QP_CREATE_SQ_SIG_ALL) {</div><div class='add'>+		/* This creation bit is uverbs one, need to mask before</div><div class='add'>+		 * calling drivers. It was added to prevent an extra user attr</div><div class='add'>+		 * only for that when using ioctl.</div><div class='add'>+		 */</div><div class='add'>+		attr.create_flags &amp;= ~IB_UVERBS_QP_CREATE_SQ_SIG_ALL;</div><div class='add'>+		attr.sq_sig_type = IB_SIGNAL_ALL_WR;</div><div class='add'>+	} else {</div><div class='add'>+		attr.sq_sig_type = IB_SIGNAL_REQ_WR;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	set_caps(&amp;attr, &amp;cap, true);</div><div class='add'>+	mutex_init(&amp;obj-&gt;mcast_lock);</div><div class='add'>+</div><div class='add'>+	qp = ib_create_qp_user(device, pd, &amp;attr, &amp;attrs-&gt;driver_udata, obj,</div><div class='add'>+			       KBUILD_MODNAME);</div><div class='add'>+	if (IS_ERR(qp)) {</div><div class='add'>+		ret = PTR_ERR(qp);</div><div class='add'>+		goto err_put;</div><div class='add'>+	}</div><div class='add'>+	ib_qp_usecnt_inc(qp);</div><div class='add'>+</div><div class='add'>+	if (attr.qp_type == IB_QPT_XRC_TGT) {</div><div class='add'>+		obj-&gt;uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,</div><div class='add'>+					  uobject);</div><div class='add'>+		atomic_inc(&amp;obj-&gt;uxrcd-&gt;refcnt);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	obj-&gt;uevent.uobject.object = qp;</div><div class='add'>+	uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_CREATE_QP_HANDLE);</div><div class='add'>+</div><div class='add'>+	set_caps(&amp;attr, &amp;cap, false);</div><div class='add'>+	ret = uverbs_copy_to_struct_or_zero(attrs,</div><div class='add'>+					UVERBS_ATTR_CREATE_QP_RESP_CAP, &amp;cap,</div><div class='add'>+					sizeof(cap));</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_QP_RESP_QP_NUM,</div><div class='add'>+			     &amp;qp-&gt;qp_num,</div><div class='add'>+			     sizeof(qp-&gt;qp_num));</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+err_put:</div><div class='add'>+	if (obj-&gt;uevent.event_file)</div><div class='add'>+		uverbs_uobject_put(&amp;obj-&gt;uevent.event_file-&gt;uobj);</div><div class='add'>+	return ret;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	UVERBS_METHOD_QP_CREATE,</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_QP_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_QP,</div><div class='add'>+			UVERBS_ACCESS_NEW,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_QP_XRCD_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_XRCD,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_QP_PD_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_PD,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_QP_SRQ_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_SRQ,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_QP_SEND_CQ_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_CQ,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_QP_RECV_CQ_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_CQ,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_QP_IND_TABLE_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_RWQ_IND_TBL,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_QP_USER_HANDLE,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_QP_CAP,</div><div class='add'>+			   UVERBS_ATTR_STRUCT(struct ib_uverbs_qp_cap,</div><div class='add'>+					      max_inline_data),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_CONST_IN(UVERBS_ATTR_CREATE_QP_TYPE,</div><div class='add'>+			     enum ib_uverbs_qp_type,</div><div class='add'>+			     UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_CREATE_QP_FLAGS,</div><div class='add'>+			     enum ib_uverbs_qp_create_flags,</div><div class='add'>+			     UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_QP_SOURCE_QPN,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_FD(UVERBS_ATTR_CREATE_QP_EVENT_FD,</div><div class='add'>+		       UVERBS_OBJECT_ASYNC_EVENT,</div><div class='add'>+		       UVERBS_ACCESS_READ,</div><div class='add'>+		       UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_QP_RESP_CAP,</div><div class='add'>+			    UVERBS_ATTR_STRUCT(struct ib_uverbs_qp_cap,</div><div class='add'>+					       max_inline_data),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_QP_RESP_QP_NUM,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_UHW());</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(UVERBS_METHOD_QP_DESTROY)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *uobj =</div><div class='add'>+		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_DESTROY_QP_HANDLE);</div><div class='add'>+	struct ib_uqp_object *obj =</div><div class='add'>+		container_of(uobj, struct ib_uqp_object, uevent.uobject);</div><div class='add'>+	struct ib_uverbs_destroy_qp_resp resp = {</div><div class='add'>+		.events_reported = obj-&gt;uevent.events_reported</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	return uverbs_copy_to(attrs, UVERBS_ATTR_DESTROY_QP_RESP, &amp;resp,</div><div class='add'>+			      sizeof(resp));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	UVERBS_METHOD_QP_DESTROY,</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_QP_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_QP,</div><div class='add'>+			UVERBS_ACCESS_DESTROY,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_DESTROY_QP_RESP,</div><div class='add'>+			    UVERBS_ATTR_TYPE(struct ib_uverbs_destroy_qp_resp),</div><div class='add'>+			    UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_OBJECT(</div><div class='add'>+	UVERBS_OBJECT_QP,</div><div class='add'>+	UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uqp_object), uverbs_free_qp),</div><div class='add'>+	&amp;UVERBS_METHOD(UVERBS_METHOD_QP_CREATE),</div><div class='add'>+	&amp;UVERBS_METHOD(UVERBS_METHOD_QP_DESTROY));</div><div class='add'>+</div><div class='add'>+const struct uapi_definition uverbs_def_obj_qp[] = {</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_QP,</div><div class='add'>+				      UAPI_DEF_OBJ_NEEDS_FN(destroy_qp)),</div><div class='add'>+	{}</div><div class='add'>+};</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_std_types_srq.c b/drivers/infiniband/core/uverbs_std_types_srq.c<br/>new file mode 100644<br/>index 000000000000..e5513f828bdc<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_srq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_std_types_srq.c</a></div><div class='hunk'>@@ -0,0 +1,234 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2020, Mellanox Technologies inc.  All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/uverbs_std_types.h&gt;</div><div class='add'>+#include "rdma_core.h"</div><div class='add'>+#include "uverbs.h"</div><div class='add'>+</div><div class='add'>+static int uverbs_free_srq(struct ib_uobject *uobject,</div><div class='add'>+		    enum rdma_remove_reason why,</div><div class='add'>+		    struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_srq *srq = uobject-&gt;object;</div><div class='add'>+	struct ib_uevent_object *uevent =</div><div class='add'>+		container_of(uobject, struct ib_uevent_object, uobject);</div><div class='add'>+	enum ib_srq_type srq_type = srq-&gt;srq_type;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = ib_destroy_srq_user(srq, &amp;attrs-&gt;driver_udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (srq_type == IB_SRQT_XRC) {</div><div class='add'>+		struct ib_usrq_object *us =</div><div class='add'>+			container_of(uobject, struct ib_usrq_object,</div><div class='add'>+				     uevent.uobject);</div><div class='add'>+</div><div class='add'>+		atomic_dec(&amp;us-&gt;uxrcd-&gt;refcnt);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ib_uverbs_release_uevent(uevent);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(UVERBS_METHOD_SRQ_CREATE)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_usrq_object *obj = container_of(</div><div class='add'>+		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_CREATE_SRQ_HANDLE),</div><div class='add'>+		typeof(*obj), uevent.uobject);</div><div class='add'>+	struct ib_pd *pd =</div><div class='add'>+		uverbs_attr_get_obj(attrs, UVERBS_ATTR_CREATE_SRQ_PD_HANDLE);</div><div class='add'>+	struct ib_srq_init_attr attr = {};</div><div class='add'>+	struct ib_uobject *xrcd_uobj;</div><div class='add'>+	struct ib_srq *srq;</div><div class='add'>+	u64 user_handle;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_from(&amp;attr.attr.max_sge, attrs,</div><div class='add'>+			       UVERBS_ATTR_CREATE_SRQ_MAX_SGE);</div><div class='add'>+	if (!ret)</div><div class='add'>+		ret = uverbs_copy_from(&amp;attr.attr.max_wr, attrs,</div><div class='add'>+				       UVERBS_ATTR_CREATE_SRQ_MAX_WR);</div><div class='add'>+	if (!ret)</div><div class='add'>+		ret = uverbs_copy_from(&amp;attr.attr.srq_limit, attrs,</div><div class='add'>+				       UVERBS_ATTR_CREATE_SRQ_LIMIT);</div><div class='add'>+	if (!ret)</div><div class='add'>+		ret = uverbs_copy_from(&amp;user_handle, attrs,</div><div class='add'>+				       UVERBS_ATTR_CREATE_SRQ_USER_HANDLE);</div><div class='add'>+	if (!ret)</div><div class='add'>+		ret = uverbs_get_const(&amp;attr.srq_type, attrs,</div><div class='add'>+				       UVERBS_ATTR_CREATE_SRQ_TYPE);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (ib_srq_has_cq(attr.srq_type)) {</div><div class='add'>+		attr.ext.cq = uverbs_attr_get_obj(attrs,</div><div class='add'>+					UVERBS_ATTR_CREATE_SRQ_CQ_HANDLE);</div><div class='add'>+		if (IS_ERR(attr.ext.cq))</div><div class='add'>+			return PTR_ERR(attr.ext.cq);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (attr.srq_type) {</div><div class='add'>+	case IB_UVERBS_SRQT_XRC:</div><div class='add'>+		xrcd_uobj = uverbs_attr_get_uobject(attrs,</div><div class='add'>+					UVERBS_ATTR_CREATE_SRQ_XRCD_HANDLE);</div><div class='add'>+		if (IS_ERR(xrcd_uobj))</div><div class='add'>+			return PTR_ERR(xrcd_uobj);</div><div class='add'>+</div><div class='add'>+		attr.ext.xrc.xrcd = (struct ib_xrcd *)xrcd_uobj-&gt;object;</div><div class='add'>+		if (!attr.ext.xrc.xrcd)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		obj-&gt;uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,</div><div class='add'>+					  uobject);</div><div class='add'>+		atomic_inc(&amp;obj-&gt;uxrcd-&gt;refcnt);</div><div class='add'>+		break;</div><div class='add'>+	case IB_UVERBS_SRQT_TM:</div><div class='add'>+		ret = uverbs_copy_from(&amp;attr.ext.tag_matching.max_num_tags,</div><div class='add'>+				       attrs,</div><div class='add'>+				       UVERBS_ATTR_CREATE_SRQ_MAX_NUM_TAGS);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+		break;</div><div class='add'>+	case IB_UVERBS_SRQT_BASIC:</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	obj-&gt;uevent.event_file = ib_uverbs_get_async_event(attrs,</div><div class='add'>+					UVERBS_ATTR_CREATE_SRQ_EVENT_FD);</div><div class='add'>+	INIT_LIST_HEAD(&amp;obj-&gt;uevent.event_list);</div><div class='add'>+	attr.event_handler = ib_uverbs_srq_event_handler;</div><div class='add'>+	obj-&gt;uevent.uobject.user_handle = user_handle;</div><div class='add'>+</div><div class='add'>+	srq = ib_create_srq_user(pd, &amp;attr, obj, &amp;attrs-&gt;driver_udata);</div><div class='add'>+	if (IS_ERR(srq)) {</div><div class='add'>+		ret = PTR_ERR(srq);</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	obj-&gt;uevent.uobject.object = srq;</div><div class='add'>+	uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_CREATE_SRQ_HANDLE);</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_SRQ_RESP_MAX_WR,</div><div class='add'>+			     &amp;attr.attr.max_wr,</div><div class='add'>+			     sizeof(attr.attr.max_wr));</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_SRQ_RESP_MAX_SGE,</div><div class='add'>+			     &amp;attr.attr.max_sge,</div><div class='add'>+			     sizeof(attr.attr.max_sge));</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (attr.srq_type == IB_SRQT_XRC) {</div><div class='add'>+		ret = uverbs_copy_to(attrs,</div><div class='add'>+				     UVERBS_ATTR_CREATE_SRQ_RESP_SRQ_NUM,</div><div class='add'>+				     &amp;srq-&gt;ext.xrc.srq_num,</div><div class='add'>+				     sizeof(srq-&gt;ext.xrc.srq_num));</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+err:</div><div class='add'>+	if (obj-&gt;uevent.event_file)</div><div class='add'>+		uverbs_uobject_put(&amp;obj-&gt;uevent.event_file-&gt;uobj);</div><div class='add'>+	if (attr.srq_type == IB_SRQT_XRC)</div><div class='add'>+		atomic_dec(&amp;obj-&gt;uxrcd-&gt;refcnt);</div><div class='add'>+	return ret;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	UVERBS_METHOD_SRQ_CREATE,</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_SRQ_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_SRQ,</div><div class='add'>+			UVERBS_ACCESS_NEW,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_SRQ_PD_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_PD,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_CONST_IN(UVERBS_ATTR_CREATE_SRQ_TYPE,</div><div class='add'>+			     enum ib_uverbs_srq_type,</div><div class='add'>+			     UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_SRQ_USER_HANDLE,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_SRQ_MAX_WR,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_SRQ_MAX_SGE,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_SRQ_LIMIT,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_SRQ_XRCD_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_XRCD,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_SRQ_CQ_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_CQ,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_SRQ_MAX_NUM_TAGS,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_FD(UVERBS_ATTR_CREATE_SRQ_EVENT_FD,</div><div class='add'>+		       UVERBS_OBJECT_ASYNC_EVENT,</div><div class='add'>+		       UVERBS_ACCESS_READ,</div><div class='add'>+		       UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_SRQ_RESP_MAX_WR,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			    UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_SRQ_RESP_MAX_SGE,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			    UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_SRQ_RESP_SRQ_NUM,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_UHW());</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(UVERBS_METHOD_SRQ_DESTROY)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *uobj =</div><div class='add'>+		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_DESTROY_SRQ_HANDLE);</div><div class='add'>+	struct ib_usrq_object *obj =</div><div class='add'>+		container_of(uobj, struct ib_usrq_object, uevent.uobject);</div><div class='add'>+	struct ib_uverbs_destroy_srq_resp resp = {</div><div class='add'>+		.events_reported = obj-&gt;uevent.events_reported</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	return uverbs_copy_to(attrs, UVERBS_ATTR_DESTROY_SRQ_RESP, &amp;resp,</div><div class='add'>+			      sizeof(resp));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	UVERBS_METHOD_SRQ_DESTROY,</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_SRQ_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_SRQ,</div><div class='add'>+			UVERBS_ACCESS_DESTROY,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_DESTROY_SRQ_RESP,</div><div class='add'>+			    UVERBS_ATTR_TYPE(struct ib_uverbs_destroy_srq_resp),</div><div class='add'>+			    UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_OBJECT(</div><div class='add'>+	UVERBS_OBJECT_SRQ,</div><div class='add'>+	UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_usrq_object),</div><div class='add'>+				 uverbs_free_srq),</div><div class='add'>+	&amp;UVERBS_METHOD(UVERBS_METHOD_SRQ_CREATE),</div><div class='add'>+	&amp;UVERBS_METHOD(UVERBS_METHOD_SRQ_DESTROY)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+const struct uapi_definition uverbs_def_obj_srq[] = {</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_SRQ,</div><div class='add'>+				      UAPI_DEF_OBJ_NEEDS_FN(destroy_srq)),</div><div class='add'>+	{}</div><div class='add'>+};</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_std_types_wq.c b/drivers/infiniband/core/uverbs_std_types_wq.c<br/>new file mode 100644<br/>index 000000000000..7ded8339346f<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_std_types_wq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_std_types_wq.c</a></div><div class='hunk'>@@ -0,0 +1,194 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2020, Mellanox Technologies inc.  All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/uverbs_std_types.h&gt;</div><div class='add'>+#include "rdma_core.h"</div><div class='add'>+#include "uverbs.h"</div><div class='add'>+</div><div class='add'>+static int uverbs_free_wq(struct ib_uobject *uobject,</div><div class='add'>+			  enum rdma_remove_reason why,</div><div class='add'>+			  struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_wq *wq = uobject-&gt;object;</div><div class='add'>+	struct ib_uwq_object *uwq =</div><div class='add'>+		container_of(uobject, struct ib_uwq_object, uevent.uobject);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = ib_destroy_wq_user(wq, &amp;attrs-&gt;driver_udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ib_uverbs_release_uevent(&amp;uwq-&gt;uevent);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(UVERBS_METHOD_WQ_CREATE)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uwq_object *obj = container_of(</div><div class='add'>+		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_CREATE_WQ_HANDLE),</div><div class='add'>+		typeof(*obj), uevent.uobject);</div><div class='add'>+	struct ib_pd *pd =</div><div class='add'>+		uverbs_attr_get_obj(attrs, UVERBS_ATTR_CREATE_WQ_PD_HANDLE);</div><div class='add'>+	struct ib_cq *cq =</div><div class='add'>+		uverbs_attr_get_obj(attrs, UVERBS_ATTR_CREATE_WQ_CQ_HANDLE);</div><div class='add'>+	struct ib_wq_init_attr wq_init_attr = {};</div><div class='add'>+	struct ib_wq *wq;</div><div class='add'>+	u64 user_handle;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_get_flags32(&amp;wq_init_attr.create_flags, attrs,</div><div class='add'>+				 UVERBS_ATTR_CREATE_WQ_FLAGS,</div><div class='add'>+				 IB_UVERBS_WQ_FLAGS_CVLAN_STRIPPING |</div><div class='add'>+				 IB_UVERBS_WQ_FLAGS_SCATTER_FCS |</div><div class='add'>+				 IB_UVERBS_WQ_FLAGS_DELAY_DROP |</div><div class='add'>+				 IB_UVERBS_WQ_FLAGS_PCI_WRITE_END_PADDING);</div><div class='add'>+	if (!ret)</div><div class='add'>+		ret = uverbs_copy_from(&amp;wq_init_attr.max_sge, attrs,</div><div class='add'>+			       UVERBS_ATTR_CREATE_WQ_MAX_SGE);</div><div class='add'>+	if (!ret)</div><div class='add'>+		ret = uverbs_copy_from(&amp;wq_init_attr.max_wr, attrs,</div><div class='add'>+				       UVERBS_ATTR_CREATE_WQ_MAX_WR);</div><div class='add'>+	if (!ret)</div><div class='add'>+		ret = uverbs_copy_from(&amp;user_handle, attrs,</div><div class='add'>+				       UVERBS_ATTR_CREATE_WQ_USER_HANDLE);</div><div class='add'>+	if (!ret)</div><div class='add'>+		ret = uverbs_get_const(&amp;wq_init_attr.wq_type, attrs,</div><div class='add'>+				       UVERBS_ATTR_CREATE_WQ_TYPE);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (wq_init_attr.wq_type != IB_WQT_RQ)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	obj-&gt;uevent.event_file = ib_uverbs_get_async_event(attrs,</div><div class='add'>+					UVERBS_ATTR_CREATE_WQ_EVENT_FD);</div><div class='add'>+	obj-&gt;uevent.uobject.user_handle = user_handle;</div><div class='add'>+	INIT_LIST_HEAD(&amp;obj-&gt;uevent.event_list);</div><div class='add'>+	wq_init_attr.event_handler = ib_uverbs_wq_event_handler;</div><div class='add'>+	wq_init_attr.wq_context = attrs-&gt;ufile;</div><div class='add'>+	wq_init_attr.cq = cq;</div><div class='add'>+</div><div class='add'>+	wq = pd-&gt;device-&gt;ops.create_wq(pd, &amp;wq_init_attr, &amp;attrs-&gt;driver_udata);</div><div class='add'>+	if (IS_ERR(wq)) {</div><div class='add'>+		ret = PTR_ERR(wq);</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	obj-&gt;uevent.uobject.object = wq;</div><div class='add'>+	wq-&gt;wq_type = wq_init_attr.wq_type;</div><div class='add'>+	wq-&gt;cq = cq;</div><div class='add'>+	wq-&gt;pd = pd;</div><div class='add'>+	wq-&gt;device = pd-&gt;device;</div><div class='add'>+	wq-&gt;wq_context = wq_init_attr.wq_context;</div><div class='add'>+	atomic_set(&amp;wq-&gt;usecnt, 0);</div><div class='add'>+	atomic_inc(&amp;pd-&gt;usecnt);</div><div class='add'>+	atomic_inc(&amp;cq-&gt;usecnt);</div><div class='add'>+	wq-&gt;uobject = obj;</div><div class='add'>+	uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_CREATE_WQ_HANDLE);</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_WQ_RESP_MAX_WR,</div><div class='add'>+			     &amp;wq_init_attr.max_wr,</div><div class='add'>+			     sizeof(wq_init_attr.max_wr));</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_WQ_RESP_MAX_SGE,</div><div class='add'>+			     &amp;wq_init_attr.max_sge,</div><div class='add'>+			     sizeof(wq_init_attr.max_sge));</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_WQ_RESP_WQ_NUM,</div><div class='add'>+			     &amp;wq-&gt;wq_num,</div><div class='add'>+			     sizeof(wq-&gt;wq_num));</div><div class='add'>+	return ret;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	if (obj-&gt;uevent.event_file)</div><div class='add'>+		uverbs_uobject_put(&amp;obj-&gt;uevent.event_file-&gt;uobj);</div><div class='add'>+	return ret;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	UVERBS_METHOD_WQ_CREATE,</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_WQ_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_WQ,</div><div class='add'>+			UVERBS_ACCESS_NEW,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_WQ_PD_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_PD,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_CONST_IN(UVERBS_ATTR_CREATE_WQ_TYPE,</div><div class='add'>+			     enum ib_wq_type,</div><div class='add'>+			     UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_WQ_USER_HANDLE,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_WQ_MAX_WR,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_WQ_MAX_SGE,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_CREATE_WQ_FLAGS,</div><div class='add'>+			     enum ib_uverbs_wq_flags,</div><div class='add'>+			     UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_WQ_CQ_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_CQ,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_FD(UVERBS_ATTR_CREATE_WQ_EVENT_FD,</div><div class='add'>+		       UVERBS_OBJECT_ASYNC_EVENT,</div><div class='add'>+		       UVERBS_ACCESS_READ,</div><div class='add'>+		       UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_WQ_RESP_MAX_WR,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			    UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_WQ_RESP_MAX_SGE,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			    UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_WQ_RESP_WQ_NUM,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_UHW());</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(UVERBS_METHOD_WQ_DESTROY)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *uobj =</div><div class='add'>+		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_DESTROY_WQ_HANDLE);</div><div class='add'>+	struct ib_uwq_object *obj =</div><div class='add'>+		container_of(uobj, struct ib_uwq_object, uevent.uobject);</div><div class='add'>+</div><div class='add'>+	return uverbs_copy_to(attrs, UVERBS_ATTR_DESTROY_WQ_RESP,</div><div class='add'>+			      &amp;obj-&gt;uevent.events_reported,</div><div class='add'>+			      sizeof(obj-&gt;uevent.events_reported));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	UVERBS_METHOD_WQ_DESTROY,</div><div class='add'>+	UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_WQ_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_WQ,</div><div class='add'>+			UVERBS_ACCESS_DESTROY,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_DESTROY_WQ_RESP,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			    UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_OBJECT(</div><div class='add'>+	UVERBS_OBJECT_WQ,</div><div class='add'>+	UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uwq_object), uverbs_free_wq),</div><div class='add'>+	&amp;UVERBS_METHOD(UVERBS_METHOD_WQ_CREATE),</div><div class='add'>+	&amp;UVERBS_METHOD(UVERBS_METHOD_WQ_DESTROY)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+const struct uapi_definition uverbs_def_obj_wq[] = {</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_WQ,</div><div class='add'>+				      UAPI_DEF_OBJ_NEEDS_FN(destroy_wq)),</div><div class='add'>+	{}</div><div class='add'>+};</div><div class='head'>diff --git a/drivers/infiniband/core/uverbs_uapi.c b/drivers/infiniband/core/uverbs_uapi.c<br/>index 7a987acf0c0b..e00ea63175bd 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_uapi.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/uverbs_uapi.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/uverbs_uapi.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/uverbs_uapi.c</a></div><div class='hunk'>@@ -22,6 +22,8 @@ static void *uapi_add_elm(struct uverbs_api *uapi, u32 key, size_t alloc_size)</div><div class='ctx'> 		return ERR_PTR(-EOVERFLOW);</div><div class='ctx'> </div><div class='ctx'> 	elm = kzalloc(alloc_size, GFP_KERNEL);</div><div class='add'>+	if (!elm)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='ctx'> 	rc = radix_tree_insert(&amp;uapi-&gt;radix, key, elm);</div><div class='ctx'> 	if (rc) {</div><div class='ctx'> 		kfree(elm);</div><div class='hunk'>@@ -77,10 +79,7 @@ static int uapi_create_write(struct uverbs_api *uapi,</div><div class='ctx'> </div><div class='ctx'> 	method_elm-&gt;is_ex = def-&gt;write.is_ex;</div><div class='ctx'> 	method_elm-&gt;handler = def-&gt;func_write;</div><div class='del'>-	if (def-&gt;write.is_ex)</div><div class='del'>-		method_elm-&gt;disabled = !(ibdev-&gt;uverbs_ex_cmd_mask &amp;</div><div class='del'>-					 BIT_ULL(def-&gt;write.command_num));</div><div class='del'>-	else</div><div class='add'>+	if (!def-&gt;write.is_ex)</div><div class='ctx'> 		method_elm-&gt;disabled = !(ibdev-&gt;uverbs_cmd_mask &amp;</div><div class='ctx'> 					 BIT_ULL(def-&gt;write.command_num));</div><div class='ctx'> </div><div class='hunk'>@@ -193,9 +192,9 @@ static int uapi_merge_obj_tree(struct uverbs_api *uapi,</div><div class='ctx'> 		 * disassociation, and the FD types require the driver to use</div><div class='ctx'> 		 * struct file_operations.owner to prevent the driver module</div><div class='ctx'> 		 * code from unloading while the file is open. This provides</div><div class='del'>-		 * enough safety that uverbs_close_fd() will continue to work.</div><div class='del'>-		 * Drivers using FD are responsible to handle disassociation of</div><div class='del'>-		 * the device on their own.</div><div class='add'>+		 * enough safety that uverbs_uobject_fd_release() will</div><div class='add'>+		 * continue to work.  Drivers using FD are responsible to</div><div class='add'>+		 * handle disassociation of the device on their own.</div><div class='ctx'> 		 */</div><div class='ctx'> 		if (WARN_ON(is_driver &amp;&amp;</div><div class='ctx'> 			    obj-&gt;type_attrs-&gt;type_class != &amp;uverbs_idr_class &amp;&amp;</div><div class='hunk'>@@ -448,6 +447,9 @@ static int uapi_finalize(struct uverbs_api *uapi)</div><div class='ctx'> 	uapi-&gt;num_write_ex = max_write_ex + 1;</div><div class='ctx'> 	data = kmalloc_array(uapi-&gt;num_write + uapi-&gt;num_write_ex,</div><div class='ctx'> 			     sizeof(*uapi-&gt;write_methods), GFP_KERNEL);</div><div class='add'>+	if (!data)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='ctx'> 	for (i = 0; i != uapi-&gt;num_write + uapi-&gt;num_write_ex; i++)</div><div class='ctx'> 		data[i] = &amp;uapi-&gt;notsupp_method;</div><div class='ctx'> 	uapi-&gt;write_methods = data;</div><div class='hunk'>@@ -518,7 +520,7 @@ static void uapi_key_okay(u32 key)</div><div class='ctx'> 		count++;</div><div class='ctx'> 	if (uapi_key_is_attr(key))</div><div class='ctx'> 		count++;</div><div class='del'>-	WARN(count != 1, "Bad count %d key=%x", count, key);</div><div class='add'>+	WARN(count != 1, "Bad count %u key=%x", count, key);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void uapi_finalize_disable(struct uverbs_api *uapi)</div><div class='hunk'>@@ -624,13 +626,18 @@ void uverbs_destroy_api(struct uverbs_api *uapi)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static const struct uapi_definition uverbs_core_api[] = {</div><div class='add'>+	UAPI_DEF_CHAIN(uverbs_def_obj_async_fd),</div><div class='ctx'> 	UAPI_DEF_CHAIN(uverbs_def_obj_counters),</div><div class='ctx'> 	UAPI_DEF_CHAIN(uverbs_def_obj_cq),</div><div class='ctx'> 	UAPI_DEF_CHAIN(uverbs_def_obj_device),</div><div class='ctx'> 	UAPI_DEF_CHAIN(uverbs_def_obj_dm),</div><div class='add'>+	UAPI_DEF_CHAIN(uverbs_def_obj_dmah),</div><div class='ctx'> 	UAPI_DEF_CHAIN(uverbs_def_obj_flow_action),</div><div class='ctx'> 	UAPI_DEF_CHAIN(uverbs_def_obj_intf),</div><div class='ctx'> 	UAPI_DEF_CHAIN(uverbs_def_obj_mr),</div><div class='add'>+	UAPI_DEF_CHAIN(uverbs_def_obj_qp),</div><div class='add'>+	UAPI_DEF_CHAIN(uverbs_def_obj_srq),</div><div class='add'>+	UAPI_DEF_CHAIN(uverbs_def_obj_wq),</div><div class='ctx'> 	UAPI_DEF_CHAIN(uverbs_def_write_intf),</div><div class='ctx'> 	{},</div><div class='ctx'> };</div><div class='hunk'>@@ -645,7 +652,7 @@ struct uverbs_api *uverbs_alloc_api(struct ib_device *ibdev)</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='ctx'> 	INIT_RADIX_TREE(&amp;uapi-&gt;radix, GFP_KERNEL);</div><div class='del'>-	uapi-&gt;driver_id = ibdev-&gt;driver_id;</div><div class='add'>+	uapi-&gt;driver_id = ibdev-&gt;ops.driver_id;</div><div class='ctx'> </div><div class='ctx'> 	rc = uapi_merge_def(uapi, ibdev, uverbs_core_api, false);</div><div class='ctx'> 	if (rc)</div><div class='head'>diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c<br/>index e666a1f7608d..11b1a194de44 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/verbs.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/core/verbs.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/core/verbs.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/core/verbs.c</a></div><div class='hunk'>@@ -50,8 +50,10 @@</div><div class='ctx'> #include &lt;rdma/ib_cache.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_addr.h&gt;</div><div class='ctx'> #include &lt;rdma/rw.h&gt;</div><div class='add'>+#include &lt;rdma/lag.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "core_priv.h"</div><div class='add'>+#include &lt;trace/events/rdma_core.h&gt;</div><div class='ctx'> </div><div class='ctx'> static int ib_resolve_eth_dmac(struct ib_device *device,</div><div class='ctx'> 			       struct rdma_ah_attr *ah_attr);</div><div class='hunk'>@@ -94,10 +96,10 @@ static const char * const wc_statuses[] = {</div><div class='ctx'> 	[IB_WC_LOC_EEC_OP_ERR]		= "local EE context operation error",</div><div class='ctx'> 	[IB_WC_LOC_PROT_ERR]		= "local protection error",</div><div class='ctx'> 	[IB_WC_WR_FLUSH_ERR]		= "WR flushed",</div><div class='del'>-	[IB_WC_MW_BIND_ERR]		= "memory management operation error",</div><div class='add'>+	[IB_WC_MW_BIND_ERR]		= "memory bind operation error",</div><div class='ctx'> 	[IB_WC_BAD_RESP_ERR]		= "bad response error",</div><div class='ctx'> 	[IB_WC_LOC_ACCESS_ERR]		= "local access error",</div><div class='del'>-	[IB_WC_REM_INV_REQ_ERR]		= "invalid request error",</div><div class='add'>+	[IB_WC_REM_INV_REQ_ERR]		= "remote invalid request error",</div><div class='ctx'> 	[IB_WC_REM_ACCESS_ERR]		= "remote access error",</div><div class='ctx'> 	[IB_WC_REM_OP_ERR]		= "remote operation error",</div><div class='ctx'> 	[IB_WC_RETRY_EXC_ERR]		= "transport retry counter exceeded",</div><div class='hunk'>@@ -145,6 +147,8 @@ __attribute_const__ int ib_rate_to_mult(enum ib_rate rate)</div><div class='ctx'> 	case IB_RATE_50_GBPS:  return  20;</div><div class='ctx'> 	case IB_RATE_400_GBPS: return 160;</div><div class='ctx'> 	case IB_RATE_600_GBPS: return 240;</div><div class='add'>+	case IB_RATE_800_GBPS: return 320;</div><div class='add'>+	case IB_RATE_1600_GBPS: return 640;</div><div class='ctx'> 	default:	       return  -1;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='hunk'>@@ -174,6 +178,8 @@ __attribute_const__ enum ib_rate mult_to_ib_rate(int mult)</div><div class='ctx'> 	case 20:  return IB_RATE_50_GBPS;</div><div class='ctx'> 	case 160: return IB_RATE_400_GBPS;</div><div class='ctx'> 	case 240: return IB_RATE_600_GBPS;</div><div class='add'>+	case 320: return IB_RATE_800_GBPS;</div><div class='add'>+	case 640: return IB_RATE_1600_GBPS;</div><div class='ctx'> 	default:  return IB_RATE_PORT_CURRENT;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='hunk'>@@ -203,13 +209,15 @@ __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate)</div><div class='ctx'> 	case IB_RATE_50_GBPS:  return 53125;</div><div class='ctx'> 	case IB_RATE_400_GBPS: return 425000;</div><div class='ctx'> 	case IB_RATE_600_GBPS: return 637500;</div><div class='add'>+	case IB_RATE_800_GBPS: return 850000;</div><div class='add'>+	case IB_RATE_1600_GBPS: return 1700000;</div><div class='ctx'> 	default:	       return -1;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_rate_to_mbps);</div><div class='ctx'> </div><div class='ctx'> __attribute_const__ enum rdma_transport_type</div><div class='del'>-rdma_node_get_transport(enum rdma_node_type node_type)</div><div class='add'>+rdma_node_get_transport(unsigned int node_type)</div><div class='ctx'> {</div><div class='ctx'> </div><div class='ctx'> 	if (node_type == RDMA_NODE_USNIC)</div><div class='hunk'>@@ -225,7 +233,8 @@ rdma_node_get_transport(enum rdma_node_type node_type)</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_node_get_transport);</div><div class='ctx'> </div><div class='del'>-enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num)</div><div class='add'>+enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device,</div><div class='add'>+					      u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	enum rdma_transport_type lt;</div><div class='ctx'> 	if (device-&gt;ops.get_link_layer)</div><div class='hunk'>@@ -242,8 +251,10 @@ EXPORT_SYMBOL(rdma_port_get_link_layer);</div><div class='ctx'> /* Protection domains */</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * ib_alloc_pd - Allocates an unused protection domain.</div><div class='add'>+ * __ib_alloc_pd - Allocates an unused protection domain.</div><div class='ctx'>  * @device: The device on which to allocate the protection domain.</div><div class='add'>+ * @flags: protection domain flags</div><div class='add'>+ * @caller: caller's build-time module name</div><div class='ctx'>  *</div><div class='ctx'>  * A protection domain object provides an association between QPs, shared</div><div class='ctx'>  * receive queues, address handles, memory regions, and memory windows.</div><div class='hunk'>@@ -263,22 +274,20 @@ struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='ctx'> 	pd-&gt;device = device;</div><div class='del'>-	pd-&gt;uobject = NULL;</div><div class='del'>-	pd-&gt;__internal_mr = NULL;</div><div class='del'>-	atomic_set(&amp;pd-&gt;usecnt, 0);</div><div class='ctx'> 	pd-&gt;flags = flags;</div><div class='ctx'> </div><div class='del'>-	pd-&gt;res.type = RDMA_RESTRACK_PD;</div><div class='del'>-	rdma_restrack_set_task(&amp;pd-&gt;res, caller);</div><div class='add'>+	rdma_restrack_new(&amp;pd-&gt;res, RDMA_RESTRACK_PD);</div><div class='add'>+	rdma_restrack_set_name(&amp;pd-&gt;res, caller);</div><div class='ctx'> </div><div class='ctx'> 	ret = device-&gt;ops.alloc_pd(pd, NULL);</div><div class='ctx'> 	if (ret) {</div><div class='add'>+		rdma_restrack_put(&amp;pd-&gt;res);</div><div class='ctx'> 		kfree(pd);</div><div class='ctx'> 		return ERR_PTR(ret);</div><div class='ctx'> 	}</div><div class='del'>-	rdma_restrack_kadd(&amp;pd-&gt;res);</div><div class='add'>+	rdma_restrack_add(&amp;pd-&gt;res);</div><div class='ctx'> </div><div class='del'>-	if (device-&gt;attrs.device_cap_flags &amp; IB_DEVICE_LOCAL_DMA_LKEY)</div><div class='add'>+	if (device-&gt;attrs.kernel_cap_flags &amp; IBK_LOCAL_DMA_LKEY)</div><div class='ctx'> 		pd-&gt;local_dma_lkey = device-&gt;local_dma_lkey;</div><div class='ctx'> 	else</div><div class='ctx'> 		mr_access_flags |= IB_ACCESS_LOCAL_WRITE;</div><div class='hunk'>@@ -299,12 +308,13 @@ struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,</div><div class='ctx'> </div><div class='ctx'> 		mr-&gt;device	= pd-&gt;device;</div><div class='ctx'> 		mr-&gt;pd		= pd;</div><div class='add'>+		mr-&gt;type        = IB_MR_TYPE_DMA;</div><div class='ctx'> 		mr-&gt;uobject	= NULL;</div><div class='ctx'> 		mr-&gt;need_inval	= false;</div><div class='ctx'> </div><div class='ctx'> 		pd-&gt;__internal_mr = mr;</div><div class='ctx'> </div><div class='del'>-		if (!(device-&gt;attrs.device_cap_flags &amp; IB_DEVICE_LOCAL_DMA_LKEY))</div><div class='add'>+		if (!(device-&gt;attrs.kernel_cap_flags &amp; IBK_LOCAL_DMA_LKEY))</div><div class='ctx'> 			pd-&gt;local_dma_lkey = pd-&gt;__internal_mr-&gt;lkey;</div><div class='ctx'> </div><div class='ctx'> 		if (flags &amp; IB_PD_UNSAFE_GLOBAL_RKEY)</div><div class='hunk'>@@ -316,7 +326,7 @@ struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,</div><div class='ctx'> EXPORT_SYMBOL(__ib_alloc_pd);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * ib_dealloc_pd - Deallocates a protection domain.</div><div class='add'>+ * ib_dealloc_pd_user - Deallocates a protection domain.</div><div class='ctx'>  * @pd: The protection domain to deallocate.</div><div class='ctx'>  * @udata: Valid user data or NULL for kernel object</div><div class='ctx'>  *</div><div class='hunk'>@@ -324,7 +334,7 @@ EXPORT_SYMBOL(__ib_alloc_pd);</div><div class='ctx'>  * exist.  The caller is responsible to synchronously destroy them and</div><div class='ctx'>  * guarantee no new allocations will happen.</div><div class='ctx'>  */</div><div class='del'>-void ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata)</div><div class='add'>+int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='hunk'>@@ -334,13 +344,13 @@ void ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata)</div><div class='ctx'> 		pd-&gt;__internal_mr = NULL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* uverbs manipulates usecnt with proper locking, while the kabi</div><div class='del'>-	   requires the caller to guarantee we can't race here. */</div><div class='del'>-	WARN_ON(atomic_read(&amp;pd-&gt;usecnt));</div><div class='add'>+	ret = pd-&gt;device-&gt;ops.dealloc_pd(pd, udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='ctx'> 	rdma_restrack_del(&amp;pd-&gt;res);</div><div class='del'>-	pd-&gt;device-&gt;ops.dealloc_pd(pd, udata);</div><div class='ctx'> 	kfree(pd);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_dealloc_pd_user);</div><div class='ctx'> </div><div class='hunk'>@@ -362,7 +372,7 @@ void rdma_copy_ah_attr(struct rdma_ah_attr *dest,</div><div class='ctx'> EXPORT_SYMBOL(rdma_copy_ah_attr);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * rdma_replace_ah_attr - Replace valid ah_attr with new new one.</div><div class='add'>+ * rdma_replace_ah_attr - Replace valid ah_attr with new one.</div><div class='ctx'>  * @old:        Pointer to existing ah_attr which needs to be replaced.</div><div class='ctx'>  *              old is assumed to be valid or zero'd</div><div class='ctx'>  * @new:        Pointer to the new ah_attr.</div><div class='hunk'>@@ -496,15 +506,17 @@ rdma_update_sgid_attr(struct rdma_ah_attr *ah_attr,</div><div class='ctx'> static struct ib_ah *_rdma_create_ah(struct ib_pd *pd,</div><div class='ctx'> 				     struct rdma_ah_attr *ah_attr,</div><div class='ctx'> 				     u32 flags,</div><div class='del'>-				     struct ib_udata *udata)</div><div class='add'>+				     struct ib_udata *udata,</div><div class='add'>+				     struct net_device *xmit_slave)</div><div class='ctx'> {</div><div class='add'>+	struct rdma_ah_init_attr init_attr = {};</div><div class='ctx'> 	struct ib_device *device = pd-&gt;device;</div><div class='ctx'> 	struct ib_ah *ah;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	might_sleep_if(flags &amp; RDMA_CREATE_AH_SLEEPABLE);</div><div class='ctx'> </div><div class='del'>-	if (!device-&gt;ops.create_ah)</div><div class='add'>+	if (!udata &amp;&amp; !device-&gt;ops.create_ah)</div><div class='ctx'> 		return ERR_PTR(-EOPNOTSUPP);</div><div class='ctx'> </div><div class='ctx'> 	ah = rdma_zalloc_drv_obj_gfp(</div><div class='hunk'>@@ -517,9 +529,17 @@ static struct ib_ah *_rdma_create_ah(struct ib_pd *pd,</div><div class='ctx'> 	ah-&gt;pd = pd;</div><div class='ctx'> 	ah-&gt;type = ah_attr-&gt;type;</div><div class='ctx'> 	ah-&gt;sgid_attr = rdma_update_sgid_attr(ah_attr, NULL);</div><div class='add'>+	init_attr.ah_attr = ah_attr;</div><div class='add'>+	init_attr.flags = flags;</div><div class='add'>+	init_attr.xmit_slave = xmit_slave;</div><div class='ctx'> </div><div class='del'>-	ret = device-&gt;ops.create_ah(ah, ah_attr, flags, udata);</div><div class='add'>+	if (udata)</div><div class='add'>+		ret = device-&gt;ops.create_user_ah(ah, &amp;init_attr, udata);</div><div class='add'>+	else</div><div class='add'>+		ret = device-&gt;ops.create_ah(ah, &amp;init_attr, NULL);</div><div class='ctx'> 	if (ret) {</div><div class='add'>+		if (ah-&gt;sgid_attr)</div><div class='add'>+			rdma_put_gid_attr(ah-&gt;sgid_attr);</div><div class='ctx'> 		kfree(ah);</div><div class='ctx'> 		return ERR_PTR(ret);</div><div class='ctx'> 	}</div><div class='hunk'>@@ -543,15 +563,22 @@ struct ib_ah *rdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,</div><div class='ctx'> 			     u32 flags)</div><div class='ctx'> {</div><div class='ctx'> 	const struct ib_gid_attr *old_sgid_attr;</div><div class='add'>+	struct net_device *slave;</div><div class='ctx'> 	struct ib_ah *ah;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	ret = rdma_fill_sgid_attr(pd-&gt;device, ah_attr, &amp;old_sgid_attr);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ERR_PTR(ret);</div><div class='del'>-</div><div class='del'>-	ah = _rdma_create_ah(pd, ah_attr, flags, NULL);</div><div class='del'>-</div><div class='add'>+	slave = rdma_lag_get_ah_roce_slave(pd-&gt;device, ah_attr,</div><div class='add'>+					   (flags &amp; RDMA_CREATE_AH_SLEEPABLE) ?</div><div class='add'>+					   GFP_KERNEL : GFP_ATOMIC);</div><div class='add'>+	if (IS_ERR(slave)) {</div><div class='add'>+		rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);</div><div class='add'>+		return ERR_CAST(slave);</div><div class='add'>+	}</div><div class='add'>+	ah = _rdma_create_ah(pd, ah_attr, flags, NULL, slave);</div><div class='add'>+	rdma_lag_put_ah_roce_slave(slave);</div><div class='ctx'> 	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);</div><div class='ctx'> 	return ah;</div><div class='ctx'> }</div><div class='hunk'>@@ -590,7 +617,8 @@ struct ib_ah *rdma_create_user_ah(struct ib_pd *pd,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ah = _rdma_create_ah(pd, ah_attr, RDMA_CREATE_AH_SLEEPABLE, udata);</div><div class='add'>+	ah = _rdma_create_ah(pd, ah_attr, RDMA_CREATE_AH_SLEEPABLE,</div><div class='add'>+			     udata, NULL);</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='ctx'> 	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);</div><div class='hunk'>@@ -632,7 +660,7 @@ int ib_get_rdma_header_version(const union rdma_network_hdr *hdr)</div><div class='ctx'> EXPORT_SYMBOL(ib_get_rdma_header_version);</div><div class='ctx'> </div><div class='ctx'> static enum rdma_network_type ib_get_net_type_by_grh(struct ib_device *device,</div><div class='del'>-						     u8 port_num,</div><div class='add'>+						     u32 port_num,</div><div class='ctx'> 						     const struct ib_grh *grh)</div><div class='ctx'> {</div><div class='ctx'> 	int grh_version;</div><div class='hunk'>@@ -661,20 +689,21 @@ static bool find_gid_index(const union ib_gid *gid,</div><div class='ctx'> 			   void *context)</div><div class='ctx'> {</div><div class='ctx'> 	struct find_gid_index_context *ctx = context;</div><div class='add'>+	u16 vlan_id = 0xffff;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (ctx-&gt;gid_type != gid_attr-&gt;gid_type)</div><div class='ctx'> 		return false;</div><div class='ctx'> </div><div class='del'>-	if ((!!(ctx-&gt;vlan_id != 0xffff) == !is_vlan_dev(gid_attr-&gt;ndev)) ||</div><div class='del'>-	    (is_vlan_dev(gid_attr-&gt;ndev) &amp;&amp;</div><div class='del'>-	     vlan_dev_vlan_id(gid_attr-&gt;ndev) != ctx-&gt;vlan_id))</div><div class='add'>+	ret = rdma_read_gid_l2_fields(gid_attr, &amp;vlan_id, NULL);</div><div class='add'>+	if (ret)</div><div class='ctx'> 		return false;</div><div class='ctx'> </div><div class='del'>-	return true;</div><div class='add'>+	return ctx-&gt;vlan_id == vlan_id;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static const struct ib_gid_attr *</div><div class='del'>-get_sgid_attr_from_eth(struct ib_device *device, u8 port_num,</div><div class='add'>+get_sgid_attr_from_eth(struct ib_device *device, u32 port_num,</div><div class='ctx'> 		       u16 vlan_id, const union ib_gid *sgid,</div><div class='ctx'> 		       enum ib_gid_type gid_type)</div><div class='ctx'> {</div><div class='hunk'>@@ -709,7 +738,7 @@ int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr,</div><div class='ctx'> 				       (struct in6_addr *)dgid);</div><div class='ctx'> 		return 0;</div><div class='ctx'> 	} else if (net_type == RDMA_NETWORK_IPV6 ||</div><div class='del'>-		   net_type == RDMA_NETWORK_IB) {</div><div class='add'>+		   net_type == RDMA_NETWORK_IB || RDMA_NETWORK_ROCE_V1) {</div><div class='ctx'> 		*dgid = hdr-&gt;ibgrh.dgid;</div><div class='ctx'> 		*sgid = hdr-&gt;ibgrh.sgid;</div><div class='ctx'> 		return 0;</div><div class='hunk'>@@ -721,7 +750,7 @@ EXPORT_SYMBOL(ib_get_gids_from_rdma_hdr);</div><div class='ctx'> </div><div class='ctx'> /* Resolve destination mac address and hop limit for unicast destination</div><div class='ctx'>  * GID entry, considering the source GID entry as well.</div><div class='del'>- * ah_attribute must have have valid port_num, sgid_index.</div><div class='add'>+ * ah_attribute must have valid port_num, sgid_index.</div><div class='ctx'>  */</div><div class='ctx'> static int ib_resolve_unicast_gid_dmac(struct ib_device *device,</div><div class='ctx'> 				       struct rdma_ah_attr *ah_attr)</div><div class='hunk'>@@ -761,7 +790,7 @@ static int ib_resolve_unicast_gid_dmac(struct ib_device *device,</div><div class='ctx'>  * On success the caller is responsible to call rdma_destroy_ah_attr on the</div><div class='ctx'>  * attr.</div><div class='ctx'>  */</div><div class='del'>-int ib_init_ah_attr_from_wc(struct ib_device *device, u8 port_num,</div><div class='add'>+int ib_init_ah_attr_from_wc(struct ib_device *device, u32 port_num,</div><div class='ctx'> 			    const struct ib_wc *wc, const struct ib_grh *grh,</div><div class='ctx'> 			    struct rdma_ah_attr *ah_attr)</div><div class='ctx'> {</div><div class='hunk'>@@ -892,7 +921,7 @@ void rdma_destroy_ah_attr(struct rdma_ah_attr *ah_attr)</div><div class='ctx'> EXPORT_SYMBOL(rdma_destroy_ah_attr);</div><div class='ctx'> </div><div class='ctx'> struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc,</div><div class='del'>-				   const struct ib_grh *grh, u8 port_num)</div><div class='add'>+				   const struct ib_grh *grh, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_ah_attr ah_attr;</div><div class='ctx'> 	struct ib_ah *ah;</div><div class='hunk'>@@ -945,32 +974,50 @@ int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	const struct ib_gid_attr *sgid_attr = ah-&gt;sgid_attr;</div><div class='ctx'> 	struct ib_pd *pd;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	might_sleep_if(flags &amp; RDMA_DESTROY_AH_SLEEPABLE);</div><div class='ctx'> </div><div class='ctx'> 	pd = ah-&gt;pd;</div><div class='ctx'> </div><div class='del'>-	ah-&gt;device-&gt;ops.destroy_ah(ah, flags);</div><div class='add'>+	ret = ah-&gt;device-&gt;ops.destroy_ah(ah, flags);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='ctx'> 	atomic_dec(&amp;pd-&gt;usecnt);</div><div class='ctx'> 	if (sgid_attr)</div><div class='ctx'> 		rdma_put_gid_attr(sgid_attr);</div><div class='ctx'> </div><div class='ctx'> 	kfree(ah);</div><div class='del'>-	return 0;</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_destroy_ah_user);</div><div class='ctx'> </div><div class='ctx'> /* Shared receive queues */</div><div class='ctx'> </div><div class='del'>-struct ib_srq *ib_create_srq(struct ib_pd *pd,</div><div class='del'>-			     struct ib_srq_init_attr *srq_init_attr)</div><div class='add'>+/**</div><div class='add'>+ * ib_create_srq_user - Creates a SRQ associated with the specified protection</div><div class='add'>+ *   domain.</div><div class='add'>+ * @pd: The protection domain associated with the SRQ.</div><div class='add'>+ * @srq_init_attr: A list of initial attributes required to create the</div><div class='add'>+ *   SRQ.  If SRQ creation succeeds, then the attributes are updated to</div><div class='add'>+ *   the actual capabilities of the created SRQ.</div><div class='add'>+ * @uobject: uobject pointer if this is not a kernel SRQ</div><div class='add'>+ * @udata: udata pointer if this is not a kernel SRQ</div><div class='add'>+ *</div><div class='add'>+ * srq_attr-&gt;max_wr and srq_attr-&gt;max_sge are read the determine the</div><div class='add'>+ * requested size of the SRQ, and set to the actual values allocated</div><div class='add'>+ * on return.  If ib_create_srq() succeeds, then max_wr and max_sge</div><div class='add'>+ * will always be at least as large as the requested values.</div><div class='add'>+ */</div><div class='add'>+struct ib_srq *ib_create_srq_user(struct ib_pd *pd,</div><div class='add'>+				  struct ib_srq_init_attr *srq_init_attr,</div><div class='add'>+				  struct ib_usrq_object *uobject,</div><div class='add'>+				  struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_srq *srq;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	if (!pd-&gt;device-&gt;ops.create_srq)</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='del'>-</div><div class='ctx'> 	srq = rdma_zalloc_drv_obj(pd-&gt;device, ib_srq);</div><div class='ctx'> 	if (!srq)</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='hunk'>@@ -980,6 +1027,7 @@ struct ib_srq *ib_create_srq(struct ib_pd *pd,</div><div class='ctx'> 	srq-&gt;event_handler = srq_init_attr-&gt;event_handler;</div><div class='ctx'> 	srq-&gt;srq_context = srq_init_attr-&gt;srq_context;</div><div class='ctx'> 	srq-&gt;srq_type = srq_init_attr-&gt;srq_type;</div><div class='add'>+	srq-&gt;uobject = uobject;</div><div class='ctx'> </div><div class='ctx'> 	if (ib_srq_has_cq(srq-&gt;srq_type)) {</div><div class='ctx'> 		srq-&gt;ext.cq = srq_init_attr-&gt;ext.cq;</div><div class='hunk'>@@ -987,14 +1035,19 @@ struct ib_srq *ib_create_srq(struct ib_pd *pd,</div><div class='ctx'> 	}</div><div class='ctx'> 	if (srq-&gt;srq_type == IB_SRQT_XRC) {</div><div class='ctx'> 		srq-&gt;ext.xrc.xrcd = srq_init_attr-&gt;ext.xrc.xrcd;</div><div class='del'>-		atomic_inc(&amp;srq-&gt;ext.xrc.xrcd-&gt;usecnt);</div><div class='add'>+		if (srq-&gt;ext.xrc.xrcd)</div><div class='add'>+			atomic_inc(&amp;srq-&gt;ext.xrc.xrcd-&gt;usecnt);</div><div class='ctx'> 	}</div><div class='ctx'> 	atomic_inc(&amp;pd-&gt;usecnt);</div><div class='ctx'> </div><div class='del'>-	ret = pd-&gt;device-&gt;ops.create_srq(srq, srq_init_attr, NULL);</div><div class='add'>+	rdma_restrack_new(&amp;srq-&gt;res, RDMA_RESTRACK_SRQ);</div><div class='add'>+	rdma_restrack_parent_name(&amp;srq-&gt;res, &amp;pd-&gt;res);</div><div class='add'>+</div><div class='add'>+	ret = pd-&gt;device-&gt;ops.create_srq(srq, srq_init_attr, udata);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		atomic_dec(&amp;srq-&gt;pd-&gt;usecnt);</div><div class='del'>-		if (srq-&gt;srq_type == IB_SRQT_XRC)</div><div class='add'>+		rdma_restrack_put(&amp;srq-&gt;res);</div><div class='add'>+		atomic_dec(&amp;pd-&gt;usecnt);</div><div class='add'>+		if (srq-&gt;srq_type == IB_SRQT_XRC &amp;&amp; srq-&gt;ext.xrc.xrcd)</div><div class='ctx'> 			atomic_dec(&amp;srq-&gt;ext.xrc.xrcd-&gt;usecnt);</div><div class='ctx'> 		if (ib_srq_has_cq(srq-&gt;srq_type))</div><div class='ctx'> 			atomic_dec(&amp;srq-&gt;ext.cq-&gt;usecnt);</div><div class='hunk'>@@ -1002,9 +1055,11 @@ struct ib_srq *ib_create_srq(struct ib_pd *pd,</div><div class='ctx'> 		return ERR_PTR(ret);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	rdma_restrack_add(&amp;srq-&gt;res);</div><div class='add'>+</div><div class='ctx'> 	return srq;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(ib_create_srq);</div><div class='add'>+EXPORT_SYMBOL(ib_create_srq_user);</div><div class='ctx'> </div><div class='ctx'> int ib_modify_srq(struct ib_srq *srq,</div><div class='ctx'> 		  struct ib_srq_attr *srq_attr,</div><div class='hunk'>@@ -1026,41 +1081,49 @@ EXPORT_SYMBOL(ib_query_srq);</div><div class='ctx'> </div><div class='ctx'> int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='ctx'> 	if (atomic_read(&amp;srq-&gt;usecnt))</div><div class='ctx'> 		return -EBUSY;</div><div class='ctx'> </div><div class='del'>-	srq-&gt;device-&gt;ops.destroy_srq(srq, udata);</div><div class='add'>+	ret = srq-&gt;device-&gt;ops.destroy_srq(srq, udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='ctx'> 	atomic_dec(&amp;srq-&gt;pd-&gt;usecnt);</div><div class='del'>-	if (srq-&gt;srq_type == IB_SRQT_XRC)</div><div class='add'>+	if (srq-&gt;srq_type == IB_SRQT_XRC &amp;&amp; srq-&gt;ext.xrc.xrcd)</div><div class='ctx'> 		atomic_dec(&amp;srq-&gt;ext.xrc.xrcd-&gt;usecnt);</div><div class='ctx'> 	if (ib_srq_has_cq(srq-&gt;srq_type))</div><div class='ctx'> 		atomic_dec(&amp;srq-&gt;ext.cq-&gt;usecnt);</div><div class='add'>+	rdma_restrack_del(&amp;srq-&gt;res);</div><div class='ctx'> 	kfree(srq);</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_destroy_srq_user);</div><div class='ctx'> </div><div class='ctx'> /* Queue pairs */</div><div class='ctx'> </div><div class='add'>+static void __ib_qp_event_handler(struct ib_event *event, void *context)</div><div class='add'>+{</div><div class='add'>+	struct ib_qp *qp = event-&gt;element.qp;</div><div class='add'>+</div><div class='add'>+	if (event-&gt;event == IB_EVENT_QP_LAST_WQE_REACHED)</div><div class='add'>+		complete(&amp;qp-&gt;srq_completion);</div><div class='add'>+	if (qp-&gt;registered_event_handler)</div><div class='add'>+		qp-&gt;registered_event_handler(event, qp-&gt;qp_context);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static void __ib_shared_qp_event_handler(struct ib_event *event, void *context)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_qp *qp = context;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> </div><div class='del'>-	spin_lock_irqsave(&amp;qp-&gt;device-&gt;event_handler_lock, flags);</div><div class='add'>+	spin_lock_irqsave(&amp;qp-&gt;device-&gt;qp_open_list_lock, flags);</div><div class='ctx'> 	list_for_each_entry(event-&gt;element.qp, &amp;qp-&gt;open_list, open_list)</div><div class='ctx'> 		if (event-&gt;element.qp-&gt;event_handler)</div><div class='ctx'> 			event-&gt;element.qp-&gt;event_handler(event, event-&gt;element.qp-&gt;qp_context);</div><div class='del'>-	spin_unlock_irqrestore(&amp;qp-&gt;device-&gt;event_handler_lock, flags);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp)</div><div class='del'>-{</div><div class='del'>-	mutex_lock(&amp;xrcd-&gt;tgt_qp_mutex);</div><div class='del'>-	list_add(&amp;qp-&gt;xrcd_list, &amp;xrcd-&gt;tgt_qp_list);</div><div class='del'>-	mutex_unlock(&amp;xrcd-&gt;tgt_qp_mutex);</div><div class='add'>+	spin_unlock_irqrestore(&amp;qp-&gt;device-&gt;qp_open_list_lock, flags);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp,</div><div class='hunk'>@@ -1090,9 +1153,9 @@ static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp,</div><div class='ctx'> 	qp-&gt;qp_num = real_qp-&gt;qp_num;</div><div class='ctx'> 	qp-&gt;qp_type = real_qp-&gt;qp_type;</div><div class='ctx'> </div><div class='del'>-	spin_lock_irqsave(&amp;real_qp-&gt;device-&gt;event_handler_lock, flags);</div><div class='add'>+	spin_lock_irqsave(&amp;real_qp-&gt;device-&gt;qp_open_list_lock, flags);</div><div class='ctx'> 	list_add(&amp;qp-&gt;open_list, &amp;real_qp-&gt;open_list);</div><div class='del'>-	spin_unlock_irqrestore(&amp;real_qp-&gt;device-&gt;event_handler_lock, flags);</div><div class='add'>+	spin_unlock_irqrestore(&amp;real_qp-&gt;device-&gt;qp_open_list_lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	return qp;</div><div class='ctx'> }</div><div class='hunk'>@@ -1105,25 +1168,24 @@ struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,</div><div class='ctx'> 	if (qp_open_attr-&gt;qp_type != IB_QPT_XRC_TGT)</div><div class='ctx'> 		return ERR_PTR(-EINVAL);</div><div class='ctx'> </div><div class='del'>-	qp = ERR_PTR(-EINVAL);</div><div class='del'>-	mutex_lock(&amp;xrcd-&gt;tgt_qp_mutex);</div><div class='del'>-	list_for_each_entry(real_qp, &amp;xrcd-&gt;tgt_qp_list, xrcd_list) {</div><div class='del'>-		if (real_qp-&gt;qp_num == qp_open_attr-&gt;qp_num) {</div><div class='del'>-			qp = __ib_open_qp(real_qp, qp_open_attr-&gt;event_handler,</div><div class='del'>-					  qp_open_attr-&gt;qp_context);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='add'>+	down_read(&amp;xrcd-&gt;tgt_qps_rwsem);</div><div class='add'>+	real_qp = xa_load(&amp;xrcd-&gt;tgt_qps, qp_open_attr-&gt;qp_num);</div><div class='add'>+	if (!real_qp) {</div><div class='add'>+		up_read(&amp;xrcd-&gt;tgt_qps_rwsem);</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='ctx'> 	}</div><div class='del'>-	mutex_unlock(&amp;xrcd-&gt;tgt_qp_mutex);</div><div class='add'>+	qp = __ib_open_qp(real_qp, qp_open_attr-&gt;event_handler,</div><div class='add'>+			  qp_open_attr-&gt;qp_context);</div><div class='add'>+	up_read(&amp;xrcd-&gt;tgt_qps_rwsem);</div><div class='ctx'> 	return qp;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_open_qp);</div><div class='ctx'> </div><div class='ctx'> static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp,</div><div class='del'>-					struct ib_qp_init_attr *qp_init_attr,</div><div class='del'>-					struct ib_udata *udata)</div><div class='add'>+					struct ib_qp_init_attr *qp_init_attr)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_qp *real_qp = qp;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='ctx'> 	qp-&gt;event_handler = __ib_shared_qp_event_handler;</div><div class='ctx'> 	qp-&gt;qp_context = qp;</div><div class='hunk'>@@ -1139,84 +1201,171 @@ static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp,</div><div class='ctx'> 	if (IS_ERR(qp))</div><div class='ctx'> 		return qp;</div><div class='ctx'> </div><div class='del'>-	__ib_insert_xrcd_qp(qp_init_attr-&gt;xrcd, real_qp);</div><div class='add'>+	err = xa_err(xa_store(&amp;qp_init_attr-&gt;xrcd-&gt;tgt_qps, real_qp-&gt;qp_num,</div><div class='add'>+			      real_qp, GFP_KERNEL));</div><div class='add'>+	if (err) {</div><div class='add'>+		ib_close_qp(qp);</div><div class='add'>+		return ERR_PTR(err);</div><div class='add'>+	}</div><div class='ctx'> 	return qp;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_qp *ib_create_qp_user(struct ib_pd *pd,</div><div class='del'>-				struct ib_qp_init_attr *qp_init_attr,</div><div class='del'>-				struct ib_udata *udata)</div><div class='add'>+static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd,</div><div class='add'>+			       struct ib_qp_init_attr *attr,</div><div class='add'>+			       struct ib_udata *udata,</div><div class='add'>+			       struct ib_uqp_object *uobj, const char *caller)</div><div class='ctx'> {</div><div class='del'>-	struct ib_device *device = pd ? pd-&gt;device : qp_init_attr-&gt;xrcd-&gt;device;</div><div class='add'>+	struct ib_udata dummy = {};</div><div class='ctx'> 	struct ib_qp *qp;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	if (qp_init_attr-&gt;rwq_ind_tbl &amp;&amp;</div><div class='del'>-	    (qp_init_attr-&gt;recv_cq ||</div><div class='del'>-	    qp_init_attr-&gt;srq || qp_init_attr-&gt;cap.max_recv_wr ||</div><div class='del'>-	    qp_init_attr-&gt;cap.max_recv_sge))</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If the callers is using the RDMA API calculate the resources</div><div class='del'>-	 * needed for the RDMA READ/WRITE operations.</div><div class='del'>-	 *</div><div class='del'>-	 * Note that these callers need to pass in a port number.</div><div class='del'>-	 */</div><div class='del'>-	if (qp_init_attr-&gt;cap.max_rdma_ctxs)</div><div class='del'>-		rdma_rw_init_qp(device, qp_init_attr);</div><div class='add'>+	if (!dev-&gt;ops.create_qp)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='ctx'> </div><div class='del'>-	qp = _ib_create_qp(device, pd, qp_init_attr, NULL, NULL);</div><div class='del'>-	if (IS_ERR(qp))</div><div class='del'>-		return qp;</div><div class='add'>+	qp = rdma_zalloc_drv_obj_numa(dev, ib_qp);</div><div class='add'>+	if (!qp)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='del'>-	ret = ib_create_qp_security(qp, device);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err;</div><div class='add'>+	qp-&gt;device = dev;</div><div class='add'>+	qp-&gt;pd = pd;</div><div class='add'>+	qp-&gt;uobject = uobj;</div><div class='add'>+	qp-&gt;real_qp = qp;</div><div class='ctx'> </div><div class='del'>-	qp-&gt;qp_type    = qp_init_attr-&gt;qp_type;</div><div class='del'>-	qp-&gt;rwq_ind_tbl = qp_init_attr-&gt;rwq_ind_tbl;</div><div class='add'>+	qp-&gt;qp_type = attr-&gt;qp_type;</div><div class='add'>+	qp-&gt;rwq_ind_tbl = attr-&gt;rwq_ind_tbl;</div><div class='add'>+	qp-&gt;srq = attr-&gt;srq;</div><div class='add'>+	qp-&gt;event_handler = __ib_qp_event_handler;</div><div class='add'>+	qp-&gt;registered_event_handler = attr-&gt;event_handler;</div><div class='add'>+	qp-&gt;port = attr-&gt;port_num;</div><div class='add'>+	qp-&gt;qp_context = attr-&gt;qp_context;</div><div class='ctx'> </div><div class='del'>-	atomic_set(&amp;qp-&gt;usecnt, 0);</div><div class='del'>-	qp-&gt;mrs_used = 0;</div><div class='ctx'> 	spin_lock_init(&amp;qp-&gt;mr_lock);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;qp-&gt;rdma_mrs);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;qp-&gt;sig_mrs);</div><div class='del'>-	qp-&gt;port = 0;</div><div class='add'>+	init_completion(&amp;qp-&gt;srq_completion);</div><div class='ctx'> </div><div class='del'>-	if (qp_init_attr-&gt;qp_type == IB_QPT_XRC_TGT) {</div><div class='del'>-		struct ib_qp *xrc_qp =</div><div class='del'>-			create_xrc_qp_user(qp, qp_init_attr, udata);</div><div class='add'>+	qp-&gt;send_cq = attr-&gt;send_cq;</div><div class='add'>+	qp-&gt;recv_cq = attr-&gt;recv_cq;</div><div class='ctx'> </div><div class='del'>-		if (IS_ERR(xrc_qp)) {</div><div class='del'>-			ret = PTR_ERR(xrc_qp);</div><div class='del'>-			goto err;</div><div class='del'>-		}</div><div class='del'>-		return xrc_qp;</div><div class='del'>-	}</div><div class='add'>+	rdma_restrack_new(&amp;qp-&gt;res, RDMA_RESTRACK_QP);</div><div class='add'>+	WARN_ONCE(!udata &amp;&amp; !caller, "Missing kernel QP owner");</div><div class='add'>+	rdma_restrack_set_name(&amp;qp-&gt;res, udata ? NULL : caller);</div><div class='add'>+	ret = dev-&gt;ops.create_qp(qp, attr, udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_create;</div><div class='ctx'> </div><div class='del'>-	qp-&gt;event_handler = qp_init_attr-&gt;event_handler;</div><div class='del'>-	qp-&gt;qp_context = qp_init_attr-&gt;qp_context;</div><div class='del'>-	if (qp_init_attr-&gt;qp_type == IB_QPT_XRC_INI) {</div><div class='del'>-		qp-&gt;recv_cq = NULL;</div><div class='del'>-		qp-&gt;srq = NULL;</div><div class='del'>-	} else {</div><div class='del'>-		qp-&gt;recv_cq = qp_init_attr-&gt;recv_cq;</div><div class='del'>-		if (qp_init_attr-&gt;recv_cq)</div><div class='del'>-			atomic_inc(&amp;qp_init_attr-&gt;recv_cq-&gt;usecnt);</div><div class='del'>-		qp-&gt;srq = qp_init_attr-&gt;srq;</div><div class='del'>-		if (qp-&gt;srq)</div><div class='del'>-			atomic_inc(&amp;qp_init_attr-&gt;srq-&gt;usecnt);</div><div class='add'>+	/*</div><div class='add'>+	 * TODO: The mlx4 internally overwrites send_cq and recv_cq.</div><div class='add'>+	 * Unfortunately, it is not an easy task to fix that driver.</div><div class='add'>+	 */</div><div class='add'>+	qp-&gt;send_cq = attr-&gt;send_cq;</div><div class='add'>+	qp-&gt;recv_cq = attr-&gt;recv_cq;</div><div class='add'>+</div><div class='add'>+	ret = ib_create_qp_security(qp, dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_security;</div><div class='add'>+</div><div class='add'>+	rdma_restrack_add(&amp;qp-&gt;res);</div><div class='add'>+	return qp;</div><div class='add'>+</div><div class='add'>+err_security:</div><div class='add'>+	qp-&gt;device-&gt;ops.destroy_qp(qp, udata ? &amp;dummy : NULL);</div><div class='add'>+err_create:</div><div class='add'>+	rdma_restrack_put(&amp;qp-&gt;res);</div><div class='add'>+	kfree(qp);</div><div class='add'>+	return ERR_PTR(ret);</div><div class='add'>+</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ib_create_qp_user - Creates a QP associated with the specified protection</div><div class='add'>+ *   domain.</div><div class='add'>+ * @dev: IB device</div><div class='add'>+ * @pd: The protection domain associated with the QP.</div><div class='add'>+ * @attr: A list of initial attributes required to create the</div><div class='add'>+ *   QP.  If QP creation succeeds, then the attributes are updated to</div><div class='add'>+ *   the actual capabilities of the created QP.</div><div class='add'>+ * @udata: User data</div><div class='add'>+ * @uobj: uverbs obect</div><div class='add'>+ * @caller: caller's build-time module name</div><div class='add'>+ */</div><div class='add'>+struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd,</div><div class='add'>+				struct ib_qp_init_attr *attr,</div><div class='add'>+				struct ib_udata *udata,</div><div class='add'>+				struct ib_uqp_object *uobj, const char *caller)</div><div class='add'>+{</div><div class='add'>+	struct ib_qp *qp, *xrc_qp;</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;qp_type == IB_QPT_XRC_TGT)</div><div class='add'>+		qp = create_qp(dev, pd, attr, NULL, NULL, caller);</div><div class='add'>+	else</div><div class='add'>+		qp = create_qp(dev, pd, attr, udata, uobj, NULL);</div><div class='add'>+	if (attr-&gt;qp_type != IB_QPT_XRC_TGT || IS_ERR(qp))</div><div class='add'>+		return qp;</div><div class='add'>+</div><div class='add'>+	xrc_qp = create_xrc_qp_user(qp, attr);</div><div class='add'>+	if (IS_ERR(xrc_qp)) {</div><div class='add'>+		ib_destroy_qp(qp);</div><div class='add'>+		return xrc_qp;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	qp-&gt;send_cq = qp_init_attr-&gt;send_cq;</div><div class='del'>-	qp-&gt;xrcd    = NULL;</div><div class='add'>+	xrc_qp-&gt;uobject = uobj;</div><div class='add'>+	return xrc_qp;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_create_qp_user);</div><div class='ctx'> </div><div class='del'>-	atomic_inc(&amp;pd-&gt;usecnt);</div><div class='del'>-	if (qp_init_attr-&gt;send_cq)</div><div class='del'>-		atomic_inc(&amp;qp_init_attr-&gt;send_cq-&gt;usecnt);</div><div class='del'>-	if (qp_init_attr-&gt;rwq_ind_tbl)</div><div class='add'>+void ib_qp_usecnt_inc(struct ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	if (qp-&gt;pd)</div><div class='add'>+		atomic_inc(&amp;qp-&gt;pd-&gt;usecnt);</div><div class='add'>+	if (qp-&gt;send_cq)</div><div class='add'>+		atomic_inc(&amp;qp-&gt;send_cq-&gt;usecnt);</div><div class='add'>+	if (qp-&gt;recv_cq)</div><div class='add'>+		atomic_inc(&amp;qp-&gt;recv_cq-&gt;usecnt);</div><div class='add'>+	if (qp-&gt;srq)</div><div class='add'>+		atomic_inc(&amp;qp-&gt;srq-&gt;usecnt);</div><div class='add'>+	if (qp-&gt;rwq_ind_tbl)</div><div class='ctx'> 		atomic_inc(&amp;qp-&gt;rwq_ind_tbl-&gt;usecnt);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_qp_usecnt_inc);</div><div class='add'>+</div><div class='add'>+void ib_qp_usecnt_dec(struct ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	if (qp-&gt;rwq_ind_tbl)</div><div class='add'>+		atomic_dec(&amp;qp-&gt;rwq_ind_tbl-&gt;usecnt);</div><div class='add'>+	if (qp-&gt;srq)</div><div class='add'>+		atomic_dec(&amp;qp-&gt;srq-&gt;usecnt);</div><div class='add'>+	if (qp-&gt;recv_cq)</div><div class='add'>+		atomic_dec(&amp;qp-&gt;recv_cq-&gt;usecnt);</div><div class='add'>+	if (qp-&gt;send_cq)</div><div class='add'>+		atomic_dec(&amp;qp-&gt;send_cq-&gt;usecnt);</div><div class='add'>+	if (qp-&gt;pd)</div><div class='add'>+		atomic_dec(&amp;qp-&gt;pd-&gt;usecnt);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_qp_usecnt_dec);</div><div class='add'>+</div><div class='add'>+struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd,</div><div class='add'>+				  struct ib_qp_init_attr *qp_init_attr,</div><div class='add'>+				  const char *caller)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *device = pd-&gt;device;</div><div class='add'>+	struct ib_qp *qp;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * If the callers is using the RDMA API calculate the resources</div><div class='add'>+	 * needed for the RDMA READ/WRITE operations.</div><div class='add'>+	 *</div><div class='add'>+	 * Note that these callers need to pass in a port number.</div><div class='add'>+	 */</div><div class='add'>+	if (qp_init_attr-&gt;cap.max_rdma_ctxs)</div><div class='add'>+		rdma_rw_init_qp(device, qp_init_attr);</div><div class='add'>+</div><div class='add'>+	qp = create_qp(device, pd, qp_init_attr, NULL, NULL, caller);</div><div class='add'>+	if (IS_ERR(qp))</div><div class='add'>+		return qp;</div><div class='add'>+</div><div class='add'>+	ib_qp_usecnt_inc(qp);</div><div class='ctx'> </div><div class='ctx'> 	if (qp_init_attr-&gt;cap.max_rdma_ctxs) {</div><div class='ctx'> 		ret = rdma_rw_init_mrs(qp, qp_init_attr);</div><div class='hunk'>@@ -1232,6 +1381,8 @@ struct ib_qp *ib_create_qp_user(struct ib_pd *pd,</div><div class='ctx'> 	qp-&gt;max_write_sge = qp_init_attr-&gt;cap.max_send_sge;</div><div class='ctx'> 	qp-&gt;max_read_sge = min_t(u32, qp_init_attr-&gt;cap.max_send_sge,</div><div class='ctx'> 				 device-&gt;attrs.max_sge_rd);</div><div class='add'>+	if (qp_init_attr-&gt;create_flags &amp; IB_QP_CREATE_INTEGRITY_EN)</div><div class='add'>+		qp-&gt;integrity_en = true;</div><div class='ctx'> </div><div class='ctx'> 	return qp;</div><div class='ctx'> </div><div class='hunk'>@@ -1240,7 +1391,7 @@ err:</div><div class='ctx'> 	return ERR_PTR(ret);</div><div class='ctx'> </div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(ib_create_qp_user);</div><div class='add'>+EXPORT_SYMBOL(ib_create_qp_kernel);</div><div class='ctx'> </div><div class='ctx'> static const struct {</div><div class='ctx'> 	int			valid;</div><div class='hunk'>@@ -1613,22 +1764,48 @@ static bool is_qp_type_connected(const struct ib_qp *qp)</div><div class='ctx'> 		qp-&gt;qp_type == IB_QPT_XRC_TGT);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/**</div><div class='add'>+/*</div><div class='ctx'>  * IB core internal function to perform QP attributes modification.</div><div class='ctx'>  */</div><div class='ctx'> static int _ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,</div><div class='ctx'> 			 int attr_mask, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	u8 port = attr_mask &amp; IB_QP_PORT ? attr-&gt;port_num : qp-&gt;port;</div><div class='add'>+	u32 port = attr_mask &amp; IB_QP_PORT ? attr-&gt;port_num : qp-&gt;port;</div><div class='ctx'> 	const struct ib_gid_attr *old_sgid_attr_av;</div><div class='ctx'> 	const struct ib_gid_attr *old_sgid_attr_alt_av;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	attr-&gt;xmit_slave = NULL;</div><div class='ctx'> 	if (attr_mask &amp; IB_QP_AV) {</div><div class='ctx'> 		ret = rdma_fill_sgid_attr(qp-&gt;device, &amp;attr-&gt;ah_attr,</div><div class='ctx'> 					  &amp;old_sgid_attr_av);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			return ret;</div><div class='add'>+</div><div class='add'>+		if (attr-&gt;ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE &amp;&amp;</div><div class='add'>+		    is_qp_type_connected(qp)) {</div><div class='add'>+			struct net_device *slave;</div><div class='add'>+</div><div class='add'>+			/*</div><div class='add'>+			 * If the user provided the qp_attr then we have to</div><div class='add'>+			 * resolve it. Kerne users have to provide already</div><div class='add'>+			 * resolved rdma_ah_attr's.</div><div class='add'>+			 */</div><div class='add'>+			if (udata) {</div><div class='add'>+				ret = ib_resolve_eth_dmac(qp-&gt;device,</div><div class='add'>+							  &amp;attr-&gt;ah_attr);</div><div class='add'>+				if (ret)</div><div class='add'>+					goto out_av;</div><div class='add'>+			}</div><div class='add'>+			slave = rdma_lag_get_ah_roce_slave(qp-&gt;device,</div><div class='add'>+							   &amp;attr-&gt;ah_attr,</div><div class='add'>+							   GFP_KERNEL);</div><div class='add'>+			if (IS_ERR(slave)) {</div><div class='add'>+				ret = PTR_ERR(slave);</div><div class='add'>+				goto out_av;</div><div class='add'>+			}</div><div class='add'>+			attr-&gt;xmit_slave = slave;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> 	if (attr_mask &amp; IB_QP_ALT_PATH) {</div><div class='ctx'> 		/*</div><div class='hunk'>@@ -1650,23 +1827,11 @@ static int _ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,</div><div class='ctx'> 		if (!(rdma_protocol_ib(qp-&gt;device,</div><div class='ctx'> 				       attr-&gt;alt_ah_attr.port_num) &amp;&amp;</div><div class='ctx'> 		      rdma_protocol_ib(qp-&gt;device, port))) {</div><div class='del'>-			ret = EINVAL;</div><div class='add'>+			ret = -EINVAL;</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * If the user provided the qp_attr then we have to resolve it. Kernel</div><div class='del'>-	 * users have to provide already resolved rdma_ah_attr's</div><div class='del'>-	 */</div><div class='del'>-	if (udata &amp;&amp; (attr_mask &amp; IB_QP_AV) &amp;&amp;</div><div class='del'>-	    attr-&gt;ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE &amp;&amp;</div><div class='del'>-	    is_qp_type_connected(qp)) {</div><div class='del'>-		ret = ib_resolve_eth_dmac(qp-&gt;device, &amp;attr-&gt;ah_attr);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	if (rdma_ib_or_roce(qp-&gt;device, port)) {</div><div class='ctx'> 		if (attr_mask &amp; IB_QP_RQ_PSN &amp;&amp; attr-&gt;rq_psn &amp; ~0xffffff) {</div><div class='ctx'> 			dev_warn(&amp;qp-&gt;device-&gt;dev,</div><div class='hunk'>@@ -1683,6 +1848,14 @@ static int _ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	/*</div><div class='add'>+	 * Bind this qp to a counter automatically based on the rdma counter</div><div class='add'>+	 * rules. This only set in RST2INIT with port specified</div><div class='add'>+	 */</div><div class='add'>+	if (!qp-&gt;counter &amp;&amp; (attr_mask &amp; IB_QP_PORT) &amp;&amp;</div><div class='add'>+	    ((attr_mask &amp; IB_QP_STATE) &amp;&amp; attr-&gt;qp_state == IB_QPS_INIT))</div><div class='add'>+		rdma_counter_bind_qp_auto(qp, attr-&gt;port_num);</div><div class='add'>+</div><div class='ctx'> 	ret = ib_security_modify_qp(qp, attr, attr_mask, udata);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto out;</div><div class='hunk'>@@ -1700,8 +1873,10 @@ out:</div><div class='ctx'> 	if (attr_mask &amp; IB_QP_ALT_PATH)</div><div class='ctx'> 		rdma_unfill_sgid_attr(&amp;attr-&gt;alt_ah_attr, old_sgid_attr_alt_av);</div><div class='ctx'> out_av:</div><div class='del'>-	if (attr_mask &amp; IB_QP_AV)</div><div class='add'>+	if (attr_mask &amp; IB_QP_AV) {</div><div class='add'>+		rdma_lag_put_ah_roce_slave(attr-&gt;xmit_slave);</div><div class='ctx'> 		rdma_unfill_sgid_attr(&amp;attr-&gt;ah_attr, old_sgid_attr_av);</div><div class='add'>+	}</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1723,12 +1898,95 @@ int ib_modify_qp_with_udata(struct ib_qp *ib_qp, struct ib_qp_attr *attr,</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_modify_qp_with_udata);</div><div class='ctx'> </div><div class='del'>-int ib_get_eth_speed(struct ib_device *dev, u8 port_num, u8 *speed, u8 *width)</div><div class='add'>+static void ib_get_width_and_speed(u32 netdev_speed, u32 lanes,</div><div class='add'>+				   u16 *speed, u8 *width)</div><div class='add'>+{</div><div class='add'>+	if (!lanes) {</div><div class='add'>+		if (netdev_speed &lt;= SPEED_1000) {</div><div class='add'>+			*width = IB_WIDTH_1X;</div><div class='add'>+			*speed = IB_SPEED_SDR;</div><div class='add'>+		} else if (netdev_speed &lt;= SPEED_10000) {</div><div class='add'>+			*width = IB_WIDTH_1X;</div><div class='add'>+			*speed = IB_SPEED_FDR10;</div><div class='add'>+		} else if (netdev_speed &lt;= SPEED_20000) {</div><div class='add'>+			*width = IB_WIDTH_4X;</div><div class='add'>+			*speed = IB_SPEED_DDR;</div><div class='add'>+		} else if (netdev_speed &lt;= SPEED_25000) {</div><div class='add'>+			*width = IB_WIDTH_1X;</div><div class='add'>+			*speed = IB_SPEED_EDR;</div><div class='add'>+		} else if (netdev_speed &lt;= SPEED_40000) {</div><div class='add'>+			*width = IB_WIDTH_4X;</div><div class='add'>+			*speed = IB_SPEED_FDR10;</div><div class='add'>+		} else if (netdev_speed &lt;= SPEED_50000) {</div><div class='add'>+			*width = IB_WIDTH_2X;</div><div class='add'>+			*speed = IB_SPEED_EDR;</div><div class='add'>+		} else if (netdev_speed &lt;= SPEED_100000) {</div><div class='add'>+			*width = IB_WIDTH_4X;</div><div class='add'>+			*speed = IB_SPEED_EDR;</div><div class='add'>+		} else if (netdev_speed &lt;= SPEED_200000) {</div><div class='add'>+			*width = IB_WIDTH_4X;</div><div class='add'>+			*speed = IB_SPEED_HDR;</div><div class='add'>+		} else {</div><div class='add'>+			*width = IB_WIDTH_4X;</div><div class='add'>+			*speed = IB_SPEED_NDR;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (lanes) {</div><div class='add'>+	case 1:</div><div class='add'>+		*width = IB_WIDTH_1X;</div><div class='add'>+		break;</div><div class='add'>+	case 2:</div><div class='add'>+		*width = IB_WIDTH_2X;</div><div class='add'>+		break;</div><div class='add'>+	case 4:</div><div class='add'>+		*width = IB_WIDTH_4X;</div><div class='add'>+		break;</div><div class='add'>+	case 8:</div><div class='add'>+		*width = IB_WIDTH_8X;</div><div class='add'>+		break;</div><div class='add'>+	case 12:</div><div class='add'>+		*width = IB_WIDTH_12X;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		*width = IB_WIDTH_1X;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (netdev_speed / lanes) {</div><div class='add'>+	case SPEED_2500:</div><div class='add'>+		*speed = IB_SPEED_SDR;</div><div class='add'>+		break;</div><div class='add'>+	case SPEED_5000:</div><div class='add'>+		*speed = IB_SPEED_DDR;</div><div class='add'>+		break;</div><div class='add'>+	case SPEED_10000:</div><div class='add'>+		*speed = IB_SPEED_FDR10;</div><div class='add'>+		break;</div><div class='add'>+	case SPEED_14000:</div><div class='add'>+		*speed = IB_SPEED_FDR;</div><div class='add'>+		break;</div><div class='add'>+	case SPEED_25000:</div><div class='add'>+		*speed = IB_SPEED_EDR;</div><div class='add'>+		break;</div><div class='add'>+	case SPEED_50000:</div><div class='add'>+		*speed = IB_SPEED_HDR;</div><div class='add'>+		break;</div><div class='add'>+	case SPEED_100000:</div><div class='add'>+		*speed = IB_SPEED_NDR;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		*speed = IB_SPEED_SDR;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u16 *speed, u8 *width)</div><div class='ctx'> {</div><div class='ctx'> 	int rc;</div><div class='ctx'> 	u32 netdev_speed;</div><div class='ctx'> 	struct net_device *netdev;</div><div class='del'>-	struct ethtool_link_ksettings lksettings;</div><div class='add'>+	struct ethtool_link_ksettings lksettings = {};</div><div class='ctx'> </div><div class='ctx'> 	if (rdma_port_get_link_layer(dev, port_num) != IB_LINK_LAYER_ETHERNET)</div><div class='ctx'> 		return -EINVAL;</div><div class='hunk'>@@ -1743,33 +2001,17 @@ int ib_get_eth_speed(struct ib_device *dev, u8 port_num, u8 *speed, u8 *width)</div><div class='ctx'> </div><div class='ctx'> 	dev_put(netdev);</div><div class='ctx'> </div><div class='del'>-	if (!rc) {</div><div class='add'>+	if (!rc &amp;&amp; lksettings.base.speed != (u32)SPEED_UNKNOWN) {</div><div class='ctx'> 		netdev_speed = lksettings.base.speed;</div><div class='ctx'> 	} else {</div><div class='ctx'> 		netdev_speed = SPEED_1000;</div><div class='del'>-		pr_warn("%s speed is unknown, defaulting to %d\n", netdev-&gt;name,</div><div class='del'>-			netdev_speed);</div><div class='add'>+		if (rc)</div><div class='add'>+			pr_warn("%s speed is unknown, defaulting to %u\n",</div><div class='add'>+				netdev-&gt;name, netdev_speed);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (netdev_speed &lt;= SPEED_1000) {</div><div class='del'>-		*width = IB_WIDTH_1X;</div><div class='del'>-		*speed = IB_SPEED_SDR;</div><div class='del'>-	} else if (netdev_speed &lt;= SPEED_10000) {</div><div class='del'>-		*width = IB_WIDTH_1X;</div><div class='del'>-		*speed = IB_SPEED_FDR10;</div><div class='del'>-	} else if (netdev_speed &lt;= SPEED_20000) {</div><div class='del'>-		*width = IB_WIDTH_4X;</div><div class='del'>-		*speed = IB_SPEED_DDR;</div><div class='del'>-	} else if (netdev_speed &lt;= SPEED_25000) {</div><div class='del'>-		*width = IB_WIDTH_1X;</div><div class='del'>-		*speed = IB_SPEED_EDR;</div><div class='del'>-	} else if (netdev_speed &lt;= SPEED_40000) {</div><div class='del'>-		*width = IB_WIDTH_4X;</div><div class='del'>-		*speed = IB_SPEED_FDR10;</div><div class='del'>-	} else {</div><div class='del'>-		*width = IB_WIDTH_4X;</div><div class='del'>-		*speed = IB_SPEED_EDR;</div><div class='del'>-	}</div><div class='add'>+	ib_get_width_and_speed(netdev_speed, lksettings.lanes,</div><div class='add'>+			       speed, width);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -1806,9 +2048,9 @@ int ib_close_qp(struct ib_qp *qp)</div><div class='ctx'> 	if (real_qp == qp)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	spin_lock_irqsave(&amp;real_qp-&gt;device-&gt;event_handler_lock, flags);</div><div class='add'>+	spin_lock_irqsave(&amp;real_qp-&gt;device-&gt;qp_open_list_lock, flags);</div><div class='ctx'> 	list_del(&amp;qp-&gt;open_list);</div><div class='del'>-	spin_unlock_irqrestore(&amp;real_qp-&gt;device-&gt;event_handler_lock, flags);</div><div class='add'>+	spin_unlock_irqrestore(&amp;real_qp-&gt;device-&gt;qp_open_list_lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	atomic_dec(&amp;real_qp-&gt;usecnt);</div><div class='ctx'> 	if (qp-&gt;qp_sec)</div><div class='hunk'>@@ -1827,21 +2069,18 @@ static int __ib_destroy_shared_qp(struct ib_qp *qp)</div><div class='ctx'> </div><div class='ctx'> 	real_qp = qp-&gt;real_qp;</div><div class='ctx'> 	xrcd = real_qp-&gt;xrcd;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;xrcd-&gt;tgt_qp_mutex);</div><div class='add'>+	down_write(&amp;xrcd-&gt;tgt_qps_rwsem);</div><div class='ctx'> 	ib_close_qp(qp);</div><div class='ctx'> 	if (atomic_read(&amp;real_qp-&gt;usecnt) == 0)</div><div class='del'>-		list_del(&amp;real_qp-&gt;xrcd_list);</div><div class='add'>+		xa_erase(&amp;xrcd-&gt;tgt_qps, real_qp-&gt;qp_num);</div><div class='ctx'> 	else</div><div class='ctx'> 		real_qp = NULL;</div><div class='del'>-	mutex_unlock(&amp;xrcd-&gt;tgt_qp_mutex);</div><div class='add'>+	up_write(&amp;xrcd-&gt;tgt_qps_rwsem);</div><div class='ctx'> </div><div class='ctx'> 	if (real_qp) {</div><div class='ctx'> 		ret = ib_destroy_qp(real_qp);</div><div class='ctx'> 		if (!ret)</div><div class='ctx'> 			atomic_dec(&amp;xrcd-&gt;usecnt);</div><div class='del'>-		else</div><div class='del'>-			__ib_insert_xrcd_qp(xrcd, real_qp);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='hunk'>@@ -1851,10 +2090,6 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	const struct ib_gid_attr *alt_path_sgid_attr = qp-&gt;alt_path_sgid_attr;</div><div class='ctx'> 	const struct ib_gid_attr *av_sgid_attr = qp-&gt;av_sgid_attr;</div><div class='del'>-	struct ib_pd *pd;</div><div class='del'>-	struct ib_cq *scq, *rcq;</div><div class='del'>-	struct ib_srq *srq;</div><div class='del'>-	struct ib_rwq_ind_table *ind_tbl;</div><div class='ctx'> 	struct ib_qp_security *sec;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='hunk'>@@ -1866,11 +2101,6 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata)</div><div class='ctx'> 	if (qp-&gt;real_qp != qp)</div><div class='ctx'> 		return __ib_destroy_shared_qp(qp);</div><div class='ctx'> </div><div class='del'>-	pd   = qp-&gt;pd;</div><div class='del'>-	scq  = qp-&gt;send_cq;</div><div class='del'>-	rcq  = qp-&gt;recv_cq;</div><div class='del'>-	srq  = qp-&gt;srq;</div><div class='del'>-	ind_tbl = qp-&gt;rwq_ind_tbl;</div><div class='ctx'> 	sec  = qp-&gt;qp_sec;</div><div class='ctx'> 	if (sec)</div><div class='ctx'> 		ib_destroy_qp_security_begin(sec);</div><div class='hunk'>@@ -1878,30 +2108,25 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata)</div><div class='ctx'> 	if (!qp-&gt;uobject)</div><div class='ctx'> 		rdma_rw_cleanup_mrs(qp);</div><div class='ctx'> </div><div class='del'>-	rdma_restrack_del(&amp;qp-&gt;res);</div><div class='add'>+	rdma_counter_unbind_qp(qp, qp-&gt;port, true);</div><div class='ctx'> 	ret = qp-&gt;device-&gt;ops.destroy_qp(qp, udata);</div><div class='del'>-	if (!ret) {</div><div class='del'>-		if (alt_path_sgid_attr)</div><div class='del'>-			rdma_put_gid_attr(alt_path_sgid_attr);</div><div class='del'>-		if (av_sgid_attr)</div><div class='del'>-			rdma_put_gid_attr(av_sgid_attr);</div><div class='del'>-		if (pd)</div><div class='del'>-			atomic_dec(&amp;pd-&gt;usecnt);</div><div class='del'>-		if (scq)</div><div class='del'>-			atomic_dec(&amp;scq-&gt;usecnt);</div><div class='del'>-		if (rcq)</div><div class='del'>-			atomic_dec(&amp;rcq-&gt;usecnt);</div><div class='del'>-		if (srq)</div><div class='del'>-			atomic_dec(&amp;srq-&gt;usecnt);</div><div class='del'>-		if (ind_tbl)</div><div class='del'>-			atomic_dec(&amp;ind_tbl-&gt;usecnt);</div><div class='del'>-		if (sec)</div><div class='del'>-			ib_destroy_qp_security_end(sec);</div><div class='del'>-	} else {</div><div class='add'>+	if (ret) {</div><div class='ctx'> 		if (sec)</div><div class='ctx'> 			ib_destroy_qp_security_abort(sec);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (alt_path_sgid_attr)</div><div class='add'>+		rdma_put_gid_attr(alt_path_sgid_attr);</div><div class='add'>+	if (av_sgid_attr)</div><div class='add'>+		rdma_put_gid_attr(av_sgid_attr);</div><div class='add'>+</div><div class='add'>+	ib_qp_usecnt_dec(qp);</div><div class='add'>+	if (sec)</div><div class='add'>+		ib_destroy_qp_security_end(sec);</div><div class='add'>+</div><div class='add'>+	rdma_restrack_del(&amp;qp-&gt;res);</div><div class='add'>+	kfree(qp);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_destroy_qp_user);</div><div class='hunk'>@@ -1916,27 +2141,39 @@ struct ib_cq *__ib_create_cq(struct ib_device *device,</div><div class='ctx'> 			     const char *caller)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_cq *cq;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	cq = rdma_zalloc_drv_obj(device, ib_cq);</div><div class='add'>+	if (!cq)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	cq-&gt;device = device;</div><div class='add'>+	cq-&gt;uobject = NULL;</div><div class='add'>+	cq-&gt;comp_handler = comp_handler;</div><div class='add'>+	cq-&gt;event_handler = event_handler;</div><div class='add'>+	cq-&gt;cq_context = cq_context;</div><div class='add'>+	atomic_set(&amp;cq-&gt;usecnt, 0);</div><div class='add'>+</div><div class='add'>+	rdma_restrack_new(&amp;cq-&gt;res, RDMA_RESTRACK_CQ);</div><div class='add'>+	rdma_restrack_set_name(&amp;cq-&gt;res, caller);</div><div class='ctx'> </div><div class='del'>-	cq = device-&gt;ops.create_cq(device, cq_attr, NULL);</div><div class='del'>-</div><div class='del'>-	if (!IS_ERR(cq)) {</div><div class='del'>-		cq-&gt;device        = device;</div><div class='del'>-		cq-&gt;uobject       = NULL;</div><div class='del'>-		cq-&gt;comp_handler  = comp_handler;</div><div class='del'>-		cq-&gt;event_handler = event_handler;</div><div class='del'>-		cq-&gt;cq_context    = cq_context;</div><div class='del'>-		atomic_set(&amp;cq-&gt;usecnt, 0);</div><div class='del'>-		cq-&gt;res.type = RDMA_RESTRACK_CQ;</div><div class='del'>-		rdma_restrack_set_task(&amp;cq-&gt;res, caller);</div><div class='del'>-		rdma_restrack_kadd(&amp;cq-&gt;res);</div><div class='add'>+	ret = device-&gt;ops.create_cq(cq, cq_attr, NULL);</div><div class='add'>+	if (ret) {</div><div class='add'>+		rdma_restrack_put(&amp;cq-&gt;res);</div><div class='add'>+		kfree(cq);</div><div class='add'>+		return ERR_PTR(ret);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	rdma_restrack_add(&amp;cq-&gt;res);</div><div class='ctx'> 	return cq;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(__ib_create_cq);</div><div class='ctx'> </div><div class='ctx'> int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period)</div><div class='ctx'> {</div><div class='add'>+	if (cq-&gt;shared)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='ctx'> 	return cq-&gt;device-&gt;ops.modify_cq ?</div><div class='ctx'> 		cq-&gt;device-&gt;ops.modify_cq(cq, cq_count,</div><div class='ctx'> 					  cq_period) : -EOPNOTSUPP;</div><div class='hunk'>@@ -1945,16 +2182,29 @@ EXPORT_SYMBOL(rdma_set_cq_moderation);</div><div class='ctx'> </div><div class='ctx'> int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (WARN_ON_ONCE(cq-&gt;shared))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='ctx'> 	if (atomic_read(&amp;cq-&gt;usecnt))</div><div class='ctx'> 		return -EBUSY;</div><div class='ctx'> </div><div class='add'>+	ret = cq-&gt;device-&gt;ops.destroy_cq(cq, udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='ctx'> 	rdma_restrack_del(&amp;cq-&gt;res);</div><div class='del'>-	return cq-&gt;device-&gt;ops.destroy_cq(cq, udata);</div><div class='add'>+	kfree(cq);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_destroy_cq_user);</div><div class='ctx'> </div><div class='ctx'> int ib_resize_cq(struct ib_cq *cq, int cqe)</div><div class='ctx'> {</div><div class='add'>+	if (cq-&gt;shared)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='ctx'> 	return cq-&gt;device-&gt;ops.resize_cq ?</div><div class='ctx'> 		cq-&gt;device-&gt;ops.resize_cq(cq, cqe, NULL) : -EOPNOTSUPP;</div><div class='ctx'> }</div><div class='hunk'>@@ -1962,18 +2212,73 @@ EXPORT_SYMBOL(ib_resize_cq);</div><div class='ctx'> </div><div class='ctx'> /* Memory regions */</div><div class='ctx'> </div><div class='add'>+struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='add'>+			     u64 virt_addr, int access_flags)</div><div class='add'>+{</div><div class='add'>+	struct ib_mr *mr;</div><div class='add'>+</div><div class='add'>+	if (access_flags &amp; IB_ACCESS_ON_DEMAND) {</div><div class='add'>+		if (!(pd-&gt;device-&gt;attrs.kernel_cap_flags &amp;</div><div class='add'>+		      IBK_ON_DEMAND_PAGING)) {</div><div class='add'>+			pr_debug("ODP support not available\n");</div><div class='add'>+			return ERR_PTR(-EINVAL);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mr = pd-&gt;device-&gt;ops.reg_user_mr(pd, start, length, virt_addr,</div><div class='add'>+					 access_flags, NULL, NULL);</div><div class='add'>+</div><div class='add'>+	if (IS_ERR(mr))</div><div class='add'>+		return mr;</div><div class='add'>+</div><div class='add'>+	mr-&gt;device = pd-&gt;device;</div><div class='add'>+	mr-&gt;type = IB_MR_TYPE_USER;</div><div class='add'>+	mr-&gt;pd = pd;</div><div class='add'>+	mr-&gt;dm = NULL;</div><div class='add'>+	atomic_inc(&amp;pd-&gt;usecnt);</div><div class='add'>+	mr-&gt;iova =  virt_addr;</div><div class='add'>+	mr-&gt;length = length;</div><div class='add'>+</div><div class='add'>+	rdma_restrack_new(&amp;mr-&gt;res, RDMA_RESTRACK_MR);</div><div class='add'>+	rdma_restrack_parent_name(&amp;mr-&gt;res, &amp;pd-&gt;res);</div><div class='add'>+	rdma_restrack_add(&amp;mr-&gt;res);</div><div class='add'>+</div><div class='add'>+	return mr;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_reg_user_mr);</div><div class='add'>+</div><div class='add'>+int ib_advise_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice,</div><div class='add'>+		 u32 flags, struct ib_sge *sg_list, u32 num_sge)</div><div class='add'>+{</div><div class='add'>+	if (!pd-&gt;device-&gt;ops.advise_mr)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (!num_sge)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	return pd-&gt;device-&gt;ops.advise_mr(pd, advice, flags, sg_list, num_sge,</div><div class='add'>+					 NULL);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_advise_mr);</div><div class='add'>+</div><div class='ctx'> int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_pd *pd = mr-&gt;pd;</div><div class='ctx'> 	struct ib_dm *dm = mr-&gt;dm;</div><div class='add'>+	struct ib_dmah *dmah = mr-&gt;dmah;</div><div class='add'>+	struct ib_sig_attrs *sig_attrs = mr-&gt;sig_attrs;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	trace_mr_dereg(mr);</div><div class='ctx'> 	rdma_restrack_del(&amp;mr-&gt;res);</div><div class='ctx'> 	ret = mr-&gt;device-&gt;ops.dereg_mr(mr, udata);</div><div class='ctx'> 	if (!ret) {</div><div class='ctx'> 		atomic_dec(&amp;pd-&gt;usecnt);</div><div class='ctx'> 		if (dm)</div><div class='ctx'> 			atomic_dec(&amp;dm-&gt;usecnt);</div><div class='add'>+		if (dmah)</div><div class='add'>+			atomic_dec(&amp;dmah-&gt;usecnt);</div><div class='add'>+		kfree(sig_attrs);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -1985,7 +2290,6 @@ EXPORT_SYMBOL(ib_dereg_mr_user);</div><div class='ctx'>  * @pd:            protection domain associated with the region</div><div class='ctx'>  * @mr_type:       memory region type</div><div class='ctx'>  * @max_num_sg:    maximum sg entries available for registration.</div><div class='del'>- * @udata:	   user data or null for kernel objects</div><div class='ctx'>  *</div><div class='ctx'>  * Notes:</div><div class='ctx'>  * Memory registeration page/sg lists must not exceed max_num_sg.</div><div class='hunk'>@@ -1993,77 +2297,104 @@ EXPORT_SYMBOL(ib_dereg_mr_user);</div><div class='ctx'>  * max_num_sg * used_page_size.</div><div class='ctx'>  *</div><div class='ctx'>  */</div><div class='del'>-struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type,</div><div class='del'>-			       u32 max_num_sg, struct ib_udata *udata)</div><div class='add'>+struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,</div><div class='add'>+			  u32 max_num_sg)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_mr *mr;</div><div class='ctx'> </div><div class='del'>-	if (!pd-&gt;device-&gt;ops.alloc_mr)</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+	if (!pd-&gt;device-&gt;ops.alloc_mr) {</div><div class='add'>+		mr = ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	mr = pd-&gt;device-&gt;ops.alloc_mr(pd, mr_type, max_num_sg, udata);</div><div class='del'>-	if (!IS_ERR(mr)) {</div><div class='del'>-		mr-&gt;device  = pd-&gt;device;</div><div class='del'>-		mr-&gt;pd      = pd;</div><div class='del'>-		mr-&gt;dm      = NULL;</div><div class='del'>-		mr-&gt;uobject = NULL;</div><div class='del'>-		atomic_inc(&amp;pd-&gt;usecnt);</div><div class='del'>-		mr-&gt;need_inval = false;</div><div class='del'>-		mr-&gt;res.type = RDMA_RESTRACK_MR;</div><div class='del'>-		rdma_restrack_kadd(&amp;mr-&gt;res);</div><div class='add'>+	if (mr_type == IB_MR_TYPE_INTEGRITY) {</div><div class='add'>+		WARN_ON_ONCE(1);</div><div class='add'>+		mr = ERR_PTR(-EINVAL);</div><div class='add'>+		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	mr = pd-&gt;device-&gt;ops.alloc_mr(pd, mr_type, max_num_sg);</div><div class='add'>+	if (IS_ERR(mr))</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	mr-&gt;device = pd-&gt;device;</div><div class='add'>+	mr-&gt;pd = pd;</div><div class='add'>+	mr-&gt;dm = NULL;</div><div class='add'>+	mr-&gt;uobject = NULL;</div><div class='add'>+	atomic_inc(&amp;pd-&gt;usecnt);</div><div class='add'>+	mr-&gt;need_inval = false;</div><div class='add'>+	mr-&gt;type = mr_type;</div><div class='add'>+	mr-&gt;sig_attrs = NULL;</div><div class='add'>+</div><div class='add'>+	rdma_restrack_new(&amp;mr-&gt;res, RDMA_RESTRACK_MR);</div><div class='add'>+	rdma_restrack_parent_name(&amp;mr-&gt;res, &amp;pd-&gt;res);</div><div class='add'>+	rdma_restrack_add(&amp;mr-&gt;res);</div><div class='add'>+out:</div><div class='add'>+	trace_mr_alloc(pd, mr_type, max_num_sg, mr);</div><div class='ctx'> 	return mr;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(ib_alloc_mr_user);</div><div class='del'>-</div><div class='del'>-/* "Fast" memory regions */</div><div class='add'>+EXPORT_SYMBOL(ib_alloc_mr);</div><div class='ctx'> </div><div class='del'>-struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,</div><div class='del'>-			    int mr_access_flags,</div><div class='del'>-			    struct ib_fmr_attr *fmr_attr)</div><div class='add'>+/**</div><div class='add'>+ * ib_alloc_mr_integrity() - Allocates an integrity memory region</div><div class='add'>+ * @pd:                      protection domain associated with the region</div><div class='add'>+ * @max_num_data_sg:         maximum data sg entries available for registration</div><div class='add'>+ * @max_num_meta_sg:         maximum metadata sg entries available for</div><div class='add'>+ *                           registration</div><div class='add'>+ *</div><div class='add'>+ * Notes:</div><div class='add'>+ * Memory registration page/sg lists must not exceed max_num_sg,</div><div class='add'>+ * also the integrity page/sg lists must not exceed max_num_meta_sg.</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,</div><div class='add'>+				    u32 max_num_data_sg,</div><div class='add'>+				    u32 max_num_meta_sg)</div><div class='ctx'> {</div><div class='del'>-	struct ib_fmr *fmr;</div><div class='del'>-</div><div class='del'>-	if (!pd-&gt;device-&gt;ops.alloc_fmr)</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+	struct ib_mr *mr;</div><div class='add'>+	struct ib_sig_attrs *sig_attrs;</div><div class='ctx'> </div><div class='del'>-	fmr = pd-&gt;device-&gt;ops.alloc_fmr(pd, mr_access_flags, fmr_attr);</div><div class='del'>-	if (!IS_ERR(fmr)) {</div><div class='del'>-		fmr-&gt;device = pd-&gt;device;</div><div class='del'>-		fmr-&gt;pd     = pd;</div><div class='del'>-		atomic_inc(&amp;pd-&gt;usecnt);</div><div class='add'>+	if (!pd-&gt;device-&gt;ops.alloc_mr_integrity ||</div><div class='add'>+	    !pd-&gt;device-&gt;ops.map_mr_sg_pi) {</div><div class='add'>+		mr = ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return fmr;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_alloc_fmr);</div><div class='del'>-</div><div class='del'>-int ib_unmap_fmr(struct list_head *fmr_list)</div><div class='del'>-{</div><div class='del'>-	struct ib_fmr *fmr;</div><div class='del'>-</div><div class='del'>-	if (list_empty(fmr_list))</div><div class='del'>-		return 0;</div><div class='add'>+	if (!max_num_meta_sg) {</div><div class='add'>+		mr = ERR_PTR(-EINVAL);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	fmr = list_entry(fmr_list-&gt;next, struct ib_fmr, list);</div><div class='del'>-	return fmr-&gt;device-&gt;ops.unmap_fmr(fmr_list);</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_unmap_fmr);</div><div class='add'>+	sig_attrs = kzalloc(sizeof(struct ib_sig_attrs), GFP_KERNEL);</div><div class='add'>+	if (!sig_attrs) {</div><div class='add'>+		mr = ERR_PTR(-ENOMEM);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-int ib_dealloc_fmr(struct ib_fmr *fmr)</div><div class='del'>-{</div><div class='del'>-	struct ib_pd *pd;</div><div class='del'>-	int ret;</div><div class='add'>+	mr = pd-&gt;device-&gt;ops.alloc_mr_integrity(pd, max_num_data_sg,</div><div class='add'>+						max_num_meta_sg);</div><div class='add'>+	if (IS_ERR(mr)) {</div><div class='add'>+		kfree(sig_attrs);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	pd = fmr-&gt;pd;</div><div class='del'>-	ret = fmr-&gt;device-&gt;ops.dealloc_fmr(fmr);</div><div class='del'>-	if (!ret)</div><div class='del'>-		atomic_dec(&amp;pd-&gt;usecnt);</div><div class='add'>+	mr-&gt;device = pd-&gt;device;</div><div class='add'>+	mr-&gt;pd = pd;</div><div class='add'>+	mr-&gt;dm = NULL;</div><div class='add'>+	mr-&gt;uobject = NULL;</div><div class='add'>+	atomic_inc(&amp;pd-&gt;usecnt);</div><div class='add'>+	mr-&gt;need_inval = false;</div><div class='add'>+	mr-&gt;type = IB_MR_TYPE_INTEGRITY;</div><div class='add'>+	mr-&gt;sig_attrs = sig_attrs;</div><div class='ctx'> </div><div class='del'>-	return ret;</div><div class='add'>+	rdma_restrack_new(&amp;mr-&gt;res, RDMA_RESTRACK_MR);</div><div class='add'>+	rdma_restrack_parent_name(&amp;mr-&gt;res, &amp;pd-&gt;res);</div><div class='add'>+	rdma_restrack_add(&amp;mr-&gt;res);</div><div class='add'>+out:</div><div class='add'>+	trace_mr_integ_alloc(pd, max_num_data_sg, max_num_meta_sg, mr);</div><div class='add'>+	return mr;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(ib_dealloc_fmr);</div><div class='add'>+EXPORT_SYMBOL(ib_alloc_mr_integrity);</div><div class='ctx'> </div><div class='ctx'> /* Multicast groups */</div><div class='ctx'> </div><div class='hunk'>@@ -2072,7 +2403,7 @@ static bool is_valid_mcast_lid(struct ib_qp *qp, u16 lid)</div><div class='ctx'> 	struct ib_qp_init_attr init_attr = {};</div><div class='ctx'> 	struct ib_qp_attr attr = {};</div><div class='ctx'> 	int num_eth_ports = 0;</div><div class='del'>-	int port;</div><div class='add'>+	unsigned int port;</div><div class='ctx'> </div><div class='ctx'> 	/* If QP state &gt;= init, it is assigned to a port and we can check this</div><div class='ctx'> 	 * port only.</div><div class='hunk'>@@ -2087,7 +2418,7 @@ static bool is_valid_mcast_lid(struct ib_qp *qp, u16 lid)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Can't get a quick answer, iterate over all ports */</div><div class='del'>-	for (port = 0; port &lt; qp-&gt;device-&gt;phys_port_cnt; port++)</div><div class='add'>+	rdma_for_each_port(qp-&gt;device, port)</div><div class='ctx'> 		if (rdma_port_get_link_layer(qp-&gt;device, port) !=</div><div class='ctx'> 		    IB_LINK_LAYER_INFINIBAND)</div><div class='ctx'> 			num_eth_ports++;</div><div class='hunk'>@@ -2141,44 +2472,61 @@ int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_detach_mcast);</div><div class='ctx'> </div><div class='del'>-struct ib_xrcd *__ib_alloc_xrcd(struct ib_device *device, const char *caller)</div><div class='add'>+/**</div><div class='add'>+ * ib_alloc_xrcd_user - Allocates an XRC domain.</div><div class='add'>+ * @device: The device on which to allocate the XRC domain.</div><div class='add'>+ * @inode: inode to connect XRCD</div><div class='add'>+ * @udata: Valid user data or NULL for kernel object</div><div class='add'>+ */</div><div class='add'>+struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device,</div><div class='add'>+				   struct inode *inode, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_xrcd *xrcd;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (!device-&gt;ops.alloc_xrcd)</div><div class='ctx'> 		return ERR_PTR(-EOPNOTSUPP);</div><div class='ctx'> </div><div class='del'>-	xrcd = device-&gt;ops.alloc_xrcd(device, NULL);</div><div class='del'>-	if (!IS_ERR(xrcd)) {</div><div class='del'>-		xrcd-&gt;device = device;</div><div class='del'>-		xrcd-&gt;inode = NULL;</div><div class='del'>-		atomic_set(&amp;xrcd-&gt;usecnt, 0);</div><div class='del'>-		mutex_init(&amp;xrcd-&gt;tgt_qp_mutex);</div><div class='del'>-		INIT_LIST_HEAD(&amp;xrcd-&gt;tgt_qp_list);</div><div class='del'>-	}</div><div class='add'>+	xrcd = rdma_zalloc_drv_obj(device, ib_xrcd);</div><div class='add'>+	if (!xrcd)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='add'>+	xrcd-&gt;device = device;</div><div class='add'>+	xrcd-&gt;inode = inode;</div><div class='add'>+	atomic_set(&amp;xrcd-&gt;usecnt, 0);</div><div class='add'>+	init_rwsem(&amp;xrcd-&gt;tgt_qps_rwsem);</div><div class='add'>+	xa_init(&amp;xrcd-&gt;tgt_qps);</div><div class='add'>+</div><div class='add'>+	ret = device-&gt;ops.alloc_xrcd(xrcd, udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err;</div><div class='ctx'> 	return xrcd;</div><div class='add'>+err:</div><div class='add'>+	kfree(xrcd);</div><div class='add'>+	return ERR_PTR(ret);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(__ib_alloc_xrcd);</div><div class='add'>+EXPORT_SYMBOL(ib_alloc_xrcd_user);</div><div class='ctx'> </div><div class='del'>-int ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)</div><div class='add'>+/**</div><div class='add'>+ * ib_dealloc_xrcd_user - Deallocates an XRC domain.</div><div class='add'>+ * @xrcd: The XRC domain to deallocate.</div><div class='add'>+ * @udata: Valid user data or NULL for kernel object</div><div class='add'>+ */</div><div class='add'>+int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	struct ib_qp *qp;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (atomic_read(&amp;xrcd-&gt;usecnt))</div><div class='ctx'> 		return -EBUSY;</div><div class='ctx'> </div><div class='del'>-	while (!list_empty(&amp;xrcd-&gt;tgt_qp_list)) {</div><div class='del'>-		qp = list_entry(xrcd-&gt;tgt_qp_list.next, struct ib_qp, xrcd_list);</div><div class='del'>-		ret = ib_destroy_qp(qp);</div><div class='del'>-		if (ret)</div><div class='del'>-			return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return xrcd-&gt;device-&gt;ops.dealloc_xrcd(xrcd, udata);</div><div class='add'>+	WARN_ON(!xa_empty(&amp;xrcd-&gt;tgt_qps));</div><div class='add'>+	ret = xrcd-&gt;device-&gt;ops.dealloc_xrcd(xrcd, udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+	kfree(xrcd);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(ib_dealloc_xrcd);</div><div class='add'>+EXPORT_SYMBOL(ib_dealloc_xrcd_user);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * ib_create_wq - Creates a WQ associated with the specified protection</div><div class='hunk'>@@ -2220,110 +2568,28 @@ struct ib_wq *ib_create_wq(struct ib_pd *pd,</div><div class='ctx'> EXPORT_SYMBOL(ib_create_wq);</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * ib_destroy_wq - Destroys the specified user WQ.</div><div class='add'>+ * ib_destroy_wq_user - Destroys the specified user WQ.</div><div class='ctx'>  * @wq: The WQ to destroy.</div><div class='ctx'>  * @udata: Valid user data</div><div class='ctx'>  */</div><div class='del'>-int ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)</div><div class='add'>+int ib_destroy_wq_user(struct ib_wq *wq, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	int err;</div><div class='ctx'> 	struct ib_cq *cq = wq-&gt;cq;</div><div class='ctx'> 	struct ib_pd *pd = wq-&gt;pd;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (atomic_read(&amp;wq-&gt;usecnt))</div><div class='ctx'> 		return -EBUSY;</div><div class='ctx'> </div><div class='del'>-	err = wq-&gt;device-&gt;ops.destroy_wq(wq, udata);</div><div class='del'>-	if (!err) {</div><div class='del'>-		atomic_dec(&amp;pd-&gt;usecnt);</div><div class='del'>-		atomic_dec(&amp;cq-&gt;usecnt);</div><div class='del'>-	}</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_destroy_wq);</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * ib_modify_wq - Modifies the specified WQ.</div><div class='del'>- * @wq: The WQ to modify.</div><div class='del'>- * @wq_attr: On input, specifies the WQ attributes to modify.</div><div class='del'>- * @wq_attr_mask: A bit-mask used to specify which attributes of the WQ</div><div class='del'>- *   are being modified.</div><div class='del'>- * On output, the current values of selected WQ attributes are returned.</div><div class='del'>- */</div><div class='del'>-int ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,</div><div class='del'>-		 u32 wq_attr_mask)</div><div class='del'>-{</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	if (!wq-&gt;device-&gt;ops.modify_wq)</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-	err = wq-&gt;device-&gt;ops.modify_wq(wq, wq_attr, wq_attr_mask, NULL);</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_modify_wq);</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * ib_create_rwq_ind_table - Creates a RQ Indirection Table.</div><div class='del'>- * @device: The device on which to create the rwq indirection table.</div><div class='del'>- * @ib_rwq_ind_table_init_attr: A list of initial attributes required to</div><div class='del'>- * create the Indirection Table.</div><div class='del'>- *</div><div class='del'>- * Note: The life time of ib_rwq_ind_table_init_attr-&gt;ind_tbl is not less</div><div class='del'>- *	than the created ib_rwq_ind_table object and the caller is responsible</div><div class='del'>- *	for its memory allocation/free.</div><div class='del'>- */</div><div class='del'>-struct ib_rwq_ind_table *ib_create_rwq_ind_table(struct ib_device *device,</div><div class='del'>-						 struct ib_rwq_ind_table_init_attr *init_attr)</div><div class='del'>-{</div><div class='del'>-	struct ib_rwq_ind_table *rwq_ind_table;</div><div class='del'>-	int i;</div><div class='del'>-	u32 table_size;</div><div class='del'>-</div><div class='del'>-	if (!device-&gt;ops.create_rwq_ind_table)</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='del'>-</div><div class='del'>-	table_size = (1 &lt;&lt; init_attr-&gt;log_ind_tbl_size);</div><div class='del'>-	rwq_ind_table = device-&gt;ops.create_rwq_ind_table(device,</div><div class='del'>-							 init_attr, NULL);</div><div class='del'>-	if (IS_ERR(rwq_ind_table))</div><div class='del'>-		return rwq_ind_table;</div><div class='del'>-</div><div class='del'>-	rwq_ind_table-&gt;ind_tbl = init_attr-&gt;ind_tbl;</div><div class='del'>-	rwq_ind_table-&gt;log_ind_tbl_size = init_attr-&gt;log_ind_tbl_size;</div><div class='del'>-	rwq_ind_table-&gt;device = device;</div><div class='del'>-	rwq_ind_table-&gt;uobject = NULL;</div><div class='del'>-	atomic_set(&amp;rwq_ind_table-&gt;usecnt, 0);</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; table_size; i++)</div><div class='del'>-		atomic_inc(&amp;rwq_ind_table-&gt;ind_tbl[i]-&gt;usecnt);</div><div class='del'>-</div><div class='del'>-	return rwq_ind_table;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL(ib_create_rwq_ind_table);</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * ib_destroy_rwq_ind_table - Destroys the specified Indirection Table.</div><div class='del'>- * @wq_ind_table: The Indirection Table to destroy.</div><div class='del'>-*/</div><div class='del'>-int ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table)</div><div class='del'>-{</div><div class='del'>-	int err, i;</div><div class='del'>-	u32 table_size = (1 &lt;&lt; rwq_ind_table-&gt;log_ind_tbl_size);</div><div class='del'>-	struct ib_wq **ind_tbl = rwq_ind_table-&gt;ind_tbl;</div><div class='del'>-</div><div class='del'>-	if (atomic_read(&amp;rwq_ind_table-&gt;usecnt))</div><div class='del'>-		return -EBUSY;</div><div class='del'>-</div><div class='del'>-	err = rwq_ind_table-&gt;device-&gt;ops.destroy_rwq_ind_table(rwq_ind_table);</div><div class='del'>-	if (!err) {</div><div class='del'>-		for (i = 0; i &lt; table_size; i++)</div><div class='del'>-			atomic_dec(&amp;ind_tbl[i]-&gt;usecnt);</div><div class='del'>-	}</div><div class='add'>+	ret = wq-&gt;device-&gt;ops.destroy_wq(wq, udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='del'>-	return err;</div><div class='add'>+	atomic_dec(&amp;pd-&gt;usecnt);</div><div class='add'>+	atomic_dec(&amp;cq-&gt;usecnt);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(ib_destroy_rwq_ind_table);</div><div class='add'>+EXPORT_SYMBOL(ib_destroy_wq_user);</div><div class='ctx'> </div><div class='ctx'> int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,</div><div class='ctx'> 		       struct ib_mr_status *mr_status)</div><div class='hunk'>@@ -2335,7 +2601,7 @@ int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_check_mr_status);</div><div class='ctx'> </div><div class='del'>-int ib_set_vf_link_state(struct ib_device *device, int vf, u8 port,</div><div class='add'>+int ib_set_vf_link_state(struct ib_device *device, int vf, u32 port,</div><div class='ctx'> 			 int state)</div><div class='ctx'> {</div><div class='ctx'> 	if (!device-&gt;ops.set_vf_link_state)</div><div class='hunk'>@@ -2345,7 +2611,7 @@ int ib_set_vf_link_state(struct ib_device *device, int vf, u8 port,</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_set_vf_link_state);</div><div class='ctx'> </div><div class='del'>-int ib_get_vf_config(struct ib_device *device, int vf, u8 port,</div><div class='add'>+int ib_get_vf_config(struct ib_device *device, int vf, u32 port,</div><div class='ctx'> 		     struct ifla_vf_info *info)</div><div class='ctx'> {</div><div class='ctx'> 	if (!device-&gt;ops.get_vf_config)</div><div class='hunk'>@@ -2355,7 +2621,7 @@ int ib_get_vf_config(struct ib_device *device, int vf, u8 port,</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_get_vf_config);</div><div class='ctx'> </div><div class='del'>-int ib_get_vf_stats(struct ib_device *device, int vf, u8 port,</div><div class='add'>+int ib_get_vf_stats(struct ib_device *device, int vf, u32 port,</div><div class='ctx'> 		    struct ifla_vf_stats *stats)</div><div class='ctx'> {</div><div class='ctx'> 	if (!device-&gt;ops.get_vf_stats)</div><div class='hunk'>@@ -2365,7 +2631,7 @@ int ib_get_vf_stats(struct ib_device *device, int vf, u8 port,</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_get_vf_stats);</div><div class='ctx'> </div><div class='del'>-int ib_set_vf_guid(struct ib_device *device, int vf, u8 port, u64 guid,</div><div class='add'>+int ib_set_vf_guid(struct ib_device *device, int vf, u32 port, u64 guid,</div><div class='ctx'> 		   int type)</div><div class='ctx'> {</div><div class='ctx'> 	if (!device-&gt;ops.set_vf_guid)</div><div class='hunk'>@@ -2375,6 +2641,53 @@ int ib_set_vf_guid(struct ib_device *device, int vf, u8 port, u64 guid,</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_set_vf_guid);</div><div class='ctx'> </div><div class='add'>+int ib_get_vf_guid(struct ib_device *device, int vf, u32 port,</div><div class='add'>+		   struct ifla_vf_guid *node_guid,</div><div class='add'>+		   struct ifla_vf_guid *port_guid)</div><div class='add'>+{</div><div class='add'>+	if (!device-&gt;ops.get_vf_guid)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	return device-&gt;ops.get_vf_guid(device, vf, port, node_guid, port_guid);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_get_vf_guid);</div><div class='add'>+/**</div><div class='add'>+ * ib_map_mr_sg_pi() - Map the dma mapped SG lists for PI (protection</div><div class='add'>+ *     information) and set an appropriate memory region for registration.</div><div class='add'>+ * @mr:             memory region</div><div class='add'>+ * @data_sg:        dma mapped scatterlist for data</div><div class='add'>+ * @data_sg_nents:  number of entries in data_sg</div><div class='add'>+ * @data_sg_offset: offset in bytes into data_sg</div><div class='add'>+ * @meta_sg:        dma mapped scatterlist for metadata</div><div class='add'>+ * @meta_sg_nents:  number of entries in meta_sg</div><div class='add'>+ * @meta_sg_offset: offset in bytes into meta_sg</div><div class='add'>+ * @page_size:      page vector desired page size</div><div class='add'>+ *</div><div class='add'>+ * Constraints:</div><div class='add'>+ * - The MR must be allocated with type IB_MR_TYPE_INTEGRITY.</div><div class='add'>+ *</div><div class='add'>+ * Return: 0 on success.</div><div class='add'>+ *</div><div class='add'>+ * After this completes successfully, the  memory region</div><div class='add'>+ * is ready for registration.</div><div class='add'>+ */</div><div class='add'>+int ib_map_mr_sg_pi(struct ib_mr *mr, struct scatterlist *data_sg,</div><div class='add'>+		    int data_sg_nents, unsigned int *data_sg_offset,</div><div class='add'>+		    struct scatterlist *meta_sg, int meta_sg_nents,</div><div class='add'>+		    unsigned int *meta_sg_offset, unsigned int page_size)</div><div class='add'>+{</div><div class='add'>+	if (unlikely(!mr-&gt;device-&gt;ops.map_mr_sg_pi ||</div><div class='add'>+		     WARN_ON_ONCE(mr-&gt;type != IB_MR_TYPE_INTEGRITY)))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	mr-&gt;page_size = page_size;</div><div class='add'>+</div><div class='add'>+	return mr-&gt;device-&gt;ops.map_mr_sg_pi(mr, data_sg, data_sg_nents,</div><div class='add'>+					    data_sg_offset, meta_sg,</div><div class='add'>+					    meta_sg_nents, meta_sg_offset);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(ib_map_mr_sg_pi);</div><div class='add'>+</div><div class='ctx'> /**</div><div class='ctx'>  * ib_map_mr_sg() - Map the largest prefix of a dma mapped SG list</div><div class='ctx'>  *     and set it the memory region.</div><div class='hunk'>@@ -2385,6 +2698,7 @@ EXPORT_SYMBOL(ib_set_vf_guid);</div><div class='ctx'>  * @page_size:     page vector desired page size</div><div class='ctx'>  *</div><div class='ctx'>  * Constraints:</div><div class='add'>+ *</div><div class='ctx'>  * - The first sg element is allowed to have an offset.</div><div class='ctx'>  * - Each sg element must either be aligned to page_size or virtually</div><div class='ctx'>  *   contiguous to the previous element. In case an sg element has a</div><div class='hunk'>@@ -2418,10 +2732,12 @@ EXPORT_SYMBOL(ib_map_mr_sg);</div><div class='ctx'>  * @mr:            memory region</div><div class='ctx'>  * @sgl:           dma mapped scatterlist</div><div class='ctx'>  * @sg_nents:      number of entries in sg</div><div class='del'>- * @sg_offset_p:   IN:  start offset in bytes into sg</div><div class='del'>- *                 OUT: offset in bytes for element n of the sg of the first</div><div class='add'>+ * @sg_offset_p:   ==== =======================================================</div><div class='add'>+ *                 IN   start offset in bytes into sg</div><div class='add'>+ *                 OUT  offset in bytes for element n of the sg of the first</div><div class='ctx'>  *                      byte that has not been processed where n is the return</div><div class='ctx'>  *                      value of this function.</div><div class='add'>+ *                 ==== =======================================================</div><div class='ctx'>  * @set_page:      driver page assignment function pointer</div><div class='ctx'>  *</div><div class='ctx'>  * Core service helper for drivers to convert the largest</div><div class='hunk'>@@ -2586,6 +2902,72 @@ static void __ib_drain_rq(struct ib_qp *qp)</div><div class='ctx'> 		wait_for_completion(&amp;rdrain.done);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+/*</div><div class='add'>+ * __ib_drain_srq() - Block until Last WQE Reached event arrives, or timeout</div><div class='add'>+ *                    expires.</div><div class='add'>+ * @qp:               queue pair associated with SRQ to drain</div><div class='add'>+ *</div><div class='add'>+ * Quoting 10.3.1 Queue Pair and EE Context States:</div><div class='add'>+ *</div><div class='add'>+ * Note, for QPs that are associated with an SRQ, the Consumer should take the</div><div class='add'>+ * QP through the Error State before invoking a Destroy QP or a Modify QP to the</div><div class='add'>+ * Reset State.  The Consumer may invoke the Destroy QP without first performing</div><div class='add'>+ * a Modify QP to the Error State and waiting for the Affiliated Asynchronous</div><div class='add'>+ * Last WQE Reached Event. However, if the Consumer does not wait for the</div><div class='add'>+ * Affiliated Asynchronous Last WQE Reached Event, then WQE and Data Segment</div><div class='add'>+ * leakage may occur. Therefore, it is good programming practice to tear down a</div><div class='add'>+ * QP that is associated with an SRQ by using the following process:</div><div class='add'>+ *</div><div class='add'>+ * - Put the QP in the Error State</div><div class='add'>+ * - Wait for the Affiliated Asynchronous Last WQE Reached Event;</div><div class='add'>+ * - either:</div><div class='add'>+ *       drain the CQ by invoking the Poll CQ verb and either wait for CQ</div><div class='add'>+ *       to be empty or the number of Poll CQ operations has exceeded</div><div class='add'>+ *       CQ capacity size;</div><div class='add'>+ * - or</div><div class='add'>+ *       post another WR that completes on the same CQ and wait for this</div><div class='add'>+ *       WR to return as a WC;</div><div class='add'>+ * - and then invoke a Destroy QP or Reset QP.</div><div class='add'>+ *</div><div class='add'>+ * We use the first option.</div><div class='add'>+ */</div><div class='add'>+static void __ib_drain_srq(struct ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };</div><div class='add'>+	struct ib_cq *cq;</div><div class='add'>+	int n, polled = 0;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!qp-&gt;srq) {</div><div class='add'>+		WARN_ONCE(1, "QP 0x%p is not associated with SRQ\n", qp);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = ib_modify_qp(qp, &amp;attr, IB_QP_STATE);</div><div class='add'>+	if (ret) {</div><div class='add'>+		WARN_ONCE(ret, "failed to drain shared recv queue: %d\n", ret);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ib_srq_has_cq(qp-&gt;srq-&gt;srq_type)) {</div><div class='add'>+		cq = qp-&gt;srq-&gt;ext.cq;</div><div class='add'>+	} else if (qp-&gt;recv_cq) {</div><div class='add'>+		cq = qp-&gt;recv_cq;</div><div class='add'>+	} else {</div><div class='add'>+		WARN_ONCE(1, "QP 0x%p has no CQ associated with SRQ\n", qp);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (wait_for_completion_timeout(&amp;qp-&gt;srq_completion, 60 * HZ) &gt; 0) {</div><div class='add'>+		while (polled != cq-&gt;cqe) {</div><div class='add'>+			n = ib_process_cq_direct(cq, cq-&gt;cqe - polled);</div><div class='add'>+			if (!n)</div><div class='add'>+				return;</div><div class='add'>+			polled += n;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /**</div><div class='ctx'>  * ib_drain_sq() - Block until all SQ CQEs have been consumed by the</div><div class='ctx'>  *		   application.</div><div class='hunk'>@@ -2611,6 +2993,7 @@ void ib_drain_sq(struct ib_qp *qp)</div><div class='ctx'> 		qp-&gt;device-&gt;ops.drain_sq(qp);</div><div class='ctx'> 	else</div><div class='ctx'> 		__ib_drain_sq(qp);</div><div class='add'>+	trace_cq_drain_complete(qp-&gt;send_cq);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_drain_sq);</div><div class='ctx'> </div><div class='hunk'>@@ -2639,6 +3022,7 @@ void ib_drain_rq(struct ib_qp *qp)</div><div class='ctx'> 		qp-&gt;device-&gt;ops.drain_rq(qp);</div><div class='ctx'> 	else</div><div class='ctx'> 		__ib_drain_rq(qp);</div><div class='add'>+	trace_cq_drain_complete(qp-&gt;recv_cq);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_drain_rq);</div><div class='ctx'> </div><div class='hunk'>@@ -2662,10 +3046,12 @@ void ib_drain_qp(struct ib_qp *qp)</div><div class='ctx'> 	ib_drain_sq(qp);</div><div class='ctx'> 	if (!qp-&gt;srq)</div><div class='ctx'> 		ib_drain_rq(qp);</div><div class='add'>+	else</div><div class='add'>+		__ib_drain_srq(qp);</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(ib_drain_qp);</div><div class='ctx'> </div><div class='del'>-struct net_device *rdma_alloc_netdev(struct ib_device *device, u8 port_num,</div><div class='add'>+struct net_device *rdma_alloc_netdev(struct ib_device *device, u32 port_num,</div><div class='ctx'> 				     enum rdma_netdev_t type, const char *name,</div><div class='ctx'> 				     unsigned char name_assign_type,</div><div class='ctx'> 				     void (*setup)(struct net_device *))</div><div class='hunk'>@@ -2691,7 +3077,7 @@ struct net_device *rdma_alloc_netdev(struct ib_device *device, u8 port_num,</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(rdma_alloc_netdev);</div><div class='ctx'> </div><div class='del'>-int rdma_init_netdev(struct ib_device *device, u8 port_num,</div><div class='add'>+int rdma_init_netdev(struct ib_device *device, u32 port_num,</div><div class='ctx'> 		     enum rdma_netdev_t type, const char *name,</div><div class='ctx'> 		     unsigned char name_assign_type,</div><div class='ctx'> 		     void (*setup)(struct net_device *),</div><div class='hunk'>@@ -2729,20 +3115,73 @@ EXPORT_SYMBOL(__rdma_block_iter_start);</div><div class='ctx'> bool __rdma_block_iter_next(struct ib_block_iter *biter)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned int block_offset;</div><div class='add'>+	unsigned int delta;</div><div class='ctx'> </div><div class='ctx'> 	if (!biter-&gt;__sg_nents || !biter-&gt;__sg)</div><div class='ctx'> 		return false;</div><div class='ctx'> </div><div class='ctx'> 	biter-&gt;__dma_addr = sg_dma_address(biter-&gt;__sg) + biter-&gt;__sg_advance;</div><div class='ctx'> 	block_offset = biter-&gt;__dma_addr &amp; (BIT_ULL(biter-&gt;__pg_bit) - 1);</div><div class='del'>-	biter-&gt;__sg_advance += BIT_ULL(biter-&gt;__pg_bit) - block_offset;</div><div class='add'>+	delta = BIT_ULL(biter-&gt;__pg_bit) - block_offset;</div><div class='ctx'> </div><div class='del'>-	if (biter-&gt;__sg_advance &gt;= sg_dma_len(biter-&gt;__sg)) {</div><div class='add'>+	while (biter-&gt;__sg_nents &amp;&amp; biter-&gt;__sg &amp;&amp;</div><div class='add'>+	       sg_dma_len(biter-&gt;__sg) - biter-&gt;__sg_advance &lt;= delta) {</div><div class='add'>+		delta -= sg_dma_len(biter-&gt;__sg) - biter-&gt;__sg_advance;</div><div class='ctx'> 		biter-&gt;__sg_advance = 0;</div><div class='ctx'> 		biter-&gt;__sg = sg_next(biter-&gt;__sg);</div><div class='ctx'> 		biter-&gt;__sg_nents--;</div><div class='ctx'> 	}</div><div class='add'>+	biter-&gt;__sg_advance += delta;</div><div class='ctx'> </div><div class='ctx'> 	return true;</div><div class='ctx'> }</div><div class='ctx'> EXPORT_SYMBOL(__rdma_block_iter_next);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * rdma_alloc_hw_stats_struct - Helper function to allocate dynamic struct</div><div class='add'>+ *   for the drivers.</div><div class='add'>+ * @descs: array of static descriptors</div><div class='add'>+ * @num_counters: number of elements in array</div><div class='add'>+ * @lifespan: milliseconds between updates</div><div class='add'>+ */</div><div class='add'>+struct rdma_hw_stats *rdma_alloc_hw_stats_struct(</div><div class='add'>+	const struct rdma_stat_desc *descs, int num_counters,</div><div class='add'>+	unsigned long lifespan)</div><div class='add'>+{</div><div class='add'>+	struct rdma_hw_stats *stats;</div><div class='add'>+</div><div class='add'>+	stats = kzalloc(struct_size(stats, value, num_counters), GFP_KERNEL);</div><div class='add'>+	if (!stats)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	stats-&gt;is_disabled = kcalloc(BITS_TO_LONGS(num_counters),</div><div class='add'>+				     sizeof(*stats-&gt;is_disabled), GFP_KERNEL);</div><div class='add'>+	if (!stats-&gt;is_disabled)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	stats-&gt;descs = descs;</div><div class='add'>+	stats-&gt;num_counters = num_counters;</div><div class='add'>+	stats-&gt;lifespan = msecs_to_jiffies(lifespan);</div><div class='add'>+	mutex_init(&amp;stats-&gt;lock);</div><div class='add'>+</div><div class='add'>+	return stats;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	kfree(stats);</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_alloc_hw_stats_struct);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * rdma_free_hw_stats_struct - Helper function to release rdma_hw_stats</div><div class='add'>+ * @stats: statistics to release</div><div class='add'>+ */</div><div class='add'>+void rdma_free_hw_stats_struct(struct rdma_hw_stats *stats)</div><div class='add'>+{</div><div class='add'>+	if (!stats)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	kfree(stats-&gt;is_disabled);</div><div class='add'>+	kfree(stats);</div><div class='add'>+}</div><div class='add'>+EXPORT_SYMBOL(rdma_free_hw_stats_struct);</div><div class='head'>diff --git a/drivers/infiniband/hw/Makefile b/drivers/infiniband/hw/Makefile<br/>index 77094be1b262..c42b22ac3303 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/Makefile?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/Makefile</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/Makefile?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/Makefile</a></div><div class='hunk'>@@ -1,17 +1,18 @@</div><div class='ctx'> # SPDX-License-Identifier: GPL-2.0</div><div class='ctx'> obj-$(CONFIG_INFINIBAND_MTHCA)		+= mthca/</div><div class='del'>-obj-$(CONFIG_INFINIBAND_QIB)		+= qib/</div><div class='del'>-obj-$(CONFIG_INFINIBAND_CXGB3)		+= cxgb3/</div><div class='ctx'> obj-$(CONFIG_INFINIBAND_CXGB4)		+= cxgb4/</div><div class='ctx'> obj-$(CONFIG_INFINIBAND_EFA)		+= efa/</div><div class='del'>-obj-$(CONFIG_INFINIBAND_I40IW)		+= i40iw/</div><div class='add'>+obj-$(CONFIG_INFINIBAND_IRDMA)		+= irdma/</div><div class='add'>+obj-$(CONFIG_MANA_INFINIBAND)		+= mana/</div><div class='ctx'> obj-$(CONFIG_MLX4_INFINIBAND)		+= mlx4/</div><div class='ctx'> obj-$(CONFIG_MLX5_INFINIBAND)		+= mlx5/</div><div class='del'>-obj-$(CONFIG_INFINIBAND_NES)		+= nes/</div><div class='ctx'> obj-$(CONFIG_INFINIBAND_OCRDMA)		+= ocrdma/</div><div class='ctx'> obj-$(CONFIG_INFINIBAND_VMWARE_PVRDMA)	+= vmw_pvrdma/</div><div class='ctx'> obj-$(CONFIG_INFINIBAND_USNIC)		+= usnic/</div><div class='ctx'> obj-$(CONFIG_INFINIBAND_HFI1)		+= hfi1/</div><div class='del'>-obj-$(CONFIG_INFINIBAND_HNS)		+= hns/</div><div class='add'>+obj-$(CONFIG_INFINIBAND_HNS_HIP08)	+= hns/</div><div class='ctx'> obj-$(CONFIG_INFINIBAND_QEDR)		+= qedr/</div><div class='ctx'> obj-$(CONFIG_INFINIBAND_BNXT_RE)	+= bnxt_re/</div><div class='add'>+obj-$(CONFIG_INFINIBAND_BNG_RE)		+= bng_re/</div><div class='add'>+obj-$(CONFIG_INFINIBAND_ERDMA)		+= erdma/</div><div class='add'>+obj-$(CONFIG_INFINIBAND_IONIC)		+= ionic/</div><div class='head'>diff --git a/drivers/infiniband/hw/bng_re/Kconfig b/drivers/infiniband/hw/bng_re/Kconfig<br/>new file mode 100644<br/>index 000000000000..85845f72c64d<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bng_re/Kconfig?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bng_re/Kconfig</a></div><div class='hunk'>@@ -0,0 +1,10 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+config INFINIBAND_BNG_RE</div><div class='add'>+	tristate "Broadcom Next generation RoCE HCA support"</div><div class='add'>+	depends on 64BIT</div><div class='add'>+	depends on INET &amp;&amp; DCB &amp;&amp; BNGE</div><div class='add'>+	help</div><div class='add'>+	  This driver supports Broadcom Next generation</div><div class='add'>+	  50/100/200/400/800 gigabit RoCE HCAs. The module</div><div class='add'>+	  will be called bng_re. To compile this driver</div><div class='add'>+	  as a module, choose M here.</div><div class='head'>diff --git a/drivers/infiniband/hw/bng_re/Makefile b/drivers/infiniband/hw/bng_re/Makefile<br/>new file mode 100644<br/>index 000000000000..c6aaaf853c77<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bng_re/Makefile?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bng_re/Makefile</a></div><div class='hunk'>@@ -0,0 +1,8 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0</div><div class='add'>+ccflags-y := -I $(srctree)/drivers/net/ethernet/broadcom/bnge -I $(srctree)/drivers/infiniband/hw/bnxt_re</div><div class='add'>+</div><div class='add'>+obj-$(CONFIG_INFINIBAND_BNG_RE) += bng_re.o</div><div class='add'>+</div><div class='add'>+bng_re-y := bng_dev.o bng_fw.o \</div><div class='add'>+	    bng_res.o bng_sp.o \</div><div class='add'>+	    bng_debugfs.o</div><div class='head'>diff --git a/drivers/infiniband/hw/bng_re/bng_debugfs.c b/drivers/infiniband/hw/bng_re/bng_debugfs.c<br/>new file mode 100644<br/>index 000000000000..9ec5a8785250<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bng_re/bng_debugfs.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bng_re/bng_debugfs.c</a></div><div class='hunk'>@@ -0,0 +1,39 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0</div><div class='add'>+// Copyright (c) 2025 Broadcom.</div><div class='add'>+#include &lt;linux/debugfs.h&gt;</div><div class='add'>+#include &lt;linux/pci.h&gt;</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+</div><div class='add'>+#include "bng_res.h"</div><div class='add'>+#include "bng_fw.h"</div><div class='add'>+#include "bnge.h"</div><div class='add'>+#include "bnge_auxr.h"</div><div class='add'>+#include "bng_re.h"</div><div class='add'>+#include "bng_debugfs.h"</div><div class='add'>+</div><div class='add'>+static struct dentry *bng_re_debugfs_root;</div><div class='add'>+</div><div class='add'>+void bng_re_debugfs_add_pdev(struct bng_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct pci_dev *pdev = rdev-&gt;aux_dev-&gt;pdev;</div><div class='add'>+</div><div class='add'>+	rdev-&gt;dbg_root =</div><div class='add'>+		debugfs_create_dir(dev_name(&amp;pdev-&gt;dev), bng_re_debugfs_root);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bng_re_debugfs_rem_pdev(struct bng_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	debugfs_remove_recursive(rdev-&gt;dbg_root);</div><div class='add'>+	rdev-&gt;dbg_root = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bng_re_register_debugfs(void)</div><div class='add'>+{</div><div class='add'>+	bng_re_debugfs_root = debugfs_create_dir("bng_re", NULL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bng_re_unregister_debugfs(void)</div><div class='add'>+{</div><div class='add'>+	debugfs_remove(bng_re_debugfs_root);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/bng_re/bng_debugfs.h b/drivers/infiniband/hw/bng_re/bng_debugfs.h<br/>new file mode 100644<br/>index 000000000000..baef71df4242<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bng_re/bng_debugfs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bng_re/bng_debugfs.h</a></div><div class='hunk'>@@ -0,0 +1,12 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 */</div><div class='add'>+// Copyright (c) 2025 Broadcom.</div><div class='add'>+</div><div class='add'>+#ifndef __BNG_RE_DEBUGFS__</div><div class='add'>+#define __BNG_RE_DEBUGFS__</div><div class='add'>+</div><div class='add'>+void bng_re_debugfs_add_pdev(struct bng_re_dev *rdev);</div><div class='add'>+void bng_re_debugfs_rem_pdev(struct bng_re_dev *rdev);</div><div class='add'>+</div><div class='add'>+void bng_re_register_debugfs(void);</div><div class='add'>+void bng_re_unregister_debugfs(void);</div><div class='add'>+#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/bng_re/bng_dev.c b/drivers/infiniband/hw/bng_re/bng_dev.c<br/>new file mode 100644<br/>index 000000000000..d8f8d7f7075f<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bng_re/bng_dev.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bng_re/bng_dev.c</a></div><div class='hunk'>@@ -0,0 +1,534 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0</div><div class='add'>+// Copyright (c) 2025 Broadcom.</div><div class='add'>+</div><div class='add'>+#include &lt;linux/module.h&gt;</div><div class='add'>+#include &lt;linux/pci.h&gt;</div><div class='add'>+#include &lt;linux/auxiliary_bus.h&gt;</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+</div><div class='add'>+#include "bng_res.h"</div><div class='add'>+#include "bng_sp.h"</div><div class='add'>+#include "bng_fw.h"</div><div class='add'>+#include "bnge.h"</div><div class='add'>+#include "bnge_auxr.h"</div><div class='add'>+#include "bng_re.h"</div><div class='add'>+#include "bnge_hwrm.h"</div><div class='add'>+#include "bng_debugfs.h"</div><div class='add'>+</div><div class='add'>+MODULE_AUTHOR("Siva Reddy Kallam &lt;siva.kallam@broadcom.com&gt;");</div><div class='add'>+MODULE_DESCRIPTION(BNG_RE_DESC);</div><div class='add'>+MODULE_LICENSE("Dual BSD/GPL");</div><div class='add'>+</div><div class='add'>+static struct bng_re_dev *bng_re_dev_add(struct auxiliary_device *adev,</div><div class='add'>+					 struct bnge_auxr_dev *aux_dev)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_dev *rdev;</div><div class='add'>+</div><div class='add'>+	/* Allocate bng_re_dev instance */</div><div class='add'>+	rdev = ib_alloc_device(bng_re_dev, ibdev);</div><div class='add'>+	if (!rdev) {</div><div class='add'>+		pr_err("%s: bng_re_dev allocation failure!", KBUILD_MODNAME);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Assign auxiliary device specific data */</div><div class='add'>+	rdev-&gt;netdev = aux_dev-&gt;net;</div><div class='add'>+	rdev-&gt;aux_dev = aux_dev;</div><div class='add'>+	rdev-&gt;adev = adev;</div><div class='add'>+	rdev-&gt;fn_id = rdev-&gt;aux_dev-&gt;pdev-&gt;devfn;</div><div class='add'>+</div><div class='add'>+	return rdev;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+</div><div class='add'>+static int bng_re_register_netdev(struct bng_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnge_auxr_dev *aux_dev;</div><div class='add'>+</div><div class='add'>+	aux_dev = rdev-&gt;aux_dev;</div><div class='add'>+	return bnge_register_dev(aux_dev, rdev-&gt;adev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bng_re_destroy_chip_ctx(struct bng_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_chip_ctx *chip_ctx;</div><div class='add'>+</div><div class='add'>+	if (!rdev-&gt;chip_ctx)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	kfree(rdev-&gt;dev_attr);</div><div class='add'>+	rdev-&gt;dev_attr = NULL;</div><div class='add'>+</div><div class='add'>+	chip_ctx = rdev-&gt;chip_ctx;</div><div class='add'>+	rdev-&gt;chip_ctx = NULL;</div><div class='add'>+	rdev-&gt;rcfw.res = NULL;</div><div class='add'>+	rdev-&gt;bng_res.cctx = NULL;</div><div class='add'>+	rdev-&gt;bng_res.pdev = NULL;</div><div class='add'>+	kfree(chip_ctx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bng_re_setup_chip_ctx(struct bng_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_chip_ctx *chip_ctx;</div><div class='add'>+	struct bnge_auxr_dev *aux_dev;</div><div class='add'>+	int rc = -ENOMEM;</div><div class='add'>+</div><div class='add'>+	aux_dev = rdev-&gt;aux_dev;</div><div class='add'>+	rdev-&gt;bng_res.pdev = aux_dev-&gt;pdev;</div><div class='add'>+	rdev-&gt;rcfw.res = &amp;rdev-&gt;bng_res;</div><div class='add'>+	chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL);</div><div class='add'>+	if (!chip_ctx)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	chip_ctx-&gt;chip_num = aux_dev-&gt;chip_num;</div><div class='add'>+	chip_ctx-&gt;hw_stats_size = aux_dev-&gt;hw_ring_stats_size;</div><div class='add'>+</div><div class='add'>+	rdev-&gt;chip_ctx = chip_ctx;</div><div class='add'>+	rdev-&gt;bng_res.cctx = rdev-&gt;chip_ctx;</div><div class='add'>+	rdev-&gt;dev_attr = kzalloc(sizeof(*rdev-&gt;dev_attr), GFP_KERNEL);</div><div class='add'>+	if (!rdev-&gt;dev_attr)</div><div class='add'>+		goto free_chip_ctx;</div><div class='add'>+	rdev-&gt;bng_res.dattr = rdev-&gt;dev_attr;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+free_chip_ctx:</div><div class='add'>+	kfree(rdev-&gt;chip_ctx);</div><div class='add'>+	rdev-&gt;chip_ctx = NULL;</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bng_re_init_hwrm_hdr(struct input *hdr, u16 opcd)</div><div class='add'>+{</div><div class='add'>+	hdr-&gt;req_type = cpu_to_le16(opcd);</div><div class='add'>+	hdr-&gt;cmpl_ring = cpu_to_le16(-1);</div><div class='add'>+	hdr-&gt;target_id = cpu_to_le16(-1);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bng_re_fill_fw_msg(struct bnge_fw_msg *fw_msg, void *msg,</div><div class='add'>+			       int msg_len, void *resp, int resp_max_len,</div><div class='add'>+			       int timeout)</div><div class='add'>+{</div><div class='add'>+	fw_msg-&gt;msg = msg;</div><div class='add'>+	fw_msg-&gt;msg_len = msg_len;</div><div class='add'>+	fw_msg-&gt;resp = resp;</div><div class='add'>+	fw_msg-&gt;resp_max_len = resp_max_len;</div><div class='add'>+	fw_msg-&gt;timeout = timeout;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bng_re_net_ring_free(struct bng_re_dev *rdev,</div><div class='add'>+				u16 fw_ring_id, int type)</div><div class='add'>+{</div><div class='add'>+	struct bnge_auxr_dev *aux_dev = rdev-&gt;aux_dev;</div><div class='add'>+	struct hwrm_ring_free_input req = {};</div><div class='add'>+	struct hwrm_ring_free_output resp;</div><div class='add'>+	struct bnge_fw_msg fw_msg = {};</div><div class='add'>+	int rc = -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!rdev)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	if (!aux_dev)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	bng_re_init_hwrm_hdr((void *)&amp;req, HWRM_RING_FREE);</div><div class='add'>+	req.ring_type = type;</div><div class='add'>+	req.ring_id = cpu_to_le16(fw_ring_id);</div><div class='add'>+	bng_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), (void *)&amp;resp,</div><div class='add'>+			    sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT);</div><div class='add'>+	rc = bnge_send_msg(aux_dev, &amp;fw_msg);</div><div class='add'>+	if (rc)</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to free HW ring:%d :%#x",</div><div class='add'>+			  req.ring_id, rc);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bng_re_net_ring_alloc(struct bng_re_dev *rdev,</div><div class='add'>+				 struct bng_re_ring_attr *ring_attr,</div><div class='add'>+				 u16 *fw_ring_id)</div><div class='add'>+{</div><div class='add'>+	struct bnge_auxr_dev *aux_dev = rdev-&gt;aux_dev;</div><div class='add'>+	struct hwrm_ring_alloc_input req = {};</div><div class='add'>+	struct hwrm_ring_alloc_output resp;</div><div class='add'>+	struct bnge_fw_msg fw_msg = {};</div><div class='add'>+	int rc = -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!aux_dev)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	bng_re_init_hwrm_hdr((void *)&amp;req, HWRM_RING_ALLOC);</div><div class='add'>+	req.enables = 0;</div><div class='add'>+	req.page_tbl_addr =  cpu_to_le64(ring_attr-&gt;dma_arr[0]);</div><div class='add'>+	if (ring_attr-&gt;pages &gt; 1) {</div><div class='add'>+		/* Page size is in log2 units */</div><div class='add'>+		req.page_size = BNGE_PAGE_SHIFT;</div><div class='add'>+		req.page_tbl_depth = 1;</div><div class='add'>+	}</div><div class='add'>+	req.fbo = 0;</div><div class='add'>+	/* Association of ring index with doorbell index and MSIX number */</div><div class='add'>+	req.logical_id = cpu_to_le16(ring_attr-&gt;lrid);</div><div class='add'>+	req.length = cpu_to_le32(ring_attr-&gt;depth + 1);</div><div class='add'>+	req.ring_type = ring_attr-&gt;type;</div><div class='add'>+	req.int_mode = ring_attr-&gt;mode;</div><div class='add'>+	bng_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), (void *)&amp;resp,</div><div class='add'>+			   sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT);</div><div class='add'>+	rc = bnge_send_msg(aux_dev, &amp;fw_msg);</div><div class='add'>+	if (!rc)</div><div class='add'>+		*fw_ring_id = le16_to_cpu(resp.ring_id);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bng_re_stats_ctx_free(struct bng_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnge_auxr_dev *aux_dev = rdev-&gt;aux_dev;</div><div class='add'>+	struct hwrm_stat_ctx_free_input req = {};</div><div class='add'>+	struct hwrm_stat_ctx_free_output resp = {};</div><div class='add'>+	struct bnge_fw_msg fw_msg = {};</div><div class='add'>+	int rc = -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!aux_dev)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	bng_re_init_hwrm_hdr((void *)&amp;req, HWRM_STAT_CTX_FREE);</div><div class='add'>+	req.stat_ctx_id = cpu_to_le32(rdev-&gt;stats_ctx.fw_id);</div><div class='add'>+	bng_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), (void *)&amp;resp,</div><div class='add'>+			   sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT);</div><div class='add'>+	rc = bnge_send_msg(aux_dev, &amp;fw_msg);</div><div class='add'>+	if (rc)</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to free HW stats context %#x",</div><div class='add'>+			  rc);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bng_re_stats_ctx_alloc(struct bng_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnge_auxr_dev *aux_dev = rdev-&gt;aux_dev;</div><div class='add'>+	struct bng_re_stats *stats = &amp;rdev-&gt;stats_ctx;</div><div class='add'>+	struct hwrm_stat_ctx_alloc_output resp = {};</div><div class='add'>+	struct hwrm_stat_ctx_alloc_input req = {};</div><div class='add'>+	struct bnge_fw_msg fw_msg = {};</div><div class='add'>+	int rc = -EINVAL;</div><div class='add'>+</div><div class='add'>+	stats-&gt;fw_id = BNGE_INVALID_STATS_CTX_ID;</div><div class='add'>+</div><div class='add'>+	if (!aux_dev)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	bng_re_init_hwrm_hdr((void *)&amp;req, HWRM_STAT_CTX_ALLOC);</div><div class='add'>+	req.update_period_ms = cpu_to_le32(1000);</div><div class='add'>+	req.stats_dma_addr = cpu_to_le64(stats-&gt;dma_map);</div><div class='add'>+	req.stats_dma_length = cpu_to_le16(rdev-&gt;chip_ctx-&gt;hw_stats_size);</div><div class='add'>+	req.stat_ctx_flags = STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE;</div><div class='add'>+	bng_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), (void *)&amp;resp,</div><div class='add'>+			   sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT);</div><div class='add'>+	rc = bnge_send_msg(aux_dev, &amp;fw_msg);</div><div class='add'>+	if (!rc)</div><div class='add'>+		stats-&gt;fw_id = le32_to_cpu(resp.stat_ctx_id);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bng_re_query_hwrm_version(struct bng_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnge_auxr_dev *aux_dev = rdev-&gt;aux_dev;</div><div class='add'>+	struct hwrm_ver_get_output ver_get_resp = {};</div><div class='add'>+	struct hwrm_ver_get_input ver_get_req = {};</div><div class='add'>+	struct bng_re_chip_ctx *cctx;</div><div class='add'>+	struct bnge_fw_msg fw_msg = {};</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	bng_re_init_hwrm_hdr((void *)&amp;ver_get_req, HWRM_VER_GET);</div><div class='add'>+	ver_get_req.hwrm_intf_maj = HWRM_VERSION_MAJOR;</div><div class='add'>+	ver_get_req.hwrm_intf_min = HWRM_VERSION_MINOR;</div><div class='add'>+	ver_get_req.hwrm_intf_upd = HWRM_VERSION_UPDATE;</div><div class='add'>+	bng_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;ver_get_req, sizeof(ver_get_req),</div><div class='add'>+			    (void *)&amp;ver_get_resp, sizeof(ver_get_resp),</div><div class='add'>+			    BNGE_DFLT_HWRM_CMD_TIMEOUT);</div><div class='add'>+	rc = bnge_send_msg(aux_dev, &amp;fw_msg);</div><div class='add'>+	if (rc) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to query HW version, rc = 0x%x",</div><div class='add'>+			  rc);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cctx = rdev-&gt;chip_ctx;</div><div class='add'>+	cctx-&gt;hwrm_intf_ver =</div><div class='add'>+		(u64)le16_to_cpu(ver_get_resp.hwrm_intf_major) &lt;&lt; 48 |</div><div class='add'>+		(u64)le16_to_cpu(ver_get_resp.hwrm_intf_minor) &lt;&lt; 32 |</div><div class='add'>+		(u64)le16_to_cpu(ver_get_resp.hwrm_intf_build) &lt;&lt; 16 |</div><div class='add'>+		le16_to_cpu(ver_get_resp.hwrm_intf_patch);</div><div class='add'>+</div><div class='add'>+	cctx-&gt;hwrm_cmd_max_timeout = le16_to_cpu(ver_get_resp.max_req_timeout);</div><div class='add'>+</div><div class='add'>+	if (!cctx-&gt;hwrm_cmd_max_timeout)</div><div class='add'>+		cctx-&gt;hwrm_cmd_max_timeout = BNG_ROCE_FW_MAX_TIMEOUT;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bng_re_dev_uninit(struct bng_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	int rc;</div><div class='add'>+	bng_re_debugfs_rem_pdev(rdev);</div><div class='add'>+</div><div class='add'>+	if (test_and_clear_bit(BNG_RE_FLAG_RCFW_CHANNEL_EN, &amp;rdev-&gt;flags)) {</div><div class='add'>+		rc = bng_re_deinit_rcfw(&amp;rdev-&gt;rcfw);</div><div class='add'>+		if (rc)</div><div class='add'>+			ibdev_warn(&amp;rdev-&gt;ibdev,</div><div class='add'>+				   "Failed to deinitialize RCFW: %#x", rc);</div><div class='add'>+		bng_re_stats_ctx_free(rdev);</div><div class='add'>+		bng_re_free_stats_ctx_mem(rdev-&gt;bng_res.pdev, &amp;rdev-&gt;stats_ctx);</div><div class='add'>+		bng_re_disable_rcfw_channel(&amp;rdev-&gt;rcfw);</div><div class='add'>+		bng_re_net_ring_free(rdev, rdev-&gt;rcfw.creq.ring_id,</div><div class='add'>+			     RING_ALLOC_REQ_RING_TYPE_NQ);</div><div class='add'>+		bng_re_free_rcfw_channel(&amp;rdev-&gt;rcfw);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	kfree(rdev-&gt;nqr);</div><div class='add'>+	rdev-&gt;nqr = NULL;</div><div class='add'>+	bng_re_destroy_chip_ctx(rdev);</div><div class='add'>+	if (test_and_clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &amp;rdev-&gt;flags))</div><div class='add'>+		bnge_unregister_dev(rdev-&gt;aux_dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bng_re_dev_init(struct bng_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_ring_attr rattr = {};</div><div class='add'>+	struct bng_re_creq_ctx *creq;</div><div class='add'>+	u32 db_offt;</div><div class='add'>+	int vid;</div><div class='add'>+	u8 type;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	/* Registered a new RoCE device instance to netdev */</div><div class='add'>+	rc = bng_re_register_netdev(rdev);</div><div class='add'>+	if (rc) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+				"Failed to register with netedev: %#x\n", rc);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	set_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &amp;rdev-&gt;flags);</div><div class='add'>+</div><div class='add'>+	if (rdev-&gt;aux_dev-&gt;auxr_info-&gt;msix_requested &lt; BNG_RE_MIN_MSIX) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "RoCE requires minimum 2 MSI-X vectors, but only %d reserved\n",</div><div class='add'>+			  rdev-&gt;aux_dev-&gt;auxr_info-&gt;msix_requested);</div><div class='add'>+		bnge_unregister_dev(rdev-&gt;aux_dev);</div><div class='add'>+		clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &amp;rdev-&gt;flags);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+	ibdev_dbg(&amp;rdev-&gt;ibdev, "Got %d MSI-X vectors\n",</div><div class='add'>+		  rdev-&gt;aux_dev-&gt;auxr_info-&gt;msix_requested);</div><div class='add'>+</div><div class='add'>+	rc = bng_re_setup_chip_ctx(rdev);</div><div class='add'>+	if (rc) {</div><div class='add'>+		bnge_unregister_dev(rdev-&gt;aux_dev);</div><div class='add'>+		clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &amp;rdev-&gt;flags);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to get chip context\n");</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	bng_re_query_hwrm_version(rdev);</div><div class='add'>+</div><div class='add'>+	rc = bng_re_alloc_fw_channel(&amp;rdev-&gt;bng_res, &amp;rdev-&gt;rcfw);</div><div class='add'>+	if (rc) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to allocate RCFW Channel: %#x\n", rc);</div><div class='add'>+		goto fail;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Allocate nq record memory */</div><div class='add'>+	rdev-&gt;nqr = kzalloc(sizeof(*rdev-&gt;nqr), GFP_KERNEL);</div><div class='add'>+	if (!rdev-&gt;nqr) {</div><div class='add'>+		bng_re_destroy_chip_ctx(rdev);</div><div class='add'>+		bnge_unregister_dev(rdev-&gt;aux_dev);</div><div class='add'>+		clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &amp;rdev-&gt;flags);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rdev-&gt;nqr-&gt;num_msix = rdev-&gt;aux_dev-&gt;auxr_info-&gt;msix_requested;</div><div class='add'>+	memcpy(rdev-&gt;nqr-&gt;msix_entries, rdev-&gt;aux_dev-&gt;msix_info,</div><div class='add'>+	       sizeof(struct bnge_msix_info) * rdev-&gt;nqr-&gt;num_msix);</div><div class='add'>+</div><div class='add'>+	type = RING_ALLOC_REQ_RING_TYPE_NQ;</div><div class='add'>+	creq = &amp;rdev-&gt;rcfw.creq;</div><div class='add'>+	rattr.dma_arr = creq-&gt;hwq.pbl[BNG_PBL_LVL_0].pg_map_arr;</div><div class='add'>+	rattr.pages = creq-&gt;hwq.pbl[creq-&gt;hwq.level].pg_count;</div><div class='add'>+	rattr.type = type;</div><div class='add'>+	rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;</div><div class='add'>+	rattr.depth = BNG_FW_CREQE_MAX_CNT - 1;</div><div class='add'>+	rattr.lrid = rdev-&gt;nqr-&gt;msix_entries[BNG_RE_CREQ_NQ_IDX].ring_idx;</div><div class='add'>+	rc = bng_re_net_ring_alloc(rdev, &amp;rattr, &amp;creq-&gt;ring_id);</div><div class='add'>+	if (rc) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to allocate CREQ: %#x\n", rc);</div><div class='add'>+		goto free_rcfw;</div><div class='add'>+	}</div><div class='add'>+	db_offt = rdev-&gt;nqr-&gt;msix_entries[BNG_RE_CREQ_NQ_IDX].db_offset;</div><div class='add'>+	vid = rdev-&gt;nqr-&gt;msix_entries[BNG_RE_CREQ_NQ_IDX].vector;</div><div class='add'>+</div><div class='add'>+	rc = bng_re_enable_fw_channel(&amp;rdev-&gt;rcfw,</div><div class='add'>+					vid, db_offt);</div><div class='add'>+	if (rc) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to enable RCFW channel: %#x\n",</div><div class='add'>+			  rc);</div><div class='add'>+		goto free_ring;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rc = bng_re_get_dev_attr(&amp;rdev-&gt;rcfw);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto disable_rcfw;</div><div class='add'>+</div><div class='add'>+	bng_re_debugfs_add_pdev(rdev);</div><div class='add'>+	rc = bng_re_alloc_stats_ctx_mem(rdev-&gt;bng_res.pdev, rdev-&gt;chip_ctx,</div><div class='add'>+					&amp;rdev-&gt;stats_ctx);</div><div class='add'>+	if (rc) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to allocate stats context: %#x\n", rc);</div><div class='add'>+		goto disable_rcfw;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rc = bng_re_stats_ctx_alloc(rdev);</div><div class='add'>+	if (rc) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to allocate QPLIB context: %#x\n", rc);</div><div class='add'>+		goto free_stats_ctx;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rc = bng_re_init_rcfw(&amp;rdev-&gt;rcfw, &amp;rdev-&gt;stats_ctx);</div><div class='add'>+	if (rc) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to initialize RCFW: %#x\n", rc);</div><div class='add'>+		goto free_sctx;</div><div class='add'>+	}</div><div class='add'>+	set_bit(BNG_RE_FLAG_RCFW_CHANNEL_EN, &amp;rdev-&gt;flags);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+free_sctx:</div><div class='add'>+	bng_re_stats_ctx_free(rdev);</div><div class='add'>+free_stats_ctx:</div><div class='add'>+	bng_re_free_stats_ctx_mem(rdev-&gt;bng_res.pdev, &amp;rdev-&gt;stats_ctx);</div><div class='add'>+disable_rcfw:</div><div class='add'>+	bng_re_disable_rcfw_channel(&amp;rdev-&gt;rcfw);</div><div class='add'>+free_ring:</div><div class='add'>+	bng_re_net_ring_free(rdev, rdev-&gt;rcfw.creq.ring_id, type);</div><div class='add'>+free_rcfw:</div><div class='add'>+	bng_re_free_rcfw_channel(&amp;rdev-&gt;rcfw);</div><div class='add'>+fail:</div><div class='add'>+	bng_re_dev_uninit(rdev);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bng_re_add_device(struct auxiliary_device *adev)</div><div class='add'>+{</div><div class='add'>+	struct bnge_auxr_priv *auxr_priv =</div><div class='add'>+		container_of(adev, struct bnge_auxr_priv, aux_dev);</div><div class='add'>+	struct bng_re_en_dev_info *dev_info;</div><div class='add'>+	struct bng_re_dev *rdev;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	dev_info = auxiliary_get_drvdata(adev);</div><div class='add'>+</div><div class='add'>+	rdev = bng_re_dev_add(adev, auxr_priv-&gt;auxr_dev);</div><div class='add'>+	if (!rdev) {</div><div class='add'>+		rc = -ENOMEM;</div><div class='add'>+		goto exit;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dev_info-&gt;rdev = rdev;</div><div class='add'>+</div><div class='add'>+	rc = bng_re_dev_init(rdev);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto re_dev_dealloc;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+re_dev_dealloc:</div><div class='add'>+	ib_dealloc_device(&amp;rdev-&gt;ibdev);</div><div class='add'>+exit:</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+</div><div class='add'>+static void bng_re_remove_device(struct bng_re_dev *rdev,</div><div class='add'>+				 struct auxiliary_device *aux_dev)</div><div class='add'>+{</div><div class='add'>+	bng_re_dev_uninit(rdev);</div><div class='add'>+	ib_dealloc_device(&amp;rdev-&gt;ibdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+</div><div class='add'>+static int bng_re_probe(struct auxiliary_device *adev,</div><div class='add'>+			const struct auxiliary_device_id *id)</div><div class='add'>+{</div><div class='add'>+	struct bnge_auxr_priv *aux_priv =</div><div class='add'>+		container_of(adev, struct bnge_auxr_priv, aux_dev);</div><div class='add'>+	struct bng_re_en_dev_info *en_info;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	en_info = kzalloc(sizeof(*en_info), GFP_KERNEL);</div><div class='add'>+	if (!en_info)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	en_info-&gt;auxr_dev = aux_priv-&gt;auxr_dev;</div><div class='add'>+</div><div class='add'>+	auxiliary_set_drvdata(adev, en_info);</div><div class='add'>+</div><div class='add'>+	rc = bng_re_add_device(adev);</div><div class='add'>+	if (rc)</div><div class='add'>+		kfree(en_info);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bng_re_remove(struct auxiliary_device *adev)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_en_dev_info *dev_info = auxiliary_get_drvdata(adev);</div><div class='add'>+	struct bng_re_dev *rdev;</div><div class='add'>+</div><div class='add'>+	rdev = dev_info-&gt;rdev;</div><div class='add'>+</div><div class='add'>+	if (rdev)</div><div class='add'>+		bng_re_remove_device(rdev, adev);</div><div class='add'>+	kfree(dev_info);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct auxiliary_device_id bng_re_id_table[] = {</div><div class='add'>+	{ .name = BNG_RE_ADEV_NAME ".rdma", },</div><div class='add'>+	{},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+MODULE_DEVICE_TABLE(auxiliary, bng_re_id_table);</div><div class='add'>+</div><div class='add'>+static struct auxiliary_driver bng_re_driver = {</div><div class='add'>+	.name = "rdma",</div><div class='add'>+	.probe = bng_re_probe,</div><div class='add'>+	.remove = bng_re_remove,</div><div class='add'>+	.id_table = bng_re_id_table,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int __init bng_re_mod_init(void)</div><div class='add'>+{</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+</div><div class='add'>+	bng_re_register_debugfs();</div><div class='add'>+</div><div class='add'>+	rc = auxiliary_driver_register(&amp;bng_re_driver);</div><div class='add'>+	if (rc) {</div><div class='add'>+		pr_err("%s: Failed to register auxiliary driver\n",</div><div class='add'>+		       KBUILD_MODNAME);</div><div class='add'>+		goto unreg_debugfs;</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+unreg_debugfs:</div><div class='add'>+	bng_re_unregister_debugfs();</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void __exit bng_re_mod_exit(void)</div><div class='add'>+{</div><div class='add'>+	auxiliary_driver_unregister(&amp;bng_re_driver);</div><div class='add'>+	bng_re_unregister_debugfs();</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+module_init(bng_re_mod_init);</div><div class='add'>+module_exit(bng_re_mod_exit);</div><div class='head'>diff --git a/drivers/infiniband/hw/bng_re/bng_fw.c b/drivers/infiniband/hw/bng_re/bng_fw.c<br/>new file mode 100644<br/>index 000000000000..7d9539113cf5<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bng_re/bng_fw.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bng_re/bng_fw.c</a></div><div class='hunk'>@@ -0,0 +1,767 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0</div><div class='add'>+// Copyright (c) 2025 Broadcom.</div><div class='add'>+#include &lt;linux/pci.h&gt;</div><div class='add'>+</div><div class='add'>+#include "roce_hsi.h"</div><div class='add'>+#include "bng_res.h"</div><div class='add'>+#include "bng_fw.h"</div><div class='add'>+#include "bng_sp.h"</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * bng_re_map_rc  -  map return type based on opcode</div><div class='add'>+ * @opcode:  roce slow path opcode</div><div class='add'>+ *</div><div class='add'>+ * case #1</div><div class='add'>+ * Firmware initiated error recovery is a safe state machine and</div><div class='add'>+ * driver can consider all the underlying rdma resources are free.</div><div class='add'>+ * In this state, it is safe to return success for opcodes related to</div><div class='add'>+ * destroying rdma resources (like destroy qp, destroy cq etc.).</div><div class='add'>+ *</div><div class='add'>+ * case #2</div><div class='add'>+ * If driver detect potential firmware stall, it is not safe state machine</div><div class='add'>+ * and the driver can not consider all the underlying rdma resources are</div><div class='add'>+ * freed.</div><div class='add'>+ * In this state, it is not safe to return success for opcodes related to</div><div class='add'>+ * destroying rdma resources (like destroy qp, destroy cq etc.).</div><div class='add'>+ *</div><div class='add'>+ * Scope of this helper function is only for case #1.</div><div class='add'>+ *</div><div class='add'>+ * Returns:</div><div class='add'>+ * 0 to communicate success to caller.</div><div class='add'>+ * Non zero error code to communicate failure to caller.</div><div class='add'>+ */</div><div class='add'>+static int bng_re_map_rc(u8 opcode)</div><div class='add'>+{</div><div class='add'>+	switch (opcode) {</div><div class='add'>+	case CMDQ_BASE_OPCODE_DESTROY_QP:</div><div class='add'>+	case CMDQ_BASE_OPCODE_DESTROY_SRQ:</div><div class='add'>+	case CMDQ_BASE_OPCODE_DESTROY_CQ:</div><div class='add'>+	case CMDQ_BASE_OPCODE_DEALLOCATE_KEY:</div><div class='add'>+	case CMDQ_BASE_OPCODE_DEREGISTER_MR:</div><div class='add'>+	case CMDQ_BASE_OPCODE_DELETE_GID:</div><div class='add'>+	case CMDQ_BASE_OPCODE_DESTROY_QP1:</div><div class='add'>+	case CMDQ_BASE_OPCODE_DESTROY_AH:</div><div class='add'>+	case CMDQ_BASE_OPCODE_DEINITIALIZE_FW:</div><div class='add'>+	case CMDQ_BASE_OPCODE_MODIFY_ROCE_CC:</div><div class='add'>+	case CMDQ_BASE_OPCODE_SET_LINK_AGGR_MODE:</div><div class='add'>+		return 0;</div><div class='add'>+	default:</div><div class='add'>+		return -ETIMEDOUT;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bng_re_free_rcfw_channel(struct bng_re_rcfw *rcfw)</div><div class='add'>+{</div><div class='add'>+	kfree(rcfw-&gt;crsqe_tbl);</div><div class='add'>+	bng_re_free_hwq(rcfw-&gt;res, &amp;rcfw-&gt;cmdq.hwq);</div><div class='add'>+	bng_re_free_hwq(rcfw-&gt;res, &amp;rcfw-&gt;creq.hwq);</div><div class='add'>+	rcfw-&gt;pdev = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bng_re_alloc_fw_channel(struct bng_re_res *res,</div><div class='add'>+			    struct bng_re_rcfw *rcfw)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_hwq_attr hwq_attr = {};</div><div class='add'>+	struct bng_re_sg_info sginfo = {};</div><div class='add'>+	struct bng_re_cmdq_ctx *cmdq;</div><div class='add'>+	struct bng_re_creq_ctx *creq;</div><div class='add'>+</div><div class='add'>+	rcfw-&gt;pdev = res-&gt;pdev;</div><div class='add'>+	cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+	creq = &amp;rcfw-&gt;creq;</div><div class='add'>+	rcfw-&gt;res = res;</div><div class='add'>+</div><div class='add'>+	sginfo.pgsize = PAGE_SIZE;</div><div class='add'>+	sginfo.pgshft = PAGE_SHIFT;</div><div class='add'>+</div><div class='add'>+	hwq_attr.sginfo = &amp;sginfo;</div><div class='add'>+	hwq_attr.res = rcfw-&gt;res;</div><div class='add'>+	hwq_attr.depth = BNG_FW_CREQE_MAX_CNT;</div><div class='add'>+	hwq_attr.stride = BNG_FW_CREQE_UNITS;</div><div class='add'>+	hwq_attr.type = BNG_HWQ_TYPE_QUEUE;</div><div class='add'>+</div><div class='add'>+	if (bng_re_alloc_init_hwq(&amp;creq-&gt;hwq, &amp;hwq_attr)) {</div><div class='add'>+		dev_err(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='add'>+			"HW channel CREQ allocation failed\n");</div><div class='add'>+		goto fail;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rcfw-&gt;cmdq_depth = BNG_FW_CMDQE_MAX_CNT;</div><div class='add'>+</div><div class='add'>+	sginfo.pgsize = bng_fw_cmdqe_page_size(rcfw-&gt;cmdq_depth);</div><div class='add'>+	hwq_attr.depth = rcfw-&gt;cmdq_depth &amp; 0x7FFFFFFF;</div><div class='add'>+	hwq_attr.stride = BNG_FW_CMDQE_UNITS;</div><div class='add'>+	hwq_attr.type = BNG_HWQ_TYPE_CTX;</div><div class='add'>+	if (bng_re_alloc_init_hwq(&amp;cmdq-&gt;hwq, &amp;hwq_attr)) {</div><div class='add'>+		dev_err(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='add'>+			"HW channel CMDQ allocation failed\n");</div><div class='add'>+		goto fail;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rcfw-&gt;crsqe_tbl = kcalloc(cmdq-&gt;hwq.max_elements,</div><div class='add'>+				  sizeof(*rcfw-&gt;crsqe_tbl), GFP_KERNEL);</div><div class='add'>+	if (!rcfw-&gt;crsqe_tbl)</div><div class='add'>+		goto fail;</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;rcfw-&gt;tbl_lock);</div><div class='add'>+</div><div class='add'>+	rcfw-&gt;max_timeout = res-&gt;cctx-&gt;hwrm_cmd_max_timeout;</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+fail:</div><div class='add'>+	bng_re_free_rcfw_channel(rcfw);</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bng_re_process_qp_event(struct bng_re_rcfw *rcfw,</div><div class='add'>+				   struct creq_qp_event *qp_event,</div><div class='add'>+				   u32 *num_wait)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_hwq *hwq = &amp;rcfw-&gt;cmdq.hwq;</div><div class='add'>+	struct bng_re_crsqe *crsqe;</div><div class='add'>+	u32 req_size;</div><div class='add'>+	u16 cookie;</div><div class='add'>+	bool is_waiter_alive;</div><div class='add'>+	struct pci_dev *pdev;</div><div class='add'>+	u32 wait_cmds = 0;</div><div class='add'>+	int rc = 0;</div><div class='add'>+</div><div class='add'>+	pdev = rcfw-&gt;pdev;</div><div class='add'>+	switch (qp_event-&gt;event) {</div><div class='add'>+	case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION:</div><div class='add'>+		dev_err(&amp;pdev-&gt;dev, "Received QP error notification\n");</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		/*</div><div class='add'>+		 * Command Response</div><div class='add'>+		 * cmdq-&gt;lock needs to be acquired to synchronie</div><div class='add'>+		 * the command send and completion reaping. This function</div><div class='add'>+		 * is always called with creq-&gt;lock held. Using</div><div class='add'>+		 * the nested variant of spin_lock.</div><div class='add'>+		 *</div><div class='add'>+		 */</div><div class='add'>+</div><div class='add'>+		spin_lock_nested(&amp;hwq-&gt;lock, SINGLE_DEPTH_NESTING);</div><div class='add'>+		cookie = le16_to_cpu(qp_event-&gt;cookie);</div><div class='add'>+		cookie &amp;= BNG_FW_MAX_COOKIE_VALUE;</div><div class='add'>+		crsqe = &amp;rcfw-&gt;crsqe_tbl[cookie];</div><div class='add'>+</div><div class='add'>+		if (WARN_ONCE(test_bit(FIRMWARE_STALL_DETECTED,</div><div class='add'>+				       &amp;rcfw-&gt;cmdq.flags),</div><div class='add'>+		    "Unreponsive rcfw channel detected.!!")) {</div><div class='add'>+			dev_info(&amp;pdev-&gt;dev,</div><div class='add'>+				 "rcfw timedout: cookie = %#x, free_slots = %d",</div><div class='add'>+				 cookie, crsqe-&gt;free_slots);</div><div class='add'>+			spin_unlock(&amp;hwq-&gt;lock);</div><div class='add'>+			return rc;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (crsqe-&gt;is_waiter_alive) {</div><div class='add'>+			if (crsqe-&gt;resp) {</div><div class='add'>+				memcpy(crsqe-&gt;resp, qp_event, sizeof(*qp_event));</div><div class='add'>+				/* Insert write memory barrier to ensure that</div><div class='add'>+				 * response data is copied before clearing the</div><div class='add'>+				 * flags</div><div class='add'>+				 */</div><div class='add'>+				smp_wmb();</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		wait_cmds++;</div><div class='add'>+</div><div class='add'>+		req_size = crsqe-&gt;req_size;</div><div class='add'>+		is_waiter_alive = crsqe-&gt;is_waiter_alive;</div><div class='add'>+</div><div class='add'>+		crsqe-&gt;req_size = 0;</div><div class='add'>+		if (!is_waiter_alive)</div><div class='add'>+			crsqe-&gt;resp = NULL;</div><div class='add'>+</div><div class='add'>+		crsqe-&gt;is_in_used = false;</div><div class='add'>+</div><div class='add'>+		hwq-&gt;cons += req_size;</div><div class='add'>+</div><div class='add'>+		spin_unlock(&amp;hwq-&gt;lock);</div><div class='add'>+	}</div><div class='add'>+	*num_wait += wait_cmds;</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* function events */</div><div class='add'>+static int bng_re_process_func_event(struct bng_re_rcfw *rcfw,</div><div class='add'>+				     struct creq_func_event *func_event)</div><div class='add'>+{</div><div class='add'>+	switch (func_event-&gt;event) {</div><div class='add'>+	case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR:</div><div class='add'>+	case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR:</div><div class='add'>+	case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR:</div><div class='add'>+	case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR:</div><div class='add'>+	case CREQ_FUNC_EVENT_EVENT_CQ_ERROR:</div><div class='add'>+	case CREQ_FUNC_EVENT_EVENT_TQM_ERROR:</div><div class='add'>+	case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR:</div><div class='add'>+	case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR:</div><div class='add'>+	case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR:</div><div class='add'>+	case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR:</div><div class='add'>+	case CREQ_FUNC_EVENT_EVENT_TIM_ERROR:</div><div class='add'>+	case CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST:</div><div class='add'>+	case CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED:</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* CREQ Completion handlers */</div><div class='add'>+static void bng_re_service_creq(struct tasklet_struct *t)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_rcfw *rcfw = from_tasklet(rcfw, t, creq.creq_tasklet);</div><div class='add'>+	struct bng_re_creq_ctx *creq = &amp;rcfw-&gt;creq;</div><div class='add'>+	u32 type, budget = BNG_FW_CREQ_ENTRY_POLL_BUDGET;</div><div class='add'>+	struct bng_re_hwq *hwq = &amp;creq-&gt;hwq;</div><div class='add'>+	struct creq_base *creqe;</div><div class='add'>+	u32 num_wakeup = 0;</div><div class='add'>+	u32 hw_polled = 0;</div><div class='add'>+</div><div class='add'>+	/* Service the CREQ until budget is over */</div><div class='add'>+	spin_lock_bh(&amp;hwq-&gt;lock);</div><div class='add'>+	while (budget &gt; 0) {</div><div class='add'>+		creqe = bng_re_get_qe(hwq, hwq-&gt;cons, NULL);</div><div class='add'>+		if (!BNG_FW_CREQ_CMP_VALID(creqe, creq-&gt;creq_db.dbinfo.flags))</div><div class='add'>+			break;</div><div class='add'>+		/* The valid test of the entry must be done first before</div><div class='add'>+		 * reading any further.</div><div class='add'>+		 */</div><div class='add'>+		dma_rmb();</div><div class='add'>+</div><div class='add'>+		type = creqe-&gt;type &amp; CREQ_BASE_TYPE_MASK;</div><div class='add'>+		switch (type) {</div><div class='add'>+		case CREQ_BASE_TYPE_QP_EVENT:</div><div class='add'>+			bng_re_process_qp_event</div><div class='add'>+				(rcfw, (struct creq_qp_event *)creqe,</div><div class='add'>+				 &amp;num_wakeup);</div><div class='add'>+			creq-&gt;stats.creq_qp_event_processed++;</div><div class='add'>+			break;</div><div class='add'>+		case CREQ_BASE_TYPE_FUNC_EVENT:</div><div class='add'>+			if (!bng_re_process_func_event</div><div class='add'>+			    (rcfw, (struct creq_func_event *)creqe))</div><div class='add'>+				creq-&gt;stats.creq_func_event_processed++;</div><div class='add'>+			else</div><div class='add'>+				dev_warn(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='add'>+					 "aeqe:%#x Not handled\n", type);</div><div class='add'>+			break;</div><div class='add'>+		default:</div><div class='add'>+			if (type != ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT)</div><div class='add'>+				dev_warn(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='add'>+					 "creqe with event 0x%x not handled\n",</div><div class='add'>+					 type);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		budget--;</div><div class='add'>+		hw_polled++;</div><div class='add'>+		bng_re_hwq_incr_cons(hwq-&gt;max_elements, &amp;hwq-&gt;cons,</div><div class='add'>+				     1, &amp;creq-&gt;creq_db.dbinfo.flags);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (hw_polled)</div><div class='add'>+		bng_re_ring_nq_db(&amp;creq-&gt;creq_db.dbinfo,</div><div class='add'>+				  rcfw-&gt;res-&gt;cctx, true);</div><div class='add'>+	spin_unlock_bh(&amp;hwq-&gt;lock);</div><div class='add'>+	if (num_wakeup)</div><div class='add'>+		wake_up_nr(&amp;rcfw-&gt;cmdq.waitq, num_wakeup);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int __send_message_basic_sanity(struct bng_re_rcfw *rcfw,</div><div class='add'>+				       struct bng_re_cmdqmsg *msg,</div><div class='add'>+				       u8 opcode)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_cmdq_ctx *cmdq;</div><div class='add'>+</div><div class='add'>+	cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+</div><div class='add'>+	if (test_bit(FIRMWARE_STALL_DETECTED, &amp;cmdq-&gt;flags))</div><div class='add'>+		return -ETIMEDOUT;</div><div class='add'>+</div><div class='add'>+	if (test_bit(FIRMWARE_INITIALIZED_FLAG, &amp;cmdq-&gt;flags) &amp;&amp;</div><div class='add'>+	    opcode == CMDQ_BASE_OPCODE_INITIALIZE_FW) {</div><div class='add'>+		dev_err(&amp;rcfw-&gt;pdev-&gt;dev, "RCFW already initialized!");</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &amp;cmdq-&gt;flags) &amp;&amp;</div><div class='add'>+	    (opcode != CMDQ_BASE_OPCODE_QUERY_FUNC &amp;&amp;</div><div class='add'>+	     opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW &amp;&amp;</div><div class='add'>+	     opcode != CMDQ_BASE_OPCODE_QUERY_VERSION)) {</div><div class='add'>+		dev_err(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='add'>+			"RCFW not initialized, reject opcode 0x%x",</div><div class='add'>+			opcode);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int __send_message(struct bng_re_rcfw *rcfw,</div><div class='add'>+			  struct bng_re_cmdqmsg *msg, u8 opcode)</div><div class='add'>+{</div><div class='add'>+	u32 bsize, free_slots, required_slots;</div><div class='add'>+	struct bng_re_cmdq_ctx *cmdq;</div><div class='add'>+	struct bng_re_crsqe *crsqe;</div><div class='add'>+	struct bng_fw_cmdqe *cmdqe;</div><div class='add'>+	struct bng_re_hwq *hwq;</div><div class='add'>+	u32 sw_prod, cmdq_prod;</div><div class='add'>+	struct pci_dev *pdev;</div><div class='add'>+	u16 cookie;</div><div class='add'>+	u8 *preq;</div><div class='add'>+</div><div class='add'>+	cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+	hwq = &amp;cmdq-&gt;hwq;</div><div class='add'>+	pdev = rcfw-&gt;pdev;</div><div class='add'>+</div><div class='add'>+	/* Cmdq are in 16-byte units, each request can consume 1 or more</div><div class='add'>+	 * cmdqe</div><div class='add'>+	 */</div><div class='add'>+	spin_lock_bh(&amp;hwq-&gt;lock);</div><div class='add'>+	required_slots = bng_re_get_cmd_slots(msg-&gt;req);</div><div class='add'>+	free_slots = HWQ_FREE_SLOTS(hwq);</div><div class='add'>+	cookie = cmdq-&gt;seq_num &amp; BNG_FW_MAX_COOKIE_VALUE;</div><div class='add'>+	crsqe = &amp;rcfw-&gt;crsqe_tbl[cookie];</div><div class='add'>+</div><div class='add'>+	if (required_slots &gt;= free_slots) {</div><div class='add'>+		dev_info_ratelimited(&amp;pdev-&gt;dev,</div><div class='add'>+				     "CMDQ is full req/free %d/%d!",</div><div class='add'>+				     required_slots, free_slots);</div><div class='add'>+		spin_unlock_bh(&amp;hwq-&gt;lock);</div><div class='add'>+		return -EAGAIN;</div><div class='add'>+	}</div><div class='add'>+	__set_cmdq_base_cookie(msg-&gt;req, msg-&gt;req_sz, cpu_to_le16(cookie));</div><div class='add'>+</div><div class='add'>+	bsize = bng_re_set_cmd_slots(msg-&gt;req);</div><div class='add'>+	crsqe-&gt;free_slots = free_slots;</div><div class='add'>+	crsqe-&gt;resp = (struct creq_qp_event *)msg-&gt;resp;</div><div class='add'>+	crsqe-&gt;is_waiter_alive = true;</div><div class='add'>+	crsqe-&gt;is_in_used = true;</div><div class='add'>+	crsqe-&gt;opcode = opcode;</div><div class='add'>+</div><div class='add'>+	crsqe-&gt;req_size = __get_cmdq_base_cmd_size(msg-&gt;req, msg-&gt;req_sz);</div><div class='add'>+	if (__get_cmdq_base_resp_size(msg-&gt;req, msg-&gt;req_sz) &amp;&amp; msg-&gt;sb) {</div><div class='add'>+		struct bng_re_rcfw_sbuf *sbuf = msg-&gt;sb;</div><div class='add'>+</div><div class='add'>+		__set_cmdq_base_resp_addr(msg-&gt;req, msg-&gt;req_sz,</div><div class='add'>+					  cpu_to_le64(sbuf-&gt;dma_addr));</div><div class='add'>+		__set_cmdq_base_resp_size(msg-&gt;req, msg-&gt;req_sz,</div><div class='add'>+					  ALIGN(sbuf-&gt;size,</div><div class='add'>+						BNG_FW_CMDQE_UNITS) /</div><div class='add'>+						BNG_FW_CMDQE_UNITS);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	preq = (u8 *)msg-&gt;req;</div><div class='add'>+	do {</div><div class='add'>+		/* Locate the next cmdq slot */</div><div class='add'>+		sw_prod = HWQ_CMP(hwq-&gt;prod, hwq);</div><div class='add'>+		cmdqe = bng_re_get_qe(hwq, sw_prod, NULL);</div><div class='add'>+		/* Copy a segment of the req cmd to the cmdq */</div><div class='add'>+		memset(cmdqe, 0, sizeof(*cmdqe));</div><div class='add'>+		memcpy(cmdqe, preq, min_t(u32, bsize, sizeof(*cmdqe)));</div><div class='add'>+		preq += min_t(u32, bsize, sizeof(*cmdqe));</div><div class='add'>+		bsize -= min_t(u32, bsize, sizeof(*cmdqe));</div><div class='add'>+		hwq-&gt;prod++;</div><div class='add'>+	} while (bsize &gt; 0);</div><div class='add'>+	cmdq-&gt;seq_num++;</div><div class='add'>+</div><div class='add'>+	cmdq_prod = hwq-&gt;prod &amp; 0xFFFF;</div><div class='add'>+	if (test_bit(FIRMWARE_FIRST_FLAG, &amp;cmdq-&gt;flags)) {</div><div class='add'>+		/* The very first doorbell write</div><div class='add'>+		 * is required to set this flag</div><div class='add'>+		 * which prompts the FW to reset</div><div class='add'>+		 * its internal pointers</div><div class='add'>+		 */</div><div class='add'>+		cmdq_prod |= BIT(FIRMWARE_FIRST_FLAG);</div><div class='add'>+		clear_bit(FIRMWARE_FIRST_FLAG, &amp;cmdq-&gt;flags);</div><div class='add'>+	}</div><div class='add'>+	/* ring CMDQ DB */</div><div class='add'>+	wmb();</div><div class='add'>+	writel(cmdq_prod, cmdq-&gt;cmdq_mbox.prod);</div><div class='add'>+	writel(BNG_FW_CMDQ_TRIG_VAL, cmdq-&gt;cmdq_mbox.db);</div><div class='add'>+	spin_unlock_bh(&amp;hwq-&gt;lock);</div><div class='add'>+	/* Return the CREQ response pointer */</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * __wait_for_resp   -	Don't hold the cpu context and wait for response</div><div class='add'>+ * @rcfw:    rcfw channel instance of rdev</div><div class='add'>+ * @cookie:  cookie to track the command</div><div class='add'>+ *</div><div class='add'>+ * Wait for command completion in sleepable context.</div><div class='add'>+ *</div><div class='add'>+ * Returns:</div><div class='add'>+ * 0 if command is completed by firmware.</div><div class='add'>+ * Non zero error code for rest of the case.</div><div class='add'>+ */</div><div class='add'>+static int __wait_for_resp(struct bng_re_rcfw *rcfw, u16 cookie)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_cmdq_ctx *cmdq;</div><div class='add'>+	struct bng_re_crsqe *crsqe;</div><div class='add'>+</div><div class='add'>+	cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+	crsqe = &amp;rcfw-&gt;crsqe_tbl[cookie];</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		wait_event_timeout(cmdq-&gt;waitq,</div><div class='add'>+				   !crsqe-&gt;is_in_used,</div><div class='add'>+				   secs_to_jiffies(rcfw-&gt;max_timeout));</div><div class='add'>+</div><div class='add'>+		if (!crsqe-&gt;is_in_used)</div><div class='add'>+			return 0;</div><div class='add'>+</div><div class='add'>+		bng_re_service_creq(&amp;rcfw-&gt;creq.creq_tasklet);</div><div class='add'>+</div><div class='add'>+		if (!crsqe-&gt;is_in_used)</div><div class='add'>+			return 0;</div><div class='add'>+	} while (true);</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * bng_re_rcfw_send_message   -	interface to send</div><div class='add'>+ * and complete rcfw command.</div><div class='add'>+ * @rcfw:   rcfw channel instance of rdev</div><div class='add'>+ * @msg:    message to send</div><div class='add'>+ *</div><div class='add'>+ * This function does not account shadow queue depth. It will send</div><div class='add'>+ * all the command unconditionally as long as send queue is not full.</div><div class='add'>+ *</div><div class='add'>+ * Returns:</div><div class='add'>+ * 0 if command completed by firmware.</div><div class='add'>+ * Non zero if the command is not completed by firmware.</div><div class='add'>+ */</div><div class='add'>+int bng_re_rcfw_send_message(struct bng_re_rcfw *rcfw,</div><div class='add'>+			     struct bng_re_cmdqmsg *msg)</div><div class='add'>+{</div><div class='add'>+	struct creq_qp_event *evnt = (struct creq_qp_event *)msg-&gt;resp;</div><div class='add'>+	struct bng_re_crsqe *crsqe;</div><div class='add'>+	u16 cookie;</div><div class='add'>+	int rc;</div><div class='add'>+	u8 opcode;</div><div class='add'>+</div><div class='add'>+	opcode = __get_cmdq_base_opcode(msg-&gt;req, msg-&gt;req_sz);</div><div class='add'>+</div><div class='add'>+	rc = __send_message_basic_sanity(rcfw, msg, opcode);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc == -ENXIO ? bng_re_map_rc(opcode) : rc;</div><div class='add'>+</div><div class='add'>+	rc = __send_message(rcfw, msg, opcode);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	cookie = le16_to_cpu(__get_cmdq_base_cookie(msg-&gt;req, msg-&gt;req_sz))</div><div class='add'>+				&amp; BNG_FW_MAX_COOKIE_VALUE;</div><div class='add'>+</div><div class='add'>+	rc = __wait_for_resp(rcfw, cookie);</div><div class='add'>+</div><div class='add'>+	if (rc) {</div><div class='add'>+		spin_lock_bh(&amp;rcfw-&gt;cmdq.hwq.lock);</div><div class='add'>+		crsqe = &amp;rcfw-&gt;crsqe_tbl[cookie];</div><div class='add'>+		crsqe-&gt;is_waiter_alive = false;</div><div class='add'>+		if (rc == -ENODEV)</div><div class='add'>+			set_bit(FIRMWARE_STALL_DETECTED, &amp;rcfw-&gt;cmdq.flags);</div><div class='add'>+		spin_unlock_bh(&amp;rcfw-&gt;cmdq.hwq.lock);</div><div class='add'>+		return -ETIMEDOUT;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (evnt-&gt;status) {</div><div class='add'>+		/* failed with status */</div><div class='add'>+		dev_err(&amp;rcfw-&gt;pdev-&gt;dev, "cmdq[%#x]=%#x status %#x\n",</div><div class='add'>+			cookie, opcode, evnt-&gt;status);</div><div class='add'>+		rc = -EIO;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bng_re_map_cmdq_mbox(struct bng_re_rcfw *rcfw)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_cmdq_mbox *mbox;</div><div class='add'>+	resource_size_t bar_reg;</div><div class='add'>+	struct pci_dev *pdev;</div><div class='add'>+</div><div class='add'>+	pdev = rcfw-&gt;pdev;</div><div class='add'>+	mbox = &amp;rcfw-&gt;cmdq.cmdq_mbox;</div><div class='add'>+</div><div class='add'>+	mbox-&gt;reg.bar_id = BNG_FW_COMM_PCI_BAR_REGION;</div><div class='add'>+	mbox-&gt;reg.len = BNG_FW_COMM_SIZE;</div><div class='add'>+	mbox-&gt;reg.bar_base = pci_resource_start(pdev, mbox-&gt;reg.bar_id);</div><div class='add'>+	if (!mbox-&gt;reg.bar_base) {</div><div class='add'>+		dev_err(&amp;pdev-&gt;dev,</div><div class='add'>+			"CMDQ BAR region %d resc start is 0!\n",</div><div class='add'>+			mbox-&gt;reg.bar_id);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	bar_reg = mbox-&gt;reg.bar_base + BNG_FW_COMM_BASE_OFFSET;</div><div class='add'>+	mbox-&gt;reg.len = BNG_FW_COMM_SIZE;</div><div class='add'>+	mbox-&gt;reg.bar_reg = ioremap(bar_reg, mbox-&gt;reg.len);</div><div class='add'>+	if (!mbox-&gt;reg.bar_reg) {</div><div class='add'>+		dev_err(&amp;pdev-&gt;dev,</div><div class='add'>+			"CMDQ BAR region %d mapping failed\n",</div><div class='add'>+			mbox-&gt;reg.bar_id);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mbox-&gt;prod = (void  __iomem *)(mbox-&gt;reg.bar_reg +</div><div class='add'>+			BNG_FW_PF_VF_COMM_PROD_OFFSET);</div><div class='add'>+	mbox-&gt;db = (void __iomem *)(mbox-&gt;reg.bar_reg + BNG_FW_COMM_TRIG_OFFSET);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static irqreturn_t bng_re_creq_irq(int irq, void *dev_instance)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_rcfw *rcfw = dev_instance;</div><div class='add'>+	struct bng_re_creq_ctx *creq;</div><div class='add'>+	struct bng_re_hwq *hwq;</div><div class='add'>+	u32 sw_cons;</div><div class='add'>+</div><div class='add'>+	creq = &amp;rcfw-&gt;creq;</div><div class='add'>+	hwq = &amp;creq-&gt;hwq;</div><div class='add'>+	/* Prefetch the CREQ element */</div><div class='add'>+	sw_cons = HWQ_CMP(hwq-&gt;cons, hwq);</div><div class='add'>+	bng_re_get_qe(hwq, sw_cons, NULL);</div><div class='add'>+</div><div class='add'>+	tasklet_schedule(&amp;creq-&gt;creq_tasklet);</div><div class='add'>+	return IRQ_HANDLED;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bng_re_rcfw_start_irq(struct bng_re_rcfw *rcfw, int msix_vector,</div><div class='add'>+			  bool need_init)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_creq_ctx *creq;</div><div class='add'>+	struct bng_re_res *res;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	creq = &amp;rcfw-&gt;creq;</div><div class='add'>+	res = rcfw-&gt;res;</div><div class='add'>+</div><div class='add'>+	if (creq-&gt;irq_handler_avail)</div><div class='add'>+		return -EFAULT;</div><div class='add'>+</div><div class='add'>+	creq-&gt;msix_vec = msix_vector;</div><div class='add'>+	if (need_init)</div><div class='add'>+		tasklet_setup(&amp;creq-&gt;creq_tasklet, bng_re_service_creq);</div><div class='add'>+	else</div><div class='add'>+		tasklet_enable(&amp;creq-&gt;creq_tasklet);</div><div class='add'>+</div><div class='add'>+	creq-&gt;irq_name = kasprintf(GFP_KERNEL, "bng_re-creq@pci:%s",</div><div class='add'>+				   pci_name(res-&gt;pdev));</div><div class='add'>+	if (!creq-&gt;irq_name)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	rc = request_irq(creq-&gt;msix_vec, bng_re_creq_irq, 0,</div><div class='add'>+			 creq-&gt;irq_name, rcfw);</div><div class='add'>+	if (rc) {</div><div class='add'>+		kfree(creq-&gt;irq_name);</div><div class='add'>+		creq-&gt;irq_name = NULL;</div><div class='add'>+		tasklet_disable(&amp;creq-&gt;creq_tasklet);</div><div class='add'>+		return rc;</div><div class='add'>+	}</div><div class='add'>+	creq-&gt;irq_handler_avail = true;</div><div class='add'>+</div><div class='add'>+	bng_re_ring_nq_db(&amp;creq-&gt;creq_db.dbinfo, res-&gt;cctx, true);</div><div class='add'>+	atomic_inc(&amp;rcfw-&gt;rcfw_intr_enabled);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bng_re_map_creq_db(struct bng_re_rcfw *rcfw, u32 reg_offt)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_creq_db *creq_db;</div><div class='add'>+	resource_size_t bar_reg;</div><div class='add'>+	struct pci_dev *pdev;</div><div class='add'>+</div><div class='add'>+	pdev = rcfw-&gt;pdev;</div><div class='add'>+	creq_db = &amp;rcfw-&gt;creq.creq_db;</div><div class='add'>+</div><div class='add'>+	creq_db-&gt;dbinfo.flags = 0;</div><div class='add'>+	creq_db-&gt;reg.bar_id = BNG_FW_COMM_CONS_PCI_BAR_REGION;</div><div class='add'>+	creq_db-&gt;reg.bar_base = pci_resource_start(pdev, creq_db-&gt;reg.bar_id);</div><div class='add'>+	if (!creq_db-&gt;reg.bar_id)</div><div class='add'>+		dev_err(&amp;pdev-&gt;dev,</div><div class='add'>+			"CREQ BAR region %d resc start is 0!",</div><div class='add'>+			creq_db-&gt;reg.bar_id);</div><div class='add'>+</div><div class='add'>+	bar_reg = creq_db-&gt;reg.bar_base + reg_offt;</div><div class='add'>+</div><div class='add'>+	creq_db-&gt;reg.len = BNG_FW_CREQ_DB_LEN;</div><div class='add'>+	creq_db-&gt;reg.bar_reg = ioremap(bar_reg, creq_db-&gt;reg.len);</div><div class='add'>+	if (!creq_db-&gt;reg.bar_reg) {</div><div class='add'>+		dev_err(&amp;pdev-&gt;dev,</div><div class='add'>+			"CREQ BAR region %d mapping failed",</div><div class='add'>+			creq_db-&gt;reg.bar_id);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+	creq_db-&gt;dbinfo.db = creq_db-&gt;reg.bar_reg;</div><div class='add'>+	creq_db-&gt;dbinfo.hwq = &amp;rcfw-&gt;creq.hwq;</div><div class='add'>+	creq_db-&gt;dbinfo.xid = rcfw-&gt;creq.ring_id;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bng_re_rcfw_stop_irq(struct bng_re_rcfw *rcfw, bool kill)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_creq_ctx *creq;</div><div class='add'>+</div><div class='add'>+	creq = &amp;rcfw-&gt;creq;</div><div class='add'>+</div><div class='add'>+	if (!creq-&gt;irq_handler_avail)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	creq-&gt;irq_handler_avail = false;</div><div class='add'>+	/* Mask h/w interrupts */</div><div class='add'>+	bng_re_ring_nq_db(&amp;creq-&gt;creq_db.dbinfo, rcfw-&gt;res-&gt;cctx, false);</div><div class='add'>+	/* Sync with last running IRQ-handler */</div><div class='add'>+	synchronize_irq(creq-&gt;msix_vec);</div><div class='add'>+	free_irq(creq-&gt;msix_vec, rcfw);</div><div class='add'>+	kfree(creq-&gt;irq_name);</div><div class='add'>+	creq-&gt;irq_name = NULL;</div><div class='add'>+	atomic_set(&amp;rcfw-&gt;rcfw_intr_enabled, 0);</div><div class='add'>+	if (kill)</div><div class='add'>+		tasklet_kill(&amp;creq-&gt;creq_tasklet);</div><div class='add'>+	tasklet_disable(&amp;creq-&gt;creq_tasklet);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bng_re_disable_rcfw_channel(struct bng_re_rcfw *rcfw)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_creq_ctx *creq;</div><div class='add'>+	struct bng_re_cmdq_ctx *cmdq;</div><div class='add'>+</div><div class='add'>+	creq = &amp;rcfw-&gt;creq;</div><div class='add'>+	cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+	/* Make sure the HW channel is stopped! */</div><div class='add'>+	bng_re_rcfw_stop_irq(rcfw, true);</div><div class='add'>+</div><div class='add'>+	iounmap(cmdq-&gt;cmdq_mbox.reg.bar_reg);</div><div class='add'>+	iounmap(creq-&gt;creq_db.reg.bar_reg);</div><div class='add'>+</div><div class='add'>+	cmdq-&gt;cmdq_mbox.reg.bar_reg = NULL;</div><div class='add'>+	creq-&gt;creq_db.reg.bar_reg = NULL;</div><div class='add'>+	creq-&gt;msix_vec = 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bng_re_start_rcfw(struct bng_re_rcfw *rcfw)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_cmdq_ctx *cmdq;</div><div class='add'>+	struct bng_re_creq_ctx *creq;</div><div class='add'>+	struct bng_re_cmdq_mbox *mbox;</div><div class='add'>+	struct cmdq_init init = {0};</div><div class='add'>+</div><div class='add'>+	cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+	creq = &amp;rcfw-&gt;creq;</div><div class='add'>+	mbox = &amp;cmdq-&gt;cmdq_mbox;</div><div class='add'>+</div><div class='add'>+	init.cmdq_pbl = cpu_to_le64(cmdq-&gt;hwq.pbl[BNG_PBL_LVL_0].pg_map_arr[0]);</div><div class='add'>+	init.cmdq_size_cmdq_lvl =</div><div class='add'>+			cpu_to_le16(((rcfw-&gt;cmdq_depth &lt;&lt;</div><div class='add'>+				      CMDQ_INIT_CMDQ_SIZE_SFT) &amp;</div><div class='add'>+				    CMDQ_INIT_CMDQ_SIZE_MASK) |</div><div class='add'>+				    ((cmdq-&gt;hwq.level &lt;&lt;</div><div class='add'>+				      CMDQ_INIT_CMDQ_LVL_SFT) &amp;</div><div class='add'>+				    CMDQ_INIT_CMDQ_LVL_MASK));</div><div class='add'>+	init.creq_ring_id = cpu_to_le16(creq-&gt;ring_id);</div><div class='add'>+	/* Write to the mailbox register */</div><div class='add'>+	__iowrite32_copy(mbox-&gt;reg.bar_reg, &amp;init, sizeof(init) / 4);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bng_re_enable_fw_channel(struct bng_re_rcfw *rcfw,</div><div class='add'>+			     int msix_vector,</div><div class='add'>+			     int cp_bar_reg_off)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_cmdq_ctx *cmdq;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+</div><div class='add'>+	/* Assign defaults */</div><div class='add'>+	cmdq-&gt;seq_num = 0;</div><div class='add'>+	set_bit(FIRMWARE_FIRST_FLAG, &amp;cmdq-&gt;flags);</div><div class='add'>+	init_waitqueue_head(&amp;cmdq-&gt;waitq);</div><div class='add'>+</div><div class='add'>+	rc = bng_re_map_cmdq_mbox(rcfw);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	rc = bng_re_map_creq_db(rcfw, cp_bar_reg_off);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	rc = bng_re_rcfw_start_irq(rcfw, msix_vector, true);</div><div class='add'>+	if (rc) {</div><div class='add'>+		dev_err(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='add'>+			"Failed to request IRQ for CREQ rc = 0x%x\n", rc);</div><div class='add'>+		bng_re_disable_rcfw_channel(rcfw);</div><div class='add'>+		return rc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	bng_re_start_rcfw(rcfw);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bng_re_deinit_rcfw(struct bng_re_rcfw *rcfw)</div><div class='add'>+{</div><div class='add'>+	struct creq_deinitialize_fw_resp resp = {};</div><div class='add'>+	struct cmdq_deinitialize_fw req = {};</div><div class='add'>+	struct bng_re_cmdqmsg msg = {};</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	bng_re_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+			     CMDQ_BASE_OPCODE_DEINITIALIZE_FW,</div><div class='add'>+			     sizeof(req));</div><div class='add'>+	bng_re_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL,</div><div class='add'>+			    sizeof(req), sizeof(resp), 0);</div><div class='add'>+	rc = bng_re_rcfw_send_message(rcfw, &amp;msg);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	clear_bit(FIRMWARE_INITIALIZED_FLAG, &amp;rcfw-&gt;cmdq.flags);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+static inline bool _is_hw_retx_supported(u16 dev_cap_flags)</div><div class='add'>+{</div><div class='add'>+	return dev_cap_flags &amp;</div><div class='add'>+		(CREQ_QUERY_FUNC_RESP_SB_HW_REQUESTER_RETX_ENABLED |</div><div class='add'>+		 CREQ_QUERY_FUNC_RESP_SB_HW_RESPONDER_RETX_ENABLED);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define BNG_RE_HW_RETX(a) _is_hw_retx_supported((a))</div><div class='add'>+static inline bool _is_optimize_modify_qp_supported(u16 dev_cap_ext_flags2)</div><div class='add'>+{</div><div class='add'>+	return dev_cap_ext_flags2 &amp;</div><div class='add'>+	       CREQ_QUERY_FUNC_RESP_SB_OPTIMIZE_MODIFY_QP_SUPPORTED;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bng_re_init_rcfw(struct bng_re_rcfw *rcfw,</div><div class='add'>+		     struct bng_re_stats *stats_ctx)</div><div class='add'>+{</div><div class='add'>+	struct creq_initialize_fw_resp resp = {};</div><div class='add'>+	struct cmdq_initialize_fw req = {};</div><div class='add'>+	struct bng_re_cmdqmsg msg = {};</div><div class='add'>+	int rc;</div><div class='add'>+	u16 flags = 0;</div><div class='add'>+</div><div class='add'>+	bng_re_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+			     CMDQ_BASE_OPCODE_INITIALIZE_FW,</div><div class='add'>+			     sizeof(req));</div><div class='add'>+	/* Supply (log-base-2-of-host-page-size - base-page-shift)</div><div class='add'>+	 * to bono to adjust the doorbell page sizes.</div><div class='add'>+	 */</div><div class='add'>+	req.log2_dbr_pg_size = cpu_to_le16(PAGE_SHIFT -</div><div class='add'>+					   BNG_FW_DBR_BASE_PAGE_SHIFT);</div><div class='add'>+	if (BNG_RE_HW_RETX(rcfw-&gt;res-&gt;dattr-&gt;dev_cap_flags))</div><div class='add'>+		flags |= CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED;</div><div class='add'>+	if (_is_optimize_modify_qp_supported(rcfw-&gt;res-&gt;dattr-&gt;dev_cap_flags2))</div><div class='add'>+		flags |= CMDQ_INITIALIZE_FW_FLAGS_OPTIMIZE_MODIFY_QP_SUPPORTED;</div><div class='add'>+	req.flags |= cpu_to_le16(flags);</div><div class='add'>+	req.stat_ctx_id = cpu_to_le32(stats_ctx-&gt;fw_id);</div><div class='add'>+	bng_re_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req), sizeof(resp), 0);</div><div class='add'>+	rc = bng_re_rcfw_send_message(rcfw, &amp;msg);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+	set_bit(FIRMWARE_INITIALIZED_FLAG, &amp;rcfw-&gt;cmdq.flags);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/bng_re/bng_fw.h b/drivers/infiniband/hw/bng_re/bng_fw.h<br/>new file mode 100644<br/>index 000000000000..c89c926ec2fc<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bng_re/bng_fw.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bng_re/bng_fw.h</a></div><div class='hunk'>@@ -0,0 +1,211 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 */</div><div class='add'>+// Copyright (c) 2025 Broadcom.</div><div class='add'>+</div><div class='add'>+#ifndef __BNG_FW_H__</div><div class='add'>+#define __BNG_FW_H__</div><div class='add'>+</div><div class='add'>+#include "bng_tlv.h"</div><div class='add'>+</div><div class='add'>+/* FW DB related */</div><div class='add'>+#define BNG_FW_CMDQ_TRIG_VAL		1</div><div class='add'>+#define BNG_FW_COMM_PCI_BAR_REGION	0</div><div class='add'>+#define BNG_FW_COMM_CONS_PCI_BAR_REGION	2</div><div class='add'>+#define BNG_FW_DBR_BASE_PAGE_SHIFT	12</div><div class='add'>+#define BNG_FW_COMM_SIZE		0x104</div><div class='add'>+#define BNG_FW_COMM_BASE_OFFSET		0x600</div><div class='add'>+#define BNG_FW_COMM_TRIG_OFFSET		0x100</div><div class='add'>+#define BNG_FW_PF_VF_COMM_PROD_OFFSET	0xc</div><div class='add'>+#define BNG_FW_CREQ_DB_LEN		8</div><div class='add'>+</div><div class='add'>+/* CREQ */</div><div class='add'>+#define BNG_FW_CREQE_MAX_CNT		(64 * 1024)</div><div class='add'>+#define BNG_FW_CREQE_UNITS		16</div><div class='add'>+#define BNG_FW_CREQ_ENTRY_POLL_BUDGET	0x100</div><div class='add'>+#define BNG_FW_CREQ_CMP_VALID(hdr, pass)			\</div><div class='add'>+	(!!((hdr)-&gt;v &amp; CREQ_BASE_V) ==				\</div><div class='add'>+	   !((pass) &amp; BNG_RE_FLAG_EPOCH_CONS_MASK))</div><div class='add'>+#define BNG_FW_CREQ_ENTRY_POLL_BUDGET	0x100</div><div class='add'>+</div><div class='add'>+/* CMDQ */</div><div class='add'>+struct bng_fw_cmdqe {</div><div class='add'>+	u8	data[16];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define BNG_FW_CMDQE_MAX_CNT		8192</div><div class='add'>+#define BNG_FW_CMDQE_UNITS		sizeof(struct bng_fw_cmdqe)</div><div class='add'>+#define BNG_FW_CMDQE_BYTES(depth)	((depth) * BNG_FW_CMDQE_UNITS)</div><div class='add'>+</div><div class='add'>+#define BNG_FW_MAX_COOKIE_VALUE		(BNG_FW_CMDQE_MAX_CNT - 1)</div><div class='add'>+#define BNG_FW_CMD_IS_BLOCKING		0x8000</div><div class='add'>+</div><div class='add'>+/* Crsq buf is 1024-Byte */</div><div class='add'>+struct bng_re_crsbe {</div><div class='add'>+	u8			data[1024];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+</div><div class='add'>+static inline u32 bng_fw_cmdqe_npages(u32 depth)</div><div class='add'>+{</div><div class='add'>+	u32 npages;</div><div class='add'>+</div><div class='add'>+	npages = BNG_FW_CMDQE_BYTES(depth) / PAGE_SIZE;</div><div class='add'>+	if (BNG_FW_CMDQE_BYTES(depth) % PAGE_SIZE)</div><div class='add'>+		npages++;</div><div class='add'>+	return npages;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 bng_fw_cmdqe_page_size(u32 depth)</div><div class='add'>+{</div><div class='add'>+	return (bng_fw_cmdqe_npages(depth) * PAGE_SIZE);</div><div class='add'>+}</div><div class='add'>+struct bng_re_cmdq_mbox {</div><div class='add'>+	struct bng_re_reg_desc		reg;</div><div class='add'>+	void __iomem			*prod;</div><div class='add'>+	void __iomem			*db;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* HWQ */</div><div class='add'>+struct bng_re_cmdq_ctx {</div><div class='add'>+	struct bng_re_hwq		hwq;</div><div class='add'>+	struct bng_re_cmdq_mbox		cmdq_mbox;</div><div class='add'>+	unsigned long			flags;</div><div class='add'>+#define FIRMWARE_INITIALIZED_FLAG	(0)</div><div class='add'>+#define FIRMWARE_STALL_DETECTED		(3)</div><div class='add'>+#define FIRMWARE_FIRST_FLAG		(31)</div><div class='add'>+	wait_queue_head_t		waitq;</div><div class='add'>+	u32				seq_num;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_creq_db {</div><div class='add'>+	struct bng_re_reg_desc	reg;</div><div class='add'>+	struct bng_re_db_info	dbinfo;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_creq_stat {</div><div class='add'>+	u64	creq_qp_event_processed;</div><div class='add'>+	u64	creq_func_event_processed;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_creq_ctx {</div><div class='add'>+	struct bng_re_hwq		hwq;</div><div class='add'>+	struct bng_re_creq_db		creq_db;</div><div class='add'>+	struct bng_re_creq_stat		stats;</div><div class='add'>+	struct tasklet_struct		creq_tasklet;</div><div class='add'>+	u16				ring_id;</div><div class='add'>+	int				msix_vec;</div><div class='add'>+	bool				irq_handler_avail;</div><div class='add'>+	char				*irq_name;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_crsqe {</div><div class='add'>+	struct creq_qp_event	*resp;</div><div class='add'>+	u32			req_size;</div><div class='add'>+	/* Free slots at the time of submission */</div><div class='add'>+	u32			free_slots;</div><div class='add'>+	u8			opcode;</div><div class='add'>+	bool			is_waiter_alive;</div><div class='add'>+	bool			is_in_used;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_rcfw_sbuf {</div><div class='add'>+	void *sb;</div><div class='add'>+	dma_addr_t dma_addr;</div><div class='add'>+	u32 size;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* RoCE FW Communication Channels */</div><div class='add'>+struct bng_re_rcfw {</div><div class='add'>+	struct pci_dev		*pdev;</div><div class='add'>+	struct bng_re_res	*res;</div><div class='add'>+	struct bng_re_cmdq_ctx	cmdq;</div><div class='add'>+	struct bng_re_creq_ctx	creq;</div><div class='add'>+	struct bng_re_crsqe	*crsqe_tbl;</div><div class='add'>+	/* To synchronize the qp-handle hash table */</div><div class='add'>+	spinlock_t		tbl_lock;</div><div class='add'>+	u32			cmdq_depth;</div><div class='add'>+	/* cached from chip cctx for quick reference in slow path */</div><div class='add'>+	u16			max_timeout;</div><div class='add'>+	atomic_t		rcfw_intr_enabled;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_cmdqmsg {</div><div class='add'>+	struct cmdq_base	*req;</div><div class='add'>+	struct creq_base	*resp;</div><div class='add'>+	void			*sb;</div><div class='add'>+	u32			req_sz;</div><div class='add'>+	u32			res_sz;</div><div class='add'>+	u8			block;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline void bng_re_rcfw_cmd_prep(struct cmdq_base *req,</div><div class='add'>+					u8 opcode, u8 cmd_size)</div><div class='add'>+{</div><div class='add'>+	req-&gt;opcode = opcode;</div><div class='add'>+	req-&gt;cmd_size = cmd_size;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void bng_re_fill_cmdqmsg(struct bng_re_cmdqmsg *msg,</div><div class='add'>+				       void *req, void *resp, void *sb,</div><div class='add'>+				       u32 req_sz, u32 res_sz, u8 block)</div><div class='add'>+{</div><div class='add'>+	msg-&gt;req = req;</div><div class='add'>+	msg-&gt;resp = resp;</div><div class='add'>+	msg-&gt;sb = sb;</div><div class='add'>+	msg-&gt;req_sz = req_sz;</div><div class='add'>+	msg-&gt;res_sz = res_sz;</div><div class='add'>+	msg-&gt;block = block;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* Get the number of command units required for the req. The</div><div class='add'>+ * function returns correct value only if called before</div><div class='add'>+ * setting using bng_re_set_cmd_slots</div><div class='add'>+ */</div><div class='add'>+static inline u32 bng_re_get_cmd_slots(struct cmdq_base *req)</div><div class='add'>+{</div><div class='add'>+	u32 cmd_units = 0;</div><div class='add'>+</div><div class='add'>+	if (HAS_TLV_HEADER(req)) {</div><div class='add'>+		struct roce_tlv *tlv_req = (struct roce_tlv *)req;</div><div class='add'>+</div><div class='add'>+		cmd_units = tlv_req-&gt;total_size;</div><div class='add'>+	} else {</div><div class='add'>+		cmd_units = (req-&gt;cmd_size + BNG_FW_CMDQE_UNITS - 1) /</div><div class='add'>+			    BNG_FW_CMDQE_UNITS;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return cmd_units;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 bng_re_set_cmd_slots(struct cmdq_base *req)</div><div class='add'>+{</div><div class='add'>+	u32 cmd_byte = 0;</div><div class='add'>+</div><div class='add'>+	if (HAS_TLV_HEADER(req)) {</div><div class='add'>+		struct roce_tlv *tlv_req = (struct roce_tlv *)req;</div><div class='add'>+</div><div class='add'>+		cmd_byte = tlv_req-&gt;total_size * BNG_FW_CMDQE_UNITS;</div><div class='add'>+	} else {</div><div class='add'>+		cmd_byte = req-&gt;cmd_size;</div><div class='add'>+		req-&gt;cmd_size = (req-&gt;cmd_size + BNG_FW_CMDQE_UNITS - 1) /</div><div class='add'>+				 BNG_FW_CMDQE_UNITS;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return cmd_byte;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bng_re_free_rcfw_channel(struct bng_re_rcfw *rcfw);</div><div class='add'>+int bng_re_alloc_fw_channel(struct bng_re_res *res,</div><div class='add'>+			    struct bng_re_rcfw *rcfw);</div><div class='add'>+int bng_re_enable_fw_channel(struct bng_re_rcfw *rcfw,</div><div class='add'>+			     int msix_vector,</div><div class='add'>+			     int cp_bar_reg_off);</div><div class='add'>+void bng_re_disable_rcfw_channel(struct bng_re_rcfw *rcfw);</div><div class='add'>+int bng_re_rcfw_start_irq(struct bng_re_rcfw *rcfw, int msix_vector,</div><div class='add'>+			  bool need_init);</div><div class='add'>+void bng_re_rcfw_stop_irq(struct bng_re_rcfw *rcfw, bool kill);</div><div class='add'>+int bng_re_rcfw_send_message(struct bng_re_rcfw *rcfw,</div><div class='add'>+			     struct bng_re_cmdqmsg *msg);</div><div class='add'>+int bng_re_init_rcfw(struct bng_re_rcfw *rcfw,</div><div class='add'>+		     struct bng_re_stats *stats_ctx);</div><div class='add'>+int bng_re_deinit_rcfw(struct bng_re_rcfw *rcfw);</div><div class='add'>+#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/bng_re/bng_re.h b/drivers/infiniband/hw/bng_re/bng_re.h<br/>new file mode 100644<br/>index 000000000000..dae4862621a7<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bng_re/bng_re.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bng_re/bng_re.h</a></div><div class='hunk'>@@ -0,0 +1,85 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 */</div><div class='add'>+// Copyright (c) 2025 Broadcom.</div><div class='add'>+</div><div class='add'>+#ifndef __BNG_RE_H__</div><div class='add'>+#define __BNG_RE_H__</div><div class='add'>+</div><div class='add'>+#include "bng_res.h"</div><div class='add'>+</div><div class='add'>+#define BNG_RE_ADEV_NAME		"bng_en"</div><div class='add'>+</div><div class='add'>+#define BNG_RE_DESC	"Broadcom 800G RoCE Driver"</div><div class='add'>+</div><div class='add'>+#define	rdev_to_dev(rdev)	((rdev) ? (&amp;(rdev)-&gt;ibdev.dev) : NULL)</div><div class='add'>+</div><div class='add'>+#define BNG_RE_MIN_MSIX		2</div><div class='add'>+#define BNG_RE_MAX_MSIX		BNGE_MAX_ROCE_MSIX</div><div class='add'>+</div><div class='add'>+#define BNG_RE_CREQ_NQ_IDX	0</div><div class='add'>+</div><div class='add'>+#define BNGE_INVALID_STATS_CTX_ID	-1</div><div class='add'>+/* NQ specific structures  */</div><div class='add'>+struct bng_re_nq_db {</div><div class='add'>+	struct bng_re_reg_desc	reg;</div><div class='add'>+	struct bng_re_db_info	dbinfo;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_nq {</div><div class='add'>+	struct pci_dev			*pdev;</div><div class='add'>+	struct bng_re_res		*res;</div><div class='add'>+	char				*name;</div><div class='add'>+	struct bng_re_hwq		hwq;</div><div class='add'>+	struct bng_re_nq_db		nq_db;</div><div class='add'>+	u16				ring_id;</div><div class='add'>+	int				msix_vec;</div><div class='add'>+	cpumask_t			mask;</div><div class='add'>+	struct tasklet_struct		nq_tasklet;</div><div class='add'>+	bool				requested;</div><div class='add'>+	int				budget;</div><div class='add'>+	u32				load;</div><div class='add'>+</div><div class='add'>+	struct workqueue_struct		*cqn_wq;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_nq_record {</div><div class='add'>+	struct bnge_msix_info	msix_entries[BNG_RE_MAX_MSIX];</div><div class='add'>+	struct bng_re_nq	nq[BNG_RE_MAX_MSIX];</div><div class='add'>+	int			num_msix;</div><div class='add'>+	/* serialize NQ access */</div><div class='add'>+	struct mutex		load_lock;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_en_dev_info {</div><div class='add'>+	struct bng_re_dev *rdev;</div><div class='add'>+	struct bnge_auxr_dev *auxr_dev;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_ring_attr {</div><div class='add'>+	dma_addr_t	*dma_arr;</div><div class='add'>+	int		pages;</div><div class='add'>+	int		type;</div><div class='add'>+	u32		depth;</div><div class='add'>+	u32		lrid; /* Logical ring id */</div><div class='add'>+	u8		mode;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_dev {</div><div class='add'>+	struct ib_device		ibdev;</div><div class='add'>+	unsigned long			flags;</div><div class='add'>+#define BNG_RE_FLAG_NETDEV_REGISTERED		0</div><div class='add'>+#define BNG_RE_FLAG_RCFW_CHANNEL_EN		1</div><div class='add'>+	struct net_device		*netdev;</div><div class='add'>+	struct auxiliary_device         *adev;</div><div class='add'>+	struct bnge_auxr_dev		*aux_dev;</div><div class='add'>+	struct bng_re_chip_ctx		*chip_ctx;</div><div class='add'>+	int				fn_id;</div><div class='add'>+	struct bng_re_res		bng_res;</div><div class='add'>+	struct bng_re_rcfw		rcfw;</div><div class='add'>+	struct bng_re_nq_record		*nqr;</div><div class='add'>+	/* Device Resources */</div><div class='add'>+	struct bng_re_dev_attr		*dev_attr;</div><div class='add'>+	struct dentry			*dbg_root;</div><div class='add'>+	struct bng_re_stats		stats_ctx;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/bng_re/bng_res.c b/drivers/infiniband/hw/bng_re/bng_res.c<br/>new file mode 100644<br/>index 000000000000..c50823758b53<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bng_re/bng_res.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bng_re/bng_res.c</a></div><div class='hunk'>@@ -0,0 +1,279 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0</div><div class='add'>+// Copyright (c) 2025 Broadcom.</div><div class='add'>+</div><div class='add'>+#include &lt;linux/pci.h&gt;</div><div class='add'>+#include &lt;linux/vmalloc.h&gt;</div><div class='add'>+#include &lt;rdma/ib_umem.h&gt;</div><div class='add'>+</div><div class='add'>+#include &lt;linux/bnxt/hsi.h&gt;</div><div class='add'>+#include "bng_res.h"</div><div class='add'>+#include "roce_hsi.h"</div><div class='add'>+</div><div class='add'>+/* Stats */</div><div class='add'>+void bng_re_free_stats_ctx_mem(struct pci_dev *pdev,</div><div class='add'>+			       struct bng_re_stats *stats)</div><div class='add'>+{</div><div class='add'>+	if (stats-&gt;dma) {</div><div class='add'>+		dma_free_coherent(&amp;pdev-&gt;dev, stats-&gt;size,</div><div class='add'>+				  stats-&gt;dma, stats-&gt;dma_map);</div><div class='add'>+	}</div><div class='add'>+	memset(stats, 0, sizeof(*stats));</div><div class='add'>+	stats-&gt;fw_id = -1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bng_re_alloc_stats_ctx_mem(struct pci_dev *pdev,</div><div class='add'>+			       struct bng_re_chip_ctx *cctx,</div><div class='add'>+			       struct bng_re_stats *stats)</div><div class='add'>+{</div><div class='add'>+	memset(stats, 0, sizeof(*stats));</div><div class='add'>+	stats-&gt;fw_id = -1;</div><div class='add'>+	stats-&gt;size = cctx-&gt;hw_stats_size;</div><div class='add'>+	stats-&gt;dma = dma_alloc_coherent(&amp;pdev-&gt;dev, stats-&gt;size,</div><div class='add'>+					&amp;stats-&gt;dma_map, GFP_KERNEL);</div><div class='add'>+	if (!stats-&gt;dma)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bng_free_pbl(struct bng_re_res  *res, struct bng_re_pbl *pbl)</div><div class='add'>+{</div><div class='add'>+	struct pci_dev *pdev = res-&gt;pdev;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; pbl-&gt;pg_count; i++) {</div><div class='add'>+		if (pbl-&gt;pg_arr[i])</div><div class='add'>+			dma_free_coherent(&amp;pdev-&gt;dev, pbl-&gt;pg_size,</div><div class='add'>+					  (void *)((unsigned long)</div><div class='add'>+					     pbl-&gt;pg_arr[i] &amp;</div><div class='add'>+						PAGE_MASK),</div><div class='add'>+					  pbl-&gt;pg_map_arr[i]);</div><div class='add'>+		else</div><div class='add'>+			dev_warn(&amp;pdev-&gt;dev,</div><div class='add'>+					"PBL free pg_arr[%d] empty?!\n", i);</div><div class='add'>+		pbl-&gt;pg_arr[i] = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	vfree(pbl-&gt;pg_arr);</div><div class='add'>+	pbl-&gt;pg_arr = NULL;</div><div class='add'>+	vfree(pbl-&gt;pg_map_arr);</div><div class='add'>+	pbl-&gt;pg_map_arr = NULL;</div><div class='add'>+	pbl-&gt;pg_count = 0;</div><div class='add'>+	pbl-&gt;pg_size = 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bng_alloc_pbl(struct bng_re_res  *res,</div><div class='add'>+			 struct bng_re_pbl *pbl,</div><div class='add'>+			 struct bng_re_sg_info *sginfo)</div><div class='add'>+{</div><div class='add'>+	struct pci_dev *pdev = res-&gt;pdev;</div><div class='add'>+	u32 pages;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (sginfo-&gt;nopte)</div><div class='add'>+		return 0;</div><div class='add'>+	pages = sginfo-&gt;npages;</div><div class='add'>+</div><div class='add'>+	/* page ptr arrays */</div><div class='add'>+	pbl-&gt;pg_arr = vmalloc_array(pages, sizeof(void *));</div><div class='add'>+	if (!pbl-&gt;pg_arr)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	pbl-&gt;pg_map_arr = vmalloc_array(pages, sizeof(dma_addr_t));</div><div class='add'>+	if (!pbl-&gt;pg_map_arr) {</div><div class='add'>+		vfree(pbl-&gt;pg_arr);</div><div class='add'>+		pbl-&gt;pg_arr = NULL;</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+	pbl-&gt;pg_count = 0;</div><div class='add'>+	pbl-&gt;pg_size = sginfo-&gt;pgsize;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; pages; i++) {</div><div class='add'>+		pbl-&gt;pg_arr[i] = dma_alloc_coherent(&amp;pdev-&gt;dev,</div><div class='add'>+				pbl-&gt;pg_size,</div><div class='add'>+				&amp;pbl-&gt;pg_map_arr[i],</div><div class='add'>+				GFP_KERNEL);</div><div class='add'>+		if (!pbl-&gt;pg_arr[i])</div><div class='add'>+			goto fail;</div><div class='add'>+		pbl-&gt;pg_count++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+fail:</div><div class='add'>+	bng_free_pbl(res, pbl);</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bng_re_free_hwq(struct bng_re_res *res,</div><div class='add'>+		     struct bng_re_hwq *hwq)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (!hwq-&gt;max_elements)</div><div class='add'>+		return;</div><div class='add'>+	if (hwq-&gt;level &gt;= BNG_PBL_LVL_MAX)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; hwq-&gt;level + 1; i++)</div><div class='add'>+		bng_free_pbl(res, &amp;hwq-&gt;pbl[i]);</div><div class='add'>+</div><div class='add'>+	hwq-&gt;level = BNG_PBL_LVL_MAX;</div><div class='add'>+	hwq-&gt;max_elements = 0;</div><div class='add'>+	hwq-&gt;element_size = 0;</div><div class='add'>+	hwq-&gt;prod = 0;</div><div class='add'>+	hwq-&gt;cons = 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* All HWQs are power of 2 in size */</div><div class='add'>+int bng_re_alloc_init_hwq(struct bng_re_hwq *hwq,</div><div class='add'>+			  struct bng_re_hwq_attr *hwq_attr)</div><div class='add'>+{</div><div class='add'>+	u32 npages, pg_size;</div><div class='add'>+	struct bng_re_sg_info sginfo = {};</div><div class='add'>+	u32 depth, stride, npbl, npde;</div><div class='add'>+	dma_addr_t *src_phys_ptr, **dst_virt_ptr;</div><div class='add'>+	struct bng_re_res *res;</div><div class='add'>+	struct pci_dev *pdev;</div><div class='add'>+	int i, rc, lvl;</div><div class='add'>+</div><div class='add'>+	res = hwq_attr-&gt;res;</div><div class='add'>+	pdev = res-&gt;pdev;</div><div class='add'>+	pg_size = hwq_attr-&gt;sginfo-&gt;pgsize;</div><div class='add'>+	hwq-&gt;level = BNG_PBL_LVL_MAX;</div><div class='add'>+</div><div class='add'>+	depth = roundup_pow_of_two(hwq_attr-&gt;depth);</div><div class='add'>+	stride = roundup_pow_of_two(hwq_attr-&gt;stride);</div><div class='add'>+</div><div class='add'>+	npages = (depth * stride) / pg_size;</div><div class='add'>+	if ((depth * stride) % pg_size)</div><div class='add'>+		npages++;</div><div class='add'>+	if (!npages)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	hwq_attr-&gt;sginfo-&gt;npages = npages;</div><div class='add'>+</div><div class='add'>+	if (npages == MAX_PBL_LVL_0_PGS &amp;&amp; !hwq_attr-&gt;sginfo-&gt;nopte) {</div><div class='add'>+		/* This request is Level 0, map PTE */</div><div class='add'>+		rc = bng_alloc_pbl(res, &amp;hwq-&gt;pbl[BNG_PBL_LVL_0], hwq_attr-&gt;sginfo);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto fail;</div><div class='add'>+		hwq-&gt;level = BNG_PBL_LVL_0;</div><div class='add'>+		goto done;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (npages &gt;= MAX_PBL_LVL_0_PGS) {</div><div class='add'>+		if (npages &gt; MAX_PBL_LVL_1_PGS) {</div><div class='add'>+			u32 flag = PTU_PTE_VALID;</div><div class='add'>+			/* 2 levels of indirection */</div><div class='add'>+			npbl = npages &gt;&gt; MAX_PBL_LVL_1_PGS_SHIFT;</div><div class='add'>+			if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))</div><div class='add'>+				npbl++;</div><div class='add'>+			npde = npbl &gt;&gt; MAX_PDL_LVL_SHIFT;</div><div class='add'>+			if (npbl % BIT(MAX_PDL_LVL_SHIFT))</div><div class='add'>+				npde++;</div><div class='add'>+			/* Alloc PDE pages */</div><div class='add'>+			sginfo.pgsize = npde * pg_size;</div><div class='add'>+			sginfo.npages = 1;</div><div class='add'>+			rc = bng_alloc_pbl(res, &amp;hwq-&gt;pbl[BNG_PBL_LVL_0], &amp;sginfo);</div><div class='add'>+			if (rc)</div><div class='add'>+				goto fail;</div><div class='add'>+</div><div class='add'>+			/* Alloc PBL pages */</div><div class='add'>+			sginfo.npages = npbl;</div><div class='add'>+			sginfo.pgsize = PAGE_SIZE;</div><div class='add'>+			rc = bng_alloc_pbl(res, &amp;hwq-&gt;pbl[BNG_PBL_LVL_1], &amp;sginfo);</div><div class='add'>+			if (rc)</div><div class='add'>+				goto fail;</div><div class='add'>+			/* Fill PDL with PBL page pointers */</div><div class='add'>+			dst_virt_ptr =</div><div class='add'>+				(dma_addr_t **)hwq-&gt;pbl[BNG_PBL_LVL_0].pg_arr;</div><div class='add'>+			src_phys_ptr = hwq-&gt;pbl[BNG_PBL_LVL_1].pg_map_arr;</div><div class='add'>+			for (i = 0; i &lt; hwq-&gt;pbl[BNG_PBL_LVL_1].pg_count; i++)</div><div class='add'>+				dst_virt_ptr[0][i] = src_phys_ptr[i] | flag;</div><div class='add'>+</div><div class='add'>+			/* Alloc or init PTEs */</div><div class='add'>+			rc = bng_alloc_pbl(res, &amp;hwq-&gt;pbl[BNG_PBL_LVL_2],</div><div class='add'>+					 hwq_attr-&gt;sginfo);</div><div class='add'>+			if (rc)</div><div class='add'>+				goto fail;</div><div class='add'>+			hwq-&gt;level = BNG_PBL_LVL_2;</div><div class='add'>+			if (hwq_attr-&gt;sginfo-&gt;nopte)</div><div class='add'>+				goto done;</div><div class='add'>+			/* Fill PBLs with PTE pointers */</div><div class='add'>+			dst_virt_ptr =</div><div class='add'>+				(dma_addr_t **)hwq-&gt;pbl[BNG_PBL_LVL_1].pg_arr;</div><div class='add'>+			src_phys_ptr = hwq-&gt;pbl[BNG_PBL_LVL_2].pg_map_arr;</div><div class='add'>+			for (i = 0; i &lt; hwq-&gt;pbl[BNG_PBL_LVL_2].pg_count; i++) {</div><div class='add'>+				dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =</div><div class='add'>+					src_phys_ptr[i] | PTU_PTE_VALID;</div><div class='add'>+			}</div><div class='add'>+			if (hwq_attr-&gt;type == BNG_HWQ_TYPE_QUEUE) {</div><div class='add'>+				/* Find the last pg of the size */</div><div class='add'>+				i = hwq-&gt;pbl[BNG_PBL_LVL_2].pg_count;</div><div class='add'>+				dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=</div><div class='add'>+								  PTU_PTE_LAST;</div><div class='add'>+				if (i &gt; 1)</div><div class='add'>+					dst_virt_ptr[PTR_PG(i - 2)]</div><div class='add'>+						    [PTR_IDX(i - 2)] |=</div><div class='add'>+						    PTU_PTE_NEXT_TO_LAST;</div><div class='add'>+			}</div><div class='add'>+		} else { /* pages &lt; 512 npbl = 1, npde = 0 */</div><div class='add'>+			u32 flag = PTU_PTE_VALID;</div><div class='add'>+</div><div class='add'>+			/* 1 level of indirection */</div><div class='add'>+			npbl = npages &gt;&gt; MAX_PBL_LVL_1_PGS_SHIFT;</div><div class='add'>+			if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))</div><div class='add'>+				npbl++;</div><div class='add'>+			sginfo.npages = npbl;</div><div class='add'>+			sginfo.pgsize = PAGE_SIZE;</div><div class='add'>+			/* Alloc PBL page */</div><div class='add'>+			rc = bng_alloc_pbl(res, &amp;hwq-&gt;pbl[BNG_PBL_LVL_0], &amp;sginfo);</div><div class='add'>+			if (rc)</div><div class='add'>+				goto fail;</div><div class='add'>+			/* Alloc or init  PTEs */</div><div class='add'>+			rc = bng_alloc_pbl(res, &amp;hwq-&gt;pbl[BNG_PBL_LVL_1],</div><div class='add'>+					 hwq_attr-&gt;sginfo);</div><div class='add'>+			if (rc)</div><div class='add'>+				goto fail;</div><div class='add'>+			hwq-&gt;level = BNG_PBL_LVL_1;</div><div class='add'>+			if (hwq_attr-&gt;sginfo-&gt;nopte)</div><div class='add'>+				goto done;</div><div class='add'>+			/* Fill PBL with PTE pointers */</div><div class='add'>+			dst_virt_ptr =</div><div class='add'>+				(dma_addr_t **)hwq-&gt;pbl[BNG_PBL_LVL_0].pg_arr;</div><div class='add'>+			src_phys_ptr = hwq-&gt;pbl[BNG_PBL_LVL_1].pg_map_arr;</div><div class='add'>+			for (i = 0; i &lt; hwq-&gt;pbl[BNG_PBL_LVL_1].pg_count; i++)</div><div class='add'>+				dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =</div><div class='add'>+					src_phys_ptr[i] | flag;</div><div class='add'>+			if (hwq_attr-&gt;type == BNG_HWQ_TYPE_QUEUE) {</div><div class='add'>+				/* Find the last pg of the size */</div><div class='add'>+				i = hwq-&gt;pbl[BNG_PBL_LVL_1].pg_count;</div><div class='add'>+				dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=</div><div class='add'>+								  PTU_PTE_LAST;</div><div class='add'>+				if (i &gt; 1)</div><div class='add'>+					dst_virt_ptr[PTR_PG(i - 2)]</div><div class='add'>+						    [PTR_IDX(i - 2)] |=</div><div class='add'>+						    PTU_PTE_NEXT_TO_LAST;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+done:</div><div class='add'>+	hwq-&gt;prod = 0;</div><div class='add'>+	hwq-&gt;cons = 0;</div><div class='add'>+	hwq-&gt;pdev = pdev;</div><div class='add'>+	hwq-&gt;depth = hwq_attr-&gt;depth;</div><div class='add'>+	hwq-&gt;max_elements = hwq-&gt;depth;</div><div class='add'>+	hwq-&gt;element_size = stride;</div><div class='add'>+	hwq-&gt;qe_ppg = pg_size / stride;</div><div class='add'>+	/* For direct access to the elements */</div><div class='add'>+	lvl = hwq-&gt;level;</div><div class='add'>+	if (hwq_attr-&gt;sginfo-&gt;nopte &amp;&amp; hwq-&gt;level)</div><div class='add'>+		lvl = hwq-&gt;level - 1;</div><div class='add'>+	hwq-&gt;pbl_ptr = hwq-&gt;pbl[lvl].pg_arr;</div><div class='add'>+	hwq-&gt;pbl_dma_ptr = hwq-&gt;pbl[lvl].pg_map_arr;</div><div class='add'>+	spin_lock_init(&amp;hwq-&gt;lock);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+fail:</div><div class='add'>+	bng_re_free_hwq(res, hwq);</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/bng_re/bng_res.h b/drivers/infiniband/hw/bng_re/bng_res.h<br/>new file mode 100644<br/>index 000000000000..9997f86d6a0e<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bng_re/bng_res.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bng_re/bng_res.h</a></div><div class='hunk'>@@ -0,0 +1,215 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 */</div><div class='add'>+// Copyright (c) 2025 Broadcom.</div><div class='add'>+</div><div class='add'>+#ifndef __BNG_RES_H__</div><div class='add'>+#define __BNG_RES_H__</div><div class='add'>+</div><div class='add'>+#include "roce_hsi.h"</div><div class='add'>+</div><div class='add'>+#define BNG_ROCE_FW_MAX_TIMEOUT	60</div><div class='add'>+</div><div class='add'>+#define PTR_CNT_PER_PG		(PAGE_SIZE / sizeof(void *))</div><div class='add'>+#define PTR_MAX_IDX_PER_PG	(PTR_CNT_PER_PG - 1)</div><div class='add'>+#define PTR_PG(x)		(((x) &amp; ~PTR_MAX_IDX_PER_PG) / PTR_CNT_PER_PG)</div><div class='add'>+#define PTR_IDX(x)		((x) &amp; PTR_MAX_IDX_PER_PG)</div><div class='add'>+</div><div class='add'>+#define HWQ_CMP(idx, hwq)	((idx) &amp; ((hwq)-&gt;max_elements - 1))</div><div class='add'>+#define HWQ_FREE_SLOTS(hwq)	(hwq-&gt;max_elements - \</div><div class='add'>+				((HWQ_CMP(hwq-&gt;prod, hwq)\</div><div class='add'>+				- HWQ_CMP(hwq-&gt;cons, hwq))\</div><div class='add'>+				&amp; (hwq-&gt;max_elements - 1)))</div><div class='add'>+</div><div class='add'>+#define MAX_PBL_LVL_0_PGS		1</div><div class='add'>+#define MAX_PBL_LVL_1_PGS		512</div><div class='add'>+#define MAX_PBL_LVL_1_PGS_SHIFT		9</div><div class='add'>+#define MAX_PBL_LVL_1_PGS_FOR_LVL_2	256</div><div class='add'>+#define MAX_PBL_LVL_2_PGS		(256 * 512)</div><div class='add'>+#define MAX_PDL_LVL_SHIFT               9</div><div class='add'>+</div><div class='add'>+#define BNG_RE_DBR_VALID		(0x1UL &lt;&lt; 26)</div><div class='add'>+#define BNG_RE_DBR_EPOCH_SHIFT	24</div><div class='add'>+#define BNG_RE_DBR_TOGGLE_SHIFT	25</div><div class='add'>+</div><div class='add'>+#define BNG_MAX_TQM_ALLOC_REQ	48</div><div class='add'>+</div><div class='add'>+struct bng_re_reg_desc {</div><div class='add'>+	u8		bar_id;</div><div class='add'>+	resource_size_t	bar_base;</div><div class='add'>+	unsigned long	offset;</div><div class='add'>+	void __iomem	*bar_reg;</div><div class='add'>+	size_t		len;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_db_info {</div><div class='add'>+	void __iomem		*db;</div><div class='add'>+	void __iomem		*priv_db;</div><div class='add'>+	struct bng_re_hwq	*hwq;</div><div class='add'>+	u32			xid;</div><div class='add'>+	u32			max_slot;</div><div class='add'>+	u32                     flags;</div><div class='add'>+	u8			toggle;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum bng_re_db_info_flags_mask {</div><div class='add'>+	BNG_RE_FLAG_EPOCH_CONS_SHIFT        = 0x0UL,</div><div class='add'>+	BNG_RE_FLAG_EPOCH_PROD_SHIFT        = 0x1UL,</div><div class='add'>+	BNG_RE_FLAG_EPOCH_CONS_MASK         = 0x1UL,</div><div class='add'>+	BNG_RE_FLAG_EPOCH_PROD_MASK         = 0x2UL,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum bng_re_db_epoch_flag_shift {</div><div class='add'>+	BNG_RE_DB_EPOCH_CONS_SHIFT  = BNG_RE_DBR_EPOCH_SHIFT,</div><div class='add'>+	BNG_RE_DB_EPOCH_PROD_SHIFT  = (BNG_RE_DBR_EPOCH_SHIFT - 1),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_chip_ctx {</div><div class='add'>+	u16	chip_num;</div><div class='add'>+	u16	hw_stats_size;</div><div class='add'>+	u64	hwrm_intf_ver;</div><div class='add'>+	u16	hwrm_cmd_max_timeout;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_pbl {</div><div class='add'>+	u32		pg_count;</div><div class='add'>+	u32		pg_size;</div><div class='add'>+	void		**pg_arr;</div><div class='add'>+	dma_addr_t	*pg_map_arr;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum bng_re_pbl_lvl {</div><div class='add'>+	BNG_PBL_LVL_0,</div><div class='add'>+	BNG_PBL_LVL_1,</div><div class='add'>+	BNG_PBL_LVL_2,</div><div class='add'>+	BNG_PBL_LVL_MAX</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum bng_re_hwq_type {</div><div class='add'>+	BNG_HWQ_TYPE_CTX,</div><div class='add'>+	BNG_HWQ_TYPE_QUEUE</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_sg_info {</div><div class='add'>+	u32	npages;</div><div class='add'>+	u32	pgshft;</div><div class='add'>+	u32	pgsize;</div><div class='add'>+	bool	nopte;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_hwq_attr {</div><div class='add'>+	struct bng_re_res		*res;</div><div class='add'>+	struct bng_re_sg_info		*sginfo;</div><div class='add'>+	enum bng_re_hwq_type		type;</div><div class='add'>+	u32				depth;</div><div class='add'>+	u32				stride;</div><div class='add'>+	u32				aux_stride;</div><div class='add'>+	u32				aux_depth;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_hwq {</div><div class='add'>+	struct pci_dev			*pdev;</div><div class='add'>+	/* lock to protect hwq */</div><div class='add'>+	spinlock_t			lock;</div><div class='add'>+	struct bng_re_pbl		pbl[BNG_PBL_LVL_MAX + 1];</div><div class='add'>+	/* Valid values: 0, 1, 2 */</div><div class='add'>+	enum bng_re_pbl_lvl		level;</div><div class='add'>+	/* PBL entries */</div><div class='add'>+	void				**pbl_ptr;</div><div class='add'>+	/* PBL  dma_addr */</div><div class='add'>+	dma_addr_t			*pbl_dma_ptr;</div><div class='add'>+	u32				max_elements;</div><div class='add'>+	u32				depth;</div><div class='add'>+	u16				element_size;</div><div class='add'>+	u32				prod;</div><div class='add'>+	u32				cons;</div><div class='add'>+	/* queue entry per page */</div><div class='add'>+	u16				qe_ppg;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_stats {</div><div class='add'>+	dma_addr_t			dma_map;</div><div class='add'>+	void				*dma;</div><div class='add'>+	u32				size;</div><div class='add'>+	u32				fw_id;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bng_re_res {</div><div class='add'>+	struct pci_dev			*pdev;</div><div class='add'>+	struct bng_re_chip_ctx		*cctx;</div><div class='add'>+	struct bng_re_dev_attr		*dattr;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline void *bng_re_get_qe(struct bng_re_hwq *hwq,</div><div class='add'>+				  u32 indx, u64 *pg)</div><div class='add'>+{</div><div class='add'>+	u32 pg_num, pg_idx;</div><div class='add'>+</div><div class='add'>+	pg_num = (indx / hwq-&gt;qe_ppg);</div><div class='add'>+	pg_idx = (indx % hwq-&gt;qe_ppg);</div><div class='add'>+	if (pg)</div><div class='add'>+		*pg = (u64)&amp;hwq-&gt;pbl_ptr[pg_num];</div><div class='add'>+	return (void *)(hwq-&gt;pbl_ptr[pg_num] + hwq-&gt;element_size * pg_idx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define BNG_RE_INIT_DBHDR(xid, type, indx, toggle) \</div><div class='add'>+	(((u64)(((xid) &amp; DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE |  \</div><div class='add'>+		(type) | BNG_RE_DBR_VALID) &lt;&lt; 32) | (indx) |  \</div><div class='add'>+	 (((u32)(toggle)) &lt;&lt; (BNG_RE_DBR_TOGGLE_SHIFT)))</div><div class='add'>+</div><div class='add'>+static inline void bng_re_ring_db(struct bng_re_db_info *info,</div><div class='add'>+				  u32 type)</div><div class='add'>+{</div><div class='add'>+	u64 key = 0;</div><div class='add'>+	u32 indx;</div><div class='add'>+	u8 toggle = 0;</div><div class='add'>+</div><div class='add'>+	if (type == DBC_DBC_TYPE_CQ_ARMALL ||</div><div class='add'>+	    type == DBC_DBC_TYPE_CQ_ARMSE)</div><div class='add'>+		toggle = info-&gt;toggle;</div><div class='add'>+</div><div class='add'>+	indx = (info-&gt;hwq-&gt;cons &amp; DBC_DBC_INDEX_MASK) |</div><div class='add'>+	       ((info-&gt;flags &amp; BNG_RE_FLAG_EPOCH_CONS_MASK) &lt;&lt;</div><div class='add'>+		 BNG_RE_DB_EPOCH_CONS_SHIFT);</div><div class='add'>+</div><div class='add'>+	key =  BNG_RE_INIT_DBHDR(info-&gt;xid, type, indx, toggle);</div><div class='add'>+	writeq(key, info-&gt;db);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void bng_re_ring_nq_db(struct bng_re_db_info *info,</div><div class='add'>+				     struct bng_re_chip_ctx *cctx,</div><div class='add'>+				     bool arm)</div><div class='add'>+{</div><div class='add'>+	u32 type;</div><div class='add'>+</div><div class='add'>+	type = arm ? DBC_DBC_TYPE_NQ_ARM : DBC_DBC_TYPE_NQ;</div><div class='add'>+	bng_re_ring_db(info, type);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void bng_re_hwq_incr_cons(u32 max_elements, u32 *cons, u32 cnt,</div><div class='add'>+					u32 *dbinfo_flags)</div><div class='add'>+{</div><div class='add'>+	/* move cons and update toggle/epoch if wrap around */</div><div class='add'>+	*cons += cnt;</div><div class='add'>+	if (*cons &gt;= max_elements) {</div><div class='add'>+		*cons %= max_elements;</div><div class='add'>+		*dbinfo_flags ^= 1UL &lt;&lt; BNG_RE_FLAG_EPOCH_CONS_SHIFT;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool _is_max_srq_ext_supported(u16 dev_cap_ext_flags_2)</div><div class='add'>+{</div><div class='add'>+	return !!(dev_cap_ext_flags_2 &amp; CREQ_QUERY_FUNC_RESP_SB_MAX_SRQ_EXTENDED);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bng_re_free_hwq(struct bng_re_res *res,</div><div class='add'>+		     struct bng_re_hwq *hwq);</div><div class='add'>+</div><div class='add'>+int bng_re_alloc_init_hwq(struct bng_re_hwq *hwq,</div><div class='add'>+			  struct bng_re_hwq_attr *hwq_attr);</div><div class='add'>+</div><div class='add'>+void bng_re_free_stats_ctx_mem(struct pci_dev *pdev,</div><div class='add'>+			       struct bng_re_stats *stats);</div><div class='add'>+</div><div class='add'>+int bng_re_alloc_stats_ctx_mem(struct pci_dev *pdev,</div><div class='add'>+			       struct bng_re_chip_ctx *cctx,</div><div class='add'>+			       struct bng_re_stats *stats);</div><div class='add'>+#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/bng_re/bng_sp.c b/drivers/infiniband/hw/bng_re/bng_sp.c<br/>new file mode 100644<br/>index 000000000000..83099e05328d<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bng_re/bng_sp.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bng_re/bng_sp.c</a></div><div class='hunk'>@@ -0,0 +1,131 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0</div><div class='add'>+// Copyright (c) 2025 Broadcom.</div><div class='add'>+#include &lt;linux/interrupt.h&gt;</div><div class='add'>+#include &lt;linux/pci.h&gt;</div><div class='add'>+</div><div class='add'>+#include "bng_res.h"</div><div class='add'>+#include "bng_fw.h"</div><div class='add'>+#include "bng_sp.h"</div><div class='add'>+#include "bng_tlv.h"</div><div class='add'>+</div><div class='add'>+static bool bng_re_is_atomic_cap(struct bng_re_rcfw *rcfw)</div><div class='add'>+{</div><div class='add'>+	u16 pcie_ctl2 = 0;</div><div class='add'>+</div><div class='add'>+	pcie_capability_read_word(rcfw-&gt;pdev, PCI_EXP_DEVCTL2, &amp;pcie_ctl2);</div><div class='add'>+	return (pcie_ctl2 &amp; PCI_EXP_DEVCTL2_ATOMIC_REQ);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bng_re_query_version(struct bng_re_rcfw *rcfw,</div><div class='add'>+				 char *fw_ver)</div><div class='add'>+{</div><div class='add'>+	struct creq_query_version_resp resp = {};</div><div class='add'>+	struct bng_re_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_query_version req = {};</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	bng_re_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+			     CMDQ_BASE_OPCODE_QUERY_VERSION,</div><div class='add'>+			     sizeof(req));</div><div class='add'>+</div><div class='add'>+	bng_re_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req), sizeof(resp), 0);</div><div class='add'>+	rc = bng_re_rcfw_send_message(rcfw, &amp;msg);</div><div class='add'>+	if (rc)</div><div class='add'>+		return;</div><div class='add'>+	fw_ver[0] = resp.fw_maj;</div><div class='add'>+	fw_ver[1] = resp.fw_minor;</div><div class='add'>+	fw_ver[2] = resp.fw_bld;</div><div class='add'>+	fw_ver[3] = resp.fw_rsvd;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bng_re_get_dev_attr(struct bng_re_rcfw *rcfw)</div><div class='add'>+{</div><div class='add'>+	struct bng_re_dev_attr *attr = rcfw-&gt;res-&gt;dattr;</div><div class='add'>+	struct creq_query_func_resp resp = {};</div><div class='add'>+	struct bng_re_cmdqmsg msg = {};</div><div class='add'>+	struct creq_query_func_resp_sb *sb;</div><div class='add'>+	struct bng_re_rcfw_sbuf sbuf;</div><div class='add'>+	struct cmdq_query_func req = {};</div><div class='add'>+	u8 *tqm_alloc;</div><div class='add'>+	int i, rc;</div><div class='add'>+	u32 temp;</div><div class='add'>+</div><div class='add'>+	bng_re_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+			     CMDQ_BASE_OPCODE_QUERY_FUNC,</div><div class='add'>+			     sizeof(req));</div><div class='add'>+</div><div class='add'>+	sbuf.size = ALIGN(sizeof(*sb), BNG_FW_CMDQE_UNITS);</div><div class='add'>+	sbuf.sb = dma_alloc_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf.size,</div><div class='add'>+				     &amp;sbuf.dma_addr, GFP_KERNEL);</div><div class='add'>+	if (!sbuf.sb)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	sb = sbuf.sb;</div><div class='add'>+	req.resp_size = sbuf.size / BNG_FW_CMDQE_UNITS;</div><div class='add'>+	bng_re_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, &amp;sbuf, sizeof(req),</div><div class='add'>+			    sizeof(resp), 0);</div><div class='add'>+	rc = bng_re_rcfw_send_message(rcfw, &amp;msg);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto bail;</div><div class='add'>+	/* Extract the context from the side buffer */</div><div class='add'>+	attr-&gt;max_qp = le32_to_cpu(sb-&gt;max_qp);</div><div class='add'>+	/* max_qp value reported by FW doesn't include the QP1 */</div><div class='add'>+	attr-&gt;max_qp += 1;</div><div class='add'>+	attr-&gt;max_qp_rd_atom =</div><div class='add'>+		sb-&gt;max_qp_rd_atom &gt; BNG_RE_MAX_OUT_RD_ATOM ?</div><div class='add'>+		BNG_RE_MAX_OUT_RD_ATOM : sb-&gt;max_qp_rd_atom;</div><div class='add'>+	attr-&gt;max_qp_init_rd_atom =</div><div class='add'>+		sb-&gt;max_qp_init_rd_atom &gt; BNG_RE_MAX_OUT_RD_ATOM ?</div><div class='add'>+		BNG_RE_MAX_OUT_RD_ATOM : sb-&gt;max_qp_init_rd_atom;</div><div class='add'>+	attr-&gt;max_qp_wqes = le16_to_cpu(sb-&gt;max_qp_wr) - 1;</div><div class='add'>+</div><div class='add'>+	/* Adjust for max_qp_wqes for variable wqe */</div><div class='add'>+	attr-&gt;max_qp_wqes = min_t(u32, attr-&gt;max_qp_wqes, BNG_VAR_MAX_WQE - 1);</div><div class='add'>+</div><div class='add'>+	attr-&gt;max_qp_sges = min_t(u32, sb-&gt;max_sge_var_wqe, BNG_VAR_MAX_SGE);</div><div class='add'>+	attr-&gt;max_cq = le32_to_cpu(sb-&gt;max_cq);</div><div class='add'>+	attr-&gt;max_cq_wqes = le32_to_cpu(sb-&gt;max_cqe);</div><div class='add'>+	attr-&gt;max_cq_sges = attr-&gt;max_qp_sges;</div><div class='add'>+	attr-&gt;max_mr = le32_to_cpu(sb-&gt;max_mr);</div><div class='add'>+	attr-&gt;max_mw = le32_to_cpu(sb-&gt;max_mw);</div><div class='add'>+</div><div class='add'>+	attr-&gt;max_mr_size = le64_to_cpu(sb-&gt;max_mr_size);</div><div class='add'>+	attr-&gt;max_pd = 64 * 1024;</div><div class='add'>+	attr-&gt;max_raw_ethy_qp = le32_to_cpu(sb-&gt;max_raw_eth_qp);</div><div class='add'>+	attr-&gt;max_ah = le32_to_cpu(sb-&gt;max_ah);</div><div class='add'>+</div><div class='add'>+	attr-&gt;max_srq = le16_to_cpu(sb-&gt;max_srq);</div><div class='add'>+	attr-&gt;max_srq_wqes = le32_to_cpu(sb-&gt;max_srq_wr) - 1;</div><div class='add'>+	attr-&gt;max_srq_sges = sb-&gt;max_srq_sge;</div><div class='add'>+	attr-&gt;max_pkey = 1;</div><div class='add'>+	attr-&gt;max_inline_data = le32_to_cpu(sb-&gt;max_inline_data);</div><div class='add'>+	/*</div><div class='add'>+	 * Read the max gid supported by HW.</div><div class='add'>+	 * For each entry in HW  GID in HW table, we consume 2</div><div class='add'>+	 * GID entries in the kernel GID table.  So max_gid reported</div><div class='add'>+	 * to stack can be up to twice the value reported by the HW, up to 256 gids.</div><div class='add'>+	 */</div><div class='add'>+	attr-&gt;max_sgid = le32_to_cpu(sb-&gt;max_gid);</div><div class='add'>+	attr-&gt;max_sgid = min_t(u32, BNG_RE_NUM_GIDS_SUPPORTED, 2 * attr-&gt;max_sgid);</div><div class='add'>+	attr-&gt;dev_cap_flags = le16_to_cpu(sb-&gt;dev_cap_flags);</div><div class='add'>+	attr-&gt;dev_cap_flags2 = le16_to_cpu(sb-&gt;dev_cap_ext_flags_2);</div><div class='add'>+</div><div class='add'>+	if (_is_max_srq_ext_supported(attr-&gt;dev_cap_flags2))</div><div class='add'>+		attr-&gt;max_srq += le16_to_cpu(sb-&gt;max_srq_ext);</div><div class='add'>+</div><div class='add'>+	bng_re_query_version(rcfw, attr-&gt;fw_ver);</div><div class='add'>+	for (i = 0; i &lt; BNG_MAX_TQM_ALLOC_REQ / 4; i++) {</div><div class='add'>+		temp = le32_to_cpu(sb-&gt;tqm_alloc_reqs[i]);</div><div class='add'>+		tqm_alloc = (u8 *)&amp;temp;</div><div class='add'>+		attr-&gt;tqm_alloc_reqs[i * 4] = *tqm_alloc;</div><div class='add'>+		attr-&gt;tqm_alloc_reqs[i * 4 + 1] = *(++tqm_alloc);</div><div class='add'>+		attr-&gt;tqm_alloc_reqs[i * 4 + 2] = *(++tqm_alloc);</div><div class='add'>+		attr-&gt;tqm_alloc_reqs[i * 4 + 3] = *(++tqm_alloc);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	attr-&gt;max_dpi = le32_to_cpu(sb-&gt;max_dpi);</div><div class='add'>+	attr-&gt;is_atomic = bng_re_is_atomic_cap(rcfw);</div><div class='add'>+bail:</div><div class='add'>+	dma_free_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf.size,</div><div class='add'>+			  sbuf.sb, sbuf.dma_addr);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/bng_re/bng_sp.h b/drivers/infiniband/hw/bng_re/bng_sp.h<br/>new file mode 100644<br/>index 000000000000..e15190515ed1<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bng_re/bng_sp.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bng_re/bng_sp.h</a></div><div class='hunk'>@@ -0,0 +1,47 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 */</div><div class='add'>+// Copyright (c) 2025 Broadcom.</div><div class='add'>+</div><div class='add'>+#ifndef __BNG_SP_H__</div><div class='add'>+#define __BNG_SP_H__</div><div class='add'>+</div><div class='add'>+#include "bng_fw.h"</div><div class='add'>+</div><div class='add'>+#define BNG_VAR_MAX_WQE		4352</div><div class='add'>+#define BNG_VAR_MAX_SGE		13</div><div class='add'>+</div><div class='add'>+struct bng_re_dev_attr {</div><div class='add'>+#define FW_VER_ARR_LEN			4</div><div class='add'>+	u8				fw_ver[FW_VER_ARR_LEN];</div><div class='add'>+#define BNG_RE_NUM_GIDS_SUPPORTED	256</div><div class='add'>+	u16				max_sgid;</div><div class='add'>+	u16				max_mrw;</div><div class='add'>+	u32				max_qp;</div><div class='add'>+#define BNG_RE_MAX_OUT_RD_ATOM		126</div><div class='add'>+	u32				max_qp_rd_atom;</div><div class='add'>+	u32				max_qp_init_rd_atom;</div><div class='add'>+	u32				max_qp_wqes;</div><div class='add'>+	u32				max_qp_sges;</div><div class='add'>+	u32				max_cq;</div><div class='add'>+	u32				max_cq_wqes;</div><div class='add'>+	u32				max_cq_sges;</div><div class='add'>+	u32				max_mr;</div><div class='add'>+	u64				max_mr_size;</div><div class='add'>+	u32				max_pd;</div><div class='add'>+	u32				max_mw;</div><div class='add'>+	u32				max_raw_ethy_qp;</div><div class='add'>+	u32				max_ah;</div><div class='add'>+	u32				max_srq;</div><div class='add'>+	u32				max_srq_wqes;</div><div class='add'>+	u32				max_srq_sges;</div><div class='add'>+	u32				max_pkey;</div><div class='add'>+	u32				max_inline_data;</div><div class='add'>+	u32				l2_db_size;</div><div class='add'>+	u8				tqm_alloc_reqs[BNG_MAX_TQM_ALLOC_REQ];</div><div class='add'>+	bool				is_atomic;</div><div class='add'>+	u16                             dev_cap_flags;</div><div class='add'>+	u16                             dev_cap_flags2;</div><div class='add'>+	u32                             max_dpi;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+int bng_re_get_dev_attr(struct bng_re_rcfw *rcfw);</div><div class='add'>+#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/bng_re/bng_tlv.h b/drivers/infiniband/hw/bng_re/bng_tlv.h<br/>new file mode 100644<br/>index 000000000000..278f4922962d<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bng_re/bng_tlv.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bng_re/bng_tlv.h</a></div><div class='hunk'>@@ -0,0 +1,128 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='add'>+</div><div class='add'>+#ifndef __BNG_TLV_H__</div><div class='add'>+#define __BNG_TLV_H__</div><div class='add'>+</div><div class='add'>+#include "roce_hsi.h"</div><div class='add'>+</div><div class='add'>+struct roce_tlv {</div><div class='add'>+	struct tlv tlv;</div><div class='add'>+	u8 total_size; // in units of 16 byte chunks</div><div class='add'>+	u8 unused[7];  // for 16 byte alignment</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * TLV size in units of 16 byte chunks</div><div class='add'>+ */</div><div class='add'>+#define TLV_SIZE ((sizeof(struct roce_tlv) + 15) / 16)</div><div class='add'>+/*</div><div class='add'>+ * TLV length in bytes</div><div class='add'>+ */</div><div class='add'>+#define TLV_BYTES (TLV_SIZE * 16)</div><div class='add'>+</div><div class='add'>+#define HAS_TLV_HEADER(msg) (le16_to_cpu(((struct tlv *)(msg))-&gt;cmd_discr) == CMD_DISCR_TLV_ENCAP)</div><div class='add'>+#define GET_TLV_DATA(tlv)   ((void *)&amp;((uint8_t *)(tlv))[TLV_BYTES])</div><div class='add'>+</div><div class='add'>+static inline u8 __get_cmdq_base_opcode(struct cmdq_base *req, u32 size)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		return ((struct cmdq_base *)GET_TLV_DATA(req))-&gt;opcode;</div><div class='add'>+	else</div><div class='add'>+		return req-&gt;opcode;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void __set_cmdq_base_opcode(struct cmdq_base *req,</div><div class='add'>+					  u32 size, u8 val)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		((struct cmdq_base *)GET_TLV_DATA(req))-&gt;opcode = val;</div><div class='add'>+	else</div><div class='add'>+		req-&gt;opcode = val;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline __le16 __get_cmdq_base_cookie(struct cmdq_base *req, u32 size)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		return ((struct cmdq_base *)GET_TLV_DATA(req))-&gt;cookie;</div><div class='add'>+	else</div><div class='add'>+		return req-&gt;cookie;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void __set_cmdq_base_cookie(struct cmdq_base *req,</div><div class='add'>+					  u32 size, __le16 val)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		((struct cmdq_base *)GET_TLV_DATA(req))-&gt;cookie = val;</div><div class='add'>+	else</div><div class='add'>+		req-&gt;cookie = val;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline __le64 __get_cmdq_base_resp_addr(struct cmdq_base *req, u32 size)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		return ((struct cmdq_base *)GET_TLV_DATA(req))-&gt;resp_addr;</div><div class='add'>+	else</div><div class='add'>+		return req-&gt;resp_addr;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void __set_cmdq_base_resp_addr(struct cmdq_base *req,</div><div class='add'>+					     u32 size, __le64 val)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		((struct cmdq_base *)GET_TLV_DATA(req))-&gt;resp_addr = val;</div><div class='add'>+	else</div><div class='add'>+		req-&gt;resp_addr = val;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u8 __get_cmdq_base_resp_size(struct cmdq_base *req, u32 size)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		return ((struct cmdq_base *)GET_TLV_DATA(req))-&gt;resp_size;</div><div class='add'>+	else</div><div class='add'>+		return req-&gt;resp_size;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void __set_cmdq_base_resp_size(struct cmdq_base *req,</div><div class='add'>+					     u32 size, u8 val)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		((struct cmdq_base *)GET_TLV_DATA(req))-&gt;resp_size = val;</div><div class='add'>+	else</div><div class='add'>+		req-&gt;resp_size = val;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u8 __get_cmdq_base_cmd_size(struct cmdq_base *req, u32 size)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		return ((struct roce_tlv *)(req))-&gt;total_size;</div><div class='add'>+	else</div><div class='add'>+		return req-&gt;cmd_size;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void __set_cmdq_base_cmd_size(struct cmdq_base *req,</div><div class='add'>+					    u32 size, u8 val)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		((struct cmdq_base *)GET_TLV_DATA(req))-&gt;cmd_size = val;</div><div class='add'>+	else</div><div class='add'>+		req-&gt;cmd_size = val;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline __le16 __get_cmdq_base_flags(struct cmdq_base *req, u32 size)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		return ((struct cmdq_base *)GET_TLV_DATA(req))-&gt;flags;</div><div class='add'>+	else</div><div class='add'>+		return req-&gt;flags;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void __set_cmdq_base_flags(struct cmdq_base *req,</div><div class='add'>+					 u32 size, __le16 val)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		((struct cmdq_base *)GET_TLV_DATA(req))-&gt;flags = val;</div><div class='add'>+	else</div><div class='add'>+		req-&gt;flags = val;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#endif /* __BNG_TLV_H__ */</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/Kconfig b/drivers/infiniband/hw/bnxt_re/Kconfig<br/>index 51e8234520a9..6a17f5cdb020 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/Kconfig?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/Kconfig</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/Kconfig?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/Kconfig</a></div><div class='hunk'>@@ -1,10 +1,9 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='ctx'> config INFINIBAND_BNXT_RE</div><div class='del'>-        tristate "Broadcom Netxtreme HCA support"</div><div class='del'>-        depends on 64BIT</div><div class='del'>-        depends on ETHERNET &amp;&amp; NETDEVICES &amp;&amp; PCI &amp;&amp; INET &amp;&amp; DCB</div><div class='del'>-        select NET_VENDOR_BROADCOM</div><div class='del'>-        select BNXT</div><div class='del'>-        ---help---</div><div class='add'>+	tristate "Broadcom Netxtreme HCA support"</div><div class='add'>+	depends on 64BIT</div><div class='add'>+	depends on INET &amp;&amp; DCB &amp;&amp; BNXT</div><div class='add'>+	help</div><div class='ctx'> 	  This driver supports Broadcom NetXtreme-E 10/25/40/50 gigabit</div><div class='ctx'> 	  RoCE HCAs.  To compile this driver as a module, choose M here:</div><div class='ctx'> 	  the module will be called bnxt_re.</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/Makefile b/drivers/infiniband/hw/bnxt_re/Makefile<br/>index ee9bb1be61ea..f63417d2ccc6 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/Makefile?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/Makefile</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/Makefile?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/Makefile</a></div><div class='hunk'>@@ -4,4 +4,5 @@ ccflags-y := -I $(srctree)/drivers/net/ethernet/broadcom/bnxt</div><div class='ctx'> obj-$(CONFIG_INFINIBAND_BNXT_RE) += bnxt_re.o</div><div class='ctx'> bnxt_re-y := main.o ib_verbs.o \</div><div class='ctx'> 	     qplib_res.o qplib_rcfw.o	\</div><div class='del'>-	     qplib_sp.o qplib_fp.o  hw_counters.o</div><div class='add'>+	     qplib_sp.o qplib_fp.o  hw_counters.o	\</div><div class='add'>+	     debugfs.o</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h<br/>index e55a1666c0cd..3a7ce4729fcf 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/bnxt_re.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/bnxt_re.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/bnxt_re.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/bnxt_re.h</a></div><div class='hunk'>@@ -39,33 +39,20 @@</div><div class='ctx'> </div><div class='ctx'> #ifndef __BNXT_RE_H__</div><div class='ctx'> #define __BNXT_RE_H__</div><div class='add'>+#include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='add'>+#include "hw_counters.h"</div><div class='add'>+#include &lt;linux/hashtable.h&gt;</div><div class='ctx'> #define ROCE_DRV_MODULE_NAME		"bnxt_re"</div><div class='ctx'> </div><div class='ctx'> #define BNXT_RE_DESC	"Broadcom NetXtreme-C/E RoCE Driver"</div><div class='del'>-#define BNXT_RE_PAGE_SHIFT_4K		(12)</div><div class='del'>-#define BNXT_RE_PAGE_SHIFT_8K		(13)</div><div class='del'>-#define BNXT_RE_PAGE_SHIFT_64K		(16)</div><div class='del'>-#define BNXT_RE_PAGE_SHIFT_2M		(21)</div><div class='del'>-#define BNXT_RE_PAGE_SHIFT_8M		(23)</div><div class='del'>-#define BNXT_RE_PAGE_SHIFT_1G		(30)</div><div class='ctx'> </div><div class='del'>-#define BNXT_RE_PAGE_SIZE_4K		BIT(BNXT_RE_PAGE_SHIFT_4K)</div><div class='del'>-#define BNXT_RE_PAGE_SIZE_8K		BIT(BNXT_RE_PAGE_SHIFT_8K)</div><div class='del'>-#define BNXT_RE_PAGE_SIZE_64K		BIT(BNXT_RE_PAGE_SHIFT_64K)</div><div class='del'>-#define BNXT_RE_PAGE_SIZE_2M		BIT(BNXT_RE_PAGE_SHIFT_2M)</div><div class='del'>-#define BNXT_RE_PAGE_SIZE_8M		BIT(BNXT_RE_PAGE_SHIFT_8M)</div><div class='del'>-#define BNXT_RE_PAGE_SIZE_1G		BIT(BNXT_RE_PAGE_SHIFT_1G)</div><div class='add'>+#define BNXT_RE_PAGE_SHIFT_1G		(30)</div><div class='add'>+#define BNXT_RE_PAGE_SIZE_SUPPORTED	0x7FFFF000 /* 4kb - 1G */</div><div class='ctx'> </div><div class='ctx'> #define BNXT_RE_MAX_MR_SIZE_LOW		BIT_ULL(BNXT_RE_PAGE_SHIFT_1G)</div><div class='ctx'> #define BNXT_RE_MAX_MR_SIZE_HIGH	BIT_ULL(39)</div><div class='ctx'> #define BNXT_RE_MAX_MR_SIZE		BNXT_RE_MAX_MR_SIZE_HIGH</div><div class='ctx'> </div><div class='del'>-#define BNXT_RE_MAX_QPC_COUNT		(64 * 1024)</div><div class='del'>-#define BNXT_RE_MAX_MRW_COUNT		(64 * 1024)</div><div class='del'>-#define BNXT_RE_MAX_SRQC_COUNT		(64 * 1024)</div><div class='del'>-#define BNXT_RE_MAX_CQ_COUNT		(64 * 1024)</div><div class='del'>-#define BNXT_RE_MAX_MRW_COUNT_64K	(64 * 1024)</div><div class='del'>-#define BNXT_RE_MAX_MRW_COUNT_256K	(256 * 1024)</div><div class='ctx'> </div><div class='ctx'> /* Number of MRs to reserve for PF, leaving remainder for VFs */</div><div class='ctx'> #define BNXT_RE_RESVD_MR_FOR_PF         (32 * 1024)</div><div class='hunk'>@@ -89,13 +76,24 @@</div><div class='ctx'> </div><div class='ctx'> #define BNXT_RE_DEFAULT_ACK_DELAY	16</div><div class='ctx'> </div><div class='del'>-struct bnxt_re_work {</div><div class='del'>-	struct work_struct	work;</div><div class='del'>-	unsigned long		event;</div><div class='del'>-	struct bnxt_re_dev      *rdev;</div><div class='del'>-	struct net_device	*vlan_dev;</div><div class='add'>+struct bnxt_re_ring_attr {</div><div class='add'>+	dma_addr_t	*dma_arr;</div><div class='add'>+	int		pages;</div><div class='add'>+	int		type;</div><div class='add'>+	u32		depth;</div><div class='add'>+	u32		lrid; /* Logical ring id */</div><div class='add'>+	u8		mode;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+/*</div><div class='add'>+ * Data structure and defines to handle</div><div class='add'>+ * recovery</div><div class='add'>+ */</div><div class='add'>+#define BNXT_RE_PRE_RECOVERY_REMOVE 0x1</div><div class='add'>+#define BNXT_RE_COMPLETE_REMOVE 0x2</div><div class='add'>+#define BNXT_RE_POST_RECOVERY_INIT 0x4</div><div class='add'>+#define BNXT_RE_COMPLETE_INIT 0x8</div><div class='add'>+</div><div class='ctx'> struct bnxt_re_sqp_entries {</div><div class='ctx'> 	struct bnxt_qplib_sge sge;</div><div class='ctx'> 	u64 wrid;</div><div class='hunk'>@@ -104,74 +102,137 @@ struct bnxt_re_sqp_entries {</div><div class='ctx'> 	struct bnxt_re_qp *qp1_qp;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define BNXT_RE_MIN_MSIX		2</div><div class='del'>-#define BNXT_RE_MAX_MSIX		9</div><div class='add'>+#define BNXT_RE_MAX_GSI_SQP_ENTRIES	1024</div><div class='add'>+struct bnxt_re_gsi_context {</div><div class='add'>+	struct	bnxt_re_qp *gsi_qp;</div><div class='add'>+	struct	bnxt_re_qp *gsi_sqp;</div><div class='add'>+	struct	bnxt_re_ah *gsi_sah;</div><div class='add'>+	struct	bnxt_re_sqp_entries *sqp_tbl;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_re_en_dev_info {</div><div class='add'>+	struct bnxt_en_dev *en_dev;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> #define BNXT_RE_AEQ_IDX			0</div><div class='ctx'> #define BNXT_RE_NQ_IDX			1</div><div class='add'>+#define BNXT_RE_GEN_P5_MAX_VF		64</div><div class='add'>+</div><div class='add'>+struct bnxt_re_pacing {</div><div class='add'>+	u64 dbr_db_fifo_reg_off;</div><div class='add'>+	void *dbr_page;</div><div class='add'>+	u64 dbr_bar_addr;</div><div class='add'>+	u32 pacing_algo_th;</div><div class='add'>+	u32 do_pacing_save;</div><div class='add'>+	u32 dbq_pacing_time; /* ms */</div><div class='add'>+	u32 dbr_def_do_pacing;</div><div class='add'>+	bool dbr_pacing;</div><div class='add'>+	struct mutex dbq_lock; /* synchronize db pacing algo */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define BNXT_RE_MAX_DBR_DO_PACING 0xFFFF</div><div class='add'>+#define BNXT_RE_DBR_PACING_TIME 5 /* ms */</div><div class='add'>+#define BNXT_RE_PACING_ALGO_THRESHOLD 250 /* Entries in DB FIFO */</div><div class='add'>+#define BNXT_RE_PACING_ALARM_TH_MULTIPLE 2 /* Multiple of pacing algo threshold */</div><div class='add'>+/* Default do_pacing value when there is no congestion */</div><div class='add'>+#define BNXT_RE_DBR_DO_PACING_NO_CONGESTION 0x7F /* 1 in 512 probability */</div><div class='add'>+</div><div class='add'>+#define BNXT_RE_MAX_FIFO_DEPTH_P5       0x2c00</div><div class='add'>+#define BNXT_RE_MAX_FIFO_DEPTH_P7       0x8000</div><div class='add'>+</div><div class='add'>+#define BNXT_RE_MAX_FIFO_DEPTH(ctx)	\</div><div class='add'>+	(bnxt_qplib_is_chip_gen_p7((ctx)) ? \</div><div class='add'>+	 BNXT_RE_MAX_FIFO_DEPTH_P7 :\</div><div class='add'>+	 BNXT_RE_MAX_FIFO_DEPTH_P5)</div><div class='add'>+</div><div class='add'>+#define BNXT_RE_GRC_FIFO_REG_BASE 0x2000</div><div class='add'>+</div><div class='add'>+#define BNXT_RE_MIN_MSIX		2</div><div class='add'>+#define BNXT_RE_MAX_MSIX		BNXT_MAX_ROCE_MSIX</div><div class='add'>+struct bnxt_re_nq_record {</div><div class='add'>+	struct bnxt_msix_entry	msix_entries[BNXT_RE_MAX_MSIX];</div><div class='add'>+	struct bnxt_qplib_nq	nq[BNXT_RE_MAX_MSIX];</div><div class='add'>+	int			num_msix;</div><div class='add'>+	/* serialize NQ access */</div><div class='add'>+	struct mutex		load_lock;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define MAX_CQ_HASH_BITS		(16)</div><div class='add'>+#define MAX_SRQ_HASH_BITS		(16)</div><div class='add'>+</div><div class='add'>+static inline bool bnxt_re_chip_gen_p7(u16 chip_num)</div><div class='add'>+{</div><div class='add'>+	return (chip_num == CHIP_NUM_58818 ||</div><div class='add'>+		chip_num == CHIP_NUM_57608);</div><div class='add'>+}</div><div class='ctx'> </div><div class='ctx'> struct bnxt_re_dev {</div><div class='ctx'> 	struct ib_device		ibdev;</div><div class='ctx'> 	struct list_head		list;</div><div class='ctx'> 	unsigned long			flags;</div><div class='ctx'> #define BNXT_RE_FLAG_NETDEV_REGISTERED		0</div><div class='del'>-#define BNXT_RE_FLAG_IBDEV_REGISTERED		1</div><div class='del'>-#define BNXT_RE_FLAG_GOT_MSIX			2</div><div class='add'>+#define BNXT_RE_FLAG_STATS_CTX3_ALLOC		1</div><div class='ctx'> #define BNXT_RE_FLAG_HAVE_L2_REF		3</div><div class='ctx'> #define BNXT_RE_FLAG_RCFW_CHANNEL_EN		4</div><div class='del'>-#define BNXT_RE_FLAG_QOS_WORK_REG		5</div><div class='ctx'> #define BNXT_RE_FLAG_RESOURCES_ALLOCATED	7</div><div class='ctx'> #define BNXT_RE_FLAG_RESOURCES_INITIALIZED	8</div><div class='add'>+#define BNXT_RE_FLAG_ERR_DEVICE_DETACHED       17</div><div class='ctx'> #define BNXT_RE_FLAG_ISSUE_ROCE_STATS          29</div><div class='ctx'> 	struct net_device		*netdev;</div><div class='add'>+	struct auxiliary_device         *adev;</div><div class='ctx'> 	unsigned int			version, major, minor;</div><div class='del'>-	struct bnxt_qplib_chip_ctx	chip_ctx;</div><div class='add'>+	struct bnxt_qplib_chip_ctx	*chip_ctx;</div><div class='ctx'> 	struct bnxt_en_dev		*en_dev;</div><div class='del'>-	struct bnxt_msix_entry		msix_entries[BNXT_RE_MAX_MSIX];</div><div class='del'>-	int				num_msix;</div><div class='ctx'> </div><div class='ctx'> 	int				id;</div><div class='ctx'> </div><div class='del'>-	struct delayed_work		worker;</div><div class='del'>-	u8				cur_prio_map;</div><div class='del'>-	u8				active_speed;</div><div class='del'>-	u8				active_width;</div><div class='del'>-</div><div class='del'>-	/* FP Notification Queue (CQ &amp; SRQ) */</div><div class='del'>-	struct tasklet_struct		nq_task;</div><div class='del'>-</div><div class='ctx'> 	/* RCFW Channel */</div><div class='ctx'> 	struct bnxt_qplib_rcfw		rcfw;</div><div class='ctx'> </div><div class='del'>-	/* NQ */</div><div class='del'>-	struct bnxt_qplib_nq		nq[BNXT_RE_MAX_MSIX];</div><div class='add'>+	/* NQ record */</div><div class='add'>+	struct bnxt_re_nq_record	*nqr;</div><div class='ctx'> </div><div class='ctx'> 	/* Device Resources */</div><div class='del'>-	struct bnxt_qplib_dev_attr	dev_attr;</div><div class='add'>+	struct bnxt_qplib_dev_attr	*dev_attr;</div><div class='ctx'> 	struct bnxt_qplib_ctx		qplib_ctx;</div><div class='ctx'> 	struct bnxt_qplib_res		qplib_res;</div><div class='ctx'> 	struct bnxt_qplib_dpi		dpi_privileged;</div><div class='add'>+	struct bnxt_qplib_cq_coal_param	cq_coalescing;</div><div class='ctx'> </div><div class='del'>-	atomic_t			qp_count;</div><div class='ctx'> 	struct mutex			qp_lock;	/* protect qp list */</div><div class='ctx'> 	struct list_head		qp_list;</div><div class='ctx'> </div><div class='del'>-	atomic_t			cq_count;</div><div class='del'>-	atomic_t			srq_count;</div><div class='del'>-	atomic_t			mr_count;</div><div class='del'>-	atomic_t			mw_count;</div><div class='del'>-	atomic_t			sched_count;</div><div class='ctx'> 	/* Max of 2 lossless traffic class supported per port */</div><div class='ctx'> 	u16				cosq[2];</div><div class='ctx'> </div><div class='del'>-	/* QP for for handling QP1 packets */</div><div class='del'>-	u32				sqp_id;</div><div class='del'>-	struct bnxt_re_qp		*qp1_sqp;</div><div class='del'>-	struct bnxt_re_ah		*sqp_ah;</div><div class='del'>-	struct bnxt_re_sqp_entries sqp_tbl[1024];</div><div class='add'>+	/* QP for handling QP1 packets */</div><div class='add'>+	struct bnxt_re_gsi_context	gsi_ctx;</div><div class='add'>+	struct bnxt_re_stats		stats;</div><div class='ctx'> 	atomic_t nq_alloc_cnt;</div><div class='ctx'> 	u32 is_virtfn;</div><div class='ctx'> 	u32 num_vfs;</div><div class='del'>-	struct bnxt_qplib_roce_stats	stats;</div><div class='add'>+	struct bnxt_re_pacing pacing;</div><div class='add'>+	struct work_struct dbq_fifo_check_work;</div><div class='add'>+	struct delayed_work dbq_pacing_work;</div><div class='add'>+	DECLARE_HASHTABLE(cq_hash, MAX_CQ_HASH_BITS);</div><div class='add'>+	DECLARE_HASHTABLE(srq_hash, MAX_SRQ_HASH_BITS);</div><div class='add'>+	struct dentry			*dbg_root;</div><div class='add'>+	struct dentry			*qp_debugfs;</div><div class='add'>+	unsigned long			event_bitmap;</div><div class='add'>+	struct bnxt_qplib_cc_param	cc_param;</div><div class='add'>+	struct workqueue_struct		*dcb_wq;</div><div class='add'>+	struct dentry                   *cc_config;</div><div class='add'>+	struct bnxt_re_dbg_cc_config_params *cc_config_params;</div><div class='add'>+	struct dentry			*cq_coal_cfg;</div><div class='add'>+	struct bnxt_re_dbg_cq_coal_params *cq_coal_cfg_params;</div><div class='add'>+#define BNXT_VPD_FLD_LEN		32</div><div class='add'>+	char board_partno[BNXT_VPD_FLD_LEN];</div><div class='add'>+	/* RoCE mirror */</div><div class='add'>+	u16 mirror_vnic_id;</div><div class='add'>+	union ib_gid ugid;</div><div class='add'>+	u32 ugid_index;</div><div class='add'>+	u8 sniffer_flow_created : 1;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> #define to_bnxt_re_dev(ptr, member)	\</div><div class='hunk'>@@ -181,6 +242,17 @@ struct bnxt_re_dev {</div><div class='ctx'> #define BNXT_RE_ROCEV2_IPV4_PACKET	2</div><div class='ctx'> #define BNXT_RE_ROCEV2_IPV6_PACKET	3</div><div class='ctx'> </div><div class='add'>+#define BNXT_RE_CHECK_RC(x) ((x) &amp;&amp; ((x) != -ETIMEDOUT))</div><div class='add'>+void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev);</div><div class='add'>+</div><div class='add'>+int bnxt_re_assign_pma_port_counters(struct bnxt_re_dev *rdev, struct ib_mad *out_mad);</div><div class='add'>+int bnxt_re_assign_pma_port_ext_counters(struct bnxt_re_dev *rdev,</div><div class='add'>+					 struct ib_mad *out_mad);</div><div class='add'>+</div><div class='add'>+void bnxt_re_hwrm_free_vnic(struct bnxt_re_dev *rdev);</div><div class='add'>+int bnxt_re_hwrm_alloc_vnic(struct bnxt_re_dev *rdev);</div><div class='add'>+int bnxt_re_hwrm_cfg_vnic(struct bnxt_re_dev *rdev, u32 qp_id);</div><div class='add'>+</div><div class='ctx'> static inline struct device *rdev_to_dev(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='ctx'> 	if (rdev)</div><div class='hunk'>@@ -188,4 +260,33 @@ static inline struct device *rdev_to_dev(struct bnxt_re_dev *rdev)</div><div class='ctx'> 	return NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+extern const struct uapi_definition bnxt_re_uapi_defs[];</div><div class='add'>+</div><div class='add'>+static inline void bnxt_re_set_pacing_dev_state(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	rdev-&gt;qplib_res.pacing_data-&gt;dev_err_state =</div><div class='add'>+		test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &amp;rdev-&gt;flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int bnxt_re_read_context_allowed(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	if (!bnxt_qplib_is_chip_gen_p5_p7(rdev-&gt;chip_ctx) ||</div><div class='add'>+	    rdev-&gt;rcfw.res-&gt;cctx-&gt;hwrm_intf_ver &lt; HWRM_VERSION_READ_CTX)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define BNXT_RE_CONTEXT_TYPE_QPC_SIZE_P5	1088</div><div class='add'>+#define BNXT_RE_CONTEXT_TYPE_CQ_SIZE_P5		128</div><div class='add'>+#define BNXT_RE_CONTEXT_TYPE_MRW_SIZE_P5	128</div><div class='add'>+#define BNXT_RE_CONTEXT_TYPE_SRQ_SIZE_P5	192</div><div class='add'>+</div><div class='add'>+#define BNXT_RE_CONTEXT_TYPE_QPC_SIZE_P7	1088</div><div class='add'>+#define BNXT_RE_CONTEXT_TYPE_CQ_SIZE_P7		192</div><div class='add'>+#define BNXT_RE_CONTEXT_TYPE_MRW_SIZE_P7	192</div><div class='add'>+#define BNXT_RE_CONTEXT_TYPE_SRQ_SIZE_P7	192</div><div class='add'>+</div><div class='add'>+#define BNXT_RE_HWRM_CMD_TIMEOUT(rdev)		\</div><div class='add'>+		((rdev)-&gt;chip_ctx-&gt;hwrm_cmd_max_timeout * 1000)</div><div class='add'>+</div><div class='ctx'> #endif</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/debugfs.c b/drivers/infiniband/hw/bnxt_re/debugfs.c<br/>new file mode 100644<br/>index 000000000000..88817c86ae24<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/debugfs.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/debugfs.c</a></div><div class='hunk'>@@ -0,0 +1,524 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2024, Broadcom. All rights reserved.  The term</div><div class='add'>+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.</div><div class='add'>+ *</div><div class='add'>+ * Description: Debugfs component of the bnxt_re driver</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/debugfs.h&gt;</div><div class='add'>+#include &lt;linux/pci.h&gt;</div><div class='add'>+#include &lt;linux/seq_file.h&gt;</div><div class='add'>+#include &lt;rdma/ib_addr.h&gt;</div><div class='add'>+</div><div class='add'>+#include "bnxt_ulp.h"</div><div class='add'>+#include "roce_hsi.h"</div><div class='add'>+#include "qplib_res.h"</div><div class='add'>+#include "qplib_sp.h"</div><div class='add'>+#include "qplib_fp.h"</div><div class='add'>+#include "qplib_rcfw.h"</div><div class='add'>+#include "bnxt_re.h"</div><div class='add'>+#include "ib_verbs.h"</div><div class='add'>+#include "debugfs.h"</div><div class='add'>+</div><div class='add'>+static struct dentry *bnxt_re_debugfs_root;</div><div class='add'>+</div><div class='add'>+static const char * const bnxt_re_cq_coal_str[] = {</div><div class='add'>+	"buf_maxtime",</div><div class='add'>+	"normal_maxbuf",</div><div class='add'>+	"during_maxbuf",</div><div class='add'>+	"en_ring_idle_mode",</div><div class='add'>+	"enable",</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const char * const bnxt_re_cc_gen0_name[] = {</div><div class='add'>+	"enable_cc",</div><div class='add'>+	"run_avg_weight_g",</div><div class='add'>+	"num_phase_per_state",</div><div class='add'>+	"init_cr",</div><div class='add'>+	"init_tr",</div><div class='add'>+	"tos_ecn",</div><div class='add'>+	"tos_dscp",</div><div class='add'>+	"alt_vlan_pcp",</div><div class='add'>+	"alt_vlan_dscp",</div><div class='add'>+	"rtt",</div><div class='add'>+	"cc_mode",</div><div class='add'>+	"tcp_cp",</div><div class='add'>+	"tx_queue",</div><div class='add'>+	"inactivity_cp",</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline const char *bnxt_re_qp_state_str(u8 state)</div><div class='add'>+{</div><div class='add'>+	switch (state) {</div><div class='add'>+	case CMDQ_MODIFY_QP_NEW_STATE_RESET:</div><div class='add'>+		return "RST";</div><div class='add'>+	case CMDQ_MODIFY_QP_NEW_STATE_INIT:</div><div class='add'>+		return "INIT";</div><div class='add'>+	case CMDQ_MODIFY_QP_NEW_STATE_RTR:</div><div class='add'>+		return "RTR";</div><div class='add'>+	case CMDQ_MODIFY_QP_NEW_STATE_RTS:</div><div class='add'>+		return "RTS";</div><div class='add'>+	case CMDQ_MODIFY_QP_NEW_STATE_SQE:</div><div class='add'>+		return "SQER";</div><div class='add'>+	case CMDQ_MODIFY_QP_NEW_STATE_SQD:</div><div class='add'>+		return "SQD";</div><div class='add'>+	case CMDQ_MODIFY_QP_NEW_STATE_ERR:</div><div class='add'>+		return "ERR";</div><div class='add'>+	default:</div><div class='add'>+		return "Invalid QP state";</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline const char *bnxt_re_qp_type_str(u8 type)</div><div class='add'>+{</div><div class='add'>+	switch (type) {</div><div class='add'>+	case CMDQ_CREATE_QP1_TYPE_GSI: return "QP1";</div><div class='add'>+	case CMDQ_CREATE_QP_TYPE_GSI: return "QP1";</div><div class='add'>+	case CMDQ_CREATE_QP_TYPE_RC: return "RC";</div><div class='add'>+	case CMDQ_CREATE_QP_TYPE_UD: return "UD";</div><div class='add'>+	case CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE: return "RAW_ETHERTYPE";</div><div class='add'>+	default: return "Invalid transport type";</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static ssize_t qp_info_read(struct file *filep,</div><div class='add'>+			    char __user *buffer,</div><div class='add'>+			    size_t count, loff_t *ppos)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_qp *qp = filep-&gt;private_data;</div><div class='add'>+	char *buf;</div><div class='add'>+	int len;</div><div class='add'>+</div><div class='add'>+	if (*ppos)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	buf = kasprintf(GFP_KERNEL,</div><div class='add'>+			"QPN\t\t: %d\n"</div><div class='add'>+			"transport\t: %s\n"</div><div class='add'>+			"state\t\t: %s\n"</div><div class='add'>+			"mtu\t\t: %d\n"</div><div class='add'>+			"timeout\t\t: %d\n"</div><div class='add'>+			"remote QPN\t: %d\n",</div><div class='add'>+			qp-&gt;qplib_qp.id,</div><div class='add'>+			bnxt_re_qp_type_str(qp-&gt;qplib_qp.type),</div><div class='add'>+			bnxt_re_qp_state_str(qp-&gt;qplib_qp.state),</div><div class='add'>+			qp-&gt;qplib_qp.mtu,</div><div class='add'>+			qp-&gt;qplib_qp.timeout,</div><div class='add'>+			qp-&gt;qplib_qp.dest_qpn);</div><div class='add'>+	if (!buf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	if (count &lt; strlen(buf)) {</div><div class='add'>+		kfree(buf);</div><div class='add'>+		return -ENOSPC;</div><div class='add'>+	}</div><div class='add'>+	len = simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf));</div><div class='add'>+	kfree(buf);</div><div class='add'>+	return len;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct file_operations debugfs_qp_fops = {</div><div class='add'>+	.owner = THIS_MODULE,</div><div class='add'>+	.open = simple_open,</div><div class='add'>+	.read = qp_info_read,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+void bnxt_re_debug_add_qpinfo(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp)</div><div class='add'>+{</div><div class='add'>+	char resn[32];</div><div class='add'>+</div><div class='add'>+	sprintf(resn, "0x%x", qp-&gt;qplib_qp.id);</div><div class='add'>+	qp-&gt;dentry = debugfs_create_file(resn, 0400, rdev-&gt;qp_debugfs, qp, &amp;debugfs_qp_fops);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bnxt_re_debug_rem_qpinfo(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp)</div><div class='add'>+{</div><div class='add'>+	debugfs_remove(qp-&gt;dentry);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int map_cc_config_offset_gen0_ext0(u32 offset, struct bnxt_qplib_cc_param *ccparam, u32 *val)</div><div class='add'>+{</div><div class='add'>+	u64 map_offset;</div><div class='add'>+</div><div class='add'>+	map_offset = BIT(offset);</div><div class='add'>+</div><div class='add'>+	switch (map_offset) {</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC:</div><div class='add'>+		*val = ccparam-&gt;enable;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_G:</div><div class='add'>+		*val = ccparam-&gt;g;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_NUMPHASEPERSTATE:</div><div class='add'>+		*val = ccparam-&gt;nph_per_state;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INIT_CR:</div><div class='add'>+		*val = ccparam-&gt;init_cr;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INIT_TR:</div><div class='add'>+		*val = ccparam-&gt;init_tr;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN:</div><div class='add'>+		*val = ccparam-&gt;tos_ecn;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP:</div><div class='add'>+		*val =  ccparam-&gt;tos_dscp;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP:</div><div class='add'>+		*val = ccparam-&gt;alt_vlan_pcp;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP:</div><div class='add'>+		*val = ccparam-&gt;alt_tos_dscp;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_RTT:</div><div class='add'>+	       *val = ccparam-&gt;rtt;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE:</div><div class='add'>+		*val = ccparam-&gt;cc_mode;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TCP_CP:</div><div class='add'>+		*val =  ccparam-&gt;tcp_cp;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INACTIVITY_CP:</div><div class='add'>+		*val = ccparam-&gt;inact_th;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static ssize_t bnxt_re_cc_config_get(struct file *filp, char __user *buffer,</div><div class='add'>+				     size_t usr_buf_len, loff_t *ppos)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_cc_param *dbg_cc_param = filp-&gt;private_data;</div><div class='add'>+	struct bnxt_re_dev *rdev = dbg_cc_param-&gt;rdev;</div><div class='add'>+	struct bnxt_qplib_cc_param ccparam = {};</div><div class='add'>+	u32 offset = dbg_cc_param-&gt;offset;</div><div class='add'>+	char buf[16];</div><div class='add'>+	u32 val;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rc = bnxt_qplib_query_cc_param(&amp;rdev-&gt;qplib_res, &amp;ccparam);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	rc = map_cc_config_offset_gen0_ext0(offset, &amp;ccparam, &amp;val);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	rc = snprintf(buf, sizeof(buf), "%d\n", val);</div><div class='add'>+	if (rc &lt; 0)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	return simple_read_from_buffer(buffer, usr_buf_len, ppos, (u8 *)(buf), rc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bnxt_re_fill_gen0_ext0(struct bnxt_qplib_cc_param *ccparam, u32 offset, u32 val)</div><div class='add'>+{</div><div class='add'>+	u32 modify_mask;</div><div class='add'>+</div><div class='add'>+	modify_mask = BIT(offset);</div><div class='add'>+</div><div class='add'>+	switch (modify_mask) {</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC:</div><div class='add'>+		ccparam-&gt;enable = val;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_G:</div><div class='add'>+		ccparam-&gt;g = val;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_NUMPHASEPERSTATE:</div><div class='add'>+		ccparam-&gt;nph_per_state = val;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INIT_CR:</div><div class='add'>+		ccparam-&gt;init_cr = val;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INIT_TR:</div><div class='add'>+		ccparam-&gt;init_tr = val;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN:</div><div class='add'>+		ccparam-&gt;tos_ecn = val;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP:</div><div class='add'>+		ccparam-&gt;tos_dscp = val;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP:</div><div class='add'>+		ccparam-&gt;alt_vlan_pcp = val;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP:</div><div class='add'>+		ccparam-&gt;alt_tos_dscp = val;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_RTT:</div><div class='add'>+		ccparam-&gt;rtt = val;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE:</div><div class='add'>+		ccparam-&gt;cc_mode = val;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TCP_CP:</div><div class='add'>+		ccparam-&gt;tcp_cp = val;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TX_QUEUE:</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INACTIVITY_CP:</div><div class='add'>+		ccparam-&gt;inact_th = val;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TIME_PER_PHASE:</div><div class='add'>+		ccparam-&gt;time_pph = val;</div><div class='add'>+		break;</div><div class='add'>+	case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_PKTS_PER_PHASE:</div><div class='add'>+		ccparam-&gt;pkts_pph = val;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ccparam-&gt;mask = modify_mask;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bnxt_re_configure_cc(struct bnxt_re_dev *rdev, u32 gen_ext, u32 offset, u32 val)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_cc_param ccparam = { };</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (gen_ext != CC_CONFIG_GEN0_EXT0)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	rc = bnxt_re_fill_gen0_ext0(&amp;ccparam, offset, val);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_modify_cc(&amp;rdev-&gt;qplib_res, &amp;ccparam);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static ssize_t bnxt_re_cc_config_set(struct file *filp, const char __user *buffer,</div><div class='add'>+				     size_t count, loff_t *ppos)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_cc_param *dbg_cc_param = filp-&gt;private_data;</div><div class='add'>+	struct bnxt_re_dev *rdev = dbg_cc_param-&gt;rdev;</div><div class='add'>+	u32 offset = dbg_cc_param-&gt;offset;</div><div class='add'>+	u8 cc_gen = dbg_cc_param-&gt;cc_gen;</div><div class='add'>+	char buf[16];</div><div class='add'>+	u32 val;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (count &gt;= sizeof(buf))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (copy_from_user(buf, buffer, count))</div><div class='add'>+		return -EFAULT;</div><div class='add'>+</div><div class='add'>+	buf[count] = '\0';</div><div class='add'>+	if (kstrtou32(buf, 0, &amp;val))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	rc = bnxt_re_configure_cc(rdev, cc_gen, offset, val);</div><div class='add'>+	return rc ? rc : count;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct file_operations bnxt_re_cc_config_ops = {</div><div class='add'>+	.owner = THIS_MODULE,</div><div class='add'>+	.open = simple_open,</div><div class='add'>+	.read = bnxt_re_cc_config_get,</div><div class='add'>+	.write = bnxt_re_cc_config_set,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int info_show(struct seq_file *m, void *unused)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_dev *rdev = m-&gt;private;</div><div class='add'>+	struct bnxt_re_res_cntrs *res_s = &amp;rdev-&gt;stats.res;</div><div class='add'>+</div><div class='add'>+	seq_puts(m, "Info:\n");</div><div class='add'>+	seq_printf(m, "Device Name\t\t: %s\n", dev_name(&amp;rdev-&gt;ibdev.dev));</div><div class='add'>+	seq_printf(m, "PD Watermark\t\t: %llu\n", res_s-&gt;pd_watermark);</div><div class='add'>+	seq_printf(m, "AH Watermark\t\t: %llu\n", res_s-&gt;ah_watermark);</div><div class='add'>+	seq_printf(m, "QP Watermark\t\t: %llu\n", res_s-&gt;qp_watermark);</div><div class='add'>+	seq_printf(m, "RC QP Watermark\t\t: %llu\n", res_s-&gt;rc_qp_watermark);</div><div class='add'>+	seq_printf(m, "UD QP Watermark\t\t: %llu\n", res_s-&gt;ud_qp_watermark);</div><div class='add'>+	seq_printf(m, "SRQ Watermark\t\t: %llu\n", res_s-&gt;srq_watermark);</div><div class='add'>+	seq_printf(m, "CQ Watermark\t\t: %llu\n", res_s-&gt;cq_watermark);</div><div class='add'>+	seq_printf(m, "MR Watermark\t\t: %llu\n", res_s-&gt;mr_watermark);</div><div class='add'>+	seq_printf(m, "MW Watermark\t\t: %llu\n", res_s-&gt;mw_watermark);</div><div class='add'>+	seq_printf(m, "CQ Resize Count\t\t: %d\n", atomic_read(&amp;res_s-&gt;resize_count));</div><div class='add'>+	if (rdev-&gt;pacing.dbr_pacing) {</div><div class='add'>+		seq_printf(m, "DB Pacing Reschedule\t: %llu\n", rdev-&gt;stats.pacing.resched);</div><div class='add'>+		seq_printf(m, "DB Pacing Complete\t: %llu\n", rdev-&gt;stats.pacing.complete);</div><div class='add'>+		seq_printf(m, "DB Pacing Alerts\t: %llu\n", rdev-&gt;stats.pacing.alerts);</div><div class='add'>+		seq_printf(m, "DB FIFO Register\t: 0x%x\n",</div><div class='add'>+			   readl(rdev-&gt;en_dev-&gt;bar0 + rdev-&gt;pacing.dbr_db_fifo_reg_off));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+DEFINE_SHOW_ATTRIBUTE(info);</div><div class='add'>+</div><div class='add'>+static void bnxt_re_debugfs_add_info(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	debugfs_create_file("info", 0400, rdev-&gt;dbg_root, rdev, &amp;info_fops);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static ssize_t cq_coal_cfg_write(struct file *file,</div><div class='add'>+				 const char __user *buf,</div><div class='add'>+				 size_t count, loff_t *pos)</div><div class='add'>+{</div><div class='add'>+	struct seq_file *s = file-&gt;private_data;</div><div class='add'>+	struct bnxt_re_cq_coal_param *param = s-&gt;private;</div><div class='add'>+	struct bnxt_re_dev *rdev = param-&gt;rdev;</div><div class='add'>+	int offset = param-&gt;offset;</div><div class='add'>+	char lbuf[16] = { };</div><div class='add'>+	u32 val;</div><div class='add'>+</div><div class='add'>+	if (count &gt; sizeof(lbuf))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (copy_from_user(lbuf, buf, count))</div><div class='add'>+		return -EFAULT;</div><div class='add'>+</div><div class='add'>+	lbuf[sizeof(lbuf) - 1] = '\0';</div><div class='add'>+</div><div class='add'>+	if (kstrtou32(lbuf, 0, &amp;val))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	switch (offset) {</div><div class='add'>+	case BNXT_RE_COAL_CQ_BUF_MAXTIME:</div><div class='add'>+		if (val &lt; 1 || val &gt; BNXT_QPLIB_CQ_COAL_MAX_BUF_MAXTIME)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		rdev-&gt;cq_coalescing.buf_maxtime = val;</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_RE_COAL_CQ_NORMAL_MAXBUF:</div><div class='add'>+		if (val &lt; 1 || val &gt; BNXT_QPLIB_CQ_COAL_MAX_NORMAL_MAXBUF)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		rdev-&gt;cq_coalescing.normal_maxbuf = val;</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_RE_COAL_CQ_DURING_MAXBUF:</div><div class='add'>+		if (val &lt; 1 || val &gt; BNXT_QPLIB_CQ_COAL_MAX_DURING_MAXBUF)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		rdev-&gt;cq_coalescing.during_maxbuf = val;</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_RE_COAL_CQ_EN_RING_IDLE_MODE:</div><div class='add'>+		if (val &gt; BNXT_QPLIB_CQ_COAL_MAX_EN_RING_IDLE_MODE)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		rdev-&gt;cq_coalescing.en_ring_idle_mode = val;</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_RE_COAL_CQ_ENABLE:</div><div class='add'>+		if (val &gt; 1)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		rdev-&gt;cq_coalescing.enable = val;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+	return  count;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int cq_coal_cfg_show(struct seq_file *s, void *unused)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_cq_coal_param *param = s-&gt;private;</div><div class='add'>+	struct bnxt_re_dev *rdev = param-&gt;rdev;</div><div class='add'>+	int offset = param-&gt;offset;</div><div class='add'>+	u32 val = 0;</div><div class='add'>+</div><div class='add'>+	switch (offset) {</div><div class='add'>+	case BNXT_RE_COAL_CQ_BUF_MAXTIME:</div><div class='add'>+		val = rdev-&gt;cq_coalescing.buf_maxtime;</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_RE_COAL_CQ_NORMAL_MAXBUF:</div><div class='add'>+		val = rdev-&gt;cq_coalescing.normal_maxbuf;</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_RE_COAL_CQ_DURING_MAXBUF:</div><div class='add'>+		val = rdev-&gt;cq_coalescing.during_maxbuf;</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_RE_COAL_CQ_EN_RING_IDLE_MODE:</div><div class='add'>+		val = rdev-&gt;cq_coalescing.en_ring_idle_mode;</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_RE_COAL_CQ_ENABLE:</div><div class='add'>+		val = rdev-&gt;cq_coalescing.enable;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	seq_printf(s, "%u\n", val);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+DEFINE_SHOW_STORE_ATTRIBUTE(cq_coal_cfg);</div><div class='add'>+</div><div class='add'>+static void bnxt_re_cleanup_cq_coal_debugfs(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	debugfs_remove_recursive(rdev-&gt;cq_coal_cfg);</div><div class='add'>+	kfree(rdev-&gt;cq_coal_cfg_params);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_init_cq_coal_debugfs(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_dbg_cq_coal_params *dbg_cq_coal_params;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (!_is_cq_coalescing_supported(rdev-&gt;dev_attr-&gt;dev_cap_flags2))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	dbg_cq_coal_params = kzalloc(sizeof(*dbg_cq_coal_params), GFP_KERNEL);</div><div class='add'>+	if (!dbg_cq_coal_params)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	rdev-&gt;cq_coal_cfg = debugfs_create_dir("cq_coal_cfg", rdev-&gt;dbg_root);</div><div class='add'>+	rdev-&gt;cq_coal_cfg_params = dbg_cq_coal_params;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; BNXT_RE_COAL_CQ_MAX; i++) {</div><div class='add'>+		dbg_cq_coal_params-&gt;params[i].offset = i;</div><div class='add'>+		dbg_cq_coal_params-&gt;params[i].rdev = rdev;</div><div class='add'>+		debugfs_create_file(bnxt_re_cq_coal_str[i],</div><div class='add'>+				    0600, rdev-&gt;cq_coal_cfg,</div><div class='add'>+				    &amp;dbg_cq_coal_params-&gt;params[i],</div><div class='add'>+				    &amp;cq_coal_cfg_fops);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bnxt_re_debugfs_add_pdev(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct pci_dev *pdev = rdev-&gt;en_dev-&gt;pdev;</div><div class='add'>+	struct bnxt_re_dbg_cc_config_params *cc_params;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	rdev-&gt;dbg_root = debugfs_create_dir(dev_name(&amp;pdev-&gt;dev), bnxt_re_debugfs_root);</div><div class='add'>+</div><div class='add'>+	rdev-&gt;qp_debugfs = debugfs_create_dir("QPs", rdev-&gt;dbg_root);</div><div class='add'>+	rdev-&gt;cc_config = debugfs_create_dir("cc_config", rdev-&gt;dbg_root);</div><div class='add'>+</div><div class='add'>+	bnxt_re_debugfs_add_info(rdev);</div><div class='add'>+</div><div class='add'>+	rdev-&gt;cc_config_params = kzalloc(sizeof(*cc_params), GFP_KERNEL);</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; BNXT_RE_CC_PARAM_GEN0; i++) {</div><div class='add'>+		struct bnxt_re_cc_param *tmp_params = &amp;rdev-&gt;cc_config_params-&gt;gen0_parms[i];</div><div class='add'>+</div><div class='add'>+		tmp_params-&gt;rdev = rdev;</div><div class='add'>+		tmp_params-&gt;offset = i;</div><div class='add'>+		tmp_params-&gt;cc_gen = CC_CONFIG_GEN0_EXT0;</div><div class='add'>+		tmp_params-&gt;dentry = debugfs_create_file(bnxt_re_cc_gen0_name[i], 0400,</div><div class='add'>+							 rdev-&gt;cc_config, tmp_params,</div><div class='add'>+							 &amp;bnxt_re_cc_config_ops);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	bnxt_re_init_cq_coal_debugfs(rdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bnxt_re_debugfs_rem_pdev(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	bnxt_re_cleanup_cq_coal_debugfs(rdev);</div><div class='add'>+	debugfs_remove_recursive(rdev-&gt;qp_debugfs);</div><div class='add'>+	debugfs_remove_recursive(rdev-&gt;cc_config);</div><div class='add'>+	kfree(rdev-&gt;cc_config_params);</div><div class='add'>+	debugfs_remove_recursive(rdev-&gt;dbg_root);</div><div class='add'>+	rdev-&gt;dbg_root = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bnxt_re_register_debugfs(void)</div><div class='add'>+{</div><div class='add'>+	bnxt_re_debugfs_root = debugfs_create_dir("bnxt_re", NULL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bnxt_re_unregister_debugfs(void)</div><div class='add'>+{</div><div class='add'>+	debugfs_remove(bnxt_re_debugfs_root);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/debugfs.h b/drivers/infiniband/hw/bnxt_re/debugfs.h<br/>new file mode 100644<br/>index 000000000000..98f4620ef245<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/debugfs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/debugfs.h</a></div><div class='hunk'>@@ -0,0 +1,55 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2024, Broadcom. All rights reserved.  The term</div><div class='add'>+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.</div><div class='add'>+ *</div><div class='add'>+ * Description: Debugfs header</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef __BNXT_RE_DEBUGFS__</div><div class='add'>+#define __BNXT_RE_DEBUGFS__</div><div class='add'>+</div><div class='add'>+void bnxt_re_debug_add_qpinfo(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp);</div><div class='add'>+void bnxt_re_debug_rem_qpinfo(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp);</div><div class='add'>+</div><div class='add'>+void bnxt_re_debugfs_add_pdev(struct bnxt_re_dev *rdev);</div><div class='add'>+void bnxt_re_debugfs_rem_pdev(struct bnxt_re_dev *rdev);</div><div class='add'>+</div><div class='add'>+void bnxt_re_register_debugfs(void);</div><div class='add'>+void bnxt_re_unregister_debugfs(void);</div><div class='add'>+</div><div class='add'>+#define CC_CONFIG_GEN_EXT(x, y)	(((x) &lt;&lt; 16) | (y))</div><div class='add'>+#define CC_CONFIG_GEN0_EXT0  CC_CONFIG_GEN_EXT(0, 0)</div><div class='add'>+</div><div class='add'>+#define BNXT_RE_CC_PARAM_GEN0  14</div><div class='add'>+</div><div class='add'>+struct bnxt_re_cc_param {</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct dentry *dentry;</div><div class='add'>+	u32 offset;</div><div class='add'>+	u8 cc_gen;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_re_dbg_cc_config_params {</div><div class='add'>+	struct bnxt_re_cc_param	gen0_parms[BNXT_RE_CC_PARAM_GEN0];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_re_cq_coal_param {</div><div class='add'>+	struct bnxt_re_dev	*rdev;</div><div class='add'>+	u32			offset;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum bnxt_re_cq_coal_types {</div><div class='add'>+	BNXT_RE_COAL_CQ_BUF_MAXTIME,</div><div class='add'>+	BNXT_RE_COAL_CQ_NORMAL_MAXBUF,</div><div class='add'>+	BNXT_RE_COAL_CQ_DURING_MAXBUF,</div><div class='add'>+	BNXT_RE_COAL_CQ_EN_RING_IDLE_MODE,</div><div class='add'>+	BNXT_RE_COAL_CQ_ENABLE,</div><div class='add'>+	BNXT_RE_COAL_CQ_MAX</div><div class='add'>+</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_re_dbg_cq_coal_params {</div><div class='add'>+	struct bnxt_re_cq_coal_param	params[BNXT_RE_COAL_CQ_MAX];</div><div class='add'>+};</div><div class='add'>+#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.c b/drivers/infiniband/hw/bnxt_re/hw_counters.c<br/>index 604b71875f5f..651cf9d0e0c7 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/hw_counters.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/hw_counters.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/hw_counters.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/hw_counters.c</a></div><div class='hunk'>@@ -37,18 +37,11 @@</div><div class='ctx'>  *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/interrupt.h&gt;</div><div class='ctx'> #include &lt;linux/types.h&gt;</div><div class='del'>-#include &lt;linux/spinlock.h&gt;</div><div class='del'>-#include &lt;linux/sched.h&gt;</div><div class='del'>-#include &lt;linux/slab.h&gt;</div><div class='ctx'> #include &lt;linux/pci.h&gt;</div><div class='del'>-#include &lt;linux/prefetch.h&gt;</div><div class='del'>-#include &lt;linux/delay.h&gt;</div><div class='add'>+#include &lt;rdma/ib_mad.h&gt;</div><div class='add'>+#include &lt;rdma/ib_pma.h&gt;</div><div class='ctx'> </div><div class='del'>-#include &lt;rdma/ib_addr.h&gt;</div><div class='del'>-</div><div class='del'>-#include "bnxt_ulp.h"</div><div class='ctx'> #include "roce_hsi.h"</div><div class='ctx'> #include "qplib_res.h"</div><div class='ctx'> #include "qplib_sp.h"</div><div class='hunk'>@@ -57,192 +50,384 @@</div><div class='ctx'> #include "bnxt_re.h"</div><div class='ctx'> #include "hw_counters.h"</div><div class='ctx'> </div><div class='del'>-static const char * const bnxt_re_stat_name[] = {</div><div class='del'>-	[BNXT_RE_ACTIVE_QP]		=  "active_qps",</div><div class='del'>-	[BNXT_RE_ACTIVE_SRQ]		=  "active_srqs",</div><div class='del'>-	[BNXT_RE_ACTIVE_CQ]		=  "active_cqs",</div><div class='del'>-	[BNXT_RE_ACTIVE_MR]		=  "active_mrs",</div><div class='del'>-	[BNXT_RE_ACTIVE_MW]		=  "active_mws",</div><div class='del'>-	[BNXT_RE_RX_PKTS]		=  "rx_pkts",</div><div class='del'>-	[BNXT_RE_RX_BYTES]		=  "rx_bytes",</div><div class='del'>-	[BNXT_RE_TX_PKTS]		=  "tx_pkts",</div><div class='del'>-	[BNXT_RE_TX_BYTES]		=  "tx_bytes",</div><div class='del'>-	[BNXT_RE_RECOVERABLE_ERRORS]	=  "recoverable_errors",</div><div class='del'>-	[BNXT_RE_RX_DROPS]		=  "rx_roce_drops",</div><div class='del'>-	[BNXT_RE_RX_DISCARDS]		=  "rx_roce_discards",</div><div class='del'>-	[BNXT_RE_TO_RETRANSMITS]        = "to_retransmits",</div><div class='del'>-	[BNXT_RE_SEQ_ERR_NAKS_RCVD]     = "seq_err_naks_rcvd",</div><div class='del'>-	[BNXT_RE_MAX_RETRY_EXCEEDED]    = "max_retry_exceeded",</div><div class='del'>-	[BNXT_RE_RNR_NAKS_RCVD]         = "rnr_naks_rcvd",</div><div class='del'>-	[BNXT_RE_MISSING_RESP]          = "missin_resp",</div><div class='del'>-	[BNXT_RE_UNRECOVERABLE_ERR]     = "unrecoverable_err",</div><div class='del'>-	[BNXT_RE_BAD_RESP_ERR]          = "bad_resp_err",</div><div class='del'>-	[BNXT_RE_LOCAL_QP_OP_ERR]       = "local_qp_op_err",</div><div class='del'>-	[BNXT_RE_LOCAL_PROTECTION_ERR]  = "local_protection_err",</div><div class='del'>-	[BNXT_RE_MEM_MGMT_OP_ERR]       = "mem_mgmt_op_err",</div><div class='del'>-	[BNXT_RE_REMOTE_INVALID_REQ_ERR] = "remote_invalid_req_err",</div><div class='del'>-	[BNXT_RE_REMOTE_ACCESS_ERR]     = "remote_access_err",</div><div class='del'>-	[BNXT_RE_REMOTE_OP_ERR]         = "remote_op_err",</div><div class='del'>-	[BNXT_RE_DUP_REQ]               = "dup_req",</div><div class='del'>-	[BNXT_RE_RES_EXCEED_MAX]        = "res_exceed_max",</div><div class='del'>-	[BNXT_RE_RES_LENGTH_MISMATCH]   = "res_length_mismatch",</div><div class='del'>-	[BNXT_RE_RES_EXCEEDS_WQE]       = "res_exceeds_wqe",</div><div class='del'>-	[BNXT_RE_RES_OPCODE_ERR]        = "res_opcode_err",</div><div class='del'>-	[BNXT_RE_RES_RX_INVALID_RKEY]   = "res_rx_invalid_rkey",</div><div class='del'>-	[BNXT_RE_RES_RX_DOMAIN_ERR]     = "res_rx_domain_err",</div><div class='del'>-	[BNXT_RE_RES_RX_NO_PERM]        = "res_rx_no_perm",</div><div class='del'>-	[BNXT_RE_RES_RX_RANGE_ERR]      = "res_rx_range_err",</div><div class='del'>-	[BNXT_RE_RES_TX_INVALID_RKEY]   = "res_tx_invalid_rkey",</div><div class='del'>-	[BNXT_RE_RES_TX_DOMAIN_ERR]     = "res_tx_domain_err",</div><div class='del'>-	[BNXT_RE_RES_TX_NO_PERM]        = "res_tx_no_perm",</div><div class='del'>-	[BNXT_RE_RES_TX_RANGE_ERR]      = "res_tx_range_err",</div><div class='del'>-	[BNXT_RE_RES_IRRQ_OFLOW]        = "res_irrq_oflow",</div><div class='del'>-	[BNXT_RE_RES_UNSUP_OPCODE]      = "res_unsup_opcode",</div><div class='del'>-	[BNXT_RE_RES_UNALIGNED_ATOMIC]  = "res_unaligned_atomic",</div><div class='del'>-	[BNXT_RE_RES_REM_INV_ERR]       = "res_rem_inv_err",</div><div class='del'>-	[BNXT_RE_RES_MEM_ERROR]         = "res_mem_err",</div><div class='del'>-	[BNXT_RE_RES_SRQ_ERR]           = "res_srq_err",</div><div class='del'>-	[BNXT_RE_RES_CMP_ERR]           = "res_cmp_err",</div><div class='del'>-	[BNXT_RE_RES_INVALID_DUP_RKEY]  = "res_invalid_dup_rkey",</div><div class='del'>-	[BNXT_RE_RES_WQE_FORMAT_ERR]    = "res_wqe_format_err",</div><div class='del'>-	[BNXT_RE_RES_CQ_LOAD_ERR]       = "res_cq_load_err",</div><div class='del'>-	[BNXT_RE_RES_SRQ_LOAD_ERR]      = "res_srq_load_err",</div><div class='del'>-	[BNXT_RE_RES_TX_PCI_ERR]        = "res_tx_pci_err",</div><div class='del'>-	[BNXT_RE_RES_RX_PCI_ERR]        = "res_rx_pci_err",</div><div class='del'>-	[BNXT_RE_OUT_OF_SEQ_ERR]        = "oos_drop_count"</div><div class='add'>+static const struct rdma_stat_desc bnxt_re_stat_descs[] = {</div><div class='add'>+	[BNXT_RE_RX_PKTS].name		=  "rx_pkts",</div><div class='add'>+	[BNXT_RE_RX_BYTES].name		=  "rx_bytes",</div><div class='add'>+	[BNXT_RE_TX_PKTS].name		=  "tx_pkts",</div><div class='add'>+	[BNXT_RE_TX_BYTES].name		=  "tx_bytes",</div><div class='add'>+	[BNXT_RE_RECOVERABLE_ERRORS].name	=  "recoverable_errors",</div><div class='add'>+	[BNXT_RE_TX_ERRORS].name                =  "tx_roce_errors",</div><div class='add'>+	[BNXT_RE_TX_DISCARDS].name              =  "tx_roce_discards",</div><div class='add'>+	[BNXT_RE_RX_ERRORS].name		=  "rx_roce_errors",</div><div class='add'>+	[BNXT_RE_RX_DISCARDS].name		=  "rx_roce_discards",</div><div class='add'>+	[BNXT_RE_TO_RETRANSMITS].name           =  "local_ack_timeout_err",</div><div class='add'>+	[BNXT_RE_SEQ_ERR_NAKS_RCVD].name        =  "packet_seq_err",</div><div class='add'>+	[BNXT_RE_MAX_RETRY_EXCEEDED].name	=  "max_retry_exceeded",</div><div class='add'>+	[BNXT_RE_RNR_NAKS_RCVD].name            =  "rnr_nak_retry_err",</div><div class='add'>+	[BNXT_RE_MISSING_RESP].name             =  "implied_nak_seq_err",</div><div class='add'>+	[BNXT_RE_UNRECOVERABLE_ERR].name     = "unrecoverable_err",</div><div class='add'>+	[BNXT_RE_BAD_RESP_ERR].name          = "bad_resp_err",</div><div class='add'>+	[BNXT_RE_LOCAL_QP_OP_ERR].name       = "local_qp_op_err",</div><div class='add'>+	[BNXT_RE_LOCAL_PROTECTION_ERR].name  = "local_protection_err",</div><div class='add'>+	[BNXT_RE_MEM_MGMT_OP_ERR].name       = "mem_mgmt_op_err",</div><div class='add'>+	[BNXT_RE_REMOTE_INVALID_REQ_ERR].name   = "req_remote_invalid_request",</div><div class='add'>+	[BNXT_RE_REMOTE_ACCESS_ERR].name        = "req_remote_access_errors",</div><div class='add'>+	[BNXT_RE_REMOTE_OP_ERR].name         = "remote_op_err",</div><div class='add'>+	[BNXT_RE_DUP_REQ].name               = "duplicate_request",</div><div class='add'>+	[BNXT_RE_RES_EXCEED_MAX].name        = "res_exceed_max",</div><div class='add'>+	[BNXT_RE_RES_LENGTH_MISMATCH].name   = "resp_local_length_error",</div><div class='add'>+	[BNXT_RE_RES_EXCEEDS_WQE].name       = "res_exceeds_wqe",</div><div class='add'>+	[BNXT_RE_RES_OPCODE_ERR].name        = "res_opcode_err",</div><div class='add'>+	[BNXT_RE_RES_RX_INVALID_RKEY].name   = "res_rx_invalid_rkey",</div><div class='add'>+	[BNXT_RE_RES_RX_DOMAIN_ERR].name     = "res_rx_domain_err",</div><div class='add'>+	[BNXT_RE_RES_RX_NO_PERM].name        = "res_rx_no_perm",</div><div class='add'>+	[BNXT_RE_RES_RX_RANGE_ERR].name      = "res_rx_range_err",</div><div class='add'>+	[BNXT_RE_RES_TX_INVALID_RKEY].name   = "res_tx_invalid_rkey",</div><div class='add'>+	[BNXT_RE_RES_TX_DOMAIN_ERR].name     = "res_tx_domain_err",</div><div class='add'>+	[BNXT_RE_RES_TX_NO_PERM].name        = "res_tx_no_perm",</div><div class='add'>+	[BNXT_RE_RES_TX_RANGE_ERR].name      = "res_tx_range_err",</div><div class='add'>+	[BNXT_RE_RES_IRRQ_OFLOW].name        = "res_irrq_oflow",</div><div class='add'>+	[BNXT_RE_RES_UNSUP_OPCODE].name      = "res_unsup_opcode",</div><div class='add'>+	[BNXT_RE_RES_UNALIGNED_ATOMIC].name  = "res_unaligned_atomic",</div><div class='add'>+	[BNXT_RE_RES_REM_INV_ERR].name       = "res_rem_inv_err",</div><div class='add'>+	[BNXT_RE_RES_MEM_ERROR].name         = "res_mem_err",</div><div class='add'>+	[BNXT_RE_RES_SRQ_ERR].name           = "res_srq_err",</div><div class='add'>+	[BNXT_RE_RES_CMP_ERR].name           = "res_cmp_err",</div><div class='add'>+	[BNXT_RE_RES_INVALID_DUP_RKEY].name  = "res_invalid_dup_rkey",</div><div class='add'>+	[BNXT_RE_RES_WQE_FORMAT_ERR].name    = "res_wqe_format_err",</div><div class='add'>+	[BNXT_RE_RES_CQ_LOAD_ERR].name       = "res_cq_load_err",</div><div class='add'>+	[BNXT_RE_RES_SRQ_LOAD_ERR].name      = "res_srq_load_err",</div><div class='add'>+	[BNXT_RE_RES_TX_PCI_ERR].name        = "res_tx_pci_err",</div><div class='add'>+	[BNXT_RE_RES_RX_PCI_ERR].name        = "res_rx_pci_err",</div><div class='add'>+	[BNXT_RE_OUT_OF_SEQ_ERR].name        = "out_of_sequence",</div><div class='add'>+	[BNXT_RE_TX_ATOMIC_REQ].name	     = "tx_atomic_req",</div><div class='add'>+	[BNXT_RE_TX_READ_REQ].name	     = "tx_read_req",</div><div class='add'>+	[BNXT_RE_TX_READ_RES].name	     = "tx_read_resp",</div><div class='add'>+	[BNXT_RE_TX_WRITE_REQ].name	     = "tx_write_req",</div><div class='add'>+	[BNXT_RE_TX_SEND_REQ].name	     = "tx_send_req",</div><div class='add'>+	[BNXT_RE_TX_ROCE_PKTS].name          = "tx_roce_only_pkts",</div><div class='add'>+	[BNXT_RE_TX_ROCE_BYTES].name         = "tx_roce_only_bytes",</div><div class='add'>+	[BNXT_RE_RX_ATOMIC_REQ].name	     = "rx_atomic_requests",</div><div class='add'>+	[BNXT_RE_RX_READ_REQ].name	     = "rx_read_requests",</div><div class='add'>+	[BNXT_RE_RX_READ_RESP].name	     = "rx_read_resp",</div><div class='add'>+	[BNXT_RE_RX_WRITE_REQ].name	     = "rx_write_requests",</div><div class='add'>+	[BNXT_RE_RX_SEND_REQ].name	     = "rx_send_req",</div><div class='add'>+	[BNXT_RE_RX_ROCE_PKTS].name          = "rx_roce_only_pkts",</div><div class='add'>+	[BNXT_RE_RX_ROCE_BYTES].name         = "rx_roce_only_bytes",</div><div class='add'>+	[BNXT_RE_RX_ROCE_GOOD_PKTS].name     = "rx_roce_good_pkts",</div><div class='add'>+	[BNXT_RE_RX_ROCE_GOOD_BYTES].name    = "rx_roce_good_bytes",</div><div class='add'>+	[BNXT_RE_OOB].name		     = "out_of_buffer",</div><div class='add'>+	[BNXT_RE_TX_CNP].name                = "np_cnp_pkts",</div><div class='add'>+	[BNXT_RE_RX_CNP].name                = "rp_cnp_handled",</div><div class='add'>+	[BNXT_RE_RX_ECN].name                = "np_ecn_marked_roce_packets",</div><div class='add'>+	[BNXT_RE_REQ_CQE_ERROR].name            = "req_cqe_error",</div><div class='add'>+	[BNXT_RE_RESP_CQE_ERROR].name           = "resp_cqe_error",</div><div class='add'>+	[BNXT_RE_RESP_REMOTE_ACCESS_ERRS].name  = "resp_remote_access_errors",</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static void bnxt_re_copy_ext_stats(struct bnxt_re_dev *rdev,</div><div class='add'>+				   struct rdma_hw_stats *stats,</div><div class='add'>+				   struct bnxt_qplib_ext_stat *s)</div><div class='add'>+{</div><div class='add'>+	stats-&gt;value[BNXT_RE_TX_ATOMIC_REQ] = s-&gt;tx_atomic_req;</div><div class='add'>+	stats-&gt;value[BNXT_RE_TX_READ_REQ]   = s-&gt;tx_read_req;</div><div class='add'>+	stats-&gt;value[BNXT_RE_TX_READ_RES]   = s-&gt;tx_read_res;</div><div class='add'>+	stats-&gt;value[BNXT_RE_TX_WRITE_REQ]  = s-&gt;tx_write_req;</div><div class='add'>+	stats-&gt;value[BNXT_RE_TX_SEND_REQ]   = s-&gt;tx_send_req;</div><div class='add'>+	stats-&gt;value[BNXT_RE_TX_ROCE_PKTS]  = s-&gt;tx_roce_pkts;</div><div class='add'>+	stats-&gt;value[BNXT_RE_TX_ROCE_BYTES] = s-&gt;tx_roce_bytes;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RX_ATOMIC_REQ] = s-&gt;rx_atomic_req;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RX_READ_REQ]   = s-&gt;rx_read_req;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RX_READ_RESP]  = s-&gt;rx_read_res;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RX_WRITE_REQ]  = s-&gt;rx_write_req;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RX_SEND_REQ]   = s-&gt;rx_send_req;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RX_ROCE_PKTS]  = s-&gt;rx_roce_pkts;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RX_ROCE_BYTES] = s-&gt;rx_roce_bytes;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RX_ROCE_GOOD_PKTS] = s-&gt;rx_roce_good_pkts;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RX_ROCE_GOOD_BYTES] = s-&gt;rx_roce_good_bytes;</div><div class='add'>+	stats-&gt;value[BNXT_RE_OOB] = s-&gt;rx_out_of_buffer;</div><div class='add'>+	stats-&gt;value[BNXT_RE_TX_CNP] = s-&gt;tx_cnp;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RX_CNP] = s-&gt;rx_cnp;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RX_ECN] = s-&gt;rx_ecn_marked;</div><div class='add'>+	stats-&gt;value[BNXT_RE_OUT_OF_SEQ_ERR] = s-&gt;rx_out_of_sequence;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bnxt_re_get_ext_stat(struct bnxt_re_dev *rdev,</div><div class='add'>+				struct rdma_hw_stats *stats)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_ext_stat *estat = &amp;rdev-&gt;stats.rstat.ext_stat;</div><div class='add'>+	u32 fid;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	fid = PCI_FUNC(rdev-&gt;en_dev-&gt;pdev-&gt;devfn);</div><div class='add'>+	rc = bnxt_qplib_qext_stat(&amp;rdev-&gt;rcfw, fid, estat);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto done;</div><div class='add'>+	bnxt_re_copy_ext_stats(rdev, stats, estat);</div><div class='add'>+</div><div class='add'>+done:</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_copy_err_stats(struct bnxt_re_dev *rdev,</div><div class='add'>+				   struct rdma_hw_stats *stats,</div><div class='add'>+				   struct bnxt_qplib_roce_stats *err_s)</div><div class='add'>+{</div><div class='add'>+	stats-&gt;value[BNXT_RE_TO_RETRANSMITS] =</div><div class='add'>+				err_s-&gt;to_retransmits;</div><div class='add'>+	stats-&gt;value[BNXT_RE_SEQ_ERR_NAKS_RCVD] =</div><div class='add'>+				err_s-&gt;seq_err_naks_rcvd;</div><div class='add'>+	stats-&gt;value[BNXT_RE_MAX_RETRY_EXCEEDED] =</div><div class='add'>+				err_s-&gt;max_retry_exceeded;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RNR_NAKS_RCVD] =</div><div class='add'>+				err_s-&gt;rnr_naks_rcvd;</div><div class='add'>+	stats-&gt;value[BNXT_RE_MISSING_RESP] =</div><div class='add'>+				err_s-&gt;missing_resp;</div><div class='add'>+	stats-&gt;value[BNXT_RE_UNRECOVERABLE_ERR] =</div><div class='add'>+				err_s-&gt;unrecoverable_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_BAD_RESP_ERR] =</div><div class='add'>+				err_s-&gt;bad_resp_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_LOCAL_QP_OP_ERR]	=</div><div class='add'>+			err_s-&gt;local_qp_op_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_LOCAL_PROTECTION_ERR] =</div><div class='add'>+			err_s-&gt;local_protection_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_MEM_MGMT_OP_ERR] =</div><div class='add'>+			err_s-&gt;mem_mgmt_op_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_REMOTE_INVALID_REQ_ERR] =</div><div class='add'>+			err_s-&gt;remote_invalid_req_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_REMOTE_ACCESS_ERR] =</div><div class='add'>+			err_s-&gt;remote_access_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_REMOTE_OP_ERR] =</div><div class='add'>+			err_s-&gt;remote_op_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_DUP_REQ] =</div><div class='add'>+			err_s-&gt;dup_req;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_EXCEED_MAX] =</div><div class='add'>+			err_s-&gt;res_exceed_max;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_LENGTH_MISMATCH] =</div><div class='add'>+			err_s-&gt;res_length_mismatch;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_EXCEEDS_WQE] =</div><div class='add'>+			err_s-&gt;res_exceeds_wqe;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_OPCODE_ERR] =</div><div class='add'>+			err_s-&gt;res_opcode_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_RX_INVALID_RKEY] =</div><div class='add'>+			err_s-&gt;res_rx_invalid_rkey;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_RX_DOMAIN_ERR] =</div><div class='add'>+			err_s-&gt;res_rx_domain_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_RX_NO_PERM] =</div><div class='add'>+			err_s-&gt;res_rx_no_perm;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_RX_RANGE_ERR]  =</div><div class='add'>+			err_s-&gt;res_rx_range_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_TX_INVALID_RKEY] =</div><div class='add'>+			err_s-&gt;res_tx_invalid_rkey;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_TX_DOMAIN_ERR] =</div><div class='add'>+			err_s-&gt;res_tx_domain_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_TX_NO_PERM] =</div><div class='add'>+			err_s-&gt;res_tx_no_perm;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_TX_RANGE_ERR]  =</div><div class='add'>+			err_s-&gt;res_tx_range_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_IRRQ_OFLOW] =</div><div class='add'>+			err_s-&gt;res_irrq_oflow;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_UNSUP_OPCODE]  =</div><div class='add'>+			err_s-&gt;res_unsup_opcode;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_UNALIGNED_ATOMIC] =</div><div class='add'>+			err_s-&gt;res_unaligned_atomic;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_REM_INV_ERR]   =</div><div class='add'>+			err_s-&gt;res_rem_inv_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_MEM_ERROR] =</div><div class='add'>+			err_s-&gt;res_mem_error;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_SRQ_ERR] =</div><div class='add'>+			err_s-&gt;res_srq_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_CMP_ERR] =</div><div class='add'>+			err_s-&gt;res_cmp_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_INVALID_DUP_RKEY] =</div><div class='add'>+			err_s-&gt;res_invalid_dup_rkey;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_WQE_FORMAT_ERR] =</div><div class='add'>+			err_s-&gt;res_wqe_format_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_CQ_LOAD_ERR]   =</div><div class='add'>+			err_s-&gt;res_cq_load_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_SRQ_LOAD_ERR]  =</div><div class='add'>+			err_s-&gt;res_srq_load_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_TX_PCI_ERR]    =</div><div class='add'>+			err_s-&gt;res_tx_pci_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RES_RX_PCI_ERR]    =</div><div class='add'>+			err_s-&gt;res_rx_pci_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_OUT_OF_SEQ_ERR]    =</div><div class='add'>+			err_s-&gt;res_oos_drop_count;</div><div class='add'>+	stats-&gt;value[BNXT_RE_REQ_CQE_ERROR]     =</div><div class='add'>+			err_s-&gt;bad_resp_err +</div><div class='add'>+			err_s-&gt;local_qp_op_err +</div><div class='add'>+			err_s-&gt;local_protection_err +</div><div class='add'>+			err_s-&gt;mem_mgmt_op_err +</div><div class='add'>+			err_s-&gt;remote_invalid_req_err +</div><div class='add'>+			err_s-&gt;remote_access_err +</div><div class='add'>+			err_s-&gt;remote_op_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RESP_CQE_ERROR] =</div><div class='add'>+			err_s-&gt;res_cmp_err +</div><div class='add'>+			err_s-&gt;res_cq_load_err;</div><div class='add'>+	stats-&gt;value[BNXT_RE_RESP_REMOTE_ACCESS_ERRS] =</div><div class='add'>+			err_s-&gt;res_rx_no_perm +</div><div class='add'>+			err_s-&gt;res_tx_no_perm;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bnxt_re_assign_pma_port_ext_counters(struct bnxt_re_dev *rdev, struct ib_mad *out_mad)</div><div class='add'>+{</div><div class='add'>+	struct ib_pma_portcounters_ext *pma_cnt_ext;</div><div class='add'>+	struct bnxt_qplib_ext_stat *estat = &amp;rdev-&gt;stats.rstat.ext_stat;</div><div class='add'>+	struct ctx_hw_stats *hw_stats = NULL;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	hw_stats = rdev-&gt;qplib_ctx.stats.dma;</div><div class='add'>+</div><div class='add'>+	pma_cnt_ext = (struct ib_pma_portcounters_ext *)(out_mad-&gt;data + 40);</div><div class='add'>+	if (_is_ext_stats_supported(rdev-&gt;dev_attr-&gt;dev_cap_flags)) {</div><div class='add'>+		u32 fid = PCI_FUNC(rdev-&gt;en_dev-&gt;pdev-&gt;devfn);</div><div class='add'>+</div><div class='add'>+		rc = bnxt_qplib_qext_stat(&amp;rdev-&gt;rcfw, fid, estat);</div><div class='add'>+		if (rc)</div><div class='add'>+			return rc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	pma_cnt_ext = (struct ib_pma_portcounters_ext *)(out_mad-&gt;data + 40);</div><div class='add'>+	if ((bnxt_qplib_is_chip_gen_p5(rdev-&gt;chip_ctx) &amp;&amp; rdev-&gt;is_virtfn) ||</div><div class='add'>+	    !bnxt_qplib_is_chip_gen_p5(rdev-&gt;chip_ctx)) {</div><div class='add'>+		pma_cnt_ext-&gt;port_xmit_data =</div><div class='add'>+			cpu_to_be64(le64_to_cpu(hw_stats-&gt;tx_ucast_bytes) / 4);</div><div class='add'>+		pma_cnt_ext-&gt;port_rcv_data =</div><div class='add'>+			cpu_to_be64(le64_to_cpu(hw_stats-&gt;rx_ucast_bytes) / 4);</div><div class='add'>+		pma_cnt_ext-&gt;port_xmit_packets =</div><div class='add'>+			cpu_to_be64(le64_to_cpu(hw_stats-&gt;tx_ucast_pkts));</div><div class='add'>+		pma_cnt_ext-&gt;port_rcv_packets =</div><div class='add'>+			cpu_to_be64(le64_to_cpu(hw_stats-&gt;rx_ucast_pkts));</div><div class='add'>+		pma_cnt_ext-&gt;port_unicast_rcv_packets =</div><div class='add'>+			cpu_to_be64(le64_to_cpu(hw_stats-&gt;rx_ucast_pkts));</div><div class='add'>+		pma_cnt_ext-&gt;port_unicast_xmit_packets =</div><div class='add'>+			cpu_to_be64(le64_to_cpu(hw_stats-&gt;tx_ucast_pkts));</div><div class='add'>+</div><div class='add'>+	} else {</div><div class='add'>+		pma_cnt_ext-&gt;port_rcv_packets = cpu_to_be64(estat-&gt;rx_roce_good_pkts);</div><div class='add'>+		pma_cnt_ext-&gt;port_rcv_data = cpu_to_be64(estat-&gt;rx_roce_good_bytes / 4);</div><div class='add'>+		pma_cnt_ext-&gt;port_xmit_packets = cpu_to_be64(estat-&gt;tx_roce_pkts);</div><div class='add'>+		pma_cnt_ext-&gt;port_xmit_data = cpu_to_be64(estat-&gt;tx_roce_bytes / 4);</div><div class='add'>+		pma_cnt_ext-&gt;port_unicast_rcv_packets = cpu_to_be64(estat-&gt;rx_roce_good_pkts);</div><div class='add'>+		pma_cnt_ext-&gt;port_unicast_xmit_packets = cpu_to_be64(estat-&gt;tx_roce_pkts);</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bnxt_re_assign_pma_port_counters(struct bnxt_re_dev *rdev, struct ib_mad *out_mad)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_ext_stat *estat = &amp;rdev-&gt;stats.rstat.ext_stat;</div><div class='add'>+	struct ib_pma_portcounters *pma_cnt;</div><div class='add'>+	struct ctx_hw_stats *hw_stats = NULL;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	hw_stats = rdev-&gt;qplib_ctx.stats.dma;</div><div class='add'>+</div><div class='add'>+	pma_cnt = (struct ib_pma_portcounters *)(out_mad-&gt;data + 40);</div><div class='add'>+	if (_is_ext_stats_supported(rdev-&gt;dev_attr-&gt;dev_cap_flags)) {</div><div class='add'>+		u32 fid = PCI_FUNC(rdev-&gt;en_dev-&gt;pdev-&gt;devfn);</div><div class='add'>+</div><div class='add'>+		rc = bnxt_qplib_qext_stat(&amp;rdev-&gt;rcfw, fid, estat);</div><div class='add'>+		if (rc)</div><div class='add'>+			return rc;</div><div class='add'>+	}</div><div class='add'>+	if ((bnxt_qplib_is_chip_gen_p5(rdev-&gt;chip_ctx) &amp;&amp; rdev-&gt;is_virtfn) ||</div><div class='add'>+	    !bnxt_qplib_is_chip_gen_p5(rdev-&gt;chip_ctx)) {</div><div class='add'>+		pma_cnt-&gt;port_rcv_packets =</div><div class='add'>+			cpu_to_be32((u32)(le64_to_cpu(hw_stats-&gt;rx_ucast_pkts)) &amp; 0xFFFFFFFF);</div><div class='add'>+		pma_cnt-&gt;port_rcv_data =</div><div class='add'>+			cpu_to_be32((u32)((le64_to_cpu(hw_stats-&gt;rx_ucast_bytes) &amp;</div><div class='add'>+					   0xFFFFFFFF) / 4));</div><div class='add'>+		pma_cnt-&gt;port_xmit_packets =</div><div class='add'>+			cpu_to_be32((u32)(le64_to_cpu(hw_stats-&gt;tx_ucast_pkts)) &amp; 0xFFFFFFFF);</div><div class='add'>+		pma_cnt-&gt;port_xmit_data =</div><div class='add'>+			cpu_to_be32((u32)((le64_to_cpu(hw_stats-&gt;tx_ucast_bytes)</div><div class='add'>+					   &amp; 0xFFFFFFFF) / 4));</div><div class='add'>+	} else {</div><div class='add'>+		pma_cnt-&gt;port_rcv_packets = cpu_to_be32(estat-&gt;rx_roce_good_pkts);</div><div class='add'>+		pma_cnt-&gt;port_rcv_data = cpu_to_be32((estat-&gt;rx_roce_good_bytes / 4));</div><div class='add'>+		pma_cnt-&gt;port_xmit_packets = cpu_to_be32(estat-&gt;tx_roce_pkts);</div><div class='add'>+		pma_cnt-&gt;port_xmit_data = cpu_to_be32((estat-&gt;tx_roce_bytes / 4));</div><div class='add'>+	}</div><div class='add'>+	pma_cnt-&gt;port_rcv_constraint_errors = (u8)(le64_to_cpu(hw_stats-&gt;rx_discard_pkts) &amp; 0xFF);</div><div class='add'>+	pma_cnt-&gt;port_rcv_errors = cpu_to_be16((u16)(le64_to_cpu(hw_stats-&gt;rx_error_pkts)</div><div class='add'>+						     &amp; 0xFFFF));</div><div class='add'>+	pma_cnt-&gt;port_xmit_constraint_errors = (u8)(le64_to_cpu(hw_stats-&gt;tx_error_pkts) &amp; 0xFF);</div><div class='add'>+	pma_cnt-&gt;port_xmit_discards = cpu_to_be16((u16)(le64_to_cpu(hw_stats-&gt;tx_discard_pkts)</div><div class='add'>+							&amp; 0xFFFF));</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev,</div><div class='ctx'> 			    struct rdma_hw_stats *stats,</div><div class='del'>-			    u8 port, int index)</div><div class='add'>+			    u32 port, int index)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);</div><div class='del'>-	struct ctx_hw_stats *bnxt_re_stats = rdev-&gt;qplib_ctx.stats.dma;</div><div class='add'>+	struct bnxt_qplib_roce_stats *err_s = NULL;</div><div class='add'>+	struct ctx_hw_stats *hw_stats = NULL;</div><div class='ctx'> 	int rc  = 0;</div><div class='ctx'> </div><div class='add'>+	hw_stats = rdev-&gt;qplib_ctx.stats.dma;</div><div class='ctx'> 	if (!port || !stats)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	stats-&gt;value[BNXT_RE_ACTIVE_QP] = atomic_read(&amp;rdev-&gt;qp_count);</div><div class='del'>-	stats-&gt;value[BNXT_RE_ACTIVE_SRQ] = atomic_read(&amp;rdev-&gt;srq_count);</div><div class='del'>-	stats-&gt;value[BNXT_RE_ACTIVE_CQ] = atomic_read(&amp;rdev-&gt;cq_count);</div><div class='del'>-	stats-&gt;value[BNXT_RE_ACTIVE_MR] = atomic_read(&amp;rdev-&gt;mr_count);</div><div class='del'>-	stats-&gt;value[BNXT_RE_ACTIVE_MW] = atomic_read(&amp;rdev-&gt;mw_count);</div><div class='del'>-	if (bnxt_re_stats) {</div><div class='add'>+	if (hw_stats) {</div><div class='ctx'> 		stats-&gt;value[BNXT_RE_RECOVERABLE_ERRORS] =</div><div class='del'>-			le64_to_cpu(bnxt_re_stats-&gt;tx_bcast_pkts);</div><div class='del'>-		stats-&gt;value[BNXT_RE_RX_DROPS] =</div><div class='del'>-			le64_to_cpu(bnxt_re_stats-&gt;rx_drop_pkts);</div><div class='add'>+			le64_to_cpu(hw_stats-&gt;tx_bcast_pkts);</div><div class='add'>+		stats-&gt;value[BNXT_RE_TX_DISCARDS] =</div><div class='add'>+			le64_to_cpu(hw_stats-&gt;tx_discard_pkts);</div><div class='add'>+		stats-&gt;value[BNXT_RE_TX_ERRORS] =</div><div class='add'>+			le64_to_cpu(hw_stats-&gt;tx_error_pkts);</div><div class='add'>+		stats-&gt;value[BNXT_RE_RX_ERRORS] =</div><div class='add'>+			le64_to_cpu(hw_stats-&gt;rx_error_pkts);</div><div class='ctx'> 		stats-&gt;value[BNXT_RE_RX_DISCARDS] =</div><div class='del'>-			le64_to_cpu(bnxt_re_stats-&gt;rx_discard_pkts);</div><div class='add'>+			le64_to_cpu(hw_stats-&gt;rx_discard_pkts);</div><div class='ctx'> 		stats-&gt;value[BNXT_RE_RX_PKTS] =</div><div class='del'>-			le64_to_cpu(bnxt_re_stats-&gt;rx_ucast_pkts);</div><div class='add'>+			le64_to_cpu(hw_stats-&gt;rx_ucast_pkts);</div><div class='ctx'> 		stats-&gt;value[BNXT_RE_RX_BYTES] =</div><div class='del'>-			le64_to_cpu(bnxt_re_stats-&gt;rx_ucast_bytes);</div><div class='add'>+			le64_to_cpu(hw_stats-&gt;rx_ucast_bytes);</div><div class='ctx'> 		stats-&gt;value[BNXT_RE_TX_PKTS] =</div><div class='del'>-			le64_to_cpu(bnxt_re_stats-&gt;tx_ucast_pkts);</div><div class='add'>+			le64_to_cpu(hw_stats-&gt;tx_ucast_pkts);</div><div class='ctx'> 		stats-&gt;value[BNXT_RE_TX_BYTES] =</div><div class='del'>-			le64_to_cpu(bnxt_re_stats-&gt;tx_ucast_bytes);</div><div class='add'>+			le64_to_cpu(hw_stats-&gt;tx_ucast_bytes);</div><div class='ctx'> 	}</div><div class='add'>+	err_s = &amp;rdev-&gt;stats.rstat.errs;</div><div class='ctx'> 	if (test_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &amp;rdev-&gt;flags)) {</div><div class='del'>-		rc = bnxt_qplib_get_roce_stats(&amp;rdev-&gt;rcfw, &amp;rdev-&gt;stats);</div><div class='del'>-		if (rc)</div><div class='add'>+		rc = bnxt_qplib_get_roce_stats(&amp;rdev-&gt;rcfw, err_s);</div><div class='add'>+		if (rc) {</div><div class='ctx'> 			clear_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS,</div><div class='ctx'> 				  &amp;rdev-&gt;flags);</div><div class='del'>-		stats-&gt;value[BNXT_RE_TO_RETRANSMITS] =</div><div class='del'>-					rdev-&gt;stats.to_retransmits;</div><div class='del'>-		stats-&gt;value[BNXT_RE_SEQ_ERR_NAKS_RCVD] =</div><div class='del'>-					rdev-&gt;stats.seq_err_naks_rcvd;</div><div class='del'>-		stats-&gt;value[BNXT_RE_MAX_RETRY_EXCEEDED] =</div><div class='del'>-					rdev-&gt;stats.max_retry_exceeded;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RNR_NAKS_RCVD] =</div><div class='del'>-					rdev-&gt;stats.rnr_naks_rcvd;</div><div class='del'>-		stats-&gt;value[BNXT_RE_MISSING_RESP] =</div><div class='del'>-					rdev-&gt;stats.missing_resp;</div><div class='del'>-		stats-&gt;value[BNXT_RE_UNRECOVERABLE_ERR] =</div><div class='del'>-					rdev-&gt;stats.unrecoverable_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_BAD_RESP_ERR] =</div><div class='del'>-					rdev-&gt;stats.bad_resp_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_LOCAL_QP_OP_ERR]	=</div><div class='del'>-				rdev-&gt;stats.local_qp_op_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_LOCAL_PROTECTION_ERR] =</div><div class='del'>-				rdev-&gt;stats.local_protection_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_MEM_MGMT_OP_ERR] =</div><div class='del'>-				rdev-&gt;stats.mem_mgmt_op_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_REMOTE_INVALID_REQ_ERR] =</div><div class='del'>-				rdev-&gt;stats.remote_invalid_req_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_REMOTE_ACCESS_ERR] =</div><div class='del'>-				rdev-&gt;stats.remote_access_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_REMOTE_OP_ERR] =</div><div class='del'>-				rdev-&gt;stats.remote_op_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_DUP_REQ] =</div><div class='del'>-				rdev-&gt;stats.dup_req;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_EXCEED_MAX] =</div><div class='del'>-				rdev-&gt;stats.res_exceed_max;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_LENGTH_MISMATCH] =</div><div class='del'>-				rdev-&gt;stats.res_length_mismatch;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_EXCEEDS_WQE] =</div><div class='del'>-				rdev-&gt;stats.res_exceeds_wqe;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_OPCODE_ERR] =</div><div class='del'>-				rdev-&gt;stats.res_opcode_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_RX_INVALID_RKEY] =</div><div class='del'>-				rdev-&gt;stats.res_rx_invalid_rkey;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_RX_DOMAIN_ERR] =</div><div class='del'>-				rdev-&gt;stats.res_rx_domain_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_RX_NO_PERM] =</div><div class='del'>-				rdev-&gt;stats.res_rx_no_perm;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_RX_RANGE_ERR]  =</div><div class='del'>-				rdev-&gt;stats.res_rx_range_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_TX_INVALID_RKEY] =</div><div class='del'>-				rdev-&gt;stats.res_tx_invalid_rkey;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_TX_DOMAIN_ERR] =</div><div class='del'>-				rdev-&gt;stats.res_tx_domain_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_TX_NO_PERM] =</div><div class='del'>-				rdev-&gt;stats.res_tx_no_perm;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_TX_RANGE_ERR]  =</div><div class='del'>-				rdev-&gt;stats.res_tx_range_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_IRRQ_OFLOW] =</div><div class='del'>-				rdev-&gt;stats.res_irrq_oflow;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_UNSUP_OPCODE]  =</div><div class='del'>-				rdev-&gt;stats.res_unsup_opcode;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_UNALIGNED_ATOMIC] =</div><div class='del'>-				rdev-&gt;stats.res_unaligned_atomic;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_REM_INV_ERR]   =</div><div class='del'>-				rdev-&gt;stats.res_rem_inv_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_MEM_ERROR] =</div><div class='del'>-				rdev-&gt;stats.res_mem_error;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_SRQ_ERR] =</div><div class='del'>-				rdev-&gt;stats.res_srq_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_CMP_ERR] =</div><div class='del'>-				rdev-&gt;stats.res_cmp_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_INVALID_DUP_RKEY] =</div><div class='del'>-				rdev-&gt;stats.res_invalid_dup_rkey;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_WQE_FORMAT_ERR] =</div><div class='del'>-				rdev-&gt;stats.res_wqe_format_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_CQ_LOAD_ERR]   =</div><div class='del'>-				rdev-&gt;stats.res_cq_load_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_SRQ_LOAD_ERR]  =</div><div class='del'>-				rdev-&gt;stats.res_srq_load_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_TX_PCI_ERR]    =</div><div class='del'>-				rdev-&gt;stats.res_tx_pci_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_RES_RX_PCI_ERR]    =</div><div class='del'>-				rdev-&gt;stats.res_rx_pci_err;</div><div class='del'>-		stats-&gt;value[BNXT_RE_OUT_OF_SEQ_ERR]    =</div><div class='del'>-				rdev-&gt;stats.res_oos_drop_count;</div><div class='add'>+			goto done;</div><div class='add'>+		}</div><div class='add'>+		bnxt_re_copy_err_stats(rdev, stats, err_s);</div><div class='add'>+		if (bnxt_ext_stats_supported(rdev-&gt;chip_ctx, rdev-&gt;dev_attr-&gt;dev_cap_flags,</div><div class='add'>+					     rdev-&gt;is_virtfn)) {</div><div class='add'>+			rc = bnxt_re_get_ext_stat(rdev, stats);</div><div class='add'>+			if (rc) {</div><div class='add'>+				clear_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS,</div><div class='add'>+					  &amp;rdev-&gt;flags);</div><div class='add'>+				goto done;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return ARRAY_SIZE(bnxt_re_stat_name);</div><div class='add'>+done:</div><div class='add'>+	return bnxt_qplib_is_chip_gen_p5_p7(rdev-&gt;chip_ctx) ?</div><div class='add'>+		BNXT_RE_NUM_EXT_COUNTERS : BNXT_RE_NUM_STD_COUNTERS;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct rdma_hw_stats *bnxt_re_ib_alloc_hw_stats(struct ib_device *ibdev,</div><div class='del'>-						u8 port_num)</div><div class='add'>+struct rdma_hw_stats *bnxt_re_ib_alloc_hw_port_stats(struct ib_device *ibdev,</div><div class='add'>+						     u32 port_num)</div><div class='ctx'> {</div><div class='del'>-	BUILD_BUG_ON(ARRAY_SIZE(bnxt_re_stat_name) != BNXT_RE_NUM_COUNTERS);</div><div class='del'>-	/* We support only per port stats */</div><div class='del'>-	if (!port_num)</div><div class='del'>-		return NULL;</div><div class='add'>+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);</div><div class='add'>+	int num_counters = 0;</div><div class='add'>+</div><div class='add'>+	if (bnxt_qplib_is_chip_gen_p5_p7(rdev-&gt;chip_ctx))</div><div class='add'>+		num_counters = BNXT_RE_NUM_EXT_COUNTERS;</div><div class='add'>+	else</div><div class='add'>+		num_counters = BNXT_RE_NUM_STD_COUNTERS;</div><div class='ctx'> </div><div class='del'>-	return rdma_alloc_hw_stats_struct(bnxt_re_stat_name,</div><div class='del'>-					  ARRAY_SIZE(bnxt_re_stat_name),</div><div class='add'>+	return rdma_alloc_hw_stats_struct(bnxt_re_stat_descs, num_counters,</div><div class='ctx'> 					  RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.h b/drivers/infiniband/hw/bnxt_re/hw_counters.h<br/>index 76399f477e5c..09d371d442aa 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/hw_counters.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/hw_counters.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/hw_counters.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/hw_counters.h</a></div><div class='hunk'>@@ -41,17 +41,14 @@</div><div class='ctx'> #define __BNXT_RE_HW_STATS_H__</div><div class='ctx'> </div><div class='ctx'> enum bnxt_re_hw_stats {</div><div class='del'>-	BNXT_RE_ACTIVE_QP,</div><div class='del'>-	BNXT_RE_ACTIVE_SRQ,</div><div class='del'>-	BNXT_RE_ACTIVE_CQ,</div><div class='del'>-	BNXT_RE_ACTIVE_MR,</div><div class='del'>-	BNXT_RE_ACTIVE_MW,</div><div class='ctx'> 	BNXT_RE_RX_PKTS,</div><div class='ctx'> 	BNXT_RE_RX_BYTES,</div><div class='ctx'> 	BNXT_RE_TX_PKTS,</div><div class='ctx'> 	BNXT_RE_TX_BYTES,</div><div class='ctx'> 	BNXT_RE_RECOVERABLE_ERRORS,</div><div class='del'>-	BNXT_RE_RX_DROPS,</div><div class='add'>+	BNXT_RE_TX_ERRORS,</div><div class='add'>+	BNXT_RE_TX_DISCARDS,</div><div class='add'>+	BNXT_RE_RX_ERRORS,</div><div class='ctx'> 	BNXT_RE_RX_DISCARDS,</div><div class='ctx'> 	BNXT_RE_TO_RETRANSMITS,</div><div class='ctx'> 	BNXT_RE_SEQ_ERR_NAKS_RCVD,</div><div class='hunk'>@@ -93,12 +90,76 @@ enum bnxt_re_hw_stats {</div><div class='ctx'> 	BNXT_RE_RES_TX_PCI_ERR,</div><div class='ctx'> 	BNXT_RE_RES_RX_PCI_ERR,</div><div class='ctx'> 	BNXT_RE_OUT_OF_SEQ_ERR,</div><div class='del'>-	BNXT_RE_NUM_COUNTERS</div><div class='add'>+	BNXT_RE_TX_ATOMIC_REQ,</div><div class='add'>+	BNXT_RE_TX_READ_REQ,</div><div class='add'>+	BNXT_RE_TX_READ_RES,</div><div class='add'>+	BNXT_RE_TX_WRITE_REQ,</div><div class='add'>+	BNXT_RE_TX_SEND_REQ,</div><div class='add'>+	BNXT_RE_TX_ROCE_PKTS,</div><div class='add'>+	BNXT_RE_TX_ROCE_BYTES,</div><div class='add'>+	BNXT_RE_RX_ATOMIC_REQ,</div><div class='add'>+	BNXT_RE_RX_READ_REQ,</div><div class='add'>+	BNXT_RE_RX_READ_RESP,</div><div class='add'>+	BNXT_RE_RX_WRITE_REQ,</div><div class='add'>+	BNXT_RE_RX_SEND_REQ,</div><div class='add'>+	BNXT_RE_RX_ROCE_PKTS,</div><div class='add'>+	BNXT_RE_RX_ROCE_BYTES,</div><div class='add'>+	BNXT_RE_RX_ROCE_GOOD_PKTS,</div><div class='add'>+	BNXT_RE_RX_ROCE_GOOD_BYTES,</div><div class='add'>+	BNXT_RE_OOB,</div><div class='add'>+	BNXT_RE_TX_CNP,</div><div class='add'>+	BNXT_RE_RX_CNP,</div><div class='add'>+	BNXT_RE_RX_ECN,</div><div class='add'>+	BNXT_RE_REQ_CQE_ERROR,</div><div class='add'>+	BNXT_RE_RESP_CQE_ERROR,</div><div class='add'>+	BNXT_RE_RESP_REMOTE_ACCESS_ERRS,</div><div class='add'>+	BNXT_RE_NUM_EXT_COUNTERS</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct rdma_hw_stats *bnxt_re_ib_alloc_hw_stats(struct ib_device *ibdev,</div><div class='del'>-						u8 port_num);</div><div class='add'>+#define BNXT_RE_NUM_STD_COUNTERS (BNXT_RE_OUT_OF_SEQ_ERR + 1)</div><div class='add'>+</div><div class='add'>+struct bnxt_re_db_pacing_stats {</div><div class='add'>+	u64 resched;</div><div class='add'>+	u64 complete;</div><div class='add'>+	u64 alerts;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_re_res_cntrs {</div><div class='add'>+	atomic_t qp_count;</div><div class='add'>+	atomic_t rc_qp_count;</div><div class='add'>+	atomic_t ud_qp_count;</div><div class='add'>+	atomic_t cq_count;</div><div class='add'>+	atomic_t srq_count;</div><div class='add'>+	atomic_t mr_count;</div><div class='add'>+	atomic_t mw_count;</div><div class='add'>+	atomic_t ah_count;</div><div class='add'>+	atomic_t pd_count;</div><div class='add'>+	atomic_t resize_count;</div><div class='add'>+	u64 qp_watermark;</div><div class='add'>+	u64 rc_qp_watermark;</div><div class='add'>+	u64 ud_qp_watermark;</div><div class='add'>+	u64 cq_watermark;</div><div class='add'>+	u64 srq_watermark;</div><div class='add'>+	u64 mr_watermark;</div><div class='add'>+	u64 mw_watermark;</div><div class='add'>+	u64 ah_watermark;</div><div class='add'>+	u64 pd_watermark;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_re_rstat {</div><div class='add'>+	struct bnxt_qplib_roce_stats    errs;</div><div class='add'>+	struct bnxt_qplib_ext_stat      ext_stat;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_re_stats {</div><div class='add'>+	struct bnxt_re_rstat            rstat;</div><div class='add'>+	struct bnxt_re_res_cntrs        res;</div><div class='add'>+	struct bnxt_re_db_pacing_stats  pacing;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct rdma_hw_stats *bnxt_re_ib_alloc_hw_port_stats(struct ib_device *ibdev,</div><div class='add'>+						     u32 port_num);</div><div class='ctx'> int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev,</div><div class='ctx'> 			    struct rdma_hw_stats *stats,</div><div class='del'>-			    u8 port, int index);</div><div class='add'>+			    u32 port, int index);</div><div class='ctx'> #endif /* __BNXT_RE_HW_STATS_H__ */</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c<br/>index 2c3685faa57a..f19b55c13d58 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/ib_verbs.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/ib_verbs.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/ib_verbs.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/ib_verbs.c</a></div><div class='hunk'>@@ -41,6 +41,7 @@</div><div class='ctx'> #include &lt;linux/pci.h&gt;</div><div class='ctx'> #include &lt;linux/netdevice.h&gt;</div><div class='ctx'> #include &lt;linux/if_ether.h&gt;</div><div class='add'>+#include &lt;net/addrconf.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include &lt;rdma/ib_verbs.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_user_verbs.h&gt;</div><div class='hunk'>@@ -48,9 +49,9 @@</div><div class='ctx'> #include &lt;rdma/ib_addr.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_mad.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_cache.h&gt;</div><div class='add'>+#include &lt;rdma/ib_pma.h&gt;</div><div class='ctx'> #include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='del'>-</div><div class='del'>-#include "bnxt_ulp.h"</div><div class='add'>+#include &lt;linux/hashtable.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "roce_hsi.h"</div><div class='ctx'> #include "qplib_res.h"</div><div class='hunk'>@@ -60,6 +61,16 @@</div><div class='ctx'> </div><div class='ctx'> #include "bnxt_re.h"</div><div class='ctx'> #include "ib_verbs.h"</div><div class='add'>+#include "debugfs.h"</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/uverbs_types.h&gt;</div><div class='add'>+#include &lt;rdma/uverbs_std_types.h&gt;</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/ib_user_ioctl_cmds.h&gt;</div><div class='add'>+</div><div class='add'>+#define UVERBS_MODULE_NAME bnxt_re</div><div class='add'>+#include &lt;rdma/uverbs_named_ioctl.h&gt;</div><div class='add'>+</div><div class='ctx'> #include &lt;rdma/bnxt_re-abi.h&gt;</div><div class='ctx'> </div><div class='ctx'> static int __from_ib_access_flags(int iflags)</div><div class='hunk'>@@ -83,9 +94,9 @@ static int __from_ib_access_flags(int iflags)</div><div class='ctx'> 	return qflags;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static enum ib_access_flags __to_ib_access_flags(int qflags)</div><div class='add'>+static int __to_ib_access_flags(int qflags)</div><div class='ctx'> {</div><div class='del'>-	enum ib_access_flags iflags = 0;</div><div class='add'>+	int iflags = 0;</div><div class='ctx'> </div><div class='ctx'> 	if (qflags &amp; BNXT_QPLIB_ACCESS_LOCAL_WRITE)</div><div class='ctx'> 		iflags |= IB_ACCESS_LOCAL_WRITE;</div><div class='hunk'>@@ -102,7 +113,57 @@ static enum ib_access_flags __to_ib_access_flags(int qflags)</div><div class='ctx'> 	if (qflags &amp; BNXT_QPLIB_ACCESS_ON_DEMAND)</div><div class='ctx'> 		iflags |= IB_ACCESS_ON_DEMAND;</div><div class='ctx'> 	return iflags;</div><div class='del'>-};</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u8 __qp_access_flags_from_ib(struct bnxt_qplib_chip_ctx *cctx, int iflags)</div><div class='add'>+{</div><div class='add'>+	u8 qflags = 0;</div><div class='add'>+</div><div class='add'>+	if (!bnxt_qplib_is_chip_gen_p5_p7(cctx))</div><div class='add'>+		/* For Wh+ */</div><div class='add'>+		return (u8)__from_ib_access_flags(iflags);</div><div class='add'>+</div><div class='add'>+	/* For P5, P7 and later chips */</div><div class='add'>+	if (iflags &amp; IB_ACCESS_LOCAL_WRITE)</div><div class='add'>+		qflags |= CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE;</div><div class='add'>+	if (iflags &amp; IB_ACCESS_REMOTE_WRITE)</div><div class='add'>+		qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE;</div><div class='add'>+	if (iflags &amp; IB_ACCESS_REMOTE_READ)</div><div class='add'>+		qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ;</div><div class='add'>+	if (iflags &amp; IB_ACCESS_REMOTE_ATOMIC)</div><div class='add'>+		qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC;</div><div class='add'>+</div><div class='add'>+	return qflags;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int __qp_access_flags_to_ib(struct bnxt_qplib_chip_ctx *cctx, u8 qflags)</div><div class='add'>+{</div><div class='add'>+	int iflags = 0;</div><div class='add'>+</div><div class='add'>+	if (!bnxt_qplib_is_chip_gen_p5_p7(cctx))</div><div class='add'>+		/* For Wh+ */</div><div class='add'>+		return __to_ib_access_flags(qflags);</div><div class='add'>+</div><div class='add'>+	/* For P5, P7 and later chips */</div><div class='add'>+	if (qflags &amp; CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE)</div><div class='add'>+		iflags |= IB_ACCESS_LOCAL_WRITE;</div><div class='add'>+	if (qflags &amp; CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE)</div><div class='add'>+		iflags |= IB_ACCESS_REMOTE_WRITE;</div><div class='add'>+	if (qflags &amp; CMDQ_MODIFY_QP_ACCESS_REMOTE_READ)</div><div class='add'>+		iflags |= IB_ACCESS_REMOTE_READ;</div><div class='add'>+	if (qflags &amp; CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC)</div><div class='add'>+		iflags |= IB_ACCESS_REMOTE_ATOMIC;</div><div class='add'>+</div><div class='add'>+	return iflags;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_check_and_set_relaxed_ordering(struct bnxt_re_dev *rdev,</div><div class='add'>+						   struct bnxt_qplib_mrw *qplib_mr)</div><div class='add'>+{</div><div class='add'>+	if (_is_relaxed_ordering_supported(rdev-&gt;dev_attr-&gt;dev_cap_flags2) &amp;&amp;</div><div class='add'>+	    pcie_relaxed_ordering_enabled(rdev-&gt;en_dev-&gt;pdev))</div><div class='add'>+		qplib_mr-&gt;flags |= CMDQ_REGISTER_MR_FLAGS_ENABLE_RO;</div><div class='add'>+}</div><div class='ctx'> </div><div class='ctx'> static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,</div><div class='ctx'> 			     struct bnxt_qplib_sge *sg_list, int num)</div><div class='hunk'>@@ -124,20 +185,20 @@ int bnxt_re_query_device(struct ib_device *ibdev,</div><div class='ctx'> 			 struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);</div><div class='del'>-	struct bnxt_qplib_dev_attr *dev_attr = &amp;rdev-&gt;dev_attr;</div><div class='add'>+	struct bnxt_qplib_dev_attr *dev_attr = rdev-&gt;dev_attr;</div><div class='ctx'> </div><div class='ctx'> 	memset(ib_attr, 0, sizeof(*ib_attr));</div><div class='ctx'> 	memcpy(&amp;ib_attr-&gt;fw_ver, dev_attr-&gt;fw_ver,</div><div class='ctx'> 	       min(sizeof(dev_attr-&gt;fw_ver),</div><div class='ctx'> 		   sizeof(ib_attr-&gt;fw_ver)));</div><div class='del'>-	bnxt_qplib_get_guid(rdev-&gt;netdev-&gt;dev_addr,</div><div class='del'>-			    (u8 *)&amp;ib_attr-&gt;sys_image_guid);</div><div class='add'>+	addrconf_addr_eui48((u8 *)&amp;ib_attr-&gt;sys_image_guid,</div><div class='add'>+			    rdev-&gt;netdev-&gt;dev_addr);</div><div class='ctx'> 	ib_attr-&gt;max_mr_size = BNXT_RE_MAX_MR_SIZE;</div><div class='del'>-	ib_attr-&gt;page_size_cap = BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_2M;</div><div class='add'>+	ib_attr-&gt;page_size_cap = BNXT_RE_PAGE_SIZE_SUPPORTED;</div><div class='ctx'> </div><div class='ctx'> 	ib_attr-&gt;vendor_id = rdev-&gt;en_dev-&gt;pdev-&gt;vendor;</div><div class='ctx'> 	ib_attr-&gt;vendor_part_id = rdev-&gt;en_dev-&gt;pdev-&gt;device;</div><div class='del'>-	ib_attr-&gt;hw_ver = rdev-&gt;en_dev-&gt;pdev-&gt;subsystem_device;</div><div class='add'>+	ib_attr-&gt;hw_ver = rdev-&gt;en_dev-&gt;pdev-&gt;revision;</div><div class='ctx'> 	ib_attr-&gt;max_qp = dev_attr-&gt;max_qp;</div><div class='ctx'> 	ib_attr-&gt;max_qp_wr = dev_attr-&gt;max_qp_wqes;</div><div class='ctx'> 	ib_attr-&gt;device_cap_flags =</div><div class='hunk'>@@ -145,13 +206,13 @@ int bnxt_re_query_device(struct ib_device *ibdev,</div><div class='ctx'> 				    | IB_DEVICE_RC_RNR_NAK_GEN</div><div class='ctx'> 				    | IB_DEVICE_SHUTDOWN_PORT</div><div class='ctx'> 				    | IB_DEVICE_SYS_IMAGE_GUID</div><div class='del'>-				    | IB_DEVICE_LOCAL_DMA_LKEY</div><div class='ctx'> 				    | IB_DEVICE_RESIZE_MAX_WR</div><div class='ctx'> 				    | IB_DEVICE_PORT_ACTIVE_EVENT</div><div class='ctx'> 				    | IB_DEVICE_N_NOTIFY_CQ</div><div class='ctx'> 				    | IB_DEVICE_MEM_WINDOW</div><div class='ctx'> 				    | IB_DEVICE_MEM_WINDOW_TYPE_2B</div><div class='ctx'> 				    | IB_DEVICE_MEM_MGT_EXTENSIONS;</div><div class='add'>+	ib_attr-&gt;kernel_cap_flags = IBK_LOCAL_DMA_LKEY;</div><div class='ctx'> 	ib_attr-&gt;max_send_sge = dev_attr-&gt;max_qp_sges;</div><div class='ctx'> 	ib_attr-&gt;max_recv_sge = dev_attr-&gt;max_qp_sges;</div><div class='ctx'> 	ib_attr-&gt;max_sge_rd = dev_attr-&gt;max_qp_sges;</div><div class='hunk'>@@ -163,6 +224,10 @@ int bnxt_re_query_device(struct ib_device *ibdev,</div><div class='ctx'> 	ib_attr-&gt;max_qp_init_rd_atom = dev_attr-&gt;max_qp_init_rd_atom;</div><div class='ctx'> 	ib_attr-&gt;atomic_cap = IB_ATOMIC_NONE;</div><div class='ctx'> 	ib_attr-&gt;masked_atomic_cap = IB_ATOMIC_NONE;</div><div class='add'>+	if (dev_attr-&gt;is_atomic) {</div><div class='add'>+		ib_attr-&gt;atomic_cap = IB_ATOMIC_GLOB;</div><div class='add'>+		ib_attr-&gt;masked_atomic_cap = IB_ATOMIC_GLOB;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	ib_attr-&gt;max_ee_rd_atom = 0;</div><div class='ctx'> 	ib_attr-&gt;max_res_rd_atom = 0;</div><div class='hunk'>@@ -177,9 +242,6 @@ int bnxt_re_query_device(struct ib_device *ibdev,</div><div class='ctx'> 	ib_attr-&gt;max_total_mcast_qp_attach = 0;</div><div class='ctx'> 	ib_attr-&gt;max_ah = dev_attr-&gt;max_ah;</div><div class='ctx'> </div><div class='del'>-	ib_attr-&gt;max_fmr = 0;</div><div class='del'>-	ib_attr-&gt;max_map_per_fmr = 0;</div><div class='del'>-</div><div class='ctx'> 	ib_attr-&gt;max_srq = dev_attr-&gt;max_srq;</div><div class='ctx'> 	ib_attr-&gt;max_srq_wr = dev_attr-&gt;max_srq_wqes;</div><div class='ctx'> 	ib_attr-&gt;max_srq_sge = dev_attr-&gt;max_srq_sges;</div><div class='hunk'>@@ -195,39 +257,40 @@ int bnxt_re_modify_device(struct ib_device *ibdev,</div><div class='ctx'> 			  int device_modify_mask,</div><div class='ctx'> 			  struct ib_device_modify *device_modify)</div><div class='ctx'> {</div><div class='del'>-	switch (device_modify_mask) {</div><div class='del'>-	case IB_DEVICE_MODIFY_SYS_IMAGE_GUID:</div><div class='del'>-		/* Modify the GUID requires the modification of the GID table */</div><div class='del'>-		/* GUID should be made as READ-ONLY */</div><div class='del'>-		break;</div><div class='del'>-	case IB_DEVICE_MODIFY_NODE_DESC:</div><div class='del'>-		/* Node Desc should be made as READ-ONLY */</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='add'>+	ibdev_dbg(ibdev, "Modify device with mask 0x%x", device_modify_mask);</div><div class='add'>+</div><div class='add'>+	if (device_modify_mask &amp; ~IB_DEVICE_MODIFY_NODE_DESC)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (!(device_modify_mask &amp; IB_DEVICE_MODIFY_NODE_DESC))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	memcpy(ibdev-&gt;node_desc, device_modify-&gt;node_desc, IB_DEVICE_NODE_DESC_MAX);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* Port */</div><div class='del'>-int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,</div><div class='add'>+int bnxt_re_query_port(struct ib_device *ibdev, u32 port_num,</div><div class='ctx'> 		       struct ib_port_attr *port_attr)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);</div><div class='del'>-	struct bnxt_qplib_dev_attr *dev_attr = &amp;rdev-&gt;dev_attr;</div><div class='add'>+	struct bnxt_qplib_dev_attr *dev_attr = rdev-&gt;dev_attr;</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='ctx'> 	memset(port_attr, 0, sizeof(*port_attr));</div><div class='ctx'> </div><div class='ctx'> 	if (netif_running(rdev-&gt;netdev) &amp;&amp; netif_carrier_ok(rdev-&gt;netdev)) {</div><div class='ctx'> 		port_attr-&gt;state = IB_PORT_ACTIVE;</div><div class='del'>-		port_attr-&gt;phys_state = 5;</div><div class='add'>+		port_attr-&gt;phys_state = IB_PORT_PHYS_STATE_LINK_UP;</div><div class='ctx'> 	} else {</div><div class='ctx'> 		port_attr-&gt;state = IB_PORT_DOWN;</div><div class='del'>-		port_attr-&gt;phys_state = 3;</div><div class='add'>+		port_attr-&gt;phys_state = IB_PORT_PHYS_STATE_DISABLED;</div><div class='ctx'> 	}</div><div class='ctx'> 	port_attr-&gt;max_mtu = IB_MTU_4096;</div><div class='ctx'> 	port_attr-&gt;active_mtu = iboe_get_mtu(rdev-&gt;netdev-&gt;mtu);</div><div class='del'>-	port_attr-&gt;gid_tbl_len = dev_attr-&gt;max_sgid;</div><div class='add'>+	/* One GID is reserved for RawEth QP. Report one less */</div><div class='add'>+	port_attr-&gt;gid_tbl_len = (rdev-&gt;rcfw.roce_mirror ? (dev_attr-&gt;max_sgid - 1) :</div><div class='add'>+				  dev_attr-&gt;max_sgid);</div><div class='ctx'> 	port_attr-&gt;port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |</div><div class='ctx'> 				    IB_PORT_DEVICE_MGMT_SUP |</div><div class='ctx'> 				    IB_PORT_VENDOR_CLASS_SUP;</div><div class='hunk'>@@ -244,13 +307,13 @@ int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,</div><div class='ctx'> 	port_attr-&gt;sm_sl = 0;</div><div class='ctx'> 	port_attr-&gt;subnet_timeout = 0;</div><div class='ctx'> 	port_attr-&gt;init_type_reply = 0;</div><div class='del'>-	port_attr-&gt;active_speed = rdev-&gt;active_speed;</div><div class='del'>-	port_attr-&gt;active_width = rdev-&gt;active_width;</div><div class='add'>+	rc = ib_get_eth_speed(&amp;rdev-&gt;ibdev, port_num, &amp;port_attr-&gt;active_speed,</div><div class='add'>+			      &amp;port_attr-&gt;active_width);</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num,</div><div class='add'>+int bnxt_re_get_port_immutable(struct ib_device *ibdev, u32 port_num,</div><div class='ctx'> 			       struct ib_port_immutable *immutable)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr port_attr;</div><div class='hunk'>@@ -271,27 +334,26 @@ void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str)</div><div class='ctx'> 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);</div><div class='ctx'> </div><div class='ctx'> 	snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d.%d",</div><div class='del'>-		 rdev-&gt;dev_attr.fw_ver[0], rdev-&gt;dev_attr.fw_ver[1],</div><div class='del'>-		 rdev-&gt;dev_attr.fw_ver[2], rdev-&gt;dev_attr.fw_ver[3]);</div><div class='add'>+		 rdev-&gt;dev_attr-&gt;fw_ver[0], rdev-&gt;dev_attr-&gt;fw_ver[1],</div><div class='add'>+		 rdev-&gt;dev_attr-&gt;fw_ver[2], rdev-&gt;dev_attr-&gt;fw_ver[3]);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num,</div><div class='add'>+int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num,</div><div class='ctx'> 		       u16 index, u16 *pkey)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);</div><div class='add'>+	if (index &gt; 0)</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	/* Ignore port_num */</div><div class='add'>+	*pkey = IB_DEFAULT_PKEY_FULL;</div><div class='ctx'> </div><div class='del'>-	memset(pkey, 0, sizeof(*pkey));</div><div class='del'>-	return bnxt_qplib_get_pkey(&amp;rdev-&gt;qplib_res,</div><div class='del'>-				   &amp;rdev-&gt;qplib_res.pkey_tbl, index, pkey);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num,</div><div class='add'>+int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num,</div><div class='ctx'> 		      int index, union ib_gid *gid)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);</div><div class='del'>-	int rc = 0;</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='ctx'> 	/* Ignore port_num */</div><div class='ctx'> 	memset(gid, 0, sizeof(*gid));</div><div class='hunk'>@@ -308,16 +370,18 @@ int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context)</div><div class='ctx'> 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr-&gt;device, ibdev);</div><div class='ctx'> 	struct bnxt_qplib_sgid_tbl *sgid_tbl = &amp;rdev-&gt;qplib_res.sgid_tbl;</div><div class='ctx'> 	struct bnxt_qplib_gid *gid_to_del;</div><div class='add'>+	u16 vlan_id = 0xFFFF;</div><div class='ctx'> </div><div class='ctx'> 	/* Delete the entry from the hardware */</div><div class='ctx'> 	ctx = *context;</div><div class='ctx'> 	if (!ctx)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (sgid_tbl &amp;&amp; sgid_tbl-&gt;active) {</div><div class='add'>+	if (sgid_tbl-&gt;active) {</div><div class='ctx'> 		if (ctx-&gt;idx &gt;= sgid_tbl-&gt;max)</div><div class='ctx'> 			return -EINVAL;</div><div class='del'>-		gid_to_del = &amp;sgid_tbl-&gt;tbl[ctx-&gt;idx];</div><div class='add'>+		gid_to_del = &amp;sgid_tbl-&gt;tbl[ctx-&gt;idx].gid;</div><div class='add'>+		vlan_id = sgid_tbl-&gt;tbl[ctx-&gt;idx].vlan_id;</div><div class='ctx'> 		/* DEL_GID is called in WQ context(netdevice_event_work_handler)</div><div class='ctx'> 		 * or via the ib_unregister_device path. In the former case QP1</div><div class='ctx'> 		 * may not be destroyed yet, in which case just return as FW</div><div class='hunk'>@@ -328,17 +392,18 @@ int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context)</div><div class='ctx'> 		 */</div><div class='ctx'> 		if (ctx-&gt;idx == 0 &amp;&amp;</div><div class='ctx'> 		    rdma_link_local_addr((struct in6_addr *)gid_to_del) &amp;&amp;</div><div class='del'>-		    ctx-&gt;refcnt == 1 &amp;&amp; rdev-&gt;qp1_sqp) {</div><div class='del'>-			dev_dbg(rdev_to_dev(rdev),</div><div class='del'>-				"Trying to delete GID0 while QP1 is alive\n");</div><div class='add'>+		    ctx-&gt;refcnt == 1 &amp;&amp; rdev-&gt;gsi_ctx.gsi_sqp) {</div><div class='add'>+			ibdev_dbg(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "Trying to delete GID0 while QP1 is alive\n");</div><div class='ctx'> 			return -EFAULT;</div><div class='ctx'> 		}</div><div class='ctx'> 		ctx-&gt;refcnt--;</div><div class='ctx'> 		if (!ctx-&gt;refcnt) {</div><div class='del'>-			rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del, true);</div><div class='add'>+			rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del,</div><div class='add'>+						 vlan_id,  true);</div><div class='ctx'> 			if (rc) {</div><div class='del'>-				dev_err(rdev_to_dev(rdev),</div><div class='del'>-					"Failed to remove GID: %#x", rc);</div><div class='add'>+				ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+					  "Failed to remove GID: %#x", rc);</div><div class='ctx'> 			} else {</div><div class='ctx'> 				ctx_tbl = sgid_tbl-&gt;ctx;</div><div class='ctx'> 				ctx_tbl[ctx-&gt;idx] = NULL;</div><div class='hunk'>@@ -366,7 +431,7 @@ int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context)</div><div class='ctx'> </div><div class='ctx'> 	rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)&amp;attr-&gt;gid,</div><div class='ctx'> 				 rdev-&gt;qplib_res.netdev-&gt;dev_addr,</div><div class='del'>-				 vlan_id, true, &amp;tbl_idx);</div><div class='add'>+				 vlan_id, true, &amp;tbl_idx, false, 0);</div><div class='ctx'> 	if (rc == -EALREADY) {</div><div class='ctx'> 		ctx_tbl = sgid_tbl-&gt;ctx;</div><div class='ctx'> 		ctx_tbl[tbl_idx]-&gt;refcnt++;</div><div class='hunk'>@@ -375,7 +440,7 @@ int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (rc &lt; 0) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to add GID: %#x", rc);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to add GID: %#x", rc);</div><div class='ctx'> 		return rc;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -392,7 +457,7 @@ int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,</div><div class='del'>-					    u8 port_num)</div><div class='add'>+					    u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	return IB_LINK_LAYER_ETHERNET;</div><div class='ctx'> }</div><div class='hunk'>@@ -404,6 +469,10 @@ static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd)</div><div class='ctx'> 	struct bnxt_re_fence_data *fence = &amp;pd-&gt;fence;</div><div class='ctx'> 	struct ib_mr *ib_mr = &amp;fence-&gt;mr-&gt;ib_mr;</div><div class='ctx'> 	struct bnxt_qplib_swqe *wqe = &amp;fence-&gt;bind_wqe;</div><div class='add'>+	struct bnxt_re_dev *rdev = pd-&gt;rdev;</div><div class='add'>+</div><div class='add'>+	if (bnxt_qplib_is_chip_gen_p5_p7(rdev-&gt;chip_ctx))</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='ctx'> 	memset(wqe, 0, sizeof(*wqe));</div><div class='ctx'> 	wqe-&gt;type = BNXT_QPLIB_SWQE_TYPE_BIND_MW;</div><div class='hunk'>@@ -438,12 +507,12 @@ static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp)</div><div class='ctx'> 	wqe.bind.r_key = fence-&gt;bind_rkey;</div><div class='ctx'> 	fence-&gt;bind_rkey = ib_inc_rkey(fence-&gt;bind_rkey);</div><div class='ctx'> </div><div class='del'>-	dev_dbg(rdev_to_dev(qp-&gt;rdev),</div><div class='del'>-		"Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n",</div><div class='add'>+	ibdev_dbg(&amp;qp-&gt;rdev-&gt;ibdev,</div><div class='add'>+		  "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n",</div><div class='ctx'> 		wqe.bind.r_key, qp-&gt;qplib_qp.id, pd);</div><div class='ctx'> 	rc = bnxt_qplib_post_send(&amp;qp-&gt;qplib_qp, &amp;wqe);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(qp-&gt;rdev), "Failed to bind fence-WQE\n");</div><div class='add'>+		ibdev_err(&amp;qp-&gt;rdev-&gt;ibdev, "Failed to bind fence-WQE\n");</div><div class='ctx'> 		return rc;</div><div class='ctx'> 	}</div><div class='ctx'> 	bnxt_qplib_post_send_db(&amp;qp-&gt;qplib_qp);</div><div class='hunk'>@@ -458,6 +527,9 @@ static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd)</div><div class='ctx'> 	struct device *dev = &amp;rdev-&gt;en_dev-&gt;pdev-&gt;dev;</div><div class='ctx'> 	struct bnxt_re_mr *mr = fence-&gt;mr;</div><div class='ctx'> </div><div class='add'>+	if (bnxt_qplib_is_chip_gen_p5_p7(rdev-&gt;chip_ctx))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='ctx'> 	if (fence-&gt;mw) {</div><div class='ctx'> 		bnxt_re_dealloc_mw(fence-&gt;mw);</div><div class='ctx'> 		fence-&gt;mw = NULL;</div><div class='hunk'>@@ -487,14 +559,16 @@ static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)</div><div class='ctx'> 	struct bnxt_re_mr *mr = NULL;</div><div class='ctx'> 	dma_addr_t dma_addr = 0;</div><div class='ctx'> 	struct ib_mw *mw;</div><div class='del'>-	u64 pbl_tbl;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='add'>+	if (bnxt_qplib_is_chip_gen_p5_p7(rdev-&gt;chip_ctx))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='ctx'> 	dma_addr = dma_map_single(dev, fence-&gt;va, BNXT_RE_FENCE_BYTES,</div><div class='ctx'> 				  DMA_BIDIRECTIONAL);</div><div class='ctx'> 	rc = dma_mapping_error(dev, dma_addr);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to dma-map fence-MR-mem\n");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to dma-map fence-MR-mem\n");</div><div class='ctx'> 		rc = -EIO;</div><div class='ctx'> 		fence-&gt;dma_addr = 0;</div><div class='ctx'> 		goto fail;</div><div class='hunk'>@@ -511,22 +585,26 @@ static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)</div><div class='ctx'> 	mr-&gt;rdev = rdev;</div><div class='ctx'> 	mr-&gt;qplib_mr.pd = &amp;pd-&gt;qplib_pd;</div><div class='ctx'> 	mr-&gt;qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;</div><div class='del'>-	mr-&gt;qplib_mr.flags = __from_ib_access_flags(mr_access_flags);</div><div class='del'>-	rc = bnxt_qplib_alloc_mrw(&amp;rdev-&gt;qplib_res, &amp;mr-&gt;qplib_mr);</div><div class='del'>-	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to alloc fence-HW-MR\n");</div><div class='del'>-		goto fail;</div><div class='del'>-	}</div><div class='add'>+	mr-&gt;qplib_mr.access_flags = __from_ib_access_flags(mr_access_flags);</div><div class='add'>+	if (!_is_alloc_mr_unified(rdev-&gt;dev_attr-&gt;dev_cap_flags)) {</div><div class='add'>+		rc = bnxt_qplib_alloc_mrw(&amp;rdev-&gt;qplib_res, &amp;mr-&gt;qplib_mr);</div><div class='add'>+		if (rc) {</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev, "Failed to alloc fence-HW-MR\n");</div><div class='add'>+			goto fail;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='del'>-	/* Register MR */</div><div class='del'>-	mr-&gt;ib_mr.lkey = mr-&gt;qplib_mr.lkey;</div><div class='add'>+		/* Register MR */</div><div class='add'>+		mr-&gt;ib_mr.lkey = mr-&gt;qplib_mr.lkey;</div><div class='add'>+	} else {</div><div class='add'>+		mr-&gt;qplib_mr.flags = CMDQ_REGISTER_MR_FLAGS_ALLOC_MR;</div><div class='add'>+	}</div><div class='ctx'> 	mr-&gt;qplib_mr.va = (u64)(unsigned long)fence-&gt;va;</div><div class='ctx'> 	mr-&gt;qplib_mr.total_size = BNXT_RE_FENCE_BYTES;</div><div class='del'>-	pbl_tbl = dma_addr;</div><div class='del'>-	rc = bnxt_qplib_reg_mr(&amp;rdev-&gt;qplib_res, &amp;mr-&gt;qplib_mr, &amp;pbl_tbl,</div><div class='del'>-			       BNXT_RE_FENCE_PBL_SIZE, false, PAGE_SIZE);</div><div class='add'>+	rc = bnxt_qplib_reg_mr(&amp;rdev-&gt;qplib_res, &amp;mr-&gt;qplib_mr, NULL,</div><div class='add'>+			       BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE,</div><div class='add'>+			       _is_alloc_mr_unified(rdev-&gt;dev_attr-&gt;dev_cap_flags));</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to register fence-MR\n");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to register fence-MR\n");</div><div class='ctx'> 		goto fail;</div><div class='ctx'> 	}</div><div class='ctx'> 	mr-&gt;ib_mr.rkey = mr-&gt;qplib_mr.rkey;</div><div class='hunk'>@@ -534,8 +612,8 @@ static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)</div><div class='ctx'> 	/* Create a fence MW only for kernel consumers */</div><div class='ctx'> 	mw = bnxt_re_alloc_mw(&amp;pd-&gt;ib_pd, IB_MW_TYPE_1, NULL);</div><div class='ctx'> 	if (IS_ERR(mw)) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev),</div><div class='del'>-			"Failed to create fence-MW for PD: %p\n", pd);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to create fence-MW for PD: %p\n", pd);</div><div class='ctx'> 		rc = PTR_ERR(mw);</div><div class='ctx'> 		goto fail;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -549,17 +627,69 @@ fail:</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static struct bnxt_re_user_mmap_entry*</div><div class='add'>+bnxt_re_mmap_entry_insert(struct bnxt_re_ucontext *uctx, u64 mem_offset,</div><div class='add'>+			  enum bnxt_re_mmap_flag mmap_flag, u64 *offset)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_user_mmap_entry *entry;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	entry = kzalloc(sizeof(*entry), GFP_KERNEL);</div><div class='add'>+	if (!entry)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	entry-&gt;mem_offset = mem_offset;</div><div class='add'>+	entry-&gt;mmap_flag = mmap_flag;</div><div class='add'>+	entry-&gt;uctx = uctx;</div><div class='add'>+</div><div class='add'>+	switch (mmap_flag) {</div><div class='add'>+	case BNXT_RE_MMAP_SH_PAGE:</div><div class='add'>+		ret = rdma_user_mmap_entry_insert_exact(&amp;uctx-&gt;ib_uctx,</div><div class='add'>+							&amp;entry-&gt;rdma_entry, PAGE_SIZE, 0);</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_RE_MMAP_UC_DB:</div><div class='add'>+	case BNXT_RE_MMAP_WC_DB:</div><div class='add'>+	case BNXT_RE_MMAP_DBR_BAR:</div><div class='add'>+	case BNXT_RE_MMAP_DBR_PAGE:</div><div class='add'>+	case BNXT_RE_MMAP_TOGGLE_PAGE:</div><div class='add'>+		ret = rdma_user_mmap_entry_insert(&amp;uctx-&gt;ib_uctx,</div><div class='add'>+						  &amp;entry-&gt;rdma_entry, PAGE_SIZE);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ret) {</div><div class='add'>+		kfree(entry);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+	if (offset)</div><div class='add'>+		*offset = rdma_user_mmap_get_offset(&amp;entry-&gt;rdma_entry);</div><div class='add'>+</div><div class='add'>+	return entry;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /* Protection Domains */</div><div class='del'>-void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)</div><div class='add'>+int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);</div><div class='ctx'> 	struct bnxt_re_dev *rdev = pd-&gt;rdev;</div><div class='ctx'> </div><div class='add'>+	if (udata) {</div><div class='add'>+		rdma_user_mmap_entry_remove(pd-&gt;pd_db_mmap);</div><div class='add'>+		pd-&gt;pd_db_mmap = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	bnxt_re_destroy_fence_mr(pd);</div><div class='ctx'> </div><div class='del'>-	if (pd-&gt;qplib_pd.id)</div><div class='del'>-		bnxt_qplib_dealloc_pd(&amp;rdev-&gt;qplib_res, &amp;rdev-&gt;qplib_res.pd_tbl,</div><div class='del'>-				      &amp;pd-&gt;qplib_pd);</div><div class='add'>+	if (pd-&gt;qplib_pd.id) {</div><div class='add'>+		if (!bnxt_qplib_dealloc_pd(&amp;rdev-&gt;qplib_res,</div><div class='add'>+					   &amp;rdev-&gt;qplib_res.pd_tbl,</div><div class='add'>+					   &amp;pd-&gt;qplib_pd))</div><div class='add'>+			atomic_dec(&amp;rdev-&gt;stats.res.pd_count);</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='hunk'>@@ -569,25 +699,27 @@ int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='ctx'> 	struct bnxt_re_ucontext *ucntx = rdma_udata_to_drv_context(</div><div class='ctx'> 		udata, struct bnxt_re_ucontext, ib_uctx);</div><div class='ctx'> 	struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd);</div><div class='del'>-	int rc;</div><div class='add'>+	struct bnxt_re_user_mmap_entry *entry = NULL;</div><div class='add'>+	u32 active_pds;</div><div class='add'>+	int rc = 0;</div><div class='ctx'> </div><div class='ctx'> 	pd-&gt;rdev = rdev;</div><div class='del'>-	if (bnxt_qplib_alloc_pd(&amp;rdev-&gt;qplib_res.pd_tbl, &amp;pd-&gt;qplib_pd)) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to allocate HW PD");</div><div class='add'>+	if (bnxt_qplib_alloc_pd(&amp;rdev-&gt;qplib_res, &amp;pd-&gt;qplib_pd)) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to allocate HW PD");</div><div class='ctx'> 		rc = -ENOMEM;</div><div class='ctx'> 		goto fail;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (udata) {</div><div class='del'>-		struct bnxt_re_pd_resp resp;</div><div class='add'>+		struct bnxt_re_pd_resp resp = {};</div><div class='ctx'> </div><div class='ctx'> 		if (!ucntx-&gt;dpi.dbr) {</div><div class='ctx'> 			/* Allocate DPI in alloc_pd to avoid failing of</div><div class='ctx'> 			 * ibv_devinfo and family of application when DPIs</div><div class='ctx'> 			 * are depleted.</div><div class='ctx'> 			 */</div><div class='del'>-			if (bnxt_qplib_alloc_dpi(&amp;rdev-&gt;qplib_res.dpi_tbl,</div><div class='del'>-						 &amp;ucntx-&gt;dpi, ucntx)) {</div><div class='add'>+			if (bnxt_qplib_alloc_dpi(&amp;rdev-&gt;qplib_res,</div><div class='add'>+						 &amp;ucntx-&gt;dpi, ucntx, BNXT_QPLIB_DPI_TYPE_UC)) {</div><div class='ctx'> 				rc = -ENOMEM;</div><div class='ctx'> 				goto dbfail;</div><div class='ctx'> 			}</div><div class='hunk'>@@ -596,20 +728,33 @@ int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='ctx'> 		resp.pdid = pd-&gt;qplib_pd.id;</div><div class='ctx'> 		/* Still allow mapping this DBR to the new user PD. */</div><div class='ctx'> 		resp.dpi = ucntx-&gt;dpi.dpi;</div><div class='del'>-		resp.dbr = (u64)ucntx-&gt;dpi.umdbr;</div><div class='ctx'> </div><div class='del'>-		rc = ib_copy_to_udata(udata, &amp;resp, sizeof(resp));</div><div class='add'>+		entry = bnxt_re_mmap_entry_insert(ucntx, (u64)ucntx-&gt;dpi.umdbr,</div><div class='add'>+						  BNXT_RE_MMAP_UC_DB, &amp;resp.dbr);</div><div class='add'>+</div><div class='add'>+		if (!entry) {</div><div class='add'>+			rc = -ENOMEM;</div><div class='add'>+			goto dbfail;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		pd-&gt;pd_db_mmap = &amp;entry-&gt;rdma_entry;</div><div class='add'>+</div><div class='add'>+		rc = ib_copy_to_udata(udata, &amp;resp, min(sizeof(resp), udata-&gt;outlen));</div><div class='ctx'> 		if (rc) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"Failed to copy user response\n");</div><div class='add'>+			rdma_user_mmap_entry_remove(pd-&gt;pd_db_mmap);</div><div class='add'>+			rc = -EFAULT;</div><div class='ctx'> 			goto dbfail;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (!udata)</div><div class='ctx'> 		if (bnxt_re_create_fence_mr(pd))</div><div class='del'>-			dev_warn(rdev_to_dev(rdev),</div><div class='del'>-				 "Failed to create Fence-MR\n");</div><div class='add'>+			ibdev_warn(&amp;rdev-&gt;ibdev,</div><div class='add'>+				   "Failed to create Fence-MR\n");</div><div class='add'>+	active_pds = atomic_inc_return(&amp;rdev-&gt;stats.res.pd_count);</div><div class='add'>+	if (active_pds &gt; rdev-&gt;stats.res.pd_watermark)</div><div class='add'>+		rdev-&gt;stats.res.pd_watermark = active_pds;</div><div class='add'>+</div><div class='ctx'> 	return 0;</div><div class='ctx'> dbfail:</div><div class='ctx'> 	bnxt_qplib_dealloc_pd(&amp;rdev-&gt;qplib_res, &amp;rdev-&gt;qplib_res.pd_tbl,</div><div class='hunk'>@@ -619,13 +764,24 @@ fail:</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* Address Handles */</div><div class='del'>-void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)</div><div class='add'>+int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);</div><div class='ctx'> 	struct bnxt_re_dev *rdev = ah-&gt;rdev;</div><div class='add'>+	bool block = true;</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='del'>-	bnxt_qplib_destroy_ah(&amp;rdev-&gt;qplib_res, &amp;ah-&gt;qplib_ah,</div><div class='del'>-			      !(flags &amp; RDMA_DESTROY_AH_SLEEPABLE));</div><div class='add'>+	block = !(flags &amp; RDMA_DESTROY_AH_SLEEPABLE);</div><div class='add'>+	rc = bnxt_qplib_destroy_ah(&amp;rdev-&gt;qplib_res, &amp;ah-&gt;qplib_ah, block);</div><div class='add'>+	if (BNXT_RE_CHECK_RC(rc)) {</div><div class='add'>+		if (rc == -ETIMEDOUT)</div><div class='add'>+			rc = 0;</div><div class='add'>+		else</div><div class='add'>+			goto fail;</div><div class='add'>+	}</div><div class='add'>+	atomic_dec(&amp;rdev-&gt;stats.res.ah_count);</div><div class='add'>+fail:</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype)</div><div class='hunk'>@@ -646,20 +802,23 @@ static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype)</div><div class='ctx'> 	return nw_type;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr,</div><div class='del'>-		      u32 flags, struct ib_udata *udata)</div><div class='add'>+int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,</div><div class='add'>+		      struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_pd *ib_pd = ib_ah-&gt;pd;</div><div class='ctx'> 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);</div><div class='add'>+	struct rdma_ah_attr *ah_attr = init_attr-&gt;ah_attr;</div><div class='ctx'> 	const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);</div><div class='ctx'> 	struct bnxt_re_dev *rdev = pd-&gt;rdev;</div><div class='ctx'> 	const struct ib_gid_attr *sgid_attr;</div><div class='add'>+	struct bnxt_re_gid_ctx *ctx;</div><div class='ctx'> 	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);</div><div class='add'>+	u32 active_ahs;</div><div class='ctx'> 	u8 nw_type;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='ctx'> 	if (!(rdma_ah_get_ah_flags(ah_attr) &amp; IB_AH_GRH)) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to alloc AH: GRH not set");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to alloc AH: GRH not set");</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -669,28 +828,28 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr,</div><div class='ctx'> 	/* Supply the configuration for the HW */</div><div class='ctx'> 	memcpy(ah-&gt;qplib_ah.dgid.data, grh-&gt;dgid.raw,</div><div class='ctx'> 	       sizeof(union ib_gid));</div><div class='del'>-	/*</div><div class='del'>-	 * If RoCE V2 is enabled, stack will have two entries for</div><div class='del'>-	 * each GID entry. Avoiding this duplicte entry in HW. Dividing</div><div class='del'>-	 * the GID index by 2 for RoCE V2</div><div class='add'>+	sgid_attr = grh-&gt;sgid_attr;</div><div class='add'>+	/* Get the HW context of the GID. The reference</div><div class='add'>+	 * of GID table entry is already taken by the caller.</div><div class='ctx'> 	 */</div><div class='del'>-	ah-&gt;qplib_ah.sgid_index = grh-&gt;sgid_index / 2;</div><div class='add'>+	ctx = rdma_read_gid_hw_context(sgid_attr);</div><div class='add'>+	ah-&gt;qplib_ah.sgid_index = ctx-&gt;idx;</div><div class='ctx'> 	ah-&gt;qplib_ah.host_sgid_index = grh-&gt;sgid_index;</div><div class='ctx'> 	ah-&gt;qplib_ah.traffic_class = grh-&gt;traffic_class;</div><div class='ctx'> 	ah-&gt;qplib_ah.flow_label = grh-&gt;flow_label;</div><div class='ctx'> 	ah-&gt;qplib_ah.hop_limit = grh-&gt;hop_limit;</div><div class='ctx'> 	ah-&gt;qplib_ah.sl = rdma_ah_get_sl(ah_attr);</div><div class='ctx'> </div><div class='del'>-	sgid_attr = grh-&gt;sgid_attr;</div><div class='ctx'> 	/* Get network header type for this GID */</div><div class='ctx'> 	nw_type = rdma_gid_attr_network_type(sgid_attr);</div><div class='ctx'> 	ah-&gt;qplib_ah.nw_type = bnxt_re_stack_to_dev_nw_type(nw_type);</div><div class='ctx'> </div><div class='ctx'> 	memcpy(ah-&gt;qplib_ah.dmac, ah_attr-&gt;roce.dmac, ETH_ALEN);</div><div class='ctx'> 	rc = bnxt_qplib_create_ah(&amp;rdev-&gt;qplib_res, &amp;ah-&gt;qplib_ah,</div><div class='del'>-				  !(flags &amp; RDMA_CREATE_AH_SLEEPABLE));</div><div class='add'>+				  !(init_attr-&gt;flags &amp;</div><div class='add'>+				    RDMA_CREATE_AH_SLEEPABLE));</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to allocate HW AH");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to allocate HW AH");</div><div class='ctx'> 		return rc;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -707,15 +866,13 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr,</div><div class='ctx'> 		wmb(); /* make sure cache is updated. */</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;uctx-&gt;sh_lock, flag);</div><div class='ctx'> 	}</div><div class='add'>+	active_ahs = atomic_inc_return(&amp;rdev-&gt;stats.res.ah_count);</div><div class='add'>+	if (active_ahs &gt; rdev-&gt;stats.res.ah_watermark)</div><div class='add'>+		rdev-&gt;stats.res.ah_watermark = active_ahs;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)</div><div class='del'>-{</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);</div><div class='hunk'>@@ -757,20 +914,77 @@ void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;qp-&gt;scq-&gt;cq_lock, flags);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_qp *gsi_sqp;</div><div class='add'>+	struct bnxt_re_ah *gsi_sah;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rdev = qp-&gt;rdev;</div><div class='add'>+	gsi_sqp = rdev-&gt;gsi_ctx.gsi_sqp;</div><div class='add'>+	gsi_sah = rdev-&gt;gsi_ctx.gsi_sah;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;rdev-&gt;ibdev, "Destroy the shadow AH\n");</div><div class='add'>+	bnxt_qplib_destroy_ah(&amp;rdev-&gt;qplib_res,</div><div class='add'>+			      &amp;gsi_sah-&gt;qplib_ah,</div><div class='add'>+			      true);</div><div class='add'>+	atomic_dec(&amp;rdev-&gt;stats.res.ah_count);</div><div class='add'>+	bnxt_qplib_clean_qp(&amp;qp-&gt;qplib_qp);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;rdev-&gt;ibdev, "Destroy the shadow QP\n");</div><div class='add'>+	rc = bnxt_qplib_destroy_qp(&amp;rdev-&gt;qplib_res, &amp;gsi_sqp-&gt;qplib_qp);</div><div class='add'>+	if (rc)</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Destroy Shadow QP failed");</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_free_qp_res(&amp;rdev-&gt;qplib_res, &amp;gsi_sqp-&gt;qplib_qp);</div><div class='add'>+</div><div class='add'>+	/* remove from active qp list */</div><div class='add'>+	mutex_lock(&amp;rdev-&gt;qp_lock);</div><div class='add'>+	list_del(&amp;gsi_sqp-&gt;list);</div><div class='add'>+	mutex_unlock(&amp;rdev-&gt;qp_lock);</div><div class='add'>+	atomic_dec(&amp;rdev-&gt;stats.res.qp_count);</div><div class='add'>+</div><div class='add'>+	kfree(rdev-&gt;gsi_ctx.sqp_tbl);</div><div class='add'>+	kfree(gsi_sah);</div><div class='add'>+	kfree(gsi_sqp);</div><div class='add'>+	rdev-&gt;gsi_ctx.gsi_sqp = NULL;</div><div class='add'>+	rdev-&gt;gsi_ctx.gsi_sah = NULL;</div><div class='add'>+	rdev-&gt;gsi_ctx.sqp_tbl = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_del_unique_gid(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (!rdev-&gt;rcfw.roce_mirror)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	rc = bnxt_qplib_del_sgid(&amp;rdev-&gt;qplib_res.sgid_tbl,</div><div class='add'>+				 (struct bnxt_qplib_gid *)&amp;rdev-&gt;ugid,</div><div class='add'>+				 0xFFFF, true);</div><div class='add'>+	if (rc)</div><div class='add'>+		dev_err(rdev_to_dev(rdev), "Failed to delete unique GID, rc: %d\n", rc);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /* Queue Pairs */</div><div class='ctx'> int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);</div><div class='add'>+	struct bnxt_qplib_qp *qplib_qp = &amp;qp-&gt;qplib_qp;</div><div class='ctx'> 	struct bnxt_re_dev *rdev = qp-&gt;rdev;</div><div class='add'>+	struct bnxt_qplib_nq *scq_nq = NULL;</div><div class='add'>+	struct bnxt_qplib_nq *rcq_nq = NULL;</div><div class='ctx'> 	unsigned int flags;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='add'>+	bnxt_re_debug_rem_qpinfo(rdev, qp);</div><div class='add'>+</div><div class='ctx'> 	bnxt_qplib_flush_cqn_wq(&amp;qp-&gt;qplib_qp);</div><div class='add'>+</div><div class='ctx'> 	rc = bnxt_qplib_destroy_qp(&amp;rdev-&gt;qplib_res, &amp;qp-&gt;qplib_qp);</div><div class='del'>-	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to destroy HW QP");</div><div class='del'>-		return rc;</div><div class='del'>-	}</div><div class='add'>+	if (rc)</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to destroy HW QP");</div><div class='ctx'> </div><div class='ctx'> 	if (rdma_is_kernel_res(&amp;qp-&gt;ib_qp.res)) {</div><div class='ctx'> 		flags = bnxt_re_lock_cqs(qp);</div><div class='hunk'>@@ -780,41 +994,33 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)</div><div class='ctx'> </div><div class='ctx'> 	bnxt_qplib_free_qp_res(&amp;rdev-&gt;qplib_res, &amp;qp-&gt;qplib_qp);</div><div class='ctx'> </div><div class='del'>-	if (ib_qp-&gt;qp_type == IB_QPT_GSI &amp;&amp; rdev-&gt;qp1_sqp) {</div><div class='del'>-		bnxt_qplib_destroy_ah(&amp;rdev-&gt;qplib_res, &amp;rdev-&gt;sqp_ah-&gt;qplib_ah,</div><div class='del'>-				      false);</div><div class='del'>-</div><div class='del'>-		bnxt_qplib_clean_qp(&amp;qp-&gt;qplib_qp);</div><div class='del'>-		rc = bnxt_qplib_destroy_qp(&amp;rdev-&gt;qplib_res,</div><div class='del'>-					   &amp;rdev-&gt;qp1_sqp-&gt;qplib_qp);</div><div class='del'>-		if (rc) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"Failed to destroy Shadow QP");</div><div class='del'>-			return rc;</div><div class='del'>-		}</div><div class='del'>-		bnxt_qplib_free_qp_res(&amp;rdev-&gt;qplib_res,</div><div class='del'>-				       &amp;rdev-&gt;qp1_sqp-&gt;qplib_qp);</div><div class='del'>-		mutex_lock(&amp;rdev-&gt;qp_lock);</div><div class='del'>-		list_del(&amp;rdev-&gt;qp1_sqp-&gt;list);</div><div class='del'>-		atomic_dec(&amp;rdev-&gt;qp_count);</div><div class='del'>-		mutex_unlock(&amp;rdev-&gt;qp_lock);</div><div class='del'>-</div><div class='del'>-		kfree(rdev-&gt;sqp_ah);</div><div class='del'>-		kfree(rdev-&gt;qp1_sqp);</div><div class='del'>-		rdev-&gt;qp1_sqp = NULL;</div><div class='del'>-		rdev-&gt;sqp_ah = NULL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!IS_ERR_OR_NULL(qp-&gt;rumem))</div><div class='del'>-		ib_umem_release(qp-&gt;rumem);</div><div class='del'>-	if (!IS_ERR_OR_NULL(qp-&gt;sumem))</div><div class='del'>-		ib_umem_release(qp-&gt;sumem);</div><div class='add'>+	if (ib_qp-&gt;qp_type == IB_QPT_GSI &amp;&amp; rdev-&gt;gsi_ctx.gsi_sqp)</div><div class='add'>+		bnxt_re_destroy_gsi_sqp(qp);</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;rdev-&gt;qp_lock);</div><div class='ctx'> 	list_del(&amp;qp-&gt;list);</div><div class='del'>-	atomic_dec(&amp;rdev-&gt;qp_count);</div><div class='ctx'> 	mutex_unlock(&amp;rdev-&gt;qp_lock);</div><div class='del'>-	kfree(qp);</div><div class='add'>+	atomic_dec(&amp;rdev-&gt;stats.res.qp_count);</div><div class='add'>+	if (qp-&gt;qplib_qp.type == CMDQ_CREATE_QP_TYPE_RC)</div><div class='add'>+		atomic_dec(&amp;rdev-&gt;stats.res.rc_qp_count);</div><div class='add'>+	else if (qp-&gt;qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD)</div><div class='add'>+		atomic_dec(&amp;rdev-&gt;stats.res.ud_qp_count);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;qplib_qp.type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE)</div><div class='add'>+		bnxt_re_del_unique_gid(rdev);</div><div class='add'>+</div><div class='add'>+	ib_umem_release(qp-&gt;rumem);</div><div class='add'>+	ib_umem_release(qp-&gt;sumem);</div><div class='add'>+</div><div class='add'>+	/* Flush all the entries of notification queue associated with</div><div class='add'>+	 * given qp.</div><div class='add'>+	 */</div><div class='add'>+	scq_nq = qplib_qp-&gt;scq-&gt;nq;</div><div class='add'>+	rcq_nq = qplib_qp-&gt;rcq-&gt;nq;</div><div class='add'>+	bnxt_re_synchronize_nq(scq_nq);</div><div class='add'>+	if (scq_nq != rcq_nq)</div><div class='add'>+		bnxt_re_synchronize_nq(rcq_nq);</div><div class='add'>+</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -827,54 +1033,124 @@ static u8 __from_ib_qp_type(enum ib_qp_type type)</div><div class='ctx'> 		return CMDQ_CREATE_QP_TYPE_RC;</div><div class='ctx'> 	case IB_QPT_UD:</div><div class='ctx'> 		return CMDQ_CREATE_QP_TYPE_UD;</div><div class='add'>+	case IB_QPT_RAW_PACKET:</div><div class='add'>+		return CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE;</div><div class='ctx'> 	default:</div><div class='ctx'> 		return IB_QPT_MAX;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static u16 bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp *qplqp,</div><div class='add'>+				   int rsge, int max)</div><div class='add'>+{</div><div class='add'>+	if (qplqp-&gt;wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)</div><div class='add'>+		rsge = max;</div><div class='add'>+	return bnxt_re_get_rwqe_size(rsge);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u16 bnxt_re_get_wqe_size(int ilsize, int nsge)</div><div class='add'>+{</div><div class='add'>+	u16 wqe_size, calc_ils;</div><div class='add'>+</div><div class='add'>+	wqe_size = bnxt_re_get_swqe_size(nsge);</div><div class='add'>+	if (ilsize) {</div><div class='add'>+		calc_ils = sizeof(struct sq_send_hdr) + ilsize;</div><div class='add'>+		wqe_size = max_t(u16, calc_ils, wqe_size);</div><div class='add'>+		wqe_size = ALIGN(wqe_size, sizeof(struct sq_send_hdr));</div><div class='add'>+	}</div><div class='add'>+	return wqe_size;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,</div><div class='add'>+				   struct ib_qp_init_attr *init_attr)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_dev_attr *dev_attr;</div><div class='add'>+	struct bnxt_qplib_qp *qplqp;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct bnxt_qplib_q *sq;</div><div class='add'>+	int align, ilsize;</div><div class='add'>+</div><div class='add'>+	rdev = qp-&gt;rdev;</div><div class='add'>+	qplqp = &amp;qp-&gt;qplib_qp;</div><div class='add'>+	sq = &amp;qplqp-&gt;sq;</div><div class='add'>+	dev_attr = rdev-&gt;dev_attr;</div><div class='add'>+</div><div class='add'>+	align = sizeof(struct sq_send_hdr);</div><div class='add'>+	ilsize = ALIGN(init_attr-&gt;cap.max_inline_data, align);</div><div class='add'>+</div><div class='add'>+	/* For gen p4 and gen p5 fixed wqe compatibility mode</div><div class='add'>+	 * wqe size is fixed to 128 bytes - ie 6 SGEs</div><div class='add'>+	 */</div><div class='add'>+	if (qplqp-&gt;wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) {</div><div class='add'>+		sq-&gt;wqe_size = bnxt_re_get_swqe_size(BNXT_STATIC_MAX_SGE);</div><div class='add'>+		sq-&gt;max_sge = BNXT_STATIC_MAX_SGE;</div><div class='add'>+	} else {</div><div class='add'>+		sq-&gt;wqe_size = bnxt_re_get_wqe_size(ilsize, sq-&gt;max_sge);</div><div class='add'>+		if (sq-&gt;wqe_size &gt; bnxt_re_get_swqe_size(dev_attr-&gt;max_qp_sges))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (init_attr-&gt;cap.max_inline_data) {</div><div class='add'>+		qplqp-&gt;max_inline_data = sq-&gt;wqe_size -</div><div class='add'>+			sizeof(struct sq_send_hdr);</div><div class='add'>+		init_attr-&gt;cap.max_inline_data = qplqp-&gt;max_inline_data;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,</div><div class='del'>-				struct bnxt_re_qp *qp, struct ib_udata *udata)</div><div class='add'>+				struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx,</div><div class='add'>+				struct bnxt_re_qp_req *ureq)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_re_qp_req ureq;</div><div class='del'>-	struct bnxt_qplib_qp *qplib_qp = &amp;qp-&gt;qplib_qp;</div><div class='del'>-	struct ib_umem *umem;</div><div class='add'>+	struct bnxt_qplib_qp *qplib_qp;</div><div class='ctx'> 	int bytes = 0, psn_sz;</div><div class='del'>-	struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context(</div><div class='del'>-		udata, struct bnxt_re_ucontext, ib_uctx);</div><div class='add'>+	struct ib_umem *umem;</div><div class='add'>+	int psn_nume;</div><div class='ctx'> </div><div class='del'>-	if (ib_copy_from_udata(&amp;ureq, udata, sizeof(ureq)))</div><div class='del'>-		return -EFAULT;</div><div class='add'>+	qplib_qp = &amp;qp-&gt;qplib_qp;</div><div class='ctx'> </div><div class='del'>-	bytes = (qplib_qp-&gt;sq.max_wqe * BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);</div><div class='add'>+	bytes = (qplib_qp-&gt;sq.max_wqe * qplib_qp-&gt;sq.wqe_size);</div><div class='ctx'> 	/* Consider mapping PSN search memory only for RC QPs. */</div><div class='ctx'> 	if (qplib_qp-&gt;type == CMDQ_CREATE_QP_TYPE_RC) {</div><div class='del'>-		psn_sz = bnxt_qplib_is_chip_gen_p5(&amp;rdev-&gt;chip_ctx) ?</div><div class='del'>-					sizeof(struct sq_psn_search_ext) :</div><div class='del'>-					sizeof(struct sq_psn_search);</div><div class='del'>-		bytes += (qplib_qp-&gt;sq.max_wqe * psn_sz);</div><div class='add'>+		psn_sz = bnxt_qplib_is_chip_gen_p5_p7(rdev-&gt;chip_ctx) ?</div><div class='add'>+						   sizeof(struct sq_psn_search_ext) :</div><div class='add'>+						   sizeof(struct sq_psn_search);</div><div class='add'>+		if (cntx &amp;&amp; bnxt_re_is_var_size_supported(rdev, cntx)) {</div><div class='add'>+			psn_nume = ureq-&gt;sq_slots;</div><div class='add'>+		} else {</div><div class='add'>+			psn_nume = (qplib_qp-&gt;wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?</div><div class='add'>+			qplib_qp-&gt;sq.max_wqe : ((qplib_qp-&gt;sq.max_wqe * qplib_qp-&gt;sq.wqe_size) /</div><div class='add'>+				 sizeof(struct bnxt_qplib_sge));</div><div class='add'>+		}</div><div class='add'>+		if (_is_host_msn_table(rdev-&gt;qplib_res.dattr-&gt;dev_cap_flags2))</div><div class='add'>+			psn_nume = roundup_pow_of_two(psn_nume);</div><div class='add'>+		bytes += (psn_nume * psn_sz);</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='ctx'> 	bytes = PAGE_ALIGN(bytes);</div><div class='del'>-	umem = ib_umem_get(udata, ureq.qpsva, bytes, IB_ACCESS_LOCAL_WRITE, 1);</div><div class='add'>+	umem = ib_umem_get(&amp;rdev-&gt;ibdev, ureq-&gt;qpsva, bytes,</div><div class='add'>+			   IB_ACCESS_LOCAL_WRITE);</div><div class='ctx'> 	if (IS_ERR(umem))</div><div class='ctx'> 		return PTR_ERR(umem);</div><div class='ctx'> </div><div class='ctx'> 	qp-&gt;sumem = umem;</div><div class='del'>-	qplib_qp-&gt;sq.sg_info.sglist = umem-&gt;sg_head.sgl;</div><div class='del'>-	qplib_qp-&gt;sq.sg_info.npages = ib_umem_num_pages(umem);</div><div class='del'>-	qplib_qp-&gt;sq.sg_info.nmap = umem-&gt;nmap;</div><div class='del'>-	qplib_qp-&gt;qp_handle = ureq.qp_handle;</div><div class='add'>+	qplib_qp-&gt;sq.sg_info.umem = umem;</div><div class='add'>+	qplib_qp-&gt;sq.sg_info.pgsize = PAGE_SIZE;</div><div class='add'>+	qplib_qp-&gt;sq.sg_info.pgshft = PAGE_SHIFT;</div><div class='add'>+	qplib_qp-&gt;qp_handle = ureq-&gt;qp_handle;</div><div class='ctx'> </div><div class='ctx'> 	if (!qp-&gt;qplib_qp.srq) {</div><div class='del'>-		bytes = (qplib_qp-&gt;rq.max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);</div><div class='add'>+		bytes = (qplib_qp-&gt;rq.max_wqe * qplib_qp-&gt;rq.wqe_size);</div><div class='ctx'> 		bytes = PAGE_ALIGN(bytes);</div><div class='del'>-		umem = ib_umem_get(udata, ureq.qprva, bytes,</div><div class='del'>-				   IB_ACCESS_LOCAL_WRITE, 1);</div><div class='add'>+		umem = ib_umem_get(&amp;rdev-&gt;ibdev, ureq-&gt;qprva, bytes,</div><div class='add'>+				   IB_ACCESS_LOCAL_WRITE);</div><div class='ctx'> 		if (IS_ERR(umem))</div><div class='ctx'> 			goto rqfail;</div><div class='ctx'> 		qp-&gt;rumem = umem;</div><div class='del'>-		qplib_qp-&gt;rq.sg_info.sglist = umem-&gt;sg_head.sgl;</div><div class='del'>-		qplib_qp-&gt;rq.sg_info.npages = ib_umem_num_pages(umem);</div><div class='del'>-		qplib_qp-&gt;rq.sg_info.nmap = umem-&gt;nmap;</div><div class='add'>+		qplib_qp-&gt;rq.sg_info.umem = umem;</div><div class='add'>+		qplib_qp-&gt;rq.sg_info.pgsize = PAGE_SIZE;</div><div class='add'>+		qplib_qp-&gt;rq.sg_info.pgshft = PAGE_SHIFT;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	qplib_qp-&gt;dpi = &amp;cntx-&gt;dpi;</div><div class='hunk'>@@ -922,10 +1198,11 @@ static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah</div><div class='ctx'> </div><div class='ctx'> 	rc = bnxt_qplib_create_ah(&amp;rdev-&gt;qplib_res, &amp;ah-&gt;qplib_ah, false);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev),</div><div class='del'>-			"Failed to allocate HW AH for Shadow QP");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to allocate HW AH for Shadow QP");</div><div class='ctx'> 		goto fail;</div><div class='ctx'> 	}</div><div class='add'>+	atomic_inc(&amp;rdev-&gt;stats.res.ah_count);</div><div class='ctx'> </div><div class='ctx'> 	return ah;</div><div class='ctx'> </div><div class='hunk'>@@ -960,18 +1237,26 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp</div><div class='ctx'> 	qp-&gt;qplib_qp.sig_type = true;</div><div class='ctx'> </div><div class='ctx'> 	/* Shadow QP SQ depth should be same as QP1 RQ depth */</div><div class='add'>+	qp-&gt;qplib_qp.sq.wqe_size = bnxt_re_get_wqe_size(0, 6);</div><div class='ctx'> 	qp-&gt;qplib_qp.sq.max_wqe = qp1_qp-&gt;rq.max_wqe;</div><div class='add'>+	qp-&gt;qplib_qp.sq.max_sw_wqe = qp1_qp-&gt;rq.max_wqe;</div><div class='ctx'> 	qp-&gt;qplib_qp.sq.max_sge = 2;</div><div class='ctx'> 	/* Q full delta can be 1 since it is internal QP */</div><div class='ctx'> 	qp-&gt;qplib_qp.sq.q_full_delta = 1;</div><div class='add'>+	qp-&gt;qplib_qp.sq.sg_info.pgsize = PAGE_SIZE;</div><div class='add'>+	qp-&gt;qplib_qp.sq.sg_info.pgshft = PAGE_SHIFT;</div><div class='ctx'> </div><div class='ctx'> 	qp-&gt;qplib_qp.scq = qp1_qp-&gt;scq;</div><div class='ctx'> 	qp-&gt;qplib_qp.rcq = qp1_qp-&gt;rcq;</div><div class='ctx'> </div><div class='add'>+	qp-&gt;qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(6);</div><div class='ctx'> 	qp-&gt;qplib_qp.rq.max_wqe = qp1_qp-&gt;rq.max_wqe;</div><div class='add'>+	qp-&gt;qplib_qp.rq.max_sw_wqe = qp1_qp-&gt;rq.max_wqe;</div><div class='ctx'> 	qp-&gt;qplib_qp.rq.max_sge = qp1_qp-&gt;rq.max_sge;</div><div class='ctx'> 	/* Q full delta can be 1 since it is internal QP */</div><div class='ctx'> 	qp-&gt;qplib_qp.rq.q_full_delta = 1;</div><div class='add'>+	qp-&gt;qplib_qp.rq.sg_info.pgsize = PAGE_SIZE;</div><div class='add'>+	qp-&gt;qplib_qp.rq.sg_info.pgshft = PAGE_SHIFT;</div><div class='ctx'> </div><div class='ctx'> 	qp-&gt;qplib_qp.mtu = qp1_qp-&gt;mtu;</div><div class='ctx'> </div><div class='hunk'>@@ -983,13 +1268,11 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='del'>-	rdev-&gt;sqp_id = qp-&gt;qplib_qp.id;</div><div class='del'>-</div><div class='ctx'> 	spin_lock_init(&amp;qp-&gt;sq_lock);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;qp-&gt;list);</div><div class='ctx'> 	mutex_lock(&amp;rdev-&gt;qp_lock);</div><div class='ctx'> 	list_add_tail(&amp;qp-&gt;list, &amp;rdev-&gt;qp_list);</div><div class='del'>-	atomic_inc(&amp;rdev-&gt;qp_count);</div><div class='add'>+	atomic_inc(&amp;rdev-&gt;stats.res.qp_count);</div><div class='ctx'> 	mutex_unlock(&amp;rdev-&gt;qp_lock);</div><div class='ctx'> 	return qp;</div><div class='ctx'> fail:</div><div class='hunk'>@@ -997,219 +1280,464 @@ fail:</div><div class='ctx'> 	return NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,</div><div class='del'>-				struct ib_qp_init_attr *qp_init_attr,</div><div class='del'>-				struct ib_udata *udata)</div><div class='add'>+static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,</div><div class='add'>+				struct ib_qp_init_attr *init_attr,</div><div class='add'>+				struct bnxt_re_ucontext *uctx)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);</div><div class='del'>-	struct bnxt_re_dev *rdev = pd-&gt;rdev;</div><div class='del'>-	struct bnxt_qplib_dev_attr *dev_attr = &amp;rdev-&gt;dev_attr;</div><div class='del'>-	struct bnxt_re_qp *qp;</div><div class='del'>-	struct bnxt_re_cq *cq;</div><div class='del'>-	struct bnxt_re_srq *srq;</div><div class='del'>-	int rc, entries;</div><div class='add'>+	struct bnxt_qplib_dev_attr *dev_attr;</div><div class='add'>+	struct bnxt_qplib_qp *qplqp;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct bnxt_qplib_q *rq;</div><div class='add'>+	int entries;</div><div class='ctx'> </div><div class='del'>-	if ((qp_init_attr-&gt;cap.max_send_wr &gt; dev_attr-&gt;max_qp_wqes) ||</div><div class='del'>-	    (qp_init_attr-&gt;cap.max_recv_wr &gt; dev_attr-&gt;max_qp_wqes) ||</div><div class='del'>-	    (qp_init_attr-&gt;cap.max_send_sge &gt; dev_attr-&gt;max_qp_sges) ||</div><div class='del'>-	    (qp_init_attr-&gt;cap.max_recv_sge &gt; dev_attr-&gt;max_qp_sges) ||</div><div class='del'>-	    (qp_init_attr-&gt;cap.max_inline_data &gt; dev_attr-&gt;max_inline_data))</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+	rdev = qp-&gt;rdev;</div><div class='add'>+	qplqp = &amp;qp-&gt;qplib_qp;</div><div class='add'>+	rq = &amp;qplqp-&gt;rq;</div><div class='add'>+	dev_attr = rdev-&gt;dev_attr;</div><div class='ctx'> </div><div class='del'>-	qp = kzalloc(sizeof(*qp), GFP_KERNEL);</div><div class='del'>-	if (!qp)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='add'>+	if (init_attr-&gt;srq) {</div><div class='add'>+		struct bnxt_re_srq *srq;</div><div class='ctx'> </div><div class='del'>-	qp-&gt;rdev = rdev;</div><div class='del'>-	ether_addr_copy(qp-&gt;qplib_qp.smac, rdev-&gt;netdev-&gt;dev_addr);</div><div class='del'>-	qp-&gt;qplib_qp.pd = &amp;pd-&gt;qplib_pd;</div><div class='del'>-	qp-&gt;qplib_qp.qp_handle = (u64)(unsigned long)(&amp;qp-&gt;qplib_qp);</div><div class='del'>-	qp-&gt;qplib_qp.type = __from_ib_qp_type(qp_init_attr-&gt;qp_type);</div><div class='del'>-</div><div class='del'>-	if (qp_init_attr-&gt;qp_type == IB_QPT_GSI &amp;&amp;</div><div class='del'>-	    bnxt_qplib_is_chip_gen_p5(&amp;rdev-&gt;chip_ctx))</div><div class='del'>-		qp-&gt;qplib_qp.type = CMDQ_CREATE_QP_TYPE_GSI;</div><div class='del'>-	if (qp-&gt;qplib_qp.type == IB_QPT_MAX) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported",</div><div class='del'>-			qp-&gt;qplib_qp.type);</div><div class='del'>-		rc = -EINVAL;</div><div class='del'>-		goto fail;</div><div class='add'>+		srq = container_of(init_attr-&gt;srq, struct bnxt_re_srq, ib_srq);</div><div class='add'>+		qplqp-&gt;srq = &amp;srq-&gt;qplib_srq;</div><div class='add'>+		rq-&gt;max_wqe = 0;</div><div class='add'>+	} else {</div><div class='add'>+		rq-&gt;max_sge = init_attr-&gt;cap.max_recv_sge;</div><div class='add'>+		if (rq-&gt;max_sge &gt; dev_attr-&gt;max_qp_sges)</div><div class='add'>+			rq-&gt;max_sge = dev_attr-&gt;max_qp_sges;</div><div class='add'>+		init_attr-&gt;cap.max_recv_sge = rq-&gt;max_sge;</div><div class='add'>+		rq-&gt;wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq-&gt;max_sge,</div><div class='add'>+						       dev_attr-&gt;max_qp_sges);</div><div class='add'>+		/* Allocate 1 more than what's provided so posting max doesn't</div><div class='add'>+		 * mean empty.</div><div class='add'>+		 */</div><div class='add'>+		entries = bnxt_re_init_depth(init_attr-&gt;cap.max_recv_wr + 1, uctx);</div><div class='add'>+		rq-&gt;max_wqe = min_t(u32, entries, dev_attr-&gt;max_qp_wqes + 1);</div><div class='add'>+		rq-&gt;max_sw_wqe = rq-&gt;max_wqe;</div><div class='add'>+		rq-&gt;q_full_delta = 0;</div><div class='add'>+		rq-&gt;sg_info.pgsize = PAGE_SIZE;</div><div class='add'>+		rq-&gt;sg_info.pgshft = PAGE_SHIFT;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	qp-&gt;qplib_qp.max_inline_data = qp_init_attr-&gt;cap.max_inline_data;</div><div class='del'>-	qp-&gt;qplib_qp.sig_type = ((qp_init_attr-&gt;sq_sig_type ==</div><div class='del'>-				  IB_SIGNAL_ALL_WR) ? true : false);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_dev_attr *dev_attr;</div><div class='add'>+	struct bnxt_qplib_qp *qplqp;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='ctx'> </div><div class='del'>-	qp-&gt;qplib_qp.sq.max_sge = qp_init_attr-&gt;cap.max_send_sge;</div><div class='del'>-	if (qp-&gt;qplib_qp.sq.max_sge &gt; dev_attr-&gt;max_qp_sges)</div><div class='del'>-		qp-&gt;qplib_qp.sq.max_sge = dev_attr-&gt;max_qp_sges;</div><div class='add'>+	rdev = qp-&gt;rdev;</div><div class='add'>+	qplqp = &amp;qp-&gt;qplib_qp;</div><div class='add'>+	dev_attr = rdev-&gt;dev_attr;</div><div class='ctx'> </div><div class='del'>-	if (qp_init_attr-&gt;send_cq) {</div><div class='del'>-		cq = container_of(qp_init_attr-&gt;send_cq, struct bnxt_re_cq,</div><div class='del'>-				  ib_cq);</div><div class='del'>-		if (!cq) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev), "Send CQ not found");</div><div class='del'>-			rc = -EINVAL;</div><div class='del'>-			goto fail;</div><div class='add'>+	if (!bnxt_qplib_is_chip_gen_p5_p7(rdev-&gt;chip_ctx)) {</div><div class='add'>+		qplqp-&gt;rq.max_sge = dev_attr-&gt;max_qp_sges;</div><div class='add'>+		if (qplqp-&gt;rq.max_sge &gt; dev_attr-&gt;max_qp_sges)</div><div class='add'>+			qplqp-&gt;rq.max_sge = dev_attr-&gt;max_qp_sges;</div><div class='add'>+		qplqp-&gt;rq.max_sge = 6;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,</div><div class='add'>+				struct ib_qp_init_attr *init_attr,</div><div class='add'>+				struct bnxt_re_ucontext *uctx,</div><div class='add'>+				struct bnxt_re_qp_req *ureq)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_dev_attr *dev_attr;</div><div class='add'>+	struct bnxt_qplib_qp *qplqp;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct bnxt_qplib_q *sq;</div><div class='add'>+	int diff = 0;</div><div class='add'>+	int entries;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rdev = qp-&gt;rdev;</div><div class='add'>+	qplqp = &amp;qp-&gt;qplib_qp;</div><div class='add'>+	sq = &amp;qplqp-&gt;sq;</div><div class='add'>+	dev_attr = rdev-&gt;dev_attr;</div><div class='add'>+</div><div class='add'>+	sq-&gt;max_sge = init_attr-&gt;cap.max_send_sge;</div><div class='add'>+	entries = init_attr-&gt;cap.max_send_wr;</div><div class='add'>+	if (uctx &amp;&amp; qplqp-&gt;wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) {</div><div class='add'>+		sq-&gt;max_wqe = ureq-&gt;sq_slots;</div><div class='add'>+		sq-&gt;max_sw_wqe = ureq-&gt;sq_slots;</div><div class='add'>+		sq-&gt;wqe_size = sizeof(struct sq_sge);</div><div class='add'>+	} else {</div><div class='add'>+		if (sq-&gt;max_sge &gt; dev_attr-&gt;max_qp_sges) {</div><div class='add'>+			sq-&gt;max_sge = dev_attr-&gt;max_qp_sges;</div><div class='add'>+			init_attr-&gt;cap.max_send_sge = sq-&gt;max_sge;</div><div class='ctx'> 		}</div><div class='del'>-		qp-&gt;qplib_qp.scq = &amp;cq-&gt;qplib_cq;</div><div class='add'>+</div><div class='add'>+		rc = bnxt_re_setup_swqe_size(qp, init_attr);</div><div class='add'>+		if (rc)</div><div class='add'>+			return rc;</div><div class='add'>+</div><div class='add'>+		/* Allocate 128 + 1 more than what's provided */</div><div class='add'>+		diff = (qplqp-&gt;wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) ?</div><div class='add'>+			0 : BNXT_QPLIB_RESERVED_QP_WRS;</div><div class='add'>+		entries = bnxt_re_init_depth(entries + diff + 1, uctx);</div><div class='add'>+		sq-&gt;max_wqe = min_t(u32, entries, dev_attr-&gt;max_qp_wqes + diff + 1);</div><div class='add'>+		if (qplqp-&gt;wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)</div><div class='add'>+			sq-&gt;max_sw_wqe = bnxt_qplib_get_depth(sq, qplqp-&gt;wqe_mode, true);</div><div class='add'>+		else</div><div class='add'>+			sq-&gt;max_sw_wqe = sq-&gt;max_wqe;</div><div class='add'>+</div><div class='add'>+	}</div><div class='add'>+	sq-&gt;q_full_delta = diff + 1;</div><div class='add'>+	/*</div><div class='add'>+	 * Reserving one slot for Phantom WQE. Application can</div><div class='add'>+	 * post one extra entry in this case. But allowing this to avoid</div><div class='add'>+	 * unexpected Queue full condition</div><div class='add'>+	 */</div><div class='add'>+	qplqp-&gt;sq.q_full_delta -= 1;</div><div class='add'>+	qplqp-&gt;sq.sg_info.pgsize = PAGE_SIZE;</div><div class='add'>+	qplqp-&gt;sq.sg_info.pgshft = PAGE_SHIFT;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,</div><div class='add'>+				       struct ib_qp_init_attr *init_attr,</div><div class='add'>+				       struct bnxt_re_ucontext *uctx)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_dev_attr *dev_attr;</div><div class='add'>+	struct bnxt_qplib_qp *qplqp;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	int entries;</div><div class='add'>+</div><div class='add'>+	rdev = qp-&gt;rdev;</div><div class='add'>+	qplqp = &amp;qp-&gt;qplib_qp;</div><div class='add'>+	dev_attr = rdev-&gt;dev_attr;</div><div class='add'>+</div><div class='add'>+	if (!bnxt_qplib_is_chip_gen_p5_p7(rdev-&gt;chip_ctx)) {</div><div class='add'>+		entries = bnxt_re_init_depth(init_attr-&gt;cap.max_send_wr + 1, uctx);</div><div class='add'>+		qplqp-&gt;sq.max_wqe = min_t(u32, entries,</div><div class='add'>+					  dev_attr-&gt;max_qp_wqes + 1);</div><div class='add'>+		qplqp-&gt;sq.q_full_delta = qplqp-&gt;sq.max_wqe -</div><div class='add'>+			init_attr-&gt;cap.max_send_wr;</div><div class='add'>+		qplqp-&gt;sq.max_sge++; /* Need one extra sge to put UD header */</div><div class='add'>+		if (qplqp-&gt;sq.max_sge &gt; dev_attr-&gt;max_qp_sges)</div><div class='add'>+			qplqp-&gt;sq.max_sge = dev_attr-&gt;max_qp_sges;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,</div><div class='add'>+				struct ib_qp_init_attr *init_attr)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_chip_ctx *chip_ctx;</div><div class='add'>+	int qptype;</div><div class='add'>+</div><div class='add'>+	chip_ctx = rdev-&gt;chip_ctx;</div><div class='add'>+</div><div class='add'>+	qptype = __from_ib_qp_type(init_attr-&gt;qp_type);</div><div class='add'>+	if (qptype == IB_QPT_MAX) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "QP type 0x%x not supported", qptype);</div><div class='add'>+		qptype = -EOPNOTSUPP;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (bnxt_qplib_is_chip_gen_p5_p7(chip_ctx) &amp;&amp;</div><div class='add'>+	    init_attr-&gt;qp_type == IB_QPT_GSI)</div><div class='add'>+		qptype = CMDQ_CREATE_QP_TYPE_GSI;</div><div class='add'>+out:</div><div class='add'>+	return qptype;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,</div><div class='add'>+				struct ib_qp_init_attr *init_attr,</div><div class='add'>+				struct bnxt_re_ucontext *uctx,</div><div class='add'>+				struct bnxt_re_qp_req *ureq)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_dev_attr *dev_attr;</div><div class='add'>+	struct bnxt_qplib_qp *qplqp;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct bnxt_re_cq *cq;</div><div class='add'>+	int rc = 0, qptype;</div><div class='add'>+</div><div class='add'>+	rdev = qp-&gt;rdev;</div><div class='add'>+	qplqp = &amp;qp-&gt;qplib_qp;</div><div class='add'>+	dev_attr = rdev-&gt;dev_attr;</div><div class='add'>+</div><div class='add'>+	/* Setup misc params */</div><div class='add'>+	ether_addr_copy(qplqp-&gt;smac, rdev-&gt;netdev-&gt;dev_addr);</div><div class='add'>+	qplqp-&gt;pd = &amp;pd-&gt;qplib_pd;</div><div class='add'>+	qplqp-&gt;qp_handle = (u64)qplqp;</div><div class='add'>+	qplqp-&gt;max_inline_data = init_attr-&gt;cap.max_inline_data;</div><div class='add'>+	qplqp-&gt;sig_type = init_attr-&gt;sq_sig_type == IB_SIGNAL_ALL_WR;</div><div class='add'>+	qptype = bnxt_re_init_qp_type(rdev, init_attr);</div><div class='add'>+	if (qptype &lt; 0) {</div><div class='add'>+		rc = qptype;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+	qplqp-&gt;type = (u8)qptype;</div><div class='add'>+	qplqp-&gt;wqe_mode = bnxt_re_is_var_size_supported(rdev, uctx);</div><div class='add'>+	if (init_attr-&gt;qp_type == IB_QPT_RC) {</div><div class='add'>+		qplqp-&gt;max_rd_atomic = dev_attr-&gt;max_qp_rd_atom;</div><div class='add'>+		qplqp-&gt;max_dest_rd_atomic = dev_attr-&gt;max_qp_init_rd_atom;</div><div class='add'>+	}</div><div class='add'>+	qplqp-&gt;mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev-&gt;netdev-&gt;mtu));</div><div class='add'>+	qplqp-&gt;dpi = &amp;rdev-&gt;dpi_privileged; /* Doorbell page */</div><div class='add'>+	if (init_attr-&gt;create_flags) {</div><div class='add'>+		ibdev_dbg(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "QP create flags 0x%x not supported",</div><div class='add'>+			  init_attr-&gt;create_flags);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Setup CQs */</div><div class='add'>+	if (init_attr-&gt;send_cq) {</div><div class='add'>+		cq = container_of(init_attr-&gt;send_cq, struct bnxt_re_cq, ib_cq);</div><div class='add'>+		qplqp-&gt;scq = &amp;cq-&gt;qplib_cq;</div><div class='ctx'> 		qp-&gt;scq = cq;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (qp_init_attr-&gt;recv_cq) {</div><div class='del'>-		cq = container_of(qp_init_attr-&gt;recv_cq, struct bnxt_re_cq,</div><div class='del'>-				  ib_cq);</div><div class='del'>-		if (!cq) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev), "Receive CQ not found");</div><div class='del'>-			rc = -EINVAL;</div><div class='del'>-			goto fail;</div><div class='del'>-		}</div><div class='del'>-		qp-&gt;qplib_qp.rcq = &amp;cq-&gt;qplib_cq;</div><div class='add'>+	if (init_attr-&gt;recv_cq) {</div><div class='add'>+		cq = container_of(init_attr-&gt;recv_cq, struct bnxt_re_cq, ib_cq);</div><div class='add'>+		qplqp-&gt;rcq = &amp;cq-&gt;qplib_cq;</div><div class='ctx'> 		qp-&gt;rcq = cq;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (qp_init_attr-&gt;srq) {</div><div class='del'>-		srq = container_of(qp_init_attr-&gt;srq, struct bnxt_re_srq,</div><div class='del'>-				   ib_srq);</div><div class='del'>-		if (!srq) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev), "SRQ not found");</div><div class='del'>-			rc = -EINVAL;</div><div class='del'>-			goto fail;</div><div class='del'>-		}</div><div class='del'>-		qp-&gt;qplib_qp.srq = &amp;srq-&gt;qplib_srq;</div><div class='del'>-		qp-&gt;qplib_qp.rq.max_wqe = 0;</div><div class='del'>-	} else {</div><div class='del'>-		/* Allocate 1 more than what's provided so posting max doesn't</div><div class='del'>-		 * mean empty</div><div class='del'>-		 */</div><div class='del'>-		entries = roundup_pow_of_two(qp_init_attr-&gt;cap.max_recv_wr + 1);</div><div class='del'>-		qp-&gt;qplib_qp.rq.max_wqe = min_t(u32, entries,</div><div class='del'>-						dev_attr-&gt;max_qp_wqes + 1);</div><div class='add'>+	/* Setup RQ/SRQ */</div><div class='add'>+	rc = bnxt_re_init_rq_attr(qp, init_attr, uctx);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto out;</div><div class='add'>+	if (init_attr-&gt;qp_type == IB_QPT_GSI)</div><div class='add'>+		bnxt_re_adjust_gsi_rq_attr(qp);</div><div class='add'>+</div><div class='add'>+	/* Setup SQ */</div><div class='add'>+	rc = bnxt_re_init_sq_attr(qp, init_attr, uctx, ureq);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto out;</div><div class='add'>+	if (init_attr-&gt;qp_type == IB_QPT_GSI)</div><div class='add'>+		bnxt_re_adjust_gsi_sq_attr(qp, init_attr, uctx);</div><div class='add'>+</div><div class='add'>+	if (uctx) /* This will update DPI and qp_handle */</div><div class='add'>+		rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq);</div><div class='add'>+out:</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		qp-&gt;qplib_qp.rq.q_full_delta = qp-&gt;qplib_qp.rq.max_wqe -</div><div class='del'>-						qp_init_attr-&gt;cap.max_recv_wr;</div><div class='add'>+static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp,</div><div class='add'>+				     struct bnxt_re_pd *pd)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_sqp_entries *sqp_tbl;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct bnxt_re_qp *sqp;</div><div class='add'>+	struct bnxt_re_ah *sah;</div><div class='add'>+	int rc = 0;</div><div class='add'>+</div><div class='add'>+	rdev = qp-&gt;rdev;</div><div class='add'>+	/* Create a shadow QP to handle the QP1 traffic */</div><div class='add'>+	sqp_tbl = kcalloc(BNXT_RE_MAX_GSI_SQP_ENTRIES, sizeof(*sqp_tbl),</div><div class='add'>+			  GFP_KERNEL);</div><div class='add'>+	if (!sqp_tbl)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	rdev-&gt;gsi_ctx.sqp_tbl = sqp_tbl;</div><div class='ctx'> </div><div class='del'>-		qp-&gt;qplib_qp.rq.max_sge = qp_init_attr-&gt;cap.max_recv_sge;</div><div class='del'>-		if (qp-&gt;qplib_qp.rq.max_sge &gt; dev_attr-&gt;max_qp_sges)</div><div class='del'>-			qp-&gt;qplib_qp.rq.max_sge = dev_attr-&gt;max_qp_sges;</div><div class='add'>+	sqp = bnxt_re_create_shadow_qp(pd, &amp;rdev-&gt;qplib_res, &amp;qp-&gt;qplib_qp);</div><div class='add'>+	if (!sqp) {</div><div class='add'>+		rc = -ENODEV;</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to create Shadow QP for QP1");</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+	rdev-&gt;gsi_ctx.gsi_sqp = sqp;</div><div class='add'>+</div><div class='add'>+	sqp-&gt;rcq = qp-&gt;rcq;</div><div class='add'>+	sqp-&gt;scq = qp-&gt;scq;</div><div class='add'>+	sah = bnxt_re_create_shadow_qp_ah(pd, &amp;rdev-&gt;qplib_res,</div><div class='add'>+					  &amp;qp-&gt;qplib_qp);</div><div class='add'>+	if (!sah) {</div><div class='add'>+		bnxt_qplib_destroy_qp(&amp;rdev-&gt;qplib_res,</div><div class='add'>+				      &amp;sqp-&gt;qplib_qp);</div><div class='add'>+		rc = -ENODEV;</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to create AH entry for ShadowQP");</div><div class='add'>+		goto out;</div><div class='ctx'> 	}</div><div class='add'>+	rdev-&gt;gsi_ctx.gsi_sah = sah;</div><div class='ctx'> </div><div class='del'>-	qp-&gt;qplib_qp.mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev-&gt;netdev-&gt;mtu));</div><div class='add'>+	return 0;</div><div class='add'>+out:</div><div class='add'>+	kfree(sqp_tbl);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (qp_init_attr-&gt;qp_type == IB_QPT_GSI &amp;&amp;</div><div class='del'>-	    !(bnxt_qplib_is_chip_gen_p5(&amp;rdev-&gt;chip_ctx))) {</div><div class='del'>-		/* Allocate 1 more than what's provided */</div><div class='del'>-		entries = roundup_pow_of_two(qp_init_attr-&gt;cap.max_send_wr + 1);</div><div class='del'>-		qp-&gt;qplib_qp.sq.max_wqe = min_t(u32, entries,</div><div class='del'>-						dev_attr-&gt;max_qp_wqes + 1);</div><div class='del'>-		qp-&gt;qplib_qp.sq.q_full_delta = qp-&gt;qplib_qp.sq.max_wqe -</div><div class='del'>-						qp_init_attr-&gt;cap.max_send_wr;</div><div class='del'>-		qp-&gt;qplib_qp.rq.max_sge = dev_attr-&gt;max_qp_sges;</div><div class='del'>-		if (qp-&gt;qplib_qp.rq.max_sge &gt; dev_attr-&gt;max_qp_sges)</div><div class='del'>-			qp-&gt;qplib_qp.rq.max_sge = dev_attr-&gt;max_qp_sges;</div><div class='del'>-		qp-&gt;qplib_qp.sq.max_sge++;</div><div class='del'>-		if (qp-&gt;qplib_qp.sq.max_sge &gt; dev_attr-&gt;max_qp_sges)</div><div class='del'>-			qp-&gt;qplib_qp.sq.max_sge = dev_attr-&gt;max_qp_sges;</div><div class='del'>-</div><div class='del'>-		qp-&gt;qplib_qp.rq_hdr_buf_size =</div><div class='del'>-					BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;</div><div class='del'>-</div><div class='del'>-		qp-&gt;qplib_qp.sq_hdr_buf_size =</div><div class='del'>-					BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2;</div><div class='del'>-		qp-&gt;qplib_qp.dpi = &amp;rdev-&gt;dpi_privileged;</div><div class='del'>-		rc = bnxt_qplib_create_qp1(&amp;rdev-&gt;qplib_res, &amp;qp-&gt;qplib_qp);</div><div class='del'>-		if (rc) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev), "Failed to create HW QP1");</div><div class='del'>-			goto fail;</div><div class='del'>-		}</div><div class='del'>-		/* Create a shadow QP to handle the QP1 traffic */</div><div class='del'>-		rdev-&gt;qp1_sqp = bnxt_re_create_shadow_qp(pd, &amp;rdev-&gt;qplib_res,</div><div class='del'>-							 &amp;qp-&gt;qplib_qp);</div><div class='del'>-		if (!rdev-&gt;qp1_sqp) {</div><div class='del'>-			rc = -EINVAL;</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"Failed to create Shadow QP for QP1");</div><div class='del'>-			goto qp_destroy;</div><div class='del'>-		}</div><div class='del'>-		rdev-&gt;sqp_ah = bnxt_re_create_shadow_qp_ah(pd, &amp;rdev-&gt;qplib_res,</div><div class='del'>-							   &amp;qp-&gt;qplib_qp);</div><div class='del'>-		if (!rdev-&gt;sqp_ah) {</div><div class='del'>-			bnxt_qplib_destroy_qp(&amp;rdev-&gt;qplib_res,</div><div class='del'>-					      &amp;rdev-&gt;qp1_sqp-&gt;qplib_qp);</div><div class='del'>-			rc = -EINVAL;</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"Failed to create AH entry for ShadowQP");</div><div class='del'>-			goto qp_destroy;</div><div class='del'>-		}</div><div class='add'>+static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,</div><div class='add'>+				 struct ib_qp_init_attr *init_attr)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct bnxt_qplib_qp *qplqp;</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='del'>-	} else {</div><div class='del'>-		/* Allocate 128 + 1 more than what's provided */</div><div class='del'>-		entries = roundup_pow_of_two(qp_init_attr-&gt;cap.max_send_wr +</div><div class='del'>-					     BNXT_QPLIB_RESERVED_QP_WRS + 1);</div><div class='del'>-		qp-&gt;qplib_qp.sq.max_wqe = min_t(u32, entries,</div><div class='del'>-						dev_attr-&gt;max_qp_wqes +</div><div class='del'>-						BNXT_QPLIB_RESERVED_QP_WRS + 1);</div><div class='del'>-		qp-&gt;qplib_qp.sq.q_full_delta = BNXT_QPLIB_RESERVED_QP_WRS + 1;</div><div class='add'>+	rdev = qp-&gt;rdev;</div><div class='add'>+	qplqp = &amp;qp-&gt;qplib_qp;</div><div class='ctx'> </div><div class='del'>-		/*</div><div class='del'>-		 * Reserving one slot for Phantom WQE. Application can</div><div class='del'>-		 * post one extra entry in this case. But allowing this to avoid</div><div class='del'>-		 * unexpected Queue full condition</div><div class='del'>-		 */</div><div class='add'>+	qplqp-&gt;rq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;</div><div class='add'>+	qplqp-&gt;sq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2;</div><div class='ctx'> </div><div class='del'>-		qp-&gt;qplib_qp.sq.q_full_delta -= 1;</div><div class='add'>+	rc = bnxt_qplib_create_qp1(&amp;rdev-&gt;qplib_res, qplqp);</div><div class='add'>+	if (rc) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "create HW QP1 failed!");</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		qp-&gt;qplib_qp.max_rd_atomic = dev_attr-&gt;max_qp_rd_atom;</div><div class='del'>-		qp-&gt;qplib_qp.max_dest_rd_atomic = dev_attr-&gt;max_qp_init_rd_atom;</div><div class='del'>-		if (udata) {</div><div class='del'>-			rc = bnxt_re_init_user_qp(rdev, pd, qp, udata);</div><div class='del'>-			if (rc)</div><div class='del'>-				goto fail;</div><div class='del'>-		} else {</div><div class='del'>-			qp-&gt;qplib_qp.dpi = &amp;rdev-&gt;dpi_privileged;</div><div class='del'>-		}</div><div class='add'>+	rc = bnxt_re_create_shadow_gsi(qp, pd);</div><div class='add'>+out:</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='ctx'> </div><div class='add'>+static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev,</div><div class='add'>+				   struct ib_qp_init_attr *init_attr,</div><div class='add'>+				   struct bnxt_qplib_dev_attr *dev_attr)</div><div class='add'>+{</div><div class='add'>+	bool rc = true;</div><div class='add'>+</div><div class='add'>+	if (init_attr-&gt;cap.max_send_wr &gt; dev_attr-&gt;max_qp_wqes ||</div><div class='add'>+	    init_attr-&gt;cap.max_recv_wr &gt; dev_attr-&gt;max_qp_wqes ||</div><div class='add'>+	    init_attr-&gt;cap.max_send_sge &gt; dev_attr-&gt;max_qp_sges ||</div><div class='add'>+	    init_attr-&gt;cap.max_recv_sge &gt; dev_attr-&gt;max_qp_sges ||</div><div class='add'>+	    init_attr-&gt;cap.max_inline_data &gt; dev_attr-&gt;max_inline_data) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Create QP failed - max exceeded! 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x",</div><div class='add'>+			  init_attr-&gt;cap.max_send_wr, dev_attr-&gt;max_qp_wqes,</div><div class='add'>+			  init_attr-&gt;cap.max_recv_wr, dev_attr-&gt;max_qp_wqes,</div><div class='add'>+			  init_attr-&gt;cap.max_send_sge, dev_attr-&gt;max_qp_sges,</div><div class='add'>+			  init_attr-&gt;cap.max_recv_sge, dev_attr-&gt;max_qp_sges,</div><div class='add'>+			  init_attr-&gt;cap.max_inline_data,</div><div class='add'>+			  dev_attr-&gt;max_inline_data);</div><div class='add'>+		rc = false;</div><div class='add'>+	}</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bnxt_re_add_unique_gid(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_ctx *hctx = &amp;rdev-&gt;qplib_ctx;</div><div class='add'>+	struct bnxt_qplib_res *res = &amp;rdev-&gt;qplib_res;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (!rdev-&gt;rcfw.roce_mirror)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	rdev-&gt;ugid.global.subnet_prefix = cpu_to_be64(0xfe8000000000abcdLL);</div><div class='add'>+	addrconf_ifid_eui48(&amp;rdev-&gt;ugid.raw[8], rdev-&gt;netdev);</div><div class='add'>+</div><div class='add'>+	rc = bnxt_qplib_add_sgid(&amp;res-&gt;sgid_tbl,</div><div class='add'>+				 (struct bnxt_qplib_gid *)&amp;rdev-&gt;ugid,</div><div class='add'>+				 rdev-&gt;qplib_res.netdev-&gt;dev_addr,</div><div class='add'>+				 0xFFFF, true, &amp;rdev-&gt;ugid_index, true,</div><div class='add'>+				 hctx-&gt;stats3.fw_id);</div><div class='add'>+	if (rc)</div><div class='add'>+		dev_err(rdev_to_dev(rdev), "Failed to add unique GID. rc = %d\n", rc);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,</div><div class='add'>+		      struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_dev_attr *dev_attr;</div><div class='add'>+	struct bnxt_re_ucontext *uctx;</div><div class='add'>+	struct bnxt_re_qp_req ureq;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct bnxt_re_pd *pd;</div><div class='add'>+	struct bnxt_re_qp *qp;</div><div class='add'>+	struct ib_pd *ib_pd;</div><div class='add'>+	u32 active_qps;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	ib_pd = ib_qp-&gt;pd;</div><div class='add'>+	pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);</div><div class='add'>+	rdev = pd-&gt;rdev;</div><div class='add'>+	dev_attr = rdev-&gt;dev_attr;</div><div class='add'>+	qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);</div><div class='add'>+</div><div class='add'>+	uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);</div><div class='add'>+	if (udata)</div><div class='add'>+		if (ib_copy_from_udata(&amp;ureq, udata,  min(udata-&gt;inlen, sizeof(ureq))))</div><div class='add'>+			return -EFAULT;</div><div class='add'>+</div><div class='add'>+	rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);</div><div class='add'>+	if (!rc) {</div><div class='add'>+		rc = -EINVAL;</div><div class='add'>+		goto fail;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	qp-&gt;rdev = rdev;</div><div class='add'>+	rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &amp;ureq);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto fail;</div><div class='add'>+</div><div class='add'>+	if (qp_init_attr-&gt;qp_type == IB_QPT_GSI &amp;&amp;</div><div class='add'>+	    !(bnxt_qplib_is_chip_gen_p5_p7(rdev-&gt;chip_ctx))) {</div><div class='add'>+		rc = bnxt_re_create_gsi_qp(qp, pd, qp_init_attr);</div><div class='add'>+		if (rc == -ENODEV)</div><div class='add'>+			goto qp_destroy;</div><div class='add'>+		if (rc)</div><div class='add'>+			goto fail;</div><div class='add'>+	} else {</div><div class='ctx'> 		rc = bnxt_qplib_create_qp(&amp;rdev-&gt;qplib_res, &amp;qp-&gt;qplib_qp);</div><div class='ctx'> 		if (rc) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev), "Failed to create HW QP");</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev, "Failed to create HW QP");</div><div class='ctx'> 			goto free_umem;</div><div class='ctx'> 		}</div><div class='add'>+		if (udata) {</div><div class='add'>+			struct bnxt_re_qp_resp resp;</div><div class='add'>+</div><div class='add'>+			resp.qpid = qp-&gt;qplib_qp.id;</div><div class='add'>+			resp.rsvd = 0;</div><div class='add'>+			rc = ib_copy_to_udata(udata, &amp;resp, sizeof(resp));</div><div class='add'>+			if (rc) {</div><div class='add'>+				ibdev_err(&amp;rdev-&gt;ibdev, "Failed to copy QP udata");</div><div class='add'>+				goto qp_destroy;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Support for RawEth QP is added to capture TCP pkt dump.</div><div class='add'>+	 * So unique SGID is used to avoid incorrect statistics on per</div><div class='add'>+	 * function stats_ctx</div><div class='add'>+	 */</div><div class='add'>+	if (qp-&gt;qplib_qp.type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE) {</div><div class='add'>+		rc = bnxt_re_add_unique_gid(rdev);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto qp_destroy;</div><div class='add'>+		qp-&gt;qplib_qp.ugid_index = rdev-&gt;ugid_index;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	qp-&gt;ib_qp.qp_num = qp-&gt;qplib_qp.id;</div><div class='add'>+	if (qp_init_attr-&gt;qp_type == IB_QPT_GSI)</div><div class='add'>+		rdev-&gt;gsi_ctx.gsi_qp = qp;</div><div class='ctx'> 	spin_lock_init(&amp;qp-&gt;sq_lock);</div><div class='ctx'> 	spin_lock_init(&amp;qp-&gt;rq_lock);</div><div class='del'>-</div><div class='del'>-	if (udata) {</div><div class='del'>-		struct bnxt_re_qp_resp resp;</div><div class='del'>-</div><div class='del'>-		resp.qpid = qp-&gt;ib_qp.qp_num;</div><div class='del'>-		resp.rsvd = 0;</div><div class='del'>-		rc = ib_copy_to_udata(udata, &amp;resp, sizeof(resp));</div><div class='del'>-		if (rc) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev), "Failed to copy QP udata");</div><div class='del'>-			goto qp_destroy;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;qp-&gt;list);</div><div class='ctx'> 	mutex_lock(&amp;rdev-&gt;qp_lock);</div><div class='ctx'> 	list_add_tail(&amp;qp-&gt;list, &amp;rdev-&gt;qp_list);</div><div class='del'>-	atomic_inc(&amp;rdev-&gt;qp_count);</div><div class='ctx'> 	mutex_unlock(&amp;rdev-&gt;qp_lock);</div><div class='add'>+	active_qps = atomic_inc_return(&amp;rdev-&gt;stats.res.qp_count);</div><div class='add'>+	if (active_qps &gt; rdev-&gt;stats.res.qp_watermark)</div><div class='add'>+		rdev-&gt;stats.res.qp_watermark = active_qps;</div><div class='add'>+	if (qp_init_attr-&gt;qp_type == IB_QPT_RC) {</div><div class='add'>+		active_qps = atomic_inc_return(&amp;rdev-&gt;stats.res.rc_qp_count);</div><div class='add'>+		if (active_qps &gt; rdev-&gt;stats.res.rc_qp_watermark)</div><div class='add'>+			rdev-&gt;stats.res.rc_qp_watermark = active_qps;</div><div class='add'>+	} else if (qp_init_attr-&gt;qp_type == IB_QPT_UD) {</div><div class='add'>+		active_qps = atomic_inc_return(&amp;rdev-&gt;stats.res.ud_qp_count);</div><div class='add'>+		if (active_qps &gt; rdev-&gt;stats.res.ud_qp_watermark)</div><div class='add'>+			rdev-&gt;stats.res.ud_qp_watermark = active_qps;</div><div class='add'>+	}</div><div class='add'>+	bnxt_re_debug_add_qpinfo(rdev, qp);</div><div class='ctx'> </div><div class='del'>-	return &amp;qp-&gt;ib_qp;</div><div class='add'>+	return 0;</div><div class='ctx'> qp_destroy:</div><div class='ctx'> 	bnxt_qplib_destroy_qp(&amp;rdev-&gt;qplib_res, &amp;qp-&gt;qplib_qp);</div><div class='ctx'> free_umem:</div><div class='del'>-	if (udata) {</div><div class='del'>-		if (qp-&gt;rumem)</div><div class='del'>-			ib_umem_release(qp-&gt;rumem);</div><div class='del'>-		if (qp-&gt;sumem)</div><div class='del'>-			ib_umem_release(qp-&gt;sumem);</div><div class='del'>-	}</div><div class='add'>+	ib_umem_release(qp-&gt;rumem);</div><div class='add'>+	ib_umem_release(qp-&gt;sumem);</div><div class='ctx'> fail:</div><div class='del'>-	kfree(qp);</div><div class='del'>-	return ERR_PTR(rc);</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static u8 __from_ib_qp_state(enum ib_qp_state state)</div><div class='hunk'>@@ -1291,22 +1819,21 @@ static enum ib_mtu __to_ib_mtu(u32 mtu)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* Shared Receive Queues */</div><div class='del'>-void bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)</div><div class='add'>+int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,</div><div class='ctx'> 					       ib_srq);</div><div class='ctx'> 	struct bnxt_re_dev *rdev = srq-&gt;rdev;</div><div class='ctx'> 	struct bnxt_qplib_srq *qplib_srq = &amp;srq-&gt;qplib_srq;</div><div class='del'>-	struct bnxt_qplib_nq *nq = NULL;</div><div class='ctx'> </div><div class='del'>-	if (qplib_srq-&gt;cq)</div><div class='del'>-		nq = qplib_srq-&gt;cq-&gt;nq;</div><div class='add'>+	if (rdev-&gt;chip_ctx-&gt;modes.toggle_bits &amp; BNXT_QPLIB_SRQ_TOGGLE_BIT) {</div><div class='add'>+		free_page((unsigned long)srq-&gt;uctx_srq_page);</div><div class='add'>+		hash_del(&amp;srq-&gt;hash_entry);</div><div class='add'>+	}</div><div class='ctx'> 	bnxt_qplib_destroy_srq(&amp;rdev-&gt;qplib_res, qplib_srq);</div><div class='del'>-	if (srq-&gt;umem)</div><div class='del'>-		ib_umem_release(srq-&gt;umem);</div><div class='del'>-	atomic_dec(&amp;rdev-&gt;srq_count);</div><div class='del'>-	if (nq)</div><div class='del'>-		nq-&gt;budget--;</div><div class='add'>+	ib_umem_release(srq-&gt;umem);</div><div class='add'>+	atomic_dec(&amp;rdev-&gt;stats.res.srq_count);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,</div><div class='hunk'>@@ -1324,16 +1851,17 @@ static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,</div><div class='ctx'> 	if (ib_copy_from_udata(&amp;ureq, udata, sizeof(ureq)))</div><div class='ctx'> 		return -EFAULT;</div><div class='ctx'> </div><div class='del'>-	bytes = (qplib_srq-&gt;max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);</div><div class='add'>+	bytes = (qplib_srq-&gt;max_wqe * qplib_srq-&gt;wqe_size);</div><div class='ctx'> 	bytes = PAGE_ALIGN(bytes);</div><div class='del'>-	umem = ib_umem_get(udata, ureq.srqva, bytes, IB_ACCESS_LOCAL_WRITE, 1);</div><div class='add'>+	umem = ib_umem_get(&amp;rdev-&gt;ibdev, ureq.srqva, bytes,</div><div class='add'>+			   IB_ACCESS_LOCAL_WRITE);</div><div class='ctx'> 	if (IS_ERR(umem))</div><div class='ctx'> 		return PTR_ERR(umem);</div><div class='ctx'> </div><div class='ctx'> 	srq-&gt;umem = umem;</div><div class='del'>-	qplib_srq-&gt;sg_info.sglist = umem-&gt;sg_head.sgl;</div><div class='del'>-	qplib_srq-&gt;sg_info.npages = ib_umem_num_pages(umem);</div><div class='del'>-	qplib_srq-&gt;sg_info.nmap = umem-&gt;nmap;</div><div class='add'>+	qplib_srq-&gt;sg_info.umem = umem;</div><div class='add'>+	qplib_srq-&gt;sg_info.pgsize = PAGE_SIZE;</div><div class='add'>+	qplib_srq-&gt;sg_info.pgshft = PAGE_SHIFT;</div><div class='ctx'> 	qplib_srq-&gt;srq_handle = ureq.srq_handle;</div><div class='ctx'> 	qplib_srq-&gt;dpi = &amp;cntx-&gt;dpi;</div><div class='ctx'> </div><div class='hunk'>@@ -1344,17 +1872,23 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq,</div><div class='ctx'> 		       struct ib_srq_init_attr *srq_init_attr,</div><div class='ctx'> 		       struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	struct ib_pd *ib_pd = ib_srq-&gt;pd;</div><div class='del'>-	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);</div><div class='del'>-	struct bnxt_re_dev *rdev = pd-&gt;rdev;</div><div class='del'>-	struct bnxt_qplib_dev_attr *dev_attr = &amp;rdev-&gt;dev_attr;</div><div class='del'>-	struct bnxt_re_srq *srq =</div><div class='del'>-		container_of(ib_srq, struct bnxt_re_srq, ib_srq);</div><div class='del'>-	struct bnxt_qplib_nq *nq = NULL;</div><div class='add'>+	struct bnxt_qplib_dev_attr *dev_attr;</div><div class='add'>+	struct bnxt_re_ucontext *uctx;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct bnxt_re_srq *srq;</div><div class='add'>+	struct bnxt_re_pd *pd;</div><div class='add'>+	struct ib_pd *ib_pd;</div><div class='add'>+	u32 active_srqs;</div><div class='ctx'> 	int rc, entries;</div><div class='ctx'> </div><div class='add'>+	ib_pd = ib_srq-&gt;pd;</div><div class='add'>+	pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);</div><div class='add'>+	rdev = pd-&gt;rdev;</div><div class='add'>+	dev_attr = rdev-&gt;dev_attr;</div><div class='add'>+	srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq);</div><div class='add'>+</div><div class='ctx'> 	if (srq_init_attr-&gt;attr.max_wr &gt;= dev_attr-&gt;max_srq_wqes) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Create CQ failed - max exceeded");</div><div class='ctx'> 		rc = -EINVAL;</div><div class='ctx'> 		goto exit;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -1364,22 +1898,26 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq,</div><div class='ctx'> 		goto exit;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);</div><div class='ctx'> 	srq-&gt;rdev = rdev;</div><div class='ctx'> 	srq-&gt;qplib_srq.pd = &amp;pd-&gt;qplib_pd;</div><div class='ctx'> 	srq-&gt;qplib_srq.dpi = &amp;rdev-&gt;dpi_privileged;</div><div class='ctx'> 	/* Allocate 1 more than what's provided so posting max doesn't</div><div class='ctx'> 	 * mean empty</div><div class='ctx'> 	 */</div><div class='del'>-	entries = roundup_pow_of_two(srq_init_attr-&gt;attr.max_wr + 1);</div><div class='add'>+	entries = bnxt_re_init_depth(srq_init_attr-&gt;attr.max_wr + 1, uctx);</div><div class='ctx'> 	if (entries &gt; dev_attr-&gt;max_srq_wqes + 1)</div><div class='ctx'> 		entries = dev_attr-&gt;max_srq_wqes + 1;</div><div class='del'>-</div><div class='ctx'> 	srq-&gt;qplib_srq.max_wqe = entries;</div><div class='add'>+</div><div class='ctx'> 	srq-&gt;qplib_srq.max_sge = srq_init_attr-&gt;attr.max_sge;</div><div class='add'>+	 /* 128 byte wqe size for SRQ . So use max sges */</div><div class='add'>+	srq-&gt;qplib_srq.wqe_size = bnxt_re_get_rwqe_size(dev_attr-&gt;max_srq_sges);</div><div class='ctx'> 	srq-&gt;qplib_srq.threshold = srq_init_attr-&gt;attr.srq_limit;</div><div class='ctx'> 	srq-&gt;srq_limit = srq_init_attr-&gt;attr.srq_limit;</div><div class='del'>-	srq-&gt;qplib_srq.eventq_hw_ring_id = rdev-&gt;nq[0].ring_id;</div><div class='del'>-	nq = &amp;rdev-&gt;nq[0];</div><div class='add'>+	srq-&gt;qplib_srq.eventq_hw_ring_id = rdev-&gt;nqr-&gt;nq[0].ring_id;</div><div class='add'>+	srq-&gt;qplib_srq.sg_info.pgsize = PAGE_SIZE;</div><div class='add'>+	srq-&gt;qplib_srq.sg_info.pgshft = PAGE_SHIFT;</div><div class='ctx'> </div><div class='ctx'> 	if (udata) {</div><div class='ctx'> 		rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);</div><div class='hunk'>@@ -1389,31 +1927,40 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq,</div><div class='ctx'> </div><div class='ctx'> 	rc = bnxt_qplib_create_srq(&amp;rdev-&gt;qplib_res, &amp;srq-&gt;qplib_srq);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Create HW SRQ failed!");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Create HW SRQ failed!");</div><div class='ctx'> 		goto fail;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (udata) {</div><div class='del'>-		struct bnxt_re_srq_resp resp;</div><div class='add'>+		struct bnxt_re_srq_resp resp = {};</div><div class='ctx'> </div><div class='ctx'> 		resp.srqid = srq-&gt;qplib_srq.id;</div><div class='add'>+		if (rdev-&gt;chip_ctx-&gt;modes.toggle_bits &amp; BNXT_QPLIB_SRQ_TOGGLE_BIT) {</div><div class='add'>+			hash_add(rdev-&gt;srq_hash, &amp;srq-&gt;hash_entry, srq-&gt;qplib_srq.id);</div><div class='add'>+			srq-&gt;uctx_srq_page = (void *)get_zeroed_page(GFP_KERNEL);</div><div class='add'>+			if (!srq-&gt;uctx_srq_page) {</div><div class='add'>+				rc = -ENOMEM;</div><div class='add'>+				goto fail;</div><div class='add'>+			}</div><div class='add'>+			resp.comp_mask |= BNXT_RE_SRQ_TOGGLE_PAGE_SUPPORT;</div><div class='add'>+		}</div><div class='ctx'> 		rc = ib_copy_to_udata(udata, &amp;resp, sizeof(resp));</div><div class='ctx'> 		if (rc) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev), "SRQ copy to udata failed!");</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev, "SRQ copy to udata failed!");</div><div class='ctx'> 			bnxt_qplib_destroy_srq(&amp;rdev-&gt;qplib_res,</div><div class='ctx'> 					       &amp;srq-&gt;qplib_srq);</div><div class='del'>-			goto exit;</div><div class='add'>+			goto fail;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='del'>-	if (nq)</div><div class='del'>-		nq-&gt;budget++;</div><div class='del'>-	atomic_inc(&amp;rdev-&gt;srq_count);</div><div class='add'>+	active_srqs = atomic_inc_return(&amp;rdev-&gt;stats.res.srq_count);</div><div class='add'>+	if (active_srqs &gt; rdev-&gt;stats.res.srq_watermark)</div><div class='add'>+		rdev-&gt;stats.res.srq_watermark = active_srqs;</div><div class='add'>+	spin_lock_init(&amp;srq-&gt;lock);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> fail:</div><div class='del'>-	if (srq-&gt;umem)</div><div class='del'>-		ib_umem_release(srq-&gt;umem);</div><div class='add'>+	ib_umem_release(srq-&gt;umem);</div><div class='ctx'> exit:</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='hunk'>@@ -1425,33 +1972,28 @@ int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,</div><div class='ctx'> 	struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,</div><div class='ctx'> 					       ib_srq);</div><div class='ctx'> 	struct bnxt_re_dev *rdev = srq-&gt;rdev;</div><div class='del'>-	int rc;</div><div class='ctx'> </div><div class='ctx'> 	switch (srq_attr_mask) {</div><div class='ctx'> 	case IB_SRQ_MAX_WR:</div><div class='ctx'> 		/* SRQ resize is not supported */</div><div class='del'>-		break;</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	case IB_SRQ_LIMIT:</div><div class='ctx'> 		/* Change the SRQ threshold */</div><div class='ctx'> 		if (srq_attr-&gt;srq_limit &gt; srq-&gt;qplib_srq.max_wqe)</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 		srq-&gt;qplib_srq.threshold = srq_attr-&gt;srq_limit;</div><div class='del'>-		rc = bnxt_qplib_modify_srq(&amp;rdev-&gt;qplib_res, &amp;srq-&gt;qplib_srq);</div><div class='del'>-		if (rc) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev), "Modify HW SRQ failed!");</div><div class='del'>-			return rc;</div><div class='del'>-		}</div><div class='add'>+		bnxt_qplib_srq_arm_db(&amp;srq-&gt;qplib_srq.dbinfo, srq-&gt;qplib_srq.threshold);</div><div class='add'>+</div><div class='ctx'> 		/* On success, update the shadow */</div><div class='ctx'> 		srq-&gt;srq_limit = srq_attr-&gt;srq_limit;</div><div class='ctx'> 		/* No need to Build and send response back to udata */</div><div class='del'>-		break;</div><div class='add'>+		return 0;</div><div class='ctx'> 	default:</div><div class='del'>-		dev_err(rdev_to_dev(rdev),</div><div class='del'>-			"Unsupported srq_attr_mask 0x%x", srq_attr_mask);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Unsupported srq_attr_mask 0x%x", srq_attr_mask);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='del'>-	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)</div><div class='hunk'>@@ -1466,7 +2008,7 @@ int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)</div><div class='ctx'> 	tsrq.qplib_srq.id = srq-&gt;qplib_srq.id;</div><div class='ctx'> 	rc = bnxt_qplib_query_srq(&amp;rdev-&gt;qplib_res, &amp;tsrq.qplib_srq);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Query HW SRQ failed!");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Query HW SRQ failed!");</div><div class='ctx'> 		return rc;</div><div class='ctx'> 	}</div><div class='ctx'> 	srq_attr-&gt;max_wr = srq-&gt;qplib_srq.max_wqe;</div><div class='hunk'>@@ -1508,8 +2050,8 @@ static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,</div><div class='ctx'> 				    struct bnxt_re_qp *qp1_qp,</div><div class='ctx'> 				    int qp_attr_mask)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_re_qp *qp = rdev-&gt;qp1_sqp;</div><div class='del'>-	int rc = 0;</div><div class='add'>+	struct bnxt_re_qp *qp = rdev-&gt;gsi_ctx.gsi_sqp;</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='ctx'> 	if (qp_attr_mask &amp; IB_QP_STATE) {</div><div class='ctx'> 		qp-&gt;qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;</div><div class='hunk'>@@ -1532,8 +2074,7 @@ static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,</div><div class='ctx'> </div><div class='ctx'> 	rc = bnxt_qplib_modify_qp(&amp;rdev-&gt;qplib_res, &amp;qp-&gt;qplib_qp);</div><div class='ctx'> 	if (rc)</div><div class='del'>-		dev_err(rdev_to_dev(rdev),</div><div class='del'>-			"Failed to modify Shadow QP for QP1");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to modify Shadow QP for QP1");</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1542,27 +2083,30 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);</div><div class='ctx'> 	struct bnxt_re_dev *rdev = qp-&gt;rdev;</div><div class='del'>-	struct bnxt_qplib_dev_attr *dev_attr = &amp;rdev-&gt;dev_attr;</div><div class='add'>+	struct bnxt_qplib_dev_attr *dev_attr = rdev-&gt;dev_attr;</div><div class='ctx'> 	enum ib_qp_state curr_qp_state, new_qp_state;</div><div class='ctx'> 	int rc, entries;</div><div class='ctx'> 	unsigned int flags;</div><div class='ctx'> 	u8 nw_type;</div><div class='ctx'> </div><div class='add'>+	if (qp_attr_mask &amp; ~IB_QP_ATTR_STANDARD_BITS)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='ctx'> 	qp-&gt;qplib_qp.modify_flags = 0;</div><div class='ctx'> 	if (qp_attr_mask &amp; IB_QP_STATE) {</div><div class='ctx'> 		curr_qp_state = __to_ib_qp_state(qp-&gt;qplib_qp.cur_qp_state);</div><div class='ctx'> 		new_qp_state = qp_attr-&gt;qp_state;</div><div class='ctx'> 		if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state,</div><div class='ctx'> 					ib_qp-&gt;qp_type, qp_attr_mask)) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"Invalid attribute mask: %#x specified ",</div><div class='del'>-				qp_attr_mask);</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"for qpn: %#x type: %#x",</div><div class='del'>-				ib_qp-&gt;qp_num, ib_qp-&gt;qp_type);</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"curr_qp_state=0x%x, new_qp_state=0x%x\n",</div><div class='del'>-				curr_qp_state, new_qp_state);</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "Invalid attribute mask: %#x specified ",</div><div class='add'>+				  qp_attr_mask);</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "for qpn: %#x type: %#x",</div><div class='add'>+				  ib_qp-&gt;qp_num, ib_qp-&gt;qp_type);</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "curr_qp_state=0x%x, new_qp_state=0x%x\n",</div><div class='add'>+				  curr_qp_state, new_qp_state);</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 		}</div><div class='ctx'> 		qp-&gt;qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;</div><div class='hunk'>@@ -1570,18 +2114,16 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> </div><div class='ctx'> 		if (!qp-&gt;sumem &amp;&amp;</div><div class='ctx'> 		    qp-&gt;qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {</div><div class='del'>-			dev_dbg(rdev_to_dev(rdev),</div><div class='del'>-				"Move QP = %p to flush list\n",</div><div class='del'>-				qp);</div><div class='add'>+			ibdev_dbg(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "Move QP = %p to flush list\n", qp);</div><div class='ctx'> 			flags = bnxt_re_lock_cqs(qp);</div><div class='ctx'> 			bnxt_qplib_add_flush_qp(&amp;qp-&gt;qplib_qp);</div><div class='ctx'> 			bnxt_re_unlock_cqs(qp, flags);</div><div class='ctx'> 		}</div><div class='ctx'> 		if (!qp-&gt;sumem &amp;&amp;</div><div class='ctx'> 		    qp-&gt;qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {</div><div class='del'>-			dev_dbg(rdev_to_dev(rdev),</div><div class='del'>-				"Move QP = %p out of flush list\n",</div><div class='del'>-				qp);</div><div class='add'>+			ibdev_dbg(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "Move QP = %p out of flush list\n", qp);</div><div class='ctx'> 			flags = bnxt_re_lock_cqs(qp);</div><div class='ctx'> 			bnxt_qplib_clean_qp(&amp;qp-&gt;qplib_qp);</div><div class='ctx'> 			bnxt_re_unlock_cqs(qp, flags);</div><div class='hunk'>@@ -1595,12 +2137,10 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 	if (qp_attr_mask &amp; IB_QP_ACCESS_FLAGS) {</div><div class='ctx'> 		qp-&gt;qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;</div><div class='ctx'> 		qp-&gt;qplib_qp.access =</div><div class='del'>-			__from_ib_access_flags(qp_attr-&gt;qp_access_flags);</div><div class='add'>+			__qp_access_flags_from_ib(qp-&gt;qplib_qp.cctx,</div><div class='add'>+						  qp_attr-&gt;qp_access_flags);</div><div class='ctx'> 		/* LOCAL_WRITE access must be set to allow RC receive */</div><div class='del'>-		qp-&gt;qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;</div><div class='del'>-		/* Temp: Set all params on QP as of now */</div><div class='del'>-		qp-&gt;qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE;</div><div class='del'>-		qp-&gt;qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ;</div><div class='add'>+		qp-&gt;qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE;</div><div class='ctx'> 	}</div><div class='ctx'> 	if (qp_attr_mask &amp; IB_QP_PKEY_INDEX) {</div><div class='ctx'> 		qp-&gt;qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;</div><div class='hunk'>@@ -1614,6 +2154,7 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 		const struct ib_global_route *grh =</div><div class='ctx'> 			rdma_ah_read_grh(&amp;qp_attr-&gt;ah_attr);</div><div class='ctx'> 		const struct ib_gid_attr *sgid_attr;</div><div class='add'>+		struct bnxt_re_gid_ctx *ctx;</div><div class='ctx'> </div><div class='ctx'> 		qp-&gt;qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |</div><div class='ctx'> 				     CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |</div><div class='hunk'>@@ -1625,19 +2166,19 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 		memcpy(qp-&gt;qplib_qp.ah.dgid.data, grh-&gt;dgid.raw,</div><div class='ctx'> 		       sizeof(qp-&gt;qplib_qp.ah.dgid.data));</div><div class='ctx'> 		qp-&gt;qplib_qp.ah.flow_label = grh-&gt;flow_label;</div><div class='del'>-		/* If RoCE V2 is enabled, stack will have two entries for</div><div class='del'>-		 * each GID entry. Avoiding this duplicte entry in HW. Dividing</div><div class='del'>-		 * the GID index by 2 for RoCE V2</div><div class='add'>+		sgid_attr = grh-&gt;sgid_attr;</div><div class='add'>+		/* Get the HW context of the GID. The reference</div><div class='add'>+		 * of GID table entry is already taken by the caller.</div><div class='ctx'> 		 */</div><div class='del'>-		qp-&gt;qplib_qp.ah.sgid_index = grh-&gt;sgid_index / 2;</div><div class='add'>+		ctx = rdma_read_gid_hw_context(sgid_attr);</div><div class='add'>+		qp-&gt;qplib_qp.ah.sgid_index = ctx-&gt;idx;</div><div class='ctx'> 		qp-&gt;qplib_qp.ah.host_sgid_index = grh-&gt;sgid_index;</div><div class='ctx'> 		qp-&gt;qplib_qp.ah.hop_limit = grh-&gt;hop_limit;</div><div class='del'>-		qp-&gt;qplib_qp.ah.traffic_class = grh-&gt;traffic_class;</div><div class='add'>+		qp-&gt;qplib_qp.ah.traffic_class = grh-&gt;traffic_class &gt;&gt; 2;</div><div class='ctx'> 		qp-&gt;qplib_qp.ah.sl = rdma_ah_get_sl(&amp;qp_attr-&gt;ah_attr);</div><div class='ctx'> 		ether_addr_copy(qp-&gt;qplib_qp.ah.dmac,</div><div class='ctx'> 				qp_attr-&gt;ah_attr.roce.dmac);</div><div class='ctx'> </div><div class='del'>-		sgid_attr = qp_attr-&gt;ah_attr.grh.sgid_attr;</div><div class='ctx'> 		rc = rdma_read_gid_l2_fields(sgid_attr, NULL,</div><div class='ctx'> 					     &amp;qp-&gt;qplib_qp.smac[0]);</div><div class='ctx'> 		if (rc)</div><div class='hunk'>@@ -1660,18 +2201,20 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (qp_attr_mask &amp; IB_QP_PATH_MTU) {</div><div class='del'>-		qp-&gt;qplib_qp.modify_flags |=</div><div class='del'>-				CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;</div><div class='del'>-		qp-&gt;qplib_qp.path_mtu = __from_ib_mtu(qp_attr-&gt;path_mtu);</div><div class='del'>-		qp-&gt;qplib_qp.mtu = ib_mtu_enum_to_int(qp_attr-&gt;path_mtu);</div><div class='del'>-	} else if (qp_attr-&gt;qp_state == IB_QPS_RTR) {</div><div class='del'>-		qp-&gt;qplib_qp.modify_flags |=</div><div class='del'>-			CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;</div><div class='del'>-		qp-&gt;qplib_qp.path_mtu =</div><div class='del'>-			__from_ib_mtu(iboe_get_mtu(rdev-&gt;netdev-&gt;mtu));</div><div class='del'>-		qp-&gt;qplib_qp.mtu =</div><div class='del'>-			ib_mtu_enum_to_int(iboe_get_mtu(rdev-&gt;netdev-&gt;mtu));</div><div class='add'>+	if (qp_attr-&gt;qp_state == IB_QPS_RTR) {</div><div class='add'>+		enum ib_mtu qpmtu;</div><div class='add'>+</div><div class='add'>+		qpmtu = iboe_get_mtu(rdev-&gt;netdev-&gt;mtu);</div><div class='add'>+		if (qp_attr_mask &amp; IB_QP_PATH_MTU) {</div><div class='add'>+			if (ib_mtu_enum_to_int(qp_attr-&gt;path_mtu) &gt;</div><div class='add'>+			    ib_mtu_enum_to_int(qpmtu))</div><div class='add'>+				return -EINVAL;</div><div class='add'>+			qpmtu = qp_attr-&gt;path_mtu;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		qp-&gt;qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;</div><div class='add'>+		qp-&gt;qplib_qp.path_mtu = __from_ib_mtu(qpmtu);</div><div class='add'>+		qp-&gt;qplib_qp.mtu = ib_mtu_enum_to_int(qpmtu);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (qp_attr_mask &amp; IB_QP_TIMEOUT) {</div><div class='hunk'>@@ -1711,10 +2254,10 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 	if (qp_attr_mask &amp; IB_QP_MAX_DEST_RD_ATOMIC) {</div><div class='ctx'> 		if (qp_attr-&gt;max_dest_rd_atomic &gt;</div><div class='ctx'> 		    dev_attr-&gt;max_qp_init_rd_atom) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"max_dest_rd_atomic requested%d is &gt; dev_max%d",</div><div class='del'>-				qp_attr-&gt;max_dest_rd_atomic,</div><div class='del'>-				dev_attr-&gt;max_qp_init_rd_atom);</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "max_dest_rd_atomic requested%d is &gt; dev_max%d",</div><div class='add'>+				  qp_attr-&gt;max_dest_rd_atomic,</div><div class='add'>+				  dev_attr-&gt;max_qp_init_rd_atom);</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='hunk'>@@ -1723,6 +2266,9 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 		qp-&gt;qplib_qp.max_dest_rd_atomic = qp_attr-&gt;max_dest_rd_atomic;</div><div class='ctx'> 	}</div><div class='ctx'> 	if (qp_attr_mask &amp; IB_QP_CAP) {</div><div class='add'>+		struct bnxt_re_ucontext *uctx =</div><div class='add'>+			rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);</div><div class='add'>+</div><div class='ctx'> 		qp-&gt;qplib_qp.modify_flags |=</div><div class='ctx'> 				CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |</div><div class='ctx'> 				CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |</div><div class='hunk'>@@ -1735,11 +2281,11 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 		    (qp_attr-&gt;cap.max_recv_sge &gt;= dev_attr-&gt;max_qp_sges) ||</div><div class='ctx'> 		    (qp_attr-&gt;cap.max_inline_data &gt;=</div><div class='ctx'> 						dev_attr-&gt;max_inline_data)) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"Create QP failed - max exceeded");</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "Create QP failed - max exceeded");</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 		}</div><div class='del'>-		entries = roundup_pow_of_two(qp_attr-&gt;cap.max_send_wr);</div><div class='add'>+		entries = bnxt_re_init_depth(qp_attr-&gt;cap.max_send_wr, uctx);</div><div class='ctx'> 		qp-&gt;qplib_qp.sq.max_wqe = min_t(u32, entries,</div><div class='ctx'> 						dev_attr-&gt;max_qp_wqes + 1);</div><div class='ctx'> 		qp-&gt;qplib_qp.sq.q_full_delta = qp-&gt;qplib_qp.sq.max_wqe -</div><div class='hunk'>@@ -1752,9 +2298,10 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 		qp-&gt;qplib_qp.sq.q_full_delta -= 1;</div><div class='ctx'> 		qp-&gt;qplib_qp.sq.max_sge = qp_attr-&gt;cap.max_send_sge;</div><div class='ctx'> 		if (qp-&gt;qplib_qp.rq.max_wqe) {</div><div class='del'>-			entries = roundup_pow_of_two(qp_attr-&gt;cap.max_recv_wr);</div><div class='add'>+			entries = bnxt_re_init_depth(qp_attr-&gt;cap.max_recv_wr, uctx);</div><div class='ctx'> 			qp-&gt;qplib_qp.rq.max_wqe =</div><div class='ctx'> 				min_t(u32, entries, dev_attr-&gt;max_qp_wqes + 1);</div><div class='add'>+			qp-&gt;qplib_qp.rq.max_sw_wqe = qp-&gt;qplib_qp.rq.max_wqe;</div><div class='ctx'> 			qp-&gt;qplib_qp.rq.q_full_delta = qp-&gt;qplib_qp.rq.max_wqe -</div><div class='ctx'> 						       qp_attr-&gt;cap.max_recv_wr;</div><div class='ctx'> 			qp-&gt;qplib_qp.rq.max_sge = qp_attr-&gt;cap.max_recv_sge;</div><div class='hunk'>@@ -1769,10 +2316,10 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 	}</div><div class='ctx'> 	rc = bnxt_qplib_modify_qp(&amp;rdev-&gt;qplib_res, &amp;qp-&gt;qplib_qp);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to modify HW QP");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to modify HW QP");</div><div class='ctx'> 		return rc;</div><div class='ctx'> 	}</div><div class='del'>-	if (ib_qp-&gt;qp_type == IB_QPT_GSI &amp;&amp; rdev-&gt;qp1_sqp)</div><div class='add'>+	if (ib_qp-&gt;qp_type == IB_QPT_GSI &amp;&amp; rdev-&gt;gsi_ctx.gsi_sqp)</div><div class='ctx'> 		rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='hunk'>@@ -1794,16 +2341,18 @@ int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> </div><div class='ctx'> 	rc = bnxt_qplib_query_qp(&amp;rdev-&gt;qplib_res, qplib_qp);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to query HW QP");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to query HW QP");</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> 	qp_attr-&gt;qp_state = __to_ib_qp_state(qplib_qp-&gt;state);</div><div class='add'>+	qp_attr-&gt;cur_qp_state = __to_ib_qp_state(qplib_qp-&gt;cur_qp_state);</div><div class='ctx'> 	qp_attr-&gt;en_sqd_async_notify = qplib_qp-&gt;en_sqd_async_notify ? 1 : 0;</div><div class='del'>-	qp_attr-&gt;qp_access_flags = __to_ib_access_flags(qplib_qp-&gt;access);</div><div class='add'>+	qp_attr-&gt;qp_access_flags = __qp_access_flags_to_ib(qp-&gt;qplib_qp.cctx,</div><div class='add'>+							   qplib_qp-&gt;access);</div><div class='ctx'> 	qp_attr-&gt;pkey_index = qplib_qp-&gt;pkey_index;</div><div class='ctx'> 	qp_attr-&gt;qkey = qplib_qp-&gt;qkey;</div><div class='ctx'> 	qp_attr-&gt;ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;</div><div class='del'>-	rdma_ah_set_grh(&amp;qp_attr-&gt;ah_attr, NULL, qplib_qp-&gt;ah.flow_label,</div><div class='add'>+	rdma_ah_set_grh(&amp;qp_attr-&gt;ah_attr, NULL, qplib_qp-&gt;udp_sport,</div><div class='ctx'> 			qplib_qp-&gt;ah.host_sgid_index,</div><div class='ctx'> 			qplib_qp-&gt;ah.hop_limit,</div><div class='ctx'> 			qplib_qp-&gt;ah.traffic_class);</div><div class='hunk'>@@ -1815,6 +2364,7 @@ int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 	qp_attr-&gt;retry_cnt = qplib_qp-&gt;retry_cnt;</div><div class='ctx'> 	qp_attr-&gt;rnr_retry = qplib_qp-&gt;rnr_retry;</div><div class='ctx'> 	qp_attr-&gt;min_rnr_timer = qplib_qp-&gt;min_rnr_timer;</div><div class='add'>+	qp_attr-&gt;port_num = __to_ib_port_num(qplib_qp-&gt;port_id);</div><div class='ctx'> 	qp_attr-&gt;rq_psn = qplib_qp-&gt;rq.psn;</div><div class='ctx'> 	qp_attr-&gt;max_rd_atomic = qplib_qp-&gt;max_rd_atomic;</div><div class='ctx'> 	qp_attr-&gt;sq_psn = qplib_qp-&gt;sq.psn;</div><div class='hunk'>@@ -1857,7 +2407,7 @@ static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,</div><div class='ctx'> 	u8 ip_version = 0;</div><div class='ctx'> 	u16 vlan_id = 0xFFFF;</div><div class='ctx'> 	void *buf;</div><div class='del'>-	int i, rc = 0;</div><div class='add'>+	int i, rc;</div><div class='ctx'> </div><div class='ctx'> 	memset(&amp;qp-&gt;qp1_hdr, 0, sizeof(qp-&gt;qp1_hdr));</div><div class='ctx'> </div><div class='hunk'>@@ -1895,7 +2445,7 @@ static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	is_eth = true;</div><div class='del'>-	is_vlan = (vlan_id &amp;&amp; (vlan_id &lt; 0x1000)) ? true : false;</div><div class='add'>+	is_vlan = vlan_id &amp;&amp; (vlan_id &lt; 0x1000);</div><div class='ctx'> </div><div class='ctx'> 	ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh,</div><div class='ctx'> 			  ip_version, is_udp, 0, &amp;qp-&gt;qp1_hdr);</div><div class='hunk'>@@ -1999,7 +2549,7 @@ static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,</div><div class='ctx'> 		wqe-&gt;num_sge++;</div><div class='ctx'> </div><div class='ctx'> 	} else {</div><div class='del'>-		dev_err(rdev_to_dev(qp-&gt;rdev), "QP1 buffer is empty!");</div><div class='add'>+		ibdev_err(&amp;qp-&gt;rdev-&gt;ibdev, "QP1 buffer is empty!");</div><div class='ctx'> 		rc = -ENOMEM;</div><div class='ctx'> 	}</div><div class='ctx'> 	return rc;</div><div class='hunk'>@@ -2016,9 +2566,12 @@ static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,</div><div class='ctx'> 					    struct bnxt_qplib_swqe *wqe,</div><div class='ctx'> 					    int payload_size)</div><div class='ctx'> {</div><div class='add'>+	struct bnxt_re_sqp_entries *sqp_entry;</div><div class='ctx'> 	struct bnxt_qplib_sge ref, sge;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='ctx'> 	u32 rq_prod_index;</div><div class='del'>-	struct bnxt_re_sqp_entries *sqp_entry;</div><div class='add'>+</div><div class='add'>+	rdev = qp-&gt;rdev;</div><div class='ctx'> </div><div class='ctx'> 	rq_prod_index = bnxt_qplib_get_rq_prod_index(&amp;qp-&gt;qplib_qp);</div><div class='ctx'> </div><div class='hunk'>@@ -2033,7 +2586,7 @@ static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,</div><div class='ctx'> 	ref.lkey = wqe-&gt;sg_list[0].lkey;</div><div class='ctx'> 	ref.size = wqe-&gt;sg_list[0].size;</div><div class='ctx'> </div><div class='del'>-	sqp_entry = &amp;qp-&gt;rdev-&gt;sqp_tbl[rq_prod_index];</div><div class='add'>+	sqp_entry = &amp;rdev-&gt;gsi_ctx.sqp_tbl[rq_prod_index];</div><div class='ctx'> </div><div class='ctx'> 	/* SGE 1 */</div><div class='ctx'> 	wqe-&gt;sg_list[0].addr = sge.addr;</div><div class='hunk'>@@ -2075,7 +2628,7 @@ static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_WR_SEND_WITH_IMM:</div><div class='ctx'> 		wqe-&gt;type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;</div><div class='del'>-		wqe-&gt;send.imm_data = wr-&gt;ex.imm_data;</div><div class='add'>+		wqe-&gt;send.imm_data = be32_to_cpu(wr-&gt;ex.imm_data);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_WR_SEND_WITH_INV:</div><div class='ctx'> 		wqe-&gt;type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;</div><div class='hunk'>@@ -2105,7 +2658,7 @@ static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr,</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='ctx'> 		wqe-&gt;type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;</div><div class='del'>-		wqe-&gt;rdma.imm_data = wr-&gt;ex.imm_data;</div><div class='add'>+		wqe-&gt;rdma.imm_data = be32_to_cpu(wr-&gt;ex.imm_data);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_WR_RDMA_READ:</div><div class='ctx'> 		wqe-&gt;type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;</div><div class='hunk'>@@ -2161,11 +2714,6 @@ static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr,</div><div class='ctx'> 	wqe-&gt;type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;</div><div class='ctx'> 	wqe-&gt;local_inv.inv_l_key = wr-&gt;ex.invalidate_rkey;</div><div class='ctx'> </div><div class='del'>-	/* Need unconditional fence for local invalidate</div><div class='del'>-	 * opcode to work as expected.</div><div class='del'>-	 */</div><div class='del'>-	wqe-&gt;flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;</div><div class='del'>-</div><div class='ctx'> 	if (wr-&gt;send_flags &amp; IB_SEND_SIGNALED)</div><div class='ctx'> 		wqe-&gt;flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;</div><div class='ctx'> 	if (wr-&gt;send_flags &amp; IB_SEND_SOLICITED)</div><div class='hunk'>@@ -2185,15 +2733,9 @@ static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr,</div><div class='ctx'> 	wqe-&gt;frmr.pbl_dma_ptr = qplib_frpl-&gt;hwq.pbl_dma_ptr[0];</div><div class='ctx'> 	wqe-&gt;frmr.page_list = mr-&gt;pages;</div><div class='ctx'> 	wqe-&gt;frmr.page_list_len = mr-&gt;npages;</div><div class='del'>-	wqe-&gt;frmr.levels = qplib_frpl-&gt;hwq.level + 1;</div><div class='add'>+	wqe-&gt;frmr.levels = qplib_frpl-&gt;hwq.level;</div><div class='ctx'> 	wqe-&gt;type = BNXT_QPLIB_SWQE_TYPE_REG_MR;</div><div class='ctx'> </div><div class='del'>-	/* Need unconditional fence for reg_mr</div><div class='del'>-	 * opcode to function as expected.</div><div class='del'>-	 */</div><div class='del'>-</div><div class='del'>-	wqe-&gt;flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;</div><div class='del'>-</div><div class='ctx'> 	if (wr-&gt;wr.send_flags &amp; IB_SEND_SIGNALED)</div><div class='ctx'> 		wqe-&gt;flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;</div><div class='ctx'> </div><div class='hunk'>@@ -2210,7 +2752,8 @@ static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr,</div><div class='ctx'> </div><div class='ctx'> 	wqe-&gt;frmr.l_key = wr-&gt;key;</div><div class='ctx'> 	wqe-&gt;frmr.length = wr-&gt;mr-&gt;length;</div><div class='del'>-	wqe-&gt;frmr.pbl_pg_sz_log = (wr-&gt;mr-&gt;page_size &gt;&gt; PAGE_SHIFT_4K) - 1;</div><div class='add'>+	wqe-&gt;frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE &gt;&gt; PAGE_SHIFT_4K);</div><div class='add'>+	wqe-&gt;frmr.pg_sz_log = ilog2(wr-&gt;mr-&gt;page_size &gt;&gt; PAGE_SHIFT_4K);</div><div class='ctx'> 	wqe-&gt;frmr.va = wr-&gt;mr-&gt;iova;</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -2232,8 +2775,8 @@ static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,</div><div class='ctx'> </div><div class='ctx'> 		if ((sge_len + wqe-&gt;inline_len) &gt;</div><div class='ctx'> 		    BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"Inline data size requested &gt; supported value");</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "Inline data size requested &gt; supported value");</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 		}</div><div class='ctx'> 		sge_len = wr-&gt;sg_list[i].length;</div><div class='hunk'>@@ -2280,21 +2823,18 @@ static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,</div><div class='ctx'> 				       struct bnxt_re_qp *qp,</div><div class='ctx'> 				       const struct ib_send_wr *wr)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_qplib_swqe wqe;</div><div class='ctx'> 	int rc = 0, payload_sz = 0;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;qp-&gt;sq_lock, flags);</div><div class='del'>-	memset(&amp;wqe, 0, sizeof(wqe));</div><div class='ctx'> 	while (wr) {</div><div class='del'>-		/* House keeping */</div><div class='del'>-		memset(&amp;wqe, 0, sizeof(wqe));</div><div class='add'>+		struct bnxt_qplib_swqe wqe = {};</div><div class='ctx'> </div><div class='ctx'> 		/* Common */</div><div class='ctx'> 		wqe.num_sge = wr-&gt;num_sge;</div><div class='ctx'> 		if (wr-&gt;num_sge &gt; qp-&gt;qplib_qp.sq.max_sge) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"Limit exceeded for Send SGEs");</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "Limit exceeded for Send SGEs");</div><div class='ctx'> 			rc = -EINVAL;</div><div class='ctx'> 			goto bad;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -2313,19 +2853,32 @@ static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,</div><div class='ctx'> 			rc = bnxt_qplib_post_send(&amp;qp-&gt;qplib_qp, &amp;wqe);</div><div class='ctx'> bad:</div><div class='ctx'> 		if (rc) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"Post send failed opcode = %#x rc = %d",</div><div class='del'>-				wr-&gt;opcode, rc);</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "Post send failed opcode = %#x rc = %d",</div><div class='add'>+				  wr-&gt;opcode, rc);</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='ctx'> 		wr = wr-&gt;next;</div><div class='ctx'> 	}</div><div class='ctx'> 	bnxt_qplib_post_send_db(&amp;qp-&gt;qplib_qp);</div><div class='del'>-	bnxt_ud_qp_hw_stall_workaround(qp);</div><div class='add'>+	if (!bnxt_qplib_is_chip_gen_p5_p7(qp-&gt;rdev-&gt;chip_ctx))</div><div class='add'>+		bnxt_ud_qp_hw_stall_workaround(qp);</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;qp-&gt;sq_lock, flags);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void bnxt_re_legacy_set_uc_fence(struct bnxt_qplib_swqe *wqe)</div><div class='add'>+{</div><div class='add'>+	/* Need unconditional fence for non-wire memory opcode</div><div class='add'>+	 * to work as expected.</div><div class='add'>+	 */</div><div class='add'>+	if (wqe-&gt;type == BNXT_QPLIB_SWQE_TYPE_LOCAL_INV ||</div><div class='add'>+	    wqe-&gt;type == BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR ||</div><div class='add'>+	    wqe-&gt;type == BNXT_QPLIB_SWQE_TYPE_REG_MR ||</div><div class='add'>+	    wqe-&gt;type == BNXT_QPLIB_SWQE_TYPE_BIND_MW)</div><div class='add'>+		wqe-&gt;flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,</div><div class='ctx'> 		      const struct ib_send_wr **bad_wr)</div><div class='ctx'> {</div><div class='hunk'>@@ -2342,8 +2895,8 @@ int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,</div><div class='ctx'> 		/* Common */</div><div class='ctx'> 		wqe.num_sge = wr-&gt;num_sge;</div><div class='ctx'> 		if (wr-&gt;num_sge &gt; qp-&gt;qplib_qp.sq.max_sge) {</div><div class='del'>-			dev_err(rdev_to_dev(qp-&gt;rdev),</div><div class='del'>-				"Limit exceeded for Send SGEs");</div><div class='add'>+			ibdev_err(&amp;qp-&gt;rdev-&gt;ibdev,</div><div class='add'>+				  "Limit exceeded for Send SGEs");</div><div class='ctx'> 			rc = -EINVAL;</div><div class='ctx'> 			goto bad;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -2374,7 +2927,7 @@ int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,</div><div class='ctx'> 			default:</div><div class='ctx'> 				break;</div><div class='ctx'> 			}</div><div class='del'>-			/* fall through */</div><div class='add'>+			fallthrough;</div><div class='ctx'> 		case IB_WR_SEND_WITH_INV:</div><div class='ctx'> 			rc = bnxt_re_build_send_wqe(qp, wr, &amp;wqe);</div><div class='ctx'> 			break;</div><div class='hunk'>@@ -2388,8 +2941,8 @@ int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,</div><div class='ctx'> 			rc = bnxt_re_build_atomic_wqe(wr, &amp;wqe);</div><div class='ctx'> 			break;</div><div class='ctx'> 		case IB_WR_RDMA_READ_WITH_INV:</div><div class='del'>-			dev_err(rdev_to_dev(qp-&gt;rdev),</div><div class='del'>-				"RDMA Read with Invalidate is not supported");</div><div class='add'>+			ibdev_err(&amp;qp-&gt;rdev-&gt;ibdev,</div><div class='add'>+				  "RDMA Read with Invalidate is not supported");</div><div class='ctx'> 			rc = -EINVAL;</div><div class='ctx'> 			goto bad;</div><div class='ctx'> 		case IB_WR_LOCAL_INV:</div><div class='hunk'>@@ -2400,25 +2953,29 @@ int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,</div><div class='ctx'> 			break;</div><div class='ctx'> 		default:</div><div class='ctx'> 			/* Unsupported WRs */</div><div class='del'>-			dev_err(rdev_to_dev(qp-&gt;rdev),</div><div class='del'>-				"WR (%#x) is not supported", wr-&gt;opcode);</div><div class='add'>+			ibdev_err(&amp;qp-&gt;rdev-&gt;ibdev,</div><div class='add'>+				  "WR (%#x) is not supported", wr-&gt;opcode);</div><div class='ctx'> 			rc = -EINVAL;</div><div class='ctx'> 			goto bad;</div><div class='ctx'> 		}</div><div class='del'>-		if (!rc)</div><div class='add'>+		if (!rc) {</div><div class='add'>+			if (!bnxt_qplib_is_chip_gen_p5_p7(qp-&gt;rdev-&gt;chip_ctx))</div><div class='add'>+				bnxt_re_legacy_set_uc_fence(&amp;wqe);</div><div class='ctx'> 			rc = bnxt_qplib_post_send(&amp;qp-&gt;qplib_qp, &amp;wqe);</div><div class='add'>+		}</div><div class='ctx'> bad:</div><div class='ctx'> 		if (rc) {</div><div class='del'>-			dev_err(rdev_to_dev(qp-&gt;rdev),</div><div class='del'>-				"post_send failed op:%#x qps = %#x rc = %d\n",</div><div class='del'>-				wr-&gt;opcode, qp-&gt;qplib_qp.state, rc);</div><div class='add'>+			ibdev_err(&amp;qp-&gt;rdev-&gt;ibdev,</div><div class='add'>+				  "post_send failed op:%#x qps = %#x rc = %d\n",</div><div class='add'>+				  wr-&gt;opcode, qp-&gt;qplib_qp.state, rc);</div><div class='ctx'> 			*bad_wr = wr;</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='ctx'> 		wr = wr-&gt;next;</div><div class='ctx'> 	}</div><div class='ctx'> 	bnxt_qplib_post_send_db(&amp;qp-&gt;qplib_qp);</div><div class='del'>-	bnxt_ud_qp_hw_stall_workaround(qp);</div><div class='add'>+	if (!bnxt_qplib_is_chip_gen_p5_p7(qp-&gt;rdev-&gt;chip_ctx))</div><div class='add'>+		bnxt_ud_qp_hw_stall_workaround(qp);</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;qp-&gt;sq_lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	return rc;</div><div class='hunk'>@@ -2431,7 +2988,6 @@ static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,</div><div class='ctx'> 	struct bnxt_qplib_swqe wqe;</div><div class='ctx'> 	int rc = 0;</div><div class='ctx'> </div><div class='del'>-	memset(&amp;wqe, 0, sizeof(wqe));</div><div class='ctx'> 	while (wr) {</div><div class='ctx'> 		/* House keeping */</div><div class='ctx'> 		memset(&amp;wqe, 0, sizeof(wqe));</div><div class='hunk'>@@ -2439,8 +2995,8 @@ static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,</div><div class='ctx'> 		/* Common */</div><div class='ctx'> 		wqe.num_sge = wr-&gt;num_sge;</div><div class='ctx'> 		if (wr-&gt;num_sge &gt; qp-&gt;qplib_qp.rq.max_sge) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"Limit exceeded for Receive SGEs");</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "Limit exceeded for Receive SGEs");</div><div class='ctx'> 			rc = -EINVAL;</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -2476,8 +3032,8 @@ int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,</div><div class='ctx'> 		/* Common */</div><div class='ctx'> 		wqe.num_sge = wr-&gt;num_sge;</div><div class='ctx'> 		if (wr-&gt;num_sge &gt; qp-&gt;qplib_qp.rq.max_sge) {</div><div class='del'>-			dev_err(rdev_to_dev(qp-&gt;rdev),</div><div class='del'>-				"Limit exceeded for Receive SGEs");</div><div class='add'>+			ibdev_err(&amp;qp-&gt;rdev-&gt;ibdev,</div><div class='add'>+				  "Limit exceeded for Receive SGEs");</div><div class='ctx'> 			rc = -EINVAL;</div><div class='ctx'> 			*bad_wr = wr;</div><div class='ctx'> 			break;</div><div class='hunk'>@@ -2516,81 +3072,103 @@ int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static struct bnxt_qplib_nq *bnxt_re_get_nq(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	int min, indx;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;rdev-&gt;nqr-&gt;load_lock);</div><div class='add'>+	for (indx = 0, min = 0; indx &lt; (rdev-&gt;nqr-&gt;num_msix - 1); indx++) {</div><div class='add'>+		if (rdev-&gt;nqr-&gt;nq[min].load &gt; rdev-&gt;nqr-&gt;nq[indx].load)</div><div class='add'>+			min = indx;</div><div class='add'>+	}</div><div class='add'>+	rdev-&gt;nqr-&gt;nq[min].load++;</div><div class='add'>+	mutex_unlock(&amp;rdev-&gt;nqr-&gt;load_lock);</div><div class='add'>+</div><div class='add'>+	return &amp;rdev-&gt;nqr-&gt;nq[min];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq)</div><div class='add'>+{</div><div class='add'>+	mutex_lock(&amp;rdev-&gt;nqr-&gt;load_lock);</div><div class='add'>+	nq-&gt;load--;</div><div class='add'>+	mutex_unlock(&amp;rdev-&gt;nqr-&gt;load_lock);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /* Completion Queues */</div><div class='ctx'> int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	int rc;</div><div class='del'>-	struct bnxt_re_cq *cq;</div><div class='add'>+	struct bnxt_qplib_chip_ctx *cctx;</div><div class='ctx'> 	struct bnxt_qplib_nq *nq;</div><div class='ctx'> 	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct bnxt_re_cq *cq;</div><div class='ctx'> </div><div class='ctx'> 	cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);</div><div class='ctx'> 	rdev = cq-&gt;rdev;</div><div class='ctx'> 	nq = cq-&gt;qplib_cq.nq;</div><div class='add'>+	cctx = rdev-&gt;chip_ctx;</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_destroy_cq(&amp;rdev-&gt;qplib_res, &amp;cq-&gt;qplib_cq);</div><div class='del'>-	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to destroy HW CQ");</div><div class='del'>-		return rc;</div><div class='add'>+	if (cctx-&gt;modes.toggle_bits &amp; BNXT_QPLIB_CQ_TOGGLE_BIT) {</div><div class='add'>+		free_page((unsigned long)cq-&gt;uctx_cq_page);</div><div class='add'>+		hash_del(&amp;cq-&gt;hash_entry);</div><div class='ctx'> 	}</div><div class='del'>-	if (!IS_ERR_OR_NULL(cq-&gt;umem))</div><div class='del'>-		ib_umem_release(cq-&gt;umem);</div><div class='add'>+	bnxt_qplib_destroy_cq(&amp;rdev-&gt;qplib_res, &amp;cq-&gt;qplib_cq);</div><div class='ctx'> </div><div class='del'>-	atomic_dec(&amp;rdev-&gt;cq_count);</div><div class='del'>-	nq-&gt;budget--;</div><div class='del'>-	kfree(cq-&gt;cql);</div><div class='del'>-	kfree(cq);</div><div class='add'>+	bnxt_re_put_nq(rdev, nq);</div><div class='add'>+	ib_umem_release(cq-&gt;umem);</div><div class='ctx'> </div><div class='add'>+	atomic_dec(&amp;rdev-&gt;stats.res.cq_count);</div><div class='add'>+	kfree(cq-&gt;cql);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,</div><div class='del'>-				const struct ib_cq_init_attr *attr,</div><div class='del'>-				struct ib_udata *udata)</div><div class='add'>+int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		      struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);</div><div class='del'>-	struct bnxt_qplib_dev_attr *dev_attr = &amp;rdev-&gt;dev_attr;</div><div class='del'>-	struct bnxt_re_cq *cq = NULL;</div><div class='del'>-	int rc, entries;</div><div class='add'>+	struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);</div><div class='add'>+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq-&gt;device, ibdev);</div><div class='add'>+	struct ib_udata *udata = &amp;attrs-&gt;driver_udata;</div><div class='add'>+	struct bnxt_re_ucontext *uctx =</div><div class='add'>+		rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);</div><div class='add'>+	struct bnxt_qplib_dev_attr *dev_attr = rdev-&gt;dev_attr;</div><div class='add'>+	struct bnxt_qplib_chip_ctx *cctx;</div><div class='ctx'> 	int cqe = attr-&gt;cqe;</div><div class='del'>-	struct bnxt_qplib_nq *nq = NULL;</div><div class='del'>-	unsigned int nq_alloc_cnt;</div><div class='add'>+	int rc, entries;</div><div class='add'>+	u32 active_cqs;</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;flags)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	/* Validate CQ fields */</div><div class='ctx'> 	if (cqe &lt; 1 || cqe &gt; dev_attr-&gt;max_cq_wqes) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to create CQ -max exceeded");</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to create CQ -max exceeded");</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	}</div><div class='del'>-	cq = kzalloc(sizeof(*cq), GFP_KERNEL);</div><div class='del'>-	if (!cq)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='ctx'> 	cq-&gt;rdev = rdev;</div><div class='add'>+	cctx = rdev-&gt;chip_ctx;</div><div class='ctx'> 	cq-&gt;qplib_cq.cq_handle = (u64)(unsigned long)(&amp;cq-&gt;qplib_cq);</div><div class='ctx'> </div><div class='del'>-	entries = roundup_pow_of_two(cqe + 1);</div><div class='add'>+	entries = bnxt_re_init_depth(cqe + 1, uctx);</div><div class='ctx'> 	if (entries &gt; dev_attr-&gt;max_cq_wqes + 1)</div><div class='ctx'> 		entries = dev_attr-&gt;max_cq_wqes + 1;</div><div class='ctx'> </div><div class='add'>+	cq-&gt;qplib_cq.sg_info.pgsize = PAGE_SIZE;</div><div class='add'>+	cq-&gt;qplib_cq.sg_info.pgshft = PAGE_SHIFT;</div><div class='ctx'> 	if (udata) {</div><div class='ctx'> 		struct bnxt_re_cq_req req;</div><div class='del'>-		struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(</div><div class='del'>-			udata, struct bnxt_re_ucontext, ib_uctx);</div><div class='ctx'> 		if (ib_copy_from_udata(&amp;req, udata, sizeof(req))) {</div><div class='ctx'> 			rc = -EFAULT;</div><div class='ctx'> 			goto fail;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		cq-&gt;umem = ib_umem_get(udata, req.cq_va,</div><div class='add'>+		cq-&gt;umem = ib_umem_get(&amp;rdev-&gt;ibdev, req.cq_va,</div><div class='ctx'> 				       entries * sizeof(struct cq_base),</div><div class='del'>-				       IB_ACCESS_LOCAL_WRITE, 1);</div><div class='add'>+				       IB_ACCESS_LOCAL_WRITE);</div><div class='ctx'> 		if (IS_ERR(cq-&gt;umem)) {</div><div class='ctx'> 			rc = PTR_ERR(cq-&gt;umem);</div><div class='ctx'> 			goto fail;</div><div class='ctx'> 		}</div><div class='del'>-		cq-&gt;qplib_cq.sg_info.sglist = cq-&gt;umem-&gt;sg_head.sgl;</div><div class='del'>-		cq-&gt;qplib_cq.sg_info.npages = ib_umem_num_pages(cq-&gt;umem);</div><div class='del'>-		cq-&gt;qplib_cq.sg_info.nmap = cq-&gt;umem-&gt;nmap;</div><div class='add'>+		cq-&gt;qplib_cq.sg_info.umem = cq-&gt;umem;</div><div class='ctx'> 		cq-&gt;qplib_cq.dpi = &amp;uctx-&gt;dpi;</div><div class='ctx'> 	} else {</div><div class='ctx'> 		cq-&gt;max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);</div><div class='hunk'>@@ -2603,53 +3181,159 @@ struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,</div><div class='ctx'> </div><div class='ctx'> 		cq-&gt;qplib_cq.dpi = &amp;rdev-&gt;dpi_privileged;</div><div class='ctx'> 	}</div><div class='del'>-	/*</div><div class='del'>-	 * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a</div><div class='del'>-	 * used for getting the NQ index.</div><div class='del'>-	 */</div><div class='del'>-	nq_alloc_cnt = atomic_inc_return(&amp;rdev-&gt;nq_alloc_cnt);</div><div class='del'>-	nq = &amp;rdev-&gt;nq[nq_alloc_cnt % (rdev-&gt;num_msix - 1)];</div><div class='ctx'> 	cq-&gt;qplib_cq.max_wqe = entries;</div><div class='del'>-	cq-&gt;qplib_cq.cnq_hw_ring_id = nq-&gt;ring_id;</div><div class='del'>-	cq-&gt;qplib_cq.nq	= nq;</div><div class='add'>+	cq-&gt;qplib_cq.coalescing = &amp;rdev-&gt;cq_coalescing;</div><div class='add'>+	cq-&gt;qplib_cq.nq = bnxt_re_get_nq(rdev);</div><div class='add'>+	cq-&gt;qplib_cq.cnq_hw_ring_id = cq-&gt;qplib_cq.nq-&gt;ring_id;</div><div class='ctx'> </div><div class='ctx'> 	rc = bnxt_qplib_create_cq(&amp;rdev-&gt;qplib_res, &amp;cq-&gt;qplib_cq);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to create HW CQ");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to create HW CQ");</div><div class='ctx'> 		goto fail;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	cq-&gt;ib_cq.cqe = entries;</div><div class='ctx'> 	cq-&gt;cq_period = cq-&gt;qplib_cq.period;</div><div class='del'>-	nq-&gt;budget++;</div><div class='ctx'> </div><div class='del'>-	atomic_inc(&amp;rdev-&gt;cq_count);</div><div class='add'>+	active_cqs = atomic_inc_return(&amp;rdev-&gt;stats.res.cq_count);</div><div class='add'>+	if (active_cqs &gt; rdev-&gt;stats.res.cq_watermark)</div><div class='add'>+		rdev-&gt;stats.res.cq_watermark = active_cqs;</div><div class='ctx'> 	spin_lock_init(&amp;cq-&gt;cq_lock);</div><div class='ctx'> </div><div class='ctx'> 	if (udata) {</div><div class='del'>-		struct bnxt_re_cq_resp resp;</div><div class='add'>+		struct bnxt_re_cq_resp resp = {};</div><div class='ctx'> </div><div class='add'>+		if (cctx-&gt;modes.toggle_bits &amp; BNXT_QPLIB_CQ_TOGGLE_BIT) {</div><div class='add'>+			hash_add(rdev-&gt;cq_hash, &amp;cq-&gt;hash_entry, cq-&gt;qplib_cq.id);</div><div class='add'>+			/* Allocate a page */</div><div class='add'>+			cq-&gt;uctx_cq_page = (void *)get_zeroed_page(GFP_KERNEL);</div><div class='add'>+			if (!cq-&gt;uctx_cq_page) {</div><div class='add'>+				rc = -ENOMEM;</div><div class='add'>+				goto c2fail;</div><div class='add'>+			}</div><div class='add'>+			resp.comp_mask |= BNXT_RE_CQ_TOGGLE_PAGE_SUPPORT;</div><div class='add'>+		}</div><div class='ctx'> 		resp.cqid = cq-&gt;qplib_cq.id;</div><div class='ctx'> 		resp.tail = cq-&gt;qplib_cq.hwq.cons;</div><div class='ctx'> 		resp.phase = cq-&gt;qplib_cq.period;</div><div class='ctx'> 		resp.rsvd = 0;</div><div class='del'>-		rc = ib_copy_to_udata(udata, &amp;resp, sizeof(resp));</div><div class='add'>+		rc = ib_copy_to_udata(udata, &amp;resp, min(sizeof(resp), udata-&gt;outlen));</div><div class='ctx'> 		if (rc) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev), "Failed to copy CQ udata");</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev, "Failed to copy CQ udata");</div><div class='ctx'> 			bnxt_qplib_destroy_cq(&amp;rdev-&gt;qplib_res, &amp;cq-&gt;qplib_cq);</div><div class='del'>-			goto c2fail;</div><div class='add'>+			goto free_mem;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return &amp;cq-&gt;ib_cq;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='add'>+free_mem:</div><div class='add'>+	free_page((unsigned long)cq-&gt;uctx_cq_page);</div><div class='ctx'> c2fail:</div><div class='del'>-	if (udata)</div><div class='del'>-		ib_umem_release(cq-&gt;umem);</div><div class='add'>+	ib_umem_release(cq-&gt;umem);</div><div class='ctx'> fail:</div><div class='ctx'> 	kfree(cq-&gt;cql);</div><div class='del'>-	kfree(cq);</div><div class='del'>-	return ERR_PTR(rc);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_dev *rdev = cq-&gt;rdev;</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_resize_cq_complete(&amp;rdev-&gt;qplib_res, &amp;cq-&gt;qplib_cq);</div><div class='add'>+</div><div class='add'>+	cq-&gt;qplib_cq.max_wqe = cq-&gt;resize_cqe;</div><div class='add'>+	if (cq-&gt;resize_umem) {</div><div class='add'>+		ib_umem_release(cq-&gt;umem);</div><div class='add'>+		cq-&gt;umem = cq-&gt;resize_umem;</div><div class='add'>+		cq-&gt;resize_umem = NULL;</div><div class='add'>+		cq-&gt;resize_cqe = 0;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bnxt_re_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_sg_info sg_info = {};</div><div class='add'>+	struct bnxt_qplib_dpi *orig_dpi = NULL;</div><div class='add'>+	struct bnxt_qplib_dev_attr *dev_attr;</div><div class='add'>+	struct bnxt_re_ucontext *uctx = NULL;</div><div class='add'>+	struct bnxt_re_resize_cq_req req;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct bnxt_re_cq *cq;</div><div class='add'>+	int rc, entries;</div><div class='add'>+</div><div class='add'>+	cq =  container_of(ibcq, struct bnxt_re_cq, ib_cq);</div><div class='add'>+	rdev = cq-&gt;rdev;</div><div class='add'>+	dev_attr = rdev-&gt;dev_attr;</div><div class='add'>+	if (!ibcq-&gt;uobject) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Kernel CQ Resize not supported");</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (cq-&gt;resize_umem) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Resize CQ %#x failed - Busy",</div><div class='add'>+			  cq-&gt;qplib_cq.id);</div><div class='add'>+		return -EBUSY;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Check the requested cq depth out of supported depth */</div><div class='add'>+	if (cqe &lt; 1 || cqe &gt; dev_attr-&gt;max_cq_wqes) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Resize CQ %#x failed - out of range cqe %d",</div><div class='add'>+			  cq-&gt;qplib_cq.id, cqe);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);</div><div class='add'>+	entries = bnxt_re_init_depth(cqe + 1, uctx);</div><div class='add'>+	if (entries &gt; dev_attr-&gt;max_cq_wqes + 1)</div><div class='add'>+		entries = dev_attr-&gt;max_cq_wqes + 1;</div><div class='add'>+</div><div class='add'>+	/* uverbs consumer */</div><div class='add'>+	if (ib_copy_from_udata(&amp;req, udata, sizeof(req))) {</div><div class='add'>+		rc = -EFAULT;</div><div class='add'>+		goto fail;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cq-&gt;resize_umem = ib_umem_get(&amp;rdev-&gt;ibdev, req.cq_va,</div><div class='add'>+				      entries * sizeof(struct cq_base),</div><div class='add'>+				      IB_ACCESS_LOCAL_WRITE);</div><div class='add'>+	if (IS_ERR(cq-&gt;resize_umem)) {</div><div class='add'>+		rc = PTR_ERR(cq-&gt;resize_umem);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "%s: ib_umem_get failed! rc = %pe\n",</div><div class='add'>+			  __func__, cq-&gt;resize_umem);</div><div class='add'>+		cq-&gt;resize_umem = NULL;</div><div class='add'>+		goto fail;</div><div class='add'>+	}</div><div class='add'>+	cq-&gt;resize_cqe = entries;</div><div class='add'>+	memcpy(&amp;sg_info, &amp;cq-&gt;qplib_cq.sg_info, sizeof(sg_info));</div><div class='add'>+	orig_dpi = cq-&gt;qplib_cq.dpi;</div><div class='add'>+</div><div class='add'>+	cq-&gt;qplib_cq.sg_info.umem = cq-&gt;resize_umem;</div><div class='add'>+	cq-&gt;qplib_cq.sg_info.pgsize = PAGE_SIZE;</div><div class='add'>+	cq-&gt;qplib_cq.sg_info.pgshft = PAGE_SHIFT;</div><div class='add'>+	cq-&gt;qplib_cq.dpi = &amp;uctx-&gt;dpi;</div><div class='add'>+</div><div class='add'>+	rc = bnxt_qplib_resize_cq(&amp;rdev-&gt;qplib_res, &amp;cq-&gt;qplib_cq, entries);</div><div class='add'>+	if (rc) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Resize HW CQ %#x failed!",</div><div class='add'>+			  cq-&gt;qplib_cq.id);</div><div class='add'>+		goto fail;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cq-&gt;ib_cq.cqe = cq-&gt;resize_cqe;</div><div class='add'>+	atomic_inc(&amp;rdev-&gt;stats.res.resize_count);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+fail:</div><div class='add'>+	if (cq-&gt;resize_umem) {</div><div class='add'>+		ib_umem_release(cq-&gt;resize_umem);</div><div class='add'>+		cq-&gt;resize_umem = NULL;</div><div class='add'>+		cq-&gt;resize_cqe = 0;</div><div class='add'>+		memcpy(&amp;cq-&gt;qplib_cq.sg_info, &amp;sg_info, sizeof(sg_info));</div><div class='add'>+		cq-&gt;qplib_cq.dpi = orig_dpi;</div><div class='add'>+	}</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static u8 __req_to_ib_wc_status(u8 qstatus)</div><div class='hunk'>@@ -2868,54 +3552,50 @@ static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev,</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *qp1_qp,</div><div class='add'>+static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *gsi_qp,</div><div class='ctx'> 					 struct bnxt_qplib_cqe *cqe)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_re_dev *rdev = qp1_qp-&gt;rdev;</div><div class='add'>+	struct bnxt_re_dev *rdev = gsi_qp-&gt;rdev;</div><div class='ctx'> 	struct bnxt_re_sqp_entries *sqp_entry = NULL;</div><div class='del'>-	struct bnxt_re_qp *qp = rdev-&gt;qp1_sqp;</div><div class='add'>+	struct bnxt_re_qp *gsi_sqp = rdev-&gt;gsi_ctx.gsi_sqp;</div><div class='add'>+	dma_addr_t shrq_hdr_buf_map;</div><div class='add'>+	struct ib_sge s_sge[2] = {};</div><div class='add'>+	struct ib_sge r_sge[2] = {};</div><div class='add'>+	struct bnxt_re_ah *gsi_sah;</div><div class='add'>+	struct ib_recv_wr rwr = {};</div><div class='add'>+	dma_addr_t rq_hdr_buf_map;</div><div class='add'>+	struct ib_ud_wr udwr = {};</div><div class='ctx'> 	struct ib_send_wr *swr;</div><div class='del'>-	struct ib_ud_wr udwr;</div><div class='del'>-	struct ib_recv_wr rwr;</div><div class='add'>+	u32 skip_bytes = 0;</div><div class='ctx'> 	int pkt_type = 0;</div><div class='del'>-	u32 tbl_idx;</div><div class='ctx'> 	void *rq_hdr_buf;</div><div class='del'>-	dma_addr_t rq_hdr_buf_map;</div><div class='del'>-	dma_addr_t shrq_hdr_buf_map;</div><div class='ctx'> 	u32 offset = 0;</div><div class='del'>-	u32 skip_bytes = 0;</div><div class='del'>-	struct ib_sge s_sge[2];</div><div class='del'>-	struct ib_sge r_sge[2];</div><div class='add'>+	u32 tbl_idx;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	memset(&amp;udwr, 0, sizeof(udwr));</div><div class='del'>-	memset(&amp;rwr, 0, sizeof(rwr));</div><div class='del'>-	memset(&amp;s_sge, 0, sizeof(s_sge));</div><div class='del'>-	memset(&amp;r_sge, 0, sizeof(r_sge));</div><div class='del'>-</div><div class='ctx'> 	swr = &amp;udwr.wr;</div><div class='ctx'> 	tbl_idx = cqe-&gt;wr_id;</div><div class='ctx'> </div><div class='del'>-	rq_hdr_buf = qp1_qp-&gt;qplib_qp.rq_hdr_buf +</div><div class='del'>-			(tbl_idx * qp1_qp-&gt;qplib_qp.rq_hdr_buf_size);</div><div class='del'>-	rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&amp;qp1_qp-&gt;qplib_qp,</div><div class='add'>+	rq_hdr_buf = gsi_qp-&gt;qplib_qp.rq_hdr_buf +</div><div class='add'>+			(tbl_idx * gsi_qp-&gt;qplib_qp.rq_hdr_buf_size);</div><div class='add'>+	rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&amp;gsi_qp-&gt;qplib_qp,</div><div class='ctx'> 							  tbl_idx);</div><div class='ctx'> </div><div class='ctx'> 	/* Shadow QP header buffer */</div><div class='del'>-	shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&amp;qp-&gt;qplib_qp,</div><div class='add'>+	shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&amp;gsi_qp-&gt;qplib_qp,</div><div class='ctx'> 							    tbl_idx);</div><div class='del'>-	sqp_entry = &amp;rdev-&gt;sqp_tbl[tbl_idx];</div><div class='add'>+	sqp_entry = &amp;rdev-&gt;gsi_ctx.sqp_tbl[tbl_idx];</div><div class='ctx'> </div><div class='ctx'> 	/* Store this cqe */</div><div class='ctx'> 	memcpy(&amp;sqp_entry-&gt;cqe, cqe, sizeof(struct bnxt_qplib_cqe));</div><div class='del'>-	sqp_entry-&gt;qp1_qp = qp1_qp;</div><div class='add'>+	sqp_entry-&gt;qp1_qp = gsi_qp;</div><div class='ctx'> </div><div class='ctx'> 	/* Find packet type from the cqe */</div><div class='ctx'> </div><div class='ctx'> 	pkt_type = bnxt_re_check_packet_type(cqe-&gt;raweth_qp1_flags,</div><div class='ctx'> 					     cqe-&gt;raweth_qp1_flags2);</div><div class='ctx'> 	if (pkt_type &lt; 0) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Invalid packet\n");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Invalid packet\n");</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -2962,10 +3642,10 @@ static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *qp1_qp,</div><div class='ctx'> 	rwr.wr_id = tbl_idx;</div><div class='ctx'> 	rwr.next = NULL;</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_re_post_recv_shadow_qp(rdev, qp, &amp;rwr);</div><div class='add'>+	rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &amp;rwr);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev),</div><div class='del'>-			"Failed to post Rx buffers to shadow QP");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to post Rx buffers to shadow QP");</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -2974,15 +3654,13 @@ static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *qp1_qp,</div><div class='ctx'> 	swr-&gt;wr_id = tbl_idx;</div><div class='ctx'> 	swr-&gt;opcode = IB_WR_SEND;</div><div class='ctx'> 	swr-&gt;next = NULL;</div><div class='del'>-</div><div class='del'>-	udwr.ah = &amp;rdev-&gt;sqp_ah-&gt;ib_ah;</div><div class='del'>-	udwr.remote_qpn = rdev-&gt;qp1_sqp-&gt;qplib_qp.id;</div><div class='del'>-	udwr.remote_qkey = rdev-&gt;qp1_sqp-&gt;qplib_qp.qkey;</div><div class='add'>+	gsi_sah = rdev-&gt;gsi_ctx.gsi_sah;</div><div class='add'>+	udwr.ah = &amp;gsi_sah-&gt;ib_ah;</div><div class='add'>+	udwr.remote_qpn = gsi_sqp-&gt;qplib_qp.id;</div><div class='add'>+	udwr.remote_qkey = gsi_sqp-&gt;qplib_qp.qkey;</div><div class='ctx'> </div><div class='ctx'> 	/* post data received  in the send queue */</div><div class='del'>-	rc = bnxt_re_post_send_shadow_qp(rdev, qp, swr);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='add'>+	return bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,</div><div class='hunk'>@@ -2993,6 +3671,19 @@ static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,</div><div class='ctx'> 	wc-&gt;wc_flags |= IB_WC_GRH;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev,</div><div class='add'>+					u16 vlan_id)</div><div class='add'>+{</div><div class='add'>+	/*</div><div class='add'>+	 * Check if the vlan is configured in the host.  If not configured, it</div><div class='add'>+	 * can be a transparent VLAN. So dont report the vlan id.</div><div class='add'>+	 */</div><div class='add'>+	if (!__vlan_find_dev_deep_rcu(rdev-&gt;netdev,</div><div class='add'>+				      htons(ETH_P_8021Q), vlan_id))</div><div class='add'>+		return false;</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static bool bnxt_re_is_vlan_pkt(struct bnxt_qplib_cqe *orig_cqe,</div><div class='ctx'> 				u16 *vid, u8 *sl)</div><div class='ctx'> {</div><div class='hunk'>@@ -3034,12 +3725,12 @@ static void bnxt_re_process_res_rc_wc(struct ib_wc *wc,</div><div class='ctx'> 		wc-&gt;opcode = IB_WC_RECV_RDMA_WITH_IMM;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *qp,</div><div class='add'>+static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp,</div><div class='ctx'> 					     struct ib_wc *wc,</div><div class='ctx'> 					     struct bnxt_qplib_cqe *cqe)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_re_dev *rdev = qp-&gt;rdev;</div><div class='del'>-	struct bnxt_re_qp *qp1_qp = NULL;</div><div class='add'>+	struct bnxt_re_dev *rdev = gsi_sqp-&gt;rdev;</div><div class='add'>+	struct bnxt_re_qp *gsi_qp = NULL;</div><div class='ctx'> 	struct bnxt_qplib_cqe *orig_cqe = NULL;</div><div class='ctx'> 	struct bnxt_re_sqp_entries *sqp_entry = NULL;</div><div class='ctx'> 	int nw_type;</div><div class='hunk'>@@ -3049,21 +3740,23 @@ static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *qp,</div><div class='ctx'> </div><div class='ctx'> 	tbl_idx = cqe-&gt;wr_id;</div><div class='ctx'> </div><div class='del'>-	sqp_entry = &amp;rdev-&gt;sqp_tbl[tbl_idx];</div><div class='del'>-	qp1_qp = sqp_entry-&gt;qp1_qp;</div><div class='add'>+	sqp_entry = &amp;rdev-&gt;gsi_ctx.sqp_tbl[tbl_idx];</div><div class='add'>+	gsi_qp = sqp_entry-&gt;qp1_qp;</div><div class='ctx'> 	orig_cqe = &amp;sqp_entry-&gt;cqe;</div><div class='ctx'> </div><div class='ctx'> 	wc-&gt;wr_id = sqp_entry-&gt;wrid;</div><div class='ctx'> 	wc-&gt;byte_len = orig_cqe-&gt;length;</div><div class='del'>-	wc-&gt;qp = &amp;qp1_qp-&gt;ib_qp;</div><div class='add'>+	wc-&gt;qp = &amp;gsi_qp-&gt;ib_qp;</div><div class='ctx'> </div><div class='del'>-	wc-&gt;ex.imm_data = orig_cqe-&gt;immdata;</div><div class='add'>+	wc-&gt;ex.imm_data = cpu_to_be32(orig_cqe-&gt;immdata);</div><div class='ctx'> 	wc-&gt;src_qp = orig_cqe-&gt;src_qp;</div><div class='ctx'> 	memcpy(wc-&gt;smac, orig_cqe-&gt;smac, ETH_ALEN);</div><div class='ctx'> 	if (bnxt_re_is_vlan_pkt(orig_cqe, &amp;vlan_id, &amp;sl)) {</div><div class='del'>-		wc-&gt;vlan_id = vlan_id;</div><div class='del'>-		wc-&gt;sl = sl;</div><div class='del'>-		wc-&gt;wc_flags |= IB_WC_WITH_VLAN;</div><div class='add'>+		if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {</div><div class='add'>+			wc-&gt;vlan_id = vlan_id;</div><div class='add'>+			wc-&gt;sl = sl;</div><div class='add'>+			wc-&gt;wc_flags |= IB_WC_WITH_VLAN;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> 	wc-&gt;port_num = 1;</div><div class='ctx'> 	wc-&gt;vendor_err = orig_cqe-&gt;status;</div><div class='hunk'>@@ -3084,8 +3777,11 @@ static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp,</div><div class='ctx'> 				      struct ib_wc *wc,</div><div class='ctx'> 				      struct bnxt_qplib_cqe *cqe)</div><div class='ctx'> {</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	u16 vlan_id = 0;</div><div class='ctx'> 	u8 nw_type;</div><div class='ctx'> </div><div class='add'>+	rdev = qp-&gt;rdev;</div><div class='ctx'> 	wc-&gt;opcode = IB_WC_RECV;</div><div class='ctx'> 	wc-&gt;status = __rc_to_ib_wc_status(cqe-&gt;status);</div><div class='ctx'> </div><div class='hunk'>@@ -3097,9 +3793,12 @@ static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp,</div><div class='ctx'> 		memcpy(wc-&gt;smac, cqe-&gt;smac, ETH_ALEN);</div><div class='ctx'> 		wc-&gt;wc_flags |= IB_WC_WITH_SMAC;</div><div class='ctx'> 		if (cqe-&gt;flags &amp; CQ_RES_UD_FLAGS_META_FORMAT_VLAN) {</div><div class='del'>-			wc-&gt;vlan_id = (cqe-&gt;cfa_meta &amp; 0xFFF);</div><div class='del'>-			if (wc-&gt;vlan_id &lt; 0x1000)</div><div class='del'>-				wc-&gt;wc_flags |= IB_WC_WITH_VLAN;</div><div class='add'>+			vlan_id = (cqe-&gt;cfa_meta &amp; 0xFFF);</div><div class='add'>+		}</div><div class='add'>+		/* Mark only if vlan_id is non zero */</div><div class='add'>+		if (vlan_id &amp;&amp; bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {</div><div class='add'>+			wc-&gt;vlan_id = vlan_id;</div><div class='add'>+			wc-&gt;wc_flags |= IB_WC_WITH_VLAN;</div><div class='ctx'> 		}</div><div class='ctx'> 		nw_type = (cqe-&gt;flags &amp; CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK) &gt;&gt;</div><div class='ctx'> 			   CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT;</div><div class='hunk'>@@ -3113,18 +3812,18 @@ static int send_phantom_wqe(struct bnxt_re_qp *qp)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_qp *lib_qp = &amp;qp-&gt;qplib_qp;</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	int rc = 0;</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;qp-&gt;sq_lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	rc = bnxt_re_bind_fence_mw(lib_qp);</div><div class='ctx'> 	if (!rc) {</div><div class='ctx'> 		lib_qp-&gt;sq.phantom_wqe_cnt++;</div><div class='del'>-		dev_dbg(&amp;lib_qp-&gt;sq.hwq.pdev-&gt;dev,</div><div class='del'>-			"qp %#x sq-&gt;prod %#x sw_prod %#x phantom_wqe_cnt %d\n",</div><div class='del'>-			lib_qp-&gt;id, lib_qp-&gt;sq.hwq.prod,</div><div class='del'>-			HWQ_CMP(lib_qp-&gt;sq.hwq.prod, &amp;lib_qp-&gt;sq.hwq),</div><div class='del'>-			lib_qp-&gt;sq.phantom_wqe_cnt);</div><div class='add'>+		ibdev_dbg(&amp;qp-&gt;rdev-&gt;ibdev,</div><div class='add'>+			  "qp %#x sq-&gt;prod %#x sw_prod %#x phantom_wqe_cnt %d\n",</div><div class='add'>+			  lib_qp-&gt;id, lib_qp-&gt;sq.hwq.prod,</div><div class='add'>+			  HWQ_CMP(lib_qp-&gt;sq.hwq.prod, &amp;lib_qp-&gt;sq.hwq),</div><div class='add'>+			  lib_qp-&gt;sq.phantom_wqe_cnt);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	spin_unlock_irqrestore(&amp;qp-&gt;sq_lock, flags);</div><div class='hunk'>@@ -3134,7 +3833,7 @@ static int send_phantom_wqe(struct bnxt_re_qp *qp)</div><div class='ctx'> int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);</div><div class='del'>-	struct bnxt_re_qp *qp;</div><div class='add'>+	struct bnxt_re_qp *qp, *sh_qp;</div><div class='ctx'> 	struct bnxt_qplib_cqe *cqe;</div><div class='ctx'> 	int i, ncqe, budget;</div><div class='ctx'> 	struct bnxt_qplib_q *sq;</div><div class='hunk'>@@ -3143,11 +3842,20 @@ int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)</div><div class='ctx'> 	struct bnxt_re_sqp_entries *sqp_entry = NULL;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> </div><div class='add'>+	/* User CQ; the only processing we do is to</div><div class='add'>+	 * complete any pending CQ resize operation.</div><div class='add'>+	 */</div><div class='add'>+	if (cq-&gt;umem) {</div><div class='add'>+		if (cq-&gt;resize_umem)</div><div class='add'>+			bnxt_re_resize_cq_complete(cq);</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	spin_lock_irqsave(&amp;cq-&gt;cq_lock, flags);</div><div class='ctx'> 	budget = min_t(u32, num_entries, cq-&gt;max_cql);</div><div class='ctx'> 	num_entries = budget;</div><div class='ctx'> 	if (!cq-&gt;cql) {</div><div class='del'>-		dev_err(rdev_to_dev(cq-&gt;rdev), "POLL CQ : no CQL to use");</div><div class='add'>+		ibdev_err(&amp;cq-&gt;rdev-&gt;ibdev, "POLL CQ : no CQL to use");</div><div class='ctx'> 		goto exit;</div><div class='ctx'> 	}</div><div class='ctx'> 	cqe = &amp;cq-&gt;cql[0];</div><div class='hunk'>@@ -3160,8 +3868,8 @@ int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)</div><div class='ctx'> 				qp = container_of(lib_qp,</div><div class='ctx'> 						  struct bnxt_re_qp, qplib_qp);</div><div class='ctx'> 				if (send_phantom_wqe(qp) == -ENOMEM)</div><div class='del'>-					dev_err(rdev_to_dev(cq-&gt;rdev),</div><div class='del'>-						"Phantom failed! Scheduled to send again\n");</div><div class='add'>+					ibdev_err(&amp;cq-&gt;rdev-&gt;ibdev,</div><div class='add'>+						  "Phantom failed! Scheduled to send again\n");</div><div class='ctx'> 				else</div><div class='ctx'> 					sq-&gt;send_phantom = false;</div><div class='ctx'> 			}</div><div class='hunk'>@@ -3184,13 +3892,11 @@ int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)</div><div class='ctx'> 				((struct bnxt_qplib_qp *)</div><div class='ctx'> 				 (unsigned long)(cqe-&gt;qp_handle),</div><div class='ctx'> 				 struct bnxt_re_qp, qplib_qp);</div><div class='del'>-			if (!qp) {</div><div class='del'>-				dev_err(rdev_to_dev(cq-&gt;rdev),</div><div class='del'>-					"POLL CQ : bad QP handle");</div><div class='del'>-				continue;</div><div class='del'>-			}</div><div class='ctx'> 			wc-&gt;qp = &amp;qp-&gt;ib_qp;</div><div class='del'>-			wc-&gt;ex.imm_data = cqe-&gt;immdata;</div><div class='add'>+			if (cqe-&gt;flags &amp; CQ_RES_RC_FLAGS_IMM)</div><div class='add'>+				wc-&gt;ex.imm_data = cpu_to_be32(cqe-&gt;immdata);</div><div class='add'>+			else</div><div class='add'>+				wc-&gt;ex.invalidate_rkey = cqe-&gt;invrkey;</div><div class='ctx'> 			wc-&gt;src_qp = cqe-&gt;src_qp;</div><div class='ctx'> 			memcpy(wc-&gt;smac, cqe-&gt;smac, ETH_ALEN);</div><div class='ctx'> 			wc-&gt;port_num = 1;</div><div class='hunk'>@@ -3198,8 +3904,9 @@ int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)</div><div class='ctx'> </div><div class='ctx'> 			switch (cqe-&gt;opcode) {</div><div class='ctx'> 			case CQ_BASE_CQE_TYPE_REQ:</div><div class='del'>-				if (qp-&gt;rdev-&gt;qp1_sqp &amp;&amp; qp-&gt;qplib_qp.id ==</div><div class='del'>-				    qp-&gt;rdev-&gt;qp1_sqp-&gt;qplib_qp.id) {</div><div class='add'>+				sh_qp = qp-&gt;rdev-&gt;gsi_ctx.gsi_sqp;</div><div class='add'>+				if (sh_qp &amp;&amp;</div><div class='add'>+				    qp-&gt;qplib_qp.id == sh_qp-&gt;qplib_qp.id) {</div><div class='ctx'> 					/* Handle this completion with</div><div class='ctx'> 					 * the stored completion</div><div class='ctx'> 					 */</div><div class='hunk'>@@ -3225,7 +3932,7 @@ int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)</div><div class='ctx'> 				 * stored in the table</div><div class='ctx'> 				 */</div><div class='ctx'> 				tbl_idx = cqe-&gt;wr_id;</div><div class='del'>-				sqp_entry = &amp;cq-&gt;rdev-&gt;sqp_tbl[tbl_idx];</div><div class='add'>+				sqp_entry = &amp;cq-&gt;rdev-&gt;gsi_ctx.sqp_tbl[tbl_idx];</div><div class='ctx'> 				wc-&gt;wr_id = sqp_entry-&gt;wrid;</div><div class='ctx'> 				bnxt_re_process_res_rawqp1_wc(wc, cqe);</div><div class='ctx'> 				break;</div><div class='hunk'>@@ -3233,8 +3940,9 @@ int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)</div><div class='ctx'> 				bnxt_re_process_res_rc_wc(wc, cqe);</div><div class='ctx'> 				break;</div><div class='ctx'> 			case CQ_BASE_CQE_TYPE_RES_UD:</div><div class='del'>-				if (qp-&gt;rdev-&gt;qp1_sqp &amp;&amp; qp-&gt;qplib_qp.id ==</div><div class='del'>-				    qp-&gt;rdev-&gt;qp1_sqp-&gt;qplib_qp.id) {</div><div class='add'>+				sh_qp = qp-&gt;rdev-&gt;gsi_ctx.gsi_sqp;</div><div class='add'>+				if (sh_qp &amp;&amp;</div><div class='add'>+				    qp-&gt;qplib_qp.id == sh_qp-&gt;qplib_qp.id) {</div><div class='ctx'> 					/* Handle this completion with</div><div class='ctx'> 					 * the stored completion</div><div class='ctx'> 					 */</div><div class='hunk'>@@ -3249,9 +3957,9 @@ int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)</div><div class='ctx'> 				bnxt_re_process_res_ud_wc(qp, wc, cqe);</div><div class='ctx'> 				break;</div><div class='ctx'> 			default:</div><div class='del'>-				dev_err(rdev_to_dev(cq-&gt;rdev),</div><div class='del'>-					"POLL CQ : type 0x%x not handled",</div><div class='del'>-					cqe-&gt;opcode);</div><div class='add'>+				ibdev_err(&amp;cq-&gt;rdev-&gt;ibdev,</div><div class='add'>+					  "POLL CQ : type 0x%x not handled",</div><div class='add'>+					  cqe-&gt;opcode);</div><div class='ctx'> 				continue;</div><div class='ctx'> 			}</div><div class='ctx'> 			wc++;</div><div class='hunk'>@@ -3297,7 +4005,7 @@ struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)</div><div class='ctx'> 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);</div><div class='ctx'> 	struct bnxt_re_dev *rdev = pd-&gt;rdev;</div><div class='ctx'> 	struct bnxt_re_mr *mr;</div><div class='del'>-	u64 pbl = 0;</div><div class='add'>+	u32 active_mrs;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='ctx'> 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='hunk'>@@ -3306,9 +4014,12 @@ struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;rdev = rdev;</div><div class='ctx'> 	mr-&gt;qplib_mr.pd = &amp;pd-&gt;qplib_pd;</div><div class='del'>-	mr-&gt;qplib_mr.flags = __from_ib_access_flags(mr_access_flags);</div><div class='add'>+	mr-&gt;qplib_mr.access_flags = __from_ib_access_flags(mr_access_flags);</div><div class='ctx'> 	mr-&gt;qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;</div><div class='ctx'> </div><div class='add'>+	if (mr_access_flags &amp; IB_ACCESS_RELAXED_ORDERING)</div><div class='add'>+		bnxt_re_check_and_set_relaxed_ordering(rdev, &amp;mr-&gt;qplib_mr);</div><div class='add'>+</div><div class='ctx'> 	/* Allocate and register 0 as the address */</div><div class='ctx'> 	rc = bnxt_qplib_alloc_mrw(&amp;rdev-&gt;qplib_res, &amp;mr-&gt;qplib_mr);</div><div class='ctx'> 	if (rc)</div><div class='hunk'>@@ -3316,8 +4027,8 @@ struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;qplib_mr.hwq.level = PBL_LVL_MAX;</div><div class='ctx'> 	mr-&gt;qplib_mr.total_size = -1; /* Infinte length */</div><div class='del'>-	rc = bnxt_qplib_reg_mr(&amp;rdev-&gt;qplib_res, &amp;mr-&gt;qplib_mr, &amp;pbl, 0, false,</div><div class='del'>-			       PAGE_SIZE);</div><div class='add'>+	rc = bnxt_qplib_reg_mr(&amp;rdev-&gt;qplib_res, &amp;mr-&gt;qplib_mr, NULL, 0,</div><div class='add'>+			       PAGE_SIZE, false);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail_mr;</div><div class='ctx'> </div><div class='hunk'>@@ -3325,7 +4036,9 @@ struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)</div><div class='ctx'> 	if (mr_access_flags &amp; (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |</div><div class='ctx'> 			       IB_ACCESS_REMOTE_ATOMIC))</div><div class='ctx'> 		mr-&gt;ib_mr.rkey = mr-&gt;ib_mr.lkey;</div><div class='del'>-	atomic_inc(&amp;rdev-&gt;mr_count);</div><div class='add'>+	active_mrs = atomic_inc_return(&amp;rdev-&gt;stats.res.mr_count);</div><div class='add'>+	if (active_mrs &gt; rdev-&gt;stats.res.mr_watermark)</div><div class='add'>+		rdev-&gt;stats.res.mr_watermark = active_mrs;</div><div class='ctx'> </div><div class='ctx'> 	return &amp;mr-&gt;ib_mr;</div><div class='ctx'> </div><div class='hunk'>@@ -3343,8 +4056,10 @@ int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='ctx'> 	rc = bnxt_qplib_free_mrw(&amp;rdev-&gt;qplib_res, &amp;mr-&gt;qplib_mr);</div><div class='del'>-	if (rc)</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Dereg MR failed: %#x\n", rc);</div><div class='add'>+	if (rc) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Dereg MR failed: %#x\n", rc);</div><div class='add'>+		return rc;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	if (mr-&gt;pages) {</div><div class='ctx'> 		rc = bnxt_qplib_free_fast_reg_page_list(&amp;rdev-&gt;qplib_res,</div><div class='hunk'>@@ -3353,11 +4068,10 @@ int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)</div><div class='ctx'> 		mr-&gt;npages = 0;</div><div class='ctx'> 		mr-&gt;pages = NULL;</div><div class='ctx'> 	}</div><div class='del'>-	if (!IS_ERR_OR_NULL(mr-&gt;ib_umem))</div><div class='del'>-		ib_umem_release(mr-&gt;ib_umem);</div><div class='add'>+	ib_umem_release(mr-&gt;ib_umem);</div><div class='ctx'> </div><div class='ctx'> 	kfree(mr);</div><div class='del'>-	atomic_dec(&amp;rdev-&gt;mr_count);</div><div class='add'>+	atomic_dec(&amp;rdev-&gt;stats.res.mr_count);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -3382,15 +4096,16 @@ int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,</div><div class='del'>-			       u32 max_num_sg, struct ib_udata *udata)</div><div class='add'>+			       u32 max_num_sg)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);</div><div class='ctx'> 	struct bnxt_re_dev *rdev = pd-&gt;rdev;</div><div class='ctx'> 	struct bnxt_re_mr *mr = NULL;</div><div class='add'>+	u32 active_mrs;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='ctx'> 	if (type != IB_MR_TYPE_MEM_REG) {</div><div class='del'>-		dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported", type);</div><div class='add'>+		ibdev_dbg(&amp;rdev-&gt;ibdev, "MR type 0x%x not supported", type);</div><div class='ctx'> 		return ERR_PTR(-EINVAL);</div><div class='ctx'> 	}</div><div class='ctx'> 	if (max_num_sg &gt; MAX_PBL_LVL_1_PGS)</div><div class='hunk'>@@ -3402,7 +4117,7 @@ struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;rdev = rdev;</div><div class='ctx'> 	mr-&gt;qplib_mr.pd = &amp;pd-&gt;qplib_pd;</div><div class='del'>-	mr-&gt;qplib_mr.flags = BNXT_QPLIB_FR_PMR;</div><div class='add'>+	mr-&gt;qplib_mr.access_flags = BNXT_QPLIB_FR_PMR;</div><div class='ctx'> 	mr-&gt;qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;</div><div class='ctx'> </div><div class='ctx'> 	rc = bnxt_qplib_alloc_mrw(&amp;rdev-&gt;qplib_res, &amp;mr-&gt;qplib_mr);</div><div class='hunk'>@@ -3420,12 +4135,14 @@ struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,</div><div class='ctx'> 	rc = bnxt_qplib_alloc_fast_reg_page_list(&amp;rdev-&gt;qplib_res,</div><div class='ctx'> 						 &amp;mr-&gt;qplib_frpl, max_num_sg);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev),</div><div class='del'>-			"Failed to allocate HW FR page list");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to allocate HW FR page list");</div><div class='ctx'> 		goto fail_mr;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	atomic_inc(&amp;rdev-&gt;mr_count);</div><div class='add'>+	active_mrs = atomic_inc_return(&amp;rdev-&gt;stats.res.mr_count);</div><div class='add'>+	if (active_mrs &gt; rdev-&gt;stats.res.mr_watermark)</div><div class='add'>+		rdev-&gt;stats.res.mr_watermark = active_mrs;</div><div class='ctx'> 	return &amp;mr-&gt;ib_mr;</div><div class='ctx'> </div><div class='ctx'> fail_mr:</div><div class='hunk'>@@ -3443,6 +4160,7 @@ struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,</div><div class='ctx'> 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);</div><div class='ctx'> 	struct bnxt_re_dev *rdev = pd-&gt;rdev;</div><div class='ctx'> 	struct bnxt_re_mw *mw;</div><div class='add'>+	u32 active_mws;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='ctx'> 	mw = kzalloc(sizeof(*mw), GFP_KERNEL);</div><div class='hunk'>@@ -3456,12 +4174,14 @@ struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,</div><div class='ctx'> 			       CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B);</div><div class='ctx'> 	rc = bnxt_qplib_alloc_mrw(&amp;rdev-&gt;qplib_res, &amp;mw-&gt;qplib_mw);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Allocate MW failed!");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Allocate MW failed!");</div><div class='ctx'> 		goto fail;</div><div class='ctx'> 	}</div><div class='ctx'> 	mw-&gt;ib_mw.rkey = mw-&gt;qplib_mw.rkey;</div><div class='ctx'> </div><div class='del'>-	atomic_inc(&amp;rdev-&gt;mw_count);</div><div class='add'>+	active_mws = atomic_inc_return(&amp;rdev-&gt;stats.res.mw_count);</div><div class='add'>+	if (active_mws &gt; rdev-&gt;stats.res.mw_watermark)</div><div class='add'>+		rdev-&gt;stats.res.mw_watermark = active_mws;</div><div class='ctx'> 	return &amp;mw-&gt;ib_mw;</div><div class='ctx'> </div><div class='ctx'> fail:</div><div class='hunk'>@@ -3477,141 +4197,83 @@ int bnxt_re_dealloc_mw(struct ib_mw *ib_mw)</div><div class='ctx'> </div><div class='ctx'> 	rc = bnxt_qplib_free_mrw(&amp;rdev-&gt;qplib_res, &amp;mw-&gt;qplib_mw);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Free MW failed: %#x\n", rc);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Free MW failed: %#x\n", rc);</div><div class='ctx'> 		return rc;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	kfree(mw);</div><div class='del'>-	atomic_dec(&amp;rdev-&gt;mw_count);</div><div class='add'>+	atomic_dec(&amp;rdev-&gt;stats.res.mw_count);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int bnxt_re_page_size_ok(int page_shift)</div><div class='del'>-{</div><div class='del'>-	switch (page_shift) {</div><div class='del'>-	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K:</div><div class='del'>-	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K:</div><div class='del'>-	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K:</div><div class='del'>-	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M:</div><div class='del'>-	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K:</div><div class='del'>-	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M:</div><div class='del'>-	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M:</div><div class='del'>-	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G:</div><div class='del'>-		return 1;</div><div class='del'>-	default:</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int fill_umem_pbl_tbl(struct ib_umem *umem, u64 *pbl_tbl_orig,</div><div class='del'>-			     int page_shift)</div><div class='del'>-{</div><div class='del'>-	u64 *pbl_tbl = pbl_tbl_orig;</div><div class='del'>-	u64 page_size =  BIT_ULL(page_shift);</div><div class='del'>-	struct ib_block_iter biter;</div><div class='del'>-</div><div class='del'>-	rdma_for_each_block(umem-&gt;sg_head.sgl, &amp;biter, umem-&gt;nmap, page_size)</div><div class='del'>-		*pbl_tbl++ = rdma_block_iter_dma_address(&amp;biter);</div><div class='del'>-</div><div class='del'>-	return pbl_tbl - pbl_tbl_orig;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* uverbs */</div><div class='del'>-struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,</div><div class='del'>-				  u64 virt_addr, int mr_access_flags,</div><div class='del'>-				  struct ib_udata *udata)</div><div class='add'>+static struct ib_mr *__bnxt_re_user_reg_mr(struct ib_pd *ib_pd, u64 length, u64 virt_addr,</div><div class='add'>+					   int mr_access_flags, struct ib_umem *umem)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);</div><div class='ctx'> 	struct bnxt_re_dev *rdev = pd-&gt;rdev;</div><div class='add'>+	unsigned long page_size;</div><div class='ctx'> 	struct bnxt_re_mr *mr;</div><div class='del'>-	struct ib_umem *umem;</div><div class='del'>-	u64 *pbl_tbl = NULL;</div><div class='del'>-	int umem_pgs, page_shift, rc;</div><div class='add'>+	int umem_pgs, rc;</div><div class='add'>+	u32 active_mrs;</div><div class='ctx'> </div><div class='ctx'> 	if (length &gt; BNXT_RE_MAX_MR_SIZE) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "MR Size: %lld &gt; Max supported:%lld\n",</div><div class='del'>-			length, BNXT_RE_MAX_MR_SIZE);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "MR Size: %lld &gt; Max supported:%lld\n",</div><div class='add'>+			  length, BNXT_RE_MAX_MR_SIZE);</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	page_size = ib_umem_find_best_pgsz(umem, BNXT_RE_PAGE_SIZE_SUPPORTED, virt_addr);</div><div class='add'>+	if (!page_size) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "umem page size unsupported!");</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='ctx'> 	if (!mr)</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;rdev = rdev;</div><div class='ctx'> 	mr-&gt;qplib_mr.pd = &amp;pd-&gt;qplib_pd;</div><div class='del'>-	mr-&gt;qplib_mr.flags = __from_ib_access_flags(mr_access_flags);</div><div class='add'>+	mr-&gt;qplib_mr.access_flags = __from_ib_access_flags(mr_access_flags);</div><div class='ctx'> 	mr-&gt;qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_alloc_mrw(&amp;rdev-&gt;qplib_res, &amp;mr-&gt;qplib_mr);</div><div class='del'>-	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to allocate MR");</div><div class='del'>-		goto free_mr;</div><div class='del'>-	}</div><div class='del'>-	/* The fixed portion of the rkey is the same as the lkey */</div><div class='del'>-	mr-&gt;ib_mr.rkey = mr-&gt;qplib_mr.rkey;</div><div class='del'>-</div><div class='del'>-	umem = ib_umem_get(udata, start, length, mr_access_flags, 0);</div><div class='del'>-	if (IS_ERR(umem)) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to get umem");</div><div class='del'>-		rc = -EFAULT;</div><div class='del'>-		goto free_mrw;</div><div class='add'>+	if (!_is_alloc_mr_unified(rdev-&gt;dev_attr-&gt;dev_cap_flags)) {</div><div class='add'>+		rc = bnxt_qplib_alloc_mrw(&amp;rdev-&gt;qplib_res, &amp;mr-&gt;qplib_mr);</div><div class='add'>+		if (rc) {</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev, "Failed to allocate MR rc = %d", rc);</div><div class='add'>+			rc = -EIO;</div><div class='add'>+			goto free_mr;</div><div class='add'>+		}</div><div class='add'>+		/* The fixed portion of the rkey is the same as the lkey */</div><div class='add'>+		mr-&gt;ib_mr.rkey = mr-&gt;qplib_mr.rkey;</div><div class='add'>+	} else {</div><div class='add'>+		mr-&gt;qplib_mr.flags = CMDQ_REGISTER_MR_FLAGS_ALLOC_MR;</div><div class='ctx'> 	}</div><div class='ctx'> 	mr-&gt;ib_umem = umem;</div><div class='del'>-</div><div class='ctx'> 	mr-&gt;qplib_mr.va = virt_addr;</div><div class='del'>-	umem_pgs = ib_umem_page_count(umem);</div><div class='del'>-	if (!umem_pgs) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "umem is invalid!");</div><div class='del'>-		rc = -EINVAL;</div><div class='del'>-		goto free_umem;</div><div class='del'>-	}</div><div class='ctx'> 	mr-&gt;qplib_mr.total_size = length;</div><div class='ctx'> </div><div class='del'>-	pbl_tbl = kcalloc(umem_pgs, sizeof(u64 *), GFP_KERNEL);</div><div class='del'>-	if (!pbl_tbl) {</div><div class='del'>-		rc = -ENOMEM;</div><div class='del'>-		goto free_umem;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	page_shift = __ffs(ib_umem_find_best_pgsz(umem,</div><div class='del'>-				BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_2M,</div><div class='del'>-				virt_addr));</div><div class='del'>-</div><div class='del'>-	if (!bnxt_re_page_size_ok(page_shift)) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "umem page size unsupported!");</div><div class='del'>-		rc = -EFAULT;</div><div class='del'>-		goto fail;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (page_shift == BNXT_RE_PAGE_SHIFT_4K &amp;&amp;</div><div class='del'>-	    length &gt; BNXT_RE_MAX_MR_SIZE_LOW) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Requested MR Sz:%llu Max sup:%llu",</div><div class='del'>-			length,	(u64)BNXT_RE_MAX_MR_SIZE_LOW);</div><div class='del'>-		rc = -EINVAL;</div><div class='del'>-		goto fail;</div><div class='del'>-	}</div><div class='add'>+	if (mr_access_flags &amp; IB_ACCESS_RELAXED_ORDERING)</div><div class='add'>+		bnxt_re_check_and_set_relaxed_ordering(rdev, &amp;mr-&gt;qplib_mr);</div><div class='ctx'> </div><div class='del'>-	/* Map umem buf ptrs to the PBL */</div><div class='del'>-	umem_pgs = fill_umem_pbl_tbl(umem, pbl_tbl, page_shift);</div><div class='del'>-	rc = bnxt_qplib_reg_mr(&amp;rdev-&gt;qplib_res, &amp;mr-&gt;qplib_mr, pbl_tbl,</div><div class='del'>-			       umem_pgs, false, 1 &lt;&lt; page_shift);</div><div class='add'>+	umem_pgs = ib_umem_num_dma_blocks(umem, page_size);</div><div class='add'>+	rc = bnxt_qplib_reg_mr(&amp;rdev-&gt;qplib_res, &amp;mr-&gt;qplib_mr, umem,</div><div class='add'>+			       umem_pgs, page_size,</div><div class='add'>+			       _is_alloc_mr_unified(rdev-&gt;dev_attr-&gt;dev_cap_flags));</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to register user MR");</div><div class='del'>-		goto fail;</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to register user MR - rc = %d\n", rc);</div><div class='add'>+		rc = -EIO;</div><div class='add'>+		goto free_mrw;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	kfree(pbl_tbl);</div><div class='del'>-</div><div class='ctx'> 	mr-&gt;ib_mr.lkey = mr-&gt;qplib_mr.lkey;</div><div class='ctx'> 	mr-&gt;ib_mr.rkey = mr-&gt;qplib_mr.lkey;</div><div class='del'>-	atomic_inc(&amp;rdev-&gt;mr_count);</div><div class='add'>+	active_mrs = atomic_inc_return(&amp;rdev-&gt;stats.res.mr_count);</div><div class='add'>+	if (active_mrs &gt; rdev-&gt;stats.res.mr_watermark)</div><div class='add'>+		rdev-&gt;stats.res.mr_watermark = active_mrs;</div><div class='ctx'> </div><div class='ctx'> 	return &amp;mr-&gt;ib_mr;</div><div class='del'>-fail:</div><div class='del'>-	kfree(pbl_tbl);</div><div class='del'>-free_umem:</div><div class='del'>-	ib_umem_release(umem);</div><div class='add'>+</div><div class='ctx'> free_mrw:</div><div class='ctx'> 	bnxt_qplib_free_mrw(&amp;rdev-&gt;qplib_res, &amp;mr-&gt;qplib_mr);</div><div class='ctx'> free_mr:</div><div class='hunk'>@@ -3619,23 +4281,75 @@ free_mr:</div><div class='ctx'> 	return ERR_PTR(rc);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,</div><div class='add'>+				  u64 virt_addr, int mr_access_flags,</div><div class='add'>+				  struct ib_dmah *dmah,</div><div class='add'>+				  struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);</div><div class='add'>+	struct bnxt_re_dev *rdev = pd-&gt;rdev;</div><div class='add'>+	struct ib_umem *umem;</div><div class='add'>+	struct ib_mr *ib_mr;</div><div class='add'>+</div><div class='add'>+	if (dmah)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	umem = ib_umem_get(&amp;rdev-&gt;ibdev, start, length, mr_access_flags);</div><div class='add'>+	if (IS_ERR(umem))</div><div class='add'>+		return ERR_CAST(umem);</div><div class='add'>+</div><div class='add'>+	ib_mr = __bnxt_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem);</div><div class='add'>+	if (IS_ERR(ib_mr))</div><div class='add'>+		ib_umem_release(umem);</div><div class='add'>+	return ib_mr;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_mr *bnxt_re_reg_user_mr_dmabuf(struct ib_pd *ib_pd, u64 start,</div><div class='add'>+					 u64 length, u64 virt_addr, int fd,</div><div class='add'>+					 int mr_access_flags,</div><div class='add'>+					 struct ib_dmah *dmah,</div><div class='add'>+					 struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);</div><div class='add'>+	struct bnxt_re_dev *rdev = pd-&gt;rdev;</div><div class='add'>+	struct ib_umem_dmabuf *umem_dmabuf;</div><div class='add'>+	struct ib_umem *umem;</div><div class='add'>+	struct ib_mr *ib_mr;</div><div class='add'>+</div><div class='add'>+	if (dmah)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	umem_dmabuf = ib_umem_dmabuf_get_pinned(&amp;rdev-&gt;ibdev, start, length,</div><div class='add'>+						fd, mr_access_flags);</div><div class='add'>+	if (IS_ERR(umem_dmabuf))</div><div class='add'>+		return ERR_CAST(umem_dmabuf);</div><div class='add'>+</div><div class='add'>+	umem = &amp;umem_dmabuf-&gt;umem;</div><div class='add'>+</div><div class='add'>+	ib_mr = __bnxt_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem);</div><div class='add'>+	if (IS_ERR(ib_mr))</div><div class='add'>+		ib_umem_release(umem);</div><div class='add'>+	return ib_mr;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *ibdev = ctx-&gt;device;</div><div class='ctx'> 	struct bnxt_re_ucontext *uctx =</div><div class='ctx'> 		container_of(ctx, struct bnxt_re_ucontext, ib_uctx);</div><div class='ctx'> 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);</div><div class='del'>-	struct bnxt_qplib_dev_attr *dev_attr = &amp;rdev-&gt;dev_attr;</div><div class='del'>-	struct bnxt_re_uctx_resp resp;</div><div class='add'>+	struct bnxt_qplib_dev_attr *dev_attr = rdev-&gt;dev_attr;</div><div class='add'>+	struct bnxt_re_user_mmap_entry *entry;</div><div class='add'>+	struct bnxt_re_uctx_resp resp = {};</div><div class='add'>+	struct bnxt_re_uctx_req ureq = {};</div><div class='ctx'> 	u32 chip_met_rev_num = 0;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	dev_dbg(rdev_to_dev(rdev), "ABI version requested %d",</div><div class='del'>-		ibdev-&gt;uverbs_abi_ver);</div><div class='add'>+	ibdev_dbg(ibdev, "ABI version requested %u", ibdev-&gt;ops.uverbs_abi_ver);</div><div class='ctx'> </div><div class='del'>-	if (ibdev-&gt;uverbs_abi_ver != BNXT_RE_ABI_VERSION) {</div><div class='del'>-		dev_dbg(rdev_to_dev(rdev), " is different from the device %d ",</div><div class='del'>-			BNXT_RE_ABI_VERSION);</div><div class='add'>+	if (ibdev-&gt;ops.uverbs_abi_ver != BNXT_RE_ABI_VERSION) {</div><div class='add'>+		ibdev_dbg(ibdev, " is different from the device %d ",</div><div class='add'>+			  BNXT_RE_ABI_VERSION);</div><div class='ctx'> 		return -EPERM;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -3649,25 +4363,53 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)</div><div class='ctx'> 	spin_lock_init(&amp;uctx-&gt;sh_lock);</div><div class='ctx'> </div><div class='ctx'> 	resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX;</div><div class='del'>-	chip_met_rev_num = rdev-&gt;chip_ctx.chip_num;</div><div class='del'>-	chip_met_rev_num |= ((u32)rdev-&gt;chip_ctx.chip_rev &amp; 0xFF) &lt;&lt;</div><div class='add'>+	chip_met_rev_num = rdev-&gt;chip_ctx-&gt;chip_num;</div><div class='add'>+	chip_met_rev_num |= ((u32)rdev-&gt;chip_ctx-&gt;chip_rev &amp; 0xFF) &lt;&lt;</div><div class='ctx'> 			     BNXT_RE_CHIP_ID0_CHIP_REV_SFT;</div><div class='del'>-	chip_met_rev_num |= ((u32)rdev-&gt;chip_ctx.chip_metal &amp; 0xFF) &lt;&lt;</div><div class='add'>+	chip_met_rev_num |= ((u32)rdev-&gt;chip_ctx-&gt;chip_metal &amp; 0xFF) &lt;&lt;</div><div class='ctx'> 			     BNXT_RE_CHIP_ID0_CHIP_MET_SFT;</div><div class='ctx'> 	resp.chip_id0 = chip_met_rev_num;</div><div class='del'>-	/* Future extension of chip info */</div><div class='del'>-	resp.chip_id1 = 0;</div><div class='ctx'> 	/*Temp, Use xa_alloc instead */</div><div class='ctx'> 	resp.dev_id = rdev-&gt;en_dev-&gt;pdev-&gt;devfn;</div><div class='ctx'> 	resp.max_qp = rdev-&gt;qplib_ctx.qpc_count;</div><div class='ctx'> 	resp.pg_size = PAGE_SIZE;</div><div class='ctx'> 	resp.cqe_sz = sizeof(struct cq_base);</div><div class='ctx'> 	resp.max_cqd = dev_attr-&gt;max_cq_wqes;</div><div class='del'>-	resp.rsvd    = 0;</div><div class='add'>+</div><div class='add'>+	if (rdev-&gt;chip_ctx-&gt;modes.db_push)</div><div class='add'>+		resp.comp_mask |= BNXT_RE_UCNTX_CMASK_WC_DPI_ENABLED;</div><div class='add'>+</div><div class='add'>+	entry = bnxt_re_mmap_entry_insert(uctx, 0, BNXT_RE_MMAP_SH_PAGE, NULL);</div><div class='add'>+	if (!entry) {</div><div class='add'>+		rc = -ENOMEM;</div><div class='add'>+		goto cfail;</div><div class='add'>+	}</div><div class='add'>+	uctx-&gt;shpage_mmap = &amp;entry-&gt;rdma_entry;</div><div class='add'>+	if (rdev-&gt;pacing.dbr_pacing)</div><div class='add'>+		resp.comp_mask |= BNXT_RE_UCNTX_CMASK_DBR_PACING_ENABLED;</div><div class='add'>+</div><div class='add'>+	if (_is_host_msn_table(rdev-&gt;qplib_res.dattr-&gt;dev_cap_flags2))</div><div class='add'>+		resp.comp_mask |= BNXT_RE_UCNTX_CMASK_MSN_TABLE_ENABLED;</div><div class='add'>+</div><div class='add'>+	if (udata-&gt;inlen &gt;= sizeof(ureq)) {</div><div class='add'>+		rc = ib_copy_from_udata(&amp;ureq, udata, min(udata-&gt;inlen, sizeof(ureq)));</div><div class='add'>+		if (rc)</div><div class='add'>+			goto cfail;</div><div class='add'>+		if (ureq.comp_mask &amp; BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT) {</div><div class='add'>+			resp.comp_mask |= BNXT_RE_UCNTX_CMASK_POW2_DISABLED;</div><div class='add'>+			uctx-&gt;cmask |= BNXT_RE_UCNTX_CAP_POW2_DISABLED;</div><div class='add'>+		}</div><div class='add'>+		if (ureq.comp_mask &amp; BNXT_RE_COMP_MASK_REQ_UCNTX_VAR_WQE_SUPPORT) {</div><div class='add'>+			resp.comp_mask |= BNXT_RE_UCNTX_CMASK_HAVE_MODE;</div><div class='add'>+			resp.mode = rdev-&gt;chip_ctx-&gt;modes.wqe_mode;</div><div class='add'>+			if (resp.mode == BNXT_QPLIB_WQE_MODE_VARIABLE)</div><div class='add'>+				uctx-&gt;cmask |= BNXT_RE_UCNTX_CAP_VAR_WQE_ENABLED;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	rc = ib_copy_to_udata(udata, &amp;resp, min(udata-&gt;outlen, sizeof(resp)));</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to copy user context");</div><div class='add'>+		ibdev_err(ibdev, "Failed to copy user context");</div><div class='ctx'> 		rc = -EFAULT;</div><div class='ctx'> 		goto cfail;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -3688,6 +4430,8 @@ void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)</div><div class='ctx'> </div><div class='ctx'> 	struct bnxt_re_dev *rdev = uctx-&gt;rdev;</div><div class='ctx'> </div><div class='add'>+	rdma_user_mmap_entry_remove(uctx-&gt;shpage_mmap);</div><div class='add'>+	uctx-&gt;shpage_mmap = NULL;</div><div class='ctx'> 	if (uctx-&gt;shpg)</div><div class='ctx'> 		free_page((unsigned long)uctx-&gt;shpg);</div><div class='ctx'> </div><div class='hunk'>@@ -3695,40 +4439,500 @@ void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)</div><div class='ctx'> 		/* Free DPI only if this is the first PD allocated by the</div><div class='ctx'> 		 * application and mark the context dpi as NULL</div><div class='ctx'> 		 */</div><div class='del'>-		bnxt_qplib_dealloc_dpi(&amp;rdev-&gt;qplib_res,</div><div class='del'>-				       &amp;rdev-&gt;qplib_res.dpi_tbl, &amp;uctx-&gt;dpi);</div><div class='add'>+		bnxt_qplib_dealloc_dpi(&amp;rdev-&gt;qplib_res, &amp;uctx-&gt;dpi);</div><div class='ctx'> 		uctx-&gt;dpi.dbr = NULL;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int bnxt_re_setup_vnic(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp)</div><div class='add'>+{</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rc = bnxt_re_hwrm_alloc_vnic(rdev);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	rc = bnxt_re_hwrm_cfg_vnic(rdev, qp-&gt;qplib_qp.id);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto out_free_vnic;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+out_free_vnic:</div><div class='add'>+	bnxt_re_hwrm_free_vnic(rdev);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_flow *bnxt_re_create_flow(struct ib_qp *ib_qp,</div><div class='add'>+				    struct ib_flow_attr *attr,</div><div class='add'>+				    struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);</div><div class='add'>+	struct bnxt_re_dev *rdev = qp-&gt;rdev;</div><div class='add'>+	struct bnxt_re_flow *flow;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;type != IB_FLOW_ATTR_SNIFFER ||</div><div class='add'>+	    !rdev-&gt;rcfw.roce_mirror)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;rdev-&gt;qp_lock);</div><div class='add'>+	if (rdev-&gt;sniffer_flow_created) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "RoCE Mirroring is already Configured\n");</div><div class='add'>+		mutex_unlock(&amp;rdev-&gt;qp_lock);</div><div class='add'>+		return ERR_PTR(-EBUSY);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	flow = kzalloc(sizeof(*flow), GFP_KERNEL);</div><div class='add'>+	if (!flow) {</div><div class='add'>+		mutex_unlock(&amp;rdev-&gt;qp_lock);</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	flow-&gt;rdev = rdev;</div><div class='add'>+</div><div class='add'>+	rc = bnxt_re_setup_vnic(rdev, qp);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto out_free_flow;</div><div class='add'>+</div><div class='add'>+	rc = bnxt_qplib_create_flow(&amp;rdev-&gt;qplib_res);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto out_free_vnic;</div><div class='add'>+</div><div class='add'>+	rdev-&gt;sniffer_flow_created = 1;</div><div class='add'>+	mutex_unlock(&amp;rdev-&gt;qp_lock);</div><div class='add'>+</div><div class='add'>+	return &amp;flow-&gt;ib_flow;</div><div class='add'>+</div><div class='add'>+out_free_vnic:</div><div class='add'>+	bnxt_re_hwrm_free_vnic(rdev);</div><div class='add'>+out_free_flow:</div><div class='add'>+	mutex_unlock(&amp;rdev-&gt;qp_lock);</div><div class='add'>+	kfree(flow);</div><div class='add'>+	return ERR_PTR(rc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bnxt_re_destroy_flow(struct ib_flow *flow_id)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_flow *flow =</div><div class='add'>+		container_of(flow_id, struct bnxt_re_flow, ib_flow);</div><div class='add'>+	struct bnxt_re_dev *rdev = flow-&gt;rdev;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;rdev-&gt;qp_lock);</div><div class='add'>+	rc = bnxt_qplib_destroy_flow(&amp;rdev-&gt;qplib_res);</div><div class='add'>+	if (rc)</div><div class='add'>+		ibdev_dbg(&amp;rdev-&gt;ibdev, "failed to destroy_flow rc = %d\n", rc);</div><div class='add'>+	rdev-&gt;sniffer_flow_created = 0;</div><div class='add'>+</div><div class='add'>+	bnxt_re_hwrm_free_vnic(rdev);</div><div class='add'>+	mutex_unlock(&amp;rdev-&gt;qp_lock);</div><div class='add'>+	kfree(flow);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct bnxt_re_cq *bnxt_re_search_for_cq(struct bnxt_re_dev *rdev, u32 cq_id)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_cq *cq = NULL, *tmp_cq;</div><div class='add'>+</div><div class='add'>+	hash_for_each_possible(rdev-&gt;cq_hash, tmp_cq, hash_entry, cq_id) {</div><div class='add'>+		if (tmp_cq-&gt;qplib_cq.id == cq_id) {</div><div class='add'>+			cq = tmp_cq;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	return cq;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct bnxt_re_srq *bnxt_re_search_for_srq(struct bnxt_re_dev *rdev, u32 srq_id)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_srq *srq = NULL, *tmp_srq;</div><div class='add'>+</div><div class='add'>+	hash_for_each_possible(rdev-&gt;srq_hash, tmp_srq, hash_entry, srq_id) {</div><div class='add'>+		if (tmp_srq-&gt;qplib_srq.id == srq_id) {</div><div class='add'>+			srq = tmp_srq;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	return srq;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /* Helper function to mmap the virtual memory from user app */</div><div class='ctx'> int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_re_ucontext *uctx = container_of(ib_uctx,</div><div class='ctx'> 						   struct bnxt_re_ucontext,</div><div class='ctx'> 						   ib_uctx);</div><div class='del'>-	struct bnxt_re_dev *rdev = uctx-&gt;rdev;</div><div class='add'>+	struct bnxt_re_user_mmap_entry *bnxt_entry;</div><div class='add'>+	struct rdma_user_mmap_entry *rdma_entry;</div><div class='add'>+	int ret = 0;</div><div class='ctx'> 	u64 pfn;</div><div class='ctx'> </div><div class='del'>-	if (vma-&gt;vm_end - vma-&gt;vm_start != PAGE_SIZE)</div><div class='add'>+	rdma_entry = rdma_user_mmap_entry_get(&amp;uctx-&gt;ib_uctx, vma);</div><div class='add'>+	if (!rdma_entry)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (vma-&gt;vm_pgoff) {</div><div class='del'>-		vma-&gt;vm_page_prot = pgprot_noncached(vma-&gt;vm_page_prot);</div><div class='del'>-		if (io_remap_pfn_range(vma, vma-&gt;vm_start, vma-&gt;vm_pgoff,</div><div class='del'>-				       PAGE_SIZE, vma-&gt;vm_page_prot)) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev), "Failed to map DPI");</div><div class='del'>-			return -EAGAIN;</div><div class='add'>+	bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry,</div><div class='add'>+				  rdma_entry);</div><div class='add'>+</div><div class='add'>+	switch (bnxt_entry-&gt;mmap_flag) {</div><div class='add'>+	case BNXT_RE_MMAP_WC_DB:</div><div class='add'>+		pfn = bnxt_entry-&gt;mem_offset &gt;&gt; PAGE_SHIFT;</div><div class='add'>+		ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,</div><div class='add'>+					pgprot_writecombine(vma-&gt;vm_page_prot),</div><div class='add'>+					rdma_entry);</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_RE_MMAP_UC_DB:</div><div class='add'>+		pfn = bnxt_entry-&gt;mem_offset &gt;&gt; PAGE_SHIFT;</div><div class='add'>+		ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,</div><div class='add'>+					pgprot_noncached(vma-&gt;vm_page_prot),</div><div class='add'>+				rdma_entry);</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_RE_MMAP_SH_PAGE:</div><div class='add'>+		ret = vm_insert_page(vma, vma-&gt;vm_start, virt_to_page(uctx-&gt;shpg));</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_RE_MMAP_DBR_BAR:</div><div class='add'>+		pfn = bnxt_entry-&gt;mem_offset &gt;&gt; PAGE_SHIFT;</div><div class='add'>+		ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,</div><div class='add'>+					pgprot_noncached(vma-&gt;vm_page_prot),</div><div class='add'>+					rdma_entry);</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_RE_MMAP_DBR_PAGE:</div><div class='add'>+	case BNXT_RE_MMAP_TOGGLE_PAGE:</div><div class='add'>+		/* Driver doesn't expect write access for user space */</div><div class='add'>+		if (vma-&gt;vm_flags &amp; VM_WRITE)</div><div class='add'>+			ret = -EFAULT;</div><div class='add'>+		else</div><div class='add'>+			ret = vm_insert_page(vma, vma-&gt;vm_start,</div><div class='add'>+					     virt_to_page((void *)bnxt_entry-&gt;mem_offset));</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rdma_user_mmap_entry_put(rdma_entry);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bnxt_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_user_mmap_entry *bnxt_entry;</div><div class='add'>+</div><div class='add'>+	bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry,</div><div class='add'>+				  rdma_entry);</div><div class='add'>+</div><div class='add'>+	kfree(bnxt_entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags,</div><div class='add'>+			u32 port_num, const struct ib_wc *in_wc,</div><div class='add'>+			const struct ib_grh *in_grh,</div><div class='add'>+			const struct ib_mad *in_mad, struct ib_mad *out_mad,</div><div class='add'>+			size_t *out_mad_size, u16 *out_mad_pkey_index)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);</div><div class='add'>+	struct ib_class_port_info cpi = {};</div><div class='add'>+	int ret = IB_MAD_RESULT_SUCCESS;</div><div class='add'>+	int rc = 0;</div><div class='add'>+</div><div class='add'>+	if (in_mad-&gt;mad_hdr.mgmt_class != IB_MGMT_CLASS_PERF_MGMT)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	switch (in_mad-&gt;mad_hdr.attr_id) {</div><div class='add'>+	case IB_PMA_CLASS_PORT_INFO:</div><div class='add'>+		cpi.capability_mask = IB_PMA_CLASS_CAP_EXT_WIDTH;</div><div class='add'>+		memcpy((out_mad-&gt;data + 40), &amp;cpi, sizeof(cpi));</div><div class='add'>+		break;</div><div class='add'>+	case IB_PMA_PORT_COUNTERS_EXT:</div><div class='add'>+		rc = bnxt_re_assign_pma_port_ext_counters(rdev, out_mad);</div><div class='add'>+		break;</div><div class='add'>+	case IB_PMA_PORT_COUNTERS:</div><div class='add'>+		rc = bnxt_re_assign_pma_port_counters(rdev, out_mad);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		rc = -EINVAL;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+	if (rc)</div><div class='add'>+		return IB_MAD_RESULT_FAILURE;</div><div class='add'>+	ret |= IB_MAD_RESULT_REPLY;</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(BNXT_RE_METHOD_NOTIFY_DRV)(struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_ucontext *uctx;</div><div class='add'>+</div><div class='add'>+	uctx = container_of(ib_uverbs_get_ucontext(attrs), struct bnxt_re_ucontext, ib_uctx);</div><div class='add'>+	bnxt_re_pacing_alert(uctx-&gt;rdev);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_ALLOC_PAGE_HANDLE);</div><div class='add'>+	enum bnxt_re_alloc_page_type alloc_type;</div><div class='add'>+	struct bnxt_re_user_mmap_entry *entry;</div><div class='add'>+	enum bnxt_re_mmap_flag mmap_flag;</div><div class='add'>+	struct bnxt_qplib_chip_ctx *cctx;</div><div class='add'>+	struct bnxt_re_ucontext *uctx;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	u64 mmap_offset;</div><div class='add'>+	u32 length;</div><div class='add'>+	u32 dpi;</div><div class='add'>+	u64 addr;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	uctx = container_of(ib_uverbs_get_ucontext(attrs), struct bnxt_re_ucontext, ib_uctx);</div><div class='add'>+	if (IS_ERR(uctx))</div><div class='add'>+		return PTR_ERR(uctx);</div><div class='add'>+</div><div class='add'>+	err = uverbs_get_const(&amp;alloc_type, attrs, BNXT_RE_ALLOC_PAGE_TYPE);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	rdev = uctx-&gt;rdev;</div><div class='add'>+	cctx = rdev-&gt;chip_ctx;</div><div class='add'>+</div><div class='add'>+	switch (alloc_type) {</div><div class='add'>+	case BNXT_RE_ALLOC_WC_PAGE:</div><div class='add'>+		if (cctx-&gt;modes.db_push)  {</div><div class='add'>+			if (bnxt_qplib_alloc_dpi(&amp;rdev-&gt;qplib_res, &amp;uctx-&gt;wcdpi,</div><div class='add'>+						 uctx, BNXT_QPLIB_DPI_TYPE_WC))</div><div class='add'>+				return -ENOMEM;</div><div class='add'>+			length = PAGE_SIZE;</div><div class='add'>+			dpi = uctx-&gt;wcdpi.dpi;</div><div class='add'>+			addr = (u64)uctx-&gt;wcdpi.umdbr;</div><div class='add'>+			mmap_flag = BNXT_RE_MMAP_WC_DB;</div><div class='add'>+		} else {</div><div class='add'>+			return -EINVAL;</div><div class='ctx'> 		}</div><div class='del'>-	} else {</div><div class='del'>-		pfn = virt_to_phys(uctx-&gt;shpg) &gt;&gt; PAGE_SHIFT;</div><div class='del'>-		if (remap_pfn_range(vma, vma-&gt;vm_start,</div><div class='del'>-				    pfn, PAGE_SIZE, vma-&gt;vm_page_prot)) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"Failed to map shared page");</div><div class='del'>-			return -EAGAIN;</div><div class='add'>+</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_RE_ALLOC_DBR_BAR_PAGE:</div><div class='add'>+		length = PAGE_SIZE;</div><div class='add'>+		addr = (u64)rdev-&gt;pacing.dbr_bar_addr;</div><div class='add'>+		mmap_flag = BNXT_RE_MMAP_DBR_BAR;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	case BNXT_RE_ALLOC_DBR_PAGE:</div><div class='add'>+		length = PAGE_SIZE;</div><div class='add'>+		addr = (u64)rdev-&gt;pacing.dbr_page;</div><div class='add'>+		mmap_flag = BNXT_RE_MMAP_DBR_PAGE;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	default:</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &amp;mmap_offset);</div><div class='add'>+	if (!entry)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	uobj-&gt;object = entry;</div><div class='add'>+	uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_PAGE_HANDLE);</div><div class='add'>+	err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_MMAP_OFFSET,</div><div class='add'>+			     &amp;mmap_offset, sizeof(mmap_offset));</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_MMAP_LENGTH,</div><div class='add'>+			     &amp;length, sizeof(length));</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_DPI,</div><div class='add'>+			     &amp;dpi, sizeof(dpi));</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int alloc_page_obj_cleanup(struct ib_uobject *uobject,</div><div class='add'>+				  enum rdma_remove_reason why,</div><div class='add'>+			    struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct  bnxt_re_user_mmap_entry *entry = uobject-&gt;object;</div><div class='add'>+	struct bnxt_re_ucontext *uctx = entry-&gt;uctx;</div><div class='add'>+</div><div class='add'>+	switch (entry-&gt;mmap_flag) {</div><div class='add'>+	case BNXT_RE_MMAP_WC_DB:</div><div class='add'>+		if (uctx &amp;&amp; uctx-&gt;wcdpi.dbr) {</div><div class='add'>+			struct bnxt_re_dev *rdev = uctx-&gt;rdev;</div><div class='add'>+</div><div class='add'>+			bnxt_qplib_dealloc_dpi(&amp;rdev-&gt;qplib_res, &amp;uctx-&gt;wcdpi);</div><div class='add'>+			uctx-&gt;wcdpi.dbr = NULL;</div><div class='ctx'> 		}</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_RE_MMAP_DBR_BAR:</div><div class='add'>+	case BNXT_RE_MMAP_DBR_PAGE:</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		goto exit;</div><div class='add'>+	}</div><div class='add'>+	rdma_user_mmap_entry_remove(&amp;entry-&gt;rdma_entry);</div><div class='add'>+exit:</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_ALLOC_PAGE,</div><div class='add'>+			    UVERBS_ATTR_IDR(BNXT_RE_ALLOC_PAGE_HANDLE,</div><div class='add'>+					    BNXT_RE_OBJECT_ALLOC_PAGE,</div><div class='add'>+					    UVERBS_ACCESS_NEW,</div><div class='add'>+					    UA_MANDATORY),</div><div class='add'>+			    UVERBS_ATTR_CONST_IN(BNXT_RE_ALLOC_PAGE_TYPE,</div><div class='add'>+						 enum bnxt_re_alloc_page_type,</div><div class='add'>+						 UA_MANDATORY),</div><div class='add'>+			    UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_OFFSET,</div><div class='add'>+						UVERBS_ATTR_TYPE(u64),</div><div class='add'>+						UA_MANDATORY),</div><div class='add'>+			    UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_LENGTH,</div><div class='add'>+						UVERBS_ATTR_TYPE(u32),</div><div class='add'>+						UA_MANDATORY),</div><div class='add'>+			    UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_DPI,</div><div class='add'>+						UVERBS_ATTR_TYPE(u32),</div><div class='add'>+						UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_DESTROY_PAGE,</div><div class='add'>+				    UVERBS_ATTR_IDR(BNXT_RE_DESTROY_PAGE_HANDLE,</div><div class='add'>+						    BNXT_RE_OBJECT_ALLOC_PAGE,</div><div class='add'>+						    UVERBS_ACCESS_DESTROY,</div><div class='add'>+						    UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_ALLOC_PAGE,</div><div class='add'>+			    UVERBS_TYPE_ALLOC_IDR(alloc_page_obj_cleanup),</div><div class='add'>+			    &amp;UVERBS_METHOD(BNXT_RE_METHOD_ALLOC_PAGE),</div><div class='add'>+			    &amp;UVERBS_METHOD(BNXT_RE_METHOD_DESTROY_PAGE));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_NOTIFY_DRV);</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_NOTIFY_DRV,</div><div class='add'>+			      &amp;UVERBS_METHOD(BNXT_RE_METHOD_NOTIFY_DRV));</div><div class='add'>+</div><div class='add'>+/* Toggle MEM */</div><div class='add'>+static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_TOGGLE_MEM_HANDLE);</div><div class='add'>+	enum bnxt_re_mmap_flag mmap_flag = BNXT_RE_MMAP_TOGGLE_PAGE;</div><div class='add'>+	enum bnxt_re_get_toggle_mem_type res_type;</div><div class='add'>+	struct bnxt_re_user_mmap_entry *entry;</div><div class='add'>+	struct bnxt_re_ucontext *uctx;</div><div class='add'>+	struct ib_ucontext *ib_uctx;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct bnxt_re_srq *srq;</div><div class='add'>+	u32 length = PAGE_SIZE;</div><div class='add'>+	struct bnxt_re_cq *cq;</div><div class='add'>+	u64 mem_offset;</div><div class='add'>+	u32 offset = 0;</div><div class='add'>+	u64 addr = 0;</div><div class='add'>+	u32 res_id;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	ib_uctx = ib_uverbs_get_ucontext(attrs);</div><div class='add'>+	if (IS_ERR(ib_uctx))</div><div class='add'>+		return PTR_ERR(ib_uctx);</div><div class='add'>+</div><div class='add'>+	err = uverbs_get_const(&amp;res_type, attrs, BNXT_RE_TOGGLE_MEM_TYPE);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);</div><div class='add'>+	rdev = uctx-&gt;rdev;</div><div class='add'>+	err = uverbs_copy_from(&amp;res_id, attrs, BNXT_RE_TOGGLE_MEM_RES_ID);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	switch (res_type) {</div><div class='add'>+	case BNXT_RE_CQ_TOGGLE_MEM:</div><div class='add'>+		cq = bnxt_re_search_for_cq(rdev, res_id);</div><div class='add'>+		if (!cq)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		addr = (u64)cq-&gt;uctx_cq_page;</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_RE_SRQ_TOGGLE_MEM:</div><div class='add'>+		srq = bnxt_re_search_for_srq(rdev, res_id);</div><div class='add'>+		if (!srq)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		addr = (u64)srq-&gt;uctx_srq_page;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	default:</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &amp;mem_offset);</div><div class='add'>+	if (!entry)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	uobj-&gt;object = entry;</div><div class='add'>+	uverbs_finalize_uobj_create(attrs, BNXT_RE_TOGGLE_MEM_HANDLE);</div><div class='add'>+	err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_PAGE,</div><div class='add'>+			     &amp;mem_offset, sizeof(mem_offset));</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_LENGTH,</div><div class='add'>+			     &amp;length, sizeof(length));</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_OFFSET,</div><div class='add'>+			     &amp;offset, sizeof(offset));</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_toggle_mem_obj_cleanup(struct ib_uobject *uobject,</div><div class='add'>+				      enum rdma_remove_reason why,</div><div class='add'>+				      struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct  bnxt_re_user_mmap_entry *entry = uobject-&gt;object;</div><div class='add'>+</div><div class='add'>+	rdma_user_mmap_entry_remove(&amp;entry-&gt;rdma_entry);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_TOGGLE_MEM,</div><div class='add'>+			    UVERBS_ATTR_IDR(BNXT_RE_TOGGLE_MEM_HANDLE,</div><div class='add'>+					    BNXT_RE_OBJECT_GET_TOGGLE_MEM,</div><div class='add'>+					    UVERBS_ACCESS_NEW,</div><div class='add'>+					    UA_MANDATORY),</div><div class='add'>+			    UVERBS_ATTR_CONST_IN(BNXT_RE_TOGGLE_MEM_TYPE,</div><div class='add'>+						 enum bnxt_re_get_toggle_mem_type,</div><div class='add'>+						 UA_MANDATORY),</div><div class='add'>+			    UVERBS_ATTR_PTR_IN(BNXT_RE_TOGGLE_MEM_RES_ID,</div><div class='add'>+					       UVERBS_ATTR_TYPE(u32),</div><div class='add'>+					       UA_MANDATORY),</div><div class='add'>+			    UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_PAGE,</div><div class='add'>+						UVERBS_ATTR_TYPE(u64),</div><div class='add'>+						UA_MANDATORY),</div><div class='add'>+			    UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_OFFSET,</div><div class='add'>+						UVERBS_ATTR_TYPE(u32),</div><div class='add'>+						UA_MANDATORY),</div><div class='add'>+			    UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_LENGTH,</div><div class='add'>+						UVERBS_ATTR_TYPE(u32),</div><div class='add'>+						UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_RELEASE_TOGGLE_MEM,</div><div class='add'>+				    UVERBS_ATTR_IDR(BNXT_RE_RELEASE_TOGGLE_MEM_HANDLE,</div><div class='add'>+						    BNXT_RE_OBJECT_GET_TOGGLE_MEM,</div><div class='add'>+						    UVERBS_ACCESS_DESTROY,</div><div class='add'>+						    UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_GET_TOGGLE_MEM,</div><div class='add'>+			    UVERBS_TYPE_ALLOC_IDR(get_toggle_mem_obj_cleanup),</div><div class='add'>+			    &amp;UVERBS_METHOD(BNXT_RE_METHOD_GET_TOGGLE_MEM),</div><div class='add'>+			    &amp;UVERBS_METHOD(BNXT_RE_METHOD_RELEASE_TOGGLE_MEM));</div><div class='add'>+</div><div class='add'>+const struct uapi_definition bnxt_re_uapi_defs[] = {</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE),</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV),</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_GET_TOGGLE_MEM),</div><div class='add'>+	{}</div><div class='add'>+};</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h<br/>index 09a33049e42f..76ba9ab04d5c 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/ib_verbs.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/ib_verbs.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/ib_verbs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/ib_verbs.h</a></div><div class='hunk'>@@ -60,6 +60,8 @@ struct bnxt_re_pd {</div><div class='ctx'> 	struct bnxt_re_dev	*rdev;</div><div class='ctx'> 	struct bnxt_qplib_pd	qplib_pd;</div><div class='ctx'> 	struct bnxt_re_fence_data fence;</div><div class='add'>+	struct rdma_user_mmap_entry *pd_db_mmap;</div><div class='add'>+	struct rdma_user_mmap_entry *pd_wcdb_mmap;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct bnxt_re_ah {</div><div class='hunk'>@@ -75,12 +77,14 @@ struct bnxt_re_srq {</div><div class='ctx'> 	struct bnxt_qplib_srq	qplib_srq;</div><div class='ctx'> 	struct ib_umem		*umem;</div><div class='ctx'> 	spinlock_t		lock;		/* protect srq */</div><div class='add'>+	void			*uctx_srq_page;</div><div class='add'>+	struct hlist_node       hash_entry;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct bnxt_re_qp {</div><div class='add'>+	struct ib_qp		ib_qp;</div><div class='ctx'> 	struct list_head	list;</div><div class='ctx'> 	struct bnxt_re_dev	*rdev;</div><div class='del'>-	struct ib_qp		ib_qp;</div><div class='ctx'> 	spinlock_t		sq_lock;	/* protect sq */</div><div class='ctx'> 	spinlock_t		rq_lock;	/* protect rq */</div><div class='ctx'> 	struct bnxt_qplib_qp	qplib_qp;</div><div class='hunk'>@@ -91,19 +95,24 @@ struct bnxt_re_qp {</div><div class='ctx'> 	struct ib_ud_header	qp1_hdr;</div><div class='ctx'> 	struct bnxt_re_cq	*scq;</div><div class='ctx'> 	struct bnxt_re_cq	*rcq;</div><div class='add'>+	struct dentry		*dentry;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct bnxt_re_cq {</div><div class='add'>+	struct ib_cq		ib_cq;</div><div class='ctx'> 	struct bnxt_re_dev	*rdev;</div><div class='ctx'> 	spinlock_t              cq_lock;	/* protect cq */</div><div class='ctx'> 	u16			cq_count;</div><div class='ctx'> 	u16			cq_period;</div><div class='del'>-	struct ib_cq		ib_cq;</div><div class='ctx'> 	struct bnxt_qplib_cq	qplib_cq;</div><div class='ctx'> 	struct bnxt_qplib_cqe	*cql;</div><div class='ctx'> #define MAX_CQL_PER_POLL	1024</div><div class='ctx'> 	u32			max_cql;</div><div class='ctx'> 	struct ib_umem		*umem;</div><div class='add'>+	struct ib_umem		*resize_umem;</div><div class='add'>+	int			resize_cqe;</div><div class='add'>+	void			*uctx_cq_page;</div><div class='add'>+	struct hlist_node	hash_entry;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct bnxt_re_mr {</div><div class='hunk'>@@ -122,12 +131,6 @@ struct bnxt_re_frpl {</div><div class='ctx'> 	u64				*page_list;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct bnxt_re_fmr {</div><div class='del'>-	struct bnxt_re_dev	*rdev;</div><div class='del'>-	struct ib_fmr		ib_fmr;</div><div class='del'>-	struct bnxt_qplib_mrw	qplib_fmr;</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> struct bnxt_re_mw {</div><div class='ctx'> 	struct bnxt_re_dev	*rdev;</div><div class='ctx'> 	struct ib_mw		ib_mw;</div><div class='hunk'>@@ -138,36 +141,89 @@ struct bnxt_re_ucontext {</div><div class='ctx'> 	struct ib_ucontext      ib_uctx;</div><div class='ctx'> 	struct bnxt_re_dev	*rdev;</div><div class='ctx'> 	struct bnxt_qplib_dpi	dpi;</div><div class='add'>+	struct bnxt_qplib_dpi   wcdpi;</div><div class='ctx'> 	void			*shpg;</div><div class='ctx'> 	spinlock_t		sh_lock;	/* protect shpg */</div><div class='add'>+	struct rdma_user_mmap_entry *shpage_mmap;</div><div class='add'>+	u64 cmask;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum bnxt_re_mmap_flag {</div><div class='add'>+	BNXT_RE_MMAP_SH_PAGE,</div><div class='add'>+	BNXT_RE_MMAP_UC_DB,</div><div class='add'>+	BNXT_RE_MMAP_WC_DB,</div><div class='add'>+	BNXT_RE_MMAP_DBR_PAGE,</div><div class='add'>+	BNXT_RE_MMAP_DBR_BAR,</div><div class='add'>+	BNXT_RE_MMAP_TOGGLE_PAGE,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_re_user_mmap_entry {</div><div class='add'>+	struct rdma_user_mmap_entry rdma_entry;</div><div class='add'>+	struct bnxt_re_ucontext *uctx;</div><div class='add'>+	u64 mem_offset;</div><div class='add'>+	u8 mmap_flag;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_re_flow {</div><div class='add'>+	struct ib_flow		ib_flow;</div><div class='add'>+	struct bnxt_re_dev	*rdev;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline u16 bnxt_re_get_swqe_size(int nsge)</div><div class='add'>+{</div><div class='add'>+	return sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u16 bnxt_re_get_rwqe_size(int nsge)</div><div class='add'>+{</div><div class='add'>+	return sizeof(struct rq_wqe_hdr) + (nsge * sizeof(struct sq_sge));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	BNXT_RE_UCNTX_CAP_POW2_DISABLED = 0x1ULL,</div><div class='add'>+	BNXT_RE_UCNTX_CAP_VAR_WQE_ENABLED = 0x2ULL,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static inline u32 bnxt_re_init_depth(u32 ent, struct bnxt_re_ucontext *uctx)</div><div class='add'>+{</div><div class='add'>+	return uctx ? (uctx-&gt;cmask &amp; BNXT_RE_UCNTX_CAP_POW2_DISABLED) ?</div><div class='add'>+		ent : roundup_pow_of_two(ent) : ent;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool bnxt_re_is_var_size_supported(struct bnxt_re_dev *rdev,</div><div class='add'>+						 struct bnxt_re_ucontext *uctx)</div><div class='add'>+{</div><div class='add'>+	if (uctx)</div><div class='add'>+		return uctx-&gt;cmask &amp; BNXT_RE_UCNTX_CAP_VAR_WQE_ENABLED;</div><div class='add'>+	else</div><div class='add'>+		return rdev-&gt;chip_ctx-&gt;modes.wqe_mode;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int bnxt_re_query_device(struct ib_device *ibdev,</div><div class='ctx'> 			 struct ib_device_attr *ib_attr,</div><div class='ctx'> 			 struct ib_udata *udata);</div><div class='ctx'> int bnxt_re_modify_device(struct ib_device *ibdev,</div><div class='ctx'> 			  int device_modify_mask,</div><div class='ctx'> 			  struct ib_device_modify *device_modify);</div><div class='del'>-int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,</div><div class='add'>+int bnxt_re_query_port(struct ib_device *ibdev, u32 port_num,</div><div class='ctx'> 		       struct ib_port_attr *port_attr);</div><div class='del'>-int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num,</div><div class='add'>+int bnxt_re_get_port_immutable(struct ib_device *ibdev, u32 port_num,</div><div class='ctx'> 			       struct ib_port_immutable *immutable);</div><div class='ctx'> void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str);</div><div class='del'>-int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num,</div><div class='add'>+int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num,</div><div class='ctx'> 		       u16 index, u16 *pkey);</div><div class='ctx'> int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context);</div><div class='ctx'> int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context);</div><div class='del'>-int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num,</div><div class='add'>+int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num,</div><div class='ctx'> 		      int index, union ib_gid *gid);</div><div class='ctx'> enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,</div><div class='del'>-					    u8 port_num);</div><div class='add'>+					    u32 port_num);</div><div class='ctx'> int bnxt_re_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);</div><div class='del'>-void bnxt_re_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);</div><div class='del'>-int bnxt_re_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,</div><div class='add'>+int bnxt_re_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);</div><div class='add'>+int bnxt_re_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr,</div><div class='ctx'> 		      struct ib_udata *udata);</div><div class='del'>-int bnxt_re_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);</div><div class='ctx'> int bnxt_re_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);</div><div class='del'>-void bnxt_re_destroy_ah(struct ib_ah *ah, u32 flags);</div><div class='add'>+int bnxt_re_destroy_ah(struct ib_ah *ah, u32 flags);</div><div class='ctx'> int bnxt_re_create_srq(struct ib_srq *srq,</div><div class='ctx'> 		       struct ib_srq_init_attr *srq_init_attr,</div><div class='ctx'> 		       struct ib_udata *udata);</div><div class='hunk'>@@ -175,12 +231,11 @@ int bnxt_re_modify_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr,</div><div class='ctx'> 		       enum ib_srq_attr_mask srq_attr_mask,</div><div class='ctx'> 		       struct ib_udata *udata);</div><div class='ctx'> int bnxt_re_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);</div><div class='del'>-void bnxt_re_destroy_srq(struct ib_srq *srq, struct ib_udata *udata);</div><div class='add'>+int bnxt_re_destroy_srq(struct ib_srq *srq, struct ib_udata *udata);</div><div class='ctx'> int bnxt_re_post_srq_recv(struct ib_srq *srq, const struct ib_recv_wr *recv_wr,</div><div class='ctx'> 			  const struct ib_recv_wr **bad_recv_wr);</div><div class='del'>-struct ib_qp *bnxt_re_create_qp(struct ib_pd *pd,</div><div class='del'>-				struct ib_qp_init_attr *qp_init_attr,</div><div class='del'>-				struct ib_udata *udata);</div><div class='add'>+int bnxt_re_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr,</div><div class='add'>+		      struct ib_udata *udata);</div><div class='ctx'> int bnxt_re_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 		      int qp_attr_mask, struct ib_udata *udata);</div><div class='ctx'> int bnxt_re_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,</div><div class='hunk'>@@ -190,9 +245,9 @@ int bnxt_re_post_send(struct ib_qp *qp, const struct ib_send_wr *send_wr,</div><div class='ctx'> 		      const struct ib_send_wr **bad_send_wr);</div><div class='ctx'> int bnxt_re_post_recv(struct ib_qp *qp, const struct ib_recv_wr *recv_wr,</div><div class='ctx'> 		      const struct ib_recv_wr **bad_recv_wr);</div><div class='del'>-struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,</div><div class='del'>-				const struct ib_cq_init_attr *attr,</div><div class='del'>-				struct ib_udata *udata);</div><div class='add'>+int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		      struct uverbs_attr_bundle *attrs);</div><div class='add'>+int bnxt_re_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata);</div><div class='ctx'> int bnxt_re_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);</div><div class='ctx'> int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc);</div><div class='ctx'> int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);</div><div class='hunk'>@@ -201,17 +256,40 @@ struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags);</div><div class='ctx'> int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,</div><div class='ctx'> 		      unsigned int *sg_offset);</div><div class='ctx'> struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type mr_type,</div><div class='del'>-			       u32 max_num_sg, struct ib_udata *udata);</div><div class='add'>+			       u32 max_num_sg);</div><div class='ctx'> int bnxt_re_dereg_mr(struct ib_mr *mr, struct ib_udata *udata);</div><div class='ctx'> struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,</div><div class='ctx'> 			       struct ib_udata *udata);</div><div class='ctx'> int bnxt_re_dealloc_mw(struct ib_mw *mw);</div><div class='ctx'> struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='ctx'> 				  u64 virt_addr, int mr_access_flags,</div><div class='add'>+				  struct ib_dmah *dmah,</div><div class='ctx'> 				  struct ib_udata *udata);</div><div class='add'>+struct ib_mr *bnxt_re_reg_user_mr_dmabuf(struct ib_pd *ib_pd, u64 start,</div><div class='add'>+					 u64 length, u64 virt_addr,</div><div class='add'>+					 int fd, int mr_access_flags,</div><div class='add'>+					 struct ib_dmah *dmah,</div><div class='add'>+					 struct uverbs_attr_bundle *attrs);</div><div class='ctx'> int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata);</div><div class='ctx'> void bnxt_re_dealloc_ucontext(struct ib_ucontext *context);</div><div class='add'>+struct ib_flow *bnxt_re_create_flow(struct ib_qp *ib_qp,</div><div class='add'>+				    struct ib_flow_attr *attr,</div><div class='add'>+				    struct ib_udata *udata);</div><div class='add'>+int bnxt_re_destroy_flow(struct ib_flow *flow_id);</div><div class='add'>+</div><div class='ctx'> int bnxt_re_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);</div><div class='add'>+void bnxt_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry);</div><div class='add'>+</div><div class='add'>+int bnxt_re_process_mad(struct ib_device *device, int process_mad_flags,</div><div class='add'>+			u32 port_num, const struct ib_wc *in_wc,</div><div class='add'>+			const struct ib_grh *in_grh,</div><div class='add'>+			const struct ib_mad *in_mad, struct ib_mad *out_mad,</div><div class='add'>+			size_t *out_mad_size, u16 *out_mad_pkey_index);</div><div class='add'>+</div><div class='add'>+static inline u32 __to_ib_port_num(u16 port_id)</div><div class='add'>+{</div><div class='add'>+	return (u32)port_id + 1;</div><div class='add'>+}</div><div class='ctx'> </div><div class='ctx'> unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp);</div><div class='ctx'> void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, unsigned long flags);</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c<br/>index 814f959c7db9..73003ad25ee8 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/main.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/main.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/main.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/main.c</a></div><div class='hunk'>@@ -48,11 +48,13 @@</div><div class='ctx'> #include &lt;net/ipv6.h&gt;</div><div class='ctx'> #include &lt;net/addrconf.h&gt;</div><div class='ctx'> #include &lt;linux/if_ether.h&gt;</div><div class='add'>+#include &lt;linux/auxiliary_bus.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include &lt;rdma/ib_verbs.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_user_verbs.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_umem.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_addr.h&gt;</div><div class='add'>+#include &lt;linux/hashtable.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "bnxt_ulp.h"</div><div class='ctx'> #include "roce_hsi.h"</div><div class='hunk'>@@ -65,52 +67,157 @@</div><div class='ctx'> #include &lt;rdma/bnxt_re-abi.h&gt;</div><div class='ctx'> #include "bnxt.h"</div><div class='ctx'> #include "hw_counters.h"</div><div class='add'>+#include "debugfs.h"</div><div class='ctx'> </div><div class='ctx'> static char version[] =</div><div class='ctx'> 		BNXT_RE_DESC "\n";</div><div class='ctx'> </div><div class='ctx'> MODULE_AUTHOR("Eddie Wai &lt;eddie.wai@broadcom.com&gt;");</div><div class='del'>-MODULE_DESCRIPTION(BNXT_RE_DESC " Driver");</div><div class='add'>+MODULE_DESCRIPTION(BNXT_RE_DESC);</div><div class='ctx'> MODULE_LICENSE("Dual BSD/GPL");</div><div class='ctx'> </div><div class='ctx'> /* globals */</div><div class='del'>-static struct list_head bnxt_re_dev_list = LIST_HEAD_INIT(bnxt_re_dev_list);</div><div class='del'>-/* Mutex to protect the list of bnxt_re devices added */</div><div class='del'>-static DEFINE_MUTEX(bnxt_re_dev_lock);</div><div class='del'>-static struct workqueue_struct *bnxt_re_wq;</div><div class='del'>-static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev);</div><div class='add'>+static DEFINE_MUTEX(bnxt_re_mutex);</div><div class='add'>+</div><div class='add'>+static int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev);</div><div class='add'>+static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev);</div><div class='add'>+</div><div class='add'>+static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len,</div><div class='add'>+			     u32 *offset);</div><div class='add'>+static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp,</div><div class='add'>+				   u8 port_num, enum ib_event_type event);</div><div class='add'>+static void bnxt_re_set_db_offset(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_chip_ctx *cctx;</div><div class='add'>+	struct bnxt_en_dev *en_dev;</div><div class='add'>+	struct bnxt_qplib_res *res;</div><div class='add'>+	u32 l2db_len = 0;</div><div class='add'>+	u32 offset = 0;</div><div class='add'>+	u32 barlen;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	res = &amp;rdev-&gt;qplib_res;</div><div class='add'>+	en_dev = rdev-&gt;en_dev;</div><div class='add'>+	cctx = rdev-&gt;chip_ctx;</div><div class='add'>+</div><div class='add'>+	/* Issue qcfg */</div><div class='add'>+	rc = bnxt_re_hwrm_qcfg(rdev, &amp;l2db_len, &amp;offset);</div><div class='add'>+	if (rc)</div><div class='add'>+		dev_info(rdev_to_dev(rdev),</div><div class='add'>+			 "Couldn't get DB bar size, Low latency framework is disabled\n");</div><div class='add'>+	/* set register offsets for both UC and WC */</div><div class='add'>+	if (bnxt_qplib_is_chip_gen_p7(cctx)) {</div><div class='add'>+		res-&gt;dpi_tbl.ucreg.offset = offset;</div><div class='add'>+		res-&gt;dpi_tbl.wcreg.offset = en_dev-&gt;l2_db_size;</div><div class='add'>+	} else {</div><div class='add'>+		res-&gt;dpi_tbl.ucreg.offset = res-&gt;is_vf ? BNXT_QPLIB_DBR_VF_DB_OFFSET :</div><div class='add'>+							 BNXT_QPLIB_DBR_PF_DB_OFFSET;</div><div class='add'>+		res-&gt;dpi_tbl.wcreg.offset = res-&gt;dpi_tbl.ucreg.offset;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* If WC mapping is disabled by L2 driver then en_dev-&gt;l2_db_size</div><div class='add'>+	 * is equal to the DB-Bar actual size. This indicates that L2</div><div class='add'>+	 * is mapping entire bar as UC-. RoCE driver can't enable WC mapping</div><div class='add'>+	 * in such cases and DB-push will be disabled.</div><div class='add'>+	 */</div><div class='add'>+	barlen = pci_resource_len(res-&gt;pdev, RCFW_DBR_PCI_BAR_REGION);</div><div class='add'>+	if (cctx-&gt;modes.db_push &amp;&amp; l2db_len &amp;&amp; en_dev-&gt;l2_db_size != barlen) {</div><div class='add'>+		res-&gt;dpi_tbl.wcreg.offset = en_dev-&gt;l2_db_size;</div><div class='add'>+		dev_info(rdev_to_dev(rdev),  "Low latency framework is enabled\n");</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_chip_ctx *cctx;</div><div class='add'>+</div><div class='add'>+	cctx = rdev-&gt;chip_ctx;</div><div class='add'>+	cctx-&gt;modes.wqe_mode = bnxt_qplib_is_chip_gen_p7(rdev-&gt;chip_ctx) ?</div><div class='add'>+			       BNXT_QPLIB_WQE_MODE_VARIABLE : BNXT_QPLIB_WQE_MODE_STATIC;</div><div class='add'>+	if (bnxt_re_hwrm_qcaps(rdev))</div><div class='add'>+		dev_err(rdev_to_dev(rdev),</div><div class='add'>+			"Failed to query hwrm qcaps\n");</div><div class='add'>+	if (bnxt_qplib_is_chip_gen_p7(rdev-&gt;chip_ctx)) {</div><div class='add'>+		cctx-&gt;modes.toggle_bits |= BNXT_QPLIB_CQ_TOGGLE_BIT;</div><div class='add'>+		cctx-&gt;modes.toggle_bits |= BNXT_QPLIB_SRQ_TOGGLE_BIT;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='ctx'> </div><div class='ctx'> static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='add'>+	struct bnxt_qplib_chip_ctx *chip_ctx;</div><div class='add'>+</div><div class='add'>+	if (!rdev-&gt;chip_ctx)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	kfree(rdev-&gt;dev_attr);</div><div class='add'>+	rdev-&gt;dev_attr = NULL;</div><div class='add'>+</div><div class='add'>+	chip_ctx = rdev-&gt;chip_ctx;</div><div class='add'>+	rdev-&gt;chip_ctx = NULL;</div><div class='ctx'> 	rdev-&gt;rcfw.res = NULL;</div><div class='ctx'> 	rdev-&gt;qplib_res.cctx = NULL;</div><div class='add'>+	rdev-&gt;qplib_res.pdev = NULL;</div><div class='add'>+	rdev-&gt;qplib_res.netdev = NULL;</div><div class='add'>+	kfree(chip_ctx);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='add'>+	struct bnxt_qplib_chip_ctx *chip_ctx;</div><div class='ctx'> 	struct bnxt_en_dev *en_dev;</div><div class='del'>-	struct bnxt *bp;</div><div class='add'>+	int rc = -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	en_dev = rdev-&gt;en_dev;</div><div class='del'>-	bp = netdev_priv(en_dev-&gt;net);</div><div class='ctx'> </div><div class='del'>-	rdev-&gt;chip_ctx.chip_num = bp-&gt;chip_num;</div><div class='add'>+	rdev-&gt;qplib_res.pdev = en_dev-&gt;pdev;</div><div class='add'>+	chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL);</div><div class='add'>+	if (!chip_ctx)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	chip_ctx-&gt;chip_num = en_dev-&gt;chip_num;</div><div class='add'>+	chip_ctx-&gt;hw_stats_size = en_dev-&gt;hw_ring_stats_size;</div><div class='add'>+</div><div class='add'>+	rdev-&gt;chip_ctx = chip_ctx;</div><div class='ctx'> 	/* rest members to follow eventually */</div><div class='ctx'> </div><div class='del'>-	rdev-&gt;qplib_res.cctx = &amp;rdev-&gt;chip_ctx;</div><div class='add'>+	rdev-&gt;qplib_res.cctx = rdev-&gt;chip_ctx;</div><div class='ctx'> 	rdev-&gt;rcfw.res = &amp;rdev-&gt;qplib_res;</div><div class='add'>+	rdev-&gt;dev_attr = kzalloc(sizeof(*rdev-&gt;dev_attr), GFP_KERNEL);</div><div class='add'>+	if (!rdev-&gt;dev_attr)</div><div class='add'>+		goto free_chip_ctx;</div><div class='add'>+	rdev-&gt;qplib_res.dattr = rdev-&gt;dev_attr;</div><div class='add'>+	rdev-&gt;qplib_res.is_vf = BNXT_EN_VF(en_dev);</div><div class='add'>+	rdev-&gt;qplib_res.en_dev = en_dev;</div><div class='add'>+</div><div class='add'>+	rc = bnxt_re_query_hwrm_intf_version(rdev);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto free_dev_attr;</div><div class='ctx'> </div><div class='add'>+	bnxt_re_set_drv_mode(rdev);</div><div class='add'>+</div><div class='add'>+	bnxt_re_set_db_offset(rdev);</div><div class='add'>+	rc = bnxt_qplib_map_db_bar(&amp;rdev-&gt;qplib_res);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto free_dev_attr;</div><div class='add'>+</div><div class='add'>+	if (bnxt_qplib_determine_atomics(en_dev-&gt;pdev))</div><div class='add'>+		ibdev_info(&amp;rdev-&gt;ibdev,</div><div class='add'>+			   "platform doesn't support global atomics.");</div><div class='ctx'> 	return 0;</div><div class='add'>+free_dev_attr:</div><div class='add'>+	kfree(rdev-&gt;dev_attr);</div><div class='add'>+	rdev-&gt;dev_attr = NULL;</div><div class='add'>+free_chip_ctx:</div><div class='add'>+	kfree(rdev-&gt;chip_ctx);</div><div class='add'>+	rdev-&gt;chip_ctx = NULL;</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* SR-IOV helper functions */</div><div class='ctx'> </div><div class='ctx'> static void bnxt_re_get_sriov_func_type(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt *bp;</div><div class='del'>-</div><div class='del'>-	bp = netdev_priv(rdev-&gt;en_dev-&gt;net);</div><div class='del'>-	if (BNXT_VF(bp))</div><div class='add'>+	if (BNXT_EN_VF(rdev-&gt;en_dev))</div><div class='ctx'> 		rdev-&gt;is_virtfn = 1;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -119,104 +226,237 @@ static void bnxt_re_get_sriov_func_type(struct bnxt_re_dev *rdev)</div><div class='ctx'>  * reserved for the function. The driver may choose to allocate fewer</div><div class='ctx'>  * resources than the firmware maximum.</div><div class='ctx'>  */</div><div class='del'>-static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev)</div><div class='add'>+static void bnxt_re_limit_pf_res(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='del'>-	u32 vf_qps = 0, vf_srqs = 0, vf_cqs = 0, vf_mrws = 0, vf_gids = 0;</div><div class='del'>-	u32 i;</div><div class='del'>-	u32 vf_pct;</div><div class='del'>-	u32 num_vfs;</div><div class='del'>-	struct bnxt_qplib_dev_attr *dev_attr = &amp;rdev-&gt;dev_attr;</div><div class='add'>+	struct bnxt_qplib_dev_attr *attr;</div><div class='add'>+	struct bnxt_qplib_ctx *ctx;</div><div class='add'>+	int i;</div><div class='ctx'> </div><div class='del'>-	rdev-&gt;qplib_ctx.qpc_count = min_t(u32, BNXT_RE_MAX_QPC_COUNT,</div><div class='del'>-					  dev_attr-&gt;max_qp);</div><div class='add'>+	attr = rdev-&gt;dev_attr;</div><div class='add'>+	ctx = &amp;rdev-&gt;qplib_ctx;</div><div class='ctx'> </div><div class='del'>-	rdev-&gt;qplib_ctx.mrw_count = BNXT_RE_MAX_MRW_COUNT_256K;</div><div class='add'>+	ctx-&gt;qpc_count = min_t(u32, BNXT_RE_MAX_QPC_COUNT,</div><div class='add'>+			       attr-&gt;max_qp);</div><div class='add'>+	ctx-&gt;mrw_count = BNXT_RE_MAX_MRW_COUNT_256K;</div><div class='ctx'> 	/* Use max_mr from fw since max_mrw does not get set */</div><div class='del'>-	rdev-&gt;qplib_ctx.mrw_count = min_t(u32, rdev-&gt;qplib_ctx.mrw_count,</div><div class='del'>-					  dev_attr-&gt;max_mr);</div><div class='del'>-	rdev-&gt;qplib_ctx.srqc_count = min_t(u32, BNXT_RE_MAX_SRQC_COUNT,</div><div class='del'>-					   dev_attr-&gt;max_srq);</div><div class='del'>-	rdev-&gt;qplib_ctx.cq_count = min_t(u32, BNXT_RE_MAX_CQ_COUNT,</div><div class='del'>-					 dev_attr-&gt;max_cq);</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; MAX_TQM_ALLOC_REQ; i++)</div><div class='del'>-		rdev-&gt;qplib_ctx.tqm_count[i] =</div><div class='del'>-		rdev-&gt;dev_attr.tqm_alloc_reqs[i];</div><div class='del'>-</div><div class='del'>-	if (rdev-&gt;num_vfs) {</div><div class='del'>-		/*</div><div class='del'>-		 * Reserve a set of resources for the PF. Divide the remaining</div><div class='del'>-		 * resources among the VFs</div><div class='del'>-		 */</div><div class='del'>-		vf_pct = 100 - BNXT_RE_PCT_RSVD_FOR_PF;</div><div class='del'>-		num_vfs = 100 * rdev-&gt;num_vfs;</div><div class='del'>-		vf_qps = (rdev-&gt;qplib_ctx.qpc_count * vf_pct) / num_vfs;</div><div class='del'>-		vf_srqs = (rdev-&gt;qplib_ctx.srqc_count * vf_pct) / num_vfs;</div><div class='del'>-		vf_cqs = (rdev-&gt;qplib_ctx.cq_count * vf_pct) / num_vfs;</div><div class='del'>-		/*</div><div class='del'>-		 * The driver allows many more MRs than other resources. If the</div><div class='del'>-		 * firmware does also, then reserve a fixed amount for the PF</div><div class='del'>-		 * and divide the rest among VFs. VFs may use many MRs for NFS</div><div class='del'>-		 * mounts, ISER, NVME applications, etc. If the firmware</div><div class='del'>-		 * severely restricts the number of MRs, then let PF have</div><div class='del'>-		 * half and divide the rest among VFs, as for the other</div><div class='del'>-		 * resource types.</div><div class='del'>-		 */</div><div class='del'>-		if (rdev-&gt;qplib_ctx.mrw_count &lt; BNXT_RE_MAX_MRW_COUNT_64K)</div><div class='del'>-			vf_mrws = rdev-&gt;qplib_ctx.mrw_count * vf_pct / num_vfs;</div><div class='del'>-		else</div><div class='del'>-			vf_mrws = (rdev-&gt;qplib_ctx.mrw_count -</div><div class='del'>-				   BNXT_RE_RESVD_MR_FOR_PF) / rdev-&gt;num_vfs;</div><div class='del'>-		vf_gids = BNXT_RE_MAX_GID_PER_VF;</div><div class='del'>-	}</div><div class='del'>-	rdev-&gt;qplib_ctx.vf_res.max_mrw_per_vf = vf_mrws;</div><div class='del'>-	rdev-&gt;qplib_ctx.vf_res.max_gid_per_vf = vf_gids;</div><div class='del'>-	rdev-&gt;qplib_ctx.vf_res.max_qp_per_vf = vf_qps;</div><div class='del'>-	rdev-&gt;qplib_ctx.vf_res.max_srq_per_vf = vf_srqs;</div><div class='del'>-	rdev-&gt;qplib_ctx.vf_res.max_cq_per_vf = vf_cqs;</div><div class='add'>+	ctx-&gt;mrw_count = min_t(u32, ctx-&gt;mrw_count, attr-&gt;max_mr);</div><div class='add'>+	ctx-&gt;srqc_count = min_t(u32, BNXT_RE_MAX_SRQC_COUNT,</div><div class='add'>+				attr-&gt;max_srq);</div><div class='add'>+	ctx-&gt;cq_count = min_t(u32, BNXT_RE_MAX_CQ_COUNT, attr-&gt;max_cq);</div><div class='add'>+	if (!bnxt_qplib_is_chip_gen_p5_p7(rdev-&gt;chip_ctx))</div><div class='add'>+		for (i = 0; i &lt; MAX_TQM_ALLOC_REQ; i++)</div><div class='add'>+			rdev-&gt;qplib_ctx.tqm_ctx.qcount[i] =</div><div class='add'>+			rdev-&gt;dev_attr-&gt;tqm_alloc_reqs[i];</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/* for handling bnxt_en callbacks later */</div><div class='del'>-static void bnxt_re_stop(void *p)</div><div class='add'>+static void bnxt_re_limit_vf_res(struct bnxt_qplib_ctx *qplib_ctx, u32 num_vf)</div><div class='ctx'> {</div><div class='add'>+	struct bnxt_qplib_vf_res *vf_res;</div><div class='add'>+	u32 mrws = 0;</div><div class='add'>+	u32 vf_pct;</div><div class='add'>+	u32 nvfs;</div><div class='add'>+</div><div class='add'>+	vf_res = &amp;qplib_ctx-&gt;vf_res;</div><div class='add'>+	/*</div><div class='add'>+	 * Reserve a set of resources for the PF. Divide the remaining</div><div class='add'>+	 * resources among the VFs</div><div class='add'>+	 */</div><div class='add'>+	vf_pct = 100 - BNXT_RE_PCT_RSVD_FOR_PF;</div><div class='add'>+	nvfs = num_vf;</div><div class='add'>+	num_vf = 100 * num_vf;</div><div class='add'>+	vf_res-&gt;max_qp_per_vf = (qplib_ctx-&gt;qpc_count * vf_pct) / num_vf;</div><div class='add'>+	vf_res-&gt;max_srq_per_vf = (qplib_ctx-&gt;srqc_count * vf_pct) / num_vf;</div><div class='add'>+	vf_res-&gt;max_cq_per_vf = (qplib_ctx-&gt;cq_count * vf_pct) / num_vf;</div><div class='add'>+	/*</div><div class='add'>+	 * The driver allows many more MRs than other resources. If the</div><div class='add'>+	 * firmware does also, then reserve a fixed amount for the PF and</div><div class='add'>+	 * divide the rest among VFs. VFs may use many MRs for NFS</div><div class='add'>+	 * mounts, ISER, NVME applications, etc. If the firmware severely</div><div class='add'>+	 * restricts the number of MRs, then let PF have half and divide</div><div class='add'>+	 * the rest among VFs, as for the other resource types.</div><div class='add'>+	 */</div><div class='add'>+	if (qplib_ctx-&gt;mrw_count &lt; BNXT_RE_MAX_MRW_COUNT_64K) {</div><div class='add'>+		mrws = qplib_ctx-&gt;mrw_count * vf_pct;</div><div class='add'>+		nvfs = num_vf;</div><div class='add'>+	} else {</div><div class='add'>+		mrws = qplib_ctx-&gt;mrw_count - BNXT_RE_RESVD_MR_FOR_PF;</div><div class='add'>+	}</div><div class='add'>+	vf_res-&gt;max_mrw_per_vf = (mrws / nvfs);</div><div class='add'>+	vf_res-&gt;max_gid_per_vf = BNXT_RE_MAX_GID_PER_VF;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void bnxt_re_start(void *p)</div><div class='add'>+static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='add'>+	u32 num_vfs;</div><div class='add'>+</div><div class='add'>+	memset(&amp;rdev-&gt;qplib_ctx.vf_res, 0, sizeof(struct bnxt_qplib_vf_res));</div><div class='add'>+	bnxt_re_limit_pf_res(rdev);</div><div class='add'>+</div><div class='add'>+	num_vfs =  bnxt_qplib_is_chip_gen_p5_p7(rdev-&gt;chip_ctx) ?</div><div class='add'>+			BNXT_RE_GEN_P5_MAX_VF : rdev-&gt;num_vfs;</div><div class='add'>+	if (num_vfs)</div><div class='add'>+		bnxt_re_limit_vf_res(&amp;rdev-&gt;qplib_ctx, num_vfs);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void bnxt_re_sriov_config(void *p, int num_vfs)</div><div class='add'>+static void bnxt_re_vf_res_config(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_re_dev *rdev = p;</div><div class='del'>-</div><div class='del'>-	if (!rdev)</div><div class='add'>+	/*</div><div class='add'>+	 * Use the total VF count since the actual VF count may not be</div><div class='add'>+	 * available at this point.</div><div class='add'>+	 */</div><div class='add'>+	rdev-&gt;num_vfs = pci_sriov_get_totalvfs(rdev-&gt;en_dev-&gt;pdev);</div><div class='add'>+	if (!rdev-&gt;num_vfs)</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='del'>-	rdev-&gt;num_vfs = num_vfs;</div><div class='ctx'> 	bnxt_re_set_resource_limits(rdev);</div><div class='ctx'> 	bnxt_qplib_set_func_resources(&amp;rdev-&gt;qplib_res, &amp;rdev-&gt;rcfw,</div><div class='ctx'> 				      &amp;rdev-&gt;qplib_ctx);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void bnxt_re_shutdown(void *p)</div><div class='add'>+struct bnxt_re_dcb_work {</div><div class='add'>+	struct work_struct work;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct hwrm_async_event_cmpl cmpl;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static bool bnxt_re_is_qp1_qp(struct bnxt_re_qp *qp)</div><div class='add'>+{</div><div class='add'>+	return qp-&gt;ib_qp.qp_type == IB_QPT_GSI;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct bnxt_re_qp *bnxt_re_get_qp1_qp(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_qp *qp;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;rdev-&gt;qp_lock);</div><div class='add'>+	list_for_each_entry(qp, &amp;rdev-&gt;qp_list, list) {</div><div class='add'>+		if (bnxt_re_is_qp1_qp(qp)) {</div><div class='add'>+			mutex_unlock(&amp;rdev-&gt;qp_lock);</div><div class='add'>+			return qp;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;rdev-&gt;qp_lock);</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bnxt_re_update_qp1_tos_dscp(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_qp *qp;</div><div class='add'>+</div><div class='add'>+	if (!bnxt_qplib_is_chip_gen_p5_p7(rdev-&gt;chip_ctx))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	qp = bnxt_re_get_qp1_qp(rdev);</div><div class='add'>+	if (!qp)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	qp-&gt;qplib_qp.modify_flags = CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP;</div><div class='add'>+	qp-&gt;qplib_qp.tos_dscp = rdev-&gt;cc_param.qp1_tos_dscp;</div><div class='add'>+</div><div class='add'>+	return bnxt_qplib_modify_qp(&amp;rdev-&gt;qplib_res, &amp;qp-&gt;qplib_qp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	rdev-&gt;dcb_wq = create_singlethread_workqueue("bnxt_re_dcb_wq");</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	if (!rdev-&gt;dcb_wq)</div><div class='add'>+		return;</div><div class='add'>+	destroy_workqueue(rdev-&gt;dcb_wq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_dcb_wq_task(struct work_struct *work)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_dcb_work *dcb_work =</div><div class='add'>+		container_of(work, struct bnxt_re_dcb_work, work);</div><div class='add'>+	struct bnxt_re_dev *rdev = dcb_work-&gt;rdev;</div><div class='add'>+	struct bnxt_qplib_cc_param *cc_param;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (!rdev)</div><div class='add'>+		goto free_dcb;</div><div class='add'>+</div><div class='add'>+	cc_param = &amp;rdev-&gt;cc_param;</div><div class='add'>+	rc = bnxt_qplib_query_cc_param(&amp;rdev-&gt;qplib_res, cc_param);</div><div class='add'>+	if (rc) {</div><div class='add'>+		ibdev_dbg(&amp;rdev-&gt;ibdev, "Failed to query ccparam rc:%d", rc);</div><div class='add'>+		goto free_dcb;</div><div class='add'>+	}</div><div class='add'>+	if (cc_param-&gt;qp1_tos_dscp != cc_param-&gt;tos_dscp) {</div><div class='add'>+		cc_param-&gt;qp1_tos_dscp = cc_param-&gt;tos_dscp;</div><div class='add'>+		rc = bnxt_re_update_qp1_tos_dscp(rdev);</div><div class='add'>+		if (rc) {</div><div class='add'>+			ibdev_dbg(&amp;rdev-&gt;ibdev, "%s: Failed to modify QP1 rc:%d",</div><div class='add'>+				  __func__, rc);</div><div class='add'>+			goto free_dcb;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+free_dcb:</div><div class='add'>+	kfree(dcb_work);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_async_notifier(void *handle, struct hwrm_async_event_cmpl *cmpl)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_re_dev *rdev = p;</div><div class='add'>+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);</div><div class='add'>+	struct bnxt_re_dcb_work *dcb_work;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	u32 data1, data2;</div><div class='add'>+	u16 event_id;</div><div class='ctx'> </div><div class='add'>+	rdev = en_info-&gt;rdev;</div><div class='ctx'> 	if (!rdev)</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='del'>-	bnxt_re_ib_unreg(rdev);</div><div class='add'>+	event_id = le16_to_cpu(cmpl-&gt;event_id);</div><div class='add'>+	data1 = le32_to_cpu(cmpl-&gt;event_data1);</div><div class='add'>+	data2 = le32_to_cpu(cmpl-&gt;event_data2);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;rdev-&gt;ibdev, "Async event_id = %d data1 = %d data2 = %d",</div><div class='add'>+		  event_id, data1, data2);</div><div class='add'>+</div><div class='add'>+	switch (event_id) {</div><div class='add'>+	case ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE:</div><div class='add'>+		dcb_work = kzalloc(sizeof(*dcb_work), GFP_ATOMIC);</div><div class='add'>+		if (!dcb_work)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		dcb_work-&gt;rdev = rdev;</div><div class='add'>+		memcpy(&amp;dcb_work-&gt;cmpl, cmpl, sizeof(*cmpl));</div><div class='add'>+		INIT_WORK(&amp;dcb_work-&gt;work, bnxt_re_dcb_wq_task);</div><div class='add'>+		queue_work(rdev-&gt;dcb_wq, &amp;dcb_work-&gt;work);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void bnxt_re_stop_irq(void *handle)</div><div class='add'>+static void bnxt_re_stop_irq(void *handle, bool reset)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_re_dev *rdev = (struct bnxt_re_dev *)handle;</div><div class='del'>-	struct bnxt_qplib_rcfw *rcfw = &amp;rdev-&gt;rcfw;</div><div class='add'>+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);</div><div class='add'>+	struct bnxt_qplib_rcfw *rcfw;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='ctx'> 	struct bnxt_qplib_nq *nq;</div><div class='ctx'> 	int indx;</div><div class='ctx'> </div><div class='del'>-	for (indx = BNXT_RE_NQ_IDX; indx &lt; rdev-&gt;num_msix; indx++) {</div><div class='del'>-		nq = &amp;rdev-&gt;nq[indx - 1];</div><div class='add'>+	rdev = en_info-&gt;rdev;</div><div class='add'>+	if (!rdev)</div><div class='add'>+		return;</div><div class='add'>+	rcfw = &amp;rdev-&gt;rcfw;</div><div class='add'>+</div><div class='add'>+	if (reset) {</div><div class='add'>+		set_bit(ERR_DEVICE_DETACHED, &amp;rdev-&gt;rcfw.cmdq.flags);</div><div class='add'>+		set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &amp;rdev-&gt;flags);</div><div class='add'>+		wake_up_all(&amp;rdev-&gt;rcfw.cmdq.waitq);</div><div class='add'>+		bnxt_re_dispatch_event(&amp;rdev-&gt;ibdev, NULL, 1,</div><div class='add'>+				       IB_EVENT_DEVICE_FATAL);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (indx = BNXT_RE_NQ_IDX; indx &lt; rdev-&gt;nqr-&gt;num_msix; indx++) {</div><div class='add'>+		nq = &amp;rdev-&gt;nqr-&gt;nq[indx - 1];</div><div class='ctx'> 		bnxt_qplib_nq_stop_irq(nq, false);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -225,207 +465,505 @@ static void bnxt_re_stop_irq(void *handle)</div><div class='ctx'> </div><div class='ctx'> static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_re_dev *rdev = (struct bnxt_re_dev *)handle;</div><div class='del'>-	struct bnxt_msix_entry *msix_ent = rdev-&gt;msix_entries;</div><div class='del'>-	struct bnxt_qplib_rcfw *rcfw = &amp;rdev-&gt;rcfw;</div><div class='add'>+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);</div><div class='add'>+	struct bnxt_msix_entry *msix_ent;</div><div class='add'>+	struct bnxt_qplib_rcfw *rcfw;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='ctx'> 	struct bnxt_qplib_nq *nq;</div><div class='ctx'> 	int indx, rc;</div><div class='ctx'> </div><div class='add'>+	rdev = en_info-&gt;rdev;</div><div class='add'>+	if (!rdev)</div><div class='add'>+		return;</div><div class='add'>+	msix_ent = rdev-&gt;nqr-&gt;msix_entries;</div><div class='add'>+	rcfw = &amp;rdev-&gt;rcfw;</div><div class='ctx'> 	if (!ent) {</div><div class='ctx'> 		/* Not setting the f/w timeout bit in rcfw.</div><div class='ctx'> 		 * During the driver unload the first command</div><div class='ctx'> 		 * to f/w will timeout and that will set the</div><div class='ctx'> 		 * timeout bit.</div><div class='ctx'> 		 */</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to re-start IRQs\n");</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to re-start IRQs\n");</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Vectors may change after restart, so update with new vectors</div><div class='ctx'> 	 * in device sctructure.</div><div class='ctx'> 	 */</div><div class='del'>-	for (indx = 0; indx &lt; rdev-&gt;num_msix; indx++)</div><div class='del'>-		rdev-&gt;msix_entries[indx].vector = ent[indx].vector;</div><div class='add'>+	for (indx = 0; indx &lt; rdev-&gt;nqr-&gt;num_msix; indx++)</div><div class='add'>+		rdev-&gt;nqr-&gt;msix_entries[indx].vector = ent[indx].vector;</div><div class='ctx'> </div><div class='del'>-	bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector,</div><div class='del'>-				  false);</div><div class='del'>-	for (indx = BNXT_RE_NQ_IDX ; indx &lt; rdev-&gt;num_msix; indx++) {</div><div class='del'>-		nq = &amp;rdev-&gt;nq[indx - 1];</div><div class='add'>+	rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector,</div><div class='add'>+				       false);</div><div class='add'>+	if (rc) {</div><div class='add'>+		ibdev_warn(&amp;rdev-&gt;ibdev, "Failed to reinit CREQ\n");</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+	for (indx = BNXT_RE_NQ_IDX ; indx &lt; rdev-&gt;nqr-&gt;num_msix; indx++) {</div><div class='add'>+		nq = &amp;rdev-&gt;nqr-&gt;nq[indx - 1];</div><div class='ctx'> 		rc = bnxt_qplib_nq_start_irq(nq, indx - 1,</div><div class='ctx'> 					     msix_ent[indx].vector, false);</div><div class='del'>-		if (rc)</div><div class='del'>-			dev_warn(rdev_to_dev(rdev),</div><div class='del'>-				 "Failed to reinit NQ index %d\n", indx - 1);</div><div class='add'>+		if (rc) {</div><div class='add'>+			ibdev_warn(&amp;rdev-&gt;ibdev, "Failed to reinit NQ index %d\n",</div><div class='add'>+				   indx - 1);</div><div class='add'>+			return;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static struct bnxt_ulp_ops bnxt_re_ulp_ops = {</div><div class='del'>-	.ulp_async_notifier = NULL,</div><div class='del'>-	.ulp_stop = bnxt_re_stop,</div><div class='del'>-	.ulp_start = bnxt_re_start,</div><div class='del'>-	.ulp_sriov_config = bnxt_re_sriov_config,</div><div class='del'>-	.ulp_shutdown = bnxt_re_shutdown,</div><div class='add'>+	.ulp_async_notifier = bnxt_re_async_notifier,</div><div class='ctx'> 	.ulp_irq_stop = bnxt_re_stop_irq,</div><div class='ctx'> 	.ulp_irq_restart = bnxt_re_start_irq</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /* RoCE -&gt; Net driver */</div><div class='ctx'> </div><div class='del'>-/* Driver registration routines used to let the networking driver (bnxt_en)</div><div class='del'>- * to know that the RoCE driver is now installed</div><div class='del'>- */</div><div class='del'>-static int bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev)</div><div class='add'>+static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_en_dev *en_dev;</div><div class='add'>+</div><div class='add'>+	en_dev = rdev-&gt;en_dev;</div><div class='add'>+	return bnxt_register_dev(en_dev, &amp;bnxt_re_ulp_ops, rdev-&gt;adev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_init_hwrm_hdr(struct input *hdr, u16 opcd)</div><div class='add'>+{</div><div class='add'>+	hdr-&gt;req_type = cpu_to_le16(opcd);</div><div class='add'>+	hdr-&gt;cmpl_ring = cpu_to_le16(-1);</div><div class='add'>+	hdr-&gt;target_id = cpu_to_le16(-1);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, void *msg,</div><div class='add'>+				int msg_len, void *resp, int resp_max_len,</div><div class='add'>+				int timeout)</div><div class='add'>+{</div><div class='add'>+	fw_msg-&gt;msg = msg;</div><div class='add'>+	fw_msg-&gt;msg_len = msg_len;</div><div class='add'>+	fw_msg-&gt;resp = resp;</div><div class='add'>+	fw_msg-&gt;resp_max_len = resp_max_len;</div><div class='add'>+	fw_msg-&gt;timeout = timeout;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bnxt_re_hwrm_free_vnic(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_en_dev *en_dev = rdev-&gt;en_dev;</div><div class='add'>+	struct hwrm_vnic_free_input req = {};</div><div class='add'>+	struct bnxt_fw_msg fw_msg = {};</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	if (!rdev)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	bnxt_re_init_hwrm_hdr((void *)&amp;req, HWRM_VNIC_FREE);</div><div class='ctx'> </div><div class='del'>-	en_dev = rdev-&gt;en_dev;</div><div class='add'>+	req.vnic_id = cpu_to_le32(rdev-&gt;mirror_vnic_id);</div><div class='add'>+	bnxt_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), NULL,</div><div class='add'>+			    0, BNXT_RE_HWRM_CMD_TIMEOUT(rdev));</div><div class='add'>+	rc = bnxt_send_msg(en_dev, &amp;fw_msg);</div><div class='add'>+	if (rc)</div><div class='add'>+		ibdev_dbg(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to free vnic, rc = %d\n", rc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bnxt_re_hwrm_alloc_vnic(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_en_dev *en_dev = rdev-&gt;en_dev;</div><div class='add'>+	struct hwrm_vnic_alloc_output resp = {};</div><div class='add'>+	struct hwrm_vnic_alloc_input req = {};</div><div class='add'>+	struct bnxt_fw_msg fw_msg = {};</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	bnxt_re_init_hwrm_hdr((void *)&amp;req, HWRM_VNIC_ALLOC);</div><div class='add'>+</div><div class='add'>+	req.vnic_id = cpu_to_le16(rdev-&gt;mirror_vnic_id);</div><div class='add'>+	req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_VNIC_ID_VALID);</div><div class='add'>+	bnxt_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), (void *)&amp;resp,</div><div class='add'>+			    sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));</div><div class='add'>+	rc = bnxt_send_msg(en_dev, &amp;fw_msg);</div><div class='add'>+	if (rc)</div><div class='add'>+		ibdev_dbg(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to alloc vnic, rc = %d\n", rc);</div><div class='ctx'> </div><div class='del'>-	rc = en_dev-&gt;en_ops-&gt;bnxt_unregister_device(rdev-&gt;en_dev,</div><div class='del'>-						    BNXT_ROCE_ULP);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev)</div><div class='add'>+int bnxt_re_hwrm_cfg_vnic(struct bnxt_re_dev *rdev, u32 qp_id)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_en_dev *en_dev;</div><div class='del'>-	int rc = 0;</div><div class='add'>+	struct bnxt_en_dev *en_dev = rdev-&gt;en_dev;</div><div class='add'>+	struct hwrm_vnic_cfg_input req = {};</div><div class='add'>+	struct bnxt_fw_msg fw_msg = {};</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='del'>-	if (!rdev)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	bnxt_re_init_hwrm_hdr((void *)&amp;req, HWRM_VNIC_CFG);</div><div class='ctx'> </div><div class='del'>-	en_dev = rdev-&gt;en_dev;</div><div class='add'>+	req.flags = cpu_to_le32(VNIC_CFG_REQ_FLAGS_ROCE_ONLY_VNIC_MODE);</div><div class='add'>+	req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_RAW_QP_ID |</div><div class='add'>+				  VNIC_CFG_REQ_ENABLES_MRU);</div><div class='add'>+	req.vnic_id = cpu_to_le16(rdev-&gt;mirror_vnic_id);</div><div class='add'>+	req.raw_qp_id = cpu_to_le32(qp_id);</div><div class='add'>+	req.mru = cpu_to_le16(rdev-&gt;netdev-&gt;mtu + VLAN_ETH_HLEN);</div><div class='add'>+</div><div class='add'>+	bnxt_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), NULL,</div><div class='add'>+			    0, BNXT_RE_HWRM_CMD_TIMEOUT(rdev));</div><div class='add'>+	rc = bnxt_send_msg(en_dev, &amp;fw_msg);</div><div class='add'>+	if (rc)</div><div class='add'>+		ibdev_dbg(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to cfg vnic, rc = %d\n", rc);</div><div class='ctx'> </div><div class='del'>-	rc = en_dev-&gt;en_ops-&gt;bnxt_register_device(en_dev, BNXT_ROCE_ULP,</div><div class='del'>-						  &amp;bnxt_re_ulp_ops, rdev);</div><div class='del'>-	rdev-&gt;qplib_res.pdev = rdev-&gt;en_dev-&gt;pdev;</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int bnxt_re_free_msix(struct bnxt_re_dev *rdev)</div><div class='add'>+/* Query device config using common hwrm */</div><div class='add'>+static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len,</div><div class='add'>+			     u32 *offset)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_en_dev *en_dev;</div><div class='add'>+	struct bnxt_en_dev *en_dev = rdev-&gt;en_dev;</div><div class='add'>+	struct hwrm_func_qcfg_output resp = {0};</div><div class='add'>+	struct hwrm_func_qcfg_input req = {0};</div><div class='add'>+	struct bnxt_fw_msg fw_msg = {};</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	if (!rdev)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	bnxt_re_init_hwrm_hdr((void *)&amp;req, HWRM_FUNC_QCFG);</div><div class='add'>+	req.fid = cpu_to_le16(0xffff);</div><div class='add'>+	bnxt_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), (void *)&amp;resp,</div><div class='add'>+			    sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));</div><div class='add'>+	rc = bnxt_send_msg(en_dev, &amp;fw_msg);</div><div class='add'>+	if (!rc) {</div><div class='add'>+		*db_len = PAGE_ALIGN(le16_to_cpu(resp.l2_doorbell_bar_size_kb) * 1024);</div><div class='add'>+		*offset = PAGE_ALIGN(le16_to_cpu(resp.legacy_l2_db_size_kb) * 1024);</div><div class='add'>+		rdev-&gt;mirror_vnic_id = le16_to_cpu(resp.mirror_vnic_id);</div><div class='add'>+	}</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	en_dev = rdev-&gt;en_dev;</div><div class='add'>+/* Query function capabilities using common hwrm */</div><div class='add'>+int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_en_dev *en_dev = rdev-&gt;en_dev;</div><div class='add'>+	struct hwrm_func_qcaps_output resp = {};</div><div class='add'>+	struct hwrm_func_qcaps_input req = {};</div><div class='add'>+	struct bnxt_qplib_chip_ctx *cctx;</div><div class='add'>+	struct bnxt_fw_msg fw_msg = {};</div><div class='add'>+	u32 flags_ext2;</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='add'>+	cctx = rdev-&gt;chip_ctx;</div><div class='add'>+	bnxt_re_init_hwrm_hdr((void *)&amp;req, HWRM_FUNC_QCAPS);</div><div class='add'>+	req.fid = cpu_to_le16(0xffff);</div><div class='add'>+	bnxt_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), (void *)&amp;resp,</div><div class='add'>+			    sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));</div><div class='ctx'> </div><div class='del'>-	rc = en_dev-&gt;en_ops-&gt;bnxt_free_msix(rdev-&gt;en_dev, BNXT_ROCE_ULP);</div><div class='add'>+	rc = bnxt_send_msg(en_dev, &amp;fw_msg);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+	cctx-&gt;modes.db_push = le32_to_cpu(resp.flags) &amp; FUNC_QCAPS_RESP_FLAGS_WCB_PUSH_MODE;</div><div class='ctx'> </div><div class='del'>-	return rc;</div><div class='add'>+	flags_ext2 = le32_to_cpu(resp.flags_ext2);</div><div class='add'>+	cctx-&gt;modes.dbr_pacing = flags_ext2 &amp; FUNC_QCAPS_RESP_FLAGS_EXT2_DBR_PACING_EXT_SUPPORTED ||</div><div class='add'>+				 flags_ext2 &amp; FUNC_QCAPS_RESP_FLAGS_EXT2_DBR_PACING_V0_SUPPORTED;</div><div class='add'>+	cctx-&gt;modes.roce_mirror = !!(le32_to_cpu(resp.flags_ext3) &amp;</div><div class='add'>+				     FUNC_QCAPS_RESP_FLAGS_EXT3_MIRROR_ON_ROCE_SUPPORTED);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int bnxt_re_request_msix(struct bnxt_re_dev *rdev)</div><div class='add'>+static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='del'>-	int rc = 0, num_msix_want = BNXT_RE_MAX_MSIX, num_msix_got;</div><div class='del'>-	struct bnxt_en_dev *en_dev;</div><div class='add'>+	struct bnxt_qplib_db_pacing_data *pacing_data = rdev-&gt;qplib_res.pacing_data;</div><div class='add'>+	struct hwrm_func_dbr_pacing_qcfg_output resp = {};</div><div class='add'>+	struct hwrm_func_dbr_pacing_qcfg_input req = {};</div><div class='add'>+	struct bnxt_en_dev *en_dev = rdev-&gt;en_dev;</div><div class='add'>+	struct bnxt_qplib_chip_ctx *cctx;</div><div class='add'>+	struct bnxt_fw_msg fw_msg = {};</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='del'>-	if (!rdev)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	cctx = rdev-&gt;chip_ctx;</div><div class='add'>+	bnxt_re_init_hwrm_hdr((void *)&amp;req, HWRM_FUNC_DBR_PACING_QCFG);</div><div class='add'>+	bnxt_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), (void *)&amp;resp,</div><div class='add'>+			    sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));</div><div class='add'>+	rc = bnxt_send_msg(en_dev, &amp;fw_msg);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='ctx'> </div><div class='del'>-	en_dev = rdev-&gt;en_dev;</div><div class='add'>+	if ((le32_to_cpu(resp.dbr_stat_db_fifo_reg) &amp;</div><div class='add'>+	    FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK) ==</div><div class='add'>+		FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_GRC)</div><div class='add'>+		cctx-&gt;dbr_stat_db_fifo =</div><div class='add'>+			le32_to_cpu(resp.dbr_stat_db_fifo_reg) &amp;</div><div class='add'>+			~FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK;</div><div class='ctx'> </div><div class='del'>-	num_msix_want = min_t(u32, BNXT_RE_MAX_MSIX, num_online_cpus());</div><div class='add'>+	pacing_data-&gt;fifo_max_depth = le32_to_cpu(resp.dbr_stat_db_max_fifo_depth);</div><div class='add'>+	if (!pacing_data-&gt;fifo_max_depth)</div><div class='add'>+		pacing_data-&gt;fifo_max_depth = BNXT_RE_MAX_FIFO_DEPTH(cctx);</div><div class='add'>+	pacing_data-&gt;fifo_room_mask = le32_to_cpu(resp.dbr_stat_db_fifo_reg_fifo_room_mask);</div><div class='add'>+	pacing_data-&gt;fifo_room_shift = resp.dbr_stat_db_fifo_reg_fifo_room_shift;</div><div class='ctx'> </div><div class='del'>-	num_msix_got = en_dev-&gt;en_ops-&gt;bnxt_request_msix(en_dev, BNXT_ROCE_ULP,</div><div class='del'>-							 rdev-&gt;msix_entries,</div><div class='del'>-							 num_msix_want);</div><div class='del'>-	if (num_msix_got &lt; BNXT_RE_MIN_MSIX) {</div><div class='del'>-		rc = -EINVAL;</div><div class='del'>-		goto done;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* Update the pacing tunable parameters to the default values */</div><div class='add'>+static void bnxt_re_set_default_pacing_data(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_db_pacing_data *pacing_data = rdev-&gt;qplib_res.pacing_data;</div><div class='add'>+</div><div class='add'>+	pacing_data-&gt;do_pacing = rdev-&gt;pacing.dbr_def_do_pacing;</div><div class='add'>+	pacing_data-&gt;pacing_th = rdev-&gt;pacing.pacing_algo_th;</div><div class='add'>+	pacing_data-&gt;alarm_th =</div><div class='add'>+		pacing_data-&gt;pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u32 __get_fifo_occupancy(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_db_pacing_data *pacing_data = rdev-&gt;qplib_res.pacing_data;</div><div class='add'>+	u32 read_val, fifo_occup;</div><div class='add'>+</div><div class='add'>+	read_val = readl(rdev-&gt;en_dev-&gt;bar0 + rdev-&gt;pacing.dbr_db_fifo_reg_off);</div><div class='add'>+	fifo_occup = pacing_data-&gt;fifo_max_depth -</div><div class='add'>+		     ((read_val &amp; pacing_data-&gt;fifo_room_mask) &gt;&gt;</div><div class='add'>+		      pacing_data-&gt;fifo_room_shift);</div><div class='add'>+	return fifo_occup;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool is_dbr_fifo_full(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	u32 max_occup, fifo_occup;</div><div class='add'>+</div><div class='add'>+	fifo_occup = __get_fifo_occupancy(rdev);</div><div class='add'>+	max_occup = BNXT_RE_MAX_FIFO_DEPTH(rdev-&gt;chip_ctx) - 1;</div><div class='add'>+	if (fifo_occup == max_occup)</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void __wait_for_fifo_occupancy_below_th(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_db_pacing_data *pacing_data = rdev-&gt;qplib_res.pacing_data;</div><div class='add'>+	u32 retry_fifo_check = 1000;</div><div class='add'>+	u32 fifo_occup;</div><div class='add'>+</div><div class='add'>+	/* loop shouldn't run infintely as the occupancy usually goes</div><div class='add'>+	 * below pacing algo threshold as soon as pacing kicks in.</div><div class='add'>+	 */</div><div class='add'>+	while (1) {</div><div class='add'>+		fifo_occup = __get_fifo_occupancy(rdev);</div><div class='add'>+		/* Fifo occupancy cannot be greater the MAX FIFO depth */</div><div class='add'>+		if (fifo_occup &gt; pacing_data-&gt;fifo_max_depth)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		if (fifo_occup &lt; pacing_data-&gt;pacing_th)</div><div class='add'>+			break;</div><div class='add'>+		if (!retry_fifo_check--) {</div><div class='add'>+			dev_info_once(rdev_to_dev(rdev),</div><div class='add'>+				      "%s: fifo_occup = 0x%xfifo_max_depth = 0x%x pacing_th = 0x%x\n",</div><div class='add'>+				      __func__, fifo_occup, pacing_data-&gt;fifo_max_depth,</div><div class='add'>+					pacing_data-&gt;pacing_th);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_db_fifo_check(struct work_struct *work)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,</div><div class='add'>+			dbq_fifo_check_work);</div><div class='add'>+	struct bnxt_qplib_db_pacing_data *pacing_data;</div><div class='add'>+	u32 pacing_save;</div><div class='add'>+</div><div class='add'>+	if (!mutex_trylock(&amp;rdev-&gt;pacing.dbq_lock))</div><div class='add'>+		return;</div><div class='add'>+	pacing_data = rdev-&gt;qplib_res.pacing_data;</div><div class='add'>+	pacing_save = rdev-&gt;pacing.do_pacing_save;</div><div class='add'>+	__wait_for_fifo_occupancy_below_th(rdev);</div><div class='add'>+	cancel_delayed_work_sync(&amp;rdev-&gt;dbq_pacing_work);</div><div class='add'>+	if (pacing_save &gt; rdev-&gt;pacing.dbr_def_do_pacing) {</div><div class='add'>+		/* Double the do_pacing value during the congestion */</div><div class='add'>+		pacing_save = pacing_save &lt;&lt; 1;</div><div class='add'>+	} else {</div><div class='add'>+		/*</div><div class='add'>+		 * when a new congestion is detected increase the do_pacing</div><div class='add'>+		 * by 8 times. And also increase the pacing_th by 4 times. The</div><div class='add'>+		 * reason to increase pacing_th is to give more space for the</div><div class='add'>+		 * queue to oscillate down without getting empty, but also more</div><div class='add'>+		 * room for the queue to increase without causing another alarm.</div><div class='add'>+		 */</div><div class='add'>+		pacing_save = pacing_save &lt;&lt; 3;</div><div class='add'>+		pacing_data-&gt;pacing_th = rdev-&gt;pacing.pacing_algo_th * 4;</div><div class='ctx'> 	}</div><div class='del'>-	if (num_msix_got != num_msix_want) {</div><div class='del'>-		dev_warn(rdev_to_dev(rdev),</div><div class='del'>-			 "Requested %d MSI-X vectors, got %d\n",</div><div class='del'>-			 num_msix_want, num_msix_got);</div><div class='add'>+</div><div class='add'>+	if (pacing_save &gt; BNXT_RE_MAX_DBR_DO_PACING)</div><div class='add'>+		pacing_save = BNXT_RE_MAX_DBR_DO_PACING;</div><div class='add'>+</div><div class='add'>+	pacing_data-&gt;do_pacing = pacing_save;</div><div class='add'>+	rdev-&gt;pacing.do_pacing_save = pacing_data-&gt;do_pacing;</div><div class='add'>+	pacing_data-&gt;alarm_th =</div><div class='add'>+		pacing_data-&gt;pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE;</div><div class='add'>+	schedule_delayed_work(&amp;rdev-&gt;dbq_pacing_work,</div><div class='add'>+			      msecs_to_jiffies(rdev-&gt;pacing.dbq_pacing_time));</div><div class='add'>+	rdev-&gt;stats.pacing.alerts++;</div><div class='add'>+	mutex_unlock(&amp;rdev-&gt;pacing.dbq_lock);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_pacing_timer_exp(struct work_struct *work)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,</div><div class='add'>+			dbq_pacing_work.work);</div><div class='add'>+	struct bnxt_qplib_db_pacing_data *pacing_data;</div><div class='add'>+	u32 fifo_occup;</div><div class='add'>+</div><div class='add'>+	if (!mutex_trylock(&amp;rdev-&gt;pacing.dbq_lock))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	pacing_data = rdev-&gt;qplib_res.pacing_data;</div><div class='add'>+	fifo_occup = __get_fifo_occupancy(rdev);</div><div class='add'>+</div><div class='add'>+	if (fifo_occup &gt; pacing_data-&gt;pacing_th)</div><div class='add'>+		goto restart_timer;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Instead of immediately going back to the default do_pacing</div><div class='add'>+	 * reduce it by 1/8 times and restart the timer.</div><div class='add'>+	 */</div><div class='add'>+	pacing_data-&gt;do_pacing = pacing_data-&gt;do_pacing - (pacing_data-&gt;do_pacing &gt;&gt; 3);</div><div class='add'>+	pacing_data-&gt;do_pacing = max_t(u32, rdev-&gt;pacing.dbr_def_do_pacing, pacing_data-&gt;do_pacing);</div><div class='add'>+	if (pacing_data-&gt;do_pacing &lt;= rdev-&gt;pacing.dbr_def_do_pacing) {</div><div class='add'>+		bnxt_re_set_default_pacing_data(rdev);</div><div class='add'>+		rdev-&gt;stats.pacing.complete++;</div><div class='add'>+		goto dbq_unlock;</div><div class='ctx'> 	}</div><div class='del'>-	rdev-&gt;num_msix = num_msix_got;</div><div class='del'>-done:</div><div class='del'>-	return rc;</div><div class='add'>+</div><div class='add'>+restart_timer:</div><div class='add'>+	schedule_delayed_work(&amp;rdev-&gt;dbq_pacing_work,</div><div class='add'>+			      msecs_to_jiffies(rdev-&gt;pacing.dbq_pacing_time));</div><div class='add'>+	rdev-&gt;stats.pacing.resched++;</div><div class='add'>+dbq_unlock:</div><div class='add'>+	rdev-&gt;pacing.do_pacing_save = pacing_data-&gt;do_pacing;</div><div class='add'>+	mutex_unlock(&amp;rdev-&gt;pacing.dbq_lock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev *rdev, struct input *hdr,</div><div class='del'>-				  u16 opcd, u16 crid, u16 trid)</div><div class='add'>+void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='del'>-	hdr-&gt;req_type = cpu_to_le16(opcd);</div><div class='del'>-	hdr-&gt;cmpl_ring = cpu_to_le16(crid);</div><div class='del'>-	hdr-&gt;target_id = cpu_to_le16(trid);</div><div class='add'>+	struct bnxt_qplib_db_pacing_data *pacing_data;</div><div class='add'>+</div><div class='add'>+	if (!rdev-&gt;pacing.dbr_pacing)</div><div class='add'>+		return;</div><div class='add'>+	mutex_lock(&amp;rdev-&gt;pacing.dbq_lock);</div><div class='add'>+	pacing_data = rdev-&gt;qplib_res.pacing_data;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Increase the alarm_th to max so that other user lib instances do not</div><div class='add'>+	 * keep alerting the driver.</div><div class='add'>+	 */</div><div class='add'>+	pacing_data-&gt;alarm_th = pacing_data-&gt;fifo_max_depth;</div><div class='add'>+	pacing_data-&gt;do_pacing = BNXT_RE_MAX_DBR_DO_PACING;</div><div class='add'>+	cancel_work_sync(&amp;rdev-&gt;dbq_fifo_check_work);</div><div class='add'>+	schedule_work(&amp;rdev-&gt;dbq_fifo_check_work);</div><div class='add'>+	mutex_unlock(&amp;rdev-&gt;pacing.dbq_lock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, void *msg,</div><div class='del'>-				int msg_len, void *resp, int resp_max_len,</div><div class='del'>-				int timeout)</div><div class='add'>+static int bnxt_re_initialize_dbr_pacing(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='del'>-	fw_msg-&gt;msg = msg;</div><div class='del'>-	fw_msg-&gt;msg_len = msg_len;</div><div class='del'>-	fw_msg-&gt;resp = resp;</div><div class='del'>-	fw_msg-&gt;resp_max_len = resp_max_len;</div><div class='del'>-	fw_msg-&gt;timeout = timeout;</div><div class='add'>+	/* Allocate a page for app use */</div><div class='add'>+	rdev-&gt;pacing.dbr_page = (void *)__get_free_page(GFP_KERNEL);</div><div class='add'>+	if (!rdev-&gt;pacing.dbr_page)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	memset((u8 *)rdev-&gt;pacing.dbr_page, 0, PAGE_SIZE);</div><div class='add'>+	rdev-&gt;qplib_res.pacing_data = (struct bnxt_qplib_db_pacing_data *)rdev-&gt;pacing.dbr_page;</div><div class='add'>+</div><div class='add'>+	if (bnxt_re_hwrm_dbr_pacing_qcfg(rdev)) {</div><div class='add'>+		free_page((u64)rdev-&gt;pacing.dbr_page);</div><div class='add'>+		rdev-&gt;pacing.dbr_page = NULL;</div><div class='add'>+		return -EIO;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* MAP HW window 2 for reading db fifo depth */</div><div class='add'>+	writel(rdev-&gt;chip_ctx-&gt;dbr_stat_db_fifo &amp; BNXT_GRC_BASE_MASK,</div><div class='add'>+	       rdev-&gt;en_dev-&gt;bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);</div><div class='add'>+	rdev-&gt;pacing.dbr_db_fifo_reg_off =</div><div class='add'>+		(rdev-&gt;chip_ctx-&gt;dbr_stat_db_fifo &amp; BNXT_GRC_OFFSET_MASK) +</div><div class='add'>+		 BNXT_RE_GRC_FIFO_REG_BASE;</div><div class='add'>+	rdev-&gt;pacing.dbr_bar_addr =</div><div class='add'>+		pci_resource_start(rdev-&gt;qplib_res.pdev, 0) + rdev-&gt;pacing.dbr_db_fifo_reg_off;</div><div class='add'>+</div><div class='add'>+	if (is_dbr_fifo_full(rdev)) {</div><div class='add'>+		free_page((u64)rdev-&gt;pacing.dbr_page);</div><div class='add'>+		rdev-&gt;pacing.dbr_page = NULL;</div><div class='add'>+		return -EIO;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rdev-&gt;pacing.pacing_algo_th = BNXT_RE_PACING_ALGO_THRESHOLD;</div><div class='add'>+	rdev-&gt;pacing.dbq_pacing_time = BNXT_RE_DBR_PACING_TIME;</div><div class='add'>+	rdev-&gt;pacing.dbr_def_do_pacing = BNXT_RE_DBR_DO_PACING_NO_CONGESTION;</div><div class='add'>+	rdev-&gt;pacing.do_pacing_save = rdev-&gt;pacing.dbr_def_do_pacing;</div><div class='add'>+	rdev-&gt;qplib_res.pacing_data-&gt;grc_reg_offset = rdev-&gt;pacing.dbr_db_fifo_reg_off;</div><div class='add'>+	bnxt_re_set_default_pacing_data(rdev);</div><div class='add'>+	/* Initialize worker for DBR Pacing */</div><div class='add'>+	INIT_WORK(&amp;rdev-&gt;dbq_fifo_check_work, bnxt_re_db_fifo_check);</div><div class='add'>+	INIT_DELAYED_WORK(&amp;rdev-&gt;dbq_pacing_work, bnxt_re_pacing_timer_exp);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_deinitialize_dbr_pacing(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	cancel_work_sync(&amp;rdev-&gt;dbq_fifo_check_work);</div><div class='add'>+	cancel_delayed_work_sync(&amp;rdev-&gt;dbq_pacing_work);</div><div class='add'>+	if (rdev-&gt;pacing.dbr_page)</div><div class='add'>+		free_page((u64)rdev-&gt;pacing.dbr_page);</div><div class='add'>+</div><div class='add'>+	rdev-&gt;pacing.dbr_page = NULL;</div><div class='add'>+	rdev-&gt;pacing.dbr_pacing = false;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev,</div><div class='ctx'> 				 u16 fw_ring_id, int type)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_en_dev *en_dev = rdev-&gt;en_dev;</div><div class='del'>-	struct hwrm_ring_free_input req = {0};</div><div class='add'>+	struct hwrm_ring_free_input req = {};</div><div class='ctx'> 	struct hwrm_ring_free_output resp;</div><div class='del'>-	struct bnxt_fw_msg fw_msg;</div><div class='add'>+	struct bnxt_fw_msg fw_msg = {};</div><div class='ctx'> 	int rc = -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (!en_dev)</div><div class='del'>-		return rc;</div><div class='del'>-</div><div class='del'>-	memset(&amp;fw_msg, 0, sizeof(fw_msg));</div><div class='add'>+	if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &amp;rdev-&gt;flags))</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-	bnxt_re_init_hwrm_hdr(rdev, (void *)&amp;req, HWRM_RING_FREE, -1, -1);</div><div class='add'>+	bnxt_re_init_hwrm_hdr((void *)&amp;req, HWRM_RING_FREE);</div><div class='ctx'> 	req.ring_type = type;</div><div class='ctx'> 	req.ring_id = cpu_to_le16(fw_ring_id);</div><div class='ctx'> 	bnxt_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), (void *)&amp;resp,</div><div class='del'>-			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);</div><div class='del'>-	rc = en_dev-&gt;en_ops-&gt;bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &amp;fw_msg);</div><div class='add'>+			    sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));</div><div class='add'>+	rc = bnxt_send_msg(en_dev, &amp;fw_msg);</div><div class='ctx'> 	if (rc)</div><div class='del'>-		dev_err(rdev_to_dev(rdev),</div><div class='del'>-			"Failed to free HW ring:%d :%#x", req.ring_id, rc);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to free HW ring:%d :%#x",</div><div class='add'>+			  req.ring_id, rc);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, dma_addr_t *dma_arr,</div><div class='del'>-				  int pages, int type, u32 ring_mask,</div><div class='del'>-				  u32 map_index, u16 *fw_ring_id)</div><div class='add'>+static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev,</div><div class='add'>+				  struct bnxt_re_ring_attr *ring_attr,</div><div class='add'>+				  u16 *fw_ring_id)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_en_dev *en_dev = rdev-&gt;en_dev;</div><div class='del'>-	struct hwrm_ring_alloc_input req = {0};</div><div class='add'>+	struct hwrm_ring_alloc_input req = {};</div><div class='ctx'> 	struct hwrm_ring_alloc_output resp;</div><div class='del'>-	struct bnxt_fw_msg fw_msg;</div><div class='add'>+	struct bnxt_fw_msg fw_msg = {};</div><div class='ctx'> 	int rc = -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (!en_dev)</div><div class='del'>-		return rc;</div><div class='del'>-</div><div class='del'>-	memset(&amp;fw_msg, 0, sizeof(fw_msg));</div><div class='del'>-	bnxt_re_init_hwrm_hdr(rdev, (void *)&amp;req, HWRM_RING_ALLOC, -1, -1);</div><div class='add'>+	bnxt_re_init_hwrm_hdr((void *)&amp;req, HWRM_RING_ALLOC);</div><div class='ctx'> 	req.enables = 0;</div><div class='del'>-	req.page_tbl_addr =  cpu_to_le64(dma_arr[0]);</div><div class='del'>-	if (pages &gt; 1) {</div><div class='add'>+	req.page_tbl_addr =  cpu_to_le64(ring_attr-&gt;dma_arr[0]);</div><div class='add'>+	if (ring_attr-&gt;pages &gt; 1) {</div><div class='ctx'> 		/* Page size is in log2 units */</div><div class='ctx'> 		req.page_size = BNXT_PAGE_SHIFT;</div><div class='ctx'> 		req.page_tbl_depth = 1;</div><div class='ctx'> 	}</div><div class='ctx'> 	req.fbo = 0;</div><div class='ctx'> 	/* Association of ring index with doorbell index and MSIX number */</div><div class='del'>-	req.logical_id = cpu_to_le16(map_index);</div><div class='del'>-	req.length = cpu_to_le32(ring_mask + 1);</div><div class='del'>-	req.ring_type = type;</div><div class='del'>-	req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;</div><div class='add'>+	req.logical_id = cpu_to_le16(ring_attr-&gt;lrid);</div><div class='add'>+	req.length = cpu_to_le32(ring_attr-&gt;depth + 1);</div><div class='add'>+	req.ring_type = ring_attr-&gt;type;</div><div class='add'>+	req.int_mode = ring_attr-&gt;mode;</div><div class='ctx'> 	bnxt_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), (void *)&amp;resp,</div><div class='del'>-			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);</div><div class='del'>-	rc = en_dev-&gt;en_ops-&gt;bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &amp;fw_msg);</div><div class='add'>+			    sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));</div><div class='add'>+	rc = bnxt_send_msg(en_dev, &amp;fw_msg);</div><div class='ctx'> 	if (!rc)</div><div class='ctx'> 		*fw_ring_id = le16_to_cpu(resp.ring_id);</div><div class='ctx'> </div><div class='hunk'>@@ -436,168 +974,361 @@ static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev,</div><div class='ctx'> 				      u32 fw_stats_ctx_id)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_en_dev *en_dev = rdev-&gt;en_dev;</div><div class='del'>-	struct hwrm_stat_ctx_free_input req = {0};</div><div class='del'>-	struct bnxt_fw_msg fw_msg;</div><div class='add'>+	struct hwrm_stat_ctx_free_input req = {};</div><div class='add'>+	struct hwrm_stat_ctx_free_output resp = {};</div><div class='add'>+	struct bnxt_fw_msg fw_msg = {};</div><div class='ctx'> 	int rc = -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (!en_dev)</div><div class='del'>-		return rc;</div><div class='del'>-</div><div class='del'>-	memset(&amp;fw_msg, 0, sizeof(fw_msg));</div><div class='add'>+	if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &amp;rdev-&gt;flags))</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-	bnxt_re_init_hwrm_hdr(rdev, (void *)&amp;req, HWRM_STAT_CTX_FREE, -1, -1);</div><div class='add'>+	bnxt_re_init_hwrm_hdr((void *)&amp;req, HWRM_STAT_CTX_FREE);</div><div class='ctx'> 	req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id);</div><div class='del'>-	bnxt_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), (void *)&amp;req,</div><div class='del'>-			    sizeof(req), DFLT_HWRM_CMD_TIMEOUT);</div><div class='del'>-	rc = en_dev-&gt;en_ops-&gt;bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &amp;fw_msg);</div><div class='add'>+	bnxt_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), (void *)&amp;resp,</div><div class='add'>+			    sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));</div><div class='add'>+	rc = bnxt_send_msg(en_dev, &amp;fw_msg);</div><div class='ctx'> 	if (rc)</div><div class='del'>-		dev_err(rdev_to_dev(rdev),</div><div class='del'>-			"Failed to free HW stats context %#x", rc);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to free HW stats context %#x",</div><div class='add'>+			  rc);</div><div class='ctx'> </div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev,</div><div class='del'>-				       dma_addr_t dma_map,</div><div class='del'>-				       u32 *fw_stats_ctx_id)</div><div class='add'>+				       struct bnxt_qplib_stats *stats)</div><div class='ctx'> {</div><div class='del'>-	struct hwrm_stat_ctx_alloc_output resp = {0};</div><div class='del'>-	struct hwrm_stat_ctx_alloc_input req = {0};</div><div class='add'>+	struct bnxt_qplib_chip_ctx *chip_ctx = rdev-&gt;chip_ctx;</div><div class='add'>+	struct hwrm_stat_ctx_alloc_output resp = {};</div><div class='add'>+	struct hwrm_stat_ctx_alloc_input req = {};</div><div class='ctx'> 	struct bnxt_en_dev *en_dev = rdev-&gt;en_dev;</div><div class='del'>-	struct bnxt_fw_msg fw_msg;</div><div class='add'>+	struct bnxt_fw_msg fw_msg = {};</div><div class='ctx'> 	int rc = -EINVAL;</div><div class='ctx'> </div><div class='del'>-	*fw_stats_ctx_id = INVALID_STATS_CTX_ID;</div><div class='del'>-</div><div class='del'>-	if (!en_dev)</div><div class='del'>-		return rc;</div><div class='del'>-</div><div class='del'>-	memset(&amp;fw_msg, 0, sizeof(fw_msg));</div><div class='add'>+	stats-&gt;fw_id = INVALID_STATS_CTX_ID;</div><div class='ctx'> </div><div class='del'>-	bnxt_re_init_hwrm_hdr(rdev, (void *)&amp;req, HWRM_STAT_CTX_ALLOC, -1, -1);</div><div class='add'>+	bnxt_re_init_hwrm_hdr((void *)&amp;req, HWRM_STAT_CTX_ALLOC);</div><div class='ctx'> 	req.update_period_ms = cpu_to_le32(1000);</div><div class='del'>-	req.stats_dma_addr = cpu_to_le64(dma_map);</div><div class='add'>+	req.stats_dma_addr = cpu_to_le64(stats-&gt;dma_map);</div><div class='add'>+	req.stats_dma_length = cpu_to_le16(chip_ctx-&gt;hw_stats_size);</div><div class='ctx'> 	req.stat_ctx_flags = STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE;</div><div class='ctx'> 	bnxt_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), (void *)&amp;resp,</div><div class='del'>-			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);</div><div class='del'>-	rc = en_dev-&gt;en_ops-&gt;bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &amp;fw_msg);</div><div class='add'>+			    sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));</div><div class='add'>+	rc = bnxt_send_msg(en_dev, &amp;fw_msg);</div><div class='ctx'> 	if (!rc)</div><div class='del'>-		*fw_stats_ctx_id = le32_to_cpu(resp.stat_ctx_id);</div><div class='add'>+		stats-&gt;fw_id = le32_to_cpu(resp.stat_ctx_id);</div><div class='ctx'> </div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void bnxt_re_disassociate_ucontext(struct ib_ucontext *ibcontext)</div><div class='add'>+{</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /* Device */</div><div class='add'>+static ssize_t hw_rev_show(struct device *device, struct device_attribute *attr,</div><div class='add'>+			   char *buf)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_dev *rdev =</div><div class='add'>+		rdma_device_to_drv_device(device, struct bnxt_re_dev, ibdev);</div><div class='add'>+</div><div class='add'>+	return sysfs_emit(buf, "0x%x\n", rdev-&gt;en_dev-&gt;pdev-&gt;revision);</div><div class='add'>+}</div><div class='add'>+static DEVICE_ATTR_RO(hw_rev);</div><div class='ctx'> </div><div class='del'>-static bool is_bnxt_re_dev(struct net_device *netdev)</div><div class='add'>+static ssize_t hca_type_show(struct device *device,</div><div class='add'>+			     struct device_attribute *attr, char *buf)</div><div class='ctx'> {</div><div class='del'>-	struct ethtool_drvinfo drvinfo;</div><div class='add'>+	struct bnxt_re_dev *rdev =</div><div class='add'>+		rdma_device_to_drv_device(device, struct bnxt_re_dev, ibdev);</div><div class='ctx'> </div><div class='del'>-	if (netdev-&gt;ethtool_ops &amp;&amp; netdev-&gt;ethtool_ops-&gt;get_drvinfo) {</div><div class='del'>-		memset(&amp;drvinfo, 0, sizeof(drvinfo));</div><div class='del'>-		netdev-&gt;ethtool_ops-&gt;get_drvinfo(netdev, &amp;drvinfo);</div><div class='add'>+	return sysfs_emit(buf, "0x%x\n", rdev-&gt;en_dev-&gt;pdev-&gt;device);</div><div class='add'>+}</div><div class='add'>+static DEVICE_ATTR_RO(hca_type);</div><div class='ctx'> </div><div class='del'>-		if (strcmp(drvinfo.driver, "bnxt_en"))</div><div class='del'>-			return false;</div><div class='del'>-		return true;</div><div class='del'>-	}</div><div class='del'>-	return false;</div><div class='add'>+static ssize_t board_id_show(struct device *device, struct device_attribute *attr,</div><div class='add'>+			     char *buf)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_dev *rdev = rdma_device_to_drv_device(device,</div><div class='add'>+							     struct bnxt_re_dev, ibdev);</div><div class='add'>+	char buffer[BNXT_VPD_FLD_LEN] = {};</div><div class='add'>+</div><div class='add'>+	if (!rdev-&gt;is_virtfn)</div><div class='add'>+		memcpy(buffer, rdev-&gt;board_partno, BNXT_VPD_FLD_LEN - 1);</div><div class='add'>+	else</div><div class='add'>+		scnprintf(buffer, BNXT_VPD_FLD_LEN, "0x%x-VF",</div><div class='add'>+			  rdev-&gt;en_dev-&gt;pdev-&gt;device);</div><div class='add'>+</div><div class='add'>+	return sysfs_emit(buf, "%s\n", buffer);</div><div class='ctx'> }</div><div class='add'>+static DEVICE_ATTR_RO(board_id);</div><div class='add'>+</div><div class='add'>+static struct attribute *bnxt_re_attributes[] = {</div><div class='add'>+	&amp;dev_attr_hw_rev.attr,</div><div class='add'>+	&amp;dev_attr_hca_type.attr,</div><div class='add'>+	&amp;dev_attr_board_id.attr,</div><div class='add'>+	NULL</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct attribute_group bnxt_re_dev_attr_group = {</div><div class='add'>+	.attrs = bnxt_re_attributes,</div><div class='add'>+};</div><div class='ctx'> </div><div class='del'>-static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev)</div><div class='add'>+static int bnxt_re_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct bnxt_qplib_hwq *mr_hwq;</div><div class='add'>+	struct nlattr *table_attr;</div><div class='add'>+	struct bnxt_re_mr *mr;</div><div class='add'>+</div><div class='add'>+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);</div><div class='add'>+	if (!table_attr)</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);</div><div class='add'>+	mr_hwq = &amp;mr-&gt;qplib_mr.hwq;</div><div class='add'>+</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "page_size",</div><div class='add'>+				   mr_hwq-&gt;qe_ppg * mr_hwq-&gt;element_size))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "max_elements", mr_hwq-&gt;max_elements))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "element_size", mr_hwq-&gt;element_size))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u64_hex(msg, "hwq", (unsigned long)mr_hwq))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u64_hex(msg, "va", mr-&gt;qplib_mr.va))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	nla_nest_end(msg, table_attr);</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-	rcu_read_lock();</div><div class='del'>-	list_for_each_entry_rcu(rdev, &amp;bnxt_re_dev_list, list) {</div><div class='del'>-		if (rdev-&gt;netdev == netdev) {</div><div class='del'>-			rcu_read_unlock();</div><div class='del'>-			return rdev;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	rcu_read_unlock();</div><div class='del'>-	return NULL;</div><div class='add'>+err:</div><div class='add'>+	nla_nest_cancel(msg, table_attr);</div><div class='add'>+	return -EMSGSIZE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void bnxt_re_dev_unprobe(struct net_device *netdev,</div><div class='del'>-				struct bnxt_en_dev *en_dev)</div><div class='add'>+static int bnxt_re_fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ib_mr)</div><div class='ctx'> {</div><div class='del'>-	dev_put(netdev);</div><div class='del'>-	module_put(en_dev-&gt;pdev-&gt;driver-&gt;driver.owner);</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct bnxt_re_mr *mr;</div><div class='add'>+	int err, len;</div><div class='add'>+	void *data;</div><div class='add'>+</div><div class='add'>+	mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);</div><div class='add'>+	rdev = mr-&gt;rdev;</div><div class='add'>+</div><div class='add'>+	err = bnxt_re_read_context_allowed(rdev);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	len = bnxt_qplib_is_chip_gen_p7(rdev-&gt;chip_ctx) ? BNXT_RE_CONTEXT_TYPE_MRW_SIZE_P7 :</div><div class='add'>+							  BNXT_RE_CONTEXT_TYPE_MRW_SIZE_P5;</div><div class='add'>+	data = kzalloc(len, GFP_KERNEL);</div><div class='add'>+	if (!data)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	err = bnxt_qplib_read_context(&amp;rdev-&gt;rcfw, CMDQ_READ_CONTEXT_TYPE_MRW,</div><div class='add'>+				      mr-&gt;qplib_mr.lkey, len, data);</div><div class='add'>+	if (!err)</div><div class='add'>+		err = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, len, data);</div><div class='add'>+</div><div class='add'>+	kfree(data);</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct bnxt_en_dev *bnxt_re_dev_probe(struct net_device *netdev)</div><div class='add'>+static int bnxt_re_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt *bp = netdev_priv(netdev);</div><div class='del'>-	struct bnxt_en_dev *en_dev;</div><div class='del'>-	struct pci_dev *pdev;</div><div class='add'>+	struct bnxt_qplib_hwq *cq_hwq;</div><div class='add'>+	struct nlattr *table_attr;</div><div class='add'>+	struct bnxt_re_cq *cq;</div><div class='add'>+</div><div class='add'>+	cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);</div><div class='add'>+	cq_hwq = &amp;cq-&gt;qplib_cq.hwq;</div><div class='add'>+</div><div class='add'>+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);</div><div class='add'>+	if (!table_attr)</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "cq_depth", cq_hwq-&gt;depth))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "max_elements", cq_hwq-&gt;max_elements))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "element_size", cq_hwq-&gt;element_size))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "max_wqe", cq-&gt;qplib_cq.max_wqe))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	nla_nest_end(msg, table_attr);</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-	/* Call bnxt_en's RoCE probe via indirect API */</div><div class='del'>-	if (!bp-&gt;ulp_probe)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+err:</div><div class='add'>+	nla_nest_cancel(msg, table_attr);</div><div class='add'>+	return -EMSGSIZE;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	en_dev = bp-&gt;ulp_probe(netdev);</div><div class='del'>-	if (IS_ERR(en_dev))</div><div class='del'>-		return en_dev;</div><div class='add'>+static int bnxt_re_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct bnxt_re_cq *cq;</div><div class='add'>+	int err, len;</div><div class='add'>+	void *data;</div><div class='ctx'> </div><div class='del'>-	pdev = en_dev-&gt;pdev;</div><div class='del'>-	if (!pdev)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+	cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);</div><div class='add'>+	rdev = cq-&gt;rdev;</div><div class='ctx'> </div><div class='del'>-	if (!(en_dev-&gt;flags &amp; BNXT_EN_FLAG_ROCE_CAP)) {</div><div class='del'>-		dev_info(&amp;pdev-&gt;dev,</div><div class='del'>-			"%s: probe error: RoCE is not supported on this device",</div><div class='del'>-			ROCE_DRV_MODULE_NAME);</div><div class='del'>-		return ERR_PTR(-ENODEV);</div><div class='del'>-	}</div><div class='add'>+	err = bnxt_re_read_context_allowed(rdev);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='ctx'> </div><div class='del'>-	/* Bump net device reference count */</div><div class='del'>-	if (!try_module_get(pdev-&gt;driver-&gt;driver.owner))</div><div class='del'>-		return ERR_PTR(-ENODEV);</div><div class='add'>+	len = bnxt_qplib_is_chip_gen_p7(rdev-&gt;chip_ctx) ? BNXT_RE_CONTEXT_TYPE_CQ_SIZE_P7 :</div><div class='add'>+					BNXT_RE_CONTEXT_TYPE_CQ_SIZE_P5;</div><div class='add'>+	data = kzalloc(len, GFP_KERNEL);</div><div class='add'>+	if (!data)</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	dev_hold(netdev);</div><div class='add'>+	err = bnxt_qplib_read_context(&amp;rdev-&gt;rcfw,</div><div class='add'>+				      CMDQ_READ_CONTEXT_TYPE_CQ,</div><div class='add'>+				      cq-&gt;qplib_cq.id, len, data);</div><div class='add'>+	if (!err)</div><div class='add'>+		err = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, len, data);</div><div class='ctx'> </div><div class='del'>-	return en_dev;</div><div class='add'>+	kfree(data);</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t hw_rev_show(struct device *device, struct device_attribute *attr,</div><div class='del'>-			   char *buf)</div><div class='add'>+static int bnxt_re_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_re_dev *rdev =</div><div class='del'>-		rdma_device_to_drv_device(device, struct bnxt_re_dev, ibdev);</div><div class='add'>+	struct bnxt_qplib_qp *qplib_qp;</div><div class='add'>+	struct nlattr *table_attr;</div><div class='add'>+	struct bnxt_re_qp *qp;</div><div class='add'>+</div><div class='add'>+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);</div><div class='add'>+	if (!table_attr)</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);</div><div class='add'>+	qplib_qp = &amp;qp-&gt;qplib_qp;</div><div class='add'>+</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "sq_max_wqe", qplib_qp-&gt;sq.max_wqe))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "sq_max_sge", qplib_qp-&gt;sq.max_sge))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "sq_wqe_size", qplib_qp-&gt;sq.wqe_size))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "sq_swq_start", qplib_qp-&gt;sq.swq_start))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "sq_swq_last", qplib_qp-&gt;sq.swq_last))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "rq_max_wqe", qplib_qp-&gt;rq.max_wqe))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "rq_max_sge", qplib_qp-&gt;rq.max_sge))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "rq_wqe_size", qplib_qp-&gt;rq.wqe_size))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "rq_swq_start", qplib_qp-&gt;rq.swq_start))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "rq_swq_last", qplib_qp-&gt;rq.swq_last))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "timeout", qplib_qp-&gt;timeout))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	nla_nest_end(msg, table_attr);</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-	return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev-&gt;en_dev-&gt;pdev-&gt;vendor);</div><div class='add'>+err:</div><div class='add'>+	nla_nest_cancel(msg, table_attr);</div><div class='add'>+	return -EMSGSIZE;</div><div class='ctx'> }</div><div class='del'>-static DEVICE_ATTR_RO(hw_rev);</div><div class='ctx'> </div><div class='del'>-static ssize_t hca_type_show(struct device *device,</div><div class='del'>-			     struct device_attribute *attr, char *buf)</div><div class='add'>+static int bnxt_re_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ibqp)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_re_dev *rdev =</div><div class='del'>-		rdma_device_to_drv_device(device, struct bnxt_re_dev, ibdev);</div><div class='add'>+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibqp-&gt;device, ibdev);</div><div class='add'>+	int err, len;</div><div class='add'>+	void *data;</div><div class='add'>+</div><div class='add'>+	err = bnxt_re_read_context_allowed(rdev);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	len = bnxt_qplib_is_chip_gen_p7(rdev-&gt;chip_ctx) ? BNXT_RE_CONTEXT_TYPE_QPC_SIZE_P7 :</div><div class='add'>+							  BNXT_RE_CONTEXT_TYPE_QPC_SIZE_P5;</div><div class='add'>+	data = kzalloc(len, GFP_KERNEL);</div><div class='add'>+	if (!data)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	err = bnxt_qplib_read_context(&amp;rdev-&gt;rcfw, CMDQ_READ_CONTEXT_TYPE_QPC,</div><div class='add'>+				      ibqp-&gt;qp_num, len, data);</div><div class='add'>+	if (!err)</div><div class='add'>+		err = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, len, data);</div><div class='ctx'> </div><div class='del'>-	return scnprintf(buf, PAGE_SIZE, "%s\n", rdev-&gt;ibdev.node_desc);</div><div class='add'>+	kfree(data);</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='del'>-static DEVICE_ATTR_RO(hca_type);</div><div class='ctx'> </div><div class='del'>-static struct attribute *bnxt_re_attributes[] = {</div><div class='del'>-	&amp;dev_attr_hw_rev.attr,</div><div class='del'>-	&amp;dev_attr_hca_type.attr,</div><div class='del'>-	NULL</div><div class='del'>-};</div><div class='add'>+static int bnxt_re_fill_res_srq_entry(struct sk_buff *msg, struct ib_srq *ib_srq)</div><div class='add'>+{</div><div class='add'>+	struct nlattr *table_attr;</div><div class='add'>+	struct bnxt_re_srq *srq;</div><div class='ctx'> </div><div class='del'>-static const struct attribute_group bnxt_re_dev_attr_group = {</div><div class='del'>-	.attrs = bnxt_re_attributes,</div><div class='del'>-};</div><div class='add'>+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);</div><div class='add'>+	if (!table_attr)</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq);</div><div class='add'>+</div><div class='add'>+	if (rdma_nl_put_driver_u32_hex(msg, "wqe_size", srq-&gt;qplib_srq.wqe_size))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32_hex(msg, "max_wqe", srq-&gt;qplib_srq.max_wqe))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32_hex(msg, "max_sge", srq-&gt;qplib_srq.max_sge))</div><div class='add'>+		goto err;</div><div class='add'>+	if (rdma_nl_put_driver_u32_hex(msg, "srq_limit", srq-&gt;qplib_srq.threshold))</div><div class='add'>+		goto err;</div><div class='ctx'> </div><div class='del'>-static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev)</div><div class='add'>+	nla_nest_end(msg, table_attr);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	nla_nest_cancel(msg, table_attr);</div><div class='add'>+	return -EMSGSIZE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bnxt_re_fill_res_srq_entry_raw(struct sk_buff *msg, struct ib_srq *ib_srq)</div><div class='ctx'> {</div><div class='del'>-	ib_unregister_device(&amp;rdev-&gt;ibdev);</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	struct bnxt_re_srq *srq;</div><div class='add'>+	int err, len;</div><div class='add'>+	void *data;</div><div class='add'>+</div><div class='add'>+	srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq);</div><div class='add'>+	rdev = srq-&gt;rdev;</div><div class='add'>+</div><div class='add'>+	err = bnxt_re_read_context_allowed(rdev);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	len = bnxt_qplib_is_chip_gen_p7(rdev-&gt;chip_ctx) ? BNXT_RE_CONTEXT_TYPE_SRQ_SIZE_P7 :</div><div class='add'>+							  BNXT_RE_CONTEXT_TYPE_SRQ_SIZE_P5;</div><div class='add'>+</div><div class='add'>+	data = kzalloc(len, GFP_KERNEL);</div><div class='add'>+	if (!data)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	err = bnxt_qplib_read_context(&amp;rdev-&gt;rcfw, CMDQ_READ_CONTEXT_TYPE_SRQ,</div><div class='add'>+				      srq-&gt;qplib_srq.id, len, data);</div><div class='add'>+	if (!err)</div><div class='add'>+		err = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, len, data);</div><div class='add'>+</div><div class='add'>+	kfree(data);</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static const struct ib_device_ops bnxt_re_dev_ops = {</div><div class='add'>+	.owner = THIS_MODULE,</div><div class='add'>+	.driver_id = RDMA_DRIVER_BNXT_RE,</div><div class='add'>+	.uverbs_abi_ver = BNXT_RE_ABI_VERSION,</div><div class='add'>+</div><div class='ctx'> 	.add_gid = bnxt_re_add_gid,</div><div class='del'>-	.alloc_hw_stats = bnxt_re_ib_alloc_hw_stats,</div><div class='add'>+	.alloc_hw_port_stats = bnxt_re_ib_alloc_hw_port_stats,</div><div class='ctx'> 	.alloc_mr = bnxt_re_alloc_mr,</div><div class='ctx'> 	.alloc_pd = bnxt_re_alloc_pd,</div><div class='ctx'> 	.alloc_ucontext = bnxt_re_alloc_ucontext,</div><div class='hunk'>@@ -605,6 +1336,7 @@ static const struct ib_device_ops bnxt_re_dev_ops = {</div><div class='ctx'> 	.create_cq = bnxt_re_create_cq,</div><div class='ctx'> 	.create_qp = bnxt_re_create_qp,</div><div class='ctx'> 	.create_srq = bnxt_re_create_srq,</div><div class='add'>+	.create_user_ah = bnxt_re_create_ah,</div><div class='ctx'> 	.dealloc_pd = bnxt_re_dealloc_pd,</div><div class='ctx'> 	.dealloc_ucontext = bnxt_re_dealloc_ucontext,</div><div class='ctx'> 	.del_gid = bnxt_re_del_gid,</div><div class='hunk'>@@ -613,6 +1345,8 @@ static const struct ib_device_ops bnxt_re_dev_ops = {</div><div class='ctx'> 	.destroy_cq = bnxt_re_destroy_cq,</div><div class='ctx'> 	.destroy_qp = bnxt_re_destroy_qp,</div><div class='ctx'> 	.destroy_srq = bnxt_re_destroy_srq,</div><div class='add'>+	.device_group = &amp;bnxt_re_dev_attr_group,</div><div class='add'>+	.disassociate_ucontext = bnxt_re_disassociate_ucontext,</div><div class='ctx'> 	.get_dev_fw_str = bnxt_re_query_fw_str,</div><div class='ctx'> 	.get_dma_mr = bnxt_re_get_dma_mr,</div><div class='ctx'> 	.get_hw_stats = bnxt_re_ib_get_hw_stats,</div><div class='hunk'>@@ -620,102 +1354,77 @@ static const struct ib_device_ops bnxt_re_dev_ops = {</div><div class='ctx'> 	.get_port_immutable = bnxt_re_get_port_immutable,</div><div class='ctx'> 	.map_mr_sg = bnxt_re_map_mr_sg,</div><div class='ctx'> 	.mmap = bnxt_re_mmap,</div><div class='del'>-	.modify_ah = bnxt_re_modify_ah,</div><div class='del'>-	.modify_device = bnxt_re_modify_device,</div><div class='add'>+	.mmap_free = bnxt_re_mmap_free,</div><div class='ctx'> 	.modify_qp = bnxt_re_modify_qp,</div><div class='ctx'> 	.modify_srq = bnxt_re_modify_srq,</div><div class='ctx'> 	.poll_cq = bnxt_re_poll_cq,</div><div class='ctx'> 	.post_recv = bnxt_re_post_recv,</div><div class='ctx'> 	.post_send = bnxt_re_post_send,</div><div class='ctx'> 	.post_srq_recv = bnxt_re_post_srq_recv,</div><div class='add'>+	.process_mad = bnxt_re_process_mad,</div><div class='ctx'> 	.query_ah = bnxt_re_query_ah,</div><div class='ctx'> 	.query_device = bnxt_re_query_device,</div><div class='add'>+	.modify_device = bnxt_re_modify_device,</div><div class='ctx'> 	.query_pkey = bnxt_re_query_pkey,</div><div class='ctx'> 	.query_port = bnxt_re_query_port,</div><div class='ctx'> 	.query_qp = bnxt_re_query_qp,</div><div class='ctx'> 	.query_srq = bnxt_re_query_srq,</div><div class='ctx'> 	.reg_user_mr = bnxt_re_reg_user_mr,</div><div class='add'>+	.reg_user_mr_dmabuf = bnxt_re_reg_user_mr_dmabuf,</div><div class='ctx'> 	.req_notify_cq = bnxt_re_req_notify_cq,</div><div class='add'>+	.resize_cq = bnxt_re_resize_cq,</div><div class='add'>+	.create_flow = bnxt_re_create_flow,</div><div class='add'>+	.destroy_flow = bnxt_re_destroy_flow,</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_ah, bnxt_re_ah, ib_ah),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_cq, bnxt_re_cq, ib_cq),</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_pd, bnxt_re_pd, ib_pd),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_qp, bnxt_re_qp, ib_qp),</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_srq, bnxt_re_srq, ib_srq),</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_ucontext, bnxt_re_ucontext, ib_uctx),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static const struct ib_device_ops restrack_ops = {</div><div class='add'>+	.fill_res_cq_entry = bnxt_re_fill_res_cq_entry,</div><div class='add'>+	.fill_res_cq_entry_raw = bnxt_re_fill_res_cq_entry_raw,</div><div class='add'>+	.fill_res_qp_entry = bnxt_re_fill_res_qp_entry,</div><div class='add'>+	.fill_res_qp_entry_raw = bnxt_re_fill_res_qp_entry_raw,</div><div class='add'>+	.fill_res_mr_entry = bnxt_re_fill_res_mr_entry,</div><div class='add'>+	.fill_res_mr_entry_raw = bnxt_re_fill_res_mr_entry_raw,</div><div class='add'>+	.fill_res_srq_entry = bnxt_re_fill_res_srq_entry,</div><div class='add'>+	.fill_res_srq_entry_raw = bnxt_re_fill_res_srq_entry_raw,</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *ibdev = &amp;rdev-&gt;ibdev;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	/* ib device init */</div><div class='del'>-	ibdev-&gt;owner = THIS_MODULE;</div><div class='ctx'> 	ibdev-&gt;node_type = RDMA_NODE_IB_CA;</div><div class='del'>-	strlcpy(ibdev-&gt;node_desc, BNXT_RE_DESC " HCA",</div><div class='del'>-		strlen(BNXT_RE_DESC) + 5);</div><div class='add'>+	strscpy(ibdev-&gt;node_desc, BNXT_RE_DESC " HCA");</div><div class='ctx'> 	ibdev-&gt;phys_port_cnt = 1;</div><div class='ctx'> </div><div class='del'>-	bnxt_qplib_get_guid(rdev-&gt;netdev-&gt;dev_addr, (u8 *)&amp;ibdev-&gt;node_guid);</div><div class='add'>+	addrconf_addr_eui48((u8 *)&amp;ibdev-&gt;node_guid, rdev-&gt;netdev-&gt;dev_addr);</div><div class='ctx'> </div><div class='del'>-	ibdev-&gt;num_comp_vectors	= 1;</div><div class='add'>+	ibdev-&gt;num_comp_vectors	= rdev-&gt;nqr-&gt;num_msix - 1;</div><div class='ctx'> 	ibdev-&gt;dev.parent = &amp;rdev-&gt;en_dev-&gt;pdev-&gt;dev;</div><div class='ctx'> 	ibdev-&gt;local_dma_lkey = BNXT_QPLIB_RSVD_LKEY;</div><div class='ctx'> </div><div class='del'>-	/* User space */</div><div class='del'>-	ibdev-&gt;uverbs_abi_ver = BNXT_RE_ABI_VERSION;</div><div class='del'>-	ibdev-&gt;uverbs_cmd_mask =</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_GET_CONTEXT)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_DEVICE)	|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_PORT)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_ALLOC_PD)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_DEALLOC_PD)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_REG_MR)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_REREG_MR)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_DEREG_MR)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_CQ)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_RESIZE_CQ)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_CQ)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_QP)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_MODIFY_QP)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_QP)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_QP)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_SRQ)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_MODIFY_SRQ)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_SRQ)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_SRQ)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_AH)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_MODIFY_AH)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_AH)		|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_AH);</div><div class='del'>-	/* POLL_CQ and REQ_NOTIFY_CQ is directly handled in libbnxt_re */</div><div class='del'>-</div><div class='del'>-</div><div class='del'>-	rdma_set_device_sysfs_group(ibdev, &amp;bnxt_re_dev_attr_group);</div><div class='del'>-	ibdev-&gt;driver_id = RDMA_DRIVER_BNXT_RE;</div><div class='add'>+	if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))</div><div class='add'>+		ibdev-&gt;driver_def = bnxt_re_uapi_defs;</div><div class='add'>+</div><div class='ctx'> 	ib_set_device_ops(ibdev, &amp;bnxt_re_dev_ops);</div><div class='add'>+	ib_set_device_ops(ibdev, &amp;restrack_ops);</div><div class='ctx'> 	ret = ib_device_set_netdev(&amp;rdev-&gt;ibdev, rdev-&gt;netdev, 1);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	return ib_register_device(ibdev, "bnxt_re%d");</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev)</div><div class='del'>-{</div><div class='del'>-	dev_put(rdev-&gt;netdev);</div><div class='del'>-	rdev-&gt;netdev = NULL;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;bnxt_re_dev_lock);</div><div class='del'>-	list_del_rcu(&amp;rdev-&gt;list);</div><div class='del'>-	mutex_unlock(&amp;bnxt_re_dev_lock);</div><div class='del'>-</div><div class='del'>-	synchronize_rcu();</div><div class='del'>-</div><div class='del'>-	ib_dealloc_device(&amp;rdev-&gt;ibdev);</div><div class='del'>-	/* rdev is gone */</div><div class='add'>+	dma_set_max_seg_size(&amp;rdev-&gt;en_dev-&gt;pdev-&gt;dev, UINT_MAX);</div><div class='add'>+	ibdev-&gt;uverbs_cmd_mask |= BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ);</div><div class='add'>+	return ib_register_device(ibdev, "bnxt_re%d", &amp;rdev-&gt;en_dev-&gt;pdev-&gt;dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,</div><div class='add'>+static struct bnxt_re_dev *bnxt_re_dev_add(struct auxiliary_device *adev,</div><div class='ctx'> 					   struct bnxt_en_dev *en_dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_re_dev *rdev;</div><div class='hunk'>@@ -723,28 +1432,38 @@ static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,</div><div class='ctx'> 	/* Allocate bnxt_re_dev instance here */</div><div class='ctx'> 	rdev = ib_alloc_device(bnxt_re_dev, ibdev);</div><div class='ctx'> 	if (!rdev) {</div><div class='del'>-		dev_err(NULL, "%s: bnxt_re_dev allocation failure!",</div><div class='del'>-			ROCE_DRV_MODULE_NAME);</div><div class='add'>+		ibdev_err(NULL, "%s: bnxt_re_dev allocation failure!",</div><div class='add'>+			  ROCE_DRV_MODULE_NAME);</div><div class='ctx'> 		return NULL;</div><div class='ctx'> 	}</div><div class='ctx'> 	/* Default values */</div><div class='del'>-	rdev-&gt;netdev = netdev;</div><div class='del'>-	dev_hold(rdev-&gt;netdev);</div><div class='add'>+	rdev-&gt;netdev = en_dev-&gt;net;</div><div class='ctx'> 	rdev-&gt;en_dev = en_dev;</div><div class='add'>+	rdev-&gt;adev = adev;</div><div class='ctx'> 	rdev-&gt;id = rdev-&gt;en_dev-&gt;pdev-&gt;devfn;</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;rdev-&gt;qp_list);</div><div class='ctx'> 	mutex_init(&amp;rdev-&gt;qp_lock);</div><div class='del'>-	atomic_set(&amp;rdev-&gt;qp_count, 0);</div><div class='del'>-	atomic_set(&amp;rdev-&gt;cq_count, 0);</div><div class='del'>-	atomic_set(&amp;rdev-&gt;srq_count, 0);</div><div class='del'>-	atomic_set(&amp;rdev-&gt;mr_count, 0);</div><div class='del'>-	atomic_set(&amp;rdev-&gt;mw_count, 0);</div><div class='add'>+	mutex_init(&amp;rdev-&gt;pacing.dbq_lock);</div><div class='add'>+	atomic_set(&amp;rdev-&gt;stats.res.qp_count, 0);</div><div class='add'>+	atomic_set(&amp;rdev-&gt;stats.res.cq_count, 0);</div><div class='add'>+	atomic_set(&amp;rdev-&gt;stats.res.srq_count, 0);</div><div class='add'>+	atomic_set(&amp;rdev-&gt;stats.res.mr_count, 0);</div><div class='add'>+	atomic_set(&amp;rdev-&gt;stats.res.mw_count, 0);</div><div class='add'>+	atomic_set(&amp;rdev-&gt;stats.res.ah_count, 0);</div><div class='add'>+	atomic_set(&amp;rdev-&gt;stats.res.pd_count, 0);</div><div class='ctx'> 	rdev-&gt;cosq[0] = 0xFFFF;</div><div class='ctx'> 	rdev-&gt;cosq[1] = 0xFFFF;</div><div class='add'>+	rdev-&gt;cq_coalescing.enable = 1;</div><div class='add'>+	rdev-&gt;cq_coalescing.buf_maxtime = BNXT_QPLIB_CQ_COAL_DEF_BUF_MAXTIME;</div><div class='add'>+	if (bnxt_re_chip_gen_p7(en_dev-&gt;chip_num)) {</div><div class='add'>+		rdev-&gt;cq_coalescing.normal_maxbuf = BNXT_QPLIB_CQ_COAL_DEF_NORMAL_MAXBUF_P7;</div><div class='add'>+		rdev-&gt;cq_coalescing.during_maxbuf = BNXT_QPLIB_CQ_COAL_DEF_DURING_MAXBUF_P7;</div><div class='add'>+	} else {</div><div class='add'>+		rdev-&gt;cq_coalescing.normal_maxbuf = BNXT_QPLIB_CQ_COAL_DEF_NORMAL_MAXBUF_P5;</div><div class='add'>+		rdev-&gt;cq_coalescing.during_maxbuf = BNXT_QPLIB_CQ_COAL_DEF_DURING_MAXBUF_P5;</div><div class='add'>+	}</div><div class='add'>+	rdev-&gt;cq_coalescing.en_ring_idle_mode = BNXT_QPLIB_CQ_COAL_DEF_EN_RING_IDLE_MODE;</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;bnxt_re_dev_lock);</div><div class='del'>-	list_add_tail_rcu(&amp;rdev-&gt;list, &amp;bnxt_re_dev_list);</div><div class='del'>-	mutex_unlock(&amp;bnxt_re_dev_lock);</div><div class='ctx'> 	return rdev;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -783,24 +1502,163 @@ static int bnxt_re_handle_unaffi_async_event(struct creq_func_event</div><div class='ctx'> static int bnxt_re_handle_qp_async_event(struct creq_qp_event *qp_event,</div><div class='ctx'> 					 struct bnxt_re_qp *qp)</div><div class='ctx'> {</div><div class='del'>-	struct ib_event event;</div><div class='add'>+	struct creq_qp_error_notification *err_event;</div><div class='add'>+	struct bnxt_re_srq *srq = NULL;</div><div class='add'>+	struct ib_event event = {};</div><div class='ctx'> 	unsigned int flags;</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {</div><div class='add'>+	if (qp-&gt;qplib_qp.srq)</div><div class='add'>+		srq =  container_of(qp-&gt;qplib_qp.srq, struct bnxt_re_srq,</div><div class='add'>+				    qplib_srq);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR &amp;&amp;</div><div class='add'>+	    rdma_is_kernel_res(&amp;qp-&gt;ib_qp.res)) {</div><div class='ctx'> 		flags = bnxt_re_lock_cqs(qp);</div><div class='ctx'> 		bnxt_qplib_add_flush_qp(&amp;qp-&gt;qplib_qp);</div><div class='ctx'> 		bnxt_re_unlock_cqs(qp, flags);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	memset(&amp;event, 0, sizeof(event));</div><div class='del'>-	if (qp-&gt;qplib_qp.srq) {</div><div class='del'>-		event.device = &amp;qp-&gt;rdev-&gt;ibdev;</div><div class='del'>-		event.element.qp = &amp;qp-&gt;ib_qp;</div><div class='del'>-		event.event = IB_EVENT_QP_LAST_WQE_REACHED;</div><div class='add'>+	event.device = &amp;qp-&gt;rdev-&gt;ibdev;</div><div class='add'>+	event.element.qp = &amp;qp-&gt;ib_qp;</div><div class='add'>+	event.event = IB_EVENT_QP_FATAL;</div><div class='add'>+</div><div class='add'>+	err_event = (struct creq_qp_error_notification *)qp_event;</div><div class='add'>+</div><div class='add'>+	switch (err_event-&gt;req_err_state_reason) {</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_OPCODE_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_TIMEOUT_RETRY_LIMIT:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RNR_TIMEOUT_RETRY_LIMIT:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_NAK_ARRIVAL_2:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_NAK_ARRIVAL_3:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_INVALID_READ_RESP:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_ILLEGAL_BIND:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_ILLEGAL_FAST_REG:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_ILLEGAL_INVALIDATE:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RETRAN_LOCAL_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_AV_DOMAIN_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_PROD_WQE_MSMTCH_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_PSN_RANGE_CHECK_ERROR:</div><div class='add'>+		event.event = IB_EVENT_QP_ACCESS_ERR;</div><div class='add'>+		break;</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_NAK_ARRIVAL_1:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_NAK_ARRIVAL_4:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_READ_RESP_LENGTH:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_WQE_FORMAT_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_ORRQ_FORMAT_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_INVALID_AVID_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_SERV_TYPE_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_INVALID_OP_ERROR:</div><div class='add'>+		event.event = IB_EVENT_QP_REQ_ERR;</div><div class='add'>+		break;</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RX_MEMORY_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_TX_MEMORY_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_CMP_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_CQ_LOAD_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_TX_PCI_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RX_PCI_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RETX_SETUP_ERROR:</div><div class='add'>+		event.event = IB_EVENT_QP_FATAL;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (err_event-&gt;res_err_state_reason) {</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_EXCEED_MAX:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_PSN_NOT_FOUND:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_INVALID_DUP_RKEY:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_IRRQ_FORMAT_ERROR:</div><div class='add'>+		event.event = IB_EVENT_QP_ACCESS_ERR;</div><div class='add'>+		break;</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_REM_INVALIDATE:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_OPCODE_ERROR:</div><div class='add'>+		event.event = IB_EVENT_QP_REQ_ERR;</div><div class='add'>+		break;</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_CMP_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_MEMORY_ERROR:</div><div class='add'>+		event.event = IB_EVENT_QP_FATAL;</div><div class='add'>+		break;</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR:</div><div class='add'>+	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_SRQ_ERROR:</div><div class='add'>+		if (srq)</div><div class='add'>+			event.event = IB_EVENT_SRQ_ERR;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (err_event-&gt;res_err_state_reason || err_event-&gt;req_err_state_reason) {</div><div class='add'>+		ibdev_dbg(&amp;qp-&gt;rdev-&gt;ibdev,</div><div class='add'>+			  "%s %s qp_id: %d cons (%d %d) req (%d %d) res (%d %d)\n",</div><div class='add'>+			   __func__, rdma_is_kernel_res(&amp;qp-&gt;ib_qp.res) ? "kernel" : "user",</div><div class='add'>+			   qp-&gt;qplib_qp.id,</div><div class='add'>+			   err_event-&gt;sq_cons_idx,</div><div class='add'>+			   err_event-&gt;rq_cons_idx,</div><div class='add'>+			   err_event-&gt;req_slow_path_state,</div><div class='add'>+			   err_event-&gt;req_err_state_reason,</div><div class='add'>+			   err_event-&gt;res_slow_path_state,</div><div class='add'>+			   err_event-&gt;res_err_state_reason);</div><div class='add'>+	} else {</div><div class='add'>+		if (srq)</div><div class='add'>+			event.event = IB_EVENT_QP_LAST_WQE_REACHED;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (event.device &amp;&amp; qp-&gt;ib_qp.event_handler)</div><div class='add'>+	if (event.event == IB_EVENT_SRQ_ERR &amp;&amp; srq-&gt;ib_srq.event_handler)  {</div><div class='add'>+		(*srq-&gt;ib_srq.event_handler)(&amp;event,</div><div class='add'>+				srq-&gt;ib_srq.srq_context);</div><div class='add'>+	} else if (event.device &amp;&amp; qp-&gt;ib_qp.event_handler) {</div><div class='ctx'> 		qp-&gt;ib_qp.event_handler(&amp;event, qp-&gt;ib_qp.qp_context);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bnxt_re_handle_cq_async_error(void *event, struct bnxt_re_cq *cq)</div><div class='add'>+{</div><div class='add'>+	struct creq_cq_error_notification *cqerr;</div><div class='add'>+	struct ib_event ibevent = {};</div><div class='add'>+</div><div class='add'>+	cqerr = event;</div><div class='add'>+	switch (cqerr-&gt;cq_err_reason) {</div><div class='add'>+	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_INVALID_ERROR:</div><div class='add'>+	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_OVERFLOW_ERROR:</div><div class='add'>+	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_LOAD_ERROR:</div><div class='add'>+	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_INVALID_ERROR:</div><div class='add'>+	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_OVERFLOW_ERROR:</div><div class='add'>+	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR:</div><div class='add'>+		ibevent.event = IB_EVENT_CQ_ERR;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ibevent.event == IB_EVENT_CQ_ERR &amp;&amp; cq-&gt;ib_cq.event_handler) {</div><div class='add'>+		ibevent.element.cq = &amp;cq-&gt;ib_cq;</div><div class='add'>+		ibevent.device = &amp;cq-&gt;rdev-&gt;ibdev;</div><div class='add'>+</div><div class='add'>+		ibdev_dbg(&amp;cq-&gt;rdev-&gt;ibdev,</div><div class='add'>+			  "%s err reason %d\n", __func__, cqerr-&gt;cq_err_reason);</div><div class='add'>+		cq-&gt;ib_cq.event_handler(&amp;ibevent, cq-&gt;ib_cq.cq_context);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -808,6 +1666,10 @@ static int bnxt_re_handle_qp_async_event(struct creq_qp_event *qp_event,</div><div class='ctx'> static int bnxt_re_handle_affi_async_event(struct creq_qp_event *affi_async,</div><div class='ctx'> 					   void *obj)</div><div class='ctx'> {</div><div class='add'>+	struct bnxt_qplib_qp *lib_qp;</div><div class='add'>+	struct bnxt_qplib_cq *lib_cq;</div><div class='add'>+	struct bnxt_re_qp *qp;</div><div class='add'>+	struct bnxt_re_cq *cq;</div><div class='ctx'> 	int rc = 0;</div><div class='ctx'> 	u8 event;</div><div class='ctx'> </div><div class='hunk'>@@ -815,11 +1677,19 @@ static int bnxt_re_handle_affi_async_event(struct creq_qp_event *affi_async,</div><div class='ctx'> 		return rc; /* QP was already dead, still return success */</div><div class='ctx'> </div><div class='ctx'> 	event = affi_async-&gt;event;</div><div class='del'>-	if (event == CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION) {</div><div class='del'>-		struct bnxt_qplib_qp *lib_qp = obj;</div><div class='del'>-		struct bnxt_re_qp *qp = container_of(lib_qp, struct bnxt_re_qp,</div><div class='del'>-						     qplib_qp);</div><div class='add'>+	switch (event) {</div><div class='add'>+	case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION:</div><div class='add'>+		lib_qp = obj;</div><div class='add'>+		qp = container_of(lib_qp, struct bnxt_re_qp, qplib_qp);</div><div class='ctx'> 		rc = bnxt_re_handle_qp_async_event(affi_async, qp);</div><div class='add'>+		break;</div><div class='add'>+	case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION:</div><div class='add'>+		lib_cq = obj;</div><div class='add'>+		cq = container_of(lib_cq, struct bnxt_re_cq, qplib_cq);</div><div class='add'>+		rc = bnxt_re_handle_cq_async_error(affi_async, cq);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		rc = -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='hunk'>@@ -850,28 +1720,17 @@ static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq,</div><div class='ctx'> 	struct bnxt_re_srq *srq = container_of(handle, struct bnxt_re_srq,</div><div class='ctx'> 					       qplib_srq);</div><div class='ctx'> 	struct ib_event ib_event;</div><div class='del'>-	int rc = 0;</div><div class='ctx'> </div><div class='del'>-	if (!srq) {</div><div class='del'>-		dev_err(NULL, "%s: SRQ is NULL, SRQN not handled",</div><div class='del'>-			ROCE_DRV_MODULE_NAME);</div><div class='del'>-		rc = -EINVAL;</div><div class='del'>-		goto done;</div><div class='del'>-	}</div><div class='ctx'> 	ib_event.device = &amp;srq-&gt;rdev-&gt;ibdev;</div><div class='ctx'> 	ib_event.element.srq = &amp;srq-&gt;ib_srq;</div><div class='del'>-	if (event == NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT)</div><div class='del'>-		ib_event.event = IB_EVENT_SRQ_LIMIT_REACHED;</div><div class='del'>-	else</div><div class='del'>-		ib_event.event = IB_EVENT_SRQ_ERR;</div><div class='ctx'> </div><div class='ctx'> 	if (srq-&gt;ib_srq.event_handler) {</div><div class='del'>-		/* Lock event_handler? */</div><div class='add'>+		if (event == NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT)</div><div class='add'>+			ib_event.event = IB_EVENT_SRQ_LIMIT_REACHED;</div><div class='ctx'> 		(*srq-&gt;ib_srq.event_handler)(&amp;ib_event,</div><div class='ctx'> 					     srq-&gt;ib_srq.srq_context);</div><div class='ctx'> 	}</div><div class='del'>-done:</div><div class='del'>-	return rc;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,</div><div class='hunk'>@@ -880,31 +1739,18 @@ static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,</div><div class='ctx'> 	struct bnxt_re_cq *cq = container_of(handle, struct bnxt_re_cq,</div><div class='ctx'> 					     qplib_cq);</div><div class='ctx'> </div><div class='del'>-	if (!cq) {</div><div class='del'>-		dev_err(NULL, "%s: CQ is NULL, CQN not handled",</div><div class='del'>-			ROCE_DRV_MODULE_NAME);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-	if (cq-&gt;ib_cq.comp_handler) {</div><div class='del'>-		/* Lock comp_handler? */</div><div class='add'>+	if (cq-&gt;ib_cq.comp_handler)</div><div class='ctx'> 		(*cq-&gt;ib_cq.comp_handler)(&amp;cq-&gt;ib_cq, cq-&gt;ib_cq.cq_context);</div><div class='del'>-	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static u32 bnxt_re_get_nqdb_offset(struct bnxt_re_dev *rdev, u16 indx)</div><div class='del'>-{</div><div class='del'>-	return bnxt_qplib_is_chip_gen_p5(&amp;rdev-&gt;chip_ctx) ?</div><div class='del'>-				0x10000 : rdev-&gt;msix_entries[indx].db_offset;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	for (i = 1; i &lt; rdev-&gt;num_msix; i++)</div><div class='del'>-		bnxt_qplib_disable_nq(&amp;rdev-&gt;nq[i - 1]);</div><div class='add'>+	for (i = 1; i &lt; rdev-&gt;nqr-&gt;num_msix; i++)</div><div class='add'>+		bnxt_qplib_disable_nq(&amp;rdev-&gt;nqr-&gt;nq[i - 1]);</div><div class='ctx'> </div><div class='ctx'> 	if (rdev-&gt;qplib_res.rcfw)</div><div class='ctx'> 		bnxt_qplib_cleanup_res(&amp;rdev-&gt;qplib_res);</div><div class='hunk'>@@ -918,15 +1764,17 @@ static int bnxt_re_init_res(struct bnxt_re_dev *rdev)</div><div class='ctx'> </div><div class='ctx'> 	bnxt_qplib_init_res(&amp;rdev-&gt;qplib_res);</div><div class='ctx'> </div><div class='del'>-	for (i = 1; i &lt; rdev-&gt;num_msix ; i++) {</div><div class='del'>-		db_offt = bnxt_re_get_nqdb_offset(rdev, i);</div><div class='del'>-		rc = bnxt_qplib_enable_nq(rdev-&gt;en_dev-&gt;pdev, &amp;rdev-&gt;nq[i - 1],</div><div class='del'>-					  i - 1, rdev-&gt;msix_entries[i].vector,</div><div class='add'>+	mutex_init(&amp;rdev-&gt;nqr-&gt;load_lock);</div><div class='add'>+</div><div class='add'>+	for (i = 1; i &lt; rdev-&gt;nqr-&gt;num_msix ; i++) {</div><div class='add'>+		db_offt = rdev-&gt;nqr-&gt;msix_entries[i].db_offset;</div><div class='add'>+		rc = bnxt_qplib_enable_nq(rdev-&gt;en_dev-&gt;pdev, &amp;rdev-&gt;nqr-&gt;nq[i - 1],</div><div class='add'>+					  i - 1, rdev-&gt;nqr-&gt;msix_entries[i].vector,</div><div class='ctx'> 					  db_offt, &amp;bnxt_re_cqn_handler,</div><div class='ctx'> 					  &amp;bnxt_re_srqn_handler);</div><div class='ctx'> 		if (rc) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"Failed to enable NQ with rc = 0x%x", rc);</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "Failed to enable NQ with rc = 0x%x", rc);</div><div class='ctx'> 			goto fail;</div><div class='ctx'> 		}</div><div class='ctx'> 		num_vec_enabled++;</div><div class='hunk'>@@ -934,20 +1782,22 @@ static int bnxt_re_init_res(struct bnxt_re_dev *rdev)</div><div class='ctx'> 	return 0;</div><div class='ctx'> fail:</div><div class='ctx'> 	for (i = num_vec_enabled; i &gt;= 0; i--)</div><div class='del'>-		bnxt_qplib_disable_nq(&amp;rdev-&gt;nq[i]);</div><div class='add'>+		bnxt_qplib_disable_nq(&amp;rdev-&gt;nqr-&gt;nq[i]);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void bnxt_re_free_nq_res(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='add'>+	struct bnxt_qplib_nq *nq;</div><div class='ctx'> 	u8 type;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; rdev-&gt;num_msix - 1; i++) {</div><div class='del'>-		type = bnxt_qplib_get_ring_type(&amp;rdev-&gt;chip_ctx);</div><div class='del'>-		bnxt_re_net_ring_free(rdev, rdev-&gt;nq[i].ring_id, type);</div><div class='del'>-		rdev-&gt;nq[i].res = NULL;</div><div class='del'>-		bnxt_qplib_free_nq(&amp;rdev-&gt;nq[i]);</div><div class='add'>+	for (i = 0; i &lt; rdev-&gt;nqr-&gt;num_msix - 1; i++) {</div><div class='add'>+		type = bnxt_qplib_get_ring_type(rdev-&gt;chip_ctx);</div><div class='add'>+		nq = &amp;rdev-&gt;nqr-&gt;nq[i];</div><div class='add'>+		bnxt_re_net_ring_free(rdev, nq-&gt;ring_id, type);</div><div class='add'>+		bnxt_qplib_free_nq(nq);</div><div class='add'>+		nq-&gt;res = NULL;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -957,7 +1807,6 @@ static void bnxt_re_free_res(struct bnxt_re_dev *rdev)</div><div class='ctx'> </div><div class='ctx'> 	if (rdev-&gt;qplib_res.dpi_tbl.max) {</div><div class='ctx'> 		bnxt_qplib_dealloc_dpi(&amp;rdev-&gt;qplib_res,</div><div class='del'>-				       &amp;rdev-&gt;qplib_res.dpi_tbl,</div><div class='ctx'> 				       &amp;rdev-&gt;dpi_privileged);</div><div class='ctx'> 	}</div><div class='ctx'> 	if (rdev-&gt;qplib_res.rcfw) {</div><div class='hunk'>@@ -968,65 +1817,63 @@ static void bnxt_re_free_res(struct bnxt_re_dev *rdev)</div><div class='ctx'> </div><div class='ctx'> static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='add'>+	struct bnxt_re_ring_attr rattr = {};</div><div class='ctx'> 	int num_vec_created = 0;</div><div class='del'>-	dma_addr_t *pg_map;</div><div class='del'>-	int rc = 0, i;</div><div class='del'>-	int pages;</div><div class='add'>+	int rc, i;</div><div class='ctx'> 	u8 type;</div><div class='ctx'> </div><div class='ctx'> 	/* Configure and allocate resources for qplib */</div><div class='ctx'> 	rdev-&gt;qplib_res.rcfw = &amp;rdev-&gt;rcfw;</div><div class='del'>-	rc = bnxt_qplib_get_dev_attr(&amp;rdev-&gt;rcfw, &amp;rdev-&gt;dev_attr,</div><div class='del'>-				     rdev-&gt;is_virtfn);</div><div class='add'>+	rc = bnxt_qplib_get_dev_attr(&amp;rdev-&gt;rcfw);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_alloc_res(&amp;rdev-&gt;qplib_res, rdev-&gt;en_dev-&gt;pdev,</div><div class='del'>-				  rdev-&gt;netdev, &amp;rdev-&gt;dev_attr);</div><div class='add'>+	rc = bnxt_qplib_alloc_res(&amp;rdev-&gt;qplib_res, rdev-&gt;netdev);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_alloc_dpi(&amp;rdev-&gt;qplib_res.dpi_tbl,</div><div class='add'>+	rc = bnxt_qplib_alloc_dpi(&amp;rdev-&gt;qplib_res,</div><div class='ctx'> 				  &amp;rdev-&gt;dpi_privileged,</div><div class='del'>-				  rdev);</div><div class='add'>+				  rdev, BNXT_QPLIB_DPI_TYPE_KERNEL);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto dealloc_res;</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; rdev-&gt;num_msix - 1; i++) {</div><div class='del'>-		rdev-&gt;nq[i].res = &amp;rdev-&gt;qplib_res;</div><div class='del'>-		rdev-&gt;nq[i].hwq.max_elements = BNXT_RE_MAX_CQ_COUNT +</div><div class='del'>-			BNXT_RE_MAX_SRQC_COUNT + 2;</div><div class='del'>-		rc = bnxt_qplib_alloc_nq(rdev-&gt;en_dev-&gt;pdev, &amp;rdev-&gt;nq[i]);</div><div class='add'>+	for (i = 0; i &lt; rdev-&gt;nqr-&gt;num_msix - 1; i++) {</div><div class='add'>+		struct bnxt_qplib_nq *nq;</div><div class='add'>+</div><div class='add'>+		nq = &amp;rdev-&gt;nqr-&gt;nq[i];</div><div class='add'>+		nq-&gt;hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;</div><div class='add'>+		rc = bnxt_qplib_alloc_nq(&amp;rdev-&gt;qplib_res, nq);</div><div class='ctx'> 		if (rc) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev), "Alloc Failed NQ%d rc:%#x",</div><div class='del'>-				i, rc);</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev, "Alloc Failed NQ%d rc:%#x",</div><div class='add'>+				  i, rc);</div><div class='ctx'> 			goto free_nq;</div><div class='ctx'> 		}</div><div class='del'>-		type = bnxt_qplib_get_ring_type(&amp;rdev-&gt;chip_ctx);</div><div class='del'>-		pg_map = rdev-&gt;nq[i].hwq.pbl[PBL_LVL_0].pg_map_arr;</div><div class='del'>-		pages = rdev-&gt;nq[i].hwq.pbl[rdev-&gt;nq[i].hwq.level].pg_count;</div><div class='del'>-		rc = bnxt_re_net_ring_alloc(rdev, pg_map, pages, type,</div><div class='del'>-					    BNXT_QPLIB_NQE_MAX_CNT - 1,</div><div class='del'>-					    rdev-&gt;msix_entries[i + 1].ring_idx,</div><div class='del'>-					    &amp;rdev-&gt;nq[i].ring_id);</div><div class='add'>+		type = bnxt_qplib_get_ring_type(rdev-&gt;chip_ctx);</div><div class='add'>+		rattr.dma_arr = nq-&gt;hwq.pbl[PBL_LVL_0].pg_map_arr;</div><div class='add'>+		rattr.pages = nq-&gt;hwq.pbl[rdev-&gt;nqr-&gt;nq[i].hwq.level].pg_count;</div><div class='add'>+		rattr.type = type;</div><div class='add'>+		rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;</div><div class='add'>+		rattr.depth = BNXT_QPLIB_NQE_MAX_CNT - 1;</div><div class='add'>+		rattr.lrid = rdev-&gt;nqr-&gt;msix_entries[i + 1].ring_idx;</div><div class='add'>+		rc = bnxt_re_net_ring_alloc(rdev, &amp;rattr, &amp;nq-&gt;ring_id);</div><div class='ctx'> 		if (rc) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"Failed to allocate NQ fw id with rc = 0x%x",</div><div class='del'>-				rc);</div><div class='del'>-			bnxt_qplib_free_nq(&amp;rdev-&gt;nq[i]);</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "Failed to allocate NQ fw id with rc = 0x%x",</div><div class='add'>+				  rc);</div><div class='add'>+			bnxt_qplib_free_nq(nq);</div><div class='ctx'> 			goto free_nq;</div><div class='ctx'> 		}</div><div class='ctx'> 		num_vec_created++;</div><div class='ctx'> 	}</div><div class='ctx'> 	return 0;</div><div class='ctx'> free_nq:</div><div class='del'>-	for (i = num_vec_created; i &gt;= 0; i--) {</div><div class='del'>-		type = bnxt_qplib_get_ring_type(&amp;rdev-&gt;chip_ctx);</div><div class='del'>-		bnxt_re_net_ring_free(rdev, rdev-&gt;nq[i].ring_id, type);</div><div class='del'>-		bnxt_qplib_free_nq(&amp;rdev-&gt;nq[i]);</div><div class='add'>+	for (i = num_vec_created - 1; i &gt;= 0; i--) {</div><div class='add'>+		type = bnxt_qplib_get_ring_type(rdev-&gt;chip_ctx);</div><div class='add'>+		bnxt_re_net_ring_free(rdev, rdev-&gt;nqr-&gt;nq[i].ring_id, type);</div><div class='add'>+		bnxt_qplib_free_nq(&amp;rdev-&gt;nqr-&gt;nq[i]);</div><div class='ctx'> 	}</div><div class='ctx'> 	bnxt_qplib_dealloc_dpi(&amp;rdev-&gt;qplib_res,</div><div class='del'>-			       &amp;rdev-&gt;qplib_res.dpi_tbl,</div><div class='ctx'> 			       &amp;rdev-&gt;dpi_privileged);</div><div class='ctx'> dealloc_res:</div><div class='ctx'> 	bnxt_qplib_free_res(&amp;rdev-&gt;qplib_res);</div><div class='hunk'>@@ -1055,63 +1902,17 @@ static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp,</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-#define HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN      0x02</div><div class='del'>-static int bnxt_re_query_hwrm_pri2cos(struct bnxt_re_dev *rdev, u8 dir,</div><div class='del'>-				      u64 *cid_map)</div><div class='del'>-{</div><div class='del'>-	struct hwrm_queue_pri2cos_qcfg_input req = {0};</div><div class='del'>-	struct bnxt *bp = netdev_priv(rdev-&gt;netdev);</div><div class='del'>-	struct hwrm_queue_pri2cos_qcfg_output resp;</div><div class='del'>-	struct bnxt_en_dev *en_dev = rdev-&gt;en_dev;</div><div class='del'>-	struct bnxt_fw_msg fw_msg;</div><div class='del'>-	u32 flags = 0;</div><div class='del'>-	u8 *qcfgmap, *tmp_map;</div><div class='del'>-	int rc = 0, i;</div><div class='del'>-</div><div class='del'>-	if (!cid_map)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	memset(&amp;fw_msg, 0, sizeof(fw_msg));</div><div class='del'>-	bnxt_re_init_hwrm_hdr(rdev, (void *)&amp;req,</div><div class='del'>-			      HWRM_QUEUE_PRI2COS_QCFG, -1, -1);</div><div class='del'>-	flags |= (dir &amp; 0x01);</div><div class='del'>-	flags |= HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN;</div><div class='del'>-	req.flags = cpu_to_le32(flags);</div><div class='del'>-	req.port_id = bp-&gt;pf.port_id;</div><div class='del'>-</div><div class='del'>-	bnxt_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), (void *)&amp;resp,</div><div class='del'>-			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);</div><div class='del'>-	rc = en_dev-&gt;en_ops-&gt;bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &amp;fw_msg);</div><div class='del'>-	if (rc)</div><div class='del'>-		return rc;</div><div class='del'>-</div><div class='del'>-	if (resp.queue_cfg_info) {</div><div class='del'>-		dev_warn(rdev_to_dev(rdev),</div><div class='del'>-			 "Asymmetric cos queue configuration detected");</div><div class='del'>-		dev_warn(rdev_to_dev(rdev),</div><div class='del'>-			 " on device, QoS may not be fully functional\n");</div><div class='del'>-	}</div><div class='del'>-	qcfgmap = &amp;resp.pri0_cos_queue_id;</div><div class='del'>-	tmp_map = (u8 *)cid_map;</div><div class='del'>-	for (i = 0; i &lt; IEEE_8021QAZ_MAX_TCS; i++)</div><div class='del'>-		tmp_map[i] = qcfgmap[i];</div><div class='del'>-</div><div class='del'>-	return rc;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev,</div><div class='ctx'> 					struct bnxt_re_qp *qp)</div><div class='ctx'> {</div><div class='del'>-	return (qp-&gt;ib_qp.qp_type == IB_QPT_GSI) || (qp == rdev-&gt;qp1_sqp);</div><div class='add'>+	return (qp-&gt;ib_qp.qp_type == IB_QPT_GSI) ||</div><div class='add'>+	       (qp == rdev-&gt;gsi_ctx.gsi_sqp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='del'>-	int mask = IB_QP_STATE;</div><div class='del'>-	struct ib_qp_attr qp_attr;</div><div class='ctx'> 	struct bnxt_re_qp *qp;</div><div class='ctx'> </div><div class='del'>-	qp_attr.qp_state = IB_QPS_ERR;</div><div class='ctx'> 	mutex_lock(&amp;rdev-&gt;qp_lock);</div><div class='ctx'> 	list_for_each_entry(qp, &amp;rdev-&gt;qp_list, list) {</div><div class='ctx'> 		/* Modify the state of all QPs except QP1/Shadow QP */</div><div class='hunk'>@@ -1119,173 +1920,233 @@ static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev)</div><div class='ctx'> 			if (qp-&gt;qplib_qp.state !=</div><div class='ctx'> 			    CMDQ_MODIFY_QP_NEW_STATE_RESET &amp;&amp;</div><div class='ctx'> 			    qp-&gt;qplib_qp.state !=</div><div class='del'>-			    CMDQ_MODIFY_QP_NEW_STATE_ERR) {</div><div class='add'>+			    CMDQ_MODIFY_QP_NEW_STATE_ERR)</div><div class='ctx'> 				bnxt_re_dispatch_event(&amp;rdev-&gt;ibdev, &amp;qp-&gt;ib_qp,</div><div class='ctx'> 						       1, IB_EVENT_QP_FATAL);</div><div class='del'>-				bnxt_re_modify_qp(&amp;qp-&gt;ib_qp, &amp;qp_attr, mask,</div><div class='del'>-						  NULL);</div><div class='del'>-			}</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> 	mutex_unlock(&amp;rdev-&gt;qp_lock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int bnxt_re_update_gid(struct bnxt_re_dev *rdev)</div><div class='add'>+static void bnxt_re_net_unregister_async_event(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_qplib_sgid_tbl *sgid_tbl = &amp;rdev-&gt;qplib_res.sgid_tbl;</div><div class='del'>-	struct bnxt_qplib_gid gid;</div><div class='del'>-	u16 gid_idx, index;</div><div class='del'>-	int rc = 0;</div><div class='add'>+	if (rdev-&gt;is_virtfn)</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-	if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &amp;rdev-&gt;flags))</div><div class='del'>-		return 0;</div><div class='add'>+	memset(&amp;rdev-&gt;event_bitmap, 0, sizeof(rdev-&gt;event_bitmap));</div><div class='add'>+	bnxt_register_async_events(rdev-&gt;en_dev, &amp;rdev-&gt;event_bitmap,</div><div class='add'>+				   ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (!sgid_tbl) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated");</div><div class='del'>-		return -EINVAL;</div><div class='add'>+static void bnxt_re_net_register_async_event(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	if (rdev-&gt;is_virtfn)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	rdev-&gt;event_bitmap |= (1 &lt;&lt; ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE);</div><div class='add'>+	bnxt_register_async_events(rdev-&gt;en_dev, &amp;rdev-&gt;event_bitmap,</div><div class='add'>+				   ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_read_vpd_info(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct pci_dev *pdev = rdev-&gt;en_dev-&gt;pdev;</div><div class='add'>+	unsigned int vpd_size, kw_len;</div><div class='add'>+	int pos, size;</div><div class='add'>+	u8 *vpd_data;</div><div class='add'>+</div><div class='add'>+	vpd_data = pci_vpd_alloc(pdev, &amp;vpd_size);</div><div class='add'>+	if (IS_ERR(vpd_data)) {</div><div class='add'>+		pci_warn(pdev, "Unable to read VPD, err=%pe\n", vpd_data);</div><div class='add'>+		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	for (index = 0; index &lt; sgid_tbl-&gt;active; index++) {</div><div class='del'>-		gid_idx = sgid_tbl-&gt;hw_id[index];</div><div class='add'>+	pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,</div><div class='add'>+					   PCI_VPD_RO_KEYWORD_PARTNO, &amp;kw_len);</div><div class='add'>+	if (pos &lt; 0)</div><div class='add'>+		goto free;</div><div class='ctx'> </div><div class='del'>-		if (!memcmp(&amp;sgid_tbl-&gt;tbl[index], &amp;bnxt_qplib_gid_zero,</div><div class='del'>-			    sizeof(bnxt_qplib_gid_zero)))</div><div class='del'>-			continue;</div><div class='del'>-		/* need to modify the VLAN enable setting of non VLAN GID only</div><div class='del'>-		 * as setting is done for VLAN GID while adding GID</div><div class='del'>-		 */</div><div class='del'>-		if (sgid_tbl-&gt;vlan[index])</div><div class='del'>-			continue;</div><div class='add'>+	size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);</div><div class='add'>+	memcpy(rdev-&gt;board_partno, &amp;vpd_data[pos], size);</div><div class='add'>+free:</div><div class='add'>+	kfree(vpd_data);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		memcpy(&amp;gid, &amp;sgid_tbl-&gt;tbl[index], sizeof(gid));</div><div class='add'>+static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_en_dev *en_dev = rdev-&gt;en_dev;</div><div class='add'>+	struct hwrm_ver_get_output resp = {};</div><div class='add'>+	struct hwrm_ver_get_input req = {};</div><div class='add'>+	struct bnxt_qplib_chip_ctx *cctx;</div><div class='add'>+	struct bnxt_fw_msg fw_msg = {};</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='del'>-		rc = bnxt_qplib_update_sgid(sgid_tbl, &amp;gid, gid_idx,</div><div class='del'>-					    rdev-&gt;qplib_res.netdev-&gt;dev_addr);</div><div class='add'>+	bnxt_re_init_hwrm_hdr((void *)&amp;req, HWRM_VER_GET);</div><div class='add'>+	req.hwrm_intf_maj = HWRM_VERSION_MAJOR;</div><div class='add'>+	req.hwrm_intf_min = HWRM_VERSION_MINOR;</div><div class='add'>+	req.hwrm_intf_upd = HWRM_VERSION_UPDATE;</div><div class='add'>+	bnxt_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), (void *)&amp;resp,</div><div class='add'>+			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);</div><div class='add'>+	rc = bnxt_send_msg(en_dev, &amp;fw_msg);</div><div class='add'>+	if (rc) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to query HW version, rc = 0x%x",</div><div class='add'>+			  rc);</div><div class='add'>+		return rc;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return rc;</div><div class='add'>+	cctx = rdev-&gt;chip_ctx;</div><div class='add'>+	cctx-&gt;hwrm_intf_ver =</div><div class='add'>+		(u64)le16_to_cpu(resp.hwrm_intf_major) &lt;&lt; 48 |</div><div class='add'>+		(u64)le16_to_cpu(resp.hwrm_intf_minor) &lt;&lt; 32 |</div><div class='add'>+		(u64)le16_to_cpu(resp.hwrm_intf_build) &lt;&lt; 16 |</div><div class='add'>+		le16_to_cpu(resp.hwrm_intf_patch);</div><div class='add'>+</div><div class='add'>+	cctx-&gt;hwrm_cmd_max_timeout = le16_to_cpu(resp.max_req_timeout);</div><div class='add'>+</div><div class='add'>+	if (!cctx-&gt;hwrm_cmd_max_timeout)</div><div class='add'>+		cctx-&gt;hwrm_cmd_max_timeout = RCFW_FW_STALL_MAX_TIMEOUT;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static u32 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev)</div><div class='add'>+static int bnxt_re_ib_init(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='del'>-	u32 prio_map = 0, tmp_map = 0;</div><div class='del'>-	struct net_device *netdev;</div><div class='del'>-	struct dcb_app app;</div><div class='add'>+	int rc;</div><div class='add'>+	u32 event;</div><div class='ctx'> </div><div class='del'>-	netdev = rdev-&gt;netdev;</div><div class='add'>+	/* Register ib dev */</div><div class='add'>+	rc = bnxt_re_register_ib(rdev);</div><div class='add'>+	if (rc) {</div><div class='add'>+		pr_err("Failed to register with IB: %#x\n", rc);</div><div class='add'>+		return rc;</div><div class='add'>+	}</div><div class='add'>+	dev_info(rdev_to_dev(rdev), "Device registered with IB successfully");</div><div class='add'>+	set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &amp;rdev-&gt;flags);</div><div class='ctx'> </div><div class='del'>-	memset(&amp;app, 0, sizeof(app));</div><div class='del'>-	app.selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE;</div><div class='del'>-	app.protocol = ETH_P_IBOE;</div><div class='del'>-	tmp_map = dcb_ieee_getapp_mask(netdev, &amp;app);</div><div class='del'>-	prio_map = tmp_map;</div><div class='add'>+	event = netif_running(rdev-&gt;netdev) &amp;&amp; netif_carrier_ok(rdev-&gt;netdev) ?</div><div class='add'>+		IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;</div><div class='ctx'> </div><div class='del'>-	app.selector = IEEE_8021QAZ_APP_SEL_DGRAM;</div><div class='del'>-	app.protocol = ROCE_V2_UDP_DPORT;</div><div class='del'>-	tmp_map = dcb_ieee_getapp_mask(netdev, &amp;app);</div><div class='del'>-	prio_map |= tmp_map;</div><div class='add'>+	bnxt_re_dispatch_event(&amp;rdev-&gt;ibdev, NULL, 1, event);</div><div class='ctx'> </div><div class='del'>-	return prio_map;</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void bnxt_re_parse_cid_map(u8 prio_map, u8 *cid_map, u16 *cosq)</div><div class='add'>+static int bnxt_re_alloc_nqr_mem(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='del'>-	u16 prio;</div><div class='del'>-	u8 id;</div><div class='add'>+	rdev-&gt;nqr = kzalloc(sizeof(*rdev-&gt;nqr), GFP_KERNEL);</div><div class='add'>+	if (!rdev-&gt;nqr)</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	for (prio = 0, id = 0; prio &lt; 8; prio++) {</div><div class='del'>-		if (prio_map &amp; (1 &lt;&lt; prio)) {</div><div class='del'>-			cosq[id] = cid_map[prio];</div><div class='del'>-			id++;</div><div class='del'>-			if (id == 2) /* Max 2 tcs supported */</div><div class='del'>-				break;</div><div class='del'>-		}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_free_nqr_mem(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	kfree(rdev-&gt;nqr);</div><div class='add'>+	rdev-&gt;nqr = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* When DEL_GID fails, driver is not freeing GID ctx memory.</div><div class='add'>+ * To avoid the memory leak, free the memory during unload</div><div class='add'>+ */</div><div class='add'>+static void bnxt_re_free_gid_ctx(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_sgid_tbl *sgid_tbl = &amp;rdev-&gt;qplib_res.sgid_tbl;</div><div class='add'>+	struct bnxt_re_gid_ctx *ctx, **ctx_tbl;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (!sgid_tbl-&gt;active)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	ctx_tbl = sgid_tbl-&gt;ctx;</div><div class='add'>+	for (i = 0; i &lt; sgid_tbl-&gt;max; i++) {</div><div class='add'>+		if (sgid_tbl-&gt;hw_id[i] == 0xFFFF)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		ctx = ctx_tbl[i];</div><div class='add'>+		kfree(ctx);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev)</div><div class='add'>+static int bnxt_re_get_stats_ctx(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='del'>-	u8 prio_map = 0;</div><div class='del'>-	u64 cid_map;</div><div class='add'>+	struct bnxt_qplib_ctx *hctx = &amp;rdev-&gt;qplib_ctx;</div><div class='add'>+	struct bnxt_qplib_res *res = &amp;rdev-&gt;qplib_res;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	/* Get priority for roce */</div><div class='del'>-	prio_map = bnxt_re_get_priority_mask(rdev);</div><div class='add'>+	rc = bnxt_qplib_alloc_stats_ctx(res-&gt;pdev, res-&gt;cctx, &amp;hctx-&gt;stats);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	rc = bnxt_re_net_stats_ctx_alloc(rdev, &amp;hctx-&gt;stats);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto free_stat_mem;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+free_stat_mem:</div><div class='add'>+	bnxt_qplib_free_stats_ctx(res-&gt;pdev, &amp;hctx-&gt;stats);</div><div class='ctx'> </div><div class='del'>-	if (prio_map == rdev-&gt;cur_prio_map)</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bnxt_re_get_stats3_ctx(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_ctx *hctx = &amp;rdev-&gt;qplib_ctx;</div><div class='add'>+	struct bnxt_qplib_res *res = &amp;rdev-&gt;qplib_res;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (!rdev-&gt;rcfw.roce_mirror)</div><div class='ctx'> 		return 0;</div><div class='del'>-	rdev-&gt;cur_prio_map = prio_map;</div><div class='del'>-	/* Get cosq id for this priority */</div><div class='del'>-	rc = bnxt_re_query_hwrm_pri2cos(rdev, 0, &amp;cid_map);</div><div class='del'>-	if (rc) {</div><div class='del'>-		dev_warn(rdev_to_dev(rdev), "no cos for p_mask %x\n", prio_map);</div><div class='del'>-		return rc;</div><div class='del'>-	}</div><div class='del'>-	/* Parse CoS IDs for app priority */</div><div class='del'>-	bnxt_re_parse_cid_map(prio_map, (u8 *)&amp;cid_map, rdev-&gt;cosq);</div><div class='ctx'> </div><div class='del'>-	/* Config BONO. */</div><div class='del'>-	rc = bnxt_qplib_map_tc2cos(&amp;rdev-&gt;qplib_res, rdev-&gt;cosq);</div><div class='del'>-	if (rc) {</div><div class='del'>-		dev_warn(rdev_to_dev(rdev), "no tc for cos{%x, %x}\n",</div><div class='del'>-			 rdev-&gt;cosq[0], rdev-&gt;cosq[1]);</div><div class='add'>+	rc = bnxt_qplib_alloc_stats_ctx(res-&gt;pdev, res-&gt;cctx, &amp;hctx-&gt;stats3);</div><div class='add'>+	if (rc)</div><div class='ctx'> 		return rc;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Actual priorities are not programmed as they are already</div><div class='del'>-	 * done by L2 driver; just enable or disable priority vlan tagging</div><div class='del'>-	 */</div><div class='del'>-	if ((prio_map == 0 &amp;&amp; rdev-&gt;qplib_res.prio) ||</div><div class='del'>-	    (prio_map != 0 &amp;&amp; !rdev-&gt;qplib_res.prio)) {</div><div class='del'>-		rdev-&gt;qplib_res.prio = prio_map ? true : false;</div><div class='ctx'> </div><div class='del'>-		bnxt_re_update_gid(rdev);</div><div class='del'>-	}</div><div class='add'>+	rc = bnxt_re_net_stats_ctx_alloc(rdev, &amp;hctx-&gt;stats3);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto free_stat_mem;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='add'>+free_stat_mem:</div><div class='add'>+	bnxt_qplib_free_stats_ctx(res-&gt;pdev, &amp;hctx-&gt;stats3);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev)</div><div class='add'>+static void bnxt_re_put_stats3_ctx(struct bnxt_re_dev *rdev)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_en_dev *en_dev = rdev-&gt;en_dev;</div><div class='del'>-	struct hwrm_ver_get_output resp = {0};</div><div class='del'>-	struct hwrm_ver_get_input req = {0};</div><div class='del'>-	struct bnxt_fw_msg fw_msg;</div><div class='del'>-	int rc = 0;</div><div class='add'>+	struct bnxt_qplib_ctx *hctx = &amp;rdev-&gt;qplib_ctx;</div><div class='add'>+	struct bnxt_qplib_res *res = &amp;rdev-&gt;qplib_res;</div><div class='ctx'> </div><div class='del'>-	memset(&amp;fw_msg, 0, sizeof(fw_msg));</div><div class='del'>-	bnxt_re_init_hwrm_hdr(rdev, (void *)&amp;req,</div><div class='del'>-			      HWRM_VER_GET, -1, -1);</div><div class='del'>-	req.hwrm_intf_maj = HWRM_VERSION_MAJOR;</div><div class='del'>-	req.hwrm_intf_min = HWRM_VERSION_MINOR;</div><div class='del'>-	req.hwrm_intf_upd = HWRM_VERSION_UPDATE;</div><div class='del'>-	bnxt_re_fill_fw_msg(&amp;fw_msg, (void *)&amp;req, sizeof(req), (void *)&amp;resp,</div><div class='del'>-			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);</div><div class='del'>-	rc = en_dev-&gt;en_ops-&gt;bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &amp;fw_msg);</div><div class='del'>-	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev),</div><div class='del'>-			"Failed to query HW version, rc = 0x%x", rc);</div><div class='add'>+	if (!rdev-&gt;rcfw.roce_mirror)</div><div class='ctx'> 		return;</div><div class='del'>-	}</div><div class='del'>-	rdev-&gt;qplib_ctx.hwrm_intf_ver =</div><div class='del'>-		(u64)resp.hwrm_intf_major &lt;&lt; 48 |</div><div class='del'>-		(u64)resp.hwrm_intf_minor &lt;&lt; 32 |</div><div class='del'>-		(u64)resp.hwrm_intf_build &lt;&lt; 16 |</div><div class='del'>-		resp.hwrm_intf_patch;</div><div class='add'>+</div><div class='add'>+	bnxt_re_net_stats_ctx_free(rdev, hctx-&gt;stats3.fw_id);</div><div class='add'>+	bnxt_qplib_free_stats_ctx(res-&gt;pdev, &amp;hctx-&gt;stats3);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev)</div><div class='add'>+static void bnxt_re_put_stats_ctx(struct bnxt_re_dev *rdev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_ctx *hctx = &amp;rdev-&gt;qplib_ctx;</div><div class='add'>+	struct bnxt_qplib_res *res = &amp;rdev-&gt;qplib_res;</div><div class='add'>+</div><div class='add'>+	bnxt_re_net_stats_ctx_free(rdev, hctx-&gt;stats.fw_id);</div><div class='add'>+	bnxt_qplib_free_stats_ctx(res-&gt;pdev, &amp;hctx-&gt;stats);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev, u8 op_type)</div><div class='ctx'> {</div><div class='ctx'> 	u8 type;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	if (test_and_clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &amp;rdev-&gt;flags)) {</div><div class='del'>-		/* Cleanup ib dev */</div><div class='del'>-		bnxt_re_unregister_ib(rdev);</div><div class='del'>-	}</div><div class='del'>-	if (test_and_clear_bit(BNXT_RE_FLAG_QOS_WORK_REG, &amp;rdev-&gt;flags))</div><div class='del'>-		cancel_delayed_work_sync(&amp;rdev-&gt;worker);</div><div class='add'>+	bnxt_re_debugfs_rem_pdev(rdev);</div><div class='add'>+</div><div class='add'>+	bnxt_re_net_unregister_async_event(rdev);</div><div class='add'>+	bnxt_re_uninit_dcb_wq(rdev);</div><div class='add'>+</div><div class='add'>+	bnxt_re_put_stats3_ctx(rdev);</div><div class='ctx'> </div><div class='add'>+	bnxt_re_free_gid_ctx(rdev);</div><div class='ctx'> 	if (test_and_clear_bit(BNXT_RE_FLAG_RESOURCES_INITIALIZED,</div><div class='ctx'> 			       &amp;rdev-&gt;flags))</div><div class='ctx'> 		bnxt_re_cleanup_res(rdev);</div><div class='hunk'>@@ -1295,138 +2156,157 @@ static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev)</div><div class='ctx'> 	if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &amp;rdev-&gt;flags)) {</div><div class='ctx'> 		rc = bnxt_qplib_deinit_rcfw(&amp;rdev-&gt;rcfw);</div><div class='ctx'> 		if (rc)</div><div class='del'>-			dev_warn(rdev_to_dev(rdev),</div><div class='del'>-				 "Failed to deinitialize RCFW: %#x", rc);</div><div class='del'>-		bnxt_re_net_stats_ctx_free(rdev, rdev-&gt;qplib_ctx.stats.fw_id);</div><div class='del'>-		bnxt_qplib_free_ctx(rdev-&gt;en_dev-&gt;pdev, &amp;rdev-&gt;qplib_ctx);</div><div class='add'>+			ibdev_warn(&amp;rdev-&gt;ibdev,</div><div class='add'>+				   "Failed to deinitialize RCFW: %#x", rc);</div><div class='add'>+		bnxt_re_put_stats_ctx(rdev);</div><div class='add'>+		bnxt_qplib_free_hwctx(&amp;rdev-&gt;qplib_res, &amp;rdev-&gt;qplib_ctx);</div><div class='ctx'> 		bnxt_qplib_disable_rcfw_channel(&amp;rdev-&gt;rcfw);</div><div class='del'>-		type = bnxt_qplib_get_ring_type(&amp;rdev-&gt;chip_ctx);</div><div class='del'>-		bnxt_re_net_ring_free(rdev, rdev-&gt;rcfw.creq_ring_id, type);</div><div class='add'>+		type = bnxt_qplib_get_ring_type(rdev-&gt;chip_ctx);</div><div class='add'>+		bnxt_re_net_ring_free(rdev, rdev-&gt;rcfw.creq.ring_id, type);</div><div class='ctx'> 		bnxt_qplib_free_rcfw_channel(&amp;rdev-&gt;rcfw);</div><div class='ctx'> 	}</div><div class='del'>-	if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &amp;rdev-&gt;flags)) {</div><div class='del'>-		rc = bnxt_re_free_msix(rdev);</div><div class='del'>-		if (rc)</div><div class='del'>-			dev_warn(rdev_to_dev(rdev),</div><div class='del'>-				 "Failed to free MSI-X vectors: %#x", rc);</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	bnxt_re_destroy_chip_ctx(rdev);</div><div class='del'>-	if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &amp;rdev-&gt;flags)) {</div><div class='del'>-		rc = bnxt_re_unregister_netdev(rdev);</div><div class='del'>-		if (rc)</div><div class='del'>-			dev_warn(rdev_to_dev(rdev),</div><div class='del'>-				 "Failed to unregister with netdev: %#x", rc);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='add'>+	rdev-&gt;nqr-&gt;num_msix = 0;</div><div class='ctx'> </div><div class='del'>-/* worker thread for polling periodic events. Now used for QoS programming*/</div><div class='del'>-static void bnxt_re_worker(struct work_struct *work)</div><div class='del'>-{</div><div class='del'>-	struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,</div><div class='del'>-						worker.work);</div><div class='add'>+	if (rdev-&gt;pacing.dbr_pacing)</div><div class='add'>+		bnxt_re_deinitialize_dbr_pacing(rdev);</div><div class='ctx'> </div><div class='del'>-	bnxt_re_setup_qos(rdev);</div><div class='del'>-	schedule_delayed_work(&amp;rdev-&gt;worker, msecs_to_jiffies(30000));</div><div class='add'>+	bnxt_re_free_nqr_mem(rdev);</div><div class='add'>+	bnxt_re_destroy_chip_ctx(rdev);</div><div class='add'>+	if (op_type == BNXT_RE_COMPLETE_REMOVE) {</div><div class='add'>+		if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &amp;rdev-&gt;flags))</div><div class='add'>+			bnxt_unregister_dev(rdev-&gt;en_dev);</div><div class='add'>+	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)</div><div class='add'>+static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type)</div><div class='ctx'> {</div><div class='del'>-	dma_addr_t *pg_map;</div><div class='del'>-	u32 db_offt, ridx;</div><div class='del'>-	int pages, vid;</div><div class='del'>-	bool locked;</div><div class='add'>+	struct bnxt_re_ring_attr rattr = {};</div><div class='add'>+	struct bnxt_qplib_creq_ctx *creq;</div><div class='add'>+	u32 db_offt;</div><div class='add'>+	int vid;</div><div class='ctx'> 	u8 type;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	/* Acquire rtnl lock through out this function */</div><div class='del'>-	rtnl_lock();</div><div class='del'>-	locked = true;</div><div class='add'>+	if (op_type == BNXT_RE_COMPLETE_INIT) {</div><div class='add'>+		/* Registered a new RoCE device instance to netdev */</div><div class='add'>+		rc = bnxt_re_register_netdev(rdev);</div><div class='add'>+		if (rc) {</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "Failed to register with Ethernet driver, rc %d\n",</div><div class='add'>+				  rc);</div><div class='add'>+			return rc;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &amp;rdev-&gt;flags);</div><div class='ctx'> </div><div class='del'>-	/* Registered a new RoCE device instance to netdev */</div><div class='del'>-	rc = bnxt_re_register_netdev(rdev);</div><div class='del'>-	if (rc) {</div><div class='del'>-		rtnl_unlock();</div><div class='del'>-		pr_err("Failed to register with netedev: %#x\n", rc);</div><div class='add'>+	if (rdev-&gt;en_dev-&gt;ulp_tbl-&gt;msix_requested &lt; BNXT_RE_MIN_MSIX) {</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "RoCE requires minimum 2 MSI-X vectors, but only %d reserved\n",</div><div class='add'>+			  rdev-&gt;en_dev-&gt;ulp_tbl-&gt;msix_requested);</div><div class='add'>+		bnxt_unregister_dev(rdev-&gt;en_dev);</div><div class='add'>+		clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &amp;rdev-&gt;flags);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='del'>-	set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &amp;rdev-&gt;flags);</div><div class='add'>+	ibdev_dbg(&amp;rdev-&gt;ibdev, "Got %d MSI-X vectors\n",</div><div class='add'>+		  rdev-&gt;en_dev-&gt;ulp_tbl-&gt;msix_requested);</div><div class='ctx'> </div><div class='ctx'> 	rc = bnxt_re_setup_chip_ctx(rdev);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		dev_err(rdev_to_dev(rdev), "Failed to get chip context\n");</div><div class='add'>+		bnxt_unregister_dev(rdev-&gt;en_dev);</div><div class='add'>+		clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &amp;rdev-&gt;flags);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to get chip context\n");</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* Check whether VF or PF */</div><div class='del'>-	bnxt_re_get_sriov_func_type(rdev);</div><div class='del'>-</div><div class='del'>-	rc = bnxt_re_request_msix(rdev);</div><div class='add'>+	rc = bnxt_re_alloc_nqr_mem(rdev);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		pr_err("Failed to get MSI-X vectors: %#x\n", rc);</div><div class='del'>-		rc = -EINVAL;</div><div class='del'>-		goto fail;</div><div class='add'>+		bnxt_re_destroy_chip_ctx(rdev);</div><div class='add'>+		bnxt_unregister_dev(rdev-&gt;en_dev);</div><div class='add'>+		clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &amp;rdev-&gt;flags);</div><div class='add'>+		return rc;</div><div class='ctx'> 	}</div><div class='del'>-	set_bit(BNXT_RE_FLAG_GOT_MSIX, &amp;rdev-&gt;flags);</div><div class='add'>+	rdev-&gt;nqr-&gt;num_msix = rdev-&gt;en_dev-&gt;ulp_tbl-&gt;msix_requested;</div><div class='add'>+	memcpy(rdev-&gt;nqr-&gt;msix_entries, rdev-&gt;en_dev-&gt;msix_entries,</div><div class='add'>+	       sizeof(struct bnxt_msix_entry) * rdev-&gt;nqr-&gt;num_msix);</div><div class='ctx'> </div><div class='del'>-	bnxt_re_query_hwrm_intf_version(rdev);</div><div class='add'>+	/* Check whether VF or PF */</div><div class='add'>+	bnxt_re_get_sriov_func_type(rdev);</div><div class='ctx'> </div><div class='ctx'> 	/* Establish RCFW Communication Channel to initialize the context</div><div class='ctx'> 	 * memory for the function and all child VFs</div><div class='ctx'> 	 */</div><div class='del'>-	rc = bnxt_qplib_alloc_rcfw_channel(rdev-&gt;en_dev-&gt;pdev, &amp;rdev-&gt;rcfw,</div><div class='del'>-					   &amp;rdev-&gt;qplib_ctx,</div><div class='del'>-					   BNXT_RE_MAX_QPC_COUNT);</div><div class='add'>+	rc = bnxt_qplib_alloc_rcfw_channel(&amp;rdev-&gt;qplib_res, &amp;rdev-&gt;rcfw,</div><div class='add'>+					   &amp;rdev-&gt;qplib_ctx);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		pr_err("Failed to allocate RCFW Channel: %#x\n", rc);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to allocate RCFW Channel: %#x\n", rc);</div><div class='ctx'> 		goto fail;</div><div class='ctx'> 	}</div><div class='del'>-	type = bnxt_qplib_get_ring_type(&amp;rdev-&gt;chip_ctx);</div><div class='del'>-	pg_map = rdev-&gt;rcfw.creq.pbl[PBL_LVL_0].pg_map_arr;</div><div class='del'>-	pages = rdev-&gt;rcfw.creq.pbl[rdev-&gt;rcfw.creq.level].pg_count;</div><div class='del'>-	ridx = rdev-&gt;msix_entries[BNXT_RE_AEQ_IDX].ring_idx;</div><div class='del'>-	rc = bnxt_re_net_ring_alloc(rdev, pg_map, pages, type,</div><div class='del'>-				    BNXT_QPLIB_CREQE_MAX_CNT - 1,</div><div class='del'>-				    ridx, &amp;rdev-&gt;rcfw.creq_ring_id);</div><div class='add'>+</div><div class='add'>+	type = bnxt_qplib_get_ring_type(rdev-&gt;chip_ctx);</div><div class='add'>+	creq = &amp;rdev-&gt;rcfw.creq;</div><div class='add'>+	rattr.dma_arr = creq-&gt;hwq.pbl[PBL_LVL_0].pg_map_arr;</div><div class='add'>+	rattr.pages = creq-&gt;hwq.pbl[creq-&gt;hwq.level].pg_count;</div><div class='add'>+	rattr.type = type;</div><div class='add'>+	rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;</div><div class='add'>+	rattr.depth = BNXT_QPLIB_CREQE_MAX_CNT - 1;</div><div class='add'>+	rattr.lrid = rdev-&gt;nqr-&gt;msix_entries[BNXT_RE_AEQ_IDX].ring_idx;</div><div class='add'>+	rc = bnxt_re_net_ring_alloc(rdev, &amp;rattr, &amp;creq-&gt;ring_id);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		pr_err("Failed to allocate CREQ: %#x\n", rc);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to allocate CREQ: %#x\n", rc);</div><div class='ctx'> 		goto free_rcfw;</div><div class='ctx'> 	}</div><div class='del'>-	db_offt = bnxt_re_get_nqdb_offset(rdev, BNXT_RE_AEQ_IDX);</div><div class='del'>-	vid = rdev-&gt;msix_entries[BNXT_RE_AEQ_IDX].vector;</div><div class='del'>-	rc = bnxt_qplib_enable_rcfw_channel(rdev-&gt;en_dev-&gt;pdev, &amp;rdev-&gt;rcfw,</div><div class='del'>-					    vid, db_offt, rdev-&gt;is_virtfn,</div><div class='add'>+	db_offt = rdev-&gt;nqr-&gt;msix_entries[BNXT_RE_AEQ_IDX].db_offset;</div><div class='add'>+	vid = rdev-&gt;nqr-&gt;msix_entries[BNXT_RE_AEQ_IDX].vector;</div><div class='add'>+	rc = bnxt_qplib_enable_rcfw_channel(&amp;rdev-&gt;rcfw,</div><div class='add'>+					    vid, db_offt,</div><div class='ctx'> 					    &amp;bnxt_re_aeq_handler);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		pr_err("Failed to enable RCFW channel: %#x\n", rc);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to enable RCFW channel: %#x\n",</div><div class='add'>+			  rc);</div><div class='ctx'> 		goto free_ring;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_get_dev_attr(&amp;rdev-&gt;rcfw, &amp;rdev-&gt;dev_attr,</div><div class='del'>-				     rdev-&gt;is_virtfn);</div><div class='add'>+	if (bnxt_qplib_dbr_pacing_en(rdev-&gt;chip_ctx)) {</div><div class='add'>+		rc = bnxt_re_initialize_dbr_pacing(rdev);</div><div class='add'>+		if (!rc) {</div><div class='add'>+			rdev-&gt;pacing.dbr_pacing = true;</div><div class='add'>+		} else {</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "DBR pacing disabled with error : %d\n", rc);</div><div class='add'>+			rdev-&gt;pacing.dbr_pacing = false;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	rc = bnxt_qplib_get_dev_attr(&amp;rdev-&gt;rcfw);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto disable_rcfw;</div><div class='del'>-	if (!rdev-&gt;is_virtfn)</div><div class='del'>-		bnxt_re_set_resource_limits(rdev);</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_alloc_ctx(rdev-&gt;en_dev-&gt;pdev, &amp;rdev-&gt;qplib_ctx, 0,</div><div class='del'>-				  bnxt_qplib_is_chip_gen_p5(&amp;rdev-&gt;chip_ctx));</div><div class='del'>-	if (rc) {</div><div class='del'>-		pr_err("Failed to allocate QPLIB context: %#x\n", rc);</div><div class='del'>-		goto disable_rcfw;</div><div class='add'>+	bnxt_qplib_query_version(&amp;rdev-&gt;rcfw);</div><div class='add'>+	bnxt_re_set_resource_limits(rdev);</div><div class='add'>+</div><div class='add'>+	if (!rdev-&gt;is_virtfn &amp;&amp;</div><div class='add'>+	    !bnxt_qplib_is_chip_gen_p5_p7(rdev-&gt;chip_ctx)) {</div><div class='add'>+		rc = bnxt_qplib_alloc_hwctx(&amp;rdev-&gt;qplib_res, &amp;rdev-&gt;qplib_ctx);</div><div class='add'>+		if (rc) {</div><div class='add'>+			ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+				  "Failed to allocate hw context: %#x\n", rc);</div><div class='add'>+			goto disable_rcfw;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='del'>-	rc = bnxt_re_net_stats_ctx_alloc(rdev,</div><div class='del'>-					 rdev-&gt;qplib_ctx.stats.dma_map,</div><div class='del'>-					 &amp;rdev-&gt;qplib_ctx.stats.fw_id);</div><div class='add'>+</div><div class='add'>+	rc = bnxt_re_get_stats_ctx(rdev);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		pr_err("Failed to allocate stats context: %#x\n", rc);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to allocate stats context: %#x\n", rc);</div><div class='ctx'> 		goto free_ctx;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	rc = bnxt_qplib_init_rcfw(&amp;rdev-&gt;rcfw, &amp;rdev-&gt;qplib_ctx,</div><div class='ctx'> 				  rdev-&gt;is_virtfn);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		pr_err("Failed to initialize RCFW: %#x\n", rc);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to initialize RCFW: %#x\n", rc);</div><div class='ctx'> 		goto free_sctx;</div><div class='ctx'> 	}</div><div class='ctx'> 	set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &amp;rdev-&gt;flags);</div><div class='hunk'>@@ -1434,306 +2314,300 @@ static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)</div><div class='ctx'> 	/* Resources based on the 'new' device caps */</div><div class='ctx'> 	rc = bnxt_re_alloc_res(rdev);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		pr_err("Failed to allocate resources: %#x\n", rc);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to allocate resources: %#x\n", rc);</div><div class='ctx'> 		goto fail;</div><div class='ctx'> 	}</div><div class='ctx'> 	set_bit(BNXT_RE_FLAG_RESOURCES_ALLOCATED, &amp;rdev-&gt;flags);</div><div class='ctx'> 	rc = bnxt_re_init_res(rdev);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		pr_err("Failed to initialize resources: %#x\n", rc);</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev,</div><div class='add'>+			  "Failed to initialize resources: %#x\n", rc);</div><div class='ctx'> 		goto fail;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	set_bit(BNXT_RE_FLAG_RESOURCES_INITIALIZED, &amp;rdev-&gt;flags);</div><div class='ctx'> </div><div class='ctx'> 	if (!rdev-&gt;is_virtfn) {</div><div class='del'>-		rc = bnxt_re_setup_qos(rdev);</div><div class='add'>+		/* Query f/w defaults of CC params */</div><div class='add'>+		rc = bnxt_qplib_query_cc_param(&amp;rdev-&gt;qplib_res, &amp;rdev-&gt;cc_param);</div><div class='ctx'> 		if (rc)</div><div class='del'>-			pr_info("RoCE priority not yet configured\n");</div><div class='add'>+			ibdev_warn(&amp;rdev-&gt;ibdev, "Failed to query CC defaults\n");</div><div class='ctx'> </div><div class='del'>-		INIT_DELAYED_WORK(&amp;rdev-&gt;worker, bnxt_re_worker);</div><div class='del'>-		set_bit(BNXT_RE_FLAG_QOS_WORK_REG, &amp;rdev-&gt;flags);</div><div class='del'>-		schedule_delayed_work(&amp;rdev-&gt;worker, msecs_to_jiffies(30000));</div><div class='add'>+		if (!(rdev-&gt;qplib_res.en_dev-&gt;flags &amp; BNXT_EN_FLAG_ROCE_VF_RES_MGMT))</div><div class='add'>+			bnxt_re_vf_res_config(rdev);</div><div class='ctx'> 	}</div><div class='add'>+	hash_init(rdev-&gt;cq_hash);</div><div class='add'>+	if (rdev-&gt;chip_ctx-&gt;modes.toggle_bits &amp; BNXT_QPLIB_SRQ_TOGGLE_BIT)</div><div class='add'>+		hash_init(rdev-&gt;srq_hash);</div><div class='ctx'> </div><div class='del'>-	rtnl_unlock();</div><div class='del'>-	locked = false;</div><div class='add'>+	bnxt_re_debugfs_add_pdev(rdev);</div><div class='ctx'> </div><div class='del'>-	/* Register ib dev */</div><div class='del'>-	rc = bnxt_re_register_ib(rdev);</div><div class='del'>-	if (rc) {</div><div class='del'>-		pr_err("Failed to register with IB: %#x\n", rc);</div><div class='add'>+	bnxt_re_init_dcb_wq(rdev);</div><div class='add'>+	bnxt_re_net_register_async_event(rdev);</div><div class='add'>+</div><div class='add'>+	if (!rdev-&gt;is_virtfn)</div><div class='add'>+		bnxt_re_read_vpd_info(rdev);</div><div class='add'>+</div><div class='add'>+	rc = bnxt_re_get_stats3_ctx(rdev);</div><div class='add'>+	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='del'>-	}</div><div class='del'>-	set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &amp;rdev-&gt;flags);</div><div class='del'>-	dev_info(rdev_to_dev(rdev), "Device registered successfully");</div><div class='del'>-	ib_get_eth_speed(&amp;rdev-&gt;ibdev, 1, &amp;rdev-&gt;active_speed,</div><div class='del'>-			 &amp;rdev-&gt;active_width);</div><div class='del'>-	set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &amp;rdev-&gt;flags);</div><div class='del'>-	bnxt_re_dispatch_event(&amp;rdev-&gt;ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE);</div><div class='del'>-	bnxt_re_dispatch_event(&amp;rdev-&gt;ibdev, NULL, 1, IB_EVENT_GID_CHANGE);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> free_sctx:</div><div class='ctx'> 	bnxt_re_net_stats_ctx_free(rdev, rdev-&gt;qplib_ctx.stats.fw_id);</div><div class='ctx'> free_ctx:</div><div class='del'>-	bnxt_qplib_free_ctx(rdev-&gt;en_dev-&gt;pdev, &amp;rdev-&gt;qplib_ctx);</div><div class='add'>+	bnxt_qplib_free_hwctx(&amp;rdev-&gt;qplib_res, &amp;rdev-&gt;qplib_ctx);</div><div class='ctx'> disable_rcfw:</div><div class='ctx'> 	bnxt_qplib_disable_rcfw_channel(&amp;rdev-&gt;rcfw);</div><div class='ctx'> free_ring:</div><div class='del'>-	type = bnxt_qplib_get_ring_type(&amp;rdev-&gt;chip_ctx);</div><div class='del'>-	bnxt_re_net_ring_free(rdev, rdev-&gt;rcfw.creq_ring_id, type);</div><div class='add'>+	type = bnxt_qplib_get_ring_type(rdev-&gt;chip_ctx);</div><div class='add'>+	bnxt_re_net_ring_free(rdev, rdev-&gt;rcfw.creq.ring_id, type);</div><div class='ctx'> free_rcfw:</div><div class='ctx'> 	bnxt_qplib_free_rcfw_channel(&amp;rdev-&gt;rcfw);</div><div class='ctx'> fail:</div><div class='del'>-	if (!locked)</div><div class='del'>-		rtnl_lock();</div><div class='del'>-	bnxt_re_ib_unreg(rdev);</div><div class='del'>-	rtnl_unlock();</div><div class='add'>+	bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE);</div><div class='ctx'> </div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev)</div><div class='add'>+static void bnxt_re_setup_cc(struct bnxt_re_dev *rdev, bool enable)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_en_dev *en_dev = rdev-&gt;en_dev;</div><div class='del'>-	struct net_device *netdev = rdev-&gt;netdev;</div><div class='add'>+	struct bnxt_qplib_cc_param cc_param = {};</div><div class='add'>+</div><div class='add'>+	/* Do not enable congestion control on VFs */</div><div class='add'>+	if (rdev-&gt;is_virtfn)</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-	bnxt_re_dev_remove(rdev);</div><div class='add'>+	/* Currently enabling only for GenP5 adapters */</div><div class='add'>+	if (!bnxt_qplib_is_chip_gen_p5_p7(rdev-&gt;chip_ctx))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	if (enable) {</div><div class='add'>+		cc_param.enable  = 1;</div><div class='add'>+		cc_param.tos_ecn = 1;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (netdev)</div><div class='del'>-		bnxt_re_dev_unprobe(netdev, en_dev);</div><div class='add'>+	cc_param.mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC |</div><div class='add'>+			 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN);</div><div class='add'>+</div><div class='add'>+	if (bnxt_qplib_modify_cc(&amp;rdev-&gt;qplib_res, &amp;cc_param))</div><div class='add'>+		ibdev_err(&amp;rdev-&gt;ibdev, "Failed to setup CC enable = %d\n", enable);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct net_device *netdev)</div><div class='add'>+static void bnxt_re_update_en_info_rdev(struct bnxt_re_dev *rdev,</div><div class='add'>+					struct bnxt_re_en_dev_info *en_info,</div><div class='add'>+					struct auxiliary_device *adev)</div><div class='ctx'> {</div><div class='add'>+	/* Before updating the rdev pointer in bnxt_re_en_dev_info structure,</div><div class='add'>+	 * take the rtnl lock to avoid accessing invalid rdev pointer from</div><div class='add'>+	 * L2 ULP callbacks. This is applicable in all the places where rdev</div><div class='add'>+	 * pointer is updated in bnxt_re_en_dev_info.</div><div class='add'>+	 */</div><div class='add'>+	rtnl_lock();</div><div class='add'>+	en_info-&gt;rdev = rdev;</div><div class='add'>+	rtnl_unlock();</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bnxt_re_add_device(struct auxiliary_device *adev, u8 op_type)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_aux_priv *aux_priv =</div><div class='add'>+		container_of(adev, struct bnxt_aux_priv, aux_dev);</div><div class='add'>+	struct bnxt_re_en_dev_info *en_info;</div><div class='ctx'> 	struct bnxt_en_dev *en_dev;</div><div class='del'>-	int rc = 0;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='del'>-	if (!is_bnxt_re_dev(netdev))</div><div class='del'>-		return -ENODEV;</div><div class='add'>+	en_info = auxiliary_get_drvdata(adev);</div><div class='add'>+	en_dev = en_info-&gt;en_dev;</div><div class='ctx'> </div><div class='del'>-	en_dev = bnxt_re_dev_probe(netdev);</div><div class='del'>-	if (IS_ERR(en_dev)) {</div><div class='del'>-		if (en_dev != ERR_PTR(-ENODEV))</div><div class='del'>-			pr_err("%s: Failed to probe\n", ROCE_DRV_MODULE_NAME);</div><div class='del'>-		rc = PTR_ERR(en_dev);</div><div class='del'>-		goto exit;</div><div class='del'>-	}</div><div class='del'>-	*rdev = bnxt_re_dev_add(netdev, en_dev);</div><div class='del'>-	if (!*rdev) {</div><div class='add'>+</div><div class='add'>+	rdev = bnxt_re_dev_add(adev, en_dev);</div><div class='add'>+	if (!rdev || !rdev_to_dev(rdev)) {</div><div class='ctx'> 		rc = -ENOMEM;</div><div class='del'>-		bnxt_re_dev_unprobe(netdev, en_dev);</div><div class='ctx'> 		goto exit;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	bnxt_re_update_en_info_rdev(rdev, en_info, adev);</div><div class='add'>+</div><div class='add'>+	rc = bnxt_re_dev_init(rdev, op_type);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto re_dev_dealloc;</div><div class='add'>+</div><div class='add'>+	rc = bnxt_re_ib_init(rdev);</div><div class='add'>+	if (rc) {</div><div class='add'>+		pr_err("Failed to register with IB: %s",</div><div class='add'>+			aux_priv-&gt;aux_dev.name);</div><div class='add'>+		goto re_dev_uninit;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	bnxt_re_setup_cc(rdev, true);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+re_dev_uninit:</div><div class='add'>+	bnxt_re_update_en_info_rdev(NULL, en_info, adev);</div><div class='add'>+	bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE);</div><div class='add'>+re_dev_dealloc:</div><div class='add'>+	ib_dealloc_device(&amp;rdev-&gt;ibdev);</div><div class='ctx'> exit:</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void bnxt_re_remove_one(struct bnxt_re_dev *rdev)</div><div class='add'>+#define BNXT_ADEV_NAME "bnxt_en"</div><div class='add'>+</div><div class='add'>+static void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 op_type,</div><div class='add'>+				  struct auxiliary_device *aux_dev)</div><div class='ctx'> {</div><div class='del'>-	pci_dev_put(rdev-&gt;en_dev-&gt;pdev);</div><div class='add'>+	bnxt_re_setup_cc(rdev, false);</div><div class='add'>+	ib_unregister_device(&amp;rdev-&gt;ibdev);</div><div class='add'>+	bnxt_re_dev_uninit(rdev, op_type);</div><div class='add'>+	ib_dealloc_device(&amp;rdev-&gt;ibdev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/* Handle all deferred netevents tasks */</div><div class='del'>-static void bnxt_re_task(struct work_struct *work)</div><div class='add'>+static void bnxt_re_remove(struct auxiliary_device *adev)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_re_work *re_work;</div><div class='add'>+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev);</div><div class='ctx'> 	struct bnxt_re_dev *rdev;</div><div class='del'>-	int rc = 0;</div><div class='ctx'> </div><div class='del'>-	re_work = container_of(work, struct bnxt_re_work, work);</div><div class='del'>-	rdev = re_work-&gt;rdev;</div><div class='add'>+	mutex_lock(&amp;bnxt_re_mutex);</div><div class='add'>+	rdev = en_info-&gt;rdev;</div><div class='ctx'> </div><div class='del'>-	if (re_work-&gt;event != NETDEV_REGISTER &amp;&amp;</div><div class='del'>-	    !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &amp;rdev-&gt;flags))</div><div class='del'>-		return;</div><div class='add'>+	if (rdev)</div><div class='add'>+		bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, adev);</div><div class='add'>+	kfree(en_info);</div><div class='add'>+	mutex_unlock(&amp;bnxt_re_mutex);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	switch (re_work-&gt;event) {</div><div class='del'>-	case NETDEV_REGISTER:</div><div class='del'>-		rc = bnxt_re_ib_reg(rdev);</div><div class='del'>-		if (rc) {</div><div class='del'>-			dev_err(rdev_to_dev(rdev),</div><div class='del'>-				"Failed to register with IB: %#x", rc);</div><div class='del'>-			bnxt_re_remove_one(rdev);</div><div class='del'>-			bnxt_re_dev_unreg(rdev);</div><div class='del'>-			goto exit;</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case NETDEV_UP:</div><div class='del'>-		bnxt_re_dispatch_event(&amp;rdev-&gt;ibdev, NULL, 1,</div><div class='del'>-				       IB_EVENT_PORT_ACTIVE);</div><div class='del'>-		break;</div><div class='del'>-	case NETDEV_DOWN:</div><div class='del'>-		bnxt_re_dev_stop(rdev);</div><div class='del'>-		break;</div><div class='del'>-	case NETDEV_CHANGE:</div><div class='del'>-		if (!netif_carrier_ok(rdev-&gt;netdev))</div><div class='del'>-			bnxt_re_dev_stop(rdev);</div><div class='del'>-		else if (netif_carrier_ok(rdev-&gt;netdev))</div><div class='del'>-			bnxt_re_dispatch_event(&amp;rdev-&gt;ibdev, NULL, 1,</div><div class='del'>-					       IB_EVENT_PORT_ACTIVE);</div><div class='del'>-		ib_get_eth_speed(&amp;rdev-&gt;ibdev, 1, &amp;rdev-&gt;active_speed,</div><div class='del'>-				 &amp;rdev-&gt;active_width);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='add'>+static int bnxt_re_probe(struct auxiliary_device *adev,</div><div class='add'>+			 const struct auxiliary_device_id *id)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_aux_priv *aux_priv =</div><div class='add'>+		container_of(adev, struct bnxt_aux_priv, aux_dev);</div><div class='add'>+	struct bnxt_re_en_dev_info *en_info;</div><div class='add'>+	struct bnxt_en_dev *en_dev;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	en_dev = aux_priv-&gt;edev;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;bnxt_re_mutex);</div><div class='add'>+	en_info = kzalloc(sizeof(*en_info), GFP_KERNEL);</div><div class='add'>+	if (!en_info) {</div><div class='add'>+		mutex_unlock(&amp;bnxt_re_mutex);</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='del'>-	smp_mb__before_atomic();</div><div class='del'>-	atomic_dec(&amp;rdev-&gt;sched_count);</div><div class='del'>-exit:</div><div class='del'>-	kfree(re_work);</div><div class='add'>+	en_info-&gt;en_dev = en_dev;</div><div class='add'>+</div><div class='add'>+	auxiliary_set_drvdata(adev, en_info);</div><div class='add'>+</div><div class='add'>+	rc = bnxt_re_add_device(adev, BNXT_RE_COMPLETE_INIT);</div><div class='add'>+	if (rc)</div><div class='add'>+		kfree(en_info);</div><div class='add'>+</div><div class='add'>+	mutex_unlock(&amp;bnxt_re_mutex);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void bnxt_re_init_one(struct bnxt_re_dev *rdev)</div><div class='add'>+static int bnxt_re_suspend(struct auxiliary_device *adev, pm_message_t state)</div><div class='ctx'> {</div><div class='del'>-	pci_dev_get(rdev-&gt;en_dev-&gt;pdev);</div><div class='add'>+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev);</div><div class='add'>+	struct bnxt_en_dev *en_dev;</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='add'>+</div><div class='add'>+	rdev = en_info-&gt;rdev;</div><div class='add'>+	en_dev = en_info-&gt;en_dev;</div><div class='add'>+	mutex_lock(&amp;bnxt_re_mutex);</div><div class='add'>+</div><div class='add'>+	ibdev_info(&amp;rdev-&gt;ibdev, "Handle device suspend call");</div><div class='add'>+	/* Check the current device state from bnxt_en_dev and move the</div><div class='add'>+	 * device to detached state if FW_FATAL_COND is set.</div><div class='add'>+	 * This prevents more commands to HW during clean-up,</div><div class='add'>+	 * in case the device is already in error.</div><div class='add'>+	 */</div><div class='add'>+	if (test_bit(BNXT_STATE_FW_FATAL_COND, &amp;rdev-&gt;en_dev-&gt;en_state)) {</div><div class='add'>+		set_bit(ERR_DEVICE_DETACHED, &amp;rdev-&gt;rcfw.cmdq.flags);</div><div class='add'>+		set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &amp;rdev-&gt;flags);</div><div class='add'>+		wake_up_all(&amp;rdev-&gt;rcfw.cmdq.waitq);</div><div class='add'>+		bnxt_re_dev_stop(rdev);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (rdev-&gt;pacing.dbr_pacing)</div><div class='add'>+		bnxt_re_set_pacing_dev_state(rdev);</div><div class='add'>+</div><div class='add'>+	ibdev_info(&amp;rdev-&gt;ibdev, "%s: L2 driver notified to stop en_state 0x%lx",</div><div class='add'>+		   __func__, en_dev-&gt;en_state);</div><div class='add'>+	bnxt_re_remove_device(rdev, BNXT_RE_PRE_RECOVERY_REMOVE, adev);</div><div class='add'>+	bnxt_re_update_en_info_rdev(NULL, en_info, adev);</div><div class='add'>+	mutex_unlock(&amp;bnxt_re_mutex);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/*</div><div class='del'>- * "Notifier chain callback can be invoked for the same chain from</div><div class='del'>- * different CPUs at the same time".</div><div class='del'>- *</div><div class='del'>- * For cases when the netdev is already present, our call to the</div><div class='del'>- * register_netdevice_notifier() will actually get the rtnl_lock()</div><div class='del'>- * before sending NETDEV_REGISTER and (if up) NETDEV_UP</div><div class='del'>- * events.</div><div class='del'>- *</div><div class='del'>- * But for cases when the netdev is not already present, the notifier</div><div class='del'>- * chain is subjected to be invoked from different CPUs simultaneously.</div><div class='del'>- *</div><div class='del'>- * This is protected by the netdev_mutex.</div><div class='del'>- */</div><div class='del'>-static int bnxt_re_netdev_event(struct notifier_block *notifier,</div><div class='del'>-				unsigned long event, void *ptr)</div><div class='add'>+static int bnxt_re_resume(struct auxiliary_device *adev)</div><div class='ctx'> {</div><div class='del'>-	struct net_device *real_dev, *netdev = netdev_notifier_info_to_dev(ptr);</div><div class='del'>-	struct bnxt_re_work *re_work;</div><div class='add'>+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev);</div><div class='ctx'> 	struct bnxt_re_dev *rdev;</div><div class='del'>-	int rc = 0;</div><div class='del'>-	bool sch_work = false;</div><div class='ctx'> </div><div class='del'>-	real_dev = rdma_vlan_dev_real_dev(netdev);</div><div class='del'>-	if (!real_dev)</div><div class='del'>-		real_dev = netdev;</div><div class='add'>+	mutex_lock(&amp;bnxt_re_mutex);</div><div class='add'>+	bnxt_re_add_device(adev, BNXT_RE_POST_RECOVERY_INIT);</div><div class='add'>+	rdev = en_info-&gt;rdev;</div><div class='add'>+	ibdev_info(&amp;rdev-&gt;ibdev, "Device resume completed");</div><div class='add'>+	mutex_unlock(&amp;bnxt_re_mutex);</div><div class='ctx'> </div><div class='del'>-	rdev = bnxt_re_from_netdev(real_dev);</div><div class='del'>-	if (!rdev &amp;&amp; event != NETDEV_REGISTER)</div><div class='del'>-		goto exit;</div><div class='del'>-	if (real_dev != netdev)</div><div class='del'>-		goto exit;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	switch (event) {</div><div class='del'>-	case NETDEV_REGISTER:</div><div class='del'>-		if (rdev)</div><div class='del'>-			break;</div><div class='del'>-		rc = bnxt_re_dev_reg(&amp;rdev, real_dev);</div><div class='del'>-		if (rc == -ENODEV)</div><div class='del'>-			break;</div><div class='del'>-		if (rc) {</div><div class='del'>-			pr_err("Failed to register with the device %s: %#x\n",</div><div class='del'>-			       real_dev-&gt;name, rc);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		bnxt_re_init_one(rdev);</div><div class='del'>-		sch_work = true;</div><div class='del'>-		break;</div><div class='add'>+static void bnxt_re_shutdown(struct auxiliary_device *adev)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev);</div><div class='add'>+	struct bnxt_re_dev *rdev;</div><div class='ctx'> </div><div class='del'>-	case NETDEV_UNREGISTER:</div><div class='del'>-		/* netdev notifier will call NETDEV_UNREGISTER again later since</div><div class='del'>-		 * we are still holding the reference to the netdev</div><div class='del'>-		 */</div><div class='del'>-		if (atomic_read(&amp;rdev-&gt;sched_count) &gt; 0)</div><div class='del'>-			goto exit;</div><div class='del'>-		bnxt_re_ib_unreg(rdev);</div><div class='del'>-		bnxt_re_remove_one(rdev);</div><div class='del'>-		bnxt_re_dev_unreg(rdev);</div><div class='del'>-		break;</div><div class='add'>+	rdev = en_info-&gt;rdev;</div><div class='add'>+	ib_unregister_device(&amp;rdev-&gt;ibdev);</div><div class='add'>+	bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	default:</div><div class='del'>-		sch_work = true;</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	if (sch_work) {</div><div class='del'>-		/* Allocate for the deferred task */</div><div class='del'>-		re_work = kzalloc(sizeof(*re_work), GFP_ATOMIC);</div><div class='del'>-		if (re_work) {</div><div class='del'>-			re_work-&gt;rdev = rdev;</div><div class='del'>-			re_work-&gt;event = event;</div><div class='del'>-			re_work-&gt;vlan_dev = (real_dev == netdev ?</div><div class='del'>-					     NULL : netdev);</div><div class='del'>-			INIT_WORK(&amp;re_work-&gt;work, bnxt_re_task);</div><div class='del'>-			atomic_inc(&amp;rdev-&gt;sched_count);</div><div class='del'>-			queue_work(bnxt_re_wq, &amp;re_work-&gt;work);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='add'>+static const struct auxiliary_device_id bnxt_re_id_table[] = {</div><div class='add'>+	{ .name = BNXT_ADEV_NAME ".rdma", },</div><div class='add'>+	{},</div><div class='add'>+};</div><div class='ctx'> </div><div class='del'>-exit:</div><div class='del'>-	return NOTIFY_DONE;</div><div class='del'>-}</div><div class='add'>+MODULE_DEVICE_TABLE(auxiliary, bnxt_re_id_table);</div><div class='ctx'> </div><div class='del'>-static struct notifier_block bnxt_re_netdev_notifier = {</div><div class='del'>-	.notifier_call = bnxt_re_netdev_event</div><div class='add'>+static struct auxiliary_driver bnxt_re_driver = {</div><div class='add'>+	.name = "rdma",</div><div class='add'>+	.probe = bnxt_re_probe,</div><div class='add'>+	.remove = bnxt_re_remove,</div><div class='add'>+	.shutdown = bnxt_re_shutdown,</div><div class='add'>+	.suspend = bnxt_re_suspend,</div><div class='add'>+	.resume = bnxt_re_resume,</div><div class='add'>+	.id_table = bnxt_re_id_table,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static int __init bnxt_re_mod_init(void)</div><div class='ctx'> {</div><div class='del'>-	int rc = 0;</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='ctx'> 	pr_info("%s: %s", ROCE_DRV_MODULE_NAME, version);</div><div class='add'>+	bnxt_re_register_debugfs();</div><div class='ctx'> </div><div class='del'>-	bnxt_re_wq = create_singlethread_workqueue("bnxt_re");</div><div class='del'>-	if (!bnxt_re_wq)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	INIT_LIST_HEAD(&amp;bnxt_re_dev_list);</div><div class='del'>-</div><div class='del'>-	rc = register_netdevice_notifier(&amp;bnxt_re_netdev_notifier);</div><div class='add'>+	rc = auxiliary_driver_register(&amp;bnxt_re_driver);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		pr_err("%s: Cannot register to netdevice_notifier",</div><div class='del'>-		       ROCE_DRV_MODULE_NAME);</div><div class='del'>-		goto err_netdev;</div><div class='add'>+		pr_err("%s: Failed to register auxiliary driver\n",</div><div class='add'>+			ROCE_DRV_MODULE_NAME);</div><div class='add'>+		goto err_debug;</div><div class='ctx'> 	}</div><div class='ctx'> 	return 0;</div><div class='del'>-</div><div class='del'>-err_netdev:</div><div class='del'>-	destroy_workqueue(bnxt_re_wq);</div><div class='del'>-</div><div class='add'>+err_debug:</div><div class='add'>+	bnxt_re_unregister_debugfs();</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void __exit bnxt_re_mod_exit(void)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_re_dev *rdev, *next;</div><div class='del'>-	LIST_HEAD(to_be_deleted);</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;bnxt_re_dev_lock);</div><div class='del'>-	/* Free all adapter allocated resources */</div><div class='del'>-	if (!list_empty(&amp;bnxt_re_dev_list))</div><div class='del'>-		list_splice_init(&amp;bnxt_re_dev_list, &amp;to_be_deleted);</div><div class='del'>-	mutex_unlock(&amp;bnxt_re_dev_lock);</div><div class='del'>-       /*</div><div class='del'>-	* Cleanup the devices in reverse order so that the VF device</div><div class='del'>-	* cleanup is done before PF cleanup</div><div class='del'>-	*/</div><div class='del'>-	list_for_each_entry_safe_reverse(rdev, next, &amp;to_be_deleted, list) {</div><div class='del'>-		dev_info(rdev_to_dev(rdev), "Unregistering Device");</div><div class='del'>-		/*</div><div class='del'>-		 * Flush out any scheduled tasks before destroying the</div><div class='del'>-		 * resources</div><div class='del'>-		 */</div><div class='del'>-		flush_workqueue(bnxt_re_wq);</div><div class='del'>-		bnxt_re_dev_stop(rdev);</div><div class='del'>-		/* Acquire the rtnl_lock as the L2 resources are freed here */</div><div class='del'>-		rtnl_lock();</div><div class='del'>-		bnxt_re_ib_unreg(rdev);</div><div class='del'>-		rtnl_unlock();</div><div class='del'>-		bnxt_re_remove_one(rdev);</div><div class='del'>-		bnxt_re_dev_unreg(rdev);</div><div class='del'>-	}</div><div class='del'>-	unregister_netdevice_notifier(&amp;bnxt_re_netdev_notifier);</div><div class='del'>-	if (bnxt_re_wq)</div><div class='del'>-		destroy_workqueue(bnxt_re_wq);</div><div class='add'>+	auxiliary_driver_unregister(&amp;bnxt_re_driver);</div><div class='add'>+	bnxt_re_unregister_debugfs();</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> module_init(bnxt_re_mod_init);</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c<br/>index 958c1ff9c515..c88f049136fc 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_fp.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_fp.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_fp.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/qplib_fp.c</a></div><div class='hunk'>@@ -43,8 +43,10 @@</div><div class='ctx'> #include &lt;linux/sched.h&gt;</div><div class='ctx'> #include &lt;linux/slab.h&gt;</div><div class='ctx'> #include &lt;linux/pci.h&gt;</div><div class='add'>+#include &lt;linux/delay.h&gt;</div><div class='ctx'> #include &lt;linux/prefetch.h&gt;</div><div class='ctx'> #include &lt;linux/if_ether.h&gt;</div><div class='add'>+#include &lt;rdma/ib_mad.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "roce_hsi.h"</div><div class='ctx'> </div><div class='hunk'>@@ -52,10 +54,12 @@</div><div class='ctx'> #include "qplib_rcfw.h"</div><div class='ctx'> #include "qplib_sp.h"</div><div class='ctx'> #include "qplib_fp.h"</div><div class='add'>+#include &lt;rdma/ib_addr.h&gt;</div><div class='add'>+#include "bnxt_ulp.h"</div><div class='add'>+#include "bnxt_re.h"</div><div class='add'>+#include "ib_verbs.h"</div><div class='ctx'> </div><div class='del'>-static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq);</div><div class='ctx'> static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp);</div><div class='del'>-static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type);</div><div class='ctx'> </div><div class='ctx'> static void bnxt_qplib_cancel_phantom_processing(struct bnxt_qplib_qp *qp)</div><div class='ctx'> {</div><div class='hunk'>@@ -179,11 +183,11 @@ static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res,</div><div class='ctx'> </div><div class='ctx'> 	if (qp-&gt;rq_hdr_buf)</div><div class='ctx'> 		dma_free_coherent(&amp;res-&gt;pdev-&gt;dev,</div><div class='del'>-				  rq-&gt;hwq.max_elements * qp-&gt;rq_hdr_buf_size,</div><div class='add'>+				  rq-&gt;max_wqe * qp-&gt;rq_hdr_buf_size,</div><div class='ctx'> 				  qp-&gt;rq_hdr_buf, qp-&gt;rq_hdr_buf_map);</div><div class='ctx'> 	if (qp-&gt;sq_hdr_buf)</div><div class='ctx'> 		dma_free_coherent(&amp;res-&gt;pdev-&gt;dev,</div><div class='del'>-				  sq-&gt;hwq.max_elements * qp-&gt;sq_hdr_buf_size,</div><div class='add'>+				  sq-&gt;max_wqe * qp-&gt;sq_hdr_buf_size,</div><div class='ctx'> 				  qp-&gt;sq_hdr_buf, qp-&gt;sq_hdr_buf_map);</div><div class='ctx'> 	qp-&gt;rq_hdr_buf = NULL;</div><div class='ctx'> 	qp-&gt;sq_hdr_buf = NULL;</div><div class='hunk'>@@ -200,10 +204,9 @@ static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res,</div><div class='ctx'> 	struct bnxt_qplib_q *sq = &amp;qp-&gt;sq;</div><div class='ctx'> 	int rc = 0;</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;sq_hdr_buf_size &amp;&amp; sq-&gt;hwq.max_elements) {</div><div class='add'>+	if (qp-&gt;sq_hdr_buf_size &amp;&amp; sq-&gt;max_wqe) {</div><div class='ctx'> 		qp-&gt;sq_hdr_buf = dma_alloc_coherent(&amp;res-&gt;pdev-&gt;dev,</div><div class='del'>-					sq-&gt;hwq.max_elements *</div><div class='del'>-					qp-&gt;sq_hdr_buf_size,</div><div class='add'>+					sq-&gt;max_wqe * qp-&gt;sq_hdr_buf_size,</div><div class='ctx'> 					&amp;qp-&gt;sq_hdr_buf_map, GFP_KERNEL);</div><div class='ctx'> 		if (!qp-&gt;sq_hdr_buf) {</div><div class='ctx'> 			rc = -ENOMEM;</div><div class='hunk'>@@ -213,9 +216,9 @@ static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;rq_hdr_buf_size &amp;&amp; rq-&gt;hwq.max_elements) {</div><div class='add'>+	if (qp-&gt;rq_hdr_buf_size &amp;&amp; rq-&gt;max_wqe) {</div><div class='ctx'> 		qp-&gt;rq_hdr_buf = dma_alloc_coherent(&amp;res-&gt;pdev-&gt;dev,</div><div class='del'>-						    rq-&gt;hwq.max_elements *</div><div class='add'>+						    rq-&gt;max_wqe *</div><div class='ctx'> 						    qp-&gt;rq_hdr_buf_size,</div><div class='ctx'> 						    &amp;qp-&gt;rq_hdr_buf_map,</div><div class='ctx'> 						    GFP_KERNEL);</div><div class='hunk'>@@ -233,27 +236,20 @@ fail:</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void bnxt_qplib_service_nq(unsigned long data)</div><div class='add'>+static void clean_nq(struct bnxt_qplib_nq *nq, struct bnxt_qplib_cq *cq)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;</div><div class='ctx'> 	struct bnxt_qplib_hwq *hwq = &amp;nq-&gt;hwq;</div><div class='ctx'> 	struct nq_base *nqe, **nq_ptr;</div><div class='del'>-	struct bnxt_qplib_cq *cq;</div><div class='del'>-	int num_cqne_processed = 0;</div><div class='del'>-	int num_srqne_processed = 0;</div><div class='del'>-	u32 sw_cons, raw_cons;</div><div class='del'>-	u16 type;</div><div class='ctx'> 	int budget = nq-&gt;budget;</div><div class='ctx'> 	uintptr_t q_handle;</div><div class='del'>-	bool gen_p5 = bnxt_qplib_is_chip_gen_p5(nq-&gt;res-&gt;cctx);</div><div class='add'>+	u16 type;</div><div class='ctx'> </div><div class='add'>+	spin_lock_bh(&amp;hwq-&gt;lock);</div><div class='ctx'> 	/* Service the NQ until empty */</div><div class='del'>-	raw_cons = hwq-&gt;cons;</div><div class='ctx'> 	while (budget--) {</div><div class='del'>-		sw_cons = HWQ_CMP(raw_cons, hwq);</div><div class='ctx'> 		nq_ptr = (struct nq_base **)hwq-&gt;pbl_ptr;</div><div class='del'>-		nqe = &amp;nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)];</div><div class='del'>-		if (!NQE_CMP_VALID(nqe, raw_cons, hwq-&gt;max_elements))</div><div class='add'>+		nqe = &amp;nq_ptr[NQE_PG(hwq-&gt;cons)][NQE_IDX(hwq-&gt;cons)];</div><div class='add'>+		if (!NQE_CMP_VALID(nqe, nq-&gt;nq_db.dbinfo.flags))</div><div class='ctx'> 			break;</div><div class='ctx'> </div><div class='ctx'> 		/*</div><div class='hunk'>@@ -271,33 +267,114 @@ static void bnxt_qplib_service_nq(unsigned long data)</div><div class='ctx'> 			q_handle = le32_to_cpu(nqcne-&gt;cq_handle_low);</div><div class='ctx'> 			q_handle |= (u64)le32_to_cpu(nqcne-&gt;cq_handle_high)</div><div class='ctx'> 						     &lt;&lt; 32;</div><div class='add'>+			if ((unsigned long)cq == q_handle) {</div><div class='add'>+				nqcne-&gt;cq_handle_low = 0;</div><div class='add'>+				nqcne-&gt;cq_handle_high = 0;</div><div class='add'>+				cq-&gt;cnq_events++;</div><div class='add'>+			}</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		default:</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		bnxt_qplib_hwq_incr_cons(hwq-&gt;max_elements, &amp;hwq-&gt;cons,</div><div class='add'>+					 1, &amp;nq-&gt;nq_db.dbinfo.flags);</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_bh(&amp;hwq-&gt;lock);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* Wait for receiving all NQEs for this CQ and clean the NQEs associated with</div><div class='add'>+ * this CQ.</div><div class='add'>+ */</div><div class='add'>+static void __wait_for_all_nqes(struct bnxt_qplib_cq *cq, u16 cnq_events)</div><div class='add'>+{</div><div class='add'>+	u32 retry_cnt = 100;</div><div class='add'>+</div><div class='add'>+	while (retry_cnt--) {</div><div class='add'>+		if (cnq_events == cq-&gt;cnq_events)</div><div class='add'>+			return;</div><div class='add'>+		usleep_range(50, 100);</div><div class='add'>+		clean_nq(cq-&gt;nq, cq);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_qplib_service_nq(struct tasklet_struct *t)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_nq *nq = from_tasklet(nq, t, nq_tasklet);</div><div class='add'>+	struct bnxt_qplib_hwq *hwq = &amp;nq-&gt;hwq;</div><div class='add'>+	struct bnxt_qplib_cq *cq;</div><div class='add'>+	int budget = nq-&gt;budget;</div><div class='add'>+	struct nq_base *nqe;</div><div class='add'>+	uintptr_t q_handle;</div><div class='add'>+	u32 hw_polled = 0;</div><div class='add'>+	u16 type;</div><div class='add'>+</div><div class='add'>+	spin_lock_bh(&amp;hwq-&gt;lock);</div><div class='add'>+	/* Service the NQ until empty */</div><div class='add'>+	while (budget--) {</div><div class='add'>+		nqe = bnxt_qplib_get_qe(hwq, hwq-&gt;cons, NULL);</div><div class='add'>+		if (!NQE_CMP_VALID(nqe, nq-&gt;nq_db.dbinfo.flags))</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		/*</div><div class='add'>+		 * The valid test of the entry must be done first before</div><div class='add'>+		 * reading any further.</div><div class='add'>+		 */</div><div class='add'>+		dma_rmb();</div><div class='add'>+</div><div class='add'>+		type = le16_to_cpu(nqe-&gt;info10_type) &amp; NQ_BASE_TYPE_MASK;</div><div class='add'>+		switch (type) {</div><div class='add'>+		case NQ_BASE_TYPE_CQ_NOTIFICATION:</div><div class='add'>+		{</div><div class='add'>+			struct nq_cn *nqcne = (struct nq_cn *)nqe;</div><div class='add'>+			struct bnxt_re_cq *cq_p;</div><div class='add'>+</div><div class='add'>+			q_handle = le32_to_cpu(nqcne-&gt;cq_handle_low);</div><div class='add'>+			q_handle |= (u64)le32_to_cpu(nqcne-&gt;cq_handle_high)</div><div class='add'>+						     &lt;&lt; 32;</div><div class='ctx'> 			cq = (struct bnxt_qplib_cq *)(unsigned long)q_handle;</div><div class='del'>-			bnxt_qplib_arm_cq_enable(cq);</div><div class='add'>+			if (!cq)</div><div class='add'>+				break;</div><div class='add'>+			cq-&gt;toggle = (le16_to_cpu(nqe-&gt;info10_type) &amp;</div><div class='add'>+					NQ_CN_TOGGLE_MASK) &gt;&gt; NQ_CN_TOGGLE_SFT;</div><div class='add'>+			cq-&gt;dbinfo.toggle = cq-&gt;toggle;</div><div class='add'>+			cq_p = container_of(cq, struct bnxt_re_cq, qplib_cq);</div><div class='add'>+			if (cq_p-&gt;uctx_cq_page)</div><div class='add'>+				*((u32 *)cq_p-&gt;uctx_cq_page) = cq-&gt;toggle;</div><div class='add'>+</div><div class='add'>+			bnxt_qplib_armen_db(&amp;cq-&gt;dbinfo,</div><div class='add'>+					    DBC_DBC_TYPE_CQ_ARMENA);</div><div class='ctx'> 			spin_lock_bh(&amp;cq-&gt;compl_lock);</div><div class='ctx'> 			atomic_set(&amp;cq-&gt;arm_state, 0);</div><div class='del'>-			if (!nq-&gt;cqn_handler(nq, (cq)))</div><div class='del'>-				num_cqne_processed++;</div><div class='del'>-			else</div><div class='add'>+			if (nq-&gt;cqn_handler(nq, (cq)))</div><div class='ctx'> 				dev_warn(&amp;nq-&gt;pdev-&gt;dev,</div><div class='ctx'> 					 "cqn - type 0x%x not handled\n", type);</div><div class='add'>+			cq-&gt;cnq_events++;</div><div class='ctx'> 			spin_unlock_bh(&amp;cq-&gt;compl_lock);</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='ctx'> 		case NQ_BASE_TYPE_SRQ_EVENT:</div><div class='ctx'> 		{</div><div class='add'>+			struct bnxt_qplib_srq *srq;</div><div class='add'>+			struct bnxt_re_srq *srq_p;</div><div class='ctx'> 			struct nq_srq_event *nqsrqe =</div><div class='ctx'> 						(struct nq_srq_event *)nqe;</div><div class='ctx'> </div><div class='ctx'> 			q_handle = le32_to_cpu(nqsrqe-&gt;srq_handle_low);</div><div class='ctx'> 			q_handle |= (u64)le32_to_cpu(nqsrqe-&gt;srq_handle_high)</div><div class='ctx'> 				     &lt;&lt; 32;</div><div class='del'>-			bnxt_qplib_arm_srq((struct bnxt_qplib_srq *)q_handle,</div><div class='del'>-					   DBC_DBC_TYPE_SRQ_ARMENA);</div><div class='del'>-			if (!nq-&gt;srqn_handler(nq,</div><div class='del'>-					      (struct bnxt_qplib_srq *)q_handle,</div><div class='del'>-					      nqsrqe-&gt;event))</div><div class='del'>-				num_srqne_processed++;</div><div class='del'>-			else</div><div class='add'>+			srq = (struct bnxt_qplib_srq *)q_handle;</div><div class='add'>+			srq-&gt;toggle = (le16_to_cpu(nqe-&gt;info10_type) &amp; NQ_CN_TOGGLE_MASK)</div><div class='add'>+				      &gt;&gt; NQ_CN_TOGGLE_SFT;</div><div class='add'>+			srq-&gt;dbinfo.toggle = srq-&gt;toggle;</div><div class='add'>+			srq_p = container_of(srq, struct bnxt_re_srq, qplib_srq);</div><div class='add'>+			if (srq_p-&gt;uctx_srq_page)</div><div class='add'>+				*((u32 *)srq_p-&gt;uctx_srq_page) = srq-&gt;toggle;</div><div class='add'>+			bnxt_qplib_armen_db(&amp;srq-&gt;dbinfo,</div><div class='add'>+					    DBC_DBC_TYPE_SRQ_ARMENA);</div><div class='add'>+			if (nq-&gt;srqn_handler(nq,</div><div class='add'>+					     (struct bnxt_qplib_srq *)q_handle,</div><div class='add'>+					     nqsrqe-&gt;event))</div><div class='ctx'> 				dev_warn(&amp;nq-&gt;pdev-&gt;dev,</div><div class='ctx'> 					 "SRQ event 0x%x not handled\n",</div><div class='ctx'> 					 nqsrqe-&gt;event);</div><div class='hunk'>@@ -310,50 +387,67 @@ static void bnxt_qplib_service_nq(unsigned long data)</div><div class='ctx'> 				 "nqe with type = 0x%x not handled\n", type);</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='del'>-		raw_cons++;</div><div class='del'>-	}</div><div class='del'>-	if (hwq-&gt;cons != raw_cons) {</div><div class='del'>-		hwq-&gt;cons = raw_cons;</div><div class='del'>-		bnxt_qplib_ring_nq_db_rearm(nq-&gt;bar_reg_iomem, hwq-&gt;cons,</div><div class='del'>-					    hwq-&gt;max_elements, nq-&gt;ring_id,</div><div class='del'>-					    gen_p5);</div><div class='add'>+		hw_polled++;</div><div class='add'>+		bnxt_qplib_hwq_incr_cons(hwq-&gt;max_elements, &amp;hwq-&gt;cons,</div><div class='add'>+					 1, &amp;nq-&gt;nq_db.dbinfo.flags);</div><div class='ctx'> 	}</div><div class='add'>+	if (hw_polled)</div><div class='add'>+		bnxt_qplib_ring_nq_db(&amp;nq-&gt;nq_db.dbinfo, nq-&gt;res-&gt;cctx, true);</div><div class='add'>+	spin_unlock_bh(&amp;hwq-&gt;lock);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* bnxt_re_synchronize_nq - self polling notification queue.</div><div class='add'>+ * @nq      -     notification queue pointer</div><div class='add'>+ *</div><div class='add'>+ * This function will start polling entries of a given notification queue</div><div class='add'>+ * for all pending  entries.</div><div class='add'>+ * This function is useful to synchronize notification entries while resources</div><div class='add'>+ * are going away.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+void bnxt_re_synchronize_nq(struct bnxt_qplib_nq *nq)</div><div class='add'>+{</div><div class='add'>+	int budget = nq-&gt;budget;</div><div class='add'>+</div><div class='add'>+	nq-&gt;budget = nq-&gt;hwq.max_elements;</div><div class='add'>+	bnxt_qplib_service_nq(&amp;nq-&gt;nq_tasklet);</div><div class='add'>+	nq-&gt;budget = budget;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_nq *nq = dev_instance;</div><div class='ctx'> 	struct bnxt_qplib_hwq *hwq = &amp;nq-&gt;hwq;</div><div class='del'>-	struct nq_base **nq_ptr;</div><div class='ctx'> 	u32 sw_cons;</div><div class='ctx'> </div><div class='ctx'> 	/* Prefetch the NQ element */</div><div class='ctx'> 	sw_cons = HWQ_CMP(hwq-&gt;cons, hwq);</div><div class='del'>-	nq_ptr = (struct nq_base **)nq-&gt;hwq.pbl_ptr;</div><div class='del'>-	prefetch(&amp;nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]);</div><div class='add'>+	prefetch(bnxt_qplib_get_qe(hwq, sw_cons, NULL));</div><div class='ctx'> </div><div class='ctx'> 	/* Fan out to CPU affinitized kthreads? */</div><div class='del'>-	tasklet_schedule(&amp;nq-&gt;worker);</div><div class='add'>+	tasklet_schedule(&amp;nq-&gt;nq_tasklet);</div><div class='ctx'> </div><div class='ctx'> 	return IRQ_HANDLED;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill)</div><div class='ctx'> {</div><div class='del'>-	bool gen_p5 = bnxt_qplib_is_chip_gen_p5(nq-&gt;res-&gt;cctx);</div><div class='del'>-	tasklet_disable(&amp;nq-&gt;worker);</div><div class='add'>+	if (!nq-&gt;requested)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	nq-&gt;requested = false;</div><div class='ctx'> 	/* Mask h/w interrupt */</div><div class='del'>-	bnxt_qplib_ring_nq_db(nq-&gt;bar_reg_iomem, nq-&gt;hwq.cons,</div><div class='del'>-			      nq-&gt;hwq.max_elements, nq-&gt;ring_id, gen_p5);</div><div class='add'>+	bnxt_qplib_ring_nq_db(&amp;nq-&gt;nq_db.dbinfo, nq-&gt;res-&gt;cctx, false);</div><div class='ctx'> 	/* Sync with last running IRQ handler */</div><div class='del'>-	synchronize_irq(nq-&gt;vector);</div><div class='add'>+	synchronize_irq(nq-&gt;msix_vec);</div><div class='add'>+	irq_set_affinity_hint(nq-&gt;msix_vec, NULL);</div><div class='add'>+	free_irq(nq-&gt;msix_vec, nq);</div><div class='add'>+	kfree(nq-&gt;name);</div><div class='add'>+	nq-&gt;name = NULL;</div><div class='add'>+</div><div class='ctx'> 	if (kill)</div><div class='del'>-		tasklet_kill(&amp;nq-&gt;worker);</div><div class='del'>-	if (nq-&gt;requested) {</div><div class='del'>-		irq_set_affinity_hint(nq-&gt;vector, NULL);</div><div class='del'>-		free_irq(nq-&gt;vector, nq);</div><div class='del'>-		nq-&gt;requested = false;</div><div class='del'>-	}</div><div class='add'>+		tasklet_kill(&amp;nq-&gt;nq_tasklet);</div><div class='add'>+	tasklet_disable(&amp;nq-&gt;nq_tasklet);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)</div><div class='hunk'>@@ -364,89 +458,114 @@ void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Make sure the HW is stopped! */</div><div class='del'>-	if (nq-&gt;requested)</div><div class='del'>-		bnxt_qplib_nq_stop_irq(nq, true);</div><div class='add'>+	bnxt_qplib_nq_stop_irq(nq, true);</div><div class='ctx'> </div><div class='del'>-	if (nq-&gt;bar_reg_iomem)</div><div class='del'>-		iounmap(nq-&gt;bar_reg_iomem);</div><div class='del'>-	nq-&gt;bar_reg_iomem = NULL;</div><div class='add'>+	if (nq-&gt;nq_db.reg.bar_reg) {</div><div class='add'>+		iounmap(nq-&gt;nq_db.reg.bar_reg);</div><div class='add'>+		nq-&gt;nq_db.reg.bar_reg = NULL;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	nq-&gt;cqn_handler = NULL;</div><div class='ctx'> 	nq-&gt;srqn_handler = NULL;</div><div class='del'>-	nq-&gt;vector = 0;</div><div class='add'>+	nq-&gt;msix_vec = 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx,</div><div class='ctx'> 			    int msix_vector, bool need_init)</div><div class='ctx'> {</div><div class='del'>-	bool gen_p5 = bnxt_qplib_is_chip_gen_p5(nq-&gt;res-&gt;cctx);</div><div class='add'>+	struct bnxt_qplib_res *res = nq-&gt;res;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='ctx'> 	if (nq-&gt;requested)</div><div class='ctx'> 		return -EFAULT;</div><div class='ctx'> </div><div class='del'>-	nq-&gt;vector = msix_vector;</div><div class='add'>+	nq-&gt;msix_vec = msix_vector;</div><div class='ctx'> 	if (need_init)</div><div class='del'>-		tasklet_init(&amp;nq-&gt;worker, bnxt_qplib_service_nq,</div><div class='del'>-			     (unsigned long)nq);</div><div class='add'>+		tasklet_setup(&amp;nq-&gt;nq_tasklet, bnxt_qplib_service_nq);</div><div class='ctx'> 	else</div><div class='del'>-		tasklet_enable(&amp;nq-&gt;worker);</div><div class='add'>+		tasklet_enable(&amp;nq-&gt;nq_tasklet);</div><div class='ctx'> </div><div class='del'>-	snprintf(nq-&gt;name, sizeof(nq-&gt;name), "bnxt_qplib_nq-%d", nq_indx);</div><div class='del'>-	rc = request_irq(nq-&gt;vector, bnxt_qplib_nq_irq, 0, nq-&gt;name, nq);</div><div class='del'>-	if (rc)</div><div class='add'>+	nq-&gt;name = kasprintf(GFP_KERNEL, "bnxt_re-nq-%d@pci:%s",</div><div class='add'>+			     nq_indx, pci_name(res-&gt;pdev));</div><div class='add'>+	if (!nq-&gt;name)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	rc = request_irq(nq-&gt;msix_vec, bnxt_qplib_nq_irq, 0, nq-&gt;name, nq);</div><div class='add'>+	if (rc) {</div><div class='add'>+		kfree(nq-&gt;name);</div><div class='add'>+		nq-&gt;name = NULL;</div><div class='add'>+		tasklet_disable(&amp;nq-&gt;nq_tasklet);</div><div class='ctx'> 		return rc;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	cpumask_clear(&amp;nq-&gt;mask);</div><div class='ctx'> 	cpumask_set_cpu(nq_indx, &amp;nq-&gt;mask);</div><div class='del'>-	rc = irq_set_affinity_hint(nq-&gt;vector, &amp;nq-&gt;mask);</div><div class='add'>+	rc = irq_set_affinity_hint(nq-&gt;msix_vec, &amp;nq-&gt;mask);</div><div class='ctx'> 	if (rc) {</div><div class='ctx'> 		dev_warn(&amp;nq-&gt;pdev-&gt;dev,</div><div class='ctx'> 			 "set affinity failed; vector: %d nq_idx: %d\n",</div><div class='del'>-			 nq-&gt;vector, nq_indx);</div><div class='add'>+			 nq-&gt;msix_vec, nq_indx);</div><div class='ctx'> 	}</div><div class='ctx'> 	nq-&gt;requested = true;</div><div class='del'>-	bnxt_qplib_ring_nq_db_rearm(nq-&gt;bar_reg_iomem, nq-&gt;hwq.cons,</div><div class='del'>-				    nq-&gt;hwq.max_elements, nq-&gt;ring_id, gen_p5);</div><div class='add'>+	bnxt_qplib_ring_nq_db(&amp;nq-&gt;nq_db.dbinfo, res-&gt;cctx, true);</div><div class='ctx'> </div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int bnxt_qplib_map_nq_db(struct bnxt_qplib_nq *nq,  u32 reg_offt)</div><div class='add'>+{</div><div class='add'>+	resource_size_t reg_base;</div><div class='add'>+	struct bnxt_qplib_nq_db *nq_db;</div><div class='add'>+	struct pci_dev *pdev;</div><div class='add'>+</div><div class='add'>+	pdev = nq-&gt;pdev;</div><div class='add'>+	nq_db = &amp;nq-&gt;nq_db;</div><div class='add'>+</div><div class='add'>+	nq_db-&gt;dbinfo.flags = 0;</div><div class='add'>+	nq_db-&gt;reg.bar_id = NQ_CONS_PCI_BAR_REGION;</div><div class='add'>+	nq_db-&gt;reg.bar_base = pci_resource_start(pdev, nq_db-&gt;reg.bar_id);</div><div class='add'>+	if (!nq_db-&gt;reg.bar_base) {</div><div class='add'>+		dev_err(&amp;pdev-&gt;dev, "QPLIB: NQ BAR region %d resc start is 0!",</div><div class='add'>+			nq_db-&gt;reg.bar_id);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	reg_base = nq_db-&gt;reg.bar_base + reg_offt;</div><div class='add'>+	/* Unconditionally map 8 bytes to support 57500 series */</div><div class='add'>+	nq_db-&gt;reg.len = 8;</div><div class='add'>+	nq_db-&gt;reg.bar_reg = ioremap(reg_base, nq_db-&gt;reg.len);</div><div class='add'>+	if (!nq_db-&gt;reg.bar_reg) {</div><div class='add'>+		dev_err(&amp;pdev-&gt;dev, "QPLIB: NQ BAR region %d mapping failed",</div><div class='add'>+			nq_db-&gt;reg.bar_id);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	nq_db-&gt;dbinfo.db = nq_db-&gt;reg.bar_reg;</div><div class='add'>+	nq_db-&gt;dbinfo.hwq = &amp;nq-&gt;hwq;</div><div class='add'>+	nq_db-&gt;dbinfo.xid = nq-&gt;ring_id;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,</div><div class='ctx'> 			 int nq_idx, int msix_vector, int bar_reg_offset,</div><div class='del'>-			 int (*cqn_handler)(struct bnxt_qplib_nq *nq,</div><div class='del'>-					    struct bnxt_qplib_cq *),</div><div class='del'>-			 int (*srqn_handler)(struct bnxt_qplib_nq *nq,</div><div class='del'>-					     struct bnxt_qplib_srq *,</div><div class='del'>-					     u8 event))</div><div class='add'>+			 cqn_handler_t cqn_handler,</div><div class='add'>+			 srqn_handler_t srqn_handler)</div><div class='ctx'> {</div><div class='del'>-	resource_size_t nq_base;</div><div class='del'>-	int rc = -1;</div><div class='del'>-</div><div class='del'>-	if (cqn_handler)</div><div class='del'>-		nq-&gt;cqn_handler = cqn_handler;</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='del'>-	if (srqn_handler)</div><div class='del'>-		nq-&gt;srqn_handler = srqn_handler;</div><div class='add'>+	nq-&gt;pdev = pdev;</div><div class='add'>+	nq-&gt;cqn_handler = cqn_handler;</div><div class='add'>+	nq-&gt;srqn_handler = srqn_handler;</div><div class='add'>+	nq-&gt;load = 0;</div><div class='ctx'> </div><div class='ctx'> 	/* Have a task to schedule CQ notifiers in post send case */</div><div class='ctx'> 	nq-&gt;cqn_wq  = create_singlethread_workqueue("bnxt_qplib_nq");</div><div class='ctx'> 	if (!nq-&gt;cqn_wq)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	nq-&gt;bar_reg = NQ_CONS_PCI_BAR_REGION;</div><div class='del'>-	nq-&gt;bar_reg_off = bar_reg_offset;</div><div class='del'>-	nq_base = pci_resource_start(pdev, nq-&gt;bar_reg);</div><div class='del'>-	if (!nq_base) {</div><div class='del'>-		rc = -ENOMEM;</div><div class='del'>-		goto fail;</div><div class='del'>-	}</div><div class='del'>-	/* Unconditionally map 8 bytes to support 57500 series */</div><div class='del'>-	nq-&gt;bar_reg_iomem = ioremap_nocache(nq_base + nq-&gt;bar_reg_off, 8);</div><div class='del'>-	if (!nq-&gt;bar_reg_iomem) {</div><div class='del'>-		rc = -ENOMEM;</div><div class='add'>+	rc = bnxt_qplib_map_nq_db(nq, bar_reg_offset);</div><div class='add'>+	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='ctx'> 	rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true);</div><div class='ctx'> 	if (rc) {</div><div class='hunk'>@@ -464,97 +583,86 @@ fail:</div><div class='ctx'> void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq)</div><div class='ctx'> {</div><div class='ctx'> 	if (nq-&gt;hwq.max_elements) {</div><div class='del'>-		bnxt_qplib_free_hwq(nq-&gt;pdev, &amp;nq-&gt;hwq);</div><div class='add'>+		bnxt_qplib_free_hwq(nq-&gt;res, &amp;nq-&gt;hwq);</div><div class='ctx'> 		nq-&gt;hwq.max_elements = 0;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq)</div><div class='add'>+int bnxt_qplib_alloc_nq(struct bnxt_qplib_res *res, struct bnxt_qplib_nq *nq)</div><div class='ctx'> {</div><div class='del'>-	u8 hwq_type;</div><div class='add'>+	struct bnxt_qplib_hwq_attr hwq_attr = {};</div><div class='add'>+	struct bnxt_qplib_sg_info sginfo = {};</div><div class='ctx'> </div><div class='del'>-	nq-&gt;pdev = pdev;</div><div class='add'>+	nq-&gt;pdev = res-&gt;pdev;</div><div class='add'>+	nq-&gt;res = res;</div><div class='ctx'> 	if (!nq-&gt;hwq.max_elements ||</div><div class='ctx'> 	    nq-&gt;hwq.max_elements &gt; BNXT_QPLIB_NQE_MAX_CNT)</div><div class='ctx'> 		nq-&gt;hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;</div><div class='del'>-	hwq_type = bnxt_qplib_get_hwq_type(nq-&gt;res);</div><div class='del'>-	if (bnxt_qplib_alloc_init_hwq(nq-&gt;pdev, &amp;nq-&gt;hwq, NULL,</div><div class='del'>-				      &amp;nq-&gt;hwq.max_elements,</div><div class='del'>-				      BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0,</div><div class='del'>-				      PAGE_SIZE, hwq_type))</div><div class='del'>-		return -ENOMEM;</div><div class='ctx'> </div><div class='add'>+	sginfo.pgsize = PAGE_SIZE;</div><div class='add'>+	sginfo.pgshft = PAGE_SHIFT;</div><div class='add'>+	hwq_attr.res = res;</div><div class='add'>+	hwq_attr.sginfo = &amp;sginfo;</div><div class='add'>+	hwq_attr.depth = nq-&gt;hwq.max_elements;</div><div class='add'>+	hwq_attr.stride = sizeof(struct nq_base);</div><div class='add'>+	hwq_attr.type = bnxt_qplib_get_hwq_type(nq-&gt;res);</div><div class='add'>+	if (bnxt_qplib_alloc_init_hwq(&amp;nq-&gt;hwq, &amp;hwq_attr)) {</div><div class='add'>+		dev_err(&amp;nq-&gt;pdev-&gt;dev, "FP NQ allocation failed");</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='ctx'> 	nq-&gt;budget = 8;</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* SRQ */</div><div class='del'>-static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type)</div><div class='del'>-{</div><div class='del'>-	struct bnxt_qplib_hwq *srq_hwq = &amp;srq-&gt;hwq;</div><div class='del'>-	void __iomem *db;</div><div class='del'>-	u32 sw_prod;</div><div class='del'>-	u64 val = 0;</div><div class='del'>-</div><div class='del'>-	/* Ring DB */</div><div class='del'>-	sw_prod = (arm_type == DBC_DBC_TYPE_SRQ_ARM) ?</div><div class='del'>-		   srq-&gt;threshold : HWQ_CMP(srq_hwq-&gt;prod, srq_hwq);</div><div class='del'>-	db = (arm_type == DBC_DBC_TYPE_SRQ_ARMENA) ? srq-&gt;dbr_base :</div><div class='del'>-						     srq-&gt;dpi-&gt;dbr;</div><div class='del'>-	val = ((srq-&gt;id &lt;&lt; DBC_DBC_XID_SFT) &amp; DBC_DBC_XID_MASK) | arm_type;</div><div class='del'>-	val &lt;&lt;= 32;</div><div class='del'>-	val |= (sw_prod &lt;&lt; DBC_DBC_INDEX_SFT) &amp; DBC_DBC_INDEX_MASK;</div><div class='del'>-	writeq(val, db);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> void bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,</div><div class='ctx'> 			   struct bnxt_qplib_srq *srq)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct cmdq_destroy_srq req;</div><div class='del'>-	struct creq_destroy_srq_resp resp;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='add'>+	struct creq_destroy_srq_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_destroy_srq req = {};</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, DESTROY_SRQ, cmd_flags);</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_DESTROY_SRQ,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='ctx'> 	/* Configure the request */</div><div class='ctx'> 	req.srq_cid = cpu_to_le32(srq-&gt;id);</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (struct cmdq_base *)&amp;req,</div><div class='del'>-					  (struct creq_base *)&amp;resp, NULL, 0);</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req), sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	kfree(srq-&gt;swq);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		return;</div><div class='del'>-	bnxt_qplib_free_hwq(res-&gt;pdev, &amp;srq-&gt;hwq);</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;srq-&gt;hwq);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,</div><div class='ctx'> 			  struct bnxt_qplib_srq *srq)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct cmdq_create_srq req;</div><div class='del'>-	struct creq_create_srq_resp resp;</div><div class='add'>+	struct bnxt_qplib_hwq_attr hwq_attr = {};</div><div class='add'>+	struct creq_create_srq_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_create_srq req = {};</div><div class='ctx'> 	struct bnxt_qplib_pbl *pbl;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='add'>+	u16 pg_sz_lvl;</div><div class='ctx'> 	int rc, idx;</div><div class='ctx'> </div><div class='del'>-	srq-&gt;hwq.max_elements = srq-&gt;max_wqe;</div><div class='del'>-	rc = bnxt_qplib_alloc_init_hwq(res-&gt;pdev, &amp;srq-&gt;hwq, &amp;srq-&gt;sg_info,</div><div class='del'>-				       &amp;srq-&gt;hwq.max_elements,</div><div class='del'>-				       BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,</div><div class='del'>-				       PAGE_SIZE, HWQ_TYPE_QUEUE);</div><div class='add'>+	hwq_attr.res = res;</div><div class='add'>+	hwq_attr.sginfo = &amp;srq-&gt;sg_info;</div><div class='add'>+	hwq_attr.depth = srq-&gt;max_wqe;</div><div class='add'>+	hwq_attr.stride = srq-&gt;wqe_size;</div><div class='add'>+	hwq_attr.type = HWQ_TYPE_QUEUE;</div><div class='add'>+	rc = bnxt_qplib_alloc_init_hwq(&amp;srq-&gt;hwq, &amp;hwq_attr);</div><div class='ctx'> 	if (rc)</div><div class='del'>-		goto exit;</div><div class='del'>-</div><div class='del'>-	srq-&gt;swq = kcalloc(srq-&gt;hwq.max_elements, sizeof(*srq-&gt;swq),</div><div class='del'>-			   GFP_KERNEL);</div><div class='del'>-	if (!srq-&gt;swq) {</div><div class='del'>-		rc = -ENOMEM;</div><div class='del'>-		goto fail;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	RCFW_CMD_PREP(req, CREATE_SRQ, cmd_flags);</div><div class='add'>+		return rc;</div><div class='add'>+	srq-&gt;dbinfo.flags = 0;</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_CREATE_SRQ,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='ctx'> 	/* Configure the request */</div><div class='ctx'> 	req.dpi = cpu_to_le32(srq-&gt;dpi-&gt;dpi);</div><div class='hunk'>@@ -562,97 +670,82 @@ int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,</div><div class='ctx'> </div><div class='ctx'> 	req.srq_size = cpu_to_le16((u16)srq-&gt;hwq.max_elements);</div><div class='ctx'> 	pbl = &amp;srq-&gt;hwq.pbl[PBL_LVL_0];</div><div class='del'>-	req.pg_size_lvl = cpu_to_le16((((u16)srq-&gt;hwq.level &amp;</div><div class='del'>-				      CMDQ_CREATE_SRQ_LVL_MASK) &lt;&lt;</div><div class='del'>-				      CMDQ_CREATE_SRQ_LVL_SFT) |</div><div class='del'>-				      (pbl-&gt;pg_size == ROCE_PG_SIZE_4K ?</div><div class='del'>-				       CMDQ_CREATE_SRQ_PG_SIZE_PG_4K :</div><div class='del'>-				       pbl-&gt;pg_size == ROCE_PG_SIZE_8K ?</div><div class='del'>-				       CMDQ_CREATE_SRQ_PG_SIZE_PG_8K :</div><div class='del'>-				       pbl-&gt;pg_size == ROCE_PG_SIZE_64K ?</div><div class='del'>-				       CMDQ_CREATE_SRQ_PG_SIZE_PG_64K :</div><div class='del'>-				       pbl-&gt;pg_size == ROCE_PG_SIZE_2M ?</div><div class='del'>-				       CMDQ_CREATE_SRQ_PG_SIZE_PG_2M :</div><div class='del'>-				       pbl-&gt;pg_size == ROCE_PG_SIZE_8M ?</div><div class='del'>-				       CMDQ_CREATE_SRQ_PG_SIZE_PG_8M :</div><div class='del'>-				       pbl-&gt;pg_size == ROCE_PG_SIZE_1G ?</div><div class='del'>-				       CMDQ_CREATE_SRQ_PG_SIZE_PG_1G :</div><div class='del'>-				       CMDQ_CREATE_SRQ_PG_SIZE_PG_4K));</div><div class='add'>+	pg_sz_lvl = ((u16)bnxt_qplib_base_pg_size(&amp;srq-&gt;hwq) &lt;&lt;</div><div class='add'>+		     CMDQ_CREATE_SRQ_PG_SIZE_SFT);</div><div class='add'>+	pg_sz_lvl |= (srq-&gt;hwq.level &amp; CMDQ_CREATE_SRQ_LVL_MASK) &lt;&lt;</div><div class='add'>+		      CMDQ_CREATE_SRQ_LVL_SFT;</div><div class='add'>+	req.pg_size_lvl = cpu_to_le16(pg_sz_lvl);</div><div class='ctx'> 	req.pbl = cpu_to_le64(pbl-&gt;pg_map_arr[0]);</div><div class='ctx'> 	req.pd_id = cpu_to_le32(srq-&gt;pd-&gt;id);</div><div class='ctx'> 	req.eventq_id = cpu_to_le16(srq-&gt;eventq_hw_ring_id);</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req,</div><div class='del'>-					  (void *)&amp;resp, NULL, 0);</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req), sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_init(&amp;srq-&gt;lock);</div><div class='ctx'> 	srq-&gt;start_idx = 0;</div><div class='ctx'> 	srq-&gt;last_idx = srq-&gt;hwq.max_elements - 1;</div><div class='del'>-	for (idx = 0; idx &lt; srq-&gt;hwq.max_elements; idx++)</div><div class='del'>-		srq-&gt;swq[idx].next_idx = idx + 1;</div><div class='del'>-	srq-&gt;swq[srq-&gt;last_idx].next_idx = -1;</div><div class='add'>+	if (!srq-&gt;hwq.is_user) {</div><div class='add'>+		srq-&gt;swq = kcalloc(srq-&gt;hwq.max_elements, sizeof(*srq-&gt;swq),</div><div class='add'>+				   GFP_KERNEL);</div><div class='add'>+		if (!srq-&gt;swq) {</div><div class='add'>+			rc = -ENOMEM;</div><div class='add'>+			goto fail;</div><div class='add'>+		}</div><div class='add'>+		for (idx = 0; idx &lt; srq-&gt;hwq.max_elements; idx++)</div><div class='add'>+			srq-&gt;swq[idx].next_idx = idx + 1;</div><div class='add'>+		srq-&gt;swq[srq-&gt;last_idx].next_idx = -1;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	srq-&gt;id = le32_to_cpu(resp.xid);</div><div class='del'>-	srq-&gt;dbr_base = res-&gt;dpi_tbl.dbr_bar_reg_iomem;</div><div class='del'>-	if (srq-&gt;threshold)</div><div class='del'>-		bnxt_qplib_arm_srq(srq, DBC_DBC_TYPE_SRQ_ARMENA);</div><div class='del'>-	srq-&gt;arm_req = false;</div><div class='add'>+	srq-&gt;dbinfo.hwq = &amp;srq-&gt;hwq;</div><div class='add'>+	srq-&gt;dbinfo.xid = srq-&gt;id;</div><div class='add'>+	srq-&gt;dbinfo.db = srq-&gt;dpi-&gt;dbr;</div><div class='add'>+	srq-&gt;dbinfo.max_slot = 1;</div><div class='add'>+	srq-&gt;dbinfo.priv_db = res-&gt;dpi_tbl.priv_db;</div><div class='add'>+	bnxt_qplib_armen_db(&amp;srq-&gt;dbinfo, DBC_DBC_TYPE_SRQ_ARMENA);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> fail:</div><div class='del'>-	bnxt_qplib_free_hwq(res-&gt;pdev, &amp;srq-&gt;hwq);</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;srq-&gt;hwq);</div><div class='ctx'> 	kfree(srq-&gt;swq);</div><div class='del'>-exit:</div><div class='del'>-	return rc;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,</div><div class='del'>-			  struct bnxt_qplib_srq *srq)</div><div class='del'>-{</div><div class='del'>-	struct bnxt_qplib_hwq *srq_hwq = &amp;srq-&gt;hwq;</div><div class='del'>-	u32 sw_prod, sw_cons, count = 0;</div><div class='ctx'> </div><div class='del'>-	sw_prod = HWQ_CMP(srq_hwq-&gt;prod, srq_hwq);</div><div class='del'>-	sw_cons = HWQ_CMP(srq_hwq-&gt;cons, srq_hwq);</div><div class='del'>-</div><div class='del'>-	count = sw_prod &gt; sw_cons ? sw_prod - sw_cons :</div><div class='del'>-				    srq_hwq-&gt;max_elements - sw_cons + sw_prod;</div><div class='del'>-	if (count &gt; srq-&gt;threshold) {</div><div class='del'>-		srq-&gt;arm_req = false;</div><div class='del'>-		bnxt_qplib_arm_srq(srq, DBC_DBC_TYPE_SRQ_ARM);</div><div class='del'>-	} else {</div><div class='del'>-		/* Deferred arming */</div><div class='del'>-		srq-&gt;arm_req = true;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,</div><div class='ctx'> 			 struct bnxt_qplib_srq *srq)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct cmdq_query_srq req;</div><div class='del'>-	struct creq_query_srq_resp resp;</div><div class='del'>-	struct bnxt_qplib_rcfw_sbuf *sbuf;</div><div class='add'>+	struct creq_query_srq_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct bnxt_qplib_rcfw_sbuf sbuf;</div><div class='ctx'> 	struct creq_query_srq_resp_sb *sb;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='del'>-	int rc = 0;</div><div class='add'>+	struct cmdq_query_srq req = {};</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, QUERY_SRQ, cmd_flags);</div><div class='del'>-	req.srq_cid = cpu_to_le32(srq-&gt;id);</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_QUERY_SRQ,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='ctx'> 	/* Configure the request */</div><div class='del'>-	sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));</div><div class='del'>-	if (!sbuf)</div><div class='add'>+	sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS);</div><div class='add'>+	sbuf.sb = dma_alloc_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf.size,</div><div class='add'>+				     &amp;sbuf.dma_addr, GFP_KERNEL);</div><div class='add'>+	if (!sbuf.sb)</div><div class='ctx'> 		return -ENOMEM;</div><div class='del'>-	sb = sbuf-&gt;sb;</div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req, (void *)&amp;resp,</div><div class='del'>-					  (void *)sbuf, 0);</div><div class='del'>-	srq-&gt;threshold = le16_to_cpu(sb-&gt;srq_limit);</div><div class='del'>-	bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);</div><div class='add'>+	req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;</div><div class='add'>+	req.srq_cid = cpu_to_le32(srq-&gt;id);</div><div class='add'>+	sb = sbuf.sb;</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, &amp;sbuf, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='add'>+	if (!rc)</div><div class='add'>+		srq-&gt;threshold = le16_to_cpu(sb-&gt;srq_limit);</div><div class='add'>+	dma_free_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf.size,</div><div class='add'>+			  sbuf.sb, sbuf.dma_addr);</div><div class='ctx'> </div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='hunk'>@@ -661,27 +754,23 @@ int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,</div><div class='ctx'> 			     struct bnxt_qplib_swqe *wqe)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_hwq *srq_hwq = &amp;srq-&gt;hwq;</div><div class='del'>-	struct rq_wqe *srqe, **srqe_ptr;</div><div class='add'>+	struct rq_wqe *srqe;</div><div class='ctx'> 	struct sq_sge *hw_sge;</div><div class='del'>-	u32 sw_prod, sw_cons, count = 0;</div><div class='del'>-	int i, rc = 0, next;</div><div class='add'>+	int i, next;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock(&amp;srq_hwq-&gt;lock);</div><div class='ctx'> 	if (srq-&gt;start_idx == srq-&gt;last_idx) {</div><div class='ctx'> 		dev_err(&amp;srq_hwq-&gt;pdev-&gt;dev,</div><div class='ctx'> 			"FP: SRQ (0x%x) is full!\n", srq-&gt;id);</div><div class='del'>-		rc = -EINVAL;</div><div class='ctx'> 		spin_unlock(&amp;srq_hwq-&gt;lock);</div><div class='del'>-		goto done;</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> 	next = srq-&gt;start_idx;</div><div class='ctx'> 	srq-&gt;start_idx = srq-&gt;swq[next].next_idx;</div><div class='ctx'> 	spin_unlock(&amp;srq_hwq-&gt;lock);</div><div class='ctx'> </div><div class='del'>-	sw_prod = HWQ_CMP(srq_hwq-&gt;prod, srq_hwq);</div><div class='del'>-	srqe_ptr = (struct rq_wqe **)srq_hwq-&gt;pbl_ptr;</div><div class='del'>-	srqe = &amp;srqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];</div><div class='del'>-	memset(srqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);</div><div class='add'>+	srqe = bnxt_qplib_get_qe(srq_hwq, srq_hwq-&gt;prod, NULL);</div><div class='add'>+	memset(srqe, 0, srq-&gt;wqe_size);</div><div class='ctx'> 	/* Calculate wqe_size16 and data_len */</div><div class='ctx'> 	for (i = 0, hw_sge = (struct sq_sge *)srqe-&gt;data;</div><div class='ctx'> 	     i &lt; wqe-&gt;num_sge; i++, hw_sge++) {</div><div class='hunk'>@@ -696,185 +785,199 @@ int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,</div><div class='ctx'> 	srqe-&gt;wr_id[0] = cpu_to_le32((u32)next);</div><div class='ctx'> 	srq-&gt;swq[next].wr_id = wqe-&gt;wr_id;</div><div class='ctx'> </div><div class='del'>-	srq_hwq-&gt;prod++;</div><div class='add'>+	bnxt_qplib_hwq_incr_prod(&amp;srq-&gt;dbinfo, srq_hwq, srq-&gt;dbinfo.max_slot);</div><div class='ctx'> </div><div class='del'>-	spin_lock(&amp;srq_hwq-&gt;lock);</div><div class='del'>-	sw_prod = HWQ_CMP(srq_hwq-&gt;prod, srq_hwq);</div><div class='del'>-	/* retaining srq_hwq-&gt;cons for this logic</div><div class='del'>-	 * actually the lock is only required to</div><div class='del'>-	 * read srq_hwq-&gt;cons.</div><div class='del'>-	 */</div><div class='del'>-	sw_cons = HWQ_CMP(srq_hwq-&gt;cons, srq_hwq);</div><div class='del'>-	count = sw_prod &gt; sw_cons ? sw_prod - sw_cons :</div><div class='del'>-				    srq_hwq-&gt;max_elements - sw_cons + sw_prod;</div><div class='del'>-	spin_unlock(&amp;srq_hwq-&gt;lock);</div><div class='ctx'> 	/* Ring DB */</div><div class='del'>-	bnxt_qplib_arm_srq(srq, DBC_DBC_TYPE_SRQ);</div><div class='del'>-	if (srq-&gt;arm_req == true &amp;&amp; count &gt; srq-&gt;threshold) {</div><div class='del'>-		srq-&gt;arm_req = false;</div><div class='del'>-		bnxt_qplib_arm_srq(srq, DBC_DBC_TYPE_SRQ_ARM);</div><div class='del'>-	}</div><div class='del'>-done:</div><div class='del'>-	return rc;</div><div class='add'>+	bnxt_qplib_ring_prod_db(&amp;srq-&gt;dbinfo, DBC_DBC_TYPE_SRQ);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* QP */</div><div class='add'>+</div><div class='add'>+static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que)</div><div class='add'>+{</div><div class='add'>+	int indx;</div><div class='add'>+</div><div class='add'>+	que-&gt;swq = kcalloc(que-&gt;max_sw_wqe, sizeof(*que-&gt;swq), GFP_KERNEL);</div><div class='add'>+	if (!que-&gt;swq)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	que-&gt;swq_start = 0;</div><div class='add'>+	que-&gt;swq_last = que-&gt;max_sw_wqe - 1;</div><div class='add'>+	for (indx = 0; indx &lt; que-&gt;max_sw_wqe; indx++)</div><div class='add'>+		que-&gt;swq[indx].next_idx = indx + 1;</div><div class='add'>+	que-&gt;swq[que-&gt;swq_last].next_idx = 0; /* Make it circular */</div><div class='add'>+	que-&gt;swq_last = 0;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)</div><div class='ctx'> {</div><div class='add'>+	struct bnxt_qplib_hwq_attr hwq_attr = {};</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct cmdq_create_qp1 req;</div><div class='del'>-	struct creq_create_qp1_resp resp;</div><div class='del'>-	struct bnxt_qplib_pbl *pbl;</div><div class='add'>+	struct creq_create_qp1_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='ctx'> 	struct bnxt_qplib_q *sq = &amp;qp-&gt;sq;</div><div class='ctx'> 	struct bnxt_qplib_q *rq = &amp;qp-&gt;rq;</div><div class='del'>-	int rc;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='add'>+	struct cmdq_create_qp1 req = {};</div><div class='add'>+	struct bnxt_qplib_pbl *pbl;</div><div class='ctx'> 	u32 qp_flags = 0;</div><div class='add'>+	u8 pg_sz_lvl;</div><div class='add'>+	u32 tbl_indx;</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags);</div><div class='del'>-</div><div class='add'>+	sq-&gt;dbinfo.flags = 0;</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_CREATE_QP1,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> 	/* General */</div><div class='ctx'> 	req.type = qp-&gt;type;</div><div class='ctx'> 	req.dpi = cpu_to_le32(qp-&gt;dpi-&gt;dpi);</div><div class='ctx'> 	req.qp_handle = cpu_to_le64(qp-&gt;qp_handle);</div><div class='ctx'> </div><div class='ctx'> 	/* SQ */</div><div class='del'>-	sq-&gt;hwq.max_elements = sq-&gt;max_wqe;</div><div class='del'>-	rc = bnxt_qplib_alloc_init_hwq(res-&gt;pdev, &amp;sq-&gt;hwq, NULL,</div><div class='del'>-				       &amp;sq-&gt;hwq.max_elements,</div><div class='del'>-				       BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0,</div><div class='del'>-				       PAGE_SIZE, HWQ_TYPE_QUEUE);</div><div class='add'>+	hwq_attr.res = res;</div><div class='add'>+	hwq_attr.sginfo = &amp;sq-&gt;sg_info;</div><div class='add'>+	hwq_attr.stride = sizeof(struct sq_sge);</div><div class='add'>+	hwq_attr.depth = bnxt_qplib_get_depth(sq, qp-&gt;wqe_mode, false);</div><div class='add'>+	hwq_attr.type = HWQ_TYPE_QUEUE;</div><div class='add'>+	rc = bnxt_qplib_alloc_init_hwq(&amp;sq-&gt;hwq, &amp;hwq_attr);</div><div class='ctx'> 	if (rc)</div><div class='del'>-		goto exit;</div><div class='add'>+		return rc;</div><div class='ctx'> </div><div class='del'>-	sq-&gt;swq = kcalloc(sq-&gt;hwq.max_elements, sizeof(*sq-&gt;swq), GFP_KERNEL);</div><div class='del'>-	if (!sq-&gt;swq) {</div><div class='del'>-		rc = -ENOMEM;</div><div class='add'>+	rc = bnxt_qplib_alloc_init_swq(sq);</div><div class='add'>+	if (rc)</div><div class='ctx'> 		goto fail_sq;</div><div class='del'>-	}</div><div class='add'>+</div><div class='add'>+	req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp-&gt;wqe_mode));</div><div class='ctx'> 	pbl = &amp;sq-&gt;hwq.pbl[PBL_LVL_0];</div><div class='ctx'> 	req.sq_pbl = cpu_to_le64(pbl-&gt;pg_map_arr[0]);</div><div class='del'>-	req.sq_pg_size_sq_lvl =</div><div class='del'>-		((sq-&gt;hwq.level &amp; CMDQ_CREATE_QP1_SQ_LVL_MASK)</div><div class='del'>-				&lt;&lt;  CMDQ_CREATE_QP1_SQ_LVL_SFT) |</div><div class='del'>-		(pbl-&gt;pg_size == ROCE_PG_SIZE_4K ?</div><div class='del'>-				CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K :</div><div class='del'>-		 pbl-&gt;pg_size == ROCE_PG_SIZE_8K ?</div><div class='del'>-				CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K :</div><div class='del'>-		 pbl-&gt;pg_size == ROCE_PG_SIZE_64K ?</div><div class='del'>-				CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K :</div><div class='del'>-		 pbl-&gt;pg_size == ROCE_PG_SIZE_2M ?</div><div class='del'>-				CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M :</div><div class='del'>-		 pbl-&gt;pg_size == ROCE_PG_SIZE_8M ?</div><div class='del'>-				CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M :</div><div class='del'>-		 pbl-&gt;pg_size == ROCE_PG_SIZE_1G ?</div><div class='del'>-				CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G :</div><div class='del'>-		 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K);</div><div class='del'>-</div><div class='del'>-	if (qp-&gt;scq)</div><div class='del'>-		req.scq_cid = cpu_to_le32(qp-&gt;scq-&gt;id);</div><div class='del'>-</div><div class='del'>-	qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;</div><div class='add'>+	pg_sz_lvl = (bnxt_qplib_base_pg_size(&amp;sq-&gt;hwq) &lt;&lt;</div><div class='add'>+		     CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT);</div><div class='add'>+	pg_sz_lvl |= (sq-&gt;hwq.level &amp; CMDQ_CREATE_QP1_SQ_LVL_MASK);</div><div class='add'>+	req.sq_pg_size_sq_lvl = pg_sz_lvl;</div><div class='add'>+	req.sq_fwo_sq_sge =</div><div class='add'>+		cpu_to_le16((sq-&gt;max_sge &amp; CMDQ_CREATE_QP1_SQ_SGE_MASK) &lt;&lt;</div><div class='add'>+			     CMDQ_CREATE_QP1_SQ_SGE_SFT);</div><div class='add'>+	req.scq_cid = cpu_to_le32(qp-&gt;scq-&gt;id);</div><div class='ctx'> </div><div class='ctx'> 	/* RQ */</div><div class='ctx'> 	if (rq-&gt;max_wqe) {</div><div class='del'>-		rq-&gt;hwq.max_elements = qp-&gt;rq.max_wqe;</div><div class='del'>-		rc = bnxt_qplib_alloc_init_hwq(res-&gt;pdev, &amp;rq-&gt;hwq, NULL,</div><div class='del'>-					       &amp;rq-&gt;hwq.max_elements,</div><div class='del'>-					       BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,</div><div class='del'>-					       PAGE_SIZE, HWQ_TYPE_QUEUE);</div><div class='add'>+		rq-&gt;dbinfo.flags = 0;</div><div class='add'>+		hwq_attr.res = res;</div><div class='add'>+		hwq_attr.sginfo = &amp;rq-&gt;sg_info;</div><div class='add'>+		hwq_attr.stride = sizeof(struct sq_sge);</div><div class='add'>+		hwq_attr.depth = bnxt_qplib_get_depth(rq, qp-&gt;wqe_mode, false);</div><div class='add'>+		hwq_attr.type = HWQ_TYPE_QUEUE;</div><div class='add'>+		rc = bnxt_qplib_alloc_init_hwq(&amp;rq-&gt;hwq, &amp;hwq_attr);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto sq_swq;</div><div class='add'>+		rc = bnxt_qplib_alloc_init_swq(rq);</div><div class='ctx'> 		if (rc)</div><div class='del'>-			goto fail_sq;</div><div class='del'>-</div><div class='del'>-		rq-&gt;swq = kcalloc(rq-&gt;hwq.max_elements, sizeof(*rq-&gt;swq),</div><div class='del'>-				  GFP_KERNEL);</div><div class='del'>-		if (!rq-&gt;swq) {</div><div class='del'>-			rc = -ENOMEM;</div><div class='ctx'> 			goto fail_rq;</div><div class='del'>-		}</div><div class='add'>+		req.rq_size = cpu_to_le32(rq-&gt;max_wqe);</div><div class='ctx'> 		pbl = &amp;rq-&gt;hwq.pbl[PBL_LVL_0];</div><div class='ctx'> 		req.rq_pbl = cpu_to_le64(pbl-&gt;pg_map_arr[0]);</div><div class='del'>-		req.rq_pg_size_rq_lvl =</div><div class='del'>-			((rq-&gt;hwq.level &amp; CMDQ_CREATE_QP1_RQ_LVL_MASK) &lt;&lt;</div><div class='del'>-			 CMDQ_CREATE_QP1_RQ_LVL_SFT) |</div><div class='del'>-				(pbl-&gt;pg_size == ROCE_PG_SIZE_4K ?</div><div class='del'>-					CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K :</div><div class='del'>-				 pbl-&gt;pg_size == ROCE_PG_SIZE_8K ?</div><div class='del'>-					CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K :</div><div class='del'>-				 pbl-&gt;pg_size == ROCE_PG_SIZE_64K ?</div><div class='del'>-					CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K :</div><div class='del'>-				 pbl-&gt;pg_size == ROCE_PG_SIZE_2M ?</div><div class='del'>-					CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M :</div><div class='del'>-				 pbl-&gt;pg_size == ROCE_PG_SIZE_8M ?</div><div class='del'>-					CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M :</div><div class='del'>-				 pbl-&gt;pg_size == ROCE_PG_SIZE_1G ?</div><div class='del'>-					CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G :</div><div class='del'>-				 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K);</div><div class='del'>-		if (qp-&gt;rcq)</div><div class='del'>-			req.rcq_cid = cpu_to_le32(qp-&gt;rcq-&gt;id);</div><div class='del'>-	}</div><div class='del'>-</div><div class='add'>+		pg_sz_lvl = (bnxt_qplib_base_pg_size(&amp;rq-&gt;hwq) &lt;&lt;</div><div class='add'>+			     CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT);</div><div class='add'>+		pg_sz_lvl |= (rq-&gt;hwq.level &amp; CMDQ_CREATE_QP1_RQ_LVL_MASK);</div><div class='add'>+		req.rq_pg_size_rq_lvl = pg_sz_lvl;</div><div class='add'>+		req.rq_fwo_rq_sge =</div><div class='add'>+			cpu_to_le16((rq-&gt;max_sge &amp;</div><div class='add'>+				     CMDQ_CREATE_QP1_RQ_SGE_MASK) &lt;&lt;</div><div class='add'>+				    CMDQ_CREATE_QP1_RQ_SGE_SFT);</div><div class='add'>+	}</div><div class='add'>+	req.rcq_cid = cpu_to_le32(qp-&gt;rcq-&gt;id);</div><div class='ctx'> 	/* Header buffer - allow hdr_buf pass in */</div><div class='ctx'> 	rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);</div><div class='ctx'> 	if (rc) {</div><div class='ctx'> 		rc = -ENOMEM;</div><div class='del'>-		goto fail;</div><div class='add'>+		goto rq_rwq;</div><div class='ctx'> 	}</div><div class='add'>+	qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;</div><div class='ctx'> 	req.qp_flags = cpu_to_le32(qp_flags);</div><div class='del'>-	req.sq_size = cpu_to_le32(sq-&gt;hwq.max_elements);</div><div class='del'>-	req.rq_size = cpu_to_le32(rq-&gt;hwq.max_elements);</div><div class='del'>-</div><div class='del'>-	req.sq_fwo_sq_sge =</div><div class='del'>-		cpu_to_le16((sq-&gt;max_sge &amp; CMDQ_CREATE_QP1_SQ_SGE_MASK) &lt;&lt;</div><div class='del'>-			    CMDQ_CREATE_QP1_SQ_SGE_SFT);</div><div class='del'>-	req.rq_fwo_rq_sge =</div><div class='del'>-		cpu_to_le16((rq-&gt;max_sge &amp; CMDQ_CREATE_QP1_RQ_SGE_MASK) &lt;&lt;</div><div class='del'>-			    CMDQ_CREATE_QP1_RQ_SGE_SFT);</div><div class='del'>-</div><div class='ctx'> 	req.pd_id = cpu_to_le32(qp-&gt;pd-&gt;id);</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req,</div><div class='del'>-					  (void *)&amp;resp, NULL, 0);</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req), sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='ctx'> 	qp-&gt;id = le32_to_cpu(resp.xid);</div><div class='ctx'> 	qp-&gt;cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;</div><div class='del'>-	rcfw-&gt;qp_tbl[qp-&gt;id].qp_id = qp-&gt;id;</div><div class='del'>-	rcfw-&gt;qp_tbl[qp-&gt;id].qp_handle = (void *)qp;</div><div class='add'>+	qp-&gt;cctx = res-&gt;cctx;</div><div class='add'>+	sq-&gt;dbinfo.hwq = &amp;sq-&gt;hwq;</div><div class='add'>+	sq-&gt;dbinfo.xid = qp-&gt;id;</div><div class='add'>+	sq-&gt;dbinfo.db = qp-&gt;dpi-&gt;dbr;</div><div class='add'>+	sq-&gt;dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp-&gt;wqe_mode);</div><div class='add'>+	if (rq-&gt;max_wqe) {</div><div class='add'>+		rq-&gt;dbinfo.hwq = &amp;rq-&gt;hwq;</div><div class='add'>+		rq-&gt;dbinfo.xid = qp-&gt;id;</div><div class='add'>+		rq-&gt;dbinfo.db = qp-&gt;dpi-&gt;dbr;</div><div class='add'>+		rq-&gt;dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq-&gt;wqe_size);</div><div class='add'>+	}</div><div class='add'>+	tbl_indx = map_qp_id_to_tbl_indx(qp-&gt;id, rcfw);</div><div class='add'>+	rcfw-&gt;qp_tbl[tbl_indx].qp_id = qp-&gt;id;</div><div class='add'>+	rcfw-&gt;qp_tbl[tbl_indx].qp_handle = (void *)qp;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> fail:</div><div class='ctx'> 	bnxt_qplib_free_qp_hdr_buf(res, qp);</div><div class='del'>-fail_rq:</div><div class='del'>-	bnxt_qplib_free_hwq(res-&gt;pdev, &amp;rq-&gt;hwq);</div><div class='add'>+rq_rwq:</div><div class='ctx'> 	kfree(rq-&gt;swq);</div><div class='del'>-fail_sq:</div><div class='del'>-	bnxt_qplib_free_hwq(res-&gt;pdev, &amp;sq-&gt;hwq);</div><div class='add'>+fail_rq:</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;rq-&gt;hwq);</div><div class='add'>+sq_swq:</div><div class='ctx'> 	kfree(sq-&gt;swq);</div><div class='del'>-exit:</div><div class='add'>+fail_sq:</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;sq-&gt;hwq);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_hwq *hwq;</div><div class='add'>+	struct bnxt_qplib_q *sq;</div><div class='add'>+	u64 fpsne, psn_pg;</div><div class='add'>+	u16 indx_pad = 0;</div><div class='add'>+</div><div class='add'>+	sq = &amp;qp-&gt;sq;</div><div class='add'>+	hwq = &amp;sq-&gt;hwq;</div><div class='add'>+	/* First psn entry */</div><div class='add'>+	fpsne = (u64)bnxt_qplib_get_qe(hwq, hwq-&gt;depth, &amp;psn_pg);</div><div class='add'>+	if (!IS_ALIGNED(fpsne, PAGE_SIZE))</div><div class='add'>+		indx_pad = (fpsne &amp; ~PAGE_MASK) / size;</div><div class='add'>+	hwq-&gt;pad_pgofft = indx_pad;</div><div class='add'>+	hwq-&gt;pad_pg = (u64 *)psn_pg;</div><div class='add'>+	hwq-&gt;pad_stride = size;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	unsigned long int psn_search, poff = 0;</div><div class='del'>-	struct sq_psn_search **psn_search_ptr;</div><div class='add'>+	struct bnxt_qplib_hwq_attr hwq_attr = {};</div><div class='add'>+	struct bnxt_qplib_sg_info sginfo = {};</div><div class='add'>+	struct creq_create_qp_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='ctx'> 	struct bnxt_qplib_q *sq = &amp;qp-&gt;sq;</div><div class='ctx'> 	struct bnxt_qplib_q *rq = &amp;qp-&gt;rq;</div><div class='del'>-	int i, rc, req_size, psn_sz = 0;</div><div class='del'>-	struct sq_send **hw_sq_send_ptr;</div><div class='del'>-	struct creq_create_qp_resp resp;</div><div class='add'>+	struct cmdq_create_qp req = {};</div><div class='add'>+	int rc, req_size, psn_sz = 0;</div><div class='ctx'> 	struct bnxt_qplib_hwq *xrrq;</div><div class='del'>-	u16 cmd_flags = 0, max_ssge;</div><div class='del'>-	struct cmdq_create_qp req;</div><div class='ctx'> 	struct bnxt_qplib_pbl *pbl;</div><div class='ctx'> 	u32 qp_flags = 0;</div><div class='del'>-	u16 max_rsge;</div><div class='add'>+	u8 pg_sz_lvl;</div><div class='add'>+	u32 tbl_indx;</div><div class='add'>+	u16 nsge;</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);</div><div class='add'>+	qp-&gt;is_host_msn_tbl = _is_host_msn_table(res-&gt;dattr-&gt;dev_cap_flags2);</div><div class='add'>+	sq-&gt;dbinfo.flags = 0;</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_CREATE_QP,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='ctx'> 	/* General */</div><div class='ctx'> 	req.type = qp-&gt;type;</div><div class='hunk'>@@ -883,144 +986,110 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)</div><div class='ctx'> </div><div class='ctx'> 	/* SQ */</div><div class='ctx'> 	if (qp-&gt;type == CMDQ_CREATE_QP_TYPE_RC) {</div><div class='del'>-		psn_sz = bnxt_qplib_is_chip_gen_p5(res-&gt;cctx) ?</div><div class='add'>+		psn_sz = bnxt_qplib_is_chip_gen_p5_p7(res-&gt;cctx) ?</div><div class='ctx'> 			 sizeof(struct sq_psn_search_ext) :</div><div class='ctx'> 			 sizeof(struct sq_psn_search);</div><div class='add'>+</div><div class='add'>+		if (qp-&gt;is_host_msn_tbl) {</div><div class='add'>+			psn_sz = sizeof(struct sq_msn_search);</div><div class='add'>+			qp-&gt;msn = 0;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='del'>-	sq-&gt;hwq.max_elements = sq-&gt;max_wqe;</div><div class='del'>-	rc = bnxt_qplib_alloc_init_hwq(res-&gt;pdev, &amp;sq-&gt;hwq, &amp;sq-&gt;sg_info,</div><div class='del'>-				       &amp;sq-&gt;hwq.max_elements,</div><div class='del'>-				       BNXT_QPLIB_MAX_SQE_ENTRY_SIZE,</div><div class='del'>-				       psn_sz,</div><div class='del'>-				       PAGE_SIZE, HWQ_TYPE_QUEUE);</div><div class='add'>+</div><div class='add'>+	hwq_attr.res = res;</div><div class='add'>+	hwq_attr.sginfo = &amp;sq-&gt;sg_info;</div><div class='add'>+	hwq_attr.stride = sizeof(struct sq_sge);</div><div class='add'>+	hwq_attr.depth = bnxt_qplib_get_depth(sq, qp-&gt;wqe_mode, true);</div><div class='add'>+	hwq_attr.aux_stride = psn_sz;</div><div class='add'>+	hwq_attr.aux_depth = psn_sz ? bnxt_qplib_set_sq_size(sq, qp-&gt;wqe_mode)</div><div class='add'>+				    : 0;</div><div class='add'>+	/* Update msn tbl size */</div><div class='add'>+	if (qp-&gt;is_host_msn_tbl &amp;&amp; psn_sz) {</div><div class='add'>+		if (qp-&gt;wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)</div><div class='add'>+			hwq_attr.aux_depth =</div><div class='add'>+				roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, qp-&gt;wqe_mode));</div><div class='add'>+		else</div><div class='add'>+			hwq_attr.aux_depth =</div><div class='add'>+				roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, qp-&gt;wqe_mode)) / 2;</div><div class='add'>+		qp-&gt;msn_tbl_sz = hwq_attr.aux_depth;</div><div class='add'>+		qp-&gt;msn = 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hwq_attr.type = HWQ_TYPE_QUEUE;</div><div class='add'>+	rc = bnxt_qplib_alloc_init_hwq(&amp;sq-&gt;hwq, &amp;hwq_attr);</div><div class='ctx'> 	if (rc)</div><div class='del'>-		goto exit;</div><div class='add'>+		return rc;</div><div class='ctx'> </div><div class='del'>-	sq-&gt;swq = kcalloc(sq-&gt;hwq.max_elements, sizeof(*sq-&gt;swq), GFP_KERNEL);</div><div class='del'>-	if (!sq-&gt;swq) {</div><div class='del'>-		rc = -ENOMEM;</div><div class='del'>-		goto fail_sq;</div><div class='del'>-	}</div><div class='del'>-	hw_sq_send_ptr = (struct sq_send **)sq-&gt;hwq.pbl_ptr;</div><div class='del'>-	if (psn_sz) {</div><div class='del'>-		psn_search_ptr = (struct sq_psn_search **)</div><div class='del'>-				  &amp;hw_sq_send_ptr[get_sqe_pg</div><div class='del'>-					(sq-&gt;hwq.max_elements)];</div><div class='del'>-		psn_search = (unsigned long int)</div><div class='del'>-			      &amp;hw_sq_send_ptr[get_sqe_pg(sq-&gt;hwq.max_elements)]</div><div class='del'>-			      [get_sqe_idx(sq-&gt;hwq.max_elements)];</div><div class='del'>-		if (psn_search &amp; ~PAGE_MASK) {</div><div class='del'>-			/* If the psn_search does not start on a page boundary,</div><div class='del'>-			 * then calculate the offset</div><div class='del'>-			 */</div><div class='del'>-			poff = (psn_search &amp; ~PAGE_MASK) /</div><div class='del'>-				BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE;</div><div class='del'>-		}</div><div class='del'>-		for (i = 0; i &lt; sq-&gt;hwq.max_elements; i++) {</div><div class='del'>-			sq-&gt;swq[i].psn_search =</div><div class='del'>-				&amp;psn_search_ptr[get_psne_pg(i + poff)]</div><div class='del'>-					       [get_psne_idx(i + poff)];</div><div class='del'>-			/*psns_ext will be used only for P5 chips. */</div><div class='del'>-			sq-&gt;swq[i].psn_ext =</div><div class='del'>-				(struct sq_psn_search_ext *)</div><div class='del'>-				&amp;psn_search_ptr[get_psne_pg(i + poff)]</div><div class='del'>-					       [get_psne_idx(i + poff)];</div><div class='del'>-		}</div><div class='add'>+	if (!sq-&gt;hwq.is_user) {</div><div class='add'>+		rc = bnxt_qplib_alloc_init_swq(sq);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto fail_sq;</div><div class='add'>+</div><div class='add'>+		if (psn_sz)</div><div class='add'>+			bnxt_qplib_init_psn_ptr(qp, psn_sz);</div><div class='ctx'> 	}</div><div class='add'>+	req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp-&gt;wqe_mode));</div><div class='ctx'> 	pbl = &amp;sq-&gt;hwq.pbl[PBL_LVL_0];</div><div class='ctx'> 	req.sq_pbl = cpu_to_le64(pbl-&gt;pg_map_arr[0]);</div><div class='del'>-	req.sq_pg_size_sq_lvl =</div><div class='del'>-		((sq-&gt;hwq.level &amp; CMDQ_CREATE_QP_SQ_LVL_MASK)</div><div class='del'>-				 &lt;&lt;  CMDQ_CREATE_QP_SQ_LVL_SFT) |</div><div class='del'>-		(pbl-&gt;pg_size == ROCE_PG_SIZE_4K ?</div><div class='del'>-				CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K :</div><div class='del'>-		 pbl-&gt;pg_size == ROCE_PG_SIZE_8K ?</div><div class='del'>-				CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K :</div><div class='del'>-		 pbl-&gt;pg_size == ROCE_PG_SIZE_64K ?</div><div class='del'>-				CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K :</div><div class='del'>-		 pbl-&gt;pg_size == ROCE_PG_SIZE_2M ?</div><div class='del'>-				CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M :</div><div class='del'>-		 pbl-&gt;pg_size == ROCE_PG_SIZE_8M ?</div><div class='del'>-				CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M :</div><div class='del'>-		 pbl-&gt;pg_size == ROCE_PG_SIZE_1G ?</div><div class='del'>-				CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G :</div><div class='del'>-		 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K);</div><div class='del'>-</div><div class='del'>-	if (qp-&gt;scq)</div><div class='del'>-		req.scq_cid = cpu_to_le32(qp-&gt;scq-&gt;id);</div><div class='del'>-</div><div class='del'>-	qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;</div><div class='del'>-	qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;</div><div class='del'>-	if (qp-&gt;sig_type)</div><div class='del'>-		qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;</div><div class='add'>+	pg_sz_lvl = (bnxt_qplib_base_pg_size(&amp;sq-&gt;hwq) &lt;&lt;</div><div class='add'>+		     CMDQ_CREATE_QP_SQ_PG_SIZE_SFT);</div><div class='add'>+	pg_sz_lvl |= (sq-&gt;hwq.level &amp; CMDQ_CREATE_QP_SQ_LVL_MASK);</div><div class='add'>+	req.sq_pg_size_sq_lvl = pg_sz_lvl;</div><div class='add'>+	req.sq_fwo_sq_sge =</div><div class='add'>+		cpu_to_le16(((sq-&gt;max_sge &amp; CMDQ_CREATE_QP_SQ_SGE_MASK) &lt;&lt;</div><div class='add'>+			     CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);</div><div class='add'>+	req.scq_cid = cpu_to_le32(qp-&gt;scq-&gt;id);</div><div class='ctx'> </div><div class='ctx'> 	/* RQ */</div><div class='del'>-	if (rq-&gt;max_wqe) {</div><div class='del'>-		rq-&gt;hwq.max_elements = rq-&gt;max_wqe;</div><div class='del'>-		rc = bnxt_qplib_alloc_init_hwq(res-&gt;pdev, &amp;rq-&gt;hwq,</div><div class='del'>-					       &amp;rq-&gt;sg_info,</div><div class='del'>-					       &amp;rq-&gt;hwq.max_elements,</div><div class='del'>-					       BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,</div><div class='del'>-					       PAGE_SIZE, HWQ_TYPE_QUEUE);</div><div class='add'>+	if (!qp-&gt;srq) {</div><div class='add'>+		rq-&gt;dbinfo.flags = 0;</div><div class='add'>+		hwq_attr.res = res;</div><div class='add'>+		hwq_attr.sginfo = &amp;rq-&gt;sg_info;</div><div class='add'>+		hwq_attr.stride = sizeof(struct sq_sge);</div><div class='add'>+		hwq_attr.depth = bnxt_qplib_get_depth(rq, qp-&gt;wqe_mode, false);</div><div class='add'>+		hwq_attr.aux_stride = 0;</div><div class='add'>+		hwq_attr.aux_depth = 0;</div><div class='add'>+		hwq_attr.type = HWQ_TYPE_QUEUE;</div><div class='add'>+		rc = bnxt_qplib_alloc_init_hwq(&amp;rq-&gt;hwq, &amp;hwq_attr);</div><div class='ctx'> 		if (rc)</div><div class='del'>-			goto fail_sq;</div><div class='del'>-</div><div class='del'>-		rq-&gt;swq = kcalloc(rq-&gt;hwq.max_elements, sizeof(*rq-&gt;swq),</div><div class='del'>-				  GFP_KERNEL);</div><div class='del'>-		if (!rq-&gt;swq) {</div><div class='del'>-			rc = -ENOMEM;</div><div class='del'>-			goto fail_rq;</div><div class='add'>+			goto sq_swq;</div><div class='add'>+		if (!rq-&gt;hwq.is_user) {</div><div class='add'>+			rc = bnxt_qplib_alloc_init_swq(rq);</div><div class='add'>+			if (rc)</div><div class='add'>+				goto fail_rq;</div><div class='ctx'> 		}</div><div class='add'>+</div><div class='add'>+		req.rq_size = cpu_to_le32(rq-&gt;max_wqe);</div><div class='ctx'> 		pbl = &amp;rq-&gt;hwq.pbl[PBL_LVL_0];</div><div class='ctx'> 		req.rq_pbl = cpu_to_le64(pbl-&gt;pg_map_arr[0]);</div><div class='del'>-		req.rq_pg_size_rq_lvl =</div><div class='del'>-			((rq-&gt;hwq.level &amp; CMDQ_CREATE_QP_RQ_LVL_MASK) &lt;&lt;</div><div class='del'>-			 CMDQ_CREATE_QP_RQ_LVL_SFT) |</div><div class='del'>-				(pbl-&gt;pg_size == ROCE_PG_SIZE_4K ?</div><div class='del'>-					CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K :</div><div class='del'>-				 pbl-&gt;pg_size == ROCE_PG_SIZE_8K ?</div><div class='del'>-					CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K :</div><div class='del'>-				 pbl-&gt;pg_size == ROCE_PG_SIZE_64K ?</div><div class='del'>-					CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K :</div><div class='del'>-				 pbl-&gt;pg_size == ROCE_PG_SIZE_2M ?</div><div class='del'>-					CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M :</div><div class='del'>-				 pbl-&gt;pg_size == ROCE_PG_SIZE_8M ?</div><div class='del'>-					CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M :</div><div class='del'>-				 pbl-&gt;pg_size == ROCE_PG_SIZE_1G ?</div><div class='del'>-					CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G :</div><div class='del'>-				 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K);</div><div class='add'>+		pg_sz_lvl = (bnxt_qplib_base_pg_size(&amp;rq-&gt;hwq) &lt;&lt;</div><div class='add'>+			     CMDQ_CREATE_QP_RQ_PG_SIZE_SFT);</div><div class='add'>+		pg_sz_lvl |= (rq-&gt;hwq.level &amp; CMDQ_CREATE_QP_RQ_LVL_MASK);</div><div class='add'>+		req.rq_pg_size_rq_lvl = pg_sz_lvl;</div><div class='add'>+		nsge = (qp-&gt;wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?</div><div class='add'>+			6 : rq-&gt;max_sge;</div><div class='add'>+		req.rq_fwo_rq_sge =</div><div class='add'>+			cpu_to_le16(((nsge &amp;</div><div class='add'>+				      CMDQ_CREATE_QP_RQ_SGE_MASK) &lt;&lt;</div><div class='add'>+				     CMDQ_CREATE_QP_RQ_SGE_SFT) | 0);</div><div class='ctx'> 	} else {</div><div class='ctx'> 		/* SRQ */</div><div class='del'>-		if (qp-&gt;srq) {</div><div class='del'>-			qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;</div><div class='del'>-			req.srq_cid = cpu_to_le32(qp-&gt;srq-&gt;id);</div><div class='del'>-		}</div><div class='add'>+		qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;</div><div class='add'>+		req.srq_cid = cpu_to_le32(qp-&gt;srq-&gt;id);</div><div class='ctx'> 	}</div><div class='add'>+	req.rcq_cid = cpu_to_le32(qp-&gt;rcq-&gt;id);</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;rcq)</div><div class='del'>-		req.rcq_cid = cpu_to_le32(qp-&gt;rcq-&gt;id);</div><div class='del'>-	req.qp_flags = cpu_to_le32(qp_flags);</div><div class='del'>-	req.sq_size = cpu_to_le32(sq-&gt;hwq.max_elements);</div><div class='del'>-	req.rq_size = cpu_to_le32(rq-&gt;hwq.max_elements);</div><div class='del'>-	qp-&gt;sq_hdr_buf = NULL;</div><div class='del'>-	qp-&gt;rq_hdr_buf = NULL;</div><div class='add'>+	qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;</div><div class='add'>+	qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;</div><div class='add'>+	if (qp-&gt;sig_type)</div><div class='add'>+		qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;</div><div class='add'>+	if (qp-&gt;wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)</div><div class='add'>+		qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED;</div><div class='add'>+	if (bnxt_ext_stats_supported(res-&gt;cctx, res-&gt;dattr-&gt;dev_cap_flags, res-&gt;is_vf))</div><div class='add'>+		qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_ENABLED;</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);</div><div class='del'>-	if (rc)</div><div class='del'>-		goto fail_rq;</div><div class='add'>+	req.qp_flags = cpu_to_le32(qp_flags);</div><div class='ctx'> </div><div class='del'>-	/* CTRL-22434: Irrespective of the requested SGE count on the SQ</div><div class='del'>-	 * always create the QP with max send sges possible if the requested</div><div class='del'>-	 * inline size is greater than 0.</div><div class='del'>-	 */</div><div class='del'>-	max_ssge = qp-&gt;max_inline_data ? 6 : sq-&gt;max_sge;</div><div class='del'>-	req.sq_fwo_sq_sge = cpu_to_le16(</div><div class='del'>-				((max_ssge &amp; CMDQ_CREATE_QP_SQ_SGE_MASK)</div><div class='del'>-				 &lt;&lt; CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);</div><div class='del'>-	max_rsge = bnxt_qplib_is_chip_gen_p5(res-&gt;cctx) ? 6 : rq-&gt;max_sge;</div><div class='del'>-	req.rq_fwo_rq_sge = cpu_to_le16(</div><div class='del'>-				((max_rsge &amp; CMDQ_CREATE_QP_RQ_SGE_MASK)</div><div class='del'>-				 &lt;&lt; CMDQ_CREATE_QP_RQ_SGE_SFT) | 0);</div><div class='ctx'> 	/* ORRQ and IRRQ */</div><div class='ctx'> 	if (psn_sz) {</div><div class='ctx'> 		xrrq = &amp;qp-&gt;orrq;</div><div class='hunk'>@@ -1029,12 +1098,19 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)</div><div class='ctx'> 		req_size = xrrq-&gt;max_elements *</div><div class='ctx'> 			   BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;</div><div class='ctx'> 		req_size &amp;= ~(PAGE_SIZE - 1);</div><div class='del'>-		rc = bnxt_qplib_alloc_init_hwq(res-&gt;pdev, xrrq, NULL,</div><div class='del'>-					       &amp;xrrq-&gt;max_elements,</div><div class='del'>-					       BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE,</div><div class='del'>-					       0, req_size, HWQ_TYPE_CTX);</div><div class='add'>+		sginfo.pgsize = req_size;</div><div class='add'>+		sginfo.pgshft = PAGE_SHIFT;</div><div class='add'>+</div><div class='add'>+		hwq_attr.res = res;</div><div class='add'>+		hwq_attr.sginfo = &amp;sginfo;</div><div class='add'>+		hwq_attr.depth = xrrq-&gt;max_elements;</div><div class='add'>+		hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE;</div><div class='add'>+		hwq_attr.aux_stride = 0;</div><div class='add'>+		hwq_attr.aux_depth = 0;</div><div class='add'>+		hwq_attr.type = HWQ_TYPE_CTX;</div><div class='add'>+		rc = bnxt_qplib_alloc_init_hwq(xrrq, &amp;hwq_attr);</div><div class='ctx'> 		if (rc)</div><div class='del'>-			goto fail_buf_free;</div><div class='add'>+			goto rq_swq;</div><div class='ctx'> 		pbl = &amp;xrrq-&gt;pbl[PBL_LVL_0];</div><div class='ctx'> 		req.orrq_addr = cpu_to_le64(pbl-&gt;pg_map_arr[0]);</div><div class='ctx'> </div><div class='hunk'>@@ -1044,11 +1120,10 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)</div><div class='ctx'> 		req_size = xrrq-&gt;max_elements *</div><div class='ctx'> 			   BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;</div><div class='ctx'> 		req_size &amp;= ~(PAGE_SIZE - 1);</div><div class='del'>-</div><div class='del'>-		rc = bnxt_qplib_alloc_init_hwq(res-&gt;pdev, xrrq, NULL,</div><div class='del'>-					       &amp;xrrq-&gt;max_elements,</div><div class='del'>-					       BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE,</div><div class='del'>-					       0, req_size, HWQ_TYPE_CTX);</div><div class='add'>+		sginfo.pgsize = req_size;</div><div class='add'>+		hwq_attr.depth =  xrrq-&gt;max_elements;</div><div class='add'>+		hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE;</div><div class='add'>+		rc = bnxt_qplib_alloc_init_hwq(xrrq, &amp;hwq_attr);</div><div class='ctx'> 		if (rc)</div><div class='ctx'> 			goto fail_orrq;</div><div class='ctx'> </div><div class='hunk'>@@ -1057,36 +1132,46 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)</div><div class='ctx'> 	}</div><div class='ctx'> 	req.pd_id = cpu_to_le32(qp-&gt;pd-&gt;id);</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req,</div><div class='del'>-					  (void *)&amp;resp, NULL, 0);</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='ctx'> 	qp-&gt;id = le32_to_cpu(resp.xid);</div><div class='ctx'> 	qp-&gt;cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;</div><div class='del'>-	qp-&gt;cctx = res-&gt;cctx;</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;qp-&gt;sq_flush);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;qp-&gt;rq_flush);</div><div class='del'>-	rcfw-&gt;qp_tbl[qp-&gt;id].qp_id = qp-&gt;id;</div><div class='del'>-	rcfw-&gt;qp_tbl[qp-&gt;id].qp_handle = (void *)qp;</div><div class='add'>+	qp-&gt;cctx = res-&gt;cctx;</div><div class='add'>+	sq-&gt;dbinfo.hwq = &amp;sq-&gt;hwq;</div><div class='add'>+	sq-&gt;dbinfo.xid = qp-&gt;id;</div><div class='add'>+	sq-&gt;dbinfo.db = qp-&gt;dpi-&gt;dbr;</div><div class='add'>+	sq-&gt;dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp-&gt;wqe_mode);</div><div class='add'>+	if (rq-&gt;max_wqe) {</div><div class='add'>+		rq-&gt;dbinfo.hwq = &amp;rq-&gt;hwq;</div><div class='add'>+		rq-&gt;dbinfo.xid = qp-&gt;id;</div><div class='add'>+		rq-&gt;dbinfo.db = qp-&gt;dpi-&gt;dbr;</div><div class='add'>+		rq-&gt;dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq-&gt;wqe_size);</div><div class='add'>+	}</div><div class='add'>+	spin_lock_bh(&amp;rcfw-&gt;tbl_lock);</div><div class='add'>+	tbl_indx = map_qp_id_to_tbl_indx(qp-&gt;id, rcfw);</div><div class='add'>+	rcfw-&gt;qp_tbl[tbl_indx].qp_id = qp-&gt;id;</div><div class='add'>+	rcfw-&gt;qp_tbl[tbl_indx].qp_handle = (void *)qp;</div><div class='add'>+	spin_unlock_bh(&amp;rcfw-&gt;tbl_lock);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='del'>-</div><div class='ctx'> fail:</div><div class='del'>-	if (qp-&gt;irrq.max_elements)</div><div class='del'>-		bnxt_qplib_free_hwq(res-&gt;pdev, &amp;qp-&gt;irrq);</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;qp-&gt;irrq);</div><div class='ctx'> fail_orrq:</div><div class='del'>-	if (qp-&gt;orrq.max_elements)</div><div class='del'>-		bnxt_qplib_free_hwq(res-&gt;pdev, &amp;qp-&gt;orrq);</div><div class='del'>-fail_buf_free:</div><div class='del'>-	bnxt_qplib_free_qp_hdr_buf(res, qp);</div><div class='del'>-fail_rq:</div><div class='del'>-	bnxt_qplib_free_hwq(res-&gt;pdev, &amp;rq-&gt;hwq);</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;qp-&gt;orrq);</div><div class='add'>+rq_swq:</div><div class='ctx'> 	kfree(rq-&gt;swq);</div><div class='del'>-fail_sq:</div><div class='del'>-	bnxt_qplib_free_hwq(res-&gt;pdev, &amp;sq-&gt;hwq);</div><div class='add'>+fail_rq:</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;rq-&gt;hwq);</div><div class='add'>+sq_swq:</div><div class='ctx'> 	kfree(sq-&gt;swq);</div><div class='del'>-exit:</div><div class='add'>+fail_sq:</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;sq-&gt;hwq);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1104,8 +1189,6 @@ static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp)</div><div class='ctx'> 			qp-&gt;path_mtu =</div><div class='ctx'> 				CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;</div><div class='ctx'> 		}</div><div class='del'>-		qp-&gt;modify_flags &amp;=</div><div class='del'>-			~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;</div><div class='ctx'> 		/* Bono FW require the max_dest_rd_atomic to be &gt;= 1 */</div><div class='ctx'> 		if (qp-&gt;max_dest_rd_atomic &lt; 1)</div><div class='ctx'> 			qp-&gt;max_dest_rd_atomic = 1;</div><div class='hunk'>@@ -1179,20 +1262,73 @@ static void __filter_modify_flags(struct bnxt_qplib_qp *qp)</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void bnxt_set_mandatory_attributes(struct bnxt_qplib_res *res,</div><div class='add'>+					  struct bnxt_qplib_qp *qp,</div><div class='add'>+					  struct cmdq_modify_qp *req)</div><div class='add'>+{</div><div class='add'>+	u32 mandatory_flags = 0;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;type == CMDQ_MODIFY_QP_QP_TYPE_RC)</div><div class='add'>+		mandatory_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_INIT &amp;&amp;</div><div class='add'>+	    qp-&gt;state == CMDQ_MODIFY_QP_NEW_STATE_RTR) {</div><div class='add'>+		if (qp-&gt;type == CMDQ_MODIFY_QP_QP_TYPE_RC &amp;&amp; qp-&gt;srq)</div><div class='add'>+			req-&gt;flags = cpu_to_le16(CMDQ_MODIFY_QP_FLAGS_SRQ_USED);</div><div class='add'>+		mandatory_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (_is_min_rnr_in_rtr_rts_mandatory(res-&gt;dattr-&gt;dev_cap_flags2) &amp;&amp;</div><div class='add'>+	    (qp-&gt;cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_RTR &amp;&amp;</div><div class='add'>+	     qp-&gt;state == CMDQ_MODIFY_QP_NEW_STATE_RTS)) {</div><div class='add'>+		if (qp-&gt;type == CMDQ_MODIFY_QP_QP_TYPE_RC)</div><div class='add'>+			mandatory_flags |=</div><div class='add'>+				CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;type == CMDQ_MODIFY_QP_QP_TYPE_UD ||</div><div class='add'>+	    qp-&gt;type == CMDQ_MODIFY_QP_QP_TYPE_GSI)</div><div class='add'>+		mandatory_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;</div><div class='add'>+</div><div class='add'>+	qp-&gt;modify_flags |= mandatory_flags;</div><div class='add'>+	req-&gt;qp_type = qp-&gt;type;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool is_optimized_state_transition(struct bnxt_qplib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	if ((qp-&gt;cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_INIT &amp;&amp;</div><div class='add'>+	     qp-&gt;state == CMDQ_MODIFY_QP_NEW_STATE_RTR) ||</div><div class='add'>+	    (qp-&gt;cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_RTR &amp;&amp;</div><div class='add'>+	     qp-&gt;state == CMDQ_MODIFY_QP_NEW_STATE_RTS))</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)</div><div class='ctx'> {</div><div class='add'>+	struct bnxt_qplib_sgid_tbl *sgid_tbl = &amp;res-&gt;sgid_tbl;</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct cmdq_modify_qp req;</div><div class='del'>-	struct creq_modify_qp_resp resp;</div><div class='del'>-	u16 cmd_flags = 0, pkey;</div><div class='add'>+	struct creq_modify_qp_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_modify_qp req = {};</div><div class='add'>+	u16 vlan_pcp_vlan_dei_vlan_id;</div><div class='ctx'> 	u32 temp32[4];</div><div class='ctx'> 	u32 bmask;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags);</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_MODIFY_QP,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='ctx'> 	/* Filter out the qp_attr_mask based on the state-&gt;new transition */</div><div class='ctx'> 	__filter_modify_flags(qp);</div><div class='add'>+	if (qp-&gt;modify_flags &amp; CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {</div><div class='add'>+		/* Set mandatory attributes for INIT -&gt; RTR and RTR -&gt; RTS transition */</div><div class='add'>+		if (_is_optimize_modify_qp_supported(res-&gt;dattr-&gt;dev_cap_flags2) &amp;&amp;</div><div class='add'>+		    is_optimized_state_transition(qp))</div><div class='add'>+			bnxt_set_mandatory_attributes(res, qp, &amp;req);</div><div class='add'>+	}</div><div class='ctx'> 	bmask = qp-&gt;modify_flags;</div><div class='ctx'> 	req.modify_mask = cpu_to_le32(qp-&gt;modify_flags);</div><div class='ctx'> 	req.qp_cid = cpu_to_le32(qp-&gt;id);</div><div class='hunk'>@@ -1207,11 +1343,9 @@ int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)</div><div class='ctx'> 	if (bmask &amp; CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS)</div><div class='ctx'> 		req.access = qp-&gt;access;</div><div class='ctx'> </div><div class='del'>-	if (bmask &amp; CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) {</div><div class='del'>-		if (!bnxt_qplib_get_pkey(res, &amp;res-&gt;pkey_tbl,</div><div class='del'>-					 qp-&gt;pkey_index, &amp;pkey))</div><div class='del'>-			req.pkey = cpu_to_le16(pkey);</div><div class='del'>-	}</div><div class='add'>+	if (bmask &amp; CMDQ_MODIFY_QP_MODIFY_MASK_PKEY)</div><div class='add'>+		req.pkey = cpu_to_le16(IB_DEFAULT_PKEY_FULL);</div><div class='add'>+</div><div class='ctx'> 	if (bmask &amp; CMDQ_MODIFY_QP_MODIFY_MASK_QKEY)</div><div class='ctx'> 		req.qkey = cpu_to_le32(qp-&gt;qkey);</div><div class='ctx'> </div><div class='hunk'>@@ -1225,9 +1359,14 @@ int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)</div><div class='ctx'> 	if (bmask &amp; CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL)</div><div class='ctx'> 		req.flow_label = cpu_to_le32(qp-&gt;ah.flow_label);</div><div class='ctx'> </div><div class='del'>-	if (bmask &amp; CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)</div><div class='del'>-		req.sgid_index = cpu_to_le16(res-&gt;sgid_tbl.hw_id</div><div class='del'>-					     [qp-&gt;ah.sgid_index]);</div><div class='add'>+	if (bmask &amp; CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX) {</div><div class='add'>+		if (qp-&gt;type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE)</div><div class='add'>+			req.sgid_index =</div><div class='add'>+				cpu_to_le16(sgid_tbl-&gt;hw_id[qp-&gt;ugid_index]);</div><div class='add'>+		else</div><div class='add'>+			req.sgid_index =</div><div class='add'>+				cpu_to_le16(sgid_tbl-&gt;hw_id[qp-&gt;ah.sgid_index]);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	if (bmask &amp; CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT)</div><div class='ctx'> 		req.hop_limit = qp-&gt;ah.hop_limit;</div><div class='hunk'>@@ -1239,7 +1378,7 @@ int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)</div><div class='ctx'> 		memcpy(req.dest_mac, qp-&gt;ah.dmac, 6);</div><div class='ctx'> </div><div class='ctx'> 	if (bmask &amp; CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)</div><div class='del'>-		req.path_mtu = qp-&gt;path_mtu;</div><div class='add'>+		req.path_mtu_pingpong_push_enable |= qp-&gt;path_mtu;</div><div class='ctx'> </div><div class='ctx'> 	if (bmask &amp; CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT)</div><div class='ctx'> 		req.timeout = qp-&gt;timeout;</div><div class='hunk'>@@ -1275,10 +1414,19 @@ int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)</div><div class='ctx'> 	if (bmask &amp; CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID)</div><div class='ctx'> 		req.dest_qp_id = cpu_to_le32(qp-&gt;dest_qpn);</div><div class='ctx'> </div><div class='del'>-	req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp-&gt;vlan_id);</div><div class='add'>+	if (bmask &amp; CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID) {</div><div class='add'>+		vlan_pcp_vlan_dei_vlan_id =</div><div class='add'>+			((res-&gt;sgid_tbl.tbl[qp-&gt;ah.sgid_index].vlan_id &lt;&lt;</div><div class='add'>+			  CMDQ_MODIFY_QP_VLAN_ID_SFT) &amp;</div><div class='add'>+			 CMDQ_MODIFY_QP_VLAN_ID_MASK);</div><div class='add'>+		vlan_pcp_vlan_dei_vlan_id |=</div><div class='add'>+			((qp-&gt;ah.sl &lt;&lt; CMDQ_MODIFY_QP_VLAN_PCP_SFT) &amp;</div><div class='add'>+			 CMDQ_MODIFY_QP_VLAN_PCP_MASK);</div><div class='add'>+		req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(vlan_pcp_vlan_dei_vlan_id);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req,</div><div class='del'>-					  (void *)&amp;resp, NULL, 0);</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),  sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		return rc;</div><div class='ctx'> 	qp-&gt;cur_qp_state = qp-&gt;state;</div><div class='hunk'>@@ -1288,36 +1436,41 @@ int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)</div><div class='ctx'> int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct cmdq_query_qp req;</div><div class='del'>-	struct creq_query_qp_resp resp;</div><div class='del'>-	struct bnxt_qplib_rcfw_sbuf *sbuf;</div><div class='add'>+	struct creq_query_qp_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct bnxt_qplib_rcfw_sbuf sbuf;</div><div class='ctx'> 	struct creq_query_qp_resp_sb *sb;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='add'>+	struct cmdq_query_qp req = {};</div><div class='ctx'> 	u32 temp32[4];</div><div class='del'>-	int i, rc = 0;</div><div class='add'>+	int i, rc;</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, QUERY_QP, cmd_flags);</div><div class='del'>-</div><div class='del'>-	sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));</div><div class='del'>-	if (!sbuf)</div><div class='add'>+	sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS);</div><div class='add'>+	sbuf.sb = dma_alloc_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf.size,</div><div class='add'>+				     &amp;sbuf.dma_addr, GFP_KERNEL);</div><div class='add'>+	if (!sbuf.sb)</div><div class='ctx'> 		return -ENOMEM;</div><div class='del'>-	sb = sbuf-&gt;sb;</div><div class='add'>+	sb = sbuf.sb;</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_QUERY_QP,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='ctx'> 	req.qp_cid = cpu_to_le32(qp-&gt;id);</div><div class='del'>-	req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;</div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req, (void *)&amp;resp,</div><div class='del'>-					  (void *)sbuf, 0);</div><div class='add'>+	req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, &amp;sbuf, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto bail;</div><div class='ctx'> 	/* Extract the context from the side buffer */</div><div class='ctx'> 	qp-&gt;state = sb-&gt;en_sqd_async_notify_state &amp;</div><div class='ctx'> 			CREQ_QUERY_QP_RESP_SB_STATE_MASK;</div><div class='ctx'> 	qp-&gt;en_sqd_async_notify = sb-&gt;en_sqd_async_notify_state &amp;</div><div class='del'>-				  CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ?</div><div class='del'>-				  true : false;</div><div class='add'>+				  CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY;</div><div class='ctx'> 	qp-&gt;access = sb-&gt;access;</div><div class='ctx'> 	qp-&gt;pkey_index = le16_to_cpu(sb-&gt;pkey);</div><div class='ctx'> 	qp-&gt;qkey = le32_to_cpu(sb-&gt;qkey);</div><div class='add'>+	qp-&gt;udp_sport = le16_to_cpu(sb-&gt;udp_src_port);</div><div class='ctx'> </div><div class='ctx'> 	temp32[0] = le32_to_cpu(sb-&gt;dgid[0]);</div><div class='ctx'> 	temp32[1] = le32_to_cpu(sb-&gt;dgid[1]);</div><div class='hunk'>@@ -1363,21 +1516,25 @@ int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)</div><div class='ctx'> 	qp-&gt;dest_qpn = le32_to_cpu(sb-&gt;dest_qp_id);</div><div class='ctx'> 	memcpy(qp-&gt;smac, sb-&gt;src_mac, 6);</div><div class='ctx'> 	qp-&gt;vlan_id = le16_to_cpu(sb-&gt;vlan_pcp_vlan_dei_vlan_id);</div><div class='add'>+	qp-&gt;port_id = le16_to_cpu(sb-&gt;port_id);</div><div class='ctx'> bail:</div><div class='del'>-	bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);</div><div class='add'>+	dma_free_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf.size,</div><div class='add'>+			  sbuf.sb, sbuf.dma_addr);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_hwq *cq_hwq = &amp;cq-&gt;hwq;</div><div class='del'>-	struct cq_base *hw_cqe, **hw_cqe_ptr;</div><div class='add'>+	u32 peek_flags, peek_cons;</div><div class='add'>+	struct cq_base *hw_cqe;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='add'>+	peek_flags = cq-&gt;dbinfo.flags;</div><div class='add'>+	peek_cons = cq_hwq-&gt;cons;</div><div class='ctx'> 	for (i = 0; i &lt; cq_hwq-&gt;max_elements; i++) {</div><div class='del'>-		hw_cqe_ptr = (struct cq_base **)cq_hwq-&gt;pbl_ptr;</div><div class='del'>-		hw_cqe = &amp;hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)];</div><div class='del'>-		if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq-&gt;max_elements))</div><div class='add'>+		hw_cqe = bnxt_qplib_get_qe(cq_hwq, peek_cons, NULL);</div><div class='add'>+		if (!CQE_CMP_VALID(hw_cqe, peek_flags))</div><div class='ctx'> 			continue;</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * The valid test of the entry must be done first before</div><div class='hunk'>@@ -1407,6 +1564,8 @@ static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)</div><div class='ctx'> 		default:</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='add'>+		bnxt_qplib_hwq_incr_cons(cq_hwq-&gt;max_elements, &amp;peek_cons,</div><div class='add'>+					 1, &amp;peek_flags);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1414,22 +1573,31 @@ int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,</div><div class='ctx'> 			  struct bnxt_qplib_qp *qp)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct cmdq_destroy_qp req;</div><div class='del'>-	struct creq_destroy_qp_resp resp;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='add'>+	struct creq_destroy_qp_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_destroy_qp req = {};</div><div class='add'>+	u32 tbl_indx;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	rcfw-&gt;qp_tbl[qp-&gt;id].qp_id = BNXT_QPLIB_QP_ID_INVALID;</div><div class='del'>-	rcfw-&gt;qp_tbl[qp-&gt;id].qp_handle = NULL;</div><div class='add'>+	spin_lock_bh(&amp;rcfw-&gt;tbl_lock);</div><div class='add'>+	tbl_indx = map_qp_id_to_tbl_indx(qp-&gt;id, rcfw);</div><div class='add'>+	rcfw-&gt;qp_tbl[tbl_indx].qp_id = BNXT_QPLIB_QP_ID_INVALID;</div><div class='add'>+	rcfw-&gt;qp_tbl[tbl_indx].qp_handle = NULL;</div><div class='add'>+	spin_unlock_bh(&amp;rcfw-&gt;tbl_lock);</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags);</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_DESTROY_QP,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='ctx'> 	req.qp_cid = cpu_to_le32(qp-&gt;id);</div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req,</div><div class='del'>-					  (void *)&amp;resp, NULL, 0);</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		rcfw-&gt;qp_tbl[qp-&gt;id].qp_id = qp-&gt;id;</div><div class='del'>-		rcfw-&gt;qp_tbl[qp-&gt;id].qp_handle = qp;</div><div class='add'>+		spin_lock_bh(&amp;rcfw-&gt;tbl_lock);</div><div class='add'>+		rcfw-&gt;qp_tbl[tbl_indx].qp_id = qp-&gt;id;</div><div class='add'>+		rcfw-&gt;qp_tbl[tbl_indx].qp_handle = qp;</div><div class='add'>+		spin_unlock_bh(&amp;rcfw-&gt;tbl_lock);</div><div class='ctx'> 		return rc;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -1440,16 +1608,16 @@ void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res,</div><div class='ctx'> 			    struct bnxt_qplib_qp *qp)</div><div class='ctx'> {</div><div class='ctx'> 	bnxt_qplib_free_qp_hdr_buf(res, qp);</div><div class='del'>-	bnxt_qplib_free_hwq(res-&gt;pdev, &amp;qp-&gt;sq.hwq);</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;qp-&gt;sq.hwq);</div><div class='ctx'> 	kfree(qp-&gt;sq.swq);</div><div class='ctx'> </div><div class='del'>-	bnxt_qplib_free_hwq(res-&gt;pdev, &amp;qp-&gt;rq.hwq);</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;qp-&gt;rq.hwq);</div><div class='ctx'> 	kfree(qp-&gt;rq.swq);</div><div class='ctx'> </div><div class='ctx'> 	if (qp-&gt;irrq.max_elements)</div><div class='del'>-		bnxt_qplib_free_hwq(res-&gt;pdev, &amp;qp-&gt;irrq);</div><div class='add'>+		bnxt_qplib_free_hwq(res, &amp;qp-&gt;irrq);</div><div class='ctx'> 	if (qp-&gt;orrq.max_elements)</div><div class='del'>-		bnxt_qplib_free_hwq(res-&gt;pdev, &amp;qp-&gt;orrq);</div><div class='add'>+		bnxt_qplib_free_hwq(res, &amp;qp-&gt;orrq);</div><div class='ctx'> </div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1462,7 +1630,7 @@ void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,</div><div class='ctx'> 	memset(sge, 0, sizeof(*sge));</div><div class='ctx'> </div><div class='ctx'> 	if (qp-&gt;sq_hdr_buf) {</div><div class='del'>-		sw_prod = HWQ_CMP(sq-&gt;hwq.prod, &amp;sq-&gt;hwq);</div><div class='add'>+		sw_prod = sq-&gt;swq_start;</div><div class='ctx'> 		sge-&gt;addr = (dma_addr_t)(qp-&gt;sq_hdr_buf_map +</div><div class='ctx'> 					 sw_prod * qp-&gt;sq_hdr_buf_size);</div><div class='ctx'> 		sge-&gt;lkey = 0xFFFFFFFF;</div><div class='hunk'>@@ -1476,7 +1644,7 @@ u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_q *rq = &amp;qp-&gt;rq;</div><div class='ctx'> </div><div class='del'>-	return HWQ_CMP(rq-&gt;hwq.prod, &amp;rq-&gt;hwq);</div><div class='add'>+	return rq-&gt;swq_start;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)</div><div class='hunk'>@@ -1493,7 +1661,7 @@ void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,</div><div class='ctx'> 	memset(sge, 0, sizeof(*sge));</div><div class='ctx'> </div><div class='ctx'> 	if (qp-&gt;rq_hdr_buf) {</div><div class='del'>-		sw_prod = HWQ_CMP(rq-&gt;hwq.prod, &amp;rq-&gt;hwq);</div><div class='add'>+		sw_prod = rq-&gt;swq_start;</div><div class='ctx'> 		sge-&gt;addr = (dma_addr_t)(qp-&gt;rq_hdr_buf_map +</div><div class='ctx'> 					 sw_prod * qp-&gt;rq_hdr_buf_size);</div><div class='ctx'> 		sge-&gt;lkey = 0xFFFFFFFF;</div><div class='hunk'>@@ -1503,142 +1671,298 @@ void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,</div><div class='ctx'> 	return NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+/* Fil the MSN table into the next psn row */</div><div class='add'>+static void bnxt_qplib_fill_msn_search(struct bnxt_qplib_qp *qp,</div><div class='add'>+				       struct bnxt_qplib_swqe *wqe,</div><div class='add'>+				       struct bnxt_qplib_swq *swq)</div><div class='add'>+{</div><div class='add'>+	struct sq_msn_search *msns;</div><div class='add'>+	u32 start_psn, next_psn;</div><div class='add'>+	u16 start_idx;</div><div class='add'>+</div><div class='add'>+	msns = (struct sq_msn_search *)swq-&gt;psn_search;</div><div class='add'>+	msns-&gt;start_idx_next_psn_start_psn = 0;</div><div class='add'>+</div><div class='add'>+	start_psn = swq-&gt;start_psn;</div><div class='add'>+	next_psn = swq-&gt;next_psn;</div><div class='add'>+	start_idx = swq-&gt;slot_idx;</div><div class='add'>+	msns-&gt;start_idx_next_psn_start_psn |=</div><div class='add'>+		bnxt_re_update_msn_tbl(start_idx, next_psn, start_psn);</div><div class='add'>+	qp-&gt;msn++;</div><div class='add'>+	qp-&gt;msn %= qp-&gt;msn_tbl_sz;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp,</div><div class='add'>+				       struct bnxt_qplib_swqe *wqe,</div><div class='add'>+				       struct bnxt_qplib_swq *swq)</div><div class='add'>+{</div><div class='add'>+	struct sq_psn_search_ext *psns_ext;</div><div class='add'>+	struct sq_psn_search *psns;</div><div class='add'>+	u32 flg_npsn;</div><div class='add'>+	u32 op_spsn;</div><div class='add'>+</div><div class='add'>+	if (!swq-&gt;psn_search)</div><div class='add'>+		return;</div><div class='add'>+	/* Handle MSN differently on cap flags  */</div><div class='add'>+	if (qp-&gt;is_host_msn_tbl) {</div><div class='add'>+		bnxt_qplib_fill_msn_search(qp, wqe, swq);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+	psns = (struct sq_psn_search *)swq-&gt;psn_search;</div><div class='add'>+	psns = swq-&gt;psn_search;</div><div class='add'>+	psns_ext = swq-&gt;psn_ext;</div><div class='add'>+</div><div class='add'>+	op_spsn = ((swq-&gt;start_psn &lt;&lt; SQ_PSN_SEARCH_START_PSN_SFT) &amp;</div><div class='add'>+		    SQ_PSN_SEARCH_START_PSN_MASK);</div><div class='add'>+	op_spsn |= ((wqe-&gt;type &lt;&lt; SQ_PSN_SEARCH_OPCODE_SFT) &amp;</div><div class='add'>+		     SQ_PSN_SEARCH_OPCODE_MASK);</div><div class='add'>+	flg_npsn = ((swq-&gt;next_psn &lt;&lt; SQ_PSN_SEARCH_NEXT_PSN_SFT) &amp;</div><div class='add'>+		     SQ_PSN_SEARCH_NEXT_PSN_MASK);</div><div class='add'>+</div><div class='add'>+	if (bnxt_qplib_is_chip_gen_p5_p7(qp-&gt;cctx)) {</div><div class='add'>+		psns_ext-&gt;opcode_start_psn = cpu_to_le32(op_spsn);</div><div class='add'>+		psns_ext-&gt;flags_next_psn = cpu_to_le32(flg_npsn);</div><div class='add'>+		psns_ext-&gt;start_slot_idx = cpu_to_le16(swq-&gt;slot_idx);</div><div class='add'>+	} else {</div><div class='add'>+		psns-&gt;opcode_start_psn = cpu_to_le32(op_spsn);</div><div class='add'>+		psns-&gt;flags_next_psn = cpu_to_le32(flg_npsn);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static unsigned int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp,</div><div class='add'>+					  struct bnxt_qplib_swqe *wqe,</div><div class='add'>+					  u32 *idx)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_hwq *hwq;</div><div class='add'>+	int len, t_len, offt;</div><div class='add'>+	bool pull_dst = true;</div><div class='add'>+	void *il_dst = NULL;</div><div class='add'>+	void *il_src = NULL;</div><div class='add'>+	int t_cplen, cplen;</div><div class='add'>+	int indx;</div><div class='add'>+</div><div class='add'>+	hwq = &amp;qp-&gt;sq.hwq;</div><div class='add'>+	t_len = 0;</div><div class='add'>+	for (indx = 0; indx &lt; wqe-&gt;num_sge; indx++) {</div><div class='add'>+		len = wqe-&gt;sg_list[indx].size;</div><div class='add'>+		il_src = (void *)wqe-&gt;sg_list[indx].addr;</div><div class='add'>+		t_len += len;</div><div class='add'>+		if (t_len &gt; qp-&gt;max_inline_data)</div><div class='add'>+			return BNXT_RE_INVAL_MSG_SIZE;</div><div class='add'>+		while (len) {</div><div class='add'>+			if (pull_dst) {</div><div class='add'>+				pull_dst = false;</div><div class='add'>+				il_dst = bnxt_qplib_get_prod_qe(hwq, *idx);</div><div class='add'>+				(*idx)++;</div><div class='add'>+				t_cplen = 0;</div><div class='add'>+				offt = 0;</div><div class='add'>+			}</div><div class='add'>+			cplen = min_t(int, len, sizeof(struct sq_sge));</div><div class='add'>+			cplen = min_t(int, cplen,</div><div class='add'>+					(sizeof(struct sq_sge) - offt));</div><div class='add'>+			memcpy(il_dst, il_src, cplen);</div><div class='add'>+			t_cplen += cplen;</div><div class='add'>+			il_src += cplen;</div><div class='add'>+			il_dst += cplen;</div><div class='add'>+			offt += cplen;</div><div class='add'>+			len -= cplen;</div><div class='add'>+			if (t_cplen == sizeof(struct sq_sge))</div><div class='add'>+				pull_dst = true;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return t_len;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static unsigned int bnxt_qplib_put_sges(struct bnxt_qplib_hwq *hwq,</div><div class='add'>+					struct bnxt_qplib_sge *ssge,</div><div class='add'>+					u32 nsge, u32 *idx)</div><div class='add'>+{</div><div class='add'>+	struct sq_sge *dsge;</div><div class='add'>+	int indx, len = 0;</div><div class='add'>+</div><div class='add'>+	for (indx = 0; indx &lt; nsge; indx++, (*idx)++) {</div><div class='add'>+		dsge = bnxt_qplib_get_prod_qe(hwq, *idx);</div><div class='add'>+		dsge-&gt;va_or_pa = cpu_to_le64(ssge[indx].addr);</div><div class='add'>+		dsge-&gt;l_key = cpu_to_le32(ssge[indx].lkey);</div><div class='add'>+		dsge-&gt;size = cpu_to_le32(ssge[indx].size);</div><div class='add'>+		len += ssge[indx].size;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return len;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u16 bnxt_qplib_required_slots(struct bnxt_qplib_qp *qp,</div><div class='add'>+				     struct bnxt_qplib_swqe *wqe,</div><div class='add'>+				     u16 *wqe_sz, u16 *qdf, u8 mode)</div><div class='add'>+{</div><div class='add'>+	u32 ilsize, bytes;</div><div class='add'>+	u16 nsge;</div><div class='add'>+	u16 slot;</div><div class='add'>+</div><div class='add'>+	nsge = wqe-&gt;num_sge;</div><div class='add'>+	/* Adding sq_send_hdr is a misnomer, for rq also hdr size is same. */</div><div class='add'>+	bytes = sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge);</div><div class='add'>+	if (wqe-&gt;flags &amp; BNXT_QPLIB_SWQE_FLAGS_INLINE) {</div><div class='add'>+		ilsize = bnxt_qplib_calc_ilsize(wqe, qp-&gt;max_inline_data);</div><div class='add'>+		bytes = ALIGN(ilsize, sizeof(struct sq_sge));</div><div class='add'>+		bytes += sizeof(struct sq_send_hdr);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*qdf =  __xlate_qfd(qp-&gt;sq.q_full_delta, bytes);</div><div class='add'>+	slot = bytes &gt;&gt; 4;</div><div class='add'>+	*wqe_sz = slot;</div><div class='add'>+	if (mode == BNXT_QPLIB_WQE_MODE_STATIC)</div><div class='add'>+		slot = 8;</div><div class='add'>+	return slot;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_qp *qp, struct bnxt_qplib_q *sq,</div><div class='add'>+				     struct bnxt_qplib_swq *swq, bool hw_retx)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_hwq *hwq;</div><div class='add'>+	u32 pg_num, pg_indx;</div><div class='add'>+	void *buff;</div><div class='add'>+	u32 tail;</div><div class='add'>+</div><div class='add'>+	hwq = &amp;sq-&gt;hwq;</div><div class='add'>+	if (!hwq-&gt;pad_pg)</div><div class='add'>+		return;</div><div class='add'>+	tail = swq-&gt;slot_idx / sq-&gt;dbinfo.max_slot;</div><div class='add'>+	if (hw_retx) {</div><div class='add'>+		/* For HW retx use qp msn index */</div><div class='add'>+		tail = qp-&gt;msn;</div><div class='add'>+		tail %= qp-&gt;msn_tbl_sz;</div><div class='add'>+	}</div><div class='add'>+	pg_num = (tail + hwq-&gt;pad_pgofft) / (PAGE_SIZE / hwq-&gt;pad_stride);</div><div class='add'>+	pg_indx = (tail + hwq-&gt;pad_pgofft) % (PAGE_SIZE / hwq-&gt;pad_stride);</div><div class='add'>+	buff = (void *)(hwq-&gt;pad_pg[pg_num] + pg_indx * hwq-&gt;pad_stride);</div><div class='add'>+	swq-&gt;psn_ext = buff;</div><div class='add'>+	swq-&gt;psn_search = buff;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_q *sq = &amp;qp-&gt;sq;</div><div class='del'>-	u32 sw_prod;</div><div class='del'>-	u64 val = 0;</div><div class='ctx'> </div><div class='del'>-	val = (((qp-&gt;id &lt;&lt; DBC_DBC_XID_SFT) &amp; DBC_DBC_XID_MASK) |</div><div class='del'>-	       DBC_DBC_TYPE_SQ);</div><div class='del'>-	val &lt;&lt;= 32;</div><div class='del'>-	sw_prod = HWQ_CMP(sq-&gt;hwq.prod, &amp;sq-&gt;hwq);</div><div class='del'>-	val |= (sw_prod &lt;&lt; DBC_DBC_INDEX_SFT) &amp; DBC_DBC_INDEX_MASK;</div><div class='del'>-	/* Flush all the WQE writes to HW */</div><div class='del'>-	writeq(val, qp-&gt;dpi-&gt;dbr);</div><div class='add'>+	bnxt_qplib_ring_prod_db(&amp;sq-&gt;dbinfo, DBC_DBC_TYPE_SQ);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,</div><div class='ctx'> 			 struct bnxt_qplib_swqe *wqe)</div><div class='ctx'> {</div><div class='add'>+	struct bnxt_qplib_nq_work *nq_work = NULL;</div><div class='add'>+	int i, rc = 0, data_len = 0, pkt_num = 0;</div><div class='ctx'> 	struct bnxt_qplib_q *sq = &amp;qp-&gt;sq;</div><div class='add'>+	struct bnxt_qplib_hwq *hwq;</div><div class='ctx'> 	struct bnxt_qplib_swq *swq;</div><div class='del'>-	struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;</div><div class='del'>-	struct sq_sge *hw_sge;</div><div class='del'>-	struct bnxt_qplib_nq_work *nq_work = NULL;</div><div class='ctx'> 	bool sch_handler = false;</div><div class='del'>-	u32 sw_prod;</div><div class='del'>-	u8 wqe_size16;</div><div class='del'>-	int i, rc = 0, data_len = 0, pkt_num = 0;</div><div class='add'>+	u32 wqe_idx, slots, idx;</div><div class='add'>+	u16 wqe_sz, qdf = 0;</div><div class='add'>+	bool msn_update;</div><div class='add'>+	void *base_hdr;</div><div class='add'>+	void *ext_hdr;</div><div class='ctx'> 	__le32 temp32;</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;state != CMDQ_MODIFY_QP_NEW_STATE_RTS) {</div><div class='del'>-		if (qp-&gt;state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {</div><div class='del'>-			sch_handler = true;</div><div class='del'>-			dev_dbg(&amp;sq-&gt;hwq.pdev-&gt;dev,</div><div class='del'>-				"%s Error QP. Scheduling for poll_cq\n",</div><div class='del'>-				__func__);</div><div class='del'>-			goto queue_err;</div><div class='del'>-		}</div><div class='add'>+	hwq = &amp;sq-&gt;hwq;</div><div class='add'>+	if (qp-&gt;state != CMDQ_MODIFY_QP_NEW_STATE_RTS &amp;&amp;</div><div class='add'>+	    qp-&gt;state != CMDQ_MODIFY_QP_NEW_STATE_ERR) {</div><div class='add'>+		dev_err(&amp;hwq-&gt;pdev-&gt;dev,</div><div class='add'>+			"QPLIB: FP: QP (0x%x) is in the 0x%x state",</div><div class='add'>+			qp-&gt;id, qp-&gt;state);</div><div class='add'>+		rc = -EINVAL;</div><div class='add'>+		goto done;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (bnxt_qplib_queue_full(sq)) {</div><div class='del'>-		dev_err(&amp;sq-&gt;hwq.pdev-&gt;dev,</div><div class='add'>+	slots = bnxt_qplib_required_slots(qp, wqe, &amp;wqe_sz, &amp;qdf, qp-&gt;wqe_mode);</div><div class='add'>+	if (bnxt_qplib_queue_full(sq, slots + qdf)) {</div><div class='add'>+		dev_err(&amp;hwq-&gt;pdev-&gt;dev,</div><div class='ctx'> 			"prod = %#x cons = %#x qdepth = %#x delta = %#x\n",</div><div class='del'>-			sq-&gt;hwq.prod, sq-&gt;hwq.cons, sq-&gt;hwq.max_elements,</div><div class='del'>-			sq-&gt;q_full_delta);</div><div class='add'>+			hwq-&gt;prod, hwq-&gt;cons, hwq-&gt;depth, sq-&gt;q_full_delta);</div><div class='ctx'> 		rc = -ENOMEM;</div><div class='ctx'> 		goto done;</div><div class='ctx'> 	}</div><div class='del'>-	sw_prod = HWQ_CMP(sq-&gt;hwq.prod, &amp;sq-&gt;hwq);</div><div class='del'>-	swq = &amp;sq-&gt;swq[sw_prod];</div><div class='add'>+</div><div class='add'>+	swq = bnxt_qplib_get_swqe(sq, &amp;wqe_idx);</div><div class='add'>+	bnxt_qplib_pull_psn_buff(qp, sq, swq, qp-&gt;is_host_msn_tbl);</div><div class='add'>+</div><div class='add'>+	idx = 0;</div><div class='add'>+	swq-&gt;slot_idx = hwq-&gt;prod;</div><div class='add'>+	swq-&gt;slots = slots;</div><div class='ctx'> 	swq-&gt;wr_id = wqe-&gt;wr_id;</div><div class='ctx'> 	swq-&gt;type = wqe-&gt;type;</div><div class='ctx'> 	swq-&gt;flags = wqe-&gt;flags;</div><div class='add'>+	swq-&gt;start_psn = sq-&gt;psn &amp; BTH_PSN_MASK;</div><div class='ctx'> 	if (qp-&gt;sig_type)</div><div class='ctx'> 		swq-&gt;flags |= SQ_SEND_FLAGS_SIGNAL_COMP;</div><div class='del'>-	swq-&gt;start_psn = sq-&gt;psn &amp; BTH_PSN_MASK;</div><div class='ctx'> </div><div class='del'>-	hw_sq_send_ptr = (struct sq_send **)sq-&gt;hwq.pbl_ptr;</div><div class='del'>-	hw_sq_send_hdr = &amp;hw_sq_send_ptr[get_sqe_pg(sw_prod)]</div><div class='del'>-					[get_sqe_idx(sw_prod)];</div><div class='add'>+	if (qp-&gt;state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {</div><div class='add'>+		sch_handler = true;</div><div class='add'>+		dev_dbg(&amp;hwq-&gt;pdev-&gt;dev,</div><div class='add'>+			"%s Error QP. Scheduling for poll_cq\n", __func__);</div><div class='add'>+		goto queue_err;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);</div><div class='add'>+	base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);</div><div class='add'>+	ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);</div><div class='add'>+	memset(base_hdr, 0, sizeof(struct sq_sge));</div><div class='add'>+	memset(ext_hdr, 0, sizeof(struct sq_sge));</div><div class='ctx'> </div><div class='del'>-	if (wqe-&gt;flags &amp; BNXT_QPLIB_SWQE_FLAGS_INLINE) {</div><div class='add'>+	if (wqe-&gt;flags &amp; BNXT_QPLIB_SWQE_FLAGS_INLINE)</div><div class='ctx'> 		/* Copy the inline data */</div><div class='del'>-		if (wqe-&gt;inline_len &gt; BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {</div><div class='del'>-			dev_warn(&amp;sq-&gt;hwq.pdev-&gt;dev,</div><div class='del'>-				 "Inline data length &gt; 96 detected\n");</div><div class='del'>-			data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH;</div><div class='del'>-		} else {</div><div class='del'>-			data_len = wqe-&gt;inline_len;</div><div class='del'>-		}</div><div class='del'>-		memcpy(hw_sq_send_hdr-&gt;data, wqe-&gt;inline_data, data_len);</div><div class='del'>-		wqe_size16 = (data_len + 15) &gt;&gt; 4;</div><div class='del'>-	} else {</div><div class='del'>-		for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr-&gt;data;</div><div class='del'>-		     i &lt; wqe-&gt;num_sge; i++, hw_sge++) {</div><div class='del'>-			hw_sge-&gt;va_or_pa = cpu_to_le64(wqe-&gt;sg_list[i].addr);</div><div class='del'>-			hw_sge-&gt;l_key = cpu_to_le32(wqe-&gt;sg_list[i].lkey);</div><div class='del'>-			hw_sge-&gt;size = cpu_to_le32(wqe-&gt;sg_list[i].size);</div><div class='del'>-			data_len += wqe-&gt;sg_list[i].size;</div><div class='del'>-		}</div><div class='del'>-		/* Each SGE entry = 1 WQE size16 */</div><div class='del'>-		wqe_size16 = wqe-&gt;num_sge;</div><div class='del'>-		/* HW requires wqe size has room for atleast one SGE even if</div><div class='del'>-		 * none was supplied by ULP</div><div class='del'>-		 */</div><div class='del'>-		if (!wqe-&gt;num_sge)</div><div class='del'>-			wqe_size16++;</div><div class='add'>+		data_len = bnxt_qplib_put_inline(qp, wqe, &amp;idx);</div><div class='add'>+	else</div><div class='add'>+		data_len = bnxt_qplib_put_sges(hwq, wqe-&gt;sg_list, wqe-&gt;num_sge,</div><div class='add'>+					       &amp;idx);</div><div class='add'>+	if (data_len &gt; BNXT_RE_MAX_MSG_SIZE) {</div><div class='add'>+		rc = -EINVAL;</div><div class='add'>+		goto done;</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='add'>+	/* Make sure we update MSN table only for wired wqes */</div><div class='add'>+	msn_update = true;</div><div class='ctx'> 	/* Specifics */</div><div class='ctx'> 	switch (wqe-&gt;type) {</div><div class='ctx'> 	case BNXT_QPLIB_SWQE_TYPE_SEND:</div><div class='ctx'> 		if (qp-&gt;type == CMDQ_CREATE_QP1_TYPE_GSI) {</div><div class='add'>+			struct sq_send_raweth_qp1_hdr *sqe = base_hdr;</div><div class='add'>+			struct sq_raw_ext_hdr *ext_sqe = ext_hdr;</div><div class='ctx'> 			/* Assemble info for Raw Ethertype QPs */</div><div class='del'>-			struct sq_send_raweth_qp1 *sqe =</div><div class='del'>-				(struct sq_send_raweth_qp1 *)hw_sq_send_hdr;</div><div class='ctx'> </div><div class='ctx'> 			sqe-&gt;wqe_type = wqe-&gt;type;</div><div class='ctx'> 			sqe-&gt;flags = wqe-&gt;flags;</div><div class='del'>-			sqe-&gt;wqe_size = wqe_size16 +</div><div class='del'>-				((offsetof(typeof(*sqe), data) + 15) &gt;&gt; 4);</div><div class='add'>+			sqe-&gt;wqe_size = wqe_sz;</div><div class='ctx'> 			sqe-&gt;cfa_action = cpu_to_le16(wqe-&gt;rawqp1.cfa_action);</div><div class='ctx'> 			sqe-&gt;lflags = cpu_to_le16(wqe-&gt;rawqp1.lflags);</div><div class='ctx'> 			sqe-&gt;length = cpu_to_le32(data_len);</div><div class='del'>-			sqe-&gt;cfa_meta = cpu_to_le32((wqe-&gt;rawqp1.cfa_meta &amp;</div><div class='add'>+			ext_sqe-&gt;cfa_meta = cpu_to_le32((wqe-&gt;rawqp1.cfa_meta &amp;</div><div class='ctx'> 				SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) &lt;&lt;</div><div class='ctx'> 				SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);</div><div class='ctx'> </div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='del'>-		/* fall thru */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:</div><div class='ctx'> 	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:</div><div class='ctx'> 	{</div><div class='del'>-		struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr;</div><div class='add'>+		struct sq_ud_ext_hdr *ext_sqe = ext_hdr;</div><div class='add'>+		struct sq_send_hdr *sqe = base_hdr;</div><div class='ctx'> </div><div class='ctx'> 		sqe-&gt;wqe_type = wqe-&gt;type;</div><div class='ctx'> 		sqe-&gt;flags = wqe-&gt;flags;</div><div class='del'>-		sqe-&gt;wqe_size = wqe_size16 +</div><div class='del'>-				((offsetof(typeof(*sqe), data) + 15) &gt;&gt; 4);</div><div class='del'>-		sqe-&gt;inv_key_or_imm_data = cpu_to_le32(</div><div class='del'>-						wqe-&gt;send.inv_key);</div><div class='add'>+		sqe-&gt;wqe_size = wqe_sz;</div><div class='add'>+		sqe-&gt;inv_key_or_imm_data = cpu_to_le32(wqe-&gt;send.inv_key);</div><div class='ctx'> 		if (qp-&gt;type == CMDQ_CREATE_QP_TYPE_UD ||</div><div class='ctx'> 		    qp-&gt;type == CMDQ_CREATE_QP_TYPE_GSI) {</div><div class='ctx'> 			sqe-&gt;q_key = cpu_to_le32(wqe-&gt;send.q_key);</div><div class='del'>-			sqe-&gt;dst_qp = cpu_to_le32(</div><div class='del'>-					wqe-&gt;send.dst_qp &amp; SQ_SEND_DST_QP_MASK);</div><div class='ctx'> 			sqe-&gt;length = cpu_to_le32(data_len);</div><div class='del'>-			sqe-&gt;avid = cpu_to_le32(wqe-&gt;send.avid &amp;</div><div class='del'>-						SQ_SEND_AVID_MASK);</div><div class='ctx'> 			sq-&gt;psn = (sq-&gt;psn + 1) &amp; BTH_PSN_MASK;</div><div class='add'>+			ext_sqe-&gt;dst_qp = cpu_to_le32(wqe-&gt;send.dst_qp &amp;</div><div class='add'>+						      SQ_SEND_DST_QP_MASK);</div><div class='add'>+			ext_sqe-&gt;avid = cpu_to_le32(wqe-&gt;send.avid &amp;</div><div class='add'>+						    SQ_SEND_AVID_MASK);</div><div class='add'>+			msn_update = false;</div><div class='ctx'> 		} else {</div><div class='ctx'> 			sqe-&gt;length = cpu_to_le32(data_len);</div><div class='del'>-			sqe-&gt;dst_qp = 0;</div><div class='del'>-			sqe-&gt;avid = 0;</div><div class='ctx'> 			if (qp-&gt;mtu)</div><div class='ctx'> 				pkt_num = (data_len + qp-&gt;mtu - 1) / qp-&gt;mtu;</div><div class='ctx'> 			if (!pkt_num)</div><div class='hunk'>@@ -1651,16 +1975,16 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,</div><div class='ctx'> 	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:</div><div class='ctx'> 	case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:</div><div class='ctx'> 	{</div><div class='del'>-		struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr;</div><div class='add'>+		struct sq_rdma_ext_hdr *ext_sqe = ext_hdr;</div><div class='add'>+		struct sq_rdma_hdr *sqe = base_hdr;</div><div class='ctx'> </div><div class='ctx'> 		sqe-&gt;wqe_type = wqe-&gt;type;</div><div class='ctx'> 		sqe-&gt;flags = wqe-&gt;flags;</div><div class='del'>-		sqe-&gt;wqe_size = wqe_size16 +</div><div class='del'>-				((offsetof(typeof(*sqe), data) + 15) &gt;&gt; 4);</div><div class='add'>+		sqe-&gt;wqe_size = wqe_sz;</div><div class='ctx'> 		sqe-&gt;imm_data = cpu_to_le32(wqe-&gt;rdma.inv_key);</div><div class='ctx'> 		sqe-&gt;length = cpu_to_le32((u32)data_len);</div><div class='del'>-		sqe-&gt;remote_va = cpu_to_le64(wqe-&gt;rdma.remote_va);</div><div class='del'>-		sqe-&gt;remote_key = cpu_to_le32(wqe-&gt;rdma.r_key);</div><div class='add'>+		ext_sqe-&gt;remote_va = cpu_to_le64(wqe-&gt;rdma.remote_va);</div><div class='add'>+		ext_sqe-&gt;remote_key = cpu_to_le32(wqe-&gt;rdma.r_key);</div><div class='ctx'> 		if (qp-&gt;mtu)</div><div class='ctx'> 			pkt_num = (data_len + qp-&gt;mtu - 1) / qp-&gt;mtu;</div><div class='ctx'> 		if (!pkt_num)</div><div class='hunk'>@@ -1671,14 +1995,15 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,</div><div class='ctx'> 	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:</div><div class='ctx'> 	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:</div><div class='ctx'> 	{</div><div class='del'>-		struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr;</div><div class='add'>+		struct sq_atomic_ext_hdr *ext_sqe = ext_hdr;</div><div class='add'>+		struct sq_atomic_hdr *sqe = base_hdr;</div><div class='ctx'> </div><div class='ctx'> 		sqe-&gt;wqe_type = wqe-&gt;type;</div><div class='ctx'> 		sqe-&gt;flags = wqe-&gt;flags;</div><div class='ctx'> 		sqe-&gt;remote_key = cpu_to_le32(wqe-&gt;atomic.r_key);</div><div class='ctx'> 		sqe-&gt;remote_va = cpu_to_le64(wqe-&gt;atomic.remote_va);</div><div class='del'>-		sqe-&gt;swap_data = cpu_to_le64(wqe-&gt;atomic.swap_data);</div><div class='del'>-		sqe-&gt;cmp_data = cpu_to_le64(wqe-&gt;atomic.cmp_data);</div><div class='add'>+		ext_sqe-&gt;swap_data = cpu_to_le64(wqe-&gt;atomic.swap_data);</div><div class='add'>+		ext_sqe-&gt;cmp_data = cpu_to_le64(wqe-&gt;atomic.cmp_data);</div><div class='ctx'> 		if (qp-&gt;mtu)</div><div class='ctx'> 			pkt_num = (data_len + qp-&gt;mtu - 1) / qp-&gt;mtu;</div><div class='ctx'> 		if (!pkt_num)</div><div class='hunk'>@@ -1688,18 +2013,18 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,</div><div class='ctx'> 	}</div><div class='ctx'> 	case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:</div><div class='ctx'> 	{</div><div class='del'>-		struct sq_localinvalidate *sqe =</div><div class='del'>-				(struct sq_localinvalidate *)hw_sq_send_hdr;</div><div class='add'>+		struct sq_localinvalidate *sqe = base_hdr;</div><div class='ctx'> </div><div class='ctx'> 		sqe-&gt;wqe_type = wqe-&gt;type;</div><div class='ctx'> 		sqe-&gt;flags = wqe-&gt;flags;</div><div class='ctx'> 		sqe-&gt;inv_l_key = cpu_to_le32(wqe-&gt;local_inv.inv_l_key);</div><div class='del'>-</div><div class='add'>+		msn_update = false;</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='ctx'> 	case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:</div><div class='ctx'> 	{</div><div class='del'>-		struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr;</div><div class='add'>+		struct sq_fr_pmr_ext_hdr *ext_sqe = ext_hdr;</div><div class='add'>+		struct sq_fr_pmr_hdr *sqe = base_hdr;</div><div class='ctx'> </div><div class='ctx'> 		sqe-&gt;wqe_type = wqe-&gt;type;</div><div class='ctx'> 		sqe-&gt;flags = wqe-&gt;flags;</div><div class='hunk'>@@ -1723,14 +2048,16 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,</div><div class='ctx'> 			wqe-&gt;frmr.pbl_ptr[i] = cpu_to_le64(</div><div class='ctx'> 						wqe-&gt;frmr.page_list[i] |</div><div class='ctx'> 						PTU_PTE_VALID);</div><div class='del'>-		sqe-&gt;pblptr = cpu_to_le64(wqe-&gt;frmr.pbl_dma_ptr);</div><div class='del'>-		sqe-&gt;va = cpu_to_le64(wqe-&gt;frmr.va);</div><div class='add'>+		ext_sqe-&gt;pblptr = cpu_to_le64(wqe-&gt;frmr.pbl_dma_ptr);</div><div class='add'>+		ext_sqe-&gt;va = cpu_to_le64(wqe-&gt;frmr.va);</div><div class='add'>+		msn_update = false;</div><div class='ctx'> </div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='ctx'> 	case BNXT_QPLIB_SWQE_TYPE_BIND_MW:</div><div class='ctx'> 	{</div><div class='del'>-		struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr;</div><div class='add'>+		struct sq_bind_ext_hdr *ext_sqe = ext_hdr;</div><div class='add'>+		struct sq_bind_hdr *sqe = base_hdr;</div><div class='ctx'> </div><div class='ctx'> 		sqe-&gt;wqe_type = wqe-&gt;type;</div><div class='ctx'> 		sqe-&gt;flags = wqe-&gt;flags;</div><div class='hunk'>@@ -1739,9 +2066,9 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,</div><div class='ctx'> 			(wqe-&gt;bind.zero_based ? SQ_BIND_ZERO_BASED : 0);</div><div class='ctx'> 		sqe-&gt;parent_l_key = cpu_to_le32(wqe-&gt;bind.parent_l_key);</div><div class='ctx'> 		sqe-&gt;l_key = cpu_to_le32(wqe-&gt;bind.r_key);</div><div class='del'>-		sqe-&gt;va = cpu_to_le64(wqe-&gt;bind.va);</div><div class='del'>-		temp32 = cpu_to_le32(wqe-&gt;bind.length);</div><div class='del'>-		memcpy(&amp;sqe-&gt;length, &amp;temp32, sizeof(wqe-&gt;bind.length));</div><div class='add'>+		ext_sqe-&gt;va = cpu_to_le64(wqe-&gt;bind.va);</div><div class='add'>+		ext_sqe-&gt;length_lo = cpu_to_le32(wqe-&gt;bind.length);</div><div class='add'>+		msn_update = false;</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='ctx'> 	default:</div><div class='hunk'>@@ -1749,44 +2076,14 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,</div><div class='ctx'> 		rc = -EINVAL;</div><div class='ctx'> 		goto done;</div><div class='ctx'> 	}</div><div class='del'>-	swq-&gt;next_psn = sq-&gt;psn &amp; BTH_PSN_MASK;</div><div class='del'>-	if (swq-&gt;psn_search) {</div><div class='del'>-		u32 opcd_spsn;</div><div class='del'>-		u32 flg_npsn;</div><div class='del'>-</div><div class='del'>-		opcd_spsn = ((swq-&gt;start_psn &lt;&lt; SQ_PSN_SEARCH_START_PSN_SFT) &amp;</div><div class='del'>-			      SQ_PSN_SEARCH_START_PSN_MASK);</div><div class='del'>-		opcd_spsn |= ((wqe-&gt;type &lt;&lt; SQ_PSN_SEARCH_OPCODE_SFT) &amp;</div><div class='del'>-			       SQ_PSN_SEARCH_OPCODE_MASK);</div><div class='del'>-		flg_npsn = ((swq-&gt;next_psn &lt;&lt; SQ_PSN_SEARCH_NEXT_PSN_SFT) &amp;</div><div class='del'>-			     SQ_PSN_SEARCH_NEXT_PSN_MASK);</div><div class='del'>-		if (bnxt_qplib_is_chip_gen_p5(qp-&gt;cctx)) {</div><div class='del'>-			swq-&gt;psn_ext-&gt;opcode_start_psn =</div><div class='del'>-						cpu_to_le32(opcd_spsn);</div><div class='del'>-			swq-&gt;psn_ext-&gt;flags_next_psn =</div><div class='del'>-						cpu_to_le32(flg_npsn);</div><div class='del'>-		} else {</div><div class='del'>-			swq-&gt;psn_search-&gt;opcode_start_psn =</div><div class='del'>-						cpu_to_le32(opcd_spsn);</div><div class='del'>-			swq-&gt;psn_search-&gt;flags_next_psn =</div><div class='del'>-						cpu_to_le32(flg_npsn);</div><div class='del'>-		}</div><div class='add'>+	if (!qp-&gt;is_host_msn_tbl || msn_update) {</div><div class='add'>+		swq-&gt;next_psn = sq-&gt;psn &amp; BTH_PSN_MASK;</div><div class='add'>+		bnxt_qplib_fill_psn_search(qp, wqe, swq);</div><div class='ctx'> 	}</div><div class='ctx'> queue_err:</div><div class='del'>-	if (sch_handler) {</div><div class='del'>-		/* Store the ULP info in the software structures */</div><div class='del'>-		sw_prod = HWQ_CMP(sq-&gt;hwq.prod, &amp;sq-&gt;hwq);</div><div class='del'>-		swq = &amp;sq-&gt;swq[sw_prod];</div><div class='del'>-		swq-&gt;wr_id = wqe-&gt;wr_id;</div><div class='del'>-		swq-&gt;type = wqe-&gt;type;</div><div class='del'>-		swq-&gt;flags = wqe-&gt;flags;</div><div class='del'>-		if (qp-&gt;sig_type)</div><div class='del'>-			swq-&gt;flags |= SQ_SEND_FLAGS_SIGNAL_COMP;</div><div class='del'>-		swq-&gt;start_psn = sq-&gt;psn &amp; BTH_PSN_MASK;</div><div class='del'>-	}</div><div class='del'>-	sq-&gt;hwq.prod++;</div><div class='add'>+	bnxt_qplib_swq_mod_start(sq, wqe_idx);</div><div class='add'>+	bnxt_qplib_hwq_incr_prod(&amp;sq-&gt;dbinfo, hwq, swq-&gt;slots);</div><div class='ctx'> 	qp-&gt;wqe_cnt++;</div><div class='del'>-</div><div class='ctx'> done:</div><div class='ctx'> 	if (sch_handler) {</div><div class='ctx'> 		nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);</div><div class='hunk'>@@ -1796,7 +2093,7 @@ done:</div><div class='ctx'> 			INIT_WORK(&amp;nq_work-&gt;work, bnxt_qpn_cqn_sched_task);</div><div class='ctx'> 			queue_work(qp-&gt;scq-&gt;nq-&gt;cqn_wq, &amp;nq_work-&gt;work);</div><div class='ctx'> 		} else {</div><div class='del'>-			dev_err(&amp;sq-&gt;hwq.pdev-&gt;dev,</div><div class='add'>+			dev_err(&amp;hwq-&gt;pdev-&gt;dev,</div><div class='ctx'> 				"FP: Failed to allocate SQ nq_work!\n");</div><div class='ctx'> 			rc = -ENOMEM;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -1807,77 +2104,74 @@ done:</div><div class='ctx'> void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_q *rq = &amp;qp-&gt;rq;</div><div class='del'>-	u32 sw_prod;</div><div class='del'>-	u64 val = 0;</div><div class='ctx'> </div><div class='del'>-	val = (((qp-&gt;id &lt;&lt; DBC_DBC_XID_SFT) &amp; DBC_DBC_XID_MASK) |</div><div class='del'>-	       DBC_DBC_TYPE_RQ);</div><div class='del'>-	val &lt;&lt;= 32;</div><div class='del'>-	sw_prod = HWQ_CMP(rq-&gt;hwq.prod, &amp;rq-&gt;hwq);</div><div class='del'>-	val |= (sw_prod &lt;&lt; DBC_DBC_INDEX_SFT) &amp; DBC_DBC_INDEX_MASK;</div><div class='del'>-	/* Flush the writes to HW Rx WQE before the ringing Rx DB */</div><div class='del'>-	writeq(val, qp-&gt;dpi-&gt;dbr);</div><div class='add'>+	bnxt_qplib_ring_prod_db(&amp;rq-&gt;dbinfo, DBC_DBC_TYPE_RQ);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,</div><div class='ctx'> 			 struct bnxt_qplib_swqe *wqe)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_qplib_q *rq = &amp;qp-&gt;rq;</div><div class='del'>-	struct rq_wqe *rqe, **rqe_ptr;</div><div class='del'>-	struct sq_sge *hw_sge;</div><div class='ctx'> 	struct bnxt_qplib_nq_work *nq_work = NULL;</div><div class='add'>+	struct bnxt_qplib_q *rq = &amp;qp-&gt;rq;</div><div class='add'>+	struct rq_wqe_hdr *base_hdr;</div><div class='add'>+	struct rq_ext_hdr *ext_hdr;</div><div class='add'>+	struct bnxt_qplib_hwq *hwq;</div><div class='add'>+	struct bnxt_qplib_swq *swq;</div><div class='ctx'> 	bool sch_handler = false;</div><div class='del'>-	u32 sw_prod;</div><div class='del'>-	int i, rc = 0;</div><div class='add'>+	u32 wqe_idx, idx;</div><div class='add'>+	u16 wqe_sz;</div><div class='add'>+	int rc = 0;</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {</div><div class='del'>-		sch_handler = true;</div><div class='del'>-		dev_dbg(&amp;rq-&gt;hwq.pdev-&gt;dev,</div><div class='del'>-			"%s: Error QP. Scheduling for poll_cq\n", __func__);</div><div class='del'>-		goto queue_err;</div><div class='add'>+	hwq = &amp;rq-&gt;hwq;</div><div class='add'>+	if (qp-&gt;state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {</div><div class='add'>+		dev_err(&amp;hwq-&gt;pdev-&gt;dev,</div><div class='add'>+			"QPLIB: FP: QP (0x%x) is in the 0x%x state",</div><div class='add'>+			qp-&gt;id, qp-&gt;state);</div><div class='add'>+		rc = -EINVAL;</div><div class='add'>+		goto done;</div><div class='ctx'> 	}</div><div class='del'>-	if (bnxt_qplib_queue_full(rq)) {</div><div class='del'>-		dev_err(&amp;rq-&gt;hwq.pdev-&gt;dev,</div><div class='add'>+</div><div class='add'>+	if (bnxt_qplib_queue_full(rq, rq-&gt;dbinfo.max_slot)) {</div><div class='add'>+		dev_err(&amp;hwq-&gt;pdev-&gt;dev,</div><div class='ctx'> 			"FP: QP (0x%x) RQ is full!\n", qp-&gt;id);</div><div class='ctx'> 		rc = -EINVAL;</div><div class='ctx'> 		goto done;</div><div class='ctx'> 	}</div><div class='del'>-	sw_prod = HWQ_CMP(rq-&gt;hwq.prod, &amp;rq-&gt;hwq);</div><div class='del'>-	rq-&gt;swq[sw_prod].wr_id = wqe-&gt;wr_id;</div><div class='del'>-</div><div class='del'>-	rqe_ptr = (struct rq_wqe **)rq-&gt;hwq.pbl_ptr;</div><div class='del'>-	rqe = &amp;rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];</div><div class='ctx'> </div><div class='del'>-	memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);</div><div class='add'>+	swq = bnxt_qplib_get_swqe(rq, &amp;wqe_idx);</div><div class='add'>+	swq-&gt;wr_id = wqe-&gt;wr_id;</div><div class='add'>+	swq-&gt;slots = rq-&gt;dbinfo.max_slot;</div><div class='ctx'> </div><div class='del'>-	/* Calculate wqe_size16 and data_len */</div><div class='del'>-	for (i = 0, hw_sge = (struct sq_sge *)rqe-&gt;data;</div><div class='del'>-	     i &lt; wqe-&gt;num_sge; i++, hw_sge++) {</div><div class='del'>-		hw_sge-&gt;va_or_pa = cpu_to_le64(wqe-&gt;sg_list[i].addr);</div><div class='del'>-		hw_sge-&gt;l_key = cpu_to_le32(wqe-&gt;sg_list[i].lkey);</div><div class='del'>-		hw_sge-&gt;size = cpu_to_le32(wqe-&gt;sg_list[i].size);</div><div class='add'>+	if (qp-&gt;state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {</div><div class='add'>+		sch_handler = true;</div><div class='add'>+		dev_dbg(&amp;hwq-&gt;pdev-&gt;dev,</div><div class='add'>+			"%s: Error QP. Scheduling for poll_cq\n", __func__);</div><div class='add'>+		goto queue_err;</div><div class='ctx'> 	}</div><div class='del'>-	rqe-&gt;wqe_type = wqe-&gt;type;</div><div class='del'>-	rqe-&gt;flags = wqe-&gt;flags;</div><div class='del'>-	rqe-&gt;wqe_size = wqe-&gt;num_sge +</div><div class='del'>-			((offsetof(typeof(*rqe), data) + 15) &gt;&gt; 4);</div><div class='del'>-	/* HW requires wqe size has room for atleast one SGE even if none</div><div class='del'>-	 * was supplied by ULP</div><div class='del'>-	 */</div><div class='del'>-	if (!wqe-&gt;num_sge)</div><div class='del'>-		rqe-&gt;wqe_size++;</div><div class='del'>-</div><div class='del'>-	/* Supply the rqe-&gt;wr_id index to the wr_id_tbl for now */</div><div class='del'>-	rqe-&gt;wr_id[0] = cpu_to_le32(sw_prod);</div><div class='ctx'> </div><div class='add'>+	idx = 0;</div><div class='add'>+	base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);</div><div class='add'>+	ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);</div><div class='add'>+	memset(base_hdr, 0, sizeof(struct sq_sge));</div><div class='add'>+	memset(ext_hdr, 0, sizeof(struct sq_sge));</div><div class='add'>+	wqe_sz = (sizeof(struct rq_wqe_hdr) +</div><div class='add'>+	wqe-&gt;num_sge * sizeof(struct sq_sge)) &gt;&gt; 4;</div><div class='add'>+	bnxt_qplib_put_sges(hwq, wqe-&gt;sg_list, wqe-&gt;num_sge, &amp;idx);</div><div class='add'>+	if (!wqe-&gt;num_sge) {</div><div class='add'>+		struct sq_sge *sge;</div><div class='add'>+</div><div class='add'>+		sge = bnxt_qplib_get_prod_qe(hwq, idx++);</div><div class='add'>+		sge-&gt;size = 0;</div><div class='add'>+		wqe_sz++;</div><div class='add'>+	}</div><div class='add'>+	base_hdr-&gt;wqe_type = wqe-&gt;type;</div><div class='add'>+	base_hdr-&gt;flags = wqe-&gt;flags;</div><div class='add'>+	base_hdr-&gt;wqe_size = wqe_sz;</div><div class='add'>+	base_hdr-&gt;wr_id[0] = cpu_to_le32(wqe_idx);</div><div class='ctx'> queue_err:</div><div class='del'>-	if (sch_handler) {</div><div class='del'>-		/* Store the ULP info in the software structures */</div><div class='del'>-		sw_prod = HWQ_CMP(rq-&gt;hwq.prod, &amp;rq-&gt;hwq);</div><div class='del'>-		rq-&gt;swq[sw_prod].wr_id = wqe-&gt;wr_id;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	rq-&gt;hwq.prod++;</div><div class='add'>+	bnxt_qplib_swq_mod_start(rq, wqe_idx);</div><div class='add'>+	bnxt_qplib_hwq_incr_prod(&amp;rq-&gt;dbinfo, hwq, swq-&gt;slots);</div><div class='add'>+done:</div><div class='ctx'> 	if (sch_handler) {</div><div class='ctx'> 		nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);</div><div class='ctx'> 		if (nq_work) {</div><div class='hunk'>@@ -1886,97 +2180,87 @@ queue_err:</div><div class='ctx'> 			INIT_WORK(&amp;nq_work-&gt;work, bnxt_qpn_cqn_sched_task);</div><div class='ctx'> 			queue_work(qp-&gt;rcq-&gt;nq-&gt;cqn_wq, &amp;nq_work-&gt;work);</div><div class='ctx'> 		} else {</div><div class='del'>-			dev_err(&amp;rq-&gt;hwq.pdev-&gt;dev,</div><div class='add'>+			dev_err(&amp;hwq-&gt;pdev-&gt;dev,</div><div class='ctx'> 				"FP: Failed to allocate RQ nq_work!\n");</div><div class='ctx'> 			rc = -ENOMEM;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='del'>-done:</div><div class='add'>+</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* CQ */</div><div class='del'>-</div><div class='del'>-/* Spinlock must be held */</div><div class='del'>-static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq)</div><div class='del'>-{</div><div class='del'>-	u64 val = 0;</div><div class='del'>-</div><div class='del'>-	val = ((cq-&gt;id &lt;&lt; DBC_DBC_XID_SFT) &amp; DBC_DBC_XID_MASK) |</div><div class='del'>-	       DBC_DBC_TYPE_CQ_ARMENA;</div><div class='del'>-	val &lt;&lt;= 32;</div><div class='del'>-	/* Flush memory writes before enabling the CQ */</div><div class='del'>-	writeq(val, cq-&gt;dbr_base);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type)</div><div class='del'>-{</div><div class='del'>-	struct bnxt_qplib_hwq *cq_hwq = &amp;cq-&gt;hwq;</div><div class='del'>-	u32 sw_cons;</div><div class='del'>-	u64 val = 0;</div><div class='del'>-</div><div class='del'>-	/* Ring DB */</div><div class='del'>-	val = ((cq-&gt;id &lt;&lt; DBC_DBC_XID_SFT) &amp; DBC_DBC_XID_MASK) | arm_type;</div><div class='del'>-	val &lt;&lt;= 32;</div><div class='del'>-	sw_cons = HWQ_CMP(cq_hwq-&gt;cons, cq_hwq);</div><div class='del'>-	val |= (sw_cons &lt;&lt; DBC_DBC_INDEX_SFT) &amp; DBC_DBC_INDEX_MASK;</div><div class='del'>-	/* flush memory writes before arming the CQ */</div><div class='del'>-	writeq(val, cq-&gt;dpi-&gt;dbr);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct cmdq_create_cq req;</div><div class='del'>-	struct creq_create_cq_resp resp;</div><div class='add'>+	struct bnxt_qplib_hwq_attr hwq_attr = {};</div><div class='add'>+	struct creq_create_cq_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_create_cq req = {};</div><div class='ctx'> 	struct bnxt_qplib_pbl *pbl;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='add'>+	u32 coalescing = 0;</div><div class='add'>+	u32 pg_sz_lvl;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	cq-&gt;hwq.max_elements = cq-&gt;max_wqe;</div><div class='del'>-	rc = bnxt_qplib_alloc_init_hwq(res-&gt;pdev, &amp;cq-&gt;hwq, &amp;cq-&gt;sg_info,</div><div class='del'>-				       &amp;cq-&gt;hwq.max_elements,</div><div class='del'>-				       BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0,</div><div class='del'>-				       PAGE_SIZE, HWQ_TYPE_QUEUE);</div><div class='del'>-	if (rc)</div><div class='del'>-		goto exit;</div><div class='del'>-</div><div class='del'>-	RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags);</div><div class='del'>-</div><div class='ctx'> 	if (!cq-&gt;dpi) {</div><div class='ctx'> 		dev_err(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='ctx'> 			"FP: CREATE_CQ failed due to NULL DPI\n");</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	cq-&gt;dbinfo.flags = 0;</div><div class='add'>+	hwq_attr.res = res;</div><div class='add'>+	hwq_attr.depth = cq-&gt;max_wqe;</div><div class='add'>+	hwq_attr.stride = sizeof(struct cq_base);</div><div class='add'>+	hwq_attr.type = HWQ_TYPE_QUEUE;</div><div class='add'>+	hwq_attr.sginfo = &amp;cq-&gt;sg_info;</div><div class='add'>+	rc = bnxt_qplib_alloc_init_hwq(&amp;cq-&gt;hwq, &amp;hwq_attr);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_CREATE_CQ,</div><div class='add'>+				 sizeof(req));</div><div class='add'>+</div><div class='ctx'> 	req.dpi = cpu_to_le32(cq-&gt;dpi-&gt;dpi);</div><div class='ctx'> 	req.cq_handle = cpu_to_le64(cq-&gt;cq_handle);</div><div class='add'>+	req.cq_size = cpu_to_le32(cq-&gt;max_wqe);</div><div class='add'>+</div><div class='add'>+	if (_is_cq_coalescing_supported(res-&gt;dattr-&gt;dev_cap_flags2) &amp;&amp;</div><div class='add'>+	    cq-&gt;coalescing-&gt;enable) {</div><div class='add'>+		req.flags |= cpu_to_le16(CMDQ_CREATE_CQ_FLAGS_COALESCING_VALID);</div><div class='add'>+		coalescing |= ((cq-&gt;coalescing-&gt;buf_maxtime &lt;&lt;</div><div class='add'>+				CMDQ_CREATE_CQ_BUF_MAXTIME_SFT) &amp;</div><div class='add'>+			       CMDQ_CREATE_CQ_BUF_MAXTIME_MASK);</div><div class='add'>+		coalescing |= ((cq-&gt;coalescing-&gt;normal_maxbuf &lt;&lt;</div><div class='add'>+				CMDQ_CREATE_CQ_NORMAL_MAXBUF_SFT) &amp;</div><div class='add'>+			       CMDQ_CREATE_CQ_NORMAL_MAXBUF_MASK);</div><div class='add'>+		coalescing |= ((cq-&gt;coalescing-&gt;during_maxbuf &lt;&lt;</div><div class='add'>+				CMDQ_CREATE_CQ_DURING_MAXBUF_SFT) &amp;</div><div class='add'>+			       CMDQ_CREATE_CQ_DURING_MAXBUF_MASK);</div><div class='add'>+		if (cq-&gt;coalescing-&gt;en_ring_idle_mode)</div><div class='add'>+			coalescing |= CMDQ_CREATE_CQ_ENABLE_RING_IDLE_MODE;</div><div class='add'>+		else</div><div class='add'>+			coalescing &amp;= ~CMDQ_CREATE_CQ_ENABLE_RING_IDLE_MODE;</div><div class='add'>+		req.coalescing = cpu_to_le32(coalescing);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	req.cq_size = cpu_to_le32(cq-&gt;hwq.max_elements);</div><div class='ctx'> 	pbl = &amp;cq-&gt;hwq.pbl[PBL_LVL_0];</div><div class='del'>-	req.pg_size_lvl = cpu_to_le32(</div><div class='del'>-	    ((cq-&gt;hwq.level &amp; CMDQ_CREATE_CQ_LVL_MASK) &lt;&lt;</div><div class='del'>-						CMDQ_CREATE_CQ_LVL_SFT) |</div><div class='del'>-	    (pbl-&gt;pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K :</div><div class='del'>-	     pbl-&gt;pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K :</div><div class='del'>-	     pbl-&gt;pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K :</div><div class='del'>-	     pbl-&gt;pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M :</div><div class='del'>-	     pbl-&gt;pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M :</div><div class='del'>-	     pbl-&gt;pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G :</div><div class='del'>-	     CMDQ_CREATE_CQ_PG_SIZE_PG_4K));</div><div class='del'>-</div><div class='add'>+	pg_sz_lvl = (bnxt_qplib_base_pg_size(&amp;cq-&gt;hwq) &lt;&lt;</div><div class='add'>+		     CMDQ_CREATE_CQ_PG_SIZE_SFT);</div><div class='add'>+	pg_sz_lvl |= (cq-&gt;hwq.level &amp; CMDQ_CREATE_CQ_LVL_MASK);</div><div class='add'>+	req.pg_size_lvl = cpu_to_le32(pg_sz_lvl);</div><div class='ctx'> 	req.pbl = cpu_to_le64(pbl-&gt;pg_map_arr[0]);</div><div class='del'>-</div><div class='ctx'> 	req.cq_fco_cnq_id = cpu_to_le32(</div><div class='ctx'> 			(cq-&gt;cnq_hw_ring_id &amp; CMDQ_CREATE_CQ_CNQ_ID_MASK) &lt;&lt;</div><div class='ctx'> 			 CMDQ_CREATE_CQ_CNQ_ID_SFT);</div><div class='del'>-</div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req,</div><div class='del'>-					  (void *)&amp;resp, NULL, 0);</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='ctx'> 	cq-&gt;id = le32_to_cpu(resp.xid);</div><div class='del'>-	cq-&gt;dbr_base = res-&gt;dpi_tbl.dbr_bar_reg_iomem;</div><div class='ctx'> 	cq-&gt;period = BNXT_QPLIB_QUEUE_START_PERIOD;</div><div class='ctx'> 	init_waitqueue_head(&amp;cq-&gt;waitq);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;cq-&gt;sqf_head);</div><div class='hunk'>@@ -1984,51 +2268,112 @@ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)</div><div class='ctx'> 	spin_lock_init(&amp;cq-&gt;compl_lock);</div><div class='ctx'> 	spin_lock_init(&amp;cq-&gt;flush_lock);</div><div class='ctx'> </div><div class='del'>-	bnxt_qplib_arm_cq_enable(cq);</div><div class='add'>+	cq-&gt;dbinfo.hwq = &amp;cq-&gt;hwq;</div><div class='add'>+	cq-&gt;dbinfo.xid = cq-&gt;id;</div><div class='add'>+	cq-&gt;dbinfo.db = cq-&gt;dpi-&gt;dbr;</div><div class='add'>+	cq-&gt;dbinfo.priv_db = res-&gt;dpi_tbl.priv_db;</div><div class='add'>+	cq-&gt;dbinfo.flags = 0;</div><div class='add'>+	cq-&gt;dbinfo.toggle = 0;</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_armen_db(&amp;cq-&gt;dbinfo, DBC_DBC_TYPE_CQ_ARMENA);</div><div class='add'>+</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> fail:</div><div class='del'>-	bnxt_qplib_free_hwq(res-&gt;pdev, &amp;cq-&gt;hwq);</div><div class='del'>-exit:</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;cq-&gt;hwq);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res,</div><div class='add'>+				   struct bnxt_qplib_cq *cq)</div><div class='add'>+{</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;cq-&gt;hwq);</div><div class='add'>+	memcpy(&amp;cq-&gt;hwq, &amp;cq-&gt;resize_hwq, sizeof(cq-&gt;hwq));</div><div class='add'>+       /* Reset only the cons bit in the flags */</div><div class='add'>+	cq-&gt;dbinfo.flags &amp;= ~(1UL &lt;&lt; BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq,</div><div class='add'>+			 int new_cqes)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_hwq_attr hwq_attr = {};</div><div class='add'>+	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='add'>+	struct creq_resize_cq_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_resize_cq req = {};</div><div class='add'>+	struct bnxt_qplib_pbl *pbl;</div><div class='add'>+	u32 pg_sz, lvl, new_sz;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_RESIZE_CQ,</div><div class='add'>+				 sizeof(req));</div><div class='add'>+	hwq_attr.sginfo = &amp;cq-&gt;sg_info;</div><div class='add'>+	hwq_attr.res = res;</div><div class='add'>+	hwq_attr.depth = new_cqes;</div><div class='add'>+	hwq_attr.stride = sizeof(struct cq_base);</div><div class='add'>+	hwq_attr.type = HWQ_TYPE_QUEUE;</div><div class='add'>+	rc = bnxt_qplib_alloc_init_hwq(&amp;cq-&gt;resize_hwq, &amp;hwq_attr);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	req.cq_cid = cpu_to_le32(cq-&gt;id);</div><div class='add'>+	pbl = &amp;cq-&gt;resize_hwq.pbl[PBL_LVL_0];</div><div class='add'>+	pg_sz = bnxt_qplib_base_pg_size(&amp;cq-&gt;resize_hwq);</div><div class='add'>+	lvl = (cq-&gt;resize_hwq.level &lt;&lt; CMDQ_RESIZE_CQ_LVL_SFT) &amp;</div><div class='add'>+				       CMDQ_RESIZE_CQ_LVL_MASK;</div><div class='add'>+	new_sz = (new_cqes &lt;&lt; CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT) &amp;</div><div class='add'>+		  CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK;</div><div class='add'>+	req.new_cq_size_pg_size_lvl = cpu_to_le32(new_sz | pg_sz | lvl);</div><div class='add'>+	req.new_pbl = cpu_to_le64(pbl-&gt;pg_map_arr[0]);</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct cmdq_destroy_cq req;</div><div class='del'>-	struct creq_destroy_cq_resp resp;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='add'>+	struct creq_destroy_cq_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_destroy_cq req = {};</div><div class='add'>+	u16 total_cnq_events;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags);</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_DESTROY_CQ,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='ctx'> 	req.cq_cid = cpu_to_le32(cq-&gt;id);</div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req,</div><div class='del'>-					  (void *)&amp;resp, NULL, 0);</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		return rc;</div><div class='del'>-	bnxt_qplib_free_hwq(res-&gt;pdev, &amp;cq-&gt;hwq);</div><div class='add'>+	total_cnq_events = le16_to_cpu(resp.total_cnq_events);</div><div class='add'>+	__wait_for_all_nqes(cq, total_cnq_events);</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;cq-&gt;hwq);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,</div><div class='ctx'> 		      struct bnxt_qplib_cqe **pcqe, int *budget)</div><div class='ctx'> {</div><div class='del'>-	u32 sw_prod, sw_cons;</div><div class='ctx'> 	struct bnxt_qplib_cqe *cqe;</div><div class='add'>+	u32 start, last;</div><div class='ctx'> 	int rc = 0;</div><div class='ctx'> </div><div class='ctx'> 	/* Now complete all outstanding SQEs with FLUSHED_ERR */</div><div class='del'>-	sw_prod = HWQ_CMP(sq-&gt;hwq.prod, &amp;sq-&gt;hwq);</div><div class='add'>+	start = sq-&gt;swq_start;</div><div class='ctx'> 	cqe = *pcqe;</div><div class='ctx'> 	while (*budget) {</div><div class='del'>-		sw_cons = HWQ_CMP(sq-&gt;hwq.cons, &amp;sq-&gt;hwq);</div><div class='del'>-		if (sw_cons == sw_prod) {</div><div class='add'>+		last = sq-&gt;swq_last;</div><div class='add'>+		if (start == last)</div><div class='ctx'> 			break;</div><div class='del'>-		}</div><div class='ctx'> 		/* Skip the FENCE WQE completions */</div><div class='del'>-		if (sq-&gt;swq[sw_cons].wr_id == BNXT_QPLIB_FENCE_WRID) {</div><div class='add'>+		if (sq-&gt;swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) {</div><div class='ctx'> 			bnxt_qplib_cancel_phantom_processing(qp);</div><div class='ctx'> 			goto skip_compl;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -2036,16 +2381,18 @@ static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,</div><div class='ctx'> 		cqe-&gt;status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;</div><div class='ctx'> 		cqe-&gt;opcode = CQ_BASE_CQE_TYPE_REQ;</div><div class='ctx'> 		cqe-&gt;qp_handle = (u64)(unsigned long)qp;</div><div class='del'>-		cqe-&gt;wr_id = sq-&gt;swq[sw_cons].wr_id;</div><div class='add'>+		cqe-&gt;wr_id = sq-&gt;swq[last].wr_id;</div><div class='ctx'> 		cqe-&gt;src_qp = qp-&gt;id;</div><div class='del'>-		cqe-&gt;type = sq-&gt;swq[sw_cons].type;</div><div class='add'>+		cqe-&gt;type = sq-&gt;swq[last].type;</div><div class='ctx'> 		cqe++;</div><div class='ctx'> 		(*budget)--;</div><div class='ctx'> skip_compl:</div><div class='del'>-		sq-&gt;hwq.cons++;</div><div class='add'>+		bnxt_qplib_hwq_incr_cons(sq-&gt;hwq.max_elements, &amp;sq-&gt;hwq.cons,</div><div class='add'>+					 sq-&gt;swq[last].slots, &amp;sq-&gt;dbinfo.flags);</div><div class='add'>+		sq-&gt;swq_last = sq-&gt;swq[last].next_idx;</div><div class='ctx'> 	}</div><div class='ctx'> 	*pcqe = cqe;</div><div class='del'>-	if (!(*budget) &amp;&amp; HWQ_CMP(sq-&gt;hwq.cons, &amp;sq-&gt;hwq) != sw_prod)</div><div class='add'>+	if (!(*budget) &amp;&amp; sq-&gt;swq_last != start)</div><div class='ctx'> 		/* Out of budget */</div><div class='ctx'> 		rc = -EAGAIN;</div><div class='ctx'> </div><div class='hunk'>@@ -2056,9 +2403,9 @@ static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,</div><div class='ctx'> 		      struct bnxt_qplib_cqe **pcqe, int *budget)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_cqe *cqe;</div><div class='del'>-	u32 sw_prod, sw_cons;</div><div class='del'>-	int rc = 0;</div><div class='add'>+	u32 start, last;</div><div class='ctx'> 	int opcode = 0;</div><div class='add'>+	int rc = 0;</div><div class='ctx'> </div><div class='ctx'> 	switch (qp-&gt;type) {</div><div class='ctx'> 	case CMDQ_CREATE_QP1_TYPE_GSI:</div><div class='hunk'>@@ -2074,24 +2421,26 @@ static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Flush the rest of the RQ */</div><div class='del'>-	sw_prod = HWQ_CMP(rq-&gt;hwq.prod, &amp;rq-&gt;hwq);</div><div class='add'>+	start = rq-&gt;swq_start;</div><div class='ctx'> 	cqe = *pcqe;</div><div class='ctx'> 	while (*budget) {</div><div class='del'>-		sw_cons = HWQ_CMP(rq-&gt;hwq.cons, &amp;rq-&gt;hwq);</div><div class='del'>-		if (sw_cons == sw_prod)</div><div class='add'>+		last = rq-&gt;swq_last;</div><div class='add'>+		if (last == start)</div><div class='ctx'> 			break;</div><div class='ctx'> 		memset(cqe, 0, sizeof(*cqe));</div><div class='ctx'> 		cqe-&gt;status =</div><div class='ctx'> 		    CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;</div><div class='ctx'> 		cqe-&gt;opcode = opcode;</div><div class='ctx'> 		cqe-&gt;qp_handle = (unsigned long)qp;</div><div class='del'>-		cqe-&gt;wr_id = rq-&gt;swq[sw_cons].wr_id;</div><div class='add'>+		cqe-&gt;wr_id = rq-&gt;swq[last].wr_id;</div><div class='ctx'> 		cqe++;</div><div class='ctx'> 		(*budget)--;</div><div class='del'>-		rq-&gt;hwq.cons++;</div><div class='add'>+		bnxt_qplib_hwq_incr_cons(rq-&gt;hwq.max_elements, &amp;rq-&gt;hwq.cons,</div><div class='add'>+					 rq-&gt;swq[last].slots, &amp;rq-&gt;dbinfo.flags);</div><div class='add'>+		rq-&gt;swq_last = rq-&gt;swq[last].next_idx;</div><div class='ctx'> 	}</div><div class='ctx'> 	*pcqe = cqe;</div><div class='del'>-	if (!*budget &amp;&amp; HWQ_CMP(rq-&gt;hwq.cons, &amp;rq-&gt;hwq) != sw_prod)</div><div class='add'>+	if (!*budget &amp;&amp; rq-&gt;swq_last != start)</div><div class='ctx'> 		/* Out of budget */</div><div class='ctx'> 		rc = -EAGAIN;</div><div class='ctx'> </div><div class='hunk'>@@ -2114,20 +2463,20 @@ void bnxt_qplib_mark_qp_error(void *qp_handle)</div><div class='ctx'>  *       CQE is track from sw_cq_cons to max_element but valid only if VALID=1</div><div class='ctx'>  */</div><div class='ctx'> static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,</div><div class='del'>-		     u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons)</div><div class='add'>+		     u32 cq_cons, u32 swq_last, u32 cqe_sq_cons)</div><div class='ctx'> {</div><div class='add'>+	u32 peek_sw_cq_cons, peek_sq_cons_idx, peek_flags;</div><div class='ctx'> 	struct bnxt_qplib_q *sq = &amp;qp-&gt;sq;</div><div class='del'>-	struct bnxt_qplib_swq *swq;</div><div class='del'>-	u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx;</div><div class='del'>-	struct cq_base *peek_hwcqe, **peek_hw_cqe_ptr;</div><div class='ctx'> 	struct cq_req *peek_req_hwcqe;</div><div class='ctx'> 	struct bnxt_qplib_qp *peek_qp;</div><div class='ctx'> 	struct bnxt_qplib_q *peek_sq;</div><div class='add'>+	struct bnxt_qplib_swq *swq;</div><div class='add'>+	struct cq_base *peek_hwcqe;</div><div class='ctx'> 	int i, rc = 0;</div><div class='ctx'> </div><div class='ctx'> 	/* Normal mode */</div><div class='ctx'> 	/* Check for the psn_search marking before completing */</div><div class='del'>-	swq = &amp;sq-&gt;swq[sw_sq_cons];</div><div class='add'>+	swq = &amp;sq-&gt;swq[swq_last];</div><div class='ctx'> 	if (swq-&gt;psn_search &amp;&amp;</div><div class='ctx'> 	    le32_to_cpu(swq-&gt;psn_search-&gt;flags_next_psn) &amp; 0x80000000) {</div><div class='ctx'> 		/* Unmark */</div><div class='hunk'>@@ -2136,29 +2485,25 @@ static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,</div><div class='ctx'> 				     &amp; ~0x80000000);</div><div class='ctx'> 		dev_dbg(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='ctx'> 			"FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",</div><div class='del'>-			cq_cons, qp-&gt;id, sw_sq_cons, cqe_sq_cons);</div><div class='add'>+			cq_cons, qp-&gt;id, swq_last, cqe_sq_cons);</div><div class='ctx'> 		sq-&gt;condition = true;</div><div class='ctx'> 		sq-&gt;send_phantom = true;</div><div class='ctx'> </div><div class='ctx'> 		/* TODO: Only ARM if the previous SQE is ARMALL */</div><div class='del'>-		bnxt_qplib_arm_cq(cq, DBC_DBC_TYPE_CQ_ARMALL);</div><div class='del'>-</div><div class='add'>+		bnxt_qplib_ring_db(&amp;cq-&gt;dbinfo, DBC_DBC_TYPE_CQ_ARMALL);</div><div class='ctx'> 		rc = -EAGAIN;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> 	if (sq-&gt;condition) {</div><div class='ctx'> 		/* Peek at the completions */</div><div class='del'>-		peek_raw_cq_cons = cq-&gt;hwq.cons;</div><div class='add'>+		peek_flags = cq-&gt;dbinfo.flags;</div><div class='ctx'> 		peek_sw_cq_cons = cq_cons;</div><div class='ctx'> 		i = cq-&gt;hwq.max_elements;</div><div class='ctx'> 		while (i--) {</div><div class='del'>-			peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &amp;cq-&gt;hwq);</div><div class='del'>-			peek_hw_cqe_ptr = (struct cq_base **)cq-&gt;hwq.pbl_ptr;</div><div class='del'>-			peek_hwcqe = &amp;peek_hw_cqe_ptr[CQE_PG(peek_sw_cq_cons)]</div><div class='del'>-						     [CQE_IDX(peek_sw_cq_cons)];</div><div class='add'>+			peek_hwcqe = bnxt_qplib_get_qe(&amp;cq-&gt;hwq,</div><div class='add'>+						       peek_sw_cq_cons, NULL);</div><div class='ctx'> 			/* If the next hwcqe is VALID */</div><div class='del'>-			if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons,</div><div class='del'>-					  cq-&gt;hwq.max_elements)) {</div><div class='add'>+			if (CQE_CMP_VALID(peek_hwcqe, peek_flags)) {</div><div class='ctx'> 			/*</div><div class='ctx'> 			 * The valid test of the entry must be done first before</div><div class='ctx'> 			 * reading any further.</div><div class='hunk'>@@ -2175,9 +2520,10 @@ static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,</div><div class='ctx'> 						 le64_to_cpu</div><div class='ctx'> 						 (peek_req_hwcqe-&gt;qp_handle));</div><div class='ctx'> 					peek_sq = &amp;peek_qp-&gt;sq;</div><div class='del'>-					peek_sq_cons_idx = HWQ_CMP(le16_to_cpu(</div><div class='del'>-						peek_req_hwcqe-&gt;sq_cons_idx) - 1</div><div class='del'>-						, &amp;sq-&gt;hwq);</div><div class='add'>+					peek_sq_cons_idx =</div><div class='add'>+						((le16_to_cpu(</div><div class='add'>+						  peek_req_hwcqe-&gt;sq_cons_idx)</div><div class='add'>+						  - 1) % sq-&gt;max_wqe);</div><div class='ctx'> 					/* If the hwcqe's sq's wr_id matches */</div><div class='ctx'> 					if (peek_sq == sq &amp;&amp;</div><div class='ctx'> 					    sq-&gt;swq[peek_sq_cons_idx].wr_id ==</div><div class='hunk'>@@ -2200,28 +2546,56 @@ static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,</div><div class='ctx'> 				rc = -EINVAL;</div><div class='ctx'> 				goto out;</div><div class='ctx'> 			}</div><div class='del'>-			peek_sw_cq_cons++;</div><div class='del'>-			peek_raw_cq_cons++;</div><div class='add'>+			bnxt_qplib_hwq_incr_cons(cq-&gt;hwq.max_elements,</div><div class='add'>+						 &amp;peek_sw_cq_cons,</div><div class='add'>+						 1, &amp;peek_flags);</div><div class='ctx'> 		}</div><div class='ctx'> 		dev_err(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='ctx'> 			"Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n",</div><div class='del'>-			cq_cons, qp-&gt;id, sw_sq_cons, cqe_sq_cons);</div><div class='add'>+			cq_cons, qp-&gt;id, swq_last, cqe_sq_cons);</div><div class='ctx'> 		rc = -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> out:</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int bnxt_qplib_get_cqe_sq_cons(struct bnxt_qplib_q *sq, u32 cqe_slot)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_hwq *sq_hwq;</div><div class='add'>+	struct bnxt_qplib_swq *swq;</div><div class='add'>+	int cqe_sq_cons = -1;</div><div class='add'>+	u32 start, last;</div><div class='add'>+</div><div class='add'>+	sq_hwq = &amp;sq-&gt;hwq;</div><div class='add'>+</div><div class='add'>+	start = sq-&gt;swq_start;</div><div class='add'>+	last = sq-&gt;swq_last;</div><div class='add'>+</div><div class='add'>+	while (last != start) {</div><div class='add'>+		swq = &amp;sq-&gt;swq[last];</div><div class='add'>+		if (swq-&gt;slot_idx  == cqe_slot) {</div><div class='add'>+			cqe_sq_cons = swq-&gt;next_idx;</div><div class='add'>+			dev_err(&amp;sq_hwq-&gt;pdev-&gt;dev, "%s: Found cons wqe = %d slot = %d\n",</div><div class='add'>+				__func__, cqe_sq_cons, cqe_slot);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		last = swq-&gt;next_idx;</div><div class='add'>+	}</div><div class='add'>+	return cqe_sq_cons;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 				     struct cq_req *hwcqe,</div><div class='ctx'> 				     struct bnxt_qplib_cqe **pcqe, int *budget,</div><div class='ctx'> 				     u32 cq_cons, struct bnxt_qplib_qp **lib_qp)</div><div class='ctx'> {</div><div class='add'>+	struct bnxt_qplib_swq *swq;</div><div class='add'>+	struct bnxt_qplib_cqe *cqe;</div><div class='add'>+	u32 cqe_sq_cons, slot_num;</div><div class='ctx'> 	struct bnxt_qplib_qp *qp;</div><div class='ctx'> 	struct bnxt_qplib_q *sq;</div><div class='del'>-	struct bnxt_qplib_cqe *cqe;</div><div class='del'>-	u32 sw_sq_cons, cqe_sq_cons;</div><div class='del'>-	struct bnxt_qplib_swq *swq;</div><div class='add'>+	int cqe_cons;</div><div class='ctx'> 	int rc = 0;</div><div class='ctx'> </div><div class='ctx'> 	qp = (struct bnxt_qplib_qp *)((unsigned long)</div><div class='hunk'>@@ -2233,31 +2607,37 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 	}</div><div class='ctx'> 	sq = &amp;qp-&gt;sq;</div><div class='ctx'> </div><div class='del'>-	cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe-&gt;sq_cons_idx), &amp;sq-&gt;hwq);</div><div class='del'>-	if (cqe_sq_cons &gt; sq-&gt;hwq.max_elements) {</div><div class='del'>-		dev_err(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='del'>-			"FP: CQ Process req reported sq_cons_idx 0x%x which exceeded max 0x%x\n",</div><div class='del'>-			cqe_sq_cons, sq-&gt;hwq.max_elements);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='add'>+	cqe_sq_cons = le16_to_cpu(hwcqe-&gt;sq_cons_idx) % sq-&gt;max_sw_wqe;</div><div class='ctx'> 	if (qp-&gt;sq.flushed) {</div><div class='ctx'> 		dev_dbg(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='ctx'> 			"%s: QP in Flush QP = %p\n", __func__, qp);</div><div class='ctx'> 		goto done;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	if (__is_err_cqe_for_var_wqe(qp, hwcqe-&gt;status)) {</div><div class='add'>+		slot_num = le16_to_cpu(hwcqe-&gt;sq_cons_idx);</div><div class='add'>+		cqe_cons = bnxt_qplib_get_cqe_sq_cons(sq, slot_num);</div><div class='add'>+		if (cqe_cons &lt; 0) {</div><div class='add'>+			dev_err(&amp;cq-&gt;hwq.pdev-&gt;dev, "%s: Wrong SQ cons cqe_slot_indx = %d\n",</div><div class='add'>+				__func__, slot_num);</div><div class='add'>+			goto done;</div><div class='add'>+		}</div><div class='add'>+		cqe_sq_cons = cqe_cons;</div><div class='add'>+		dev_err(&amp;cq-&gt;hwq.pdev-&gt;dev, "%s: cqe_sq_cons = %d swq_last = %d swq_start = %d\n",</div><div class='add'>+			__func__, cqe_sq_cons, sq-&gt;swq_last, sq-&gt;swq_start);</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	/* Require to walk the sq's swq to fabricate CQEs for all previously</div><div class='ctx'> 	 * signaled SWQEs due to CQE aggregation from the current sq cons</div><div class='ctx'> 	 * to the cqe_sq_cons</div><div class='ctx'> 	 */</div><div class='ctx'> 	cqe = *pcqe;</div><div class='ctx'> 	while (*budget) {</div><div class='del'>-		sw_sq_cons = HWQ_CMP(sq-&gt;hwq.cons, &amp;sq-&gt;hwq);</div><div class='del'>-		if (sw_sq_cons == cqe_sq_cons)</div><div class='add'>+		if (sq-&gt;swq_last == cqe_sq_cons)</div><div class='ctx'> 			/* Done */</div><div class='ctx'> 			break;</div><div class='ctx'> </div><div class='del'>-		swq = &amp;sq-&gt;swq[sw_sq_cons];</div><div class='add'>+		swq = &amp;sq-&gt;swq[sq-&gt;swq_last];</div><div class='ctx'> 		memset(cqe, 0, sizeof(*cqe));</div><div class='ctx'> 		cqe-&gt;opcode = CQ_BASE_CQE_TYPE_REQ;</div><div class='ctx'> 		cqe-&gt;qp_handle = (u64)(unsigned long)qp;</div><div class='hunk'>@@ -2271,38 +2651,42 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 		 * of the request being signaled or not, it must complete with</div><div class='ctx'> 		 * the hwcqe error status</div><div class='ctx'> 		 */</div><div class='del'>-		if (HWQ_CMP((sw_sq_cons + 1), &amp;sq-&gt;hwq) == cqe_sq_cons &amp;&amp;</div><div class='add'>+		if (swq-&gt;next_idx == cqe_sq_cons &amp;&amp;</div><div class='ctx'> 		    hwcqe-&gt;status != CQ_REQ_STATUS_OK) {</div><div class='ctx'> 			cqe-&gt;status = hwcqe-&gt;status;</div><div class='ctx'> 			dev_err(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='ctx'> 				"FP: CQ Processed Req wr_id[%d] = 0x%llx with status 0x%x\n",</div><div class='del'>-				sw_sq_cons, cqe-&gt;wr_id, cqe-&gt;status);</div><div class='add'>+				sq-&gt;swq_last, cqe-&gt;wr_id, cqe-&gt;status);</div><div class='ctx'> 			cqe++;</div><div class='ctx'> 			(*budget)--;</div><div class='ctx'> 			bnxt_qplib_mark_qp_error(qp);</div><div class='ctx'> 			/* Add qp to flush list of the CQ */</div><div class='ctx'> 			bnxt_qplib_add_flush_qp(qp);</div><div class='ctx'> 		} else {</div><div class='del'>-			if (swq-&gt;flags &amp; SQ_SEND_FLAGS_SIGNAL_COMP) {</div><div class='del'>-				/* Before we complete, do WA 9060 */</div><div class='del'>-				if (do_wa9060(qp, cq, cq_cons, sw_sq_cons,</div><div class='add'>+			/* Before we complete, do WA 9060 */</div><div class='add'>+			if (!bnxt_qplib_is_chip_gen_p5_p7(qp-&gt;cctx)) {</div><div class='add'>+				if (do_wa9060(qp, cq, cq_cons, sq-&gt;swq_last,</div><div class='ctx'> 					      cqe_sq_cons)) {</div><div class='ctx'> 					*lib_qp = qp;</div><div class='ctx'> 					goto out;</div><div class='ctx'> 				}</div><div class='add'>+			}</div><div class='add'>+			if (swq-&gt;flags &amp; SQ_SEND_FLAGS_SIGNAL_COMP) {</div><div class='ctx'> 				cqe-&gt;status = CQ_REQ_STATUS_OK;</div><div class='ctx'> 				cqe++;</div><div class='ctx'> 				(*budget)--;</div><div class='ctx'> 			}</div><div class='ctx'> 		}</div><div class='ctx'> skip:</div><div class='del'>-		sq-&gt;hwq.cons++;</div><div class='add'>+		bnxt_qplib_hwq_incr_cons(sq-&gt;hwq.max_elements, &amp;sq-&gt;hwq.cons,</div><div class='add'>+					 swq-&gt;slots, &amp;sq-&gt;dbinfo.flags);</div><div class='add'>+		sq-&gt;swq_last = swq-&gt;next_idx;</div><div class='ctx'> 		if (sq-&gt;single)</div><div class='ctx'> 			break;</div><div class='ctx'> 	}</div><div class='ctx'> out:</div><div class='ctx'> 	*pcqe = cqe;</div><div class='del'>-	if (HWQ_CMP(sq-&gt;hwq.cons, &amp;sq-&gt;hwq) != cqe_sq_cons) {</div><div class='add'>+	if (sq-&gt;swq_last != cqe_sq_cons) {</div><div class='ctx'> 		/* Out of budget */</div><div class='ctx'> 		rc = -EAGAIN;</div><div class='ctx'> 		goto done;</div><div class='hunk'>@@ -2322,7 +2706,8 @@ static void bnxt_qplib_release_srqe(struct bnxt_qplib_srq *srq, u32 tag)</div><div class='ctx'> 	srq-&gt;swq[srq-&gt;last_idx].next_idx = (int)tag;</div><div class='ctx'> 	srq-&gt;last_idx = (int)tag;</div><div class='ctx'> 	srq-&gt;swq[srq-&gt;last_idx].next_idx = -1;</div><div class='del'>-	srq-&gt;hwq.cons++; /* Support for SRQE counter */</div><div class='add'>+	bnxt_qplib_hwq_incr_cons(srq-&gt;hwq.max_elements, &amp;srq-&gt;hwq.cons,</div><div class='add'>+				 srq-&gt;dbinfo.max_slot, &amp;srq-&gt;dbinfo.flags);</div><div class='ctx'> 	spin_unlock(&amp;srq-&gt;hwq.lock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -2331,12 +2716,11 @@ static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 					struct bnxt_qplib_cqe **pcqe,</div><div class='ctx'> 					int *budget)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_qplib_qp *qp;</div><div class='del'>-	struct bnxt_qplib_q *rq;</div><div class='ctx'> 	struct bnxt_qplib_srq *srq;</div><div class='ctx'> 	struct bnxt_qplib_cqe *cqe;</div><div class='add'>+	struct bnxt_qplib_qp *qp;</div><div class='add'>+	struct bnxt_qplib_q *rq;</div><div class='ctx'> 	u32 wr_id_idx;</div><div class='del'>-	int rc = 0;</div><div class='ctx'> </div><div class='ctx'> 	qp = (struct bnxt_qplib_qp *)((unsigned long)</div><div class='ctx'> 				      le64_to_cpu(hwcqe-&gt;qp_handle));</div><div class='hunk'>@@ -2347,7 +2731,7 @@ static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 	if (qp-&gt;rq.flushed) {</div><div class='ctx'> 		dev_dbg(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='ctx'> 			"%s: QP in Flush QP = %p\n", __func__, qp);</div><div class='del'>-		goto done;</div><div class='add'>+		return 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	cqe = *pcqe;</div><div class='hunk'>@@ -2377,17 +2761,24 @@ static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 		(*budget)--;</div><div class='ctx'> 		*pcqe = cqe;</div><div class='ctx'> 	} else {</div><div class='add'>+		struct bnxt_qplib_swq *swq;</div><div class='add'>+</div><div class='ctx'> 		rq = &amp;qp-&gt;rq;</div><div class='del'>-		if (wr_id_idx &gt;= rq-&gt;hwq.max_elements) {</div><div class='add'>+		if (wr_id_idx &gt; (rq-&gt;max_wqe - 1)) {</div><div class='ctx'> 			dev_err(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='ctx'> 				"FP: CQ Process RC wr_id idx 0x%x exceeded RQ max 0x%x\n",</div><div class='del'>-				wr_id_idx, rq-&gt;hwq.max_elements);</div><div class='add'>+				wr_id_idx, rq-&gt;max_wqe);</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 		}</div><div class='del'>-		cqe-&gt;wr_id = rq-&gt;swq[wr_id_idx].wr_id;</div><div class='add'>+		if (wr_id_idx != rq-&gt;swq_last)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		swq = &amp;rq-&gt;swq[rq-&gt;swq_last];</div><div class='add'>+		cqe-&gt;wr_id = swq-&gt;wr_id;</div><div class='ctx'> 		cqe++;</div><div class='ctx'> 		(*budget)--;</div><div class='del'>-		rq-&gt;hwq.cons++;</div><div class='add'>+		bnxt_qplib_hwq_incr_cons(rq-&gt;hwq.max_elements, &amp;rq-&gt;hwq.cons,</div><div class='add'>+					 swq-&gt;slots, &amp;rq-&gt;dbinfo.flags);</div><div class='add'>+		rq-&gt;swq_last = swq-&gt;next_idx;</div><div class='ctx'> 		*pcqe = cqe;</div><div class='ctx'> </div><div class='ctx'> 		if (hwcqe-&gt;status != CQ_RES_RC_STATUS_OK) {</div><div class='hunk'>@@ -2397,8 +2788,7 @@ static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-done:</div><div class='del'>-	return rc;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,</div><div class='hunk'>@@ -2406,12 +2796,11 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 					struct bnxt_qplib_cqe **pcqe,</div><div class='ctx'> 					int *budget)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_qplib_qp *qp;</div><div class='del'>-	struct bnxt_qplib_q *rq;</div><div class='ctx'> 	struct bnxt_qplib_srq *srq;</div><div class='ctx'> 	struct bnxt_qplib_cqe *cqe;</div><div class='add'>+	struct bnxt_qplib_qp *qp;</div><div class='add'>+	struct bnxt_qplib_q *rq;</div><div class='ctx'> 	u32 wr_id_idx;</div><div class='del'>-	int rc = 0;</div><div class='ctx'> </div><div class='ctx'> 	qp = (struct bnxt_qplib_qp *)((unsigned long)</div><div class='ctx'> 				      le64_to_cpu(hwcqe-&gt;qp_handle));</div><div class='hunk'>@@ -2422,11 +2811,11 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 	if (qp-&gt;rq.flushed) {</div><div class='ctx'> 		dev_dbg(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='ctx'> 			"%s: QP in Flush QP = %p\n", __func__, qp);</div><div class='del'>-		goto done;</div><div class='add'>+		return 0;</div><div class='ctx'> 	}</div><div class='ctx'> 	cqe = *pcqe;</div><div class='ctx'> 	cqe-&gt;opcode = hwcqe-&gt;cqe_type_toggle &amp; CQ_BASE_CQE_TYPE_MASK;</div><div class='del'>-	cqe-&gt;length = (u32)le16_to_cpu(hwcqe-&gt;length);</div><div class='add'>+	cqe-&gt;length = le16_to_cpu(hwcqe-&gt;length) &amp; CQ_RES_UD_LENGTH_MASK;</div><div class='ctx'> 	cqe-&gt;cfa_meta = le16_to_cpu(hwcqe-&gt;cfa_metadata);</div><div class='ctx'> 	cqe-&gt;invrkey = le32_to_cpu(hwcqe-&gt;imm_data);</div><div class='ctx'> 	cqe-&gt;flags = le16_to_cpu(hwcqe-&gt;flags);</div><div class='hunk'>@@ -2458,18 +2847,25 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 		(*budget)--;</div><div class='ctx'> 		*pcqe = cqe;</div><div class='ctx'> 	} else {</div><div class='add'>+		struct bnxt_qplib_swq *swq;</div><div class='add'>+</div><div class='ctx'> 		rq = &amp;qp-&gt;rq;</div><div class='del'>-		if (wr_id_idx &gt;= rq-&gt;hwq.max_elements) {</div><div class='add'>+		if (wr_id_idx &gt; (rq-&gt;max_wqe - 1)) {</div><div class='ctx'> 			dev_err(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='ctx'> 				"FP: CQ Process UD wr_id idx 0x%x exceeded RQ max 0x%x\n",</div><div class='del'>-				wr_id_idx, rq-&gt;hwq.max_elements);</div><div class='add'>+				wr_id_idx, rq-&gt;max_wqe);</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		cqe-&gt;wr_id = rq-&gt;swq[wr_id_idx].wr_id;</div><div class='add'>+		if (rq-&gt;swq_last != wr_id_idx)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		swq = &amp;rq-&gt;swq[rq-&gt;swq_last];</div><div class='add'>+		cqe-&gt;wr_id = swq-&gt;wr_id;</div><div class='ctx'> 		cqe++;</div><div class='ctx'> 		(*budget)--;</div><div class='del'>-		rq-&gt;hwq.cons++;</div><div class='add'>+		bnxt_qplib_hwq_incr_cons(rq-&gt;hwq.max_elements, &amp;rq-&gt;hwq.cons,</div><div class='add'>+					 swq-&gt;slots, &amp;rq-&gt;dbinfo.flags);</div><div class='add'>+		rq-&gt;swq_last = swq-&gt;next_idx;</div><div class='ctx'> 		*pcqe = cqe;</div><div class='ctx'> </div><div class='ctx'> 		if (hwcqe-&gt;status != CQ_RES_RC_STATUS_OK) {</div><div class='hunk'>@@ -2478,23 +2874,18 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 			bnxt_qplib_add_flush_qp(qp);</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='del'>-done:</div><div class='del'>-	return rc;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq)</div><div class='ctx'> {</div><div class='del'>-	struct cq_base *hw_cqe, **hw_cqe_ptr;</div><div class='del'>-	u32 sw_cons, raw_cons;</div><div class='add'>+	struct cq_base *hw_cqe;</div><div class='ctx'> 	bool rc = true;</div><div class='ctx'> </div><div class='del'>-	raw_cons = cq-&gt;hwq.cons;</div><div class='del'>-	sw_cons = HWQ_CMP(raw_cons, &amp;cq-&gt;hwq);</div><div class='del'>-	hw_cqe_ptr = (struct cq_base **)cq-&gt;hwq.pbl_ptr;</div><div class='del'>-	hw_cqe = &amp;hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];</div><div class='del'>-</div><div class='add'>+	hw_cqe = bnxt_qplib_get_qe(&amp;cq-&gt;hwq, cq-&gt;hwq.cons, NULL);</div><div class='ctx'> 	 /* Check for Valid bit. If the CQE is valid, return false */</div><div class='del'>-	rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq-&gt;hwq.max_elements);</div><div class='add'>+	rc = !CQE_CMP_VALID(hw_cqe, cq-&gt;dbinfo.flags);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -2508,7 +2899,6 @@ static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 	struct bnxt_qplib_srq *srq;</div><div class='ctx'> 	struct bnxt_qplib_cqe *cqe;</div><div class='ctx'> 	u32 wr_id_idx;</div><div class='del'>-	int rc = 0;</div><div class='ctx'> </div><div class='ctx'> 	qp = (struct bnxt_qplib_qp *)((unsigned long)</div><div class='ctx'> 				      le64_to_cpu(hwcqe-&gt;qp_handle));</div><div class='hunk'>@@ -2519,7 +2909,7 @@ static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 	if (qp-&gt;rq.flushed) {</div><div class='ctx'> 		dev_dbg(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='ctx'> 			"%s: QP in Flush QP = %p\n", __func__, qp);</div><div class='del'>-		goto done;</div><div class='add'>+		return 0;</div><div class='ctx'> 	}</div><div class='ctx'> 	cqe = *pcqe;</div><div class='ctx'> 	cqe-&gt;opcode = hwcqe-&gt;cqe_type_toggle &amp; CQ_BASE_CQE_TYPE_MASK;</div><div class='hunk'>@@ -2562,17 +2952,24 @@ static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 		(*budget)--;</div><div class='ctx'> 		*pcqe = cqe;</div><div class='ctx'> 	} else {</div><div class='add'>+		struct bnxt_qplib_swq *swq;</div><div class='add'>+</div><div class='ctx'> 		rq = &amp;qp-&gt;rq;</div><div class='del'>-		if (wr_id_idx &gt;= rq-&gt;hwq.max_elements) {</div><div class='add'>+		if (wr_id_idx &gt; (rq-&gt;max_wqe - 1)) {</div><div class='ctx'> 			dev_err(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='ctx'> 				"FP: CQ Process Raw/QP1 RQ wr_id idx 0x%x exceeded RQ max 0x%x\n",</div><div class='del'>-				wr_id_idx, rq-&gt;hwq.max_elements);</div><div class='add'>+				wr_id_idx, rq-&gt;max_wqe);</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 		}</div><div class='del'>-		cqe-&gt;wr_id = rq-&gt;swq[wr_id_idx].wr_id;</div><div class='add'>+		if (rq-&gt;swq_last != wr_id_idx)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		swq = &amp;rq-&gt;swq[rq-&gt;swq_last];</div><div class='add'>+		cqe-&gt;wr_id = swq-&gt;wr_id;</div><div class='ctx'> 		cqe++;</div><div class='ctx'> 		(*budget)--;</div><div class='del'>-		rq-&gt;hwq.cons++;</div><div class='add'>+		bnxt_qplib_hwq_incr_cons(rq-&gt;hwq.max_elements, &amp;rq-&gt;hwq.cons,</div><div class='add'>+					 swq-&gt;slots, &amp;rq-&gt;dbinfo.flags);</div><div class='add'>+		rq-&gt;swq_last = swq-&gt;next_idx;</div><div class='ctx'> 		*pcqe = cqe;</div><div class='ctx'> </div><div class='ctx'> 		if (hwcqe-&gt;status != CQ_RES_RC_STATUS_OK) {</div><div class='hunk'>@@ -2582,8 +2979,7 @@ static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-done:</div><div class='del'>-	return rc;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,</div><div class='hunk'>@@ -2594,7 +2990,7 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 	struct bnxt_qplib_qp *qp;</div><div class='ctx'> 	struct bnxt_qplib_q *sq, *rq;</div><div class='ctx'> 	struct bnxt_qplib_cqe *cqe;</div><div class='del'>-	u32 sw_cons = 0, cqe_cons;</div><div class='add'>+	u32 swq_last = 0, cqe_cons;</div><div class='ctx'> 	int rc = 0;</div><div class='ctx'> </div><div class='ctx'> 	/* Check the Status */</div><div class='hunk'>@@ -2605,11 +3001,8 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,</div><div class='ctx'> </div><div class='ctx'> 	qp = (struct bnxt_qplib_qp *)((unsigned long)</div><div class='ctx'> 				      le64_to_cpu(hwcqe-&gt;qp_handle));</div><div class='del'>-	if (!qp) {</div><div class='del'>-		dev_err(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='del'>-			"FP: CQ Process terminal qp is NULL\n");</div><div class='add'>+	if (!qp)</div><div class='ctx'> 		return -EINVAL;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='ctx'> 	/* Must block new posting of SQ and RQ */</div><div class='ctx'> 	qp-&gt;state = CMDQ_MODIFY_QP_NEW_STATE_ERR;</div><div class='hunk'>@@ -2620,13 +3013,7 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 	cqe_cons = le16_to_cpu(hwcqe-&gt;sq_cons_idx);</div><div class='ctx'> 	if (cqe_cons == 0xFFFF)</div><div class='ctx'> 		goto do_rq;</div><div class='del'>-</div><div class='del'>-	if (cqe_cons &gt; sq-&gt;hwq.max_elements) {</div><div class='del'>-		dev_err(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='del'>-			"FP: CQ Process terminal reported sq_cons_idx 0x%x which exceeded max 0x%x\n",</div><div class='del'>-			cqe_cons, sq-&gt;hwq.max_elements);</div><div class='del'>-		goto do_rq;</div><div class='del'>-	}</div><div class='add'>+	cqe_cons %= sq-&gt;max_sw_wqe;</div><div class='ctx'> </div><div class='ctx'> 	if (qp-&gt;sq.flushed) {</div><div class='ctx'> 		dev_dbg(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='hunk'>@@ -2640,24 +3027,26 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 	 */</div><div class='ctx'> 	cqe = *pcqe;</div><div class='ctx'> 	while (*budget) {</div><div class='del'>-		sw_cons = HWQ_CMP(sq-&gt;hwq.cons, &amp;sq-&gt;hwq);</div><div class='del'>-		if (sw_cons == cqe_cons)</div><div class='add'>+		swq_last = sq-&gt;swq_last;</div><div class='add'>+		if (swq_last == cqe_cons)</div><div class='ctx'> 			break;</div><div class='del'>-		if (sq-&gt;swq[sw_cons].flags &amp; SQ_SEND_FLAGS_SIGNAL_COMP) {</div><div class='add'>+		if (sq-&gt;swq[swq_last].flags &amp; SQ_SEND_FLAGS_SIGNAL_COMP) {</div><div class='ctx'> 			memset(cqe, 0, sizeof(*cqe));</div><div class='ctx'> 			cqe-&gt;status = CQ_REQ_STATUS_OK;</div><div class='ctx'> 			cqe-&gt;opcode = CQ_BASE_CQE_TYPE_REQ;</div><div class='ctx'> 			cqe-&gt;qp_handle = (u64)(unsigned long)qp;</div><div class='ctx'> 			cqe-&gt;src_qp = qp-&gt;id;</div><div class='del'>-			cqe-&gt;wr_id = sq-&gt;swq[sw_cons].wr_id;</div><div class='del'>-			cqe-&gt;type = sq-&gt;swq[sw_cons].type;</div><div class='add'>+			cqe-&gt;wr_id = sq-&gt;swq[swq_last].wr_id;</div><div class='add'>+			cqe-&gt;type = sq-&gt;swq[swq_last].type;</div><div class='ctx'> 			cqe++;</div><div class='ctx'> 			(*budget)--;</div><div class='ctx'> 		}</div><div class='del'>-		sq-&gt;hwq.cons++;</div><div class='add'>+		bnxt_qplib_hwq_incr_cons(sq-&gt;hwq.max_elements, &amp;sq-&gt;hwq.cons,</div><div class='add'>+					 sq-&gt;swq[swq_last].slots, &amp;sq-&gt;dbinfo.flags);</div><div class='add'>+		sq-&gt;swq_last = sq-&gt;swq[swq_last].next_idx;</div><div class='ctx'> 	}</div><div class='ctx'> 	*pcqe = cqe;</div><div class='del'>-	if (!(*budget) &amp;&amp; sw_cons != cqe_cons) {</div><div class='add'>+	if (!(*budget) &amp;&amp; swq_last != cqe_cons) {</div><div class='ctx'> 		/* Out of budget */</div><div class='ctx'> 		rc = -EAGAIN;</div><div class='ctx'> 		goto sq_done;</div><div class='hunk'>@@ -2669,10 +3058,11 @@ do_rq:</div><div class='ctx'> 	cqe_cons = le16_to_cpu(hwcqe-&gt;rq_cons_idx);</div><div class='ctx'> 	if (cqe_cons == 0xFFFF) {</div><div class='ctx'> 		goto done;</div><div class='del'>-	} else if (cqe_cons &gt; rq-&gt;hwq.max_elements) {</div><div class='add'>+	} else if (cqe_cons &gt; rq-&gt;max_wqe - 1) {</div><div class='ctx'> 		dev_err(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='ctx'> 			"FP: CQ Processed terminal reported rq_cons_idx 0x%x exceeds max 0x%x\n",</div><div class='del'>-			cqe_cons, rq-&gt;hwq.max_elements);</div><div class='add'>+			cqe_cons, rq-&gt;max_wqe);</div><div class='add'>+		rc = -EINVAL;</div><div class='ctx'> 		goto done;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -2736,20 +3126,18 @@ int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,</div><div class='ctx'> int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,</div><div class='ctx'> 		       int num_cqes, struct bnxt_qplib_qp **lib_qp)</div><div class='ctx'> {</div><div class='del'>-	struct cq_base *hw_cqe, **hw_cqe_ptr;</div><div class='del'>-	u32 sw_cons, raw_cons;</div><div class='add'>+	struct cq_base *hw_cqe;</div><div class='ctx'> 	int budget, rc = 0;</div><div class='add'>+	u32 hw_polled = 0;</div><div class='add'>+	u8 type;</div><div class='ctx'> </div><div class='del'>-	raw_cons = cq-&gt;hwq.cons;</div><div class='ctx'> 	budget = num_cqes;</div><div class='ctx'> </div><div class='ctx'> 	while (budget) {</div><div class='del'>-		sw_cons = HWQ_CMP(raw_cons, &amp;cq-&gt;hwq);</div><div class='del'>-		hw_cqe_ptr = (struct cq_base **)cq-&gt;hwq.pbl_ptr;</div><div class='del'>-		hw_cqe = &amp;hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];</div><div class='add'>+		hw_cqe = bnxt_qplib_get_qe(&amp;cq-&gt;hwq, cq-&gt;hwq.cons, NULL);</div><div class='ctx'> </div><div class='ctx'> 		/* Check for Valid bit */</div><div class='del'>-		if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq-&gt;hwq.max_elements))</div><div class='add'>+		if (!CQE_CMP_VALID(hw_cqe, cq-&gt;dbinfo.flags))</div><div class='ctx'> 			break;</div><div class='ctx'> </div><div class='ctx'> 		/*</div><div class='hunk'>@@ -2758,12 +3146,13 @@ int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,</div><div class='ctx'> 		 */</div><div class='ctx'> 		dma_rmb();</div><div class='ctx'> 		/* From the device's respective CQE format to qplib_wc*/</div><div class='del'>-		switch (hw_cqe-&gt;cqe_type_toggle &amp; CQ_BASE_CQE_TYPE_MASK) {</div><div class='add'>+		type = hw_cqe-&gt;cqe_type_toggle &amp; CQ_BASE_CQE_TYPE_MASK;</div><div class='add'>+		switch (type) {</div><div class='ctx'> 		case CQ_BASE_CQE_TYPE_REQ:</div><div class='ctx'> 			rc = bnxt_qplib_cq_process_req(cq,</div><div class='ctx'> 						       (struct cq_req *)hw_cqe,</div><div class='ctx'> 						       &amp;cqe, &amp;budget,</div><div class='del'>-						       sw_cons, lib_qp);</div><div class='add'>+						       cq-&gt;hwq.cons, lib_qp);</div><div class='ctx'> 			break;</div><div class='ctx'> 		case CQ_BASE_CQE_TYPE_RES_RC:</div><div class='ctx'> 			rc = bnxt_qplib_cq_process_res_rc(cq,</div><div class='hunk'>@@ -2805,23 +3194,26 @@ int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,</div><div class='ctx'> 			/* Error while processing the CQE, just skip to the</div><div class='ctx'> 			 * next one</div><div class='ctx'> 			 */</div><div class='del'>-			dev_err(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='del'>-				"process_cqe error rc = 0x%x\n", rc);</div><div class='add'>+			if (type != CQ_BASE_CQE_TYPE_TERMINAL)</div><div class='add'>+				dev_err(&amp;cq-&gt;hwq.pdev-&gt;dev,</div><div class='add'>+					"process_cqe error rc = 0x%x\n", rc);</div><div class='ctx'> 		}</div><div class='del'>-		raw_cons++;</div><div class='del'>-	}</div><div class='del'>-	if (cq-&gt;hwq.cons != raw_cons) {</div><div class='del'>-		cq-&gt;hwq.cons = raw_cons;</div><div class='del'>-		bnxt_qplib_arm_cq(cq, DBC_DBC_TYPE_CQ);</div><div class='add'>+		hw_polled++;</div><div class='add'>+		bnxt_qplib_hwq_incr_cons(cq-&gt;hwq.max_elements, &amp;cq-&gt;hwq.cons,</div><div class='add'>+					 1, &amp;cq-&gt;dbinfo.flags);</div><div class='add'>+</div><div class='ctx'> 	}</div><div class='add'>+	if (hw_polled)</div><div class='add'>+		bnxt_qplib_ring_db(&amp;cq-&gt;dbinfo, DBC_DBC_TYPE_CQ);</div><div class='ctx'> exit:</div><div class='ctx'> 	return num_cqes - budget;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)</div><div class='ctx'> {</div><div class='add'>+	cq-&gt;dbinfo.toggle = cq-&gt;toggle;</div><div class='ctx'> 	if (arm_type)</div><div class='del'>-		bnxt_qplib_arm_cq(cq, arm_type);</div><div class='add'>+		bnxt_qplib_ring_db(&amp;cq-&gt;dbinfo, arm_type);</div><div class='ctx'> 	/* Using cq-&gt;arm_state variable to track whether to issue cq handler */</div><div class='ctx'> 	atomic_set(&amp;cq-&gt;arm_state, 1);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h<br/>index 99e0a13cbefa..1b414a73b46d 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_fp.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_fp.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_fp.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/qplib_fp.h</a></div><div class='hunk'>@@ -39,12 +39,60 @@</div><div class='ctx'> #ifndef __BNXT_QPLIB_FP_H__</div><div class='ctx'> #define __BNXT_QPLIB_FP_H__</div><div class='ctx'> </div><div class='add'>+#include &lt;rdma/bnxt_re-abi.h&gt;</div><div class='add'>+</div><div class='add'>+/* Few helper structures temporarily defined here</div><div class='add'>+ * should get rid of these when roce_hsi.h is updated</div><div class='add'>+ * in original code base</div><div class='add'>+ */</div><div class='add'>+struct sq_ud_ext_hdr {</div><div class='add'>+	__le32 dst_qp;</div><div class='add'>+	__le32 avid;</div><div class='add'>+	__le64 rsvd;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct sq_raw_ext_hdr {</div><div class='add'>+	__le32 cfa_meta;</div><div class='add'>+	__le32 rsvd0;</div><div class='add'>+	__le64 rsvd1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct sq_rdma_ext_hdr {</div><div class='add'>+	__le64 remote_va;</div><div class='add'>+	__le32 remote_key;</div><div class='add'>+	__le32 rsvd;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct sq_atomic_ext_hdr {</div><div class='add'>+	__le64 swap_data;</div><div class='add'>+	__le64 cmp_data;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct sq_fr_pmr_ext_hdr {</div><div class='add'>+	__le64 pblptr;</div><div class='add'>+	__le64 va;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct sq_bind_ext_hdr {</div><div class='add'>+	__le64 va;</div><div class='add'>+	__le32 length_lo;</div><div class='add'>+	__le32 length_hi;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct rq_ext_hdr {</div><div class='add'>+	__le64 rsvd1;</div><div class='add'>+	__le64 rsvd2;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* Helper structures end */</div><div class='add'>+</div><div class='ctx'> struct bnxt_qplib_srq {</div><div class='ctx'> 	struct bnxt_qplib_pd		*pd;</div><div class='ctx'> 	struct bnxt_qplib_dpi		*dpi;</div><div class='del'>-	void __iomem			*dbr_base;</div><div class='add'>+	struct bnxt_qplib_db_info	dbinfo;</div><div class='ctx'> 	u64				srq_handle;</div><div class='ctx'> 	u32				id;</div><div class='add'>+	u16				wqe_size;</div><div class='ctx'> 	u32				max_wqe;</div><div class='ctx'> 	u32				max_sge;</div><div class='ctx'> 	u32				threshold;</div><div class='hunk'>@@ -57,6 +105,7 @@ struct bnxt_qplib_srq {</div><div class='ctx'> 	struct bnxt_qplib_sg_info	sg_info;</div><div class='ctx'> 	u16				eventq_hw_ring_id;</div><div class='ctx'> 	spinlock_t			lock; /* protect SRQE link list */</div><div class='add'>+	u8				toggle;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct bnxt_qplib_sge {</div><div class='hunk'>@@ -65,38 +114,6 @@ struct bnxt_qplib_sge {</div><div class='ctx'> 	u32				size;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define BNXT_QPLIB_MAX_SQE_ENTRY_SIZE	sizeof(struct sq_send)</div><div class='del'>-</div><div class='del'>-#define SQE_CNT_PER_PG		(PAGE_SIZE / BNXT_QPLIB_MAX_SQE_ENTRY_SIZE)</div><div class='del'>-#define SQE_MAX_IDX_PER_PG	(SQE_CNT_PER_PG - 1)</div><div class='del'>-</div><div class='del'>-static inline u32 get_sqe_pg(u32 val)</div><div class='del'>-{</div><div class='del'>-	return ((val &amp; ~SQE_MAX_IDX_PER_PG) / SQE_CNT_PER_PG);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u32 get_sqe_idx(u32 val)</div><div class='del'>-{</div><div class='del'>-	return (val &amp; SQE_MAX_IDX_PER_PG);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-#define BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE	sizeof(struct sq_psn_search)</div><div class='del'>-</div><div class='del'>-#define PSNE_CNT_PER_PG		(PAGE_SIZE / BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE)</div><div class='del'>-#define PSNE_MAX_IDX_PER_PG	(PSNE_CNT_PER_PG - 1)</div><div class='del'>-</div><div class='del'>-static inline u32 get_psne_pg(u32 val)</div><div class='del'>-{</div><div class='del'>-	return ((val &amp; ~PSNE_MAX_IDX_PER_PG) / PSNE_CNT_PER_PG);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u32 get_psne_idx(u32 val)</div><div class='del'>-{</div><div class='del'>-	return (val &amp; PSNE_MAX_IDX_PER_PG);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-#define BNXT_QPLIB_QP_MAX_SGL	6</div><div class='del'>-</div><div class='ctx'> struct bnxt_qplib_swq {</div><div class='ctx'> 	u64				wr_id;</div><div class='ctx'> 	int				next_idx;</div><div class='hunk'>@@ -104,6 +121,8 @@ struct bnxt_qplib_swq {</div><div class='ctx'> 	u8				flags;</div><div class='ctx'> 	u32				start_psn;</div><div class='ctx'> 	u32				next_psn;</div><div class='add'>+	u32				slot_idx;</div><div class='add'>+	u8				slots;</div><div class='ctx'> 	struct sq_psn_search		*psn_search;</div><div class='ctx'> 	struct sq_psn_search_ext	*psn_ext;</div><div class='ctx'> };</div><div class='hunk'>@@ -134,7 +153,7 @@ struct bnxt_qplib_swqe {</div><div class='ctx'> #define BNXT_QPLIB_SWQE_FLAGS_UC_FENCE			BIT(2)</div><div class='ctx'> #define BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT		BIT(3)</div><div class='ctx'> #define BNXT_QPLIB_SWQE_FLAGS_INLINE			BIT(4)</div><div class='del'>-	struct bnxt_qplib_sge		sg_list[BNXT_QPLIB_QP_MAX_SGL];</div><div class='add'>+	struct bnxt_qplib_sge		sg_list[BNXT_VAR_MAX_SGE];</div><div class='ctx'> 	int				num_sge;</div><div class='ctx'> 	/* Max inline data is 96 bytes */</div><div class='ctx'> 	u32				inline_len;</div><div class='hunk'>@@ -145,12 +164,12 @@ struct bnxt_qplib_swqe {</div><div class='ctx'> 		/* Send, with imm, inval key */</div><div class='ctx'> 		struct {</div><div class='ctx'> 			union {</div><div class='del'>-				__be32	imm_data;</div><div class='add'>+				u32	imm_data;</div><div class='ctx'> 				u32	inv_key;</div><div class='ctx'> 			};</div><div class='ctx'> 			u32		q_key;</div><div class='ctx'> 			u32		dst_qp;</div><div class='del'>-			u16		avid;</div><div class='add'>+			u32		avid;</div><div class='ctx'> 		} send;</div><div class='ctx'> </div><div class='ctx'> 		/* Send Raw Ethernet and QP1 */</div><div class='hunk'>@@ -163,7 +182,7 @@ struct bnxt_qplib_swqe {</div><div class='ctx'> 		/* RDMA write, with imm, read */</div><div class='ctx'> 		struct {</div><div class='ctx'> 			union {</div><div class='del'>-				__be32	imm_data;</div><div class='add'>+				u32	imm_data;</div><div class='ctx'> 				u32	inv_key;</div><div class='ctx'> 			};</div><div class='ctx'> 			u64		remote_va;</div><div class='hunk'>@@ -226,18 +245,14 @@ struct bnxt_qplib_swqe {</div><div class='ctx'> 	};</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define BNXT_QPLIB_MAX_RQE_ENTRY_SIZE	sizeof(struct rq_wqe)</div><div class='del'>-</div><div class='del'>-#define RQE_CNT_PER_PG		(PAGE_SIZE / BNXT_QPLIB_MAX_RQE_ENTRY_SIZE)</div><div class='del'>-#define RQE_MAX_IDX_PER_PG	(RQE_CNT_PER_PG - 1)</div><div class='del'>-#define RQE_PG(x)		(((x) &amp; ~RQE_MAX_IDX_PER_PG) / RQE_CNT_PER_PG)</div><div class='del'>-#define RQE_IDX(x)		((x) &amp; RQE_MAX_IDX_PER_PG)</div><div class='del'>-</div><div class='ctx'> struct bnxt_qplib_q {</div><div class='ctx'> 	struct bnxt_qplib_hwq		hwq;</div><div class='ctx'> 	struct bnxt_qplib_swq		*swq;</div><div class='add'>+	struct bnxt_qplib_db_info	dbinfo;</div><div class='ctx'> 	struct bnxt_qplib_sg_info	sg_info;</div><div class='ctx'> 	u32				max_wqe;</div><div class='add'>+	u32				max_sw_wqe;</div><div class='add'>+	u16				wqe_size;</div><div class='ctx'> 	u16				q_full_delta;</div><div class='ctx'> 	u16				max_sge;</div><div class='ctx'> 	u32				psn;</div><div class='hunk'>@@ -248,6 +263,8 @@ struct bnxt_qplib_q {</div><div class='ctx'> 	u32				phantom_cqe_cnt;</div><div class='ctx'> 	u32				next_cq_cons;</div><div class='ctx'> 	bool				flushed;</div><div class='add'>+	u32				swq_start;</div><div class='add'>+	u32				swq_last;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct bnxt_qplib_qp {</div><div class='hunk'>@@ -255,13 +272,14 @@ struct bnxt_qplib_qp {</div><div class='ctx'> 	struct bnxt_qplib_dpi		*dpi;</div><div class='ctx'> 	struct bnxt_qplib_chip_ctx	*cctx;</div><div class='ctx'> 	u64				qp_handle;</div><div class='del'>-#define        BNXT_QPLIB_QP_ID_INVALID        0xFFFFFFFF</div><div class='add'>+#define BNXT_QPLIB_QP_ID_INVALID        0xFFFFFFFF</div><div class='ctx'> 	u32				id;</div><div class='ctx'> 	u8				type;</div><div class='ctx'> 	u8				sig_type;</div><div class='del'>-	u32				modify_flags;</div><div class='add'>+	u8				wqe_mode;</div><div class='ctx'> 	u8				state;</div><div class='ctx'> 	u8				cur_qp_state;</div><div class='add'>+	u64				modify_flags;</div><div class='ctx'> 	u32				max_inline_data;</div><div class='ctx'> 	u32				mtu;</div><div class='ctx'> 	u8				path_mtu;</div><div class='hunk'>@@ -280,6 +298,8 @@ struct bnxt_qplib_qp {</div><div class='ctx'> 	u32				dest_qpn;</div><div class='ctx'> 	u8				smac[6];</div><div class='ctx'> 	u16				vlan_id;</div><div class='add'>+	u16				port_id;</div><div class='add'>+	u16				udp_sport;</div><div class='ctx'> 	u8				nw_type;</div><div class='ctx'> 	struct bnxt_qplib_ah		ah;</div><div class='ctx'> </div><div class='hunk'>@@ -321,8 +341,16 @@ struct bnxt_qplib_qp {</div><div class='ctx'> 	dma_addr_t			rq_hdr_buf_map;</div><div class='ctx'> 	struct list_head		sq_flush;</div><div class='ctx'> 	struct list_head		rq_flush;</div><div class='add'>+	u32				msn;</div><div class='add'>+	u32				msn_tbl_sz;</div><div class='add'>+	bool				is_host_msn_tbl;</div><div class='add'>+	u8				tos_dscp;</div><div class='add'>+	u32				ugid_index;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+#define BNXT_RE_MAX_MSG_SIZE	0x80000000</div><div class='add'>+#define BNXT_RE_INVAL_MSG_SIZE	0xFFFFFFFF</div><div class='add'>+</div><div class='ctx'> #define BNXT_QPLIB_MAX_CQE_ENTRY_SIZE	sizeof(struct cq_base)</div><div class='ctx'> </div><div class='ctx'> #define CQE_CNT_PER_PG		(PAGE_SIZE / BNXT_QPLIB_MAX_CQE_ENTRY_SIZE)</div><div class='hunk'>@@ -331,17 +359,56 @@ struct bnxt_qplib_qp {</div><div class='ctx'> #define CQE_IDX(x)		((x) &amp; CQE_MAX_IDX_PER_PG)</div><div class='ctx'> </div><div class='ctx'> #define ROCE_CQE_CMP_V			0</div><div class='del'>-#define CQE_CMP_VALID(hdr, raw_cons, cp_bit)			\</div><div class='add'>+#define CQE_CMP_VALID(hdr, pass)			\</div><div class='ctx'> 	(!!((hdr)-&gt;cqe_type_toggle &amp; CQ_BASE_TOGGLE) ==		\</div><div class='del'>-	   !((raw_cons) &amp; (cp_bit)))</div><div class='add'>+	   !((pass) &amp; BNXT_QPLIB_FLAG_EPOCH_CONS_MASK))</div><div class='add'>+</div><div class='add'>+static inline u32 __bnxt_qplib_get_avail(struct bnxt_qplib_hwq *hwq)</div><div class='add'>+{</div><div class='add'>+	int cons, prod, avail;</div><div class='add'>+</div><div class='add'>+	cons = hwq-&gt;cons;</div><div class='add'>+	prod = hwq-&gt;prod;</div><div class='add'>+	avail = cons - prod;</div><div class='add'>+	if (cons &lt;= prod)</div><div class='add'>+		avail += hwq-&gt;depth;</div><div class='add'>+	return avail;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_q *qplib_q)</div><div class='add'>+static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_q *que,</div><div class='add'>+					 u8 slots)</div><div class='ctx'> {</div><div class='del'>-	return HWQ_CMP((qplib_q-&gt;hwq.prod + qplib_q-&gt;q_full_delta),</div><div class='del'>-		       &amp;qplib_q-&gt;hwq) == HWQ_CMP(qplib_q-&gt;hwq.cons,</div><div class='del'>-						 &amp;qplib_q-&gt;hwq);</div><div class='add'>+	struct bnxt_qplib_hwq *hwq;</div><div class='add'>+	int avail;</div><div class='add'>+</div><div class='add'>+	hwq = &amp;que-&gt;hwq;</div><div class='add'>+	/* False full is possible, retrying post-send makes sense */</div><div class='add'>+	avail = hwq-&gt;cons - hwq-&gt;prod;</div><div class='add'>+	if (hwq-&gt;cons &lt;= hwq-&gt;prod)</div><div class='add'>+		avail += hwq-&gt;depth;</div><div class='add'>+	return avail &lt;= slots;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+/* CQ coalescing parameters */</div><div class='add'>+struct bnxt_qplib_cq_coal_param {</div><div class='add'>+	u16 buf_maxtime;</div><div class='add'>+	u8 normal_maxbuf;</div><div class='add'>+	u8 during_maxbuf;</div><div class='add'>+	u8 en_ring_idle_mode;</div><div class='add'>+	u8 enable;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define BNXT_QPLIB_CQ_COAL_DEF_BUF_MAXTIME		0x1</div><div class='add'>+#define BNXT_QPLIB_CQ_COAL_DEF_NORMAL_MAXBUF_P7		0x8</div><div class='add'>+#define BNXT_QPLIB_CQ_COAL_DEF_DURING_MAXBUF_P7		0x8</div><div class='add'>+#define BNXT_QPLIB_CQ_COAL_DEF_NORMAL_MAXBUF_P5		0x1</div><div class='add'>+#define BNXT_QPLIB_CQ_COAL_DEF_DURING_MAXBUF_P5		0x1</div><div class='add'>+#define BNXT_QPLIB_CQ_COAL_DEF_EN_RING_IDLE_MODE	0x1</div><div class='add'>+#define BNXT_QPLIB_CQ_COAL_MAX_BUF_MAXTIME		0x1bf</div><div class='add'>+#define BNXT_QPLIB_CQ_COAL_MAX_NORMAL_MAXBUF		0x1f</div><div class='add'>+#define BNXT_QPLIB_CQ_COAL_MAX_DURING_MAXBUF		0x1f</div><div class='add'>+#define BNXT_QPLIB_CQ_COAL_MAX_EN_RING_IDLE_MODE	0x1</div><div class='add'>+</div><div class='ctx'> struct bnxt_qplib_cqe {</div><div class='ctx'> 	u8				status;</div><div class='ctx'> 	u8				type;</div><div class='hunk'>@@ -350,7 +417,7 @@ struct bnxt_qplib_cqe {</div><div class='ctx'> 	u16				cfa_meta;</div><div class='ctx'> 	u64				wr_id;</div><div class='ctx'> 	union {</div><div class='del'>-		__be32			immdata;</div><div class='add'>+		u32			immdata;</div><div class='ctx'> 		u32			invrkey;</div><div class='ctx'> 	};</div><div class='ctx'> 	u64				qp_handle;</div><div class='hunk'>@@ -370,17 +437,19 @@ struct bnxt_qplib_cqe {</div><div class='ctx'> #define BNXT_QPLIB_QUEUE_START_PERIOD		0x01</div><div class='ctx'> struct bnxt_qplib_cq {</div><div class='ctx'> 	struct bnxt_qplib_dpi		*dpi;</div><div class='del'>-	void __iomem			*dbr_base;</div><div class='add'>+	struct bnxt_qplib_db_info	dbinfo;</div><div class='ctx'> 	u32				max_wqe;</div><div class='ctx'> 	u32				id;</div><div class='ctx'> 	u16				count;</div><div class='ctx'> 	u16				period;</div><div class='ctx'> 	struct bnxt_qplib_hwq		hwq;</div><div class='add'>+	struct bnxt_qplib_hwq		resize_hwq;</div><div class='ctx'> 	u32				cnq_hw_ring_id;</div><div class='ctx'> 	struct bnxt_qplib_nq		*nq;</div><div class='ctx'> 	bool				resize_in_progress;</div><div class='ctx'> 	struct bnxt_qplib_sg_info	sg_info;</div><div class='ctx'> 	u64				cq_handle;</div><div class='add'>+	u8				toggle;</div><div class='ctx'> </div><div class='ctx'> #define CQ_RESIZE_WAIT_TIME_MS		500</div><div class='ctx'> 	unsigned long			flags;</div><div class='hunk'>@@ -401,6 +470,8 @@ struct bnxt_qplib_cq {</div><div class='ctx'>  * of the same QP while manipulating the flush list.</div><div class='ctx'>  */</div><div class='ctx'> 	spinlock_t			flush_lock; /* QP flush management */</div><div class='add'>+	u16				cnq_events;</div><div class='add'>+	struct bnxt_qplib_cq_coal_param	*coalescing;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> #define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE	sizeof(struct xrrq_irrq)</div><div class='hunk'>@@ -417,9 +488,9 @@ struct bnxt_qplib_cq {</div><div class='ctx'> #define NQE_PG(x)		(((x) &amp; ~NQE_MAX_IDX_PER_PG) / NQE_CNT_PER_PG)</div><div class='ctx'> #define NQE_IDX(x)		((x) &amp; NQE_MAX_IDX_PER_PG)</div><div class='ctx'> </div><div class='del'>-#define NQE_CMP_VALID(hdr, raw_cons, cp_bit)			\</div><div class='add'>+#define NQE_CMP_VALID(hdr, pass)			\</div><div class='ctx'> 	(!!(le32_to_cpu((hdr)-&gt;info63_v[0]) &amp; NQ_BASE_V) ==	\</div><div class='del'>-	   !((raw_cons) &amp; (cp_bit)))</div><div class='add'>+	   !((pass) &amp; BNXT_QPLIB_FLAG_EPOCH_CONS_MASK))</div><div class='ctx'> </div><div class='ctx'> #define BNXT_QPLIB_NQE_MAX_CNT		(128 * 1024)</div><div class='ctx'> </div><div class='hunk'>@@ -433,66 +504,33 @@ struct bnxt_qplib_cq {</div><div class='ctx'> 					 NQ_DB_IDX_VALID |	\</div><div class='ctx'> 					 NQ_DB_IRQ_DIS)</div><div class='ctx'> </div><div class='del'>-static inline void bnxt_qplib_ring_nq_db64(void __iomem *db, u32 index,</div><div class='del'>-					   u32 xid, bool arm)</div><div class='del'>-{</div><div class='del'>-	u64 val;</div><div class='del'>-</div><div class='del'>-	val = xid &amp; DBC_DBC_XID_MASK;</div><div class='del'>-	val |= DBC_DBC_PATH_ROCE;</div><div class='del'>-	val |= arm ? DBC_DBC_TYPE_NQ_ARM : DBC_DBC_TYPE_NQ;</div><div class='del'>-	val &lt;&lt;= 32;</div><div class='del'>-	val |= index &amp; DBC_DBC_INDEX_MASK;</div><div class='del'>-	writeq(val, db);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void bnxt_qplib_ring_nq_db_rearm(void __iomem *db, u32 raw_cons,</div><div class='del'>-					       u32 max_elements, u32 xid,</div><div class='del'>-					       bool gen_p5)</div><div class='del'>-{</div><div class='del'>-	u32 index = raw_cons &amp; (max_elements - 1);</div><div class='del'>-</div><div class='del'>-	if (gen_p5)</div><div class='del'>-		bnxt_qplib_ring_nq_db64(db, index, xid, true);</div><div class='del'>-	else</div><div class='del'>-		writel(NQ_DB_CP_FLAGS_REARM | (index &amp; DBC_DBC32_XID_MASK), db);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void bnxt_qplib_ring_nq_db(void __iomem *db, u32 raw_cons,</div><div class='del'>-					 u32 max_elements, u32 xid,</div><div class='del'>-					 bool gen_p5)</div><div class='del'>-{</div><div class='del'>-	u32 index = raw_cons &amp; (max_elements - 1);</div><div class='add'>+struct bnxt_qplib_nq_db {</div><div class='add'>+	struct bnxt_qplib_reg_desc	reg;</div><div class='add'>+	struct bnxt_qplib_db_info	dbinfo;</div><div class='add'>+};</div><div class='ctx'> </div><div class='del'>-	if (gen_p5)</div><div class='del'>-		bnxt_qplib_ring_nq_db64(db, index, xid, false);</div><div class='del'>-	else</div><div class='del'>-		writel(NQ_DB_CP_FLAGS | (index &amp; DBC_DBC32_XID_MASK), db);</div><div class='del'>-}</div><div class='add'>+typedef int (*cqn_handler_t)(struct bnxt_qplib_nq *nq,</div><div class='add'>+		struct bnxt_qplib_cq *cq);</div><div class='add'>+typedef int (*srqn_handler_t)(struct bnxt_qplib_nq *nq,</div><div class='add'>+		struct bnxt_qplib_srq *srq, u8 event);</div><div class='ctx'> </div><div class='ctx'> struct bnxt_qplib_nq {</div><div class='del'>-	struct pci_dev		*pdev;</div><div class='del'>-	struct bnxt_qplib_res	*res;</div><div class='del'>-</div><div class='del'>-	int			vector;</div><div class='del'>-	cpumask_t		mask;</div><div class='del'>-	int			budget;</div><div class='del'>-	bool			requested;</div><div class='del'>-	struct tasklet_struct	worker;</div><div class='del'>-	struct bnxt_qplib_hwq	hwq;</div><div class='del'>-</div><div class='del'>-	u16			bar_reg;</div><div class='del'>-	u32			bar_reg_off;</div><div class='del'>-	u16			ring_id;</div><div class='del'>-	void __iomem		*bar_reg_iomem;</div><div class='del'>-</div><div class='del'>-	int			(*cqn_handler)(struct bnxt_qplib_nq *nq,</div><div class='del'>-					       struct bnxt_qplib_cq *cq);</div><div class='del'>-	int			(*srqn_handler)(struct bnxt_qplib_nq *nq,</div><div class='del'>-						struct bnxt_qplib_srq *srq,</div><div class='del'>-						u8 event);</div><div class='del'>-	struct workqueue_struct	*cqn_wq;</div><div class='del'>-	char			name[32];</div><div class='add'>+	struct pci_dev			*pdev;</div><div class='add'>+	struct bnxt_qplib_res		*res;</div><div class='add'>+	char				*name;</div><div class='add'>+	struct bnxt_qplib_hwq		hwq;</div><div class='add'>+	struct bnxt_qplib_nq_db		nq_db;</div><div class='add'>+	u16				ring_id;</div><div class='add'>+	int				msix_vec;</div><div class='add'>+	cpumask_t			mask;</div><div class='add'>+	struct tasklet_struct		nq_tasklet;</div><div class='add'>+	bool				requested;</div><div class='add'>+	int				budget;</div><div class='add'>+	u32				load;</div><div class='add'>+</div><div class='add'>+	cqn_handler_t			cqn_handler;</div><div class='add'>+	srqn_handler_t			srqn_handler;</div><div class='add'>+	struct workqueue_struct		*cqn_wq;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct bnxt_qplib_nq_work {</div><div class='hunk'>@@ -507,15 +545,10 @@ int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx,</div><div class='ctx'> 			    int msix_vector, bool need_init);</div><div class='ctx'> int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,</div><div class='ctx'> 			 int nq_idx, int msix_vector, int bar_reg_offset,</div><div class='del'>-			 int (*cqn_handler)(struct bnxt_qplib_nq *nq,</div><div class='del'>-					    struct bnxt_qplib_cq *cq),</div><div class='del'>-			 int (*srqn_handler)(struct bnxt_qplib_nq *nq,</div><div class='del'>-					     struct bnxt_qplib_srq *srq,</div><div class='del'>-					     u8 event));</div><div class='add'>+			 cqn_handler_t cqn_handler,</div><div class='add'>+			 srqn_handler_t srq_handler);</div><div class='ctx'> int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,</div><div class='ctx'> 			  struct bnxt_qplib_srq *srq);</div><div class='del'>-int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,</div><div class='del'>-			  struct bnxt_qplib_srq *srq);</div><div class='ctx'> int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,</div><div class='ctx'> 			 struct bnxt_qplib_srq *srq);</div><div class='ctx'> void bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,</div><div class='hunk'>@@ -544,13 +577,17 @@ void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp);</div><div class='ctx'> int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,</div><div class='ctx'> 			 struct bnxt_qplib_swqe *wqe);</div><div class='ctx'> int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);</div><div class='add'>+int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq,</div><div class='add'>+			 int new_cqes);</div><div class='add'>+void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res,</div><div class='add'>+				   struct bnxt_qplib_cq *cq);</div><div class='ctx'> int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);</div><div class='ctx'> int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,</div><div class='ctx'> 		       int num, struct bnxt_qplib_qp **qp);</div><div class='ctx'> bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq);</div><div class='ctx'> void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type);</div><div class='ctx'> void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq);</div><div class='del'>-int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq);</div><div class='add'>+int bnxt_qplib_alloc_nq(struct bnxt_qplib_res *res, struct bnxt_qplib_nq *nq);</div><div class='ctx'> void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp);</div><div class='ctx'> void bnxt_qplib_acquire_cq_locks(struct bnxt_qplib_qp *qp,</div><div class='ctx'> 				 unsigned long *flags);</div><div class='hunk'>@@ -560,4 +597,93 @@ int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,</div><div class='ctx'> 				  struct bnxt_qplib_cqe *cqe,</div><div class='ctx'> 				  int num_cqes);</div><div class='ctx'> void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp);</div><div class='add'>+void bnxt_re_synchronize_nq(struct bnxt_qplib_nq *nq);</div><div class='add'>+</div><div class='add'>+static inline void *bnxt_qplib_get_swqe(struct bnxt_qplib_q *que, u32 *swq_idx)</div><div class='add'>+{</div><div class='add'>+	u32 idx;</div><div class='add'>+</div><div class='add'>+	idx = que-&gt;swq_start;</div><div class='add'>+	if (swq_idx)</div><div class='add'>+		*swq_idx = idx;</div><div class='add'>+	return &amp;que-&gt;swq[idx];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void bnxt_qplib_swq_mod_start(struct bnxt_qplib_q *que, u32 idx)</div><div class='add'>+{</div><div class='add'>+	que-&gt;swq_start = que-&gt;swq[idx].next_idx;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que, u8 wqe_mode, bool is_sq)</div><div class='add'>+{</div><div class='add'>+	u32 slots;</div><div class='add'>+</div><div class='add'>+	/* Queue depth is the number of slots. */</div><div class='add'>+	slots = (que-&gt;wqe_size * que-&gt;max_wqe) / sizeof(struct sq_sge);</div><div class='add'>+	/* For variable WQE mode, need to align the slots to 256 */</div><div class='add'>+	if (wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE &amp;&amp; is_sq)</div><div class='add'>+		slots = ALIGN(slots, BNXT_VAR_MAX_SLOT_ALIGN);</div><div class='add'>+	return slots;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 bnxt_qplib_set_sq_size(struct bnxt_qplib_q *que, u8 wqe_mode)</div><div class='add'>+{</div><div class='add'>+	return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?</div><div class='add'>+		que-&gt;max_wqe : bnxt_qplib_get_depth(que, wqe_mode, true);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 bnxt_qplib_set_sq_max_slot(u8 wqe_mode)</div><div class='add'>+{</div><div class='add'>+	return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?</div><div class='add'>+		sizeof(struct sq_send) / sizeof(struct sq_sge) : 1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 bnxt_qplib_set_rq_max_slot(u32 wqe_size)</div><div class='add'>+{</div><div class='add'>+	return (wqe_size / sizeof(struct sq_sge));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u16 __xlate_qfd(u16 delta, u16 wqe_bytes)</div><div class='add'>+{</div><div class='add'>+	/* For Cu/Wh delta = 128, stride = 16, wqe_bytes = 128</div><div class='add'>+	 * For Gen-p5 B/C mode delta = 0, stride = 16, wqe_bytes = 128.</div><div class='add'>+	 * For Gen-p5 delta = 0, stride = 16, 32 &lt;= wqe_bytes &lt;= 512.</div><div class='add'>+	 * when 8916 is disabled.</div><div class='add'>+	 */</div><div class='add'>+	return (delta * wqe_bytes) / sizeof(struct sq_sge);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u16 bnxt_qplib_calc_ilsize(struct bnxt_qplib_swqe *wqe, u16 max)</div><div class='add'>+{</div><div class='add'>+	u16 size = 0;</div><div class='add'>+	int indx;</div><div class='add'>+</div><div class='add'>+	for (indx = 0; indx &lt; wqe-&gt;num_sge; indx++)</div><div class='add'>+		size += wqe-&gt;sg_list[indx].size;</div><div class='add'>+	if (size &gt; max)</div><div class='add'>+		size = max;</div><div class='add'>+</div><div class='add'>+	return size;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* MSN table update inlin */</div><div class='add'>+static inline __le64 bnxt_re_update_msn_tbl(u32 st_idx, u32 npsn, u32 start_psn)</div><div class='add'>+{</div><div class='add'>+	return cpu_to_le64((((u64)(st_idx) &lt;&lt; SQ_MSN_SEARCH_START_IDX_SFT) &amp;</div><div class='add'>+		SQ_MSN_SEARCH_START_IDX_MASK) |</div><div class='add'>+		(((u64)(npsn) &lt;&lt; SQ_MSN_SEARCH_NEXT_PSN_SFT) &amp;</div><div class='add'>+		SQ_MSN_SEARCH_NEXT_PSN_MASK) |</div><div class='add'>+		(((start_psn) &lt;&lt; SQ_MSN_SEARCH_START_PSN_SFT) &amp;</div><div class='add'>+		SQ_MSN_SEARCH_START_PSN_MASK));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool __is_var_wqe(struct bnxt_qplib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	return (qp-&gt;wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool __is_err_cqe_for_var_wqe(struct bnxt_qplib_qp *qp, u8 status)</div><div class='add'>+{</div><div class='add'>+	return (status != CQ_REQ_STATUS_OK) &amp;&amp; __is_var_wqe(qp);</div><div class='add'>+}</div><div class='ctx'> #endif /* __BNXT_QPLIB_FP_H__ */</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c<br/>index 48b04d2f175f..295a9610f3e6 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_rcfw.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/qplib_rcfw.c</a></div><div class='hunk'>@@ -49,204 +49,535 @@</div><div class='ctx'> #include "qplib_rcfw.h"</div><div class='ctx'> #include "qplib_sp.h"</div><div class='ctx'> #include "qplib_fp.h"</div><div class='add'>+#include "qplib_tlv.h"</div><div class='ctx'> </div><div class='del'>-static void bnxt_qplib_service_creq(unsigned long data);</div><div class='add'>+static void bnxt_qplib_service_creq(struct tasklet_struct *t);</div><div class='ctx'> </div><div class='del'>-/* Hardware communication channel */</div><div class='add'>+/**</div><div class='add'>+ * bnxt_qplib_map_rc  -  map return type based on opcode</div><div class='add'>+ * @opcode:  roce slow path opcode</div><div class='add'>+ *</div><div class='add'>+ * case #1</div><div class='add'>+ * Firmware initiated error recovery is a safe state machine and</div><div class='add'>+ * driver can consider all the underlying rdma resources are free.</div><div class='add'>+ * In this state, it is safe to return success for opcodes related to</div><div class='add'>+ * destroying rdma resources (like destroy qp, destroy cq etc.).</div><div class='add'>+ *</div><div class='add'>+ * case #2</div><div class='add'>+ * If driver detect potential firmware stall, it is not safe state machine</div><div class='add'>+ * and the driver can not consider all the underlying rdma resources are</div><div class='add'>+ * freed.</div><div class='add'>+ * In this state, it is not safe to return success for opcodes related to</div><div class='add'>+ * destroying rdma resources (like destroy qp, destroy cq etc.).</div><div class='add'>+ *</div><div class='add'>+ * Scope of this helper function is only for case #1.</div><div class='add'>+ *</div><div class='add'>+ * Returns:</div><div class='add'>+ * 0 to communicate success to caller.</div><div class='add'>+ * Non zero error code to communicate failure to caller.</div><div class='add'>+ */</div><div class='add'>+static int bnxt_qplib_map_rc(u8 opcode)</div><div class='add'>+{</div><div class='add'>+	switch (opcode) {</div><div class='add'>+	case CMDQ_BASE_OPCODE_DESTROY_QP:</div><div class='add'>+	case CMDQ_BASE_OPCODE_DESTROY_SRQ:</div><div class='add'>+	case CMDQ_BASE_OPCODE_DESTROY_CQ:</div><div class='add'>+	case CMDQ_BASE_OPCODE_DEALLOCATE_KEY:</div><div class='add'>+	case CMDQ_BASE_OPCODE_DEREGISTER_MR:</div><div class='add'>+	case CMDQ_BASE_OPCODE_DELETE_GID:</div><div class='add'>+	case CMDQ_BASE_OPCODE_DESTROY_QP1:</div><div class='add'>+	case CMDQ_BASE_OPCODE_DESTROY_AH:</div><div class='add'>+	case CMDQ_BASE_OPCODE_DEINITIALIZE_FW:</div><div class='add'>+	case CMDQ_BASE_OPCODE_MODIFY_ROCE_CC:</div><div class='add'>+	case CMDQ_BASE_OPCODE_SET_LINK_AGGR_MODE:</div><div class='add'>+		return 0;</div><div class='add'>+	default:</div><div class='add'>+		return -ETIMEDOUT;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * bnxt_re_is_fw_stalled   -	Check firmware health</div><div class='add'>+ * @rcfw:     rcfw channel instance of rdev</div><div class='add'>+ * @cookie:   cookie to track the command</div><div class='add'>+ *</div><div class='add'>+ * If firmware has not responded any rcfw command within</div><div class='add'>+ * rcfw-&gt;max_timeout, consider firmware as stalled.</div><div class='add'>+ *</div><div class='add'>+ * Returns:</div><div class='add'>+ * 0 if firmware is responding</div><div class='add'>+ * -ENODEV if firmware is not responding</div><div class='add'>+ */</div><div class='add'>+static int bnxt_re_is_fw_stalled(struct bnxt_qplib_rcfw *rcfw,</div><div class='add'>+				 u16 cookie)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_cmdq_ctx *cmdq;</div><div class='add'>+	struct bnxt_qplib_crsqe *crsqe;</div><div class='add'>+</div><div class='add'>+	crsqe = &amp;rcfw-&gt;crsqe_tbl[cookie];</div><div class='add'>+	cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+</div><div class='add'>+	if (time_after(jiffies, cmdq-&gt;last_seen +</div><div class='add'>+		      (rcfw-&gt;max_timeout * HZ))) {</div><div class='add'>+		dev_warn_ratelimited(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='add'>+				     "%s: FW STALL Detected. cmdq[%#x]=%#x waited (%d &gt; %d) msec active %d ",</div><div class='add'>+				     __func__, cookie, crsqe-&gt;opcode,</div><div class='add'>+				     jiffies_to_msecs(jiffies - cmdq-&gt;last_seen),</div><div class='add'>+				     rcfw-&gt;max_timeout * 1000,</div><div class='add'>+				     crsqe-&gt;is_in_used);</div><div class='add'>+		return -ENODEV;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * __wait_for_resp   -	Don't hold the cpu context and wait for response</div><div class='add'>+ * @rcfw:    rcfw channel instance of rdev</div><div class='add'>+ * @cookie:  cookie to track the command</div><div class='add'>+ *</div><div class='add'>+ * Wait for command completion in sleepable context.</div><div class='add'>+ *</div><div class='add'>+ * Returns:</div><div class='add'>+ * 0 if command is completed by firmware.</div><div class='add'>+ * Non zero error code for rest of the case.</div><div class='add'>+ */</div><div class='ctx'> static int __wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)</div><div class='ctx'> {</div><div class='del'>-	u16 cbit;</div><div class='del'>-	int rc;</div><div class='add'>+	struct bnxt_qplib_cmdq_ctx *cmdq;</div><div class='add'>+	struct bnxt_qplib_crsqe *crsqe;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+	crsqe = &amp;rcfw-&gt;crsqe_tbl[cookie];</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		if (test_bit(ERR_DEVICE_DETACHED, &amp;cmdq-&gt;flags))</div><div class='add'>+			return bnxt_qplib_map_rc(crsqe-&gt;opcode);</div><div class='add'>+		if (test_bit(FIRMWARE_STALL_DETECTED, &amp;cmdq-&gt;flags))</div><div class='add'>+			return -ETIMEDOUT;</div><div class='add'>+</div><div class='add'>+		wait_event_timeout(cmdq-&gt;waitq,</div><div class='add'>+				   !crsqe-&gt;is_in_used ||</div><div class='add'>+				   test_bit(ERR_DEVICE_DETACHED, &amp;cmdq-&gt;flags),</div><div class='add'>+				   secs_to_jiffies(rcfw-&gt;max_timeout));</div><div class='ctx'> </div><div class='del'>-	cbit = cookie % rcfw-&gt;cmdq_depth;</div><div class='del'>-	rc = wait_event_timeout(rcfw-&gt;waitq,</div><div class='del'>-				!test_bit(cbit, rcfw-&gt;cmdq_bitmap),</div><div class='del'>-				msecs_to_jiffies(RCFW_CMD_WAIT_TIME_MS));</div><div class='del'>-	return rc ? 0 : -ETIMEDOUT;</div><div class='add'>+		if (!crsqe-&gt;is_in_used)</div><div class='add'>+			return 0;</div><div class='add'>+</div><div class='add'>+		bnxt_qplib_service_creq(&amp;rcfw-&gt;creq.creq_tasklet);</div><div class='add'>+</div><div class='add'>+		if (!crsqe-&gt;is_in_used)</div><div class='add'>+			return 0;</div><div class='add'>+</div><div class='add'>+		ret = bnxt_re_is_fw_stalled(rcfw, cookie);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+</div><div class='add'>+	} while (true);</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+/**</div><div class='add'>+ * __block_for_resp   -	hold the cpu context and wait for response</div><div class='add'>+ * @rcfw:    rcfw channel instance of rdev</div><div class='add'>+ * @cookie:  cookie to track the command</div><div class='add'>+ *</div><div class='add'>+ * This function will hold the cpu (non-sleepable context) and</div><div class='add'>+ * wait for command completion. Maximum holding interval is 8 second.</div><div class='add'>+ *</div><div class='add'>+ * Returns:</div><div class='add'>+ * -ETIMEDOUT if command is not completed in specific time interval.</div><div class='add'>+ * 0 if command is completed by firmware.</div><div class='add'>+ */</div><div class='ctx'> static int __block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)</div><div class='ctx'> {</div><div class='del'>-	u32 count = RCFW_BLOCKED_CMD_WAIT_COUNT;</div><div class='del'>-	u16 cbit;</div><div class='add'>+	struct bnxt_qplib_cmdq_ctx *cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+	struct bnxt_qplib_crsqe *crsqe;</div><div class='add'>+	unsigned long issue_time = 0;</div><div class='add'>+</div><div class='add'>+	issue_time = jiffies;</div><div class='add'>+	crsqe = &amp;rcfw-&gt;crsqe_tbl[cookie];</div><div class='ctx'> </div><div class='del'>-	cbit = cookie % rcfw-&gt;cmdq_depth;</div><div class='del'>-	if (!test_bit(cbit, rcfw-&gt;cmdq_bitmap))</div><div class='del'>-		goto done;</div><div class='ctx'> 	do {</div><div class='del'>-		mdelay(1); /* 1m sec */</div><div class='del'>-		bnxt_qplib_service_creq((unsigned long)rcfw);</div><div class='del'>-	} while (test_bit(cbit, rcfw-&gt;cmdq_bitmap) &amp;&amp; --count);</div><div class='del'>-done:</div><div class='del'>-	return count ? 0 : -ETIMEDOUT;</div><div class='add'>+		if (test_bit(ERR_DEVICE_DETACHED, &amp;cmdq-&gt;flags))</div><div class='add'>+			return bnxt_qplib_map_rc(crsqe-&gt;opcode);</div><div class='add'>+		if (test_bit(FIRMWARE_STALL_DETECTED, &amp;cmdq-&gt;flags))</div><div class='add'>+			return -ETIMEDOUT;</div><div class='add'>+</div><div class='add'>+		udelay(1);</div><div class='add'>+</div><div class='add'>+		bnxt_qplib_service_creq(&amp;rcfw-&gt;creq.creq_tasklet);</div><div class='add'>+		if (!crsqe-&gt;is_in_used)</div><div class='add'>+			return 0;</div><div class='add'>+</div><div class='add'>+	} while (time_before(jiffies, issue_time + (8 * HZ)));</div><div class='add'>+</div><div class='add'>+	return -ETIMEDOUT;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static int __send_message(struct bnxt_qplib_rcfw *rcfw, struct cmdq_base *req,</div><div class='del'>-			  struct creq_base *resp, void *sb, u8 is_block)</div><div class='add'>+/*  __send_message_no_waiter -	get cookie and post the message.</div><div class='add'>+ * @rcfw:   rcfw channel instance of rdev</div><div class='add'>+ * @msg:    qplib message internal</div><div class='add'>+ *</div><div class='add'>+ * This function will just post and don't bother about completion.</div><div class='add'>+ * Current design of this function is -</div><div class='add'>+ * user must hold the completion queue hwq-&gt;lock.</div><div class='add'>+ * user must have used existing completion and free the resources.</div><div class='add'>+ * this function will not check queue full condition.</div><div class='add'>+ * this function will explicitly set is_waiter_alive=false.</div><div class='add'>+ * current use case is - send destroy_ah if create_ah is return</div><div class='add'>+ * after waiter of create_ah is lost. It can be extended for other</div><div class='add'>+ * use case as well.</div><div class='add'>+ *</div><div class='add'>+ * Returns: Nothing</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+static void __send_message_no_waiter(struct bnxt_qplib_rcfw *rcfw,</div><div class='add'>+				     struct bnxt_qplib_cmdqmsg *msg)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_qplib_cmdqe *cmdqe, **cmdq_ptr;</div><div class='del'>-	struct bnxt_qplib_hwq *cmdq = &amp;rcfw-&gt;cmdq;</div><div class='del'>-	u32 cmdq_depth = rcfw-&gt;cmdq_depth;</div><div class='del'>-	struct bnxt_qplib_crsq *crsqe;</div><div class='add'>+	struct bnxt_qplib_cmdq_ctx *cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+	struct bnxt_qplib_hwq *hwq = &amp;cmdq-&gt;hwq;</div><div class='add'>+	struct bnxt_qplib_crsqe *crsqe;</div><div class='add'>+	struct bnxt_qplib_cmdqe *cmdqe;</div><div class='ctx'> 	u32 sw_prod, cmdq_prod;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	u32 size, opcode;</div><div class='del'>-	u16 cookie, cbit;</div><div class='add'>+	u16 cookie;</div><div class='add'>+	u32 bsize;</div><div class='ctx'> 	u8 *preq;</div><div class='ctx'> </div><div class='del'>-	opcode = req-&gt;opcode;</div><div class='del'>-	if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &amp;rcfw-&gt;flags) &amp;&amp;</div><div class='del'>-	    (opcode != CMDQ_BASE_OPCODE_QUERY_FUNC &amp;&amp;</div><div class='del'>-	     opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW &amp;&amp;</div><div class='del'>-	     opcode != CMDQ_BASE_OPCODE_QUERY_VERSION)) {</div><div class='del'>-		dev_err(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='del'>-			"RCFW not initialized, reject opcode 0x%x\n", opcode);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='add'>+	cookie = cmdq-&gt;seq_num &amp; RCFW_MAX_COOKIE_VALUE;</div><div class='add'>+	__set_cmdq_base_cookie(msg-&gt;req, msg-&gt;req_sz, cpu_to_le16(cookie));</div><div class='add'>+	crsqe = &amp;rcfw-&gt;crsqe_tbl[cookie];</div><div class='ctx'> </div><div class='del'>-	if (test_bit(FIRMWARE_INITIALIZED_FLAG, &amp;rcfw-&gt;flags) &amp;&amp;</div><div class='del'>-	    opcode == CMDQ_BASE_OPCODE_INITIALIZE_FW) {</div><div class='del'>-		dev_err(&amp;rcfw-&gt;pdev-&gt;dev, "RCFW already initialized!\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='add'>+	/* Set cmd_size in terms of 16B slots in req. */</div><div class='add'>+	bsize = bnxt_qplib_set_cmd_slots(msg-&gt;req);</div><div class='add'>+	/* GET_CMD_SIZE would return number of slots in either case of tlv</div><div class='add'>+	 * and non-tlv commands after call to bnxt_qplib_set_cmd_slots()</div><div class='add'>+	 */</div><div class='add'>+	crsqe-&gt;is_internal_cmd = true;</div><div class='add'>+	crsqe-&gt;is_waiter_alive = false;</div><div class='add'>+	crsqe-&gt;is_in_used = true;</div><div class='add'>+	crsqe-&gt;req_size = __get_cmdq_base_cmd_size(msg-&gt;req, msg-&gt;req_sz);</div><div class='ctx'> </div><div class='del'>-	if (test_bit(FIRMWARE_TIMED_OUT, &amp;rcfw-&gt;flags))</div><div class='del'>-		return -ETIMEDOUT;</div><div class='add'>+	preq = (u8 *)msg-&gt;req;</div><div class='add'>+	do {</div><div class='add'>+		/* Locate the next cmdq slot */</div><div class='add'>+		sw_prod = HWQ_CMP(hwq-&gt;prod, hwq);</div><div class='add'>+		cmdqe = bnxt_qplib_get_qe(hwq, sw_prod, NULL);</div><div class='add'>+		/* Copy a segment of the req cmd to the cmdq */</div><div class='add'>+		memset(cmdqe, 0, sizeof(*cmdqe));</div><div class='add'>+		memcpy(cmdqe, preq, min_t(u32, bsize, sizeof(*cmdqe)));</div><div class='add'>+		preq += min_t(u32, bsize, sizeof(*cmdqe));</div><div class='add'>+		bsize -= min_t(u32, bsize, sizeof(*cmdqe));</div><div class='add'>+		hwq-&gt;prod++;</div><div class='add'>+	} while (bsize &gt; 0);</div><div class='add'>+	cmdq-&gt;seq_num++;</div><div class='add'>+</div><div class='add'>+	cmdq_prod = hwq-&gt;prod;</div><div class='add'>+	atomic_inc(&amp;rcfw-&gt;timeout_send);</div><div class='add'>+	/* ring CMDQ DB */</div><div class='add'>+	wmb();</div><div class='add'>+	writel(cmdq_prod, cmdq-&gt;cmdq_mbox.prod);</div><div class='add'>+	writel(RCFW_CMDQ_TRIG_VAL, cmdq-&gt;cmdq_mbox.db);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int __send_message(struct bnxt_qplib_rcfw *rcfw,</div><div class='add'>+			  struct bnxt_qplib_cmdqmsg *msg, u8 opcode)</div><div class='add'>+{</div><div class='add'>+	u32 bsize, free_slots, required_slots;</div><div class='add'>+	struct bnxt_qplib_cmdq_ctx *cmdq;</div><div class='add'>+	struct bnxt_qplib_crsqe *crsqe;</div><div class='add'>+	struct bnxt_qplib_cmdqe *cmdqe;</div><div class='add'>+	struct bnxt_qplib_hwq *hwq;</div><div class='add'>+	u32 sw_prod, cmdq_prod;</div><div class='add'>+	struct pci_dev *pdev;</div><div class='add'>+	u16 cookie;</div><div class='add'>+	u8 *preq;</div><div class='add'>+</div><div class='add'>+	cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+	hwq = &amp;cmdq-&gt;hwq;</div><div class='add'>+	pdev = rcfw-&gt;pdev;</div><div class='ctx'> </div><div class='ctx'> 	/* Cmdq are in 16-byte units, each request can consume 1 or more</div><div class='ctx'> 	 * cmdqe</div><div class='ctx'> 	 */</div><div class='del'>-	spin_lock_irqsave(&amp;cmdq-&gt;lock, flags);</div><div class='del'>-	if (req-&gt;cmd_size &gt;= HWQ_FREE_SLOTS(cmdq)) {</div><div class='del'>-		dev_err(&amp;rcfw-&gt;pdev-&gt;dev, "RCFW: CMDQ is full!\n");</div><div class='del'>-		spin_unlock_irqrestore(&amp;cmdq-&gt;lock, flags);</div><div class='add'>+	spin_lock_bh(&amp;hwq-&gt;lock);</div><div class='add'>+	required_slots = bnxt_qplib_get_cmd_slots(msg-&gt;req);</div><div class='add'>+	free_slots = HWQ_FREE_SLOTS(hwq);</div><div class='add'>+	cookie = cmdq-&gt;seq_num &amp; RCFW_MAX_COOKIE_VALUE;</div><div class='add'>+	crsqe = &amp;rcfw-&gt;crsqe_tbl[cookie];</div><div class='add'>+</div><div class='add'>+	if (required_slots &gt;= free_slots) {</div><div class='add'>+		dev_info_ratelimited(&amp;pdev-&gt;dev,</div><div class='add'>+				     "CMDQ is full req/free %d/%d!",</div><div class='add'>+				     required_slots, free_slots);</div><div class='add'>+		spin_unlock_bh(&amp;hwq-&gt;lock);</div><div class='ctx'> 		return -EAGAIN;</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='del'>-</div><div class='del'>-	cookie = rcfw-&gt;seq_num &amp; RCFW_MAX_COOKIE_VALUE;</div><div class='del'>-	cbit = cookie % rcfw-&gt;cmdq_depth;</div><div class='del'>-	if (is_block)</div><div class='add'>+	if (msg-&gt;block)</div><div class='ctx'> 		cookie |= RCFW_CMD_IS_BLOCKING;</div><div class='del'>-</div><div class='del'>-	set_bit(cbit, rcfw-&gt;cmdq_bitmap);</div><div class='del'>-	req-&gt;cookie = cpu_to_le16(cookie);</div><div class='del'>-	crsqe = &amp;rcfw-&gt;crsqe_tbl[cbit];</div><div class='del'>-	if (crsqe-&gt;resp) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;cmdq-&gt;lock, flags);</div><div class='del'>-		return -EBUSY;</div><div class='del'>-	}</div><div class='del'>-	memset(resp, 0, sizeof(*resp));</div><div class='del'>-	crsqe-&gt;resp = (struct creq_qp_event *)resp;</div><div class='del'>-	crsqe-&gt;resp-&gt;cookie = req-&gt;cookie;</div><div class='del'>-	crsqe-&gt;req_size = req-&gt;cmd_size;</div><div class='del'>-	if (req-&gt;resp_size &amp;&amp; sb) {</div><div class='del'>-		struct bnxt_qplib_rcfw_sbuf *sbuf = sb;</div><div class='del'>-</div><div class='del'>-		req-&gt;resp_addr = cpu_to_le64(sbuf-&gt;dma_addr);</div><div class='del'>-		req-&gt;resp_size = (sbuf-&gt;size + BNXT_QPLIB_CMDQE_UNITS - 1) /</div><div class='del'>-				  BNXT_QPLIB_CMDQE_UNITS;</div><div class='add'>+	__set_cmdq_base_cookie(msg-&gt;req, msg-&gt;req_sz, cpu_to_le16(cookie));</div><div class='add'>+</div><div class='add'>+	bsize = bnxt_qplib_set_cmd_slots(msg-&gt;req);</div><div class='add'>+	crsqe-&gt;free_slots = free_slots;</div><div class='add'>+	crsqe-&gt;resp = (struct creq_qp_event *)msg-&gt;resp;</div><div class='add'>+	crsqe-&gt;resp-&gt;cookie = cpu_to_le16(cookie);</div><div class='add'>+	crsqe-&gt;is_internal_cmd = false;</div><div class='add'>+	crsqe-&gt;is_waiter_alive = true;</div><div class='add'>+	crsqe-&gt;is_in_used = true;</div><div class='add'>+	crsqe-&gt;opcode = opcode;</div><div class='add'>+</div><div class='add'>+	crsqe-&gt;req_size = __get_cmdq_base_cmd_size(msg-&gt;req, msg-&gt;req_sz);</div><div class='add'>+	if (__get_cmdq_base_resp_size(msg-&gt;req, msg-&gt;req_sz) &amp;&amp; msg-&gt;sb) {</div><div class='add'>+		struct bnxt_qplib_rcfw_sbuf *sbuf = msg-&gt;sb;</div><div class='add'>+</div><div class='add'>+		__set_cmdq_base_resp_addr(msg-&gt;req, msg-&gt;req_sz,</div><div class='add'>+					  cpu_to_le64(sbuf-&gt;dma_addr));</div><div class='add'>+		__set_cmdq_base_resp_size(msg-&gt;req, msg-&gt;req_sz,</div><div class='add'>+					  ALIGN(sbuf-&gt;size,</div><div class='add'>+						BNXT_QPLIB_CMDQE_UNITS) /</div><div class='add'>+						BNXT_QPLIB_CMDQE_UNITS);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	cmdq_ptr = (struct bnxt_qplib_cmdqe **)cmdq-&gt;pbl_ptr;</div><div class='del'>-	preq = (u8 *)req;</div><div class='del'>-	size = req-&gt;cmd_size * BNXT_QPLIB_CMDQE_UNITS;</div><div class='add'>+	preq = (u8 *)msg-&gt;req;</div><div class='ctx'> 	do {</div><div class='ctx'> 		/* Locate the next cmdq slot */</div><div class='del'>-		sw_prod = HWQ_CMP(cmdq-&gt;prod, cmdq);</div><div class='del'>-		cmdqe = &amp;cmdq_ptr[get_cmdq_pg(sw_prod, cmdq_depth)]</div><div class='del'>-				[get_cmdq_idx(sw_prod, cmdq_depth)];</div><div class='del'>-		if (!cmdqe) {</div><div class='del'>-			dev_err(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='del'>-				"RCFW request failed with no cmdqe!\n");</div><div class='del'>-			goto done;</div><div class='del'>-		}</div><div class='add'>+		sw_prod = HWQ_CMP(hwq-&gt;prod, hwq);</div><div class='add'>+		cmdqe = bnxt_qplib_get_qe(hwq, sw_prod, NULL);</div><div class='ctx'> 		/* Copy a segment of the req cmd to the cmdq */</div><div class='ctx'> 		memset(cmdqe, 0, sizeof(*cmdqe));</div><div class='del'>-		memcpy(cmdqe, preq, min_t(u32, size, sizeof(*cmdqe)));</div><div class='del'>-		preq += min_t(u32, size, sizeof(*cmdqe));</div><div class='del'>-		size -= min_t(u32, size, sizeof(*cmdqe));</div><div class='del'>-		cmdq-&gt;prod++;</div><div class='del'>-		rcfw-&gt;seq_num++;</div><div class='del'>-	} while (size &gt; 0);</div><div class='del'>-</div><div class='del'>-	rcfw-&gt;seq_num++;</div><div class='del'>-</div><div class='del'>-	cmdq_prod = cmdq-&gt;prod;</div><div class='del'>-	if (test_bit(FIRMWARE_FIRST_FLAG, &amp;rcfw-&gt;flags)) {</div><div class='add'>+		memcpy(cmdqe, preq, min_t(u32, bsize, sizeof(*cmdqe)));</div><div class='add'>+		preq += min_t(u32, bsize, sizeof(*cmdqe));</div><div class='add'>+		bsize -= min_t(u32, bsize, sizeof(*cmdqe));</div><div class='add'>+		hwq-&gt;prod++;</div><div class='add'>+	} while (bsize &gt; 0);</div><div class='add'>+	cmdq-&gt;seq_num++;</div><div class='add'>+</div><div class='add'>+	cmdq_prod = hwq-&gt;prod &amp; 0xFFFF;</div><div class='add'>+	if (test_bit(FIRMWARE_FIRST_FLAG, &amp;cmdq-&gt;flags)) {</div><div class='ctx'> 		/* The very first doorbell write</div><div class='ctx'> 		 * is required to set this flag</div><div class='ctx'> 		 * which prompts the FW to reset</div><div class='ctx'> 		 * its internal pointers</div><div class='ctx'> 		 */</div><div class='ctx'> 		cmdq_prod |= BIT(FIRMWARE_FIRST_FLAG);</div><div class='del'>-		clear_bit(FIRMWARE_FIRST_FLAG, &amp;rcfw-&gt;flags);</div><div class='add'>+		clear_bit(FIRMWARE_FIRST_FLAG, &amp;cmdq-&gt;flags);</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='ctx'> 	/* ring CMDQ DB */</div><div class='ctx'> 	wmb();</div><div class='del'>-	writel(cmdq_prod, rcfw-&gt;cmdq_bar_reg_iomem +</div><div class='del'>-	       rcfw-&gt;cmdq_bar_reg_prod_off);</div><div class='del'>-	writel(RCFW_CMDQ_TRIG_VAL, rcfw-&gt;cmdq_bar_reg_iomem +</div><div class='del'>-	       rcfw-&gt;cmdq_bar_reg_trig_off);</div><div class='del'>-done:</div><div class='del'>-	spin_unlock_irqrestore(&amp;cmdq-&gt;lock, flags);</div><div class='add'>+	writel(cmdq_prod, cmdq-&gt;cmdq_mbox.prod);</div><div class='add'>+	writel(RCFW_CMDQ_TRIG_VAL, cmdq-&gt;cmdq_mbox.db);</div><div class='add'>+	print_hex_dump_bytes("req: ", DUMP_PREFIX_OFFSET, msg-&gt;req, msg-&gt;req_sz);</div><div class='add'>+	spin_unlock_bh(&amp;hwq-&gt;lock);</div><div class='ctx'> 	/* Return the CREQ response pointer */</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,</div><div class='del'>-				 struct cmdq_base *req,</div><div class='del'>-				 struct creq_base *resp,</div><div class='del'>-				 void *sb, u8 is_block)</div><div class='add'>+/**</div><div class='add'>+ * __poll_for_resp   -	self poll completion for rcfw command</div><div class='add'>+ * @rcfw:     rcfw channel instance of rdev</div><div class='add'>+ * @cookie:   cookie to track the command</div><div class='add'>+ *</div><div class='add'>+ * It works same as __wait_for_resp except this function will</div><div class='add'>+ * do self polling in sort interval since interrupt is disabled.</div><div class='add'>+ * This function can not be called from non-sleepable context.</div><div class='add'>+ *</div><div class='add'>+ * Returns:</div><div class='add'>+ * -ETIMEDOUT if command is not completed in specific time interval.</div><div class='add'>+ * 0 if command is completed by firmware.</div><div class='add'>+ */</div><div class='add'>+static int __poll_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)</div><div class='ctx'> {</div><div class='del'>-	struct creq_qp_event *evnt = (struct creq_qp_event *)resp;</div><div class='del'>-	u16 cookie;</div><div class='del'>-	u8 opcode, retry_cnt = 0xFF;</div><div class='del'>-	int rc = 0;</div><div class='add'>+	struct bnxt_qplib_cmdq_ctx *cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+	struct bnxt_qplib_crsqe *crsqe;</div><div class='add'>+	unsigned long issue_time;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	do {</div><div class='del'>-		opcode = req-&gt;opcode;</div><div class='del'>-		rc = __send_message(rcfw, req, resp, sb, is_block);</div><div class='del'>-		cookie = le16_to_cpu(req-&gt;cookie) &amp; RCFW_MAX_COOKIE_VALUE;</div><div class='del'>-		if (!rc)</div><div class='del'>-			break;</div><div class='add'>+	issue_time = jiffies;</div><div class='add'>+	crsqe = &amp;rcfw-&gt;crsqe_tbl[cookie];</div><div class='ctx'> </div><div class='del'>-		if (!retry_cnt || (rc != -EAGAIN &amp;&amp; rc != -EBUSY)) {</div><div class='del'>-			/* send failed */</div><div class='del'>-			dev_err(&amp;rcfw-&gt;pdev-&gt;dev, "cmdq[%#x]=%#x send failed\n",</div><div class='del'>-				cookie, opcode);</div><div class='del'>-			return rc;</div><div class='add'>+	do {</div><div class='add'>+		if (test_bit(ERR_DEVICE_DETACHED, &amp;cmdq-&gt;flags))</div><div class='add'>+			return bnxt_qplib_map_rc(crsqe-&gt;opcode);</div><div class='add'>+		if (test_bit(FIRMWARE_STALL_DETECTED, &amp;cmdq-&gt;flags))</div><div class='add'>+			return -ETIMEDOUT;</div><div class='add'>+</div><div class='add'>+		usleep_range(1000, 1001);</div><div class='add'>+</div><div class='add'>+		bnxt_qplib_service_creq(&amp;rcfw-&gt;creq.creq_tasklet);</div><div class='add'>+		if (!crsqe-&gt;is_in_used)</div><div class='add'>+			return 0;</div><div class='add'>+		if (jiffies_to_msecs(jiffies - issue_time) &gt;</div><div class='add'>+		    (rcfw-&gt;max_timeout * 1000)) {</div><div class='add'>+			ret = bnxt_re_is_fw_stalled(rcfw, cookie);</div><div class='add'>+			if (ret)</div><div class='add'>+				return ret;</div><div class='ctx'> 		}</div><div class='del'>-		is_block ? mdelay(1) : usleep_range(500, 1000);</div><div class='add'>+	} while (true);</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int __send_message_basic_sanity(struct bnxt_qplib_rcfw *rcfw,</div><div class='add'>+				       struct bnxt_qplib_cmdqmsg *msg,</div><div class='add'>+				       u8 opcode)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_cmdq_ctx *cmdq;</div><div class='add'>+</div><div class='add'>+	cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+</div><div class='add'>+	/* Prevent posting if f/w is not in a state to process */</div><div class='add'>+	if (test_bit(ERR_DEVICE_DETACHED, &amp;rcfw-&gt;cmdq.flags))</div><div class='add'>+		return -ENXIO;</div><div class='add'>+</div><div class='add'>+	if (test_bit(FIRMWARE_STALL_DETECTED, &amp;cmdq-&gt;flags))</div><div class='add'>+		return -ETIMEDOUT;</div><div class='add'>+</div><div class='add'>+	if (test_bit(FIRMWARE_INITIALIZED_FLAG, &amp;cmdq-&gt;flags) &amp;&amp;</div><div class='add'>+	    opcode == CMDQ_BASE_OPCODE_INITIALIZE_FW) {</div><div class='add'>+		dev_err(&amp;rcfw-&gt;pdev-&gt;dev, "QPLIB: RCFW already initialized!");</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	} while (retry_cnt--);</div><div class='add'>+	if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &amp;cmdq-&gt;flags) &amp;&amp;</div><div class='add'>+	    (opcode != CMDQ_BASE_OPCODE_QUERY_FUNC &amp;&amp;</div><div class='add'>+	     opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW &amp;&amp;</div><div class='add'>+	     opcode != CMDQ_BASE_OPCODE_QUERY_VERSION)) {</div><div class='add'>+		dev_err(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='add'>+			"QPLIB: RCFW not initialized, reject opcode 0x%x",</div><div class='add'>+			opcode);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* This function will just post and do not bother about completion */</div><div class='add'>+static void __destroy_timedout_ah(struct bnxt_qplib_rcfw *rcfw,</div><div class='add'>+				  struct creq_create_ah_resp *create_ah_resp)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_destroy_ah req = {};</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_DESTROY_AH,</div><div class='add'>+				 sizeof(req));</div><div class='add'>+	req.ah_cid = create_ah_resp-&gt;xid;</div><div class='add'>+	msg.req = (struct cmdq_base *)&amp;req;</div><div class='add'>+	msg.req_sz = sizeof(req);</div><div class='add'>+	__send_message_no_waiter(rcfw, &amp;msg);</div><div class='add'>+	dev_info_ratelimited(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='add'>+			     "From %s: ah_cid = %d timeout_send %d\n",</div><div class='add'>+			     __func__, req.ah_cid,</div><div class='add'>+			     atomic_read(&amp;rcfw-&gt;timeout_send));</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (is_block)</div><div class='add'>+/**</div><div class='add'>+ * __bnxt_qplib_rcfw_send_message   -	qplib interface to send</div><div class='add'>+ * and complete rcfw command.</div><div class='add'>+ * @rcfw:   rcfw channel instance of rdev</div><div class='add'>+ * @msg:    qplib message internal</div><div class='add'>+ *</div><div class='add'>+ * This function does not account shadow queue depth. It will send</div><div class='add'>+ * all the command unconditionally as long as send queue is not full.</div><div class='add'>+ *</div><div class='add'>+ * Returns:</div><div class='add'>+ * 0 if command completed by firmware.</div><div class='add'>+ * Non zero if the command is not completed by firmware.</div><div class='add'>+ */</div><div class='add'>+static int __bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,</div><div class='add'>+					  struct bnxt_qplib_cmdqmsg *msg)</div><div class='add'>+{</div><div class='add'>+	struct creq_qp_event *evnt = (struct creq_qp_event *)msg-&gt;resp;</div><div class='add'>+	struct bnxt_qplib_crsqe *crsqe;</div><div class='add'>+	u16 cookie;</div><div class='add'>+	int rc;</div><div class='add'>+	u8 opcode;</div><div class='add'>+</div><div class='add'>+	opcode = __get_cmdq_base_opcode(msg-&gt;req, msg-&gt;req_sz);</div><div class='add'>+</div><div class='add'>+	rc = __send_message_basic_sanity(rcfw, msg, opcode);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc == -ENXIO ? bnxt_qplib_map_rc(opcode) : rc;</div><div class='add'>+</div><div class='add'>+	rc = __send_message(rcfw, msg, opcode);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	cookie = le16_to_cpu(__get_cmdq_base_cookie(msg-&gt;req, msg-&gt;req_sz))</div><div class='add'>+				&amp; RCFW_MAX_COOKIE_VALUE;</div><div class='add'>+</div><div class='add'>+	if (msg-&gt;block)</div><div class='ctx'> 		rc = __block_for_resp(rcfw, cookie);</div><div class='del'>-	else</div><div class='add'>+	else if (atomic_read(&amp;rcfw-&gt;rcfw_intr_enabled))</div><div class='ctx'> 		rc = __wait_for_resp(rcfw, cookie);</div><div class='add'>+	else</div><div class='add'>+		rc = __poll_for_resp(rcfw, cookie);</div><div class='add'>+</div><div class='ctx'> 	if (rc) {</div><div class='del'>-		/* timed out */</div><div class='del'>-		dev_err(&amp;rcfw-&gt;pdev-&gt;dev, "cmdq[%#x]=%#x timedout (%d)msec\n",</div><div class='del'>-			cookie, opcode, RCFW_CMD_WAIT_TIME_MS);</div><div class='del'>-		set_bit(FIRMWARE_TIMED_OUT, &amp;rcfw-&gt;flags);</div><div class='del'>-		return rc;</div><div class='add'>+		spin_lock_bh(&amp;rcfw-&gt;cmdq.hwq.lock);</div><div class='add'>+		crsqe = &amp;rcfw-&gt;crsqe_tbl[cookie];</div><div class='add'>+		crsqe-&gt;is_waiter_alive = false;</div><div class='add'>+		if (rc == -ENODEV)</div><div class='add'>+			set_bit(FIRMWARE_STALL_DETECTED, &amp;rcfw-&gt;cmdq.flags);</div><div class='add'>+		spin_unlock_bh(&amp;rcfw-&gt;cmdq.hwq.lock);</div><div class='add'>+		return -ETIMEDOUT;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (evnt-&gt;status) {</div><div class='ctx'> 		/* failed with status */</div><div class='ctx'> 		dev_err(&amp;rcfw-&gt;pdev-&gt;dev, "cmdq[%#x]=%#x status %#x\n",</div><div class='ctx'> 			cookie, opcode, evnt-&gt;status);</div><div class='del'>-		rc = -EFAULT;</div><div class='add'>+		rc = -EIO;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * bnxt_qplib_rcfw_send_message   -	qplib interface to send</div><div class='add'>+ * and complete rcfw command.</div><div class='add'>+ * @rcfw:   rcfw channel instance of rdev</div><div class='add'>+ * @msg:    qplib message internal</div><div class='add'>+ *</div><div class='add'>+ * Driver interact with Firmware through rcfw channel/slow path in two ways.</div><div class='add'>+ * a. Blocking rcfw command send. In this path, driver cannot hold</div><div class='add'>+ * the context for longer period since it is holding cpu until</div><div class='add'>+ * command is not completed.</div><div class='add'>+ * b. Non-blocking rcfw command send. In this path, driver can hold the</div><div class='add'>+ * context for longer period. There may be many pending command waiting</div><div class='add'>+ * for completion because of non-blocking nature.</div><div class='add'>+ *</div><div class='add'>+ * Driver will use shadow queue depth. Current queue depth of 8K</div><div class='add'>+ * (due to size of rcfw message there can be actual ~4K rcfw outstanding)</div><div class='add'>+ * is not optimal for rcfw command processing in firmware.</div><div class='add'>+ *</div><div class='add'>+ * Restrict at max #RCFW_CMD_NON_BLOCKING_SHADOW_QD Non-Blocking rcfw commands.</div><div class='add'>+ * Allow all blocking commands until there is no queue full.</div><div class='add'>+ *</div><div class='add'>+ * Returns:</div><div class='add'>+ * 0 if command completed by firmware.</div><div class='add'>+ * Non zero if the command is not completed by firmware.</div><div class='add'>+ */</div><div class='add'>+int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,</div><div class='add'>+				 struct bnxt_qplib_cmdqmsg *msg)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!msg-&gt;block) {</div><div class='add'>+		down(&amp;rcfw-&gt;rcfw_inflight);</div><div class='add'>+		ret = __bnxt_qplib_rcfw_send_message(rcfw, msg);</div><div class='add'>+		up(&amp;rcfw-&gt;rcfw_inflight);</div><div class='add'>+	} else {</div><div class='add'>+		ret = __bnxt_qplib_rcfw_send_message(rcfw, msg);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /* Completions */</div><div class='ctx'> static int bnxt_qplib_process_func_event(struct bnxt_qplib_rcfw *rcfw,</div><div class='ctx'> 					 struct creq_func_event *func_event)</div><div class='ctx'> {</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='ctx'> 	switch (func_event-&gt;event) {</div><div class='ctx'> 	case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR:</div><div class='ctx'> 		break;</div><div class='hunk'>@@ -280,37 +611,47 @@ static int bnxt_qplib_process_func_event(struct bnxt_qplib_rcfw *rcfw,</div><div class='ctx'> 	default:</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='del'>-	return 0;</div><div class='add'>+</div><div class='add'>+	rc = rcfw-&gt;creq.aeq_handler(rcfw, (void *)func_event, NULL);</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw,</div><div class='del'>-				       struct creq_qp_event *qp_event)</div><div class='add'>+				       struct creq_qp_event *qp_event,</div><div class='add'>+				       u32 *num_wait)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_qplib_hwq *cmdq = &amp;rcfw-&gt;cmdq;</div><div class='ctx'> 	struct creq_qp_error_notification *err_event;</div><div class='del'>-	struct bnxt_qplib_crsq *crsqe;</div><div class='del'>-	unsigned long flags;</div><div class='add'>+	struct bnxt_qplib_hwq *hwq = &amp;rcfw-&gt;cmdq.hwq;</div><div class='add'>+	struct bnxt_qplib_crsqe *crsqe;</div><div class='add'>+	u32 qp_id, tbl_indx, req_size;</div><div class='ctx'> 	struct bnxt_qplib_qp *qp;</div><div class='del'>-	u16 cbit, blocked = 0;</div><div class='del'>-	u16 cookie;</div><div class='del'>-	__le16  mcookie;</div><div class='del'>-	u32 qp_id;</div><div class='add'>+	u16 cookie, blocked = 0;</div><div class='add'>+	bool is_waiter_alive;</div><div class='add'>+	struct pci_dev *pdev;</div><div class='add'>+	u32 wait_cmds = 0;</div><div class='add'>+	int rc = 0;</div><div class='ctx'> </div><div class='add'>+	pdev = rcfw-&gt;pdev;</div><div class='add'>+	print_hex_dump_bytes("event: ", DUMP_PREFIX_OFFSET, qp_event, sizeof(*qp_event));</div><div class='ctx'> 	switch (qp_event-&gt;event) {</div><div class='ctx'> 	case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION:</div><div class='ctx'> 		err_event = (struct creq_qp_error_notification *)qp_event;</div><div class='ctx'> 		qp_id = le32_to_cpu(err_event-&gt;xid);</div><div class='del'>-		qp = rcfw-&gt;qp_tbl[qp_id].qp_handle;</div><div class='del'>-		dev_dbg(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='del'>-			"Received QP error notification\n");</div><div class='del'>-		dev_dbg(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='add'>+		spin_lock(&amp;rcfw-&gt;tbl_lock);</div><div class='add'>+		tbl_indx = map_qp_id_to_tbl_indx(qp_id, rcfw);</div><div class='add'>+		qp = rcfw-&gt;qp_tbl[tbl_indx].qp_handle;</div><div class='add'>+		if (!qp) {</div><div class='add'>+			spin_unlock(&amp;rcfw-&gt;tbl_lock);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		bnxt_qplib_mark_qp_error(qp);</div><div class='add'>+		rc = rcfw-&gt;creq.aeq_handler(rcfw, qp_event, qp);</div><div class='add'>+		spin_unlock(&amp;rcfw-&gt;tbl_lock);</div><div class='add'>+		dev_dbg(&amp;pdev-&gt;dev, "Received QP error notification\n");</div><div class='add'>+		dev_dbg(&amp;pdev-&gt;dev,</div><div class='ctx'> 			"qpid 0x%x, req_err=0x%x, resp_err=0x%x\n",</div><div class='ctx'> 			qp_id, err_event-&gt;req_err_state_reason,</div><div class='ctx'> 			err_event-&gt;res_err_state_reason);</div><div class='del'>-		if (!qp)</div><div class='del'>-			break;</div><div class='del'>-		bnxt_qplib_mark_qp_error(qp);</div><div class='del'>-		rcfw-&gt;aeq_handler(rcfw, qp_event, qp);</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		/*</div><div class='hunk'>@@ -322,75 +663,107 @@ static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw,</div><div class='ctx'> 		 *</div><div class='ctx'> 		 */</div><div class='ctx'> </div><div class='del'>-		spin_lock_irqsave_nested(&amp;cmdq-&gt;lock, flags,</div><div class='del'>-					 SINGLE_DEPTH_NESTING);</div><div class='add'>+		spin_lock_nested(&amp;hwq-&gt;lock, SINGLE_DEPTH_NESTING);</div><div class='ctx'> 		cookie = le16_to_cpu(qp_event-&gt;cookie);</div><div class='del'>-		mcookie = qp_event-&gt;cookie;</div><div class='ctx'> 		blocked = cookie &amp; RCFW_CMD_IS_BLOCKING;</div><div class='ctx'> 		cookie &amp;= RCFW_MAX_COOKIE_VALUE;</div><div class='del'>-		cbit = cookie % rcfw-&gt;cmdq_depth;</div><div class='del'>-		crsqe = &amp;rcfw-&gt;crsqe_tbl[cbit];</div><div class='del'>-		if (crsqe-&gt;resp &amp;&amp;</div><div class='del'>-		    crsqe-&gt;resp-&gt;cookie  == mcookie) {</div><div class='del'>-			memcpy(crsqe-&gt;resp, qp_event, sizeof(*qp_event));</div><div class='del'>-			crsqe-&gt;resp = NULL;</div><div class='del'>-		} else {</div><div class='del'>-			if (crsqe-&gt;resp &amp;&amp; crsqe-&gt;resp-&gt;cookie)</div><div class='del'>-				dev_err(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='del'>-					"CMD %s cookie sent=%#x, recd=%#x\n",</div><div class='del'>-					crsqe-&gt;resp ? "mismatch" : "collision",</div><div class='del'>-					crsqe-&gt;resp ? crsqe-&gt;resp-&gt;cookie : 0,</div><div class='del'>-					mcookie);</div><div class='add'>+		crsqe = &amp;rcfw-&gt;crsqe_tbl[cookie];</div><div class='add'>+</div><div class='add'>+		if (WARN_ONCE(test_bit(FIRMWARE_STALL_DETECTED,</div><div class='add'>+				       &amp;rcfw-&gt;cmdq.flags),</div><div class='add'>+		    "QPLIB: Unreponsive rcfw channel detected.!!")) {</div><div class='add'>+			dev_info(&amp;pdev-&gt;dev,</div><div class='add'>+				 "rcfw timedout: cookie = %#x, free_slots = %d",</div><div class='add'>+				 cookie, crsqe-&gt;free_slots);</div><div class='add'>+			spin_unlock(&amp;hwq-&gt;lock);</div><div class='add'>+			return rc;</div><div class='ctx'> 		}</div><div class='del'>-		if (!test_and_clear_bit(cbit, rcfw-&gt;cmdq_bitmap))</div><div class='del'>-			dev_warn(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='del'>-				 "CMD bit %d was not requested\n", cbit);</div><div class='del'>-		cmdq-&gt;cons += crsqe-&gt;req_size;</div><div class='add'>+</div><div class='add'>+		if (crsqe-&gt;is_internal_cmd &amp;&amp; !qp_event-&gt;status)</div><div class='add'>+			atomic_dec(&amp;rcfw-&gt;timeout_send);</div><div class='add'>+</div><div class='add'>+		if (crsqe-&gt;is_waiter_alive) {</div><div class='add'>+			if (crsqe-&gt;resp) {</div><div class='add'>+				memcpy(crsqe-&gt;resp, qp_event, sizeof(*qp_event));</div><div class='add'>+				/* Insert write memory barrier to ensure that</div><div class='add'>+				 * response data is copied before clearing the</div><div class='add'>+				 * flags</div><div class='add'>+				 */</div><div class='add'>+				smp_wmb();</div><div class='add'>+			}</div><div class='add'>+			if (!blocked)</div><div class='add'>+				wait_cmds++;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		req_size = crsqe-&gt;req_size;</div><div class='add'>+		is_waiter_alive = crsqe-&gt;is_waiter_alive;</div><div class='add'>+</div><div class='ctx'> 		crsqe-&gt;req_size = 0;</div><div class='add'>+		if (!is_waiter_alive)</div><div class='add'>+			crsqe-&gt;resp = NULL;</div><div class='ctx'> </div><div class='del'>-		if (!blocked)</div><div class='del'>-			wake_up(&amp;rcfw-&gt;waitq);</div><div class='del'>-		spin_unlock_irqrestore(&amp;cmdq-&gt;lock, flags);</div><div class='add'>+		crsqe-&gt;is_in_used = false;</div><div class='add'>+</div><div class='add'>+		hwq-&gt;cons += req_size;</div><div class='add'>+</div><div class='add'>+		/* This is a case to handle below scenario -</div><div class='add'>+		 * Create AH is completed successfully by firmware,</div><div class='add'>+		 * but completion took more time and driver already lost</div><div class='add'>+		 * the context of create_ah from caller.</div><div class='add'>+		 * We have already return failure for create_ah verbs,</div><div class='add'>+		 * so let's destroy the same address vector since it is</div><div class='add'>+		 * no more used in stack. We don't care about completion</div><div class='add'>+		 * in __send_message_no_waiter.</div><div class='add'>+		 * If destroy_ah is failued by firmware, there will be AH</div><div class='add'>+		 * resource leak and relatively not critical +  unlikely</div><div class='add'>+		 * scenario. Current design is not to handle such case.</div><div class='add'>+		 */</div><div class='add'>+		if (!is_waiter_alive &amp;&amp; !qp_event-&gt;status &amp;&amp;</div><div class='add'>+		    qp_event-&gt;event == CREQ_QP_EVENT_EVENT_CREATE_AH)</div><div class='add'>+			__destroy_timedout_ah(rcfw,</div><div class='add'>+					      (struct creq_create_ah_resp *)</div><div class='add'>+					      qp_event);</div><div class='add'>+		spin_unlock(&amp;hwq-&gt;lock);</div><div class='ctx'> 	}</div><div class='del'>-	return 0;</div><div class='add'>+	*num_wait += wait_cmds;</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* SP - CREQ Completion handlers */</div><div class='del'>-static void bnxt_qplib_service_creq(unsigned long data)</div><div class='add'>+static void bnxt_qplib_service_creq(struct tasklet_struct *t)</div><div class='ctx'> {</div><div class='del'>-	struct bnxt_qplib_rcfw *rcfw = (struct bnxt_qplib_rcfw *)data;</div><div class='del'>-	bool gen_p5 = bnxt_qplib_is_chip_gen_p5(rcfw-&gt;res-&gt;cctx);</div><div class='del'>-	struct bnxt_qplib_hwq *creq = &amp;rcfw-&gt;creq;</div><div class='add'>+	struct bnxt_qplib_rcfw *rcfw = from_tasklet(rcfw, t, creq.creq_tasklet);</div><div class='add'>+	struct bnxt_qplib_creq_ctx *creq = &amp;rcfw-&gt;creq;</div><div class='ctx'> 	u32 type, budget = CREQ_ENTRY_POLL_BUDGET;</div><div class='del'>-	struct creq_base *creqe, **creq_ptr;</div><div class='del'>-	u32 sw_cons, raw_cons;</div><div class='del'>-	unsigned long flags;</div><div class='add'>+	struct bnxt_qplib_hwq *hwq = &amp;creq-&gt;hwq;</div><div class='add'>+	struct creq_base *creqe;</div><div class='add'>+	u32 num_wakeup = 0;</div><div class='add'>+	u32 hw_polled = 0;</div><div class='ctx'> </div><div class='ctx'> 	/* Service the CREQ until budget is over */</div><div class='del'>-	spin_lock_irqsave(&amp;creq-&gt;lock, flags);</div><div class='del'>-	raw_cons = creq-&gt;cons;</div><div class='add'>+	spin_lock_bh(&amp;hwq-&gt;lock);</div><div class='ctx'> 	while (budget &gt; 0) {</div><div class='del'>-		sw_cons = HWQ_CMP(raw_cons, creq);</div><div class='del'>-		creq_ptr = (struct creq_base **)creq-&gt;pbl_ptr;</div><div class='del'>-		creqe = &amp;creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)];</div><div class='del'>-		if (!CREQ_CMP_VALID(creqe, raw_cons, creq-&gt;max_elements))</div><div class='add'>+		creqe = bnxt_qplib_get_qe(hwq, hwq-&gt;cons, NULL);</div><div class='add'>+		if (!CREQ_CMP_VALID(creqe, creq-&gt;creq_db.dbinfo.flags))</div><div class='ctx'> 			break;</div><div class='ctx'> 		/* The valid test of the entry must be done first before</div><div class='ctx'> 		 * reading any further.</div><div class='ctx'> 		 */</div><div class='ctx'> 		dma_rmb();</div><div class='add'>+		rcfw-&gt;cmdq.last_seen = jiffies;</div><div class='ctx'> </div><div class='ctx'> 		type = creqe-&gt;type &amp; CREQ_BASE_TYPE_MASK;</div><div class='ctx'> 		switch (type) {</div><div class='ctx'> 		case CREQ_BASE_TYPE_QP_EVENT:</div><div class='ctx'> 			bnxt_qplib_process_qp_event</div><div class='del'>-				(rcfw, (struct creq_qp_event *)creqe);</div><div class='del'>-			rcfw-&gt;creq_qp_event_processed++;</div><div class='add'>+				(rcfw, (struct creq_qp_event *)creqe,</div><div class='add'>+				 &amp;num_wakeup);</div><div class='add'>+			creq-&gt;stats.creq_qp_event_processed++;</div><div class='ctx'> 			break;</div><div class='ctx'> 		case CREQ_BASE_TYPE_FUNC_EVENT:</div><div class='ctx'> 			if (!bnxt_qplib_process_func_event</div><div class='ctx'> 			    (rcfw, (struct creq_func_event *)creqe))</div><div class='del'>-				rcfw-&gt;creq_func_event_processed++;</div><div class='add'>+				creq-&gt;stats.creq_func_event_processed++;</div><div class='ctx'> 			else</div><div class='ctx'> 				dev_warn(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='ctx'> 					 "aeqe:%#x Not handled\n", type);</div><div class='hunk'>@@ -402,32 +775,34 @@ static void bnxt_qplib_service_creq(unsigned long data)</div><div class='ctx'> 					 type);</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='del'>-		raw_cons++;</div><div class='ctx'> 		budget--;</div><div class='add'>+		hw_polled++;</div><div class='add'>+		bnxt_qplib_hwq_incr_cons(hwq-&gt;max_elements, &amp;hwq-&gt;cons,</div><div class='add'>+					 1, &amp;creq-&gt;creq_db.dbinfo.flags);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (creq-&gt;cons != raw_cons) {</div><div class='del'>-		creq-&gt;cons = raw_cons;</div><div class='del'>-		bnxt_qplib_ring_creq_db_rearm(rcfw-&gt;creq_bar_reg_iomem,</div><div class='del'>-					      raw_cons, creq-&gt;max_elements,</div><div class='del'>-					      rcfw-&gt;creq_ring_id, gen_p5);</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;creq-&gt;lock, flags);</div><div class='add'>+	if (hw_polled)</div><div class='add'>+		bnxt_qplib_ring_nq_db(&amp;creq-&gt;creq_db.dbinfo,</div><div class='add'>+				      rcfw-&gt;res-&gt;cctx, true);</div><div class='add'>+	spin_unlock_bh(&amp;hwq-&gt;lock);</div><div class='add'>+	if (num_wakeup)</div><div class='add'>+		wake_up_nr(&amp;rcfw-&gt;cmdq.waitq, num_wakeup);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = dev_instance;</div><div class='del'>-	struct bnxt_qplib_hwq *creq = &amp;rcfw-&gt;creq;</div><div class='del'>-	struct creq_base **creq_ptr;</div><div class='add'>+	struct bnxt_qplib_creq_ctx *creq;</div><div class='add'>+	struct bnxt_qplib_hwq *hwq;</div><div class='ctx'> 	u32 sw_cons;</div><div class='ctx'> </div><div class='add'>+	creq = &amp;rcfw-&gt;creq;</div><div class='add'>+	hwq = &amp;creq-&gt;hwq;</div><div class='ctx'> 	/* Prefetch the CREQ element */</div><div class='del'>-	sw_cons = HWQ_CMP(creq-&gt;cons, creq);</div><div class='del'>-	creq_ptr = (struct creq_base **)rcfw-&gt;creq.pbl_ptr;</div><div class='del'>-	prefetch(&amp;creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)]);</div><div class='add'>+	sw_cons = HWQ_CMP(hwq-&gt;cons, hwq);</div><div class='add'>+	prefetch(bnxt_qplib_get_qe(hwq, sw_cons, NULL));</div><div class='ctx'> </div><div class='del'>-	tasklet_schedule(&amp;rcfw-&gt;worker);</div><div class='add'>+	tasklet_schedule(&amp;creq-&gt;creq_tasklet);</div><div class='ctx'> </div><div class='ctx'> 	return IRQ_HANDLED;</div><div class='ctx'> }</div><div class='hunk'>@@ -435,47 +810,37 @@ static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance)</div><div class='ctx'> /* RCFW */</div><div class='ctx'> int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw)</div><div class='ctx'> {</div><div class='del'>-	struct cmdq_deinitialize_fw req;</div><div class='del'>-	struct creq_deinitialize_fw_resp resp;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='add'>+	struct creq_deinitialize_fw_resp resp = {};</div><div class='add'>+	struct cmdq_deinitialize_fw req = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, DEINITIALIZE_FW, cmd_flags);</div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req, (void *)&amp;resp,</div><div class='del'>-					  NULL, 0);</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_DEINITIALIZE_FW,</div><div class='add'>+				 sizeof(req));</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL,</div><div class='add'>+				sizeof(req), sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		return rc;</div><div class='ctx'> </div><div class='del'>-	clear_bit(FIRMWARE_INITIALIZED_FLAG, &amp;rcfw-&gt;flags);</div><div class='add'>+	clear_bit(FIRMWARE_INITIALIZED_FLAG, &amp;rcfw-&gt;cmdq.flags);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int __get_pbl_pg_idx(struct bnxt_qplib_pbl *pbl)</div><div class='del'>-{</div><div class='del'>-	return (pbl-&gt;pg_size == ROCE_PG_SIZE_4K ?</div><div class='del'>-				      CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K :</div><div class='del'>-		pbl-&gt;pg_size == ROCE_PG_SIZE_8K ?</div><div class='del'>-				      CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K :</div><div class='del'>-		pbl-&gt;pg_size == ROCE_PG_SIZE_64K ?</div><div class='del'>-				      CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K :</div><div class='del'>-		pbl-&gt;pg_size == ROCE_PG_SIZE_2M ?</div><div class='del'>-				      CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M :</div><div class='del'>-		pbl-&gt;pg_size == ROCE_PG_SIZE_8M ?</div><div class='del'>-				      CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M :</div><div class='del'>-		pbl-&gt;pg_size == ROCE_PG_SIZE_1G ?</div><div class='del'>-				      CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G :</div><div class='del'>-				      CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,</div><div class='ctx'> 			 struct bnxt_qplib_ctx *ctx, int is_virtfn)</div><div class='ctx'> {</div><div class='del'>-	struct cmdq_initialize_fw req;</div><div class='del'>-	struct creq_initialize_fw_resp resp;</div><div class='del'>-	u16 cmd_flags = 0, level;</div><div class='add'>+	struct creq_initialize_fw_resp resp = {};</div><div class='add'>+	struct cmdq_initialize_fw req = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	u16 flags = 0;</div><div class='add'>+	u8 pgsz, lvl;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, INITIALIZE_FW, cmd_flags);</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_INITIALIZE_FW,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> 	/* Supply (log-base-2-of-host-page-size - base-page-shift)</div><div class='ctx'> 	 * to bono to adjust the doorbell page sizes.</div><div class='ctx'> 	 */</div><div class='hunk'>@@ -488,34 +853,33 @@ int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,</div><div class='ctx'> 	 * shall setup this area for VF. Skipping the</div><div class='ctx'> 	 * HW programming</div><div class='ctx'> 	 */</div><div class='del'>-	if (is_virtfn || bnxt_qplib_is_chip_gen_p5(rcfw-&gt;res-&gt;cctx))</div><div class='add'>+	if (is_virtfn || bnxt_qplib_is_chip_gen_p5_p7(rcfw-&gt;res-&gt;cctx))</div><div class='ctx'> 		goto skip_ctx_setup;</div><div class='ctx'> </div><div class='del'>-	level = ctx-&gt;qpc_tbl.level;</div><div class='del'>-	req.qpc_pg_size_qpc_lvl = (level &lt;&lt; CMDQ_INITIALIZE_FW_QPC_LVL_SFT) |</div><div class='del'>-				__get_pbl_pg_idx(&amp;ctx-&gt;qpc_tbl.pbl[level]);</div><div class='del'>-	level = ctx-&gt;mrw_tbl.level;</div><div class='del'>-	req.mrw_pg_size_mrw_lvl = (level &lt;&lt; CMDQ_INITIALIZE_FW_MRW_LVL_SFT) |</div><div class='del'>-				__get_pbl_pg_idx(&amp;ctx-&gt;mrw_tbl.pbl[level]);</div><div class='del'>-	level = ctx-&gt;srqc_tbl.level;</div><div class='del'>-	req.srq_pg_size_srq_lvl = (level &lt;&lt; CMDQ_INITIALIZE_FW_SRQ_LVL_SFT) |</div><div class='del'>-				__get_pbl_pg_idx(&amp;ctx-&gt;srqc_tbl.pbl[level]);</div><div class='del'>-	level = ctx-&gt;cq_tbl.level;</div><div class='del'>-	req.cq_pg_size_cq_lvl = (level &lt;&lt; CMDQ_INITIALIZE_FW_CQ_LVL_SFT) |</div><div class='del'>-				__get_pbl_pg_idx(&amp;ctx-&gt;cq_tbl.pbl[level]);</div><div class='del'>-	level = ctx-&gt;srqc_tbl.level;</div><div class='del'>-	req.srq_pg_size_srq_lvl = (level &lt;&lt; CMDQ_INITIALIZE_FW_SRQ_LVL_SFT) |</div><div class='del'>-				__get_pbl_pg_idx(&amp;ctx-&gt;srqc_tbl.pbl[level]);</div><div class='del'>-	level = ctx-&gt;cq_tbl.level;</div><div class='del'>-	req.cq_pg_size_cq_lvl = (level &lt;&lt; CMDQ_INITIALIZE_FW_CQ_LVL_SFT) |</div><div class='del'>-				__get_pbl_pg_idx(&amp;ctx-&gt;cq_tbl.pbl[level]);</div><div class='del'>-	level = ctx-&gt;tim_tbl.level;</div><div class='del'>-	req.tim_pg_size_tim_lvl = (level &lt;&lt; CMDQ_INITIALIZE_FW_TIM_LVL_SFT) |</div><div class='del'>-				  __get_pbl_pg_idx(&amp;ctx-&gt;tim_tbl.pbl[level]);</div><div class='del'>-	level = ctx-&gt;tqm_pde_level;</div><div class='del'>-	req.tqm_pg_size_tqm_lvl = (level &lt;&lt; CMDQ_INITIALIZE_FW_TQM_LVL_SFT) |</div><div class='del'>-				  __get_pbl_pg_idx(&amp;ctx-&gt;tqm_pde.pbl[level]);</div><div class='del'>-</div><div class='add'>+	lvl = ctx-&gt;qpc_tbl.level;</div><div class='add'>+	pgsz = bnxt_qplib_base_pg_size(&amp;ctx-&gt;qpc_tbl);</div><div class='add'>+	req.qpc_pg_size_qpc_lvl = (pgsz &lt;&lt; CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) |</div><div class='add'>+				   lvl;</div><div class='add'>+	lvl = ctx-&gt;mrw_tbl.level;</div><div class='add'>+	pgsz = bnxt_qplib_base_pg_size(&amp;ctx-&gt;mrw_tbl);</div><div class='add'>+	req.mrw_pg_size_mrw_lvl = (pgsz &lt;&lt; CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) |</div><div class='add'>+				   lvl;</div><div class='add'>+	lvl = ctx-&gt;srqc_tbl.level;</div><div class='add'>+	pgsz = bnxt_qplib_base_pg_size(&amp;ctx-&gt;srqc_tbl);</div><div class='add'>+	req.srq_pg_size_srq_lvl = (pgsz &lt;&lt; CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) |</div><div class='add'>+				   lvl;</div><div class='add'>+	lvl = ctx-&gt;cq_tbl.level;</div><div class='add'>+	pgsz = bnxt_qplib_base_pg_size(&amp;ctx-&gt;cq_tbl);</div><div class='add'>+	req.cq_pg_size_cq_lvl = (pgsz &lt;&lt; CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) |</div><div class='add'>+				 lvl;</div><div class='add'>+	lvl = ctx-&gt;tim_tbl.level;</div><div class='add'>+	pgsz = bnxt_qplib_base_pg_size(&amp;ctx-&gt;tim_tbl);</div><div class='add'>+	req.tim_pg_size_tim_lvl = (pgsz &lt;&lt; CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) |</div><div class='add'>+				   lvl;</div><div class='add'>+	lvl = ctx-&gt;tqm_ctx.pde.level;</div><div class='add'>+	pgsz = bnxt_qplib_base_pg_size(&amp;ctx-&gt;tqm_ctx.pde);</div><div class='add'>+	req.tqm_pg_size_tqm_lvl = (pgsz &lt;&lt; CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) |</div><div class='add'>+				   lvl;</div><div class='ctx'> 	req.qpc_page_dir =</div><div class='ctx'> 		cpu_to_le64(ctx-&gt;qpc_tbl.pbl[PBL_LVL_0].pg_map_arr[0]);</div><div class='ctx'> 	req.mrw_page_dir =</div><div class='hunk'>@@ -527,83 +891,91 @@ int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,</div><div class='ctx'> 	req.tim_page_dir =</div><div class='ctx'> 		cpu_to_le64(ctx-&gt;tim_tbl.pbl[PBL_LVL_0].pg_map_arr[0]);</div><div class='ctx'> 	req.tqm_page_dir =</div><div class='del'>-		cpu_to_le64(ctx-&gt;tqm_pde.pbl[PBL_LVL_0].pg_map_arr[0]);</div><div class='add'>+		cpu_to_le64(ctx-&gt;tqm_ctx.pde.pbl[PBL_LVL_0].pg_map_arr[0]);</div><div class='ctx'> </div><div class='ctx'> 	req.number_of_qp = cpu_to_le32(ctx-&gt;qpc_tbl.max_elements);</div><div class='ctx'> 	req.number_of_mrw = cpu_to_le32(ctx-&gt;mrw_tbl.max_elements);</div><div class='ctx'> 	req.number_of_srq = cpu_to_le32(ctx-&gt;srqc_tbl.max_elements);</div><div class='ctx'> 	req.number_of_cq = cpu_to_le32(ctx-&gt;cq_tbl.max_elements);</div><div class='ctx'> </div><div class='del'>-	req.max_qp_per_vf = cpu_to_le32(ctx-&gt;vf_res.max_qp_per_vf);</div><div class='del'>-	req.max_mrw_per_vf = cpu_to_le32(ctx-&gt;vf_res.max_mrw_per_vf);</div><div class='del'>-	req.max_srq_per_vf = cpu_to_le32(ctx-&gt;vf_res.max_srq_per_vf);</div><div class='del'>-	req.max_cq_per_vf = cpu_to_le32(ctx-&gt;vf_res.max_cq_per_vf);</div><div class='del'>-	req.max_gid_per_vf = cpu_to_le32(ctx-&gt;vf_res.max_gid_per_vf);</div><div class='del'>-</div><div class='ctx'> skip_ctx_setup:</div><div class='add'>+	if (BNXT_RE_HW_RETX(rcfw-&gt;res-&gt;dattr-&gt;dev_cap_flags))</div><div class='add'>+		flags |= CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED;</div><div class='add'>+	if (_is_optimize_modify_qp_supported(rcfw-&gt;res-&gt;dattr-&gt;dev_cap_flags2))</div><div class='add'>+		flags |= CMDQ_INITIALIZE_FW_FLAGS_OPTIMIZE_MODIFY_QP_SUPPORTED;</div><div class='add'>+	if (rcfw-&gt;res-&gt;en_dev-&gt;flags &amp; BNXT_EN_FLAG_ROCE_VF_RES_MGMT)</div><div class='add'>+		flags |= CMDQ_INITIALIZE_FW_FLAGS_L2_VF_RESOURCE_MGMT;</div><div class='add'>+	if (bnxt_qplib_roce_mirror_supported(rcfw-&gt;res-&gt;cctx)) {</div><div class='add'>+		flags |= CMDQ_INITIALIZE_FW_FLAGS_MIRROR_ON_ROCE_SUPPORTED;</div><div class='add'>+		rcfw-&gt;roce_mirror = true;</div><div class='add'>+	}</div><div class='add'>+	req.flags |= cpu_to_le16(flags);</div><div class='ctx'> 	req.stat_ctx_id = cpu_to_le32(ctx-&gt;stats.fw_id);</div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req, (void *)&amp;resp,</div><div class='del'>-					  NULL, 0);</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req), sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		return rc;</div><div class='del'>-	set_bit(FIRMWARE_INITIALIZED_FLAG, &amp;rcfw-&gt;flags);</div><div class='add'>+	set_bit(FIRMWARE_INITIALIZED_FLAG, &amp;rcfw-&gt;cmdq.flags);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)</div><div class='ctx'> {</div><div class='del'>-	kfree(rcfw-&gt;qp_tbl);</div><div class='ctx'> 	kfree(rcfw-&gt;crsqe_tbl);</div><div class='del'>-	bnxt_qplib_free_hwq(rcfw-&gt;pdev, &amp;rcfw-&gt;cmdq);</div><div class='del'>-	bnxt_qplib_free_hwq(rcfw-&gt;pdev, &amp;rcfw-&gt;creq);</div><div class='add'>+	bnxt_qplib_free_hwq(rcfw-&gt;res, &amp;rcfw-&gt;cmdq.hwq);</div><div class='add'>+	bnxt_qplib_free_hwq(rcfw-&gt;res, &amp;rcfw-&gt;creq.hwq);</div><div class='ctx'> 	rcfw-&gt;pdev = NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev,</div><div class='add'>+int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res,</div><div class='ctx'> 				  struct bnxt_qplib_rcfw *rcfw,</div><div class='del'>-				  struct bnxt_qplib_ctx *ctx,</div><div class='del'>-				  int qp_tbl_sz)</div><div class='add'>+				  struct bnxt_qplib_ctx *ctx)</div><div class='ctx'> {</div><div class='del'>-	u8 hwq_type;</div><div class='del'>-</div><div class='del'>-	rcfw-&gt;pdev = pdev;</div><div class='del'>-	rcfw-&gt;creq.max_elements = BNXT_QPLIB_CREQE_MAX_CNT;</div><div class='del'>-	hwq_type = bnxt_qplib_get_hwq_type(rcfw-&gt;res);</div><div class='del'>-	if (bnxt_qplib_alloc_init_hwq(rcfw-&gt;pdev, &amp;rcfw-&gt;creq, NULL,</div><div class='del'>-				      &amp;rcfw-&gt;creq.max_elements,</div><div class='del'>-				      BNXT_QPLIB_CREQE_UNITS,</div><div class='del'>-				      0, PAGE_SIZE, hwq_type)) {</div><div class='add'>+	struct bnxt_qplib_hwq_attr hwq_attr = {};</div><div class='add'>+	struct bnxt_qplib_sg_info sginfo = {};</div><div class='add'>+	struct bnxt_qplib_cmdq_ctx *cmdq;</div><div class='add'>+	struct bnxt_qplib_creq_ctx *creq;</div><div class='add'>+</div><div class='add'>+	rcfw-&gt;pdev = res-&gt;pdev;</div><div class='add'>+	cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+	creq = &amp;rcfw-&gt;creq;</div><div class='add'>+	rcfw-&gt;res = res;</div><div class='add'>+</div><div class='add'>+	sginfo.pgsize = PAGE_SIZE;</div><div class='add'>+	sginfo.pgshft = PAGE_SHIFT;</div><div class='add'>+</div><div class='add'>+	hwq_attr.sginfo = &amp;sginfo;</div><div class='add'>+	hwq_attr.res = rcfw-&gt;res;</div><div class='add'>+	hwq_attr.depth = BNXT_QPLIB_CREQE_MAX_CNT;</div><div class='add'>+	hwq_attr.stride = BNXT_QPLIB_CREQE_UNITS;</div><div class='add'>+	hwq_attr.type = bnxt_qplib_get_hwq_type(res);</div><div class='add'>+</div><div class='add'>+	if (bnxt_qplib_alloc_init_hwq(&amp;creq-&gt;hwq, &amp;hwq_attr)) {</div><div class='ctx'> 		dev_err(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='ctx'> 			"HW channel CREQ allocation failed\n");</div><div class='ctx'> 		goto fail;</div><div class='ctx'> 	}</div><div class='del'>-	if (ctx-&gt;hwrm_intf_ver &lt; HWRM_VERSION_RCFW_CMDQ_DEPTH_CHECK)</div><div class='del'>-		rcfw-&gt;cmdq_depth = BNXT_QPLIB_CMDQE_MAX_CNT_256;</div><div class='del'>-	else</div><div class='del'>-		rcfw-&gt;cmdq_depth = BNXT_QPLIB_CMDQE_MAX_CNT_8192;</div><div class='del'>-</div><div class='del'>-	rcfw-&gt;cmdq.max_elements = rcfw-&gt;cmdq_depth;</div><div class='del'>-	if (bnxt_qplib_alloc_init_hwq</div><div class='del'>-			(rcfw-&gt;pdev, &amp;rcfw-&gt;cmdq, NULL,</div><div class='del'>-			 &amp;rcfw-&gt;cmdq.max_elements,</div><div class='del'>-			 BNXT_QPLIB_CMDQE_UNITS, 0,</div><div class='del'>-			 bnxt_qplib_cmdqe_page_size(rcfw-&gt;cmdq_depth),</div><div class='del'>-			 HWQ_TYPE_CTX)) {</div><div class='add'>+</div><div class='add'>+	rcfw-&gt;cmdq_depth = BNXT_QPLIB_CMDQE_MAX_CNT;</div><div class='add'>+</div><div class='add'>+	sginfo.pgsize = bnxt_qplib_cmdqe_page_size(rcfw-&gt;cmdq_depth);</div><div class='add'>+	hwq_attr.depth = rcfw-&gt;cmdq_depth &amp; 0x7FFFFFFF;</div><div class='add'>+	hwq_attr.stride = BNXT_QPLIB_CMDQE_UNITS;</div><div class='add'>+	hwq_attr.type = HWQ_TYPE_CTX;</div><div class='add'>+	if (bnxt_qplib_alloc_init_hwq(&amp;cmdq-&gt;hwq, &amp;hwq_attr)) {</div><div class='ctx'> 		dev_err(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='ctx'> 			"HW channel CMDQ allocation failed\n");</div><div class='ctx'> 		goto fail;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	rcfw-&gt;crsqe_tbl = kcalloc(rcfw-&gt;cmdq.max_elements,</div><div class='add'>+	rcfw-&gt;crsqe_tbl = kcalloc(cmdq-&gt;hwq.max_elements,</div><div class='ctx'> 				  sizeof(*rcfw-&gt;crsqe_tbl), GFP_KERNEL);</div><div class='ctx'> 	if (!rcfw-&gt;crsqe_tbl)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='del'>-	rcfw-&gt;qp_tbl_size = qp_tbl_sz;</div><div class='del'>-	rcfw-&gt;qp_tbl = kcalloc(qp_tbl_sz, sizeof(struct bnxt_qplib_qp_node),</div><div class='del'>-			       GFP_KERNEL);</div><div class='del'>-	if (!rcfw-&gt;qp_tbl)</div><div class='del'>-		goto fail;</div><div class='add'>+	spin_lock_init(&amp;rcfw-&gt;tbl_lock);</div><div class='add'>+</div><div class='add'>+	rcfw-&gt;max_timeout = res-&gt;cctx-&gt;hwrm_cmd_max_timeout;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='hunk'>@@ -614,137 +986,206 @@ fail:</div><div class='ctx'> </div><div class='ctx'> void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill)</div><div class='ctx'> {</div><div class='del'>-	bool gen_p5 = bnxt_qplib_is_chip_gen_p5(rcfw-&gt;res-&gt;cctx);</div><div class='add'>+	struct bnxt_qplib_creq_ctx *creq;</div><div class='add'>+</div><div class='add'>+	creq = &amp;rcfw-&gt;creq;</div><div class='ctx'> </div><div class='del'>-	tasklet_disable(&amp;rcfw-&gt;worker);</div><div class='add'>+	if (!creq-&gt;requested)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	creq-&gt;requested = false;</div><div class='ctx'> 	/* Mask h/w interrupts */</div><div class='del'>-	bnxt_qplib_ring_creq_db(rcfw-&gt;creq_bar_reg_iomem, rcfw-&gt;creq.cons,</div><div class='del'>-				rcfw-&gt;creq.max_elements, rcfw-&gt;creq_ring_id,</div><div class='del'>-				gen_p5);</div><div class='add'>+	bnxt_qplib_ring_nq_db(&amp;creq-&gt;creq_db.dbinfo, rcfw-&gt;res-&gt;cctx, false);</div><div class='ctx'> 	/* Sync with last running IRQ-handler */</div><div class='del'>-	synchronize_irq(rcfw-&gt;vector);</div><div class='add'>+	synchronize_irq(creq-&gt;msix_vec);</div><div class='add'>+	free_irq(creq-&gt;msix_vec, rcfw);</div><div class='add'>+	kfree(creq-&gt;irq_name);</div><div class='add'>+	creq-&gt;irq_name = NULL;</div><div class='add'>+	atomic_set(&amp;rcfw-&gt;rcfw_intr_enabled, 0);</div><div class='ctx'> 	if (kill)</div><div class='del'>-		tasklet_kill(&amp;rcfw-&gt;worker);</div><div class='del'>-</div><div class='del'>-	if (rcfw-&gt;requested) {</div><div class='del'>-		free_irq(rcfw-&gt;vector, rcfw);</div><div class='del'>-		rcfw-&gt;requested = false;</div><div class='del'>-	}</div><div class='add'>+		tasklet_kill(&amp;creq-&gt;creq_tasklet);</div><div class='add'>+	tasklet_disable(&amp;creq-&gt;creq_tasklet);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)</div><div class='ctx'> {</div><div class='del'>-	unsigned long indx;</div><div class='add'>+	struct bnxt_qplib_creq_ctx *creq;</div><div class='add'>+	struct bnxt_qplib_cmdq_ctx *cmdq;</div><div class='ctx'> </div><div class='add'>+	creq = &amp;rcfw-&gt;creq;</div><div class='add'>+	cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+	/* Make sure the HW channel is stopped! */</div><div class='ctx'> 	bnxt_qplib_rcfw_stop_irq(rcfw, true);</div><div class='ctx'> </div><div class='del'>-	iounmap(rcfw-&gt;cmdq_bar_reg_iomem);</div><div class='del'>-	iounmap(rcfw-&gt;creq_bar_reg_iomem);</div><div class='add'>+	iounmap(cmdq-&gt;cmdq_mbox.reg.bar_reg);</div><div class='add'>+	iounmap(creq-&gt;creq_db.reg.bar_reg);</div><div class='ctx'> </div><div class='del'>-	indx = find_first_bit(rcfw-&gt;cmdq_bitmap, rcfw-&gt;bmap_size);</div><div class='del'>-	if (indx != rcfw-&gt;bmap_size)</div><div class='del'>-		dev_err(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='del'>-			"disabling RCFW with pending cmd-bit %lx\n", indx);</div><div class='del'>-	kfree(rcfw-&gt;cmdq_bitmap);</div><div class='del'>-	rcfw-&gt;bmap_size = 0;</div><div class='del'>-</div><div class='del'>-	rcfw-&gt;cmdq_bar_reg_iomem = NULL;</div><div class='del'>-	rcfw-&gt;creq_bar_reg_iomem = NULL;</div><div class='del'>-	rcfw-&gt;aeq_handler = NULL;</div><div class='del'>-	rcfw-&gt;vector = 0;</div><div class='add'>+	cmdq-&gt;cmdq_mbox.reg.bar_reg = NULL;</div><div class='add'>+	creq-&gt;creq_db.reg.bar_reg = NULL;</div><div class='add'>+	creq-&gt;aeq_handler = NULL;</div><div class='add'>+	creq-&gt;msix_vec = 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector,</div><div class='ctx'> 			      bool need_init)</div><div class='ctx'> {</div><div class='del'>-	bool gen_p5 = bnxt_qplib_is_chip_gen_p5(rcfw-&gt;res-&gt;cctx);</div><div class='add'>+	struct bnxt_qplib_creq_ctx *creq;</div><div class='add'>+	struct bnxt_qplib_res *res;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	if (rcfw-&gt;requested)</div><div class='add'>+	creq = &amp;rcfw-&gt;creq;</div><div class='add'>+	res = rcfw-&gt;res;</div><div class='add'>+</div><div class='add'>+	if (creq-&gt;requested)</div><div class='ctx'> 		return -EFAULT;</div><div class='ctx'> </div><div class='del'>-	rcfw-&gt;vector = msix_vector;</div><div class='add'>+	creq-&gt;msix_vec = msix_vector;</div><div class='ctx'> 	if (need_init)</div><div class='del'>-		tasklet_init(&amp;rcfw-&gt;worker,</div><div class='del'>-			     bnxt_qplib_service_creq, (unsigned long)rcfw);</div><div class='add'>+		tasklet_setup(&amp;creq-&gt;creq_tasklet, bnxt_qplib_service_creq);</div><div class='ctx'> 	else</div><div class='del'>-		tasklet_enable(&amp;rcfw-&gt;worker);</div><div class='del'>-	rc = request_irq(rcfw-&gt;vector, bnxt_qplib_creq_irq, 0,</div><div class='del'>-			 "bnxt_qplib_creq", rcfw);</div><div class='del'>-	if (rc)</div><div class='add'>+		tasklet_enable(&amp;creq-&gt;creq_tasklet);</div><div class='add'>+</div><div class='add'>+	creq-&gt;irq_name = kasprintf(GFP_KERNEL, "bnxt_re-creq@pci:%s",</div><div class='add'>+				   pci_name(res-&gt;pdev));</div><div class='add'>+	if (!creq-&gt;irq_name)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	rc = request_irq(creq-&gt;msix_vec, bnxt_qplib_creq_irq, 0,</div><div class='add'>+			 creq-&gt;irq_name, rcfw);</div><div class='add'>+	if (rc) {</div><div class='add'>+		kfree(creq-&gt;irq_name);</div><div class='add'>+		creq-&gt;irq_name = NULL;</div><div class='add'>+		tasklet_disable(&amp;creq-&gt;creq_tasklet);</div><div class='ctx'> 		return rc;</div><div class='del'>-	rcfw-&gt;requested = true;</div><div class='del'>-	bnxt_qplib_ring_creq_db_rearm(rcfw-&gt;creq_bar_reg_iomem,</div><div class='del'>-				      rcfw-&gt;creq.cons, rcfw-&gt;creq.max_elements,</div><div class='del'>-				      rcfw-&gt;creq_ring_id, gen_p5);</div><div class='add'>+	}</div><div class='add'>+	creq-&gt;requested = true;</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_ring_nq_db(&amp;creq-&gt;creq_db.dbinfo, res-&gt;cctx, true);</div><div class='add'>+	atomic_inc(&amp;rcfw-&gt;rcfw_intr_enabled);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev,</div><div class='del'>-				   struct bnxt_qplib_rcfw *rcfw,</div><div class='del'>-				   int msix_vector,</div><div class='del'>-				   int cp_bar_reg_off, int virt_fn,</div><div class='del'>-				   int (*aeq_handler)(struct bnxt_qplib_rcfw *,</div><div class='del'>-						      void *, void *))</div><div class='add'>+static int bnxt_qplib_map_cmdq_mbox(struct bnxt_qplib_rcfw *rcfw)</div><div class='ctx'> {</div><div class='del'>-	resource_size_t res_base;</div><div class='del'>-	struct cmdq_init init;</div><div class='del'>-	u16 bmap_size;</div><div class='del'>-	int rc;</div><div class='del'>-</div><div class='del'>-	/* General */</div><div class='del'>-	rcfw-&gt;seq_num = 0;</div><div class='del'>-	set_bit(FIRMWARE_FIRST_FLAG, &amp;rcfw-&gt;flags);</div><div class='del'>-	bmap_size = BITS_TO_LONGS(rcfw-&gt;cmdq_depth) * sizeof(unsigned long);</div><div class='del'>-	rcfw-&gt;cmdq_bitmap = kzalloc(bmap_size, GFP_KERNEL);</div><div class='del'>-	if (!rcfw-&gt;cmdq_bitmap)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	rcfw-&gt;bmap_size = bmap_size;</div><div class='del'>-</div><div class='del'>-	/* CMDQ */</div><div class='del'>-	rcfw-&gt;cmdq_bar_reg = RCFW_COMM_PCI_BAR_REGION;</div><div class='del'>-	res_base = pci_resource_start(pdev, rcfw-&gt;cmdq_bar_reg);</div><div class='del'>-	if (!res_base)</div><div class='add'>+	struct bnxt_qplib_cmdq_mbox *mbox;</div><div class='add'>+	resource_size_t bar_reg;</div><div class='add'>+	struct pci_dev *pdev;</div><div class='add'>+</div><div class='add'>+	pdev = rcfw-&gt;pdev;</div><div class='add'>+	mbox = &amp;rcfw-&gt;cmdq.cmdq_mbox;</div><div class='add'>+</div><div class='add'>+	mbox-&gt;reg.bar_id = RCFW_COMM_PCI_BAR_REGION;</div><div class='add'>+	mbox-&gt;reg.len = RCFW_COMM_SIZE;</div><div class='add'>+	mbox-&gt;reg.bar_base = pci_resource_start(pdev, mbox-&gt;reg.bar_id);</div><div class='add'>+	if (!mbox-&gt;reg.bar_base) {</div><div class='add'>+		dev_err(&amp;pdev-&gt;dev,</div><div class='add'>+			"QPLIB: CMDQ BAR region %d resc start is 0!\n",</div><div class='add'>+			mbox-&gt;reg.bar_id);</div><div class='ctx'> 		return -ENOMEM;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	rcfw-&gt;cmdq_bar_reg_iomem = ioremap_nocache(res_base +</div><div class='del'>-					      RCFW_COMM_BASE_OFFSET,</div><div class='del'>-					      RCFW_COMM_SIZE);</div><div class='del'>-	if (!rcfw-&gt;cmdq_bar_reg_iomem) {</div><div class='del'>-		dev_err(&amp;rcfw-&gt;pdev-&gt;dev, "CMDQ BAR region %d mapping failed\n",</div><div class='del'>-			rcfw-&gt;cmdq_bar_reg);</div><div class='add'>+	bar_reg = mbox-&gt;reg.bar_base + RCFW_COMM_BASE_OFFSET;</div><div class='add'>+	mbox-&gt;reg.len = RCFW_COMM_SIZE;</div><div class='add'>+	mbox-&gt;reg.bar_reg = ioremap(bar_reg, mbox-&gt;reg.len);</div><div class='add'>+	if (!mbox-&gt;reg.bar_reg) {</div><div class='add'>+		dev_err(&amp;pdev-&gt;dev,</div><div class='add'>+			"QPLIB: CMDQ BAR region %d mapping failed\n",</div><div class='add'>+			mbox-&gt;reg.bar_id);</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	rcfw-&gt;cmdq_bar_reg_prod_off = virt_fn ? RCFW_VF_COMM_PROD_OFFSET :</div><div class='del'>-					RCFW_PF_COMM_PROD_OFFSET;</div><div class='del'>-</div><div class='del'>-	rcfw-&gt;cmdq_bar_reg_trig_off = RCFW_COMM_TRIG_OFFSET;</div><div class='add'>+	mbox-&gt;prod = (void  __iomem *)(mbox-&gt;reg.bar_reg +</div><div class='add'>+			RCFW_PF_VF_COMM_PROD_OFFSET);</div><div class='add'>+	mbox-&gt;db = (void __iomem *)(mbox-&gt;reg.bar_reg + RCFW_COMM_TRIG_OFFSET);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	/* CREQ */</div><div class='del'>-	rcfw-&gt;creq_bar_reg = RCFW_COMM_CONS_PCI_BAR_REGION;</div><div class='del'>-	res_base = pci_resource_start(pdev, rcfw-&gt;creq_bar_reg);</div><div class='del'>-	if (!res_base)</div><div class='del'>-		dev_err(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='del'>-			"CREQ BAR region %d resc start is 0!\n",</div><div class='del'>-			rcfw-&gt;creq_bar_reg);</div><div class='add'>+static int bnxt_qplib_map_creq_db(struct bnxt_qplib_rcfw *rcfw, u32 reg_offt)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_creq_db *creq_db;</div><div class='add'>+	resource_size_t bar_reg;</div><div class='add'>+	struct pci_dev *pdev;</div><div class='add'>+</div><div class='add'>+	pdev = rcfw-&gt;pdev;</div><div class='add'>+	creq_db = &amp;rcfw-&gt;creq.creq_db;</div><div class='add'>+</div><div class='add'>+	creq_db-&gt;dbinfo.flags = 0;</div><div class='add'>+	creq_db-&gt;reg.bar_id = RCFW_COMM_CONS_PCI_BAR_REGION;</div><div class='add'>+	creq_db-&gt;reg.bar_base = pci_resource_start(pdev, creq_db-&gt;reg.bar_id);</div><div class='add'>+	if (!creq_db-&gt;reg.bar_id)</div><div class='add'>+		dev_err(&amp;pdev-&gt;dev,</div><div class='add'>+			"QPLIB: CREQ BAR region %d resc start is 0!",</div><div class='add'>+			creq_db-&gt;reg.bar_id);</div><div class='add'>+</div><div class='add'>+	bar_reg = creq_db-&gt;reg.bar_base + reg_offt;</div><div class='ctx'> 	/* Unconditionally map 8 bytes to support 57500 series */</div><div class='del'>-	rcfw-&gt;creq_bar_reg_iomem = ioremap_nocache(res_base + cp_bar_reg_off,</div><div class='del'>-						   8);</div><div class='del'>-	if (!rcfw-&gt;creq_bar_reg_iomem) {</div><div class='del'>-		dev_err(&amp;rcfw-&gt;pdev-&gt;dev, "CREQ BAR region %d mapping failed\n",</div><div class='del'>-			rcfw-&gt;creq_bar_reg);</div><div class='del'>-		iounmap(rcfw-&gt;cmdq_bar_reg_iomem);</div><div class='del'>-		rcfw-&gt;cmdq_bar_reg_iomem = NULL;</div><div class='add'>+	creq_db-&gt;reg.len = 8;</div><div class='add'>+	creq_db-&gt;reg.bar_reg = ioremap(bar_reg, creq_db-&gt;reg.len);</div><div class='add'>+	if (!creq_db-&gt;reg.bar_reg) {</div><div class='add'>+		dev_err(&amp;pdev-&gt;dev,</div><div class='add'>+			"QPLIB: CREQ BAR region %d mapping failed",</div><div class='add'>+			creq_db-&gt;reg.bar_id);</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='del'>-	rcfw-&gt;creq_qp_event_processed = 0;</div><div class='del'>-	rcfw-&gt;creq_func_event_processed = 0;</div><div class='add'>+	creq_db-&gt;dbinfo.db = creq_db-&gt;reg.bar_reg;</div><div class='add'>+	creq_db-&gt;dbinfo.hwq = &amp;rcfw-&gt;creq.hwq;</div><div class='add'>+	creq_db-&gt;dbinfo.xid = rcfw-&gt;creq.ring_id;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_qplib_start_rcfw(struct bnxt_qplib_rcfw *rcfw)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_cmdq_ctx *cmdq;</div><div class='add'>+	struct bnxt_qplib_creq_ctx *creq;</div><div class='add'>+	struct bnxt_qplib_cmdq_mbox *mbox;</div><div class='add'>+	struct cmdq_init init = {0};</div><div class='add'>+</div><div class='add'>+	cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+	creq = &amp;rcfw-&gt;creq;</div><div class='add'>+	mbox = &amp;cmdq-&gt;cmdq_mbox;</div><div class='add'>+</div><div class='add'>+	init.cmdq_pbl = cpu_to_le64(cmdq-&gt;hwq.pbl[PBL_LVL_0].pg_map_arr[0]);</div><div class='add'>+	init.cmdq_size_cmdq_lvl =</div><div class='add'>+			cpu_to_le16(((rcfw-&gt;cmdq_depth &lt;&lt;</div><div class='add'>+				      CMDQ_INIT_CMDQ_SIZE_SFT) &amp;</div><div class='add'>+				    CMDQ_INIT_CMDQ_SIZE_MASK) |</div><div class='add'>+				    ((cmdq-&gt;hwq.level &lt;&lt;</div><div class='add'>+				      CMDQ_INIT_CMDQ_LVL_SFT) &amp;</div><div class='add'>+				    CMDQ_INIT_CMDQ_LVL_MASK));</div><div class='add'>+	init.creq_ring_id = cpu_to_le16(creq-&gt;ring_id);</div><div class='add'>+	/* Write to the Bono mailbox register */</div><div class='add'>+	__iowrite32_copy(mbox-&gt;reg.bar_reg, &amp;init, sizeof(init) / 4);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw,</div><div class='add'>+				   int msix_vector,</div><div class='add'>+				   int cp_bar_reg_off,</div><div class='add'>+				   aeq_handler_t aeq_handler)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_cmdq_ctx *cmdq;</div><div class='add'>+	struct bnxt_qplib_creq_ctx *creq;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	cmdq = &amp;rcfw-&gt;cmdq;</div><div class='add'>+	creq = &amp;rcfw-&gt;creq;</div><div class='add'>+</div><div class='add'>+	/* Clear to defaults */</div><div class='add'>+</div><div class='add'>+	cmdq-&gt;seq_num = 0;</div><div class='add'>+	set_bit(FIRMWARE_FIRST_FLAG, &amp;cmdq-&gt;flags);</div><div class='add'>+	init_waitqueue_head(&amp;cmdq-&gt;waitq);</div><div class='ctx'> </div><div class='del'>-	if (aeq_handler)</div><div class='del'>-		rcfw-&gt;aeq_handler = aeq_handler;</div><div class='del'>-	init_waitqueue_head(&amp;rcfw-&gt;waitq);</div><div class='add'>+	creq-&gt;stats.creq_qp_event_processed = 0;</div><div class='add'>+	creq-&gt;stats.creq_func_event_processed = 0;</div><div class='add'>+	creq-&gt;aeq_handler = aeq_handler;</div><div class='add'>+</div><div class='add'>+	rc = bnxt_qplib_map_cmdq_mbox(rcfw);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	rc = bnxt_qplib_map_creq_db(rcfw, cp_bar_reg_off);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='ctx'> </div><div class='ctx'> 	rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_vector, true);</div><div class='ctx'> 	if (rc) {</div><div class='hunk'>@@ -754,46 +1195,8 @@ int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev,</div><div class='ctx'> 		return rc;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	init.cmdq_pbl = cpu_to_le64(rcfw-&gt;cmdq.pbl[PBL_LVL_0].pg_map_arr[0]);</div><div class='del'>-	init.cmdq_size_cmdq_lvl = cpu_to_le16(</div><div class='del'>-		((rcfw-&gt;cmdq_depth &lt;&lt; CMDQ_INIT_CMDQ_SIZE_SFT) &amp;</div><div class='del'>-		 CMDQ_INIT_CMDQ_SIZE_MASK) |</div><div class='del'>-		((rcfw-&gt;cmdq.level &lt;&lt; CMDQ_INIT_CMDQ_LVL_SFT) &amp;</div><div class='del'>-		 CMDQ_INIT_CMDQ_LVL_MASK));</div><div class='del'>-	init.creq_ring_id = cpu_to_le16(rcfw-&gt;creq_ring_id);</div><div class='add'>+	sema_init(&amp;rcfw-&gt;rcfw_inflight, RCFW_CMD_NON_BLOCKING_SHADOW_QD);</div><div class='add'>+	bnxt_qplib_start_rcfw(rcfw);</div><div class='ctx'> </div><div class='del'>-	/* Write to the Bono mailbox register */</div><div class='del'>-	__iowrite32_copy(rcfw-&gt;cmdq_bar_reg_iomem, &amp;init, sizeof(init) / 4);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='del'>-</div><div class='del'>-struct bnxt_qplib_rcfw_sbuf *bnxt_qplib_rcfw_alloc_sbuf(</div><div class='del'>-		struct bnxt_qplib_rcfw *rcfw,</div><div class='del'>-		u32 size)</div><div class='del'>-{</div><div class='del'>-	struct bnxt_qplib_rcfw_sbuf *sbuf;</div><div class='del'>-</div><div class='del'>-	sbuf = kzalloc(sizeof(*sbuf), GFP_ATOMIC);</div><div class='del'>-	if (!sbuf)</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	sbuf-&gt;size = size;</div><div class='del'>-	sbuf-&gt;sb = dma_alloc_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf-&gt;size,</div><div class='del'>-				      &amp;sbuf-&gt;dma_addr, GFP_ATOMIC);</div><div class='del'>-	if (!sbuf-&gt;sb)</div><div class='del'>-		goto bail;</div><div class='del'>-</div><div class='del'>-	return sbuf;</div><div class='del'>-bail:</div><div class='del'>-	kfree(sbuf);</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void bnxt_qplib_rcfw_free_sbuf(struct bnxt_qplib_rcfw *rcfw,</div><div class='del'>-			       struct bnxt_qplib_rcfw_sbuf *sbuf)</div><div class='del'>-{</div><div class='del'>-	if (sbuf-&gt;sb)</div><div class='del'>-		dma_free_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf-&gt;size,</div><div class='del'>-				  sbuf-&gt;sb, sbuf-&gt;dma_addr);</div><div class='del'>-	kfree(sbuf);</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h<br/>index 2138533bb642..988c89b4232e 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_rcfw.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/qplib_rcfw.h</a></div><div class='hunk'>@@ -39,39 +39,40 @@</div><div class='ctx'> #ifndef __BNXT_QPLIB_RCFW_H__</div><div class='ctx'> #define __BNXT_QPLIB_RCFW_H__</div><div class='ctx'> </div><div class='add'>+#include "qplib_tlv.h"</div><div class='add'>+</div><div class='ctx'> #define RCFW_CMDQ_TRIG_VAL		1</div><div class='ctx'> #define RCFW_COMM_PCI_BAR_REGION	0</div><div class='ctx'> #define RCFW_COMM_CONS_PCI_BAR_REGION	2</div><div class='ctx'> #define RCFW_COMM_BASE_OFFSET		0x600</div><div class='del'>-#define RCFW_PF_COMM_PROD_OFFSET	0xc</div><div class='del'>-#define RCFW_VF_COMM_PROD_OFFSET	0xc</div><div class='add'>+#define RCFW_PF_VF_COMM_PROD_OFFSET	0xc</div><div class='ctx'> #define RCFW_COMM_TRIG_OFFSET		0x100</div><div class='ctx'> #define RCFW_COMM_SIZE			0x104</div><div class='ctx'> </div><div class='ctx'> #define RCFW_DBR_PCI_BAR_REGION		2</div><div class='ctx'> #define RCFW_DBR_BASE_PAGE_SHIFT	12</div><div class='del'>-</div><div class='del'>-#define RCFW_CMD_PREP(req, CMD, cmd_flags)				\</div><div class='del'>-	do {								\</div><div class='del'>-		memset(&amp;(req), 0, sizeof((req)));			\</div><div class='del'>-		(req).opcode = CMDQ_BASE_OPCODE_##CMD;			\</div><div class='del'>-		(req).cmd_size = (sizeof((req)) +			\</div><div class='del'>-				BNXT_QPLIB_CMDQE_UNITS - 1) /		\</div><div class='del'>-				BNXT_QPLIB_CMDQE_UNITS;			\</div><div class='del'>-		(req).flags = cpu_to_le16(cmd_flags);			\</div><div class='del'>-	} while (0)</div><div class='del'>-</div><div class='del'>-#define RCFW_CMD_WAIT_TIME_MS		20000 /* 20 Seconds timeout */</div><div class='add'>+#define RCFW_FW_STALL_MAX_TIMEOUT	40</div><div class='ctx'> </div><div class='ctx'> /* Cmdq contains a fix number of a 16-Byte slots */</div><div class='ctx'> struct bnxt_qplib_cmdqe {</div><div class='ctx'> 	u8		data[16];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* CMDQ elements */</div><div class='del'>-#define BNXT_QPLIB_CMDQE_MAX_CNT_256	256</div><div class='del'>-#define BNXT_QPLIB_CMDQE_MAX_CNT_8192	8192</div><div class='ctx'> #define BNXT_QPLIB_CMDQE_UNITS		sizeof(struct bnxt_qplib_cmdqe)</div><div class='add'>+</div><div class='add'>+static inline void bnxt_qplib_rcfw_cmd_prep(struct cmdq_base *req,</div><div class='add'>+					    u8 opcode, u8 cmd_size)</div><div class='add'>+{</div><div class='add'>+	req-&gt;opcode = opcode;</div><div class='add'>+	req-&gt;cmd_size = cmd_size;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* Shadow queue depth for non blocking command */</div><div class='add'>+#define RCFW_CMD_NON_BLOCKING_SHADOW_QD	64</div><div class='add'>+#define RCFW_CMD_WAIT_TIME_MS		20000 /* 20 Seconds timeout */</div><div class='add'>+</div><div class='add'>+/* CMDQ elements */</div><div class='add'>+#define BNXT_QPLIB_CMDQE_MAX_CNT	8192</div><div class='ctx'> #define BNXT_QPLIB_CMDQE_BYTES(depth)	((depth) * BNXT_QPLIB_CMDQE_UNITS)</div><div class='ctx'> </div><div class='ctx'> static inline u32 bnxt_qplib_cmdqe_npages(u32 depth)</div><div class='hunk'>@@ -89,36 +90,50 @@ static inline u32 bnxt_qplib_cmdqe_page_size(u32 depth)</div><div class='ctx'> 	return (bnxt_qplib_cmdqe_npages(depth) * PAGE_SIZE);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline u32 bnxt_qplib_cmdqe_cnt_per_pg(u32 depth)</div><div class='add'>+/* Get the number of command units required for the req. The</div><div class='add'>+ * function returns correct value only if called before</div><div class='add'>+ * setting using bnxt_qplib_set_cmd_slots</div><div class='add'>+ */</div><div class='add'>+static inline u32 bnxt_qplib_get_cmd_slots(struct cmdq_base *req)</div><div class='ctx'> {</div><div class='del'>-	return (bnxt_qplib_cmdqe_page_size(depth) /</div><div class='del'>-		 BNXT_QPLIB_CMDQE_UNITS);</div><div class='del'>-}</div><div class='add'>+	u32 cmd_units = 0;</div><div class='ctx'> </div><div class='del'>-#define MAX_CMDQ_IDX(depth)		((depth) - 1)</div><div class='add'>+	if (HAS_TLV_HEADER(req)) {</div><div class='add'>+		struct roce_tlv *tlv_req = (struct roce_tlv *)req;</div><div class='ctx'> </div><div class='del'>-static inline u32 bnxt_qplib_max_cmdq_idx_per_pg(u32 depth)</div><div class='del'>-{</div><div class='del'>-	return (bnxt_qplib_cmdqe_cnt_per_pg(depth) - 1);</div><div class='add'>+		cmd_units = tlv_req-&gt;total_size;</div><div class='add'>+	} else {</div><div class='add'>+		cmd_units = (req-&gt;cmd_size + BNXT_QPLIB_CMDQE_UNITS - 1) /</div><div class='add'>+			    BNXT_QPLIB_CMDQE_UNITS;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return cmd_units;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-#define RCFW_MAX_COOKIE_VALUE		0x7FFF</div><div class='del'>-#define RCFW_CMD_IS_BLOCKING		0x8000</div><div class='del'>-#define RCFW_BLOCKED_CMD_WAIT_COUNT	0x4E20</div><div class='add'>+static inline u32 bnxt_qplib_set_cmd_slots(struct cmdq_base *req)</div><div class='add'>+{</div><div class='add'>+	u32 cmd_byte = 0;</div><div class='ctx'> </div><div class='del'>-#define HWRM_VERSION_RCFW_CMDQ_DEPTH_CHECK 0x1000900020011ULL</div><div class='add'>+	if (HAS_TLV_HEADER(req)) {</div><div class='add'>+		struct roce_tlv *tlv_req = (struct roce_tlv *)req;</div><div class='ctx'> </div><div class='del'>-static inline u32 get_cmdq_pg(u32 val, u32 depth)</div><div class='del'>-{</div><div class='del'>-	return (val &amp; ~(bnxt_qplib_max_cmdq_idx_per_pg(depth))) /</div><div class='del'>-		(bnxt_qplib_cmdqe_cnt_per_pg(depth));</div><div class='del'>-}</div><div class='add'>+		cmd_byte = tlv_req-&gt;total_size * BNXT_QPLIB_CMDQE_UNITS;</div><div class='add'>+	} else {</div><div class='add'>+		cmd_byte = req-&gt;cmd_size;</div><div class='add'>+		req-&gt;cmd_size = (req-&gt;cmd_size + BNXT_QPLIB_CMDQE_UNITS - 1) /</div><div class='add'>+				 BNXT_QPLIB_CMDQE_UNITS;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-static inline u32 get_cmdq_idx(u32 val, u32 depth)</div><div class='del'>-{</div><div class='del'>-	return val &amp; (bnxt_qplib_max_cmdq_idx_per_pg(depth));</div><div class='add'>+	return cmd_byte;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+#define RCFW_MAX_COOKIE_VALUE		(BNXT_QPLIB_CMDQE_MAX_CNT - 1)</div><div class='add'>+#define RCFW_CMD_IS_BLOCKING		0x8000</div><div class='add'>+</div><div class='add'>+#define HWRM_VERSION_DEV_ATTR_MAX_DPI  0x1000A0000000DULL</div><div class='add'>+/* HWRM version 1.10.3.18 */</div><div class='add'>+#define HWRM_VERSION_READ_CTX           0x1000A00030012</div><div class='add'>+</div><div class='ctx'> /* Crsq buf is 1024-Byte */</div><div class='ctx'> struct bnxt_qplib_crsbe {</div><div class='ctx'> 	u8			data[1024];</div><div class='hunk'>@@ -128,83 +143,23 @@ struct bnxt_qplib_crsbe {</div><div class='ctx'> /* Allocate 1 per QP for async error notification for now */</div><div class='ctx'> #define BNXT_QPLIB_CREQE_MAX_CNT	(64 * 1024)</div><div class='ctx'> #define BNXT_QPLIB_CREQE_UNITS		16	/* 16-Bytes per prod unit */</div><div class='del'>-#define BNXT_QPLIB_CREQE_CNT_PER_PG	(PAGE_SIZE / BNXT_QPLIB_CREQE_UNITS)</div><div class='del'>-</div><div class='del'>-#define MAX_CREQ_IDX			(BNXT_QPLIB_CREQE_MAX_CNT - 1)</div><div class='del'>-#define MAX_CREQ_IDX_PER_PG		(BNXT_QPLIB_CREQE_CNT_PER_PG - 1)</div><div class='del'>-</div><div class='del'>-static inline u32 get_creq_pg(u32 val)</div><div class='del'>-{</div><div class='del'>-	return (val &amp; ~MAX_CREQ_IDX_PER_PG) / BNXT_QPLIB_CREQE_CNT_PER_PG;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u32 get_creq_idx(u32 val)</div><div class='del'>-{</div><div class='del'>-	return val &amp; MAX_CREQ_IDX_PER_PG;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-#define BNXT_QPLIB_CREQE_PER_PG	(PAGE_SIZE / sizeof(struct creq_base))</div><div class='del'>-</div><div class='del'>-#define CREQ_CMP_VALID(hdr, raw_cons, cp_bit)			\</div><div class='add'>+#define CREQ_CMP_VALID(hdr, pass)			\</div><div class='ctx'> 	(!!((hdr)-&gt;v &amp; CREQ_BASE_V) ==				\</div><div class='del'>-	   !((raw_cons) &amp; (cp_bit)))</div><div class='del'>-</div><div class='del'>-#define CREQ_DB_KEY_CP			(0x2 &lt;&lt; CMPL_DOORBELL_KEY_SFT)</div><div class='del'>-#define CREQ_DB_IDX_VALID		CMPL_DOORBELL_IDX_VALID</div><div class='del'>-#define CREQ_DB_IRQ_DIS			CMPL_DOORBELL_MASK</div><div class='del'>-#define CREQ_DB_CP_FLAGS_REARM		(CREQ_DB_KEY_CP |	\</div><div class='del'>-					 CREQ_DB_IDX_VALID)</div><div class='del'>-#define CREQ_DB_CP_FLAGS		(CREQ_DB_KEY_CP |	\</div><div class='del'>-					 CREQ_DB_IDX_VALID |	\</div><div class='del'>-					 CREQ_DB_IRQ_DIS)</div><div class='del'>-</div><div class='del'>-static inline void bnxt_qplib_ring_creq_db64(void __iomem *db, u32 index,</div><div class='del'>-					     u32 xid, bool arm)</div><div class='del'>-{</div><div class='del'>-	u64 val = 0;</div><div class='del'>-</div><div class='del'>-	val = xid &amp; DBC_DBC_XID_MASK;</div><div class='del'>-	val |= DBC_DBC_PATH_ROCE;</div><div class='del'>-	val |= arm ? DBC_DBC_TYPE_NQ_ARM : DBC_DBC_TYPE_NQ;</div><div class='del'>-	val &lt;&lt;= 32;</div><div class='del'>-	val |= index &amp; DBC_DBC_INDEX_MASK;</div><div class='del'>-</div><div class='del'>-	writeq(val, db);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void bnxt_qplib_ring_creq_db_rearm(void __iomem *db, u32 raw_cons,</div><div class='del'>-						 u32 max_elements, u32 xid,</div><div class='del'>-						 bool gen_p5)</div><div class='del'>-{</div><div class='del'>-	u32 index = raw_cons &amp; (max_elements - 1);</div><div class='del'>-</div><div class='del'>-	if (gen_p5)</div><div class='del'>-		bnxt_qplib_ring_creq_db64(db, index, xid, true);</div><div class='del'>-	else</div><div class='del'>-		writel(CREQ_DB_CP_FLAGS_REARM | (index &amp; DBC_DBC32_XID_MASK),</div><div class='del'>-		       db);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void bnxt_qplib_ring_creq_db(void __iomem *db, u32 raw_cons,</div><div class='del'>-					   u32 max_elements, u32 xid,</div><div class='del'>-					   bool gen_p5)</div><div class='del'>-{</div><div class='del'>-	u32 index = raw_cons &amp; (max_elements - 1);</div><div class='del'>-</div><div class='del'>-	if (gen_p5)</div><div class='del'>-		bnxt_qplib_ring_creq_db64(db, index, xid, true);</div><div class='del'>-	else</div><div class='del'>-		writel(CREQ_DB_CP_FLAGS | (index &amp; DBC_DBC32_XID_MASK),</div><div class='del'>-		       db);</div><div class='del'>-}</div><div class='del'>-</div><div class='add'>+	   !((pass) &amp; BNXT_QPLIB_FLAG_EPOCH_CONS_MASK))</div><div class='ctx'> #define CREQ_ENTRY_POLL_BUDGET		0x100</div><div class='ctx'> </div><div class='ctx'> /* HWQ */</div><div class='add'>+typedef int (*aeq_handler_t)(struct bnxt_qplib_rcfw *, void *, void *);</div><div class='ctx'> </div><div class='del'>-struct bnxt_qplib_crsq {</div><div class='add'>+struct bnxt_qplib_crsqe {</div><div class='ctx'> 	struct creq_qp_event	*resp;</div><div class='ctx'> 	u32			req_size;</div><div class='add'>+	/* Free slots at the time of submission */</div><div class='add'>+	u32			free_slots;</div><div class='add'>+	u8			opcode;</div><div class='add'>+	bool			is_waiter_alive;</div><div class='add'>+	bool			is_internal_cmd;</div><div class='add'>+	bool			is_in_used;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct bnxt_qplib_rcfw_sbuf {</div><div class='hunk'>@@ -220,63 +175,103 @@ struct bnxt_qplib_qp_node {</div><div class='ctx'> </div><div class='ctx'> #define BNXT_QPLIB_OOS_COUNT_MASK 0xFFFFFFFF</div><div class='ctx'> </div><div class='add'>+#define FIRMWARE_INITIALIZED_FLAG	(0)</div><div class='add'>+#define FIRMWARE_FIRST_FLAG		(31)</div><div class='add'>+#define FIRMWARE_STALL_DETECTED		(3)</div><div class='add'>+#define ERR_DEVICE_DETACHED             (4)</div><div class='add'>+</div><div class='add'>+struct bnxt_qplib_cmdq_mbox {</div><div class='add'>+	struct bnxt_qplib_reg_desc	reg;</div><div class='add'>+	void __iomem			*prod;</div><div class='add'>+	void __iomem			*db;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_qplib_cmdq_ctx {</div><div class='add'>+	struct bnxt_qplib_hwq		hwq;</div><div class='add'>+	struct bnxt_qplib_cmdq_mbox	cmdq_mbox;</div><div class='add'>+	wait_queue_head_t		waitq;</div><div class='add'>+	unsigned long			flags;</div><div class='add'>+	unsigned long			last_seen;</div><div class='add'>+	u32				seq_num;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_qplib_creq_db {</div><div class='add'>+	struct bnxt_qplib_reg_desc	reg;</div><div class='add'>+	struct bnxt_qplib_db_info	dbinfo;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_qplib_creq_stat {</div><div class='add'>+	u64	creq_qp_event_processed;</div><div class='add'>+	u64	creq_func_event_processed;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_qplib_creq_ctx {</div><div class='add'>+	struct bnxt_qplib_hwq		hwq;</div><div class='add'>+	struct bnxt_qplib_creq_db	creq_db;</div><div class='add'>+	struct bnxt_qplib_creq_stat	stats;</div><div class='add'>+	struct tasklet_struct		creq_tasklet;</div><div class='add'>+	aeq_handler_t			aeq_handler;</div><div class='add'>+	u16				ring_id;</div><div class='add'>+	int				msix_vec;</div><div class='add'>+	bool				requested; /*irq handler installed */</div><div class='add'>+	char				*irq_name;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> /* RCFW Communication Channels */</div><div class='ctx'> struct bnxt_qplib_rcfw {</div><div class='ctx'> 	struct pci_dev		*pdev;</div><div class='ctx'> 	struct bnxt_qplib_res	*res;</div><div class='del'>-	int			vector;</div><div class='del'>-	struct tasklet_struct	worker;</div><div class='del'>-	bool			requested;</div><div class='del'>-	unsigned long		*cmdq_bitmap;</div><div class='del'>-	u32			bmap_size;</div><div class='del'>-	unsigned long		flags;</div><div class='del'>-#define FIRMWARE_INITIALIZED_FLAG	0</div><div class='del'>-#define FIRMWARE_FIRST_FLAG		31</div><div class='del'>-#define FIRMWARE_TIMED_OUT		3</div><div class='del'>-	wait_queue_head_t	waitq;</div><div class='del'>-	int			(*aeq_handler)(struct bnxt_qplib_rcfw *,</div><div class='del'>-					       void *, void *);</div><div class='del'>-	u32			seq_num;</div><div class='del'>-</div><div class='del'>-	/* Bar region info */</div><div class='del'>-	void __iomem		*cmdq_bar_reg_iomem;</div><div class='del'>-	u16			cmdq_bar_reg;</div><div class='del'>-	u16			cmdq_bar_reg_prod_off;</div><div class='del'>-	u16			cmdq_bar_reg_trig_off;</div><div class='del'>-	u16			creq_ring_id;</div><div class='del'>-	u16			creq_bar_reg;</div><div class='del'>-	void __iomem		*creq_bar_reg_iomem;</div><div class='del'>-</div><div class='del'>-	/* Cmd-Resp and Async Event notification queue */</div><div class='del'>-	struct bnxt_qplib_hwq	creq;</div><div class='del'>-	u64			creq_qp_event_processed;</div><div class='del'>-	u64			creq_func_event_processed;</div><div class='del'>-</div><div class='del'>-	/* Actual Cmd and Resp Queues */</div><div class='del'>-	struct bnxt_qplib_hwq	cmdq;</div><div class='del'>-	struct bnxt_qplib_crsq	*crsqe_tbl;</div><div class='add'>+	struct bnxt_qplib_cmdq_ctx	cmdq;</div><div class='add'>+	struct bnxt_qplib_creq_ctx	creq;</div><div class='add'>+	struct bnxt_qplib_crsqe		*crsqe_tbl;</div><div class='ctx'> 	int qp_tbl_size;</div><div class='ctx'> 	struct bnxt_qplib_qp_node *qp_tbl;</div><div class='add'>+	/* To synchronize the qp-handle hash table */</div><div class='add'>+	spinlock_t			tbl_lock;</div><div class='ctx'> 	u64 oos_prev;</div><div class='ctx'> 	u32 init_oos_stats;</div><div class='ctx'> 	u32 cmdq_depth;</div><div class='add'>+	atomic_t rcfw_intr_enabled;</div><div class='add'>+	struct semaphore rcfw_inflight;</div><div class='add'>+	atomic_t timeout_send;</div><div class='add'>+	/* cached from chip cctx for quick reference in slow path */</div><div class='add'>+	u16 max_timeout;</div><div class='add'>+	bool roce_mirror;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct bnxt_qplib_cmdqmsg {</div><div class='add'>+	struct cmdq_base	*req;</div><div class='add'>+	struct creq_base	*resp;</div><div class='add'>+	void			*sb;</div><div class='add'>+	u32			req_sz;</div><div class='add'>+	u32			res_sz;</div><div class='add'>+	u8			block;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline void bnxt_qplib_fill_cmdqmsg(struct bnxt_qplib_cmdqmsg *msg,</div><div class='add'>+					   void *req, void *resp, void *sb,</div><div class='add'>+					   u32 req_sz, u32 res_sz, u8 block)</div><div class='add'>+{</div><div class='add'>+	msg-&gt;req = req;</div><div class='add'>+	msg-&gt;resp = resp;</div><div class='add'>+	msg-&gt;sb = sb;</div><div class='add'>+	msg-&gt;req_sz = req_sz;</div><div class='add'>+	msg-&gt;res_sz = res_sz;</div><div class='add'>+	msg-&gt;block = block;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw);</div><div class='del'>-int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev,</div><div class='add'>+int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res,</div><div class='ctx'> 				  struct bnxt_qplib_rcfw *rcfw,</div><div class='del'>-				  struct bnxt_qplib_ctx *ctx,</div><div class='del'>-				  int qp_tbl_sz);</div><div class='add'>+				  struct bnxt_qplib_ctx *ctx);</div><div class='ctx'> void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill);</div><div class='ctx'> void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw);</div><div class='ctx'> int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector,</div><div class='ctx'> 			      bool need_init);</div><div class='del'>-int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev,</div><div class='del'>-				   struct bnxt_qplib_rcfw *rcfw,</div><div class='add'>+int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw,</div><div class='ctx'> 				   int msix_vector,</div><div class='del'>-				   int cp_bar_reg_off, int virt_fn,</div><div class='del'>-				   int (*aeq_handler)(struct bnxt_qplib_rcfw *,</div><div class='del'>-						      void *aeqe, void *obj));</div><div class='add'>+				   int cp_bar_reg_off,</div><div class='add'>+				   aeq_handler_t aeq_handler);</div><div class='ctx'> </div><div class='ctx'> struct bnxt_qplib_rcfw_sbuf *bnxt_qplib_rcfw_alloc_sbuf(</div><div class='ctx'> 				struct bnxt_qplib_rcfw *rcfw,</div><div class='hunk'>@@ -284,11 +279,16 @@ struct bnxt_qplib_rcfw_sbuf *bnxt_qplib_rcfw_alloc_sbuf(</div><div class='ctx'> void bnxt_qplib_rcfw_free_sbuf(struct bnxt_qplib_rcfw *rcfw,</div><div class='ctx'> 			       struct bnxt_qplib_rcfw_sbuf *sbuf);</div><div class='ctx'> int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,</div><div class='del'>-				 struct cmdq_base *req, struct creq_base *resp,</div><div class='del'>-				 void *sbuf, u8 is_block);</div><div class='add'>+				 struct bnxt_qplib_cmdqmsg *msg);</div><div class='ctx'> </div><div class='ctx'> int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw);</div><div class='ctx'> int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,</div><div class='ctx'> 			 struct bnxt_qplib_ctx *ctx, int is_virtfn);</div><div class='ctx'> void bnxt_qplib_mark_qp_error(void *qp_handle);</div><div class='add'>+</div><div class='add'>+static inline u32 map_qp_id_to_tbl_indx(u32 qid, struct bnxt_qplib_rcfw *rcfw)</div><div class='add'>+{</div><div class='add'>+	/* Last index of the qp_tbl is for QP1 ie. qp_tbl_size - 1*/</div><div class='add'>+	return (qid == 1) ? rcfw-&gt;qp_tbl_size - 1 : (qid % (rcfw-&gt;qp_tbl_size - 2));</div><div class='add'>+}</div><div class='ctx'> #endif /* __BNXT_QPLIB_RCFW_H__ */</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c<br/>index 37928b1111df..875d7b52c06a 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_res.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_res.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_res.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/qplib_res.c</a></div><div class='hunk'>@@ -44,20 +44,20 @@</div><div class='ctx'> #include &lt;linux/inetdevice.h&gt;</div><div class='ctx'> #include &lt;linux/dma-mapping.h&gt;</div><div class='ctx'> #include &lt;linux/if_vlan.h&gt;</div><div class='add'>+#include &lt;linux/vmalloc.h&gt;</div><div class='add'>+#include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+#include &lt;rdma/ib_umem.h&gt;</div><div class='add'>+</div><div class='ctx'> #include "roce_hsi.h"</div><div class='ctx'> #include "qplib_res.h"</div><div class='ctx'> #include "qplib_sp.h"</div><div class='ctx'> #include "qplib_rcfw.h"</div><div class='ctx'> </div><div class='del'>-static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,</div><div class='del'>-				      struct bnxt_qplib_stats *stats);</div><div class='del'>-static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,</div><div class='del'>-				      struct bnxt_qplib_stats *stats);</div><div class='del'>-</div><div class='ctx'> /* PBL */</div><div class='del'>-static void __free_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl,</div><div class='add'>+static void __free_pbl(struct bnxt_qplib_res *res, struct bnxt_qplib_pbl *pbl,</div><div class='ctx'> 		       bool is_umem)</div><div class='ctx'> {</div><div class='add'>+	struct pci_dev *pdev = res-&gt;pdev;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='ctx'> 	if (!is_umem) {</div><div class='hunk'>@@ -74,37 +74,60 @@ static void __free_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl,</div><div class='ctx'> 			pbl-&gt;pg_arr[i] = NULL;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='del'>-	kfree(pbl-&gt;pg_arr);</div><div class='add'>+	vfree(pbl-&gt;pg_arr);</div><div class='ctx'> 	pbl-&gt;pg_arr = NULL;</div><div class='del'>-	kfree(pbl-&gt;pg_map_arr);</div><div class='add'>+	vfree(pbl-&gt;pg_map_arr);</div><div class='ctx'> 	pbl-&gt;pg_map_arr = NULL;</div><div class='ctx'> 	pbl-&gt;pg_count = 0;</div><div class='ctx'> 	pbl-&gt;pg_size = 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int __alloc_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl,</div><div class='del'>-		       struct scatterlist *sghead, u32 pages,</div><div class='del'>-		       u32 nmaps, u32 pg_size)</div><div class='add'>+static void bnxt_qplib_fill_user_dma_pages(struct bnxt_qplib_pbl *pbl,</div><div class='add'>+					   struct bnxt_qplib_sg_info *sginfo)</div><div class='add'>+{</div><div class='add'>+	struct ib_block_iter biter;</div><div class='add'>+	int i = 0;</div><div class='add'>+</div><div class='add'>+	rdma_umem_for_each_dma_block(sginfo-&gt;umem, &amp;biter, sginfo-&gt;pgsize) {</div><div class='add'>+		pbl-&gt;pg_map_arr[i] = rdma_block_iter_dma_address(&amp;biter);</div><div class='add'>+		pbl-&gt;pg_arr[i] = NULL;</div><div class='add'>+		pbl-&gt;pg_count++;</div><div class='add'>+		i++;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int __alloc_pbl(struct bnxt_qplib_res *res,</div><div class='add'>+		       struct bnxt_qplib_pbl *pbl,</div><div class='add'>+		       struct bnxt_qplib_sg_info *sginfo)</div><div class='ctx'> {</div><div class='del'>-	struct sg_dma_page_iter sg_iter;</div><div class='add'>+	struct pci_dev *pdev = res-&gt;pdev;</div><div class='ctx'> 	bool is_umem = false;</div><div class='add'>+	u32 pages;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='add'>+	if (sginfo-&gt;nopte)</div><div class='add'>+		return 0;</div><div class='add'>+	if (sginfo-&gt;umem)</div><div class='add'>+		pages = ib_umem_num_dma_blocks(sginfo-&gt;umem, sginfo-&gt;pgsize);</div><div class='add'>+	else</div><div class='add'>+		pages = sginfo-&gt;npages;</div><div class='ctx'> 	/* page ptr arrays */</div><div class='del'>-	pbl-&gt;pg_arr = kcalloc(pages, sizeof(void *), GFP_KERNEL);</div><div class='add'>+	pbl-&gt;pg_arr = vmalloc_array(pages, sizeof(void *));</div><div class='ctx'> 	if (!pbl-&gt;pg_arr)</div><div class='ctx'> 		return -ENOMEM;</div><div class='add'>+	memset(pbl-&gt;pg_arr, 0, pages * sizeof(void *));</div><div class='ctx'> </div><div class='del'>-	pbl-&gt;pg_map_arr = kcalloc(pages, sizeof(dma_addr_t), GFP_KERNEL);</div><div class='add'>+	pbl-&gt;pg_map_arr = vmalloc_array(pages, sizeof(dma_addr_t));</div><div class='ctx'> 	if (!pbl-&gt;pg_map_arr) {</div><div class='del'>-		kfree(pbl-&gt;pg_arr);</div><div class='add'>+		vfree(pbl-&gt;pg_arr);</div><div class='ctx'> 		pbl-&gt;pg_arr = NULL;</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='add'>+	memset(pbl-&gt;pg_map_arr, 0, pages * sizeof(dma_addr_t));</div><div class='ctx'> 	pbl-&gt;pg_count = 0;</div><div class='del'>-	pbl-&gt;pg_size = pg_size;</div><div class='add'>+	pbl-&gt;pg_size = sginfo-&gt;pgsize;</div><div class='ctx'> </div><div class='del'>-	if (!sghead) {</div><div class='add'>+	if (!sginfo-&gt;umem) {</div><div class='ctx'> 		for (i = 0; i &lt; pages; i++) {</div><div class='ctx'> 			pbl-&gt;pg_arr[i] = dma_alloc_coherent(&amp;pdev-&gt;dev,</div><div class='ctx'> 							    pbl-&gt;pg_size,</div><div class='hunk'>@@ -115,25 +138,19 @@ static int __alloc_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl,</div><div class='ctx'> 			pbl-&gt;pg_count++;</div><div class='ctx'> 		}</div><div class='ctx'> 	} else {</div><div class='del'>-		i = 0;</div><div class='ctx'> 		is_umem = true;</div><div class='del'>-		for_each_sg_dma_page(sghead, &amp;sg_iter, nmaps, 0) {</div><div class='del'>-			pbl-&gt;pg_map_arr[i] = sg_page_iter_dma_address(&amp;sg_iter);</div><div class='del'>-			pbl-&gt;pg_arr[i] = NULL;</div><div class='del'>-			pbl-&gt;pg_count++;</div><div class='del'>-			i++;</div><div class='del'>-		}</div><div class='add'>+		bnxt_qplib_fill_user_dma_pages(pbl, sginfo);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='del'>-</div><div class='ctx'> fail:</div><div class='del'>-	__free_pbl(pdev, pbl, is_umem);</div><div class='add'>+	__free_pbl(res, pbl, is_umem);</div><div class='ctx'> 	return -ENOMEM;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* HWQ */</div><div class='del'>-void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq)</div><div class='add'>+void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res,</div><div class='add'>+			 struct bnxt_qplib_hwq *hwq)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='hunk'>@@ -144,9 +161,9 @@ void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq)</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; hwq-&gt;level + 1; i++) {</div><div class='ctx'> 		if (i == hwq-&gt;level)</div><div class='del'>-			__free_pbl(pdev, &amp;hwq-&gt;pbl[i], hwq-&gt;is_user);</div><div class='add'>+			__free_pbl(res, &amp;hwq-&gt;pbl[i], hwq-&gt;is_user);</div><div class='ctx'> 		else</div><div class='del'>-			__free_pbl(pdev, &amp;hwq-&gt;pbl[i], false);</div><div class='add'>+			__free_pbl(res, &amp;hwq-&gt;pbl[i], false);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	hwq-&gt;level = PBL_LVL_MAX;</div><div class='hunk'>@@ -158,79 +175,96 @@ void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* All HWQs are power of 2 in size */</div><div class='del'>-int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,</div><div class='del'>-			      struct bnxt_qplib_sg_info *sg_info,</div><div class='del'>-			      u32 *elements, u32 element_size, u32 aux,</div><div class='del'>-			      u32 pg_size, enum bnxt_qplib_hwq_type hwq_type)</div><div class='add'>+</div><div class='add'>+int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq,</div><div class='add'>+			      struct bnxt_qplib_hwq_attr *hwq_attr)</div><div class='ctx'> {</div><div class='del'>-	u32 pages, maps, slots, size, aux_pages = 0, aux_size = 0;</div><div class='add'>+	u32 npages, aux_slots, pg_size, aux_pages = 0, aux_size = 0;</div><div class='add'>+	struct bnxt_qplib_sg_info sginfo = {};</div><div class='add'>+	u32 depth, stride, npbl, npde;</div><div class='ctx'> 	dma_addr_t *src_phys_ptr, **dst_virt_ptr;</div><div class='del'>-	struct scatterlist *sghead = NULL;</div><div class='del'>-	int i, rc;</div><div class='add'>+	struct bnxt_qplib_res *res;</div><div class='add'>+	struct pci_dev *pdev;</div><div class='add'>+	int i, rc, lvl;</div><div class='ctx'> </div><div class='add'>+	res = hwq_attr-&gt;res;</div><div class='add'>+	pdev = res-&gt;pdev;</div><div class='add'>+	pg_size = hwq_attr-&gt;sginfo-&gt;pgsize;</div><div class='ctx'> 	hwq-&gt;level = PBL_LVL_MAX;</div><div class='ctx'> </div><div class='del'>-	slots = roundup_pow_of_two(*elements);</div><div class='del'>-	if (aux) {</div><div class='del'>-		aux_size = roundup_pow_of_two(aux);</div><div class='del'>-		aux_pages = (slots * aux_size) / pg_size;</div><div class='del'>-		if ((slots * aux_size) % pg_size)</div><div class='add'>+	depth = roundup_pow_of_two(hwq_attr-&gt;depth);</div><div class='add'>+	stride = roundup_pow_of_two(hwq_attr-&gt;stride);</div><div class='add'>+	if (hwq_attr-&gt;aux_depth) {</div><div class='add'>+		aux_slots = hwq_attr-&gt;aux_depth;</div><div class='add'>+		aux_size = roundup_pow_of_two(hwq_attr-&gt;aux_stride);</div><div class='add'>+		aux_pages = (aux_slots * aux_size) / pg_size;</div><div class='add'>+		if ((aux_slots * aux_size) % pg_size)</div><div class='ctx'> 			aux_pages++;</div><div class='ctx'> 	}</div><div class='del'>-	size = roundup_pow_of_two(element_size);</div><div class='ctx'> </div><div class='del'>-	if (sg_info)</div><div class='del'>-		sghead = sg_info-&gt;sglist;</div><div class='del'>-</div><div class='del'>-	if (!sghead) {</div><div class='add'>+	if (!hwq_attr-&gt;sginfo-&gt;umem) {</div><div class='ctx'> 		hwq-&gt;is_user = false;</div><div class='del'>-		pages = (slots * size) / pg_size + aux_pages;</div><div class='del'>-		if ((slots * size) % pg_size)</div><div class='del'>-			pages++;</div><div class='del'>-		if (!pages)</div><div class='add'>+		npages = (depth * stride) / pg_size + aux_pages;</div><div class='add'>+		if ((depth * stride) % pg_size)</div><div class='add'>+			npages++;</div><div class='add'>+		if (!npages)</div><div class='ctx'> 			return -EINVAL;</div><div class='del'>-		maps = 0;</div><div class='add'>+		hwq_attr-&gt;sginfo-&gt;npages = npages;</div><div class='ctx'> 	} else {</div><div class='add'>+		npages = ib_umem_num_dma_blocks(hwq_attr-&gt;sginfo-&gt;umem,</div><div class='add'>+						hwq_attr-&gt;sginfo-&gt;pgsize);</div><div class='ctx'> 		hwq-&gt;is_user = true;</div><div class='del'>-		pages = sg_info-&gt;npages;</div><div class='del'>-		maps = sg_info-&gt;nmap;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* Alloc the 1st memory block; can be a PDL/PTL/PBL */</div><div class='del'>-	if (sghead &amp;&amp; (pages == MAX_PBL_LVL_0_PGS))</div><div class='del'>-		rc = __alloc_pbl(pdev, &amp;hwq-&gt;pbl[PBL_LVL_0], sghead,</div><div class='del'>-				 pages, maps, pg_size);</div><div class='del'>-	else</div><div class='del'>-		rc = __alloc_pbl(pdev, &amp;hwq-&gt;pbl[PBL_LVL_0], NULL,</div><div class='del'>-				 1, 0, pg_size);</div><div class='del'>-	if (rc)</div><div class='del'>-		goto fail;</div><div class='del'>-</div><div class='del'>-	hwq-&gt;level = PBL_LVL_0;</div><div class='add'>+	if (npages == MAX_PBL_LVL_0_PGS &amp;&amp; !hwq_attr-&gt;sginfo-&gt;nopte) {</div><div class='add'>+		/* This request is Level 0, map PTE */</div><div class='add'>+		rc = __alloc_pbl(res, &amp;hwq-&gt;pbl[PBL_LVL_0], hwq_attr-&gt;sginfo);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto fail;</div><div class='add'>+		hwq-&gt;level = PBL_LVL_0;</div><div class='add'>+		goto done;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (pages &gt; MAX_PBL_LVL_0_PGS) {</div><div class='del'>-		if (pages &gt; MAX_PBL_LVL_1_PGS) {</div><div class='add'>+	if (npages &gt;= MAX_PBL_LVL_0_PGS) {</div><div class='add'>+		if (npages &gt; MAX_PBL_LVL_1_PGS) {</div><div class='add'>+			u32 flag = (hwq_attr-&gt;type == HWQ_TYPE_L2_CMPL) ?</div><div class='add'>+				    0 : PTU_PTE_VALID;</div><div class='ctx'> 			/* 2 levels of indirection */</div><div class='del'>-			rc = __alloc_pbl(pdev, &amp;hwq-&gt;pbl[PBL_LVL_1], NULL,</div><div class='del'>-					 MAX_PBL_LVL_1_PGS_FOR_LVL_2,</div><div class='del'>-					 0, pg_size);</div><div class='add'>+			npbl = npages &gt;&gt; MAX_PBL_LVL_1_PGS_SHIFT;</div><div class='add'>+			if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))</div><div class='add'>+				npbl++;</div><div class='add'>+			npde = npbl &gt;&gt; MAX_PDL_LVL_SHIFT;</div><div class='add'>+			if (npbl % BIT(MAX_PDL_LVL_SHIFT))</div><div class='add'>+				npde++;</div><div class='add'>+			/* Alloc PDE pages */</div><div class='add'>+			sginfo.pgsize = npde * pg_size;</div><div class='add'>+			sginfo.npages = 1;</div><div class='add'>+			rc = __alloc_pbl(res, &amp;hwq-&gt;pbl[PBL_LVL_0], &amp;sginfo);</div><div class='add'>+			if (rc)</div><div class='add'>+				goto fail;</div><div class='add'>+</div><div class='add'>+			/* Alloc PBL pages */</div><div class='add'>+			sginfo.npages = npbl;</div><div class='add'>+			sginfo.pgsize = PAGE_SIZE;</div><div class='add'>+			rc = __alloc_pbl(res, &amp;hwq-&gt;pbl[PBL_LVL_1], &amp;sginfo);</div><div class='ctx'> 			if (rc)</div><div class='ctx'> 				goto fail;</div><div class='del'>-			/* Fill in lvl0 PBL */</div><div class='add'>+			/* Fill PDL with PBL page pointers */</div><div class='ctx'> 			dst_virt_ptr =</div><div class='ctx'> 				(dma_addr_t **)hwq-&gt;pbl[PBL_LVL_0].pg_arr;</div><div class='ctx'> 			src_phys_ptr = hwq-&gt;pbl[PBL_LVL_1].pg_map_arr;</div><div class='ctx'> 			for (i = 0; i &lt; hwq-&gt;pbl[PBL_LVL_1].pg_count; i++)</div><div class='del'>-				dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =</div><div class='del'>-					src_phys_ptr[i] | PTU_PDE_VALID;</div><div class='del'>-			hwq-&gt;level = PBL_LVL_1;</div><div class='add'>+				dst_virt_ptr[0][i] = src_phys_ptr[i] | flag;</div><div class='ctx'> </div><div class='del'>-			rc = __alloc_pbl(pdev, &amp;hwq-&gt;pbl[PBL_LVL_2], sghead,</div><div class='del'>-					 pages, maps, pg_size);</div><div class='add'>+			/* Alloc or init PTEs */</div><div class='add'>+			rc = __alloc_pbl(res, &amp;hwq-&gt;pbl[PBL_LVL_2],</div><div class='add'>+					 hwq_attr-&gt;sginfo);</div><div class='ctx'> 			if (rc)</div><div class='ctx'> 				goto fail;</div><div class='del'>-</div><div class='del'>-			/* Fill in lvl1 PBL */</div><div class='add'>+			hwq-&gt;level = PBL_LVL_2;</div><div class='add'>+			if (hwq_attr-&gt;sginfo-&gt;nopte)</div><div class='add'>+				goto done;</div><div class='add'>+			/* Fill PBLs with PTE pointers */</div><div class='ctx'> 			dst_virt_ptr =</div><div class='ctx'> 				(dma_addr_t **)hwq-&gt;pbl[PBL_LVL_1].pg_arr;</div><div class='ctx'> 			src_phys_ptr = hwq-&gt;pbl[PBL_LVL_2].pg_map_arr;</div><div class='hunk'>@@ -238,7 +272,7 @@ int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,</div><div class='ctx'> 				dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =</div><div class='ctx'> 					src_phys_ptr[i] | PTU_PTE_VALID;</div><div class='ctx'> 			}</div><div class='del'>-			if (hwq_type == HWQ_TYPE_QUEUE) {</div><div class='add'>+			if (hwq_attr-&gt;type == HWQ_TYPE_QUEUE) {</div><div class='ctx'> 				/* Find the last pg of the size */</div><div class='ctx'> 				i = hwq-&gt;pbl[PBL_LVL_2].pg_count;</div><div class='ctx'> 				dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=</div><div class='hunk'>@@ -248,25 +282,36 @@ int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,</div><div class='ctx'> 						    [PTR_IDX(i - 2)] |=</div><div class='ctx'> 						    PTU_PTE_NEXT_TO_LAST;</div><div class='ctx'> 			}</div><div class='del'>-			hwq-&gt;level = PBL_LVL_2;</div><div class='del'>-		} else {</div><div class='del'>-			u32 flag = hwq_type == HWQ_TYPE_L2_CMPL ? 0 :</div><div class='del'>-						PTU_PTE_VALID;</div><div class='add'>+		} else { /* pages &lt; 512 npbl = 1, npde = 0 */</div><div class='add'>+			u32 flag = (hwq_attr-&gt;type == HWQ_TYPE_L2_CMPL) ?</div><div class='add'>+				    0 : PTU_PTE_VALID;</div><div class='ctx'> </div><div class='ctx'> 			/* 1 level of indirection */</div><div class='del'>-			rc = __alloc_pbl(pdev, &amp;hwq-&gt;pbl[PBL_LVL_1], sghead,</div><div class='del'>-					 pages, maps, pg_size);</div><div class='add'>+			npbl = npages &gt;&gt; MAX_PBL_LVL_1_PGS_SHIFT;</div><div class='add'>+			if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))</div><div class='add'>+				npbl++;</div><div class='add'>+			sginfo.npages = npbl;</div><div class='add'>+			sginfo.pgsize = PAGE_SIZE;</div><div class='add'>+			/* Alloc PBL page */</div><div class='add'>+			rc = __alloc_pbl(res, &amp;hwq-&gt;pbl[PBL_LVL_0], &amp;sginfo);</div><div class='add'>+			if (rc)</div><div class='add'>+				goto fail;</div><div class='add'>+			/* Alloc or init  PTEs */</div><div class='add'>+			rc = __alloc_pbl(res, &amp;hwq-&gt;pbl[PBL_LVL_1],</div><div class='add'>+					 hwq_attr-&gt;sginfo);</div><div class='ctx'> 			if (rc)</div><div class='ctx'> 				goto fail;</div><div class='del'>-			/* Fill in lvl0 PBL */</div><div class='add'>+			hwq-&gt;level = PBL_LVL_1;</div><div class='add'>+			if (hwq_attr-&gt;sginfo-&gt;nopte)</div><div class='add'>+				goto done;</div><div class='add'>+			/* Fill PBL with PTE pointers */</div><div class='ctx'> 			dst_virt_ptr =</div><div class='ctx'> 				(dma_addr_t **)hwq-&gt;pbl[PBL_LVL_0].pg_arr;</div><div class='ctx'> 			src_phys_ptr = hwq-&gt;pbl[PBL_LVL_1].pg_map_arr;</div><div class='del'>-			for (i = 0; i &lt; hwq-&gt;pbl[PBL_LVL_1].pg_count; i++) {</div><div class='add'>+			for (i = 0; i &lt; hwq-&gt;pbl[PBL_LVL_1].pg_count; i++)</div><div class='ctx'> 				dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =</div><div class='ctx'> 					src_phys_ptr[i] | flag;</div><div class='del'>-			}</div><div class='del'>-			if (hwq_type == HWQ_TYPE_QUEUE) {</div><div class='add'>+			if (hwq_attr-&gt;type == HWQ_TYPE_QUEUE) {</div><div class='ctx'> 				/* Find the last pg of the size */</div><div class='ctx'> 				i = hwq-&gt;pbl[PBL_LVL_1].pg_count;</div><div class='ctx'> 				dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=</div><div class='hunk'>@@ -276,46 +321,145 @@ int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,</div><div class='ctx'> 						    [PTR_IDX(i - 2)] |=</div><div class='ctx'> 						    PTU_PTE_NEXT_TO_LAST;</div><div class='ctx'> 			}</div><div class='del'>-			hwq-&gt;level = PBL_LVL_1;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='del'>-	hwq-&gt;pdev = pdev;</div><div class='del'>-	spin_lock_init(&amp;hwq-&gt;lock);</div><div class='add'>+done:</div><div class='ctx'> 	hwq-&gt;prod = 0;</div><div class='ctx'> 	hwq-&gt;cons = 0;</div><div class='del'>-	*elements = hwq-&gt;max_elements = slots;</div><div class='del'>-	hwq-&gt;element_size = size;</div><div class='del'>-</div><div class='add'>+	hwq-&gt;pdev = pdev;</div><div class='add'>+	hwq-&gt;depth = hwq_attr-&gt;depth;</div><div class='add'>+	hwq-&gt;max_elements = hwq-&gt;depth;</div><div class='add'>+	hwq-&gt;element_size = stride;</div><div class='add'>+	hwq-&gt;qe_ppg = pg_size / stride;</div><div class='ctx'> 	/* For direct access to the elements */</div><div class='del'>-	hwq-&gt;pbl_ptr = hwq-&gt;pbl[hwq-&gt;level].pg_arr;</div><div class='del'>-	hwq-&gt;pbl_dma_ptr = hwq-&gt;pbl[hwq-&gt;level].pg_map_arr;</div><div class='add'>+	lvl = hwq-&gt;level;</div><div class='add'>+	if (hwq_attr-&gt;sginfo-&gt;nopte &amp;&amp; hwq-&gt;level)</div><div class='add'>+		lvl = hwq-&gt;level - 1;</div><div class='add'>+	hwq-&gt;pbl_ptr = hwq-&gt;pbl[lvl].pg_arr;</div><div class='add'>+	hwq-&gt;pbl_dma_ptr = hwq-&gt;pbl[lvl].pg_map_arr;</div><div class='add'>+	spin_lock_init(&amp;hwq-&gt;lock);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='del'>-</div><div class='ctx'> fail:</div><div class='del'>-	bnxt_qplib_free_hwq(pdev, hwq);</div><div class='add'>+	bnxt_qplib_free_hwq(res, hwq);</div><div class='ctx'> 	return -ENOMEM;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* Context Tables */</div><div class='del'>-void bnxt_qplib_free_ctx(struct pci_dev *pdev,</div><div class='del'>-			 struct bnxt_qplib_ctx *ctx)</div><div class='add'>+void bnxt_qplib_free_hwctx(struct bnxt_qplib_res *res,</div><div class='add'>+			   struct bnxt_qplib_ctx *ctx)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	bnxt_qplib_free_hwq(pdev, &amp;ctx-&gt;qpc_tbl);</div><div class='del'>-	bnxt_qplib_free_hwq(pdev, &amp;ctx-&gt;mrw_tbl);</div><div class='del'>-	bnxt_qplib_free_hwq(pdev, &amp;ctx-&gt;srqc_tbl);</div><div class='del'>-	bnxt_qplib_free_hwq(pdev, &amp;ctx-&gt;cq_tbl);</div><div class='del'>-	bnxt_qplib_free_hwq(pdev, &amp;ctx-&gt;tim_tbl);</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;ctx-&gt;qpc_tbl);</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;ctx-&gt;mrw_tbl);</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;ctx-&gt;srqc_tbl);</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;ctx-&gt;cq_tbl);</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;ctx-&gt;tim_tbl);</div><div class='ctx'> 	for (i = 0; i &lt; MAX_TQM_ALLOC_REQ; i++)</div><div class='del'>-		bnxt_qplib_free_hwq(pdev, &amp;ctx-&gt;tqm_tbl[i]);</div><div class='del'>-	bnxt_qplib_free_hwq(pdev, &amp;ctx-&gt;tqm_pde);</div><div class='del'>-	bnxt_qplib_free_stats_ctx(pdev, &amp;ctx-&gt;stats);</div><div class='add'>+		bnxt_qplib_free_hwq(res, &amp;ctx-&gt;tqm_ctx.qtbl[i]);</div><div class='add'>+	/* restore original pde level before destroy */</div><div class='add'>+	ctx-&gt;tqm_ctx.pde.level = ctx-&gt;tqm_ctx.pde_level;</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;ctx-&gt;tqm_ctx.pde);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bnxt_qplib_alloc_tqm_rings(struct bnxt_qplib_res *res,</div><div class='add'>+				      struct bnxt_qplib_ctx *ctx)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_hwq_attr hwq_attr = {};</div><div class='add'>+	struct bnxt_qplib_sg_info sginfo = {};</div><div class='add'>+	struct bnxt_qplib_tqm_ctx *tqmctx;</div><div class='add'>+	int rc;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	tqmctx = &amp;ctx-&gt;tqm_ctx;</div><div class='add'>+</div><div class='add'>+	sginfo.pgsize = PAGE_SIZE;</div><div class='add'>+	sginfo.pgshft = PAGE_SHIFT;</div><div class='add'>+	hwq_attr.sginfo = &amp;sginfo;</div><div class='add'>+	hwq_attr.res = res;</div><div class='add'>+	hwq_attr.type = HWQ_TYPE_CTX;</div><div class='add'>+	hwq_attr.depth = 512;</div><div class='add'>+	hwq_attr.stride = sizeof(u64);</div><div class='add'>+	/* Alloc pdl buffer */</div><div class='add'>+	rc = bnxt_qplib_alloc_init_hwq(&amp;tqmctx-&gt;pde, &amp;hwq_attr);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto out;</div><div class='add'>+	/* Save original pdl level */</div><div class='add'>+	tqmctx-&gt;pde_level = tqmctx-&gt;pde.level;</div><div class='add'>+</div><div class='add'>+	hwq_attr.stride = 1;</div><div class='add'>+	for (i = 0; i &lt; MAX_TQM_ALLOC_REQ; i++) {</div><div class='add'>+		if (!tqmctx-&gt;qcount[i])</div><div class='add'>+			continue;</div><div class='add'>+		hwq_attr.depth = ctx-&gt;qpc_count * tqmctx-&gt;qcount[i];</div><div class='add'>+		rc = bnxt_qplib_alloc_init_hwq(&amp;tqmctx-&gt;qtbl[i], &amp;hwq_attr);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto out;</div><div class='add'>+	}</div><div class='add'>+out:</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_qplib_map_tqm_pgtbl(struct bnxt_qplib_tqm_ctx *ctx)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_hwq *tbl;</div><div class='add'>+	dma_addr_t *dma_ptr;</div><div class='add'>+	__le64 **pbl_ptr, *ptr;</div><div class='add'>+	int i, j, k;</div><div class='add'>+	int fnz_idx = -1;</div><div class='add'>+	int pg_count;</div><div class='add'>+</div><div class='add'>+	pbl_ptr = (__le64 **)ctx-&gt;pde.pbl_ptr;</div><div class='add'>+</div><div class='add'>+	for (i = 0, j = 0; i &lt; MAX_TQM_ALLOC_REQ;</div><div class='add'>+	     i++, j += MAX_TQM_ALLOC_BLK_SIZE) {</div><div class='add'>+		tbl = &amp;ctx-&gt;qtbl[i];</div><div class='add'>+		if (!tbl-&gt;max_elements)</div><div class='add'>+			continue;</div><div class='add'>+		if (fnz_idx == -1)</div><div class='add'>+			fnz_idx = i; /* first non-zero index */</div><div class='add'>+		switch (tbl-&gt;level) {</div><div class='add'>+		case PBL_LVL_2:</div><div class='add'>+			pg_count = tbl-&gt;pbl[PBL_LVL_1].pg_count;</div><div class='add'>+			for (k = 0; k &lt; pg_count; k++) {</div><div class='add'>+				ptr = &amp;pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)];</div><div class='add'>+				dma_ptr = &amp;tbl-&gt;pbl[PBL_LVL_1].pg_map_arr[k];</div><div class='add'>+				*ptr = cpu_to_le64(*dma_ptr | PTU_PTE_VALID);</div><div class='add'>+			}</div><div class='add'>+			break;</div><div class='add'>+		case PBL_LVL_1:</div><div class='add'>+		case PBL_LVL_0:</div><div class='add'>+		default:</div><div class='add'>+			ptr = &amp;pbl_ptr[PTR_PG(j)][PTR_IDX(j)];</div><div class='add'>+			*ptr = cpu_to_le64(tbl-&gt;pbl[PBL_LVL_0].pg_map_arr[0] |</div><div class='add'>+					   PTU_PTE_VALID);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	if (fnz_idx == -1)</div><div class='add'>+		fnz_idx = 0;</div><div class='add'>+	/* update pde level as per page table programming */</div><div class='add'>+	ctx-&gt;pde.level = (ctx-&gt;qtbl[fnz_idx].level == PBL_LVL_2) ? PBL_LVL_2 :</div><div class='add'>+			  ctx-&gt;qtbl[fnz_idx].level + 1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int bnxt_qplib_setup_tqm_rings(struct bnxt_qplib_res *res,</div><div class='add'>+				      struct bnxt_qplib_ctx *ctx)</div><div class='add'>+{</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rc = bnxt_qplib_alloc_tqm_rings(res, ctx);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto fail;</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_map_tqm_pgtbl(&amp;ctx-&gt;tqm_ctx);</div><div class='add'>+fail:</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='del'>- * Routine: bnxt_qplib_alloc_ctx</div><div class='add'>+ * Routine: bnxt_qplib_alloc_hwctx</div><div class='ctx'>  * Description:</div><div class='ctx'>  *     Context tables are memories which are used by the chip fw.</div><div class='ctx'>  *     The 6 tables defined are:</div><div class='hunk'>@@ -335,140 +479,66 @@ void bnxt_qplib_free_ctx(struct pci_dev *pdev,</div><div class='ctx'>  * Returns:</div><div class='ctx'>  *     0 if success, else -ERRORS</div><div class='ctx'>  */</div><div class='del'>-int bnxt_qplib_alloc_ctx(struct pci_dev *pdev,</div><div class='del'>-			 struct bnxt_qplib_ctx *ctx,</div><div class='del'>-			 bool virt_fn, bool is_p5)</div><div class='add'>+int bnxt_qplib_alloc_hwctx(struct bnxt_qplib_res *res,</div><div class='add'>+			   struct bnxt_qplib_ctx *ctx)</div><div class='ctx'> {</div><div class='del'>-	int i, j, k, rc = 0;</div><div class='del'>-	int fnz_idx = -1;</div><div class='del'>-	__le64 **pbl_ptr;</div><div class='del'>-</div><div class='del'>-	if (virt_fn || is_p5)</div><div class='del'>-		goto stats_alloc;</div><div class='add'>+	struct bnxt_qplib_hwq_attr hwq_attr = {};</div><div class='add'>+	struct bnxt_qplib_sg_info sginfo = {};</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='ctx'> 	/* QPC Tables */</div><div class='del'>-	ctx-&gt;qpc_tbl.max_elements = ctx-&gt;qpc_count;</div><div class='del'>-	rc = bnxt_qplib_alloc_init_hwq(pdev, &amp;ctx-&gt;qpc_tbl, NULL,</div><div class='del'>-				       &amp;ctx-&gt;qpc_tbl.max_elements,</div><div class='del'>-				       BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE, 0,</div><div class='del'>-				       PAGE_SIZE, HWQ_TYPE_CTX);</div><div class='add'>+	sginfo.pgsize = PAGE_SIZE;</div><div class='add'>+	sginfo.pgshft = PAGE_SHIFT;</div><div class='add'>+	hwq_attr.sginfo = &amp;sginfo;</div><div class='add'>+</div><div class='add'>+	hwq_attr.res = res;</div><div class='add'>+	hwq_attr.depth = ctx-&gt;qpc_count;</div><div class='add'>+	hwq_attr.stride = BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE;</div><div class='add'>+	hwq_attr.type = HWQ_TYPE_CTX;</div><div class='add'>+	rc = bnxt_qplib_alloc_init_hwq(&amp;ctx-&gt;qpc_tbl, &amp;hwq_attr);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='ctx'> 	/* MRW Tables */</div><div class='del'>-	ctx-&gt;mrw_tbl.max_elements = ctx-&gt;mrw_count;</div><div class='del'>-	rc = bnxt_qplib_alloc_init_hwq(pdev, &amp;ctx-&gt;mrw_tbl, NULL,</div><div class='del'>-				       &amp;ctx-&gt;mrw_tbl.max_elements,</div><div class='del'>-				       BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE, 0,</div><div class='del'>-				       PAGE_SIZE, HWQ_TYPE_CTX);</div><div class='add'>+	hwq_attr.depth = ctx-&gt;mrw_count;</div><div class='add'>+	hwq_attr.stride = BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE;</div><div class='add'>+	rc = bnxt_qplib_alloc_init_hwq(&amp;ctx-&gt;mrw_tbl, &amp;hwq_attr);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='ctx'> 	/* SRQ Tables */</div><div class='del'>-	ctx-&gt;srqc_tbl.max_elements = ctx-&gt;srqc_count;</div><div class='del'>-	rc = bnxt_qplib_alloc_init_hwq(pdev, &amp;ctx-&gt;srqc_tbl, NULL,</div><div class='del'>-				       &amp;ctx-&gt;srqc_tbl.max_elements,</div><div class='del'>-				       BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE, 0,</div><div class='del'>-				       PAGE_SIZE, HWQ_TYPE_CTX);</div><div class='add'>+	hwq_attr.depth = ctx-&gt;srqc_count;</div><div class='add'>+	hwq_attr.stride = BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE;</div><div class='add'>+	rc = bnxt_qplib_alloc_init_hwq(&amp;ctx-&gt;srqc_tbl, &amp;hwq_attr);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='ctx'> 	/* CQ Tables */</div><div class='del'>-	ctx-&gt;cq_tbl.max_elements = ctx-&gt;cq_count;</div><div class='del'>-	rc = bnxt_qplib_alloc_init_hwq(pdev, &amp;ctx-&gt;cq_tbl, NULL,</div><div class='del'>-				       &amp;ctx-&gt;cq_tbl.max_elements,</div><div class='del'>-				       BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE, 0,</div><div class='del'>-				       PAGE_SIZE, HWQ_TYPE_CTX);</div><div class='add'>+	hwq_attr.depth = ctx-&gt;cq_count;</div><div class='add'>+	hwq_attr.stride = BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE;</div><div class='add'>+	rc = bnxt_qplib_alloc_init_hwq(&amp;ctx-&gt;cq_tbl, &amp;hwq_attr);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='ctx'> 	/* TQM Buffer */</div><div class='del'>-	ctx-&gt;tqm_pde.max_elements = 512;</div><div class='del'>-	rc = bnxt_qplib_alloc_init_hwq(pdev, &amp;ctx-&gt;tqm_pde, NULL,</div><div class='del'>-				       &amp;ctx-&gt;tqm_pde.max_elements, sizeof(u64),</div><div class='del'>-				       0, PAGE_SIZE, HWQ_TYPE_CTX);</div><div class='add'>+	rc = bnxt_qplib_setup_tqm_rings(res, ctx);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; MAX_TQM_ALLOC_REQ; i++) {</div><div class='del'>-		if (!ctx-&gt;tqm_count[i])</div><div class='del'>-			continue;</div><div class='del'>-		ctx-&gt;tqm_tbl[i].max_elements = ctx-&gt;qpc_count *</div><div class='del'>-					       ctx-&gt;tqm_count[i];</div><div class='del'>-		rc = bnxt_qplib_alloc_init_hwq(pdev, &amp;ctx-&gt;tqm_tbl[i], NULL,</div><div class='del'>-					       &amp;ctx-&gt;tqm_tbl[i].max_elements, 1,</div><div class='del'>-					       0, PAGE_SIZE, HWQ_TYPE_CTX);</div><div class='del'>-		if (rc)</div><div class='del'>-			goto fail;</div><div class='del'>-	}</div><div class='del'>-	pbl_ptr = (__le64 **)ctx-&gt;tqm_pde.pbl_ptr;</div><div class='del'>-	for (i = 0, j = 0; i &lt; MAX_TQM_ALLOC_REQ;</div><div class='del'>-	     i++, j += MAX_TQM_ALLOC_BLK_SIZE) {</div><div class='del'>-		if (!ctx-&gt;tqm_tbl[i].max_elements)</div><div class='del'>-			continue;</div><div class='del'>-		if (fnz_idx == -1)</div><div class='del'>-			fnz_idx = i;</div><div class='del'>-		switch (ctx-&gt;tqm_tbl[i].level) {</div><div class='del'>-		case PBL_LVL_2:</div><div class='del'>-			for (k = 0; k &lt; ctx-&gt;tqm_tbl[i].pbl[PBL_LVL_1].pg_count;</div><div class='del'>-			     k++)</div><div class='del'>-				pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)] =</div><div class='del'>-				  cpu_to_le64(</div><div class='del'>-				    ctx-&gt;tqm_tbl[i].pbl[PBL_LVL_1].pg_map_arr[k]</div><div class='del'>-				    | PTU_PTE_VALID);</div><div class='del'>-			break;</div><div class='del'>-		case PBL_LVL_1:</div><div class='del'>-		case PBL_LVL_0:</div><div class='del'>-		default:</div><div class='del'>-			pbl_ptr[PTR_PG(j)][PTR_IDX(j)] = cpu_to_le64(</div><div class='del'>-				ctx-&gt;tqm_tbl[i].pbl[PBL_LVL_0].pg_map_arr[0] |</div><div class='del'>-				PTU_PTE_VALID);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	if (fnz_idx == -1)</div><div class='del'>-		fnz_idx = 0;</div><div class='del'>-	ctx-&gt;tqm_pde_level = ctx-&gt;tqm_tbl[fnz_idx].level == PBL_LVL_2 ?</div><div class='del'>-			     PBL_LVL_2 : ctx-&gt;tqm_tbl[fnz_idx].level + 1;</div><div class='del'>-</div><div class='ctx'> 	/* TIM Buffer */</div><div class='ctx'> 	ctx-&gt;tim_tbl.max_elements = ctx-&gt;qpc_count * 16;</div><div class='del'>-	rc = bnxt_qplib_alloc_init_hwq(pdev, &amp;ctx-&gt;tim_tbl, NULL,</div><div class='del'>-				       &amp;ctx-&gt;tim_tbl.max_elements, 1,</div><div class='del'>-				       0, PAGE_SIZE, HWQ_TYPE_CTX);</div><div class='del'>-	if (rc)</div><div class='del'>-		goto fail;</div><div class='del'>-</div><div class='del'>-stats_alloc:</div><div class='del'>-	/* Stats */</div><div class='del'>-	rc = bnxt_qplib_alloc_stats_ctx(pdev, &amp;ctx-&gt;stats);</div><div class='add'>+	hwq_attr.depth = ctx-&gt;qpc_count * 16;</div><div class='add'>+	hwq_attr.stride = 1;</div><div class='add'>+	rc = bnxt_qplib_alloc_init_hwq(&amp;ctx-&gt;tim_tbl, &amp;hwq_attr);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> fail:</div><div class='del'>-	bnxt_qplib_free_ctx(pdev, ctx);</div><div class='add'>+	bnxt_qplib_free_hwctx(res, ctx);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/* GUID */</div><div class='del'>-void bnxt_qplib_get_guid(u8 *dev_addr, u8 *guid)</div><div class='del'>-{</div><div class='del'>-	u8 mac[ETH_ALEN];</div><div class='del'>-</div><div class='del'>-	/* MAC-48 to EUI-64 mapping */</div><div class='del'>-	memcpy(mac, dev_addr, ETH_ALEN);</div><div class='del'>-	guid[0] = mac[0] ^ 2;</div><div class='del'>-	guid[1] = mac[1];</div><div class='del'>-	guid[2] = mac[2];</div><div class='del'>-	guid[3] = 0xff;</div><div class='del'>-	guid[4] = 0xfe;</div><div class='del'>-	guid[5] = mac[3];</div><div class='del'>-	guid[6] = mac[4];</div><div class='del'>-	guid[7] = mac[5];</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res,</div><div class='ctx'> 				     struct bnxt_qplib_sgid_tbl *sgid_tbl)</div><div class='ctx'> {</div><div class='hunk'>@@ -488,7 +558,7 @@ static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res,</div><div class='ctx'> 				     struct bnxt_qplib_sgid_tbl *sgid_tbl,</div><div class='ctx'> 				     u16 max)</div><div class='ctx'> {</div><div class='del'>-	sgid_tbl-&gt;tbl = kcalloc(max, sizeof(struct bnxt_qplib_gid), GFP_KERNEL);</div><div class='add'>+	sgid_tbl-&gt;tbl = kcalloc(max, sizeof(*sgid_tbl-&gt;tbl), GFP_KERNEL);</div><div class='ctx'> 	if (!sgid_tbl-&gt;tbl)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='hunk'>@@ -526,9 +596,10 @@ static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res,</div><div class='ctx'> 	for (i = 0; i &lt; sgid_tbl-&gt;max; i++) {</div><div class='ctx'> 		if (memcmp(&amp;sgid_tbl-&gt;tbl[i], &amp;bnxt_qplib_gid_zero,</div><div class='ctx'> 			   sizeof(bnxt_qplib_gid_zero)))</div><div class='del'>-			bnxt_qplib_del_sgid(sgid_tbl, &amp;sgid_tbl-&gt;tbl[i], true);</div><div class='add'>+			bnxt_qplib_del_sgid(sgid_tbl, &amp;sgid_tbl-&gt;tbl[i].gid,</div><div class='add'>+					    sgid_tbl-&gt;tbl[i].vlan_id, true);</div><div class='ctx'> 	}</div><div class='del'>-	memset(sgid_tbl-&gt;tbl, 0, sizeof(struct bnxt_qplib_gid) * sgid_tbl-&gt;max);</div><div class='add'>+	memset(sgid_tbl-&gt;tbl, 0, sizeof(*sgid_tbl-&gt;tbl) * sgid_tbl-&gt;max);</div><div class='ctx'> 	memset(sgid_tbl-&gt;hw_id, -1, sizeof(u16) * sgid_tbl-&gt;max);</div><div class='ctx'> 	memset(sgid_tbl-&gt;vlan, 0, sizeof(u8) * sgid_tbl-&gt;max);</div><div class='ctx'> 	sgid_tbl-&gt;active = 0;</div><div class='hunk'>@@ -537,61 +608,53 @@ static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res,</div><div class='ctx'> static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl,</div><div class='ctx'> 				     struct net_device *netdev)</div><div class='ctx'> {</div><div class='del'>-	memset(sgid_tbl-&gt;tbl, 0, sizeof(struct bnxt_qplib_gid) * sgid_tbl-&gt;max);</div><div class='del'>-	memset(sgid_tbl-&gt;hw_id, -1, sizeof(u16) * sgid_tbl-&gt;max);</div><div class='del'>-}</div><div class='add'>+	u32 i;</div><div class='ctx'> </div><div class='del'>-static void bnxt_qplib_free_pkey_tbl(struct bnxt_qplib_res *res,</div><div class='del'>-				     struct bnxt_qplib_pkey_tbl *pkey_tbl)</div><div class='del'>-{</div><div class='del'>-	if (!pkey_tbl-&gt;tbl)</div><div class='del'>-		dev_dbg(&amp;res-&gt;pdev-&gt;dev, "PKEY tbl not present\n");</div><div class='del'>-	else</div><div class='del'>-		kfree(pkey_tbl-&gt;tbl);</div><div class='add'>+	for (i = 0; i &lt; sgid_tbl-&gt;max; i++)</div><div class='add'>+		sgid_tbl-&gt;tbl[i].vlan_id = 0xffff;</div><div class='ctx'> </div><div class='del'>-	pkey_tbl-&gt;tbl = NULL;</div><div class='del'>-	pkey_tbl-&gt;max = 0;</div><div class='del'>-	pkey_tbl-&gt;active = 0;</div><div class='add'>+	memset(sgid_tbl-&gt;hw_id, -1, sizeof(u16) * sgid_tbl-&gt;max);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int bnxt_qplib_alloc_pkey_tbl(struct bnxt_qplib_res *res,</div><div class='del'>-				     struct bnxt_qplib_pkey_tbl *pkey_tbl,</div><div class='del'>-				     u16 max)</div><div class='del'>-{</div><div class='del'>-	pkey_tbl-&gt;tbl = kcalloc(max, sizeof(u16), GFP_KERNEL);</div><div class='del'>-	if (!pkey_tbl-&gt;tbl)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	pkey_tbl-&gt;max = max;</div><div class='del'>-	return 0;</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> /* PDs */</div><div class='del'>-int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pdt, struct bnxt_qplib_pd *pd)</div><div class='add'>+int bnxt_qplib_alloc_pd(struct bnxt_qplib_res  *res, struct bnxt_qplib_pd *pd)</div><div class='ctx'> {</div><div class='add'>+	struct bnxt_qplib_pd_tbl *pdt = &amp;res-&gt;pd_tbl;</div><div class='ctx'> 	u32 bit_num;</div><div class='add'>+	int rc = 0;</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;res-&gt;pd_tbl_lock);</div><div class='ctx'> 	bit_num = find_first_bit(pdt-&gt;tbl, pdt-&gt;max);</div><div class='del'>-	if (bit_num == pdt-&gt;max)</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+	if (bit_num == pdt-&gt;max) {</div><div class='add'>+		rc = -ENOMEM;</div><div class='add'>+		goto exit;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	/* Found unused PD */</div><div class='ctx'> 	clear_bit(bit_num, pdt-&gt;tbl);</div><div class='ctx'> 	pd-&gt;id = bit_num;</div><div class='del'>-	return 0;</div><div class='add'>+exit:</div><div class='add'>+	mutex_unlock(&amp;res-&gt;pd_tbl_lock);</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res,</div><div class='ctx'> 			  struct bnxt_qplib_pd_tbl *pdt,</div><div class='ctx'> 			  struct bnxt_qplib_pd *pd)</div><div class='ctx'> {</div><div class='add'>+	int rc = 0;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;res-&gt;pd_tbl_lock);</div><div class='ctx'> 	if (test_and_set_bit(pd-&gt;id, pdt-&gt;tbl)) {</div><div class='ctx'> 		dev_warn(&amp;res-&gt;pdev-&gt;dev, "Freeing an unused PD? pdn = %d\n",</div><div class='ctx'> 			 pd-&gt;id);</div><div class='del'>-		return -EINVAL;</div><div class='add'>+		rc = -EINVAL;</div><div class='add'>+		goto exit;</div><div class='ctx'> 	}</div><div class='ctx'> 	pd-&gt;id = 0;</div><div class='del'>-	return 0;</div><div class='add'>+exit:</div><div class='add'>+	mutex_unlock(&amp;res-&gt;pd_tbl_lock);</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt)</div><div class='hunk'>@@ -616,49 +679,82 @@ static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res,</div><div class='ctx'> </div><div class='ctx'> 	pdt-&gt;max = max;</div><div class='ctx'> 	memset((u8 *)pdt-&gt;tbl, 0xFF, bytes);</div><div class='add'>+	mutex_init(&amp;res-&gt;pd_tbl_lock);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* DPIs */</div><div class='del'>-int bnxt_qplib_alloc_dpi(struct bnxt_qplib_dpi_tbl *dpit,</div><div class='del'>-			 struct bnxt_qplib_dpi     *dpi,</div><div class='del'>-			 void                      *app)</div><div class='add'>+int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res,</div><div class='add'>+			 struct bnxt_qplib_dpi *dpi,</div><div class='add'>+			 void *app, u8 type)</div><div class='ctx'> {</div><div class='add'>+	struct bnxt_qplib_dpi_tbl *dpit = &amp;res-&gt;dpi_tbl;</div><div class='add'>+	struct bnxt_qplib_reg_desc *reg;</div><div class='ctx'> 	u32 bit_num;</div><div class='add'>+	u64 umaddr;</div><div class='add'>+</div><div class='add'>+	reg = &amp;dpit-&gt;wcreg;</div><div class='add'>+	mutex_lock(&amp;res-&gt;dpi_tbl_lock);</div><div class='ctx'> </div><div class='ctx'> 	bit_num = find_first_bit(dpit-&gt;tbl, dpit-&gt;max);</div><div class='del'>-	if (bit_num == dpit-&gt;max)</div><div class='add'>+	if (bit_num == dpit-&gt;max) {</div><div class='add'>+		mutex_unlock(&amp;res-&gt;dpi_tbl_lock);</div><div class='ctx'> 		return -ENOMEM;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	/* Found unused DPI */</div><div class='ctx'> 	clear_bit(bit_num, dpit-&gt;tbl);</div><div class='ctx'> 	dpit-&gt;app_tbl[bit_num] = app;</div><div class='ctx'> </div><div class='del'>-	dpi-&gt;dpi = bit_num;</div><div class='del'>-	dpi-&gt;dbr = dpit-&gt;dbr_bar_reg_iomem + (bit_num * PAGE_SIZE);</div><div class='del'>-	dpi-&gt;umdbr = dpit-&gt;unmapped_dbr + (bit_num * PAGE_SIZE);</div><div class='add'>+	dpi-&gt;bit = bit_num;</div><div class='add'>+	dpi-&gt;dpi = bit_num + (reg-&gt;offset - dpit-&gt;ucreg.offset) / PAGE_SIZE;</div><div class='add'>+</div><div class='add'>+	umaddr = reg-&gt;bar_base + reg-&gt;offset + bit_num * PAGE_SIZE;</div><div class='add'>+	dpi-&gt;umdbr = umaddr;</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case BNXT_QPLIB_DPI_TYPE_KERNEL:</div><div class='add'>+		/* privileged dbr was already mapped just initialize it. */</div><div class='add'>+		dpi-&gt;umdbr = dpit-&gt;ucreg.bar_base +</div><div class='add'>+			     dpit-&gt;ucreg.offset + bit_num * PAGE_SIZE;</div><div class='add'>+		dpi-&gt;dbr = dpit-&gt;priv_db;</div><div class='add'>+		dpi-&gt;dpi = dpi-&gt;bit;</div><div class='add'>+		break;</div><div class='add'>+	case BNXT_QPLIB_DPI_TYPE_WC:</div><div class='add'>+		dpi-&gt;dbr = ioremap_wc(umaddr, PAGE_SIZE);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		dpi-&gt;dbr = ioremap(umaddr, PAGE_SIZE);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='add'>+	dpi-&gt;type = type;</div><div class='add'>+	mutex_unlock(&amp;res-&gt;dpi_tbl_lock);</div><div class='ctx'> 	return 0;</div><div class='add'>+</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,</div><div class='del'>-			   struct bnxt_qplib_dpi_tbl *dpit,</div><div class='del'>-			   struct bnxt_qplib_dpi     *dpi)</div><div class='add'>+			   struct bnxt_qplib_dpi *dpi)</div><div class='ctx'> {</div><div class='del'>-	if (dpi-&gt;dpi &gt;= dpit-&gt;max) {</div><div class='del'>-		dev_warn(&amp;res-&gt;pdev-&gt;dev, "Invalid DPI? dpi = %d\n", dpi-&gt;dpi);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-	if (test_and_set_bit(dpi-&gt;dpi, dpit-&gt;tbl)) {</div><div class='del'>-		dev_warn(&amp;res-&gt;pdev-&gt;dev, "Freeing an unused DPI? dpi = %d\n",</div><div class='del'>-			 dpi-&gt;dpi);</div><div class='add'>+	struct bnxt_qplib_dpi_tbl *dpit = &amp;res-&gt;dpi_tbl;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;res-&gt;dpi_tbl_lock);</div><div class='add'>+	if (dpi-&gt;dpi &amp;&amp; dpi-&gt;type != BNXT_QPLIB_DPI_TYPE_KERNEL)</div><div class='add'>+		pci_iounmap(res-&gt;pdev, dpi-&gt;dbr);</div><div class='add'>+</div><div class='add'>+	if (test_and_set_bit(dpi-&gt;bit, dpit-&gt;tbl)) {</div><div class='add'>+		dev_warn(&amp;res-&gt;pdev-&gt;dev,</div><div class='add'>+			 "Freeing an unused DPI? dpi = %d, bit = %d\n",</div><div class='add'>+				dpi-&gt;dpi, dpi-&gt;bit);</div><div class='add'>+		mutex_unlock(&amp;res-&gt;dpi_tbl_lock);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> 	if (dpit-&gt;app_tbl)</div><div class='del'>-		dpit-&gt;app_tbl[dpi-&gt;dpi] = NULL;</div><div class='add'>+		dpit-&gt;app_tbl[dpi-&gt;bit] = NULL;</div><div class='ctx'> 	memset(dpi, 0, sizeof(*dpi));</div><div class='del'>-</div><div class='add'>+	mutex_unlock(&amp;res-&gt;dpi_tbl_lock);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -667,52 +763,38 @@ static void bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_res     *res,</div><div class='ctx'> {</div><div class='ctx'> 	kfree(dpit-&gt;tbl);</div><div class='ctx'> 	kfree(dpit-&gt;app_tbl);</div><div class='del'>-	if (dpit-&gt;dbr_bar_reg_iomem)</div><div class='del'>-		pci_iounmap(res-&gt;pdev, dpit-&gt;dbr_bar_reg_iomem);</div><div class='del'>-	memset(dpit, 0, sizeof(*dpit));</div><div class='add'>+	dpit-&gt;tbl = NULL;</div><div class='add'>+	dpit-&gt;app_tbl = NULL;</div><div class='add'>+	dpit-&gt;max = 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res     *res,</div><div class='del'>-				    struct bnxt_qplib_dpi_tbl *dpit,</div><div class='del'>-				    u32                       dbr_offset)</div><div class='add'>+static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res,</div><div class='add'>+				    struct bnxt_qplib_dev_attr *dev_attr)</div><div class='ctx'> {</div><div class='del'>-	u32 dbr_bar_reg = RCFW_DBR_PCI_BAR_REGION;</div><div class='del'>-	resource_size_t bar_reg_base;</div><div class='del'>-	u32 dbr_len, bytes;</div><div class='del'>-</div><div class='del'>-	if (dpit-&gt;dbr_bar_reg_iomem) {</div><div class='del'>-		dev_err(&amp;res-&gt;pdev-&gt;dev, "DBR BAR region %d already mapped\n",</div><div class='del'>-			dbr_bar_reg);</div><div class='del'>-		return -EALREADY;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	bar_reg_base = pci_resource_start(res-&gt;pdev, dbr_bar_reg);</div><div class='del'>-	if (!bar_reg_base) {</div><div class='del'>-		dev_err(&amp;res-&gt;pdev-&gt;dev, "BAR region %d resc start failed\n",</div><div class='del'>-			dbr_bar_reg);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='add'>+	struct bnxt_qplib_dpi_tbl *dpit;</div><div class='add'>+	struct bnxt_qplib_reg_desc *reg;</div><div class='add'>+	unsigned long bar_len;</div><div class='add'>+	u32 dbr_offset;</div><div class='add'>+	u32 bytes;</div><div class='ctx'> </div><div class='del'>-	dbr_len = pci_resource_len(res-&gt;pdev, dbr_bar_reg) - dbr_offset;</div><div class='del'>-	if (!dbr_len || ((dbr_len &amp; (PAGE_SIZE - 1)) != 0)) {</div><div class='del'>-		dev_err(&amp;res-&gt;pdev-&gt;dev, "Invalid DBR length %d\n", dbr_len);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='add'>+	dpit = &amp;res-&gt;dpi_tbl;</div><div class='add'>+	reg = &amp;dpit-&gt;wcreg;</div><div class='ctx'> </div><div class='del'>-	dpit-&gt;dbr_bar_reg_iomem = ioremap_nocache(bar_reg_base + dbr_offset,</div><div class='del'>-						  dbr_len);</div><div class='del'>-	if (!dpit-&gt;dbr_bar_reg_iomem) {</div><div class='del'>-		dev_err(&amp;res-&gt;pdev-&gt;dev,</div><div class='del'>-			"FP: DBR BAR region %d mapping failed\n", dbr_bar_reg);</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+	if (!bnxt_qplib_is_chip_gen_p5_p7(res-&gt;cctx)) {</div><div class='add'>+		/* Offest should come from L2 driver */</div><div class='add'>+		dbr_offset = dev_attr-&gt;l2_db_size;</div><div class='add'>+		dpit-&gt;ucreg.offset = dbr_offset;</div><div class='add'>+		dpit-&gt;wcreg.offset = dbr_offset;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	dpit-&gt;unmapped_dbr = bar_reg_base + dbr_offset;</div><div class='del'>-	dpit-&gt;max = dbr_len / PAGE_SIZE;</div><div class='add'>+	bar_len = pci_resource_len(res-&gt;pdev, reg-&gt;bar_id);</div><div class='add'>+	dpit-&gt;max = (bar_len - reg-&gt;offset) / PAGE_SIZE;</div><div class='add'>+	if (dev_attr-&gt;max_dpi)</div><div class='add'>+		dpit-&gt;max = min_t(u32, dpit-&gt;max, dev_attr-&gt;max_dpi);</div><div class='ctx'> </div><div class='del'>-	dpit-&gt;app_tbl = kcalloc(dpit-&gt;max, sizeof(void *), GFP_KERNEL);</div><div class='add'>+	dpit-&gt;app_tbl = kcalloc(dpit-&gt;max,  sizeof(void *), GFP_KERNEL);</div><div class='ctx'> 	if (!dpit-&gt;app_tbl)</div><div class='del'>-		goto unmap_io;</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	bytes = dpit-&gt;max &gt;&gt; 3;</div><div class='ctx'> 	if (!bytes)</div><div class='hunk'>@@ -722,39 +804,20 @@ static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res     *res,</div><div class='ctx'> 	if (!dpit-&gt;tbl) {</div><div class='ctx'> 		kfree(dpit-&gt;app_tbl);</div><div class='ctx'> 		dpit-&gt;app_tbl = NULL;</div><div class='del'>-		goto unmap_io;</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	memset((u8 *)dpit-&gt;tbl, 0xFF, bytes);</div><div class='add'>+	mutex_init(&amp;res-&gt;dpi_tbl_lock);</div><div class='add'>+	dpit-&gt;priv_db = dpit-&gt;ucreg.bar_reg + dpit-&gt;ucreg.offset;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-unmap_io:</div><div class='del'>-	pci_iounmap(res-&gt;pdev, dpit-&gt;dbr_bar_reg_iomem);</div><div class='del'>-	return -ENOMEM;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* PKEYs */</div><div class='del'>-static void bnxt_qplib_cleanup_pkey_tbl(struct bnxt_qplib_pkey_tbl *pkey_tbl)</div><div class='del'>-{</div><div class='del'>-	memset(pkey_tbl-&gt;tbl, 0, sizeof(u16) * pkey_tbl-&gt;max);</div><div class='del'>-	pkey_tbl-&gt;active = 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void bnxt_qplib_init_pkey_tbl(struct bnxt_qplib_res *res,</div><div class='del'>-				     struct bnxt_qplib_pkey_tbl *pkey_tbl)</div><div class='del'>-{</div><div class='del'>-	u16 pkey = 0xFFFF;</div><div class='del'>-</div><div class='del'>-	memset(pkey_tbl-&gt;tbl, 0, sizeof(u16) * pkey_tbl-&gt;max);</div><div class='del'>-</div><div class='del'>-	/* pkey default = 0xFFFF */</div><div class='del'>-	bnxt_qplib_add_pkey(res, pkey_tbl, &amp;pkey, false);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* Stats */</div><div class='del'>-static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,</div><div class='del'>-				      struct bnxt_qplib_stats *stats)</div><div class='add'>+void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,</div><div class='add'>+			       struct bnxt_qplib_stats *stats)</div><div class='ctx'> {</div><div class='ctx'> 	if (stats-&gt;dma) {</div><div class='ctx'> 		dma_free_coherent(&amp;pdev-&gt;dev, stats-&gt;size,</div><div class='hunk'>@@ -764,16 +827,13 @@ static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,</div><div class='ctx'> 	stats-&gt;fw_id = -1;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,</div><div class='del'>-				      struct bnxt_qplib_stats *stats)</div><div class='add'>+int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,</div><div class='add'>+			       struct bnxt_qplib_chip_ctx *cctx,</div><div class='add'>+			       struct bnxt_qplib_stats *stats)</div><div class='ctx'> {</div><div class='ctx'> 	memset(stats, 0, sizeof(*stats));</div><div class='ctx'> 	stats-&gt;fw_id = -1;</div><div class='del'>-	/* 128 byte aligned context memory is required only for 57500.</div><div class='del'>-	 * However making this unconditional, it does not harm previous</div><div class='del'>-	 * generation.</div><div class='del'>-	 */</div><div class='del'>-	stats-&gt;size = ALIGN(sizeof(struct ctx_hw_stats), 128);</div><div class='add'>+	stats-&gt;size = cctx-&gt;hw_stats_size;</div><div class='ctx'> 	stats-&gt;dma = dma_alloc_coherent(&amp;pdev-&gt;dev, stats-&gt;size,</div><div class='ctx'> 					&amp;stats-&gt;dma_map, GFP_KERNEL);</div><div class='ctx'> 	if (!stats-&gt;dma) {</div><div class='hunk'>@@ -785,43 +845,41 @@ static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,</div><div class='ctx'> </div><div class='ctx'> void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res)</div><div class='ctx'> {</div><div class='del'>-	bnxt_qplib_cleanup_pkey_tbl(&amp;res-&gt;pkey_tbl);</div><div class='ctx'> 	bnxt_qplib_cleanup_sgid_tbl(res, &amp;res-&gt;sgid_tbl);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int bnxt_qplib_init_res(struct bnxt_qplib_res *res)</div><div class='ctx'> {</div><div class='ctx'> 	bnxt_qplib_init_sgid_tbl(&amp;res-&gt;sgid_tbl, res-&gt;netdev);</div><div class='del'>-	bnxt_qplib_init_pkey_tbl(res, &amp;res-&gt;pkey_tbl);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void bnxt_qplib_free_res(struct bnxt_qplib_res *res)</div><div class='ctx'> {</div><div class='del'>-	bnxt_qplib_free_pkey_tbl(res, &amp;res-&gt;pkey_tbl);</div><div class='add'>+	kfree(res-&gt;rcfw-&gt;qp_tbl);</div><div class='ctx'> 	bnxt_qplib_free_sgid_tbl(res, &amp;res-&gt;sgid_tbl);</div><div class='ctx'> 	bnxt_qplib_free_pd_tbl(&amp;res-&gt;pd_tbl);</div><div class='ctx'> 	bnxt_qplib_free_dpi_tbl(res, &amp;res-&gt;dpi_tbl);</div><div class='del'>-</div><div class='del'>-	res-&gt;netdev = NULL;</div><div class='del'>-	res-&gt;pdev = NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev,</div><div class='del'>-			 struct net_device *netdev,</div><div class='del'>-			 struct bnxt_qplib_dev_attr *dev_attr)</div><div class='add'>+int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct net_device *netdev)</div><div class='ctx'> {</div><div class='del'>-	int rc = 0;</div><div class='add'>+	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='add'>+	struct bnxt_qplib_dev_attr *dev_attr;</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='del'>-	res-&gt;pdev = pdev;</div><div class='ctx'> 	res-&gt;netdev = netdev;</div><div class='add'>+	dev_attr = res-&gt;dattr;</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_alloc_sgid_tbl(res, &amp;res-&gt;sgid_tbl, dev_attr-&gt;max_sgid);</div><div class='del'>-	if (rc)</div><div class='del'>-		goto fail;</div><div class='add'>+	/* Allocate one extra to hold the QP1 entries */</div><div class='add'>+	rcfw-&gt;qp_tbl_size = max_t(u32, BNXT_RE_MAX_QPC_COUNT + 1, dev_attr-&gt;max_qp);</div><div class='add'>+	rcfw-&gt;qp_tbl = kcalloc(rcfw-&gt;qp_tbl_size, sizeof(struct bnxt_qplib_qp_node),</div><div class='add'>+			       GFP_KERNEL);</div><div class='add'>+	if (!rcfw-&gt;qp_tbl)</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_alloc_pkey_tbl(res, &amp;res-&gt;pkey_tbl, dev_attr-&gt;max_pkey);</div><div class='add'>+	rc = bnxt_qplib_alloc_sgid_tbl(res, &amp;res-&gt;sgid_tbl, dev_attr-&gt;max_sgid);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='hunk'>@@ -829,7 +887,7 @@ int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev,</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_alloc_dpi_tbl(res, &amp;res-&gt;dpi_tbl, dev_attr-&gt;l2_db_size);</div><div class='add'>+	rc = bnxt_qplib_alloc_dpi_tbl(res, dev_attr);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='hunk'>@@ -838,3 +896,60 @@ fail:</div><div class='ctx'> 	bnxt_qplib_free_res(res);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='add'>+</div><div class='add'>+void bnxt_qplib_unmap_db_bar(struct bnxt_qplib_res *res)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_reg_desc *reg;</div><div class='add'>+</div><div class='add'>+	reg = &amp;res-&gt;dpi_tbl.ucreg;</div><div class='add'>+	if (reg-&gt;bar_reg)</div><div class='add'>+		pci_iounmap(res-&gt;pdev, reg-&gt;bar_reg);</div><div class='add'>+	reg-&gt;bar_reg = NULL;</div><div class='add'>+	reg-&gt;bar_base = 0;</div><div class='add'>+	reg-&gt;len = 0;</div><div class='add'>+	reg-&gt;bar_id = 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bnxt_qplib_map_db_bar(struct bnxt_qplib_res *res)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_reg_desc *ucreg;</div><div class='add'>+	struct bnxt_qplib_reg_desc *wcreg;</div><div class='add'>+</div><div class='add'>+	wcreg = &amp;res-&gt;dpi_tbl.wcreg;</div><div class='add'>+	wcreg-&gt;bar_id = RCFW_DBR_PCI_BAR_REGION;</div><div class='add'>+	wcreg-&gt;bar_base = pci_resource_start(res-&gt;pdev, wcreg-&gt;bar_id);</div><div class='add'>+</div><div class='add'>+	ucreg = &amp;res-&gt;dpi_tbl.ucreg;</div><div class='add'>+	ucreg-&gt;bar_id = RCFW_DBR_PCI_BAR_REGION;</div><div class='add'>+	ucreg-&gt;bar_base = pci_resource_start(res-&gt;pdev, ucreg-&gt;bar_id);</div><div class='add'>+	ucreg-&gt;len = ucreg-&gt;offset + PAGE_SIZE;</div><div class='add'>+	if (!ucreg-&gt;len || ((ucreg-&gt;len &amp; (PAGE_SIZE - 1)) != 0)) {</div><div class='add'>+		dev_err(&amp;res-&gt;pdev-&gt;dev, "QPLIB: invalid dbr length %d",</div><div class='add'>+			(int)ucreg-&gt;len);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+	ucreg-&gt;bar_reg = ioremap(ucreg-&gt;bar_base, ucreg-&gt;len);</div><div class='add'>+	if (!ucreg-&gt;bar_reg) {</div><div class='add'>+		dev_err(&amp;res-&gt;pdev-&gt;dev, "privileged dpi map failed!");</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bnxt_qplib_determine_atomics(struct pci_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int comp;</div><div class='add'>+	u16 ctl2;</div><div class='add'>+</div><div class='add'>+	comp = pci_enable_atomic_ops_to_root(dev,</div><div class='add'>+					     PCI_EXP_DEVCAP2_ATOMIC_COMP32);</div><div class='add'>+	if (comp)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	comp = pci_enable_atomic_ops_to_root(dev,</div><div class='add'>+					     PCI_EXP_DEVCAP2_ATOMIC_COMP64);</div><div class='add'>+	if (comp)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &amp;ctl2);</div><div class='add'>+	return !(ctl2 &amp; PCI_EXP_DEVCTL2_ATOMIC_REQ);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h<br/>index 30c42c92fac7..2ea3b7f232a3 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_res.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_res.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_res.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/qplib_res.h</a></div><div class='hunk'>@@ -39,8 +39,65 @@</div><div class='ctx'> #ifndef __BNXT_QPLIB_RES_H__</div><div class='ctx'> #define __BNXT_QPLIB_RES_H__</div><div class='ctx'> </div><div class='add'>+#include "bnxt_ulp.h"</div><div class='add'>+</div><div class='ctx'> extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero;</div><div class='ctx'> </div><div class='add'>+#define CHIP_NUM_57508		0x1750</div><div class='add'>+#define CHIP_NUM_57504		0x1751</div><div class='add'>+#define CHIP_NUM_57502		0x1752</div><div class='add'>+#define CHIP_NUM_58818          0xd818</div><div class='add'>+#define CHIP_NUM_57608          0x1760</div><div class='add'>+</div><div class='add'>+#define BNXT_RE_MAX_QPC_COUNT		(64 * 1024)</div><div class='add'>+#define BNXT_RE_MAX_MRW_COUNT		(64 * 1024)</div><div class='add'>+#define BNXT_RE_MAX_SRQC_COUNT		(64 * 1024)</div><div class='add'>+#define BNXT_RE_MAX_CQ_COUNT		(64 * 1024)</div><div class='add'>+#define BNXT_RE_MAX_MRW_COUNT_64K	(64 * 1024)</div><div class='add'>+#define BNXT_RE_MAX_MRW_COUNT_256K	(256 * 1024)</div><div class='add'>+</div><div class='add'>+#define BNXT_QPLIB_DBR_VALID		(0x1UL &lt;&lt; 26)</div><div class='add'>+#define BNXT_QPLIB_DBR_EPOCH_SHIFT	24</div><div class='add'>+#define BNXT_QPLIB_DBR_TOGGLE_SHIFT	25</div><div class='add'>+</div><div class='add'>+struct bnxt_qplib_drv_modes {</div><div class='add'>+	u8	wqe_mode;</div><div class='add'>+	bool db_push;</div><div class='add'>+	bool dbr_pacing;</div><div class='add'>+	u32 toggle_bits;</div><div class='add'>+	u8 roce_mirror;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum bnxt_re_toggle_modes {</div><div class='add'>+	BNXT_QPLIB_CQ_TOGGLE_BIT = 0x1,</div><div class='add'>+	BNXT_QPLIB_SRQ_TOGGLE_BIT = 0x2,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_qplib_chip_ctx {</div><div class='add'>+	u16	chip_num;</div><div class='add'>+	u8	chip_rev;</div><div class='add'>+	u8	chip_metal;</div><div class='add'>+	u16	hw_stats_size;</div><div class='add'>+	u16	hwrm_cmd_max_timeout;</div><div class='add'>+	struct bnxt_qplib_drv_modes modes;</div><div class='add'>+	u64	hwrm_intf_ver;</div><div class='add'>+	u32     dbr_stat_db_fifo;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_qplib_db_pacing_data {</div><div class='add'>+	u32 do_pacing;</div><div class='add'>+	u32 pacing_th;</div><div class='add'>+	u32 alarm_th;</div><div class='add'>+	u32 fifo_max_depth;</div><div class='add'>+	u32 fifo_room_mask;</div><div class='add'>+	u32 fifo_room_shift;</div><div class='add'>+	u32 grc_reg_offset;</div><div class='add'>+	u32 dev_err_state;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define BNXT_QPLIB_DBR_PF_DB_OFFSET     0x10000</div><div class='add'>+#define BNXT_QPLIB_DBR_VF_DB_OFFSET     0x4000</div><div class='add'>+</div><div class='ctx'> #define PTR_CNT_PER_PG		(PAGE_SIZE / sizeof(void *))</div><div class='ctx'> #define PTR_MAX_IDX_PER_PG	(PTR_CNT_PER_PG - 1)</div><div class='ctx'> #define PTR_PG(x)		(((x) &amp; ~PTR_MAX_IDX_PER_PG) / PTR_CNT_PER_PG)</div><div class='hunk'>@@ -55,7 +112,8 @@ extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero;</div><div class='ctx'> enum bnxt_qplib_hwq_type {</div><div class='ctx'> 	HWQ_TYPE_CTX,</div><div class='ctx'> 	HWQ_TYPE_QUEUE,</div><div class='del'>-	HWQ_TYPE_L2_CMPL</div><div class='add'>+	HWQ_TYPE_L2_CMPL,</div><div class='add'>+	HWQ_TYPE_MR</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> #define MAX_PBL_LVL_0_PGS		1</div><div class='hunk'>@@ -63,6 +121,7 @@ enum bnxt_qplib_hwq_type {</div><div class='ctx'> #define MAX_PBL_LVL_1_PGS_SHIFT		9</div><div class='ctx'> #define MAX_PBL_LVL_1_PGS_FOR_LVL_2	256</div><div class='ctx'> #define MAX_PBL_LVL_2_PGS		(256 * 512)</div><div class='add'>+#define MAX_PDL_LVL_SHIFT               9</div><div class='ctx'> </div><div class='ctx'> enum bnxt_qplib_pbl_lvl {</div><div class='ctx'> 	PBL_LVL_0,</div><div class='hunk'>@@ -78,6 +137,23 @@ enum bnxt_qplib_pbl_lvl {</div><div class='ctx'> #define ROCE_PG_SIZE_8M		(8 * 1024 * 1024)</div><div class='ctx'> #define ROCE_PG_SIZE_1G		(1024 * 1024 * 1024)</div><div class='ctx'> </div><div class='add'>+enum bnxt_qplib_hwrm_pg_size {</div><div class='add'>+	BNXT_QPLIB_HWRM_PG_SIZE_4K	= 0,</div><div class='add'>+	BNXT_QPLIB_HWRM_PG_SIZE_8K	= 1,</div><div class='add'>+	BNXT_QPLIB_HWRM_PG_SIZE_64K	= 2,</div><div class='add'>+	BNXT_QPLIB_HWRM_PG_SIZE_2M	= 3,</div><div class='add'>+	BNXT_QPLIB_HWRM_PG_SIZE_8M	= 4,</div><div class='add'>+	BNXT_QPLIB_HWRM_PG_SIZE_1G	= 5,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_qplib_reg_desc {</div><div class='add'>+	u8		bar_id;</div><div class='add'>+	resource_size_t	bar_base;</div><div class='add'>+	unsigned long	offset;</div><div class='add'>+	void __iomem	*bar_reg;</div><div class='add'>+	size_t		len;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct bnxt_qplib_pbl {</div><div class='ctx'> 	u32				pg_count;</div><div class='ctx'> 	u32				pg_size;</div><div class='hunk'>@@ -85,23 +161,68 @@ struct bnxt_qplib_pbl {</div><div class='ctx'> 	dma_addr_t			*pg_map_arr;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct bnxt_qplib_sg_info {</div><div class='add'>+	struct ib_umem			*umem;</div><div class='add'>+	u32				npages;</div><div class='add'>+	u32				pgshft;</div><div class='add'>+	u32				pgsize;</div><div class='add'>+	bool				nopte;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_qplib_hwq_attr {</div><div class='add'>+	struct bnxt_qplib_res		*res;</div><div class='add'>+	struct bnxt_qplib_sg_info	*sginfo;</div><div class='add'>+	enum bnxt_qplib_hwq_type	type;</div><div class='add'>+	u32				depth;</div><div class='add'>+	u32				stride;</div><div class='add'>+	u32				aux_stride;</div><div class='add'>+	u32				aux_depth;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct bnxt_qplib_hwq {</div><div class='ctx'> 	struct pci_dev			*pdev;</div><div class='ctx'> 	/* lock to protect qplib_hwq */</div><div class='ctx'> 	spinlock_t			lock;</div><div class='del'>-	struct bnxt_qplib_pbl		pbl[PBL_LVL_MAX];</div><div class='add'>+	struct bnxt_qplib_pbl		pbl[PBL_LVL_MAX + 1];</div><div class='ctx'> 	enum bnxt_qplib_pbl_lvl		level;		/* 0, 1, or 2 */</div><div class='ctx'> 	/* ptr for easy access to the PBL entries */</div><div class='ctx'> 	void				**pbl_ptr;</div><div class='ctx'> 	/* ptr for easy access to the dma_addr */</div><div class='ctx'> 	dma_addr_t			*pbl_dma_ptr;</div><div class='ctx'> 	u32				max_elements;</div><div class='add'>+	u32				depth;</div><div class='ctx'> 	u16				element_size;	/* Size of each entry */</div><div class='add'>+	u16				qe_ppg;	/* queue entry per page */</div><div class='ctx'> </div><div class='ctx'> 	u32				prod;		/* raw */</div><div class='ctx'> 	u32				cons;		/* raw */</div><div class='ctx'> 	u8				cp_bit;</div><div class='ctx'> 	u8				is_user;</div><div class='add'>+	u64				*pad_pg;</div><div class='add'>+	u32				pad_stride;</div><div class='add'>+	u32				pad_pgofft;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_qplib_db_info {</div><div class='add'>+	void __iomem		*db;</div><div class='add'>+	void __iomem		*priv_db;</div><div class='add'>+	struct bnxt_qplib_hwq	*hwq;</div><div class='add'>+	u32			xid;</div><div class='add'>+	u32			max_slot;</div><div class='add'>+	u32                     flags;</div><div class='add'>+	u8			toggle;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum bnxt_qplib_db_info_flags_mask {</div><div class='add'>+	BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT        = 0x0UL,</div><div class='add'>+	BNXT_QPLIB_FLAG_EPOCH_PROD_SHIFT        = 0x1UL,</div><div class='add'>+	BNXT_QPLIB_FLAG_EPOCH_CONS_MASK         = 0x1UL,</div><div class='add'>+	BNXT_QPLIB_FLAG_EPOCH_PROD_MASK         = 0x2UL,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum bnxt_qplib_db_epoch_flag_shift {</div><div class='add'>+	BNXT_QPLIB_DB_EPOCH_CONS_SHIFT  = BNXT_QPLIB_DBR_EPOCH_SHIFT,</div><div class='add'>+	BNXT_QPLIB_DB_EPOCH_PROD_SHIFT  = (BNXT_QPLIB_DBR_EPOCH_SHIFT - 1),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /* Tables */</div><div class='hunk'>@@ -111,7 +232,7 @@ struct bnxt_qplib_pd_tbl {</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct bnxt_qplib_sgid_tbl {</div><div class='del'>-	struct bnxt_qplib_gid		*tbl;</div><div class='add'>+	struct bnxt_qplib_gid_info	*tbl;</div><div class='ctx'> 	u16				*hw_id;</div><div class='ctx'> 	u16				max;</div><div class='ctx'> 	u16				active;</div><div class='hunk'>@@ -119,24 +240,27 @@ struct bnxt_qplib_sgid_tbl {</div><div class='ctx'> 	u8				*vlan;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct bnxt_qplib_pkey_tbl {</div><div class='del'>-	u16				*tbl;</div><div class='del'>-	u16				max;</div><div class='del'>-	u16				active;</div><div class='add'>+enum {</div><div class='add'>+	BNXT_QPLIB_DPI_TYPE_KERNEL      = 0,</div><div class='add'>+	BNXT_QPLIB_DPI_TYPE_UC          = 1,</div><div class='add'>+	BNXT_QPLIB_DPI_TYPE_WC          = 2</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct bnxt_qplib_dpi {</div><div class='ctx'> 	u32				dpi;</div><div class='add'>+	u32				bit;</div><div class='ctx'> 	void __iomem			*dbr;</div><div class='ctx'> 	u64				umdbr;</div><div class='add'>+	u8				type;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct bnxt_qplib_dpi_tbl {</div><div class='ctx'> 	void				**app_tbl;</div><div class='ctx'> 	unsigned long			*tbl;</div><div class='ctx'> 	u16				max;</div><div class='del'>-	void __iomem			*dbr_bar_reg_iomem;</div><div class='del'>-	u64				unmapped_dbr;</div><div class='add'>+	struct bnxt_qplib_reg_desc	ucreg; /* Hold entire DB bar. */</div><div class='add'>+	struct bnxt_qplib_reg_desc	wcreg;</div><div class='add'>+	void __iomem			*priv_db;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct bnxt_qplib_stats {</div><div class='hunk'>@@ -159,6 +283,15 @@ struct bnxt_qplib_vf_res {</div><div class='ctx'> #define BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE	64</div><div class='ctx'> #define BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE	128</div><div class='ctx'> </div><div class='add'>+#define MAX_TQM_ALLOC_REQ               48</div><div class='add'>+#define MAX_TQM_ALLOC_BLK_SIZE          8</div><div class='add'>+struct bnxt_qplib_tqm_ctx {</div><div class='add'>+	struct bnxt_qplib_hwq           pde;</div><div class='add'>+	u8                              pde_level; /* Original level */</div><div class='add'>+	struct bnxt_qplib_hwq           qtbl[MAX_TQM_ALLOC_REQ];</div><div class='add'>+	u8                              qcount[MAX_TQM_ALLOC_REQ];</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct bnxt_qplib_ctx {</div><div class='ctx'> 	u32				qpc_count;</div><div class='ctx'> 	struct bnxt_qplib_hwq		qpc_tbl;</div><div class='hunk'>@@ -169,61 +302,113 @@ struct bnxt_qplib_ctx {</div><div class='ctx'> 	u32				cq_count;</div><div class='ctx'> 	struct bnxt_qplib_hwq		cq_tbl;</div><div class='ctx'> 	struct bnxt_qplib_hwq		tim_tbl;</div><div class='del'>-#define MAX_TQM_ALLOC_REQ		48</div><div class='del'>-#define MAX_TQM_ALLOC_BLK_SIZE		8</div><div class='del'>-	u8				tqm_count[MAX_TQM_ALLOC_REQ];</div><div class='del'>-	struct bnxt_qplib_hwq		tqm_pde;</div><div class='del'>-	u32				tqm_pde_level;</div><div class='del'>-	struct bnxt_qplib_hwq		tqm_tbl[MAX_TQM_ALLOC_REQ];</div><div class='add'>+	struct bnxt_qplib_tqm_ctx	tqm_ctx;</div><div class='ctx'> 	struct bnxt_qplib_stats		stats;</div><div class='add'>+	struct bnxt_qplib_stats		stats3;</div><div class='ctx'> 	struct bnxt_qplib_vf_res	vf_res;</div><div class='del'>-	u64				hwrm_intf_ver;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct bnxt_qplib_chip_ctx {</div><div class='del'>-	u16	chip_num;</div><div class='del'>-	u8	chip_rev;</div><div class='del'>-	u8	chip_metal;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define CHIP_NUM_57500          0x1750</div><div class='del'>-</div><div class='ctx'> struct bnxt_qplib_res {</div><div class='ctx'> 	struct pci_dev			*pdev;</div><div class='ctx'> 	struct bnxt_qplib_chip_ctx	*cctx;</div><div class='add'>+	struct bnxt_qplib_dev_attr      *dattr;</div><div class='ctx'> 	struct net_device		*netdev;</div><div class='del'>-</div><div class='add'>+	struct bnxt_en_dev		*en_dev;</div><div class='ctx'> 	struct bnxt_qplib_rcfw		*rcfw;</div><div class='ctx'> 	struct bnxt_qplib_pd_tbl	pd_tbl;</div><div class='add'>+	/* To protect the pd table bit map */</div><div class='add'>+	struct mutex			pd_tbl_lock;</div><div class='ctx'> 	struct bnxt_qplib_sgid_tbl	sgid_tbl;</div><div class='del'>-	struct bnxt_qplib_pkey_tbl	pkey_tbl;</div><div class='ctx'> 	struct bnxt_qplib_dpi_tbl	dpi_tbl;</div><div class='add'>+	/* To protect the dpi table bit map */</div><div class='add'>+	struct mutex                    dpi_tbl_lock;</div><div class='ctx'> 	bool				prio;</div><div class='add'>+	bool                            is_vf;</div><div class='add'>+	struct bnxt_qplib_db_pacing_data *pacing_data;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static inline bool bnxt_qplib_is_chip_gen_p7(struct bnxt_qplib_chip_ctx *cctx)</div><div class='add'>+{</div><div class='add'>+	return (cctx-&gt;chip_num == CHIP_NUM_58818 ||</div><div class='add'>+		cctx-&gt;chip_num == CHIP_NUM_57608);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static inline bool bnxt_qplib_is_chip_gen_p5(struct bnxt_qplib_chip_ctx *cctx)</div><div class='ctx'> {</div><div class='del'>-	return (cctx-&gt;chip_num == CHIP_NUM_57500);</div><div class='add'>+	return (cctx-&gt;chip_num == CHIP_NUM_57508 ||</div><div class='add'>+		cctx-&gt;chip_num == CHIP_NUM_57504 ||</div><div class='add'>+		cctx-&gt;chip_num == CHIP_NUM_57502);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool bnxt_qplib_is_chip_gen_p5_p7(struct bnxt_qplib_chip_ctx *cctx)</div><div class='add'>+{</div><div class='add'>+	return bnxt_qplib_is_chip_gen_p5(cctx) || bnxt_qplib_is_chip_gen_p7(cctx);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static inline u8 bnxt_qplib_get_hwq_type(struct bnxt_qplib_res *res)</div><div class='ctx'> {</div><div class='del'>-	return bnxt_qplib_is_chip_gen_p5(res-&gt;cctx) ?</div><div class='add'>+	return bnxt_qplib_is_chip_gen_p5_p7(res-&gt;cctx) ?</div><div class='ctx'> 					HWQ_TYPE_QUEUE : HWQ_TYPE_L2_CMPL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static inline u8 bnxt_qplib_get_ring_type(struct bnxt_qplib_chip_ctx *cctx)</div><div class='ctx'> {</div><div class='del'>-	return bnxt_qplib_is_chip_gen_p5(cctx) ?</div><div class='add'>+	return bnxt_qplib_is_chip_gen_p5_p7(cctx) ?</div><div class='ctx'> 	       RING_ALLOC_REQ_RING_TYPE_NQ :</div><div class='ctx'> 	       RING_ALLOC_REQ_RING_TYPE_ROCE_CMPL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct bnxt_qplib_sg_info {</div><div class='del'>-	struct scatterlist		*sglist;</div><div class='del'>-	u32				nmap;</div><div class='del'>-	u32				npages;</div><div class='del'>-};</div><div class='add'>+static inline u8 bnxt_qplib_base_pg_size(struct bnxt_qplib_hwq *hwq)</div><div class='add'>+{</div><div class='add'>+	u8 pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K;</div><div class='add'>+	struct bnxt_qplib_pbl *pbl;</div><div class='add'>+</div><div class='add'>+	pbl = &amp;hwq-&gt;pbl[PBL_LVL_0];</div><div class='add'>+	switch (pbl-&gt;pg_size) {</div><div class='add'>+	case ROCE_PG_SIZE_4K:</div><div class='add'>+		pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K;</div><div class='add'>+		break;</div><div class='add'>+	case ROCE_PG_SIZE_8K:</div><div class='add'>+		pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8K;</div><div class='add'>+		break;</div><div class='add'>+	case ROCE_PG_SIZE_64K:</div><div class='add'>+		pg_size = BNXT_QPLIB_HWRM_PG_SIZE_64K;</div><div class='add'>+		break;</div><div class='add'>+	case ROCE_PG_SIZE_2M:</div><div class='add'>+		pg_size = BNXT_QPLIB_HWRM_PG_SIZE_2M;</div><div class='add'>+		break;</div><div class='add'>+	case ROCE_PG_SIZE_8M:</div><div class='add'>+		pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8M;</div><div class='add'>+		break;</div><div class='add'>+	case ROCE_PG_SIZE_1G:</div><div class='add'>+		pg_size = BNXT_QPLIB_HWRM_PG_SIZE_1G;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return pg_size;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void *bnxt_qplib_get_qe(struct bnxt_qplib_hwq *hwq,</div><div class='add'>+				      u32 indx, u64 *pg)</div><div class='add'>+{</div><div class='add'>+	u32 pg_num, pg_idx;</div><div class='add'>+</div><div class='add'>+	pg_num = (indx / hwq-&gt;qe_ppg);</div><div class='add'>+	pg_idx = (indx % hwq-&gt;qe_ppg);</div><div class='add'>+	if (pg)</div><div class='add'>+		*pg = (u64)&amp;hwq-&gt;pbl_ptr[pg_num];</div><div class='add'>+	return (void *)(hwq-&gt;pbl_ptr[pg_num] + hwq-&gt;element_size * pg_idx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void *bnxt_qplib_get_prod_qe(struct bnxt_qplib_hwq *hwq, u32 idx)</div><div class='add'>+{</div><div class='add'>+	idx += hwq-&gt;prod;</div><div class='add'>+	if (idx &gt;= hwq-&gt;depth)</div><div class='add'>+		idx -= hwq-&gt;depth;</div><div class='add'>+	return bnxt_qplib_get_qe(hwq, idx, NULL);</div><div class='add'>+}</div><div class='ctx'> </div><div class='ctx'> #define to_bnxt_qplib(ptr, type, member)	\</div><div class='ctx'> 	container_of(ptr, type, member)</div><div class='hunk'>@@ -231,32 +416,211 @@ struct bnxt_qplib_sg_info {</div><div class='ctx'> struct bnxt_qplib_pd;</div><div class='ctx'> struct bnxt_qplib_dev_attr;</div><div class='ctx'> </div><div class='del'>-void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq);</div><div class='del'>-int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,</div><div class='del'>-			      struct bnxt_qplib_sg_info *sg_info, u32 *elements,</div><div class='del'>-			      u32 elements_per_page, u32 aux, u32 pg_size,</div><div class='del'>-			      enum bnxt_qplib_hwq_type hwq_type);</div><div class='del'>-void bnxt_qplib_get_guid(u8 *dev_addr, u8 *guid);</div><div class='del'>-int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pd_tbl,</div><div class='add'>+void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res,</div><div class='add'>+			 struct bnxt_qplib_hwq *hwq);</div><div class='add'>+int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq,</div><div class='add'>+			      struct bnxt_qplib_hwq_attr *hwq_attr);</div><div class='add'>+int bnxt_qplib_alloc_pd(struct bnxt_qplib_res *res,</div><div class='ctx'> 			struct bnxt_qplib_pd *pd);</div><div class='ctx'> int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res,</div><div class='ctx'> 			  struct bnxt_qplib_pd_tbl *pd_tbl,</div><div class='ctx'> 			  struct bnxt_qplib_pd *pd);</div><div class='del'>-int bnxt_qplib_alloc_dpi(struct bnxt_qplib_dpi_tbl *dpit,</div><div class='del'>-			 struct bnxt_qplib_dpi     *dpi,</div><div class='del'>-			 void                      *app);</div><div class='add'>+int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res,</div><div class='add'>+			 struct bnxt_qplib_dpi *dpi,</div><div class='add'>+			 void *app, u8 type);</div><div class='ctx'> int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,</div><div class='del'>-			   struct bnxt_qplib_dpi_tbl *dpi_tbl,</div><div class='ctx'> 			   struct bnxt_qplib_dpi *dpi);</div><div class='ctx'> void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res);</div><div class='ctx'> int bnxt_qplib_init_res(struct bnxt_qplib_res *res);</div><div class='ctx'> void bnxt_qplib_free_res(struct bnxt_qplib_res *res);</div><div class='del'>-int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev,</div><div class='del'>-			 struct net_device *netdev,</div><div class='del'>-			 struct bnxt_qplib_dev_attr *dev_attr);</div><div class='del'>-void bnxt_qplib_free_ctx(struct pci_dev *pdev,</div><div class='del'>-			 struct bnxt_qplib_ctx *ctx);</div><div class='del'>-int bnxt_qplib_alloc_ctx(struct pci_dev *pdev,</div><div class='del'>-			 struct bnxt_qplib_ctx *ctx,</div><div class='del'>-			 bool virt_fn, bool is_p5);</div><div class='add'>+int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct net_device *netdev);</div><div class='add'>+void bnxt_qplib_free_hwctx(struct bnxt_qplib_res *res,</div><div class='add'>+			   struct bnxt_qplib_ctx *ctx);</div><div class='add'>+int bnxt_qplib_alloc_hwctx(struct bnxt_qplib_res *res,</div><div class='add'>+			   struct bnxt_qplib_ctx *ctx);</div><div class='add'>+int bnxt_qplib_map_db_bar(struct bnxt_qplib_res *res);</div><div class='add'>+void bnxt_qplib_unmap_db_bar(struct bnxt_qplib_res *res);</div><div class='add'>+</div><div class='add'>+int bnxt_qplib_determine_atomics(struct pci_dev *dev);</div><div class='add'>+int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,</div><div class='add'>+			       struct bnxt_qplib_chip_ctx *cctx,</div><div class='add'>+			       struct bnxt_qplib_stats *stats);</div><div class='add'>+void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,</div><div class='add'>+			       struct bnxt_qplib_stats *stats);</div><div class='add'>+</div><div class='add'>+static inline void bnxt_qplib_hwq_incr_prod(struct bnxt_qplib_db_info *dbinfo,</div><div class='add'>+					    struct bnxt_qplib_hwq *hwq, u32 cnt)</div><div class='add'>+{</div><div class='add'>+	/* move prod and update toggle/epoch if wrap around */</div><div class='add'>+	hwq-&gt;prod += cnt;</div><div class='add'>+	if (hwq-&gt;prod &gt;= hwq-&gt;depth) {</div><div class='add'>+		hwq-&gt;prod %= hwq-&gt;depth;</div><div class='add'>+		dbinfo-&gt;flags ^= 1UL &lt;&lt; BNXT_QPLIB_FLAG_EPOCH_PROD_SHIFT;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void bnxt_qplib_hwq_incr_cons(u32 max_elements, u32 *cons, u32 cnt,</div><div class='add'>+					    u32 *dbinfo_flags)</div><div class='add'>+{</div><div class='add'>+	/* move cons and update toggle/epoch if wrap around */</div><div class='add'>+	*cons += cnt;</div><div class='add'>+	if (*cons &gt;= max_elements) {</div><div class='add'>+		*cons %= max_elements;</div><div class='add'>+		*dbinfo_flags ^= 1UL &lt;&lt; BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void bnxt_qplib_ring_db32(struct bnxt_qplib_db_info *info,</div><div class='add'>+					bool arm)</div><div class='add'>+{</div><div class='add'>+	u32 key = 0;</div><div class='add'>+</div><div class='add'>+	key |= info-&gt;hwq-&gt;cons | (CMPL_DOORBELL_IDX_VALID |</div><div class='add'>+		(CMPL_DOORBELL_KEY_CMPL &amp; CMPL_DOORBELL_KEY_MASK));</div><div class='add'>+	if (!arm)</div><div class='add'>+		key |= CMPL_DOORBELL_MASK;</div><div class='add'>+	writel(key, info-&gt;db);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define BNXT_QPLIB_INIT_DBHDR(xid, type, indx, toggle) \</div><div class='add'>+	(((u64)(((xid) &amp; DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE |  \</div><div class='add'>+		(type) | BNXT_QPLIB_DBR_VALID) &lt;&lt; 32) | (indx) |  \</div><div class='add'>+	 (((u32)(toggle)) &lt;&lt; (BNXT_QPLIB_DBR_TOGGLE_SHIFT)))</div><div class='add'>+</div><div class='add'>+static inline void bnxt_qplib_ring_db(struct bnxt_qplib_db_info *info,</div><div class='add'>+				      u32 type)</div><div class='add'>+{</div><div class='add'>+	u64 key = 0;</div><div class='add'>+	u32 indx;</div><div class='add'>+	u8 toggle = 0;</div><div class='add'>+</div><div class='add'>+	if (type == DBC_DBC_TYPE_CQ_ARMALL ||</div><div class='add'>+	    type == DBC_DBC_TYPE_CQ_ARMSE)</div><div class='add'>+		toggle = info-&gt;toggle;</div><div class='add'>+</div><div class='add'>+	indx = (info-&gt;hwq-&gt;cons &amp; DBC_DBC_INDEX_MASK) |</div><div class='add'>+	       ((info-&gt;flags &amp; BNXT_QPLIB_FLAG_EPOCH_CONS_MASK) &lt;&lt;</div><div class='add'>+		 BNXT_QPLIB_DB_EPOCH_CONS_SHIFT);</div><div class='add'>+</div><div class='add'>+	key =  BNXT_QPLIB_INIT_DBHDR(info-&gt;xid, type, indx, toggle);</div><div class='add'>+	writeq(key, info-&gt;db);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void bnxt_qplib_ring_prod_db(struct bnxt_qplib_db_info *info,</div><div class='add'>+					   u32 type)</div><div class='add'>+{</div><div class='add'>+	u64 key = 0;</div><div class='add'>+	u32 indx;</div><div class='add'>+</div><div class='add'>+	indx = (((info-&gt;hwq-&gt;prod / info-&gt;max_slot) &amp; DBC_DBC_INDEX_MASK) |</div><div class='add'>+		((info-&gt;flags &amp; BNXT_QPLIB_FLAG_EPOCH_PROD_MASK) &lt;&lt;</div><div class='add'>+		 BNXT_QPLIB_DB_EPOCH_PROD_SHIFT));</div><div class='add'>+	key = BNXT_QPLIB_INIT_DBHDR(info-&gt;xid, type, indx, 0);</div><div class='add'>+	writeq(key, info-&gt;db);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void bnxt_qplib_armen_db(struct bnxt_qplib_db_info *info,</div><div class='add'>+				       u32 type)</div><div class='add'>+{</div><div class='add'>+	u64 key = 0;</div><div class='add'>+	u8 toggle = 0;</div><div class='add'>+</div><div class='add'>+	if (type == DBC_DBC_TYPE_CQ_ARMENA || type == DBC_DBC_TYPE_SRQ_ARMENA)</div><div class='add'>+		toggle = info-&gt;toggle;</div><div class='add'>+	/* Index always at 0 */</div><div class='add'>+	key = BNXT_QPLIB_INIT_DBHDR(info-&gt;xid, type, 0, toggle);</div><div class='add'>+	writeq(key, info-&gt;priv_db);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void bnxt_qplib_srq_arm_db(struct bnxt_qplib_db_info *info,</div><div class='add'>+					 u32 th)</div><div class='add'>+{</div><div class='add'>+	u64 key = 0;</div><div class='add'>+</div><div class='add'>+	key = BNXT_QPLIB_INIT_DBHDR(info-&gt;xid, DBC_DBC_TYPE_SRQ_ARM, th, info-&gt;toggle);</div><div class='add'>+	writeq(key, info-&gt;priv_db);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void bnxt_qplib_ring_nq_db(struct bnxt_qplib_db_info *info,</div><div class='add'>+					 struct bnxt_qplib_chip_ctx *cctx,</div><div class='add'>+					 bool arm)</div><div class='add'>+{</div><div class='add'>+	u32 type;</div><div class='add'>+</div><div class='add'>+	type = arm ? DBC_DBC_TYPE_NQ_ARM : DBC_DBC_TYPE_NQ;</div><div class='add'>+	if (bnxt_qplib_is_chip_gen_p5_p7(cctx))</div><div class='add'>+		bnxt_qplib_ring_db(info, type);</div><div class='add'>+	else</div><div class='add'>+		bnxt_qplib_ring_db32(info, arm);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool _is_ext_stats_supported(u16 dev_cap_flags)</div><div class='add'>+{</div><div class='add'>+	return dev_cap_flags &amp;</div><div class='add'>+		CREQ_QUERY_FUNC_RESP_SB_EXT_STATS;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int bnxt_ext_stats_supported(struct bnxt_qplib_chip_ctx *ctx,</div><div class='add'>+					   u16 flags, bool virtfn)</div><div class='add'>+{</div><div class='add'>+	/* ext stats supported if cap flag is set AND is a PF OR a Thor2 VF */</div><div class='add'>+	return (_is_ext_stats_supported(flags) &amp;&amp;</div><div class='add'>+		((virtfn &amp;&amp; bnxt_qplib_is_chip_gen_p7(ctx)) || (!virtfn)));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool _is_hw_retx_supported(u16 dev_cap_flags)</div><div class='add'>+{</div><div class='add'>+	return dev_cap_flags &amp;</div><div class='add'>+		(CREQ_QUERY_FUNC_RESP_SB_HW_REQUESTER_RETX_ENABLED |</div><div class='add'>+		 CREQ_QUERY_FUNC_RESP_SB_HW_RESPONDER_RETX_ENABLED);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define BNXT_RE_HW_RETX(a) _is_hw_retx_supported((a))</div><div class='add'>+</div><div class='add'>+static inline bool _is_host_msn_table(u16 dev_cap_ext_flags2)</div><div class='add'>+{</div><div class='add'>+	return (dev_cap_ext_flags2 &amp; CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_MASK) ==</div><div class='add'>+		CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_MSN_TABLE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u8 bnxt_qplib_dbr_pacing_en(struct bnxt_qplib_chip_ctx *cctx)</div><div class='add'>+{</div><div class='add'>+	return cctx-&gt;modes.dbr_pacing;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u8 bnxt_qplib_roce_mirror_supported(struct bnxt_qplib_chip_ctx *cctx)</div><div class='add'>+{</div><div class='add'>+	return cctx-&gt;modes.roce_mirror;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool _is_alloc_mr_unified(u16 dev_cap_flags)</div><div class='add'>+{</div><div class='add'>+	return dev_cap_flags &amp; CREQ_QUERY_FUNC_RESP_SB_MR_REGISTER_ALLOC;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool _is_relaxed_ordering_supported(u16 dev_cap_ext_flags2)</div><div class='add'>+{</div><div class='add'>+	return dev_cap_ext_flags2 &amp; CREQ_QUERY_FUNC_RESP_SB_MEMORY_REGION_RO_SUPPORTED;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool _is_optimize_modify_qp_supported(u16 dev_cap_ext_flags2)</div><div class='add'>+{</div><div class='add'>+	return dev_cap_ext_flags2 &amp; CREQ_QUERY_FUNC_RESP_SB_OPTIMIZE_MODIFY_QP_SUPPORTED;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool _is_min_rnr_in_rtr_rts_mandatory(u16 dev_cap_ext_flags2)</div><div class='add'>+{</div><div class='add'>+	return !!(dev_cap_ext_flags2 &amp; CREQ_QUERY_FUNC_RESP_SB_MIN_RNR_RTR_RTS_OPT_SUPPORTED);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool _is_cq_coalescing_supported(u16 dev_cap_ext_flags2)</div><div class='add'>+{</div><div class='add'>+	return dev_cap_ext_flags2 &amp; CREQ_QUERY_FUNC_RESP_SB_CQ_COALESCING_SUPPORTED;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool _is_max_srq_ext_supported(u16 dev_cap_ext_flags_2)</div><div class='add'>+{</div><div class='add'>+	return !!(dev_cap_ext_flags_2 &amp; CREQ_QUERY_FUNC_RESP_SB_MAX_SRQ_EXTENDED);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> #endif /* __BNXT_QPLIB_RES_H__ */</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c<br/>index 48793d3512ac..408a34df2667 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_sp.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_sp.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_sp.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/qplib_sp.c</a></div><div class='hunk'>@@ -48,81 +48,107 @@</div><div class='ctx'> #include "qplib_res.h"</div><div class='ctx'> #include "qplib_rcfw.h"</div><div class='ctx'> #include "qplib_sp.h"</div><div class='add'>+#include "qplib_tlv.h"</div><div class='ctx'> </div><div class='ctx'> const struct bnxt_qplib_gid bnxt_qplib_gid_zero = {{ 0, 0, 0, 0, 0, 0, 0, 0,</div><div class='ctx'> 						     0, 0, 0, 0, 0, 0, 0, 0 } };</div><div class='ctx'> </div><div class='ctx'> /* Device */</div><div class='ctx'> </div><div class='del'>-static void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw,</div><div class='del'>-				     char *fw_ver)</div><div class='add'>+static bool bnxt_qplib_is_atomic_cap(struct bnxt_qplib_rcfw *rcfw)</div><div class='ctx'> {</div><div class='del'>-	struct cmdq_query_version req;</div><div class='del'>-	struct creq_query_version_resp resp;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='del'>-	int rc = 0;</div><div class='add'>+	u16 pcie_ctl2 = 0;</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, QUERY_VERSION, cmd_flags);</div><div class='add'>+	if (!bnxt_qplib_is_chip_gen_p5_p7(rcfw-&gt;res-&gt;cctx))</div><div class='add'>+		return false;</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req,</div><div class='del'>-					  (void *)&amp;resp, NULL, 0);</div><div class='add'>+	pcie_capability_read_word(rcfw-&gt;pdev, PCI_EXP_DEVCTL2, &amp;pcie_ctl2);</div><div class='add'>+	return (pcie_ctl2 &amp; PCI_EXP_DEVCTL2_ATOMIC_REQ);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw)</div><div class='add'>+{</div><div class='add'>+	struct creq_query_version_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_query_version req = {};</div><div class='add'>+	struct bnxt_qplib_dev_attr *attr;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	attr = rcfw-&gt;res-&gt;dattr;</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_QUERY_VERSION,</div><div class='add'>+				 sizeof(req));</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req), sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		return;</div><div class='del'>-	fw_ver[0] = resp.fw_maj;</div><div class='del'>-	fw_ver[1] = resp.fw_minor;</div><div class='del'>-	fw_ver[2] = resp.fw_bld;</div><div class='del'>-	fw_ver[3] = resp.fw_rsvd;</div><div class='add'>+	attr-&gt;fw_ver[0] = resp.fw_maj;</div><div class='add'>+	attr-&gt;fw_ver[1] = resp.fw_minor;</div><div class='add'>+	attr-&gt;fw_ver[2] = resp.fw_bld;</div><div class='add'>+	attr-&gt;fw_ver[3] = resp.fw_rsvd;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,</div><div class='del'>-			    struct bnxt_qplib_dev_attr *attr, bool vf)</div><div class='add'>+int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw)</div><div class='ctx'> {</div><div class='del'>-	struct cmdq_query_func req;</div><div class='del'>-	struct creq_query_func_resp resp;</div><div class='del'>-	struct bnxt_qplib_rcfw_sbuf *sbuf;</div><div class='add'>+	struct bnxt_qplib_dev_attr *attr = rcfw-&gt;res-&gt;dattr;</div><div class='add'>+	struct creq_query_func_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='ctx'> 	struct creq_query_func_resp_sb *sb;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='del'>-	u32 temp;</div><div class='add'>+	struct bnxt_qplib_rcfw_sbuf sbuf;</div><div class='add'>+	struct bnxt_qplib_chip_ctx *cctx;</div><div class='add'>+	struct cmdq_query_func req = {};</div><div class='ctx'> 	u8 *tqm_alloc;</div><div class='del'>-	int i, rc = 0;</div><div class='add'>+	int i, rc;</div><div class='add'>+	u32 temp;</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, QUERY_FUNC, cmd_flags);</div><div class='add'>+	cctx = rcfw-&gt;res-&gt;cctx;</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_QUERY_FUNC,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='del'>-	sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));</div><div class='del'>-	if (!sbuf) {</div><div class='del'>-		dev_err(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='del'>-			"SP: QUERY_FUNC alloc side buffer failed\n");</div><div class='add'>+	sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS);</div><div class='add'>+	sbuf.sb = dma_alloc_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf.size,</div><div class='add'>+				     &amp;sbuf.dma_addr, GFP_KERNEL);</div><div class='add'>+	if (!sbuf.sb)</div><div class='ctx'> 		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	sb = sbuf-&gt;sb;</div><div class='del'>-	req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;</div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req, (void *)&amp;resp,</div><div class='del'>-					  (void *)sbuf, 0);</div><div class='add'>+	sb = sbuf.sb;</div><div class='add'>+	req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, &amp;sbuf, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto bail;</div><div class='ctx'> </div><div class='ctx'> 	/* Extract the context from the side buffer */</div><div class='ctx'> 	attr-&gt;max_qp = le32_to_cpu(sb-&gt;max_qp);</div><div class='del'>-	/* max_qp value reported by FW for PF doesn't include the QP1 for PF */</div><div class='del'>-	if (!vf)</div><div class='del'>-		attr-&gt;max_qp += 1;</div><div class='add'>+	/* max_qp value reported by FW doesn't include the QP1 */</div><div class='add'>+	attr-&gt;max_qp += 1;</div><div class='ctx'> 	attr-&gt;max_qp_rd_atom =</div><div class='ctx'> 		sb-&gt;max_qp_rd_atom &gt; BNXT_QPLIB_MAX_OUT_RD_ATOM ?</div><div class='ctx'> 		BNXT_QPLIB_MAX_OUT_RD_ATOM : sb-&gt;max_qp_rd_atom;</div><div class='ctx'> 	attr-&gt;max_qp_init_rd_atom =</div><div class='ctx'> 		sb-&gt;max_qp_init_rd_atom &gt; BNXT_QPLIB_MAX_OUT_RD_ATOM ?</div><div class='ctx'> 		BNXT_QPLIB_MAX_OUT_RD_ATOM : sb-&gt;max_qp_init_rd_atom;</div><div class='del'>-	attr-&gt;max_qp_wqes = le16_to_cpu(sb-&gt;max_qp_wr);</div><div class='del'>-	/*</div><div class='del'>-	 * 128 WQEs needs to be reserved for the HW (8916). Prevent</div><div class='del'>-	 * reporting the max number</div><div class='del'>-	 */</div><div class='del'>-	attr-&gt;max_qp_wqes -= BNXT_QPLIB_RESERVED_QP_WRS;</div><div class='del'>-	attr-&gt;max_qp_sges = bnxt_qplib_is_chip_gen_p5(rcfw-&gt;res-&gt;cctx) ?</div><div class='del'>-			    6 : sb-&gt;max_sge;</div><div class='add'>+	attr-&gt;max_qp_wqes = le16_to_cpu(sb-&gt;max_qp_wr) - 1;</div><div class='add'>+	if (!bnxt_qplib_is_chip_gen_p5_p7(rcfw-&gt;res-&gt;cctx)) {</div><div class='add'>+		/*</div><div class='add'>+		 * 128 WQEs needs to be reserved for the HW (8916). Prevent</div><div class='add'>+		 * reporting the max number on legacy devices</div><div class='add'>+		 */</div><div class='add'>+		attr-&gt;max_qp_wqes -= BNXT_QPLIB_RESERVED_QP_WRS + 1;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Adjust for max_qp_wqes for variable wqe */</div><div class='add'>+	if (cctx-&gt;modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)</div><div class='add'>+		attr-&gt;max_qp_wqes = BNXT_VAR_MAX_WQE - 1;</div><div class='add'>+</div><div class='add'>+	attr-&gt;max_qp_sges = cctx-&gt;modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE ?</div><div class='add'>+			    min_t(u32, sb-&gt;max_sge_var_wqe, BNXT_VAR_MAX_SGE) : 6;</div><div class='ctx'> 	attr-&gt;max_cq = le32_to_cpu(sb-&gt;max_cq);</div><div class='ctx'> 	attr-&gt;max_cq_wqes = le32_to_cpu(sb-&gt;max_cqe);</div><div class='add'>+	if (!bnxt_qplib_is_chip_gen_p7(rcfw-&gt;res-&gt;cctx))</div><div class='add'>+		attr-&gt;max_cq_wqes = min_t(u32, BNXT_QPLIB_MAX_CQ_WQES, attr-&gt;max_cq_wqes);</div><div class='ctx'> 	attr-&gt;max_cq_sges = attr-&gt;max_qp_sges;</div><div class='ctx'> 	attr-&gt;max_mr = le32_to_cpu(sb-&gt;max_mr);</div><div class='ctx'> 	attr-&gt;max_mw = le32_to_cpu(sb-&gt;max_mw);</div><div class='hunk'>@@ -132,29 +158,27 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,</div><div class='ctx'> 	attr-&gt;max_raw_ethy_qp = le32_to_cpu(sb-&gt;max_raw_eth_qp);</div><div class='ctx'> 	attr-&gt;max_ah = le32_to_cpu(sb-&gt;max_ah);</div><div class='ctx'> </div><div class='del'>-	attr-&gt;max_fmr = le32_to_cpu(sb-&gt;max_fmr);</div><div class='del'>-	attr-&gt;max_map_per_fmr = sb-&gt;max_map_per_fmr;</div><div class='del'>-</div><div class='ctx'> 	attr-&gt;max_srq = le16_to_cpu(sb-&gt;max_srq);</div><div class='ctx'> 	attr-&gt;max_srq_wqes = le32_to_cpu(sb-&gt;max_srq_wr) - 1;</div><div class='ctx'> 	attr-&gt;max_srq_sges = sb-&gt;max_srq_sge;</div><div class='del'>-	attr-&gt;max_pkey = le32_to_cpu(sb-&gt;max_pkeys);</div><div class='add'>+	attr-&gt;max_pkey = 1;</div><div class='add'>+	attr-&gt;max_inline_data = attr-&gt;max_qp_sges * sizeof(struct sq_sge);</div><div class='add'>+	if (!bnxt_qplib_is_chip_gen_p7(rcfw-&gt;res-&gt;cctx))</div><div class='add'>+		attr-&gt;l2_db_size = (sb-&gt;l2_db_space_size + 1) *</div><div class='add'>+				    (0x01 &lt;&lt; RCFW_DBR_BASE_PAGE_SHIFT);</div><div class='ctx'> 	/*</div><div class='del'>-	 * Some versions of FW reports more than 0xFFFF.</div><div class='del'>-	 * Restrict it for now to 0xFFFF to avoid</div><div class='del'>-	 * reporting trucated value</div><div class='add'>+	 * Read the max gid supported by HW.</div><div class='add'>+	 * For each entry in HW  GID in HW table, we consume 2</div><div class='add'>+	 * GID entries in the kernel GID table.  So max_gid reported</div><div class='add'>+	 * to stack can be up to twice the value reported by the HW, up to 256 gids.</div><div class='ctx'> 	 */</div><div class='del'>-	if (attr-&gt;max_pkey &gt; 0xFFFF) {</div><div class='del'>-		/* ib_port_attr::pkey_tbl_len is u16 */</div><div class='del'>-		attr-&gt;max_pkey = 0xFFFF;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	attr-&gt;max_inline_data = le32_to_cpu(sb-&gt;max_inline_data);</div><div class='del'>-	attr-&gt;l2_db_size = (sb-&gt;l2_db_space_size + 1) *</div><div class='del'>-			    (0x01 &lt;&lt; RCFW_DBR_BASE_PAGE_SHIFT);</div><div class='ctx'> 	attr-&gt;max_sgid = le32_to_cpu(sb-&gt;max_gid);</div><div class='add'>+	attr-&gt;max_sgid = min_t(u32, BNXT_QPLIB_NUM_GIDS_SUPPORTED, 2 * attr-&gt;max_sgid);</div><div class='add'>+	attr-&gt;dev_cap_flags = le16_to_cpu(sb-&gt;dev_cap_flags);</div><div class='add'>+	attr-&gt;dev_cap_flags2 = le16_to_cpu(sb-&gt;dev_cap_ext_flags_2);</div><div class='ctx'> </div><div class='del'>-	bnxt_qplib_query_version(rcfw, attr-&gt;fw_ver);</div><div class='add'>+	if (_is_max_srq_ext_supported(attr-&gt;dev_cap_flags2))</div><div class='add'>+		attr-&gt;max_srq += le16_to_cpu(sb-&gt;max_srq_ext);</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; MAX_TQM_ALLOC_REQ / 4; i++) {</div><div class='ctx'> 		temp = le32_to_cpu(sb-&gt;tqm_alloc_reqs[i]);</div><div class='hunk'>@@ -165,9 +189,13 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,</div><div class='ctx'> 		attr-&gt;tqm_alloc_reqs[i * 4 + 3] = *(++tqm_alloc);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	attr-&gt;is_atomic = false;</div><div class='add'>+	if (rcfw-&gt;res-&gt;cctx-&gt;hwrm_intf_ver &gt;= HWRM_VERSION_DEV_ATTR_MAX_DPI)</div><div class='add'>+		attr-&gt;max_dpi = le32_to_cpu(sb-&gt;max_dpi);</div><div class='add'>+</div><div class='add'>+	attr-&gt;is_atomic = bnxt_qplib_is_atomic_cap(rcfw);</div><div class='ctx'> bail:</div><div class='del'>-	bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);</div><div class='add'>+	dma_free_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf.size,</div><div class='add'>+			  sbuf.sb, sbuf.dma_addr);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -175,12 +203,14 @@ int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res,</div><div class='ctx'> 				  struct bnxt_qplib_rcfw *rcfw,</div><div class='ctx'> 				  struct bnxt_qplib_ctx *ctx)</div><div class='ctx'> {</div><div class='del'>-	struct cmdq_set_func_resources req;</div><div class='del'>-	struct creq_set_func_resources_resp resp;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='del'>-	int rc = 0;</div><div class='add'>+	struct creq_set_func_resources_resp resp = {};</div><div class='add'>+	struct cmdq_set_func_resources req = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, SET_FUNC_RESOURCES, cmd_flags);</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='ctx'> 	req.number_of_qp = cpu_to_le32(ctx-&gt;qpc_count);</div><div class='ctx'> 	req.number_of_mrw = cpu_to_le32(ctx-&gt;mrw_count);</div><div class='hunk'>@@ -193,9 +223,9 @@ int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res,</div><div class='ctx'> 	req.max_cq_per_vf = cpu_to_le32(ctx-&gt;vf_res.max_cq_per_vf);</div><div class='ctx'> 	req.max_gid_per_vf = cpu_to_le32(ctx-&gt;vf_res.max_gid_per_vf);</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req,</div><div class='del'>-					  (void *)&amp;resp,</div><div class='del'>-					  NULL, 0);</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc) {</div><div class='ctx'> 		dev_err(&amp;res-&gt;pdev-&gt;dev, "Failed to set function resources\n");</div><div class='ctx'> 	}</div><div class='hunk'>@@ -213,12 +243,12 @@ int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res,</div><div class='ctx'> 			index, sgid_tbl-&gt;max);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='del'>-	memcpy(gid, &amp;sgid_tbl-&gt;tbl[index], sizeof(*gid));</div><div class='add'>+	memcpy(gid, &amp;sgid_tbl-&gt;tbl[index].gid, sizeof(*gid));</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,</div><div class='del'>-			struct bnxt_qplib_gid *gid, bool update)</div><div class='add'>+			struct bnxt_qplib_gid *gid, u16 vlan_id, bool update)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl,</div><div class='ctx'> 						   struct bnxt_qplib_res,</div><div class='hunk'>@@ -226,17 +256,14 @@ int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='ctx'> 	int index;</div><div class='ctx'> </div><div class='del'>-	if (!sgid_tbl) {</div><div class='del'>-		dev_err(&amp;res-&gt;pdev-&gt;dev, "SGID table not allocated\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='ctx'> 	/* Do we need a sgid_lock here? */</div><div class='ctx'> 	if (!sgid_tbl-&gt;active) {</div><div class='ctx'> 		dev_err(&amp;res-&gt;pdev-&gt;dev, "SGID table has no active entries\n");</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='ctx'> 	for (index = 0; index &lt; sgid_tbl-&gt;max; index++) {</div><div class='del'>-		if (!memcmp(&amp;sgid_tbl-&gt;tbl[index], gid, sizeof(*gid)))</div><div class='add'>+		if (!memcmp(&amp;sgid_tbl-&gt;tbl[index].gid, gid, sizeof(*gid)) &amp;&amp;</div><div class='add'>+		    vlan_id == sgid_tbl-&gt;tbl[index].vlan_id)</div><div class='ctx'> 			break;</div><div class='ctx'> 	}</div><div class='ctx'> 	if (index == sgid_tbl-&gt;max) {</div><div class='hunk'>@@ -245,25 +272,29 @@ int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,</div><div class='ctx'> 	}</div><div class='ctx'> 	/* Remove GID from the SGID table */</div><div class='ctx'> 	if (update) {</div><div class='del'>-		struct cmdq_delete_gid req;</div><div class='del'>-		struct creq_delete_gid_resp resp;</div><div class='del'>-		u16 cmd_flags = 0;</div><div class='add'>+		struct creq_delete_gid_resp resp = {};</div><div class='add'>+		struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+		struct cmdq_delete_gid req = {};</div><div class='ctx'> 		int rc;</div><div class='ctx'> </div><div class='del'>-		RCFW_CMD_PREP(req, DELETE_GID, cmd_flags);</div><div class='add'>+		bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+					 CMDQ_BASE_OPCODE_DELETE_GID,</div><div class='add'>+					 sizeof(req));</div><div class='ctx'> 		if (sgid_tbl-&gt;hw_id[index] == 0xFFFF) {</div><div class='ctx'> 			dev_err(&amp;res-&gt;pdev-&gt;dev,</div><div class='ctx'> 				"GID entry contains an invalid HW id\n");</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 		}</div><div class='ctx'> 		req.gid_index = cpu_to_le16(sgid_tbl-&gt;hw_id[index]);</div><div class='del'>-		rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req,</div><div class='del'>-						  (void *)&amp;resp, NULL, 0);</div><div class='add'>+		bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),</div><div class='add'>+					sizeof(resp), 0);</div><div class='add'>+		rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 		if (rc)</div><div class='ctx'> 			return rc;</div><div class='ctx'> 	}</div><div class='del'>-	memcpy(&amp;sgid_tbl-&gt;tbl[index], &amp;bnxt_qplib_gid_zero,</div><div class='add'>+	memcpy(&amp;sgid_tbl-&gt;tbl[index].gid, &amp;bnxt_qplib_gid_zero,</div><div class='ctx'> 	       sizeof(bnxt_qplib_gid_zero));</div><div class='add'>+	sgid_tbl-&gt;tbl[index].vlan_id = 0xFFFF;</div><div class='ctx'> 	sgid_tbl-&gt;vlan[index] = 0;</div><div class='ctx'> 	sgid_tbl-&gt;active--;</div><div class='ctx'> 	dev_dbg(&amp;res-&gt;pdev-&gt;dev,</div><div class='hunk'>@@ -276,8 +307,9 @@ int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,</div><div class='del'>-			struct bnxt_qplib_gid *gid, u8 *smac, u16 vlan_id,</div><div class='del'>-			bool update, u32 *index)</div><div class='add'>+			struct bnxt_qplib_gid *gid, const u8 *smac,</div><div class='add'>+			u16 vlan_id, bool update, u32 *index,</div><div class='add'>+			bool is_ugid, u32 stats_ctx_id)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl,</div><div class='ctx'> 						   struct bnxt_qplib_res,</div><div class='hunk'>@@ -285,10 +317,6 @@ int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='ctx'> 	int i, free_idx;</div><div class='ctx'> </div><div class='del'>-	if (!sgid_tbl) {</div><div class='del'>-		dev_err(&amp;res-&gt;pdev-&gt;dev, "SGID table not allocated\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='ctx'> 	/* Do we need a sgid_lock here? */</div><div class='ctx'> 	if (sgid_tbl-&gt;active == sgid_tbl-&gt;max) {</div><div class='ctx'> 		dev_err(&amp;res-&gt;pdev-&gt;dev, "SGID table is full\n");</div><div class='hunk'>@@ -296,7 +324,8 @@ int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,</div><div class='ctx'> 	}</div><div class='ctx'> 	free_idx = sgid_tbl-&gt;max;</div><div class='ctx'> 	for (i = 0; i &lt; sgid_tbl-&gt;max; i++) {</div><div class='del'>-		if (!memcmp(&amp;sgid_tbl-&gt;tbl[i], gid, sizeof(*gid))) {</div><div class='add'>+		if (!memcmp(&amp;sgid_tbl-&gt;tbl[i], gid, sizeof(*gid)) &amp;&amp;</div><div class='add'>+		    sgid_tbl-&gt;tbl[i].vlan_id == vlan_id) {</div><div class='ctx'> 			dev_dbg(&amp;res-&gt;pdev-&gt;dev,</div><div class='ctx'> 				"SGID entry already exist in entry %d!\n", i);</div><div class='ctx'> 			*index = i;</div><div class='hunk'>@@ -313,12 +342,14 @@ int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='ctx'> 	if (update) {</div><div class='del'>-		struct cmdq_add_gid req;</div><div class='del'>-		struct creq_add_gid_resp resp;</div><div class='del'>-		u16 cmd_flags = 0;</div><div class='add'>+		struct creq_add_gid_resp resp = {};</div><div class='add'>+		struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+		struct cmdq_add_gid req = {};</div><div class='ctx'> 		int rc;</div><div class='ctx'> </div><div class='del'>-		RCFW_CMD_PREP(req, ADD_GID, cmd_flags);</div><div class='add'>+		bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+					 CMDQ_BASE_OPCODE_ADD_GID,</div><div class='add'>+					 sizeof(req));</div><div class='ctx'> </div><div class='ctx'> 		req.gid[0] = cpu_to_be32(((u32 *)gid-&gt;data)[3]);</div><div class='ctx'> 		req.gid[1] = cpu_to_be32(((u32 *)gid-&gt;data)[2]);</div><div class='hunk'>@@ -343,14 +374,19 @@ int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,</div><div class='ctx'> 		req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]);</div><div class='ctx'> 		req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]);</div><div class='ctx'> </div><div class='del'>-		rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req,</div><div class='del'>-						  (void *)&amp;resp, NULL, 0);</div><div class='add'>+		req.stats_ctx = cpu_to_le16(CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID |</div><div class='add'>+					    (u16)stats_ctx_id);</div><div class='add'>+</div><div class='add'>+		bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),</div><div class='add'>+					sizeof(resp), 0);</div><div class='add'>+		rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 		if (rc)</div><div class='ctx'> 			return rc;</div><div class='ctx'> 		sgid_tbl-&gt;hw_id[free_idx] = le32_to_cpu(resp.xid);</div><div class='ctx'> 	}</div><div class='ctx'> 	/* Add GID to the sgid_tbl */</div><div class='ctx'> 	memcpy(&amp;sgid_tbl-&gt;tbl[free_idx], gid, sizeof(*gid));</div><div class='add'>+	sgid_tbl-&gt;tbl[free_idx].vlan_id = vlan_id;</div><div class='ctx'> 	sgid_tbl-&gt;active++;</div><div class='ctx'> 	if (vlan_id != 0xFFFF)</div><div class='ctx'> 		sgid_tbl-&gt;vlan[free_idx] = 1;</div><div class='hunk'>@@ -364,143 +400,21 @@ int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,</div><div class='del'>-			   struct bnxt_qplib_gid *gid, u16 gid_idx,</div><div class='del'>-			   u8 *smac)</div><div class='del'>-{</div><div class='del'>-	struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl,</div><div class='del'>-						   struct bnxt_qplib_res,</div><div class='del'>-						   sgid_tbl);</div><div class='del'>-	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct creq_modify_gid_resp resp;</div><div class='del'>-	struct cmdq_modify_gid req;</div><div class='del'>-	int rc;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='del'>-</div><div class='del'>-	RCFW_CMD_PREP(req, MODIFY_GID, cmd_flags);</div><div class='del'>-</div><div class='del'>-	req.gid[0] = cpu_to_be32(((u32 *)gid-&gt;data)[3]);</div><div class='del'>-	req.gid[1] = cpu_to_be32(((u32 *)gid-&gt;data)[2]);</div><div class='del'>-	req.gid[2] = cpu_to_be32(((u32 *)gid-&gt;data)[1]);</div><div class='del'>-	req.gid[3] = cpu_to_be32(((u32 *)gid-&gt;data)[0]);</div><div class='del'>-	if (res-&gt;prio) {</div><div class='del'>-		req.vlan |= cpu_to_le16</div><div class='del'>-			(CMDQ_ADD_GID_VLAN_TPID_TPID_8100 |</div><div class='del'>-			 CMDQ_ADD_GID_VLAN_VLAN_EN);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* MAC in network format */</div><div class='del'>-	req.src_mac[0] = cpu_to_be16(((u16 *)smac)[0]);</div><div class='del'>-	req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]);</div><div class='del'>-	req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]);</div><div class='del'>-</div><div class='del'>-	req.gid_index = cpu_to_le16(gid_idx);</div><div class='del'>-</div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req,</div><div class='del'>-					  (void *)&amp;resp, NULL, 0);</div><div class='del'>-	return rc;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* pkeys */</div><div class='del'>-int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res,</div><div class='del'>-			struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index,</div><div class='del'>-			u16 *pkey)</div><div class='del'>-{</div><div class='del'>-	if (index == 0xFFFF) {</div><div class='del'>-		*pkey = 0xFFFF;</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='del'>-	if (index &gt;= pkey_tbl-&gt;max) {</div><div class='del'>-		dev_err(&amp;res-&gt;pdev-&gt;dev,</div><div class='del'>-			"Index %d exceeded PKEY table max (%d)\n",</div><div class='del'>-			index, pkey_tbl-&gt;max);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-	memcpy(pkey, &amp;pkey_tbl-&gt;tbl[index], sizeof(*pkey));</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int bnxt_qplib_del_pkey(struct bnxt_qplib_res *res,</div><div class='del'>-			struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,</div><div class='del'>-			bool update)</div><div class='del'>-{</div><div class='del'>-	int i, rc = 0;</div><div class='del'>-</div><div class='del'>-	if (!pkey_tbl) {</div><div class='del'>-		dev_err(&amp;res-&gt;pdev-&gt;dev, "PKEY table not allocated\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Do we need a pkey_lock here? */</div><div class='del'>-	if (!pkey_tbl-&gt;active) {</div><div class='del'>-		dev_err(&amp;res-&gt;pdev-&gt;dev, "PKEY table has no active entries\n");</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	for (i = 0; i &lt; pkey_tbl-&gt;max; i++) {</div><div class='del'>-		if (!memcmp(&amp;pkey_tbl-&gt;tbl[i], pkey, sizeof(*pkey)))</div><div class='del'>-			break;</div><div class='del'>-	}</div><div class='del'>-	if (i == pkey_tbl-&gt;max) {</div><div class='del'>-		dev_err(&amp;res-&gt;pdev-&gt;dev,</div><div class='del'>-			"PKEY 0x%04x not found in the pkey table\n", *pkey);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	memset(&amp;pkey_tbl-&gt;tbl[i], 0, sizeof(*pkey));</div><div class='del'>-	pkey_tbl-&gt;active--;</div><div class='del'>-</div><div class='del'>-	/* unlock */</div><div class='del'>-	return rc;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res,</div><div class='del'>-			struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,</div><div class='del'>-			bool update)</div><div class='del'>-{</div><div class='del'>-	int i, free_idx, rc = 0;</div><div class='del'>-</div><div class='del'>-	if (!pkey_tbl) {</div><div class='del'>-		dev_err(&amp;res-&gt;pdev-&gt;dev, "PKEY table not allocated\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Do we need a pkey_lock here? */</div><div class='del'>-	if (pkey_tbl-&gt;active == pkey_tbl-&gt;max) {</div><div class='del'>-		dev_err(&amp;res-&gt;pdev-&gt;dev, "PKEY table is full\n");</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	free_idx = pkey_tbl-&gt;max;</div><div class='del'>-	for (i = 0; i &lt; pkey_tbl-&gt;max; i++) {</div><div class='del'>-		if (!memcmp(&amp;pkey_tbl-&gt;tbl[i], pkey, sizeof(*pkey)))</div><div class='del'>-			return -EALREADY;</div><div class='del'>-		else if (!pkey_tbl-&gt;tbl[i] &amp;&amp; free_idx == pkey_tbl-&gt;max)</div><div class='del'>-			free_idx = i;</div><div class='del'>-	}</div><div class='del'>-	if (free_idx == pkey_tbl-&gt;max) {</div><div class='del'>-		dev_err(&amp;res-&gt;pdev-&gt;dev,</div><div class='del'>-			"PKEY table is FULL but count is not MAX??\n");</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	/* Add PKEY to the pkey_tbl */</div><div class='del'>-	memcpy(&amp;pkey_tbl-&gt;tbl[free_idx], pkey, sizeof(*pkey));</div><div class='del'>-	pkey_tbl-&gt;active++;</div><div class='del'>-</div><div class='del'>-	/* unlock */</div><div class='del'>-	return rc;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> /* AH */</div><div class='ctx'> int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,</div><div class='ctx'> 			 bool block)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct cmdq_create_ah req;</div><div class='del'>-	struct creq_create_ah_resp resp;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='add'>+	struct creq_create_ah_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_create_ah req = {};</div><div class='ctx'> 	u32 temp32[4];</div><div class='ctx'> 	u16 temp16[3];</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, CREATE_AH, cmd_flags);</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_CREATE_AH,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='ctx'> 	memcpy(temp32, ah-&gt;dgid.data, sizeof(struct bnxt_qplib_gid));</div><div class='ctx'> 	req.dgid[0] = cpu_to_le32(temp32[0]);</div><div class='hunk'>@@ -523,8 +437,9 @@ int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,</div><div class='ctx'> 	req.dest_mac[1] = cpu_to_le16(temp16[1]);</div><div class='ctx'> 	req.dest_mac[2] = cpu_to_le16(temp16[2]);</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req, (void *)&amp;resp,</div><div class='del'>-					  NULL, block);</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),</div><div class='add'>+				sizeof(resp), block);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		return rc;</div><div class='ctx'> </div><div class='hunk'>@@ -532,30 +447,35 @@ int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,</div><div class='del'>-			   bool block)</div><div class='add'>+int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,</div><div class='add'>+			  bool block)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct cmdq_destroy_ah req;</div><div class='del'>-	struct creq_destroy_ah_resp resp;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='add'>+	struct creq_destroy_ah_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_destroy_ah req = {};</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='ctx'> 	/* Clean up the AH table in the device */</div><div class='del'>-	RCFW_CMD_PREP(req, DESTROY_AH, cmd_flags);</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_DESTROY_AH,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='ctx'> 	req.ah_cid = cpu_to_le32(ah-&gt;id);</div><div class='ctx'> </div><div class='del'>-	bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req, (void *)&amp;resp, NULL,</div><div class='del'>-				     block);</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),</div><div class='add'>+				sizeof(resp), block);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* MRW */</div><div class='ctx'> int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)</div><div class='ctx'> {</div><div class='add'>+	struct creq_deallocate_key_resp resp = {};</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct cmdq_deallocate_key req;</div><div class='del'>-	struct creq_deallocate_key_resp resp;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='add'>+	struct cmdq_deallocate_key req = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='ctx'> 	if (mrw-&gt;lkey == 0xFFFFFFFF) {</div><div class='hunk'>@@ -563,7 +483,9 @@ int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)</div><div class='ctx'> 		return 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, DEALLOCATE_KEY, cmd_flags);</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_DEALLOCATE_KEY,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='ctx'> 	req.mrw_flags = mrw-&gt;type;</div><div class='ctx'> </div><div class='hunk'>@@ -574,14 +496,15 @@ int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)</div><div class='ctx'> 	else</div><div class='ctx'> 		req.key = cpu_to_le32(mrw-&gt;lkey);</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req, (void *)&amp;resp,</div><div class='del'>-					  NULL, 0);</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		return rc;</div><div class='ctx'> </div><div class='ctx'> 	/* Free the qplib's MRW memory */</div><div class='ctx'> 	if (mrw-&gt;hwq.max_elements)</div><div class='del'>-		bnxt_qplib_free_hwq(res-&gt;pdev, &amp;mrw-&gt;hwq);</div><div class='add'>+		bnxt_qplib_free_hwq(res, &amp;mrw-&gt;hwq);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -589,26 +512,29 @@ int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)</div><div class='ctx'> int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct cmdq_allocate_mrw req;</div><div class='del'>-	struct creq_allocate_mrw_resp resp;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='add'>+	struct creq_allocate_mrw_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_allocate_mrw req = {};</div><div class='ctx'> 	unsigned long tmp;</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, ALLOCATE_MRW, cmd_flags);</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_ALLOCATE_MRW,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='ctx'> 	req.pd_id = cpu_to_le32(mrw-&gt;pd-&gt;id);</div><div class='ctx'> 	req.mrw_flags = mrw-&gt;type;</div><div class='ctx'> 	if ((mrw-&gt;type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR &amp;&amp;</div><div class='del'>-	     mrw-&gt;flags &amp; BNXT_QPLIB_FR_PMR) ||</div><div class='add'>+	     mrw-&gt;access_flags &amp; BNXT_QPLIB_FR_PMR) ||</div><div class='ctx'> 	    mrw-&gt;type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A ||</div><div class='ctx'> 	    mrw-&gt;type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)</div><div class='ctx'> 		req.access = CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY;</div><div class='ctx'> 	tmp = (unsigned long)mrw;</div><div class='ctx'> 	req.mrw_handle = cpu_to_le64(tmp);</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req,</div><div class='del'>-					  (void *)&amp;resp, NULL, 0);</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		return rc;</div><div class='ctx'> </div><div class='hunk'>@@ -625,16 +551,19 @@ int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,</div><div class='ctx'> 			 bool block)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct cmdq_deregister_mr req;</div><div class='del'>-	struct creq_deregister_mr_resp resp;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='add'>+	struct creq_deregister_mr_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_deregister_mr req = {};</div><div class='ctx'> 	int rc;</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, DEREGISTER_MR, cmd_flags);</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_DEREGISTER_MR,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='ctx'> 	req.lkey = cpu_to_le32(mrw-&gt;lkey);</div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req,</div><div class='del'>-					  (void *)&amp;resp, NULL, block);</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),</div><div class='add'>+				sizeof(resp), block);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		return rc;</div><div class='ctx'> </div><div class='hunk'>@@ -642,61 +571,53 @@ int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,</div><div class='ctx'> 	if (mrw-&gt;hwq.max_elements) {</div><div class='ctx'> 		mrw-&gt;va = 0;</div><div class='ctx'> 		mrw-&gt;total_size = 0;</div><div class='del'>-		bnxt_qplib_free_hwq(res-&gt;pdev, &amp;mrw-&gt;hwq);</div><div class='add'>+		bnxt_qplib_free_hwq(res, &amp;mrw-&gt;hwq);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,</div><div class='del'>-		      u64 *pbl_tbl, int num_pbls, bool block, u32 buf_pg_size)</div><div class='add'>+		      struct ib_umem *umem, int num_pbls, u32 buf_pg_size, bool unified_mr)</div><div class='ctx'> {</div><div class='ctx'> 	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct cmdq_register_mr req;</div><div class='del'>-	struct creq_register_mr_resp resp;</div><div class='del'>-	u16 cmd_flags = 0, level;</div><div class='del'>-	int pg_ptrs, pages, i, rc;</div><div class='del'>-	dma_addr_t **pbl_ptr;</div><div class='add'>+	struct bnxt_qplib_hwq_attr hwq_attr = {};</div><div class='add'>+	struct bnxt_qplib_sg_info sginfo = {};</div><div class='add'>+	struct creq_register_mr_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_register_mr req = {};</div><div class='add'>+	int pages, rc;</div><div class='ctx'> 	u32 pg_size;</div><div class='add'>+	u16 level;</div><div class='ctx'> </div><div class='ctx'> 	if (num_pbls) {</div><div class='add'>+		pages = roundup_pow_of_two(num_pbls);</div><div class='ctx'> 		/* Allocate memory for the non-leaf pages to store buf ptrs.</div><div class='ctx'> 		 * Non-leaf pages always uses system PAGE_SIZE</div><div class='ctx'> 		 */</div><div class='del'>-		pg_ptrs = roundup_pow_of_two(num_pbls);</div><div class='del'>-		pages = pg_ptrs &gt;&gt; MAX_PBL_LVL_1_PGS_SHIFT;</div><div class='del'>-		if (!pages)</div><div class='del'>-			pages++;</div><div class='del'>-</div><div class='del'>-		if (pages &gt; MAX_PBL_LVL_1_PGS) {</div><div class='del'>-			dev_err(&amp;res-&gt;pdev-&gt;dev,</div><div class='del'>-				"SP: Reg MR pages requested (0x%x) exceeded max (0x%x)\n",</div><div class='del'>-				pages, MAX_PBL_LVL_1_PGS);</div><div class='del'>-			return -ENOMEM;</div><div class='del'>-		}</div><div class='ctx'> 		/* Free the hwq if it already exist, must be a rereg */</div><div class='ctx'> 		if (mr-&gt;hwq.max_elements)</div><div class='del'>-			bnxt_qplib_free_hwq(res-&gt;pdev, &amp;mr-&gt;hwq);</div><div class='del'>-</div><div class='del'>-		mr-&gt;hwq.max_elements = pages;</div><div class='del'>-		/* Use system PAGE_SIZE */</div><div class='del'>-		rc = bnxt_qplib_alloc_init_hwq(res-&gt;pdev, &amp;mr-&gt;hwq, NULL,</div><div class='del'>-					       &amp;mr-&gt;hwq.max_elements,</div><div class='del'>-					       PAGE_SIZE, 0, PAGE_SIZE,</div><div class='del'>-					       HWQ_TYPE_CTX);</div><div class='add'>+			bnxt_qplib_free_hwq(res, &amp;mr-&gt;hwq);</div><div class='add'>+		hwq_attr.res = res;</div><div class='add'>+		hwq_attr.depth = pages;</div><div class='add'>+		hwq_attr.stride = sizeof(dma_addr_t);</div><div class='add'>+		hwq_attr.type = HWQ_TYPE_MR;</div><div class='add'>+		hwq_attr.sginfo = &amp;sginfo;</div><div class='add'>+		hwq_attr.sginfo-&gt;umem = umem;</div><div class='add'>+		hwq_attr.sginfo-&gt;npages = pages;</div><div class='add'>+		hwq_attr.sginfo-&gt;pgsize = buf_pg_size;</div><div class='add'>+		hwq_attr.sginfo-&gt;pgshft = ilog2(buf_pg_size);</div><div class='add'>+		rc = bnxt_qplib_alloc_init_hwq(&amp;mr-&gt;hwq, &amp;hwq_attr);</div><div class='ctx'> 		if (rc) {</div><div class='ctx'> 			dev_err(&amp;res-&gt;pdev-&gt;dev,</div><div class='ctx'> 				"SP: Reg MR memory allocation failed\n");</div><div class='ctx'> 			return -ENOMEM;</div><div class='ctx'> 		}</div><div class='del'>-		/* Write to the hwq */</div><div class='del'>-		pbl_ptr = (dma_addr_t **)mr-&gt;hwq.pbl_ptr;</div><div class='del'>-		for (i = 0; i &lt; num_pbls; i++)</div><div class='del'>-			pbl_ptr[PTR_PG(i)][PTR_IDX(i)] =</div><div class='del'>-				(pbl_tbl[i] &amp; PAGE_MASK) | PTU_PTE_VALID;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, REGISTER_MR, cmd_flags);</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_REGISTER_MR,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='ctx'> 	/* Configure the request */</div><div class='ctx'> 	if (mr-&gt;hwq.level == PBL_LVL_MAX) {</div><div class='hunk'>@@ -705,7 +626,7 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,</div><div class='ctx'> 		req.pbl = 0;</div><div class='ctx'> 		pg_size = PAGE_SIZE;</div><div class='ctx'> 	} else {</div><div class='del'>-		level = mr-&gt;hwq.level + 1;</div><div class='add'>+		level = mr-&gt;hwq.level;</div><div class='ctx'> 		req.pbl = cpu_to_le64(mr-&gt;hwq.pbl[PBL_LVL_0].pg_map_arr[0]);</div><div class='ctx'> 	}</div><div class='ctx'> 	pg_size = buf_pg_size ? buf_pg_size : PAGE_SIZE;</div><div class='hunk'>@@ -716,21 +637,30 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,</div><div class='ctx'> 	req.log2_pbl_pg_size = cpu_to_le16(((ilog2(PAGE_SIZE) &lt;&lt;</div><div class='ctx'> 				 CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_SFT) &amp;</div><div class='ctx'> 				CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_MASK));</div><div class='del'>-	req.access = (mr-&gt;flags &amp; 0xFFFF);</div><div class='add'>+	req.access = (mr-&gt;access_flags &amp; BNXT_QPLIB_MR_ACCESS_MASK);</div><div class='ctx'> 	req.va = cpu_to_le64(mr-&gt;va);</div><div class='ctx'> 	req.key = cpu_to_le32(mr-&gt;lkey);</div><div class='add'>+	if (unified_mr)</div><div class='add'>+		req.key = cpu_to_le32(mr-&gt;pd-&gt;id);</div><div class='add'>+	req.flags = cpu_to_le16(mr-&gt;flags);</div><div class='ctx'> 	req.mr_size = cpu_to_le64(mr-&gt;total_size);</div><div class='ctx'> </div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req,</div><div class='del'>-					  (void *)&amp;resp, NULL, block);</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto fail;</div><div class='ctx'> </div><div class='add'>+	if (unified_mr) {</div><div class='add'>+		mr-&gt;lkey = le32_to_cpu(resp.xid);</div><div class='add'>+		mr-&gt;rkey = mr-&gt;lkey;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> fail:</div><div class='ctx'> 	if (mr-&gt;hwq.max_elements)</div><div class='del'>-		bnxt_qplib_free_hwq(res-&gt;pdev, &amp;mr-&gt;hwq);</div><div class='add'>+		bnxt_qplib_free_hwq(res, &amp;mr-&gt;hwq);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -738,6 +668,8 @@ int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,</div><div class='ctx'> 					struct bnxt_qplib_frpl *frpl,</div><div class='ctx'> 					int max_pg_ptrs)</div><div class='ctx'> {</div><div class='add'>+	struct bnxt_qplib_hwq_attr hwq_attr = {};</div><div class='add'>+	struct bnxt_qplib_sg_info sginfo = {};</div><div class='ctx'> 	int pg_ptrs, pages, rc;</div><div class='ctx'> </div><div class='ctx'> 	/* Re-calculate the max to fit the HWQ allocation model */</div><div class='hunk'>@@ -749,10 +681,15 @@ int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,</div><div class='ctx'> 	if (pages &gt; MAX_PBL_LVL_1_PGS)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	frpl-&gt;hwq.max_elements = pages;</div><div class='del'>-	rc = bnxt_qplib_alloc_init_hwq(res-&gt;pdev, &amp;frpl-&gt;hwq, NULL,</div><div class='del'>-				       &amp;frpl-&gt;hwq.max_elements, PAGE_SIZE, 0,</div><div class='del'>-				       PAGE_SIZE, HWQ_TYPE_CTX);</div><div class='add'>+	sginfo.pgsize = PAGE_SIZE;</div><div class='add'>+	sginfo.nopte = true;</div><div class='add'>+</div><div class='add'>+	hwq_attr.res = res;</div><div class='add'>+	hwq_attr.depth = pg_ptrs;</div><div class='add'>+	hwq_attr.stride = PAGE_SIZE;</div><div class='add'>+	hwq_attr.sginfo = &amp;sginfo;</div><div class='add'>+	hwq_attr.type = HWQ_TYPE_CTX;</div><div class='add'>+	rc = bnxt_qplib_alloc_init_hwq(&amp;frpl-&gt;hwq, &amp;hwq_attr);</div><div class='ctx'> 	if (!rc)</div><div class='ctx'> 		frpl-&gt;max_pg_ptrs = pg_ptrs;</div><div class='ctx'> </div><div class='hunk'>@@ -762,48 +699,35 @@ int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,</div><div class='ctx'> int bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res,</div><div class='ctx'> 				       struct bnxt_qplib_frpl *frpl)</div><div class='ctx'> {</div><div class='del'>-	bnxt_qplib_free_hwq(res-&gt;pdev, &amp;frpl-&gt;hwq);</div><div class='add'>+	bnxt_qplib_free_hwq(res, &amp;frpl-&gt;hwq);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids)</div><div class='del'>-{</div><div class='del'>-	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='del'>-	struct cmdq_map_tc_to_cos req;</div><div class='del'>-	struct creq_map_tc_to_cos_resp resp;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='del'>-</div><div class='del'>-	RCFW_CMD_PREP(req, MAP_TC_TO_COS, cmd_flags);</div><div class='del'>-	req.cos0 = cpu_to_le16(cids[0]);</div><div class='del'>-	req.cos1 = cpu_to_le16(cids[1]);</div><div class='del'>-</div><div class='del'>-	return bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req, (void *)&amp;resp,</div><div class='del'>-						NULL, 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> int bnxt_qplib_get_roce_stats(struct bnxt_qplib_rcfw *rcfw,</div><div class='ctx'> 			      struct bnxt_qplib_roce_stats *stats)</div><div class='ctx'> {</div><div class='del'>-	struct cmdq_query_roce_stats req;</div><div class='del'>-	struct creq_query_roce_stats_resp resp;</div><div class='del'>-	struct bnxt_qplib_rcfw_sbuf *sbuf;</div><div class='add'>+	struct creq_query_roce_stats_resp resp = {};</div><div class='ctx'> 	struct creq_query_roce_stats_resp_sb *sb;</div><div class='del'>-	u16 cmd_flags = 0;</div><div class='del'>-	int rc = 0;</div><div class='add'>+	struct cmdq_query_roce_stats req = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct bnxt_qplib_rcfw_sbuf sbuf;</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='del'>-	RCFW_CMD_PREP(req, QUERY_ROCE_STATS, cmd_flags);</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_QUERY_ROCE_STATS,</div><div class='add'>+				 sizeof(req));</div><div class='ctx'> </div><div class='del'>-	sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));</div><div class='del'>-	if (!sbuf) {</div><div class='del'>-		dev_err(&amp;rcfw-&gt;pdev-&gt;dev,</div><div class='del'>-			"SP: QUERY_ROCE_STATS alloc side buffer failed\n");</div><div class='add'>+	sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS);</div><div class='add'>+	sbuf.sb = dma_alloc_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf.size,</div><div class='add'>+				     &amp;sbuf.dma_addr, GFP_KERNEL);</div><div class='add'>+	if (!sbuf.sb)</div><div class='ctx'> 		return -ENOMEM;</div><div class='del'>-	}</div><div class='add'>+	sb = sbuf.sb;</div><div class='ctx'> </div><div class='del'>-	sb = sbuf-&gt;sb;</div><div class='del'>-	req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;</div><div class='del'>-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&amp;req, (void *)&amp;resp,</div><div class='del'>-					  (void *)sbuf, 0);</div><div class='add'>+	req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, &amp;sbuf, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='ctx'> 	if (rc)</div><div class='ctx'> 		goto bail;</div><div class='ctx'> 	/* Extract the context from the side buffer */</div><div class='hunk'>@@ -857,6 +781,365 @@ int bnxt_qplib_get_roce_stats(struct bnxt_qplib_rcfw *rcfw,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> bail:</div><div class='del'>-	bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);</div><div class='add'>+	dma_free_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf.size,</div><div class='add'>+			  sbuf.sb, sbuf.dma_addr);</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='add'>+</div><div class='add'>+int bnxt_qplib_qext_stat(struct bnxt_qplib_rcfw *rcfw, u32 fid,</div><div class='add'>+			 struct bnxt_qplib_ext_stat *estat)</div><div class='add'>+{</div><div class='add'>+	struct creq_query_roce_stats_ext_resp resp = {};</div><div class='add'>+	struct creq_query_roce_stats_ext_resp_sb *sb;</div><div class='add'>+	struct cmdq_query_roce_stats_ext req = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct bnxt_qplib_rcfw_sbuf sbuf;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS);</div><div class='add'>+	sbuf.sb = dma_alloc_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf.size,</div><div class='add'>+				     &amp;sbuf.dma_addr, GFP_KERNEL);</div><div class='add'>+	if (!sbuf.sb)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	sb = sbuf.sb;</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_QUERY_ROCE_STATS,</div><div class='add'>+				 sizeof(req));</div><div class='add'>+</div><div class='add'>+	req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;</div><div class='add'>+	req.resp_addr = cpu_to_le64(sbuf.dma_addr);</div><div class='add'>+	if (bnxt_qplib_is_chip_gen_p7(rcfw-&gt;res-&gt;cctx) &amp;&amp; rcfw-&gt;res-&gt;is_vf)</div><div class='add'>+		req.function_id =</div><div class='add'>+			cpu_to_le32(CMDQ_QUERY_ROCE_STATS_EXT_VF_VALID |</div><div class='add'>+				    (fid &lt;&lt; CMDQ_QUERY_ROCE_STATS_EXT_VF_NUM_SFT));</div><div class='add'>+	else</div><div class='add'>+		req.function_id = cpu_to_le32(fid);</div><div class='add'>+	req.flags = cpu_to_le16(CMDQ_QUERY_ROCE_STATS_EXT_FLAGS_FUNCTION_ID);</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, &amp;sbuf, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto bail;</div><div class='add'>+</div><div class='add'>+	estat-&gt;tx_atomic_req = le64_to_cpu(sb-&gt;tx_atomic_req_pkts);</div><div class='add'>+	estat-&gt;tx_read_req = le64_to_cpu(sb-&gt;tx_read_req_pkts);</div><div class='add'>+	estat-&gt;tx_read_res = le64_to_cpu(sb-&gt;tx_read_res_pkts);</div><div class='add'>+	estat-&gt;tx_write_req = le64_to_cpu(sb-&gt;tx_write_req_pkts);</div><div class='add'>+	estat-&gt;tx_send_req = le64_to_cpu(sb-&gt;tx_send_req_pkts);</div><div class='add'>+	estat-&gt;tx_roce_pkts = le64_to_cpu(sb-&gt;tx_roce_pkts);</div><div class='add'>+	estat-&gt;tx_roce_bytes = le64_to_cpu(sb-&gt;tx_roce_bytes);</div><div class='add'>+	estat-&gt;rx_atomic_req = le64_to_cpu(sb-&gt;rx_atomic_req_pkts);</div><div class='add'>+	estat-&gt;rx_read_req = le64_to_cpu(sb-&gt;rx_read_req_pkts);</div><div class='add'>+	estat-&gt;rx_read_res = le64_to_cpu(sb-&gt;rx_read_res_pkts);</div><div class='add'>+	estat-&gt;rx_write_req = le64_to_cpu(sb-&gt;rx_write_req_pkts);</div><div class='add'>+	estat-&gt;rx_send_req = le64_to_cpu(sb-&gt;rx_send_req_pkts);</div><div class='add'>+	estat-&gt;rx_roce_pkts = le64_to_cpu(sb-&gt;rx_roce_pkts);</div><div class='add'>+	estat-&gt;rx_roce_bytes = le64_to_cpu(sb-&gt;rx_roce_bytes);</div><div class='add'>+	estat-&gt;rx_roce_good_pkts = le64_to_cpu(sb-&gt;rx_roce_good_pkts);</div><div class='add'>+	estat-&gt;rx_roce_good_bytes = le64_to_cpu(sb-&gt;rx_roce_good_bytes);</div><div class='add'>+	estat-&gt;rx_out_of_buffer = le64_to_cpu(sb-&gt;rx_out_of_buffer_pkts);</div><div class='add'>+	estat-&gt;rx_out_of_sequence = le64_to_cpu(sb-&gt;rx_out_of_sequence_pkts);</div><div class='add'>+	estat-&gt;tx_cnp = le64_to_cpu(sb-&gt;tx_cnp_pkts);</div><div class='add'>+	estat-&gt;rx_cnp = le64_to_cpu(sb-&gt;rx_cnp_pkts);</div><div class='add'>+	estat-&gt;rx_ecn_marked = le64_to_cpu(sb-&gt;rx_ecn_marked_pkts);</div><div class='add'>+</div><div class='add'>+bail:</div><div class='add'>+	dma_free_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf.size,</div><div class='add'>+			  sbuf.sb, sbuf.dma_addr);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_qplib_fill_cc_gen1(struct cmdq_modify_roce_cc_gen1_tlv *ext_req,</div><div class='add'>+				    struct bnxt_qplib_cc_param_ext *cc_ext)</div><div class='add'>+{</div><div class='add'>+	ext_req-&gt;modify_mask = cpu_to_le64(cc_ext-&gt;ext_mask);</div><div class='add'>+	cc_ext-&gt;ext_mask = 0;</div><div class='add'>+	ext_req-&gt;inactivity_th_hi = cpu_to_le16(cc_ext-&gt;inact_th_hi);</div><div class='add'>+	ext_req-&gt;min_time_between_cnps = cpu_to_le16(cc_ext-&gt;min_delta_cnp);</div><div class='add'>+	ext_req-&gt;init_cp = cpu_to_le16(cc_ext-&gt;init_cp);</div><div class='add'>+	ext_req-&gt;tr_update_mode = cc_ext-&gt;tr_update_mode;</div><div class='add'>+	ext_req-&gt;tr_update_cycles = cc_ext-&gt;tr_update_cyls;</div><div class='add'>+	ext_req-&gt;fr_num_rtts = cc_ext-&gt;fr_rtt;</div><div class='add'>+	ext_req-&gt;ai_rate_increase = cc_ext-&gt;ai_rate_incr;</div><div class='add'>+	ext_req-&gt;reduction_relax_rtts_th = cpu_to_le16(cc_ext-&gt;rr_rtt_th);</div><div class='add'>+	ext_req-&gt;additional_relax_cr_th = cpu_to_le16(cc_ext-&gt;ar_cr_th);</div><div class='add'>+	ext_req-&gt;cr_min_th = cpu_to_le16(cc_ext-&gt;cr_min_th);</div><div class='add'>+	ext_req-&gt;bw_avg_weight = cc_ext-&gt;bw_avg_weight;</div><div class='add'>+	ext_req-&gt;actual_cr_factor = cc_ext-&gt;cr_factor;</div><div class='add'>+	ext_req-&gt;max_cp_cr_th = cpu_to_le16(cc_ext-&gt;cr_th_max_cp);</div><div class='add'>+	ext_req-&gt;cp_bias_en = cc_ext-&gt;cp_bias_en;</div><div class='add'>+	ext_req-&gt;cp_bias = cc_ext-&gt;cp_bias;</div><div class='add'>+	ext_req-&gt;cnp_ecn = cc_ext-&gt;cnp_ecn;</div><div class='add'>+	ext_req-&gt;rtt_jitter_en = cc_ext-&gt;rtt_jitter_en;</div><div class='add'>+	ext_req-&gt;link_bytes_per_usec = cpu_to_le16(cc_ext-&gt;bytes_per_usec);</div><div class='add'>+	ext_req-&gt;reset_cc_cr_th = cpu_to_le16(cc_ext-&gt;cc_cr_reset_th);</div><div class='add'>+	ext_req-&gt;cr_width = cc_ext-&gt;cr_width;</div><div class='add'>+	ext_req-&gt;quota_period_min = cc_ext-&gt;min_quota;</div><div class='add'>+	ext_req-&gt;quota_period_max = cc_ext-&gt;max_quota;</div><div class='add'>+	ext_req-&gt;quota_period_abs_max = cc_ext-&gt;abs_max_quota;</div><div class='add'>+	ext_req-&gt;tr_lower_bound = cpu_to_le16(cc_ext-&gt;tr_lb);</div><div class='add'>+	ext_req-&gt;cr_prob_factor = cc_ext-&gt;cr_prob_fac;</div><div class='add'>+	ext_req-&gt;tr_prob_factor = cc_ext-&gt;tr_prob_fac;</div><div class='add'>+	ext_req-&gt;fairness_cr_th = cpu_to_le16(cc_ext-&gt;fair_cr_th);</div><div class='add'>+	ext_req-&gt;red_div = cc_ext-&gt;red_div;</div><div class='add'>+	ext_req-&gt;cnp_ratio_th = cc_ext-&gt;cnp_ratio_th;</div><div class='add'>+	ext_req-&gt;exp_ai_rtts = cpu_to_le16(cc_ext-&gt;ai_ext_rtt);</div><div class='add'>+	ext_req-&gt;exp_ai_cr_cp_ratio = cc_ext-&gt;exp_crcp_ratio;</div><div class='add'>+	ext_req-&gt;use_rate_table = cc_ext-&gt;low_rate_en;</div><div class='add'>+	ext_req-&gt;cp_exp_update_th = cpu_to_le16(cc_ext-&gt;cpcr_update_th);</div><div class='add'>+	ext_req-&gt;high_exp_ai_rtts_th1 = cpu_to_le16(cc_ext-&gt;ai_rtt_th1);</div><div class='add'>+	ext_req-&gt;high_exp_ai_rtts_th2 = cpu_to_le16(cc_ext-&gt;ai_rtt_th2);</div><div class='add'>+	ext_req-&gt;actual_cr_cong_free_rtts_th = cpu_to_le16(cc_ext-&gt;cf_rtt_th);</div><div class='add'>+	ext_req-&gt;severe_cong_cr_th1 = cpu_to_le16(cc_ext-&gt;sc_cr_th1);</div><div class='add'>+	ext_req-&gt;severe_cong_cr_th2 = cpu_to_le16(cc_ext-&gt;sc_cr_th2);</div><div class='add'>+	ext_req-&gt;link64B_per_rtt = cpu_to_le32(cc_ext-&gt;l64B_per_rtt);</div><div class='add'>+	ext_req-&gt;cc_ack_bytes = cc_ext-&gt;cc_ack_bytes;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bnxt_qplib_modify_cc(struct bnxt_qplib_res *res,</div><div class='add'>+			 struct bnxt_qplib_cc_param *cc_param)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_tlv_modify_cc_req tlv_req = {};</div><div class='add'>+	struct creq_modify_roce_cc_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_modify_roce_cc *req;</div><div class='add'>+	int req_size;</div><div class='add'>+	void *cmd;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	/* Prepare the older base command */</div><div class='add'>+	req = &amp;tlv_req.base_req;</div><div class='add'>+	cmd = req;</div><div class='add'>+	req_size = sizeof(*req);</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)req, CMDQ_BASE_OPCODE_MODIFY_ROCE_CC,</div><div class='add'>+				 sizeof(*req));</div><div class='add'>+	req-&gt;modify_mask = cpu_to_le32(cc_param-&gt;mask);</div><div class='add'>+	req-&gt;enable_cc = cc_param-&gt;enable;</div><div class='add'>+	req-&gt;g = cc_param-&gt;g;</div><div class='add'>+	req-&gt;num_phases_per_state = cc_param-&gt;nph_per_state;</div><div class='add'>+	req-&gt;time_per_phase = cc_param-&gt;time_pph;</div><div class='add'>+	req-&gt;pkts_per_phase = cc_param-&gt;pkts_pph;</div><div class='add'>+	req-&gt;init_cr = cpu_to_le16(cc_param-&gt;init_cr);</div><div class='add'>+	req-&gt;init_tr = cpu_to_le16(cc_param-&gt;init_tr);</div><div class='add'>+	req-&gt;tos_dscp_tos_ecn = (cc_param-&gt;tos_dscp &lt;&lt; CMDQ_MODIFY_ROCE_CC_TOS_DSCP_SFT) |</div><div class='add'>+				(cc_param-&gt;tos_ecn &amp; CMDQ_MODIFY_ROCE_CC_TOS_ECN_MASK);</div><div class='add'>+	req-&gt;alt_vlan_pcp = cc_param-&gt;alt_vlan_pcp;</div><div class='add'>+	req-&gt;alt_tos_dscp = cpu_to_le16(cc_param-&gt;alt_tos_dscp);</div><div class='add'>+	req-&gt;rtt = cpu_to_le16(cc_param-&gt;rtt);</div><div class='add'>+	req-&gt;tcp_cp = cpu_to_le16(cc_param-&gt;tcp_cp);</div><div class='add'>+	req-&gt;cc_mode = cc_param-&gt;cc_mode;</div><div class='add'>+	req-&gt;inactivity_th = cpu_to_le16(cc_param-&gt;inact_th);</div><div class='add'>+</div><div class='add'>+	/* For chip gen P5 onwards fill extended cmd and header */</div><div class='add'>+	if (bnxt_qplib_is_chip_gen_p5_p7(res-&gt;cctx)) {</div><div class='add'>+		struct roce_tlv *hdr;</div><div class='add'>+		u32 payload;</div><div class='add'>+		u32 chunks;</div><div class='add'>+</div><div class='add'>+		cmd = &amp;tlv_req;</div><div class='add'>+		req_size = sizeof(tlv_req);</div><div class='add'>+		/* Prepare primary tlv header */</div><div class='add'>+		hdr = &amp;tlv_req.tlv_hdr;</div><div class='add'>+		chunks = CHUNKS(sizeof(struct bnxt_qplib_tlv_modify_cc_req));</div><div class='add'>+		payload = sizeof(struct cmdq_modify_roce_cc);</div><div class='add'>+		__roce_1st_tlv_prep(hdr, chunks, payload, true);</div><div class='add'>+		/* Prepare secondary tlv header */</div><div class='add'>+		hdr = (struct roce_tlv *)&amp;tlv_req.ext_req;</div><div class='add'>+		payload = sizeof(struct cmdq_modify_roce_cc_gen1_tlv) -</div><div class='add'>+			  sizeof(struct roce_tlv);</div><div class='add'>+		__roce_ext_tlv_prep(hdr, TLV_TYPE_MODIFY_ROCE_CC_GEN1, payload, false, true);</div><div class='add'>+		bnxt_qplib_fill_cc_gen1(&amp;tlv_req.ext_req, &amp;cc_param-&gt;cc_ext);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, cmd, &amp;resp, NULL, req_size,</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(res-&gt;rcfw, &amp;msg);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bnxt_qplib_read_context(struct bnxt_qplib_rcfw *rcfw, u8 res_type,</div><div class='add'>+			    u32 xid, u32 resp_size, void *resp_va)</div><div class='add'>+{</div><div class='add'>+	struct creq_read_context resp = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_read_context req = {};</div><div class='add'>+	struct bnxt_qplib_rcfw_sbuf sbuf;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	sbuf.size = resp_size;</div><div class='add'>+	sbuf.sb = dma_alloc_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf.size,</div><div class='add'>+				     &amp;sbuf.dma_addr, GFP_KERNEL);</div><div class='add'>+	if (!sbuf.sb)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_READ_CONTEXT, sizeof(req));</div><div class='add'>+	req.resp_addr = cpu_to_le64(sbuf.dma_addr);</div><div class='add'>+	req.resp_size = resp_size / BNXT_QPLIB_CMDQE_UNITS;</div><div class='add'>+</div><div class='add'>+	req.xid = cpu_to_le32(xid);</div><div class='add'>+	req.type = res_type;</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, &amp;sbuf, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto free_mem;</div><div class='add'>+</div><div class='add'>+	memcpy(resp_va, sbuf.sb, resp_size);</div><div class='add'>+free_mem:</div><div class='add'>+	dma_free_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf.size, sbuf.sb, sbuf.dma_addr);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void bnxt_qplib_read_cc_gen1(struct bnxt_qplib_cc_param_ext *cc_ext,</div><div class='add'>+				    struct creq_query_roce_cc_gen1_resp_sb_tlv *sb)</div><div class='add'>+{</div><div class='add'>+	cc_ext-&gt;inact_th_hi = le16_to_cpu(sb-&gt;inactivity_th_hi);</div><div class='add'>+	cc_ext-&gt;min_delta_cnp = le16_to_cpu(sb-&gt;min_time_between_cnps);</div><div class='add'>+	cc_ext-&gt;init_cp = le16_to_cpu(sb-&gt;init_cp);</div><div class='add'>+	cc_ext-&gt;tr_update_mode = sb-&gt;tr_update_mode;</div><div class='add'>+	cc_ext-&gt;tr_update_cyls = sb-&gt;tr_update_cycles;</div><div class='add'>+	cc_ext-&gt;fr_rtt = sb-&gt;fr_num_rtts;</div><div class='add'>+	cc_ext-&gt;ai_rate_incr = sb-&gt;ai_rate_increase;</div><div class='add'>+	cc_ext-&gt;rr_rtt_th = le16_to_cpu(sb-&gt;reduction_relax_rtts_th);</div><div class='add'>+	cc_ext-&gt;ar_cr_th = le16_to_cpu(sb-&gt;additional_relax_cr_th);</div><div class='add'>+	cc_ext-&gt;cr_min_th = le16_to_cpu(sb-&gt;cr_min_th);</div><div class='add'>+	cc_ext-&gt;bw_avg_weight = sb-&gt;bw_avg_weight;</div><div class='add'>+	cc_ext-&gt;cr_factor = sb-&gt;actual_cr_factor;</div><div class='add'>+	cc_ext-&gt;cr_th_max_cp = le16_to_cpu(sb-&gt;max_cp_cr_th);</div><div class='add'>+	cc_ext-&gt;cp_bias_en = sb-&gt;cp_bias_en;</div><div class='add'>+	cc_ext-&gt;cp_bias = sb-&gt;cp_bias;</div><div class='add'>+	cc_ext-&gt;cnp_ecn = sb-&gt;cnp_ecn;</div><div class='add'>+	cc_ext-&gt;rtt_jitter_en = sb-&gt;rtt_jitter_en;</div><div class='add'>+	cc_ext-&gt;bytes_per_usec = le16_to_cpu(sb-&gt;link_bytes_per_usec);</div><div class='add'>+	cc_ext-&gt;cc_cr_reset_th = le16_to_cpu(sb-&gt;reset_cc_cr_th);</div><div class='add'>+	cc_ext-&gt;cr_width = sb-&gt;cr_width;</div><div class='add'>+	cc_ext-&gt;min_quota = sb-&gt;quota_period_min;</div><div class='add'>+	cc_ext-&gt;max_quota = sb-&gt;quota_period_max;</div><div class='add'>+	cc_ext-&gt;abs_max_quota = sb-&gt;quota_period_abs_max;</div><div class='add'>+	cc_ext-&gt;tr_lb = le16_to_cpu(sb-&gt;tr_lower_bound);</div><div class='add'>+	cc_ext-&gt;cr_prob_fac = sb-&gt;cr_prob_factor;</div><div class='add'>+	cc_ext-&gt;tr_prob_fac = sb-&gt;tr_prob_factor;</div><div class='add'>+	cc_ext-&gt;fair_cr_th = le16_to_cpu(sb-&gt;fairness_cr_th);</div><div class='add'>+	cc_ext-&gt;red_div = sb-&gt;red_div;</div><div class='add'>+	cc_ext-&gt;cnp_ratio_th = sb-&gt;cnp_ratio_th;</div><div class='add'>+	cc_ext-&gt;ai_ext_rtt = le16_to_cpu(sb-&gt;exp_ai_rtts);</div><div class='add'>+	cc_ext-&gt;exp_crcp_ratio = sb-&gt;exp_ai_cr_cp_ratio;</div><div class='add'>+	cc_ext-&gt;low_rate_en = sb-&gt;use_rate_table;</div><div class='add'>+	cc_ext-&gt;cpcr_update_th = le16_to_cpu(sb-&gt;cp_exp_update_th);</div><div class='add'>+	cc_ext-&gt;ai_rtt_th1 = le16_to_cpu(sb-&gt;high_exp_ai_rtts_th1);</div><div class='add'>+	cc_ext-&gt;ai_rtt_th2 = le16_to_cpu(sb-&gt;high_exp_ai_rtts_th2);</div><div class='add'>+	cc_ext-&gt;cf_rtt_th = le16_to_cpu(sb-&gt;actual_cr_cong_free_rtts_th);</div><div class='add'>+	cc_ext-&gt;sc_cr_th1 = le16_to_cpu(sb-&gt;severe_cong_cr_th1);</div><div class='add'>+	cc_ext-&gt;sc_cr_th2 = le16_to_cpu(sb-&gt;severe_cong_cr_th2);</div><div class='add'>+	cc_ext-&gt;l64B_per_rtt = le32_to_cpu(sb-&gt;link64B_per_rtt);</div><div class='add'>+	cc_ext-&gt;cc_ack_bytes = sb-&gt;cc_ack_bytes;</div><div class='add'>+	cc_ext-&gt;reduce_cf_rtt_th = le16_to_cpu(sb-&gt;reduce_init_cong_free_rtts_th);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bnxt_qplib_query_cc_param(struct bnxt_qplib_res *res,</div><div class='add'>+			      struct bnxt_qplib_cc_param *cc_param)</div><div class='add'>+{</div><div class='add'>+	struct bnxt_qplib_tlv_query_rcc_sb *ext_sb;</div><div class='add'>+	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='add'>+	struct creq_query_roce_cc_resp resp = {};</div><div class='add'>+	struct creq_query_roce_cc_resp_sb *sb;</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+	struct cmdq_query_roce_cc req = {};</div><div class='add'>+	struct bnxt_qplib_rcfw_sbuf sbuf;</div><div class='add'>+	size_t resp_size;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	/* Query the parameters from chip */</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req, CMDQ_BASE_OPCODE_QUERY_ROCE_CC,</div><div class='add'>+				 sizeof(req));</div><div class='add'>+	if (bnxt_qplib_is_chip_gen_p5_p7(res-&gt;cctx))</div><div class='add'>+		resp_size = sizeof(*ext_sb);</div><div class='add'>+	else</div><div class='add'>+		resp_size = sizeof(*sb);</div><div class='add'>+</div><div class='add'>+	sbuf.size = ALIGN(resp_size, BNXT_QPLIB_CMDQE_UNITS);</div><div class='add'>+	sbuf.sb = dma_alloc_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf.size,</div><div class='add'>+				     &amp;sbuf.dma_addr, GFP_KERNEL);</div><div class='add'>+	if (!sbuf.sb)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, &amp;sbuf, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	rc = bnxt_qplib_rcfw_send_message(res-&gt;rcfw, &amp;msg);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	ext_sb = sbuf.sb;</div><div class='add'>+	sb = bnxt_qplib_is_chip_gen_p5_p7(res-&gt;cctx) ? &amp;ext_sb-&gt;base_sb :</div><div class='add'>+		(struct creq_query_roce_cc_resp_sb *)ext_sb;</div><div class='add'>+</div><div class='add'>+	cc_param-&gt;enable = sb-&gt;enable_cc &amp; CREQ_QUERY_ROCE_CC_RESP_SB_ENABLE_CC;</div><div class='add'>+	cc_param-&gt;tos_ecn = (sb-&gt;tos_dscp_tos_ecn &amp;</div><div class='add'>+			     CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_MASK) &gt;&gt;</div><div class='add'>+			    CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_SFT;</div><div class='add'>+	cc_param-&gt;tos_dscp = (sb-&gt;tos_dscp_tos_ecn &amp;</div><div class='add'>+			      CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_MASK) &gt;&gt;</div><div class='add'>+			     CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_SFT;</div><div class='add'>+	cc_param-&gt;alt_tos_dscp = sb-&gt;alt_tos_dscp;</div><div class='add'>+	cc_param-&gt;alt_vlan_pcp = sb-&gt;alt_vlan_pcp;</div><div class='add'>+</div><div class='add'>+	cc_param-&gt;g = sb-&gt;g;</div><div class='add'>+	cc_param-&gt;nph_per_state = sb-&gt;num_phases_per_state;</div><div class='add'>+	cc_param-&gt;init_cr = le16_to_cpu(sb-&gt;init_cr);</div><div class='add'>+	cc_param-&gt;init_tr = le16_to_cpu(sb-&gt;init_tr);</div><div class='add'>+	cc_param-&gt;cc_mode = sb-&gt;cc_mode;</div><div class='add'>+	cc_param-&gt;inact_th = le16_to_cpu(sb-&gt;inactivity_th);</div><div class='add'>+	cc_param-&gt;rtt = le16_to_cpu(sb-&gt;rtt);</div><div class='add'>+	cc_param-&gt;tcp_cp = le16_to_cpu(sb-&gt;tcp_cp);</div><div class='add'>+	cc_param-&gt;time_pph = sb-&gt;time_per_phase;</div><div class='add'>+	cc_param-&gt;pkts_pph = sb-&gt;pkts_per_phase;</div><div class='add'>+	if (bnxt_qplib_is_chip_gen_p5_p7(res-&gt;cctx)) {</div><div class='add'>+		bnxt_qplib_read_cc_gen1(&amp;cc_param-&gt;cc_ext, &amp;ext_sb-&gt;gen1_sb);</div><div class='add'>+		cc_param-&gt;inact_th |= (cc_param-&gt;cc_ext.inact_th_hi &amp; 0x3F) &lt;&lt; 16;</div><div class='add'>+	}</div><div class='add'>+out:</div><div class='add'>+	dma_free_coherent(&amp;rcfw-&gt;pdev-&gt;dev, sbuf.size, sbuf.sb, sbuf.dma_addr);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bnxt_qplib_create_flow(struct bnxt_qplib_res *res)</div><div class='add'>+{</div><div class='add'>+	struct creq_roce_mirror_cfg_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='add'>+	struct cmdq_roce_mirror_cfg req = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_ROCE_MIRROR_CFG,</div><div class='add'>+				 sizeof(req));</div><div class='add'>+</div><div class='add'>+	req.mirror_flags = (u8)CMDQ_ROCE_MIRROR_CFG_MIRROR_ENABLE;</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+	return bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int bnxt_qplib_destroy_flow(struct bnxt_qplib_res *res)</div><div class='add'>+{</div><div class='add'>+	struct creq_roce_mirror_cfg_resp resp = {};</div><div class='add'>+	struct bnxt_qplib_rcfw *rcfw = res-&gt;rcfw;</div><div class='add'>+	struct cmdq_roce_mirror_cfg req = {};</div><div class='add'>+	struct bnxt_qplib_cmdqmsg msg = {};</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&amp;req,</div><div class='add'>+				 CMDQ_BASE_OPCODE_ROCE_MIRROR_CFG,</div><div class='add'>+				 sizeof(req));</div><div class='add'>+</div><div class='add'>+	req.mirror_flags &amp;= ~((u8)CMDQ_ROCE_MIRROR_CFG_MIRROR_ENABLE);</div><div class='add'>+</div><div class='add'>+	bnxt_qplib_fill_cmdqmsg(&amp;msg, &amp;req, &amp;resp, NULL, sizeof(req),</div><div class='add'>+				sizeof(resp), 0);</div><div class='add'>+</div><div class='add'>+	return bnxt_qplib_rcfw_send_message(rcfw, &amp;msg);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h<br/>index 0ec3b12b0bcd..5a45c55c6464 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_sp.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/qplib_sp.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_sp.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/qplib_sp.h</a></div><div class='hunk'>@@ -40,13 +40,13 @@</div><div class='ctx'> #ifndef __BNXT_QPLIB_SP_H__</div><div class='ctx'> #define __BNXT_QPLIB_SP_H__</div><div class='ctx'> </div><div class='add'>+#include &lt;rdma/bnxt_re-abi.h&gt;</div><div class='ctx'> #define BNXT_QPLIB_RESERVED_QP_WRS	128</div><div class='ctx'> </div><div class='del'>-#define PCI_EXP_DEVCTL2_ATOMIC_REQ      0x0040</div><div class='del'>-</div><div class='ctx'> struct bnxt_qplib_dev_attr {</div><div class='ctx'> #define FW_VER_ARR_LEN			4</div><div class='ctx'> 	u8				fw_ver[FW_VER_ARR_LEN];</div><div class='add'>+#define BNXT_QPLIB_NUM_GIDS_SUPPORTED	256</div><div class='ctx'> 	u16				max_sgid;</div><div class='ctx'> 	u16				max_mrw;</div><div class='ctx'> 	u32				max_qp;</div><div class='hunk'>@@ -56,6 +56,7 @@ struct bnxt_qplib_dev_attr {</div><div class='ctx'> 	u32				max_qp_wqes;</div><div class='ctx'> 	u32				max_qp_sges;</div><div class='ctx'> 	u32				max_cq;</div><div class='add'>+#define BNXT_QPLIB_MAX_CQ_WQES          0xfffff</div><div class='ctx'> 	u32				max_cq_wqes;</div><div class='ctx'> 	u32				max_cq_sges;</div><div class='ctx'> 	u32				max_mr;</div><div class='hunk'>@@ -64,8 +65,6 @@ struct bnxt_qplib_dev_attr {</div><div class='ctx'> 	u32				max_mw;</div><div class='ctx'> 	u32				max_raw_ethy_qp;</div><div class='ctx'> 	u32				max_ah;</div><div class='del'>-	u32				max_fmr;</div><div class='del'>-	u32				max_map_per_fmr;</div><div class='ctx'> 	u32				max_srq;</div><div class='ctx'> 	u32				max_srq_wqes;</div><div class='ctx'> 	u32				max_srq_sges;</div><div class='hunk'>@@ -74,6 +73,9 @@ struct bnxt_qplib_dev_attr {</div><div class='ctx'> 	u32				l2_db_size;</div><div class='ctx'> 	u8				tqm_alloc_reqs[MAX_TQM_ALLOC_REQ];</div><div class='ctx'> 	bool				is_atomic;</div><div class='add'>+	u16                             dev_cap_flags;</div><div class='add'>+	u16                             dev_cap_flags2;</div><div class='add'>+	u32                             max_dpi;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct bnxt_qplib_pd {</div><div class='hunk'>@@ -84,6 +86,11 @@ struct bnxt_qplib_gid {</div><div class='ctx'> 	u8				data[16];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct bnxt_qplib_gid_info {</div><div class='add'>+	struct bnxt_qplib_gid gid;</div><div class='add'>+	u16 vlan_id;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct bnxt_qplib_ah {</div><div class='ctx'> 	struct bnxt_qplib_gid		dgid;</div><div class='ctx'> 	struct bnxt_qplib_pd		*pd;</div><div class='hunk'>@@ -103,7 +110,8 @@ struct bnxt_qplib_ah {</div><div class='ctx'> struct bnxt_qplib_mrw {</div><div class='ctx'> 	struct bnxt_qplib_pd		*pd;</div><div class='ctx'> 	int				type;</div><div class='del'>-	u32				flags;</div><div class='add'>+	u32				access_flags;</div><div class='add'>+#define BNXT_QPLIB_MR_ACCESS_MASK	0xFF</div><div class='ctx'> #define BNXT_QPLIB_FR_PMR		0x80000000</div><div class='ctx'> 	u32				lkey;</div><div class='ctx'> 	u32				rkey;</div><div class='hunk'>@@ -111,6 +119,7 @@ struct bnxt_qplib_mrw {</div><div class='ctx'> 	u64				va;</div><div class='ctx'> 	u64				total_size;</div><div class='ctx'> 	u32				npages;</div><div class='add'>+	u16				flags;</div><div class='ctx'> 	u64				mr_handle;</div><div class='ctx'> 	struct bnxt_qplib_hwq		hwq;</div><div class='ctx'> };</div><div class='hunk'>@@ -217,40 +226,122 @@ struct bnxt_qplib_roce_stats {</div><div class='ctx'> 	/* port 3 active qps */</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct bnxt_qplib_ext_stat {</div><div class='add'>+	u64  tx_atomic_req;</div><div class='add'>+	u64  tx_read_req;</div><div class='add'>+	u64  tx_read_res;</div><div class='add'>+	u64  tx_write_req;</div><div class='add'>+	u64  tx_send_req;</div><div class='add'>+	u64  tx_roce_pkts;</div><div class='add'>+	u64  tx_roce_bytes;</div><div class='add'>+	u64  rx_atomic_req;</div><div class='add'>+	u64  rx_read_req;</div><div class='add'>+	u64  rx_read_res;</div><div class='add'>+	u64  rx_write_req;</div><div class='add'>+	u64  rx_send_req;</div><div class='add'>+	u64  rx_roce_pkts;</div><div class='add'>+	u64  rx_roce_bytes;</div><div class='add'>+	u64  rx_roce_good_pkts;</div><div class='add'>+	u64  rx_roce_good_bytes;</div><div class='add'>+	u64  rx_out_of_buffer;</div><div class='add'>+	u64  rx_out_of_sequence;</div><div class='add'>+	u64  tx_cnp;</div><div class='add'>+	u64  rx_cnp;</div><div class='add'>+	u64  rx_ecn_marked;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_qplib_cc_param_ext {</div><div class='add'>+	u64 ext_mask;</div><div class='add'>+	u16 inact_th_hi;</div><div class='add'>+	u16 min_delta_cnp;</div><div class='add'>+	u16 init_cp;</div><div class='add'>+	u8 tr_update_mode;</div><div class='add'>+	u8 tr_update_cyls;</div><div class='add'>+	u8 fr_rtt;</div><div class='add'>+	u8 ai_rate_incr;</div><div class='add'>+	u16 rr_rtt_th;</div><div class='add'>+	u16 ar_cr_th;</div><div class='add'>+	u16 cr_min_th;</div><div class='add'>+	u8 bw_avg_weight;</div><div class='add'>+	u8 cr_factor;</div><div class='add'>+	u16 cr_th_max_cp;</div><div class='add'>+	u8 cp_bias_en;</div><div class='add'>+	u8 cp_bias;</div><div class='add'>+	u8 cnp_ecn;</div><div class='add'>+	u8 rtt_jitter_en;</div><div class='add'>+	u16 bytes_per_usec;</div><div class='add'>+	u16 cc_cr_reset_th;</div><div class='add'>+	u8 cr_width;</div><div class='add'>+	u8 min_quota;</div><div class='add'>+	u8 max_quota;</div><div class='add'>+	u8 abs_max_quota;</div><div class='add'>+	u16 tr_lb;</div><div class='add'>+	u8 cr_prob_fac;</div><div class='add'>+	u8 tr_prob_fac;</div><div class='add'>+	u16 fair_cr_th;</div><div class='add'>+	u8 red_div;</div><div class='add'>+	u8 cnp_ratio_th;</div><div class='add'>+	u16 ai_ext_rtt;</div><div class='add'>+	u8 exp_crcp_ratio;</div><div class='add'>+	u8 low_rate_en;</div><div class='add'>+	u16 cpcr_update_th;</div><div class='add'>+	u16 ai_rtt_th1;</div><div class='add'>+	u16 ai_rtt_th2;</div><div class='add'>+	u16 cf_rtt_th;</div><div class='add'>+	u16 sc_cr_th1; /* severe congestion cr threshold 1 */</div><div class='add'>+	u16 sc_cr_th2; /* severe congestion cr threshold 2 */</div><div class='add'>+	u32 l64B_per_rtt;</div><div class='add'>+	u8 cc_ack_bytes;</div><div class='add'>+	u16 reduce_cf_rtt_th;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_qplib_cc_param {</div><div class='add'>+	u8 alt_vlan_pcp;</div><div class='add'>+	u8 qp1_tos_dscp;</div><div class='add'>+	u16 alt_tos_dscp;</div><div class='add'>+	u8 cc_mode;</div><div class='add'>+	u8 enable;</div><div class='add'>+	u16 inact_th;</div><div class='add'>+	u16 init_cr;</div><div class='add'>+	u16 init_tr;</div><div class='add'>+	u16 rtt;</div><div class='add'>+	u8 g;</div><div class='add'>+	u8 nph_per_state;</div><div class='add'>+	u8 time_pph;</div><div class='add'>+	u8 pkts_pph;</div><div class='add'>+	u8 tos_ecn;</div><div class='add'>+	u8 tos_dscp;</div><div class='add'>+	u16 tcp_cp;</div><div class='add'>+	struct bnxt_qplib_cc_param_ext cc_ext;</div><div class='add'>+	u32 mask;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res,</div><div class='ctx'> 			struct bnxt_qplib_sgid_tbl *sgid_tbl, int index,</div><div class='ctx'> 			struct bnxt_qplib_gid *gid);</div><div class='ctx'> int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,</div><div class='del'>-			struct bnxt_qplib_gid *gid, bool update);</div><div class='add'>+			struct bnxt_qplib_gid *gid, u16 vlan_id, bool update);</div><div class='ctx'> int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,</div><div class='del'>-			struct bnxt_qplib_gid *gid, u8 *mac, u16 vlan_id,</div><div class='del'>-			bool update, u32 *index);</div><div class='add'>+			struct bnxt_qplib_gid *gid, const u8 *mac, u16 vlan_id,</div><div class='add'>+			bool update, u32 *index,</div><div class='add'>+			bool is_ugid, u32 stats_ctx_id);</div><div class='ctx'> int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,</div><div class='del'>-			   struct bnxt_qplib_gid *gid, u16 gid_idx, u8 *smac);</div><div class='del'>-int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res,</div><div class='del'>-			struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index,</div><div class='del'>-			u16 *pkey);</div><div class='del'>-int bnxt_qplib_del_pkey(struct bnxt_qplib_res *res,</div><div class='del'>-			struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,</div><div class='del'>-			bool update);</div><div class='del'>-int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res,</div><div class='del'>-			struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,</div><div class='del'>-			bool update);</div><div class='del'>-int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,</div><div class='del'>-			    struct bnxt_qplib_dev_attr *attr, bool vf);</div><div class='add'>+			   struct bnxt_qplib_gid *gid, u16 gid_idx,</div><div class='add'>+			   const u8 *smac);</div><div class='add'>+int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw);</div><div class='ctx'> int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res,</div><div class='ctx'> 				  struct bnxt_qplib_rcfw *rcfw,</div><div class='ctx'> 				  struct bnxt_qplib_ctx *ctx);</div><div class='ctx'> int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,</div><div class='ctx'> 			 bool block);</div><div class='del'>-void bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,</div><div class='del'>-			   bool block);</div><div class='add'>+int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,</div><div class='add'>+			  bool block);</div><div class='ctx'> int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res,</div><div class='ctx'> 			 struct bnxt_qplib_mrw *mrw);</div><div class='ctx'> int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,</div><div class='ctx'> 			 bool block);</div><div class='ctx'> int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,</div><div class='del'>-		      u64 *pbl_tbl, int num_pbls, bool block, u32 buf_pg_size);</div><div class='add'>+		      struct ib_umem *umem, int num_pbls, u32 buf_pg_size, bool unified_mr);</div><div class='ctx'> int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr);</div><div class='ctx'> int bnxt_qplib_alloc_fast_reg_mr(struct bnxt_qplib_res *res,</div><div class='ctx'> 				 struct bnxt_qplib_mrw *mr, int max);</div><div class='hunk'>@@ -258,7 +349,25 @@ int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,</div><div class='ctx'> 					struct bnxt_qplib_frpl *frpl, int max);</div><div class='ctx'> int bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res,</div><div class='ctx'> 				       struct bnxt_qplib_frpl *frpl);</div><div class='del'>-int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids);</div><div class='ctx'> int bnxt_qplib_get_roce_stats(struct bnxt_qplib_rcfw *rcfw,</div><div class='ctx'> 			      struct bnxt_qplib_roce_stats *stats);</div><div class='add'>+int bnxt_qplib_qext_stat(struct bnxt_qplib_rcfw *rcfw, u32 fid,</div><div class='add'>+			 struct bnxt_qplib_ext_stat *estat);</div><div class='add'>+int bnxt_qplib_modify_cc(struct bnxt_qplib_res *res,</div><div class='add'>+			 struct bnxt_qplib_cc_param *cc_param);</div><div class='add'>+int bnxt_qplib_read_context(struct bnxt_qplib_rcfw *rcfw, u8 type, u32 xid,</div><div class='add'>+			    u32 resp_size, void *resp_va);</div><div class='add'>+int bnxt_qplib_query_cc_param(struct bnxt_qplib_res *res,</div><div class='add'>+			      struct bnxt_qplib_cc_param *cc_param);</div><div class='add'>+void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw);</div><div class='add'>+int bnxt_qplib_create_flow(struct bnxt_qplib_res *res);</div><div class='add'>+int bnxt_qplib_destroy_flow(struct bnxt_qplib_res *res);</div><div class='add'>+</div><div class='add'>+#define BNXT_VAR_MAX_WQE       4352</div><div class='add'>+#define BNXT_VAR_MAX_SLOT_ALIGN 256</div><div class='add'>+#define BNXT_VAR_MAX_SGE        13</div><div class='add'>+#define BNXT_RE_MAX_RQ_WQES     65536</div><div class='add'>+</div><div class='add'>+#define BNXT_STATIC_MAX_SGE	6</div><div class='add'>+</div><div class='ctx'> #endif /* __BNXT_QPLIB_SP_H__*/</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/qplib_tlv.h b/drivers/infiniband/hw/bnxt_re/qplib_tlv.h<br/>new file mode 100644<br/>index 000000000000..ae96a75d7f31<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/qplib_tlv.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/qplib_tlv.h</a></div><div class='hunk'>@@ -0,0 +1,162 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='add'>+</div><div class='add'>+#ifndef __QPLIB_TLV_H__</div><div class='add'>+#define __QPLIB_TLV_H__</div><div class='add'>+</div><div class='add'>+struct roce_tlv {</div><div class='add'>+	struct tlv tlv;</div><div class='add'>+	u8 total_size; // in units of 16 byte chunks</div><div class='add'>+	u8 unused[7];  // for 16 byte alignment</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define CHUNK_SIZE 16</div><div class='add'>+#define CHUNKS(x) (((x) + CHUNK_SIZE - 1) / CHUNK_SIZE)</div><div class='add'>+</div><div class='add'>+static inline  void __roce_1st_tlv_prep(struct roce_tlv *rtlv, u8 tot_chunks,</div><div class='add'>+					u16 content_bytes, u8 flags)</div><div class='add'>+{</div><div class='add'>+	rtlv-&gt;tlv.cmd_discr = cpu_to_le16(CMD_DISCR_TLV_ENCAP);</div><div class='add'>+	rtlv-&gt;tlv.tlv_type = cpu_to_le16(TLV_TYPE_ROCE_SP_COMMAND);</div><div class='add'>+	rtlv-&gt;tlv.length = cpu_to_le16(content_bytes);</div><div class='add'>+	rtlv-&gt;tlv.flags = TLV_FLAGS_REQUIRED;</div><div class='add'>+	rtlv-&gt;tlv.flags |= flags ? TLV_FLAGS_MORE : 0;</div><div class='add'>+	rtlv-&gt;total_size = (tot_chunks);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void __roce_ext_tlv_prep(struct roce_tlv *rtlv, u16 tlv_type,</div><div class='add'>+				       u16 content_bytes, u8 more, u8 flags)</div><div class='add'>+{</div><div class='add'>+	rtlv-&gt;tlv.cmd_discr = cpu_to_le16(CMD_DISCR_TLV_ENCAP);</div><div class='add'>+	rtlv-&gt;tlv.tlv_type = cpu_to_le16(tlv_type);</div><div class='add'>+	rtlv-&gt;tlv.length = cpu_to_le16(content_bytes);</div><div class='add'>+	rtlv-&gt;tlv.flags |= more ? TLV_FLAGS_MORE : 0;</div><div class='add'>+	rtlv-&gt;tlv.flags |= flags ? TLV_FLAGS_REQUIRED : 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * TLV size in units of 16 byte chunks</div><div class='add'>+ */</div><div class='add'>+#define TLV_SIZE ((sizeof(struct roce_tlv) + 15) / 16)</div><div class='add'>+/*</div><div class='add'>+ * TLV length in bytes</div><div class='add'>+ */</div><div class='add'>+#define TLV_BYTES (TLV_SIZE * 16)</div><div class='add'>+</div><div class='add'>+#define HAS_TLV_HEADER(msg) (le16_to_cpu(((struct tlv *)(msg))-&gt;cmd_discr) == CMD_DISCR_TLV_ENCAP)</div><div class='add'>+#define GET_TLV_DATA(tlv)   ((void *)&amp;((uint8_t *)(tlv))[TLV_BYTES])</div><div class='add'>+</div><div class='add'>+static inline u8 __get_cmdq_base_opcode(struct cmdq_base *req, u32 size)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		return ((struct cmdq_base *)GET_TLV_DATA(req))-&gt;opcode;</div><div class='add'>+	else</div><div class='add'>+		return req-&gt;opcode;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void __set_cmdq_base_opcode(struct cmdq_base *req,</div><div class='add'>+					  u32 size, u8 val)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		((struct cmdq_base *)GET_TLV_DATA(req))-&gt;opcode = val;</div><div class='add'>+	else</div><div class='add'>+		req-&gt;opcode = val;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline __le16 __get_cmdq_base_cookie(struct cmdq_base *req, u32 size)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		return ((struct cmdq_base *)GET_TLV_DATA(req))-&gt;cookie;</div><div class='add'>+	else</div><div class='add'>+		return req-&gt;cookie;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void __set_cmdq_base_cookie(struct cmdq_base *req,</div><div class='add'>+					  u32 size, __le16 val)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		((struct cmdq_base *)GET_TLV_DATA(req))-&gt;cookie = val;</div><div class='add'>+	else</div><div class='add'>+		req-&gt;cookie = val;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline __le64 __get_cmdq_base_resp_addr(struct cmdq_base *req, u32 size)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		return ((struct cmdq_base *)GET_TLV_DATA(req))-&gt;resp_addr;</div><div class='add'>+	else</div><div class='add'>+		return req-&gt;resp_addr;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void __set_cmdq_base_resp_addr(struct cmdq_base *req,</div><div class='add'>+					     u32 size, __le64 val)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		((struct cmdq_base *)GET_TLV_DATA(req))-&gt;resp_addr = val;</div><div class='add'>+	else</div><div class='add'>+		req-&gt;resp_addr = val;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u8 __get_cmdq_base_resp_size(struct cmdq_base *req, u32 size)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		return ((struct cmdq_base *)GET_TLV_DATA(req))-&gt;resp_size;</div><div class='add'>+	else</div><div class='add'>+		return req-&gt;resp_size;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void __set_cmdq_base_resp_size(struct cmdq_base *req,</div><div class='add'>+					     u32 size, u8 val)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		((struct cmdq_base *)GET_TLV_DATA(req))-&gt;resp_size = val;</div><div class='add'>+	else</div><div class='add'>+		req-&gt;resp_size = val;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u8 __get_cmdq_base_cmd_size(struct cmdq_base *req, u32 size)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		return ((struct roce_tlv *)(req))-&gt;total_size;</div><div class='add'>+	else</div><div class='add'>+		return req-&gt;cmd_size;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void __set_cmdq_base_cmd_size(struct cmdq_base *req,</div><div class='add'>+					    u32 size, u8 val)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		((struct cmdq_base *)GET_TLV_DATA(req))-&gt;cmd_size = val;</div><div class='add'>+	else</div><div class='add'>+		req-&gt;cmd_size = val;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline __le16 __get_cmdq_base_flags(struct cmdq_base *req, u32 size)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		return ((struct cmdq_base *)GET_TLV_DATA(req))-&gt;flags;</div><div class='add'>+	else</div><div class='add'>+		return req-&gt;flags;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void __set_cmdq_base_flags(struct cmdq_base *req,</div><div class='add'>+					 u32 size, __le16 val)</div><div class='add'>+{</div><div class='add'>+	if (HAS_TLV_HEADER(req) &amp;&amp; size &gt; TLV_BYTES)</div><div class='add'>+		((struct cmdq_base *)GET_TLV_DATA(req))-&gt;flags = val;</div><div class='add'>+	else</div><div class='add'>+		req-&gt;flags = val;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct bnxt_qplib_tlv_modify_cc_req {</div><div class='add'>+	struct roce_tlv				tlv_hdr;</div><div class='add'>+	struct cmdq_modify_roce_cc		base_req;</div><div class='add'>+	__le64					tlvpad;</div><div class='add'>+	struct cmdq_modify_roce_cc_gen1_tlv	ext_req;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct bnxt_qplib_tlv_query_rcc_sb {</div><div class='add'>+	struct roce_tlv					tlv_hdr;</div><div class='add'>+	struct creq_query_roce_cc_resp_sb		base_sb;</div><div class='add'>+	struct creq_query_roce_cc_gen1_resp_sb_tlv	gen1_sb;</div><div class='add'>+};</div><div class='add'>+#endif /* __QPLIB_TLV_H__ */</div><div class='head'>diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h<br/>index e4b09e7c2175..99ecd72e72e2 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/roce_hsi.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/bnxt_re/roce_hsi.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/bnxt_re/roce_hsi.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/bnxt_re/roce_hsi.h</a></div><div class='hunk'>@@ -39,1396 +39,1381 @@</div><div class='ctx'> #ifndef __BNXT_RE_HSI_H__</div><div class='ctx'> #define __BNXT_RE_HSI_H__</div><div class='ctx'> </div><div class='del'>-/* include bnxt_hsi.h from bnxt_en driver */</div><div class='del'>-#include "bnxt_hsi.h"</div><div class='add'>+/* include linux/bnxt/hsi.h */</div><div class='add'>+#include &lt;linux/bnxt/hsi.h&gt;</div><div class='ctx'> </div><div class='del'>-/* CMP Door Bell Format (4 bytes) */</div><div class='add'>+/* tx_doorbell (size:32b/4B) */</div><div class='add'>+struct tx_doorbell {</div><div class='add'>+	__le32	key_idx;</div><div class='add'>+	#define TX_DOORBELL_IDX_MASK 0xffffffUL</div><div class='add'>+	#define TX_DOORBELL_IDX_SFT 0</div><div class='add'>+	#define TX_DOORBELL_KEY_MASK 0xf0000000UL</div><div class='add'>+	#define TX_DOORBELL_KEY_SFT 28</div><div class='add'>+	#define TX_DOORBELL_KEY_TX    (0x0UL &lt;&lt; 28)</div><div class='add'>+	#define TX_DOORBELL_KEY_LAST TX_DOORBELL_KEY_TX</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* rx_doorbell (size:32b/4B) */</div><div class='add'>+struct rx_doorbell {</div><div class='add'>+	__le32	key_idx;</div><div class='add'>+	#define RX_DOORBELL_IDX_MASK 0xffffffUL</div><div class='add'>+	#define RX_DOORBELL_IDX_SFT 0</div><div class='add'>+	#define RX_DOORBELL_KEY_MASK 0xf0000000UL</div><div class='add'>+	#define RX_DOORBELL_KEY_SFT 28</div><div class='add'>+	#define RX_DOORBELL_KEY_RX    (0x1UL &lt;&lt; 28)</div><div class='add'>+	#define RX_DOORBELL_KEY_LAST RX_DOORBELL_KEY_RX</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmpl_doorbell (size:32b/4B) */</div><div class='ctx'> struct cmpl_doorbell {</div><div class='del'>-	__le32 key_mask_valid_idx;</div><div class='del'>-	#define CMPL_DOORBELL_IDX_MASK				    0xffffffUL</div><div class='del'>-	#define CMPL_DOORBELL_IDX_SFT				    0</div><div class='del'>-	#define CMPL_DOORBELL_RESERVED_MASK			    0x3000000UL</div><div class='del'>-	#define CMPL_DOORBELL_RESERVED_SFT			    24</div><div class='del'>-	#define CMPL_DOORBELL_IDX_VALID				    0x4000000UL</div><div class='del'>-	#define CMPL_DOORBELL_MASK				    0x8000000UL</div><div class='del'>-	#define CMPL_DOORBELL_KEY_MASK				    0xf0000000UL</div><div class='del'>-	#define CMPL_DOORBELL_KEY_SFT				    28</div><div class='del'>-	#define CMPL_DOORBELL_KEY_CMPL				(0x2UL &lt;&lt; 28)</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Status Door Bell Format (4 bytes) */</div><div class='add'>+	__le32	key_mask_valid_idx;</div><div class='add'>+	#define CMPL_DOORBELL_IDX_MASK      0xffffffUL</div><div class='add'>+	#define CMPL_DOORBELL_IDX_SFT       0</div><div class='add'>+	#define CMPL_DOORBELL_IDX_VALID     0x4000000UL</div><div class='add'>+	#define CMPL_DOORBELL_MASK          0x8000000UL</div><div class='add'>+	#define CMPL_DOORBELL_KEY_MASK      0xf0000000UL</div><div class='add'>+	#define CMPL_DOORBELL_KEY_SFT       28</div><div class='add'>+	#define CMPL_DOORBELL_KEY_CMPL        (0x2UL &lt;&lt; 28)</div><div class='add'>+	#define CMPL_DOORBELL_KEY_LAST       CMPL_DOORBELL_KEY_CMPL</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* status_doorbell (size:32b/4B) */</div><div class='ctx'> struct status_doorbell {</div><div class='del'>-	__le32 key_idx;</div><div class='del'>-	#define STATUS_DOORBELL_IDX_MASK			    0xffffffUL</div><div class='del'>-	#define STATUS_DOORBELL_IDX_SFT			    0</div><div class='del'>-	#define STATUS_DOORBELL_RESERVED_MASK			    0xf000000UL</div><div class='del'>-	#define STATUS_DOORBELL_RESERVED_SFT			    24</div><div class='del'>-	#define STATUS_DOORBELL_KEY_MASK			    0xf0000000UL</div><div class='del'>-	#define STATUS_DOORBELL_KEY_SFT			    28</div><div class='del'>-	#define STATUS_DOORBELL_KEY_STAT			   (0x3UL &lt;&lt; 28)</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* RoCE Host Structures */</div><div class='del'>-</div><div class='del'>-/* Doorbell Structures */</div><div class='del'>-/* dbc_dbc (size:64b/8B) */</div><div class='del'>-struct dbc_dbc {</div><div class='del'>-	__le32  index;</div><div class='del'>-	#define DBC_DBC_INDEX_MASK		0xffffffUL</div><div class='del'>-	#define DBC_DBC_INDEX_SFT		0</div><div class='del'>-	__le32  type_path_xid;</div><div class='del'>-	#define DBC_DBC_XID_MASK		0xfffffUL</div><div class='del'>-	#define DBC_DBC_XID_SFT			0</div><div class='del'>-	#define DBC_DBC_PATH_MASK		0x3000000UL</div><div class='del'>-	#define DBC_DBC_PATH_SFT		24</div><div class='del'>-	#define DBC_DBC_PATH_ROCE		(0x0UL &lt;&lt; 24)</div><div class='del'>-	#define DBC_DBC_PATH_L2			(0x1UL &lt;&lt; 24)</div><div class='del'>-	#define DBC_DBC_PATH_ENGINE		(0x2UL &lt;&lt; 24)</div><div class='del'>-	#define DBC_DBC_PATH_LAST		DBC_DBC_PATH_ENGINE</div><div class='del'>-	#define DBC_DBC_DEBUG_TRACE		0x8000000UL</div><div class='del'>-	#define DBC_DBC_TYPE_MASK		0xf0000000UL</div><div class='del'>-	#define DBC_DBC_TYPE_SFT		28</div><div class='del'>-	#define DBC_DBC_TYPE_SQ			(0x0UL &lt;&lt; 28)</div><div class='del'>-	#define DBC_DBC_TYPE_RQ			(0x1UL &lt;&lt; 28)</div><div class='del'>-	#define DBC_DBC_TYPE_SRQ		(0x2UL &lt;&lt; 28)</div><div class='del'>-	#define DBC_DBC_TYPE_SRQ_ARM		(0x3UL &lt;&lt; 28)</div><div class='del'>-	#define DBC_DBC_TYPE_CQ			(0x4UL &lt;&lt; 28)</div><div class='del'>-	#define DBC_DBC_TYPE_CQ_ARMSE		(0x5UL &lt;&lt; 28)</div><div class='del'>-	#define DBC_DBC_TYPE_CQ_ARMALL		(0x6UL &lt;&lt; 28)</div><div class='del'>-	#define DBC_DBC_TYPE_CQ_ARMENA		(0x7UL &lt;&lt; 28)</div><div class='del'>-	#define DBC_DBC_TYPE_SRQ_ARMENA		(0x8UL &lt;&lt; 28)</div><div class='del'>-	#define DBC_DBC_TYPE_CQ_CUTOFF_ACK	(0x9UL &lt;&lt; 28)</div><div class='del'>-	#define DBC_DBC_TYPE_NQ			(0xaUL &lt;&lt; 28)</div><div class='del'>-	#define DBC_DBC_TYPE_NQ_ARM		(0xbUL &lt;&lt; 28)</div><div class='del'>-	#define DBC_DBC_TYPE_NULL		(0xfUL &lt;&lt; 28)</div><div class='del'>-	#define DBC_DBC_TYPE_LAST		DBC_DBC_TYPE_NULL</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* dbc_dbc32 (size:32b/4B) */</div><div class='del'>-struct dbc_dbc32 {</div><div class='del'>-	__le32  type_abs_incr_xid;</div><div class='del'>-	#define DBC_DBC32_XID_MASK		0xfffffUL</div><div class='del'>-	#define DBC_DBC32_XID_SFT		0</div><div class='del'>-	#define DBC_DBC32_PATH_MASK		0xc00000UL</div><div class='del'>-	#define DBC_DBC32_PATH_SFT		22</div><div class='del'>-	#define DBC_DBC32_PATH_ROCE		(0x0UL &lt;&lt; 22)</div><div class='del'>-	#define DBC_DBC32_PATH_L2		(0x1UL &lt;&lt; 22)</div><div class='del'>-	#define DBC_DBC32_PATH_LAST		DBC_DBC32_PATH_L2</div><div class='del'>-	#define DBC_DBC32_INCR_MASK		0xf000000UL</div><div class='del'>-	#define DBC_DBC32_INCR_SFT		24</div><div class='del'>-	#define DBC_DBC32_ABS			0x10000000UL</div><div class='del'>-	#define DBC_DBC32_TYPE_MASK		0xe0000000UL</div><div class='del'>-	#define DBC_DBC32_TYPE_SFT		29</div><div class='del'>-	#define DBC_DBC32_TYPE_SQ		(0x0UL &lt;&lt; 29)</div><div class='del'>-	#define DBC_DBC32_TYPE_LAST		DBC_DBC32_TYPE_SQ</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* SQ WQE Structures */</div><div class='del'>-/* Base SQ WQE (8 bytes) */</div><div class='del'>-struct sq_base {</div><div class='del'>-	u8 wqe_type;</div><div class='del'>-	#define SQ_BASE_WQE_TYPE_SEND				   0x0UL</div><div class='del'>-	#define SQ_BASE_WQE_TYPE_SEND_W_IMMEAD			   0x1UL</div><div class='del'>-	#define SQ_BASE_WQE_TYPE_SEND_W_INVALID		   0x2UL</div><div class='del'>-	#define SQ_BASE_WQE_TYPE_WRITE_WQE			   0x4UL</div><div class='del'>-	#define SQ_BASE_WQE_TYPE_WRITE_W_IMMEAD		   0x5UL</div><div class='del'>-	#define SQ_BASE_WQE_TYPE_READ_WQE			   0x6UL</div><div class='del'>-	#define SQ_BASE_WQE_TYPE_ATOMIC_CS			   0x8UL</div><div class='del'>-	#define SQ_BASE_WQE_TYPE_ATOMIC_FA			   0xbUL</div><div class='del'>-	#define SQ_BASE_WQE_TYPE_LOCAL_INVALID			   0xcUL</div><div class='del'>-	#define SQ_BASE_WQE_TYPE_FR_PMR			   0xdUL</div><div class='del'>-	#define SQ_BASE_WQE_TYPE_BIND				   0xeUL</div><div class='del'>-	u8 unused_0[7];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* WQE SGE (16 bytes) */</div><div class='del'>-struct sq_sge {</div><div class='del'>-	__le64 va_or_pa;</div><div class='del'>-	__le32 l_key;</div><div class='del'>-	__le32 size;</div><div class='add'>+	__le32	key_idx;</div><div class='add'>+	#define STATUS_DOORBELL_IDX_MASK 0xffffffUL</div><div class='add'>+	#define STATUS_DOORBELL_IDX_SFT 0</div><div class='add'>+	#define STATUS_DOORBELL_KEY_MASK 0xf0000000UL</div><div class='add'>+	#define STATUS_DOORBELL_KEY_SFT 28</div><div class='add'>+	#define STATUS_DOORBELL_KEY_STAT  (0x3UL &lt;&lt; 28)</div><div class='add'>+	#define STATUS_DOORBELL_KEY_LAST STATUS_DOORBELL_KEY_STAT</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* PSN Search Structure (8 bytes) */</div><div class='del'>-struct sq_psn_search {</div><div class='del'>-	__le32 opcode_start_psn;</div><div class='del'>-	#define SQ_PSN_SEARCH_START_PSN_MASK			    0xffffffUL</div><div class='del'>-	#define SQ_PSN_SEARCH_START_PSN_SFT			    0</div><div class='del'>-	#define SQ_PSN_SEARCH_OPCODE_MASK			    0xff000000UL</div><div class='del'>-	#define SQ_PSN_SEARCH_OPCODE_SFT			    24</div><div class='del'>-	__le32 flags_next_psn;</div><div class='del'>-	#define SQ_PSN_SEARCH_NEXT_PSN_MASK			    0xffffffUL</div><div class='del'>-	#define SQ_PSN_SEARCH_NEXT_PSN_SFT			    0</div><div class='del'>-	#define SQ_PSN_SEARCH_FLAGS_MASK			    0xff000000UL</div><div class='del'>-	#define SQ_PSN_SEARCH_FLAGS_SFT				    24</div><div class='add'>+/* cmdq_init (size:128b/16B) */</div><div class='add'>+struct cmdq_init {</div><div class='add'>+	__le64	cmdq_pbl;</div><div class='add'>+	__le16	cmdq_size_cmdq_lvl;</div><div class='add'>+	#define CMDQ_INIT_CMDQ_LVL_MASK 0x3UL</div><div class='add'>+	#define CMDQ_INIT_CMDQ_LVL_SFT  0</div><div class='add'>+	#define CMDQ_INIT_CMDQ_SIZE_MASK 0xfffcUL</div><div class='add'>+	#define CMDQ_INIT_CMDQ_SIZE_SFT 2</div><div class='add'>+	__le16	creq_ring_id;</div><div class='add'>+	__le32	prod_idx;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* sq_psn_search_ext (size:128b/16B) */</div><div class='del'>-struct sq_psn_search_ext {</div><div class='del'>-	__le32  opcode_start_psn;</div><div class='del'>-	#define SQ_PSN_SEARCH_EXT_START_PSN_MASK		    0xffffffUL</div><div class='del'>-	#define SQ_PSN_SEARCH_EXT_START_PSN_SFT			    0</div><div class='del'>-	#define SQ_PSN_SEARCH_EXT_OPCODE_MASK			    0xff000000UL</div><div class='del'>-	#define SQ_PSN_SEARCH_EXT_OPCODE_SFT			    24</div><div class='del'>-	__le32  flags_next_psn;</div><div class='del'>-	#define SQ_PSN_SEARCH_EXT_NEXT_PSN_MASK			    0xffffffUL</div><div class='del'>-	#define SQ_PSN_SEARCH_EXT_NEXT_PSN_SFT			    0</div><div class='del'>-	#define SQ_PSN_SEARCH_EXT_FLAGS_MASK			    0xff000000UL</div><div class='del'>-	#define SQ_PSN_SEARCH_EXT_FLAGS_SFT			    24</div><div class='del'>-	__le16  start_slot_idx;</div><div class='del'>-	__le16  reserved16;</div><div class='del'>-	__le32  reserved32;</div><div class='add'>+/* cmdq_base (size:128b/16B) */</div><div class='add'>+struct cmdq_base {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_BASE_OPCODE_CREATE_QP              0x1UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_DESTROY_QP             0x2UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_MODIFY_QP              0x3UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_QUERY_QP               0x4UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_CREATE_SRQ             0x5UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_DESTROY_SRQ            0x6UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_QUERY_SRQ              0x8UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_CREATE_CQ              0x9UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_DESTROY_CQ             0xaUL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_RESIZE_CQ              0xcUL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_ALLOCATE_MRW           0xdUL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_DEALLOCATE_KEY         0xeUL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_REGISTER_MR            0xfUL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_DEREGISTER_MR          0x10UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_ADD_GID                0x11UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_DELETE_GID             0x12UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_MODIFY_GID             0x17UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_QUERY_GID              0x18UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_CREATE_QP1             0x13UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_DESTROY_QP1            0x14UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_CREATE_AH              0x15UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_DESTROY_AH             0x16UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_INITIALIZE_FW          0x80UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_DEINITIALIZE_FW        0x81UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_STOP_FUNC              0x82UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_QUERY_FUNC             0x83UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES     0x84UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_READ_CONTEXT           0x85UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_VF_BACKCHANNEL_REQUEST 0x86UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_READ_VF_MEMORY         0x87UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_COMPLETE_VF_REQUEST    0x88UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_EXTEND_CONTEXT_ARRRAY  0x89UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_MAP_TC_TO_COS          0x8aUL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_QUERY_VERSION          0x8bUL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_MODIFY_ROCE_CC         0x8cUL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_QUERY_ROCE_CC          0x8dUL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_QUERY_ROCE_STATS       0x8eUL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_SET_LINK_AGGR_MODE     0x8fUL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_MODIFY_CQ              0x90UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_QUERY_QP_EXTEND        0x91UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_QUERY_ROCE_STATS_EXT   0x92UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_ROCE_MIRROR_CFG	0x99UL</div><div class='add'>+	#define CMDQ_BASE_OPCODE_LAST                   CMDQ_BASE_OPCODE_ROCE_MIRROR_CFG</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* Send SQ WQE (40 bytes) */</div><div class='del'>-struct sq_send {</div><div class='del'>-	u8 wqe_type;</div><div class='del'>-	#define SQ_SEND_WQE_TYPE_SEND				   0x0UL</div><div class='del'>-	#define SQ_SEND_WQE_TYPE_SEND_W_IMMEAD			   0x1UL</div><div class='del'>-	#define SQ_SEND_WQE_TYPE_SEND_W_INVALID		   0x2UL</div><div class='del'>-	u8 flags;</div><div class='del'>-	#define SQ_SEND_FLAGS_SIGNAL_COMP			    0x1UL</div><div class='del'>-	#define SQ_SEND_FLAGS_RD_OR_ATOMIC_FENCE		    0x2UL</div><div class='del'>-	#define SQ_SEND_FLAGS_UC_FENCE				    0x4UL</div><div class='del'>-	#define SQ_SEND_FLAGS_SE				    0x8UL</div><div class='del'>-	#define SQ_SEND_FLAGS_INLINE				    0x10UL</div><div class='del'>-	u8 wqe_size;</div><div class='del'>-	u8 reserved8_1;</div><div class='del'>-	__le32 inv_key_or_imm_data;</div><div class='del'>-	__le32 length;</div><div class='del'>-	__le32 q_key;</div><div class='del'>-	__le32 dst_qp;</div><div class='del'>-	#define SQ_SEND_DST_QP_MASK				    0xffffffUL</div><div class='del'>-	#define SQ_SEND_DST_QP_SFT				    0</div><div class='del'>-	#define SQ_SEND_RESERVED8_2_MASK			    0xff000000UL</div><div class='del'>-	#define SQ_SEND_RESERVED8_2_SFT			    24</div><div class='del'>-	__le32 avid;</div><div class='del'>-	#define SQ_SEND_AVID_MASK				    0xfffffUL</div><div class='del'>-	#define SQ_SEND_AVID_SFT				    0</div><div class='del'>-	#define SQ_SEND_RESERVED_AVID_MASK			    0xfff00000UL</div><div class='del'>-	#define SQ_SEND_RESERVED_AVID_SFT			    20</div><div class='del'>-	__le64 reserved64;</div><div class='del'>-	__le32 data[24];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Send Raw Ethernet and QP1 SQ WQE (40 bytes) */</div><div class='del'>-struct sq_send_raweth_qp1 {</div><div class='del'>-	u8 wqe_type;</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_WQE_TYPE_SEND		   0x0UL</div><div class='del'>-	u8 flags;</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_FLAGS_SIGNAL_COMP		    0x1UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_FLAGS_RD_OR_ATOMIC_FENCE	    0x2UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_FLAGS_UC_FENCE		    0x4UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_FLAGS_SE			    0x8UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_FLAGS_INLINE		    0x10UL</div><div class='del'>-	u8 wqe_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le16 lflags;</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_LFLAGS_TCP_UDP_CHKSUM	    0x1UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM		    0x2UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_LFLAGS_NOCRC		    0x4UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_LFLAGS_STAMP		    0x8UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_LFLAGS_T_IP_CHKSUM		    0x10UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_1		    0x20UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_2		    0x40UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_3		    0x80UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC		    0x100UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_LFLAGS_FCOE_CRC		    0x200UL</div><div class='del'>-	__le16 cfa_action;</div><div class='del'>-	__le32 length;</div><div class='del'>-	__le32 reserved32_1;</div><div class='del'>-	__le32 cfa_meta;</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK	    0xfffUL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT	    0</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_DE		    0x1000UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_MASK	    0xe000UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_SFT	    13</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_MASK	    0x70000UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_SFT	    16</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID88A8    (0x0UL &lt;&lt; 16)</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID8100    (0x1UL &lt;&lt; 16)</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9100    (0x2UL &lt;&lt; 16)</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9200    (0x3UL &lt;&lt; 16)</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9300    (0x4UL &lt;&lt; 16)</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG     (0x5UL &lt;&lt; 16)</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_LAST	\</div><div class='del'>-				SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_MASK     0xff80000UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_SFT      19</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_KEY_MASK		    0xf0000000UL</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_KEY_SFT		    28</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_KEY_NONE		   (0x0UL &lt;&lt; 28)</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG	   (0x1UL &lt;&lt; 28)</div><div class='del'>-	#define SQ_SEND_RAWETH_QP1_CFA_META_KEY_LAST		\</div><div class='del'>-				SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG</div><div class='del'>-	__le32 reserved32_2;</div><div class='del'>-	__le64 reserved64;</div><div class='del'>-	__le32 data[24];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* RDMA SQ WQE (40 bytes) */</div><div class='del'>-struct sq_rdma {</div><div class='del'>-	u8 wqe_type;</div><div class='del'>-	#define SQ_RDMA_WQE_TYPE_WRITE_WQE			   0x4UL</div><div class='del'>-	#define SQ_RDMA_WQE_TYPE_WRITE_W_IMMEAD		   0x5UL</div><div class='del'>-	#define SQ_RDMA_WQE_TYPE_READ_WQE			   0x6UL</div><div class='del'>-	u8 flags;</div><div class='del'>-	#define SQ_RDMA_FLAGS_SIGNAL_COMP			    0x1UL</div><div class='del'>-	#define SQ_RDMA_FLAGS_RD_OR_ATOMIC_FENCE		    0x2UL</div><div class='del'>-	#define SQ_RDMA_FLAGS_UC_FENCE				    0x4UL</div><div class='del'>-	#define SQ_RDMA_FLAGS_SE				    0x8UL</div><div class='del'>-	#define SQ_RDMA_FLAGS_INLINE				    0x10UL</div><div class='del'>-	u8 wqe_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le32 imm_data;</div><div class='del'>-	__le32 length;</div><div class='del'>-	__le32 reserved32_1;</div><div class='del'>-	__le64 remote_va;</div><div class='del'>-	__le32 remote_key;</div><div class='del'>-	__le32 reserved32_2;</div><div class='del'>-	__le32 data[24];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Atomic SQ WQE (40 bytes) */</div><div class='del'>-struct sq_atomic {</div><div class='del'>-	u8 wqe_type;</div><div class='del'>-	#define SQ_ATOMIC_WQE_TYPE_ATOMIC_CS			   0x8UL</div><div class='del'>-	#define SQ_ATOMIC_WQE_TYPE_ATOMIC_FA			   0xbUL</div><div class='del'>-	u8 flags;</div><div class='del'>-	#define SQ_ATOMIC_FLAGS_SIGNAL_COMP			    0x1UL</div><div class='del'>-	#define SQ_ATOMIC_FLAGS_RD_OR_ATOMIC_FENCE		    0x2UL</div><div class='del'>-	#define SQ_ATOMIC_FLAGS_UC_FENCE			    0x4UL</div><div class='del'>-	#define SQ_ATOMIC_FLAGS_SE				    0x8UL</div><div class='del'>-	#define SQ_ATOMIC_FLAGS_INLINE				    0x10UL</div><div class='del'>-	__le16 reserved16;</div><div class='del'>-	__le32 remote_key;</div><div class='del'>-	__le64 remote_va;</div><div class='del'>-	__le64 swap_data;</div><div class='del'>-	__le64 cmp_data;</div><div class='del'>-	__le32 data[24];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Local Invalidate SQ WQE (40 bytes) */</div><div class='del'>-struct sq_localinvalidate {</div><div class='del'>-	u8 wqe_type;</div><div class='del'>-	#define SQ_LOCALINVALIDATE_WQE_TYPE_LOCAL_INVALID	   0xcUL</div><div class='del'>-	u8 flags;</div><div class='del'>-	#define SQ_LOCALINVALIDATE_FLAGS_SIGNAL_COMP		    0x1UL</div><div class='del'>-	#define SQ_LOCALINVALIDATE_FLAGS_RD_OR_ATOMIC_FENCE	    0x2UL</div><div class='del'>-	#define SQ_LOCALINVALIDATE_FLAGS_UC_FENCE		    0x4UL</div><div class='del'>-	#define SQ_LOCALINVALIDATE_FLAGS_SE			    0x8UL</div><div class='del'>-	#define SQ_LOCALINVALIDATE_FLAGS_INLINE		    0x10UL</div><div class='del'>-	__le16 reserved16;</div><div class='del'>-	__le32 inv_l_key;</div><div class='del'>-	__le64 reserved64;</div><div class='del'>-	__le32 reserved128[4];</div><div class='del'>-	__le32 data[24];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* FR-PMR SQ WQE (40 bytes) */</div><div class='del'>-struct sq_fr_pmr {</div><div class='del'>-	u8 wqe_type;</div><div class='del'>-	#define SQ_FR_PMR_WQE_TYPE_FR_PMR			   0xdUL</div><div class='del'>-	u8 flags;</div><div class='del'>-	#define SQ_FR_PMR_FLAGS_SIGNAL_COMP			    0x1UL</div><div class='del'>-	#define SQ_FR_PMR_FLAGS_RD_OR_ATOMIC_FENCE		    0x2UL</div><div class='del'>-	#define SQ_FR_PMR_FLAGS_UC_FENCE			    0x4UL</div><div class='del'>-	#define SQ_FR_PMR_FLAGS_SE				    0x8UL</div><div class='del'>-	#define SQ_FR_PMR_FLAGS_INLINE				    0x10UL</div><div class='del'>-	u8 access_cntl;</div><div class='del'>-	#define SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE		    0x1UL</div><div class='del'>-	#define SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ		    0x2UL</div><div class='del'>-	#define SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE		    0x4UL</div><div class='del'>-	#define SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC		    0x8UL</div><div class='del'>-	#define SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND		    0x10UL</div><div class='del'>-	u8 zero_based_page_size_log;</div><div class='del'>-	#define SQ_FR_PMR_PAGE_SIZE_LOG_MASK			    0x1fUL</div><div class='del'>-	#define SQ_FR_PMR_PAGE_SIZE_LOG_SFT			    0</div><div class='del'>-	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4K		   0x0UL</div><div class='del'>-	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8K		   0x1UL</div><div class='del'>-	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64K		   0x4UL</div><div class='del'>-	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256K		   0x6UL</div><div class='del'>-	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1M		   0x8UL</div><div class='del'>-	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2M		   0x9UL</div><div class='del'>-	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4M		   0xaUL</div><div class='del'>-	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1G		   0x12UL</div><div class='del'>-	#define SQ_FR_PMR_ZERO_BASED				    0x20UL</div><div class='del'>-	#define SQ_FR_PMR_RESERVED2_MASK			    0xc0UL</div><div class='del'>-	#define SQ_FR_PMR_RESERVED2_SFT			    6</div><div class='del'>-	__le32 l_key;</div><div class='del'>-	u8 length[5];</div><div class='del'>-	u8 reserved8_1;</div><div class='del'>-	u8 reserved8_2;</div><div class='del'>-	u8 numlevels_pbl_page_size_log;</div><div class='del'>-	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK		    0x1fUL</div><div class='del'>-	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT		    0</div><div class='del'>-	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4K		   0x0UL</div><div class='del'>-	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8K		   0x1UL</div><div class='del'>-	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64K		   0x4UL</div><div class='del'>-	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256K		   0x6UL</div><div class='del'>-	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1M		   0x8UL</div><div class='del'>-	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2M		   0x9UL</div><div class='del'>-	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4M		   0xaUL</div><div class='del'>-	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1G		   0x12UL</div><div class='del'>-	#define SQ_FR_PMR_RESERVED1				    0x20UL</div><div class='del'>-	#define SQ_FR_PMR_NUMLEVELS_MASK			    0xc0UL</div><div class='del'>-	#define SQ_FR_PMR_NUMLEVELS_SFT			    6</div><div class='del'>-	#define SQ_FR_PMR_NUMLEVELS_PHYSICAL			   (0x0UL &lt;&lt; 6)</div><div class='del'>-	#define SQ_FR_PMR_NUMLEVELS_LAYER1			   (0x1UL &lt;&lt; 6)</div><div class='del'>-	#define SQ_FR_PMR_NUMLEVELS_LAYER2			   (0x2UL &lt;&lt; 6)</div><div class='del'>-	__le64 pblptr;</div><div class='del'>-	__le64 va;</div><div class='del'>-	__le32 data[24];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Bind SQ WQE (40 bytes) */</div><div class='del'>-struct sq_bind {</div><div class='del'>-	u8 wqe_type;</div><div class='del'>-	#define SQ_BIND_WQE_TYPE_BIND				   0xeUL</div><div class='del'>-	u8 flags;</div><div class='del'>-	#define SQ_BIND_FLAGS_SIGNAL_COMP			    0x1UL</div><div class='del'>-	#define SQ_BIND_FLAGS_RD_OR_ATOMIC_FENCE		    0x2UL</div><div class='del'>-	#define SQ_BIND_FLAGS_UC_FENCE				    0x4UL</div><div class='del'>-	#define SQ_BIND_FLAGS_SE				    0x8UL</div><div class='del'>-	#define SQ_BIND_FLAGS_INLINE				    0x10UL</div><div class='del'>-	u8 access_cntl;</div><div class='del'>-	#define SQ_BIND_ACCESS_CNTL_LOCAL_WRITE		    0x1UL</div><div class='del'>-	#define SQ_BIND_ACCESS_CNTL_REMOTE_READ		    0x2UL</div><div class='del'>-	#define SQ_BIND_ACCESS_CNTL_REMOTE_WRITE		    0x4UL</div><div class='del'>-	#define SQ_BIND_ACCESS_CNTL_REMOTE_ATOMIC		    0x8UL</div><div class='del'>-	#define SQ_BIND_ACCESS_CNTL_WINDOW_BIND		    0x10UL</div><div class='del'>-	u8 reserved8_1;</div><div class='del'>-	u8 mw_type_zero_based;</div><div class='del'>-	#define SQ_BIND_ZERO_BASED				    0x1UL</div><div class='del'>-	#define SQ_BIND_MW_TYPE				    0x2UL</div><div class='del'>-	#define SQ_BIND_MW_TYPE_TYPE1				   (0x0UL &lt;&lt; 1)</div><div class='del'>-	#define SQ_BIND_MW_TYPE_TYPE2				   (0x1UL &lt;&lt; 1)</div><div class='del'>-	#define SQ_BIND_RESERVED6_MASK				    0xfcUL</div><div class='del'>-	#define SQ_BIND_RESERVED6_SFT				    2</div><div class='del'>-	u8 reserved8_2;</div><div class='del'>-	__le16 reserved16;</div><div class='del'>-	__le32 parent_l_key;</div><div class='del'>-	__le32 l_key;</div><div class='del'>-	__le64 va;</div><div class='del'>-	u8 length[5];</div><div class='del'>-	u8 data_reserved24[99];</div><div class='del'>-	#define SQ_BIND_RESERVED24_MASK			    0xffffff00UL</div><div class='del'>-	#define SQ_BIND_RESERVED24_SFT				    8</div><div class='del'>-	#define SQ_BIND_DATA_MASK				    0xffffffffUL</div><div class='del'>-	#define SQ_BIND_DATA_SFT				    0</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* RQ/SRQ WQE Structures */</div><div class='del'>-/* RQ/SRQ WQE (40 bytes) */</div><div class='del'>-struct rq_wqe {</div><div class='del'>-	u8 wqe_type;</div><div class='del'>-	#define RQ_WQE_WQE_TYPE_RCV				   0x80UL</div><div class='del'>-	u8 flags;</div><div class='del'>-	u8 wqe_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le32 reserved32;</div><div class='del'>-	__le32 wr_id[2];</div><div class='del'>-	#define RQ_WQE_WR_ID_MASK				    0xfffffUL</div><div class='del'>-	#define RQ_WQE_WR_ID_SFT				    0</div><div class='del'>-	#define RQ_WQE_RESERVED44_MASK				    0xfff00000UL</div><div class='del'>-	#define RQ_WQE_RESERVED44_SFT				    20</div><div class='del'>-	__le32 reserved128[4];</div><div class='del'>-	__le32 data[24];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* CQ CQE Structures */</div><div class='del'>-/* Base CQE (32 bytes) */</div><div class='del'>-struct cq_base {</div><div class='del'>-	__le64 reserved64_1;</div><div class='del'>-	__le64 reserved64_2;</div><div class='del'>-	__le64 reserved64_3;</div><div class='del'>-	u8 cqe_type_toggle;</div><div class='del'>-	#define CQ_BASE_TOGGLE					    0x1UL</div><div class='del'>-	#define CQ_BASE_CQE_TYPE_MASK				    0x1eUL</div><div class='del'>-	#define CQ_BASE_CQE_TYPE_SFT				    1</div><div class='del'>-	#define CQ_BASE_CQE_TYPE_REQ				   (0x0UL &lt;&lt; 1)</div><div class='del'>-	#define CQ_BASE_CQE_TYPE_RES_RC			   (0x1UL &lt;&lt; 1)</div><div class='del'>-	#define CQ_BASE_CQE_TYPE_RES_UD			   (0x2UL &lt;&lt; 1)</div><div class='del'>-	#define CQ_BASE_CQE_TYPE_RES_RAWETH_QP1		   (0x3UL &lt;&lt; 1)</div><div class='del'>-	#define CQ_BASE_CQE_TYPE_TERMINAL			   (0xeUL &lt;&lt; 1)</div><div class='del'>-	#define CQ_BASE_CQE_TYPE_CUT_OFF			   (0xfUL &lt;&lt; 1)</div><div class='del'>-	#define CQ_BASE_RESERVED3_MASK				    0xe0UL</div><div class='del'>-	#define CQ_BASE_RESERVED3_SFT				    5</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 reserved16;</div><div class='del'>-	__le32 reserved32;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Requester CQ CQE (32 bytes) */</div><div class='del'>-struct cq_req {</div><div class='del'>-	__le64 qp_handle;</div><div class='del'>-	__le16 sq_cons_idx;</div><div class='del'>-	__le16 reserved16_1;</div><div class='del'>-	__le32 reserved32_2;</div><div class='del'>-	__le64 reserved64;</div><div class='del'>-	u8 cqe_type_toggle;</div><div class='del'>-	#define CQ_REQ_TOGGLE					    0x1UL</div><div class='del'>-	#define CQ_REQ_CQE_TYPE_MASK				    0x1eUL</div><div class='del'>-	#define CQ_REQ_CQE_TYPE_SFT				    1</div><div class='del'>-	#define CQ_REQ_CQE_TYPE_REQ				   (0x0UL &lt;&lt; 1)</div><div class='del'>-	#define CQ_REQ_RESERVED3_MASK				    0xe0UL</div><div class='del'>-	#define CQ_REQ_RESERVED3_SFT				    5</div><div class='del'>-	u8 status;</div><div class='del'>-	#define CQ_REQ_STATUS_OK				   0x0UL</div><div class='del'>-	#define CQ_REQ_STATUS_BAD_RESPONSE_ERR			   0x1UL</div><div class='del'>-	#define CQ_REQ_STATUS_LOCAL_LENGTH_ERR			   0x2UL</div><div class='del'>-	#define CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR		   0x3UL</div><div class='del'>-	#define CQ_REQ_STATUS_LOCAL_PROTECTION_ERR		   0x4UL</div><div class='del'>-	#define CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR		   0x5UL</div><div class='del'>-	#define CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR	   0x6UL</div><div class='del'>-	#define CQ_REQ_STATUS_REMOTE_ACCESS_ERR		   0x7UL</div><div class='del'>-	#define CQ_REQ_STATUS_REMOTE_OPERATION_ERR		   0x8UL</div><div class='del'>-	#define CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR		   0x9UL</div><div class='del'>-	#define CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR		   0xaUL</div><div class='del'>-	#define CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR		   0xbUL</div><div class='del'>-	__le16 reserved16_2;</div><div class='del'>-	__le32 reserved32_1;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Responder RC CQE (32 bytes) */</div><div class='del'>-struct cq_res_rc {</div><div class='del'>-	__le32 length;</div><div class='del'>-	__le32 imm_data_or_inv_r_key;</div><div class='del'>-	__le64 qp_handle;</div><div class='del'>-	__le64 mr_handle;</div><div class='del'>-	u8 cqe_type_toggle;</div><div class='del'>-	#define CQ_RES_RC_TOGGLE				    0x1UL</div><div class='del'>-	#define CQ_RES_RC_CQE_TYPE_MASK			    0x1eUL</div><div class='del'>-	#define CQ_RES_RC_CQE_TYPE_SFT				    1</div><div class='del'>-	#define CQ_RES_RC_CQE_TYPE_RES_RC			   (0x1UL &lt;&lt; 1)</div><div class='del'>-	#define CQ_RES_RC_RESERVED3_MASK			    0xe0UL</div><div class='del'>-	#define CQ_RES_RC_RESERVED3_SFT			    5</div><div class='del'>-	u8 status;</div><div class='del'>-	#define CQ_RES_RC_STATUS_OK				   0x0UL</div><div class='del'>-	#define CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR		   0x1UL</div><div class='del'>-	#define CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR		   0x2UL</div><div class='del'>-	#define CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR		   0x3UL</div><div class='del'>-	#define CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR	   0x4UL</div><div class='del'>-	#define CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR	   0x5UL</div><div class='del'>-	#define CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR       0x6UL</div><div class='del'>-	#define CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR	   0x7UL</div><div class='del'>-	#define CQ_RES_RC_STATUS_HW_FLUSH_ERR			   0x8UL</div><div class='del'>-	__le16 flags;</div><div class='del'>-	#define CQ_RES_RC_FLAGS_SRQ				    0x1UL</div><div class='del'>-	#define CQ_RES_RC_FLAGS_SRQ_RQ				   (0x0UL &lt;&lt; 0)</div><div class='del'>-	#define CQ_RES_RC_FLAGS_SRQ_SRQ			   (0x1UL &lt;&lt; 0)</div><div class='del'>-	#define CQ_RES_RC_FLAGS_SRQ_LAST    CQ_RES_RC_FLAGS_SRQ_SRQ</div><div class='del'>-	#define CQ_RES_RC_FLAGS_IMM				    0x2UL</div><div class='del'>-	#define CQ_RES_RC_FLAGS_INV				    0x4UL</div><div class='del'>-	#define CQ_RES_RC_FLAGS_RDMA				    0x8UL</div><div class='del'>-	#define CQ_RES_RC_FLAGS_RDMA_SEND			   (0x0UL &lt;&lt; 3)</div><div class='del'>-	#define CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE		   (0x1UL &lt;&lt; 3)</div><div class='del'>-	#define CQ_RES_RC_FLAGS_RDMA_LAST    CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE</div><div class='del'>-	__le32 srq_or_rq_wr_id;</div><div class='del'>-	#define CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK			    0xfffffUL</div><div class='del'>-	#define CQ_RES_RC_SRQ_OR_RQ_WR_ID_SFT			    0</div><div class='del'>-	#define CQ_RES_RC_RESERVED12_MASK			    0xfff00000UL</div><div class='del'>-	#define CQ_RES_RC_RESERVED12_SFT			    20</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Responder UD CQE (32 bytes) */</div><div class='del'>-struct cq_res_ud {</div><div class='del'>-	__le16 length;</div><div class='del'>-	#define CQ_RES_UD_LENGTH_MASK				    0x3fffUL</div><div class='del'>-	#define CQ_RES_UD_LENGTH_SFT				    0</div><div class='del'>-	__le16 cfa_metadata;</div><div class='del'>-	#define CQ_RES_UD_CFA_METADATA_VID_MASK			0xfffUL</div><div class='del'>-	#define CQ_RES_UD_CFA_METADATA_VID_SFT			0</div><div class='del'>-	#define CQ_RES_UD_CFA_METADATA_DE			0x1000UL</div><div class='del'>-	#define CQ_RES_UD_CFA_METADATA_PRI_MASK			0xe000UL</div><div class='del'>-	#define CQ_RES_UD_CFA_METADATA_PRI_SFT			13</div><div class='del'>-	__le32 imm_data;</div><div class='del'>-	__le64 qp_handle;</div><div class='del'>-	__le16 src_mac[3];</div><div class='del'>-	__le16 src_qp_low;</div><div class='del'>-	u8 cqe_type_toggle;</div><div class='del'>-	#define CQ_RES_UD_TOGGLE				   0x1UL</div><div class='del'>-	#define CQ_RES_UD_CQE_TYPE_MASK				   0x1eUL</div><div class='del'>-	#define CQ_RES_UD_CQE_TYPE_SFT				   1</div><div class='del'>-	#define CQ_RES_UD_CQE_TYPE_RES_UD			   (0x2UL &lt;&lt; 1)</div><div class='del'>-	u8 status;</div><div class='del'>-	#define CQ_RES_UD_STATUS_OK				   0x0UL</div><div class='del'>-	#define CQ_RES_UD_STATUS_LOCAL_ACCESS_ERROR		   0x1UL</div><div class='del'>-	#define CQ_RES_UD_STATUS_HW_LOCAL_LENGTH_ERR		   0x2UL</div><div class='del'>-	#define CQ_RES_UD_STATUS_LOCAL_PROTECTION_ERR		   0x3UL</div><div class='del'>-	#define CQ_RES_UD_STATUS_LOCAL_QP_OPERATION_ERR	   0x4UL</div><div class='del'>-	#define CQ_RES_UD_STATUS_MEMORY_MGT_OPERATION_ERR	   0x5UL</div><div class='del'>-	#define CQ_RES_UD_STATUS_WORK_REQUEST_FLUSHED_ERR	   0x7UL</div><div class='del'>-	#define CQ_RES_UD_STATUS_HW_FLUSH_ERR			   0x8UL</div><div class='del'>-	__le16 flags;</div><div class='del'>-	#define CQ_RES_UD_FLAGS_SRQ				    0x1UL</div><div class='del'>-	#define CQ_RES_UD_FLAGS_SRQ_RQ				   (0x0UL &lt;&lt; 0)</div><div class='del'>-	#define CQ_RES_UD_FLAGS_SRQ_SRQ			   (0x1UL &lt;&lt; 0)</div><div class='del'>-	#define CQ_RES_UD_FLAGS_SRQ_LAST    CQ_RES_UD_FLAGS_SRQ_SRQ</div><div class='del'>-	#define CQ_RES_UD_FLAGS_IMM				    0x2UL</div><div class='del'>-	#define CQ_RES_UD_FLAGS_UNUSED_MASK			0xcUL</div><div class='del'>-	#define CQ_RES_UD_FLAGS_UNUSED_SFT			2</div><div class='del'>-	#define CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK		0x30UL</div><div class='del'>-	#define CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT			4</div><div class='del'>-	#define CQ_RES_UD_FLAGS_ROCE_IP_VER_V1			(0x0UL &lt;&lt; 4)</div><div class='del'>-	#define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV4		(0x2UL &lt;&lt; 4)</div><div class='del'>-	#define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6		(0x3UL &lt;&lt; 4)</div><div class='del'>-	#define CQ_RES_UD_FLAGS_ROCE_IP_VER_LAST		\</div><div class='del'>-					CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6</div><div class='del'>-	#define CQ_RES_UD_FLAGS_META_FORMAT_MASK		0x3c0UL</div><div class='del'>-	#define CQ_RES_UD_FLAGS_META_FORMAT_SFT			6</div><div class='del'>-	#define CQ_RES_UD_FLAGS_META_FORMAT_NONE		(0x0UL &lt;&lt; 6)</div><div class='del'>-	#define CQ_RES_UD_FLAGS_META_FORMAT_VLAN		(0x1UL &lt;&lt; 6)</div><div class='del'>-	#define CQ_RES_UD_FLAGS_META_FORMAT_TUNNEL_ID		(0x2UL &lt;&lt; 6)</div><div class='del'>-	#define CQ_RES_UD_FLAGS_META_FORMAT_CHDR_DATA		(0x3UL &lt;&lt; 6)</div><div class='del'>-	#define CQ_RES_UD_FLAGS_META_FORMAT_HDR_OFFSET		(0x4UL &lt;&lt; 6)</div><div class='del'>-	#define CQ_RES_UD_FLAGS_META_FORMAT_LAST		\</div><div class='del'>-					CQ_RES_UD_FLAGS_META_FORMAT_HDR_OFFSET</div><div class='del'>-	#define CQ_RES_UD_FLAGS_EXT_META_FORMAT_MASK		0xc00UL</div><div class='del'>-	#define CQ_RES_UD_FLAGS_EXT_META_FORMAT_SFT		10</div><div class='del'>-</div><div class='del'>-	__le32 src_qp_high_srq_or_rq_wr_id;</div><div class='del'>-	#define CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK			    0xfffffUL</div><div class='del'>-	#define CQ_RES_UD_SRQ_OR_RQ_WR_ID_SFT			    0</div><div class='del'>-	#define CQ_RES_UD_SRC_QP_HIGH_MASK			    0xff000000UL</div><div class='del'>-	#define CQ_RES_UD_SRC_QP_HIGH_SFT			    24</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Responder RawEth and QP1 CQE (32 bytes) */</div><div class='del'>-struct cq_res_raweth_qp1 {</div><div class='del'>-	__le16 length;</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_LENGTH_MASK			    0x3fffUL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_LENGTH_SFT			    0</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RESERVED2_MASK		    0xc000UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RESERVED2_SFT		    14</div><div class='del'>-	__le16 raweth_qp1_flags;</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ERROR	    0x1UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_RESERVED5_1_MASK 0x3eUL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_RESERVED5_1_SFT 1</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_MASK      0x3c0UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_SFT       6</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_NOT_KNOWN (0x0UL &lt;&lt; 6)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_IP       (0x1UL &lt;&lt; 6)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_TCP      (0x2UL &lt;&lt; 6)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_UDP      (0x3UL &lt;&lt; 6)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_FCOE     (0x4UL &lt;&lt; 6)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE     (0x5UL &lt;&lt; 6)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ICMP     (0x7UL &lt;&lt; 6)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_WO_TIMESTAMP \</div><div class='del'>-								 (0x8UL &lt;&lt; 6)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP \</div><div class='del'>-								 (0x9UL &lt;&lt; 6)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_LAST	\</div><div class='del'>-		CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_MASK	    0x3ffUL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_SFT		    0</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RESERVED6_MASK		    0xfc00UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RESERVED6_SFT		    10</div><div class='del'>-	__le16 raweth_qp1_errors;</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_RESERVED4_MASK 0xfUL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_RESERVED4_SFT  0</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_IP_CS_ERROR    0x10UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_L4_CS_ERROR    0x20UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_IP_CS_ERROR  0x40UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_L4_CS_ERROR  0x80UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_CRC_ERROR      0x100UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_MASK 0xe00UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_SFT 9</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_NO_ERROR \</div><div class='del'>-								(0x0UL &lt;&lt; 9)</div><div class='del'>-	#define \</div><div class='del'>-	   CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION \</div><div class='del'>-								(0x1UL &lt;&lt; 9)</div><div class='del'>-	#define \</div><div class='del'>-	   CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN \</div><div class='del'>-								(0x2UL &lt;&lt; 9)</div><div class='del'>-	#define \</div><div class='del'>-	   CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR \</div><div class='del'>-								(0x3UL &lt;&lt; 9)</div><div class='del'>-	#define \</div><div class='del'>-	   CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR \</div><div class='del'>-								(0x4UL &lt;&lt; 9)</div><div class='del'>-	#define \</div><div class='del'>-	   CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR \</div><div class='del'>-								(0x5UL &lt;&lt; 9)</div><div class='del'>-	#define \</div><div class='del'>-	   CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL \</div><div class='del'>-								(0x6UL &lt;&lt; 9)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_LAST \</div><div class='del'>-		CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_MASK 0xf000UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_SFT  12</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_NO_ERROR \</div><div class='del'>-								(0x0UL &lt;&lt; 12)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_VERSION \</div><div class='del'>-								(0x1UL &lt;&lt; 12)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN \</div><div class='del'>-								 (0x2UL &lt;&lt; 12)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_TTL \</div><div class='del'>-								 (0x3UL &lt;&lt; 12)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_IP_TOTAL_ERROR \</div><div class='del'>-								 (0x4UL &lt;&lt; 12)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR \</div><div class='del'>-								 (0x5UL &lt;&lt; 12)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN \</div><div class='del'>-								 (0x6UL &lt;&lt; 12)</div><div class='del'>-	#define \</div><div class='del'>-	 CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL\</div><div class='del'>-								 (0x7UL &lt;&lt; 12)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN \</div><div class='del'>-								 (0x8UL &lt;&lt; 12)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_LAST \</div><div class='del'>-		CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN</div><div class='del'>-	__le16 raweth_qp1_cfa_code;</div><div class='del'>-	__le64 qp_handle;</div><div class='del'>-	__le32 raweth_qp1_flags2;</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC     0x1UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC     0x2UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_IP_CS_CALC   0x4UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_L4_CS_CALC   0x8UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_MASK 0xf0UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_SFT 4</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_NONE \</div><div class='del'>-								(0x0UL &lt;&lt; 4)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN \</div><div class='del'>-								(0x1UL &lt;&lt; 4)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_LAST\</div><div class='del'>-			CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE	    0x100UL</div><div class='del'>-	__le32 raweth_qp1_metadata;</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK     0xfffUL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_SFT      0</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_DE	    0x1000UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK     0xe000UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT      13</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK    0xffff0000UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT     16</div><div class='del'>-	u8 cqe_type_toggle;</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_TOGGLE			    0x1UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_CQE_TYPE_MASK		    0x1eUL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_CQE_TYPE_SFT			    1</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_CQE_TYPE_RES_RAWETH_QP1	   (0x3UL &lt;&lt; 1)</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RESERVED3_MASK		    0xe0UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RESERVED3_SFT		    5</div><div class='del'>-	u8 status;</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_STATUS_OK			   0x0UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR       0x1UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR      0x2UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR     0x3UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR   0x4UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR		   0x8UL</div><div class='del'>-	__le16 flags;</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_FLAGS_SRQ			    0x1UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_FLAGS_SRQ_RQ			   0x0UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ		   0x1UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_FLAGS_SRQ_LAST \</div><div class='del'>-					CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ</div><div class='del'>-	__le32 raweth_qp1_payload_offset_srq_or_rq_wr_id;</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK		    0xfffffUL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_SFT		    0</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RESERVED4_MASK		    0xf00000UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RESERVED4_SFT		    20</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_MASK   0xff000000UL</div><div class='del'>-	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_SFT    24</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Terminal CQE (32 bytes) */</div><div class='del'>-struct cq_terminal {</div><div class='del'>-	__le64 qp_handle;</div><div class='del'>-	__le16 sq_cons_idx;</div><div class='del'>-	__le16 rq_cons_idx;</div><div class='del'>-	__le32 reserved32_1;</div><div class='del'>-	__le64 reserved64_3;</div><div class='del'>-	u8 cqe_type_toggle;</div><div class='del'>-	#define CQ_TERMINAL_TOGGLE				    0x1UL</div><div class='del'>-	#define CQ_TERMINAL_CQE_TYPE_MASK			    0x1eUL</div><div class='del'>-	#define CQ_TERMINAL_CQE_TYPE_SFT			    1</div><div class='del'>-	#define CQ_TERMINAL_CQE_TYPE_TERMINAL			   (0xeUL &lt;&lt; 1)</div><div class='del'>-	#define CQ_TERMINAL_RESERVED3_MASK			    0xe0UL</div><div class='del'>-	#define CQ_TERMINAL_RESERVED3_SFT			    5</div><div class='del'>-	u8 status;</div><div class='del'>-	#define CQ_TERMINAL_STATUS_OK				   0x0UL</div><div class='del'>-	__le16 reserved16;</div><div class='del'>-	__le32 reserved32_2;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Cutoff CQE (32 bytes) */</div><div class='del'>-struct cq_cutoff {</div><div class='del'>-	__le64 reserved64_1;</div><div class='del'>-	__le64 reserved64_2;</div><div class='del'>-	__le64 reserved64_3;</div><div class='del'>-	u8 cqe_type_toggle;</div><div class='del'>-	#define CQ_CUTOFF_TOGGLE				    0x1UL</div><div class='del'>-	#define CQ_CUTOFF_CQE_TYPE_MASK			    0x1eUL</div><div class='del'>-	#define CQ_CUTOFF_CQE_TYPE_SFT				    1</div><div class='del'>-	#define CQ_CUTOFF_CQE_TYPE_CUT_OFF			   (0xfUL &lt;&lt; 1)</div><div class='del'>-	#define CQ_CUTOFF_RESERVED3_MASK			    0xe0UL</div><div class='del'>-	#define CQ_CUTOFF_RESERVED3_SFT			    5</div><div class='del'>-	u8 status;</div><div class='del'>-	#define CQ_CUTOFF_STATUS_OK				   0x0UL</div><div class='del'>-	__le16 reserved16;</div><div class='del'>-	__le32 reserved32;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Notification Queue (NQ) Structures */</div><div class='del'>-/* Base NQ Record (16 bytes) */</div><div class='del'>-struct nq_base {</div><div class='del'>-	__le16 info10_type;</div><div class='del'>-	#define NQ_BASE_TYPE_MASK				    0x3fUL</div><div class='del'>-	#define NQ_BASE_TYPE_SFT				    0</div><div class='del'>-	#define NQ_BASE_TYPE_CQ_NOTIFICATION			   0x30UL</div><div class='del'>-	#define NQ_BASE_TYPE_SRQ_EVENT				   0x32UL</div><div class='del'>-	#define NQ_BASE_TYPE_DBQ_EVENT				   0x34UL</div><div class='del'>-	#define NQ_BASE_TYPE_QP_EVENT				   0x38UL</div><div class='del'>-	#define NQ_BASE_TYPE_FUNC_EVENT			   0x3aUL</div><div class='del'>-	#define NQ_BASE_INFO10_MASK				    0xffc0UL</div><div class='del'>-	#define NQ_BASE_INFO10_SFT				    6</div><div class='del'>-	__le16 info16;</div><div class='del'>-	__le32 info32;</div><div class='del'>-	__le32 info63_v[2];</div><div class='del'>-	#define NQ_BASE_V					    0x1UL</div><div class='del'>-	#define NQ_BASE_INFO63_MASK				    0xfffffffeUL</div><div class='del'>-	#define NQ_BASE_INFO63_SFT				    1</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Completion Queue Notification (16 bytes) */</div><div class='del'>-struct nq_cn {</div><div class='del'>-	__le16 type;</div><div class='del'>-	#define NQ_CN_TYPE_MASK				    0x3fUL</div><div class='del'>-	#define NQ_CN_TYPE_SFT					    0</div><div class='del'>-	#define NQ_CN_TYPE_CQ_NOTIFICATION			   0x30UL</div><div class='del'>-	#define NQ_CN_RESERVED9_MASK				    0xffc0UL</div><div class='del'>-	#define NQ_CN_RESERVED9_SFT				    6</div><div class='del'>-	__le16 reserved16;</div><div class='del'>-	__le32 cq_handle_low;</div><div class='del'>-	__le32 v;</div><div class='del'>-	#define NQ_CN_V					    0x1UL</div><div class='del'>-	#define NQ_CN_RESERVED31_MASK				    0xfffffffeUL</div><div class='del'>-	#define NQ_CN_RESERVED31_SFT				    1</div><div class='del'>-	__le32 cq_handle_high;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* SRQ Event Notification (16 bytes) */</div><div class='del'>-struct nq_srq_event {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define NQ_SRQ_EVENT_TYPE_MASK				    0x3fUL</div><div class='del'>-	#define NQ_SRQ_EVENT_TYPE_SFT				    0</div><div class='del'>-	#define NQ_SRQ_EVENT_TYPE_SRQ_EVENT			   0x32UL</div><div class='del'>-	#define NQ_SRQ_EVENT_RESERVED1_MASK			    0xc0UL</div><div class='del'>-	#define NQ_SRQ_EVENT_RESERVED1_SFT			    6</div><div class='del'>-	u8 event;</div><div class='del'>-	#define NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT		   0x1UL</div><div class='del'>-	__le16 reserved16;</div><div class='del'>-	__le32 srq_handle_low;</div><div class='del'>-	__le32 v;</div><div class='del'>-	#define NQ_SRQ_EVENT_V					    0x1UL</div><div class='del'>-	#define NQ_SRQ_EVENT_RESERVED31_MASK			    0xfffffffeUL</div><div class='del'>-	#define NQ_SRQ_EVENT_RESERVED31_SFT			    1</div><div class='del'>-	__le32 srq_handle_high;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* DBQ Async Event Notification (16 bytes) */</div><div class='del'>-struct nq_dbq_event {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define NQ_DBQ_EVENT_TYPE_MASK				    0x3fUL</div><div class='del'>-	#define NQ_DBQ_EVENT_TYPE_SFT				    0</div><div class='del'>-	#define NQ_DBQ_EVENT_TYPE_DBQ_EVENT			   0x34UL</div><div class='del'>-	#define NQ_DBQ_EVENT_RESERVED1_MASK			    0xc0UL</div><div class='del'>-	#define NQ_DBQ_EVENT_RESERVED1_SFT			    6</div><div class='del'>-	u8 event;</div><div class='del'>-	#define NQ_DBQ_EVENT_EVENT_DBQ_THRESHOLD_EVENT		   0x1UL</div><div class='del'>-	__le16 db_pfid;</div><div class='del'>-	#define NQ_DBQ_EVENT_DB_PFID_MASK			    0xfUL</div><div class='del'>-	#define NQ_DBQ_EVENT_DB_PFID_SFT			    0</div><div class='del'>-	#define NQ_DBQ_EVENT_RESERVED12_MASK			    0xfff0UL</div><div class='del'>-	#define NQ_DBQ_EVENT_RESERVED12_SFT			    4</div><div class='del'>-	__le32 db_dpi;</div><div class='del'>-	#define NQ_DBQ_EVENT_DB_DPI_MASK			    0xfffffUL</div><div class='del'>-	#define NQ_DBQ_EVENT_DB_DPI_SFT			    0</div><div class='del'>-	#define NQ_DBQ_EVENT_RESERVED12_2_MASK			    0xfff00000UL</div><div class='del'>-	#define NQ_DBQ_EVENT_RESERVED12_2_SFT			    20</div><div class='del'>-	__le32 v;</div><div class='del'>-	#define NQ_DBQ_EVENT_V					    0x1UL</div><div class='del'>-	#define NQ_DBQ_EVENT_RESERVED32_MASK			    0xfffffffeUL</div><div class='del'>-	#define NQ_DBQ_EVENT_RESERVED32_SFT			    1</div><div class='del'>-	__le32 db_type_db_xid;</div><div class='del'>-	#define NQ_DBQ_EVENT_DB_XID_MASK			    0xfffffUL</div><div class='del'>-	#define NQ_DBQ_EVENT_DB_XID_SFT			    0</div><div class='del'>-	#define NQ_DBQ_EVENT_RESERVED8_MASK			    0xff00000UL</div><div class='del'>-	#define NQ_DBQ_EVENT_RESERVED8_SFT			    20</div><div class='del'>-	#define NQ_DBQ_EVENT_DB_TYPE_MASK			    0xf0000000UL</div><div class='del'>-	#define NQ_DBQ_EVENT_DB_TYPE_SFT			    28</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Read Request/Response Queue Structures */</div><div class='del'>-/* Input Read Request Queue (IRRQ) Message (32 bytes) */</div><div class='del'>-struct xrrq_irrq {</div><div class='del'>-	__le16 credits_type;</div><div class='del'>-	#define XRRQ_IRRQ_TYPE					    0x1UL</div><div class='del'>-	#define XRRQ_IRRQ_TYPE_READ_REQ			   0x0UL</div><div class='del'>-	#define XRRQ_IRRQ_TYPE_ATOMIC_REQ			   0x1UL</div><div class='del'>-	#define XRRQ_IRRQ_RESERVED10_MASK			    0x7feUL</div><div class='del'>-	#define XRRQ_IRRQ_RESERVED10_SFT			    1</div><div class='del'>-	#define XRRQ_IRRQ_CREDITS_MASK				    0xf800UL</div><div class='del'>-	#define XRRQ_IRRQ_CREDITS_SFT				    11</div><div class='del'>-	__le16 reserved16;</div><div class='del'>-	__le32 reserved32;</div><div class='del'>-	__le32 psn;</div><div class='del'>-	#define XRRQ_IRRQ_PSN_MASK				    0xffffffUL</div><div class='del'>-	#define XRRQ_IRRQ_PSN_SFT				    0</div><div class='del'>-	#define XRRQ_IRRQ_RESERVED8_1_MASK			    0xff000000UL</div><div class='del'>-	#define XRRQ_IRRQ_RESERVED8_1_SFT			    24</div><div class='del'>-	__le32 msn;</div><div class='del'>-	#define XRRQ_IRRQ_MSN_MASK				    0xffffffUL</div><div class='del'>-	#define XRRQ_IRRQ_MSN_SFT				    0</div><div class='del'>-	#define XRRQ_IRRQ_RESERVED8_2_MASK			    0xff000000UL</div><div class='del'>-	#define XRRQ_IRRQ_RESERVED8_2_SFT			    24</div><div class='del'>-	__le64 va_or_atomic_result;</div><div class='del'>-	__le32 rdma_r_key;</div><div class='del'>-	__le32 length;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Output Read Request Queue (ORRQ) Message (32 bytes) */</div><div class='del'>-struct xrrq_orrq {</div><div class='del'>-	__le16 num_sges_type;</div><div class='del'>-	#define XRRQ_ORRQ_TYPE					    0x1UL</div><div class='del'>-	#define XRRQ_ORRQ_TYPE_READ_REQ			   0x0UL</div><div class='del'>-	#define XRRQ_ORRQ_TYPE_ATOMIC_REQ			   0x1UL</div><div class='del'>-	#define XRRQ_ORRQ_RESERVED10_MASK			    0x7feUL</div><div class='del'>-	#define XRRQ_ORRQ_RESERVED10_SFT			    1</div><div class='del'>-	#define XRRQ_ORRQ_NUM_SGES_MASK			    0xf800UL</div><div class='del'>-	#define XRRQ_ORRQ_NUM_SGES_SFT				    11</div><div class='del'>-	__le16 reserved16;</div><div class='del'>-	__le32 length;</div><div class='del'>-	__le32 psn;</div><div class='del'>-	#define XRRQ_ORRQ_PSN_MASK				    0xffffffUL</div><div class='del'>-	#define XRRQ_ORRQ_PSN_SFT				    0</div><div class='del'>-	#define XRRQ_ORRQ_RESERVED8_1_MASK			    0xff000000UL</div><div class='del'>-	#define XRRQ_ORRQ_RESERVED8_1_SFT			    24</div><div class='del'>-	__le32 end_psn;</div><div class='del'>-	#define XRRQ_ORRQ_END_PSN_MASK				    0xffffffUL</div><div class='del'>-	#define XRRQ_ORRQ_END_PSN_SFT				    0</div><div class='del'>-	#define XRRQ_ORRQ_RESERVED8_2_MASK			    0xff000000UL</div><div class='del'>-	#define XRRQ_ORRQ_RESERVED8_2_SFT			    24</div><div class='del'>-	__le64 first_sge_phy_or_sing_sge_va;</div><div class='del'>-	__le32 single_sge_l_key;</div><div class='del'>-	__le32 single_sge_size;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Page Buffer List Memory Structures (PBL) */</div><div class='del'>-/* Page Table Entry (PTE) (8 bytes) */</div><div class='del'>-struct ptu_pte {</div><div class='del'>-	__le32 page_next_to_last_last_valid[2];</div><div class='del'>-	#define PTU_PTE_VALID					    0x1UL</div><div class='del'>-	#define PTU_PTE_LAST					    0x2UL</div><div class='del'>-	#define PTU_PTE_NEXT_TO_LAST				    0x4UL</div><div class='del'>-	#define PTU_PTE_PAGE_MASK				    0xfffff000UL</div><div class='del'>-	#define PTU_PTE_PAGE_SFT				    12</div><div class='add'>+/* creq_base (size:128b/16B) */</div><div class='add'>+struct creq_base {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_BASE_TYPE_MASK      0x3fUL</div><div class='add'>+	#define CREQ_BASE_TYPE_SFT       0</div><div class='add'>+	#define CREQ_BASE_TYPE_QP_EVENT    0x38UL</div><div class='add'>+	#define CREQ_BASE_TYPE_FUNC_EVENT  0x3aUL</div><div class='add'>+	#define CREQ_BASE_TYPE_LAST       CREQ_BASE_TYPE_FUNC_EVENT</div><div class='add'>+	u8	reserved56[7];</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_BASE_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	u8	reserved48[6];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* Page Directory Entry (PDE) (8 bytes) */</div><div class='del'>-struct ptu_pde {</div><div class='del'>-	__le32 page_valid[2];</div><div class='del'>-	#define PTU_PDE_VALID					    0x1UL</div><div class='del'>-	#define PTU_PDE_PAGE_MASK				    0xfffff000UL</div><div class='del'>-	#define PTU_PDE_PAGE_SFT				    12</div><div class='add'>+/* cmdq_query_version (size:128b/16B) */</div><div class='add'>+struct cmdq_query_version {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_QUERY_VERSION_OPCODE_QUERY_VERSION 0x8bUL</div><div class='add'>+	#define CMDQ_QUERY_VERSION_OPCODE_LAST         CMDQ_QUERY_VERSION_OPCODE_QUERY_VERSION</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* RoCE Fastpath Host Structures */</div><div class='del'>-/* Command Queue (CMDQ) Interface */</div><div class='del'>-/* Init CMDQ (16 bytes) */</div><div class='del'>-struct cmdq_init {</div><div class='del'>-	__le64 cmdq_pbl;</div><div class='del'>-	__le16 cmdq_size_cmdq_lvl;</div><div class='del'>-	#define CMDQ_INIT_CMDQ_LVL_MASK			    0x3UL</div><div class='del'>-	#define CMDQ_INIT_CMDQ_LVL_SFT				    0</div><div class='del'>-	#define CMDQ_INIT_CMDQ_SIZE_MASK			    0xfffcUL</div><div class='del'>-	#define CMDQ_INIT_CMDQ_SIZE_SFT			    2</div><div class='del'>-	__le16 creq_ring_id;</div><div class='del'>-	__le32 prod_idx;</div><div class='add'>+/* creq_query_version_resp (size:128b/16B) */</div><div class='add'>+struct creq_query_version_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_QUERY_VERSION_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_QUERY_VERSION_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_QUERY_VERSION_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_QUERY_VERSION_RESP_TYPE_LAST     CREQ_QUERY_VERSION_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	fw_maj;</div><div class='add'>+	u8	fw_minor;</div><div class='add'>+	u8	fw_bld;</div><div class='add'>+	u8	fw_rsvd;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_QUERY_VERSION_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_QUERY_VERSION_RESP_EVENT_QUERY_VERSION 0x8bUL</div><div class='add'>+	#define CREQ_QUERY_VERSION_RESP_EVENT_LAST         \</div><div class='add'>+		CREQ_QUERY_VERSION_RESP_EVENT_QUERY_VERSION</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	u8	intf_maj;</div><div class='add'>+	u8	intf_minor;</div><div class='add'>+	u8	intf_bld;</div><div class='add'>+	u8	intf_rsvd;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* Update CMDQ producer index (16 bytes) */</div><div class='del'>-struct cmdq_update {</div><div class='del'>-	__le64 reserved64;</div><div class='del'>-	__le32 reserved32;</div><div class='del'>-	__le32 prod_idx;</div><div class='add'>+/* cmdq_initialize_fw (size:896b/112B) */</div><div class='add'>+struct cmdq_initialize_fw {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_OPCODE_INITIALIZE_FW 0x80UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_OPCODE_LAST         CMDQ_INITIALIZE_FW_OPCODE_INITIALIZE_FW</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_FLAGS_MRAV_RESERVATION_SPLIT          0x1UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED     0x2UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_FLAGS_OPTIMIZE_MODIFY_QP_SUPPORTED    0x8UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_FLAGS_L2_VF_RESOURCE_MGMT		 0x10UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_FLAGS_MIRROR_ON_ROCE_SUPPORTED        0x80UL</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	u8	qpc_pg_size_qpc_lvl;</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_QPC_LVL_MASK      0xfUL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_QPC_LVL_SFT       0</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_0       0x0UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_1       0x1UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_2       0x2UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_QPC_LVL_LAST       CMDQ_INITIALIZE_FW_QPC_LVL_LVL_2</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_MASK  0xf0UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT   4</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K   (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K   (0x1UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K  (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M   (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M   (0x4UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G   (0x5UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_LAST   CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G</div><div class='add'>+	u8	mrw_pg_size_mrw_lvl;</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_MRW_LVL_MASK      0xfUL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_MRW_LVL_SFT       0</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_0       0x0UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_1       0x1UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_2       0x2UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_MRW_LVL_LAST       CMDQ_INITIALIZE_FW_MRW_LVL_LVL_2</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_MASK  0xf0UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT   4</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_4K   (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8K   (0x1UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_64K  (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_2M   (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8M   (0x4UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_1G   (0x5UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_LAST   CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_1G</div><div class='add'>+	u8	srq_pg_size_srq_lvl;</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_SRQ_LVL_MASK      0xfUL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_SRQ_LVL_SFT       0</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_0       0x0UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_1       0x1UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_2       0x2UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_SRQ_LVL_LAST       CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_2</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_MASK  0xf0UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT   4</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_4K   (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8K   (0x1UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_64K  (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_2M   (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8M   (0x4UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_1G   (0x5UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_LAST   CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_1G</div><div class='add'>+	u8	cq_pg_size_cq_lvl;</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_CQ_LVL_MASK      0xfUL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_CQ_LVL_SFT       0</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_0       0x0UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_1       0x1UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_2       0x2UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_CQ_LVL_LAST       CMDQ_INITIALIZE_FW_CQ_LVL_LVL_2</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_MASK  0xf0UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT   4</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_4K   (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8K   (0x1UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_64K  (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_2M   (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8M   (0x4UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_1G   (0x5UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_LAST   CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_1G</div><div class='add'>+	u8	tqm_pg_size_tqm_lvl;</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TQM_LVL_MASK      0xfUL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TQM_LVL_SFT       0</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_0       0x0UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_1       0x1UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_2       0x2UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TQM_LVL_LAST       CMDQ_INITIALIZE_FW_TQM_LVL_LVL_2</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_MASK  0xf0UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT   4</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_4K   (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8K   (0x1UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_64K  (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_2M   (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8M   (0x4UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_1G   (0x5UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_LAST   CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_1G</div><div class='add'>+	u8	tim_pg_size_tim_lvl;</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TIM_LVL_MASK      0xfUL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TIM_LVL_SFT       0</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_0       0x0UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_1       0x1UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_2       0x2UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TIM_LVL_LAST       CMDQ_INITIALIZE_FW_TIM_LVL_LVL_2</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_MASK  0xf0UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT   4</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_4K   (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8K   (0x1UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_64K  (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_2M   (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8M   (0x4UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_1G   (0x5UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_LAST   CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_1G</div><div class='add'>+	__le16	log2_dbr_pg_size;</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_MASK   0xfUL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_SFT    0</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_4K    0x0UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_8K    0x1UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_16K   0x2UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_32K   0x3UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_64K   0x4UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128K  0x5UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_256K  0x6UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_512K  0x7UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_1M    0x8UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_2M    0x9UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_4M    0xaUL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_8M    0xbUL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_16M   0xcUL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_32M   0xdUL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_64M   0xeUL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128M  0xfUL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_LAST    \</div><div class='add'>+		CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128M</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_RSVD_MASK               0xfff0UL</div><div class='add'>+	#define CMDQ_INITIALIZE_FW_RSVD_SFT                4</div><div class='add'>+	__le64	qpc_page_dir;</div><div class='add'>+	__le64	mrw_page_dir;</div><div class='add'>+	__le64	srq_page_dir;</div><div class='add'>+	__le64	cq_page_dir;</div><div class='add'>+	__le64	tqm_page_dir;</div><div class='add'>+	__le64	tim_page_dir;</div><div class='add'>+	__le32	number_of_qp;</div><div class='add'>+	__le32	number_of_mrw;</div><div class='add'>+	__le32	number_of_srq;</div><div class='add'>+	__le32	number_of_cq;</div><div class='add'>+	__le32	max_qp_per_vf;</div><div class='add'>+	__le32	max_mrw_per_vf;</div><div class='add'>+	__le32	max_srq_per_vf;</div><div class='add'>+	__le32	max_cq_per_vf;</div><div class='add'>+	__le32	max_gid_per_vf;</div><div class='add'>+	__le32	stat_ctx_id;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* CMDQ common header structure (16 bytes) */</div><div class='del'>-struct cmdq_base {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_BASE_OPCODE_CREATE_QP			   0x1UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_DESTROY_QP			   0x2UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_MODIFY_QP			   0x3UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_QUERY_QP			   0x4UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_CREATE_SRQ			   0x5UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_DESTROY_SRQ			   0x6UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_QUERY_SRQ			   0x8UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_CREATE_CQ			   0x9UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_DESTROY_CQ			   0xaUL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_RESIZE_CQ			   0xcUL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_ALLOCATE_MRW			   0xdUL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_DEALLOCATE_KEY		   0xeUL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_REGISTER_MR			   0xfUL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_DEREGISTER_MR			   0x10UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_ADD_GID			   0x11UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_DELETE_GID			   0x12UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_MODIFY_GID			   0x17UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_QUERY_GID			   0x18UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_CREATE_QP1			   0x13UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_DESTROY_QP1			   0x14UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_CREATE_AH			   0x15UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_DESTROY_AH			   0x16UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_INITIALIZE_FW			   0x80UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_DEINITIALIZE_FW		   0x81UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_STOP_FUNC			   0x82UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_QUERY_FUNC			   0x83UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES		   0x84UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_READ_CONTEXT			   0x85UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_VF_BACKCHANNEL_REQUEST	   0x86UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_READ_VF_MEMORY		   0x87UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_COMPLETE_VF_REQUEST		   0x88UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_EXTEND_CONTEXT_ARRRAY		   0x89UL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_MAP_TC_TO_COS			   0x8aUL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_QUERY_VERSION			   0x8bUL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_MODIFY_CC			   0x8cUL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_QUERY_CC			   0x8dUL</div><div class='del'>-	#define CMDQ_BASE_OPCODE_QUERY_ROCE_STATS	   0x8eUL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Create QP command (96 bytes) */</div><div class='add'>+/* creq_initialize_fw_resp (size:128b/16B) */</div><div class='add'>+struct creq_initialize_fw_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_INITIALIZE_FW_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_INITIALIZE_FW_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_INITIALIZE_FW_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_INITIALIZE_FW_RESP_TYPE_LAST     CREQ_INITIALIZE_FW_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_INITIALIZE_FW_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_INITIALIZE_FW_RESP_EVENT_INITIALIZE_FW 0x80UL</div><div class='add'>+	#define CREQ_INITIALIZE_FW_RESP_EVENT_LAST         \</div><div class='add'>+		CREQ_INITIALIZE_FW_RESP_EVENT_INITIALIZE_FW</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_deinitialize_fw (size:128b/16B) */</div><div class='add'>+struct cmdq_deinitialize_fw {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_DEINITIALIZE_FW_OPCODE_DEINITIALIZE_FW 0x81UL</div><div class='add'>+	#define CMDQ_DEINITIALIZE_FW_OPCODE_LAST           \</div><div class='add'>+		CMDQ_DEINITIALIZE_FW_OPCODE_DEINITIALIZE_FW</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_deinitialize_fw_resp (size:128b/16B) */</div><div class='add'>+struct creq_deinitialize_fw_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_DEINITIALIZE_FW_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_DEINITIALIZE_FW_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_DEINITIALIZE_FW_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_DEINITIALIZE_FW_RESP_TYPE_LAST     CREQ_DEINITIALIZE_FW_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_DEINITIALIZE_FW_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_DEINITIALIZE_FW_RESP_EVENT_DEINITIALIZE_FW 0x81UL</div><div class='add'>+	#define CREQ_DEINITIALIZE_FW_RESP_EVENT_LAST           \</div><div class='add'>+		CREQ_DEINITIALIZE_FW_RESP_EVENT_DEINITIALIZE_FW</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_create_qp (size:832b/104B) */</div><div class='ctx'> struct cmdq_create_qp {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_CREATE_QP_OPCODE_CREATE_QP		   0x1UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le64 qp_handle;</div><div class='del'>-	__le32 qp_flags;</div><div class='del'>-	#define CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED		   0x1UL</div><div class='del'>-	#define CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION	   0x2UL</div><div class='del'>-	#define CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE      0x4UL</div><div class='del'>-	#define CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED		   0x8UL</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CMDQ_CREATE_QP_TYPE_RC				   0x2UL</div><div class='del'>-	#define CMDQ_CREATE_QP_TYPE_UD				   0x4UL</div><div class='del'>-	#define CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE		   0x6UL</div><div class='del'>-	#define CMDQ_CREATE_QP_TYPE_GSI				   0x7UL</div><div class='del'>-	u8 sq_pg_size_sq_lvl;</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_LVL_MASK			    0xfUL</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_LVL_SFT			    0</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_LVL_LVL_0			   0x0UL</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_LVL_LVL_1			   0x1UL</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_LVL_LVL_2			   0x2UL</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_PG_SIZE_MASK			    0xf0UL</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_PG_SIZE_SFT			    4</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K		   (0x0UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K		   (0x1UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K		   (0x2UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M		   (0x3UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M		   (0x4UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G		   (0x5UL &lt;&lt; 4)</div><div class='del'>-	u8 rq_pg_size_rq_lvl;</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_LVL_MASK			    0xfUL</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_LVL_SFT			    0</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_LVL_LVL_0			   0x0UL</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_LVL_LVL_1			   0x1UL</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_LVL_LVL_2			   0x2UL</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_PG_SIZE_MASK			    0xf0UL</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_PG_SIZE_SFT			    4</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K		   (0x0UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K		   (0x1UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K		   (0x2UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M		   (0x3UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M		   (0x4UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G		   (0x5UL &lt;&lt; 4)</div><div class='del'>-	u8 unused_0;</div><div class='del'>-	__le32 dpi;</div><div class='del'>-	__le32 sq_size;</div><div class='del'>-	__le32 rq_size;</div><div class='del'>-	__le16 sq_fwo_sq_sge;</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_SGE_MASK			    0xfUL</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_SGE_SFT			    0</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_FWO_MASK			    0xfff0UL</div><div class='del'>-	#define CMDQ_CREATE_QP_SQ_FWO_SFT			    4</div><div class='del'>-	__le16 rq_fwo_rq_sge;</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_SGE_MASK			    0xfUL</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_SGE_SFT			    0</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_FWO_MASK			    0xfff0UL</div><div class='del'>-	#define CMDQ_CREATE_QP_RQ_FWO_SFT			    4</div><div class='del'>-	__le32 scq_cid;</div><div class='del'>-	__le32 rcq_cid;</div><div class='del'>-	__le32 srq_cid;</div><div class='del'>-	__le32 pd_id;</div><div class='del'>-	__le64 sq_pbl;</div><div class='del'>-	__le64 rq_pbl;</div><div class='del'>-	__le64 irrq_addr;</div><div class='del'>-	__le64 orrq_addr;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Destroy QP command (24 bytes) */</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_CREATE_QP_OPCODE_CREATE_QP 0x1UL</div><div class='add'>+	#define CMDQ_CREATE_QP_OPCODE_LAST     CMDQ_CREATE_QP_OPCODE_CREATE_QP</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le64	qp_handle;</div><div class='add'>+	__le32	qp_flags;</div><div class='add'>+	#define CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED                   0x1UL</div><div class='add'>+	#define CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION           0x2UL</div><div class='add'>+	#define CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE       0x4UL</div><div class='add'>+	#define CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED             0x8UL</div><div class='add'>+	#define CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED 0x10UL</div><div class='add'>+	#define CMDQ_CREATE_QP_QP_FLAGS_OPTIMIZED_TRANSMIT_ENABLED 0x20UL</div><div class='add'>+	#define CMDQ_CREATE_QP_QP_FLAGS_RESPONDER_UD_CQE_WITH_CFA  0x40UL</div><div class='add'>+	#define CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_ENABLED          0x80UL</div><div class='add'>+	#define CMDQ_CREATE_QP_QP_FLAGS_EXPRESS_MODE_ENABLED       0x100UL</div><div class='add'>+	#define CMDQ_CREATE_QP_QP_FLAGS_STEERING_TAG_VALID         0x200UL</div><div class='add'>+	#define CMDQ_CREATE_QP_QP_FLAGS_RDMA_READ_OR_ATOMICS_USED  0x400UL</div><div class='add'>+	#define CMDQ_CREATE_QP_QP_FLAGS_LAST                      \</div><div class='add'>+		CMDQ_CREATE_QP_QP_FLAGS_RDMA_READ_OR_ATOMICS_USED</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CMDQ_CREATE_QP_TYPE_RC            0x2UL</div><div class='add'>+	#define CMDQ_CREATE_QP_TYPE_UD            0x4UL</div><div class='add'>+	#define CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE 0x6UL</div><div class='add'>+	#define CMDQ_CREATE_QP_TYPE_GSI           0x7UL</div><div class='add'>+	#define CMDQ_CREATE_QP_TYPE_LAST         CMDQ_CREATE_QP_TYPE_GSI</div><div class='add'>+	u8	sq_pg_size_sq_lvl;</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_LVL_MASK      0xfUL</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_LVL_SFT       0</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_LVL_LVL_0       0x0UL</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_LVL_LVL_1       0x1UL</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_LVL_LVL_2       0x2UL</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_LVL_LAST       CMDQ_CREATE_QP_SQ_LVL_LVL_2</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_PG_SIZE_MASK  0xf0UL</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_PG_SIZE_SFT   4</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K   (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K   (0x1UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K  (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M   (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M   (0x4UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G   (0x5UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_PG_SIZE_LAST   CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G</div><div class='add'>+	u8	rq_pg_size_rq_lvl;</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_LVL_MASK      0xfUL</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_LVL_SFT       0</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_LVL_LVL_0       0x0UL</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_LVL_LVL_1       0x1UL</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_LVL_LVL_2       0x2UL</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_LVL_LAST       CMDQ_CREATE_QP_RQ_LVL_LVL_2</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_PG_SIZE_MASK  0xf0UL</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_PG_SIZE_SFT   4</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K   (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K   (0x1UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K  (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M   (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M   (0x4UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G   (0x5UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_PG_SIZE_LAST   CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G</div><div class='add'>+	u8	unused_0;</div><div class='add'>+	__le32	dpi;</div><div class='add'>+	__le32	sq_size;</div><div class='add'>+	__le32	rq_size;</div><div class='add'>+	__le16	sq_fwo_sq_sge;</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_SGE_MASK 0xfUL</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_SGE_SFT 0</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_FWO_MASK 0xfff0UL</div><div class='add'>+	#define CMDQ_CREATE_QP_SQ_FWO_SFT 4</div><div class='add'>+	__le16	rq_fwo_rq_sge;</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_SGE_MASK 0xfUL</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_SGE_SFT 0</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_FWO_MASK 0xfff0UL</div><div class='add'>+	#define CMDQ_CREATE_QP_RQ_FWO_SFT 4</div><div class='add'>+	__le32	scq_cid;</div><div class='add'>+	__le32	rcq_cid;</div><div class='add'>+	__le32	srq_cid;</div><div class='add'>+	__le32	pd_id;</div><div class='add'>+	__le64	sq_pbl;</div><div class='add'>+	__le64	rq_pbl;</div><div class='add'>+	__le64	irrq_addr;</div><div class='add'>+	__le64	orrq_addr;</div><div class='add'>+	__le32	request_xid;</div><div class='add'>+	__le16	steering_tag;</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_create_qp_resp (size:128b/16B) */</div><div class='add'>+struct creq_create_qp_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_CREATE_QP_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_CREATE_QP_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_CREATE_QP_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_CREATE_QP_RESP_TYPE_LAST     CREQ_CREATE_QP_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_CREATE_QP_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_CREATE_QP_RESP_EVENT_CREATE_QP 0x1UL</div><div class='add'>+	#define CREQ_CREATE_QP_RESP_EVENT_LAST     CREQ_CREATE_QP_RESP_EVENT_CREATE_QP</div><div class='add'>+	u8	optimized_transmit_enabled;</div><div class='add'>+	u8	reserved48[5];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_destroy_qp (size:192b/24B) */</div><div class='ctx'> struct cmdq_destroy_qp {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_DESTROY_QP_OPCODE_DESTROY_QP		   0x2UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le32 qp_cid;</div><div class='del'>-	__le32 unused_0;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Modify QP command (112 bytes) */</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_DESTROY_QP_OPCODE_DESTROY_QP 0x2UL</div><div class='add'>+	#define CMDQ_DESTROY_QP_OPCODE_LAST      CMDQ_DESTROY_QP_OPCODE_DESTROY_QP</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	qp_cid;</div><div class='add'>+	__le32	unused_0;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_destroy_qp_resp (size:128b/16B) */</div><div class='add'>+struct creq_destroy_qp_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_DESTROY_QP_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_DESTROY_QP_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_DESTROY_QP_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_DESTROY_QP_RESP_TYPE_LAST     CREQ_DESTROY_QP_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_DESTROY_QP_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_DESTROY_QP_RESP_EVENT_DESTROY_QP 0x2UL</div><div class='add'>+	#define CREQ_DESTROY_QP_RESP_EVENT_LAST      CREQ_DESTROY_QP_RESP_EVENT_DESTROY_QP</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_modify_qp (size:1024b/128B) */</div><div class='ctx'> struct cmdq_modify_qp {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_MODIFY_QP_OPCODE_MODIFY_QP		   0x3UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le32 modify_mask;</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_STATE		    0x1UL</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_MODIFY_QP_OPCODE_MODIFY_QP 0x3UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_OPCODE_LAST     CMDQ_MODIFY_QP_OPCODE_MODIFY_QP</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	 #define CMDQ_MODIFY_QP_FLAGS_SRQ_USED       0x1UL</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	qp_type;</div><div class='add'>+	#define CMDQ_MODIFY_QP_QP_TYPE_RC            0x2UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_QP_TYPE_UD            0x4UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_QP_TYPE_RAW_ETHERTYPE 0x6UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_QP_TYPE_GSI           0x7UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_QP_TYPE_LAST         CMDQ_MODIFY_QP_QP_TYPE_GSI</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	modify_mask;</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_STATE                   0x1UL</div><div class='ctx'> 	#define CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY     0x2UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS		    0x4UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_PKEY		    0x8UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_QKEY		    0x10UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_DGID		    0x20UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL		    0x40UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX		    0x80UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT		    0x100UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS	    0x200UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC		    0x400UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU		    0x1000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT		    0x2000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT		    0x4000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY		    0x8000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN		    0x10000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC	    0x20000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER	    0x40000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN		    0x80000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS                  0x4UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_PKEY                    0x8UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_QKEY                    0x10UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_DGID                    0x20UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL              0x40UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX              0x80UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT               0x100UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS           0x200UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC                0x400UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_PINGPONG_PUSH_MODE      0x800UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU                0x1000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT                 0x2000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT               0x4000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY               0x8000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN                  0x10000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC           0x20000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER           0x40000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN                  0x80000UL</div><div class='ctx'> 	#define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC      0x100000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE		    0x200000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE		    0x400000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE		    0x800000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE		    0x1000000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA	    0x2000000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID		    0x4000000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC		    0x8000000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID		    0x10000000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC		    0x20000000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN		    0x40000000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP		    0x80000000UL</div><div class='del'>-	__le32 qp_cid;</div><div class='del'>-	u8 network_type_en_sqd_async_notify_new_state;</div><div class='del'>-	#define CMDQ_MODIFY_QP_NEW_STATE_MASK			    0xfUL</div><div class='del'>-	#define CMDQ_MODIFY_QP_NEW_STATE_SFT			    0</div><div class='del'>-	#define CMDQ_MODIFY_QP_NEW_STATE_RESET			   0x0UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_NEW_STATE_INIT			   0x1UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_NEW_STATE_RTR			   0x2UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_NEW_STATE_RTS			   0x3UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_NEW_STATE_SQD			   0x4UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_NEW_STATE_SQE			   0x5UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_NEW_STATE_ERR			   0x6UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY		    0x10UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_NETWORK_TYPE_MASK		    0xc0UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_NETWORK_TYPE_SFT		    6</div><div class='del'>-	#define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1		   (0x0UL &lt;&lt; 6)</div><div class='del'>-	#define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4	   (0x2UL &lt;&lt; 6)</div><div class='del'>-	#define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6	   (0x3UL &lt;&lt; 6)</div><div class='del'>-	u8 access;</div><div class='del'>-	#define CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE		    0x1UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE		    0x2UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_ACCESS_REMOTE_READ		    0x4UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC		    0x8UL</div><div class='del'>-	__le16 pkey;</div><div class='del'>-	__le32 qkey;</div><div class='del'>-	__le32 dgid[4];</div><div class='del'>-	__le32 flow_label;</div><div class='del'>-	__le16 sgid_index;</div><div class='del'>-	u8 hop_limit;</div><div class='del'>-	u8 traffic_class;</div><div class='del'>-	__le16 dest_mac[3];</div><div class='del'>-	u8 tos_dscp_tos_ecn;</div><div class='del'>-	#define CMDQ_MODIFY_QP_TOS_ECN_MASK			    0x3UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_TOS_ECN_SFT			    0</div><div class='del'>-	#define CMDQ_MODIFY_QP_TOS_DSCP_MASK			    0xfcUL</div><div class='del'>-	#define CMDQ_MODIFY_QP_TOS_DSCP_SFT			    2</div><div class='del'>-	u8 path_mtu;</div><div class='del'>-	#define CMDQ_MODIFY_QP_PATH_MTU_MASK			    0xf0UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_PATH_MTU_SFT			    4</div><div class='del'>-	#define CMDQ_MODIFY_QP_PATH_MTU_MTU_256		   (0x0UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_MODIFY_QP_PATH_MTU_MTU_512		   (0x1UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_MODIFY_QP_PATH_MTU_MTU_1024		   (0x2UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_MODIFY_QP_PATH_MTU_MTU_2048		   (0x3UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_MODIFY_QP_PATH_MTU_MTU_4096		   (0x4UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_MODIFY_QP_PATH_MTU_MTU_8192		   (0x5UL &lt;&lt; 4)</div><div class='del'>-	u8 timeout;</div><div class='del'>-	u8 retry_cnt;</div><div class='del'>-	u8 rnr_retry;</div><div class='del'>-	u8 min_rnr_timer;</div><div class='del'>-	__le32 rq_psn;</div><div class='del'>-	__le32 sq_psn;</div><div class='del'>-	u8 max_rd_atomic;</div><div class='del'>-	u8 max_dest_rd_atomic;</div><div class='del'>-	__le16 enable_cc;</div><div class='del'>-	#define CMDQ_MODIFY_QP_ENABLE_CC			    0x1UL</div><div class='del'>-	__le32 sq_size;</div><div class='del'>-	__le32 rq_size;</div><div class='del'>-	__le16 sq_sge;</div><div class='del'>-	__le16 rq_sge;</div><div class='del'>-	__le32 max_inline_data;</div><div class='del'>-	__le32 dest_qp_id;</div><div class='del'>-	__le32 unused_3;</div><div class='del'>-	__le16 src_mac[3];</div><div class='del'>-	__le16 vlan_pcp_vlan_dei_vlan_id;</div><div class='del'>-	#define CMDQ_MODIFY_QP_VLAN_ID_MASK			    0xfffUL</div><div class='del'>-	#define CMDQ_MODIFY_QP_VLAN_ID_SFT			    0</div><div class='del'>-	#define CMDQ_MODIFY_QP_VLAN_DEI			    0x1000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_VLAN_PCP_MASK			    0xe000UL</div><div class='del'>-	#define CMDQ_MODIFY_QP_VLAN_PCP_SFT			    13</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Query QP command (24 bytes) */</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE                 0x200000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE                 0x400000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE                  0x800000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE                  0x1000000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA         0x2000000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID              0x4000000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC                 0x8000000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID                 0x10000000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC               0x20000000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN                 0x40000000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP                0x80000000UL</div><div class='add'>+	__le32	qp_cid;</div><div class='add'>+	u8	network_type_en_sqd_async_notify_new_state;</div><div class='add'>+	#define CMDQ_MODIFY_QP_NEW_STATE_MASK          0xfUL</div><div class='add'>+	#define CMDQ_MODIFY_QP_NEW_STATE_SFT           0</div><div class='add'>+	#define CMDQ_MODIFY_QP_NEW_STATE_RESET           0x0UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_NEW_STATE_INIT            0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_NEW_STATE_RTR             0x2UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_NEW_STATE_RTS             0x3UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_NEW_STATE_SQD             0x4UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_NEW_STATE_SQE             0x5UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_NEW_STATE_ERR             0x6UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_NEW_STATE_LAST           CMDQ_MODIFY_QP_NEW_STATE_ERR</div><div class='add'>+	#define CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY     0x10UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_UNUSED1                 0x20UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_NETWORK_TYPE_MASK       0xc0UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_NETWORK_TYPE_SFT        6</div><div class='add'>+	#define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1       (0x0UL &lt;&lt; 6)</div><div class='add'>+	#define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4  (0x2UL &lt;&lt; 6)</div><div class='add'>+	#define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6  (0x3UL &lt;&lt; 6)</div><div class='add'>+	#define CMDQ_MODIFY_QP_NETWORK_TYPE_LAST        CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6</div><div class='add'>+	u8	access;</div><div class='add'>+	#define CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC_REMOTE_READ_REMOTE_WRITE_LOCAL_WRITE_MASK 0xffUL</div><div class='add'>+	#define CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC_REMOTE_READ_REMOTE_WRITE_LOCAL_WRITE_SFT 0</div><div class='add'>+	#define CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE   0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE  0x2UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_ACCESS_REMOTE_READ   0x4UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC 0x8UL</div><div class='add'>+	__le16	pkey;</div><div class='add'>+	__le32	qkey;</div><div class='add'>+	__le32	dgid[4];</div><div class='add'>+	__le32	flow_label;</div><div class='add'>+	__le16	sgid_index;</div><div class='add'>+	u8	hop_limit;</div><div class='add'>+	u8	traffic_class;</div><div class='add'>+	__le16	dest_mac[3];</div><div class='add'>+	u8	tos_dscp_tos_ecn;</div><div class='add'>+	#define CMDQ_MODIFY_QP_TOS_ECN_MASK 0x3UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_TOS_ECN_SFT  0</div><div class='add'>+	#define CMDQ_MODIFY_QP_TOS_DSCP_MASK 0xfcUL</div><div class='add'>+	#define CMDQ_MODIFY_QP_TOS_DSCP_SFT 2</div><div class='add'>+	u8	path_mtu_pingpong_push_enable;</div><div class='add'>+	#define CMDQ_MODIFY_QP_PINGPONG_PUSH_ENABLE     0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_UNUSED3_MASK             0xeUL</div><div class='add'>+	#define CMDQ_MODIFY_QP_UNUSED3_SFT              1</div><div class='add'>+	#define CMDQ_MODIFY_QP_PATH_MTU_MASK            0xf0UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_PATH_MTU_SFT             4</div><div class='add'>+	#define CMDQ_MODIFY_QP_PATH_MTU_MTU_256           (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_MODIFY_QP_PATH_MTU_MTU_512           (0x1UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_MODIFY_QP_PATH_MTU_MTU_1024          (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_MODIFY_QP_PATH_MTU_MTU_2048          (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_MODIFY_QP_PATH_MTU_MTU_4096          (0x4UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_MODIFY_QP_PATH_MTU_MTU_8192          (0x5UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_MODIFY_QP_PATH_MTU_LAST             CMDQ_MODIFY_QP_PATH_MTU_MTU_8192</div><div class='add'>+	u8	timeout;</div><div class='add'>+	u8	retry_cnt;</div><div class='add'>+	u8	rnr_retry;</div><div class='add'>+	u8	min_rnr_timer;</div><div class='add'>+	__le32	rq_psn;</div><div class='add'>+	__le32	sq_psn;</div><div class='add'>+	u8	max_rd_atomic;</div><div class='add'>+	u8	max_dest_rd_atomic;</div><div class='add'>+	__le16	enable_cc;</div><div class='add'>+	#define CMDQ_MODIFY_QP_ENABLE_CC     0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_UNUSED15_MASK 0xfffeUL</div><div class='add'>+	#define CMDQ_MODIFY_QP_UNUSED15_SFT  1</div><div class='add'>+	__le32	sq_size;</div><div class='add'>+	__le32	rq_size;</div><div class='add'>+	__le16	sq_sge;</div><div class='add'>+	__le16	rq_sge;</div><div class='add'>+	__le32	max_inline_data;</div><div class='add'>+	__le32	dest_qp_id;</div><div class='add'>+	__le32	pingpong_push_dpi;</div><div class='add'>+	__le16	src_mac[3];</div><div class='add'>+	__le16	vlan_pcp_vlan_dei_vlan_id;</div><div class='add'>+	#define CMDQ_MODIFY_QP_VLAN_ID_MASK 0xfffUL</div><div class='add'>+	#define CMDQ_MODIFY_QP_VLAN_ID_SFT  0</div><div class='add'>+	#define CMDQ_MODIFY_QP_VLAN_DEI     0x1000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_VLAN_PCP_MASK 0xe000UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_VLAN_PCP_SFT 13</div><div class='add'>+	__le64	irrq_addr;</div><div class='add'>+	__le64	orrq_addr;</div><div class='add'>+	__le32	ext_modify_mask;</div><div class='add'>+	#define CMDQ_MODIFY_QP_EXT_MODIFY_MASK_EXT_STATS_CTX     0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_QP_EXT_MODIFY_MASK_SCHQ_ID_VALID     0x2UL</div><div class='add'>+	__le32	ext_stats_ctx_id;</div><div class='add'>+	__le16	schq_id;</div><div class='add'>+	__le16	unused_0;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_modify_qp_resp (size:128b/16B) */</div><div class='add'>+struct creq_modify_qp_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_MODIFY_QP_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_MODIFY_QP_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_MODIFY_QP_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_MODIFY_QP_RESP_TYPE_LAST     CREQ_MODIFY_QP_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_MODIFY_QP_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_MODIFY_QP_RESP_EVENT_MODIFY_QP 0x3UL</div><div class='add'>+	#define CREQ_MODIFY_QP_RESP_EVENT_LAST     CREQ_MODIFY_QP_RESP_EVENT_MODIFY_QP</div><div class='add'>+	u8	pingpong_push_state_index_enabled;</div><div class='add'>+	#define CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED     0x1UL</div><div class='add'>+	#define CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_INDEX_MASK  0xeUL</div><div class='add'>+	#define CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_INDEX_SFT   1</div><div class='add'>+	#define CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_STATE       0x10UL</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le32	lag_src_mac;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_query_qp (size:192b/24B) */</div><div class='ctx'> struct cmdq_query_qp {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_QUERY_QP_OPCODE_QUERY_QP			   0x4UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le32 qp_cid;</div><div class='del'>-	__le32 unused_0;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Create SRQ command (48 bytes) */</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_QUERY_QP_OPCODE_QUERY_QP 0x4UL</div><div class='add'>+	#define CMDQ_QUERY_QP_OPCODE_LAST    CMDQ_QUERY_QP_OPCODE_QUERY_QP</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	qp_cid;</div><div class='add'>+	__le32	unused_0;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_qp_resp (size:128b/16B) */</div><div class='add'>+struct creq_query_qp_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_TYPE_LAST     CREQ_QUERY_QP_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	size;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_EVENT_QUERY_QP 0x4UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_EVENT_LAST    CREQ_QUERY_QP_RESP_EVENT_QUERY_QP</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_qp_resp_sb (size:832b/104B) */</div><div class='add'>+struct creq_query_qp_resp_sb {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_OPCODE_QUERY_QP 0x4UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_OPCODE_LAST    CREQ_QUERY_QP_RESP_SB_OPCODE_QUERY_QP</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	en_sqd_async_notify_state;</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_STATE_MASK              0xfUL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_STATE_SFT               0</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_STATE_RESET               0x0UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_STATE_INIT                0x1UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_STATE_RTR                 0x2UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_STATE_RTS                 0x3UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_STATE_SQD                 0x4UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_STATE_SQE                 0x5UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_STATE_ERR                 0x6UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_STATE_LAST               CREQ_QUERY_QP_RESP_SB_STATE_ERR</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY     0x10UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_UNUSED3_MASK            0xe0UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_UNUSED3_SFT             5</div><div class='add'>+	u8	access;</div><div class='add'>+	#define \</div><div class='add'>+	CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC_REMOTE_READ_REMOTE_WRITE_LOCAL_WRITE_MASK\</div><div class='add'>+		0xffUL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC_REMOTE_READ_REMOTE_WRITE_LOCAL_WRITE_SFT\</div><div class='add'>+		0</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_ACCESS_LOCAL_WRITE 0x1UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_WRITE 0x2UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_READ 0x4UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC 0x8UL</div><div class='add'>+	__le16	pkey;</div><div class='add'>+	__le32	qkey;</div><div class='add'>+	__le16	udp_src_port;</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	dgid[4];</div><div class='add'>+	__le32	flow_label;</div><div class='add'>+	__le16	sgid_index;</div><div class='add'>+	u8	hop_limit;</div><div class='add'>+	u8	traffic_class;</div><div class='add'>+	__le16	dest_mac[3];</div><div class='add'>+	__le16	path_mtu_dest_vlan_id;</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_MASK 0xfffUL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_SFT 0</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK    0xf000UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT     12</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_256   (0x0UL &lt;&lt; 12)</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_512   (0x1UL &lt;&lt; 12)</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_1024  (0x2UL &lt;&lt; 12)</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_2048  (0x3UL &lt;&lt; 12)</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_4096  (0x4UL &lt;&lt; 12)</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_8192  (0x5UL &lt;&lt; 12)</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_LAST     CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_8192</div><div class='add'>+	u8	timeout;</div><div class='add'>+	u8	retry_cnt;</div><div class='add'>+	u8	rnr_retry;</div><div class='add'>+	u8	min_rnr_timer;</div><div class='add'>+	__le32	rq_psn;</div><div class='add'>+	__le32	sq_psn;</div><div class='add'>+	u8	max_rd_atomic;</div><div class='add'>+	u8	max_dest_rd_atomic;</div><div class='add'>+	u8	tos_dscp_tos_ecn;</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_TOS_ECN_MASK 0x3UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_TOS_ECN_SFT  0</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_MASK 0xfcUL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_SFT 2</div><div class='add'>+	u8	enable_cc;</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_ENABLE_CC     0x1UL</div><div class='add'>+	__le32	sq_size;</div><div class='add'>+	__le32	rq_size;</div><div class='add'>+	__le16	sq_sge;</div><div class='add'>+	__le16	rq_sge;</div><div class='add'>+	__le32	max_inline_data;</div><div class='add'>+	__le32	dest_qp_id;</div><div class='add'>+	__le16	port_id;</div><div class='add'>+	u8	unused_0;</div><div class='add'>+	u8	stat_collection_id;</div><div class='add'>+	__le16	src_mac[3];</div><div class='add'>+	__le16	vlan_pcp_vlan_dei_vlan_id;</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK 0xfffUL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT  0</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_VLAN_DEI     0x1000UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_MASK 0xe000UL</div><div class='add'>+	#define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_SFT 13</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_query_qp_extend (size:192b/24B) */</div><div class='add'>+struct cmdq_query_qp_extend {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_QUERY_QP_EXTEND_OPCODE_QUERY_QP_EXTEND 0x91UL</div><div class='add'>+	#define CMDQ_QUERY_QP_EXTEND_OPCODE_LAST CMDQ_QUERY_QP_EXTEND_OPCODE_QUERY_QP_EXTEND</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	num_qps;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	function_id;</div><div class='add'>+	#define CMDQ_QUERY_QP_EXTEND_PF_NUM_MASK  0xffUL</div><div class='add'>+	#define CMDQ_QUERY_QP_EXTEND_PF_NUM_SFT   0</div><div class='add'>+	#define CMDQ_QUERY_QP_EXTEND_VF_NUM_MASK  0xffff00UL</div><div class='add'>+	#define CMDQ_QUERY_QP_EXTEND_VF_NUM_SFT   8</div><div class='add'>+	#define CMDQ_QUERY_QP_EXTEND_VF_VALID     0x1000000UL</div><div class='add'>+	__le32	current_index;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_qp_extend_resp (size:128b/16B) */</div><div class='add'>+struct creq_query_qp_extend_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_TYPE_LAST     CREQ_QUERY_QP_EXTEND_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	size;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_EVENT_QUERY_QP_EXTEND 0x91UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_EVENT_LAST CREQ_QUERY_QP_EXTEND_RESP_EVENT_QUERY_QP_EXTEND</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	current_index;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_qp_extend_resp_sb (size:384b/48B) */</div><div class='add'>+struct creq_query_qp_extend_resp_sb {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_OPCODE_QUERY_QP_EXTEND 0x91UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_OPCODE_LAST \</div><div class='add'>+		CREQ_QUERY_QP_EXTEND_RESP_SB_OPCODE_QUERY_QP_EXTEND</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	state;</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_MASK  0xfUL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_SFT   0</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_RESET   0x0UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_INIT    0x1UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_RTR     0x2UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_RTS     0x3UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_SQD     0x4UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_SQE     0x5UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_ERR     0x6UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_LAST   CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_ERR</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_UNUSED4_MASK 0xf0UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_UNUSED4_SFT 4</div><div class='add'>+	u8	reserved_8;</div><div class='add'>+	__le16	port_id;</div><div class='add'>+	__le32	qkey;</div><div class='add'>+	__le16	sgid_index;</div><div class='add'>+	u8	network_type;</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_NETWORK_TYPE_ROCEV1      0x0UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_NETWORK_TYPE_ROCEV2_IPV4 0x2UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_NETWORK_TYPE_ROCEV2_IPV6 0x3UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_NETWORK_TYPE_LAST \</div><div class='add'>+		CREQ_QUERY_QP_EXTEND_RESP_SB_NETWORK_TYPE_ROCEV2_IPV6</div><div class='add'>+	u8	unused_0;</div><div class='add'>+	__le32	dgid[4];</div><div class='add'>+	__le32	dest_qp_id;</div><div class='add'>+	u8	stat_collection_id;</div><div class='add'>+	u8	reservred_8;</div><div class='add'>+	__le16	reserved_16;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_qp_extend_resp_sb_tlv (size:512b/64B) */</div><div class='add'>+struct creq_query_qp_extend_resp_sb_tlv {</div><div class='add'>+	__le16	cmd_discr;</div><div class='add'>+	u8	reserved_8b;</div><div class='add'>+	u8	tlv_flags;</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_MORE         0x1UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_MORE_LAST      0x0UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_MORE_NOT_LAST  0x1UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_REQUIRED     0x2UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_REQUIRED_NO    (0x0UL &lt;&lt; 1)</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES   (0x1UL &lt;&lt; 1)</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_REQUIRED_LAST \</div><div class='add'>+		CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES</div><div class='add'>+	__le16	tlv_type;</div><div class='add'>+	__le16	length;</div><div class='add'>+	u8	total_size;</div><div class='add'>+	u8	reserved56[7];</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_OPCODE_QUERY_QP_EXTEND 0x91UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_OPCODE_LAST \</div><div class='add'>+		CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_OPCODE_QUERY_QP_EXTEND</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	state;</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_MASK  0xfUL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_SFT   0</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_RESET   0x0UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_INIT    0x1UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_RTR     0x2UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_RTS     0x3UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_SQD     0x4UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_SQE     0x5UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_ERR     0x6UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_LAST \</div><div class='add'>+		CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_ERR</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_UNUSED4_MASK 0xf0UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_UNUSED4_SFT 4</div><div class='add'>+	u8	reserved_8;</div><div class='add'>+	__le16	port_id;</div><div class='add'>+	__le32	qkey;</div><div class='add'>+	__le16	sgid_index;</div><div class='add'>+	u8	network_type;</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_NETWORK_TYPE_ROCEV1      0x0UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_NETWORK_TYPE_ROCEV2_IPV4 0x2UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_NETWORK_TYPE_ROCEV2_IPV6 0x3UL</div><div class='add'>+	#define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_NETWORK_TYPE_LAST \</div><div class='add'>+		CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_NETWORK_TYPE_ROCEV2_IPV6</div><div class='add'>+	u8	unused_0;</div><div class='add'>+	__le32	dgid[4];</div><div class='add'>+	__le32	dest_qp_id;</div><div class='add'>+	u8	stat_collection_id;</div><div class='add'>+	u8	reservred_8;</div><div class='add'>+	__le16	reserved_16;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_create_srq (size:448b/56B) */</div><div class='ctx'> struct cmdq_create_srq {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_CREATE_SRQ_OPCODE_CREATE_SRQ		   0x5UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le64 srq_handle;</div><div class='del'>-	__le16 pg_size_lvl;</div><div class='del'>-	#define CMDQ_CREATE_SRQ_LVL_MASK			    0x3UL</div><div class='del'>-	#define CMDQ_CREATE_SRQ_LVL_SFT			    0</div><div class='del'>-	#define CMDQ_CREATE_SRQ_LVL_LVL_0			   0x0UL</div><div class='del'>-	#define CMDQ_CREATE_SRQ_LVL_LVL_1			   0x1UL</div><div class='del'>-	#define CMDQ_CREATE_SRQ_LVL_LVL_2			   0x2UL</div><div class='del'>-	#define CMDQ_CREATE_SRQ_PG_SIZE_MASK			    0x1cUL</div><div class='del'>-	#define CMDQ_CREATE_SRQ_PG_SIZE_SFT			    2</div><div class='del'>-	#define CMDQ_CREATE_SRQ_PG_SIZE_PG_4K			   (0x0UL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_CREATE_SRQ_PG_SIZE_PG_8K			   (0x1UL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_CREATE_SRQ_PG_SIZE_PG_64K			   (0x2UL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_CREATE_SRQ_PG_SIZE_PG_2M			   (0x3UL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_CREATE_SRQ_PG_SIZE_PG_8M			   (0x4UL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_CREATE_SRQ_PG_SIZE_PG_1G			   (0x5UL &lt;&lt; 2)</div><div class='del'>-	__le16 eventq_id;</div><div class='del'>-	#define CMDQ_CREATE_SRQ_EVENTQ_ID_MASK			    0xfffUL</div><div class='del'>-	#define CMDQ_CREATE_SRQ_EVENTQ_ID_SFT			    0</div><div class='del'>-	__le16 srq_size;</div><div class='del'>-	__le16 srq_fwo;</div><div class='del'>-	__le32 dpi;</div><div class='del'>-	__le32 pd_id;</div><div class='del'>-	__le64 pbl;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Destroy SRQ command (24 bytes) */</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_CREATE_SRQ_OPCODE_CREATE_SRQ 0x5UL</div><div class='add'>+	#define CMDQ_CREATE_SRQ_OPCODE_LAST      CMDQ_CREATE_SRQ_OPCODE_CREATE_SRQ</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	#define CMDQ_CREATE_SRQ_FLAGS_STEERING_TAG_VALID	0x1UL</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le64	srq_handle;</div><div class='add'>+	__le16	pg_size_lvl;</div><div class='add'>+	#define CMDQ_CREATE_SRQ_LVL_MASK      0x3UL</div><div class='add'>+	#define CMDQ_CREATE_SRQ_LVL_SFT       0</div><div class='add'>+	#define CMDQ_CREATE_SRQ_LVL_LVL_0       0x0UL</div><div class='add'>+	#define CMDQ_CREATE_SRQ_LVL_LVL_1       0x1UL</div><div class='add'>+	#define CMDQ_CREATE_SRQ_LVL_LVL_2       0x2UL</div><div class='add'>+	#define CMDQ_CREATE_SRQ_LVL_LAST       CMDQ_CREATE_SRQ_LVL_LVL_2</div><div class='add'>+	#define CMDQ_CREATE_SRQ_PG_SIZE_MASK  0x1cUL</div><div class='add'>+	#define CMDQ_CREATE_SRQ_PG_SIZE_SFT   2</div><div class='add'>+	#define CMDQ_CREATE_SRQ_PG_SIZE_PG_4K   (0x0UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_CREATE_SRQ_PG_SIZE_PG_8K   (0x1UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_CREATE_SRQ_PG_SIZE_PG_64K  (0x2UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_CREATE_SRQ_PG_SIZE_PG_2M   (0x3UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_CREATE_SRQ_PG_SIZE_PG_8M   (0x4UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_CREATE_SRQ_PG_SIZE_PG_1G   (0x5UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_CREATE_SRQ_PG_SIZE_LAST   CMDQ_CREATE_SRQ_PG_SIZE_PG_1G</div><div class='add'>+	#define CMDQ_CREATE_SRQ_UNUSED11_MASK 0xffe0UL</div><div class='add'>+	#define CMDQ_CREATE_SRQ_UNUSED11_SFT  5</div><div class='add'>+	__le16	eventq_id;</div><div class='add'>+	#define CMDQ_CREATE_SRQ_EVENTQ_ID_MASK 0xfffUL</div><div class='add'>+	#define CMDQ_CREATE_SRQ_EVENTQ_ID_SFT 0</div><div class='add'>+	#define CMDQ_CREATE_SRQ_UNUSED4_MASK  0xf000UL</div><div class='add'>+	#define CMDQ_CREATE_SRQ_UNUSED4_SFT   12</div><div class='add'>+	__le16	srq_size;</div><div class='add'>+	__le16	srq_fwo;</div><div class='add'>+	__le32	dpi;</div><div class='add'>+	__le32	pd_id;</div><div class='add'>+	__le64	pbl;</div><div class='add'>+	__le16	steering_tag;</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_create_srq_resp (size:128b/16B) */</div><div class='add'>+struct creq_create_srq_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_CREATE_SRQ_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_CREATE_SRQ_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_CREATE_SRQ_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_CREATE_SRQ_RESP_TYPE_LAST     CREQ_CREATE_SRQ_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_CREATE_SRQ_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_CREATE_SRQ_RESP_EVENT_CREATE_SRQ 0x5UL</div><div class='add'>+	#define CREQ_CREATE_SRQ_RESP_EVENT_LAST      CREQ_CREATE_SRQ_RESP_EVENT_CREATE_SRQ</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_destroy_srq (size:192b/24B) */</div><div class='ctx'> struct cmdq_destroy_srq {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_DESTROY_SRQ_OPCODE_DESTROY_SRQ		   0x6UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le32 srq_cid;</div><div class='del'>-	__le32 unused_0;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Query SRQ command (24 bytes) */</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_DESTROY_SRQ_OPCODE_DESTROY_SRQ 0x6UL</div><div class='add'>+	#define CMDQ_DESTROY_SRQ_OPCODE_LAST       CMDQ_DESTROY_SRQ_OPCODE_DESTROY_SRQ</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	srq_cid;</div><div class='add'>+	__le32	unused_0;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_destroy_srq_resp (size:128b/16B) */</div><div class='add'>+struct creq_destroy_srq_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_DESTROY_SRQ_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_DESTROY_SRQ_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_DESTROY_SRQ_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_DESTROY_SRQ_RESP_TYPE_LAST     CREQ_DESTROY_SRQ_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_DESTROY_SRQ_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_DESTROY_SRQ_RESP_EVENT_DESTROY_SRQ 0x6UL</div><div class='add'>+	#define CREQ_DESTROY_SRQ_RESP_EVENT_LAST       CREQ_DESTROY_SRQ_RESP_EVENT_DESTROY_SRQ</div><div class='add'>+	__le16	enable_for_arm[3];</div><div class='add'>+	#define CREQ_DESTROY_SRQ_RESP_UNUSED0_MASK       0xffffUL</div><div class='add'>+	#define CREQ_DESTROY_SRQ_RESP_UNUSED0_SFT        0</div><div class='add'>+	#define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_MASK 0x30000UL</div><div class='add'>+	#define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_SFT 16</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_query_srq (size:192b/24B) */</div><div class='ctx'> struct cmdq_query_srq {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_QUERY_SRQ_OPCODE_QUERY_SRQ		   0x8UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le32 srq_cid;</div><div class='del'>-	__le32 unused_0;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Create CQ command (48 bytes) */</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_QUERY_SRQ_OPCODE_QUERY_SRQ 0x8UL</div><div class='add'>+	#define CMDQ_QUERY_SRQ_OPCODE_LAST     CMDQ_QUERY_SRQ_OPCODE_QUERY_SRQ</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	srq_cid;</div><div class='add'>+	__le32	unused_0;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_srq_resp (size:128b/16B) */</div><div class='add'>+struct creq_query_srq_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_QUERY_SRQ_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_QUERY_SRQ_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_QUERY_SRQ_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_QUERY_SRQ_RESP_TYPE_LAST     CREQ_QUERY_SRQ_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	size;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_QUERY_SRQ_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_QUERY_SRQ_RESP_EVENT_QUERY_SRQ 0x8UL</div><div class='add'>+	#define CREQ_QUERY_SRQ_RESP_EVENT_LAST     CREQ_QUERY_SRQ_RESP_EVENT_QUERY_SRQ</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_srq_resp_sb (size:256b/32B) */</div><div class='add'>+struct creq_query_srq_resp_sb {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CREQ_QUERY_SRQ_RESP_SB_OPCODE_QUERY_SRQ 0x8UL</div><div class='add'>+	#define CREQ_QUERY_SRQ_RESP_SB_OPCODE_LAST     CREQ_QUERY_SRQ_RESP_SB_OPCODE_QUERY_SRQ</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	__le16	srq_limit;</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	data[4];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_create_cq (size:448b/56B) */</div><div class='ctx'> struct cmdq_create_cq {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_CREATE_CQ_OPCODE_CREATE_CQ		   0x9UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le64 cq_handle;</div><div class='del'>-	__le32 pg_size_lvl;</div><div class='del'>-	#define CMDQ_CREATE_CQ_LVL_MASK			    0x3UL</div><div class='del'>-	#define CMDQ_CREATE_CQ_LVL_SFT				    0</div><div class='del'>-	#define CMDQ_CREATE_CQ_LVL_LVL_0			   0x0UL</div><div class='del'>-	#define CMDQ_CREATE_CQ_LVL_LVL_1			   0x1UL</div><div class='del'>-	#define CMDQ_CREATE_CQ_LVL_LVL_2			   0x2UL</div><div class='del'>-	#define CMDQ_CREATE_CQ_PG_SIZE_MASK			    0x1cUL</div><div class='del'>-	#define CMDQ_CREATE_CQ_PG_SIZE_SFT			    2</div><div class='del'>-	#define CMDQ_CREATE_CQ_PG_SIZE_PG_4K			   (0x0UL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_CREATE_CQ_PG_SIZE_PG_8K			   (0x1UL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_CREATE_CQ_PG_SIZE_PG_64K			   (0x2UL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_CREATE_CQ_PG_SIZE_PG_2M			   (0x3UL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_CREATE_CQ_PG_SIZE_PG_8M			   (0x4UL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_CREATE_CQ_PG_SIZE_PG_1G			   (0x5UL &lt;&lt; 2)</div><div class='del'>-	__le32 cq_fco_cnq_id;</div><div class='del'>-	#define CMDQ_CREATE_CQ_CNQ_ID_MASK			    0xfffUL</div><div class='del'>-	#define CMDQ_CREATE_CQ_CNQ_ID_SFT			    0</div><div class='del'>-	#define CMDQ_CREATE_CQ_CQ_FCO_MASK			    0xfffff000UL</div><div class='del'>-	#define CMDQ_CREATE_CQ_CQ_FCO_SFT			    12</div><div class='del'>-	__le32 dpi;</div><div class='del'>-	__le32 cq_size;</div><div class='del'>-	__le64 pbl;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Destroy CQ command (24 bytes) */</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_CREATE_CQ_OPCODE_CREATE_CQ 0x9UL</div><div class='add'>+	#define CMDQ_CREATE_CQ_OPCODE_LAST     CMDQ_CREATE_CQ_OPCODE_CREATE_CQ</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	#define CMDQ_CREATE_CQ_FLAGS_DISABLE_CQ_OVERFLOW_DETECTION     0x1UL</div><div class='add'>+	#define CMDQ_CREATE_CQ_FLAGS_STEERING_TAG_VALID                0x2UL</div><div class='add'>+	#define CMDQ_CREATE_CQ_FLAGS_INFINITE_CQ_MODE                  0x4UL</div><div class='add'>+	#define CMDQ_CREATE_CQ_FLAGS_COALESCING_VALID                  0x8UL</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le64	cq_handle;</div><div class='add'>+	__le32	pg_size_lvl;</div><div class='add'>+	#define CMDQ_CREATE_CQ_LVL_MASK      0x3UL</div><div class='add'>+	#define CMDQ_CREATE_CQ_LVL_SFT       0</div><div class='add'>+	#define CMDQ_CREATE_CQ_LVL_LVL_0       0x0UL</div><div class='add'>+	#define CMDQ_CREATE_CQ_LVL_LVL_1       0x1UL</div><div class='add'>+	#define CMDQ_CREATE_CQ_LVL_LVL_2       0x2UL</div><div class='add'>+	#define CMDQ_CREATE_CQ_LVL_LAST       CMDQ_CREATE_CQ_LVL_LVL_2</div><div class='add'>+	#define CMDQ_CREATE_CQ_PG_SIZE_MASK  0x1cUL</div><div class='add'>+	#define CMDQ_CREATE_CQ_PG_SIZE_SFT   2</div><div class='add'>+	#define CMDQ_CREATE_CQ_PG_SIZE_PG_4K   (0x0UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_CREATE_CQ_PG_SIZE_PG_8K   (0x1UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_CREATE_CQ_PG_SIZE_PG_64K  (0x2UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_CREATE_CQ_PG_SIZE_PG_2M   (0x3UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_CREATE_CQ_PG_SIZE_PG_8M   (0x4UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_CREATE_CQ_PG_SIZE_PG_1G   (0x5UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_CREATE_CQ_PG_SIZE_LAST   CMDQ_CREATE_CQ_PG_SIZE_PG_1G</div><div class='add'>+	#define CMDQ_CREATE_CQ_UNUSED27_MASK 0xffffffe0UL</div><div class='add'>+	#define CMDQ_CREATE_CQ_UNUSED27_SFT  5</div><div class='add'>+	__le32	cq_fco_cnq_id;</div><div class='add'>+	#define CMDQ_CREATE_CQ_CNQ_ID_MASK 0xfffUL</div><div class='add'>+	#define CMDQ_CREATE_CQ_CNQ_ID_SFT 0</div><div class='add'>+	#define CMDQ_CREATE_CQ_CQ_FCO_MASK 0xfffff000UL</div><div class='add'>+	#define CMDQ_CREATE_CQ_CQ_FCO_SFT 12</div><div class='add'>+	__le32	dpi;</div><div class='add'>+	__le32	cq_size;</div><div class='add'>+	__le64	pbl;</div><div class='add'>+	__le16	steering_tag;</div><div class='add'>+	u8	reserved48[2];</div><div class='add'>+	__le32  coalescing;</div><div class='add'>+	#define CMDQ_CREATE_CQ_BUF_MAXTIME_MASK          0x1ffUL</div><div class='add'>+	#define CMDQ_CREATE_CQ_BUF_MAXTIME_SFT           0</div><div class='add'>+	#define CMDQ_CREATE_CQ_NORMAL_MAXBUF_MASK        0x3e00UL</div><div class='add'>+	#define CMDQ_CREATE_CQ_NORMAL_MAXBUF_SFT         9</div><div class='add'>+	#define CMDQ_CREATE_CQ_DURING_MAXBUF_MASK        0x7c000UL</div><div class='add'>+	#define CMDQ_CREATE_CQ_DURING_MAXBUF_SFT         14</div><div class='add'>+	#define CMDQ_CREATE_CQ_ENABLE_RING_IDLE_MODE     0x80000UL</div><div class='add'>+	#define CMDQ_CREATE_CQ_UNUSED12_MASK             0xfff00000UL</div><div class='add'>+	#define CMDQ_CREATE_CQ_UNUSED12_SFT              20</div><div class='add'>+	__le64  reserved64;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_create_cq_resp (size:128b/16B) */</div><div class='add'>+struct creq_create_cq_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_CREATE_CQ_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_CREATE_CQ_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_CREATE_CQ_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_CREATE_CQ_RESP_TYPE_LAST     CREQ_CREATE_CQ_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_CREATE_CQ_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_CREATE_CQ_RESP_EVENT_CREATE_CQ 0x9UL</div><div class='add'>+	#define CREQ_CREATE_CQ_RESP_EVENT_LAST     CREQ_CREATE_CQ_RESP_EVENT_CREATE_CQ</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_destroy_cq (size:192b/24B) */</div><div class='ctx'> struct cmdq_destroy_cq {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_DESTROY_CQ_OPCODE_DESTROY_CQ		   0xaUL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le32 cq_cid;</div><div class='del'>-	__le32 unused_0;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Resize CQ command (40 bytes) */</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_DESTROY_CQ_OPCODE_DESTROY_CQ 0xaUL</div><div class='add'>+	#define CMDQ_DESTROY_CQ_OPCODE_LAST      CMDQ_DESTROY_CQ_OPCODE_DESTROY_CQ</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	cq_cid;</div><div class='add'>+	__le32	unused_0;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_destroy_cq_resp (size:128b/16B) */</div><div class='add'>+struct creq_destroy_cq_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_DESTROY_CQ_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_DESTROY_CQ_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_DESTROY_CQ_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_DESTROY_CQ_RESP_TYPE_LAST     CREQ_DESTROY_CQ_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_DESTROY_CQ_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_DESTROY_CQ_RESP_EVENT_DESTROY_CQ 0xaUL</div><div class='add'>+	#define CREQ_DESTROY_CQ_RESP_EVENT_LAST      CREQ_DESTROY_CQ_RESP_EVENT_DESTROY_CQ</div><div class='add'>+	__le16	cq_arm_lvl;</div><div class='add'>+	#define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_MASK 0x3UL</div><div class='add'>+	#define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_SFT 0</div><div class='add'>+	__le16	total_cnq_events;</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_resize_cq (size:320b/40B) */</div><div class='ctx'> struct cmdq_resize_cq {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_RESIZE_CQ_OPCODE_RESIZE_CQ		   0xcUL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le32 cq_cid;</div><div class='del'>-	__le32 new_cq_size_pg_size_lvl;</div><div class='del'>-	#define CMDQ_RESIZE_CQ_LVL_MASK			    0x3UL</div><div class='del'>-	#define CMDQ_RESIZE_CQ_LVL_SFT				    0</div><div class='del'>-	#define CMDQ_RESIZE_CQ_LVL_LVL_0			   0x0UL</div><div class='del'>-	#define CMDQ_RESIZE_CQ_LVL_LVL_1			   0x1UL</div><div class='del'>-	#define CMDQ_RESIZE_CQ_LVL_LVL_2			   0x2UL</div><div class='del'>-	#define CMDQ_RESIZE_CQ_PG_SIZE_MASK			    0x1cUL</div><div class='del'>-	#define CMDQ_RESIZE_CQ_PG_SIZE_SFT			    2</div><div class='del'>-	#define CMDQ_RESIZE_CQ_PG_SIZE_PG_4K			   (0x0UL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_RESIZE_CQ_PG_SIZE_PG_8K			   (0x1UL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_RESIZE_CQ_PG_SIZE_PG_64K			   (0x2UL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_RESIZE_CQ_PG_SIZE_PG_2M			   (0x3UL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_RESIZE_CQ_PG_SIZE_PG_8M			   (0x4UL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_RESIZE_CQ_PG_SIZE_PG_1G			   (0x5UL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK		    0x1fffe0UL</div><div class='del'>-	#define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT			    5</div><div class='del'>-	__le64 new_pbl;</div><div class='del'>-	__le32 new_cq_fco;</div><div class='del'>-	__le32 unused_2;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Allocate MRW command (32 bytes) */</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_RESIZE_CQ_OPCODE_RESIZE_CQ 0xcUL</div><div class='add'>+	#define CMDQ_RESIZE_CQ_OPCODE_LAST     CMDQ_RESIZE_CQ_OPCODE_RESIZE_CQ</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	cq_cid;</div><div class='add'>+	__le32	new_cq_size_pg_size_lvl;</div><div class='add'>+	#define CMDQ_RESIZE_CQ_LVL_MASK        0x3UL</div><div class='add'>+	#define CMDQ_RESIZE_CQ_LVL_SFT         0</div><div class='add'>+	#define CMDQ_RESIZE_CQ_LVL_LVL_0         0x0UL</div><div class='add'>+	#define CMDQ_RESIZE_CQ_LVL_LVL_1         0x1UL</div><div class='add'>+	#define CMDQ_RESIZE_CQ_LVL_LVL_2         0x2UL</div><div class='add'>+	#define CMDQ_RESIZE_CQ_LVL_LAST         CMDQ_RESIZE_CQ_LVL_LVL_2</div><div class='add'>+	#define CMDQ_RESIZE_CQ_PG_SIZE_MASK    0x1cUL</div><div class='add'>+	#define CMDQ_RESIZE_CQ_PG_SIZE_SFT     2</div><div class='add'>+	#define CMDQ_RESIZE_CQ_PG_SIZE_PG_4K     (0x0UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_RESIZE_CQ_PG_SIZE_PG_8K     (0x1UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_RESIZE_CQ_PG_SIZE_PG_64K    (0x2UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_RESIZE_CQ_PG_SIZE_PG_2M     (0x3UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_RESIZE_CQ_PG_SIZE_PG_8M     (0x4UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_RESIZE_CQ_PG_SIZE_PG_1G     (0x5UL &lt;&lt; 2)</div><div class='add'>+	#define CMDQ_RESIZE_CQ_PG_SIZE_LAST     CMDQ_RESIZE_CQ_PG_SIZE_PG_1G</div><div class='add'>+	#define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK 0x1fffffe0UL</div><div class='add'>+	#define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT 5</div><div class='add'>+	__le64	new_pbl;</div><div class='add'>+	__le32	new_cq_fco;</div><div class='add'>+	__le32	unused_0;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_resize_cq_resp (size:128b/16B) */</div><div class='add'>+struct creq_resize_cq_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_RESIZE_CQ_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_RESIZE_CQ_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_RESIZE_CQ_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_RESIZE_CQ_RESP_TYPE_LAST     CREQ_RESIZE_CQ_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_RESIZE_CQ_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_RESIZE_CQ_RESP_EVENT_RESIZE_CQ 0xcUL</div><div class='add'>+	#define CREQ_RESIZE_CQ_RESP_EVENT_LAST     CREQ_RESIZE_CQ_RESP_EVENT_RESIZE_CQ</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_allocate_mrw (size:256b/32B) */</div><div class='ctx'> struct cmdq_allocate_mrw {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_ALLOCATE_MRW_OPCODE_ALLOCATE_MRW		   0xdUL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le64 mrw_handle;</div><div class='del'>-	u8 mrw_flags;</div><div class='del'>-	#define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MASK		    0xfUL</div><div class='del'>-	#define CMDQ_ALLOCATE_MRW_MRW_FLAGS_SFT		    0</div><div class='del'>-	#define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR			   0x0UL</div><div class='del'>-	#define CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR		   0x1UL</div><div class='del'>-	#define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1		   0x2UL</div><div class='del'>-	#define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A		   0x3UL</div><div class='del'>-	#define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B		   0x4UL</div><div class='del'>-	u8 access;</div><div class='del'>-	#define CMDQ_ALLOCATE_MRW_ACCESS_RESERVED_MASK		    0x1fUL</div><div class='del'>-	#define CMDQ_ALLOCATE_MRW_ACCESS_RESERVED_SFT		    0</div><div class='del'>-	#define CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY	    0x20UL</div><div class='del'>-	__le16 unused_1;</div><div class='del'>-	__le32 pd_id;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* De-allocate key command (24 bytes) */</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_ALLOCATE_MRW_OPCODE_ALLOCATE_MRW 0xdUL</div><div class='add'>+	#define CMDQ_ALLOCATE_MRW_OPCODE_LAST        CMDQ_ALLOCATE_MRW_OPCODE_ALLOCATE_MRW</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le64	mrw_handle;</div><div class='add'>+	u8	mrw_flags;</div><div class='add'>+	#define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MASK     0xfUL</div><div class='add'>+	#define CMDQ_ALLOCATE_MRW_MRW_FLAGS_SFT      0</div><div class='add'>+	#define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR         0x0UL</div><div class='add'>+	#define CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR        0x1UL</div><div class='add'>+	#define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1   0x2UL</div><div class='add'>+	#define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A  0x3UL</div><div class='add'>+	#define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B  0x4UL</div><div class='add'>+	#define CMDQ_ALLOCATE_MRW_MRW_FLAGS_LAST      CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B</div><div class='add'>+	#define CMDQ_ALLOCATE_MRW_STEERING_TAG_VALID     0x10UL</div><div class='add'>+	#define CMDQ_ALLOCATE_MRW_UNUSED4_MASK       0xe0UL</div><div class='add'>+	#define CMDQ_ALLOCATE_MRW_UNUSED4_SFT        5</div><div class='add'>+	u8	access;</div><div class='add'>+	#define CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY     0x20UL</div><div class='add'>+	__le16	steering_tag;</div><div class='add'>+	__le32	pd_id;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_allocate_mrw_resp (size:128b/16B) */</div><div class='add'>+struct creq_allocate_mrw_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_ALLOCATE_MRW_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_ALLOCATE_MRW_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_ALLOCATE_MRW_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_ALLOCATE_MRW_RESP_TYPE_LAST     CREQ_ALLOCATE_MRW_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_ALLOCATE_MRW_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_ALLOCATE_MRW_RESP_EVENT_ALLOCATE_MRW 0xdUL</div><div class='add'>+	#define CREQ_ALLOCATE_MRW_RESP_EVENT_LAST        CREQ_ALLOCATE_MRW_RESP_EVENT_ALLOCATE_MRW</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_deallocate_key (size:192b/24B) */</div><div class='ctx'> struct cmdq_deallocate_key {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_DEALLOCATE_KEY_OPCODE_DEALLOCATE_KEY	   0xeUL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	u8 mrw_flags;</div><div class='del'>-	#define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MASK		    0xfUL</div><div class='del'>-	#define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_SFT		    0</div><div class='del'>-	#define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MR		   0x0UL</div><div class='del'>-	#define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_PMR		   0x1UL</div><div class='del'>-	#define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE1		   0x2UL</div><div class='del'>-	#define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2A	   0x3UL</div><div class='del'>-	#define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2B	   0x4UL</div><div class='del'>-	u8 unused_1[3];</div><div class='del'>-	__le32 key;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Register MR command (48 bytes) */</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_DEALLOCATE_KEY_OPCODE_DEALLOCATE_KEY 0xeUL</div><div class='add'>+	#define CMDQ_DEALLOCATE_KEY_OPCODE_LAST          CMDQ_DEALLOCATE_KEY_OPCODE_DEALLOCATE_KEY</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	u8	mrw_flags;</div><div class='add'>+	#define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MASK     0xfUL</div><div class='add'>+	#define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_SFT      0</div><div class='add'>+	#define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MR         0x0UL</div><div class='add'>+	#define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_PMR        0x1UL</div><div class='add'>+	#define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE1   0x2UL</div><div class='add'>+	#define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2A  0x3UL</div><div class='add'>+	#define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2B  0x4UL</div><div class='add'>+	#define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_LAST      CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2B</div><div class='add'>+	#define CMDQ_DEALLOCATE_KEY_UNUSED4_MASK       0xf0UL</div><div class='add'>+	#define CMDQ_DEALLOCATE_KEY_UNUSED4_SFT        4</div><div class='add'>+	u8	unused24[3];</div><div class='add'>+	__le32	key;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_deallocate_key_resp (size:128b/16B) */</div><div class='add'>+struct creq_deallocate_key_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_DEALLOCATE_KEY_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_DEALLOCATE_KEY_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_DEALLOCATE_KEY_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_DEALLOCATE_KEY_RESP_TYPE_LAST     CREQ_DEALLOCATE_KEY_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_DEALLOCATE_KEY_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_DEALLOCATE_KEY_RESP_EVENT_DEALLOCATE_KEY 0xeUL</div><div class='add'>+	#define CREQ_DEALLOCATE_KEY_RESP_EVENT_LAST CREQ_DEALLOCATE_KEY_RESP_EVENT_DEALLOCATE_KEY</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	bound_window_info;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_register_mr (size:448b/56B) */</div><div class='ctx'> struct cmdq_register_mr {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_REGISTER_MR_OPCODE_REGISTER_MR		   0xfUL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	u8 log2_pg_size_lvl;</div><div class='del'>-	#define CMDQ_REGISTER_MR_LVL_MASK			    0x3UL</div><div class='del'>-	#define CMDQ_REGISTER_MR_LVL_SFT			    0</div><div class='del'>-	#define CMDQ_REGISTER_MR_LVL_LVL_0			   0x0UL</div><div class='del'>-	#define CMDQ_REGISTER_MR_LVL_LVL_1			   0x1UL</div><div class='del'>-	#define CMDQ_REGISTER_MR_LVL_LVL_2			   0x2UL</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_REGISTER_MR_OPCODE_REGISTER_MR 0xfUL</div><div class='add'>+	#define CMDQ_REGISTER_MR_OPCODE_LAST       CMDQ_REGISTER_MR_OPCODE_REGISTER_MR</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	#define CMDQ_REGISTER_MR_FLAGS_ALLOC_MR			0x1UL</div><div class='add'>+	#define CMDQ_REGISTER_MR_FLAGS_STEERING_TAG_VALID	0x2UL</div><div class='add'>+	#define CMDQ_REGISTER_MR_FLAGS_ENABLE_RO		0x4UL</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	u8	log2_pg_size_lvl;</div><div class='add'>+	#define CMDQ_REGISTER_MR_LVL_MASK            0x3UL</div><div class='add'>+	#define CMDQ_REGISTER_MR_LVL_SFT             0</div><div class='add'>+	#define CMDQ_REGISTER_MR_LVL_LVL_0             0x0UL</div><div class='add'>+	#define CMDQ_REGISTER_MR_LVL_LVL_1             0x1UL</div><div class='add'>+	#define CMDQ_REGISTER_MR_LVL_LVL_2             0x2UL</div><div class='ctx'> 	#define CMDQ_REGISTER_MR_LVL_LAST             CMDQ_REGISTER_MR_LVL_LVL_2</div><div class='del'>-	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK		    0x7cUL</div><div class='del'>-	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT		    2</div><div class='add'>+	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK   0x7cUL</div><div class='add'>+	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT    2</div><div class='ctx'> 	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_4K    (0xcUL &lt;&lt; 2)</div><div class='ctx'> 	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_8K    (0xdUL &lt;&lt; 2)</div><div class='ctx'> 	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_64K   (0x10UL &lt;&lt; 2)</div><div class='hunk'>@@ -1437,16 +1422,15 @@ struct cmdq_register_mr {</div><div class='ctx'> 	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_2M    (0x15UL &lt;&lt; 2)</div><div class='ctx'> 	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_4M    (0x16UL &lt;&lt; 2)</div><div class='ctx'> 	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1G    (0x1eUL &lt;&lt; 2)</div><div class='del'>-	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_LAST	\</div><div class='del'>-					CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1G</div><div class='add'>+	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_LAST    CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1G</div><div class='ctx'> 	#define CMDQ_REGISTER_MR_UNUSED1             0x80UL</div><div class='del'>-	u8 access;</div><div class='del'>-	#define CMDQ_REGISTER_MR_ACCESS_LOCAL_WRITE		    0x1UL</div><div class='del'>-	#define CMDQ_REGISTER_MR_ACCESS_REMOTE_READ		    0x2UL</div><div class='del'>-	#define CMDQ_REGISTER_MR_ACCESS_REMOTE_WRITE		    0x4UL</div><div class='del'>-	#define CMDQ_REGISTER_MR_ACCESS_REMOTE_ATOMIC		    0x8UL</div><div class='del'>-	#define CMDQ_REGISTER_MR_ACCESS_MW_BIND		    0x10UL</div><div class='del'>-	#define CMDQ_REGISTER_MR_ACCESS_ZERO_BASED		    0x20UL</div><div class='add'>+	u8	access;</div><div class='add'>+	#define CMDQ_REGISTER_MR_ACCESS_LOCAL_WRITE       0x1UL</div><div class='add'>+	#define CMDQ_REGISTER_MR_ACCESS_REMOTE_READ       0x2UL</div><div class='add'>+	#define CMDQ_REGISTER_MR_ACCESS_REMOTE_WRITE      0x4UL</div><div class='add'>+	#define CMDQ_REGISTER_MR_ACCESS_REMOTE_ATOMIC     0x8UL</div><div class='add'>+	#define CMDQ_REGISTER_MR_ACCESS_MW_BIND           0x10UL</div><div class='add'>+	#define CMDQ_REGISTER_MR_ACCESS_ZERO_BASED        0x20UL</div><div class='ctx'> 	__le16	log2_pbl_pg_size;</div><div class='ctx'> 	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_MASK   0x1fUL</div><div class='ctx'> 	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_SFT    0</div><div class='hunk'>@@ -1458,1447 +1442,511 @@ struct cmdq_register_mr {</div><div class='ctx'> 	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M    0x15UL</div><div class='ctx'> 	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M    0x16UL</div><div class='ctx'> 	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G    0x1eUL</div><div class='del'>-	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_LAST    \</div><div class='del'>-				CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G</div><div class='add'>+	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_LAST    CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G</div><div class='ctx'> 	#define CMDQ_REGISTER_MR_UNUSED11_MASK           0xffe0UL</div><div class='ctx'> 	#define CMDQ_REGISTER_MR_UNUSED11_SFT            5</div><div class='del'>-	__le32 key;</div><div class='del'>-	__le64 pbl;</div><div class='del'>-	__le64 va;</div><div class='del'>-	__le64 mr_size;</div><div class='add'>+	__le32	key;</div><div class='add'>+	__le64	pbl;</div><div class='add'>+	__le64	va;</div><div class='add'>+	__le64	mr_size;</div><div class='add'>+	__le16  steering_tag;</div><div class='add'>+	u8      reserved48[6];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* Deregister MR command (24 bytes) */</div><div class='del'>-struct cmdq_deregister_mr {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_DEREGISTER_MR_OPCODE_DEREGISTER_MR	   0x10UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le32 lkey;</div><div class='del'>-	__le32 unused_0;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Add GID command (48 bytes) */</div><div class='del'>-struct cmdq_add_gid {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_ADD_GID_OPCODE_ADD_GID			   0x11UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__be32 gid[4];</div><div class='del'>-	__be16 src_mac[3];</div><div class='del'>-	__le16 vlan;</div><div class='del'>-	#define CMDQ_ADD_GID_VLAN_VLAN_ID_MASK			    0xfffUL</div><div class='del'>-	#define CMDQ_ADD_GID_VLAN_VLAN_ID_SFT			    0</div><div class='del'>-	#define CMDQ_ADD_GID_VLAN_TPID_MASK			    0x7000UL</div><div class='del'>-	#define CMDQ_ADD_GID_VLAN_TPID_SFT			    12</div><div class='del'>-	#define CMDQ_ADD_GID_VLAN_TPID_TPID_88A8		   (0x0UL &lt;&lt; 12)</div><div class='del'>-	#define CMDQ_ADD_GID_VLAN_TPID_TPID_8100		   (0x1UL &lt;&lt; 12)</div><div class='del'>-	#define CMDQ_ADD_GID_VLAN_TPID_TPID_9100		   (0x2UL &lt;&lt; 12)</div><div class='del'>-	#define CMDQ_ADD_GID_VLAN_TPID_TPID_9200		   (0x3UL &lt;&lt; 12)</div><div class='del'>-	#define CMDQ_ADD_GID_VLAN_TPID_TPID_9300		   (0x4UL &lt;&lt; 12)</div><div class='del'>-	#define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG1		   (0x5UL &lt;&lt; 12)</div><div class='del'>-	#define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG2		   (0x6UL &lt;&lt; 12)</div><div class='del'>-	#define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3		   (0x7UL &lt;&lt; 12)</div><div class='del'>-	#define CMDQ_ADD_GID_VLAN_TPID_LAST    CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3</div><div class='del'>-	#define CMDQ_ADD_GID_VLAN_VLAN_EN			    0x8000UL</div><div class='del'>-	__le16 ipid;</div><div class='del'>-	__le16 stats_ctx;</div><div class='del'>-	#define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_MASK	    0x7fffUL</div><div class='del'>-	#define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_SFT	    0</div><div class='del'>-	#define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID		    0x8000UL</div><div class='del'>-	__le32 unused_0;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Delete GID command (24 bytes) */</div><div class='del'>-struct cmdq_delete_gid {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_DELETE_GID_OPCODE_DELETE_GID		   0x12UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le16 gid_index;</div><div class='del'>-	__le16 unused_0;</div><div class='del'>-	__le32 unused_1;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Modify GID command (48 bytes) */</div><div class='del'>-struct cmdq_modify_gid {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_MODIFY_GID_OPCODE_MODIFY_GID		   0x17UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__be32 gid[4];</div><div class='del'>-	__be16 src_mac[3];</div><div class='del'>-	__le16 vlan;</div><div class='del'>-	#define CMDQ_MODIFY_GID_VLAN_VLAN_ID_MASK		    0xfffUL</div><div class='del'>-	#define CMDQ_MODIFY_GID_VLAN_VLAN_ID_SFT		    0</div><div class='del'>-	#define CMDQ_MODIFY_GID_VLAN_TPID_MASK			    0x7000UL</div><div class='del'>-	#define CMDQ_MODIFY_GID_VLAN_TPID_SFT			    12</div><div class='del'>-	#define CMDQ_MODIFY_GID_VLAN_TPID_TPID_88A8		   (0x0UL &lt;&lt; 12)</div><div class='del'>-	#define CMDQ_MODIFY_GID_VLAN_TPID_TPID_8100		   (0x1UL &lt;&lt; 12)</div><div class='del'>-	#define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9100		   (0x2UL &lt;&lt; 12)</div><div class='del'>-	#define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9200		   (0x3UL &lt;&lt; 12)</div><div class='del'>-	#define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9300		   (0x4UL &lt;&lt; 12)</div><div class='del'>-	#define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG1		   (0x5UL &lt;&lt; 12)</div><div class='del'>-	#define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG2		   (0x6UL &lt;&lt; 12)</div><div class='del'>-	#define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3		   (0x7UL &lt;&lt; 12)</div><div class='del'>-	#define CMDQ_MODIFY_GID_VLAN_TPID_LAST		\</div><div class='del'>-					CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3</div><div class='del'>-	#define CMDQ_MODIFY_GID_VLAN_VLAN_EN			    0x8000UL</div><div class='del'>-	__le16 ipid;</div><div class='del'>-	__le16 gid_index;</div><div class='del'>-	__le16 stats_ctx;</div><div class='del'>-	#define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_MASK	    0x7fffUL</div><div class='del'>-	#define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_SFT	    0</div><div class='del'>-	#define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_VALID	    0x8000UL</div><div class='del'>-	__le16 unused_0;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Query GID command (24 bytes) */</div><div class='del'>-struct cmdq_query_gid {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_QUERY_GID_OPCODE_QUERY_GID		   0x18UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le16 gid_index;</div><div class='del'>-	__le16 unused_0;</div><div class='del'>-	__le32 unused_1;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Create QP1 command (80 bytes) */</div><div class='del'>-struct cmdq_create_qp1 {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_CREATE_QP1_OPCODE_CREATE_QP1		   0x13UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le64 qp_handle;</div><div class='del'>-	__le32 qp_flags;</div><div class='del'>-	#define CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED		   0x1UL</div><div class='del'>-	#define CMDQ_CREATE_QP1_QP_FLAGS_FORCE_COMPLETION	   0x2UL</div><div class='del'>-	#define CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE     0x4UL</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CMDQ_CREATE_QP1_TYPE_GSI			   0x1UL</div><div class='del'>-	u8 sq_pg_size_sq_lvl;</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_LVL_MASK			    0xfUL</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_LVL_SFT			    0</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_LVL_LVL_0			   0x0UL</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_LVL_LVL_1			   0x1UL</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_LVL_LVL_2			   0x2UL</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_MASK		    0xf0UL</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT			    4</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K		   (0x0UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K		   (0x1UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K		   (0x2UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M		   (0x3UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M		   (0x4UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G		   (0x5UL &lt;&lt; 4)</div><div class='del'>-	u8 rq_pg_size_rq_lvl;</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_LVL_MASK			    0xfUL</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_LVL_SFT			    0</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_LVL_LVL_0			   0x0UL</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_LVL_LVL_1			   0x1UL</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_LVL_LVL_2			   0x2UL</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_MASK		    0xf0UL</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT			    4</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K		   (0x0UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K		   (0x1UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K		   (0x2UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M		   (0x3UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M		   (0x4UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G		   (0x5UL &lt;&lt; 4)</div><div class='del'>-	u8 unused_0;</div><div class='del'>-	__le32 dpi;</div><div class='del'>-	__le32 sq_size;</div><div class='del'>-	__le32 rq_size;</div><div class='del'>-	__le16 sq_fwo_sq_sge;</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_SGE_MASK			    0xfUL</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_SGE_SFT			    0</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_FWO_MASK			    0xfff0UL</div><div class='del'>-	#define CMDQ_CREATE_QP1_SQ_FWO_SFT			    4</div><div class='del'>-	__le16 rq_fwo_rq_sge;</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_SGE_MASK			    0xfUL</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_SGE_SFT			    0</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_FWO_MASK			    0xfff0UL</div><div class='del'>-	#define CMDQ_CREATE_QP1_RQ_FWO_SFT			    4</div><div class='del'>-	__le32 scq_cid;</div><div class='del'>-	__le32 rcq_cid;</div><div class='del'>-	__le32 srq_cid;</div><div class='del'>-	__le32 pd_id;</div><div class='del'>-	__le64 sq_pbl;</div><div class='del'>-	__le64 rq_pbl;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Destroy QP1 command (24 bytes) */</div><div class='del'>-struct cmdq_destroy_qp1 {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_DESTROY_QP1_OPCODE_DESTROY_QP1		   0x14UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le32 qp1_cid;</div><div class='del'>-	__le32 unused_0;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Create AH command (64 bytes) */</div><div class='del'>-struct cmdq_create_ah {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_CREATE_AH_OPCODE_CREATE_AH		   0x15UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le64 ah_handle;</div><div class='del'>-	__le32 dgid[4];</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CMDQ_CREATE_AH_TYPE_V1				   0x0UL</div><div class='del'>-	#define CMDQ_CREATE_AH_TYPE_V2IPV4			   0x2UL</div><div class='del'>-	#define CMDQ_CREATE_AH_TYPE_V2IPV6			   0x3UL</div><div class='del'>-	u8 hop_limit;</div><div class='del'>-	__le16 sgid_index;</div><div class='del'>-	__le32 dest_vlan_id_flow_label;</div><div class='del'>-	#define CMDQ_CREATE_AH_FLOW_LABEL_MASK			    0xfffffUL</div><div class='del'>-	#define CMDQ_CREATE_AH_FLOW_LABEL_SFT			    0</div><div class='del'>-	#define CMDQ_CREATE_AH_DEST_VLAN_ID_MASK		    0xfff00000UL</div><div class='del'>-	#define CMDQ_CREATE_AH_DEST_VLAN_ID_SFT		    20</div><div class='del'>-	__le32 pd_id;</div><div class='del'>-	__le32 unused_0;</div><div class='del'>-	__le16 dest_mac[3];</div><div class='del'>-	u8 traffic_class;</div><div class='del'>-	u8 unused_1;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Destroy AH command (24 bytes) */</div><div class='del'>-struct cmdq_destroy_ah {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_DESTROY_AH_OPCODE_DESTROY_AH		   0x16UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le32 ah_cid;</div><div class='del'>-	__le32 unused_0;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Initialize Firmware command (112 bytes) */</div><div class='del'>-struct cmdq_initialize_fw {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_OPCODE_INITIALIZE_FW	   0x80UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	u8 qpc_pg_size_qpc_lvl;</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_QPC_LVL_MASK		    0xfUL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_QPC_LVL_SFT			    0</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_0		   0x0UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_1		   0x1UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_2		   0x2UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_MASK		    0xf0UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT		    4</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K		   (0x0UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K		   (0x1UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K		   (0x2UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M		   (0x3UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M		   (0x4UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G		   (0x5UL &lt;&lt; 4)</div><div class='del'>-	u8 mrw_pg_size_mrw_lvl;</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_MRW_LVL_MASK		    0xfUL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_MRW_LVL_SFT			    0</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_0		   0x0UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_1		   0x1UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_2		   0x2UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_MASK		    0xf0UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT		    4</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_4K		   (0x0UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8K		   (0x1UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_64K		   (0x2UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_2M		   (0x3UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8M		   (0x4UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_1G		   (0x5UL &lt;&lt; 4)</div><div class='del'>-	u8 srq_pg_size_srq_lvl;</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_SRQ_LVL_MASK		    0xfUL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_SRQ_LVL_SFT			    0</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_0		   0x0UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_1		   0x1UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_2		   0x2UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_MASK		    0xf0UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT		    4</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_4K		   (0x0UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8K		   (0x1UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_64K		   (0x2UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_2M		   (0x3UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8M		   (0x4UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_1G		   (0x5UL &lt;&lt; 4)</div><div class='del'>-	u8 cq_pg_size_cq_lvl;</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_CQ_LVL_MASK			    0xfUL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_CQ_LVL_SFT			    0</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_0		   0x0UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_1		   0x1UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_2		   0x2UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_MASK		    0xf0UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT		    4</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_4K		   (0x0UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8K		   (0x1UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_64K		   (0x2UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_2M		   (0x3UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8M		   (0x4UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_1G		   (0x5UL &lt;&lt; 4)</div><div class='del'>-	u8 tqm_pg_size_tqm_lvl;</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TQM_LVL_MASK		    0xfUL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TQM_LVL_SFT			    0</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_0		   0x0UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_1		   0x1UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_2		   0x2UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_MASK		    0xf0UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT		    4</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_4K		   (0x0UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8K		   (0x1UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_64K		   (0x2UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_2M		   (0x3UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8M		   (0x4UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_1G		   (0x5UL &lt;&lt; 4)</div><div class='del'>-	u8 tim_pg_size_tim_lvl;</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TIM_LVL_MASK		    0xfUL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TIM_LVL_SFT			    0</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_0		   0x0UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_1		   0x1UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_2		   0x2UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_MASK		    0xf0UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT		    4</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_4K		   (0x0UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8K		   (0x1UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_64K		   (0x2UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_2M		   (0x3UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8M		   (0x4UL &lt;&lt; 4)</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_1G		   (0x5UL &lt;&lt; 4)</div><div class='del'>-	/* This value is (log-base-2-of-DBR-page-size - 12).</div><div class='del'>-	 * 0 for 4KB. HW supported values are enumerated below.</div><div class='del'>-	 */</div><div class='del'>-	__le16  log2_dbr_pg_size;</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_MASK	0xfUL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_SFT		0</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_4K	0x0UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_8K	0x1UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_16K	0x2UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_32K	0x3UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_64K	0x4UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128K	0x5UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_256K	0x6UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_512K	0x7UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_1M	0x8UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_2M	0x9UL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_4M	0xaUL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_8M	0xbUL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_16M	0xcUL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_32M	0xdUL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_64M	0xeUL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128M	0xfUL</div><div class='del'>-	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_LAST		\</div><div class='del'>-			CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128M</div><div class='del'>-	__le64 qpc_page_dir;</div><div class='del'>-	__le64 mrw_page_dir;</div><div class='del'>-	__le64 srq_page_dir;</div><div class='del'>-	__le64 cq_page_dir;</div><div class='del'>-	__le64 tqm_page_dir;</div><div class='del'>-	__le64 tim_page_dir;</div><div class='del'>-	__le32 number_of_qp;</div><div class='del'>-	__le32 number_of_mrw;</div><div class='del'>-	__le32 number_of_srq;</div><div class='del'>-	__le32 number_of_cq;</div><div class='del'>-	__le32 max_qp_per_vf;</div><div class='del'>-	__le32 max_mrw_per_vf;</div><div class='del'>-	__le32 max_srq_per_vf;</div><div class='del'>-	__le32 max_cq_per_vf;</div><div class='del'>-	__le32 max_gid_per_vf;</div><div class='del'>-	__le32 stat_ctx_id;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* De-initialize Firmware command (16 bytes) */</div><div class='del'>-struct cmdq_deinitialize_fw {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_DEINITIALIZE_FW_OPCODE_DEINITIALIZE_FW       0x81UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Stop function command (16 bytes) */</div><div class='del'>-struct cmdq_stop_func {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_STOP_FUNC_OPCODE_STOP_FUNC		   0x82UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Query function command (16 bytes) */</div><div class='del'>-struct cmdq_query_func {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_QUERY_FUNC_OPCODE_QUERY_FUNC		   0x83UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='add'>+/* creq_register_mr_resp (size:128b/16B) */</div><div class='add'>+struct creq_register_mr_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_REGISTER_MR_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_REGISTER_MR_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_REGISTER_MR_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_REGISTER_MR_RESP_TYPE_LAST     CREQ_REGISTER_MR_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_REGISTER_MR_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_REGISTER_MR_RESP_EVENT_REGISTER_MR 0xfUL</div><div class='add'>+	#define CREQ_REGISTER_MR_RESP_EVENT_LAST       CREQ_REGISTER_MR_RESP_EVENT_REGISTER_MR</div><div class='add'>+	u8	reserved48[6];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* Set function resources command (16 bytes) */</div><div class='del'>-struct cmdq_set_func_resources {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_SET_FUNC_RESOURCES_OPCODE_SET_FUNC_RESOURCES 0x84UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le32 number_of_qp;</div><div class='del'>-	__le32 number_of_mrw;</div><div class='del'>-	__le32 number_of_srq;</div><div class='del'>-	__le32 number_of_cq;</div><div class='del'>-	__le32 max_qp_per_vf;</div><div class='del'>-	__le32 max_mrw_per_vf;</div><div class='del'>-	__le32 max_srq_per_vf;</div><div class='del'>-	__le32 max_cq_per_vf;</div><div class='del'>-	__le32 max_gid_per_vf;</div><div class='del'>-	__le32 stat_ctx_id;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Read hardware resource context command (24 bytes) */</div><div class='del'>-struct cmdq_read_context {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_READ_CONTEXT_OPCODE_READ_CONTEXT		   0x85UL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le32 type_xid;</div><div class='del'>-	#define CMDQ_READ_CONTEXT_XID_MASK			    0xffffffUL</div><div class='del'>-	#define CMDQ_READ_CONTEXT_XID_SFT			    0</div><div class='del'>-	#define CMDQ_READ_CONTEXT_TYPE_MASK			    0xff000000UL</div><div class='del'>-	#define CMDQ_READ_CONTEXT_TYPE_SFT			    24</div><div class='del'>-	#define CMDQ_READ_CONTEXT_TYPE_QPC			   (0x0UL &lt;&lt; 24)</div><div class='del'>-	#define CMDQ_READ_CONTEXT_TYPE_CQ			   (0x1UL &lt;&lt; 24)</div><div class='del'>-	#define CMDQ_READ_CONTEXT_TYPE_MRW			   (0x2UL &lt;&lt; 24)</div><div class='del'>-	#define CMDQ_READ_CONTEXT_TYPE_SRQ			   (0x3UL &lt;&lt; 24)</div><div class='del'>-	__le32 unused_0;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Map TC to COS. Can only be issued from a PF (24 bytes) */</div><div class='del'>-struct cmdq_map_tc_to_cos {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_MAP_TC_TO_COS_OPCODE_MAP_TC_TO_COS	   0x8aUL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-	__le16 cos0;</div><div class='del'>-	#define CMDQ_MAP_TC_TO_COS_COS0_NO_CHANGE		   0xffffUL</div><div class='del'>-	__le16 cos1;</div><div class='del'>-	#define CMDQ_MAP_TC_TO_COS_COS1_DISABLE		   0x8000UL</div><div class='del'>-	#define CMDQ_MAP_TC_TO_COS_COS1_NO_CHANGE		   0xffffUL</div><div class='del'>-	__le32 unused_0;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Query version command (16 bytes) */</div><div class='del'>-struct cmdq_query_version {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CMDQ_QUERY_VERSION_OPCODE_QUERY_VERSION	   0x8bUL</div><div class='del'>-	u8 cmd_size;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 resp_addr;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Command-Response Event Queue (CREQ) Structures */</div><div class='del'>-/* Base CREQ Record (16 bytes) */</div><div class='del'>-struct creq_base {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_BASE_TYPE_MASK				    0x3fUL</div><div class='del'>-	#define CREQ_BASE_TYPE_SFT				    0</div><div class='del'>-	#define CREQ_BASE_TYPE_QP_EVENT			   0x38UL</div><div class='del'>-	#define CREQ_BASE_TYPE_FUNC_EVENT			   0x3aUL</div><div class='del'>-	#define CREQ_BASE_RESERVED2_MASK			    0xc0UL</div><div class='del'>-	#define CREQ_BASE_RESERVED2_SFT			    6</div><div class='del'>-	u8 reserved56[7];</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_BASE_V					    0x1UL</div><div class='del'>-	#define CREQ_BASE_RESERVED7_MASK			    0xfeUL</div><div class='del'>-	#define CREQ_BASE_RESERVED7_SFT			    1</div><div class='del'>-	u8 event;</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* RoCE Function Async Event Notification (16 bytes) */</div><div class='del'>-struct creq_func_event {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_FUNC_EVENT_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_FUNC_EVENT_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_FUNC_EVENT_TYPE_FUNC_EVENT		   0x3aUL</div><div class='del'>-	#define CREQ_FUNC_EVENT_RESERVED2_MASK			    0xc0UL</div><div class='del'>-	#define CREQ_FUNC_EVENT_RESERVED2_SFT			    6</div><div class='del'>-	u8 reserved56[7];</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_FUNC_EVENT_V				    0x1UL</div><div class='del'>-	#define CREQ_FUNC_EVENT_RESERVED7_MASK			    0xfeUL</div><div class='del'>-	#define CREQ_FUNC_EVENT_RESERVED7_SFT			    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR		   0x1UL</div><div class='del'>-	#define CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR		   0x2UL</div><div class='del'>-	#define CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR		   0x3UL</div><div class='del'>-	#define CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR		   0x4UL</div><div class='del'>-	#define CREQ_FUNC_EVENT_EVENT_CQ_ERROR			   0x5UL</div><div class='del'>-	#define CREQ_FUNC_EVENT_EVENT_TQM_ERROR		   0x6UL</div><div class='del'>-	#define CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR		   0x7UL</div><div class='del'>-	#define CREQ_FUNC_EVENT_EVENT_CFCS_ERROR		   0x8UL</div><div class='del'>-	#define CREQ_FUNC_EVENT_EVENT_CFCC_ERROR		   0x9UL</div><div class='del'>-	#define CREQ_FUNC_EVENT_EVENT_CFCM_ERROR		   0xaUL</div><div class='del'>-	#define CREQ_FUNC_EVENT_EVENT_TIM_ERROR		   0xbUL</div><div class='del'>-	#define CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST		   0x80UL</div><div class='del'>-	#define CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED	   0x81UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* RoCE Slowpath Command Completion (16 bytes) */</div><div class='del'>-struct creq_qp_event {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_QP_EVENT_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_QP_EVENT_TYPE_SFT				    0</div><div class='del'>-	#define CREQ_QP_EVENT_TYPE_QP_EVENT			   0x38UL</div><div class='del'>-	#define CREQ_QP_EVENT_RESERVED2_MASK			    0xc0UL</div><div class='del'>-	#define CREQ_QP_EVENT_RESERVED2_SFT			    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 reserved32;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_QP_EVENT_V				    0x1UL</div><div class='del'>-	#define CREQ_QP_EVENT_RESERVED7_MASK			    0xfeUL</div><div class='del'>-	#define CREQ_QP_EVENT_RESERVED7_SFT			    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_CREATE_QP			   0x1UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_DESTROY_QP			   0x2UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_MODIFY_QP			   0x3UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_QUERY_QP			   0x4UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_CREATE_SRQ			   0x5UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_DESTROY_SRQ		   0x6UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_QUERY_SRQ			   0x8UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_CREATE_CQ			   0x9UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_DESTROY_CQ			   0xaUL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_RESIZE_CQ			   0xcUL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_ALLOCATE_MRW		   0xdUL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_DEALLOCATE_KEY		   0xeUL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_REGISTER_MR		   0xfUL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_DEREGISTER_MR		   0x10UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_ADD_GID			   0x11UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_DELETE_GID			   0x12UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_MODIFY_GID			   0x17UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_QUERY_GID			   0x18UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_CREATE_QP1			   0x13UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_DESTROY_QP1		   0x14UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_CREATE_AH			   0x15UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_DESTROY_AH			   0x16UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_INITIALIZE_FW		   0x80UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_DEINITIALIZE_FW		   0x81UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_STOP_FUNC			   0x82UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_QUERY_FUNC			   0x83UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_SET_FUNC_RESOURCES		   0x84UL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_MAP_TC_TO_COS		   0x8aUL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_QUERY_VERSION		   0x8bUL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_MODIFY_CC			   0x8cUL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_QUERY_CC			   0x8dUL</div><div class='del'>-	#define CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION	   0xc0UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Create QP command response (16 bytes) */</div><div class='del'>-struct creq_create_qp_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_CREATE_QP_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_CREATE_QP_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_CREATE_QP_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_CREATE_QP_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_CREATE_QP_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_CREATE_QP_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_CREATE_QP_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_CREATE_QP_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_CREATE_QP_RESP_EVENT_CREATE_QP		   0x1UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Destroy QP command response (16 bytes) */</div><div class='del'>-struct creq_destroy_qp_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_DESTROY_QP_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_DESTROY_QP_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_DESTROY_QP_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_DESTROY_QP_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_DESTROY_QP_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_DESTROY_QP_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_DESTROY_QP_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_DESTROY_QP_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_DESTROY_QP_RESP_EVENT_DESTROY_QP		   0x2UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Modify QP command response (16 bytes) */</div><div class='del'>-struct creq_modify_qp_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_MODIFY_QP_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_MODIFY_QP_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_MODIFY_QP_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_MODIFY_QP_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_MODIFY_QP_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_MODIFY_QP_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_MODIFY_QP_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_MODIFY_QP_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_MODIFY_QP_RESP_EVENT_MODIFY_QP		   0x3UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* cmdq_query_roce_stats (size:128b/16B) */</div><div class='del'>-struct cmdq_query_roce_stats {</div><div class='add'>+/* cmdq_deregister_mr (size:192b/24B) */</div><div class='add'>+struct cmdq_deregister_mr {</div><div class='ctx'> 	u8	opcode;</div><div class='del'>-	#define CMDQ_QUERY_ROCE_STATS_OPCODE_QUERY_ROCE_STATS 0x8eUL</div><div class='del'>-	#define CMDQ_QUERY_ROCE_STATS_OPCODE_LAST	\</div><div class='del'>-				CMDQ_QUERY_ROCE_STATS_OPCODE_QUERY_ROCE_STATS</div><div class='add'>+	#define CMDQ_DEREGISTER_MR_OPCODE_DEREGISTER_MR 0x10UL</div><div class='add'>+	#define CMDQ_DEREGISTER_MR_OPCODE_LAST         CMDQ_DEREGISTER_MR_OPCODE_DEREGISTER_MR</div><div class='ctx'> 	u8	cmd_size;</div><div class='ctx'> 	__le16	flags;</div><div class='ctx'> 	__le16	cookie;</div><div class='ctx'> 	u8	resp_size;</div><div class='ctx'> 	u8	reserved8;</div><div class='ctx'> 	__le64	resp_addr;</div><div class='add'>+	__le32	lkey;</div><div class='add'>+	__le32	unused_0;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* Query QP command response (16 bytes) */</div><div class='del'>-struct creq_query_qp_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 size;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_EVENT_QUERY_QP		   0x4UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Query QP command response side buffer structure (104 bytes) */</div><div class='del'>-struct creq_query_qp_resp_sb {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_OPCODE_QUERY_QP		   0x4UL</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 en_sqd_async_notify_state;</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_STATE_MASK		    0xfUL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_STATE_SFT		    0</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_STATE_RESET		   0x0UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_STATE_INIT		   0x1UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_STATE_RTR		   0x2UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_STATE_RTS		   0x3UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_STATE_SQD		   0x4UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_STATE_SQE		   0x5UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_STATE_ERR		   0x6UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY	    0x10UL</div><div class='del'>-	u8 access;</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_ACCESS_LOCAL_WRITE	    0x1UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_WRITE	    0x2UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_READ	    0x4UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC	    0x8UL</div><div class='del'>-	__le16 pkey;</div><div class='del'>-	__le32 qkey;</div><div class='del'>-	__le32 reserved32;</div><div class='del'>-	__le32 dgid[4];</div><div class='del'>-	__le32 flow_label;</div><div class='del'>-	__le16 sgid_index;</div><div class='del'>-	u8 hop_limit;</div><div class='del'>-	u8 traffic_class;</div><div class='del'>-	__le16 dest_mac[3];</div><div class='del'>-	__le16 path_mtu_dest_vlan_id;</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_MASK	    0xfffUL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_SFT		    0</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK		    0xf000UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT		    12</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_256		   (0x0UL &lt;&lt; 12)</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_512		   (0x1UL &lt;&lt; 12)</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_1024	   (0x2UL &lt;&lt; 12)</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_2048	   (0x3UL &lt;&lt; 12)</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_4096	   (0x4UL &lt;&lt; 12)</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_8192	   (0x5UL &lt;&lt; 12)</div><div class='del'>-	u8 timeout;</div><div class='del'>-	u8 retry_cnt;</div><div class='del'>-	u8 rnr_retry;</div><div class='del'>-	u8 min_rnr_timer;</div><div class='del'>-	__le32 rq_psn;</div><div class='del'>-	__le32 sq_psn;</div><div class='del'>-	u8 max_rd_atomic;</div><div class='del'>-	u8 max_dest_rd_atomic;</div><div class='del'>-	u8 tos_dscp_tos_ecn;</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_TOS_ECN_MASK		    0x3UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_TOS_ECN_SFT		    0</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_MASK		    0xfcUL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_SFT		    2</div><div class='del'>-	u8 enable_cc;</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_ENABLE_CC		    0x1UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_RESERVED7_SFT		    1</div><div class='del'>-	__le32 sq_size;</div><div class='del'>-	__le32 rq_size;</div><div class='del'>-	__le16 sq_sge;</div><div class='del'>-	__le16 rq_sge;</div><div class='del'>-	__le32 max_inline_data;</div><div class='del'>-	__le32 dest_qp_id;</div><div class='del'>-	__le32 unused_1;</div><div class='del'>-	__le16 src_mac[3];</div><div class='del'>-	__le16 vlan_pcp_vlan_dei_vlan_id;</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK		    0xfffUL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT		    0</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_VLAN_DEI			    0x1000UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_MASK		    0xe000UL</div><div class='del'>-	#define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_SFT		    13</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Create SRQ command response (16 bytes) */</div><div class='del'>-struct creq_create_srq_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_CREATE_SRQ_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_CREATE_SRQ_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_CREATE_SRQ_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_CREATE_SRQ_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_CREATE_SRQ_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_CREATE_SRQ_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_CREATE_SRQ_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_CREATE_SRQ_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_CREATE_SRQ_RESP_EVENT_CREATE_SRQ		   0x5UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Destroy SRQ command response (16 bytes) */</div><div class='del'>-struct creq_destroy_srq_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_DESTROY_SRQ_RESP_TYPE_MASK		    0x3fUL</div><div class='del'>-	#define CREQ_DESTROY_SRQ_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_DESTROY_SRQ_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_DESTROY_SRQ_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_DESTROY_SRQ_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_DESTROY_SRQ_RESP_V			    0x1UL</div><div class='del'>-	#define CREQ_DESTROY_SRQ_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_DESTROY_SRQ_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_DESTROY_SRQ_RESP_EVENT_DESTROY_SRQ	   0x6UL</div><div class='del'>-	__le16 enable_for_arm[3];</div><div class='del'>-	#define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_MASK	    0x30000UL</div><div class='del'>-	#define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_SFT	    16</div><div class='del'>-	#define CREQ_DESTROY_SRQ_RESP_RESERVED46_MASK		    0xfffc0000UL</div><div class='del'>-	#define CREQ_DESTROY_SRQ_RESP_RESERVED46_SFT		    18</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Query SRQ command response (16 bytes) */</div><div class='del'>-struct creq_query_srq_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_QUERY_SRQ_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_QUERY_SRQ_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_QUERY_SRQ_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_QUERY_SRQ_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_QUERY_SRQ_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 size;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_QUERY_SRQ_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_QUERY_SRQ_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_QUERY_SRQ_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_QUERY_SRQ_RESP_EVENT_QUERY_SRQ		   0x8UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Query SRQ command response side buffer structure (24 bytes) */</div><div class='del'>-struct creq_query_srq_resp_sb {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CREQ_QUERY_SRQ_RESP_SB_OPCODE_QUERY_SRQ	   0x8UL</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	__le16 srq_limit;</div><div class='del'>-	__le16 reserved16;</div><div class='del'>-	__le32 data[4];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Create CQ command Response (16 bytes) */</div><div class='del'>-struct creq_create_cq_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_CREATE_CQ_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_CREATE_CQ_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_CREATE_CQ_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_CREATE_CQ_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_CREATE_CQ_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_CREATE_CQ_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_CREATE_CQ_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_CREATE_CQ_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_CREATE_CQ_RESP_EVENT_CREATE_CQ		   0x9UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Destroy CQ command response (16 bytes) */</div><div class='del'>-struct creq_destroy_cq_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_DESTROY_CQ_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_DESTROY_CQ_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_DESTROY_CQ_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_DESTROY_CQ_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_DESTROY_CQ_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_DESTROY_CQ_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_DESTROY_CQ_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_DESTROY_CQ_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_DESTROY_CQ_RESP_EVENT_DESTROY_CQ		   0xaUL</div><div class='del'>-	__le16 cq_arm_lvl;</div><div class='del'>-	#define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_MASK		    0x3UL</div><div class='del'>-	#define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_SFT		    0</div><div class='del'>-	#define CREQ_DESTROY_CQ_RESP_RESERVED14_MASK		    0xfffcUL</div><div class='del'>-	#define CREQ_DESTROY_CQ_RESP_RESERVED14_SFT		    2</div><div class='del'>-	__le16 total_cnq_events;</div><div class='del'>-	__le16 reserved16;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Resize CQ command response (16 bytes) */</div><div class='del'>-struct creq_resize_cq_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_RESIZE_CQ_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_RESIZE_CQ_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_RESIZE_CQ_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_RESIZE_CQ_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_RESIZE_CQ_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_RESIZE_CQ_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_RESIZE_CQ_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_RESIZE_CQ_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_RESIZE_CQ_RESP_EVENT_RESIZE_CQ		   0xcUL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Allocate MRW command response (16 bytes) */</div><div class='del'>-struct creq_allocate_mrw_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_ALLOCATE_MRW_RESP_TYPE_MASK		    0x3fUL</div><div class='del'>-	#define CREQ_ALLOCATE_MRW_RESP_TYPE_SFT		    0</div><div class='del'>-	#define CREQ_ALLOCATE_MRW_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_ALLOCATE_MRW_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_ALLOCATE_MRW_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_ALLOCATE_MRW_RESP_V			    0x1UL</div><div class='del'>-	#define CREQ_ALLOCATE_MRW_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_ALLOCATE_MRW_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_ALLOCATE_MRW_RESP_EVENT_ALLOCATE_MRW	   0xdUL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* De-allocate key command response (16 bytes) */</div><div class='del'>-struct creq_deallocate_key_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_DEALLOCATE_KEY_RESP_TYPE_MASK		    0x3fUL</div><div class='del'>-	#define CREQ_DEALLOCATE_KEY_RESP_TYPE_SFT		    0</div><div class='del'>-	#define CREQ_DEALLOCATE_KEY_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_DEALLOCATE_KEY_RESP_RESERVED2_MASK	    0xc0UL</div><div class='del'>-	#define CREQ_DEALLOCATE_KEY_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_DEALLOCATE_KEY_RESP_V			    0x1UL</div><div class='del'>-	#define CREQ_DEALLOCATE_KEY_RESP_RESERVED7_MASK	    0xfeUL</div><div class='del'>-	#define CREQ_DEALLOCATE_KEY_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_DEALLOCATE_KEY_RESP_EVENT_DEALLOCATE_KEY     0xeUL</div><div class='del'>-	__le16 reserved16;</div><div class='del'>-	__le32 bound_window_info;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Register MR command response (16 bytes) */</div><div class='del'>-struct creq_register_mr_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_REGISTER_MR_RESP_TYPE_MASK		    0x3fUL</div><div class='del'>-	#define CREQ_REGISTER_MR_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_REGISTER_MR_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_REGISTER_MR_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_REGISTER_MR_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_REGISTER_MR_RESP_V			    0x1UL</div><div class='del'>-	#define CREQ_REGISTER_MR_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_REGISTER_MR_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_REGISTER_MR_RESP_EVENT_REGISTER_MR	   0xfUL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Deregister MR command response (16 bytes) */</div><div class='add'>+/* creq_deregister_mr_resp (size:128b/16B) */</div><div class='ctx'> struct creq_deregister_mr_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_DEREGISTER_MR_RESP_TYPE_MASK		    0x3fUL</div><div class='del'>-	#define CREQ_DEREGISTER_MR_RESP_TYPE_SFT		    0</div><div class='del'>-	#define CREQ_DEREGISTER_MR_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_DEREGISTER_MR_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_DEREGISTER_MR_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_DEREGISTER_MR_RESP_V			    0x1UL</div><div class='del'>-	#define CREQ_DEREGISTER_MR_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_DEREGISTER_MR_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_DEREGISTER_MR_RESP_EVENT_DEREGISTER_MR       0x10UL</div><div class='del'>-	__le16 reserved16;</div><div class='del'>-	__le32 bound_windows;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Add GID command response (16 bytes) */</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_DEREGISTER_MR_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_DEREGISTER_MR_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_DEREGISTER_MR_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_DEREGISTER_MR_RESP_TYPE_LAST     CREQ_DEREGISTER_MR_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_DEREGISTER_MR_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_DEREGISTER_MR_RESP_EVENT_DEREGISTER_MR 0x10UL</div><div class='add'>+	#define CREQ_DEREGISTER_MR_RESP_EVENT_LAST CREQ_DEREGISTER_MR_RESP_EVENT_DEREGISTER_MR</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	bound_windows;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_add_gid (size:384b/48B) */</div><div class='add'>+struct cmdq_add_gid {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_ADD_GID_OPCODE_ADD_GID 0x11UL</div><div class='add'>+	#define CMDQ_ADD_GID_OPCODE_LAST   CMDQ_ADD_GID_OPCODE_ADD_GID</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__be32	gid[4];</div><div class='add'>+	__be16	src_mac[3];</div><div class='add'>+	__le16	vlan;</div><div class='add'>+	#define CMDQ_ADD_GID_VLAN_VLAN_EN_TPID_VLAN_ID_MASK          0xffffUL</div><div class='add'>+	#define CMDQ_ADD_GID_VLAN_VLAN_EN_TPID_VLAN_ID_SFT           0</div><div class='add'>+	#define CMDQ_ADD_GID_VLAN_VLAN_ID_MASK                       0xfffUL</div><div class='add'>+	#define CMDQ_ADD_GID_VLAN_VLAN_ID_SFT                        0</div><div class='add'>+	#define CMDQ_ADD_GID_VLAN_TPID_MASK                          0x7000UL</div><div class='add'>+	#define CMDQ_ADD_GID_VLAN_TPID_SFT                           12</div><div class='add'>+	#define CMDQ_ADD_GID_VLAN_TPID_TPID_88A8                       (0x0UL &lt;&lt; 12)</div><div class='add'>+	#define CMDQ_ADD_GID_VLAN_TPID_TPID_8100                       (0x1UL &lt;&lt; 12)</div><div class='add'>+	#define CMDQ_ADD_GID_VLAN_TPID_TPID_9100                       (0x2UL &lt;&lt; 12)</div><div class='add'>+	#define CMDQ_ADD_GID_VLAN_TPID_TPID_9200                       (0x3UL &lt;&lt; 12)</div><div class='add'>+	#define CMDQ_ADD_GID_VLAN_TPID_TPID_9300                       (0x4UL &lt;&lt; 12)</div><div class='add'>+	#define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG1                       (0x5UL &lt;&lt; 12)</div><div class='add'>+	#define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG2                       (0x6UL &lt;&lt; 12)</div><div class='add'>+	#define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3                       (0x7UL &lt;&lt; 12)</div><div class='add'>+	#define CMDQ_ADD_GID_VLAN_TPID_LAST CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3</div><div class='add'>+	#define CMDQ_ADD_GID_VLAN_VLAN_EN                            0x8000UL</div><div class='add'>+	__le16	ipid;</div><div class='add'>+	__le16	stats_ctx;</div><div class='add'>+	#define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID_STATS_CTX_ID_MASK                0xffffUL</div><div class='add'>+	#define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID_STATS_CTX_ID_SFT                 0</div><div class='add'>+	#define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_MASK                                0x7fffUL</div><div class='add'>+	#define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_SFT                                 0</div><div class='add'>+	#define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID                                  0x8000UL</div><div class='add'>+	__le32	unused_0;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_add_gid_resp (size:128b/16B) */</div><div class='ctx'> struct creq_add_gid_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_ADD_GID_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_ADD_GID_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_ADD_GID_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_ADD_GID_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_ADD_GID_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_ADD_GID_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_ADD_GID_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_ADD_GID_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_ADD_GID_RESP_EVENT_ADD_GID		   0x11UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Delete GID command response (16 bytes) */</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_ADD_GID_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_ADD_GID_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_ADD_GID_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_ADD_GID_RESP_TYPE_LAST     CREQ_ADD_GID_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_ADD_GID_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_ADD_GID_RESP_EVENT_ADD_GID 0x11UL</div><div class='add'>+	#define CREQ_ADD_GID_RESP_EVENT_LAST   CREQ_ADD_GID_RESP_EVENT_ADD_GID</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_delete_gid (size:192b/24B) */</div><div class='add'>+struct cmdq_delete_gid {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_DELETE_GID_OPCODE_DELETE_GID 0x12UL</div><div class='add'>+	#define CMDQ_DELETE_GID_OPCODE_LAST      CMDQ_DELETE_GID_OPCODE_DELETE_GID</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le16	gid_index;</div><div class='add'>+	u8	unused_0[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_delete_gid_resp (size:128b/16B) */</div><div class='ctx'> struct creq_delete_gid_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_DELETE_GID_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_DELETE_GID_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_DELETE_GID_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_DELETE_GID_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_DELETE_GID_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_DELETE_GID_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_DELETE_GID_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_DELETE_GID_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_DELETE_GID_RESP_EVENT_DELETE_GID		   0x12UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Modify GID command response (16 bytes) */</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_DELETE_GID_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_DELETE_GID_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_DELETE_GID_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_DELETE_GID_RESP_TYPE_LAST     CREQ_DELETE_GID_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_DELETE_GID_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_DELETE_GID_RESP_EVENT_DELETE_GID 0x12UL</div><div class='add'>+	#define CREQ_DELETE_GID_RESP_EVENT_LAST      CREQ_DELETE_GID_RESP_EVENT_DELETE_GID</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_modify_gid (size:384b/48B) */</div><div class='add'>+struct cmdq_modify_gid {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_MODIFY_GID_OPCODE_MODIFY_GID 0x17UL</div><div class='add'>+	#define CMDQ_MODIFY_GID_OPCODE_LAST      CMDQ_MODIFY_GID_OPCODE_MODIFY_GID</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__be32	gid[4];</div><div class='add'>+	__be16	src_mac[3];</div><div class='add'>+	__le16	vlan;</div><div class='add'>+	#define CMDQ_MODIFY_GID_VLAN_VLAN_ID_MASK  0xfffUL</div><div class='add'>+	#define CMDQ_MODIFY_GID_VLAN_VLAN_ID_SFT   0</div><div class='add'>+	#define CMDQ_MODIFY_GID_VLAN_TPID_MASK     0x7000UL</div><div class='add'>+	#define CMDQ_MODIFY_GID_VLAN_TPID_SFT      12</div><div class='add'>+	#define CMDQ_MODIFY_GID_VLAN_TPID_TPID_88A8  (0x0UL &lt;&lt; 12)</div><div class='add'>+	#define CMDQ_MODIFY_GID_VLAN_TPID_TPID_8100  (0x1UL &lt;&lt; 12)</div><div class='add'>+	#define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9100  (0x2UL &lt;&lt; 12)</div><div class='add'>+	#define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9200  (0x3UL &lt;&lt; 12)</div><div class='add'>+	#define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9300  (0x4UL &lt;&lt; 12)</div><div class='add'>+	#define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG1  (0x5UL &lt;&lt; 12)</div><div class='add'>+	#define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG2  (0x6UL &lt;&lt; 12)</div><div class='add'>+	#define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3  (0x7UL &lt;&lt; 12)</div><div class='add'>+	#define CMDQ_MODIFY_GID_VLAN_TPID_LAST      CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3</div><div class='add'>+	#define CMDQ_MODIFY_GID_VLAN_VLAN_EN       0x8000UL</div><div class='add'>+	__le16	ipid;</div><div class='add'>+	__le16	gid_index;</div><div class='add'>+	__le16	stats_ctx;</div><div class='add'>+	#define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_MASK   0x7fffUL</div><div class='add'>+	#define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_SFT    0</div><div class='add'>+	#define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_VALID     0x8000UL</div><div class='add'>+	__le16	unused_0;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_modify_gid_resp (size:128b/16B) */</div><div class='ctx'> struct creq_modify_gid_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_MODIFY_GID_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_MODIFY_GID_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_MODIFY_GID_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_MODIFY_GID_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_MODIFY_GID_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_MODIFY_GID_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_MODIFY_GID_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_MODIFY_GID_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_MODIFY_GID_RESP_EVENT_ADD_GID		   0x11UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Query GID command response (16 bytes) */</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_MODIFY_GID_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_MODIFY_GID_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_MODIFY_GID_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_MODIFY_GID_RESP_TYPE_LAST     CREQ_MODIFY_GID_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_MODIFY_GID_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_MODIFY_GID_RESP_EVENT_ADD_GID 0x11UL</div><div class='add'>+	#define CREQ_MODIFY_GID_RESP_EVENT_LAST   CREQ_MODIFY_GID_RESP_EVENT_ADD_GID</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_query_gid (size:192b/24B) */</div><div class='add'>+struct cmdq_query_gid {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_QUERY_GID_OPCODE_QUERY_GID 0x18UL</div><div class='add'>+	#define CMDQ_QUERY_GID_OPCODE_LAST     CMDQ_QUERY_GID_OPCODE_QUERY_GID</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le16	gid_index;</div><div class='add'>+	u8	unused16[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_gid_resp (size:128b/16B) */</div><div class='ctx'> struct creq_query_gid_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 size;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_EVENT_QUERY_GID		   0x18UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Query GID command response side buffer structure (40 bytes) */</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_TYPE_LAST     CREQ_QUERY_GID_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	size;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_EVENT_QUERY_GID 0x18UL</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_EVENT_LAST     CREQ_QUERY_GID_RESP_EVENT_QUERY_GID</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_gid_resp_sb (size:320b/40B) */</div><div class='ctx'> struct creq_query_gid_resp_sb {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_SB_OPCODE_QUERY_GID	   0x18UL</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le32 gid[4];</div><div class='del'>-	__le16 src_mac[3];</div><div class='del'>-	__le16 vlan;</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_MASK	    0xfffUL</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_SFT	    0</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_MASK		    0x7000UL</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_SFT		    12</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_88A8	   (0x0UL &lt;&lt; 12)</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_8100	   (0x1UL &lt;&lt; 12)</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9100	   (0x2UL &lt;&lt; 12)</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9200	   (0x3UL &lt;&lt; 12)</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9300	   (0x4UL &lt;&lt; 12)</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG1	   (0x5UL &lt;&lt; 12)</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG2	   (0x6UL &lt;&lt; 12)</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3	   (0x7UL &lt;&lt; 12)</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_LAST	\</div><div class='del'>-				CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3</div><div class='del'>-	#define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN		    0x8000UL</div><div class='del'>-	__le16 ipid;</div><div class='del'>-	__le16 gid_index;</div><div class='del'>-	__le32 unused_0;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Create QP1 command response (16 bytes) */</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_OPCODE_QUERY_GID 0x18UL</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_OPCODE_LAST     CREQ_QUERY_GID_RESP_SB_OPCODE_QUERY_GID</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le32	gid[4];</div><div class='add'>+	__le16	src_mac[3];</div><div class='add'>+	__le16	vlan;</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN_TPID_VLAN_ID_MASK          0xffffUL</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN_TPID_VLAN_ID_SFT           0</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_MASK                       0xfffUL</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_SFT                        0</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_MASK                          0x7000UL</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_SFT                           12</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_88A8                       (0x0UL &lt;&lt; 12)</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_8100                       (0x1UL &lt;&lt; 12)</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9100                       (0x2UL &lt;&lt; 12)</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9200                       (0x3UL &lt;&lt; 12)</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9300                       (0x4UL &lt;&lt; 12)</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG1                       (0x5UL &lt;&lt; 12)</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG2                       (0x6UL &lt;&lt; 12)</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3                       (0x7UL &lt;&lt; 12)</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_LAST CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3</div><div class='add'>+	#define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN                            0x8000UL</div><div class='add'>+	__le16	ipid;</div><div class='add'>+	__le16	gid_index;</div><div class='add'>+	__le32	unused_0;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_create_qp1 (size:640b/80B) */</div><div class='add'>+struct cmdq_create_qp1 {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_CREATE_QP1_OPCODE_CREATE_QP1 0x13UL</div><div class='add'>+	#define CMDQ_CREATE_QP1_OPCODE_LAST      CMDQ_CREATE_QP1_OPCODE_CREATE_QP1</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le64	qp_handle;</div><div class='add'>+	__le32	qp_flags;</div><div class='add'>+	#define CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED             0x1UL</div><div class='add'>+	#define CMDQ_CREATE_QP1_QP_FLAGS_FORCE_COMPLETION     0x2UL</div><div class='add'>+	#define CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL</div><div class='add'>+	#define CMDQ_CREATE_QP1_QP_FLAGS_LAST     CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CMDQ_CREATE_QP1_TYPE_GSI 0x1UL</div><div class='add'>+	#define CMDQ_CREATE_QP1_TYPE_LAST CMDQ_CREATE_QP1_TYPE_GSI</div><div class='add'>+	u8	sq_pg_size_sq_lvl;</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_LVL_MASK      0xfUL</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_LVL_SFT       0</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_LVL_LVL_0       0x0UL</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_LVL_LVL_1       0x1UL</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_LVL_LVL_2       0x2UL</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_LVL_LAST       CMDQ_CREATE_QP1_SQ_LVL_LVL_2</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_MASK  0xf0UL</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT   4</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K   (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K   (0x1UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K  (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M   (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M   (0x4UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G   (0x5UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_PG_SIZE_LAST   CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G</div><div class='add'>+	u8	rq_pg_size_rq_lvl;</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_LVL_MASK      0xfUL</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_LVL_SFT       0</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_LVL_LVL_0       0x0UL</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_LVL_LVL_1       0x1UL</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_LVL_LVL_2       0x2UL</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_LVL_LAST       CMDQ_CREATE_QP1_RQ_LVL_LVL_2</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_MASK  0xf0UL</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT   4</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K   (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K   (0x1UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K  (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M   (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M   (0x4UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G   (0x5UL &lt;&lt; 4)</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_PG_SIZE_LAST   CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G</div><div class='add'>+	u8	unused_0;</div><div class='add'>+	__le32	dpi;</div><div class='add'>+	__le32	sq_size;</div><div class='add'>+	__le32	rq_size;</div><div class='add'>+	__le16	sq_fwo_sq_sge;</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_SGE_MASK 0xfUL</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_SGE_SFT 0</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_FWO_MASK 0xfff0UL</div><div class='add'>+	#define CMDQ_CREATE_QP1_SQ_FWO_SFT 4</div><div class='add'>+	__le16	rq_fwo_rq_sge;</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_SGE_MASK 0xfUL</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_SGE_SFT 0</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_FWO_MASK 0xfff0UL</div><div class='add'>+	#define CMDQ_CREATE_QP1_RQ_FWO_SFT 4</div><div class='add'>+	__le32	scq_cid;</div><div class='add'>+	__le32	rcq_cid;</div><div class='add'>+	__le32	srq_cid;</div><div class='add'>+	__le32	pd_id;</div><div class='add'>+	__le64	sq_pbl;</div><div class='add'>+	__le64	rq_pbl;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_create_qp1_resp (size:128b/16B) */</div><div class='ctx'> struct creq_create_qp1_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_CREATE_QP1_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_CREATE_QP1_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_CREATE_QP1_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_CREATE_QP1_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_CREATE_QP1_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_CREATE_QP1_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_CREATE_QP1_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_CREATE_QP1_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_CREATE_QP1_RESP_EVENT_CREATE_QP1		   0x13UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Destroy QP1 command response (16 bytes) */</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_CREATE_QP1_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_CREATE_QP1_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_CREATE_QP1_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_CREATE_QP1_RESP_TYPE_LAST     CREQ_CREATE_QP1_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_CREATE_QP1_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_CREATE_QP1_RESP_EVENT_CREATE_QP1 0x13UL</div><div class='add'>+	#define CREQ_CREATE_QP1_RESP_EVENT_LAST      CREQ_CREATE_QP1_RESP_EVENT_CREATE_QP1</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_destroy_qp1 (size:192b/24B) */</div><div class='add'>+struct cmdq_destroy_qp1 {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_DESTROY_QP1_OPCODE_DESTROY_QP1 0x14UL</div><div class='add'>+	#define CMDQ_DESTROY_QP1_OPCODE_LAST       CMDQ_DESTROY_QP1_OPCODE_DESTROY_QP1</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	qp1_cid;</div><div class='add'>+	__le32	unused_0;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_destroy_qp1_resp (size:128b/16B) */</div><div class='ctx'> struct creq_destroy_qp1_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_DESTROY_QP1_RESP_TYPE_MASK		    0x3fUL</div><div class='del'>-	#define CREQ_DESTROY_QP1_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_DESTROY_QP1_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_DESTROY_QP1_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_DESTROY_QP1_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_DESTROY_QP1_RESP_V			    0x1UL</div><div class='del'>-	#define CREQ_DESTROY_QP1_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_DESTROY_QP1_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_DESTROY_QP1_RESP_EVENT_DESTROY_QP1	   0x14UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Create AH command response (16 bytes) */</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_DESTROY_QP1_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_DESTROY_QP1_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_DESTROY_QP1_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_DESTROY_QP1_RESP_TYPE_LAST     CREQ_DESTROY_QP1_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_DESTROY_QP1_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_DESTROY_QP1_RESP_EVENT_DESTROY_QP1 0x14UL</div><div class='add'>+	#define CREQ_DESTROY_QP1_RESP_EVENT_LAST       CREQ_DESTROY_QP1_RESP_EVENT_DESTROY_QP1</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_create_ah (size:512b/64B) */</div><div class='add'>+struct cmdq_create_ah {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_CREATE_AH_OPCODE_CREATE_AH 0x15UL</div><div class='add'>+	#define CMDQ_CREATE_AH_OPCODE_LAST     CMDQ_CREATE_AH_OPCODE_CREATE_AH</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le64	ah_handle;</div><div class='add'>+	__le32	dgid[4];</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CMDQ_CREATE_AH_TYPE_V1     0x0UL</div><div class='add'>+	#define CMDQ_CREATE_AH_TYPE_V2IPV4 0x2UL</div><div class='add'>+	#define CMDQ_CREATE_AH_TYPE_V2IPV6 0x3UL</div><div class='add'>+	#define CMDQ_CREATE_AH_TYPE_LAST  CMDQ_CREATE_AH_TYPE_V2IPV6</div><div class='add'>+	u8	hop_limit;</div><div class='add'>+	__le16	sgid_index;</div><div class='add'>+	__le32	dest_vlan_id_flow_label;</div><div class='add'>+	#define CMDQ_CREATE_AH_FLOW_LABEL_MASK  0xfffffUL</div><div class='add'>+	#define CMDQ_CREATE_AH_FLOW_LABEL_SFT   0</div><div class='add'>+	#define CMDQ_CREATE_AH_DEST_VLAN_ID_MASK 0xfff00000UL</div><div class='add'>+	#define CMDQ_CREATE_AH_DEST_VLAN_ID_SFT 20</div><div class='add'>+	__le32	pd_id;</div><div class='add'>+	__le32	unused_0;</div><div class='add'>+	__le16	dest_mac[3];</div><div class='add'>+	u8	traffic_class;</div><div class='add'>+	u8	enable_cc;</div><div class='add'>+	#define CMDQ_CREATE_AH_ENABLE_CC     0x1UL</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_create_ah_resp (size:128b/16B) */</div><div class='ctx'> struct creq_create_ah_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_CREATE_AH_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_CREATE_AH_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_CREATE_AH_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_CREATE_AH_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_CREATE_AH_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_CREATE_AH_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_CREATE_AH_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_CREATE_AH_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_CREATE_AH_RESP_EVENT_CREATE_AH		   0x15UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Destroy AH command response (16 bytes) */</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_CREATE_AH_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_CREATE_AH_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_CREATE_AH_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_CREATE_AH_RESP_TYPE_LAST     CREQ_CREATE_AH_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_CREATE_AH_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_CREATE_AH_RESP_EVENT_CREATE_AH 0x15UL</div><div class='add'>+	#define CREQ_CREATE_AH_RESP_EVENT_LAST     CREQ_CREATE_AH_RESP_EVENT_CREATE_AH</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_destroy_ah (size:192b/24B) */</div><div class='add'>+struct cmdq_destroy_ah {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_DESTROY_AH_OPCODE_DESTROY_AH 0x16UL</div><div class='add'>+	#define CMDQ_DESTROY_AH_OPCODE_LAST      CMDQ_DESTROY_AH_OPCODE_DESTROY_AH</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	ah_cid;</div><div class='add'>+	__le32	unused_0;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_destroy_ah_resp (size:128b/16B) */</div><div class='ctx'> struct creq_destroy_ah_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_DESTROY_AH_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_DESTROY_AH_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_DESTROY_AH_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_DESTROY_AH_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_DESTROY_AH_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_DESTROY_AH_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_DESTROY_AH_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_DESTROY_AH_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_DESTROY_AH_RESP_EVENT_DESTROY_AH		   0x16UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Initialize Firmware command response (16 bytes) */</div><div class='del'>-struct creq_initialize_fw_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_INITIALIZE_FW_RESP_TYPE_MASK		    0x3fUL</div><div class='del'>-	#define CREQ_INITIALIZE_FW_RESP_TYPE_SFT		    0</div><div class='del'>-	#define CREQ_INITIALIZE_FW_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_INITIALIZE_FW_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_INITIALIZE_FW_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 reserved32;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_INITIALIZE_FW_RESP_V			    0x1UL</div><div class='del'>-	#define CREQ_INITIALIZE_FW_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_INITIALIZE_FW_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_INITIALIZE_FW_RESP_EVENT_INITIALIZE_FW       0x80UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* De-initialize Firmware command response (16 bytes) */</div><div class='del'>-struct creq_deinitialize_fw_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_DEINITIALIZE_FW_RESP_TYPE_MASK		    0x3fUL</div><div class='del'>-	#define CREQ_DEINITIALIZE_FW_RESP_TYPE_SFT		    0</div><div class='del'>-	#define CREQ_DEINITIALIZE_FW_RESP_TYPE_QP_EVENT	   0x38UL</div><div class='del'>-	#define CREQ_DEINITIALIZE_FW_RESP_RESERVED2_MASK	    0xc0UL</div><div class='del'>-	#define CREQ_DEINITIALIZE_FW_RESP_RESERVED2_SFT	    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 reserved32;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_DEINITIALIZE_FW_RESP_V			    0x1UL</div><div class='del'>-	#define CREQ_DEINITIALIZE_FW_RESP_RESERVED7_MASK	    0xfeUL</div><div class='del'>-	#define CREQ_DEINITIALIZE_FW_RESP_RESERVED7_SFT	    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_DEINITIALIZE_FW_RESP_EVENT_DEINITIALIZE_FW   0x81UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Stop function command response (16 bytes) */</div><div class='del'>-struct creq_stop_func_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_STOP_FUNC_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_STOP_FUNC_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_STOP_FUNC_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_STOP_FUNC_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_STOP_FUNC_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 reserved32;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_STOP_FUNC_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_STOP_FUNC_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_STOP_FUNC_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_STOP_FUNC_RESP_EVENT_STOP_FUNC		   0x82UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Query function command response (16 bytes) */</div><div class='del'>-struct creq_query_func_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_QUERY_FUNC_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_QUERY_FUNC_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_QUERY_FUNC_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_QUERY_FUNC_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_QUERY_FUNC_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 size;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_QUERY_FUNC_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_QUERY_FUNC_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_QUERY_FUNC_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_QUERY_FUNC_RESP_EVENT_QUERY_FUNC		   0x83UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Query function command response side buffer structure (88 bytes) */</div><div class='del'>-struct creq_query_func_resp_sb {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CREQ_QUERY_FUNC_RESP_SB_OPCODE_QUERY_FUNC	   0x83UL</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	__le64 max_mr_size;</div><div class='del'>-	__le32 max_qp;</div><div class='del'>-	__le16 max_qp_wr;</div><div class='del'>-	__le16 dev_cap_flags;</div><div class='del'>-	#define CREQ_QUERY_FUNC_RESP_SB_DEV_CAP_FLAGS_RESIZE_QP   0x1UL</div><div class='del'>-	__le32 max_cq;</div><div class='del'>-	__le32 max_cqe;</div><div class='del'>-	__le32 max_pd;</div><div class='del'>-	u8 max_sge;</div><div class='del'>-	u8 max_srq_sge;</div><div class='del'>-	u8 max_qp_rd_atom;</div><div class='del'>-	u8 max_qp_init_rd_atom;</div><div class='del'>-	__le32 max_mr;</div><div class='del'>-	__le32 max_mw;</div><div class='del'>-	__le32 max_raw_eth_qp;</div><div class='del'>-	__le32 max_ah;</div><div class='del'>-	__le32 max_fmr;</div><div class='del'>-	__le32 max_srq_wr;</div><div class='del'>-	__le32 max_pkeys;</div><div class='del'>-	__le32 max_inline_data;</div><div class='del'>-	u8 max_map_per_fmr;</div><div class='del'>-	u8 l2_db_space_size;</div><div class='del'>-	__le16 max_srq;</div><div class='del'>-	__le32 max_gid;</div><div class='del'>-	__le32 tqm_alloc_reqs[12];</div><div class='del'>-	__le32 max_dpi;</div><div class='del'>-	__le32 reserved_32;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Set resources command response (16 bytes) */</div><div class='del'>-struct creq_set_func_resources_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_MASK		    0x3fUL</div><div class='del'>-	#define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_SFT		    0</div><div class='del'>-	#define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_QP_EVENT	   0x38UL</div><div class='del'>-	#define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED2_MASK	    0xc0UL</div><div class='del'>-	#define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED2_SFT	    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 reserved32;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_SET_FUNC_RESOURCES_RESP_V			    0x1UL</div><div class='del'>-	#define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED7_MASK	    0xfeUL</div><div class='del'>-	#define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED7_SFT	    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_SET_FUNC_RESOURCES_RESP_EVENT_SET_FUNC_RESOURCES 0x84UL</div><div class='del'>-	__le16 reserved48[3];</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_DESTROY_AH_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_DESTROY_AH_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_DESTROY_AH_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_DESTROY_AH_RESP_TYPE_LAST     CREQ_DESTROY_AH_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_DESTROY_AH_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_DESTROY_AH_RESP_EVENT_DESTROY_AH 0x16UL</div><div class='add'>+	#define CREQ_DESTROY_AH_RESP_EVENT_LAST      CREQ_DESTROY_AH_RESP_EVENT_DESTROY_AH</div><div class='add'>+	u8	reserved48[6];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* Map TC to COS response (16 bytes) */</div><div class='del'>-struct creq_map_tc_to_cos_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_MAP_TC_TO_COS_RESP_TYPE_MASK		    0x3fUL</div><div class='del'>-	#define CREQ_MAP_TC_TO_COS_RESP_TYPE_SFT		    0</div><div class='del'>-	#define CREQ_MAP_TC_TO_COS_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_MAP_TC_TO_COS_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_MAP_TC_TO_COS_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 reserved32;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_MAP_TC_TO_COS_RESP_V			    0x1UL</div><div class='del'>-	#define CREQ_MAP_TC_TO_COS_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_MAP_TC_TO_COS_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_MAP_TC_TO_COS_RESP_EVENT_MAP_TC_TO_COS       0x8aUL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Query version response (16 bytes) */</div><div class='del'>-struct creq_query_version_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_QUERY_VERSION_RESP_TYPE_MASK		    0x3fUL</div><div class='del'>-	#define CREQ_QUERY_VERSION_RESP_TYPE_SFT		    0</div><div class='del'>-	#define CREQ_QUERY_VERSION_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_QUERY_VERSION_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_QUERY_VERSION_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	u8 fw_maj;</div><div class='del'>-	u8 fw_minor;</div><div class='del'>-	u8 fw_bld;</div><div class='del'>-	u8 fw_rsvd;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_QUERY_VERSION_RESP_V			    0x1UL</div><div class='del'>-	#define CREQ_QUERY_VERSION_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_QUERY_VERSION_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_QUERY_VERSION_RESP_EVENT_QUERY_VERSION       0x8bUL</div><div class='del'>-	__le16 reserved16;</div><div class='del'>-	u8 intf_maj;</div><div class='del'>-	u8 intf_minor;</div><div class='del'>-	u8 intf_bld;</div><div class='del'>-	u8 intf_rsvd;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Modify congestion control command response (16 bytes) */</div><div class='del'>-struct creq_modify_cc_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_MODIFY_CC_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_MODIFY_CC_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_MODIFY_CC_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_MODIFY_CC_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_MODIFY_CC_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 reserved32;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_MODIFY_CC_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_MODIFY_CC_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_MODIFY_CC_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_MODIFY_CC_RESP_EVENT_MODIFY_CC		   0x8cUL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Query congestion control command response (16 bytes) */</div><div class='del'>-struct creq_query_cc_resp {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_TYPE_MASK			    0x3fUL</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_TYPE_SFT			    0</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_TYPE_QP_EVENT		   0x38UL</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_RESERVED2_MASK		    0xc0UL</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_RESERVED2_SFT		    6</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le32 size;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_V				    0x1UL</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_RESERVED7_MASK		    0xfeUL</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_RESERVED7_SFT		    1</div><div class='del'>-	u8 event;</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_EVENT_QUERY_CC		   0x8dUL</div><div class='del'>-	__le16 reserved48[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Query congestion control command response side buffer structure (32 bytes) */</div><div class='del'>-struct creq_query_cc_resp_sb {</div><div class='del'>-	u8 opcode;</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_SB_OPCODE_QUERY_CC		   0x8dUL</div><div class='del'>-	u8 status;</div><div class='del'>-	__le16 cookie;</div><div class='del'>-	__le16 flags;</div><div class='del'>-	u8 resp_size;</div><div class='del'>-	u8 reserved8;</div><div class='del'>-	u8 enable_cc;</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_SB_ENABLE_CC		    0x1UL</div><div class='del'>-	u8 g;</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_SB_G_MASK			    0x7UL</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_SB_G_SFT			    0</div><div class='del'>-	u8 num_phases_per_state;</div><div class='del'>-	__le16 init_cr;</div><div class='del'>-	u8 unused_2;</div><div class='del'>-	__le16 unused_3;</div><div class='del'>-	u8 unused_4;</div><div class='del'>-	__le16 init_tr;</div><div class='del'>-	u8 tos_dscp_tos_ecn;</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_SB_TOS_ECN_MASK		    0x3UL</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_SB_TOS_ECN_SFT		    0</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_SB_TOS_DSCP_MASK		    0xfcUL</div><div class='del'>-	#define CREQ_QUERY_CC_RESP_SB_TOS_DSCP_SFT		    2</div><div class='del'>-	__le64 reserved64;</div><div class='del'>-	__le64 reserved64_1;</div><div class='add'>+/* cmdq_query_roce_stats (size:192b/24B) */</div><div class='add'>+struct cmdq_query_roce_stats {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_OPCODE_QUERY_ROCE_STATS 0x8eUL</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_OPCODE_LAST    CMDQ_QUERY_ROCE_STATS_OPCODE_QUERY_ROCE_STATS</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_FLAGS_COLLECTION_ID     0x1UL</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_FLAGS_FUNCTION_ID       0x2UL</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	collection_id;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	function_id;</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_PF_NUM_MASK  0xffUL</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_PF_NUM_SFT   0</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_VF_NUM_MASK  0xffff00UL</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_VF_NUM_SFT   8</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_VF_VALID     0x1000000UL</div><div class='add'>+	__le32	reserved32;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /* creq_query_roce_stats_resp (size:128b/16B) */</div><div class='hunk'>@@ -2907,8 +1955,7 @@ struct creq_query_roce_stats_resp {</div><div class='ctx'> 	#define CREQ_QUERY_ROCE_STATS_RESP_TYPE_MASK    0x3fUL</div><div class='ctx'> 	#define CREQ_QUERY_ROCE_STATS_RESP_TYPE_SFT     0</div><div class='ctx'> 	#define CREQ_QUERY_ROCE_STATS_RESP_TYPE_QP_EVENT  0x38UL</div><div class='del'>-	#define CREQ_QUERY_ROCE_STATS_RESP_TYPE_LAST	\</div><div class='del'>-				CREQ_QUERY_ROCE_STATS_RESP_TYPE_QP_EVENT</div><div class='add'>+	#define CREQ_QUERY_ROCE_STATS_RESP_TYPE_LAST     CREQ_QUERY_ROCE_STATS_RESP_TYPE_QP_EVENT</div><div class='ctx'> 	u8	status;</div><div class='ctx'> 	__le16	cookie;</div><div class='ctx'> 	__le32	size;</div><div class='hunk'>@@ -2916,17 +1963,17 @@ struct creq_query_roce_stats_resp {</div><div class='ctx'> 	#define CREQ_QUERY_ROCE_STATS_RESP_V     0x1UL</div><div class='ctx'> 	u8	event;</div><div class='ctx'> 	#define CREQ_QUERY_ROCE_STATS_RESP_EVENT_QUERY_ROCE_STATS 0x8eUL</div><div class='del'>-	#define CREQ_QUERY_ROCE_STATS_RESP_EVENT_LAST	\</div><div class='del'>-			CREQ_QUERY_ROCE_STATS_RESP_EVENT_QUERY_ROCE_STATS</div><div class='add'>+	#define CREQ_QUERY_ROCE_STATS_RESP_EVENT_LAST \</div><div class='add'>+		CREQ_QUERY_ROCE_STATS_RESP_EVENT_QUERY_ROCE_STATS</div><div class='ctx'> 	u8	reserved48[6];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* creq_query_roce_stats_resp_sb (size:2624b/328B) */</div><div class='add'>+/* creq_query_roce_stats_resp_sb (size:2944b/368B) */</div><div class='ctx'> struct creq_query_roce_stats_resp_sb {</div><div class='ctx'> 	u8	opcode;</div><div class='ctx'> 	#define CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_QUERY_ROCE_STATS 0x8eUL</div><div class='ctx'> 	#define CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_LAST \</div><div class='del'>-			CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_QUERY_ROCE_STATS</div><div class='add'>+		CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_QUERY_ROCE_STATS</div><div class='ctx'> 	u8	status;</div><div class='ctx'> 	__le16	cookie;</div><div class='ctx'> 	__le16	flags;</div><div class='hunk'>@@ -2973,46 +2020,2710 @@ struct creq_query_roce_stats_resp_sb {</div><div class='ctx'> 	__le64	res_srq_load_err;</div><div class='ctx'> 	__le64	res_tx_pci_err;</div><div class='ctx'> 	__le64	res_rx_pci_err;</div><div class='del'>-	__le64  res_oos_drop_count;</div><div class='del'>-	__le64  active_qp_count_p0;</div><div class='del'>-	__le64  active_qp_count_p1;</div><div class='del'>-	__le64  active_qp_count_p2;</div><div class='del'>-	__le64  active_qp_count_p3;</div><div class='add'>+	__le64	res_oos_drop_count;</div><div class='add'>+	__le64	active_qp_count_p0;</div><div class='add'>+	__le64	active_qp_count_p1;</div><div class='add'>+	__le64	active_qp_count_p2;</div><div class='add'>+	__le64	active_qp_count_p3;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_query_roce_stats_ext (size:192b/24B) */</div><div class='add'>+struct cmdq_query_roce_stats_ext {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_QUERY_ROCE_STATS 0x92UL</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_LAST \</div><div class='add'>+			CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_QUERY_ROCE_STATS</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_EXT_FLAGS_COLLECTION_ID     0x1UL</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_EXT_FLAGS_FUNCTION_ID       0x2UL</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	collection_id;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	function_id;</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_EXT_PF_NUM_MASK  0xffUL</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_EXT_PF_NUM_SFT   0</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_EXT_VF_NUM_MASK  0xffff00UL</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_EXT_VF_NUM_SFT   8</div><div class='add'>+	#define CMDQ_QUERY_ROCE_STATS_EXT_VF_VALID     0x1000000UL</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_roce_stats_ext_resp (size:128b/16B) */</div><div class='add'>+struct creq_query_roce_stats_ext_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_QUERY_ROCE_STATS_EXT_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_QUERY_ROCE_STATS_EXT_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_QUERY_ROCE_STATS_EXT_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_STATS_EXT_RESP_TYPE_LAST \</div><div class='add'>+		CREQ_QUERY_ROCE_STATS_EXT_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	size;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_QUERY_ROCE_STATS_EXT_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_QUERY_ROCE_STATS_EXT_RESP_EVENT_QUERY_ROCE_STATS_EXT 0x92UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_STATS_EXT_RESP_EVENT_LAST \</div><div class='add'>+		CREQ_QUERY_ROCE_STATS_EXT_RESP_EVENT_QUERY_ROCE_STATS_EXT</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_roce_stats_ext_resp_sb (size:1856b/232B) */</div><div class='add'>+struct creq_query_roce_stats_ext_resp_sb {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CREQ_QUERY_ROCE_STATS_EXT_RESP_SB_OPCODE_QUERY_ROCE_STATS_EXT 0x92UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_STATS_EXT_RESP_SB_OPCODE_LAST \</div><div class='add'>+		CREQ_QUERY_ROCE_STATS_EXT_RESP_SB_OPCODE_QUERY_ROCE_STATS_EXT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	rsvd;</div><div class='add'>+	__le64	tx_atomic_req_pkts;</div><div class='add'>+	__le64	tx_read_req_pkts;</div><div class='add'>+	__le64	tx_read_res_pkts;</div><div class='add'>+	__le64	tx_write_req_pkts;</div><div class='add'>+	__le64	tx_send_req_pkts;</div><div class='add'>+	__le64	tx_roce_pkts;</div><div class='add'>+	__le64	tx_roce_bytes;</div><div class='add'>+	__le64	rx_atomic_req_pkts;</div><div class='add'>+	__le64	rx_read_req_pkts;</div><div class='add'>+	__le64	rx_read_res_pkts;</div><div class='add'>+	__le64	rx_write_req_pkts;</div><div class='add'>+	__le64	rx_send_req_pkts;</div><div class='add'>+	__le64	rx_roce_pkts;</div><div class='add'>+	__le64	rx_roce_bytes;</div><div class='add'>+	__le64	rx_roce_good_pkts;</div><div class='add'>+	__le64	rx_roce_good_bytes;</div><div class='add'>+	__le64	rx_out_of_buffer_pkts;</div><div class='add'>+	__le64	rx_out_of_sequence_pkts;</div><div class='add'>+	__le64	tx_cnp_pkts;</div><div class='add'>+	__le64	rx_cnp_pkts;</div><div class='add'>+	__le64	rx_ecn_marked_pkts;</div><div class='add'>+	__le64	tx_cnp_bytes;</div><div class='add'>+	__le64	rx_cnp_bytes;</div><div class='add'>+	__le64	seq_err_naks_rcvd;</div><div class='add'>+	__le64	rnr_naks_rcvd;</div><div class='add'>+	__le64	missing_resp;</div><div class='add'>+	__le64	to_retransmit;</div><div class='add'>+	__le64	dup_req;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_roce_mirror_cfg (size:192b/24B) */</div><div class='add'>+struct cmdq_roce_mirror_cfg {</div><div class='add'>+	u8      opcode;</div><div class='add'>+	#define CMDQ_ROCE_MIRROR_CFG_OPCODE_ROCE_MIRROR_CFG	0x99UL</div><div class='add'>+	#define CMDQ_ROCE_MIRROR_CFG_OPCODE_LAST		\</div><div class='add'>+				CMDQ_ROCE_MIRROR_CFG_OPCODE_ROCE_MIRROR_CFG</div><div class='add'>+	u8      cmd_size;</div><div class='add'>+	__le16  flags;</div><div class='add'>+	__le16  cookie;</div><div class='add'>+	u8      resp_size;</div><div class='add'>+	u8      reserved8;</div><div class='add'>+	__le64  resp_addr;</div><div class='add'>+	u8      mirror_flags;</div><div class='add'>+	#define CMDQ_ROCE_MIRROR_CFG_MIRROR_ENABLE		0x1UL</div><div class='add'>+	u8      rsvd[7];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_roce_mirror_cfg_resp (size:128b/16B) */</div><div class='add'>+struct creq_roce_mirror_cfg_resp {</div><div class='add'>+	u8      type;</div><div class='add'>+	#define CREQ_ROCE_MIRROR_CFG_RESP_TYPE_MASK	0x3fUL</div><div class='add'>+	#define CREQ_ROCE_MIRROR_CFG_RESP_TYPE_SFT	0</div><div class='add'>+	#define CREQ_ROCE_MIRROR_CFG_RESP_TYPE_QP_EVENT	0x38UL</div><div class='add'>+	#define CREQ_ROCE_MIRROR_CFG_RESP_TYPE_LAST	\</div><div class='add'>+			CREQ_ROCE_MIRROR_CFG_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8      status;</div><div class='add'>+	__le16  cookie;</div><div class='add'>+	__le32  reserved32;</div><div class='add'>+	u8      v;</div><div class='add'>+	#define CREQ_ROCE_MIRROR_CFG_RESP_V		0x1UL</div><div class='add'>+	u8      event;</div><div class='add'>+	#define CREQ_ROCE_MIRROR_CFG_RESP_EVENT_ROCE_MIRROR_CFG	0x99UL</div><div class='add'>+	#define CREQ_ROCE_MIRROR_CFG_RESP_EVENT_LAST	\</div><div class='add'>+			CREQ_ROCE_MIRROR_CFG_RESP_EVENT_ROCE_MIRROR_CFG</div><div class='add'>+	u8      reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_query_func (size:128b/16B) */</div><div class='add'>+struct cmdq_query_func {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_QUERY_FUNC_OPCODE_QUERY_FUNC 0x83UL</div><div class='add'>+	#define CMDQ_QUERY_FUNC_OPCODE_LAST      CMDQ_QUERY_FUNC_OPCODE_QUERY_FUNC</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_func_resp (size:128b/16B) */</div><div class='add'>+struct creq_query_func_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_TYPE_LAST     CREQ_QUERY_FUNC_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	size;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_EVENT_QUERY_FUNC 0x83UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_EVENT_LAST      CREQ_QUERY_FUNC_RESP_EVENT_QUERY_FUNC</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_func_resp_sb (size:1088b/136B) */</div><div class='add'>+struct creq_query_func_resp_sb {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_OPCODE_QUERY_FUNC 0x83UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_OPCODE_LAST      CREQ_QUERY_FUNC_RESP_SB_OPCODE_QUERY_FUNC</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	max_mr_size;</div><div class='add'>+	__le32	max_qp;</div><div class='add'>+	__le16	max_qp_wr;</div><div class='add'>+	__le16	dev_cap_flags;</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_RESIZE_QP                      0x1UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_MASK             0xeUL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_SFT              1</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_CC_GEN0            (0x0UL &lt;&lt; 1)</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_CC_GEN1            (0x1UL &lt;&lt; 1)</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_CC_GEN1_EXT        (0x2UL &lt;&lt; 1)</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_LAST \</div><div class='add'>+		CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_CC_GEN1_EXT</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_EXT_STATS                      0x10UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_MR_REGISTER_ALLOC              0x20UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_OPTIMIZED_TRANSMIT_ENABLED     0x40UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_CQE_V2                         0x80UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_PINGPONG_PUSH_MODE             0x100UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_HW_REQUESTER_RETX_ENABLED      0x200UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_HW_RESPONDER_RETX_ENABLED      0x400UL</div><div class='add'>+	__le32	max_cq;</div><div class='add'>+	__le32	max_cqe;</div><div class='add'>+	__le32	max_pd;</div><div class='add'>+	u8	max_sge;</div><div class='add'>+	u8	max_srq_sge;</div><div class='add'>+	u8	max_qp_rd_atom;</div><div class='add'>+	u8	max_qp_init_rd_atom;</div><div class='add'>+	__le32	max_mr;</div><div class='add'>+	__le32	max_mw;</div><div class='add'>+	__le32	max_raw_eth_qp;</div><div class='add'>+	__le32	max_ah;</div><div class='add'>+	__le32	max_fmr;</div><div class='add'>+	__le32	max_srq_wr;</div><div class='add'>+	__le32	max_pkeys;</div><div class='add'>+	__le32	max_inline_data;</div><div class='add'>+	u8	max_map_per_fmr;</div><div class='add'>+	u8	l2_db_space_size;</div><div class='add'>+	__le16	max_srq;</div><div class='add'>+	__le32	max_gid;</div><div class='add'>+	__le32	tqm_alloc_reqs[12];</div><div class='add'>+	__le32	max_dpi;</div><div class='add'>+	u8	max_sge_var_wqe;</div><div class='add'>+	u8	dev_cap_ext_flags;</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_ATOMIC_OPS_NOT_SUPPORTED         0x1UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_DRV_VERSION_RGTR_SUPPORTED       0x2UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_CREATE_QP_BATCH_SUPPORTED        0x4UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_DESTROY_QP_BATCH_SUPPORTED       0x8UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_ROCE_STATS_EXT_CTX_SUPPORTED     0x10UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_CREATE_SRQ_SGE_SUPPORTED         0x20UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_FIXED_SIZE_WQE_DISABLED          0x40UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_DCN_SUPPORTED                    0x80UL</div><div class='add'>+	__le16	max_inline_data_var_wqe;</div><div class='add'>+	__le32	start_qid;</div><div class='add'>+	u8	max_msn_table_size;</div><div class='add'>+	u8	reserved8_1;</div><div class='add'>+	__le16	dev_cap_ext_flags_2;</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_OPTIMIZE_MODIFY_QP_SUPPORTED             0x1UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_CHANGE_UDP_SRC_PORT_WQE_SUPPORTED        0x2UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_CQ_COALESCING_SUPPORTED                  0x4UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_MEMORY_REGION_RO_SUPPORTED               0x8UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_MASK          0x30UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_SFT           4</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_PSN_TABLE  (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_MSN_TABLE  (0x1UL &lt;&lt; 4)</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_IQM_MSN_TABLE   (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_LAST	\</div><div class='add'>+			CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_IQM_MSN_TABLE</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_MAX_SRQ_EXTENDED                         0x40UL</div><div class='add'>+	#define CREQ_QUERY_FUNC_RESP_SB_MIN_RNR_RTR_RTS_OPT_SUPPORTED            0x1000UL</div><div class='add'>+	__le16	max_xp_qp_size;</div><div class='add'>+	__le16	create_qp_batch_size;</div><div class='add'>+	__le16	destroy_qp_batch_size;</div><div class='add'>+	__le16  max_srq_ext;</div><div class='add'>+	__le64	reserved64;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_set_func_resources (size:448b/56B) */</div><div class='add'>+struct cmdq_set_func_resources {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_SET_FUNC_RESOURCES_OPCODE_SET_FUNC_RESOURCES 0x84UL</div><div class='add'>+	#define CMDQ_SET_FUNC_RESOURCES_OPCODE_LAST\</div><div class='add'>+			CMDQ_SET_FUNC_RESOURCES_OPCODE_SET_FUNC_RESOURCES</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	#define CMDQ_SET_FUNC_RESOURCES_FLAGS_MRAV_RESERVATION_SPLIT     0x1UL</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	number_of_qp;</div><div class='add'>+	__le32	number_of_mrw;</div><div class='add'>+	__le32	number_of_srq;</div><div class='add'>+	__le32	number_of_cq;</div><div class='add'>+	__le32	max_qp_per_vf;</div><div class='add'>+	__le32	max_mrw_per_vf;</div><div class='add'>+	__le32	max_srq_per_vf;</div><div class='add'>+	__le32	max_cq_per_vf;</div><div class='add'>+	__le32	max_gid_per_vf;</div><div class='add'>+	__le32	stat_ctx_id;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* QP error notification event (16 bytes) */</div><div class='add'>+/* creq_set_func_resources_resp (size:128b/16B) */</div><div class='add'>+struct creq_set_func_resources_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_LAST CREQ_SET_FUNC_RESOURCES_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_SET_FUNC_RESOURCES_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_SET_FUNC_RESOURCES_RESP_EVENT_SET_FUNC_RESOURCES 0x84UL</div><div class='add'>+	#define CREQ_SET_FUNC_RESOURCES_RESP_EVENT_LAST \</div><div class='add'>+		CREQ_SET_FUNC_RESOURCES_RESP_EVENT_SET_FUNC_RESOURCES</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_read_context (size:192b/24B) */</div><div class='add'>+struct cmdq_read_context {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_READ_CONTEXT_OPCODE_READ_CONTEXT 0x85UL</div><div class='add'>+	#define CMDQ_READ_CONTEXT_OPCODE_LAST        CMDQ_READ_CONTEXT_OPCODE_READ_CONTEXT</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CMDQ_READ_CONTEXT_TYPE_QPC 0x0UL</div><div class='add'>+	#define CMDQ_READ_CONTEXT_TYPE_CQ  0x1UL</div><div class='add'>+	#define CMDQ_READ_CONTEXT_TYPE_MRW 0x2UL</div><div class='add'>+	#define CMDQ_READ_CONTEXT_TYPE_SRQ 0x3UL</div><div class='add'>+	#define CMDQ_READ_CONTEXT_TYPE_LAST CMDQ_READ_CONTEXT_TYPE_SRQ</div><div class='add'>+	u8	unused_0[3];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_read_context (size:128b/16B) */</div><div class='add'>+struct creq_read_context {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_READ_CONTEXT_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_READ_CONTEXT_TYPE_SFT     0</div><div class='add'>+	#define CREQ_READ_CONTEXT_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_READ_CONTEXT_TYPE_LAST     CREQ_READ_CONTEXT_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_READ_CONTEXT_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_READ_CONTEXT_EVENT_READ_CONTEXT 0x85UL</div><div class='add'>+	#define CREQ_READ_CONTEXT_EVENT_LAST        CREQ_READ_CONTEXT_EVENT_READ_CONTEXT</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	reserved_32;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_map_tc_to_cos (size:192b/24B) */</div><div class='add'>+struct cmdq_map_tc_to_cos {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_MAP_TC_TO_COS_OPCODE_MAP_TC_TO_COS 0x8aUL</div><div class='add'>+	#define CMDQ_MAP_TC_TO_COS_OPCODE_LAST         CMDQ_MAP_TC_TO_COS_OPCODE_MAP_TC_TO_COS</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le16	cos0;</div><div class='add'>+	#define CMDQ_MAP_TC_TO_COS_COS0_NO_CHANGE 0xffffUL</div><div class='add'>+	#define CMDQ_MAP_TC_TO_COS_COS0_LAST     CMDQ_MAP_TC_TO_COS_COS0_NO_CHANGE</div><div class='add'>+	__le16	cos1;</div><div class='add'>+	#define CMDQ_MAP_TC_TO_COS_COS1_DISABLE   0x8000UL</div><div class='add'>+	#define CMDQ_MAP_TC_TO_COS_COS1_NO_CHANGE 0xffffUL</div><div class='add'>+	#define CMDQ_MAP_TC_TO_COS_COS1_LAST     CMDQ_MAP_TC_TO_COS_COS1_NO_CHANGE</div><div class='add'>+	__le32	unused_0;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_map_tc_to_cos_resp (size:128b/16B) */</div><div class='add'>+struct creq_map_tc_to_cos_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_MAP_TC_TO_COS_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_MAP_TC_TO_COS_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_MAP_TC_TO_COS_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_MAP_TC_TO_COS_RESP_TYPE_LAST     CREQ_MAP_TC_TO_COS_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_MAP_TC_TO_COS_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_MAP_TC_TO_COS_RESP_EVENT_MAP_TC_TO_COS 0x8aUL</div><div class='add'>+	#define CREQ_MAP_TC_TO_COS_RESP_EVENT_LAST CREQ_MAP_TC_TO_COS_RESP_EVENT_MAP_TC_TO_COS</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_query_roce_cc (size:128b/16B) */</div><div class='add'>+struct cmdq_query_roce_cc {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_QUERY_ROCE_CC_OPCODE_QUERY_ROCE_CC 0x8dUL</div><div class='add'>+	#define CMDQ_QUERY_ROCE_CC_OPCODE_LAST CMDQ_QUERY_ROCE_CC_OPCODE_QUERY_ROCE_CC</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_roce_cc_resp (size:128b/16B) */</div><div class='add'>+struct creq_query_roce_cc_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_TYPE_LAST     CREQ_QUERY_ROCE_CC_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	size;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_EVENT_QUERY_ROCE_CC 0x8dUL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_EVENT_LAST  CREQ_QUERY_ROCE_CC_RESP_EVENT_QUERY_ROCE_CC</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_roce_cc_resp_sb (size:256b/32B) */</div><div class='add'>+struct creq_query_roce_cc_resp_sb {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_OPCODE_QUERY_ROCE_CC 0x8dUL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_OPCODE_LAST \</div><div class='add'>+		CREQ_QUERY_ROCE_CC_RESP_SB_OPCODE_QUERY_ROCE_CC</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	u8	enable_cc;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_ENABLE_CC     0x1UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_UNUSED7_MASK  0xfeUL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_UNUSED7_SFT   1</div><div class='add'>+	u8	tos_dscp_tos_ecn;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_MASK 0x3UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_SFT  0</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_MASK 0xfcUL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_SFT 2</div><div class='add'>+	u8	g;</div><div class='add'>+	u8	num_phases_per_state;</div><div class='add'>+	__le16	init_cr;</div><div class='add'>+	__le16	init_tr;</div><div class='add'>+	u8	alt_vlan_pcp;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_ALT_VLAN_PCP_MASK 0x7UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_ALT_VLAN_PCP_SFT 0</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD1_MASK       0xf8UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD1_SFT        3</div><div class='add'>+	u8	alt_tos_dscp;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_ALT_TOS_DSCP_MASK 0x3fUL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_ALT_TOS_DSCP_SFT 0</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD4_MASK       0xc0UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD4_SFT        6</div><div class='add'>+	u8	cc_mode;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_CC_MODE_DCTCP         0x0UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_CC_MODE_PROBABILISTIC 0x1UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_CC_MODE_LAST \</div><div class='add'>+		CREQ_QUERY_ROCE_CC_RESP_SB_CC_MODE_PROBABILISTIC</div><div class='add'>+	u8	tx_queue;</div><div class='add'>+	__le16	rtt;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_RTT_MASK  0x3fffUL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_RTT_SFT   0</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD5_MASK 0xc000UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD5_SFT 14</div><div class='add'>+	__le16	tcp_cp;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TCP_CP_MASK 0x3ffUL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TCP_CP_SFT 0</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD6_MASK 0xfc00UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD6_SFT  10</div><div class='add'>+	__le16	inactivity_th;</div><div class='add'>+	u8	pkts_per_phase;</div><div class='add'>+	u8	time_per_phase;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_roce_cc_resp_sb_tlv (size:384b/48B) */</div><div class='add'>+struct creq_query_roce_cc_resp_sb_tlv {</div><div class='add'>+	__le16	cmd_discr;</div><div class='add'>+	u8	reserved_8b;</div><div class='add'>+	u8	tlv_flags;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_MORE         0x1UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_MORE_LAST      0x0UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_MORE_NOT_LAST  0x1UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_REQUIRED     0x2UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_REQUIRED_NO    (0x0UL &lt;&lt; 1)</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES   (0x1UL &lt;&lt; 1)</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_REQUIRED_LAST \</div><div class='add'>+		CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES</div><div class='add'>+	__le16	tlv_type;</div><div class='add'>+	__le16	length;</div><div class='add'>+	u8	total_size;</div><div class='add'>+	u8	reserved56[7];</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_OPCODE_QUERY_ROCE_CC 0x8dUL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_OPCODE_LAST \</div><div class='add'>+		CREQ_QUERY_ROCE_CC_RESP_SB_TLV_OPCODE_QUERY_ROCE_CC</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	u8	enable_cc;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_ENABLE_CC     0x1UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_UNUSED7_MASK  0xfeUL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_UNUSED7_SFT   1</div><div class='add'>+	u8	tos_dscp_tos_ecn;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TOS_ECN_MASK 0x3UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TOS_ECN_SFT  0</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TOS_DSCP_MASK 0xfcUL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TOS_DSCP_SFT 2</div><div class='add'>+	u8	g;</div><div class='add'>+	u8	num_phases_per_state;</div><div class='add'>+	__le16	init_cr;</div><div class='add'>+	__le16	init_tr;</div><div class='add'>+	u8	alt_vlan_pcp;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_ALT_VLAN_PCP_MASK 0x7UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_ALT_VLAN_PCP_SFT 0</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD1_MASK       0xf8UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD1_SFT        3</div><div class='add'>+	u8	alt_tos_dscp;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_ALT_TOS_DSCP_MASK 0x3fUL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_ALT_TOS_DSCP_SFT 0</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD4_MASK       0xc0UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD4_SFT        6</div><div class='add'>+	u8	cc_mode;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_CC_MODE_DCTCP         0x0UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_CC_MODE_PROBABILISTIC 0x1UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_CC_MODE_LAST\</div><div class='add'>+		CREQ_QUERY_ROCE_CC_RESP_SB_TLV_CC_MODE_PROBABILISTIC</div><div class='add'>+	u8	tx_queue;</div><div class='add'>+	__le16	rtt;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RTT_MASK  0x3fffUL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RTT_SFT   0</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD5_MASK 0xc000UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD5_SFT 14</div><div class='add'>+	__le16	tcp_cp;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TCP_CP_MASK 0x3ffUL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TCP_CP_SFT 0</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD6_MASK 0xfc00UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD6_SFT  10</div><div class='add'>+	__le16	inactivity_th;</div><div class='add'>+	u8	pkts_per_phase;</div><div class='add'>+	u8	time_per_phase;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_query_roce_cc_gen1_resp_sb_tlv (size:704b/88B) */</div><div class='add'>+struct creq_query_roce_cc_gen1_resp_sb_tlv {</div><div class='add'>+	__le16	cmd_discr;</div><div class='add'>+	u8	reserved_8b;</div><div class='add'>+	u8	tlv_flags;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_MORE         0x1UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_MORE_LAST      0x0UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_MORE_NOT_LAST  0x1UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_REQUIRED     0x2UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_REQUIRED_NO    (0x0UL &lt;&lt; 1)</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES   (0x1UL &lt;&lt; 1)</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_REQUIRED_LAST \</div><div class='add'>+		CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES</div><div class='add'>+	__le16	tlv_type;</div><div class='add'>+	__le16	length;</div><div class='add'>+	__le64	reserved64;</div><div class='add'>+	__le16	inactivity_th_hi;</div><div class='add'>+	__le16	min_time_between_cnps;</div><div class='add'>+	__le16	init_cp;</div><div class='add'>+	u8	tr_update_mode;</div><div class='add'>+	u8	tr_update_cycles;</div><div class='add'>+	u8	fr_num_rtts;</div><div class='add'>+	u8	ai_rate_increase;</div><div class='add'>+	__le16	reduction_relax_rtts_th;</div><div class='add'>+	__le16	additional_relax_cr_th;</div><div class='add'>+	__le16	cr_min_th;</div><div class='add'>+	u8	bw_avg_weight;</div><div class='add'>+	u8	actual_cr_factor;</div><div class='add'>+	__le16	max_cp_cr_th;</div><div class='add'>+	u8	cp_bias_en;</div><div class='add'>+	u8	cp_bias;</div><div class='add'>+	u8	cnp_ecn;</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_CNP_ECN_NOT_ECT 0x0UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_CNP_ECN_ECT_1   0x1UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_CNP_ECN_ECT_0   0x2UL</div><div class='add'>+	#define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_CNP_ECN_LAST \</div><div class='add'>+		CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_CNP_ECN_ECT_0</div><div class='add'>+	u8	rtt_jitter_en;</div><div class='add'>+	__le16	link_bytes_per_usec;</div><div class='add'>+	__le16	reset_cc_cr_th;</div><div class='add'>+	u8	cr_width;</div><div class='add'>+	u8	quota_period_min;</div><div class='add'>+	u8	quota_period_max;</div><div class='add'>+	u8	quota_period_abs_max;</div><div class='add'>+	__le16	tr_lower_bound;</div><div class='add'>+	u8	cr_prob_factor;</div><div class='add'>+	u8	tr_prob_factor;</div><div class='add'>+	__le16	fairness_cr_th;</div><div class='add'>+	u8	red_div;</div><div class='add'>+	u8	cnp_ratio_th;</div><div class='add'>+	__le16	exp_ai_rtts;</div><div class='add'>+	u8	exp_ai_cr_cp_ratio;</div><div class='add'>+	u8	use_rate_table;</div><div class='add'>+	__le16	cp_exp_update_th;</div><div class='add'>+	__le16	high_exp_ai_rtts_th1;</div><div class='add'>+	__le16	high_exp_ai_rtts_th2;</div><div class='add'>+	__le16	actual_cr_cong_free_rtts_th;</div><div class='add'>+	__le16	severe_cong_cr_th1;</div><div class='add'>+	__le16	severe_cong_cr_th2;</div><div class='add'>+	__le32	link64B_per_rtt;</div><div class='add'>+	u8	cc_ack_bytes;</div><div class='add'>+	u8	reduce_init_en;</div><div class='add'>+	__le16	reduce_init_cong_free_rtts_th;</div><div class='add'>+	u8	random_no_red_en;</div><div class='add'>+	u8	actual_cr_shift_correction_en;</div><div class='add'>+	u8	quota_period_adjust_en;</div><div class='add'>+	u8	reserved[5];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_modify_roce_cc (size:448b/56B) */</div><div class='add'>+struct cmdq_modify_roce_cc {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_OPCODE_MODIFY_ROCE_CC 0x8cUL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_OPCODE_LAST          CMDQ_MODIFY_ROCE_CC_OPCODE_MODIFY_ROCE_CC</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	modify_mask;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC            0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_G                    0x2UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_NUMPHASEPERSTATE     0x4UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INIT_CR              0x8UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INIT_TR              0x10UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN              0x20UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP             0x40UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP         0x80UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP         0x100UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_RTT                  0x200UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE              0x400UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TCP_CP               0x800UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TX_QUEUE             0x1000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INACTIVITY_CP        0x2000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TIME_PER_PHASE       0x4000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_PKTS_PER_PHASE       0x8000UL</div><div class='add'>+	u8	enable_cc;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_ENABLE_CC     0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_RSVD1_MASK    0xfeUL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_RSVD1_SFT     1</div><div class='add'>+	u8	g;</div><div class='add'>+	u8	num_phases_per_state;</div><div class='add'>+	u8	pkts_per_phase;</div><div class='add'>+	__le16	init_cr;</div><div class='add'>+	__le16	init_tr;</div><div class='add'>+	u8	tos_dscp_tos_ecn;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TOS_ECN_MASK 0x3UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TOS_ECN_SFT  0</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TOS_DSCP_MASK 0xfcUL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TOS_DSCP_SFT 2</div><div class='add'>+	u8	alt_vlan_pcp;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_ALT_VLAN_PCP_MASK 0x7UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_ALT_VLAN_PCP_SFT 0</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_RSVD3_MASK       0xf8UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_RSVD3_SFT        3</div><div class='add'>+	__le16	alt_tos_dscp;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_ALT_TOS_DSCP_MASK 0x3fUL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_ALT_TOS_DSCP_SFT 0</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_RSVD4_MASK       0xffc0UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_RSVD4_SFT        6</div><div class='add'>+	__le16	rtt;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_RTT_MASK  0x3fffUL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_RTT_SFT   0</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_RSVD5_MASK 0xc000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_RSVD5_SFT 14</div><div class='add'>+	__le16	tcp_cp;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TCP_CP_MASK 0x3ffUL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TCP_CP_SFT 0</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_RSVD6_MASK 0xfc00UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_RSVD6_SFT  10</div><div class='add'>+	u8	cc_mode;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_CC_MODE_DCTCP_CC_MODE         0x0UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_CC_MODE_PROBABILISTIC_CC_MODE 0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_CC_MODE_LAST CMDQ_MODIFY_ROCE_CC_CC_MODE_PROBABILISTIC_CC_MODE</div><div class='add'>+	u8	tx_queue;</div><div class='add'>+	__le16	inactivity_th;</div><div class='add'>+	u8	time_per_phase;</div><div class='add'>+	u8	reserved8_1;</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+	__le64	reserved64;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_modify_roce_cc_tlv (size:640b/80B) */</div><div class='add'>+struct cmdq_modify_roce_cc_tlv {</div><div class='add'>+	__le16	cmd_discr;</div><div class='add'>+	u8	reserved_8b;</div><div class='add'>+	u8	tlv_flags;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_MORE         0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_MORE_LAST      0x0UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_MORE_NOT_LAST  0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_REQUIRED     0x2UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_REQUIRED_NO    (0x0UL &lt;&lt; 1)</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_REQUIRED_YES   (0x1UL &lt;&lt; 1)</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_REQUIRED_LAST \</div><div class='add'>+		CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_REQUIRED_YES</div><div class='add'>+	__le16	tlv_type;</div><div class='add'>+	__le16	length;</div><div class='add'>+	u8	total_size;</div><div class='add'>+	u8	reserved56[7];</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_OPCODE_MODIFY_ROCE_CC 0x8cUL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_OPCODE_LAST CMDQ_MODIFY_ROCE_CC_TLV_OPCODE_MODIFY_ROCE_CC</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	modify_mask;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_ENABLE_CC            0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_G                    0x2UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_NUMPHASEPERSTATE     0x4UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_INIT_CR              0x8UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_INIT_TR              0x10UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_TOS_ECN              0x20UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_TOS_DSCP             0x40UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_ALT_VLAN_PCP         0x80UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_ALT_TOS_DSCP         0x100UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_RTT                  0x200UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_CC_MODE              0x400UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_TCP_CP               0x800UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_TX_QUEUE             0x1000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_INACTIVITY_CP        0x2000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_TIME_PER_PHASE       0x4000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_PKTS_PER_PHASE       0x8000UL</div><div class='add'>+	u8	enable_cc;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_ENABLE_CC     0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_RSVD1_MASK    0xfeUL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_RSVD1_SFT     1</div><div class='add'>+	u8	g;</div><div class='add'>+	u8	num_phases_per_state;</div><div class='add'>+	u8	pkts_per_phase;</div><div class='add'>+	__le16	init_cr;</div><div class='add'>+	__le16	init_tr;</div><div class='add'>+	u8	tos_dscp_tos_ecn;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_TOS_ECN_MASK 0x3UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_TOS_ECN_SFT  0</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_TOS_DSCP_MASK 0xfcUL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_TOS_DSCP_SFT 2</div><div class='add'>+	u8	alt_vlan_pcp;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_ALT_VLAN_PCP_MASK 0x7UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_ALT_VLAN_PCP_SFT 0</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_RSVD3_MASK       0xf8UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_RSVD3_SFT        3</div><div class='add'>+	__le16	alt_tos_dscp;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_ALT_TOS_DSCP_MASK 0x3fUL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_ALT_TOS_DSCP_SFT 0</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_RSVD4_MASK       0xffc0UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_RSVD4_SFT        6</div><div class='add'>+	__le16	rtt;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_RTT_MASK  0x3fffUL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_RTT_SFT   0</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_RSVD5_MASK 0xc000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_RSVD5_SFT 14</div><div class='add'>+	__le16	tcp_cp;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_TCP_CP_MASK 0x3ffUL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_TCP_CP_SFT 0</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_RSVD6_MASK 0xfc00UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_RSVD6_SFT  10</div><div class='add'>+	u8	cc_mode;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_CC_MODE_DCTCP_CC_MODE         0x0UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_CC_MODE_PROBABILISTIC_CC_MODE 0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_TLV_CC_MODE_LAST\</div><div class='add'>+		CMDQ_MODIFY_ROCE_CC_TLV_CC_MODE_PROBABILISTIC_CC_MODE</div><div class='add'>+	u8	tx_queue;</div><div class='add'>+	__le16	inactivity_th;</div><div class='add'>+	u8	time_per_phase;</div><div class='add'>+	u8	reserved8_1;</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+	__le64	reserved64;</div><div class='add'>+	__le64	reservedtlvpad;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_modify_roce_cc_gen1_tlv (size:768b/96B) */</div><div class='add'>+struct cmdq_modify_roce_cc_gen1_tlv {</div><div class='add'>+	__le16	cmd_discr;</div><div class='add'>+	u8	reserved_8b;</div><div class='add'>+	u8	tlv_flags;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_MORE         0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_MORE_LAST      0x0UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_MORE_NOT_LAST  0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_REQUIRED     0x2UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_REQUIRED_NO    (0x0UL &lt;&lt; 1)</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_REQUIRED_YES   (0x1UL &lt;&lt; 1)</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_REQUIRED_LAST\</div><div class='add'>+		CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_REQUIRED_YES</div><div class='add'>+	__le16	tlv_type;</div><div class='add'>+	__le16	length;</div><div class='add'>+	__le64	reserved64;</div><div class='add'>+	__le64	modify_mask;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_MIN_TIME_BETWEEN_CNPS       0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_INIT_CP                     0x2UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_TR_UPDATE_MODE              0x4UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_TR_UPDATE_CYCLES            0x8UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_FR_NUM_RTTS                 0x10UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_AI_RATE_INCREASE            0x20UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_REDUCTION_RELAX_RTTS_TH     0x40UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_ADDITIONAL_RELAX_CR_TH      0x80UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CR_MIN_TH                   0x100UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_BW_AVG_WEIGHT               0x200UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_ACTUAL_CR_FACTOR            0x400UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_MAX_CP_CR_TH                0x800UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CP_BIAS_EN                  0x1000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CP_BIAS                     0x2000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CNP_ECN                     0x4000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_RTT_JITTER_EN               0x8000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_LINK_BYTES_PER_USEC         0x10000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_RESET_CC_CR_TH              0x20000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CR_WIDTH                    0x40000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_QUOTA_PERIOD_MIN            0x80000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_QUOTA_PERIOD_MAX            0x100000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_QUOTA_PERIOD_ABS_MAX        0x200000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_TR_LOWER_BOUND              0x400000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CR_PROB_FACTOR              0x800000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_TR_PROB_FACTOR              0x1000000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_FAIRNESS_CR_TH              0x2000000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_RED_DIV                     0x4000000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CNP_RATIO_TH                0x8000000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_EXP_AI_RTTS                 0x10000000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_EXP_AI_CR_CP_RATIO          0x20000000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CP_EXP_UPDATE_TH            0x40000000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_HIGH_EXP_AI_RTTS_TH1        0x80000000UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_HIGH_EXP_AI_RTTS_TH2        0x100000000ULL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_USE_RATE_TABLE              0x200000000ULL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_LINK64B_PER_RTT             0x400000000ULL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_ACTUAL_CR_CONG_FREE_RTTS_TH 0x800000000ULL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_SEVERE_CONG_CR_TH1          0x1000000000ULL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_SEVERE_CONG_CR_TH2          0x2000000000ULL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CC_ACK_BYTES                0x4000000000ULL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_REDUCE_INIT_EN              0x8000000000ULL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_REDUCE_INIT_CONG_FREE_RTTS_TH \</div><div class='add'>+										0x10000000000ULL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_RANDOM_NO_RED_EN 0x20000000000ULL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_ACTUAL_CR_SHIFT_CORRECTION_EN \</div><div class='add'>+										0x40000000000ULL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_QUOTA_PERIOD_ADJUST_EN 0x80000000000ULL</div><div class='add'>+	__le16	inactivity_th_hi;</div><div class='add'>+	__le16	min_time_between_cnps;</div><div class='add'>+	__le16	init_cp;</div><div class='add'>+	u8	tr_update_mode;</div><div class='add'>+	u8	tr_update_cycles;</div><div class='add'>+	u8	fr_num_rtts;</div><div class='add'>+	u8	ai_rate_increase;</div><div class='add'>+	__le16	reduction_relax_rtts_th;</div><div class='add'>+	__le16	additional_relax_cr_th;</div><div class='add'>+	__le16	cr_min_th;</div><div class='add'>+	u8	bw_avg_weight;</div><div class='add'>+	u8	actual_cr_factor;</div><div class='add'>+	__le16	max_cp_cr_th;</div><div class='add'>+	u8	cp_bias_en;</div><div class='add'>+	u8	cp_bias;</div><div class='add'>+	u8	cnp_ecn;</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_CNP_ECN_NOT_ECT 0x0UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_CNP_ECN_ECT_1   0x1UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_CNP_ECN_ECT_0   0x2UL</div><div class='add'>+	#define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_CNP_ECN_LAST CMDQ_MODIFY_ROCE_CC_GEN1_TLV_CNP_ECN_ECT_0</div><div class='add'>+	u8	rtt_jitter_en;</div><div class='add'>+	__le16	link_bytes_per_usec;</div><div class='add'>+	__le16	reset_cc_cr_th;</div><div class='add'>+	u8	cr_width;</div><div class='add'>+	u8	quota_period_min;</div><div class='add'>+	u8	quota_period_max;</div><div class='add'>+	u8	quota_period_abs_max;</div><div class='add'>+	__le16	tr_lower_bound;</div><div class='add'>+	u8	cr_prob_factor;</div><div class='add'>+	u8	tr_prob_factor;</div><div class='add'>+	__le16	fairness_cr_th;</div><div class='add'>+	u8	red_div;</div><div class='add'>+	u8	cnp_ratio_th;</div><div class='add'>+	__le16	exp_ai_rtts;</div><div class='add'>+	u8	exp_ai_cr_cp_ratio;</div><div class='add'>+	u8	use_rate_table;</div><div class='add'>+	__le16	cp_exp_update_th;</div><div class='add'>+	__le16	high_exp_ai_rtts_th1;</div><div class='add'>+	__le16	high_exp_ai_rtts_th2;</div><div class='add'>+	__le16	actual_cr_cong_free_rtts_th;</div><div class='add'>+	__le16	severe_cong_cr_th1;</div><div class='add'>+	__le16	severe_cong_cr_th2;</div><div class='add'>+	__le32	link64B_per_rtt;</div><div class='add'>+	u8	cc_ack_bytes;</div><div class='add'>+	u8	reduce_init_en;</div><div class='add'>+	__le16	reduce_init_cong_free_rtts_th;</div><div class='add'>+	u8	random_no_red_en;</div><div class='add'>+	u8	actual_cr_shift_correction_en;</div><div class='add'>+	u8	quota_period_adjust_en;</div><div class='add'>+	u8	reserved[5];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_modify_roce_cc_resp (size:128b/16B) */</div><div class='add'>+struct creq_modify_roce_cc_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_MODIFY_ROCE_CC_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_MODIFY_ROCE_CC_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_MODIFY_ROCE_CC_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_MODIFY_ROCE_CC_RESP_TYPE_LAST     CREQ_MODIFY_ROCE_CC_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_MODIFY_ROCE_CC_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_MODIFY_ROCE_CC_RESP_EVENT_MODIFY_ROCE_CC 0x8cUL</div><div class='add'>+	#define CREQ_MODIFY_ROCE_CC_RESP_EVENT_LAST   CREQ_MODIFY_ROCE_CC_RESP_EVENT_MODIFY_ROCE_CC</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq_set_link_aggr_mode_cc (size:320b/40B) */</div><div class='add'>+struct cmdq_set_link_aggr_mode_cc {</div><div class='add'>+	u8	opcode;</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_OPCODE_SET_LINK_AGGR_MODE 0x8fUL</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_OPCODE_LAST \</div><div class='add'>+		CMDQ_SET_LINK_AGGR_MODE_OPCODE_SET_LINK_AGGR_MODE</div><div class='add'>+	u8	cmd_size;</div><div class='add'>+	__le16	flags;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	u8	resp_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le64	resp_addr;</div><div class='add'>+	__le32	modify_mask;</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_AGGR_EN             0x1UL</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_ACTIVE_PORT_MAP     0x2UL</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_MEMBER_PORT_MAP     0x4UL</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_AGGR_MODE           0x8UL</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_STAT_CTX_ID         0x10UL</div><div class='add'>+	u8	aggr_enable;</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_AGGR_ENABLE     0x1UL</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_RSVD1_MASK      0xfeUL</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_RSVD1_SFT       1</div><div class='add'>+	u8	active_port_map;</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_ACTIVE_PORT_MAP_MASK 0xfUL</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_ACTIVE_PORT_MAP_SFT 0</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_RSVD2_MASK          0xf0UL</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_RSVD2_SFT           4</div><div class='add'>+	u8	member_port_map;</div><div class='add'>+	u8	link_aggr_mode;</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_ACTIVE_ACTIVE 0x1UL</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_ACTIVE_BACKUP 0x2UL</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_BALANCE_XOR   0x3UL</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_802_3_AD      0x4UL</div><div class='add'>+	#define CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_LAST CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_802_3_AD</div><div class='add'>+	__le16	stat_ctx_id[4];</div><div class='add'>+	__le64	rsvd1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_set_link_aggr_mode_resources_resp (size:128b/16B) */</div><div class='add'>+struct creq_set_link_aggr_mode_resources_resp {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_SET_LINK_AGGR_MODE_RESP_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_SET_LINK_AGGR_MODE_RESP_TYPE_SFT     0</div><div class='add'>+	#define CREQ_SET_LINK_AGGR_MODE_RESP_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_SET_LINK_AGGR_MODE_RESP_TYPE_LAST CREQ_SET_LINK_AGGR_MODE_RESP_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_SET_LINK_AGGR_MODE_RESP_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_SET_LINK_AGGR_MODE_RESP_EVENT_SET_LINK_AGGR_MODE 0x8fUL</div><div class='add'>+	#define CREQ_SET_LINK_AGGR_MODE_RESP_EVENT_LAST\</div><div class='add'>+		CREQ_SET_LINK_AGGR_MODE_RESP_EVENT_SET_LINK_AGGR_MODE</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_func_event (size:128b/16B) */</div><div class='add'>+struct creq_func_event {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_FUNC_EVENT_TYPE_MASK      0x3fUL</div><div class='add'>+	#define CREQ_FUNC_EVENT_TYPE_SFT       0</div><div class='add'>+	#define CREQ_FUNC_EVENT_TYPE_FUNC_EVENT  0x3aUL</div><div class='add'>+	#define CREQ_FUNC_EVENT_TYPE_LAST       CREQ_FUNC_EVENT_TYPE_FUNC_EVENT</div><div class='add'>+	u8	reserved56[7];</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_FUNC_EVENT_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR       0x1UL</div><div class='add'>+	#define CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR      0x2UL</div><div class='add'>+	#define CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR       0x3UL</div><div class='add'>+	#define CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR      0x4UL</div><div class='add'>+	#define CREQ_FUNC_EVENT_EVENT_CQ_ERROR           0x5UL</div><div class='add'>+	#define CREQ_FUNC_EVENT_EVENT_TQM_ERROR          0x6UL</div><div class='add'>+	#define CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR         0x7UL</div><div class='add'>+	#define CREQ_FUNC_EVENT_EVENT_CFCS_ERROR         0x8UL</div><div class='add'>+	#define CREQ_FUNC_EVENT_EVENT_CFCC_ERROR         0x9UL</div><div class='add'>+	#define CREQ_FUNC_EVENT_EVENT_CFCM_ERROR         0xaUL</div><div class='add'>+	#define CREQ_FUNC_EVENT_EVENT_TIM_ERROR          0xbUL</div><div class='add'>+	#define CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST    0x80UL</div><div class='add'>+	#define CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED 0x81UL</div><div class='add'>+	#define CREQ_FUNC_EVENT_EVENT_LAST              CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_qp_event (size:128b/16B) */</div><div class='add'>+struct creq_qp_event {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_QP_EVENT_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_QP_EVENT_TYPE_SFT     0</div><div class='add'>+	#define CREQ_QP_EVENT_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_QP_EVENT_TYPE_LAST     CREQ_QP_EVENT_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	#define CREQ_QP_EVENT_STATUS_SUCCESS           0x0UL</div><div class='add'>+	#define CREQ_QP_EVENT_STATUS_FAIL              0x1UL</div><div class='add'>+	#define CREQ_QP_EVENT_STATUS_RESOURCES         0x2UL</div><div class='add'>+	#define CREQ_QP_EVENT_STATUS_INVALID_CMD       0x3UL</div><div class='add'>+	#define CREQ_QP_EVENT_STATUS_NOT_IMPLEMENTED   0x4UL</div><div class='add'>+	#define CREQ_QP_EVENT_STATUS_INVALID_PARAMETER 0x5UL</div><div class='add'>+	#define CREQ_QP_EVENT_STATUS_HARDWARE_ERROR    0x6UL</div><div class='add'>+	#define CREQ_QP_EVENT_STATUS_INTERNAL_ERROR    0x7UL</div><div class='add'>+	#define CREQ_QP_EVENT_STATUS_LAST             CREQ_QP_EVENT_STATUS_INTERNAL_ERROR</div><div class='add'>+	__le16	cookie;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_QP_EVENT_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_CREATE_QP             0x1UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_DESTROY_QP            0x2UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_MODIFY_QP             0x3UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_QUERY_QP              0x4UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_CREATE_SRQ            0x5UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_DESTROY_SRQ           0x6UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_QUERY_SRQ             0x8UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_CREATE_CQ             0x9UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_DESTROY_CQ            0xaUL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_RESIZE_CQ             0xcUL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_ALLOCATE_MRW          0xdUL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_DEALLOCATE_KEY        0xeUL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_REGISTER_MR           0xfUL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_DEREGISTER_MR         0x10UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_ADD_GID               0x11UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_DELETE_GID            0x12UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_MODIFY_GID            0x17UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_QUERY_GID             0x18UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_CREATE_QP1            0x13UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_DESTROY_QP1           0x14UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_CREATE_AH             0x15UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_DESTROY_AH            0x16UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_INITIALIZE_FW         0x80UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_DEINITIALIZE_FW       0x81UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_STOP_FUNC             0x82UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_QUERY_FUNC            0x83UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_SET_FUNC_RESOURCES    0x84UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_READ_CONTEXT          0x85UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_MAP_TC_TO_COS         0x8aUL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_QUERY_VERSION         0x8bUL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_MODIFY_CC             0x8cUL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_QUERY_CC              0x8dUL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_QUERY_ROCE_STATS      0x8eUL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_SET_LINK_AGGR_MODE    0x8fUL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_QUERY_QP_EXTEND       0x91UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION 0xc0UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION 0xc1UL</div><div class='add'>+	#define CREQ_QP_EVENT_EVENT_LAST                 CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* creq_qp_error_notification (size:128b/16B) */</div><div class='ctx'> struct creq_qp_error_notification {</div><div class='del'>-	u8 type;</div><div class='del'>-	#define CREQ_QP_ERROR_NOTIFICATION_TYPE_MASK		    0x3fUL</div><div class='del'>-	#define CREQ_QP_ERROR_NOTIFICATION_TYPE_SFT		    0</div><div class='del'>-	#define CREQ_QP_ERROR_NOTIFICATION_TYPE_QP_EVENT	   0x38UL</div><div class='del'>-	#define CREQ_QP_ERROR_NOTIFICATION_RESERVED2_MASK	    0xc0UL</div><div class='del'>-	#define CREQ_QP_ERROR_NOTIFICATION_RESERVED2_SFT	    6</div><div class='del'>-	u8 status;</div><div class='del'>-	u8 req_slow_path_state;</div><div class='del'>-	u8 req_err_state_reason;</div><div class='del'>-	__le32 xid;</div><div class='del'>-	u8 v;</div><div class='del'>-	#define CREQ_QP_ERROR_NOTIFICATION_V			    0x1UL</div><div class='del'>-	#define CREQ_QP_ERROR_NOTIFICATION_RESERVED7_MASK	    0xfeUL</div><div class='del'>-	#define CREQ_QP_ERROR_NOTIFICATION_RESERVED7_SFT	    1</div><div class='del'>-	u8 event;</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_TYPE_SFT     0</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_TYPE_QP_EVENT  0x38UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_TYPE_LAST     CREQ_QP_ERROR_NOTIFICATION_TYPE_QP_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	u8	req_slow_path_state;</div><div class='add'>+	u8	req_err_state_reason;</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_NO_ERROR                    0X0UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_OPCODE_ERROR            0X1UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_TIMEOUT_RETRY_LIMIT     0X2UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RNR_TIMEOUT_RETRY_LIMIT 0X3UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_NAK_ARRIVAL_1           0X4UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_NAK_ARRIVAL_2           0X5UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_NAK_ARRIVAL_3           0X6UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_NAK_ARRIVAL_4           0X7UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RX_MEMORY_ERROR         0X8UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_TX_MEMORY_ERROR         0X9UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_READ_RESP_LENGTH        0XAUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_INVALID_READ_RESP       0XBUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_ILLEGAL_BIND            0XCUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_ILLEGAL_FAST_REG        0XDUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_ILLEGAL_INVALIDATE      0XEUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_CMP_ERROR               0XFUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RETRAN_LOCAL_ERROR      0X10UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_WQE_FORMAT_ERROR        0X11UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_ORRQ_FORMAT_ERROR       0X12UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_INVALID_AVID_ERROR      0X13UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_AV_DOMAIN_ERROR         0X14UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_CQ_LOAD_ERROR           0X15UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_SERV_TYPE_ERROR         0X16UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_INVALID_OP_ERROR        0X17UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_TX_PCI_ERROR            0X18UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RX_PCI_ERROR            0X19UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_PROD_WQE_MSMTCH_ERROR   0X1AUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_PSN_RANGE_CHECK_ERROR   0X1BUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RETX_SETUP_ERROR        0X1CUL</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='ctx'> 	#define CREQ_QP_ERROR_NOTIFICATION_EVENT_QP_ERROR_NOTIFICATION 0xc0UL</div><div class='del'>-	u8 res_slow_path_state;</div><div class='del'>-	u8 res_err_state_reason;</div><div class='del'>-	__le16 sq_cons_idx;</div><div class='del'>-	__le16 rq_cons_idx;</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_EVENT_LAST \</div><div class='add'>+		CREQ_QP_ERROR_NOTIFICATION_EVENT_QP_ERROR_NOTIFICATION</div><div class='add'>+	u8	res_slow_path_state;</div><div class='add'>+	u8	res_err_state_reason;</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_NO_ERROR                      0x0UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_EXCEED_MAX                0x1UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH   0x2UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE               0x3UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_OPCODE_ERROR              0x4UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT 0x5UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY          0x6UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR           0x7UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION          0x8UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR            0x9UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY          0xaUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR           0xbUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION          0xcUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR            0xdUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW                0xeUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE        0xfUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC            0x10UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_REM_INVALIDATE            0x11UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_MEMORY_ERROR              0x12UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_SRQ_ERROR                 0x13UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_CMP_ERROR                 0x14UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_INVALID_DUP_RKEY          0x15UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR          0x16UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_IRRQ_FORMAT_ERROR         0x17UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR             0x18UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR            0x19UL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR              0x1bUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR              0x1cUL</div><div class='add'>+	#define CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_PSN_NOT_FOUND             0x1dUL</div><div class='add'>+	__le16	sq_cons_idx;</div><div class='add'>+	__le16	rq_cons_idx;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* RoCE Slowpath HSI Specification 1.6.0 */</div><div class='del'>-#define ROCE_SP_HSI_VERSION_MAJOR	1</div><div class='del'>-#define ROCE_SP_HSI_VERSION_MINOR	6</div><div class='del'>-#define ROCE_SP_HSI_VERSION_UPDATE	0</div><div class='add'>+/* creq_cq_error_notification (size:128b/16B) */</div><div class='add'>+struct creq_cq_error_notification {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define CREQ_CQ_ERROR_NOTIFICATION_TYPE_MASK    0x3fUL</div><div class='add'>+	#define CREQ_CQ_ERROR_NOTIFICATION_TYPE_SFT     0</div><div class='add'>+	#define CREQ_CQ_ERROR_NOTIFICATION_TYPE_CQ_EVENT  0x38UL</div><div class='add'>+	#define CREQ_CQ_ERROR_NOTIFICATION_TYPE_LAST     CREQ_CQ_ERROR_NOTIFICATION_TYPE_CQ_EVENT</div><div class='add'>+	u8	status;</div><div class='add'>+	u8	cq_err_reason;</div><div class='add'>+	#define CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_INVALID_ERROR  0x1UL</div><div class='add'>+	#define CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_OVERFLOW_ERROR 0x2UL</div><div class='add'>+	#define CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_LOAD_ERROR     0x3UL</div><div class='add'>+	#define CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_INVALID_ERROR  0x4UL</div><div class='add'>+	#define CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_OVERFLOW_ERROR 0x5UL</div><div class='add'>+	#define CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR     0x6UL</div><div class='add'>+	#define CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_LAST \</div><div class='add'>+			CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le32	xid;</div><div class='add'>+	u8	v;</div><div class='add'>+	#define CREQ_CQ_ERROR_NOTIFICATION_V     0x1UL</div><div class='add'>+	u8	event;</div><div class='add'>+	#define CREQ_CQ_ERROR_NOTIFICATION_EVENT_CQ_ERROR_NOTIFICATION 0xc1UL</div><div class='add'>+	#define CREQ_CQ_ERROR_NOTIFICATION_EVENT_LAST \</div><div class='add'>+		CREQ_CQ_ERROR_NOTIFICATION_EVENT_CQ_ERROR_NOTIFICATION</div><div class='add'>+	u8	reserved48[6];</div><div class='add'>+};</div><div class='ctx'> </div><div class='del'>-#define ROCE_SP_HSI_VERSION_STR	"1.6.0"</div><div class='del'>-/*</div><div class='del'>- * Following is the signature for ROCE_SP_HSI message field that indicates not</div><div class='del'>- * applicable (All F's). Need to cast it the size of the field if needed.</div><div class='del'>- */</div><div class='del'>-#define ROCE_SP_HSI_NA_SIGNATURE	((__le32)(-1))</div><div class='del'>-#endif /* __BNXT_RE_HSI_H__ */</div><div class='add'>+/* sq_base (size:64b/8B) */</div><div class='add'>+struct sq_base {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define SQ_BASE_WQE_TYPE_SEND           0x0UL</div><div class='add'>+	#define SQ_BASE_WQE_TYPE_SEND_W_IMMEAD  0x1UL</div><div class='add'>+	#define SQ_BASE_WQE_TYPE_SEND_W_INVALID 0x2UL</div><div class='add'>+	#define SQ_BASE_WQE_TYPE_WRITE_WQE      0x4UL</div><div class='add'>+	#define SQ_BASE_WQE_TYPE_WRITE_W_IMMEAD 0x5UL</div><div class='add'>+	#define SQ_BASE_WQE_TYPE_READ_WQE       0x6UL</div><div class='add'>+	#define SQ_BASE_WQE_TYPE_ATOMIC_CS      0x8UL</div><div class='add'>+	#define SQ_BASE_WQE_TYPE_ATOMIC_FA      0xbUL</div><div class='add'>+	#define SQ_BASE_WQE_TYPE_LOCAL_INVALID  0xcUL</div><div class='add'>+	#define SQ_BASE_WQE_TYPE_FR_PMR         0xdUL</div><div class='add'>+	#define SQ_BASE_WQE_TYPE_BIND           0xeUL</div><div class='add'>+	#define SQ_BASE_WQE_TYPE_FR_PPMR        0xfUL</div><div class='add'>+	#define SQ_BASE_WQE_TYPE_LAST          SQ_BASE_WQE_TYPE_FR_PPMR</div><div class='add'>+	u8	unused_0[7];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_sge (size:128b/16B) */</div><div class='add'>+struct sq_sge {</div><div class='add'>+	__le64	va_or_pa;</div><div class='add'>+	__le32	l_key;</div><div class='add'>+	__le32	size;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_psn_search (size:64b/8B) */</div><div class='add'>+struct sq_psn_search {</div><div class='add'>+	__le32	opcode_start_psn;</div><div class='add'>+	#define SQ_PSN_SEARCH_START_PSN_MASK 0xffffffUL</div><div class='add'>+	#define SQ_PSN_SEARCH_START_PSN_SFT 0</div><div class='add'>+	#define SQ_PSN_SEARCH_OPCODE_MASK   0xff000000UL</div><div class='add'>+	#define SQ_PSN_SEARCH_OPCODE_SFT    24</div><div class='add'>+	__le32	flags_next_psn;</div><div class='add'>+	#define SQ_PSN_SEARCH_NEXT_PSN_MASK 0xffffffUL</div><div class='add'>+	#define SQ_PSN_SEARCH_NEXT_PSN_SFT 0</div><div class='add'>+	#define SQ_PSN_SEARCH_FLAGS_MASK   0xff000000UL</div><div class='add'>+	#define SQ_PSN_SEARCH_FLAGS_SFT    24</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_psn_search_ext (size:128b/16B) */</div><div class='add'>+struct sq_psn_search_ext {</div><div class='add'>+	__le32	opcode_start_psn;</div><div class='add'>+	#define SQ_PSN_SEARCH_EXT_START_PSN_MASK 0xffffffUL</div><div class='add'>+	#define SQ_PSN_SEARCH_EXT_START_PSN_SFT 0</div><div class='add'>+	#define SQ_PSN_SEARCH_EXT_OPCODE_MASK   0xff000000UL</div><div class='add'>+	#define SQ_PSN_SEARCH_EXT_OPCODE_SFT    24</div><div class='add'>+	__le32	flags_next_psn;</div><div class='add'>+	#define SQ_PSN_SEARCH_EXT_NEXT_PSN_MASK 0xffffffUL</div><div class='add'>+	#define SQ_PSN_SEARCH_EXT_NEXT_PSN_SFT 0</div><div class='add'>+	#define SQ_PSN_SEARCH_EXT_FLAGS_MASK   0xff000000UL</div><div class='add'>+	#define SQ_PSN_SEARCH_EXT_FLAGS_SFT    24</div><div class='add'>+	__le16	start_slot_idx;</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_msn_search (size:64b/8B) */</div><div class='add'>+struct sq_msn_search {</div><div class='add'>+	__le64	start_idx_next_psn_start_psn;</div><div class='add'>+	#define SQ_MSN_SEARCH_START_PSN_MASK 0xffffffUL</div><div class='add'>+	#define SQ_MSN_SEARCH_START_PSN_SFT 0</div><div class='add'>+	#define SQ_MSN_SEARCH_NEXT_PSN_MASK 0xffffff000000ULL</div><div class='add'>+	#define SQ_MSN_SEARCH_NEXT_PSN_SFT  24</div><div class='add'>+	#define SQ_MSN_SEARCH_START_IDX_MASK 0xffff000000000000ULL</div><div class='add'>+	#define SQ_MSN_SEARCH_START_IDX_SFT 48</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_send (size:1024b/128B) */</div><div class='add'>+struct sq_send {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define SQ_SEND_WQE_TYPE_SEND           0x0UL</div><div class='add'>+	#define SQ_SEND_WQE_TYPE_SEND_W_IMMEAD  0x1UL</div><div class='add'>+	#define SQ_SEND_WQE_TYPE_SEND_W_INVALID 0x2UL</div><div class='add'>+	#define SQ_SEND_WQE_TYPE_LAST          SQ_SEND_WQE_TYPE_SEND_W_INVALID</div><div class='add'>+	u8	flags;</div><div class='add'>+	#define SQ_SEND_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL</div><div class='add'>+	#define SQ_SEND_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT  0</div><div class='add'>+	#define SQ_SEND_FLAGS_SIGNAL_COMP                                            0x1UL</div><div class='add'>+	#define SQ_SEND_FLAGS_RD_OR_ATOMIC_FENCE                                     0x2UL</div><div class='add'>+	#define SQ_SEND_FLAGS_UC_FENCE                                               0x4UL</div><div class='add'>+	#define SQ_SEND_FLAGS_SE                                                     0x8UL</div><div class='add'>+	#define SQ_SEND_FLAGS_INLINE                                                 0x10UL</div><div class='add'>+	#define SQ_SEND_FLAGS_WQE_TS_EN                                              0x20UL</div><div class='add'>+	#define SQ_SEND_FLAGS_DEBUG_TRACE                                            0x40UL</div><div class='add'>+	u8	wqe_size;</div><div class='add'>+	u8	reserved8_1;</div><div class='add'>+	__le32	inv_key_or_imm_data;</div><div class='add'>+	__le32	length;</div><div class='add'>+	__le32	q_key;</div><div class='add'>+	__le32	dst_qp;</div><div class='add'>+	#define SQ_SEND_DST_QP_MASK 0xffffffUL</div><div class='add'>+	#define SQ_SEND_DST_QP_SFT 0</div><div class='add'>+	__le32	avid;</div><div class='add'>+	#define SQ_SEND_AVID_MASK 0xfffffUL</div><div class='add'>+	#define SQ_SEND_AVID_SFT 0</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+	__le32	timestamp;</div><div class='add'>+	#define SQ_SEND_TIMESTAMP_MASK 0xffffffUL</div><div class='add'>+	#define SQ_SEND_TIMESTAMP_SFT 0</div><div class='add'>+	__le32	data[24];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_send_hdr (size:256b/32B) */</div><div class='add'>+struct sq_send_hdr {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define SQ_SEND_HDR_WQE_TYPE_SEND           0x0UL</div><div class='add'>+	#define SQ_SEND_HDR_WQE_TYPE_SEND_W_IMMEAD  0x1UL</div><div class='add'>+	#define SQ_SEND_HDR_WQE_TYPE_SEND_W_INVALID 0x2UL</div><div class='add'>+	#define SQ_SEND_HDR_WQE_TYPE_LAST          SQ_SEND_HDR_WQE_TYPE_SEND_W_INVALID</div><div class='add'>+	u8	flags;</div><div class='add'>+	#define SQ_SEND_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL</div><div class='add'>+	#define SQ_SEND_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT  0</div><div class='add'>+	#define SQ_SEND_HDR_FLAGS_SIGNAL_COMP                                            0x1UL</div><div class='add'>+	#define SQ_SEND_HDR_FLAGS_RD_OR_ATOMIC_FENCE                                     0x2UL</div><div class='add'>+	#define SQ_SEND_HDR_FLAGS_UC_FENCE                                               0x4UL</div><div class='add'>+	#define SQ_SEND_HDR_FLAGS_SE                                                     0x8UL</div><div class='add'>+	#define SQ_SEND_HDR_FLAGS_INLINE                                                 0x10UL</div><div class='add'>+	#define SQ_SEND_HDR_FLAGS_WQE_TS_EN                                              0x20UL</div><div class='add'>+	#define SQ_SEND_HDR_FLAGS_DEBUG_TRACE                                            0x40UL</div><div class='add'>+	u8	wqe_size;</div><div class='add'>+	u8	reserved8_1;</div><div class='add'>+	__le32	inv_key_or_imm_data;</div><div class='add'>+	__le32	length;</div><div class='add'>+	__le32	q_key;</div><div class='add'>+	__le32	dst_qp;</div><div class='add'>+	#define SQ_SEND_HDR_DST_QP_MASK 0xffffffUL</div><div class='add'>+	#define SQ_SEND_HDR_DST_QP_SFT 0</div><div class='add'>+	__le32	avid;</div><div class='add'>+	#define SQ_SEND_HDR_AVID_MASK 0xfffffUL</div><div class='add'>+	#define SQ_SEND_HDR_AVID_SFT 0</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+	__le32	timestamp;</div><div class='add'>+	#define SQ_SEND_HDR_TIMESTAMP_MASK 0xffffffUL</div><div class='add'>+	#define SQ_SEND_HDR_TIMESTAMP_SFT 0</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_send_raweth_qp1 (size:1024b/128B) */</div><div class='add'>+struct sq_send_raweth_qp1 {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_WQE_TYPE_SEND 0x0UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_WQE_TYPE_LAST SQ_SEND_RAWETH_QP1_WQE_TYPE_SEND</div><div class='add'>+	u8	flags;</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK \</div><div class='add'>+		0xffUL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT \</div><div class='add'>+		0</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_FLAGS_SIGNAL_COMP  0x1UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_FLAGS_RD_OR_ATOMIC_FENCE  0x2UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_FLAGS_UC_FENCE 0x4UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_FLAGS_SE	0x8UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_FLAGS_INLINE 0x10UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_FLAGS_WQE_TS_EN 0x20UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_FLAGS_DEBUG_TRACE 0x40UL</div><div class='add'>+	u8	wqe_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le16	lflags;</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_LFLAGS_TCP_UDP_CHKSUM     0x1UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM          0x2UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_LFLAGS_NOCRC              0x4UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_LFLAGS_STAMP              0x8UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_LFLAGS_T_IP_CHKSUM        0x10UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC           0x100UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_LFLAGS_FCOE_CRC           0x200UL</div><div class='add'>+	__le16	cfa_action;</div><div class='add'>+	__le32	length;</div><div class='add'>+	__le32	reserved32_1;</div><div class='add'>+	__le32	cfa_meta;</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK     0xfffUL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT      0</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_DE           0x1000UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_MASK     0xe000UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_SFT      13</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_MASK    0x70000UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_SFT     16</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID88A8  (0x0UL &lt;&lt; 16)</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID8100  (0x1UL &lt;&lt; 16)</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9100  (0x2UL &lt;&lt; 16)</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9200  (0x3UL &lt;&lt; 16)</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9300  (0x4UL &lt;&lt; 16)</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG   (0x5UL &lt;&lt; 16)</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_LAST\</div><div class='add'>+		SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_MASK 0xff80000UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_SFT 19</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_KEY_MASK          0xf0000000UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_KEY_SFT           28</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_KEY_NONE            (0x0UL &lt;&lt; 28)</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG        (0x1UL &lt;&lt; 28)</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_CFA_META_KEY_LAST SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG</div><div class='add'>+	__le32	reserved32_2;</div><div class='add'>+	__le32	reserved32_3;</div><div class='add'>+	__le32	timestamp;</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_TIMESTAMP_MASK 0xffffffUL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_TIMESTAMP_SFT 0</div><div class='add'>+	__le32	data[24];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_send_raweth_qp1_hdr (size:256b/32B) */</div><div class='add'>+struct sq_send_raweth_qp1_hdr {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_WQE_TYPE_SEND 0x0UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_WQE_TYPE_LAST SQ_SEND_RAWETH_QP1_HDR_WQE_TYPE_SEND</div><div class='add'>+	u8	flags;</div><div class='add'>+	#define \</div><div class='add'>+	SQ_SEND_RAWETH_QP1_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT\</div><div class='add'>+		0</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_FLAGS_SIGNAL_COMP 0x1UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_FLAGS_UC_FENCE 0x4UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_FLAGS_SE 0x8UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_FLAGS_INLINE 0x10UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_FLAGS_WQE_TS_EN 0x20UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_FLAGS_DEBUG_TRACE 0x40UL</div><div class='add'>+	u8	wqe_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le16	lflags;</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_LFLAGS_TCP_UDP_CHKSUM     0x1UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_LFLAGS_IP_CHKSUM          0x2UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_LFLAGS_NOCRC              0x4UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_LFLAGS_STAMP              0x8UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_LFLAGS_T_IP_CHKSUM        0x10UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_LFLAGS_ROCE_CRC           0x100UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_LFLAGS_FCOE_CRC           0x200UL</div><div class='add'>+	__le16	cfa_action;</div><div class='add'>+	__le32	length;</div><div class='add'>+	__le32	reserved32_1;</div><div class='add'>+	__le32	cfa_meta;</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_VID_MASK     0xfffUL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_VID_SFT      0</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_DE           0x1000UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_PRI_MASK     0xe000UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_PRI_SFT      13</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_MASK    0x70000UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_SFT     16</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPID88A8  (0x0UL &lt;&lt; 16)</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPID8100  (0x1UL &lt;&lt; 16)</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPID9100  (0x2UL &lt;&lt; 16)</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPID9200  (0x3UL &lt;&lt; 16)</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPID9300  (0x4UL &lt;&lt; 16)</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPIDCFG   (0x5UL &lt;&lt; 16)</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_LAST\</div><div class='add'>+			SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPIDCFG</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_RESERVED_MASK 0xff80000UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_RESERVED_SFT 19</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_MASK          0xf0000000UL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_SFT           28</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_NONE            (0x0UL &lt;&lt; 28)</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_VLAN_TAG        (0x1UL &lt;&lt; 28)</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_LAST\</div><div class='add'>+		SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_VLAN_TAG</div><div class='add'>+	__le32	reserved32_2;</div><div class='add'>+	__le32	reserved32_3;</div><div class='add'>+	__le32	timestamp;</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_TIMESTAMP_MASK 0xffffffUL</div><div class='add'>+	#define SQ_SEND_RAWETH_QP1_HDR_TIMESTAMP_SFT 0</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_rdma (size:1024b/128B) */</div><div class='add'>+struct sq_rdma {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define SQ_RDMA_WQE_TYPE_WRITE_WQE      0x4UL</div><div class='add'>+	#define SQ_RDMA_WQE_TYPE_WRITE_W_IMMEAD 0x5UL</div><div class='add'>+	#define SQ_RDMA_WQE_TYPE_READ_WQE       0x6UL</div><div class='add'>+	#define SQ_RDMA_WQE_TYPE_LAST          SQ_RDMA_WQE_TYPE_READ_WQE</div><div class='add'>+	u8	flags;</div><div class='add'>+	#define SQ_RDMA_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL</div><div class='add'>+	#define SQ_RDMA_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT  0</div><div class='add'>+	#define SQ_RDMA_FLAGS_SIGNAL_COMP                                            0x1UL</div><div class='add'>+	#define SQ_RDMA_FLAGS_RD_OR_ATOMIC_FENCE                                     0x2UL</div><div class='add'>+	#define SQ_RDMA_FLAGS_UC_FENCE                                               0x4UL</div><div class='add'>+	#define SQ_RDMA_FLAGS_SE                                                     0x8UL</div><div class='add'>+	#define SQ_RDMA_FLAGS_INLINE                                                 0x10UL</div><div class='add'>+	#define SQ_RDMA_FLAGS_WQE_TS_EN                                              0x20UL</div><div class='add'>+	#define SQ_RDMA_FLAGS_DEBUG_TRACE                                            0x40UL</div><div class='add'>+	u8	wqe_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le32	imm_data;</div><div class='add'>+	__le32	length;</div><div class='add'>+	__le32	reserved32_1;</div><div class='add'>+	__le64	remote_va;</div><div class='add'>+	__le32	remote_key;</div><div class='add'>+	__le32	timestamp;</div><div class='add'>+	#define SQ_RDMA_TIMESTAMP_MASK 0xffffffUL</div><div class='add'>+	#define SQ_RDMA_TIMESTAMP_SFT 0</div><div class='add'>+	__le32	data[24];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_rdma_hdr (size:256b/32B) */</div><div class='add'>+struct sq_rdma_hdr {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define SQ_RDMA_HDR_WQE_TYPE_WRITE_WQE      0x4UL</div><div class='add'>+	#define SQ_RDMA_HDR_WQE_TYPE_WRITE_W_IMMEAD 0x5UL</div><div class='add'>+	#define SQ_RDMA_HDR_WQE_TYPE_READ_WQE       0x6UL</div><div class='add'>+	#define SQ_RDMA_HDR_WQE_TYPE_LAST          SQ_RDMA_HDR_WQE_TYPE_READ_WQE</div><div class='add'>+	u8	flags;</div><div class='add'>+	#define SQ_RDMA_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL</div><div class='add'>+	#define SQ_RDMA_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT  0</div><div class='add'>+	#define SQ_RDMA_HDR_FLAGS_SIGNAL_COMP                                            0x1UL</div><div class='add'>+	#define SQ_RDMA_HDR_FLAGS_RD_OR_ATOMIC_FENCE                                     0x2UL</div><div class='add'>+	#define SQ_RDMA_HDR_FLAGS_UC_FENCE                                               0x4UL</div><div class='add'>+	#define SQ_RDMA_HDR_FLAGS_SE                                                     0x8UL</div><div class='add'>+	#define SQ_RDMA_HDR_FLAGS_INLINE                                                 0x10UL</div><div class='add'>+	#define SQ_RDMA_HDR_FLAGS_WQE_TS_EN                                              0x20UL</div><div class='add'>+	#define SQ_RDMA_HDR_FLAGS_DEBUG_TRACE                                            0x40UL</div><div class='add'>+	u8	wqe_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le32	imm_data;</div><div class='add'>+	__le32	length;</div><div class='add'>+	__le32	reserved32_1;</div><div class='add'>+	__le64	remote_va;</div><div class='add'>+	__le32	remote_key;</div><div class='add'>+	__le32	timestamp;</div><div class='add'>+	#define SQ_RDMA_HDR_TIMESTAMP_MASK 0xffffffUL</div><div class='add'>+	#define SQ_RDMA_HDR_TIMESTAMP_SFT 0</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_atomic (size:1024b/128B) */</div><div class='add'>+struct sq_atomic {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define SQ_ATOMIC_WQE_TYPE_ATOMIC_CS 0x8UL</div><div class='add'>+	#define SQ_ATOMIC_WQE_TYPE_ATOMIC_FA 0xbUL</div><div class='add'>+	#define SQ_ATOMIC_WQE_TYPE_LAST     SQ_ATOMIC_WQE_TYPE_ATOMIC_FA</div><div class='add'>+	u8	flags;</div><div class='add'>+	#define SQ_ATOMIC_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK   0xffUL</div><div class='add'>+	#define SQ_ATOMIC_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT    0</div><div class='add'>+	#define SQ_ATOMIC_FLAGS_SIGNAL_COMP                                              0x1UL</div><div class='add'>+	#define SQ_ATOMIC_FLAGS_RD_OR_ATOMIC_FENCE                                       0x2UL</div><div class='add'>+	#define SQ_ATOMIC_FLAGS_UC_FENCE                                                 0x4UL</div><div class='add'>+	#define SQ_ATOMIC_FLAGS_SE                                                       0x8UL</div><div class='add'>+	#define SQ_ATOMIC_FLAGS_INLINE                                                   0x10UL</div><div class='add'>+	#define SQ_ATOMIC_FLAGS_WQE_TS_EN                                                0x20UL</div><div class='add'>+	#define SQ_ATOMIC_FLAGS_DEBUG_TRACE                                              0x40UL</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	remote_key;</div><div class='add'>+	__le64	remote_va;</div><div class='add'>+	__le64	swap_data;</div><div class='add'>+	__le64	cmp_data;</div><div class='add'>+	__le32	data[24];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_atomic_hdr (size:256b/32B) */</div><div class='add'>+struct sq_atomic_hdr {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define SQ_ATOMIC_HDR_WQE_TYPE_ATOMIC_CS 0x8UL</div><div class='add'>+	#define SQ_ATOMIC_HDR_WQE_TYPE_ATOMIC_FA 0xbUL</div><div class='add'>+	#define SQ_ATOMIC_HDR_WQE_TYPE_LAST     SQ_ATOMIC_HDR_WQE_TYPE_ATOMIC_FA</div><div class='add'>+	u8	flags;</div><div class='add'>+	#define SQ_ATOMIC_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL</div><div class='add'>+	#define SQ_ATOMIC_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT 0</div><div class='add'>+	#define SQ_ATOMIC_HDR_FLAGS_SIGNAL_COMP  0x1UL</div><div class='add'>+	#define SQ_ATOMIC_HDR_FLAGS_RD_OR_ATOMIC_FENCE  0x2UL</div><div class='add'>+	#define SQ_ATOMIC_HDR_FLAGS_UC_FENCE            0x4UL</div><div class='add'>+	#define SQ_ATOMIC_HDR_FLAGS_SE                  0x8UL</div><div class='add'>+	#define SQ_ATOMIC_HDR_FLAGS_INLINE              0x10UL</div><div class='add'>+	#define SQ_ATOMIC_HDR_FLAGS_WQE_TS_EN           0x20UL</div><div class='add'>+	#define SQ_ATOMIC_HDR_FLAGS_DEBUG_TRACE         0x40UL</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	remote_key;</div><div class='add'>+	__le64	remote_va;</div><div class='add'>+	__le64	swap_data;</div><div class='add'>+	__le64	cmp_data;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_localinvalidate (size:1024b/128B) */</div><div class='add'>+struct sq_localinvalidate {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define SQ_LOCALINVALIDATE_WQE_TYPE_LOCAL_INVALID 0xcUL</div><div class='add'>+	#define SQ_LOCALINVALIDATE_WQE_TYPE_LAST         SQ_LOCALINVALIDATE_WQE_TYPE_LOCAL_INVALID</div><div class='add'>+	u8	flags;</div><div class='add'>+	#define SQ_LOCALINVALIDATE_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK\</div><div class='add'>+		0xffUL</div><div class='add'>+	#define SQ_LOCALINVALIDATE_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT\</div><div class='add'>+		0</div><div class='add'>+	#define SQ_LOCALINVALIDATE_FLAGS_SIGNAL_COMP   0x1UL</div><div class='add'>+	#define SQ_LOCALINVALIDATE_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL</div><div class='add'>+	#define SQ_LOCALINVALIDATE_FLAGS_UC_FENCE 0x4UL</div><div class='add'>+	#define SQ_LOCALINVALIDATE_FLAGS_SE 0x8UL</div><div class='add'>+	#define SQ_LOCALINVALIDATE_FLAGS_INLINE 0x10UL</div><div class='add'>+	#define SQ_LOCALINVALIDATE_FLAGS_WQE_TS_EN 0x20UL</div><div class='add'>+	#define SQ_LOCALINVALIDATE_FLAGS_DEBUG_TRACE 0x40UL</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	inv_l_key;</div><div class='add'>+	__le64	reserved64;</div><div class='add'>+	u8	reserved128[16];</div><div class='add'>+	__le32	data[24];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_localinvalidate_hdr (size:256b/32B) */</div><div class='add'>+struct sq_localinvalidate_hdr {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define SQ_LOCALINVALIDATE_HDR_WQE_TYPE_LOCAL_INVALID 0xcUL</div><div class='add'>+	#define SQ_LOCALINVALIDATE_HDR_WQE_TYPE_LAST SQ_LOCALINVALIDATE_HDR_WQE_TYPE_LOCAL_INVALID</div><div class='add'>+	u8	flags;</div><div class='add'>+	#define \</div><div class='add'>+	SQ_LOCALINVALIDATE_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL</div><div class='add'>+	#define SQ_LOCALINVALIDATE_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT\</div><div class='add'>+		0</div><div class='add'>+	#define SQ_LOCALINVALIDATE_HDR_FLAGS_SIGNAL_COMP 0x1UL</div><div class='add'>+	#define SQ_LOCALINVALIDATE_HDR_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL</div><div class='add'>+	#define SQ_LOCALINVALIDATE_HDR_FLAGS_UC_FENCE 0x4UL</div><div class='add'>+	#define SQ_LOCALINVALIDATE_HDR_FLAGS_SE 0x8UL</div><div class='add'>+	#define SQ_LOCALINVALIDATE_HDR_FLAGS_INLINE 0x10UL</div><div class='add'>+	#define SQ_LOCALINVALIDATE_HDR_FLAGS_WQE_TS_EN  0x20UL</div><div class='add'>+	#define SQ_LOCALINVALIDATE_HDR_FLAGS_DEBUG_TRACE 0x40UL</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	inv_l_key;</div><div class='add'>+	__le64	reserved64;</div><div class='add'>+	u8	reserved128[16];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_fr_pmr (size:1024b/128B) */</div><div class='add'>+struct sq_fr_pmr {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define SQ_FR_PMR_WQE_TYPE_FR_PMR 0xdUL</div><div class='add'>+	#define SQ_FR_PMR_WQE_TYPE_LAST  SQ_FR_PMR_WQE_TYPE_FR_PMR</div><div class='add'>+	u8	flags;</div><div class='add'>+	#define SQ_FR_PMR_FLAGS_SIGNAL_COMP            0x1UL</div><div class='add'>+	#define SQ_FR_PMR_FLAGS_RD_OR_ATOMIC_FENCE     0x2UL</div><div class='add'>+	#define SQ_FR_PMR_FLAGS_UC_FENCE               0x4UL</div><div class='add'>+	#define SQ_FR_PMR_FLAGS_SE                     0x8UL</div><div class='add'>+	#define SQ_FR_PMR_FLAGS_INLINE                 0x10UL</div><div class='add'>+	#define SQ_FR_PMR_FLAGS_WQE_TS_EN              0x20UL</div><div class='add'>+	#define SQ_FR_PMR_FLAGS_DEBUG_TRACE            0x40UL</div><div class='add'>+	u8	access_cntl;</div><div class='add'>+	#define SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE       0x1UL</div><div class='add'>+	#define SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ       0x2UL</div><div class='add'>+	#define SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE      0x4UL</div><div class='add'>+	#define SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC     0x8UL</div><div class='add'>+	#define SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND       0x10UL</div><div class='add'>+	u8	zero_based_page_size_log;</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_MASK     0x1fUL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_SFT      0</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4K    0x0UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8K    0x1UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_16K   0x2UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_32K   0x3UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64K   0x4UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_128K  0x5UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256K  0x6UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_512K  0x7UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1M    0x8UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2M    0x9UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4M    0xaUL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8M    0xbUL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_16M   0xcUL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_32M   0xdUL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64M   0xeUL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_128M  0xfUL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256M  0x10UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_512M  0x11UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1G    0x12UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2G    0x13UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4G    0x14UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8G    0x15UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_16G   0x16UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_32G   0x17UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64G   0x18UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_128G  0x19UL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256G  0x1aUL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_512G  0x1bUL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1T    0x1cUL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2T    0x1dUL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4T    0x1eUL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8T    0x1fUL</div><div class='add'>+	#define SQ_FR_PMR_PAGE_SIZE_LOG_LAST      SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8T</div><div class='add'>+	#define SQ_FR_PMR_ZERO_BASED             0x20UL</div><div class='add'>+	__le32	l_key;</div><div class='add'>+	u8	length[5];</div><div class='add'>+	u8	reserved8_1;</div><div class='add'>+	u8	reserved8_2;</div><div class='add'>+	u8	numlevels_pbl_page_size_log;</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK     0x1fUL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT      0</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4K    0x0UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8K    0x1UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_16K   0x2UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_32K   0x3UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64K   0x4UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_128K  0x5UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256K  0x6UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_512K  0x7UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1M    0x8UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2M    0x9UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4M    0xaUL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8M    0xbUL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_16M   0xcUL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_32M   0xdUL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64M   0xeUL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_128M  0xfUL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256M  0x10UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_512M  0x11UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1G    0x12UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2G    0x13UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4G    0x14UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8G    0x15UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_16G   0x16UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_32G   0x17UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64G   0x18UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_128G  0x19UL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256G  0x1aUL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_512G  0x1bUL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1T    0x1cUL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2T    0x1dUL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4T    0x1eUL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8T    0x1fUL</div><div class='add'>+	#define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_LAST      SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8T</div><div class='add'>+	#define SQ_FR_PMR_NUMLEVELS_MASK             0xc0UL</div><div class='add'>+	#define SQ_FR_PMR_NUMLEVELS_SFT              6</div><div class='add'>+	#define SQ_FR_PMR_NUMLEVELS_PHYSICAL           (0x0UL &lt;&lt; 6)</div><div class='add'>+	#define SQ_FR_PMR_NUMLEVELS_LAYER1             (0x1UL &lt;&lt; 6)</div><div class='add'>+	#define SQ_FR_PMR_NUMLEVELS_LAYER2             (0x2UL &lt;&lt; 6)</div><div class='add'>+	#define SQ_FR_PMR_NUMLEVELS_LAST              SQ_FR_PMR_NUMLEVELS_LAYER2</div><div class='add'>+	__le64	pblptr;</div><div class='add'>+	__le64	va;</div><div class='add'>+	__le32	data[24];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_fr_pmr_hdr (size:256b/32B) */</div><div class='add'>+struct sq_fr_pmr_hdr {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define SQ_FR_PMR_HDR_WQE_TYPE_FR_PMR 0xdUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_WQE_TYPE_LAST  SQ_FR_PMR_HDR_WQE_TYPE_FR_PMR</div><div class='add'>+	u8	flags;</div><div class='add'>+	#define SQ_FR_PMR_HDR_FLAGS_SIGNAL_COMP            0x1UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_FLAGS_RD_OR_ATOMIC_FENCE     0x2UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_FLAGS_UC_FENCE               0x4UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_FLAGS_SE                     0x8UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_FLAGS_INLINE                 0x10UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_FLAGS_WQE_TS_EN              0x20UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_FLAGS_DEBUG_TRACE            0x40UL</div><div class='add'>+	u8	access_cntl;</div><div class='add'>+	#define SQ_FR_PMR_HDR_ACCESS_CNTL_LOCAL_WRITE       0x1UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_ACCESS_CNTL_REMOTE_READ       0x2UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_ACCESS_CNTL_REMOTE_WRITE      0x4UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_ACCESS_CNTL_REMOTE_ATOMIC     0x8UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_ACCESS_CNTL_WINDOW_BIND       0x10UL</div><div class='add'>+	u8	zero_based_page_size_log;</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_MASK     0x1fUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_SFT      0</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_4K    0x0UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_8K    0x1UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_16K   0x2UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_32K   0x3UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_64K   0x4UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_128K  0x5UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_256K  0x6UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_512K  0x7UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_1M    0x8UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_2M    0x9UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_4M    0xaUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_8M    0xbUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_16M   0xcUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_32M   0xdUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_64M   0xeUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_128M  0xfUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_256M  0x10UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_512M  0x11UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_1G    0x12UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_2G    0x13UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_4G    0x14UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_8G    0x15UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_16G   0x16UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_32G   0x17UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_64G   0x18UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_128G  0x19UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_256G  0x1aUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_512G  0x1bUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_1T    0x1cUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_2T    0x1dUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_4T    0x1eUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_8T    0x1fUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_LAST      SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_8T</div><div class='add'>+	#define SQ_FR_PMR_HDR_ZERO_BASED             0x20UL</div><div class='add'>+	__le32	l_key;</div><div class='add'>+	u8	length[5];</div><div class='add'>+	u8	reserved8_1;</div><div class='add'>+	u8	reserved8_2;</div><div class='add'>+	u8	numlevels_pbl_page_size_log;</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_MASK     0x1fUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_SFT      0</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_4K    0x0UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_8K    0x1UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_16K   0x2UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_32K   0x3UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_64K   0x4UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_128K  0x5UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_256K  0x6UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_512K  0x7UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_1M    0x8UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_2M    0x9UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_4M    0xaUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_8M    0xbUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_16M   0xcUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_32M   0xdUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_64M   0xeUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_128M  0xfUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_256M  0x10UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_512M  0x11UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_1G    0x12UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_2G    0x13UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_4G    0x14UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_8G    0x15UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_16G   0x16UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_32G   0x17UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_64G   0x18UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_128G  0x19UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_256G  0x1aUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_512G  0x1bUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_1T    0x1cUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_2T    0x1dUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_4T    0x1eUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_8T    0x1fUL</div><div class='add'>+	#define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_LAST      SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_8T</div><div class='add'>+	#define SQ_FR_PMR_HDR_NUMLEVELS_MASK             0xc0UL</div><div class='add'>+	#define SQ_FR_PMR_HDR_NUMLEVELS_SFT              6</div><div class='add'>+	#define SQ_FR_PMR_HDR_NUMLEVELS_PHYSICAL           (0x0UL &lt;&lt; 6)</div><div class='add'>+	#define SQ_FR_PMR_HDR_NUMLEVELS_LAYER1             (0x1UL &lt;&lt; 6)</div><div class='add'>+	#define SQ_FR_PMR_HDR_NUMLEVELS_LAYER2             (0x2UL &lt;&lt; 6)</div><div class='add'>+	#define SQ_FR_PMR_HDR_NUMLEVELS_LAST              SQ_FR_PMR_HDR_NUMLEVELS_LAYER2</div><div class='add'>+	__le64	pblptr;</div><div class='add'>+	__le64	va;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_bind (size:1024b/128B) */</div><div class='add'>+struct sq_bind {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define SQ_BIND_WQE_TYPE_BIND 0xeUL</div><div class='add'>+	#define SQ_BIND_WQE_TYPE_LAST SQ_BIND_WQE_TYPE_BIND</div><div class='add'>+	u8	flags;</div><div class='add'>+	#define SQ_BIND_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL</div><div class='add'>+	#define SQ_BIND_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT  0</div><div class='add'>+	#define SQ_BIND_FLAGS_SIGNAL_COMP                                            0x1UL</div><div class='add'>+	#define SQ_BIND_FLAGS_RD_OR_ATOMIC_FENCE                                     0x2UL</div><div class='add'>+	#define SQ_BIND_FLAGS_UC_FENCE                                               0x4UL</div><div class='add'>+	#define SQ_BIND_FLAGS_SE                                                     0x8UL</div><div class='add'>+	#define SQ_BIND_FLAGS_INLINE                                                 0x10UL</div><div class='add'>+	#define SQ_BIND_FLAGS_WQE_TS_EN                                              0x20UL</div><div class='add'>+	#define SQ_BIND_FLAGS_DEBUG_TRACE                                            0x40UL</div><div class='add'>+	u8	access_cntl;</div><div class='add'>+	#define \</div><div class='add'>+	SQ_BIND_ACCESS_CNTL_WINDOW_BIND_REMOTE_ATOMIC_REMOTE_WRITE_REMOTE_READ_LOCAL_WRITE_MASK\</div><div class='add'>+		0xffUL</div><div class='add'>+	#define \</div><div class='add'>+	SQ_BIND_ACCESS_CNTL_WINDOW_BIND_REMOTE_ATOMIC_REMOTE_WRITE_REMOTE_READ_LOCAL_WRITE_SFT 0</div><div class='add'>+	#define SQ_BIND_ACCESS_CNTL_LOCAL_WRITE       0x1UL</div><div class='add'>+	#define SQ_BIND_ACCESS_CNTL_REMOTE_READ       0x2UL</div><div class='add'>+	#define SQ_BIND_ACCESS_CNTL_REMOTE_WRITE      0x4UL</div><div class='add'>+	#define SQ_BIND_ACCESS_CNTL_REMOTE_ATOMIC     0x8UL</div><div class='add'>+	#define SQ_BIND_ACCESS_CNTL_WINDOW_BIND       0x10UL</div><div class='add'>+	u8	reserved8_1;</div><div class='add'>+	u8	mw_type_zero_based;</div><div class='add'>+	#define SQ_BIND_ZERO_BASED     0x1UL</div><div class='add'>+	#define SQ_BIND_MW_TYPE        0x2UL</div><div class='add'>+	#define SQ_BIND_MW_TYPE_TYPE1    (0x0UL &lt;&lt; 1)</div><div class='add'>+	#define SQ_BIND_MW_TYPE_TYPE2    (0x1UL &lt;&lt; 1)</div><div class='add'>+	#define SQ_BIND_MW_TYPE_LAST    SQ_BIND_MW_TYPE_TYPE2</div><div class='add'>+	u8	reserved8_2;</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	parent_l_key;</div><div class='add'>+	__le32	l_key;</div><div class='add'>+	__le64	va;</div><div class='add'>+	u8	length[5];</div><div class='add'>+	u8	reserved24[3];</div><div class='add'>+	__le32	data[24];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* sq_bind_hdr (size:256b/32B) */</div><div class='add'>+struct sq_bind_hdr {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define SQ_BIND_HDR_WQE_TYPE_BIND 0xeUL</div><div class='add'>+	#define SQ_BIND_HDR_WQE_TYPE_LAST SQ_BIND_HDR_WQE_TYPE_BIND</div><div class='add'>+	u8	flags;</div><div class='add'>+	#define SQ_BIND_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL</div><div class='add'>+	#define SQ_BIND_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT  0</div><div class='add'>+	#define SQ_BIND_HDR_FLAGS_SIGNAL_COMP		0x1UL</div><div class='add'>+	#define SQ_BIND_HDR_FLAGS_RD_OR_ATOMIC_FENCE	0x2UL</div><div class='add'>+	#define SQ_BIND_HDR_FLAGS_UC_FENCE		0x4UL</div><div class='add'>+	#define SQ_BIND_HDR_FLAGS_SE                    0x8UL</div><div class='add'>+	#define SQ_BIND_HDR_FLAGS_INLINE                0x10UL</div><div class='add'>+	#define SQ_BIND_HDR_FLAGS_WQE_TS_EN             0x20UL</div><div class='add'>+	#define SQ_BIND_HDR_FLAGS_DEBUG_TRACE           0x40UL</div><div class='add'>+	u8	access_cntl;</div><div class='add'>+	#define \</div><div class='add'>+	SQ_BIND_HDR_ACCESS_CNTL_WINDOW_BIND_REMOTE_ATOMIC_REMOTE_WRITE_REMOTE_READ_LOCAL_WRITE_MASK\</div><div class='add'>+		0xffUL</div><div class='add'>+	#define \</div><div class='add'>+	SQ_BIND_HDR_ACCESS_CNTL_WINDOW_BIND_REMOTE_ATOMIC_REMOTE_WRITE_REMOTE_READ_LOCAL_WRITE_SFT \</div><div class='add'>+		0</div><div class='add'>+	#define SQ_BIND_HDR_ACCESS_CNTL_LOCAL_WRITE	0x1UL</div><div class='add'>+	#define SQ_BIND_HDR_ACCESS_CNTL_REMOTE_READ	0x2UL</div><div class='add'>+	#define SQ_BIND_HDR_ACCESS_CNTL_REMOTE_WRITE    0x4UL</div><div class='add'>+	#define SQ_BIND_HDR_ACCESS_CNTL_REMOTE_ATOMIC   0x8UL</div><div class='add'>+	#define SQ_BIND_HDR_ACCESS_CNTL_WINDOW_BIND     0x10UL</div><div class='add'>+	u8	reserved8_1;</div><div class='add'>+	u8	mw_type_zero_based;</div><div class='add'>+	#define SQ_BIND_HDR_ZERO_BASED     0x1UL</div><div class='add'>+	#define SQ_BIND_HDR_MW_TYPE        0x2UL</div><div class='add'>+	#define SQ_BIND_HDR_MW_TYPE_TYPE1    (0x0UL &lt;&lt; 1)</div><div class='add'>+	#define SQ_BIND_HDR_MW_TYPE_TYPE2    (0x1UL &lt;&lt; 1)</div><div class='add'>+	#define SQ_BIND_HDR_MW_TYPE_LAST    SQ_BIND_HDR_MW_TYPE_TYPE2</div><div class='add'>+	u8	reserved8_2;</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	parent_l_key;</div><div class='add'>+	__le32	l_key;</div><div class='add'>+	__le64	va;</div><div class='add'>+	u8	length[5];</div><div class='add'>+	u8	reserved24[3];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* rq_wqe (size:1024b/128B) */</div><div class='add'>+struct rq_wqe {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define RQ_WQE_WQE_TYPE_RCV 0x80UL</div><div class='add'>+	#define RQ_WQE_WQE_TYPE_LAST RQ_WQE_WQE_TYPE_RCV</div><div class='add'>+	u8	flags;</div><div class='add'>+	u8	wqe_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+	__le32	wr_id[2];</div><div class='add'>+	#define RQ_WQE_WR_ID_MASK 0xfffffUL</div><div class='add'>+	#define RQ_WQE_WR_ID_SFT 0</div><div class='add'>+	u8	reserved128[16];</div><div class='add'>+	__le32	data[24];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* rq_wqe_hdr (size:256b/32B) */</div><div class='add'>+struct rq_wqe_hdr {</div><div class='add'>+	u8	wqe_type;</div><div class='add'>+	#define RQ_WQE_HDR_WQE_TYPE_RCV 0x80UL</div><div class='add'>+	#define RQ_WQE_HDR_WQE_TYPE_LAST RQ_WQE_HDR_WQE_TYPE_RCV</div><div class='add'>+	u8	flags;</div><div class='add'>+	u8	wqe_size;</div><div class='add'>+	u8	reserved8;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+	__le32	wr_id[2];</div><div class='add'>+	#define RQ_WQE_HDR_WR_ID_MASK 0xfffffUL</div><div class='add'>+	#define RQ_WQE_HDR_WR_ID_SFT 0</div><div class='add'>+	u8	reserved128[16];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cq_base (size:256b/32B) */</div><div class='add'>+struct cq_base {</div><div class='add'>+	__le64	reserved64_1;</div><div class='add'>+	__le64	reserved64_2;</div><div class='add'>+	__le64	reserved64_3;</div><div class='add'>+	u8	cqe_type_toggle;</div><div class='add'>+	#define CQ_BASE_TOGGLE                 0x1UL</div><div class='add'>+	#define CQ_BASE_CQE_TYPE_MASK          0x1eUL</div><div class='add'>+	#define CQ_BASE_CQE_TYPE_SFT           1</div><div class='add'>+	#define CQ_BASE_CQE_TYPE_REQ             (0x0UL &lt;&lt; 1)</div><div class='add'>+	#define CQ_BASE_CQE_TYPE_RES_RC          (0x1UL &lt;&lt; 1)</div><div class='add'>+	#define CQ_BASE_CQE_TYPE_RES_UD          (0x2UL &lt;&lt; 1)</div><div class='add'>+	#define CQ_BASE_CQE_TYPE_RES_RAWETH_QP1  (0x3UL &lt;&lt; 1)</div><div class='add'>+	#define CQ_BASE_CQE_TYPE_RES_UD_CFA      (0x4UL &lt;&lt; 1)</div><div class='add'>+	#define CQ_BASE_CQE_TYPE_REQ_V3             (0x8UL &lt;&lt; 1)</div><div class='add'>+	#define CQ_BASE_CQE_TYPE_RES_RC_V3          (0x9UL &lt;&lt; 1)</div><div class='add'>+	#define CQ_BASE_CQE_TYPE_RES_UD_V3          (0xaUL &lt;&lt; 1)</div><div class='add'>+	#define CQ_BASE_CQE_TYPE_RES_RAWETH_QP1_V3  (0xbUL &lt;&lt; 1)</div><div class='add'>+	#define CQ_BASE_CQE_TYPE_RES_UD_CFA_V3      (0xcUL &lt;&lt; 1)</div><div class='add'>+	#define CQ_BASE_CQE_TYPE_NO_OP           (0xdUL &lt;&lt; 1)</div><div class='add'>+	#define CQ_BASE_CQE_TYPE_TERMINAL        (0xeUL &lt;&lt; 1)</div><div class='add'>+	#define CQ_BASE_CQE_TYPE_CUT_OFF         (0xfUL &lt;&lt; 1)</div><div class='add'>+	#define CQ_BASE_CQE_TYPE_LAST           CQ_BASE_CQE_TYPE_CUT_OFF</div><div class='add'>+	u8	status;</div><div class='add'>+	#define CQ_BASE_STATUS_OK                         0x0UL</div><div class='add'>+	#define CQ_BASE_STATUS_BAD_RESPONSE_ERR           0x1UL</div><div class='add'>+	#define CQ_BASE_STATUS_LOCAL_LENGTH_ERR           0x2UL</div><div class='add'>+	#define CQ_BASE_STATUS_HW_LOCAL_LENGTH_ERR        0x3UL</div><div class='add'>+	#define CQ_BASE_STATUS_LOCAL_QP_OPERATION_ERR     0x4UL</div><div class='add'>+	#define CQ_BASE_STATUS_LOCAL_PROTECTION_ERR       0x5UL</div><div class='add'>+	#define CQ_BASE_STATUS_LOCAL_ACCESS_ERROR         0x6UL</div><div class='add'>+	#define CQ_BASE_STATUS_MEMORY_MGT_OPERATION_ERR   0x7UL</div><div class='add'>+	#define CQ_BASE_STATUS_REMOTE_INVALID_REQUEST_ERR 0x8UL</div><div class='add'>+	#define CQ_BASE_STATUS_REMOTE_ACCESS_ERR          0x9UL</div><div class='add'>+	#define CQ_BASE_STATUS_REMOTE_OPERATION_ERR       0xaUL</div><div class='add'>+	#define CQ_BASE_STATUS_RNR_NAK_RETRY_CNT_ERR      0xbUL</div><div class='add'>+	#define CQ_BASE_STATUS_TRANSPORT_RETRY_CNT_ERR    0xcUL</div><div class='add'>+	#define CQ_BASE_STATUS_WORK_REQUEST_FLUSHED_ERR   0xdUL</div><div class='add'>+	#define CQ_BASE_STATUS_HW_FLUSH_ERR               0xeUL</div><div class='add'>+	#define CQ_BASE_STATUS_OVERFLOW_ERR               0xfUL</div><div class='add'>+	#define CQ_BASE_STATUS_LAST                      CQ_BASE_STATUS_OVERFLOW_ERR</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	opaque;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cq_req (size:256b/32B) */</div><div class='add'>+struct cq_req {</div><div class='add'>+	__le64	qp_handle;</div><div class='add'>+	__le16	sq_cons_idx;</div><div class='add'>+	__le16	reserved16_1;</div><div class='add'>+	__le32	reserved32_2;</div><div class='add'>+	__le64	reserved64;</div><div class='add'>+	u8	cqe_type_toggle;</div><div class='add'>+	#define CQ_REQ_TOGGLE       0x1UL</div><div class='add'>+	#define CQ_REQ_CQE_TYPE_MASK 0x1eUL</div><div class='add'>+	#define CQ_REQ_CQE_TYPE_SFT 1</div><div class='add'>+	#define CQ_REQ_CQE_TYPE_REQ   (0x0UL &lt;&lt; 1)</div><div class='add'>+	#define CQ_REQ_CQE_TYPE_LAST CQ_REQ_CQE_TYPE_REQ</div><div class='add'>+	#define CQ_REQ_PUSH         0x20UL</div><div class='add'>+	u8	status;</div><div class='add'>+	#define CQ_REQ_STATUS_OK                         0x0UL</div><div class='add'>+	#define CQ_REQ_STATUS_BAD_RESPONSE_ERR           0x1UL</div><div class='add'>+	#define CQ_REQ_STATUS_LOCAL_LENGTH_ERR           0x2UL</div><div class='add'>+	#define CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR     0x3UL</div><div class='add'>+	#define CQ_REQ_STATUS_LOCAL_PROTECTION_ERR       0x4UL</div><div class='add'>+	#define CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR   0x5UL</div><div class='add'>+	#define CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR 0x6UL</div><div class='add'>+	#define CQ_REQ_STATUS_REMOTE_ACCESS_ERR          0x7UL</div><div class='add'>+	#define CQ_REQ_STATUS_REMOTE_OPERATION_ERR       0x8UL</div><div class='add'>+	#define CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR      0x9UL</div><div class='add'>+	#define CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR    0xaUL</div><div class='add'>+	#define CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR   0xbUL</div><div class='add'>+	#define CQ_REQ_STATUS_LAST                      CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR</div><div class='add'>+	__le16	reserved16_2;</div><div class='add'>+	__le32	reserved32_1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cq_res_rc (size:256b/32B) */</div><div class='add'>+struct cq_res_rc {</div><div class='add'>+	__le32	length;</div><div class='add'>+	__le32	imm_data_or_inv_r_key;</div><div class='add'>+	__le64	qp_handle;</div><div class='add'>+	__le64	mr_handle;</div><div class='add'>+	u8	cqe_type_toggle;</div><div class='add'>+	#define CQ_RES_RC_TOGGLE         0x1UL</div><div class='add'>+	#define CQ_RES_RC_CQE_TYPE_MASK  0x1eUL</div><div class='add'>+	#define CQ_RES_RC_CQE_TYPE_SFT   1</div><div class='add'>+	#define CQ_RES_RC_CQE_TYPE_RES_RC  (0x1UL &lt;&lt; 1)</div><div class='add'>+	#define CQ_RES_RC_CQE_TYPE_LAST   CQ_RES_RC_CQE_TYPE_RES_RC</div><div class='add'>+	u8	status;</div><div class='add'>+	#define CQ_RES_RC_STATUS_OK                         0x0UL</div><div class='add'>+	#define CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR         0x1UL</div><div class='add'>+	#define CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR           0x2UL</div><div class='add'>+	#define CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR       0x3UL</div><div class='add'>+	#define CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR     0x4UL</div><div class='add'>+	#define CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR   0x5UL</div><div class='add'>+	#define CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR 0x6UL</div><div class='add'>+	#define CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR   0x7UL</div><div class='add'>+	#define CQ_RES_RC_STATUS_HW_FLUSH_ERR               0x8UL</div><div class='add'>+	#define CQ_RES_RC_STATUS_LAST                      CQ_RES_RC_STATUS_HW_FLUSH_ERR</div><div class='add'>+	__le16	flags;</div><div class='add'>+	#define CQ_RES_RC_FLAGS_SRQ            0x1UL</div><div class='add'>+	#define CQ_RES_RC_FLAGS_SRQ_RQ           0x0UL</div><div class='add'>+	#define CQ_RES_RC_FLAGS_SRQ_SRQ          0x1UL</div><div class='add'>+	#define CQ_RES_RC_FLAGS_SRQ_LAST        CQ_RES_RC_FLAGS_SRQ_SRQ</div><div class='add'>+	#define CQ_RES_RC_FLAGS_IMM            0x2UL</div><div class='add'>+	#define CQ_RES_RC_FLAGS_INV            0x4UL</div><div class='add'>+	#define CQ_RES_RC_FLAGS_RDMA           0x8UL</div><div class='add'>+	#define CQ_RES_RC_FLAGS_RDMA_SEND        (0x0UL &lt;&lt; 3)</div><div class='add'>+	#define CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE  (0x1UL &lt;&lt; 3)</div><div class='add'>+	#define CQ_RES_RC_FLAGS_RDMA_LAST       CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE</div><div class='add'>+	__le32	srq_or_rq_wr_id;</div><div class='add'>+	#define CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL</div><div class='add'>+	#define CQ_RES_RC_SRQ_OR_RQ_WR_ID_SFT 0</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cq_res_ud (size:256b/32B) */</div><div class='add'>+struct cq_res_ud {</div><div class='add'>+	__le16	length;</div><div class='add'>+	#define CQ_RES_UD_LENGTH_MASK 0x3fffUL</div><div class='add'>+	#define CQ_RES_UD_LENGTH_SFT 0</div><div class='add'>+	__le16	cfa_metadata;</div><div class='add'>+	#define CQ_RES_UD_CFA_METADATA_VID_MASK 0xfffUL</div><div class='add'>+	#define CQ_RES_UD_CFA_METADATA_VID_SFT 0</div><div class='add'>+	#define CQ_RES_UD_CFA_METADATA_DE      0x1000UL</div><div class='add'>+	#define CQ_RES_UD_CFA_METADATA_PRI_MASK 0xe000UL</div><div class='add'>+	#define CQ_RES_UD_CFA_METADATA_PRI_SFT 13</div><div class='add'>+	__le32	imm_data;</div><div class='add'>+	__le64	qp_handle;</div><div class='add'>+	__le16	src_mac[3];</div><div class='add'>+	__le16	src_qp_low;</div><div class='add'>+	u8	cqe_type_toggle;</div><div class='add'>+	#define CQ_RES_UD_TOGGLE         0x1UL</div><div class='add'>+	#define CQ_RES_UD_CQE_TYPE_MASK  0x1eUL</div><div class='add'>+	#define CQ_RES_UD_CQE_TYPE_SFT   1</div><div class='add'>+	#define CQ_RES_UD_CQE_TYPE_RES_UD  (0x2UL &lt;&lt; 1)</div><div class='add'>+	#define CQ_RES_UD_CQE_TYPE_LAST   CQ_RES_UD_CQE_TYPE_RES_UD</div><div class='add'>+	u8	status;</div><div class='add'>+	#define CQ_RES_UD_STATUS_OK                       0x0UL</div><div class='add'>+	#define CQ_RES_UD_STATUS_LOCAL_ACCESS_ERROR       0x1UL</div><div class='add'>+	#define CQ_RES_UD_STATUS_HW_LOCAL_LENGTH_ERR      0x2UL</div><div class='add'>+	#define CQ_RES_UD_STATUS_LOCAL_PROTECTION_ERR     0x3UL</div><div class='add'>+	#define CQ_RES_UD_STATUS_LOCAL_QP_OPERATION_ERR   0x4UL</div><div class='add'>+	#define CQ_RES_UD_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL</div><div class='add'>+	#define CQ_RES_UD_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL</div><div class='add'>+	#define CQ_RES_UD_STATUS_HW_FLUSH_ERR             0x8UL</div><div class='add'>+	#define CQ_RES_UD_STATUS_LAST                    CQ_RES_UD_STATUS_HW_FLUSH_ERR</div><div class='add'>+	__le16	flags;</div><div class='add'>+	#define CQ_RES_UD_FLAGS_SRQ                   0x1UL</div><div class='add'>+	#define CQ_RES_UD_FLAGS_SRQ_RQ                  0x0UL</div><div class='add'>+	#define CQ_RES_UD_FLAGS_SRQ_SRQ                 0x1UL</div><div class='add'>+	#define CQ_RES_UD_FLAGS_SRQ_LAST               CQ_RES_UD_FLAGS_SRQ_SRQ</div><div class='add'>+	#define CQ_RES_UD_FLAGS_IMM                   0x2UL</div><div class='add'>+	#define CQ_RES_UD_FLAGS_UNUSED_MASK           0xcUL</div><div class='add'>+	#define CQ_RES_UD_FLAGS_UNUSED_SFT            2</div><div class='add'>+	#define CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK      0x30UL</div><div class='add'>+	#define CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT       4</div><div class='add'>+	#define CQ_RES_UD_FLAGS_ROCE_IP_VER_V1          (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV4      (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6      (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_UD_FLAGS_ROCE_IP_VER_LAST       CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6</div><div class='add'>+	#define CQ_RES_UD_FLAGS_META_FORMAT_MASK      0x3c0UL</div><div class='add'>+	#define CQ_RES_UD_FLAGS_META_FORMAT_SFT       6</div><div class='add'>+	#define CQ_RES_UD_FLAGS_META_FORMAT_NONE        (0x0UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_FLAGS_META_FORMAT_VLAN        (0x1UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_FLAGS_META_FORMAT_TUNNEL_ID   (0x2UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_FLAGS_META_FORMAT_CHDR_DATA   (0x3UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_FLAGS_META_FORMAT_HDR_OFFSET  (0x4UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_FLAGS_META_FORMAT_LAST       CQ_RES_UD_FLAGS_META_FORMAT_HDR_OFFSET</div><div class='add'>+	#define CQ_RES_UD_FLAGS_EXT_META_FORMAT_MASK  0xc00UL</div><div class='add'>+	#define CQ_RES_UD_FLAGS_EXT_META_FORMAT_SFT   10</div><div class='add'>+	__le32	src_qp_high_srq_or_rq_wr_id;</div><div class='add'>+	#define CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL</div><div class='add'>+	#define CQ_RES_UD_SRQ_OR_RQ_WR_ID_SFT 0</div><div class='add'>+	#define CQ_RES_UD_SRC_QP_HIGH_MASK    0xff000000UL</div><div class='add'>+	#define CQ_RES_UD_SRC_QP_HIGH_SFT     24</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cq_res_ud_v2 (size:256b/32B) */</div><div class='add'>+struct cq_res_ud_v2 {</div><div class='add'>+	__le16	length;</div><div class='add'>+	#define CQ_RES_UD_V2_LENGTH_MASK 0x3fffUL</div><div class='add'>+	#define CQ_RES_UD_V2_LENGTH_SFT 0</div><div class='add'>+	__le16	cfa_metadata0;</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA0_VID_MASK 0xfffUL</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA0_VID_SFT 0</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA0_DE      0x1000UL</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA0_PRI_MASK 0xe000UL</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA0_PRI_SFT 13</div><div class='add'>+	__le32	imm_data;</div><div class='add'>+	__le64	qp_handle;</div><div class='add'>+	__le16	src_mac[3];</div><div class='add'>+	__le16	src_qp_low;</div><div class='add'>+	u8	cqe_type_toggle;</div><div class='add'>+	#define CQ_RES_UD_V2_TOGGLE         0x1UL</div><div class='add'>+	#define CQ_RES_UD_V2_CQE_TYPE_MASK  0x1eUL</div><div class='add'>+	#define CQ_RES_UD_V2_CQE_TYPE_SFT   1</div><div class='add'>+	#define CQ_RES_UD_V2_CQE_TYPE_RES_UD  (0x2UL &lt;&lt; 1)</div><div class='add'>+	#define CQ_RES_UD_V2_CQE_TYPE_LAST   CQ_RES_UD_V2_CQE_TYPE_RES_UD</div><div class='add'>+	u8	status;</div><div class='add'>+	#define CQ_RES_UD_V2_STATUS_OK                       0x0UL</div><div class='add'>+	#define CQ_RES_UD_V2_STATUS_LOCAL_ACCESS_ERROR       0x1UL</div><div class='add'>+	#define CQ_RES_UD_V2_STATUS_HW_LOCAL_LENGTH_ERR      0x2UL</div><div class='add'>+	#define CQ_RES_UD_V2_STATUS_LOCAL_PROTECTION_ERR     0x3UL</div><div class='add'>+	#define CQ_RES_UD_V2_STATUS_LOCAL_QP_OPERATION_ERR   0x4UL</div><div class='add'>+	#define CQ_RES_UD_V2_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL</div><div class='add'>+	#define CQ_RES_UD_V2_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL</div><div class='add'>+	#define CQ_RES_UD_V2_STATUS_HW_FLUSH_ERR             0x8UL</div><div class='add'>+	#define CQ_RES_UD_V2_STATUS_LAST                    CQ_RES_UD_V2_STATUS_HW_FLUSH_ERR</div><div class='add'>+	__le16	flags;</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_SRQ                    0x1UL</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_SRQ_RQ                   0x0UL</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_SRQ_SRQ                  0x1UL</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_SRQ_LAST                CQ_RES_UD_V2_FLAGS_SRQ_SRQ</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_IMM                    0x2UL</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_UNUSED_MASK            0xcUL</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_UNUSED_SFT             2</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_MASK       0x30UL</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_SFT        4</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_V1           (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_V2IPV4       (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_V2IPV6       (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_LAST        CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_V2IPV6</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_META_FORMAT_MASK       0x3c0UL</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_META_FORMAT_SFT        6</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_META_FORMAT_NONE         (0x0UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_META_FORMAT_ACT_REC_PTR  (0x1UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_META_FORMAT_TUNNEL_ID    (0x2UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_META_FORMAT_CHDR_DATA    (0x3UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_META_FORMAT_HDR_OFFSET   (0x4UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_V2_FLAGS_META_FORMAT_LAST        CQ_RES_UD_V2_FLAGS_META_FORMAT_HDR_OFFSET</div><div class='add'>+	__le32	src_qp_high_srq_or_rq_wr_id;</div><div class='add'>+	#define CQ_RES_UD_V2_SRQ_OR_RQ_WR_ID_MASK           0xfffffUL</div><div class='add'>+	#define CQ_RES_UD_V2_SRQ_OR_RQ_WR_ID_SFT            0</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA1_MASK             0xf00000UL</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA1_SFT              20</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_MASK     0x700000UL</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_SFT      20</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPID88A8   (0x0UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPID8100   (0x1UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPID9100   (0x2UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPID9200   (0x3UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPID9300   (0x4UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPIDCFG    (0x5UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_LAST CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPIDCFG</div><div class='add'>+	#define CQ_RES_UD_V2_CFA_METADATA1_VALID             0x800000UL</div><div class='add'>+	#define CQ_RES_UD_V2_SRC_QP_HIGH_MASK               0xff000000UL</div><div class='add'>+	#define CQ_RES_UD_V2_SRC_QP_HIGH_SFT                24</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cq_res_ud_cfa (size:256b/32B) */</div><div class='add'>+struct cq_res_ud_cfa {</div><div class='add'>+	__le16	length;</div><div class='add'>+	#define CQ_RES_UD_CFA_LENGTH_MASK 0x3fffUL</div><div class='add'>+	#define CQ_RES_UD_CFA_LENGTH_SFT 0</div><div class='add'>+	__le16	cfa_code;</div><div class='add'>+	__le32	imm_data;</div><div class='add'>+	__le32	qid;</div><div class='add'>+	#define CQ_RES_UD_CFA_QID_MASK 0xfffffUL</div><div class='add'>+	#define CQ_RES_UD_CFA_QID_SFT 0</div><div class='add'>+	__le32	cfa_metadata;</div><div class='add'>+	#define CQ_RES_UD_CFA_CFA_METADATA_VID_MASK 0xfffUL</div><div class='add'>+	#define CQ_RES_UD_CFA_CFA_METADATA_VID_SFT  0</div><div class='add'>+	#define CQ_RES_UD_CFA_CFA_METADATA_DE       0x1000UL</div><div class='add'>+	#define CQ_RES_UD_CFA_CFA_METADATA_PRI_MASK 0xe000UL</div><div class='add'>+	#define CQ_RES_UD_CFA_CFA_METADATA_PRI_SFT  13</div><div class='add'>+	#define CQ_RES_UD_CFA_CFA_METADATA_TPID_MASK 0xffff0000UL</div><div class='add'>+	#define CQ_RES_UD_CFA_CFA_METADATA_TPID_SFT 16</div><div class='add'>+	__le16	src_mac[3];</div><div class='add'>+	__le16	src_qp_low;</div><div class='add'>+	u8	cqe_type_toggle;</div><div class='add'>+	#define CQ_RES_UD_CFA_TOGGLE             0x1UL</div><div class='add'>+	#define CQ_RES_UD_CFA_CQE_TYPE_MASK      0x1eUL</div><div class='add'>+	#define CQ_RES_UD_CFA_CQE_TYPE_SFT       1</div><div class='add'>+	#define CQ_RES_UD_CFA_CQE_TYPE_RES_UD_CFA  (0x4UL &lt;&lt; 1)</div><div class='add'>+	#define CQ_RES_UD_CFA_CQE_TYPE_LAST       CQ_RES_UD_CFA_CQE_TYPE_RES_UD_CFA</div><div class='add'>+	u8	status;</div><div class='add'>+	#define CQ_RES_UD_CFA_STATUS_OK                       0x0UL</div><div class='add'>+	#define CQ_RES_UD_CFA_STATUS_LOCAL_ACCESS_ERROR       0x1UL</div><div class='add'>+	#define CQ_RES_UD_CFA_STATUS_HW_LOCAL_LENGTH_ERR      0x2UL</div><div class='add'>+	#define CQ_RES_UD_CFA_STATUS_LOCAL_PROTECTION_ERR     0x3UL</div><div class='add'>+	#define CQ_RES_UD_CFA_STATUS_LOCAL_QP_OPERATION_ERR   0x4UL</div><div class='add'>+	#define CQ_RES_UD_CFA_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL</div><div class='add'>+	#define CQ_RES_UD_CFA_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL</div><div class='add'>+	#define CQ_RES_UD_CFA_STATUS_HW_FLUSH_ERR             0x8UL</div><div class='add'>+	#define CQ_RES_UD_CFA_STATUS_LAST                    CQ_RES_UD_CFA_STATUS_HW_FLUSH_ERR</div><div class='add'>+	__le16	flags;</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_SRQ                   0x1UL</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_SRQ_RQ                  0x0UL</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_SRQ_SRQ                 0x1UL</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_SRQ_LAST               CQ_RES_UD_CFA_FLAGS_SRQ_SRQ</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_IMM                   0x2UL</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_UNUSED_MASK           0xcUL</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_UNUSED_SFT            2</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_MASK      0x30UL</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_SFT       4</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_V1          (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_V2IPV4      (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_V2IPV6      (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_LAST       CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_V2IPV6</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_META_FORMAT_MASK      0x3c0UL</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_META_FORMAT_SFT       6</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_META_FORMAT_NONE        (0x0UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_META_FORMAT_VLAN        (0x1UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_META_FORMAT_TUNNEL_ID   (0x2UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_META_FORMAT_CHDR_DATA   (0x3UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_META_FORMAT_HDR_OFFSET  (0x4UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_META_FORMAT_LAST	CQ_RES_UD_CFA_FLAGS_META_FORMAT_HDR_OFFSET</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_EXT_META_FORMAT_MASK  0xc00UL</div><div class='add'>+	#define CQ_RES_UD_CFA_FLAGS_EXT_META_FORMAT_SFT   10</div><div class='add'>+	__le32	src_qp_high_srq_or_rq_wr_id;</div><div class='add'>+	#define CQ_RES_UD_CFA_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL</div><div class='add'>+	#define CQ_RES_UD_CFA_SRQ_OR_RQ_WR_ID_SFT 0</div><div class='add'>+	#define CQ_RES_UD_CFA_SRC_QP_HIGH_MASK    0xff000000UL</div><div class='add'>+	#define CQ_RES_UD_CFA_SRC_QP_HIGH_SFT     24</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cq_res_ud_cfa_v2 (size:256b/32B) */</div><div class='add'>+struct cq_res_ud_cfa_v2 {</div><div class='add'>+	__le16	length;</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_LENGTH_MASK 0x3fffUL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_LENGTH_SFT 0</div><div class='add'>+	__le16	cfa_metadata0;</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA0_VID_MASK 0xfffUL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA0_VID_SFT 0</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA0_DE      0x1000UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA0_PRI_MASK 0xe000UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA0_PRI_SFT 13</div><div class='add'>+	__le32	imm_data;</div><div class='add'>+	__le32	qid;</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_QID_MASK 0xfffffUL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_QID_SFT 0</div><div class='add'>+	__le32	cfa_metadata2;</div><div class='add'>+	__le16	src_mac[3];</div><div class='add'>+	__le16	src_qp_low;</div><div class='add'>+	u8	cqe_type_toggle;</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_TOGGLE             0x1UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CQE_TYPE_MASK      0x1eUL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CQE_TYPE_SFT       1</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CQE_TYPE_RES_UD_CFA  (0x4UL &lt;&lt; 1)</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CQE_TYPE_LAST       CQ_RES_UD_CFA_V2_CQE_TYPE_RES_UD_CFA</div><div class='add'>+	u8	status;</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_STATUS_OK                       0x0UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_STATUS_LOCAL_ACCESS_ERROR       0x1UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_STATUS_HW_LOCAL_LENGTH_ERR      0x2UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_STATUS_LOCAL_PROTECTION_ERR     0x3UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_STATUS_LOCAL_QP_OPERATION_ERR   0x4UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_STATUS_HW_FLUSH_ERR             0x8UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_STATUS_LAST   CQ_RES_UD_CFA_V2_STATUS_HW_FLUSH_ERR</div><div class='add'>+	__le16	flags;</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_SRQ                    0x1UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_SRQ_RQ                   0x0UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_SRQ_SRQ                  0x1UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_SRQ_LAST                CQ_RES_UD_CFA_V2_FLAGS_SRQ_SRQ</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_IMM                    0x2UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_UNUSED_MASK            0xcUL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_UNUSED_SFT             2</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_MASK       0x30UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_SFT        4</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_V1           (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_V2IPV4       (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_V2IPV6       (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_LAST  CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_V2IPV6</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_MASK       0x3c0UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_SFT        6</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_NONE         (0x0UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_ACT_REC_PTR  (0x1UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_TUNNEL_ID    (0x2UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_CHDR_DATA    (0x3UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_HDR_OFFSET   (0x4UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_LAST \</div><div class='add'>+		CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_HDR_OFFSET</div><div class='add'>+	__le32	src_qp_high_srq_or_rq_wr_id;</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_SRQ_OR_RQ_WR_ID_MASK           0xfffffUL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_SRQ_OR_RQ_WR_ID_SFT            0</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA1_MASK             0xf00000UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA1_SFT              20</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_MASK     0x700000UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_SFT      20</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPID88A8   (0x0UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPID8100   (0x1UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPID9100   (0x2UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPID9200   (0x3UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPID9300   (0x4UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPIDCFG    (0x5UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_LAST \</div><div class='add'>+		CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPIDCFG</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_CFA_METADATA1_VALID             0x800000UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_SRC_QP_HIGH_MASK               0xff000000UL</div><div class='add'>+	#define CQ_RES_UD_CFA_V2_SRC_QP_HIGH_SFT                24</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cq_res_raweth_qp1 (size:256b/32B) */</div><div class='add'>+struct cq_res_raweth_qp1 {</div><div class='add'>+	__le16	length;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_LENGTH_MASK 0x3fffUL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_LENGTH_SFT 0</div><div class='add'>+	__le16	raweth_qp1_flags;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_MASK                  0x3ffUL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_SFT                   0</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ERROR                  0x1UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_MASK             0x3c0UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_SFT              6</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_NOT_KNOWN          (0x0UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_IP                 (0x1UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_TCP                (0x2UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_UDP                (0x3UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_FCOE               (0x4UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE               (0x5UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ICMP               (0x7UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_WO_TIMESTAMP   (0x8UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP    (0x9UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_LAST \</div><div class='add'>+		CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP</div><div class='add'>+	__le16	raweth_qp1_errors;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_IP_CS_ERROR   0x10UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_L4_CS_ERROR   0x20UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_IP_CS_ERROR 0x40UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_L4_CS_ERROR 0x80UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_CRC_ERROR     0x100UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_MASK  0xe00UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_SFT   9</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_NO_ERROR                (0x0UL &lt;&lt; 9)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION        (0x1UL &lt;&lt; 9)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN        (0x2UL &lt;&lt; 9)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR      (0x3UL &lt;&lt; 9)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR        (0x4UL &lt;&lt; 9)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR       (0x5UL &lt;&lt; 9)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL            (0x6UL &lt;&lt; 9)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_LAST \</div><div class='add'>+		CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_MASK                    0xf000UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_SFT                     12</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_NO_ERROR	(0x0UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_VERSION	(0x1UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN    (0x2UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_TTL        (0x3UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_IP_TOTAL_ERROR    (0x4UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR   (0x5UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN    (0x6UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL (0x7UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN (0x8UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_LAST \</div><div class='add'>+		CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN</div><div class='add'>+	__le16	raweth_qp1_cfa_code;</div><div class='add'>+	__le64	qp_handle;</div><div class='add'>+	__le32	raweth_qp1_flags2;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC                 0x1UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC                 0x2UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_IP_CS_CALC               0x4UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_L4_CS_CALC               0x8UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_MASK           0xf0UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_SFT            4</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_NONE             (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN             (0x1UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_TUNNEL_ID        (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_CHDR_DATA        (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_HDR_OFFSET       (0x4UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_LAST \</div><div class='add'>+		CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_HDR_OFFSET</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE                    0x100UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_CALC     0x200UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_EXT_META_FORMAT_MASK       0xc00UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_EXT_META_FORMAT_SFT        10</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_MASK     0xffff0000UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_SFT      16</div><div class='add'>+	__le32	raweth_qp1_metadata;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_DE_VID_MASK    0xffffUL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_DE_VID_SFT     0</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK           0xfffUL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_SFT            0</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_DE                 0x1000UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK           0xe000UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT            13</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK          0xffff0000UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT           16</div><div class='add'>+	u8	cqe_type_toggle;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_TOGGLE                 0x1UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_CQE_TYPE_MASK          0x1eUL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_CQE_TYPE_SFT           1</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_CQE_TYPE_RES_RAWETH_QP1  (0x3UL &lt;&lt; 1)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_CQE_TYPE_LAST           CQ_RES_RAWETH_QP1_CQE_TYPE_RES_RAWETH_QP1</div><div class='add'>+	u8	status;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_STATUS_OK                       0x0UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR       0x1UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR      0x2UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR     0x3UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR   0x4UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR             0x8UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_STATUS_LAST  CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR</div><div class='add'>+	__le16	flags;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_FLAGS_SRQ     0x1UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_FLAGS_SRQ_RQ    0x0UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ   0x1UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_FLAGS_SRQ_LAST CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ</div><div class='add'>+	__le32	raweth_qp1_payload_offset_srq_or_rq_wr_id;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK          0xfffffUL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_SFT           0</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_MASK 0xff000000UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_SFT 24</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cq_res_raweth_qp1_v2 (size:256b/32B) */</div><div class='add'>+struct cq_res_raweth_qp1_v2 {</div><div class='add'>+	__le16	length;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_LENGTH_MASK 0x3fffUL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_LENGTH_SFT 0</div><div class='add'>+	__le16	raweth_qp1_flags;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_MASK                  0x3ffUL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_SFT                   0</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ERROR                  0x1UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_MASK             0x3c0UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_SFT              6</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_NOT_KNOWN          (0x0UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_IP                 (0x1UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_TCP                (0x2UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_UDP                (0x3UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_FCOE               (0x4UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_ROCE               (0x5UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_ICMP               (0x7UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_PTP_WO_TIMESTAMP   (0x8UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP    (0x9UL &lt;&lt; 6)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_LAST \</div><div class='add'>+		CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP</div><div class='add'>+	__le16	raweth_qp1_errors;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_IP_CS_ERROR                       0x10UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_L4_CS_ERROR                       0x20UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_IP_CS_ERROR                     0x40UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_L4_CS_ERROR                     0x80UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_CRC_ERROR                         0x100UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_MASK                  0xe00UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_SFT                   9</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_NO_ERROR (0x0UL &lt;&lt; 9)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION (0x1UL &lt;&lt; 9)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN (0x2UL &lt;&lt; 9)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR (0x3UL &lt;&lt; 9)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR (0x4UL &lt;&lt; 9)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR (0x5UL &lt;&lt; 9)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL (0x6UL &lt;&lt; 9)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_LAST \</div><div class='add'>+		CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_MASK    0xf000UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_SFT 12</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_NO_ERROR   (0x0UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_VERSION  (0x1UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN  (0x2UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_TTL      (0x3UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_IP_TOTAL_ERROR  (0x4UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR (0x5UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN  (0x6UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL \</div><div class='add'>+		(0x7UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN \</div><div class='add'>+		(0x8UL &lt;&lt; 12)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_LAST \</div><div class='add'>+		CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN</div><div class='add'>+	__le16	cfa_metadata0;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA0_VID_MASK 0xfffUL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA0_VID_SFT 0</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA0_DE      0x1000UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA0_PRI_MASK 0xe000UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA0_PRI_SFT 13</div><div class='add'>+	__le64	qp_handle;</div><div class='add'>+	__le32	raweth_qp1_flags2;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_CS_ALL_OK_MODE             0x8UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_MASK           0xf0UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_SFT            4</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_NONE             (0x0UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_ACT_REC_PTR      (0x1UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_TUNNEL_ID        (0x2UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_CHDR_DATA        (0x3UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_HDR_OFFSET       (0x4UL &lt;&lt; 4)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_LAST \</div><div class='add'>+		CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_HDR_OFFSET</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_IP_TYPE                    0x100UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_CALC     0x200UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_CS_OK_MASK                 0xfc00UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_CS_OK_SFT                  10</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_MASK     0xffff0000UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_SFT      16</div><div class='add'>+	__le32	cfa_metadata2;</div><div class='add'>+	u8	cqe_type_toggle;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_TOGGLE                 0x1UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CQE_TYPE_MASK          0x1eUL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CQE_TYPE_SFT           1</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CQE_TYPE_RES_RAWETH_QP1  (0x3UL &lt;&lt; 1)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CQE_TYPE_LAST CQ_RES_RAWETH_QP1_V2_CQE_TYPE_RES_RAWETH_QP1</div><div class='add'>+	u8	status;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_STATUS_OK                       0x0UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_STATUS_LOCAL_ACCESS_ERROR       0x1UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_STATUS_HW_LOCAL_LENGTH_ERR      0x2UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_STATUS_LOCAL_PROTECTION_ERR     0x3UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_STATUS_LOCAL_QP_OPERATION_ERR   0x4UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_STATUS_HW_FLUSH_ERR             0x8UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_STATUS_LAST CQ_RES_RAWETH_QP1_V2_STATUS_HW_FLUSH_ERR</div><div class='add'>+	__le16	flags;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_FLAGS_SRQ     0x1UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_FLAGS_SRQ_RQ    0x0UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_FLAGS_SRQ_SRQ   0x1UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_FLAGS_SRQ_LAST CQ_RES_RAWETH_QP1_V2_FLAGS_SRQ_SRQ</div><div class='add'>+	__le32	raweth_qp1_payload_offset_srq_or_rq_wr_id;</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_SRQ_OR_RQ_WR_ID_MASK           0xfffffUL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_SRQ_OR_RQ_WR_ID_SFT            0</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_MASK             0xf00000UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_SFT              20</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_MASK     0x700000UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_SFT      20</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPID88A8   (0x0UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPID8100   (0x1UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPID9100   (0x2UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPID9200   (0x3UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPID9300   (0x4UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPIDCFG    (0x5UL &lt;&lt; 20)</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_LAST \</div><div class='add'>+		CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPIDCFG</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_VALID             0x800000UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_PAYLOAD_OFFSET_MASK 0xff000000UL</div><div class='add'>+	#define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_PAYLOAD_OFFSET_SFT  24</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cq_terminal (size:256b/32B) */</div><div class='add'>+struct cq_terminal {</div><div class='add'>+	__le64	qp_handle;</div><div class='add'>+	__le16	sq_cons_idx;</div><div class='add'>+	__le16	rq_cons_idx;</div><div class='add'>+	__le32	reserved32_1;</div><div class='add'>+	__le64	reserved64_3;</div><div class='add'>+	u8	cqe_type_toggle;</div><div class='add'>+	#define CQ_TERMINAL_TOGGLE           0x1UL</div><div class='add'>+	#define CQ_TERMINAL_CQE_TYPE_MASK    0x1eUL</div><div class='add'>+	#define CQ_TERMINAL_CQE_TYPE_SFT     1</div><div class='add'>+	#define CQ_TERMINAL_CQE_TYPE_TERMINAL  (0xeUL &lt;&lt; 1)</div><div class='add'>+	#define CQ_TERMINAL_CQE_TYPE_LAST     CQ_TERMINAL_CQE_TYPE_TERMINAL</div><div class='add'>+	u8	status;</div><div class='add'>+	#define CQ_TERMINAL_STATUS_OK 0x0UL</div><div class='add'>+	#define CQ_TERMINAL_STATUS_LAST CQ_TERMINAL_STATUS_OK</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	reserved32_2;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cq_cutoff (size:256b/32B) */</div><div class='add'>+struct cq_cutoff {</div><div class='add'>+	__le64	reserved64_1;</div><div class='add'>+	__le64	reserved64_2;</div><div class='add'>+	__le64	reserved64_3;</div><div class='add'>+	u8	cqe_type_toggle;</div><div class='add'>+	#define CQ_CUTOFF_TOGGLE          0x1UL</div><div class='add'>+	#define CQ_CUTOFF_CQE_TYPE_MASK   0x1eUL</div><div class='add'>+	#define CQ_CUTOFF_CQE_TYPE_SFT    1</div><div class='add'>+	#define CQ_CUTOFF_CQE_TYPE_CUT_OFF  (0xfUL &lt;&lt; 1)</div><div class='add'>+	#define CQ_CUTOFF_CQE_TYPE_LAST    CQ_CUTOFF_CQE_TYPE_CUT_OFF</div><div class='add'>+	#define CQ_CUTOFF_RESIZE_TOGGLE_MASK 0x60UL</div><div class='add'>+	#define CQ_CUTOFF_RESIZE_TOGGLE_SFT 5</div><div class='add'>+	u8	status;</div><div class='add'>+	#define CQ_CUTOFF_STATUS_OK 0x0UL</div><div class='add'>+	#define CQ_CUTOFF_STATUS_LAST CQ_CUTOFF_STATUS_OK</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* nq_base (size:128b/16B) */</div><div class='add'>+struct nq_base {</div><div class='add'>+	__le16	info10_type;</div><div class='add'>+	#define NQ_BASE_TYPE_MASK           0x3fUL</div><div class='add'>+	#define NQ_BASE_TYPE_SFT            0</div><div class='add'>+	#define NQ_BASE_TYPE_CQ_NOTIFICATION  0x30UL</div><div class='add'>+	#define NQ_BASE_TYPE_SRQ_EVENT        0x32UL</div><div class='add'>+	#define NQ_BASE_TYPE_DBQ_EVENT        0x34UL</div><div class='add'>+	#define NQ_BASE_TYPE_QP_EVENT         0x38UL</div><div class='add'>+	#define NQ_BASE_TYPE_FUNC_EVENT       0x3aUL</div><div class='add'>+	#define NQ_BASE_TYPE_LAST            NQ_BASE_TYPE_FUNC_EVENT</div><div class='add'>+	#define NQ_BASE_INFO10_MASK         0xffc0UL</div><div class='add'>+	#define NQ_BASE_INFO10_SFT          6</div><div class='add'>+	__le16	info16;</div><div class='add'>+	__le32	info32;</div><div class='add'>+	__le32	info63_v[2];</div><div class='add'>+	#define NQ_BASE_V          0x1UL</div><div class='add'>+	#define NQ_BASE_INFO63_MASK 0xfffffffeUL</div><div class='add'>+	#define NQ_BASE_INFO63_SFT 1</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* nq_cn (size:128b/16B) */</div><div class='add'>+struct nq_cn {</div><div class='add'>+	__le16	type;</div><div class='add'>+	#define NQ_CN_TYPE_MASK           0x3fUL</div><div class='add'>+	#define NQ_CN_TYPE_SFT            0</div><div class='add'>+	#define NQ_CN_TYPE_CQ_NOTIFICATION  0x30UL</div><div class='add'>+	#define NQ_CN_TYPE_LAST            NQ_CN_TYPE_CQ_NOTIFICATION</div><div class='add'>+	#define NQ_CN_TOGGLE_MASK         0xc0UL</div><div class='add'>+	#define NQ_CN_TOGGLE_SFT          6</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	cq_handle_low;</div><div class='add'>+	__le32	v;</div><div class='add'>+	#define NQ_CN_V     0x1UL</div><div class='add'>+	__le32	cq_handle_high;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* nq_srq_event (size:128b/16B) */</div><div class='add'>+struct nq_srq_event {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define NQ_SRQ_EVENT_TYPE_MASK     0x3fUL</div><div class='add'>+	#define NQ_SRQ_EVENT_TYPE_SFT      0</div><div class='add'>+	#define NQ_SRQ_EVENT_TYPE_SRQ_EVENT  0x32UL</div><div class='add'>+	#define NQ_SRQ_EVENT_TYPE_LAST      NQ_SRQ_EVENT_TYPE_SRQ_EVENT</div><div class='add'>+	#define NQ_SRQ_EVENT_TOGGLE_MASK   0xc0UL</div><div class='add'>+	#define NQ_SRQ_EVENT_TOGGLE_SFT    6</div><div class='add'>+	u8	event;</div><div class='add'>+	#define NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT 0x1UL</div><div class='add'>+	#define NQ_SRQ_EVENT_EVENT_LAST               NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	srq_handle_low;</div><div class='add'>+	__le32	v;</div><div class='add'>+	#define NQ_SRQ_EVENT_V     0x1UL</div><div class='add'>+	__le32	srq_handle_high;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* nq_dbq_event (size:128b/16B) */</div><div class='add'>+struct nq_dbq_event {</div><div class='add'>+	u8	type;</div><div class='add'>+	#define NQ_DBQ_EVENT_TYPE_MASK     0x3fUL</div><div class='add'>+	#define NQ_DBQ_EVENT_TYPE_SFT      0</div><div class='add'>+	#define NQ_DBQ_EVENT_TYPE_DBQ_EVENT  0x34UL</div><div class='add'>+	#define NQ_DBQ_EVENT_TYPE_LAST      NQ_DBQ_EVENT_TYPE_DBQ_EVENT</div><div class='add'>+	u8	event;</div><div class='add'>+	#define NQ_DBQ_EVENT_EVENT_DBQ_THRESHOLD_EVENT 0x1UL</div><div class='add'>+	#define NQ_DBQ_EVENT_EVENT_LAST               NQ_DBQ_EVENT_EVENT_DBQ_THRESHOLD_EVENT</div><div class='add'>+	__le16	db_pfid;</div><div class='add'>+	#define NQ_DBQ_EVENT_DB_PFID_MASK 0xfUL</div><div class='add'>+	#define NQ_DBQ_EVENT_DB_PFID_SFT 0</div><div class='add'>+	__le32	db_dpi;</div><div class='add'>+	#define NQ_DBQ_EVENT_DB_DPI_MASK 0xfffffUL</div><div class='add'>+	#define NQ_DBQ_EVENT_DB_DPI_SFT 0</div><div class='add'>+	__le32	v;</div><div class='add'>+	#define NQ_DBQ_EVENT_V     0x1UL</div><div class='add'>+	__le32	db_type_db_xid;</div><div class='add'>+	#define NQ_DBQ_EVENT_DB_XID_MASK 0xfffffUL</div><div class='add'>+	#define NQ_DBQ_EVENT_DB_XID_SFT  0</div><div class='add'>+	#define NQ_DBQ_EVENT_DB_TYPE_MASK 0xf0000000UL</div><div class='add'>+	#define NQ_DBQ_EVENT_DB_TYPE_SFT 28</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* xrrq_irrq (size:256b/32B) */</div><div class='add'>+struct xrrq_irrq {</div><div class='add'>+	__le16	credits_type;</div><div class='add'>+	#define XRRQ_IRRQ_TYPE           0x1UL</div><div class='add'>+	#define XRRQ_IRRQ_TYPE_READ_REQ    0x0UL</div><div class='add'>+	#define XRRQ_IRRQ_TYPE_ATOMIC_REQ  0x1UL</div><div class='add'>+	#define XRRQ_IRRQ_TYPE_LAST       XRRQ_IRRQ_TYPE_ATOMIC_REQ</div><div class='add'>+	#define XRRQ_IRRQ_CREDITS_MASK   0xf800UL</div><div class='add'>+	#define XRRQ_IRRQ_CREDITS_SFT    11</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	reserved32;</div><div class='add'>+	__le32	psn;</div><div class='add'>+	#define XRRQ_IRRQ_PSN_MASK 0xffffffUL</div><div class='add'>+	#define XRRQ_IRRQ_PSN_SFT 0</div><div class='add'>+	__le32	msn;</div><div class='add'>+	#define XRRQ_IRRQ_MSN_MASK 0xffffffUL</div><div class='add'>+	#define XRRQ_IRRQ_MSN_SFT 0</div><div class='add'>+	__le64	va_or_atomic_result;</div><div class='add'>+	__le32	rdma_r_key;</div><div class='add'>+	__le32	length;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* xrrq_orrq (size:256b/32B) */</div><div class='add'>+struct xrrq_orrq {</div><div class='add'>+	__le16	num_sges_type;</div><div class='add'>+	#define XRRQ_ORRQ_TYPE           0x1UL</div><div class='add'>+	#define XRRQ_ORRQ_TYPE_READ_REQ    0x0UL</div><div class='add'>+	#define XRRQ_ORRQ_TYPE_ATOMIC_REQ  0x1UL</div><div class='add'>+	#define XRRQ_ORRQ_TYPE_LAST       XRRQ_ORRQ_TYPE_ATOMIC_REQ</div><div class='add'>+	#define XRRQ_ORRQ_NUM_SGES_MASK  0xf800UL</div><div class='add'>+	#define XRRQ_ORRQ_NUM_SGES_SFT   11</div><div class='add'>+	__le16	reserved16;</div><div class='add'>+	__le32	length;</div><div class='add'>+	__le32	psn;</div><div class='add'>+	#define XRRQ_ORRQ_PSN_MASK 0xffffffUL</div><div class='add'>+	#define XRRQ_ORRQ_PSN_SFT 0</div><div class='add'>+	__le32	end_psn;</div><div class='add'>+	#define XRRQ_ORRQ_END_PSN_MASK 0xffffffUL</div><div class='add'>+	#define XRRQ_ORRQ_END_PSN_SFT 0</div><div class='add'>+	__le64	first_sge_phy_or_sing_sge_va;</div><div class='add'>+	__le32	single_sge_l_key;</div><div class='add'>+	__le32	single_sge_size;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* ptu_pte (size:64b/8B) */</div><div class='add'>+struct ptu_pte {</div><div class='add'>+	__le32	page_next_to_last_last_valid[2];</div><div class='add'>+	#define PTU_PTE_VALID            0x1UL</div><div class='add'>+	#define PTU_PTE_LAST             0x2UL</div><div class='add'>+	#define PTU_PTE_NEXT_TO_LAST     0x4UL</div><div class='add'>+	#define PTU_PTE_UNUSED_MASK      0xff8UL</div><div class='add'>+	#define PTU_PTE_UNUSED_SFT       3</div><div class='add'>+	#define PTU_PTE_PAGE_MASK        0xfffff000UL</div><div class='add'>+	#define PTU_PTE_PAGE_SFT         12</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* ptu_pde (size:64b/8B) */</div><div class='add'>+struct ptu_pde {</div><div class='add'>+	__le32	page_valid[2];</div><div class='add'>+	#define PTU_PDE_VALID      0x1UL</div><div class='add'>+	#define PTU_PDE_UNUSED_MASK 0xffeUL</div><div class='add'>+	#define PTU_PDE_UNUSED_SFT 1</div><div class='add'>+	#define PTU_PDE_PAGE_MASK  0xfffff000UL</div><div class='add'>+	#define PTU_PDE_PAGE_SFT   12</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#endif /* ___BNXT_RE_HSI_H__ */</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/Kconfig b/drivers/infiniband/hw/cxgb3/Kconfig<br/>deleted file mode 100644<br/>index a7b77cb3d5d5..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/Kconfig?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/Kconfig</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,18 +0,0 @@</div><div class='del'>-config INFINIBAND_CXGB3</div><div class='del'>-	tristate "Chelsio RDMA Driver"</div><div class='del'>-	depends on CHELSIO_T3</div><div class='del'>-	select GENERIC_ALLOCATOR</div><div class='del'>-	---help---</div><div class='del'>-	  This is an iWARP/RDMA driver for the Chelsio T3 1GbE and</div><div class='del'>-	  10GbE adapters.</div><div class='del'>-</div><div class='del'>-	  For general information about Chelsio and our products, visit</div><div class='del'>-	  our website at &lt;http://www.chelsio.com&gt;.</div><div class='del'>-</div><div class='del'>-	  For customer support, please visit our customer support page at</div><div class='del'>-	  &lt;http://www.chelsio.com/support.html&gt;.</div><div class='del'>-</div><div class='del'>-	  Please send feedback to &lt;linux-bugs@chelsio.com&gt;.</div><div class='del'>-</div><div class='del'>-	  To compile this driver as a module, choose M here: the module</div><div class='del'>-	  will be called iw_cxgb3.</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/Makefile b/drivers/infiniband/hw/cxgb3/Makefile<br/>deleted file mode 100644<br/>index 34bb86a6ae3a..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/Makefile?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/Makefile</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,7 +0,0 @@</div><div class='del'>-# SPDX-License-Identifier: GPL-2.0</div><div class='del'>-ccflags-y := -I $(srctree)/drivers/net/ethernet/chelsio/cxgb3</div><div class='del'>-</div><div class='del'>-obj-$(CONFIG_INFINIBAND_CXGB3) += iw_cxgb3.o</div><div class='del'>-</div><div class='del'>-iw_cxgb3-y :=  iwch_cm.o iwch_ev.o iwch_cq.o iwch_qp.o iwch_mem.o \</div><div class='del'>-	       iwch_provider.o iwch.o cxio_hal.o cxio_resource.o</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.c b/drivers/infiniband/hw/cxgb3/cxio_hal.c<br/>deleted file mode 100644<br/>index 8ac72ac7cbac..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/cxio_hal.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/cxio_hal.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,1331 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-#include &lt;asm/delay.h&gt;</div><div class='del'>-</div><div class='del'>-#include &lt;linux/mutex.h&gt;</div><div class='del'>-#include &lt;linux/netdevice.h&gt;</div><div class='del'>-#include &lt;linux/sched.h&gt;</div><div class='del'>-#include &lt;linux/spinlock.h&gt;</div><div class='del'>-#include &lt;linux/pci.h&gt;</div><div class='del'>-#include &lt;linux/dma-mapping.h&gt;</div><div class='del'>-#include &lt;linux/slab.h&gt;</div><div class='del'>-#include &lt;net/net_namespace.h&gt;</div><div class='del'>-</div><div class='del'>-#include "cxio_resource.h"</div><div class='del'>-#include "cxio_hal.h"</div><div class='del'>-#include "cxgb3_offload.h"</div><div class='del'>-#include "sge_defs.h"</div><div class='del'>-</div><div class='del'>-static LIST_HEAD(rdev_list);</div><div class='del'>-static cxio_hal_ev_callback_func_t cxio_ev_cb = NULL;</div><div class='del'>-</div><div class='del'>-static struct cxio_rdev *cxio_hal_find_rdev_by_name(char *dev_name)</div><div class='del'>-{</div><div class='del'>-	struct cxio_rdev *rdev;</div><div class='del'>-</div><div class='del'>-	list_for_each_entry(rdev, &amp;rdev_list, entry)</div><div class='del'>-		if (!strcmp(rdev-&gt;dev_name, dev_name))</div><div class='del'>-			return rdev;</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct cxio_rdev *cxio_hal_find_rdev_by_t3cdev(struct t3cdev *tdev)</div><div class='del'>-{</div><div class='del'>-	struct cxio_rdev *rdev;</div><div class='del'>-</div><div class='del'>-	list_for_each_entry(rdev, &amp;rdev_list, entry)</div><div class='del'>-		if (rdev-&gt;t3cdev_p == tdev)</div><div class='del'>-			return rdev;</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_hal_cq_op(struct cxio_rdev *rdev_p, struct t3_cq *cq,</div><div class='del'>-		   enum t3_cq_opcode op, u32 credit)</div><div class='del'>-{</div><div class='del'>-	int ret;</div><div class='del'>-	struct t3_cqe *cqe;</div><div class='del'>-	u32 rptr;</div><div class='del'>-</div><div class='del'>-	struct rdma_cq_op setup;</div><div class='del'>-	setup.id = cq-&gt;cqid;</div><div class='del'>-	setup.credits = (op == CQ_CREDIT_UPDATE) ? credit : 0;</div><div class='del'>-	setup.op = op;</div><div class='del'>-	ret = rdev_p-&gt;t3cdev_p-&gt;ctl(rdev_p-&gt;t3cdev_p, RDMA_CQ_OP, &amp;setup);</div><div class='del'>-</div><div class='del'>-	if ((ret &lt; 0) || (op == CQ_CREDIT_UPDATE))</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If the rearm returned an index other than our current index,</div><div class='del'>-	 * then there might be CQE's in flight (being DMA'd).  We must wait</div><div class='del'>-	 * here for them to complete or the consumer can miss a notification.</div><div class='del'>-	 */</div><div class='del'>-	if (Q_PTR2IDX((cq-&gt;rptr), cq-&gt;size_log2) != ret) {</div><div class='del'>-		int i=0;</div><div class='del'>-</div><div class='del'>-		rptr = cq-&gt;rptr;</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * Keep the generation correct by bumping rptr until it</div><div class='del'>-		 * matches the index returned by the rearm - 1.</div><div class='del'>-		 */</div><div class='del'>-		while (Q_PTR2IDX((rptr+1), cq-&gt;size_log2) != ret)</div><div class='del'>-			rptr++;</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * Now rptr is the index for the (last) cqe that was</div><div class='del'>-		 * in-flight at the time the HW rearmed the CQ.  We</div><div class='del'>-		 * spin until that CQE is valid.</div><div class='del'>-		 */</div><div class='del'>-		cqe = cq-&gt;queue + Q_PTR2IDX(rptr, cq-&gt;size_log2);</div><div class='del'>-		while (!CQ_VLD_ENTRY(rptr, cq-&gt;size_log2, cqe)) {</div><div class='del'>-			udelay(1);</div><div class='del'>-			if (i++ &gt; 1000000) {</div><div class='del'>-				pr_err("%s: stalled rnic\n", rdev_p-&gt;dev_name);</div><div class='del'>-				return -EIO;</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		return 1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int cxio_hal_clear_cq_ctx(struct cxio_rdev *rdev_p, u32 cqid)</div><div class='del'>-{</div><div class='del'>-	struct rdma_cq_setup setup;</div><div class='del'>-	setup.id = cqid;</div><div class='del'>-	setup.base_addr = 0;	/* NULL address */</div><div class='del'>-	setup.size = 0;		/* disaable the CQ */</div><div class='del'>-	setup.credits = 0;</div><div class='del'>-	setup.credit_thres = 0;</div><div class='del'>-	setup.ovfl_mode = 0;</div><div class='del'>-	return (rdev_p-&gt;t3cdev_p-&gt;ctl(rdev_p-&gt;t3cdev_p, RDMA_CQ_SETUP, &amp;setup));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int cxio_hal_clear_qp_ctx(struct cxio_rdev *rdev_p, u32 qpid)</div><div class='del'>-{</div><div class='del'>-	u64 sge_cmd;</div><div class='del'>-	struct t3_modify_qp_wr *wqe;</div><div class='del'>-	struct sk_buff *skb = alloc_skb(sizeof(*wqe), GFP_KERNEL);</div><div class='del'>-	if (!skb) {</div><div class='del'>-		pr_debug("%s alloc_skb failed\n", __func__);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	wqe = skb_put_zero(skb, sizeof(*wqe));</div><div class='del'>-	build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD,</div><div class='del'>-		       T3_COMPLETION_FLAG | T3_NOTIFY_FLAG, 0, qpid, 7,</div><div class='del'>-		       T3_SOPEOP);</div><div class='del'>-	wqe-&gt;flags = cpu_to_be32(MODQP_WRITE_EC);</div><div class='del'>-	sge_cmd = qpid &lt;&lt; 8 | 3;</div><div class='del'>-	wqe-&gt;sge_cmd = cpu_to_be64(sge_cmd);</div><div class='del'>-	skb-&gt;priority = CPL_PRIORITY_CONTROL;</div><div class='del'>-	return iwch_cxgb3_ofld_send(rdev_p-&gt;t3cdev_p, skb);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq, int kernel)</div><div class='del'>-{</div><div class='del'>-	struct rdma_cq_setup setup;</div><div class='del'>-	int size = (1UL &lt;&lt; (cq-&gt;size_log2)) * sizeof(struct t3_cqe);</div><div class='del'>-</div><div class='del'>-	size += 1; /* one extra page for storing cq-in-err state */</div><div class='del'>-	cq-&gt;cqid = cxio_hal_get_cqid(rdev_p-&gt;rscp);</div><div class='del'>-	if (!cq-&gt;cqid)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	if (kernel) {</div><div class='del'>-		cq-&gt;sw_queue = kzalloc(size, GFP_KERNEL);</div><div class='del'>-		if (!cq-&gt;sw_queue)</div><div class='del'>-			return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	cq-&gt;queue = dma_alloc_coherent(&amp;(rdev_p-&gt;rnic_info.pdev-&gt;dev), size,</div><div class='del'>-					     &amp;(cq-&gt;dma_addr), GFP_KERNEL);</div><div class='del'>-	if (!cq-&gt;queue) {</div><div class='del'>-		kfree(cq-&gt;sw_queue);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	dma_unmap_addr_set(cq, mapping, cq-&gt;dma_addr);</div><div class='del'>-	memset(cq-&gt;queue, 0, size);</div><div class='del'>-	setup.id = cq-&gt;cqid;</div><div class='del'>-	setup.base_addr = (u64) (cq-&gt;dma_addr);</div><div class='del'>-	setup.size = 1UL &lt;&lt; cq-&gt;size_log2;</div><div class='del'>-	setup.credits = 65535;</div><div class='del'>-	setup.credit_thres = 1;</div><div class='del'>-	if (rdev_p-&gt;t3cdev_p-&gt;type != T3A)</div><div class='del'>-		setup.ovfl_mode = 0;</div><div class='del'>-	else</div><div class='del'>-		setup.ovfl_mode = 1;</div><div class='del'>-	return (rdev_p-&gt;t3cdev_p-&gt;ctl(rdev_p-&gt;t3cdev_p, RDMA_CQ_SETUP, &amp;setup));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-#ifdef notyet</div><div class='del'>-int cxio_resize_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)</div><div class='del'>-{</div><div class='del'>-	struct rdma_cq_setup setup;</div><div class='del'>-	setup.id = cq-&gt;cqid;</div><div class='del'>-	setup.base_addr = (u64) (cq-&gt;dma_addr);</div><div class='del'>-	setup.size = 1UL &lt;&lt; cq-&gt;size_log2;</div><div class='del'>-	setup.credits = setup.size;</div><div class='del'>-	setup.credit_thres = setup.size;	/* TBD: overflow recovery */</div><div class='del'>-	setup.ovfl_mode = 1;</div><div class='del'>-	return (rdev_p-&gt;t3cdev_p-&gt;ctl(rdev_p-&gt;t3cdev_p, RDMA_CQ_SETUP, &amp;setup));</div><div class='del'>-}</div><div class='del'>-#endif</div><div class='del'>-</div><div class='del'>-static u32 get_qpid(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)</div><div class='del'>-{</div><div class='del'>-	struct cxio_qpid_list *entry;</div><div class='del'>-	u32 qpid;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;uctx-&gt;lock);</div><div class='del'>-	if (!list_empty(&amp;uctx-&gt;qpids)) {</div><div class='del'>-		entry = list_entry(uctx-&gt;qpids.next, struct cxio_qpid_list,</div><div class='del'>-				   entry);</div><div class='del'>-		list_del(&amp;entry-&gt;entry);</div><div class='del'>-		qpid = entry-&gt;qpid;</div><div class='del'>-		kfree(entry);</div><div class='del'>-	} else {</div><div class='del'>-		qpid = cxio_hal_get_qpid(rdev_p-&gt;rscp);</div><div class='del'>-		if (!qpid)</div><div class='del'>-			goto out;</div><div class='del'>-		for (i = qpid+1; i &amp; rdev_p-&gt;qpmask; i++) {</div><div class='del'>-			entry = kmalloc(sizeof *entry, GFP_KERNEL);</div><div class='del'>-			if (!entry)</div><div class='del'>-				break;</div><div class='del'>-			entry-&gt;qpid = i;</div><div class='del'>-			list_add_tail(&amp;entry-&gt;entry, &amp;uctx-&gt;qpids);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-out:</div><div class='del'>-	mutex_unlock(&amp;uctx-&gt;lock);</div><div class='del'>-	pr_debug("%s qpid 0x%x\n", __func__, qpid);</div><div class='del'>-	return qpid;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void put_qpid(struct cxio_rdev *rdev_p, u32 qpid,</div><div class='del'>-		     struct cxio_ucontext *uctx)</div><div class='del'>-{</div><div class='del'>-	struct cxio_qpid_list *entry;</div><div class='del'>-</div><div class='del'>-	entry = kmalloc(sizeof *entry, GFP_KERNEL);</div><div class='del'>-	if (!entry)</div><div class='del'>-		return;</div><div class='del'>-	pr_debug("%s qpid 0x%x\n", __func__, qpid);</div><div class='del'>-	entry-&gt;qpid = qpid;</div><div class='del'>-	mutex_lock(&amp;uctx-&gt;lock);</div><div class='del'>-	list_add_tail(&amp;entry-&gt;entry, &amp;uctx-&gt;qpids);</div><div class='del'>-	mutex_unlock(&amp;uctx-&gt;lock);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_release_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)</div><div class='del'>-{</div><div class='del'>-	struct list_head *pos, *nxt;</div><div class='del'>-	struct cxio_qpid_list *entry;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;uctx-&gt;lock);</div><div class='del'>-	list_for_each_safe(pos, nxt, &amp;uctx-&gt;qpids) {</div><div class='del'>-		entry = list_entry(pos, struct cxio_qpid_list, entry);</div><div class='del'>-		list_del_init(&amp;entry-&gt;entry);</div><div class='del'>-		if (!(entry-&gt;qpid &amp; rdev_p-&gt;qpmask))</div><div class='del'>-			cxio_hal_put_qpid(rdev_p-&gt;rscp, entry-&gt;qpid);</div><div class='del'>-		kfree(entry);</div><div class='del'>-	}</div><div class='del'>-	mutex_unlock(&amp;uctx-&gt;lock);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_init_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)</div><div class='del'>-{</div><div class='del'>-	INIT_LIST_HEAD(&amp;uctx-&gt;qpids);</div><div class='del'>-	mutex_init(&amp;uctx-&gt;lock);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_create_qp(struct cxio_rdev *rdev_p, u32 kernel_domain,</div><div class='del'>-		   struct t3_wq *wq, struct cxio_ucontext *uctx)</div><div class='del'>-{</div><div class='del'>-	int depth = 1UL &lt;&lt; wq-&gt;size_log2;</div><div class='del'>-	int rqsize = 1UL &lt;&lt; wq-&gt;rq_size_log2;</div><div class='del'>-</div><div class='del'>-	wq-&gt;qpid = get_qpid(rdev_p, uctx);</div><div class='del'>-	if (!wq-&gt;qpid)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	wq-&gt;rq = kcalloc(depth, sizeof(struct t3_swrq), GFP_KERNEL);</div><div class='del'>-	if (!wq-&gt;rq)</div><div class='del'>-		goto err1;</div><div class='del'>-</div><div class='del'>-	wq-&gt;rq_addr = cxio_hal_rqtpool_alloc(rdev_p, rqsize);</div><div class='del'>-	if (!wq-&gt;rq_addr)</div><div class='del'>-		goto err2;</div><div class='del'>-</div><div class='del'>-	wq-&gt;sq = kcalloc(depth, sizeof(struct t3_swsq), GFP_KERNEL);</div><div class='del'>-	if (!wq-&gt;sq)</div><div class='del'>-		goto err3;</div><div class='del'>-</div><div class='del'>-	wq-&gt;queue = dma_alloc_coherent(&amp;(rdev_p-&gt;rnic_info.pdev-&gt;dev),</div><div class='del'>-				       depth * sizeof(union t3_wr),</div><div class='del'>-				       &amp;(wq-&gt;dma_addr), GFP_KERNEL);</div><div class='del'>-	if (!wq-&gt;queue)</div><div class='del'>-		goto err4;</div><div class='del'>-</div><div class='del'>-	dma_unmap_addr_set(wq, mapping, wq-&gt;dma_addr);</div><div class='del'>-	wq-&gt;doorbell = (void __iomem *)rdev_p-&gt;rnic_info.kdb_addr;</div><div class='del'>-	if (!kernel_domain)</div><div class='del'>-		wq-&gt;udb = (u64)rdev_p-&gt;rnic_info.udbell_physbase +</div><div class='del'>-					(wq-&gt;qpid &lt;&lt; rdev_p-&gt;qpshift);</div><div class='del'>-	wq-&gt;rdev = rdev_p;</div><div class='del'>-	pr_debug("%s qpid 0x%x doorbell 0x%p udb 0x%llx\n",</div><div class='del'>-		 __func__, wq-&gt;qpid, wq-&gt;doorbell, (unsigned long long)wq-&gt;udb);</div><div class='del'>-	return 0;</div><div class='del'>-err4:</div><div class='del'>-	kfree(wq-&gt;sq);</div><div class='del'>-err3:</div><div class='del'>-	cxio_hal_rqtpool_free(rdev_p, wq-&gt;rq_addr, rqsize);</div><div class='del'>-err2:</div><div class='del'>-	kfree(wq-&gt;rq);</div><div class='del'>-err1:</div><div class='del'>-	put_qpid(rdev_p, wq-&gt;qpid, uctx);</div><div class='del'>-	return -ENOMEM;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)</div><div class='del'>-{</div><div class='del'>-	int err;</div><div class='del'>-	err = cxio_hal_clear_cq_ctx(rdev_p, cq-&gt;cqid);</div><div class='del'>-	kfree(cq-&gt;sw_queue);</div><div class='del'>-	dma_free_coherent(&amp;(rdev_p-&gt;rnic_info.pdev-&gt;dev),</div><div class='del'>-			  (1UL &lt;&lt; (cq-&gt;size_log2))</div><div class='del'>-			  * sizeof(struct t3_cqe) + 1, cq-&gt;queue,</div><div class='del'>-			  dma_unmap_addr(cq, mapping));</div><div class='del'>-	cxio_hal_put_cqid(rdev_p-&gt;rscp, cq-&gt;cqid);</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_destroy_qp(struct cxio_rdev *rdev_p, struct t3_wq *wq,</div><div class='del'>-		    struct cxio_ucontext *uctx)</div><div class='del'>-{</div><div class='del'>-	dma_free_coherent(&amp;(rdev_p-&gt;rnic_info.pdev-&gt;dev),</div><div class='del'>-			  (1UL &lt;&lt; (wq-&gt;size_log2))</div><div class='del'>-			  * sizeof(union t3_wr), wq-&gt;queue,</div><div class='del'>-			  dma_unmap_addr(wq, mapping));</div><div class='del'>-	kfree(wq-&gt;sq);</div><div class='del'>-	cxio_hal_rqtpool_free(rdev_p, wq-&gt;rq_addr, (1UL &lt;&lt; wq-&gt;rq_size_log2));</div><div class='del'>-	kfree(wq-&gt;rq);</div><div class='del'>-	put_qpid(rdev_p, wq-&gt;qpid, uctx);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void insert_recv_cqe(struct t3_wq *wq, struct t3_cq *cq)</div><div class='del'>-{</div><div class='del'>-	struct t3_cqe cqe;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x\n", __func__,</div><div class='del'>-		 wq, cq, cq-&gt;sw_rptr, cq-&gt;sw_wptr);</div><div class='del'>-	memset(&amp;cqe, 0, sizeof(cqe));</div><div class='del'>-	cqe.header = cpu_to_be32(V_CQE_STATUS(TPT_ERR_SWFLUSH) |</div><div class='del'>-			         V_CQE_OPCODE(T3_SEND) |</div><div class='del'>-				 V_CQE_TYPE(0) |</div><div class='del'>-				 V_CQE_SWCQE(1) |</div><div class='del'>-				 V_CQE_QPID(wq-&gt;qpid) |</div><div class='del'>-				 V_CQE_GENBIT(Q_GENBIT(cq-&gt;sw_wptr,</div><div class='del'>-						       cq-&gt;size_log2)));</div><div class='del'>-	*(cq-&gt;sw_queue + Q_PTR2IDX(cq-&gt;sw_wptr, cq-&gt;size_log2)) = cqe;</div><div class='del'>-	cq-&gt;sw_wptr++;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count)</div><div class='del'>-{</div><div class='del'>-	u32 ptr;</div><div class='del'>-	int flushed = 0;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s wq %p cq %p\n", __func__, wq, cq);</div><div class='del'>-</div><div class='del'>-	/* flush RQ */</div><div class='del'>-	pr_debug("%s rq_rptr %u rq_wptr %u skip count %u\n", __func__,</div><div class='del'>-		 wq-&gt;rq_rptr, wq-&gt;rq_wptr, count);</div><div class='del'>-	ptr = wq-&gt;rq_rptr + count;</div><div class='del'>-	while (ptr++ != wq-&gt;rq_wptr) {</div><div class='del'>-		insert_recv_cqe(wq, cq);</div><div class='del'>-		flushed++;</div><div class='del'>-	}</div><div class='del'>-	return flushed;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void insert_sq_cqe(struct t3_wq *wq, struct t3_cq *cq,</div><div class='del'>-		          struct t3_swsq *sqp)</div><div class='del'>-{</div><div class='del'>-	struct t3_cqe cqe;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x\n", __func__,</div><div class='del'>-		 wq, cq, cq-&gt;sw_rptr, cq-&gt;sw_wptr);</div><div class='del'>-	memset(&amp;cqe, 0, sizeof(cqe));</div><div class='del'>-	cqe.header = cpu_to_be32(V_CQE_STATUS(TPT_ERR_SWFLUSH) |</div><div class='del'>-			         V_CQE_OPCODE(sqp-&gt;opcode) |</div><div class='del'>-			         V_CQE_TYPE(1) |</div><div class='del'>-			         V_CQE_SWCQE(1) |</div><div class='del'>-			         V_CQE_QPID(wq-&gt;qpid) |</div><div class='del'>-			         V_CQE_GENBIT(Q_GENBIT(cq-&gt;sw_wptr,</div><div class='del'>-						       cq-&gt;size_log2)));</div><div class='del'>-	cqe.u.scqe.wrid_hi = sqp-&gt;sq_wptr;</div><div class='del'>-</div><div class='del'>-	*(cq-&gt;sw_queue + Q_PTR2IDX(cq-&gt;sw_wptr, cq-&gt;size_log2)) = cqe;</div><div class='del'>-	cq-&gt;sw_wptr++;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count)</div><div class='del'>-{</div><div class='del'>-	__u32 ptr = wq-&gt;sq_rptr + count;</div><div class='del'>-	int flushed = 0;</div><div class='del'>-	struct t3_swsq *sqp = wq-&gt;sq + Q_PTR2IDX(ptr, wq-&gt;sq_size_log2);</div><div class='del'>-</div><div class='del'>-	while (ptr != wq-&gt;sq_wptr) {</div><div class='del'>-		sqp-&gt;signaled = 0;</div><div class='del'>-		insert_sq_cqe(wq, cq, sqp);</div><div class='del'>-		ptr++;</div><div class='del'>-		sqp = wq-&gt;sq + Q_PTR2IDX(ptr, wq-&gt;sq_size_log2);</div><div class='del'>-		flushed++;</div><div class='del'>-	}</div><div class='del'>-	return flushed;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Move all CQEs from the HWCQ into the SWCQ.</div><div class='del'>- */</div><div class='del'>-void cxio_flush_hw_cq(struct t3_cq *cq)</div><div class='del'>-{</div><div class='del'>-	struct t3_cqe *cqe, *swcqe;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s cq %p cqid 0x%x\n", __func__, cq, cq-&gt;cqid);</div><div class='del'>-	cqe = cxio_next_hw_cqe(cq);</div><div class='del'>-	while (cqe) {</div><div class='del'>-		pr_debug("%s flushing hwcq rptr 0x%x to swcq wptr 0x%x\n",</div><div class='del'>-			 __func__, cq-&gt;rptr, cq-&gt;sw_wptr);</div><div class='del'>-		swcqe = cq-&gt;sw_queue + Q_PTR2IDX(cq-&gt;sw_wptr, cq-&gt;size_log2);</div><div class='del'>-		*swcqe = *cqe;</div><div class='del'>-		swcqe-&gt;header |= cpu_to_be32(V_CQE_SWCQE(1));</div><div class='del'>-		cq-&gt;sw_wptr++;</div><div class='del'>-		cq-&gt;rptr++;</div><div class='del'>-		cqe = cxio_next_hw_cqe(cq);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int cqe_completes_wr(struct t3_cqe *cqe, struct t3_wq *wq)</div><div class='del'>-{</div><div class='del'>-	if (CQE_OPCODE(*cqe) == T3_TERMINATE)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	if ((CQE_OPCODE(*cqe) == T3_RDMA_WRITE) &amp;&amp; RQ_TYPE(*cqe))</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	if ((CQE_OPCODE(*cqe) == T3_READ_RESP) &amp;&amp; SQ_TYPE(*cqe))</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	if (CQE_SEND_OPCODE(*cqe) &amp;&amp; RQ_TYPE(*cqe) &amp;&amp;</div><div class='del'>-	    Q_EMPTY(wq-&gt;rq_rptr, wq-&gt;rq_wptr))</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	return 1;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count)</div><div class='del'>-{</div><div class='del'>-	struct t3_cqe *cqe;</div><div class='del'>-	u32 ptr;</div><div class='del'>-</div><div class='del'>-	*count = 0;</div><div class='del'>-	ptr = cq-&gt;sw_rptr;</div><div class='del'>-	while (!Q_EMPTY(ptr, cq-&gt;sw_wptr)) {</div><div class='del'>-		cqe = cq-&gt;sw_queue + (Q_PTR2IDX(ptr, cq-&gt;size_log2));</div><div class='del'>-		if ((SQ_TYPE(*cqe) ||</div><div class='del'>-		     ((CQE_OPCODE(*cqe) == T3_READ_RESP) &amp;&amp; wq-&gt;oldest_read)) &amp;&amp;</div><div class='del'>-		    (CQE_QPID(*cqe) == wq-&gt;qpid))</div><div class='del'>-			(*count)++;</div><div class='del'>-		ptr++;</div><div class='del'>-	}</div><div class='del'>-	pr_debug("%s cq %p count %d\n", __func__, cq, *count);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_count_rcqes(struct t3_cq *cq, struct t3_wq *wq, int *count)</div><div class='del'>-{</div><div class='del'>-	struct t3_cqe *cqe;</div><div class='del'>-	u32 ptr;</div><div class='del'>-</div><div class='del'>-	*count = 0;</div><div class='del'>-	pr_debug("%s count zero %d\n", __func__, *count);</div><div class='del'>-	ptr = cq-&gt;sw_rptr;</div><div class='del'>-	while (!Q_EMPTY(ptr, cq-&gt;sw_wptr)) {</div><div class='del'>-		cqe = cq-&gt;sw_queue + (Q_PTR2IDX(ptr, cq-&gt;size_log2));</div><div class='del'>-		if (RQ_TYPE(*cqe) &amp;&amp; (CQE_OPCODE(*cqe) != T3_READ_RESP) &amp;&amp;</div><div class='del'>-		    (CQE_QPID(*cqe) == wq-&gt;qpid) &amp;&amp; cqe_completes_wr(cqe, wq))</div><div class='del'>-			(*count)++;</div><div class='del'>-		ptr++;</div><div class='del'>-	}</div><div class='del'>-	pr_debug("%s cq %p count %d\n", __func__, cq, *count);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int cxio_hal_init_ctrl_cq(struct cxio_rdev *rdev_p)</div><div class='del'>-{</div><div class='del'>-	struct rdma_cq_setup setup;</div><div class='del'>-	setup.id = 0;</div><div class='del'>-	setup.base_addr = 0;	/* NULL address */</div><div class='del'>-	setup.size = 1;		/* enable the CQ */</div><div class='del'>-	setup.credits = 0;</div><div class='del'>-</div><div class='del'>-	/* force SGE to redirect to RspQ and interrupt */</div><div class='del'>-	setup.credit_thres = 0;</div><div class='del'>-	setup.ovfl_mode = 1;</div><div class='del'>-	return (rdev_p-&gt;t3cdev_p-&gt;ctl(rdev_p-&gt;t3cdev_p, RDMA_CQ_SETUP, &amp;setup));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p)</div><div class='del'>-{</div><div class='del'>-	int err;</div><div class='del'>-	u64 sge_cmd, ctx0, ctx1;</div><div class='del'>-	u64 base_addr;</div><div class='del'>-	struct t3_modify_qp_wr *wqe;</div><div class='del'>-	struct sk_buff *skb;</div><div class='del'>-</div><div class='del'>-	skb = alloc_skb(sizeof(*wqe), GFP_KERNEL);</div><div class='del'>-	if (!skb) {</div><div class='del'>-		pr_debug("%s alloc_skb failed\n", __func__);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	err = cxio_hal_init_ctrl_cq(rdev_p);</div><div class='del'>-	if (err) {</div><div class='del'>-		pr_debug("%s err %d initializing ctrl_cq\n", __func__, err);</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-	rdev_p-&gt;ctrl_qp.workq = dma_alloc_coherent(</div><div class='del'>-					&amp;(rdev_p-&gt;rnic_info.pdev-&gt;dev),</div><div class='del'>-					(1 &lt;&lt; T3_CTRL_QP_SIZE_LOG2) *</div><div class='del'>-					sizeof(union t3_wr),</div><div class='del'>-					&amp;(rdev_p-&gt;ctrl_qp.dma_addr),</div><div class='del'>-					GFP_KERNEL);</div><div class='del'>-	if (!rdev_p-&gt;ctrl_qp.workq) {</div><div class='del'>-		pr_debug("%s dma_alloc_coherent failed\n", __func__);</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-	dma_unmap_addr_set(&amp;rdev_p-&gt;ctrl_qp, mapping,</div><div class='del'>-			   rdev_p-&gt;ctrl_qp.dma_addr);</div><div class='del'>-	rdev_p-&gt;ctrl_qp.doorbell = (void __iomem *)rdev_p-&gt;rnic_info.kdb_addr;</div><div class='del'>-	memset(rdev_p-&gt;ctrl_qp.workq, 0,</div><div class='del'>-	       (1 &lt;&lt; T3_CTRL_QP_SIZE_LOG2) * sizeof(union t3_wr));</div><div class='del'>-</div><div class='del'>-	mutex_init(&amp;rdev_p-&gt;ctrl_qp.lock);</div><div class='del'>-	init_waitqueue_head(&amp;rdev_p-&gt;ctrl_qp.waitq);</div><div class='del'>-</div><div class='del'>-	/* update HW Ctrl QP context */</div><div class='del'>-	base_addr = rdev_p-&gt;ctrl_qp.dma_addr;</div><div class='del'>-	base_addr &gt;&gt;= 12;</div><div class='del'>-	ctx0 = (V_EC_SIZE((1 &lt;&lt; T3_CTRL_QP_SIZE_LOG2)) |</div><div class='del'>-		V_EC_BASE_LO((u32) base_addr &amp; 0xffff));</div><div class='del'>-	ctx0 &lt;&lt;= 32;</div><div class='del'>-	ctx0 |= V_EC_CREDITS(FW_WR_NUM);</div><div class='del'>-	base_addr &gt;&gt;= 16;</div><div class='del'>-	ctx1 = (u32) base_addr;</div><div class='del'>-	base_addr &gt;&gt;= 32;</div><div class='del'>-	ctx1 |= ((u64) (V_EC_BASE_HI((u32) base_addr &amp; 0xf) | V_EC_RESPQ(0) |</div><div class='del'>-			V_EC_TYPE(0) | V_EC_GEN(1) |</div><div class='del'>-			V_EC_UP_TOKEN(T3_CTL_QP_TID) | F_EC_VALID)) &lt;&lt; 32;</div><div class='del'>-	wqe = skb_put_zero(skb, sizeof(*wqe));</div><div class='del'>-	build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 0, 0,</div><div class='del'>-		       T3_CTL_QP_TID, 7, T3_SOPEOP);</div><div class='del'>-	wqe-&gt;flags = cpu_to_be32(MODQP_WRITE_EC);</div><div class='del'>-	sge_cmd = (3ULL &lt;&lt; 56) | FW_RI_SGEEC_START &lt;&lt; 8 | 3;</div><div class='del'>-	wqe-&gt;sge_cmd = cpu_to_be64(sge_cmd);</div><div class='del'>-	wqe-&gt;ctx1 = cpu_to_be64(ctx1);</div><div class='del'>-	wqe-&gt;ctx0 = cpu_to_be64(ctx0);</div><div class='del'>-	pr_debug("CtrlQP dma_addr 0x%llx workq %p size %d\n",</div><div class='del'>-		 (unsigned long long)rdev_p-&gt;ctrl_qp.dma_addr,</div><div class='del'>-		 rdev_p-&gt;ctrl_qp.workq, 1 &lt;&lt; T3_CTRL_QP_SIZE_LOG2);</div><div class='del'>-	skb-&gt;priority = CPL_PRIORITY_CONTROL;</div><div class='del'>-	return iwch_cxgb3_ofld_send(rdev_p-&gt;t3cdev_p, skb);</div><div class='del'>-err:</div><div class='del'>-	kfree_skb(skb);</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int cxio_hal_destroy_ctrl_qp(struct cxio_rdev *rdev_p)</div><div class='del'>-{</div><div class='del'>-	dma_free_coherent(&amp;(rdev_p-&gt;rnic_info.pdev-&gt;dev),</div><div class='del'>-			  (1UL &lt;&lt; T3_CTRL_QP_SIZE_LOG2)</div><div class='del'>-			  * sizeof(union t3_wr), rdev_p-&gt;ctrl_qp.workq,</div><div class='del'>-			  dma_unmap_addr(&amp;rdev_p-&gt;ctrl_qp, mapping));</div><div class='del'>-	return cxio_hal_clear_qp_ctx(rdev_p, T3_CTRL_QP_ID);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* write len bytes of data into addr (32B aligned address)</div><div class='del'>- * If data is NULL, clear len byte of memory to zero.</div><div class='del'>- * caller acquires the ctrl_qp lock before the call</div><div class='del'>- */</div><div class='del'>-static int cxio_hal_ctrl_qp_write_mem(struct cxio_rdev *rdev_p, u32 addr,</div><div class='del'>-				      u32 len, void *data)</div><div class='del'>-{</div><div class='del'>-	u32 i, nr_wqe, copy_len;</div><div class='del'>-	u8 *copy_data;</div><div class='del'>-	u8 wr_len, utx_len;	/* length in 8 byte flit */</div><div class='del'>-	enum t3_wr_flags flag;</div><div class='del'>-	__be64 *wqe;</div><div class='del'>-	u64 utx_cmd;</div><div class='del'>-	addr &amp;= 0x7FFFFFF;</div><div class='del'>-	nr_wqe = len % 96 ? len / 96 + 1 : len / 96;	/* 96B max per WQE */</div><div class='del'>-	pr_debug("%s wptr 0x%x rptr 0x%x len %d, nr_wqe %d data %p addr 0x%0x\n",</div><div class='del'>-		 __func__, rdev_p-&gt;ctrl_qp.wptr, rdev_p-&gt;ctrl_qp.rptr, len,</div><div class='del'>-		 nr_wqe, data, addr);</div><div class='del'>-	utx_len = 3;		/* in 32B unit */</div><div class='del'>-	for (i = 0; i &lt; nr_wqe; i++) {</div><div class='del'>-		if (Q_FULL(rdev_p-&gt;ctrl_qp.rptr, rdev_p-&gt;ctrl_qp.wptr,</div><div class='del'>-		           T3_CTRL_QP_SIZE_LOG2)) {</div><div class='del'>-			pr_debug("%s ctrl_qp full wtpr 0x%0x rptr 0x%0x, wait for more space i %d\n",</div><div class='del'>-				 __func__,</div><div class='del'>-				 rdev_p-&gt;ctrl_qp.wptr, rdev_p-&gt;ctrl_qp.rptr, i);</div><div class='del'>-			if (wait_event_interruptible(rdev_p-&gt;ctrl_qp.waitq,</div><div class='del'>-					     !Q_FULL(rdev_p-&gt;ctrl_qp.rptr,</div><div class='del'>-						     rdev_p-&gt;ctrl_qp.wptr,</div><div class='del'>-						     T3_CTRL_QP_SIZE_LOG2))) {</div><div class='del'>-				pr_debug("%s ctrl_qp workq interrupted\n",</div><div class='del'>-					 __func__);</div><div class='del'>-				return -ERESTARTSYS;</div><div class='del'>-			}</div><div class='del'>-			pr_debug("%s ctrl_qp wakeup, continue posting work request i %d\n",</div><div class='del'>-				 __func__, i);</div><div class='del'>-		}</div><div class='del'>-		wqe = (__be64 *)(rdev_p-&gt;ctrl_qp.workq + (rdev_p-&gt;ctrl_qp.wptr %</div><div class='del'>-						(1 &lt;&lt; T3_CTRL_QP_SIZE_LOG2)));</div><div class='del'>-		flag = 0;</div><div class='del'>-		if (i == (nr_wqe - 1)) {</div><div class='del'>-			/* last WQE */</div><div class='del'>-			flag = T3_COMPLETION_FLAG;</div><div class='del'>-			if (len % 32)</div><div class='del'>-				utx_len = len / 32 + 1;</div><div class='del'>-			else</div><div class='del'>-				utx_len = len / 32;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * Force a CQE to return the credit to the workq in case</div><div class='del'>-		 * we posted more than half the max QP size of WRs</div><div class='del'>-		 */</div><div class='del'>-		if ((i != 0) &amp;&amp;</div><div class='del'>-		    (i % (((1 &lt;&lt; T3_CTRL_QP_SIZE_LOG2)) &gt;&gt; 1) == 0)) {</div><div class='del'>-			flag = T3_COMPLETION_FLAG;</div><div class='del'>-			pr_debug("%s force completion at i %d\n", __func__, i);</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		/* build the utx mem command */</div><div class='del'>-		wqe += (sizeof(struct t3_bypass_wr) &gt;&gt; 3);</div><div class='del'>-		utx_cmd = (T3_UTX_MEM_WRITE &lt;&lt; 28) | (addr + i * 3);</div><div class='del'>-		utx_cmd &lt;&lt;= 32;</div><div class='del'>-		utx_cmd |= (utx_len &lt;&lt; 28) | ((utx_len &lt;&lt; 2) + 1);</div><div class='del'>-		*wqe = cpu_to_be64(utx_cmd);</div><div class='del'>-		wqe++;</div><div class='del'>-		copy_data = (u8 *) data + i * 96;</div><div class='del'>-		copy_len = len &gt; 96 ? 96 : len;</div><div class='del'>-</div><div class='del'>-		/* clear memory content if data is NULL */</div><div class='del'>-		if (data)</div><div class='del'>-			memcpy(wqe, copy_data, copy_len);</div><div class='del'>-		else</div><div class='del'>-			memset(wqe, 0, copy_len);</div><div class='del'>-		if (copy_len % 32)</div><div class='del'>-			memset(((u8 *) wqe) + copy_len, 0,</div><div class='del'>-			       32 - (copy_len % 32));</div><div class='del'>-		wr_len = ((sizeof(struct t3_bypass_wr)) &gt;&gt; 3) + 1 +</div><div class='del'>-			 (utx_len &lt;&lt; 2);</div><div class='del'>-		wqe = (__be64 *)(rdev_p-&gt;ctrl_qp.workq + (rdev_p-&gt;ctrl_qp.wptr %</div><div class='del'>-			      (1 &lt;&lt; T3_CTRL_QP_SIZE_LOG2)));</div><div class='del'>-</div><div class='del'>-		/* wptr in the WRID[31:0] */</div><div class='del'>-		((union t3_wrid *)(wqe+1))-&gt;id0.low = rdev_p-&gt;ctrl_qp.wptr;</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * This must be the last write with a memory barrier</div><div class='del'>-		 * for the genbit</div><div class='del'>-		 */</div><div class='del'>-		build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_BP, flag,</div><div class='del'>-			       Q_GENBIT(rdev_p-&gt;ctrl_qp.wptr,</div><div class='del'>-					T3_CTRL_QP_SIZE_LOG2), T3_CTRL_QP_ID,</div><div class='del'>-			       wr_len, T3_SOPEOP);</div><div class='del'>-		if (flag == T3_COMPLETION_FLAG)</div><div class='del'>-			ring_doorbell(rdev_p-&gt;ctrl_qp.doorbell, T3_CTRL_QP_ID);</div><div class='del'>-		len -= 96;</div><div class='del'>-		rdev_p-&gt;ctrl_qp.wptr++;</div><div class='del'>-	}</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* IN: stag key, pdid, perm, zbva, to, len, page_size, pbl_size and pbl_addr</div><div class='del'>- * OUT: stag index</div><div class='del'>- * TBD: shared memory region support</div><div class='del'>- */</div><div class='del'>-static int __cxio_tpt_op(struct cxio_rdev *rdev_p, u32 reset_tpt_entry,</div><div class='del'>-			 u32 *stag, u8 stag_state, u32 pdid,</div><div class='del'>-			 enum tpt_mem_type type, enum tpt_mem_perm perm,</div><div class='del'>-			 u32 zbva, u64 to, u32 len, u8 page_size,</div><div class='del'>-			 u32 pbl_size, u32 pbl_addr)</div><div class='del'>-{</div><div class='del'>-	int err;</div><div class='del'>-	struct tpt_entry tpt;</div><div class='del'>-	u32 stag_idx;</div><div class='del'>-	u32 wptr;</div><div class='del'>-</div><div class='del'>-	if (cxio_fatal_error(rdev_p))</div><div class='del'>-		return -EIO;</div><div class='del'>-</div><div class='del'>-	stag_state = stag_state &gt; 0;</div><div class='del'>-	stag_idx = (*stag) &gt;&gt; 8;</div><div class='del'>-</div><div class='del'>-	if ((!reset_tpt_entry) &amp;&amp; !(*stag != T3_STAG_UNSET)) {</div><div class='del'>-		stag_idx = cxio_hal_get_stag(rdev_p-&gt;rscp);</div><div class='del'>-		if (!stag_idx)</div><div class='del'>-			return -ENOMEM;</div><div class='del'>-		*stag = (stag_idx &lt;&lt; 8) | ((*stag) &amp; 0xFF);</div><div class='del'>-	}</div><div class='del'>-	pr_debug("%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x\n",</div><div class='del'>-		 __func__, stag_state, type, pdid, stag_idx);</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;rdev_p-&gt;ctrl_qp.lock);</div><div class='del'>-</div><div class='del'>-	/* write TPT entry */</div><div class='del'>-	if (reset_tpt_entry)</div><div class='del'>-		memset(&amp;tpt, 0, sizeof(tpt));</div><div class='del'>-	else {</div><div class='del'>-		tpt.valid_stag_pdid = cpu_to_be32(F_TPT_VALID |</div><div class='del'>-				V_TPT_STAG_KEY((*stag) &amp; M_TPT_STAG_KEY) |</div><div class='del'>-				V_TPT_STAG_STATE(stag_state) |</div><div class='del'>-				V_TPT_STAG_TYPE(type) | V_TPT_PDID(pdid));</div><div class='del'>-		BUG_ON(page_size &gt;= 28);</div><div class='del'>-		tpt.flags_pagesize_qpid = cpu_to_be32(V_TPT_PERM(perm) |</div><div class='del'>-			((perm &amp; TPT_MW_BIND) ? F_TPT_MW_BIND_ENABLE : 0) |</div><div class='del'>-			V_TPT_ADDR_TYPE((zbva ? TPT_ZBTO : TPT_VATO)) |</div><div class='del'>-			V_TPT_PAGE_SIZE(page_size));</div><div class='del'>-		tpt.rsvd_pbl_addr = cpu_to_be32(V_TPT_PBL_ADDR(PBL_OFF(rdev_p, pbl_addr)&gt;&gt;3));</div><div class='del'>-		tpt.len = cpu_to_be32(len);</div><div class='del'>-		tpt.va_hi = cpu_to_be32((u32) (to &gt;&gt; 32));</div><div class='del'>-		tpt.va_low_or_fbo = cpu_to_be32((u32) (to &amp; 0xFFFFFFFFULL));</div><div class='del'>-		tpt.rsvd_bind_cnt_or_pstag = 0;</div><div class='del'>-		tpt.rsvd_pbl_size = cpu_to_be32(V_TPT_PBL_SIZE(pbl_size &gt;&gt; 2));</div><div class='del'>-	}</div><div class='del'>-	err = cxio_hal_ctrl_qp_write_mem(rdev_p,</div><div class='del'>-				       stag_idx +</div><div class='del'>-				       (rdev_p-&gt;rnic_info.tpt_base &gt;&gt; 5),</div><div class='del'>-				       sizeof(tpt), &amp;tpt);</div><div class='del'>-</div><div class='del'>-	/* release the stag index to free pool */</div><div class='del'>-	if (reset_tpt_entry)</div><div class='del'>-		cxio_hal_put_stag(rdev_p-&gt;rscp, stag_idx);</div><div class='del'>-</div><div class='del'>-	wptr = rdev_p-&gt;ctrl_qp.wptr;</div><div class='del'>-	mutex_unlock(&amp;rdev_p-&gt;ctrl_qp.lock);</div><div class='del'>-	if (!err)</div><div class='del'>-		if (wait_event_interruptible(rdev_p-&gt;ctrl_qp.waitq,</div><div class='del'>-					     SEQ32_GE(rdev_p-&gt;ctrl_qp.rptr,</div><div class='del'>-						      wptr)))</div><div class='del'>-			return -ERESTARTSYS;</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_write_pbl(struct cxio_rdev *rdev_p, __be64 *pbl,</div><div class='del'>-		   u32 pbl_addr, u32 pbl_size)</div><div class='del'>-{</div><div class='del'>-	u32 wptr;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d\n",</div><div class='del'>-		 __func__, pbl_addr, rdev_p-&gt;rnic_info.pbl_base,</div><div class='del'>-		 pbl_size);</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;rdev_p-&gt;ctrl_qp.lock);</div><div class='del'>-	err = cxio_hal_ctrl_qp_write_mem(rdev_p, pbl_addr &gt;&gt; 5, pbl_size &lt;&lt; 3,</div><div class='del'>-					 pbl);</div><div class='del'>-	wptr = rdev_p-&gt;ctrl_qp.wptr;</div><div class='del'>-	mutex_unlock(&amp;rdev_p-&gt;ctrl_qp.lock);</div><div class='del'>-	if (err)</div><div class='del'>-		return err;</div><div class='del'>-</div><div class='del'>-	if (wait_event_interruptible(rdev_p-&gt;ctrl_qp.waitq,</div><div class='del'>-				     SEQ32_GE(rdev_p-&gt;ctrl_qp.rptr,</div><div class='del'>-					      wptr)))</div><div class='del'>-		return -ERESTARTSYS;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_register_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,</div><div class='del'>-			   enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,</div><div class='del'>-			   u8 page_size, u32 pbl_size, u32 pbl_addr)</div><div class='del'>-{</div><div class='del'>-	*stag = T3_STAG_UNSET;</div><div class='del'>-	return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,</div><div class='del'>-			     zbva, to, len, page_size, pbl_size, pbl_addr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_reregister_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,</div><div class='del'>-			   enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,</div><div class='del'>-			   u8 page_size, u32 pbl_size, u32 pbl_addr)</div><div class='del'>-{</div><div class='del'>-	return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,</div><div class='del'>-			     zbva, to, len, page_size, pbl_size, pbl_addr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_dereg_mem(struct cxio_rdev *rdev_p, u32 stag, u32 pbl_size,</div><div class='del'>-		   u32 pbl_addr)</div><div class='del'>-{</div><div class='del'>-	return __cxio_tpt_op(rdev_p, 1, &amp;stag, 0, 0, 0, 0, 0, 0ULL, 0, 0,</div><div class='del'>-			     pbl_size, pbl_addr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_allocate_window(struct cxio_rdev *rdev_p, u32 * stag, u32 pdid)</div><div class='del'>-{</div><div class='del'>-	*stag = T3_STAG_UNSET;</div><div class='del'>-	return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_MW, 0, 0, 0ULL, 0, 0,</div><div class='del'>-			     0, 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_deallocate_window(struct cxio_rdev *rdev_p, u32 stag)</div><div class='del'>-{</div><div class='del'>-	return __cxio_tpt_op(rdev_p, 1, &amp;stag, 0, 0, 0, 0, 0, 0ULL, 0, 0,</div><div class='del'>-			     0, 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_allocate_stag(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid, u32 pbl_size, u32 pbl_addr)</div><div class='del'>-{</div><div class='del'>-	*stag = T3_STAG_UNSET;</div><div class='del'>-	return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_NON_SHARED_MR,</div><div class='del'>-			     0, 0, 0ULL, 0, 0, pbl_size, pbl_addr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr)</div><div class='del'>-{</div><div class='del'>-	struct t3_rdma_init_wr *wqe;</div><div class='del'>-	struct sk_buff *skb = alloc_skb(sizeof(*wqe), GFP_ATOMIC);</div><div class='del'>-	if (!skb)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	pr_debug("%s rdev_p %p\n", __func__, rdev_p);</div><div class='del'>-	wqe = __skb_put(skb, sizeof(*wqe));</div><div class='del'>-	wqe-&gt;wrh.op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(T3_WR_INIT));</div><div class='del'>-	wqe-&gt;wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(attr-&gt;tid) |</div><div class='del'>-					   V_FW_RIWR_LEN(sizeof(*wqe) &gt;&gt; 3));</div><div class='del'>-	wqe-&gt;wrid.id1 = 0;</div><div class='del'>-	wqe-&gt;qpid = cpu_to_be32(attr-&gt;qpid);</div><div class='del'>-	wqe-&gt;pdid = cpu_to_be32(attr-&gt;pdid);</div><div class='del'>-	wqe-&gt;scqid = cpu_to_be32(attr-&gt;scqid);</div><div class='del'>-	wqe-&gt;rcqid = cpu_to_be32(attr-&gt;rcqid);</div><div class='del'>-	wqe-&gt;rq_addr = cpu_to_be32(attr-&gt;rq_addr - rdev_p-&gt;rnic_info.rqt_base);</div><div class='del'>-	wqe-&gt;rq_size = cpu_to_be32(attr-&gt;rq_size);</div><div class='del'>-	wqe-&gt;mpaattrs = attr-&gt;mpaattrs;</div><div class='del'>-	wqe-&gt;qpcaps = attr-&gt;qpcaps;</div><div class='del'>-	wqe-&gt;ulpdu_size = cpu_to_be16(attr-&gt;tcp_emss);</div><div class='del'>-	wqe-&gt;rqe_count = cpu_to_be16(attr-&gt;rqe_count);</div><div class='del'>-	wqe-&gt;flags_rtr_type = cpu_to_be16(attr-&gt;flags |</div><div class='del'>-					  V_RTR_TYPE(attr-&gt;rtr_type) |</div><div class='del'>-					  V_CHAN(attr-&gt;chan));</div><div class='del'>-	wqe-&gt;ord = cpu_to_be32(attr-&gt;ord);</div><div class='del'>-	wqe-&gt;ird = cpu_to_be32(attr-&gt;ird);</div><div class='del'>-	wqe-&gt;qp_dma_addr = cpu_to_be64(attr-&gt;qp_dma_addr);</div><div class='del'>-	wqe-&gt;qp_dma_size = cpu_to_be32(attr-&gt;qp_dma_size);</div><div class='del'>-	wqe-&gt;irs = cpu_to_be32(attr-&gt;irs);</div><div class='del'>-	skb-&gt;priority = 0;	/* 0=&gt;ToeQ; 1=&gt;CtrlQ */</div><div class='del'>-	return iwch_cxgb3_ofld_send(rdev_p-&gt;t3cdev_p, skb);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_register_ev_cb(cxio_hal_ev_callback_func_t ev_cb)</div><div class='del'>-{</div><div class='del'>-	cxio_ev_cb = ev_cb;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_unregister_ev_cb(cxio_hal_ev_callback_func_t ev_cb)</div><div class='del'>-{</div><div class='del'>-	cxio_ev_cb = NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int cxio_hal_ev_handler(struct t3cdev *t3cdev_p, struct sk_buff *skb)</div><div class='del'>-{</div><div class='del'>-	static int cnt;</div><div class='del'>-	struct cxio_rdev *rdev_p = NULL;</div><div class='del'>-	struct respQ_msg_t *rsp_msg = (struct respQ_msg_t *) skb-&gt;data;</div><div class='del'>-	pr_debug("%d: %s cq_id 0x%x cq_ptr 0x%x genbit %0x overflow %0x an %0x se %0x notify %0x cqbranch %0x creditth %0x\n",</div><div class='del'>-		 cnt, __func__, RSPQ_CQID(rsp_msg), RSPQ_CQPTR(rsp_msg),</div><div class='del'>-		 RSPQ_GENBIT(rsp_msg), RSPQ_OVERFLOW(rsp_msg), RSPQ_AN(rsp_msg),</div><div class='del'>-		 RSPQ_SE(rsp_msg), RSPQ_NOTIFY(rsp_msg), RSPQ_CQBRANCH(rsp_msg),</div><div class='del'>-		 RSPQ_CREDIT_THRESH(rsp_msg));</div><div class='del'>-	pr_debug("CQE: QPID 0x%0x genbit %0x type 0x%0x status 0x%0x opcode %d len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x\n",</div><div class='del'>-		 CQE_QPID(rsp_msg-&gt;cqe), CQE_GENBIT(rsp_msg-&gt;cqe),</div><div class='del'>-		 CQE_TYPE(rsp_msg-&gt;cqe), CQE_STATUS(rsp_msg-&gt;cqe),</div><div class='del'>-		 CQE_OPCODE(rsp_msg-&gt;cqe), CQE_LEN(rsp_msg-&gt;cqe),</div><div class='del'>-		 CQE_WRID_HI(rsp_msg-&gt;cqe), CQE_WRID_LOW(rsp_msg-&gt;cqe));</div><div class='del'>-	rdev_p = (struct cxio_rdev *)t3cdev_p-&gt;ulp;</div><div class='del'>-	if (!rdev_p) {</div><div class='del'>-		pr_debug("%s called by t3cdev %p with null ulp\n", __func__,</div><div class='del'>-			 t3cdev_p);</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='del'>-	if (CQE_QPID(rsp_msg-&gt;cqe) == T3_CTRL_QP_ID) {</div><div class='del'>-		rdev_p-&gt;ctrl_qp.rptr = CQE_WRID_LOW(rsp_msg-&gt;cqe) + 1;</div><div class='del'>-		wake_up_interruptible(&amp;rdev_p-&gt;ctrl_qp.waitq);</div><div class='del'>-		dev_kfree_skb_irq(skb);</div><div class='del'>-	} else if (CQE_QPID(rsp_msg-&gt;cqe) == 0xfff8)</div><div class='del'>-		dev_kfree_skb_irq(skb);</div><div class='del'>-	else if (cxio_ev_cb)</div><div class='del'>-		(*cxio_ev_cb) (rdev_p, skb);</div><div class='del'>-	else</div><div class='del'>-		dev_kfree_skb_irq(skb);</div><div class='del'>-	cnt++;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* Caller takes care of locking if needed */</div><div class='del'>-int cxio_rdev_open(struct cxio_rdev *rdev_p)</div><div class='del'>-{</div><div class='del'>-	struct net_device *netdev_p = NULL;</div><div class='del'>-	int err = 0;</div><div class='del'>-	if (strlen(rdev_p-&gt;dev_name)) {</div><div class='del'>-		if (cxio_hal_find_rdev_by_name(rdev_p-&gt;dev_name)) {</div><div class='del'>-			return -EBUSY;</div><div class='del'>-		}</div><div class='del'>-		netdev_p = dev_get_by_name(&amp;init_net, rdev_p-&gt;dev_name);</div><div class='del'>-		if (!netdev_p) {</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-		dev_put(netdev_p);</div><div class='del'>-	} else if (rdev_p-&gt;t3cdev_p) {</div><div class='del'>-		if (cxio_hal_find_rdev_by_t3cdev(rdev_p-&gt;t3cdev_p)) {</div><div class='del'>-			return -EBUSY;</div><div class='del'>-		}</div><div class='del'>-		netdev_p = rdev_p-&gt;t3cdev_p-&gt;lldev;</div><div class='del'>-		strncpy(rdev_p-&gt;dev_name, rdev_p-&gt;t3cdev_p-&gt;name,</div><div class='del'>-			T3_MAX_DEV_NAME_LEN);</div><div class='del'>-	} else {</div><div class='del'>-		pr_debug("%s t3cdev_p or dev_name must be set\n", __func__);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	list_add_tail(&amp;rdev_p-&gt;entry, &amp;rdev_list);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s opening rnic dev %s\n", __func__, rdev_p-&gt;dev_name);</div><div class='del'>-	memset(&amp;rdev_p-&gt;ctrl_qp, 0, sizeof(rdev_p-&gt;ctrl_qp));</div><div class='del'>-	if (!rdev_p-&gt;t3cdev_p)</div><div class='del'>-		rdev_p-&gt;t3cdev_p = dev2t3cdev(netdev_p);</div><div class='del'>-	rdev_p-&gt;t3cdev_p-&gt;ulp = (void *) rdev_p;</div><div class='del'>-</div><div class='del'>-	err = rdev_p-&gt;t3cdev_p-&gt;ctl(rdev_p-&gt;t3cdev_p, GET_EMBEDDED_INFO,</div><div class='del'>-					 &amp;(rdev_p-&gt;fw_info));</div><div class='del'>-	if (err) {</div><div class='del'>-		pr_err("%s t3cdev_p(%p)-&gt;ctl returned error %d\n",</div><div class='del'>-		       __func__, rdev_p-&gt;t3cdev_p, err);</div><div class='del'>-		goto err1;</div><div class='del'>-	}</div><div class='del'>-	if (G_FW_VERSION_MAJOR(rdev_p-&gt;fw_info.fw_vers) != CXIO_FW_MAJ) {</div><div class='del'>-		pr_err("fatal firmware version mismatch: need version %u but adapter has version %u\n",</div><div class='del'>-		       CXIO_FW_MAJ,</div><div class='del'>-		       G_FW_VERSION_MAJOR(rdev_p-&gt;fw_info.fw_vers));</div><div class='del'>-		err = -EINVAL;</div><div class='del'>-		goto err1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	err = rdev_p-&gt;t3cdev_p-&gt;ctl(rdev_p-&gt;t3cdev_p, RDMA_GET_PARAMS,</div><div class='del'>-					 &amp;(rdev_p-&gt;rnic_info));</div><div class='del'>-	if (err) {</div><div class='del'>-		pr_err("%s t3cdev_p(%p)-&gt;ctl returned error %d\n",</div><div class='del'>-		       __func__, rdev_p-&gt;t3cdev_p, err);</div><div class='del'>-		goto err1;</div><div class='del'>-	}</div><div class='del'>-	err = rdev_p-&gt;t3cdev_p-&gt;ctl(rdev_p-&gt;t3cdev_p, GET_PORTS,</div><div class='del'>-				    &amp;(rdev_p-&gt;port_info));</div><div class='del'>-	if (err) {</div><div class='del'>-		pr_err("%s t3cdev_p(%p)-&gt;ctl returned error %d\n",</div><div class='del'>-		       __func__, rdev_p-&gt;t3cdev_p, err);</div><div class='del'>-		goto err1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * qpshift is the number of bits to shift the qpid left in order</div><div class='del'>-	 * to get the correct address of the doorbell for that qp.</div><div class='del'>-	 */</div><div class='del'>-	cxio_init_ucontext(rdev_p, &amp;rdev_p-&gt;uctx);</div><div class='del'>-	rdev_p-&gt;qpshift = PAGE_SHIFT -</div><div class='del'>-			  ilog2(65536 &gt;&gt;</div><div class='del'>-			            ilog2(rdev_p-&gt;rnic_info.udbell_len &gt;&gt;</div><div class='del'>-					      PAGE_SHIFT));</div><div class='del'>-	rdev_p-&gt;qpnr = rdev_p-&gt;rnic_info.udbell_len &gt;&gt; PAGE_SHIFT;</div><div class='del'>-	rdev_p-&gt;qpmask = (65536 &gt;&gt; ilog2(rdev_p-&gt;qpnr)) - 1;</div><div class='del'>-	pr_debug("%s rnic %s info: tpt_base 0x%0x tpt_top 0x%0x num stags %d pbl_base 0x%0x pbl_top 0x%0x rqt_base 0x%0x, rqt_top 0x%0x\n",</div><div class='del'>-		 __func__, rdev_p-&gt;dev_name, rdev_p-&gt;rnic_info.tpt_base,</div><div class='del'>-		 rdev_p-&gt;rnic_info.tpt_top, cxio_num_stags(rdev_p),</div><div class='del'>-		 rdev_p-&gt;rnic_info.pbl_base,</div><div class='del'>-		 rdev_p-&gt;rnic_info.pbl_top, rdev_p-&gt;rnic_info.rqt_base,</div><div class='del'>-		 rdev_p-&gt;rnic_info.rqt_top);</div><div class='del'>-	pr_debug("udbell_len 0x%0x udbell_physbase 0x%lx kdb_addr %p qpshift %lu qpnr %d qpmask 0x%x\n",</div><div class='del'>-		 rdev_p-&gt;rnic_info.udbell_len,</div><div class='del'>-		 rdev_p-&gt;rnic_info.udbell_physbase, rdev_p-&gt;rnic_info.kdb_addr,</div><div class='del'>-		 rdev_p-&gt;qpshift, rdev_p-&gt;qpnr, rdev_p-&gt;qpmask);</div><div class='del'>-</div><div class='del'>-	err = cxio_hal_init_ctrl_qp(rdev_p);</div><div class='del'>-	if (err) {</div><div class='del'>-		pr_err("%s error %d initializing ctrl_qp\n", __func__, err);</div><div class='del'>-		goto err1;</div><div class='del'>-	}</div><div class='del'>-	err = cxio_hal_init_resource(rdev_p, cxio_num_stags(rdev_p), 0,</div><div class='del'>-				     0, T3_MAX_NUM_QP, T3_MAX_NUM_CQ,</div><div class='del'>-				     T3_MAX_NUM_PD);</div><div class='del'>-	if (err) {</div><div class='del'>-		pr_err("%s error %d initializing hal resources\n",</div><div class='del'>-		       __func__, err);</div><div class='del'>-		goto err2;</div><div class='del'>-	}</div><div class='del'>-	err = cxio_hal_pblpool_create(rdev_p);</div><div class='del'>-	if (err) {</div><div class='del'>-		pr_err("%s error %d initializing pbl mem pool\n",</div><div class='del'>-		       __func__, err);</div><div class='del'>-		goto err3;</div><div class='del'>-	}</div><div class='del'>-	err = cxio_hal_rqtpool_create(rdev_p);</div><div class='del'>-	if (err) {</div><div class='del'>-		pr_err("%s error %d initializing rqt mem pool\n",</div><div class='del'>-		       __func__, err);</div><div class='del'>-		goto err4;</div><div class='del'>-	}</div><div class='del'>-	return 0;</div><div class='del'>-err4:</div><div class='del'>-	cxio_hal_pblpool_destroy(rdev_p);</div><div class='del'>-err3:</div><div class='del'>-	cxio_hal_destroy_resource(rdev_p-&gt;rscp);</div><div class='del'>-err2:</div><div class='del'>-	cxio_hal_destroy_ctrl_qp(rdev_p);</div><div class='del'>-err1:</div><div class='del'>-	rdev_p-&gt;t3cdev_p-&gt;ulp = NULL;</div><div class='del'>-	list_del(&amp;rdev_p-&gt;entry);</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_rdev_close(struct cxio_rdev *rdev_p)</div><div class='del'>-{</div><div class='del'>-	if (rdev_p) {</div><div class='del'>-		cxio_hal_pblpool_destroy(rdev_p);</div><div class='del'>-		cxio_hal_rqtpool_destroy(rdev_p);</div><div class='del'>-		list_del(&amp;rdev_p-&gt;entry);</div><div class='del'>-		cxio_hal_destroy_ctrl_qp(rdev_p);</div><div class='del'>-		cxio_hal_destroy_resource(rdev_p-&gt;rscp);</div><div class='del'>-		rdev_p-&gt;t3cdev_p-&gt;ulp = NULL;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int __init cxio_hal_init(void)</div><div class='del'>-{</div><div class='del'>-	if (cxio_hal_init_rhdl_resource(T3_MAX_NUM_RI))</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	t3_register_cpl_handler(CPL_ASYNC_NOTIF, cxio_hal_ev_handler);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void __exit cxio_hal_exit(void)</div><div class='del'>-{</div><div class='del'>-	struct cxio_rdev *rdev, *tmp;</div><div class='del'>-</div><div class='del'>-	t3_register_cpl_handler(CPL_ASYNC_NOTIF, NULL);</div><div class='del'>-	list_for_each_entry_safe(rdev, tmp, &amp;rdev_list, entry)</div><div class='del'>-		cxio_rdev_close(rdev);</div><div class='del'>-	cxio_hal_destroy_rhdl_resource();</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void flush_completed_wrs(struct t3_wq *wq, struct t3_cq *cq)</div><div class='del'>-{</div><div class='del'>-	struct t3_swsq *sqp;</div><div class='del'>-	__u32 ptr = wq-&gt;sq_rptr;</div><div class='del'>-	int count = Q_COUNT(wq-&gt;sq_rptr, wq-&gt;sq_wptr);</div><div class='del'>-</div><div class='del'>-	sqp = wq-&gt;sq + Q_PTR2IDX(ptr, wq-&gt;sq_size_log2);</div><div class='del'>-	while (count--)</div><div class='del'>-		if (!sqp-&gt;signaled) {</div><div class='del'>-			ptr++;</div><div class='del'>-			sqp = wq-&gt;sq + Q_PTR2IDX(ptr,  wq-&gt;sq_size_log2);</div><div class='del'>-		} else if (sqp-&gt;complete) {</div><div class='del'>-</div><div class='del'>-			/*</div><div class='del'>-			 * Insert this completed cqe into the swcq.</div><div class='del'>-			 */</div><div class='del'>-			pr_debug("%s moving cqe into swcq sq idx %ld cq idx %ld\n",</div><div class='del'>-				 __func__, Q_PTR2IDX(ptr,  wq-&gt;sq_size_log2),</div><div class='del'>-				 Q_PTR2IDX(cq-&gt;sw_wptr, cq-&gt;size_log2));</div><div class='del'>-			sqp-&gt;cqe.header |= htonl(V_CQE_SWCQE(1));</div><div class='del'>-			*(cq-&gt;sw_queue + Q_PTR2IDX(cq-&gt;sw_wptr, cq-&gt;size_log2))</div><div class='del'>-				= sqp-&gt;cqe;</div><div class='del'>-			cq-&gt;sw_wptr++;</div><div class='del'>-			sqp-&gt;signaled = 0;</div><div class='del'>-			break;</div><div class='del'>-		} else</div><div class='del'>-			break;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void create_read_req_cqe(struct t3_wq *wq, struct t3_cqe *hw_cqe,</div><div class='del'>-				struct t3_cqe *read_cqe)</div><div class='del'>-{</div><div class='del'>-	read_cqe-&gt;u.scqe.wrid_hi = wq-&gt;oldest_read-&gt;sq_wptr;</div><div class='del'>-	read_cqe-&gt;len = wq-&gt;oldest_read-&gt;read_len;</div><div class='del'>-	read_cqe-&gt;header = htonl(V_CQE_QPID(CQE_QPID(*hw_cqe)) |</div><div class='del'>-				 V_CQE_SWCQE(SW_CQE(*hw_cqe)) |</div><div class='del'>-				 V_CQE_OPCODE(T3_READ_REQ) |</div><div class='del'>-				 V_CQE_TYPE(1));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Return a ptr to the next read wr in the SWSQ or NULL.</div><div class='del'>- */</div><div class='del'>-static void advance_oldest_read(struct t3_wq *wq)</div><div class='del'>-{</div><div class='del'>-</div><div class='del'>-	u32 rptr = wq-&gt;oldest_read - wq-&gt;sq + 1;</div><div class='del'>-	u32 wptr = Q_PTR2IDX(wq-&gt;sq_wptr, wq-&gt;sq_size_log2);</div><div class='del'>-</div><div class='del'>-	while (Q_PTR2IDX(rptr, wq-&gt;sq_size_log2) != wptr) {</div><div class='del'>-		wq-&gt;oldest_read = wq-&gt;sq + Q_PTR2IDX(rptr, wq-&gt;sq_size_log2);</div><div class='del'>-</div><div class='del'>-		if (wq-&gt;oldest_read-&gt;opcode == T3_READ_REQ)</div><div class='del'>-			return;</div><div class='del'>-		rptr++;</div><div class='del'>-	}</div><div class='del'>-	wq-&gt;oldest_read = NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * cxio_poll_cq</div><div class='del'>- *</div><div class='del'>- * Caller must:</div><div class='del'>- *     check the validity of the first CQE,</div><div class='del'>- *     supply the wq assicated with the qpid.</div><div class='del'>- *</div><div class='del'>- * credit: cq credit to return to sge.</div><div class='del'>- * cqe_flushed: 1 iff the CQE is flushed.</div><div class='del'>- * cqe: copy of the polled CQE.</div><div class='del'>- *</div><div class='del'>- * return value:</div><div class='del'>- *     0       CQE returned,</div><div class='del'>- *    -1       CQE skipped, try again.</div><div class='del'>- */</div><div class='del'>-int cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe,</div><div class='del'>-		     u8 *cqe_flushed, u64 *cookie, u32 *credit)</div><div class='del'>-{</div><div class='del'>-	int ret = 0;</div><div class='del'>-	struct t3_cqe *hw_cqe, read_cqe;</div><div class='del'>-</div><div class='del'>-	*cqe_flushed = 0;</div><div class='del'>-	*credit = 0;</div><div class='del'>-	hw_cqe = cxio_next_cqe(cq);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s CQE OOO %d qpid 0x%0x genbit %d type %d status 0x%0x opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x\n",</div><div class='del'>-		 __func__, CQE_OOO(*hw_cqe), CQE_QPID(*hw_cqe),</div><div class='del'>-		 CQE_GENBIT(*hw_cqe), CQE_TYPE(*hw_cqe), CQE_STATUS(*hw_cqe),</div><div class='del'>-		 CQE_OPCODE(*hw_cqe), CQE_LEN(*hw_cqe), CQE_WRID_HI(*hw_cqe),</div><div class='del'>-		 CQE_WRID_LOW(*hw_cqe));</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * skip cqe's not affiliated with a QP.</div><div class='del'>-	 */</div><div class='del'>-	if (wq == NULL) {</div><div class='del'>-		ret = -1;</div><div class='del'>-		goto skip_cqe;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Gotta tweak READ completions:</div><div class='del'>-	 *	1) the cqe doesn't contain the sq_wptr from the wr.</div><div class='del'>-	 *	2) opcode not reflected from the wr.</div><div class='del'>-	 *	3) read_len not reflected from the wr.</div><div class='del'>-	 *	4) cq_type is RQ_TYPE not SQ_TYPE.</div><div class='del'>-	 */</div><div class='del'>-	if (RQ_TYPE(*hw_cqe) &amp;&amp; (CQE_OPCODE(*hw_cqe) == T3_READ_RESP)) {</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * If this is an unsolicited read response, then the read</div><div class='del'>-		 * was generated by the kernel driver as part of peer-2-peer</div><div class='del'>-		 * connection setup.  So ignore the completion.</div><div class='del'>-		 */</div><div class='del'>-		if (!wq-&gt;oldest_read) {</div><div class='del'>-			if (CQE_STATUS(*hw_cqe))</div><div class='del'>-				wq-&gt;error = 1;</div><div class='del'>-			ret = -1;</div><div class='del'>-			goto skip_cqe;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * Don't write to the HWCQ, so create a new read req CQE</div><div class='del'>-		 * in local memory.</div><div class='del'>-		 */</div><div class='del'>-		create_read_req_cqe(wq, hw_cqe, &amp;read_cqe);</div><div class='del'>-		hw_cqe = &amp;read_cqe;</div><div class='del'>-		advance_oldest_read(wq);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * T3A: Discard TERMINATE CQEs.</div><div class='del'>-	 */</div><div class='del'>-	if (CQE_OPCODE(*hw_cqe) == T3_TERMINATE) {</div><div class='del'>-		ret = -1;</div><div class='del'>-		wq-&gt;error = 1;</div><div class='del'>-		goto skip_cqe;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (CQE_STATUS(*hw_cqe) || wq-&gt;error) {</div><div class='del'>-		*cqe_flushed = wq-&gt;error;</div><div class='del'>-		wq-&gt;error = 1;</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * T3A inserts errors into the CQE.  We cannot return</div><div class='del'>-		 * these as work completions.</div><div class='del'>-		 */</div><div class='del'>-		/* incoming write failures */</div><div class='del'>-		if ((CQE_OPCODE(*hw_cqe) == T3_RDMA_WRITE)</div><div class='del'>-		     &amp;&amp; RQ_TYPE(*hw_cqe)) {</div><div class='del'>-			ret = -1;</div><div class='del'>-			goto skip_cqe;</div><div class='del'>-		}</div><div class='del'>-		/* incoming read request failures */</div><div class='del'>-		if ((CQE_OPCODE(*hw_cqe) == T3_READ_RESP) &amp;&amp; SQ_TYPE(*hw_cqe)) {</div><div class='del'>-			ret = -1;</div><div class='del'>-			goto skip_cqe;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		/* incoming SEND with no receive posted failures */</div><div class='del'>-		if (CQE_SEND_OPCODE(*hw_cqe) &amp;&amp; RQ_TYPE(*hw_cqe) &amp;&amp;</div><div class='del'>-		    Q_EMPTY(wq-&gt;rq_rptr, wq-&gt;rq_wptr)) {</div><div class='del'>-			ret = -1;</div><div class='del'>-			goto skip_cqe;</div><div class='del'>-		}</div><div class='del'>-		BUG_ON((*cqe_flushed == 0) &amp;&amp; !SW_CQE(*hw_cqe));</div><div class='del'>-		goto proc_cqe;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * RECV completion.</div><div class='del'>-	 */</div><div class='del'>-	if (RQ_TYPE(*hw_cqe)) {</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * HW only validates 4 bits of MSN.  So we must validate that</div><div class='del'>-		 * the MSN in the SEND is the next expected MSN.  If its not,</div><div class='del'>-		 * then we complete this with TPT_ERR_MSN and mark the wq in</div><div class='del'>-		 * error.</div><div class='del'>-		 */</div><div class='del'>-</div><div class='del'>-		if (Q_EMPTY(wq-&gt;rq_rptr, wq-&gt;rq_wptr)) {</div><div class='del'>-			wq-&gt;error = 1;</div><div class='del'>-			ret = -1;</div><div class='del'>-			goto skip_cqe;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (unlikely((CQE_WRID_MSN(*hw_cqe) != (wq-&gt;rq_rptr + 1)))) {</div><div class='del'>-			wq-&gt;error = 1;</div><div class='del'>-			hw_cqe-&gt;header |= htonl(V_CQE_STATUS(TPT_ERR_MSN));</div><div class='del'>-			goto proc_cqe;</div><div class='del'>-		}</div><div class='del'>-		goto proc_cqe;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If we get here its a send completion.</div><div class='del'>-	 *</div><div class='del'>-	 * Handle out of order completion. These get stuffed</div><div class='del'>-	 * in the SW SQ. Then the SW SQ is walked to move any</div><div class='del'>-	 * now in-order completions into the SW CQ.  This handles</div><div class='del'>-	 * 2 cases:</div><div class='del'>-	 *	1) reaping unsignaled WRs when the first subsequent</div><div class='del'>-	 *	   signaled WR is completed.</div><div class='del'>-	 *	2) out of order read completions.</div><div class='del'>-	 */</div><div class='del'>-	if (!SW_CQE(*hw_cqe) &amp;&amp; (CQE_WRID_SQ_WPTR(*hw_cqe) != wq-&gt;sq_rptr)) {</div><div class='del'>-		struct t3_swsq *sqp;</div><div class='del'>-</div><div class='del'>-		pr_debug("%s out of order completion going in swsq at idx %ld\n",</div><div class='del'>-			 __func__,</div><div class='del'>-			 Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe),</div><div class='del'>-				   wq-&gt;sq_size_log2));</div><div class='del'>-		sqp = wq-&gt;sq +</div><div class='del'>-		      Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq-&gt;sq_size_log2);</div><div class='del'>-		sqp-&gt;cqe = *hw_cqe;</div><div class='del'>-		sqp-&gt;complete = 1;</div><div class='del'>-		ret = -1;</div><div class='del'>-		goto flush_wq;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-proc_cqe:</div><div class='del'>-	*cqe = *hw_cqe;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Reap the associated WR(s) that are freed up with this</div><div class='del'>-	 * completion.</div><div class='del'>-	 */</div><div class='del'>-	if (SQ_TYPE(*hw_cqe)) {</div><div class='del'>-		wq-&gt;sq_rptr = CQE_WRID_SQ_WPTR(*hw_cqe);</div><div class='del'>-		pr_debug("%s completing sq idx %ld\n", __func__,</div><div class='del'>-			 Q_PTR2IDX(wq-&gt;sq_rptr, wq-&gt;sq_size_log2));</div><div class='del'>-		*cookie = wq-&gt;sq[Q_PTR2IDX(wq-&gt;sq_rptr, wq-&gt;sq_size_log2)].wr_id;</div><div class='del'>-		wq-&gt;sq_rptr++;</div><div class='del'>-	} else {</div><div class='del'>-		pr_debug("%s completing rq idx %ld\n", __func__,</div><div class='del'>-			 Q_PTR2IDX(wq-&gt;rq_rptr, wq-&gt;rq_size_log2));</div><div class='del'>-		*cookie = wq-&gt;rq[Q_PTR2IDX(wq-&gt;rq_rptr, wq-&gt;rq_size_log2)].wr_id;</div><div class='del'>-		if (wq-&gt;rq[Q_PTR2IDX(wq-&gt;rq_rptr, wq-&gt;rq_size_log2)].pbl_addr)</div><div class='del'>-			cxio_hal_pblpool_free(wq-&gt;rdev,</div><div class='del'>-				wq-&gt;rq[Q_PTR2IDX(wq-&gt;rq_rptr,</div><div class='del'>-				wq-&gt;rq_size_log2)].pbl_addr, T3_STAG0_PBL_SIZE);</div><div class='del'>-		BUG_ON(Q_EMPTY(wq-&gt;rq_rptr, wq-&gt;rq_wptr));</div><div class='del'>-		wq-&gt;rq_rptr++;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-flush_wq:</div><div class='del'>-	/*</div><div class='del'>-	 * Flush any completed cqes that are now in-order.</div><div class='del'>-	 */</div><div class='del'>-	flush_completed_wrs(wq, cq);</div><div class='del'>-</div><div class='del'>-skip_cqe:</div><div class='del'>-	if (SW_CQE(*hw_cqe)) {</div><div class='del'>-		pr_debug("%s cq %p cqid 0x%x skip sw cqe sw_rptr 0x%x\n",</div><div class='del'>-			 __func__, cq, cq-&gt;cqid, cq-&gt;sw_rptr);</div><div class='del'>-		++cq-&gt;sw_rptr;</div><div class='del'>-	} else {</div><div class='del'>-		pr_debug("%s cq %p cqid 0x%x skip hw cqe rptr 0x%x\n",</div><div class='del'>-			 __func__, cq, cq-&gt;cqid, cq-&gt;rptr);</div><div class='del'>-		++cq-&gt;rptr;</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * T3A: compute credits.</div><div class='del'>-		 */</div><div class='del'>-		if (((cq-&gt;rptr - cq-&gt;wptr) &gt; (1 &lt;&lt; (cq-&gt;size_log2 - 1)))</div><div class='del'>-		    || ((cq-&gt;rptr - cq-&gt;wptr) &gt;= 128)) {</div><div class='del'>-			*credit = cq-&gt;rptr - cq-&gt;wptr;</div><div class='del'>-			cq-&gt;wptr = cq-&gt;rptr;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.h b/drivers/infiniband/hw/cxgb3/cxio_hal.h<br/>deleted file mode 100644<br/>index c64e50b5a548..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/cxio_hal.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/cxio_hal.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,205 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-#ifndef  __CXIO_HAL_H__</div><div class='del'>-#define  __CXIO_HAL_H__</div><div class='del'>-</div><div class='del'>-#include &lt;linux/list.h&gt;</div><div class='del'>-#include &lt;linux/mutex.h&gt;</div><div class='del'>-#include &lt;linux/kfifo.h&gt;</div><div class='del'>-</div><div class='del'>-#include "t3_cpl.h"</div><div class='del'>-#include "t3cdev.h"</div><div class='del'>-#include "cxgb3_ctl_defs.h"</div><div class='del'>-#include "cxio_wr.h"</div><div class='del'>-</div><div class='del'>-#define T3_CTRL_QP_ID    FW_RI_SGEEC_START</div><div class='del'>-#define T3_CTL_QP_TID	 FW_RI_TID_START</div><div class='del'>-#define T3_CTRL_QP_SIZE_LOG2  8</div><div class='del'>-#define T3_CTRL_CQ_ID    0</div><div class='del'>-</div><div class='del'>-#define T3_MAX_NUM_RI (1&lt;&lt;15)</div><div class='del'>-#define T3_MAX_NUM_QP (1&lt;&lt;15)</div><div class='del'>-#define T3_MAX_NUM_CQ (1&lt;&lt;15)</div><div class='del'>-#define T3_MAX_NUM_PD (1&lt;&lt;15)</div><div class='del'>-#define T3_MAX_PBL_SIZE 256</div><div class='del'>-#define T3_MAX_RQ_SIZE 1024</div><div class='del'>-#define T3_MAX_QP_DEPTH (T3_MAX_RQ_SIZE-1)</div><div class='del'>-#define T3_MAX_CQ_DEPTH 65536</div><div class='del'>-#define T3_MAX_NUM_STAG (1&lt;&lt;15)</div><div class='del'>-#define T3_MAX_MR_SIZE 0x100000000ULL</div><div class='del'>-#define T3_PAGESIZE_MASK 0xffff000  /* 4KB-128MB */</div><div class='del'>-</div><div class='del'>-#define T3_STAG_UNSET 0xffffffff</div><div class='del'>-</div><div class='del'>-#define T3_MAX_DEV_NAME_LEN 32</div><div class='del'>-</div><div class='del'>-#define CXIO_FW_MAJ 7</div><div class='del'>-</div><div class='del'>-struct cxio_hal_ctrl_qp {</div><div class='del'>-	u32 wptr;</div><div class='del'>-	u32 rptr;</div><div class='del'>-	struct mutex lock;	/* for the wtpr, can sleep */</div><div class='del'>-	wait_queue_head_t waitq;/* wait for RspQ/CQE msg */</div><div class='del'>-	union t3_wr *workq;	/* the work request queue */</div><div class='del'>-	dma_addr_t dma_addr;	/* pci bus address of the workq */</div><div class='del'>-	DEFINE_DMA_UNMAP_ADDR(mapping);</div><div class='del'>-	void __iomem *doorbell;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct cxio_hal_resource {</div><div class='del'>-	struct kfifo tpt_fifo;</div><div class='del'>-	spinlock_t tpt_fifo_lock;</div><div class='del'>-	struct kfifo qpid_fifo;</div><div class='del'>-	spinlock_t qpid_fifo_lock;</div><div class='del'>-	struct kfifo cqid_fifo;</div><div class='del'>-	spinlock_t cqid_fifo_lock;</div><div class='del'>-	struct kfifo pdid_fifo;</div><div class='del'>-	spinlock_t pdid_fifo_lock;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct cxio_qpid_list {</div><div class='del'>-	struct list_head entry;</div><div class='del'>-	u32 qpid;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct cxio_ucontext {</div><div class='del'>-	struct list_head qpids;</div><div class='del'>-	struct mutex lock;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct cxio_rdev {</div><div class='del'>-	char dev_name[T3_MAX_DEV_NAME_LEN];</div><div class='del'>-	struct t3cdev *t3cdev_p;</div><div class='del'>-	struct rdma_info rnic_info;</div><div class='del'>-	struct adap_ports port_info;</div><div class='del'>-	struct cxio_hal_resource *rscp;</div><div class='del'>-	struct cxio_hal_ctrl_qp ctrl_qp;</div><div class='del'>-	void *ulp;</div><div class='del'>-	unsigned long qpshift;</div><div class='del'>-	u32 qpnr;</div><div class='del'>-	u32 qpmask;</div><div class='del'>-	struct cxio_ucontext uctx;</div><div class='del'>-	struct gen_pool *pbl_pool;</div><div class='del'>-	struct gen_pool *rqt_pool;</div><div class='del'>-	struct list_head entry;</div><div class='del'>-	struct ch_embedded_info fw_info;</div><div class='del'>-	u32	flags;</div><div class='del'>-#define	CXIO_ERROR_FATAL	1</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static inline int cxio_fatal_error(struct cxio_rdev *rdev_p)</div><div class='del'>-{</div><div class='del'>-	return rdev_p-&gt;flags &amp; CXIO_ERROR_FATAL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline int cxio_num_stags(struct cxio_rdev *rdev_p)</div><div class='del'>-{</div><div class='del'>-	return min((int)T3_MAX_NUM_STAG, (int)((rdev_p-&gt;rnic_info.tpt_top - rdev_p-&gt;rnic_info.tpt_base) &gt;&gt; 5));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-typedef void (*cxio_hal_ev_callback_func_t) (struct cxio_rdev * rdev_p,</div><div class='del'>-					     struct sk_buff * skb);</div><div class='del'>-</div><div class='del'>-#define RSPQ_CQID(rsp) (be32_to_cpu(rsp-&gt;cq_ptrid) &amp; 0xffff)</div><div class='del'>-#define RSPQ_CQPTR(rsp) ((be32_to_cpu(rsp-&gt;cq_ptrid) &gt;&gt; 16) &amp; 0xffff)</div><div class='del'>-#define RSPQ_GENBIT(rsp) ((be32_to_cpu(rsp-&gt;flags) &gt;&gt; 16) &amp; 1)</div><div class='del'>-#define RSPQ_OVERFLOW(rsp) ((be32_to_cpu(rsp-&gt;flags) &gt;&gt; 17) &amp; 1)</div><div class='del'>-#define RSPQ_AN(rsp) ((be32_to_cpu(rsp-&gt;flags) &gt;&gt; 18) &amp; 1)</div><div class='del'>-#define RSPQ_SE(rsp) ((be32_to_cpu(rsp-&gt;flags) &gt;&gt; 19) &amp; 1)</div><div class='del'>-#define RSPQ_NOTIFY(rsp) ((be32_to_cpu(rsp-&gt;flags) &gt;&gt; 20) &amp; 1)</div><div class='del'>-#define RSPQ_CQBRANCH(rsp) ((be32_to_cpu(rsp-&gt;flags) &gt;&gt; 21) &amp; 1)</div><div class='del'>-#define RSPQ_CREDIT_THRESH(rsp) ((be32_to_cpu(rsp-&gt;flags) &gt;&gt; 22) &amp; 1)</div><div class='del'>-</div><div class='del'>-struct respQ_msg_t {</div><div class='del'>-	__be32 flags;		/* flit 0 */</div><div class='del'>-	__be32 cq_ptrid;</div><div class='del'>-	__be64 rsvd;		/* flit 1 */</div><div class='del'>-	struct t3_cqe cqe;	/* flits 2-3 */</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum t3_cq_opcode {</div><div class='del'>-	CQ_ARM_AN = 0x2,</div><div class='del'>-	CQ_ARM_SE = 0x6,</div><div class='del'>-	CQ_FORCE_AN = 0x3,</div><div class='del'>-	CQ_CREDIT_UPDATE = 0x7</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-int cxio_rdev_open(struct cxio_rdev *rdev);</div><div class='del'>-void cxio_rdev_close(struct cxio_rdev *rdev);</div><div class='del'>-int cxio_hal_cq_op(struct cxio_rdev *rdev, struct t3_cq *cq,</div><div class='del'>-		   enum t3_cq_opcode op, u32 credit);</div><div class='del'>-int cxio_create_cq(struct cxio_rdev *rdev, struct t3_cq *cq, int kernel);</div><div class='del'>-int cxio_destroy_cq(struct cxio_rdev *rdev, struct t3_cq *cq);</div><div class='del'>-int cxio_resize_cq(struct cxio_rdev *rdev, struct t3_cq *cq);</div><div class='del'>-void cxio_release_ucontext(struct cxio_rdev *rdev, struct cxio_ucontext *uctx);</div><div class='del'>-void cxio_init_ucontext(struct cxio_rdev *rdev, struct cxio_ucontext *uctx);</div><div class='del'>-int cxio_create_qp(struct cxio_rdev *rdev, u32 kernel_domain, struct t3_wq *wq,</div><div class='del'>-		   struct cxio_ucontext *uctx);</div><div class='del'>-int cxio_destroy_qp(struct cxio_rdev *rdev, struct t3_wq *wq,</div><div class='del'>-		    struct cxio_ucontext *uctx);</div><div class='del'>-int cxio_peek_cq(struct t3_wq *wr, struct t3_cq *cq, int opcode);</div><div class='del'>-int cxio_write_pbl(struct cxio_rdev *rdev_p, __be64 *pbl,</div><div class='del'>-		   u32 pbl_addr, u32 pbl_size);</div><div class='del'>-int cxio_register_phys_mem(struct cxio_rdev *rdev, u32 * stag, u32 pdid,</div><div class='del'>-			   enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,</div><div class='del'>-			   u8 page_size, u32 pbl_size, u32 pbl_addr);</div><div class='del'>-int cxio_reregister_phys_mem(struct cxio_rdev *rdev, u32 * stag, u32 pdid,</div><div class='del'>-			   enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,</div><div class='del'>-			   u8 page_size, u32 pbl_size, u32 pbl_addr);</div><div class='del'>-int cxio_dereg_mem(struct cxio_rdev *rdev, u32 stag, u32 pbl_size,</div><div class='del'>-		   u32 pbl_addr);</div><div class='del'>-int cxio_allocate_window(struct cxio_rdev *rdev, u32 * stag, u32 pdid);</div><div class='del'>-int cxio_allocate_stag(struct cxio_rdev *rdev, u32 *stag, u32 pdid, u32 pbl_size, u32 pbl_addr);</div><div class='del'>-int cxio_deallocate_window(struct cxio_rdev *rdev, u32 stag);</div><div class='del'>-int cxio_rdma_init(struct cxio_rdev *rdev, struct t3_rdma_init_attr *attr);</div><div class='del'>-void cxio_register_ev_cb(cxio_hal_ev_callback_func_t ev_cb);</div><div class='del'>-void cxio_unregister_ev_cb(cxio_hal_ev_callback_func_t ev_cb);</div><div class='del'>-u32 cxio_hal_get_pdid(struct cxio_hal_resource *rscp);</div><div class='del'>-void cxio_hal_put_pdid(struct cxio_hal_resource *rscp, u32 pdid);</div><div class='del'>-int __init cxio_hal_init(void);</div><div class='del'>-void __exit cxio_hal_exit(void);</div><div class='del'>-int cxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count);</div><div class='del'>-int cxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count);</div><div class='del'>-void cxio_count_rcqes(struct t3_cq *cq, struct t3_wq *wq, int *count);</div><div class='del'>-void cxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count);</div><div class='del'>-void cxio_flush_hw_cq(struct t3_cq *cq);</div><div class='del'>-int cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe,</div><div class='del'>-		     u8 *cqe_flushed, u64 *cookie, u32 *credit);</div><div class='del'>-int iwch_cxgb3_ofld_send(struct t3cdev *tdev, struct sk_buff *skb);</div><div class='del'>-</div><div class='del'>-#ifdef pr_fmt</div><div class='del'>-#undef pr_fmt</div><div class='del'>-#endif</div><div class='del'>-</div><div class='del'>-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt</div><div class='del'>-</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/cxio_resource.c b/drivers/infiniband/hw/cxgb3/cxio_resource.c<br/>deleted file mode 100644<br/>index c6e7bc4420b6..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/cxio_resource.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/cxio_resource.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,344 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-/* Crude resource management */</div><div class='del'>-#include &lt;linux/kernel.h&gt;</div><div class='del'>-#include &lt;linux/random.h&gt;</div><div class='del'>-#include &lt;linux/slab.h&gt;</div><div class='del'>-#include &lt;linux/kfifo.h&gt;</div><div class='del'>-#include &lt;linux/spinlock.h&gt;</div><div class='del'>-#include &lt;linux/errno.h&gt;</div><div class='del'>-#include "cxio_resource.h"</div><div class='del'>-#include "cxio_hal.h"</div><div class='del'>-</div><div class='del'>-static struct kfifo rhdl_fifo;</div><div class='del'>-static spinlock_t rhdl_fifo_lock;</div><div class='del'>-</div><div class='del'>-#define RANDOM_SIZE 16</div><div class='del'>-</div><div class='del'>-static int __cxio_init_resource_fifo(struct kfifo *fifo,</div><div class='del'>-				   spinlock_t *fifo_lock,</div><div class='del'>-				   u32 nr, u32 skip_low,</div><div class='del'>-				   u32 skip_high,</div><div class='del'>-				   int random)</div><div class='del'>-{</div><div class='del'>-	u32 i, j, entry = 0, idx;</div><div class='del'>-	u32 random_bytes;</div><div class='del'>-	u32 rarray[16];</div><div class='del'>-	spin_lock_init(fifo_lock);</div><div class='del'>-</div><div class='del'>-	if (kfifo_alloc(fifo, nr * sizeof(u32), GFP_KERNEL))</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; skip_low + skip_high; i++)</div><div class='del'>-		kfifo_in(fifo, (unsigned char *) &amp;entry, sizeof(u32));</div><div class='del'>-	if (random) {</div><div class='del'>-		j = 0;</div><div class='del'>-		random_bytes = prandom_u32();</div><div class='del'>-		for (i = 0; i &lt; RANDOM_SIZE; i++)</div><div class='del'>-			rarray[i] = i + skip_low;</div><div class='del'>-		for (i = skip_low + RANDOM_SIZE; i &lt; nr - skip_high; i++) {</div><div class='del'>-			if (j &gt;= RANDOM_SIZE) {</div><div class='del'>-				j = 0;</div><div class='del'>-				random_bytes = prandom_u32();</div><div class='del'>-			}</div><div class='del'>-			idx = (random_bytes &gt;&gt; (j * 2)) &amp; 0xF;</div><div class='del'>-			kfifo_in(fifo,</div><div class='del'>-				(unsigned char *) &amp;rarray[idx],</div><div class='del'>-				sizeof(u32));</div><div class='del'>-			rarray[idx] = i;</div><div class='del'>-			j++;</div><div class='del'>-		}</div><div class='del'>-		for (i = 0; i &lt; RANDOM_SIZE; i++)</div><div class='del'>-			kfifo_in(fifo,</div><div class='del'>-				(unsigned char *) &amp;rarray[i],</div><div class='del'>-				sizeof(u32));</div><div class='del'>-	} else</div><div class='del'>-		for (i = skip_low; i &lt; nr - skip_high; i++)</div><div class='del'>-			kfifo_in(fifo, (unsigned char *) &amp;i, sizeof(u32));</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; skip_low + skip_high; i++)</div><div class='del'>-		if (kfifo_out_locked(fifo, (unsigned char *) &amp;entry,</div><div class='del'>-				sizeof(u32), fifo_lock) != sizeof(u32))</div><div class='del'>-					break;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int cxio_init_resource_fifo(struct kfifo *fifo, spinlock_t * fifo_lock,</div><div class='del'>-				   u32 nr, u32 skip_low, u32 skip_high)</div><div class='del'>-{</div><div class='del'>-	return (__cxio_init_resource_fifo(fifo, fifo_lock, nr, skip_low,</div><div class='del'>-					  skip_high, 0));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int cxio_init_resource_fifo_random(struct kfifo *fifo,</div><div class='del'>-				   spinlock_t * fifo_lock,</div><div class='del'>-				   u32 nr, u32 skip_low, u32 skip_high)</div><div class='del'>-{</div><div class='del'>-</div><div class='del'>-	return (__cxio_init_resource_fifo(fifo, fifo_lock, nr, skip_low,</div><div class='del'>-					  skip_high, 1));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int cxio_init_qpid_fifo(struct cxio_rdev *rdev_p)</div><div class='del'>-{</div><div class='del'>-	u32 i;</div><div class='del'>-</div><div class='del'>-	spin_lock_init(&amp;rdev_p-&gt;rscp-&gt;qpid_fifo_lock);</div><div class='del'>-</div><div class='del'>-	if (kfifo_alloc(&amp;rdev_p-&gt;rscp-&gt;qpid_fifo, T3_MAX_NUM_QP * sizeof(u32),</div><div class='del'>-					      GFP_KERNEL))</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	for (i = 16; i &lt; T3_MAX_NUM_QP; i++)</div><div class='del'>-		if (!(i &amp; rdev_p-&gt;qpmask))</div><div class='del'>-			kfifo_in(&amp;rdev_p-&gt;rscp-&gt;qpid_fifo,</div><div class='del'>-				    (unsigned char *) &amp;i, sizeof(u32));</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_hal_init_rhdl_resource(u32 nr_rhdl)</div><div class='del'>-{</div><div class='del'>-	return cxio_init_resource_fifo(&amp;rhdl_fifo, &amp;rhdl_fifo_lock, nr_rhdl, 1,</div><div class='del'>-				       0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_hal_destroy_rhdl_resource(void)</div><div class='del'>-{</div><div class='del'>-	kfifo_free(&amp;rhdl_fifo);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* nr_* must be power of 2 */</div><div class='del'>-int cxio_hal_init_resource(struct cxio_rdev *rdev_p,</div><div class='del'>-			   u32 nr_tpt, u32 nr_pbl,</div><div class='del'>-			   u32 nr_rqt, u32 nr_qpid, u32 nr_cqid, u32 nr_pdid)</div><div class='del'>-{</div><div class='del'>-	int err = 0;</div><div class='del'>-	struct cxio_hal_resource *rscp;</div><div class='del'>-</div><div class='del'>-	rscp = kmalloc(sizeof(*rscp), GFP_KERNEL);</div><div class='del'>-	if (!rscp)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	rdev_p-&gt;rscp = rscp;</div><div class='del'>-	err = cxio_init_resource_fifo_random(&amp;rscp-&gt;tpt_fifo,</div><div class='del'>-				      &amp;rscp-&gt;tpt_fifo_lock,</div><div class='del'>-				      nr_tpt, 1, 0);</div><div class='del'>-	if (err)</div><div class='del'>-		goto tpt_err;</div><div class='del'>-	err = cxio_init_qpid_fifo(rdev_p);</div><div class='del'>-	if (err)</div><div class='del'>-		goto qpid_err;</div><div class='del'>-	err = cxio_init_resource_fifo(&amp;rscp-&gt;cqid_fifo, &amp;rscp-&gt;cqid_fifo_lock,</div><div class='del'>-				      nr_cqid, 1, 0);</div><div class='del'>-	if (err)</div><div class='del'>-		goto cqid_err;</div><div class='del'>-	err = cxio_init_resource_fifo(&amp;rscp-&gt;pdid_fifo, &amp;rscp-&gt;pdid_fifo_lock,</div><div class='del'>-				      nr_pdid, 1, 0);</div><div class='del'>-	if (err)</div><div class='del'>-		goto pdid_err;</div><div class='del'>-	return 0;</div><div class='del'>-pdid_err:</div><div class='del'>-	kfifo_free(&amp;rscp-&gt;cqid_fifo);</div><div class='del'>-cqid_err:</div><div class='del'>-	kfifo_free(&amp;rscp-&gt;qpid_fifo);</div><div class='del'>-qpid_err:</div><div class='del'>-	kfifo_free(&amp;rscp-&gt;tpt_fifo);</div><div class='del'>-tpt_err:</div><div class='del'>-	return -ENOMEM;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * returns 0 if no resource available</div><div class='del'>- */</div><div class='del'>-static u32 cxio_hal_get_resource(struct kfifo *fifo, spinlock_t * lock)</div><div class='del'>-{</div><div class='del'>-	u32 entry;</div><div class='del'>-	if (kfifo_out_locked(fifo, (unsigned char *) &amp;entry, sizeof(u32), lock))</div><div class='del'>-		return entry;</div><div class='del'>-	else</div><div class='del'>-		return 0;	/* fifo emptry */</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void cxio_hal_put_resource(struct kfifo *fifo, spinlock_t * lock,</div><div class='del'>-		u32 entry)</div><div class='del'>-{</div><div class='del'>-	BUG_ON(</div><div class='del'>-	kfifo_in_locked(fifo, (unsigned char *) &amp;entry, sizeof(u32), lock)</div><div class='del'>-	== 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-u32 cxio_hal_get_stag(struct cxio_hal_resource *rscp)</div><div class='del'>-{</div><div class='del'>-	return cxio_hal_get_resource(&amp;rscp-&gt;tpt_fifo, &amp;rscp-&gt;tpt_fifo_lock);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_hal_put_stag(struct cxio_hal_resource *rscp, u32 stag)</div><div class='del'>-{</div><div class='del'>-	cxio_hal_put_resource(&amp;rscp-&gt;tpt_fifo, &amp;rscp-&gt;tpt_fifo_lock, stag);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-u32 cxio_hal_get_qpid(struct cxio_hal_resource *rscp)</div><div class='del'>-{</div><div class='del'>-	u32 qpid = cxio_hal_get_resource(&amp;rscp-&gt;qpid_fifo,</div><div class='del'>-			&amp;rscp-&gt;qpid_fifo_lock);</div><div class='del'>-	pr_debug("%s qpid 0x%x\n", __func__, qpid);</div><div class='del'>-	return qpid;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_hal_put_qpid(struct cxio_hal_resource *rscp, u32 qpid)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s qpid 0x%x\n", __func__, qpid);</div><div class='del'>-	cxio_hal_put_resource(&amp;rscp-&gt;qpid_fifo, &amp;rscp-&gt;qpid_fifo_lock, qpid);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-u32 cxio_hal_get_cqid(struct cxio_hal_resource *rscp)</div><div class='del'>-{</div><div class='del'>-	return cxio_hal_get_resource(&amp;rscp-&gt;cqid_fifo, &amp;rscp-&gt;cqid_fifo_lock);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_hal_put_cqid(struct cxio_hal_resource *rscp, u32 cqid)</div><div class='del'>-{</div><div class='del'>-	cxio_hal_put_resource(&amp;rscp-&gt;cqid_fifo, &amp;rscp-&gt;cqid_fifo_lock, cqid);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-u32 cxio_hal_get_pdid(struct cxio_hal_resource *rscp)</div><div class='del'>-{</div><div class='del'>-	return cxio_hal_get_resource(&amp;rscp-&gt;pdid_fifo, &amp;rscp-&gt;pdid_fifo_lock);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_hal_put_pdid(struct cxio_hal_resource *rscp, u32 pdid)</div><div class='del'>-{</div><div class='del'>-	cxio_hal_put_resource(&amp;rscp-&gt;pdid_fifo, &amp;rscp-&gt;pdid_fifo_lock, pdid);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_hal_destroy_resource(struct cxio_hal_resource *rscp)</div><div class='del'>-{</div><div class='del'>-	kfifo_free(&amp;rscp-&gt;tpt_fifo);</div><div class='del'>-	kfifo_free(&amp;rscp-&gt;cqid_fifo);</div><div class='del'>-	kfifo_free(&amp;rscp-&gt;qpid_fifo);</div><div class='del'>-	kfifo_free(&amp;rscp-&gt;pdid_fifo);</div><div class='del'>-	kfree(rscp);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * PBL Memory Manager.  Uses Linux generic allocator.</div><div class='del'>- */</div><div class='del'>-</div><div class='del'>-#define MIN_PBL_SHIFT 8			/* 256B == min PBL size (32 entries) */</div><div class='del'>-</div><div class='del'>-u32 cxio_hal_pblpool_alloc(struct cxio_rdev *rdev_p, int size)</div><div class='del'>-{</div><div class='del'>-	unsigned long addr = gen_pool_alloc(rdev_p-&gt;pbl_pool, size);</div><div class='del'>-	pr_debug("%s addr 0x%x size %d\n", __func__, (u32)addr, size);</div><div class='del'>-	return (u32)addr;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_hal_pblpool_free(struct cxio_rdev *rdev_p, u32 addr, int size)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s addr 0x%x size %d\n", __func__, addr, size);</div><div class='del'>-	gen_pool_free(rdev_p-&gt;pbl_pool, (unsigned long)addr, size);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_hal_pblpool_create(struct cxio_rdev *rdev_p)</div><div class='del'>-{</div><div class='del'>-	unsigned pbl_start, pbl_chunk;</div><div class='del'>-</div><div class='del'>-	rdev_p-&gt;pbl_pool = gen_pool_create(MIN_PBL_SHIFT, -1);</div><div class='del'>-	if (!rdev_p-&gt;pbl_pool)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	pbl_start = rdev_p-&gt;rnic_info.pbl_base;</div><div class='del'>-	pbl_chunk = rdev_p-&gt;rnic_info.pbl_top - pbl_start + 1;</div><div class='del'>-</div><div class='del'>-	while (pbl_start &lt; rdev_p-&gt;rnic_info.pbl_top) {</div><div class='del'>-		pbl_chunk = min(rdev_p-&gt;rnic_info.pbl_top - pbl_start + 1,</div><div class='del'>-				pbl_chunk);</div><div class='del'>-		if (gen_pool_add(rdev_p-&gt;pbl_pool, pbl_start, pbl_chunk, -1)) {</div><div class='del'>-			pr_debug("%s failed to add PBL chunk (%x/%x)\n",</div><div class='del'>-				 __func__, pbl_start, pbl_chunk);</div><div class='del'>-			if (pbl_chunk &lt;= 1024 &lt;&lt; MIN_PBL_SHIFT) {</div><div class='del'>-				pr_warn("%s: Failed to add all PBL chunks (%x/%x)\n",</div><div class='del'>-					__func__, pbl_start,</div><div class='del'>-					rdev_p-&gt;rnic_info.pbl_top - pbl_start);</div><div class='del'>-				return 0;</div><div class='del'>-			}</div><div class='del'>-			pbl_chunk &gt;&gt;= 1;</div><div class='del'>-		} else {</div><div class='del'>-			pr_debug("%s added PBL chunk (%x/%x)\n",</div><div class='del'>-				 __func__, pbl_start, pbl_chunk);</div><div class='del'>-			pbl_start += pbl_chunk;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_hal_pblpool_destroy(struct cxio_rdev *rdev_p)</div><div class='del'>-{</div><div class='del'>-	gen_pool_destroy(rdev_p-&gt;pbl_pool);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * RQT Memory Manager.  Uses Linux generic allocator.</div><div class='del'>- */</div><div class='del'>-</div><div class='del'>-#define MIN_RQT_SHIFT 10	/* 1KB == mini RQT size (16 entries) */</div><div class='del'>-#define RQT_CHUNK 2*1024*1024</div><div class='del'>-</div><div class='del'>-u32 cxio_hal_rqtpool_alloc(struct cxio_rdev *rdev_p, int size)</div><div class='del'>-{</div><div class='del'>-	unsigned long addr = gen_pool_alloc(rdev_p-&gt;rqt_pool, size &lt;&lt; 6);</div><div class='del'>-	pr_debug("%s addr 0x%x size %d\n", __func__, (u32)addr, size &lt;&lt; 6);</div><div class='del'>-	return (u32)addr;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_hal_rqtpool_free(struct cxio_rdev *rdev_p, u32 addr, int size)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s addr 0x%x size %d\n", __func__, addr, size &lt;&lt; 6);</div><div class='del'>-	gen_pool_free(rdev_p-&gt;rqt_pool, (unsigned long)addr, size &lt;&lt; 6);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int cxio_hal_rqtpool_create(struct cxio_rdev *rdev_p)</div><div class='del'>-{</div><div class='del'>-	unsigned long i;</div><div class='del'>-	rdev_p-&gt;rqt_pool = gen_pool_create(MIN_RQT_SHIFT, -1);</div><div class='del'>-	if (rdev_p-&gt;rqt_pool)</div><div class='del'>-		for (i = rdev_p-&gt;rnic_info.rqt_base;</div><div class='del'>-		     i &lt;= rdev_p-&gt;rnic_info.rqt_top - RQT_CHUNK + 1;</div><div class='del'>-		     i += RQT_CHUNK)</div><div class='del'>-			gen_pool_add(rdev_p-&gt;rqt_pool, i, RQT_CHUNK, -1);</div><div class='del'>-	return rdev_p-&gt;rqt_pool ? 0 : -ENOMEM;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void cxio_hal_rqtpool_destroy(struct cxio_rdev *rdev_p)</div><div class='del'>-{</div><div class='del'>-	gen_pool_destroy(rdev_p-&gt;rqt_pool);</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/cxio_resource.h b/drivers/infiniband/hw/cxgb3/cxio_resource.h<br/>deleted file mode 100644<br/>index a2703a3d882d..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/cxio_resource.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/cxio_resource.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,69 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-#ifndef __CXIO_RESOURCE_H__</div><div class='del'>-#define __CXIO_RESOURCE_H__</div><div class='del'>-</div><div class='del'>-#include &lt;linux/kernel.h&gt;</div><div class='del'>-#include &lt;linux/random.h&gt;</div><div class='del'>-#include &lt;linux/slab.h&gt;</div><div class='del'>-#include &lt;linux/kfifo.h&gt;</div><div class='del'>-#include &lt;linux/spinlock.h&gt;</div><div class='del'>-#include &lt;linux/errno.h&gt;</div><div class='del'>-#include &lt;linux/genalloc.h&gt;</div><div class='del'>-#include "cxio_hal.h"</div><div class='del'>-</div><div class='del'>-extern int cxio_hal_init_rhdl_resource(u32 nr_rhdl);</div><div class='del'>-extern void cxio_hal_destroy_rhdl_resource(void);</div><div class='del'>-extern int cxio_hal_init_resource(struct cxio_rdev *rdev_p,</div><div class='del'>-				  u32 nr_tpt, u32 nr_pbl,</div><div class='del'>-				  u32 nr_rqt, u32 nr_qpid, u32 nr_cqid,</div><div class='del'>-				  u32 nr_pdid);</div><div class='del'>-extern u32 cxio_hal_get_stag(struct cxio_hal_resource *rscp);</div><div class='del'>-extern void cxio_hal_put_stag(struct cxio_hal_resource *rscp, u32 stag);</div><div class='del'>-extern u32 cxio_hal_get_qpid(struct cxio_hal_resource *rscp);</div><div class='del'>-extern void cxio_hal_put_qpid(struct cxio_hal_resource *rscp, u32 qpid);</div><div class='del'>-extern u32 cxio_hal_get_cqid(struct cxio_hal_resource *rscp);</div><div class='del'>-extern void cxio_hal_put_cqid(struct cxio_hal_resource *rscp, u32 cqid);</div><div class='del'>-extern void cxio_hal_destroy_resource(struct cxio_hal_resource *rscp);</div><div class='del'>-</div><div class='del'>-#define PBL_OFF(rdev_p, a) ( (a) - (rdev_p)-&gt;rnic_info.pbl_base )</div><div class='del'>-extern int cxio_hal_pblpool_create(struct cxio_rdev *rdev_p);</div><div class='del'>-extern void cxio_hal_pblpool_destroy(struct cxio_rdev *rdev_p);</div><div class='del'>-extern u32 cxio_hal_pblpool_alloc(struct cxio_rdev *rdev_p, int size);</div><div class='del'>-extern void cxio_hal_pblpool_free(struct cxio_rdev *rdev_p, u32 addr, int size);</div><div class='del'>-</div><div class='del'>-#define RQT_OFF(rdev_p, a) ( (a) - (rdev_p)-&gt;rnic_info.rqt_base )</div><div class='del'>-extern int cxio_hal_rqtpool_create(struct cxio_rdev *rdev_p);</div><div class='del'>-extern void cxio_hal_rqtpool_destroy(struct cxio_rdev *rdev_p);</div><div class='del'>-extern u32 cxio_hal_rqtpool_alloc(struct cxio_rdev *rdev_p, int size);</div><div class='del'>-extern void cxio_hal_rqtpool_free(struct cxio_rdev *rdev_p, u32 addr, int size);</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/cxio_wr.h b/drivers/infiniband/hw/cxgb3/cxio_wr.h<br/>deleted file mode 100644<br/>index 53aa5c36247a..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/cxio_wr.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/cxio_wr.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,802 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-#ifndef __CXIO_WR_H__</div><div class='del'>-#define __CXIO_WR_H__</div><div class='del'>-</div><div class='del'>-#include &lt;asm/io.h&gt;</div><div class='del'>-#include &lt;linux/pci.h&gt;</div><div class='del'>-#include &lt;linux/timer.h&gt;</div><div class='del'>-#include "firmware_exports.h"</div><div class='del'>-</div><div class='del'>-#define T3_MAX_SGE      4</div><div class='del'>-#define T3_MAX_INLINE	64</div><div class='del'>-#define T3_STAG0_PBL_SIZE (2 * T3_MAX_SGE &lt;&lt; 3)</div><div class='del'>-#define T3_STAG0_MAX_PBE_LEN (128 * 1024 * 1024)</div><div class='del'>-#define T3_STAG0_PAGE_SHIFT 15</div><div class='del'>-</div><div class='del'>-#define Q_EMPTY(rptr,wptr) ((rptr)==(wptr))</div><div class='del'>-#define Q_FULL(rptr,wptr,size_log2)  ( (((wptr)-(rptr))&gt;&gt;(size_log2)) &amp;&amp; \</div><div class='del'>-				       ((rptr)!=(wptr)) )</div><div class='del'>-#define Q_GENBIT(ptr,size_log2) (!(((ptr)&gt;&gt;size_log2)&amp;0x1))</div><div class='del'>-#define Q_FREECNT(rptr,wptr,size_log2) ((1UL&lt;&lt;size_log2)-((wptr)-(rptr)))</div><div class='del'>-#define Q_COUNT(rptr,wptr) ((wptr)-(rptr))</div><div class='del'>-#define Q_PTR2IDX(ptr,size_log2) (ptr &amp; ((1UL&lt;&lt;size_log2)-1))</div><div class='del'>-</div><div class='del'>-static inline void ring_doorbell(void __iomem *doorbell, u32 qpid)</div><div class='del'>-{</div><div class='del'>-	writel(((1&lt;&lt;31) | qpid), doorbell);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-#define SEQ32_GE(x,y) (!( (((u32) (x)) - ((u32) (y))) &amp; 0x80000000 ))</div><div class='del'>-</div><div class='del'>-enum t3_wr_flags {</div><div class='del'>-	T3_COMPLETION_FLAG = 0x01,</div><div class='del'>-	T3_NOTIFY_FLAG = 0x02,</div><div class='del'>-	T3_SOLICITED_EVENT_FLAG = 0x04,</div><div class='del'>-	T3_READ_FENCE_FLAG = 0x08,</div><div class='del'>-	T3_LOCAL_FENCE_FLAG = 0x10</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-enum t3_wr_opcode {</div><div class='del'>-	T3_WR_BP = FW_WROPCODE_RI_BYPASS,</div><div class='del'>-	T3_WR_SEND = FW_WROPCODE_RI_SEND,</div><div class='del'>-	T3_WR_WRITE = FW_WROPCODE_RI_RDMA_WRITE,</div><div class='del'>-	T3_WR_READ = FW_WROPCODE_RI_RDMA_READ,</div><div class='del'>-	T3_WR_INV_STAG = FW_WROPCODE_RI_LOCAL_INV,</div><div class='del'>-	T3_WR_BIND = FW_WROPCODE_RI_BIND_MW,</div><div class='del'>-	T3_WR_RCV = FW_WROPCODE_RI_RECEIVE,</div><div class='del'>-	T3_WR_INIT = FW_WROPCODE_RI_RDMA_INIT,</div><div class='del'>-	T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP,</div><div class='del'>-	T3_WR_FASTREG = FW_WROPCODE_RI_FASTREGISTER_MR</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-enum t3_rdma_opcode {</div><div class='del'>-	T3_RDMA_WRITE,		/* IETF RDMAP v1.0 ... */</div><div class='del'>-	T3_READ_REQ,</div><div class='del'>-	T3_READ_RESP,</div><div class='del'>-	T3_SEND,</div><div class='del'>-	T3_SEND_WITH_INV,</div><div class='del'>-	T3_SEND_WITH_SE,</div><div class='del'>-	T3_SEND_WITH_SE_INV,</div><div class='del'>-	T3_TERMINATE,</div><div class='del'>-	T3_RDMA_INIT,		/* CHELSIO RI specific ... */</div><div class='del'>-	T3_BIND_MW,</div><div class='del'>-	T3_FAST_REGISTER,</div><div class='del'>-	T3_LOCAL_INV,</div><div class='del'>-	T3_QP_MOD,</div><div class='del'>-	T3_BYPASS,</div><div class='del'>-	T3_RDMA_READ_REQ_WITH_INV,</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-static inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop)</div><div class='del'>-{</div><div class='del'>-	switch (wrop) {</div><div class='del'>-		case T3_WR_BP: return T3_BYPASS;</div><div class='del'>-		case T3_WR_SEND: return T3_SEND;</div><div class='del'>-		case T3_WR_WRITE: return T3_RDMA_WRITE;</div><div class='del'>-		case T3_WR_READ: return T3_READ_REQ;</div><div class='del'>-		case T3_WR_INV_STAG: return T3_LOCAL_INV;</div><div class='del'>-		case T3_WR_BIND: return T3_BIND_MW;</div><div class='del'>-		case T3_WR_INIT: return T3_RDMA_INIT;</div><div class='del'>-		case T3_WR_QP_MOD: return T3_QP_MOD;</div><div class='del'>-		case T3_WR_FASTREG: return T3_FAST_REGISTER;</div><div class='del'>-		default: break;</div><div class='del'>-	}</div><div class='del'>-	return -1;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-</div><div class='del'>-/* Work request id */</div><div class='del'>-union t3_wrid {</div><div class='del'>-	struct {</div><div class='del'>-		u32 hi;</div><div class='del'>-		u32 low;</div><div class='del'>-	} id0;</div><div class='del'>-	u64 id1;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define WRID(wrid)		(wrid.id1)</div><div class='del'>-#define WRID_GEN(wrid)		(wrid.id0.wr_gen)</div><div class='del'>-#define WRID_IDX(wrid)		(wrid.id0.wr_idx)</div><div class='del'>-#define WRID_LO(wrid)		(wrid.id0.wr_lo)</div><div class='del'>-</div><div class='del'>-struct fw_riwrh {</div><div class='del'>-	__be32 op_seop_flags;</div><div class='del'>-	__be32 gen_tid_len;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define S_FW_RIWR_OP		24</div><div class='del'>-#define M_FW_RIWR_OP		0xff</div><div class='del'>-#define V_FW_RIWR_OP(x)		((x) &lt;&lt; S_FW_RIWR_OP)</div><div class='del'>-#define G_FW_RIWR_OP(x)	((((x) &gt;&gt; S_FW_RIWR_OP)) &amp; M_FW_RIWR_OP)</div><div class='del'>-</div><div class='del'>-#define S_FW_RIWR_SOPEOP	22</div><div class='del'>-#define M_FW_RIWR_SOPEOP	0x3</div><div class='del'>-#define V_FW_RIWR_SOPEOP(x)	((x) &lt;&lt; S_FW_RIWR_SOPEOP)</div><div class='del'>-</div><div class='del'>-#define S_FW_RIWR_FLAGS		8</div><div class='del'>-#define M_FW_RIWR_FLAGS		0x3fffff</div><div class='del'>-#define V_FW_RIWR_FLAGS(x)	((x) &lt;&lt; S_FW_RIWR_FLAGS)</div><div class='del'>-#define G_FW_RIWR_FLAGS(x)	((((x) &gt;&gt; S_FW_RIWR_FLAGS)) &amp; M_FW_RIWR_FLAGS)</div><div class='del'>-</div><div class='del'>-#define S_FW_RIWR_TID		8</div><div class='del'>-#define V_FW_RIWR_TID(x)	((x) &lt;&lt; S_FW_RIWR_TID)</div><div class='del'>-</div><div class='del'>-#define S_FW_RIWR_LEN		0</div><div class='del'>-#define V_FW_RIWR_LEN(x)	((x) &lt;&lt; S_FW_RIWR_LEN)</div><div class='del'>-</div><div class='del'>-#define S_FW_RIWR_GEN           31</div><div class='del'>-#define V_FW_RIWR_GEN(x)        ((x)  &lt;&lt; S_FW_RIWR_GEN)</div><div class='del'>-</div><div class='del'>-struct t3_sge {</div><div class='del'>-	__be32 stag;</div><div class='del'>-	__be32 len;</div><div class='del'>-	__be64 to;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* If num_sgle is zero, flit 5+ contains immediate data.*/</div><div class='del'>-struct t3_send_wr {</div><div class='del'>-	struct fw_riwrh wrh;	/* 0 */</div><div class='del'>-	union t3_wrid wrid;	/* 1 */</div><div class='del'>-</div><div class='del'>-	u8 rdmaop;		/* 2 */</div><div class='del'>-	u8 reserved[3];</div><div class='del'>-	__be32 rem_stag;</div><div class='del'>-	__be32 plen;		/* 3 */</div><div class='del'>-	__be32 num_sgle;</div><div class='del'>-	struct t3_sge sgl[T3_MAX_SGE];	/* 4+ */</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define T3_MAX_FASTREG_DEPTH 10</div><div class='del'>-#define T3_MAX_FASTREG_FRAG 10</div><div class='del'>-</div><div class='del'>-struct t3_fastreg_wr {</div><div class='del'>-	struct fw_riwrh wrh;	/* 0 */</div><div class='del'>-	union t3_wrid wrid;	/* 1 */</div><div class='del'>-	__be32 stag;		/* 2 */</div><div class='del'>-	__be32 len;</div><div class='del'>-	__be32 va_base_hi;	/* 3 */</div><div class='del'>-	__be32 va_base_lo_fbo;</div><div class='del'>-	__be32 page_type_perms; /* 4 */</div><div class='del'>-	__be32 reserved1;</div><div class='del'>-	__be64 pbl_addrs[0];	/* 5+ */</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * If a fastreg wr spans multiple wqes, then the 2nd fragment look like this.</div><div class='del'>- */</div><div class='del'>-struct t3_pbl_frag {</div><div class='del'>-	struct fw_riwrh wrh;	/* 0 */</div><div class='del'>-	__be64 pbl_addrs[14];	/* 1..14 */</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define S_FR_PAGE_COUNT		24</div><div class='del'>-#define M_FR_PAGE_COUNT		0xff</div><div class='del'>-#define V_FR_PAGE_COUNT(x)	((x) &lt;&lt; S_FR_PAGE_COUNT)</div><div class='del'>-#define G_FR_PAGE_COUNT(x)	((((x) &gt;&gt; S_FR_PAGE_COUNT)) &amp; M_FR_PAGE_COUNT)</div><div class='del'>-</div><div class='del'>-#define S_FR_PAGE_SIZE		16</div><div class='del'>-#define M_FR_PAGE_SIZE		0x1f</div><div class='del'>-#define V_FR_PAGE_SIZE(x)	((x) &lt;&lt; S_FR_PAGE_SIZE)</div><div class='del'>-#define G_FR_PAGE_SIZE(x)	((((x) &gt;&gt; S_FR_PAGE_SIZE)) &amp; M_FR_PAGE_SIZE)</div><div class='del'>-</div><div class='del'>-#define S_FR_TYPE		8</div><div class='del'>-#define M_FR_TYPE		0x1</div><div class='del'>-#define V_FR_TYPE(x)		((x) &lt;&lt; S_FR_TYPE)</div><div class='del'>-#define G_FR_TYPE(x)		((((x) &gt;&gt; S_FR_TYPE)) &amp; M_FR_TYPE)</div><div class='del'>-</div><div class='del'>-#define S_FR_PERMS		0</div><div class='del'>-#define M_FR_PERMS		0xff</div><div class='del'>-#define V_FR_PERMS(x)		((x) &lt;&lt; S_FR_PERMS)</div><div class='del'>-#define G_FR_PERMS(x)		((((x) &gt;&gt; S_FR_PERMS)) &amp; M_FR_PERMS)</div><div class='del'>-</div><div class='del'>-struct t3_local_inv_wr {</div><div class='del'>-	struct fw_riwrh wrh;	/* 0 */</div><div class='del'>-	union t3_wrid wrid;	/* 1 */</div><div class='del'>-	__be32 stag;		/* 2 */</div><div class='del'>-	__be32 reserved;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct t3_rdma_write_wr {</div><div class='del'>-	struct fw_riwrh wrh;	/* 0 */</div><div class='del'>-	union t3_wrid wrid;	/* 1 */</div><div class='del'>-	u8 rdmaop;		/* 2 */</div><div class='del'>-	u8 reserved[3];</div><div class='del'>-	__be32 stag_sink;</div><div class='del'>-	__be64 to_sink;		/* 3 */</div><div class='del'>-	__be32 plen;		/* 4 */</div><div class='del'>-	__be32 num_sgle;</div><div class='del'>-	struct t3_sge sgl[T3_MAX_SGE];	/* 5+ */</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct t3_rdma_read_wr {</div><div class='del'>-	struct fw_riwrh wrh;	/* 0 */</div><div class='del'>-	union t3_wrid wrid;	/* 1 */</div><div class='del'>-	u8 rdmaop;		/* 2 */</div><div class='del'>-	u8 local_inv;</div><div class='del'>-	u8 reserved[2];</div><div class='del'>-	__be32 rem_stag;</div><div class='del'>-	__be64 rem_to;		/* 3 */</div><div class='del'>-	__be32 local_stag;	/* 4 */</div><div class='del'>-	__be32 local_len;</div><div class='del'>-	__be64 local_to;	/* 5 */</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct t3_bind_mw_wr {</div><div class='del'>-	struct fw_riwrh wrh;	/* 0 */</div><div class='del'>-	union t3_wrid wrid;	/* 1 */</div><div class='del'>-	u16 reserved;		/* 2 */</div><div class='del'>-	u8 type;</div><div class='del'>-	u8 perms;</div><div class='del'>-	__be32 mr_stag;</div><div class='del'>-	__be32 mw_stag;		/* 3 */</div><div class='del'>-	__be32 mw_len;</div><div class='del'>-	__be64 mw_va;		/* 4 */</div><div class='del'>-	__be32 mr_pbl_addr;	/* 5 */</div><div class='del'>-	u8 reserved2[3];</div><div class='del'>-	u8 mr_pagesz;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct t3_receive_wr {</div><div class='del'>-	struct fw_riwrh wrh;	/* 0 */</div><div class='del'>-	union t3_wrid wrid;	/* 1 */</div><div class='del'>-	u8 pagesz[T3_MAX_SGE];</div><div class='del'>-	__be32 num_sgle;		/* 2 */</div><div class='del'>-	struct t3_sge sgl[T3_MAX_SGE];	/* 3+ */</div><div class='del'>-	__be32 pbl_addr[T3_MAX_SGE];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct t3_bypass_wr {</div><div class='del'>-	struct fw_riwrh wrh;</div><div class='del'>-	union t3_wrid wrid;	/* 1 */</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct t3_modify_qp_wr {</div><div class='del'>-	struct fw_riwrh wrh;	/* 0 */</div><div class='del'>-	union t3_wrid wrid;	/* 1 */</div><div class='del'>-	__be32 flags;		/* 2 */</div><div class='del'>-	__be32 quiesce;		/* 2 */</div><div class='del'>-	__be32 max_ird;		/* 3 */</div><div class='del'>-	__be32 max_ord;		/* 3 */</div><div class='del'>-	__be64 sge_cmd;		/* 4 */</div><div class='del'>-	__be64 ctx1;		/* 5 */</div><div class='del'>-	__be64 ctx0;		/* 6 */</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum t3_modify_qp_flags {</div><div class='del'>-	MODQP_QUIESCE  = 0x01,</div><div class='del'>-	MODQP_MAX_IRD  = 0x02,</div><div class='del'>-	MODQP_MAX_ORD  = 0x04,</div><div class='del'>-	MODQP_WRITE_EC = 0x08,</div><div class='del'>-	MODQP_READ_EC  = 0x10,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-</div><div class='del'>-enum t3_mpa_attrs {</div><div class='del'>-	uP_RI_MPA_RX_MARKER_ENABLE = 0x1,</div><div class='del'>-	uP_RI_MPA_TX_MARKER_ENABLE = 0x2,</div><div class='del'>-	uP_RI_MPA_CRC_ENABLE = 0x4,</div><div class='del'>-	uP_RI_MPA_IETF_ENABLE = 0x8</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-enum t3_qp_caps {</div><div class='del'>-	uP_RI_QP_RDMA_READ_ENABLE = 0x01,</div><div class='del'>-	uP_RI_QP_RDMA_WRITE_ENABLE = 0x02,</div><div class='del'>-	uP_RI_QP_BIND_ENABLE = 0x04,</div><div class='del'>-	uP_RI_QP_FAST_REGISTER_ENABLE = 0x08,</div><div class='del'>-	uP_RI_QP_STAG0_ENABLE = 0x10</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-enum rdma_init_rtr_types {</div><div class='del'>-	RTR_READ = 1,</div><div class='del'>-	RTR_WRITE = 2,</div><div class='del'>-	RTR_SEND = 3,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define S_RTR_TYPE	2</div><div class='del'>-#define M_RTR_TYPE	0x3</div><div class='del'>-#define V_RTR_TYPE(x)	((x) &lt;&lt; S_RTR_TYPE)</div><div class='del'>-#define G_RTR_TYPE(x)	((((x) &gt;&gt; S_RTR_TYPE)) &amp; M_RTR_TYPE)</div><div class='del'>-</div><div class='del'>-#define S_CHAN		4</div><div class='del'>-#define M_CHAN		0x3</div><div class='del'>-#define V_CHAN(x)	((x) &lt;&lt; S_CHAN)</div><div class='del'>-#define G_CHAN(x)	((((x) &gt;&gt; S_CHAN)) &amp; M_CHAN)</div><div class='del'>-</div><div class='del'>-struct t3_rdma_init_attr {</div><div class='del'>-	u32 tid;</div><div class='del'>-	u32 qpid;</div><div class='del'>-	u32 pdid;</div><div class='del'>-	u32 scqid;</div><div class='del'>-	u32 rcqid;</div><div class='del'>-	u32 rq_addr;</div><div class='del'>-	u32 rq_size;</div><div class='del'>-	enum t3_mpa_attrs mpaattrs;</div><div class='del'>-	enum t3_qp_caps qpcaps;</div><div class='del'>-	u16 tcp_emss;</div><div class='del'>-	u32 ord;</div><div class='del'>-	u32 ird;</div><div class='del'>-	u64 qp_dma_addr;</div><div class='del'>-	u32 qp_dma_size;</div><div class='del'>-	enum rdma_init_rtr_types rtr_type;</div><div class='del'>-	u16 flags;</div><div class='del'>-	u16 rqe_count;</div><div class='del'>-	u32 irs;</div><div class='del'>-	u32 chan;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct t3_rdma_init_wr {</div><div class='del'>-	struct fw_riwrh wrh;	/* 0 */</div><div class='del'>-	union t3_wrid wrid;	/* 1 */</div><div class='del'>-	__be32 qpid;		/* 2 */</div><div class='del'>-	__be32 pdid;</div><div class='del'>-	__be32 scqid;		/* 3 */</div><div class='del'>-	__be32 rcqid;</div><div class='del'>-	__be32 rq_addr;		/* 4 */</div><div class='del'>-	__be32 rq_size;</div><div class='del'>-	u8 mpaattrs;		/* 5 */</div><div class='del'>-	u8 qpcaps;</div><div class='del'>-	__be16 ulpdu_size;</div><div class='del'>-	__be16 flags_rtr_type;</div><div class='del'>-	__be16 rqe_count;</div><div class='del'>-	__be32 ord;		/* 6 */</div><div class='del'>-	__be32 ird;</div><div class='del'>-	__be64 qp_dma_addr;	/* 7 */</div><div class='del'>-	__be32 qp_dma_size;	/* 8 */</div><div class='del'>-	__be32 irs;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct t3_genbit {</div><div class='del'>-	u64 flit[15];</div><div class='del'>-	__be64 genbit;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct t3_wq_in_err {</div><div class='del'>-	u64 flit[13];</div><div class='del'>-	u64 err;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum rdma_init_wr_flags {</div><div class='del'>-	MPA_INITIATOR = (1&lt;&lt;0),</div><div class='del'>-	PRIV_QP = (1&lt;&lt;1),</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-union t3_wr {</div><div class='del'>-	struct t3_send_wr send;</div><div class='del'>-	struct t3_rdma_write_wr write;</div><div class='del'>-	struct t3_rdma_read_wr read;</div><div class='del'>-	struct t3_receive_wr recv;</div><div class='del'>-	struct t3_fastreg_wr fastreg;</div><div class='del'>-	struct t3_pbl_frag pbl_frag;</div><div class='del'>-	struct t3_local_inv_wr local_inv;</div><div class='del'>-	struct t3_bind_mw_wr bind;</div><div class='del'>-	struct t3_bypass_wr bypass;</div><div class='del'>-	struct t3_rdma_init_wr init;</div><div class='del'>-	struct t3_modify_qp_wr qp_mod;</div><div class='del'>-	struct t3_genbit genbit;</div><div class='del'>-	struct t3_wq_in_err wq_in_err;</div><div class='del'>-	__be64 flit[16];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define T3_SQ_CQE_FLIT	  13</div><div class='del'>-#define T3_SQ_COOKIE_FLIT 14</div><div class='del'>-</div><div class='del'>-#define T3_RQ_COOKIE_FLIT 13</div><div class='del'>-#define T3_RQ_CQE_FLIT	  14</div><div class='del'>-</div><div class='del'>-static inline enum t3_wr_opcode fw_riwrh_opcode(struct fw_riwrh *wqe)</div><div class='del'>-{</div><div class='del'>-	return G_FW_RIWR_OP(be32_to_cpu(wqe-&gt;op_seop_flags));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-enum t3_wr_hdr_bits {</div><div class='del'>-	T3_EOP = 1,</div><div class='del'>-	T3_SOP = 2,</div><div class='del'>-	T3_SOPEOP = T3_EOP|T3_SOP,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static inline void build_fw_riwrh(struct fw_riwrh *wqe, enum t3_wr_opcode op,</div><div class='del'>-				  enum t3_wr_flags flags, u8 genbit, u32 tid,</div><div class='del'>-				  u8 len, u8 sopeop)</div><div class='del'>-{</div><div class='del'>-	wqe-&gt;op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(op) |</div><div class='del'>-					 V_FW_RIWR_SOPEOP(sopeop) |</div><div class='del'>-					 V_FW_RIWR_FLAGS(flags));</div><div class='del'>-	wmb();</div><div class='del'>-	wqe-&gt;gen_tid_len = cpu_to_be32(V_FW_RIWR_GEN(genbit) |</div><div class='del'>-				       V_FW_RIWR_TID(tid) |</div><div class='del'>-				       V_FW_RIWR_LEN(len));</div><div class='del'>-	/* 2nd gen bit... */</div><div class='del'>-	((union t3_wr *)wqe)-&gt;genbit.genbit = cpu_to_be64(genbit);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * T3 ULP2_TX commands</div><div class='del'>- */</div><div class='del'>-enum t3_utx_mem_op {</div><div class='del'>-	T3_UTX_MEM_READ = 2,</div><div class='del'>-	T3_UTX_MEM_WRITE = 3</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* T3 MC7 RDMA TPT entry format */</div><div class='del'>-</div><div class='del'>-enum tpt_mem_type {</div><div class='del'>-	TPT_NON_SHARED_MR = 0x0,</div><div class='del'>-	TPT_SHARED_MR = 0x1,</div><div class='del'>-	TPT_MW = 0x2,</div><div class='del'>-	TPT_MW_RELAXED_PROTECTION = 0x3</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum tpt_addr_type {</div><div class='del'>-	TPT_ZBTO = 0,</div><div class='del'>-	TPT_VATO = 1</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum tpt_mem_perm {</div><div class='del'>-	TPT_MW_BIND = 0x10,</div><div class='del'>-	TPT_LOCAL_READ = 0x8,</div><div class='del'>-	TPT_LOCAL_WRITE = 0x4,</div><div class='del'>-	TPT_REMOTE_READ = 0x2,</div><div class='del'>-	TPT_REMOTE_WRITE = 0x1</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct tpt_entry {</div><div class='del'>-	__be32 valid_stag_pdid;</div><div class='del'>-	__be32 flags_pagesize_qpid;</div><div class='del'>-</div><div class='del'>-	__be32 rsvd_pbl_addr;</div><div class='del'>-	__be32 len;</div><div class='del'>-	__be32 va_hi;</div><div class='del'>-	__be32 va_low_or_fbo;</div><div class='del'>-</div><div class='del'>-	__be32 rsvd_bind_cnt_or_pstag;</div><div class='del'>-	__be32 rsvd_pbl_size;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define S_TPT_VALID		31</div><div class='del'>-#define V_TPT_VALID(x)		((x) &lt;&lt; S_TPT_VALID)</div><div class='del'>-#define F_TPT_VALID		V_TPT_VALID(1U)</div><div class='del'>-</div><div class='del'>-#define S_TPT_STAG_KEY		23</div><div class='del'>-#define M_TPT_STAG_KEY		0xFF</div><div class='del'>-#define V_TPT_STAG_KEY(x)	((x) &lt;&lt; S_TPT_STAG_KEY)</div><div class='del'>-#define G_TPT_STAG_KEY(x)	(((x) &gt;&gt; S_TPT_STAG_KEY) &amp; M_TPT_STAG_KEY)</div><div class='del'>-</div><div class='del'>-#define S_TPT_STAG_STATE	22</div><div class='del'>-#define V_TPT_STAG_STATE(x)	((x) &lt;&lt; S_TPT_STAG_STATE)</div><div class='del'>-#define F_TPT_STAG_STATE	V_TPT_STAG_STATE(1U)</div><div class='del'>-</div><div class='del'>-#define S_TPT_STAG_TYPE		20</div><div class='del'>-#define M_TPT_STAG_TYPE		0x3</div><div class='del'>-#define V_TPT_STAG_TYPE(x)	((x) &lt;&lt; S_TPT_STAG_TYPE)</div><div class='del'>-#define G_TPT_STAG_TYPE(x)	(((x) &gt;&gt; S_TPT_STAG_TYPE) &amp; M_TPT_STAG_TYPE)</div><div class='del'>-</div><div class='del'>-#define S_TPT_PDID		0</div><div class='del'>-#define M_TPT_PDID		0xFFFFF</div><div class='del'>-#define V_TPT_PDID(x)		((x) &lt;&lt; S_TPT_PDID)</div><div class='del'>-#define G_TPT_PDID(x)		(((x) &gt;&gt; S_TPT_PDID) &amp; M_TPT_PDID)</div><div class='del'>-</div><div class='del'>-#define S_TPT_PERM		28</div><div class='del'>-#define M_TPT_PERM		0xF</div><div class='del'>-#define V_TPT_PERM(x)		((x) &lt;&lt; S_TPT_PERM)</div><div class='del'>-#define G_TPT_PERM(x)		(((x) &gt;&gt; S_TPT_PERM) &amp; M_TPT_PERM)</div><div class='del'>-</div><div class='del'>-#define S_TPT_REM_INV_DIS	27</div><div class='del'>-#define V_TPT_REM_INV_DIS(x)	((x) &lt;&lt; S_TPT_REM_INV_DIS)</div><div class='del'>-#define F_TPT_REM_INV_DIS	V_TPT_REM_INV_DIS(1U)</div><div class='del'>-</div><div class='del'>-#define S_TPT_ADDR_TYPE		26</div><div class='del'>-#define V_TPT_ADDR_TYPE(x)	((x) &lt;&lt; S_TPT_ADDR_TYPE)</div><div class='del'>-#define F_TPT_ADDR_TYPE		V_TPT_ADDR_TYPE(1U)</div><div class='del'>-</div><div class='del'>-#define S_TPT_MW_BIND_ENABLE	25</div><div class='del'>-#define V_TPT_MW_BIND_ENABLE(x)	((x) &lt;&lt; S_TPT_MW_BIND_ENABLE)</div><div class='del'>-#define F_TPT_MW_BIND_ENABLE    V_TPT_MW_BIND_ENABLE(1U)</div><div class='del'>-</div><div class='del'>-#define S_TPT_PAGE_SIZE		20</div><div class='del'>-#define M_TPT_PAGE_SIZE		0x1F</div><div class='del'>-#define V_TPT_PAGE_SIZE(x)	((x) &lt;&lt; S_TPT_PAGE_SIZE)</div><div class='del'>-#define G_TPT_PAGE_SIZE(x)	(((x) &gt;&gt; S_TPT_PAGE_SIZE) &amp; M_TPT_PAGE_SIZE)</div><div class='del'>-</div><div class='del'>-#define S_TPT_PBL_ADDR		0</div><div class='del'>-#define M_TPT_PBL_ADDR		0x1FFFFFFF</div><div class='del'>-#define V_TPT_PBL_ADDR(x)	((x) &lt;&lt; S_TPT_PBL_ADDR)</div><div class='del'>-#define G_TPT_PBL_ADDR(x)       (((x) &gt;&gt; S_TPT_PBL_ADDR) &amp; M_TPT_PBL_ADDR)</div><div class='del'>-</div><div class='del'>-#define S_TPT_QPID		0</div><div class='del'>-#define M_TPT_QPID		0xFFFFF</div><div class='del'>-#define V_TPT_QPID(x)		((x) &lt;&lt; S_TPT_QPID)</div><div class='del'>-#define G_TPT_QPID(x)		(((x) &gt;&gt; S_TPT_QPID) &amp; M_TPT_QPID)</div><div class='del'>-</div><div class='del'>-#define S_TPT_PSTAG		0</div><div class='del'>-#define M_TPT_PSTAG		0xFFFFFF</div><div class='del'>-#define V_TPT_PSTAG(x)		((x) &lt;&lt; S_TPT_PSTAG)</div><div class='del'>-#define G_TPT_PSTAG(x)		(((x) &gt;&gt; S_TPT_PSTAG) &amp; M_TPT_PSTAG)</div><div class='del'>-</div><div class='del'>-#define S_TPT_PBL_SIZE		0</div><div class='del'>-#define M_TPT_PBL_SIZE		0xFFFFF</div><div class='del'>-#define V_TPT_PBL_SIZE(x)	((x) &lt;&lt; S_TPT_PBL_SIZE)</div><div class='del'>-#define G_TPT_PBL_SIZE(x)	(((x) &gt;&gt; S_TPT_PBL_SIZE) &amp; M_TPT_PBL_SIZE)</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * CQE defs</div><div class='del'>- */</div><div class='del'>-struct t3_cqe {</div><div class='del'>-	__be32 header;</div><div class='del'>-	__be32 len;</div><div class='del'>-	union {</div><div class='del'>-		struct {</div><div class='del'>-			__be32 stag;</div><div class='del'>-			__be32 msn;</div><div class='del'>-		} rcqe;</div><div class='del'>-		struct {</div><div class='del'>-			u32 wrid_hi;</div><div class='del'>-			u32 wrid_low;</div><div class='del'>-		} scqe;</div><div class='del'>-	} u;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define S_CQE_OOO	  31</div><div class='del'>-#define M_CQE_OOO	  0x1</div><div class='del'>-#define G_CQE_OOO(x)	  ((((x) &gt;&gt; S_CQE_OOO)) &amp; M_CQE_OOO)</div><div class='del'>-#define V_CEQ_OOO(x)	  ((x)&lt;&lt;S_CQE_OOO)</div><div class='del'>-</div><div class='del'>-#define S_CQE_QPID        12</div><div class='del'>-#define M_CQE_QPID        0x7FFFF</div><div class='del'>-#define G_CQE_QPID(x)     ((((x) &gt;&gt; S_CQE_QPID)) &amp; M_CQE_QPID)</div><div class='del'>-#define V_CQE_QPID(x)	  ((x)&lt;&lt;S_CQE_QPID)</div><div class='del'>-</div><div class='del'>-#define S_CQE_SWCQE       11</div><div class='del'>-#define M_CQE_SWCQE       0x1</div><div class='del'>-#define G_CQE_SWCQE(x)    ((((x) &gt;&gt; S_CQE_SWCQE)) &amp; M_CQE_SWCQE)</div><div class='del'>-#define V_CQE_SWCQE(x)	  ((x)&lt;&lt;S_CQE_SWCQE)</div><div class='del'>-</div><div class='del'>-#define S_CQE_GENBIT      10</div><div class='del'>-#define M_CQE_GENBIT      0x1</div><div class='del'>-#define G_CQE_GENBIT(x)   (((x) &gt;&gt; S_CQE_GENBIT) &amp; M_CQE_GENBIT)</div><div class='del'>-#define V_CQE_GENBIT(x)	  ((x)&lt;&lt;S_CQE_GENBIT)</div><div class='del'>-</div><div class='del'>-#define S_CQE_STATUS      5</div><div class='del'>-#define M_CQE_STATUS      0x1F</div><div class='del'>-#define G_CQE_STATUS(x)   ((((x) &gt;&gt; S_CQE_STATUS)) &amp; M_CQE_STATUS)</div><div class='del'>-#define V_CQE_STATUS(x)   ((x)&lt;&lt;S_CQE_STATUS)</div><div class='del'>-</div><div class='del'>-#define S_CQE_TYPE        4</div><div class='del'>-#define M_CQE_TYPE        0x1</div><div class='del'>-#define G_CQE_TYPE(x)     ((((x) &gt;&gt; S_CQE_TYPE)) &amp; M_CQE_TYPE)</div><div class='del'>-#define V_CQE_TYPE(x)     ((x)&lt;&lt;S_CQE_TYPE)</div><div class='del'>-</div><div class='del'>-#define S_CQE_OPCODE      0</div><div class='del'>-#define M_CQE_OPCODE      0xF</div><div class='del'>-#define G_CQE_OPCODE(x)   ((((x) &gt;&gt; S_CQE_OPCODE)) &amp; M_CQE_OPCODE)</div><div class='del'>-#define V_CQE_OPCODE(x)   ((x)&lt;&lt;S_CQE_OPCODE)</div><div class='del'>-</div><div class='del'>-#define SW_CQE(x)         (G_CQE_SWCQE(be32_to_cpu((x).header)))</div><div class='del'>-#define CQE_OOO(x)        (G_CQE_OOO(be32_to_cpu((x).header)))</div><div class='del'>-#define CQE_QPID(x)       (G_CQE_QPID(be32_to_cpu((x).header)))</div><div class='del'>-#define CQE_GENBIT(x)     (G_CQE_GENBIT(be32_to_cpu((x).header)))</div><div class='del'>-#define CQE_TYPE(x)       (G_CQE_TYPE(be32_to_cpu((x).header)))</div><div class='del'>-#define SQ_TYPE(x)	  (CQE_TYPE((x)))</div><div class='del'>-#define RQ_TYPE(x)	  (!CQE_TYPE((x)))</div><div class='del'>-#define CQE_STATUS(x)     (G_CQE_STATUS(be32_to_cpu((x).header)))</div><div class='del'>-#define CQE_OPCODE(x)     (G_CQE_OPCODE(be32_to_cpu((x).header)))</div><div class='del'>-</div><div class='del'>-#define CQE_SEND_OPCODE(x)( \</div><div class='del'>-	(G_CQE_OPCODE(be32_to_cpu((x).header)) == T3_SEND) || \</div><div class='del'>-	(G_CQE_OPCODE(be32_to_cpu((x).header)) == T3_SEND_WITH_SE) || \</div><div class='del'>-	(G_CQE_OPCODE(be32_to_cpu((x).header)) == T3_SEND_WITH_INV) || \</div><div class='del'>-	(G_CQE_OPCODE(be32_to_cpu((x).header)) == T3_SEND_WITH_SE_INV))</div><div class='del'>-</div><div class='del'>-#define CQE_LEN(x)        (be32_to_cpu((x).len))</div><div class='del'>-</div><div class='del'>-/* used for RQ completion processing */</div><div class='del'>-#define CQE_WRID_STAG(x)  (be32_to_cpu((x).u.rcqe.stag))</div><div class='del'>-#define CQE_WRID_MSN(x)   (be32_to_cpu((x).u.rcqe.msn))</div><div class='del'>-</div><div class='del'>-/* used for SQ completion processing */</div><div class='del'>-#define CQE_WRID_SQ_WPTR(x)	((x).u.scqe.wrid_hi)</div><div class='del'>-#define CQE_WRID_WPTR(x)	((x).u.scqe.wrid_low)</div><div class='del'>-</div><div class='del'>-/* generic accessor macros */</div><div class='del'>-#define CQE_WRID_HI(x)		((x).u.scqe.wrid_hi)</div><div class='del'>-#define CQE_WRID_LOW(x)		((x).u.scqe.wrid_low)</div><div class='del'>-</div><div class='del'>-#define TPT_ERR_SUCCESS                     0x0</div><div class='del'>-#define TPT_ERR_STAG                        0x1	 /* STAG invalid: either the */</div><div class='del'>-						 /* STAG is offlimt, being 0, */</div><div class='del'>-						 /* or STAG_key mismatch */</div><div class='del'>-#define TPT_ERR_PDID                        0x2	 /* PDID mismatch */</div><div class='del'>-#define TPT_ERR_QPID                        0x3	 /* QPID mismatch */</div><div class='del'>-#define TPT_ERR_ACCESS                      0x4	 /* Invalid access right */</div><div class='del'>-#define TPT_ERR_WRAP                        0x5	 /* Wrap error */</div><div class='del'>-#define TPT_ERR_BOUND                       0x6	 /* base and bounds voilation */</div><div class='del'>-#define TPT_ERR_INVALIDATE_SHARED_MR        0x7	 /* attempt to invalidate a  */</div><div class='del'>-						 /* shared memory region */</div><div class='del'>-#define TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND 0x8	 /* attempt to invalidate a  */</div><div class='del'>-						 /* shared memory region */</div><div class='del'>-#define TPT_ERR_ECC                         0x9	 /* ECC error detected */</div><div class='del'>-#define TPT_ERR_ECC_PSTAG                   0xA	 /* ECC error detected when  */</div><div class='del'>-						 /* reading PSTAG for a MW  */</div><div class='del'>-						 /* Invalidate */</div><div class='del'>-#define TPT_ERR_PBL_ADDR_BOUND              0xB	 /* pbl addr out of bounds:  */</div><div class='del'>-						 /* software error */</div><div class='del'>-#define TPT_ERR_SWFLUSH			    0xC	 /* SW FLUSHED */</div><div class='del'>-#define TPT_ERR_CRC                         0x10 /* CRC error */</div><div class='del'>-#define TPT_ERR_MARKER                      0x11 /* Marker error */</div><div class='del'>-#define TPT_ERR_PDU_LEN_ERR                 0x12 /* invalid PDU length */</div><div class='del'>-#define TPT_ERR_OUT_OF_RQE                  0x13 /* out of RQE */</div><div class='del'>-#define TPT_ERR_DDP_VERSION                 0x14 /* wrong DDP version */</div><div class='del'>-#define TPT_ERR_RDMA_VERSION                0x15 /* wrong RDMA version */</div><div class='del'>-#define TPT_ERR_OPCODE                      0x16 /* invalid rdma opcode */</div><div class='del'>-#define TPT_ERR_DDP_QUEUE_NUM               0x17 /* invalid ddp queue number */</div><div class='del'>-#define TPT_ERR_MSN                         0x18 /* MSN error */</div><div class='del'>-#define TPT_ERR_TBIT                        0x19 /* tag bit not set correctly */</div><div class='del'>-#define TPT_ERR_MO                          0x1A /* MO not 0 for TERMINATE  */</div><div class='del'>-						 /* or READ_REQ */</div><div class='del'>-#define TPT_ERR_MSN_GAP                     0x1B</div><div class='del'>-#define TPT_ERR_MSN_RANGE                   0x1C</div><div class='del'>-#define TPT_ERR_IRD_OVERFLOW                0x1D</div><div class='del'>-#define TPT_ERR_RQE_ADDR_BOUND              0x1E /* RQE addr out of bounds:  */</div><div class='del'>-						 /* software error */</div><div class='del'>-#define TPT_ERR_INTERNAL_ERR                0x1F /* internal error (opcode  */</div><div class='del'>-						 /* mismatch) */</div><div class='del'>-</div><div class='del'>-struct t3_swsq {</div><div class='del'>-	__u64			wr_id;</div><div class='del'>-	struct t3_cqe		cqe;</div><div class='del'>-	__u32			sq_wptr;</div><div class='del'>-	__be32			read_len;</div><div class='del'>-	int			opcode;</div><div class='del'>-	int			complete;</div><div class='del'>-	int			signaled;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct t3_swrq {</div><div class='del'>-	__u64			wr_id;</div><div class='del'>-	__u32			pbl_addr;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * A T3 WQ implements both the SQ and RQ.</div><div class='del'>- */</div><div class='del'>-struct t3_wq {</div><div class='del'>-	union t3_wr *queue;		/* DMA accessible memory */</div><div class='del'>-	dma_addr_t dma_addr;		/* DMA address for HW */</div><div class='del'>-	DEFINE_DMA_UNMAP_ADDR(mapping); /* unmap kruft */</div><div class='del'>-	u32 error;			/* 1 once we go to ERROR */</div><div class='del'>-	u32 qpid;</div><div class='del'>-	u32 wptr;			/* idx to next available WR slot */</div><div class='del'>-	u32 size_log2;			/* total wq size */</div><div class='del'>-	struct t3_swsq *sq;		/* SW SQ */</div><div class='del'>-	struct t3_swsq *oldest_read;	/* tracks oldest pending read */</div><div class='del'>-	u32 sq_wptr;			/* sq_wptr - sq_rptr == count of */</div><div class='del'>-	u32 sq_rptr;			/* pending wrs */</div><div class='del'>-	u32 sq_size_log2;		/* sq size */</div><div class='del'>-	struct t3_swrq *rq;		/* SW RQ (holds consumer wr_ids */</div><div class='del'>-	u32 rq_wptr;			/* rq_wptr - rq_rptr == count of */</div><div class='del'>-	u32 rq_rptr;			/* pending wrs */</div><div class='del'>-	struct t3_swrq *rq_oldest_wr;	/* oldest wr on the SW RQ */</div><div class='del'>-	u32 rq_size_log2;		/* rq size */</div><div class='del'>-	u32 rq_addr;			/* rq adapter address */</div><div class='del'>-	void __iomem *doorbell;		/* kernel db */</div><div class='del'>-	u64 udb;			/* user db if any */</div><div class='del'>-	struct cxio_rdev *rdev;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct t3_cq {</div><div class='del'>-	u32 cqid;</div><div class='del'>-	u32 rptr;</div><div class='del'>-	u32 wptr;</div><div class='del'>-	u32 size_log2;</div><div class='del'>-	dma_addr_t dma_addr;</div><div class='del'>-	DEFINE_DMA_UNMAP_ADDR(mapping);</div><div class='del'>-	struct t3_cqe *queue;</div><div class='del'>-	struct t3_cqe *sw_queue;</div><div class='del'>-	u32 sw_rptr;</div><div class='del'>-	u32 sw_wptr;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define CQ_VLD_ENTRY(ptr,size_log2,cqe) (Q_GENBIT(ptr,size_log2) == \</div><div class='del'>-					 CQE_GENBIT(*cqe))</div><div class='del'>-</div><div class='del'>-struct t3_cq_status_page {</div><div class='del'>-	u32 cq_err;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static inline int cxio_cq_in_error(struct t3_cq *cq)</div><div class='del'>-{</div><div class='del'>-	return ((struct t3_cq_status_page *)</div><div class='del'>-		&amp;cq-&gt;queue[1 &lt;&lt; cq-&gt;size_log2])-&gt;cq_err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cxio_set_cq_in_error(struct t3_cq *cq)</div><div class='del'>-{</div><div class='del'>-	((struct t3_cq_status_page *)</div><div class='del'>-	 &amp;cq-&gt;queue[1 &lt;&lt; cq-&gt;size_log2])-&gt;cq_err = 1;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cxio_set_wq_in_error(struct t3_wq *wq)</div><div class='del'>-{</div><div class='del'>-	wq-&gt;queue-&gt;wq_in_err.err |= 1;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cxio_disable_wq_db(struct t3_wq *wq)</div><div class='del'>-{</div><div class='del'>-	wq-&gt;queue-&gt;wq_in_err.err |= 2;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void cxio_enable_wq_db(struct t3_wq *wq)</div><div class='del'>-{</div><div class='del'>-	wq-&gt;queue-&gt;wq_in_err.err &amp;= ~2;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline int cxio_wq_db_enabled(struct t3_wq *wq)</div><div class='del'>-{</div><div class='del'>-	return !(wq-&gt;queue-&gt;wq_in_err.err &amp; 2);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline struct t3_cqe *cxio_next_hw_cqe(struct t3_cq *cq)</div><div class='del'>-{</div><div class='del'>-	struct t3_cqe *cqe;</div><div class='del'>-</div><div class='del'>-	cqe = cq-&gt;queue + (Q_PTR2IDX(cq-&gt;rptr, cq-&gt;size_log2));</div><div class='del'>-	if (CQ_VLD_ENTRY(cq-&gt;rptr, cq-&gt;size_log2, cqe))</div><div class='del'>-		return cqe;</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline struct t3_cqe *cxio_next_sw_cqe(struct t3_cq *cq)</div><div class='del'>-{</div><div class='del'>-	struct t3_cqe *cqe;</div><div class='del'>-</div><div class='del'>-	if (!Q_EMPTY(cq-&gt;sw_rptr, cq-&gt;sw_wptr)) {</div><div class='del'>-		cqe = cq-&gt;sw_queue + (Q_PTR2IDX(cq-&gt;sw_rptr, cq-&gt;size_log2));</div><div class='del'>-		return cqe;</div><div class='del'>-	}</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline struct t3_cqe *cxio_next_cqe(struct t3_cq *cq)</div><div class='del'>-{</div><div class='del'>-	struct t3_cqe *cqe;</div><div class='del'>-</div><div class='del'>-	if (!Q_EMPTY(cq-&gt;sw_rptr, cq-&gt;sw_wptr)) {</div><div class='del'>-		cqe = cq-&gt;sw_queue + (Q_PTR2IDX(cq-&gt;sw_rptr, cq-&gt;size_log2));</div><div class='del'>-		return cqe;</div><div class='del'>-	}</div><div class='del'>-	cqe = cq-&gt;queue + (Q_PTR2IDX(cq-&gt;rptr, cq-&gt;size_log2));</div><div class='del'>-	if (CQ_VLD_ENTRY(cq-&gt;rptr, cq-&gt;size_log2, cqe))</div><div class='del'>-		return cqe;</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/iwch.c b/drivers/infiniband/hw/cxgb3/iwch.c<br/>deleted file mode 100644<br/>index 56a8ab6210cf..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/iwch.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,282 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='del'>-#include &lt;linux/moduleparam.h&gt;</div><div class='del'>-</div><div class='del'>-#include &lt;rdma/ib_verbs.h&gt;</div><div class='del'>-</div><div class='del'>-#include "cxgb3_offload.h"</div><div class='del'>-#include "iwch_provider.h"</div><div class='del'>-#include &lt;rdma/cxgb3-abi.h&gt;</div><div class='del'>-#include "iwch.h"</div><div class='del'>-#include "iwch_cm.h"</div><div class='del'>-</div><div class='del'>-#define DRV_VERSION "1.1"</div><div class='del'>-</div><div class='del'>-MODULE_AUTHOR("Boyd Faulkner, Steve Wise");</div><div class='del'>-MODULE_DESCRIPTION("Chelsio T3 RDMA Driver");</div><div class='del'>-MODULE_LICENSE("Dual BSD/GPL");</div><div class='del'>-</div><div class='del'>-static void open_rnic_dev(struct t3cdev *);</div><div class='del'>-static void close_rnic_dev(struct t3cdev *);</div><div class='del'>-static void iwch_event_handler(struct t3cdev *, u32, u32);</div><div class='del'>-</div><div class='del'>-struct cxgb3_client t3c_client = {</div><div class='del'>-	.name = "iw_cxgb3",</div><div class='del'>-	.add = open_rnic_dev,</div><div class='del'>-	.remove = close_rnic_dev,</div><div class='del'>-	.handlers = t3c_handlers,</div><div class='del'>-	.redirect = iwch_ep_redirect,</div><div class='del'>-	.event_handler = iwch_event_handler</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static LIST_HEAD(dev_list);</div><div class='del'>-static DEFINE_MUTEX(dev_mutex);</div><div class='del'>-</div><div class='del'>-static void disable_dbs(struct iwch_dev *rnicp)</div><div class='del'>-{</div><div class='del'>-	unsigned long index;</div><div class='del'>-	struct iwch_qp *qhp;</div><div class='del'>-</div><div class='del'>-	xa_lock_irq(&amp;rnicp-&gt;qps);</div><div class='del'>-	xa_for_each(&amp;rnicp-&gt;qps, index, qhp)</div><div class='del'>-		cxio_disable_wq_db(&amp;qhp-&gt;wq);</div><div class='del'>-	xa_unlock_irq(&amp;rnicp-&gt;qps);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void enable_dbs(struct iwch_dev *rnicp, int ring_db)</div><div class='del'>-{</div><div class='del'>-	unsigned long index;</div><div class='del'>-	struct iwch_qp *qhp;</div><div class='del'>-</div><div class='del'>-	xa_lock_irq(&amp;rnicp-&gt;qps);</div><div class='del'>-	xa_for_each(&amp;rnicp-&gt;qps, index, qhp) {</div><div class='del'>-		if (ring_db)</div><div class='del'>-			ring_doorbell(qhp-&gt;rhp-&gt;rdev.ctrl_qp.doorbell,</div><div class='del'>-					qhp-&gt;wq.qpid);</div><div class='del'>-		cxio_enable_wq_db(&amp;qhp-&gt;wq);</div><div class='del'>-	}</div><div class='del'>-	xa_unlock_irq(&amp;rnicp-&gt;qps);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void iwch_db_drop_task(struct work_struct *work)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *rnicp = container_of(work, struct iwch_dev,</div><div class='del'>-					      db_drop_task.work);</div><div class='del'>-	enable_dbs(rnicp, 1);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void rnic_init(struct iwch_dev *rnicp)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s iwch_dev %p\n", __func__,  rnicp);</div><div class='del'>-	xa_init_flags(&amp;rnicp-&gt;cqs, XA_FLAGS_LOCK_IRQ);</div><div class='del'>-	xa_init_flags(&amp;rnicp-&gt;qps, XA_FLAGS_LOCK_IRQ);</div><div class='del'>-	xa_init_flags(&amp;rnicp-&gt;mrs, XA_FLAGS_LOCK_IRQ);</div><div class='del'>-	INIT_DELAYED_WORK(&amp;rnicp-&gt;db_drop_task, iwch_db_drop_task);</div><div class='del'>-</div><div class='del'>-	rnicp-&gt;attr.max_qps = T3_MAX_NUM_QP - 32;</div><div class='del'>-	rnicp-&gt;attr.max_wrs = T3_MAX_QP_DEPTH;</div><div class='del'>-	rnicp-&gt;attr.max_sge_per_wr = T3_MAX_SGE;</div><div class='del'>-	rnicp-&gt;attr.max_sge_per_rdma_write_wr = T3_MAX_SGE;</div><div class='del'>-	rnicp-&gt;attr.max_cqs = T3_MAX_NUM_CQ - 1;</div><div class='del'>-	rnicp-&gt;attr.max_cqes_per_cq = T3_MAX_CQ_DEPTH;</div><div class='del'>-	rnicp-&gt;attr.max_mem_regs = cxio_num_stags(&amp;rnicp-&gt;rdev);</div><div class='del'>-	rnicp-&gt;attr.max_phys_buf_entries = T3_MAX_PBL_SIZE;</div><div class='del'>-	rnicp-&gt;attr.max_pds = T3_MAX_NUM_PD - 1;</div><div class='del'>-	rnicp-&gt;attr.mem_pgsizes_bitmask = T3_PAGESIZE_MASK;</div><div class='del'>-	rnicp-&gt;attr.max_mr_size = T3_MAX_MR_SIZE;</div><div class='del'>-	rnicp-&gt;attr.can_resize_wq = 0;</div><div class='del'>-	rnicp-&gt;attr.max_rdma_reads_per_qp = 8;</div><div class='del'>-	rnicp-&gt;attr.max_rdma_read_resources =</div><div class='del'>-	    rnicp-&gt;attr.max_rdma_reads_per_qp * rnicp-&gt;attr.max_qps;</div><div class='del'>-	rnicp-&gt;attr.max_rdma_read_qp_depth = 8;	/* IRD */</div><div class='del'>-	rnicp-&gt;attr.max_rdma_read_depth =</div><div class='del'>-	    rnicp-&gt;attr.max_rdma_read_qp_depth * rnicp-&gt;attr.max_qps;</div><div class='del'>-	rnicp-&gt;attr.rq_overflow_handled = 0;</div><div class='del'>-	rnicp-&gt;attr.can_modify_ird = 0;</div><div class='del'>-	rnicp-&gt;attr.can_modify_ord = 0;</div><div class='del'>-	rnicp-&gt;attr.max_mem_windows = rnicp-&gt;attr.max_mem_regs - 1;</div><div class='del'>-	rnicp-&gt;attr.stag0_value = 1;</div><div class='del'>-	rnicp-&gt;attr.zbva_support = 1;</div><div class='del'>-	rnicp-&gt;attr.local_invalidate_fence = 1;</div><div class='del'>-	rnicp-&gt;attr.cq_overflow_detection = 1;</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void open_rnic_dev(struct t3cdev *tdev)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *rnicp;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s t3cdev %p\n", __func__,  tdev);</div><div class='del'>-	pr_info_once("Chelsio T3 RDMA Driver - version %s\n", DRV_VERSION);</div><div class='del'>-	rnicp = ib_alloc_device(iwch_dev, ibdev);</div><div class='del'>-	if (!rnicp) {</div><div class='del'>-		pr_err("Cannot allocate ib device\n");</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-	rnicp-&gt;rdev.ulp = rnicp;</div><div class='del'>-	rnicp-&gt;rdev.t3cdev_p = tdev;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;dev_mutex);</div><div class='del'>-</div><div class='del'>-	if (cxio_rdev_open(&amp;rnicp-&gt;rdev)) {</div><div class='del'>-		mutex_unlock(&amp;dev_mutex);</div><div class='del'>-		pr_err("Unable to open CXIO rdev\n");</div><div class='del'>-		ib_dealloc_device(&amp;rnicp-&gt;ibdev);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	rnic_init(rnicp);</div><div class='del'>-</div><div class='del'>-	list_add_tail(&amp;rnicp-&gt;entry, &amp;dev_list);</div><div class='del'>-	mutex_unlock(&amp;dev_mutex);</div><div class='del'>-</div><div class='del'>-	if (iwch_register_device(rnicp)) {</div><div class='del'>-		pr_err("Unable to register device\n");</div><div class='del'>-		close_rnic_dev(tdev);</div><div class='del'>-	}</div><div class='del'>-	pr_info("Initialized device %s\n",</div><div class='del'>-		pci_name(rnicp-&gt;rdev.rnic_info.pdev));</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void close_rnic_dev(struct t3cdev *tdev)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *dev, *tmp;</div><div class='del'>-	pr_debug("%s t3cdev %p\n", __func__,  tdev);</div><div class='del'>-	mutex_lock(&amp;dev_mutex);</div><div class='del'>-	list_for_each_entry_safe(dev, tmp, &amp;dev_list, entry) {</div><div class='del'>-		if (dev-&gt;rdev.t3cdev_p == tdev) {</div><div class='del'>-			dev-&gt;rdev.flags = CXIO_ERROR_FATAL;</div><div class='del'>-			synchronize_net();</div><div class='del'>-			cancel_delayed_work_sync(&amp;dev-&gt;db_drop_task);</div><div class='del'>-			list_del(&amp;dev-&gt;entry);</div><div class='del'>-			iwch_unregister_device(dev);</div><div class='del'>-			cxio_rdev_close(&amp;dev-&gt;rdev);</div><div class='del'>-			WARN_ON(!xa_empty(&amp;dev-&gt;cqs));</div><div class='del'>-			WARN_ON(!xa_empty(&amp;dev-&gt;qps));</div><div class='del'>-			WARN_ON(!xa_empty(&amp;dev-&gt;mrs));</div><div class='del'>-			ib_dealloc_device(&amp;dev-&gt;ibdev);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	mutex_unlock(&amp;dev_mutex);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void iwch_event_handler(struct t3cdev *tdev, u32 evt, u32 port_id)</div><div class='del'>-{</div><div class='del'>-	struct cxio_rdev *rdev = tdev-&gt;ulp;</div><div class='del'>-	struct iwch_dev *rnicp;</div><div class='del'>-	struct ib_event event;</div><div class='del'>-	u32 portnum = port_id + 1;</div><div class='del'>-	int dispatch = 0;</div><div class='del'>-</div><div class='del'>-	if (!rdev)</div><div class='del'>-		return;</div><div class='del'>-	rnicp = rdev_to_iwch_dev(rdev);</div><div class='del'>-	switch (evt) {</div><div class='del'>-	case OFFLOAD_STATUS_DOWN: {</div><div class='del'>-		rdev-&gt;flags = CXIO_ERROR_FATAL;</div><div class='del'>-		synchronize_net();</div><div class='del'>-		event.event  = IB_EVENT_DEVICE_FATAL;</div><div class='del'>-		dispatch = 1;</div><div class='del'>-		break;</div><div class='del'>-		}</div><div class='del'>-	case OFFLOAD_PORT_DOWN: {</div><div class='del'>-		event.event  = IB_EVENT_PORT_ERR;</div><div class='del'>-		dispatch = 1;</div><div class='del'>-		break;</div><div class='del'>-		}</div><div class='del'>-	case OFFLOAD_PORT_UP: {</div><div class='del'>-		event.event  = IB_EVENT_PORT_ACTIVE;</div><div class='del'>-		dispatch = 1;</div><div class='del'>-		break;</div><div class='del'>-		}</div><div class='del'>-	case OFFLOAD_DB_FULL: {</div><div class='del'>-		disable_dbs(rnicp);</div><div class='del'>-		break;</div><div class='del'>-		}</div><div class='del'>-	case OFFLOAD_DB_EMPTY: {</div><div class='del'>-		enable_dbs(rnicp, 1);</div><div class='del'>-		break;</div><div class='del'>-		}</div><div class='del'>-	case OFFLOAD_DB_DROP: {</div><div class='del'>-		unsigned long delay = 1000;</div><div class='del'>-		unsigned short r;</div><div class='del'>-</div><div class='del'>-		disable_dbs(rnicp);</div><div class='del'>-		get_random_bytes(&amp;r, 2);</div><div class='del'>-		delay += r &amp; 1023;</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * delay is between 1000-2023 usecs.</div><div class='del'>-		 */</div><div class='del'>-		schedule_delayed_work(&amp;rnicp-&gt;db_drop_task,</div><div class='del'>-			usecs_to_jiffies(delay));</div><div class='del'>-		break;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (dispatch) {</div><div class='del'>-		event.device = &amp;rnicp-&gt;ibdev;</div><div class='del'>-		event.element.port_num = portnum;</div><div class='del'>-		ib_dispatch_event(&amp;event);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int __init iwch_init_module(void)</div><div class='del'>-{</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	err = cxio_hal_init();</div><div class='del'>-	if (err)</div><div class='del'>-		return err;</div><div class='del'>-	err = iwch_cm_init();</div><div class='del'>-	if (err)</div><div class='del'>-		return err;</div><div class='del'>-	cxio_register_ev_cb(iwch_ev_dispatch);</div><div class='del'>-	cxgb3_register_client(&amp;t3c_client);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void __exit iwch_exit_module(void)</div><div class='del'>-{</div><div class='del'>-	cxgb3_unregister_client(&amp;t3c_client);</div><div class='del'>-	cxio_unregister_ev_cb(iwch_ev_dispatch);</div><div class='del'>-	iwch_cm_term();</div><div class='del'>-	cxio_hal_exit();</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-module_init(iwch_init_module);</div><div class='del'>-module_exit(iwch_exit_module);</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/iwch.h b/drivers/infiniband/hw/cxgb3/iwch.h<br/>deleted file mode 100644<br/>index 310a937bffcf..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/iwch.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,155 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-#ifndef __IWCH_H__</div><div class='del'>-#define __IWCH_H__</div><div class='del'>-</div><div class='del'>-#include &lt;linux/mutex.h&gt;</div><div class='del'>-#include &lt;linux/list.h&gt;</div><div class='del'>-#include &lt;linux/spinlock.h&gt;</div><div class='del'>-#include &lt;linux/xarray.h&gt;</div><div class='del'>-#include &lt;linux/workqueue.h&gt;</div><div class='del'>-</div><div class='del'>-#include &lt;rdma/ib_verbs.h&gt;</div><div class='del'>-</div><div class='del'>-#include "cxio_hal.h"</div><div class='del'>-#include "cxgb3_offload.h"</div><div class='del'>-</div><div class='del'>-struct iwch_pd;</div><div class='del'>-struct iwch_cq;</div><div class='del'>-struct iwch_qp;</div><div class='del'>-struct iwch_mr;</div><div class='del'>-</div><div class='del'>-struct iwch_rnic_attributes {</div><div class='del'>-	u32 max_qps;</div><div class='del'>-	u32 max_wrs;				/* Max for any SQ/RQ */</div><div class='del'>-	u32 max_sge_per_wr;</div><div class='del'>-	u32 max_sge_per_rdma_write_wr;	/* for RDMA Write WR */</div><div class='del'>-	u32 max_cqs;</div><div class='del'>-	u32 max_cqes_per_cq;</div><div class='del'>-	u32 max_mem_regs;</div><div class='del'>-	u32 max_phys_buf_entries;		/* for phys buf list */</div><div class='del'>-	u32 max_pds;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * The memory page sizes supported by this RNIC.</div><div class='del'>-	 * Bit position i in bitmap indicates page of</div><div class='del'>-	 * size (4k)^i.  Phys block list mode unsupported.</div><div class='del'>-	 */</div><div class='del'>-	u32 mem_pgsizes_bitmask;</div><div class='del'>-	u64 max_mr_size;</div><div class='del'>-	u8 can_resize_wq;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * The maximum number of RDMA Reads that can be outstanding</div><div class='del'>-	 * per QP with this RNIC as the target.</div><div class='del'>-	 */</div><div class='del'>-	u32 max_rdma_reads_per_qp;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * The maximum number of resources used for RDMA Reads</div><div class='del'>-	 * by this RNIC with this RNIC as the target.</div><div class='del'>-	 */</div><div class='del'>-	u32 max_rdma_read_resources;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * The max depth per QP for initiation of RDMA Read</div><div class='del'>-	 * by this RNIC.</div><div class='del'>-	 */</div><div class='del'>-	u32 max_rdma_read_qp_depth;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * The maximum depth for initiation of RDMA Read</div><div class='del'>-	 * operations by this RNIC on all QPs</div><div class='del'>-	 */</div><div class='del'>-	u32 max_rdma_read_depth;</div><div class='del'>-	u8 rq_overflow_handled;</div><div class='del'>-	u32 can_modify_ird;</div><div class='del'>-	u32 can_modify_ord;</div><div class='del'>-	u32 max_mem_windows;</div><div class='del'>-	u32 stag0_value;</div><div class='del'>-	u8 zbva_support;</div><div class='del'>-	u8 local_invalidate_fence;</div><div class='del'>-	u32 cq_overflow_detection;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct iwch_dev {</div><div class='del'>-	struct ib_device ibdev;</div><div class='del'>-	struct cxio_rdev rdev;</div><div class='del'>-	u32 device_cap_flags;</div><div class='del'>-	struct iwch_rnic_attributes attr;</div><div class='del'>-	struct xarray cqs;</div><div class='del'>-	struct xarray qps;</div><div class='del'>-	struct xarray mrs;</div><div class='del'>-	struct list_head entry;</div><div class='del'>-	struct delayed_work db_drop_task;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static inline struct iwch_dev *to_iwch_dev(struct ib_device *ibdev)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibdev, struct iwch_dev, ibdev);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline struct iwch_dev *rdev_to_iwch_dev(struct cxio_rdev *rdev)</div><div class='del'>-{</div><div class='del'>-	return container_of(rdev, struct iwch_dev, rdev);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline int t3b_device(const struct iwch_dev *rhp)</div><div class='del'>-{</div><div class='del'>-	return rhp-&gt;rdev.t3cdev_p-&gt;type == T3B;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline int t3a_device(const struct iwch_dev *rhp)</div><div class='del'>-{</div><div class='del'>-	return rhp-&gt;rdev.t3cdev_p-&gt;type == T3A;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline struct iwch_cq *get_chp(struct iwch_dev *rhp, u32 cqid)</div><div class='del'>-{</div><div class='del'>-	return xa_load(&amp;rhp-&gt;cqs, cqid);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline struct iwch_qp *get_qhp(struct iwch_dev *rhp, u32 qpid)</div><div class='del'>-{</div><div class='del'>-	return xa_load(&amp;rhp-&gt;qps, qpid);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline struct iwch_mr *get_mhp(struct iwch_dev *rhp, u32 mmid)</div><div class='del'>-{</div><div class='del'>-	return xa_load(&amp;rhp-&gt;mrs, mmid);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-extern struct cxgb3_client t3c_client;</div><div class='del'>-extern cxgb3_cpl_handler_func t3c_handlers[NUM_CPL_CMDS];</div><div class='del'>-extern void iwch_ev_dispatch(struct cxio_rdev *rdev_p, struct sk_buff *skb);</div><div class='del'>-</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/iwch_cm.c b/drivers/infiniband/hw/cxgb3/iwch_cm.c<br/>deleted file mode 100644<br/>index 1c90c86fc8b8..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/iwch_cm.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch_cm.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,2258 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='del'>-#include &lt;linux/list.h&gt;</div><div class='del'>-#include &lt;linux/slab.h&gt;</div><div class='del'>-#include &lt;linux/workqueue.h&gt;</div><div class='del'>-#include &lt;linux/skbuff.h&gt;</div><div class='del'>-#include &lt;linux/timer.h&gt;</div><div class='del'>-#include &lt;linux/notifier.h&gt;</div><div class='del'>-#include &lt;linux/inetdevice.h&gt;</div><div class='del'>-</div><div class='del'>-#include &lt;net/neighbour.h&gt;</div><div class='del'>-#include &lt;net/netevent.h&gt;</div><div class='del'>-#include &lt;net/route.h&gt;</div><div class='del'>-</div><div class='del'>-#include "tcb.h"</div><div class='del'>-#include "cxgb3_offload.h"</div><div class='del'>-#include "iwch.h"</div><div class='del'>-#include "iwch_provider.h"</div><div class='del'>-#include "iwch_cm.h"</div><div class='del'>-</div><div class='del'>-static char *states[] = {</div><div class='del'>-	"idle",</div><div class='del'>-	"listen",</div><div class='del'>-	"connecting",</div><div class='del'>-	"mpa_wait_req",</div><div class='del'>-	"mpa_req_sent",</div><div class='del'>-	"mpa_req_rcvd",</div><div class='del'>-	"mpa_rep_sent",</div><div class='del'>-	"fpdu_mode",</div><div class='del'>-	"aborting",</div><div class='del'>-	"closing",</div><div class='del'>-	"moribund",</div><div class='del'>-	"dead",</div><div class='del'>-	NULL,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-int peer2peer = 0;</div><div class='del'>-module_param(peer2peer, int, 0644);</div><div class='del'>-MODULE_PARM_DESC(peer2peer, "Support peer2peer ULPs (default=0)");</div><div class='del'>-</div><div class='del'>-static int ep_timeout_secs = 60;</div><div class='del'>-module_param(ep_timeout_secs, int, 0644);</div><div class='del'>-MODULE_PARM_DESC(ep_timeout_secs, "CM Endpoint operation timeout "</div><div class='del'>-				   "in seconds (default=60)");</div><div class='del'>-</div><div class='del'>-static int mpa_rev = 1;</div><div class='del'>-module_param(mpa_rev, int, 0644);</div><div class='del'>-MODULE_PARM_DESC(mpa_rev, "MPA Revision, 0 supports amso1100, "</div><div class='del'>-		 "1 is spec compliant. (default=1)");</div><div class='del'>-</div><div class='del'>-static int markers_enabled = 0;</div><div class='del'>-module_param(markers_enabled, int, 0644);</div><div class='del'>-MODULE_PARM_DESC(markers_enabled, "Enable MPA MARKERS (default(0)=disabled)");</div><div class='del'>-</div><div class='del'>-static int crc_enabled = 1;</div><div class='del'>-module_param(crc_enabled, int, 0644);</div><div class='del'>-MODULE_PARM_DESC(crc_enabled, "Enable MPA CRC (default(1)=enabled)");</div><div class='del'>-</div><div class='del'>-static int rcv_win = 256 * 1024;</div><div class='del'>-module_param(rcv_win, int, 0644);</div><div class='del'>-MODULE_PARM_DESC(rcv_win, "TCP receive window in bytes (default=256)");</div><div class='del'>-</div><div class='del'>-static int snd_win = 32 * 1024;</div><div class='del'>-module_param(snd_win, int, 0644);</div><div class='del'>-MODULE_PARM_DESC(snd_win, "TCP send window in bytes (default=32KB)");</div><div class='del'>-</div><div class='del'>-static unsigned int nocong = 0;</div><div class='del'>-module_param(nocong, uint, 0644);</div><div class='del'>-MODULE_PARM_DESC(nocong, "Turn off congestion control (default=0)");</div><div class='del'>-</div><div class='del'>-static unsigned int cong_flavor = 1;</div><div class='del'>-module_param(cong_flavor, uint, 0644);</div><div class='del'>-MODULE_PARM_DESC(cong_flavor, "TCP Congestion control flavor (default=1)");</div><div class='del'>-</div><div class='del'>-static struct workqueue_struct *workq;</div><div class='del'>-</div><div class='del'>-static struct sk_buff_head rxq;</div><div class='del'>-</div><div class='del'>-static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp);</div><div class='del'>-static void ep_timeout(struct timer_list *t);</div><div class='del'>-static void connect_reply_upcall(struct iwch_ep *ep, int status);</div><div class='del'>-</div><div class='del'>-static void start_ep_timer(struct iwch_ep *ep)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	if (timer_pending(&amp;ep-&gt;timer)) {</div><div class='del'>-		pr_debug("%s stopped / restarted timer ep %p\n", __func__, ep);</div><div class='del'>-		del_timer_sync(&amp;ep-&gt;timer);</div><div class='del'>-	} else</div><div class='del'>-		get_ep(&amp;ep-&gt;com);</div><div class='del'>-	ep-&gt;timer.expires = jiffies + ep_timeout_secs * HZ;</div><div class='del'>-	add_timer(&amp;ep-&gt;timer);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void stop_ep_timer(struct iwch_ep *ep)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	if (!timer_pending(&amp;ep-&gt;timer)) {</div><div class='del'>-		WARN(1, "%s timer stopped when its not running!  ep %p state %u\n",</div><div class='del'>-			__func__, ep, ep-&gt;com.state);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-	del_timer_sync(&amp;ep-&gt;timer);</div><div class='del'>-	put_ep(&amp;ep-&gt;com);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_l2t_send(struct t3cdev *tdev, struct sk_buff *skb, struct l2t_entry *l2e)</div><div class='del'>-{</div><div class='del'>-	int	error = 0;</div><div class='del'>-	struct cxio_rdev *rdev;</div><div class='del'>-</div><div class='del'>-	rdev = (struct cxio_rdev *)tdev-&gt;ulp;</div><div class='del'>-	if (cxio_fatal_error(rdev)) {</div><div class='del'>-		kfree_skb(skb);</div><div class='del'>-		return -EIO;</div><div class='del'>-	}</div><div class='del'>-	error = l2t_send(tdev, skb, l2e);</div><div class='del'>-	if (error &lt; 0)</div><div class='del'>-		kfree_skb(skb);</div><div class='del'>-	return error &lt; 0 ? error : 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_cxgb3_ofld_send(struct t3cdev *tdev, struct sk_buff *skb)</div><div class='del'>-{</div><div class='del'>-	int	error = 0;</div><div class='del'>-	struct cxio_rdev *rdev;</div><div class='del'>-</div><div class='del'>-	rdev = (struct cxio_rdev *)tdev-&gt;ulp;</div><div class='del'>-	if (cxio_fatal_error(rdev)) {</div><div class='del'>-		kfree_skb(skb);</div><div class='del'>-		return -EIO;</div><div class='del'>-	}</div><div class='del'>-	error = cxgb3_ofld_send(tdev, skb);</div><div class='del'>-	if (error &lt; 0)</div><div class='del'>-		kfree_skb(skb);</div><div class='del'>-	return error &lt; 0 ? error : 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void release_tid(struct t3cdev *tdev, u32 hwtid, struct sk_buff *skb)</div><div class='del'>-{</div><div class='del'>-	struct cpl_tid_release *req;</div><div class='del'>-</div><div class='del'>-	skb = get_skb(skb, sizeof *req, GFP_KERNEL);</div><div class='del'>-	if (!skb)</div><div class='del'>-		return;</div><div class='del'>-	req = skb_put(skb, sizeof(*req));</div><div class='del'>-	req-&gt;wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));</div><div class='del'>-	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, hwtid));</div><div class='del'>-	skb-&gt;priority = CPL_PRIORITY_SETUP;</div><div class='del'>-	iwch_cxgb3_ofld_send(tdev, skb);</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_quiesce_tid(struct iwch_ep *ep)</div><div class='del'>-{</div><div class='del'>-	struct cpl_set_tcb_field *req;</div><div class='del'>-	struct sk_buff *skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);</div><div class='del'>-</div><div class='del'>-	if (!skb)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	req = skb_put(skb, sizeof(*req));</div><div class='del'>-	req-&gt;wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));</div><div class='del'>-	req-&gt;wr.wr_lo = htonl(V_WR_TID(ep-&gt;hwtid));</div><div class='del'>-	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, ep-&gt;hwtid));</div><div class='del'>-	req-&gt;reply = 0;</div><div class='del'>-	req-&gt;cpu_idx = 0;</div><div class='del'>-	req-&gt;word = htons(W_TCB_RX_QUIESCE);</div><div class='del'>-	req-&gt;mask = cpu_to_be64(1ULL &lt;&lt; S_TCB_RX_QUIESCE);</div><div class='del'>-	req-&gt;val = cpu_to_be64(1 &lt;&lt; S_TCB_RX_QUIESCE);</div><div class='del'>-</div><div class='del'>-	skb-&gt;priority = CPL_PRIORITY_DATA;</div><div class='del'>-	return iwch_cxgb3_ofld_send(ep-&gt;com.tdev, skb);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_resume_tid(struct iwch_ep *ep)</div><div class='del'>-{</div><div class='del'>-	struct cpl_set_tcb_field *req;</div><div class='del'>-	struct sk_buff *skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);</div><div class='del'>-</div><div class='del'>-	if (!skb)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	req = skb_put(skb, sizeof(*req));</div><div class='del'>-	req-&gt;wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));</div><div class='del'>-	req-&gt;wr.wr_lo = htonl(V_WR_TID(ep-&gt;hwtid));</div><div class='del'>-	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, ep-&gt;hwtid));</div><div class='del'>-	req-&gt;reply = 0;</div><div class='del'>-	req-&gt;cpu_idx = 0;</div><div class='del'>-	req-&gt;word = htons(W_TCB_RX_QUIESCE);</div><div class='del'>-	req-&gt;mask = cpu_to_be64(1ULL &lt;&lt; S_TCB_RX_QUIESCE);</div><div class='del'>-	req-&gt;val = 0;</div><div class='del'>-</div><div class='del'>-	skb-&gt;priority = CPL_PRIORITY_DATA;</div><div class='del'>-	return iwch_cxgb3_ofld_send(ep-&gt;com.tdev, skb);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_emss(struct iwch_ep *ep, u16 opt)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s ep %p opt %u\n", __func__, ep, opt);</div><div class='del'>-	ep-&gt;emss = T3C_DATA(ep-&gt;com.tdev)-&gt;mtus[G_TCPOPT_MSS(opt)] - 40;</div><div class='del'>-	if (G_TCPOPT_TSTAMP(opt))</div><div class='del'>-		ep-&gt;emss -= 12;</div><div class='del'>-	if (ep-&gt;emss &lt; 128)</div><div class='del'>-		ep-&gt;emss = 128;</div><div class='del'>-	pr_debug("emss=%d\n", ep-&gt;emss);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static enum iwch_ep_state state_read(struct iwch_ep_common *epc)</div><div class='del'>-{</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	enum iwch_ep_state state;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;epc-&gt;lock, flags);</div><div class='del'>-	state = epc-&gt;state;</div><div class='del'>-	spin_unlock_irqrestore(&amp;epc-&gt;lock, flags);</div><div class='del'>-	return state;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void __state_set(struct iwch_ep_common *epc, enum iwch_ep_state new)</div><div class='del'>-{</div><div class='del'>-	epc-&gt;state = new;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void state_set(struct iwch_ep_common *epc, enum iwch_ep_state new)</div><div class='del'>-{</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;epc-&gt;lock, flags);</div><div class='del'>-	pr_debug("%s - %s -&gt; %s\n", __func__, states[epc-&gt;state], states[new]);</div><div class='del'>-	__state_set(epc, new);</div><div class='del'>-	spin_unlock_irqrestore(&amp;epc-&gt;lock, flags);</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void *alloc_ep(int size, gfp_t gfp)</div><div class='del'>-{</div><div class='del'>-	struct iwch_ep_common *epc;</div><div class='del'>-</div><div class='del'>-	epc = kzalloc(size, gfp);</div><div class='del'>-	if (epc) {</div><div class='del'>-		kref_init(&amp;epc-&gt;kref);</div><div class='del'>-		spin_lock_init(&amp;epc-&gt;lock);</div><div class='del'>-		init_waitqueue_head(&amp;epc-&gt;waitq);</div><div class='del'>-	}</div><div class='del'>-	pr_debug("%s alloc ep %p\n", __func__, epc);</div><div class='del'>-	return epc;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void __free_ep(struct kref *kref)</div><div class='del'>-{</div><div class='del'>-	struct iwch_ep *ep;</div><div class='del'>-	ep = container_of(container_of(kref, struct iwch_ep_common, kref),</div><div class='del'>-			  struct iwch_ep, com);</div><div class='del'>-	pr_debug("%s ep %p state %s\n",</div><div class='del'>-		 __func__, ep, states[state_read(&amp;ep-&gt;com)]);</div><div class='del'>-	if (test_bit(RELEASE_RESOURCES, &amp;ep-&gt;com.flags)) {</div><div class='del'>-		cxgb3_remove_tid(ep-&gt;com.tdev, (void *)ep, ep-&gt;hwtid);</div><div class='del'>-		dst_release(ep-&gt;dst);</div><div class='del'>-		l2t_release(ep-&gt;com.tdev, ep-&gt;l2t);</div><div class='del'>-	}</div><div class='del'>-	kfree(ep);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void release_ep_resources(struct iwch_ep *ep)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s ep %p tid %d\n", __func__, ep, ep-&gt;hwtid);</div><div class='del'>-	set_bit(RELEASE_RESOURCES, &amp;ep-&gt;com.flags);</div><div class='del'>-	put_ep(&amp;ep-&gt;com);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int status2errno(int status)</div><div class='del'>-{</div><div class='del'>-	switch (status) {</div><div class='del'>-	case CPL_ERR_NONE:</div><div class='del'>-		return 0;</div><div class='del'>-	case CPL_ERR_CONN_RESET:</div><div class='del'>-		return -ECONNRESET;</div><div class='del'>-	case CPL_ERR_ARP_MISS:</div><div class='del'>-		return -EHOSTUNREACH;</div><div class='del'>-	case CPL_ERR_CONN_TIMEDOUT:</div><div class='del'>-		return -ETIMEDOUT;</div><div class='del'>-	case CPL_ERR_TCAM_FULL:</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	case CPL_ERR_CONN_EXIST:</div><div class='del'>-		return -EADDRINUSE;</div><div class='del'>-	default:</div><div class='del'>-		return -EIO;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Try and reuse skbs already allocated...</div><div class='del'>- */</div><div class='del'>-static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp)</div><div class='del'>-{</div><div class='del'>-	if (skb &amp;&amp; !skb_is_nonlinear(skb) &amp;&amp; !skb_cloned(skb)) {</div><div class='del'>-		skb_trim(skb, 0);</div><div class='del'>-		skb_get(skb);</div><div class='del'>-	} else {</div><div class='del'>-		skb = alloc_skb(len, gfp);</div><div class='del'>-	}</div><div class='del'>-	return skb;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct rtable *find_route(struct t3cdev *dev, __be32 local_ip,</div><div class='del'>-				 __be32 peer_ip, __be16 local_port,</div><div class='del'>-				 __be16 peer_port, u8 tos)</div><div class='del'>-{</div><div class='del'>-	struct rtable *rt;</div><div class='del'>-	struct flowi4 fl4;</div><div class='del'>-</div><div class='del'>-	rt = ip_route_output_ports(&amp;init_net, &amp;fl4, NULL, peer_ip, local_ip,</div><div class='del'>-				   peer_port, local_port, IPPROTO_TCP,</div><div class='del'>-				   tos, 0);</div><div class='del'>-	if (IS_ERR(rt))</div><div class='del'>-		return NULL;</div><div class='del'>-	return rt;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static unsigned int find_best_mtu(const struct t3c_data *d, unsigned short mtu)</div><div class='del'>-{</div><div class='del'>-	int i = 0;</div><div class='del'>-</div><div class='del'>-	while (i &lt; d-&gt;nmtus - 1 &amp;&amp; d-&gt;mtus[i + 1] &lt;= mtu)</div><div class='del'>-		++i;</div><div class='del'>-	return i;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void arp_failure_discard(struct t3cdev *dev, struct sk_buff *skb)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s t3cdev %p\n", __func__, dev);</div><div class='del'>-	kfree_skb(skb);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Handle an ARP failure for an active open.</div><div class='del'>- */</div><div class='del'>-static void act_open_req_arp_failure(struct t3cdev *dev, struct sk_buff *skb)</div><div class='del'>-{</div><div class='del'>-	pr_err("ARP failure during connect\n");</div><div class='del'>-	kfree_skb(skb);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Handle an ARP failure for a CPL_ABORT_REQ.  Change it into a no RST variant</div><div class='del'>- * and send it along.</div><div class='del'>- */</div><div class='del'>-static void abort_arp_failure(struct t3cdev *dev, struct sk_buff *skb)</div><div class='del'>-{</div><div class='del'>-	struct cpl_abort_req *req = cplhdr(skb);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s t3cdev %p\n", __func__, dev);</div><div class='del'>-	req-&gt;cmd = CPL_ABORT_NO_RST;</div><div class='del'>-	iwch_cxgb3_ofld_send(dev, skb);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int send_halfclose(struct iwch_ep *ep, gfp_t gfp)</div><div class='del'>-{</div><div class='del'>-	struct cpl_close_con_req *req;</div><div class='del'>-	struct sk_buff *skb;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	skb = get_skb(NULL, sizeof(*req), gfp);</div><div class='del'>-	if (!skb) {</div><div class='del'>-		pr_err("%s - failed to alloc skb\n", __func__);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	skb-&gt;priority = CPL_PRIORITY_DATA;</div><div class='del'>-	set_arp_failure_handler(skb, arp_failure_discard);</div><div class='del'>-	req = skb_put(skb, sizeof(*req));</div><div class='del'>-	req-&gt;wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));</div><div class='del'>-	req-&gt;wr.wr_lo = htonl(V_WR_TID(ep-&gt;hwtid));</div><div class='del'>-	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, ep-&gt;hwtid));</div><div class='del'>-	return iwch_l2t_send(ep-&gt;com.tdev, skb, ep-&gt;l2t);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int send_abort(struct iwch_ep *ep, struct sk_buff *skb, gfp_t gfp)</div><div class='del'>-{</div><div class='del'>-	struct cpl_abort_req *req;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	skb = get_skb(skb, sizeof(*req), gfp);</div><div class='del'>-	if (!skb) {</div><div class='del'>-		pr_err("%s - failed to alloc skb\n", __func__);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	skb-&gt;priority = CPL_PRIORITY_DATA;</div><div class='del'>-	set_arp_failure_handler(skb, abort_arp_failure);</div><div class='del'>-	req = skb_put_zero(skb, sizeof(*req));</div><div class='del'>-	req-&gt;wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));</div><div class='del'>-	req-&gt;wr.wr_lo = htonl(V_WR_TID(ep-&gt;hwtid));</div><div class='del'>-	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, ep-&gt;hwtid));</div><div class='del'>-	req-&gt;cmd = CPL_ABORT_SEND_RST;</div><div class='del'>-	return iwch_l2t_send(ep-&gt;com.tdev, skb, ep-&gt;l2t);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int send_connect(struct iwch_ep *ep)</div><div class='del'>-{</div><div class='del'>-	struct cpl_act_open_req *req;</div><div class='del'>-	struct sk_buff *skb;</div><div class='del'>-	u32 opt0h, opt0l, opt2;</div><div class='del'>-	unsigned int mtu_idx;</div><div class='del'>-	int wscale;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-</div><div class='del'>-	skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);</div><div class='del'>-	if (!skb) {</div><div class='del'>-		pr_err("%s - failed to alloc skb\n", __func__);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	mtu_idx = find_best_mtu(T3C_DATA(ep-&gt;com.tdev), dst_mtu(ep-&gt;dst));</div><div class='del'>-	wscale = compute_wscale(rcv_win);</div><div class='del'>-	opt0h = V_NAGLE(0) |</div><div class='del'>-	    V_NO_CONG(nocong) |</div><div class='del'>-	    V_KEEP_ALIVE(1) |</div><div class='del'>-	    F_TCAM_BYPASS |</div><div class='del'>-	    V_WND_SCALE(wscale) |</div><div class='del'>-	    V_MSS_IDX(mtu_idx) |</div><div class='del'>-	    V_L2T_IDX(ep-&gt;l2t-&gt;idx) | V_TX_CHANNEL(ep-&gt;l2t-&gt;smt_idx);</div><div class='del'>-	opt0l = V_TOS((ep-&gt;tos &gt;&gt; 2) &amp; M_TOS) | V_RCV_BUFSIZ(rcv_win&gt;&gt;10);</div><div class='del'>-	opt2 = F_RX_COALESCE_VALID | V_RX_COALESCE(0) | V_FLAVORS_VALID(1) |</div><div class='del'>-	       V_CONG_CONTROL_FLAVOR(cong_flavor);</div><div class='del'>-	skb-&gt;priority = CPL_PRIORITY_SETUP;</div><div class='del'>-	set_arp_failure_handler(skb, act_open_req_arp_failure);</div><div class='del'>-</div><div class='del'>-	req = skb_put(skb, sizeof(*req));</div><div class='del'>-	req-&gt;wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));</div><div class='del'>-	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, ep-&gt;atid));</div><div class='del'>-	req-&gt;local_port = ep-&gt;com.local_addr.sin_port;</div><div class='del'>-	req-&gt;peer_port = ep-&gt;com.remote_addr.sin_port;</div><div class='del'>-	req-&gt;local_ip = ep-&gt;com.local_addr.sin_addr.s_addr;</div><div class='del'>-	req-&gt;peer_ip = ep-&gt;com.remote_addr.sin_addr.s_addr;</div><div class='del'>-	req-&gt;opt0h = htonl(opt0h);</div><div class='del'>-	req-&gt;opt0l = htonl(opt0l);</div><div class='del'>-	req-&gt;params = 0;</div><div class='del'>-	req-&gt;opt2 = htonl(opt2);</div><div class='del'>-	return iwch_l2t_send(ep-&gt;com.tdev, skb, ep-&gt;l2t);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void send_mpa_req(struct iwch_ep *ep, struct sk_buff *skb)</div><div class='del'>-{</div><div class='del'>-	int mpalen;</div><div class='del'>-	struct tx_data_wr *req;</div><div class='del'>-	struct mpa_message *mpa;</div><div class='del'>-	int len;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p pd_len %d\n", __func__, ep, ep-&gt;plen);</div><div class='del'>-</div><div class='del'>-	BUG_ON(skb_cloned(skb));</div><div class='del'>-</div><div class='del'>-	mpalen = sizeof(*mpa) + ep-&gt;plen;</div><div class='del'>-	if (skb-&gt;data + mpalen + sizeof(*req) &gt; skb_end_pointer(skb)) {</div><div class='del'>-		kfree_skb(skb);</div><div class='del'>-		skb=alloc_skb(mpalen + sizeof(*req), GFP_KERNEL);</div><div class='del'>-		if (!skb) {</div><div class='del'>-			connect_reply_upcall(ep, -ENOMEM);</div><div class='del'>-			return;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	skb_trim(skb, 0);</div><div class='del'>-	skb_reserve(skb, sizeof(*req));</div><div class='del'>-	skb_put(skb, mpalen);</div><div class='del'>-	skb-&gt;priority = CPL_PRIORITY_DATA;</div><div class='del'>-	mpa = (struct mpa_message *) skb-&gt;data;</div><div class='del'>-	memset(mpa, 0, sizeof(*mpa));</div><div class='del'>-	memcpy(mpa-&gt;key, MPA_KEY_REQ, sizeof(mpa-&gt;key));</div><div class='del'>-	mpa-&gt;flags = (crc_enabled ? MPA_CRC : 0) |</div><div class='del'>-		     (markers_enabled ? MPA_MARKERS : 0);</div><div class='del'>-	mpa-&gt;private_data_size = htons(ep-&gt;plen);</div><div class='del'>-	mpa-&gt;revision = mpa_rev;</div><div class='del'>-</div><div class='del'>-	if (ep-&gt;plen)</div><div class='del'>-		memcpy(mpa-&gt;private_data, ep-&gt;mpa_pkt + sizeof(*mpa), ep-&gt;plen);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Reference the mpa skb.  This ensures the data area</div><div class='del'>-	 * will remain in memory until the hw acks the tx.</div><div class='del'>-	 * Function tx_ack() will deref it.</div><div class='del'>-	 */</div><div class='del'>-	skb_get(skb);</div><div class='del'>-	set_arp_failure_handler(skb, arp_failure_discard);</div><div class='del'>-	skb_reset_transport_header(skb);</div><div class='del'>-	len = skb-&gt;len;</div><div class='del'>-	req = skb_push(skb, sizeof(*req));</div><div class='del'>-	req-&gt;wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)|F_WR_COMPL);</div><div class='del'>-	req-&gt;wr_lo = htonl(V_WR_TID(ep-&gt;hwtid));</div><div class='del'>-	req-&gt;len = htonl(len);</div><div class='del'>-	req-&gt;param = htonl(V_TX_PORT(ep-&gt;l2t-&gt;smt_idx) |</div><div class='del'>-			   V_TX_SNDBUF(snd_win&gt;&gt;15));</div><div class='del'>-	req-&gt;flags = htonl(F_TX_INIT);</div><div class='del'>-	req-&gt;sndseq = htonl(ep-&gt;snd_seq);</div><div class='del'>-	BUG_ON(ep-&gt;mpa_skb);</div><div class='del'>-	ep-&gt;mpa_skb = skb;</div><div class='del'>-	iwch_l2t_send(ep-&gt;com.tdev, skb, ep-&gt;l2t);</div><div class='del'>-	start_ep_timer(ep);</div><div class='del'>-	state_set(&amp;ep-&gt;com, MPA_REQ_SENT);</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int send_mpa_reject(struct iwch_ep *ep, const void *pdata, u8 plen)</div><div class='del'>-{</div><div class='del'>-	int mpalen;</div><div class='del'>-	struct tx_data_wr *req;</div><div class='del'>-	struct mpa_message *mpa;</div><div class='del'>-	struct sk_buff *skb;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p plen %d\n", __func__, ep, plen);</div><div class='del'>-</div><div class='del'>-	mpalen = sizeof(*mpa) + plen;</div><div class='del'>-</div><div class='del'>-	skb = get_skb(NULL, mpalen + sizeof(*req), GFP_KERNEL);</div><div class='del'>-	if (!skb) {</div><div class='del'>-		pr_err("%s - cannot alloc skb!\n", __func__);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	skb_reserve(skb, sizeof(*req));</div><div class='del'>-	mpa = skb_put(skb, mpalen);</div><div class='del'>-	memset(mpa, 0, sizeof(*mpa));</div><div class='del'>-	memcpy(mpa-&gt;key, MPA_KEY_REP, sizeof(mpa-&gt;key));</div><div class='del'>-	mpa-&gt;flags = MPA_REJECT;</div><div class='del'>-	mpa-&gt;revision = mpa_rev;</div><div class='del'>-	mpa-&gt;private_data_size = htons(plen);</div><div class='del'>-	if (plen)</div><div class='del'>-		memcpy(mpa-&gt;private_data, pdata, plen);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Reference the mpa skb again.  This ensures the data area</div><div class='del'>-	 * will remain in memory until the hw acks the tx.</div><div class='del'>-	 * Function tx_ack() will deref it.</div><div class='del'>-	 */</div><div class='del'>-	skb_get(skb);</div><div class='del'>-	skb-&gt;priority = CPL_PRIORITY_DATA;</div><div class='del'>-	set_arp_failure_handler(skb, arp_failure_discard);</div><div class='del'>-	skb_reset_transport_header(skb);</div><div class='del'>-	req = skb_push(skb, sizeof(*req));</div><div class='del'>-	req-&gt;wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)|F_WR_COMPL);</div><div class='del'>-	req-&gt;wr_lo = htonl(V_WR_TID(ep-&gt;hwtid));</div><div class='del'>-	req-&gt;len = htonl(mpalen);</div><div class='del'>-	req-&gt;param = htonl(V_TX_PORT(ep-&gt;l2t-&gt;smt_idx) |</div><div class='del'>-			   V_TX_SNDBUF(snd_win&gt;&gt;15));</div><div class='del'>-	req-&gt;flags = htonl(F_TX_INIT);</div><div class='del'>-	req-&gt;sndseq = htonl(ep-&gt;snd_seq);</div><div class='del'>-	BUG_ON(ep-&gt;mpa_skb);</div><div class='del'>-	ep-&gt;mpa_skb = skb;</div><div class='del'>-	return iwch_l2t_send(ep-&gt;com.tdev, skb, ep-&gt;l2t);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int send_mpa_reply(struct iwch_ep *ep, const void *pdata, u8 plen)</div><div class='del'>-{</div><div class='del'>-	int mpalen;</div><div class='del'>-	struct tx_data_wr *req;</div><div class='del'>-	struct mpa_message *mpa;</div><div class='del'>-	int len;</div><div class='del'>-	struct sk_buff *skb;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p plen %d\n", __func__, ep, plen);</div><div class='del'>-</div><div class='del'>-	mpalen = sizeof(*mpa) + plen;</div><div class='del'>-</div><div class='del'>-	skb = get_skb(NULL, mpalen + sizeof(*req), GFP_KERNEL);</div><div class='del'>-	if (!skb) {</div><div class='del'>-		pr_err("%s - cannot alloc skb!\n", __func__);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	skb-&gt;priority = CPL_PRIORITY_DATA;</div><div class='del'>-	skb_reserve(skb, sizeof(*req));</div><div class='del'>-	mpa = skb_put(skb, mpalen);</div><div class='del'>-	memset(mpa, 0, sizeof(*mpa));</div><div class='del'>-	memcpy(mpa-&gt;key, MPA_KEY_REP, sizeof(mpa-&gt;key));</div><div class='del'>-	mpa-&gt;flags = (ep-&gt;mpa_attr.crc_enabled ? MPA_CRC : 0) |</div><div class='del'>-		     (markers_enabled ? MPA_MARKERS : 0);</div><div class='del'>-	mpa-&gt;revision = mpa_rev;</div><div class='del'>-	mpa-&gt;private_data_size = htons(plen);</div><div class='del'>-	if (plen)</div><div class='del'>-		memcpy(mpa-&gt;private_data, pdata, plen);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Reference the mpa skb.  This ensures the data area</div><div class='del'>-	 * will remain in memory until the hw acks the tx.</div><div class='del'>-	 * Function tx_ack() will deref it.</div><div class='del'>-	 */</div><div class='del'>-	skb_get(skb);</div><div class='del'>-	set_arp_failure_handler(skb, arp_failure_discard);</div><div class='del'>-	skb_reset_transport_header(skb);</div><div class='del'>-	len = skb-&gt;len;</div><div class='del'>-	req = skb_push(skb, sizeof(*req));</div><div class='del'>-	req-&gt;wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)|F_WR_COMPL);</div><div class='del'>-	req-&gt;wr_lo = htonl(V_WR_TID(ep-&gt;hwtid));</div><div class='del'>-	req-&gt;len = htonl(len);</div><div class='del'>-	req-&gt;param = htonl(V_TX_PORT(ep-&gt;l2t-&gt;smt_idx) |</div><div class='del'>-			   V_TX_SNDBUF(snd_win&gt;&gt;15));</div><div class='del'>-	req-&gt;flags = htonl(F_TX_INIT);</div><div class='del'>-	req-&gt;sndseq = htonl(ep-&gt;snd_seq);</div><div class='del'>-	ep-&gt;mpa_skb = skb;</div><div class='del'>-	state_set(&amp;ep-&gt;com, MPA_REP_SENT);</div><div class='del'>-	return iwch_l2t_send(ep-&gt;com.tdev, skb, ep-&gt;l2t);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)</div><div class='del'>-{</div><div class='del'>-	struct iwch_ep *ep = ctx;</div><div class='del'>-	struct cpl_act_establish *req = cplhdr(skb);</div><div class='del'>-	unsigned int tid = GET_TID(req);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p tid %d\n", __func__, ep, tid);</div><div class='del'>-</div><div class='del'>-	dst_confirm(ep-&gt;dst);</div><div class='del'>-</div><div class='del'>-	/* setup the hwtid for this connection */</div><div class='del'>-	ep-&gt;hwtid = tid;</div><div class='del'>-	cxgb3_insert_tid(ep-&gt;com.tdev, &amp;t3c_client, ep, tid);</div><div class='del'>-</div><div class='del'>-	ep-&gt;snd_seq = ntohl(req-&gt;snd_isn);</div><div class='del'>-	ep-&gt;rcv_seq = ntohl(req-&gt;rcv_isn);</div><div class='del'>-</div><div class='del'>-	set_emss(ep, ntohs(req-&gt;tcp_opt));</div><div class='del'>-</div><div class='del'>-	/* dealloc the atid */</div><div class='del'>-	cxgb3_free_atid(ep-&gt;com.tdev, ep-&gt;atid);</div><div class='del'>-</div><div class='del'>-	/* start MPA negotiation */</div><div class='del'>-	send_mpa_req(ep, skb);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void abort_connection(struct iwch_ep *ep, struct sk_buff *skb, gfp_t gfp)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s ep %p\n", __FILE__, ep);</div><div class='del'>-	state_set(&amp;ep-&gt;com, ABORTING);</div><div class='del'>-	send_abort(ep, skb, gfp);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void close_complete_upcall(struct iwch_ep *ep)</div><div class='del'>-{</div><div class='del'>-	struct iw_cm_event event;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	memset(&amp;event, 0, sizeof(event));</div><div class='del'>-	event.event = IW_CM_EVENT_CLOSE;</div><div class='del'>-	if (ep-&gt;com.cm_id) {</div><div class='del'>-		pr_debug("close complete delivered ep %p cm_id %p tid %d\n",</div><div class='del'>-			 ep, ep-&gt;com.cm_id, ep-&gt;hwtid);</div><div class='del'>-		ep-&gt;com.cm_id-&gt;event_handler(ep-&gt;com.cm_id, &amp;event);</div><div class='del'>-		ep-&gt;com.cm_id-&gt;rem_ref(ep-&gt;com.cm_id);</div><div class='del'>-		ep-&gt;com.cm_id = NULL;</div><div class='del'>-		ep-&gt;com.qp = NULL;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void peer_close_upcall(struct iwch_ep *ep)</div><div class='del'>-{</div><div class='del'>-	struct iw_cm_event event;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	memset(&amp;event, 0, sizeof(event));</div><div class='del'>-	event.event = IW_CM_EVENT_DISCONNECT;</div><div class='del'>-	if (ep-&gt;com.cm_id) {</div><div class='del'>-		pr_debug("peer close delivered ep %p cm_id %p tid %d\n",</div><div class='del'>-			 ep, ep-&gt;com.cm_id, ep-&gt;hwtid);</div><div class='del'>-		ep-&gt;com.cm_id-&gt;event_handler(ep-&gt;com.cm_id, &amp;event);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void peer_abort_upcall(struct iwch_ep *ep)</div><div class='del'>-{</div><div class='del'>-	struct iw_cm_event event;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	memset(&amp;event, 0, sizeof(event));</div><div class='del'>-	event.event = IW_CM_EVENT_CLOSE;</div><div class='del'>-	event.status = -ECONNRESET;</div><div class='del'>-	if (ep-&gt;com.cm_id) {</div><div class='del'>-		pr_debug("abort delivered ep %p cm_id %p tid %d\n", ep,</div><div class='del'>-			 ep-&gt;com.cm_id, ep-&gt;hwtid);</div><div class='del'>-		ep-&gt;com.cm_id-&gt;event_handler(ep-&gt;com.cm_id, &amp;event);</div><div class='del'>-		ep-&gt;com.cm_id-&gt;rem_ref(ep-&gt;com.cm_id);</div><div class='del'>-		ep-&gt;com.cm_id = NULL;</div><div class='del'>-		ep-&gt;com.qp = NULL;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void connect_reply_upcall(struct iwch_ep *ep, int status)</div><div class='del'>-{</div><div class='del'>-	struct iw_cm_event event;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p status %d\n", __func__, ep, status);</div><div class='del'>-	memset(&amp;event, 0, sizeof(event));</div><div class='del'>-	event.event = IW_CM_EVENT_CONNECT_REPLY;</div><div class='del'>-	event.status = status;</div><div class='del'>-	memcpy(&amp;event.local_addr, &amp;ep-&gt;com.local_addr,</div><div class='del'>-	       sizeof(ep-&gt;com.local_addr));</div><div class='del'>-	memcpy(&amp;event.remote_addr, &amp;ep-&gt;com.remote_addr,</div><div class='del'>-	       sizeof(ep-&gt;com.remote_addr));</div><div class='del'>-</div><div class='del'>-	if ((status == 0) || (status == -ECONNREFUSED)) {</div><div class='del'>-		event.private_data_len = ep-&gt;plen;</div><div class='del'>-		event.private_data = ep-&gt;mpa_pkt + sizeof(struct mpa_message);</div><div class='del'>-	}</div><div class='del'>-	if (ep-&gt;com.cm_id) {</div><div class='del'>-		pr_debug("%s ep %p tid %d status %d\n", __func__, ep,</div><div class='del'>-			 ep-&gt;hwtid, status);</div><div class='del'>-		ep-&gt;com.cm_id-&gt;event_handler(ep-&gt;com.cm_id, &amp;event);</div><div class='del'>-	}</div><div class='del'>-	if (status &lt; 0) {</div><div class='del'>-		ep-&gt;com.cm_id-&gt;rem_ref(ep-&gt;com.cm_id);</div><div class='del'>-		ep-&gt;com.cm_id = NULL;</div><div class='del'>-		ep-&gt;com.qp = NULL;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void connect_request_upcall(struct iwch_ep *ep)</div><div class='del'>-{</div><div class='del'>-	struct iw_cm_event event;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p tid %d\n", __func__, ep, ep-&gt;hwtid);</div><div class='del'>-	memset(&amp;event, 0, sizeof(event));</div><div class='del'>-	event.event = IW_CM_EVENT_CONNECT_REQUEST;</div><div class='del'>-	memcpy(&amp;event.local_addr, &amp;ep-&gt;com.local_addr,</div><div class='del'>-	       sizeof(ep-&gt;com.local_addr));</div><div class='del'>-	memcpy(&amp;event.remote_addr, &amp;ep-&gt;com.remote_addr,</div><div class='del'>-	       sizeof(ep-&gt;com.local_addr));</div><div class='del'>-	event.private_data_len = ep-&gt;plen;</div><div class='del'>-	event.private_data = ep-&gt;mpa_pkt + sizeof(struct mpa_message);</div><div class='del'>-	event.provider_data = ep;</div><div class='del'>-	/*</div><div class='del'>-	 * Until ird/ord negotiation via MPAv2 support is added, send max</div><div class='del'>-	 * supported values</div><div class='del'>-	 */</div><div class='del'>-	event.ird = event.ord = 8;</div><div class='del'>-	if (state_read(&amp;ep-&gt;parent_ep-&gt;com) != DEAD) {</div><div class='del'>-		get_ep(&amp;ep-&gt;com);</div><div class='del'>-		ep-&gt;parent_ep-&gt;com.cm_id-&gt;event_handler(</div><div class='del'>-						ep-&gt;parent_ep-&gt;com.cm_id,</div><div class='del'>-						&amp;event);</div><div class='del'>-	}</div><div class='del'>-	put_ep(&amp;ep-&gt;parent_ep-&gt;com);</div><div class='del'>-	ep-&gt;parent_ep = NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void established_upcall(struct iwch_ep *ep)</div><div class='del'>-{</div><div class='del'>-	struct iw_cm_event event;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	memset(&amp;event, 0, sizeof(event));</div><div class='del'>-	event.event = IW_CM_EVENT_ESTABLISHED;</div><div class='del'>-	/*</div><div class='del'>-	 * Until ird/ord negotiation via MPAv2 support is added, send max</div><div class='del'>-	 * supported values</div><div class='del'>-	 */</div><div class='del'>-	event.ird = event.ord = 8;</div><div class='del'>-	if (ep-&gt;com.cm_id) {</div><div class='del'>-		pr_debug("%s ep %p tid %d\n", __func__, ep, ep-&gt;hwtid);</div><div class='del'>-		ep-&gt;com.cm_id-&gt;event_handler(ep-&gt;com.cm_id, &amp;event);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int update_rx_credits(struct iwch_ep *ep, u32 credits)</div><div class='del'>-{</div><div class='del'>-	struct cpl_rx_data_ack *req;</div><div class='del'>-	struct sk_buff *skb;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p credits %u\n", __func__, ep, credits);</div><div class='del'>-	skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);</div><div class='del'>-	if (!skb) {</div><div class='del'>-		pr_err("update_rx_credits - cannot alloc skb!\n");</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	req = skb_put(skb, sizeof(*req));</div><div class='del'>-	req-&gt;wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));</div><div class='del'>-	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, ep-&gt;hwtid));</div><div class='del'>-	req-&gt;credit_dack = htonl(V_RX_CREDITS(credits) | V_RX_FORCE_ACK(1));</div><div class='del'>-	skb-&gt;priority = CPL_PRIORITY_ACK;</div><div class='del'>-	iwch_cxgb3_ofld_send(ep-&gt;com.tdev, skb);</div><div class='del'>-	return credits;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void process_mpa_reply(struct iwch_ep *ep, struct sk_buff *skb)</div><div class='del'>-{</div><div class='del'>-	struct mpa_message *mpa;</div><div class='del'>-	u16 plen;</div><div class='del'>-	struct iwch_qp_attributes attrs;</div><div class='del'>-	enum iwch_qp_attr_mask mask;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Stop mpa timer.  If it expired, then the state has</div><div class='del'>-	 * changed and we bail since ep_timeout already aborted</div><div class='del'>-	 * the connection.</div><div class='del'>-	 */</div><div class='del'>-	stop_ep_timer(ep);</div><div class='del'>-	if (state_read(&amp;ep-&gt;com) != MPA_REQ_SENT)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If we get more than the supported amount of private data</div><div class='del'>-	 * then we must fail this connection.</div><div class='del'>-	 */</div><div class='del'>-	if (ep-&gt;mpa_pkt_len + skb-&gt;len &gt; sizeof(ep-&gt;mpa_pkt)) {</div><div class='del'>-		err = -EINVAL;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * copy the new data into our accumulation buffer.</div><div class='del'>-	 */</div><div class='del'>-	skb_copy_from_linear_data(skb, &amp;(ep-&gt;mpa_pkt[ep-&gt;mpa_pkt_len]),</div><div class='del'>-				  skb-&gt;len);</div><div class='del'>-	ep-&gt;mpa_pkt_len += skb-&gt;len;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * if we don't even have the mpa message, then bail.</div><div class='del'>-	 */</div><div class='del'>-	if (ep-&gt;mpa_pkt_len &lt; sizeof(*mpa))</div><div class='del'>-		return;</div><div class='del'>-	mpa = (struct mpa_message *) ep-&gt;mpa_pkt;</div><div class='del'>-</div><div class='del'>-	/* Validate MPA header. */</div><div class='del'>-	if (mpa-&gt;revision != mpa_rev) {</div><div class='del'>-		err = -EPROTO;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-	if (memcmp(mpa-&gt;key, MPA_KEY_REP, sizeof(mpa-&gt;key))) {</div><div class='del'>-		err = -EPROTO;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	plen = ntohs(mpa-&gt;private_data_size);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Fail if there's too much private data.</div><div class='del'>-	 */</div><div class='del'>-	if (plen &gt; MPA_MAX_PRIVATE_DATA) {</div><div class='del'>-		err = -EPROTO;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If plen does not account for pkt size</div><div class='del'>-	 */</div><div class='del'>-	if (ep-&gt;mpa_pkt_len &gt; (sizeof(*mpa) + plen)) {</div><div class='del'>-		err = -EPROTO;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ep-&gt;plen = (u8) plen;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If we don't have all the pdata yet, then bail.</div><div class='del'>-	 * We'll continue process when more data arrives.</div><div class='del'>-	 */</div><div class='del'>-	if (ep-&gt;mpa_pkt_len &lt; (sizeof(*mpa) + plen))</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	if (mpa-&gt;flags &amp; MPA_REJECT) {</div><div class='del'>-		err = -ECONNREFUSED;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If we get here we have accumulated the entire mpa</div><div class='del'>-	 * start reply message including private data. And</div><div class='del'>-	 * the MPA header is valid.</div><div class='del'>-	 */</div><div class='del'>-	state_set(&amp;ep-&gt;com, FPDU_MODE);</div><div class='del'>-	ep-&gt;mpa_attr.initiator = 1;</div><div class='del'>-	ep-&gt;mpa_attr.crc_enabled = (mpa-&gt;flags &amp; MPA_CRC) | crc_enabled ? 1 : 0;</div><div class='del'>-	ep-&gt;mpa_attr.recv_marker_enabled = markers_enabled;</div><div class='del'>-	ep-&gt;mpa_attr.xmit_marker_enabled = mpa-&gt;flags &amp; MPA_MARKERS ? 1 : 0;</div><div class='del'>-	ep-&gt;mpa_attr.version = mpa_rev;</div><div class='del'>-	pr_debug("%s - crc_enabled=%d, recv_marker_enabled=%d, xmit_marker_enabled=%d, version=%d\n",</div><div class='del'>-		 __func__,</div><div class='del'>-		 ep-&gt;mpa_attr.crc_enabled, ep-&gt;mpa_attr.recv_marker_enabled,</div><div class='del'>-		 ep-&gt;mpa_attr.xmit_marker_enabled, ep-&gt;mpa_attr.version);</div><div class='del'>-</div><div class='del'>-	attrs.mpa_attr = ep-&gt;mpa_attr;</div><div class='del'>-	attrs.max_ird = ep-&gt;ird;</div><div class='del'>-	attrs.max_ord = ep-&gt;ord;</div><div class='del'>-	attrs.llp_stream_handle = ep;</div><div class='del'>-	attrs.next_state = IWCH_QP_STATE_RTS;</div><div class='del'>-</div><div class='del'>-	mask = IWCH_QP_ATTR_NEXT_STATE |</div><div class='del'>-	    IWCH_QP_ATTR_LLP_STREAM_HANDLE | IWCH_QP_ATTR_MPA_ATTR |</div><div class='del'>-	    IWCH_QP_ATTR_MAX_IRD | IWCH_QP_ATTR_MAX_ORD;</div><div class='del'>-</div><div class='del'>-	/* bind QP and TID with INIT_WR */</div><div class='del'>-	err = iwch_modify_qp(ep-&gt;com.qp-&gt;rhp,</div><div class='del'>-			     ep-&gt;com.qp, mask, &amp;attrs, 1);</div><div class='del'>-	if (err)</div><div class='del'>-		goto err;</div><div class='del'>-</div><div class='del'>-	if (peer2peer &amp;&amp; iwch_rqes_posted(ep-&gt;com.qp) == 0) {</div><div class='del'>-		iwch_post_zb_read(ep);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	goto out;</div><div class='del'>-err:</div><div class='del'>-	abort_connection(ep, skb, GFP_KERNEL);</div><div class='del'>-out:</div><div class='del'>-	connect_reply_upcall(ep, err);</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void process_mpa_request(struct iwch_ep *ep, struct sk_buff *skb)</div><div class='del'>-{</div><div class='del'>-	struct mpa_message *mpa;</div><div class='del'>-	u16 plen;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Stop mpa timer.  If it expired, then the state has</div><div class='del'>-	 * changed and we bail since ep_timeout already aborted</div><div class='del'>-	 * the connection.</div><div class='del'>-	 */</div><div class='del'>-	stop_ep_timer(ep);</div><div class='del'>-	if (state_read(&amp;ep-&gt;com) != MPA_REQ_WAIT)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If we get more than the supported amount of private data</div><div class='del'>-	 * then we must fail this connection.</div><div class='del'>-	 */</div><div class='del'>-	if (ep-&gt;mpa_pkt_len + skb-&gt;len &gt; sizeof(ep-&gt;mpa_pkt)) {</div><div class='del'>-		abort_connection(ep, skb, GFP_KERNEL);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	pr_debug("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Copy the new data into our accumulation buffer.</div><div class='del'>-	 */</div><div class='del'>-	skb_copy_from_linear_data(skb, &amp;(ep-&gt;mpa_pkt[ep-&gt;mpa_pkt_len]),</div><div class='del'>-				  skb-&gt;len);</div><div class='del'>-	ep-&gt;mpa_pkt_len += skb-&gt;len;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If we don't even have the mpa message, then bail.</div><div class='del'>-	 * We'll continue process when more data arrives.</div><div class='del'>-	 */</div><div class='del'>-	if (ep-&gt;mpa_pkt_len &lt; sizeof(*mpa))</div><div class='del'>-		return;</div><div class='del'>-	pr_debug("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__);</div><div class='del'>-	mpa = (struct mpa_message *) ep-&gt;mpa_pkt;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Validate MPA Header.</div><div class='del'>-	 */</div><div class='del'>-	if (mpa-&gt;revision != mpa_rev) {</div><div class='del'>-		abort_connection(ep, skb, GFP_KERNEL);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (memcmp(mpa-&gt;key, MPA_KEY_REQ, sizeof(mpa-&gt;key))) {</div><div class='del'>-		abort_connection(ep, skb, GFP_KERNEL);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	plen = ntohs(mpa-&gt;private_data_size);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Fail if there's too much private data.</div><div class='del'>-	 */</div><div class='del'>-	if (plen &gt; MPA_MAX_PRIVATE_DATA) {</div><div class='del'>-		abort_connection(ep, skb, GFP_KERNEL);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If plen does not account for pkt size</div><div class='del'>-	 */</div><div class='del'>-	if (ep-&gt;mpa_pkt_len &gt; (sizeof(*mpa) + plen)) {</div><div class='del'>-		abort_connection(ep, skb, GFP_KERNEL);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-	ep-&gt;plen = (u8) plen;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If we don't have all the pdata yet, then bail.</div><div class='del'>-	 */</div><div class='del'>-	if (ep-&gt;mpa_pkt_len &lt; (sizeof(*mpa) + plen))</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If we get here we have accumulated the entire mpa</div><div class='del'>-	 * start reply message including private data.</div><div class='del'>-	 */</div><div class='del'>-	ep-&gt;mpa_attr.initiator = 0;</div><div class='del'>-	ep-&gt;mpa_attr.crc_enabled = (mpa-&gt;flags &amp; MPA_CRC) | crc_enabled ? 1 : 0;</div><div class='del'>-	ep-&gt;mpa_attr.recv_marker_enabled = markers_enabled;</div><div class='del'>-	ep-&gt;mpa_attr.xmit_marker_enabled = mpa-&gt;flags &amp; MPA_MARKERS ? 1 : 0;</div><div class='del'>-	ep-&gt;mpa_attr.version = mpa_rev;</div><div class='del'>-	pr_debug("%s - crc_enabled=%d, recv_marker_enabled=%d, xmit_marker_enabled=%d, version=%d\n",</div><div class='del'>-		 __func__,</div><div class='del'>-		 ep-&gt;mpa_attr.crc_enabled, ep-&gt;mpa_attr.recv_marker_enabled,</div><div class='del'>-		 ep-&gt;mpa_attr.xmit_marker_enabled, ep-&gt;mpa_attr.version);</div><div class='del'>-</div><div class='del'>-	state_set(&amp;ep-&gt;com, MPA_REQ_RCVD);</div><div class='del'>-</div><div class='del'>-	/* drive upcall */</div><div class='del'>-	connect_request_upcall(ep);</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int rx_data(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)</div><div class='del'>-{</div><div class='del'>-	struct iwch_ep *ep = ctx;</div><div class='del'>-	struct cpl_rx_data *hdr = cplhdr(skb);</div><div class='del'>-	unsigned int dlen = ntohs(hdr-&gt;len);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p dlen %u\n", __func__, ep, dlen);</div><div class='del'>-</div><div class='del'>-	skb_pull(skb, sizeof(*hdr));</div><div class='del'>-	skb_trim(skb, dlen);</div><div class='del'>-</div><div class='del'>-	ep-&gt;rcv_seq += dlen;</div><div class='del'>-	BUG_ON(ep-&gt;rcv_seq != (ntohl(hdr-&gt;seq) + dlen));</div><div class='del'>-</div><div class='del'>-	switch (state_read(&amp;ep-&gt;com)) {</div><div class='del'>-	case MPA_REQ_SENT:</div><div class='del'>-		process_mpa_reply(ep, skb);</div><div class='del'>-		break;</div><div class='del'>-	case MPA_REQ_WAIT:</div><div class='del'>-		process_mpa_request(ep, skb);</div><div class='del'>-		break;</div><div class='del'>-	case MPA_REP_SENT:</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		pr_err("%s Unexpected streaming data. ep %p state %d tid %d\n",</div><div class='del'>-		       __func__, ep, state_read(&amp;ep-&gt;com), ep-&gt;hwtid);</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * The ep will timeout and inform the ULP of the failure.</div><div class='del'>-		 * See ep_timeout().</div><div class='del'>-		 */</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* update RX credits */</div><div class='del'>-	update_rx_credits(ep, dlen);</div><div class='del'>-</div><div class='del'>-	return CPL_RET_BUF_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Upcall from the adapter indicating data has been transmitted.</div><div class='del'>- * For us its just the single MPA request or reply.  We can now free</div><div class='del'>- * the skb holding the mpa message.</div><div class='del'>- */</div><div class='del'>-static int tx_ack(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)</div><div class='del'>-{</div><div class='del'>-	struct iwch_ep *ep = ctx;</div><div class='del'>-	struct cpl_wr_ack *hdr = cplhdr(skb);</div><div class='del'>-	unsigned int credits = ntohs(hdr-&gt;credits);</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int post_zb = 0;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p credits %u\n", __func__, ep, credits);</div><div class='del'>-</div><div class='del'>-	if (credits == 0) {</div><div class='del'>-		pr_debug("%s 0 credit ack  ep %p state %u\n",</div><div class='del'>-			 __func__, ep, state_read(&amp;ep-&gt;com));</div><div class='del'>-		return CPL_RET_BUF_DONE;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;ep-&gt;com.lock, flags);</div><div class='del'>-	BUG_ON(credits != 1);</div><div class='del'>-	dst_confirm(ep-&gt;dst);</div><div class='del'>-	if (!ep-&gt;mpa_skb) {</div><div class='del'>-		pr_debug("%s rdma_init wr_ack ep %p state %u\n",</div><div class='del'>-			 __func__, ep, ep-&gt;com.state);</div><div class='del'>-		if (ep-&gt;mpa_attr.initiator) {</div><div class='del'>-			pr_debug("%s initiator ep %p state %u\n",</div><div class='del'>-				 __func__, ep, ep-&gt;com.state);</div><div class='del'>-			if (peer2peer &amp;&amp; ep-&gt;com.state == FPDU_MODE)</div><div class='del'>-				post_zb = 1;</div><div class='del'>-		} else {</div><div class='del'>-			pr_debug("%s responder ep %p state %u\n",</div><div class='del'>-				 __func__, ep, ep-&gt;com.state);</div><div class='del'>-			if (ep-&gt;com.state == MPA_REQ_RCVD) {</div><div class='del'>-				ep-&gt;com.rpl_done = 1;</div><div class='del'>-				wake_up(&amp;ep-&gt;com.waitq);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		pr_debug("%s lsm ack ep %p state %u freeing skb\n",</div><div class='del'>-			 __func__, ep, ep-&gt;com.state);</div><div class='del'>-		kfree_skb(ep-&gt;mpa_skb);</div><div class='del'>-		ep-&gt;mpa_skb = NULL;</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;ep-&gt;com.lock, flags);</div><div class='del'>-	if (post_zb)</div><div class='del'>-		iwch_post_zb_read(ep);</div><div class='del'>-	return CPL_RET_BUF_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int abort_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)</div><div class='del'>-{</div><div class='del'>-	struct iwch_ep *ep = ctx;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int release = 0;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	BUG_ON(!ep);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * We get 2 abort replies from the HW.  The first one must</div><div class='del'>-	 * be ignored except for scribbling that we need one more.</div><div class='del'>-	 */</div><div class='del'>-	if (!test_and_set_bit(ABORT_REQ_IN_PROGRESS, &amp;ep-&gt;com.flags)) {</div><div class='del'>-		return CPL_RET_BUF_DONE;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;ep-&gt;com.lock, flags);</div><div class='del'>-	switch (ep-&gt;com.state) {</div><div class='del'>-	case ABORTING:</div><div class='del'>-		close_complete_upcall(ep);</div><div class='del'>-		__state_set(&amp;ep-&gt;com, DEAD);</div><div class='del'>-		release = 1;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		pr_err("%s ep %p state %d\n", __func__, ep, ep-&gt;com.state);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;ep-&gt;com.lock, flags);</div><div class='del'>-</div><div class='del'>-	if (release)</div><div class='del'>-		release_ep_resources(ep);</div><div class='del'>-	return CPL_RET_BUF_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Return whether a failed active open has allocated a TID</div><div class='del'>- */</div><div class='del'>-static inline int act_open_has_tid(int status)</div><div class='del'>-{</div><div class='del'>-	return status != CPL_ERR_TCAM_FULL &amp;&amp; status != CPL_ERR_CONN_EXIST &amp;&amp;</div><div class='del'>-	       status != CPL_ERR_ARP_MISS;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)</div><div class='del'>-{</div><div class='del'>-	struct iwch_ep *ep = ctx;</div><div class='del'>-	struct cpl_act_open_rpl *rpl = cplhdr(skb);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p status %u errno %d\n", __func__, ep, rpl-&gt;status,</div><div class='del'>-		 status2errno(rpl-&gt;status));</div><div class='del'>-	connect_reply_upcall(ep, status2errno(rpl-&gt;status));</div><div class='del'>-	state_set(&amp;ep-&gt;com, DEAD);</div><div class='del'>-	if (ep-&gt;com.tdev-&gt;type != T3A &amp;&amp; act_open_has_tid(rpl-&gt;status))</div><div class='del'>-		release_tid(ep-&gt;com.tdev, GET_TID(rpl), NULL);</div><div class='del'>-	cxgb3_free_atid(ep-&gt;com.tdev, ep-&gt;atid);</div><div class='del'>-	dst_release(ep-&gt;dst);</div><div class='del'>-	l2t_release(ep-&gt;com.tdev, ep-&gt;l2t);</div><div class='del'>-	put_ep(&amp;ep-&gt;com);</div><div class='del'>-	return CPL_RET_BUF_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int listen_start(struct iwch_listen_ep *ep)</div><div class='del'>-{</div><div class='del'>-	struct sk_buff *skb;</div><div class='del'>-	struct cpl_pass_open_req *req;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);</div><div class='del'>-	if (!skb) {</div><div class='del'>-		pr_err("t3c_listen_start failed to alloc skb!\n");</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	req = skb_put(skb, sizeof(*req));</div><div class='del'>-	req-&gt;wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));</div><div class='del'>-	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, ep-&gt;stid));</div><div class='del'>-	req-&gt;local_port = ep-&gt;com.local_addr.sin_port;</div><div class='del'>-	req-&gt;local_ip = ep-&gt;com.local_addr.sin_addr.s_addr;</div><div class='del'>-	req-&gt;peer_port = 0;</div><div class='del'>-	req-&gt;peer_ip = 0;</div><div class='del'>-	req-&gt;peer_netmask = 0;</div><div class='del'>-	req-&gt;opt0h = htonl(F_DELACK | F_TCAM_BYPASS);</div><div class='del'>-	req-&gt;opt0l = htonl(V_RCV_BUFSIZ(rcv_win&gt;&gt;10));</div><div class='del'>-	req-&gt;opt1 = htonl(V_CONN_POLICY(CPL_CONN_POLICY_ASK));</div><div class='del'>-</div><div class='del'>-	skb-&gt;priority = 1;</div><div class='del'>-	return iwch_cxgb3_ofld_send(ep-&gt;com.tdev, skb);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int pass_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)</div><div class='del'>-{</div><div class='del'>-	struct iwch_listen_ep *ep = ctx;</div><div class='del'>-	struct cpl_pass_open_rpl *rpl = cplhdr(skb);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p status %d error %d\n", __func__, ep,</div><div class='del'>-		 rpl-&gt;status, status2errno(rpl-&gt;status));</div><div class='del'>-	ep-&gt;com.rpl_err = status2errno(rpl-&gt;status);</div><div class='del'>-	ep-&gt;com.rpl_done = 1;</div><div class='del'>-	wake_up(&amp;ep-&gt;com.waitq);</div><div class='del'>-</div><div class='del'>-	return CPL_RET_BUF_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int listen_stop(struct iwch_listen_ep *ep)</div><div class='del'>-{</div><div class='del'>-	struct sk_buff *skb;</div><div class='del'>-	struct cpl_close_listserv_req *req;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);</div><div class='del'>-	if (!skb) {</div><div class='del'>-		pr_err("%s - failed to alloc skb\n", __func__);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	req = skb_put(skb, sizeof(*req));</div><div class='del'>-	req-&gt;wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));</div><div class='del'>-	req-&gt;cpu_idx = 0;</div><div class='del'>-	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, ep-&gt;stid));</div><div class='del'>-	skb-&gt;priority = 1;</div><div class='del'>-	return iwch_cxgb3_ofld_send(ep-&gt;com.tdev, skb);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int close_listsrv_rpl(struct t3cdev *tdev, struct sk_buff *skb,</div><div class='del'>-			     void *ctx)</div><div class='del'>-{</div><div class='del'>-	struct iwch_listen_ep *ep = ctx;</div><div class='del'>-	struct cpl_close_listserv_rpl *rpl = cplhdr(skb);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	ep-&gt;com.rpl_err = status2errno(rpl-&gt;status);</div><div class='del'>-	ep-&gt;com.rpl_done = 1;</div><div class='del'>-	wake_up(&amp;ep-&gt;com.waitq);</div><div class='del'>-	return CPL_RET_BUF_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void accept_cr(struct iwch_ep *ep, __be32 peer_ip, struct sk_buff *skb)</div><div class='del'>-{</div><div class='del'>-	struct cpl_pass_accept_rpl *rpl;</div><div class='del'>-	unsigned int mtu_idx;</div><div class='del'>-	u32 opt0h, opt0l, opt2;</div><div class='del'>-	int wscale;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	BUG_ON(skb_cloned(skb));</div><div class='del'>-	skb_trim(skb, sizeof(*rpl));</div><div class='del'>-	skb_get(skb);</div><div class='del'>-	mtu_idx = find_best_mtu(T3C_DATA(ep-&gt;com.tdev), dst_mtu(ep-&gt;dst));</div><div class='del'>-	wscale = compute_wscale(rcv_win);</div><div class='del'>-	opt0h = V_NAGLE(0) |</div><div class='del'>-	    V_NO_CONG(nocong) |</div><div class='del'>-	    V_KEEP_ALIVE(1) |</div><div class='del'>-	    F_TCAM_BYPASS |</div><div class='del'>-	    V_WND_SCALE(wscale) |</div><div class='del'>-	    V_MSS_IDX(mtu_idx) |</div><div class='del'>-	    V_L2T_IDX(ep-&gt;l2t-&gt;idx) | V_TX_CHANNEL(ep-&gt;l2t-&gt;smt_idx);</div><div class='del'>-	opt0l = V_TOS((ep-&gt;tos &gt;&gt; 2) &amp; M_TOS) | V_RCV_BUFSIZ(rcv_win&gt;&gt;10);</div><div class='del'>-	opt2 = F_RX_COALESCE_VALID | V_RX_COALESCE(0) | V_FLAVORS_VALID(1) |</div><div class='del'>-	       V_CONG_CONTROL_FLAVOR(cong_flavor);</div><div class='del'>-</div><div class='del'>-	rpl = cplhdr(skb);</div><div class='del'>-	rpl-&gt;wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));</div><div class='del'>-	OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, ep-&gt;hwtid));</div><div class='del'>-	rpl-&gt;peer_ip = peer_ip;</div><div class='del'>-	rpl-&gt;opt0h = htonl(opt0h);</div><div class='del'>-	rpl-&gt;opt0l_status = htonl(opt0l | CPL_PASS_OPEN_ACCEPT);</div><div class='del'>-	rpl-&gt;opt2 = htonl(opt2);</div><div class='del'>-	rpl-&gt;rsvd = rpl-&gt;opt2;	/* workaround for HW bug */</div><div class='del'>-	skb-&gt;priority = CPL_PRIORITY_SETUP;</div><div class='del'>-	iwch_l2t_send(ep-&gt;com.tdev, skb, ep-&gt;l2t);</div><div class='del'>-</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void reject_cr(struct t3cdev *tdev, u32 hwtid, __be32 peer_ip,</div><div class='del'>-		      struct sk_buff *skb)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s t3cdev %p tid %u peer_ip %x\n", __func__, tdev, hwtid,</div><div class='del'>-		 peer_ip);</div><div class='del'>-	BUG_ON(skb_cloned(skb));</div><div class='del'>-	skb_trim(skb, sizeof(struct cpl_tid_release));</div><div class='del'>-	skb_get(skb);</div><div class='del'>-</div><div class='del'>-	if (tdev-&gt;type != T3A)</div><div class='del'>-		release_tid(tdev, hwtid, skb);</div><div class='del'>-	else {</div><div class='del'>-		struct cpl_pass_accept_rpl *rpl;</div><div class='del'>-</div><div class='del'>-		rpl = cplhdr(skb);</div><div class='del'>-		skb-&gt;priority = CPL_PRIORITY_SETUP;</div><div class='del'>-		rpl-&gt;wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));</div><div class='del'>-		OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,</div><div class='del'>-						      hwtid));</div><div class='del'>-		rpl-&gt;peer_ip = peer_ip;</div><div class='del'>-		rpl-&gt;opt0h = htonl(F_TCAM_BYPASS);</div><div class='del'>-		rpl-&gt;opt0l_status = htonl(CPL_PASS_OPEN_REJECT);</div><div class='del'>-		rpl-&gt;opt2 = 0;</div><div class='del'>-		rpl-&gt;rsvd = rpl-&gt;opt2;</div><div class='del'>-		iwch_cxgb3_ofld_send(tdev, skb);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int pass_accept_req(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)</div><div class='del'>-{</div><div class='del'>-	struct iwch_ep *child_ep, *parent_ep = ctx;</div><div class='del'>-	struct cpl_pass_accept_req *req = cplhdr(skb);</div><div class='del'>-	unsigned int hwtid = GET_TID(req);</div><div class='del'>-	struct dst_entry *dst;</div><div class='del'>-	struct l2t_entry *l2t;</div><div class='del'>-	struct rtable *rt;</div><div class='del'>-	struct iff_mac tim;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s parent ep %p tid %u\n", __func__, parent_ep, hwtid);</div><div class='del'>-</div><div class='del'>-	if (state_read(&amp;parent_ep-&gt;com) != LISTEN) {</div><div class='del'>-		pr_err("%s - listening ep not in LISTEN\n", __func__);</div><div class='del'>-		goto reject;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Find the netdev for this connection request.</div><div class='del'>-	 */</div><div class='del'>-	tim.mac_addr = req-&gt;dst_mac;</div><div class='del'>-	tim.vlan_tag = ntohs(req-&gt;vlan_tag);</div><div class='del'>-	if (tdev-&gt;ctl(tdev, GET_IFF_FROM_MAC, &amp;tim) &lt; 0 || !tim.dev) {</div><div class='del'>-		pr_err("%s bad dst mac %pM\n", __func__, req-&gt;dst_mac);</div><div class='del'>-		goto reject;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Find output route */</div><div class='del'>-	rt = find_route(tdev,</div><div class='del'>-			req-&gt;local_ip,</div><div class='del'>-			req-&gt;peer_ip,</div><div class='del'>-			req-&gt;local_port,</div><div class='del'>-			req-&gt;peer_port, G_PASS_OPEN_TOS(ntohl(req-&gt;tos_tid)));</div><div class='del'>-	if (!rt) {</div><div class='del'>-		pr_err("%s - failed to find dst entry!\n", __func__);</div><div class='del'>-		goto reject;</div><div class='del'>-	}</div><div class='del'>-	dst = &amp;rt-&gt;dst;</div><div class='del'>-	l2t = t3_l2t_get(tdev, dst, NULL, &amp;req-&gt;peer_ip);</div><div class='del'>-	if (!l2t) {</div><div class='del'>-		pr_err("%s - failed to allocate l2t entry!\n", __func__);</div><div class='del'>-		dst_release(dst);</div><div class='del'>-		goto reject;</div><div class='del'>-	}</div><div class='del'>-	child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL);</div><div class='del'>-	if (!child_ep) {</div><div class='del'>-		pr_err("%s - failed to allocate ep entry!\n", __func__);</div><div class='del'>-		l2t_release(tdev, l2t);</div><div class='del'>-		dst_release(dst);</div><div class='del'>-		goto reject;</div><div class='del'>-	}</div><div class='del'>-	state_set(&amp;child_ep-&gt;com, CONNECTING);</div><div class='del'>-	child_ep-&gt;com.tdev = tdev;</div><div class='del'>-	child_ep-&gt;com.cm_id = NULL;</div><div class='del'>-	child_ep-&gt;com.local_addr.sin_family = AF_INET;</div><div class='del'>-	child_ep-&gt;com.local_addr.sin_port = req-&gt;local_port;</div><div class='del'>-	child_ep-&gt;com.local_addr.sin_addr.s_addr = req-&gt;local_ip;</div><div class='del'>-	child_ep-&gt;com.remote_addr.sin_family = AF_INET;</div><div class='del'>-	child_ep-&gt;com.remote_addr.sin_port = req-&gt;peer_port;</div><div class='del'>-	child_ep-&gt;com.remote_addr.sin_addr.s_addr = req-&gt;peer_ip;</div><div class='del'>-	get_ep(&amp;parent_ep-&gt;com);</div><div class='del'>-	child_ep-&gt;parent_ep = parent_ep;</div><div class='del'>-	child_ep-&gt;tos = G_PASS_OPEN_TOS(ntohl(req-&gt;tos_tid));</div><div class='del'>-	child_ep-&gt;l2t = l2t;</div><div class='del'>-	child_ep-&gt;dst = dst;</div><div class='del'>-	child_ep-&gt;hwtid = hwtid;</div><div class='del'>-	timer_setup(&amp;child_ep-&gt;timer, ep_timeout, 0);</div><div class='del'>-	cxgb3_insert_tid(tdev, &amp;t3c_client, child_ep, hwtid);</div><div class='del'>-	accept_cr(child_ep, req-&gt;peer_ip, skb);</div><div class='del'>-	goto out;</div><div class='del'>-reject:</div><div class='del'>-	reject_cr(tdev, hwtid, req-&gt;peer_ip, skb);</div><div class='del'>-out:</div><div class='del'>-	return CPL_RET_BUF_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int pass_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)</div><div class='del'>-{</div><div class='del'>-	struct iwch_ep *ep = ctx;</div><div class='del'>-	struct cpl_pass_establish *req = cplhdr(skb);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	ep-&gt;snd_seq = ntohl(req-&gt;snd_isn);</div><div class='del'>-	ep-&gt;rcv_seq = ntohl(req-&gt;rcv_isn);</div><div class='del'>-</div><div class='del'>-	set_emss(ep, ntohs(req-&gt;tcp_opt));</div><div class='del'>-</div><div class='del'>-	dst_confirm(ep-&gt;dst);</div><div class='del'>-	state_set(&amp;ep-&gt;com, MPA_REQ_WAIT);</div><div class='del'>-	start_ep_timer(ep);</div><div class='del'>-</div><div class='del'>-	return CPL_RET_BUF_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int peer_close(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)</div><div class='del'>-{</div><div class='del'>-	struct iwch_ep *ep = ctx;</div><div class='del'>-	struct iwch_qp_attributes attrs;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int disconnect = 1;</div><div class='del'>-	int release = 0;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	dst_confirm(ep-&gt;dst);</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;ep-&gt;com.lock, flags);</div><div class='del'>-	switch (ep-&gt;com.state) {</div><div class='del'>-	case MPA_REQ_WAIT:</div><div class='del'>-		__state_set(&amp;ep-&gt;com, CLOSING);</div><div class='del'>-		break;</div><div class='del'>-	case MPA_REQ_SENT:</div><div class='del'>-		__state_set(&amp;ep-&gt;com, CLOSING);</div><div class='del'>-		connect_reply_upcall(ep, -ECONNRESET);</div><div class='del'>-		break;</div><div class='del'>-	case MPA_REQ_RCVD:</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * We're gonna mark this puppy DEAD, but keep</div><div class='del'>-		 * the reference on it until the ULP accepts or</div><div class='del'>-		 * rejects the CR. Also wake up anyone waiting</div><div class='del'>-		 * in rdma connection migration (see iwch_accept_cr()).</div><div class='del'>-		 */</div><div class='del'>-		__state_set(&amp;ep-&gt;com, CLOSING);</div><div class='del'>-		ep-&gt;com.rpl_done = 1;</div><div class='del'>-		ep-&gt;com.rpl_err = -ECONNRESET;</div><div class='del'>-		pr_debug("waking up ep %p\n", ep);</div><div class='del'>-		wake_up(&amp;ep-&gt;com.waitq);</div><div class='del'>-		break;</div><div class='del'>-	case MPA_REP_SENT:</div><div class='del'>-		__state_set(&amp;ep-&gt;com, CLOSING);</div><div class='del'>-		ep-&gt;com.rpl_done = 1;</div><div class='del'>-		ep-&gt;com.rpl_err = -ECONNRESET;</div><div class='del'>-		pr_debug("waking up ep %p\n", ep);</div><div class='del'>-		wake_up(&amp;ep-&gt;com.waitq);</div><div class='del'>-		break;</div><div class='del'>-	case FPDU_MODE:</div><div class='del'>-		start_ep_timer(ep);</div><div class='del'>-		__state_set(&amp;ep-&gt;com, CLOSING);</div><div class='del'>-		attrs.next_state = IWCH_QP_STATE_CLOSING;</div><div class='del'>-		iwch_modify_qp(ep-&gt;com.qp-&gt;rhp, ep-&gt;com.qp,</div><div class='del'>-			       IWCH_QP_ATTR_NEXT_STATE, &amp;attrs, 1);</div><div class='del'>-		peer_close_upcall(ep);</div><div class='del'>-		break;</div><div class='del'>-	case ABORTING:</div><div class='del'>-		disconnect = 0;</div><div class='del'>-		break;</div><div class='del'>-	case CLOSING:</div><div class='del'>-		__state_set(&amp;ep-&gt;com, MORIBUND);</div><div class='del'>-		disconnect = 0;</div><div class='del'>-		break;</div><div class='del'>-	case MORIBUND:</div><div class='del'>-		stop_ep_timer(ep);</div><div class='del'>-		if (ep-&gt;com.cm_id &amp;&amp; ep-&gt;com.qp) {</div><div class='del'>-			attrs.next_state = IWCH_QP_STATE_IDLE;</div><div class='del'>-			iwch_modify_qp(ep-&gt;com.qp-&gt;rhp, ep-&gt;com.qp,</div><div class='del'>-				       IWCH_QP_ATTR_NEXT_STATE, &amp;attrs, 1);</div><div class='del'>-		}</div><div class='del'>-		close_complete_upcall(ep);</div><div class='del'>-		__state_set(&amp;ep-&gt;com, DEAD);</div><div class='del'>-		release = 1;</div><div class='del'>-		disconnect = 0;</div><div class='del'>-		break;</div><div class='del'>-	case DEAD:</div><div class='del'>-		disconnect = 0;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		BUG_ON(1);</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;ep-&gt;com.lock, flags);</div><div class='del'>-	if (disconnect)</div><div class='del'>-		iwch_ep_disconnect(ep, 0, GFP_KERNEL);</div><div class='del'>-	if (release)</div><div class='del'>-		release_ep_resources(ep);</div><div class='del'>-	return CPL_RET_BUF_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Returns whether an ABORT_REQ_RSS message is a negative advice.</div><div class='del'>- */</div><div class='del'>-static int is_neg_adv_abort(unsigned int status)</div><div class='del'>-{</div><div class='del'>-	return status == CPL_ERR_RTX_NEG_ADVICE ||</div><div class='del'>-	       status == CPL_ERR_PERSIST_NEG_ADVICE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int peer_abort(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)</div><div class='del'>-{</div><div class='del'>-	struct cpl_abort_req_rss *req = cplhdr(skb);</div><div class='del'>-	struct iwch_ep *ep = ctx;</div><div class='del'>-	struct cpl_abort_rpl *rpl;</div><div class='del'>-	struct sk_buff *rpl_skb;</div><div class='del'>-	struct iwch_qp_attributes attrs;</div><div class='del'>-	int ret;</div><div class='del'>-	int release = 0;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	if (is_neg_adv_abort(req-&gt;status)) {</div><div class='del'>-		pr_debug("%s neg_adv_abort ep %p tid %d\n", __func__, ep,</div><div class='del'>-			 ep-&gt;hwtid);</div><div class='del'>-		t3_l2t_send_event(ep-&gt;com.tdev, ep-&gt;l2t);</div><div class='del'>-		return CPL_RET_BUF_DONE;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * We get 2 peer aborts from the HW.  The first one must</div><div class='del'>-	 * be ignored except for scribbling that we need one more.</div><div class='del'>-	 */</div><div class='del'>-	if (!test_and_set_bit(PEER_ABORT_IN_PROGRESS, &amp;ep-&gt;com.flags)) {</div><div class='del'>-		return CPL_RET_BUF_DONE;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;ep-&gt;com.lock, flags);</div><div class='del'>-	pr_debug("%s ep %p state %u\n", __func__, ep, ep-&gt;com.state);</div><div class='del'>-	switch (ep-&gt;com.state) {</div><div class='del'>-	case CONNECTING:</div><div class='del'>-		break;</div><div class='del'>-	case MPA_REQ_WAIT:</div><div class='del'>-		stop_ep_timer(ep);</div><div class='del'>-		break;</div><div class='del'>-	case MPA_REQ_SENT:</div><div class='del'>-		stop_ep_timer(ep);</div><div class='del'>-		connect_reply_upcall(ep, -ECONNRESET);</div><div class='del'>-		break;</div><div class='del'>-	case MPA_REP_SENT:</div><div class='del'>-		ep-&gt;com.rpl_done = 1;</div><div class='del'>-		ep-&gt;com.rpl_err = -ECONNRESET;</div><div class='del'>-		pr_debug("waking up ep %p\n", ep);</div><div class='del'>-		wake_up(&amp;ep-&gt;com.waitq);</div><div class='del'>-		break;</div><div class='del'>-	case MPA_REQ_RCVD:</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * We're gonna mark this puppy DEAD, but keep</div><div class='del'>-		 * the reference on it until the ULP accepts or</div><div class='del'>-		 * rejects the CR. Also wake up anyone waiting</div><div class='del'>-		 * in rdma connection migration (see iwch_accept_cr()).</div><div class='del'>-		 */</div><div class='del'>-		ep-&gt;com.rpl_done = 1;</div><div class='del'>-		ep-&gt;com.rpl_err = -ECONNRESET;</div><div class='del'>-		pr_debug("waking up ep %p\n", ep);</div><div class='del'>-		wake_up(&amp;ep-&gt;com.waitq);</div><div class='del'>-		break;</div><div class='del'>-	case MORIBUND:</div><div class='del'>-	case CLOSING:</div><div class='del'>-		stop_ep_timer(ep);</div><div class='del'>-		/*FALLTHROUGH*/</div><div class='del'>-	case FPDU_MODE:</div><div class='del'>-		if (ep-&gt;com.cm_id &amp;&amp; ep-&gt;com.qp) {</div><div class='del'>-			attrs.next_state = IWCH_QP_STATE_ERROR;</div><div class='del'>-			ret = iwch_modify_qp(ep-&gt;com.qp-&gt;rhp,</div><div class='del'>-				     ep-&gt;com.qp, IWCH_QP_ATTR_NEXT_STATE,</div><div class='del'>-				     &amp;attrs, 1);</div><div class='del'>-			if (ret)</div><div class='del'>-				pr_err("%s - qp &lt;- error failed!\n", __func__);</div><div class='del'>-		}</div><div class='del'>-		peer_abort_upcall(ep);</div><div class='del'>-		break;</div><div class='del'>-	case ABORTING:</div><div class='del'>-		break;</div><div class='del'>-	case DEAD:</div><div class='del'>-		pr_debug("%s PEER_ABORT IN DEAD STATE!!!!\n", __func__);</div><div class='del'>-		spin_unlock_irqrestore(&amp;ep-&gt;com.lock, flags);</div><div class='del'>-		return CPL_RET_BUF_DONE;</div><div class='del'>-	default:</div><div class='del'>-		BUG_ON(1);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	dst_confirm(ep-&gt;dst);</div><div class='del'>-	if (ep-&gt;com.state != ABORTING) {</div><div class='del'>-		__state_set(&amp;ep-&gt;com, DEAD);</div><div class='del'>-		release = 1;</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;ep-&gt;com.lock, flags);</div><div class='del'>-</div><div class='del'>-	rpl_skb = get_skb(skb, sizeof(*rpl), GFP_KERNEL);</div><div class='del'>-	if (!rpl_skb) {</div><div class='del'>-		pr_err("%s - cannot allocate skb!\n", __func__);</div><div class='del'>-		release = 1;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-	rpl_skb-&gt;priority = CPL_PRIORITY_DATA;</div><div class='del'>-	rpl = skb_put(rpl_skb, sizeof(*rpl));</div><div class='del'>-	rpl-&gt;wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));</div><div class='del'>-	rpl-&gt;wr.wr_lo = htonl(V_WR_TID(ep-&gt;hwtid));</div><div class='del'>-	OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, ep-&gt;hwtid));</div><div class='del'>-	rpl-&gt;cmd = CPL_ABORT_NO_RST;</div><div class='del'>-	iwch_cxgb3_ofld_send(ep-&gt;com.tdev, rpl_skb);</div><div class='del'>-out:</div><div class='del'>-	if (release)</div><div class='del'>-		release_ep_resources(ep);</div><div class='del'>-	return CPL_RET_BUF_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int close_con_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)</div><div class='del'>-{</div><div class='del'>-	struct iwch_ep *ep = ctx;</div><div class='del'>-	struct iwch_qp_attributes attrs;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int release = 0;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	BUG_ON(!ep);</div><div class='del'>-</div><div class='del'>-	/* The cm_id may be null if we failed to connect */</div><div class='del'>-	spin_lock_irqsave(&amp;ep-&gt;com.lock, flags);</div><div class='del'>-	switch (ep-&gt;com.state) {</div><div class='del'>-	case CLOSING:</div><div class='del'>-		__state_set(&amp;ep-&gt;com, MORIBUND);</div><div class='del'>-		break;</div><div class='del'>-	case MORIBUND:</div><div class='del'>-		stop_ep_timer(ep);</div><div class='del'>-		if ((ep-&gt;com.cm_id) &amp;&amp; (ep-&gt;com.qp)) {</div><div class='del'>-			attrs.next_state = IWCH_QP_STATE_IDLE;</div><div class='del'>-			iwch_modify_qp(ep-&gt;com.qp-&gt;rhp,</div><div class='del'>-					     ep-&gt;com.qp,</div><div class='del'>-					     IWCH_QP_ATTR_NEXT_STATE,</div><div class='del'>-					     &amp;attrs, 1);</div><div class='del'>-		}</div><div class='del'>-		close_complete_upcall(ep);</div><div class='del'>-		__state_set(&amp;ep-&gt;com, DEAD);</div><div class='del'>-		release = 1;</div><div class='del'>-		break;</div><div class='del'>-	case ABORTING:</div><div class='del'>-	case DEAD:</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		BUG_ON(1);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;ep-&gt;com.lock, flags);</div><div class='del'>-	if (release)</div><div class='del'>-		release_ep_resources(ep);</div><div class='del'>-	return CPL_RET_BUF_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * T3A does 3 things when a TERM is received:</div><div class='del'>- * 1) send up a CPL_RDMA_TERMINATE message with the TERM packet</div><div class='del'>- * 2) generate an async event on the QP with the TERMINATE opcode</div><div class='del'>- * 3) post a TERMINATE opcode cqe into the associated CQ.</div><div class='del'>- *</div><div class='del'>- * For (1), we save the message in the qp for later consumer consumption.</div><div class='del'>- * For (2), we move the QP into TERMINATE, post a QP event and disconnect.</div><div class='del'>- * For (3), we toss the CQE in cxio_poll_cq().</div><div class='del'>- *</div><div class='del'>- * terminate() handles case (1)...</div><div class='del'>- */</div><div class='del'>-static int terminate(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)</div><div class='del'>-{</div><div class='del'>-	struct iwch_ep *ep = ctx;</div><div class='del'>-</div><div class='del'>-	if (state_read(&amp;ep-&gt;com) != FPDU_MODE)</div><div class='del'>-		return CPL_RET_BUF_DONE;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	skb_pull(skb, sizeof(struct cpl_rdma_terminate));</div><div class='del'>-	pr_debug("%s saving %d bytes of term msg\n", __func__, skb-&gt;len);</div><div class='del'>-	skb_copy_from_linear_data(skb, ep-&gt;com.qp-&gt;attr.terminate_buffer,</div><div class='del'>-				  skb-&gt;len);</div><div class='del'>-	ep-&gt;com.qp-&gt;attr.terminate_msg_len = skb-&gt;len;</div><div class='del'>-	ep-&gt;com.qp-&gt;attr.is_terminate_local = 0;</div><div class='del'>-	return CPL_RET_BUF_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int ec_status(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)</div><div class='del'>-{</div><div class='del'>-	struct cpl_rdma_ec_status *rep = cplhdr(skb);</div><div class='del'>-	struct iwch_ep *ep = ctx;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p tid %u status %d\n", __func__, ep, ep-&gt;hwtid,</div><div class='del'>-		 rep-&gt;status);</div><div class='del'>-	if (rep-&gt;status) {</div><div class='del'>-		struct iwch_qp_attributes attrs;</div><div class='del'>-</div><div class='del'>-		pr_err("%s BAD CLOSE - Aborting tid %u\n",</div><div class='del'>-		       __func__, ep-&gt;hwtid);</div><div class='del'>-		stop_ep_timer(ep);</div><div class='del'>-		attrs.next_state = IWCH_QP_STATE_ERROR;</div><div class='del'>-		iwch_modify_qp(ep-&gt;com.qp-&gt;rhp,</div><div class='del'>-			       ep-&gt;com.qp, IWCH_QP_ATTR_NEXT_STATE,</div><div class='del'>-			       &amp;attrs, 1);</div><div class='del'>-		abort_connection(ep, NULL, GFP_KERNEL);</div><div class='del'>-	}</div><div class='del'>-	return CPL_RET_BUF_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void ep_timeout(struct timer_list *t)</div><div class='del'>-{</div><div class='del'>-	struct iwch_ep *ep = from_timer(ep, t, timer);</div><div class='del'>-	struct iwch_qp_attributes attrs;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int abort = 1;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;ep-&gt;com.lock, flags);</div><div class='del'>-	pr_debug("%s ep %p tid %u state %d\n", __func__, ep, ep-&gt;hwtid,</div><div class='del'>-		 ep-&gt;com.state);</div><div class='del'>-	switch (ep-&gt;com.state) {</div><div class='del'>-	case MPA_REQ_SENT:</div><div class='del'>-		__state_set(&amp;ep-&gt;com, ABORTING);</div><div class='del'>-		connect_reply_upcall(ep, -ETIMEDOUT);</div><div class='del'>-		break;</div><div class='del'>-	case MPA_REQ_WAIT:</div><div class='del'>-		__state_set(&amp;ep-&gt;com, ABORTING);</div><div class='del'>-		break;</div><div class='del'>-	case CLOSING:</div><div class='del'>-	case MORIBUND:</div><div class='del'>-		if (ep-&gt;com.cm_id &amp;&amp; ep-&gt;com.qp) {</div><div class='del'>-			attrs.next_state = IWCH_QP_STATE_ERROR;</div><div class='del'>-			iwch_modify_qp(ep-&gt;com.qp-&gt;rhp,</div><div class='del'>-				     ep-&gt;com.qp, IWCH_QP_ATTR_NEXT_STATE,</div><div class='del'>-				     &amp;attrs, 1);</div><div class='del'>-		}</div><div class='del'>-		__state_set(&amp;ep-&gt;com, ABORTING);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		WARN(1, "%s unexpected state ep %p state %u\n",</div><div class='del'>-			__func__, ep, ep-&gt;com.state);</div><div class='del'>-		abort = 0;</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;ep-&gt;com.lock, flags);</div><div class='del'>-	if (abort)</div><div class='del'>-		abort_connection(ep, NULL, GFP_ATOMIC);</div><div class='del'>-	put_ep(&amp;ep-&gt;com);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)</div><div class='del'>-{</div><div class='del'>-	struct iwch_ep *ep = to_ep(cm_id);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p tid %u\n", __func__, ep, ep-&gt;hwtid);</div><div class='del'>-</div><div class='del'>-	if (state_read(&amp;ep-&gt;com) == DEAD) {</div><div class='del'>-		put_ep(&amp;ep-&gt;com);</div><div class='del'>-		return -ECONNRESET;</div><div class='del'>-	}</div><div class='del'>-	BUG_ON(state_read(&amp;ep-&gt;com) != MPA_REQ_RCVD);</div><div class='del'>-	if (mpa_rev == 0)</div><div class='del'>-		abort_connection(ep, NULL, GFP_KERNEL);</div><div class='del'>-	else {</div><div class='del'>-		send_mpa_reject(ep, pdata, pdata_len);</div><div class='del'>-		iwch_ep_disconnect(ep, 0, GFP_KERNEL);</div><div class='del'>-	}</div><div class='del'>-	put_ep(&amp;ep-&gt;com);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)</div><div class='del'>-{</div><div class='del'>-	int err;</div><div class='del'>-	struct iwch_qp_attributes attrs;</div><div class='del'>-	enum iwch_qp_attr_mask mask;</div><div class='del'>-	struct iwch_ep *ep = to_ep(cm_id);</div><div class='del'>-	struct iwch_dev *h = to_iwch_dev(cm_id-&gt;device);</div><div class='del'>-	struct iwch_qp *qp = get_qhp(h, conn_param-&gt;qpn);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p tid %u\n", __func__, ep, ep-&gt;hwtid);</div><div class='del'>-	if (state_read(&amp;ep-&gt;com) == DEAD) {</div><div class='del'>-		err = -ECONNRESET;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	BUG_ON(state_read(&amp;ep-&gt;com) != MPA_REQ_RCVD);</div><div class='del'>-	BUG_ON(!qp);</div><div class='del'>-</div><div class='del'>-	if ((conn_param-&gt;ord &gt; qp-&gt;rhp-&gt;attr.max_rdma_read_qp_depth) ||</div><div class='del'>-	    (conn_param-&gt;ird &gt; qp-&gt;rhp-&gt;attr.max_rdma_reads_per_qp)) {</div><div class='del'>-		abort_connection(ep, NULL, GFP_KERNEL);</div><div class='del'>-		err = -EINVAL;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cm_id-&gt;add_ref(cm_id);</div><div class='del'>-	ep-&gt;com.cm_id = cm_id;</div><div class='del'>-	ep-&gt;com.qp = qp;</div><div class='del'>-</div><div class='del'>-	ep-&gt;ird = conn_param-&gt;ird;</div><div class='del'>-	ep-&gt;ord = conn_param-&gt;ord;</div><div class='del'>-</div><div class='del'>-	if (peer2peer &amp;&amp; ep-&gt;ird == 0)</div><div class='del'>-		ep-&gt;ird = 1;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s %d ird %d ord %d\n", __func__, __LINE__, ep-&gt;ird, ep-&gt;ord);</div><div class='del'>-</div><div class='del'>-	/* bind QP to EP and move to RTS */</div><div class='del'>-	attrs.mpa_attr = ep-&gt;mpa_attr;</div><div class='del'>-	attrs.max_ird = ep-&gt;ird;</div><div class='del'>-	attrs.max_ord = ep-&gt;ord;</div><div class='del'>-	attrs.llp_stream_handle = ep;</div><div class='del'>-	attrs.next_state = IWCH_QP_STATE_RTS;</div><div class='del'>-</div><div class='del'>-	/* bind QP and TID with INIT_WR */</div><div class='del'>-	mask = IWCH_QP_ATTR_NEXT_STATE |</div><div class='del'>-			     IWCH_QP_ATTR_LLP_STREAM_HANDLE |</div><div class='del'>-			     IWCH_QP_ATTR_MPA_ATTR |</div><div class='del'>-			     IWCH_QP_ATTR_MAX_IRD |</div><div class='del'>-			     IWCH_QP_ATTR_MAX_ORD;</div><div class='del'>-</div><div class='del'>-	err = iwch_modify_qp(ep-&gt;com.qp-&gt;rhp,</div><div class='del'>-			     ep-&gt;com.qp, mask, &amp;attrs, 1);</div><div class='del'>-	if (err)</div><div class='del'>-		goto err1;</div><div class='del'>-</div><div class='del'>-	/* if needed, wait for wr_ack */</div><div class='del'>-	if (iwch_rqes_posted(qp)) {</div><div class='del'>-		wait_event(ep-&gt;com.waitq, ep-&gt;com.rpl_done);</div><div class='del'>-		err = ep-&gt;com.rpl_err;</div><div class='del'>-		if (err)</div><div class='del'>-			goto err1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	err = send_mpa_reply(ep, conn_param-&gt;private_data,</div><div class='del'>-			     conn_param-&gt;private_data_len);</div><div class='del'>-	if (err)</div><div class='del'>-		goto err1;</div><div class='del'>-</div><div class='del'>-</div><div class='del'>-	state_set(&amp;ep-&gt;com, FPDU_MODE);</div><div class='del'>-	established_upcall(ep);</div><div class='del'>-	put_ep(&amp;ep-&gt;com);</div><div class='del'>-	return 0;</div><div class='del'>-err1:</div><div class='del'>-	ep-&gt;com.cm_id = NULL;</div><div class='del'>-	ep-&gt;com.qp = NULL;</div><div class='del'>-	cm_id-&gt;rem_ref(cm_id);</div><div class='del'>-err:</div><div class='del'>-	put_ep(&amp;ep-&gt;com);</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int is_loopback_dst(struct iw_cm_id *cm_id)</div><div class='del'>-{</div><div class='del'>-	struct net_device *dev;</div><div class='del'>-	struct sockaddr_in *raddr = (struct sockaddr_in *)&amp;cm_id-&gt;m_remote_addr;</div><div class='del'>-</div><div class='del'>-	dev = ip_dev_find(&amp;init_net, raddr-&gt;sin_addr.s_addr);</div><div class='del'>-	if (!dev)</div><div class='del'>-		return 0;</div><div class='del'>-	dev_put(dev);</div><div class='del'>-	return 1;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *h = to_iwch_dev(cm_id-&gt;device);</div><div class='del'>-	struct iwch_ep *ep;</div><div class='del'>-	struct rtable *rt;</div><div class='del'>-	int err = 0;</div><div class='del'>-	struct sockaddr_in *laddr = (struct sockaddr_in *)&amp;cm_id-&gt;m_local_addr;</div><div class='del'>-	struct sockaddr_in *raddr = (struct sockaddr_in *)&amp;cm_id-&gt;m_remote_addr;</div><div class='del'>-</div><div class='del'>-	if (cm_id-&gt;m_remote_addr.ss_family != PF_INET) {</div><div class='del'>-		err = -ENOSYS;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (is_loopback_dst(cm_id)) {</div><div class='del'>-		err = -ENOSYS;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ep = alloc_ep(sizeof(*ep), GFP_KERNEL);</div><div class='del'>-	if (!ep) {</div><div class='del'>-		pr_err("%s - cannot alloc ep\n", __func__);</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-	timer_setup(&amp;ep-&gt;timer, ep_timeout, 0);</div><div class='del'>-	ep-&gt;plen = conn_param-&gt;private_data_len;</div><div class='del'>-	if (ep-&gt;plen)</div><div class='del'>-		memcpy(ep-&gt;mpa_pkt + sizeof(struct mpa_message),</div><div class='del'>-		       conn_param-&gt;private_data, ep-&gt;plen);</div><div class='del'>-	ep-&gt;ird = conn_param-&gt;ird;</div><div class='del'>-	ep-&gt;ord = conn_param-&gt;ord;</div><div class='del'>-</div><div class='del'>-	if (peer2peer &amp;&amp; ep-&gt;ord == 0)</div><div class='del'>-		ep-&gt;ord = 1;</div><div class='del'>-</div><div class='del'>-	ep-&gt;com.tdev = h-&gt;rdev.t3cdev_p;</div><div class='del'>-</div><div class='del'>-	cm_id-&gt;add_ref(cm_id);</div><div class='del'>-	ep-&gt;com.cm_id = cm_id;</div><div class='del'>-	ep-&gt;com.qp = get_qhp(h, conn_param-&gt;qpn);</div><div class='del'>-	BUG_ON(!ep-&gt;com.qp);</div><div class='del'>-	pr_debug("%s qpn 0x%x qp %p cm_id %p\n", __func__, conn_param-&gt;qpn,</div><div class='del'>-		 ep-&gt;com.qp, cm_id);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Allocate an active TID to initiate a TCP connection.</div><div class='del'>-	 */</div><div class='del'>-	ep-&gt;atid = cxgb3_alloc_atid(h-&gt;rdev.t3cdev_p, &amp;t3c_client, ep);</div><div class='del'>-	if (ep-&gt;atid == -1) {</div><div class='del'>-		pr_err("%s - cannot alloc atid\n", __func__);</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto fail2;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* find a route */</div><div class='del'>-	rt = find_route(h-&gt;rdev.t3cdev_p, laddr-&gt;sin_addr.s_addr,</div><div class='del'>-			raddr-&gt;sin_addr.s_addr, laddr-&gt;sin_port,</div><div class='del'>-			raddr-&gt;sin_port, IPTOS_LOWDELAY);</div><div class='del'>-	if (!rt) {</div><div class='del'>-		pr_err("%s - cannot find route\n", __func__);</div><div class='del'>-		err = -EHOSTUNREACH;</div><div class='del'>-		goto fail3;</div><div class='del'>-	}</div><div class='del'>-	ep-&gt;dst = &amp;rt-&gt;dst;</div><div class='del'>-	ep-&gt;l2t = t3_l2t_get(ep-&gt;com.tdev, ep-&gt;dst, NULL,</div><div class='del'>-			     &amp;raddr-&gt;sin_addr.s_addr);</div><div class='del'>-	if (!ep-&gt;l2t) {</div><div class='del'>-		pr_err("%s - cannot alloc l2e\n", __func__);</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto fail4;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	state_set(&amp;ep-&gt;com, CONNECTING);</div><div class='del'>-	ep-&gt;tos = IPTOS_LOWDELAY;</div><div class='del'>-	memcpy(&amp;ep-&gt;com.local_addr, &amp;cm_id-&gt;m_local_addr,</div><div class='del'>-	       sizeof(ep-&gt;com.local_addr));</div><div class='del'>-	memcpy(&amp;ep-&gt;com.remote_addr, &amp;cm_id-&gt;m_remote_addr,</div><div class='del'>-	       sizeof(ep-&gt;com.remote_addr));</div><div class='del'>-</div><div class='del'>-	/* send connect request to rnic */</div><div class='del'>-	err = send_connect(ep);</div><div class='del'>-	if (!err)</div><div class='del'>-		goto out;</div><div class='del'>-</div><div class='del'>-	l2t_release(h-&gt;rdev.t3cdev_p, ep-&gt;l2t);</div><div class='del'>-fail4:</div><div class='del'>-	dst_release(ep-&gt;dst);</div><div class='del'>-fail3:</div><div class='del'>-	cxgb3_free_atid(ep-&gt;com.tdev, ep-&gt;atid);</div><div class='del'>-fail2:</div><div class='del'>-	cm_id-&gt;rem_ref(cm_id);</div><div class='del'>-	put_ep(&amp;ep-&gt;com);</div><div class='del'>-out:</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_create_listen(struct iw_cm_id *cm_id, int backlog)</div><div class='del'>-{</div><div class='del'>-	int err = 0;</div><div class='del'>-	struct iwch_dev *h = to_iwch_dev(cm_id-&gt;device);</div><div class='del'>-	struct iwch_listen_ep *ep;</div><div class='del'>-</div><div class='del'>-</div><div class='del'>-	might_sleep();</div><div class='del'>-</div><div class='del'>-	if (cm_id-&gt;m_local_addr.ss_family != PF_INET) {</div><div class='del'>-		err = -ENOSYS;</div><div class='del'>-		goto fail1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ep = alloc_ep(sizeof(*ep), GFP_KERNEL);</div><div class='del'>-	if (!ep) {</div><div class='del'>-		pr_err("%s - cannot alloc ep\n", __func__);</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto fail1;</div><div class='del'>-	}</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-	ep-&gt;com.tdev = h-&gt;rdev.t3cdev_p;</div><div class='del'>-	cm_id-&gt;add_ref(cm_id);</div><div class='del'>-	ep-&gt;com.cm_id = cm_id;</div><div class='del'>-	ep-&gt;backlog = backlog;</div><div class='del'>-	memcpy(&amp;ep-&gt;com.local_addr, &amp;cm_id-&gt;m_local_addr,</div><div class='del'>-	       sizeof(ep-&gt;com.local_addr));</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Allocate a server TID.</div><div class='del'>-	 */</div><div class='del'>-	ep-&gt;stid = cxgb3_alloc_stid(h-&gt;rdev.t3cdev_p, &amp;t3c_client, ep);</div><div class='del'>-	if (ep-&gt;stid == -1) {</div><div class='del'>-		pr_err("%s - cannot alloc atid\n", __func__);</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto fail2;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	state_set(&amp;ep-&gt;com, LISTEN);</div><div class='del'>-	err = listen_start(ep);</div><div class='del'>-	if (err)</div><div class='del'>-		goto fail3;</div><div class='del'>-</div><div class='del'>-	/* wait for pass_open_rpl */</div><div class='del'>-	wait_event(ep-&gt;com.waitq, ep-&gt;com.rpl_done);</div><div class='del'>-	err = ep-&gt;com.rpl_err;</div><div class='del'>-	if (!err) {</div><div class='del'>-		cm_id-&gt;provider_data = ep;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-fail3:</div><div class='del'>-	cxgb3_free_stid(ep-&gt;com.tdev, ep-&gt;stid);</div><div class='del'>-fail2:</div><div class='del'>-	cm_id-&gt;rem_ref(cm_id);</div><div class='del'>-	put_ep(&amp;ep-&gt;com);</div><div class='del'>-fail1:</div><div class='del'>-out:</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_destroy_listen(struct iw_cm_id *cm_id)</div><div class='del'>-{</div><div class='del'>-	int err;</div><div class='del'>-	struct iwch_listen_ep *ep = to_listen_ep(cm_id);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p\n", __func__, ep);</div><div class='del'>-</div><div class='del'>-	might_sleep();</div><div class='del'>-	state_set(&amp;ep-&gt;com, DEAD);</div><div class='del'>-	ep-&gt;com.rpl_done = 0;</div><div class='del'>-	ep-&gt;com.rpl_err = 0;</div><div class='del'>-	err = listen_stop(ep);</div><div class='del'>-	if (err)</div><div class='del'>-		goto done;</div><div class='del'>-	wait_event(ep-&gt;com.waitq, ep-&gt;com.rpl_done);</div><div class='del'>-	cxgb3_free_stid(ep-&gt;com.tdev, ep-&gt;stid);</div><div class='del'>-done:</div><div class='del'>-	err = ep-&gt;com.rpl_err;</div><div class='del'>-	cm_id-&gt;rem_ref(cm_id);</div><div class='del'>-	put_ep(&amp;ep-&gt;com);</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_ep_disconnect(struct iwch_ep *ep, int abrupt, gfp_t gfp)</div><div class='del'>-{</div><div class='del'>-	int ret=0;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int close = 0;</div><div class='del'>-	int fatal = 0;</div><div class='del'>-	struct t3cdev *tdev;</div><div class='del'>-	struct cxio_rdev *rdev;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;ep-&gt;com.lock, flags);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p state %s, abrupt %d\n", __func__, ep,</div><div class='del'>-		 states[ep-&gt;com.state], abrupt);</div><div class='del'>-</div><div class='del'>-	tdev = (struct t3cdev *)ep-&gt;com.tdev;</div><div class='del'>-	rdev = (struct cxio_rdev *)tdev-&gt;ulp;</div><div class='del'>-	if (cxio_fatal_error(rdev)) {</div><div class='del'>-		fatal = 1;</div><div class='del'>-		close_complete_upcall(ep);</div><div class='del'>-		ep-&gt;com.state = DEAD;</div><div class='del'>-	}</div><div class='del'>-	switch (ep-&gt;com.state) {</div><div class='del'>-	case MPA_REQ_WAIT:</div><div class='del'>-	case MPA_REQ_SENT:</div><div class='del'>-	case MPA_REQ_RCVD:</div><div class='del'>-	case MPA_REP_SENT:</div><div class='del'>-	case FPDU_MODE:</div><div class='del'>-		close = 1;</div><div class='del'>-		if (abrupt)</div><div class='del'>-			ep-&gt;com.state = ABORTING;</div><div class='del'>-		else {</div><div class='del'>-			ep-&gt;com.state = CLOSING;</div><div class='del'>-			start_ep_timer(ep);</div><div class='del'>-		}</div><div class='del'>-		set_bit(CLOSE_SENT, &amp;ep-&gt;com.flags);</div><div class='del'>-		break;</div><div class='del'>-	case CLOSING:</div><div class='del'>-		if (!test_and_set_bit(CLOSE_SENT, &amp;ep-&gt;com.flags)) {</div><div class='del'>-			close = 1;</div><div class='del'>-			if (abrupt) {</div><div class='del'>-				stop_ep_timer(ep);</div><div class='del'>-				ep-&gt;com.state = ABORTING;</div><div class='del'>-			} else</div><div class='del'>-				ep-&gt;com.state = MORIBUND;</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case MORIBUND:</div><div class='del'>-	case ABORTING:</div><div class='del'>-	case DEAD:</div><div class='del'>-		pr_debug("%s ignoring disconnect ep %p state %u\n",</div><div class='del'>-			 __func__, ep, ep-&gt;com.state);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		BUG();</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	spin_unlock_irqrestore(&amp;ep-&gt;com.lock, flags);</div><div class='del'>-	if (close) {</div><div class='del'>-		if (abrupt)</div><div class='del'>-			ret = send_abort(ep, NULL, gfp);</div><div class='del'>-		else</div><div class='del'>-			ret = send_halfclose(ep, gfp);</div><div class='del'>-		if (ret)</div><div class='del'>-			fatal = 1;</div><div class='del'>-	}</div><div class='del'>-	if (fatal)</div><div class='del'>-		release_ep_resources(ep);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new,</div><div class='del'>-		     struct l2t_entry *l2t)</div><div class='del'>-{</div><div class='del'>-	struct iwch_ep *ep = ctx;</div><div class='del'>-</div><div class='del'>-	if (ep-&gt;dst != old)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ep %p redirect to dst %p l2t %p\n", __func__, ep, new,</div><div class='del'>-		 l2t);</div><div class='del'>-	dst_hold(new);</div><div class='del'>-	l2t_release(ep-&gt;com.tdev, ep-&gt;l2t);</div><div class='del'>-	ep-&gt;l2t = l2t;</div><div class='del'>-	dst_release(old);</div><div class='del'>-	ep-&gt;dst = new;</div><div class='del'>-	return 1;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * All the CM events are handled on a work queue to have a safe context.</div><div class='del'>- * These are the real handlers that are called from the work queue.</div><div class='del'>- */</div><div class='del'>-static const cxgb3_cpl_handler_func work_handlers[NUM_CPL_CMDS] = {</div><div class='del'>-	[CPL_ACT_ESTABLISH]	= act_establish,</div><div class='del'>-	[CPL_ACT_OPEN_RPL]	= act_open_rpl,</div><div class='del'>-	[CPL_RX_DATA]		= rx_data,</div><div class='del'>-	[CPL_TX_DMA_ACK]	= tx_ack,</div><div class='del'>-	[CPL_ABORT_RPL_RSS]	= abort_rpl,</div><div class='del'>-	[CPL_ABORT_RPL]		= abort_rpl,</div><div class='del'>-	[CPL_PASS_OPEN_RPL]	= pass_open_rpl,</div><div class='del'>-	[CPL_CLOSE_LISTSRV_RPL]	= close_listsrv_rpl,</div><div class='del'>-	[CPL_PASS_ACCEPT_REQ]	= pass_accept_req,</div><div class='del'>-	[CPL_PASS_ESTABLISH]	= pass_establish,</div><div class='del'>-	[CPL_PEER_CLOSE]	= peer_close,</div><div class='del'>-	[CPL_ABORT_REQ_RSS]	= peer_abort,</div><div class='del'>-	[CPL_CLOSE_CON_RPL]	= close_con_rpl,</div><div class='del'>-	[CPL_RDMA_TERMINATE]	= terminate,</div><div class='del'>-	[CPL_RDMA_EC_STATUS]	= ec_status,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static void process_work(struct work_struct *work)</div><div class='del'>-{</div><div class='del'>-	struct sk_buff *skb = NULL;</div><div class='del'>-	void *ep;</div><div class='del'>-	struct t3cdev *tdev;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	while ((skb = skb_dequeue(&amp;rxq))) {</div><div class='del'>-		ep = *((void **) (skb-&gt;cb));</div><div class='del'>-		tdev = *((struct t3cdev **) (skb-&gt;cb + sizeof(void *)));</div><div class='del'>-		ret = work_handlers[G_OPCODE(ntohl((__force __be32)skb-&gt;csum))](tdev, skb, ep);</div><div class='del'>-		if (ret &amp; CPL_RET_BUF_DONE)</div><div class='del'>-			kfree_skb(skb);</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * ep was referenced in sched(), and is freed here.</div><div class='del'>-		 */</div><div class='del'>-		put_ep((struct iwch_ep_common *)ep);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static DECLARE_WORK(skb_work, process_work);</div><div class='del'>-</div><div class='del'>-static int sched(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)</div><div class='del'>-{</div><div class='del'>-	struct iwch_ep_common *epc = ctx;</div><div class='del'>-</div><div class='del'>-	get_ep(epc);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Save ctx and tdev in the skb-&gt;cb area.</div><div class='del'>-	 */</div><div class='del'>-	*((void **) skb-&gt;cb) = ctx;</div><div class='del'>-	*((struct t3cdev **) (skb-&gt;cb + sizeof(void *))) = tdev;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Queue the skb and schedule the worker thread.</div><div class='del'>-	 */</div><div class='del'>-	skb_queue_tail(&amp;rxq, skb);</div><div class='del'>-	queue_work(workq, &amp;skb_work);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int set_tcb_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)</div><div class='del'>-{</div><div class='del'>-	struct cpl_set_tcb_rpl *rpl = cplhdr(skb);</div><div class='del'>-</div><div class='del'>-	if (rpl-&gt;status != CPL_ERR_NONE) {</div><div class='del'>-		pr_err("Unexpected SET_TCB_RPL status %u for tid %u\n",</div><div class='del'>-		       rpl-&gt;status, GET_TID(rpl));</div><div class='del'>-	}</div><div class='del'>-	return CPL_RET_BUF_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * All upcalls from the T3 Core go to sched() to schedule the</div><div class='del'>- * processing on a work queue.</div><div class='del'>- */</div><div class='del'>-cxgb3_cpl_handler_func t3c_handlers[NUM_CPL_CMDS] = {</div><div class='del'>-	[CPL_ACT_ESTABLISH]	= sched,</div><div class='del'>-	[CPL_ACT_OPEN_RPL]	= sched,</div><div class='del'>-	[CPL_RX_DATA]		= sched,</div><div class='del'>-	[CPL_TX_DMA_ACK]	= sched,</div><div class='del'>-	[CPL_ABORT_RPL_RSS]	= sched,</div><div class='del'>-	[CPL_ABORT_RPL]		= sched,</div><div class='del'>-	[CPL_PASS_OPEN_RPL]	= sched,</div><div class='del'>-	[CPL_CLOSE_LISTSRV_RPL]	= sched,</div><div class='del'>-	[CPL_PASS_ACCEPT_REQ]	= sched,</div><div class='del'>-	[CPL_PASS_ESTABLISH]	= sched,</div><div class='del'>-	[CPL_PEER_CLOSE]	= sched,</div><div class='del'>-	[CPL_CLOSE_CON_RPL]	= sched,</div><div class='del'>-	[CPL_ABORT_REQ_RSS]	= sched,</div><div class='del'>-	[CPL_RDMA_TERMINATE]	= sched,</div><div class='del'>-	[CPL_RDMA_EC_STATUS]	= sched,</div><div class='del'>-	[CPL_SET_TCB_RPL]	= set_tcb_rpl,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-int __init iwch_cm_init(void)</div><div class='del'>-{</div><div class='del'>-	skb_queue_head_init(&amp;rxq);</div><div class='del'>-</div><div class='del'>-	workq = alloc_ordered_workqueue("iw_cxgb3", WQ_MEM_RECLAIM);</div><div class='del'>-	if (!workq)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void __exit iwch_cm_term(void)</div><div class='del'>-{</div><div class='del'>-	flush_workqueue(workq);</div><div class='del'>-	destroy_workqueue(workq);</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/iwch_cm.h b/drivers/infiniband/hw/cxgb3/iwch_cm.h<br/>deleted file mode 100644<br/>index cc7fe644d260..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/iwch_cm.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch_cm.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,233 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-#ifndef _IWCH_CM_H_</div><div class='del'>-#define _IWCH_CM_H_</div><div class='del'>-</div><div class='del'>-#include &lt;linux/inet.h&gt;</div><div class='del'>-#include &lt;linux/wait.h&gt;</div><div class='del'>-#include &lt;linux/spinlock.h&gt;</div><div class='del'>-#include &lt;linux/kref.h&gt;</div><div class='del'>-</div><div class='del'>-#include &lt;rdma/ib_verbs.h&gt;</div><div class='del'>-#include &lt;rdma/iw_cm.h&gt;</div><div class='del'>-</div><div class='del'>-#include "cxgb3_offload.h"</div><div class='del'>-#include "iwch_provider.h"</div><div class='del'>-</div><div class='del'>-#define MPA_KEY_REQ "MPA ID Req Frame"</div><div class='del'>-#define MPA_KEY_REP "MPA ID Rep Frame"</div><div class='del'>-</div><div class='del'>-#define MPA_MAX_PRIVATE_DATA	256</div><div class='del'>-#define MPA_REV		0	/* XXX - amso1100 uses rev 0 ! */</div><div class='del'>-#define MPA_REJECT		0x20</div><div class='del'>-#define MPA_CRC			0x40</div><div class='del'>-#define MPA_MARKERS		0x80</div><div class='del'>-#define MPA_FLAGS_MASK		0xE0</div><div class='del'>-</div><div class='del'>-#define put_ep(ep) {							\</div><div class='del'>-	pr_debug("put_ep (via %s:%u) ep %p refcnt %d\n",		\</div><div class='del'>-		 __func__, __LINE__, ep, kref_read(&amp;((ep)-&gt;kref)));	\</div><div class='del'>-	WARN_ON(kref_read(&amp;((ep)-&gt;kref)) &lt; 1);				\</div><div class='del'>-	kref_put(&amp;((ep)-&gt;kref), __free_ep);				\</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-#define get_ep(ep) {							\</div><div class='del'>-	pr_debug("get_ep (via %s:%u) ep %p, refcnt %d\n",		\</div><div class='del'>-		 __func__, __LINE__, ep, kref_read(&amp;((ep)-&gt;kref)));	\</div><div class='del'>-	kref_get(&amp;((ep)-&gt;kref));					\</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct mpa_message {</div><div class='del'>-	u8 key[16];</div><div class='del'>-	u8 flags;</div><div class='del'>-	u8 revision;</div><div class='del'>-	__be16 private_data_size;</div><div class='del'>-	u8 private_data[0];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct terminate_message {</div><div class='del'>-	u8 layer_etype;</div><div class='del'>-	u8 ecode;</div><div class='del'>-	__be16 hdrct_rsvd;</div><div class='del'>-	u8 len_hdrs[0];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define TERM_MAX_LENGTH (sizeof(struct terminate_message) + 2 + 18 + 28)</div><div class='del'>-</div><div class='del'>-enum iwch_layers_types {</div><div class='del'>-	LAYER_RDMAP		= 0x00,</div><div class='del'>-	LAYER_DDP		= 0x10,</div><div class='del'>-	LAYER_MPA		= 0x20,</div><div class='del'>-	RDMAP_LOCAL_CATA	= 0x00,</div><div class='del'>-	RDMAP_REMOTE_PROT	= 0x01,</div><div class='del'>-	RDMAP_REMOTE_OP		= 0x02,</div><div class='del'>-	DDP_LOCAL_CATA		= 0x00,</div><div class='del'>-	DDP_TAGGED_ERR		= 0x01,</div><div class='del'>-	DDP_UNTAGGED_ERR	= 0x02,</div><div class='del'>-	DDP_LLP			= 0x03</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum iwch_rdma_ecodes {</div><div class='del'>-	RDMAP_INV_STAG		= 0x00,</div><div class='del'>-	RDMAP_BASE_BOUNDS	= 0x01,</div><div class='del'>-	RDMAP_ACC_VIOL		= 0x02,</div><div class='del'>-	RDMAP_STAG_NOT_ASSOC	= 0x03,</div><div class='del'>-	RDMAP_TO_WRAP		= 0x04,</div><div class='del'>-	RDMAP_INV_VERS		= 0x05,</div><div class='del'>-	RDMAP_INV_OPCODE	= 0x06,</div><div class='del'>-	RDMAP_STREAM_CATA	= 0x07,</div><div class='del'>-	RDMAP_GLOBAL_CATA	= 0x08,</div><div class='del'>-	RDMAP_CANT_INV_STAG	= 0x09,</div><div class='del'>-	RDMAP_UNSPECIFIED	= 0xff</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum iwch_ddp_ecodes {</div><div class='del'>-	DDPT_INV_STAG		= 0x00,</div><div class='del'>-	DDPT_BASE_BOUNDS	= 0x01,</div><div class='del'>-	DDPT_STAG_NOT_ASSOC	= 0x02,</div><div class='del'>-	DDPT_TO_WRAP		= 0x03,</div><div class='del'>-	DDPT_INV_VERS		= 0x04,</div><div class='del'>-	DDPU_INV_QN		= 0x01,</div><div class='del'>-	DDPU_INV_MSN_NOBUF	= 0x02,</div><div class='del'>-	DDPU_INV_MSN_RANGE	= 0x03,</div><div class='del'>-	DDPU_INV_MO		= 0x04,</div><div class='del'>-	DDPU_MSG_TOOBIG		= 0x05,</div><div class='del'>-	DDPU_INV_VERS		= 0x06</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum iwch_mpa_ecodes {</div><div class='del'>-	MPA_CRC_ERR		= 0x02,</div><div class='del'>-	MPA_MARKER_ERR		= 0x03</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum iwch_ep_state {</div><div class='del'>-	IDLE = 0,</div><div class='del'>-	LISTEN,</div><div class='del'>-	CONNECTING,</div><div class='del'>-	MPA_REQ_WAIT,</div><div class='del'>-	MPA_REQ_SENT,</div><div class='del'>-	MPA_REQ_RCVD,</div><div class='del'>-	MPA_REP_SENT,</div><div class='del'>-	FPDU_MODE,</div><div class='del'>-	ABORTING,</div><div class='del'>-	CLOSING,</div><div class='del'>-	MORIBUND,</div><div class='del'>-	DEAD,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum iwch_ep_flags {</div><div class='del'>-	PEER_ABORT_IN_PROGRESS	= 0,</div><div class='del'>-	ABORT_REQ_IN_PROGRESS	= 1,</div><div class='del'>-	RELEASE_RESOURCES	= 2,</div><div class='del'>-	CLOSE_SENT		= 3,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct iwch_ep_common {</div><div class='del'>-	struct iw_cm_id *cm_id;</div><div class='del'>-	struct iwch_qp *qp;</div><div class='del'>-	struct t3cdev *tdev;</div><div class='del'>-	enum iwch_ep_state state;</div><div class='del'>-	struct kref kref;</div><div class='del'>-	spinlock_t lock;</div><div class='del'>-	struct sockaddr_in local_addr;</div><div class='del'>-	struct sockaddr_in remote_addr;</div><div class='del'>-	wait_queue_head_t waitq;</div><div class='del'>-	int rpl_done;</div><div class='del'>-	int rpl_err;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct iwch_listen_ep {</div><div class='del'>-	struct iwch_ep_common com;</div><div class='del'>-	unsigned int stid;</div><div class='del'>-	int backlog;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct iwch_ep {</div><div class='del'>-	struct iwch_ep_common com;</div><div class='del'>-	struct iwch_ep *parent_ep;</div><div class='del'>-	struct timer_list timer;</div><div class='del'>-	unsigned int atid;</div><div class='del'>-	u32 hwtid;</div><div class='del'>-	u32 snd_seq;</div><div class='del'>-	u32 rcv_seq;</div><div class='del'>-	struct l2t_entry *l2t;</div><div class='del'>-	struct dst_entry *dst;</div><div class='del'>-	struct sk_buff *mpa_skb;</div><div class='del'>-	struct iwch_mpa_attributes mpa_attr;</div><div class='del'>-	unsigned int mpa_pkt_len;</div><div class='del'>-	u8 mpa_pkt[sizeof(struct mpa_message) + MPA_MAX_PRIVATE_DATA];</div><div class='del'>-	u8 tos;</div><div class='del'>-	u16 emss;</div><div class='del'>-	u16 plen;</div><div class='del'>-	u32 ird;</div><div class='del'>-	u32 ord;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static inline struct iwch_ep *to_ep(struct iw_cm_id *cm_id)</div><div class='del'>-{</div><div class='del'>-	return cm_id-&gt;provider_data;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline struct iwch_listen_ep *to_listen_ep(struct iw_cm_id *cm_id)</div><div class='del'>-{</div><div class='del'>-	return cm_id-&gt;provider_data;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline int compute_wscale(int win)</div><div class='del'>-{</div><div class='del'>-	int wscale = 0;</div><div class='del'>-</div><div class='del'>-	while (wscale &lt; 14 &amp;&amp; (65535&lt;&lt;wscale) &lt; win)</div><div class='del'>-		wscale++;</div><div class='del'>-	return wscale;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* CM prototypes */</div><div class='del'>-</div><div class='del'>-int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);</div><div class='del'>-int iwch_create_listen(struct iw_cm_id *cm_id, int backlog);</div><div class='del'>-int iwch_destroy_listen(struct iw_cm_id *cm_id);</div><div class='del'>-int iwch_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);</div><div class='del'>-int iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);</div><div class='del'>-int iwch_ep_disconnect(struct iwch_ep *ep, int abrupt, gfp_t gfp);</div><div class='del'>-int iwch_quiesce_tid(struct iwch_ep *ep);</div><div class='del'>-int iwch_resume_tid(struct iwch_ep *ep);</div><div class='del'>-void __free_ep(struct kref *kref);</div><div class='del'>-void iwch_rearp(struct iwch_ep *ep);</div><div class='del'>-int iwch_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new, struct l2t_entry *l2t);</div><div class='del'>-</div><div class='del'>-int __init iwch_cm_init(void);</div><div class='del'>-void __exit iwch_cm_term(void);</div><div class='del'>-extern int peer2peer;</div><div class='del'>-</div><div class='del'>-#endif				/* _IWCH_CM_H_ */</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/iwch_cq.c b/drivers/infiniband/hw/cxgb3/iwch_cq.c<br/>deleted file mode 100644<br/>index a098c0140580..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/iwch_cq.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch_cq.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,230 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-#include "iwch_provider.h"</div><div class='del'>-#include "iwch.h"</div><div class='del'>-</div><div class='del'>-static int __iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp,</div><div class='del'>-			      struct iwch_qp *qhp, struct ib_wc *wc)</div><div class='del'>-{</div><div class='del'>-	struct t3_wq *wq = qhp ? &amp;qhp-&gt;wq : NULL;</div><div class='del'>-	struct t3_cqe cqe;</div><div class='del'>-	u32 credit = 0;</div><div class='del'>-	u8 cqe_flushed;</div><div class='del'>-	u64 cookie;</div><div class='del'>-	int ret = 1;</div><div class='del'>-</div><div class='del'>-	ret = cxio_poll_cq(wq, &amp;(chp-&gt;cq), &amp;cqe, &amp;cqe_flushed, &amp;cookie,</div><div class='del'>-				   &amp;credit);</div><div class='del'>-	if (t3a_device(chp-&gt;rhp) &amp;&amp; credit) {</div><div class='del'>-		pr_debug("%s updating %d cq credits on id %d\n", __func__,</div><div class='del'>-			 credit, chp-&gt;cq.cqid);</div><div class='del'>-		cxio_hal_cq_op(&amp;rhp-&gt;rdev, &amp;chp-&gt;cq, CQ_CREDIT_UPDATE, credit);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (ret) {</div><div class='del'>-		ret = -EAGAIN;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-	ret = 1;</div><div class='del'>-</div><div class='del'>-	wc-&gt;wr_id = cookie;</div><div class='del'>-	wc-&gt;qp = qhp ? &amp;qhp-&gt;ibqp : NULL;</div><div class='del'>-	wc-&gt;vendor_err = CQE_STATUS(cqe);</div><div class='del'>-	wc-&gt;wc_flags = 0;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s qpid 0x%x type %d opcode %d status 0x%x wrid hi 0x%x lo 0x%x cookie 0x%llx\n",</div><div class='del'>-		 __func__,</div><div class='del'>-		 CQE_QPID(cqe), CQE_TYPE(cqe),</div><div class='del'>-		 CQE_OPCODE(cqe), CQE_STATUS(cqe), CQE_WRID_HI(cqe),</div><div class='del'>-		 CQE_WRID_LOW(cqe), (unsigned long long)cookie);</div><div class='del'>-</div><div class='del'>-	if (CQE_TYPE(cqe) == 0) {</div><div class='del'>-		if (!CQE_STATUS(cqe))</div><div class='del'>-			wc-&gt;byte_len = CQE_LEN(cqe);</div><div class='del'>-		else</div><div class='del'>-			wc-&gt;byte_len = 0;</div><div class='del'>-		wc-&gt;opcode = IB_WC_RECV;</div><div class='del'>-		if (CQE_OPCODE(cqe) == T3_SEND_WITH_INV ||</div><div class='del'>-		    CQE_OPCODE(cqe) == T3_SEND_WITH_SE_INV) {</div><div class='del'>-			wc-&gt;ex.invalidate_rkey = CQE_WRID_STAG(cqe);</div><div class='del'>-			wc-&gt;wc_flags |= IB_WC_WITH_INVALIDATE;</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		switch (CQE_OPCODE(cqe)) {</div><div class='del'>-		case T3_RDMA_WRITE:</div><div class='del'>-			wc-&gt;opcode = IB_WC_RDMA_WRITE;</div><div class='del'>-			break;</div><div class='del'>-		case T3_READ_REQ:</div><div class='del'>-			wc-&gt;opcode = IB_WC_RDMA_READ;</div><div class='del'>-			wc-&gt;byte_len = CQE_LEN(cqe);</div><div class='del'>-			break;</div><div class='del'>-		case T3_SEND:</div><div class='del'>-		case T3_SEND_WITH_SE:</div><div class='del'>-		case T3_SEND_WITH_INV:</div><div class='del'>-		case T3_SEND_WITH_SE_INV:</div><div class='del'>-			wc-&gt;opcode = IB_WC_SEND;</div><div class='del'>-			break;</div><div class='del'>-		case T3_LOCAL_INV:</div><div class='del'>-			wc-&gt;opcode = IB_WC_LOCAL_INV;</div><div class='del'>-			break;</div><div class='del'>-		case T3_FAST_REGISTER:</div><div class='del'>-			wc-&gt;opcode = IB_WC_REG_MR;</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			pr_err("Unexpected opcode %d in the CQE received for QPID=0x%0x\n",</div><div class='del'>-			       CQE_OPCODE(cqe), CQE_QPID(cqe));</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (cqe_flushed)</div><div class='del'>-		wc-&gt;status = IB_WC_WR_FLUSH_ERR;</div><div class='del'>-	else {</div><div class='del'>-</div><div class='del'>-		switch (CQE_STATUS(cqe)) {</div><div class='del'>-		case TPT_ERR_SUCCESS:</div><div class='del'>-			wc-&gt;status = IB_WC_SUCCESS;</div><div class='del'>-			break;</div><div class='del'>-		case TPT_ERR_STAG:</div><div class='del'>-			wc-&gt;status = IB_WC_LOC_ACCESS_ERR;</div><div class='del'>-			break;</div><div class='del'>-		case TPT_ERR_PDID:</div><div class='del'>-			wc-&gt;status = IB_WC_LOC_PROT_ERR;</div><div class='del'>-			break;</div><div class='del'>-		case TPT_ERR_QPID:</div><div class='del'>-		case TPT_ERR_ACCESS:</div><div class='del'>-			wc-&gt;status = IB_WC_LOC_ACCESS_ERR;</div><div class='del'>-			break;</div><div class='del'>-		case TPT_ERR_WRAP:</div><div class='del'>-			wc-&gt;status = IB_WC_GENERAL_ERR;</div><div class='del'>-			break;</div><div class='del'>-		case TPT_ERR_BOUND:</div><div class='del'>-			wc-&gt;status = IB_WC_LOC_LEN_ERR;</div><div class='del'>-			break;</div><div class='del'>-		case TPT_ERR_INVALIDATE_SHARED_MR:</div><div class='del'>-		case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND:</div><div class='del'>-			wc-&gt;status = IB_WC_MW_BIND_ERR;</div><div class='del'>-			break;</div><div class='del'>-		case TPT_ERR_CRC:</div><div class='del'>-		case TPT_ERR_MARKER:</div><div class='del'>-		case TPT_ERR_PDU_LEN_ERR:</div><div class='del'>-		case TPT_ERR_OUT_OF_RQE:</div><div class='del'>-		case TPT_ERR_DDP_VERSION:</div><div class='del'>-		case TPT_ERR_RDMA_VERSION:</div><div class='del'>-		case TPT_ERR_DDP_QUEUE_NUM:</div><div class='del'>-		case TPT_ERR_MSN:</div><div class='del'>-		case TPT_ERR_TBIT:</div><div class='del'>-		case TPT_ERR_MO:</div><div class='del'>-		case TPT_ERR_MSN_RANGE:</div><div class='del'>-		case TPT_ERR_IRD_OVERFLOW:</div><div class='del'>-		case TPT_ERR_OPCODE:</div><div class='del'>-			wc-&gt;status = IB_WC_FATAL_ERR;</div><div class='del'>-			break;</div><div class='del'>-		case TPT_ERR_SWFLUSH:</div><div class='del'>-			wc-&gt;status = IB_WC_WR_FLUSH_ERR;</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			pr_err("Unexpected cqe_status 0x%x for QPID=0x%0x\n",</div><div class='del'>-			       CQE_STATUS(cqe), CQE_QPID(cqe));</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-out:</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Get one cq entry from cxio and map it to openib.</div><div class='del'>- *</div><div class='del'>- * Returns:</div><div class='del'>- *	0			EMPTY;</div><div class='del'>- *	1			cqe returned</div><div class='del'>- *	-EAGAIN		caller must try again</div><div class='del'>- *	any other -errno	fatal error</div><div class='del'>- */</div><div class='del'>-static int iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp,</div><div class='del'>-			    struct ib_wc *wc)</div><div class='del'>-{</div><div class='del'>-	struct iwch_qp *qhp;</div><div class='del'>-	struct t3_cqe *rd_cqe;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	rd_cqe = cxio_next_cqe(&amp;chp-&gt;cq);</div><div class='del'>-</div><div class='del'>-	if (!rd_cqe)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	qhp = get_qhp(rhp, CQE_QPID(*rd_cqe));</div><div class='del'>-	if (qhp) {</div><div class='del'>-		spin_lock(&amp;qhp-&gt;lock);</div><div class='del'>-		ret = __iwch_poll_cq_one(rhp, chp, qhp, wc);</div><div class='del'>-		spin_unlock(&amp;qhp-&gt;lock);</div><div class='del'>-	} else {</div><div class='del'>-		ret = __iwch_poll_cq_one(rhp, chp, NULL, wc);</div><div class='del'>-	}</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-	struct iwch_cq *chp;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int npolled;</div><div class='del'>-	int err = 0;</div><div class='del'>-</div><div class='del'>-	chp = to_iwch_cq(ibcq);</div><div class='del'>-	rhp = chp-&gt;rhp;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;chp-&gt;lock, flags);</div><div class='del'>-	for (npolled = 0; npolled &lt; num_entries; ++npolled) {</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * Because T3 can post CQEs that are _not_ associated</div><div class='del'>-		 * with a WR, we might have to poll again after removing</div><div class='del'>-		 * one of these.</div><div class='del'>-		 */</div><div class='del'>-		do {</div><div class='del'>-			err = iwch_poll_cq_one(rhp, chp, wc + npolled);</div><div class='del'>-		} while (err == -EAGAIN);</div><div class='del'>-		if (err &lt;= 0)</div><div class='del'>-			break;</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;chp-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-	if (err &lt; 0)</div><div class='del'>-		return err;</div><div class='del'>-	else {</div><div class='del'>-		return npolled;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/iwch_ev.c b/drivers/infiniband/hw/cxgb3/iwch_ev.c<br/>deleted file mode 100644<br/>index 9d356c1301c7..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/iwch_ev.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch_ev.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,232 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-#include &lt;linux/gfp.h&gt;</div><div class='del'>-#include &lt;linux/mman.h&gt;</div><div class='del'>-#include &lt;net/sock.h&gt;</div><div class='del'>-#include "iwch_provider.h"</div><div class='del'>-#include "iwch.h"</div><div class='del'>-#include "iwch_cm.h"</div><div class='del'>-#include "cxio_hal.h"</div><div class='del'>-#include "cxio_wr.h"</div><div class='del'>-</div><div class='del'>-static void post_qp_event(struct iwch_dev *rnicp, struct iwch_cq *chp,</div><div class='del'>-			  struct respQ_msg_t *rsp_msg,</div><div class='del'>-			  enum ib_event_type ib_event,</div><div class='del'>-			  int send_term)</div><div class='del'>-{</div><div class='del'>-	struct ib_event event;</div><div class='del'>-	struct iwch_qp_attributes attrs;</div><div class='del'>-	struct iwch_qp *qhp;</div><div class='del'>-	unsigned long flag;</div><div class='del'>-</div><div class='del'>-	xa_lock(&amp;rnicp-&gt;qps);</div><div class='del'>-	qhp = xa_load(&amp;rnicp-&gt;qps, CQE_QPID(rsp_msg-&gt;cqe));</div><div class='del'>-</div><div class='del'>-	if (!qhp) {</div><div class='del'>-		pr_err("%s unaffiliated error 0x%x qpid 0x%x\n",</div><div class='del'>-		       __func__, CQE_STATUS(rsp_msg-&gt;cqe),</div><div class='del'>-		       CQE_QPID(rsp_msg-&gt;cqe));</div><div class='del'>-		xa_unlock(&amp;rnicp-&gt;qps);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if ((qhp-&gt;attr.state == IWCH_QP_STATE_ERROR) ||</div><div class='del'>-	    (qhp-&gt;attr.state == IWCH_QP_STATE_TERMINATE)) {</div><div class='del'>-		pr_debug("%s AE received after RTS - qp state %d qpid 0x%x status 0x%x\n",</div><div class='del'>-			 __func__,</div><div class='del'>-			 qhp-&gt;attr.state, qhp-&gt;wq.qpid,</div><div class='del'>-			 CQE_STATUS(rsp_msg-&gt;cqe));</div><div class='del'>-		xa_unlock(&amp;rnicp-&gt;qps);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	pr_err("%s - AE qpid 0x%x opcode %d status 0x%x type %d wrid.hi 0x%x wrid.lo 0x%x\n",</div><div class='del'>-	       __func__,</div><div class='del'>-	       CQE_QPID(rsp_msg-&gt;cqe), CQE_OPCODE(rsp_msg-&gt;cqe),</div><div class='del'>-	       CQE_STATUS(rsp_msg-&gt;cqe), CQE_TYPE(rsp_msg-&gt;cqe),</div><div class='del'>-	       CQE_WRID_HI(rsp_msg-&gt;cqe), CQE_WRID_LOW(rsp_msg-&gt;cqe));</div><div class='del'>-</div><div class='del'>-	atomic_inc(&amp;qhp-&gt;refcnt);</div><div class='del'>-	xa_unlock(&amp;rnicp-&gt;qps);</div><div class='del'>-</div><div class='del'>-	if (qhp-&gt;attr.state == IWCH_QP_STATE_RTS) {</div><div class='del'>-		attrs.next_state = IWCH_QP_STATE_TERMINATE;</div><div class='del'>-		iwch_modify_qp(qhp-&gt;rhp, qhp, IWCH_QP_ATTR_NEXT_STATE,</div><div class='del'>-			       &amp;attrs, 1);</div><div class='del'>-		if (send_term)</div><div class='del'>-			iwch_post_terminate(qhp, rsp_msg);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	event.event = ib_event;</div><div class='del'>-	event.device = chp-&gt;ibcq.device;</div><div class='del'>-	if (ib_event == IB_EVENT_CQ_ERR)</div><div class='del'>-		event.element.cq = &amp;chp-&gt;ibcq;</div><div class='del'>-	else</div><div class='del'>-		event.element.qp = &amp;qhp-&gt;ibqp;</div><div class='del'>-</div><div class='del'>-	if (qhp-&gt;ibqp.event_handler)</div><div class='del'>-		(*qhp-&gt;ibqp.event_handler)(&amp;event, qhp-&gt;ibqp.qp_context);</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;chp-&gt;comp_handler_lock, flag);</div><div class='del'>-	(*chp-&gt;ibcq.comp_handler)(&amp;chp-&gt;ibcq, chp-&gt;ibcq.cq_context);</div><div class='del'>-	spin_unlock_irqrestore(&amp;chp-&gt;comp_handler_lock, flag);</div><div class='del'>-</div><div class='del'>-	if (atomic_dec_and_test(&amp;qhp-&gt;refcnt))</div><div class='del'>-		wake_up(&amp;qhp-&gt;wait);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void iwch_ev_dispatch(struct cxio_rdev *rdev_p, struct sk_buff *skb)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *rnicp;</div><div class='del'>-	struct respQ_msg_t *rsp_msg = (struct respQ_msg_t *) skb-&gt;data;</div><div class='del'>-	struct iwch_cq *chp;</div><div class='del'>-	struct iwch_qp *qhp;</div><div class='del'>-	u32 cqid = RSPQ_CQID(rsp_msg);</div><div class='del'>-	unsigned long flag;</div><div class='del'>-</div><div class='del'>-	rnicp = (struct iwch_dev *) rdev_p-&gt;ulp;</div><div class='del'>-	xa_lock(&amp;rnicp-&gt;qps);</div><div class='del'>-	chp = get_chp(rnicp, cqid);</div><div class='del'>-	qhp = xa_load(&amp;rnicp-&gt;qps, CQE_QPID(rsp_msg-&gt;cqe));</div><div class='del'>-	if (!chp || !qhp) {</div><div class='del'>-		pr_err("BAD AE cqid 0x%x qpid 0x%x opcode %d status 0x%x type %d wrid.hi 0x%x wrid.lo 0x%x\n",</div><div class='del'>-		       cqid, CQE_QPID(rsp_msg-&gt;cqe),</div><div class='del'>-		       CQE_OPCODE(rsp_msg-&gt;cqe), CQE_STATUS(rsp_msg-&gt;cqe),</div><div class='del'>-		       CQE_TYPE(rsp_msg-&gt;cqe), CQE_WRID_HI(rsp_msg-&gt;cqe),</div><div class='del'>-		       CQE_WRID_LOW(rsp_msg-&gt;cqe));</div><div class='del'>-		xa_unlock(&amp;rnicp-&gt;qps);</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-	iwch_qp_add_ref(&amp;qhp-&gt;ibqp);</div><div class='del'>-	atomic_inc(&amp;chp-&gt;refcnt);</div><div class='del'>-	xa_unlock(&amp;rnicp-&gt;qps);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * 1) completion of our sending a TERMINATE.</div><div class='del'>-	 * 2) incoming TERMINATE message.</div><div class='del'>-	 */</div><div class='del'>-	if ((CQE_OPCODE(rsp_msg-&gt;cqe) == T3_TERMINATE) &amp;&amp;</div><div class='del'>-	    (CQE_STATUS(rsp_msg-&gt;cqe) == 0)) {</div><div class='del'>-		if (SQ_TYPE(rsp_msg-&gt;cqe)) {</div><div class='del'>-			pr_debug("%s QPID 0x%x ep %p disconnecting\n",</div><div class='del'>-				 __func__, qhp-&gt;wq.qpid, qhp-&gt;ep);</div><div class='del'>-			iwch_ep_disconnect(qhp-&gt;ep, 0, GFP_ATOMIC);</div><div class='del'>-		} else {</div><div class='del'>-			pr_debug("%s post REQ_ERR AE QPID 0x%x\n", __func__,</div><div class='del'>-				 qhp-&gt;wq.qpid);</div><div class='del'>-			post_qp_event(rnicp, chp, rsp_msg,</div><div class='del'>-				      IB_EVENT_QP_REQ_ERR, 0);</div><div class='del'>-			iwch_ep_disconnect(qhp-&gt;ep, 0, GFP_ATOMIC);</div><div class='del'>-		}</div><div class='del'>-		goto done;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Bad incoming Read request */</div><div class='del'>-	if (SQ_TYPE(rsp_msg-&gt;cqe) &amp;&amp;</div><div class='del'>-	    (CQE_OPCODE(rsp_msg-&gt;cqe) == T3_READ_RESP)) {</div><div class='del'>-		post_qp_event(rnicp, chp, rsp_msg, IB_EVENT_QP_REQ_ERR, 1);</div><div class='del'>-		goto done;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Bad incoming write */</div><div class='del'>-	if (RQ_TYPE(rsp_msg-&gt;cqe) &amp;&amp;</div><div class='del'>-	    (CQE_OPCODE(rsp_msg-&gt;cqe) == T3_RDMA_WRITE)) {</div><div class='del'>-		post_qp_event(rnicp, chp, rsp_msg, IB_EVENT_QP_REQ_ERR, 1);</div><div class='del'>-		goto done;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	switch (CQE_STATUS(rsp_msg-&gt;cqe)) {</div><div class='del'>-</div><div class='del'>-	/* Completion Events */</div><div class='del'>-	case TPT_ERR_SUCCESS:</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * Confirm the destination entry if this is a RECV completion.</div><div class='del'>-		 */</div><div class='del'>-		if (qhp-&gt;ep &amp;&amp; SQ_TYPE(rsp_msg-&gt;cqe))</div><div class='del'>-			dst_confirm(qhp-&gt;ep-&gt;dst);</div><div class='del'>-		spin_lock_irqsave(&amp;chp-&gt;comp_handler_lock, flag);</div><div class='del'>-		(*chp-&gt;ibcq.comp_handler)(&amp;chp-&gt;ibcq, chp-&gt;ibcq.cq_context);</div><div class='del'>-		spin_unlock_irqrestore(&amp;chp-&gt;comp_handler_lock, flag);</div><div class='del'>-		break;</div><div class='del'>-</div><div class='del'>-	case TPT_ERR_STAG:</div><div class='del'>-	case TPT_ERR_PDID:</div><div class='del'>-	case TPT_ERR_QPID:</div><div class='del'>-	case TPT_ERR_ACCESS:</div><div class='del'>-	case TPT_ERR_WRAP:</div><div class='del'>-	case TPT_ERR_BOUND:</div><div class='del'>-	case TPT_ERR_INVALIDATE_SHARED_MR:</div><div class='del'>-	case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND:</div><div class='del'>-		post_qp_event(rnicp, chp, rsp_msg, IB_EVENT_QP_ACCESS_ERR, 1);</div><div class='del'>-		break;</div><div class='del'>-</div><div class='del'>-	/* Device Fatal Errors */</div><div class='del'>-	case TPT_ERR_ECC:</div><div class='del'>-	case TPT_ERR_ECC_PSTAG:</div><div class='del'>-	case TPT_ERR_INTERNAL_ERR:</div><div class='del'>-		post_qp_event(rnicp, chp, rsp_msg, IB_EVENT_DEVICE_FATAL, 1);</div><div class='del'>-		break;</div><div class='del'>-</div><div class='del'>-	/* QP Fatal Errors */</div><div class='del'>-	case TPT_ERR_OUT_OF_RQE:</div><div class='del'>-	case TPT_ERR_PBL_ADDR_BOUND:</div><div class='del'>-	case TPT_ERR_CRC:</div><div class='del'>-	case TPT_ERR_MARKER:</div><div class='del'>-	case TPT_ERR_PDU_LEN_ERR:</div><div class='del'>-	case TPT_ERR_DDP_VERSION:</div><div class='del'>-	case TPT_ERR_RDMA_VERSION:</div><div class='del'>-	case TPT_ERR_OPCODE:</div><div class='del'>-	case TPT_ERR_DDP_QUEUE_NUM:</div><div class='del'>-	case TPT_ERR_MSN:</div><div class='del'>-	case TPT_ERR_TBIT:</div><div class='del'>-	case TPT_ERR_MO:</div><div class='del'>-	case TPT_ERR_MSN_GAP:</div><div class='del'>-	case TPT_ERR_MSN_RANGE:</div><div class='del'>-	case TPT_ERR_RQE_ADDR_BOUND:</div><div class='del'>-	case TPT_ERR_IRD_OVERFLOW:</div><div class='del'>-		post_qp_event(rnicp, chp, rsp_msg, IB_EVENT_QP_FATAL, 1);</div><div class='del'>-		break;</div><div class='del'>-</div><div class='del'>-	default:</div><div class='del'>-		pr_err("Unknown T3 status 0x%x QPID 0x%x\n",</div><div class='del'>-		       CQE_STATUS(rsp_msg-&gt;cqe), qhp-&gt;wq.qpid);</div><div class='del'>-		post_qp_event(rnicp, chp, rsp_msg, IB_EVENT_QP_FATAL, 1);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-done:</div><div class='del'>-	if (atomic_dec_and_test(&amp;chp-&gt;refcnt))</div><div class='del'>-	        wake_up(&amp;chp-&gt;wait);</div><div class='del'>-	iwch_qp_rem_ref(&amp;qhp-&gt;ibqp);</div><div class='del'>-out:</div><div class='del'>-	dev_kfree_skb_irq(skb);</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/iwch_mem.c b/drivers/infiniband/hw/cxgb3/iwch_mem.c<br/>deleted file mode 100644<br/>index ce0f2741821d..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/iwch_mem.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch_mem.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,101 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-#include &lt;linux/slab.h&gt;</div><div class='del'>-#include &lt;asm/byteorder.h&gt;</div><div class='del'>-</div><div class='del'>-#include &lt;rdma/iw_cm.h&gt;</div><div class='del'>-#include &lt;rdma/ib_verbs.h&gt;</div><div class='del'>-</div><div class='del'>-#include "cxio_hal.h"</div><div class='del'>-#include "cxio_resource.h"</div><div class='del'>-#include "iwch.h"</div><div class='del'>-#include "iwch_provider.h"</div><div class='del'>-</div><div class='del'>-static int iwch_finish_mem_reg(struct iwch_mr *mhp, u32 stag)</div><div class='del'>-{</div><div class='del'>-	u32 mmid;</div><div class='del'>-</div><div class='del'>-	mhp-&gt;attr.state = 1;</div><div class='del'>-	mhp-&gt;attr.stag = stag;</div><div class='del'>-	mmid = stag &gt;&gt; 8;</div><div class='del'>-	mhp-&gt;ibmr.rkey = mhp-&gt;ibmr.lkey = stag;</div><div class='del'>-	pr_debug("%s mmid 0x%x mhp %p\n", __func__, mmid, mhp);</div><div class='del'>-	return xa_insert_irq(&amp;mhp-&gt;rhp-&gt;mrs, mmid, mhp, GFP_KERNEL);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_register_mem(struct iwch_dev *rhp, struct iwch_pd *php,</div><div class='del'>-		      struct iwch_mr *mhp, int shift)</div><div class='del'>-{</div><div class='del'>-	u32 stag;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if (cxio_register_phys_mem(&amp;rhp-&gt;rdev,</div><div class='del'>-				   &amp;stag, mhp-&gt;attr.pdid,</div><div class='del'>-				   mhp-&gt;attr.perms,</div><div class='del'>-				   mhp-&gt;attr.zbva,</div><div class='del'>-				   mhp-&gt;attr.va_fbo,</div><div class='del'>-				   mhp-&gt;attr.len,</div><div class='del'>-				   shift - 12,</div><div class='del'>-				   mhp-&gt;attr.pbl_size, mhp-&gt;attr.pbl_addr))</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	ret = iwch_finish_mem_reg(mhp, stag);</div><div class='del'>-	if (ret)</div><div class='del'>-		cxio_dereg_mem(&amp;rhp-&gt;rdev, mhp-&gt;attr.stag, mhp-&gt;attr.pbl_size,</div><div class='del'>-		       mhp-&gt;attr.pbl_addr);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_alloc_pbl(struct iwch_mr *mhp, int npages)</div><div class='del'>-{</div><div class='del'>-	mhp-&gt;attr.pbl_addr = cxio_hal_pblpool_alloc(&amp;mhp-&gt;rhp-&gt;rdev,</div><div class='del'>-						    npages &lt;&lt; 3);</div><div class='del'>-</div><div class='del'>-	if (!mhp-&gt;attr.pbl_addr)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	mhp-&gt;attr.pbl_size = npages;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void iwch_free_pbl(struct iwch_mr *mhp)</div><div class='del'>-{</div><div class='del'>-	cxio_hal_pblpool_free(&amp;mhp-&gt;rhp-&gt;rdev, mhp-&gt;attr.pbl_addr,</div><div class='del'>-			      mhp-&gt;attr.pbl_size &lt;&lt; 3);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_write_pbl(struct iwch_mr *mhp, __be64 *pages, int npages, int offset)</div><div class='del'>-{</div><div class='del'>-	return cxio_write_pbl(&amp;mhp-&gt;rhp-&gt;rdev, pages,</div><div class='del'>-			      mhp-&gt;attr.pbl_addr + (offset &lt;&lt; 3), npages);</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c<br/>deleted file mode 100644<br/>index 3a481dfb1607..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/iwch_provider.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch_provider.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,1402 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='del'>-#include &lt;linux/moduleparam.h&gt;</div><div class='del'>-#include &lt;linux/device.h&gt;</div><div class='del'>-#include &lt;linux/netdevice.h&gt;</div><div class='del'>-#include &lt;linux/etherdevice.h&gt;</div><div class='del'>-#include &lt;linux/delay.h&gt;</div><div class='del'>-#include &lt;linux/errno.h&gt;</div><div class='del'>-#include &lt;linux/list.h&gt;</div><div class='del'>-#include &lt;linux/sched/mm.h&gt;</div><div class='del'>-#include &lt;linux/spinlock.h&gt;</div><div class='del'>-#include &lt;linux/ethtool.h&gt;</div><div class='del'>-#include &lt;linux/rtnetlink.h&gt;</div><div class='del'>-#include &lt;linux/inetdevice.h&gt;</div><div class='del'>-#include &lt;linux/slab.h&gt;</div><div class='del'>-</div><div class='del'>-#include &lt;asm/io.h&gt;</div><div class='del'>-#include &lt;asm/irq.h&gt;</div><div class='del'>-#include &lt;asm/byteorder.h&gt;</div><div class='del'>-</div><div class='del'>-#include &lt;rdma/iw_cm.h&gt;</div><div class='del'>-#include &lt;rdma/ib_verbs.h&gt;</div><div class='del'>-#include &lt;rdma/ib_smi.h&gt;</div><div class='del'>-#include &lt;rdma/ib_umem.h&gt;</div><div class='del'>-#include &lt;rdma/ib_user_verbs.h&gt;</div><div class='del'>-#include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='del'>-</div><div class='del'>-#include "cxio_hal.h"</div><div class='del'>-#include "iwch.h"</div><div class='del'>-#include "iwch_provider.h"</div><div class='del'>-#include "iwch_cm.h"</div><div class='del'>-#include &lt;rdma/cxgb3-abi.h&gt;</div><div class='del'>-#include "common.h"</div><div class='del'>-</div><div class='del'>-static void iwch_dealloc_ucontext(struct ib_ucontext *context)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *rhp = to_iwch_dev(context-&gt;device);</div><div class='del'>-	struct iwch_ucontext *ucontext = to_iwch_ucontext(context);</div><div class='del'>-	struct iwch_mm_entry *mm, *tmp;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s context %p\n", __func__, context);</div><div class='del'>-	list_for_each_entry_safe(mm, tmp, &amp;ucontext-&gt;mmaps, entry)</div><div class='del'>-		kfree(mm);</div><div class='del'>-	cxio_release_ucontext(&amp;rhp-&gt;rdev, &amp;ucontext-&gt;uctx);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_alloc_ucontext(struct ib_ucontext *ucontext,</div><div class='del'>-			       struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct ib_device *ibdev = ucontext-&gt;device;</div><div class='del'>-	struct iwch_ucontext *context = to_iwch_ucontext(ucontext);</div><div class='del'>-	struct iwch_dev *rhp = to_iwch_dev(ibdev);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ibdev %p\n", __func__, ibdev);</div><div class='del'>-	cxio_init_ucontext(&amp;rhp-&gt;rdev, &amp;context-&gt;uctx);</div><div class='del'>-	INIT_LIST_HEAD(&amp;context-&gt;mmaps);</div><div class='del'>-	spin_lock_init(&amp;context-&gt;mmap_lock);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct iwch_cq *chp;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ib_cq %p\n", __func__, ib_cq);</div><div class='del'>-	chp = to_iwch_cq(ib_cq);</div><div class='del'>-</div><div class='del'>-	xa_erase_irq(&amp;chp-&gt;rhp-&gt;cqs, chp-&gt;cq.cqid);</div><div class='del'>-	atomic_dec(&amp;chp-&gt;refcnt);</div><div class='del'>-	wait_event(chp-&gt;wait, !atomic_read(&amp;chp-&gt;refcnt));</div><div class='del'>-</div><div class='del'>-	cxio_destroy_cq(&amp;chp-&gt;rhp-&gt;rdev, &amp;chp-&gt;cq);</div><div class='del'>-	kfree(chp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,</div><div class='del'>-				    const struct ib_cq_init_attr *attr,</div><div class='del'>-				    struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	int entries = attr-&gt;cqe;</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-	struct iwch_cq *chp;</div><div class='del'>-	struct iwch_create_cq_resp uresp;</div><div class='del'>-	struct iwch_create_cq_req ureq;</div><div class='del'>-	static int warned;</div><div class='del'>-	size_t resplen;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ib_dev %p entries %d\n", __func__, ibdev, entries);</div><div class='del'>-	if (attr-&gt;flags)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	rhp = to_iwch_dev(ibdev);</div><div class='del'>-	chp = kzalloc(sizeof(*chp), GFP_KERNEL);</div><div class='del'>-	if (!chp)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	if (udata) {</div><div class='del'>-		if (!t3a_device(rhp)) {</div><div class='del'>-			if (ib_copy_from_udata(&amp;ureq, udata, sizeof (ureq))) {</div><div class='del'>-				kfree(chp);</div><div class='del'>-				return ERR_PTR(-EFAULT);</div><div class='del'>-			}</div><div class='del'>-			chp-&gt;user_rptr_addr = (u32 __user *)(unsigned long)ureq.user_rptr_addr;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (t3a_device(rhp)) {</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * T3A: Add some fluff to handle extra CQEs inserted</div><div class='del'>-		 * for various errors.</div><div class='del'>-		 * Additional CQE possibilities:</div><div class='del'>-		 *      TERMINATE,</div><div class='del'>-		 *      incoming RDMA WRITE Failures</div><div class='del'>-		 *      incoming RDMA READ REQUEST FAILUREs</div><div class='del'>-		 * NOTE: We cannot ensure the CQ won't overflow.</div><div class='del'>-		 */</div><div class='del'>-		entries += 16;</div><div class='del'>-	}</div><div class='del'>-	entries = roundup_pow_of_two(entries);</div><div class='del'>-	chp-&gt;cq.size_log2 = ilog2(entries);</div><div class='del'>-</div><div class='del'>-	if (cxio_create_cq(&amp;rhp-&gt;rdev, &amp;chp-&gt;cq, !udata)) {</div><div class='del'>-		kfree(chp);</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-	}</div><div class='del'>-	chp-&gt;rhp = rhp;</div><div class='del'>-	chp-&gt;ibcq.cqe = 1 &lt;&lt; chp-&gt;cq.size_log2;</div><div class='del'>-	spin_lock_init(&amp;chp-&gt;lock);</div><div class='del'>-	spin_lock_init(&amp;chp-&gt;comp_handler_lock);</div><div class='del'>-	atomic_set(&amp;chp-&gt;refcnt, 1);</div><div class='del'>-	init_waitqueue_head(&amp;chp-&gt;wait);</div><div class='del'>-	if (xa_store_irq(&amp;rhp-&gt;cqs, chp-&gt;cq.cqid, chp, GFP_KERNEL)) {</div><div class='del'>-		cxio_destroy_cq(&amp;chp-&gt;rhp-&gt;rdev, &amp;chp-&gt;cq);</div><div class='del'>-		kfree(chp);</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (udata) {</div><div class='del'>-		struct iwch_mm_entry *mm;</div><div class='del'>-		struct iwch_ucontext *ucontext = rdma_udata_to_drv_context(</div><div class='del'>-			udata, struct iwch_ucontext, ibucontext);</div><div class='del'>-</div><div class='del'>-		mm = kmalloc(sizeof *mm, GFP_KERNEL);</div><div class='del'>-		if (!mm) {</div><div class='del'>-			iwch_destroy_cq(&amp;chp-&gt;ibcq, udata);</div><div class='del'>-			return ERR_PTR(-ENOMEM);</div><div class='del'>-		}</div><div class='del'>-		uresp.cqid = chp-&gt;cq.cqid;</div><div class='del'>-		uresp.size_log2 = chp-&gt;cq.size_log2;</div><div class='del'>-		spin_lock(&amp;ucontext-&gt;mmap_lock);</div><div class='del'>-		uresp.key = ucontext-&gt;key;</div><div class='del'>-		ucontext-&gt;key += PAGE_SIZE;</div><div class='del'>-		spin_unlock(&amp;ucontext-&gt;mmap_lock);</div><div class='del'>-		mm-&gt;key = uresp.key;</div><div class='del'>-		mm-&gt;addr = virt_to_phys(chp-&gt;cq.queue);</div><div class='del'>-		if (udata-&gt;outlen &lt; sizeof uresp) {</div><div class='del'>-			if (!warned++)</div><div class='del'>-				pr_warn("Warning - downlevel libcxgb3 (non-fatal)\n");</div><div class='del'>-			mm-&gt;len = PAGE_ALIGN((1UL &lt;&lt; uresp.size_log2) *</div><div class='del'>-					     sizeof(struct t3_cqe));</div><div class='del'>-			resplen = sizeof(struct iwch_create_cq_resp_v0);</div><div class='del'>-		} else {</div><div class='del'>-			mm-&gt;len = PAGE_ALIGN(((1UL &lt;&lt; uresp.size_log2) + 1) *</div><div class='del'>-					     sizeof(struct t3_cqe));</div><div class='del'>-			uresp.memsize = mm-&gt;len;</div><div class='del'>-			uresp.reserved = 0;</div><div class='del'>-			resplen = sizeof uresp;</div><div class='del'>-		}</div><div class='del'>-		if (ib_copy_to_udata(udata, &amp;uresp, resplen)) {</div><div class='del'>-			kfree(mm);</div><div class='del'>-			iwch_destroy_cq(&amp;chp-&gt;ibcq, udata);</div><div class='del'>-			return ERR_PTR(-EFAULT);</div><div class='del'>-		}</div><div class='del'>-		insert_mmap(ucontext, mm);</div><div class='del'>-	}</div><div class='del'>-	pr_debug("created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx\n",</div><div class='del'>-		 chp-&gt;cq.cqid, chp, (1 &lt;&lt; chp-&gt;cq.size_log2),</div><div class='del'>-		 (unsigned long long)chp-&gt;cq.dma_addr);</div><div class='del'>-	return &amp;chp-&gt;ibcq;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-#ifdef notyet</div><div class='del'>-	struct iwch_cq *chp = to_iwch_cq(cq);</div><div class='del'>-	struct t3_cq oldcq, newcq;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ib_cq %p cqe %d\n", __func__, cq, cqe);</div><div class='del'>-</div><div class='del'>-	/* We don't downsize... */</div><div class='del'>-	if (cqe &lt;= cq-&gt;cqe)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	/* create new t3_cq with new size */</div><div class='del'>-	cqe = roundup_pow_of_two(cqe+1);</div><div class='del'>-	newcq.size_log2 = ilog2(cqe);</div><div class='del'>-</div><div class='del'>-	/* Dont allow resize to less than the current wce count */</div><div class='del'>-	if (cqe &lt; Q_COUNT(chp-&gt;cq.rptr, chp-&gt;cq.wptr)) {</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Quiesce all QPs using this CQ */</div><div class='del'>-	ret = iwch_quiesce_qps(chp);</div><div class='del'>-	if (ret) {</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = cxio_create_cq(&amp;chp-&gt;rhp-&gt;rdev, &amp;newcq);</div><div class='del'>-	if (ret) {</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* copy CQEs */</div><div class='del'>-	memcpy(newcq.queue, chp-&gt;cq.queue, (1 &lt;&lt; chp-&gt;cq.size_log2) *</div><div class='del'>-				        sizeof(struct t3_cqe));</div><div class='del'>-</div><div class='del'>-	/* old iwch_qp gets new t3_cq but keeps old cqid */</div><div class='del'>-	oldcq = chp-&gt;cq;</div><div class='del'>-	chp-&gt;cq = newcq;</div><div class='del'>-	chp-&gt;cq.cqid = oldcq.cqid;</div><div class='del'>-</div><div class='del'>-	/* resize new t3_cq to update the HW context */</div><div class='del'>-	ret = cxio_resize_cq(&amp;chp-&gt;rhp-&gt;rdev, &amp;chp-&gt;cq);</div><div class='del'>-	if (ret) {</div><div class='del'>-		chp-&gt;cq = oldcq;</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-	chp-&gt;ibcq.cqe = (1&lt;&lt;chp-&gt;cq.size_log2) - 1;</div><div class='del'>-</div><div class='del'>-	/* destroy old t3_cq */</div><div class='del'>-	oldcq.cqid = newcq.cqid;</div><div class='del'>-	ret = cxio_destroy_cq(&amp;chp-&gt;rhp-&gt;rdev, &amp;oldcq);</div><div class='del'>-	if (ret) {</div><div class='del'>-		pr_err("%s - cxio_destroy_cq failed %d\n", __func__, ret);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* add user hooks here */</div><div class='del'>-</div><div class='del'>-	/* resume qps */</div><div class='del'>-	ret = iwch_resume_qps(chp);</div><div class='del'>-	return ret;</div><div class='del'>-#else</div><div class='del'>-	return -ENOSYS;</div><div class='del'>-#endif</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-	struct iwch_cq *chp;</div><div class='del'>-	enum t3_cq_opcode cq_op;</div><div class='del'>-	int err;</div><div class='del'>-	unsigned long flag;</div><div class='del'>-	u32 rptr;</div><div class='del'>-</div><div class='del'>-	chp = to_iwch_cq(ibcq);</div><div class='del'>-	rhp = chp-&gt;rhp;</div><div class='del'>-	if ((flags &amp; IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)</div><div class='del'>-		cq_op = CQ_ARM_SE;</div><div class='del'>-	else</div><div class='del'>-		cq_op = CQ_ARM_AN;</div><div class='del'>-	if (chp-&gt;user_rptr_addr) {</div><div class='del'>-		if (get_user(rptr, chp-&gt;user_rptr_addr))</div><div class='del'>-			return -EFAULT;</div><div class='del'>-		spin_lock_irqsave(&amp;chp-&gt;lock, flag);</div><div class='del'>-		chp-&gt;cq.rptr = rptr;</div><div class='del'>-	} else</div><div class='del'>-		spin_lock_irqsave(&amp;chp-&gt;lock, flag);</div><div class='del'>-	pr_debug("%s rptr 0x%x\n", __func__, chp-&gt;cq.rptr);</div><div class='del'>-	err = cxio_hal_cq_op(&amp;rhp-&gt;rdev, &amp;chp-&gt;cq, cq_op, 0);</div><div class='del'>-	spin_unlock_irqrestore(&amp;chp-&gt;lock, flag);</div><div class='del'>-	if (err &lt; 0)</div><div class='del'>-		pr_err("Error %d rearming CQID 0x%x\n", err, chp-&gt;cq.cqid);</div><div class='del'>-	if (err &gt; 0 &amp;&amp; !(flags &amp; IB_CQ_REPORT_MISSED_EVENTS))</div><div class='del'>-		err = 0;</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)</div><div class='del'>-{</div><div class='del'>-	int len = vma-&gt;vm_end - vma-&gt;vm_start;</div><div class='del'>-	u32 key = vma-&gt;vm_pgoff &lt;&lt; PAGE_SHIFT;</div><div class='del'>-	struct cxio_rdev *rdev_p;</div><div class='del'>-	int ret = 0;</div><div class='del'>-	struct iwch_mm_entry *mm;</div><div class='del'>-	struct iwch_ucontext *ucontext;</div><div class='del'>-	u64 addr;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s pgoff 0x%lx key 0x%x len %d\n", __func__, vma-&gt;vm_pgoff,</div><div class='del'>-		 key, len);</div><div class='del'>-</div><div class='del'>-	if (vma-&gt;vm_start &amp; (PAGE_SIZE-1)) {</div><div class='del'>-	        return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	rdev_p = &amp;(to_iwch_dev(context-&gt;device)-&gt;rdev);</div><div class='del'>-	ucontext = to_iwch_ucontext(context);</div><div class='del'>-</div><div class='del'>-	mm = remove_mmap(ucontext, key, len);</div><div class='del'>-	if (!mm)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	addr = mm-&gt;addr;</div><div class='del'>-	kfree(mm);</div><div class='del'>-</div><div class='del'>-	if ((addr &gt;= rdev_p-&gt;rnic_info.udbell_physbase) &amp;&amp;</div><div class='del'>-	    (addr &lt; (rdev_p-&gt;rnic_info.udbell_physbase +</div><div class='del'>-		       rdev_p-&gt;rnic_info.udbell_len))) {</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * Map T3 DB register.</div><div class='del'>-		 */</div><div class='del'>-		if (vma-&gt;vm_flags &amp; VM_READ) {</div><div class='del'>-			return -EPERM;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		vma-&gt;vm_page_prot = pgprot_noncached(vma-&gt;vm_page_prot);</div><div class='del'>-		vma-&gt;vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;</div><div class='del'>-		vma-&gt;vm_flags &amp;= ~VM_MAYREAD;</div><div class='del'>-		ret = io_remap_pfn_range(vma, vma-&gt;vm_start,</div><div class='del'>-					 addr &gt;&gt; PAGE_SHIFT,</div><div class='del'>-				         len, vma-&gt;vm_page_prot);</div><div class='del'>-	} else {</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * Map WQ or CQ contig dma memory...</div><div class='del'>-		 */</div><div class='del'>-		ret = remap_pfn_range(vma, vma-&gt;vm_start,</div><div class='del'>-				      addr &gt;&gt; PAGE_SHIFT,</div><div class='del'>-				      len, vma-&gt;vm_page_prot);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void iwch_deallocate_pd(struct ib_pd *pd, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-	struct iwch_pd *php;</div><div class='del'>-</div><div class='del'>-	php = to_iwch_pd(pd);</div><div class='del'>-	rhp = php-&gt;rhp;</div><div class='del'>-	pr_debug("%s ibpd %p pdid 0x%x\n", __func__, pd, php-&gt;pdid);</div><div class='del'>-	cxio_hal_put_pdid(rhp-&gt;rdev.rscp, php-&gt;pdid);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_allocate_pd(struct ib_pd *pd, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct iwch_pd *php = to_iwch_pd(pd);</div><div class='del'>-	struct ib_device *ibdev = pd-&gt;device;</div><div class='del'>-	u32 pdid;</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ibdev %p\n", __func__, ibdev);</div><div class='del'>-	rhp = (struct iwch_dev *) ibdev;</div><div class='del'>-	pdid = cxio_hal_get_pdid(rhp-&gt;rdev.rscp);</div><div class='del'>-	if (!pdid)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	php-&gt;pdid = pdid;</div><div class='del'>-	php-&gt;rhp = rhp;</div><div class='del'>-	if (udata) {</div><div class='del'>-		struct iwch_alloc_pd_resp resp = {.pdid = php-&gt;pdid};</div><div class='del'>-</div><div class='del'>-		if (ib_copy_to_udata(udata, &amp;resp, sizeof(resp))) {</div><div class='del'>-			iwch_deallocate_pd(&amp;php-&gt;ibpd, udata);</div><div class='del'>-			return -EFAULT;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	pr_debug("%s pdid 0x%0x ptr 0x%p\n", __func__, pdid, php);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-	struct iwch_mr *mhp;</div><div class='del'>-	u32 mmid;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ib_mr %p\n", __func__, ib_mr);</div><div class='del'>-</div><div class='del'>-	mhp = to_iwch_mr(ib_mr);</div><div class='del'>-	kfree(mhp-&gt;pages);</div><div class='del'>-	rhp = mhp-&gt;rhp;</div><div class='del'>-	mmid = mhp-&gt;attr.stag &gt;&gt; 8;</div><div class='del'>-	cxio_dereg_mem(&amp;rhp-&gt;rdev, mhp-&gt;attr.stag, mhp-&gt;attr.pbl_size,</div><div class='del'>-		       mhp-&gt;attr.pbl_addr);</div><div class='del'>-	iwch_free_pbl(mhp);</div><div class='del'>-	xa_erase_irq(&amp;rhp-&gt;mrs, mmid);</div><div class='del'>-	if (mhp-&gt;kva)</div><div class='del'>-		kfree((void *) (unsigned long) mhp-&gt;kva);</div><div class='del'>-	if (mhp-&gt;umem)</div><div class='del'>-		ib_umem_release(mhp-&gt;umem);</div><div class='del'>-	pr_debug("%s mmid 0x%x ptr %p\n", __func__, mmid, mhp);</div><div class='del'>-	kfree(mhp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct ib_mr *iwch_get_dma_mr(struct ib_pd *pd, int acc)</div><div class='del'>-{</div><div class='del'>-	const u64 total_size = 0xffffffff;</div><div class='del'>-	const u64 mask = (total_size + PAGE_SIZE - 1) &amp; PAGE_MASK;</div><div class='del'>-	struct iwch_pd *php = to_iwch_pd(pd);</div><div class='del'>-	struct iwch_dev *rhp = php-&gt;rhp;</div><div class='del'>-	struct iwch_mr *mhp;</div><div class='del'>-	__be64 *page_list;</div><div class='del'>-	int shift = 26, npages, ret, i;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ib_pd %p\n", __func__, pd);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * T3 only supports 32 bits of size.</div><div class='del'>-	 */</div><div class='del'>-	if (sizeof(phys_addr_t) &gt; 4) {</div><div class='del'>-		pr_warn_once("Cannot support dma_mrs on this platform\n");</div><div class='del'>-		return ERR_PTR(-ENOTSUPP);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);</div><div class='del'>-	if (!mhp)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	mhp-&gt;rhp = rhp;</div><div class='del'>-</div><div class='del'>-	npages = (total_size + (1ULL &lt;&lt; shift) - 1) &gt;&gt; shift;</div><div class='del'>-	if (!npages) {</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	page_list = kmalloc_array(npages, sizeof(u64), GFP_KERNEL);</div><div class='del'>-	if (!page_list) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; npages; i++)</div><div class='del'>-		page_list[i] = cpu_to_be64((u64)i &lt;&lt; shift);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s mask 0x%llx shift %d len %lld pbl_size %d\n",</div><div class='del'>-		 __func__, mask, shift, total_size, npages);</div><div class='del'>-</div><div class='del'>-	ret = iwch_alloc_pbl(mhp, npages);</div><div class='del'>-	if (ret) {</div><div class='del'>-		kfree(page_list);</div><div class='del'>-		goto err_pbl;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = iwch_write_pbl(mhp, page_list, npages, 0);</div><div class='del'>-	kfree(page_list);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_pbl;</div><div class='del'>-</div><div class='del'>-	mhp-&gt;attr.pdid = php-&gt;pdid;</div><div class='del'>-	mhp-&gt;attr.zbva = 0;</div><div class='del'>-</div><div class='del'>-	mhp-&gt;attr.perms = iwch_ib_to_tpt_access(acc);</div><div class='del'>-	mhp-&gt;attr.va_fbo = 0;</div><div class='del'>-	mhp-&gt;attr.page_size = shift - 12;</div><div class='del'>-</div><div class='del'>-	mhp-&gt;attr.len = (u32) total_size;</div><div class='del'>-	mhp-&gt;attr.pbl_size = npages;</div><div class='del'>-	ret = iwch_register_mem(rhp, php, mhp, shift);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_pbl;</div><div class='del'>-</div><div class='del'>-	return &amp;mhp-&gt;ibmr;</div><div class='del'>-</div><div class='del'>-err_pbl:</div><div class='del'>-	iwch_free_pbl(mhp);</div><div class='del'>-</div><div class='del'>-err:</div><div class='del'>-	kfree(mhp);</div><div class='del'>-	return ERR_PTR(ret);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='del'>-				      u64 virt, int acc, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	__be64 *pages;</div><div class='del'>-	int shift, n, i;</div><div class='del'>-	int err = 0;</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-	struct iwch_pd *php;</div><div class='del'>-	struct iwch_mr *mhp;</div><div class='del'>-	struct iwch_reg_user_mr_resp uresp;</div><div class='del'>-	struct sg_dma_page_iter sg_iter;</div><div class='del'>-	pr_debug("%s ib_pd %p\n", __func__, pd);</div><div class='del'>-</div><div class='del'>-	php = to_iwch_pd(pd);</div><div class='del'>-	rhp = php-&gt;rhp;</div><div class='del'>-	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);</div><div class='del'>-	if (!mhp)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	mhp-&gt;rhp = rhp;</div><div class='del'>-</div><div class='del'>-	mhp-&gt;umem = ib_umem_get(udata, start, length, acc, 0);</div><div class='del'>-	if (IS_ERR(mhp-&gt;umem)) {</div><div class='del'>-		err = PTR_ERR(mhp-&gt;umem);</div><div class='del'>-		kfree(mhp);</div><div class='del'>-		return ERR_PTR(err);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	shift = PAGE_SHIFT;</div><div class='del'>-</div><div class='del'>-	n = ib_umem_num_pages(mhp-&gt;umem);</div><div class='del'>-</div><div class='del'>-	err = iwch_alloc_pbl(mhp, n);</div><div class='del'>-	if (err)</div><div class='del'>-		goto err;</div><div class='del'>-</div><div class='del'>-	pages = (__be64 *) __get_free_page(GFP_KERNEL);</div><div class='del'>-	if (!pages) {</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto err_pbl;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i = n = 0;</div><div class='del'>-</div><div class='del'>-	for_each_sg_dma_page(mhp-&gt;umem-&gt;sg_head.sgl, &amp;sg_iter, mhp-&gt;umem-&gt;nmap, 0) {</div><div class='del'>-		pages[i++] = cpu_to_be64(sg_page_iter_dma_address(&amp;sg_iter));</div><div class='del'>-		if (i == PAGE_SIZE / sizeof *pages) {</div><div class='del'>-			err = iwch_write_pbl(mhp, pages, i, n);</div><div class='del'>-			if (err)</div><div class='del'>-				goto pbl_done;</div><div class='del'>-			n += i;</div><div class='del'>-			i = 0;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (i)</div><div class='del'>-		err = iwch_write_pbl(mhp, pages, i, n);</div><div class='del'>-</div><div class='del'>-pbl_done:</div><div class='del'>-	free_page((unsigned long) pages);</div><div class='del'>-	if (err)</div><div class='del'>-		goto err_pbl;</div><div class='del'>-</div><div class='del'>-	mhp-&gt;attr.pdid = php-&gt;pdid;</div><div class='del'>-	mhp-&gt;attr.zbva = 0;</div><div class='del'>-	mhp-&gt;attr.perms = iwch_ib_to_tpt_access(acc);</div><div class='del'>-	mhp-&gt;attr.va_fbo = virt;</div><div class='del'>-	mhp-&gt;attr.page_size = shift - 12;</div><div class='del'>-	mhp-&gt;attr.len = (u32) length;</div><div class='del'>-</div><div class='del'>-	err = iwch_register_mem(rhp, php, mhp, shift);</div><div class='del'>-	if (err)</div><div class='del'>-		goto err_pbl;</div><div class='del'>-</div><div class='del'>-	if (udata &amp;&amp; !t3a_device(rhp)) {</div><div class='del'>-		uresp.pbl_addr = (mhp-&gt;attr.pbl_addr -</div><div class='del'>-				 rhp-&gt;rdev.rnic_info.pbl_base) &gt;&gt; 3;</div><div class='del'>-		pr_debug("%s user resp pbl_addr 0x%x\n", __func__,</div><div class='del'>-			 uresp.pbl_addr);</div><div class='del'>-</div><div class='del'>-		if (ib_copy_to_udata(udata, &amp;uresp, sizeof (uresp))) {</div><div class='del'>-			iwch_dereg_mr(&amp;mhp-&gt;ibmr, udata);</div><div class='del'>-			err = -EFAULT;</div><div class='del'>-			goto err;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return &amp;mhp-&gt;ibmr;</div><div class='del'>-</div><div class='del'>-err_pbl:</div><div class='del'>-	iwch_free_pbl(mhp);</div><div class='del'>-</div><div class='del'>-err:</div><div class='del'>-	ib_umem_release(mhp-&gt;umem);</div><div class='del'>-	kfree(mhp);</div><div class='del'>-	return ERR_PTR(err);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,</div><div class='del'>-				   struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-	struct iwch_pd *php;</div><div class='del'>-	struct iwch_mw *mhp;</div><div class='del'>-	u32 mmid;</div><div class='del'>-	u32 stag = 0;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if (type != IB_MW_TYPE_1)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	php = to_iwch_pd(pd);</div><div class='del'>-	rhp = php-&gt;rhp;</div><div class='del'>-	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);</div><div class='del'>-	if (!mhp)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-	ret = cxio_allocate_window(&amp;rhp-&gt;rdev, &amp;stag, php-&gt;pdid);</div><div class='del'>-	if (ret) {</div><div class='del'>-		kfree(mhp);</div><div class='del'>-		return ERR_PTR(ret);</div><div class='del'>-	}</div><div class='del'>-	mhp-&gt;rhp = rhp;</div><div class='del'>-	mhp-&gt;attr.pdid = php-&gt;pdid;</div><div class='del'>-	mhp-&gt;attr.type = TPT_MW;</div><div class='del'>-	mhp-&gt;attr.stag = stag;</div><div class='del'>-	mmid = (stag) &gt;&gt; 8;</div><div class='del'>-	mhp-&gt;ibmw.rkey = stag;</div><div class='del'>-	if (xa_insert_irq(&amp;rhp-&gt;mrs, mmid, mhp, GFP_KERNEL)) {</div><div class='del'>-		cxio_deallocate_window(&amp;rhp-&gt;rdev, mhp-&gt;attr.stag);</div><div class='del'>-		kfree(mhp);</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-	}</div><div class='del'>-	pr_debug("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);</div><div class='del'>-	return &amp;(mhp-&gt;ibmw);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_dealloc_mw(struct ib_mw *mw)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-	struct iwch_mw *mhp;</div><div class='del'>-	u32 mmid;</div><div class='del'>-</div><div class='del'>-	mhp = to_iwch_mw(mw);</div><div class='del'>-	rhp = mhp-&gt;rhp;</div><div class='del'>-	mmid = (mw-&gt;rkey) &gt;&gt; 8;</div><div class='del'>-	cxio_deallocate_window(&amp;rhp-&gt;rdev, mhp-&gt;attr.stag);</div><div class='del'>-	xa_erase_irq(&amp;rhp-&gt;mrs, mmid);</div><div class='del'>-	pr_debug("%s ib_mw %p mmid 0x%x ptr %p\n", __func__, mw, mmid, mhp);</div><div class='del'>-	kfree(mhp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct ib_mr *iwch_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,</div><div class='del'>-				   u32 max_num_sg, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-	struct iwch_pd *php;</div><div class='del'>-	struct iwch_mr *mhp;</div><div class='del'>-	u32 mmid;</div><div class='del'>-	u32 stag = 0;</div><div class='del'>-	int ret = -ENOMEM;</div><div class='del'>-</div><div class='del'>-	if (mr_type != IB_MR_TYPE_MEM_REG ||</div><div class='del'>-	    max_num_sg &gt; T3_MAX_FASTREG_DEPTH)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	php = to_iwch_pd(pd);</div><div class='del'>-	rhp = php-&gt;rhp;</div><div class='del'>-	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);</div><div class='del'>-	if (!mhp)</div><div class='del'>-		goto err;</div><div class='del'>-</div><div class='del'>-	mhp-&gt;pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);</div><div class='del'>-	if (!mhp-&gt;pages)</div><div class='del'>-		goto pl_err;</div><div class='del'>-</div><div class='del'>-	mhp-&gt;rhp = rhp;</div><div class='del'>-	ret = iwch_alloc_pbl(mhp, max_num_sg);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err1;</div><div class='del'>-	mhp-&gt;attr.pbl_size = max_num_sg;</div><div class='del'>-	ret = cxio_allocate_stag(&amp;rhp-&gt;rdev, &amp;stag, php-&gt;pdid,</div><div class='del'>-				 mhp-&gt;attr.pbl_size, mhp-&gt;attr.pbl_addr);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err2;</div><div class='del'>-	mhp-&gt;attr.pdid = php-&gt;pdid;</div><div class='del'>-	mhp-&gt;attr.type = TPT_NON_SHARED_MR;</div><div class='del'>-	mhp-&gt;attr.stag = stag;</div><div class='del'>-	mhp-&gt;attr.state = 1;</div><div class='del'>-	mmid = (stag) &gt;&gt; 8;</div><div class='del'>-	mhp-&gt;ibmr.rkey = mhp-&gt;ibmr.lkey = stag;</div><div class='del'>-	ret = xa_insert_irq(&amp;rhp-&gt;mrs, mmid, mhp, GFP_KERNEL);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err3;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);</div><div class='del'>-	return &amp;(mhp-&gt;ibmr);</div><div class='del'>-err3:</div><div class='del'>-	cxio_dereg_mem(&amp;rhp-&gt;rdev, stag, mhp-&gt;attr.pbl_size,</div><div class='del'>-		       mhp-&gt;attr.pbl_addr);</div><div class='del'>-err2:</div><div class='del'>-	iwch_free_pbl(mhp);</div><div class='del'>-err1:</div><div class='del'>-	kfree(mhp-&gt;pages);</div><div class='del'>-pl_err:</div><div class='del'>-	kfree(mhp);</div><div class='del'>-err:</div><div class='del'>-	return ERR_PTR(ret);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_set_page(struct ib_mr *ibmr, u64 addr)</div><div class='del'>-{</div><div class='del'>-	struct iwch_mr *mhp = to_iwch_mr(ibmr);</div><div class='del'>-</div><div class='del'>-	if (unlikely(mhp-&gt;npages == mhp-&gt;attr.pbl_size))</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	mhp-&gt;pages[mhp-&gt;npages++] = addr;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,</div><div class='del'>-			  int sg_nents, unsigned int *sg_offset)</div><div class='del'>-{</div><div class='del'>-	struct iwch_mr *mhp = to_iwch_mr(ibmr);</div><div class='del'>-</div><div class='del'>-	mhp-&gt;npages = 0;</div><div class='del'>-</div><div class='del'>-	return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, iwch_set_page);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-	struct iwch_qp *qhp;</div><div class='del'>-	struct iwch_qp_attributes attrs;</div><div class='del'>-	struct iwch_ucontext *ucontext;</div><div class='del'>-</div><div class='del'>-	qhp = to_iwch_qp(ib_qp);</div><div class='del'>-	rhp = qhp-&gt;rhp;</div><div class='del'>-</div><div class='del'>-	attrs.next_state = IWCH_QP_STATE_ERROR;</div><div class='del'>-	iwch_modify_qp(rhp, qhp, IWCH_QP_ATTR_NEXT_STATE, &amp;attrs, 0);</div><div class='del'>-	wait_event(qhp-&gt;wait, !qhp-&gt;ep);</div><div class='del'>-</div><div class='del'>-	xa_erase_irq(&amp;rhp-&gt;qps, qhp-&gt;wq.qpid);</div><div class='del'>-</div><div class='del'>-	atomic_dec(&amp;qhp-&gt;refcnt);</div><div class='del'>-	wait_event(qhp-&gt;wait, !atomic_read(&amp;qhp-&gt;refcnt));</div><div class='del'>-</div><div class='del'>-	ucontext = rdma_udata_to_drv_context(udata, struct iwch_ucontext,</div><div class='del'>-					     ibucontext);</div><div class='del'>-	cxio_destroy_qp(&amp;rhp-&gt;rdev, &amp;qhp-&gt;wq,</div><div class='del'>-			ucontext ? &amp;ucontext-&gt;uctx : &amp;rhp-&gt;rdev.uctx);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ib_qp %p qpid 0x%0x qhp %p\n", __func__,</div><div class='del'>-		 ib_qp, qhp-&gt;wq.qpid, qhp);</div><div class='del'>-	kfree(qhp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct ib_qp *iwch_create_qp(struct ib_pd *pd,</div><div class='del'>-			     struct ib_qp_init_attr *attrs,</div><div class='del'>-			     struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-	struct iwch_qp *qhp;</div><div class='del'>-	struct iwch_pd *php;</div><div class='del'>-	struct iwch_cq *schp;</div><div class='del'>-	struct iwch_cq *rchp;</div><div class='del'>-	struct iwch_create_qp_resp uresp;</div><div class='del'>-	int wqsize, sqsize, rqsize;</div><div class='del'>-	struct iwch_ucontext *ucontext;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ib_pd %p\n", __func__, pd);</div><div class='del'>-	if (attrs-&gt;qp_type != IB_QPT_RC)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-	php = to_iwch_pd(pd);</div><div class='del'>-	rhp = php-&gt;rhp;</div><div class='del'>-	schp = get_chp(rhp, ((struct iwch_cq *) attrs-&gt;send_cq)-&gt;cq.cqid);</div><div class='del'>-	rchp = get_chp(rhp, ((struct iwch_cq *) attrs-&gt;recv_cq)-&gt;cq.cqid);</div><div class='del'>-	if (!schp || !rchp)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	/* The RQT size must be # of entries + 1 rounded up to a power of two */</div><div class='del'>-	rqsize = roundup_pow_of_two(attrs-&gt;cap.max_recv_wr);</div><div class='del'>-	if (rqsize == attrs-&gt;cap.max_recv_wr)</div><div class='del'>-		rqsize = roundup_pow_of_two(attrs-&gt;cap.max_recv_wr+1);</div><div class='del'>-</div><div class='del'>-	/* T3 doesn't support RQT depth &lt; 16 */</div><div class='del'>-	if (rqsize &lt; 16)</div><div class='del'>-		rqsize = 16;</div><div class='del'>-</div><div class='del'>-	if (rqsize &gt; T3_MAX_RQ_SIZE)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	if (attrs-&gt;cap.max_inline_data &gt; T3_MAX_INLINE)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * NOTE: The SQ and total WQ sizes don't need to be</div><div class='del'>-	 * a power of two.  However, all the code assumes</div><div class='del'>-	 * they are. EG: Q_FREECNT() and friends.</div><div class='del'>-	 */</div><div class='del'>-	sqsize = roundup_pow_of_two(attrs-&gt;cap.max_send_wr);</div><div class='del'>-	wqsize = roundup_pow_of_two(rqsize + sqsize);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Kernel users need more wq space for fastreg WRs which can take</div><div class='del'>-	 * 2 WR fragments.</div><div class='del'>-	 */</div><div class='del'>-	ucontext = rdma_udata_to_drv_context(udata, struct iwch_ucontext,</div><div class='del'>-					     ibucontext);</div><div class='del'>-	if (!ucontext &amp;&amp; wqsize &lt; (rqsize + (2 * sqsize)))</div><div class='del'>-		wqsize = roundup_pow_of_two(rqsize +</div><div class='del'>-				roundup_pow_of_two(attrs-&gt;cap.max_send_wr * 2));</div><div class='del'>-	pr_debug("%s wqsize %d sqsize %d rqsize %d\n", __func__,</div><div class='del'>-		 wqsize, sqsize, rqsize);</div><div class='del'>-	qhp = kzalloc(sizeof(*qhp), GFP_KERNEL);</div><div class='del'>-	if (!qhp)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-	qhp-&gt;wq.size_log2 = ilog2(wqsize);</div><div class='del'>-	qhp-&gt;wq.rq_size_log2 = ilog2(rqsize);</div><div class='del'>-	qhp-&gt;wq.sq_size_log2 = ilog2(sqsize);</div><div class='del'>-	if (cxio_create_qp(&amp;rhp-&gt;rdev, !udata, &amp;qhp-&gt;wq,</div><div class='del'>-			   ucontext ? &amp;ucontext-&gt;uctx : &amp;rhp-&gt;rdev.uctx)) {</div><div class='del'>-		kfree(qhp);</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	attrs-&gt;cap.max_recv_wr = rqsize - 1;</div><div class='del'>-	attrs-&gt;cap.max_send_wr = sqsize;</div><div class='del'>-	attrs-&gt;cap.max_inline_data = T3_MAX_INLINE;</div><div class='del'>-</div><div class='del'>-	qhp-&gt;rhp = rhp;</div><div class='del'>-	qhp-&gt;attr.pd = php-&gt;pdid;</div><div class='del'>-	qhp-&gt;attr.scq = ((struct iwch_cq *) attrs-&gt;send_cq)-&gt;cq.cqid;</div><div class='del'>-	qhp-&gt;attr.rcq = ((struct iwch_cq *) attrs-&gt;recv_cq)-&gt;cq.cqid;</div><div class='del'>-	qhp-&gt;attr.sq_num_entries = attrs-&gt;cap.max_send_wr;</div><div class='del'>-	qhp-&gt;attr.rq_num_entries = attrs-&gt;cap.max_recv_wr;</div><div class='del'>-	qhp-&gt;attr.sq_max_sges = attrs-&gt;cap.max_send_sge;</div><div class='del'>-	qhp-&gt;attr.sq_max_sges_rdma_write = attrs-&gt;cap.max_send_sge;</div><div class='del'>-	qhp-&gt;attr.rq_max_sges = attrs-&gt;cap.max_recv_sge;</div><div class='del'>-	qhp-&gt;attr.state = IWCH_QP_STATE_IDLE;</div><div class='del'>-	qhp-&gt;attr.next_state = IWCH_QP_STATE_IDLE;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * XXX - These don't get passed in from the openib user</div><div class='del'>-	 * at create time.  The CM sets them via a QP modify.</div><div class='del'>-	 * Need to fix...  I think the CM should</div><div class='del'>-	 */</div><div class='del'>-	qhp-&gt;attr.enable_rdma_read = 1;</div><div class='del'>-	qhp-&gt;attr.enable_rdma_write = 1;</div><div class='del'>-	qhp-&gt;attr.enable_bind = 1;</div><div class='del'>-	qhp-&gt;attr.max_ord = 1;</div><div class='del'>-	qhp-&gt;attr.max_ird = 1;</div><div class='del'>-</div><div class='del'>-	spin_lock_init(&amp;qhp-&gt;lock);</div><div class='del'>-	init_waitqueue_head(&amp;qhp-&gt;wait);</div><div class='del'>-	atomic_set(&amp;qhp-&gt;refcnt, 1);</div><div class='del'>-</div><div class='del'>-	if (xa_store_irq(&amp;rhp-&gt;qps, qhp-&gt;wq.qpid, qhp, GFP_KERNEL)) {</div><div class='del'>-		cxio_destroy_qp(&amp;rhp-&gt;rdev, &amp;qhp-&gt;wq,</div><div class='del'>-			ucontext ? &amp;ucontext-&gt;uctx : &amp;rhp-&gt;rdev.uctx);</div><div class='del'>-		kfree(qhp);</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (udata) {</div><div class='del'>-</div><div class='del'>-		struct iwch_mm_entry *mm1, *mm2;</div><div class='del'>-</div><div class='del'>-		mm1 = kmalloc(sizeof *mm1, GFP_KERNEL);</div><div class='del'>-		if (!mm1) {</div><div class='del'>-			iwch_destroy_qp(&amp;qhp-&gt;ibqp, udata);</div><div class='del'>-			return ERR_PTR(-ENOMEM);</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		mm2 = kmalloc(sizeof *mm2, GFP_KERNEL);</div><div class='del'>-		if (!mm2) {</div><div class='del'>-			kfree(mm1);</div><div class='del'>-			iwch_destroy_qp(&amp;qhp-&gt;ibqp, udata);</div><div class='del'>-			return ERR_PTR(-ENOMEM);</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		uresp.qpid = qhp-&gt;wq.qpid;</div><div class='del'>-		uresp.size_log2 = qhp-&gt;wq.size_log2;</div><div class='del'>-		uresp.sq_size_log2 = qhp-&gt;wq.sq_size_log2;</div><div class='del'>-		uresp.rq_size_log2 = qhp-&gt;wq.rq_size_log2;</div><div class='del'>-		spin_lock(&amp;ucontext-&gt;mmap_lock);</div><div class='del'>-		uresp.key = ucontext-&gt;key;</div><div class='del'>-		ucontext-&gt;key += PAGE_SIZE;</div><div class='del'>-		uresp.db_key = ucontext-&gt;key;</div><div class='del'>-		ucontext-&gt;key += PAGE_SIZE;</div><div class='del'>-		spin_unlock(&amp;ucontext-&gt;mmap_lock);</div><div class='del'>-		if (ib_copy_to_udata(udata, &amp;uresp, sizeof (uresp))) {</div><div class='del'>-			kfree(mm1);</div><div class='del'>-			kfree(mm2);</div><div class='del'>-			iwch_destroy_qp(&amp;qhp-&gt;ibqp, udata);</div><div class='del'>-			return ERR_PTR(-EFAULT);</div><div class='del'>-		}</div><div class='del'>-		mm1-&gt;key = uresp.key;</div><div class='del'>-		mm1-&gt;addr = virt_to_phys(qhp-&gt;wq.queue);</div><div class='del'>-		mm1-&gt;len = PAGE_ALIGN(wqsize * sizeof (union t3_wr));</div><div class='del'>-		insert_mmap(ucontext, mm1);</div><div class='del'>-		mm2-&gt;key = uresp.db_key;</div><div class='del'>-		mm2-&gt;addr = qhp-&gt;wq.udb &amp; PAGE_MASK;</div><div class='del'>-		mm2-&gt;len = PAGE_SIZE;</div><div class='del'>-		insert_mmap(ucontext, mm2);</div><div class='del'>-	}</div><div class='del'>-	qhp-&gt;ibqp.qp_num = qhp-&gt;wq.qpid;</div><div class='del'>-	pr_debug("%s sq_num_entries %d, rq_num_entries %d qpid 0x%0x qhp %p dma_addr 0x%llx size %d rq_addr 0x%x\n",</div><div class='del'>-		 __func__, qhp-&gt;attr.sq_num_entries, qhp-&gt;attr.rq_num_entries,</div><div class='del'>-		 qhp-&gt;wq.qpid, qhp, (unsigned long long)qhp-&gt;wq.dma_addr,</div><div class='del'>-		 1 &lt;&lt; qhp-&gt;wq.size_log2, qhp-&gt;wq.rq_addr);</div><div class='del'>-	return &amp;qhp-&gt;ibqp;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='del'>-		      int attr_mask, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-	struct iwch_qp *qhp;</div><div class='del'>-	enum iwch_qp_attr_mask mask = 0;</div><div class='del'>-	struct iwch_qp_attributes attrs;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ib_qp %p\n", __func__, ibqp);</div><div class='del'>-</div><div class='del'>-	/* iwarp does not support the RTR state */</div><div class='del'>-	if ((attr_mask &amp; IB_QP_STATE) &amp;&amp; (attr-&gt;qp_state == IB_QPS_RTR))</div><div class='del'>-		attr_mask &amp;= ~IB_QP_STATE;</div><div class='del'>-</div><div class='del'>-	/* Make sure we still have something left to do */</div><div class='del'>-	if (!attr_mask)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	memset(&amp;attrs, 0, sizeof attrs);</div><div class='del'>-	qhp = to_iwch_qp(ibqp);</div><div class='del'>-	rhp = qhp-&gt;rhp;</div><div class='del'>-</div><div class='del'>-	attrs.next_state = iwch_convert_state(attr-&gt;qp_state);</div><div class='del'>-	attrs.enable_rdma_read = (attr-&gt;qp_access_flags &amp;</div><div class='del'>-			       IB_ACCESS_REMOTE_READ) ?  1 : 0;</div><div class='del'>-	attrs.enable_rdma_write = (attr-&gt;qp_access_flags &amp;</div><div class='del'>-				IB_ACCESS_REMOTE_WRITE) ? 1 : 0;</div><div class='del'>-	attrs.enable_bind = (attr-&gt;qp_access_flags &amp; IB_ACCESS_MW_BIND) ? 1 : 0;</div><div class='del'>-</div><div class='del'>-</div><div class='del'>-	mask |= (attr_mask &amp; IB_QP_STATE) ? IWCH_QP_ATTR_NEXT_STATE : 0;</div><div class='del'>-	mask |= (attr_mask &amp; IB_QP_ACCESS_FLAGS) ?</div><div class='del'>-			(IWCH_QP_ATTR_ENABLE_RDMA_READ |</div><div class='del'>-			 IWCH_QP_ATTR_ENABLE_RDMA_WRITE |</div><div class='del'>-			 IWCH_QP_ATTR_ENABLE_RDMA_BIND) : 0;</div><div class='del'>-</div><div class='del'>-	return iwch_modify_qp(rhp, qhp, mask, &amp;attrs, 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void iwch_qp_add_ref(struct ib_qp *qp)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s ib_qp %p\n", __func__, qp);</div><div class='del'>-	atomic_inc(&amp;(to_iwch_qp(qp)-&gt;refcnt));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void iwch_qp_rem_ref(struct ib_qp *qp)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s ib_qp %p\n", __func__, qp);</div><div class='del'>-	if (atomic_dec_and_test(&amp;(to_iwch_qp(qp)-&gt;refcnt)))</div><div class='del'>-	        wake_up(&amp;(to_iwch_qp(qp)-&gt;wait));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct ib_qp *iwch_get_qp(struct ib_device *dev, int qpn)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s ib_dev %p qpn 0x%x\n", __func__, dev, qpn);</div><div class='del'>-	return (struct ib_qp *)get_qhp(to_iwch_dev(dev), qpn);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-</div><div class='del'>-static int iwch_query_pkey(struct ib_device *ibdev,</div><div class='del'>-			   u8 port, u16 index, u16 * pkey)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s ibdev %p\n", __func__, ibdev);</div><div class='del'>-	*pkey = 0;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_query_gid(struct ib_device *ibdev, u8 port,</div><div class='del'>-			  int index, union ib_gid *gid)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *dev;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ibdev %p, port %d, index %d, gid %p\n",</div><div class='del'>-		 __func__, ibdev, port, index, gid);</div><div class='del'>-	dev = to_iwch_dev(ibdev);</div><div class='del'>-	BUG_ON(port == 0 || port &gt; 2);</div><div class='del'>-	memset(&amp;(gid-&gt;raw[0]), 0, sizeof(gid-&gt;raw));</div><div class='del'>-	memcpy(&amp;(gid-&gt;raw[0]), dev-&gt;rdev.port_info.lldevs[port-1]-&gt;dev_addr, 6);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static u64 fw_vers_string_to_u64(struct iwch_dev *iwch_dev)</div><div class='del'>-{</div><div class='del'>-	struct ethtool_drvinfo info;</div><div class='del'>-	struct net_device *lldev = iwch_dev-&gt;rdev.t3cdev_p-&gt;lldev;</div><div class='del'>-	char *cp, *next;</div><div class='del'>-	unsigned fw_maj, fw_min, fw_mic;</div><div class='del'>-</div><div class='del'>-	lldev-&gt;ethtool_ops-&gt;get_drvinfo(lldev, &amp;info);</div><div class='del'>-</div><div class='del'>-	next = info.fw_version + 1;</div><div class='del'>-	cp = strsep(&amp;next, ".");</div><div class='del'>-	sscanf(cp, "%i", &amp;fw_maj);</div><div class='del'>-	cp = strsep(&amp;next, ".");</div><div class='del'>-	sscanf(cp, "%i", &amp;fw_min);</div><div class='del'>-	cp = strsep(&amp;next, ".");</div><div class='del'>-	sscanf(cp, "%i", &amp;fw_mic);</div><div class='del'>-</div><div class='del'>-	return (((u64)fw_maj &amp; 0xffff) &lt;&lt; 32) | ((fw_min &amp; 0xffff) &lt;&lt; 16) |</div><div class='del'>-	       (fw_mic &amp; 0xffff);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_query_device(struct ib_device *ibdev, struct ib_device_attr *props,</div><div class='del'>-			     struct ib_udata *uhw)</div><div class='del'>-{</div><div class='del'>-</div><div class='del'>-	struct iwch_dev *dev;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ibdev %p\n", __func__, ibdev);</div><div class='del'>-</div><div class='del'>-	if (uhw-&gt;inlen || uhw-&gt;outlen)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	dev = to_iwch_dev(ibdev);</div><div class='del'>-	memset(props, 0, sizeof *props);</div><div class='del'>-	memcpy(&amp;props-&gt;sys_image_guid, dev-&gt;rdev.t3cdev_p-&gt;lldev-&gt;dev_addr, 6);</div><div class='del'>-	props-&gt;hw_ver = dev-&gt;rdev.t3cdev_p-&gt;type;</div><div class='del'>-	props-&gt;fw_ver = fw_vers_string_to_u64(dev);</div><div class='del'>-	props-&gt;device_cap_flags = dev-&gt;device_cap_flags;</div><div class='del'>-	props-&gt;page_size_cap = dev-&gt;attr.mem_pgsizes_bitmask;</div><div class='del'>-	props-&gt;vendor_id = (u32)dev-&gt;rdev.rnic_info.pdev-&gt;vendor;</div><div class='del'>-	props-&gt;vendor_part_id = (u32)dev-&gt;rdev.rnic_info.pdev-&gt;device;</div><div class='del'>-	props-&gt;max_mr_size = dev-&gt;attr.max_mr_size;</div><div class='del'>-	props-&gt;max_qp = dev-&gt;attr.max_qps;</div><div class='del'>-	props-&gt;max_qp_wr = dev-&gt;attr.max_wrs;</div><div class='del'>-	props-&gt;max_send_sge = dev-&gt;attr.max_sge_per_wr;</div><div class='del'>-	props-&gt;max_recv_sge = dev-&gt;attr.max_sge_per_wr;</div><div class='del'>-	props-&gt;max_sge_rd = 1;</div><div class='del'>-	props-&gt;max_qp_rd_atom = dev-&gt;attr.max_rdma_reads_per_qp;</div><div class='del'>-	props-&gt;max_qp_init_rd_atom = dev-&gt;attr.max_rdma_reads_per_qp;</div><div class='del'>-	props-&gt;max_cq = dev-&gt;attr.max_cqs;</div><div class='del'>-	props-&gt;max_cqe = dev-&gt;attr.max_cqes_per_cq;</div><div class='del'>-	props-&gt;max_mr = dev-&gt;attr.max_mem_regs;</div><div class='del'>-	props-&gt;max_pd = dev-&gt;attr.max_pds;</div><div class='del'>-	props-&gt;local_ca_ack_delay = 0;</div><div class='del'>-	props-&gt;max_fast_reg_page_list_len = T3_MAX_FASTREG_DEPTH;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_query_port(struct ib_device *ibdev,</div><div class='del'>-			   u8 port, struct ib_port_attr *props)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *dev;</div><div class='del'>-	struct net_device *netdev;</div><div class='del'>-	struct in_device *inetdev;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ibdev %p\n", __func__, ibdev);</div><div class='del'>-</div><div class='del'>-	dev = to_iwch_dev(ibdev);</div><div class='del'>-	netdev = dev-&gt;rdev.port_info.lldevs[port-1];</div><div class='del'>-</div><div class='del'>-	/* props being zeroed by the caller, avoid zeroing it here */</div><div class='del'>-	props-&gt;max_mtu = IB_MTU_4096;</div><div class='del'>-	props-&gt;active_mtu = ib_mtu_int_to_enum(netdev-&gt;mtu);</div><div class='del'>-</div><div class='del'>-	if (!netif_carrier_ok(netdev))</div><div class='del'>-		props-&gt;state = IB_PORT_DOWN;</div><div class='del'>-	else {</div><div class='del'>-		inetdev = in_dev_get(netdev);</div><div class='del'>-		if (inetdev) {</div><div class='del'>-			if (inetdev-&gt;ifa_list)</div><div class='del'>-				props-&gt;state = IB_PORT_ACTIVE;</div><div class='del'>-			else</div><div class='del'>-				props-&gt;state = IB_PORT_INIT;</div><div class='del'>-			in_dev_put(inetdev);</div><div class='del'>-		} else</div><div class='del'>-			props-&gt;state = IB_PORT_INIT;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	props-&gt;port_cap_flags =</div><div class='del'>-	    IB_PORT_CM_SUP |</div><div class='del'>-	    IB_PORT_SNMP_TUNNEL_SUP |</div><div class='del'>-	    IB_PORT_REINIT_SUP |</div><div class='del'>-	    IB_PORT_DEVICE_MGMT_SUP |</div><div class='del'>-	    IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;</div><div class='del'>-	props-&gt;gid_tbl_len = 1;</div><div class='del'>-	props-&gt;pkey_tbl_len = 1;</div><div class='del'>-	props-&gt;active_width = 2;</div><div class='del'>-	props-&gt;active_speed = IB_SPEED_DDR;</div><div class='del'>-	props-&gt;max_msg_sz = -1;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t hw_rev_show(struct device *dev,</div><div class='del'>-			   struct device_attribute *attr, char *buf)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *iwch_dev =</div><div class='del'>-			rdma_device_to_drv_device(dev, struct iwch_dev, ibdev);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s dev 0x%p\n", __func__, dev);</div><div class='del'>-	return sprintf(buf, "%d\n", iwch_dev-&gt;rdev.t3cdev_p-&gt;type);</div><div class='del'>-}</div><div class='del'>-static DEVICE_ATTR_RO(hw_rev);</div><div class='del'>-</div><div class='del'>-static ssize_t hca_type_show(struct device *dev,</div><div class='del'>-			     struct device_attribute *attr, char *buf)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *iwch_dev =</div><div class='del'>-			rdma_device_to_drv_device(dev, struct iwch_dev, ibdev);</div><div class='del'>-	struct ethtool_drvinfo info;</div><div class='del'>-	struct net_device *lldev = iwch_dev-&gt;rdev.t3cdev_p-&gt;lldev;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s dev 0x%p\n", __func__, dev);</div><div class='del'>-	lldev-&gt;ethtool_ops-&gt;get_drvinfo(lldev, &amp;info);</div><div class='del'>-	return sprintf(buf, "%s\n", info.driver);</div><div class='del'>-}</div><div class='del'>-static DEVICE_ATTR_RO(hca_type);</div><div class='del'>-</div><div class='del'>-static ssize_t board_id_show(struct device *dev,</div><div class='del'>-			     struct device_attribute *attr, char *buf)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *iwch_dev =</div><div class='del'>-			rdma_device_to_drv_device(dev, struct iwch_dev, ibdev);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s dev 0x%p\n", __func__, dev);</div><div class='del'>-	return sprintf(buf, "%x.%x\n", iwch_dev-&gt;rdev.rnic_info.pdev-&gt;vendor,</div><div class='del'>-		       iwch_dev-&gt;rdev.rnic_info.pdev-&gt;device);</div><div class='del'>-}</div><div class='del'>-static DEVICE_ATTR_RO(board_id);</div><div class='del'>-</div><div class='del'>-enum counters {</div><div class='del'>-	IPINRECEIVES,</div><div class='del'>-	IPINHDRERRORS,</div><div class='del'>-	IPINADDRERRORS,</div><div class='del'>-	IPINUNKNOWNPROTOS,</div><div class='del'>-	IPINDISCARDS,</div><div class='del'>-	IPINDELIVERS,</div><div class='del'>-	IPOUTREQUESTS,</div><div class='del'>-	IPOUTDISCARDS,</div><div class='del'>-	IPOUTNOROUTES,</div><div class='del'>-	IPREASMTIMEOUT,</div><div class='del'>-	IPREASMREQDS,</div><div class='del'>-	IPREASMOKS,</div><div class='del'>-	IPREASMFAILS,</div><div class='del'>-	TCPACTIVEOPENS,</div><div class='del'>-	TCPPASSIVEOPENS,</div><div class='del'>-	TCPATTEMPTFAILS,</div><div class='del'>-	TCPESTABRESETS,</div><div class='del'>-	TCPCURRESTAB,</div><div class='del'>-	TCPINSEGS,</div><div class='del'>-	TCPOUTSEGS,</div><div class='del'>-	TCPRETRANSSEGS,</div><div class='del'>-	TCPINERRS,</div><div class='del'>-	TCPOUTRSTS,</div><div class='del'>-	TCPRTOMIN,</div><div class='del'>-	TCPRTOMAX,</div><div class='del'>-	NR_COUNTERS</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static const char * const names[] = {</div><div class='del'>-	[IPINRECEIVES] = "ipInReceives",</div><div class='del'>-	[IPINHDRERRORS] = "ipInHdrErrors",</div><div class='del'>-	[IPINADDRERRORS] = "ipInAddrErrors",</div><div class='del'>-	[IPINUNKNOWNPROTOS] = "ipInUnknownProtos",</div><div class='del'>-	[IPINDISCARDS] = "ipInDiscards",</div><div class='del'>-	[IPINDELIVERS] = "ipInDelivers",</div><div class='del'>-	[IPOUTREQUESTS] = "ipOutRequests",</div><div class='del'>-	[IPOUTDISCARDS] = "ipOutDiscards",</div><div class='del'>-	[IPOUTNOROUTES] = "ipOutNoRoutes",</div><div class='del'>-	[IPREASMTIMEOUT] = "ipReasmTimeout",</div><div class='del'>-	[IPREASMREQDS] = "ipReasmReqds",</div><div class='del'>-	[IPREASMOKS] = "ipReasmOKs",</div><div class='del'>-	[IPREASMFAILS] = "ipReasmFails",</div><div class='del'>-	[TCPACTIVEOPENS] = "tcpActiveOpens",</div><div class='del'>-	[TCPPASSIVEOPENS] = "tcpPassiveOpens",</div><div class='del'>-	[TCPATTEMPTFAILS] = "tcpAttemptFails",</div><div class='del'>-	[TCPESTABRESETS] = "tcpEstabResets",</div><div class='del'>-	[TCPCURRESTAB] = "tcpCurrEstab",</div><div class='del'>-	[TCPINSEGS] = "tcpInSegs",</div><div class='del'>-	[TCPOUTSEGS] = "tcpOutSegs",</div><div class='del'>-	[TCPRETRANSSEGS] = "tcpRetransSegs",</div><div class='del'>-	[TCPINERRS] = "tcpInErrs",</div><div class='del'>-	[TCPOUTRSTS] = "tcpOutRsts",</div><div class='del'>-	[TCPRTOMIN] = "tcpRtoMin",</div><div class='del'>-	[TCPRTOMAX] = "tcpRtoMax",</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct rdma_hw_stats *iwch_alloc_stats(struct ib_device *ibdev,</div><div class='del'>-					      u8 port_num)</div><div class='del'>-{</div><div class='del'>-	BUILD_BUG_ON(ARRAY_SIZE(names) != NR_COUNTERS);</div><div class='del'>-</div><div class='del'>-	/* Our driver only supports device level stats */</div><div class='del'>-	if (port_num != 0)</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	return rdma_alloc_hw_stats_struct(names, NR_COUNTERS,</div><div class='del'>-					  RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_get_mib(struct ib_device *ibdev, struct rdma_hw_stats *stats,</div><div class='del'>-			u8 port, int index)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *dev;</div><div class='del'>-	struct tp_mib_stats m;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if (port != 0 || !stats)</div><div class='del'>-		return -ENOSYS;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s ibdev %p\n", __func__, ibdev);</div><div class='del'>-	dev = to_iwch_dev(ibdev);</div><div class='del'>-	ret = dev-&gt;rdev.t3cdev_p-&gt;ctl(dev-&gt;rdev.t3cdev_p, RDMA_GET_MIB, &amp;m);</div><div class='del'>-	if (ret)</div><div class='del'>-		return -ENOSYS;</div><div class='del'>-</div><div class='del'>-	stats-&gt;value[IPINRECEIVES] = ((u64)m.ipInReceive_hi &lt;&lt; 32) +	m.ipInReceive_lo;</div><div class='del'>-	stats-&gt;value[IPINHDRERRORS] = ((u64)m.ipInHdrErrors_hi &lt;&lt; 32) + m.ipInHdrErrors_lo;</div><div class='del'>-	stats-&gt;value[IPINADDRERRORS] = ((u64)m.ipInAddrErrors_hi &lt;&lt; 32) + m.ipInAddrErrors_lo;</div><div class='del'>-	stats-&gt;value[IPINUNKNOWNPROTOS] = ((u64)m.ipInUnknownProtos_hi &lt;&lt; 32) + m.ipInUnknownProtos_lo;</div><div class='del'>-	stats-&gt;value[IPINDISCARDS] = ((u64)m.ipInDiscards_hi &lt;&lt; 32) + m.ipInDiscards_lo;</div><div class='del'>-	stats-&gt;value[IPINDELIVERS] = ((u64)m.ipInDelivers_hi &lt;&lt; 32) + m.ipInDelivers_lo;</div><div class='del'>-	stats-&gt;value[IPOUTREQUESTS] = ((u64)m.ipOutRequests_hi &lt;&lt; 32) + m.ipOutRequests_lo;</div><div class='del'>-	stats-&gt;value[IPOUTDISCARDS] = ((u64)m.ipOutDiscards_hi &lt;&lt; 32) + m.ipOutDiscards_lo;</div><div class='del'>-	stats-&gt;value[IPOUTNOROUTES] = ((u64)m.ipOutNoRoutes_hi &lt;&lt; 32) + m.ipOutNoRoutes_lo;</div><div class='del'>-	stats-&gt;value[IPREASMTIMEOUT] = 	m.ipReasmTimeout;</div><div class='del'>-	stats-&gt;value[IPREASMREQDS] = m.ipReasmReqds;</div><div class='del'>-	stats-&gt;value[IPREASMOKS] = m.ipReasmOKs;</div><div class='del'>-	stats-&gt;value[IPREASMFAILS] = m.ipReasmFails;</div><div class='del'>-	stats-&gt;value[TCPACTIVEOPENS] =	m.tcpActiveOpens;</div><div class='del'>-	stats-&gt;value[TCPPASSIVEOPENS] =	m.tcpPassiveOpens;</div><div class='del'>-	stats-&gt;value[TCPATTEMPTFAILS] = m.tcpAttemptFails;</div><div class='del'>-	stats-&gt;value[TCPESTABRESETS] = m.tcpEstabResets;</div><div class='del'>-	stats-&gt;value[TCPCURRESTAB] = m.tcpOutRsts;</div><div class='del'>-	stats-&gt;value[TCPINSEGS] = m.tcpCurrEstab;</div><div class='del'>-	stats-&gt;value[TCPOUTSEGS] = ((u64)m.tcpInSegs_hi &lt;&lt; 32) + m.tcpInSegs_lo;</div><div class='del'>-	stats-&gt;value[TCPRETRANSSEGS] = ((u64)m.tcpOutSegs_hi &lt;&lt; 32) + m.tcpOutSegs_lo;</div><div class='del'>-	stats-&gt;value[TCPINERRS] = ((u64)m.tcpRetransSeg_hi &lt;&lt; 32) + m.tcpRetransSeg_lo,</div><div class='del'>-	stats-&gt;value[TCPOUTRSTS] = ((u64)m.tcpInErrs_hi &lt;&lt; 32) + m.tcpInErrs_lo;</div><div class='del'>-	stats-&gt;value[TCPRTOMIN] = m.tcpRtoMin;</div><div class='del'>-	stats-&gt;value[TCPRTOMAX] = m.tcpRtoMax;</div><div class='del'>-</div><div class='del'>-	return stats-&gt;num_counters;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct attribute *iwch_class_attributes[] = {</div><div class='del'>-	&amp;dev_attr_hw_rev.attr,</div><div class='del'>-	&amp;dev_attr_hca_type.attr,</div><div class='del'>-	&amp;dev_attr_board_id.attr,</div><div class='del'>-	NULL</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static const struct attribute_group iwch_attr_group = {</div><div class='del'>-	.attrs = iwch_class_attributes,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static int iwch_port_immutable(struct ib_device *ibdev, u8 port_num,</div><div class='del'>-			       struct ib_port_immutable *immutable)</div><div class='del'>-{</div><div class='del'>-	struct ib_port_attr attr;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	immutable-&gt;core_cap_flags = RDMA_CORE_PORT_IWARP;</div><div class='del'>-</div><div class='del'>-	err = ib_query_port(ibdev, port_num, &amp;attr);</div><div class='del'>-	if (err)</div><div class='del'>-		return err;</div><div class='del'>-</div><div class='del'>-	immutable-&gt;pkey_tbl_len = attr.pkey_tbl_len;</div><div class='del'>-	immutable-&gt;gid_tbl_len = attr.gid_tbl_len;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void get_dev_fw_ver_str(struct ib_device *ibdev, char *str)</div><div class='del'>-{</div><div class='del'>-	struct iwch_dev *iwch_dev = to_iwch_dev(ibdev);</div><div class='del'>-	struct ethtool_drvinfo info;</div><div class='del'>-	struct net_device *lldev = iwch_dev-&gt;rdev.t3cdev_p-&gt;lldev;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s dev 0x%p\n", __func__, iwch_dev);</div><div class='del'>-	lldev-&gt;ethtool_ops-&gt;get_drvinfo(lldev, &amp;info);</div><div class='del'>-	snprintf(str, IB_FW_VERSION_NAME_MAX, "%s", info.fw_version);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static const struct ib_device_ops iwch_dev_ops = {</div><div class='del'>-	.alloc_hw_stats	= iwch_alloc_stats,</div><div class='del'>-	.alloc_mr = iwch_alloc_mr,</div><div class='del'>-	.alloc_mw = iwch_alloc_mw,</div><div class='del'>-	.alloc_pd = iwch_allocate_pd,</div><div class='del'>-	.alloc_ucontext = iwch_alloc_ucontext,</div><div class='del'>-	.create_cq = iwch_create_cq,</div><div class='del'>-	.create_qp = iwch_create_qp,</div><div class='del'>-	.dealloc_mw = iwch_dealloc_mw,</div><div class='del'>-	.dealloc_pd = iwch_deallocate_pd,</div><div class='del'>-	.dealloc_ucontext = iwch_dealloc_ucontext,</div><div class='del'>-	.dereg_mr = iwch_dereg_mr,</div><div class='del'>-	.destroy_cq = iwch_destroy_cq,</div><div class='del'>-	.destroy_qp = iwch_destroy_qp,</div><div class='del'>-	.get_dev_fw_str = get_dev_fw_ver_str,</div><div class='del'>-	.get_dma_mr = iwch_get_dma_mr,</div><div class='del'>-	.get_hw_stats = iwch_get_mib,</div><div class='del'>-	.get_port_immutable = iwch_port_immutable,</div><div class='del'>-	.iw_accept = iwch_accept_cr,</div><div class='del'>-	.iw_add_ref = iwch_qp_add_ref,</div><div class='del'>-	.iw_connect = iwch_connect,</div><div class='del'>-	.iw_create_listen = iwch_create_listen,</div><div class='del'>-	.iw_destroy_listen = iwch_destroy_listen,</div><div class='del'>-	.iw_get_qp = iwch_get_qp,</div><div class='del'>-	.iw_reject = iwch_reject_cr,</div><div class='del'>-	.iw_rem_ref = iwch_qp_rem_ref,</div><div class='del'>-	.map_mr_sg = iwch_map_mr_sg,</div><div class='del'>-	.mmap = iwch_mmap,</div><div class='del'>-	.modify_qp = iwch_ib_modify_qp,</div><div class='del'>-	.poll_cq = iwch_poll_cq,</div><div class='del'>-	.post_recv = iwch_post_receive,</div><div class='del'>-	.post_send = iwch_post_send,</div><div class='del'>-	.query_device = iwch_query_device,</div><div class='del'>-	.query_gid = iwch_query_gid,</div><div class='del'>-	.query_pkey = iwch_query_pkey,</div><div class='del'>-	.query_port = iwch_query_port,</div><div class='del'>-	.reg_user_mr = iwch_reg_user_mr,</div><div class='del'>-	.req_notify_cq = iwch_arm_cq,</div><div class='del'>-	.resize_cq = iwch_resize_cq,</div><div class='del'>-	INIT_RDMA_OBJ_SIZE(ib_pd, iwch_pd, ibpd),</div><div class='del'>-	INIT_RDMA_OBJ_SIZE(ib_ucontext, iwch_ucontext, ibucontext),</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-int iwch_register_device(struct iwch_dev *dev)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s iwch_dev %p\n", __func__, dev);</div><div class='del'>-	memset(&amp;dev-&gt;ibdev.node_guid, 0, sizeof(dev-&gt;ibdev.node_guid));</div><div class='del'>-	memcpy(&amp;dev-&gt;ibdev.node_guid, dev-&gt;rdev.t3cdev_p-&gt;lldev-&gt;dev_addr, 6);</div><div class='del'>-	dev-&gt;ibdev.owner = THIS_MODULE;</div><div class='del'>-	dev-&gt;device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |</div><div class='del'>-				IB_DEVICE_MEM_WINDOW |</div><div class='del'>-				IB_DEVICE_MEM_MGT_EXTENSIONS;</div><div class='del'>-</div><div class='del'>-	/* cxgb3 supports STag 0. */</div><div class='del'>-	dev-&gt;ibdev.local_dma_lkey = 0;</div><div class='del'>-</div><div class='del'>-	dev-&gt;ibdev.uverbs_cmd_mask =</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_GET_CONTEXT) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_DEVICE) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_PORT) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_ALLOC_PD) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_DEALLOC_PD) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_REG_MR) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_DEREG_MR) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_CQ) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_CQ) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_QP) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_MODIFY_QP) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_POLL_CQ) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_QP) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_POST_SEND) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_POST_RECV);</div><div class='del'>-	dev-&gt;ibdev.node_type = RDMA_NODE_RNIC;</div><div class='del'>-	BUILD_BUG_ON(sizeof(IWCH_NODE_DESC) &gt; IB_DEVICE_NODE_DESC_MAX);</div><div class='del'>-	memcpy(dev-&gt;ibdev.node_desc, IWCH_NODE_DESC, sizeof(IWCH_NODE_DESC));</div><div class='del'>-	dev-&gt;ibdev.phys_port_cnt = dev-&gt;rdev.port_info.nports;</div><div class='del'>-	dev-&gt;ibdev.num_comp_vectors = 1;</div><div class='del'>-	dev-&gt;ibdev.dev.parent = &amp;dev-&gt;rdev.rnic_info.pdev-&gt;dev;</div><div class='del'>-	dev-&gt;ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION;</div><div class='del'>-</div><div class='del'>-	memcpy(dev-&gt;ibdev.iw_ifname, dev-&gt;rdev.t3cdev_p-&gt;lldev-&gt;name,</div><div class='del'>-	       sizeof(dev-&gt;ibdev.iw_ifname));</div><div class='del'>-</div><div class='del'>-	dev-&gt;ibdev.driver_id = RDMA_DRIVER_CXGB3;</div><div class='del'>-	rdma_set_device_sysfs_group(&amp;dev-&gt;ibdev, &amp;iwch_attr_group);</div><div class='del'>-	ib_set_device_ops(&amp;dev-&gt;ibdev, &amp;iwch_dev_ops);</div><div class='del'>-	return ib_register_device(&amp;dev-&gt;ibdev, "cxgb3_%d");</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void iwch_unregister_device(struct iwch_dev *dev)</div><div class='del'>-{</div><div class='del'>-	pr_debug("%s iwch_dev %p\n", __func__, dev);</div><div class='del'>-	ib_unregister_device(&amp;dev-&gt;ibdev);</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.h b/drivers/infiniband/hw/cxgb3/iwch_provider.h<br/>deleted file mode 100644<br/>index 8adbe9658935..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/iwch_provider.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch_provider.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,347 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-#ifndef __IWCH_PROVIDER_H__</div><div class='del'>-#define __IWCH_PROVIDER_H__</div><div class='del'>-</div><div class='del'>-#include &lt;linux/list.h&gt;</div><div class='del'>-#include &lt;linux/spinlock.h&gt;</div><div class='del'>-#include &lt;rdma/ib_verbs.h&gt;</div><div class='del'>-#include &lt;asm/types.h&gt;</div><div class='del'>-#include "t3cdev.h"</div><div class='del'>-#include "iwch.h"</div><div class='del'>-#include "cxio_wr.h"</div><div class='del'>-#include "cxio_hal.h"</div><div class='del'>-</div><div class='del'>-struct iwch_pd {</div><div class='del'>-	struct ib_pd ibpd;</div><div class='del'>-	u32 pdid;</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static inline struct iwch_pd *to_iwch_pd(struct ib_pd *ibpd)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibpd, struct iwch_pd, ibpd);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct tpt_attributes {</div><div class='del'>-	u32 stag;</div><div class='del'>-	u32 state:1;</div><div class='del'>-	u32 type:2;</div><div class='del'>-	u32 rsvd:1;</div><div class='del'>-	enum tpt_mem_perm perms;</div><div class='del'>-	u32 remote_invaliate_disable:1;</div><div class='del'>-	u32 zbva:1;</div><div class='del'>-	u32 mw_bind_enable:1;</div><div class='del'>-	u32 page_size:5;</div><div class='del'>-</div><div class='del'>-	u32 pdid;</div><div class='del'>-	u32 qpid;</div><div class='del'>-	u32 pbl_addr;</div><div class='del'>-	u32 len;</div><div class='del'>-	u64 va_fbo;</div><div class='del'>-	u32 pbl_size;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct iwch_mr {</div><div class='del'>-	struct ib_mr ibmr;</div><div class='del'>-	struct ib_umem *umem;</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-	u64 kva;</div><div class='del'>-	struct tpt_attributes attr;</div><div class='del'>-	u64 *pages;</div><div class='del'>-	u32 npages;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-typedef struct iwch_mw iwch_mw_handle;</div><div class='del'>-</div><div class='del'>-static inline struct iwch_mr *to_iwch_mr(struct ib_mr *ibmr)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibmr, struct iwch_mr, ibmr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct iwch_mw {</div><div class='del'>-	struct ib_mw ibmw;</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-	u64 kva;</div><div class='del'>-	struct tpt_attributes attr;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static inline struct iwch_mw *to_iwch_mw(struct ib_mw *ibmw)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibmw, struct iwch_mw, ibmw);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct iwch_cq {</div><div class='del'>-	struct ib_cq ibcq;</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-	struct t3_cq cq;</div><div class='del'>-	spinlock_t lock;</div><div class='del'>-	spinlock_t comp_handler_lock;</div><div class='del'>-	atomic_t refcnt;</div><div class='del'>-	wait_queue_head_t wait;</div><div class='del'>-	u32 __user *user_rptr_addr;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static inline struct iwch_cq *to_iwch_cq(struct ib_cq *ibcq)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibcq, struct iwch_cq, ibcq);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-enum IWCH_QP_FLAGS {</div><div class='del'>-	QP_QUIESCED = 0x01</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct iwch_mpa_attributes {</div><div class='del'>-	u8 initiator;</div><div class='del'>-	u8 recv_marker_enabled;</div><div class='del'>-	u8 xmit_marker_enabled;	/* iWARP: enable inbound Read Resp. */</div><div class='del'>-	u8 crc_enabled;</div><div class='del'>-	u8 version;	/* 0 or 1 */</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct iwch_qp_attributes {</div><div class='del'>-	u32 scq;</div><div class='del'>-	u32 rcq;</div><div class='del'>-	u32 sq_num_entries;</div><div class='del'>-	u32 rq_num_entries;</div><div class='del'>-	u32 sq_max_sges;</div><div class='del'>-	u32 sq_max_sges_rdma_write;</div><div class='del'>-	u32 rq_max_sges;</div><div class='del'>-	u32 state;</div><div class='del'>-	u8 enable_rdma_read;</div><div class='del'>-	u8 enable_rdma_write;	/* enable inbound Read Resp. */</div><div class='del'>-	u8 enable_bind;</div><div class='del'>-	u8 enable_mmid0_fastreg;	/* Enable STAG0 + Fast-register */</div><div class='del'>-	/*</div><div class='del'>-	 * Next QP state. If specify the current state, only the</div><div class='del'>-	 * QP attributes will be modified.</div><div class='del'>-	 */</div><div class='del'>-	u32 max_ord;</div><div class='del'>-	u32 max_ird;</div><div class='del'>-	u32 pd;	/* IN */</div><div class='del'>-	u32 next_state;</div><div class='del'>-	char terminate_buffer[52];</div><div class='del'>-	u32 terminate_msg_len;</div><div class='del'>-	u8 is_terminate_local;</div><div class='del'>-	struct iwch_mpa_attributes mpa_attr;	/* IN-OUT */</div><div class='del'>-	struct iwch_ep *llp_stream_handle;</div><div class='del'>-	char *stream_msg_buf;	/* Last stream msg. before Idle -&gt; RTS */</div><div class='del'>-	u32 stream_msg_buf_len;	/* Only on Idle -&gt; RTS */</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct iwch_qp {</div><div class='del'>-	struct ib_qp ibqp;</div><div class='del'>-	struct iwch_dev *rhp;</div><div class='del'>-	struct iwch_ep *ep;</div><div class='del'>-	struct iwch_qp_attributes attr;</div><div class='del'>-	struct t3_wq wq;</div><div class='del'>-	spinlock_t lock;</div><div class='del'>-	atomic_t refcnt;</div><div class='del'>-	wait_queue_head_t wait;</div><div class='del'>-	enum IWCH_QP_FLAGS flags;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static inline int qp_quiesced(struct iwch_qp *qhp)</div><div class='del'>-{</div><div class='del'>-	return qhp-&gt;flags &amp; QP_QUIESCED;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline struct iwch_qp *to_iwch_qp(struct ib_qp *ibqp)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibqp, struct iwch_qp, ibqp);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void iwch_qp_add_ref(struct ib_qp *qp);</div><div class='del'>-void iwch_qp_rem_ref(struct ib_qp *qp);</div><div class='del'>-</div><div class='del'>-struct iwch_ucontext {</div><div class='del'>-	struct ib_ucontext ibucontext;</div><div class='del'>-	struct cxio_ucontext uctx;</div><div class='del'>-	u32 key;</div><div class='del'>-	spinlock_t mmap_lock;</div><div class='del'>-	struct list_head mmaps;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static inline struct iwch_ucontext *to_iwch_ucontext(struct ib_ucontext *c)</div><div class='del'>-{</div><div class='del'>-	return container_of(c, struct iwch_ucontext, ibucontext);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct iwch_mm_entry {</div><div class='del'>-	struct list_head entry;</div><div class='del'>-	u64 addr;</div><div class='del'>-	u32 key;</div><div class='del'>-	unsigned len;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static inline struct iwch_mm_entry *remove_mmap(struct iwch_ucontext *ucontext,</div><div class='del'>-						u32 key, unsigned len)</div><div class='del'>-{</div><div class='del'>-	struct list_head *pos, *nxt;</div><div class='del'>-	struct iwch_mm_entry *mm;</div><div class='del'>-</div><div class='del'>-	spin_lock(&amp;ucontext-&gt;mmap_lock);</div><div class='del'>-	list_for_each_safe(pos, nxt, &amp;ucontext-&gt;mmaps) {</div><div class='del'>-</div><div class='del'>-		mm = list_entry(pos, struct iwch_mm_entry, entry);</div><div class='del'>-		if (mm-&gt;key == key &amp;&amp; mm-&gt;len == len) {</div><div class='del'>-			list_del_init(&amp;mm-&gt;entry);</div><div class='del'>-			spin_unlock(&amp;ucontext-&gt;mmap_lock);</div><div class='del'>-			pr_debug("%s key 0x%x addr 0x%llx len %d\n",</div><div class='del'>-				 __func__, key,</div><div class='del'>-				 (unsigned long long)mm-&gt;addr, mm-&gt;len);</div><div class='del'>-			return mm;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	spin_unlock(&amp;ucontext-&gt;mmap_lock);</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void insert_mmap(struct iwch_ucontext *ucontext,</div><div class='del'>-			       struct iwch_mm_entry *mm)</div><div class='del'>-{</div><div class='del'>-	spin_lock(&amp;ucontext-&gt;mmap_lock);</div><div class='del'>-	pr_debug("%s key 0x%x addr 0x%llx len %d\n",</div><div class='del'>-		 __func__, mm-&gt;key, (unsigned long long)mm-&gt;addr, mm-&gt;len);</div><div class='del'>-	list_add_tail(&amp;mm-&gt;entry, &amp;ucontext-&gt;mmaps);</div><div class='del'>-	spin_unlock(&amp;ucontext-&gt;mmap_lock);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-enum iwch_qp_attr_mask {</div><div class='del'>-	IWCH_QP_ATTR_NEXT_STATE = 1 &lt;&lt; 0,</div><div class='del'>-	IWCH_QP_ATTR_ENABLE_RDMA_READ = 1 &lt;&lt; 7,</div><div class='del'>-	IWCH_QP_ATTR_ENABLE_RDMA_WRITE = 1 &lt;&lt; 8,</div><div class='del'>-	IWCH_QP_ATTR_ENABLE_RDMA_BIND = 1 &lt;&lt; 9,</div><div class='del'>-	IWCH_QP_ATTR_MAX_ORD = 1 &lt;&lt; 11,</div><div class='del'>-	IWCH_QP_ATTR_MAX_IRD = 1 &lt;&lt; 12,</div><div class='del'>-	IWCH_QP_ATTR_LLP_STREAM_HANDLE = 1 &lt;&lt; 22,</div><div class='del'>-	IWCH_QP_ATTR_STREAM_MSG_BUFFER = 1 &lt;&lt; 23,</div><div class='del'>-	IWCH_QP_ATTR_MPA_ATTR = 1 &lt;&lt; 24,</div><div class='del'>-	IWCH_QP_ATTR_QP_CONTEXT_ACTIVATE = 1 &lt;&lt; 25,</div><div class='del'>-	IWCH_QP_ATTR_VALID_MODIFY = (IWCH_QP_ATTR_ENABLE_RDMA_READ |</div><div class='del'>-				     IWCH_QP_ATTR_ENABLE_RDMA_WRITE |</div><div class='del'>-				     IWCH_QP_ATTR_MAX_ORD |</div><div class='del'>-				     IWCH_QP_ATTR_MAX_IRD |</div><div class='del'>-				     IWCH_QP_ATTR_LLP_STREAM_HANDLE |</div><div class='del'>-				     IWCH_QP_ATTR_STREAM_MSG_BUFFER |</div><div class='del'>-				     IWCH_QP_ATTR_MPA_ATTR |</div><div class='del'>-				     IWCH_QP_ATTR_QP_CONTEXT_ACTIVATE)</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-int iwch_modify_qp(struct iwch_dev *rhp,</div><div class='del'>-				struct iwch_qp *qhp,</div><div class='del'>-				enum iwch_qp_attr_mask mask,</div><div class='del'>-				struct iwch_qp_attributes *attrs,</div><div class='del'>-				int internal);</div><div class='del'>-</div><div class='del'>-enum iwch_qp_state {</div><div class='del'>-	IWCH_QP_STATE_IDLE,</div><div class='del'>-	IWCH_QP_STATE_RTS,</div><div class='del'>-	IWCH_QP_STATE_ERROR,</div><div class='del'>-	IWCH_QP_STATE_TERMINATE,</div><div class='del'>-	IWCH_QP_STATE_CLOSING,</div><div class='del'>-	IWCH_QP_STATE_TOT</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static inline int iwch_convert_state(enum ib_qp_state ib_state)</div><div class='del'>-{</div><div class='del'>-	switch (ib_state) {</div><div class='del'>-	case IB_QPS_RESET:</div><div class='del'>-	case IB_QPS_INIT:</div><div class='del'>-		return IWCH_QP_STATE_IDLE;</div><div class='del'>-	case IB_QPS_RTS:</div><div class='del'>-		return IWCH_QP_STATE_RTS;</div><div class='del'>-	case IB_QPS_SQD:</div><div class='del'>-		return IWCH_QP_STATE_CLOSING;</div><div class='del'>-	case IB_QPS_SQE:</div><div class='del'>-		return IWCH_QP_STATE_TERMINATE;</div><div class='del'>-	case IB_QPS_ERR:</div><div class='del'>-		return IWCH_QP_STATE_ERROR;</div><div class='del'>-	default:</div><div class='del'>-		return -1;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u32 iwch_ib_to_tpt_access(int acc)</div><div class='del'>-{</div><div class='del'>-	return (acc &amp; IB_ACCESS_REMOTE_WRITE ? TPT_REMOTE_WRITE : 0) |</div><div class='del'>-	       (acc &amp; IB_ACCESS_REMOTE_READ ? TPT_REMOTE_READ : 0) |</div><div class='del'>-	       (acc &amp; IB_ACCESS_LOCAL_WRITE ? TPT_LOCAL_WRITE : 0) |</div><div class='del'>-	       (acc &amp; IB_ACCESS_MW_BIND ? TPT_MW_BIND : 0) |</div><div class='del'>-	       TPT_LOCAL_READ;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u32 iwch_ib_to_tpt_bind_access(int acc)</div><div class='del'>-{</div><div class='del'>-	return (acc &amp; IB_ACCESS_REMOTE_WRITE ? TPT_REMOTE_WRITE : 0) |</div><div class='del'>-	       (acc &amp; IB_ACCESS_REMOTE_READ ? TPT_REMOTE_READ : 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-enum iwch_mmid_state {</div><div class='del'>-	IWCH_STAG_STATE_VALID,</div><div class='del'>-	IWCH_STAG_STATE_INVALID</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum iwch_qp_query_flags {</div><div class='del'>-	IWCH_QP_QUERY_CONTEXT_NONE = 0x0,	/* No ctx; Only attrs */</div><div class='del'>-	IWCH_QP_QUERY_CONTEXT_GET = 0x1,	/* Get ctx + attrs */</div><div class='del'>-	IWCH_QP_QUERY_CONTEXT_SUSPEND = 0x2,	/* Not Supported */</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Quiesce QP context; Consumer</div><div class='del'>-	 * will NOT replay outstanding WR</div><div class='del'>-	 */</div><div class='del'>-	IWCH_QP_QUERY_CONTEXT_QUIESCE = 0x4,</div><div class='del'>-	IWCH_QP_QUERY_CONTEXT_REMOVE = 0x8,</div><div class='del'>-	IWCH_QP_QUERY_TEST_USERWRITE = 0x32	/* Test special */</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-u16 iwch_rqes_posted(struct iwch_qp *qhp);</div><div class='del'>-int iwch_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,</div><div class='del'>-		   const struct ib_send_wr **bad_wr);</div><div class='del'>-int iwch_post_receive(struct ib_qp *ibqp, const struct ib_recv_wr *wr,</div><div class='del'>-		      const struct ib_recv_wr **bad_wr);</div><div class='del'>-int iwch_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);</div><div class='del'>-int iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg);</div><div class='del'>-int iwch_post_zb_read(struct iwch_ep *ep);</div><div class='del'>-int iwch_register_device(struct iwch_dev *dev);</div><div class='del'>-void iwch_unregister_device(struct iwch_dev *dev);</div><div class='del'>-void stop_read_rep_timer(struct iwch_qp *qhp);</div><div class='del'>-int iwch_register_mem(struct iwch_dev *rhp, struct iwch_pd *php,</div><div class='del'>-		      struct iwch_mr *mhp, int shift);</div><div class='del'>-int iwch_alloc_pbl(struct iwch_mr *mhp, int npages);</div><div class='del'>-void iwch_free_pbl(struct iwch_mr *mhp);</div><div class='del'>-int iwch_write_pbl(struct iwch_mr *mhp, __be64 *pages, int npages, int offset);</div><div class='del'>-</div><div class='del'>-#define IWCH_NODE_DESC "cxgb3 Chelsio Communications"</div><div class='del'>-</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/iwch_qp.c b/drivers/infiniband/hw/cxgb3/iwch_qp.c<br/>deleted file mode 100644<br/>index c649faad63f9..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/iwch_qp.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/iwch_qp.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,1082 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-#include &lt;linux/sched.h&gt;</div><div class='del'>-#include &lt;linux/gfp.h&gt;</div><div class='del'>-#include "iwch_provider.h"</div><div class='del'>-#include "iwch.h"</div><div class='del'>-#include "iwch_cm.h"</div><div class='del'>-#include "cxio_hal.h"</div><div class='del'>-#include "cxio_resource.h"</div><div class='del'>-</div><div class='del'>-#define NO_SUPPORT -1</div><div class='del'>-</div><div class='del'>-static int build_rdma_send(union t3_wr *wqe, const struct ib_send_wr *wr,</div><div class='del'>-			   u8 *flit_cnt)</div><div class='del'>-{</div><div class='del'>-	int i;</div><div class='del'>-	u32 plen;</div><div class='del'>-</div><div class='del'>-	switch (wr-&gt;opcode) {</div><div class='del'>-	case IB_WR_SEND:</div><div class='del'>-		if (wr-&gt;send_flags &amp; IB_SEND_SOLICITED)</div><div class='del'>-			wqe-&gt;send.rdmaop = T3_SEND_WITH_SE;</div><div class='del'>-		else</div><div class='del'>-			wqe-&gt;send.rdmaop = T3_SEND;</div><div class='del'>-		wqe-&gt;send.rem_stag = 0;</div><div class='del'>-		break;</div><div class='del'>-	case IB_WR_SEND_WITH_INV:</div><div class='del'>-		if (wr-&gt;send_flags &amp; IB_SEND_SOLICITED)</div><div class='del'>-			wqe-&gt;send.rdmaop = T3_SEND_WITH_SE_INV;</div><div class='del'>-		else</div><div class='del'>-			wqe-&gt;send.rdmaop = T3_SEND_WITH_INV;</div><div class='del'>-		wqe-&gt;send.rem_stag = cpu_to_be32(wr-&gt;ex.invalidate_rkey);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-	if (wr-&gt;num_sge &gt; T3_MAX_SGE)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	wqe-&gt;send.reserved[0] = 0;</div><div class='del'>-	wqe-&gt;send.reserved[1] = 0;</div><div class='del'>-	wqe-&gt;send.reserved[2] = 0;</div><div class='del'>-	plen = 0;</div><div class='del'>-	for (i = 0; i &lt; wr-&gt;num_sge; i++) {</div><div class='del'>-		if ((plen + wr-&gt;sg_list[i].length) &lt; plen)</div><div class='del'>-			return -EMSGSIZE;</div><div class='del'>-</div><div class='del'>-		plen += wr-&gt;sg_list[i].length;</div><div class='del'>-		wqe-&gt;send.sgl[i].stag = cpu_to_be32(wr-&gt;sg_list[i].lkey);</div><div class='del'>-		wqe-&gt;send.sgl[i].len = cpu_to_be32(wr-&gt;sg_list[i].length);</div><div class='del'>-		wqe-&gt;send.sgl[i].to = cpu_to_be64(wr-&gt;sg_list[i].addr);</div><div class='del'>-	}</div><div class='del'>-	wqe-&gt;send.num_sgle = cpu_to_be32(wr-&gt;num_sge);</div><div class='del'>-	*flit_cnt = 4 + ((wr-&gt;num_sge) &lt;&lt; 1);</div><div class='del'>-	wqe-&gt;send.plen = cpu_to_be32(plen);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int build_rdma_write(union t3_wr *wqe, const struct ib_send_wr *wr,</div><div class='del'>-			    u8 *flit_cnt)</div><div class='del'>-{</div><div class='del'>-	int i;</div><div class='del'>-	u32 plen;</div><div class='del'>-	if (wr-&gt;num_sge &gt; T3_MAX_SGE)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	wqe-&gt;write.rdmaop = T3_RDMA_WRITE;</div><div class='del'>-	wqe-&gt;write.reserved[0] = 0;</div><div class='del'>-	wqe-&gt;write.reserved[1] = 0;</div><div class='del'>-	wqe-&gt;write.reserved[2] = 0;</div><div class='del'>-	wqe-&gt;write.stag_sink = cpu_to_be32(rdma_wr(wr)-&gt;rkey);</div><div class='del'>-	wqe-&gt;write.to_sink = cpu_to_be64(rdma_wr(wr)-&gt;remote_addr);</div><div class='del'>-</div><div class='del'>-	if (wr-&gt;opcode == IB_WR_RDMA_WRITE_WITH_IMM) {</div><div class='del'>-		plen = 4;</div><div class='del'>-		wqe-&gt;write.sgl[0].stag = wr-&gt;ex.imm_data;</div><div class='del'>-		wqe-&gt;write.sgl[0].len = cpu_to_be32(0);</div><div class='del'>-		wqe-&gt;write.num_sgle = cpu_to_be32(0);</div><div class='del'>-		*flit_cnt = 6;</div><div class='del'>-	} else {</div><div class='del'>-		plen = 0;</div><div class='del'>-		for (i = 0; i &lt; wr-&gt;num_sge; i++) {</div><div class='del'>-			if ((plen + wr-&gt;sg_list[i].length) &lt; plen) {</div><div class='del'>-				return -EMSGSIZE;</div><div class='del'>-			}</div><div class='del'>-			plen += wr-&gt;sg_list[i].length;</div><div class='del'>-			wqe-&gt;write.sgl[i].stag =</div><div class='del'>-			    cpu_to_be32(wr-&gt;sg_list[i].lkey);</div><div class='del'>-			wqe-&gt;write.sgl[i].len =</div><div class='del'>-			    cpu_to_be32(wr-&gt;sg_list[i].length);</div><div class='del'>-			wqe-&gt;write.sgl[i].to =</div><div class='del'>-			    cpu_to_be64(wr-&gt;sg_list[i].addr);</div><div class='del'>-		}</div><div class='del'>-		wqe-&gt;write.num_sgle = cpu_to_be32(wr-&gt;num_sge);</div><div class='del'>-		*flit_cnt = 5 + ((wr-&gt;num_sge) &lt;&lt; 1);</div><div class='del'>-	}</div><div class='del'>-	wqe-&gt;write.plen = cpu_to_be32(plen);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int build_rdma_read(union t3_wr *wqe, const struct ib_send_wr *wr,</div><div class='del'>-			   u8 *flit_cnt)</div><div class='del'>-{</div><div class='del'>-	if (wr-&gt;num_sge &gt; 1)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	wqe-&gt;read.rdmaop = T3_READ_REQ;</div><div class='del'>-	if (wr-&gt;opcode == IB_WR_RDMA_READ_WITH_INV)</div><div class='del'>-		wqe-&gt;read.local_inv = 1;</div><div class='del'>-	else</div><div class='del'>-		wqe-&gt;read.local_inv = 0;</div><div class='del'>-	wqe-&gt;read.reserved[0] = 0;</div><div class='del'>-	wqe-&gt;read.reserved[1] = 0;</div><div class='del'>-	wqe-&gt;read.rem_stag = cpu_to_be32(rdma_wr(wr)-&gt;rkey);</div><div class='del'>-	wqe-&gt;read.rem_to = cpu_to_be64(rdma_wr(wr)-&gt;remote_addr);</div><div class='del'>-	wqe-&gt;read.local_stag = cpu_to_be32(wr-&gt;sg_list[0].lkey);</div><div class='del'>-	wqe-&gt;read.local_len = cpu_to_be32(wr-&gt;sg_list[0].length);</div><div class='del'>-	wqe-&gt;read.local_to = cpu_to_be64(wr-&gt;sg_list[0].addr);</div><div class='del'>-	*flit_cnt = sizeof(struct t3_rdma_read_wr) &gt;&gt; 3;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int build_memreg(union t3_wr *wqe, const struct ib_reg_wr *wr,</div><div class='del'>-			u8 *flit_cnt, int *wr_cnt, struct t3_wq *wq)</div><div class='del'>-{</div><div class='del'>-	struct iwch_mr *mhp = to_iwch_mr(wr-&gt;mr);</div><div class='del'>-	int i;</div><div class='del'>-	__be64 *p;</div><div class='del'>-</div><div class='del'>-	if (mhp-&gt;npages &gt; T3_MAX_FASTREG_DEPTH)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	*wr_cnt = 1;</div><div class='del'>-	wqe-&gt;fastreg.stag = cpu_to_be32(wr-&gt;key);</div><div class='del'>-	wqe-&gt;fastreg.len = cpu_to_be32(mhp-&gt;ibmr.length);</div><div class='del'>-	wqe-&gt;fastreg.va_base_hi = cpu_to_be32(mhp-&gt;ibmr.iova &gt;&gt; 32);</div><div class='del'>-	wqe-&gt;fastreg.va_base_lo_fbo =</div><div class='del'>-				cpu_to_be32(mhp-&gt;ibmr.iova &amp; 0xffffffff);</div><div class='del'>-	wqe-&gt;fastreg.page_type_perms = cpu_to_be32(</div><div class='del'>-		V_FR_PAGE_COUNT(mhp-&gt;npages) |</div><div class='del'>-		V_FR_PAGE_SIZE(ilog2(wr-&gt;mr-&gt;page_size) - 12) |</div><div class='del'>-		V_FR_TYPE(TPT_VATO) |</div><div class='del'>-		V_FR_PERMS(iwch_ib_to_tpt_access(wr-&gt;access)));</div><div class='del'>-	p = &amp;wqe-&gt;fastreg.pbl_addrs[0];</div><div class='del'>-	for (i = 0; i &lt; mhp-&gt;npages; i++, p++) {</div><div class='del'>-</div><div class='del'>-		/* If we need a 2nd WR, then set it up */</div><div class='del'>-		if (i == T3_MAX_FASTREG_FRAG) {</div><div class='del'>-			*wr_cnt = 2;</div><div class='del'>-			wqe = (union t3_wr *)(wq-&gt;queue +</div><div class='del'>-				Q_PTR2IDX((wq-&gt;wptr+1), wq-&gt;size_log2));</div><div class='del'>-			build_fw_riwrh((void *)wqe, T3_WR_FASTREG, 0,</div><div class='del'>-			       Q_GENBIT(wq-&gt;wptr + 1, wq-&gt;size_log2),</div><div class='del'>-			       0, 1 + mhp-&gt;npages - T3_MAX_FASTREG_FRAG,</div><div class='del'>-			       T3_EOP);</div><div class='del'>-</div><div class='del'>-			p = &amp;wqe-&gt;pbl_frag.pbl_addrs[0];</div><div class='del'>-		}</div><div class='del'>-		*p = cpu_to_be64((u64)mhp-&gt;pages[i]);</div><div class='del'>-	}</div><div class='del'>-	*flit_cnt = 5 + mhp-&gt;npages;</div><div class='del'>-	if (*flit_cnt &gt; 15)</div><div class='del'>-		*flit_cnt = 15;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int build_inv_stag(union t3_wr *wqe, const struct ib_send_wr *wr,</div><div class='del'>-			  u8 *flit_cnt)</div><div class='del'>-{</div><div class='del'>-	wqe-&gt;local_inv.stag = cpu_to_be32(wr-&gt;ex.invalidate_rkey);</div><div class='del'>-	wqe-&gt;local_inv.reserved = 0;</div><div class='del'>-	*flit_cnt = sizeof(struct t3_local_inv_wr) &gt;&gt; 3;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int iwch_sgl2pbl_map(struct iwch_dev *rhp, struct ib_sge *sg_list,</div><div class='del'>-			    u32 num_sgle, u32 * pbl_addr, u8 * page_size)</div><div class='del'>-{</div><div class='del'>-	int i;</div><div class='del'>-	struct iwch_mr *mhp;</div><div class='del'>-	u64 offset;</div><div class='del'>-	for (i = 0; i &lt; num_sgle; i++) {</div><div class='del'>-</div><div class='del'>-		mhp = get_mhp(rhp, (sg_list[i].lkey) &gt;&gt; 8);</div><div class='del'>-		if (!mhp) {</div><div class='del'>-			pr_debug("%s %d\n", __func__, __LINE__);</div><div class='del'>-			return -EIO;</div><div class='del'>-		}</div><div class='del'>-		if (!mhp-&gt;attr.state) {</div><div class='del'>-			pr_debug("%s %d\n", __func__, __LINE__);</div><div class='del'>-			return -EIO;</div><div class='del'>-		}</div><div class='del'>-		if (mhp-&gt;attr.zbva) {</div><div class='del'>-			pr_debug("%s %d\n", __func__, __LINE__);</div><div class='del'>-			return -EIO;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (sg_list[i].addr &lt; mhp-&gt;attr.va_fbo) {</div><div class='del'>-			pr_debug("%s %d\n", __func__, __LINE__);</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-		if (sg_list[i].addr + ((u64) sg_list[i].length) &lt;</div><div class='del'>-		    sg_list[i].addr) {</div><div class='del'>-			pr_debug("%s %d\n", __func__, __LINE__);</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-		if (sg_list[i].addr + ((u64) sg_list[i].length) &gt;</div><div class='del'>-		    mhp-&gt;attr.va_fbo + ((u64) mhp-&gt;attr.len)) {</div><div class='del'>-			pr_debug("%s %d\n", __func__, __LINE__);</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-		offset = sg_list[i].addr - mhp-&gt;attr.va_fbo;</div><div class='del'>-		offset += mhp-&gt;attr.va_fbo &amp;</div><div class='del'>-			  ((1UL &lt;&lt; (12 + mhp-&gt;attr.page_size)) - 1);</div><div class='del'>-		pbl_addr[i] = ((mhp-&gt;attr.pbl_addr -</div><div class='del'>-			        rhp-&gt;rdev.rnic_info.pbl_base) &gt;&gt; 3) +</div><div class='del'>-			      (offset &gt;&gt; (12 + mhp-&gt;attr.page_size));</div><div class='del'>-		page_size[i] = mhp-&gt;attr.page_size;</div><div class='del'>-	}</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int build_rdma_recv(struct iwch_qp *qhp, union t3_wr *wqe,</div><div class='del'>-			   const struct ib_recv_wr *wr)</div><div class='del'>-{</div><div class='del'>-	int i, err = 0;</div><div class='del'>-	u32 pbl_addr[T3_MAX_SGE];</div><div class='del'>-	u8 page_size[T3_MAX_SGE];</div><div class='del'>-</div><div class='del'>-	err = iwch_sgl2pbl_map(qhp-&gt;rhp, wr-&gt;sg_list, wr-&gt;num_sge, pbl_addr,</div><div class='del'>-			       page_size);</div><div class='del'>-	if (err)</div><div class='del'>-		return err;</div><div class='del'>-	wqe-&gt;recv.pagesz[0] = page_size[0];</div><div class='del'>-	wqe-&gt;recv.pagesz[1] = page_size[1];</div><div class='del'>-	wqe-&gt;recv.pagesz[2] = page_size[2];</div><div class='del'>-	wqe-&gt;recv.pagesz[3] = page_size[3];</div><div class='del'>-	wqe-&gt;recv.num_sgle = cpu_to_be32(wr-&gt;num_sge);</div><div class='del'>-	for (i = 0; i &lt; wr-&gt;num_sge; i++) {</div><div class='del'>-		wqe-&gt;recv.sgl[i].stag = cpu_to_be32(wr-&gt;sg_list[i].lkey);</div><div class='del'>-		wqe-&gt;recv.sgl[i].len = cpu_to_be32(wr-&gt;sg_list[i].length);</div><div class='del'>-</div><div class='del'>-		/* to in the WQE == the offset into the page */</div><div class='del'>-		wqe-&gt;recv.sgl[i].to = cpu_to_be64(((u32)wr-&gt;sg_list[i].addr) &amp;</div><div class='del'>-				((1UL &lt;&lt; (12 + page_size[i])) - 1));</div><div class='del'>-</div><div class='del'>-		/* pbl_addr is the adapters address in the PBL */</div><div class='del'>-		wqe-&gt;recv.pbl_addr[i] = cpu_to_be32(pbl_addr[i]);</div><div class='del'>-	}</div><div class='del'>-	for (; i &lt; T3_MAX_SGE; i++) {</div><div class='del'>-		wqe-&gt;recv.sgl[i].stag = 0;</div><div class='del'>-		wqe-&gt;recv.sgl[i].len = 0;</div><div class='del'>-		wqe-&gt;recv.sgl[i].to = 0;</div><div class='del'>-		wqe-&gt;recv.pbl_addr[i] = 0;</div><div class='del'>-	}</div><div class='del'>-	qhp-&gt;wq.rq[Q_PTR2IDX(qhp-&gt;wq.rq_wptr,</div><div class='del'>-			     qhp-&gt;wq.rq_size_log2)].wr_id = wr-&gt;wr_id;</div><div class='del'>-	qhp-&gt;wq.rq[Q_PTR2IDX(qhp-&gt;wq.rq_wptr,</div><div class='del'>-			     qhp-&gt;wq.rq_size_log2)].pbl_addr = 0;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int build_zero_stag_recv(struct iwch_qp *qhp, union t3_wr *wqe,</div><div class='del'>-				const struct ib_recv_wr *wr)</div><div class='del'>-{</div><div class='del'>-	int i;</div><div class='del'>-	u32 pbl_addr;</div><div class='del'>-	u32 pbl_offset;</div><div class='del'>-</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * The T3 HW requires the PBL in the HW recv descriptor to reference</div><div class='del'>-	 * a PBL entry.  So we allocate the max needed PBL memory here and pass</div><div class='del'>-	 * it to the uP in the recv WR.  The uP will build the PBL and setup</div><div class='del'>-	 * the HW recv descriptor.</div><div class='del'>-	 */</div><div class='del'>-	pbl_addr = cxio_hal_pblpool_alloc(&amp;qhp-&gt;rhp-&gt;rdev, T3_STAG0_PBL_SIZE);</div><div class='del'>-	if (!pbl_addr)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Compute the 8B aligned offset.</div><div class='del'>-	 */</div><div class='del'>-	pbl_offset = (pbl_addr - qhp-&gt;rhp-&gt;rdev.rnic_info.pbl_base) &gt;&gt; 3;</div><div class='del'>-</div><div class='del'>-	wqe-&gt;recv.num_sgle = cpu_to_be32(wr-&gt;num_sge);</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; wr-&gt;num_sge; i++) {</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * Use a 128MB page size. This and an imposed 128MB</div><div class='del'>-		 * sge length limit allows us to require only a 2-entry HW</div><div class='del'>-		 * PBL for each SGE.  This restriction is acceptable since</div><div class='del'>-		 * since it is not possible to allocate 128MB of contiguous</div><div class='del'>-		 * DMA coherent memory!</div><div class='del'>-		 */</div><div class='del'>-		if (wr-&gt;sg_list[i].length &gt; T3_STAG0_MAX_PBE_LEN)</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		wqe-&gt;recv.pagesz[i] = T3_STAG0_PAGE_SHIFT;</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * T3 restricts a recv to all zero-stag or all non-zero-stag.</div><div class='del'>-		 */</div><div class='del'>-		if (wr-&gt;sg_list[i].lkey != 0)</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		wqe-&gt;recv.sgl[i].stag = 0;</div><div class='del'>-		wqe-&gt;recv.sgl[i].len = cpu_to_be32(wr-&gt;sg_list[i].length);</div><div class='del'>-		wqe-&gt;recv.sgl[i].to = cpu_to_be64(wr-&gt;sg_list[i].addr);</div><div class='del'>-		wqe-&gt;recv.pbl_addr[i] = cpu_to_be32(pbl_offset);</div><div class='del'>-		pbl_offset += 2;</div><div class='del'>-	}</div><div class='del'>-	for (; i &lt; T3_MAX_SGE; i++) {</div><div class='del'>-		wqe-&gt;recv.pagesz[i] = 0;</div><div class='del'>-		wqe-&gt;recv.sgl[i].stag = 0;</div><div class='del'>-		wqe-&gt;recv.sgl[i].len = 0;</div><div class='del'>-		wqe-&gt;recv.sgl[i].to = 0;</div><div class='del'>-		wqe-&gt;recv.pbl_addr[i] = 0;</div><div class='del'>-	}</div><div class='del'>-	qhp-&gt;wq.rq[Q_PTR2IDX(qhp-&gt;wq.rq_wptr,</div><div class='del'>-			     qhp-&gt;wq.rq_size_log2)].wr_id = wr-&gt;wr_id;</div><div class='del'>-	qhp-&gt;wq.rq[Q_PTR2IDX(qhp-&gt;wq.rq_wptr,</div><div class='del'>-			     qhp-&gt;wq.rq_size_log2)].pbl_addr = pbl_addr;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,</div><div class='del'>-		   const struct ib_send_wr **bad_wr)</div><div class='del'>-{</div><div class='del'>-	int err = 0;</div><div class='del'>-	u8 uninitialized_var(t3_wr_flit_cnt);</div><div class='del'>-	enum t3_wr_opcode t3_wr_opcode = 0;</div><div class='del'>-	enum t3_wr_flags t3_wr_flags;</div><div class='del'>-	struct iwch_qp *qhp;</div><div class='del'>-	u32 idx;</div><div class='del'>-	union t3_wr *wqe;</div><div class='del'>-	u32 num_wrs;</div><div class='del'>-	unsigned long flag;</div><div class='del'>-	struct t3_swsq *sqp;</div><div class='del'>-	int wr_cnt = 1;</div><div class='del'>-</div><div class='del'>-	qhp = to_iwch_qp(ibqp);</div><div class='del'>-	spin_lock_irqsave(&amp;qhp-&gt;lock, flag);</div><div class='del'>-	if (qhp-&gt;attr.state &gt; IWCH_QP_STATE_RTS) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;qhp-&gt;lock, flag);</div><div class='del'>-		err = -EINVAL;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-	num_wrs = Q_FREECNT(qhp-&gt;wq.sq_rptr, qhp-&gt;wq.sq_wptr,</div><div class='del'>-		  qhp-&gt;wq.sq_size_log2);</div><div class='del'>-	if (num_wrs == 0) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;qhp-&gt;lock, flag);</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-	while (wr) {</div><div class='del'>-		if (num_wrs == 0) {</div><div class='del'>-			err = -ENOMEM;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		idx = Q_PTR2IDX(qhp-&gt;wq.wptr, qhp-&gt;wq.size_log2);</div><div class='del'>-		wqe = (union t3_wr *) (qhp-&gt;wq.queue + idx);</div><div class='del'>-		t3_wr_flags = 0;</div><div class='del'>-		if (wr-&gt;send_flags &amp; IB_SEND_SOLICITED)</div><div class='del'>-			t3_wr_flags |= T3_SOLICITED_EVENT_FLAG;</div><div class='del'>-		if (wr-&gt;send_flags &amp; IB_SEND_SIGNALED)</div><div class='del'>-			t3_wr_flags |= T3_COMPLETION_FLAG;</div><div class='del'>-		sqp = qhp-&gt;wq.sq +</div><div class='del'>-		      Q_PTR2IDX(qhp-&gt;wq.sq_wptr, qhp-&gt;wq.sq_size_log2);</div><div class='del'>-		switch (wr-&gt;opcode) {</div><div class='del'>-		case IB_WR_SEND:</div><div class='del'>-		case IB_WR_SEND_WITH_INV:</div><div class='del'>-			if (wr-&gt;send_flags &amp; IB_SEND_FENCE)</div><div class='del'>-				t3_wr_flags |= T3_READ_FENCE_FLAG;</div><div class='del'>-			t3_wr_opcode = T3_WR_SEND;</div><div class='del'>-			err = build_rdma_send(wqe, wr, &amp;t3_wr_flit_cnt);</div><div class='del'>-			break;</div><div class='del'>-		case IB_WR_RDMA_WRITE:</div><div class='del'>-		case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='del'>-			t3_wr_opcode = T3_WR_WRITE;</div><div class='del'>-			err = build_rdma_write(wqe, wr, &amp;t3_wr_flit_cnt);</div><div class='del'>-			break;</div><div class='del'>-		case IB_WR_RDMA_READ:</div><div class='del'>-		case IB_WR_RDMA_READ_WITH_INV:</div><div class='del'>-			t3_wr_opcode = T3_WR_READ;</div><div class='del'>-			t3_wr_flags = 0; /* T3 reads are always signaled */</div><div class='del'>-			err = build_rdma_read(wqe, wr, &amp;t3_wr_flit_cnt);</div><div class='del'>-			if (err)</div><div class='del'>-				break;</div><div class='del'>-			sqp-&gt;read_len = wqe-&gt;read.local_len;</div><div class='del'>-			if (!qhp-&gt;wq.oldest_read)</div><div class='del'>-				qhp-&gt;wq.oldest_read = sqp;</div><div class='del'>-			break;</div><div class='del'>-		case IB_WR_REG_MR:</div><div class='del'>-			t3_wr_opcode = T3_WR_FASTREG;</div><div class='del'>-			err = build_memreg(wqe, reg_wr(wr), &amp;t3_wr_flit_cnt,</div><div class='del'>-					   &amp;wr_cnt, &amp;qhp-&gt;wq);</div><div class='del'>-			break;</div><div class='del'>-		case IB_WR_LOCAL_INV:</div><div class='del'>-			if (wr-&gt;send_flags &amp; IB_SEND_FENCE)</div><div class='del'>-				t3_wr_flags |= T3_LOCAL_FENCE_FLAG;</div><div class='del'>-			t3_wr_opcode = T3_WR_INV_STAG;</div><div class='del'>-			err = build_inv_stag(wqe, wr, &amp;t3_wr_flit_cnt);</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			pr_debug("%s post of type=%d TBD!\n", __func__,</div><div class='del'>-				 wr-&gt;opcode);</div><div class='del'>-			err = -EINVAL;</div><div class='del'>-		}</div><div class='del'>-		if (err)</div><div class='del'>-			break;</div><div class='del'>-		wqe-&gt;send.wrid.id0.hi = qhp-&gt;wq.sq_wptr;</div><div class='del'>-		sqp-&gt;wr_id = wr-&gt;wr_id;</div><div class='del'>-		sqp-&gt;opcode = wr2opcode(t3_wr_opcode);</div><div class='del'>-		sqp-&gt;sq_wptr = qhp-&gt;wq.sq_wptr;</div><div class='del'>-		sqp-&gt;complete = 0;</div><div class='del'>-		sqp-&gt;signaled = (wr-&gt;send_flags &amp; IB_SEND_SIGNALED);</div><div class='del'>-</div><div class='del'>-		build_fw_riwrh((void *) wqe, t3_wr_opcode, t3_wr_flags,</div><div class='del'>-			       Q_GENBIT(qhp-&gt;wq.wptr, qhp-&gt;wq.size_log2),</div><div class='del'>-			       0, t3_wr_flit_cnt,</div><div class='del'>-			       (wr_cnt == 1) ? T3_SOPEOP : T3_SOP);</div><div class='del'>-		pr_debug("%s cookie 0x%llx wq idx 0x%x swsq idx %ld opcode %d\n",</div><div class='del'>-			 __func__, (unsigned long long)wr-&gt;wr_id, idx,</div><div class='del'>-			 Q_PTR2IDX(qhp-&gt;wq.sq_wptr, qhp-&gt;wq.sq_size_log2),</div><div class='del'>-			 sqp-&gt;opcode);</div><div class='del'>-		wr = wr-&gt;next;</div><div class='del'>-		num_wrs--;</div><div class='del'>-		qhp-&gt;wq.wptr += wr_cnt;</div><div class='del'>-		++(qhp-&gt;wq.sq_wptr);</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;qhp-&gt;lock, flag);</div><div class='del'>-	if (cxio_wq_db_enabled(&amp;qhp-&gt;wq))</div><div class='del'>-		ring_doorbell(qhp-&gt;wq.doorbell, qhp-&gt;wq.qpid);</div><div class='del'>-</div><div class='del'>-out:</div><div class='del'>-	if (err)</div><div class='del'>-		*bad_wr = wr;</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_post_receive(struct ib_qp *ibqp, const struct ib_recv_wr *wr,</div><div class='del'>-		      const struct ib_recv_wr **bad_wr)</div><div class='del'>-{</div><div class='del'>-	int err = 0;</div><div class='del'>-	struct iwch_qp *qhp;</div><div class='del'>-	u32 idx;</div><div class='del'>-	union t3_wr *wqe;</div><div class='del'>-	u32 num_wrs;</div><div class='del'>-	unsigned long flag;</div><div class='del'>-</div><div class='del'>-	qhp = to_iwch_qp(ibqp);</div><div class='del'>-	spin_lock_irqsave(&amp;qhp-&gt;lock, flag);</div><div class='del'>-	if (qhp-&gt;attr.state &gt; IWCH_QP_STATE_RTS) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;qhp-&gt;lock, flag);</div><div class='del'>-		err = -EINVAL;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-	num_wrs = Q_FREECNT(qhp-&gt;wq.rq_rptr, qhp-&gt;wq.rq_wptr,</div><div class='del'>-			    qhp-&gt;wq.rq_size_log2) - 1;</div><div class='del'>-	if (!wr) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;qhp-&gt;lock, flag);</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-	while (wr) {</div><div class='del'>-		if (wr-&gt;num_sge &gt; T3_MAX_SGE) {</div><div class='del'>-			err = -EINVAL;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		idx = Q_PTR2IDX(qhp-&gt;wq.wptr, qhp-&gt;wq.size_log2);</div><div class='del'>-		wqe = (union t3_wr *) (qhp-&gt;wq.queue + idx);</div><div class='del'>-		if (num_wrs)</div><div class='del'>-			if (wr-&gt;sg_list[0].lkey)</div><div class='del'>-				err = build_rdma_recv(qhp, wqe, wr);</div><div class='del'>-			else</div><div class='del'>-				err = build_zero_stag_recv(qhp, wqe, wr);</div><div class='del'>-		else</div><div class='del'>-			err = -ENOMEM;</div><div class='del'>-</div><div class='del'>-		if (err)</div><div class='del'>-			break;</div><div class='del'>-</div><div class='del'>-		build_fw_riwrh((void *) wqe, T3_WR_RCV, T3_COMPLETION_FLAG,</div><div class='del'>-			       Q_GENBIT(qhp-&gt;wq.wptr, qhp-&gt;wq.size_log2),</div><div class='del'>-			       0, sizeof(struct t3_receive_wr) &gt;&gt; 3, T3_SOPEOP);</div><div class='del'>-		pr_debug("%s cookie 0x%llx idx 0x%x rq_wptr 0x%x rw_rptr 0x%x wqe %p\n",</div><div class='del'>-			 __func__, (unsigned long long)wr-&gt;wr_id,</div><div class='del'>-			 idx, qhp-&gt;wq.rq_wptr, qhp-&gt;wq.rq_rptr, wqe);</div><div class='del'>-		++(qhp-&gt;wq.rq_wptr);</div><div class='del'>-		++(qhp-&gt;wq.wptr);</div><div class='del'>-		wr = wr-&gt;next;</div><div class='del'>-		num_wrs--;</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;qhp-&gt;lock, flag);</div><div class='del'>-	if (cxio_wq_db_enabled(&amp;qhp-&gt;wq))</div><div class='del'>-		ring_doorbell(qhp-&gt;wq.doorbell, qhp-&gt;wq.qpid);</div><div class='del'>-</div><div class='del'>-out:</div><div class='del'>-	if (err)</div><div class='del'>-		*bad_wr = wr;</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void build_term_codes(struct respQ_msg_t *rsp_msg,</div><div class='del'>-				    u8 *layer_type, u8 *ecode)</div><div class='del'>-{</div><div class='del'>-	int status = TPT_ERR_INTERNAL_ERR;</div><div class='del'>-	int tagged = 0;</div><div class='del'>-	int opcode = -1;</div><div class='del'>-	int rqtype = 0;</div><div class='del'>-	int send_inv = 0;</div><div class='del'>-</div><div class='del'>-	if (rsp_msg) {</div><div class='del'>-		status = CQE_STATUS(rsp_msg-&gt;cqe);</div><div class='del'>-		opcode = CQE_OPCODE(rsp_msg-&gt;cqe);</div><div class='del'>-		rqtype = RQ_TYPE(rsp_msg-&gt;cqe);</div><div class='del'>-		send_inv = (opcode == T3_SEND_WITH_INV) ||</div><div class='del'>-		           (opcode == T3_SEND_WITH_SE_INV);</div><div class='del'>-		tagged = (opcode == T3_RDMA_WRITE) ||</div><div class='del'>-			 (rqtype &amp;&amp; (opcode == T3_READ_RESP));</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	switch (status) {</div><div class='del'>-	case TPT_ERR_STAG:</div><div class='del'>-		if (send_inv) {</div><div class='del'>-			*layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;</div><div class='del'>-			*ecode = RDMAP_CANT_INV_STAG;</div><div class='del'>-		} else {</div><div class='del'>-			*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;</div><div class='del'>-			*ecode = RDMAP_INV_STAG;</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_PDID:</div><div class='del'>-		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;</div><div class='del'>-		if ((opcode == T3_SEND_WITH_INV) ||</div><div class='del'>-		    (opcode == T3_SEND_WITH_SE_INV))</div><div class='del'>-			*ecode = RDMAP_CANT_INV_STAG;</div><div class='del'>-		else</div><div class='del'>-			*ecode = RDMAP_STAG_NOT_ASSOC;</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_QPID:</div><div class='del'>-		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;</div><div class='del'>-		*ecode = RDMAP_STAG_NOT_ASSOC;</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_ACCESS:</div><div class='del'>-		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;</div><div class='del'>-		*ecode = RDMAP_ACC_VIOL;</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_WRAP:</div><div class='del'>-		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;</div><div class='del'>-		*ecode = RDMAP_TO_WRAP;</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_BOUND:</div><div class='del'>-		if (tagged) {</div><div class='del'>-			*layer_type = LAYER_DDP|DDP_TAGGED_ERR;</div><div class='del'>-			*ecode = DDPT_BASE_BOUNDS;</div><div class='del'>-		} else {</div><div class='del'>-			*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;</div><div class='del'>-			*ecode = RDMAP_BASE_BOUNDS;</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_INVALIDATE_SHARED_MR:</div><div class='del'>-	case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND:</div><div class='del'>-		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;</div><div class='del'>-		*ecode = RDMAP_CANT_INV_STAG;</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_ECC:</div><div class='del'>-	case TPT_ERR_ECC_PSTAG:</div><div class='del'>-	case TPT_ERR_INTERNAL_ERR:</div><div class='del'>-		*layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA;</div><div class='del'>-		*ecode = 0;</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_OUT_OF_RQE:</div><div class='del'>-		*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;</div><div class='del'>-		*ecode = DDPU_INV_MSN_NOBUF;</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_PBL_ADDR_BOUND:</div><div class='del'>-		*layer_type = LAYER_DDP|DDP_TAGGED_ERR;</div><div class='del'>-		*ecode = DDPT_BASE_BOUNDS;</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_CRC:</div><div class='del'>-		*layer_type = LAYER_MPA|DDP_LLP;</div><div class='del'>-		*ecode = MPA_CRC_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_MARKER:</div><div class='del'>-		*layer_type = LAYER_MPA|DDP_LLP;</div><div class='del'>-		*ecode = MPA_MARKER_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_PDU_LEN_ERR:</div><div class='del'>-		*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;</div><div class='del'>-		*ecode = DDPU_MSG_TOOBIG;</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_DDP_VERSION:</div><div class='del'>-		if (tagged) {</div><div class='del'>-			*layer_type = LAYER_DDP|DDP_TAGGED_ERR;</div><div class='del'>-			*ecode = DDPT_INV_VERS;</div><div class='del'>-		} else {</div><div class='del'>-			*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;</div><div class='del'>-			*ecode = DDPU_INV_VERS;</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_RDMA_VERSION:</div><div class='del'>-		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;</div><div class='del'>-		*ecode = RDMAP_INV_VERS;</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_OPCODE:</div><div class='del'>-		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;</div><div class='del'>-		*ecode = RDMAP_INV_OPCODE;</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_DDP_QUEUE_NUM:</div><div class='del'>-		*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;</div><div class='del'>-		*ecode = DDPU_INV_QN;</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_MSN:</div><div class='del'>-	case TPT_ERR_MSN_GAP:</div><div class='del'>-	case TPT_ERR_MSN_RANGE:</div><div class='del'>-	case TPT_ERR_IRD_OVERFLOW:</div><div class='del'>-		*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;</div><div class='del'>-		*ecode = DDPU_INV_MSN_RANGE;</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_TBIT:</div><div class='del'>-		*layer_type = LAYER_DDP|DDP_LOCAL_CATA;</div><div class='del'>-		*ecode = 0;</div><div class='del'>-		break;</div><div class='del'>-	case TPT_ERR_MO:</div><div class='del'>-		*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;</div><div class='del'>-		*ecode = DDPU_INV_MO;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		*layer_type = LAYER_RDMAP|DDP_LOCAL_CATA;</div><div class='del'>-		*ecode = 0;</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_post_zb_read(struct iwch_ep *ep)</div><div class='del'>-{</div><div class='del'>-	union t3_wr *wqe;</div><div class='del'>-	struct sk_buff *skb;</div><div class='del'>-	u8 flit_cnt = sizeof(struct t3_rdma_read_wr) &gt;&gt; 3;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s enter\n", __func__);</div><div class='del'>-	skb = alloc_skb(40, GFP_KERNEL);</div><div class='del'>-	if (!skb) {</div><div class='del'>-		pr_err("%s cannot send zb_read!!\n", __func__);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	wqe = skb_put_zero(skb, sizeof(struct t3_rdma_read_wr));</div><div class='del'>-	wqe-&gt;read.rdmaop = T3_READ_REQ;</div><div class='del'>-	wqe-&gt;read.reserved[0] = 0;</div><div class='del'>-	wqe-&gt;read.reserved[1] = 0;</div><div class='del'>-	wqe-&gt;read.rem_stag = cpu_to_be32(1);</div><div class='del'>-	wqe-&gt;read.rem_to = cpu_to_be64(1);</div><div class='del'>-	wqe-&gt;read.local_stag = cpu_to_be32(1);</div><div class='del'>-	wqe-&gt;read.local_len = cpu_to_be32(0);</div><div class='del'>-	wqe-&gt;read.local_to = cpu_to_be64(1);</div><div class='del'>-	wqe-&gt;send.wrh.op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(T3_WR_READ));</div><div class='del'>-	wqe-&gt;send.wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(ep-&gt;hwtid)|</div><div class='del'>-						V_FW_RIWR_LEN(flit_cnt));</div><div class='del'>-	skb-&gt;priority = CPL_PRIORITY_DATA;</div><div class='del'>-	return iwch_cxgb3_ofld_send(ep-&gt;com.qp-&gt;rhp-&gt;rdev.t3cdev_p, skb);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * This posts a TERMINATE with layer=RDMA, type=catastrophic.</div><div class='del'>- */</div><div class='del'>-int iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg)</div><div class='del'>-{</div><div class='del'>-	union t3_wr *wqe;</div><div class='del'>-	struct terminate_message *term;</div><div class='del'>-	struct sk_buff *skb;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s %d\n", __func__, __LINE__);</div><div class='del'>-	skb = alloc_skb(40, GFP_ATOMIC);</div><div class='del'>-	if (!skb) {</div><div class='del'>-		pr_err("%s cannot send TERMINATE!\n", __func__);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	wqe = skb_put_zero(skb, 40);</div><div class='del'>-	wqe-&gt;send.rdmaop = T3_TERMINATE;</div><div class='del'>-</div><div class='del'>-	/* immediate data length */</div><div class='del'>-	wqe-&gt;send.plen = htonl(4);</div><div class='del'>-</div><div class='del'>-	/* immediate data starts here. */</div><div class='del'>-	term = (struct terminate_message *)wqe-&gt;send.sgl;</div><div class='del'>-	build_term_codes(rsp_msg, &amp;term-&gt;layer_etype, &amp;term-&gt;ecode);</div><div class='del'>-	wqe-&gt;send.wrh.op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(T3_WR_SEND) |</div><div class='del'>-			 V_FW_RIWR_FLAGS(T3_COMPLETION_FLAG | T3_NOTIFY_FLAG));</div><div class='del'>-	wqe-&gt;send.wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(qhp-&gt;ep-&gt;hwtid));</div><div class='del'>-	skb-&gt;priority = CPL_PRIORITY_DATA;</div><div class='del'>-	return iwch_cxgb3_ofld_send(qhp-&gt;rhp-&gt;rdev.t3cdev_p, skb);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Assumes qhp lock is held.</div><div class='del'>- */</div><div class='del'>-static void __flush_qp(struct iwch_qp *qhp, struct iwch_cq *rchp,</div><div class='del'>-				struct iwch_cq *schp)</div><div class='del'>-	__releases(&amp;qhp-&gt;lock)</div><div class='del'>-	__acquires(&amp;qhp-&gt;lock)</div><div class='del'>-{</div><div class='del'>-	int count;</div><div class='del'>-	int flushed;</div><div class='del'>-</div><div class='del'>-	lockdep_assert_held(&amp;qhp-&gt;lock);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s qhp %p rchp %p schp %p\n", __func__, qhp, rchp, schp);</div><div class='del'>-	/* take a ref on the qhp since we must release the lock */</div><div class='del'>-	atomic_inc(&amp;qhp-&gt;refcnt);</div><div class='del'>-	spin_unlock(&amp;qhp-&gt;lock);</div><div class='del'>-</div><div class='del'>-	/* locking hierarchy: cq lock first, then qp lock. */</div><div class='del'>-	spin_lock(&amp;rchp-&gt;lock);</div><div class='del'>-	spin_lock(&amp;qhp-&gt;lock);</div><div class='del'>-	cxio_flush_hw_cq(&amp;rchp-&gt;cq);</div><div class='del'>-	cxio_count_rcqes(&amp;rchp-&gt;cq, &amp;qhp-&gt;wq, &amp;count);</div><div class='del'>-	flushed = cxio_flush_rq(&amp;qhp-&gt;wq, &amp;rchp-&gt;cq, count);</div><div class='del'>-	spin_unlock(&amp;qhp-&gt;lock);</div><div class='del'>-	spin_unlock(&amp;rchp-&gt;lock);</div><div class='del'>-	if (flushed) {</div><div class='del'>-		spin_lock(&amp;rchp-&gt;comp_handler_lock);</div><div class='del'>-		(*rchp-&gt;ibcq.comp_handler)(&amp;rchp-&gt;ibcq, rchp-&gt;ibcq.cq_context);</div><div class='del'>-		spin_unlock(&amp;rchp-&gt;comp_handler_lock);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* locking hierarchy: cq lock first, then qp lock. */</div><div class='del'>-	spin_lock(&amp;schp-&gt;lock);</div><div class='del'>-	spin_lock(&amp;qhp-&gt;lock);</div><div class='del'>-	cxio_flush_hw_cq(&amp;schp-&gt;cq);</div><div class='del'>-	cxio_count_scqes(&amp;schp-&gt;cq, &amp;qhp-&gt;wq, &amp;count);</div><div class='del'>-	flushed = cxio_flush_sq(&amp;qhp-&gt;wq, &amp;schp-&gt;cq, count);</div><div class='del'>-	spin_unlock(&amp;qhp-&gt;lock);</div><div class='del'>-	spin_unlock(&amp;schp-&gt;lock);</div><div class='del'>-	if (flushed) {</div><div class='del'>-		spin_lock(&amp;schp-&gt;comp_handler_lock);</div><div class='del'>-		(*schp-&gt;ibcq.comp_handler)(&amp;schp-&gt;ibcq, schp-&gt;ibcq.cq_context);</div><div class='del'>-		spin_unlock(&amp;schp-&gt;comp_handler_lock);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* deref */</div><div class='del'>-	if (atomic_dec_and_test(&amp;qhp-&gt;refcnt))</div><div class='del'>-	        wake_up(&amp;qhp-&gt;wait);</div><div class='del'>-</div><div class='del'>-	spin_lock(&amp;qhp-&gt;lock);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void flush_qp(struct iwch_qp *qhp)</div><div class='del'>-{</div><div class='del'>-	struct iwch_cq *rchp, *schp;</div><div class='del'>-</div><div class='del'>-	rchp = get_chp(qhp-&gt;rhp, qhp-&gt;attr.rcq);</div><div class='del'>-	schp = get_chp(qhp-&gt;rhp, qhp-&gt;attr.scq);</div><div class='del'>-</div><div class='del'>-	if (qhp-&gt;ibqp.uobject) {</div><div class='del'>-		cxio_set_wq_in_error(&amp;qhp-&gt;wq);</div><div class='del'>-		cxio_set_cq_in_error(&amp;rchp-&gt;cq);</div><div class='del'>-		spin_lock(&amp;rchp-&gt;comp_handler_lock);</div><div class='del'>-		(*rchp-&gt;ibcq.comp_handler)(&amp;rchp-&gt;ibcq, rchp-&gt;ibcq.cq_context);</div><div class='del'>-		spin_unlock(&amp;rchp-&gt;comp_handler_lock);</div><div class='del'>-		if (schp != rchp) {</div><div class='del'>-			cxio_set_cq_in_error(&amp;schp-&gt;cq);</div><div class='del'>-			spin_lock(&amp;schp-&gt;comp_handler_lock);</div><div class='del'>-			(*schp-&gt;ibcq.comp_handler)(&amp;schp-&gt;ibcq,</div><div class='del'>-						   schp-&gt;ibcq.cq_context);</div><div class='del'>-			spin_unlock(&amp;schp-&gt;comp_handler_lock);</div><div class='del'>-		}</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-	__flush_qp(qhp, rchp, schp);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Return count of RECV WRs posted</div><div class='del'>- */</div><div class='del'>-u16 iwch_rqes_posted(struct iwch_qp *qhp)</div><div class='del'>-{</div><div class='del'>-	union t3_wr *wqe = qhp-&gt;wq.queue;</div><div class='del'>-	u16 count = 0;</div><div class='del'>-</div><div class='del'>-	while (count &lt; USHRT_MAX &amp;&amp; fw_riwrh_opcode((struct fw_riwrh *)wqe) == T3_WR_RCV) {</div><div class='del'>-		count++;</div><div class='del'>-		wqe++;</div><div class='del'>-	}</div><div class='del'>-	pr_debug("%s qhp %p count %u\n", __func__, qhp, count);</div><div class='del'>-	return count;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp,</div><div class='del'>-				enum iwch_qp_attr_mask mask,</div><div class='del'>-				struct iwch_qp_attributes *attrs)</div><div class='del'>-{</div><div class='del'>-	struct t3_rdma_init_attr init_attr;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	init_attr.tid = qhp-&gt;ep-&gt;hwtid;</div><div class='del'>-	init_attr.qpid = qhp-&gt;wq.qpid;</div><div class='del'>-	init_attr.pdid = qhp-&gt;attr.pd;</div><div class='del'>-	init_attr.scqid = qhp-&gt;attr.scq;</div><div class='del'>-	init_attr.rcqid = qhp-&gt;attr.rcq;</div><div class='del'>-	init_attr.rq_addr = qhp-&gt;wq.rq_addr;</div><div class='del'>-	init_attr.rq_size = 1 &lt;&lt; qhp-&gt;wq.rq_size_log2;</div><div class='del'>-	init_attr.mpaattrs = uP_RI_MPA_IETF_ENABLE |</div><div class='del'>-		qhp-&gt;attr.mpa_attr.recv_marker_enabled |</div><div class='del'>-		(qhp-&gt;attr.mpa_attr.xmit_marker_enabled &lt;&lt; 1) |</div><div class='del'>-		(qhp-&gt;attr.mpa_attr.crc_enabled &lt;&lt; 2);</div><div class='del'>-</div><div class='del'>-	init_attr.qpcaps = uP_RI_QP_RDMA_READ_ENABLE |</div><div class='del'>-			   uP_RI_QP_RDMA_WRITE_ENABLE |</div><div class='del'>-			   uP_RI_QP_BIND_ENABLE;</div><div class='del'>-	if (!qhp-&gt;ibqp.uobject)</div><div class='del'>-		init_attr.qpcaps |= uP_RI_QP_STAG0_ENABLE |</div><div class='del'>-				    uP_RI_QP_FAST_REGISTER_ENABLE;</div><div class='del'>-</div><div class='del'>-	init_attr.tcp_emss = qhp-&gt;ep-&gt;emss;</div><div class='del'>-	init_attr.ord = qhp-&gt;attr.max_ord;</div><div class='del'>-	init_attr.ird = qhp-&gt;attr.max_ird;</div><div class='del'>-	init_attr.qp_dma_addr = qhp-&gt;wq.dma_addr;</div><div class='del'>-	init_attr.qp_dma_size = (1UL &lt;&lt; qhp-&gt;wq.size_log2);</div><div class='del'>-	init_attr.rqe_count = iwch_rqes_posted(qhp);</div><div class='del'>-	init_attr.flags = qhp-&gt;attr.mpa_attr.initiator ? MPA_INITIATOR : 0;</div><div class='del'>-	init_attr.chan = qhp-&gt;ep-&gt;l2t-&gt;smt_idx;</div><div class='del'>-	if (peer2peer) {</div><div class='del'>-		init_attr.rtr_type = RTR_READ;</div><div class='del'>-		if (init_attr.ord == 0 &amp;&amp; qhp-&gt;attr.mpa_attr.initiator)</div><div class='del'>-			init_attr.ord = 1;</div><div class='del'>-		if (init_attr.ird == 0 &amp;&amp; !qhp-&gt;attr.mpa_attr.initiator)</div><div class='del'>-			init_attr.ird = 1;</div><div class='del'>-	} else</div><div class='del'>-		init_attr.rtr_type = 0;</div><div class='del'>-	init_attr.irs = qhp-&gt;ep-&gt;rcv_seq;</div><div class='del'>-	pr_debug("%s init_attr.rq_addr 0x%x init_attr.rq_size = %d flags 0x%x qpcaps 0x%x\n",</div><div class='del'>-		 __func__,</div><div class='del'>-		 init_attr.rq_addr, init_attr.rq_size,</div><div class='del'>-		 init_attr.flags, init_attr.qpcaps);</div><div class='del'>-	ret = cxio_rdma_init(&amp;rhp-&gt;rdev, &amp;init_attr);</div><div class='del'>-	pr_debug("%s ret %d\n", __func__, ret);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int iwch_modify_qp(struct iwch_dev *rhp, struct iwch_qp *qhp,</div><div class='del'>-				enum iwch_qp_attr_mask mask,</div><div class='del'>-				struct iwch_qp_attributes *attrs,</div><div class='del'>-				int internal)</div><div class='del'>-{</div><div class='del'>-	int ret = 0;</div><div class='del'>-	struct iwch_qp_attributes newattr = qhp-&gt;attr;</div><div class='del'>-	unsigned long flag;</div><div class='del'>-	int disconnect = 0;</div><div class='del'>-	int terminate = 0;</div><div class='del'>-	int abort = 0;</div><div class='del'>-	int free = 0;</div><div class='del'>-	struct iwch_ep *ep = NULL;</div><div class='del'>-</div><div class='del'>-	pr_debug("%s qhp %p qpid 0x%x ep %p state %d -&gt; %d\n", __func__,</div><div class='del'>-		 qhp, qhp-&gt;wq.qpid, qhp-&gt;ep, qhp-&gt;attr.state,</div><div class='del'>-		 (mask &amp; IWCH_QP_ATTR_NEXT_STATE) ? attrs-&gt;next_state : -1);</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;qhp-&gt;lock, flag);</div><div class='del'>-</div><div class='del'>-	/* Process attr changes if in IDLE */</div><div class='del'>-	if (mask &amp; IWCH_QP_ATTR_VALID_MODIFY) {</div><div class='del'>-		if (qhp-&gt;attr.state != IWCH_QP_STATE_IDLE) {</div><div class='del'>-			ret = -EIO;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-		if (mask &amp; IWCH_QP_ATTR_ENABLE_RDMA_READ)</div><div class='del'>-			newattr.enable_rdma_read = attrs-&gt;enable_rdma_read;</div><div class='del'>-		if (mask &amp; IWCH_QP_ATTR_ENABLE_RDMA_WRITE)</div><div class='del'>-			newattr.enable_rdma_write = attrs-&gt;enable_rdma_write;</div><div class='del'>-		if (mask &amp; IWCH_QP_ATTR_ENABLE_RDMA_BIND)</div><div class='del'>-			newattr.enable_bind = attrs-&gt;enable_bind;</div><div class='del'>-		if (mask &amp; IWCH_QP_ATTR_MAX_ORD) {</div><div class='del'>-			if (attrs-&gt;max_ord &gt;</div><div class='del'>-			    rhp-&gt;attr.max_rdma_read_qp_depth) {</div><div class='del'>-				ret = -EINVAL;</div><div class='del'>-				goto out;</div><div class='del'>-			}</div><div class='del'>-			newattr.max_ord = attrs-&gt;max_ord;</div><div class='del'>-		}</div><div class='del'>-		if (mask &amp; IWCH_QP_ATTR_MAX_IRD) {</div><div class='del'>-			if (attrs-&gt;max_ird &gt;</div><div class='del'>-			    rhp-&gt;attr.max_rdma_reads_per_qp) {</div><div class='del'>-				ret = -EINVAL;</div><div class='del'>-				goto out;</div><div class='del'>-			}</div><div class='del'>-			newattr.max_ird = attrs-&gt;max_ird;</div><div class='del'>-		}</div><div class='del'>-		qhp-&gt;attr = newattr;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!(mask &amp; IWCH_QP_ATTR_NEXT_STATE))</div><div class='del'>-		goto out;</div><div class='del'>-	if (qhp-&gt;attr.state == attrs-&gt;next_state)</div><div class='del'>-		goto out;</div><div class='del'>-</div><div class='del'>-	switch (qhp-&gt;attr.state) {</div><div class='del'>-	case IWCH_QP_STATE_IDLE:</div><div class='del'>-		switch (attrs-&gt;next_state) {</div><div class='del'>-		case IWCH_QP_STATE_RTS:</div><div class='del'>-			if (!(mask &amp; IWCH_QP_ATTR_LLP_STREAM_HANDLE)) {</div><div class='del'>-				ret = -EINVAL;</div><div class='del'>-				goto out;</div><div class='del'>-			}</div><div class='del'>-			if (!(mask &amp; IWCH_QP_ATTR_MPA_ATTR)) {</div><div class='del'>-				ret = -EINVAL;</div><div class='del'>-				goto out;</div><div class='del'>-			}</div><div class='del'>-			qhp-&gt;attr.mpa_attr = attrs-&gt;mpa_attr;</div><div class='del'>-			qhp-&gt;attr.llp_stream_handle = attrs-&gt;llp_stream_handle;</div><div class='del'>-			qhp-&gt;ep = qhp-&gt;attr.llp_stream_handle;</div><div class='del'>-			qhp-&gt;attr.state = IWCH_QP_STATE_RTS;</div><div class='del'>-</div><div class='del'>-			/*</div><div class='del'>-			 * Ref the endpoint here and deref when we</div><div class='del'>-			 * disassociate the endpoint from the QP.  This</div><div class='del'>-			 * happens in CLOSING-&gt;IDLE transition or *-&gt;ERROR</div><div class='del'>-			 * transition.</div><div class='del'>-			 */</div><div class='del'>-			get_ep(&amp;qhp-&gt;ep-&gt;com);</div><div class='del'>-			spin_unlock_irqrestore(&amp;qhp-&gt;lock, flag);</div><div class='del'>-			ret = rdma_init(rhp, qhp, mask, attrs);</div><div class='del'>-			spin_lock_irqsave(&amp;qhp-&gt;lock, flag);</div><div class='del'>-			if (ret)</div><div class='del'>-				goto err;</div><div class='del'>-			break;</div><div class='del'>-		case IWCH_QP_STATE_ERROR:</div><div class='del'>-			qhp-&gt;attr.state = IWCH_QP_STATE_ERROR;</div><div class='del'>-			flush_qp(qhp);</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case IWCH_QP_STATE_RTS:</div><div class='del'>-		switch (attrs-&gt;next_state) {</div><div class='del'>-		case IWCH_QP_STATE_CLOSING:</div><div class='del'>-			BUG_ON(kref_read(&amp;qhp-&gt;ep-&gt;com.kref) &lt; 2);</div><div class='del'>-			qhp-&gt;attr.state = IWCH_QP_STATE_CLOSING;</div><div class='del'>-			if (!internal) {</div><div class='del'>-				abort=0;</div><div class='del'>-				disconnect = 1;</div><div class='del'>-				ep = qhp-&gt;ep;</div><div class='del'>-				get_ep(&amp;ep-&gt;com);</div><div class='del'>-			}</div><div class='del'>-			break;</div><div class='del'>-		case IWCH_QP_STATE_TERMINATE:</div><div class='del'>-			qhp-&gt;attr.state = IWCH_QP_STATE_TERMINATE;</div><div class='del'>-			if (qhp-&gt;ibqp.uobject)</div><div class='del'>-				cxio_set_wq_in_error(&amp;qhp-&gt;wq);</div><div class='del'>-			if (!internal)</div><div class='del'>-				terminate = 1;</div><div class='del'>-			break;</div><div class='del'>-		case IWCH_QP_STATE_ERROR:</div><div class='del'>-			qhp-&gt;attr.state = IWCH_QP_STATE_ERROR;</div><div class='del'>-			if (!internal) {</div><div class='del'>-				abort=1;</div><div class='del'>-				disconnect = 1;</div><div class='del'>-				ep = qhp-&gt;ep;</div><div class='del'>-				get_ep(&amp;ep-&gt;com);</div><div class='del'>-			}</div><div class='del'>-			goto err;</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case IWCH_QP_STATE_CLOSING:</div><div class='del'>-		if (!internal) {</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-		switch (attrs-&gt;next_state) {</div><div class='del'>-			case IWCH_QP_STATE_IDLE:</div><div class='del'>-				flush_qp(qhp);</div><div class='del'>-				qhp-&gt;attr.state = IWCH_QP_STATE_IDLE;</div><div class='del'>-				qhp-&gt;attr.llp_stream_handle = NULL;</div><div class='del'>-				put_ep(&amp;qhp-&gt;ep-&gt;com);</div><div class='del'>-				qhp-&gt;ep = NULL;</div><div class='del'>-				wake_up(&amp;qhp-&gt;wait);</div><div class='del'>-				break;</div><div class='del'>-			case IWCH_QP_STATE_ERROR:</div><div class='del'>-				goto err;</div><div class='del'>-			default:</div><div class='del'>-				ret = -EINVAL;</div><div class='del'>-				goto err;</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case IWCH_QP_STATE_ERROR:</div><div class='del'>-		if (attrs-&gt;next_state != IWCH_QP_STATE_IDLE) {</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (!Q_EMPTY(qhp-&gt;wq.sq_rptr, qhp-&gt;wq.sq_wptr) ||</div><div class='del'>-		    !Q_EMPTY(qhp-&gt;wq.rq_rptr, qhp-&gt;wq.rq_wptr)) {</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-		qhp-&gt;attr.state = IWCH_QP_STATE_IDLE;</div><div class='del'>-		break;</div><div class='del'>-	case IWCH_QP_STATE_TERMINATE:</div><div class='del'>-		if (!internal) {</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-		goto err;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		pr_err("%s in a bad state %d\n", __func__, qhp-&gt;attr.state);</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-		goto err;</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	goto out;</div><div class='del'>-err:</div><div class='del'>-	pr_debug("%s disassociating ep %p qpid 0x%x\n", __func__, qhp-&gt;ep,</div><div class='del'>-		 qhp-&gt;wq.qpid);</div><div class='del'>-</div><div class='del'>-	/* disassociate the LLP connection */</div><div class='del'>-	qhp-&gt;attr.llp_stream_handle = NULL;</div><div class='del'>-	ep = qhp-&gt;ep;</div><div class='del'>-	qhp-&gt;ep = NULL;</div><div class='del'>-	qhp-&gt;attr.state = IWCH_QP_STATE_ERROR;</div><div class='del'>-	free=1;</div><div class='del'>-	wake_up(&amp;qhp-&gt;wait);</div><div class='del'>-	BUG_ON(!ep);</div><div class='del'>-	flush_qp(qhp);</div><div class='del'>-out:</div><div class='del'>-	spin_unlock_irqrestore(&amp;qhp-&gt;lock, flag);</div><div class='del'>-</div><div class='del'>-	if (terminate)</div><div class='del'>-		iwch_post_terminate(qhp, NULL);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If disconnect is 1, then we need to initiate a disconnect</div><div class='del'>-	 * on the EP.  This can be a normal close (RTS-&gt;CLOSING) or</div><div class='del'>-	 * an abnormal close (RTS/CLOSING-&gt;ERROR).</div><div class='del'>-	 */</div><div class='del'>-	if (disconnect) {</div><div class='del'>-		iwch_ep_disconnect(ep, abort, GFP_KERNEL);</div><div class='del'>-		put_ep(&amp;ep-&gt;com);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If free is 1, then we've disassociated the EP from the QP</div><div class='del'>-	 * and we need to dereference the EP.</div><div class='del'>-	 */</div><div class='del'>-	if (free)</div><div class='del'>-		put_ep(&amp;ep-&gt;com);</div><div class='del'>-</div><div class='del'>-	pr_debug("%s exit state %d\n", __func__, qhp-&gt;attr.state);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb3/tcb.h b/drivers/infiniband/hw/cxgb3/tcb.h<br/>deleted file mode 100644<br/>index c702dc199e18..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb3/tcb.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb3/tcb.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,632 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2007 Chelsio, Inc. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-#ifndef _TCB_DEFS_H</div><div class='del'>-#define _TCB_DEFS_H</div><div class='del'>-</div><div class='del'>-#define W_TCB_T_STATE    0</div><div class='del'>-#define S_TCB_T_STATE    0</div><div class='del'>-#define M_TCB_T_STATE    0xfULL</div><div class='del'>-#define V_TCB_T_STATE(x) ((x) &lt;&lt; S_TCB_T_STATE)</div><div class='del'>-</div><div class='del'>-#define W_TCB_TIMER    0</div><div class='del'>-#define S_TCB_TIMER    4</div><div class='del'>-#define M_TCB_TIMER    0x1ULL</div><div class='del'>-#define V_TCB_TIMER(x) ((x) &lt;&lt; S_TCB_TIMER)</div><div class='del'>-</div><div class='del'>-#define W_TCB_DACK_TIMER    0</div><div class='del'>-#define S_TCB_DACK_TIMER    5</div><div class='del'>-#define M_TCB_DACK_TIMER    0x1ULL</div><div class='del'>-#define V_TCB_DACK_TIMER(x) ((x) &lt;&lt; S_TCB_DACK_TIMER)</div><div class='del'>-</div><div class='del'>-#define W_TCB_DEL_FLAG    0</div><div class='del'>-#define S_TCB_DEL_FLAG    6</div><div class='del'>-#define M_TCB_DEL_FLAG    0x1ULL</div><div class='del'>-#define V_TCB_DEL_FLAG(x) ((x) &lt;&lt; S_TCB_DEL_FLAG)</div><div class='del'>-</div><div class='del'>-#define W_TCB_L2T_IX    0</div><div class='del'>-#define S_TCB_L2T_IX    7</div><div class='del'>-#define M_TCB_L2T_IX    0x7ffULL</div><div class='del'>-#define V_TCB_L2T_IX(x) ((x) &lt;&lt; S_TCB_L2T_IX)</div><div class='del'>-</div><div class='del'>-#define W_TCB_SMAC_SEL    0</div><div class='del'>-#define S_TCB_SMAC_SEL    18</div><div class='del'>-#define M_TCB_SMAC_SEL    0x3ULL</div><div class='del'>-#define V_TCB_SMAC_SEL(x) ((x) &lt;&lt; S_TCB_SMAC_SEL)</div><div class='del'>-</div><div class='del'>-#define W_TCB_TOS    0</div><div class='del'>-#define S_TCB_TOS    20</div><div class='del'>-#define M_TCB_TOS    0x3fULL</div><div class='del'>-#define V_TCB_TOS(x) ((x) &lt;&lt; S_TCB_TOS)</div><div class='del'>-</div><div class='del'>-#define W_TCB_MAX_RT    0</div><div class='del'>-#define S_TCB_MAX_RT    26</div><div class='del'>-#define M_TCB_MAX_RT    0xfULL</div><div class='del'>-#define V_TCB_MAX_RT(x) ((x) &lt;&lt; S_TCB_MAX_RT)</div><div class='del'>-</div><div class='del'>-#define W_TCB_T_RXTSHIFT    0</div><div class='del'>-#define S_TCB_T_RXTSHIFT    30</div><div class='del'>-#define M_TCB_T_RXTSHIFT    0xfULL</div><div class='del'>-#define V_TCB_T_RXTSHIFT(x) ((x) &lt;&lt; S_TCB_T_RXTSHIFT)</div><div class='del'>-</div><div class='del'>-#define W_TCB_T_DUPACKS    1</div><div class='del'>-#define S_TCB_T_DUPACKS    2</div><div class='del'>-#define M_TCB_T_DUPACKS    0xfULL</div><div class='del'>-#define V_TCB_T_DUPACKS(x) ((x) &lt;&lt; S_TCB_T_DUPACKS)</div><div class='del'>-</div><div class='del'>-#define W_TCB_T_MAXSEG    1</div><div class='del'>-#define S_TCB_T_MAXSEG    6</div><div class='del'>-#define M_TCB_T_MAXSEG    0xfULL</div><div class='del'>-#define V_TCB_T_MAXSEG(x) ((x) &lt;&lt; S_TCB_T_MAXSEG)</div><div class='del'>-</div><div class='del'>-#define W_TCB_T_FLAGS1    1</div><div class='del'>-#define S_TCB_T_FLAGS1    10</div><div class='del'>-#define M_TCB_T_FLAGS1    0xffffffffULL</div><div class='del'>-#define V_TCB_T_FLAGS1(x) ((x) &lt;&lt; S_TCB_T_FLAGS1)</div><div class='del'>-</div><div class='del'>-#define W_TCB_T_MIGRATION    1</div><div class='del'>-#define S_TCB_T_MIGRATION    20</div><div class='del'>-#define M_TCB_T_MIGRATION    0x1ULL</div><div class='del'>-#define V_TCB_T_MIGRATION(x) ((x) &lt;&lt; S_TCB_T_MIGRATION)</div><div class='del'>-</div><div class='del'>-#define W_TCB_T_FLAGS2    2</div><div class='del'>-#define S_TCB_T_FLAGS2    10</div><div class='del'>-#define M_TCB_T_FLAGS2    0x7fULL</div><div class='del'>-#define V_TCB_T_FLAGS2(x) ((x) &lt;&lt; S_TCB_T_FLAGS2)</div><div class='del'>-</div><div class='del'>-#define W_TCB_SND_SCALE    2</div><div class='del'>-#define S_TCB_SND_SCALE    17</div><div class='del'>-#define M_TCB_SND_SCALE    0xfULL</div><div class='del'>-#define V_TCB_SND_SCALE(x) ((x) &lt;&lt; S_TCB_SND_SCALE)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RCV_SCALE    2</div><div class='del'>-#define S_TCB_RCV_SCALE    21</div><div class='del'>-#define M_TCB_RCV_SCALE    0xfULL</div><div class='del'>-#define V_TCB_RCV_SCALE(x) ((x) &lt;&lt; S_TCB_RCV_SCALE)</div><div class='del'>-</div><div class='del'>-#define W_TCB_SND_UNA_RAW    2</div><div class='del'>-#define S_TCB_SND_UNA_RAW    25</div><div class='del'>-#define M_TCB_SND_UNA_RAW    0x7ffffffULL</div><div class='del'>-#define V_TCB_SND_UNA_RAW(x) ((x) &lt;&lt; S_TCB_SND_UNA_RAW)</div><div class='del'>-</div><div class='del'>-#define W_TCB_SND_NXT_RAW    3</div><div class='del'>-#define S_TCB_SND_NXT_RAW    20</div><div class='del'>-#define M_TCB_SND_NXT_RAW    0x7ffffffULL</div><div class='del'>-#define V_TCB_SND_NXT_RAW(x) ((x) &lt;&lt; S_TCB_SND_NXT_RAW)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RCV_NXT    4</div><div class='del'>-#define S_TCB_RCV_NXT    15</div><div class='del'>-#define M_TCB_RCV_NXT    0xffffffffULL</div><div class='del'>-#define V_TCB_RCV_NXT(x) ((x) &lt;&lt; S_TCB_RCV_NXT)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RCV_ADV    5</div><div class='del'>-#define S_TCB_RCV_ADV    15</div><div class='del'>-#define M_TCB_RCV_ADV    0xffffULL</div><div class='del'>-#define V_TCB_RCV_ADV(x) ((x) &lt;&lt; S_TCB_RCV_ADV)</div><div class='del'>-</div><div class='del'>-#define W_TCB_SND_MAX_RAW    5</div><div class='del'>-#define S_TCB_SND_MAX_RAW    31</div><div class='del'>-#define M_TCB_SND_MAX_RAW    0x7ffffffULL</div><div class='del'>-#define V_TCB_SND_MAX_RAW(x) ((x) &lt;&lt; S_TCB_SND_MAX_RAW)</div><div class='del'>-</div><div class='del'>-#define W_TCB_SND_CWND    6</div><div class='del'>-#define S_TCB_SND_CWND    26</div><div class='del'>-#define M_TCB_SND_CWND    0x7ffffffULL</div><div class='del'>-#define V_TCB_SND_CWND(x) ((x) &lt;&lt; S_TCB_SND_CWND)</div><div class='del'>-</div><div class='del'>-#define W_TCB_SND_SSTHRESH    7</div><div class='del'>-#define S_TCB_SND_SSTHRESH    21</div><div class='del'>-#define M_TCB_SND_SSTHRESH    0x7ffffffULL</div><div class='del'>-#define V_TCB_SND_SSTHRESH(x) ((x) &lt;&lt; S_TCB_SND_SSTHRESH)</div><div class='del'>-</div><div class='del'>-#define W_TCB_T_RTT_TS_RECENT_AGE    8</div><div class='del'>-#define S_TCB_T_RTT_TS_RECENT_AGE    16</div><div class='del'>-#define M_TCB_T_RTT_TS_RECENT_AGE    0xffffffffULL</div><div class='del'>-#define V_TCB_T_RTT_TS_RECENT_AGE(x) ((x) &lt;&lt; S_TCB_T_RTT_TS_RECENT_AGE)</div><div class='del'>-</div><div class='del'>-#define W_TCB_T_RTSEQ_RECENT    9</div><div class='del'>-#define S_TCB_T_RTSEQ_RECENT    16</div><div class='del'>-#define M_TCB_T_RTSEQ_RECENT    0xffffffffULL</div><div class='del'>-#define V_TCB_T_RTSEQ_RECENT(x) ((x) &lt;&lt; S_TCB_T_RTSEQ_RECENT)</div><div class='del'>-</div><div class='del'>-#define W_TCB_T_SRTT    10</div><div class='del'>-#define S_TCB_T_SRTT    16</div><div class='del'>-#define M_TCB_T_SRTT    0xffffULL</div><div class='del'>-#define V_TCB_T_SRTT(x) ((x) &lt;&lt; S_TCB_T_SRTT)</div><div class='del'>-</div><div class='del'>-#define W_TCB_T_RTTVAR    11</div><div class='del'>-#define S_TCB_T_RTTVAR    0</div><div class='del'>-#define M_TCB_T_RTTVAR    0xffffULL</div><div class='del'>-#define V_TCB_T_RTTVAR(x) ((x) &lt;&lt; S_TCB_T_RTTVAR)</div><div class='del'>-</div><div class='del'>-#define W_TCB_TS_LAST_ACK_SENT_RAW    11</div><div class='del'>-#define S_TCB_TS_LAST_ACK_SENT_RAW    16</div><div class='del'>-#define M_TCB_TS_LAST_ACK_SENT_RAW    0x7ffffffULL</div><div class='del'>-#define V_TCB_TS_LAST_ACK_SENT_RAW(x) ((x) &lt;&lt; S_TCB_TS_LAST_ACK_SENT_RAW)</div><div class='del'>-</div><div class='del'>-#define W_TCB_DIP    12</div><div class='del'>-#define S_TCB_DIP    11</div><div class='del'>-#define M_TCB_DIP    0xffffffffULL</div><div class='del'>-#define V_TCB_DIP(x) ((x) &lt;&lt; S_TCB_DIP)</div><div class='del'>-</div><div class='del'>-#define W_TCB_SIP    13</div><div class='del'>-#define S_TCB_SIP    11</div><div class='del'>-#define M_TCB_SIP    0xffffffffULL</div><div class='del'>-#define V_TCB_SIP(x) ((x) &lt;&lt; S_TCB_SIP)</div><div class='del'>-</div><div class='del'>-#define W_TCB_DP    14</div><div class='del'>-#define S_TCB_DP    11</div><div class='del'>-#define M_TCB_DP    0xffffULL</div><div class='del'>-#define V_TCB_DP(x) ((x) &lt;&lt; S_TCB_DP)</div><div class='del'>-</div><div class='del'>-#define W_TCB_SP    14</div><div class='del'>-#define S_TCB_SP    27</div><div class='del'>-#define M_TCB_SP    0xffffULL</div><div class='del'>-#define V_TCB_SP(x) ((x) &lt;&lt; S_TCB_SP)</div><div class='del'>-</div><div class='del'>-#define W_TCB_TIMESTAMP    15</div><div class='del'>-#define S_TCB_TIMESTAMP    11</div><div class='del'>-#define M_TCB_TIMESTAMP    0xffffffffULL</div><div class='del'>-#define V_TCB_TIMESTAMP(x) ((x) &lt;&lt; S_TCB_TIMESTAMP)</div><div class='del'>-</div><div class='del'>-#define W_TCB_TIMESTAMP_OFFSET    16</div><div class='del'>-#define S_TCB_TIMESTAMP_OFFSET    11</div><div class='del'>-#define M_TCB_TIMESTAMP_OFFSET    0xfULL</div><div class='del'>-#define V_TCB_TIMESTAMP_OFFSET(x) ((x) &lt;&lt; S_TCB_TIMESTAMP_OFFSET)</div><div class='del'>-</div><div class='del'>-#define W_TCB_TX_MAX    16</div><div class='del'>-#define S_TCB_TX_MAX    15</div><div class='del'>-#define M_TCB_TX_MAX    0xffffffffULL</div><div class='del'>-#define V_TCB_TX_MAX(x) ((x) &lt;&lt; S_TCB_TX_MAX)</div><div class='del'>-</div><div class='del'>-#define W_TCB_TX_HDR_PTR_RAW    17</div><div class='del'>-#define S_TCB_TX_HDR_PTR_RAW    15</div><div class='del'>-#define M_TCB_TX_HDR_PTR_RAW    0x1ffffULL</div><div class='del'>-#define V_TCB_TX_HDR_PTR_RAW(x) ((x) &lt;&lt; S_TCB_TX_HDR_PTR_RAW)</div><div class='del'>-</div><div class='del'>-#define W_TCB_TX_LAST_PTR_RAW    18</div><div class='del'>-#define S_TCB_TX_LAST_PTR_RAW    0</div><div class='del'>-#define M_TCB_TX_LAST_PTR_RAW    0x1ffffULL</div><div class='del'>-#define V_TCB_TX_LAST_PTR_RAW(x) ((x) &lt;&lt; S_TCB_TX_LAST_PTR_RAW)</div><div class='del'>-</div><div class='del'>-#define W_TCB_TX_COMPACT    18</div><div class='del'>-#define S_TCB_TX_COMPACT    17</div><div class='del'>-#define M_TCB_TX_COMPACT    0x1ULL</div><div class='del'>-#define V_TCB_TX_COMPACT(x) ((x) &lt;&lt; S_TCB_TX_COMPACT)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_COMPACT    18</div><div class='del'>-#define S_TCB_RX_COMPACT    18</div><div class='del'>-#define M_TCB_RX_COMPACT    0x1ULL</div><div class='del'>-#define V_TCB_RX_COMPACT(x) ((x) &lt;&lt; S_TCB_RX_COMPACT)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RCV_WND    18</div><div class='del'>-#define S_TCB_RCV_WND    19</div><div class='del'>-#define M_TCB_RCV_WND    0x7ffffffULL</div><div class='del'>-#define V_TCB_RCV_WND(x) ((x) &lt;&lt; S_TCB_RCV_WND)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_HDR_OFFSET    19</div><div class='del'>-#define S_TCB_RX_HDR_OFFSET    14</div><div class='del'>-#define M_TCB_RX_HDR_OFFSET    0x7ffffffULL</div><div class='del'>-#define V_TCB_RX_HDR_OFFSET(x) ((x) &lt;&lt; S_TCB_RX_HDR_OFFSET)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_FRAG0_START_IDX_RAW    20</div><div class='del'>-#define S_TCB_RX_FRAG0_START_IDX_RAW    9</div><div class='del'>-#define M_TCB_RX_FRAG0_START_IDX_RAW    0x7ffffffULL</div><div class='del'>-#define V_TCB_RX_FRAG0_START_IDX_RAW(x) ((x) &lt;&lt; S_TCB_RX_FRAG0_START_IDX_RAW)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_FRAG1_START_IDX_OFFSET    21</div><div class='del'>-#define S_TCB_RX_FRAG1_START_IDX_OFFSET    4</div><div class='del'>-#define M_TCB_RX_FRAG1_START_IDX_OFFSET    0x7ffffffULL</div><div class='del'>-#define V_TCB_RX_FRAG1_START_IDX_OFFSET(x) ((x) &lt;&lt; S_TCB_RX_FRAG1_START_IDX_OFFSET)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_FRAG0_LEN    21</div><div class='del'>-#define S_TCB_RX_FRAG0_LEN    31</div><div class='del'>-#define M_TCB_RX_FRAG0_LEN    0x7ffffffULL</div><div class='del'>-#define V_TCB_RX_FRAG0_LEN(x) ((x) &lt;&lt; S_TCB_RX_FRAG0_LEN)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_FRAG1_LEN    22</div><div class='del'>-#define S_TCB_RX_FRAG1_LEN    26</div><div class='del'>-#define M_TCB_RX_FRAG1_LEN    0x7ffffffULL</div><div class='del'>-#define V_TCB_RX_FRAG1_LEN(x) ((x) &lt;&lt; S_TCB_RX_FRAG1_LEN)</div><div class='del'>-</div><div class='del'>-#define W_TCB_NEWRENO_RECOVER    23</div><div class='del'>-#define S_TCB_NEWRENO_RECOVER    21</div><div class='del'>-#define M_TCB_NEWRENO_RECOVER    0x7ffffffULL</div><div class='del'>-#define V_TCB_NEWRENO_RECOVER(x) ((x) &lt;&lt; S_TCB_NEWRENO_RECOVER)</div><div class='del'>-</div><div class='del'>-#define W_TCB_PDU_HAVE_LEN    24</div><div class='del'>-#define S_TCB_PDU_HAVE_LEN    16</div><div class='del'>-#define M_TCB_PDU_HAVE_LEN    0x1ULL</div><div class='del'>-#define V_TCB_PDU_HAVE_LEN(x) ((x) &lt;&lt; S_TCB_PDU_HAVE_LEN)</div><div class='del'>-</div><div class='del'>-#define W_TCB_PDU_LEN    24</div><div class='del'>-#define S_TCB_PDU_LEN    17</div><div class='del'>-#define M_TCB_PDU_LEN    0xffffULL</div><div class='del'>-#define V_TCB_PDU_LEN(x) ((x) &lt;&lt; S_TCB_PDU_LEN)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_QUIESCE    25</div><div class='del'>-#define S_TCB_RX_QUIESCE    1</div><div class='del'>-#define M_TCB_RX_QUIESCE    0x1ULL</div><div class='del'>-#define V_TCB_RX_QUIESCE(x) ((x) &lt;&lt; S_TCB_RX_QUIESCE)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_PTR_RAW    25</div><div class='del'>-#define S_TCB_RX_PTR_RAW    2</div><div class='del'>-#define M_TCB_RX_PTR_RAW    0x1ffffULL</div><div class='del'>-#define V_TCB_RX_PTR_RAW(x) ((x) &lt;&lt; S_TCB_RX_PTR_RAW)</div><div class='del'>-</div><div class='del'>-#define W_TCB_CPU_NO    25</div><div class='del'>-#define S_TCB_CPU_NO    19</div><div class='del'>-#define M_TCB_CPU_NO    0x7fULL</div><div class='del'>-#define V_TCB_CPU_NO(x) ((x) &lt;&lt; S_TCB_CPU_NO)</div><div class='del'>-</div><div class='del'>-#define W_TCB_ULP_TYPE    25</div><div class='del'>-#define S_TCB_ULP_TYPE    26</div><div class='del'>-#define M_TCB_ULP_TYPE    0xfULL</div><div class='del'>-#define V_TCB_ULP_TYPE(x) ((x) &lt;&lt; S_TCB_ULP_TYPE)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_FRAG1_PTR_RAW    25</div><div class='del'>-#define S_TCB_RX_FRAG1_PTR_RAW    30</div><div class='del'>-#define M_TCB_RX_FRAG1_PTR_RAW    0x1ffffULL</div><div class='del'>-#define V_TCB_RX_FRAG1_PTR_RAW(x) ((x) &lt;&lt; S_TCB_RX_FRAG1_PTR_RAW)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_FRAG2_START_IDX_OFFSET_RAW    26</div><div class='del'>-#define S_TCB_RX_FRAG2_START_IDX_OFFSET_RAW    15</div><div class='del'>-#define M_TCB_RX_FRAG2_START_IDX_OFFSET_RAW    0x7ffffffULL</div><div class='del'>-#define V_TCB_RX_FRAG2_START_IDX_OFFSET_RAW(x) ((x) &lt;&lt; S_TCB_RX_FRAG2_START_IDX_OFFSET_RAW)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_FRAG2_PTR_RAW    27</div><div class='del'>-#define S_TCB_RX_FRAG2_PTR_RAW    10</div><div class='del'>-#define M_TCB_RX_FRAG2_PTR_RAW    0x1ffffULL</div><div class='del'>-#define V_TCB_RX_FRAG2_PTR_RAW(x) ((x) &lt;&lt; S_TCB_RX_FRAG2_PTR_RAW)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_FRAG2_LEN_RAW    27</div><div class='del'>-#define S_TCB_RX_FRAG2_LEN_RAW    27</div><div class='del'>-#define M_TCB_RX_FRAG2_LEN_RAW    0x7ffffffULL</div><div class='del'>-#define V_TCB_RX_FRAG2_LEN_RAW(x) ((x) &lt;&lt; S_TCB_RX_FRAG2_LEN_RAW)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_FRAG3_PTR_RAW    28</div><div class='del'>-#define S_TCB_RX_FRAG3_PTR_RAW    22</div><div class='del'>-#define M_TCB_RX_FRAG3_PTR_RAW    0x1ffffULL</div><div class='del'>-#define V_TCB_RX_FRAG3_PTR_RAW(x) ((x) &lt;&lt; S_TCB_RX_FRAG3_PTR_RAW)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_FRAG3_LEN_RAW    29</div><div class='del'>-#define S_TCB_RX_FRAG3_LEN_RAW    7</div><div class='del'>-#define M_TCB_RX_FRAG3_LEN_RAW    0x7ffffffULL</div><div class='del'>-#define V_TCB_RX_FRAG3_LEN_RAW(x) ((x) &lt;&lt; S_TCB_RX_FRAG3_LEN_RAW)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_FRAG3_START_IDX_OFFSET_RAW    30</div><div class='del'>-#define S_TCB_RX_FRAG3_START_IDX_OFFSET_RAW    2</div><div class='del'>-#define M_TCB_RX_FRAG3_START_IDX_OFFSET_RAW    0x7ffffffULL</div><div class='del'>-#define V_TCB_RX_FRAG3_START_IDX_OFFSET_RAW(x) ((x) &lt;&lt; S_TCB_RX_FRAG3_START_IDX_OFFSET_RAW)</div><div class='del'>-</div><div class='del'>-#define W_TCB_PDU_HDR_LEN    30</div><div class='del'>-#define S_TCB_PDU_HDR_LEN    29</div><div class='del'>-#define M_TCB_PDU_HDR_LEN    0xffULL</div><div class='del'>-#define V_TCB_PDU_HDR_LEN(x) ((x) &lt;&lt; S_TCB_PDU_HDR_LEN)</div><div class='del'>-</div><div class='del'>-#define W_TCB_SLUSH1    31</div><div class='del'>-#define S_TCB_SLUSH1    5</div><div class='del'>-#define M_TCB_SLUSH1    0x7ffffULL</div><div class='del'>-#define V_TCB_SLUSH1(x) ((x) &lt;&lt; S_TCB_SLUSH1)</div><div class='del'>-</div><div class='del'>-#define W_TCB_ULP_RAW    31</div><div class='del'>-#define S_TCB_ULP_RAW    24</div><div class='del'>-#define M_TCB_ULP_RAW    0xffULL</div><div class='del'>-#define V_TCB_ULP_RAW(x) ((x) &lt;&lt; S_TCB_ULP_RAW)</div><div class='del'>-</div><div class='del'>-#define W_TCB_DDP_RDMAP_VERSION    25</div><div class='del'>-#define S_TCB_DDP_RDMAP_VERSION    30</div><div class='del'>-#define M_TCB_DDP_RDMAP_VERSION    0x1ULL</div><div class='del'>-#define V_TCB_DDP_RDMAP_VERSION(x) ((x) &lt;&lt; S_TCB_DDP_RDMAP_VERSION)</div><div class='del'>-</div><div class='del'>-#define W_TCB_MARKER_ENABLE_RX    25</div><div class='del'>-#define S_TCB_MARKER_ENABLE_RX    31</div><div class='del'>-#define M_TCB_MARKER_ENABLE_RX    0x1ULL</div><div class='del'>-#define V_TCB_MARKER_ENABLE_RX(x) ((x) &lt;&lt; S_TCB_MARKER_ENABLE_RX)</div><div class='del'>-</div><div class='del'>-#define W_TCB_MARKER_ENABLE_TX    26</div><div class='del'>-#define S_TCB_MARKER_ENABLE_TX    0</div><div class='del'>-#define M_TCB_MARKER_ENABLE_TX    0x1ULL</div><div class='del'>-#define V_TCB_MARKER_ENABLE_TX(x) ((x) &lt;&lt; S_TCB_MARKER_ENABLE_TX)</div><div class='del'>-</div><div class='del'>-#define W_TCB_CRC_ENABLE    26</div><div class='del'>-#define S_TCB_CRC_ENABLE    1</div><div class='del'>-#define M_TCB_CRC_ENABLE    0x1ULL</div><div class='del'>-#define V_TCB_CRC_ENABLE(x) ((x) &lt;&lt; S_TCB_CRC_ENABLE)</div><div class='del'>-</div><div class='del'>-#define W_TCB_IRS_ULP    26</div><div class='del'>-#define S_TCB_IRS_ULP    2</div><div class='del'>-#define M_TCB_IRS_ULP    0x1ffULL</div><div class='del'>-#define V_TCB_IRS_ULP(x) ((x) &lt;&lt; S_TCB_IRS_ULP)</div><div class='del'>-</div><div class='del'>-#define W_TCB_ISS_ULP    26</div><div class='del'>-#define S_TCB_ISS_ULP    11</div><div class='del'>-#define M_TCB_ISS_ULP    0x1ffULL</div><div class='del'>-#define V_TCB_ISS_ULP(x) ((x) &lt;&lt; S_TCB_ISS_ULP)</div><div class='del'>-</div><div class='del'>-#define W_TCB_TX_PDU_LEN    26</div><div class='del'>-#define S_TCB_TX_PDU_LEN    20</div><div class='del'>-#define M_TCB_TX_PDU_LEN    0x3fffULL</div><div class='del'>-#define V_TCB_TX_PDU_LEN(x) ((x) &lt;&lt; S_TCB_TX_PDU_LEN)</div><div class='del'>-</div><div class='del'>-#define W_TCB_TX_PDU_OUT    27</div><div class='del'>-#define S_TCB_TX_PDU_OUT    2</div><div class='del'>-#define M_TCB_TX_PDU_OUT    0x1ULL</div><div class='del'>-#define V_TCB_TX_PDU_OUT(x) ((x) &lt;&lt; S_TCB_TX_PDU_OUT)</div><div class='del'>-</div><div class='del'>-#define W_TCB_CQ_IDX_SQ    27</div><div class='del'>-#define S_TCB_CQ_IDX_SQ    3</div><div class='del'>-#define M_TCB_CQ_IDX_SQ    0xffffULL</div><div class='del'>-#define V_TCB_CQ_IDX_SQ(x) ((x) &lt;&lt; S_TCB_CQ_IDX_SQ)</div><div class='del'>-</div><div class='del'>-#define W_TCB_CQ_IDX_RQ    27</div><div class='del'>-#define S_TCB_CQ_IDX_RQ    19</div><div class='del'>-#define M_TCB_CQ_IDX_RQ    0xffffULL</div><div class='del'>-#define V_TCB_CQ_IDX_RQ(x) ((x) &lt;&lt; S_TCB_CQ_IDX_RQ)</div><div class='del'>-</div><div class='del'>-#define W_TCB_QP_ID    28</div><div class='del'>-#define S_TCB_QP_ID    3</div><div class='del'>-#define M_TCB_QP_ID    0xffffULL</div><div class='del'>-#define V_TCB_QP_ID(x) ((x) &lt;&lt; S_TCB_QP_ID)</div><div class='del'>-</div><div class='del'>-#define W_TCB_PD_ID    28</div><div class='del'>-#define S_TCB_PD_ID    19</div><div class='del'>-#define M_TCB_PD_ID    0xffffULL</div><div class='del'>-#define V_TCB_PD_ID(x) ((x) &lt;&lt; S_TCB_PD_ID)</div><div class='del'>-</div><div class='del'>-#define W_TCB_STAG    29</div><div class='del'>-#define S_TCB_STAG    3</div><div class='del'>-#define M_TCB_STAG    0xffffffffULL</div><div class='del'>-#define V_TCB_STAG(x) ((x) &lt;&lt; S_TCB_STAG)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RQ_START    30</div><div class='del'>-#define S_TCB_RQ_START    3</div><div class='del'>-#define M_TCB_RQ_START    0x3ffffffULL</div><div class='del'>-#define V_TCB_RQ_START(x) ((x) &lt;&lt; S_TCB_RQ_START)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RQ_MSN    30</div><div class='del'>-#define S_TCB_RQ_MSN    29</div><div class='del'>-#define M_TCB_RQ_MSN    0x3ffULL</div><div class='del'>-#define V_TCB_RQ_MSN(x) ((x) &lt;&lt; S_TCB_RQ_MSN)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RQ_MAX_OFFSET    31</div><div class='del'>-#define S_TCB_RQ_MAX_OFFSET    7</div><div class='del'>-#define M_TCB_RQ_MAX_OFFSET    0xfULL</div><div class='del'>-#define V_TCB_RQ_MAX_OFFSET(x) ((x) &lt;&lt; S_TCB_RQ_MAX_OFFSET)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RQ_WRITE_PTR    31</div><div class='del'>-#define S_TCB_RQ_WRITE_PTR    11</div><div class='del'>-#define M_TCB_RQ_WRITE_PTR    0x3ffULL</div><div class='del'>-#define V_TCB_RQ_WRITE_PTR(x) ((x) &lt;&lt; S_TCB_RQ_WRITE_PTR)</div><div class='del'>-</div><div class='del'>-#define W_TCB_INB_WRITE_PERM    31</div><div class='del'>-#define S_TCB_INB_WRITE_PERM    21</div><div class='del'>-#define M_TCB_INB_WRITE_PERM    0x1ULL</div><div class='del'>-#define V_TCB_INB_WRITE_PERM(x) ((x) &lt;&lt; S_TCB_INB_WRITE_PERM)</div><div class='del'>-</div><div class='del'>-#define W_TCB_INB_READ_PERM    31</div><div class='del'>-#define S_TCB_INB_READ_PERM    22</div><div class='del'>-#define M_TCB_INB_READ_PERM    0x1ULL</div><div class='del'>-#define V_TCB_INB_READ_PERM(x) ((x) &lt;&lt; S_TCB_INB_READ_PERM)</div><div class='del'>-</div><div class='del'>-#define W_TCB_ORD_L_BIT_VLD    31</div><div class='del'>-#define S_TCB_ORD_L_BIT_VLD    23</div><div class='del'>-#define M_TCB_ORD_L_BIT_VLD    0x1ULL</div><div class='del'>-#define V_TCB_ORD_L_BIT_VLD(x) ((x) &lt;&lt; S_TCB_ORD_L_BIT_VLD)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RDMAP_OPCODE    31</div><div class='del'>-#define S_TCB_RDMAP_OPCODE    24</div><div class='del'>-#define M_TCB_RDMAP_OPCODE    0xfULL</div><div class='del'>-#define V_TCB_RDMAP_OPCODE(x) ((x) &lt;&lt; S_TCB_RDMAP_OPCODE)</div><div class='del'>-</div><div class='del'>-#define W_TCB_TX_FLUSH    31</div><div class='del'>-#define S_TCB_TX_FLUSH    28</div><div class='del'>-#define M_TCB_TX_FLUSH    0x1ULL</div><div class='del'>-#define V_TCB_TX_FLUSH(x) ((x) &lt;&lt; S_TCB_TX_FLUSH)</div><div class='del'>-</div><div class='del'>-#define W_TCB_TX_OOS_RXMT    31</div><div class='del'>-#define S_TCB_TX_OOS_RXMT    29</div><div class='del'>-#define M_TCB_TX_OOS_RXMT    0x1ULL</div><div class='del'>-#define V_TCB_TX_OOS_RXMT(x) ((x) &lt;&lt; S_TCB_TX_OOS_RXMT)</div><div class='del'>-</div><div class='del'>-#define W_TCB_TX_OOS_TXMT    31</div><div class='del'>-#define S_TCB_TX_OOS_TXMT    30</div><div class='del'>-#define M_TCB_TX_OOS_TXMT    0x1ULL</div><div class='del'>-#define V_TCB_TX_OOS_TXMT(x) ((x) &lt;&lt; S_TCB_TX_OOS_TXMT)</div><div class='del'>-</div><div class='del'>-#define W_TCB_SLUSH_AUX2    31</div><div class='del'>-#define S_TCB_SLUSH_AUX2    31</div><div class='del'>-#define M_TCB_SLUSH_AUX2    0x1ULL</div><div class='del'>-#define V_TCB_SLUSH_AUX2(x) ((x) &lt;&lt; S_TCB_SLUSH_AUX2)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_FRAG1_PTR_RAW2    25</div><div class='del'>-#define S_TCB_RX_FRAG1_PTR_RAW2    30</div><div class='del'>-#define M_TCB_RX_FRAG1_PTR_RAW2    0x1ffffULL</div><div class='del'>-#define V_TCB_RX_FRAG1_PTR_RAW2(x) ((x) &lt;&lt; S_TCB_RX_FRAG1_PTR_RAW2)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_DDP_FLAGS    26</div><div class='del'>-#define S_TCB_RX_DDP_FLAGS    15</div><div class='del'>-#define M_TCB_RX_DDP_FLAGS    0x3ffULL</div><div class='del'>-#define V_TCB_RX_DDP_FLAGS(x) ((x) &lt;&lt; S_TCB_RX_DDP_FLAGS)</div><div class='del'>-</div><div class='del'>-#define W_TCB_SLUSH_AUX3    26</div><div class='del'>-#define S_TCB_SLUSH_AUX3    31</div><div class='del'>-#define M_TCB_SLUSH_AUX3    0x1ffULL</div><div class='del'>-#define V_TCB_SLUSH_AUX3(x) ((x) &lt;&lt; S_TCB_SLUSH_AUX3)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_DDP_BUF0_OFFSET    27</div><div class='del'>-#define S_TCB_RX_DDP_BUF0_OFFSET    8</div><div class='del'>-#define M_TCB_RX_DDP_BUF0_OFFSET    0x3fffffULL</div><div class='del'>-#define V_TCB_RX_DDP_BUF0_OFFSET(x) ((x) &lt;&lt; S_TCB_RX_DDP_BUF0_OFFSET)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_DDP_BUF0_LEN    27</div><div class='del'>-#define S_TCB_RX_DDP_BUF0_LEN    30</div><div class='del'>-#define M_TCB_RX_DDP_BUF0_LEN    0x3fffffULL</div><div class='del'>-#define V_TCB_RX_DDP_BUF0_LEN(x) ((x) &lt;&lt; S_TCB_RX_DDP_BUF0_LEN)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_DDP_BUF1_OFFSET    28</div><div class='del'>-#define S_TCB_RX_DDP_BUF1_OFFSET    20</div><div class='del'>-#define M_TCB_RX_DDP_BUF1_OFFSET    0x3fffffULL</div><div class='del'>-#define V_TCB_RX_DDP_BUF1_OFFSET(x) ((x) &lt;&lt; S_TCB_RX_DDP_BUF1_OFFSET)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_DDP_BUF1_LEN    29</div><div class='del'>-#define S_TCB_RX_DDP_BUF1_LEN    10</div><div class='del'>-#define M_TCB_RX_DDP_BUF1_LEN    0x3fffffULL</div><div class='del'>-#define V_TCB_RX_DDP_BUF1_LEN(x) ((x) &lt;&lt; S_TCB_RX_DDP_BUF1_LEN)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_DDP_BUF0_TAG    30</div><div class='del'>-#define S_TCB_RX_DDP_BUF0_TAG    0</div><div class='del'>-#define M_TCB_RX_DDP_BUF0_TAG    0xffffffffULL</div><div class='del'>-#define V_TCB_RX_DDP_BUF0_TAG(x) ((x) &lt;&lt; S_TCB_RX_DDP_BUF0_TAG)</div><div class='del'>-</div><div class='del'>-#define W_TCB_RX_DDP_BUF1_TAG    31</div><div class='del'>-#define S_TCB_RX_DDP_BUF1_TAG    0</div><div class='del'>-#define M_TCB_RX_DDP_BUF1_TAG    0xffffffffULL</div><div class='del'>-#define V_TCB_RX_DDP_BUF1_TAG(x) ((x) &lt;&lt; S_TCB_RX_DDP_BUF1_TAG)</div><div class='del'>-</div><div class='del'>-#define S_TF_DACK    10</div><div class='del'>-#define V_TF_DACK(x) ((x) &lt;&lt; S_TF_DACK)</div><div class='del'>-</div><div class='del'>-#define S_TF_NAGLE    11</div><div class='del'>-#define V_TF_NAGLE(x) ((x) &lt;&lt; S_TF_NAGLE)</div><div class='del'>-</div><div class='del'>-#define S_TF_RECV_SCALE    12</div><div class='del'>-#define V_TF_RECV_SCALE(x) ((x) &lt;&lt; S_TF_RECV_SCALE)</div><div class='del'>-</div><div class='del'>-#define S_TF_RECV_TSTMP    13</div><div class='del'>-#define V_TF_RECV_TSTMP(x) ((x) &lt;&lt; S_TF_RECV_TSTMP)</div><div class='del'>-</div><div class='del'>-#define S_TF_RECV_SACK    14</div><div class='del'>-#define V_TF_RECV_SACK(x) ((x) &lt;&lt; S_TF_RECV_SACK)</div><div class='del'>-</div><div class='del'>-#define S_TF_TURBO    15</div><div class='del'>-#define V_TF_TURBO(x) ((x) &lt;&lt; S_TF_TURBO)</div><div class='del'>-</div><div class='del'>-#define S_TF_KEEPALIVE    16</div><div class='del'>-#define V_TF_KEEPALIVE(x) ((x) &lt;&lt; S_TF_KEEPALIVE)</div><div class='del'>-</div><div class='del'>-#define S_TF_TCAM_BYPASS    17</div><div class='del'>-#define V_TF_TCAM_BYPASS(x) ((x) &lt;&lt; S_TF_TCAM_BYPASS)</div><div class='del'>-</div><div class='del'>-#define S_TF_CORE_FIN    18</div><div class='del'>-#define V_TF_CORE_FIN(x) ((x) &lt;&lt; S_TF_CORE_FIN)</div><div class='del'>-</div><div class='del'>-#define S_TF_CORE_MORE    19</div><div class='del'>-#define V_TF_CORE_MORE(x) ((x) &lt;&lt; S_TF_CORE_MORE)</div><div class='del'>-</div><div class='del'>-#define S_TF_MIGRATING    20</div><div class='del'>-#define V_TF_MIGRATING(x) ((x) &lt;&lt; S_TF_MIGRATING)</div><div class='del'>-</div><div class='del'>-#define S_TF_ACTIVE_OPEN    21</div><div class='del'>-#define V_TF_ACTIVE_OPEN(x) ((x) &lt;&lt; S_TF_ACTIVE_OPEN)</div><div class='del'>-</div><div class='del'>-#define S_TF_ASK_MODE    22</div><div class='del'>-#define V_TF_ASK_MODE(x) ((x) &lt;&lt; S_TF_ASK_MODE)</div><div class='del'>-</div><div class='del'>-#define S_TF_NON_OFFLOAD    23</div><div class='del'>-#define V_TF_NON_OFFLOAD(x) ((x) &lt;&lt; S_TF_NON_OFFLOAD)</div><div class='del'>-</div><div class='del'>-#define S_TF_MOD_SCHD    24</div><div class='del'>-#define V_TF_MOD_SCHD(x) ((x) &lt;&lt; S_TF_MOD_SCHD)</div><div class='del'>-</div><div class='del'>-#define S_TF_MOD_SCHD_REASON0    25</div><div class='del'>-#define V_TF_MOD_SCHD_REASON0(x) ((x) &lt;&lt; S_TF_MOD_SCHD_REASON0)</div><div class='del'>-</div><div class='del'>-#define S_TF_MOD_SCHD_REASON1    26</div><div class='del'>-#define V_TF_MOD_SCHD_REASON1(x) ((x) &lt;&lt; S_TF_MOD_SCHD_REASON1)</div><div class='del'>-</div><div class='del'>-#define S_TF_MOD_SCHD_RX    27</div><div class='del'>-#define V_TF_MOD_SCHD_RX(x) ((x) &lt;&lt; S_TF_MOD_SCHD_RX)</div><div class='del'>-</div><div class='del'>-#define S_TF_CORE_PUSH    28</div><div class='del'>-#define V_TF_CORE_PUSH(x) ((x) &lt;&lt; S_TF_CORE_PUSH)</div><div class='del'>-</div><div class='del'>-#define S_TF_RCV_COALESCE_ENABLE    29</div><div class='del'>-#define V_TF_RCV_COALESCE_ENABLE(x) ((x) &lt;&lt; S_TF_RCV_COALESCE_ENABLE)</div><div class='del'>-</div><div class='del'>-#define S_TF_RCV_COALESCE_PUSH    30</div><div class='del'>-#define V_TF_RCV_COALESCE_PUSH(x) ((x) &lt;&lt; S_TF_RCV_COALESCE_PUSH)</div><div class='del'>-</div><div class='del'>-#define S_TF_RCV_COALESCE_LAST_PSH    31</div><div class='del'>-#define V_TF_RCV_COALESCE_LAST_PSH(x) ((x) &lt;&lt; S_TF_RCV_COALESCE_LAST_PSH)</div><div class='del'>-</div><div class='del'>-#define S_TF_RCV_COALESCE_HEARTBEAT    32</div><div class='del'>-#define V_TF_RCV_COALESCE_HEARTBEAT(x) ((x) &lt;&lt; S_TF_RCV_COALESCE_HEARTBEAT)</div><div class='del'>-</div><div class='del'>-#define S_TF_HALF_CLOSE    33</div><div class='del'>-#define V_TF_HALF_CLOSE(x) ((x) &lt;&lt; S_TF_HALF_CLOSE)</div><div class='del'>-</div><div class='del'>-#define S_TF_DACK_MSS    34</div><div class='del'>-#define V_TF_DACK_MSS(x) ((x) &lt;&lt; S_TF_DACK_MSS)</div><div class='del'>-</div><div class='del'>-#define S_TF_CCTRL_SEL0    35</div><div class='del'>-#define V_TF_CCTRL_SEL0(x) ((x) &lt;&lt; S_TF_CCTRL_SEL0)</div><div class='del'>-</div><div class='del'>-#define S_TF_CCTRL_SEL1    36</div><div class='del'>-#define V_TF_CCTRL_SEL1(x) ((x) &lt;&lt; S_TF_CCTRL_SEL1)</div><div class='del'>-</div><div class='del'>-#define S_TF_TCP_NEWRENO_FAST_RECOVERY    37</div><div class='del'>-#define V_TF_TCP_NEWRENO_FAST_RECOVERY(x) ((x) &lt;&lt; S_TF_TCP_NEWRENO_FAST_RECOVERY)</div><div class='del'>-</div><div class='del'>-#define S_TF_TX_PACE_AUTO    38</div><div class='del'>-#define V_TF_TX_PACE_AUTO(x) ((x) &lt;&lt; S_TF_TX_PACE_AUTO)</div><div class='del'>-</div><div class='del'>-#define S_TF_PEER_FIN_HELD    39</div><div class='del'>-#define V_TF_PEER_FIN_HELD(x) ((x) &lt;&lt; S_TF_PEER_FIN_HELD)</div><div class='del'>-</div><div class='del'>-#define S_TF_CORE_URG    40</div><div class='del'>-#define V_TF_CORE_URG(x) ((x) &lt;&lt; S_TF_CORE_URG)</div><div class='del'>-</div><div class='del'>-#define S_TF_RDMA_ERROR    41</div><div class='del'>-#define V_TF_RDMA_ERROR(x) ((x) &lt;&lt; S_TF_RDMA_ERROR)</div><div class='del'>-</div><div class='del'>-#define S_TF_SSWS_DISABLED    42</div><div class='del'>-#define V_TF_SSWS_DISABLED(x) ((x) &lt;&lt; S_TF_SSWS_DISABLED)</div><div class='del'>-</div><div class='del'>-#define S_TF_DUPACK_COUNT_ODD    43</div><div class='del'>-#define V_TF_DUPACK_COUNT_ODD(x) ((x) &lt;&lt; S_TF_DUPACK_COUNT_ODD)</div><div class='del'>-</div><div class='del'>-#define S_TF_TX_CHANNEL    44</div><div class='del'>-#define V_TF_TX_CHANNEL(x) ((x) &lt;&lt; S_TF_TX_CHANNEL)</div><div class='del'>-</div><div class='del'>-#define S_TF_RX_CHANNEL    45</div><div class='del'>-#define V_TF_RX_CHANNEL(x) ((x) &lt;&lt; S_TF_RX_CHANNEL)</div><div class='del'>-</div><div class='del'>-#define S_TF_TX_PACE_FIXED    46</div><div class='del'>-#define V_TF_TX_PACE_FIXED(x) ((x) &lt;&lt; S_TF_TX_PACE_FIXED)</div><div class='del'>-</div><div class='del'>-#define S_TF_RDMA_FLM_ERROR    47</div><div class='del'>-#define V_TF_RDMA_FLM_ERROR(x) ((x) &lt;&lt; S_TF_RDMA_FLM_ERROR)</div><div class='del'>-</div><div class='del'>-#define S_TF_RX_FLOW_CONTROL_DISABLE    48</div><div class='del'>-#define V_TF_RX_FLOW_CONTROL_DISABLE(x) ((x) &lt;&lt; S_TF_RX_FLOW_CONTROL_DISABLE)</div><div class='del'>-</div><div class='del'>-#endif /* _TCB_DEFS_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb4/Kconfig b/drivers/infiniband/hw/cxgb4/Kconfig<br/>index e0522a5d5a06..9e2b2c348afd 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/Kconfig?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/Kconfig</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/Kconfig?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/cxgb4/Kconfig</a></div><div class='hunk'>@@ -1,10 +1,11 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='ctx'> config INFINIBAND_CXGB4</div><div class='ctx'> 	tristate "Chelsio T4/T5 RDMA Driver"</div><div class='ctx'> 	depends on CHELSIO_T4 &amp;&amp; INET</div><div class='ctx'> 	depends on INFINIBAND_ADDR_TRANS</div><div class='ctx'> 	select CHELSIO_LIB</div><div class='ctx'> 	select GENERIC_ALLOCATOR</div><div class='del'>-	---help---</div><div class='add'>+	help</div><div class='ctx'> 	  This is an iWARP/RDMA driver for the Chelsio T4 and T5</div><div class='ctx'> 	  1GbE, 10GbE adapters and T5 40GbE adapter.</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb4/Makefile b/drivers/infiniband/hw/cxgb4/Makefile<br/>index 31a87d90a40b..291d259d2319 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/Makefile?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/Makefile</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/Makefile?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/cxgb4/Makefile</a></div><div class='hunk'>@@ -1,3 +1,4 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='ctx'> ccflags-y := -I $(srctree)/drivers/net/ethernet/chelsio/cxgb4</div><div class='ctx'> ccflags-y += -I $(srctree)/drivers/net/ethernet/chelsio/libcxgb</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c<br/>index 0f3b1193d5f8..b3b45c49077d 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/cm.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/cm.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/cm.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/cxgb4/cm.c</a></div><div class='hunk'>@@ -77,9 +77,9 @@ static int enable_ecn;</div><div class='ctx'> module_param(enable_ecn, int, 0644);</div><div class='ctx'> MODULE_PARM_DESC(enable_ecn, "Enable ECN (default=0/disabled)");</div><div class='ctx'> </div><div class='del'>-static int dack_mode = 1;</div><div class='add'>+static int dack_mode;</div><div class='ctx'> module_param(dack_mode, int, 0644);</div><div class='del'>-MODULE_PARM_DESC(dack_mode, "Delayed ack mode (default=1)");</div><div class='add'>+MODULE_PARM_DESC(dack_mode, "Delayed ack mode (default=0)");</div><div class='ctx'> </div><div class='ctx'> uint c4iw_max_read_depth = 32;</div><div class='ctx'> module_param(c4iw_max_read_depth, int, 0644);</div><div class='hunk'>@@ -145,7 +145,7 @@ static void connect_reply_upcall(struct c4iw_ep *ep, int status);</div><div class='ctx'> static int sched(struct c4iw_dev *dev, struct sk_buff *skb);</div><div class='ctx'> </div><div class='ctx'> static LIST_HEAD(timeout_list);</div><div class='del'>-static spinlock_t timeout_lock;</div><div class='add'>+static DEFINE_SPINLOCK(timeout_lock);</div><div class='ctx'> </div><div class='ctx'> static void deref_cm_id(struct c4iw_ep_common *epc)</div><div class='ctx'> {</div><div class='hunk'>@@ -191,7 +191,7 @@ static void start_ep_timer(struct c4iw_ep *ep)</div><div class='ctx'> static int stop_ep_timer(struct c4iw_ep *ep)</div><div class='ctx'> {</div><div class='ctx'> 	pr_debug("ep %p stopping\n", ep);</div><div class='del'>-	del_timer_sync(&amp;ep-&gt;timer);</div><div class='add'>+	timer_delete_sync(&amp;ep-&gt;timer);</div><div class='ctx'> 	if (!test_and_set_bit(TIMEOUT, &amp;ep-&gt;com.flags)) {</div><div class='ctx'> 		c4iw_put_ep(&amp;ep-&gt;com);</div><div class='ctx'> 		return 0;</div><div class='hunk'>@@ -495,7 +495,6 @@ static int _put_ep_safe(struct c4iw_dev *dev, struct sk_buff *skb)</div><div class='ctx'> </div><div class='ctx'> 	ep = *((struct c4iw_ep **)(skb-&gt;cb + 2 * sizeof(void *)));</div><div class='ctx'> 	release_ep_resources(ep);</div><div class='del'>-	kfree_skb(skb);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -506,7 +505,6 @@ static int _put_pass_ep_safe(struct c4iw_dev *dev, struct sk_buff *skb)</div><div class='ctx'> 	ep = *((struct c4iw_ep **)(skb-&gt;cb + 2 * sizeof(void *)));</div><div class='ctx'> 	c4iw_put_ep(&amp;ep-&gt;parent_ep-&gt;com);</div><div class='ctx'> 	release_ep_resources(ep);</div><div class='del'>-	kfree_skb(skb);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -736,7 +734,7 @@ static int send_connect(struct c4iw_ep *ep)</div><div class='ctx'> 				   &amp;ep-&gt;com.remote_addr;</div><div class='ctx'> 	int ret;</div><div class='ctx'> 	enum chip_type adapter_type = ep-&gt;com.dev-&gt;rdev.lldi.adapter_type;</div><div class='del'>-	u32 isn = (prandom_u32() &amp; ~7UL) - 1;</div><div class='add'>+	u32 isn = (get_random_u32() &amp; ~7UL) - 1;</div><div class='ctx'> 	struct net_device *netdev;</div><div class='ctx'> 	u64 params;</div><div class='ctx'> </div><div class='hunk'>@@ -953,7 +951,7 @@ static int send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb,</div><div class='ctx'> 	mpalen = sizeof(*mpa) + ep-&gt;plen;</div><div class='ctx'> 	if (mpa_rev_to_use == 2)</div><div class='ctx'> 		mpalen += sizeof(struct mpa_v2_conn_params);</div><div class='del'>-	wrlen = roundup(mpalen + sizeof *req, 16);</div><div class='add'>+	wrlen = roundup(mpalen + sizeof(*req), 16);</div><div class='ctx'> 	skb = get_skb(skb, wrlen, GFP_KERNEL);</div><div class='ctx'> 	if (!skb) {</div><div class='ctx'> 		connect_reply_upcall(ep, -ENOMEM);</div><div class='hunk'>@@ -997,8 +995,9 @@ static int send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (mpa_rev_to_use == 2) {</div><div class='del'>-		mpa-&gt;private_data_size = htons(ntohs(mpa-&gt;private_data_size) +</div><div class='del'>-					       sizeof (struct mpa_v2_conn_params));</div><div class='add'>+		mpa-&gt;private_data_size =</div><div class='add'>+			htons(ntohs(mpa-&gt;private_data_size) +</div><div class='add'>+			      sizeof(struct mpa_v2_conn_params));</div><div class='ctx'> 		pr_debug("initiator ird %u ord %u\n", ep-&gt;ird,</div><div class='ctx'> 			 ep-&gt;ord);</div><div class='ctx'> 		mpa_v2_params.ird = htons((u16)ep-&gt;ird);</div><div class='hunk'>@@ -1057,7 +1056,7 @@ static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen)</div><div class='ctx'> 	mpalen = sizeof(*mpa) + plen;</div><div class='ctx'> 	if (ep-&gt;mpa_attr.version == 2 &amp;&amp; ep-&gt;mpa_attr.enhanced_rdma_conn)</div><div class='ctx'> 		mpalen += sizeof(struct mpa_v2_conn_params);</div><div class='del'>-	wrlen = roundup(mpalen + sizeof *req, 16);</div><div class='add'>+	wrlen = roundup(mpalen + sizeof(*req), 16);</div><div class='ctx'> </div><div class='ctx'> 	skb = get_skb(NULL, wrlen, GFP_KERNEL);</div><div class='ctx'> 	if (!skb) {</div><div class='hunk'>@@ -1088,8 +1087,9 @@ static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen)</div><div class='ctx'> </div><div class='ctx'> 	if (ep-&gt;mpa_attr.version == 2 &amp;&amp; ep-&gt;mpa_attr.enhanced_rdma_conn) {</div><div class='ctx'> 		mpa-&gt;flags |= MPA_ENHANCED_RDMA_CONN;</div><div class='del'>-		mpa-&gt;private_data_size = htons(ntohs(mpa-&gt;private_data_size) +</div><div class='del'>-					       sizeof (struct mpa_v2_conn_params));</div><div class='add'>+		mpa-&gt;private_data_size =</div><div class='add'>+			htons(ntohs(mpa-&gt;private_data_size) +</div><div class='add'>+			      sizeof(struct mpa_v2_conn_params));</div><div class='ctx'> 		mpa_v2_params.ird = htons(((u16)ep-&gt;ird) |</div><div class='ctx'> 					  (peer2peer ? MPA_V2_PEER2PEER_MODEL :</div><div class='ctx'> 					   0));</div><div class='hunk'>@@ -1136,7 +1136,7 @@ static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen)</div><div class='ctx'> 	mpalen = sizeof(*mpa) + plen;</div><div class='ctx'> 	if (ep-&gt;mpa_attr.version == 2 &amp;&amp; ep-&gt;mpa_attr.enhanced_rdma_conn)</div><div class='ctx'> 		mpalen += sizeof(struct mpa_v2_conn_params);</div><div class='del'>-	wrlen = roundup(mpalen + sizeof *req, 16);</div><div class='add'>+	wrlen = roundup(mpalen + sizeof(*req), 16);</div><div class='ctx'> </div><div class='ctx'> 	skb = get_skb(NULL, wrlen, GFP_KERNEL);</div><div class='ctx'> 	if (!skb) {</div><div class='hunk'>@@ -1171,8 +1171,9 @@ static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen)</div><div class='ctx'> </div><div class='ctx'> 	if (ep-&gt;mpa_attr.version == 2 &amp;&amp; ep-&gt;mpa_attr.enhanced_rdma_conn) {</div><div class='ctx'> 		mpa-&gt;flags |= MPA_ENHANCED_RDMA_CONN;</div><div class='del'>-		mpa-&gt;private_data_size = htons(ntohs(mpa-&gt;private_data_size) +</div><div class='del'>-					       sizeof (struct mpa_v2_conn_params));</div><div class='add'>+		mpa-&gt;private_data_size =</div><div class='add'>+			htons(ntohs(mpa-&gt;private_data_size) +</div><div class='add'>+			      sizeof(struct mpa_v2_conn_params));</div><div class='ctx'> 		mpa_v2_params.ird = htons((u16)ep-&gt;ird);</div><div class='ctx'> 		mpa_v2_params.ord = htons((u16)ep-&gt;ord);</div><div class='ctx'> 		if (peer2peer &amp;&amp; (ep-&gt;mpa_attr.p2p_type !=</div><div class='hunk'>@@ -1221,6 +1222,8 @@ static int act_establish(struct c4iw_dev *dev, struct sk_buff *skb)</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	ep = lookup_atid(t, atid);</div><div class='add'>+	if (!ep)</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	pr_debug("ep %p tid %u snd_isn %u rcv_isn %u\n", ep, tid,</div><div class='ctx'> 		 be32_to_cpu(req-&gt;snd_isn), be32_to_cpu(req-&gt;rcv_isn));</div><div class='hunk'>@@ -1964,6 +1967,9 @@ static int send_fw_act_open_req(struct c4iw_ep *ep, unsigned int atid)</div><div class='ctx'> 	int win;</div><div class='ctx'> </div><div class='ctx'> 	skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);</div><div class='add'>+	if (!skb)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='ctx'> 	req = __skb_put_zero(skb, sizeof(*req));</div><div class='ctx'> 	req-&gt;op_compl = htonl(WR_OP_V(FW_OFLD_CONNECTION_WR));</div><div class='ctx'> 	req-&gt;len16_pkd = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)));</div><div class='hunk'>@@ -2080,7 +2086,7 @@ static int import_ep(struct c4iw_ep *ep, int iptype, __u8 *peer_ip,</div><div class='ctx'> 	err = -ENOMEM;</div><div class='ctx'> 	if (n-&gt;dev-&gt;flags &amp; IFF_LOOPBACK) {</div><div class='ctx'> 		if (iptype == 4)</div><div class='del'>-			pdev = ip_dev_find(&amp;init_net, *(__be32 *)peer_ip);</div><div class='add'>+			pdev = __ip_dev_find(&amp;init_net, *(__be32 *)peer_ip, false);</div><div class='ctx'> 		else if (IS_ENABLED(CONFIG_IPV6))</div><div class='ctx'> 			for_each_netdev(&amp;init_net, pdev) {</div><div class='ctx'> 				if (ipv6_chk_addr(&amp;init_net,</div><div class='hunk'>@@ -2095,12 +2101,12 @@ static int import_ep(struct c4iw_ep *ep, int iptype, __u8 *peer_ip,</div><div class='ctx'> 			err = -ENODEV;</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		}</div><div class='add'>+		if (is_vlan_dev(pdev))</div><div class='add'>+			pdev = vlan_dev_real_dev(pdev);</div><div class='ctx'> 		ep-&gt;l2t = cxgb4_l2t_get(cdev-&gt;rdev.lldi.l2t,</div><div class='ctx'> 					n, pdev, rt_tos2priority(tos));</div><div class='del'>-		if (!ep-&gt;l2t) {</div><div class='del'>-			dev_put(pdev);</div><div class='add'>+		if (!ep-&gt;l2t)</div><div class='ctx'> 			goto out;</div><div class='del'>-		}</div><div class='ctx'> 		ep-&gt;mtu = pdev-&gt;mtu;</div><div class='ctx'> 		ep-&gt;tx_chan = cxgb4_port_chan(pdev);</div><div class='ctx'> 		ep-&gt;smac_idx = ((struct port_info *)netdev_priv(pdev))-&gt;smt_idx;</div><div class='hunk'>@@ -2113,7 +2119,6 @@ static int import_ep(struct c4iw_ep *ep, int iptype, __u8 *peer_ip,</div><div class='ctx'> 		ep-&gt;rss_qid = cdev-&gt;rdev.lldi.rxq_ids[</div><div class='ctx'> 			cxgb4_port_idx(pdev) * step];</div><div class='ctx'> 		set_tcp_window(ep, (struct port_info *)netdev_priv(pdev));</div><div class='del'>-		dev_put(pdev);</div><div class='ctx'> 	} else {</div><div class='ctx'> 		pdev = get_real_dev(n-&gt;dev);</div><div class='ctx'> 		ep-&gt;l2t = cxgb4_l2t_get(cdev-&gt;rdev.lldi.l2t,</div><div class='hunk'>@@ -2275,6 +2280,9 @@ static int act_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb)</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='ctx'> 	ep = lookup_atid(t, atid);</div><div class='add'>+	if (!ep)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='ctx'> 	la = (struct sockaddr_in *)&amp;ep-&gt;com.local_addr;</div><div class='ctx'> 	ra = (struct sockaddr_in *)&amp;ep-&gt;com.remote_addr;</div><div class='ctx'> 	la6 = (struct sockaddr_in6 *)&amp;ep-&gt;com.local_addr;</div><div class='hunk'>@@ -2421,20 +2429,6 @@ static int accept_cr(struct c4iw_ep *ep, struct sk_buff *skb,</div><div class='ctx'> 	enum chip_type adapter_type = ep-&gt;com.dev-&gt;rdev.lldi.adapter_type;</div><div class='ctx'> </div><div class='ctx'> 	pr_debug("ep %p tid %u\n", ep, ep-&gt;hwtid);</div><div class='del'>-</div><div class='del'>-	skb_get(skb);</div><div class='del'>-	rpl = cplhdr(skb);</div><div class='del'>-	if (!is_t4(adapter_type)) {</div><div class='del'>-		skb_trim(skb, roundup(sizeof(*rpl5), 16));</div><div class='del'>-		rpl5 = (void *)rpl;</div><div class='del'>-		INIT_TP_WR(rpl5, ep-&gt;hwtid);</div><div class='del'>-	} else {</div><div class='del'>-		skb_trim(skb, sizeof(*rpl));</div><div class='del'>-		INIT_TP_WR(rpl, ep-&gt;hwtid);</div><div class='del'>-	}</div><div class='del'>-	OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,</div><div class='del'>-						    ep-&gt;hwtid));</div><div class='del'>-</div><div class='ctx'> 	cxgb_best_mtu(ep-&gt;com.dev-&gt;rdev.lldi.mtus, ep-&gt;mtu, &amp;mtu_idx,</div><div class='ctx'> 		      enable_tcp_timestamps &amp;&amp; req-&gt;tcpopt.tstamp,</div><div class='ctx'> 		      (ep-&gt;com.remote_addr.ss_family == AF_INET) ? 0 : 1);</div><div class='hunk'>@@ -2480,17 +2474,25 @@ static int accept_cr(struct c4iw_ep *ep, struct sk_buff *skb,</div><div class='ctx'> 		if (tcph-&gt;ece &amp;&amp; tcph-&gt;cwr)</div><div class='ctx'> 			opt2 |= CCTRL_ECN_V(1);</div><div class='ctx'> 	}</div><div class='del'>-	if (CHELSIO_CHIP_VERSION(adapter_type) &gt; CHELSIO_T4) {</div><div class='del'>-		u32 isn = (prandom_u32() &amp; ~7UL) - 1;</div><div class='add'>+</div><div class='add'>+	if (!is_t4(adapter_type)) {</div><div class='add'>+		u32 isn = (get_random_u32() &amp; ~7UL) - 1;</div><div class='add'>+</div><div class='add'>+		skb = get_skb(skb, roundup(sizeof(*rpl5), 16), GFP_KERNEL);</div><div class='add'>+		rpl5 = __skb_put_zero(skb, roundup(sizeof(*rpl5), 16));</div><div class='add'>+		rpl = (void *)rpl5;</div><div class='add'>+		INIT_TP_WR_CPL(rpl5, CPL_PASS_ACCEPT_RPL, ep-&gt;hwtid);</div><div class='ctx'> 		opt2 |= T5_OPT_2_VALID_F;</div><div class='ctx'> 		opt2 |= CONG_CNTRL_V(CONG_ALG_TAHOE);</div><div class='ctx'> 		opt2 |= T5_ISS_F;</div><div class='del'>-		rpl5 = (void *)rpl;</div><div class='del'>-		memset(&amp;rpl5-&gt;iss, 0, roundup(sizeof(*rpl5)-sizeof(*rpl), 16));</div><div class='ctx'> 		if (peer2peer)</div><div class='ctx'> 			isn += 4;</div><div class='ctx'> 		rpl5-&gt;iss = cpu_to_be32(isn);</div><div class='ctx'> 		pr_debug("iss %u\n", be32_to_cpu(rpl5-&gt;iss));</div><div class='add'>+	} else {</div><div class='add'>+		skb = get_skb(skb, sizeof(*rpl), GFP_KERNEL);</div><div class='add'>+		rpl = __skb_put_zero(skb, sizeof(*rpl));</div><div class='add'>+		INIT_TP_WR_CPL(rpl, CPL_PASS_ACCEPT_RPL, ep-&gt;hwtid);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	rpl-&gt;opt0 = cpu_to_be64(opt0);</div><div class='hunk'>@@ -2681,6 +2683,9 @@ static int pass_establish(struct c4iw_dev *dev, struct sk_buff *skb)</div><div class='ctx'> 	u16 tcp_opt = ntohs(req-&gt;tcp_opt);</div><div class='ctx'> </div><div class='ctx'> 	ep = get_ep_from_tid(dev, tid);</div><div class='add'>+	if (!ep)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='ctx'> 	pr_debug("ep %p tid %u\n", ep, ep-&gt;hwtid);</div><div class='ctx'> 	ep-&gt;snd_seq = be32_to_cpu(req-&gt;snd_isn);</div><div class='ctx'> 	ep-&gt;rcv_seq = be32_to_cpu(req-&gt;rcv_isn);</div><div class='hunk'>@@ -2884,14 +2889,13 @@ static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb)</div><div class='ctx'> 	case MORIBUND:</div><div class='ctx'> 	case CLOSING:</div><div class='ctx'> 		stop_ep_timer(ep);</div><div class='del'>-		/*FALLTHROUGH*/</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case FPDU_MODE:</div><div class='ctx'> 		if (ep-&gt;com.qp &amp;&amp; ep-&gt;com.qp-&gt;srq) {</div><div class='ctx'> 			srqidx = ABORT_RSS_SRQIDX_G(</div><div class='ctx'> 					be32_to_cpu(req-&gt;srqidx_status));</div><div class='ctx'> 			if (srqidx) {</div><div class='del'>-				complete_cached_srq_buffers(ep,</div><div class='del'>-							    req-&gt;srqidx_status);</div><div class='add'>+				complete_cached_srq_buffers(ep, srqidx);</div><div class='ctx'> 			} else {</div><div class='ctx'> 				/* Hold ep ref until finish_peer_abort() */</div><div class='ctx'> 				c4iw_get_ep(&amp;ep-&gt;com);</div><div class='hunk'>@@ -3035,6 +3039,10 @@ static int terminate(struct c4iw_dev *dev, struct sk_buff *skb)</div><div class='ctx'> 				       C4IW_QP_ATTR_NEXT_STATE, &amp;attrs, 1);</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='add'>+		/* As per draft-hilland-iwarp-verbs-v1.0, sec 6.2.3,</div><div class='add'>+		 * when entering the TERM state the RNIC MUST initiate a CLOSE.</div><div class='add'>+		 */</div><div class='add'>+		c4iw_ep_disconnect(ep, 1, GFP_KERNEL);</div><div class='ctx'> 		c4iw_put_ep(&amp;ep-&gt;com);</div><div class='ctx'> 	} else</div><div class='ctx'> 		pr_warn("TERM received tid %u no ep/qp\n", tid);</div><div class='hunk'>@@ -3230,17 +3238,22 @@ static int pick_local_ipaddrs(struct c4iw_dev *dev, struct iw_cm_id *cm_id)</div><div class='ctx'> 	int found = 0;</div><div class='ctx'> 	struct sockaddr_in *laddr = (struct sockaddr_in *)&amp;cm_id-&gt;m_local_addr;</div><div class='ctx'> 	struct sockaddr_in *raddr = (struct sockaddr_in *)&amp;cm_id-&gt;m_remote_addr;</div><div class='add'>+	const struct in_ifaddr *ifa;</div><div class='ctx'> </div><div class='ctx'> 	ind = in_dev_get(dev-&gt;rdev.lldi.ports[0]);</div><div class='ctx'> 	if (!ind)</div><div class='ctx'> 		return -EADDRNOTAVAIL;</div><div class='del'>-	for_primary_ifa(ind) {</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	in_dev_for_each_ifa_rcu(ifa, ind) {</div><div class='add'>+		if (ifa-&gt;ifa_flags &amp; IFA_F_SECONDARY)</div><div class='add'>+			continue;</div><div class='ctx'> 		laddr-&gt;sin_addr.s_addr = ifa-&gt;ifa_address;</div><div class='ctx'> 		raddr-&gt;sin_addr.s_addr = ifa-&gt;ifa_address;</div><div class='ctx'> 		found = 1;</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='del'>-	endfor_ifa(ind);</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+</div><div class='ctx'> 	in_dev_put(ind);</div><div class='ctx'> 	return found ? 0 : -EADDRNOTAVAIL;</div><div class='ctx'> }</div><div class='hunk'>@@ -3273,7 +3286,7 @@ static int get_lladdr(struct net_device *dev, struct in6_addr *addr,</div><div class='ctx'> </div><div class='ctx'> static int pick_local_ip6addrs(struct c4iw_dev *dev, struct iw_cm_id *cm_id)</div><div class='ctx'> {</div><div class='del'>-	struct in6_addr uninitialized_var(addr);</div><div class='add'>+	struct in6_addr addr;</div><div class='ctx'> 	struct sockaddr_in6 *la6 = (struct sockaddr_in6 *)&amp;cm_id-&gt;m_local_addr;</div><div class='ctx'> 	struct sockaddr_in6 *ra6 = (struct sockaddr_in6 *)&amp;cm_id-&gt;m_remote_addr;</div><div class='ctx'> </div><div class='hunk'>@@ -3373,7 +3386,7 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)</div><div class='ctx'> 		if (raddr-&gt;sin_addr.s_addr == htonl(INADDR_ANY)) {</div><div class='ctx'> 			err = pick_local_ipaddrs(dev, cm_id);</div><div class='ctx'> 			if (err)</div><div class='del'>-				goto fail2;</div><div class='add'>+				goto fail3;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='ctx'> 		/* find a route */</div><div class='hunk'>@@ -3395,7 +3408,7 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)</div><div class='ctx'> 		if (ipv6_addr_type(&amp;raddr6-&gt;sin6_addr) == IPV6_ADDR_ANY) {</div><div class='ctx'> 			err = pick_local_ip6addrs(dev, cm_id);</div><div class='ctx'> 			if (err)</div><div class='del'>-				goto fail2;</div><div class='add'>+				goto fail3;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='ctx'> 		/* find a route */</div><div class='hunk'>@@ -3601,13 +3614,14 @@ int c4iw_destroy_listen(struct iw_cm_id *cm_id)</div><div class='ctx'> 	    ep-&gt;com.local_addr.ss_family == AF_INET) {</div><div class='ctx'> 		err = cxgb4_remove_server_filter(</div><div class='ctx'> 			ep-&gt;com.dev-&gt;rdev.lldi.ports[0], ep-&gt;stid,</div><div class='del'>-			ep-&gt;com.dev-&gt;rdev.lldi.rxq_ids[0], 0);</div><div class='add'>+			ep-&gt;com.dev-&gt;rdev.lldi.rxq_ids[0], false);</div><div class='ctx'> 	} else {</div><div class='ctx'> 		struct sockaddr_in6 *sin6;</div><div class='ctx'> 		c4iw_init_wr_wait(ep-&gt;com.wr_waitp);</div><div class='ctx'> 		err = cxgb4_remove_server(</div><div class='ctx'> 				ep-&gt;com.dev-&gt;rdev.lldi.ports[0], ep-&gt;stid,</div><div class='del'>-				ep-&gt;com.dev-&gt;rdev.lldi.rxq_ids[0], 0);</div><div class='add'>+				ep-&gt;com.dev-&gt;rdev.lldi.rxq_ids[0],</div><div class='add'>+				ep-&gt;com.local_addr.ss_family == AF_INET6);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto done;</div><div class='ctx'> 		err = c4iw_wait_for_reply(&amp;ep-&gt;com.dev-&gt;rdev, ep-&gt;com.wr_waitp,</div><div class='hunk'>@@ -3750,7 +3764,7 @@ static void active_ofld_conn_reply(struct c4iw_dev *dev, struct sk_buff *skb,</div><div class='ctx'> 			send_fw_act_open_req(ep, atid);</div><div class='ctx'> 			return;</div><div class='ctx'> 		}</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case FW_EADDRINUSE:</div><div class='ctx'> 		set_bit(ACT_RETRY_INUSE, &amp;ep-&gt;com.history);</div><div class='ctx'> 		if (ep-&gt;retry_count++ &lt; ACT_OPEN_RETRY_COUNT) {</div><div class='hunk'>@@ -3868,8 +3882,8 @@ static int read_tcb_rpl(struct c4iw_dev *dev, struct sk_buff *skb)</div><div class='ctx'> 		return 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ep-&gt;srqe_idx = t4_tcb_get_field32(tcb, TCB_RQ_START_W, TCB_RQ_START_W,</div><div class='del'>-			TCB_RQ_START_S);</div><div class='add'>+	ep-&gt;srqe_idx = t4_tcb_get_field32(tcb, TCB_RQ_START_W, TCB_RQ_START_M,</div><div class='add'>+					  TCB_RQ_START_S);</div><div class='ctx'> cleanup:</div><div class='ctx'> 	pr_debug("ep %p tid %u %016x\n", ep, ep-&gt;hwtid, ep-&gt;srqe_idx);</div><div class='ctx'> </div><div class='hunk'>@@ -4140,6 +4154,10 @@ static int rx_pkt(struct c4iw_dev *dev, struct sk_buff *skb)</div><div class='ctx'> </div><div class='ctx'> 	if (neigh-&gt;dev-&gt;flags &amp; IFF_LOOPBACK) {</div><div class='ctx'> 		pdev = ip_dev_find(&amp;init_net, iph-&gt;daddr);</div><div class='add'>+		if (!pdev) {</div><div class='add'>+			pr_err("%s - failed to find device!\n", __func__);</div><div class='add'>+			goto free_dst;</div><div class='add'>+		}</div><div class='ctx'> 		e = cxgb4_l2t_get(dev-&gt;rdev.lldi.l2t, neigh,</div><div class='ctx'> 				    pdev, 0);</div><div class='ctx'> 		pi = (struct port_info *)netdev_priv(pdev);</div><div class='hunk'>@@ -4309,7 +4327,7 @@ static DECLARE_WORK(skb_work, process_work);</div><div class='ctx'> </div><div class='ctx'> static void ep_timeout(struct timer_list *t)</div><div class='ctx'> {</div><div class='del'>-	struct c4iw_ep *ep = from_timer(ep, t, timer);</div><div class='add'>+	struct c4iw_ep *ep = timer_container_of(ep, t, timer);</div><div class='ctx'> 	int kickit = 0;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock(&amp;timeout_lock);</div><div class='hunk'>@@ -4442,7 +4460,6 @@ c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS] = {</div><div class='ctx'> </div><div class='ctx'> int __init c4iw_cm_init(void)</div><div class='ctx'> {</div><div class='del'>-	spin_lock_init(&amp;timeout_lock);</div><div class='ctx'> 	skb_queue_head_init(&amp;rxq);</div><div class='ctx'> </div><div class='ctx'> 	workq = alloc_ordered_workqueue("iw_cxgb4", WQ_MEM_RECLAIM);</div><div class='hunk'>@@ -4455,6 +4472,5 @@ int __init c4iw_cm_init(void)</div><div class='ctx'> void c4iw_cm_term(void)</div><div class='ctx'> {</div><div class='ctx'> 	WARN_ON(!list_empty(&amp;timeout_list));</div><div class='del'>-	flush_workqueue(workq);</div><div class='ctx'> 	destroy_workqueue(workq);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c<br/>index 52ce586621c6..14ced7b667fa 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/cq.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/cq.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/cq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/cxgb4/cq.c</a></div><div class='hunk'>@@ -34,16 +34,15 @@</div><div class='ctx'> </div><div class='ctx'> #include "iw_cxgb4.h"</div><div class='ctx'> </div><div class='del'>-static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,</div><div class='del'>-		      struct c4iw_dev_ucontext *uctx, struct sk_buff *skb,</div><div class='del'>-		      struct c4iw_wr_wait *wr_waitp)</div><div class='add'>+static void destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,</div><div class='add'>+		       struct c4iw_dev_ucontext *uctx, struct sk_buff *skb,</div><div class='add'>+		       struct c4iw_wr_wait *wr_waitp)</div><div class='ctx'> {</div><div class='ctx'> 	struct fw_ri_res_wr *res_wr;</div><div class='ctx'> 	struct fw_ri_res *res;</div><div class='ctx'> 	int wr_len;</div><div class='del'>-	int ret;</div><div class='ctx'> </div><div class='del'>-	wr_len = sizeof *res_wr + sizeof *res;</div><div class='add'>+	wr_len = sizeof(*res_wr) + sizeof(*res);</div><div class='ctx'> 	set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0);</div><div class='ctx'> </div><div class='ctx'> 	res_wr = __skb_put_zero(skb, wr_len);</div><div class='hunk'>@@ -59,14 +58,13 @@ static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,</div><div class='ctx'> 	res-&gt;u.cq.iqid = cpu_to_be32(cq-&gt;cqid);</div><div class='ctx'> </div><div class='ctx'> 	c4iw_init_wr_wait(wr_waitp);</div><div class='del'>-	ret = c4iw_ref_send_wait(rdev, skb, wr_waitp, 0, 0, __func__);</div><div class='add'>+	c4iw_ref_send_wait(rdev, skb, wr_waitp, 0, 0, __func__);</div><div class='ctx'> </div><div class='ctx'> 	kfree(cq-&gt;sw_queue);</div><div class='ctx'> 	dma_free_coherent(&amp;(rdev-&gt;lldi.pdev-&gt;dev),</div><div class='ctx'> 			  cq-&gt;memsize, cq-&gt;queue,</div><div class='ctx'> 			  dma_unmap_addr(cq, mapping));</div><div class='ctx'> 	c4iw_put_cqid(rdev, cq-&gt;cqid, uctx);</div><div class='del'>-	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,</div><div class='hunk'>@@ -104,7 +102,6 @@ static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,</div><div class='ctx'> 		goto err3;</div><div class='ctx'> 	}</div><div class='ctx'> 	dma_unmap_addr_set(cq, mapping, cq-&gt;dma_addr);</div><div class='del'>-	memset(cq-&gt;queue, 0, cq-&gt;memsize);</div><div class='ctx'> </div><div class='ctx'> 	if (user &amp;&amp; ucontext-&gt;is_32b_cqe) {</div><div class='ctx'> 		cq-&gt;qp_errp = &amp;((struct t4_status_page *)</div><div class='hunk'>@@ -117,7 +114,7 @@ static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* build fw_ri_res_wr */</div><div class='del'>-	wr_len = sizeof *res_wr + sizeof *res;</div><div class='add'>+	wr_len = sizeof(*res_wr) + sizeof(*res);</div><div class='ctx'> </div><div class='ctx'> 	skb = alloc_skb(wr_len, GFP_KERNEL);</div><div class='ctx'> 	if (!skb) {</div><div class='hunk'>@@ -757,7 +754,7 @@ skip_cqe:</div><div class='ctx'> static int __c4iw_poll_cq_one(struct c4iw_cq *chp, struct c4iw_qp *qhp,</div><div class='ctx'> 			      struct ib_wc *wc, struct c4iw_srq *srq)</div><div class='ctx'> {</div><div class='del'>-	struct t4_cqe uninitialized_var(cqe);</div><div class='add'>+	struct t4_cqe cqe;</div><div class='ctx'> 	struct t4_wq *wq = qhp ? &amp;qhp-&gt;wq : NULL;</div><div class='ctx'> 	u32 credit = 0;</div><div class='ctx'> 	u8 cqe_flushed;</div><div class='hunk'>@@ -770,7 +767,7 @@ static int __c4iw_poll_cq_one(struct c4iw_cq *chp, struct c4iw_qp *qhp,</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='ctx'> 	wc-&gt;wr_id = cookie;</div><div class='del'>-	wc-&gt;qp = qhp ? &amp;qhp-&gt;ibqp : NULL;</div><div class='add'>+	wc-&gt;qp = &amp;qhp-&gt;ibqp;</div><div class='ctx'> 	wc-&gt;vendor_err = CQE_STATUS(&amp;cqe);</div><div class='ctx'> 	wc-&gt;wc_flags = 0;</div><div class='ctx'> </div><div class='hunk'>@@ -970,6 +967,12 @@ int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)</div><div class='ctx'> 	return !err || err == -ENODATA ? npolled : err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+void c4iw_cq_rem_ref(struct c4iw_cq *chp)</div><div class='add'>+{</div><div class='add'>+	if (refcount_dec_and_test(&amp;chp-&gt;refcnt))</div><div class='add'>+		complete(&amp;chp-&gt;cq_rel_comp);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct c4iw_cq *chp;</div><div class='hunk'>@@ -979,8 +982,8 @@ int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)</div><div class='ctx'> 	chp = to_c4iw_cq(ib_cq);</div><div class='ctx'> </div><div class='ctx'> 	xa_erase_irq(&amp;chp-&gt;rhp-&gt;cqs, chp-&gt;cq.cqid);</div><div class='del'>-	atomic_dec(&amp;chp-&gt;refcnt);</div><div class='del'>-	wait_event(chp-&gt;wait, !atomic_read(&amp;chp-&gt;refcnt));</div><div class='add'>+	c4iw_cq_rem_ref(chp);</div><div class='add'>+	wait_for_completion(&amp;chp-&gt;cq_rel_comp);</div><div class='ctx'> </div><div class='ctx'> 	ucontext = rdma_udata_to_drv_context(udata, struct c4iw_ucontext,</div><div class='ctx'> 					     ibucontext);</div><div class='hunk'>@@ -988,18 +991,18 @@ int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)</div><div class='ctx'> 		   ucontext ? &amp;ucontext-&gt;uctx : &amp;chp-&gt;cq.rdev-&gt;uctx,</div><div class='ctx'> 		   chp-&gt;destroy_skb, chp-&gt;wr_waitp);</div><div class='ctx'> 	c4iw_put_wr_wait(chp-&gt;wr_waitp);</div><div class='del'>-	kfree(chp);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,</div><div class='del'>-			     const struct ib_cq_init_attr *attr,</div><div class='del'>-			     struct ib_udata *udata)</div><div class='add'>+int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		   struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='add'>+	struct ib_udata *udata = &amp;attrs-&gt;driver_udata;</div><div class='add'>+	struct ib_device *ibdev = ibcq-&gt;device;</div><div class='ctx'> 	int entries = attr-&gt;cqe;</div><div class='ctx'> 	int vector = attr-&gt;comp_vector;</div><div class='del'>-	struct c4iw_dev *rhp;</div><div class='del'>-	struct c4iw_cq *chp;</div><div class='add'>+	struct c4iw_dev *rhp = to_c4iw_dev(ibcq-&gt;device);</div><div class='add'>+	struct c4iw_cq *chp = to_c4iw_cq(ibcq);</div><div class='ctx'> 	struct c4iw_create_cq ucmd;</div><div class='ctx'> 	struct c4iw_create_cq_resp uresp;</div><div class='ctx'> 	int ret, wr_len;</div><div class='hunk'>@@ -1010,22 +1013,19 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,</div><div class='ctx'> </div><div class='ctx'> 	pr_debug("ib_dev %p entries %d\n", ibdev, entries);</div><div class='ctx'> 	if (attr-&gt;flags)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='del'>-	rhp = to_c4iw_dev(ibdev);</div><div class='add'>+	if (entries &lt; 1 || entries &gt; ibdev-&gt;attrs.max_cqe)</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	if (vector &gt;= rhp-&gt;rdev.lldi.nciq)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	if (udata) {</div><div class='ctx'> 		if (udata-&gt;inlen &lt; sizeof(ucmd))</div><div class='ctx'> 			ucontext-&gt;is_32b_cqe = 1;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	chp = kzalloc(sizeof(*chp), GFP_KERNEL);</div><div class='del'>-	if (!chp)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='ctx'> 	chp-&gt;wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL);</div><div class='ctx'> 	if (!chp-&gt;wr_waitp) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='hunk'>@@ -1087,18 +1087,18 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,</div><div class='ctx'> 	chp-&gt;ibcq.cqe = entries - 2;</div><div class='ctx'> 	spin_lock_init(&amp;chp-&gt;lock);</div><div class='ctx'> 	spin_lock_init(&amp;chp-&gt;comp_handler_lock);</div><div class='del'>-	atomic_set(&amp;chp-&gt;refcnt, 1);</div><div class='del'>-	init_waitqueue_head(&amp;chp-&gt;wait);</div><div class='add'>+	refcount_set(&amp;chp-&gt;refcnt, 1);</div><div class='add'>+	init_completion(&amp;chp-&gt;cq_rel_comp);</div><div class='ctx'> 	ret = xa_insert_irq(&amp;rhp-&gt;cqs, chp-&gt;cq.cqid, chp, GFP_KERNEL);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err_destroy_cq;</div><div class='ctx'> </div><div class='ctx'> 	if (ucontext) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='del'>-		mm = kmalloc(sizeof *mm, GFP_KERNEL);</div><div class='add'>+		mm = kmalloc(sizeof(*mm), GFP_KERNEL);</div><div class='ctx'> 		if (!mm)</div><div class='ctx'> 			goto err_remove_handle;</div><div class='del'>-		mm2 = kmalloc(sizeof *mm2, GFP_KERNEL);</div><div class='add'>+		mm2 = kmalloc(sizeof(*mm2), GFP_KERNEL);</div><div class='ctx'> 		if (!mm2)</div><div class='ctx'> 			goto err_free_mm;</div><div class='ctx'> </div><div class='hunk'>@@ -1126,19 +1126,26 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,</div><div class='ctx'> 			goto err_free_mm2;</div><div class='ctx'> </div><div class='ctx'> 		mm-&gt;key = uresp.key;</div><div class='del'>-		mm-&gt;addr = virt_to_phys(chp-&gt;cq.queue);</div><div class='add'>+		mm-&gt;addr = 0;</div><div class='add'>+		mm-&gt;vaddr = chp-&gt;cq.queue;</div><div class='add'>+		mm-&gt;dma_addr = chp-&gt;cq.dma_addr;</div><div class='ctx'> 		mm-&gt;len = chp-&gt;cq.memsize;</div><div class='add'>+		insert_flag_to_mmap(&amp;rhp-&gt;rdev, mm, mm-&gt;addr);</div><div class='ctx'> 		insert_mmap(ucontext, mm);</div><div class='ctx'> </div><div class='ctx'> 		mm2-&gt;key = uresp.gts_key;</div><div class='ctx'> 		mm2-&gt;addr = chp-&gt;cq.bar2_pa;</div><div class='ctx'> 		mm2-&gt;len = PAGE_SIZE;</div><div class='add'>+		mm2-&gt;vaddr = NULL;</div><div class='add'>+		mm2-&gt;dma_addr = 0;</div><div class='add'>+		insert_flag_to_mmap(&amp;rhp-&gt;rdev, mm2, mm2-&gt;addr);</div><div class='ctx'> 		insert_mmap(ucontext, mm2);</div><div class='ctx'> 	}</div><div class='del'>-	pr_debug("cqid 0x%0x chp %p size %u memsize %zu, dma_addr 0x%0llx\n",</div><div class='del'>-		 chp-&gt;cq.cqid, chp, chp-&gt;cq.size,</div><div class='del'>-		 chp-&gt;cq.memsize, (unsigned long long)chp-&gt;cq.dma_addr);</div><div class='del'>-	return &amp;chp-&gt;ibcq;</div><div class='add'>+</div><div class='add'>+	pr_debug("cqid 0x%0x chp %p size %u memsize %zu, dma_addr %pad\n",</div><div class='add'>+		 chp-&gt;cq.cqid, chp, chp-&gt;cq.size, chp-&gt;cq.memsize,</div><div class='add'>+		 &amp;chp-&gt;cq.dma_addr);</div><div class='add'>+	return 0;</div><div class='ctx'> err_free_mm2:</div><div class='ctx'> 	kfree(mm2);</div><div class='ctx'> err_free_mm:</div><div class='hunk'>@@ -1154,8 +1161,7 @@ err_free_skb:</div><div class='ctx'> err_free_wr_wait:</div><div class='ctx'> 	c4iw_put_wr_wait(chp-&gt;wr_waitp);</div><div class='ctx'> err_free_chp:</div><div class='del'>-	kfree(chp);</div><div class='del'>-	return ERR_PTR(ret);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c<br/>index 4c0d925c5ff5..d892f55febe2 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/device.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/device.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/device.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/cxgb4/device.c</a></div><div class='hunk'>@@ -242,10 +242,13 @@ static void set_ep_sin6_addrs(struct c4iw_ep *ep,</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int dump_qp(struct c4iw_qp *qp, struct c4iw_debugfs_data *qpd)</div><div class='add'>+static int dump_qp(unsigned long id, struct c4iw_qp *qp,</div><div class='add'>+		   struct c4iw_debugfs_data *qpd)</div><div class='ctx'> {</div><div class='ctx'> 	int space;</div><div class='ctx'> 	int cc;</div><div class='add'>+	if (id != qp-&gt;wq.sq.qid)</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='ctx'> 	space = qpd-&gt;bufsize - qpd-&gt;pos - 1;</div><div class='ctx'> 	if (space == 0)</div><div class='hunk'>@@ -327,7 +330,7 @@ static int qp_open(struct inode *inode, struct file *file)</div><div class='ctx'> 	unsigned long index;</div><div class='ctx'> 	int count = 1;</div><div class='ctx'> </div><div class='del'>-	qpd = kmalloc(sizeof *qpd, GFP_KERNEL);</div><div class='add'>+	qpd = kmalloc(sizeof(*qpd), GFP_KERNEL);</div><div class='ctx'> 	if (!qpd)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='hunk'>@@ -350,7 +353,7 @@ static int qp_open(struct inode *inode, struct file *file)</div><div class='ctx'> </div><div class='ctx'> 	xa_lock_irq(&amp;qpd-&gt;devp-&gt;qps);</div><div class='ctx'> 	xa_for_each(&amp;qpd-&gt;devp-&gt;qps, index, qp)</div><div class='del'>-		dump_qp(qp, qpd);</div><div class='add'>+		dump_qp(index, qp, qpd);</div><div class='ctx'> 	xa_unlock_irq(&amp;qpd-&gt;devp-&gt;qps);</div><div class='ctx'> </div><div class='ctx'> 	qpd-&gt;buf[qpd-&gt;pos++] = 0;</div><div class='hunk'>@@ -421,7 +424,7 @@ static int stag_open(struct inode *inode, struct file *file)</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> 	int count = 1;</div><div class='ctx'> </div><div class='del'>-	stagd = kmalloc(sizeof *stagd, GFP_KERNEL);</div><div class='add'>+	stagd = kmalloc(sizeof(*stagd), GFP_KERNEL);</div><div class='ctx'> 	if (!stagd) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='ctx'> 		goto out;</div><div class='hunk'>@@ -902,8 +905,7 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev)</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> err_free_status_page_and_wr_log:</div><div class='del'>-	if (c4iw_wr_log &amp;&amp; rdev-&gt;wr_log)</div><div class='del'>-		kfree(rdev-&gt;wr_log);</div><div class='add'>+	kfree(rdev-&gt;wr_log);</div><div class='ctx'> 	free_page((unsigned long)rdev-&gt;status_page);</div><div class='ctx'> destroy_ocqp_pool:</div><div class='ctx'> 	c4iw_ocqp_pool_destroy(rdev);</div><div class='hunk'>@@ -950,6 +952,7 @@ void c4iw_dealloc(struct uld_ctx *ctx)</div><div class='ctx'> static void c4iw_remove(struct uld_ctx *ctx)</div><div class='ctx'> {</div><div class='ctx'> 	pr_debug("c4iw_dev %p\n", ctx-&gt;dev);</div><div class='add'>+	debugfs_remove_recursive(ctx-&gt;dev-&gt;debugfs_root);</div><div class='ctx'> 	c4iw_unregister_device(ctx-&gt;dev);</div><div class='ctx'> 	c4iw_dealloc(ctx);</div><div class='ctx'> }</div><div class='hunk'>@@ -1075,7 +1078,7 @@ static void *c4iw_uld_add(const struct cxgb4_lld_info *infop)</div><div class='ctx'> 		pr_info("Chelsio T4/T5 RDMA Driver - version %s\n",</div><div class='ctx'> 			DRV_VERSION);</div><div class='ctx'> </div><div class='del'>-	ctx = kzalloc(sizeof *ctx, GFP_KERNEL);</div><div class='add'>+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);</div><div class='ctx'> 	if (!ctx) {</div><div class='ctx'> 		ctx = ERR_PTR(-ENOMEM);</div><div class='ctx'> 		goto out;</div><div class='hunk'>@@ -1110,8 +1113,10 @@ static inline struct sk_buff *copy_gl_to_skb_pkt(const struct pkt_gl *gl,</div><div class='ctx'> 	 * The math here assumes sizeof cpl_pass_accept_req &gt;= sizeof</div><div class='ctx'> 	 * cpl_rx_pkt.</div><div class='ctx'> 	 */</div><div class='del'>-	skb = alloc_skb(gl-&gt;tot_len + sizeof(struct cpl_pass_accept_req) +</div><div class='del'>-			sizeof(struct rss_header) - pktshift, GFP_ATOMIC);</div><div class='add'>+	skb = alloc_skb(size_add(gl-&gt;tot_len,</div><div class='add'>+				 sizeof(struct cpl_pass_accept_req) +</div><div class='add'>+				 sizeof(struct rss_header)) - pktshift,</div><div class='add'>+			GFP_ATOMIC);</div><div class='ctx'> 	if (unlikely(!skb))</div><div class='ctx'> 		return NULL;</div><div class='ctx'> </div><div class='hunk'>@@ -1223,9 +1228,8 @@ static int c4iw_uld_state_change(void *handle, enum cxgb4_state new_state)</div><div class='ctx'> 		if (!ctx-&gt;dev) {</div><div class='ctx'> 			ctx-&gt;dev = c4iw_alloc(&amp;ctx-&gt;lldi);</div><div class='ctx'> 			if (IS_ERR(ctx-&gt;dev)) {</div><div class='del'>-				pr_err("%s: initialization failed: %ld\n",</div><div class='del'>-				       pci_name(ctx-&gt;lldi.pdev),</div><div class='del'>-				       PTR_ERR(ctx-&gt;dev));</div><div class='add'>+				pr_err("%s: initialization failed: %pe\n",</div><div class='add'>+				       pci_name(ctx-&gt;lldi.pdev), ctx-&gt;dev);</div><div class='ctx'> 				ctx-&gt;dev = NULL;</div><div class='ctx'> 				break;</div><div class='ctx'> 			}</div><div class='hunk'>@@ -1243,10 +1247,9 @@ static int c4iw_uld_state_change(void *handle, enum cxgb4_state new_state)</div><div class='ctx'> 	case CXGB4_STATE_START_RECOVERY:</div><div class='ctx'> 		pr_info("%s: Fatal Error\n", pci_name(ctx-&gt;lldi.pdev));</div><div class='ctx'> 		if (ctx-&gt;dev) {</div><div class='del'>-			struct ib_event event;</div><div class='add'>+			struct ib_event event = {};</div><div class='ctx'> </div><div class='ctx'> 			ctx-&gt;dev-&gt;rdev.flags |= T4_FATAL_ERROR;</div><div class='del'>-			memset(&amp;event, 0, sizeof event);</div><div class='ctx'> 			event.event  = IB_EVENT_DEVICE_FATAL;</div><div class='ctx'> 			event.device = &amp;ctx-&gt;dev-&gt;ibdev;</div><div class='ctx'> 			ib_dispatch_event(&amp;event);</div><div class='hunk'>@@ -1559,7 +1562,6 @@ static void __exit c4iw_exit_module(void)</div><div class='ctx'> 		kfree(ctx);</div><div class='ctx'> 	}</div><div class='ctx'> 	mutex_unlock(&amp;dev_mutex);</div><div class='del'>-	flush_workqueue(reg_workq);</div><div class='ctx'> 	destroy_workqueue(reg_workq);</div><div class='ctx'> 	cxgb4_unregister_uld(CXGB4_ULD_RDMA);</div><div class='ctx'> 	c4iw_cm_term();</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb4/ev.c b/drivers/infiniband/hw/cxgb4/ev.c<br/>index 4cd877bd2f56..34211a533d5c 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/ev.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/ev.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/ev.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/cxgb4/ev.c</a></div><div class='hunk'>@@ -151,7 +151,7 @@ void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	c4iw_qp_add_ref(&amp;qhp-&gt;ibqp);</div><div class='del'>-	atomic_inc(&amp;chp-&gt;refcnt);</div><div class='add'>+	refcount_inc(&amp;chp-&gt;refcnt);</div><div class='ctx'> 	xa_unlock_irq(&amp;dev-&gt;qps);</div><div class='ctx'> </div><div class='ctx'> 	/* Bad incoming write */</div><div class='hunk'>@@ -213,8 +213,7 @@ void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe)</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='ctx'> done:</div><div class='del'>-	if (atomic_dec_and_test(&amp;chp-&gt;refcnt))</div><div class='del'>-		wake_up(&amp;chp-&gt;wait);</div><div class='add'>+	c4iw_cq_rem_ref(chp);</div><div class='ctx'> 	c4iw_qp_rem_ref(&amp;qhp-&gt;ibqp);</div><div class='ctx'> out:</div><div class='ctx'> 	return;</div><div class='hunk'>@@ -228,14 +227,13 @@ int c4iw_ev_handler(struct c4iw_dev *dev, u32 qid)</div><div class='ctx'> 	xa_lock_irqsave(&amp;dev-&gt;cqs, flag);</div><div class='ctx'> 	chp = xa_load(&amp;dev-&gt;cqs, qid);</div><div class='ctx'> 	if (chp) {</div><div class='del'>-		atomic_inc(&amp;chp-&gt;refcnt);</div><div class='add'>+		refcount_inc(&amp;chp-&gt;refcnt);</div><div class='ctx'> 		xa_unlock_irqrestore(&amp;dev-&gt;cqs, flag);</div><div class='ctx'> 		t4_clear_cq_armed(&amp;chp-&gt;cq);</div><div class='ctx'> 		spin_lock_irqsave(&amp;chp-&gt;comp_handler_lock, flag);</div><div class='ctx'> 		(*chp-&gt;ibcq.comp_handler)(&amp;chp-&gt;ibcq, chp-&gt;ibcq.cq_context);</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;chp-&gt;comp_handler_lock, flag);</div><div class='del'>-		if (atomic_dec_and_test(&amp;chp-&gt;refcnt))</div><div class='del'>-			wake_up(&amp;chp-&gt;wait);</div><div class='add'>+		c4iw_cq_rem_ref(chp);</div><div class='ctx'> 	} else {</div><div class='ctx'> 		pr_debug("unknown cqid 0x%x\n", qid);</div><div class='ctx'> 		xa_unlock_irqrestore(&amp;dev-&gt;cqs, flag);</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb4/id_table.c b/drivers/infiniband/hw/cxgb4/id_table.c<br/>index 724d23297b35..e2188b335e76 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/id_table.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/id_table.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/id_table.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/cxgb4/id_table.c</a></div><div class='hunk'>@@ -54,12 +54,12 @@ u32 c4iw_id_alloc(struct c4iw_id_table *alloc)</div><div class='ctx'> </div><div class='ctx'> 	if (obj &lt; alloc-&gt;max) {</div><div class='ctx'> 		if (alloc-&gt;flags &amp; C4IW_ID_TABLE_F_RANDOM)</div><div class='del'>-			alloc-&gt;last += prandom_u32() % RANDOM_SKIP;</div><div class='add'>+			alloc-&gt;last += get_random_u32_below(RANDOM_SKIP);</div><div class='ctx'> 		else</div><div class='ctx'> 			alloc-&gt;last = obj + 1;</div><div class='ctx'> 		if (alloc-&gt;last &gt;= alloc-&gt;max)</div><div class='ctx'> 			alloc-&gt;last = 0;</div><div class='del'>-		set_bit(obj, alloc-&gt;table);</div><div class='add'>+		__set_bit(obj, alloc-&gt;table);</div><div class='ctx'> 		obj += alloc-&gt;start;</div><div class='ctx'> 	} else</div><div class='ctx'> 		obj = -1;</div><div class='hunk'>@@ -75,37 +75,32 @@ void c4iw_id_free(struct c4iw_id_table *alloc, u32 obj)</div><div class='ctx'> 	obj -= alloc-&gt;start;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;alloc-&gt;lock, flags);</div><div class='del'>-	clear_bit(obj, alloc-&gt;table);</div><div class='add'>+	__clear_bit(obj, alloc-&gt;table);</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;alloc-&gt;lock, flags);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int c4iw_id_table_alloc(struct c4iw_id_table *alloc, u32 start, u32 num,</div><div class='ctx'> 			u32 reserved, u32 flags)</div><div class='ctx'> {</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='ctx'> 	alloc-&gt;start = start;</div><div class='ctx'> 	alloc-&gt;flags = flags;</div><div class='ctx'> 	if (flags &amp; C4IW_ID_TABLE_F_RANDOM)</div><div class='del'>-		alloc-&gt;last = prandom_u32() % RANDOM_SKIP;</div><div class='add'>+		alloc-&gt;last = get_random_u32_below(RANDOM_SKIP);</div><div class='ctx'> 	else</div><div class='ctx'> 		alloc-&gt;last = 0;</div><div class='del'>-	alloc-&gt;max  = num;</div><div class='add'>+	alloc-&gt;max = num;</div><div class='ctx'> 	spin_lock_init(&amp;alloc-&gt;lock);</div><div class='del'>-	alloc-&gt;table = kmalloc_array(BITS_TO_LONGS(num), sizeof(long),</div><div class='del'>-				     GFP_KERNEL);</div><div class='add'>+	alloc-&gt;table = bitmap_zalloc(num, GFP_KERNEL);</div><div class='ctx'> 	if (!alloc-&gt;table)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	bitmap_zero(alloc-&gt;table, num);</div><div class='ctx'> 	if (!(alloc-&gt;flags &amp; C4IW_ID_TABLE_F_EMPTY))</div><div class='del'>-		for (i = 0; i &lt; reserved; ++i)</div><div class='del'>-			set_bit(i, alloc-&gt;table);</div><div class='add'>+		bitmap_set(alloc-&gt;table, 0, reserved);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void c4iw_id_table_free(struct c4iw_id_table *alloc)</div><div class='ctx'> {</div><div class='del'>-	kfree(alloc-&gt;table);</div><div class='add'>+	bitmap_free(alloc-&gt;table);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h<br/>index 916ef982172e..e17c1252536b 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/iw_cxgb4.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/iw_cxgb4.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/iw_cxgb4.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/cxgb4/iw_cxgb4.h</a></div><div class='hunk'>@@ -314,7 +314,6 @@ enum db_state {</div><div class='ctx'> struct c4iw_dev {</div><div class='ctx'> 	struct ib_device ibdev;</div><div class='ctx'> 	struct c4iw_rdev rdev;</div><div class='del'>-	u32 device_cap_flags;</div><div class='ctx'> 	struct xarray cqs;</div><div class='ctx'> 	struct xarray qps;</div><div class='ctx'> 	struct xarray mrs;</div><div class='hunk'>@@ -341,11 +340,6 @@ static inline struct c4iw_dev *to_c4iw_dev(struct ib_device *ibdev)</div><div class='ctx'> 	return container_of(ibdev, struct c4iw_dev, ibdev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline struct c4iw_dev *rdev_to_c4iw_dev(struct c4iw_rdev *rdev)</div><div class='del'>-{</div><div class='del'>-	return container_of(rdev, struct c4iw_dev, rdev);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static inline struct c4iw_cq *get_chp(struct c4iw_dev *rhp, u32 cqid)</div><div class='ctx'> {</div><div class='ctx'> 	return xa_load(&amp;rhp-&gt;cqs, cqid);</div><div class='hunk'>@@ -432,8 +426,8 @@ struct c4iw_cq {</div><div class='ctx'> 	struct t4_cq cq;</div><div class='ctx'> 	spinlock_t lock;</div><div class='ctx'> 	spinlock_t comp_handler_lock;</div><div class='del'>-	atomic_t refcnt;</div><div class='del'>-	wait_queue_head_t wait;</div><div class='add'>+	refcount_t refcnt;</div><div class='add'>+	struct completion cq_rel_comp;</div><div class='ctx'> 	struct c4iw_wr_wait *wr_waitp;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -490,13 +484,13 @@ struct c4iw_qp {</div><div class='ctx'> 	struct t4_wq wq;</div><div class='ctx'> 	spinlock_t lock;</div><div class='ctx'> 	struct mutex mutex;</div><div class='del'>-	struct kref kref;</div><div class='ctx'> 	wait_queue_head_t wait;</div><div class='ctx'> 	int sq_sig_all;</div><div class='ctx'> 	struct c4iw_srq *srq;</div><div class='del'>-	struct work_struct free_work;</div><div class='ctx'> 	struct c4iw_ucontext *ucontext;</div><div class='ctx'> 	struct c4iw_wr_wait *wr_waitp;</div><div class='add'>+	struct completion qp_rel_comp;</div><div class='add'>+	refcount_t qp_refcnt;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp)</div><div class='hunk'>@@ -538,11 +532,21 @@ static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c)</div><div class='ctx'> 	return container_of(c, struct c4iw_ucontext, ibucontext);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+enum {</div><div class='add'>+	CXGB4_MMAP_BAR,</div><div class='add'>+	CXGB4_MMAP_BAR_WC,</div><div class='add'>+	CXGB4_MMAP_CONTIG,</div><div class='add'>+	CXGB4_MMAP_NON_CONTIG,</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct c4iw_mm_entry {</div><div class='ctx'> 	struct list_head entry;</div><div class='ctx'> 	u64 addr;</div><div class='ctx'> 	u32 key;</div><div class='add'>+	void *vaddr;</div><div class='add'>+	dma_addr_t dma_addr;</div><div class='ctx'> 	unsigned len;</div><div class='add'>+	u8 mmap_flag;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static inline struct c4iw_mm_entry *remove_mmap(struct c4iw_ucontext *ucontext,</div><div class='hunk'>@@ -567,6 +571,32 @@ static inline struct c4iw_mm_entry *remove_mmap(struct c4iw_ucontext *ucontext,</div><div class='ctx'> 	return NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static inline void insert_flag_to_mmap(struct c4iw_rdev *rdev,</div><div class='add'>+				       struct c4iw_mm_entry *mm, u64 addr)</div><div class='add'>+{</div><div class='add'>+	if (addr &gt;= pci_resource_start(rdev-&gt;lldi.pdev, 0) &amp;&amp;</div><div class='add'>+	    (addr &lt; (pci_resource_start(rdev-&gt;lldi.pdev, 0) +</div><div class='add'>+		    pci_resource_len(rdev-&gt;lldi.pdev, 0))))</div><div class='add'>+		mm-&gt;mmap_flag = CXGB4_MMAP_BAR;</div><div class='add'>+	else if (addr &gt;= pci_resource_start(rdev-&gt;lldi.pdev, 2) &amp;&amp;</div><div class='add'>+		 (addr &lt; (pci_resource_start(rdev-&gt;lldi.pdev, 2) +</div><div class='add'>+			 pci_resource_len(rdev-&gt;lldi.pdev, 2)))) {</div><div class='add'>+		if (addr &gt;= rdev-&gt;oc_mw_pa) {</div><div class='add'>+			mm-&gt;mmap_flag = CXGB4_MMAP_BAR_WC;</div><div class='add'>+		} else {</div><div class='add'>+			if (is_t4(rdev-&gt;lldi.adapter_type))</div><div class='add'>+				mm-&gt;mmap_flag = CXGB4_MMAP_BAR;</div><div class='add'>+			else</div><div class='add'>+				mm-&gt;mmap_flag = CXGB4_MMAP_BAR_WC;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		if (addr)</div><div class='add'>+			mm-&gt;mmap_flag = CXGB4_MMAP_CONTIG;</div><div class='add'>+		else</div><div class='add'>+			mm-&gt;mmap_flag = CXGB4_MMAP_NON_CONTIG;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static inline void insert_mmap(struct c4iw_ucontext *ucontext,</div><div class='ctx'> 			       struct c4iw_mm_entry *mm)</div><div class='ctx'> {</div><div class='hunk'>@@ -659,12 +689,6 @@ static inline u32 c4iw_ib_to_tpt_access(int a)</div><div class='ctx'> 	       FW_RI_MEM_ACCESS_LOCAL_READ;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline u32 c4iw_ib_to_tpt_bind_access(int acc)</div><div class='del'>-{</div><div class='del'>-	return (acc &amp; IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) |</div><div class='del'>-	       (acc &amp; IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> enum c4iw_mmid_state {</div><div class='ctx'> 	C4IW_STAG_STATE_VALID,</div><div class='ctx'> 	C4IW_STAG_STATE_INVALID</div><div class='hunk'>@@ -707,7 +731,7 @@ struct mpa_message {</div><div class='ctx'> 	u8 flags;</div><div class='ctx'> 	u8 revision;</div><div class='ctx'> 	__be16 private_data_size;</div><div class='del'>-	u8 private_data[0];</div><div class='add'>+	u8 private_data[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct mpa_v2_conn_params {</div><div class='hunk'>@@ -719,7 +743,7 @@ struct terminate_message {</div><div class='ctx'> 	u8 layer_etype;</div><div class='ctx'> 	u8 ecode;</div><div class='ctx'> 	__be16 hdrct_rsvd;</div><div class='del'>-	u8 len_hdrs[0];</div><div class='add'>+	u8 len_hdrs[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> #define TERM_MAX_LENGTH (sizeof(struct terminate_message) + 2 + 18 + 28)</div><div class='hunk'>@@ -942,15 +966,12 @@ void c4iw_id_table_free(struct c4iw_id_table *alloc);</div><div class='ctx'> </div><div class='ctx'> typedef int (*c4iw_handler_func)(struct c4iw_dev *dev, struct sk_buff *skb);</div><div class='ctx'> </div><div class='del'>-int c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new,</div><div class='del'>-		     struct l2t_entry *l2t);</div><div class='ctx'> void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qpid,</div><div class='ctx'> 		   struct c4iw_dev_ucontext *uctx);</div><div class='ctx'> u32 c4iw_get_resource(struct c4iw_id_table *id_table);</div><div class='ctx'> void c4iw_put_resource(struct c4iw_id_table *id_table, u32 entry);</div><div class='ctx'> int c4iw_init_resource(struct c4iw_rdev *rdev, u32 nr_tpt,</div><div class='ctx'> 		       u32 nr_pdid, u32 nr_srqt);</div><div class='del'>-int c4iw_init_ctrl_qp(struct c4iw_rdev *rdev);</div><div class='ctx'> int c4iw_pblpool_create(struct c4iw_rdev *rdev);</div><div class='ctx'> int c4iw_rqtpool_create(struct c4iw_rdev *rdev);</div><div class='ctx'> int c4iw_ocqp_pool_create(struct c4iw_rdev *rdev);</div><div class='hunk'>@@ -958,7 +979,6 @@ void c4iw_pblpool_destroy(struct c4iw_rdev *rdev);</div><div class='ctx'> void c4iw_rqtpool_destroy(struct c4iw_rdev *rdev);</div><div class='ctx'> void c4iw_ocqp_pool_destroy(struct c4iw_rdev *rdev);</div><div class='ctx'> void c4iw_destroy_resource(struct c4iw_resource *rscp);</div><div class='del'>-int c4iw_destroy_ctrl_qp(struct c4iw_rdev *rdev);</div><div class='ctx'> void c4iw_register_device(struct work_struct *work);</div><div class='ctx'> void c4iw_unregister_device(struct c4iw_dev *dev);</div><div class='ctx'> int __init c4iw_cm_init(void);</div><div class='hunk'>@@ -980,33 +1000,30 @@ int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);</div><div class='ctx'> void c4iw_qp_add_ref(struct ib_qp *qp);</div><div class='ctx'> void c4iw_qp_rem_ref(struct ib_qp *qp);</div><div class='ctx'> struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,</div><div class='del'>-			    u32 max_num_sg, struct ib_udata *udata);</div><div class='add'>+			    u32 max_num_sg);</div><div class='ctx'> int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,</div><div class='ctx'> 		   unsigned int *sg_offset);</div><div class='del'>-int c4iw_dealloc_mw(struct ib_mw *mw);</div><div class='ctx'> void c4iw_dealloc(struct uld_ctx *ctx);</div><div class='del'>-struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,</div><div class='del'>-			    struct ib_udata *udata);</div><div class='ctx'> struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start,</div><div class='ctx'> 					   u64 length, u64 virt, int acc,</div><div class='add'>+					   struct ib_dmah *dmah,</div><div class='ctx'> 					   struct ib_udata *udata);</div><div class='ctx'> struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc);</div><div class='ctx'> int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);</div><div class='ctx'> int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);</div><div class='del'>-struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,</div><div class='del'>-			     const struct ib_cq_init_attr *attr,</div><div class='del'>-			     struct ib_udata *udata);</div><div class='add'>+void c4iw_cq_rem_ref(struct c4iw_cq *chp);</div><div class='add'>+int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		   struct uverbs_attr_bundle *attrs);</div><div class='ctx'> int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);</div><div class='ctx'> int c4iw_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *attr,</div><div class='ctx'> 		    enum ib_srq_attr_mask srq_attr_mask,</div><div class='ctx'> 		    struct ib_udata *udata);</div><div class='del'>-void c4iw_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata);</div><div class='add'>+int c4iw_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata);</div><div class='ctx'> int c4iw_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *attrs,</div><div class='ctx'> 		    struct ib_udata *udata);</div><div class='ctx'> int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata);</div><div class='del'>-struct ib_qp *c4iw_create_qp(struct ib_pd *pd,</div><div class='del'>-			     struct ib_qp_init_attr *attrs,</div><div class='del'>-			     struct ib_udata *udata);</div><div class='add'>+int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs,</div><div class='add'>+		   struct ib_udata *udata);</div><div class='ctx'> int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> 				 int attr_mask, struct ib_udata *udata);</div><div class='ctx'> int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='hunk'>@@ -1024,8 +1041,6 @@ int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp);</div><div class='ctx'> int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count);</div><div class='ctx'> int c4iw_flush_sq(struct c4iw_qp *qhp);</div><div class='ctx'> int c4iw_ev_handler(struct c4iw_dev *rnicp, u32 qid);</div><div class='del'>-u16 c4iw_rqes_posted(struct c4iw_qp *qhp);</div><div class='del'>-int c4iw_post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe);</div><div class='ctx'> u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx);</div><div class='ctx'> void c4iw_put_cqid(struct c4iw_rdev *rdev, u32 qid,</div><div class='ctx'> 		struct c4iw_dev_ucontext *uctx);</div><div class='hunk'>@@ -1054,8 +1069,9 @@ int c4iw_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,</div><div class='ctx'> 		       const struct ib_recv_wr **bad_wr);</div><div class='ctx'> struct c4iw_wr_wait *c4iw_alloc_wr_wait(gfp_t gfp);</div><div class='ctx'> </div><div class='del'>-typedef int c4iw_restrack_func(struct sk_buff *msg,</div><div class='del'>-			       struct rdma_restrack_entry *res);</div><div class='del'>-extern c4iw_restrack_func *c4iw_restrack_funcs[RDMA_RESTRACK_MAX];</div><div class='add'>+int c4iw_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr);</div><div class='add'>+int c4iw_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq);</div><div class='add'>+int c4iw_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp);</div><div class='add'>+int c4iw_fill_res_cm_id_entry(struct sk_buff *msg, struct rdma_cm_id *cm_id);</div><div class='ctx'> </div><div class='ctx'> #endif</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c<br/>index 811c0c8c5b16..adeed7447e7b 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/mem.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/mem.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/mem.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/cxgb4/mem.c</a></div><div class='hunk'>@@ -130,8 +130,9 @@ static int _c4iw_write_mem_inline(struct c4iw_rdev *rdev, u32 addr, u32 len,</div><div class='ctx'> </div><div class='ctx'> 		copy_len = len &gt; C4IW_MAX_INLINE_SIZE ? C4IW_MAX_INLINE_SIZE :</div><div class='ctx'> 			   len;</div><div class='del'>-		wr_len = roundup(sizeof *req + sizeof *sc +</div><div class='del'>-				 roundup(copy_len, T4_ULPTX_MIN_IO), 16);</div><div class='add'>+		wr_len = roundup(sizeof(*req) + sizeof(*sc) +</div><div class='add'>+					 roundup(copy_len, T4_ULPTX_MIN_IO),</div><div class='add'>+				 16);</div><div class='ctx'> </div><div class='ctx'> 		if (!skb) {</div><div class='ctx'> 			skb = alloc_skb(wr_len, GFP_KERNEL | __GFP_NOFAIL);</div><div class='hunk'>@@ -274,13 +275,17 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry,</div><div class='ctx'> 			   struct sk_buff *skb, struct c4iw_wr_wait *wr_waitp)</div><div class='ctx'> {</div><div class='ctx'> 	int err;</div><div class='del'>-	struct fw_ri_tpte tpt;</div><div class='add'>+	struct fw_ri_tpte *tpt;</div><div class='ctx'> 	u32 stag_idx;</div><div class='ctx'> 	static atomic_t key;</div><div class='ctx'> </div><div class='ctx'> 	if (c4iw_fatal_error(rdev))</div><div class='ctx'> 		return -EIO;</div><div class='ctx'> </div><div class='add'>+	tpt = kmalloc(sizeof(*tpt), GFP_KERNEL);</div><div class='add'>+	if (!tpt)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='ctx'> 	stag_state = stag_state &gt; 0;</div><div class='ctx'> 	stag_idx = (*stag) &gt;&gt; 8;</div><div class='ctx'> </div><div class='hunk'>@@ -290,6 +295,7 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry,</div><div class='ctx'> 			mutex_lock(&amp;rdev-&gt;stats.lock);</div><div class='ctx'> 			rdev-&gt;stats.stag.fail++;</div><div class='ctx'> 			mutex_unlock(&amp;rdev-&gt;stats.lock);</div><div class='add'>+			kfree(tpt);</div><div class='ctx'> 			return -ENOMEM;</div><div class='ctx'> 		}</div><div class='ctx'> 		mutex_lock(&amp;rdev-&gt;stats.lock);</div><div class='hunk'>@@ -304,28 +310,28 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry,</div><div class='ctx'> </div><div class='ctx'> 	/* write TPT entry */</div><div class='ctx'> 	if (reset_tpt_entry)</div><div class='del'>-		memset(&amp;tpt, 0, sizeof(tpt));</div><div class='add'>+		memset(tpt, 0, sizeof(*tpt));</div><div class='ctx'> 	else {</div><div class='del'>-		tpt.valid_to_pdid = cpu_to_be32(FW_RI_TPTE_VALID_F |</div><div class='add'>+		tpt-&gt;valid_to_pdid = cpu_to_be32(FW_RI_TPTE_VALID_F |</div><div class='ctx'> 			FW_RI_TPTE_STAGKEY_V((*stag &amp; FW_RI_TPTE_STAGKEY_M)) |</div><div class='ctx'> 			FW_RI_TPTE_STAGSTATE_V(stag_state) |</div><div class='ctx'> 			FW_RI_TPTE_STAGTYPE_V(type) | FW_RI_TPTE_PDID_V(pdid));</div><div class='del'>-		tpt.locread_to_qpid = cpu_to_be32(FW_RI_TPTE_PERM_V(perm) |</div><div class='add'>+		tpt-&gt;locread_to_qpid = cpu_to_be32(FW_RI_TPTE_PERM_V(perm) |</div><div class='ctx'> 			(bind_enabled ? FW_RI_TPTE_MWBINDEN_F : 0) |</div><div class='ctx'> 			FW_RI_TPTE_ADDRTYPE_V((zbva ? FW_RI_ZERO_BASED_TO :</div><div class='ctx'> 						      FW_RI_VA_BASED_TO))|</div><div class='ctx'> 			FW_RI_TPTE_PS_V(page_size));</div><div class='del'>-		tpt.nosnoop_pbladdr = !pbl_size ? 0 : cpu_to_be32(</div><div class='add'>+		tpt-&gt;nosnoop_pbladdr = !pbl_size ? 0 : cpu_to_be32(</div><div class='ctx'> 			FW_RI_TPTE_PBLADDR_V(PBL_OFF(rdev, pbl_addr)&gt;&gt;3));</div><div class='del'>-		tpt.len_lo = cpu_to_be32((u32)(len &amp; 0xffffffffUL));</div><div class='del'>-		tpt.va_hi = cpu_to_be32((u32)(to &gt;&gt; 32));</div><div class='del'>-		tpt.va_lo_fbo = cpu_to_be32((u32)(to &amp; 0xffffffffUL));</div><div class='del'>-		tpt.dca_mwbcnt_pstag = cpu_to_be32(0);</div><div class='del'>-		tpt.len_hi = cpu_to_be32((u32)(len &gt;&gt; 32));</div><div class='add'>+		tpt-&gt;len_lo = cpu_to_be32((u32)(len &amp; 0xffffffffUL));</div><div class='add'>+		tpt-&gt;va_hi = cpu_to_be32((u32)(to &gt;&gt; 32));</div><div class='add'>+		tpt-&gt;va_lo_fbo = cpu_to_be32((u32)(to &amp; 0xffffffffUL));</div><div class='add'>+		tpt-&gt;dca_mwbcnt_pstag = cpu_to_be32(0);</div><div class='add'>+		tpt-&gt;len_hi = cpu_to_be32((u32)(len &gt;&gt; 32));</div><div class='ctx'> 	}</div><div class='ctx'> 	err = write_adapter_mem(rdev, stag_idx +</div><div class='ctx'> 				(rdev-&gt;lldi.vr-&gt;stag.start &gt;&gt; 5),</div><div class='del'>-				sizeof(tpt), &amp;tpt, skb, wr_waitp);</div><div class='add'>+				sizeof(*tpt), tpt, skb, wr_waitp);</div><div class='ctx'> </div><div class='ctx'> 	if (reset_tpt_entry) {</div><div class='ctx'> 		c4iw_put_resource(&amp;rdev-&gt;resource.tpt_table, stag_idx);</div><div class='hunk'>@@ -333,6 +339,7 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry,</div><div class='ctx'> 		rdev-&gt;stats.stag.cur -= 32;</div><div class='ctx'> 		mutex_unlock(&amp;rdev-&gt;stats.lock);</div><div class='ctx'> 	}</div><div class='add'>+	kfree(tpt);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -341,7 +348,7 @@ static int write_pbl(struct c4iw_rdev *rdev, __be64 *pbl,</div><div class='ctx'> {</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	pr_debug("*pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d\n",</div><div class='add'>+	pr_debug("*pbl_addr 0x%x, pbl_base 0x%x, pbl_size %d\n",</div><div class='ctx'> 		 pbl_addr, rdev-&gt;lldi.vr-&gt;pbl.start,</div><div class='ctx'> 		 pbl_size);</div><div class='ctx'> </div><div class='hunk'>@@ -358,22 +365,6 @@ static int dereg_mem(struct c4iw_rdev *rdev, u32 stag, u32 pbl_size,</div><div class='ctx'> 			       pbl_size, pbl_addr, skb, wr_waitp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int allocate_window(struct c4iw_rdev *rdev, u32 *stag, u32 pdid,</div><div class='del'>-			   struct c4iw_wr_wait *wr_waitp)</div><div class='del'>-{</div><div class='del'>-	*stag = T4_STAG_UNSET;</div><div class='del'>-	return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_MW, 0, 0, 0,</div><div class='del'>-			       0UL, 0, 0, 0, 0, NULL, wr_waitp);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int deallocate_window(struct c4iw_rdev *rdev, u32 stag,</div><div class='del'>-			     struct sk_buff *skb,</div><div class='del'>-			     struct c4iw_wr_wait *wr_waitp)</div><div class='del'>-{</div><div class='del'>-	return write_tpt_entry(rdev, 1, &amp;stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, 0,</div><div class='del'>-			       0, skb, wr_waitp);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int allocate_stag(struct c4iw_rdev *rdev, u32 *stag, u32 pdid,</div><div class='ctx'> 			 u32 pbl_size, u32 pbl_addr,</div><div class='ctx'> 			 struct c4iw_wr_wait *wr_waitp)</div><div class='hunk'>@@ -392,7 +383,6 @@ static int finish_mem_reg(struct c4iw_mr *mhp, u32 stag)</div><div class='ctx'> 	mmid = stag &gt;&gt; 8;</div><div class='ctx'> 	mhp-&gt;ibmr.rkey = mhp-&gt;ibmr.lkey = stag;</div><div class='ctx'> 	mhp-&gt;ibmr.length = mhp-&gt;attr.len;</div><div class='del'>-	mhp-&gt;ibmr.iova = mhp-&gt;attr.va_fbo;</div><div class='ctx'> 	mhp-&gt;ibmr.page_size = 1U &lt;&lt; (mhp-&gt;attr.page_size + 12);</div><div class='ctx'> 	pr_debug("mmid 0x%x mhp %p\n", mmid, mhp);</div><div class='ctx'> 	return xa_insert_irq(&amp;mhp-&gt;rhp-&gt;mrs, mmid, mhp, GFP_KERNEL);</div><div class='hunk'>@@ -499,18 +489,22 @@ err_free_mhp:</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='del'>-			       u64 virt, int acc, struct ib_udata *udata)</div><div class='add'>+			       u64 virt, int acc, struct ib_dmah *dmah,</div><div class='add'>+			       struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	__be64 *pages;</div><div class='ctx'> 	int shift, n, i;</div><div class='ctx'> 	int err = -ENOMEM;</div><div class='del'>-	struct sg_dma_page_iter sg_iter;</div><div class='add'>+	struct ib_block_iter biter;</div><div class='ctx'> 	struct c4iw_dev *rhp;</div><div class='ctx'> 	struct c4iw_pd *php;</div><div class='ctx'> 	struct c4iw_mr *mhp;</div><div class='ctx'> </div><div class='ctx'> 	pr_debug("ib_pd %p\n", pd);</div><div class='ctx'> </div><div class='add'>+	if (dmah)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='ctx'> 	if (length == ~0ULL)</div><div class='ctx'> 		return ERR_PTR(-EINVAL);</div><div class='ctx'> </div><div class='hunk'>@@ -536,13 +530,13 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='ctx'> </div><div class='ctx'> 	mhp-&gt;rhp = rhp;</div><div class='ctx'> </div><div class='del'>-	mhp-&gt;umem = ib_umem_get(udata, start, length, acc, 0);</div><div class='add'>+	mhp-&gt;umem = ib_umem_get(pd-&gt;device, start, length, acc);</div><div class='ctx'> 	if (IS_ERR(mhp-&gt;umem))</div><div class='ctx'> 		goto err_free_skb;</div><div class='ctx'> </div><div class='ctx'> 	shift = PAGE_SHIFT;</div><div class='ctx'> </div><div class='del'>-	n = ib_umem_num_pages(mhp-&gt;umem);</div><div class='add'>+	n = ib_umem_num_dma_blocks(mhp-&gt;umem, 1 &lt;&lt; shift);</div><div class='ctx'> 	err = alloc_pbl(mhp, n);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_umem_release;</div><div class='hunk'>@@ -555,8 +549,8 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='ctx'> </div><div class='ctx'> 	i = n = 0;</div><div class='ctx'> </div><div class='del'>-	for_each_sg_dma_page(mhp-&gt;umem-&gt;sg_head.sgl, &amp;sg_iter, mhp-&gt;umem-&gt;nmap, 0) {</div><div class='del'>-		pages[i++] = cpu_to_be64(sg_page_iter_dma_address(&amp;sg_iter));</div><div class='add'>+	rdma_umem_for_each_dma_block(mhp-&gt;umem, &amp;biter, 1 &lt;&lt; shift) {</div><div class='add'>+		pages[i++] = cpu_to_be64(rdma_block_iter_dma_address(&amp;biter));</div><div class='ctx'> 		if (i == PAGE_SIZE / sizeof(*pages)) {</div><div class='ctx'> 			err = write_pbl(&amp;mhp-&gt;rhp-&gt;rdev, pages,</div><div class='ctx'> 					mhp-&gt;attr.pbl_addr + (n &lt;&lt; 3), i,</div><div class='hunk'>@@ -605,86 +599,8 @@ err_free_mhp:</div><div class='ctx'> 	return ERR_PTR(err);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,</div><div class='del'>-			    struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct c4iw_dev *rhp;</div><div class='del'>-	struct c4iw_pd *php;</div><div class='del'>-	struct c4iw_mw *mhp;</div><div class='del'>-	u32 mmid;</div><div class='del'>-	u32 stag = 0;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if (type != IB_MW_TYPE_1)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	php = to_c4iw_pd(pd);</div><div class='del'>-	rhp = php-&gt;rhp;</div><div class='del'>-	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);</div><div class='del'>-	if (!mhp)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	mhp-&gt;wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL);</div><div class='del'>-	if (!mhp-&gt;wr_waitp) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto free_mhp;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mhp-&gt;dereg_skb = alloc_skb(SGE_MAX_WR_LEN, GFP_KERNEL);</div><div class='del'>-	if (!mhp-&gt;dereg_skb) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto free_wr_wait;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = allocate_window(&amp;rhp-&gt;rdev, &amp;stag, php-&gt;pdid, mhp-&gt;wr_waitp);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto free_skb;</div><div class='del'>-	mhp-&gt;rhp = rhp;</div><div class='del'>-	mhp-&gt;attr.pdid = php-&gt;pdid;</div><div class='del'>-	mhp-&gt;attr.type = FW_RI_STAG_MW;</div><div class='del'>-	mhp-&gt;attr.stag = stag;</div><div class='del'>-	mmid = (stag) &gt;&gt; 8;</div><div class='del'>-	mhp-&gt;ibmw.rkey = stag;</div><div class='del'>-	if (xa_insert_irq(&amp;rhp-&gt;mrs, mmid, mhp, GFP_KERNEL)) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto dealloc_win;</div><div class='del'>-	}</div><div class='del'>-	pr_debug("mmid 0x%x mhp %p stag 0x%x\n", mmid, mhp, stag);</div><div class='del'>-	return &amp;(mhp-&gt;ibmw);</div><div class='del'>-</div><div class='del'>-dealloc_win:</div><div class='del'>-	deallocate_window(&amp;rhp-&gt;rdev, mhp-&gt;attr.stag, mhp-&gt;dereg_skb,</div><div class='del'>-			  mhp-&gt;wr_waitp);</div><div class='del'>-free_skb:</div><div class='del'>-	kfree_skb(mhp-&gt;dereg_skb);</div><div class='del'>-free_wr_wait:</div><div class='del'>-	c4iw_put_wr_wait(mhp-&gt;wr_waitp);</div><div class='del'>-free_mhp:</div><div class='del'>-	kfree(mhp);</div><div class='del'>-	return ERR_PTR(ret);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int c4iw_dealloc_mw(struct ib_mw *mw)</div><div class='del'>-{</div><div class='del'>-	struct c4iw_dev *rhp;</div><div class='del'>-	struct c4iw_mw *mhp;</div><div class='del'>-	u32 mmid;</div><div class='del'>-</div><div class='del'>-	mhp = to_c4iw_mw(mw);</div><div class='del'>-	rhp = mhp-&gt;rhp;</div><div class='del'>-	mmid = (mw-&gt;rkey) &gt;&gt; 8;</div><div class='del'>-	xa_erase_irq(&amp;rhp-&gt;mrs, mmid);</div><div class='del'>-	deallocate_window(&amp;rhp-&gt;rdev, mhp-&gt;attr.stag, mhp-&gt;dereg_skb,</div><div class='del'>-			  mhp-&gt;wr_waitp);</div><div class='del'>-	kfree_skb(mhp-&gt;dereg_skb);</div><div class='del'>-	c4iw_put_wr_wait(mhp-&gt;wr_waitp);</div><div class='del'>-	pr_debug("ib_mw %p mmid 0x%x ptr %p\n", mw, mmid, mhp);</div><div class='del'>-	kfree(mhp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,</div><div class='del'>-			    u32 max_num_sg, struct ib_udata *udata)</div><div class='add'>+			    u32 max_num_sg)</div><div class='ctx'> {</div><div class='ctx'> 	struct c4iw_dev *rhp;</div><div class='ctx'> 	struct c4iw_pd *php;</div><div class='hunk'>@@ -807,8 +723,7 @@ int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)</div><div class='ctx'> 				  mhp-&gt;attr.pbl_size &lt;&lt; 3);</div><div class='ctx'> 	if (mhp-&gt;kva)</div><div class='ctx'> 		kfree((void *) (unsigned long) mhp-&gt;kva);</div><div class='del'>-	if (mhp-&gt;umem)</div><div class='del'>-		ib_umem_release(mhp-&gt;umem);</div><div class='add'>+	ib_umem_release(mhp-&gt;umem);</div><div class='ctx'> 	pr_debug("mmid 0x%x ptr %p\n", mmid, mhp);</div><div class='ctx'> 	c4iw_put_wr_wait(mhp-&gt;wr_waitp);</div><div class='ctx'> 	kfree(mhp);</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c<br/>index 74b795642fca..e059f92d90fd 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/provider.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/provider.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/provider.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/cxgb4/provider.c</a></div><div class='hunk'>@@ -41,6 +41,7 @@</div><div class='ctx'> #include &lt;linux/ethtool.h&gt;</div><div class='ctx'> #include &lt;linux/rtnetlink.h&gt;</div><div class='ctx'> #include &lt;linux/inetdevice.h&gt;</div><div class='add'>+#include &lt;net/addrconf.h&gt;</div><div class='ctx'> #include &lt;linux/io.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include &lt;asm/irq.h&gt;</div><div class='hunk'>@@ -112,6 +113,9 @@ static int c4iw_alloc_ucontext(struct ib_ucontext *ucontext,</div><div class='ctx'> 		mm-&gt;key = uresp.status_page_key;</div><div class='ctx'> 		mm-&gt;addr = virt_to_phys(rhp-&gt;rdev.status_page);</div><div class='ctx'> 		mm-&gt;len = PAGE_SIZE;</div><div class='add'>+		mm-&gt;vaddr = NULL;</div><div class='add'>+		mm-&gt;dma_addr = 0;</div><div class='add'>+		insert_flag_to_mmap(&amp;rhp-&gt;rdev, mm, mm-&gt;addr);</div><div class='ctx'> 		insert_mmap(context, mm);</div><div class='ctx'> 	}</div><div class='ctx'> 	return 0;</div><div class='hunk'>@@ -130,6 +134,11 @@ static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)</div><div class='ctx'> 	struct c4iw_mm_entry *mm;</div><div class='ctx'> 	struct c4iw_ucontext *ucontext;</div><div class='ctx'> 	u64 addr;</div><div class='add'>+	u8 mmap_flag;</div><div class='add'>+	size_t size;</div><div class='add'>+	void *vaddr;</div><div class='add'>+	unsigned long vm_pgoff;</div><div class='add'>+	dma_addr_t dma_addr;</div><div class='ctx'> </div><div class='ctx'> 	pr_debug("pgoff 0x%lx key 0x%x len %d\n", vma-&gt;vm_pgoff,</div><div class='ctx'> 		 key, len);</div><div class='hunk'>@@ -144,53 +153,44 @@ static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)</div><div class='ctx'> 	if (!mm)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	addr = mm-&gt;addr;</div><div class='add'>+	vaddr = mm-&gt;vaddr;</div><div class='add'>+	dma_addr = mm-&gt;dma_addr;</div><div class='add'>+	size = mm-&gt;len;</div><div class='add'>+	mmap_flag = mm-&gt;mmap_flag;</div><div class='ctx'> 	kfree(mm);</div><div class='ctx'> </div><div class='del'>-	if ((addr &gt;= pci_resource_start(rdev-&gt;lldi.pdev, 0)) &amp;&amp;</div><div class='del'>-	    (addr &lt; (pci_resource_start(rdev-&gt;lldi.pdev, 0) +</div><div class='del'>-		    pci_resource_len(rdev-&gt;lldi.pdev, 0)))) {</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * MA_SYNC register...</div><div class='del'>-		 */</div><div class='del'>-		vma-&gt;vm_page_prot = pgprot_noncached(vma-&gt;vm_page_prot);</div><div class='add'>+	switch (mmap_flag) {</div><div class='add'>+	case CXGB4_MMAP_BAR:</div><div class='add'>+		ret = io_remap_pfn_range(vma, vma-&gt;vm_start, addr &gt;&gt; PAGE_SHIFT,</div><div class='add'>+					 len,</div><div class='add'>+					 pgprot_noncached(vma-&gt;vm_page_prot));</div><div class='add'>+		break;</div><div class='add'>+	case CXGB4_MMAP_BAR_WC:</div><div class='ctx'> 		ret = io_remap_pfn_range(vma, vma-&gt;vm_start,</div><div class='ctx'> 					 addr &gt;&gt; PAGE_SHIFT,</div><div class='del'>-					 len, vma-&gt;vm_page_prot);</div><div class='del'>-	} else if ((addr &gt;= pci_resource_start(rdev-&gt;lldi.pdev, 2)) &amp;&amp;</div><div class='del'>-		   (addr &lt; (pci_resource_start(rdev-&gt;lldi.pdev, 2) +</div><div class='del'>-		    pci_resource_len(rdev-&gt;lldi.pdev, 2)))) {</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * Map user DB or OCQP memory...</div><div class='del'>-		 */</div><div class='del'>-		if (addr &gt;= rdev-&gt;oc_mw_pa)</div><div class='del'>-			vma-&gt;vm_page_prot = t4_pgprot_wc(vma-&gt;vm_page_prot);</div><div class='del'>-		else {</div><div class='del'>-			if (!is_t4(rdev-&gt;lldi.adapter_type))</div><div class='del'>-				vma-&gt;vm_page_prot =</div><div class='del'>-					t4_pgprot_wc(vma-&gt;vm_page_prot);</div><div class='del'>-			else</div><div class='del'>-				vma-&gt;vm_page_prot =</div><div class='del'>-					pgprot_noncached(vma-&gt;vm_page_prot);</div><div class='del'>-		}</div><div class='add'>+					 len, t4_pgprot_wc(vma-&gt;vm_page_prot));</div><div class='add'>+		break;</div><div class='add'>+	case CXGB4_MMAP_CONTIG:</div><div class='ctx'> 		ret = io_remap_pfn_range(vma, vma-&gt;vm_start,</div><div class='ctx'> 					 addr &gt;&gt; PAGE_SHIFT,</div><div class='ctx'> 					 len, vma-&gt;vm_page_prot);</div><div class='del'>-	} else {</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * Map WQ or CQ contig dma memory...</div><div class='del'>-		 */</div><div class='del'>-		ret = remap_pfn_range(vma, vma-&gt;vm_start,</div><div class='del'>-				      addr &gt;&gt; PAGE_SHIFT,</div><div class='del'>-				      len, vma-&gt;vm_page_prot);</div><div class='add'>+		break;</div><div class='add'>+	case CXGB4_MMAP_NON_CONTIG:</div><div class='add'>+		vm_pgoff = vma-&gt;vm_pgoff;</div><div class='add'>+		vma-&gt;vm_pgoff = 0;</div><div class='add'>+		ret = dma_mmap_coherent(&amp;rdev-&gt;lldi.pdev-&gt;dev, vma,</div><div class='add'>+					vaddr, dma_addr, size);</div><div class='add'>+		vma-&gt;vm_pgoff = vm_pgoff;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		break;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void c4iw_deallocate_pd(struct ib_pd *pd, struct ib_udata *udata)</div><div class='add'>+static int c4iw_deallocate_pd(struct ib_pd *pd, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct c4iw_dev *rhp;</div><div class='ctx'> 	struct c4iw_pd *php;</div><div class='hunk'>@@ -202,6 +202,7 @@ static void c4iw_deallocate_pd(struct ib_pd *pd, struct ib_udata *udata)</div><div class='ctx'> 	mutex_lock(&amp;rhp-&gt;rdev.stats.lock);</div><div class='ctx'> 	rhp-&gt;rdev.stats.pd.cur--;</div><div class='ctx'> 	mutex_unlock(&amp;rhp-&gt;rdev.stats.lock);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata)</div><div class='hunk'>@@ -236,20 +237,12 @@ static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int c4iw_query_pkey(struct ib_device *ibdev, u8 port, u16 index,</div><div class='del'>-			   u16 *pkey)</div><div class='del'>-{</div><div class='del'>-	pr_debug("ibdev %p\n", ibdev);</div><div class='del'>-	*pkey = 0;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int c4iw_query_gid(struct ib_device *ibdev, u8 port, int index,</div><div class='add'>+static int c4iw_query_gid(struct ib_device *ibdev, u32 port, int index,</div><div class='ctx'> 			  union ib_gid *gid)</div><div class='ctx'> {</div><div class='ctx'> 	struct c4iw_dev *dev;</div><div class='ctx'> </div><div class='del'>-	pr_debug("ibdev %p, port %d, index %d, gid %p\n",</div><div class='add'>+	pr_debug("ibdev %p, port %u, index %d, gid %p\n",</div><div class='ctx'> 		 ibdev, port, index, gid);</div><div class='ctx'> 	if (!port)</div><div class='ctx'> 		return -EINVAL;</div><div class='hunk'>@@ -271,11 +264,14 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	dev = to_c4iw_dev(ibdev);</div><div class='del'>-	memset(props, 0, sizeof *props);</div><div class='del'>-	memcpy(&amp;props-&gt;sys_image_guid, dev-&gt;rdev.lldi.ports[0]-&gt;dev_addr, 6);</div><div class='add'>+	addrconf_addr_eui48((u8 *)&amp;props-&gt;sys_image_guid,</div><div class='add'>+			    dev-&gt;rdev.lldi.ports[0]-&gt;dev_addr);</div><div class='ctx'> 	props-&gt;hw_ver = CHELSIO_CHIP_RELEASE(dev-&gt;rdev.lldi.adapter_type);</div><div class='ctx'> 	props-&gt;fw_ver = dev-&gt;rdev.lldi.fw_vers;</div><div class='del'>-	props-&gt;device_cap_flags = dev-&gt;device_cap_flags;</div><div class='add'>+	props-&gt;device_cap_flags = IB_DEVICE_MEM_WINDOW;</div><div class='add'>+	props-&gt;kernel_cap_flags = IBK_LOCAL_DMA_LKEY;</div><div class='add'>+	if (fastreg_support)</div><div class='add'>+		props-&gt;device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;</div><div class='ctx'> 	props-&gt;page_size_cap = T4_PAGESIZE_MASK;</div><div class='ctx'> 	props-&gt;vendor_id = (u32)dev-&gt;rdev.lldi.pdev-&gt;vendor;</div><div class='ctx'> 	props-&gt;vendor_part_id = (u32)dev-&gt;rdev.lldi.pdev-&gt;device;</div><div class='hunk'>@@ -303,34 +299,13 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int c4iw_query_port(struct ib_device *ibdev, u8 port,</div><div class='add'>+static int c4iw_query_port(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 			   struct ib_port_attr *props)</div><div class='ctx'> {</div><div class='del'>-	struct c4iw_dev *dev;</div><div class='del'>-	struct net_device *netdev;</div><div class='del'>-	struct in_device *inetdev;</div><div class='del'>-</div><div class='add'>+	int ret = 0;</div><div class='ctx'> 	pr_debug("ibdev %p\n", ibdev);</div><div class='del'>-</div><div class='del'>-	dev = to_c4iw_dev(ibdev);</div><div class='del'>-	netdev = dev-&gt;rdev.lldi.ports[port-1];</div><div class='del'>-	/* props being zeroed by the caller, avoid zeroing it here */</div><div class='del'>-	props-&gt;max_mtu = IB_MTU_4096;</div><div class='del'>-	props-&gt;active_mtu = ib_mtu_int_to_enum(netdev-&gt;mtu);</div><div class='del'>-</div><div class='del'>-	if (!netif_carrier_ok(netdev))</div><div class='del'>-		props-&gt;state = IB_PORT_DOWN;</div><div class='del'>-	else {</div><div class='del'>-		inetdev = in_dev_get(netdev);</div><div class='del'>-		if (inetdev) {</div><div class='del'>-			if (inetdev-&gt;ifa_list)</div><div class='del'>-				props-&gt;state = IB_PORT_ACTIVE;</div><div class='del'>-			else</div><div class='del'>-				props-&gt;state = IB_PORT_INIT;</div><div class='del'>-			in_dev_put(inetdev);</div><div class='del'>-		} else</div><div class='del'>-			props-&gt;state = IB_PORT_INIT;</div><div class='del'>-	}</div><div class='add'>+	ret = ib_get_eth_speed(ibdev, port, &amp;props-&gt;active_speed,</div><div class='add'>+			       &amp;props-&gt;active_width);</div><div class='ctx'> </div><div class='ctx'> 	props-&gt;port_cap_flags =</div><div class='ctx'> 	    IB_PORT_CM_SUP |</div><div class='hunk'>@@ -339,12 +314,9 @@ static int c4iw_query_port(struct ib_device *ibdev, u8 port,</div><div class='ctx'> 	    IB_PORT_DEVICE_MGMT_SUP |</div><div class='ctx'> 	    IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;</div><div class='ctx'> 	props-&gt;gid_tbl_len = 1;</div><div class='del'>-	props-&gt;pkey_tbl_len = 1;</div><div class='del'>-	props-&gt;active_width = 2;</div><div class='del'>-	props-&gt;active_speed = IB_SPEED_DDR;</div><div class='ctx'> 	props-&gt;max_msg_sz = -1;</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t hw_rev_show(struct device *dev,</div><div class='hunk'>@@ -354,8 +326,9 @@ static ssize_t hw_rev_show(struct device *dev,</div><div class='ctx'> 			rdma_device_to_drv_device(dev, struct c4iw_dev, ibdev);</div><div class='ctx'> </div><div class='ctx'> 	pr_debug("dev 0x%p\n", dev);</div><div class='del'>-	return sprintf(buf, "%d\n",</div><div class='del'>-		       CHELSIO_CHIP_RELEASE(c4iw_dev-&gt;rdev.lldi.adapter_type));</div><div class='add'>+	return sysfs_emit(</div><div class='add'>+		buf, "%d\n",</div><div class='add'>+		CHELSIO_CHIP_RELEASE(c4iw_dev-&gt;rdev.lldi.adapter_type));</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(hw_rev);</div><div class='ctx'> </div><div class='hunk'>@@ -369,7 +342,7 @@ static ssize_t hca_type_show(struct device *dev,</div><div class='ctx'> </div><div class='ctx'> 	pr_debug("dev 0x%p\n", dev);</div><div class='ctx'> 	lldev-&gt;ethtool_ops-&gt;get_drvinfo(lldev, &amp;info);</div><div class='del'>-	return sprintf(buf, "%s\n", info.driver);</div><div class='add'>+	return sysfs_emit(buf, "%s\n", info.driver);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(hca_type);</div><div class='ctx'> </div><div class='hunk'>@@ -380,8 +353,8 @@ static ssize_t board_id_show(struct device *dev, struct device_attribute *attr,</div><div class='ctx'> 			rdma_device_to_drv_device(dev, struct c4iw_dev, ibdev);</div><div class='ctx'> </div><div class='ctx'> 	pr_debug("dev 0x%p\n", dev);</div><div class='del'>-	return sprintf(buf, "%x.%x\n", c4iw_dev-&gt;rdev.lldi.pdev-&gt;vendor,</div><div class='del'>-		       c4iw_dev-&gt;rdev.lldi.pdev-&gt;device);</div><div class='add'>+	return sysfs_emit(buf, "%x.%x\n", c4iw_dev-&gt;rdev.lldi.pdev-&gt;vendor,</div><div class='add'>+			  c4iw_dev-&gt;rdev.lldi.pdev-&gt;device);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(board_id);</div><div class='ctx'> </div><div class='hunk'>@@ -397,32 +370,29 @@ enum counters {</div><div class='ctx'> 	NR_COUNTERS</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static const char * const names[] = {</div><div class='del'>-	[IP4INSEGS] = "ip4InSegs",</div><div class='del'>-	[IP4OUTSEGS] = "ip4OutSegs",</div><div class='del'>-	[IP4RETRANSSEGS] = "ip4RetransSegs",</div><div class='del'>-	[IP4OUTRSTS] = "ip4OutRsts",</div><div class='del'>-	[IP6INSEGS] = "ip6InSegs",</div><div class='del'>-	[IP6OUTSEGS] = "ip6OutSegs",</div><div class='del'>-	[IP6RETRANSSEGS] = "ip6RetransSegs",</div><div class='del'>-	[IP6OUTRSTS] = "ip6OutRsts"</div><div class='add'>+static const struct rdma_stat_desc cxgb4_descs[] = {</div><div class='add'>+	[IP4INSEGS].name = "ip4InSegs",</div><div class='add'>+	[IP4OUTSEGS].name = "ip4OutSegs",</div><div class='add'>+	[IP4RETRANSSEGS].name = "ip4RetransSegs",</div><div class='add'>+	[IP4OUTRSTS].name = "ip4OutRsts",</div><div class='add'>+	[IP6INSEGS].name = "ip6InSegs",</div><div class='add'>+	[IP6OUTSEGS].name = "ip6OutSegs",</div><div class='add'>+	[IP6RETRANSSEGS].name = "ip6RetransSegs",</div><div class='add'>+	[IP6OUTRSTS].name = "ip6OutRsts"</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static struct rdma_hw_stats *c4iw_alloc_stats(struct ib_device *ibdev,</div><div class='del'>-					      u8 port_num)</div><div class='add'>+static struct rdma_hw_stats *c4iw_alloc_device_stats(struct ib_device *ibdev)</div><div class='ctx'> {</div><div class='del'>-	BUILD_BUG_ON(ARRAY_SIZE(names) != NR_COUNTERS);</div><div class='del'>-</div><div class='del'>-	if (port_num != 0)</div><div class='del'>-		return NULL;</div><div class='add'>+	BUILD_BUG_ON(ARRAY_SIZE(cxgb4_descs) != NR_COUNTERS);</div><div class='ctx'> </div><div class='del'>-	return rdma_alloc_hw_stats_struct(names, NR_COUNTERS,</div><div class='add'>+	/* FIXME: these look like port stats */</div><div class='add'>+	return rdma_alloc_hw_stats_struct(cxgb4_descs, NR_COUNTERS,</div><div class='ctx'> 					  RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int c4iw_get_mib(struct ib_device *ibdev,</div><div class='ctx'> 			struct rdma_hw_stats *stats,</div><div class='del'>-			u8 port, int index)</div><div class='add'>+			u32 port, int index)</div><div class='ctx'> {</div><div class='ctx'> 	struct tp_tcp_stats v4, v6;</div><div class='ctx'> 	struct c4iw_dev *c4iw_dev = to_c4iw_dev(ibdev);</div><div class='hunk'>@@ -451,7 +421,7 @@ static const struct attribute_group c4iw_attr_group = {</div><div class='ctx'> 	.attrs = c4iw_class_attributes,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static int c4iw_port_immutable(struct ib_device *ibdev, u8 port_num,</div><div class='add'>+static int c4iw_port_immutable(struct ib_device *ibdev, u32 port_num,</div><div class='ctx'> 			       struct ib_port_immutable *immutable)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr attr;</div><div class='hunk'>@@ -463,7 +433,6 @@ static int c4iw_port_immutable(struct ib_device *ibdev, u8 port_num,</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='del'>-	immutable-&gt;pkey_tbl_len = attr.pkey_tbl_len;</div><div class='ctx'> 	immutable-&gt;gid_tbl_len = attr.gid_tbl_len;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='hunk'>@@ -482,30 +451,29 @@ static void get_dev_fw_str(struct ib_device *dev, char *str)</div><div class='ctx'> 		 FW_HDR_FW_VER_BUILD_G(c4iw_dev-&gt;rdev.lldi.fw_vers));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int fill_res_entry(struct sk_buff *msg, struct rdma_restrack_entry *res)</div><div class='del'>-{</div><div class='del'>-	return (res-&gt;type &lt; ARRAY_SIZE(c4iw_restrack_funcs) &amp;&amp;</div><div class='del'>-		c4iw_restrack_funcs[res-&gt;type]) ?</div><div class='del'>-		c4iw_restrack_funcs[res-&gt;type](msg, res) : 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static const struct ib_device_ops c4iw_dev_ops = {</div><div class='del'>-	.alloc_hw_stats = c4iw_alloc_stats,</div><div class='add'>+	.owner = THIS_MODULE,</div><div class='add'>+	.driver_id = RDMA_DRIVER_CXGB4,</div><div class='add'>+	.uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION,</div><div class='add'>+</div><div class='add'>+	.alloc_hw_device_stats = c4iw_alloc_device_stats,</div><div class='ctx'> 	.alloc_mr = c4iw_alloc_mr,</div><div class='del'>-	.alloc_mw = c4iw_alloc_mw,</div><div class='ctx'> 	.alloc_pd = c4iw_allocate_pd,</div><div class='ctx'> 	.alloc_ucontext = c4iw_alloc_ucontext,</div><div class='ctx'> 	.create_cq = c4iw_create_cq,</div><div class='ctx'> 	.create_qp = c4iw_create_qp,</div><div class='ctx'> 	.create_srq = c4iw_create_srq,</div><div class='del'>-	.dealloc_mw = c4iw_dealloc_mw,</div><div class='ctx'> 	.dealloc_pd = c4iw_deallocate_pd,</div><div class='ctx'> 	.dealloc_ucontext = c4iw_dealloc_ucontext,</div><div class='ctx'> 	.dereg_mr = c4iw_dereg_mr,</div><div class='ctx'> 	.destroy_cq = c4iw_destroy_cq,</div><div class='ctx'> 	.destroy_qp = c4iw_destroy_qp,</div><div class='ctx'> 	.destroy_srq = c4iw_destroy_srq,</div><div class='del'>-	.fill_res_entry = fill_res_entry,</div><div class='add'>+	.device_group = &amp;c4iw_attr_group,</div><div class='add'>+	.fill_res_cq_entry = c4iw_fill_res_cq_entry,</div><div class='add'>+	.fill_res_cm_id_entry = c4iw_fill_res_cm_id_entry,</div><div class='add'>+	.fill_res_mr_entry = c4iw_fill_res_mr_entry,</div><div class='add'>+	.fill_res_qp_entry = c4iw_fill_res_qp_entry,</div><div class='ctx'> 	.get_dev_fw_str = get_dev_fw_str,</div><div class='ctx'> 	.get_dma_mr = c4iw_get_dma_mr,</div><div class='ctx'> 	.get_hw_stats = c4iw_get_mib,</div><div class='hunk'>@@ -528,12 +496,15 @@ static const struct ib_device_ops c4iw_dev_ops = {</div><div class='ctx'> 	.post_srq_recv = c4iw_post_srq_recv,</div><div class='ctx'> 	.query_device = c4iw_query_device,</div><div class='ctx'> 	.query_gid = c4iw_query_gid,</div><div class='del'>-	.query_pkey = c4iw_query_pkey,</div><div class='ctx'> 	.query_port = c4iw_query_port,</div><div class='ctx'> 	.query_qp = c4iw_ib_query_qp,</div><div class='ctx'> 	.reg_user_mr = c4iw_reg_user_mr,</div><div class='ctx'> 	.req_notify_cq = c4iw_arm_cq,</div><div class='add'>+</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_cq, c4iw_cq, ibcq),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_mw, c4iw_mw, ibmw),</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_pd, c4iw_pd, ibpd),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_qp, c4iw_qp, ibqp),</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_srq, c4iw_srq, ibsrq),</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_ucontext, c4iw_ucontext, ibucontext),</div><div class='ctx'> };</div><div class='hunk'>@@ -559,53 +530,26 @@ void c4iw_register_device(struct work_struct *work)</div><div class='ctx'> 	struct c4iw_dev *dev = ctx-&gt;dev;</div><div class='ctx'> </div><div class='ctx'> 	pr_debug("c4iw_dev %p\n", dev);</div><div class='del'>-	memset(&amp;dev-&gt;ibdev.node_guid, 0, sizeof(dev-&gt;ibdev.node_guid));</div><div class='del'>-	memcpy(&amp;dev-&gt;ibdev.node_guid, dev-&gt;rdev.lldi.ports[0]-&gt;dev_addr, 6);</div><div class='del'>-	dev-&gt;ibdev.owner = THIS_MODULE;</div><div class='del'>-	dev-&gt;device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW;</div><div class='del'>-	if (fastreg_support)</div><div class='del'>-		dev-&gt;device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;</div><div class='add'>+	addrconf_addr_eui48((u8 *)&amp;dev-&gt;ibdev.node_guid,</div><div class='add'>+			    dev-&gt;rdev.lldi.ports[0]-&gt;dev_addr);</div><div class='ctx'> 	dev-&gt;ibdev.local_dma_lkey = 0;</div><div class='del'>-	dev-&gt;ibdev.uverbs_cmd_mask =</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_GET_CONTEXT) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_DEVICE) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_PORT) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_ALLOC_PD) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_DEALLOC_PD) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_REG_MR) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_DEREG_MR) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_CQ) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_CQ) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_QP) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_MODIFY_QP) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_QP) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_POLL_CQ) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_QP) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_POST_SEND) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_POST_RECV) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_SRQ) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_MODIFY_SRQ) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_SRQ);</div><div class='ctx'> 	dev-&gt;ibdev.node_type = RDMA_NODE_RNIC;</div><div class='ctx'> 	BUILD_BUG_ON(sizeof(C4IW_NODE_DESC) &gt; IB_DEVICE_NODE_DESC_MAX);</div><div class='ctx'> 	memcpy(dev-&gt;ibdev.node_desc, C4IW_NODE_DESC, sizeof(C4IW_NODE_DESC));</div><div class='ctx'> 	dev-&gt;ibdev.phys_port_cnt = dev-&gt;rdev.lldi.nports;</div><div class='ctx'> 	dev-&gt;ibdev.num_comp_vectors =  dev-&gt;rdev.lldi.nciq;</div><div class='ctx'> 	dev-&gt;ibdev.dev.parent = &amp;dev-&gt;rdev.lldi.pdev-&gt;dev;</div><div class='del'>-	dev-&gt;ibdev.uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION;</div><div class='ctx'> </div><div class='ctx'> 	memcpy(dev-&gt;ibdev.iw_ifname, dev-&gt;rdev.lldi.ports[0]-&gt;name,</div><div class='ctx'> 	       sizeof(dev-&gt;ibdev.iw_ifname));</div><div class='ctx'> </div><div class='del'>-	rdma_set_device_sysfs_group(&amp;dev-&gt;ibdev, &amp;c4iw_attr_group);</div><div class='del'>-	dev-&gt;ibdev.driver_id = RDMA_DRIVER_CXGB4;</div><div class='ctx'> 	ib_set_device_ops(&amp;dev-&gt;ibdev, &amp;c4iw_dev_ops);</div><div class='ctx'> 	ret = set_netdevs(&amp;dev-&gt;ibdev, &amp;dev-&gt;rdev);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err_dealloc_ctx;</div><div class='del'>-	ret = ib_register_device(&amp;dev-&gt;ibdev, "cxgb4_%d");</div><div class='add'>+	dma_set_max_seg_size(&amp;dev-&gt;rdev.lldi.pdev-&gt;dev, UINT_MAX);</div><div class='add'>+	ret = ib_register_device(&amp;dev-&gt;ibdev, "cxgb4_%d",</div><div class='add'>+				 &amp;dev-&gt;rdev.lldi.pdev-&gt;dev);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err_dealloc_ctx;</div><div class='ctx'> 	return;</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c<br/>index e92b9544357a..955f061a55e9 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/qp.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/qp.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/qp.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/cxgb4/qp.c</a></div><div class='hunk'>@@ -274,7 +274,6 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,</div><div class='ctx'> 			 (unsigned long long)virt_to_phys(wq-&gt;sq.queue),</div><div class='ctx'> 			 wq-&gt;rq.queue,</div><div class='ctx'> 			 (unsigned long long)virt_to_phys(wq-&gt;rq.queue));</div><div class='del'>-		memset(wq-&gt;rq.queue, 0, wq-&gt;rq.memsize);</div><div class='ctx'> 		dma_unmap_addr_set(&amp;wq-&gt;rq, mapping, wq-&gt;rq.dma_addr);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -296,6 +295,7 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,</div><div class='ctx'> 	if (user &amp;&amp; (!wq-&gt;sq.bar2_pa || (need_rq &amp;&amp; !wq-&gt;rq.bar2_pa))) {</div><div class='ctx'> 		pr_warn("%s: sqid %u or rqid %u not in BAR2 range\n",</div><div class='ctx'> 			pci_name(rdev-&gt;lldi.pdev), wq-&gt;sq.qid, wq-&gt;rq.qid);</div><div class='add'>+		ret = -EINVAL;</div><div class='ctx'> 		goto free_dma;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -303,7 +303,7 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,</div><div class='ctx'> 	wq-&gt;rq.msn = 1;</div><div class='ctx'> </div><div class='ctx'> 	/* build fw_ri_res_wr */</div><div class='del'>-	wr_len = sizeof *res_wr + 2 * sizeof *res;</div><div class='add'>+	wr_len = sizeof(*res_wr) + 2 * sizeof(*res);</div><div class='ctx'> 	if (need_rq)</div><div class='ctx'> 		wr_len += sizeof(*res);</div><div class='ctx'> 	skb = alloc_skb(wr_len, GFP_KERNEL);</div><div class='hunk'>@@ -439,7 +439,7 @@ static int build_immd(struct t4_sq *sq, struct fw_ri_immd *immdp,</div><div class='ctx'> 			rem -= len;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='del'>-	len = roundup(plen + sizeof *immdp, 16) - (plen + sizeof *immdp);</div><div class='add'>+	len = roundup(plen + sizeof(*immdp), 16) - (plen + sizeof(*immdp));</div><div class='ctx'> 	if (len)</div><div class='ctx'> 		memset(dstp, 0, len);</div><div class='ctx'> 	immdp-&gt;op = FW_RI_DATA_IMMD;</div><div class='hunk'>@@ -528,7 +528,7 @@ static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe,</div><div class='ctx'> 					 T4_MAX_SEND_INLINE, &amp;plen);</div><div class='ctx'> 			if (ret)</div><div class='ctx'> 				return ret;</div><div class='del'>-			size = sizeof wqe-&gt;send + sizeof(struct fw_ri_immd) +</div><div class='add'>+			size = sizeof(wqe-&gt;send) + sizeof(struct fw_ri_immd) +</div><div class='ctx'> 			       plen;</div><div class='ctx'> 		} else {</div><div class='ctx'> 			ret = build_isgl((__be64 *)sq-&gt;queue,</div><div class='hunk'>@@ -537,7 +537,7 @@ static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe,</div><div class='ctx'> 					 wr-&gt;sg_list, wr-&gt;num_sge, &amp;plen);</div><div class='ctx'> 			if (ret)</div><div class='ctx'> 				return ret;</div><div class='del'>-			size = sizeof wqe-&gt;send + sizeof(struct fw_ri_isgl) +</div><div class='add'>+			size = sizeof(wqe-&gt;send) + sizeof(struct fw_ri_isgl) +</div><div class='ctx'> 			       wr-&gt;num_sge * sizeof(struct fw_ri_sge);</div><div class='ctx'> 		}</div><div class='ctx'> 	} else {</div><div class='hunk'>@@ -545,7 +545,7 @@ static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe,</div><div class='ctx'> 		wqe-&gt;send.u.immd_src[0].r1 = 0;</div><div class='ctx'> 		wqe-&gt;send.u.immd_src[0].r2 = 0;</div><div class='ctx'> 		wqe-&gt;send.u.immd_src[0].immdlen = 0;</div><div class='del'>-		size = sizeof wqe-&gt;send + sizeof(struct fw_ri_immd);</div><div class='add'>+		size = sizeof(wqe-&gt;send) + sizeof(struct fw_ri_immd);</div><div class='ctx'> 		plen = 0;</div><div class='ctx'> 	}</div><div class='ctx'> 	*len16 = DIV_ROUND_UP(size, 16);</div><div class='hunk'>@@ -579,7 +579,7 @@ static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe,</div><div class='ctx'> 					 T4_MAX_WRITE_INLINE, &amp;plen);</div><div class='ctx'> 			if (ret)</div><div class='ctx'> 				return ret;</div><div class='del'>-			size = sizeof wqe-&gt;write + sizeof(struct fw_ri_immd) +</div><div class='add'>+			size = sizeof(wqe-&gt;write) + sizeof(struct fw_ri_immd) +</div><div class='ctx'> 			       plen;</div><div class='ctx'> 		} else {</div><div class='ctx'> 			ret = build_isgl((__be64 *)sq-&gt;queue,</div><div class='hunk'>@@ -588,7 +588,7 @@ static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe,</div><div class='ctx'> 					 wr-&gt;sg_list, wr-&gt;num_sge, &amp;plen);</div><div class='ctx'> 			if (ret)</div><div class='ctx'> 				return ret;</div><div class='del'>-			size = sizeof wqe-&gt;write + sizeof(struct fw_ri_isgl) +</div><div class='add'>+			size = sizeof(wqe-&gt;write) + sizeof(struct fw_ri_isgl) +</div><div class='ctx'> 			       wr-&gt;num_sge * sizeof(struct fw_ri_sge);</div><div class='ctx'> 		}</div><div class='ctx'> 	} else {</div><div class='hunk'>@@ -596,7 +596,7 @@ static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe,</div><div class='ctx'> 		wqe-&gt;write.u.immd_src[0].r1 = 0;</div><div class='ctx'> 		wqe-&gt;write.u.immd_src[0].r2 = 0;</div><div class='ctx'> 		wqe-&gt;write.u.immd_src[0].immdlen = 0;</div><div class='del'>-		size = sizeof wqe-&gt;write + sizeof(struct fw_ri_immd);</div><div class='add'>+		size = sizeof(wqe-&gt;write) + sizeof(struct fw_ri_immd);</div><div class='ctx'> 		plen = 0;</div><div class='ctx'> 	}</div><div class='ctx'> 	*len16 = DIV_ROUND_UP(size, 16);</div><div class='hunk'>@@ -683,7 +683,7 @@ static int build_rdma_read(union t4_wr *wqe, const struct ib_send_wr *wr,</div><div class='ctx'> 	}</div><div class='ctx'> 	wqe-&gt;read.r2 = 0;</div><div class='ctx'> 	wqe-&gt;read.r5 = 0;</div><div class='del'>-	*len16 = DIV_ROUND_UP(sizeof wqe-&gt;read, 16);</div><div class='add'>+	*len16 = DIV_ROUND_UP(sizeof(wqe-&gt;read), 16);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -766,8 +766,8 @@ static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe,</div><div class='ctx'> 			 &amp;wqe-&gt;recv.isgl, wr-&gt;sg_list, wr-&gt;num_sge, NULL);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='del'>-	*len16 = DIV_ROUND_UP(sizeof wqe-&gt;recv +</div><div class='del'>-			      wr-&gt;num_sge * sizeof(struct fw_ri_sge), 16);</div><div class='add'>+	*len16 = DIV_ROUND_UP(</div><div class='add'>+		sizeof(wqe-&gt;recv) + wr-&gt;num_sge * sizeof(struct fw_ri_sge), 16);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -886,47 +886,21 @@ static int build_inv_stag(union t4_wr *wqe, const struct ib_send_wr *wr,</div><div class='ctx'> {</div><div class='ctx'> 	wqe-&gt;inv.stag_inv = cpu_to_be32(wr-&gt;ex.invalidate_rkey);</div><div class='ctx'> 	wqe-&gt;inv.r2 = 0;</div><div class='del'>-	*len16 = DIV_ROUND_UP(sizeof wqe-&gt;inv, 16);</div><div class='add'>+	*len16 = DIV_ROUND_UP(sizeof(wqe-&gt;inv), 16);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void free_qp_work(struct work_struct *work)</div><div class='del'>-{</div><div class='del'>-	struct c4iw_ucontext *ucontext;</div><div class='del'>-	struct c4iw_qp *qhp;</div><div class='del'>-	struct c4iw_dev *rhp;</div><div class='del'>-</div><div class='del'>-	qhp = container_of(work, struct c4iw_qp, free_work);</div><div class='del'>-	ucontext = qhp-&gt;ucontext;</div><div class='del'>-	rhp = qhp-&gt;rhp;</div><div class='del'>-</div><div class='del'>-	pr_debug("qhp %p ucontext %p\n", qhp, ucontext);</div><div class='del'>-	destroy_qp(&amp;rhp-&gt;rdev, &amp;qhp-&gt;wq,</div><div class='del'>-		   ucontext ? &amp;ucontext-&gt;uctx : &amp;rhp-&gt;rdev.uctx, !qhp-&gt;srq);</div><div class='del'>-</div><div class='del'>-	c4iw_put_wr_wait(qhp-&gt;wr_waitp);</div><div class='del'>-	kfree(qhp);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void queue_qp_free(struct kref *kref)</div><div class='del'>-{</div><div class='del'>-	struct c4iw_qp *qhp;</div><div class='del'>-</div><div class='del'>-	qhp = container_of(kref, struct c4iw_qp, kref);</div><div class='del'>-	pr_debug("qhp %p\n", qhp);</div><div class='del'>-	queue_work(qhp-&gt;rhp-&gt;rdev.free_workq, &amp;qhp-&gt;free_work);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> void c4iw_qp_add_ref(struct ib_qp *qp)</div><div class='ctx'> {</div><div class='ctx'> 	pr_debug("ib_qp %p\n", qp);</div><div class='del'>-	kref_get(&amp;to_c4iw_qp(qp)-&gt;kref);</div><div class='add'>+	refcount_inc(&amp;to_c4iw_qp(qp)-&gt;qp_refcnt);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void c4iw_qp_rem_ref(struct ib_qp *qp)</div><div class='ctx'> {</div><div class='ctx'> 	pr_debug("ib_qp %p\n", qp);</div><div class='del'>-	kref_put(&amp;to_c4iw_qp(qp)-&gt;kref, queue_qp_free);</div><div class='add'>+	if (refcount_dec_and_test(&amp;to_c4iw_qp(qp)-&gt;qp_refcnt))</div><div class='add'>+		complete(&amp;to_c4iw_qp(qp)-&gt;qp_rel_comp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void add_to_fc_list(struct list_head *head, struct list_head *entry)</div><div class='hunk'>@@ -1192,7 +1166,7 @@ int c4iw_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,</div><div class='ctx'> 				break;</div><div class='ctx'> 			}</div><div class='ctx'> 			fw_flags |= FW_RI_RDMA_WRITE_WITH_IMMEDIATE;</div><div class='del'>-			/*FALLTHROUGH*/</div><div class='add'>+			fallthrough;</div><div class='ctx'> 		case IB_WR_RDMA_WRITE:</div><div class='ctx'> 			fw_opcode = FW_RI_RDMA_WRITE_WR;</div><div class='ctx'> 			swsqe-&gt;opcode = FW_RI_RDMA_WRITE;</div><div class='hunk'>@@ -1606,7 +1580,7 @@ static void post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe,</div><div class='ctx'> 		FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe), 16)));</div><div class='ctx'> </div><div class='ctx'> 	wqe-&gt;u.terminate.type = FW_RI_TYPE_TERMINATE;</div><div class='del'>-	wqe-&gt;u.terminate.immdlen = cpu_to_be32(sizeof *term);</div><div class='add'>+	wqe-&gt;u.terminate.immdlen = cpu_to_be32(sizeof(*term));</div><div class='ctx'> 	term = (struct terminate_message *)wqe-&gt;u.terminate.termmsg;</div><div class='ctx'> 	if (qhp-&gt;attr.layer_etype == (LAYER_MPA|DDP_LLP)) {</div><div class='ctx'> 		term-&gt;layer_etype = qhp-&gt;attr.layer_etype;</div><div class='hunk'>@@ -1625,6 +1599,7 @@ static void __flush_qp(struct c4iw_qp *qhp, struct c4iw_cq *rchp,</div><div class='ctx'> 	int count;</div><div class='ctx'> 	int rq_flushed = 0, sq_flushed;</div><div class='ctx'> 	unsigned long flag;</div><div class='add'>+	struct ib_event ev;</div><div class='ctx'> </div><div class='ctx'> 	pr_debug("qhp %p rchp %p schp %p\n", qhp, rchp, schp);</div><div class='ctx'> </div><div class='hunk'>@@ -1633,6 +1608,13 @@ static void __flush_qp(struct c4iw_qp *qhp, struct c4iw_cq *rchp,</div><div class='ctx'> 	if (schp != rchp)</div><div class='ctx'> 		spin_lock(&amp;schp-&gt;lock);</div><div class='ctx'> 	spin_lock(&amp;qhp-&gt;lock);</div><div class='add'>+	if (qhp-&gt;srq &amp;&amp; qhp-&gt;attr.state == C4IW_QP_STATE_ERROR &amp;&amp;</div><div class='add'>+	    qhp-&gt;ibqp.event_handler) {</div><div class='add'>+		ev.device = qhp-&gt;ibqp.device;</div><div class='add'>+		ev.element.qp = &amp;qhp-&gt;ibqp;</div><div class='add'>+		ev.event = IB_EVENT_QP_LAST_WQE_REACHED;</div><div class='add'>+		qhp-&gt;ibqp.event_handler(&amp;ev, qhp-&gt;ibqp.qp_context);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	if (qhp-&gt;wq.flushed) {</div><div class='ctx'> 		spin_unlock(&amp;qhp-&gt;lock);</div><div class='hunk'>@@ -1751,16 +1733,15 @@ static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp,</div><div class='ctx'> static void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init)</div><div class='ctx'> {</div><div class='ctx'> 	pr_debug("p2p_type = %d\n", p2p_type);</div><div class='del'>-	memset(&amp;init-&gt;u, 0, sizeof init-&gt;u);</div><div class='add'>+	memset(&amp;init-&gt;u, 0, sizeof(init-&gt;u));</div><div class='ctx'> 	switch (p2p_type) {</div><div class='ctx'> 	case FW_RI_INIT_P2PTYPE_RDMA_WRITE:</div><div class='ctx'> 		init-&gt;u.write.opcode = FW_RI_RDMA_WRITE_WR;</div><div class='ctx'> 		init-&gt;u.write.stag_sink = cpu_to_be32(1);</div><div class='ctx'> 		init-&gt;u.write.to_sink = cpu_to_be64(1);</div><div class='ctx'> 		init-&gt;u.write.u.immd_src[0].op = FW_RI_DATA_IMMD;</div><div class='del'>-		init-&gt;u.write.len16 = DIV_ROUND_UP(sizeof init-&gt;u.write +</div><div class='del'>-						   sizeof(struct fw_ri_immd),</div><div class='del'>-						   16);</div><div class='add'>+		init-&gt;u.write.len16 = DIV_ROUND_UP(</div><div class='add'>+			sizeof(init-&gt;u.write) + sizeof(struct fw_ri_immd), 16);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case FW_RI_INIT_P2PTYPE_READ_REQ:</div><div class='ctx'> 		init-&gt;u.write.opcode = FW_RI_RDMA_READ_WR;</div><div class='hunk'>@@ -1768,7 +1749,7 @@ static void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init)</div><div class='ctx'> 		init-&gt;u.read.to_src_lo = cpu_to_be32(1);</div><div class='ctx'> 		init-&gt;u.read.stag_sink = cpu_to_be32(1);</div><div class='ctx'> 		init-&gt;u.read.to_sink_lo = cpu_to_be32(1);</div><div class='del'>-		init-&gt;u.read.len16 = DIV_ROUND_UP(sizeof init-&gt;u.read, 16);</div><div class='add'>+		init-&gt;u.read.len16 = DIV_ROUND_UP(sizeof(init-&gt;u.read), 16);</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='hunk'>@@ -1782,7 +1763,7 @@ static int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp)</div><div class='ctx'> 	pr_debug("qhp %p qid 0x%x tid %u ird %u ord %u\n", qhp,</div><div class='ctx'> 		 qhp-&gt;wq.sq.qid, qhp-&gt;ep-&gt;hwtid, qhp-&gt;ep-&gt;ird, qhp-&gt;ep-&gt;ord);</div><div class='ctx'> </div><div class='del'>-	skb = alloc_skb(sizeof *wqe, GFP_KERNEL);</div><div class='add'>+	skb = alloc_skb(sizeof(*wqe), GFP_KERNEL);</div><div class='ctx'> 	if (!skb) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='ctx'> 		goto out;</div><div class='hunk'>@@ -1976,10 +1957,10 @@ int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp,</div><div class='ctx'> 			qhp-&gt;attr.layer_etype = attrs-&gt;layer_etype;</div><div class='ctx'> 			qhp-&gt;attr.ecode = attrs-&gt;ecode;</div><div class='ctx'> 			ep = qhp-&gt;ep;</div><div class='del'>-			c4iw_get_ep(&amp;ep-&gt;com);</div><div class='del'>-			disconnect = 1;</div><div class='ctx'> 			if (!internal) {</div><div class='add'>+				c4iw_get_ep(&amp;ep-&gt;com);</div><div class='ctx'> 				terminate = 1;</div><div class='add'>+				disconnect = 1;</div><div class='ctx'> 			} else {</div><div class='ctx'> 				terminate = qhp-&gt;attr.send_term;</div><div class='ctx'> 				ret = rdma_fini(rhp, qhp, ep);</div><div class='hunk'>@@ -1991,7 +1972,6 @@ int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp,</div><div class='ctx'> 			t4_set_wq_in_error(&amp;qhp-&gt;wq, 0);</div><div class='ctx'> 			set_state(qhp, C4IW_QP_STATE_ERROR);</div><div class='ctx'> 			if (!internal) {</div><div class='del'>-				abort = 1;</div><div class='ctx'> 				disconnect = 1;</div><div class='ctx'> 				ep = qhp-&gt;ep;</div><div class='ctx'> 				c4iw_get_ep(&amp;qhp-&gt;ep-&gt;com);</div><div class='hunk'>@@ -2099,10 +2079,12 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct c4iw_dev *rhp;</div><div class='ctx'> 	struct c4iw_qp *qhp;</div><div class='add'>+	struct c4iw_ucontext *ucontext;</div><div class='ctx'> 	struct c4iw_qp_attributes attrs;</div><div class='ctx'> </div><div class='ctx'> 	qhp = to_c4iw_qp(ib_qp);</div><div class='ctx'> 	rhp = qhp-&gt;rhp;</div><div class='add'>+	ucontext = qhp-&gt;ucontext;</div><div class='ctx'> </div><div class='ctx'> 	attrs.next_state = C4IW_QP_STATE_ERROR;</div><div class='ctx'> 	if (qhp-&gt;attr.state == C4IW_QP_STATE_TERMINATE)</div><div class='hunk'>@@ -2120,15 +2102,24 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)</div><div class='ctx'> </div><div class='ctx'> 	c4iw_qp_rem_ref(ib_qp);</div><div class='ctx'> </div><div class='add'>+	wait_for_completion(&amp;qhp-&gt;qp_rel_comp);</div><div class='add'>+</div><div class='ctx'> 	pr_debug("ib_qp %p qpid 0x%0x\n", ib_qp, qhp-&gt;wq.sq.qid);</div><div class='add'>+	pr_debug("qhp %p ucontext %p\n", qhp, ucontext);</div><div class='add'>+</div><div class='add'>+	destroy_qp(&amp;rhp-&gt;rdev, &amp;qhp-&gt;wq,</div><div class='add'>+		   ucontext ? &amp;ucontext-&gt;uctx : &amp;rhp-&gt;rdev.uctx, !qhp-&gt;srq);</div><div class='add'>+</div><div class='add'>+	c4iw_put_wr_wait(qhp-&gt;wr_waitp);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,</div><div class='del'>-			     struct ib_udata *udata)</div><div class='add'>+int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs,</div><div class='add'>+		   struct ib_udata *udata)</div><div class='ctx'> {</div><div class='add'>+	struct ib_pd *pd = qp-&gt;pd;</div><div class='ctx'> 	struct c4iw_dev *rhp;</div><div class='del'>-	struct c4iw_qp *qhp;</div><div class='add'>+	struct c4iw_qp *qhp = to_c4iw_qp(qp);</div><div class='ctx'> 	struct c4iw_pd *php;</div><div class='ctx'> 	struct c4iw_cq *schp;</div><div class='ctx'> 	struct c4iw_cq *rchp;</div><div class='hunk'>@@ -2140,44 +2131,36 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,</div><div class='ctx'> 	struct c4iw_mm_entry *sq_key_mm, *rq_key_mm = NULL, *sq_db_key_mm;</div><div class='ctx'> 	struct c4iw_mm_entry *rq_db_key_mm = NULL, *ma_sync_key_mm = NULL;</div><div class='ctx'> </div><div class='del'>-	pr_debug("ib_pd %p\n", pd);</div><div class='del'>-</div><div class='del'>-	if (attrs-&gt;qp_type != IB_QPT_RC)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+	if (attrs-&gt;qp_type != IB_QPT_RC || attrs-&gt;create_flags)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	php = to_c4iw_pd(pd);</div><div class='ctx'> 	rhp = php-&gt;rhp;</div><div class='ctx'> 	schp = get_chp(rhp, ((struct c4iw_cq *)attrs-&gt;send_cq)-&gt;cq.cqid);</div><div class='ctx'> 	rchp = get_chp(rhp, ((struct c4iw_cq *)attrs-&gt;recv_cq)-&gt;cq.cqid);</div><div class='ctx'> 	if (!schp || !rchp)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	if (attrs-&gt;cap.max_inline_data &gt; T4_MAX_SEND_INLINE)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	if (!attrs-&gt;srq) {</div><div class='ctx'> 		if (attrs-&gt;cap.max_recv_wr &gt; rhp-&gt;rdev.hw_queue.t4_max_rq_size)</div><div class='del'>-			return ERR_PTR(-E2BIG);</div><div class='add'>+			return -E2BIG;</div><div class='ctx'> 		rqsize = attrs-&gt;cap.max_recv_wr + 1;</div><div class='ctx'> 		if (rqsize &lt; 8)</div><div class='ctx'> 			rqsize = 8;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (attrs-&gt;cap.max_send_wr &gt; rhp-&gt;rdev.hw_queue.t4_max_sq_size)</div><div class='del'>-		return ERR_PTR(-E2BIG);</div><div class='add'>+		return -E2BIG;</div><div class='ctx'> 	sqsize = attrs-&gt;cap.max_send_wr + 1;</div><div class='ctx'> 	if (sqsize &lt; 8)</div><div class='ctx'> 		sqsize = 8;</div><div class='ctx'> </div><div class='del'>-	qhp = kzalloc(sizeof(*qhp), GFP_KERNEL);</div><div class='del'>-	if (!qhp)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='ctx'> 	qhp-&gt;wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL);</div><div class='del'>-	if (!qhp-&gt;wr_waitp) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err_free_qhp;</div><div class='del'>-	}</div><div class='add'>+	if (!qhp-&gt;wr_waitp)</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	qhp-&gt;wq.sq.size = sqsize;</div><div class='ctx'> 	qhp-&gt;wq.sq.memsize =</div><div class='hunk'>@@ -2230,8 +2213,8 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,</div><div class='ctx'> 	spin_lock_init(&amp;qhp-&gt;lock);</div><div class='ctx'> 	mutex_init(&amp;qhp-&gt;mutex);</div><div class='ctx'> 	init_waitqueue_head(&amp;qhp-&gt;wait);</div><div class='del'>-	kref_init(&amp;qhp-&gt;kref);</div><div class='del'>-	INIT_WORK(&amp;qhp-&gt;free_work, free_qp_work);</div><div class='add'>+	init_completion(&amp;qhp-&gt;qp_rel_comp);</div><div class='add'>+	refcount_set(&amp;qhp-&gt;qp_refcnt, 1);</div><div class='ctx'> </div><div class='ctx'> 	ret = xa_insert_irq(&amp;rhp-&gt;qps, qhp-&gt;wq.sq.qid, qhp, GFP_KERNEL);</div><div class='ctx'> 	if (ret)</div><div class='hunk'>@@ -2302,28 +2285,43 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,</div><div class='ctx'> 			ucontext-&gt;key += PAGE_SIZE;</div><div class='ctx'> 		}</div><div class='ctx'> 		spin_unlock(&amp;ucontext-&gt;mmap_lock);</div><div class='del'>-		ret = ib_copy_to_udata(udata, &amp;uresp, sizeof uresp);</div><div class='add'>+		ret = ib_copy_to_udata(udata, &amp;uresp, sizeof(uresp));</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto err_free_ma_sync_key;</div><div class='ctx'> 		sq_key_mm-&gt;key = uresp.sq_key;</div><div class='del'>-		sq_key_mm-&gt;addr = qhp-&gt;wq.sq.phys_addr;</div><div class='add'>+		sq_key_mm-&gt;addr = 0;</div><div class='add'>+		sq_key_mm-&gt;vaddr = qhp-&gt;wq.sq.queue;</div><div class='add'>+		sq_key_mm-&gt;dma_addr = qhp-&gt;wq.sq.dma_addr;</div><div class='ctx'> 		sq_key_mm-&gt;len = PAGE_ALIGN(qhp-&gt;wq.sq.memsize);</div><div class='add'>+		insert_flag_to_mmap(&amp;rhp-&gt;rdev, sq_key_mm, sq_key_mm-&gt;addr);</div><div class='ctx'> 		insert_mmap(ucontext, sq_key_mm);</div><div class='ctx'> 		if (!attrs-&gt;srq) {</div><div class='ctx'> 			rq_key_mm-&gt;key = uresp.rq_key;</div><div class='del'>-			rq_key_mm-&gt;addr = virt_to_phys(qhp-&gt;wq.rq.queue);</div><div class='add'>+			rq_key_mm-&gt;addr = 0;</div><div class='add'>+			rq_key_mm-&gt;vaddr = qhp-&gt;wq.rq.queue;</div><div class='add'>+			rq_key_mm-&gt;dma_addr = qhp-&gt;wq.rq.dma_addr;</div><div class='ctx'> 			rq_key_mm-&gt;len = PAGE_ALIGN(qhp-&gt;wq.rq.memsize);</div><div class='add'>+			insert_flag_to_mmap(&amp;rhp-&gt;rdev, rq_key_mm,</div><div class='add'>+					    rq_key_mm-&gt;addr);</div><div class='ctx'> 			insert_mmap(ucontext, rq_key_mm);</div><div class='ctx'> 		}</div><div class='ctx'> 		sq_db_key_mm-&gt;key = uresp.sq_db_gts_key;</div><div class='ctx'> 		sq_db_key_mm-&gt;addr = (u64)(unsigned long)qhp-&gt;wq.sq.bar2_pa;</div><div class='add'>+		sq_db_key_mm-&gt;vaddr = NULL;</div><div class='add'>+		sq_db_key_mm-&gt;dma_addr = 0;</div><div class='ctx'> 		sq_db_key_mm-&gt;len = PAGE_SIZE;</div><div class='add'>+		insert_flag_to_mmap(&amp;rhp-&gt;rdev, sq_db_key_mm,</div><div class='add'>+				    sq_db_key_mm-&gt;addr);</div><div class='ctx'> 		insert_mmap(ucontext, sq_db_key_mm);</div><div class='ctx'> 		if (!attrs-&gt;srq) {</div><div class='ctx'> 			rq_db_key_mm-&gt;key = uresp.rq_db_gts_key;</div><div class='ctx'> 			rq_db_key_mm-&gt;addr =</div><div class='ctx'> 				(u64)(unsigned long)qhp-&gt;wq.rq.bar2_pa;</div><div class='ctx'> 			rq_db_key_mm-&gt;len = PAGE_SIZE;</div><div class='add'>+			rq_db_key_mm-&gt;vaddr = NULL;</div><div class='add'>+			rq_db_key_mm-&gt;dma_addr = 0;</div><div class='add'>+			insert_flag_to_mmap(&amp;rhp-&gt;rdev, rq_db_key_mm,</div><div class='add'>+					    rq_db_key_mm-&gt;addr);</div><div class='ctx'> 			insert_mmap(ucontext, rq_db_key_mm);</div><div class='ctx'> 		}</div><div class='ctx'> 		if (ma_sync_key_mm) {</div><div class='hunk'>@@ -2332,6 +2330,10 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,</div><div class='ctx'> 				(pci_resource_start(rhp-&gt;rdev.lldi.pdev, 0) +</div><div class='ctx'> 				PCIE_MA_SYNC_A) &amp; PAGE_MASK;</div><div class='ctx'> 			ma_sync_key_mm-&gt;len = PAGE_SIZE;</div><div class='add'>+			ma_sync_key_mm-&gt;vaddr = NULL;</div><div class='add'>+			ma_sync_key_mm-&gt;dma_addr = 0;</div><div class='add'>+			insert_flag_to_mmap(&amp;rhp-&gt;rdev, ma_sync_key_mm,</div><div class='add'>+					    ma_sync_key_mm-&gt;addr);</div><div class='ctx'> 			insert_mmap(ucontext, ma_sync_key_mm);</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='hunk'>@@ -2355,7 +2357,7 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,</div><div class='ctx'> 		 qhp-&gt;wq.sq.qid, qhp-&gt;wq.sq.size, qhp-&gt;wq.sq.memsize,</div><div class='ctx'> 		 attrs-&gt;cap.max_send_wr, qhp-&gt;wq.rq.qid, qhp-&gt;wq.rq.size,</div><div class='ctx'> 		 qhp-&gt;wq.rq.memsize, attrs-&gt;cap.max_recv_wr);</div><div class='del'>-	return &amp;qhp-&gt;ibqp;</div><div class='add'>+	return 0;</div><div class='ctx'> err_free_ma_sync_key:</div><div class='ctx'> 	kfree(ma_sync_key_mm);</div><div class='ctx'> err_free_rq_db_key:</div><div class='hunk'>@@ -2375,9 +2377,7 @@ err_destroy_qp:</div><div class='ctx'> 		   ucontext ? &amp;ucontext-&gt;uctx : &amp;rhp-&gt;rdev.uctx, !attrs-&gt;srq);</div><div class='ctx'> err_free_wr_wait:</div><div class='ctx'> 	c4iw_put_wr_wait(qhp-&gt;wr_waitp);</div><div class='del'>-err_free_qhp:</div><div class='del'>-	kfree(qhp);</div><div class='del'>-	return ERR_PTR(ret);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='hunk'>@@ -2386,10 +2386,13 @@ int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> 	struct c4iw_dev *rhp;</div><div class='ctx'> 	struct c4iw_qp *qhp;</div><div class='ctx'> 	enum c4iw_qp_attr_mask mask = 0;</div><div class='del'>-	struct c4iw_qp_attributes attrs;</div><div class='add'>+	struct c4iw_qp_attributes attrs = {};</div><div class='ctx'> </div><div class='ctx'> 	pr_debug("ib_qp %p\n", ibqp);</div><div class='ctx'> </div><div class='add'>+	if (attr_mask &amp; ~IB_QP_ATTR_STANDARD_BITS)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='ctx'> 	/* iwarp does not support the RTR state */</div><div class='ctx'> 	if ((attr_mask &amp; IB_QP_STATE) &amp;&amp; (attr-&gt;qp_state == IB_QPS_RTR))</div><div class='ctx'> 		attr_mask &amp;= ~IB_QP_STATE;</div><div class='hunk'>@@ -2398,7 +2401,6 @@ int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> 	if (!attr_mask)</div><div class='ctx'> 		return 0;</div><div class='ctx'> </div><div class='del'>-	memset(&amp;attrs, 0, sizeof attrs);</div><div class='ctx'> 	qhp = to_c4iw_qp(ibqp);</div><div class='ctx'> 	rhp = qhp-&gt;rhp;</div><div class='ctx'> </div><div class='hunk'>@@ -2482,15 +2484,16 @@ int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> {</div><div class='ctx'> 	struct c4iw_qp *qhp = to_c4iw_qp(ibqp);</div><div class='ctx'> </div><div class='del'>-	memset(attr, 0, sizeof *attr);</div><div class='del'>-	memset(init_attr, 0, sizeof *init_attr);</div><div class='add'>+	memset(attr, 0, sizeof(*attr));</div><div class='add'>+	memset(init_attr, 0, sizeof(*init_attr));</div><div class='ctx'> 	attr-&gt;qp_state = to_ib_qp_state(qhp-&gt;attr.state);</div><div class='add'>+	attr-&gt;cur_qp_state = to_ib_qp_state(qhp-&gt;attr.state);</div><div class='ctx'> 	init_attr-&gt;cap.max_send_wr = qhp-&gt;attr.sq_num_entries;</div><div class='ctx'> 	init_attr-&gt;cap.max_recv_wr = qhp-&gt;attr.rq_num_entries;</div><div class='ctx'> 	init_attr-&gt;cap.max_send_sge = qhp-&gt;attr.sq_max_sges;</div><div class='del'>-	init_attr-&gt;cap.max_recv_sge = qhp-&gt;attr.sq_max_sges;</div><div class='add'>+	init_attr-&gt;cap.max_recv_sge = qhp-&gt;attr.rq_max_sges;</div><div class='ctx'> 	init_attr-&gt;cap.max_inline_data = T4_MAX_SEND_INLINE;</div><div class='del'>-	init_attr-&gt;sq_sig_type = qhp-&gt;sq_sig_all ? IB_SIGNAL_ALL_WR : 0;</div><div class='add'>+	init_attr-&gt;sq_sig_type = qhp-&gt;sq_sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -2697,6 +2700,9 @@ int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs,</div><div class='ctx'> 	int ret;</div><div class='ctx'> 	int wr_len;</div><div class='ctx'> </div><div class='add'>+	if (attrs-&gt;srq_type != IB_SRQT_BASIC)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='ctx'> 	pr_debug("%s ib_pd %p\n", __func__, pd);</div><div class='ctx'> </div><div class='ctx'> 	php = to_c4iw_pd(pd);</div><div class='hunk'>@@ -2754,15 +2760,11 @@ int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs,</div><div class='ctx'> 	if (CHELSIO_CHIP_VERSION(rhp-&gt;rdev.lldi.adapter_type) &gt; CHELSIO_T6)</div><div class='ctx'> 		srq-&gt;flags = T4_SRQ_LIMIT_SUPPORT;</div><div class='ctx'> </div><div class='del'>-	ret = xa_insert_irq(&amp;rhp-&gt;qps, srq-&gt;wq.qid, srq, GFP_KERNEL);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_free_queue;</div><div class='del'>-</div><div class='ctx'> 	if (udata) {</div><div class='ctx'> 		srq_key_mm = kmalloc(sizeof(*srq_key_mm), GFP_KERNEL);</div><div class='ctx'> 		if (!srq_key_mm) {</div><div class='ctx'> 			ret = -ENOMEM;</div><div class='del'>-			goto err_remove_handle;</div><div class='add'>+			goto err_free_queue;</div><div class='ctx'> 		}</div><div class='ctx'> 		srq_db_key_mm = kmalloc(sizeof(*srq_db_key_mm), GFP_KERNEL);</div><div class='ctx'> 		if (!srq_db_key_mm) {</div><div class='hunk'>@@ -2786,12 +2788,19 @@ int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs,</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto err_free_srq_db_key_mm;</div><div class='ctx'> 		srq_key_mm-&gt;key = uresp.srq_key;</div><div class='del'>-		srq_key_mm-&gt;addr = virt_to_phys(srq-&gt;wq.queue);</div><div class='add'>+		srq_key_mm-&gt;addr = 0;</div><div class='ctx'> 		srq_key_mm-&gt;len = PAGE_ALIGN(srq-&gt;wq.memsize);</div><div class='add'>+		srq_key_mm-&gt;vaddr = srq-&gt;wq.queue;</div><div class='add'>+		srq_key_mm-&gt;dma_addr = srq-&gt;wq.dma_addr;</div><div class='add'>+		insert_flag_to_mmap(&amp;rhp-&gt;rdev, srq_key_mm, srq_key_mm-&gt;addr);</div><div class='ctx'> 		insert_mmap(ucontext, srq_key_mm);</div><div class='ctx'> 		srq_db_key_mm-&gt;key = uresp.srq_db_gts_key;</div><div class='ctx'> 		srq_db_key_mm-&gt;addr = (u64)(unsigned long)srq-&gt;wq.bar2_pa;</div><div class='ctx'> 		srq_db_key_mm-&gt;len = PAGE_SIZE;</div><div class='add'>+		srq_db_key_mm-&gt;vaddr = NULL;</div><div class='add'>+		srq_db_key_mm-&gt;dma_addr = 0;</div><div class='add'>+		insert_flag_to_mmap(&amp;rhp-&gt;rdev, srq_db_key_mm,</div><div class='add'>+				    srq_db_key_mm-&gt;addr);</div><div class='ctx'> 		insert_mmap(ucontext, srq_db_key_mm);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -2806,8 +2815,6 @@ err_free_srq_db_key_mm:</div><div class='ctx'> 	kfree(srq_db_key_mm);</div><div class='ctx'> err_free_srq_key_mm:</div><div class='ctx'> 	kfree(srq_key_mm);</div><div class='del'>-err_remove_handle:</div><div class='del'>-	xa_erase_irq(&amp;rhp-&gt;qps, srq-&gt;wq.qid);</div><div class='ctx'> err_free_queue:</div><div class='ctx'> 	free_srq_queue(srq, ucontext ? &amp;ucontext-&gt;uctx : &amp;rhp-&gt;rdev.uctx,</div><div class='ctx'> 		       srq-&gt;wr_waitp);</div><div class='hunk'>@@ -2820,7 +2827,7 @@ err_free_wr_wait:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void c4iw_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)</div><div class='add'>+int c4iw_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct c4iw_dev *rhp;</div><div class='ctx'> 	struct c4iw_srq *srq;</div><div class='hunk'>@@ -2830,12 +2837,11 @@ void c4iw_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)</div><div class='ctx'> 	rhp = srq-&gt;rhp;</div><div class='ctx'> </div><div class='ctx'> 	pr_debug("%s id %d\n", __func__, srq-&gt;wq.qid);</div><div class='del'>-</div><div class='del'>-	xa_erase_irq(&amp;rhp-&gt;qps, srq-&gt;wq.qid);</div><div class='ctx'> 	ucontext = rdma_udata_to_drv_context(udata, struct c4iw_ucontext,</div><div class='ctx'> 					     ibucontext);</div><div class='ctx'> 	free_srq_queue(srq, ucontext ? &amp;ucontext-&gt;uctx : &amp;rhp-&gt;rdev.uctx,</div><div class='ctx'> 		       srq-&gt;wr_waitp);</div><div class='ctx'> 	c4iw_free_srq_idx(&amp;rhp-&gt;rdev, srq-&gt;idx);</div><div class='ctx'> 	c4iw_put_wr_wait(srq-&gt;wr_waitp);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb4/resource.c b/drivers/infiniband/hw/cxgb4/resource.c<br/>index 57ed26b3cc21..e800e8e8bed5 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/resource.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/resource.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/resource.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/cxgb4/resource.c</a></div><div class='hunk'>@@ -126,7 +126,7 @@ u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)</div><div class='ctx'> 		rdev-&gt;stats.qid.cur += rdev-&gt;qpmask + 1;</div><div class='ctx'> 		mutex_unlock(&amp;rdev-&gt;stats.lock);</div><div class='ctx'> 		for (i = qid+1; i &amp; rdev-&gt;qpmask; i++) {</div><div class='del'>-			entry = kmalloc(sizeof *entry, GFP_KERNEL);</div><div class='add'>+			entry = kmalloc(sizeof(*entry), GFP_KERNEL);</div><div class='ctx'> 			if (!entry)</div><div class='ctx'> 				goto out;</div><div class='ctx'> 			entry-&gt;qid = i;</div><div class='hunk'>@@ -137,13 +137,13 @@ u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)</div><div class='ctx'> 		 * now put the same ids on the qp list since they all</div><div class='ctx'> 		 * map to the same db/gts page.</div><div class='ctx'> 		 */</div><div class='del'>-		entry = kmalloc(sizeof *entry, GFP_KERNEL);</div><div class='add'>+		entry = kmalloc(sizeof(*entry), GFP_KERNEL);</div><div class='ctx'> 		if (!entry)</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		entry-&gt;qid = qid;</div><div class='ctx'> 		list_add_tail(&amp;entry-&gt;entry, &amp;uctx-&gt;qpids);</div><div class='ctx'> 		for (i = qid+1; i &amp; rdev-&gt;qpmask; i++) {</div><div class='del'>-			entry = kmalloc(sizeof *entry, GFP_KERNEL);</div><div class='add'>+			entry = kmalloc(sizeof(*entry), GFP_KERNEL);</div><div class='ctx'> 			if (!entry)</div><div class='ctx'> 				goto out;</div><div class='ctx'> 			entry-&gt;qid = i;</div><div class='hunk'>@@ -165,7 +165,7 @@ void c4iw_put_cqid(struct c4iw_rdev *rdev, u32 qid,</div><div class='ctx'> {</div><div class='ctx'> 	struct c4iw_qid_list *entry;</div><div class='ctx'> </div><div class='del'>-	entry = kmalloc(sizeof *entry, GFP_KERNEL);</div><div class='add'>+	entry = kmalloc(sizeof(*entry), GFP_KERNEL);</div><div class='ctx'> 	if (!entry)</div><div class='ctx'> 		return;</div><div class='ctx'> 	pr_debug("qid 0x%x\n", qid);</div><div class='hunk'>@@ -200,7 +200,7 @@ u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)</div><div class='ctx'> 		rdev-&gt;stats.qid.cur += rdev-&gt;qpmask + 1;</div><div class='ctx'> 		mutex_unlock(&amp;rdev-&gt;stats.lock);</div><div class='ctx'> 		for (i = qid+1; i &amp; rdev-&gt;qpmask; i++) {</div><div class='del'>-			entry = kmalloc(sizeof *entry, GFP_KERNEL);</div><div class='add'>+			entry = kmalloc(sizeof(*entry), GFP_KERNEL);</div><div class='ctx'> 			if (!entry)</div><div class='ctx'> 				goto out;</div><div class='ctx'> 			entry-&gt;qid = i;</div><div class='hunk'>@@ -211,13 +211,13 @@ u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)</div><div class='ctx'> 		 * now put the same ids on the cq list since they all</div><div class='ctx'> 		 * map to the same db/gts page.</div><div class='ctx'> 		 */</div><div class='del'>-		entry = kmalloc(sizeof *entry, GFP_KERNEL);</div><div class='add'>+		entry = kmalloc(sizeof(*entry), GFP_KERNEL);</div><div class='ctx'> 		if (!entry)</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		entry-&gt;qid = qid;</div><div class='ctx'> 		list_add_tail(&amp;entry-&gt;entry, &amp;uctx-&gt;cqids);</div><div class='del'>-		for (i = qid; i &amp; rdev-&gt;qpmask; i++) {</div><div class='del'>-			entry = kmalloc(sizeof *entry, GFP_KERNEL);</div><div class='add'>+		for (i = qid + 1; i &amp; rdev-&gt;qpmask; i++) {</div><div class='add'>+			entry = kmalloc(sizeof(*entry), GFP_KERNEL);</div><div class='ctx'> 			if (!entry)</div><div class='ctx'> 				goto out;</div><div class='ctx'> 			entry-&gt;qid = i;</div><div class='hunk'>@@ -239,7 +239,7 @@ void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qid,</div><div class='ctx'> {</div><div class='ctx'> 	struct c4iw_qid_list *entry;</div><div class='ctx'> </div><div class='del'>-	entry = kmalloc(sizeof *entry, GFP_KERNEL);</div><div class='add'>+	entry = kmalloc(sizeof(*entry), GFP_KERNEL);</div><div class='ctx'> 	if (!entry)</div><div class='ctx'> 		return;</div><div class='ctx'> 	pr_debug("qid 0x%x\n", qid);</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb4/restrack.c b/drivers/infiniband/hw/cxgb4/restrack.c<br/>index f82d46ed969d..fd22c85d35f4 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/restrack.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/restrack.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/restrack.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/cxgb4/restrack.c</a></div><div class='hunk'>@@ -134,10 +134,8 @@ err:</div><div class='ctx'> 	return -EMSGSIZE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int fill_res_qp_entry(struct sk_buff *msg,</div><div class='del'>-			     struct rdma_restrack_entry *res)</div><div class='add'>+int c4iw_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp)</div><div class='ctx'> {</div><div class='del'>-	struct ib_qp *ibqp = container_of(res, struct ib_qp, res);</div><div class='ctx'> 	struct t4_swsqe *fsp = NULL, *lsp = NULL;</div><div class='ctx'> 	struct c4iw_qp *qhp = to_c4iw_qp(ibqp);</div><div class='ctx'> 	u16 first_sq_idx = 0, last_sq_idx = 0;</div><div class='hunk'>@@ -195,10 +193,9 @@ union union_ep {</div><div class='ctx'> 	struct c4iw_ep ep;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static int fill_res_ep_entry(struct sk_buff *msg,</div><div class='del'>-			     struct rdma_restrack_entry *res)</div><div class='add'>+int c4iw_fill_res_cm_id_entry(struct sk_buff *msg,</div><div class='add'>+			      struct rdma_cm_id *cm_id)</div><div class='ctx'> {</div><div class='del'>-	struct rdma_cm_id *cm_id = rdma_res_to_id(res);</div><div class='ctx'> 	struct nlattr *table_attr;</div><div class='ctx'> 	struct c4iw_ep_common *epcp;</div><div class='ctx'> 	struct c4iw_listen_ep *listen_ep = NULL;</div><div class='hunk'>@@ -212,7 +209,7 @@ static int fill_res_ep_entry(struct sk_buff *msg,</div><div class='ctx'> 	epcp = (struct c4iw_ep_common *)iw_cm_id-&gt;provider_data;</div><div class='ctx'> 	if (!epcp)</div><div class='ctx'> 		return 0;</div><div class='del'>-	uep = kcalloc(1, sizeof(*uep), GFP_KERNEL);</div><div class='add'>+	uep = kzalloc(sizeof(*uep), GFP_KERNEL);</div><div class='ctx'> 	if (!uep)</div><div class='ctx'> 		return 0;</div><div class='ctx'> </div><div class='hunk'>@@ -241,7 +238,7 @@ static int fill_res_ep_entry(struct sk_buff *msg,</div><div class='ctx'> 	if (rdma_nl_put_driver_u64_hex(msg, "history", epcp-&gt;history))</div><div class='ctx'> 		goto err_cancel_table;</div><div class='ctx'> </div><div class='del'>-	if (epcp-&gt;state == LISTEN) {</div><div class='add'>+	if (listen_ep) {</div><div class='ctx'> 		if (rdma_nl_put_driver_u32(msg, "stid", listen_ep-&gt;stid))</div><div class='ctx'> 			goto err_cancel_table;</div><div class='ctx'> 		if (rdma_nl_put_driver_u32(msg, "backlog", listen_ep-&gt;backlog))</div><div class='hunk'>@@ -372,10 +369,8 @@ err:</div><div class='ctx'> 	return -EMSGSIZE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int fill_res_cq_entry(struct sk_buff *msg,</div><div class='del'>-			     struct rdma_restrack_entry *res)</div><div class='add'>+int c4iw_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq)</div><div class='ctx'> {</div><div class='del'>-	struct ib_cq *ibcq = container_of(res, struct ib_cq, res);</div><div class='ctx'> 	struct c4iw_cq *chp = to_c4iw_cq(ibcq);</div><div class='ctx'> 	struct nlattr *table_attr;</div><div class='ctx'> 	struct t4_cqe hwcqes[2];</div><div class='hunk'>@@ -433,10 +428,8 @@ err:</div><div class='ctx'> 	return -EMSGSIZE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int fill_res_mr_entry(struct sk_buff *msg,</div><div class='del'>-			     struct rdma_restrack_entry *res)</div><div class='add'>+int c4iw_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr)</div><div class='ctx'> {</div><div class='del'>-	struct ib_mr *ibmr = container_of(res, struct ib_mr, res);</div><div class='ctx'> 	struct c4iw_mr *mhp = to_c4iw_mr(ibmr);</div><div class='ctx'> 	struct c4iw_dev *dev = mhp-&gt;rhp;</div><div class='ctx'> 	u32 stag = mhp-&gt;attr.stag;</div><div class='hunk'>@@ -492,10 +485,3 @@ err_cancel_table:</div><div class='ctx'> err:</div><div class='ctx'> 	return -EMSGSIZE;</div><div class='ctx'> }</div><div class='del'>-</div><div class='del'>-c4iw_restrack_func *c4iw_restrack_funcs[RDMA_RESTRACK_MAX] = {</div><div class='del'>-	[RDMA_RESTRACK_QP]	= fill_res_qp_entry,</div><div class='del'>-	[RDMA_RESTRACK_CM_ID]	= fill_res_ep_entry,</div><div class='del'>-	[RDMA_RESTRACK_CQ]	= fill_res_cq_entry,</div><div class='del'>-	[RDMA_RESTRACK_MR]	= fill_res_mr_entry,</div><div class='del'>-};</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb4/t4.h b/drivers/infiniband/hw/cxgb4/t4.h<br/>index b170817b2741..c3b0e2896475 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/t4.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/t4.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/t4.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/cxgb4/t4.h</a></div><div class='hunk'>@@ -487,11 +487,6 @@ static inline int t4_rq_empty(struct t4_wq *wq)</div><div class='ctx'> 	return wq-&gt;rq.in_use == 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline int t4_rq_full(struct t4_wq *wq)</div><div class='del'>-{</div><div class='del'>-	return wq-&gt;rq.in_use == (wq-&gt;rq.size - 1);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static inline u32 t4_rq_avail(struct t4_wq *wq)</div><div class='ctx'> {</div><div class='ctx'> 	return wq-&gt;rq.size - 1 - wq-&gt;rq.in_use;</div><div class='hunk'>@@ -534,11 +529,6 @@ static inline int t4_sq_empty(struct t4_wq *wq)</div><div class='ctx'> 	return wq-&gt;sq.in_use == 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline int t4_sq_full(struct t4_wq *wq)</div><div class='del'>-{</div><div class='del'>-	return wq-&gt;sq.in_use == (wq-&gt;sq.size - 1);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static inline u32 t4_sq_avail(struct t4_wq *wq)</div><div class='ctx'> {</div><div class='ctx'> 	return wq-&gt;sq.size - 1 - wq-&gt;sq.in_use;</div><div class='hunk'>@@ -679,11 +669,6 @@ static inline void t4_enable_wq_db(struct t4_wq *wq)</div><div class='ctx'> 	wq-&gt;rq.queue[wq-&gt;rq.size].status.db_off = 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline int t4_wq_db_enabled(struct t4_wq *wq)</div><div class='del'>-{</div><div class='del'>-	return !wq-&gt;rq.queue[wq-&gt;rq.size].status.db_off;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> enum t4_cq_flags {</div><div class='ctx'> 	CQ_ARMED	= 1,</div><div class='ctx'> };</div><div class='hunk'>@@ -817,19 +802,6 @@ static inline int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe)</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline struct t4_cqe *t4_next_sw_cqe(struct t4_cq *cq)</div><div class='del'>-{</div><div class='del'>-	if (cq-&gt;sw_in_use == cq-&gt;size) {</div><div class='del'>-		pr_warn("%s cxgb4 sw cq overflow cqid %u\n",</div><div class='del'>-			__func__, cq-&gt;cqid);</div><div class='del'>-		cq-&gt;error = 1;</div><div class='del'>-		return NULL;</div><div class='del'>-	}</div><div class='del'>-	if (cq-&gt;sw_in_use)</div><div class='del'>-		return &amp;cq-&gt;sw_queue[cq-&gt;sw_cidx];</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static inline int t4_next_cqe(struct t4_cq *cq, struct t4_cqe **cqe)</div><div class='ctx'> {</div><div class='ctx'> 	int ret = 0;</div><div class='hunk'>@@ -843,11 +815,6 @@ static inline int t4_next_cqe(struct t4_cq *cq, struct t4_cqe **cqe)</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline int t4_cq_in_error(struct t4_cq *cq)</div><div class='del'>-{</div><div class='del'>-	return *cq-&gt;qp_errp;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static inline void t4_set_cq_in_error(struct t4_cq *cq)</div><div class='ctx'> {</div><div class='ctx'> 	*cq-&gt;qp_errp = 1;</div><div class='head'>diff --git a/drivers/infiniband/hw/cxgb4/t4fw_ri_api.h b/drivers/infiniband/hw/cxgb4/t4fw_ri_api.h<br/>index cbdb300a4794..1f79537fc8d1 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/t4fw_ri_api.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/cxgb4/t4fw_ri_api.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/cxgb4/t4fw_ri_api.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/cxgb4/t4fw_ri_api.h</a></div><div class='hunk'>@@ -122,9 +122,7 @@ struct fw_ri_dsgl {</div><div class='ctx'> 	__be16	nsge;</div><div class='ctx'> 	__be32	len0;</div><div class='ctx'> 	__be64	addr0;</div><div class='del'>-#ifndef C99_NOT_SUPPORTED</div><div class='del'>-	struct fw_ri_dsge_pair sge[0];</div><div class='del'>-#endif</div><div class='add'>+	struct fw_ri_dsge_pair sge[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct fw_ri_sge {</div><div class='hunk'>@@ -138,9 +136,7 @@ struct fw_ri_isgl {</div><div class='ctx'> 	__u8	r1;</div><div class='ctx'> 	__be16	nsge;</div><div class='ctx'> 	__be32	r2;</div><div class='del'>-#ifndef C99_NOT_SUPPORTED</div><div class='del'>-	struct fw_ri_sge sge[0];</div><div class='del'>-#endif</div><div class='add'>+	struct fw_ri_sge sge[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct fw_ri_immd {</div><div class='hunk'>@@ -148,9 +144,7 @@ struct fw_ri_immd {</div><div class='ctx'> 	__u8	r1;</div><div class='ctx'> 	__be16	r2;</div><div class='ctx'> 	__be32	immdlen;</div><div class='del'>-#ifndef C99_NOT_SUPPORTED</div><div class='del'>-	__u8	data[0];</div><div class='del'>-#endif</div><div class='add'>+	__u8	data[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct fw_ri_tpte {</div><div class='hunk'>@@ -320,9 +314,7 @@ struct fw_ri_res_wr {</div><div class='ctx'> 	__be32 op_nres;</div><div class='ctx'> 	__be32 len16_pkd;</div><div class='ctx'> 	__u64  cookie;</div><div class='del'>-#ifndef C99_NOT_SUPPORTED</div><div class='del'>-	struct fw_ri_res res[0];</div><div class='del'>-#endif</div><div class='add'>+	struct fw_ri_res res[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> #define FW_RI_RES_WR_NRES_S	0</div><div class='hunk'>@@ -562,12 +554,10 @@ struct fw_ri_rdma_write_wr {</div><div class='ctx'> 	__be32 plen;</div><div class='ctx'> 	__be32 stag_sink;</div><div class='ctx'> 	__be64 to_sink;</div><div class='del'>-#ifndef C99_NOT_SUPPORTED</div><div class='ctx'> 	union {</div><div class='del'>-		struct fw_ri_immd immd_src[0];</div><div class='del'>-		struct fw_ri_isgl isgl_src[0];</div><div class='add'>+		DECLARE_FLEX_ARRAY(struct fw_ri_immd, immd_src);</div><div class='add'>+		DECLARE_FLEX_ARRAY(struct fw_ri_isgl, isgl_src);</div><div class='ctx'> 	} u;</div><div class='del'>-#endif</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct fw_ri_send_wr {</div><div class='hunk'>@@ -581,12 +571,10 @@ struct fw_ri_send_wr {</div><div class='ctx'> 	__be32 plen;</div><div class='ctx'> 	__be32 r3;</div><div class='ctx'> 	__be64 r4;</div><div class='del'>-#ifndef C99_NOT_SUPPORTED</div><div class='ctx'> 	union {</div><div class='del'>-		struct fw_ri_immd immd_src[0];</div><div class='del'>-		struct fw_ri_isgl isgl_src[0];</div><div class='add'>+		DECLARE_FLEX_ARRAY(struct fw_ri_immd, immd_src);</div><div class='add'>+		DECLARE_FLEX_ARRAY(struct fw_ri_isgl, isgl_src);</div><div class='ctx'> 	} u;</div><div class='del'>-#endif</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> #define FW_RI_SEND_WR_SENDOP_S		0</div><div class='hunk'>@@ -618,12 +606,10 @@ struct fw_ri_rdma_write_cmpl_wr {</div><div class='ctx'> 		struct fw_ri_isgl isgl_src;</div><div class='ctx'> 	} u_cmpl;</div><div class='ctx'> 	__be64 r3;</div><div class='del'>-#ifndef C99_NOT_SUPPORTED</div><div class='ctx'> 	union fw_ri_write {</div><div class='del'>-		struct fw_ri_immd immd_src[0];</div><div class='del'>-		struct fw_ri_isgl isgl_src[0];</div><div class='add'>+		DECLARE_FLEX_ARRAY(struct fw_ri_immd, immd_src);</div><div class='add'>+		DECLARE_FLEX_ARRAY(struct fw_ri_isgl, isgl_src);</div><div class='ctx'> 	} u;</div><div class='del'>-#endif</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct fw_ri_rdma_read_wr {</div><div class='head'>diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h<br/>index 9e3cc3239c13..96f9c3bc98b2 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/efa/efa.h</a></div><div class='hunk'>@@ -1,16 +1,14 @@</div><div class='ctx'> /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */</div><div class='ctx'> /*</div><div class='del'>- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='add'>+ * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #ifndef _EFA_H_</div><div class='ctx'> #define _EFA_H_</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/bitops.h&gt;</div><div class='del'>-#include &lt;linux/idr.h&gt;</div><div class='ctx'> #include &lt;linux/interrupt.h&gt;</div><div class='ctx'> #include &lt;linux/pci.h&gt;</div><div class='del'>-#include &lt;linux/sched.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include &lt;rdma/efa-abi.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_verbs.h&gt;</div><div class='hunk'>@@ -22,31 +20,27 @@</div><div class='ctx'> </div><div class='ctx'> #define EFA_IRQNAME_SIZE        40</div><div class='ctx'> </div><div class='del'>-/* 1 for AENQ + ADMIN */</div><div class='del'>-#define EFA_NUM_MSIX_VEC                  1</div><div class='ctx'> #define EFA_MGMNT_MSIX_VEC_IDX            0</div><div class='add'>+#define EFA_COMP_EQS_VEC_BASE             1</div><div class='ctx'> </div><div class='ctx'> struct efa_irq {</div><div class='ctx'> 	irq_handler_t handler;</div><div class='ctx'> 	void *data;</div><div class='del'>-	int cpu;</div><div class='add'>+	u32 irqn;</div><div class='ctx'> 	u32 vector;</div><div class='ctx'> 	cpumask_t affinity_hint_mask;</div><div class='ctx'> 	char name[EFA_IRQNAME_SIZE];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct efa_sw_stats {</div><div class='add'>+/* Don't use anything other than atomic64 */</div><div class='add'>+struct efa_stats {</div><div class='ctx'> 	atomic64_t alloc_pd_err;</div><div class='ctx'> 	atomic64_t create_qp_err;</div><div class='ctx'> 	atomic64_t create_cq_err;</div><div class='ctx'> 	atomic64_t reg_mr_err;</div><div class='ctx'> 	atomic64_t alloc_ucontext_err;</div><div class='ctx'> 	atomic64_t create_ah_err;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Don't use anything other than atomic64 */</div><div class='del'>-struct efa_stats {</div><div class='del'>-	struct efa_sw_stats sw_stats;</div><div class='add'>+	atomic64_t mmap_err;</div><div class='ctx'> 	atomic64_t keep_alive_rcvd;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -62,19 +56,23 @@ struct efa_dev {</div><div class='ctx'> 	u64 mem_bar_len;</div><div class='ctx'> 	u64 db_bar_addr;</div><div class='ctx'> 	u64 db_bar_len;</div><div class='del'>-	u8 addr[EFA_GID_SIZE];</div><div class='del'>-	u32 mtu;</div><div class='ctx'> </div><div class='del'>-	int admin_msix_vector_idx;</div><div class='add'>+	u32 num_irq_vectors;</div><div class='add'>+	u32 admin_msix_vector_idx;</div><div class='ctx'> 	struct efa_irq admin_irq;</div><div class='ctx'> </div><div class='ctx'> 	struct efa_stats stats;</div><div class='add'>+</div><div class='add'>+	/* Array of completion EQs */</div><div class='add'>+	struct efa_eq *eqs;</div><div class='add'>+	u32 neqs;</div><div class='add'>+</div><div class='add'>+	/* Only stores CQs with interrupts enabled */</div><div class='add'>+	struct xarray cqs_xa;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_ucontext {</div><div class='ctx'> 	struct ib_ucontext ibucontext;</div><div class='del'>-	struct xarray mmap_xa;</div><div class='del'>-	u32 mmap_xa_page;</div><div class='ctx'> 	u16 uarn;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -83,9 +81,19 @@ struct efa_pd {</div><div class='ctx'> 	u16 pdn;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct efa_mr_interconnect_info {</div><div class='add'>+	u16 recv_ic_id;</div><div class='add'>+	u16 rdma_read_ic_id;</div><div class='add'>+	u16 rdma_recv_ic_id;</div><div class='add'>+	u8 recv_ic_id_valid : 1;</div><div class='add'>+	u8 rdma_read_ic_id_valid : 1;</div><div class='add'>+	u8 rdma_recv_ic_id_valid : 1;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct efa_mr {</div><div class='ctx'> 	struct ib_mr ibmr;</div><div class='ctx'> 	struct ib_umem *umem;</div><div class='add'>+	struct efa_mr_interconnect_info ic_info;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_cq {</div><div class='hunk'>@@ -93,8 +101,13 @@ struct efa_cq {</div><div class='ctx'> 	struct efa_ucontext *ucontext;</div><div class='ctx'> 	dma_addr_t dma_addr;</div><div class='ctx'> 	void *cpu_addr;</div><div class='add'>+	struct rdma_user_mmap_entry *mmap_entry;</div><div class='add'>+	struct rdma_user_mmap_entry *db_mmap_entry;</div><div class='ctx'> 	size_t size;</div><div class='ctx'> 	u16 cq_idx;</div><div class='add'>+	/* NULL when no interrupts requested */</div><div class='add'>+	struct efa_eq *eq;</div><div class='add'>+	struct ib_umem *umem;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_qp {</div><div class='hunk'>@@ -103,6 +116,13 @@ struct efa_qp {</div><div class='ctx'> 	void *rq_cpu_addr;</div><div class='ctx'> 	size_t rq_size;</div><div class='ctx'> 	enum ib_qp_state state;</div><div class='add'>+</div><div class='add'>+	/* Used for saving mmap_xa entries */</div><div class='add'>+	struct rdma_user_mmap_entry *sq_db_mmap_entry;</div><div class='add'>+	struct rdma_user_mmap_entry *llq_desc_mmap_entry;</div><div class='add'>+	struct rdma_user_mmap_entry *rq_db_mmap_entry;</div><div class='add'>+	struct rdma_user_mmap_entry *rq_mmap_entry;</div><div class='add'>+</div><div class='ctx'> 	u32 qp_handle;</div><div class='ctx'> 	u32 max_send_wr;</div><div class='ctx'> 	u32 max_recv_wr;</div><div class='hunk'>@@ -118,46 +138,61 @@ struct efa_ah {</div><div class='ctx'> 	u8 id[EFA_GID_SIZE];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct efa_eq {</div><div class='add'>+	struct efa_com_eq eeq;</div><div class='add'>+	struct efa_irq irq;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> int efa_query_device(struct ib_device *ibdev,</div><div class='ctx'> 		     struct ib_device_attr *props,</div><div class='ctx'> 		     struct ib_udata *udata);</div><div class='del'>-int efa_query_port(struct ib_device *ibdev, u8 port,</div><div class='add'>+int efa_query_port(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 		   struct ib_port_attr *props);</div><div class='ctx'> int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 		 int qp_attr_mask,</div><div class='ctx'> 		 struct ib_qp_init_attr *qp_init_attr);</div><div class='del'>-int efa_query_gid(struct ib_device *ibdev, u8 port, int index,</div><div class='add'>+int efa_query_gid(struct ib_device *ibdev, u32 port, int index,</div><div class='ctx'> 		  union ib_gid *gid);</div><div class='del'>-int efa_query_pkey(struct ib_device *ibdev, u8 port, u16 index,</div><div class='add'>+int efa_query_pkey(struct ib_device *ibdev, u32 port, u16 index,</div><div class='ctx'> 		   u16 *pkey);</div><div class='ctx'> int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);</div><div class='del'>-void efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);</div><div class='add'>+int efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);</div><div class='ctx'> int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);</div><div class='del'>-struct ib_qp *efa_create_qp(struct ib_pd *ibpd,</div><div class='del'>-			    struct ib_qp_init_attr *init_attr,</div><div class='del'>-			    struct ib_udata *udata);</div><div class='add'>+int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,</div><div class='add'>+		  struct ib_udata *udata);</div><div class='ctx'> int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);</div><div class='del'>-struct ib_cq *efa_create_cq(struct ib_device *ibdev,</div><div class='del'>-			    const struct ib_cq_init_attr *attr,</div><div class='del'>-			    struct ib_udata *udata);</div><div class='add'>+int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		  struct uverbs_attr_bundle *attrs);</div><div class='add'>+int efa_create_cq_umem(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		       struct ib_umem *umem, struct uverbs_attr_bundle *attrs);</div><div class='ctx'> struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,</div><div class='ctx'> 			 u64 virt_addr, int access_flags,</div><div class='add'>+			 struct ib_dmah *dmah,</div><div class='ctx'> 			 struct ib_udata *udata);</div><div class='add'>+struct ib_mr *efa_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start,</div><div class='add'>+				     u64 length, u64 virt_addr,</div><div class='add'>+				     int fd, int access_flags,</div><div class='add'>+				     struct ib_dmah *dmah,</div><div class='add'>+				     struct uverbs_attr_bundle *attrs);</div><div class='ctx'> int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);</div><div class='del'>-int efa_get_port_immutable(struct ib_device *ibdev, u8 port_num,</div><div class='add'>+int efa_get_port_immutable(struct ib_device *ibdev, u32 port_num,</div><div class='ctx'> 			   struct ib_port_immutable *immutable);</div><div class='ctx'> int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata);</div><div class='ctx'> void efa_dealloc_ucontext(struct ib_ucontext *ibucontext);</div><div class='ctx'> int efa_mmap(struct ib_ucontext *ibucontext,</div><div class='ctx'> 	     struct vm_area_struct *vma);</div><div class='add'>+void efa_mmap_free(struct rdma_user_mmap_entry *rdma_entry);</div><div class='ctx'> int efa_create_ah(struct ib_ah *ibah,</div><div class='del'>-		  struct rdma_ah_attr *ah_attr,</div><div class='del'>-		  u32 flags,</div><div class='add'>+		  struct rdma_ah_init_attr *init_attr,</div><div class='ctx'> 		  struct ib_udata *udata);</div><div class='del'>-void efa_destroy_ah(struct ib_ah *ibah, u32 flags);</div><div class='add'>+int efa_destroy_ah(struct ib_ah *ibah, u32 flags);</div><div class='ctx'> int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 		  int qp_attr_mask, struct ib_udata *udata);</div><div class='ctx'> enum rdma_link_layer efa_port_link_layer(struct ib_device *ibdev,</div><div class='del'>-					 u8 port_num);</div><div class='add'>+					 u32 port_num);</div><div class='add'>+struct rdma_hw_stats *efa_alloc_hw_port_stats(struct ib_device *ibdev, u32 port_num);</div><div class='add'>+struct rdma_hw_stats *efa_alloc_hw_device_stats(struct ib_device *ibdev);</div><div class='add'>+int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,</div><div class='add'>+		     u32 port_num, int index);</div><div class='ctx'> </div><div class='ctx'> #endif /* _EFA_H_ */</div><div class='head'>diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h<br/>index 2be0469d545f..57178dad5eb7 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_admin_cmds_defs.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/efa/efa_admin_cmds_defs.h</a></div><div class='hunk'>@@ -1,6 +1,6 @@</div><div class='ctx'> /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */</div><div class='ctx'> /*</div><div class='del'>- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='add'>+ * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #ifndef _EFA_ADMIN_CMDS_H_</div><div class='hunk'>@@ -28,7 +28,10 @@ enum efa_admin_aq_opcode {</div><div class='ctx'> 	EFA_ADMIN_DEALLOC_PD                        = 15,</div><div class='ctx'> 	EFA_ADMIN_ALLOC_UAR                         = 16,</div><div class='ctx'> 	EFA_ADMIN_DEALLOC_UAR                       = 17,</div><div class='del'>-	EFA_ADMIN_MAX_OPCODE                        = 17,</div><div class='add'>+	EFA_ADMIN_CREATE_EQ                         = 18,</div><div class='add'>+	EFA_ADMIN_DESTROY_EQ                        = 19,</div><div class='add'>+	EFA_ADMIN_ALLOC_MR                          = 20,</div><div class='add'>+	EFA_ADMIN_MAX_OPCODE                        = 20,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum efa_admin_aq_feature_id {</div><div class='hunk'>@@ -37,7 +40,8 @@ enum efa_admin_aq_feature_id {</div><div class='ctx'> 	EFA_ADMIN_NETWORK_ATTR                      = 3,</div><div class='ctx'> 	EFA_ADMIN_QUEUE_ATTR                        = 4,</div><div class='ctx'> 	EFA_ADMIN_HW_HINTS                          = 5,</div><div class='del'>-	EFA_ADMIN_FEATURES_OPCODE_NUM               = 8,</div><div class='add'>+	EFA_ADMIN_HOST_INFO                         = 6,</div><div class='add'>+	EFA_ADMIN_EVENT_QUEUE_ATTR                  = 7,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /* QP transport type */</div><div class='hunk'>@@ -61,6 +65,10 @@ enum efa_admin_qp_state {</div><div class='ctx'> </div><div class='ctx'> enum efa_admin_get_stats_type {</div><div class='ctx'> 	EFA_ADMIN_GET_STATS_TYPE_BASIC              = 0,</div><div class='add'>+	EFA_ADMIN_GET_STATS_TYPE_MESSAGES           = 1,</div><div class='add'>+	EFA_ADMIN_GET_STATS_TYPE_RDMA_READ          = 2,</div><div class='add'>+	EFA_ADMIN_GET_STATS_TYPE_RDMA_WRITE         = 3,</div><div class='add'>+	EFA_ADMIN_GET_STATS_TYPE_NETWORK            = 4,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum efa_admin_get_stats_scope {</div><div class='hunk'>@@ -68,14 +76,6 @@ enum efa_admin_get_stats_scope {</div><div class='ctx'> 	EFA_ADMIN_GET_STATS_SCOPE_QUEUE             = 1,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-enum efa_admin_modify_qp_mask_bits {</div><div class='del'>-	EFA_ADMIN_QP_STATE_BIT                      = 0,</div><div class='del'>-	EFA_ADMIN_CUR_QP_STATE_BIT                  = 1,</div><div class='del'>-	EFA_ADMIN_QKEY_BIT                          = 2,</div><div class='del'>-	EFA_ADMIN_SQ_PSN_BIT                        = 3,</div><div class='del'>-	EFA_ADMIN_SQ_DRAINED_ASYNC_NOTIFY_BIT       = 4,</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> /*</div><div class='ctx'>  * QP allocation sizes, converted by fabric QueuePair (QP) create command</div><div class='ctx'>  * from QP capabilities.</div><div class='hunk'>@@ -112,7 +112,10 @@ struct efa_admin_create_qp_cmd {</div><div class='ctx'> 	 *    virtual (IOVA returned by MR registration)</div><div class='ctx'> 	 * 1 : rq_virt - If set, RQ ring base address is</div><div class='ctx'> 	 *    virtual (IOVA returned by MR registration)</div><div class='del'>-	 * 7:2 : reserved - MBZ</div><div class='add'>+	 * 2 : unsolicited_write_recv - If set, work requests</div><div class='add'>+	 *    will not be consumed for incoming RDMA write with</div><div class='add'>+	 *    immediate</div><div class='add'>+	 * 7:3 : reserved - MBZ</div><div class='ctx'> 	 */</div><div class='ctx'> 	u8 flags;</div><div class='ctx'> </div><div class='hunk'>@@ -149,8 +152,11 @@ struct efa_admin_create_qp_cmd {</div><div class='ctx'> 	/* UAR number */</div><div class='ctx'> 	u16 uar;</div><div class='ctx'> </div><div class='add'>+	/* Requested service level for the QP, 0 is the default SL */</div><div class='add'>+	u8 sl;</div><div class='add'>+</div><div class='ctx'> 	/* MBZ */</div><div class='del'>-	u16 reserved;</div><div class='add'>+	u8 reserved;</div><div class='ctx'> </div><div class='ctx'> 	/* MBZ */</div><div class='ctx'> 	u32 reserved2;</div><div class='hunk'>@@ -160,10 +166,16 @@ struct efa_admin_create_qp_resp {</div><div class='ctx'> 	/* Common Admin Queue completion descriptor */</div><div class='ctx'> 	struct efa_admin_acq_common_desc acq_common_desc;</div><div class='ctx'> </div><div class='del'>-	/* Opaque handle to be used for consequent operations on the QP */</div><div class='add'>+	/*</div><div class='add'>+	 * Opaque handle to be used for consequent admin operations on the</div><div class='add'>+	 * QP</div><div class='add'>+	 */</div><div class='ctx'> 	u32 qp_handle;</div><div class='ctx'> </div><div class='del'>-	/* QP number in the given EFA virtual device */</div><div class='add'>+	/*</div><div class='add'>+	 * QP number in the given EFA virtual device. Least-significant bits (as</div><div class='add'>+	 * needed according to max_qp) carry unique QP ID</div><div class='add'>+	 */</div><div class='ctx'> 	u16 qp_num;</div><div class='ctx'> </div><div class='ctx'> 	/* MBZ */</div><div class='hunk'>@@ -193,8 +205,14 @@ struct efa_admin_modify_qp_cmd {</div><div class='ctx'> 	struct efa_admin_aq_common_desc aq_common_desc;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='del'>-	 * Mask indicating which fields should be updated see enum</div><div class='del'>-	 * efa_admin_modify_qp_mask_bits</div><div class='add'>+	 * Mask indicating which fields should be updated</div><div class='add'>+	 * 0 : qp_state</div><div class='add'>+	 * 1 : cur_qp_state</div><div class='add'>+	 * 2 : qkey</div><div class='add'>+	 * 3 : sq_psn</div><div class='add'>+	 * 4 : sq_drained_async_notify</div><div class='add'>+	 * 5 : rnr_retry</div><div class='add'>+	 * 31:6 : reserved</div><div class='ctx'> 	 */</div><div class='ctx'> 	u32 modify_mask;</div><div class='ctx'> </div><div class='hunk'>@@ -216,8 +234,8 @@ struct efa_admin_modify_qp_cmd {</div><div class='ctx'> 	/* Enable async notification when SQ is drained */</div><div class='ctx'> 	u8 sq_drained_async_notify;</div><div class='ctx'> </div><div class='del'>-	/* MBZ */</div><div class='del'>-	u8 reserved1;</div><div class='add'>+	/* Number of RNR retries (valid only for SRD QPs) */</div><div class='add'>+	u8 rnr_retry;</div><div class='ctx'> </div><div class='ctx'> 	/* MBZ */</div><div class='ctx'> 	u16 reserved2;</div><div class='hunk'>@@ -252,8 +270,8 @@ struct efa_admin_query_qp_resp {</div><div class='ctx'> 	/* Indicates that draining is in progress */</div><div class='ctx'> 	u8 sq_draining;</div><div class='ctx'> </div><div class='del'>-	/* MBZ */</div><div class='del'>-	u8 reserved1;</div><div class='add'>+	/* Number of RNR retries (valid only for SRD QPs) */</div><div class='add'>+	u8 rnr_retry;</div><div class='ctx'> </div><div class='ctx'> 	/* MBZ */</div><div class='ctx'> 	u16 reserved2;</div><div class='hunk'>@@ -286,6 +304,7 @@ struct efa_admin_create_ah_cmd {</div><div class='ctx'> 	/* PD number */</div><div class='ctx'> 	u16 pd;</div><div class='ctx'> </div><div class='add'>+	/* MBZ */</div><div class='ctx'> 	u16 reserved;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -296,6 +315,7 @@ struct efa_admin_create_ah_resp {</div><div class='ctx'> 	/* Target interface address handle (opaque) */</div><div class='ctx'> 	u16 ah;</div><div class='ctx'> </div><div class='add'>+	/* MBZ */</div><div class='ctx'> 	u16 reserved;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -362,12 +382,19 @@ struct efa_admin_reg_mr_cmd {</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * permissions</div><div class='del'>-	 * 0 : local_write_enable - Write permissions: value</div><div class='del'>-	 *    of 1 needed for RQ buffers and for RDMA write</div><div class='del'>-	 * 7:1 : reserved1 - remote access flags, etc</div><div class='add'>+	 * 0 : local_write_enable - Local write permissions:</div><div class='add'>+	 *    must be set for RQ buffers and buffers posted for</div><div class='add'>+	 *    RDMA Read requests</div><div class='add'>+	 * 1 : remote_write_enable - Remote write</div><div class='add'>+	 *    permissions: must be set to enable RDMA write to</div><div class='add'>+	 *    the region</div><div class='add'>+	 * 2 : remote_read_enable - Remote read permissions:</div><div class='add'>+	 *    must be set to enable RDMA read from the region</div><div class='add'>+	 * 7:3 : reserved2 - MBZ</div><div class='ctx'> 	 */</div><div class='ctx'> 	u8 permissions;</div><div class='ctx'> </div><div class='add'>+	/* MBZ */</div><div class='ctx'> 	u16 reserved16_w5;</div><div class='ctx'> </div><div class='ctx'> 	/* number of pages in PBL (redundant, could be calculated) */</div><div class='hunk'>@@ -396,6 +423,32 @@ struct efa_admin_reg_mr_resp {</div><div class='ctx'> 	 * memory region</div><div class='ctx'> 	 */</div><div class='ctx'> 	u32 r_key;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Mask indicating which fields have valid values</div><div class='add'>+	 * 0 : recv_ic_id</div><div class='add'>+	 * 1 : rdma_read_ic_id</div><div class='add'>+	 * 2 : rdma_recv_ic_id</div><div class='add'>+	 */</div><div class='add'>+	u8 validity;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Physical interconnect used by the device to reach the MR for receive</div><div class='add'>+	 * operation</div><div class='add'>+	 */</div><div class='add'>+	u8 recv_ic_id;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Physical interconnect used by the device to reach the MR for RDMA</div><div class='add'>+	 * read operation</div><div class='add'>+	 */</div><div class='add'>+	u8 rdma_read_ic_id;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Physical interconnect used by the device to reach the MR for RDMA</div><div class='add'>+	 * write receive</div><div class='add'>+	 */</div><div class='add'>+	u8 rdma_recv_ic_id;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_admin_dereg_mr_cmd {</div><div class='hunk'>@@ -411,32 +464,73 @@ struct efa_admin_dereg_mr_resp {</div><div class='ctx'> 	struct efa_admin_acq_common_desc acq_common_desc;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+/*</div><div class='add'>+ * Allocation of MemoryRegion, required for QP working with Virtual</div><div class='add'>+ * Addresses in kernel verbs semantics, ready for fast registration use.</div><div class='add'>+ */</div><div class='add'>+struct efa_admin_alloc_mr_cmd {</div><div class='add'>+	/* Common Admin Queue descriptor */</div><div class='add'>+	struct efa_admin_aq_common_desc aq_common_desc;</div><div class='add'>+</div><div class='add'>+	/* Protection Domain */</div><div class='add'>+	u16 pd;</div><div class='add'>+</div><div class='add'>+	/* MBZ */</div><div class='add'>+	u16 reserved1;</div><div class='add'>+</div><div class='add'>+	/* Maximum number of pages this MR supports. */</div><div class='add'>+	u32 max_pages;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_admin_alloc_mr_resp {</div><div class='add'>+	/* Common Admin Queue completion descriptor */</div><div class='add'>+	struct efa_admin_acq_common_desc acq_common_desc;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * L_Key, to be used in conjunction with local buffer references in</div><div class='add'>+	 * SQ and RQ WQE, or with virtual RQ/CQ rings</div><div class='add'>+	 */</div><div class='add'>+	u32 l_key;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * R_Key, to be used in RDMA messages to refer to remotely accessed</div><div class='add'>+	 * memory region</div><div class='add'>+	 */</div><div class='add'>+	u32 r_key;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct efa_admin_create_cq_cmd {</div><div class='ctx'> 	struct efa_admin_aq_common_desc aq_common_desc;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='del'>-	 * 4:0 : reserved5</div><div class='add'>+	 * 4:0 : reserved5 - MBZ</div><div class='ctx'> 	 * 5 : interrupt_mode_enabled - if set, cq operates</div><div class='del'>-	 *    in interrupt mode (i.e. CQ events and MSI-X are</div><div class='del'>-	 *    generated), otherwise - polling</div><div class='add'>+	 *    in interrupt mode (i.e. CQ events and EQ elements</div><div class='add'>+	 *    are generated), otherwise - polling</div><div class='ctx'> 	 * 6 : virt - If set, ring base address is virtual</div><div class='ctx'> 	 *    (IOVA returned by MR registration)</div><div class='del'>-	 * 7 : reserved6</div><div class='add'>+	 * 7 : reserved6 - MBZ</div><div class='ctx'> 	 */</div><div class='ctx'> 	u8 cq_caps_1;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * 4:0 : cq_entry_size_words - size of CQ entry in</div><div class='ctx'> 	 *    32-bit words, valid values: 4, 8.</div><div class='del'>-	 * 7:5 : reserved7</div><div class='add'>+	 * 5 : set_src_addr - If set, source address will be</div><div class='add'>+	 *    filled on RX completions from unknown senders.</div><div class='add'>+	 *    Requires 8 words CQ entry size.</div><div class='add'>+	 * 7:6 : reserved7 - MBZ</div><div class='ctx'> 	 */</div><div class='ctx'> 	u8 cq_caps_2;</div><div class='ctx'> </div><div class='del'>-	/* completion queue depth in # of entries. must be power of 2 */</div><div class='del'>-	u16 cq_depth;</div><div class='add'>+	/* Sub completion queue depth in # of entries. must be power of 2 */</div><div class='add'>+	u16 sub_cq_depth;</div><div class='ctx'> </div><div class='del'>-	/* msix vector assigned to this cq */</div><div class='del'>-	u32 msix_vector_idx;</div><div class='add'>+	/* EQ number assigned to this cq */</div><div class='add'>+	u16 eqn;</div><div class='add'>+</div><div class='add'>+	/* MBZ */</div><div class='add'>+	u16 reserved;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * CQ ring base address, virtual or physical depending on 'virt'</div><div class='hunk'>@@ -452,7 +546,7 @@ struct efa_admin_create_cq_cmd {</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * number of sub cqs - must be equal to sub_cqs_per_cq of queue</div><div class='del'>-	 *    attributes.</div><div class='add'>+	 * attributes.</div><div class='ctx'> 	 */</div><div class='ctx'> 	u16 num_sub_cqs;</div><div class='ctx'> </div><div class='hunk'>@@ -465,8 +559,17 @@ struct efa_admin_create_cq_resp {</div><div class='ctx'> </div><div class='ctx'> 	u16 cq_idx;</div><div class='ctx'> </div><div class='del'>-	/* actual cq depth in number of entries */</div><div class='del'>-	u16 cq_actual_depth;</div><div class='add'>+	/* actual sub cq depth in number of entries */</div><div class='add'>+	u16 sub_cq_actual_depth;</div><div class='add'>+</div><div class='add'>+	/* CQ doorbell address, as offset to PCIe DB BAR */</div><div class='add'>+	u32 db_offset;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * 0 : db_valid - If set, doorbell offset is valid.</div><div class='add'>+	 *    Always set when interrupts are requested.</div><div class='add'>+	 */</div><div class='add'>+	u32 flags;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_admin_destroy_cq_cmd {</div><div class='hunk'>@@ -474,6 +577,7 @@ struct efa_admin_destroy_cq_cmd {</div><div class='ctx'> </div><div class='ctx'> 	u16 cq_idx;</div><div class='ctx'> </div><div class='add'>+	/* MBZ */</div><div class='ctx'> 	u16 reserved1;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -514,21 +618,71 @@ struct efa_admin_basic_stats {</div><div class='ctx'> 	u64 rx_pkts;</div><div class='ctx'> </div><div class='ctx'> 	u64 rx_drops;</div><div class='add'>+</div><div class='add'>+	u64 qkey_viol;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_admin_messages_stats {</div><div class='add'>+	u64 send_bytes;</div><div class='add'>+</div><div class='add'>+	u64 send_wrs;</div><div class='add'>+</div><div class='add'>+	u64 recv_bytes;</div><div class='add'>+</div><div class='add'>+	u64 recv_wrs;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_admin_rdma_read_stats {</div><div class='add'>+	u64 read_wrs;</div><div class='add'>+</div><div class='add'>+	u64 read_bytes;</div><div class='add'>+</div><div class='add'>+	u64 read_wr_err;</div><div class='add'>+</div><div class='add'>+	u64 read_resp_bytes;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_admin_rdma_write_stats {</div><div class='add'>+	u64 write_wrs;</div><div class='add'>+</div><div class='add'>+	u64 write_bytes;</div><div class='add'>+</div><div class='add'>+	u64 write_wr_err;</div><div class='add'>+</div><div class='add'>+	u64 write_recv_bytes;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_admin_network_stats {</div><div class='add'>+	u64 retrans_bytes;</div><div class='add'>+</div><div class='add'>+	u64 retrans_pkts;</div><div class='add'>+</div><div class='add'>+	u64 retrans_timeout_events;</div><div class='add'>+</div><div class='add'>+	u64 unresponsive_remote_events;</div><div class='add'>+</div><div class='add'>+	u64 impaired_remote_conn_events;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_admin_acq_get_stats_resp {</div><div class='ctx'> 	struct efa_admin_acq_common_desc acq_common_desc;</div><div class='ctx'> </div><div class='del'>-	struct efa_admin_basic_stats basic_stats;</div><div class='add'>+	union {</div><div class='add'>+		struct efa_admin_basic_stats basic_stats;</div><div class='add'>+</div><div class='add'>+		struct efa_admin_messages_stats messages_stats;</div><div class='add'>+</div><div class='add'>+		struct efa_admin_rdma_read_stats rdma_read_stats;</div><div class='add'>+</div><div class='add'>+		struct efa_admin_rdma_write_stats rdma_write_stats;</div><div class='add'>+</div><div class='add'>+		struct efa_admin_network_stats network_stats;</div><div class='add'>+	} u;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_admin_get_set_feature_common_desc {</div><div class='del'>-	/*</div><div class='del'>-	 * 1:0 : select - 0x1 - current value; 0x3 - default</div><div class='del'>-	 *    value</div><div class='del'>-	 * 7:3 : reserved3</div><div class='del'>-	 */</div><div class='del'>-	u8 flags;</div><div class='add'>+	/* MBZ */</div><div class='add'>+	u8 reserved0;</div><div class='ctx'> </div><div class='ctx'> 	/* as appears in efa_admin_aq_feature_id */</div><div class='ctx'> 	u8 feature_id;</div><div class='hunk'>@@ -553,38 +707,69 @@ struct efa_admin_feature_device_attr_desc {</div><div class='ctx'> 	/* Bar used for SQ and RQ doorbells */</div><div class='ctx'> 	u16 db_bar;</div><div class='ctx'> </div><div class='del'>-	/* Indicates how many bits are used physical address access */</div><div class='add'>+	/* Indicates how many bits are used on physical address access */</div><div class='ctx'> 	u8 phys_addr_width;</div><div class='ctx'> </div><div class='del'>-	/* Indicates how many bits are used virtual address access */</div><div class='add'>+	/* Indicates how many bits are used on virtual address access */</div><div class='ctx'> 	u8 virt_addr_width;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * 0 : rdma_read - If set, RDMA Read is supported on</div><div class='add'>+	 *    TX queues</div><div class='add'>+	 * 1 : rnr_retry - If set, RNR retry is supported on</div><div class='add'>+	 *    modify QP command</div><div class='add'>+	 * 2 : data_polling_128 - If set, 128 bytes data</div><div class='add'>+	 *    polling is supported</div><div class='add'>+	 * 3 : rdma_write - If set, RDMA Write is supported</div><div class='add'>+	 *    on TX queues</div><div class='add'>+	 * 4 : unsolicited_write_recv - If set, unsolicited</div><div class='add'>+	 *    write with imm. receive is supported</div><div class='add'>+	 * 31:5 : reserved - MBZ</div><div class='add'>+	 */</div><div class='add'>+	u32 device_caps;</div><div class='add'>+</div><div class='add'>+	/* Max RDMA transfer size in bytes */</div><div class='add'>+	u32 max_rdma_size;</div><div class='add'>+</div><div class='add'>+	/* Unique global ID for an EFA device */</div><div class='add'>+	u64 guid;</div><div class='add'>+</div><div class='add'>+	/* The device maximum link speed in Gbit/sec */</div><div class='add'>+	u16 max_link_speed_gbps;</div><div class='add'>+</div><div class='add'>+	/* MBZ */</div><div class='add'>+	u16 reserved0;</div><div class='add'>+</div><div class='add'>+	/* MBZ */</div><div class='add'>+	u32 reserved1;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_admin_feature_queue_attr_desc {</div><div class='ctx'> 	/* The maximum number of queue pairs supported */</div><div class='ctx'> 	u32 max_qp;</div><div class='ctx'> </div><div class='add'>+	/* Maximum number of WQEs per Send Queue */</div><div class='ctx'> 	u32 max_sq_depth;</div><div class='ctx'> </div><div class='del'>-	/* max send wr used in inline-buf */</div><div class='add'>+	/* Maximum size of data that can be sent inline in a Send WQE */</div><div class='ctx'> 	u32 inline_buf_size;</div><div class='ctx'> </div><div class='add'>+	/* Maximum number of buffer descriptors per Recv Queue */</div><div class='ctx'> 	u32 max_rq_depth;</div><div class='ctx'> </div><div class='ctx'> 	/* The maximum number of completion queues supported per VF */</div><div class='ctx'> 	u32 max_cq;</div><div class='ctx'> </div><div class='add'>+	/* Maximum number of CQEs per Completion Queue */</div><div class='ctx'> 	u32 max_cq_depth;</div><div class='ctx'> </div><div class='ctx'> 	/* Number of sub-CQs to be created for each CQ */</div><div class='ctx'> 	u16 sub_cqs_per_cq;</div><div class='ctx'> </div><div class='del'>-	u16 reserved;</div><div class='add'>+	/* Minimum number of WQEs per SQ */</div><div class='add'>+	u16 min_sq_depth;</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * Maximum number of SGEs (buffs) allowed for a single send work</div><div class='del'>-	 *    queue element (WQE)</div><div class='del'>-	 */</div><div class='add'>+	/* Maximum number of SGEs (buffers) allowed for a single send WQE */</div><div class='ctx'> 	u16 max_wr_send_sges;</div><div class='ctx'> </div><div class='ctx'> 	/* Maximum number of SGEs allowed for a single recv WQE */</div><div class='hunk'>@@ -604,6 +789,31 @@ struct efa_admin_feature_queue_attr_desc {</div><div class='ctx'> </div><div class='ctx'> 	/* The maximum size of LLQ in bytes */</div><div class='ctx'> 	u32 max_llq_size;</div><div class='add'>+</div><div class='add'>+	/* Maximum number of SGEs for a single RDMA read/write WQE */</div><div class='add'>+	u16 max_wr_rdma_sges;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Maximum number of bytes that can be written to SQ between two</div><div class='add'>+	 * consecutive doorbells (in units of 64B). Driver must ensure that only</div><div class='add'>+	 * complete WQEs are written to queue before issuing a doorbell.</div><div class='add'>+	 * Examples: max_tx_batch=16 and WQE size = 64B, means up to 16 WQEs can</div><div class='add'>+	 * be written to SQ between two consecutive doorbells. max_tx_batch=11</div><div class='add'>+	 * and WQE size = 128B, means up to 5 WQEs can be written to SQ between</div><div class='add'>+	 * two consecutive doorbells. Zero means unlimited.</div><div class='add'>+	 */</div><div class='add'>+	u16 max_tx_batch;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_admin_event_queue_attr_desc {</div><div class='add'>+	/* The maximum number of event queues supported */</div><div class='add'>+	u32 max_eq;</div><div class='add'>+</div><div class='add'>+	/* Maximum number of EQEs per Event Queue */</div><div class='add'>+	u32 max_eq_depth;</div><div class='add'>+</div><div class='add'>+	/* Supported events bitmask */</div><div class='add'>+	u32 event_bitmask;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_admin_feature_aenq_desc {</div><div class='hunk'>@@ -618,6 +828,7 @@ struct efa_admin_feature_network_attr_desc {</div><div class='ctx'> 	/* Raw address data in network byte order */</div><div class='ctx'> 	u8 addr[16];</div><div class='ctx'> </div><div class='add'>+	/* max packet payload size in bytes */</div><div class='ctx'> 	u32 mtu;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -663,6 +874,8 @@ struct efa_admin_get_feature_resp {</div><div class='ctx'> </div><div class='ctx'> 		struct efa_admin_feature_queue_attr_desc queue_attr;</div><div class='ctx'> </div><div class='add'>+		struct efa_admin_event_queue_attr_desc event_queue_attr;</div><div class='add'>+</div><div class='ctx'> 		struct efa_admin_hw_hints hw_hints;</div><div class='ctx'> 	} u;</div><div class='ctx'> };</div><div class='hunk'>@@ -746,6 +959,60 @@ struct efa_admin_dealloc_uar_resp {</div><div class='ctx'> 	struct efa_admin_acq_common_desc acq_common_desc;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct efa_admin_create_eq_cmd {</div><div class='add'>+	struct efa_admin_aq_common_desc aq_common_descriptor;</div><div class='add'>+</div><div class='add'>+	/* Size of the EQ in entries, must be power of 2 */</div><div class='add'>+	u16 depth;</div><div class='add'>+</div><div class='add'>+	/* MSI-X table entry index */</div><div class='add'>+	u8 msix_vec;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * 4:0 : entry_size_words - size of EQ entry in</div><div class='add'>+	 *    32-bit words</div><div class='add'>+	 * 7:5 : reserved - MBZ</div><div class='add'>+	 */</div><div class='add'>+	u8 caps;</div><div class='add'>+</div><div class='add'>+	/* EQ ring base address */</div><div class='add'>+	struct efa_common_mem_addr ba;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Enabled events on this EQ</div><div class='add'>+	 * 0 : completion_events - Enable completion events</div><div class='add'>+	 * 31:1 : reserved - MBZ</div><div class='add'>+	 */</div><div class='add'>+	u32 event_bitmask;</div><div class='add'>+</div><div class='add'>+	/* MBZ */</div><div class='add'>+	u32 reserved;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_admin_create_eq_resp {</div><div class='add'>+	struct efa_admin_acq_common_desc acq_common_desc;</div><div class='add'>+</div><div class='add'>+	/* EQ number */</div><div class='add'>+	u16 eqn;</div><div class='add'>+</div><div class='add'>+	/* MBZ */</div><div class='add'>+	u16 reserved;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_admin_destroy_eq_cmd {</div><div class='add'>+	struct efa_admin_aq_common_desc aq_common_descriptor;</div><div class='add'>+</div><div class='add'>+	/* EQ number */</div><div class='add'>+	u16 eqn;</div><div class='add'>+</div><div class='add'>+	/* MBZ */</div><div class='add'>+	u16 reserved;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_admin_destroy_eq_resp {</div><div class='add'>+	struct efa_admin_acq_common_desc acq_common_desc;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> /* asynchronous event notification groups */</div><div class='ctx'> enum efa_admin_aenq_group {</div><div class='ctx'> 	EFA_ADMIN_FATAL_ERROR                       = 1,</div><div class='hunk'>@@ -755,12 +1022,6 @@ enum efa_admin_aenq_group {</div><div class='ctx'> 	EFA_ADMIN_AENQ_GROUPS_NUM                   = 5,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-enum efa_admin_aenq_notification_syndrom {</div><div class='del'>-	EFA_ADMIN_SUSPEND                           = 0,</div><div class='del'>-	EFA_ADMIN_RESUME                            = 1,</div><div class='del'>-	EFA_ADMIN_UPDATE_HINTS                      = 2,</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> struct efa_admin_mmio_req_read_less_resp {</div><div class='ctx'> 	u16 req_id;</div><div class='ctx'> </div><div class='hunk'>@@ -770,25 +1031,110 @@ struct efa_admin_mmio_req_read_less_resp {</div><div class='ctx'> 	u32 reg_val;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+enum efa_admin_os_type {</div><div class='add'>+	EFA_ADMIN_OS_LINUX                          = 0,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_admin_host_info {</div><div class='add'>+	/* OS distribution string format */</div><div class='add'>+	u8 os_dist_str[128];</div><div class='add'>+</div><div class='add'>+	/* Defined in enum efa_admin_os_type */</div><div class='add'>+	u32 os_type;</div><div class='add'>+</div><div class='add'>+	/* Kernel version string format */</div><div class='add'>+	u8 kernel_ver_str[32];</div><div class='add'>+</div><div class='add'>+	/* Kernel version numeric format */</div><div class='add'>+	u32 kernel_ver;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * 7:0 : driver_module_type</div><div class='add'>+	 * 15:8 : driver_sub_minor</div><div class='add'>+	 * 23:16 : driver_minor</div><div class='add'>+	 * 31:24 : driver_major</div><div class='add'>+	 */</div><div class='add'>+	u32 driver_ver;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Device's Bus, Device and Function</div><div class='add'>+	 * 2:0 : function</div><div class='add'>+	 * 7:3 : device</div><div class='add'>+	 * 15:8 : bus</div><div class='add'>+	 */</div><div class='add'>+	u16 bdf;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Spec version</div><div class='add'>+	 * 7:0 : spec_minor</div><div class='add'>+	 * 15:8 : spec_major</div><div class='add'>+	 */</div><div class='add'>+	u16 spec_ver;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * 0 : intree - Intree driver</div><div class='add'>+	 * 1 : gdr - GPUDirect RDMA supported</div><div class='add'>+	 * 31:2 : reserved2</div><div class='add'>+	 */</div><div class='add'>+	u32 flags;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> /* create_qp_cmd */</div><div class='ctx'> #define EFA_ADMIN_CREATE_QP_CMD_SQ_VIRT_MASK                BIT(0)</div><div class='del'>-#define EFA_ADMIN_CREATE_QP_CMD_RQ_VIRT_SHIFT               1</div><div class='ctx'> #define EFA_ADMIN_CREATE_QP_CMD_RQ_VIRT_MASK                BIT(1)</div><div class='add'>+#define EFA_ADMIN_CREATE_QP_CMD_UNSOLICITED_WRITE_RECV_MASK BIT(2)</div><div class='add'>+</div><div class='add'>+/* modify_qp_cmd */</div><div class='add'>+#define EFA_ADMIN_MODIFY_QP_CMD_QP_STATE_MASK               BIT(0)</div><div class='add'>+#define EFA_ADMIN_MODIFY_QP_CMD_CUR_QP_STATE_MASK           BIT(1)</div><div class='add'>+#define EFA_ADMIN_MODIFY_QP_CMD_QKEY_MASK                   BIT(2)</div><div class='add'>+#define EFA_ADMIN_MODIFY_QP_CMD_SQ_PSN_MASK                 BIT(3)</div><div class='add'>+#define EFA_ADMIN_MODIFY_QP_CMD_SQ_DRAINED_ASYNC_NOTIFY_MASK BIT(4)</div><div class='add'>+#define EFA_ADMIN_MODIFY_QP_CMD_RNR_RETRY_MASK              BIT(5)</div><div class='ctx'> </div><div class='ctx'> /* reg_mr_cmd */</div><div class='ctx'> #define EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK      GENMASK(4, 0)</div><div class='del'>-#define EFA_ADMIN_REG_MR_CMD_MEM_ADDR_PHY_MODE_EN_SHIFT     7</div><div class='ctx'> #define EFA_ADMIN_REG_MR_CMD_MEM_ADDR_PHY_MODE_EN_MASK      BIT(7)</div><div class='ctx'> #define EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE_MASK        BIT(0)</div><div class='add'>+#define EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE_MASK       BIT(1)</div><div class='add'>+#define EFA_ADMIN_REG_MR_CMD_REMOTE_READ_ENABLE_MASK        BIT(2)</div><div class='add'>+</div><div class='add'>+/* reg_mr_resp */</div><div class='add'>+#define EFA_ADMIN_REG_MR_RESP_RECV_IC_ID_MASK               BIT(0)</div><div class='add'>+#define EFA_ADMIN_REG_MR_RESP_RDMA_READ_IC_ID_MASK          BIT(1)</div><div class='add'>+#define EFA_ADMIN_REG_MR_RESP_RDMA_RECV_IC_ID_MASK          BIT(2)</div><div class='ctx'> </div><div class='ctx'> /* create_cq_cmd */</div><div class='del'>-#define EFA_ADMIN_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_SHIFT 5</div><div class='ctx'> #define EFA_ADMIN_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK BIT(5)</div><div class='del'>-#define EFA_ADMIN_CREATE_CQ_CMD_VIRT_SHIFT                  6</div><div class='ctx'> #define EFA_ADMIN_CREATE_CQ_CMD_VIRT_MASK                   BIT(6)</div><div class='ctx'> #define EFA_ADMIN_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK    GENMASK(4, 0)</div><div class='del'>-</div><div class='del'>-/* get_set_feature_common_desc */</div><div class='del'>-#define EFA_ADMIN_GET_SET_FEATURE_COMMON_DESC_SELECT_MASK   GENMASK(1, 0)</div><div class='add'>+#define EFA_ADMIN_CREATE_CQ_CMD_SET_SRC_ADDR_MASK           BIT(5)</div><div class='add'>+</div><div class='add'>+/* create_cq_resp */</div><div class='add'>+#define EFA_ADMIN_CREATE_CQ_RESP_DB_VALID_MASK              BIT(0)</div><div class='add'>+</div><div class='add'>+/* feature_device_attr_desc */</div><div class='add'>+#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_READ_MASK   BIT(0)</div><div class='add'>+#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RNR_RETRY_MASK   BIT(1)</div><div class='add'>+#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_DATA_POLLING_128_MASK BIT(2)</div><div class='add'>+#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_WRITE_MASK  BIT(3)</div><div class='add'>+#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_UNSOLICITED_WRITE_RECV_MASK BIT(4)</div><div class='add'>+</div><div class='add'>+/* create_eq_cmd */</div><div class='add'>+#define EFA_ADMIN_CREATE_EQ_CMD_ENTRY_SIZE_WORDS_MASK       GENMASK(4, 0)</div><div class='add'>+#define EFA_ADMIN_CREATE_EQ_CMD_COMPLETION_EVENTS_MASK      BIT(0)</div><div class='add'>+</div><div class='add'>+/* host_info */</div><div class='add'>+#define EFA_ADMIN_HOST_INFO_DRIVER_MODULE_TYPE_MASK         GENMASK(7, 0)</div><div class='add'>+#define EFA_ADMIN_HOST_INFO_DRIVER_SUB_MINOR_MASK           GENMASK(15, 8)</div><div class='add'>+#define EFA_ADMIN_HOST_INFO_DRIVER_MINOR_MASK               GENMASK(23, 16)</div><div class='add'>+#define EFA_ADMIN_HOST_INFO_DRIVER_MAJOR_MASK               GENMASK(31, 24)</div><div class='add'>+#define EFA_ADMIN_HOST_INFO_FUNCTION_MASK                   GENMASK(2, 0)</div><div class='add'>+#define EFA_ADMIN_HOST_INFO_DEVICE_MASK                     GENMASK(7, 3)</div><div class='add'>+#define EFA_ADMIN_HOST_INFO_BUS_MASK                        GENMASK(15, 8)</div><div class='add'>+#define EFA_ADMIN_HOST_INFO_SPEC_MINOR_MASK                 GENMASK(7, 0)</div><div class='add'>+#define EFA_ADMIN_HOST_INFO_SPEC_MAJOR_MASK                 GENMASK(15, 8)</div><div class='add'>+#define EFA_ADMIN_HOST_INFO_INTREE_MASK                     BIT(0)</div><div class='add'>+#define EFA_ADMIN_HOST_INFO_GDR_MASK                        BIT(1)</div><div class='ctx'> </div><div class='ctx'> #endif /* _EFA_ADMIN_CMDS_H_ */</div><div class='head'>diff --git a/drivers/infiniband/hw/efa/efa_admin_defs.h b/drivers/infiniband/hw/efa/efa_admin_defs.h<br/>index c8e0c8b905be..35700c93e639 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_admin_defs.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_admin_defs.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_admin_defs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/efa/efa_admin_defs.h</a></div><div class='hunk'>@@ -1,6 +1,6 @@</div><div class='ctx'> /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */</div><div class='ctx'> /*</div><div class='del'>- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='add'>+ * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #ifndef _EFA_ADMIN_H_</div><div class='hunk'>@@ -82,7 +82,7 @@ struct efa_admin_acq_common_desc {</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * indicates to the driver which AQ entry has been consumed by the</div><div class='del'>-	 *    device and could be reused</div><div class='add'>+	 * device and could be reused</div><div class='ctx'> 	 */</div><div class='ctx'> 	u16 sq_head_indx;</div><div class='ctx'> };</div><div class='hunk'>@@ -96,7 +96,7 @@ struct efa_admin_acq_entry {</div><div class='ctx'> struct efa_admin_aenq_common_desc {</div><div class='ctx'> 	u16 group;</div><div class='ctx'> </div><div class='del'>-	u16 syndrom;</div><div class='add'>+	u16 syndrome;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * 0 : phase</div><div class='hunk'>@@ -118,12 +118,47 @@ struct efa_admin_aenq_entry {</div><div class='ctx'> 	u32 inline_data_w4[12];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+enum efa_admin_eqe_event_type {</div><div class='add'>+	EFA_ADMIN_EQE_EVENT_TYPE_COMPLETION         = 0,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* Completion event */</div><div class='add'>+struct efa_admin_comp_event {</div><div class='add'>+	/* CQ number */</div><div class='add'>+	u16 cqn;</div><div class='add'>+</div><div class='add'>+	/* MBZ */</div><div class='add'>+	u16 reserved;</div><div class='add'>+</div><div class='add'>+	/* MBZ */</div><div class='add'>+	u32 reserved2;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* Event Queue Element */</div><div class='add'>+struct efa_admin_eqe {</div><div class='add'>+	/*</div><div class='add'>+	 * 0 : phase</div><div class='add'>+	 * 8:1 : event_type - Event type</div><div class='add'>+	 * 31:9 : reserved - MBZ</div><div class='add'>+	 */</div><div class='add'>+	u32 common;</div><div class='add'>+</div><div class='add'>+	/* MBZ */</div><div class='add'>+	u32 reserved;</div><div class='add'>+</div><div class='add'>+	union {</div><div class='add'>+		/* Event data */</div><div class='add'>+		u32 event_data[2];</div><div class='add'>+</div><div class='add'>+		/* Completion Event */</div><div class='add'>+		struct efa_admin_comp_event comp_event;</div><div class='add'>+	} u;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> /* aq_common_desc */</div><div class='ctx'> #define EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK            GENMASK(11, 0)</div><div class='ctx'> #define EFA_ADMIN_AQ_COMMON_DESC_PHASE_MASK                 BIT(0)</div><div class='del'>-#define EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT            1</div><div class='ctx'> #define EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK             BIT(1)</div><div class='del'>-#define EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT   2</div><div class='ctx'> #define EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK    BIT(2)</div><div class='ctx'> </div><div class='ctx'> /* acq_common_desc */</div><div class='hunk'>@@ -133,4 +168,8 @@ struct efa_admin_aenq_entry {</div><div class='ctx'> /* aenq_common_desc */</div><div class='ctx'> #define EFA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK               BIT(0)</div><div class='ctx'> </div><div class='add'>+/* eqe */</div><div class='add'>+#define EFA_ADMIN_EQE_PHASE_MASK                            BIT(0)</div><div class='add'>+#define EFA_ADMIN_EQE_EVENT_TYPE_MASK                       GENMASK(8, 1)</div><div class='add'>+</div><div class='ctx'> #endif /* _EFA_ADMIN_H_ */</div><div class='head'>diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c<br/>index a5c788741a04..0e979ca10d24 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_com.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_com.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_com.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/efa/efa_com.c</a></div><div class='hunk'>@@ -1,6 +1,6 @@</div><div class='ctx'> // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause</div><div class='ctx'> /*</div><div class='del'>- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='add'>+ * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include "efa_com.h"</div><div class='hunk'>@@ -16,31 +16,13 @@</div><div class='ctx'> #define EFA_ASYNC_QUEUE_DEPTH 16</div><div class='ctx'> #define EFA_ADMIN_QUEUE_DEPTH 32</div><div class='ctx'> </div><div class='del'>-#define MIN_EFA_VER\</div><div class='del'>-	((EFA_ADMIN_API_VERSION_MAJOR &lt;&lt; EFA_REGS_VERSION_MAJOR_VERSION_SHIFT) | \</div><div class='del'>-	 (EFA_ADMIN_API_VERSION_MINOR &amp; EFA_REGS_VERSION_MINOR_VERSION_MASK))</div><div class='del'>-</div><div class='ctx'> #define EFA_CTRL_MAJOR          0</div><div class='ctx'> #define EFA_CTRL_MINOR          0</div><div class='ctx'> #define EFA_CTRL_SUB_MINOR      1</div><div class='ctx'> </div><div class='del'>-#define MIN_EFA_CTRL_VER \</div><div class='del'>-	(((EFA_CTRL_MAJOR) &lt;&lt; \</div><div class='del'>-	(EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT)) | \</div><div class='del'>-	((EFA_CTRL_MINOR) &lt;&lt; \</div><div class='del'>-	(EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT)) | \</div><div class='del'>-	(EFA_CTRL_SUB_MINOR))</div><div class='del'>-</div><div class='del'>-#define EFA_DMA_ADDR_TO_UINT32_LOW(x)   ((u32)((u64)(x)))</div><div class='del'>-#define EFA_DMA_ADDR_TO_UINT32_HIGH(x)  ((u32)(((u64)(x)) &gt;&gt; 32))</div><div class='del'>-</div><div class='del'>-#define EFA_REGS_ADMIN_INTR_MASK 1</div><div class='del'>-</div><div class='ctx'> enum efa_cmd_status {</div><div class='ctx'> 	EFA_CMD_SUBMITTED,</div><div class='ctx'> 	EFA_CMD_COMPLETED,</div><div class='del'>-	/* Abort - canceled by the driver */</div><div class='del'>-	EFA_CMD_ABORTED,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_comp_ctx {</div><div class='hunk'>@@ -48,8 +30,7 @@ struct efa_comp_ctx {</div><div class='ctx'> 	struct efa_admin_acq_entry *user_cqe;</div><div class='ctx'> 	u32 comp_size;</div><div class='ctx'> 	enum efa_cmd_status status;</div><div class='del'>-	/* status from the device */</div><div class='del'>-	u8 comp_status;</div><div class='add'>+	u16 cmd_id;</div><div class='ctx'> 	u8 cmd_opcode;</div><div class='ctx'> 	u8 occupied;</div><div class='ctx'> };</div><div class='hunk'>@@ -76,17 +57,25 @@ static const char *efa_com_cmd_str(u8 cmd)</div><div class='ctx'> 	EFA_CMD_STR_CASE(DEALLOC_PD);</div><div class='ctx'> 	EFA_CMD_STR_CASE(ALLOC_UAR);</div><div class='ctx'> 	EFA_CMD_STR_CASE(DEALLOC_UAR);</div><div class='add'>+	EFA_CMD_STR_CASE(CREATE_EQ);</div><div class='add'>+	EFA_CMD_STR_CASE(DESTROY_EQ);</div><div class='ctx'> 	default: return "unknown command opcode";</div><div class='ctx'> 	}</div><div class='ctx'> #undef EFA_CMD_STR_CASE</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+void efa_com_set_dma_addr(dma_addr_t addr, u32 *addr_high, u32 *addr_low)</div><div class='add'>+{</div><div class='add'>+	*addr_low = lower_32_bits(addr);</div><div class='add'>+	*addr_high = upper_32_bits(addr);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static u32 efa_com_reg_read32(struct efa_com_dev *edev, u16 offset)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_com_mmio_read *mmio_read = &amp;edev-&gt;mmio_read;</div><div class='ctx'> 	struct efa_admin_mmio_req_read_less_resp *read_resp;</div><div class='ctx'> 	unsigned long exp_time;</div><div class='del'>-	u32 mmio_read_reg;</div><div class='add'>+	u32 mmio_read_reg = 0;</div><div class='ctx'> 	u32 err;</div><div class='ctx'> </div><div class='ctx'> 	read_resp = mmio_read-&gt;read_resp;</div><div class='hunk'>@@ -96,10 +85,9 @@ static u32 efa_com_reg_read32(struct efa_com_dev *edev, u16 offset)</div><div class='ctx'> </div><div class='ctx'> 	/* trash DMA req_id to identify when hardware is done */</div><div class='ctx'> 	read_resp-&gt;req_id = mmio_read-&gt;seq_num + 0x9aL;</div><div class='del'>-	mmio_read_reg = (offset &lt;&lt; EFA_REGS_MMIO_REG_READ_REG_OFF_SHIFT) &amp;</div><div class='del'>-			EFA_REGS_MMIO_REG_READ_REG_OFF_MASK;</div><div class='del'>-	mmio_read_reg |= mmio_read-&gt;seq_num &amp;</div><div class='del'>-			 EFA_REGS_MMIO_REG_READ_REQ_ID_MASK;</div><div class='add'>+	EFA_SET(&amp;mmio_read_reg, EFA_REGS_MMIO_REG_READ_REG_OFF, offset);</div><div class='add'>+	EFA_SET(&amp;mmio_read_reg, EFA_REGS_MMIO_REG_READ_REQ_ID,</div><div class='add'>+		mmio_read-&gt;seq_num);</div><div class='ctx'> </div><div class='ctx'> 	writel(mmio_read_reg, edev-&gt;reg_bar + EFA_REGS_MMIO_REG_READ_OFF);</div><div class='ctx'> </div><div class='hunk'>@@ -111,17 +99,19 @@ static u32 efa_com_reg_read32(struct efa_com_dev *edev, u16 offset)</div><div class='ctx'> 	} while (time_is_after_jiffies(exp_time));</div><div class='ctx'> </div><div class='ctx'> 	if (read_resp-&gt;req_id != mmio_read-&gt;seq_num) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev,</div><div class='del'>-			  "Reading register timed out. expected: req id[%u] offset[%#x] actual: req id[%u] offset[%#x]\n",</div><div class='del'>-			  mmio_read-&gt;seq_num, offset, read_resp-&gt;req_id,</div><div class='del'>-			  read_resp-&gt;reg_off);</div><div class='add'>+		ibdev_err_ratelimited(</div><div class='add'>+			edev-&gt;efa_dev,</div><div class='add'>+			"Reading register timed out. expected: req id[%u] offset[%#x] actual: req id[%u] offset[%#x]\n",</div><div class='add'>+			mmio_read-&gt;seq_num, offset, read_resp-&gt;req_id,</div><div class='add'>+			read_resp-&gt;reg_off);</div><div class='ctx'> 		err = EFA_MMIO_READ_INVALID;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (read_resp-&gt;reg_off != offset) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev,</div><div class='del'>-			  "Reading register failed: wrong offset provided\n");</div><div class='add'>+		ibdev_err_ratelimited(</div><div class='add'>+			edev-&gt;efa_dev,</div><div class='add'>+			"Reading register failed: wrong offset provided\n");</div><div class='ctx'> 		err = EFA_MMIO_READ_INVALID;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -137,9 +127,9 @@ static int efa_com_admin_init_sq(struct efa_com_dev *edev)</div><div class='ctx'> 	struct efa_com_admin_queue *aq = &amp;edev-&gt;aq;</div><div class='ctx'> 	struct efa_com_admin_sq *sq = &amp;aq-&gt;sq;</div><div class='ctx'> 	u16 size = aq-&gt;depth * sizeof(*sq-&gt;entries);</div><div class='add'>+	u32 aq_caps = 0;</div><div class='ctx'> 	u32 addr_high;</div><div class='ctx'> 	u32 addr_low;</div><div class='del'>-	u32 aq_caps;</div><div class='ctx'> </div><div class='ctx'> 	sq-&gt;entries =</div><div class='ctx'> 		dma_alloc_coherent(aq-&gt;dmadev, size, &amp;sq-&gt;dma_addr, GFP_KERNEL);</div><div class='hunk'>@@ -154,16 +144,15 @@ static int efa_com_admin_init_sq(struct efa_com_dev *edev)</div><div class='ctx'> </div><div class='ctx'> 	sq-&gt;db_addr = (u32 __iomem *)(edev-&gt;reg_bar + EFA_REGS_AQ_PROD_DB_OFF);</div><div class='ctx'> </div><div class='del'>-	addr_high = EFA_DMA_ADDR_TO_UINT32_HIGH(sq-&gt;dma_addr);</div><div class='del'>-	addr_low = EFA_DMA_ADDR_TO_UINT32_LOW(sq-&gt;dma_addr);</div><div class='add'>+	addr_high = upper_32_bits(sq-&gt;dma_addr);</div><div class='add'>+	addr_low = lower_32_bits(sq-&gt;dma_addr);</div><div class='ctx'> </div><div class='ctx'> 	writel(addr_low, edev-&gt;reg_bar + EFA_REGS_AQ_BASE_LO_OFF);</div><div class='ctx'> 	writel(addr_high, edev-&gt;reg_bar + EFA_REGS_AQ_BASE_HI_OFF);</div><div class='ctx'> </div><div class='del'>-	aq_caps = aq-&gt;depth &amp; EFA_REGS_AQ_CAPS_AQ_DEPTH_MASK;</div><div class='del'>-	aq_caps |= (sizeof(struct efa_admin_aq_entry) &lt;&lt;</div><div class='del'>-			EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT) &amp;</div><div class='del'>-			EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK;</div><div class='add'>+	EFA_SET(&amp;aq_caps, EFA_REGS_AQ_CAPS_AQ_DEPTH, aq-&gt;depth);</div><div class='add'>+	EFA_SET(&amp;aq_caps, EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE,</div><div class='add'>+		sizeof(struct efa_admin_aq_entry));</div><div class='ctx'> </div><div class='ctx'> 	writel(aq_caps, edev-&gt;reg_bar + EFA_REGS_AQ_CAPS_OFF);</div><div class='ctx'> </div><div class='hunk'>@@ -175,9 +164,9 @@ static int efa_com_admin_init_cq(struct efa_com_dev *edev)</div><div class='ctx'> 	struct efa_com_admin_queue *aq = &amp;edev-&gt;aq;</div><div class='ctx'> 	struct efa_com_admin_cq *cq = &amp;aq-&gt;cq;</div><div class='ctx'> 	u16 size = aq-&gt;depth * sizeof(*cq-&gt;entries);</div><div class='add'>+	u32 acq_caps = 0;</div><div class='ctx'> 	u32 addr_high;</div><div class='ctx'> 	u32 addr_low;</div><div class='del'>-	u32 acq_caps;</div><div class='ctx'> </div><div class='ctx'> 	cq-&gt;entries =</div><div class='ctx'> 		dma_alloc_coherent(aq-&gt;dmadev, size, &amp;cq-&gt;dma_addr, GFP_KERNEL);</div><div class='hunk'>@@ -189,19 +178,17 @@ static int efa_com_admin_init_cq(struct efa_com_dev *edev)</div><div class='ctx'> 	cq-&gt;cc = 0;</div><div class='ctx'> 	cq-&gt;phase = 1;</div><div class='ctx'> </div><div class='del'>-	addr_high = EFA_DMA_ADDR_TO_UINT32_HIGH(cq-&gt;dma_addr);</div><div class='del'>-	addr_low = EFA_DMA_ADDR_TO_UINT32_LOW(cq-&gt;dma_addr);</div><div class='add'>+	addr_high = upper_32_bits(cq-&gt;dma_addr);</div><div class='add'>+	addr_low = lower_32_bits(cq-&gt;dma_addr);</div><div class='ctx'> </div><div class='ctx'> 	writel(addr_low, edev-&gt;reg_bar + EFA_REGS_ACQ_BASE_LO_OFF);</div><div class='ctx'> 	writel(addr_high, edev-&gt;reg_bar + EFA_REGS_ACQ_BASE_HI_OFF);</div><div class='ctx'> </div><div class='del'>-	acq_caps = aq-&gt;depth &amp; EFA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;</div><div class='del'>-	acq_caps |= (sizeof(struct efa_admin_acq_entry) &lt;&lt;</div><div class='del'>-			EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT) &amp;</div><div class='del'>-			EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK;</div><div class='del'>-	acq_caps |= (aq-&gt;msix_vector_idx &lt;&lt;</div><div class='del'>-			EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR_SHIFT) &amp;</div><div class='del'>-			EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR_MASK;</div><div class='add'>+	EFA_SET(&amp;acq_caps, EFA_REGS_ACQ_CAPS_ACQ_DEPTH, aq-&gt;depth);</div><div class='add'>+	EFA_SET(&amp;acq_caps, EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE,</div><div class='add'>+		sizeof(struct efa_admin_acq_entry));</div><div class='add'>+	EFA_SET(&amp;acq_caps, EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR,</div><div class='add'>+		aq-&gt;msix_vector_idx);</div><div class='ctx'> </div><div class='ctx'> 	writel(acq_caps, edev-&gt;reg_bar + EFA_REGS_ACQ_CAPS_OFF);</div><div class='ctx'> </div><div class='hunk'>@@ -212,7 +199,8 @@ static int efa_com_admin_init_aenq(struct efa_com_dev *edev,</div><div class='ctx'> 				   struct efa_aenq_handlers *aenq_handlers)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_com_aenq *aenq = &amp;edev-&gt;aenq;</div><div class='del'>-	u32 addr_low, addr_high, aenq_caps;</div><div class='add'>+	u32 addr_low, addr_high;</div><div class='add'>+	u32 aenq_caps = 0;</div><div class='ctx'> 	u16 size;</div><div class='ctx'> </div><div class='ctx'> 	if (!aenq_handlers) {</div><div class='hunk'>@@ -231,19 +219,17 @@ static int efa_com_admin_init_aenq(struct efa_com_dev *edev,</div><div class='ctx'> 	aenq-&gt;cc = 0;</div><div class='ctx'> 	aenq-&gt;phase = 1;</div><div class='ctx'> </div><div class='del'>-	addr_low = EFA_DMA_ADDR_TO_UINT32_LOW(aenq-&gt;dma_addr);</div><div class='del'>-	addr_high = EFA_DMA_ADDR_TO_UINT32_HIGH(aenq-&gt;dma_addr);</div><div class='add'>+	addr_low = lower_32_bits(aenq-&gt;dma_addr);</div><div class='add'>+	addr_high = upper_32_bits(aenq-&gt;dma_addr);</div><div class='ctx'> </div><div class='ctx'> 	writel(addr_low, edev-&gt;reg_bar + EFA_REGS_AENQ_BASE_LO_OFF);</div><div class='ctx'> 	writel(addr_high, edev-&gt;reg_bar + EFA_REGS_AENQ_BASE_HI_OFF);</div><div class='ctx'> </div><div class='del'>-	aenq_caps = aenq-&gt;depth &amp; EFA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK;</div><div class='del'>-	aenq_caps |= (sizeof(struct efa_admin_aenq_entry) &lt;&lt;</div><div class='del'>-		EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) &amp;</div><div class='del'>-		EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK;</div><div class='del'>-	aenq_caps |= (aenq-&gt;msix_vector_idx</div><div class='del'>-		      &lt;&lt; EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR_SHIFT) &amp;</div><div class='del'>-		     EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR_MASK;</div><div class='add'>+	EFA_SET(&amp;aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_DEPTH, aenq-&gt;depth);</div><div class='add'>+	EFA_SET(&amp;aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE,</div><div class='add'>+		sizeof(struct efa_admin_aenq_entry));</div><div class='add'>+	EFA_SET(&amp;aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR,</div><div class='add'>+		aenq-&gt;msix_vector_idx);</div><div class='ctx'> 	writel(aenq_caps, edev-&gt;reg_bar + EFA_REGS_AENQ_CAPS_OFF);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='hunk'>@@ -280,36 +266,35 @@ static void efa_com_dealloc_ctx_id(struct efa_com_admin_queue *aq,</div><div class='ctx'> static inline void efa_com_put_comp_ctx(struct efa_com_admin_queue *aq,</div><div class='ctx'> 					struct efa_comp_ctx *comp_ctx)</div><div class='ctx'> {</div><div class='del'>-	u16 comp_id = comp_ctx-&gt;user_cqe-&gt;acq_common_descriptor.command &amp;</div><div class='del'>-		      EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK;</div><div class='add'>+	u16 cmd_id = EFA_GET(&amp;comp_ctx-&gt;user_cqe-&gt;acq_common_descriptor.command,</div><div class='add'>+			     EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID);</div><div class='add'>+	u16 ctx_id = cmd_id &amp; (aq-&gt;depth - 1);</div><div class='ctx'> </div><div class='del'>-	ibdev_dbg(aq-&gt;efa_dev, "Putting completion command_id %d\n", comp_id);</div><div class='add'>+	ibdev_dbg(aq-&gt;efa_dev, "Put completion command_id %#x\n", cmd_id);</div><div class='ctx'> 	comp_ctx-&gt;occupied = 0;</div><div class='del'>-	efa_com_dealloc_ctx_id(aq, comp_id);</div><div class='add'>+	efa_com_dealloc_ctx_id(aq, ctx_id);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static struct efa_comp_ctx *efa_com_get_comp_ctx(struct efa_com_admin_queue *aq,</div><div class='del'>-						 u16 command_id, bool capture)</div><div class='add'>+						 u16 cmd_id, bool capture)</div><div class='ctx'> {</div><div class='del'>-	if (command_id &gt;= aq-&gt;depth) {</div><div class='del'>-		ibdev_err(aq-&gt;efa_dev,</div><div class='del'>-			  "command id is larger than the queue size. cmd_id: %u queue size %d\n",</div><div class='del'>-			  command_id, aq-&gt;depth);</div><div class='del'>-		return NULL;</div><div class='del'>-	}</div><div class='add'>+	u16 ctx_id = cmd_id &amp; (aq-&gt;depth - 1);</div><div class='ctx'> </div><div class='del'>-	if (aq-&gt;comp_ctx[command_id].occupied &amp;&amp; capture) {</div><div class='del'>-		ibdev_err(aq-&gt;efa_dev, "Completion context is occupied\n");</div><div class='add'>+	if (aq-&gt;comp_ctx[ctx_id].occupied &amp;&amp; capture) {</div><div class='add'>+		ibdev_err_ratelimited(</div><div class='add'>+			aq-&gt;efa_dev,</div><div class='add'>+			"Completion context for command_id %#x is occupied\n",</div><div class='add'>+			cmd_id);</div><div class='ctx'> 		return NULL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (capture) {</div><div class='del'>-		aq-&gt;comp_ctx[command_id].occupied = 1;</div><div class='del'>-		ibdev_dbg(aq-&gt;efa_dev, "Taking completion ctxt command_id %d\n",</div><div class='del'>-			  command_id);</div><div class='add'>+		aq-&gt;comp_ctx[ctx_id].occupied = 1;</div><div class='add'>+		ibdev_dbg(aq-&gt;efa_dev,</div><div class='add'>+			  "Take completion ctxt for command_id %#x\n", cmd_id);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return &amp;aq-&gt;comp_ctx[command_id];</div><div class='add'>+	return &amp;aq-&gt;comp_ctx[ctx_id];</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,</div><div class='hunk'>@@ -318,8 +303,10 @@ static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queu</div><div class='ctx'> 						       struct efa_admin_acq_entry *comp,</div><div class='ctx'> 						       size_t comp_size_in_bytes)</div><div class='ctx'> {</div><div class='add'>+	struct efa_admin_aq_entry *aqe;</div><div class='ctx'> 	struct efa_comp_ctx *comp_ctx;</div><div class='ctx'> 	u16 queue_size_mask;</div><div class='add'>+	u16 cmd_id;</div><div class='ctx'> 	u16 ctx_id;</div><div class='ctx'> 	u16 pi;</div><div class='ctx'> </div><div class='hunk'>@@ -328,13 +315,16 @@ static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queu</div><div class='ctx'> </div><div class='ctx'> 	ctx_id = efa_com_alloc_ctx_id(aq);</div><div class='ctx'> </div><div class='del'>-	cmd-&gt;aq_common_descriptor.flags |= aq-&gt;sq.phase &amp;</div><div class='del'>-		EFA_ADMIN_AQ_COMMON_DESC_PHASE_MASK;</div><div class='add'>+	/* cmd_id LSBs are the ctx_id and MSBs are entropy bits from pc */</div><div class='add'>+	cmd_id = ctx_id &amp; queue_size_mask;</div><div class='add'>+	cmd_id |= aq-&gt;sq.pc &amp; ~queue_size_mask;</div><div class='add'>+	cmd_id &amp;= EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK;</div><div class='ctx'> </div><div class='del'>-	cmd-&gt;aq_common_descriptor.command_id |= ctx_id &amp;</div><div class='del'>-		EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK;</div><div class='add'>+	cmd-&gt;aq_common_descriptor.command_id = cmd_id;</div><div class='add'>+	EFA_SET(&amp;cmd-&gt;aq_common_descriptor.flags,</div><div class='add'>+		EFA_ADMIN_AQ_COMMON_DESC_PHASE, aq-&gt;sq.phase);</div><div class='ctx'> </div><div class='del'>-	comp_ctx = efa_com_get_comp_ctx(aq, ctx_id, true);</div><div class='add'>+	comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, true);</div><div class='ctx'> 	if (!comp_ctx) {</div><div class='ctx'> 		efa_com_dealloc_ctx_id(aq, ctx_id);</div><div class='ctx'> 		return ERR_PTR(-EINVAL);</div><div class='hunk'>@@ -344,10 +334,13 @@ static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queu</div><div class='ctx'> 	comp_ctx-&gt;comp_size = comp_size_in_bytes;</div><div class='ctx'> 	comp_ctx-&gt;user_cqe = comp;</div><div class='ctx'> 	comp_ctx-&gt;cmd_opcode = cmd-&gt;aq_common_descriptor.opcode;</div><div class='add'>+	comp_ctx-&gt;cmd_id = cmd_id;</div><div class='ctx'> </div><div class='ctx'> 	reinit_completion(&amp;comp_ctx-&gt;wait_event);</div><div class='ctx'> </div><div class='del'>-	memcpy(&amp;aq-&gt;sq.entries[pi], cmd, cmd_size_in_bytes);</div><div class='add'>+	aqe = &amp;aq-&gt;sq.entries[pi];</div><div class='add'>+	memset(aqe, 0, sizeof(*aqe));</div><div class='add'>+	memcpy(aqe, cmd, cmd_size_in_bytes);</div><div class='ctx'> </div><div class='ctx'> 	aq-&gt;sq.pc++;</div><div class='ctx'> 	atomic64_inc(&amp;aq-&gt;stats.submitted_cmd);</div><div class='hunk'>@@ -401,7 +394,7 @@ static struct efa_comp_ctx *efa_com_submit_admin_cmd(struct efa_com_admin_queue</div><div class='ctx'> </div><div class='ctx'> 	spin_lock(&amp;aq-&gt;sq.lock);</div><div class='ctx'> 	if (!test_bit(EFA_AQ_STATE_RUNNING_BIT, &amp;aq-&gt;state)) {</div><div class='del'>-		ibdev_err(aq-&gt;efa_dev, "Admin queue is closed\n");</div><div class='add'>+		ibdev_err_ratelimited(aq-&gt;efa_dev, "Admin queue is closed\n");</div><div class='ctx'> 		spin_unlock(&amp;aq-&gt;sq.lock);</div><div class='ctx'> 		return ERR_PTR(-ENODEV);</div><div class='ctx'> 	}</div><div class='hunk'>@@ -415,38 +408,39 @@ static struct efa_comp_ctx *efa_com_submit_admin_cmd(struct efa_com_admin_queue</div><div class='ctx'> 	return comp_ctx;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq,</div><div class='del'>-						   struct efa_admin_acq_entry *cqe)</div><div class='add'>+static int efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq,</div><div class='add'>+						  struct efa_admin_acq_entry *cqe)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_comp_ctx *comp_ctx;</div><div class='ctx'> 	u16 cmd_id;</div><div class='ctx'> </div><div class='del'>-	cmd_id = cqe-&gt;acq_common_descriptor.command &amp;</div><div class='del'>-		 EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK;</div><div class='add'>+	cmd_id = EFA_GET(&amp;cqe-&gt;acq_common_descriptor.command,</div><div class='add'>+			 EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID);</div><div class='ctx'> </div><div class='ctx'> 	comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, false);</div><div class='del'>-	if (!comp_ctx) {</div><div class='add'>+	if (comp_ctx-&gt;status != EFA_CMD_SUBMITTED) {</div><div class='ctx'> 		ibdev_err(aq-&gt;efa_dev,</div><div class='del'>-			  "comp_ctx is NULL. Changing the admin queue running state\n");</div><div class='del'>-		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &amp;aq-&gt;state);</div><div class='del'>-		return;</div><div class='add'>+			  "Received completion with unexpected command id[%d], sq producer: %d, sq consumer: %d, cq consumer: %d\n",</div><div class='add'>+			  cmd_id, aq-&gt;sq.pc, aq-&gt;sq.cc, aq-&gt;cq.cc);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	comp_ctx-&gt;status = EFA_CMD_COMPLETED;</div><div class='del'>-	comp_ctx-&gt;comp_status = cqe-&gt;acq_common_descriptor.status;</div><div class='del'>-	if (comp_ctx-&gt;user_cqe)</div><div class='del'>-		memcpy(comp_ctx-&gt;user_cqe, cqe, comp_ctx-&gt;comp_size);</div><div class='add'>+	memcpy(comp_ctx-&gt;user_cqe, cqe, comp_ctx-&gt;comp_size);</div><div class='ctx'> </div><div class='ctx'> 	if (!test_bit(EFA_AQ_STATE_POLLING_BIT, &amp;aq-&gt;state))</div><div class='ctx'> 		complete(&amp;comp_ctx-&gt;wait_event);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void efa_com_handle_admin_completion(struct efa_com_admin_queue *aq)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_admin_acq_entry *cqe;</div><div class='ctx'> 	u16 queue_size_mask;</div><div class='del'>-	u16 comp_num = 0;</div><div class='add'>+	u16 comp_cmds = 0;</div><div class='ctx'> 	u8 phase;</div><div class='add'>+	int err;</div><div class='ctx'> 	u16 ci;</div><div class='ctx'> </div><div class='ctx'> 	queue_size_mask = aq-&gt;depth - 1;</div><div class='hunk'>@@ -464,10 +458,12 @@ static void efa_com_handle_admin_completion(struct efa_com_admin_queue *aq)</div><div class='ctx'> 		 * phase bit was validated</div><div class='ctx'> 		 */</div><div class='ctx'> 		dma_rmb();</div><div class='del'>-		efa_com_handle_single_admin_completion(aq, cqe);</div><div class='add'>+		err = efa_com_handle_single_admin_completion(aq, cqe);</div><div class='add'>+		if (!err)</div><div class='add'>+			comp_cmds++;</div><div class='ctx'> </div><div class='add'>+		aq-&gt;cq.cc++;</div><div class='ctx'> 		ci++;</div><div class='del'>-		comp_num++;</div><div class='ctx'> 		if (ci == aq-&gt;depth) {</div><div class='ctx'> 			ci = 0;</div><div class='ctx'> 			phase = !phase;</div><div class='hunk'>@@ -476,10 +472,9 @@ static void efa_com_handle_admin_completion(struct efa_com_admin_queue *aq)</div><div class='ctx'> 		cqe = &amp;aq-&gt;cq.entries[ci];</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	aq-&gt;cq.cc += comp_num;</div><div class='ctx'> 	aq-&gt;cq.phase = phase;</div><div class='del'>-	aq-&gt;sq.cc += comp_num;</div><div class='del'>-	atomic64_add(comp_num, &amp;aq-&gt;stats.completed_cmd);</div><div class='add'>+	aq-&gt;sq.cc += comp_cmds;</div><div class='add'>+	atomic64_add(comp_cmds, &amp;aq-&gt;stats.completed_cmd);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int efa_com_comp_status_to_errno(u8 comp_status)</div><div class='hunk'>@@ -519,8 +514,9 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c</div><div class='ctx'> 			break;</div><div class='ctx'> </div><div class='ctx'> 		if (time_is_before_jiffies(timeout)) {</div><div class='del'>-			ibdev_err(aq-&gt;efa_dev,</div><div class='del'>-				  "Wait for completion (polling) timeout\n");</div><div class='add'>+			ibdev_err_ratelimited(</div><div class='add'>+				aq-&gt;efa_dev,</div><div class='add'>+				"Wait for completion (polling) timeout\n");</div><div class='ctx'> 			/* EFA didn't have any completion */</div><div class='ctx'> 			atomic64_inc(&amp;aq-&gt;stats.no_completion);</div><div class='ctx'> </div><div class='hunk'>@@ -532,17 +528,7 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c</div><div class='ctx'> 		msleep(aq-&gt;poll_interval);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (comp_ctx-&gt;status == EFA_CMD_ABORTED) {</div><div class='del'>-		ibdev_err(aq-&gt;efa_dev, "Command was aborted\n");</div><div class='del'>-		atomic64_inc(&amp;aq-&gt;stats.aborted_cmd);</div><div class='del'>-		err = -ENODEV;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	WARN_ONCE(comp_ctx-&gt;status != EFA_CMD_COMPLETED,</div><div class='del'>-		  "Invalid completion status %d\n", comp_ctx-&gt;status);</div><div class='del'>-</div><div class='del'>-	err = efa_com_comp_status_to_errno(comp_ctx-&gt;comp_status);</div><div class='add'>+	err = efa_com_comp_status_to_errno(comp_ctx-&gt;user_cqe-&gt;acq_common_descriptor.status);</div><div class='ctx'> out:</div><div class='ctx'> 	efa_com_put_comp_ctx(aq, comp_ctx);</div><div class='ctx'> 	return err;</div><div class='hunk'>@@ -571,24 +557,28 @@ static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *com</div><div class='ctx'> 		atomic64_inc(&amp;aq-&gt;stats.no_completion);</div><div class='ctx'> </div><div class='ctx'> 		if (comp_ctx-&gt;status == EFA_CMD_COMPLETED)</div><div class='del'>-			ibdev_err(aq-&gt;efa_dev,</div><div class='del'>-				  "The device sent a completion but the driver didn't receive any MSI-X interrupt for admin cmd %s(%d) status %d (ctx: 0x%p, sq producer: %d, sq consumer: %d, cq consumer: %d)\n",</div><div class='del'>-				  efa_com_cmd_str(comp_ctx-&gt;cmd_opcode),</div><div class='del'>-				  comp_ctx-&gt;cmd_opcode, comp_ctx-&gt;status,</div><div class='del'>-				  comp_ctx, aq-&gt;sq.pc, aq-&gt;sq.cc, aq-&gt;cq.cc);</div><div class='add'>+			ibdev_err_ratelimited(</div><div class='add'>+				aq-&gt;efa_dev,</div><div class='add'>+				"The device sent a completion but the driver didn't receive any MSI-X interrupt for admin cmd %s(%d) status %d (id: %d, sq producer: %d, sq consumer: %d, cq consumer: %d)\n",</div><div class='add'>+				efa_com_cmd_str(comp_ctx-&gt;cmd_opcode),</div><div class='add'>+				comp_ctx-&gt;cmd_opcode, comp_ctx-&gt;status,</div><div class='add'>+				comp_ctx-&gt;cmd_id, aq-&gt;sq.pc, aq-&gt;sq.cc,</div><div class='add'>+				aq-&gt;cq.cc);</div><div class='ctx'> 		else</div><div class='del'>-			ibdev_err(aq-&gt;efa_dev,</div><div class='del'>-				  "The device didn't send any completion for admin cmd %s(%d) status %d (ctx 0x%p, sq producer: %d, sq consumer: %d, cq consumer: %d)\n",</div><div class='del'>-				  efa_com_cmd_str(comp_ctx-&gt;cmd_opcode),</div><div class='del'>-				  comp_ctx-&gt;cmd_opcode, comp_ctx-&gt;status,</div><div class='del'>-				  comp_ctx, aq-&gt;sq.pc, aq-&gt;sq.cc, aq-&gt;cq.cc);</div><div class='add'>+			ibdev_err_ratelimited(</div><div class='add'>+				aq-&gt;efa_dev,</div><div class='add'>+				"The device didn't send any completion for admin cmd %s(%d) status %d (id: %d, sq producer: %d, sq consumer: %d, cq consumer: %d)\n",</div><div class='add'>+				efa_com_cmd_str(comp_ctx-&gt;cmd_opcode),</div><div class='add'>+				comp_ctx-&gt;cmd_opcode, comp_ctx-&gt;status,</div><div class='add'>+				comp_ctx-&gt;cmd_id, aq-&gt;sq.pc, aq-&gt;sq.cc,</div><div class='add'>+				aq-&gt;cq.cc);</div><div class='ctx'> </div><div class='ctx'> 		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &amp;aq-&gt;state);</div><div class='ctx'> 		err = -ETIME;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	err = efa_com_comp_status_to_errno(comp_ctx-&gt;comp_status);</div><div class='add'>+	err = efa_com_comp_status_to_errno(comp_ctx-&gt;user_cqe-&gt;acq_common_descriptor.status);</div><div class='ctx'> out:</div><div class='ctx'> 	efa_com_put_comp_ctx(aq, comp_ctx);</div><div class='ctx'> 	return err;</div><div class='hunk'>@@ -643,22 +633,27 @@ int efa_com_cmd_exec(struct efa_com_admin_queue *aq,</div><div class='ctx'> 		  cmd-&gt;aq_common_descriptor.opcode);</div><div class='ctx'> 	comp_ctx = efa_com_submit_admin_cmd(aq, cmd, cmd_size, comp, comp_size);</div><div class='ctx'> 	if (IS_ERR(comp_ctx)) {</div><div class='del'>-		ibdev_err(aq-&gt;efa_dev,</div><div class='del'>-			  "Failed to submit command %s (opcode %u) err %ld\n",</div><div class='del'>-			  efa_com_cmd_str(cmd-&gt;aq_common_descriptor.opcode),</div><div class='del'>-			  cmd-&gt;aq_common_descriptor.opcode, PTR_ERR(comp_ctx));</div><div class='add'>+		ibdev_err_ratelimited(</div><div class='add'>+			aq-&gt;efa_dev,</div><div class='add'>+			"Failed to submit command %s (opcode %u) err %pe\n",</div><div class='add'>+			efa_com_cmd_str(cmd-&gt;aq_common_descriptor.opcode),</div><div class='add'>+			cmd-&gt;aq_common_descriptor.opcode, comp_ctx);</div><div class='ctx'> </div><div class='ctx'> 		up(&amp;aq-&gt;avail_cmds);</div><div class='add'>+		atomic64_inc(&amp;aq-&gt;stats.cmd_err);</div><div class='ctx'> 		return PTR_ERR(comp_ctx);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	err = efa_com_wait_and_process_admin_cq(comp_ctx, aq);</div><div class='del'>-	if (err)</div><div class='del'>-		ibdev_err(aq-&gt;efa_dev,</div><div class='del'>-			  "Failed to process command %s (opcode %u) comp_status %d err %d\n",</div><div class='del'>-			  efa_com_cmd_str(cmd-&gt;aq_common_descriptor.opcode),</div><div class='del'>-			  cmd-&gt;aq_common_descriptor.opcode,</div><div class='del'>-			  comp_ctx-&gt;comp_status, err);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err_ratelimited(</div><div class='add'>+			aq-&gt;efa_dev,</div><div class='add'>+			"Failed to process command %s (opcode %u) comp_status %d err %d\n",</div><div class='add'>+			efa_com_cmd_str(cmd-&gt;aq_common_descriptor.opcode),</div><div class='add'>+			cmd-&gt;aq_common_descriptor.opcode,</div><div class='add'>+			comp_ctx-&gt;user_cqe-&gt;acq_common_descriptor.status, err);</div><div class='add'>+		atomic64_inc(&amp;aq-&gt;stats.cmd_err);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	up(&amp;aq-&gt;avail_cmds);</div><div class='ctx'> </div><div class='hunk'>@@ -666,66 +661,6 @@ int efa_com_cmd_exec(struct efa_com_admin_queue *aq,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * efa_com_abort_admin_commands - Abort all the outstanding admin commands.</div><div class='del'>- * @edev: EFA communication layer struct</div><div class='del'>- *</div><div class='del'>- * This method aborts all the outstanding admin commands.</div><div class='del'>- * The caller should then call efa_com_wait_for_abort_completion to make sure</div><div class='del'>- * all the commands were completed.</div><div class='del'>- */</div><div class='del'>-static void efa_com_abort_admin_commands(struct efa_com_dev *edev)</div><div class='del'>-{</div><div class='del'>-	struct efa_com_admin_queue *aq = &amp;edev-&gt;aq;</div><div class='del'>-	struct efa_comp_ctx *comp_ctx;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	u16 i;</div><div class='del'>-</div><div class='del'>-	spin_lock(&amp;aq-&gt;sq.lock);</div><div class='del'>-	spin_lock_irqsave(&amp;aq-&gt;cq.lock, flags);</div><div class='del'>-	for (i = 0; i &lt; aq-&gt;depth; i++) {</div><div class='del'>-		comp_ctx = efa_com_get_comp_ctx(aq, i, false);</div><div class='del'>-		if (!comp_ctx)</div><div class='del'>-			break;</div><div class='del'>-</div><div class='del'>-		comp_ctx-&gt;status = EFA_CMD_ABORTED;</div><div class='del'>-</div><div class='del'>-		complete(&amp;comp_ctx-&gt;wait_event);</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;aq-&gt;cq.lock, flags);</div><div class='del'>-	spin_unlock(&amp;aq-&gt;sq.lock);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * efa_com_wait_for_abort_completion - Wait for admin commands abort.</div><div class='del'>- * @edev: EFA communication layer struct</div><div class='del'>- *</div><div class='del'>- * This method wait until all the outstanding admin commands will be completed.</div><div class='del'>- */</div><div class='del'>-static void efa_com_wait_for_abort_completion(struct efa_com_dev *edev)</div><div class='del'>-{</div><div class='del'>-	struct efa_com_admin_queue *aq = &amp;edev-&gt;aq;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	/* all mine */</div><div class='del'>-	for (i = 0; i &lt; aq-&gt;depth; i++)</div><div class='del'>-		down(&amp;aq-&gt;avail_cmds);</div><div class='del'>-</div><div class='del'>-	/* let it go */</div><div class='del'>-	for (i = 0; i &lt; aq-&gt;depth; i++)</div><div class='del'>-		up(&amp;aq-&gt;avail_cmds);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void efa_com_admin_flush(struct efa_com_dev *edev)</div><div class='del'>-{</div><div class='del'>-	struct efa_com_admin_queue *aq = &amp;edev-&gt;aq;</div><div class='del'>-</div><div class='del'>-	clear_bit(EFA_AQ_STATE_RUNNING_BIT, &amp;aq-&gt;state);</div><div class='del'>-</div><div class='del'>-	efa_com_abort_admin_commands(edev);</div><div class='del'>-	efa_com_wait_for_abort_completion(edev);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='ctx'>  * efa_com_admin_destroy - Destroy the admin and the async events queues.</div><div class='ctx'>  * @edev: EFA communication layer struct</div><div class='ctx'>  */</div><div class='hunk'>@@ -737,7 +672,7 @@ void efa_com_admin_destroy(struct efa_com_dev *edev)</div><div class='ctx'> 	struct efa_com_admin_sq *sq = &amp;aq-&gt;sq;</div><div class='ctx'> 	u16 size;</div><div class='ctx'> </div><div class='del'>-	efa_com_admin_flush(edev);</div><div class='add'>+	clear_bit(EFA_AQ_STATE_RUNNING_BIT, &amp;aq-&gt;state);</div><div class='ctx'> </div><div class='ctx'> 	devm_kfree(edev-&gt;dmadev, aq-&gt;comp_ctx_pool);</div><div class='ctx'> 	devm_kfree(edev-&gt;dmadev, aq-&gt;comp_ctx);</div><div class='hunk'>@@ -764,7 +699,7 @@ void efa_com_set_admin_polling_mode(struct efa_com_dev *edev, bool polling)</div><div class='ctx'> 	u32 mask_value = 0;</div><div class='ctx'> </div><div class='ctx'> 	if (polling)</div><div class='del'>-		mask_value = EFA_REGS_ADMIN_INTR_MASK;</div><div class='add'>+		EFA_SET(&amp;mask_value, EFA_REGS_INTR_MASK_EN, 1);</div><div class='ctx'> </div><div class='ctx'> 	writel(mask_value, edev-&gt;reg_bar + EFA_REGS_INTR_MASK_OFF);</div><div class='ctx'> 	if (polling)</div><div class='hunk'>@@ -802,7 +737,7 @@ int efa_com_admin_init(struct efa_com_dev *edev,</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	dev_sts = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF);</div><div class='del'>-	if (!(dev_sts &amp; EFA_REGS_DEV_STS_READY_MASK)) {</div><div class='add'>+	if (!EFA_GET(&amp;dev_sts, EFA_REGS_DEV_STS_READY)) {</div><div class='ctx'> 		ibdev_err(edev-&gt;efa_dev,</div><div class='ctx'> 			  "Device isn't ready, abort com init %#x\n", dev_sts);</div><div class='ctx'> 		return -ENODEV;</div><div class='hunk'>@@ -837,8 +772,7 @@ int efa_com_admin_init(struct efa_com_dev *edev,</div><div class='ctx'> 		goto err_destroy_cq;</div><div class='ctx'> </div><div class='ctx'> 	cap = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF);</div><div class='del'>-	timeout = (cap &amp; EFA_REGS_CAPS_ADMIN_CMD_TO_MASK) &gt;&gt;</div><div class='del'>-		  EFA_REGS_CAPS_ADMIN_CMD_TO_SHIFT;</div><div class='add'>+	timeout = EFA_GET(&amp;cap, EFA_REGS_CAPS_ADMIN_CMD_TO);</div><div class='ctx'> 	if (timeout)</div><div class='ctx'> 		/* the resolution of timeout reg is 100ms */</div><div class='ctx'> 		aq-&gt;completion_timeout = timeout * 100000;</div><div class='hunk'>@@ -870,7 +804,7 @@ err_destroy_comp_ctxt:</div><div class='ctx'>  * This method goes over the admin completion queue and wakes up</div><div class='ctx'>  * all the pending threads that wait on the commands wait event.</div><div class='ctx'>  *</div><div class='del'>- * @note: Should be called after MSI-X interrupt.</div><div class='add'>+ * Note: Should be called after MSI-X interrupt.</div><div class='ctx'>  */</div><div class='ctx'> void efa_com_admin_q_comp_intr_handler(struct efa_com_dev *edev)</div><div class='ctx'> {</div><div class='hunk'>@@ -999,7 +933,9 @@ void efa_com_mmio_reg_read_destroy(struct efa_com_dev *edev)</div><div class='ctx'> </div><div class='ctx'> int efa_com_validate_version(struct efa_com_dev *edev)</div><div class='ctx'> {</div><div class='add'>+	u32 min_ctrl_ver = 0;</div><div class='ctx'> 	u32 ctrl_ver_masked;</div><div class='add'>+	u32 min_ver = 0;</div><div class='ctx'> 	u32 ctrl_ver;</div><div class='ctx'> 	u32 ver;</div><div class='ctx'> </div><div class='hunk'>@@ -1012,33 +948,42 @@ int efa_com_validate_version(struct efa_com_dev *edev)</div><div class='ctx'> 				      EFA_REGS_CONTROLLER_VERSION_OFF);</div><div class='ctx'> </div><div class='ctx'> 	ibdev_dbg(edev-&gt;efa_dev, "efa device version: %d.%d\n",</div><div class='del'>-		  (ver &amp; EFA_REGS_VERSION_MAJOR_VERSION_MASK) &gt;&gt;</div><div class='del'>-			  EFA_REGS_VERSION_MAJOR_VERSION_SHIFT,</div><div class='del'>-		  ver &amp; EFA_REGS_VERSION_MINOR_VERSION_MASK);</div><div class='del'>-</div><div class='del'>-	if (ver &lt; MIN_EFA_VER) {</div><div class='add'>+		  EFA_GET(&amp;ver, EFA_REGS_VERSION_MAJOR_VERSION),</div><div class='add'>+		  EFA_GET(&amp;ver, EFA_REGS_VERSION_MINOR_VERSION));</div><div class='add'>+</div><div class='add'>+	EFA_SET(&amp;min_ver, EFA_REGS_VERSION_MAJOR_VERSION,</div><div class='add'>+		EFA_ADMIN_API_VERSION_MAJOR);</div><div class='add'>+	EFA_SET(&amp;min_ver, EFA_REGS_VERSION_MINOR_VERSION,</div><div class='add'>+		EFA_ADMIN_API_VERSION_MINOR);</div><div class='add'>+	if (ver &lt; min_ver) {</div><div class='ctx'> 		ibdev_err(edev-&gt;efa_dev,</div><div class='ctx'> 			  "EFA version is lower than the minimal version the driver supports\n");</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ibdev_dbg(edev-&gt;efa_dev,</div><div class='del'>-		  "efa controller version: %d.%d.%d implementation version %d\n",</div><div class='del'>-		  (ctrl_ver &amp; EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) &gt;&gt;</div><div class='del'>-			  EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT,</div><div class='del'>-		  (ctrl_ver &amp; EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) &gt;&gt;</div><div class='del'>-			  EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT,</div><div class='del'>-		  (ctrl_ver &amp; EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK),</div><div class='del'>-		  (ctrl_ver &amp; EFA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK) &gt;&gt;</div><div class='del'>-			  EFA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT);</div><div class='add'>+	ibdev_dbg(</div><div class='add'>+		edev-&gt;efa_dev,</div><div class='add'>+		"efa controller version: %d.%d.%d implementation version %d\n",</div><div class='add'>+		EFA_GET(&amp;ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION),</div><div class='add'>+		EFA_GET(&amp;ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION),</div><div class='add'>+		EFA_GET(&amp;ctrl_ver,</div><div class='add'>+			EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION),</div><div class='add'>+		EFA_GET(&amp;ctrl_ver, EFA_REGS_CONTROLLER_VERSION_IMPL_ID));</div><div class='ctx'> </div><div class='ctx'> 	ctrl_ver_masked =</div><div class='del'>-		(ctrl_ver &amp; EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) |</div><div class='del'>-		(ctrl_ver &amp; EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) |</div><div class='del'>-		(ctrl_ver &amp; EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK);</div><div class='del'>-</div><div class='add'>+		EFA_GET(&amp;ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION) |</div><div class='add'>+		EFA_GET(&amp;ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION) |</div><div class='add'>+		EFA_GET(&amp;ctrl_ver,</div><div class='add'>+			EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION);</div><div class='add'>+</div><div class='add'>+	EFA_SET(&amp;min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION,</div><div class='add'>+		EFA_CTRL_MAJOR);</div><div class='add'>+	EFA_SET(&amp;min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION,</div><div class='add'>+		EFA_CTRL_MINOR);</div><div class='add'>+	EFA_SET(&amp;min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION,</div><div class='add'>+		EFA_CTRL_SUB_MINOR);</div><div class='ctx'> 	/* Validate the ctrl version without the implementation ID */</div><div class='del'>-	if (ctrl_ver_masked &lt; MIN_EFA_CTRL_VER) {</div><div class='add'>+	if (ctrl_ver_masked &lt; min_ctrl_ver) {</div><div class='ctx'> 		ibdev_err(edev-&gt;efa_dev,</div><div class='ctx'> 			  "EFA ctrl version is lower than the minimal ctrl version the driver supports\n");</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='hunk'>@@ -1061,8 +1006,7 @@ int efa_com_get_dma_width(struct efa_com_dev *edev)</div><div class='ctx'> 	u32 caps = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF);</div><div class='ctx'> 	int width;</div><div class='ctx'> </div><div class='del'>-	width = (caps &amp; EFA_REGS_CAPS_DMA_ADDR_WIDTH_MASK) &gt;&gt;</div><div class='del'>-		EFA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT;</div><div class='add'>+	width = EFA_GET(&amp;caps, EFA_REGS_CAPS_DMA_ADDR_WIDTH);</div><div class='ctx'> </div><div class='ctx'> 	ibdev_dbg(edev-&gt;efa_dev, "DMA width: %d\n", width);</div><div class='ctx'> </div><div class='hunk'>@@ -1076,16 +1020,14 @@ int efa_com_get_dma_width(struct efa_com_dev *edev)</div><div class='ctx'> 	return width;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int wait_for_reset_state(struct efa_com_dev *edev, u32 timeout,</div><div class='del'>-				u16 exp_state)</div><div class='add'>+static int wait_for_reset_state(struct efa_com_dev *edev, u32 timeout, int on)</div><div class='ctx'> {</div><div class='ctx'> 	u32 val, i;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; timeout; i++) {</div><div class='ctx'> 		val = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF);</div><div class='ctx'> </div><div class='del'>-		if ((val &amp; EFA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK) ==</div><div class='del'>-		    exp_state)</div><div class='add'>+		if (EFA_GET(&amp;val, EFA_REGS_DEV_STS_RESET_IN_PROGRESS) == on)</div><div class='ctx'> 			return 0;</div><div class='ctx'> </div><div class='ctx'> 		ibdev_dbg(edev-&gt;efa_dev, "Reset indication val %d\n", val);</div><div class='hunk'>@@ -1105,36 +1047,34 @@ static int wait_for_reset_state(struct efa_com_dev *edev, u32 timeout,</div><div class='ctx'> int efa_com_dev_reset(struct efa_com_dev *edev,</div><div class='ctx'> 		      enum efa_regs_reset_reason_types reset_reason)</div><div class='ctx'> {</div><div class='del'>-	u32 stat, timeout, cap, reset_val;</div><div class='add'>+	u32 stat, timeout, cap;</div><div class='add'>+	u32 reset_val = 0;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	stat = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF);</div><div class='ctx'> 	cap = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF);</div><div class='ctx'> </div><div class='del'>-	if (!(stat &amp; EFA_REGS_DEV_STS_READY_MASK)) {</div><div class='add'>+	if (!EFA_GET(&amp;stat, EFA_REGS_DEV_STS_READY)) {</div><div class='ctx'> 		ibdev_err(edev-&gt;efa_dev,</div><div class='ctx'> 			  "Device isn't ready, can't reset device\n");</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	timeout = (cap &amp; EFA_REGS_CAPS_RESET_TIMEOUT_MASK) &gt;&gt;</div><div class='del'>-		  EFA_REGS_CAPS_RESET_TIMEOUT_SHIFT;</div><div class='add'>+	timeout = EFA_GET(&amp;cap, EFA_REGS_CAPS_RESET_TIMEOUT);</div><div class='ctx'> 	if (!timeout) {</div><div class='ctx'> 		ibdev_err(edev-&gt;efa_dev, "Invalid timeout value\n");</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* start reset */</div><div class='del'>-	reset_val = EFA_REGS_DEV_CTL_DEV_RESET_MASK;</div><div class='del'>-	reset_val |= (reset_reason &lt;&lt; EFA_REGS_DEV_CTL_RESET_REASON_SHIFT) &amp;</div><div class='del'>-		     EFA_REGS_DEV_CTL_RESET_REASON_MASK;</div><div class='add'>+	EFA_SET(&amp;reset_val, EFA_REGS_DEV_CTL_DEV_RESET, 1);</div><div class='add'>+	EFA_SET(&amp;reset_val, EFA_REGS_DEV_CTL_RESET_REASON, reset_reason);</div><div class='ctx'> 	writel(reset_val, edev-&gt;reg_bar + EFA_REGS_DEV_CTL_OFF);</div><div class='ctx'> </div><div class='ctx'> 	/* reset clears the mmio readless address, restore it */</div><div class='ctx'> 	efa_com_mmio_reg_read_resp_addr_init(edev);</div><div class='ctx'> </div><div class='del'>-	err = wait_for_reset_state(edev, timeout,</div><div class='del'>-				   EFA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK);</div><div class='add'>+	err = wait_for_reset_state(edev, timeout, 1);</div><div class='ctx'> 	if (err) {</div><div class='ctx'> 		ibdev_err(edev-&gt;efa_dev, "Reset indication didn't turn on\n");</div><div class='ctx'> 		return err;</div><div class='hunk'>@@ -1148,8 +1088,7 @@ int efa_com_dev_reset(struct efa_com_dev *edev,</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	timeout = (cap &amp; EFA_REGS_CAPS_ADMIN_CMD_TO_MASK) &gt;&gt;</div><div class='del'>-		  EFA_REGS_CAPS_ADMIN_CMD_TO_SHIFT;</div><div class='add'>+	timeout = EFA_GET(&amp;cap, EFA_REGS_CAPS_ADMIN_CMD_TO);</div><div class='ctx'> 	if (timeout)</div><div class='ctx'> 		/* the resolution of timeout reg is 100ms */</div><div class='ctx'> 		edev-&gt;aq.completion_timeout = timeout * 100000;</div><div class='hunk'>@@ -1158,3 +1097,159 @@ int efa_com_dev_reset(struct efa_com_dev *edev,</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='add'>+</div><div class='add'>+static int efa_com_create_eq(struct efa_com_dev *edev,</div><div class='add'>+			     struct efa_com_create_eq_params *params,</div><div class='add'>+			     struct efa_com_create_eq_result *result)</div><div class='add'>+{</div><div class='add'>+	struct efa_com_admin_queue *aq = &amp;edev-&gt;aq;</div><div class='add'>+	struct efa_admin_create_eq_resp resp = {};</div><div class='add'>+	struct efa_admin_create_eq_cmd cmd = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	cmd.aq_common_descriptor.opcode = EFA_ADMIN_CREATE_EQ;</div><div class='add'>+	EFA_SET(&amp;cmd.caps, EFA_ADMIN_CREATE_EQ_CMD_ENTRY_SIZE_WORDS,</div><div class='add'>+		params-&gt;entry_size_in_bytes / 4);</div><div class='add'>+	cmd.depth = params-&gt;depth;</div><div class='add'>+	cmd.event_bitmask = params-&gt;event_bitmask;</div><div class='add'>+	cmd.msix_vec = params-&gt;msix_vec;</div><div class='add'>+</div><div class='add'>+	efa_com_set_dma_addr(params-&gt;dma_addr, &amp;cmd.ba.mem_addr_high,</div><div class='add'>+			     &amp;cmd.ba.mem_addr_low);</div><div class='add'>+</div><div class='add'>+	err = efa_com_cmd_exec(aq,</div><div class='add'>+			       (struct efa_admin_aq_entry *)&amp;cmd,</div><div class='add'>+			       sizeof(cmd),</div><div class='add'>+			       (struct efa_admin_acq_entry *)&amp;resp,</div><div class='add'>+			       sizeof(resp));</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to create eq[%d]\n", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	result-&gt;eqn = resp.eqn;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void efa_com_destroy_eq(struct efa_com_dev *edev,</div><div class='add'>+			       struct efa_com_destroy_eq_params *params)</div><div class='add'>+{</div><div class='add'>+	struct efa_com_admin_queue *aq = &amp;edev-&gt;aq;</div><div class='add'>+	struct efa_admin_destroy_eq_resp resp = {};</div><div class='add'>+	struct efa_admin_destroy_eq_cmd cmd = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	cmd.aq_common_descriptor.opcode = EFA_ADMIN_DESTROY_EQ;</div><div class='add'>+	cmd.eqn = params-&gt;eqn;</div><div class='add'>+</div><div class='add'>+	err = efa_com_cmd_exec(aq,</div><div class='add'>+			       (struct efa_admin_aq_entry *)&amp;cmd,</div><div class='add'>+			       sizeof(cmd),</div><div class='add'>+			       (struct efa_admin_acq_entry *)&amp;resp,</div><div class='add'>+			       sizeof(resp));</div><div class='add'>+	if (err)</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to destroy EQ-%u [%d]\n", cmd.eqn,</div><div class='add'>+				      err);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void efa_com_arm_eq(struct efa_com_dev *edev, struct efa_com_eq *eeq)</div><div class='add'>+{</div><div class='add'>+	u32 val = 0;</div><div class='add'>+</div><div class='add'>+	EFA_SET(&amp;val, EFA_REGS_EQ_DB_EQN, eeq-&gt;eqn);</div><div class='add'>+	EFA_SET(&amp;val, EFA_REGS_EQ_DB_ARM, 1);</div><div class='add'>+</div><div class='add'>+	writel(val, edev-&gt;reg_bar + EFA_REGS_EQ_DB_OFF);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void efa_com_eq_comp_intr_handler(struct efa_com_dev *edev,</div><div class='add'>+				  struct efa_com_eq *eeq)</div><div class='add'>+{</div><div class='add'>+	struct efa_admin_eqe *eqe;</div><div class='add'>+	u32 processed = 0;</div><div class='add'>+	u8 phase;</div><div class='add'>+	u32 ci;</div><div class='add'>+</div><div class='add'>+	ci = eeq-&gt;cc &amp; (eeq-&gt;depth - 1);</div><div class='add'>+	phase = eeq-&gt;phase;</div><div class='add'>+	eqe = &amp;eeq-&gt;eqes[ci];</div><div class='add'>+</div><div class='add'>+	/* Go over all the events */</div><div class='add'>+	while ((READ_ONCE(eqe-&gt;common) &amp; EFA_ADMIN_EQE_PHASE_MASK) == phase) {</div><div class='add'>+		/*</div><div class='add'>+		 * Do not read the rest of the completion entry before the</div><div class='add'>+		 * phase bit was validated</div><div class='add'>+		 */</div><div class='add'>+		dma_rmb();</div><div class='add'>+</div><div class='add'>+		eeq-&gt;cb(eeq, eqe);</div><div class='add'>+</div><div class='add'>+		/* Get next event entry */</div><div class='add'>+		ci++;</div><div class='add'>+		processed++;</div><div class='add'>+</div><div class='add'>+		if (ci == eeq-&gt;depth) {</div><div class='add'>+			ci = 0;</div><div class='add'>+			phase = !phase;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		eqe = &amp;eeq-&gt;eqes[ci];</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	eeq-&gt;cc += processed;</div><div class='add'>+	eeq-&gt;phase = phase;</div><div class='add'>+	efa_com_arm_eq(eeq-&gt;edev, eeq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void efa_com_eq_destroy(struct efa_com_dev *edev, struct efa_com_eq *eeq)</div><div class='add'>+{</div><div class='add'>+	struct efa_com_destroy_eq_params params = {</div><div class='add'>+		.eqn = eeq-&gt;eqn,</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	efa_com_destroy_eq(edev, &amp;params);</div><div class='add'>+	dma_free_coherent(edev-&gt;dmadev, eeq-&gt;depth * sizeof(*eeq-&gt;eqes),</div><div class='add'>+			  eeq-&gt;eqes, eeq-&gt;dma_addr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int efa_com_eq_init(struct efa_com_dev *edev, struct efa_com_eq *eeq,</div><div class='add'>+		    efa_eqe_handler cb, u16 depth, u8 msix_vec)</div><div class='add'>+{</div><div class='add'>+	struct efa_com_create_eq_params params = {};</div><div class='add'>+	struct efa_com_create_eq_result result = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	params.depth = depth;</div><div class='add'>+	params.entry_size_in_bytes = sizeof(*eeq-&gt;eqes);</div><div class='add'>+	EFA_SET(&amp;params.event_bitmask,</div><div class='add'>+		EFA_ADMIN_CREATE_EQ_CMD_COMPLETION_EVENTS, 1);</div><div class='add'>+	params.msix_vec = msix_vec;</div><div class='add'>+</div><div class='add'>+	eeq-&gt;eqes = dma_alloc_coherent(edev-&gt;dmadev,</div><div class='add'>+				       params.depth * sizeof(*eeq-&gt;eqes),</div><div class='add'>+				       &amp;params.dma_addr, GFP_KERNEL);</div><div class='add'>+	if (!eeq-&gt;eqes)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	err = efa_com_create_eq(edev, &amp;params, &amp;result);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_free_coherent;</div><div class='add'>+</div><div class='add'>+	eeq-&gt;eqn = result.eqn;</div><div class='add'>+	eeq-&gt;edev = edev;</div><div class='add'>+	eeq-&gt;dma_addr = params.dma_addr;</div><div class='add'>+	eeq-&gt;phase = 1;</div><div class='add'>+	eeq-&gt;depth = params.depth;</div><div class='add'>+	eeq-&gt;cb = cb;</div><div class='add'>+	efa_com_arm_eq(edev, eeq);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_free_coherent:</div><div class='add'>+	dma_free_coherent(edev-&gt;dmadev, params.depth * sizeof(*eeq-&gt;eqes),</div><div class='add'>+			  eeq-&gt;eqes, params.dma_addr);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/efa/efa_com.h b/drivers/infiniband/hw/efa/efa_com.h<br/>index 84d96724a74b..4d9ca97e4296 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_com.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_com.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_com.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/efa/efa_com.h</a></div><div class='hunk'>@@ -1,6 +1,6 @@</div><div class='ctx'> /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */</div><div class='ctx'> /*</div><div class='del'>- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='add'>+ * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #ifndef _EFA_COM_H_</div><div class='hunk'>@@ -45,9 +45,9 @@ struct efa_com_admin_sq {</div><div class='ctx'> </div><div class='ctx'> /* Don't use anything other than atomic64 */</div><div class='ctx'> struct efa_com_stats_admin {</div><div class='del'>-	atomic64_t aborted_cmd;</div><div class='ctx'> 	atomic64_t submitted_cmd;</div><div class='ctx'> 	atomic64_t completed_cmd;</div><div class='add'>+	atomic64_t cmd_err;</div><div class='ctx'> 	atomic64_t no_completion;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -65,7 +65,7 @@ struct efa_com_admin_queue {</div><div class='ctx'> 	u16 depth;</div><div class='ctx'> 	struct efa_com_admin_cq cq;</div><div class='ctx'> 	struct efa_com_admin_sq sq;</div><div class='del'>-	u16 msix_vector_idx;</div><div class='add'>+	u32 msix_vector_idx;</div><div class='ctx'> </div><div class='ctx'> 	unsigned long state;</div><div class='ctx'> </div><div class='hunk'>@@ -80,13 +80,16 @@ struct efa_com_admin_queue {</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_aenq_handlers;</div><div class='add'>+struct efa_com_eq;</div><div class='add'>+typedef void (*efa_eqe_handler)(struct efa_com_eq *eeq,</div><div class='add'>+				struct efa_admin_eqe *eqe);</div><div class='ctx'> </div><div class='ctx'> struct efa_com_aenq {</div><div class='ctx'> 	struct efa_admin_aenq_entry *entries;</div><div class='ctx'> 	struct efa_aenq_handlers *aenq_handlers;</div><div class='ctx'> 	dma_addr_t dma_addr;</div><div class='ctx'> 	u32 cc; /* consumer counter */</div><div class='del'>-	u16 msix_vector_idx;</div><div class='add'>+	u32 msix_vector_idx;</div><div class='ctx'> 	u16 depth;</div><div class='ctx'> 	u8 phase;</div><div class='ctx'> };</div><div class='hunk'>@@ -112,6 +115,33 @@ struct efa_com_dev {</div><div class='ctx'> 	struct efa_com_mmio_read mmio_read;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct efa_com_eq {</div><div class='add'>+	struct efa_com_dev *edev;</div><div class='add'>+	struct efa_admin_eqe *eqes;</div><div class='add'>+	dma_addr_t dma_addr;</div><div class='add'>+	u32 cc; /* Consumer counter */</div><div class='add'>+	u16 eqn;</div><div class='add'>+	u16 depth;</div><div class='add'>+	u8 phase;</div><div class='add'>+	efa_eqe_handler cb;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_com_create_eq_params {</div><div class='add'>+	dma_addr_t dma_addr;</div><div class='add'>+	u32 event_bitmask;</div><div class='add'>+	u16 depth;</div><div class='add'>+	u8 entry_size_in_bytes;</div><div class='add'>+	u8 msix_vec;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_com_create_eq_result {</div><div class='add'>+	u16 eqn;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_com_destroy_eq_params {</div><div class='add'>+	u16 eqn;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> typedef void (*efa_aenq_handler)(void *data,</div><div class='ctx'> 	      struct efa_admin_aenq_entry *aenq_e);</div><div class='ctx'> </div><div class='hunk'>@@ -121,9 +151,13 @@ struct efa_aenq_handlers {</div><div class='ctx'> 	efa_aenq_handler unimplemented_handler;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+void efa_com_set_dma_addr(dma_addr_t addr, u32 *addr_high, u32 *addr_low);</div><div class='ctx'> int efa_com_admin_init(struct efa_com_dev *edev,</div><div class='ctx'> 		       struct efa_aenq_handlers *aenq_handlers);</div><div class='ctx'> void efa_com_admin_destroy(struct efa_com_dev *edev);</div><div class='add'>+int efa_com_eq_init(struct efa_com_dev *edev, struct efa_com_eq *eeq,</div><div class='add'>+		    efa_eqe_handler cb, u16 depth, u8 msix_vec);</div><div class='add'>+void efa_com_eq_destroy(struct efa_com_dev *edev, struct efa_com_eq *eeq);</div><div class='ctx'> int efa_com_dev_reset(struct efa_com_dev *edev,</div><div class='ctx'> 		      enum efa_regs_reset_reason_types reset_reason);</div><div class='ctx'> void efa_com_set_admin_polling_mode(struct efa_com_dev *edev, bool polling);</div><div class='hunk'>@@ -140,5 +174,7 @@ int efa_com_cmd_exec(struct efa_com_admin_queue *aq,</div><div class='ctx'> 		     struct efa_admin_acq_entry *comp,</div><div class='ctx'> 		     size_t comp_size);</div><div class='ctx'> void efa_com_aenq_intr_handler(struct efa_com_dev *edev, void *data);</div><div class='add'>+void efa_com_eq_comp_intr_handler(struct efa_com_dev *edev,</div><div class='add'>+				  struct efa_com_eq *eeq);</div><div class='ctx'> </div><div class='ctx'> #endif /* _EFA_COM_H_ */</div><div class='head'>diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c<br/>index 14227725521c..9ead02800ac7 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_com_cmd.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_com_cmd.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_com_cmd.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/efa/efa_com_cmd.c</a></div><div class='hunk'>@@ -1,18 +1,11 @@</div><div class='ctx'> // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause</div><div class='ctx'> /*</div><div class='del'>- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='add'>+ * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include "efa.h"</div><div class='ctx'> #include "efa_com.h"</div><div class='ctx'> #include "efa_com_cmd.h"</div><div class='ctx'> </div><div class='del'>-void efa_com_set_dma_addr(dma_addr_t addr, u32 *addr_high, u32 *addr_low)</div><div class='del'>-{</div><div class='del'>-	*addr_low = lower_32_bits(addr);</div><div class='del'>-	*addr_high = upper_32_bits(addr);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> int efa_com_create_qp(struct efa_com_dev *edev,</div><div class='ctx'> 		      struct efa_com_create_qp_params *params,</div><div class='ctx'> 		      struct efa_com_create_qp_result *res)</div><div class='hunk'>@@ -38,6 +31,10 @@ int efa_com_create_qp(struct efa_com_dev *edev,</div><div class='ctx'> 	create_qp_cmd.qp_alloc_size.recv_queue_depth =</div><div class='ctx'> 			params-&gt;rq_depth;</div><div class='ctx'> 	create_qp_cmd.uar = params-&gt;uarn;</div><div class='add'>+	create_qp_cmd.sl = params-&gt;sl;</div><div class='add'>+</div><div class='add'>+	if (params-&gt;unsolicited_write_recv)</div><div class='add'>+		EFA_SET(&amp;create_qp_cmd.flags, EFA_ADMIN_CREATE_QP_CMD_UNSOLICITED_WRITE_RECV, 1);</div><div class='ctx'> </div><div class='ctx'> 	err = efa_com_cmd_exec(aq,</div><div class='ctx'> 			       (struct efa_admin_aq_entry *)&amp;create_qp_cmd,</div><div class='hunk'>@@ -45,7 +42,8 @@ int efa_com_create_qp(struct efa_com_dev *edev,</div><div class='ctx'> 			       (struct efa_admin_acq_entry *)&amp;cmd_completion,</div><div class='ctx'> 			       sizeof(cmd_completion));</div><div class='ctx'> 	if (err) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Failed to create qp [%d]\n", err);</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to create qp [%d]\n", err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -57,7 +55,7 @@ int efa_com_create_qp(struct efa_com_dev *edev,</div><div class='ctx'> 	res-&gt;send_sub_cq_idx = cmd_completion.send_sub_cq_idx;</div><div class='ctx'> 	res-&gt;recv_sub_cq_idx = cmd_completion.recv_sub_cq_idx;</div><div class='ctx'> </div><div class='del'>-	return err;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int efa_com_modify_qp(struct efa_com_dev *edev,</div><div class='hunk'>@@ -76,6 +74,7 @@ int efa_com_modify_qp(struct efa_com_dev *edev,</div><div class='ctx'> 	cmd.qkey = params-&gt;qkey;</div><div class='ctx'> 	cmd.sq_psn = params-&gt;sq_psn;</div><div class='ctx'> 	cmd.sq_drained_async_notify = params-&gt;sq_drained_async_notify;</div><div class='add'>+	cmd.rnr_retry = params-&gt;rnr_retry;</div><div class='ctx'> </div><div class='ctx'> 	err = efa_com_cmd_exec(aq,</div><div class='ctx'> 			       (struct efa_admin_aq_entry *)&amp;cmd,</div><div class='hunk'>@@ -83,9 +82,10 @@ int efa_com_modify_qp(struct efa_com_dev *edev,</div><div class='ctx'> 			       (struct efa_admin_acq_entry *)&amp;resp,</div><div class='ctx'> 			       sizeof(resp));</div><div class='ctx'> 	if (err) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev,</div><div class='del'>-			  "Failed to modify qp-%u modify_mask[%#x] [%d]\n",</div><div class='del'>-			  cmd.qp_handle, cmd.modify_mask, err);</div><div class='add'>+		ibdev_err_ratelimited(</div><div class='add'>+			edev-&gt;efa_dev,</div><div class='add'>+			"Failed to modify qp-%u modify_mask[%#x] [%d]\n",</div><div class='add'>+			cmd.qp_handle, cmd.modify_mask, err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -110,8 +110,9 @@ int efa_com_query_qp(struct efa_com_dev *edev,</div><div class='ctx'> 			       (struct efa_admin_acq_entry *)&amp;resp,</div><div class='ctx'> 			       sizeof(resp));</div><div class='ctx'> 	if (err) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Failed to query qp-%u [%d]\n",</div><div class='del'>-			  cmd.qp_handle, err);</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to query qp-%u [%d]\n",</div><div class='add'>+				      cmd.qp_handle, err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -119,6 +120,7 @@ int efa_com_query_qp(struct efa_com_dev *edev,</div><div class='ctx'> 	result-&gt;qkey = resp.qkey;</div><div class='ctx'> 	result-&gt;sq_draining = resp.sq_draining;</div><div class='ctx'> 	result-&gt;sq_psn = resp.sq_psn;</div><div class='add'>+	result-&gt;rnr_retry = resp.rnr_retry;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -139,9 +141,12 @@ int efa_com_destroy_qp(struct efa_com_dev *edev,</div><div class='ctx'> 			       sizeof(qp_cmd),</div><div class='ctx'> 			       (struct efa_admin_acq_entry *)&amp;cmd_completion,</div><div class='ctx'> 			       sizeof(cmd_completion));</div><div class='del'>-	if (err)</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Failed to destroy qp-%u [%d]\n",</div><div class='del'>-			  qp_cmd.qp_handle, err);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to destroy qp-%u [%d]\n",</div><div class='add'>+				      qp_cmd.qp_handle, err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -150,18 +155,27 @@ int efa_com_create_cq(struct efa_com_dev *edev,</div><div class='ctx'> 		      struct efa_com_create_cq_params *params,</div><div class='ctx'> 		      struct efa_com_create_cq_result *result)</div><div class='ctx'> {</div><div class='del'>-	struct efa_admin_create_cq_resp cmd_completion;</div><div class='add'>+	struct efa_admin_create_cq_resp cmd_completion = {};</div><div class='ctx'> 	struct efa_admin_create_cq_cmd create_cmd = {};</div><div class='ctx'> 	struct efa_com_admin_queue *aq = &amp;edev-&gt;aq;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	create_cmd.aq_common_desc.opcode = EFA_ADMIN_CREATE_CQ;</div><div class='del'>-	create_cmd.cq_caps_2 = (params-&gt;entry_size_in_bytes / 4) &amp;</div><div class='del'>-				EFA_ADMIN_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK;</div><div class='del'>-	create_cmd.cq_depth = params-&gt;cq_depth;</div><div class='add'>+	EFA_SET(&amp;create_cmd.cq_caps_2,</div><div class='add'>+		EFA_ADMIN_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS,</div><div class='add'>+		params-&gt;entry_size_in_bytes / 4);</div><div class='add'>+	create_cmd.sub_cq_depth = params-&gt;sub_cq_depth;</div><div class='ctx'> 	create_cmd.num_sub_cqs = params-&gt;num_sub_cqs;</div><div class='ctx'> 	create_cmd.uar = params-&gt;uarn;</div><div class='del'>-</div><div class='add'>+	if (params-&gt;interrupt_mode_enabled) {</div><div class='add'>+		EFA_SET(&amp;create_cmd.cq_caps_1,</div><div class='add'>+			EFA_ADMIN_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED, 1);</div><div class='add'>+		create_cmd.eqn = params-&gt;eqn;</div><div class='add'>+	}</div><div class='add'>+	if (params-&gt;set_src_addr) {</div><div class='add'>+		EFA_SET(&amp;create_cmd.cq_caps_2,</div><div class='add'>+			EFA_ADMIN_CREATE_CQ_CMD_SET_SRC_ADDR, 1);</div><div class='add'>+	}</div><div class='ctx'> 	efa_com_set_dma_addr(params-&gt;dma_addr,</div><div class='ctx'> 			     &amp;create_cmd.cq_ba.mem_addr_high,</div><div class='ctx'> 			     &amp;create_cmd.cq_ba.mem_addr_low);</div><div class='hunk'>@@ -172,14 +186,18 @@ int efa_com_create_cq(struct efa_com_dev *edev,</div><div class='ctx'> 			       (struct efa_admin_acq_entry *)&amp;cmd_completion,</div><div class='ctx'> 			       sizeof(cmd_completion));</div><div class='ctx'> 	if (err) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Failed to create cq[%d]\n", err);</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to create cq[%d]\n", err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	result-&gt;cq_idx = cmd_completion.cq_idx;</div><div class='del'>-	result-&gt;actual_depth = params-&gt;cq_depth;</div><div class='add'>+	result-&gt;actual_depth = params-&gt;sub_cq_depth;</div><div class='add'>+	result-&gt;db_off = cmd_completion.db_offset;</div><div class='add'>+	result-&gt;db_valid = EFA_GET(&amp;cmd_completion.flags,</div><div class='add'>+				   EFA_ADMIN_CREATE_CQ_RESP_DB_VALID);</div><div class='ctx'> </div><div class='del'>-	return err;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int efa_com_destroy_cq(struct efa_com_dev *edev,</div><div class='hunk'>@@ -199,9 +217,12 @@ int efa_com_destroy_cq(struct efa_com_dev *edev,</div><div class='ctx'> 			       (struct efa_admin_acq_entry *)&amp;destroy_resp,</div><div class='ctx'> 			       sizeof(destroy_resp));</div><div class='ctx'> </div><div class='del'>-	if (err)</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Failed to destroy CQ-%u [%d]\n",</div><div class='del'>-			  params-&gt;cq_idx, err);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to destroy CQ-%u [%d]\n",</div><div class='add'>+				      params-&gt;cq_idx, err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -218,11 +239,10 @@ int efa_com_register_mr(struct efa_com_dev *edev,</div><div class='ctx'> 	mr_cmd.aq_common_desc.opcode = EFA_ADMIN_REG_MR;</div><div class='ctx'> 	mr_cmd.pd = params-&gt;pd;</div><div class='ctx'> 	mr_cmd.mr_length = params-&gt;mr_length_in_bytes;</div><div class='del'>-	mr_cmd.flags |= params-&gt;page_shift &amp;</div><div class='del'>-		EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK;</div><div class='add'>+	EFA_SET(&amp;mr_cmd.flags, EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT,</div><div class='add'>+		params-&gt;page_shift);</div><div class='ctx'> 	mr_cmd.iova = params-&gt;iova;</div><div class='del'>-	mr_cmd.permissions |= params-&gt;permissions &amp;</div><div class='del'>-			      EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE_MASK;</div><div class='add'>+	mr_cmd.permissions = params-&gt;permissions;</div><div class='ctx'> </div><div class='ctx'> 	if (params-&gt;inline_pbl) {</div><div class='ctx'> 		memcpy(mr_cmd.pbl.inline_pbl_array,</div><div class='hunk'>@@ -234,11 +254,11 @@ int efa_com_register_mr(struct efa_com_dev *edev,</div><div class='ctx'> 			params-&gt;pbl.pbl.address.mem_addr_low;</div><div class='ctx'> 		mr_cmd.pbl.pbl.address.mem_addr_high =</div><div class='ctx'> 			params-&gt;pbl.pbl.address.mem_addr_high;</div><div class='del'>-		mr_cmd.aq_common_desc.flags |=</div><div class='del'>-			EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK;</div><div class='add'>+		EFA_SET(&amp;mr_cmd.aq_common_desc.flags,</div><div class='add'>+			EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA, 1);</div><div class='ctx'> 		if (params-&gt;indirect)</div><div class='del'>-			mr_cmd.aq_common_desc.flags |=</div><div class='del'>-				EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;</div><div class='add'>+			EFA_SET(&amp;mr_cmd.aq_common_desc.flags,</div><div class='add'>+				EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT, 1);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	err = efa_com_cmd_exec(aq,</div><div class='hunk'>@@ -247,12 +267,22 @@ int efa_com_register_mr(struct efa_com_dev *edev,</div><div class='ctx'> 			       (struct efa_admin_acq_entry *)&amp;cmd_completion,</div><div class='ctx'> 			       sizeof(cmd_completion));</div><div class='ctx'> 	if (err) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Failed to register mr [%d]\n", err);</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to register mr [%d]\n", err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	result-&gt;l_key = cmd_completion.l_key;</div><div class='ctx'> 	result-&gt;r_key = cmd_completion.r_key;</div><div class='add'>+	result-&gt;ic_info.recv_ic_id = cmd_completion.recv_ic_id;</div><div class='add'>+	result-&gt;ic_info.rdma_read_ic_id = cmd_completion.rdma_read_ic_id;</div><div class='add'>+	result-&gt;ic_info.rdma_recv_ic_id = cmd_completion.rdma_recv_ic_id;</div><div class='add'>+	result-&gt;ic_info.recv_ic_id_valid = EFA_GET(&amp;cmd_completion.validity,</div><div class='add'>+						   EFA_ADMIN_REG_MR_RESP_RECV_IC_ID);</div><div class='add'>+	result-&gt;ic_info.rdma_read_ic_id_valid = EFA_GET(&amp;cmd_completion.validity,</div><div class='add'>+							EFA_ADMIN_REG_MR_RESP_RDMA_READ_IC_ID);</div><div class='add'>+	result-&gt;ic_info.rdma_recv_ic_id_valid = EFA_GET(&amp;cmd_completion.validity,</div><div class='add'>+							EFA_ADMIN_REG_MR_RESP_RDMA_RECV_IC_ID);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -273,10 +303,12 @@ int efa_com_dereg_mr(struct efa_com_dev *edev,</div><div class='ctx'> 			       sizeof(mr_cmd),</div><div class='ctx'> 			       (struct efa_admin_acq_entry *)&amp;cmd_completion,</div><div class='ctx'> 			       sizeof(cmd_completion));</div><div class='del'>-	if (err)</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev,</div><div class='del'>-			  "Failed to de-register mr(lkey-%u) [%d]\n",</div><div class='del'>-			  mr_cmd.l_key, err);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to de-register mr(lkey-%u) [%d]\n",</div><div class='add'>+				      mr_cmd.l_key, err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -301,7 +333,9 @@ int efa_com_create_ah(struct efa_com_dev *edev,</div><div class='ctx'> 			       (struct efa_admin_acq_entry *)&amp;cmd_completion,</div><div class='ctx'> 			       sizeof(cmd_completion));</div><div class='ctx'> 	if (err) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Failed to create ah [%d]\n", err);</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to create ah for %pI6 [%d]\n",</div><div class='add'>+				      ah_cmd.dest_addr, err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -327,14 +361,17 @@ int efa_com_destroy_ah(struct efa_com_dev *edev,</div><div class='ctx'> 			       sizeof(ah_cmd),</div><div class='ctx'> 			       (struct efa_admin_acq_entry *)&amp;cmd_completion,</div><div class='ctx'> 			       sizeof(cmd_completion));</div><div class='del'>-	if (err)</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Failed to destroy ah-%d pd-%d [%d]\n",</div><div class='del'>-			  ah_cmd.ah, ah_cmd.pd, err);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to destroy ah-%d pd-%d [%d]\n",</div><div class='add'>+				      ah_cmd.ah, ah_cmd.pd, err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static bool</div><div class='add'>+bool</div><div class='ctx'> efa_com_check_supported_feature_id(struct efa_com_dev *edev,</div><div class='ctx'> 				   enum efa_admin_aq_feature_id feature_id)</div><div class='ctx'> {</div><div class='hunk'>@@ -359,8 +396,9 @@ static int efa_com_get_feature_ex(struct efa_com_dev *edev,</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	if (!efa_com_check_supported_feature_id(edev, feature_id)) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Feature %d isn't supported\n",</div><div class='del'>-			  feature_id);</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Feature %d isn't supported\n",</div><div class='add'>+				      feature_id);</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -369,9 +407,8 @@ static int efa_com_get_feature_ex(struct efa_com_dev *edev,</div><div class='ctx'> 	get_cmd.aq_common_descriptor.opcode = EFA_ADMIN_GET_FEATURE;</div><div class='ctx'> </div><div class='ctx'> 	if (control_buff_size)</div><div class='del'>-		get_cmd.aq_common_descriptor.flags =</div><div class='del'>-			EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;</div><div class='del'>-</div><div class='add'>+		EFA_SET(&amp;get_cmd.aq_common_descriptor.flags,</div><div class='add'>+			EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA, 1);</div><div class='ctx'> </div><div class='ctx'> 	efa_com_set_dma_addr(control_buf_dma_addr,</div><div class='ctx'> 			     &amp;get_cmd.control_buffer.address.mem_addr_high,</div><div class='hunk'>@@ -387,10 +424,13 @@ static int efa_com_get_feature_ex(struct efa_com_dev *edev,</div><div class='ctx'> 			       get_resp,</div><div class='ctx'> 			       sizeof(*get_resp));</div><div class='ctx'> </div><div class='del'>-	if (err)</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev,</div><div class='del'>-			  "Failed to submit get_feature command %d [%d]\n",</div><div class='del'>-			  feature_id, err);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err_ratelimited(</div><div class='add'>+			edev-&gt;efa_dev,</div><div class='add'>+			"Failed to submit get_feature command %d [%d]\n",</div><div class='add'>+			feature_id, err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -402,27 +442,6 @@ static int efa_com_get_feature(struct efa_com_dev *edev,</div><div class='ctx'> 	return efa_com_get_feature_ex(edev, get_resp, feature_id, 0, 0);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int efa_com_get_network_attr(struct efa_com_dev *edev,</div><div class='del'>-			     struct efa_com_get_network_attr_result *result)</div><div class='del'>-{</div><div class='del'>-	struct efa_admin_get_feature_resp resp;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	err = efa_com_get_feature(edev, &amp;resp,</div><div class='del'>-				  EFA_ADMIN_NETWORK_ATTR);</div><div class='del'>-	if (err) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev,</div><div class='del'>-			  "Failed to get network attributes %d\n", err);</div><div class='del'>-		return err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	memcpy(result-&gt;addr, resp.u.network_attr.addr,</div><div class='del'>-	       sizeof(resp.u.network_attr.addr));</div><div class='del'>-	result-&gt;mtu = resp.u.network_attr.mtu;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> int efa_com_get_device_attr(struct efa_com_dev *edev,</div><div class='ctx'> 			    struct efa_com_get_device_attr_result *result)</div><div class='ctx'> {</div><div class='hunk'>@@ -431,8 +450,9 @@ int efa_com_get_device_attr(struct efa_com_dev *edev,</div><div class='ctx'> </div><div class='ctx'> 	err = efa_com_get_feature(edev, &amp;resp, EFA_ADMIN_DEVICE_ATTR);</div><div class='ctx'> 	if (err) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Failed to get device attributes %d\n",</div><div class='del'>-			  err);</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to get device attributes %d\n",</div><div class='add'>+				      err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -444,11 +464,16 @@ int efa_com_get_device_attr(struct efa_com_dev *edev,</div><div class='ctx'> 	result-&gt;phys_addr_width = resp.u.device_attr.phys_addr_width;</div><div class='ctx'> 	result-&gt;virt_addr_width = resp.u.device_attr.virt_addr_width;</div><div class='ctx'> 	result-&gt;db_bar = resp.u.device_attr.db_bar;</div><div class='add'>+	result-&gt;max_rdma_size = resp.u.device_attr.max_rdma_size;</div><div class='add'>+	result-&gt;device_caps = resp.u.device_attr.device_caps;</div><div class='add'>+	result-&gt;guid = resp.u.device_attr.guid;</div><div class='add'>+	result-&gt;max_link_speed_gbps = resp.u.device_attr.max_link_speed_gbps;</div><div class='ctx'> </div><div class='ctx'> 	if (result-&gt;admin_api_version &lt; 1) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev,</div><div class='del'>-			  "Failed to get device attr api version [%u &lt; 1]\n",</div><div class='del'>-			  result-&gt;admin_api_version);</div><div class='add'>+		ibdev_err_ratelimited(</div><div class='add'>+			edev-&gt;efa_dev,</div><div class='add'>+			"Failed to get device attr api version [%u &lt; 1]\n",</div><div class='add'>+			result-&gt;admin_api_version);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -456,8 +481,9 @@ int efa_com_get_device_attr(struct efa_com_dev *edev,</div><div class='ctx'> 	err = efa_com_get_feature(edev, &amp;resp,</div><div class='ctx'> 				  EFA_ADMIN_QUEUE_ATTR);</div><div class='ctx'> 	if (err) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev,</div><div class='del'>-			  "Failed to get network attributes %d\n", err);</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to get queue attributes %d\n",</div><div class='add'>+				      err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -475,6 +501,38 @@ int efa_com_get_device_attr(struct efa_com_dev *edev,</div><div class='ctx'> 	result-&gt;max_ah = resp.u.queue_attr.max_ah;</div><div class='ctx'> 	result-&gt;max_llq_size = resp.u.queue_attr.max_llq_size;</div><div class='ctx'> 	result-&gt;sub_cqs_per_cq = resp.u.queue_attr.sub_cqs_per_cq;</div><div class='add'>+	result-&gt;max_wr_rdma_sge = resp.u.queue_attr.max_wr_rdma_sges;</div><div class='add'>+	result-&gt;max_tx_batch = resp.u.queue_attr.max_tx_batch;</div><div class='add'>+	result-&gt;min_sq_depth = resp.u.queue_attr.min_sq_depth;</div><div class='add'>+</div><div class='add'>+	err = efa_com_get_feature(edev, &amp;resp, EFA_ADMIN_NETWORK_ATTR);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to get network attributes %d\n",</div><div class='add'>+				      err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	memcpy(result-&gt;addr, resp.u.network_attr.addr,</div><div class='add'>+	       sizeof(resp.u.network_attr.addr));</div><div class='add'>+	result-&gt;mtu = resp.u.network_attr.mtu;</div><div class='add'>+</div><div class='add'>+	if (efa_com_check_supported_feature_id(edev,</div><div class='add'>+					       EFA_ADMIN_EVENT_QUEUE_ATTR)) {</div><div class='add'>+		err = efa_com_get_feature(edev, &amp;resp,</div><div class='add'>+					  EFA_ADMIN_EVENT_QUEUE_ATTR);</div><div class='add'>+		if (err) {</div><div class='add'>+			ibdev_err_ratelimited(</div><div class='add'>+				edev-&gt;efa_dev,</div><div class='add'>+				"Failed to get event queue attributes %d\n",</div><div class='add'>+				err);</div><div class='add'>+			return err;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		result-&gt;max_eq = resp.u.event_queue_attr.max_eq;</div><div class='add'>+		result-&gt;max_eq_depth = resp.u.event_queue_attr.max_eq_depth;</div><div class='add'>+		result-&gt;event_bitmask = resp.u.event_queue_attr.event_bitmask;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -487,7 +545,8 @@ int efa_com_get_hw_hints(struct efa_com_dev *edev,</div><div class='ctx'> </div><div class='ctx'> 	err = efa_com_get_feature(edev, &amp;resp, EFA_ADMIN_HW_HINTS);</div><div class='ctx'> 	if (err) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Failed to get hw hints %d\n", err);</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to get hw hints %d\n", err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -499,19 +558,20 @@ int efa_com_get_hw_hints(struct efa_com_dev *edev,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int efa_com_set_feature_ex(struct efa_com_dev *edev,</div><div class='del'>-				  struct efa_admin_set_feature_resp *set_resp,</div><div class='del'>-				  struct efa_admin_set_feature_cmd *set_cmd,</div><div class='del'>-				  enum efa_admin_aq_feature_id feature_id,</div><div class='del'>-				  dma_addr_t control_buf_dma_addr,</div><div class='del'>-				  u32 control_buff_size)</div><div class='add'>+int efa_com_set_feature_ex(struct efa_com_dev *edev,</div><div class='add'>+			   struct efa_admin_set_feature_resp *set_resp,</div><div class='add'>+			   struct efa_admin_set_feature_cmd *set_cmd,</div><div class='add'>+			   enum efa_admin_aq_feature_id feature_id,</div><div class='add'>+			   dma_addr_t control_buf_dma_addr,</div><div class='add'>+			   u32 control_buff_size)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_com_admin_queue *aq;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	if (!efa_com_check_supported_feature_id(edev, feature_id)) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Feature %d isn't supported\n",</div><div class='del'>-			  feature_id);</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Feature %d isn't supported\n",</div><div class='add'>+				      feature_id);</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -519,8 +579,9 @@ static int efa_com_set_feature_ex(struct efa_com_dev *edev,</div><div class='ctx'> </div><div class='ctx'> 	set_cmd-&gt;aq_common_descriptor.opcode = EFA_ADMIN_SET_FEATURE;</div><div class='ctx'> 	if (control_buff_size) {</div><div class='del'>-		set_cmd-&gt;aq_common_descriptor.flags =</div><div class='del'>-			EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;</div><div class='add'>+		set_cmd-&gt;aq_common_descriptor.flags = 0;</div><div class='add'>+		EFA_SET(&amp;set_cmd-&gt;aq_common_descriptor.flags,</div><div class='add'>+			EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA, 1);</div><div class='ctx'> 		efa_com_set_dma_addr(control_buf_dma_addr,</div><div class='ctx'> 				     &amp;set_cmd-&gt;control_buffer.address.mem_addr_high,</div><div class='ctx'> 				     &amp;set_cmd-&gt;control_buffer.address.mem_addr_low);</div><div class='hunk'>@@ -534,10 +595,13 @@ static int efa_com_set_feature_ex(struct efa_com_dev *edev,</div><div class='ctx'> 			       (struct efa_admin_acq_entry *)set_resp,</div><div class='ctx'> 			       sizeof(*set_resp));</div><div class='ctx'> </div><div class='del'>-	if (err)</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev,</div><div class='del'>-			  "Failed to submit set_feature command %d error: %d\n",</div><div class='del'>-			  feature_id, err);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err_ratelimited(</div><div class='add'>+			edev-&gt;efa_dev,</div><div class='add'>+			"Failed to submit set_feature command %d error: %d\n",</div><div class='add'>+			feature_id, err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -562,8 +626,9 @@ int efa_com_set_aenq_config(struct efa_com_dev *edev, u32 groups)</div><div class='ctx'> </div><div class='ctx'> 	err = efa_com_get_feature(edev, &amp;get_resp, EFA_ADMIN_AENQ_CONFIG);</div><div class='ctx'> 	if (err) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Failed to get aenq attributes: %d\n",</div><div class='del'>-			  err);</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to get aenq attributes: %d\n",</div><div class='add'>+				      err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -573,9 +638,10 @@ int efa_com_set_aenq_config(struct efa_com_dev *edev, u32 groups)</div><div class='ctx'> 		  get_resp.u.aenq.enabled_groups);</div><div class='ctx'> </div><div class='ctx'> 	if ((get_resp.u.aenq.supported_groups &amp; groups) != groups) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev,</div><div class='del'>-			  "Trying to set unsupported aenq groups[%#x] supported[%#x]\n",</div><div class='del'>-			  groups, get_resp.u.aenq.supported_groups);</div><div class='add'>+		ibdev_err_ratelimited(</div><div class='add'>+			edev-&gt;efa_dev,</div><div class='add'>+			"Trying to set unsupported aenq groups[%#x] supported[%#x]\n",</div><div class='add'>+			groups, get_resp.u.aenq.supported_groups);</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -583,8 +649,9 @@ int efa_com_set_aenq_config(struct efa_com_dev *edev, u32 groups)</div><div class='ctx'> 	err = efa_com_set_feature(edev, &amp;set_resp, &amp;cmd,</div><div class='ctx'> 				  EFA_ADMIN_AENQ_CONFIG);</div><div class='ctx'> 	if (err) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Failed to set aenq attributes: %d\n",</div><div class='del'>-			  err);</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to set aenq attributes: %d\n",</div><div class='add'>+				      err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -607,7 +674,8 @@ int efa_com_alloc_pd(struct efa_com_dev *edev,</div><div class='ctx'> 			       (struct efa_admin_acq_entry *)&amp;resp,</div><div class='ctx'> 			       sizeof(resp));</div><div class='ctx'> 	if (err) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Failed to allocate pd[%d]\n", err);</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to allocate pd[%d]\n", err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -633,8 +701,9 @@ int efa_com_dealloc_pd(struct efa_com_dev *edev,</div><div class='ctx'> 			       (struct efa_admin_acq_entry *)&amp;resp,</div><div class='ctx'> 			       sizeof(resp));</div><div class='ctx'> 	if (err) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Failed to deallocate pd-%u [%d]\n",</div><div class='del'>-			  cmd.pd, err);</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to deallocate pd-%u [%d]\n",</div><div class='add'>+				      cmd.pd, err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -657,7 +726,8 @@ int efa_com_alloc_uar(struct efa_com_dev *edev,</div><div class='ctx'> 			       (struct efa_admin_acq_entry *)&amp;resp,</div><div class='ctx'> 			       sizeof(resp));</div><div class='ctx'> 	if (err) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Failed to allocate uar[%d]\n", err);</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to allocate uar[%d]\n", err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -683,10 +753,86 @@ int efa_com_dealloc_uar(struct efa_com_dev *edev,</div><div class='ctx'> 			       (struct efa_admin_acq_entry *)&amp;resp,</div><div class='ctx'> 			       sizeof(resp));</div><div class='ctx'> 	if (err) {</div><div class='del'>-		ibdev_err(edev-&gt;efa_dev, "Failed to deallocate uar-%u [%d]\n",</div><div class='del'>-			  cmd.uar, err);</div><div class='add'>+		ibdev_err_ratelimited(edev-&gt;efa_dev,</div><div class='add'>+				      "Failed to deallocate uar-%u [%d]\n",</div><div class='add'>+				      cmd.uar, err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='add'>+</div><div class='add'>+int efa_com_get_stats(struct efa_com_dev *edev,</div><div class='add'>+		      struct efa_com_get_stats_params *params,</div><div class='add'>+		      union efa_com_get_stats_result *result)</div><div class='add'>+{</div><div class='add'>+	struct efa_com_admin_queue *aq = &amp;edev-&gt;aq;</div><div class='add'>+	struct efa_admin_aq_get_stats_cmd cmd = {};</div><div class='add'>+	struct efa_admin_acq_get_stats_resp resp;</div><div class='add'>+	struct efa_admin_rdma_write_stats *rws;</div><div class='add'>+	struct efa_admin_rdma_read_stats *rrs;</div><div class='add'>+	struct efa_admin_messages_stats *ms;</div><div class='add'>+	struct efa_admin_network_stats *ns;</div><div class='add'>+	struct efa_admin_basic_stats *bs;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	cmd.aq_common_descriptor.opcode = EFA_ADMIN_GET_STATS;</div><div class='add'>+	cmd.type = params-&gt;type;</div><div class='add'>+	cmd.scope = params-&gt;scope;</div><div class='add'>+	cmd.scope_modifier = params-&gt;scope_modifier;</div><div class='add'>+</div><div class='add'>+	err = efa_com_cmd_exec(aq,</div><div class='add'>+			       (struct efa_admin_aq_entry *)&amp;cmd,</div><div class='add'>+			       sizeof(cmd),</div><div class='add'>+			       (struct efa_admin_acq_entry *)&amp;resp,</div><div class='add'>+			       sizeof(resp));</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err_ratelimited(</div><div class='add'>+			edev-&gt;efa_dev,</div><div class='add'>+			"Failed to get stats type-%u scope-%u.%u [%d]\n",</div><div class='add'>+			cmd.type, cmd.scope, cmd.scope_modifier, err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (cmd.type) {</div><div class='add'>+	case EFA_ADMIN_GET_STATS_TYPE_BASIC:</div><div class='add'>+		bs = &amp;resp.u.basic_stats;</div><div class='add'>+		result-&gt;basic_stats.tx_bytes = bs-&gt;tx_bytes;</div><div class='add'>+		result-&gt;basic_stats.tx_pkts = bs-&gt;tx_pkts;</div><div class='add'>+		result-&gt;basic_stats.rx_bytes = bs-&gt;rx_bytes;</div><div class='add'>+		result-&gt;basic_stats.rx_pkts = bs-&gt;rx_pkts;</div><div class='add'>+		result-&gt;basic_stats.rx_drops = bs-&gt;rx_drops;</div><div class='add'>+		break;</div><div class='add'>+	case EFA_ADMIN_GET_STATS_TYPE_MESSAGES:</div><div class='add'>+		ms = &amp;resp.u.messages_stats;</div><div class='add'>+		result-&gt;messages_stats.send_bytes = ms-&gt;send_bytes;</div><div class='add'>+		result-&gt;messages_stats.send_wrs = ms-&gt;send_wrs;</div><div class='add'>+		result-&gt;messages_stats.recv_bytes = ms-&gt;recv_bytes;</div><div class='add'>+		result-&gt;messages_stats.recv_wrs = ms-&gt;recv_wrs;</div><div class='add'>+		break;</div><div class='add'>+	case EFA_ADMIN_GET_STATS_TYPE_RDMA_READ:</div><div class='add'>+		rrs = &amp;resp.u.rdma_read_stats;</div><div class='add'>+		result-&gt;rdma_read_stats.read_wrs = rrs-&gt;read_wrs;</div><div class='add'>+		result-&gt;rdma_read_stats.read_bytes = rrs-&gt;read_bytes;</div><div class='add'>+		result-&gt;rdma_read_stats.read_wr_err = rrs-&gt;read_wr_err;</div><div class='add'>+		result-&gt;rdma_read_stats.read_resp_bytes = rrs-&gt;read_resp_bytes;</div><div class='add'>+		break;</div><div class='add'>+	case EFA_ADMIN_GET_STATS_TYPE_RDMA_WRITE:</div><div class='add'>+		rws = &amp;resp.u.rdma_write_stats;</div><div class='add'>+		result-&gt;rdma_write_stats.write_wrs = rws-&gt;write_wrs;</div><div class='add'>+		result-&gt;rdma_write_stats.write_bytes = rws-&gt;write_bytes;</div><div class='add'>+		result-&gt;rdma_write_stats.write_wr_err = rws-&gt;write_wr_err;</div><div class='add'>+		result-&gt;rdma_write_stats.write_recv_bytes = rws-&gt;write_recv_bytes;</div><div class='add'>+		break;</div><div class='add'>+	case EFA_ADMIN_GET_STATS_TYPE_NETWORK:</div><div class='add'>+		ns = &amp;resp.u.network_stats;</div><div class='add'>+		result-&gt;network_stats.retrans_bytes = ns-&gt;retrans_bytes;</div><div class='add'>+		result-&gt;network_stats.retrans_pkts = ns-&gt;retrans_pkts;</div><div class='add'>+		result-&gt;network_stats.retrans_timeout_events = ns-&gt;retrans_timeout_events;</div><div class='add'>+		result-&gt;network_stats.unresponsive_remote_events = ns-&gt;unresponsive_remote_events;</div><div class='add'>+		result-&gt;network_stats.impaired_remote_conn_events = ns-&gt;impaired_remote_conn_events;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h<br/>index a1174380462c..3ac2686abba1 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_com_cmd.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_com_cmd.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_com_cmd.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/efa/efa_com_cmd.h</a></div><div class='hunk'>@@ -1,6 +1,6 @@</div><div class='ctx'> /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */</div><div class='ctx'> /*</div><div class='del'>- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='add'>+ * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #ifndef _EFA_COM_CMD_H_</div><div class='hunk'>@@ -27,6 +27,8 @@ struct efa_com_create_qp_params {</div><div class='ctx'> 	u16 pd;</div><div class='ctx'> 	u16 uarn;</div><div class='ctx'> 	u8 qp_type;</div><div class='add'>+	u8 sl;</div><div class='add'>+	u8 unsolicited_write_recv : 1;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_com_create_qp_result {</div><div class='hunk'>@@ -47,6 +49,7 @@ struct efa_com_modify_qp_params {</div><div class='ctx'> 	u32 qkey;</div><div class='ctx'> 	u32 sq_psn;</div><div class='ctx'> 	u8 sq_drained_async_notify;</div><div class='add'>+	u8 rnr_retry;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_com_query_qp_params {</div><div class='hunk'>@@ -58,6 +61,7 @@ struct efa_com_query_qp_result {</div><div class='ctx'> 	u32 qkey;</div><div class='ctx'> 	u32 sq_draining;</div><div class='ctx'> 	u32 sq_psn;</div><div class='add'>+	u8 rnr_retry;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_com_destroy_qp_params {</div><div class='hunk'>@@ -68,10 +72,13 @@ struct efa_com_create_cq_params {</div><div class='ctx'> 	/* cq physical base address in OS memory */</div><div class='ctx'> 	dma_addr_t dma_addr;</div><div class='ctx'> 	/* completion queue depth in # of entries */</div><div class='del'>-	u16 cq_depth;</div><div class='add'>+	u16 sub_cq_depth;</div><div class='ctx'> 	u16 num_sub_cqs;</div><div class='ctx'> 	u16 uarn;</div><div class='add'>+	u16 eqn;</div><div class='ctx'> 	u8 entry_size_in_bytes;</div><div class='add'>+	u8 interrupt_mode_enabled : 1;</div><div class='add'>+	u8 set_src_addr : 1;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_com_create_cq_result {</div><div class='hunk'>@@ -79,6 +86,8 @@ struct efa_com_create_cq_result {</div><div class='ctx'> 	u16 cq_idx;</div><div class='ctx'> 	/* actual cq depth in # of entries */</div><div class='ctx'> 	u16 actual_depth;</div><div class='add'>+	u32 db_off;</div><div class='add'>+	bool db_valid;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_com_destroy_cq_params {</div><div class='hunk'>@@ -100,14 +109,12 @@ struct efa_com_destroy_ah_params {</div><div class='ctx'> 	u16 pdn;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct efa_com_get_network_attr_result {</div><div class='del'>-	u8 addr[EFA_GID_SIZE];</div><div class='del'>-	u32 mtu;</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> struct efa_com_get_device_attr_result {</div><div class='add'>+	u8 addr[EFA_GID_SIZE];</div><div class='ctx'> 	u64 page_size_cap;</div><div class='ctx'> 	u64 max_mr_pages;</div><div class='add'>+	u64 guid;</div><div class='add'>+	u32 mtu;</div><div class='ctx'> 	u32 fw_version;</div><div class='ctx'> 	u32 admin_api_version;</div><div class='ctx'> 	u32 device_version;</div><div class='hunk'>@@ -124,9 +131,18 @@ struct efa_com_get_device_attr_result {</div><div class='ctx'> 	u32 max_pd;</div><div class='ctx'> 	u32 max_ah;</div><div class='ctx'> 	u32 max_llq_size;</div><div class='add'>+	u32 max_rdma_size;</div><div class='add'>+	u32 device_caps;</div><div class='add'>+	u32 max_eq;</div><div class='add'>+	u32 max_eq_depth;</div><div class='add'>+	u32 event_bitmask; /* EQ events bitmask */</div><div class='ctx'> 	u16 sub_cqs_per_cq;</div><div class='ctx'> 	u16 max_sq_sge;</div><div class='ctx'> 	u16 max_rq_sge;</div><div class='add'>+	u16 max_wr_rdma_sge;</div><div class='add'>+	u16 max_tx_batch;</div><div class='add'>+	u16 min_sq_depth;</div><div class='add'>+	u16 max_link_speed_gbps;</div><div class='ctx'> 	u8 db_bar;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -181,17 +197,21 @@ struct efa_com_reg_mr_params {</div><div class='ctx'> 	 * address mapping</div><div class='ctx'> 	 */</div><div class='ctx'> 	u8 page_shift;</div><div class='del'>-	/*</div><div class='del'>-	 * permissions</div><div class='del'>-	 * 0: local_write_enable - Write permissions: value of 1 needed</div><div class='del'>-	 * for RQ buffers and for RDMA write:1: reserved1 - remote</div><div class='del'>-	 * access flags, etc</div><div class='del'>-	 */</div><div class='add'>+	/* see permissions field of struct efa_admin_reg_mr_cmd */</div><div class='ctx'> 	u8 permissions;</div><div class='ctx'> 	u8 inline_pbl;</div><div class='ctx'> 	u8 indirect;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct efa_com_mr_interconnect_info {</div><div class='add'>+	u16 recv_ic_id;</div><div class='add'>+	u16 rdma_read_ic_id;</div><div class='add'>+	u16 rdma_recv_ic_id;</div><div class='add'>+	u8 recv_ic_id_valid : 1;</div><div class='add'>+	u8 rdma_read_ic_id_valid : 1;</div><div class='add'>+	u8 rdma_recv_ic_id_valid : 1;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct efa_com_reg_mr_result {</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * To be used in conjunction with local buffers references in SQ and</div><div class='hunk'>@@ -203,6 +223,7 @@ struct efa_com_reg_mr_result {</div><div class='ctx'> 	 * accessed memory region</div><div class='ctx'> 	 */</div><div class='ctx'> 	u32 r_key;</div><div class='add'>+	struct efa_com_mr_interconnect_info ic_info;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct efa_com_dereg_mr_params {</div><div class='hunk'>@@ -225,7 +246,59 @@ struct efa_com_dealloc_uar_params {</div><div class='ctx'> 	u16 uarn;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-void efa_com_set_dma_addr(dma_addr_t addr, u32 *addr_high, u32 *addr_low);</div><div class='add'>+struct efa_com_get_stats_params {</div><div class='add'>+	/* see enum efa_admin_get_stats_type */</div><div class='add'>+	u8 type;</div><div class='add'>+	/* see enum efa_admin_get_stats_scope */</div><div class='add'>+	u8 scope;</div><div class='add'>+	u16 scope_modifier;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_com_basic_stats {</div><div class='add'>+	u64 tx_bytes;</div><div class='add'>+	u64 tx_pkts;</div><div class='add'>+	u64 rx_bytes;</div><div class='add'>+	u64 rx_pkts;</div><div class='add'>+	u64 rx_drops;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_com_messages_stats {</div><div class='add'>+	u64 send_bytes;</div><div class='add'>+	u64 send_wrs;</div><div class='add'>+	u64 recv_bytes;</div><div class='add'>+	u64 recv_wrs;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_com_rdma_read_stats {</div><div class='add'>+	u64 read_wrs;</div><div class='add'>+	u64 read_bytes;</div><div class='add'>+	u64 read_wr_err;</div><div class='add'>+	u64 read_resp_bytes;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_com_rdma_write_stats {</div><div class='add'>+	u64 write_wrs;</div><div class='add'>+	u64 write_bytes;</div><div class='add'>+	u64 write_wr_err;</div><div class='add'>+	u64 write_recv_bytes;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_com_network_stats {</div><div class='add'>+	u64 retrans_bytes;</div><div class='add'>+	u64 retrans_pkts;</div><div class='add'>+	u64 retrans_timeout_events;</div><div class='add'>+	u64 unresponsive_remote_events;</div><div class='add'>+	u64 impaired_remote_conn_events;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+union efa_com_get_stats_result {</div><div class='add'>+	struct efa_com_basic_stats basic_stats;</div><div class='add'>+	struct efa_com_messages_stats messages_stats;</div><div class='add'>+	struct efa_com_rdma_read_stats rdma_read_stats;</div><div class='add'>+	struct efa_com_rdma_write_stats rdma_write_stats;</div><div class='add'>+	struct efa_com_network_stats network_stats;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> int efa_com_create_qp(struct efa_com_dev *edev,</div><div class='ctx'> 		      struct efa_com_create_qp_params *params,</div><div class='ctx'> 		      struct efa_com_create_qp_result *res);</div><div class='hunk'>@@ -251,12 +324,19 @@ int efa_com_create_ah(struct efa_com_dev *edev,</div><div class='ctx'> 		      struct efa_com_create_ah_result *result);</div><div class='ctx'> int efa_com_destroy_ah(struct efa_com_dev *edev,</div><div class='ctx'> 		       struct efa_com_destroy_ah_params *params);</div><div class='del'>-int efa_com_get_network_attr(struct efa_com_dev *edev,</div><div class='del'>-			     struct efa_com_get_network_attr_result *result);</div><div class='ctx'> int efa_com_get_device_attr(struct efa_com_dev *edev,</div><div class='ctx'> 			    struct efa_com_get_device_attr_result *result);</div><div class='ctx'> int efa_com_get_hw_hints(struct efa_com_dev *edev,</div><div class='ctx'> 			 struct efa_com_get_hw_hints_result *result);</div><div class='add'>+bool</div><div class='add'>+efa_com_check_supported_feature_id(struct efa_com_dev *edev,</div><div class='add'>+				   enum efa_admin_aq_feature_id feature_id);</div><div class='add'>+int efa_com_set_feature_ex(struct efa_com_dev *edev,</div><div class='add'>+			   struct efa_admin_set_feature_resp *set_resp,</div><div class='add'>+			   struct efa_admin_set_feature_cmd *set_cmd,</div><div class='add'>+			   enum efa_admin_aq_feature_id feature_id,</div><div class='add'>+			   dma_addr_t control_buf_dma_addr,</div><div class='add'>+			   u32 control_buff_size);</div><div class='ctx'> int efa_com_set_aenq_config(struct efa_com_dev *edev, u32 groups);</div><div class='ctx'> int efa_com_alloc_pd(struct efa_com_dev *edev,</div><div class='ctx'> 		     struct efa_com_alloc_pd_result *result);</div><div class='hunk'>@@ -266,5 +346,8 @@ int efa_com_alloc_uar(struct efa_com_dev *edev,</div><div class='ctx'> 		      struct efa_com_alloc_uar_result *result);</div><div class='ctx'> int efa_com_dealloc_uar(struct efa_com_dev *edev,</div><div class='ctx'> 			struct efa_com_dealloc_uar_params *params);</div><div class='add'>+int efa_com_get_stats(struct efa_com_dev *edev,</div><div class='add'>+		      struct efa_com_get_stats_params *params,</div><div class='add'>+		      union efa_com_get_stats_result *result);</div><div class='ctx'> </div><div class='ctx'> #endif /* _EFA_COM_CMD_H_ */</div><div class='head'>diff --git a/drivers/infiniband/hw/efa/efa_common_defs.h b/drivers/infiniband/hw/efa/efa_common_defs.h<br/>index c559ec08898e..90af1c82c9c6 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_common_defs.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_common_defs.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_common_defs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/efa/efa_common_defs.h</a></div><div class='hunk'>@@ -1,14 +1,25 @@</div><div class='ctx'> /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */</div><div class='ctx'> /*</div><div class='del'>- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='add'>+ * Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #ifndef _EFA_COMMON_H_</div><div class='ctx'> #define _EFA_COMMON_H_</div><div class='ctx'> </div><div class='add'>+#include &lt;linux/bitfield.h&gt;</div><div class='add'>+</div><div class='ctx'> #define EFA_COMMON_SPEC_VERSION_MAJOR        2</div><div class='ctx'> #define EFA_COMMON_SPEC_VERSION_MINOR        0</div><div class='ctx'> </div><div class='add'>+#define EFA_GET(ptr, mask) FIELD_GET(mask##_MASK, *(ptr))</div><div class='add'>+</div><div class='add'>+#define EFA_SET(ptr, mask, value)                                              \</div><div class='add'>+	({                                                                     \</div><div class='add'>+		typeof(ptr) _ptr = ptr;                                        \</div><div class='add'>+		*_ptr = (*_ptr &amp; ~(mask##_MASK)) |                             \</div><div class='add'>+			FIELD_PREP(mask##_MASK, value);                        \</div><div class='add'>+	})</div><div class='add'>+</div><div class='ctx'> struct efa_common_mem_addr {</div><div class='ctx'> 	u32 mem_addr_low;</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/hw/efa/efa_io_defs.h b/drivers/infiniband/hw/efa/efa_io_defs.h<br/>new file mode 100644<br/>index 000000000000..a4c9fd33da38<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_io_defs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/efa/efa_io_defs.h</a></div><div class='hunk'>@@ -0,0 +1,391 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */</div><div class='add'>+/*</div><div class='add'>+ * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef _EFA_IO_H_</div><div class='add'>+#define _EFA_IO_H_</div><div class='add'>+</div><div class='add'>+#define EFA_IO_TX_DESC_NUM_BUFS              2</div><div class='add'>+#define EFA_IO_TX_DESC_NUM_RDMA_BUFS         1</div><div class='add'>+#define EFA_IO_TX_DESC_INLINE_MAX_SIZE       32</div><div class='add'>+#define EFA_IO_TX_DESC_IMM_DATA_SIZE         4</div><div class='add'>+#define EFA_IO_TX_DESC_INLINE_PBL_SIZE       1</div><div class='add'>+</div><div class='add'>+enum efa_io_queue_type {</div><div class='add'>+	/* send queue (of a QP) */</div><div class='add'>+	EFA_IO_SEND_QUEUE                           = 1,</div><div class='add'>+	/* recv queue (of a QP) */</div><div class='add'>+	EFA_IO_RECV_QUEUE                           = 2,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum efa_io_send_op_type {</div><div class='add'>+	/* send message */</div><div class='add'>+	EFA_IO_SEND                                 = 0,</div><div class='add'>+	/* RDMA read */</div><div class='add'>+	EFA_IO_RDMA_READ                            = 1,</div><div class='add'>+	/* RDMA write */</div><div class='add'>+	EFA_IO_RDMA_WRITE                           = 2,</div><div class='add'>+	/* Fast MR registration */</div><div class='add'>+	EFA_IO_FAST_REG                             = 3,</div><div class='add'>+	/* Fast MR invalidation */</div><div class='add'>+	EFA_IO_FAST_INV                             = 4,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum efa_io_comp_status {</div><div class='add'>+	/* Successful completion */</div><div class='add'>+	EFA_IO_COMP_STATUS_OK                       = 0,</div><div class='add'>+	/* Flushed during QP destroy */</div><div class='add'>+	EFA_IO_COMP_STATUS_FLUSHED                  = 1,</div><div class='add'>+	/* Internal QP error */</div><div class='add'>+	EFA_IO_COMP_STATUS_LOCAL_ERROR_QP_INTERNAL_ERROR = 2,</div><div class='add'>+	/* Unsupported operation */</div><div class='add'>+	EFA_IO_COMP_STATUS_LOCAL_ERROR_UNSUPPORTED_OP = 3,</div><div class='add'>+	/* Bad AH */</div><div class='add'>+	EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_AH   = 4,</div><div class='add'>+	/* LKEY not registered or does not match IOVA */</div><div class='add'>+	EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_LKEY = 5,</div><div class='add'>+	/* Message too long */</div><div class='add'>+	EFA_IO_COMP_STATUS_LOCAL_ERROR_BAD_LENGTH   = 6,</div><div class='add'>+	/* RKEY not registered or does not match remote IOVA */</div><div class='add'>+	EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_ADDRESS = 7,</div><div class='add'>+	/* Connection was reset by remote side */</div><div class='add'>+	EFA_IO_COMP_STATUS_REMOTE_ERROR_ABORT       = 8,</div><div class='add'>+	/* Bad dest QP number (QP does not exist or is in error state) */</div><div class='add'>+	EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_DEST_QPN = 9,</div><div class='add'>+	/* Destination resource not ready (no WQEs posted on RQ) */</div><div class='add'>+	EFA_IO_COMP_STATUS_REMOTE_ERROR_RNR         = 10,</div><div class='add'>+	/* Receiver SGL too short */</div><div class='add'>+	EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_LENGTH  = 11,</div><div class='add'>+	/* Unexpected status returned by responder */</div><div class='add'>+	EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_STATUS  = 12,</div><div class='add'>+	/* Unresponsive remote - was previously responsive */</div><div class='add'>+	EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE = 13,</div><div class='add'>+	/* No valid AH at remote side (required for RDMA operations) */</div><div class='add'>+	EFA_IO_COMP_STATUS_REMOTE_ERROR_UNKNOWN_PEER = 14,</div><div class='add'>+	/* Unreachable remote - never received a response */</div><div class='add'>+	EFA_IO_COMP_STATUS_LOCAL_ERROR_UNREACH_REMOTE = 15,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum efa_io_frwr_pbl_mode {</div><div class='add'>+	EFA_IO_FRWR_INLINE_PBL                      = 0,</div><div class='add'>+	EFA_IO_FRWR_DIRECT_PBL                      = 1,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_io_tx_meta_desc {</div><div class='add'>+	/* Verbs-generated Request ID */</div><div class='add'>+	u16 req_id;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * control flags</div><div class='add'>+	 * 3:0 : op_type - enum efa_io_send_op_type</div><div class='add'>+	 * 4 : has_imm - immediate_data field carries valid</div><div class='add'>+	 *    data.</div><div class='add'>+	 * 5 : inline_msg - inline mode - inline message data</div><div class='add'>+	 *    follows this descriptor (no buffer descriptors).</div><div class='add'>+	 *    Note that it is different from immediate data</div><div class='add'>+	 * 6 : meta_extension - Extended metadata. MBZ</div><div class='add'>+	 * 7 : meta_desc - Indicates metadata descriptor.</div><div class='add'>+	 *    Must be set.</div><div class='add'>+	 */</div><div class='add'>+	u8 ctrl1;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * control flags</div><div class='add'>+	 * 0 : phase</div><div class='add'>+	 * 1 : reserved25 - MBZ</div><div class='add'>+	 * 2 : first - Indicates first descriptor in</div><div class='add'>+	 *    transaction. Must be set.</div><div class='add'>+	 * 3 : last - Indicates last descriptor in</div><div class='add'>+	 *    transaction. Must be set.</div><div class='add'>+	 * 4 : comp_req - Indicates whether completion should</div><div class='add'>+	 *    be posted, after packet is transmitted. Valid only</div><div class='add'>+	 *    for the first descriptor</div><div class='add'>+	 * 7:5 : reserved29 - MBZ</div><div class='add'>+	 */</div><div class='add'>+	u8 ctrl2;</div><div class='add'>+</div><div class='add'>+	u16 dest_qp_num;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * If inline_msg bit is set, length of inline message in bytes,</div><div class='add'>+	 * otherwise length of SGL (number of buffers).</div><div class='add'>+	 */</div><div class='add'>+	u16 length;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * immediate data: if has_imm is set, then this field is included within</div><div class='add'>+	 * Tx message and reported in remote Rx completion.</div><div class='add'>+	 */</div><div class='add'>+	u32 immediate_data;</div><div class='add'>+</div><div class='add'>+	u16 ah;</div><div class='add'>+</div><div class='add'>+	u16 reserved;</div><div class='add'>+</div><div class='add'>+	/* Queue key */</div><div class='add'>+	u32 qkey;</div><div class='add'>+</div><div class='add'>+	u8 reserved2[12];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * Tx queue buffer descriptor, for any transport type. Preceded by metadata</div><div class='add'>+ * descriptor.</div><div class='add'>+ */</div><div class='add'>+struct efa_io_tx_buf_desc {</div><div class='add'>+	/* length in bytes */</div><div class='add'>+	u32 length;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * 23:0 : lkey - local memory translation key</div><div class='add'>+	 * 31:24 : reserved - MBZ</div><div class='add'>+	 */</div><div class='add'>+	u32 lkey;</div><div class='add'>+</div><div class='add'>+	/* Buffer address bits[31:0] */</div><div class='add'>+	u32 buf_addr_lo;</div><div class='add'>+</div><div class='add'>+	/* Buffer address bits[63:32] */</div><div class='add'>+	u32 buf_addr_hi;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_io_remote_mem_addr {</div><div class='add'>+	/* length in bytes */</div><div class='add'>+	u32 length;</div><div class='add'>+</div><div class='add'>+	/* remote memory translation key */</div><div class='add'>+	u32 rkey;</div><div class='add'>+</div><div class='add'>+	/* Buffer address bits[31:0] */</div><div class='add'>+	u32 buf_addr_lo;</div><div class='add'>+</div><div class='add'>+	/* Buffer address bits[63:32] */</div><div class='add'>+	u32 buf_addr_hi;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_io_rdma_req {</div><div class='add'>+	/* Remote memory address */</div><div class='add'>+	struct efa_io_remote_mem_addr remote_mem;</div><div class='add'>+</div><div class='add'>+	/* Local memory address */</div><div class='add'>+	struct efa_io_tx_buf_desc local_mem[1];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_io_fast_mr_reg_req {</div><div class='add'>+	/* Updated local key of the MR after lkey/rkey increment */</div><div class='add'>+	u32 lkey;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * permissions</div><div class='add'>+	 * 0 : local_write_enable - Local write permissions:</div><div class='add'>+	 *    must be set for RQ buffers and buffers posted for</div><div class='add'>+	 *    RDMA Read requests</div><div class='add'>+	 * 1 : remote_write_enable - Remote write</div><div class='add'>+	 *    permissions: must be set to enable RDMA write to</div><div class='add'>+	 *    the region</div><div class='add'>+	 * 2 : remote_read_enable - Remote read permissions:</div><div class='add'>+	 *    must be set to enable RDMA read from the region</div><div class='add'>+	 * 7:3 : reserved2 - MBZ</div><div class='add'>+	 */</div><div class='add'>+	u8 permissions;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * control flags</div><div class='add'>+	 * 4:0 : phys_page_size_shift - page size is (1 &lt;&lt;</div><div class='add'>+	 *    phys_page_size_shift)</div><div class='add'>+	 * 6:5 : pbl_mode - enum efa_io_frwr_pbl_mode</div><div class='add'>+	 * 7 : reserved - MBZ</div><div class='add'>+	 */</div><div class='add'>+	u8 flags;</div><div class='add'>+</div><div class='add'>+	/* MBZ */</div><div class='add'>+	u8 reserved[2];</div><div class='add'>+</div><div class='add'>+	/* IO Virtual Address associated with this MR */</div><div class='add'>+	u64 iova;</div><div class='add'>+</div><div class='add'>+	/* Memory region length, in bytes */</div><div class='add'>+	u64 mr_length;</div><div class='add'>+</div><div class='add'>+	/* Physical Buffer List, each element is page-aligned. */</div><div class='add'>+	union {</div><div class='add'>+		/*</div><div class='add'>+		 * Inline array of physical page addresses (optimization</div><div class='add'>+		 * for short region activation).</div><div class='add'>+		 */</div><div class='add'>+		u64 inline_array[1];</div><div class='add'>+</div><div class='add'>+		/* points to PBL (Currently only direct) */</div><div class='add'>+		u64 dma_addr;</div><div class='add'>+	} pbl;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct efa_io_fast_mr_inv_req {</div><div class='add'>+	/* Local key of the MR to invalidate */</div><div class='add'>+	u32 lkey;</div><div class='add'>+</div><div class='add'>+	/* MBZ */</div><div class='add'>+	u8 reserved[28];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * Tx WQE, composed of tx meta descriptors followed by either tx buffer</div><div class='add'>+ * descriptors or inline data</div><div class='add'>+ */</div><div class='add'>+struct efa_io_tx_wqe {</div><div class='add'>+	/* TX meta */</div><div class='add'>+	struct efa_io_tx_meta_desc meta;</div><div class='add'>+</div><div class='add'>+	union {</div><div class='add'>+		/* Send buffer descriptors */</div><div class='add'>+		struct efa_io_tx_buf_desc sgl[2];</div><div class='add'>+</div><div class='add'>+		u8 inline_data[32];</div><div class='add'>+</div><div class='add'>+		/* RDMA local and remote memory addresses */</div><div class='add'>+		struct efa_io_rdma_req rdma_req;</div><div class='add'>+</div><div class='add'>+		/* Fast registration */</div><div class='add'>+		struct efa_io_fast_mr_reg_req reg_mr_req;</div><div class='add'>+</div><div class='add'>+		/* Fast invalidation */</div><div class='add'>+		struct efa_io_fast_mr_inv_req inv_mr_req;</div><div class='add'>+	} data;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * Rx buffer descriptor; RX WQE is composed of one or more RX buffer</div><div class='add'>+ * descriptors.</div><div class='add'>+ */</div><div class='add'>+struct efa_io_rx_desc {</div><div class='add'>+	/* Buffer address bits[31:0] */</div><div class='add'>+	u32 buf_addr_lo;</div><div class='add'>+</div><div class='add'>+	/* Buffer Pointer[63:32] */</div><div class='add'>+	u32 buf_addr_hi;</div><div class='add'>+</div><div class='add'>+	/* Verbs-generated request id. */</div><div class='add'>+	u16 req_id;</div><div class='add'>+</div><div class='add'>+	/* Length in bytes. */</div><div class='add'>+	u16 length;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * LKey and control flags</div><div class='add'>+	 * 23:0 : lkey</div><div class='add'>+	 * 29:24 : reserved - MBZ</div><div class='add'>+	 * 30 : first - Indicates first descriptor in WQE</div><div class='add'>+	 * 31 : last - Indicates last descriptor in WQE</div><div class='add'>+	 */</div><div class='add'>+	u32 lkey_ctrl;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* Common IO completion descriptor */</div><div class='add'>+struct efa_io_cdesc_common {</div><div class='add'>+	/*</div><div class='add'>+	 * verbs-generated request ID, as provided in the completed tx or rx</div><div class='add'>+	 * descriptor.</div><div class='add'>+	 */</div><div class='add'>+	u16 req_id;</div><div class='add'>+</div><div class='add'>+	u8 status;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * flags</div><div class='add'>+	 * 0 : phase - Phase bit</div><div class='add'>+	 * 2:1 : q_type - enum efa_io_queue_type: send/recv</div><div class='add'>+	 * 3 : has_imm - indicates that immediate data is</div><div class='add'>+	 *    present - for RX completions only</div><div class='add'>+	 * 6:4 : op_type - enum efa_io_send_op_type</div><div class='add'>+	 * 7 : unsolicited - indicates that there is no</div><div class='add'>+	 *    matching request - for RDMA with imm. RX only</div><div class='add'>+	 */</div><div class='add'>+	u8 flags;</div><div class='add'>+</div><div class='add'>+	/* local QP number */</div><div class='add'>+	u16 qp_num;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* Tx completion descriptor */</div><div class='add'>+struct efa_io_tx_cdesc {</div><div class='add'>+	/* Common completion info */</div><div class='add'>+	struct efa_io_cdesc_common common;</div><div class='add'>+</div><div class='add'>+	/* MBZ */</div><div class='add'>+	u16 reserved16;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* Rx Completion Descriptor */</div><div class='add'>+struct efa_io_rx_cdesc {</div><div class='add'>+	/* Common completion info */</div><div class='add'>+	struct efa_io_cdesc_common common;</div><div class='add'>+</div><div class='add'>+	/* Transferred length bits[15:0] */</div><div class='add'>+	u16 length;</div><div class='add'>+</div><div class='add'>+	/* Remote Address Handle FW index, 0xFFFF indicates invalid ah */</div><div class='add'>+	u16 ah;</div><div class='add'>+</div><div class='add'>+	u16 src_qp_num;</div><div class='add'>+</div><div class='add'>+	/* Immediate data */</div><div class='add'>+	u32 imm;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* Rx Completion Descriptor RDMA write info */</div><div class='add'>+struct efa_io_rx_cdesc_rdma_write {</div><div class='add'>+	/* Transferred length bits[31:16] */</div><div class='add'>+	u16 length_hi;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* Extended Rx Completion Descriptor */</div><div class='add'>+struct efa_io_rx_cdesc_ex {</div><div class='add'>+	/* Base RX completion info */</div><div class='add'>+	struct efa_io_rx_cdesc base;</div><div class='add'>+</div><div class='add'>+	union {</div><div class='add'>+		struct efa_io_rx_cdesc_rdma_write rdma_write;</div><div class='add'>+</div><div class='add'>+		/*</div><div class='add'>+		 * Valid only in case of unknown AH (0xFFFF) and CQ</div><div class='add'>+		 * set_src_addr is enabled.</div><div class='add'>+		 */</div><div class='add'>+		u8 src_addr[16];</div><div class='add'>+	} u;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* tx_meta_desc */</div><div class='add'>+#define EFA_IO_TX_META_DESC_OP_TYPE_MASK                    GENMASK(3, 0)</div><div class='add'>+#define EFA_IO_TX_META_DESC_HAS_IMM_MASK                    BIT(4)</div><div class='add'>+#define EFA_IO_TX_META_DESC_INLINE_MSG_MASK                 BIT(5)</div><div class='add'>+#define EFA_IO_TX_META_DESC_META_EXTENSION_MASK             BIT(6)</div><div class='add'>+#define EFA_IO_TX_META_DESC_META_DESC_MASK                  BIT(7)</div><div class='add'>+#define EFA_IO_TX_META_DESC_PHASE_MASK                      BIT(0)</div><div class='add'>+#define EFA_IO_TX_META_DESC_FIRST_MASK                      BIT(2)</div><div class='add'>+#define EFA_IO_TX_META_DESC_LAST_MASK                       BIT(3)</div><div class='add'>+#define EFA_IO_TX_META_DESC_COMP_REQ_MASK                   BIT(4)</div><div class='add'>+</div><div class='add'>+/* tx_buf_desc */</div><div class='add'>+#define EFA_IO_TX_BUF_DESC_LKEY_MASK                        GENMASK(23, 0)</div><div class='add'>+</div><div class='add'>+/* fast_mr_reg_req */</div><div class='add'>+#define EFA_IO_FAST_MR_REG_REQ_LOCAL_WRITE_ENABLE_MASK      BIT(0)</div><div class='add'>+#define EFA_IO_FAST_MR_REG_REQ_REMOTE_WRITE_ENABLE_MASK     BIT(1)</div><div class='add'>+#define EFA_IO_FAST_MR_REG_REQ_REMOTE_READ_ENABLE_MASK      BIT(2)</div><div class='add'>+#define EFA_IO_FAST_MR_REG_REQ_PHYS_PAGE_SIZE_SHIFT_MASK    GENMASK(4, 0)</div><div class='add'>+#define EFA_IO_FAST_MR_REG_REQ_PBL_MODE_MASK                GENMASK(6, 5)</div><div class='add'>+</div><div class='add'>+/* rx_desc */</div><div class='add'>+#define EFA_IO_RX_DESC_LKEY_MASK                            GENMASK(23, 0)</div><div class='add'>+#define EFA_IO_RX_DESC_FIRST_MASK                           BIT(30)</div><div class='add'>+#define EFA_IO_RX_DESC_LAST_MASK                            BIT(31)</div><div class='add'>+</div><div class='add'>+/* cdesc_common */</div><div class='add'>+#define EFA_IO_CDESC_COMMON_PHASE_MASK                      BIT(0)</div><div class='add'>+#define EFA_IO_CDESC_COMMON_Q_TYPE_MASK                     GENMASK(2, 1)</div><div class='add'>+#define EFA_IO_CDESC_COMMON_HAS_IMM_MASK                    BIT(3)</div><div class='add'>+#define EFA_IO_CDESC_COMMON_OP_TYPE_MASK                    GENMASK(6, 4)</div><div class='add'>+#define EFA_IO_CDESC_COMMON_UNSOLICITED_MASK                BIT(7)</div><div class='add'>+</div><div class='add'>+#endif /* _EFA_IO_H_ */</div><div class='head'>diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c<br/>index db974caf1eb1..6c415b9adb5f 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_main.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_main.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_main.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/efa/efa_main.c</a></div><div class='hunk'>@@ -1,19 +1,28 @@</div><div class='ctx'> // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause</div><div class='ctx'> /*</div><div class='del'>- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='add'>+ * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/module.h&gt;</div><div class='ctx'> #include &lt;linux/pci.h&gt;</div><div class='add'>+#include &lt;linux/utsname.h&gt;</div><div class='add'>+#include &lt;linux/version.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include &lt;rdma/ib_user_verbs.h&gt;</div><div class='add'>+#include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "efa.h"</div><div class='ctx'> </div><div class='del'>-#define PCI_DEV_ID_EFA_VF 0xefa0</div><div class='add'>+#define PCI_DEV_ID_EFA0_VF 0xefa0</div><div class='add'>+#define PCI_DEV_ID_EFA1_VF 0xefa1</div><div class='add'>+#define PCI_DEV_ID_EFA2_VF 0xefa2</div><div class='add'>+#define PCI_DEV_ID_EFA3_VF 0xefa3</div><div class='ctx'> </div><div class='ctx'> static const struct pci_device_id efa_pci_tbl[] = {</div><div class='del'>-	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA_VF) },</div><div class='add'>+	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA0_VF) },</div><div class='add'>+	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA1_VF) },</div><div class='add'>+	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA2_VF) },</div><div class='add'>+	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA3_VF) },</div><div class='ctx'> 	{ }</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -30,14 +39,7 @@ MODULE_DEVICE_TABLE(pci, efa_pci_tbl);</div><div class='ctx'> 	(BIT(EFA_ADMIN_FATAL_ERROR) | BIT(EFA_ADMIN_WARNING) | \</div><div class='ctx'> 	 BIT(EFA_ADMIN_NOTIFICATION) | BIT(EFA_ADMIN_KEEP_ALIVE))</div><div class='ctx'> </div><div class='del'>-static void efa_update_network_attr(struct efa_dev *dev,</div><div class='del'>-				    struct efa_com_get_network_attr_result *network_attr)</div><div class='del'>-{</div><div class='del'>-	memcpy(dev-&gt;addr, network_attr-&gt;addr, sizeof(network_attr-&gt;addr));</div><div class='del'>-	dev-&gt;mtu = network_attr-&gt;mtu;</div><div class='del'>-</div><div class='del'>-	dev_dbg(&amp;dev-&gt;pdev-&gt;dev, "Full address %pI6\n", dev-&gt;addr);</div><div class='del'>-}</div><div class='add'>+extern const struct uapi_definition efa_uapi_defs[];</div><div class='ctx'> </div><div class='ctx'> /* This handler will called for unknown event group or unimplemented handlers */</div><div class='ctx'> static void unimplemented_aenq_handler(void *data,</div><div class='hunk'>@@ -72,6 +74,47 @@ static void efa_release_bars(struct efa_dev *dev, int bars_mask)</div><div class='ctx'> 	pci_release_selected_regions(pdev, release_bars);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void efa_process_comp_eqe(struct efa_dev *dev, struct efa_admin_eqe *eqe)</div><div class='add'>+{</div><div class='add'>+	u16 cqn = eqe-&gt;u.comp_event.cqn;</div><div class='add'>+	struct efa_cq *cq;</div><div class='add'>+</div><div class='add'>+	/* Safe to load as we're in irq and removal calls synchronize_irq() */</div><div class='add'>+	cq = xa_load(&amp;dev-&gt;cqs_xa, cqn);</div><div class='add'>+	if (unlikely(!cq)) {</div><div class='add'>+		ibdev_err_ratelimited(&amp;dev-&gt;ibdev,</div><div class='add'>+				      "Completion event on non-existent CQ[%u]",</div><div class='add'>+				      cqn);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cq-&gt;ibcq.comp_handler(&amp;cq-&gt;ibcq, cq-&gt;ibcq.cq_context);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void efa_process_eqe(struct efa_com_eq *eeq, struct efa_admin_eqe *eqe)</div><div class='add'>+{</div><div class='add'>+	struct efa_dev *dev = container_of(eeq-&gt;edev, struct efa_dev, edev);</div><div class='add'>+</div><div class='add'>+	if (likely(EFA_GET(&amp;eqe-&gt;common, EFA_ADMIN_EQE_EVENT_TYPE) ==</div><div class='add'>+			   EFA_ADMIN_EQE_EVENT_TYPE_COMPLETION))</div><div class='add'>+		efa_process_comp_eqe(dev, eqe);</div><div class='add'>+	else</div><div class='add'>+		ibdev_err_ratelimited(&amp;dev-&gt;ibdev,</div><div class='add'>+				      "Unknown event type received %lu",</div><div class='add'>+				      EFA_GET(&amp;eqe-&gt;common,</div><div class='add'>+					      EFA_ADMIN_EQE_EVENT_TYPE));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static irqreturn_t efa_intr_msix_comp(int irq, void *data)</div><div class='add'>+{</div><div class='add'>+	struct efa_eq *eq = data;</div><div class='add'>+	struct efa_com_dev *edev = eq-&gt;eeq.edev;</div><div class='add'>+</div><div class='add'>+	efa_com_eq_comp_intr_handler(edev, &amp;eq-&gt;eeq);</div><div class='add'>+</div><div class='add'>+	return IRQ_HANDLED;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static irqreturn_t efa_intr_msix_mgmnt(int irq, void *data)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_dev *dev = data;</div><div class='hunk'>@@ -82,25 +125,40 @@ static irqreturn_t efa_intr_msix_mgmnt(int irq, void *data)</div><div class='ctx'> 	return IRQ_HANDLED;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int efa_request_mgmnt_irq(struct efa_dev *dev)</div><div class='add'>+static int efa_request_irq(struct efa_dev *dev, struct efa_irq *irq)</div><div class='ctx'> {</div><div class='del'>-	struct efa_irq *irq;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	irq = &amp;dev-&gt;admin_irq;</div><div class='del'>-	err = request_irq(irq-&gt;vector, irq-&gt;handler, 0, irq-&gt;name,</div><div class='del'>-			  irq-&gt;data);</div><div class='add'>+	err = request_irq(irq-&gt;irqn, irq-&gt;handler, 0, irq-&gt;name, irq-&gt;data);</div><div class='ctx'> 	if (err) {</div><div class='del'>-		dev_err(&amp;dev-&gt;pdev-&gt;dev, "Failed to request admin irq (%d)\n",</div><div class='del'>-			err);</div><div class='add'>+		dev_err(&amp;dev-&gt;pdev-&gt;dev, "Failed to request irq %s (%d)\n",</div><div class='add'>+			irq-&gt;name, err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	dev_dbg(&amp;dev-&gt;pdev-&gt;dev, "Set affinity hint of mgmnt irq to %*pbl (irq vector: %d)\n",</div><div class='del'>-		nr_cpumask_bits, &amp;irq-&gt;affinity_hint_mask, irq-&gt;vector);</div><div class='del'>-	irq_set_affinity_hint(irq-&gt;vector, &amp;irq-&gt;affinity_hint_mask);</div><div class='add'>+	irq_set_affinity_hint(irq-&gt;irqn, &amp;irq-&gt;affinity_hint_mask);</div><div class='ctx'> </div><div class='del'>-	return err;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void efa_setup_comp_irq(struct efa_dev *dev, struct efa_eq *eq, u32 vector)</div><div class='add'>+{</div><div class='add'>+	u32 cpu;</div><div class='add'>+</div><div class='add'>+	cpu = vector - EFA_COMP_EQS_VEC_BASE;</div><div class='add'>+	snprintf(eq-&gt;irq.name, EFA_IRQNAME_SIZE, "efa-comp%d@pci:%s", cpu,</div><div class='add'>+		 pci_name(dev-&gt;pdev));</div><div class='add'>+	eq-&gt;irq.handler = efa_intr_msix_comp;</div><div class='add'>+	eq-&gt;irq.data = eq;</div><div class='add'>+	eq-&gt;irq.vector = vector;</div><div class='add'>+	eq-&gt;irq.irqn = pci_irq_vector(dev-&gt;pdev, vector);</div><div class='add'>+	cpumask_set_cpu(cpu, &amp;eq-&gt;irq.affinity_hint_mask);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void efa_free_irq(struct efa_dev *dev, struct efa_irq *irq)</div><div class='add'>+{</div><div class='add'>+	irq_set_affinity_hint(irq-&gt;irqn, NULL);</div><div class='add'>+	free_irq(irq-&gt;irqn, irq-&gt;data);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void efa_setup_mgmnt_irq(struct efa_dev *dev)</div><div class='hunk'>@@ -111,47 +169,45 @@ static void efa_setup_mgmnt_irq(struct efa_dev *dev)</div><div class='ctx'> 		 "efa-mgmnt@pci:%s", pci_name(dev-&gt;pdev));</div><div class='ctx'> 	dev-&gt;admin_irq.handler = efa_intr_msix_mgmnt;</div><div class='ctx'> 	dev-&gt;admin_irq.data = dev;</div><div class='del'>-	dev-&gt;admin_irq.vector =</div><div class='del'>-		pci_irq_vector(dev-&gt;pdev, dev-&gt;admin_msix_vector_idx);</div><div class='add'>+	dev-&gt;admin_irq.vector = dev-&gt;admin_msix_vector_idx;</div><div class='add'>+	dev-&gt;admin_irq.irqn = pci_irq_vector(dev-&gt;pdev,</div><div class='add'>+					     dev-&gt;admin_msix_vector_idx);</div><div class='ctx'> 	cpu = cpumask_first(cpu_online_mask);</div><div class='del'>-	dev-&gt;admin_irq.cpu = cpu;</div><div class='ctx'> 	cpumask_set_cpu(cpu,</div><div class='ctx'> 			&amp;dev-&gt;admin_irq.affinity_hint_mask);</div><div class='del'>-	dev_info(&amp;dev-&gt;pdev-&gt;dev, "Setup irq:0x%p vector:%d name:%s\n",</div><div class='del'>-		 &amp;dev-&gt;admin_irq,</div><div class='del'>-		 dev-&gt;admin_irq.vector,</div><div class='add'>+	dev_info(&amp;dev-&gt;pdev-&gt;dev, "Setup irq:%d name:%s\n",</div><div class='add'>+		 dev-&gt;admin_irq.irqn,</div><div class='ctx'> 		 dev-&gt;admin_irq.name);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void efa_free_mgmnt_irq(struct efa_dev *dev)</div><div class='del'>-{</div><div class='del'>-	struct efa_irq *irq;</div><div class='del'>-</div><div class='del'>-	irq = &amp;dev-&gt;admin_irq;</div><div class='del'>-	irq_set_affinity_hint(irq-&gt;vector, NULL);</div><div class='del'>-	free_irq(irq-&gt;vector, irq-&gt;data);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int efa_set_mgmnt_irq(struct efa_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	efa_setup_mgmnt_irq(dev);</div><div class='ctx'> </div><div class='del'>-	return efa_request_mgmnt_irq(dev);</div><div class='add'>+	return efa_request_irq(dev, &amp;dev-&gt;admin_irq);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int efa_request_doorbell_bar(struct efa_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	u8 db_bar_idx = dev-&gt;dev_attr.db_bar;</div><div class='ctx'> 	struct pci_dev *pdev = dev-&gt;pdev;</div><div class='del'>-	int bars;</div><div class='add'>+	int pci_mem_bars;</div><div class='add'>+	int db_bar;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	if (!(BIT(db_bar_idx) &amp; EFA_BASE_BAR_MASK)) {</div><div class='del'>-		bars = pci_select_bars(pdev, IORESOURCE_MEM) &amp; BIT(db_bar_idx);</div><div class='add'>+	db_bar = BIT(db_bar_idx);</div><div class='add'>+	if (!(db_bar &amp; EFA_BASE_BAR_MASK)) {</div><div class='add'>+		pci_mem_bars = pci_select_bars(pdev, IORESOURCE_MEM);</div><div class='add'>+		if (db_bar &amp; ~pci_mem_bars) {</div><div class='add'>+			dev_err(&amp;pdev-&gt;dev,</div><div class='add'>+				"Doorbells BAR unavailable. Requested %#x, available %#x\n",</div><div class='add'>+				db_bar, pci_mem_bars);</div><div class='add'>+			return -ENODEV;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='del'>-		err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);</div><div class='add'>+		err = pci_request_selected_regions(pdev, db_bar, DRV_MODULE_NAME);</div><div class='ctx'> 		if (err) {</div><div class='del'>-			dev_err(&amp;dev-&gt;pdev-&gt;dev,</div><div class='add'>+			dev_err(&amp;pdev-&gt;dev,</div><div class='ctx'> 				"pci_request_selected_regions for bar %d failed %d\n",</div><div class='ctx'> 				db_bar_idx, err);</div><div class='ctx'> 			return err;</div><div class='hunk'>@@ -196,21 +252,140 @@ static void efa_stats_init(struct efa_dev *dev)</div><div class='ctx'> 		atomic64_set(s, 0);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void efa_set_host_info(struct efa_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct efa_admin_set_feature_resp resp = {};</div><div class='add'>+	struct efa_admin_set_feature_cmd cmd = {};</div><div class='add'>+	struct efa_admin_host_info *hinf;</div><div class='add'>+	u32 bufsz = sizeof(*hinf);</div><div class='add'>+	dma_addr_t hinf_dma;</div><div class='add'>+</div><div class='add'>+	if (!efa_com_check_supported_feature_id(&amp;dev-&gt;edev,</div><div class='add'>+						EFA_ADMIN_HOST_INFO))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	/* Failures in host info set shall not disturb probe */</div><div class='add'>+	hinf = dma_alloc_coherent(&amp;dev-&gt;pdev-&gt;dev, bufsz, &amp;hinf_dma,</div><div class='add'>+				  GFP_KERNEL);</div><div class='add'>+	if (!hinf)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	strscpy(hinf-&gt;os_dist_str, utsname()-&gt;release,</div><div class='add'>+		sizeof(hinf-&gt;os_dist_str));</div><div class='add'>+	hinf-&gt;os_type = EFA_ADMIN_OS_LINUX;</div><div class='add'>+	strscpy(hinf-&gt;kernel_ver_str, utsname()-&gt;version,</div><div class='add'>+		sizeof(hinf-&gt;kernel_ver_str));</div><div class='add'>+	hinf-&gt;kernel_ver = LINUX_VERSION_CODE;</div><div class='add'>+	EFA_SET(&amp;hinf-&gt;driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MAJOR, 0);</div><div class='add'>+	EFA_SET(&amp;hinf-&gt;driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MINOR, 0);</div><div class='add'>+	EFA_SET(&amp;hinf-&gt;driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_SUB_MINOR, 0);</div><div class='add'>+	EFA_SET(&amp;hinf-&gt;driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MODULE_TYPE, 0);</div><div class='add'>+	EFA_SET(&amp;hinf-&gt;bdf, EFA_ADMIN_HOST_INFO_BUS, dev-&gt;pdev-&gt;bus-&gt;number);</div><div class='add'>+	EFA_SET(&amp;hinf-&gt;bdf, EFA_ADMIN_HOST_INFO_DEVICE,</div><div class='add'>+		PCI_SLOT(dev-&gt;pdev-&gt;devfn));</div><div class='add'>+	EFA_SET(&amp;hinf-&gt;bdf, EFA_ADMIN_HOST_INFO_FUNCTION,</div><div class='add'>+		PCI_FUNC(dev-&gt;pdev-&gt;devfn));</div><div class='add'>+	EFA_SET(&amp;hinf-&gt;spec_ver, EFA_ADMIN_HOST_INFO_SPEC_MAJOR,</div><div class='add'>+		EFA_COMMON_SPEC_VERSION_MAJOR);</div><div class='add'>+	EFA_SET(&amp;hinf-&gt;spec_ver, EFA_ADMIN_HOST_INFO_SPEC_MINOR,</div><div class='add'>+		EFA_COMMON_SPEC_VERSION_MINOR);</div><div class='add'>+	EFA_SET(&amp;hinf-&gt;flags, EFA_ADMIN_HOST_INFO_INTREE, 1);</div><div class='add'>+	EFA_SET(&amp;hinf-&gt;flags, EFA_ADMIN_HOST_INFO_GDR, 0);</div><div class='add'>+</div><div class='add'>+	efa_com_set_feature_ex(&amp;dev-&gt;edev, &amp;resp, &amp;cmd, EFA_ADMIN_HOST_INFO,</div><div class='add'>+			       hinf_dma, bufsz);</div><div class='add'>+</div><div class='add'>+	dma_free_coherent(&amp;dev-&gt;pdev-&gt;dev, bufsz, hinf, hinf_dma);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void efa_destroy_eq(struct efa_dev *dev, struct efa_eq *eq)</div><div class='add'>+{</div><div class='add'>+	efa_com_eq_destroy(&amp;dev-&gt;edev, &amp;eq-&gt;eeq);</div><div class='add'>+	efa_free_irq(dev, &amp;eq-&gt;irq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int efa_create_eq(struct efa_dev *dev, struct efa_eq *eq, u32 msix_vec)</div><div class='add'>+{</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	efa_setup_comp_irq(dev, eq, msix_vec);</div><div class='add'>+	err = efa_request_irq(dev, &amp;eq-&gt;irq);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = efa_com_eq_init(&amp;dev-&gt;edev, &amp;eq-&gt;eeq, efa_process_eqe,</div><div class='add'>+			      dev-&gt;dev_attr.max_eq_depth, msix_vec);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_free_comp_irq;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_free_comp_irq:</div><div class='add'>+	efa_free_irq(dev, &amp;eq-&gt;irq);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int efa_create_eqs(struct efa_dev *dev)</div><div class='add'>+{</div><div class='add'>+	u32 neqs = dev-&gt;dev_attr.max_eq;</div><div class='add'>+	int err, i;</div><div class='add'>+</div><div class='add'>+	neqs = min_t(u32, neqs, dev-&gt;num_irq_vectors - EFA_COMP_EQS_VEC_BASE);</div><div class='add'>+	dev-&gt;neqs = neqs;</div><div class='add'>+	dev-&gt;eqs = kcalloc(neqs, sizeof(*dev-&gt;eqs), GFP_KERNEL);</div><div class='add'>+	if (!dev-&gt;eqs)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; neqs; i++) {</div><div class='add'>+		err = efa_create_eq(dev, &amp;dev-&gt;eqs[i], i + EFA_COMP_EQS_VEC_BASE);</div><div class='add'>+		if (err)</div><div class='add'>+			goto err_destroy_eqs;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_destroy_eqs:</div><div class='add'>+	for (i--; i &gt;= 0; i--)</div><div class='add'>+		efa_destroy_eq(dev, &amp;dev-&gt;eqs[i]);</div><div class='add'>+	kfree(dev-&gt;eqs);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void efa_destroy_eqs(struct efa_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; dev-&gt;neqs; i++)</div><div class='add'>+		efa_destroy_eq(dev, &amp;dev-&gt;eqs[i]);</div><div class='add'>+</div><div class='add'>+	kfree(dev-&gt;eqs);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static const struct ib_device_ops efa_dev_ops = {</div><div class='add'>+	.owner = THIS_MODULE,</div><div class='add'>+	.driver_id = RDMA_DRIVER_EFA,</div><div class='add'>+	.uverbs_abi_ver = EFA_UVERBS_ABI_VERSION,</div><div class='add'>+</div><div class='add'>+	.alloc_hw_port_stats = efa_alloc_hw_port_stats,</div><div class='add'>+	.alloc_hw_device_stats = efa_alloc_hw_device_stats,</div><div class='ctx'> 	.alloc_pd = efa_alloc_pd,</div><div class='ctx'> 	.alloc_ucontext = efa_alloc_ucontext,</div><div class='del'>-	.create_ah = efa_create_ah,</div><div class='ctx'> 	.create_cq = efa_create_cq,</div><div class='add'>+	.create_cq_umem = efa_create_cq_umem,</div><div class='ctx'> 	.create_qp = efa_create_qp,</div><div class='add'>+	.create_user_ah = efa_create_ah,</div><div class='ctx'> 	.dealloc_pd = efa_dealloc_pd,</div><div class='ctx'> 	.dealloc_ucontext = efa_dealloc_ucontext,</div><div class='ctx'> 	.dereg_mr = efa_dereg_mr,</div><div class='ctx'> 	.destroy_ah = efa_destroy_ah,</div><div class='ctx'> 	.destroy_cq = efa_destroy_cq,</div><div class='ctx'> 	.destroy_qp = efa_destroy_qp,</div><div class='add'>+	.get_hw_stats = efa_get_hw_stats,</div><div class='ctx'> 	.get_link_layer = efa_port_link_layer,</div><div class='ctx'> 	.get_port_immutable = efa_get_port_immutable,</div><div class='ctx'> 	.mmap = efa_mmap,</div><div class='add'>+	.mmap_free = efa_mmap_free,</div><div class='ctx'> 	.modify_qp = efa_modify_qp,</div><div class='ctx'> 	.query_device = efa_query_device,</div><div class='ctx'> 	.query_gid = efa_query_gid,</div><div class='hunk'>@@ -218,15 +393,17 @@ static const struct ib_device_ops efa_dev_ops = {</div><div class='ctx'> 	.query_port = efa_query_port,</div><div class='ctx'> 	.query_qp = efa_query_qp,</div><div class='ctx'> 	.reg_user_mr = efa_reg_mr,</div><div class='add'>+	.reg_user_mr_dmabuf = efa_reg_user_mr_dmabuf,</div><div class='ctx'> </div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_ah, efa_ah, ibah),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_cq, efa_cq, ibcq),</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_pd, efa_pd, ibpd),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_qp, efa_qp, ibqp),</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_ucontext, efa_ucontext, ibucontext),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static int efa_ib_device_add(struct efa_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	struct efa_com_get_network_attr_result network_attr;</div><div class='ctx'> 	struct efa_com_get_hw_hints_result hw_hints;</div><div class='ctx'> 	struct pci_dev *pdev = dev-&gt;pdev;</div><div class='ctx'> 	int err;</div><div class='hunk'>@@ -242,12 +419,6 @@ static int efa_ib_device_add(struct efa_dev *dev)</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='del'>-	err = efa_com_get_network_attr(&amp;dev-&gt;edev, &amp;network_attr);</div><div class='del'>-	if (err)</div><div class='del'>-		goto err_release_doorbell_bar;</div><div class='del'>-</div><div class='del'>-	efa_update_network_attr(dev, &amp;network_attr);</div><div class='del'>-</div><div class='ctx'> 	err = efa_com_get_hw_hints(&amp;dev-&gt;edev, &amp;hw_hints);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_release_doorbell_bar;</div><div class='hunk'>@@ -259,45 +430,32 @@ static int efa_ib_device_add(struct efa_dev *dev)</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_release_doorbell_bar;</div><div class='ctx'> </div><div class='del'>-	dev-&gt;ibdev.owner = THIS_MODULE;</div><div class='add'>+	err = efa_create_eqs(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_release_doorbell_bar;</div><div class='add'>+</div><div class='add'>+	efa_set_host_info(dev);</div><div class='add'>+</div><div class='ctx'> 	dev-&gt;ibdev.node_type = RDMA_NODE_UNSPECIFIED;</div><div class='add'>+	dev-&gt;ibdev.node_guid = dev-&gt;dev_attr.guid;</div><div class='ctx'> 	dev-&gt;ibdev.phys_port_cnt = 1;</div><div class='del'>-	dev-&gt;ibdev.num_comp_vectors = 1;</div><div class='add'>+	dev-&gt;ibdev.num_comp_vectors = dev-&gt;neqs ?: 1;</div><div class='ctx'> 	dev-&gt;ibdev.dev.parent = &amp;pdev-&gt;dev;</div><div class='del'>-	dev-&gt;ibdev.uverbs_abi_ver = EFA_UVERBS_ABI_VERSION;</div><div class='del'>-</div><div class='del'>-	dev-&gt;ibdev.uverbs_cmd_mask =</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_GET_CONTEXT) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_DEVICE) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_PORT) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_ALLOC_PD) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DEALLOC_PD) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_REG_MR) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DEREG_MR) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_CQ) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_CQ) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_QP) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_MODIFY_QP) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_QP) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_QP) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_AH) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_AH);</div><div class='del'>-</div><div class='del'>-	dev-&gt;ibdev.uverbs_ex_cmd_mask =</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_QUERY_DEVICE);</div><div class='del'>-</div><div class='del'>-	dev-&gt;ibdev.driver_id = RDMA_DRIVER_EFA;</div><div class='add'>+</div><div class='ctx'> 	ib_set_device_ops(&amp;dev-&gt;ibdev, &amp;efa_dev_ops);</div><div class='ctx'> </div><div class='del'>-	err = ib_register_device(&amp;dev-&gt;ibdev, "efa_%d");</div><div class='add'>+	dev-&gt;ibdev.driver_def = efa_uapi_defs;</div><div class='add'>+</div><div class='add'>+	err = ib_register_device(&amp;dev-&gt;ibdev, "efa_%d", &amp;pdev-&gt;dev);</div><div class='ctx'> 	if (err)</div><div class='del'>-		goto err_release_doorbell_bar;</div><div class='add'>+		goto err_destroy_eqs;</div><div class='ctx'> </div><div class='ctx'> 	ibdev_info(&amp;dev-&gt;ibdev, "IB device registered\n");</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='add'>+err_destroy_eqs:</div><div class='add'>+	efa_destroy_eqs(dev);</div><div class='ctx'> err_release_doorbell_bar:</div><div class='ctx'> 	efa_release_doorbell_bar(dev);</div><div class='ctx'> 	return err;</div><div class='hunk'>@@ -305,9 +463,9 @@ err_release_doorbell_bar:</div><div class='ctx'> </div><div class='ctx'> static void efa_ib_device_remove(struct efa_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	efa_com_dev_reset(&amp;dev-&gt;edev, EFA_REGS_RESET_NORMAL);</div><div class='ctx'> 	ibdev_info(&amp;dev-&gt;ibdev, "Unregister ib device\n");</div><div class='ctx'> 	ib_unregister_device(&amp;dev-&gt;ibdev);</div><div class='add'>+	efa_destroy_eqs(dev);</div><div class='ctx'> 	efa_release_doorbell_bar(dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -318,29 +476,30 @@ static void efa_disable_msix(struct efa_dev *dev)</div><div class='ctx'> </div><div class='ctx'> static int efa_enable_msix(struct efa_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	int msix_vecs, irq_num;</div><div class='del'>-</div><div class='del'>-	/* Reserve the max msix vectors we might need */</div><div class='del'>-	msix_vecs = EFA_NUM_MSIX_VEC;</div><div class='add'>+	int max_vecs, num_vecs;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Reserve the max msix vectors we might need, one vector is reserved</div><div class='add'>+	 * for admin.</div><div class='add'>+	 */</div><div class='add'>+	max_vecs = min_t(int, pci_msix_vec_count(dev-&gt;pdev),</div><div class='add'>+			 num_online_cpus() + 1);</div><div class='ctx'> 	dev_dbg(&amp;dev-&gt;pdev-&gt;dev, "Trying to enable MSI-X, vectors %d\n",</div><div class='del'>-		msix_vecs);</div><div class='add'>+		max_vecs);</div><div class='ctx'> </div><div class='ctx'> 	dev-&gt;admin_msix_vector_idx = EFA_MGMNT_MSIX_VEC_IDX;</div><div class='del'>-	irq_num = pci_alloc_irq_vectors(dev-&gt;pdev, msix_vecs,</div><div class='del'>-					msix_vecs, PCI_IRQ_MSIX);</div><div class='add'>+	num_vecs = pci_alloc_irq_vectors(dev-&gt;pdev, 1,</div><div class='add'>+					 max_vecs, PCI_IRQ_MSIX);</div><div class='ctx'> </div><div class='del'>-	if (irq_num &lt; 0) {</div><div class='del'>-		dev_err(&amp;dev-&gt;pdev-&gt;dev, "Failed to enable MSI-X. irq_num %d\n",</div><div class='del'>-			irq_num);</div><div class='add'>+	if (num_vecs &lt; 0) {</div><div class='add'>+		dev_err(&amp;dev-&gt;pdev-&gt;dev, "Failed to enable MSI-X. error %d\n",</div><div class='add'>+			num_vecs);</div><div class='ctx'> 		return -ENOSPC;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (irq_num != msix_vecs) {</div><div class='del'>-		dev_err(&amp;dev-&gt;pdev-&gt;dev,</div><div class='del'>-			"Allocated %d MSI-X (out of %d requested)\n",</div><div class='del'>-			irq_num, msix_vecs);</div><div class='del'>-		return -ENOSPC;</div><div class='del'>-	}</div><div class='add'>+	dev_dbg(&amp;dev-&gt;pdev-&gt;dev, "Allocated %d MSI-X vectors\n", num_vecs);</div><div class='add'>+</div><div class='add'>+	dev-&gt;num_irq_vectors = num_vecs;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -364,20 +523,13 @@ static int efa_device_init(struct efa_com_dev *edev, struct pci_dev *pdev)</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(dma_width));</div><div class='del'>-	if (err) {</div><div class='del'>-		dev_err(&amp;pdev-&gt;dev, "pci_set_dma_mask failed %d\n", err);</div><div class='del'>-		return err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(dma_width));</div><div class='add'>+	err = dma_set_mask_and_coherent(&amp;pdev-&gt;dev, DMA_BIT_MASK(dma_width));</div><div class='ctx'> 	if (err) {</div><div class='del'>-		dev_err(&amp;pdev-&gt;dev,</div><div class='del'>-			"err_pci_set_consistent_dma_mask failed %d\n",</div><div class='del'>-			err);</div><div class='add'>+		dev_err(&amp;pdev-&gt;dev, "dma_set_mask_and_coherent failed %d\n", err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	dma_set_max_seg_size(&amp;pdev-&gt;dev, UINT_MAX);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -385,7 +537,7 @@ static struct efa_dev *efa_probe_device(struct pci_dev *pdev)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_com_dev *edev;</div><div class='ctx'> 	struct efa_dev *dev;</div><div class='del'>-	int bars;</div><div class='add'>+	int pci_mem_bars;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	err = pci_enable_device_mem(pdev);</div><div class='hunk'>@@ -408,9 +560,16 @@ static struct efa_dev *efa_probe_device(struct pci_dev *pdev)</div><div class='ctx'> 	edev-&gt;efa_dev = dev;</div><div class='ctx'> 	edev-&gt;dmadev = &amp;pdev-&gt;dev;</div><div class='ctx'> 	dev-&gt;pdev = pdev;</div><div class='add'>+	xa_init(&amp;dev-&gt;cqs_xa);</div><div class='ctx'> </div><div class='del'>-	bars = pci_select_bars(pdev, IORESOURCE_MEM) &amp; EFA_BASE_BAR_MASK;</div><div class='del'>-	err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);</div><div class='add'>+	pci_mem_bars = pci_select_bars(pdev, IORESOURCE_MEM);</div><div class='add'>+	if (EFA_BASE_BAR_MASK &amp; ~pci_mem_bars) {</div><div class='add'>+		dev_err(&amp;pdev-&gt;dev, "BARs unavailable. Requested %#x, available %#x\n",</div><div class='add'>+			(int)EFA_BASE_BAR_MASK, pci_mem_bars);</div><div class='add'>+		err = -ENODEV;</div><div class='add'>+		goto err_ibdev_destroy;</div><div class='add'>+	}</div><div class='add'>+	err = pci_request_selected_regions(pdev, EFA_BASE_BAR_MASK, DRV_MODULE_NAME);</div><div class='ctx'> 	if (err) {</div><div class='ctx'> 		dev_err(&amp;pdev-&gt;dev, "pci_request_selected_regions failed %d\n",</div><div class='ctx'> 			err);</div><div class='hunk'>@@ -463,7 +622,7 @@ static struct efa_dev *efa_probe_device(struct pci_dev *pdev)</div><div class='ctx'> 	return dev;</div><div class='ctx'> </div><div class='ctx'> err_free_mgmnt_irq:</div><div class='del'>-	efa_free_mgmnt_irq(dev);</div><div class='add'>+	efa_free_irq(dev, &amp;dev-&gt;admin_irq);</div><div class='ctx'> err_disable_msix:</div><div class='ctx'> 	efa_disable_msix(dev);</div><div class='ctx'> err_reg_read_destroy:</div><div class='hunk'>@@ -479,18 +638,21 @@ err_disable_device:</div><div class='ctx'> 	return ERR_PTR(err);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void efa_remove_device(struct pci_dev *pdev)</div><div class='add'>+static void efa_remove_device(struct pci_dev *pdev,</div><div class='add'>+			      enum efa_regs_reset_reason_types reset_reason)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_dev *dev = pci_get_drvdata(pdev);</div><div class='ctx'> 	struct efa_com_dev *edev;</div><div class='ctx'> </div><div class='ctx'> 	edev = &amp;dev-&gt;edev;</div><div class='add'>+	efa_com_dev_reset(edev, reset_reason);</div><div class='ctx'> 	efa_com_admin_destroy(edev);</div><div class='del'>-	efa_free_mgmnt_irq(dev);</div><div class='add'>+	efa_free_irq(dev, &amp;dev-&gt;admin_irq);</div><div class='ctx'> 	efa_disable_msix(dev);</div><div class='ctx'> 	efa_com_mmio_reg_read_destroy(edev);</div><div class='ctx'> 	devm_iounmap(&amp;pdev-&gt;dev, edev-&gt;reg_bar);</div><div class='ctx'> 	efa_release_bars(dev, EFA_BASE_BAR_MASK);</div><div class='add'>+	xa_destroy(&amp;dev-&gt;cqs_xa);</div><div class='ctx'> 	ib_dealloc_device(&amp;dev-&gt;ibdev);</div><div class='ctx'> 	pci_disable_device(pdev);</div><div class='ctx'> }</div><div class='hunk'>@@ -511,7 +673,7 @@ static int efa_probe(struct pci_dev *pdev, const struct pci_device_id *ent)</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> err_remove_device:</div><div class='del'>-	efa_remove_device(pdev);</div><div class='add'>+	efa_remove_device(pdev, EFA_REGS_RESET_INIT_ERR);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -520,7 +682,17 @@ static void efa_remove(struct pci_dev *pdev)</div><div class='ctx'> 	struct efa_dev *dev = pci_get_drvdata(pdev);</div><div class='ctx'> </div><div class='ctx'> 	efa_ib_device_remove(dev);</div><div class='del'>-	efa_remove_device(pdev);</div><div class='add'>+	efa_remove_device(pdev, EFA_REGS_RESET_NORMAL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void efa_shutdown(struct pci_dev *pdev)</div><div class='add'>+{</div><div class='add'>+	struct efa_dev *dev = pci_get_drvdata(pdev);</div><div class='add'>+</div><div class='add'>+	efa_destroy_eqs(dev);</div><div class='add'>+	efa_com_dev_reset(&amp;dev-&gt;edev, EFA_REGS_RESET_SHUTDOWN);</div><div class='add'>+	efa_free_irq(dev, &amp;dev-&gt;admin_irq);</div><div class='add'>+	efa_disable_msix(dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static struct pci_driver efa_pci_driver = {</div><div class='hunk'>@@ -528,6 +700,7 @@ static struct pci_driver efa_pci_driver = {</div><div class='ctx'> 	.id_table       = efa_pci_tbl,</div><div class='ctx'> 	.probe          = efa_probe,</div><div class='ctx'> 	.remove         = efa_remove,</div><div class='add'>+	.shutdown       = efa_shutdown,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> module_pci_driver(efa_pci_driver);</div><div class='head'>diff --git a/drivers/infiniband/hw/efa/efa_regs_defs.h b/drivers/infiniband/hw/efa/efa_regs_defs.h<br/>index bb9cad3d6a15..714ae6258800 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_regs_defs.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_regs_defs.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_regs_defs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/efa/efa_regs_defs.h</a></div><div class='hunk'>@@ -1,6 +1,6 @@</div><div class='ctx'> /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */</div><div class='ctx'> /*</div><div class='del'>- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='add'>+ * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #ifndef _EFA_REGS_H_</div><div class='hunk'>@@ -42,72 +42,60 @@ enum efa_regs_reset_reason_types {</div><div class='ctx'> #define EFA_REGS_MMIO_REG_READ_OFF                          0x5c</div><div class='ctx'> #define EFA_REGS_MMIO_RESP_LO_OFF                           0x60</div><div class='ctx'> #define EFA_REGS_MMIO_RESP_HI_OFF                           0x64</div><div class='add'>+#define EFA_REGS_EQ_DB_OFF                                  0x68</div><div class='ctx'> </div><div class='ctx'> /* version register */</div><div class='ctx'> #define EFA_REGS_VERSION_MINOR_VERSION_MASK                 0xff</div><div class='del'>-#define EFA_REGS_VERSION_MAJOR_VERSION_SHIFT                8</div><div class='ctx'> #define EFA_REGS_VERSION_MAJOR_VERSION_MASK                 0xff00</div><div class='ctx'> </div><div class='ctx'> /* controller_version register */</div><div class='ctx'> #define EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK   0xff</div><div class='del'>-#define EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT     8</div><div class='ctx'> #define EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK      0xff00</div><div class='del'>-#define EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT     16</div><div class='ctx'> #define EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK      0xff0000</div><div class='del'>-#define EFA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT           24</div><div class='ctx'> #define EFA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK            0xff000000</div><div class='ctx'> </div><div class='ctx'> /* caps register */</div><div class='ctx'> #define EFA_REGS_CAPS_CONTIGUOUS_QUEUE_REQUIRED_MASK        0x1</div><div class='del'>-#define EFA_REGS_CAPS_RESET_TIMEOUT_SHIFT                   1</div><div class='ctx'> #define EFA_REGS_CAPS_RESET_TIMEOUT_MASK                    0x3e</div><div class='del'>-#define EFA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT                  8</div><div class='ctx'> #define EFA_REGS_CAPS_DMA_ADDR_WIDTH_MASK                   0xff00</div><div class='del'>-#define EFA_REGS_CAPS_ADMIN_CMD_TO_SHIFT                    16</div><div class='ctx'> #define EFA_REGS_CAPS_ADMIN_CMD_TO_MASK                     0xf0000</div><div class='ctx'> </div><div class='ctx'> /* aq_caps register */</div><div class='ctx'> #define EFA_REGS_AQ_CAPS_AQ_DEPTH_MASK                      0xffff</div><div class='del'>-#define EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT                16</div><div class='ctx'> #define EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK                 0xffff0000</div><div class='ctx'> </div><div class='ctx'> /* acq_caps register */</div><div class='ctx'> #define EFA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK                    0xffff</div><div class='del'>-#define EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT              16</div><div class='ctx'> #define EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK               0xff0000</div><div class='del'>-#define EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR_SHIFT             24</div><div class='ctx'> #define EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR_MASK              0xff000000</div><div class='ctx'> </div><div class='ctx'> /* aenq_caps register */</div><div class='ctx'> #define EFA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK                  0xffff</div><div class='del'>-#define EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT            16</div><div class='ctx'> #define EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK             0xff0000</div><div class='del'>-#define EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR_SHIFT           24</div><div class='ctx'> #define EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR_MASK            0xff000000</div><div class='ctx'> </div><div class='add'>+/* intr_mask register */</div><div class='add'>+#define EFA_REGS_INTR_MASK_EN_MASK                          0x1</div><div class='add'>+</div><div class='ctx'> /* dev_ctl register */</div><div class='ctx'> #define EFA_REGS_DEV_CTL_DEV_RESET_MASK                     0x1</div><div class='del'>-#define EFA_REGS_DEV_CTL_AQ_RESTART_SHIFT                   1</div><div class='ctx'> #define EFA_REGS_DEV_CTL_AQ_RESTART_MASK                    0x2</div><div class='del'>-#define EFA_REGS_DEV_CTL_RESET_REASON_SHIFT                 28</div><div class='ctx'> #define EFA_REGS_DEV_CTL_RESET_REASON_MASK                  0xf0000000</div><div class='ctx'> </div><div class='ctx'> /* dev_sts register */</div><div class='ctx'> #define EFA_REGS_DEV_STS_READY_MASK                         0x1</div><div class='del'>-#define EFA_REGS_DEV_STS_AQ_RESTART_IN_PROGRESS_SHIFT       1</div><div class='ctx'> #define EFA_REGS_DEV_STS_AQ_RESTART_IN_PROGRESS_MASK        0x2</div><div class='del'>-#define EFA_REGS_DEV_STS_AQ_RESTART_FINISHED_SHIFT          2</div><div class='ctx'> #define EFA_REGS_DEV_STS_AQ_RESTART_FINISHED_MASK           0x4</div><div class='del'>-#define EFA_REGS_DEV_STS_RESET_IN_PROGRESS_SHIFT            3</div><div class='ctx'> #define EFA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK             0x8</div><div class='del'>-#define EFA_REGS_DEV_STS_RESET_FINISHED_SHIFT               4</div><div class='ctx'> #define EFA_REGS_DEV_STS_RESET_FINISHED_MASK                0x10</div><div class='del'>-#define EFA_REGS_DEV_STS_FATAL_ERROR_SHIFT                  5</div><div class='ctx'> #define EFA_REGS_DEV_STS_FATAL_ERROR_MASK                   0x20</div><div class='ctx'> </div><div class='ctx'> /* mmio_reg_read register */</div><div class='ctx'> #define EFA_REGS_MMIO_REG_READ_REQ_ID_MASK                  0xffff</div><div class='del'>-#define EFA_REGS_MMIO_REG_READ_REG_OFF_SHIFT                16</div><div class='ctx'> #define EFA_REGS_MMIO_REG_READ_REG_OFF_MASK                 0xffff0000</div><div class='ctx'> </div><div class='add'>+/* eq_db register */</div><div class='add'>+#define EFA_REGS_EQ_DB_EQN_MASK                             0xffff</div><div class='add'>+#define EFA_REGS_EQ_DB_ARM_MASK                             0x80000000</div><div class='add'>+</div><div class='ctx'> #endif /* _EFA_REGS_H_ */</div><div class='head'>diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c<br/>index 6d6886c9009f..22d3e25c3b9d 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_verbs.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/efa/efa_verbs.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/efa/efa_verbs.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/efa/efa_verbs.c</a></div><div class='hunk'>@@ -1,21 +1,24 @@</div><div class='ctx'> // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='ctx'> /*</div><div class='del'>- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='add'>+ * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='add'>+#include &lt;linux/dma-buf.h&gt;</div><div class='add'>+#include &lt;linux/dma-resv.h&gt;</div><div class='ctx'> #include &lt;linux/vmalloc.h&gt;</div><div class='add'>+#include &lt;linux/log2.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include &lt;rdma/ib_addr.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_umem.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_user_verbs.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_verbs.h&gt;</div><div class='ctx'> #include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='add'>+#define UVERBS_MODULE_NAME efa_ib</div><div class='add'>+#include &lt;rdma/uverbs_named_ioctl.h&gt;</div><div class='add'>+#include &lt;rdma/ib_user_ioctl_cmds.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "efa.h"</div><div class='del'>-</div><div class='del'>-#define EFA_MMAP_FLAG_SHIFT 56</div><div class='del'>-#define EFA_MMAP_PAGE_MASK GENMASK(EFA_MMAP_FLAG_SHIFT - 1, 0)</div><div class='del'>-#define EFA_MMAP_INVALID U64_MAX</div><div class='add'>+#include "efa_io_defs.h"</div><div class='ctx'> </div><div class='ctx'> enum {</div><div class='ctx'> 	EFA_MMAP_DMA_PAGE = 0,</div><div class='hunk'>@@ -23,23 +26,71 @@ enum {</div><div class='ctx'> 	EFA_MMAP_IO_NC,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define EFA_AENQ_ENABLED_GROUPS \</div><div class='del'>-	(BIT(EFA_ADMIN_FATAL_ERROR) | BIT(EFA_ADMIN_WARNING) | \</div><div class='del'>-	 BIT(EFA_ADMIN_NOTIFICATION) | BIT(EFA_ADMIN_KEEP_ALIVE))</div><div class='del'>-</div><div class='del'>-struct efa_mmap_entry {</div><div class='del'>-	void  *obj;</div><div class='add'>+struct efa_user_mmap_entry {</div><div class='add'>+	struct rdma_user_mmap_entry rdma_entry;</div><div class='ctx'> 	u64 address;</div><div class='del'>-	u64 length;</div><div class='del'>-	u32 mmap_page;</div><div class='ctx'> 	u8 mmap_flag;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static inline u64 get_mmap_key(const struct efa_mmap_entry *efa)</div><div class='del'>-{</div><div class='del'>-	return ((u64)efa-&gt;mmap_flag &lt;&lt; EFA_MMAP_FLAG_SHIFT) |</div><div class='del'>-	       ((u64)efa-&gt;mmap_page &lt;&lt; PAGE_SHIFT);</div><div class='del'>-}</div><div class='add'>+#define EFA_DEFINE_DEVICE_STATS(op) \</div><div class='add'>+	op(EFA_SUBMITTED_CMDS, "submitted_cmds") \</div><div class='add'>+	op(EFA_COMPLETED_CMDS, "completed_cmds") \</div><div class='add'>+	op(EFA_CMDS_ERR, "cmds_err") \</div><div class='add'>+	op(EFA_NO_COMPLETION_CMDS, "no_completion_cmds") \</div><div class='add'>+	op(EFA_KEEP_ALIVE_RCVD, "keep_alive_rcvd") \</div><div class='add'>+	op(EFA_ALLOC_PD_ERR, "alloc_pd_err") \</div><div class='add'>+	op(EFA_CREATE_QP_ERR, "create_qp_err") \</div><div class='add'>+	op(EFA_CREATE_CQ_ERR, "create_cq_err") \</div><div class='add'>+	op(EFA_REG_MR_ERR, "reg_mr_err") \</div><div class='add'>+	op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \</div><div class='add'>+	op(EFA_CREATE_AH_ERR, "create_ah_err") \</div><div class='add'>+	op(EFA_MMAP_ERR, "mmap_err")</div><div class='add'>+</div><div class='add'>+#define EFA_DEFINE_PORT_STATS(op) \</div><div class='add'>+	op(EFA_TX_BYTES, "tx_bytes") \</div><div class='add'>+	op(EFA_TX_PKTS, "tx_pkts") \</div><div class='add'>+	op(EFA_RX_BYTES, "rx_bytes") \</div><div class='add'>+	op(EFA_RX_PKTS, "rx_pkts") \</div><div class='add'>+	op(EFA_RX_DROPS, "rx_drops") \</div><div class='add'>+	op(EFA_SEND_BYTES, "send_bytes") \</div><div class='add'>+	op(EFA_SEND_WRS, "send_wrs") \</div><div class='add'>+	op(EFA_RECV_BYTES, "recv_bytes") \</div><div class='add'>+	op(EFA_RECV_WRS, "recv_wrs") \</div><div class='add'>+	op(EFA_RDMA_READ_WRS, "rdma_read_wrs") \</div><div class='add'>+	op(EFA_RDMA_READ_BYTES, "rdma_read_bytes") \</div><div class='add'>+	op(EFA_RDMA_READ_WR_ERR, "rdma_read_wr_err") \</div><div class='add'>+	op(EFA_RDMA_READ_RESP_BYTES, "rdma_read_resp_bytes") \</div><div class='add'>+	op(EFA_RDMA_WRITE_WRS, "rdma_write_wrs") \</div><div class='add'>+	op(EFA_RDMA_WRITE_BYTES, "rdma_write_bytes") \</div><div class='add'>+	op(EFA_RDMA_WRITE_WR_ERR, "rdma_write_wr_err") \</div><div class='add'>+	op(EFA_RDMA_WRITE_RECV_BYTES, "rdma_write_recv_bytes") \</div><div class='add'>+	op(EFA_RETRANS_BYTES, "retrans_bytes") \</div><div class='add'>+	op(EFA_RETRANS_PKTS, "retrans_pkts") \</div><div class='add'>+	op(EFA_RETRANS_TIMEOUT_EVENS, "retrans_timeout_events") \</div><div class='add'>+	op(EFA_UNRESPONSIVE_REMOTE_EVENTS, "unresponsive_remote_events") \</div><div class='add'>+	op(EFA_IMPAIRED_REMOTE_CONN_EVENTS, "impaired_remote_conn_events") \</div><div class='add'>+</div><div class='add'>+#define EFA_STATS_ENUM(ename, name) ename,</div><div class='add'>+#define EFA_STATS_STR(ename, nam) \</div><div class='add'>+	[ename].name = nam,</div><div class='add'>+</div><div class='add'>+enum efa_hw_device_stats {</div><div class='add'>+	EFA_DEFINE_DEVICE_STATS(EFA_STATS_ENUM)</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct rdma_stat_desc efa_device_stats_descs[] = {</div><div class='add'>+	EFA_DEFINE_DEVICE_STATS(EFA_STATS_STR)</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum efa_hw_port_stats {</div><div class='add'>+	EFA_DEFINE_PORT_STATS(EFA_STATS_ENUM)</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct rdma_stat_desc efa_port_stats_descs[] = {</div><div class='add'>+	EFA_DEFINE_PORT_STATS(EFA_STATS_STR)</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define EFA_DEFAULT_LINK_SPEED_GBPS   100</div><div class='ctx'> </div><div class='ctx'> #define EFA_CHUNK_PAYLOAD_SHIFT       12</div><div class='ctx'> #define EFA_CHUNK_PAYLOAD_SIZE        BIT(EFA_CHUNK_PAYLOAD_SHIFT)</div><div class='hunk'>@@ -55,8 +106,6 @@ static inline u64 get_mmap_key(const struct efa_mmap_entry *efa)</div><div class='ctx'> #define EFA_CHUNK_USED_SIZE \</div><div class='ctx'> 	((EFA_PTRS_PER_CHUNK * EFA_CHUNK_PAYLOAD_PTR_SIZE) + EFA_CHUNK_PTR_SIZE)</div><div class='ctx'> </div><div class='del'>-#define EFA_SUPPORTED_ACCESS_FLAGS IB_ACCESS_LOCAL_WRITE</div><div class='del'>-</div><div class='ctx'> struct pbl_chunk {</div><div class='ctx'> 	dma_addr_t dma_addr;</div><div class='ctx'> 	u64 *buf;</div><div class='hunk'>@@ -120,8 +169,15 @@ static inline struct efa_ah *to_eah(struct ib_ah *ibah)</div><div class='ctx'> 	return container_of(ibah, struct efa_ah, ibah);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-#define field_avail(x, fld, sz) (offsetof(typeof(x), fld) + \</div><div class='del'>-				 sizeof(((typeof(x) *)0)-&gt;fld) &lt;= (sz))</div><div class='add'>+static inline struct efa_user_mmap_entry *</div><div class='add'>+to_emmap(struct rdma_user_mmap_entry *rdma_entry)</div><div class='add'>+{</div><div class='add'>+	return container_of(rdma_entry, struct efa_user_mmap_entry, rdma_entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define EFA_DEV_CAP(dev, cap) \</div><div class='add'>+	((dev)-&gt;dev_attr.device_caps &amp; \</div><div class='add'>+	 EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_##cap##_MASK)</div><div class='ctx'> </div><div class='ctx'> #define is_reserved_cleared(reserved) \</div><div class='ctx'> 	!memchr_inv(reserved, 0, sizeof(reserved))</div><div class='hunk'>@@ -145,93 +201,12 @@ static void *efa_zalloc_mapped(struct efa_dev *dev, dma_addr_t *dma_addr,</div><div class='ctx'> 	return addr;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/*</div><div class='del'>- * This is only called when the ucontext is destroyed and there can be no</div><div class='del'>- * concurrent query via mmap or allocate on the xarray, thus we can be sure no</div><div class='del'>- * other thread is using the entry pointer. We also know that all the BAR</div><div class='del'>- * pages have either been zap'd or munmaped at this point.  Normal pages are</div><div class='del'>- * refcounted and will be freed at the proper time.</div><div class='del'>- */</div><div class='del'>-static void mmap_entries_remove_free(struct efa_dev *dev,</div><div class='del'>-				     struct efa_ucontext *ucontext)</div><div class='del'>-{</div><div class='del'>-	struct efa_mmap_entry *entry;</div><div class='del'>-	unsigned long mmap_page;</div><div class='del'>-</div><div class='del'>-	xa_for_each(&amp;ucontext-&gt;mmap_xa, mmap_page, entry) {</div><div class='del'>-		xa_erase(&amp;ucontext-&gt;mmap_xa, mmap_page);</div><div class='del'>-</div><div class='del'>-		ibdev_dbg(</div><div class='del'>-			&amp;dev-&gt;ibdev,</div><div class='del'>-			"mmap: obj[0x%p] key[%#llx] addr[%#llx] len[%#llx] removed\n",</div><div class='del'>-			entry-&gt;obj, get_mmap_key(entry), entry-&gt;address,</div><div class='del'>-			entry-&gt;length);</div><div class='del'>-		if (entry-&gt;mmap_flag == EFA_MMAP_DMA_PAGE)</div><div class='del'>-			/* DMA mapping is already gone, now free the pages */</div><div class='del'>-			free_pages_exact(phys_to_virt(entry-&gt;address),</div><div class='del'>-					 entry-&gt;length);</div><div class='del'>-		kfree(entry);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct efa_mmap_entry *mmap_entry_get(struct efa_dev *dev,</div><div class='del'>-					     struct efa_ucontext *ucontext,</div><div class='del'>-					     u64 key, u64 len)</div><div class='del'>-{</div><div class='del'>-	struct efa_mmap_entry *entry;</div><div class='del'>-	u64 mmap_page;</div><div class='del'>-</div><div class='del'>-	mmap_page = (key &amp; EFA_MMAP_PAGE_MASK) &gt;&gt; PAGE_SHIFT;</div><div class='del'>-	if (mmap_page &gt; U32_MAX)</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	entry = xa_load(&amp;ucontext-&gt;mmap_xa, mmap_page);</div><div class='del'>-	if (!entry || get_mmap_key(entry) != key || entry-&gt;length != len)</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='del'>-		  "mmap: obj[0x%p] key[%#llx] addr[%#llx] len[%#llx] removed\n",</div><div class='del'>-		  entry-&gt;obj, key, entry-&gt;address, entry-&gt;length);</div><div class='del'>-</div><div class='del'>-	return entry;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Note this locking scheme cannot support removal of entries, except during</div><div class='del'>- * ucontext destruction when the core code guarentees no concurrency.</div><div class='del'>- */</div><div class='del'>-static u64 mmap_entry_insert(struct efa_dev *dev, struct efa_ucontext *ucontext,</div><div class='del'>-			     void *obj, u64 address, u64 length, u8 mmap_flag)</div><div class='add'>+static void efa_free_mapped(struct efa_dev *dev, void *cpu_addr,</div><div class='add'>+			    dma_addr_t dma_addr,</div><div class='add'>+			    size_t size, enum dma_data_direction dir)</div><div class='ctx'> {</div><div class='del'>-	struct efa_mmap_entry *entry;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	entry = kmalloc(sizeof(*entry), GFP_KERNEL);</div><div class='del'>-	if (!entry)</div><div class='del'>-		return EFA_MMAP_INVALID;</div><div class='del'>-</div><div class='del'>-	entry-&gt;obj = obj;</div><div class='del'>-	entry-&gt;address = address;</div><div class='del'>-	entry-&gt;length = length;</div><div class='del'>-	entry-&gt;mmap_flag = mmap_flag;</div><div class='del'>-</div><div class='del'>-	xa_lock(&amp;ucontext-&gt;mmap_xa);</div><div class='del'>-	entry-&gt;mmap_page = ucontext-&gt;mmap_xa_page;</div><div class='del'>-	ucontext-&gt;mmap_xa_page += DIV_ROUND_UP(length, PAGE_SIZE);</div><div class='del'>-	err = __xa_insert(&amp;ucontext-&gt;mmap_xa, entry-&gt;mmap_page, entry,</div><div class='del'>-			  GFP_KERNEL);</div><div class='del'>-	xa_unlock(&amp;ucontext-&gt;mmap_xa);</div><div class='del'>-	if (err){</div><div class='del'>-		kfree(entry);</div><div class='del'>-		return EFA_MMAP_INVALID;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ibdev_dbg(</div><div class='del'>-		&amp;dev-&gt;ibdev,</div><div class='del'>-		"mmap: obj[0x%p] addr[%#llx], len[%#llx], key[%#llx] inserted\n",</div><div class='del'>-		entry-&gt;obj, entry-&gt;address, entry-&gt;length, get_mmap_key(entry));</div><div class='del'>-</div><div class='del'>-	return get_mmap_key(entry);</div><div class='add'>+	dma_unmap_single(&amp;dev-&gt;pdev-&gt;dev, dma_addr, size, dir);</div><div class='add'>+	free_pages_exact(cpu_addr, size);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int efa_query_device(struct ib_device *ibdev,</div><div class='hunk'>@@ -268,12 +243,35 @@ int efa_query_device(struct ib_device *ibdev,</div><div class='ctx'> 				 dev_attr-&gt;max_rq_depth);</div><div class='ctx'> 	props-&gt;max_send_sge = dev_attr-&gt;max_sq_sge;</div><div class='ctx'> 	props-&gt;max_recv_sge = dev_attr-&gt;max_rq_sge;</div><div class='add'>+	props-&gt;max_sge_rd = dev_attr-&gt;max_wr_rdma_sge;</div><div class='add'>+	props-&gt;max_pkeys = 1;</div><div class='ctx'> </div><div class='ctx'> 	if (udata &amp;&amp; udata-&gt;outlen) {</div><div class='ctx'> 		resp.max_sq_sge = dev_attr-&gt;max_sq_sge;</div><div class='ctx'> 		resp.max_rq_sge = dev_attr-&gt;max_rq_sge;</div><div class='ctx'> 		resp.max_sq_wr = dev_attr-&gt;max_sq_depth;</div><div class='ctx'> 		resp.max_rq_wr = dev_attr-&gt;max_rq_depth;</div><div class='add'>+		resp.max_rdma_size = dev_attr-&gt;max_rdma_size;</div><div class='add'>+</div><div class='add'>+		resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_WITH_SGID;</div><div class='add'>+		resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_WITH_EXT_MEM;</div><div class='add'>+		if (EFA_DEV_CAP(dev, RDMA_READ))</div><div class='add'>+			resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RDMA_READ;</div><div class='add'>+</div><div class='add'>+		if (EFA_DEV_CAP(dev, RNR_RETRY))</div><div class='add'>+			resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RNR_RETRY;</div><div class='add'>+</div><div class='add'>+		if (EFA_DEV_CAP(dev, DATA_POLLING_128))</div><div class='add'>+			resp.device_caps |= EFA_QUERY_DEVICE_CAPS_DATA_POLLING_128;</div><div class='add'>+</div><div class='add'>+		if (EFA_DEV_CAP(dev, RDMA_WRITE))</div><div class='add'>+			resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RDMA_WRITE;</div><div class='add'>+</div><div class='add'>+		if (EFA_DEV_CAP(dev, UNSOLICITED_WRITE_RECV))</div><div class='add'>+			resp.device_caps |= EFA_QUERY_DEVICE_CAPS_UNSOLICITED_WRITE_RECV;</div><div class='add'>+</div><div class='add'>+		if (dev-&gt;neqs)</div><div class='add'>+			resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_NOTIFICATIONS;</div><div class='ctx'> </div><div class='ctx'> 		err = ib_copy_to_udata(udata, &amp;resp,</div><div class='ctx'> 				       min(sizeof(resp), udata-&gt;outlen));</div><div class='hunk'>@@ -287,22 +285,61 @@ int efa_query_device(struct ib_device *ibdev,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int efa_query_port(struct ib_device *ibdev, u8 port,</div><div class='add'>+static void efa_link_gbps_to_speed_and_width(u16 gbps,</div><div class='add'>+					     enum ib_port_speed *speed,</div><div class='add'>+					     enum ib_port_width *width)</div><div class='add'>+{</div><div class='add'>+	if (gbps &gt;= 400) {</div><div class='add'>+		*width = IB_WIDTH_8X;</div><div class='add'>+		*speed = IB_SPEED_HDR;</div><div class='add'>+	} else if (gbps &gt;= 200) {</div><div class='add'>+		*width = IB_WIDTH_4X;</div><div class='add'>+		*speed = IB_SPEED_HDR;</div><div class='add'>+	} else if (gbps &gt;= 120) {</div><div class='add'>+		*width = IB_WIDTH_12X;</div><div class='add'>+		*speed = IB_SPEED_FDR10;</div><div class='add'>+	} else if (gbps &gt;= 100) {</div><div class='add'>+		*width = IB_WIDTH_4X;</div><div class='add'>+		*speed = IB_SPEED_EDR;</div><div class='add'>+	} else if (gbps &gt;= 60) {</div><div class='add'>+		*width = IB_WIDTH_12X;</div><div class='add'>+		*speed = IB_SPEED_DDR;</div><div class='add'>+	} else if (gbps &gt;= 50) {</div><div class='add'>+		*width = IB_WIDTH_1X;</div><div class='add'>+		*speed = IB_SPEED_HDR;</div><div class='add'>+	} else if (gbps &gt;= 40) {</div><div class='add'>+		*width = IB_WIDTH_4X;</div><div class='add'>+		*speed = IB_SPEED_FDR10;</div><div class='add'>+	} else if (gbps &gt;= 30) {</div><div class='add'>+		*width = IB_WIDTH_12X;</div><div class='add'>+		*speed = IB_SPEED_SDR;</div><div class='add'>+	} else {</div><div class='add'>+		*width = IB_WIDTH_1X;</div><div class='add'>+		*speed = IB_SPEED_EDR;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int efa_query_port(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 		   struct ib_port_attr *props)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_dev *dev = to_edev(ibdev);</div><div class='add'>+	enum ib_port_speed link_speed;</div><div class='add'>+	enum ib_port_width link_width;</div><div class='add'>+	u16 link_gbps;</div><div class='ctx'> </div><div class='ctx'> 	props-&gt;lmc = 1;</div><div class='ctx'> </div><div class='ctx'> 	props-&gt;state = IB_PORT_ACTIVE;</div><div class='del'>-	props-&gt;phys_state = 5;</div><div class='add'>+	props-&gt;phys_state = IB_PORT_PHYS_STATE_LINK_UP;</div><div class='ctx'> 	props-&gt;gid_tbl_len = 1;</div><div class='ctx'> 	props-&gt;pkey_tbl_len = 1;</div><div class='del'>-	props-&gt;active_speed = IB_SPEED_EDR;</div><div class='del'>-	props-&gt;active_width = IB_WIDTH_4X;</div><div class='del'>-	props-&gt;max_mtu = ib_mtu_int_to_enum(dev-&gt;mtu);</div><div class='del'>-	props-&gt;active_mtu = ib_mtu_int_to_enum(dev-&gt;mtu);</div><div class='del'>-	props-&gt;max_msg_sz = dev-&gt;mtu;</div><div class='add'>+	link_gbps = dev-&gt;dev_attr.max_link_speed_gbps ?: EFA_DEFAULT_LINK_SPEED_GBPS;</div><div class='add'>+	efa_link_gbps_to_speed_and_width(link_gbps, &amp;link_speed, &amp;link_width);</div><div class='add'>+	props-&gt;active_speed = link_speed;</div><div class='add'>+	props-&gt;active_width = link_width;</div><div class='add'>+	props-&gt;max_mtu = ib_mtu_int_to_enum(dev-&gt;dev_attr.mtu);</div><div class='add'>+	props-&gt;active_mtu = ib_mtu_int_to_enum(dev-&gt;dev_attr.mtu);</div><div class='add'>+	props-&gt;max_msg_sz = dev-&gt;dev_attr.mtu;</div><div class='ctx'> 	props-&gt;max_vl_num = 1;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='hunk'>@@ -320,7 +357,7 @@ int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> </div><div class='ctx'> #define EFA_QUERY_QP_SUPP_MASK \</div><div class='ctx'> 	(IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | \</div><div class='del'>-	 IB_QP_QKEY | IB_QP_SQ_PSN | IB_QP_CAP)</div><div class='add'>+	 IB_QP_QKEY | IB_QP_SQ_PSN | IB_QP_CAP | IB_QP_RNR_RETRY)</div><div class='ctx'> </div><div class='ctx'> 	if (qp_attr_mask &amp; ~EFA_QUERY_QP_SUPP_MASK) {</div><div class='ctx'> 		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='hunk'>@@ -342,6 +379,7 @@ int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 	qp_attr-&gt;sq_psn = result.sq_psn;</div><div class='ctx'> 	qp_attr-&gt;sq_draining = result.sq_draining;</div><div class='ctx'> 	qp_attr-&gt;port_num = 1;</div><div class='add'>+	qp_attr-&gt;rnr_retry = result.rnr_retry;</div><div class='ctx'> </div><div class='ctx'> 	qp_attr-&gt;cap.max_send_wr = qp-&gt;max_send_wr;</div><div class='ctx'> 	qp_attr-&gt;cap.max_recv_wr = qp-&gt;max_recv_wr;</div><div class='hunk'>@@ -358,17 +396,17 @@ int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int efa_query_gid(struct ib_device *ibdev, u8 port, int index,</div><div class='add'>+int efa_query_gid(struct ib_device *ibdev, u32 port, int index,</div><div class='ctx'> 		  union ib_gid *gid)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_dev *dev = to_edev(ibdev);</div><div class='ctx'> </div><div class='del'>-	memcpy(gid-&gt;raw, dev-&gt;addr, sizeof(dev-&gt;addr));</div><div class='add'>+	memcpy(gid-&gt;raw, dev-&gt;dev_attr.addr, sizeof(dev-&gt;dev_attr.addr));</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int efa_query_pkey(struct ib_device *ibdev, u8 port, u16 index,</div><div class='add'>+int efa_query_pkey(struct ib_device *ibdev, u32 port, u16 index,</div><div class='ctx'> 		   u16 *pkey)</div><div class='ctx'> {</div><div class='ctx'> 	if (index &gt; 0)</div><div class='hunk'>@@ -427,23 +465,18 @@ int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='ctx'> err_dealloc_pd:</div><div class='ctx'> 	efa_pd_dealloc(dev, result.pdn);</div><div class='ctx'> err_out:</div><div class='del'>-	atomic64_inc(&amp;dev-&gt;stats.sw_stats.alloc_pd_err);</div><div class='add'>+	atomic64_inc(&amp;dev-&gt;stats.alloc_pd_err);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='add'>+int efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_dev *dev = to_edev(ibpd-&gt;device);</div><div class='ctx'> 	struct efa_pd *pd = to_epd(ibpd);</div><div class='ctx'> </div><div class='del'>-	if (udata-&gt;inlen &amp;&amp;</div><div class='del'>-	    !ib_is_udata_cleared(udata, 0, udata-&gt;inlen)) {</div><div class='del'>-		ibdev_dbg(&amp;dev-&gt;ibdev, "Incompatible ABI params\n");</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	ibdev_dbg(&amp;dev-&gt;ibdev, "Dealloc pd[%d]\n", pd-&gt;pdn);</div><div class='ctx'> 	efa_pd_dealloc(dev, pd-&gt;pdn);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int efa_destroy_qp_handle(struct efa_dev *dev, u32 qp_handle)</div><div class='hunk'>@@ -453,87 +486,130 @@ static int efa_destroy_qp_handle(struct efa_dev *dev, u32 qp_handle)</div><div class='ctx'> 	return efa_com_destroy_qp(&amp;dev-&gt;edev, &amp;params);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void efa_qp_user_mmap_entries_remove(struct efa_qp *qp)</div><div class='add'>+{</div><div class='add'>+	rdma_user_mmap_entry_remove(qp-&gt;rq_mmap_entry);</div><div class='add'>+	rdma_user_mmap_entry_remove(qp-&gt;rq_db_mmap_entry);</div><div class='add'>+	rdma_user_mmap_entry_remove(qp-&gt;llq_desc_mmap_entry);</div><div class='add'>+	rdma_user_mmap_entry_remove(qp-&gt;sq_db_mmap_entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_dev *dev = to_edev(ibqp-&gt;pd-&gt;device);</div><div class='ctx'> 	struct efa_qp *qp = to_eqp(ibqp);</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	if (udata-&gt;inlen &amp;&amp;</div><div class='del'>-	    !ib_is_udata_cleared(udata, 0, udata-&gt;inlen)) {</div><div class='del'>-		ibdev_dbg(&amp;dev-&gt;ibdev, "Incompatible ABI params\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	ibdev_dbg(&amp;dev-&gt;ibdev, "Destroy qp[%u]\n", ibqp-&gt;qp_num);</div><div class='add'>+</div><div class='ctx'> 	err = efa_destroy_qp_handle(dev, qp-&gt;qp_handle);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='add'>+	efa_qp_user_mmap_entries_remove(qp);</div><div class='add'>+</div><div class='ctx'> 	if (qp-&gt;rq_cpu_addr) {</div><div class='ctx'> 		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='ctx'> 			  "qp-&gt;cpu_addr[0x%p] freed: size[%lu], dma[%pad]\n",</div><div class='ctx'> 			  qp-&gt;rq_cpu_addr, qp-&gt;rq_size,</div><div class='ctx'> 			  &amp;qp-&gt;rq_dma_addr);</div><div class='del'>-		dma_unmap_single(&amp;dev-&gt;pdev-&gt;dev, qp-&gt;rq_dma_addr, qp-&gt;rq_size,</div><div class='del'>-				 DMA_TO_DEVICE);</div><div class='add'>+		efa_free_mapped(dev, qp-&gt;rq_cpu_addr, qp-&gt;rq_dma_addr,</div><div class='add'>+				qp-&gt;rq_size, DMA_TO_DEVICE);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	kfree(qp);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static struct rdma_user_mmap_entry*</div><div class='add'>+efa_user_mmap_entry_insert(struct ib_ucontext *ucontext,</div><div class='add'>+			   u64 address, size_t length,</div><div class='add'>+			   u8 mmap_flag, u64 *offset)</div><div class='add'>+{</div><div class='add'>+	struct efa_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL);</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (!entry)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	entry-&gt;address = address;</div><div class='add'>+	entry-&gt;mmap_flag = mmap_flag;</div><div class='add'>+</div><div class='add'>+	err = rdma_user_mmap_entry_insert(ucontext, &amp;entry-&gt;rdma_entry,</div><div class='add'>+					  length);</div><div class='add'>+	if (err) {</div><div class='add'>+		kfree(entry);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+	*offset = rdma_user_mmap_get_offset(&amp;entry-&gt;rdma_entry);</div><div class='add'>+</div><div class='add'>+	return &amp;entry-&gt;rdma_entry;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int qp_mmap_entries_setup(struct efa_qp *qp,</div><div class='ctx'> 				 struct efa_dev *dev,</div><div class='ctx'> 				 struct efa_ucontext *ucontext,</div><div class='ctx'> 				 struct efa_com_create_qp_params *params,</div><div class='ctx'> 				 struct efa_ibv_create_qp_resp *resp)</div><div class='ctx'> {</div><div class='del'>-	/*</div><div class='del'>-	 * Once an entry is inserted it might be mmapped, hence cannot be</div><div class='del'>-	 * cleaned up until dealloc_ucontext.</div><div class='del'>-	 */</div><div class='del'>-	resp-&gt;sq_db_mmap_key =</div><div class='del'>-		mmap_entry_insert(dev, ucontext, qp,</div><div class='del'>-				  dev-&gt;db_bar_addr + resp-&gt;sq_db_offset,</div><div class='del'>-				  PAGE_SIZE, EFA_MMAP_IO_NC);</div><div class='del'>-	if (resp-&gt;sq_db_mmap_key == EFA_MMAP_INVALID)</div><div class='add'>+	size_t length;</div><div class='add'>+	u64 address;</div><div class='add'>+</div><div class='add'>+	address = dev-&gt;db_bar_addr + resp-&gt;sq_db_offset;</div><div class='add'>+	qp-&gt;sq_db_mmap_entry =</div><div class='add'>+		efa_user_mmap_entry_insert(&amp;ucontext-&gt;ibucontext,</div><div class='add'>+					   address,</div><div class='add'>+					   PAGE_SIZE, EFA_MMAP_IO_NC,</div><div class='add'>+					   &amp;resp-&gt;sq_db_mmap_key);</div><div class='add'>+	if (!qp-&gt;sq_db_mmap_entry)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	resp-&gt;sq_db_offset &amp;= ~PAGE_MASK;</div><div class='ctx'> </div><div class='del'>-	resp-&gt;llq_desc_mmap_key =</div><div class='del'>-		mmap_entry_insert(dev, ucontext, qp,</div><div class='del'>-				  dev-&gt;mem_bar_addr + resp-&gt;llq_desc_offset,</div><div class='del'>-				  PAGE_ALIGN(params-&gt;sq_ring_size_in_bytes +</div><div class='del'>-					     (resp-&gt;llq_desc_offset &amp; ~PAGE_MASK)),</div><div class='del'>-				  EFA_MMAP_IO_WC);</div><div class='del'>-	if (resp-&gt;llq_desc_mmap_key == EFA_MMAP_INVALID)</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+	address = dev-&gt;mem_bar_addr + resp-&gt;llq_desc_offset;</div><div class='add'>+	length = PAGE_ALIGN(params-&gt;sq_ring_size_in_bytes +</div><div class='add'>+			    offset_in_page(resp-&gt;llq_desc_offset));</div><div class='add'>+</div><div class='add'>+	qp-&gt;llq_desc_mmap_entry =</div><div class='add'>+		efa_user_mmap_entry_insert(&amp;ucontext-&gt;ibucontext,</div><div class='add'>+					   address, length,</div><div class='add'>+					   EFA_MMAP_IO_WC,</div><div class='add'>+					   &amp;resp-&gt;llq_desc_mmap_key);</div><div class='add'>+	if (!qp-&gt;llq_desc_mmap_entry)</div><div class='add'>+		goto err_remove_mmap;</div><div class='ctx'> </div><div class='ctx'> 	resp-&gt;llq_desc_offset &amp;= ~PAGE_MASK;</div><div class='ctx'> </div><div class='ctx'> 	if (qp-&gt;rq_size) {</div><div class='del'>-		resp-&gt;rq_db_mmap_key =</div><div class='del'>-			mmap_entry_insert(dev, ucontext, qp,</div><div class='del'>-					  dev-&gt;db_bar_addr + resp-&gt;rq_db_offset,</div><div class='del'>-					  PAGE_SIZE, EFA_MMAP_IO_NC);</div><div class='del'>-		if (resp-&gt;rq_db_mmap_key == EFA_MMAP_INVALID)</div><div class='del'>-			return -ENOMEM;</div><div class='add'>+		address = dev-&gt;db_bar_addr + resp-&gt;rq_db_offset;</div><div class='add'>+</div><div class='add'>+		qp-&gt;rq_db_mmap_entry =</div><div class='add'>+			efa_user_mmap_entry_insert(&amp;ucontext-&gt;ibucontext,</div><div class='add'>+						   address, PAGE_SIZE,</div><div class='add'>+						   EFA_MMAP_IO_NC,</div><div class='add'>+						   &amp;resp-&gt;rq_db_mmap_key);</div><div class='add'>+		if (!qp-&gt;rq_db_mmap_entry)</div><div class='add'>+			goto err_remove_mmap;</div><div class='ctx'> </div><div class='ctx'> 		resp-&gt;rq_db_offset &amp;= ~PAGE_MASK;</div><div class='ctx'> </div><div class='del'>-		resp-&gt;rq_mmap_key =</div><div class='del'>-			mmap_entry_insert(dev, ucontext, qp,</div><div class='del'>-					  virt_to_phys(qp-&gt;rq_cpu_addr),</div><div class='del'>-					  qp-&gt;rq_size, EFA_MMAP_DMA_PAGE);</div><div class='del'>-		if (resp-&gt;rq_mmap_key == EFA_MMAP_INVALID)</div><div class='del'>-			return -ENOMEM;</div><div class='add'>+		address = virt_to_phys(qp-&gt;rq_cpu_addr);</div><div class='add'>+		qp-&gt;rq_mmap_entry =</div><div class='add'>+			efa_user_mmap_entry_insert(&amp;ucontext-&gt;ibucontext,</div><div class='add'>+						   address, qp-&gt;rq_size,</div><div class='add'>+						   EFA_MMAP_DMA_PAGE,</div><div class='add'>+						   &amp;resp-&gt;rq_mmap_key);</div><div class='add'>+		if (!qp-&gt;rq_mmap_entry)</div><div class='add'>+			goto err_remove_mmap;</div><div class='ctx'> </div><div class='ctx'> 		resp-&gt;rq_mmap_size = qp-&gt;rq_size;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='add'>+</div><div class='add'>+err_remove_mmap:</div><div class='add'>+	efa_qp_user_mmap_entries_remove(qp);</div><div class='add'>+</div><div class='add'>+	return -ENOMEM;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int efa_qp_validate_cap(struct efa_dev *dev,</div><div class='hunk'>@@ -599,18 +675,17 @@ static int efa_qp_validate_attr(struct efa_dev *dev,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_qp *efa_create_qp(struct ib_pd *ibpd,</div><div class='del'>-			    struct ib_qp_init_attr *init_attr,</div><div class='del'>-			    struct ib_udata *udata)</div><div class='add'>+int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,</div><div class='add'>+		  struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_com_create_qp_params create_qp_params = {};</div><div class='ctx'> 	struct efa_com_create_qp_result create_qp_resp;</div><div class='del'>-	struct efa_dev *dev = to_edev(ibpd-&gt;device);</div><div class='add'>+	struct efa_dev *dev = to_edev(ibqp-&gt;device);</div><div class='ctx'> 	struct efa_ibv_create_qp_resp resp = {};</div><div class='ctx'> 	struct efa_ibv_create_qp cmd = {};</div><div class='del'>-	bool rq_entry_inserted = false;</div><div class='add'>+	struct efa_qp *qp = to_eqp(ibqp);</div><div class='ctx'> 	struct efa_ucontext *ucontext;</div><div class='del'>-	struct efa_qp *qp;</div><div class='add'>+	u16 supported_efa_flags = 0;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	ucontext = rdma_udata_to_drv_context(udata, struct efa_ucontext,</div><div class='hunk'>@@ -624,7 +699,7 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd,</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_out;</div><div class='ctx'> </div><div class='del'>-	if (!field_avail(cmd, driver_qp_type, udata-&gt;inlen)) {</div><div class='add'>+	if (offsetofend(typeof(cmd), driver_qp_type) &gt; udata-&gt;inlen) {</div><div class='ctx'> 		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='ctx'> 			  "Incompatible ABI params, no input udata\n");</div><div class='ctx'> 		err = -EINVAL;</div><div class='hunk'>@@ -648,21 +723,25 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd,</div><div class='ctx'> 		goto err_out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (cmd.comp_mask) {</div><div class='add'>+	if (cmd.comp_mask || !is_reserved_cleared(cmd.reserved_98)) {</div><div class='ctx'> 		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='ctx'> 			  "Incompatible ABI params, unknown fields in udata\n");</div><div class='ctx'> 		err = -EINVAL;</div><div class='ctx'> 		goto err_out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	qp = kzalloc(sizeof(*qp), GFP_KERNEL);</div><div class='del'>-	if (!qp) {</div><div class='del'>-		err = -ENOMEM;</div><div class='add'>+	if (EFA_DEV_CAP(dev, UNSOLICITED_WRITE_RECV))</div><div class='add'>+		supported_efa_flags |= EFA_CREATE_QP_WITH_UNSOLICITED_WRITE_RECV;</div><div class='add'>+</div><div class='add'>+	if (cmd.flags &amp; ~supported_efa_flags) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev, "Unsupported EFA QP create flags[%#x], supported[%#x]\n",</div><div class='add'>+			  cmd.flags, supported_efa_flags);</div><div class='add'>+		err = -EOPNOTSUPP;</div><div class='ctx'> 		goto err_out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	create_qp_params.uarn = ucontext-&gt;uarn;</div><div class='del'>-	create_qp_params.pd = to_epd(ibpd)-&gt;pdn;</div><div class='add'>+	create_qp_params.pd = to_epd(ibqp-&gt;pd)-&gt;pdn;</div><div class='ctx'> </div><div class='ctx'> 	if (init_attr-&gt;qp_type == IB_QPT_UD) {</div><div class='ctx'> 		create_qp_params.qp_type = EFA_ADMIN_QP_TYPE_UD;</div><div class='hunk'>@@ -673,7 +752,7 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd,</div><div class='ctx'> 			  "Unsupported qp type %d driver qp type %d\n",</div><div class='ctx'> 			  init_attr-&gt;qp_type, cmd.driver_qp_type);</div><div class='ctx'> 		err = -EOPNOTSUPP;</div><div class='del'>-		goto err_free_qp;</div><div class='add'>+		goto err_out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ibdev_dbg(&amp;dev-&gt;ibdev, "Create QP: qp type %d driver qp type %#x\n",</div><div class='hunk'>@@ -691,7 +770,7 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd,</div><div class='ctx'> 						    qp-&gt;rq_size, DMA_TO_DEVICE);</div><div class='ctx'> 		if (!qp-&gt;rq_cpu_addr) {</div><div class='ctx'> 			err = -ENOMEM;</div><div class='del'>-			goto err_free_qp;</div><div class='add'>+			goto err_out;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='ctx'> 		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='hunk'>@@ -700,6 +779,11 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd,</div><div class='ctx'> 		create_qp_params.rq_base_addr = qp-&gt;rq_dma_addr;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	create_qp_params.sl = cmd.sl;</div><div class='add'>+</div><div class='add'>+	if (cmd.flags &amp; EFA_CREATE_QP_WITH_UNSOLICITED_WRITE_RECV)</div><div class='add'>+		create_qp_params.unsolicited_write_recv = true;</div><div class='add'>+</div><div class='ctx'> 	err = efa_com_create_qp(&amp;dev-&gt;edev, &amp;create_qp_params,</div><div class='ctx'> 				&amp;create_qp_resp);</div><div class='ctx'> 	if (err)</div><div class='hunk'>@@ -716,10 +800,8 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd,</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_destroy_qp;</div><div class='ctx'> </div><div class='del'>-	rq_entry_inserted = true;</div><div class='ctx'> 	qp-&gt;qp_handle = create_qp_resp.qp_handle;</div><div class='ctx'> 	qp-&gt;ibqp.qp_num = create_qp_resp.qp_num;</div><div class='del'>-	qp-&gt;ibqp.qp_type = init_attr-&gt;qp_type;</div><div class='ctx'> 	qp-&gt;max_send_wr = init_attr-&gt;cap.max_send_wr;</div><div class='ctx'> 	qp-&gt;max_recv_wr = init_attr-&gt;cap.max_recv_wr;</div><div class='ctx'> 	qp-&gt;max_send_sge = init_attr-&gt;cap.max_send_sge;</div><div class='hunk'>@@ -733,28 +815,134 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd,</div><div class='ctx'> 			ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='ctx'> 				  "Failed to copy udata for qp[%u]\n",</div><div class='ctx'> 				  create_qp_resp.qp_num);</div><div class='del'>-			goto err_destroy_qp;</div><div class='add'>+			goto err_remove_mmap_entries;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ibdev_dbg(&amp;dev-&gt;ibdev, "Created qp[%d]\n", qp-&gt;ibqp.qp_num);</div><div class='ctx'> </div><div class='del'>-	return &amp;qp-&gt;ibqp;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='add'>+err_remove_mmap_entries:</div><div class='add'>+	efa_qp_user_mmap_entries_remove(qp);</div><div class='ctx'> err_destroy_qp:</div><div class='ctx'> 	efa_destroy_qp_handle(dev, create_qp_resp.qp_handle);</div><div class='ctx'> err_free_mapped:</div><div class='del'>-	if (qp-&gt;rq_size) {</div><div class='del'>-		dma_unmap_single(&amp;dev-&gt;pdev-&gt;dev, qp-&gt;rq_dma_addr, qp-&gt;rq_size,</div><div class='del'>-				 DMA_TO_DEVICE);</div><div class='del'>-		if (!rq_entry_inserted)</div><div class='del'>-			free_pages_exact(qp-&gt;rq_cpu_addr, qp-&gt;rq_size);</div><div class='del'>-	}</div><div class='del'>-err_free_qp:</div><div class='del'>-	kfree(qp);</div><div class='add'>+	if (qp-&gt;rq_size)</div><div class='add'>+		efa_free_mapped(dev, qp-&gt;rq_cpu_addr, qp-&gt;rq_dma_addr,</div><div class='add'>+				qp-&gt;rq_size, DMA_TO_DEVICE);</div><div class='ctx'> err_out:</div><div class='del'>-	atomic64_inc(&amp;dev-&gt;stats.sw_stats.create_qp_err);</div><div class='del'>-	return ERR_PTR(err);</div><div class='add'>+	atomic64_inc(&amp;dev-&gt;stats.create_qp_err);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct {</div><div class='add'>+	int			valid;</div><div class='add'>+	enum ib_qp_attr_mask	req_param;</div><div class='add'>+	enum ib_qp_attr_mask	opt_param;</div><div class='add'>+} srd_qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {</div><div class='add'>+	[IB_QPS_RESET] = {</div><div class='add'>+		[IB_QPS_RESET] = { .valid = 1 },</div><div class='add'>+		[IB_QPS_INIT]  = {</div><div class='add'>+			.valid = 1,</div><div class='add'>+			.req_param = IB_QP_PKEY_INDEX |</div><div class='add'>+				     IB_QP_PORT |</div><div class='add'>+				     IB_QP_QKEY,</div><div class='add'>+		},</div><div class='add'>+	},</div><div class='add'>+	[IB_QPS_INIT] = {</div><div class='add'>+		[IB_QPS_RESET] = { .valid = 1 },</div><div class='add'>+		[IB_QPS_ERR]   = { .valid = 1 },</div><div class='add'>+		[IB_QPS_INIT]  = {</div><div class='add'>+			.valid = 1,</div><div class='add'>+			.opt_param = IB_QP_PKEY_INDEX |</div><div class='add'>+				     IB_QP_PORT |</div><div class='add'>+				     IB_QP_QKEY,</div><div class='add'>+		},</div><div class='add'>+		[IB_QPS_RTR]   = {</div><div class='add'>+			.valid = 1,</div><div class='add'>+			.opt_param = IB_QP_PKEY_INDEX |</div><div class='add'>+				     IB_QP_QKEY,</div><div class='add'>+		},</div><div class='add'>+	},</div><div class='add'>+	[IB_QPS_RTR] = {</div><div class='add'>+		[IB_QPS_RESET] = { .valid = 1 },</div><div class='add'>+		[IB_QPS_ERR]   = { .valid = 1 },</div><div class='add'>+		[IB_QPS_RTS]   = {</div><div class='add'>+			.valid = 1,</div><div class='add'>+			.req_param = IB_QP_SQ_PSN,</div><div class='add'>+			.opt_param = IB_QP_CUR_STATE |</div><div class='add'>+				     IB_QP_QKEY |</div><div class='add'>+				     IB_QP_RNR_RETRY,</div><div class='add'>+</div><div class='add'>+		}</div><div class='add'>+	},</div><div class='add'>+	[IB_QPS_RTS] = {</div><div class='add'>+		[IB_QPS_RESET] = { .valid = 1 },</div><div class='add'>+		[IB_QPS_ERR]   = { .valid = 1 },</div><div class='add'>+		[IB_QPS_RTS]   = {</div><div class='add'>+			.valid = 1,</div><div class='add'>+			.opt_param = IB_QP_CUR_STATE |</div><div class='add'>+				     IB_QP_QKEY,</div><div class='add'>+		},</div><div class='add'>+		[IB_QPS_SQD] = {</div><div class='add'>+			.valid = 1,</div><div class='add'>+			.opt_param = IB_QP_EN_SQD_ASYNC_NOTIFY,</div><div class='add'>+		},</div><div class='add'>+	},</div><div class='add'>+	[IB_QPS_SQD] = {</div><div class='add'>+		[IB_QPS_RESET] = { .valid = 1 },</div><div class='add'>+		[IB_QPS_ERR]   = { .valid = 1 },</div><div class='add'>+		[IB_QPS_RTS]   = {</div><div class='add'>+			.valid = 1,</div><div class='add'>+			.opt_param = IB_QP_CUR_STATE |</div><div class='add'>+				     IB_QP_QKEY,</div><div class='add'>+		},</div><div class='add'>+		[IB_QPS_SQD] = {</div><div class='add'>+			.valid = 1,</div><div class='add'>+			.opt_param = IB_QP_PKEY_INDEX |</div><div class='add'>+				     IB_QP_QKEY,</div><div class='add'>+		}</div><div class='add'>+	},</div><div class='add'>+	[IB_QPS_SQE] = {</div><div class='add'>+		[IB_QPS_RESET] = { .valid = 1 },</div><div class='add'>+		[IB_QPS_ERR]   = { .valid = 1 },</div><div class='add'>+		[IB_QPS_RTS]   = {</div><div class='add'>+			.valid = 1,</div><div class='add'>+			.opt_param = IB_QP_CUR_STATE |</div><div class='add'>+				     IB_QP_QKEY,</div><div class='add'>+		}</div><div class='add'>+	},</div><div class='add'>+	[IB_QPS_ERR] = {</div><div class='add'>+		[IB_QPS_RESET] = { .valid = 1 },</div><div class='add'>+		[IB_QPS_ERR]   = { .valid = 1 },</div><div class='add'>+	}</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static bool efa_modify_srd_qp_is_ok(enum ib_qp_state cur_state,</div><div class='add'>+				    enum ib_qp_state next_state,</div><div class='add'>+				    enum ib_qp_attr_mask mask)</div><div class='add'>+{</div><div class='add'>+	enum ib_qp_attr_mask req_param, opt_param;</div><div class='add'>+</div><div class='add'>+	if (mask &amp; IB_QP_CUR_STATE  &amp;&amp;</div><div class='add'>+	    cur_state != IB_QPS_RTR &amp;&amp; cur_state != IB_QPS_RTS &amp;&amp;</div><div class='add'>+	    cur_state != IB_QPS_SQD &amp;&amp; cur_state != IB_QPS_SQE)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	if (!srd_qp_state_table[cur_state][next_state].valid)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	req_param = srd_qp_state_table[cur_state][next_state].req_param;</div><div class='add'>+	opt_param = srd_qp_state_table[cur_state][next_state].opt_param;</div><div class='add'>+</div><div class='add'>+	if ((mask &amp; req_param) != req_param)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	if (mask &amp; ~(req_param | opt_param | IB_QP_STATE))</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int efa_modify_qp_validate(struct efa_dev *dev, struct efa_qp *qp,</div><div class='hunk'>@@ -762,9 +950,12 @@ static int efa_modify_qp_validate(struct efa_dev *dev, struct efa_qp *qp,</div><div class='ctx'> 				  enum ib_qp_state cur_state,</div><div class='ctx'> 				  enum ib_qp_state new_state)</div><div class='ctx'> {</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='ctx'> #define EFA_MODIFY_QP_SUPP_MASK \</div><div class='ctx'> 	(IB_QP_STATE | IB_QP_CUR_STATE | IB_QP_EN_SQD_ASYNC_NOTIFY | \</div><div class='del'>-	 IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_QKEY | IB_QP_SQ_PSN)</div><div class='add'>+	 IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_QKEY | IB_QP_SQ_PSN | \</div><div class='add'>+	 IB_QP_RNR_RETRY)</div><div class='ctx'> </div><div class='ctx'> 	if (qp_attr_mask &amp; ~EFA_MODIFY_QP_SUPP_MASK) {</div><div class='ctx'> 		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='hunk'>@@ -773,8 +964,14 @@ static int efa_modify_qp_validate(struct efa_dev *dev, struct efa_qp *qp,</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!ib_modify_qp_is_ok(cur_state, new_state, IB_QPT_UD,</div><div class='del'>-				qp_attr_mask)) {</div><div class='add'>+	if (qp-&gt;ibqp.qp_type == IB_QPT_DRIVER)</div><div class='add'>+		err = !efa_modify_srd_qp_is_ok(cur_state, new_state,</div><div class='add'>+					       qp_attr_mask);</div><div class='add'>+	else</div><div class='add'>+		err = !ib_modify_qp_is_ok(cur_state, new_state, IB_QPT_UD,</div><div class='add'>+					  qp_attr_mask);</div><div class='add'>+</div><div class='add'>+	if (err) {</div><div class='ctx'> 		ibdev_dbg(&amp;dev-&gt;ibdev, "Invalid modify QP parameters\n");</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -802,6 +999,9 @@ int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 	enum ib_qp_state new_state;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='add'>+	if (qp_attr_mask &amp; ~IB_QP_ATTR_STANDARD_BITS)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='ctx'> 	if (udata-&gt;inlen &amp;&amp;</div><div class='ctx'> 	    !ib_is_udata_cleared(udata, 0, udata-&gt;inlen)) {</div><div class='ctx'> 		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='hunk'>@@ -821,28 +1021,36 @@ int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 	params.qp_handle = qp-&gt;qp_handle;</div><div class='ctx'> </div><div class='ctx'> 	if (qp_attr_mask &amp; IB_QP_STATE) {</div><div class='del'>-		params.modify_mask |= BIT(EFA_ADMIN_QP_STATE_BIT) |</div><div class='del'>-				      BIT(EFA_ADMIN_CUR_QP_STATE_BIT);</div><div class='del'>-		params.cur_qp_state = qp_attr-&gt;cur_qp_state;</div><div class='del'>-		params.qp_state = qp_attr-&gt;qp_state;</div><div class='add'>+		EFA_SET(&amp;params.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_QP_STATE,</div><div class='add'>+			1);</div><div class='add'>+		EFA_SET(&amp;params.modify_mask,</div><div class='add'>+			EFA_ADMIN_MODIFY_QP_CMD_CUR_QP_STATE, 1);</div><div class='add'>+		params.cur_qp_state = cur_state;</div><div class='add'>+		params.qp_state = new_state;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (qp_attr_mask &amp; IB_QP_EN_SQD_ASYNC_NOTIFY) {</div><div class='del'>-		params.modify_mask |=</div><div class='del'>-			BIT(EFA_ADMIN_SQ_DRAINED_ASYNC_NOTIFY_BIT);</div><div class='add'>+		EFA_SET(&amp;params.modify_mask,</div><div class='add'>+			EFA_ADMIN_MODIFY_QP_CMD_SQ_DRAINED_ASYNC_NOTIFY, 1);</div><div class='ctx'> 		params.sq_drained_async_notify = qp_attr-&gt;en_sqd_async_notify;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (qp_attr_mask &amp; IB_QP_QKEY) {</div><div class='del'>-		params.modify_mask |= BIT(EFA_ADMIN_QKEY_BIT);</div><div class='add'>+		EFA_SET(&amp;params.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_QKEY, 1);</div><div class='ctx'> 		params.qkey = qp_attr-&gt;qkey;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (qp_attr_mask &amp; IB_QP_SQ_PSN) {</div><div class='del'>-		params.modify_mask |= BIT(EFA_ADMIN_SQ_PSN_BIT);</div><div class='add'>+		EFA_SET(&amp;params.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_SQ_PSN, 1);</div><div class='ctx'> 		params.sq_psn = qp_attr-&gt;sq_psn;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (qp_attr_mask &amp; IB_QP_RNR_RETRY) {</div><div class='add'>+		EFA_SET(&amp;params.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_RNR_RETRY,</div><div class='add'>+			1);</div><div class='add'>+		params.rnr_retry = qp_attr-&gt;rnr_retry;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	err = efa_com_modify_qp(&amp;dev-&gt;edev, &amp;params);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		return err;</div><div class='hunk'>@@ -859,61 +1067,92 @@ static int efa_destroy_cq_idx(struct efa_dev *dev, int cq_idx)</div><div class='ctx'> 	return efa_com_destroy_cq(&amp;dev-&gt;edev, &amp;params);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void efa_cq_user_mmap_entries_remove(struct efa_cq *cq)</div><div class='add'>+{</div><div class='add'>+	rdma_user_mmap_entry_remove(cq-&gt;db_mmap_entry);</div><div class='add'>+	rdma_user_mmap_entry_remove(cq-&gt;mmap_entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_dev *dev = to_edev(ibcq-&gt;device);</div><div class='ctx'> 	struct efa_cq *cq = to_ecq(ibcq);</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	if (udata-&gt;inlen &amp;&amp;</div><div class='del'>-	    !ib_is_udata_cleared(udata, 0, udata-&gt;inlen)) {</div><div class='del'>-		ibdev_dbg(&amp;dev-&gt;ibdev, "Incompatible ABI params\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='ctx'> 	ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='ctx'> 		  "Destroy cq[%d] virt[0x%p] freed: size[%lu], dma[%pad]\n",</div><div class='ctx'> 		  cq-&gt;cq_idx, cq-&gt;cpu_addr, cq-&gt;size, &amp;cq-&gt;dma_addr);</div><div class='ctx'> </div><div class='del'>-	err = efa_destroy_cq_idx(dev, cq-&gt;cq_idx);</div><div class='del'>-	if (err)</div><div class='del'>-		return err;</div><div class='del'>-</div><div class='del'>-	dma_unmap_single(&amp;dev-&gt;pdev-&gt;dev, cq-&gt;dma_addr, cq-&gt;size,</div><div class='del'>-			 DMA_FROM_DEVICE);</div><div class='add'>+	efa_destroy_cq_idx(dev, cq-&gt;cq_idx);</div><div class='add'>+	efa_cq_user_mmap_entries_remove(cq);</div><div class='add'>+	if (cq-&gt;eq) {</div><div class='add'>+		xa_erase(&amp;dev-&gt;cqs_xa, cq-&gt;cq_idx);</div><div class='add'>+		synchronize_irq(cq-&gt;eq-&gt;irq.irqn);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	kfree(cq);</div><div class='add'>+	if (cq-&gt;umem)</div><div class='add'>+		ib_umem_release(cq-&gt;umem);</div><div class='add'>+	else</div><div class='add'>+		efa_free_mapped(dev, cq-&gt;cpu_addr, cq-&gt;dma_addr, cq-&gt;size, DMA_FROM_DEVICE);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static struct efa_eq *efa_vec2eq(struct efa_dev *dev, int vec)</div><div class='add'>+{</div><div class='add'>+	return &amp;dev-&gt;eqs[vec];</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int cq_mmap_entries_setup(struct efa_dev *dev, struct efa_cq *cq,</div><div class='del'>-				 struct efa_ibv_create_cq_resp *resp)</div><div class='add'>+				 struct efa_ibv_create_cq_resp *resp,</div><div class='add'>+				 bool db_valid)</div><div class='ctx'> {</div><div class='ctx'> 	resp-&gt;q_mmap_size = cq-&gt;size;</div><div class='del'>-	resp-&gt;q_mmap_key = mmap_entry_insert(dev, cq-&gt;ucontext, cq,</div><div class='del'>-					     virt_to_phys(cq-&gt;cpu_addr),</div><div class='del'>-					     cq-&gt;size, EFA_MMAP_DMA_PAGE);</div><div class='del'>-	if (resp-&gt;q_mmap_key == EFA_MMAP_INVALID)</div><div class='add'>+	cq-&gt;mmap_entry = efa_user_mmap_entry_insert(&amp;cq-&gt;ucontext-&gt;ibucontext,</div><div class='add'>+						    virt_to_phys(cq-&gt;cpu_addr),</div><div class='add'>+						    cq-&gt;size, EFA_MMAP_DMA_PAGE,</div><div class='add'>+						    &amp;resp-&gt;q_mmap_key);</div><div class='add'>+	if (!cq-&gt;mmap_entry)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='add'>+	if (db_valid) {</div><div class='add'>+		cq-&gt;db_mmap_entry =</div><div class='add'>+			efa_user_mmap_entry_insert(&amp;cq-&gt;ucontext-&gt;ibucontext,</div><div class='add'>+						   dev-&gt;db_bar_addr + resp-&gt;db_off,</div><div class='add'>+						   PAGE_SIZE, EFA_MMAP_IO_NC,</div><div class='add'>+						   &amp;resp-&gt;db_mmap_key);</div><div class='add'>+		if (!cq-&gt;db_mmap_entry) {</div><div class='add'>+			rdma_user_mmap_entry_remove(cq-&gt;mmap_entry);</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		resp-&gt;db_off &amp;= ~PAGE_MASK;</div><div class='add'>+		resp-&gt;comp_mask |= EFA_CREATE_CQ_RESP_DB_OFF;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries,</div><div class='del'>-				  int vector, struct ib_ucontext *ibucontext,</div><div class='del'>-				  struct ib_udata *udata)</div><div class='add'>+int efa_create_cq_umem(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		       struct ib_umem *umem, struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='add'>+	struct ib_udata *udata = &amp;attrs-&gt;driver_udata;</div><div class='add'>+	struct efa_ucontext *ucontext = rdma_udata_to_drv_context(</div><div class='add'>+		udata, struct efa_ucontext, ibucontext);</div><div class='add'>+	struct efa_com_create_cq_params params = {};</div><div class='ctx'> 	struct efa_ibv_create_cq_resp resp = {};</div><div class='del'>-	struct efa_com_create_cq_params params;</div><div class='ctx'> 	struct efa_com_create_cq_result result;</div><div class='add'>+	struct ib_device *ibdev = ibcq-&gt;device;</div><div class='ctx'> 	struct efa_dev *dev = to_edev(ibdev);</div><div class='ctx'> 	struct efa_ibv_create_cq cmd = {};</div><div class='del'>-	bool cq_entry_inserted = false;</div><div class='del'>-	struct efa_cq *cq;</div><div class='add'>+	struct efa_cq *cq = to_ecq(ibcq);</div><div class='add'>+	int entries = attr-&gt;cqe;</div><div class='add'>+	bool set_src_addr;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	ibdev_dbg(ibdev, "create_cq entries %d\n", entries);</div><div class='ctx'> </div><div class='add'>+	if (attr-&gt;flags)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='ctx'> 	if (entries &lt; 1 || entries &gt; dev-&gt;dev_attr.max_cq_depth) {</div><div class='ctx'> 		ibdev_dbg(ibdev,</div><div class='ctx'> 			  "cq: requested entries[%u] non-positive or greater than max[%u]\n",</div><div class='hunk'>@@ -922,7 +1161,7 @@ static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries,</div><div class='ctx'> 		goto err_out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!field_avail(cmd, num_sub_cqs, udata-&gt;inlen)) {</div><div class='add'>+	if (offsetofend(typeof(cmd), num_sub_cqs) &gt; udata-&gt;inlen) {</div><div class='ctx'> 		ibdev_dbg(ibdev,</div><div class='ctx'> 			  "Incompatible ABI params, no input udata\n");</div><div class='ctx'> 		err = -EINVAL;</div><div class='hunk'>@@ -945,14 +1184,17 @@ static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries,</div><div class='ctx'> 		goto err_out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (cmd.comp_mask || !is_reserved_cleared(cmd.reserved_50)) {</div><div class='add'>+	if (cmd.comp_mask || !is_reserved_cleared(cmd.reserved_58)) {</div><div class='ctx'> 		ibdev_dbg(ibdev,</div><div class='ctx'> 			  "Incompatible ABI params, unknown fields in udata\n");</div><div class='ctx'> 		err = -EINVAL;</div><div class='ctx'> 		goto err_out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!cmd.cq_entry_size) {</div><div class='add'>+	set_src_addr = !!(cmd.flags &amp; EFA_CREATE_CQ_WITH_SGID);</div><div class='add'>+	if ((cmd.cq_entry_size != sizeof(struct efa_io_rx_cdesc_ex)) &amp;&amp;</div><div class='add'>+	    (set_src_addr ||</div><div class='add'>+	     cmd.cq_entry_size != sizeof(struct efa_io_rx_cdesc))) {</div><div class='ctx'> 		ibdev_dbg(ibdev,</div><div class='ctx'> 			  "Invalid entry size [%u]\n", cmd.cq_entry_size);</div><div class='ctx'> 		err = -EINVAL;</div><div class='hunk'>@@ -967,43 +1209,73 @@ static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries,</div><div class='ctx'> 		goto err_out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	cq = kzalloc(sizeof(*cq), GFP_KERNEL);</div><div class='del'>-	if (!cq) {</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto err_out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cq-&gt;ucontext = to_eucontext(ibucontext);</div><div class='add'>+	cq-&gt;ucontext = ucontext;</div><div class='ctx'> 	cq-&gt;size = PAGE_ALIGN(cmd.cq_entry_size * entries * cmd.num_sub_cqs);</div><div class='del'>-	cq-&gt;cpu_addr = efa_zalloc_mapped(dev, &amp;cq-&gt;dma_addr, cq-&gt;size,</div><div class='del'>-					 DMA_FROM_DEVICE);</div><div class='del'>-	if (!cq-&gt;cpu_addr) {</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto err_free_cq;</div><div class='add'>+</div><div class='add'>+	if (umem) {</div><div class='add'>+		if (umem-&gt;length &lt; cq-&gt;size) {</div><div class='add'>+			ibdev_dbg(&amp;dev-&gt;ibdev, "External memory too small\n");</div><div class='add'>+			err = -EINVAL;</div><div class='add'>+			goto err_out;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (!ib_umem_is_contiguous(umem)) {</div><div class='add'>+			ibdev_dbg(&amp;dev-&gt;ibdev, "Non contiguous CQ unsupported\n");</div><div class='add'>+			err = -EINVAL;</div><div class='add'>+			goto err_out;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		cq-&gt;cpu_addr = NULL;</div><div class='add'>+		cq-&gt;dma_addr = ib_umem_start_dma_addr(umem);</div><div class='add'>+		cq-&gt;umem = umem;</div><div class='add'>+	} else {</div><div class='add'>+		cq-&gt;cpu_addr = efa_zalloc_mapped(dev, &amp;cq-&gt;dma_addr, cq-&gt;size,</div><div class='add'>+						 DMA_FROM_DEVICE);</div><div class='add'>+		if (!cq-&gt;cpu_addr) {</div><div class='add'>+			err = -ENOMEM;</div><div class='add'>+			goto err_out;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	params.uarn = cq-&gt;ucontext-&gt;uarn;</div><div class='del'>-	params.cq_depth = entries;</div><div class='add'>+	params.sub_cq_depth = entries;</div><div class='ctx'> 	params.dma_addr = cq-&gt;dma_addr;</div><div class='ctx'> 	params.entry_size_in_bytes = cmd.cq_entry_size;</div><div class='ctx'> 	params.num_sub_cqs = cmd.num_sub_cqs;</div><div class='add'>+	params.set_src_addr = set_src_addr;</div><div class='add'>+	if (cmd.flags &amp; EFA_CREATE_CQ_WITH_COMPLETION_CHANNEL) {</div><div class='add'>+		cq-&gt;eq = efa_vec2eq(dev, attr-&gt;comp_vector);</div><div class='add'>+		params.eqn = cq-&gt;eq-&gt;eeq.eqn;</div><div class='add'>+		params.interrupt_mode_enabled = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	err = efa_com_create_cq(&amp;dev-&gt;edev, &amp;params, &amp;result);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_free_mapped;</div><div class='ctx'> </div><div class='add'>+	resp.db_off = result.db_off;</div><div class='ctx'> 	resp.cq_idx = result.cq_idx;</div><div class='ctx'> 	cq-&gt;cq_idx = result.cq_idx;</div><div class='ctx'> 	cq-&gt;ibcq.cqe = result.actual_depth;</div><div class='ctx'> 	WARN_ON_ONCE(entries != result.actual_depth);</div><div class='ctx'> </div><div class='del'>-	err = cq_mmap_entries_setup(dev, cq, &amp;resp);</div><div class='add'>+	if (!umem)</div><div class='add'>+		err = cq_mmap_entries_setup(dev, cq, &amp;resp, result.db_valid);</div><div class='add'>+</div><div class='ctx'> 	if (err) {</div><div class='del'>-		ibdev_dbg(ibdev,</div><div class='del'>-			  "Could not setup cq[%u] mmap entries\n", cq-&gt;cq_idx);</div><div class='add'>+		ibdev_dbg(ibdev, "Could not setup cq[%u] mmap entries\n",</div><div class='add'>+			  cq-&gt;cq_idx);</div><div class='ctx'> 		goto err_destroy_cq;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	cq_entry_inserted = true;</div><div class='add'>+	if (cq-&gt;eq) {</div><div class='add'>+		err = xa_err(xa_store(&amp;dev-&gt;cqs_xa, cq-&gt;cq_idx, cq, GFP_KERNEL));</div><div class='add'>+		if (err) {</div><div class='add'>+			ibdev_dbg(ibdev, "Failed to store cq[%u] in xarray\n",</div><div class='add'>+				  cq-&gt;cq_idx);</div><div class='add'>+			goto err_remove_mmap;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	if (udata-&gt;outlen) {</div><div class='ctx'> 		err = ib_copy_to_udata(udata, &amp;resp,</div><div class='hunk'>@@ -1011,40 +1283,35 @@ static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries,</div><div class='ctx'> 		if (err) {</div><div class='ctx'> 			ibdev_dbg(ibdev,</div><div class='ctx'> 				  "Failed to copy udata for create_cq\n");</div><div class='del'>-			goto err_destroy_cq;</div><div class='add'>+			goto err_xa_erase;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ibdev_dbg(ibdev,</div><div class='del'>-		  "Created cq[%d], cq depth[%u]. dma[%pad] virt[0x%p]\n",</div><div class='add'>+	ibdev_dbg(ibdev, "Created cq[%d], cq depth[%u]. dma[%pad] virt[0x%p]\n",</div><div class='ctx'> 		  cq-&gt;cq_idx, result.actual_depth, &amp;cq-&gt;dma_addr, cq-&gt;cpu_addr);</div><div class='ctx'> </div><div class='del'>-	return &amp;cq-&gt;ibcq;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='add'>+err_xa_erase:</div><div class='add'>+	if (cq-&gt;eq)</div><div class='add'>+		xa_erase(&amp;dev-&gt;cqs_xa, cq-&gt;cq_idx);</div><div class='add'>+err_remove_mmap:</div><div class='add'>+	efa_cq_user_mmap_entries_remove(cq);</div><div class='ctx'> err_destroy_cq:</div><div class='ctx'> 	efa_destroy_cq_idx(dev, cq-&gt;cq_idx);</div><div class='ctx'> err_free_mapped:</div><div class='del'>-	dma_unmap_single(&amp;dev-&gt;pdev-&gt;dev, cq-&gt;dma_addr, cq-&gt;size,</div><div class='del'>-			 DMA_FROM_DEVICE);</div><div class='del'>-	if (!cq_entry_inserted)</div><div class='del'>-		free_pages_exact(cq-&gt;cpu_addr, cq-&gt;size);</div><div class='del'>-err_free_cq:</div><div class='del'>-	kfree(cq);</div><div class='add'>+	if (!umem)</div><div class='add'>+		efa_free_mapped(dev, cq-&gt;cpu_addr, cq-&gt;dma_addr, cq-&gt;size,</div><div class='add'>+				DMA_FROM_DEVICE);</div><div class='ctx'> err_out:</div><div class='del'>-	atomic64_inc(&amp;dev-&gt;stats.sw_stats.create_cq_err);</div><div class='del'>-	return ERR_PTR(err);</div><div class='add'>+	atomic64_inc(&amp;dev-&gt;stats.create_cq_err);</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_cq *efa_create_cq(struct ib_device *ibdev,</div><div class='del'>-			    const struct ib_cq_init_attr *attr,</div><div class='del'>-			    struct ib_udata *udata)</div><div class='add'>+int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		  struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='del'>-	struct efa_ucontext *ucontext = rdma_udata_to_drv_context(udata,</div><div class='del'>-								  struct efa_ucontext,</div><div class='del'>-								  ibucontext);</div><div class='del'>-</div><div class='del'>-	return do_create_cq(ibdev, attr-&gt;cqe, attr-&gt;comp_vector,</div><div class='del'>-			    &amp;ucontext-&gt;ibucontext, udata);</div><div class='add'>+	return efa_create_cq_umem(ibcq, attr, NULL, attrs);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int umem_to_page_list(struct efa_dev *dev,</div><div class='hunk'>@@ -1054,21 +1321,14 @@ static int umem_to_page_list(struct efa_dev *dev,</div><div class='ctx'> 			     u8 hp_shift)</div><div class='ctx'> {</div><div class='ctx'> 	u32 pages_in_hp = BIT(hp_shift - PAGE_SHIFT);</div><div class='del'>-	struct sg_dma_page_iter sg_iter;</div><div class='del'>-	unsigned int page_idx = 0;</div><div class='add'>+	struct ib_block_iter biter;</div><div class='ctx'> 	unsigned int hp_idx = 0;</div><div class='ctx'> </div><div class='ctx'> 	ibdev_dbg(&amp;dev-&gt;ibdev, "hp_cnt[%u], pages_in_hp[%u]\n",</div><div class='ctx'> 		  hp_cnt, pages_in_hp);</div><div class='ctx'> </div><div class='del'>-	for_each_sg_dma_page(umem-&gt;sg_head.sgl, &amp;sg_iter, umem-&gt;nmap, 0) {</div><div class='del'>-		if (page_idx % pages_in_hp == 0) {</div><div class='del'>-			page_list[hp_idx] = sg_page_iter_dma_address(&amp;sg_iter);</div><div class='del'>-			hp_idx++;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		page_idx++;</div><div class='del'>-	}</div><div class='add'>+	rdma_umem_for_each_dma_block(umem, &amp;biter, BIT(hp_shift))</div><div class='add'>+		page_list[hp_idx++] = rdma_block_iter_dma_address(&amp;biter);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -1079,7 +1339,7 @@ static struct scatterlist *efa_vmalloc_buf_to_sg(u64 *buf, int page_cnt)</div><div class='ctx'> 	struct page *pg;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	sglist = kcalloc(page_cnt, sizeof(*sglist), GFP_KERNEL);</div><div class='add'>+	sglist = kmalloc_array(page_cnt, sizeof(*sglist), GFP_KERNEL);</div><div class='ctx'> 	if (!sglist)</div><div class='ctx'> 		return NULL;</div><div class='ctx'> 	sg_init_table(sglist, page_cnt);</div><div class='hunk'>@@ -1103,14 +1363,14 @@ err:</div><div class='ctx'>  */</div><div class='ctx'> static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl)</div><div class='ctx'> {</div><div class='del'>-	unsigned int entry, payloads_in_sg, chunk_list_size, chunk_idx, payload_idx;</div><div class='ctx'> 	struct pbl_chunk_list *chunk_list = &amp;pbl-&gt;phys.indirect.chunk_list;</div><div class='ctx'> 	int page_cnt = pbl-&gt;phys.indirect.pbl_buf_size_in_pages;</div><div class='ctx'> 	struct scatterlist *pages_sgl = pbl-&gt;phys.indirect.sgl;</div><div class='add'>+	unsigned int chunk_list_size, chunk_idx, payload_idx;</div><div class='ctx'> 	int sg_dma_cnt = pbl-&gt;phys.indirect.sg_dma_cnt;</div><div class='ctx'> 	struct efa_com_ctrl_buff_info *ctrl_buf;</div><div class='ctx'> 	u64 *cur_chunk_buf, *prev_chunk_buf;</div><div class='del'>-	struct scatterlist *sg;</div><div class='add'>+	struct ib_block_iter biter;</div><div class='ctx'> 	dma_addr_t dma_addr;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='hunk'>@@ -1144,18 +1404,15 @@ static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl)</div><div class='ctx'> 	chunk_idx = 0;</div><div class='ctx'> 	payload_idx = 0;</div><div class='ctx'> 	cur_chunk_buf = chunk_list-&gt;chunks[0].buf;</div><div class='del'>-	for_each_sg(pages_sgl, sg, sg_dma_cnt, entry) {</div><div class='del'>-		payloads_in_sg = sg_dma_len(sg) &gt;&gt; EFA_CHUNK_PAYLOAD_SHIFT;</div><div class='del'>-		for (i = 0; i &lt; payloads_in_sg; i++) {</div><div class='del'>-			cur_chunk_buf[payload_idx++] =</div><div class='del'>-				(sg_dma_address(sg) &amp; ~(EFA_CHUNK_PAYLOAD_SIZE - 1)) +</div><div class='del'>-				(EFA_CHUNK_PAYLOAD_SIZE * i);</div><div class='del'>-</div><div class='del'>-			if (payload_idx == EFA_PTRS_PER_CHUNK) {</div><div class='del'>-				chunk_idx++;</div><div class='del'>-				cur_chunk_buf = chunk_list-&gt;chunks[chunk_idx].buf;</div><div class='del'>-				payload_idx = 0;</div><div class='del'>-			}</div><div class='add'>+	rdma_for_each_block(pages_sgl, &amp;biter, sg_dma_cnt,</div><div class='add'>+			    EFA_CHUNK_PAYLOAD_SIZE) {</div><div class='add'>+		cur_chunk_buf[payload_idx++] =</div><div class='add'>+			rdma_block_iter_dma_address(&amp;biter);</div><div class='add'>+</div><div class='add'>+		if (payload_idx == EFA_PTRS_PER_CHUNK) {</div><div class='add'>+			chunk_idx++;</div><div class='add'>+			cur_chunk_buf = chunk_list-&gt;chunks[chunk_idx].buf;</div><div class='add'>+			payload_idx = 0;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -1246,7 +1503,7 @@ static int pbl_continuous_initialize(struct efa_dev *dev,</div><div class='ctx'>  */</div><div class='ctx'> static int pbl_indirect_initialize(struct efa_dev *dev, struct pbl_context *pbl)</div><div class='ctx'> {</div><div class='del'>-	u32 size_in_pages = DIV_ROUND_UP(pbl-&gt;pbl_buf_size_in_bytes, PAGE_SIZE);</div><div class='add'>+	u32 size_in_pages = DIV_ROUND_UP(pbl-&gt;pbl_buf_size_in_bytes, EFA_CHUNK_PAYLOAD_SIZE);</div><div class='ctx'> 	struct scatterlist *sgl;</div><div class='ctx'> 	int sg_dma_cnt, err;</div><div class='ctx'> </div><div class='hunk'>@@ -1303,30 +1560,30 @@ static int pbl_create(struct efa_dev *dev,</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	pbl-&gt;pbl_buf_size_in_bytes = hp_cnt * EFA_CHUNK_PAYLOAD_PTR_SIZE;</div><div class='del'>-	pbl-&gt;pbl_buf = kzalloc(pbl-&gt;pbl_buf_size_in_bytes,</div><div class='del'>-			       GFP_KERNEL | __GFP_NOWARN);</div><div class='del'>-	if (pbl-&gt;pbl_buf) {</div><div class='del'>-		pbl-&gt;physically_continuous = 1;</div><div class='add'>+	pbl-&gt;pbl_buf = kvzalloc(pbl-&gt;pbl_buf_size_in_bytes, GFP_KERNEL);</div><div class='add'>+	if (!pbl-&gt;pbl_buf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	if (is_vmalloc_addr(pbl-&gt;pbl_buf)) {</div><div class='add'>+		pbl-&gt;physically_continuous = 0;</div><div class='ctx'> 		err = umem_to_page_list(dev, umem, pbl-&gt;pbl_buf, hp_cnt,</div><div class='ctx'> 					hp_shift);</div><div class='ctx'> 		if (err)</div><div class='del'>-			goto err_continuous;</div><div class='del'>-		err = pbl_continuous_initialize(dev, pbl);</div><div class='add'>+			goto err_free;</div><div class='add'>+</div><div class='add'>+		err = pbl_indirect_initialize(dev, pbl);</div><div class='ctx'> 		if (err)</div><div class='del'>-			goto err_continuous;</div><div class='add'>+			goto err_free;</div><div class='ctx'> 	} else {</div><div class='del'>-		pbl-&gt;physically_continuous = 0;</div><div class='del'>-		pbl-&gt;pbl_buf = vzalloc(pbl-&gt;pbl_buf_size_in_bytes);</div><div class='del'>-		if (!pbl-&gt;pbl_buf)</div><div class='del'>-			return -ENOMEM;</div><div class='del'>-</div><div class='add'>+		pbl-&gt;physically_continuous = 1;</div><div class='ctx'> 		err = umem_to_page_list(dev, umem, pbl-&gt;pbl_buf, hp_cnt,</div><div class='ctx'> 					hp_shift);</div><div class='ctx'> 		if (err)</div><div class='del'>-			goto err_indirect;</div><div class='del'>-		err = pbl_indirect_initialize(dev, pbl);</div><div class='add'>+			goto err_free;</div><div class='add'>+</div><div class='add'>+		err = pbl_continuous_initialize(dev, pbl);</div><div class='ctx'> 		if (err)</div><div class='del'>-			goto err_indirect;</div><div class='add'>+			goto err_free;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='hunk'>@@ -1335,24 +1592,20 @@ static int pbl_create(struct efa_dev *dev,</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-err_continuous:</div><div class='del'>-	kfree(pbl-&gt;pbl_buf);</div><div class='del'>-	return err;</div><div class='del'>-err_indirect:</div><div class='del'>-	vfree(pbl-&gt;pbl_buf);</div><div class='add'>+err_free:</div><div class='add'>+	kvfree(pbl-&gt;pbl_buf);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void pbl_destroy(struct efa_dev *dev, struct pbl_context *pbl)</div><div class='ctx'> {</div><div class='del'>-	if (pbl-&gt;physically_continuous) {</div><div class='add'>+	if (pbl-&gt;physically_continuous)</div><div class='ctx'> 		dma_unmap_single(&amp;dev-&gt;pdev-&gt;dev, pbl-&gt;phys.continuous.dma_addr,</div><div class='ctx'> 				 pbl-&gt;pbl_buf_size_in_bytes, DMA_TO_DEVICE);</div><div class='del'>-		kfree(pbl-&gt;pbl_buf);</div><div class='del'>-	} else {</div><div class='add'>+	else</div><div class='ctx'> 		pbl_indirect_terminate(dev, pbl);</div><div class='del'>-		vfree(pbl-&gt;pbl_buf);</div><div class='del'>-	}</div><div class='add'>+</div><div class='add'>+	kvfree(pbl-&gt;pbl_buf);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int efa_create_inline_pbl(struct efa_dev *dev, struct efa_mr *mr,</div><div class='hunk'>@@ -1406,149 +1659,231 @@ static int efa_create_pbl(struct efa_dev *dev,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void efa_cont_pages(struct ib_umem *umem, u64 addr,</div><div class='del'>-			   unsigned long max_page_shift,</div><div class='del'>-			   int *count, u8 *shift, u32 *ncont)</div><div class='del'>-{</div><div class='del'>-	struct scatterlist *sg;</div><div class='del'>-	u64 base = ~0, p = 0;</div><div class='del'>-	unsigned long tmp;</div><div class='del'>-	unsigned long m;</div><div class='del'>-	u64 len, pfn;</div><div class='del'>-	int i = 0;</div><div class='del'>-	int entry;</div><div class='del'>-</div><div class='del'>-	addr = addr &gt;&gt; PAGE_SHIFT;</div><div class='del'>-	tmp = (unsigned long)addr;</div><div class='del'>-	m = find_first_bit(&amp;tmp, BITS_PER_LONG);</div><div class='del'>-	if (max_page_shift)</div><div class='del'>-		m = min_t(unsigned long, max_page_shift - PAGE_SHIFT, m);</div><div class='del'>-</div><div class='del'>-	for_each_sg(umem-&gt;sg_head.sgl, sg, umem-&gt;nmap, entry) {</div><div class='del'>-		len = DIV_ROUND_UP(sg_dma_len(sg), PAGE_SIZE);</div><div class='del'>-		pfn = sg_dma_address(sg) &gt;&gt; PAGE_SHIFT;</div><div class='del'>-		if (base + p != pfn) {</div><div class='del'>-			/*</div><div class='del'>-			 * If either the offset or the new</div><div class='del'>-			 * base are unaligned update m</div><div class='del'>-			 */</div><div class='del'>-			tmp = (unsigned long)(pfn | p);</div><div class='del'>-			if (!IS_ALIGNED(tmp, 1 &lt;&lt; m))</div><div class='del'>-				m = find_first_bit(&amp;tmp, BITS_PER_LONG);</div><div class='del'>-</div><div class='del'>-			base = pfn;</div><div class='del'>-			p = 0;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		p += len;</div><div class='del'>-		i += len;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (i) {</div><div class='del'>-		m = min_t(unsigned long, ilog2(roundup_pow_of_two(i)), m);</div><div class='del'>-		*ncont = DIV_ROUND_UP(i, (1 &lt;&lt; m));</div><div class='del'>-	} else {</div><div class='del'>-		m = 0;</div><div class='del'>-		*ncont = 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	*shift = PAGE_SHIFT + m;</div><div class='del'>-	*count = i;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,</div><div class='del'>-			 u64 virt_addr, int access_flags,</div><div class='del'>-			 struct ib_udata *udata)</div><div class='add'>+static struct efa_mr *efa_alloc_mr(struct ib_pd *ibpd, int access_flags,</div><div class='add'>+				   struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_dev *dev = to_edev(ibpd-&gt;device);</div><div class='del'>-	struct efa_com_reg_mr_params params = {};</div><div class='del'>-	struct efa_com_reg_mr_result result = {};</div><div class='del'>-	unsigned long max_page_shift;</div><div class='del'>-	struct pbl_context pbl;</div><div class='add'>+	int supp_access_flags;</div><div class='ctx'> 	struct efa_mr *mr;</div><div class='del'>-	int inline_size;</div><div class='del'>-	int npages;</div><div class='del'>-	int err;</div><div class='ctx'> </div><div class='del'>-	if (udata-&gt;inlen &amp;&amp;</div><div class='add'>+	if (udata &amp;&amp; udata-&gt;inlen &amp;&amp;</div><div class='ctx'> 	    !ib_is_udata_cleared(udata, 0, sizeof(udata-&gt;inlen))) {</div><div class='ctx'> 		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='ctx'> 			  "Incompatible ABI params, udata not cleared\n");</div><div class='del'>-		err = -EINVAL;</div><div class='del'>-		goto err_out;</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (access_flags &amp; ~EFA_SUPPORTED_ACCESS_FLAGS) {</div><div class='add'>+	supp_access_flags =</div><div class='add'>+		IB_ACCESS_LOCAL_WRITE |</div><div class='add'>+		(EFA_DEV_CAP(dev, RDMA_READ) ? IB_ACCESS_REMOTE_READ : 0) |</div><div class='add'>+		(EFA_DEV_CAP(dev, RDMA_WRITE) ? IB_ACCESS_REMOTE_WRITE : 0);</div><div class='add'>+</div><div class='add'>+	access_flags &amp;= ~IB_ACCESS_OPTIONAL;</div><div class='add'>+	if (access_flags &amp; ~supp_access_flags) {</div><div class='ctx'> 		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='ctx'> 			  "Unsupported access flags[%#x], supported[%#x]\n",</div><div class='del'>-			  access_flags, EFA_SUPPORTED_ACCESS_FLAGS);</div><div class='del'>-		err = -EOPNOTSUPP;</div><div class='del'>-		goto err_out;</div><div class='add'>+			  access_flags, supp_access_flags);</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='del'>-	if (!mr) {</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto err_out;</div><div class='del'>-	}</div><div class='add'>+	if (!mr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='del'>-	mr-&gt;umem = ib_umem_get(udata, start, length, access_flags, 0);</div><div class='del'>-	if (IS_ERR(mr-&gt;umem)) {</div><div class='del'>-		err = PTR_ERR(mr-&gt;umem);</div><div class='del'>-		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='del'>-			  "Failed to pin and map user space memory[%d]\n", err);</div><div class='del'>-		goto err_free;</div><div class='del'>-	}</div><div class='add'>+	return mr;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int efa_register_mr(struct ib_pd *ibpd, struct efa_mr *mr, u64 start,</div><div class='add'>+			   u64 length, u64 virt_addr, int access_flags)</div><div class='add'>+{</div><div class='add'>+	struct efa_dev *dev = to_edev(ibpd-&gt;device);</div><div class='add'>+	struct efa_com_reg_mr_params params = {};</div><div class='add'>+	struct efa_com_reg_mr_result result = {};</div><div class='add'>+	struct pbl_context pbl;</div><div class='add'>+	unsigned int pg_sz;</div><div class='add'>+	int inline_size;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='ctx'> 	params.pd = to_epd(ibpd)-&gt;pdn;</div><div class='ctx'> 	params.iova = virt_addr;</div><div class='ctx'> 	params.mr_length_in_bytes = length;</div><div class='del'>-	params.permissions = access_flags &amp; 0x1;</div><div class='del'>-	max_page_shift = fls64(dev-&gt;dev_attr.page_size_cap);</div><div class='add'>+	params.permissions = access_flags;</div><div class='add'>+</div><div class='add'>+	pg_sz = ib_umem_find_best_pgsz(mr-&gt;umem,</div><div class='add'>+				       dev-&gt;dev_attr.page_size_cap,</div><div class='add'>+				       virt_addr);</div><div class='add'>+	if (!pg_sz) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev, "Failed to find a suitable page size in page_size_cap %#llx\n",</div><div class='add'>+			  dev-&gt;dev_attr.page_size_cap);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	params.page_shift = order_base_2(pg_sz);</div><div class='add'>+	params.page_num = ib_umem_num_dma_blocks(mr-&gt;umem, pg_sz);</div><div class='ctx'> </div><div class='del'>-	efa_cont_pages(mr-&gt;umem, start, max_page_shift, &amp;npages,</div><div class='del'>-		       &amp;params.page_shift, &amp;params.page_num);</div><div class='ctx'> 	ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='del'>-		  "start %#llx length %#llx npages %d params.page_shift %u params.page_num %u\n",</div><div class='del'>-		  start, length, npages, params.page_shift, params.page_num);</div><div class='add'>+		  "start %#llx length %#llx params.page_shift %u params.page_num %u\n",</div><div class='add'>+		  start, length, params.page_shift, params.page_num);</div><div class='ctx'> </div><div class='ctx'> 	inline_size = ARRAY_SIZE(params.pbl.inline_pbl_array);</div><div class='ctx'> 	if (params.page_num &lt;= inline_size) {</div><div class='ctx'> 		err = efa_create_inline_pbl(dev, mr, &amp;params);</div><div class='ctx'> 		if (err)</div><div class='del'>-			goto err_unmap;</div><div class='add'>+			return err;</div><div class='ctx'> </div><div class='ctx'> 		err = efa_com_register_mr(&amp;dev-&gt;edev, &amp;params, &amp;result);</div><div class='ctx'> 		if (err)</div><div class='del'>-			goto err_unmap;</div><div class='add'>+			return err;</div><div class='ctx'> 	} else {</div><div class='ctx'> 		err = efa_create_pbl(dev, &amp;pbl, mr, &amp;params);</div><div class='ctx'> 		if (err)</div><div class='del'>-			goto err_unmap;</div><div class='add'>+			return err;</div><div class='ctx'> </div><div class='ctx'> 		err = efa_com_register_mr(&amp;dev-&gt;edev, &amp;params, &amp;result);</div><div class='ctx'> 		pbl_destroy(dev, &amp;pbl);</div><div class='ctx'> </div><div class='ctx'> 		if (err)</div><div class='del'>-			goto err_unmap;</div><div class='add'>+			return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;ibmr.lkey = result.l_key;</div><div class='ctx'> 	mr-&gt;ibmr.rkey = result.r_key;</div><div class='ctx'> 	mr-&gt;ibmr.length = length;</div><div class='add'>+	mr-&gt;ic_info.recv_ic_id = result.ic_info.recv_ic_id;</div><div class='add'>+	mr-&gt;ic_info.rdma_read_ic_id = result.ic_info.rdma_read_ic_id;</div><div class='add'>+	mr-&gt;ic_info.rdma_recv_ic_id = result.ic_info.rdma_recv_ic_id;</div><div class='add'>+	mr-&gt;ic_info.recv_ic_id_valid = result.ic_info.recv_ic_id_valid;</div><div class='add'>+	mr-&gt;ic_info.rdma_read_ic_id_valid = result.ic_info.rdma_read_ic_id_valid;</div><div class='add'>+	mr-&gt;ic_info.rdma_recv_ic_id_valid = result.ic_info.rdma_recv_ic_id_valid;</div><div class='ctx'> 	ibdev_dbg(&amp;dev-&gt;ibdev, "Registered mr[%d]\n", mr-&gt;ibmr.lkey);</div><div class='ctx'> </div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_mr *efa_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start,</div><div class='add'>+				     u64 length, u64 virt_addr,</div><div class='add'>+				     int fd, int access_flags,</div><div class='add'>+				     struct ib_dmah *dmah,</div><div class='add'>+				     struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct efa_dev *dev = to_edev(ibpd-&gt;device);</div><div class='add'>+	struct ib_umem_dmabuf *umem_dmabuf;</div><div class='add'>+	struct efa_mr *mr;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (dmah) {</div><div class='add'>+		err = -EOPNOTSUPP;</div><div class='add'>+		goto err_out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mr = efa_alloc_mr(ibpd, access_flags, &amp;attrs-&gt;driver_udata);</div><div class='add'>+	if (IS_ERR(mr)) {</div><div class='add'>+		err = PTR_ERR(mr);</div><div class='add'>+		goto err_out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	umem_dmabuf = ib_umem_dmabuf_get_pinned(ibpd-&gt;device, start, length, fd,</div><div class='add'>+						access_flags);</div><div class='add'>+	if (IS_ERR(umem_dmabuf)) {</div><div class='add'>+		err = PTR_ERR(umem_dmabuf);</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev, "Failed to get dmabuf umem[%pe]\n",</div><div class='add'>+			  umem_dmabuf);</div><div class='add'>+		goto err_free;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mr-&gt;umem = &amp;umem_dmabuf-&gt;umem;</div><div class='add'>+	err = efa_register_mr(ibpd, mr, start, length, virt_addr, access_flags);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_release;</div><div class='add'>+</div><div class='ctx'> 	return &amp;mr-&gt;ibmr;</div><div class='ctx'> </div><div class='del'>-err_unmap:</div><div class='add'>+err_release:</div><div class='ctx'> 	ib_umem_release(mr-&gt;umem);</div><div class='ctx'> err_free:</div><div class='ctx'> 	kfree(mr);</div><div class='ctx'> err_out:</div><div class='del'>-	atomic64_inc(&amp;dev-&gt;stats.sw_stats.reg_mr_err);</div><div class='add'>+	atomic64_inc(&amp;dev-&gt;stats.reg_mr_err);</div><div class='ctx'> 	return ERR_PTR(err);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,</div><div class='add'>+			 u64 virt_addr, int access_flags,</div><div class='add'>+			 struct ib_dmah *dmah,</div><div class='add'>+			 struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct efa_dev *dev = to_edev(ibpd-&gt;device);</div><div class='add'>+	struct efa_mr *mr;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (dmah) {</div><div class='add'>+		err = -EOPNOTSUPP;</div><div class='add'>+		goto err_out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mr = efa_alloc_mr(ibpd, access_flags, udata);</div><div class='add'>+	if (IS_ERR(mr)) {</div><div class='add'>+		err = PTR_ERR(mr);</div><div class='add'>+		goto err_out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mr-&gt;umem = ib_umem_get(ibpd-&gt;device, start, length, access_flags);</div><div class='add'>+	if (IS_ERR(mr-&gt;umem)) {</div><div class='add'>+		err = PTR_ERR(mr-&gt;umem);</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='add'>+			  "Failed to pin and map user space memory[%pe]\n",</div><div class='add'>+			  mr-&gt;umem);</div><div class='add'>+		goto err_free;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = efa_register_mr(ibpd, mr, start, length, virt_addr, access_flags);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_release;</div><div class='add'>+</div><div class='add'>+	return &amp;mr-&gt;ibmr;</div><div class='add'>+</div><div class='add'>+err_release:</div><div class='add'>+	ib_umem_release(mr-&gt;umem);</div><div class='add'>+err_free:</div><div class='add'>+	kfree(mr);</div><div class='add'>+err_out:</div><div class='add'>+	atomic64_inc(&amp;dev-&gt;stats.reg_mr_err);</div><div class='add'>+	return ERR_PTR(err);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(EFA_IB_METHOD_MR_QUERY)(struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_mr *ibmr = uverbs_attr_get_obj(attrs, EFA_IB_ATTR_QUERY_MR_HANDLE);</div><div class='add'>+	struct efa_mr *mr = to_emr(ibmr);</div><div class='add'>+	u16 ic_id_validity = 0;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID,</div><div class='add'>+			     &amp;mr-&gt;ic_info.recv_ic_id, sizeof(mr-&gt;ic_info.recv_ic_id));</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RDMA_READ_IC_ID,</div><div class='add'>+			     &amp;mr-&gt;ic_info.rdma_read_ic_id, sizeof(mr-&gt;ic_info.rdma_read_ic_id));</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RDMA_RECV_IC_ID,</div><div class='add'>+			     &amp;mr-&gt;ic_info.rdma_recv_ic_id, sizeof(mr-&gt;ic_info.rdma_recv_ic_id));</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (mr-&gt;ic_info.recv_ic_id_valid)</div><div class='add'>+		ic_id_validity |= EFA_QUERY_MR_VALIDITY_RECV_IC_ID;</div><div class='add'>+	if (mr-&gt;ic_info.rdma_read_ic_id_valid)</div><div class='add'>+		ic_id_validity |= EFA_QUERY_MR_VALIDITY_RDMA_READ_IC_ID;</div><div class='add'>+	if (mr-&gt;ic_info.rdma_recv_ic_id_valid)</div><div class='add'>+		ic_id_validity |= EFA_QUERY_MR_VALIDITY_RDMA_RECV_IC_ID;</div><div class='add'>+</div><div class='add'>+	return uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_IC_ID_VALIDITY,</div><div class='add'>+			      &amp;ic_id_validity, sizeof(ic_id_validity));</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_dev *dev = to_edev(ibmr-&gt;device);</div><div class='hunk'>@@ -1556,28 +1891,20 @@ int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)</div><div class='ctx'> 	struct efa_mr *mr = to_emr(ibmr);</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	if (udata-&gt;inlen &amp;&amp;</div><div class='del'>-	    !ib_is_udata_cleared(udata, 0, udata-&gt;inlen)) {</div><div class='del'>-		ibdev_dbg(&amp;dev-&gt;ibdev, "Incompatible ABI params\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	ibdev_dbg(&amp;dev-&gt;ibdev, "Deregister mr[%d]\n", ibmr-&gt;lkey);</div><div class='ctx'> </div><div class='del'>-	if (mr-&gt;umem) {</div><div class='del'>-		params.l_key = mr-&gt;ibmr.lkey;</div><div class='del'>-		err = efa_com_dereg_mr(&amp;dev-&gt;edev, &amp;params);</div><div class='del'>-		if (err)</div><div class='del'>-			return err;</div><div class='del'>-		ib_umem_release(mr-&gt;umem);</div><div class='del'>-	}</div><div class='add'>+	params.l_key = mr-&gt;ibmr.lkey;</div><div class='add'>+	err = efa_com_dereg_mr(&amp;dev-&gt;edev, &amp;params);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='ctx'> </div><div class='add'>+	ib_umem_release(mr-&gt;umem);</div><div class='ctx'> 	kfree(mr);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int efa_get_port_immutable(struct ib_device *ibdev, u8 port_num,</div><div class='add'>+int efa_get_port_immutable(struct ib_device *ibdev, u32 port_num,</div><div class='ctx'> 			   struct ib_port_immutable *immutable)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr attr;</div><div class='hunk'>@@ -1604,11 +1931,39 @@ static int efa_dealloc_uar(struct efa_dev *dev, u16 uarn)</div><div class='ctx'> 	return efa_com_dealloc_uar(&amp;dev-&gt;edev, &amp;params);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+#define EFA_CHECK_USER_COMP(_dev, _comp_mask, _attr, _mask, _attr_str) \</div><div class='add'>+	(_attr_str = (!(_dev)-&gt;dev_attr._attr || ((_comp_mask) &amp; (_mask))) ? \</div><div class='add'>+		     NULL : #_attr)</div><div class='add'>+</div><div class='add'>+static int efa_user_comp_handshake(const struct ib_ucontext *ibucontext,</div><div class='add'>+				   const struct efa_ibv_alloc_ucontext_cmd *cmd)</div><div class='add'>+{</div><div class='add'>+	struct efa_dev *dev = to_edev(ibucontext-&gt;device);</div><div class='add'>+	char *attr_str;</div><div class='add'>+</div><div class='add'>+	if (EFA_CHECK_USER_COMP(dev, cmd-&gt;comp_mask, max_tx_batch,</div><div class='add'>+				EFA_ALLOC_UCONTEXT_CMD_COMP_TX_BATCH, attr_str))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	if (EFA_CHECK_USER_COMP(dev, cmd-&gt;comp_mask, min_sq_depth,</div><div class='add'>+				EFA_ALLOC_UCONTEXT_CMD_COMP_MIN_SQ_WR,</div><div class='add'>+				attr_str))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ibdev, "Userspace handshake failed for %s attribute\n",</div><div class='add'>+		  attr_str);</div><div class='add'>+	return -EOPNOTSUPP;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_ucontext *ucontext = to_eucontext(ibucontext);</div><div class='ctx'> 	struct efa_dev *dev = to_edev(ibucontext-&gt;device);</div><div class='ctx'> 	struct efa_ibv_alloc_ucontext_resp resp = {};</div><div class='add'>+	struct efa_ibv_alloc_ucontext_cmd cmd = {};</div><div class='ctx'> 	struct efa_com_alloc_uar_result result;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='hunk'>@@ -1617,32 +1972,43 @@ int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata)</div><div class='ctx'> 	 * we will ack input fields in our response.</div><div class='ctx'> 	 */</div><div class='ctx'> </div><div class='add'>+	err = ib_copy_from_udata(&amp;cmd, udata,</div><div class='add'>+				 min(sizeof(cmd), udata-&gt;inlen));</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='add'>+			  "Cannot copy udata for alloc_ucontext\n");</div><div class='add'>+		goto err_out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = efa_user_comp_handshake(ibucontext, &amp;cmd);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_out;</div><div class='add'>+</div><div class='ctx'> 	err = efa_com_alloc_uar(&amp;dev-&gt;edev, &amp;result);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_out;</div><div class='ctx'> </div><div class='ctx'> 	ucontext-&gt;uarn = result.uarn;</div><div class='del'>-	xa_init(&amp;ucontext-&gt;mmap_xa);</div><div class='ctx'> </div><div class='ctx'> 	resp.cmds_supp_udata_mask |= EFA_USER_CMDS_SUPP_UDATA_QUERY_DEVICE;</div><div class='ctx'> 	resp.cmds_supp_udata_mask |= EFA_USER_CMDS_SUPP_UDATA_CREATE_AH;</div><div class='ctx'> 	resp.sub_cqs_per_cq = dev-&gt;dev_attr.sub_cqs_per_cq;</div><div class='ctx'> 	resp.inline_buf_size = dev-&gt;dev_attr.inline_buf_size;</div><div class='ctx'> 	resp.max_llq_size = dev-&gt;dev_attr.max_llq_size;</div><div class='add'>+	resp.max_tx_batch = dev-&gt;dev_attr.max_tx_batch;</div><div class='add'>+	resp.min_sq_wr = dev-&gt;dev_attr.min_sq_depth;</div><div class='ctx'> </div><div class='del'>-	if (udata &amp;&amp; udata-&gt;outlen) {</div><div class='del'>-		err = ib_copy_to_udata(udata, &amp;resp,</div><div class='del'>-				       min(sizeof(resp), udata-&gt;outlen));</div><div class='del'>-		if (err)</div><div class='del'>-			goto err_dealloc_uar;</div><div class='del'>-	}</div><div class='add'>+	err = ib_copy_to_udata(udata, &amp;resp,</div><div class='add'>+			       min(sizeof(resp), udata-&gt;outlen));</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_dealloc_uar;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> err_dealloc_uar:</div><div class='ctx'> 	efa_dealloc_uar(dev, result.uarn);</div><div class='ctx'> err_out:</div><div class='del'>-	atomic64_inc(&amp;dev-&gt;stats.sw_stats.alloc_ucontext_err);</div><div class='add'>+	atomic64_inc(&amp;dev-&gt;stats.alloc_ucontext_err);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1651,38 +2017,53 @@ void efa_dealloc_ucontext(struct ib_ucontext *ibucontext)</div><div class='ctx'> 	struct efa_ucontext *ucontext = to_eucontext(ibucontext);</div><div class='ctx'> 	struct efa_dev *dev = to_edev(ibucontext-&gt;device);</div><div class='ctx'> </div><div class='del'>-	mmap_entries_remove_free(dev, ucontext);</div><div class='ctx'> 	efa_dealloc_uar(dev, ucontext-&gt;uarn);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+void efa_mmap_free(struct rdma_user_mmap_entry *rdma_entry)</div><div class='add'>+{</div><div class='add'>+	struct efa_user_mmap_entry *entry = to_emmap(rdma_entry);</div><div class='add'>+</div><div class='add'>+	kfree(entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,</div><div class='del'>-		      struct vm_area_struct *vma, u64 key, u64 length)</div><div class='add'>+		      struct vm_area_struct *vma)</div><div class='ctx'> {</div><div class='del'>-	struct efa_mmap_entry *entry;</div><div class='add'>+	struct rdma_user_mmap_entry *rdma_entry;</div><div class='add'>+	struct efa_user_mmap_entry *entry;</div><div class='ctx'> 	unsigned long va;</div><div class='add'>+	int err = 0;</div><div class='ctx'> 	u64 pfn;</div><div class='del'>-	int err;</div><div class='ctx'> </div><div class='del'>-	entry = mmap_entry_get(dev, ucontext, key, length);</div><div class='del'>-	if (!entry) {</div><div class='del'>-		ibdev_dbg(&amp;dev-&gt;ibdev, "key[%#llx] does not have valid entry\n",</div><div class='del'>-			  key);</div><div class='add'>+	rdma_entry = rdma_user_mmap_entry_get(&amp;ucontext-&gt;ibucontext, vma);</div><div class='add'>+	if (!rdma_entry) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='add'>+			  "pgoff[%#lx] does not have valid entry\n",</div><div class='add'>+			  vma-&gt;vm_pgoff);</div><div class='add'>+		atomic64_inc(&amp;dev-&gt;stats.mmap_err);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='add'>+	entry = to_emmap(rdma_entry);</div><div class='ctx'> </div><div class='ctx'> 	ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='del'>-		  "Mapping address[%#llx], length[%#llx], mmap_flag[%d]\n",</div><div class='del'>-		  entry-&gt;address, length, entry-&gt;mmap_flag);</div><div class='add'>+		  "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",</div><div class='add'>+		  entry-&gt;address, rdma_entry-&gt;npages * PAGE_SIZE,</div><div class='add'>+		  entry-&gt;mmap_flag);</div><div class='ctx'> </div><div class='ctx'> 	pfn = entry-&gt;address &gt;&gt; PAGE_SHIFT;</div><div class='ctx'> 	switch (entry-&gt;mmap_flag) {</div><div class='ctx'> 	case EFA_MMAP_IO_NC:</div><div class='del'>-		err = rdma_user_mmap_io(&amp;ucontext-&gt;ibucontext, vma, pfn, length,</div><div class='del'>-					pgprot_noncached(vma-&gt;vm_page_prot));</div><div class='add'>+		err = rdma_user_mmap_io(&amp;ucontext-&gt;ibucontext, vma, pfn,</div><div class='add'>+					entry-&gt;rdma_entry.npages * PAGE_SIZE,</div><div class='add'>+					pgprot_noncached(vma-&gt;vm_page_prot),</div><div class='add'>+					rdma_entry);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case EFA_MMAP_IO_WC:</div><div class='del'>-		err = rdma_user_mmap_io(&amp;ucontext-&gt;ibucontext, vma, pfn, length,</div><div class='del'>-					pgprot_writecombine(vma-&gt;vm_page_prot));</div><div class='add'>+		err = rdma_user_mmap_io(&amp;ucontext-&gt;ibucontext, vma, pfn,</div><div class='add'>+					entry-&gt;rdma_entry.npages * PAGE_SIZE,</div><div class='add'>+					pgprot_writecombine(vma-&gt;vm_page_prot),</div><div class='add'>+					rdma_entry);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case EFA_MMAP_DMA_PAGE:</div><div class='ctx'> 		for (va = vma-&gt;vm_start; va &lt; vma-&gt;vm_end;</div><div class='hunk'>@@ -1696,12 +2077,16 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,</div><div class='ctx'> 		err = -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (err)</div><div class='add'>+	if (err) {</div><div class='ctx'> 		ibdev_dbg(</div><div class='ctx'> 			&amp;dev-&gt;ibdev,</div><div class='del'>-			"Couldn't mmap address[%#llx] length[%#llx] mmap_flag[%d] err[%d]\n",</div><div class='del'>-			entry-&gt;address, length, entry-&gt;mmap_flag, err);</div><div class='add'>+			"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",</div><div class='add'>+			entry-&gt;address, rdma_entry-&gt;npages * PAGE_SIZE,</div><div class='add'>+			entry-&gt;mmap_flag, err);</div><div class='add'>+		atomic64_inc(&amp;dev-&gt;stats.mmap_err);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='add'>+	rdma_user_mmap_entry_put(rdma_entry);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1710,27 +2095,13 @@ int efa_mmap(struct ib_ucontext *ibucontext,</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_ucontext *ucontext = to_eucontext(ibucontext);</div><div class='ctx'> 	struct efa_dev *dev = to_edev(ibucontext-&gt;device);</div><div class='del'>-	u64 length = vma-&gt;vm_end - vma-&gt;vm_start;</div><div class='del'>-	u64 key = vma-&gt;vm_pgoff &lt;&lt; PAGE_SHIFT;</div><div class='add'>+	size_t length = vma-&gt;vm_end - vma-&gt;vm_start;</div><div class='ctx'> </div><div class='ctx'> 	ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='del'>-		  "start %#lx, end %#lx, length = %#llx, key = %#llx\n",</div><div class='del'>-		  vma-&gt;vm_start, vma-&gt;vm_end, length, key);</div><div class='del'>-</div><div class='del'>-	if (length % PAGE_SIZE != 0 || !(vma-&gt;vm_flags &amp; VM_SHARED)) {</div><div class='del'>-		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='del'>-			  "length[%#llx] is not page size aligned[%#lx] or VM_SHARED is not set [%#lx]\n",</div><div class='del'>-			  length, PAGE_SIZE, vma-&gt;vm_flags);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='add'>+		  "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",</div><div class='add'>+		  vma-&gt;vm_start, vma-&gt;vm_end, length, vma-&gt;vm_pgoff);</div><div class='ctx'> </div><div class='del'>-	if (vma-&gt;vm_flags &amp; VM_EXEC) {</div><div class='del'>-		ibdev_dbg(&amp;dev-&gt;ibdev, "Mapping executable pages is not permitted\n");</div><div class='del'>-		return -EPERM;</div><div class='del'>-	}</div><div class='del'>-	vma-&gt;vm_flags &amp;= ~VM_MAYEXEC;</div><div class='del'>-</div><div class='del'>-	return __efa_mmap(dev, ucontext, vma, key, length);</div><div class='add'>+	return __efa_mmap(dev, ucontext, vma);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int efa_ah_destroy(struct efa_dev *dev, struct efa_ah *ah)</div><div class='hunk'>@@ -1744,10 +2115,10 @@ static int efa_ah_destroy(struct efa_dev *dev, struct efa_ah *ah)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int efa_create_ah(struct ib_ah *ibah,</div><div class='del'>-		  struct rdma_ah_attr *ah_attr,</div><div class='del'>-		  u32 flags,</div><div class='add'>+		  struct rdma_ah_init_attr *init_attr,</div><div class='ctx'> 		  struct ib_udata *udata)</div><div class='ctx'> {</div><div class='add'>+	struct rdma_ah_attr *ah_attr = init_attr-&gt;ah_attr;</div><div class='ctx'> 	struct efa_dev *dev = to_edev(ibah-&gt;device);</div><div class='ctx'> 	struct efa_com_create_ah_params params = {};</div><div class='ctx'> 	struct efa_ibv_create_ah_resp resp = {};</div><div class='hunk'>@@ -1755,7 +2126,7 @@ int efa_create_ah(struct ib_ah *ibah,</div><div class='ctx'> 	struct efa_ah *ah = to_eah(ibah);</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	if (!(flags &amp; RDMA_CREATE_AH_SLEEPABLE)) {</div><div class='add'>+	if (!(init_attr-&gt;flags &amp; RDMA_CREATE_AH_SLEEPABLE)) {</div><div class='ctx'> 		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='ctx'> 			  "Create address handle is not supported in atomic context\n");</div><div class='ctx'> 		err = -EOPNOTSUPP;</div><div class='hunk'>@@ -1797,11 +2168,11 @@ int efa_create_ah(struct ib_ah *ibah,</div><div class='ctx'> err_destroy_ah:</div><div class='ctx'> 	efa_ah_destroy(dev, ah);</div><div class='ctx'> err_out:</div><div class='del'>-	atomic64_inc(&amp;dev-&gt;stats.sw_stats.create_ah_err);</div><div class='add'>+	atomic64_inc(&amp;dev-&gt;stats.create_ah_err);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void efa_destroy_ah(struct ib_ah *ibah, u32 flags)</div><div class='add'>+int efa_destroy_ah(struct ib_ah *ibah, u32 flags)</div><div class='ctx'> {</div><div class='ctx'> 	struct efa_dev *dev = to_edev(ibah-&gt;pd-&gt;device);</div><div class='ctx'> 	struct efa_ah *ah = to_eah(ibah);</div><div class='hunk'>@@ -1811,15 +2182,167 @@ void efa_destroy_ah(struct ib_ah *ibah, u32 flags)</div><div class='ctx'> 	if (!(flags &amp; RDMA_DESTROY_AH_SLEEPABLE)) {</div><div class='ctx'> 		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='ctx'> 			  "Destroy address handle is not supported in atomic context\n");</div><div class='del'>-		return;</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	efa_ah_destroy(dev, ah);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct rdma_hw_stats *efa_alloc_hw_port_stats(struct ib_device *ibdev,</div><div class='add'>+					      u32 port_num)</div><div class='add'>+{</div><div class='add'>+	return rdma_alloc_hw_stats_struct(efa_port_stats_descs,</div><div class='add'>+					  ARRAY_SIZE(efa_port_stats_descs),</div><div class='add'>+					  RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct rdma_hw_stats *efa_alloc_hw_device_stats(struct ib_device *ibdev)</div><div class='add'>+{</div><div class='add'>+	return rdma_alloc_hw_stats_struct(efa_device_stats_descs,</div><div class='add'>+					  ARRAY_SIZE(efa_device_stats_descs),</div><div class='add'>+					  RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int efa_fill_device_stats(struct efa_dev *dev,</div><div class='add'>+				 struct rdma_hw_stats *stats)</div><div class='add'>+{</div><div class='add'>+	struct efa_com_stats_admin *as = &amp;dev-&gt;edev.aq.stats;</div><div class='add'>+	struct efa_stats *s = &amp;dev-&gt;stats;</div><div class='add'>+</div><div class='add'>+	stats-&gt;value[EFA_SUBMITTED_CMDS] = atomic64_read(&amp;as-&gt;submitted_cmd);</div><div class='add'>+	stats-&gt;value[EFA_COMPLETED_CMDS] = atomic64_read(&amp;as-&gt;completed_cmd);</div><div class='add'>+	stats-&gt;value[EFA_CMDS_ERR] = atomic64_read(&amp;as-&gt;cmd_err);</div><div class='add'>+	stats-&gt;value[EFA_NO_COMPLETION_CMDS] = atomic64_read(&amp;as-&gt;no_completion);</div><div class='add'>+</div><div class='add'>+	stats-&gt;value[EFA_KEEP_ALIVE_RCVD] = atomic64_read(&amp;s-&gt;keep_alive_rcvd);</div><div class='add'>+	stats-&gt;value[EFA_ALLOC_PD_ERR] = atomic64_read(&amp;s-&gt;alloc_pd_err);</div><div class='add'>+	stats-&gt;value[EFA_CREATE_QP_ERR] = atomic64_read(&amp;s-&gt;create_qp_err);</div><div class='add'>+	stats-&gt;value[EFA_CREATE_CQ_ERR] = atomic64_read(&amp;s-&gt;create_cq_err);</div><div class='add'>+	stats-&gt;value[EFA_REG_MR_ERR] = atomic64_read(&amp;s-&gt;reg_mr_err);</div><div class='add'>+	stats-&gt;value[EFA_ALLOC_UCONTEXT_ERR] =</div><div class='add'>+		atomic64_read(&amp;s-&gt;alloc_ucontext_err);</div><div class='add'>+	stats-&gt;value[EFA_CREATE_AH_ERR] = atomic64_read(&amp;s-&gt;create_ah_err);</div><div class='add'>+	stats-&gt;value[EFA_MMAP_ERR] = atomic64_read(&amp;s-&gt;mmap_err);</div><div class='add'>+</div><div class='add'>+	return ARRAY_SIZE(efa_device_stats_descs);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int efa_fill_port_stats(struct efa_dev *dev, struct rdma_hw_stats *stats,</div><div class='add'>+			       u32 port_num)</div><div class='add'>+{</div><div class='add'>+	struct efa_com_get_stats_params params = {};</div><div class='add'>+	union efa_com_get_stats_result result;</div><div class='add'>+	struct efa_com_rdma_write_stats *rws;</div><div class='add'>+	struct efa_com_rdma_read_stats *rrs;</div><div class='add'>+	struct efa_com_messages_stats *ms;</div><div class='add'>+	struct efa_com_network_stats *ns;</div><div class='add'>+	struct efa_com_basic_stats *bs;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	params.scope = EFA_ADMIN_GET_STATS_SCOPE_ALL;</div><div class='add'>+	params.type = EFA_ADMIN_GET_STATS_TYPE_BASIC;</div><div class='add'>+</div><div class='add'>+	err = efa_com_get_stats(&amp;dev-&gt;edev, &amp;params, &amp;result);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	bs = &amp;result.basic_stats;</div><div class='add'>+	stats-&gt;value[EFA_TX_BYTES] = bs-&gt;tx_bytes;</div><div class='add'>+	stats-&gt;value[EFA_TX_PKTS] = bs-&gt;tx_pkts;</div><div class='add'>+	stats-&gt;value[EFA_RX_BYTES] = bs-&gt;rx_bytes;</div><div class='add'>+	stats-&gt;value[EFA_RX_PKTS] = bs-&gt;rx_pkts;</div><div class='add'>+	stats-&gt;value[EFA_RX_DROPS] = bs-&gt;rx_drops;</div><div class='add'>+</div><div class='add'>+	params.type = EFA_ADMIN_GET_STATS_TYPE_MESSAGES;</div><div class='add'>+	err = efa_com_get_stats(&amp;dev-&gt;edev, &amp;params, &amp;result);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	ms = &amp;result.messages_stats;</div><div class='add'>+	stats-&gt;value[EFA_SEND_BYTES] = ms-&gt;send_bytes;</div><div class='add'>+	stats-&gt;value[EFA_SEND_WRS] = ms-&gt;send_wrs;</div><div class='add'>+	stats-&gt;value[EFA_RECV_BYTES] = ms-&gt;recv_bytes;</div><div class='add'>+	stats-&gt;value[EFA_RECV_WRS] = ms-&gt;recv_wrs;</div><div class='add'>+</div><div class='add'>+	params.type = EFA_ADMIN_GET_STATS_TYPE_RDMA_READ;</div><div class='add'>+	err = efa_com_get_stats(&amp;dev-&gt;edev, &amp;params, &amp;result);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	rrs = &amp;result.rdma_read_stats;</div><div class='add'>+	stats-&gt;value[EFA_RDMA_READ_WRS] = rrs-&gt;read_wrs;</div><div class='add'>+	stats-&gt;value[EFA_RDMA_READ_BYTES] = rrs-&gt;read_bytes;</div><div class='add'>+	stats-&gt;value[EFA_RDMA_READ_WR_ERR] = rrs-&gt;read_wr_err;</div><div class='add'>+	stats-&gt;value[EFA_RDMA_READ_RESP_BYTES] = rrs-&gt;read_resp_bytes;</div><div class='add'>+</div><div class='add'>+	if (EFA_DEV_CAP(dev, RDMA_WRITE)) {</div><div class='add'>+		params.type = EFA_ADMIN_GET_STATS_TYPE_RDMA_WRITE;</div><div class='add'>+		err = efa_com_get_stats(&amp;dev-&gt;edev, &amp;params, &amp;result);</div><div class='add'>+		if (err)</div><div class='add'>+			return err;</div><div class='add'>+</div><div class='add'>+		rws = &amp;result.rdma_write_stats;</div><div class='add'>+		stats-&gt;value[EFA_RDMA_WRITE_WRS] = rws-&gt;write_wrs;</div><div class='add'>+		stats-&gt;value[EFA_RDMA_WRITE_BYTES] = rws-&gt;write_bytes;</div><div class='add'>+		stats-&gt;value[EFA_RDMA_WRITE_WR_ERR] = rws-&gt;write_wr_err;</div><div class='add'>+		stats-&gt;value[EFA_RDMA_WRITE_RECV_BYTES] = rws-&gt;write_recv_bytes;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	params.type = EFA_ADMIN_GET_STATS_TYPE_NETWORK;</div><div class='add'>+	err = efa_com_get_stats(&amp;dev-&gt;edev, &amp;params, &amp;result);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	ns = &amp;result.network_stats;</div><div class='add'>+	stats-&gt;value[EFA_RETRANS_BYTES] = ns-&gt;retrans_bytes;</div><div class='add'>+	stats-&gt;value[EFA_RETRANS_PKTS] = ns-&gt;retrans_pkts;</div><div class='add'>+	stats-&gt;value[EFA_RETRANS_TIMEOUT_EVENS] = ns-&gt;retrans_timeout_events;</div><div class='add'>+	stats-&gt;value[EFA_UNRESPONSIVE_REMOTE_EVENTS] = ns-&gt;unresponsive_remote_events;</div><div class='add'>+	stats-&gt;value[EFA_IMPAIRED_REMOTE_CONN_EVENTS] = ns-&gt;impaired_remote_conn_events;</div><div class='add'>+</div><div class='add'>+	return ARRAY_SIZE(efa_port_stats_descs);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,</div><div class='add'>+		     u32 port_num, int index)</div><div class='add'>+{</div><div class='add'>+	if (port_num)</div><div class='add'>+		return efa_fill_port_stats(to_edev(ibdev), stats, port_num);</div><div class='add'>+	else</div><div class='add'>+		return efa_fill_device_stats(to_edev(ibdev), stats);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> enum rdma_link_layer efa_port_link_layer(struct ib_device *ibdev,</div><div class='del'>-					 u8 port_num)</div><div class='add'>+					 u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	return IB_LINK_LAYER_UNSPECIFIED;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(EFA_IB_METHOD_MR_QUERY,</div><div class='add'>+			    UVERBS_ATTR_IDR(EFA_IB_ATTR_QUERY_MR_HANDLE,</div><div class='add'>+					    UVERBS_OBJECT_MR,</div><div class='add'>+					    UVERBS_ACCESS_READ,</div><div class='add'>+					    UA_MANDATORY),</div><div class='add'>+			    UVERBS_ATTR_PTR_OUT(EFA_IB_ATTR_QUERY_MR_RESP_IC_ID_VALIDITY,</div><div class='add'>+						UVERBS_ATTR_TYPE(u16),</div><div class='add'>+						UA_MANDATORY),</div><div class='add'>+			    UVERBS_ATTR_PTR_OUT(EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID,</div><div class='add'>+						UVERBS_ATTR_TYPE(u16),</div><div class='add'>+						UA_MANDATORY),</div><div class='add'>+			    UVERBS_ATTR_PTR_OUT(EFA_IB_ATTR_QUERY_MR_RESP_RDMA_READ_IC_ID,</div><div class='add'>+						UVERBS_ATTR_TYPE(u16),</div><div class='add'>+						UA_MANDATORY),</div><div class='add'>+			    UVERBS_ATTR_PTR_OUT(EFA_IB_ATTR_QUERY_MR_RESP_RDMA_RECV_IC_ID,</div><div class='add'>+						UVERBS_ATTR_TYPE(u16),</div><div class='add'>+						UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+ADD_UVERBS_METHODS(efa_mr,</div><div class='add'>+		   UVERBS_OBJECT_MR,</div><div class='add'>+		   &amp;UVERBS_METHOD(EFA_IB_METHOD_MR_QUERY));</div><div class='add'>+</div><div class='add'>+const struct uapi_definition efa_uapi_defs[] = {</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_MR,</div><div class='add'>+				&amp;efa_mr),</div><div class='add'>+	{},</div><div class='add'>+};</div><div class='head'>diff --git a/drivers/infiniband/hw/erdma/Kconfig b/drivers/infiniband/hw/erdma/Kconfig<br/>new file mode 100644<br/>index 000000000000..267fc1f3c42a<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/erdma/Kconfig?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/erdma/Kconfig</a></div><div class='hunk'>@@ -0,0 +1,12 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+config INFINIBAND_ERDMA</div><div class='add'>+	tristate "Alibaba Elastic RDMA Adapter (ERDMA) support"</div><div class='add'>+	depends on PCI_MSI &amp;&amp; 64BIT</div><div class='add'>+	depends on INFINIBAND_ADDR_TRANS</div><div class='add'>+	depends on INFINIBAND_USER_ACCESS</div><div class='add'>+	help</div><div class='add'>+	  This is a RDMA driver for Alibaba Elastic RDMA Adapter(ERDMA),</div><div class='add'>+	  which supports RDMA features in Alibaba cloud environment.</div><div class='add'>+</div><div class='add'>+	  To compile this driver as module, choose M here. The module will be</div><div class='add'>+	  called erdma.</div><div class='head'>diff --git a/drivers/infiniband/hw/erdma/Makefile b/drivers/infiniband/hw/erdma/Makefile<br/>new file mode 100644<br/>index 000000000000..51d2ef91905a<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/erdma/Makefile?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/erdma/Makefile</a></div><div class='hunk'>@@ -0,0 +1,4 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0</div><div class='add'>+obj-$(CONFIG_INFINIBAND_ERDMA) := erdma.o</div><div class='add'>+</div><div class='add'>+erdma-y := erdma_cm.o erdma_main.o erdma_cmdq.o erdma_cq.o erdma_verbs.o erdma_qp.o erdma_eq.o</div><div class='head'>diff --git a/drivers/infiniband/hw/erdma/erdma.h b/drivers/infiniband/hw/erdma/erdma.h<br/>new file mode 100644<br/>index 000000000000..2a023b99f992<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/erdma/erdma.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/erdma/erdma.h</a></div><div class='hunk'>@@ -0,0 +1,283 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='add'>+</div><div class='add'>+/* Authors: Cheng Xu &lt;chengyou@linux.alibaba.com&gt; */</div><div class='add'>+/*          Kai Shen &lt;kaishen@linux.alibaba.com&gt; */</div><div class='add'>+/* Copyright (c) 2020-2022, Alibaba Group. */</div><div class='add'>+</div><div class='add'>+#ifndef __ERDMA_H__</div><div class='add'>+#define __ERDMA_H__</div><div class='add'>+</div><div class='add'>+#include &lt;linux/bitfield.h&gt;</div><div class='add'>+#include &lt;linux/netdevice.h&gt;</div><div class='add'>+#include &lt;linux/pci.h&gt;</div><div class='add'>+#include &lt;linux/xarray.h&gt;</div><div class='add'>+#include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+</div><div class='add'>+#include "erdma_hw.h"</div><div class='add'>+</div><div class='add'>+#define DRV_MODULE_NAME "erdma"</div><div class='add'>+#define ERDMA_NODE_DESC "Elastic RDMA Adapter stack"</div><div class='add'>+</div><div class='add'>+struct erdma_eq {</div><div class='add'>+	void *qbuf;</div><div class='add'>+	dma_addr_t qbuf_dma_addr;</div><div class='add'>+</div><div class='add'>+	spinlock_t lock;</div><div class='add'>+</div><div class='add'>+	u32 depth;</div><div class='add'>+</div><div class='add'>+	u16 ci;</div><div class='add'>+	u16 rsvd;</div><div class='add'>+</div><div class='add'>+	atomic64_t event_num;</div><div class='add'>+	atomic64_t notify_num;</div><div class='add'>+</div><div class='add'>+	void __iomem *db;</div><div class='add'>+	u64 *dbrec;</div><div class='add'>+	dma_addr_t dbrec_dma;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_sq {</div><div class='add'>+	void *qbuf;</div><div class='add'>+	dma_addr_t qbuf_dma_addr;</div><div class='add'>+</div><div class='add'>+	spinlock_t lock;</div><div class='add'>+</div><div class='add'>+	u32 depth;</div><div class='add'>+	u16 ci;</div><div class='add'>+	u16 pi;</div><div class='add'>+</div><div class='add'>+	u16 wqebb_cnt;</div><div class='add'>+</div><div class='add'>+	u64 *dbrec;</div><div class='add'>+	dma_addr_t dbrec_dma;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_cq {</div><div class='add'>+	void *qbuf;</div><div class='add'>+	dma_addr_t qbuf_dma_addr;</div><div class='add'>+</div><div class='add'>+	spinlock_t lock;</div><div class='add'>+</div><div class='add'>+	u32 depth;</div><div class='add'>+	u32 ci;</div><div class='add'>+	u32 cmdsn;</div><div class='add'>+</div><div class='add'>+	u64 *dbrec;</div><div class='add'>+	dma_addr_t dbrec_dma;</div><div class='add'>+</div><div class='add'>+	atomic64_t armed_num;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	ERDMA_CMD_STATUS_INIT,</div><div class='add'>+	ERDMA_CMD_STATUS_ISSUED,</div><div class='add'>+	ERDMA_CMD_STATUS_FINISHED,</div><div class='add'>+	ERDMA_CMD_STATUS_TIMEOUT</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_comp_wait {</div><div class='add'>+	struct completion wait_event;</div><div class='add'>+	u32 cmd_status;</div><div class='add'>+	u32 ctx_id;</div><div class='add'>+	u16 sq_pi;</div><div class='add'>+	u8 comp_status;</div><div class='add'>+	u8 rsvd;</div><div class='add'>+	u32 comp_data[4];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	ERDMA_CMDQ_STATE_OK_BIT = 0,</div><div class='add'>+	ERDMA_CMDQ_STATE_TIMEOUT_BIT = 1,</div><div class='add'>+	ERDMA_CMDQ_STATE_CTX_ERR_BIT = 2,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define ERDMA_CMDQ_TIMEOUT_MS 15000</div><div class='add'>+#define ERDMA_REG_ACCESS_WAIT_MS 20</div><div class='add'>+#define ERDMA_WAIT_DEV_DONE_CNT 500</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq {</div><div class='add'>+	unsigned long *comp_wait_bitmap;</div><div class='add'>+	struct erdma_comp_wait *wait_pool;</div><div class='add'>+	spinlock_t lock;</div><div class='add'>+</div><div class='add'>+	struct erdma_cmdq_sq sq;</div><div class='add'>+	struct erdma_cmdq_cq cq;</div><div class='add'>+	struct erdma_eq eq;</div><div class='add'>+</div><div class='add'>+	unsigned long state;</div><div class='add'>+</div><div class='add'>+	struct semaphore credits;</div><div class='add'>+	u16 max_outstandings;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define COMPROMISE_CC ERDMA_CC_CUBIC</div><div class='add'>+enum erdma_cc_alg {</div><div class='add'>+	ERDMA_CC_NEWRENO = 0,</div><div class='add'>+	ERDMA_CC_CUBIC,</div><div class='add'>+	ERDMA_CC_HPCC_RTT,</div><div class='add'>+	ERDMA_CC_HPCC_ECN,</div><div class='add'>+	ERDMA_CC_HPCC_INT,</div><div class='add'>+	ERDMA_CC_METHODS_NUM</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_devattr {</div><div class='add'>+	u32 fw_version;</div><div class='add'>+</div><div class='add'>+	unsigned char peer_addr[ETH_ALEN];</div><div class='add'>+	unsigned long cap_flags;</div><div class='add'>+</div><div class='add'>+	int numa_node;</div><div class='add'>+	enum erdma_cc_alg cc;</div><div class='add'>+	u32 irq_num;</div><div class='add'>+</div><div class='add'>+	u32 max_qp;</div><div class='add'>+	u32 max_send_wr;</div><div class='add'>+	u32 max_recv_wr;</div><div class='add'>+	u32 max_ord;</div><div class='add'>+	u32 max_ird;</div><div class='add'>+</div><div class='add'>+	u32 max_send_sge;</div><div class='add'>+	u32 max_recv_sge;</div><div class='add'>+	u32 max_sge_rd;</div><div class='add'>+	u32 max_cq;</div><div class='add'>+	u32 max_cqe;</div><div class='add'>+	u64 max_mr_size;</div><div class='add'>+	u32 max_mr;</div><div class='add'>+	u32 max_pd;</div><div class='add'>+	u32 max_mw;</div><div class='add'>+	u32 max_gid;</div><div class='add'>+	u32 max_ah;</div><div class='add'>+	u32 local_dma_key;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define ERDMA_IRQNAME_SIZE 50</div><div class='add'>+</div><div class='add'>+struct erdma_irq {</div><div class='add'>+	char name[ERDMA_IRQNAME_SIZE];</div><div class='add'>+	u32 msix_vector;</div><div class='add'>+	cpumask_t affinity_hint_mask;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_eq_cb {</div><div class='add'>+	bool ready;</div><div class='add'>+	void *dev; /* All EQs use this fields to get erdma_dev struct */</div><div class='add'>+	struct erdma_irq irq;</div><div class='add'>+	struct erdma_eq eq;</div><div class='add'>+	struct tasklet_struct tasklet;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_resource_cb {</div><div class='add'>+	unsigned long *bitmap;</div><div class='add'>+	spinlock_t lock;</div><div class='add'>+	u32 next_alloc_idx;</div><div class='add'>+	u32 max_cap;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	ERDMA_RES_TYPE_PD = 0,</div><div class='add'>+	ERDMA_RES_TYPE_STAG_IDX = 1,</div><div class='add'>+	ERDMA_RES_TYPE_AH = 2,</div><div class='add'>+	ERDMA_RES_CNT = 3,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_dev {</div><div class='add'>+	struct ib_device ibdev;</div><div class='add'>+	struct net_device *netdev;</div><div class='add'>+	struct pci_dev *pdev;</div><div class='add'>+	struct notifier_block netdev_nb;</div><div class='add'>+	struct workqueue_struct *reflush_wq;</div><div class='add'>+</div><div class='add'>+	resource_size_t func_bar_addr;</div><div class='add'>+	resource_size_t func_bar_len;</div><div class='add'>+	u8 __iomem *func_bar;</div><div class='add'>+</div><div class='add'>+	struct erdma_devattr attrs;</div><div class='add'>+	u32 mtu;</div><div class='add'>+</div><div class='add'>+	/* cmdq and aeq use the same msix vector */</div><div class='add'>+	struct erdma_irq comm_irq;</div><div class='add'>+	struct erdma_cmdq cmdq;</div><div class='add'>+	struct erdma_eq aeq;</div><div class='add'>+	struct erdma_eq_cb ceqs[ERDMA_NUM_MSIX_VEC - 1];</div><div class='add'>+</div><div class='add'>+	spinlock_t lock;</div><div class='add'>+	struct erdma_resource_cb res_cb[ERDMA_RES_CNT];</div><div class='add'>+	struct xarray qp_xa;</div><div class='add'>+	struct xarray cq_xa;</div><div class='add'>+</div><div class='add'>+	u32 next_alloc_qpn;</div><div class='add'>+	u32 next_alloc_cqn;</div><div class='add'>+</div><div class='add'>+	atomic_t num_ctx;</div><div class='add'>+	struct list_head cep_list;</div><div class='add'>+</div><div class='add'>+	struct dma_pool *db_pool;</div><div class='add'>+	struct dma_pool *resp_pool;</div><div class='add'>+	enum erdma_proto_type proto;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline void *get_queue_entry(void *qbuf, u32 idx, u32 depth, u32 shift)</div><div class='add'>+{</div><div class='add'>+	idx &amp;= (depth - 1);</div><div class='add'>+</div><div class='add'>+	return qbuf + (idx &lt;&lt; shift);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct erdma_dev *to_edev(struct ib_device *ibdev)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibdev, struct erdma_dev, ibdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 erdma_reg_read32(struct erdma_dev *dev, u32 reg)</div><div class='add'>+{</div><div class='add'>+	return readl(dev-&gt;func_bar + reg);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u64 erdma_reg_read64(struct erdma_dev *dev, u32 reg)</div><div class='add'>+{</div><div class='add'>+	return readq(dev-&gt;func_bar + reg);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void erdma_reg_write32(struct erdma_dev *dev, u32 reg, u32 value)</div><div class='add'>+{</div><div class='add'>+	writel(value, dev-&gt;func_bar + reg);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void erdma_reg_write64(struct erdma_dev *dev, u32 reg, u64 value)</div><div class='add'>+{</div><div class='add'>+	writeq(value, dev-&gt;func_bar + reg);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 erdma_reg_read32_filed(struct erdma_dev *dev, u32 reg,</div><div class='add'>+					 u32 filed_mask)</div><div class='add'>+{</div><div class='add'>+	u32 val = erdma_reg_read32(dev, reg);</div><div class='add'>+</div><div class='add'>+	return FIELD_GET(filed_mask, val);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define ERDMA_GET(val, name) FIELD_GET(ERDMA_CMD_##name##_MASK, val)</div><div class='add'>+</div><div class='add'>+int erdma_cmdq_init(struct erdma_dev *dev);</div><div class='add'>+void erdma_finish_cmdq_init(struct erdma_dev *dev);</div><div class='add'>+void erdma_cmdq_destroy(struct erdma_dev *dev);</div><div class='add'>+</div><div class='add'>+void erdma_cmdq_build_reqhdr(u64 *hdr, u32 mod, u32 op);</div><div class='add'>+int erdma_post_cmd_wait(struct erdma_cmdq *cmdq, void *req, u32 req_size,</div><div class='add'>+			u64 *resp0, u64 *resp1, bool sleepable);</div><div class='add'>+void erdma_cmdq_completion_handler(struct erdma_cmdq *cmdq);</div><div class='add'>+</div><div class='add'>+int erdma_ceqs_init(struct erdma_dev *dev);</div><div class='add'>+void erdma_ceqs_uninit(struct erdma_dev *dev);</div><div class='add'>+void notify_eq(struct erdma_eq *eq);</div><div class='add'>+void *get_next_valid_eqe(struct erdma_eq *eq);</div><div class='add'>+</div><div class='add'>+int erdma_aeq_init(struct erdma_dev *dev);</div><div class='add'>+int erdma_eq_common_init(struct erdma_dev *dev, struct erdma_eq *eq, u32 depth);</div><div class='add'>+void erdma_eq_destroy(struct erdma_dev *dev, struct erdma_eq *eq);</div><div class='add'>+</div><div class='add'>+void erdma_aeq_event_handler(struct erdma_dev *dev);</div><div class='add'>+void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb);</div><div class='add'>+</div><div class='add'>+#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/erdma/erdma_cm.c b/drivers/infiniband/hw/erdma/erdma_cm.c<br/>new file mode 100644<br/>index 000000000000..ed21ba0037a4<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/erdma/erdma_cm.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/erdma/erdma_cm.c</a></div><div class='hunk'>@@ -0,0 +1,1431 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='add'>+</div><div class='add'>+/* Authors: Cheng Xu &lt;chengyou@linux.alibaba.com&gt; */</div><div class='add'>+/*          Kai Shen &lt;kaishen@linux.alibaba.com&gt; */</div><div class='add'>+/* Copyright (c) 2020-2022, Alibaba Group. */</div><div class='add'>+</div><div class='add'>+/* Authors: Bernard Metzler &lt;bmt@zurich.ibm.com&gt; */</div><div class='add'>+/*          Fredy Neeser */</div><div class='add'>+/*          Greg Joyce &lt;greg@opengridcomputing.com&gt; */</div><div class='add'>+/* Copyright (c) 2008-2019, IBM Corporation */</div><div class='add'>+/* Copyright (c) 2017, Open Grid Computing, Inc. */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/workqueue.h&gt;</div><div class='add'>+#include &lt;trace/events/sock.h&gt;</div><div class='add'>+</div><div class='add'>+#include "erdma.h"</div><div class='add'>+#include "erdma_cm.h"</div><div class='add'>+#include "erdma_verbs.h"</div><div class='add'>+</div><div class='add'>+static struct workqueue_struct *erdma_cm_wq;</div><div class='add'>+</div><div class='add'>+static void erdma_cm_llp_state_change(struct sock *sk);</div><div class='add'>+static void erdma_cm_llp_data_ready(struct sock *sk);</div><div class='add'>+static void erdma_cm_llp_error_report(struct sock *sk);</div><div class='add'>+</div><div class='add'>+static void erdma_sk_assign_cm_upcalls(struct sock *sk)</div><div class='add'>+{</div><div class='add'>+	write_lock_bh(&amp;sk-&gt;sk_callback_lock);</div><div class='add'>+	sk-&gt;sk_state_change = erdma_cm_llp_state_change;</div><div class='add'>+	sk-&gt;sk_data_ready = erdma_cm_llp_data_ready;</div><div class='add'>+	sk-&gt;sk_error_report = erdma_cm_llp_error_report;</div><div class='add'>+	write_unlock_bh(&amp;sk-&gt;sk_callback_lock);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_sk_save_upcalls(struct sock *sk)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cep *cep = sk_to_cep(sk);</div><div class='add'>+</div><div class='add'>+	write_lock_bh(&amp;sk-&gt;sk_callback_lock);</div><div class='add'>+	cep-&gt;sk_state_change = sk-&gt;sk_state_change;</div><div class='add'>+	cep-&gt;sk_data_ready = sk-&gt;sk_data_ready;</div><div class='add'>+	cep-&gt;sk_error_report = sk-&gt;sk_error_report;</div><div class='add'>+	write_unlock_bh(&amp;sk-&gt;sk_callback_lock);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_sk_restore_upcalls(struct sock *sk, struct erdma_cep *cep)</div><div class='add'>+{</div><div class='add'>+	sk-&gt;sk_state_change = cep-&gt;sk_state_change;</div><div class='add'>+	sk-&gt;sk_data_ready = cep-&gt;sk_data_ready;</div><div class='add'>+	sk-&gt;sk_error_report = cep-&gt;sk_error_report;</div><div class='add'>+	sk-&gt;sk_user_data = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_socket_disassoc(struct socket *s)</div><div class='add'>+{</div><div class='add'>+	struct sock *sk = s-&gt;sk;</div><div class='add'>+	struct erdma_cep *cep;</div><div class='add'>+</div><div class='add'>+	if (sk) {</div><div class='add'>+		write_lock_bh(&amp;sk-&gt;sk_callback_lock);</div><div class='add'>+		cep = sk_to_cep(sk);</div><div class='add'>+		if (cep) {</div><div class='add'>+			erdma_sk_restore_upcalls(sk, cep);</div><div class='add'>+			erdma_cep_put(cep);</div><div class='add'>+		} else {</div><div class='add'>+			WARN_ON_ONCE(1);</div><div class='add'>+		}</div><div class='add'>+		write_unlock_bh(&amp;sk-&gt;sk_callback_lock);</div><div class='add'>+	} else {</div><div class='add'>+		WARN_ON_ONCE(1);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_cep_socket_assoc(struct erdma_cep *cep, struct socket *s)</div><div class='add'>+{</div><div class='add'>+	cep-&gt;sock = s;</div><div class='add'>+	erdma_cep_get(cep);</div><div class='add'>+	s-&gt;sk-&gt;sk_user_data = cep;</div><div class='add'>+</div><div class='add'>+	erdma_sk_save_upcalls(s-&gt;sk);</div><div class='add'>+	erdma_sk_assign_cm_upcalls(s-&gt;sk);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_disassoc_listen_cep(struct erdma_cep *cep)</div><div class='add'>+{</div><div class='add'>+	if (cep-&gt;listen_cep) {</div><div class='add'>+		erdma_cep_put(cep-&gt;listen_cep);</div><div class='add'>+		cep-&gt;listen_cep = NULL;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct erdma_cep *erdma_cep_alloc(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cep *cep = kzalloc(sizeof(*cep), GFP_KERNEL);</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	if (!cep)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;cep-&gt;listenq);</div><div class='add'>+	INIT_LIST_HEAD(&amp;cep-&gt;devq);</div><div class='add'>+	INIT_LIST_HEAD(&amp;cep-&gt;work_freelist);</div><div class='add'>+</div><div class='add'>+	kref_init(&amp;cep-&gt;ref);</div><div class='add'>+	cep-&gt;state = ERDMA_EPSTATE_IDLE;</div><div class='add'>+	init_waitqueue_head(&amp;cep-&gt;waitq);</div><div class='add'>+	spin_lock_init(&amp;cep-&gt;lock);</div><div class='add'>+	cep-&gt;dev = dev;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;dev-&gt;lock, flags);</div><div class='add'>+	list_add_tail(&amp;cep-&gt;devq, &amp;dev-&gt;cep_list);</div><div class='add'>+	spin_unlock_irqrestore(&amp;dev-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	return cep;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_cm_free_work(struct erdma_cep *cep)</div><div class='add'>+{</div><div class='add'>+	struct list_head *w, *tmp;</div><div class='add'>+	struct erdma_cm_work *work;</div><div class='add'>+</div><div class='add'>+	list_for_each_safe(w, tmp, &amp;cep-&gt;work_freelist) {</div><div class='add'>+		work = list_entry(w, struct erdma_cm_work, list);</div><div class='add'>+		list_del(&amp;work-&gt;list);</div><div class='add'>+		kfree(work);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_cancel_mpatimer(struct erdma_cep *cep)</div><div class='add'>+{</div><div class='add'>+	spin_lock_bh(&amp;cep-&gt;lock);</div><div class='add'>+	if (cep-&gt;mpa_timer) {</div><div class='add'>+		if (cancel_delayed_work(&amp;cep-&gt;mpa_timer-&gt;work)) {</div><div class='add'>+			erdma_cep_put(cep);</div><div class='add'>+			kfree(cep-&gt;mpa_timer);</div><div class='add'>+		}</div><div class='add'>+		cep-&gt;mpa_timer = NULL;</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_bh(&amp;cep-&gt;lock);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_put_work(struct erdma_cm_work *work)</div><div class='add'>+{</div><div class='add'>+	INIT_LIST_HEAD(&amp;work-&gt;list);</div><div class='add'>+	spin_lock_bh(&amp;work-&gt;cep-&gt;lock);</div><div class='add'>+	list_add(&amp;work-&gt;list, &amp;work-&gt;cep-&gt;work_freelist);</div><div class='add'>+	spin_unlock_bh(&amp;work-&gt;cep-&gt;lock);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_cep_set_inuse(struct erdma_cep *cep)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cep-&gt;lock, flags);</div><div class='add'>+	while (cep-&gt;in_use) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;cep-&gt;lock, flags);</div><div class='add'>+		wait_event_interruptible(cep-&gt;waitq, !cep-&gt;in_use);</div><div class='add'>+		if (signal_pending(current))</div><div class='add'>+			flush_signals(current);</div><div class='add'>+</div><div class='add'>+		spin_lock_irqsave(&amp;cep-&gt;lock, flags);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cep-&gt;in_use = 1;</div><div class='add'>+	spin_unlock_irqrestore(&amp;cep-&gt;lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_cep_set_free(struct erdma_cep *cep)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cep-&gt;lock, flags);</div><div class='add'>+	cep-&gt;in_use = 0;</div><div class='add'>+	spin_unlock_irqrestore(&amp;cep-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	wake_up(&amp;cep-&gt;waitq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void __erdma_cep_dealloc(struct kref *ref)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cep *cep = container_of(ref, struct erdma_cep, ref);</div><div class='add'>+	struct erdma_dev *dev = cep-&gt;dev;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	WARN_ON(cep-&gt;listen_cep);</div><div class='add'>+</div><div class='add'>+	kfree(cep-&gt;private_data);</div><div class='add'>+	kfree(cep-&gt;mpa.pdata);</div><div class='add'>+	spin_lock_bh(&amp;cep-&gt;lock);</div><div class='add'>+	if (!list_empty(&amp;cep-&gt;work_freelist))</div><div class='add'>+		erdma_cm_free_work(cep);</div><div class='add'>+	spin_unlock_bh(&amp;cep-&gt;lock);</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;dev-&gt;lock, flags);</div><div class='add'>+	list_del(&amp;cep-&gt;devq);</div><div class='add'>+	spin_unlock_irqrestore(&amp;dev-&gt;lock, flags);</div><div class='add'>+	kfree(cep);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct erdma_cm_work *erdma_get_work(struct erdma_cep *cep)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cm_work *work = NULL;</div><div class='add'>+</div><div class='add'>+	spin_lock_bh(&amp;cep-&gt;lock);</div><div class='add'>+	if (!list_empty(&amp;cep-&gt;work_freelist)) {</div><div class='add'>+		work = list_entry(cep-&gt;work_freelist.next, struct erdma_cm_work,</div><div class='add'>+				  list);</div><div class='add'>+		list_del_init(&amp;work-&gt;list);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_unlock_bh(&amp;cep-&gt;lock);</div><div class='add'>+	return work;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_cm_alloc_work(struct erdma_cep *cep, int num)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cm_work *work;</div><div class='add'>+</div><div class='add'>+	while (num--) {</div><div class='add'>+		work = kmalloc(sizeof(*work), GFP_KERNEL);</div><div class='add'>+		if (!work) {</div><div class='add'>+			if (!(list_empty(&amp;cep-&gt;work_freelist)))</div><div class='add'>+				erdma_cm_free_work(cep);</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		}</div><div class='add'>+		work-&gt;cep = cep;</div><div class='add'>+		INIT_LIST_HEAD(&amp;work-&gt;list);</div><div class='add'>+		list_add(&amp;work-&gt;list, &amp;cep-&gt;work_freelist);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_cm_upcall(struct erdma_cep *cep, enum iw_cm_event_type reason,</div><div class='add'>+			   int status)</div><div class='add'>+{</div><div class='add'>+	struct iw_cm_event event;</div><div class='add'>+	struct iw_cm_id *cm_id;</div><div class='add'>+</div><div class='add'>+	memset(&amp;event, 0, sizeof(event));</div><div class='add'>+	event.status = status;</div><div class='add'>+	event.event = reason;</div><div class='add'>+</div><div class='add'>+	if (reason == IW_CM_EVENT_CONNECT_REQUEST) {</div><div class='add'>+		event.provider_data = cep;</div><div class='add'>+		cm_id = cep-&gt;listen_cep-&gt;cm_id;</div><div class='add'>+</div><div class='add'>+		event.ird = cep-&gt;dev-&gt;attrs.max_ird;</div><div class='add'>+		event.ord = cep-&gt;dev-&gt;attrs.max_ord;</div><div class='add'>+	} else {</div><div class='add'>+		cm_id = cep-&gt;cm_id;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (reason == IW_CM_EVENT_CONNECT_REQUEST ||</div><div class='add'>+	    reason == IW_CM_EVENT_CONNECT_REPLY) {</div><div class='add'>+		u16 pd_len = be16_to_cpu(cep-&gt;mpa.hdr.params.pd_len);</div><div class='add'>+</div><div class='add'>+		if (pd_len &amp;&amp; cep-&gt;mpa.pdata) {</div><div class='add'>+			event.private_data_len = pd_len;</div><div class='add'>+			event.private_data = cep-&gt;mpa.pdata;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		getname_local(cep-&gt;sock, &amp;event.local_addr);</div><div class='add'>+		getname_peer(cep-&gt;sock, &amp;event.remote_addr);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return cm_id-&gt;event_handler(cm_id, &amp;event);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_qp_cm_drop(struct erdma_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cep *cep = qp-&gt;cep;</div><div class='add'>+</div><div class='add'>+	if (!qp-&gt;cep)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	erdma_cep_set_inuse(cep);</div><div class='add'>+</div><div class='add'>+	/* already closed. */</div><div class='add'>+	if (cep-&gt;state == ERDMA_EPSTATE_CLOSED)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	if (cep-&gt;cm_id) {</div><div class='add'>+		switch (cep-&gt;state) {</div><div class='add'>+		case ERDMA_EPSTATE_AWAIT_MPAREP:</div><div class='add'>+			erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY,</div><div class='add'>+					-EINVAL);</div><div class='add'>+			break;</div><div class='add'>+		case ERDMA_EPSTATE_RDMA_MODE:</div><div class='add'>+			erdma_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0);</div><div class='add'>+			break;</div><div class='add'>+		case ERDMA_EPSTATE_IDLE:</div><div class='add'>+		case ERDMA_EPSTATE_LISTENING:</div><div class='add'>+		case ERDMA_EPSTATE_CONNECTING:</div><div class='add'>+		case ERDMA_EPSTATE_AWAIT_MPAREQ:</div><div class='add'>+		case ERDMA_EPSTATE_RECVD_MPAREQ:</div><div class='add'>+		case ERDMA_EPSTATE_CLOSED:</div><div class='add'>+		default:</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		cep-&gt;cm_id-&gt;rem_ref(cep-&gt;cm_id);</div><div class='add'>+		cep-&gt;cm_id = NULL;</div><div class='add'>+		erdma_cep_put(cep);</div><div class='add'>+	}</div><div class='add'>+	cep-&gt;state = ERDMA_EPSTATE_CLOSED;</div><div class='add'>+</div><div class='add'>+	if (cep-&gt;sock) {</div><div class='add'>+		erdma_socket_disassoc(cep-&gt;sock);</div><div class='add'>+		sock_release(cep-&gt;sock);</div><div class='add'>+		cep-&gt;sock = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (cep-&gt;qp) {</div><div class='add'>+		cep-&gt;qp = NULL;</div><div class='add'>+		erdma_qp_put(qp);</div><div class='add'>+	}</div><div class='add'>+out:</div><div class='add'>+	erdma_cep_set_free(cep);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_cep_put(struct erdma_cep *cep)</div><div class='add'>+{</div><div class='add'>+	WARN_ON(kref_read(&amp;cep-&gt;ref) &lt; 1);</div><div class='add'>+	kref_put(&amp;cep-&gt;ref, __erdma_cep_dealloc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_cep_get(struct erdma_cep *cep)</div><div class='add'>+{</div><div class='add'>+	kref_get(&amp;cep-&gt;ref);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_send_mpareqrep(struct erdma_cep *cep, const void *pdata,</div><div class='add'>+				u8 pd_len)</div><div class='add'>+{</div><div class='add'>+	struct socket *s = cep-&gt;sock;</div><div class='add'>+	struct mpa_rr *rr = &amp;cep-&gt;mpa.hdr;</div><div class='add'>+	struct kvec iov[3];</div><div class='add'>+	struct msghdr msg;</div><div class='add'>+	int iovec_num = 0;</div><div class='add'>+	int ret;</div><div class='add'>+	int mpa_len;</div><div class='add'>+</div><div class='add'>+	memset(&amp;msg, 0, sizeof(msg));</div><div class='add'>+</div><div class='add'>+	rr-&gt;params.pd_len = cpu_to_be16(pd_len);</div><div class='add'>+</div><div class='add'>+	iov[iovec_num].iov_base = rr;</div><div class='add'>+	iov[iovec_num].iov_len = sizeof(*rr);</div><div class='add'>+	iovec_num++;</div><div class='add'>+	mpa_len = sizeof(*rr);</div><div class='add'>+</div><div class='add'>+	iov[iovec_num].iov_base = &amp;cep-&gt;mpa.ext_data;</div><div class='add'>+	iov[iovec_num].iov_len = sizeof(cep-&gt;mpa.ext_data);</div><div class='add'>+	iovec_num++;</div><div class='add'>+	mpa_len += sizeof(cep-&gt;mpa.ext_data);</div><div class='add'>+</div><div class='add'>+	if (pd_len) {</div><div class='add'>+		iov[iovec_num].iov_base = (char *)pdata;</div><div class='add'>+		iov[iovec_num].iov_len = pd_len;</div><div class='add'>+		mpa_len += pd_len;</div><div class='add'>+		iovec_num++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = kernel_sendmsg(s, &amp;msg, iov, iovec_num, mpa_len);</div><div class='add'>+</div><div class='add'>+	return ret &lt; 0 ? ret : 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int ksock_recv(struct socket *sock, char *buf, size_t size,</div><div class='add'>+			     int flags)</div><div class='add'>+{</div><div class='add'>+	struct kvec iov = { buf, size };</div><div class='add'>+	struct msghdr msg = { .msg_name = NULL, .msg_flags = flags };</div><div class='add'>+</div><div class='add'>+	return kernel_recvmsg(sock, &amp;msg, &amp;iov, 1, size, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int __recv_mpa_hdr(struct erdma_cep *cep, int hdr_rcvd, char *hdr,</div><div class='add'>+			  int hdr_size, int *rcvd_out)</div><div class='add'>+{</div><div class='add'>+	struct socket *s = cep-&gt;sock;</div><div class='add'>+	int rcvd;</div><div class='add'>+</div><div class='add'>+	*rcvd_out = 0;</div><div class='add'>+	if (hdr_rcvd &lt; hdr_size) {</div><div class='add'>+		rcvd = ksock_recv(s, hdr + hdr_rcvd, hdr_size - hdr_rcvd,</div><div class='add'>+				  MSG_DONTWAIT);</div><div class='add'>+		if (rcvd == -EAGAIN)</div><div class='add'>+			return -EAGAIN;</div><div class='add'>+</div><div class='add'>+		if (rcvd &lt;= 0)</div><div class='add'>+			return -ECONNABORTED;</div><div class='add'>+</div><div class='add'>+		hdr_rcvd += rcvd;</div><div class='add'>+		*rcvd_out = rcvd;</div><div class='add'>+</div><div class='add'>+		if (hdr_rcvd &lt; hdr_size)</div><div class='add'>+			return -EAGAIN;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void __mpa_rr_set_revision(__be16 *bits, u8 rev)</div><div class='add'>+{</div><div class='add'>+	*bits = (*bits &amp; ~MPA_RR_MASK_REVISION) |</div><div class='add'>+		(cpu_to_be16(rev) &amp; MPA_RR_MASK_REVISION);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u8 __mpa_rr_revision(__be16 mpa_rr_bits)</div><div class='add'>+{</div><div class='add'>+	__be16 rev = mpa_rr_bits &amp; MPA_RR_MASK_REVISION;</div><div class='add'>+</div><div class='add'>+	return (u8)be16_to_cpu(rev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void __mpa_ext_set_cc(__be32 *bits, u32 cc)</div><div class='add'>+{</div><div class='add'>+	*bits = (*bits &amp; ~MPA_EXT_FLAG_CC) |</div><div class='add'>+		(cpu_to_be32(cc) &amp; MPA_EXT_FLAG_CC);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u8 __mpa_ext_cc(__be32 mpa_ext_bits)</div><div class='add'>+{</div><div class='add'>+	__be32 cc = mpa_ext_bits &amp; MPA_EXT_FLAG_CC;</div><div class='add'>+</div><div class='add'>+	return (u8)be32_to_cpu(cc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * Receive MPA Request/Reply header.</div><div class='add'>+ *</div><div class='add'>+ * Returns 0 if complete MPA Request/Reply haeder including</div><div class='add'>+ * eventual private data was received. Returns -EAGAIN if</div><div class='add'>+ * header was partially received or negative error code otherwise.</div><div class='add'>+ *</div><div class='add'>+ * Context: May be called in process context only</div><div class='add'>+ */</div><div class='add'>+static int erdma_recv_mpa_rr(struct erdma_cep *cep)</div><div class='add'>+{</div><div class='add'>+	struct mpa_rr *hdr = &amp;cep-&gt;mpa.hdr;</div><div class='add'>+	struct socket *s = cep-&gt;sock;</div><div class='add'>+	u16 pd_len;</div><div class='add'>+	int rcvd, to_rcv, ret, pd_rcvd;</div><div class='add'>+</div><div class='add'>+	if (cep-&gt;mpa.bytes_rcvd &lt; sizeof(struct mpa_rr)) {</div><div class='add'>+		ret = __recv_mpa_hdr(cep, cep-&gt;mpa.bytes_rcvd,</div><div class='add'>+				     (char *)&amp;cep-&gt;mpa.hdr,</div><div class='add'>+				     sizeof(struct mpa_rr), &amp;rcvd);</div><div class='add'>+		cep-&gt;mpa.bytes_rcvd += rcvd;</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (be16_to_cpu(hdr-&gt;params.pd_len) &gt; MPA_MAX_PRIVDATA ||</div><div class='add'>+	    __mpa_rr_revision(hdr-&gt;params.bits) != MPA_REVISION_EXT_1)</div><div class='add'>+		return -EPROTO;</div><div class='add'>+</div><div class='add'>+	if (cep-&gt;mpa.bytes_rcvd - sizeof(struct mpa_rr) &lt;</div><div class='add'>+	    sizeof(struct erdma_mpa_ext)) {</div><div class='add'>+		ret = __recv_mpa_hdr(</div><div class='add'>+			cep, cep-&gt;mpa.bytes_rcvd - sizeof(struct mpa_rr),</div><div class='add'>+			(char *)&amp;cep-&gt;mpa.ext_data,</div><div class='add'>+			sizeof(struct erdma_mpa_ext), &amp;rcvd);</div><div class='add'>+		cep-&gt;mpa.bytes_rcvd += rcvd;</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	pd_len = be16_to_cpu(hdr-&gt;params.pd_len);</div><div class='add'>+	pd_rcvd = cep-&gt;mpa.bytes_rcvd - sizeof(struct mpa_rr) -</div><div class='add'>+		  sizeof(struct erdma_mpa_ext);</div><div class='add'>+	to_rcv = pd_len - pd_rcvd;</div><div class='add'>+</div><div class='add'>+	if (!to_rcv) {</div><div class='add'>+		/*</div><div class='add'>+		 * We have received the whole MPA Request/Reply message.</div><div class='add'>+		 * Check against peer protocol violation.</div><div class='add'>+		 */</div><div class='add'>+		u32 word;</div><div class='add'>+</div><div class='add'>+		ret = __recv_mpa_hdr(cep, 0, (char *)&amp;word, sizeof(word),</div><div class='add'>+				     &amp;rcvd);</div><div class='add'>+		if (ret == -EAGAIN &amp;&amp; rcvd == 0)</div><div class='add'>+			return 0;</div><div class='add'>+</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+</div><div class='add'>+		return -EPROTO;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * At this point, MPA header has been fully received, and pd_len != 0.</div><div class='add'>+	 * So, begin to receive private data.</div><div class='add'>+	 */</div><div class='add'>+	if (!cep-&gt;mpa.pdata) {</div><div class='add'>+		cep-&gt;mpa.pdata = kmalloc(pd_len + 4, GFP_KERNEL);</div><div class='add'>+		if (!cep-&gt;mpa.pdata)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rcvd = ksock_recv(s, cep-&gt;mpa.pdata + pd_rcvd, to_rcv + 4,</div><div class='add'>+			  MSG_DONTWAIT);</div><div class='add'>+	if (rcvd &lt; 0)</div><div class='add'>+		return rcvd;</div><div class='add'>+</div><div class='add'>+	if (rcvd &gt; to_rcv)</div><div class='add'>+		return -EPROTO;</div><div class='add'>+</div><div class='add'>+	cep-&gt;mpa.bytes_rcvd += rcvd;</div><div class='add'>+</div><div class='add'>+	if (to_rcv == rcvd)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	return -EAGAIN;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * erdma_proc_mpareq()</div><div class='add'>+ *</div><div class='add'>+ * Read MPA Request from socket and signal new connection to IWCM</div><div class='add'>+ * if success. Caller must hold lock on corresponding listening CEP.</div><div class='add'>+ */</div><div class='add'>+static int erdma_proc_mpareq(struct erdma_cep *cep)</div><div class='add'>+{</div><div class='add'>+	struct mpa_rr *req;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = erdma_recv_mpa_rr(cep);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	req = &amp;cep-&gt;mpa.hdr;</div><div class='add'>+</div><div class='add'>+	if (memcmp(req-&gt;key, MPA_KEY_REQ, MPA_KEY_SIZE))</div><div class='add'>+		return -EPROTO;</div><div class='add'>+</div><div class='add'>+	memcpy(req-&gt;key, MPA_KEY_REP, MPA_KEY_SIZE);</div><div class='add'>+</div><div class='add'>+	/* Currently does not support marker and crc. */</div><div class='add'>+	if (req-&gt;params.bits &amp; MPA_RR_FLAG_MARKERS ||</div><div class='add'>+	    req-&gt;params.bits &amp; MPA_RR_FLAG_CRC)</div><div class='add'>+		goto reject_conn;</div><div class='add'>+</div><div class='add'>+	cep-&gt;state = ERDMA_EPSTATE_RECVD_MPAREQ;</div><div class='add'>+</div><div class='add'>+	/* Keep reference until IWCM accepts/rejects */</div><div class='add'>+	erdma_cep_get(cep);</div><div class='add'>+	ret = erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REQUEST, 0);</div><div class='add'>+	if (ret)</div><div class='add'>+		erdma_cep_put(cep);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+</div><div class='add'>+reject_conn:</div><div class='add'>+	req-&gt;params.bits &amp;= ~MPA_RR_FLAG_MARKERS;</div><div class='add'>+	req-&gt;params.bits |= MPA_RR_FLAG_REJECT;</div><div class='add'>+	req-&gt;params.bits &amp;= ~MPA_RR_FLAG_CRC;</div><div class='add'>+</div><div class='add'>+	kfree(cep-&gt;mpa.pdata);</div><div class='add'>+	cep-&gt;mpa.pdata = NULL;</div><div class='add'>+	erdma_send_mpareqrep(cep, NULL, 0);</div><div class='add'>+</div><div class='add'>+	return -EOPNOTSUPP;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_proc_mpareply(struct erdma_cep *cep)</div><div class='add'>+{</div><div class='add'>+	enum erdma_qpa_mask_iwarp to_modify_attrs = 0;</div><div class='add'>+	struct erdma_mod_qp_params_iwarp params;</div><div class='add'>+	struct erdma_qp *qp = cep-&gt;qp;</div><div class='add'>+	struct mpa_rr *rep;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = erdma_recv_mpa_rr(cep);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out_err;</div><div class='add'>+</div><div class='add'>+	erdma_cancel_mpatimer(cep);</div><div class='add'>+</div><div class='add'>+	rep = &amp;cep-&gt;mpa.hdr;</div><div class='add'>+</div><div class='add'>+	if (memcmp(rep-&gt;key, MPA_KEY_REP, MPA_KEY_SIZE)) {</div><div class='add'>+		ret = -EPROTO;</div><div class='add'>+		goto out_err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (rep-&gt;params.bits &amp; MPA_RR_FLAG_REJECT) {</div><div class='add'>+		erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -ECONNRESET);</div><div class='add'>+		return -ECONNRESET;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Currently does not support marker and crc. */</div><div class='add'>+	if ((rep-&gt;params.bits &amp; MPA_RR_FLAG_MARKERS) ||</div><div class='add'>+	    (rep-&gt;params.bits &amp; MPA_RR_FLAG_CRC)) {</div><div class='add'>+		erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -ECONNREFUSED);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	memset(&amp;params, 0, sizeof(params));</div><div class='add'>+	params.state = ERDMA_QPS_IWARP_RTS;</div><div class='add'>+	params.irq_size = cep-&gt;ird;</div><div class='add'>+	params.orq_size = cep-&gt;ord;</div><div class='add'>+</div><div class='add'>+	down_write(&amp;qp-&gt;state_lock);</div><div class='add'>+	if (qp-&gt;attrs.iwarp.state &gt; ERDMA_QPS_IWARP_RTR) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		up_write(&amp;qp-&gt;state_lock);</div><div class='add'>+		goto out_err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	to_modify_attrs = ERDMA_QPA_IWARP_STATE | ERDMA_QPA_IWARP_LLP_HANDLE |</div><div class='add'>+			  ERDMA_QPA_IWARP_MPA | ERDMA_QPA_IWARP_IRD |</div><div class='add'>+			  ERDMA_QPA_IWARP_ORD;</div><div class='add'>+</div><div class='add'>+	params.qp_type = ERDMA_QP_ACTIVE;</div><div class='add'>+	if (__mpa_ext_cc(cep-&gt;mpa.ext_data.bits) != qp-&gt;attrs.cc) {</div><div class='add'>+		to_modify_attrs |= ERDMA_QPA_IWARP_CC;</div><div class='add'>+		params.cc = COMPROMISE_CC;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = erdma_modify_qp_state_iwarp(qp, &amp;params, to_modify_attrs);</div><div class='add'>+</div><div class='add'>+	up_write(&amp;qp-&gt;state_lock);</div><div class='add'>+</div><div class='add'>+	if (!ret) {</div><div class='add'>+		ret = erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 0);</div><div class='add'>+		if (!ret)</div><div class='add'>+			cep-&gt;state = ERDMA_EPSTATE_RDMA_MODE;</div><div class='add'>+</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out_err:</div><div class='add'>+	if (ret != -EAGAIN)</div><div class='add'>+		erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -EINVAL);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_accept_newconn(struct erdma_cep *cep)</div><div class='add'>+{</div><div class='add'>+	struct socket *s = cep-&gt;sock;</div><div class='add'>+	struct socket *new_s = NULL;</div><div class='add'>+	struct erdma_cep *new_cep = NULL;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	if (cep-&gt;state != ERDMA_EPSTATE_LISTENING)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	new_cep = erdma_cep_alloc(cep-&gt;dev);</div><div class='add'>+	if (!new_cep)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * 4: Allocate a sufficient number of work elements</div><div class='add'>+	 * to allow concurrent handling of local + peer close</div><div class='add'>+	 * events, MPA header processing + MPA timeout.</div><div class='add'>+	 */</div><div class='add'>+	if (erdma_cm_alloc_work(new_cep, 4) != 0)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Copy saved socket callbacks from listening CEP</div><div class='add'>+	 * and assign new socket with new CEP</div><div class='add'>+	 */</div><div class='add'>+	new_cep-&gt;sk_state_change = cep-&gt;sk_state_change;</div><div class='add'>+	new_cep-&gt;sk_data_ready = cep-&gt;sk_data_ready;</div><div class='add'>+	new_cep-&gt;sk_error_report = cep-&gt;sk_error_report;</div><div class='add'>+</div><div class='add'>+	ret = kernel_accept(s, &amp;new_s, O_NONBLOCK);</div><div class='add'>+	if (ret != 0)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	new_cep-&gt;sock = new_s;</div><div class='add'>+	erdma_cep_get(new_cep);</div><div class='add'>+	new_s-&gt;sk-&gt;sk_user_data = new_cep;</div><div class='add'>+</div><div class='add'>+	tcp_sock_set_nodelay(new_s-&gt;sk);</div><div class='add'>+	new_cep-&gt;state = ERDMA_EPSTATE_AWAIT_MPAREQ;</div><div class='add'>+</div><div class='add'>+	ret = erdma_cm_queue_work(new_cep, ERDMA_CM_WORK_MPATIMEOUT);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	new_cep-&gt;listen_cep = cep;</div><div class='add'>+	erdma_cep_get(cep);</div><div class='add'>+</div><div class='add'>+	if (atomic_read(&amp;new_s-&gt;sk-&gt;sk_rmem_alloc)) {</div><div class='add'>+		/* MPA REQ already queued */</div><div class='add'>+		erdma_cep_set_inuse(new_cep);</div><div class='add'>+		ret = erdma_proc_mpareq(new_cep);</div><div class='add'>+		if (ret != -EAGAIN) {</div><div class='add'>+			erdma_cep_put(cep);</div><div class='add'>+			new_cep-&gt;listen_cep = NULL;</div><div class='add'>+			if (ret) {</div><div class='add'>+				erdma_cep_set_free(new_cep);</div><div class='add'>+				goto error;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+		erdma_cep_set_free(new_cep);</div><div class='add'>+	}</div><div class='add'>+	return;</div><div class='add'>+</div><div class='add'>+error:</div><div class='add'>+	if (new_cep) {</div><div class='add'>+		new_cep-&gt;state = ERDMA_EPSTATE_CLOSED;</div><div class='add'>+		erdma_cancel_mpatimer(new_cep);</div><div class='add'>+</div><div class='add'>+		erdma_cep_put(new_cep);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (new_s) {</div><div class='add'>+		erdma_socket_disassoc(new_s);</div><div class='add'>+		sock_release(new_s);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_newconn_connected(struct erdma_cep *cep)</div><div class='add'>+{</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	cep-&gt;mpa.hdr.params.bits = 0;</div><div class='add'>+	__mpa_rr_set_revision(&amp;cep-&gt;mpa.hdr.params.bits, MPA_REVISION_EXT_1);</div><div class='add'>+</div><div class='add'>+	memcpy(cep-&gt;mpa.hdr.key, MPA_KEY_REQ, MPA_KEY_SIZE);</div><div class='add'>+	cep-&gt;mpa.ext_data.cookie = cpu_to_be32(cep-&gt;qp-&gt;attrs.iwarp.cookie);</div><div class='add'>+	__mpa_ext_set_cc(&amp;cep-&gt;mpa.ext_data.bits, cep-&gt;qp-&gt;attrs.cc);</div><div class='add'>+</div><div class='add'>+	ret = erdma_send_mpareqrep(cep, cep-&gt;private_data, cep-&gt;pd_len);</div><div class='add'>+	cep-&gt;state = ERDMA_EPSTATE_AWAIT_MPAREP;</div><div class='add'>+	cep-&gt;mpa.hdr.params.pd_len = 0;</div><div class='add'>+</div><div class='add'>+	if (ret &gt;= 0)</div><div class='add'>+		ret = erdma_cm_queue_work(cep, ERDMA_CM_WORK_MPATIMEOUT);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_cm_work_handler(struct work_struct *w)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cm_work *work;</div><div class='add'>+	struct erdma_cep *cep;</div><div class='add'>+	int release_cep = 0, ret = 0;</div><div class='add'>+</div><div class='add'>+	work = container_of(w, struct erdma_cm_work, work.work);</div><div class='add'>+	cep = work-&gt;cep;</div><div class='add'>+</div><div class='add'>+	erdma_cep_set_inuse(cep);</div><div class='add'>+</div><div class='add'>+	switch (work-&gt;type) {</div><div class='add'>+	case ERDMA_CM_WORK_CONNECTED:</div><div class='add'>+		erdma_cancel_mpatimer(cep);</div><div class='add'>+		if (cep-&gt;state == ERDMA_EPSTATE_CONNECTING) {</div><div class='add'>+			ret = erdma_newconn_connected(cep);</div><div class='add'>+			if (ret) {</div><div class='add'>+				erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY,</div><div class='add'>+						-EIO);</div><div class='add'>+				release_cep = 1;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	case ERDMA_CM_WORK_CONNECTTIMEOUT:</div><div class='add'>+		if (cep-&gt;state == ERDMA_EPSTATE_CONNECTING) {</div><div class='add'>+			cep-&gt;mpa_timer = NULL;</div><div class='add'>+			erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY,</div><div class='add'>+					-ETIMEDOUT);</div><div class='add'>+			release_cep = 1;</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	case ERDMA_CM_WORK_ACCEPT:</div><div class='add'>+		erdma_accept_newconn(cep);</div><div class='add'>+		break;</div><div class='add'>+	case ERDMA_CM_WORK_READ_MPAHDR:</div><div class='add'>+		if (cep-&gt;state == ERDMA_EPSTATE_AWAIT_MPAREQ) {</div><div class='add'>+			if (cep-&gt;listen_cep) {</div><div class='add'>+				erdma_cep_set_inuse(cep-&gt;listen_cep);</div><div class='add'>+</div><div class='add'>+				if (cep-&gt;listen_cep-&gt;state ==</div><div class='add'>+				    ERDMA_EPSTATE_LISTENING)</div><div class='add'>+					ret = erdma_proc_mpareq(cep);</div><div class='add'>+				else</div><div class='add'>+					ret = -EFAULT;</div><div class='add'>+</div><div class='add'>+				erdma_cep_set_free(cep-&gt;listen_cep);</div><div class='add'>+</div><div class='add'>+				if (ret != -EAGAIN) {</div><div class='add'>+					erdma_cep_put(cep-&gt;listen_cep);</div><div class='add'>+					cep-&gt;listen_cep = NULL;</div><div class='add'>+					if (ret)</div><div class='add'>+						erdma_cep_put(cep);</div><div class='add'>+				}</div><div class='add'>+			}</div><div class='add'>+		} else if (cep-&gt;state == ERDMA_EPSTATE_AWAIT_MPAREP) {</div><div class='add'>+			ret = erdma_proc_mpareply(cep);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (ret &amp;&amp; ret != -EAGAIN)</div><div class='add'>+			release_cep = 1;</div><div class='add'>+		break;</div><div class='add'>+	case ERDMA_CM_WORK_CLOSE_LLP:</div><div class='add'>+		if (cep-&gt;cm_id)</div><div class='add'>+			erdma_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0);</div><div class='add'>+		release_cep = 1;</div><div class='add'>+		break;</div><div class='add'>+	case ERDMA_CM_WORK_PEER_CLOSE:</div><div class='add'>+		if (cep-&gt;cm_id) {</div><div class='add'>+			if (cep-&gt;state == ERDMA_EPSTATE_CONNECTING ||</div><div class='add'>+			    cep-&gt;state == ERDMA_EPSTATE_AWAIT_MPAREP) {</div><div class='add'>+				/*</div><div class='add'>+				 * MPA reply not received, but connection drop</div><div class='add'>+				 */</div><div class='add'>+				erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY,</div><div class='add'>+						-ECONNRESET);</div><div class='add'>+			} else if (cep-&gt;state == ERDMA_EPSTATE_RDMA_MODE) {</div><div class='add'>+				/*</div><div class='add'>+				 * NOTE: IW_CM_EVENT_DISCONNECT is given just</div><div class='add'>+				 *       to transition IWCM into CLOSING.</div><div class='add'>+				 */</div><div class='add'>+				erdma_cm_upcall(cep, IW_CM_EVENT_DISCONNECT, 0);</div><div class='add'>+				erdma_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0);</div><div class='add'>+			}</div><div class='add'>+		} else if (cep-&gt;state == ERDMA_EPSTATE_AWAIT_MPAREQ) {</div><div class='add'>+			/* Socket close before MPA request received. */</div><div class='add'>+			erdma_disassoc_listen_cep(cep);</div><div class='add'>+			erdma_cep_put(cep);</div><div class='add'>+		}</div><div class='add'>+		release_cep = 1;</div><div class='add'>+		break;</div><div class='add'>+	case ERDMA_CM_WORK_MPATIMEOUT:</div><div class='add'>+		cep-&gt;mpa_timer = NULL;</div><div class='add'>+		if (cep-&gt;state == ERDMA_EPSTATE_AWAIT_MPAREP) {</div><div class='add'>+			/*</div><div class='add'>+			 * MPA request timed out:</div><div class='add'>+			 * Hide any partially received private data and signal</div><div class='add'>+			 * timeout</div><div class='add'>+			 */</div><div class='add'>+			cep-&gt;mpa.hdr.params.pd_len = 0;</div><div class='add'>+</div><div class='add'>+			if (cep-&gt;cm_id)</div><div class='add'>+				erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY,</div><div class='add'>+						-ETIMEDOUT);</div><div class='add'>+			release_cep = 1;</div><div class='add'>+		} else if (cep-&gt;state == ERDMA_EPSTATE_AWAIT_MPAREQ) {</div><div class='add'>+			/* No MPA req received after peer TCP stream setup. */</div><div class='add'>+			erdma_disassoc_listen_cep(cep);</div><div class='add'>+</div><div class='add'>+			erdma_cep_put(cep);</div><div class='add'>+			release_cep = 1;</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		WARN(1, "Undefined CM work type: %d\n", work-&gt;type);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (release_cep) {</div><div class='add'>+		erdma_cancel_mpatimer(cep);</div><div class='add'>+		cep-&gt;state = ERDMA_EPSTATE_CLOSED;</div><div class='add'>+		if (cep-&gt;qp) {</div><div class='add'>+			struct erdma_qp *qp = cep-&gt;qp;</div><div class='add'>+			/*</div><div class='add'>+			 * Serialize a potential race with application</div><div class='add'>+			 * closing the QP and calling erdma_qp_cm_drop()</div><div class='add'>+			 */</div><div class='add'>+			erdma_qp_get(qp);</div><div class='add'>+			erdma_cep_set_free(cep);</div><div class='add'>+</div><div class='add'>+			erdma_qp_llp_close(qp);</div><div class='add'>+			erdma_qp_put(qp);</div><div class='add'>+</div><div class='add'>+			erdma_cep_set_inuse(cep);</div><div class='add'>+			cep-&gt;qp = NULL;</div><div class='add'>+			erdma_qp_put(qp);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (cep-&gt;sock) {</div><div class='add'>+			erdma_socket_disassoc(cep-&gt;sock);</div><div class='add'>+			sock_release(cep-&gt;sock);</div><div class='add'>+			cep-&gt;sock = NULL;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (cep-&gt;cm_id) {</div><div class='add'>+			cep-&gt;cm_id-&gt;rem_ref(cep-&gt;cm_id);</div><div class='add'>+			cep-&gt;cm_id = NULL;</div><div class='add'>+			if (cep-&gt;state != ERDMA_EPSTATE_LISTENING)</div><div class='add'>+				erdma_cep_put(cep);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	erdma_cep_set_free(cep);</div><div class='add'>+	erdma_put_work(work);</div><div class='add'>+	erdma_cep_put(cep);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_cm_queue_work(struct erdma_cep *cep, enum erdma_work_type type)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cm_work *work = erdma_get_work(cep);</div><div class='add'>+	unsigned long delay = 0;</div><div class='add'>+</div><div class='add'>+	if (!work)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	work-&gt;type = type;</div><div class='add'>+	work-&gt;cep = cep;</div><div class='add'>+</div><div class='add'>+	erdma_cep_get(cep);</div><div class='add'>+</div><div class='add'>+	INIT_DELAYED_WORK(&amp;work-&gt;work, erdma_cm_work_handler);</div><div class='add'>+</div><div class='add'>+	if (type == ERDMA_CM_WORK_MPATIMEOUT) {</div><div class='add'>+		cep-&gt;mpa_timer = work;</div><div class='add'>+</div><div class='add'>+		if (cep-&gt;state == ERDMA_EPSTATE_AWAIT_MPAREP)</div><div class='add'>+			delay = MPAREP_TIMEOUT;</div><div class='add'>+		else</div><div class='add'>+			delay = MPAREQ_TIMEOUT;</div><div class='add'>+	} else if (type == ERDMA_CM_WORK_CONNECTTIMEOUT) {</div><div class='add'>+		cep-&gt;mpa_timer = work;</div><div class='add'>+</div><div class='add'>+		delay = CONNECT_TIMEOUT;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	queue_delayed_work(erdma_cm_wq, &amp;work-&gt;work, delay);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_cm_llp_data_ready(struct sock *sk)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cep *cep;</div><div class='add'>+</div><div class='add'>+	trace_sk_data_ready(sk);</div><div class='add'>+</div><div class='add'>+	read_lock(&amp;sk-&gt;sk_callback_lock);</div><div class='add'>+</div><div class='add'>+	cep = sk_to_cep(sk);</div><div class='add'>+	if (!cep)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	if (cep-&gt;state == ERDMA_EPSTATE_AWAIT_MPAREQ ||</div><div class='add'>+	    cep-&gt;state == ERDMA_EPSTATE_AWAIT_MPAREP)</div><div class='add'>+		erdma_cm_queue_work(cep, ERDMA_CM_WORK_READ_MPAHDR);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	read_unlock(&amp;sk-&gt;sk_callback_lock);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_cm_llp_error_report(struct sock *sk)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cep *cep = sk_to_cep(sk);</div><div class='add'>+</div><div class='add'>+	if (cep)</div><div class='add'>+		cep-&gt;sk_error_report(sk);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_cm_llp_state_change(struct sock *sk)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cep *cep;</div><div class='add'>+	void (*orig_state_change)(struct sock *sk);</div><div class='add'>+</div><div class='add'>+	read_lock(&amp;sk-&gt;sk_callback_lock);</div><div class='add'>+</div><div class='add'>+	cep = sk_to_cep(sk);</div><div class='add'>+	if (!cep) {</div><div class='add'>+		read_unlock(&amp;sk-&gt;sk_callback_lock);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+	orig_state_change = cep-&gt;sk_state_change;</div><div class='add'>+</div><div class='add'>+	switch (sk-&gt;sk_state) {</div><div class='add'>+	case TCP_ESTABLISHED:</div><div class='add'>+		if (cep-&gt;state == ERDMA_EPSTATE_CONNECTING)</div><div class='add'>+			erdma_cm_queue_work(cep, ERDMA_CM_WORK_CONNECTED);</div><div class='add'>+		else</div><div class='add'>+			erdma_cm_queue_work(cep, ERDMA_CM_WORK_ACCEPT);</div><div class='add'>+		break;</div><div class='add'>+	case TCP_CLOSE:</div><div class='add'>+	case TCP_CLOSE_WAIT:</div><div class='add'>+		if (cep-&gt;state != ERDMA_EPSTATE_LISTENING)</div><div class='add'>+			erdma_cm_queue_work(cep, ERDMA_CM_WORK_PEER_CLOSE);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+	read_unlock(&amp;sk-&gt;sk_callback_lock);</div><div class='add'>+	orig_state_change(sk);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int kernel_bindconnect(struct socket *s, struct sockaddr *laddr,</div><div class='add'>+			      int laddrlen, struct sockaddr *raddr,</div><div class='add'>+			      int raddrlen, int flags)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	sock_set_reuseaddr(s-&gt;sk);</div><div class='add'>+	ret = s-&gt;ops-&gt;bind(s, (struct sockaddr_unsized *)laddr, laddrlen);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+	ret = s-&gt;ops-&gt;connect(s, (struct sockaddr_unsized *)raddr, raddrlen, flags);</div><div class='add'>+	return ret &lt; 0 ? ret : 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_connect(struct iw_cm_id *id, struct iw_cm_conn_param *params)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = to_edev(id-&gt;device);</div><div class='add'>+	struct erdma_qp *qp;</div><div class='add'>+	struct erdma_cep *cep = NULL;</div><div class='add'>+	struct socket *s = NULL;</div><div class='add'>+	struct sockaddr *laddr = (struct sockaddr *)&amp;id-&gt;m_local_addr;</div><div class='add'>+	struct sockaddr *raddr = (struct sockaddr *)&amp;id-&gt;m_remote_addr;</div><div class='add'>+	u16 pd_len = params-&gt;private_data_len;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (pd_len &gt; MPA_MAX_PRIVDATA)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (params-&gt;ird &gt; dev-&gt;attrs.max_ird ||</div><div class='add'>+	    params-&gt;ord &gt; dev-&gt;attrs.max_ord)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (laddr-&gt;sa_family != AF_INET || raddr-&gt;sa_family != AF_INET)</div><div class='add'>+		return -EAFNOSUPPORT;</div><div class='add'>+</div><div class='add'>+	qp = find_qp_by_qpn(dev, params-&gt;qpn);</div><div class='add'>+	if (!qp)</div><div class='add'>+		return -ENOENT;</div><div class='add'>+	erdma_qp_get(qp);</div><div class='add'>+</div><div class='add'>+	ret = sock_create(AF_INET, SOCK_STREAM, IPPROTO_TCP, &amp;s);</div><div class='add'>+	if (ret &lt; 0)</div><div class='add'>+		goto error_put_qp;</div><div class='add'>+</div><div class='add'>+	cep = erdma_cep_alloc(dev);</div><div class='add'>+	if (!cep) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto error_release_sock;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	erdma_cep_set_inuse(cep);</div><div class='add'>+</div><div class='add'>+	/* Associate QP with CEP */</div><div class='add'>+	erdma_cep_get(cep);</div><div class='add'>+	qp-&gt;cep = cep;</div><div class='add'>+	cep-&gt;qp = qp;</div><div class='add'>+</div><div class='add'>+	/* Associate cm_id with CEP */</div><div class='add'>+	id-&gt;add_ref(id);</div><div class='add'>+	cep-&gt;cm_id = id;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * 6: Allocate a sufficient number of work elements</div><div class='add'>+	 * to allow concurrent handling of local + peer close</div><div class='add'>+	 * events, MPA header processing + MPA timeout, connected event</div><div class='add'>+	 * and connect timeout.</div><div class='add'>+	 */</div><div class='add'>+	ret = erdma_cm_alloc_work(cep, 6);</div><div class='add'>+	if (ret != 0) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto error_release_cep;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cep-&gt;ird = params-&gt;ird;</div><div class='add'>+	cep-&gt;ord = params-&gt;ord;</div><div class='add'>+	cep-&gt;state = ERDMA_EPSTATE_CONNECTING;</div><div class='add'>+</div><div class='add'>+	erdma_cep_socket_assoc(cep, s);</div><div class='add'>+</div><div class='add'>+	if (pd_len) {</div><div class='add'>+		cep-&gt;pd_len = pd_len;</div><div class='add'>+		cep-&gt;private_data = kmalloc(pd_len, GFP_KERNEL);</div><div class='add'>+		if (!cep-&gt;private_data) {</div><div class='add'>+			ret = -ENOMEM;</div><div class='add'>+			goto error_disassoc;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		memcpy(cep-&gt;private_data, params-&gt;private_data,</div><div class='add'>+		       params-&gt;private_data_len);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = kernel_bindconnect(s, laddr, sizeof(*laddr), raddr,</div><div class='add'>+				 sizeof(*raddr), O_NONBLOCK);</div><div class='add'>+	if (ret != -EINPROGRESS &amp;&amp; ret != 0) {</div><div class='add'>+		goto error_disassoc;</div><div class='add'>+	} else if (ret == 0) {</div><div class='add'>+		ret = erdma_cm_queue_work(cep, ERDMA_CM_WORK_CONNECTED);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto error_disassoc;</div><div class='add'>+	} else {</div><div class='add'>+		ret = erdma_cm_queue_work(cep, ERDMA_CM_WORK_CONNECTTIMEOUT);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto error_disassoc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	erdma_cep_set_free(cep);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+error_disassoc:</div><div class='add'>+	kfree(cep-&gt;private_data);</div><div class='add'>+	cep-&gt;private_data = NULL;</div><div class='add'>+	cep-&gt;pd_len = 0;</div><div class='add'>+</div><div class='add'>+	erdma_socket_disassoc(s);</div><div class='add'>+</div><div class='add'>+error_release_cep:</div><div class='add'>+	/* disassoc with cm_id */</div><div class='add'>+	cep-&gt;cm_id = NULL;</div><div class='add'>+	id-&gt;rem_ref(id);</div><div class='add'>+</div><div class='add'>+	/* disassoc with qp */</div><div class='add'>+	qp-&gt;cep = NULL;</div><div class='add'>+	erdma_cep_put(cep);</div><div class='add'>+	cep-&gt;qp = NULL;</div><div class='add'>+</div><div class='add'>+	cep-&gt;state = ERDMA_EPSTATE_CLOSED;</div><div class='add'>+</div><div class='add'>+	erdma_cep_set_free(cep);</div><div class='add'>+</div><div class='add'>+	/* release the cep. */</div><div class='add'>+	erdma_cep_put(cep);</div><div class='add'>+</div><div class='add'>+error_release_sock:</div><div class='add'>+	if (s)</div><div class='add'>+		sock_release(s);</div><div class='add'>+error_put_qp:</div><div class='add'>+	erdma_qp_put(qp);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_accept(struct iw_cm_id *id, struct iw_cm_conn_param *params)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cep *cep = (struct erdma_cep *)id-&gt;provider_data;</div><div class='add'>+	struct erdma_mod_qp_params_iwarp mod_qp_params;</div><div class='add'>+	enum erdma_qpa_mask_iwarp to_modify_attrs = 0;</div><div class='add'>+	struct erdma_dev *dev = to_edev(id-&gt;device);</div><div class='add'>+	struct erdma_qp *qp;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	erdma_cep_set_inuse(cep);</div><div class='add'>+	erdma_cep_put(cep);</div><div class='add'>+</div><div class='add'>+	/* Free lingering inbound private data */</div><div class='add'>+	if (cep-&gt;mpa.hdr.params.pd_len) {</div><div class='add'>+		cep-&gt;mpa.hdr.params.pd_len = 0;</div><div class='add'>+		kfree(cep-&gt;mpa.pdata);</div><div class='add'>+		cep-&gt;mpa.pdata = NULL;</div><div class='add'>+	}</div><div class='add'>+	erdma_cancel_mpatimer(cep);</div><div class='add'>+</div><div class='add'>+	if (cep-&gt;state != ERDMA_EPSTATE_RECVD_MPAREQ) {</div><div class='add'>+		erdma_cep_set_free(cep);</div><div class='add'>+		erdma_cep_put(cep);</div><div class='add'>+</div><div class='add'>+		return -ECONNRESET;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	qp = find_qp_by_qpn(dev, params-&gt;qpn);</div><div class='add'>+	if (!qp)</div><div class='add'>+		return -ENOENT;</div><div class='add'>+	erdma_qp_get(qp);</div><div class='add'>+</div><div class='add'>+	down_write(&amp;qp-&gt;state_lock);</div><div class='add'>+	if (qp-&gt;attrs.iwarp.state &gt; ERDMA_QPS_IWARP_RTR) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		up_write(&amp;qp-&gt;state_lock);</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (params-&gt;ord &gt; dev-&gt;attrs.max_ord ||</div><div class='add'>+	    params-&gt;ird &gt; dev-&gt;attrs.max_ord) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		up_write(&amp;qp-&gt;state_lock);</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (params-&gt;private_data_len &gt; MPA_MAX_PRIVDATA) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		up_write(&amp;qp-&gt;state_lock);</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cep-&gt;ird = params-&gt;ird;</div><div class='add'>+	cep-&gt;ord = params-&gt;ord;</div><div class='add'>+</div><div class='add'>+	cep-&gt;cm_id = id;</div><div class='add'>+	id-&gt;add_ref(id);</div><div class='add'>+</div><div class='add'>+	memset(&amp;mod_qp_params, 0, sizeof(mod_qp_params));</div><div class='add'>+</div><div class='add'>+	mod_qp_params.irq_size = params-&gt;ird;</div><div class='add'>+	mod_qp_params.orq_size = params-&gt;ord;</div><div class='add'>+	mod_qp_params.state = ERDMA_QPS_IWARP_RTS;</div><div class='add'>+</div><div class='add'>+	/* Associate QP with CEP */</div><div class='add'>+	erdma_cep_get(cep);</div><div class='add'>+	qp-&gt;cep = cep;</div><div class='add'>+	cep-&gt;qp = qp;</div><div class='add'>+</div><div class='add'>+	cep-&gt;state = ERDMA_EPSTATE_RDMA_MODE;</div><div class='add'>+</div><div class='add'>+	mod_qp_params.qp_type = ERDMA_QP_PASSIVE;</div><div class='add'>+	mod_qp_params.pd_len = params-&gt;private_data_len;</div><div class='add'>+</div><div class='add'>+	to_modify_attrs = ERDMA_QPA_IWARP_STATE | ERDMA_QPA_IWARP_ORD |</div><div class='add'>+			  ERDMA_QPA_IWARP_LLP_HANDLE | ERDMA_QPA_IWARP_IRD |</div><div class='add'>+			  ERDMA_QPA_IWARP_MPA;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;attrs.cc != __mpa_ext_cc(cep-&gt;mpa.ext_data.bits)) {</div><div class='add'>+		to_modify_attrs |= ERDMA_QPA_IWARP_CC;</div><div class='add'>+		mod_qp_params.cc = COMPROMISE_CC;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* move to rts */</div><div class='add'>+	ret = erdma_modify_qp_state_iwarp(qp, &amp;mod_qp_params, to_modify_attrs);</div><div class='add'>+</div><div class='add'>+	up_write(&amp;qp-&gt;state_lock);</div><div class='add'>+</div><div class='add'>+	if (ret)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	cep-&gt;mpa.ext_data.bits = 0;</div><div class='add'>+	__mpa_ext_set_cc(&amp;cep-&gt;mpa.ext_data.bits, qp-&gt;attrs.cc);</div><div class='add'>+	cep-&gt;mpa.ext_data.cookie = cpu_to_be32(cep-&gt;qp-&gt;attrs.iwarp.cookie);</div><div class='add'>+</div><div class='add'>+	ret = erdma_send_mpareqrep(cep, params-&gt;private_data,</div><div class='add'>+				   params-&gt;private_data_len);</div><div class='add'>+	if (!ret) {</div><div class='add'>+		ret = erdma_cm_upcall(cep, IW_CM_EVENT_ESTABLISHED, 0);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto error;</div><div class='add'>+</div><div class='add'>+		erdma_cep_set_free(cep);</div><div class='add'>+</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+error:</div><div class='add'>+	erdma_socket_disassoc(cep-&gt;sock);</div><div class='add'>+	sock_release(cep-&gt;sock);</div><div class='add'>+	cep-&gt;sock = NULL;</div><div class='add'>+</div><div class='add'>+	cep-&gt;state = ERDMA_EPSTATE_CLOSED;</div><div class='add'>+</div><div class='add'>+	if (cep-&gt;cm_id) {</div><div class='add'>+		cep-&gt;cm_id-&gt;rem_ref(id);</div><div class='add'>+		cep-&gt;cm_id = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;cep) {</div><div class='add'>+		erdma_cep_put(cep);</div><div class='add'>+		qp-&gt;cep = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cep-&gt;qp = NULL;</div><div class='add'>+	erdma_qp_put(qp);</div><div class='add'>+</div><div class='add'>+	erdma_cep_set_free(cep);</div><div class='add'>+	erdma_cep_put(cep);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_reject(struct iw_cm_id *id, const void *pdata, u8 plen)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cep *cep = (struct erdma_cep *)id-&gt;provider_data;</div><div class='add'>+</div><div class='add'>+	erdma_cep_set_inuse(cep);</div><div class='add'>+	erdma_cep_put(cep);</div><div class='add'>+</div><div class='add'>+	erdma_cancel_mpatimer(cep);</div><div class='add'>+</div><div class='add'>+	if (cep-&gt;state != ERDMA_EPSTATE_RECVD_MPAREQ) {</div><div class='add'>+		erdma_cep_set_free(cep);</div><div class='add'>+		erdma_cep_put(cep);</div><div class='add'>+</div><div class='add'>+		return -ECONNRESET;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (__mpa_rr_revision(cep-&gt;mpa.hdr.params.bits) == MPA_REVISION_EXT_1) {</div><div class='add'>+		cep-&gt;mpa.hdr.params.bits |= MPA_RR_FLAG_REJECT; /* reject */</div><div class='add'>+		erdma_send_mpareqrep(cep, pdata, plen);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	erdma_socket_disassoc(cep-&gt;sock);</div><div class='add'>+	sock_release(cep-&gt;sock);</div><div class='add'>+	cep-&gt;sock = NULL;</div><div class='add'>+</div><div class='add'>+	cep-&gt;state = ERDMA_EPSTATE_CLOSED;</div><div class='add'>+</div><div class='add'>+	erdma_cep_set_free(cep);</div><div class='add'>+	erdma_cep_put(cep);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_create_listen(struct iw_cm_id *id, int backlog)</div><div class='add'>+{</div><div class='add'>+	struct socket *s;</div><div class='add'>+	struct erdma_cep *cep = NULL;</div><div class='add'>+	int ret = 0;</div><div class='add'>+	struct erdma_dev *dev = to_edev(id-&gt;device);</div><div class='add'>+	int addr_family = id-&gt;local_addr.ss_family;</div><div class='add'>+	struct sockaddr_in *laddr = &amp;to_sockaddr_in(id-&gt;local_addr);</div><div class='add'>+</div><div class='add'>+	if (addr_family != AF_INET)</div><div class='add'>+		return -EAFNOSUPPORT;</div><div class='add'>+</div><div class='add'>+	ret = sock_create(addr_family, SOCK_STREAM, IPPROTO_TCP, &amp;s);</div><div class='add'>+	if (ret &lt; 0)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	sock_set_reuseaddr(s-&gt;sk);</div><div class='add'>+</div><div class='add'>+	/* For wildcard addr, limit binding to current device only */</div><div class='add'>+	if (ipv4_is_zeronet(laddr-&gt;sin_addr.s_addr))</div><div class='add'>+		s-&gt;sk-&gt;sk_bound_dev_if = dev-&gt;netdev-&gt;ifindex;</div><div class='add'>+</div><div class='add'>+	ret = s-&gt;ops-&gt;bind(s, (struct sockaddr_unsized *)laddr,</div><div class='add'>+			   sizeof(struct sockaddr_in));</div><div class='add'>+	if (ret)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	cep = erdma_cep_alloc(dev);</div><div class='add'>+	if (!cep) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+	erdma_cep_socket_assoc(cep, s);</div><div class='add'>+</div><div class='add'>+	ret = erdma_cm_alloc_work(cep, backlog);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	ret = s-&gt;ops-&gt;listen(s, backlog);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	cep-&gt;cm_id = id;</div><div class='add'>+	id-&gt;add_ref(id);</div><div class='add'>+</div><div class='add'>+	if (!id-&gt;provider_data) {</div><div class='add'>+		id-&gt;provider_data =</div><div class='add'>+			kmalloc(sizeof(struct list_head), GFP_KERNEL);</div><div class='add'>+		if (!id-&gt;provider_data) {</div><div class='add'>+			ret = -ENOMEM;</div><div class='add'>+			goto error;</div><div class='add'>+		}</div><div class='add'>+		INIT_LIST_HEAD((struct list_head *)id-&gt;provider_data);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	list_add_tail(&amp;cep-&gt;listenq, (struct list_head *)id-&gt;provider_data);</div><div class='add'>+	cep-&gt;state = ERDMA_EPSTATE_LISTENING;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+error:</div><div class='add'>+	if (cep) {</div><div class='add'>+		erdma_cep_set_inuse(cep);</div><div class='add'>+</div><div class='add'>+		if (cep-&gt;cm_id) {</div><div class='add'>+			cep-&gt;cm_id-&gt;rem_ref(cep-&gt;cm_id);</div><div class='add'>+			cep-&gt;cm_id = NULL;</div><div class='add'>+		}</div><div class='add'>+		cep-&gt;sock = NULL;</div><div class='add'>+		erdma_socket_disassoc(s);</div><div class='add'>+		cep-&gt;state = ERDMA_EPSTATE_CLOSED;</div><div class='add'>+</div><div class='add'>+		erdma_cep_set_free(cep);</div><div class='add'>+		erdma_cep_put(cep);</div><div class='add'>+	}</div><div class='add'>+	sock_release(s);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_drop_listeners(struct iw_cm_id *id)</div><div class='add'>+{</div><div class='add'>+	struct list_head *p, *tmp;</div><div class='add'>+	/*</div><div class='add'>+	 * In case of a wildcard rdma_listen on a multi-homed device,</div><div class='add'>+	 * a listener's IWCM id is associated with more than one listening CEP.</div><div class='add'>+	 */</div><div class='add'>+	list_for_each_safe(p, tmp, (struct list_head *)id-&gt;provider_data) {</div><div class='add'>+		struct erdma_cep *cep =</div><div class='add'>+			list_entry(p, struct erdma_cep, listenq);</div><div class='add'>+</div><div class='add'>+		list_del(p);</div><div class='add'>+</div><div class='add'>+		erdma_cep_set_inuse(cep);</div><div class='add'>+</div><div class='add'>+		if (cep-&gt;cm_id) {</div><div class='add'>+			cep-&gt;cm_id-&gt;rem_ref(cep-&gt;cm_id);</div><div class='add'>+			cep-&gt;cm_id = NULL;</div><div class='add'>+		}</div><div class='add'>+		if (cep-&gt;sock) {</div><div class='add'>+			erdma_socket_disassoc(cep-&gt;sock);</div><div class='add'>+			sock_release(cep-&gt;sock);</div><div class='add'>+			cep-&gt;sock = NULL;</div><div class='add'>+		}</div><div class='add'>+		cep-&gt;state = ERDMA_EPSTATE_CLOSED;</div><div class='add'>+		erdma_cep_set_free(cep);</div><div class='add'>+		erdma_cep_put(cep);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_destroy_listen(struct iw_cm_id *id)</div><div class='add'>+{</div><div class='add'>+	if (!id-&gt;provider_data)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	erdma_drop_listeners(id);</div><div class='add'>+	kfree(id-&gt;provider_data);</div><div class='add'>+	id-&gt;provider_data = NULL;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_cm_init(void)</div><div class='add'>+{</div><div class='add'>+	erdma_cm_wq = create_singlethread_workqueue("erdma_cm_wq");</div><div class='add'>+	if (!erdma_cm_wq)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_cm_exit(void)</div><div class='add'>+{</div><div class='add'>+	if (erdma_cm_wq)</div><div class='add'>+		destroy_workqueue(erdma_cm_wq);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/erdma/erdma_cm.h b/drivers/infiniband/hw/erdma/erdma_cm.h<br/>new file mode 100644<br/>index 000000000000..a26d80770188<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/erdma/erdma_cm.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/erdma/erdma_cm.h</a></div><div class='hunk'>@@ -0,0 +1,167 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='add'>+</div><div class='add'>+/* Authors: Cheng Xu &lt;chengyou@linux.alibaba.com&gt; */</div><div class='add'>+/*          Kai Shen &lt;kaishen@linux.alibaba.com&gt; */</div><div class='add'>+/* Copyright (c) 2020-2022, Alibaba Group. */</div><div class='add'>+</div><div class='add'>+/* Authors: Bernard Metzler &lt;bmt@zurich.ibm.com&gt; */</div><div class='add'>+/*          Greg Joyce &lt;greg@opengridcomputing.com&gt; */</div><div class='add'>+/* Copyright (c) 2008-2019, IBM Corporation */</div><div class='add'>+/* Copyright (c) 2017, Open Grid Computing, Inc. */</div><div class='add'>+</div><div class='add'>+#ifndef __ERDMA_CM_H__</div><div class='add'>+#define __ERDMA_CM_H__</div><div class='add'>+</div><div class='add'>+#include &lt;linux/tcp.h&gt;</div><div class='add'>+#include &lt;net/sock.h&gt;</div><div class='add'>+#include &lt;rdma/iw_cm.h&gt;</div><div class='add'>+</div><div class='add'>+/* iWarp MPA protocol defs */</div><div class='add'>+#define MPA_REVISION_EXT_1 129</div><div class='add'>+#define MPA_MAX_PRIVDATA RDMA_MAX_PRIVATE_DATA</div><div class='add'>+#define MPA_KEY_REQ "MPA ID Req Frame"</div><div class='add'>+#define MPA_KEY_REP "MPA ID Rep Frame"</div><div class='add'>+#define MPA_KEY_SIZE 16</div><div class='add'>+#define MPA_DEFAULT_HDR_LEN 28</div><div class='add'>+</div><div class='add'>+struct mpa_rr_params {</div><div class='add'>+	__be16 bits;</div><div class='add'>+	__be16 pd_len;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * MPA request/response Hdr bits &amp; fields</div><div class='add'>+ */</div><div class='add'>+enum {</div><div class='add'>+	MPA_RR_FLAG_MARKERS = cpu_to_be16(0x8000),</div><div class='add'>+	MPA_RR_FLAG_CRC = cpu_to_be16(0x4000),</div><div class='add'>+	MPA_RR_FLAG_REJECT = cpu_to_be16(0x2000),</div><div class='add'>+	MPA_RR_RESERVED = cpu_to_be16(0x1f00),</div><div class='add'>+	MPA_RR_MASK_REVISION = cpu_to_be16(0x00ff)</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * MPA request/reply header</div><div class='add'>+ */</div><div class='add'>+struct mpa_rr {</div><div class='add'>+	u8 key[16];</div><div class='add'>+	struct mpa_rr_params params;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_mpa_ext {</div><div class='add'>+	__be32 cookie;</div><div class='add'>+	__be32 bits;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	MPA_EXT_FLAG_CC = cpu_to_be32(0x0000000f),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_mpa_info {</div><div class='add'>+	struct mpa_rr hdr; /* peer mpa hdr in host byte order */</div><div class='add'>+	struct erdma_mpa_ext ext_data;</div><div class='add'>+	char *pdata;</div><div class='add'>+	int bytes_rcvd;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_sk_upcalls {</div><div class='add'>+	void (*sk_state_change)(struct sock *sk);</div><div class='add'>+	void (*sk_data_ready)(struct sock *sk, int bytes);</div><div class='add'>+	void (*sk_error_report)(struct sock *sk);</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_dev;</div><div class='add'>+</div><div class='add'>+enum erdma_cep_state {</div><div class='add'>+	ERDMA_EPSTATE_IDLE = 1,</div><div class='add'>+	ERDMA_EPSTATE_LISTENING,</div><div class='add'>+	ERDMA_EPSTATE_CONNECTING,</div><div class='add'>+	ERDMA_EPSTATE_AWAIT_MPAREQ,</div><div class='add'>+	ERDMA_EPSTATE_RECVD_MPAREQ,</div><div class='add'>+	ERDMA_EPSTATE_AWAIT_MPAREP,</div><div class='add'>+	ERDMA_EPSTATE_RDMA_MODE,</div><div class='add'>+	ERDMA_EPSTATE_CLOSED</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_cep {</div><div class='add'>+	struct iw_cm_id *cm_id;</div><div class='add'>+	struct erdma_dev *dev;</div><div class='add'>+	struct list_head devq;</div><div class='add'>+	spinlock_t lock;</div><div class='add'>+	struct kref ref;</div><div class='add'>+	int in_use;</div><div class='add'>+	wait_queue_head_t waitq;</div><div class='add'>+	enum erdma_cep_state state;</div><div class='add'>+</div><div class='add'>+	struct list_head listenq;</div><div class='add'>+	struct erdma_cep *listen_cep;</div><div class='add'>+</div><div class='add'>+	struct erdma_qp *qp;</div><div class='add'>+	struct socket *sock;</div><div class='add'>+</div><div class='add'>+	struct erdma_cm_work *mpa_timer;</div><div class='add'>+	struct list_head work_freelist;</div><div class='add'>+</div><div class='add'>+	struct erdma_mpa_info mpa;</div><div class='add'>+	int ord;</div><div class='add'>+	int ird;</div><div class='add'>+</div><div class='add'>+	int pd_len;</div><div class='add'>+	/* hold user's private data. */</div><div class='add'>+	void *private_data;</div><div class='add'>+</div><div class='add'>+	/* Saved upcalls of socket llp.sock */</div><div class='add'>+	void (*sk_state_change)(struct sock *sk);</div><div class='add'>+	void (*sk_data_ready)(struct sock *sk);</div><div class='add'>+	void (*sk_error_report)(struct sock *sk);</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define MPAREQ_TIMEOUT (HZ * 20)</div><div class='add'>+#define MPAREP_TIMEOUT (HZ * 10)</div><div class='add'>+#define CONNECT_TIMEOUT (HZ * 10)</div><div class='add'>+</div><div class='add'>+enum erdma_work_type {</div><div class='add'>+	ERDMA_CM_WORK_ACCEPT = 1,</div><div class='add'>+	ERDMA_CM_WORK_READ_MPAHDR,</div><div class='add'>+	ERDMA_CM_WORK_CLOSE_LLP, /* close socket */</div><div class='add'>+	ERDMA_CM_WORK_PEER_CLOSE, /* socket indicated peer close */</div><div class='add'>+	ERDMA_CM_WORK_MPATIMEOUT,</div><div class='add'>+	ERDMA_CM_WORK_CONNECTED,</div><div class='add'>+	ERDMA_CM_WORK_CONNECTTIMEOUT</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_cm_work {</div><div class='add'>+	struct delayed_work work;</div><div class='add'>+	struct list_head list;</div><div class='add'>+	enum erdma_work_type type;</div><div class='add'>+	struct erdma_cep *cep;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define to_sockaddr_in(a) (*(struct sockaddr_in *)(&amp;(a)))</div><div class='add'>+</div><div class='add'>+static inline int getname_peer(struct socket *s, struct sockaddr_storage *a)</div><div class='add'>+{</div><div class='add'>+	return s-&gt;ops-&gt;getname(s, (struct sockaddr *)a, 1);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int getname_local(struct socket *s, struct sockaddr_storage *a)</div><div class='add'>+{</div><div class='add'>+	return s-&gt;ops-&gt;getname(s, (struct sockaddr *)a, 0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_connect(struct iw_cm_id *id, struct iw_cm_conn_param *param);</div><div class='add'>+int erdma_accept(struct iw_cm_id *id, struct iw_cm_conn_param *param);</div><div class='add'>+int erdma_reject(struct iw_cm_id *id, const void *pdata, u8 plen);</div><div class='add'>+int erdma_create_listen(struct iw_cm_id *id, int backlog);</div><div class='add'>+int erdma_destroy_listen(struct iw_cm_id *id);</div><div class='add'>+</div><div class='add'>+void erdma_cep_get(struct erdma_cep *ceq);</div><div class='add'>+void erdma_cep_put(struct erdma_cep *ceq);</div><div class='add'>+int erdma_cm_queue_work(struct erdma_cep *ceq, enum erdma_work_type type);</div><div class='add'>+</div><div class='add'>+int erdma_cm_init(void);</div><div class='add'>+void erdma_cm_exit(void);</div><div class='add'>+</div><div class='add'>+#define sk_to_cep(sk) ((struct erdma_cep *)((sk)-&gt;sk_user_data))</div><div class='add'>+</div><div class='add'>+#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/erdma/erdma_cmdq.c b/drivers/infiniband/hw/erdma/erdma_cmdq.c<br/>new file mode 100644<br/>index 000000000000..b867aefe83b2<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/erdma/erdma_cmdq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/erdma/erdma_cmdq.c</a></div><div class='hunk'>@@ -0,0 +1,452 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='add'>+</div><div class='add'>+/* Authors: Cheng Xu &lt;chengyou@linux.alibaba.com&gt; */</div><div class='add'>+/*          Kai Shen &lt;kaishen@linux.alibaba.com&gt; */</div><div class='add'>+/* Copyright (c) 2020-2022, Alibaba Group. */</div><div class='add'>+</div><div class='add'>+#include "erdma.h"</div><div class='add'>+</div><div class='add'>+static void arm_cmdq_cq(struct erdma_cmdq *cmdq)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = container_of(cmdq, struct erdma_dev, cmdq);</div><div class='add'>+	u64 db_data = FIELD_PREP(ERDMA_CQDB_CI_MASK, cmdq-&gt;cq.ci) |</div><div class='add'>+		      FIELD_PREP(ERDMA_CQDB_ARM_MASK, 1) |</div><div class='add'>+		      FIELD_PREP(ERDMA_CQDB_CMDSN_MASK, cmdq-&gt;cq.cmdsn) |</div><div class='add'>+		      FIELD_PREP(ERDMA_CQDB_IDX_MASK, cmdq-&gt;cq.cmdsn);</div><div class='add'>+</div><div class='add'>+	*cmdq-&gt;cq.dbrec = db_data;</div><div class='add'>+	writeq(db_data, dev-&gt;func_bar + ERDMA_CMDQ_CQDB_REG);</div><div class='add'>+</div><div class='add'>+	atomic64_inc(&amp;cmdq-&gt;cq.armed_num);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void kick_cmdq_db(struct erdma_cmdq *cmdq)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = container_of(cmdq, struct erdma_dev, cmdq);</div><div class='add'>+	u64 db_data = FIELD_PREP(ERDMA_CMD_HDR_WQEBB_INDEX_MASK, cmdq-&gt;sq.pi);</div><div class='add'>+</div><div class='add'>+	*cmdq-&gt;sq.dbrec = db_data;</div><div class='add'>+	writeq(db_data, dev-&gt;func_bar + ERDMA_CMDQ_SQDB_REG);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct erdma_comp_wait *get_comp_wait(struct erdma_cmdq *cmdq)</div><div class='add'>+{</div><div class='add'>+	int comp_idx;</div><div class='add'>+</div><div class='add'>+	spin_lock(&amp;cmdq-&gt;lock);</div><div class='add'>+	comp_idx = find_first_zero_bit(cmdq-&gt;comp_wait_bitmap,</div><div class='add'>+				       cmdq-&gt;max_outstandings);</div><div class='add'>+	if (comp_idx == cmdq-&gt;max_outstandings) {</div><div class='add'>+		spin_unlock(&amp;cmdq-&gt;lock);</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	__set_bit(comp_idx, cmdq-&gt;comp_wait_bitmap);</div><div class='add'>+	spin_unlock(&amp;cmdq-&gt;lock);</div><div class='add'>+</div><div class='add'>+	return &amp;cmdq-&gt;wait_pool[comp_idx];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void put_comp_wait(struct erdma_cmdq *cmdq,</div><div class='add'>+			  struct erdma_comp_wait *comp_wait)</div><div class='add'>+{</div><div class='add'>+	int used;</div><div class='add'>+</div><div class='add'>+	cmdq-&gt;wait_pool[comp_wait-&gt;ctx_id].cmd_status = ERDMA_CMD_STATUS_INIT;</div><div class='add'>+	spin_lock(&amp;cmdq-&gt;lock);</div><div class='add'>+	used = __test_and_clear_bit(comp_wait-&gt;ctx_id, cmdq-&gt;comp_wait_bitmap);</div><div class='add'>+	spin_unlock(&amp;cmdq-&gt;lock);</div><div class='add'>+</div><div class='add'>+	WARN_ON(!used);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_cmdq_wait_res_init(struct erdma_dev *dev,</div><div class='add'>+				    struct erdma_cmdq *cmdq)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	cmdq-&gt;wait_pool =</div><div class='add'>+		devm_kcalloc(&amp;dev-&gt;pdev-&gt;dev, cmdq-&gt;max_outstandings,</div><div class='add'>+			     sizeof(struct erdma_comp_wait), GFP_KERNEL);</div><div class='add'>+	if (!cmdq-&gt;wait_pool)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;cmdq-&gt;lock);</div><div class='add'>+	cmdq-&gt;comp_wait_bitmap = devm_bitmap_zalloc(</div><div class='add'>+		&amp;dev-&gt;pdev-&gt;dev, cmdq-&gt;max_outstandings, GFP_KERNEL);</div><div class='add'>+	if (!cmdq-&gt;comp_wait_bitmap)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; cmdq-&gt;max_outstandings; i++) {</div><div class='add'>+		init_completion(&amp;cmdq-&gt;wait_pool[i].wait_event);</div><div class='add'>+		cmdq-&gt;wait_pool[i].ctx_id = i;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_cmdq_sq_init(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cmdq *cmdq = &amp;dev-&gt;cmdq;</div><div class='add'>+	struct erdma_cmdq_sq *sq = &amp;cmdq-&gt;sq;</div><div class='add'>+</div><div class='add'>+	sq-&gt;wqebb_cnt = SQEBB_COUNT(ERDMA_CMDQ_SQE_SIZE);</div><div class='add'>+	sq-&gt;depth = cmdq-&gt;max_outstandings * sq-&gt;wqebb_cnt;</div><div class='add'>+</div><div class='add'>+	sq-&gt;qbuf = dma_alloc_coherent(&amp;dev-&gt;pdev-&gt;dev, sq-&gt;depth &lt;&lt; SQEBB_SHIFT,</div><div class='add'>+				      &amp;sq-&gt;qbuf_dma_addr, GFP_KERNEL);</div><div class='add'>+	if (!sq-&gt;qbuf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	sq-&gt;dbrec = dma_pool_zalloc(dev-&gt;db_pool, GFP_KERNEL, &amp;sq-&gt;dbrec_dma);</div><div class='add'>+	if (!sq-&gt;dbrec)</div><div class='add'>+		goto err_out;</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;sq-&gt;lock);</div><div class='add'>+</div><div class='add'>+	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_SQ_ADDR_H_REG,</div><div class='add'>+			  upper_32_bits(sq-&gt;qbuf_dma_addr));</div><div class='add'>+	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_SQ_ADDR_L_REG,</div><div class='add'>+			  lower_32_bits(sq-&gt;qbuf_dma_addr));</div><div class='add'>+	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_DEPTH_REG, sq-&gt;depth);</div><div class='add'>+	erdma_reg_write64(dev, ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG, sq-&gt;dbrec_dma);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_out:</div><div class='add'>+	dma_free_coherent(&amp;dev-&gt;pdev-&gt;dev, sq-&gt;depth &lt;&lt; SQEBB_SHIFT,</div><div class='add'>+			  sq-&gt;qbuf, sq-&gt;qbuf_dma_addr);</div><div class='add'>+</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_cmdq_cq_init(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cmdq *cmdq = &amp;dev-&gt;cmdq;</div><div class='add'>+	struct erdma_cmdq_cq *cq = &amp;cmdq-&gt;cq;</div><div class='add'>+</div><div class='add'>+	cq-&gt;depth = cmdq-&gt;sq.depth;</div><div class='add'>+	cq-&gt;qbuf = dma_alloc_coherent(&amp;dev-&gt;pdev-&gt;dev, cq-&gt;depth &lt;&lt; CQE_SHIFT,</div><div class='add'>+				      &amp;cq-&gt;qbuf_dma_addr, GFP_KERNEL);</div><div class='add'>+	if (!cq-&gt;qbuf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;cq-&gt;lock);</div><div class='add'>+</div><div class='add'>+	cq-&gt;dbrec = dma_pool_zalloc(dev-&gt;db_pool, GFP_KERNEL, &amp;cq-&gt;dbrec_dma);</div><div class='add'>+	if (!cq-&gt;dbrec)</div><div class='add'>+		goto err_out;</div><div class='add'>+</div><div class='add'>+	atomic64_set(&amp;cq-&gt;armed_num, 0);</div><div class='add'>+</div><div class='add'>+	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_CQ_ADDR_H_REG,</div><div class='add'>+			  upper_32_bits(cq-&gt;qbuf_dma_addr));</div><div class='add'>+	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_CQ_ADDR_L_REG,</div><div class='add'>+			  lower_32_bits(cq-&gt;qbuf_dma_addr));</div><div class='add'>+	erdma_reg_write64(dev, ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG, cq-&gt;dbrec_dma);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_out:</div><div class='add'>+	dma_free_coherent(&amp;dev-&gt;pdev-&gt;dev, cq-&gt;depth &lt;&lt; CQE_SHIFT, cq-&gt;qbuf,</div><div class='add'>+			  cq-&gt;qbuf_dma_addr);</div><div class='add'>+</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_cmdq_eq_init(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cmdq *cmdq = &amp;dev-&gt;cmdq;</div><div class='add'>+	struct erdma_eq *eq = &amp;cmdq-&gt;eq;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = erdma_eq_common_init(dev, eq, cmdq-&gt;max_outstandings);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	eq-&gt;db = dev-&gt;func_bar + ERDMA_REGS_CEQ_DB_BASE_REG;</div><div class='add'>+</div><div class='add'>+	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_H_REG,</div><div class='add'>+			  upper_32_bits(eq-&gt;qbuf_dma_addr));</div><div class='add'>+	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_L_REG,</div><div class='add'>+			  lower_32_bits(eq-&gt;qbuf_dma_addr));</div><div class='add'>+	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_DEPTH_REG, eq-&gt;depth);</div><div class='add'>+	erdma_reg_write64(dev, ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG, eq-&gt;dbrec_dma);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_cmdq_init(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cmdq *cmdq = &amp;dev-&gt;cmdq;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	cmdq-&gt;max_outstandings = ERDMA_CMDQ_MAX_OUTSTANDING;</div><div class='add'>+</div><div class='add'>+	sema_init(&amp;cmdq-&gt;credits, cmdq-&gt;max_outstandings);</div><div class='add'>+</div><div class='add'>+	err = erdma_cmdq_wait_res_init(dev, cmdq);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = erdma_cmdq_sq_init(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = erdma_cmdq_cq_init(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_destroy_sq;</div><div class='add'>+</div><div class='add'>+	err = erdma_cmdq_eq_init(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_destroy_cq;</div><div class='add'>+</div><div class='add'>+	set_bit(ERDMA_CMDQ_STATE_OK_BIT, &amp;cmdq-&gt;state);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_destroy_cq:</div><div class='add'>+	dma_free_coherent(&amp;dev-&gt;pdev-&gt;dev, cmdq-&gt;cq.depth &lt;&lt; CQE_SHIFT,</div><div class='add'>+			  cmdq-&gt;cq.qbuf, cmdq-&gt;cq.qbuf_dma_addr);</div><div class='add'>+</div><div class='add'>+	dma_pool_free(dev-&gt;db_pool, cmdq-&gt;cq.dbrec, cmdq-&gt;cq.dbrec_dma);</div><div class='add'>+</div><div class='add'>+err_destroy_sq:</div><div class='add'>+	dma_free_coherent(&amp;dev-&gt;pdev-&gt;dev, cmdq-&gt;sq.depth &lt;&lt; SQEBB_SHIFT,</div><div class='add'>+			  cmdq-&gt;sq.qbuf, cmdq-&gt;sq.qbuf_dma_addr);</div><div class='add'>+</div><div class='add'>+	dma_pool_free(dev-&gt;db_pool, cmdq-&gt;sq.dbrec, cmdq-&gt;sq.dbrec_dma);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_finish_cmdq_init(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	arm_cmdq_cq(&amp;dev-&gt;cmdq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_cmdq_destroy(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cmdq *cmdq = &amp;dev-&gt;cmdq;</div><div class='add'>+</div><div class='add'>+	clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &amp;cmdq-&gt;state);</div><div class='add'>+</div><div class='add'>+	erdma_eq_destroy(dev, &amp;cmdq-&gt;eq);</div><div class='add'>+</div><div class='add'>+	dma_free_coherent(&amp;dev-&gt;pdev-&gt;dev, cmdq-&gt;sq.depth &lt;&lt; SQEBB_SHIFT,</div><div class='add'>+			  cmdq-&gt;sq.qbuf, cmdq-&gt;sq.qbuf_dma_addr);</div><div class='add'>+</div><div class='add'>+	dma_pool_free(dev-&gt;db_pool, cmdq-&gt;sq.dbrec, cmdq-&gt;sq.dbrec_dma);</div><div class='add'>+</div><div class='add'>+	dma_free_coherent(&amp;dev-&gt;pdev-&gt;dev, cmdq-&gt;cq.depth &lt;&lt; CQE_SHIFT,</div><div class='add'>+			  cmdq-&gt;cq.qbuf, cmdq-&gt;cq.qbuf_dma_addr);</div><div class='add'>+</div><div class='add'>+	dma_pool_free(dev-&gt;db_pool, cmdq-&gt;cq.dbrec, cmdq-&gt;cq.dbrec_dma);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void *get_next_valid_cmdq_cqe(struct erdma_cmdq *cmdq)</div><div class='add'>+{</div><div class='add'>+	__be32 *cqe = get_queue_entry(cmdq-&gt;cq.qbuf, cmdq-&gt;cq.ci,</div><div class='add'>+				      cmdq-&gt;cq.depth, CQE_SHIFT);</div><div class='add'>+	u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,</div><div class='add'>+			      be32_to_cpu(READ_ONCE(*cqe)));</div><div class='add'>+</div><div class='add'>+	return owner ^ !!(cmdq-&gt;cq.ci &amp; cmdq-&gt;cq.depth) ? cqe : NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void push_cmdq_sqe(struct erdma_cmdq *cmdq, u64 *req, size_t req_len,</div><div class='add'>+			  struct erdma_comp_wait *comp_wait)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr = *req;</div><div class='add'>+</div><div class='add'>+	comp_wait-&gt;cmd_status = ERDMA_CMD_STATUS_ISSUED;</div><div class='add'>+	reinit_completion(&amp;comp_wait-&gt;wait_event);</div><div class='add'>+	comp_wait-&gt;sq_pi = cmdq-&gt;sq.pi;</div><div class='add'>+</div><div class='add'>+	wqe = get_queue_entry(cmdq-&gt;sq.qbuf, cmdq-&gt;sq.pi, cmdq-&gt;sq.depth,</div><div class='add'>+			      SQEBB_SHIFT);</div><div class='add'>+	memcpy(wqe, req, req_len);</div><div class='add'>+</div><div class='add'>+	cmdq-&gt;sq.pi += cmdq-&gt;sq.wqebb_cnt;</div><div class='add'>+	hdr |= FIELD_PREP(ERDMA_CMD_HDR_WQEBB_INDEX_MASK, cmdq-&gt;sq.pi) |</div><div class='add'>+	       FIELD_PREP(ERDMA_CMD_HDR_CONTEXT_COOKIE_MASK,</div><div class='add'>+			  comp_wait-&gt;ctx_id) |</div><div class='add'>+	       FIELD_PREP(ERDMA_CMD_HDR_WQEBB_CNT_MASK, cmdq-&gt;sq.wqebb_cnt - 1);</div><div class='add'>+	*wqe = cpu_to_le64(hdr);</div><div class='add'>+</div><div class='add'>+	kick_cmdq_db(cmdq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_poll_single_cmd_completion(struct erdma_cmdq *cmdq)</div><div class='add'>+{</div><div class='add'>+	struct erdma_comp_wait *comp_wait;</div><div class='add'>+	u32 hdr0, sqe_idx;</div><div class='add'>+	__be32 *cqe;</div><div class='add'>+	u16 ctx_id;</div><div class='add'>+	u64 *sqe;</div><div class='add'>+</div><div class='add'>+	cqe = get_next_valid_cmdq_cqe(cmdq);</div><div class='add'>+	if (!cqe)</div><div class='add'>+		return -EAGAIN;</div><div class='add'>+</div><div class='add'>+	cmdq-&gt;cq.ci++;</div><div class='add'>+</div><div class='add'>+	dma_rmb();</div><div class='add'>+	hdr0 = be32_to_cpu(*cqe);</div><div class='add'>+	sqe_idx = be32_to_cpu(*(cqe + 1));</div><div class='add'>+</div><div class='add'>+	sqe = get_queue_entry(cmdq-&gt;sq.qbuf, sqe_idx, cmdq-&gt;sq.depth,</div><div class='add'>+			      SQEBB_SHIFT);</div><div class='add'>+	ctx_id = FIELD_GET(ERDMA_CMD_HDR_CONTEXT_COOKIE_MASK, *sqe);</div><div class='add'>+	comp_wait = &amp;cmdq-&gt;wait_pool[ctx_id];</div><div class='add'>+	if (comp_wait-&gt;cmd_status != ERDMA_CMD_STATUS_ISSUED)</div><div class='add'>+		return -EIO;</div><div class='add'>+</div><div class='add'>+	comp_wait-&gt;cmd_status = ERDMA_CMD_STATUS_FINISHED;</div><div class='add'>+	comp_wait-&gt;comp_status = FIELD_GET(ERDMA_CQE_HDR_SYNDROME_MASK, hdr0);</div><div class='add'>+	cmdq-&gt;sq.ci += cmdq-&gt;sq.wqebb_cnt;</div><div class='add'>+	/* Copy 16B comp data after cqe hdr to outer */</div><div class='add'>+	be32_to_cpu_array(comp_wait-&gt;comp_data, cqe + 2, 4);</div><div class='add'>+</div><div class='add'>+	complete(&amp;comp_wait-&gt;wait_event);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_polling_cmd_completions(struct erdma_cmdq *cmdq)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	u16 comp_num;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cmdq-&gt;cq.lock, flags);</div><div class='add'>+</div><div class='add'>+	/* We must have less than # of max_outstandings</div><div class='add'>+	 * completions at one time.</div><div class='add'>+	 */</div><div class='add'>+	for (comp_num = 0; comp_num &lt; cmdq-&gt;max_outstandings; comp_num++)</div><div class='add'>+		if (erdma_poll_single_cmd_completion(cmdq))</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+	spin_unlock_irqrestore(&amp;cmdq-&gt;cq.lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_cmdq_completion_handler(struct erdma_cmdq *cmdq)</div><div class='add'>+{</div><div class='add'>+	int got_event = 0;</div><div class='add'>+</div><div class='add'>+	if (!test_bit(ERDMA_CMDQ_STATE_OK_BIT, &amp;cmdq-&gt;state))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	while (get_next_valid_eqe(&amp;cmdq-&gt;eq)) {</div><div class='add'>+		cmdq-&gt;eq.ci++;</div><div class='add'>+		got_event++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (got_event) {</div><div class='add'>+		cmdq-&gt;cq.cmdsn++;</div><div class='add'>+		erdma_polling_cmd_completions(cmdq);</div><div class='add'>+		arm_cmdq_cq(cmdq);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	notify_eq(&amp;cmdq-&gt;eq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_poll_cmd_completion(struct erdma_comp_wait *comp_ctx,</div><div class='add'>+				     struct erdma_cmdq *cmdq, u32 timeout)</div><div class='add'>+{</div><div class='add'>+	unsigned long comp_timeout = jiffies + msecs_to_jiffies(timeout);</div><div class='add'>+</div><div class='add'>+	while (1) {</div><div class='add'>+		erdma_polling_cmd_completions(cmdq);</div><div class='add'>+		if (comp_ctx-&gt;cmd_status != ERDMA_CMD_STATUS_ISSUED)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		if (time_is_before_jiffies(comp_timeout))</div><div class='add'>+			return -ETIME;</div><div class='add'>+</div><div class='add'>+		udelay(20);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_wait_cmd_completion(struct erdma_comp_wait *comp_ctx,</div><div class='add'>+				     struct erdma_cmdq *cmdq, u32 timeout)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags = 0;</div><div class='add'>+</div><div class='add'>+	wait_for_completion_timeout(&amp;comp_ctx-&gt;wait_event,</div><div class='add'>+				    msecs_to_jiffies(timeout));</div><div class='add'>+</div><div class='add'>+	if (unlikely(comp_ctx-&gt;cmd_status != ERDMA_CMD_STATUS_FINISHED)) {</div><div class='add'>+		spin_lock_irqsave(&amp;cmdq-&gt;cq.lock, flags);</div><div class='add'>+		comp_ctx-&gt;cmd_status = ERDMA_CMD_STATUS_TIMEOUT;</div><div class='add'>+		spin_unlock_irqrestore(&amp;cmdq-&gt;cq.lock, flags);</div><div class='add'>+		return -ETIME;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_cmdq_build_reqhdr(u64 *hdr, u32 mod, u32 op)</div><div class='add'>+{</div><div class='add'>+	*hdr = FIELD_PREP(ERDMA_CMD_HDR_SUB_MOD_MASK, mod) |</div><div class='add'>+	       FIELD_PREP(ERDMA_CMD_HDR_OPCODE_MASK, op);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_post_cmd_wait(struct erdma_cmdq *cmdq, void *req, u32 req_size,</div><div class='add'>+			u64 *resp0, u64 *resp1, bool sleepable)</div><div class='add'>+{</div><div class='add'>+	struct erdma_comp_wait *comp_wait;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!test_bit(ERDMA_CMDQ_STATE_OK_BIT, &amp;cmdq-&gt;state))</div><div class='add'>+		return -ENODEV;</div><div class='add'>+</div><div class='add'>+	if (!sleepable) {</div><div class='add'>+		while (down_trylock(&amp;cmdq-&gt;credits))</div><div class='add'>+			;</div><div class='add'>+	} else {</div><div class='add'>+		down(&amp;cmdq-&gt;credits);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	comp_wait = get_comp_wait(cmdq);</div><div class='add'>+	if (IS_ERR(comp_wait)) {</div><div class='add'>+		clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &amp;cmdq-&gt;state);</div><div class='add'>+		set_bit(ERDMA_CMDQ_STATE_CTX_ERR_BIT, &amp;cmdq-&gt;state);</div><div class='add'>+		up(&amp;cmdq-&gt;credits);</div><div class='add'>+		return PTR_ERR(comp_wait);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_lock(&amp;cmdq-&gt;sq.lock);</div><div class='add'>+	push_cmdq_sqe(cmdq, req, req_size, comp_wait);</div><div class='add'>+	spin_unlock(&amp;cmdq-&gt;sq.lock);</div><div class='add'>+</div><div class='add'>+	if (sleepable)</div><div class='add'>+		ret = erdma_wait_cmd_completion(comp_wait, cmdq,</div><div class='add'>+						ERDMA_CMDQ_TIMEOUT_MS);</div><div class='add'>+	else</div><div class='add'>+		ret = erdma_poll_cmd_completion(comp_wait, cmdq,</div><div class='add'>+						ERDMA_CMDQ_TIMEOUT_MS);</div><div class='add'>+</div><div class='add'>+	if (ret) {</div><div class='add'>+		set_bit(ERDMA_CMDQ_STATE_TIMEOUT_BIT, &amp;cmdq-&gt;state);</div><div class='add'>+		clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &amp;cmdq-&gt;state);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (comp_wait-&gt;comp_status)</div><div class='add'>+		ret = -EIO;</div><div class='add'>+</div><div class='add'>+	if (resp0 &amp;&amp; resp1) {</div><div class='add'>+		*resp0 = *((u64 *)&amp;comp_wait-&gt;comp_data[0]);</div><div class='add'>+		*resp1 = *((u64 *)&amp;comp_wait-&gt;comp_data[2]);</div><div class='add'>+	}</div><div class='add'>+	put_comp_wait(cmdq, comp_wait);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	up(&amp;cmdq-&gt;credits);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/erdma/erdma_cq.c b/drivers/infiniband/hw/erdma/erdma_cq.c<br/>new file mode 100644<br/>index 000000000000..1f456327e63c<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/erdma/erdma_cq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/erdma/erdma_cq.c</a></div><div class='hunk'>@@ -0,0 +1,268 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='add'>+</div><div class='add'>+/* Authors: Cheng Xu &lt;chengyou@linux.alibaba.com&gt; */</div><div class='add'>+/*          Kai Shen &lt;kaishen@linux.alibaba.com&gt; */</div><div class='add'>+/* Copyright (c) 2020-2022, Alibaba Group. */</div><div class='add'>+</div><div class='add'>+#include "erdma_verbs.h"</div><div class='add'>+</div><div class='add'>+static void *get_next_valid_cqe(struct erdma_cq *cq)</div><div class='add'>+{</div><div class='add'>+	__be32 *cqe = get_queue_entry(cq-&gt;kern_cq.qbuf, cq-&gt;kern_cq.ci,</div><div class='add'>+				      cq-&gt;depth, CQE_SHIFT);</div><div class='add'>+	u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,</div><div class='add'>+			      be32_to_cpu(READ_ONCE(*cqe)));</div><div class='add'>+</div><div class='add'>+	return owner ^ !!(cq-&gt;kern_cq.ci &amp; cq-&gt;depth) ? cqe : NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void notify_cq(struct erdma_cq *cq, u8 solcitied)</div><div class='add'>+{</div><div class='add'>+	u64 db_data =</div><div class='add'>+		FIELD_PREP(ERDMA_CQDB_IDX_MASK, (cq-&gt;kern_cq.notify_cnt)) |</div><div class='add'>+		FIELD_PREP(ERDMA_CQDB_CQN_MASK, cq-&gt;cqn) |</div><div class='add'>+		FIELD_PREP(ERDMA_CQDB_ARM_MASK, 1) |</div><div class='add'>+		FIELD_PREP(ERDMA_CQDB_SOL_MASK, solcitied) |</div><div class='add'>+		FIELD_PREP(ERDMA_CQDB_CMDSN_MASK, cq-&gt;kern_cq.cmdsn) |</div><div class='add'>+		FIELD_PREP(ERDMA_CQDB_CI_MASK, cq-&gt;kern_cq.ci);</div><div class='add'>+</div><div class='add'>+	*cq-&gt;kern_cq.dbrec = db_data;</div><div class='add'>+	writeq(db_data, cq-&gt;kern_cq.db);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cq *cq = to_ecq(ibcq);</div><div class='add'>+	unsigned long irq_flags;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cq-&gt;kern_cq.lock, irq_flags);</div><div class='add'>+</div><div class='add'>+	notify_cq(cq, (flags &amp; IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED);</div><div class='add'>+</div><div class='add'>+	if ((flags &amp; IB_CQ_REPORT_MISSED_EVENTS) &amp;&amp; get_next_valid_cqe(cq))</div><div class='add'>+		ret = 1;</div><div class='add'>+</div><div class='add'>+	cq-&gt;kern_cq.notify_cnt++;</div><div class='add'>+</div><div class='add'>+	spin_unlock_irqrestore(&amp;cq-&gt;kern_cq.lock, irq_flags);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const enum ib_wc_opcode wc_mapping_table[ERDMA_NUM_OPCODES] = {</div><div class='add'>+	[ERDMA_OP_WRITE] = IB_WC_RDMA_WRITE,</div><div class='add'>+	[ERDMA_OP_READ] = IB_WC_RDMA_READ,</div><div class='add'>+	[ERDMA_OP_SEND] = IB_WC_SEND,</div><div class='add'>+	[ERDMA_OP_SEND_WITH_IMM] = IB_WC_SEND,</div><div class='add'>+	[ERDMA_OP_RECEIVE] = IB_WC_RECV,</div><div class='add'>+	[ERDMA_OP_RECV_IMM] = IB_WC_RECV_RDMA_WITH_IMM,</div><div class='add'>+	[ERDMA_OP_RECV_INV] = IB_WC_RECV,</div><div class='add'>+	[ERDMA_OP_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE,</div><div class='add'>+	[ERDMA_OP_RSP_SEND_IMM] = IB_WC_RECV,</div><div class='add'>+	[ERDMA_OP_SEND_WITH_INV] = IB_WC_SEND,</div><div class='add'>+	[ERDMA_OP_REG_MR] = IB_WC_REG_MR,</div><div class='add'>+	[ERDMA_OP_LOCAL_INV] = IB_WC_LOCAL_INV,</div><div class='add'>+	[ERDMA_OP_READ_WITH_INV] = IB_WC_RDMA_READ,</div><div class='add'>+	[ERDMA_OP_ATOMIC_CAS] = IB_WC_COMP_SWAP,</div><div class='add'>+	[ERDMA_OP_ATOMIC_FAA] = IB_WC_FETCH_ADD,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct {</div><div class='add'>+	enum erdma_wc_status erdma;</div><div class='add'>+	enum ib_wc_status base;</div><div class='add'>+	enum erdma_vendor_err vendor;</div><div class='add'>+} map_cqe_status[ERDMA_NUM_WC_STATUS] = {</div><div class='add'>+	{ ERDMA_WC_SUCCESS, IB_WC_SUCCESS, ERDMA_WC_VENDOR_NO_ERR },</div><div class='add'>+	{ ERDMA_WC_GENERAL_ERR, IB_WC_GENERAL_ERR, ERDMA_WC_VENDOR_NO_ERR },</div><div class='add'>+	{ ERDMA_WC_RECV_WQE_FORMAT_ERR, IB_WC_GENERAL_ERR,</div><div class='add'>+	  ERDMA_WC_VENDOR_INVALID_RQE },</div><div class='add'>+	{ ERDMA_WC_RECV_STAG_INVALID_ERR, IB_WC_REM_ACCESS_ERR,</div><div class='add'>+	  ERDMA_WC_VENDOR_RQE_INVALID_STAG },</div><div class='add'>+	{ ERDMA_WC_RECV_ADDR_VIOLATION_ERR, IB_WC_REM_ACCESS_ERR,</div><div class='add'>+	  ERDMA_WC_VENDOR_RQE_ADDR_VIOLATION },</div><div class='add'>+	{ ERDMA_WC_RECV_RIGHT_VIOLATION_ERR, IB_WC_REM_ACCESS_ERR,</div><div class='add'>+	  ERDMA_WC_VENDOR_RQE_ACCESS_RIGHT_ERR },</div><div class='add'>+	{ ERDMA_WC_RECV_PDID_ERR, IB_WC_REM_ACCESS_ERR,</div><div class='add'>+	  ERDMA_WC_VENDOR_RQE_INVALID_PD },</div><div class='add'>+	{ ERDMA_WC_RECV_WARRPING_ERR, IB_WC_REM_ACCESS_ERR,</div><div class='add'>+	  ERDMA_WC_VENDOR_RQE_WRAP_ERR },</div><div class='add'>+	{ ERDMA_WC_SEND_WQE_FORMAT_ERR, IB_WC_LOC_QP_OP_ERR,</div><div class='add'>+	  ERDMA_WC_VENDOR_INVALID_SQE },</div><div class='add'>+	{ ERDMA_WC_SEND_WQE_ORD_EXCEED, IB_WC_GENERAL_ERR,</div><div class='add'>+	  ERDMA_WC_VENDOR_ZERO_ORD },</div><div class='add'>+	{ ERDMA_WC_SEND_STAG_INVALID_ERR, IB_WC_LOC_ACCESS_ERR,</div><div class='add'>+	  ERDMA_WC_VENDOR_SQE_INVALID_STAG },</div><div class='add'>+	{ ERDMA_WC_SEND_ADDR_VIOLATION_ERR, IB_WC_LOC_ACCESS_ERR,</div><div class='add'>+	  ERDMA_WC_VENDOR_SQE_ADDR_VIOLATION },</div><div class='add'>+	{ ERDMA_WC_SEND_RIGHT_VIOLATION_ERR, IB_WC_LOC_ACCESS_ERR,</div><div class='add'>+	  ERDMA_WC_VENDOR_SQE_ACCESS_ERR },</div><div class='add'>+	{ ERDMA_WC_SEND_PDID_ERR, IB_WC_LOC_ACCESS_ERR,</div><div class='add'>+	  ERDMA_WC_VENDOR_SQE_INVALID_PD },</div><div class='add'>+	{ ERDMA_WC_SEND_WARRPING_ERR, IB_WC_LOC_ACCESS_ERR,</div><div class='add'>+	  ERDMA_WC_VENDOR_SQE_WARP_ERR },</div><div class='add'>+	{ ERDMA_WC_FLUSH_ERR, IB_WC_WR_FLUSH_ERR, ERDMA_WC_VENDOR_NO_ERR },</div><div class='add'>+	{ ERDMA_WC_RETRY_EXC_ERR, IB_WC_RETRY_EXC_ERR, ERDMA_WC_VENDOR_NO_ERR },</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static void erdma_process_ud_cqe(struct erdma_cqe *cqe, struct ib_wc *wc)</div><div class='add'>+{</div><div class='add'>+	u32 ud_info;</div><div class='add'>+</div><div class='add'>+	wc-&gt;wc_flags |= (IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE);</div><div class='add'>+	ud_info = be32_to_cpu(cqe-&gt;ud.info);</div><div class='add'>+	wc-&gt;network_hdr_type = FIELD_GET(ERDMA_CQE_NTYPE_MASK, ud_info);</div><div class='add'>+	if (wc-&gt;network_hdr_type == ERDMA_NETWORK_TYPE_IPV4)</div><div class='add'>+		wc-&gt;network_hdr_type = RDMA_NETWORK_IPV4;</div><div class='add'>+	else</div><div class='add'>+		wc-&gt;network_hdr_type = RDMA_NETWORK_IPV6;</div><div class='add'>+	wc-&gt;src_qp = FIELD_GET(ERDMA_CQE_SQPN_MASK, ud_info);</div><div class='add'>+	wc-&gt;sl = FIELD_GET(ERDMA_CQE_SL_MASK, ud_info);</div><div class='add'>+	wc-&gt;pkey_index = 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define ERDMA_POLLCQ_NO_QP 1</div><div class='add'>+</div><div class='add'>+static int erdma_poll_one_cqe(struct erdma_cq *cq, struct ib_wc *wc)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = to_edev(cq-&gt;ibcq.device);</div><div class='add'>+	u8 opcode, syndrome, qtype;</div><div class='add'>+	struct erdma_kqp *kern_qp;</div><div class='add'>+	struct erdma_cqe *cqe;</div><div class='add'>+	struct erdma_qp *qp;</div><div class='add'>+	u16 wqe_idx, depth;</div><div class='add'>+	u32 qpn, cqe_hdr;</div><div class='add'>+	u64 *id_table;</div><div class='add'>+	u64 *wqe_hdr;</div><div class='add'>+</div><div class='add'>+	cqe = get_next_valid_cqe(cq);</div><div class='add'>+	if (!cqe)</div><div class='add'>+		return -EAGAIN;</div><div class='add'>+</div><div class='add'>+	cq-&gt;kern_cq.ci++;</div><div class='add'>+</div><div class='add'>+	/* cqbuf should be ready when we poll */</div><div class='add'>+	dma_rmb();</div><div class='add'>+</div><div class='add'>+	qpn = be32_to_cpu(cqe-&gt;qpn);</div><div class='add'>+	wqe_idx = be32_to_cpu(cqe-&gt;qe_idx);</div><div class='add'>+	cqe_hdr = be32_to_cpu(cqe-&gt;hdr);</div><div class='add'>+</div><div class='add'>+	qp = find_qp_by_qpn(dev, qpn);</div><div class='add'>+	if (!qp)</div><div class='add'>+		return ERDMA_POLLCQ_NO_QP;</div><div class='add'>+</div><div class='add'>+	kern_qp = &amp;qp-&gt;kern_qp;</div><div class='add'>+</div><div class='add'>+	qtype = FIELD_GET(ERDMA_CQE_HDR_QTYPE_MASK, cqe_hdr);</div><div class='add'>+	syndrome = FIELD_GET(ERDMA_CQE_HDR_SYNDROME_MASK, cqe_hdr);</div><div class='add'>+	opcode = FIELD_GET(ERDMA_CQE_HDR_OPCODE_MASK, cqe_hdr);</div><div class='add'>+</div><div class='add'>+	if (qtype == ERDMA_CQE_QTYPE_SQ) {</div><div class='add'>+		id_table = kern_qp-&gt;swr_tbl;</div><div class='add'>+		depth = qp-&gt;attrs.sq_size;</div><div class='add'>+		wqe_hdr = get_queue_entry(qp-&gt;kern_qp.sq_buf, wqe_idx,</div><div class='add'>+					  qp-&gt;attrs.sq_size, SQEBB_SHIFT);</div><div class='add'>+		kern_qp-&gt;sq_ci =</div><div class='add'>+			FIELD_GET(ERDMA_SQE_HDR_WQEBB_CNT_MASK, *wqe_hdr) +</div><div class='add'>+			wqe_idx + 1;</div><div class='add'>+	} else {</div><div class='add'>+		id_table = kern_qp-&gt;rwr_tbl;</div><div class='add'>+		depth = qp-&gt;attrs.rq_size;</div><div class='add'>+	}</div><div class='add'>+	wc-&gt;wr_id = id_table[wqe_idx &amp; (depth - 1)];</div><div class='add'>+	wc-&gt;byte_len = be32_to_cpu(cqe-&gt;size);</div><div class='add'>+</div><div class='add'>+	wc-&gt;wc_flags = 0;</div><div class='add'>+</div><div class='add'>+	wc-&gt;opcode = wc_mapping_table[opcode];</div><div class='add'>+	if (opcode == ERDMA_OP_RECV_IMM || opcode == ERDMA_OP_RSP_SEND_IMM) {</div><div class='add'>+		wc-&gt;ex.imm_data = cpu_to_be32(le32_to_cpu(cqe-&gt;imm_data));</div><div class='add'>+		wc-&gt;wc_flags |= IB_WC_WITH_IMM;</div><div class='add'>+	} else if (opcode == ERDMA_OP_RECV_INV) {</div><div class='add'>+		wc-&gt;ex.invalidate_rkey = be32_to_cpu(cqe-&gt;inv_rkey);</div><div class='add'>+		wc-&gt;wc_flags |= IB_WC_WITH_INVALIDATE;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (erdma_device_rocev2(dev) &amp;&amp;</div><div class='add'>+	    (qp-&gt;ibqp.qp_type == IB_QPT_UD || qp-&gt;ibqp.qp_type == IB_QPT_GSI))</div><div class='add'>+		erdma_process_ud_cqe(cqe, wc);</div><div class='add'>+</div><div class='add'>+	if (syndrome &gt;= ERDMA_NUM_WC_STATUS)</div><div class='add'>+		syndrome = ERDMA_WC_GENERAL_ERR;</div><div class='add'>+</div><div class='add'>+	wc-&gt;status = map_cqe_status[syndrome].base;</div><div class='add'>+	wc-&gt;vendor_err = map_cqe_status[syndrome].vendor;</div><div class='add'>+	wc-&gt;qp = &amp;qp-&gt;ibqp;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cq *cq = to_ecq(ibcq);</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int npolled, ret;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cq-&gt;kern_cq.lock, flags);</div><div class='add'>+</div><div class='add'>+	for (npolled = 0; npolled &lt; num_entries;) {</div><div class='add'>+		ret = erdma_poll_one_cqe(cq, wc + npolled);</div><div class='add'>+</div><div class='add'>+		if (ret == -EAGAIN) /* no received new CQEs. */</div><div class='add'>+			break;</div><div class='add'>+		else if (ret) /* ignore invalid CQEs. */</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		npolled++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_unlock_irqrestore(&amp;cq-&gt;kern_cq.lock, flags);</div><div class='add'>+</div><div class='add'>+	return npolled;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_remove_cqes_of_qp(struct ib_cq *ibcq, u32 qpn)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cq *cq = to_ecq(ibcq);</div><div class='add'>+	struct erdma_cqe *cqe, *dst_cqe;</div><div class='add'>+	u32 prev_cq_ci, cur_cq_ci;</div><div class='add'>+	u32 ncqe = 0, nqp_cqe = 0;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	u8 owner;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cq-&gt;kern_cq.lock, flags);</div><div class='add'>+</div><div class='add'>+	prev_cq_ci = cq-&gt;kern_cq.ci;</div><div class='add'>+</div><div class='add'>+	while (ncqe &lt; cq-&gt;depth &amp;&amp; (cqe = get_next_valid_cqe(cq)) != NULL) {</div><div class='add'>+		++cq-&gt;kern_cq.ci;</div><div class='add'>+		++ncqe;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	while (ncqe &gt; 0) {</div><div class='add'>+		cur_cq_ci = prev_cq_ci + ncqe - 1;</div><div class='add'>+		cqe = get_queue_entry(cq-&gt;kern_cq.qbuf, cur_cq_ci, cq-&gt;depth,</div><div class='add'>+				      CQE_SHIFT);</div><div class='add'>+</div><div class='add'>+		if (be32_to_cpu(cqe-&gt;qpn) == qpn) {</div><div class='add'>+			++nqp_cqe;</div><div class='add'>+		} else if (nqp_cqe) {</div><div class='add'>+			dst_cqe = get_queue_entry(cq-&gt;kern_cq.qbuf,</div><div class='add'>+						  cur_cq_ci + nqp_cqe,</div><div class='add'>+						  cq-&gt;depth, CQE_SHIFT);</div><div class='add'>+			owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,</div><div class='add'>+					  be32_to_cpu(dst_cqe-&gt;hdr));</div><div class='add'>+			cqe-&gt;hdr = cpu_to_be32(</div><div class='add'>+				(be32_to_cpu(cqe-&gt;hdr) &amp;</div><div class='add'>+				 ~ERDMA_CQE_HDR_OWNER_MASK) |</div><div class='add'>+				FIELD_PREP(ERDMA_CQE_HDR_OWNER_MASK, owner));</div><div class='add'>+			memcpy(dst_cqe, cqe, sizeof(*cqe));</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		--ncqe;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cq-&gt;kern_cq.ci = prev_cq_ci + nqp_cqe;</div><div class='add'>+	spin_unlock_irqrestore(&amp;cq-&gt;kern_cq.lock, flags);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/erdma/erdma_eq.c b/drivers/infiniband/hw/erdma/erdma_eq.c<br/>new file mode 100644<br/>index 000000000000..6486234a2360<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/erdma/erdma_eq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/erdma/erdma_eq.c</a></div><div class='hunk'>@@ -0,0 +1,326 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='add'>+</div><div class='add'>+/* Authors: Cheng Xu &lt;chengyou@linux.alibaba.com&gt; */</div><div class='add'>+/*          Kai Shen &lt;kaishen@linux.alibaba.com&gt; */</div><div class='add'>+/* Copyright (c) 2020-2022, Alibaba Group. */</div><div class='add'>+</div><div class='add'>+#include "erdma_verbs.h"</div><div class='add'>+</div><div class='add'>+#define MAX_POLL_CHUNK_SIZE 16</div><div class='add'>+</div><div class='add'>+void notify_eq(struct erdma_eq *eq)</div><div class='add'>+{</div><div class='add'>+	u64 db_data = FIELD_PREP(ERDMA_EQDB_CI_MASK, eq-&gt;ci) |</div><div class='add'>+		      FIELD_PREP(ERDMA_EQDB_ARM_MASK, 1);</div><div class='add'>+</div><div class='add'>+	*eq-&gt;dbrec = db_data;</div><div class='add'>+	writeq(db_data, eq-&gt;db);</div><div class='add'>+</div><div class='add'>+	atomic64_inc(&amp;eq-&gt;notify_num);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void *get_next_valid_eqe(struct erdma_eq *eq)</div><div class='add'>+{</div><div class='add'>+	u64 *eqe = get_queue_entry(eq-&gt;qbuf, eq-&gt;ci, eq-&gt;depth, EQE_SHIFT);</div><div class='add'>+	u32 owner = FIELD_GET(ERDMA_CEQE_HDR_O_MASK, READ_ONCE(*eqe));</div><div class='add'>+</div><div class='add'>+	return owner ^ !!(eq-&gt;ci &amp; eq-&gt;depth) ? eqe : NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_aeq_event_handler(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct erdma_aeqe *aeqe;</div><div class='add'>+	u32 cqn, qpn;</div><div class='add'>+	struct erdma_qp *qp;</div><div class='add'>+	struct erdma_cq *cq;</div><div class='add'>+	struct ib_event event;</div><div class='add'>+	u32 poll_cnt = 0;</div><div class='add'>+</div><div class='add'>+	memset(&amp;event, 0, sizeof(event));</div><div class='add'>+</div><div class='add'>+	while (poll_cnt &lt; MAX_POLL_CHUNK_SIZE) {</div><div class='add'>+		aeqe = get_next_valid_eqe(&amp;dev-&gt;aeq);</div><div class='add'>+		if (!aeqe)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		dma_rmb();</div><div class='add'>+</div><div class='add'>+		dev-&gt;aeq.ci++;</div><div class='add'>+		atomic64_inc(&amp;dev-&gt;aeq.event_num);</div><div class='add'>+		poll_cnt++;</div><div class='add'>+</div><div class='add'>+		if (FIELD_GET(ERDMA_AEQE_HDR_TYPE_MASK,</div><div class='add'>+			      le32_to_cpu(aeqe-&gt;hdr)) == ERDMA_AE_TYPE_CQ_ERR) {</div><div class='add'>+			cqn = le32_to_cpu(aeqe-&gt;event_data0);</div><div class='add'>+			cq = find_cq_by_cqn(dev, cqn);</div><div class='add'>+			if (!cq)</div><div class='add'>+				continue;</div><div class='add'>+</div><div class='add'>+			event.device = cq-&gt;ibcq.device;</div><div class='add'>+			event.element.cq = &amp;cq-&gt;ibcq;</div><div class='add'>+			event.event = IB_EVENT_CQ_ERR;</div><div class='add'>+			if (cq-&gt;ibcq.event_handler)</div><div class='add'>+				cq-&gt;ibcq.event_handler(&amp;event,</div><div class='add'>+						       cq-&gt;ibcq.cq_context);</div><div class='add'>+		} else {</div><div class='add'>+			qpn = le32_to_cpu(aeqe-&gt;event_data0);</div><div class='add'>+			qp = find_qp_by_qpn(dev, qpn);</div><div class='add'>+			if (!qp)</div><div class='add'>+				continue;</div><div class='add'>+</div><div class='add'>+			event.device = qp-&gt;ibqp.device;</div><div class='add'>+			event.element.qp = &amp;qp-&gt;ibqp;</div><div class='add'>+			event.event = IB_EVENT_QP_FATAL;</div><div class='add'>+			if (qp-&gt;ibqp.event_handler)</div><div class='add'>+				qp-&gt;ibqp.event_handler(&amp;event,</div><div class='add'>+						       qp-&gt;ibqp.qp_context);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	notify_eq(&amp;dev-&gt;aeq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_eq_common_init(struct erdma_dev *dev, struct erdma_eq *eq, u32 depth)</div><div class='add'>+{</div><div class='add'>+	u32 buf_size = depth &lt;&lt; EQE_SHIFT;</div><div class='add'>+</div><div class='add'>+	eq-&gt;qbuf = dma_alloc_coherent(&amp;dev-&gt;pdev-&gt;dev, buf_size,</div><div class='add'>+				      &amp;eq-&gt;qbuf_dma_addr, GFP_KERNEL);</div><div class='add'>+	if (!eq-&gt;qbuf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	eq-&gt;dbrec = dma_pool_zalloc(dev-&gt;db_pool, GFP_KERNEL, &amp;eq-&gt;dbrec_dma);</div><div class='add'>+	if (!eq-&gt;dbrec)</div><div class='add'>+		goto err_free_qbuf;</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;eq-&gt;lock);</div><div class='add'>+	atomic64_set(&amp;eq-&gt;event_num, 0);</div><div class='add'>+	atomic64_set(&amp;eq-&gt;notify_num, 0);</div><div class='add'>+	eq-&gt;ci = 0;</div><div class='add'>+	eq-&gt;depth = depth;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_free_qbuf:</div><div class='add'>+	dma_free_coherent(&amp;dev-&gt;pdev-&gt;dev, buf_size, eq-&gt;qbuf,</div><div class='add'>+			  eq-&gt;qbuf_dma_addr);</div><div class='add'>+</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_eq_destroy(struct erdma_dev *dev, struct erdma_eq *eq)</div><div class='add'>+{</div><div class='add'>+	dma_pool_free(dev-&gt;db_pool, eq-&gt;dbrec, eq-&gt;dbrec_dma);</div><div class='add'>+	dma_free_coherent(&amp;dev-&gt;pdev-&gt;dev, eq-&gt;depth &lt;&lt; EQE_SHIFT, eq-&gt;qbuf,</div><div class='add'>+			  eq-&gt;qbuf_dma_addr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_aeq_init(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct erdma_eq *eq = &amp;dev-&gt;aeq;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = erdma_eq_common_init(dev, &amp;dev-&gt;aeq, ERDMA_DEFAULT_EQ_DEPTH);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	eq-&gt;db = dev-&gt;func_bar + ERDMA_REGS_AEQ_DB_REG;</div><div class='add'>+</div><div class='add'>+	erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_H_REG,</div><div class='add'>+			  upper_32_bits(eq-&gt;qbuf_dma_addr));</div><div class='add'>+	erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_L_REG,</div><div class='add'>+			  lower_32_bits(eq-&gt;qbuf_dma_addr));</div><div class='add'>+	erdma_reg_write32(dev, ERDMA_REGS_AEQ_DEPTH_REG, eq-&gt;depth);</div><div class='add'>+	erdma_reg_write64(dev, ERDMA_AEQ_DB_HOST_ADDR_REG, eq-&gt;dbrec_dma);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = ceq_cb-&gt;dev;</div><div class='add'>+	struct erdma_cq *cq;</div><div class='add'>+	u32 poll_cnt = 0;</div><div class='add'>+	u64 *ceqe;</div><div class='add'>+	int cqn;</div><div class='add'>+</div><div class='add'>+	if (!ceq_cb-&gt;ready)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	while (poll_cnt &lt; MAX_POLL_CHUNK_SIZE) {</div><div class='add'>+		ceqe = get_next_valid_eqe(&amp;ceq_cb-&gt;eq);</div><div class='add'>+		if (!ceqe)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		dma_rmb();</div><div class='add'>+		ceq_cb-&gt;eq.ci++;</div><div class='add'>+		poll_cnt++;</div><div class='add'>+		cqn = FIELD_GET(ERDMA_CEQE_HDR_CQN_MASK, READ_ONCE(*ceqe));</div><div class='add'>+</div><div class='add'>+		cq = find_cq_by_cqn(dev, cqn);</div><div class='add'>+		if (!cq)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		if (rdma_is_kernel_res(&amp;cq-&gt;ibcq.res))</div><div class='add'>+			cq-&gt;kern_cq.cmdsn++;</div><div class='add'>+</div><div class='add'>+		if (cq-&gt;ibcq.comp_handler)</div><div class='add'>+			cq-&gt;ibcq.comp_handler(&amp;cq-&gt;ibcq, cq-&gt;ibcq.cq_context);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	notify_eq(&amp;ceq_cb-&gt;eq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static irqreturn_t erdma_intr_ceq_handler(int irq, void *data)</div><div class='add'>+{</div><div class='add'>+	struct erdma_eq_cb *ceq_cb = data;</div><div class='add'>+</div><div class='add'>+	tasklet_schedule(&amp;ceq_cb-&gt;tasklet);</div><div class='add'>+</div><div class='add'>+	return IRQ_HANDLED;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_intr_ceq_task(unsigned long data)</div><div class='add'>+{</div><div class='add'>+	erdma_ceq_completion_handler((struct erdma_eq_cb *)data);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_set_ceq_irq(struct erdma_dev *dev, u16 ceqn)</div><div class='add'>+{</div><div class='add'>+	struct erdma_eq_cb *eqc = &amp;dev-&gt;ceqs[ceqn];</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	snprintf(eqc-&gt;irq.name, ERDMA_IRQNAME_SIZE, "erdma-ceq%u@pci:%s", ceqn,</div><div class='add'>+		 pci_name(dev-&gt;pdev));</div><div class='add'>+	eqc-&gt;irq.msix_vector = pci_irq_vector(dev-&gt;pdev, ceqn + 1);</div><div class='add'>+</div><div class='add'>+	tasklet_init(&amp;dev-&gt;ceqs[ceqn].tasklet, erdma_intr_ceq_task,</div><div class='add'>+		     (unsigned long)&amp;dev-&gt;ceqs[ceqn]);</div><div class='add'>+</div><div class='add'>+	cpumask_set_cpu(cpumask_local_spread(ceqn + 1, dev-&gt;attrs.numa_node),</div><div class='add'>+			&amp;eqc-&gt;irq.affinity_hint_mask);</div><div class='add'>+</div><div class='add'>+	err = request_irq(eqc-&gt;irq.msix_vector, erdma_intr_ceq_handler, 0,</div><div class='add'>+			  eqc-&gt;irq.name, eqc);</div><div class='add'>+	if (err) {</div><div class='add'>+		dev_err(&amp;dev-&gt;pdev-&gt;dev, "failed to request_irq(%d)\n", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irq_set_affinity_hint(eqc-&gt;irq.msix_vector,</div><div class='add'>+			      &amp;eqc-&gt;irq.affinity_hint_mask);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_free_ceq_irq(struct erdma_dev *dev, u16 ceqn)</div><div class='add'>+{</div><div class='add'>+	struct erdma_eq_cb *eqc = &amp;dev-&gt;ceqs[ceqn];</div><div class='add'>+</div><div class='add'>+	irq_set_affinity_hint(eqc-&gt;irq.msix_vector, NULL);</div><div class='add'>+	free_irq(eqc-&gt;irq.msix_vector, eqc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cmdq_create_eq_req req;</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_COMMON,</div><div class='add'>+				CMDQ_OPCODE_CREATE_EQ);</div><div class='add'>+	req.eqn = eqn;</div><div class='add'>+	req.depth = ilog2(eq-&gt;depth);</div><div class='add'>+	req.qbuf_addr = eq-&gt;qbuf_dma_addr;</div><div class='add'>+	req.qtype = ERDMA_EQ_TYPE_CEQ;</div><div class='add'>+	/* Vector index is the same as EQN. */</div><div class='add'>+	req.vector_idx = eqn;</div><div class='add'>+	req.db_dma_addr_l = lower_32_bits(eq-&gt;dbrec_dma);</div><div class='add'>+	req.db_dma_addr_h = upper_32_bits(eq-&gt;dbrec_dma);</div><div class='add'>+</div><div class='add'>+	return erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL,</div><div class='add'>+				   false);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn)</div><div class='add'>+{</div><div class='add'>+	struct erdma_eq *eq = &amp;dev-&gt;ceqs[ceqn].eq;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = erdma_eq_common_init(dev, eq, ERDMA_DEFAULT_EQ_DEPTH);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	eq-&gt;db = dev-&gt;func_bar + ERDMA_REGS_CEQ_DB_BASE_REG +</div><div class='add'>+		 (ceqn + 1) * ERDMA_DB_SIZE;</div><div class='add'>+	dev-&gt;ceqs[ceqn].dev = dev;</div><div class='add'>+	dev-&gt;ceqs[ceqn].ready = true;</div><div class='add'>+</div><div class='add'>+	/* CEQ indexed from 1, 0 rsvd for CMDQ-EQ. */</div><div class='add'>+	ret = create_eq_cmd(dev, ceqn + 1, eq);</div><div class='add'>+	if (ret) {</div><div class='add'>+		erdma_eq_destroy(dev, eq);</div><div class='add'>+		dev-&gt;ceqs[ceqn].ready = false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_ceq_uninit_one(struct erdma_dev *dev, u16 ceqn)</div><div class='add'>+{</div><div class='add'>+	struct erdma_eq *eq = &amp;dev-&gt;ceqs[ceqn].eq;</div><div class='add'>+	struct erdma_cmdq_destroy_eq_req req;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	dev-&gt;ceqs[ceqn].ready = 0;</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_COMMON,</div><div class='add'>+				CMDQ_OPCODE_DESTROY_EQ);</div><div class='add'>+	/* CEQ indexed from 1, 0 rsvd for CMDQ-EQ. */</div><div class='add'>+	req.eqn = ceqn + 1;</div><div class='add'>+	req.qtype = ERDMA_EQ_TYPE_CEQ;</div><div class='add'>+	req.vector_idx = ceqn + 1;</div><div class='add'>+</div><div class='add'>+	err = erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL,</div><div class='add'>+				  false);</div><div class='add'>+	if (err)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	erdma_eq_destroy(dev, eq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_ceqs_init(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	u32 i, j;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; dev-&gt;attrs.irq_num - 1; i++) {</div><div class='add'>+		err = erdma_ceq_init_one(dev, i);</div><div class='add'>+		if (err)</div><div class='add'>+			goto out_err;</div><div class='add'>+</div><div class='add'>+		err = erdma_set_ceq_irq(dev, i);</div><div class='add'>+		if (err) {</div><div class='add'>+			erdma_ceq_uninit_one(dev, i);</div><div class='add'>+			goto out_err;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+out_err:</div><div class='add'>+	for (j = 0; j &lt; i; j++) {</div><div class='add'>+		erdma_free_ceq_irq(dev, j);</div><div class='add'>+		erdma_ceq_uninit_one(dev, j);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_ceqs_uninit(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	u32 i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; dev-&gt;attrs.irq_num - 1; i++) {</div><div class='add'>+		erdma_free_ceq_irq(dev, i);</div><div class='add'>+		erdma_ceq_uninit_one(dev, i);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/erdma/erdma_hw.h b/drivers/infiniband/hw/erdma/erdma_hw.h<br/>new file mode 100644<br/>index 000000000000..ea4db53901a4<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/erdma/erdma_hw.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/erdma/erdma_hw.h</a></div><div class='hunk'>@@ -0,0 +1,753 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='add'>+</div><div class='add'>+/* Authors: Cheng Xu &lt;chengyou@linux.alibaba.com&gt; */</div><div class='add'>+/*          Kai Shen &lt;kaishen@linux.alibaba.com&gt; */</div><div class='add'>+/* Copyright (c) 2020-2022, Alibaba Group. */</div><div class='add'>+</div><div class='add'>+#ifndef __ERDMA_HW_H__</div><div class='add'>+#define __ERDMA_HW_H__</div><div class='add'>+</div><div class='add'>+#include &lt;linux/kernel.h&gt;</div><div class='add'>+#include &lt;linux/types.h&gt;</div><div class='add'>+#include &lt;linux/if_ether.h&gt;</div><div class='add'>+</div><div class='add'>+/* PCIe device related definition. */</div><div class='add'>+#define ERDMA_PCI_WIDTH 64</div><div class='add'>+#define ERDMA_FUNC_BAR 0</div><div class='add'>+#define ERDMA_MISX_BAR 2</div><div class='add'>+</div><div class='add'>+#define ERDMA_BAR_MASK (BIT(ERDMA_FUNC_BAR) | BIT(ERDMA_MISX_BAR))</div><div class='add'>+</div><div class='add'>+/* MSI-X related. */</div><div class='add'>+#define ERDMA_NUM_MSIX_VEC 32U</div><div class='add'>+#define ERDMA_MSIX_VECTOR_CMDQ 0</div><div class='add'>+</div><div class='add'>+/* RoCEv2 related */</div><div class='add'>+#define ERDMA_ROCEV2_GID_SIZE 16</div><div class='add'>+#define ERDMA_MAX_PKEYS 1</div><div class='add'>+#define ERDMA_DEFAULT_PKEY 0xFFFF</div><div class='add'>+</div><div class='add'>+/* erdma device protocol type */</div><div class='add'>+enum erdma_proto_type {</div><div class='add'>+	ERDMA_PROTO_IWARP = 0,</div><div class='add'>+	ERDMA_PROTO_ROCEV2 = 1,</div><div class='add'>+	ERDMA_PROTO_COUNT = 2,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* PCIe Bar0 Registers. */</div><div class='add'>+#define ERDMA_REGS_VERSION_REG 0x0</div><div class='add'>+#define ERDMA_REGS_DEV_PROTO_REG 0xC</div><div class='add'>+#define ERDMA_REGS_DEV_CTRL_REG 0x10</div><div class='add'>+#define ERDMA_REGS_DEV_ST_REG 0x14</div><div class='add'>+#define ERDMA_REGS_NETDEV_MAC_L_REG 0x18</div><div class='add'>+#define ERDMA_REGS_NETDEV_MAC_H_REG 0x1C</div><div class='add'>+#define ERDMA_REGS_CMDQ_SQ_ADDR_L_REG 0x20</div><div class='add'>+#define ERDMA_REGS_CMDQ_SQ_ADDR_H_REG 0x24</div><div class='add'>+#define ERDMA_REGS_CMDQ_CQ_ADDR_L_REG 0x28</div><div class='add'>+#define ERDMA_REGS_CMDQ_CQ_ADDR_H_REG 0x2C</div><div class='add'>+#define ERDMA_REGS_CMDQ_DEPTH_REG 0x30</div><div class='add'>+#define ERDMA_REGS_CMDQ_EQ_DEPTH_REG 0x34</div><div class='add'>+#define ERDMA_REGS_CMDQ_EQ_ADDR_L_REG 0x38</div><div class='add'>+#define ERDMA_REGS_CMDQ_EQ_ADDR_H_REG 0x3C</div><div class='add'>+#define ERDMA_REGS_AEQ_ADDR_L_REG 0x40</div><div class='add'>+#define ERDMA_REGS_AEQ_ADDR_H_REG 0x44</div><div class='add'>+#define ERDMA_REGS_AEQ_DEPTH_REG 0x48</div><div class='add'>+#define ERDMA_REGS_GRP_NUM_REG 0x4c</div><div class='add'>+#define ERDMA_REGS_AEQ_DB_REG 0x50</div><div class='add'>+#define ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG 0x60</div><div class='add'>+#define ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG 0x68</div><div class='add'>+#define ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG 0x70</div><div class='add'>+#define ERDMA_AEQ_DB_HOST_ADDR_REG 0x78</div><div class='add'>+#define ERDMA_REGS_STATS_TSO_IN_PKTS_REG 0x80</div><div class='add'>+#define ERDMA_REGS_STATS_TSO_OUT_PKTS_REG 0x88</div><div class='add'>+#define ERDMA_REGS_STATS_TSO_OUT_BYTES_REG 0x90</div><div class='add'>+#define ERDMA_REGS_STATS_TX_DROP_PKTS_REG 0x98</div><div class='add'>+#define ERDMA_REGS_STATS_TX_BPS_METER_DROP_PKTS_REG 0xa0</div><div class='add'>+#define ERDMA_REGS_STATS_TX_PPS_METER_DROP_PKTS_REG 0xa8</div><div class='add'>+#define ERDMA_REGS_STATS_RX_PKTS_REG 0xc0</div><div class='add'>+#define ERDMA_REGS_STATS_RX_BYTES_REG 0xc8</div><div class='add'>+#define ERDMA_REGS_STATS_RX_DROP_PKTS_REG 0xd0</div><div class='add'>+#define ERDMA_REGS_STATS_RX_BPS_METER_DROP_PKTS_REG 0xd8</div><div class='add'>+#define ERDMA_REGS_STATS_RX_PPS_METER_DROP_PKTS_REG 0xe0</div><div class='add'>+#define ERDMA_REGS_CEQ_DB_BASE_REG 0x100</div><div class='add'>+#define ERDMA_CMDQ_SQDB_REG 0x200</div><div class='add'>+#define ERDMA_CMDQ_CQDB_REG 0x300</div><div class='add'>+</div><div class='add'>+/* DEV_CTRL_REG details. */</div><div class='add'>+#define ERDMA_REG_DEV_CTRL_RESET_MASK 0x00000001</div><div class='add'>+#define ERDMA_REG_DEV_CTRL_INIT_MASK 0x00000002</div><div class='add'>+</div><div class='add'>+/* DEV_ST_REG details. */</div><div class='add'>+#define ERDMA_REG_DEV_ST_RESET_DONE_MASK 0x00000001U</div><div class='add'>+#define ERDMA_REG_DEV_ST_INIT_DONE_MASK 0x00000002U</div><div class='add'>+</div><div class='add'>+/* eRDMA PCIe DBs definition. */</div><div class='add'>+#define ERDMA_BAR_DB_SPACE_BASE 4096</div><div class='add'>+</div><div class='add'>+#define ERDMA_BAR_SQDB_SPACE_OFFSET ERDMA_BAR_DB_SPACE_BASE</div><div class='add'>+#define ERDMA_BAR_SQDB_SPACE_SIZE (384 * 1024)</div><div class='add'>+</div><div class='add'>+#define ERDMA_BAR_RQDB_SPACE_OFFSET \</div><div class='add'>+	(ERDMA_BAR_SQDB_SPACE_OFFSET + ERDMA_BAR_SQDB_SPACE_SIZE)</div><div class='add'>+#define ERDMA_BAR_RQDB_SPACE_SIZE (96 * 1024)</div><div class='add'>+</div><div class='add'>+#define ERDMA_BAR_CQDB_SPACE_OFFSET \</div><div class='add'>+	(ERDMA_BAR_RQDB_SPACE_OFFSET + ERDMA_BAR_RQDB_SPACE_SIZE)</div><div class='add'>+</div><div class='add'>+#define ERDMA_SDB_SHARED_PAGE_INDEX 95</div><div class='add'>+</div><div class='add'>+/* Doorbell related. */</div><div class='add'>+#define ERDMA_DB_SIZE 8</div><div class='add'>+</div><div class='add'>+#define ERDMA_CQDB_IDX_MASK GENMASK_ULL(63, 56)</div><div class='add'>+#define ERDMA_CQDB_CQN_MASK GENMASK_ULL(55, 32)</div><div class='add'>+#define ERDMA_CQDB_ARM_MASK BIT_ULL(31)</div><div class='add'>+#define ERDMA_CQDB_SOL_MASK BIT_ULL(30)</div><div class='add'>+#define ERDMA_CQDB_CMDSN_MASK GENMASK_ULL(29, 28)</div><div class='add'>+#define ERDMA_CQDB_CI_MASK GENMASK_ULL(23, 0)</div><div class='add'>+</div><div class='add'>+#define ERDMA_EQDB_ARM_MASK BIT(31)</div><div class='add'>+#define ERDMA_EQDB_CI_MASK GENMASK_ULL(23, 0)</div><div class='add'>+</div><div class='add'>+#define ERDMA_PAGE_SIZE_SUPPORT 0x7FFFF000</div><div class='add'>+</div><div class='add'>+/* Hardware page size definition */</div><div class='add'>+#define ERDMA_HW_PAGE_SHIFT 12</div><div class='add'>+#define ERDMA_HW_PAGE_SIZE 4096</div><div class='add'>+</div><div class='add'>+/* WQE related. */</div><div class='add'>+#define EQE_SIZE 16</div><div class='add'>+#define EQE_SHIFT 4</div><div class='add'>+#define RQE_SIZE 32</div><div class='add'>+#define RQE_SHIFT 5</div><div class='add'>+#define CQE_SIZE 32</div><div class='add'>+#define CQE_SHIFT 5</div><div class='add'>+#define SQEBB_SIZE 32</div><div class='add'>+#define SQEBB_SHIFT 5</div><div class='add'>+#define SQEBB_MASK (~(SQEBB_SIZE - 1))</div><div class='add'>+#define SQEBB_ALIGN(size) ((size + SQEBB_SIZE - 1) &amp; SQEBB_MASK)</div><div class='add'>+#define SQEBB_COUNT(size) (SQEBB_ALIGN(size) &gt;&gt; SQEBB_SHIFT)</div><div class='add'>+</div><div class='add'>+#define ERDMA_MAX_SQE_SIZE 128</div><div class='add'>+#define ERDMA_MAX_WQEBB_PER_SQE 4</div><div class='add'>+</div><div class='add'>+/* CMDQ related. */</div><div class='add'>+#define ERDMA_CMDQ_MAX_OUTSTANDING 128</div><div class='add'>+#define ERDMA_CMDQ_SQE_SIZE 128</div><div class='add'>+</div><div class='add'>+/* cmdq sub module definition. */</div><div class='add'>+enum CMDQ_WQE_SUB_MOD {</div><div class='add'>+	CMDQ_SUBMOD_RDMA = 0,</div><div class='add'>+	CMDQ_SUBMOD_COMMON = 1</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum CMDQ_RDMA_OPCODE {</div><div class='add'>+	CMDQ_OPCODE_QUERY_DEVICE = 0,</div><div class='add'>+	CMDQ_OPCODE_CREATE_QP = 1,</div><div class='add'>+	CMDQ_OPCODE_DESTROY_QP = 2,</div><div class='add'>+	CMDQ_OPCODE_MODIFY_QP = 3,</div><div class='add'>+	CMDQ_OPCODE_CREATE_CQ = 4,</div><div class='add'>+	CMDQ_OPCODE_DESTROY_CQ = 5,</div><div class='add'>+	CMDQ_OPCODE_REFLUSH = 6,</div><div class='add'>+	CMDQ_OPCODE_REG_MR = 8,</div><div class='add'>+	CMDQ_OPCODE_DEREG_MR = 9,</div><div class='add'>+	CMDQ_OPCODE_SET_GID = 14,</div><div class='add'>+	CMDQ_OPCODE_CREATE_AH = 15,</div><div class='add'>+	CMDQ_OPCODE_DESTROY_AH = 16,</div><div class='add'>+	CMDQ_OPCODE_QUERY_QP = 17,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum CMDQ_COMMON_OPCODE {</div><div class='add'>+	CMDQ_OPCODE_CREATE_EQ = 0,</div><div class='add'>+	CMDQ_OPCODE_DESTROY_EQ = 1,</div><div class='add'>+	CMDQ_OPCODE_QUERY_FW_INFO = 2,</div><div class='add'>+	CMDQ_OPCODE_CONF_MTU = 3,</div><div class='add'>+	CMDQ_OPCODE_GET_STATS = 4,</div><div class='add'>+	CMDQ_OPCODE_CONF_DEVICE = 5,</div><div class='add'>+	CMDQ_OPCODE_ALLOC_DB = 8,</div><div class='add'>+	CMDQ_OPCODE_FREE_DB = 9,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cmdq-SQE HDR */</div><div class='add'>+#define ERDMA_CMD_HDR_WQEBB_CNT_MASK GENMASK_ULL(54, 52)</div><div class='add'>+#define ERDMA_CMD_HDR_CONTEXT_COOKIE_MASK GENMASK_ULL(47, 32)</div><div class='add'>+#define ERDMA_CMD_HDR_SUB_MOD_MASK GENMASK_ULL(25, 24)</div><div class='add'>+#define ERDMA_CMD_HDR_OPCODE_MASK GENMASK_ULL(23, 16)</div><div class='add'>+#define ERDMA_CMD_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0)</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_destroy_cq_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 cqn;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define ERDMA_EQ_TYPE_AEQ 0</div><div class='add'>+#define ERDMA_EQ_TYPE_CEQ 1</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_create_eq_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u64 qbuf_addr;</div><div class='add'>+	u8 vector_idx;</div><div class='add'>+	u8 eqn;</div><div class='add'>+	u8 depth;</div><div class='add'>+	u8 qtype;</div><div class='add'>+	u32 db_dma_addr_l;</div><div class='add'>+	u32 db_dma_addr_h;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_destroy_eq_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u64 rsvd0;</div><div class='add'>+	u8 vector_idx;</div><div class='add'>+	u8 eqn;</div><div class='add'>+	u8 rsvd1;</div><div class='add'>+	u8 qtype;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* config device cfg */</div><div class='add'>+#define ERDMA_CMD_CONFIG_DEVICE_PS_EN_MASK BIT(31)</div><div class='add'>+#define ERDMA_CMD_CONFIG_DEVICE_PGSHIFT_MASK GENMASK(4, 0)</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_config_device_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 cfg;</div><div class='add'>+	u32 rsvd[5];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_config_mtu_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 mtu;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* ext db requests(alloc and free) cfg */</div><div class='add'>+#define ERDMA_CMD_EXT_DB_CQ_EN_MASK BIT(2)</div><div class='add'>+#define ERDMA_CMD_EXT_DB_RQ_EN_MASK BIT(1)</div><div class='add'>+#define ERDMA_CMD_EXT_DB_SQ_EN_MASK BIT(0)</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_ext_db_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 cfg;</div><div class='add'>+	u16 rdb_off;</div><div class='add'>+	u16 sdb_off;</div><div class='add'>+	u16 rsvd0;</div><div class='add'>+	u16 cdb_off;</div><div class='add'>+	u32 rsvd1[3];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* alloc db response qword 0 definition */</div><div class='add'>+#define ERDMA_CMD_ALLOC_DB_RESP_RDB_MASK GENMASK_ULL(63, 48)</div><div class='add'>+#define ERDMA_CMD_ALLOC_DB_RESP_CDB_MASK GENMASK_ULL(47, 32)</div><div class='add'>+#define ERDMA_CMD_ALLOC_DB_RESP_SDB_MASK GENMASK_ULL(15, 0)</div><div class='add'>+</div><div class='add'>+/* create_cq cfg0 */</div><div class='add'>+#define ERDMA_CMD_CREATE_CQ_DEPTH_MASK GENMASK(31, 24)</div><div class='add'>+#define ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK GENMASK(23, 20)</div><div class='add'>+#define ERDMA_CMD_CREATE_CQ_CQN_MASK GENMASK(19, 0)</div><div class='add'>+</div><div class='add'>+/* create_cq cfg1 */</div><div class='add'>+#define ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK GENMASK(31, 16)</div><div class='add'>+#define ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK BIT(15)</div><div class='add'>+#define ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK BIT(11)</div><div class='add'>+#define ERDMA_CMD_CREATE_CQ_EQN_MASK GENMASK(9, 0)</div><div class='add'>+</div><div class='add'>+/* create_cq cfg2 */</div><div class='add'>+#define ERDMA_CMD_CREATE_CQ_DB_CFG_MASK GENMASK(15, 0)</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_create_cq_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 cfg0;</div><div class='add'>+	u32 qbuf_addr_l;</div><div class='add'>+	u32 qbuf_addr_h;</div><div class='add'>+	u32 cfg1;</div><div class='add'>+	u64 cq_dbrec_dma;</div><div class='add'>+	u32 first_page_offset;</div><div class='add'>+	u32 cfg2;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* regmr/deregmr cfg0 */</div><div class='add'>+#define ERDMA_CMD_MR_VALID_MASK BIT(31)</div><div class='add'>+#define ERDMA_CMD_MR_VERSION_MASK GENMASK(30, 28)</div><div class='add'>+#define ERDMA_CMD_MR_KEY_MASK GENMASK(27, 20)</div><div class='add'>+#define ERDMA_CMD_MR_MPT_IDX_MASK GENMASK(19, 0)</div><div class='add'>+</div><div class='add'>+/* regmr cfg1 */</div><div class='add'>+#define ERDMA_CMD_REGMR_PD_MASK GENMASK(31, 12)</div><div class='add'>+#define ERDMA_CMD_REGMR_TYPE_MASK GENMASK(7, 6)</div><div class='add'>+#define ERDMA_CMD_REGMR_RIGHT_MASK GENMASK(5, 1)</div><div class='add'>+</div><div class='add'>+/* regmr cfg2 */</div><div class='add'>+#define ERDMA_CMD_REGMR_PAGESIZE_MASK GENMASK(31, 27)</div><div class='add'>+#define ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK GENMASK(26, 24)</div><div class='add'>+#define ERDMA_CMD_REGMR_MTT_LEVEL_MASK GENMASK(21, 20)</div><div class='add'>+#define ERDMA_CMD_REGMR_MTT_CNT_MASK GENMASK(19, 0)</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_reg_mr_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 cfg0;</div><div class='add'>+	u32 cfg1;</div><div class='add'>+	u64 start_va;</div><div class='add'>+	u32 size;</div><div class='add'>+	u32 cfg2;</div><div class='add'>+	union {</div><div class='add'>+		u64 phy_addr[4];</div><div class='add'>+		struct {</div><div class='add'>+			u64 rsvd;</div><div class='add'>+			u32 size_h;</div><div class='add'>+			u32 mtt_cnt_h;</div><div class='add'>+		};</div><div class='add'>+	};</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_dereg_mr_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 cfg;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* create_av cfg0 */</div><div class='add'>+#define ERDMA_CMD_CREATE_AV_FL_MASK GENMASK(19, 0)</div><div class='add'>+#define ERDMA_CMD_CREATE_AV_NTYPE_MASK BIT(20)</div><div class='add'>+</div><div class='add'>+struct erdma_av_cfg {</div><div class='add'>+	u32 cfg0;</div><div class='add'>+	u8 traffic_class;</div><div class='add'>+	u8 hop_limit;</div><div class='add'>+	u8 sl;</div><div class='add'>+	u8 rsvd;</div><div class='add'>+	u16 udp_sport;</div><div class='add'>+	u16 sgid_index;</div><div class='add'>+	u8 dmac[ETH_ALEN];</div><div class='add'>+	u8 padding[2];</div><div class='add'>+	u8 dgid[ERDMA_ROCEV2_GID_SIZE];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_create_ah_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 pdn;</div><div class='add'>+	u32 ahn;</div><div class='add'>+	struct erdma_av_cfg av_cfg;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_destroy_ah_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 pdn;</div><div class='add'>+	u32 ahn;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* modify qp cfg */</div><div class='add'>+#define ERDMA_CMD_MODIFY_QP_STATE_MASK GENMASK(31, 24)</div><div class='add'>+#define ERDMA_CMD_MODIFY_QP_CC_MASK GENMASK(23, 20)</div><div class='add'>+#define ERDMA_CMD_MODIFY_QP_QPN_MASK GENMASK(19, 0)</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_modify_qp_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 cfg;</div><div class='add'>+	u32 cookie;</div><div class='add'>+	__be32 dip;</div><div class='add'>+	__be32 sip;</div><div class='add'>+	__be16 sport;</div><div class='add'>+	__be16 dport;</div><div class='add'>+	u32 send_nxt;</div><div class='add'>+	u32 recv_nxt;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* modify qp cfg1 for roce device */</div><div class='add'>+#define ERDMA_CMD_MODIFY_QP_DQPN_MASK GENMASK(19, 0)</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_mod_qp_req_rocev2 {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 cfg0;</div><div class='add'>+	u32 cfg1;</div><div class='add'>+	u32 attr_mask;</div><div class='add'>+	u32 qkey;</div><div class='add'>+	u32 rq_psn;</div><div class='add'>+	u32 sq_psn;</div><div class='add'>+	struct erdma_av_cfg av_cfg;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* query qp response mask */</div><div class='add'>+#define ERDMA_CMD_QUERY_QP_RESP_SQ_PSN_MASK GENMASK_ULL(23, 0)</div><div class='add'>+#define ERDMA_CMD_QUERY_QP_RESP_RQ_PSN_MASK GENMASK_ULL(47, 24)</div><div class='add'>+#define ERDMA_CMD_QUERY_QP_RESP_QP_STATE_MASK GENMASK_ULL(55, 48)</div><div class='add'>+#define ERDMA_CMD_QUERY_QP_RESP_SQ_DRAINING_MASK GENMASK_ULL(56, 56)</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_query_qp_req_rocev2 {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 qpn;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum erdma_qp_type {</div><div class='add'>+	ERDMA_QPT_RC = 0,</div><div class='add'>+	ERDMA_QPT_UD = 1,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* create qp cfg0 */</div><div class='add'>+#define ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK GENMASK(31, 20)</div><div class='add'>+#define ERDMA_CMD_CREATE_QP_QPN_MASK GENMASK(19, 0)</div><div class='add'>+</div><div class='add'>+/* create qp cfg1 */</div><div class='add'>+#define ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK GENMASK(31, 20)</div><div class='add'>+#define ERDMA_CMD_CREATE_QP_PD_MASK GENMASK(19, 0)</div><div class='add'>+</div><div class='add'>+/* create qp cfg2 */</div><div class='add'>+#define ERDMA_CMD_CREATE_QP_TYPE_MASK GENMASK(3, 0)</div><div class='add'>+</div><div class='add'>+/* create qp cqn_mtt_cfg */</div><div class='add'>+#define ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK GENMASK(31, 28)</div><div class='add'>+#define ERDMA_CMD_CREATE_QP_DB_CFG_MASK BIT(25)</div><div class='add'>+#define ERDMA_CMD_CREATE_QP_CQN_MASK GENMASK(23, 0)</div><div class='add'>+</div><div class='add'>+/* create qp mtt_cfg */</div><div class='add'>+#define ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK GENMASK(31, 12)</div><div class='add'>+#define ERDMA_CMD_CREATE_QP_MTT_CNT_MASK GENMASK(11, 1)</div><div class='add'>+#define ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK BIT(0)</div><div class='add'>+</div><div class='add'>+/* create qp db cfg */</div><div class='add'>+#define ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK GENMASK(31, 16)</div><div class='add'>+#define ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK GENMASK(15, 0)</div><div class='add'>+</div><div class='add'>+#define ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK GENMASK_ULL(31, 0)</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_create_qp_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 cfg0;</div><div class='add'>+	u32 cfg1;</div><div class='add'>+	u32 sq_cqn_mtt_cfg;</div><div class='add'>+	u32 rq_cqn_mtt_cfg;</div><div class='add'>+	u64 sq_buf_addr;</div><div class='add'>+	u64 rq_buf_addr;</div><div class='add'>+	u32 sq_mtt_cfg;</div><div class='add'>+	u32 rq_mtt_cfg;</div><div class='add'>+	u64 sq_dbrec_dma;</div><div class='add'>+	u64 rq_dbrec_dma;</div><div class='add'>+</div><div class='add'>+	u64 sq_mtt_entry[3];</div><div class='add'>+	u64 rq_mtt_entry[3];</div><div class='add'>+</div><div class='add'>+	u32 db_cfg;</div><div class='add'>+	u32 cfg2;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_destroy_qp_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 qpn;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_reflush_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 qpn;</div><div class='add'>+	u32 sq_pi;</div><div class='add'>+	u32 rq_pi;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define ERDMA_HW_RESP_SIZE 256</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_query_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 rsvd;</div><div class='add'>+	u32 index;</div><div class='add'>+</div><div class='add'>+	u64 target_addr;</div><div class='add'>+	u32 target_length;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define ERDMA_HW_RESP_MAGIC 0x5566</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_query_resp_hdr {</div><div class='add'>+	u16 magic;</div><div class='add'>+	u8 ver;</div><div class='add'>+	u8 length;</div><div class='add'>+</div><div class='add'>+	u32 index;</div><div class='add'>+	u32 rsvd[2];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_query_stats_resp {</div><div class='add'>+	struct erdma_cmdq_query_resp_hdr hdr;</div><div class='add'>+</div><div class='add'>+	u64 tx_req_cnt;</div><div class='add'>+	u64 tx_packets_cnt;</div><div class='add'>+	u64 tx_bytes_cnt;</div><div class='add'>+	u64 tx_drop_packets_cnt;</div><div class='add'>+	u64 tx_bps_meter_drop_packets_cnt;</div><div class='add'>+	u64 tx_pps_meter_drop_packets_cnt;</div><div class='add'>+	u64 rx_packets_cnt;</div><div class='add'>+	u64 rx_bytes_cnt;</div><div class='add'>+	u64 rx_drop_packets_cnt;</div><div class='add'>+	u64 rx_bps_meter_drop_packets_cnt;</div><div class='add'>+	u64 rx_pps_meter_drop_packets_cnt;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum erdma_network_type {</div><div class='add'>+	ERDMA_NETWORK_TYPE_IPV4 = 0,</div><div class='add'>+	ERDMA_NETWORK_TYPE_IPV6 = 1,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum erdma_set_gid_op {</div><div class='add'>+	ERDMA_SET_GID_OP_ADD = 0,</div><div class='add'>+	ERDMA_SET_GID_OP_DEL = 1,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* set gid cfg */</div><div class='add'>+#define ERDMA_CMD_SET_GID_SGID_IDX_MASK GENMASK(15, 0)</div><div class='add'>+#define ERDMA_CMD_SET_GID_NTYPE_MASK BIT(16)</div><div class='add'>+#define ERDMA_CMD_SET_GID_OP_MASK BIT(31)</div><div class='add'>+</div><div class='add'>+struct erdma_cmdq_set_gid_req {</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 cfg;</div><div class='add'>+	u8 gid[ERDMA_ROCEV2_GID_SIZE];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cap qword 0 definition */</div><div class='add'>+#define ERDMA_CMD_DEV_CAP_MAX_GID_MASK GENMASK_ULL(51, 48)</div><div class='add'>+#define ERDMA_CMD_DEV_CAP_MAX_CQE_MASK GENMASK_ULL(47, 40)</div><div class='add'>+#define ERDMA_CMD_DEV_CAP_FLAGS_MASK GENMASK_ULL(31, 24)</div><div class='add'>+#define ERDMA_CMD_DEV_CAP_MAX_RECV_WR_MASK GENMASK_ULL(23, 16)</div><div class='add'>+#define ERDMA_CMD_DEV_CAP_MAX_AH_MASK GENMASK_ULL(15, 8)</div><div class='add'>+#define ERDMA_CMD_DEV_CAP_MAX_MR_SIZE_MASK GENMASK_ULL(7, 0)</div><div class='add'>+</div><div class='add'>+/* cap qword 1 definition */</div><div class='add'>+#define ERDMA_CMD_DEV_CAP_DMA_LOCAL_KEY_MASK GENMASK_ULL(63, 32)</div><div class='add'>+#define ERDMA_CMD_DEV_CAP_DEFAULT_CC_MASK GENMASK_ULL(31, 28)</div><div class='add'>+#define ERDMA_CMD_DEV_CAP_QBLOCK_MASK GENMASK_ULL(27, 16)</div><div class='add'>+#define ERDMA_CMD_DEV_CAP_MAX_MW_MASK GENMASK_ULL(7, 0)</div><div class='add'>+</div><div class='add'>+#define ERDMA_NQP_PER_QBLOCK 1024</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	ERDMA_DEV_CAP_FLAGS_ATOMIC = 1 &lt;&lt; 7,</div><div class='add'>+	ERDMA_DEV_CAP_FLAGS_MTT_VA = 1 &lt;&lt; 5,</div><div class='add'>+	ERDMA_DEV_CAP_FLAGS_EXTEND_DB = 1 &lt;&lt; 3,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define ERDMA_CMD_INFO0_FW_VER_MASK GENMASK_ULL(31, 0)</div><div class='add'>+</div><div class='add'>+/* CQE hdr */</div><div class='add'>+#define ERDMA_CQE_HDR_OWNER_MASK BIT(31)</div><div class='add'>+#define ERDMA_CQE_HDR_OPCODE_MASK GENMASK(23, 16)</div><div class='add'>+#define ERDMA_CQE_HDR_QTYPE_MASK GENMASK(15, 8)</div><div class='add'>+#define ERDMA_CQE_HDR_SYNDROME_MASK GENMASK(7, 0)</div><div class='add'>+</div><div class='add'>+#define ERDMA_CQE_QTYPE_SQ 0</div><div class='add'>+#define ERDMA_CQE_QTYPE_RQ 1</div><div class='add'>+#define ERDMA_CQE_QTYPE_CMDQ 2</div><div class='add'>+</div><div class='add'>+#define ERDMA_CQE_NTYPE_MASK BIT(31)</div><div class='add'>+#define ERDMA_CQE_SL_MASK GENMASK(27, 20)</div><div class='add'>+#define ERDMA_CQE_SQPN_MASK GENMASK(19, 0)</div><div class='add'>+</div><div class='add'>+struct erdma_cqe {</div><div class='add'>+	__be32 hdr;</div><div class='add'>+	__be32 qe_idx;</div><div class='add'>+	__be32 qpn;</div><div class='add'>+	union {</div><div class='add'>+		__le32 imm_data;</div><div class='add'>+		__be32 inv_rkey;</div><div class='add'>+	};</div><div class='add'>+	__be32 size;</div><div class='add'>+	union {</div><div class='add'>+		struct {</div><div class='add'>+			__be32 rsvd[3];</div><div class='add'>+		} rc;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			__be32 rsvd[2];</div><div class='add'>+			__be32 info;</div><div class='add'>+		} ud;</div><div class='add'>+	};</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_sge {</div><div class='add'>+	__aligned_le64 addr;</div><div class='add'>+	__le32 length;</div><div class='add'>+	__le32 key;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* Receive Queue Element */</div><div class='add'>+struct erdma_rqe {</div><div class='add'>+	__le16 qe_idx;</div><div class='add'>+	__le16 rsvd0;</div><div class='add'>+	__le32 qpn;</div><div class='add'>+	__le32 rsvd1;</div><div class='add'>+	__le32 rsvd2;</div><div class='add'>+	__le64 to;</div><div class='add'>+	__le32 length;</div><div class='add'>+	__le32 stag;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* SQE */</div><div class='add'>+#define ERDMA_SQE_HDR_SGL_LEN_MASK GENMASK_ULL(63, 56)</div><div class='add'>+#define ERDMA_SQE_HDR_WQEBB_CNT_MASK GENMASK_ULL(54, 52)</div><div class='add'>+#define ERDMA_SQE_HDR_QPN_MASK GENMASK_ULL(51, 32)</div><div class='add'>+#define ERDMA_SQE_HDR_OPCODE_MASK GENMASK_ULL(31, 27)</div><div class='add'>+#define ERDMA_SQE_HDR_DWQE_MASK BIT_ULL(26)</div><div class='add'>+#define ERDMA_SQE_HDR_INLINE_MASK BIT_ULL(25)</div><div class='add'>+#define ERDMA_SQE_HDR_FENCE_MASK BIT_ULL(24)</div><div class='add'>+#define ERDMA_SQE_HDR_SE_MASK BIT_ULL(23)</div><div class='add'>+#define ERDMA_SQE_HDR_CE_MASK BIT_ULL(22)</div><div class='add'>+#define ERDMA_SQE_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0)</div><div class='add'>+</div><div class='add'>+/* REG MR attrs */</div><div class='add'>+#define ERDMA_SQE_MR_ACCESS_MASK GENMASK(5, 1)</div><div class='add'>+#define ERDMA_SQE_MR_MTT_TYPE_MASK GENMASK(7, 6)</div><div class='add'>+#define ERDMA_SQE_MR_MTT_CNT_MASK GENMASK(31, 12)</div><div class='add'>+</div><div class='add'>+struct erdma_write_sqe {</div><div class='add'>+	__le64 hdr;</div><div class='add'>+	__be32 imm_data;</div><div class='add'>+	__le32 length;</div><div class='add'>+</div><div class='add'>+	__le32 sink_stag;</div><div class='add'>+	__le32 sink_to_l;</div><div class='add'>+	__le32 sink_to_h;</div><div class='add'>+</div><div class='add'>+	__le32 rsvd;</div><div class='add'>+</div><div class='add'>+	struct erdma_sge sgl[];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_send_sqe_rc {</div><div class='add'>+	__le64 hdr;</div><div class='add'>+	union {</div><div class='add'>+		__be32 imm_data;</div><div class='add'>+		__le32 invalid_stag;</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	__le32 length;</div><div class='add'>+	struct erdma_sge sgl[];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_send_sqe_ud {</div><div class='add'>+	__le64 hdr;</div><div class='add'>+	__be32 imm_data;</div><div class='add'>+	__le32 length;</div><div class='add'>+	__le32 qkey;</div><div class='add'>+	__le32 dst_qpn;</div><div class='add'>+	__le32 ahn;</div><div class='add'>+	__le32 rsvd;</div><div class='add'>+	struct erdma_sge sgl[];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_readreq_sqe {</div><div class='add'>+	__le64 hdr;</div><div class='add'>+	__le32 invalid_stag;</div><div class='add'>+	__le32 length;</div><div class='add'>+	__le32 sink_stag;</div><div class='add'>+	__le32 sink_to_l;</div><div class='add'>+	__le32 sink_to_h;</div><div class='add'>+	__le32 rsvd;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_atomic_sqe {</div><div class='add'>+	__le64 hdr;</div><div class='add'>+	__le64 rsvd;</div><div class='add'>+	__le64 fetchadd_swap_data;</div><div class='add'>+	__le64 cmp_data;</div><div class='add'>+</div><div class='add'>+	struct erdma_sge remote;</div><div class='add'>+	struct erdma_sge sgl;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_reg_mr_sqe {</div><div class='add'>+	__le64 hdr;</div><div class='add'>+	__le64 addr;</div><div class='add'>+	__le32 length;</div><div class='add'>+	__le32 stag;</div><div class='add'>+	__le32 attrs;</div><div class='add'>+	__le32 rsvd;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* EQ related. */</div><div class='add'>+#define ERDMA_DEFAULT_EQ_DEPTH 4096</div><div class='add'>+</div><div class='add'>+/* ceqe */</div><div class='add'>+#define ERDMA_CEQE_HDR_DB_MASK BIT_ULL(63)</div><div class='add'>+#define ERDMA_CEQE_HDR_PI_MASK GENMASK_ULL(55, 32)</div><div class='add'>+#define ERDMA_CEQE_HDR_O_MASK BIT_ULL(31)</div><div class='add'>+#define ERDMA_CEQE_HDR_CQN_MASK GENMASK_ULL(19, 0)</div><div class='add'>+</div><div class='add'>+/* aeqe */</div><div class='add'>+#define ERDMA_AEQE_HDR_O_MASK BIT(31)</div><div class='add'>+#define ERDMA_AEQE_HDR_TYPE_MASK GENMASK(23, 16)</div><div class='add'>+#define ERDMA_AEQE_HDR_SUBTYPE_MASK GENMASK(7, 0)</div><div class='add'>+</div><div class='add'>+#define ERDMA_AE_TYPE_QP_FATAL_EVENT 0</div><div class='add'>+#define ERDMA_AE_TYPE_QP_ERQ_ERR_EVENT 1</div><div class='add'>+#define ERDMA_AE_TYPE_ACC_ERR_EVENT 2</div><div class='add'>+#define ERDMA_AE_TYPE_CQ_ERR 3</div><div class='add'>+#define ERDMA_AE_TYPE_OTHER_ERROR 4</div><div class='add'>+</div><div class='add'>+struct erdma_aeqe {</div><div class='add'>+	__le32 hdr;</div><div class='add'>+	__le32 event_data0;</div><div class='add'>+	__le32 event_data1;</div><div class='add'>+	__le32 rsvd;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum erdma_opcode {</div><div class='add'>+	ERDMA_OP_WRITE = 0,</div><div class='add'>+	ERDMA_OP_READ = 1,</div><div class='add'>+	ERDMA_OP_SEND = 2,</div><div class='add'>+	ERDMA_OP_SEND_WITH_IMM = 3,</div><div class='add'>+</div><div class='add'>+	ERDMA_OP_RECEIVE = 4,</div><div class='add'>+	ERDMA_OP_RECV_IMM = 5,</div><div class='add'>+	ERDMA_OP_RECV_INV = 6,</div><div class='add'>+</div><div class='add'>+	ERDMA_OP_RSVD0 = 7,</div><div class='add'>+	ERDMA_OP_RSVD1 = 8,</div><div class='add'>+	ERDMA_OP_WRITE_WITH_IMM = 9,</div><div class='add'>+</div><div class='add'>+	ERDMA_OP_RSVD2 = 10,</div><div class='add'>+	ERDMA_OP_RSVD3 = 11,</div><div class='add'>+</div><div class='add'>+	ERDMA_OP_RSP_SEND_IMM = 12,</div><div class='add'>+	ERDMA_OP_SEND_WITH_INV = 13,</div><div class='add'>+</div><div class='add'>+	ERDMA_OP_REG_MR = 14,</div><div class='add'>+	ERDMA_OP_LOCAL_INV = 15,</div><div class='add'>+	ERDMA_OP_READ_WITH_INV = 16,</div><div class='add'>+	ERDMA_OP_ATOMIC_CAS = 17,</div><div class='add'>+	ERDMA_OP_ATOMIC_FAA = 18,</div><div class='add'>+	ERDMA_NUM_OPCODES = 19,</div><div class='add'>+	ERDMA_OP_INVALID = ERDMA_NUM_OPCODES + 1</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum erdma_wc_status {</div><div class='add'>+	ERDMA_WC_SUCCESS = 0,</div><div class='add'>+	ERDMA_WC_GENERAL_ERR = 1,</div><div class='add'>+	ERDMA_WC_RECV_WQE_FORMAT_ERR = 2,</div><div class='add'>+	ERDMA_WC_RECV_STAG_INVALID_ERR = 3,</div><div class='add'>+	ERDMA_WC_RECV_ADDR_VIOLATION_ERR = 4,</div><div class='add'>+	ERDMA_WC_RECV_RIGHT_VIOLATION_ERR = 5,</div><div class='add'>+	ERDMA_WC_RECV_PDID_ERR = 6,</div><div class='add'>+	ERDMA_WC_RECV_WARRPING_ERR = 7,</div><div class='add'>+	ERDMA_WC_SEND_WQE_FORMAT_ERR = 8,</div><div class='add'>+	ERDMA_WC_SEND_WQE_ORD_EXCEED = 9,</div><div class='add'>+	ERDMA_WC_SEND_STAG_INVALID_ERR = 10,</div><div class='add'>+	ERDMA_WC_SEND_ADDR_VIOLATION_ERR = 11,</div><div class='add'>+	ERDMA_WC_SEND_RIGHT_VIOLATION_ERR = 12,</div><div class='add'>+	ERDMA_WC_SEND_PDID_ERR = 13,</div><div class='add'>+	ERDMA_WC_SEND_WARRPING_ERR = 14,</div><div class='add'>+	ERDMA_WC_FLUSH_ERR = 15,</div><div class='add'>+	ERDMA_WC_RETRY_EXC_ERR = 16,</div><div class='add'>+	ERDMA_NUM_WC_STATUS</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum erdma_vendor_err {</div><div class='add'>+	ERDMA_WC_VENDOR_NO_ERR = 0,</div><div class='add'>+	ERDMA_WC_VENDOR_INVALID_RQE = 1,</div><div class='add'>+	ERDMA_WC_VENDOR_RQE_INVALID_STAG = 2,</div><div class='add'>+	ERDMA_WC_VENDOR_RQE_ADDR_VIOLATION = 3,</div><div class='add'>+	ERDMA_WC_VENDOR_RQE_ACCESS_RIGHT_ERR = 4,</div><div class='add'>+	ERDMA_WC_VENDOR_RQE_INVALID_PD = 5,</div><div class='add'>+	ERDMA_WC_VENDOR_RQE_WRAP_ERR = 6,</div><div class='add'>+	ERDMA_WC_VENDOR_INVALID_SQE = 0x20,</div><div class='add'>+	ERDMA_WC_VENDOR_ZERO_ORD = 0x21,</div><div class='add'>+	ERDMA_WC_VENDOR_SQE_INVALID_STAG = 0x30,</div><div class='add'>+	ERDMA_WC_VENDOR_SQE_ADDR_VIOLATION = 0x31,</div><div class='add'>+	ERDMA_WC_VENDOR_SQE_ACCESS_ERR = 0x32,</div><div class='add'>+	ERDMA_WC_VENDOR_SQE_INVALID_PD = 0x33,</div><div class='add'>+	ERDMA_WC_VENDOR_SQE_WARP_ERR = 0x34</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/erdma/erdma_main.c b/drivers/infiniband/hw/erdma/erdma_main.c<br/>new file mode 100644<br/>index 000000000000..f35b30235018<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/erdma/erdma_main.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/erdma/erdma_main.c</a></div><div class='hunk'>@@ -0,0 +1,684 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='add'>+</div><div class='add'>+/* Authors: Cheng Xu &lt;chengyou@linux.alibaba.com&gt; */</div><div class='add'>+/*          Kai Shen &lt;kaishen@linux.alibaba.com&gt; */</div><div class='add'>+/* Copyright (c) 2020-2022, Alibaba Group. */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/module.h&gt;</div><div class='add'>+#include &lt;net/addrconf.h&gt;</div><div class='add'>+#include &lt;rdma/erdma-abi.h&gt;</div><div class='add'>+</div><div class='add'>+#include "erdma.h"</div><div class='add'>+#include "erdma_cm.h"</div><div class='add'>+#include "erdma_verbs.h"</div><div class='add'>+</div><div class='add'>+MODULE_AUTHOR("Cheng Xu &lt;chengyou@linux.alibaba.com&gt;");</div><div class='add'>+MODULE_DESCRIPTION("Alibaba elasticRDMA adapter driver");</div><div class='add'>+MODULE_LICENSE("Dual BSD/GPL");</div><div class='add'>+</div><div class='add'>+static int erdma_netdev_event(struct notifier_block *nb, unsigned long event,</div><div class='add'>+			      void *arg)</div><div class='add'>+{</div><div class='add'>+	struct net_device *netdev = netdev_notifier_info_to_dev(arg);</div><div class='add'>+	struct erdma_dev *dev = container_of(nb, struct erdma_dev, netdev_nb);</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;netdev == NULL || dev-&gt;netdev != netdev)</div><div class='add'>+		goto done;</div><div class='add'>+</div><div class='add'>+	switch (event) {</div><div class='add'>+	case NETDEV_CHANGEMTU:</div><div class='add'>+		if (dev-&gt;mtu != netdev-&gt;mtu) {</div><div class='add'>+			erdma_set_mtu(dev, netdev-&gt;mtu);</div><div class='add'>+			dev-&gt;mtu = netdev-&gt;mtu;</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	case NETDEV_REGISTER:</div><div class='add'>+	case NETDEV_UNREGISTER:</div><div class='add'>+	case NETDEV_CHANGEADDR:</div><div class='add'>+	case NETDEV_GOING_DOWN:</div><div class='add'>+	case NETDEV_CHANGE:</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+done:</div><div class='add'>+	return NOTIFY_OK;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_enum_and_get_netdev(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct net_device *netdev;</div><div class='add'>+	int ret = -EPROBE_DEFER;</div><div class='add'>+</div><div class='add'>+	/* Already binded to a net_device, so we skip. */</div><div class='add'>+	if (dev-&gt;netdev)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	rtnl_lock();</div><div class='add'>+	for_each_netdev(&amp;init_net, netdev) {</div><div class='add'>+		/*</div><div class='add'>+		 * In erdma, the paired netdev and ibdev should have the same</div><div class='add'>+		 * MAC address. erdma can get the value from its PCIe bar</div><div class='add'>+		 * registers. Since erdma can not get the paired netdev</div><div class='add'>+		 * reference directly, we do a traverse here to get the paired</div><div class='add'>+		 * netdev.</div><div class='add'>+		 */</div><div class='add'>+		if (ether_addr_equal_unaligned(netdev-&gt;perm_addr,</div><div class='add'>+					       dev-&gt;attrs.peer_addr)) {</div><div class='add'>+			ret = ib_device_set_netdev(&amp;dev-&gt;ibdev, netdev, 1);</div><div class='add'>+			if (ret) {</div><div class='add'>+				rtnl_unlock();</div><div class='add'>+				ibdev_warn(&amp;dev-&gt;ibdev,</div><div class='add'>+					   "failed (%d) to link netdev", ret);</div><div class='add'>+				return ret;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			dev-&gt;netdev = netdev;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rtnl_unlock();</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_device_register(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = &amp;dev-&gt;ibdev;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = erdma_enum_and_get_netdev(dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	dev-&gt;mtu = dev-&gt;netdev-&gt;mtu;</div><div class='add'>+	addrconf_addr_eui48((u8 *)&amp;ibdev-&gt;node_guid, dev-&gt;netdev-&gt;dev_addr);</div><div class='add'>+</div><div class='add'>+	ret = ib_register_device(ibdev, "erdma_%d", &amp;dev-&gt;pdev-&gt;dev);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(&amp;dev-&gt;pdev-&gt;dev,</div><div class='add'>+			"ib_register_device failed: ret = %d\n", ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dev-&gt;netdev_nb.notifier_call = erdma_netdev_event;</div><div class='add'>+	ret = register_netdevice_notifier(&amp;dev-&gt;netdev_nb);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(&amp;dev-&gt;ibdev, "failed to register notifier.\n");</div><div class='add'>+		ib_unregister_device(ibdev);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static irqreturn_t erdma_comm_irq_handler(int irq, void *data)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = data;</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_completion_handler(&amp;dev-&gt;cmdq);</div><div class='add'>+	erdma_aeq_event_handler(dev);</div><div class='add'>+</div><div class='add'>+	return IRQ_HANDLED;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_request_vectors(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int expect_irq_num = min(num_possible_cpus() + 1, ERDMA_NUM_MSIX_VEC);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = pci_alloc_irq_vectors(dev-&gt;pdev, 1, expect_irq_num, PCI_IRQ_MSIX);</div><div class='add'>+	if (ret &lt; 0) {</div><div class='add'>+		dev_err(&amp;dev-&gt;pdev-&gt;dev, "request irq vectors failed(%d)\n",</div><div class='add'>+			ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+	dev-&gt;attrs.irq_num = ret;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_comm_irq_init(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	snprintf(dev-&gt;comm_irq.name, ERDMA_IRQNAME_SIZE, "erdma-common@pci:%s",</div><div class='add'>+		 pci_name(dev-&gt;pdev));</div><div class='add'>+	dev-&gt;comm_irq.msix_vector =</div><div class='add'>+		pci_irq_vector(dev-&gt;pdev, ERDMA_MSIX_VECTOR_CMDQ);</div><div class='add'>+</div><div class='add'>+	cpumask_set_cpu(cpumask_first(cpumask_of_pcibus(dev-&gt;pdev-&gt;bus)),</div><div class='add'>+			&amp;dev-&gt;comm_irq.affinity_hint_mask);</div><div class='add'>+	irq_set_affinity_hint(dev-&gt;comm_irq.msix_vector,</div><div class='add'>+			      &amp;dev-&gt;comm_irq.affinity_hint_mask);</div><div class='add'>+</div><div class='add'>+	return request_irq(dev-&gt;comm_irq.msix_vector, erdma_comm_irq_handler, 0,</div><div class='add'>+			   dev-&gt;comm_irq.name, dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_comm_irq_uninit(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	irq_set_affinity_hint(dev-&gt;comm_irq.msix_vector, NULL);</div><div class='add'>+	free_irq(dev-&gt;comm_irq.msix_vector, dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	dev-&gt;proto = erdma_reg_read32(dev, ERDMA_REGS_DEV_PROTO_REG);</div><div class='add'>+</div><div class='add'>+	dev-&gt;resp_pool = dma_pool_create("erdma_resp_pool", &amp;pdev-&gt;dev,</div><div class='add'>+					 ERDMA_HW_RESP_SIZE, ERDMA_HW_RESP_SIZE,</div><div class='add'>+					 0);</div><div class='add'>+	if (!dev-&gt;resp_pool)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	dev-&gt;db_pool = dma_pool_create("erdma_db_pool", &amp;pdev-&gt;dev,</div><div class='add'>+				       ERDMA_DB_SIZE, ERDMA_DB_SIZE, 0);</div><div class='add'>+	if (!dev-&gt;db_pool) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto destroy_resp_pool;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = dma_set_mask_and_coherent(&amp;pdev-&gt;dev,</div><div class='add'>+					DMA_BIT_MASK(ERDMA_PCI_WIDTH));</div><div class='add'>+	if (ret)</div><div class='add'>+		goto destroy_db_pool;</div><div class='add'>+</div><div class='add'>+	dma_set_max_seg_size(&amp;pdev-&gt;dev, UINT_MAX);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+destroy_db_pool:</div><div class='add'>+	dma_pool_destroy(dev-&gt;db_pool);</div><div class='add'>+</div><div class='add'>+destroy_resp_pool:</div><div class='add'>+	dma_pool_destroy(dev-&gt;resp_pool);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_device_uninit(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	dma_pool_destroy(dev-&gt;db_pool);</div><div class='add'>+	dma_pool_destroy(dev-&gt;resp_pool);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_hw_reset(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	u32 ctrl = FIELD_PREP(ERDMA_REG_DEV_CTRL_RESET_MASK, 1);</div><div class='add'>+</div><div class='add'>+	erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, ctrl);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_wait_hw_init_done(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG,</div><div class='add'>+			  FIELD_PREP(ERDMA_REG_DEV_CTRL_INIT_MASK, 1));</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ERDMA_WAIT_DEV_DONE_CNT; i++) {</div><div class='add'>+		if (erdma_reg_read32_filed(dev, ERDMA_REGS_DEV_ST_REG,</div><div class='add'>+					   ERDMA_REG_DEV_ST_INIT_DONE_MASK))</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		msleep(ERDMA_REG_ACCESS_WAIT_MS);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (i == ERDMA_WAIT_DEV_DONE_CNT) {</div><div class='add'>+		dev_err(&amp;dev-&gt;pdev-&gt;dev, "wait init done failed.\n");</div><div class='add'>+		return -ETIMEDOUT;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct pci_device_id erdma_pci_tbl[] = {</div><div class='add'>+	{ PCI_DEVICE(PCI_VENDOR_ID_ALIBABA, 0x107f) },</div><div class='add'>+	{}</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int erdma_probe_dev(struct pci_dev *pdev)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev;</div><div class='add'>+	int bars, err;</div><div class='add'>+	u32 version;</div><div class='add'>+</div><div class='add'>+	err = pci_enable_device(pdev);</div><div class='add'>+	if (err) {</div><div class='add'>+		dev_err(&amp;pdev-&gt;dev, "pci_enable_device failed(%d)\n", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	pci_set_master(pdev);</div><div class='add'>+</div><div class='add'>+	dev = ib_alloc_device(erdma_dev, ibdev);</div><div class='add'>+	if (!dev) {</div><div class='add'>+		dev_err(&amp;pdev-&gt;dev, "ib_alloc_device failed\n");</div><div class='add'>+		err = -ENOMEM;</div><div class='add'>+		goto err_disable_device;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	pci_set_drvdata(pdev, dev);</div><div class='add'>+	dev-&gt;pdev = pdev;</div><div class='add'>+	dev-&gt;attrs.numa_node = dev_to_node(&amp;pdev-&gt;dev);</div><div class='add'>+</div><div class='add'>+	bars = pci_select_bars(pdev, IORESOURCE_MEM);</div><div class='add'>+	err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);</div><div class='add'>+	if (bars != ERDMA_BAR_MASK || err) {</div><div class='add'>+		err = err ? err : -EINVAL;</div><div class='add'>+		goto err_ib_device_release;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dev-&gt;func_bar_addr = pci_resource_start(pdev, ERDMA_FUNC_BAR);</div><div class='add'>+	dev-&gt;func_bar_len = pci_resource_len(pdev, ERDMA_FUNC_BAR);</div><div class='add'>+</div><div class='add'>+	dev-&gt;func_bar =</div><div class='add'>+		devm_ioremap(&amp;pdev-&gt;dev, dev-&gt;func_bar_addr, dev-&gt;func_bar_len);</div><div class='add'>+	if (!dev-&gt;func_bar) {</div><div class='add'>+		dev_err(&amp;pdev-&gt;dev, "devm_ioremap failed.\n");</div><div class='add'>+		err = -EFAULT;</div><div class='add'>+		goto err_release_bars;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	version = erdma_reg_read32(dev, ERDMA_REGS_VERSION_REG);</div><div class='add'>+	if (version == 0) {</div><div class='add'>+		/* we knows that it is a non-functional function. */</div><div class='add'>+		err = -ENODEV;</div><div class='add'>+		goto err_iounmap_func_bar;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = erdma_device_init(dev, pdev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_iounmap_func_bar;</div><div class='add'>+</div><div class='add'>+	err = erdma_request_vectors(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_uninit_device;</div><div class='add'>+</div><div class='add'>+	err = erdma_comm_irq_init(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_free_vectors;</div><div class='add'>+</div><div class='add'>+	err = erdma_aeq_init(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_uninit_comm_irq;</div><div class='add'>+</div><div class='add'>+	err = erdma_cmdq_init(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_uninit_aeq;</div><div class='add'>+</div><div class='add'>+	err = erdma_wait_hw_init_done(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_uninit_cmdq;</div><div class='add'>+</div><div class='add'>+	err = erdma_ceqs_init(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_reset_hw;</div><div class='add'>+</div><div class='add'>+	erdma_finish_cmdq_init(dev);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_reset_hw:</div><div class='add'>+	erdma_hw_reset(dev);</div><div class='add'>+</div><div class='add'>+err_uninit_cmdq:</div><div class='add'>+	erdma_cmdq_destroy(dev);</div><div class='add'>+</div><div class='add'>+err_uninit_aeq:</div><div class='add'>+	erdma_eq_destroy(dev, &amp;dev-&gt;aeq);</div><div class='add'>+</div><div class='add'>+err_uninit_comm_irq:</div><div class='add'>+	erdma_comm_irq_uninit(dev);</div><div class='add'>+</div><div class='add'>+err_free_vectors:</div><div class='add'>+	pci_free_irq_vectors(dev-&gt;pdev);</div><div class='add'>+</div><div class='add'>+err_uninit_device:</div><div class='add'>+	erdma_device_uninit(dev);</div><div class='add'>+</div><div class='add'>+err_iounmap_func_bar:</div><div class='add'>+	devm_iounmap(&amp;pdev-&gt;dev, dev-&gt;func_bar);</div><div class='add'>+</div><div class='add'>+err_release_bars:</div><div class='add'>+	pci_release_selected_regions(pdev, bars);</div><div class='add'>+</div><div class='add'>+err_ib_device_release:</div><div class='add'>+	ib_dealloc_device(&amp;dev-&gt;ibdev);</div><div class='add'>+</div><div class='add'>+err_disable_device:</div><div class='add'>+	pci_disable_device(pdev);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_remove_dev(struct pci_dev *pdev)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = pci_get_drvdata(pdev);</div><div class='add'>+</div><div class='add'>+	erdma_ceqs_uninit(dev);</div><div class='add'>+	erdma_hw_reset(dev);</div><div class='add'>+	erdma_cmdq_destroy(dev);</div><div class='add'>+	erdma_eq_destroy(dev, &amp;dev-&gt;aeq);</div><div class='add'>+	erdma_comm_irq_uninit(dev);</div><div class='add'>+	pci_free_irq_vectors(dev-&gt;pdev);</div><div class='add'>+	erdma_device_uninit(dev);</div><div class='add'>+</div><div class='add'>+	devm_iounmap(&amp;pdev-&gt;dev, dev-&gt;func_bar);</div><div class='add'>+	pci_release_selected_regions(pdev, ERDMA_BAR_MASK);</div><div class='add'>+</div><div class='add'>+	ib_dealloc_device(&amp;dev-&gt;ibdev);</div><div class='add'>+</div><div class='add'>+	pci_disable_device(pdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define ERDMA_GET_CAP(name, cap) FIELD_GET(ERDMA_CMD_DEV_CAP_##name##_MASK, cap)</div><div class='add'>+</div><div class='add'>+static int erdma_dev_attrs_init(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int err;</div><div class='add'>+	u64 req_hdr, cap0, cap1;</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req_hdr, CMDQ_SUBMOD_RDMA,</div><div class='add'>+				CMDQ_OPCODE_QUERY_DEVICE);</div><div class='add'>+</div><div class='add'>+	err = erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req_hdr, sizeof(req_hdr), &amp;cap0,</div><div class='add'>+				  &amp;cap1, true);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	dev-&gt;attrs.max_cqe = 1 &lt;&lt; ERDMA_GET_CAP(MAX_CQE, cap0);</div><div class='add'>+	dev-&gt;attrs.max_mr_size = 1ULL &lt;&lt; ERDMA_GET_CAP(MAX_MR_SIZE, cap0);</div><div class='add'>+	dev-&gt;attrs.max_mw = 1 &lt;&lt; ERDMA_GET_CAP(MAX_MW, cap1);</div><div class='add'>+	dev-&gt;attrs.max_recv_wr = 1 &lt;&lt; ERDMA_GET_CAP(MAX_RECV_WR, cap0);</div><div class='add'>+	dev-&gt;attrs.max_gid = 1 &lt;&lt; ERDMA_GET_CAP(MAX_GID, cap0);</div><div class='add'>+	dev-&gt;attrs.max_ah = 1 &lt;&lt; ERDMA_GET_CAP(MAX_AH, cap0);</div><div class='add'>+	dev-&gt;attrs.local_dma_key = ERDMA_GET_CAP(DMA_LOCAL_KEY, cap1);</div><div class='add'>+	dev-&gt;attrs.cc = ERDMA_GET_CAP(DEFAULT_CC, cap1);</div><div class='add'>+	dev-&gt;attrs.max_qp = ERDMA_NQP_PER_QBLOCK * ERDMA_GET_CAP(QBLOCK, cap1);</div><div class='add'>+	dev-&gt;attrs.max_mr = dev-&gt;attrs.max_qp &lt;&lt; 1;</div><div class='add'>+	dev-&gt;attrs.max_cq = dev-&gt;attrs.max_qp &lt;&lt; 1;</div><div class='add'>+	dev-&gt;attrs.cap_flags = ERDMA_GET_CAP(FLAGS, cap0);</div><div class='add'>+</div><div class='add'>+	dev-&gt;attrs.max_send_wr = ERDMA_MAX_SEND_WR;</div><div class='add'>+	dev-&gt;attrs.max_ord = ERDMA_MAX_ORD;</div><div class='add'>+	dev-&gt;attrs.max_ird = ERDMA_MAX_IRD;</div><div class='add'>+	dev-&gt;attrs.max_send_sge = ERDMA_MAX_SEND_SGE;</div><div class='add'>+	dev-&gt;attrs.max_recv_sge = ERDMA_MAX_RECV_SGE;</div><div class='add'>+	dev-&gt;attrs.max_sge_rd = ERDMA_MAX_SGE_RD;</div><div class='add'>+	dev-&gt;attrs.max_pd = ERDMA_MAX_PD;</div><div class='add'>+</div><div class='add'>+	dev-&gt;res_cb[ERDMA_RES_TYPE_PD].max_cap = ERDMA_MAX_PD;</div><div class='add'>+	dev-&gt;res_cb[ERDMA_RES_TYPE_STAG_IDX].max_cap = dev-&gt;attrs.max_mr;</div><div class='add'>+	dev-&gt;res_cb[ERDMA_RES_TYPE_AH].max_cap = dev-&gt;attrs.max_ah;</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req_hdr, CMDQ_SUBMOD_COMMON,</div><div class='add'>+				CMDQ_OPCODE_QUERY_FW_INFO);</div><div class='add'>+</div><div class='add'>+	err = erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req_hdr, sizeof(req_hdr), &amp;cap0,</div><div class='add'>+				  &amp;cap1, true);</div><div class='add'>+	if (!err)</div><div class='add'>+		dev-&gt;attrs.fw_version =</div><div class='add'>+			FIELD_GET(ERDMA_CMD_INFO0_FW_VER_MASK, cap0);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_device_config(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cmdq_config_device_req req = {};</div><div class='add'>+</div><div class='add'>+	if (!(dev-&gt;attrs.cap_flags &amp; ERDMA_DEV_CAP_FLAGS_EXTEND_DB))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_COMMON,</div><div class='add'>+				CMDQ_OPCODE_CONF_DEVICE);</div><div class='add'>+</div><div class='add'>+	req.cfg = FIELD_PREP(ERDMA_CMD_CONFIG_DEVICE_PGSHIFT_MASK, PAGE_SHIFT) |</div><div class='add'>+		  FIELD_PREP(ERDMA_CMD_CONFIG_DEVICE_PS_EN_MASK, 1);</div><div class='add'>+</div><div class='add'>+	return erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL,</div><div class='add'>+				   true);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_res_cb_init(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int i, j;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ERDMA_RES_CNT; i++) {</div><div class='add'>+		dev-&gt;res_cb[i].next_alloc_idx = 1;</div><div class='add'>+		spin_lock_init(&amp;dev-&gt;res_cb[i].lock);</div><div class='add'>+		dev-&gt;res_cb[i].bitmap =</div><div class='add'>+			bitmap_zalloc(dev-&gt;res_cb[i].max_cap, GFP_KERNEL);</div><div class='add'>+		if (!dev-&gt;res_cb[i].bitmap)</div><div class='add'>+			goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	for (j = 0; j &lt; i; j++)</div><div class='add'>+		bitmap_free(dev-&gt;res_cb[j].bitmap);</div><div class='add'>+</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_res_cb_free(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ERDMA_RES_CNT; i++)</div><div class='add'>+		bitmap_free(dev-&gt;res_cb[i].bitmap);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops erdma_device_ops_rocev2 = {</div><div class='add'>+	.get_link_layer = erdma_get_link_layer,</div><div class='add'>+	.add_gid = erdma_add_gid,</div><div class='add'>+	.del_gid = erdma_del_gid,</div><div class='add'>+	.query_pkey = erdma_query_pkey,</div><div class='add'>+	.create_ah = erdma_create_ah,</div><div class='add'>+	.destroy_ah = erdma_destroy_ah,</div><div class='add'>+	.query_ah = erdma_query_ah,</div><div class='add'>+</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_ah, erdma_ah, ibah),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops erdma_device_ops_iwarp = {</div><div class='add'>+	.iw_accept = erdma_accept,</div><div class='add'>+	.iw_add_ref = erdma_qp_get_ref,</div><div class='add'>+	.iw_connect = erdma_connect,</div><div class='add'>+	.iw_create_listen = erdma_create_listen,</div><div class='add'>+	.iw_destroy_listen = erdma_destroy_listen,</div><div class='add'>+	.iw_get_qp = erdma_get_ibqp,</div><div class='add'>+	.iw_reject = erdma_reject,</div><div class='add'>+	.iw_rem_ref = erdma_qp_put_ref,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops erdma_device_ops = {</div><div class='add'>+	.owner = THIS_MODULE,</div><div class='add'>+	.driver_id = RDMA_DRIVER_ERDMA,</div><div class='add'>+	.uverbs_abi_ver = ERDMA_ABI_VERSION,</div><div class='add'>+</div><div class='add'>+	.alloc_hw_port_stats = erdma_alloc_hw_port_stats,</div><div class='add'>+	.alloc_mr = erdma_ib_alloc_mr,</div><div class='add'>+	.alloc_pd = erdma_alloc_pd,</div><div class='add'>+	.alloc_ucontext = erdma_alloc_ucontext,</div><div class='add'>+	.create_cq = erdma_create_cq,</div><div class='add'>+	.create_qp = erdma_create_qp,</div><div class='add'>+	.dealloc_pd = erdma_dealloc_pd,</div><div class='add'>+	.dealloc_ucontext = erdma_dealloc_ucontext,</div><div class='add'>+	.dereg_mr = erdma_dereg_mr,</div><div class='add'>+	.destroy_cq = erdma_destroy_cq,</div><div class='add'>+	.destroy_qp = erdma_destroy_qp,</div><div class='add'>+	.disassociate_ucontext = erdma_disassociate_ucontext,</div><div class='add'>+	.get_dma_mr = erdma_get_dma_mr,</div><div class='add'>+	.get_hw_stats = erdma_get_hw_stats,</div><div class='add'>+	.get_port_immutable = erdma_get_port_immutable,</div><div class='add'>+	.map_mr_sg = erdma_map_mr_sg,</div><div class='add'>+	.mmap = erdma_mmap,</div><div class='add'>+	.mmap_free = erdma_mmap_free,</div><div class='add'>+	.post_recv = erdma_post_recv,</div><div class='add'>+	.post_send = erdma_post_send,</div><div class='add'>+	.poll_cq = erdma_poll_cq,</div><div class='add'>+	.query_device = erdma_query_device,</div><div class='add'>+	.query_gid = erdma_query_gid,</div><div class='add'>+	.query_port = erdma_query_port,</div><div class='add'>+	.query_qp = erdma_query_qp,</div><div class='add'>+	.req_notify_cq = erdma_req_notify_cq,</div><div class='add'>+	.reg_user_mr = erdma_reg_user_mr,</div><div class='add'>+	.modify_qp = erdma_modify_qp,</div><div class='add'>+</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_cq, erdma_cq, ibcq),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_pd, erdma_pd, ibpd),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_ucontext, erdma_ucontext, ibucontext),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_qp, erdma_qp, ibqp),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int erdma_ib_device_add(struct pci_dev *pdev)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = pci_get_drvdata(pdev);</div><div class='add'>+	struct ib_device *ibdev = &amp;dev-&gt;ibdev;</div><div class='add'>+	u64 mac;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = erdma_dev_attrs_init(dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = erdma_device_config(dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (erdma_device_iwarp(dev)) {</div><div class='add'>+		ibdev-&gt;node_type = RDMA_NODE_RNIC;</div><div class='add'>+		ib_set_device_ops(ibdev, &amp;erdma_device_ops_iwarp);</div><div class='add'>+	} else {</div><div class='add'>+		ibdev-&gt;node_type = RDMA_NODE_IB_CA;</div><div class='add'>+		ib_set_device_ops(ibdev, &amp;erdma_device_ops_rocev2);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	memcpy(ibdev-&gt;node_desc, ERDMA_NODE_DESC, sizeof(ERDMA_NODE_DESC));</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Current model (one-to-one device association):</div><div class='add'>+	 * One ERDMA device per net_device or, equivalently,</div><div class='add'>+	 * per physical port.</div><div class='add'>+	 */</div><div class='add'>+	ibdev-&gt;phys_port_cnt = 1;</div><div class='add'>+	ibdev-&gt;num_comp_vectors = dev-&gt;attrs.irq_num - 1;</div><div class='add'>+</div><div class='add'>+	ib_set_device_ops(ibdev, &amp;erdma_device_ops);</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;dev-&gt;cep_list);</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;dev-&gt;lock);</div><div class='add'>+	xa_init_flags(&amp;dev-&gt;qp_xa, XA_FLAGS_ALLOC1);</div><div class='add'>+	xa_init_flags(&amp;dev-&gt;cq_xa, XA_FLAGS_ALLOC1);</div><div class='add'>+	dev-&gt;next_alloc_cqn = 1;</div><div class='add'>+	dev-&gt;next_alloc_qpn = 1;</div><div class='add'>+</div><div class='add'>+	ret = erdma_res_cb_init(dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	atomic_set(&amp;dev-&gt;num_ctx, 0);</div><div class='add'>+</div><div class='add'>+	mac = erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_L_REG);</div><div class='add'>+	mac |= (u64)erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_H_REG) &lt;&lt; 32;</div><div class='add'>+</div><div class='add'>+	u64_to_ether_addr(mac, dev-&gt;attrs.peer_addr);</div><div class='add'>+</div><div class='add'>+	dev-&gt;reflush_wq = alloc_workqueue("erdma-reflush-wq", WQ_UNBOUND,</div><div class='add'>+					  WQ_UNBOUND_MAX_ACTIVE);</div><div class='add'>+	if (!dev-&gt;reflush_wq) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err_alloc_workqueue;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = erdma_device_register(dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_register;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_register:</div><div class='add'>+	destroy_workqueue(dev-&gt;reflush_wq);</div><div class='add'>+err_alloc_workqueue:</div><div class='add'>+	xa_destroy(&amp;dev-&gt;qp_xa);</div><div class='add'>+	xa_destroy(&amp;dev-&gt;cq_xa);</div><div class='add'>+</div><div class='add'>+	erdma_res_cb_free(dev);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_ib_device_remove(struct pci_dev *pdev)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = pci_get_drvdata(pdev);</div><div class='add'>+</div><div class='add'>+	unregister_netdevice_notifier(&amp;dev-&gt;netdev_nb);</div><div class='add'>+	ib_unregister_device(&amp;dev-&gt;ibdev);</div><div class='add'>+</div><div class='add'>+	destroy_workqueue(dev-&gt;reflush_wq);</div><div class='add'>+	erdma_res_cb_free(dev);</div><div class='add'>+	xa_destroy(&amp;dev-&gt;qp_xa);</div><div class='add'>+	xa_destroy(&amp;dev-&gt;cq_xa);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_probe(struct pci_dev *pdev, const struct pci_device_id *ent)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = erdma_probe_dev(pdev);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = erdma_ib_device_add(pdev);</div><div class='add'>+	if (ret) {</div><div class='add'>+		erdma_remove_dev(pdev);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_remove(struct pci_dev *pdev)</div><div class='add'>+{</div><div class='add'>+	erdma_ib_device_remove(pdev);</div><div class='add'>+	erdma_remove_dev(pdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct pci_driver erdma_pci_driver = {</div><div class='add'>+	.name = DRV_MODULE_NAME,</div><div class='add'>+	.id_table = erdma_pci_tbl,</div><div class='add'>+	.probe = erdma_probe,</div><div class='add'>+	.remove = erdma_remove</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+MODULE_DEVICE_TABLE(pci, erdma_pci_tbl);</div><div class='add'>+</div><div class='add'>+static __init int erdma_init_module(void)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = erdma_cm_init();</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = pci_register_driver(&amp;erdma_pci_driver);</div><div class='add'>+	if (ret)</div><div class='add'>+		erdma_cm_exit();</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void __exit erdma_exit_module(void)</div><div class='add'>+{</div><div class='add'>+	pci_unregister_driver(&amp;erdma_pci_driver);</div><div class='add'>+</div><div class='add'>+	erdma_cm_exit();</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+module_init(erdma_init_module);</div><div class='add'>+module_exit(erdma_exit_module);</div><div class='head'>diff --git a/drivers/infiniband/hw/erdma/erdma_qp.c b/drivers/infiniband/hw/erdma/erdma_qp.c<br/>new file mode 100644<br/>index 000000000000..25f6c49aec77<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/erdma/erdma_qp.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/erdma/erdma_qp.c</a></div><div class='hunk'>@@ -0,0 +1,757 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='add'>+</div><div class='add'>+/* Authors: Cheng Xu &lt;chengyou@linux.alibaba.com&gt; */</div><div class='add'>+/*          Kai Shen &lt;kaishen@linux.alibaba.com&gt; */</div><div class='add'>+/* Copyright (c) 2020-2021, Alibaba Group */</div><div class='add'>+/* Authors: Bernard Metzler &lt;bmt@zurich.ibm.com&gt; */</div><div class='add'>+/* Copyright (c) 2008-2019, IBM Corporation */</div><div class='add'>+</div><div class='add'>+#include "erdma_cm.h"</div><div class='add'>+#include "erdma_verbs.h"</div><div class='add'>+</div><div class='add'>+void erdma_qp_llp_close(struct erdma_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct erdma_mod_qp_params_iwarp params;</div><div class='add'>+</div><div class='add'>+	down_write(&amp;qp-&gt;state_lock);</div><div class='add'>+</div><div class='add'>+	switch (qp-&gt;attrs.iwarp.state) {</div><div class='add'>+	case ERDMA_QPS_IWARP_RTS:</div><div class='add'>+	case ERDMA_QPS_IWARP_RTR:</div><div class='add'>+	case ERDMA_QPS_IWARP_IDLE:</div><div class='add'>+	case ERDMA_QPS_IWARP_TERMINATE:</div><div class='add'>+		params.state = ERDMA_QPS_IWARP_CLOSING;</div><div class='add'>+		erdma_modify_qp_state_iwarp(qp, &amp;params, ERDMA_QPA_IWARP_STATE);</div><div class='add'>+		break;</div><div class='add'>+	case ERDMA_QPS_IWARP_CLOSING:</div><div class='add'>+		qp-&gt;attrs.iwarp.state = ERDMA_QPS_IWARP_IDLE;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;cep) {</div><div class='add'>+		erdma_cep_put(qp-&gt;cep);</div><div class='add'>+		qp-&gt;cep = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	up_write(&amp;qp-&gt;state_lock);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_qp *erdma_get_ibqp(struct ib_device *ibdev, int id)</div><div class='add'>+{</div><div class='add'>+	struct erdma_qp *qp = find_qp_by_qpn(to_edev(ibdev), id);</div><div class='add'>+</div><div class='add'>+	if (qp)</div><div class='add'>+		return &amp;qp-&gt;ibqp;</div><div class='add'>+</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int</div><div class='add'>+erdma_modify_qp_state_to_rts(struct erdma_qp *qp,</div><div class='add'>+			     struct erdma_mod_qp_params_iwarp *params,</div><div class='add'>+			     enum erdma_qpa_mask_iwarp mask)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+	struct erdma_dev *dev = qp-&gt;dev;</div><div class='add'>+	struct erdma_cmdq_modify_qp_req req;</div><div class='add'>+	struct tcp_sock *tp;</div><div class='add'>+	struct erdma_cep *cep = qp-&gt;cep;</div><div class='add'>+	struct sockaddr_storage local_addr, remote_addr;</div><div class='add'>+</div><div class='add'>+	if (!(mask &amp; ERDMA_QPA_IWARP_LLP_HANDLE))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!(mask &amp; ERDMA_QPA_IWARP_MPA))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!(mask &amp; ERDMA_QPA_IWARP_CC))</div><div class='add'>+		params-&gt;cc = qp-&gt;attrs.cc;</div><div class='add'>+</div><div class='add'>+	ret = getname_local(cep-&gt;sock, &amp;local_addr);</div><div class='add'>+	if (ret &lt; 0)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = getname_peer(cep-&gt;sock, &amp;remote_addr);</div><div class='add'>+	if (ret &lt; 0)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	tp = tcp_sk(qp-&gt;cep-&gt;sock-&gt;sk);</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_RDMA,</div><div class='add'>+				CMDQ_OPCODE_MODIFY_QP);</div><div class='add'>+</div><div class='add'>+	req.cfg = FIELD_PREP(ERDMA_CMD_MODIFY_QP_STATE_MASK, params-&gt;state) |</div><div class='add'>+		  FIELD_PREP(ERDMA_CMD_MODIFY_QP_CC_MASK, params-&gt;cc) |</div><div class='add'>+		  FIELD_PREP(ERDMA_CMD_MODIFY_QP_QPN_MASK, QP_ID(qp));</div><div class='add'>+</div><div class='add'>+	req.cookie = be32_to_cpu(cep-&gt;mpa.ext_data.cookie);</div><div class='add'>+	req.dip = to_sockaddr_in(remote_addr).sin_addr.s_addr;</div><div class='add'>+	req.sip = to_sockaddr_in(local_addr).sin_addr.s_addr;</div><div class='add'>+	req.dport = to_sockaddr_in(remote_addr).sin_port;</div><div class='add'>+	req.sport = to_sockaddr_in(local_addr).sin_port;</div><div class='add'>+</div><div class='add'>+	req.send_nxt = tp-&gt;snd_nxt;</div><div class='add'>+	/* rsvd tcp seq for mpa-rsp in server. */</div><div class='add'>+	if (params-&gt;qp_type == ERDMA_QP_PASSIVE)</div><div class='add'>+		req.send_nxt += MPA_DEFAULT_HDR_LEN + params-&gt;pd_len;</div><div class='add'>+	req.recv_nxt = tp-&gt;rcv_nxt;</div><div class='add'>+</div><div class='add'>+	ret = erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL,</div><div class='add'>+				  true);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (mask &amp; ERDMA_QPA_IWARP_IRD)</div><div class='add'>+		qp-&gt;attrs.irq_size = params-&gt;irq_size;</div><div class='add'>+</div><div class='add'>+	if (mask &amp; ERDMA_QPA_IWARP_ORD)</div><div class='add'>+		qp-&gt;attrs.orq_size = params-&gt;orq_size;</div><div class='add'>+</div><div class='add'>+	if (mask &amp; ERDMA_QPA_IWARP_CC)</div><div class='add'>+		qp-&gt;attrs.cc = params-&gt;cc;</div><div class='add'>+</div><div class='add'>+	qp-&gt;attrs.iwarp.state = ERDMA_QPS_IWARP_RTS;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int</div><div class='add'>+erdma_modify_qp_state_to_stop(struct erdma_qp *qp,</div><div class='add'>+			      struct erdma_mod_qp_params_iwarp *params,</div><div class='add'>+			      enum erdma_qpa_mask_iwarp mask)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = qp-&gt;dev;</div><div class='add'>+	struct erdma_cmdq_modify_qp_req req;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_RDMA,</div><div class='add'>+				CMDQ_OPCODE_MODIFY_QP);</div><div class='add'>+</div><div class='add'>+	req.cfg = FIELD_PREP(ERDMA_CMD_MODIFY_QP_STATE_MASK, params-&gt;state) |</div><div class='add'>+		  FIELD_PREP(ERDMA_CMD_MODIFY_QP_QPN_MASK, QP_ID(qp));</div><div class='add'>+</div><div class='add'>+	ret = erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL,</div><div class='add'>+				  true);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	qp-&gt;attrs.iwarp.state = params-&gt;state;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_modify_qp_state_iwarp(struct erdma_qp *qp,</div><div class='add'>+				struct erdma_mod_qp_params_iwarp *params,</div><div class='add'>+				int mask)</div><div class='add'>+{</div><div class='add'>+	bool need_reflush = false;</div><div class='add'>+	int drop_conn, ret = 0;</div><div class='add'>+</div><div class='add'>+	if (!mask)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (!(mask &amp; ERDMA_QPA_IWARP_STATE))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	switch (qp-&gt;attrs.iwarp.state) {</div><div class='add'>+	case ERDMA_QPS_IWARP_IDLE:</div><div class='add'>+	case ERDMA_QPS_IWARP_RTR:</div><div class='add'>+		if (params-&gt;state == ERDMA_QPS_IWARP_RTS) {</div><div class='add'>+			ret = erdma_modify_qp_state_to_rts(qp, params, mask);</div><div class='add'>+		} else if (params-&gt;state == ERDMA_QPS_IWARP_ERROR) {</div><div class='add'>+			qp-&gt;attrs.iwarp.state = ERDMA_QPS_IWARP_ERROR;</div><div class='add'>+			need_reflush = true;</div><div class='add'>+			if (qp-&gt;cep) {</div><div class='add'>+				erdma_cep_put(qp-&gt;cep);</div><div class='add'>+				qp-&gt;cep = NULL;</div><div class='add'>+			}</div><div class='add'>+			ret = erdma_modify_qp_state_to_stop(qp, params, mask);</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	case ERDMA_QPS_IWARP_RTS:</div><div class='add'>+		drop_conn = 0;</div><div class='add'>+</div><div class='add'>+		if (params-&gt;state == ERDMA_QPS_IWARP_CLOSING ||</div><div class='add'>+		    params-&gt;state == ERDMA_QPS_IWARP_TERMINATE ||</div><div class='add'>+		    params-&gt;state == ERDMA_QPS_IWARP_ERROR) {</div><div class='add'>+			ret = erdma_modify_qp_state_to_stop(qp, params, mask);</div><div class='add'>+			drop_conn = 1;</div><div class='add'>+			need_reflush = true;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (drop_conn)</div><div class='add'>+			erdma_qp_cm_drop(qp);</div><div class='add'>+</div><div class='add'>+		break;</div><div class='add'>+	case ERDMA_QPS_IWARP_TERMINATE:</div><div class='add'>+		if (params-&gt;state == ERDMA_QPS_IWARP_ERROR)</div><div class='add'>+			qp-&gt;attrs.iwarp.state = ERDMA_QPS_IWARP_ERROR;</div><div class='add'>+		break;</div><div class='add'>+	case ERDMA_QPS_IWARP_CLOSING:</div><div class='add'>+		if (params-&gt;state == ERDMA_QPS_IWARP_IDLE) {</div><div class='add'>+			qp-&gt;attrs.iwarp.state = ERDMA_QPS_IWARP_IDLE;</div><div class='add'>+		} else if (params-&gt;state == ERDMA_QPS_IWARP_ERROR) {</div><div class='add'>+			ret = erdma_modify_qp_state_to_stop(qp, params, mask);</div><div class='add'>+			qp-&gt;attrs.iwarp.state = ERDMA_QPS_IWARP_ERROR;</div><div class='add'>+		} else if (params-&gt;state != ERDMA_QPS_IWARP_CLOSING) {</div><div class='add'>+			return -ECONNABORTED;</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (need_reflush &amp;&amp; !ret &amp;&amp; rdma_is_kernel_res(&amp;qp-&gt;ibqp.res)) {</div><div class='add'>+		qp-&gt;flags |= ERDMA_QP_IN_FLUSHING;</div><div class='add'>+		mod_delayed_work(qp-&gt;dev-&gt;reflush_wq, &amp;qp-&gt;reflush_dwork,</div><div class='add'>+				 usecs_to_jiffies(100));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int modify_qp_cmd_rocev2(struct erdma_qp *qp,</div><div class='add'>+				struct erdma_mod_qp_params_rocev2 *params,</div><div class='add'>+				enum erdma_qpa_mask_rocev2 attr_mask)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cmdq_mod_qp_req_rocev2 req;</div><div class='add'>+</div><div class='add'>+	memset(&amp;req, 0, sizeof(req));</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_RDMA,</div><div class='add'>+				CMDQ_OPCODE_MODIFY_QP);</div><div class='add'>+</div><div class='add'>+	req.cfg0 = FIELD_PREP(ERDMA_CMD_MODIFY_QP_QPN_MASK, QP_ID(qp));</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; ERDMA_QPA_ROCEV2_STATE)</div><div class='add'>+		req.cfg0 |= FIELD_PREP(ERDMA_CMD_MODIFY_QP_STATE_MASK,</div><div class='add'>+				       params-&gt;state);</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; ERDMA_QPA_ROCEV2_DST_QPN)</div><div class='add'>+		req.cfg1 = FIELD_PREP(ERDMA_CMD_MODIFY_QP_DQPN_MASK,</div><div class='add'>+				      params-&gt;dst_qpn);</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; ERDMA_QPA_ROCEV2_QKEY)</div><div class='add'>+		req.qkey = params-&gt;qkey;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; ERDMA_QPA_ROCEV2_AV)</div><div class='add'>+		erdma_set_av_cfg(&amp;req.av_cfg, &amp;params-&gt;av);</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; ERDMA_QPA_ROCEV2_SQ_PSN)</div><div class='add'>+		req.sq_psn = params-&gt;sq_psn;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; ERDMA_QPA_ROCEV2_RQ_PSN)</div><div class='add'>+		req.rq_psn = params-&gt;rq_psn;</div><div class='add'>+</div><div class='add'>+	req.attr_mask = attr_mask;</div><div class='add'>+</div><div class='add'>+	return erdma_post_cmd_wait(&amp;qp-&gt;dev-&gt;cmdq, &amp;req, sizeof(req), NULL,</div><div class='add'>+				   NULL, true);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_reset_qp(struct erdma_qp *qp)</div><div class='add'>+{</div><div class='add'>+	qp-&gt;kern_qp.sq_pi = 0;</div><div class='add'>+	qp-&gt;kern_qp.sq_ci = 0;</div><div class='add'>+	qp-&gt;kern_qp.rq_pi = 0;</div><div class='add'>+	qp-&gt;kern_qp.rq_ci = 0;</div><div class='add'>+	memset(qp-&gt;kern_qp.swr_tbl, 0, qp-&gt;attrs.sq_size * sizeof(u64));</div><div class='add'>+	memset(qp-&gt;kern_qp.rwr_tbl, 0, qp-&gt;attrs.rq_size * sizeof(u64));</div><div class='add'>+	memset(qp-&gt;kern_qp.sq_buf, 0, qp-&gt;attrs.sq_size &lt;&lt; SQEBB_SHIFT);</div><div class='add'>+	memset(qp-&gt;kern_qp.rq_buf, 0, qp-&gt;attrs.rq_size &lt;&lt; RQE_SHIFT);</div><div class='add'>+	erdma_remove_cqes_of_qp(&amp;qp-&gt;scq-&gt;ibcq, QP_ID(qp));</div><div class='add'>+	if (qp-&gt;rcq != qp-&gt;scq)</div><div class='add'>+		erdma_remove_cqes_of_qp(&amp;qp-&gt;rcq-&gt;ibcq, QP_ID(qp));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_modify_qp_state_rocev2(struct erdma_qp *qp,</div><div class='add'>+				 struct erdma_mod_qp_params_rocev2 *params,</div><div class='add'>+				 int attr_mask)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = to_edev(qp-&gt;ibqp.device);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = modify_qp_cmd_rocev2(qp, params, attr_mask);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; ERDMA_QPA_ROCEV2_STATE)</div><div class='add'>+		qp-&gt;attrs.rocev2.state = params-&gt;state;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; ERDMA_QPA_ROCEV2_QKEY)</div><div class='add'>+		qp-&gt;attrs.rocev2.qkey = params-&gt;qkey;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; ERDMA_QPA_ROCEV2_DST_QPN)</div><div class='add'>+		qp-&gt;attrs.rocev2.dst_qpn = params-&gt;dst_qpn;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; ERDMA_QPA_ROCEV2_AV)</div><div class='add'>+		memcpy(&amp;qp-&gt;attrs.rocev2.av, &amp;params-&gt;av,</div><div class='add'>+		       sizeof(struct erdma_av));</div><div class='add'>+</div><div class='add'>+	if (rdma_is_kernel_res(&amp;qp-&gt;ibqp.res) &amp;&amp;</div><div class='add'>+	    params-&gt;state == ERDMA_QPS_ROCEV2_RESET)</div><div class='add'>+		erdma_reset_qp(qp);</div><div class='add'>+</div><div class='add'>+	if (rdma_is_kernel_res(&amp;qp-&gt;ibqp.res) &amp;&amp;</div><div class='add'>+	    params-&gt;state == ERDMA_QPS_ROCEV2_ERROR) {</div><div class='add'>+		qp-&gt;flags |= ERDMA_QP_IN_FLUSHING;</div><div class='add'>+		mod_delayed_work(dev-&gt;reflush_wq, &amp;qp-&gt;reflush_dwork,</div><div class='add'>+				 usecs_to_jiffies(100));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_qp_safe_free(struct kref *ref)</div><div class='add'>+{</div><div class='add'>+	struct erdma_qp *qp = container_of(ref, struct erdma_qp, ref);</div><div class='add'>+</div><div class='add'>+	complete(&amp;qp-&gt;safe_free);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_qp_put(struct erdma_qp *qp)</div><div class='add'>+{</div><div class='add'>+	WARN_ON(kref_read(&amp;qp-&gt;ref) &lt; 1);</div><div class='add'>+	kref_put(&amp;qp-&gt;ref, erdma_qp_safe_free);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_qp_get(struct erdma_qp *qp)</div><div class='add'>+{</div><div class='add'>+	kref_get(&amp;qp-&gt;ref);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_inline_data(struct erdma_qp *qp,</div><div class='add'>+			    const struct ib_send_wr *send_wr, u16 wqe_idx,</div><div class='add'>+			    u32 sgl_offset, __le32 *length_field)</div><div class='add'>+{</div><div class='add'>+	u32 remain_size, copy_size, data_off, bytes = 0;</div><div class='add'>+	char *data;</div><div class='add'>+	int i = 0;</div><div class='add'>+</div><div class='add'>+	wqe_idx += (sgl_offset &gt;&gt; SQEBB_SHIFT);</div><div class='add'>+	sgl_offset &amp;= (SQEBB_SIZE - 1);</div><div class='add'>+	data = get_queue_entry(qp-&gt;kern_qp.sq_buf, wqe_idx, qp-&gt;attrs.sq_size,</div><div class='add'>+			       SQEBB_SHIFT);</div><div class='add'>+</div><div class='add'>+	while (i &lt; send_wr-&gt;num_sge) {</div><div class='add'>+		bytes += send_wr-&gt;sg_list[i].length;</div><div class='add'>+		if (bytes &gt; (int)ERDMA_MAX_INLINE)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		remain_size = send_wr-&gt;sg_list[i].length;</div><div class='add'>+		data_off = 0;</div><div class='add'>+</div><div class='add'>+		while (1) {</div><div class='add'>+			copy_size = min(remain_size, SQEBB_SIZE - sgl_offset);</div><div class='add'>+</div><div class='add'>+			memcpy(data + sgl_offset,</div><div class='add'>+			       (void *)(uintptr_t)send_wr-&gt;sg_list[i].addr +</div><div class='add'>+				       data_off,</div><div class='add'>+			       copy_size);</div><div class='add'>+			remain_size -= copy_size;</div><div class='add'>+			data_off += copy_size;</div><div class='add'>+			sgl_offset += copy_size;</div><div class='add'>+			wqe_idx += (sgl_offset &gt;&gt; SQEBB_SHIFT);</div><div class='add'>+			sgl_offset &amp;= (SQEBB_SIZE - 1);</div><div class='add'>+</div><div class='add'>+			data = get_queue_entry(qp-&gt;kern_qp.sq_buf, wqe_idx,</div><div class='add'>+					       qp-&gt;attrs.sq_size, SQEBB_SHIFT);</div><div class='add'>+			if (!remain_size)</div><div class='add'>+				break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		i++;</div><div class='add'>+	}</div><div class='add'>+	*length_field = cpu_to_le32(bytes);</div><div class='add'>+</div><div class='add'>+	return bytes;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_sgl(struct erdma_qp *qp, const struct ib_send_wr *send_wr,</div><div class='add'>+		    u16 wqe_idx, u32 sgl_offset, __le32 *length_field)</div><div class='add'>+{</div><div class='add'>+	int i = 0;</div><div class='add'>+	u32 bytes = 0;</div><div class='add'>+	char *sgl;</div><div class='add'>+</div><div class='add'>+	if (send_wr-&gt;num_sge &gt; qp-&gt;dev-&gt;attrs.max_send_sge)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (sgl_offset &amp; 0xF)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	while (i &lt; send_wr-&gt;num_sge) {</div><div class='add'>+		wqe_idx += (sgl_offset &gt;&gt; SQEBB_SHIFT);</div><div class='add'>+		sgl_offset &amp;= (SQEBB_SIZE - 1);</div><div class='add'>+		sgl = get_queue_entry(qp-&gt;kern_qp.sq_buf, wqe_idx,</div><div class='add'>+				      qp-&gt;attrs.sq_size, SQEBB_SHIFT);</div><div class='add'>+</div><div class='add'>+		bytes += send_wr-&gt;sg_list[i].length;</div><div class='add'>+		memcpy(sgl + sgl_offset, &amp;send_wr-&gt;sg_list[i],</div><div class='add'>+		       sizeof(struct ib_sge));</div><div class='add'>+</div><div class='add'>+		sgl_offset += sizeof(struct ib_sge);</div><div class='add'>+		i++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*length_field = cpu_to_le32(bytes);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void init_send_sqe_rc(struct erdma_qp *qp, struct erdma_send_sqe_rc *sqe,</div><div class='add'>+			     const struct ib_send_wr *wr, u32 *hw_op)</div><div class='add'>+{</div><div class='add'>+	u32 op = ERDMA_OP_SEND;</div><div class='add'>+</div><div class='add'>+	if (wr-&gt;opcode == IB_WR_SEND_WITH_IMM) {</div><div class='add'>+		op = ERDMA_OP_SEND_WITH_IMM;</div><div class='add'>+		sqe-&gt;imm_data = wr-&gt;ex.imm_data;</div><div class='add'>+	} else if (wr-&gt;opcode == IB_WR_SEND_WITH_INV) {</div><div class='add'>+		op = ERDMA_OP_SEND_WITH_INV;</div><div class='add'>+		sqe-&gt;invalid_stag = cpu_to_le32(wr-&gt;ex.invalidate_rkey);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*hw_op = op;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void init_send_sqe_ud(struct erdma_qp *qp, struct erdma_send_sqe_ud *sqe,</div><div class='add'>+			     const struct ib_send_wr *wr, u32 *hw_op)</div><div class='add'>+{</div><div class='add'>+	const struct ib_ud_wr *uwr = ud_wr(wr);</div><div class='add'>+	struct erdma_ah *ah = to_eah(uwr-&gt;ah);</div><div class='add'>+	u32 op = ERDMA_OP_SEND;</div><div class='add'>+</div><div class='add'>+	if (wr-&gt;opcode == IB_WR_SEND_WITH_IMM) {</div><div class='add'>+		op = ERDMA_OP_SEND_WITH_IMM;</div><div class='add'>+		sqe-&gt;imm_data = wr-&gt;ex.imm_data;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*hw_op = op;</div><div class='add'>+</div><div class='add'>+	sqe-&gt;ahn = cpu_to_le32(ah-&gt;ahn);</div><div class='add'>+	sqe-&gt;dst_qpn = cpu_to_le32(uwr-&gt;remote_qpn);</div><div class='add'>+	/* Not allowed to send control qkey */</div><div class='add'>+	if (uwr-&gt;remote_qkey &amp; 0x80000000)</div><div class='add'>+		sqe-&gt;qkey = cpu_to_le32(qp-&gt;attrs.rocev2.qkey);</div><div class='add'>+	else</div><div class='add'>+		sqe-&gt;qkey = cpu_to_le32(uwr-&gt;remote_qkey);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,</div><div class='add'>+			      const struct ib_send_wr *send_wr)</div><div class='add'>+{</div><div class='add'>+	u32 wqe_size, wqebb_cnt, hw_op, flags, sgl_offset;</div><div class='add'>+	u32 idx = *pi &amp; (qp-&gt;attrs.sq_size - 1);</div><div class='add'>+	enum ib_wr_opcode op = send_wr-&gt;opcode;</div><div class='add'>+	struct erdma_send_sqe_rc *rc_send_sqe;</div><div class='add'>+	struct erdma_send_sqe_ud *ud_send_sqe;</div><div class='add'>+	struct erdma_atomic_sqe *atomic_sqe;</div><div class='add'>+	struct erdma_readreq_sqe *read_sqe;</div><div class='add'>+	struct erdma_reg_mr_sqe *regmr_sge;</div><div class='add'>+	struct erdma_write_sqe *write_sqe;</div><div class='add'>+	struct ib_rdma_wr *rdma_wr;</div><div class='add'>+	struct erdma_sge *sge;</div><div class='add'>+	__le32 *length_field;</div><div class='add'>+	struct erdma_mr *mr;</div><div class='add'>+	u64 wqe_hdr, *entry;</div><div class='add'>+	u32 attrs;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;ibqp.qp_type != IB_QPT_RC &amp;&amp; send_wr-&gt;opcode != IB_WR_SEND &amp;&amp;</div><div class='add'>+	    send_wr-&gt;opcode != IB_WR_SEND_WITH_IMM)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	entry = get_queue_entry(qp-&gt;kern_qp.sq_buf, idx, qp-&gt;attrs.sq_size,</div><div class='add'>+				SQEBB_SHIFT);</div><div class='add'>+</div><div class='add'>+	/* Clear the SQE header section. */</div><div class='add'>+	*entry = 0;</div><div class='add'>+</div><div class='add'>+	qp-&gt;kern_qp.swr_tbl[idx] = send_wr-&gt;wr_id;</div><div class='add'>+	flags = send_wr-&gt;send_flags;</div><div class='add'>+	wqe_hdr = FIELD_PREP(</div><div class='add'>+		ERDMA_SQE_HDR_CE_MASK,</div><div class='add'>+		((flags &amp; IB_SEND_SIGNALED) || qp-&gt;kern_qp.sig_all) ? 1 : 0);</div><div class='add'>+	wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_SE_MASK,</div><div class='add'>+			      flags &amp; IB_SEND_SOLICITED ? 1 : 0);</div><div class='add'>+	wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_FENCE_MASK,</div><div class='add'>+			      flags &amp; IB_SEND_FENCE ? 1 : 0);</div><div class='add'>+	wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_INLINE_MASK,</div><div class='add'>+			      flags &amp; IB_SEND_INLINE ? 1 : 0);</div><div class='add'>+	wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_QPN_MASK, QP_ID(qp));</div><div class='add'>+</div><div class='add'>+	switch (op) {</div><div class='add'>+	case IB_WR_RDMA_WRITE:</div><div class='add'>+	case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='add'>+		hw_op = ERDMA_OP_WRITE;</div><div class='add'>+		if (op == IB_WR_RDMA_WRITE_WITH_IMM)</div><div class='add'>+			hw_op = ERDMA_OP_WRITE_WITH_IMM;</div><div class='add'>+		wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, hw_op);</div><div class='add'>+		rdma_wr = container_of(send_wr, struct ib_rdma_wr, wr);</div><div class='add'>+		write_sqe = (struct erdma_write_sqe *)entry;</div><div class='add'>+</div><div class='add'>+		write_sqe-&gt;imm_data = send_wr-&gt;ex.imm_data;</div><div class='add'>+		write_sqe-&gt;sink_stag = cpu_to_le32(rdma_wr-&gt;rkey);</div><div class='add'>+		write_sqe-&gt;sink_to_h =</div><div class='add'>+			cpu_to_le32(upper_32_bits(rdma_wr-&gt;remote_addr));</div><div class='add'>+		write_sqe-&gt;sink_to_l =</div><div class='add'>+			cpu_to_le32(lower_32_bits(rdma_wr-&gt;remote_addr));</div><div class='add'>+</div><div class='add'>+		length_field = &amp;write_sqe-&gt;length;</div><div class='add'>+		wqe_size = sizeof(struct erdma_write_sqe);</div><div class='add'>+		sgl_offset = wqe_size;</div><div class='add'>+		break;</div><div class='add'>+	case IB_WR_RDMA_READ:</div><div class='add'>+	case IB_WR_RDMA_READ_WITH_INV:</div><div class='add'>+		read_sqe = (struct erdma_readreq_sqe *)entry;</div><div class='add'>+		if (unlikely(send_wr-&gt;num_sge != 1))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		hw_op = ERDMA_OP_READ;</div><div class='add'>+		if (op == IB_WR_RDMA_READ_WITH_INV) {</div><div class='add'>+			hw_op = ERDMA_OP_READ_WITH_INV;</div><div class='add'>+			read_sqe-&gt;invalid_stag =</div><div class='add'>+				cpu_to_le32(send_wr-&gt;ex.invalidate_rkey);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, hw_op);</div><div class='add'>+		rdma_wr = container_of(send_wr, struct ib_rdma_wr, wr);</div><div class='add'>+		read_sqe-&gt;length = cpu_to_le32(send_wr-&gt;sg_list[0].length);</div><div class='add'>+		read_sqe-&gt;sink_stag = cpu_to_le32(send_wr-&gt;sg_list[0].lkey);</div><div class='add'>+		read_sqe-&gt;sink_to_l =</div><div class='add'>+			cpu_to_le32(lower_32_bits(send_wr-&gt;sg_list[0].addr));</div><div class='add'>+		read_sqe-&gt;sink_to_h =</div><div class='add'>+			cpu_to_le32(upper_32_bits(send_wr-&gt;sg_list[0].addr));</div><div class='add'>+</div><div class='add'>+		sge = get_queue_entry(qp-&gt;kern_qp.sq_buf, idx + 1,</div><div class='add'>+				      qp-&gt;attrs.sq_size, SQEBB_SHIFT);</div><div class='add'>+		sge-&gt;addr = cpu_to_le64(rdma_wr-&gt;remote_addr);</div><div class='add'>+		sge-&gt;key = cpu_to_le32(rdma_wr-&gt;rkey);</div><div class='add'>+		sge-&gt;length = cpu_to_le32(send_wr-&gt;sg_list[0].length);</div><div class='add'>+		wqe_size = sizeof(struct erdma_readreq_sqe) +</div><div class='add'>+			   send_wr-&gt;num_sge * sizeof(struct ib_sge);</div><div class='add'>+</div><div class='add'>+		goto out;</div><div class='add'>+	case IB_WR_SEND:</div><div class='add'>+	case IB_WR_SEND_WITH_IMM:</div><div class='add'>+	case IB_WR_SEND_WITH_INV:</div><div class='add'>+		if (qp-&gt;ibqp.qp_type == IB_QPT_RC) {</div><div class='add'>+			rc_send_sqe = (struct erdma_send_sqe_rc *)entry;</div><div class='add'>+			init_send_sqe_rc(qp, rc_send_sqe, send_wr, &amp;hw_op);</div><div class='add'>+			length_field = &amp;rc_send_sqe-&gt;length;</div><div class='add'>+			wqe_size = sizeof(struct erdma_send_sqe_rc);</div><div class='add'>+		} else {</div><div class='add'>+			ud_send_sqe = (struct erdma_send_sqe_ud *)entry;</div><div class='add'>+			init_send_sqe_ud(qp, ud_send_sqe, send_wr, &amp;hw_op);</div><div class='add'>+			length_field = &amp;ud_send_sqe-&gt;length;</div><div class='add'>+			wqe_size = sizeof(struct erdma_send_sqe_ud);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		sgl_offset = wqe_size;</div><div class='add'>+		wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, hw_op);</div><div class='add'>+		break;</div><div class='add'>+	case IB_WR_REG_MR:</div><div class='add'>+		wqe_hdr |=</div><div class='add'>+			FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, ERDMA_OP_REG_MR);</div><div class='add'>+		regmr_sge = (struct erdma_reg_mr_sqe *)entry;</div><div class='add'>+		mr = to_emr(reg_wr(send_wr)-&gt;mr);</div><div class='add'>+</div><div class='add'>+		mr-&gt;access = ERDMA_MR_ACC_LR |</div><div class='add'>+			     to_erdma_access_flags(reg_wr(send_wr)-&gt;access);</div><div class='add'>+		regmr_sge-&gt;addr = cpu_to_le64(mr-&gt;ibmr.iova);</div><div class='add'>+		regmr_sge-&gt;length = cpu_to_le32(mr-&gt;ibmr.length);</div><div class='add'>+		regmr_sge-&gt;stag = cpu_to_le32(reg_wr(send_wr)-&gt;key);</div><div class='add'>+		attrs = FIELD_PREP(ERDMA_SQE_MR_ACCESS_MASK, mr-&gt;access) |</div><div class='add'>+			FIELD_PREP(ERDMA_SQE_MR_MTT_CNT_MASK,</div><div class='add'>+				   mr-&gt;mem.mtt_nents);</div><div class='add'>+</div><div class='add'>+		if (mr-&gt;mem.mtt_nents &lt;= ERDMA_MAX_INLINE_MTT_ENTRIES) {</div><div class='add'>+			attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 0);</div><div class='add'>+			/* Copy SGLs to SQE content to accelerate */</div><div class='add'>+			memcpy(get_queue_entry(qp-&gt;kern_qp.sq_buf, idx + 1,</div><div class='add'>+					       qp-&gt;attrs.sq_size, SQEBB_SHIFT),</div><div class='add'>+			       mr-&gt;mem.mtt-&gt;buf, MTT_SIZE(mr-&gt;mem.mtt_nents));</div><div class='add'>+			wqe_size = sizeof(struct erdma_reg_mr_sqe) +</div><div class='add'>+				   MTT_SIZE(mr-&gt;mem.mtt_nents);</div><div class='add'>+		} else {</div><div class='add'>+			attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 1);</div><div class='add'>+			wqe_size = sizeof(struct erdma_reg_mr_sqe);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		regmr_sge-&gt;attrs = cpu_to_le32(attrs);</div><div class='add'>+		goto out;</div><div class='add'>+	case IB_WR_LOCAL_INV:</div><div class='add'>+		wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK,</div><div class='add'>+				      ERDMA_OP_LOCAL_INV);</div><div class='add'>+		regmr_sge = (struct erdma_reg_mr_sqe *)entry;</div><div class='add'>+		regmr_sge-&gt;stag = cpu_to_le32(send_wr-&gt;ex.invalidate_rkey);</div><div class='add'>+		wqe_size = sizeof(struct erdma_reg_mr_sqe);</div><div class='add'>+		goto out;</div><div class='add'>+	case IB_WR_ATOMIC_CMP_AND_SWP:</div><div class='add'>+	case IB_WR_ATOMIC_FETCH_AND_ADD:</div><div class='add'>+		atomic_sqe = (struct erdma_atomic_sqe *)entry;</div><div class='add'>+		if (op == IB_WR_ATOMIC_CMP_AND_SWP) {</div><div class='add'>+			wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK,</div><div class='add'>+					      ERDMA_OP_ATOMIC_CAS);</div><div class='add'>+			atomic_sqe-&gt;fetchadd_swap_data =</div><div class='add'>+				cpu_to_le64(atomic_wr(send_wr)-&gt;swap);</div><div class='add'>+			atomic_sqe-&gt;cmp_data =</div><div class='add'>+				cpu_to_le64(atomic_wr(send_wr)-&gt;compare_add);</div><div class='add'>+		} else {</div><div class='add'>+			wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK,</div><div class='add'>+					      ERDMA_OP_ATOMIC_FAA);</div><div class='add'>+			atomic_sqe-&gt;fetchadd_swap_data =</div><div class='add'>+				cpu_to_le64(atomic_wr(send_wr)-&gt;compare_add);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		sge = get_queue_entry(qp-&gt;kern_qp.sq_buf, idx + 1,</div><div class='add'>+				      qp-&gt;attrs.sq_size, SQEBB_SHIFT);</div><div class='add'>+		sge-&gt;addr = cpu_to_le64(atomic_wr(send_wr)-&gt;remote_addr);</div><div class='add'>+		sge-&gt;key = cpu_to_le32(atomic_wr(send_wr)-&gt;rkey);</div><div class='add'>+		sge++;</div><div class='add'>+</div><div class='add'>+		sge-&gt;addr = cpu_to_le64(send_wr-&gt;sg_list[0].addr);</div><div class='add'>+		sge-&gt;key = cpu_to_le32(send_wr-&gt;sg_list[0].lkey);</div><div class='add'>+		sge-&gt;length = cpu_to_le32(send_wr-&gt;sg_list[0].length);</div><div class='add'>+</div><div class='add'>+		wqe_size = sizeof(*atomic_sqe);</div><div class='add'>+		goto out;</div><div class='add'>+	default:</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (flags &amp; IB_SEND_INLINE) {</div><div class='add'>+		ret = fill_inline_data(qp, send_wr, idx, sgl_offset,</div><div class='add'>+				       length_field);</div><div class='add'>+		if (ret &lt; 0)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		wqe_size += ret;</div><div class='add'>+		wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_SGL_LEN_MASK, ret);</div><div class='add'>+	} else {</div><div class='add'>+		ret = fill_sgl(qp, send_wr, idx, sgl_offset, length_field);</div><div class='add'>+		if (ret)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		wqe_size += send_wr-&gt;num_sge * sizeof(struct ib_sge);</div><div class='add'>+		wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_SGL_LEN_MASK,</div><div class='add'>+				      send_wr-&gt;num_sge);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	wqebb_cnt = SQEBB_COUNT(wqe_size);</div><div class='add'>+	wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_WQEBB_CNT_MASK, wqebb_cnt - 1);</div><div class='add'>+	*pi += wqebb_cnt;</div><div class='add'>+	wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_WQEBB_INDEX_MASK, *pi);</div><div class='add'>+</div><div class='add'>+	*entry = wqe_hdr;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void kick_sq_db(struct erdma_qp *qp, u16 pi)</div><div class='add'>+{</div><div class='add'>+	u64 db_data = FIELD_PREP(ERDMA_SQE_HDR_QPN_MASK, QP_ID(qp)) |</div><div class='add'>+		      FIELD_PREP(ERDMA_SQE_HDR_WQEBB_INDEX_MASK, pi);</div><div class='add'>+</div><div class='add'>+	*(u64 *)qp-&gt;kern_qp.sq_dbrec = db_data;</div><div class='add'>+	writeq(db_data, qp-&gt;kern_qp.hw_sq_db);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *send_wr,</div><div class='add'>+		    const struct ib_send_wr **bad_send_wr)</div><div class='add'>+{</div><div class='add'>+	struct erdma_qp *qp = to_eqp(ibqp);</div><div class='add'>+	int ret = 0;</div><div class='add'>+	const struct ib_send_wr *wr = send_wr;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	u16 sq_pi;</div><div class='add'>+</div><div class='add'>+	if (!send_wr)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;qp-&gt;lock, flags);</div><div class='add'>+	sq_pi = qp-&gt;kern_qp.sq_pi;</div><div class='add'>+</div><div class='add'>+	while (wr) {</div><div class='add'>+		if ((u16)(sq_pi - qp-&gt;kern_qp.sq_ci) &gt;= qp-&gt;attrs.sq_size) {</div><div class='add'>+			ret = -ENOMEM;</div><div class='add'>+			*bad_send_wr = send_wr;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ret = erdma_push_one_sqe(qp, &amp;sq_pi, wr);</div><div class='add'>+		if (ret) {</div><div class='add'>+			*bad_send_wr = wr;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		qp-&gt;kern_qp.sq_pi = sq_pi;</div><div class='add'>+		kick_sq_db(qp, sq_pi);</div><div class='add'>+</div><div class='add'>+		wr = wr-&gt;next;</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;qp-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	if (unlikely(qp-&gt;flags &amp; ERDMA_QP_IN_FLUSHING))</div><div class='add'>+		mod_delayed_work(qp-&gt;dev-&gt;reflush_wq, &amp;qp-&gt;reflush_dwork,</div><div class='add'>+				 usecs_to_jiffies(100));</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_post_recv_one(struct erdma_qp *qp,</div><div class='add'>+			       const struct ib_recv_wr *recv_wr)</div><div class='add'>+{</div><div class='add'>+	struct erdma_rqe *rqe =</div><div class='add'>+		get_queue_entry(qp-&gt;kern_qp.rq_buf, qp-&gt;kern_qp.rq_pi,</div><div class='add'>+				qp-&gt;attrs.rq_size, RQE_SHIFT);</div><div class='add'>+</div><div class='add'>+	rqe-&gt;qe_idx = cpu_to_le16(qp-&gt;kern_qp.rq_pi + 1);</div><div class='add'>+	rqe-&gt;qpn = cpu_to_le32(QP_ID(qp));</div><div class='add'>+</div><div class='add'>+	if (recv_wr-&gt;num_sge == 0) {</div><div class='add'>+		rqe-&gt;length = 0;</div><div class='add'>+	} else if (recv_wr-&gt;num_sge == 1) {</div><div class='add'>+		rqe-&gt;stag = cpu_to_le32(recv_wr-&gt;sg_list[0].lkey);</div><div class='add'>+		rqe-&gt;to = cpu_to_le64(recv_wr-&gt;sg_list[0].addr);</div><div class='add'>+		rqe-&gt;length = cpu_to_le32(recv_wr-&gt;sg_list[0].length);</div><div class='add'>+	} else {</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*(u64 *)qp-&gt;kern_qp.rq_dbrec = *(u64 *)rqe;</div><div class='add'>+	writeq(*(u64 *)rqe, qp-&gt;kern_qp.hw_rq_db);</div><div class='add'>+</div><div class='add'>+	qp-&gt;kern_qp.rwr_tbl[qp-&gt;kern_qp.rq_pi &amp; (qp-&gt;attrs.rq_size - 1)] =</div><div class='add'>+		recv_wr-&gt;wr_id;</div><div class='add'>+	qp-&gt;kern_qp.rq_pi++;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *recv_wr,</div><div class='add'>+		    const struct ib_recv_wr **bad_recv_wr)</div><div class='add'>+{</div><div class='add'>+	const struct ib_recv_wr *wr = recv_wr;</div><div class='add'>+	struct erdma_qp *qp = to_eqp(ibqp);</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;qp-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	while (wr) {</div><div class='add'>+		ret = erdma_post_recv_one(qp, wr);</div><div class='add'>+		if (ret) {</div><div class='add'>+			*bad_recv_wr = wr;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		wr = wr-&gt;next;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_unlock_irqrestore(&amp;qp-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	if (unlikely(qp-&gt;flags &amp; ERDMA_QP_IN_FLUSHING))</div><div class='add'>+		mod_delayed_work(qp-&gt;dev-&gt;reflush_wq, &amp;qp-&gt;reflush_dwork,</div><div class='add'>+				 usecs_to_jiffies(100));</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c<br/>new file mode 100644<br/>index 000000000000..109a3f3de911<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/erdma/erdma_verbs.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/erdma/erdma_verbs.c</a></div><div class='hunk'>@@ -0,0 +1,2300 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0</div><div class='add'>+</div><div class='add'>+/* Authors: Cheng Xu &lt;chengyou@linux.alibaba.com&gt; */</div><div class='add'>+/*          Kai Shen &lt;kaishen@linux.alibaba.com&gt; */</div><div class='add'>+/* Copyright (c) 2020-2022, Alibaba Group. */</div><div class='add'>+</div><div class='add'>+/* Authors: Bernard Metzler &lt;bmt@zurich.ibm.com&gt; */</div><div class='add'>+/* Copyright (c) 2008-2019, IBM Corporation */</div><div class='add'>+</div><div class='add'>+/* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/vmalloc.h&gt;</div><div class='add'>+#include &lt;net/addrconf.h&gt;</div><div class='add'>+#include &lt;rdma/erdma-abi.h&gt;</div><div class='add'>+#include &lt;rdma/ib_umem.h&gt;</div><div class='add'>+#include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='add'>+</div><div class='add'>+#include "erdma.h"</div><div class='add'>+#include "erdma_cm.h"</div><div class='add'>+#include "erdma_verbs.h"</div><div class='add'>+</div><div class='add'>+static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg,</div><div class='add'>+				      u64 *addr0, u64 *addr1)</div><div class='add'>+{</div><div class='add'>+	struct erdma_mtt *mtt = mem-&gt;mtt;</div><div class='add'>+</div><div class='add'>+	if (mem-&gt;mtt_nents &gt; ERDMA_MAX_INLINE_MTT_ENTRIES) {</div><div class='add'>+		*addr0 = mtt-&gt;buf_dma;</div><div class='add'>+		*cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,</div><div class='add'>+				   ERDMA_MR_MTT_1LEVEL);</div><div class='add'>+	} else {</div><div class='add'>+		*addr0 = mtt-&gt;buf[0];</div><div class='add'>+		memcpy(addr1, mtt-&gt;buf + 1, MTT_SIZE(mem-&gt;mtt_nents - 1));</div><div class='add'>+		*cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,</div><div class='add'>+				   ERDMA_MR_MTT_0LEVEL);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = to_edev(qp-&gt;ibqp.device);</div><div class='add'>+	struct erdma_pd *pd = to_epd(qp-&gt;ibqp.pd);</div><div class='add'>+	struct erdma_cmdq_create_qp_req req;</div><div class='add'>+	struct erdma_uqp *user_qp;</div><div class='add'>+	u64 resp0, resp1;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_RDMA,</div><div class='add'>+				CMDQ_OPCODE_CREATE_QP);</div><div class='add'>+</div><div class='add'>+	req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK,</div><div class='add'>+			      ilog2(qp-&gt;attrs.sq_size)) |</div><div class='add'>+		   FIELD_PREP(ERDMA_CMD_CREATE_QP_QPN_MASK, QP_ID(qp));</div><div class='add'>+	req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK,</div><div class='add'>+			      ilog2(qp-&gt;attrs.rq_size)) |</div><div class='add'>+		   FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd-&gt;pdn);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;ibqp.qp_type == IB_QPT_RC)</div><div class='add'>+		req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_QP_TYPE_MASK,</div><div class='add'>+				      ERDMA_QPT_RC);</div><div class='add'>+	else</div><div class='add'>+		req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_QP_TYPE_MASK,</div><div class='add'>+				      ERDMA_QPT_UD);</div><div class='add'>+</div><div class='add'>+	if (rdma_is_kernel_res(&amp;qp-&gt;ibqp.res)) {</div><div class='add'>+		u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT;</div><div class='add'>+</div><div class='add'>+		req.sq_cqn_mtt_cfg =</div><div class='add'>+			FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,</div><div class='add'>+				   pgsz_range) |</div><div class='add'>+			FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp-&gt;scq-&gt;cqn);</div><div class='add'>+		req.rq_cqn_mtt_cfg =</div><div class='add'>+			FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,</div><div class='add'>+				   pgsz_range) |</div><div class='add'>+			FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp-&gt;rcq-&gt;cqn);</div><div class='add'>+</div><div class='add'>+		req.sq_mtt_cfg =</div><div class='add'>+			FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) |</div><div class='add'>+			FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) |</div><div class='add'>+			FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,</div><div class='add'>+				   ERDMA_MR_MTT_0LEVEL);</div><div class='add'>+		req.rq_mtt_cfg = req.sq_mtt_cfg;</div><div class='add'>+</div><div class='add'>+		req.rq_buf_addr = qp-&gt;kern_qp.rq_buf_dma_addr;</div><div class='add'>+		req.sq_buf_addr = qp-&gt;kern_qp.sq_buf_dma_addr;</div><div class='add'>+		req.sq_dbrec_dma = qp-&gt;kern_qp.sq_dbrec_dma;</div><div class='add'>+		req.rq_dbrec_dma = qp-&gt;kern_qp.rq_dbrec_dma;</div><div class='add'>+	} else {</div><div class='add'>+		user_qp = &amp;qp-&gt;user_qp;</div><div class='add'>+		req.sq_cqn_mtt_cfg = FIELD_PREP(</div><div class='add'>+			ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,</div><div class='add'>+			ilog2(user_qp-&gt;sq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);</div><div class='add'>+		req.sq_cqn_mtt_cfg |=</div><div class='add'>+			FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp-&gt;scq-&gt;cqn);</div><div class='add'>+</div><div class='add'>+		req.rq_cqn_mtt_cfg = FIELD_PREP(</div><div class='add'>+			ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,</div><div class='add'>+			ilog2(user_qp-&gt;rq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);</div><div class='add'>+		req.rq_cqn_mtt_cfg |=</div><div class='add'>+			FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp-&gt;rcq-&gt;cqn);</div><div class='add'>+</div><div class='add'>+		req.sq_mtt_cfg = user_qp-&gt;sq_mem.page_offset;</div><div class='add'>+		req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,</div><div class='add'>+					     user_qp-&gt;sq_mem.mtt_nents);</div><div class='add'>+</div><div class='add'>+		req.rq_mtt_cfg = user_qp-&gt;rq_mem.page_offset;</div><div class='add'>+		req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,</div><div class='add'>+					     user_qp-&gt;rq_mem.mtt_nents);</div><div class='add'>+</div><div class='add'>+		assemble_qbuf_mtt_for_cmd(&amp;user_qp-&gt;sq_mem, &amp;req.sq_mtt_cfg,</div><div class='add'>+					  &amp;req.sq_buf_addr, req.sq_mtt_entry);</div><div class='add'>+		assemble_qbuf_mtt_for_cmd(&amp;user_qp-&gt;rq_mem, &amp;req.rq_mtt_cfg,</div><div class='add'>+					  &amp;req.rq_buf_addr, req.rq_mtt_entry);</div><div class='add'>+</div><div class='add'>+		req.sq_dbrec_dma = user_qp-&gt;sq_dbrec_dma;</div><div class='add'>+		req.rq_dbrec_dma = user_qp-&gt;rq_dbrec_dma;</div><div class='add'>+</div><div class='add'>+		if (uctx-&gt;ext_db.enable) {</div><div class='add'>+			req.sq_cqn_mtt_cfg |=</div><div class='add'>+				FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);</div><div class='add'>+			req.db_cfg =</div><div class='add'>+				FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,</div><div class='add'>+					   uctx-&gt;ext_db.sdb_off) |</div><div class='add'>+				FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,</div><div class='add'>+					   uctx-&gt;ext_db.rdb_off);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), &amp;resp0, &amp;resp1,</div><div class='add'>+				  true);</div><div class='add'>+	if (!err &amp;&amp; erdma_device_iwarp(dev))</div><div class='add'>+		qp-&gt;attrs.iwarp.cookie =</div><div class='add'>+			FIELD_GET(ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK, resp0);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr)</div><div class='add'>+{</div><div class='add'>+	struct erdma_pd *pd = to_epd(mr-&gt;ibmr.pd);</div><div class='add'>+	u32 mtt_level = ERDMA_MR_MTT_0LEVEL;</div><div class='add'>+	struct erdma_cmdq_reg_mr_req req;</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_REG_MR);</div><div class='add'>+</div><div class='add'>+	if (mr-&gt;type == ERDMA_MR_TYPE_FRMR ||</div><div class='add'>+	    mr-&gt;mem.page_cnt &gt; ERDMA_MAX_INLINE_MTT_ENTRIES) {</div><div class='add'>+		if (mr-&gt;mem.mtt-&gt;continuous) {</div><div class='add'>+			req.phy_addr[0] = mr-&gt;mem.mtt-&gt;buf_dma;</div><div class='add'>+			mtt_level = ERDMA_MR_MTT_1LEVEL;</div><div class='add'>+		} else {</div><div class='add'>+			req.phy_addr[0] = mr-&gt;mem.mtt-&gt;dma_addrs[0];</div><div class='add'>+			mtt_level = mr-&gt;mem.mtt-&gt;level;</div><div class='add'>+		}</div><div class='add'>+	} else if (mr-&gt;type != ERDMA_MR_TYPE_DMA) {</div><div class='add'>+		memcpy(req.phy_addr, mr-&gt;mem.mtt-&gt;buf,</div><div class='add'>+		       MTT_SIZE(mr-&gt;mem.page_cnt));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	req.cfg0 = FIELD_PREP(ERDMA_CMD_MR_VALID_MASK, mr-&gt;valid) |</div><div class='add'>+		   FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr-&gt;ibmr.lkey &amp; 0xFF) |</div><div class='add'>+		   FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr-&gt;ibmr.lkey &gt;&gt; 8);</div><div class='add'>+	req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd-&gt;pdn) |</div><div class='add'>+		   FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr-&gt;type) |</div><div class='add'>+		   FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr-&gt;access);</div><div class='add'>+	req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK,</div><div class='add'>+			      ilog2(mr-&gt;mem.page_size)) |</div><div class='add'>+		   FIELD_PREP(ERDMA_CMD_REGMR_MTT_LEVEL_MASK, mtt_level) |</div><div class='add'>+		   FIELD_PREP(ERDMA_CMD_REGMR_MTT_CNT_MASK, mr-&gt;mem.page_cnt);</div><div class='add'>+</div><div class='add'>+	if (mr-&gt;type == ERDMA_MR_TYPE_DMA)</div><div class='add'>+		goto post_cmd;</div><div class='add'>+</div><div class='add'>+	if (mr-&gt;type == ERDMA_MR_TYPE_NORMAL) {</div><div class='add'>+		req.start_va = mr-&gt;mem.va;</div><div class='add'>+		req.size = mr-&gt;mem.len;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!mr-&gt;mem.mtt-&gt;continuous &amp;&amp; mr-&gt;mem.mtt-&gt;level &gt; 1) {</div><div class='add'>+		req.cfg0 |= FIELD_PREP(ERDMA_CMD_MR_VERSION_MASK, 1);</div><div class='add'>+		req.cfg2 |= FIELD_PREP(ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK,</div><div class='add'>+				       PAGE_SHIFT - ERDMA_HW_PAGE_SHIFT);</div><div class='add'>+		req.size_h = upper_32_bits(mr-&gt;mem.len);</div><div class='add'>+		req.mtt_cnt_h = mr-&gt;mem.page_cnt &gt;&gt; 20;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+post_cmd:</div><div class='add'>+	return erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL,</div><div class='add'>+				   true);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = to_edev(cq-&gt;ibcq.device);</div><div class='add'>+	struct erdma_cmdq_create_cq_req req;</div><div class='add'>+	struct erdma_mem *mem;</div><div class='add'>+	u32 page_size;</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_RDMA,</div><div class='add'>+				CMDQ_OPCODE_CREATE_CQ);</div><div class='add'>+</div><div class='add'>+	req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_CQN_MASK, cq-&gt;cqn) |</div><div class='add'>+		   FIELD_PREP(ERDMA_CMD_CREATE_CQ_DEPTH_MASK, ilog2(cq-&gt;depth));</div><div class='add'>+	req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq-&gt;assoc_eqn);</div><div class='add'>+</div><div class='add'>+	if (rdma_is_kernel_res(&amp;cq-&gt;ibcq.res)) {</div><div class='add'>+		page_size = SZ_32M;</div><div class='add'>+		req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,</div><div class='add'>+				       ilog2(page_size) - ERDMA_HW_PAGE_SHIFT);</div><div class='add'>+		req.qbuf_addr_l = lower_32_bits(cq-&gt;kern_cq.qbuf_dma_addr);</div><div class='add'>+		req.qbuf_addr_h = upper_32_bits(cq-&gt;kern_cq.qbuf_dma_addr);</div><div class='add'>+</div><div class='add'>+		req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) |</div><div class='add'>+			    FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,</div><div class='add'>+				       ERDMA_MR_MTT_0LEVEL);</div><div class='add'>+</div><div class='add'>+		req.first_page_offset = 0;</div><div class='add'>+		req.cq_dbrec_dma = cq-&gt;kern_cq.dbrec_dma;</div><div class='add'>+	} else {</div><div class='add'>+		mem = &amp;cq-&gt;user_cq.qbuf_mem;</div><div class='add'>+		req.cfg0 |=</div><div class='add'>+			FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,</div><div class='add'>+				   ilog2(mem-&gt;page_size) - ERDMA_HW_PAGE_SHIFT);</div><div class='add'>+		if (mem-&gt;mtt_nents == 1) {</div><div class='add'>+			req.qbuf_addr_l = lower_32_bits(mem-&gt;mtt-&gt;buf[0]);</div><div class='add'>+			req.qbuf_addr_h = upper_32_bits(mem-&gt;mtt-&gt;buf[0]);</div><div class='add'>+			req.cfg1 |=</div><div class='add'>+				FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,</div><div class='add'>+					   ERDMA_MR_MTT_0LEVEL);</div><div class='add'>+		} else {</div><div class='add'>+			req.qbuf_addr_l = lower_32_bits(mem-&gt;mtt-&gt;buf_dma);</div><div class='add'>+			req.qbuf_addr_h = upper_32_bits(mem-&gt;mtt-&gt;buf_dma);</div><div class='add'>+			req.cfg1 |=</div><div class='add'>+				FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,</div><div class='add'>+					   ERDMA_MR_MTT_1LEVEL);</div><div class='add'>+		}</div><div class='add'>+		req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK,</div><div class='add'>+				       mem-&gt;mtt_nents);</div><div class='add'>+</div><div class='add'>+		req.first_page_offset = mem-&gt;page_offset;</div><div class='add'>+		req.cq_dbrec_dma = cq-&gt;user_cq.dbrec_dma;</div><div class='add'>+</div><div class='add'>+		if (uctx-&gt;ext_db.enable) {</div><div class='add'>+			req.cfg1 |= FIELD_PREP(</div><div class='add'>+				ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);</div><div class='add'>+			req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,</div><div class='add'>+					      uctx-&gt;ext_db.cdb_off);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL,</div><div class='add'>+				   true);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_alloc_idx(struct erdma_resource_cb *res_cb)</div><div class='add'>+{</div><div class='add'>+	int idx;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;res_cb-&gt;lock, flags);</div><div class='add'>+	idx = find_next_zero_bit(res_cb-&gt;bitmap, res_cb-&gt;max_cap,</div><div class='add'>+				 res_cb-&gt;next_alloc_idx);</div><div class='add'>+	if (idx == res_cb-&gt;max_cap) {</div><div class='add'>+		idx = find_first_zero_bit(res_cb-&gt;bitmap, res_cb-&gt;max_cap);</div><div class='add'>+		if (idx == res_cb-&gt;max_cap) {</div><div class='add'>+			res_cb-&gt;next_alloc_idx = 1;</div><div class='add'>+			spin_unlock_irqrestore(&amp;res_cb-&gt;lock, flags);</div><div class='add'>+			return -ENOSPC;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	set_bit(idx, res_cb-&gt;bitmap);</div><div class='add'>+	res_cb-&gt;next_alloc_idx = idx + 1;</div><div class='add'>+	spin_unlock_irqrestore(&amp;res_cb-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	return idx;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	u32 used;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;res_cb-&gt;lock, flags);</div><div class='add'>+	used = __test_and_clear_bit(idx, res_cb-&gt;bitmap);</div><div class='add'>+	spin_unlock_irqrestore(&amp;res_cb-&gt;lock, flags);</div><div class='add'>+	WARN_ON(!used);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct rdma_user_mmap_entry *</div><div class='add'>+erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address,</div><div class='add'>+			     u32 size, u8 mmap_flag, u64 *mmap_offset)</div><div class='add'>+{</div><div class='add'>+	struct erdma_user_mmap_entry *entry =</div><div class='add'>+		kzalloc(sizeof(*entry), GFP_KERNEL);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!entry)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	entry-&gt;address = (u64)address;</div><div class='add'>+	entry-&gt;mmap_flag = mmap_flag;</div><div class='add'>+</div><div class='add'>+	size = PAGE_ALIGN(size);</div><div class='add'>+</div><div class='add'>+	ret = rdma_user_mmap_entry_insert(&amp;uctx-&gt;ibucontext, &amp;entry-&gt;rdma_entry,</div><div class='add'>+					  size);</div><div class='add'>+	if (ret) {</div><div class='add'>+		kfree(entry);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*mmap_offset = rdma_user_mmap_get_offset(&amp;entry-&gt;rdma_entry);</div><div class='add'>+</div><div class='add'>+	return &amp;entry-&gt;rdma_entry;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,</div><div class='add'>+		       struct ib_udata *unused)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibdev);</div><div class='add'>+</div><div class='add'>+	memset(attr, 0, sizeof(*attr));</div><div class='add'>+</div><div class='add'>+	attr-&gt;max_mr_size = dev-&gt;attrs.max_mr_size;</div><div class='add'>+	attr-&gt;vendor_id = PCI_VENDOR_ID_ALIBABA;</div><div class='add'>+	attr-&gt;vendor_part_id = dev-&gt;pdev-&gt;device;</div><div class='add'>+	attr-&gt;hw_ver = dev-&gt;pdev-&gt;revision;</div><div class='add'>+	attr-&gt;max_qp = dev-&gt;attrs.max_qp - 1;</div><div class='add'>+	attr-&gt;max_qp_wr = min(dev-&gt;attrs.max_send_wr, dev-&gt;attrs.max_recv_wr);</div><div class='add'>+	attr-&gt;max_qp_rd_atom = dev-&gt;attrs.max_ord;</div><div class='add'>+	attr-&gt;max_qp_init_rd_atom = dev-&gt;attrs.max_ird;</div><div class='add'>+	attr-&gt;max_res_rd_atom = dev-&gt;attrs.max_qp * dev-&gt;attrs.max_ird;</div><div class='add'>+	attr-&gt;device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;</div><div class='add'>+	attr-&gt;kernel_cap_flags = IBK_LOCAL_DMA_LKEY;</div><div class='add'>+	ibdev-&gt;local_dma_lkey = dev-&gt;attrs.local_dma_key;</div><div class='add'>+	attr-&gt;max_send_sge = dev-&gt;attrs.max_send_sge;</div><div class='add'>+	attr-&gt;max_recv_sge = dev-&gt;attrs.max_recv_sge;</div><div class='add'>+	attr-&gt;max_sge_rd = dev-&gt;attrs.max_sge_rd;</div><div class='add'>+	attr-&gt;max_cq = dev-&gt;attrs.max_cq - 1;</div><div class='add'>+	attr-&gt;max_cqe = dev-&gt;attrs.max_cqe;</div><div class='add'>+	attr-&gt;max_mr = dev-&gt;attrs.max_mr;</div><div class='add'>+	attr-&gt;max_pd = dev-&gt;attrs.max_pd;</div><div class='add'>+	attr-&gt;max_mw = dev-&gt;attrs.max_mw;</div><div class='add'>+	attr-&gt;max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA;</div><div class='add'>+	attr-&gt;page_size_cap = ERDMA_PAGE_SIZE_SUPPORT;</div><div class='add'>+</div><div class='add'>+	if (erdma_device_rocev2(dev)) {</div><div class='add'>+		attr-&gt;max_pkeys = ERDMA_MAX_PKEYS;</div><div class='add'>+		attr-&gt;max_ah = dev-&gt;attrs.max_ah;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;attrs.cap_flags &amp; ERDMA_DEV_CAP_FLAGS_ATOMIC)</div><div class='add'>+		attr-&gt;atomic_cap = IB_ATOMIC_GLOB;</div><div class='add'>+</div><div class='add'>+	attr-&gt;fw_ver = dev-&gt;attrs.fw_version;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;netdev)</div><div class='add'>+		addrconf_addr_eui48((u8 *)&amp;attr-&gt;sys_image_guid,</div><div class='add'>+				    dev-&gt;netdev-&gt;dev_addr);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx,</div><div class='add'>+		    union ib_gid *gid)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibdev);</div><div class='add'>+</div><div class='add'>+	memset(gid, 0, sizeof(*gid));</div><div class='add'>+	ether_addr_copy(gid-&gt;raw, dev-&gt;attrs.peer_addr);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_query_port(struct ib_device *ibdev, u32 port,</div><div class='add'>+		     struct ib_port_attr *attr)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibdev);</div><div class='add'>+	struct net_device *ndev = dev-&gt;netdev;</div><div class='add'>+</div><div class='add'>+	memset(attr, 0, sizeof(*attr));</div><div class='add'>+</div><div class='add'>+	if (erdma_device_iwarp(dev)) {</div><div class='add'>+		attr-&gt;gid_tbl_len = 1;</div><div class='add'>+	} else {</div><div class='add'>+		attr-&gt;gid_tbl_len = dev-&gt;attrs.max_gid;</div><div class='add'>+		attr-&gt;ip_gids = true;</div><div class='add'>+		attr-&gt;pkey_tbl_len = ERDMA_MAX_PKEYS;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	attr-&gt;port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;</div><div class='add'>+	attr-&gt;max_msg_sz = -1;</div><div class='add'>+</div><div class='add'>+	if (!ndev)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	ib_get_eth_speed(ibdev, port, &amp;attr-&gt;active_speed, &amp;attr-&gt;active_width);</div><div class='add'>+	attr-&gt;max_mtu = ib_mtu_int_to_enum(ndev-&gt;mtu);</div><div class='add'>+	attr-&gt;active_mtu = ib_mtu_int_to_enum(ndev-&gt;mtu);</div><div class='add'>+	attr-&gt;state = ib_get_curr_port_state(ndev);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	if (attr-&gt;state == IB_PORT_ACTIVE)</div><div class='add'>+		attr-&gt;phys_state = IB_PORT_PHYS_STATE_LINK_UP;</div><div class='add'>+	else</div><div class='add'>+		attr-&gt;phys_state = IB_PORT_PHYS_STATE_DISABLED;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_get_port_immutable(struct ib_device *ibdev, u32 port,</div><div class='add'>+			     struct ib_port_immutable *port_immutable)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibdev);</div><div class='add'>+</div><div class='add'>+	if (erdma_device_iwarp(dev)) {</div><div class='add'>+		port_immutable-&gt;core_cap_flags = RDMA_CORE_PORT_IWARP;</div><div class='add'>+		port_immutable-&gt;gid_tbl_len = 1;</div><div class='add'>+	} else {</div><div class='add'>+		port_immutable-&gt;core_cap_flags =</div><div class='add'>+			RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;</div><div class='add'>+		port_immutable-&gt;max_mad_size = IB_MGMT_MAD_SIZE;</div><div class='add'>+		port_immutable-&gt;gid_tbl_len = dev-&gt;attrs.max_gid;</div><div class='add'>+		port_immutable-&gt;pkey_tbl_len = ERDMA_MAX_PKEYS;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct erdma_pd *pd = to_epd(ibpd);</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibpd-&gt;device);</div><div class='add'>+	int pdn;</div><div class='add'>+</div><div class='add'>+	pdn = erdma_alloc_idx(&amp;dev-&gt;res_cb[ERDMA_RES_TYPE_PD]);</div><div class='add'>+	if (pdn &lt; 0)</div><div class='add'>+		return pdn;</div><div class='add'>+</div><div class='add'>+	pd-&gt;pdn = pdn;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct erdma_pd *pd = to_epd(ibpd);</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibpd-&gt;device);</div><div class='add'>+</div><div class='add'>+	erdma_free_idx(&amp;dev-&gt;res_cb[ERDMA_RES_TYPE_PD], pd-&gt;pdn);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_flush_worker(struct work_struct *work)</div><div class='add'>+{</div><div class='add'>+	struct delayed_work *dwork = to_delayed_work(work);</div><div class='add'>+	struct erdma_qp *qp =</div><div class='add'>+		container_of(dwork, struct erdma_qp, reflush_dwork);</div><div class='add'>+	struct erdma_cmdq_reflush_req req;</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_RDMA,</div><div class='add'>+				CMDQ_OPCODE_REFLUSH);</div><div class='add'>+	req.qpn = QP_ID(qp);</div><div class='add'>+	req.sq_pi = qp-&gt;kern_qp.sq_pi;</div><div class='add'>+	req.rq_pi = qp-&gt;kern_qp.rq_pi;</div><div class='add'>+	erdma_post_cmd_wait(&amp;qp-&gt;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL,</div><div class='add'>+			    true);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_qp_validate_cap(struct erdma_dev *dev,</div><div class='add'>+				 struct ib_qp_init_attr *attrs)</div><div class='add'>+{</div><div class='add'>+	if ((attrs-&gt;cap.max_send_wr &gt; dev-&gt;attrs.max_send_wr) ||</div><div class='add'>+	    (attrs-&gt;cap.max_recv_wr &gt; dev-&gt;attrs.max_recv_wr) ||</div><div class='add'>+	    (attrs-&gt;cap.max_send_sge &gt; dev-&gt;attrs.max_send_sge) ||</div><div class='add'>+	    (attrs-&gt;cap.max_recv_sge &gt; dev-&gt;attrs.max_recv_sge) ||</div><div class='add'>+	    (attrs-&gt;cap.max_inline_data &gt; ERDMA_MAX_INLINE) ||</div><div class='add'>+	    !attrs-&gt;cap.max_send_wr || !attrs-&gt;cap.max_recv_wr) {</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_qp_validate_attr(struct erdma_dev *dev,</div><div class='add'>+				  struct ib_qp_init_attr *attrs)</div><div class='add'>+{</div><div class='add'>+	if (erdma_device_iwarp(dev) &amp;&amp; attrs-&gt;qp_type != IB_QPT_RC)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (erdma_device_rocev2(dev) &amp;&amp; attrs-&gt;qp_type != IB_QPT_RC &amp;&amp;</div><div class='add'>+	    attrs-&gt;qp_type != IB_QPT_UD &amp;&amp; attrs-&gt;qp_type != IB_QPT_GSI)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (attrs-&gt;srq)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (!attrs-&gt;send_cq || !attrs-&gt;recv_cq)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void free_kernel_qp(struct erdma_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = qp-&gt;dev;</div><div class='add'>+</div><div class='add'>+	vfree(qp-&gt;kern_qp.swr_tbl);</div><div class='add'>+	vfree(qp-&gt;kern_qp.rwr_tbl);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;kern_qp.sq_buf)</div><div class='add'>+		dma_free_coherent(&amp;dev-&gt;pdev-&gt;dev,</div><div class='add'>+				  qp-&gt;attrs.sq_size &lt;&lt; SQEBB_SHIFT,</div><div class='add'>+				  qp-&gt;kern_qp.sq_buf,</div><div class='add'>+				  qp-&gt;kern_qp.sq_buf_dma_addr);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;kern_qp.sq_dbrec)</div><div class='add'>+		dma_pool_free(dev-&gt;db_pool, qp-&gt;kern_qp.sq_dbrec,</div><div class='add'>+			      qp-&gt;kern_qp.sq_dbrec_dma);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;kern_qp.rq_buf)</div><div class='add'>+		dma_free_coherent(&amp;dev-&gt;pdev-&gt;dev,</div><div class='add'>+				  qp-&gt;attrs.rq_size &lt;&lt; RQE_SHIFT,</div><div class='add'>+				  qp-&gt;kern_qp.rq_buf,</div><div class='add'>+				  qp-&gt;kern_qp.rq_buf_dma_addr);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;kern_qp.rq_dbrec)</div><div class='add'>+		dma_pool_free(dev-&gt;db_pool, qp-&gt;kern_qp.rq_dbrec,</div><div class='add'>+			      qp-&gt;kern_qp.rq_dbrec_dma);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,</div><div class='add'>+			  struct ib_qp_init_attr *attrs)</div><div class='add'>+{</div><div class='add'>+	struct erdma_kqp *kqp = &amp;qp-&gt;kern_qp;</div><div class='add'>+	int size;</div><div class='add'>+</div><div class='add'>+	if (attrs-&gt;sq_sig_type == IB_SIGNAL_ALL_WR)</div><div class='add'>+		kqp-&gt;sig_all = 1;</div><div class='add'>+</div><div class='add'>+	kqp-&gt;sq_pi = 0;</div><div class='add'>+	kqp-&gt;sq_ci = 0;</div><div class='add'>+	kqp-&gt;rq_pi = 0;</div><div class='add'>+	kqp-&gt;rq_ci = 0;</div><div class='add'>+	kqp-&gt;hw_sq_db =</div><div class='add'>+		dev-&gt;func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX &lt;&lt; PAGE_SHIFT);</div><div class='add'>+	kqp-&gt;hw_rq_db = dev-&gt;func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;</div><div class='add'>+</div><div class='add'>+	kqp-&gt;swr_tbl = vmalloc_array(qp-&gt;attrs.sq_size, sizeof(u64));</div><div class='add'>+	kqp-&gt;rwr_tbl = vmalloc_array(qp-&gt;attrs.rq_size, sizeof(u64));</div><div class='add'>+	if (!kqp-&gt;swr_tbl || !kqp-&gt;rwr_tbl)</div><div class='add'>+		goto err_out;</div><div class='add'>+</div><div class='add'>+	size = qp-&gt;attrs.sq_size &lt;&lt; SQEBB_SHIFT;</div><div class='add'>+	kqp-&gt;sq_buf = dma_alloc_coherent(&amp;dev-&gt;pdev-&gt;dev, size,</div><div class='add'>+					 &amp;kqp-&gt;sq_buf_dma_addr, GFP_KERNEL);</div><div class='add'>+	if (!kqp-&gt;sq_buf)</div><div class='add'>+		goto err_out;</div><div class='add'>+</div><div class='add'>+	kqp-&gt;sq_dbrec =</div><div class='add'>+		dma_pool_zalloc(dev-&gt;db_pool, GFP_KERNEL, &amp;kqp-&gt;sq_dbrec_dma);</div><div class='add'>+	if (!kqp-&gt;sq_dbrec)</div><div class='add'>+		goto err_out;</div><div class='add'>+</div><div class='add'>+	size = qp-&gt;attrs.rq_size &lt;&lt; RQE_SHIFT;</div><div class='add'>+	kqp-&gt;rq_buf = dma_alloc_coherent(&amp;dev-&gt;pdev-&gt;dev, size,</div><div class='add'>+					 &amp;kqp-&gt;rq_buf_dma_addr, GFP_KERNEL);</div><div class='add'>+	if (!kqp-&gt;rq_buf)</div><div class='add'>+		goto err_out;</div><div class='add'>+</div><div class='add'>+	kqp-&gt;rq_dbrec =</div><div class='add'>+		dma_pool_zalloc(dev-&gt;db_pool, GFP_KERNEL, &amp;kqp-&gt;rq_dbrec_dma);</div><div class='add'>+	if (!kqp-&gt;rq_dbrec)</div><div class='add'>+		goto err_out;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_out:</div><div class='add'>+	free_kernel_qp(qp);</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem)</div><div class='add'>+{</div><div class='add'>+	struct erdma_mtt *mtt = mem-&gt;mtt;</div><div class='add'>+	struct ib_block_iter biter;</div><div class='add'>+	u32 idx = 0;</div><div class='add'>+</div><div class='add'>+	while (mtt-&gt;low_level)</div><div class='add'>+		mtt = mtt-&gt;low_level;</div><div class='add'>+</div><div class='add'>+	rdma_umem_for_each_dma_block(mem-&gt;umem, &amp;biter, mem-&gt;page_size)</div><div class='add'>+		mtt-&gt;buf[idx++] = rdma_block_iter_dma_address(&amp;biter);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev,</div><div class='add'>+					       size_t size)</div><div class='add'>+{</div><div class='add'>+	struct erdma_mtt *mtt;</div><div class='add'>+</div><div class='add'>+	mtt = kzalloc(sizeof(*mtt), GFP_KERNEL);</div><div class='add'>+	if (!mtt)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	mtt-&gt;size = size;</div><div class='add'>+	mtt-&gt;buf = kzalloc(mtt-&gt;size, GFP_KERNEL);</div><div class='add'>+	if (!mtt-&gt;buf)</div><div class='add'>+		goto err_free_mtt;</div><div class='add'>+</div><div class='add'>+	mtt-&gt;continuous = true;</div><div class='add'>+	mtt-&gt;buf_dma = dma_map_single(&amp;dev-&gt;pdev-&gt;dev, mtt-&gt;buf, mtt-&gt;size,</div><div class='add'>+				      DMA_TO_DEVICE);</div><div class='add'>+	if (dma_mapping_error(&amp;dev-&gt;pdev-&gt;dev, mtt-&gt;buf_dma))</div><div class='add'>+		goto err_free_mtt_buf;</div><div class='add'>+</div><div class='add'>+	return mtt;</div><div class='add'>+</div><div class='add'>+err_free_mtt_buf:</div><div class='add'>+	kfree(mtt-&gt;buf);</div><div class='add'>+</div><div class='add'>+err_free_mtt:</div><div class='add'>+	kfree(mtt);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(-ENOMEM);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_unmap_page_list(struct erdma_dev *dev, dma_addr_t *pg_dma,</div><div class='add'>+				  u32 npages)</div><div class='add'>+{</div><div class='add'>+	u32 i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; npages; i++)</div><div class='add'>+		dma_unmap_page(&amp;dev-&gt;pdev-&gt;dev, pg_dma[i], PAGE_SIZE,</div><div class='add'>+			       DMA_TO_DEVICE);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_destroy_mtt_buf_dma_addrs(struct erdma_dev *dev,</div><div class='add'>+					    struct erdma_mtt *mtt)</div><div class='add'>+{</div><div class='add'>+	erdma_unmap_page_list(dev, mtt-&gt;dma_addrs, mtt-&gt;npages);</div><div class='add'>+	vfree(mtt-&gt;dma_addrs);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_destroy_scatter_mtt(struct erdma_dev *dev,</div><div class='add'>+				      struct erdma_mtt *mtt)</div><div class='add'>+{</div><div class='add'>+	erdma_destroy_mtt_buf_dma_addrs(dev, mtt);</div><div class='add'>+	vfree(mtt-&gt;buf);</div><div class='add'>+	kfree(mtt);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_init_middle_mtt(struct erdma_mtt *mtt,</div><div class='add'>+				  struct erdma_mtt *low_mtt)</div><div class='add'>+{</div><div class='add'>+	dma_addr_t *pg_addr = mtt-&gt;buf;</div><div class='add'>+	u32 i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; low_mtt-&gt;npages; i++)</div><div class='add'>+		pg_addr[i] = low_mtt-&gt;dma_addrs[i];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u32 vmalloc_to_dma_addrs(struct erdma_dev *dev, dma_addr_t **dma_addrs,</div><div class='add'>+				void *buf, u64 len)</div><div class='add'>+{</div><div class='add'>+	dma_addr_t *pg_dma;</div><div class='add'>+	struct page *pg;</div><div class='add'>+	u32 npages, i;</div><div class='add'>+	void *addr;</div><div class='add'>+</div><div class='add'>+	npages = (PAGE_ALIGN((u64)buf + len) - PAGE_ALIGN_DOWN((u64)buf)) &gt;&gt;</div><div class='add'>+		 PAGE_SHIFT;</div><div class='add'>+	pg_dma = vcalloc(npages, sizeof(*pg_dma));</div><div class='add'>+	if (!pg_dma)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	addr = buf;</div><div class='add'>+	for (i = 0; i &lt; npages; i++) {</div><div class='add'>+		pg = vmalloc_to_page(addr);</div><div class='add'>+		if (!pg)</div><div class='add'>+			goto err;</div><div class='add'>+</div><div class='add'>+		pg_dma[i] = dma_map_page(&amp;dev-&gt;pdev-&gt;dev, pg, 0, PAGE_SIZE,</div><div class='add'>+					 DMA_TO_DEVICE);</div><div class='add'>+		if (dma_mapping_error(&amp;dev-&gt;pdev-&gt;dev, pg_dma[i]))</div><div class='add'>+			goto err;</div><div class='add'>+</div><div class='add'>+		addr += PAGE_SIZE;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*dma_addrs = pg_dma;</div><div class='add'>+</div><div class='add'>+	return npages;</div><div class='add'>+err:</div><div class='add'>+	erdma_unmap_page_list(dev, pg_dma, i);</div><div class='add'>+	vfree(pg_dma);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_create_mtt_buf_dma_addrs(struct erdma_dev *dev,</div><div class='add'>+					  struct erdma_mtt *mtt)</div><div class='add'>+{</div><div class='add'>+	dma_addr_t *addrs;</div><div class='add'>+	u32 npages;</div><div class='add'>+</div><div class='add'>+	/* Failed if buf is not page aligned */</div><div class='add'>+	if ((uintptr_t)mtt-&gt;buf &amp; ~PAGE_MASK)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	npages = vmalloc_to_dma_addrs(dev, &amp;addrs, mtt-&gt;buf, mtt-&gt;size);</div><div class='add'>+	if (!npages)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	mtt-&gt;dma_addrs = addrs;</div><div class='add'>+	mtt-&gt;npages = npages;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct erdma_mtt *erdma_create_scatter_mtt(struct erdma_dev *dev,</div><div class='add'>+						  size_t size)</div><div class='add'>+{</div><div class='add'>+	struct erdma_mtt *mtt;</div><div class='add'>+	int ret = -ENOMEM;</div><div class='add'>+</div><div class='add'>+	mtt = kzalloc(sizeof(*mtt), GFP_KERNEL);</div><div class='add'>+	if (!mtt)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	mtt-&gt;size = ALIGN(size, PAGE_SIZE);</div><div class='add'>+	mtt-&gt;buf = vzalloc(mtt-&gt;size);</div><div class='add'>+	mtt-&gt;continuous = false;</div><div class='add'>+	if (!mtt-&gt;buf)</div><div class='add'>+		goto err_free_mtt;</div><div class='add'>+</div><div class='add'>+	ret = erdma_create_mtt_buf_dma_addrs(dev, mtt);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_free_mtt_buf;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ibdev, "create scatter mtt, size:%lu, npages:%u\n",</div><div class='add'>+		  mtt-&gt;size, mtt-&gt;npages);</div><div class='add'>+</div><div class='add'>+	return mtt;</div><div class='add'>+</div><div class='add'>+err_free_mtt_buf:</div><div class='add'>+	vfree(mtt-&gt;buf);</div><div class='add'>+</div><div class='add'>+err_free_mtt:</div><div class='add'>+	kfree(mtt);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(ret);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct erdma_mtt *erdma_create_mtt(struct erdma_dev *dev, size_t size,</div><div class='add'>+					  bool force_continuous)</div><div class='add'>+{</div><div class='add'>+	struct erdma_mtt *mtt, *tmp_mtt;</div><div class='add'>+	int ret, level = 0;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ibdev, "create_mtt, size:%lu, force cont:%d\n", size,</div><div class='add'>+		  force_continuous);</div><div class='add'>+</div><div class='add'>+	if (!(dev-&gt;attrs.cap_flags &amp; ERDMA_DEV_CAP_FLAGS_MTT_VA))</div><div class='add'>+		force_continuous = true;</div><div class='add'>+</div><div class='add'>+	if (force_continuous)</div><div class='add'>+		return erdma_create_cont_mtt(dev, size);</div><div class='add'>+</div><div class='add'>+	mtt = erdma_create_scatter_mtt(dev, size);</div><div class='add'>+	if (IS_ERR(mtt))</div><div class='add'>+		return mtt;</div><div class='add'>+	level = 1;</div><div class='add'>+</div><div class='add'>+	/* convergence the mtt table. */</div><div class='add'>+	while (mtt-&gt;npages != 1 &amp;&amp; level &lt;= 3) {</div><div class='add'>+		tmp_mtt = erdma_create_scatter_mtt(dev, MTT_SIZE(mtt-&gt;npages));</div><div class='add'>+		if (IS_ERR(tmp_mtt)) {</div><div class='add'>+			ret = PTR_ERR(tmp_mtt);</div><div class='add'>+			goto err_free_mtt;</div><div class='add'>+		}</div><div class='add'>+		erdma_init_middle_mtt(tmp_mtt, mtt);</div><div class='add'>+		tmp_mtt-&gt;low_level = mtt;</div><div class='add'>+		mtt = tmp_mtt;</div><div class='add'>+		level++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (level &gt; 3) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err_free_mtt;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mtt-&gt;level = level;</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ibdev, "top mtt: level:%d, dma_addr 0x%llx\n",</div><div class='add'>+		  mtt-&gt;level, mtt-&gt;dma_addrs[0]);</div><div class='add'>+</div><div class='add'>+	return mtt;</div><div class='add'>+err_free_mtt:</div><div class='add'>+	while (mtt) {</div><div class='add'>+		tmp_mtt = mtt-&gt;low_level;</div><div class='add'>+		erdma_destroy_scatter_mtt(dev, mtt);</div><div class='add'>+		mtt = tmp_mtt;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(ret);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)</div><div class='add'>+{</div><div class='add'>+	struct erdma_mtt *tmp_mtt;</div><div class='add'>+</div><div class='add'>+	if (mtt-&gt;continuous) {</div><div class='add'>+		dma_unmap_single(&amp;dev-&gt;pdev-&gt;dev, mtt-&gt;buf_dma, mtt-&gt;size,</div><div class='add'>+				 DMA_TO_DEVICE);</div><div class='add'>+		kfree(mtt-&gt;buf);</div><div class='add'>+		kfree(mtt);</div><div class='add'>+	} else {</div><div class='add'>+		while (mtt) {</div><div class='add'>+			tmp_mtt = mtt-&gt;low_level;</div><div class='add'>+			erdma_destroy_scatter_mtt(dev, mtt);</div><div class='add'>+			mtt = tmp_mtt;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,</div><div class='add'>+			   u64 start, u64 len, int access, u64 virt,</div><div class='add'>+			   unsigned long req_page_size, bool force_continuous)</div><div class='add'>+{</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	mem-&gt;umem = ib_umem_get(&amp;dev-&gt;ibdev, start, len, access);</div><div class='add'>+	if (IS_ERR(mem-&gt;umem)) {</div><div class='add'>+		ret = PTR_ERR(mem-&gt;umem);</div><div class='add'>+		mem-&gt;umem = NULL;</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mem-&gt;va = virt;</div><div class='add'>+	mem-&gt;len = len;</div><div class='add'>+	mem-&gt;page_size = ib_umem_find_best_pgsz(mem-&gt;umem, req_page_size, virt);</div><div class='add'>+	mem-&gt;page_offset = start &amp; (mem-&gt;page_size - 1);</div><div class='add'>+	mem-&gt;mtt_nents = ib_umem_num_dma_blocks(mem-&gt;umem, mem-&gt;page_size);</div><div class='add'>+	mem-&gt;page_cnt = mem-&gt;mtt_nents;</div><div class='add'>+	mem-&gt;mtt = erdma_create_mtt(dev, MTT_SIZE(mem-&gt;page_cnt),</div><div class='add'>+				    force_continuous);</div><div class='add'>+	if (IS_ERR(mem-&gt;mtt)) {</div><div class='add'>+		ret = PTR_ERR(mem-&gt;mtt);</div><div class='add'>+		goto error_ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	erdma_fill_bottom_mtt(dev, mem);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+error_ret:</div><div class='add'>+	if (mem-&gt;umem) {</div><div class='add'>+		ib_umem_release(mem-&gt;umem);</div><div class='add'>+		mem-&gt;umem = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)</div><div class='add'>+{</div><div class='add'>+	if (mem-&gt;mtt)</div><div class='add'>+		erdma_destroy_mtt(dev, mem-&gt;mtt);</div><div class='add'>+</div><div class='add'>+	if (mem-&gt;umem) {</div><div class='add'>+		ib_umem_release(mem-&gt;umem);</div><div class='add'>+		mem-&gt;umem = NULL;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,</div><div class='add'>+				    u64 dbrecords_va,</div><div class='add'>+				    struct erdma_user_dbrecords_page **dbr_page,</div><div class='add'>+				    dma_addr_t *dma_addr)</div><div class='add'>+{</div><div class='add'>+	struct erdma_user_dbrecords_page *page = NULL;</div><div class='add'>+	int rv = 0;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;ctx-&gt;dbrecords_page_mutex);</div><div class='add'>+</div><div class='add'>+	list_for_each_entry(page, &amp;ctx-&gt;dbrecords_page_list, list)</div><div class='add'>+		if (page-&gt;va == (dbrecords_va &amp; PAGE_MASK))</div><div class='add'>+			goto found;</div><div class='add'>+</div><div class='add'>+	page = kmalloc(sizeof(*page), GFP_KERNEL);</div><div class='add'>+	if (!page) {</div><div class='add'>+		rv = -ENOMEM;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	page-&gt;va = (dbrecords_va &amp; PAGE_MASK);</div><div class='add'>+	page-&gt;refcnt = 0;</div><div class='add'>+</div><div class='add'>+	page-&gt;umem = ib_umem_get(ctx-&gt;ibucontext.device,</div><div class='add'>+				 dbrecords_va &amp; PAGE_MASK, PAGE_SIZE, 0);</div><div class='add'>+	if (IS_ERR(page-&gt;umem)) {</div><div class='add'>+		rv = PTR_ERR(page-&gt;umem);</div><div class='add'>+		kfree(page);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	list_add(&amp;page-&gt;list, &amp;ctx-&gt;dbrecords_page_list);</div><div class='add'>+</div><div class='add'>+found:</div><div class='add'>+	*dma_addr = sg_dma_address(page-&gt;umem-&gt;sgt_append.sgt.sgl) +</div><div class='add'>+		    (dbrecords_va &amp; ~PAGE_MASK);</div><div class='add'>+	*dbr_page = page;</div><div class='add'>+	page-&gt;refcnt++;</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;dbrecords_page_mutex);</div><div class='add'>+	return rv;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void</div><div class='add'>+erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,</div><div class='add'>+			   struct erdma_user_dbrecords_page **dbr_page)</div><div class='add'>+{</div><div class='add'>+	if (!ctx || !(*dbr_page))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;ctx-&gt;dbrecords_page_mutex);</div><div class='add'>+	if (--(*dbr_page)-&gt;refcnt == 0) {</div><div class='add'>+		list_del(&amp;(*dbr_page)-&gt;list);</div><div class='add'>+		ib_umem_release((*dbr_page)-&gt;umem);</div><div class='add'>+		kfree(*dbr_page);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*dbr_page = NULL;</div><div class='add'>+	mutex_unlock(&amp;ctx-&gt;dbrecords_page_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,</div><div class='add'>+			u64 va, u32 len, u64 dbrec_va)</div><div class='add'>+{</div><div class='add'>+	dma_addr_t dbrec_dma;</div><div class='add'>+	u32 rq_offset;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (len &lt; (ALIGN(qp-&gt;attrs.sq_size * SQEBB_SIZE, ERDMA_HW_PAGE_SIZE) +</div><div class='add'>+		   qp-&gt;attrs.rq_size * RQE_SIZE))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ret = get_mtt_entries(qp-&gt;dev, &amp;qp-&gt;user_qp.sq_mem, va,</div><div class='add'>+			      qp-&gt;attrs.sq_size &lt;&lt; SQEBB_SHIFT, 0, va,</div><div class='add'>+			      (SZ_1M - SZ_4K), true);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	rq_offset = ALIGN(qp-&gt;attrs.sq_size &lt;&lt; SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);</div><div class='add'>+	qp-&gt;user_qp.rq_offset = rq_offset;</div><div class='add'>+</div><div class='add'>+	ret = get_mtt_entries(qp-&gt;dev, &amp;qp-&gt;user_qp.rq_mem, va + rq_offset,</div><div class='add'>+			      qp-&gt;attrs.rq_size &lt;&lt; RQE_SHIFT, 0, va + rq_offset,</div><div class='add'>+			      (SZ_1M - SZ_4K), true);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto put_sq_mtt;</div><div class='add'>+</div><div class='add'>+	ret = erdma_map_user_dbrecords(uctx, dbrec_va,</div><div class='add'>+				       &amp;qp-&gt;user_qp.user_dbr_page,</div><div class='add'>+				       &amp;dbrec_dma);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto put_rq_mtt;</div><div class='add'>+</div><div class='add'>+	qp-&gt;user_qp.sq_dbrec_dma = dbrec_dma;</div><div class='add'>+	qp-&gt;user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+put_rq_mtt:</div><div class='add'>+	put_mtt_entries(qp-&gt;dev, &amp;qp-&gt;user_qp.rq_mem);</div><div class='add'>+</div><div class='add'>+put_sq_mtt:</div><div class='add'>+	put_mtt_entries(qp-&gt;dev, &amp;qp-&gt;user_qp.sq_mem);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)</div><div class='add'>+{</div><div class='add'>+	put_mtt_entries(qp-&gt;dev, &amp;qp-&gt;user_qp.sq_mem);</div><div class='add'>+	put_mtt_entries(qp-&gt;dev, &amp;qp-&gt;user_qp.rq_mem);</div><div class='add'>+	erdma_unmap_user_dbrecords(uctx, &amp;qp-&gt;user_qp.user_dbr_page);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,</div><div class='add'>+		    struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct erdma_qp *qp = to_eqp(ibqp);</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibqp-&gt;device);</div><div class='add'>+	struct erdma_ucontext *uctx = rdma_udata_to_drv_context(</div><div class='add'>+		udata, struct erdma_ucontext, ibucontext);</div><div class='add'>+	struct erdma_ureq_create_qp ureq;</div><div class='add'>+	struct erdma_uresp_create_qp uresp;</div><div class='add'>+	void *old_entry;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	ret = erdma_qp_validate_cap(dev, attrs);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_out;</div><div class='add'>+</div><div class='add'>+	ret = erdma_qp_validate_attr(dev, attrs);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_out;</div><div class='add'>+</div><div class='add'>+	qp-&gt;scq = to_ecq(attrs-&gt;send_cq);</div><div class='add'>+	qp-&gt;rcq = to_ecq(attrs-&gt;recv_cq);</div><div class='add'>+	qp-&gt;dev = dev;</div><div class='add'>+	qp-&gt;attrs.cc = dev-&gt;attrs.cc;</div><div class='add'>+</div><div class='add'>+	init_rwsem(&amp;qp-&gt;state_lock);</div><div class='add'>+	kref_init(&amp;qp-&gt;ref);</div><div class='add'>+	init_completion(&amp;qp-&gt;safe_free);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;ibqp.qp_type == IB_QPT_GSI) {</div><div class='add'>+		old_entry = xa_store(&amp;dev-&gt;qp_xa, 1, qp, GFP_KERNEL);</div><div class='add'>+		if (xa_is_err(old_entry))</div><div class='add'>+			ret = xa_err(old_entry);</div><div class='add'>+		else</div><div class='add'>+			qp-&gt;ibqp.qp_num = 1;</div><div class='add'>+	} else {</div><div class='add'>+		ret = xa_alloc_cyclic(&amp;dev-&gt;qp_xa, &amp;qp-&gt;ibqp.qp_num, qp,</div><div class='add'>+				      XA_LIMIT(1, dev-&gt;attrs.max_qp - 1),</div><div class='add'>+				      &amp;dev-&gt;next_alloc_qpn, GFP_KERNEL);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ret &lt; 0) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err_out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	qp-&gt;attrs.sq_size = roundup_pow_of_two(attrs-&gt;cap.max_send_wr *</div><div class='add'>+					       ERDMA_MAX_WQEBB_PER_SQE);</div><div class='add'>+	qp-&gt;attrs.rq_size = roundup_pow_of_two(attrs-&gt;cap.max_recv_wr);</div><div class='add'>+</div><div class='add'>+	if (uctx) {</div><div class='add'>+		ret = ib_copy_from_udata(&amp;ureq, udata,</div><div class='add'>+					 min(sizeof(ureq), udata-&gt;inlen));</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_out_xa;</div><div class='add'>+</div><div class='add'>+		ret = init_user_qp(qp, uctx, ureq.qbuf_va, ureq.qbuf_len,</div><div class='add'>+				   ureq.db_record_va);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_out_xa;</div><div class='add'>+</div><div class='add'>+		memset(&amp;uresp, 0, sizeof(uresp));</div><div class='add'>+</div><div class='add'>+		uresp.num_sqe = qp-&gt;attrs.sq_size;</div><div class='add'>+		uresp.num_rqe = qp-&gt;attrs.rq_size;</div><div class='add'>+		uresp.qp_id = QP_ID(qp);</div><div class='add'>+		uresp.rq_offset = qp-&gt;user_qp.rq_offset;</div><div class='add'>+</div><div class='add'>+		ret = ib_copy_to_udata(udata, &amp;uresp, sizeof(uresp));</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_out_cmd;</div><div class='add'>+	} else {</div><div class='add'>+		ret = init_kernel_qp(dev, qp, attrs);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_out_xa;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	qp-&gt;attrs.max_send_sge = attrs-&gt;cap.max_send_sge;</div><div class='add'>+	qp-&gt;attrs.max_recv_sge = attrs-&gt;cap.max_recv_sge;</div><div class='add'>+</div><div class='add'>+	if (erdma_device_iwarp(qp-&gt;dev))</div><div class='add'>+		qp-&gt;attrs.iwarp.state = ERDMA_QPS_IWARP_IDLE;</div><div class='add'>+	else</div><div class='add'>+		qp-&gt;attrs.rocev2.state = ERDMA_QPS_ROCEV2_RESET;</div><div class='add'>+</div><div class='add'>+	INIT_DELAYED_WORK(&amp;qp-&gt;reflush_dwork, erdma_flush_worker);</div><div class='add'>+</div><div class='add'>+	ret = create_qp_cmd(uctx, qp);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_out_cmd;</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;qp-&gt;lock);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_out_cmd:</div><div class='add'>+	if (uctx)</div><div class='add'>+		free_user_qp(qp, uctx);</div><div class='add'>+	else</div><div class='add'>+		free_kernel_qp(qp);</div><div class='add'>+err_out_xa:</div><div class='add'>+	xa_erase(&amp;dev-&gt;qp_xa, QP_ID(qp));</div><div class='add'>+err_out:</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_create_stag(struct erdma_dev *dev, u32 *stag)</div><div class='add'>+{</div><div class='add'>+	int stag_idx;</div><div class='add'>+</div><div class='add'>+	stag_idx = erdma_alloc_idx(&amp;dev-&gt;res_cb[ERDMA_RES_TYPE_STAG_IDX]);</div><div class='add'>+	if (stag_idx &lt; 0)</div><div class='add'>+		return stag_idx;</div><div class='add'>+</div><div class='add'>+	/* For now, we always let key field be zero. */</div><div class='add'>+	*stag = (stag_idx &lt;&lt; 8);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibpd-&gt;device);</div><div class='add'>+	struct erdma_mr *mr;</div><div class='add'>+	u32 stag;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='add'>+	if (!mr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	ret = erdma_create_stag(dev, &amp;stag);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out_free;</div><div class='add'>+</div><div class='add'>+	mr-&gt;type = ERDMA_MR_TYPE_DMA;</div><div class='add'>+</div><div class='add'>+	mr-&gt;ibmr.lkey = stag;</div><div class='add'>+	mr-&gt;ibmr.rkey = stag;</div><div class='add'>+	mr-&gt;ibmr.pd = ibpd;</div><div class='add'>+	mr-&gt;access = ERDMA_MR_ACC_LR | to_erdma_access_flags(acc);</div><div class='add'>+	ret = regmr_cmd(dev, mr);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out_remove_stag;</div><div class='add'>+</div><div class='add'>+	return &amp;mr-&gt;ibmr;</div><div class='add'>+</div><div class='add'>+out_remove_stag:</div><div class='add'>+	erdma_free_idx(&amp;dev-&gt;res_cb[ERDMA_RES_TYPE_STAG_IDX],</div><div class='add'>+		       mr-&gt;ibmr.lkey &gt;&gt; 8);</div><div class='add'>+</div><div class='add'>+out_free:</div><div class='add'>+	kfree(mr);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(ret);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,</div><div class='add'>+				u32 max_num_sg)</div><div class='add'>+{</div><div class='add'>+	struct erdma_mr *mr;</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibpd-&gt;device);</div><div class='add'>+	int ret;</div><div class='add'>+	u32 stag;</div><div class='add'>+</div><div class='add'>+	if (mr_type != IB_MR_TYPE_MEM_REG)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	if (max_num_sg &gt; ERDMA_MR_MAX_MTT_CNT)</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='add'>+	if (!mr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	ret = erdma_create_stag(dev, &amp;stag);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out_free;</div><div class='add'>+</div><div class='add'>+	mr-&gt;type = ERDMA_MR_TYPE_FRMR;</div><div class='add'>+</div><div class='add'>+	mr-&gt;ibmr.lkey = stag;</div><div class='add'>+	mr-&gt;ibmr.rkey = stag;</div><div class='add'>+	mr-&gt;ibmr.pd = ibpd;</div><div class='add'>+	/* update it in FRMR. */</div><div class='add'>+	mr-&gt;access = ERDMA_MR_ACC_LR | ERDMA_MR_ACC_LW | ERDMA_MR_ACC_RR |</div><div class='add'>+		     ERDMA_MR_ACC_RW;</div><div class='add'>+</div><div class='add'>+	mr-&gt;mem.page_size = PAGE_SIZE; /* update it later. */</div><div class='add'>+	mr-&gt;mem.page_cnt = max_num_sg;</div><div class='add'>+	mr-&gt;mem.mtt = erdma_create_mtt(dev, MTT_SIZE(max_num_sg), true);</div><div class='add'>+	if (IS_ERR(mr-&gt;mem.mtt)) {</div><div class='add'>+		ret = PTR_ERR(mr-&gt;mem.mtt);</div><div class='add'>+		goto out_remove_stag;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = regmr_cmd(dev, mr);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out_destroy_mtt;</div><div class='add'>+</div><div class='add'>+	return &amp;mr-&gt;ibmr;</div><div class='add'>+</div><div class='add'>+out_destroy_mtt:</div><div class='add'>+	erdma_destroy_mtt(dev, mr-&gt;mem.mtt);</div><div class='add'>+</div><div class='add'>+out_remove_stag:</div><div class='add'>+	erdma_free_idx(&amp;dev-&gt;res_cb[ERDMA_RES_TYPE_STAG_IDX],</div><div class='add'>+		       mr-&gt;ibmr.lkey &gt;&gt; 8);</div><div class='add'>+</div><div class='add'>+out_free:</div><div class='add'>+	kfree(mr);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(ret);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_set_page(struct ib_mr *ibmr, u64 addr)</div><div class='add'>+{</div><div class='add'>+	struct erdma_mr *mr = to_emr(ibmr);</div><div class='add'>+</div><div class='add'>+	if (mr-&gt;mem.mtt_nents &gt;= mr-&gt;mem.page_cnt)</div><div class='add'>+		return -1;</div><div class='add'>+</div><div class='add'>+	mr-&gt;mem.mtt-&gt;buf[mr-&gt;mem.mtt_nents] = addr;</div><div class='add'>+	mr-&gt;mem.mtt_nents++;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,</div><div class='add'>+		    unsigned int *sg_offset)</div><div class='add'>+{</div><div class='add'>+	struct erdma_mr *mr = to_emr(ibmr);</div><div class='add'>+	int num;</div><div class='add'>+</div><div class='add'>+	mr-&gt;mem.mtt_nents = 0;</div><div class='add'>+</div><div class='add'>+	num = ib_sg_to_pages(&amp;mr-&gt;ibmr, sg, sg_nents, sg_offset,</div><div class='add'>+			     erdma_set_page);</div><div class='add'>+</div><div class='add'>+	return num;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,</div><div class='add'>+				u64 virt, int access, struct ib_dmah *dmah,</div><div class='add'>+				struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct erdma_mr *mr = NULL;</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibpd-&gt;device);</div><div class='add'>+	u32 stag;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (dmah)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	if (!len || len &gt; dev-&gt;attrs.max_mr_size)</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='add'>+	if (!mr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	ret = get_mtt_entries(dev, &amp;mr-&gt;mem, start, len, access, virt,</div><div class='add'>+			      SZ_2G - SZ_4K, false);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_out_free;</div><div class='add'>+</div><div class='add'>+	ret = erdma_create_stag(dev, &amp;stag);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_out_put_mtt;</div><div class='add'>+</div><div class='add'>+	mr-&gt;ibmr.lkey = mr-&gt;ibmr.rkey = stag;</div><div class='add'>+	mr-&gt;ibmr.pd = ibpd;</div><div class='add'>+	mr-&gt;mem.va = virt;</div><div class='add'>+	mr-&gt;mem.len = len;</div><div class='add'>+	mr-&gt;access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);</div><div class='add'>+	mr-&gt;valid = 1;</div><div class='add'>+	mr-&gt;type = ERDMA_MR_TYPE_NORMAL;</div><div class='add'>+</div><div class='add'>+	ret = regmr_cmd(dev, mr);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_out_mr;</div><div class='add'>+</div><div class='add'>+	return &amp;mr-&gt;ibmr;</div><div class='add'>+</div><div class='add'>+err_out_mr:</div><div class='add'>+	erdma_free_idx(&amp;dev-&gt;res_cb[ERDMA_RES_TYPE_STAG_IDX],</div><div class='add'>+		       mr-&gt;ibmr.lkey &gt;&gt; 8);</div><div class='add'>+</div><div class='add'>+err_out_put_mtt:</div><div class='add'>+	put_mtt_entries(dev, &amp;mr-&gt;mem);</div><div class='add'>+</div><div class='add'>+err_out_free:</div><div class='add'>+	kfree(mr);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(ret);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct erdma_mr *mr;</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibmr-&gt;device);</div><div class='add'>+	struct erdma_cmdq_dereg_mr_req req;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	mr = to_emr(ibmr);</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_RDMA,</div><div class='add'>+				CMDQ_OPCODE_DEREG_MR);</div><div class='add'>+</div><div class='add'>+	req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr-&gt;lkey &gt;&gt; 8) |</div><div class='add'>+		  FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr-&gt;lkey &amp; 0xFF);</div><div class='add'>+</div><div class='add'>+	ret = erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL,</div><div class='add'>+				  true);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	erdma_free_idx(&amp;dev-&gt;res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr-&gt;lkey &gt;&gt; 8);</div><div class='add'>+</div><div class='add'>+	put_mtt_entries(dev, &amp;mr-&gt;mem);</div><div class='add'>+</div><div class='add'>+	kfree(mr);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cq *cq = to_ecq(ibcq);</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibcq-&gt;device);</div><div class='add'>+	struct erdma_ucontext *ctx = rdma_udata_to_drv_context(</div><div class='add'>+		udata, struct erdma_ucontext, ibucontext);</div><div class='add'>+	int err;</div><div class='add'>+	struct erdma_cmdq_destroy_cq_req req;</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_RDMA,</div><div class='add'>+				CMDQ_OPCODE_DESTROY_CQ);</div><div class='add'>+	req.cqn = cq-&gt;cqn;</div><div class='add'>+</div><div class='add'>+	err = erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL,</div><div class='add'>+				  true);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	if (rdma_is_kernel_res(&amp;cq-&gt;ibcq.res)) {</div><div class='add'>+		dma_free_coherent(&amp;dev-&gt;pdev-&gt;dev, cq-&gt;depth &lt;&lt; CQE_SHIFT,</div><div class='add'>+				  cq-&gt;kern_cq.qbuf, cq-&gt;kern_cq.qbuf_dma_addr);</div><div class='add'>+		dma_pool_free(dev-&gt;db_pool, cq-&gt;kern_cq.dbrec,</div><div class='add'>+			      cq-&gt;kern_cq.dbrec_dma);</div><div class='add'>+	} else {</div><div class='add'>+		erdma_unmap_user_dbrecords(ctx, &amp;cq-&gt;user_cq.user_dbr_page);</div><div class='add'>+		put_mtt_entries(dev, &amp;cq-&gt;user_cq.qbuf_mem);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	xa_erase(&amp;dev-&gt;cq_xa, cq-&gt;cqn);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct erdma_qp *qp = to_eqp(ibqp);</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibqp-&gt;device);</div><div class='add'>+	struct erdma_ucontext *ctx = rdma_udata_to_drv_context(</div><div class='add'>+		udata, struct erdma_ucontext, ibucontext);</div><div class='add'>+	struct erdma_cmdq_destroy_qp_req req;</div><div class='add'>+	union erdma_mod_qp_params params;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	down_write(&amp;qp-&gt;state_lock);</div><div class='add'>+	if (erdma_device_iwarp(dev)) {</div><div class='add'>+		params.iwarp.state = ERDMA_QPS_IWARP_ERROR;</div><div class='add'>+		erdma_modify_qp_state_iwarp(qp, &amp;params.iwarp,</div><div class='add'>+					    ERDMA_QPA_IWARP_STATE);</div><div class='add'>+	} else {</div><div class='add'>+		params.rocev2.state = ERDMA_QPS_ROCEV2_ERROR;</div><div class='add'>+		erdma_modify_qp_state_rocev2(qp, &amp;params.rocev2,</div><div class='add'>+					     ERDMA_QPA_ROCEV2_STATE);</div><div class='add'>+	}</div><div class='add'>+	up_write(&amp;qp-&gt;state_lock);</div><div class='add'>+</div><div class='add'>+	cancel_delayed_work_sync(&amp;qp-&gt;reflush_dwork);</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_RDMA,</div><div class='add'>+				CMDQ_OPCODE_DESTROY_QP);</div><div class='add'>+	req.qpn = QP_ID(qp);</div><div class='add'>+</div><div class='add'>+	err = erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL,</div><div class='add'>+				  true);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	erdma_qp_put(qp);</div><div class='add'>+	wait_for_completion(&amp;qp-&gt;safe_free);</div><div class='add'>+</div><div class='add'>+	if (rdma_is_kernel_res(&amp;qp-&gt;ibqp.res)) {</div><div class='add'>+		free_kernel_qp(qp);</div><div class='add'>+	} else {</div><div class='add'>+		put_mtt_entries(dev, &amp;qp-&gt;user_qp.sq_mem);</div><div class='add'>+		put_mtt_entries(dev, &amp;qp-&gt;user_qp.rq_mem);</div><div class='add'>+		erdma_unmap_user_dbrecords(ctx, &amp;qp-&gt;user_qp.user_dbr_page);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;cep)</div><div class='add'>+		erdma_cep_put(qp-&gt;cep);</div><div class='add'>+	xa_erase(&amp;dev-&gt;qp_xa, QP_ID(qp));</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_qp_get_ref(struct ib_qp *ibqp)</div><div class='add'>+{</div><div class='add'>+	erdma_qp_get(to_eqp(ibqp));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_qp_put_ref(struct ib_qp *ibqp)</div><div class='add'>+{</div><div class='add'>+	erdma_qp_put(to_eqp(ibqp));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)</div><div class='add'>+{</div><div class='add'>+	struct rdma_user_mmap_entry *rdma_entry;</div><div class='add'>+	struct erdma_user_mmap_entry *entry;</div><div class='add'>+	pgprot_t prot;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	rdma_entry = rdma_user_mmap_entry_get(ctx, vma);</div><div class='add'>+	if (!rdma_entry)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	entry = to_emmap(rdma_entry);</div><div class='add'>+</div><div class='add'>+	switch (entry-&gt;mmap_flag) {</div><div class='add'>+	case ERDMA_MMAP_IO_NC:</div><div class='add'>+		/* map doorbell. */</div><div class='add'>+		prot = pgprot_device(vma-&gt;vm_page_prot);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+		goto put_entry;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry-&gt;address), PAGE_SIZE,</div><div class='add'>+				prot, rdma_entry);</div><div class='add'>+</div><div class='add'>+put_entry:</div><div class='add'>+	rdma_user_mmap_entry_put(rdma_entry);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry)</div><div class='add'>+{</div><div class='add'>+	struct erdma_user_mmap_entry *entry = to_emmap(rdma_entry);</div><div class='add'>+</div><div class='add'>+	kfree(entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int alloc_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx,</div><div class='add'>+			      bool ext_db_en)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cmdq_ext_db_req req = {};</div><div class='add'>+	u64 val0, val1;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * CAP_SYS_RAWIO is required if hardware does not support extend</div><div class='add'>+	 * doorbell mechanism.</div><div class='add'>+	 */</div><div class='add'>+	if (!ext_db_en &amp;&amp; !capable(CAP_SYS_RAWIO))</div><div class='add'>+		return -EPERM;</div><div class='add'>+</div><div class='add'>+	if (!ext_db_en) {</div><div class='add'>+		ctx-&gt;sdb = dev-&gt;func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET;</div><div class='add'>+		ctx-&gt;rdb = dev-&gt;func_bar_addr + ERDMA_BAR_RQDB_SPACE_OFFSET;</div><div class='add'>+		ctx-&gt;cdb = dev-&gt;func_bar_addr + ERDMA_BAR_CQDB_SPACE_OFFSET;</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_COMMON,</div><div class='add'>+				CMDQ_OPCODE_ALLOC_DB);</div><div class='add'>+</div><div class='add'>+	req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |</div><div class='add'>+		  FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |</div><div class='add'>+		  FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);</div><div class='add'>+</div><div class='add'>+	ret = erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), &amp;val0, &amp;val1,</div><div class='add'>+				  true);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ctx-&gt;ext_db.enable = true;</div><div class='add'>+	ctx-&gt;ext_db.sdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_SDB);</div><div class='add'>+	ctx-&gt;ext_db.rdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_RDB);</div><div class='add'>+	ctx-&gt;ext_db.cdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_CDB);</div><div class='add'>+</div><div class='add'>+	ctx-&gt;sdb = dev-&gt;func_bar_addr + (ctx-&gt;ext_db.sdb_off &lt;&lt; PAGE_SHIFT);</div><div class='add'>+	ctx-&gt;cdb = dev-&gt;func_bar_addr + (ctx-&gt;ext_db.rdb_off &lt;&lt; PAGE_SHIFT);</div><div class='add'>+	ctx-&gt;rdb = dev-&gt;func_bar_addr + (ctx-&gt;ext_db.cdb_off &lt;&lt; PAGE_SHIFT);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void free_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cmdq_ext_db_req req = {};</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!ctx-&gt;ext_db.enable)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_COMMON,</div><div class='add'>+				CMDQ_OPCODE_FREE_DB);</div><div class='add'>+</div><div class='add'>+	req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |</div><div class='add'>+		  FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |</div><div class='add'>+		  FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);</div><div class='add'>+</div><div class='add'>+	req.sdb_off = ctx-&gt;ext_db.sdb_off;</div><div class='add'>+	req.rdb_off = ctx-&gt;ext_db.rdb_off;</div><div class='add'>+	req.cdb_off = ctx-&gt;ext_db.cdb_off;</div><div class='add'>+</div><div class='add'>+	ret = erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL,</div><div class='add'>+				  true);</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_err_ratelimited(&amp;dev-&gt;ibdev,</div><div class='add'>+				      "free db resources failed %d", ret);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx)</div><div class='add'>+{</div><div class='add'>+	rdma_user_mmap_entry_remove(uctx-&gt;sq_db_mmap_entry);</div><div class='add'>+	rdma_user_mmap_entry_remove(uctx-&gt;rq_db_mmap_entry);</div><div class='add'>+	rdma_user_mmap_entry_remove(uctx-&gt;cq_db_mmap_entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct erdma_ucontext *ctx = to_ectx(ibctx);</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibctx-&gt;device);</div><div class='add'>+	int ret;</div><div class='add'>+	struct erdma_uresp_alloc_ctx uresp = {};</div><div class='add'>+</div><div class='add'>+	if (atomic_inc_return(&amp;dev-&gt;num_ctx) &gt; ERDMA_MAX_CONTEXT) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err_out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (udata-&gt;outlen &lt; sizeof(uresp)) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err_out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;ctx-&gt;dbrecords_page_list);</div><div class='add'>+	mutex_init(&amp;ctx-&gt;dbrecords_page_mutex);</div><div class='add'>+</div><div class='add'>+	ret = alloc_db_resources(dev, ctx,</div><div class='add'>+				 !!(dev-&gt;attrs.cap_flags &amp;</div><div class='add'>+				    ERDMA_DEV_CAP_FLAGS_EXTEND_DB));</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_out;</div><div class='add'>+</div><div class='add'>+	ctx-&gt;sq_db_mmap_entry = erdma_user_mmap_entry_insert(</div><div class='add'>+		ctx, (void *)ctx-&gt;sdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &amp;uresp.sdb);</div><div class='add'>+	if (!ctx-&gt;sq_db_mmap_entry) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err_free_ext_db;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ctx-&gt;rq_db_mmap_entry = erdma_user_mmap_entry_insert(</div><div class='add'>+		ctx, (void *)ctx-&gt;rdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &amp;uresp.rdb);</div><div class='add'>+	if (!ctx-&gt;rq_db_mmap_entry) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err_put_mmap_entries;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ctx-&gt;cq_db_mmap_entry = erdma_user_mmap_entry_insert(</div><div class='add'>+		ctx, (void *)ctx-&gt;cdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &amp;uresp.cdb);</div><div class='add'>+	if (!ctx-&gt;cq_db_mmap_entry) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err_put_mmap_entries;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	uresp.dev_id = dev-&gt;pdev-&gt;device;</div><div class='add'>+</div><div class='add'>+	ret = ib_copy_to_udata(udata, &amp;uresp, sizeof(uresp));</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_put_mmap_entries;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_put_mmap_entries:</div><div class='add'>+	erdma_uctx_user_mmap_entries_remove(ctx);</div><div class='add'>+</div><div class='add'>+err_free_ext_db:</div><div class='add'>+	free_db_resources(dev, ctx);</div><div class='add'>+</div><div class='add'>+err_out:</div><div class='add'>+	atomic_dec(&amp;dev-&gt;num_ctx);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_dealloc_ucontext(struct ib_ucontext *ibctx)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibctx-&gt;device);</div><div class='add'>+	struct erdma_ucontext *ctx = to_ectx(ibctx);</div><div class='add'>+</div><div class='add'>+	erdma_uctx_user_mmap_entries_remove(ctx);</div><div class='add'>+	free_db_resources(dev, ctx);</div><div class='add'>+	atomic_dec(&amp;dev-&gt;num_ctx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_attr_to_av(const struct rdma_ah_attr *ah_attr,</div><div class='add'>+			     struct erdma_av *av, u16 sport)</div><div class='add'>+{</div><div class='add'>+	const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);</div><div class='add'>+</div><div class='add'>+	av-&gt;port = rdma_ah_get_port_num(ah_attr);</div><div class='add'>+	av-&gt;sgid_index = grh-&gt;sgid_index;</div><div class='add'>+	av-&gt;hop_limit = grh-&gt;hop_limit;</div><div class='add'>+	av-&gt;traffic_class = grh-&gt;traffic_class;</div><div class='add'>+	av-&gt;sl = rdma_ah_get_sl(ah_attr);</div><div class='add'>+</div><div class='add'>+	av-&gt;flow_label = grh-&gt;flow_label;</div><div class='add'>+	av-&gt;udp_sport = sport;</div><div class='add'>+</div><div class='add'>+	ether_addr_copy(av-&gt;dmac, ah_attr-&gt;roce.dmac);</div><div class='add'>+	memcpy(av-&gt;dgid, grh-&gt;dgid.raw, ERDMA_ROCEV2_GID_SIZE);</div><div class='add'>+</div><div class='add'>+	if (ipv6_addr_v4mapped((struct in6_addr *)&amp;grh-&gt;dgid))</div><div class='add'>+		av-&gt;ntype = ERDMA_NETWORK_TYPE_IPV4;</div><div class='add'>+	else</div><div class='add'>+		av-&gt;ntype = ERDMA_NETWORK_TYPE_IPV6;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_av_to_attr(struct erdma_av *av, struct rdma_ah_attr *ah_attr)</div><div class='add'>+{</div><div class='add'>+	ah_attr-&gt;type = RDMA_AH_ATTR_TYPE_ROCE;</div><div class='add'>+</div><div class='add'>+	rdma_ah_set_sl(ah_attr, av-&gt;sl);</div><div class='add'>+	rdma_ah_set_port_num(ah_attr, av-&gt;port);</div><div class='add'>+	rdma_ah_set_ah_flags(ah_attr, IB_AH_GRH);</div><div class='add'>+</div><div class='add'>+	rdma_ah_set_grh(ah_attr, NULL, av-&gt;flow_label, av-&gt;sgid_index,</div><div class='add'>+			av-&gt;hop_limit, av-&gt;traffic_class);</div><div class='add'>+	rdma_ah_set_dgid_raw(ah_attr, av-&gt;dgid);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ib_qps_to_erdma_qps[ERDMA_PROTO_COUNT][IB_QPS_ERR + 1] = {</div><div class='add'>+	[ERDMA_PROTO_IWARP] = {</div><div class='add'>+		[IB_QPS_RESET] = ERDMA_QPS_IWARP_IDLE,</div><div class='add'>+		[IB_QPS_INIT] = ERDMA_QPS_IWARP_IDLE,</div><div class='add'>+		[IB_QPS_RTR] = ERDMA_QPS_IWARP_RTR,</div><div class='add'>+		[IB_QPS_RTS] = ERDMA_QPS_IWARP_RTS,</div><div class='add'>+		[IB_QPS_SQD] = ERDMA_QPS_IWARP_CLOSING,</div><div class='add'>+		[IB_QPS_SQE] = ERDMA_QPS_IWARP_TERMINATE,</div><div class='add'>+		[IB_QPS_ERR] = ERDMA_QPS_IWARP_ERROR,</div><div class='add'>+	},</div><div class='add'>+	[ERDMA_PROTO_ROCEV2] = {</div><div class='add'>+		[IB_QPS_RESET] = ERDMA_QPS_ROCEV2_RESET,</div><div class='add'>+		[IB_QPS_INIT] = ERDMA_QPS_ROCEV2_INIT,</div><div class='add'>+		[IB_QPS_RTR] = ERDMA_QPS_ROCEV2_RTR,</div><div class='add'>+		[IB_QPS_RTS] = ERDMA_QPS_ROCEV2_RTS,</div><div class='add'>+		[IB_QPS_SQD] = ERDMA_QPS_ROCEV2_SQD,</div><div class='add'>+		[IB_QPS_SQE] = ERDMA_QPS_ROCEV2_SQE,</div><div class='add'>+		[IB_QPS_ERR] = ERDMA_QPS_ROCEV2_ERROR,</div><div class='add'>+	},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int erdma_qps_to_ib_qps[ERDMA_PROTO_COUNT][ERDMA_QPS_ROCEV2_COUNT] = {</div><div class='add'>+	[ERDMA_PROTO_IWARP] = {</div><div class='add'>+		[ERDMA_QPS_IWARP_IDLE] = IB_QPS_INIT,</div><div class='add'>+		[ERDMA_QPS_IWARP_RTR] = IB_QPS_RTR,</div><div class='add'>+		[ERDMA_QPS_IWARP_RTS] = IB_QPS_RTS,</div><div class='add'>+		[ERDMA_QPS_IWARP_CLOSING] = IB_QPS_ERR,</div><div class='add'>+		[ERDMA_QPS_IWARP_TERMINATE] = IB_QPS_ERR,</div><div class='add'>+		[ERDMA_QPS_IWARP_ERROR] = IB_QPS_ERR,</div><div class='add'>+	},</div><div class='add'>+	[ERDMA_PROTO_ROCEV2] = {</div><div class='add'>+		[ERDMA_QPS_ROCEV2_RESET] = IB_QPS_RESET,</div><div class='add'>+		[ERDMA_QPS_ROCEV2_INIT] = IB_QPS_INIT,</div><div class='add'>+		[ERDMA_QPS_ROCEV2_RTR] = IB_QPS_RTR,</div><div class='add'>+		[ERDMA_QPS_ROCEV2_RTS] = IB_QPS_RTS,</div><div class='add'>+		[ERDMA_QPS_ROCEV2_SQD] = IB_QPS_SQD,</div><div class='add'>+		[ERDMA_QPS_ROCEV2_SQE] = IB_QPS_SQE,</div><div class='add'>+		[ERDMA_QPS_ROCEV2_ERROR] = IB_QPS_ERR,</div><div class='add'>+	},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline enum erdma_qps_iwarp ib_to_iwarp_qps(enum ib_qp_state state)</div><div class='add'>+{</div><div class='add'>+	return ib_qps_to_erdma_qps[ERDMA_PROTO_IWARP][state];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline enum erdma_qps_rocev2 ib_to_rocev2_qps(enum ib_qp_state state)</div><div class='add'>+{</div><div class='add'>+	return ib_qps_to_erdma_qps[ERDMA_PROTO_ROCEV2][state];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline enum ib_qp_state iwarp_to_ib_qps(enum erdma_qps_iwarp state)</div><div class='add'>+{</div><div class='add'>+	return erdma_qps_to_ib_qps[ERDMA_PROTO_IWARP][state];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline enum ib_qp_state rocev2_to_ib_qps(enum erdma_qps_rocev2 state)</div><div class='add'>+{</div><div class='add'>+	return erdma_qps_to_ib_qps[ERDMA_PROTO_ROCEV2][state];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_check_qp_attrs(struct erdma_qp *qp, struct ib_qp_attr *attr,</div><div class='add'>+				int attr_mask)</div><div class='add'>+{</div><div class='add'>+	enum ib_qp_state cur_state, nxt_state;</div><div class='add'>+	struct erdma_dev *dev = qp-&gt;dev;</div><div class='add'>+	int ret = -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; ~IB_QP_ATTR_STANDARD_BITS) {</div><div class='add'>+		ret = -EOPNOTSUPP;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if ((attr_mask &amp; IB_QP_PORT) &amp;&amp;</div><div class='add'>+	    !rdma_is_port_valid(&amp;dev-&gt;ibdev, attr-&gt;port_num))</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	if (erdma_device_rocev2(dev)) {</div><div class='add'>+		cur_state = (attr_mask &amp; IB_QP_CUR_STATE) ?</div><div class='add'>+				    attr-&gt;cur_qp_state :</div><div class='add'>+				    rocev2_to_ib_qps(qp-&gt;attrs.rocev2.state);</div><div class='add'>+</div><div class='add'>+		nxt_state = (attr_mask &amp; IB_QP_STATE) ? attr-&gt;qp_state :</div><div class='add'>+							cur_state;</div><div class='add'>+</div><div class='add'>+		if (!ib_modify_qp_is_ok(cur_state, nxt_state, qp-&gt;ibqp.qp_type,</div><div class='add'>+					attr_mask))</div><div class='add'>+			goto out;</div><div class='add'>+</div><div class='add'>+		if ((attr_mask &amp; IB_QP_AV) &amp;&amp;</div><div class='add'>+		    erdma_check_gid_attr(</div><div class='add'>+			    rdma_ah_read_grh(&amp;attr-&gt;ah_attr)-&gt;sgid_attr))</div><div class='add'>+			goto out;</div><div class='add'>+</div><div class='add'>+		if ((attr_mask &amp; IB_QP_PKEY_INDEX) &amp;&amp;</div><div class='add'>+		    attr-&gt;pkey_index &gt;= ERDMA_MAX_PKEYS)</div><div class='add'>+			goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void erdma_init_mod_qp_params_rocev2(</div><div class='add'>+	struct erdma_qp *qp, struct erdma_mod_qp_params_rocev2 *params,</div><div class='add'>+	int *erdma_attr_mask, struct ib_qp_attr *attr, int ib_attr_mask)</div><div class='add'>+{</div><div class='add'>+	enum erdma_qpa_mask_rocev2 to_modify_attrs = 0;</div><div class='add'>+	enum erdma_qps_rocev2 cur_state, nxt_state;</div><div class='add'>+	u16 udp_sport;</div><div class='add'>+</div><div class='add'>+	if (ib_attr_mask &amp; IB_QP_CUR_STATE)</div><div class='add'>+		cur_state = ib_to_rocev2_qps(attr-&gt;cur_qp_state);</div><div class='add'>+	else</div><div class='add'>+		cur_state = qp-&gt;attrs.rocev2.state;</div><div class='add'>+</div><div class='add'>+	if (ib_attr_mask &amp; IB_QP_STATE)</div><div class='add'>+		nxt_state = ib_to_rocev2_qps(attr-&gt;qp_state);</div><div class='add'>+	else</div><div class='add'>+		nxt_state = cur_state;</div><div class='add'>+</div><div class='add'>+	to_modify_attrs |= ERDMA_QPA_ROCEV2_STATE;</div><div class='add'>+	params-&gt;state = nxt_state;</div><div class='add'>+</div><div class='add'>+	if (ib_attr_mask &amp; IB_QP_QKEY) {</div><div class='add'>+		to_modify_attrs |= ERDMA_QPA_ROCEV2_QKEY;</div><div class='add'>+		params-&gt;qkey = attr-&gt;qkey;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ib_attr_mask &amp; IB_QP_SQ_PSN) {</div><div class='add'>+		to_modify_attrs |= ERDMA_QPA_ROCEV2_SQ_PSN;</div><div class='add'>+		params-&gt;sq_psn = attr-&gt;sq_psn;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ib_attr_mask &amp; IB_QP_RQ_PSN) {</div><div class='add'>+		to_modify_attrs |= ERDMA_QPA_ROCEV2_RQ_PSN;</div><div class='add'>+		params-&gt;rq_psn = attr-&gt;rq_psn;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ib_attr_mask &amp; IB_QP_DEST_QPN) {</div><div class='add'>+		to_modify_attrs |= ERDMA_QPA_ROCEV2_DST_QPN;</div><div class='add'>+		params-&gt;dst_qpn = attr-&gt;dest_qp_num;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ib_attr_mask &amp; IB_QP_AV) {</div><div class='add'>+		to_modify_attrs |= ERDMA_QPA_ROCEV2_AV;</div><div class='add'>+		udp_sport = rdma_get_udp_sport(attr-&gt;ah_attr.grh.flow_label,</div><div class='add'>+					       QP_ID(qp), params-&gt;dst_qpn);</div><div class='add'>+		erdma_attr_to_av(&amp;attr-&gt;ah_attr, &amp;params-&gt;av, udp_sport);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*erdma_attr_mask = to_modify_attrs;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,</div><div class='add'>+		    struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct erdma_qp *qp = to_eqp(ibqp);</div><div class='add'>+	union erdma_mod_qp_params params;</div><div class='add'>+	int ret = 0, erdma_attr_mask = 0;</div><div class='add'>+</div><div class='add'>+	down_write(&amp;qp-&gt;state_lock);</div><div class='add'>+</div><div class='add'>+	ret = erdma_check_qp_attrs(qp, attr, attr_mask);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	if (erdma_device_iwarp(qp-&gt;dev)) {</div><div class='add'>+		if (attr_mask &amp; IB_QP_STATE) {</div><div class='add'>+			erdma_attr_mask |= ERDMA_QPA_IWARP_STATE;</div><div class='add'>+			params.iwarp.state = ib_to_iwarp_qps(attr-&gt;qp_state);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ret = erdma_modify_qp_state_iwarp(qp, &amp;params.iwarp,</div><div class='add'>+						  erdma_attr_mask);</div><div class='add'>+	} else {</div><div class='add'>+		erdma_init_mod_qp_params_rocev2(</div><div class='add'>+			qp, &amp;params.rocev2, &amp;erdma_attr_mask, attr, attr_mask);</div><div class='add'>+</div><div class='add'>+		ret = erdma_modify_qp_state_rocev2(qp, &amp;params.rocev2,</div><div class='add'>+						   erdma_attr_mask);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	up_write(&amp;qp-&gt;state_lock);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static enum ib_qp_state query_qp_state(struct erdma_qp *qp)</div><div class='add'>+{</div><div class='add'>+	if (erdma_device_iwarp(qp-&gt;dev))</div><div class='add'>+		return iwarp_to_ib_qps(qp-&gt;attrs.iwarp.state);</div><div class='add'>+	else</div><div class='add'>+		return rocev2_to_ib_qps(qp-&gt;attrs.rocev2.state);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='add'>+		   int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cmdq_query_qp_req_rocev2 req;</div><div class='add'>+	struct erdma_dev *dev;</div><div class='add'>+	struct erdma_qp *qp;</div><div class='add'>+	u64 resp0, resp1;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (ibqp &amp;&amp; qp_attr &amp;&amp; qp_init_attr) {</div><div class='add'>+		qp = to_eqp(ibqp);</div><div class='add'>+		dev = to_edev(ibqp-&gt;device);</div><div class='add'>+	} else {</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	qp_attr-&gt;cap.max_inline_data = ERDMA_MAX_INLINE;</div><div class='add'>+	qp_init_attr-&gt;cap.max_inline_data = ERDMA_MAX_INLINE;</div><div class='add'>+</div><div class='add'>+	qp_attr-&gt;cap.max_send_wr = qp-&gt;attrs.sq_size;</div><div class='add'>+	qp_attr-&gt;cap.max_recv_wr = qp-&gt;attrs.rq_size;</div><div class='add'>+	qp_attr-&gt;cap.max_send_sge = qp-&gt;attrs.max_send_sge;</div><div class='add'>+	qp_attr-&gt;cap.max_recv_sge = qp-&gt;attrs.max_recv_sge;</div><div class='add'>+</div><div class='add'>+	qp_attr-&gt;path_mtu = ib_mtu_int_to_enum(dev-&gt;netdev-&gt;mtu);</div><div class='add'>+	qp_attr-&gt;max_rd_atomic = qp-&gt;attrs.irq_size;</div><div class='add'>+	qp_attr-&gt;max_dest_rd_atomic = qp-&gt;attrs.orq_size;</div><div class='add'>+</div><div class='add'>+	qp_attr-&gt;qp_access_flags = IB_ACCESS_LOCAL_WRITE |</div><div class='add'>+				   IB_ACCESS_REMOTE_WRITE |</div><div class='add'>+				   IB_ACCESS_REMOTE_READ;</div><div class='add'>+</div><div class='add'>+	qp_init_attr-&gt;cap = qp_attr-&gt;cap;</div><div class='add'>+</div><div class='add'>+	if (erdma_device_rocev2(dev)) {</div><div class='add'>+		/* Query hardware to get some attributes */</div><div class='add'>+		erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_RDMA,</div><div class='add'>+					CMDQ_OPCODE_QUERY_QP);</div><div class='add'>+		req.qpn = QP_ID(qp);</div><div class='add'>+</div><div class='add'>+		ret = erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), &amp;resp0,</div><div class='add'>+					  &amp;resp1, true);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+</div><div class='add'>+		qp_attr-&gt;sq_psn =</div><div class='add'>+			FIELD_GET(ERDMA_CMD_QUERY_QP_RESP_SQ_PSN_MASK, resp0);</div><div class='add'>+		qp_attr-&gt;rq_psn =</div><div class='add'>+			FIELD_GET(ERDMA_CMD_QUERY_QP_RESP_RQ_PSN_MASK, resp0);</div><div class='add'>+		qp_attr-&gt;qp_state = rocev2_to_ib_qps(FIELD_GET(</div><div class='add'>+			ERDMA_CMD_QUERY_QP_RESP_QP_STATE_MASK, resp0));</div><div class='add'>+		qp_attr-&gt;cur_qp_state = qp_attr-&gt;qp_state;</div><div class='add'>+		qp_attr-&gt;sq_draining = FIELD_GET(</div><div class='add'>+			ERDMA_CMD_QUERY_QP_RESP_SQ_DRAINING_MASK, resp0);</div><div class='add'>+</div><div class='add'>+		qp_attr-&gt;pkey_index = 0;</div><div class='add'>+		qp_attr-&gt;dest_qp_num = qp-&gt;attrs.rocev2.dst_qpn;</div><div class='add'>+</div><div class='add'>+		if (qp-&gt;ibqp.qp_type == IB_QPT_RC)</div><div class='add'>+			erdma_av_to_attr(&amp;qp-&gt;attrs.rocev2.av,</div><div class='add'>+					 &amp;qp_attr-&gt;ah_attr);</div><div class='add'>+	} else {</div><div class='add'>+		qp_attr-&gt;qp_state = query_qp_state(qp);</div><div class='add'>+		qp_attr-&gt;cur_qp_state = qp_attr-&gt;qp_state;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,</div><div class='add'>+			      struct erdma_ureq_create_cq *ureq)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+	struct erdma_dev *dev = to_edev(cq-&gt;ibcq.device);</div><div class='add'>+</div><div class='add'>+	ret = get_mtt_entries(dev, &amp;cq-&gt;user_cq.qbuf_mem, ureq-&gt;qbuf_va,</div><div class='add'>+			      ureq-&gt;qbuf_len, 0, ureq-&gt;qbuf_va, SZ_64M - SZ_4K,</div><div class='add'>+			      true);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = erdma_map_user_dbrecords(ctx, ureq-&gt;db_record_va,</div><div class='add'>+				       &amp;cq-&gt;user_cq.user_dbr_page,</div><div class='add'>+				       &amp;cq-&gt;user_cq.dbrec_dma);</div><div class='add'>+	if (ret)</div><div class='add'>+		put_mtt_entries(dev, &amp;cq-&gt;user_cq.qbuf_mem);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_init_kernel_cq(struct erdma_cq *cq)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = to_edev(cq-&gt;ibcq.device);</div><div class='add'>+</div><div class='add'>+	cq-&gt;kern_cq.qbuf =</div><div class='add'>+		dma_alloc_coherent(&amp;dev-&gt;pdev-&gt;dev, cq-&gt;depth &lt;&lt; CQE_SHIFT,</div><div class='add'>+				   &amp;cq-&gt;kern_cq.qbuf_dma_addr, GFP_KERNEL);</div><div class='add'>+	if (!cq-&gt;kern_cq.qbuf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cq-&gt;kern_cq.dbrec = dma_pool_zalloc(dev-&gt;db_pool, GFP_KERNEL,</div><div class='add'>+					    &amp;cq-&gt;kern_cq.dbrec_dma);</div><div class='add'>+	if (!cq-&gt;kern_cq.dbrec)</div><div class='add'>+		goto err_out;</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;cq-&gt;kern_cq.lock);</div><div class='add'>+	/* use default cqdb addr */</div><div class='add'>+	cq-&gt;kern_cq.db = dev-&gt;func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_out:</div><div class='add'>+	dma_free_coherent(&amp;dev-&gt;pdev-&gt;dev, cq-&gt;depth &lt;&lt; CQE_SHIFT,</div><div class='add'>+			  cq-&gt;kern_cq.qbuf, cq-&gt;kern_cq.qbuf_dma_addr);</div><div class='add'>+</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		    struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_udata *udata = &amp;attrs-&gt;driver_udata;</div><div class='add'>+	struct erdma_cq *cq = to_ecq(ibcq);</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibcq-&gt;device);</div><div class='add'>+	unsigned int depth = attr-&gt;cqe;</div><div class='add'>+	int ret;</div><div class='add'>+	struct erdma_ucontext *ctx = rdma_udata_to_drv_context(</div><div class='add'>+		udata, struct erdma_ucontext, ibucontext);</div><div class='add'>+</div><div class='add'>+	if (depth &gt; dev-&gt;attrs.max_cqe)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	depth = roundup_pow_of_two(depth);</div><div class='add'>+	cq-&gt;ibcq.cqe = depth;</div><div class='add'>+	cq-&gt;depth = depth;</div><div class='add'>+	cq-&gt;assoc_eqn = attr-&gt;comp_vector + 1;</div><div class='add'>+</div><div class='add'>+	ret = xa_alloc_cyclic(&amp;dev-&gt;cq_xa, &amp;cq-&gt;cqn, cq,</div><div class='add'>+			      XA_LIMIT(1, dev-&gt;attrs.max_cq - 1),</div><div class='add'>+			      &amp;dev-&gt;next_alloc_cqn, GFP_KERNEL);</div><div class='add'>+	if (ret &lt; 0)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (!rdma_is_kernel_res(&amp;ibcq-&gt;res)) {</div><div class='add'>+		struct erdma_ureq_create_cq ureq;</div><div class='add'>+		struct erdma_uresp_create_cq uresp;</div><div class='add'>+</div><div class='add'>+		ret = ib_copy_from_udata(&amp;ureq, udata,</div><div class='add'>+					 min(udata-&gt;inlen, sizeof(ureq)));</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_out_xa;</div><div class='add'>+</div><div class='add'>+		ret = erdma_init_user_cq(ctx, cq, &amp;ureq);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_out_xa;</div><div class='add'>+</div><div class='add'>+		uresp.cq_id = cq-&gt;cqn;</div><div class='add'>+		uresp.num_cqe = depth;</div><div class='add'>+</div><div class='add'>+		ret = ib_copy_to_udata(udata, &amp;uresp,</div><div class='add'>+				       min(sizeof(uresp), udata-&gt;outlen));</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_free_res;</div><div class='add'>+	} else {</div><div class='add'>+		ret = erdma_init_kernel_cq(cq);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_out_xa;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = create_cq_cmd(ctx, cq);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_free_res;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_free_res:</div><div class='add'>+	if (!rdma_is_kernel_res(&amp;ibcq-&gt;res)) {</div><div class='add'>+		erdma_unmap_user_dbrecords(ctx, &amp;cq-&gt;user_cq.user_dbr_page);</div><div class='add'>+		put_mtt_entries(dev, &amp;cq-&gt;user_cq.qbuf_mem);</div><div class='add'>+	} else {</div><div class='add'>+		dma_free_coherent(&amp;dev-&gt;pdev-&gt;dev, depth &lt;&lt; CQE_SHIFT,</div><div class='add'>+				  cq-&gt;kern_cq.qbuf, cq-&gt;kern_cq.qbuf_dma_addr);</div><div class='add'>+		dma_pool_free(dev-&gt;db_pool, cq-&gt;kern_cq.dbrec,</div><div class='add'>+			      cq-&gt;kern_cq.dbrec_dma);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+err_out_xa:</div><div class='add'>+	xa_erase(&amp;dev-&gt;cq_xa, cq-&gt;cqn);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_disassociate_ucontext(struct ib_ucontext *ibcontext)</div><div class='add'>+{</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_set_mtu(struct erdma_dev *dev, u32 mtu)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cmdq_config_mtu_req req;</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_COMMON,</div><div class='add'>+				CMDQ_OPCODE_CONF_MTU);</div><div class='add'>+	req.mtu = mtu;</div><div class='add'>+</div><div class='add'>+	erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL, true);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason)</div><div class='add'>+{</div><div class='add'>+	struct ib_event event;</div><div class='add'>+</div><div class='add'>+	event.device = &amp;dev-&gt;ibdev;</div><div class='add'>+	event.element.port_num = 1;</div><div class='add'>+	event.event = reason;</div><div class='add'>+</div><div class='add'>+	ib_dispatch_event(&amp;event);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+enum counters {</div><div class='add'>+	ERDMA_STATS_TX_REQS_CNT,</div><div class='add'>+	ERDMA_STATS_TX_PACKETS_CNT,</div><div class='add'>+	ERDMA_STATS_TX_BYTES_CNT,</div><div class='add'>+	ERDMA_STATS_TX_DISABLE_DROP_CNT,</div><div class='add'>+	ERDMA_STATS_TX_BPS_METER_DROP_CNT,</div><div class='add'>+	ERDMA_STATS_TX_PPS_METER_DROP_CNT,</div><div class='add'>+</div><div class='add'>+	ERDMA_STATS_RX_PACKETS_CNT,</div><div class='add'>+	ERDMA_STATS_RX_BYTES_CNT,</div><div class='add'>+	ERDMA_STATS_RX_DISABLE_DROP_CNT,</div><div class='add'>+	ERDMA_STATS_RX_BPS_METER_DROP_CNT,</div><div class='add'>+	ERDMA_STATS_RX_PPS_METER_DROP_CNT,</div><div class='add'>+</div><div class='add'>+	ERDMA_STATS_MAX</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct rdma_stat_desc erdma_descs[] = {</div><div class='add'>+	[ERDMA_STATS_TX_REQS_CNT].name = "tx_reqs_cnt",</div><div class='add'>+	[ERDMA_STATS_TX_PACKETS_CNT].name = "tx_packets_cnt",</div><div class='add'>+	[ERDMA_STATS_TX_BYTES_CNT].name = "tx_bytes_cnt",</div><div class='add'>+	[ERDMA_STATS_TX_DISABLE_DROP_CNT].name = "tx_disable_drop_cnt",</div><div class='add'>+	[ERDMA_STATS_TX_BPS_METER_DROP_CNT].name = "tx_bps_limit_drop_cnt",</div><div class='add'>+	[ERDMA_STATS_TX_PPS_METER_DROP_CNT].name = "tx_pps_limit_drop_cnt",</div><div class='add'>+	[ERDMA_STATS_RX_PACKETS_CNT].name = "rx_packets_cnt",</div><div class='add'>+	[ERDMA_STATS_RX_BYTES_CNT].name = "rx_bytes_cnt",</div><div class='add'>+	[ERDMA_STATS_RX_DISABLE_DROP_CNT].name = "rx_disable_drop_cnt",</div><div class='add'>+	[ERDMA_STATS_RX_BPS_METER_DROP_CNT].name = "rx_bps_limit_drop_cnt",</div><div class='add'>+	[ERDMA_STATS_RX_PPS_METER_DROP_CNT].name = "rx_pps_limit_drop_cnt",</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct rdma_hw_stats *erdma_alloc_hw_port_stats(struct ib_device *device,</div><div class='add'>+						u32 port_num)</div><div class='add'>+{</div><div class='add'>+	return rdma_alloc_hw_stats_struct(erdma_descs, ERDMA_STATS_MAX,</div><div class='add'>+					  RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_query_hw_stats(struct erdma_dev *dev,</div><div class='add'>+				struct rdma_hw_stats *stats)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cmdq_query_stats_resp *resp;</div><div class='add'>+	struct erdma_cmdq_query_req req;</div><div class='add'>+	dma_addr_t dma_addr;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_COMMON,</div><div class='add'>+				CMDQ_OPCODE_GET_STATS);</div><div class='add'>+</div><div class='add'>+	resp = dma_pool_zalloc(dev-&gt;resp_pool, GFP_KERNEL, &amp;dma_addr);</div><div class='add'>+	if (!resp)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	req.target_addr = dma_addr;</div><div class='add'>+	req.target_length = ERDMA_HW_RESP_SIZE;</div><div class='add'>+</div><div class='add'>+	err = erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL,</div><div class='add'>+				  true);</div><div class='add'>+	if (err)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	if (resp-&gt;hdr.magic != ERDMA_HW_RESP_MAGIC) {</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	memcpy(&amp;stats-&gt;value[0], &amp;resp-&gt;tx_req_cnt,</div><div class='add'>+	       sizeof(u64) * stats-&gt;num_counters);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	dma_pool_free(dev-&gt;resp_pool, resp, dma_addr);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,</div><div class='add'>+		       u32 port, int index)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibdev);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (port == 0)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	ret = erdma_query_hw_stats(dev, stats);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	return stats-&gt;num_counters;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+enum rdma_link_layer erdma_get_link_layer(struct ib_device *ibdev, u32 port_num)</div><div class='add'>+{</div><div class='add'>+	return IB_LINK_LAYER_ETHERNET;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int erdma_set_gid(struct erdma_dev *dev, u8 op, u32 idx,</div><div class='add'>+			 const union ib_gid *gid)</div><div class='add'>+{</div><div class='add'>+	struct erdma_cmdq_set_gid_req req;</div><div class='add'>+	u8 ntype;</div><div class='add'>+</div><div class='add'>+	req.cfg = FIELD_PREP(ERDMA_CMD_SET_GID_SGID_IDX_MASK, idx) |</div><div class='add'>+		  FIELD_PREP(ERDMA_CMD_SET_GID_OP_MASK, op);</div><div class='add'>+</div><div class='add'>+	if (op == ERDMA_SET_GID_OP_ADD) {</div><div class='add'>+		if (ipv6_addr_v4mapped((struct in6_addr *)gid))</div><div class='add'>+			ntype = ERDMA_NETWORK_TYPE_IPV4;</div><div class='add'>+		else</div><div class='add'>+			ntype = ERDMA_NETWORK_TYPE_IPV6;</div><div class='add'>+</div><div class='add'>+		req.cfg |= FIELD_PREP(ERDMA_CMD_SET_GID_NTYPE_MASK, ntype);</div><div class='add'>+</div><div class='add'>+		memcpy(&amp;req.gid, gid, ERDMA_ROCEV2_GID_SIZE);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_RDMA,</div><div class='add'>+				CMDQ_OPCODE_SET_GID);</div><div class='add'>+	return erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL,</div><div class='add'>+				   true);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_add_gid(const struct ib_gid_attr *attr, void **context)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = to_edev(attr-&gt;device);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = erdma_check_gid_attr(attr);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	return erdma_set_gid(dev, ERDMA_SET_GID_OP_ADD, attr-&gt;index,</div><div class='add'>+			     &amp;attr-&gt;gid);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_del_gid(const struct ib_gid_attr *attr, void **context)</div><div class='add'>+{</div><div class='add'>+	return erdma_set_gid(to_edev(attr-&gt;device), ERDMA_SET_GID_OP_DEL,</div><div class='add'>+			     attr-&gt;index, NULL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)</div><div class='add'>+{</div><div class='add'>+	if (index &gt;= ERDMA_MAX_PKEYS)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	*pkey = ERDMA_DEFAULT_PKEY;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_set_av_cfg(struct erdma_av_cfg *av_cfg, struct erdma_av *av)</div><div class='add'>+{</div><div class='add'>+	av_cfg-&gt;cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_AV_FL_MASK, av-&gt;flow_label) |</div><div class='add'>+		       FIELD_PREP(ERDMA_CMD_CREATE_AV_NTYPE_MASK, av-&gt;ntype);</div><div class='add'>+</div><div class='add'>+	av_cfg-&gt;traffic_class = av-&gt;traffic_class;</div><div class='add'>+	av_cfg-&gt;hop_limit = av-&gt;hop_limit;</div><div class='add'>+	av_cfg-&gt;sl = av-&gt;sl;</div><div class='add'>+</div><div class='add'>+	av_cfg-&gt;udp_sport = av-&gt;udp_sport;</div><div class='add'>+	av_cfg-&gt;sgid_index = av-&gt;sgid_index;</div><div class='add'>+</div><div class='add'>+	ether_addr_copy(av_cfg-&gt;dmac, av-&gt;dmac);</div><div class='add'>+	memcpy(av_cfg-&gt;dgid, av-&gt;dgid, ERDMA_ROCEV2_GID_SIZE);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,</div><div class='add'>+		    struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	const struct ib_global_route *grh =</div><div class='add'>+		rdma_ah_read_grh(init_attr-&gt;ah_attr);</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibah-&gt;device);</div><div class='add'>+	struct erdma_pd *pd = to_epd(ibah-&gt;pd);</div><div class='add'>+	struct erdma_ah *ah = to_eah(ibah);</div><div class='add'>+	struct erdma_cmdq_create_ah_req req;</div><div class='add'>+	u32 udp_sport;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = erdma_check_gid_attr(grh-&gt;sgid_attr);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = erdma_alloc_idx(&amp;dev-&gt;res_cb[ERDMA_RES_TYPE_AH]);</div><div class='add'>+	if (ret &lt; 0)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ah-&gt;ahn = ret;</div><div class='add'>+</div><div class='add'>+	if (grh-&gt;flow_label)</div><div class='add'>+		udp_sport = rdma_flow_label_to_udp_sport(grh-&gt;flow_label);</div><div class='add'>+	else</div><div class='add'>+		udp_sport =</div><div class='add'>+			IB_ROCE_UDP_ENCAP_VALID_PORT_MIN + (ah-&gt;ahn &amp; 0x3FFF);</div><div class='add'>+</div><div class='add'>+	erdma_attr_to_av(init_attr-&gt;ah_attr, &amp;ah-&gt;av, udp_sport);</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_RDMA,</div><div class='add'>+				CMDQ_OPCODE_CREATE_AH);</div><div class='add'>+</div><div class='add'>+	req.pdn = pd-&gt;pdn;</div><div class='add'>+	req.ahn = ah-&gt;ahn;</div><div class='add'>+	erdma_set_av_cfg(&amp;req.av_cfg, &amp;ah-&gt;av);</div><div class='add'>+</div><div class='add'>+	ret = erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL,</div><div class='add'>+				  init_attr-&gt;flags &amp; RDMA_CREATE_AH_SLEEPABLE);</div><div class='add'>+	if (ret) {</div><div class='add'>+		erdma_free_idx(&amp;dev-&gt;res_cb[ERDMA_RES_TYPE_AH], ah-&gt;ahn);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_destroy_ah(struct ib_ah *ibah, u32 flags)</div><div class='add'>+{</div><div class='add'>+	struct erdma_dev *dev = to_edev(ibah-&gt;device);</div><div class='add'>+	struct erdma_pd *pd = to_epd(ibah-&gt;pd);</div><div class='add'>+	struct erdma_ah *ah = to_eah(ibah);</div><div class='add'>+	struct erdma_cmdq_destroy_ah_req req;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	erdma_cmdq_build_reqhdr(&amp;req.hdr, CMDQ_SUBMOD_RDMA,</div><div class='add'>+				CMDQ_OPCODE_DESTROY_AH);</div><div class='add'>+</div><div class='add'>+	req.pdn = pd-&gt;pdn;</div><div class='add'>+	req.ahn = ah-&gt;ahn;</div><div class='add'>+</div><div class='add'>+	ret = erdma_post_cmd_wait(&amp;dev-&gt;cmdq, &amp;req, sizeof(req), NULL, NULL,</div><div class='add'>+				  flags &amp; RDMA_DESTROY_AH_SLEEPABLE);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	erdma_free_idx(&amp;dev-&gt;res_cb[ERDMA_RES_TYPE_AH], ah-&gt;ahn);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)</div><div class='add'>+{</div><div class='add'>+	struct erdma_ah *ah = to_eah(ibah);</div><div class='add'>+</div><div class='add'>+	memset(ah_attr, 0, sizeof(*ah_attr));</div><div class='add'>+	erdma_av_to_attr(&amp;ah-&gt;av, ah_attr);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h<br/>new file mode 100644<br/>index 000000000000..7d8d3fe501d5<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/erdma/erdma_verbs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/erdma/erdma_verbs.h</a></div><div class='hunk'>@@ -0,0 +1,491 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='add'>+</div><div class='add'>+/* Authors: Cheng Xu &lt;chengyou@linux.alibaba.com&gt; */</div><div class='add'>+/*          Kai Shen &lt;kaishen@linux.alibaba.com&gt; */</div><div class='add'>+/* Copyright (c) 2020-2022, Alibaba Group. */</div><div class='add'>+</div><div class='add'>+#ifndef __ERDMA_VERBS_H__</div><div class='add'>+#define __ERDMA_VERBS_H__</div><div class='add'>+</div><div class='add'>+#include "erdma.h"</div><div class='add'>+</div><div class='add'>+/* RDMA Capability. */</div><div class='add'>+#define ERDMA_MAX_PD (128 * 1024)</div><div class='add'>+#define ERDMA_MAX_SEND_WR 8192</div><div class='add'>+#define ERDMA_MAX_ORD 128</div><div class='add'>+#define ERDMA_MAX_IRD 128</div><div class='add'>+#define ERDMA_MAX_SGE_RD 1</div><div class='add'>+#define ERDMA_MAX_CONTEXT (128 * 1024)</div><div class='add'>+#define ERDMA_MAX_SEND_SGE 6</div><div class='add'>+#define ERDMA_MAX_RECV_SGE 1</div><div class='add'>+#define ERDMA_MAX_INLINE (sizeof(struct erdma_sge) * (ERDMA_MAX_SEND_SGE))</div><div class='add'>+#define ERDMA_MAX_FRMR_PA 512</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	ERDMA_MMAP_IO_NC = 0, /* no cache */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_user_mmap_entry {</div><div class='add'>+	struct rdma_user_mmap_entry rdma_entry;</div><div class='add'>+	u64 address;</div><div class='add'>+	u8 mmap_flag;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_ext_db_info {</div><div class='add'>+	bool enable;</div><div class='add'>+	u16 sdb_off;</div><div class='add'>+	u16 rdb_off;</div><div class='add'>+	u16 cdb_off;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_ucontext {</div><div class='add'>+	struct ib_ucontext ibucontext;</div><div class='add'>+</div><div class='add'>+	struct erdma_ext_db_info ext_db;</div><div class='add'>+</div><div class='add'>+	u64 sdb;</div><div class='add'>+	u64 rdb;</div><div class='add'>+	u64 cdb;</div><div class='add'>+</div><div class='add'>+	struct rdma_user_mmap_entry *sq_db_mmap_entry;</div><div class='add'>+	struct rdma_user_mmap_entry *rq_db_mmap_entry;</div><div class='add'>+	struct rdma_user_mmap_entry *cq_db_mmap_entry;</div><div class='add'>+</div><div class='add'>+	/* doorbell records */</div><div class='add'>+	struct list_head dbrecords_page_list;</div><div class='add'>+	struct mutex dbrecords_page_mutex;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_pd {</div><div class='add'>+	struct ib_pd ibpd;</div><div class='add'>+	u32 pdn;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * MemoryRegion definition.</div><div class='add'>+ */</div><div class='add'>+#define ERDMA_MAX_INLINE_MTT_ENTRIES 4</div><div class='add'>+#define MTT_SIZE(mtt_cnt) ((mtt_cnt) &lt;&lt; 3) /* per mtt entry takes 8 Bytes. */</div><div class='add'>+#define ERDMA_MR_MAX_MTT_CNT 524288</div><div class='add'>+#define ERDMA_MTT_ENTRY_SIZE 8</div><div class='add'>+</div><div class='add'>+#define ERDMA_MR_TYPE_NORMAL 0</div><div class='add'>+#define ERDMA_MR_TYPE_FRMR 1</div><div class='add'>+#define ERDMA_MR_TYPE_DMA 2</div><div class='add'>+</div><div class='add'>+#define ERDMA_MR_MTT_0LEVEL 0</div><div class='add'>+#define ERDMA_MR_MTT_1LEVEL 1</div><div class='add'>+</div><div class='add'>+#define ERDMA_MR_ACC_RA BIT(0)</div><div class='add'>+#define ERDMA_MR_ACC_LR BIT(1)</div><div class='add'>+#define ERDMA_MR_ACC_LW BIT(2)</div><div class='add'>+#define ERDMA_MR_ACC_RR BIT(3)</div><div class='add'>+#define ERDMA_MR_ACC_RW BIT(4)</div><div class='add'>+</div><div class='add'>+static inline u8 to_erdma_access_flags(int access)</div><div class='add'>+{</div><div class='add'>+	return (access &amp; IB_ACCESS_REMOTE_READ ? ERDMA_MR_ACC_RR : 0) |</div><div class='add'>+	       (access &amp; IB_ACCESS_LOCAL_WRITE ? ERDMA_MR_ACC_LW : 0) |</div><div class='add'>+	       (access &amp; IB_ACCESS_REMOTE_WRITE ? ERDMA_MR_ACC_RW : 0) |</div><div class='add'>+	       (access &amp; IB_ACCESS_REMOTE_ATOMIC ? ERDMA_MR_ACC_RA : 0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* Hierarchical storage structure for MTT entries */</div><div class='add'>+struct erdma_mtt {</div><div class='add'>+	u64 *buf;</div><div class='add'>+	size_t size;</div><div class='add'>+</div><div class='add'>+	bool continuous;</div><div class='add'>+	union {</div><div class='add'>+		dma_addr_t buf_dma;</div><div class='add'>+		struct {</div><div class='add'>+			dma_addr_t *dma_addrs;</div><div class='add'>+			u32 npages;</div><div class='add'>+			u32 level;</div><div class='add'>+		};</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	struct erdma_mtt *low_level;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_mem {</div><div class='add'>+	struct ib_umem *umem;</div><div class='add'>+	struct erdma_mtt *mtt;</div><div class='add'>+</div><div class='add'>+	u32 page_size;</div><div class='add'>+	u32 page_offset;</div><div class='add'>+	u32 page_cnt;</div><div class='add'>+	u32 mtt_nents;</div><div class='add'>+</div><div class='add'>+	u64 va;</div><div class='add'>+	u64 len;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_mr {</div><div class='add'>+	struct ib_mr ibmr;</div><div class='add'>+	struct erdma_mem mem;</div><div class='add'>+	u8 type;</div><div class='add'>+	u8 access;</div><div class='add'>+	u8 valid;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_user_dbrecords_page {</div><div class='add'>+	struct list_head list;</div><div class='add'>+	struct ib_umem *umem;</div><div class='add'>+	u64 va;</div><div class='add'>+	int refcnt;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_av {</div><div class='add'>+	u8 port;</div><div class='add'>+	u8 hop_limit;</div><div class='add'>+	u8 traffic_class;</div><div class='add'>+	u8 sl;</div><div class='add'>+	u8 sgid_index;</div><div class='add'>+	u16 udp_sport;</div><div class='add'>+	u32 flow_label;</div><div class='add'>+	u8 dmac[ETH_ALEN];</div><div class='add'>+	u8 dgid[ERDMA_ROCEV2_GID_SIZE];</div><div class='add'>+	enum erdma_network_type ntype;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_ah {</div><div class='add'>+	struct ib_ah ibah;</div><div class='add'>+	struct erdma_av av;</div><div class='add'>+	u32 ahn;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_uqp {</div><div class='add'>+	struct erdma_mem sq_mem;</div><div class='add'>+	struct erdma_mem rq_mem;</div><div class='add'>+</div><div class='add'>+	dma_addr_t sq_dbrec_dma;</div><div class='add'>+	dma_addr_t rq_dbrec_dma;</div><div class='add'>+</div><div class='add'>+	struct erdma_user_dbrecords_page *user_dbr_page;</div><div class='add'>+</div><div class='add'>+	u32 rq_offset;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_kqp {</div><div class='add'>+	u16 sq_pi;</div><div class='add'>+	u16 sq_ci;</div><div class='add'>+</div><div class='add'>+	u16 rq_pi;</div><div class='add'>+	u16 rq_ci;</div><div class='add'>+</div><div class='add'>+	u64 *swr_tbl;</div><div class='add'>+	u64 *rwr_tbl;</div><div class='add'>+</div><div class='add'>+	void __iomem *hw_sq_db;</div><div class='add'>+	void __iomem *hw_rq_db;</div><div class='add'>+</div><div class='add'>+	void *sq_buf;</div><div class='add'>+	dma_addr_t sq_buf_dma_addr;</div><div class='add'>+</div><div class='add'>+	void *rq_buf;</div><div class='add'>+	dma_addr_t rq_buf_dma_addr;</div><div class='add'>+</div><div class='add'>+	void *sq_dbrec;</div><div class='add'>+	void *rq_dbrec;</div><div class='add'>+</div><div class='add'>+	dma_addr_t sq_dbrec_dma;</div><div class='add'>+	dma_addr_t rq_dbrec_dma;</div><div class='add'>+</div><div class='add'>+	u8 sig_all;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum erdma_qps_iwarp {</div><div class='add'>+	ERDMA_QPS_IWARP_IDLE = 0,</div><div class='add'>+	ERDMA_QPS_IWARP_RTR = 1,</div><div class='add'>+	ERDMA_QPS_IWARP_RTS = 2,</div><div class='add'>+	ERDMA_QPS_IWARP_CLOSING = 3,</div><div class='add'>+	ERDMA_QPS_IWARP_TERMINATE = 4,</div><div class='add'>+	ERDMA_QPS_IWARP_ERROR = 5,</div><div class='add'>+	ERDMA_QPS_IWARP_UNDEF = 6,</div><div class='add'>+	ERDMA_QPS_IWARP_COUNT = 7,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum erdma_qpa_mask_iwarp {</div><div class='add'>+	ERDMA_QPA_IWARP_STATE = (1 &lt;&lt; 0),</div><div class='add'>+	ERDMA_QPA_IWARP_LLP_HANDLE = (1 &lt;&lt; 2),</div><div class='add'>+	ERDMA_QPA_IWARP_ORD = (1 &lt;&lt; 3),</div><div class='add'>+	ERDMA_QPA_IWARP_IRD = (1 &lt;&lt; 4),</div><div class='add'>+	ERDMA_QPA_IWARP_SQ_SIZE = (1 &lt;&lt; 5),</div><div class='add'>+	ERDMA_QPA_IWARP_RQ_SIZE = (1 &lt;&lt; 6),</div><div class='add'>+	ERDMA_QPA_IWARP_MPA = (1 &lt;&lt; 7),</div><div class='add'>+	ERDMA_QPA_IWARP_CC = (1 &lt;&lt; 8),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum erdma_qps_rocev2 {</div><div class='add'>+	ERDMA_QPS_ROCEV2_RESET = 0,</div><div class='add'>+	ERDMA_QPS_ROCEV2_INIT = 1,</div><div class='add'>+	ERDMA_QPS_ROCEV2_RTR = 2,</div><div class='add'>+	ERDMA_QPS_ROCEV2_RTS = 3,</div><div class='add'>+	ERDMA_QPS_ROCEV2_SQD = 4,</div><div class='add'>+	ERDMA_QPS_ROCEV2_SQE = 5,</div><div class='add'>+	ERDMA_QPS_ROCEV2_ERROR = 6,</div><div class='add'>+	ERDMA_QPS_ROCEV2_COUNT = 7,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum erdma_qpa_mask_rocev2 {</div><div class='add'>+	ERDMA_QPA_ROCEV2_STATE = (1 &lt;&lt; 0),</div><div class='add'>+	ERDMA_QPA_ROCEV2_QKEY = (1 &lt;&lt; 1),</div><div class='add'>+	ERDMA_QPA_ROCEV2_AV = (1 &lt;&lt; 2),</div><div class='add'>+	ERDMA_QPA_ROCEV2_SQ_PSN = (1 &lt;&lt; 3),</div><div class='add'>+	ERDMA_QPA_ROCEV2_RQ_PSN = (1 &lt;&lt; 4),</div><div class='add'>+	ERDMA_QPA_ROCEV2_DST_QPN = (1 &lt;&lt; 5),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum erdma_qp_flags {</div><div class='add'>+	ERDMA_QP_IN_FLUSHING = (1 &lt;&lt; 0),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define ERDMA_QP_ACTIVE 0</div><div class='add'>+#define ERDMA_QP_PASSIVE 1</div><div class='add'>+</div><div class='add'>+struct erdma_mod_qp_params_iwarp {</div><div class='add'>+	enum erdma_qps_iwarp state;</div><div class='add'>+	enum erdma_cc_alg cc;</div><div class='add'>+	u8 qp_type;</div><div class='add'>+	u8 pd_len;</div><div class='add'>+	u32 irq_size;</div><div class='add'>+	u32 orq_size;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_qp_attrs_iwarp {</div><div class='add'>+	enum erdma_qps_iwarp state;</div><div class='add'>+	u32 cookie;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_mod_qp_params_rocev2 {</div><div class='add'>+	enum erdma_qps_rocev2 state;</div><div class='add'>+	u32 qkey;</div><div class='add'>+	u32 sq_psn;</div><div class='add'>+	u32 rq_psn;</div><div class='add'>+	u32 dst_qpn;</div><div class='add'>+	struct erdma_av av;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+union erdma_mod_qp_params {</div><div class='add'>+	struct erdma_mod_qp_params_iwarp iwarp;</div><div class='add'>+	struct erdma_mod_qp_params_rocev2 rocev2;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_qp_attrs_rocev2 {</div><div class='add'>+	enum erdma_qps_rocev2 state;</div><div class='add'>+	u32 qkey;</div><div class='add'>+	u32 dst_qpn;</div><div class='add'>+	struct erdma_av av;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_qp_attrs {</div><div class='add'>+	enum erdma_cc_alg cc; /* Congestion control algorithm */</div><div class='add'>+	u32 sq_size;</div><div class='add'>+	u32 rq_size;</div><div class='add'>+	u32 orq_size;</div><div class='add'>+	u32 irq_size;</div><div class='add'>+	u32 max_send_sge;</div><div class='add'>+	u32 max_recv_sge;</div><div class='add'>+	union {</div><div class='add'>+		struct erdma_qp_attrs_iwarp iwarp;</div><div class='add'>+		struct erdma_qp_attrs_rocev2 rocev2;</div><div class='add'>+	};</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_qp {</div><div class='add'>+	struct ib_qp ibqp;</div><div class='add'>+	struct kref ref;</div><div class='add'>+	struct completion safe_free;</div><div class='add'>+	struct erdma_dev *dev;</div><div class='add'>+	struct erdma_cep *cep;</div><div class='add'>+	struct rw_semaphore state_lock;</div><div class='add'>+</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	struct delayed_work reflush_dwork;</div><div class='add'>+</div><div class='add'>+	union {</div><div class='add'>+		struct erdma_kqp kern_qp;</div><div class='add'>+		struct erdma_uqp user_qp;</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	struct erdma_cq *scq;</div><div class='add'>+	struct erdma_cq *rcq;</div><div class='add'>+</div><div class='add'>+	struct erdma_qp_attrs attrs;</div><div class='add'>+	spinlock_t lock;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_kcq_info {</div><div class='add'>+	void *qbuf;</div><div class='add'>+	dma_addr_t qbuf_dma_addr;</div><div class='add'>+	u32 ci;</div><div class='add'>+	u32 cmdsn;</div><div class='add'>+	u32 notify_cnt;</div><div class='add'>+</div><div class='add'>+	spinlock_t lock;</div><div class='add'>+	u8 __iomem *db;</div><div class='add'>+	u64 *dbrec;</div><div class='add'>+	dma_addr_t dbrec_dma;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_ucq_info {</div><div class='add'>+	struct erdma_mem qbuf_mem;</div><div class='add'>+	struct erdma_user_dbrecords_page *user_dbr_page;</div><div class='add'>+	dma_addr_t dbrec_dma;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct erdma_cq {</div><div class='add'>+	struct ib_cq ibcq;</div><div class='add'>+	u32 cqn;</div><div class='add'>+</div><div class='add'>+	u32 depth;</div><div class='add'>+	u32 assoc_eqn;</div><div class='add'>+</div><div class='add'>+	union {</div><div class='add'>+		struct erdma_kcq_info kern_cq;</div><div class='add'>+		struct erdma_ucq_info user_cq;</div><div class='add'>+	};</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define QP_ID(qp) ((qp)-&gt;ibqp.qp_num)</div><div class='add'>+</div><div class='add'>+static inline struct erdma_qp *find_qp_by_qpn(struct erdma_dev *dev, int id)</div><div class='add'>+{</div><div class='add'>+	return (struct erdma_qp *)xa_load(&amp;dev-&gt;qp_xa, id);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct erdma_cq *find_cq_by_cqn(struct erdma_dev *dev, int id)</div><div class='add'>+{</div><div class='add'>+	return (struct erdma_cq *)xa_load(&amp;dev-&gt;cq_xa, id);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void erdma_qp_get(struct erdma_qp *qp);</div><div class='add'>+void erdma_qp_put(struct erdma_qp *qp);</div><div class='add'>+int erdma_modify_qp_state_iwarp(struct erdma_qp *qp,</div><div class='add'>+				struct erdma_mod_qp_params_iwarp *params,</div><div class='add'>+				int mask);</div><div class='add'>+int erdma_modify_qp_state_rocev2(struct erdma_qp *qp,</div><div class='add'>+				 struct erdma_mod_qp_params_rocev2 *params,</div><div class='add'>+				 int attr_mask);</div><div class='add'>+void erdma_qp_llp_close(struct erdma_qp *qp);</div><div class='add'>+void erdma_qp_cm_drop(struct erdma_qp *qp);</div><div class='add'>+</div><div class='add'>+static inline bool erdma_device_iwarp(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	return dev-&gt;proto == ERDMA_PROTO_IWARP;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool erdma_device_rocev2(struct erdma_dev *dev)</div><div class='add'>+{</div><div class='add'>+	return dev-&gt;proto == ERDMA_PROTO_ROCEV2;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct erdma_ucontext *to_ectx(struct ib_ucontext *ibctx)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibctx, struct erdma_ucontext, ibucontext);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct erdma_pd *to_epd(struct ib_pd *pd)</div><div class='add'>+{</div><div class='add'>+	return container_of(pd, struct erdma_pd, ibpd);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct erdma_mr *to_emr(struct ib_mr *ibmr)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibmr, struct erdma_mr, ibmr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct erdma_qp *to_eqp(struct ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	return container_of(qp, struct erdma_qp, ibqp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct erdma_cq *to_ecq(struct ib_cq *ibcq)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibcq, struct erdma_cq, ibcq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct erdma_ah *to_eah(struct ib_ah *ibah)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibah, struct erdma_ah, ibah);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int erdma_check_gid_attr(const struct ib_gid_attr *attr)</div><div class='add'>+{</div><div class='add'>+	u8 ntype = rdma_gid_attr_network_type(attr);</div><div class='add'>+</div><div class='add'>+	if (ntype != RDMA_NETWORK_IPV4 &amp;&amp; ntype != RDMA_NETWORK_IPV6)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct erdma_user_mmap_entry *</div><div class='add'>+to_emmap(struct rdma_user_mmap_entry *ibmmap)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibmmap, struct erdma_user_mmap_entry, rdma_entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *data);</div><div class='add'>+void erdma_dealloc_ucontext(struct ib_ucontext *ibctx);</div><div class='add'>+int erdma_query_device(struct ib_device *dev, struct ib_device_attr *attr,</div><div class='add'>+		       struct ib_udata *data);</div><div class='add'>+int erdma_get_port_immutable(struct ib_device *dev, u32 port,</div><div class='add'>+			     struct ib_port_immutable *ib_port_immutable);</div><div class='add'>+int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		    struct uverbs_attr_bundle *attrs);</div><div class='add'>+int erdma_query_port(struct ib_device *dev, u32 port,</div><div class='add'>+		     struct ib_port_attr *attr);</div><div class='add'>+int erdma_query_gid(struct ib_device *dev, u32 port, int idx,</div><div class='add'>+		    union ib_gid *gid);</div><div class='add'>+int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *data);</div><div class='add'>+int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);</div><div class='add'>+int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,</div><div class='add'>+		    struct ib_udata *data);</div><div class='add'>+int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,</div><div class='add'>+		   struct ib_qp_init_attr *init_attr);</div><div class='add'>+int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,</div><div class='add'>+		    struct ib_udata *data);</div><div class='add'>+int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);</div><div class='add'>+int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);</div><div class='add'>+void erdma_disassociate_ucontext(struct ib_ucontext *ibcontext);</div><div class='add'>+int erdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);</div><div class='add'>+struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,</div><div class='add'>+				u64 virt, int access, struct ib_dmah *dmah,</div><div class='add'>+				struct ib_udata *udata);</div><div class='add'>+struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int rights);</div><div class='add'>+int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *data);</div><div class='add'>+int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma);</div><div class='add'>+void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry);</div><div class='add'>+void erdma_qp_get_ref(struct ib_qp *ibqp);</div><div class='add'>+void erdma_qp_put_ref(struct ib_qp *ibqp);</div><div class='add'>+struct ib_qp *erdma_get_ibqp(struct ib_device *dev, int id);</div><div class='add'>+int erdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *send_wr,</div><div class='add'>+		    const struct ib_send_wr **bad_send_wr);</div><div class='add'>+int erdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *recv_wr,</div><div class='add'>+		    const struct ib_recv_wr **bad_recv_wr);</div><div class='add'>+int erdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);</div><div class='add'>+void erdma_remove_cqes_of_qp(struct ib_cq *ibcq, u32 qpn);</div><div class='add'>+struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,</div><div class='add'>+				u32 max_num_sg);</div><div class='add'>+int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,</div><div class='add'>+		    unsigned int *sg_offset);</div><div class='add'>+void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason);</div><div class='add'>+void erdma_set_mtu(struct erdma_dev *dev, u32 mtu);</div><div class='add'>+struct rdma_hw_stats *erdma_alloc_hw_port_stats(struct ib_device *device,</div><div class='add'>+						u32 port_num);</div><div class='add'>+int erdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,</div><div class='add'>+		       u32 port, int index);</div><div class='add'>+enum rdma_link_layer erdma_get_link_layer(struct ib_device *ibdev,</div><div class='add'>+					  u32 port_num);</div><div class='add'>+int erdma_add_gid(const struct ib_gid_attr *attr, void **context);</div><div class='add'>+int erdma_del_gid(const struct ib_gid_attr *attr, void **context);</div><div class='add'>+int erdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey);</div><div class='add'>+void erdma_set_av_cfg(struct erdma_av_cfg *av_cfg, struct erdma_av *av);</div><div class='add'>+int erdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,</div><div class='add'>+		    struct ib_udata *udata);</div><div class='add'>+int erdma_destroy_ah(struct ib_ah *ibah, u32 flags);</div><div class='add'>+int erdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);</div><div class='add'>+</div><div class='add'>+#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/Kconfig b/drivers/infiniband/hw/hfi1/Kconfig<br/>index 7b146b67a80f..14b92e12bf29 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/Kconfig?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/Kconfig</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/Kconfig?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/Kconfig</a></div><div class='hunk'>@@ -1,22 +1,23 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='ctx'> config INFINIBAND_HFI1</div><div class='del'>-	tristate "Intel OPA Gen1 support"</div><div class='del'>-	depends on X86_64 &amp;&amp; INFINIBAND_RDMAVT &amp;&amp; I2C</div><div class='add'>+	tristate "Cornelis OPX Gen1 support"</div><div class='add'>+	depends on X86_64 &amp;&amp; INFINIBAND_RDMAVT &amp;&amp; I2C &amp;&amp; !UML</div><div class='ctx'> 	select MMU_NOTIFIER</div><div class='ctx'> 	select CRC32</div><div class='ctx'> 	select I2C_ALGOBIT</div><div class='del'>-	---help---</div><div class='del'>-	This is a low-level driver for Intel OPA Gen1 adapter.</div><div class='add'>+	help</div><div class='add'>+	This is a low-level driver for Cornelis OPX Gen1 adapter.</div><div class='ctx'> config HFI1_DEBUG_SDMA_ORDER</div><div class='ctx'> 	bool "HFI1 SDMA Order debug"</div><div class='ctx'> 	depends on INFINIBAND_HFI1</div><div class='ctx'> 	default n</div><div class='del'>-	---help---</div><div class='add'>+	help</div><div class='ctx'> 	This is a debug flag to test for out of order</div><div class='ctx'> 	sdma completions for unit testing</div><div class='ctx'> config SDMA_VERBOSITY</div><div class='ctx'> 	bool "Config SDMA Verbosity"</div><div class='ctx'> 	depends on INFINIBAND_HFI1</div><div class='ctx'> 	default n</div><div class='del'>-	---help---</div><div class='add'>+	help</div><div class='ctx'> 	This is a configuration flag to enable verbose</div><div class='ctx'> 	SDMA debug</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/Makefile b/drivers/infiniband/hw/hfi1/Makefile<br/>index 4044a8c8dbf4..5d977f363684 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/Makefile?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/Makefile</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/Makefile?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/Makefile</a></div><div class='hunk'>@@ -10,6 +10,7 @@ obj-$(CONFIG_INFINIBAND_HFI1) += hfi1.o</div><div class='ctx'> </div><div class='ctx'> hfi1-y := \</div><div class='ctx'> 	affinity.o \</div><div class='add'>+	aspm.o \</div><div class='ctx'> 	chip.o \</div><div class='ctx'> 	device.o \</div><div class='ctx'> 	driver.o \</div><div class='hunk'>@@ -21,11 +22,16 @@ hfi1-y := \</div><div class='ctx'> 	init.o \</div><div class='ctx'> 	intr.o \</div><div class='ctx'> 	iowait.o \</div><div class='add'>+	ipoib_main.o \</div><div class='add'>+	ipoib_rx.o \</div><div class='add'>+	ipoib_tx.o \</div><div class='ctx'> 	mad.o \</div><div class='ctx'> 	mmu_rb.o \</div><div class='ctx'> 	msix.o \</div><div class='add'>+	netdev_rx.o \</div><div class='ctx'> 	opfn.o \</div><div class='ctx'> 	pcie.o \</div><div class='add'>+	pin_system.o \</div><div class='ctx'> 	pio.o \</div><div class='ctx'> 	pio_copy.o \</div><div class='ctx'> 	platform.o \</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/affinity.c b/drivers/infiniband/hw/hfi1/affinity.c<br/>index 4fe662c3bbc1..ee7fedc67b86 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/affinity.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/affinity.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/affinity.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/affinity.c</a></div><div class='hunk'>@@ -1,52 +1,10 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2015 - 2020 Intel Corporation.</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #include &lt;linux/topology.h&gt;</div><div class='ctx'> #include &lt;linux/cpumask.h&gt;</div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='ctx'> #include &lt;linux/interrupt.h&gt;</div><div class='ctx'> #include &lt;linux/numa.h&gt;</div><div class='ctx'> </div><div class='hunk'>@@ -64,6 +22,7 @@ struct hfi1_affinity_node_list node_affinity = {</div><div class='ctx'> static const char * const irq_type_names[] = {</div><div class='ctx'> 	"SDMA",</div><div class='ctx'> 	"RCVCTXT",</div><div class='add'>+	"NETDEVCTXT",</div><div class='ctx'> 	"GENERAL",</div><div class='ctx'> 	"OTHER",</div><div class='ctx'> };</div><div class='hunk'>@@ -133,9 +92,7 @@ static void cpu_mask_set_put(struct cpu_mask_set *set, int cpu)</div><div class='ctx'> /* Initialize non-HT cpu cores mask */</div><div class='ctx'> void init_real_cpu_mask(void)</div><div class='ctx'> {</div><div class='del'>-	int possible, curr_cpu, i, ht;</div><div class='del'>-</div><div class='del'>-	cpumask_clear(&amp;node_affinity.real_cpu_mask);</div><div class='add'>+	int possible, curr_cpu, ht;</div><div class='ctx'> </div><div class='ctx'> 	/* Start with cpu online mask as the real cpu mask */</div><div class='ctx'> 	cpumask_copy(&amp;node_affinity.real_cpu_mask, cpu_online_mask);</div><div class='hunk'>@@ -151,17 +108,10 @@ void init_real_cpu_mask(void)</div><div class='ctx'> 	 * "real" cores.  Assumes that HT cores are not enumerated in</div><div class='ctx'> 	 * succession (except in the single core case).</div><div class='ctx'> 	 */</div><div class='del'>-	curr_cpu = cpumask_first(&amp;node_affinity.real_cpu_mask);</div><div class='del'>-	for (i = 0; i &lt; possible / ht; i++)</div><div class='del'>-		curr_cpu = cpumask_next(curr_cpu, &amp;node_affinity.real_cpu_mask);</div><div class='del'>-	/*</div><div class='del'>-	 * Step 2.  Remove the remaining HT siblings.  Use cpumask_next() to</div><div class='del'>-	 * skip any gaps.</div><div class='del'>-	 */</div><div class='del'>-	for (; i &lt; possible; i++) {</div><div class='del'>-		cpumask_clear_cpu(curr_cpu, &amp;node_affinity.real_cpu_mask);</div><div class='del'>-		curr_cpu = cpumask_next(curr_cpu, &amp;node_affinity.real_cpu_mask);</div><div class='del'>-	}</div><div class='add'>+	curr_cpu = cpumask_nth(possible / ht, &amp;node_affinity.real_cpu_mask) + 1;</div><div class='add'>+</div><div class='add'>+	/* Step 2.  Remove the remaining HT siblings. */</div><div class='add'>+	cpumask_clear_cpus(&amp;node_affinity.real_cpu_mask, curr_cpu, nr_cpu_ids - curr_cpu);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int node_affinity_init(void)</div><div class='hunk'>@@ -218,6 +168,8 @@ out:</div><div class='ctx'> 	for (node = 0; node &lt; node_affinity.num_possible_nodes; node++)</div><div class='ctx'> 		hfi1_per_node_cntr[node] = 1;</div><div class='ctx'> </div><div class='add'>+	pci_dev_put(dev);</div><div class='add'>+</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -269,11 +221,9 @@ static void node_affinity_add_tail(struct hfi1_affinity_node *entry)</div><div class='ctx'> /* It must be called with node_affinity.lock held */</div><div class='ctx'> static struct hfi1_affinity_node *node_affinity_lookup(int node)</div><div class='ctx'> {</div><div class='del'>-	struct list_head *pos;</div><div class='ctx'> 	struct hfi1_affinity_node *entry;</div><div class='ctx'> </div><div class='del'>-	list_for_each(pos, &amp;node_affinity.list) {</div><div class='del'>-		entry = list_entry(pos, struct hfi1_affinity_node, list);</div><div class='add'>+	list_for_each_entry(entry, &amp;node_affinity.list, list) {</div><div class='ctx'> 		if (entry-&gt;node == node)</div><div class='ctx'> 			return entry;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -387,9 +337,10 @@ static int _dev_comp_vect_cpu_get(struct hfi1_devdata *dd,</div><div class='ctx'> 		       &amp;entry-&gt;def_intr.used);</div><div class='ctx'> </div><div class='ctx'> 	/* If there are non-interrupt CPUs available, use them first */</div><div class='del'>-	if (!cpumask_empty(non_intr_cpus))</div><div class='del'>-		cpu = cpumask_first(non_intr_cpus);</div><div class='del'>-	else /* Otherwise, use interrupt CPUs */</div><div class='add'>+	cpu = cpumask_first(non_intr_cpus);</div><div class='add'>+</div><div class='add'>+	/* Otherwise, use interrupt CPUs */</div><div class='add'>+	if (cpu &gt;= nr_cpu_ids)</div><div class='ctx'> 		cpu = cpumask_first(available_cpus);</div><div class='ctx'> </div><div class='ctx'> 	if (cpu &gt;= nr_cpu_ids) { /* empty */</div><div class='hunk'>@@ -479,6 +430,8 @@ static int _dev_comp_vect_mappings_create(struct hfi1_devdata *dd,</div><div class='ctx'> 			  rvt_get_ibdev_name(&amp;(dd)-&gt;verbs_dev.rdi), i, cpu);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	free_cpumask_var(available_cpus);</div><div class='add'>+	free_cpumask_var(non_intr_cpus);</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> fail:</div><div class='hunk'>@@ -629,22 +582,11 @@ static void _dev_comp_vect_cpu_mask_clean_up(struct hfi1_devdata *dd,</div><div class='ctx'>  */</div><div class='ctx'> int hfi1_dev_affinity_init(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='del'>-	int node = pcibus_to_node(dd-&gt;pcidev-&gt;bus);</div><div class='ctx'> 	struct hfi1_affinity_node *entry;</div><div class='ctx'> 	const struct cpumask *local_mask;</div><div class='ctx'> 	int curr_cpu, possible, i, ret;</div><div class='ctx'> 	bool new_entry = false;</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * If the BIOS does not have the NUMA node information set, select</div><div class='del'>-	 * NUMA 0 so we get consistent performance.</div><div class='del'>-	 */</div><div class='del'>-	if (node &lt; 0) {</div><div class='del'>-		dd_dev_err(dd, "Invalid PCI NUMA node. Performance may be affected\n");</div><div class='del'>-		node = 0;</div><div class='del'>-	}</div><div class='del'>-	dd-&gt;node = node;</div><div class='del'>-</div><div class='ctx'> 	local_mask = cpumask_of_node(dd-&gt;node);</div><div class='ctx'> 	if (cpumask_first(local_mask) &gt;= nr_cpu_ids)</div><div class='ctx'> 		local_mask = topology_core_cpumask(0);</div><div class='hunk'>@@ -657,7 +599,7 @@ int hfi1_dev_affinity_init(struct hfi1_devdata *dd)</div><div class='ctx'> 	 * create an entry in the global affinity structure and initialize it.</div><div class='ctx'> 	 */</div><div class='ctx'> 	if (!entry) {</div><div class='del'>-		entry = node_affinity_allocate(node);</div><div class='add'>+		entry = node_affinity_allocate(dd-&gt;node);</div><div class='ctx'> 		if (!entry) {</div><div class='ctx'> 			dd_dev_err(dd,</div><div class='ctx'> 				   "Unable to allocate global affinity node\n");</div><div class='hunk'>@@ -716,7 +658,7 @@ int hfi1_dev_affinity_init(struct hfi1_devdata *dd)</div><div class='ctx'> 			 * engines, use the same CPU cores as general/control</div><div class='ctx'> 			 * context.</div><div class='ctx'> 			 */</div><div class='del'>-			if (cpumask_weight(&amp;entry-&gt;def_intr.mask) == 0)</div><div class='add'>+			if (cpumask_empty(&amp;entry-&gt;def_intr.mask))</div><div class='ctx'> 				cpumask_copy(&amp;entry-&gt;def_intr.mask,</div><div class='ctx'> 					     &amp;entry-&gt;general_intr_mask);</div><div class='ctx'> 		}</div><div class='hunk'>@@ -736,7 +678,7 @@ int hfi1_dev_affinity_init(struct hfi1_devdata *dd)</div><div class='ctx'> 		 * vectors, use the same CPU core as the general/control</div><div class='ctx'> 		 * context.</div><div class='ctx'> 		 */</div><div class='del'>-		if (cpumask_weight(&amp;entry-&gt;comp_vect_mask) == 0)</div><div class='add'>+		if (cpumask_empty(&amp;entry-&gt;comp_vect_mask))</div><div class='ctx'> 			cpumask_copy(&amp;entry-&gt;comp_vect_mask,</div><div class='ctx'> 				     &amp;entry-&gt;general_intr_mask);</div><div class='ctx'> 	}</div><div class='hunk'>@@ -748,6 +690,7 @@ int hfi1_dev_affinity_init(struct hfi1_devdata *dd)</div><div class='ctx'> 	if (new_entry)</div><div class='ctx'> 		node_affinity_add_tail(entry);</div><div class='ctx'> </div><div class='add'>+	dd-&gt;affinity_entry = entry;</div><div class='ctx'> 	mutex_unlock(&amp;node_affinity.lock);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='hunk'>@@ -763,10 +706,9 @@ void hfi1_dev_affinity_clean_up(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_affinity_node *entry;</div><div class='ctx'> </div><div class='del'>-	if (dd-&gt;node &lt; 0)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='ctx'> 	mutex_lock(&amp;node_affinity.lock);</div><div class='add'>+	if (!dd-&gt;affinity_entry)</div><div class='add'>+		goto unlock;</div><div class='ctx'> 	entry = node_affinity_lookup(dd-&gt;node);</div><div class='ctx'> 	if (!entry)</div><div class='ctx'> 		goto unlock;</div><div class='hunk'>@@ -777,8 +719,8 @@ void hfi1_dev_affinity_clean_up(struct hfi1_devdata *dd)</div><div class='ctx'> 	 */</div><div class='ctx'> 	_dev_comp_vect_cpu_mask_clean_up(dd, entry);</div><div class='ctx'> unlock:</div><div class='add'>+	dd-&gt;affinity_entry = NULL;</div><div class='ctx'> 	mutex_unlock(&amp;node_affinity.lock);</div><div class='del'>-	dd-&gt;node = NUMA_NO_NODE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -913,6 +855,11 @@ static int get_irq_affinity(struct hfi1_devdata *dd,</div><div class='ctx'> 			set = &amp;entry-&gt;rcv_intr;</div><div class='ctx'> 		scnprintf(extra, 64, "ctxt %u", rcd-&gt;ctxt);</div><div class='ctx'> 		break;</div><div class='add'>+	case IRQ_NETDEVCTXT:</div><div class='add'>+		rcd = (struct hfi1_ctxtdata *)msix-&gt;arg;</div><div class='add'>+		set = &amp;entry-&gt;def_intr;</div><div class='add'>+		scnprintf(extra, 64, "ctxt %u", rcd-&gt;ctxt);</div><div class='add'>+		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		dd_dev_err(dd, "Invalid IRQ type %d\n", msix-&gt;type);</div><div class='ctx'> 		return -EINVAL;</div><div class='hunk'>@@ -965,7 +912,6 @@ void hfi1_put_irq_affinity(struct hfi1_devdata *dd,</div><div class='ctx'> 			   struct hfi1_msix_entry *msix)</div><div class='ctx'> {</div><div class='ctx'> 	struct cpu_mask_set *set = NULL;</div><div class='del'>-	struct hfi1_ctxtdata *rcd;</div><div class='ctx'> 	struct hfi1_affinity_node *entry;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;node_affinity.lock);</div><div class='hunk'>@@ -979,12 +925,17 @@ void hfi1_put_irq_affinity(struct hfi1_devdata *dd,</div><div class='ctx'> 	case IRQ_GENERAL:</div><div class='ctx'> 		/* Don't do accounting for general contexts */</div><div class='ctx'> 		break;</div><div class='del'>-	case IRQ_RCVCTXT:</div><div class='del'>-		rcd = (struct hfi1_ctxtdata *)msix-&gt;arg;</div><div class='add'>+	case IRQ_RCVCTXT: {</div><div class='add'>+		struct hfi1_ctxtdata *rcd = msix-&gt;arg;</div><div class='add'>+</div><div class='ctx'> 		/* Don't do accounting for control contexts */</div><div class='ctx'> 		if (rcd-&gt;ctxt != HFI1_CTRL_CTXT)</div><div class='ctx'> 			set = &amp;entry-&gt;rcv_intr;</div><div class='ctx'> 		break;</div><div class='add'>+	}</div><div class='add'>+	case IRQ_NETDEVCTXT:</div><div class='add'>+		set = &amp;entry-&gt;def_intr;</div><div class='add'>+		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		mutex_unlock(&amp;node_affinity.lock);</div><div class='ctx'> 		return;</div><div class='hunk'>@@ -1004,32 +955,23 @@ void hfi1_put_irq_affinity(struct hfi1_devdata *dd,</div><div class='ctx'> static void find_hw_thread_mask(uint hw_thread_no, cpumask_var_t hw_thread_mask,</div><div class='ctx'> 				struct hfi1_affinity_node_list *affinity)</div><div class='ctx'> {</div><div class='del'>-	int possible, curr_cpu, i;</div><div class='del'>-	uint num_cores_per_socket = node_affinity.num_online_cpus /</div><div class='del'>-					affinity-&gt;num_core_siblings /</div><div class='del'>-						node_affinity.num_online_nodes;</div><div class='add'>+	int curr_cpu;</div><div class='add'>+	uint num_cores;</div><div class='ctx'> </div><div class='ctx'> 	cpumask_copy(hw_thread_mask, &amp;affinity-&gt;proc.mask);</div><div class='del'>-	if (affinity-&gt;num_core_siblings &gt; 0) {</div><div class='del'>-		/* Removing other siblings not needed for now */</div><div class='del'>-		possible = cpumask_weight(hw_thread_mask);</div><div class='del'>-		curr_cpu = cpumask_first(hw_thread_mask);</div><div class='del'>-		for (i = 0;</div><div class='del'>-		     i &lt; num_cores_per_socket * node_affinity.num_online_nodes;</div><div class='del'>-		     i++)</div><div class='del'>-			curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);</div><div class='del'>-</div><div class='del'>-		for (; i &lt; possible; i++) {</div><div class='del'>-			cpumask_clear_cpu(curr_cpu, hw_thread_mask);</div><div class='del'>-			curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);</div><div class='del'>-		}</div><div class='ctx'> </div><div class='del'>-		/* Identifying correct HW threads within physical cores */</div><div class='del'>-		cpumask_shift_left(hw_thread_mask, hw_thread_mask,</div><div class='del'>-				   num_cores_per_socket *</div><div class='del'>-				   node_affinity.num_online_nodes *</div><div class='del'>-				   hw_thread_no);</div><div class='del'>-	}</div><div class='add'>+	if (affinity-&gt;num_core_siblings == 0)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	num_cores = rounddown(node_affinity.num_online_cpus / affinity-&gt;num_core_siblings,</div><div class='add'>+				node_affinity.num_online_nodes);</div><div class='add'>+</div><div class='add'>+	/* Removing other siblings not needed for now */</div><div class='add'>+	curr_cpu = cpumask_nth(num_cores * node_affinity.num_online_nodes, hw_thread_mask) + 1;</div><div class='add'>+	cpumask_clear_cpus(hw_thread_mask, curr_cpu, nr_cpu_ids - curr_cpu);</div><div class='add'>+</div><div class='add'>+	/* Identifying correct HW threads within physical cores */</div><div class='add'>+	cpumask_shift_left(hw_thread_mask, hw_thread_mask, num_cores * hw_thread_no);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int hfi1_get_proc_affinity(int node)</div><div class='hunk'>@@ -1038,7 +980,7 @@ int hfi1_get_proc_affinity(int node)</div><div class='ctx'> 	struct hfi1_affinity_node *entry;</div><div class='ctx'> 	cpumask_var_t diff, hw_thread_mask, available_mask, intrs_mask;</div><div class='ctx'> 	const struct cpumask *node_mask,</div><div class='del'>-		*proc_mask = &amp;current-&gt;cpus_allowed;</div><div class='add'>+		*proc_mask = current-&gt;cpus_ptr;</div><div class='ctx'> 	struct hfi1_affinity_node_list *affinity = &amp;node_affinity;</div><div class='ctx'> 	struct cpu_mask_set *set = &amp;affinity-&gt;proc;</div><div class='ctx'> </div><div class='hunk'>@@ -1046,7 +988,7 @@ int hfi1_get_proc_affinity(int node)</div><div class='ctx'> 	 * check whether process/context affinity has already</div><div class='ctx'> 	 * been set</div><div class='ctx'> 	 */</div><div class='del'>-	if (cpumask_weight(proc_mask) == 1) {</div><div class='add'>+	if (current-&gt;nr_cpus_allowed == 1) {</div><div class='ctx'> 		hfi1_cdbg(PROC, "PID %u %s affinity set to CPU %*pbl",</div><div class='ctx'> 			  current-&gt;pid, current-&gt;comm,</div><div class='ctx'> 			  cpumask_pr_args(proc_mask));</div><div class='hunk'>@@ -1057,7 +999,7 @@ int hfi1_get_proc_affinity(int node)</div><div class='ctx'> 		cpu = cpumask_first(proc_mask);</div><div class='ctx'> 		cpumask_set_cpu(cpu, &amp;set-&gt;used);</div><div class='ctx'> 		goto done;</div><div class='del'>-	} else if (cpumask_weight(proc_mask) &lt; cpumask_weight(&amp;set-&gt;mask)) {</div><div class='add'>+	} else if (current-&gt;nr_cpus_allowed &lt; cpumask_weight(&amp;set-&gt;mask)) {</div><div class='ctx'> 		hfi1_cdbg(PROC, "PID %u %s affinity set to CPU set(s) %*pbl",</div><div class='ctx'> 			  current-&gt;pid, current-&gt;comm,</div><div class='ctx'> 			  cpumask_pr_args(proc_mask));</div><div class='hunk'>@@ -1128,22 +1070,19 @@ int hfi1_get_proc_affinity(int node)</div><div class='ctx'> 	 * If HT cores are enabled, identify which HW threads within the</div><div class='ctx'> 	 * physical cores should be used.</div><div class='ctx'> 	 */</div><div class='del'>-	if (affinity-&gt;num_core_siblings &gt; 0) {</div><div class='del'>-		for (i = 0; i &lt; affinity-&gt;num_core_siblings; i++) {</div><div class='del'>-			find_hw_thread_mask(i, hw_thread_mask, affinity);</div><div class='add'>+	for (i = 0; i &lt; affinity-&gt;num_core_siblings; i++) {</div><div class='add'>+		find_hw_thread_mask(i, hw_thread_mask, affinity);</div><div class='ctx'> </div><div class='del'>-			/*</div><div class='del'>-			 * If there's at least one available core for this HW</div><div class='del'>-			 * thread number, stop looking for a core.</div><div class='del'>-			 *</div><div class='del'>-			 * diff will always be not empty at least once in this</div><div class='del'>-			 * loop as the used mask gets reset when</div><div class='del'>-			 * (set-&gt;mask == set-&gt;used) before this loop.</div><div class='del'>-			 */</div><div class='del'>-			cpumask_andnot(diff, hw_thread_mask, &amp;set-&gt;used);</div><div class='del'>-			if (!cpumask_empty(diff))</div><div class='del'>-				break;</div><div class='del'>-		}</div><div class='add'>+		/*</div><div class='add'>+		 * If there's at least one available core for this HW</div><div class='add'>+		 * thread number, stop looking for a core.</div><div class='add'>+		 *</div><div class='add'>+		 * diff will always be not empty at least once in this</div><div class='add'>+		 * loop as the used mask gets reset when</div><div class='add'>+		 * (set-&gt;mask == set-&gt;used) before this loop.</div><div class='add'>+		 */</div><div class='add'>+		if (cpumask_andnot(diff, hw_thread_mask, &amp;set-&gt;used))</div><div class='add'>+			break;</div><div class='ctx'> 	}</div><div class='ctx'> 	hfi1_cdbg(PROC, "Same available HW thread on all physical CPUs: %*pbl",</div><div class='ctx'> 		  cpumask_pr_args(hw_thread_mask));</div><div class='hunk'>@@ -1174,8 +1113,7 @@ int hfi1_get_proc_affinity(int node)</div><div class='ctx'> 	 *    used for process assignments using the same method as</div><div class='ctx'> 	 *    the preferred NUMA node.</div><div class='ctx'> 	 */</div><div class='del'>-	cpumask_andnot(diff, available_mask, intrs_mask);</div><div class='del'>-	if (!cpumask_empty(diff))</div><div class='add'>+	if (cpumask_andnot(diff, available_mask, intrs_mask))</div><div class='ctx'> 		cpumask_copy(available_mask, diff);</div><div class='ctx'> </div><div class='ctx'> 	/* If we don't have CPUs on the preferred node, use other NUMA nodes */</div><div class='hunk'>@@ -1191,8 +1129,7 @@ int hfi1_get_proc_affinity(int node)</div><div class='ctx'> 		 * At first, we don't want to place processes on the same</div><div class='ctx'> 		 * CPUs as interrupt handlers.</div><div class='ctx'> 		 */</div><div class='del'>-		cpumask_andnot(diff, available_mask, intrs_mask);</div><div class='del'>-		if (!cpumask_empty(diff))</div><div class='add'>+		if (cpumask_andnot(diff, available_mask, intrs_mask))</div><div class='ctx'> 			cpumask_copy(available_mask, diff);</div><div class='ctx'> 	}</div><div class='ctx'> 	hfi1_cdbg(PROC, "Possible CPUs for process: %*pbl",</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/affinity.h b/drivers/infiniband/hw/hfi1/affinity.h<br/>index 6a7e6ea4e426..ffdd0d571c7a 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/affinity.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/affinity.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/affinity.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/affinity.h</a></div><div class='hunk'>@@ -1,49 +1,8 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2015 - 2020 Intel Corporation.</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #ifndef _HFI1_AFFINITY_H</div><div class='ctx'> #define _HFI1_AFFINITY_H</div><div class='ctx'> </div><div class='hunk'>@@ -52,6 +11,7 @@</div><div class='ctx'> enum irq_type {</div><div class='ctx'> 	IRQ_SDMA,</div><div class='ctx'> 	IRQ_RCVCTXT,</div><div class='add'>+	IRQ_NETDEVCTXT,</div><div class='ctx'> 	IRQ_GENERAL,</div><div class='ctx'> 	IRQ_OTHER</div><div class='ctx'> };</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/aspm.c b/drivers/infiniband/hw/hfi1/aspm.c<br/>new file mode 100644<br/>index 000000000000..79990d09522b<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/aspm.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/aspm.c</a></div><div class='hunk'>@@ -0,0 +1,270 @@</div><div class='add'>+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)</div><div class='add'>+/*</div><div class='add'>+ * Copyright(c) 2019 Intel Corporation.</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include "aspm.h"</div><div class='add'>+</div><div class='add'>+/* Time after which the timer interrupt will re-enable ASPM */</div><div class='add'>+#define ASPM_TIMER_MS 1000</div><div class='add'>+/* Time for which interrupts are ignored after a timer has been scheduled */</div><div class='add'>+#define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2)</div><div class='add'>+/* Two interrupts within this time trigger ASPM disable */</div><div class='add'>+#define ASPM_TRIGGER_MS 1</div><div class='add'>+#define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull)</div><div class='add'>+#define ASPM_L1_SUPPORTED(reg) \</div><div class='add'>+	((((reg) &amp; PCI_EXP_LNKCAP_ASPMS) &gt;&gt; 10) &amp; 0x2)</div><div class='add'>+</div><div class='add'>+uint aspm_mode = ASPM_MODE_DISABLED;</div><div class='add'>+module_param_named(aspm, aspm_mode, uint, 0444);</div><div class='add'>+MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic");</div><div class='add'>+</div><div class='add'>+static bool aspm_hw_l1_supported(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	struct pci_dev *parent = dd-&gt;pcidev-&gt;bus-&gt;self;</div><div class='add'>+	u32 up, dn;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * If the driver does not have access to the upstream component,</div><div class='add'>+	 * it cannot support ASPM L1 at all.</div><div class='add'>+	 */</div><div class='add'>+	if (!parent)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	pcie_capability_read_dword(dd-&gt;pcidev, PCI_EXP_LNKCAP, &amp;dn);</div><div class='add'>+	dn = ASPM_L1_SUPPORTED(dn);</div><div class='add'>+</div><div class='add'>+	pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &amp;up);</div><div class='add'>+	up = ASPM_L1_SUPPORTED(up);</div><div class='add'>+</div><div class='add'>+	/* ASPM works on A-step but is reported as not supported */</div><div class='add'>+	return (!!dn || is_ax(dd)) &amp;&amp; !!up;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* Set L1 entrance latency for slower entry to L1 */</div><div class='add'>+static void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	u32 l1_ent_lat = 0x4u;</div><div class='add'>+	u32 reg32;</div><div class='add'>+</div><div class='add'>+	pci_read_config_dword(dd-&gt;pcidev, PCIE_CFG_REG_PL3, &amp;reg32);</div><div class='add'>+	reg32 &amp;= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK;</div><div class='add'>+	reg32 |= l1_ent_lat &lt;&lt; PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT;</div><div class='add'>+	pci_write_config_dword(dd-&gt;pcidev, PCIE_CFG_REG_PL3, reg32);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void aspm_hw_enable_l1(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	struct pci_dev *parent = dd-&gt;pcidev-&gt;bus-&gt;self;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * If the driver does not have access to the upstream component,</div><div class='add'>+	 * it cannot support ASPM L1 at all.</div><div class='add'>+	 */</div><div class='add'>+	if (!parent)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	/* Enable ASPM L1 first in upstream component and then downstream */</div><div class='add'>+	pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,</div><div class='add'>+					   PCI_EXP_LNKCTL_ASPMC,</div><div class='add'>+					   PCI_EXP_LNKCTL_ASPM_L1);</div><div class='add'>+	pcie_capability_clear_and_set_word(dd-&gt;pcidev, PCI_EXP_LNKCTL,</div><div class='add'>+					   PCI_EXP_LNKCTL_ASPMC,</div><div class='add'>+					   PCI_EXP_LNKCTL_ASPM_L1);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void aspm_hw_disable_l1(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	struct pci_dev *parent = dd-&gt;pcidev-&gt;bus-&gt;self;</div><div class='add'>+</div><div class='add'>+	/* Disable ASPM L1 first in downstream component and then upstream */</div><div class='add'>+	pcie_capability_clear_and_set_word(dd-&gt;pcidev, PCI_EXP_LNKCTL,</div><div class='add'>+					   PCI_EXP_LNKCTL_ASPMC, 0x0);</div><div class='add'>+	if (parent)</div><div class='add'>+		pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,</div><div class='add'>+						   PCI_EXP_LNKCTL_ASPMC, 0x0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static  void aspm_enable(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	if (dd-&gt;aspm_enabled || aspm_mode == ASPM_MODE_DISABLED ||</div><div class='add'>+	    !dd-&gt;aspm_supported)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	aspm_hw_enable_l1(dd);</div><div class='add'>+	dd-&gt;aspm_enabled = true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static  void aspm_disable(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	if (!dd-&gt;aspm_enabled || aspm_mode == ASPM_MODE_ENABLED)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	aspm_hw_disable_l1(dd);</div><div class='add'>+	dd-&gt;aspm_enabled = false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static  void aspm_disable_inc(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;dd-&gt;aspm_lock, flags);</div><div class='add'>+	aspm_disable(dd);</div><div class='add'>+	atomic_inc(&amp;dd-&gt;aspm_disabled_cnt);</div><div class='add'>+	spin_unlock_irqrestore(&amp;dd-&gt;aspm_lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static  void aspm_enable_dec(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;dd-&gt;aspm_lock, flags);</div><div class='add'>+	if (atomic_dec_and_test(&amp;dd-&gt;aspm_disabled_cnt))</div><div class='add'>+		aspm_enable(dd);</div><div class='add'>+	spin_unlock_irqrestore(&amp;dd-&gt;aspm_lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* ASPM processing for each receive context interrupt */</div><div class='add'>+void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	bool restart_timer;</div><div class='add'>+	bool close_interrupts;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	ktime_t now, prev;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;rcd-&gt;aspm_lock, flags);</div><div class='add'>+	/* PSM contexts are open */</div><div class='add'>+	if (!rcd-&gt;aspm_intr_enable)</div><div class='add'>+		goto unlock;</div><div class='add'>+</div><div class='add'>+	prev = rcd-&gt;aspm_ts_last_intr;</div><div class='add'>+	now = ktime_get();</div><div class='add'>+	rcd-&gt;aspm_ts_last_intr = now;</div><div class='add'>+</div><div class='add'>+	/* An interrupt pair close together in time */</div><div class='add'>+	close_interrupts = ktime_to_ns(ktime_sub(now, prev)) &lt; ASPM_TRIGGER_NS;</div><div class='add'>+</div><div class='add'>+	/* Don't push out our timer till this much time has elapsed */</div><div class='add'>+	restart_timer = ktime_to_ns(ktime_sub(now, rcd-&gt;aspm_ts_timer_sched)) &gt;</div><div class='add'>+				    ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC;</div><div class='add'>+	restart_timer = restart_timer &amp;&amp; close_interrupts;</div><div class='add'>+</div><div class='add'>+	/* Disable ASPM and schedule timer */</div><div class='add'>+	if (rcd-&gt;aspm_enabled &amp;&amp; close_interrupts) {</div><div class='add'>+		aspm_disable_inc(rcd-&gt;dd);</div><div class='add'>+		rcd-&gt;aspm_enabled = false;</div><div class='add'>+		restart_timer = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (restart_timer) {</div><div class='add'>+		mod_timer(&amp;rcd-&gt;aspm_timer,</div><div class='add'>+			  jiffies + msecs_to_jiffies(ASPM_TIMER_MS));</div><div class='add'>+		rcd-&gt;aspm_ts_timer_sched = now;</div><div class='add'>+	}</div><div class='add'>+unlock:</div><div class='add'>+	spin_unlock_irqrestore(&amp;rcd-&gt;aspm_lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* Timer function for re-enabling ASPM in the absence of interrupt activity */</div><div class='add'>+static  void aspm_ctx_timer_function(struct timer_list *t)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ctxtdata *rcd = timer_container_of(rcd, t, aspm_timer);</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;rcd-&gt;aspm_lock, flags);</div><div class='add'>+	aspm_enable_dec(rcd-&gt;dd);</div><div class='add'>+	rcd-&gt;aspm_enabled = true;</div><div class='add'>+	spin_unlock_irqrestore(&amp;rcd-&gt;aspm_lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * Disable interrupt processing for verbs contexts when PSM or VNIC contexts</div><div class='add'>+ * are open.</div><div class='add'>+ */</div><div class='add'>+void aspm_disable_all(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ctxtdata *rcd;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	u16 i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; dd-&gt;first_dyn_alloc_ctxt; i++) {</div><div class='add'>+		rcd = hfi1_rcd_get_by_index(dd, i);</div><div class='add'>+		if (rcd) {</div><div class='add'>+			timer_delete_sync(&amp;rcd-&gt;aspm_timer);</div><div class='add'>+			spin_lock_irqsave(&amp;rcd-&gt;aspm_lock, flags);</div><div class='add'>+			rcd-&gt;aspm_intr_enable = false;</div><div class='add'>+			spin_unlock_irqrestore(&amp;rcd-&gt;aspm_lock, flags);</div><div class='add'>+			hfi1_rcd_put(rcd);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	aspm_disable(dd);</div><div class='add'>+	atomic_set(&amp;dd-&gt;aspm_disabled_cnt, 0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* Re-enable interrupt processing for verbs contexts */</div><div class='add'>+void aspm_enable_all(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ctxtdata *rcd;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	u16 i;</div><div class='add'>+</div><div class='add'>+	aspm_enable(dd);</div><div class='add'>+</div><div class='add'>+	if (aspm_mode != ASPM_MODE_DYNAMIC)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; dd-&gt;first_dyn_alloc_ctxt; i++) {</div><div class='add'>+		rcd = hfi1_rcd_get_by_index(dd, i);</div><div class='add'>+		if (rcd) {</div><div class='add'>+			spin_lock_irqsave(&amp;rcd-&gt;aspm_lock, flags);</div><div class='add'>+			rcd-&gt;aspm_intr_enable = true;</div><div class='add'>+			rcd-&gt;aspm_enabled = true;</div><div class='add'>+			spin_unlock_irqrestore(&amp;rcd-&gt;aspm_lock, flags);</div><div class='add'>+			hfi1_rcd_put(rcd);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static  void aspm_ctx_init(struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	spin_lock_init(&amp;rcd-&gt;aspm_lock);</div><div class='add'>+	timer_setup(&amp;rcd-&gt;aspm_timer, aspm_ctx_timer_function, 0);</div><div class='add'>+	rcd-&gt;aspm_intr_supported = rcd-&gt;dd-&gt;aspm_supported &amp;&amp;</div><div class='add'>+		aspm_mode == ASPM_MODE_DYNAMIC &amp;&amp;</div><div class='add'>+		rcd-&gt;ctxt &lt; rcd-&gt;dd-&gt;first_dyn_alloc_ctxt;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void aspm_init(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ctxtdata *rcd;</div><div class='add'>+	u16 i;</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;dd-&gt;aspm_lock);</div><div class='add'>+	dd-&gt;aspm_supported = aspm_hw_l1_supported(dd);</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; dd-&gt;first_dyn_alloc_ctxt; i++) {</div><div class='add'>+		rcd = hfi1_rcd_get_by_index(dd, i);</div><div class='add'>+		if (rcd)</div><div class='add'>+			aspm_ctx_init(rcd);</div><div class='add'>+		hfi1_rcd_put(rcd);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Start with ASPM disabled */</div><div class='add'>+	aspm_hw_set_l1_ent_latency(dd);</div><div class='add'>+	dd-&gt;aspm_enabled = false;</div><div class='add'>+	aspm_hw_disable_l1(dd);</div><div class='add'>+</div><div class='add'>+	/* Now turn on ASPM if configured */</div><div class='add'>+	aspm_enable_all(dd);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void aspm_exit(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	aspm_disable_all(dd);</div><div class='add'>+</div><div class='add'>+	/* Turn on ASPM on exit to conserve power */</div><div class='add'>+	aspm_enable(dd);</div><div class='add'>+}</div><div class='add'>+</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/aspm.h b/drivers/infiniband/hw/hfi1/aspm.h<br/>index e8133870ee87..c8d92dc13daa 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/aspm.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/aspm.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/aspm.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/aspm.h</a></div><div class='hunk'>@@ -1,49 +1,8 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015-2017 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #ifndef _ASPM_H</div><div class='ctx'> #define _ASPM_H</div><div class='ctx'> </div><div class='hunk'>@@ -57,266 +16,20 @@ enum aspm_mode {</div><div class='ctx'> 	ASPM_MODE_DYNAMIC = 2,	/* ASPM enabled/disabled dynamically */</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* Time after which the timer interrupt will re-enable ASPM */</div><div class='del'>-#define ASPM_TIMER_MS 1000</div><div class='del'>-/* Time for which interrupts are ignored after a timer has been scheduled */</div><div class='del'>-#define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2)</div><div class='del'>-/* Two interrupts within this time trigger ASPM disable */</div><div class='del'>-#define ASPM_TRIGGER_MS 1</div><div class='del'>-#define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull)</div><div class='del'>-#define ASPM_L1_SUPPORTED(reg) \</div><div class='del'>-	(((reg &amp; PCI_EXP_LNKCAP_ASPMS) &gt;&gt; 10) &amp; 0x2)</div><div class='del'>-</div><div class='del'>-static inline bool aspm_hw_l1_supported(struct hfi1_devdata *dd)</div><div class='del'>-{</div><div class='del'>-	struct pci_dev *parent = dd-&gt;pcidev-&gt;bus-&gt;self;</div><div class='del'>-	u32 up, dn;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If the driver does not have access to the upstream component,</div><div class='del'>-	 * it cannot support ASPM L1 at all.</div><div class='del'>-	 */</div><div class='del'>-	if (!parent)</div><div class='del'>-		return false;</div><div class='del'>-</div><div class='del'>-	pcie_capability_read_dword(dd-&gt;pcidev, PCI_EXP_LNKCAP, &amp;dn);</div><div class='del'>-	dn = ASPM_L1_SUPPORTED(dn);</div><div class='del'>-</div><div class='del'>-	pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &amp;up);</div><div class='del'>-	up = ASPM_L1_SUPPORTED(up);</div><div class='del'>-</div><div class='del'>-	/* ASPM works on A-step but is reported as not supported */</div><div class='del'>-	return (!!dn || is_ax(dd)) &amp;&amp; !!up;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* Set L1 entrance latency for slower entry to L1 */</div><div class='del'>-static inline void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd)</div><div class='del'>-{</div><div class='del'>-	u32 l1_ent_lat = 0x4u;</div><div class='del'>-	u32 reg32;</div><div class='del'>-</div><div class='del'>-	pci_read_config_dword(dd-&gt;pcidev, PCIE_CFG_REG_PL3, &amp;reg32);</div><div class='del'>-	reg32 &amp;= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK;</div><div class='del'>-	reg32 |= l1_ent_lat &lt;&lt; PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT;</div><div class='del'>-	pci_write_config_dword(dd-&gt;pcidev, PCIE_CFG_REG_PL3, reg32);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void aspm_hw_enable_l1(struct hfi1_devdata *dd)</div><div class='del'>-{</div><div class='del'>-	struct pci_dev *parent = dd-&gt;pcidev-&gt;bus-&gt;self;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * If the driver does not have access to the upstream component,</div><div class='del'>-	 * it cannot support ASPM L1 at all.</div><div class='del'>-	 */</div><div class='del'>-	if (!parent)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	/* Enable ASPM L1 first in upstream component and then downstream */</div><div class='del'>-	pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,</div><div class='del'>-					   PCI_EXP_LNKCTL_ASPMC,</div><div class='del'>-					   PCI_EXP_LNKCTL_ASPM_L1);</div><div class='del'>-	pcie_capability_clear_and_set_word(dd-&gt;pcidev, PCI_EXP_LNKCTL,</div><div class='del'>-					   PCI_EXP_LNKCTL_ASPMC,</div><div class='del'>-					   PCI_EXP_LNKCTL_ASPM_L1);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void aspm_hw_disable_l1(struct hfi1_devdata *dd)</div><div class='del'>-{</div><div class='del'>-	struct pci_dev *parent = dd-&gt;pcidev-&gt;bus-&gt;self;</div><div class='del'>-</div><div class='del'>-	/* Disable ASPM L1 first in downstream component and then upstream */</div><div class='del'>-	pcie_capability_clear_and_set_word(dd-&gt;pcidev, PCI_EXP_LNKCTL,</div><div class='del'>-					   PCI_EXP_LNKCTL_ASPMC, 0x0);</div><div class='del'>-	if (parent)</div><div class='del'>-		pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,</div><div class='del'>-						   PCI_EXP_LNKCTL_ASPMC, 0x0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void aspm_enable(struct hfi1_devdata *dd)</div><div class='del'>-{</div><div class='del'>-	if (dd-&gt;aspm_enabled || aspm_mode == ASPM_MODE_DISABLED ||</div><div class='del'>-	    !dd-&gt;aspm_supported)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	aspm_hw_enable_l1(dd);</div><div class='del'>-	dd-&gt;aspm_enabled = true;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void aspm_disable(struct hfi1_devdata *dd)</div><div class='del'>-{</div><div class='del'>-	if (!dd-&gt;aspm_enabled || aspm_mode == ASPM_MODE_ENABLED)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	aspm_hw_disable_l1(dd);</div><div class='del'>-	dd-&gt;aspm_enabled = false;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void aspm_disable_inc(struct hfi1_devdata *dd)</div><div class='del'>-{</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;dd-&gt;aspm_lock, flags);</div><div class='del'>-	aspm_disable(dd);</div><div class='del'>-	atomic_inc(&amp;dd-&gt;aspm_disabled_cnt);</div><div class='del'>-	spin_unlock_irqrestore(&amp;dd-&gt;aspm_lock, flags);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void aspm_enable_dec(struct hfi1_devdata *dd)</div><div class='del'>-{</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;dd-&gt;aspm_lock, flags);</div><div class='del'>-	if (atomic_dec_and_test(&amp;dd-&gt;aspm_disabled_cnt))</div><div class='del'>-		aspm_enable(dd);</div><div class='del'>-	spin_unlock_irqrestore(&amp;dd-&gt;aspm_lock, flags);</div><div class='del'>-}</div><div class='add'>+void aspm_init(struct hfi1_devdata *dd);</div><div class='add'>+void aspm_exit(struct hfi1_devdata *dd);</div><div class='add'>+void aspm_hw_disable_l1(struct hfi1_devdata *dd);</div><div class='add'>+void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd);</div><div class='add'>+void aspm_disable_all(struct hfi1_devdata *dd);</div><div class='add'>+void aspm_enable_all(struct hfi1_devdata *dd);</div><div class='ctx'> </div><div class='del'>-/* ASPM processing for each receive context interrupt */</div><div class='ctx'> static inline void aspm_ctx_disable(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> {</div><div class='del'>-	bool restart_timer;</div><div class='del'>-	bool close_interrupts;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	ktime_t now, prev;</div><div class='del'>-</div><div class='ctx'> 	/* Quickest exit for minimum impact */</div><div class='del'>-	if (!rcd-&gt;aspm_intr_supported)</div><div class='add'>+	if (likely(!rcd-&gt;aspm_intr_supported))</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='del'>-	spin_lock_irqsave(&amp;rcd-&gt;aspm_lock, flags);</div><div class='del'>-	/* PSM contexts are open */</div><div class='del'>-	if (!rcd-&gt;aspm_intr_enable)</div><div class='del'>-		goto unlock;</div><div class='del'>-</div><div class='del'>-	prev = rcd-&gt;aspm_ts_last_intr;</div><div class='del'>-	now = ktime_get();</div><div class='del'>-	rcd-&gt;aspm_ts_last_intr = now;</div><div class='del'>-</div><div class='del'>-	/* An interrupt pair close together in time */</div><div class='del'>-	close_interrupts = ktime_to_ns(ktime_sub(now, prev)) &lt; ASPM_TRIGGER_NS;</div><div class='del'>-</div><div class='del'>-	/* Don't push out our timer till this much time has elapsed */</div><div class='del'>-	restart_timer = ktime_to_ns(ktime_sub(now, rcd-&gt;aspm_ts_timer_sched)) &gt;</div><div class='del'>-				    ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC;</div><div class='del'>-	restart_timer = restart_timer &amp;&amp; close_interrupts;</div><div class='del'>-</div><div class='del'>-	/* Disable ASPM and schedule timer */</div><div class='del'>-	if (rcd-&gt;aspm_enabled &amp;&amp; close_interrupts) {</div><div class='del'>-		aspm_disable_inc(rcd-&gt;dd);</div><div class='del'>-		rcd-&gt;aspm_enabled = false;</div><div class='del'>-		restart_timer = true;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (restart_timer) {</div><div class='del'>-		mod_timer(&amp;rcd-&gt;aspm_timer,</div><div class='del'>-			  jiffies + msecs_to_jiffies(ASPM_TIMER_MS));</div><div class='del'>-		rcd-&gt;aspm_ts_timer_sched = now;</div><div class='del'>-	}</div><div class='del'>-unlock:</div><div class='del'>-	spin_unlock_irqrestore(&amp;rcd-&gt;aspm_lock, flags);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* Timer function for re-enabling ASPM in the absence of interrupt activity */</div><div class='del'>-static inline void aspm_ctx_timer_function(struct timer_list *t)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer);</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;rcd-&gt;aspm_lock, flags);</div><div class='del'>-	aspm_enable_dec(rcd-&gt;dd);</div><div class='del'>-	rcd-&gt;aspm_enabled = true;</div><div class='del'>-	spin_unlock_irqrestore(&amp;rcd-&gt;aspm_lock, flags);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Disable interrupt processing for verbs contexts when PSM or VNIC contexts</div><div class='del'>- * are open.</div><div class='del'>- */</div><div class='del'>-static inline void aspm_disable_all(struct hfi1_devdata *dd)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_ctxtdata *rcd;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	u16 i;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; dd-&gt;first_dyn_alloc_ctxt; i++) {</div><div class='del'>-		rcd = hfi1_rcd_get_by_index(dd, i);</div><div class='del'>-		if (rcd) {</div><div class='del'>-			del_timer_sync(&amp;rcd-&gt;aspm_timer);</div><div class='del'>-			spin_lock_irqsave(&amp;rcd-&gt;aspm_lock, flags);</div><div class='del'>-			rcd-&gt;aspm_intr_enable = false;</div><div class='del'>-			spin_unlock_irqrestore(&amp;rcd-&gt;aspm_lock, flags);</div><div class='del'>-			hfi1_rcd_put(rcd);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	aspm_disable(dd);</div><div class='del'>-	atomic_set(&amp;dd-&gt;aspm_disabled_cnt, 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* Re-enable interrupt processing for verbs contexts */</div><div class='del'>-static inline void aspm_enable_all(struct hfi1_devdata *dd)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_ctxtdata *rcd;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	u16 i;</div><div class='del'>-</div><div class='del'>-	aspm_enable(dd);</div><div class='del'>-</div><div class='del'>-	if (aspm_mode != ASPM_MODE_DYNAMIC)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; dd-&gt;first_dyn_alloc_ctxt; i++) {</div><div class='del'>-		rcd = hfi1_rcd_get_by_index(dd, i);</div><div class='del'>-		if (rcd) {</div><div class='del'>-			spin_lock_irqsave(&amp;rcd-&gt;aspm_lock, flags);</div><div class='del'>-			rcd-&gt;aspm_intr_enable = true;</div><div class='del'>-			rcd-&gt;aspm_enabled = true;</div><div class='del'>-			spin_unlock_irqrestore(&amp;rcd-&gt;aspm_lock, flags);</div><div class='del'>-			hfi1_rcd_put(rcd);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void aspm_ctx_init(struct hfi1_ctxtdata *rcd)</div><div class='del'>-{</div><div class='del'>-	spin_lock_init(&amp;rcd-&gt;aspm_lock);</div><div class='del'>-	timer_setup(&amp;rcd-&gt;aspm_timer, aspm_ctx_timer_function, 0);</div><div class='del'>-	rcd-&gt;aspm_intr_supported = rcd-&gt;dd-&gt;aspm_supported &amp;&amp;</div><div class='del'>-		aspm_mode == ASPM_MODE_DYNAMIC &amp;&amp;</div><div class='del'>-		rcd-&gt;ctxt &lt; rcd-&gt;dd-&gt;first_dyn_alloc_ctxt;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void aspm_init(struct hfi1_devdata *dd)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_ctxtdata *rcd;</div><div class='del'>-	u16 i;</div><div class='del'>-</div><div class='del'>-	spin_lock_init(&amp;dd-&gt;aspm_lock);</div><div class='del'>-	dd-&gt;aspm_supported = aspm_hw_l1_supported(dd);</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; dd-&gt;first_dyn_alloc_ctxt; i++) {</div><div class='del'>-		rcd = hfi1_rcd_get_by_index(dd, i);</div><div class='del'>-		if (rcd)</div><div class='del'>-			aspm_ctx_init(rcd);</div><div class='del'>-		hfi1_rcd_put(rcd);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Start with ASPM disabled */</div><div class='del'>-	aspm_hw_set_l1_ent_latency(dd);</div><div class='del'>-	dd-&gt;aspm_enabled = false;</div><div class='del'>-	aspm_hw_disable_l1(dd);</div><div class='del'>-</div><div class='del'>-	/* Now turn on ASPM if configured */</div><div class='del'>-	aspm_enable_all(dd);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void aspm_exit(struct hfi1_devdata *dd)</div><div class='del'>-{</div><div class='del'>-	aspm_disable_all(dd);</div><div class='del'>-</div><div class='del'>-	/* Turn on ASPM on exit to conserve power */</div><div class='del'>-	aspm_enable(dd);</div><div class='add'>+	__aspm_ctx_disable(rcd);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> #endif /* _ASPM_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c<br/>index 310105d4e3de..0781ab756d44 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/chip.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/chip.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/chip.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/chip.c</a></div><div class='hunk'>@@ -1,48 +1,7 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2015 - 2020 Intel Corporation.</div><div class='add'>+ * Copyright(c) 2021 Cornelis Networks.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -66,10 +25,7 @@</div><div class='ctx'> #include "affinity.h"</div><div class='ctx'> #include "debugfs.h"</div><div class='ctx'> #include "fault.h"</div><div class='del'>-</div><div class='del'>-uint kdeth_qp;</div><div class='del'>-module_param_named(kdeth_qp, kdeth_qp, uint, S_IRUGO);</div><div class='del'>-MODULE_PARM_DESC(kdeth_qp, "Set the KDETH queue pair prefix");</div><div class='add'>+#include "netdev.h"</div><div class='ctx'> </div><div class='ctx'> uint num_vls = HFI1_MAX_VLS_SUPPORTED;</div><div class='ctx'> module_param(num_vls, uint, S_IRUGO);</div><div class='hunk'>@@ -128,13 +84,15 @@ struct flag_table {</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * RSM instance allocation</div><div class='del'>- *   0 - Verbs</div><div class='del'>- *   1 - User Fecn Handling</div><div class='del'>- *   2 - Vnic</div><div class='add'>+ *   0 - User Fecn Handling</div><div class='add'>+ *   1 - Vnic</div><div class='add'>+ *   2 - AIP</div><div class='add'>+ *   3 - Verbs</div><div class='ctx'>  */</div><div class='del'>-#define RSM_INS_VERBS             0</div><div class='del'>-#define RSM_INS_FECN              1</div><div class='del'>-#define RSM_INS_VNIC              2</div><div class='add'>+#define RSM_INS_FECN              0</div><div class='add'>+#define RSM_INS_VNIC              1</div><div class='add'>+#define RSM_INS_AIP               2</div><div class='add'>+#define RSM_INS_VERBS             3</div><div class='ctx'> </div><div class='ctx'> /* Bit offset into the GUID which carries HFI id information */</div><div class='ctx'> #define GUID_HFI_INDEX_SHIFT     39</div><div class='hunk'>@@ -175,6 +133,25 @@ struct flag_table {</div><div class='ctx'> /* QPN[m+n:1] QW 1, OFFSET 1 */</div><div class='ctx'> #define QPN_SELECT_OFFSET      ((1ull &lt;&lt; QW_SHIFT) | (1ull))</div><div class='ctx'> </div><div class='add'>+/* RSM fields for AIP */</div><div class='add'>+/* LRH.BTH above is reused for this rule */</div><div class='add'>+</div><div class='add'>+/* BTH.DESTQP: QW 1, OFFSET 16 for match */</div><div class='add'>+#define BTH_DESTQP_QW           1ull</div><div class='add'>+#define BTH_DESTQP_BIT_OFFSET   16ull</div><div class='add'>+#define BTH_DESTQP_OFFSET(off) ((BTH_DESTQP_QW &lt;&lt; QW_SHIFT) | (off))</div><div class='add'>+#define BTH_DESTQP_MATCH_OFFSET BTH_DESTQP_OFFSET(BTH_DESTQP_BIT_OFFSET)</div><div class='add'>+#define BTH_DESTQP_MASK         0xFFull</div><div class='add'>+#define BTH_DESTQP_VALUE        0x81ull</div><div class='add'>+</div><div class='add'>+/* DETH.SQPN: QW 1 Offset 56 for select */</div><div class='add'>+/* We use 8 most significant Soure QPN bits as entropy fpr AIP */</div><div class='add'>+#define DETH_AIP_SQPN_QW 3ull</div><div class='add'>+#define DETH_AIP_SQPN_BIT_OFFSET 56ull</div><div class='add'>+#define DETH_AIP_SQPN_OFFSET(off) ((DETH_AIP_SQPN_QW &lt;&lt; QW_SHIFT) | (off))</div><div class='add'>+#define DETH_AIP_SQPN_SELECT_OFFSET \</div><div class='add'>+	DETH_AIP_SQPN_OFFSET(DETH_AIP_SQPN_BIT_OFFSET)</div><div class='add'>+</div><div class='ctx'> /* RSM fields for Vnic */</div><div class='ctx'> /* L2_TYPE: QW 0, OFFSET 61 - for match */</div><div class='ctx'> #define L2_TYPE_QW             0ull</div><div class='hunk'>@@ -274,7 +251,7 @@ struct flag_table {</div><div class='ctx'> /*</div><div class='ctx'>  * CCE Error flags.</div><div class='ctx'>  */</div><div class='del'>-static struct flag_table cce_err_status_flags[] = {</div><div class='add'>+static const struct flag_table cce_err_status_flags[] = {</div><div class='ctx'> /* 0*/	FLAG_ENTRY0("CceCsrParityErr",</div><div class='ctx'> 		CCE_ERR_STATUS_CCE_CSR_PARITY_ERR_SMASK),</div><div class='ctx'> /* 1*/	FLAG_ENTRY0("CceCsrReadBadAddrErr",</div><div class='hunk'>@@ -364,7 +341,7 @@ static struct flag_table cce_err_status_flags[] = {</div><div class='ctx'>  * Misc Error flags</div><div class='ctx'>  */</div><div class='ctx'> #define MES(text) MISC_ERR_STATUS_MISC_##text##_ERR_SMASK</div><div class='del'>-static struct flag_table misc_err_status_flags[] = {</div><div class='add'>+static const struct flag_table misc_err_status_flags[] = {</div><div class='ctx'> /* 0*/	FLAG_ENTRY0("CSR_PARITY", MES(CSR_PARITY)),</div><div class='ctx'> /* 1*/	FLAG_ENTRY0("CSR_READ_BAD_ADDR", MES(CSR_READ_BAD_ADDR)),</div><div class='ctx'> /* 2*/	FLAG_ENTRY0("CSR_WRITE_BAD_ADDR", MES(CSR_WRITE_BAD_ADDR)),</div><div class='hunk'>@@ -383,7 +360,7 @@ static struct flag_table misc_err_status_flags[] = {</div><div class='ctx'> /*</div><div class='ctx'>  * TXE PIO Error flags and consequences</div><div class='ctx'>  */</div><div class='del'>-static struct flag_table pio_err_status_flags[] = {</div><div class='add'>+static const struct flag_table pio_err_status_flags[] = {</div><div class='ctx'> /* 0*/	FLAG_ENTRY("PioWriteBadCtxt",</div><div class='ctx'> 	SEC_WRITE_DROPPED,</div><div class='ctx'> 	SEND_PIO_ERR_STATUS_PIO_WRITE_BAD_CTXT_ERR_SMASK),</div><div class='hunk'>@@ -525,7 +502,7 @@ static struct flag_table pio_err_status_flags[] = {</div><div class='ctx'> /*</div><div class='ctx'>  * TXE SDMA Error flags</div><div class='ctx'>  */</div><div class='del'>-static struct flag_table sdma_err_status_flags[] = {</div><div class='add'>+static const struct flag_table sdma_err_status_flags[] = {</div><div class='ctx'> /* 0*/	FLAG_ENTRY0("SDmaRpyTagErr",</div><div class='ctx'> 		SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK),</div><div class='ctx'> /* 1*/	FLAG_ENTRY0("SDmaCsrParityErr",</div><div class='hunk'>@@ -553,7 +530,7 @@ static struct flag_table sdma_err_status_flags[] = {</div><div class='ctx'>  * TXE Egress Error flags</div><div class='ctx'>  */</div><div class='ctx'> #define SEES(text) SEND_EGRESS_ERR_STATUS_##text##_ERR_SMASK</div><div class='del'>-static struct flag_table egress_err_status_flags[] = {</div><div class='add'>+static const struct flag_table egress_err_status_flags[] = {</div><div class='ctx'> /* 0*/	FLAG_ENTRY0("TxPktIntegrityMemCorErr", SEES(TX_PKT_INTEGRITY_MEM_COR)),</div><div class='ctx'> /* 1*/	FLAG_ENTRY0("TxPktIntegrityMemUncErr", SEES(TX_PKT_INTEGRITY_MEM_UNC)),</div><div class='ctx'> /* 2 reserved */</div><div class='hunk'>@@ -654,7 +631,7 @@ static struct flag_table egress_err_status_flags[] = {</div><div class='ctx'>  * TXE Egress Error Info flags</div><div class='ctx'>  */</div><div class='ctx'> #define SEEI(text) SEND_EGRESS_ERR_INFO_##text##_ERR_SMASK</div><div class='del'>-static struct flag_table egress_err_info_flags[] = {</div><div class='add'>+static const struct flag_table egress_err_info_flags[] = {</div><div class='ctx'> /* 0*/	FLAG_ENTRY0("Reserved", 0ull),</div><div class='ctx'> /* 1*/	FLAG_ENTRY0("VLErr", SEEI(VL)),</div><div class='ctx'> /* 2*/	FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),</div><div class='hunk'>@@ -703,7 +680,7 @@ static struct flag_table egress_err_info_flags[] = {</div><div class='ctx'>  * TXE Send error flags</div><div class='ctx'>  */</div><div class='ctx'> #define SES(name) SEND_ERR_STATUS_SEND_##name##_ERR_SMASK</div><div class='del'>-static struct flag_table send_err_status_flags[] = {</div><div class='add'>+static const struct flag_table send_err_status_flags[] = {</div><div class='ctx'> /* 0*/	FLAG_ENTRY0("SendCsrParityErr", SES(CSR_PARITY)),</div><div class='ctx'> /* 1*/	FLAG_ENTRY0("SendCsrReadBadAddrErr", SES(CSR_READ_BAD_ADDR)),</div><div class='ctx'> /* 2*/	FLAG_ENTRY0("SendCsrWriteBadAddrErr", SES(CSR_WRITE_BAD_ADDR))</div><div class='hunk'>@@ -712,7 +689,7 @@ static struct flag_table send_err_status_flags[] = {</div><div class='ctx'> /*</div><div class='ctx'>  * TXE Send Context Error flags and consequences</div><div class='ctx'>  */</div><div class='del'>-static struct flag_table sc_err_status_flags[] = {</div><div class='add'>+static const struct flag_table sc_err_status_flags[] = {</div><div class='ctx'> /* 0*/	FLAG_ENTRY("InconsistentSop",</div><div class='ctx'> 		SEC_PACKET_DROPPED | SEC_SC_HALTED,</div><div class='ctx'> 		SEND_CTXT_ERR_STATUS_PIO_INCONSISTENT_SOP_ERR_SMASK),</div><div class='hunk'>@@ -735,7 +712,7 @@ static struct flag_table sc_err_status_flags[] = {</div><div class='ctx'>  * RXE Receive Error flags</div><div class='ctx'>  */</div><div class='ctx'> #define RXES(name) RCV_ERR_STATUS_RX_##name##_ERR_SMASK</div><div class='del'>-static struct flag_table rxe_err_status_flags[] = {</div><div class='add'>+static const struct flag_table rxe_err_status_flags[] = {</div><div class='ctx'> /* 0*/	FLAG_ENTRY0("RxDmaCsrCorErr", RXES(DMA_CSR_COR)),</div><div class='ctx'> /* 1*/	FLAG_ENTRY0("RxDcIntfParityErr", RXES(DC_INTF_PARITY)),</div><div class='ctx'> /* 2*/	FLAG_ENTRY0("RxRcvHdrUncErr", RXES(RCV_HDR_UNC)),</div><div class='hunk'>@@ -870,7 +847,7 @@ static struct flag_table rxe_err_status_flags[] = {</div><div class='ctx'>  * DCC Error Flags</div><div class='ctx'>  */</div><div class='ctx'> #define DCCE(name) DCC_ERR_FLG_##name##_SMASK</div><div class='del'>-static struct flag_table dcc_err_flags[] = {</div><div class='add'>+static const struct flag_table dcc_err_flags[] = {</div><div class='ctx'> 	FLAG_ENTRY0("bad_l2_err", DCCE(BAD_L2_ERR)),</div><div class='ctx'> 	FLAG_ENTRY0("bad_sc_err", DCCE(BAD_SC_ERR)),</div><div class='ctx'> 	FLAG_ENTRY0("bad_mid_tail_err", DCCE(BAD_MID_TAIL_ERR)),</div><div class='hunk'>@@ -923,7 +900,7 @@ static struct flag_table dcc_err_flags[] = {</div><div class='ctx'>  * LCB error flags</div><div class='ctx'>  */</div><div class='ctx'> #define LCBE(name) DC_LCB_ERR_FLG_##name##_SMASK</div><div class='del'>-static struct flag_table lcb_err_flags[] = {</div><div class='add'>+static const struct flag_table lcb_err_flags[] = {</div><div class='ctx'> /* 0*/	FLAG_ENTRY0("CSR_PARITY_ERR", LCBE(CSR_PARITY_ERR)),</div><div class='ctx'> /* 1*/	FLAG_ENTRY0("INVALID_CSR_ADDR", LCBE(INVALID_CSR_ADDR)),</div><div class='ctx'> /* 2*/	FLAG_ENTRY0("RST_FOR_FAILED_DESKEW", LCBE(RST_FOR_FAILED_DESKEW)),</div><div class='hunk'>@@ -966,7 +943,7 @@ static struct flag_table lcb_err_flags[] = {</div><div class='ctx'>  * DC8051 Error Flags</div><div class='ctx'>  */</div><div class='ctx'> #define D8E(name) DC_DC8051_ERR_FLG_##name##_SMASK</div><div class='del'>-static struct flag_table dc8051_err_flags[] = {</div><div class='add'>+static const struct flag_table dc8051_err_flags[] = {</div><div class='ctx'> 	FLAG_ENTRY0("SET_BY_8051", D8E(SET_BY_8051)),</div><div class='ctx'> 	FLAG_ENTRY0("LOST_8051_HEART_BEAT", D8E(LOST_8051_HEART_BEAT)),</div><div class='ctx'> 	FLAG_ENTRY0("CRAM_MBE", D8E(CRAM_MBE)),</div><div class='hunk'>@@ -985,7 +962,7 @@ static struct flag_table dc8051_err_flags[] = {</div><div class='ctx'>  *</div><div class='ctx'>  * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.ERROR field.</div><div class='ctx'>  */</div><div class='del'>-static struct flag_table dc8051_info_err_flags[] = {</div><div class='add'>+static const struct flag_table dc8051_info_err_flags[] = {</div><div class='ctx'> 	FLAG_ENTRY0("Spico ROM check failed",  SPICO_ROM_FAILED),</div><div class='ctx'> 	FLAG_ENTRY0("Unknown frame received",  UNKNOWN_FRAME),</div><div class='ctx'> 	FLAG_ENTRY0("Target BER not met",      TARGET_BER_NOT_MET),</div><div class='hunk'>@@ -1009,7 +986,7 @@ static struct flag_table dc8051_info_err_flags[] = {</div><div class='ctx'>  *</div><div class='ctx'>  * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG field.</div><div class='ctx'>  */</div><div class='del'>-static struct flag_table dc8051_info_host_msg_flags[] = {</div><div class='add'>+static const struct flag_table dc8051_info_host_msg_flags[] = {</div><div class='ctx'> 	FLAG_ENTRY0("Host request done", 0x0001),</div><div class='ctx'> 	FLAG_ENTRY0("BC PWR_MGM message", 0x0002),</div><div class='ctx'> 	FLAG_ENTRY0("BC SMA message", 0x0004),</div><div class='hunk'>@@ -1079,7 +1056,7 @@ static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr);</div><div class='ctx'> static void handle_temp_err(struct hfi1_devdata *dd);</div><div class='ctx'> static void dc_shutdown(struct hfi1_devdata *dd);</div><div class='ctx'> static void dc_start(struct hfi1_devdata *dd);</div><div class='del'>-static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,</div><div class='add'>+static int qos_rmt_entries(unsigned int n_krcv_queues, unsigned int *mp,</div><div class='ctx'> 			   unsigned int *np);</div><div class='ctx'> static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd);</div><div class='ctx'> static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms);</div><div class='hunk'>@@ -1304,9 +1281,9 @@ CNTR_ELEM(#name, \</div><div class='ctx'> 	  access_ibp_##cntr)</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * hfi_addr_from_offset - return addr for readq/writeq</div><div class='del'>- * @dd - the dd device</div><div class='del'>- * @offset - the offset of the CSR within bar0</div><div class='add'>+ * hfi1_addr_from_offset - return addr for readq/writeq</div><div class='add'>+ * @dd: the dd device</div><div class='add'>+ * @offset: the offset of the CSR within bar0</div><div class='ctx'>  *</div><div class='ctx'>  * This routine selects the appropriate base address</div><div class='ctx'>  * based on the indicated offset.</div><div class='hunk'>@@ -1322,8 +1299,8 @@ static inline void __iomem *hfi1_addr_from_offset(</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * read_csr - read CSR at the indicated offset</div><div class='del'>- * @dd - the dd device</div><div class='del'>- * @offset - the offset of the CSR within bar0</div><div class='add'>+ * @dd: the dd device</div><div class='add'>+ * @offset: the offset of the CSR within bar0</div><div class='ctx'>  *</div><div class='ctx'>  * Return: the value read or all FF's if there</div><div class='ctx'>  * is no mapping</div><div class='hunk'>@@ -1337,9 +1314,9 @@ u64 read_csr(const struct hfi1_devdata *dd, u32 offset)</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * write_csr - write CSR at the indicated offset</div><div class='del'>- * @dd - the dd device</div><div class='del'>- * @offset - the offset of the CSR within bar0</div><div class='del'>- * @value - value to write</div><div class='add'>+ * @dd: the dd device</div><div class='add'>+ * @offset: the offset of the CSR within bar0</div><div class='add'>+ * @value: value to write</div><div class='ctx'>  */</div><div class='ctx'> void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value)</div><div class='ctx'> {</div><div class='hunk'>@@ -1355,8 +1332,8 @@ void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value)</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * get_csr_addr - return te iomem address for offset</div><div class='del'>- * @dd - the dd device</div><div class='del'>- * @offset - the offset of the CSR within bar0</div><div class='add'>+ * @dd: the dd device</div><div class='add'>+ * @offset: the offset of the CSR within bar0</div><div class='ctx'>  *</div><div class='ctx'>  * Return: The iomem address to use in subsequent</div><div class='ctx'>  * writeq/readq operations.</div><div class='hunk'>@@ -1484,7 +1461,8 @@ static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context,</div><div class='ctx'> 		ret = write_lcb_csr(dd, csr, data);</div><div class='ctx'> </div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr);</div><div class='add'>+		if (!(dd-&gt;flags &amp; HFI1_SHUTDOWN))</div><div class='add'>+			dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr);</div><div class='ctx'> 		return 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -1685,6 +1663,14 @@ static u64 access_sw_pio_drain(const struct cntr_entry *entry,</div><div class='ctx'> 	return dd-&gt;verbs_dev.n_piodrain;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static u64 access_sw_ctx0_seq_drop(const struct cntr_entry *entry,</div><div class='add'>+				   void *context, int vl, int mode, u64 data)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_devdata *dd = context;</div><div class='add'>+</div><div class='add'>+	return dd-&gt;ctx0_seq_drop;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static u64 access_sw_vtx_wait(const struct cntr_entry *entry,</div><div class='ctx'> 			      void *context, int vl, int mode, u64 data)</div><div class='ctx'> {</div><div class='hunk'>@@ -4101,10 +4087,12 @@ def_access_ibp_counter(rc_dupreq);</div><div class='ctx'> def_access_ibp_counter(rdma_seq);</div><div class='ctx'> def_access_ibp_counter(unaligned);</div><div class='ctx'> def_access_ibp_counter(seq_naks);</div><div class='add'>+def_access_ibp_counter(rc_crwaits);</div><div class='ctx'> </div><div class='ctx'> static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = {</div><div class='ctx'> [C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH),</div><div class='ctx'> [C_RX_LEN_ERR] = RXE32_DEV_CNTR_ELEM(RxLenErr, RCV_LENGTH_ERR_CNT, CNTR_SYNTH),</div><div class='add'>+[C_RX_SHORT_ERR] = RXE32_DEV_CNTR_ELEM(RxShrErr, RCV_SHORT_ERR_CNT, CNTR_SYNTH),</div><div class='ctx'> [C_RX_ICRC_ERR] = RXE32_DEV_CNTR_ELEM(RxICrcErr, RCV_ICRC_ERR_CNT, CNTR_SYNTH),</div><div class='ctx'> [C_RX_EBP] = RXE32_DEV_CNTR_ELEM(RxEbpCnt, RCV_EBP_CNT, CNTR_SYNTH),</div><div class='ctx'> [C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT,</div><div class='hunk'>@@ -4248,6 +4236,8 @@ static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = {</div><div class='ctx'> 			    access_sw_cpu_intr),</div><div class='ctx'> [C_SW_CPU_RCV_LIM] = CNTR_ELEM("RcvLimit", 0, 0, CNTR_NORMAL,</div><div class='ctx'> 			    access_sw_cpu_rcv_limit),</div><div class='add'>+[C_SW_CTX0_SEQ_DROP] = CNTR_ELEM("SeqDrop0", 0, 0, CNTR_NORMAL,</div><div class='add'>+			    access_sw_ctx0_seq_drop),</div><div class='ctx'> [C_SW_VTX_WAIT] = CNTR_ELEM("vTxWait", 0, 0, CNTR_NORMAL,</div><div class='ctx'> 			    access_sw_vtx_wait),</div><div class='ctx'> [C_SW_PIO_WAIT] = CNTR_ELEM("PioWait", 0, 0, CNTR_NORMAL,</div><div class='hunk'>@@ -5119,6 +5109,7 @@ static struct cntr_entry port_cntrs[PORT_CNTR_LAST] = {</div><div class='ctx'> [C_SW_IBP_RDMA_SEQ] = SW_IBP_CNTR(RdmaSeq, rdma_seq),</div><div class='ctx'> [C_SW_IBP_UNALIGNED] = SW_IBP_CNTR(Unaligned, unaligned),</div><div class='ctx'> [C_SW_IBP_SEQ_NAK] = SW_IBP_CNTR(SeqNak, seq_naks),</div><div class='add'>+[C_SW_IBP_RC_CRWAITS] = SW_IBP_CNTR(RcCrWait, rc_crwaits),</div><div class='ctx'> [C_SW_CPU_RC_ACKS] = CNTR_ELEM("RcAcks", 0, 0, CNTR_NORMAL,</div><div class='ctx'> 			       access_sw_cpu_rc_acks),</div><div class='ctx'> [C_SW_CPU_RC_QACKS] = CNTR_ELEM("RcQacks", 0, 0, CNTR_NORMAL,</div><div class='hunk'>@@ -5284,7 +5275,7 @@ done:</div><div class='ctx'>  * the buffer.  End in '*' if the buffer is too short.</div><div class='ctx'>  */</div><div class='ctx'> static char *flag_string(char *buf, int buf_len, u64 flags,</div><div class='del'>-			 struct flag_table *table, int table_size)</div><div class='add'>+			 const struct flag_table *table, int table_size)</div><div class='ctx'> {</div><div class='ctx'> 	char extra[32];</div><div class='ctx'> 	char *p = buf;</div><div class='hunk'>@@ -5343,7 +5334,7 @@ static const char * const cce_misc_names[] = {</div><div class='ctx'> static char *is_misc_err_name(char *buf, size_t bsize, unsigned int source)</div><div class='ctx'> {</div><div class='ctx'> 	if (source &lt; ARRAY_SIZE(cce_misc_names))</div><div class='del'>-		strncpy(buf, cce_misc_names[source], bsize);</div><div class='add'>+		strscpy_pad(buf, cce_misc_names[source], bsize);</div><div class='ctx'> 	else</div><div class='ctx'> 		snprintf(buf, bsize, "Reserved%u",</div><div class='ctx'> 			 source + IS_GENERAL_ERR_START);</div><div class='hunk'>@@ -5383,7 +5374,7 @@ static const char * const various_names[] = {</div><div class='ctx'> static char *is_various_name(char *buf, size_t bsize, unsigned int source)</div><div class='ctx'> {</div><div class='ctx'> 	if (source &lt; ARRAY_SIZE(various_names))</div><div class='del'>-		strncpy(buf, various_names[source], bsize);</div><div class='add'>+		strscpy_pad(buf, various_names[source], bsize);</div><div class='ctx'> 	else</div><div class='ctx'> 		snprintf(buf, bsize, "Reserved%u", source + IS_VARIOUS_START);</div><div class='ctx'> 	return buf;</div><div class='hunk'>@@ -5557,7 +5548,7 @@ static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg)</div><div class='ctx'> #define RCVERR_CHECK_TIME 10</div><div class='ctx'> static void update_rcverr_timer(struct timer_list *t)</div><div class='ctx'> {</div><div class='del'>-	struct hfi1_devdata *dd = from_timer(dd, t, rcverr_timer);</div><div class='add'>+	struct hfi1_devdata *dd = timer_container_of(dd, t, rcverr_timer);</div><div class='ctx'> 	struct hfi1_pportdata *ppd = dd-&gt;pport;</div><div class='ctx'> 	u32 cur_ovfl_cnt = read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL);</div><div class='ctx'> </div><div class='hunk'>@@ -5585,7 +5576,7 @@ static int init_rcverr(struct hfi1_devdata *dd)</div><div class='ctx'> static void free_rcverr(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='ctx'> 	if (dd-&gt;rcverr_timer.function)</div><div class='del'>-		del_timer_sync(&amp;dd-&gt;rcverr_timer);</div><div class='add'>+		timer_delete_sync(&amp;dd-&gt;rcverr_timer);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)</div><div class='hunk'>@@ -6170,7 +6161,7 @@ static int request_host_lcb_access(struct hfi1_devdata *dd)</div><div class='ctx'> 	ret = do_8051_command(dd, HCMD_MISC,</div><div class='ctx'> 			      (u64)HCMD_MISC_REQUEST_LCB_ACCESS &lt;&lt;</div><div class='ctx'> 			      LOAD_DATA_FIELD_ID_SHIFT, NULL);</div><div class='del'>-	if (ret != HCMD_SUCCESS) {</div><div class='add'>+	if (ret != HCMD_SUCCESS &amp;&amp; !(dd-&gt;flags &amp; HFI1_SHUTDOWN)) {</div><div class='ctx'> 		dd_dev_err(dd, "%s: command failed with error %d\n",</div><div class='ctx'> 			   __func__, ret);</div><div class='ctx'> 	}</div><div class='hunk'>@@ -6251,7 +6242,8 @@ int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok)</div><div class='ctx'> 	if (dd-&gt;lcb_access_count == 0) {</div><div class='ctx'> 		ret = request_host_lcb_access(dd);</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			dd_dev_err(dd,</div><div class='add'>+			if (!(dd-&gt;flags &amp; HFI1_SHUTDOWN))</div><div class='add'>+				dd_dev_err(dd,</div><div class='ctx'> 				   "%s: unable to acquire LCB access, err %d\n",</div><div class='ctx'> 				   __func__, ret);</div><div class='ctx'> 			goto done;</div><div class='hunk'>@@ -6860,7 +6852,7 @@ static void rxe_kernel_unfreeze(struct hfi1_devdata *dd)</div><div class='ctx'> 		}</div><div class='ctx'> 		rcvmask = HFI1_RCVCTRL_CTXT_ENB;</div><div class='ctx'> 		/* HFI1_RCVCTRL_TAILUPD_[ENB|DIS] needs to be set explicitly */</div><div class='del'>-		rcvmask |= rcd-&gt;rcvhdrtail_kvaddr ?</div><div class='add'>+		rcvmask |= hfi1_rcvhdrtail_kvaddr(rcd) ?</div><div class='ctx'> 			HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS;</div><div class='ctx'> 		hfi1_rcvctrl(dd, rcvmask, rcd);</div><div class='ctx'> 		hfi1_rcd_put(rcd);</div><div class='hunk'>@@ -7286,11 +7278,11 @@ static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width)</div><div class='ctx'> 	case 1: return OPA_LINK_WIDTH_1X;</div><div class='ctx'> 	case 2: return OPA_LINK_WIDTH_2X;</div><div class='ctx'> 	case 3: return OPA_LINK_WIDTH_3X;</div><div class='add'>+	case 4: return OPA_LINK_WIDTH_4X;</div><div class='ctx'> 	default:</div><div class='ctx'> 		dd_dev_info(dd, "%s: invalid width %d, using 4\n",</div><div class='ctx'> 			    __func__, width);</div><div class='del'>-		/* fall through */</div><div class='del'>-	case 4: return OPA_LINK_WIDTH_4X;</div><div class='add'>+		return OPA_LINK_WIDTH_4X;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -7345,12 +7337,13 @@ static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width,</div><div class='ctx'> 		case 0:</div><div class='ctx'> 			dd-&gt;pport[0].link_speed_active = OPA_LINK_SPEED_12_5G;</div><div class='ctx'> 			break;</div><div class='add'>+		case 1:</div><div class='add'>+			dd-&gt;pport[0].link_speed_active = OPA_LINK_SPEED_25G;</div><div class='add'>+			break;</div><div class='ctx'> 		default:</div><div class='ctx'> 			dd_dev_err(dd,</div><div class='ctx'> 				   "%s: unexpected max rate %d, using 25Gb\n",</div><div class='ctx'> 				   __func__, (int)max_rate);</div><div class='del'>-			/* fall through */</div><div class='del'>-		case 1:</div><div class='ctx'> 			dd-&gt;pport[0].link_speed_active = OPA_LINK_SPEED_25G;</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -8284,7 +8277,7 @@ static void is_interrupt(struct hfi1_devdata *dd, unsigned int source)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * gerneral_interrupt() -  General interrupt handler</div><div class='add'>+ * general_interrupt -  General interrupt handler</div><div class='ctx'>  * @irq: MSIx IRQ vector</div><div class='ctx'>  * @data: hfi1 devdata</div><div class='ctx'>  *</div><div class='hunk'>@@ -8392,20 +8385,107 @@ void force_recv_intr(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> static inline int check_packet_present(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> {</div><div class='ctx'> 	u32 tail;</div><div class='del'>-	int present;</div><div class='ctx'> </div><div class='del'>-	if (!rcd-&gt;rcvhdrtail_kvaddr)</div><div class='del'>-		present = (rcd-&gt;seq_cnt ==</div><div class='del'>-				rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd))));</div><div class='del'>-	else /* is RDMA rtail */</div><div class='del'>-		present = (rcd-&gt;head != get_rcvhdrtail(rcd));</div><div class='del'>-</div><div class='del'>-	if (present)</div><div class='add'>+	if (hfi1_packet_present(rcd))</div><div class='ctx'> 		return 1;</div><div class='ctx'> </div><div class='ctx'> 	/* fall back to a CSR read, correct indpendent of DMA_RTAIL */</div><div class='ctx'> 	tail = (u32)read_uctxt_csr(rcd-&gt;dd, rcd-&gt;ctxt, RCV_HDR_TAIL);</div><div class='del'>-	return rcd-&gt;head != tail;</div><div class='add'>+	return hfi1_rcd_head(rcd) != tail;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * Common code for receive contexts interrupt handlers.</div><div class='add'>+ * Update traces, increment kernel IRQ counter and</div><div class='add'>+ * setup ASPM when needed.</div><div class='add'>+ */</div><div class='add'>+static void receive_interrupt_common(struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_devdata *dd = rcd-&gt;dd;</div><div class='add'>+</div><div class='add'>+	trace_hfi1_receive_interrupt(dd, rcd);</div><div class='add'>+	this_cpu_inc(*dd-&gt;int_counter);</div><div class='add'>+	aspm_ctx_disable(rcd);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * __hfi1_rcd_eoi_intr() - Make HW issue receive interrupt</div><div class='add'>+ * when there are packets present in the queue. When calling</div><div class='add'>+ * with interrupts enabled please use hfi1_rcd_eoi_intr.</div><div class='add'>+ *</div><div class='add'>+ * @rcd: valid receive context</div><div class='add'>+ */</div><div class='add'>+static void __hfi1_rcd_eoi_intr(struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	if (!rcd-&gt;rcvhdrq)</div><div class='add'>+		return;</div><div class='add'>+	clear_recv_intr(rcd);</div><div class='add'>+	if (check_packet_present(rcd))</div><div class='add'>+		force_recv_intr(rcd);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_rcd_eoi_intr() - End of Interrupt processing action</div><div class='add'>+ *</div><div class='add'>+ * @rcd: Ptr to hfi1_ctxtdata of receive context</div><div class='add'>+ *</div><div class='add'>+ *  Hold IRQs so we can safely clear the interrupt and</div><div class='add'>+ *  recheck for a packet that may have arrived after the previous</div><div class='add'>+ *  check and the interrupt clear.  If a packet arrived, force another</div><div class='add'>+ *  interrupt. This routine can be called at the end of receive packet</div><div class='add'>+ *  processing in interrupt service routines, interrupt service thread</div><div class='add'>+ *  and softirqs</div><div class='add'>+ */</div><div class='add'>+static void hfi1_rcd_eoi_intr(struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	local_irq_save(flags);</div><div class='add'>+	__hfi1_rcd_eoi_intr(rcd);</div><div class='add'>+	local_irq_restore(flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_netdev_rx_napi - napi poll function to move eoi inline</div><div class='add'>+ * @napi: pointer to napi object</div><div class='add'>+ * @budget: netdev budget</div><div class='add'>+ */</div><div class='add'>+int hfi1_netdev_rx_napi(struct napi_struct *napi, int budget)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_netdev_rxq *rxq = container_of(napi,</div><div class='add'>+			struct hfi1_netdev_rxq, napi);</div><div class='add'>+	struct hfi1_ctxtdata *rcd = rxq-&gt;rcd;</div><div class='add'>+	int work_done = 0;</div><div class='add'>+</div><div class='add'>+	work_done = rcd-&gt;do_interrupt(rcd, budget);</div><div class='add'>+</div><div class='add'>+	if (work_done &lt; budget) {</div><div class='add'>+		napi_complete_done(napi, work_done);</div><div class='add'>+		hfi1_rcd_eoi_intr(rcd);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return work_done;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* Receive packet napi handler for netdevs VNIC and AIP  */</div><div class='add'>+irqreturn_t receive_context_interrupt_napi(int irq, void *data)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ctxtdata *rcd = data;</div><div class='add'>+</div><div class='add'>+	receive_interrupt_common(rcd);</div><div class='add'>+</div><div class='add'>+	if (likely(rcd-&gt;napi)) {</div><div class='add'>+		if (likely(napi_schedule_prep(rcd-&gt;napi)))</div><div class='add'>+			__napi_schedule_irqoff(rcd-&gt;napi);</div><div class='add'>+		else</div><div class='add'>+			__hfi1_rcd_eoi_intr(rcd);</div><div class='add'>+	} else {</div><div class='add'>+		WARN_ONCE(1, "Napi IRQ handler without napi set up ctxt=%d\n",</div><div class='add'>+			  rcd-&gt;ctxt);</div><div class='add'>+		__hfi1_rcd_eoi_intr(rcd);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return IRQ_HANDLED;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -8419,13 +8499,9 @@ static inline int check_packet_present(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> irqreturn_t receive_context_interrupt(int irq, void *data)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_ctxtdata *rcd = data;</div><div class='del'>-	struct hfi1_devdata *dd = rcd-&gt;dd;</div><div class='ctx'> 	int disposition;</div><div class='del'>-	int present;</div><div class='ctx'> </div><div class='del'>-	trace_hfi1_receive_interrupt(dd, rcd);</div><div class='del'>-	this_cpu_inc(*dd-&gt;int_counter);</div><div class='del'>-	aspm_ctx_disable(rcd);</div><div class='add'>+	receive_interrupt_common(rcd);</div><div class='ctx'> </div><div class='ctx'> 	/* receive interrupt remains blocked while processing packets */</div><div class='ctx'> 	disposition = rcd-&gt;do_interrupt(rcd, 0);</div><div class='hunk'>@@ -8438,17 +8514,7 @@ irqreturn_t receive_context_interrupt(int irq, void *data)</div><div class='ctx'> 	if (disposition == RCV_PKT_LIMIT)</div><div class='ctx'> 		return IRQ_WAKE_THREAD;</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * The packet processor detected no more packets.  Clear the receive</div><div class='del'>-	 * interrupt and recheck for a packet packet that may have arrived</div><div class='del'>-	 * after the previous check and interrupt clear.  If a packet arrived,</div><div class='del'>-	 * force another interrupt.</div><div class='del'>-	 */</div><div class='del'>-	clear_recv_intr(rcd);</div><div class='del'>-	present = check_packet_present(rcd);</div><div class='del'>-	if (present)</div><div class='del'>-		force_recv_intr(rcd);</div><div class='del'>-</div><div class='add'>+	__hfi1_rcd_eoi_intr(rcd);</div><div class='ctx'> 	return IRQ_HANDLED;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -8459,24 +8525,11 @@ irqreturn_t receive_context_interrupt(int irq, void *data)</div><div class='ctx'> irqreturn_t receive_context_thread(int irq, void *data)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_ctxtdata *rcd = data;</div><div class='del'>-	int present;</div><div class='ctx'> </div><div class='ctx'> 	/* receive interrupt is still blocked from the IRQ handler */</div><div class='ctx'> 	(void)rcd-&gt;do_interrupt(rcd, 1);</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * The packet processor will only return if it detected no more</div><div class='del'>-	 * packets.  Hold IRQs here so we can safely clear the interrupt and</div><div class='del'>-	 * recheck for a packet that may have arrived after the previous</div><div class='del'>-	 * check and the interrupt clear.  If a packet arrived, force another</div><div class='del'>-	 * interrupt.</div><div class='del'>-	 */</div><div class='del'>-	local_irq_disable();</div><div class='del'>-	clear_recv_intr(rcd);</div><div class='del'>-	present = check_packet_present(rcd);</div><div class='del'>-	if (present)</div><div class='del'>-		force_recv_intr(rcd);</div><div class='del'>-	local_irq_enable();</div><div class='add'>+	hfi1_rcd_eoi_intr(rcd);</div><div class='ctx'> </div><div class='ctx'> 	return IRQ_HANDLED;</div><div class='ctx'> }</div><div class='hunk'>@@ -8702,7 +8755,7 @@ static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * When writing a LCB CSR, out_data contains the full value to</div><div class='del'>-	 * to be written, while in_data contains the relative LCB</div><div class='add'>+	 * be written, while in_data contains the relative LCB</div><div class='ctx'> 	 * address in 7:0.  Do the work here, rather than the caller,</div><div class='ctx'> 	 * of distrubting the write data to where it needs to go:</div><div class='ctx'> 	 *</div><div class='hunk'>@@ -9850,6 +9903,7 @@ void hfi1_quiet_serdes(struct hfi1_pportdata *ppd)</div><div class='ctx'> </div><div class='ctx'> 	/* disable the port */</div><div class='ctx'> 	clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);</div><div class='add'>+	cancel_work_sync(&amp;ppd-&gt;freeze_work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static inline int init_cpu_counters(struct hfi1_devdata *dd)</div><div class='hunk'>@@ -10046,12 +10100,12 @@ u32 lrh_max_header_bytes(struct hfi1_devdata *dd)</div><div class='ctx'> 	 * the first kernel context would have been allocated by now so</div><div class='ctx'> 	 * we are guaranteed a valid value.</div><div class='ctx'> 	 */</div><div class='del'>-	return (dd-&gt;rcd[0]-&gt;rcvhdrqentsize - 2/*PBC/RHF*/ + 1/*ICRC*/) &lt;&lt; 2;</div><div class='add'>+	return (get_hdrqentsize(dd-&gt;rcd[0]) - 2/*PBC/RHF*/ + 1/*ICRC*/) &lt;&lt; 2;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * Set Send Length</div><div class='del'>- * @ppd - per port data</div><div class='add'>+ * @ppd: per port data</div><div class='ctx'>  *</div><div class='ctx'>  * Set the MTU by limiting how many DWs may be sent.  The SendLenCheck*</div><div class='ctx'>  * registers compare against LRH.PktLen, so use the max bytes included</div><div class='hunk'>@@ -10091,7 +10145,7 @@ static void set_send_length(struct hfi1_pportdata *ppd)</div><div class='ctx'> 		thres = min(sc_percent_to_threshold(dd-&gt;vld[i].sc, 50),</div><div class='ctx'> 			    sc_mtu_to_threshold(dd-&gt;vld[i].sc,</div><div class='ctx'> 						dd-&gt;vld[i].mtu,</div><div class='del'>-						dd-&gt;rcd[0]-&gt;rcvhdrqentsize));</div><div class='add'>+						get_hdrqentsize(dd-&gt;rcd[0])));</div><div class='ctx'> 		for (j = 0; j &lt; INIT_SC_PER_VL; j++)</div><div class='ctx'> 			sc_set_cr_threshold(</div><div class='ctx'> 					pio_select_send_context_vl(dd, j, i),</div><div class='hunk'>@@ -11818,7 +11872,7 @@ u32 hdrqempty(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> 	head = (read_uctxt_csr(rcd-&gt;dd, rcd-&gt;ctxt, RCV_HDR_HEAD)</div><div class='ctx'> 		&amp; RCV_HDR_HEAD_HEAD_SMASK) &gt;&gt; RCV_HDR_HEAD_HEAD_SHIFT;</div><div class='ctx'> </div><div class='del'>-	if (rcd-&gt;rcvhdrtail_kvaddr)</div><div class='add'>+	if (hfi1_rcvhdrtail_kvaddr(rcd))</div><div class='ctx'> 		tail = get_rcvhdrtail(rcd);</div><div class='ctx'> 	else</div><div class='ctx'> 		tail = read_uctxt_csr(rcd-&gt;dd, rcd-&gt;ctxt, RCV_HDR_TAIL);</div><div class='hunk'>@@ -11862,6 +11916,84 @@ static u32 encoded_size(u32 size)</div><div class='ctx'> 	return 0x1;	/* if invalid, go with the minimum size */</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+/**</div><div class='add'>+ * encode_rcv_header_entry_size - return chip specific encoding for size</div><div class='add'>+ * @size: size in dwords</div><div class='add'>+ *</div><div class='add'>+ * Convert a receive header entry size that to the encoding used in the CSR.</div><div class='add'>+ *</div><div class='add'>+ * Return a zero if the given size is invalid, otherwise the encoding.</div><div class='add'>+ */</div><div class='add'>+u8 encode_rcv_header_entry_size(u8 size)</div><div class='add'>+{</div><div class='add'>+	/* there are only 3 valid receive header entry sizes */</div><div class='add'>+	if (size == 2)</div><div class='add'>+		return 1;</div><div class='add'>+	if (size == 16)</div><div class='add'>+		return 2;</div><div class='add'>+	if (size == 32)</div><div class='add'>+		return 4;</div><div class='add'>+	return 0; /* invalid */</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_validate_rcvhdrcnt - validate hdrcnt</div><div class='add'>+ * @dd: the device data</div><div class='add'>+ * @thecnt: the header count</div><div class='add'>+ */</div><div class='add'>+int hfi1_validate_rcvhdrcnt(struct hfi1_devdata *dd, uint thecnt)</div><div class='add'>+{</div><div class='add'>+	if (thecnt &lt;= HFI1_MIN_HDRQ_EGRBUF_CNT) {</div><div class='add'>+		dd_dev_err(dd, "Receive header queue count too small\n");</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (thecnt &gt; HFI1_MAX_HDRQ_EGRBUF_CNT) {</div><div class='add'>+		dd_dev_err(dd,</div><div class='add'>+			   "Receive header queue count cannot be greater than %u\n",</div><div class='add'>+			   HFI1_MAX_HDRQ_EGRBUF_CNT);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (thecnt % HDRQ_INCREMENT) {</div><div class='add'>+		dd_dev_err(dd, "Receive header queue count %d must be divisible by %lu\n",</div><div class='add'>+			   thecnt, HDRQ_INCREMENT);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * set_hdrq_regs - set header queue registers for context</div><div class='add'>+ * @dd: the device data</div><div class='add'>+ * @ctxt: the context</div><div class='add'>+ * @entsize: the dword entry size</div><div class='add'>+ * @hdrcnt: the number of header entries</div><div class='add'>+ */</div><div class='add'>+void set_hdrq_regs(struct hfi1_devdata *dd, u8 ctxt, u8 entsize, u16 hdrcnt)</div><div class='add'>+{</div><div class='add'>+	u64 reg;</div><div class='add'>+</div><div class='add'>+	reg = (((u64)hdrcnt &gt;&gt; HDRQ_SIZE_SHIFT) &amp; RCV_HDR_CNT_CNT_MASK) &lt;&lt;</div><div class='add'>+	      RCV_HDR_CNT_CNT_SHIFT;</div><div class='add'>+	write_kctxt_csr(dd, ctxt, RCV_HDR_CNT, reg);</div><div class='add'>+	reg = ((u64)encode_rcv_header_entry_size(entsize) &amp;</div><div class='add'>+	       RCV_HDR_ENT_SIZE_ENT_SIZE_MASK) &lt;&lt;</div><div class='add'>+	      RCV_HDR_ENT_SIZE_ENT_SIZE_SHIFT;</div><div class='add'>+	write_kctxt_csr(dd, ctxt, RCV_HDR_ENT_SIZE, reg);</div><div class='add'>+	reg = ((u64)DEFAULT_RCVHDRSIZE &amp; RCV_HDR_SIZE_HDR_SIZE_MASK) &lt;&lt;</div><div class='add'>+	      RCV_HDR_SIZE_HDR_SIZE_SHIFT;</div><div class='add'>+	write_kctxt_csr(dd, ctxt, RCV_HDR_SIZE, reg);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Program dummy tail address for every receive context</div><div class='add'>+	 * before enabling any receive context</div><div class='add'>+	 */</div><div class='add'>+	write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,</div><div class='add'>+			dd-&gt;rcvhdrtail_dummy_dma);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op,</div><div class='ctx'> 		  struct hfi1_ctxtdata *rcd)</div><div class='ctx'> {</div><div class='hunk'>@@ -11883,13 +12015,13 @@ void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op,</div><div class='ctx'> 		/* reset the tail and hdr addresses, and sequence count */</div><div class='ctx'> 		write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR,</div><div class='ctx'> 				rcd-&gt;rcvhdrq_dma);</div><div class='del'>-		if (rcd-&gt;rcvhdrtail_kvaddr)</div><div class='add'>+		if (hfi1_rcvhdrtail_kvaddr(rcd))</div><div class='ctx'> 			write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,</div><div class='ctx'> 					rcd-&gt;rcvhdrqtailaddr_dma);</div><div class='del'>-		rcd-&gt;seq_cnt = 1;</div><div class='add'>+		hfi1_set_seq_cnt(rcd, 1);</div><div class='ctx'> </div><div class='ctx'> 		/* reset the cached receive header queue head value */</div><div class='del'>-		rcd-&gt;head = 0;</div><div class='add'>+		hfi1_set_rcd_head(rcd, 0);</div><div class='ctx'> </div><div class='ctx'> 		/*</div><div class='ctx'> 		 * Zero the receive header queue so we don't get false</div><div class='hunk'>@@ -11969,7 +12101,7 @@ void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op,</div><div class='ctx'> 			      IS_RCVAVAIL_START + rcd-&gt;ctxt, false);</div><div class='ctx'> 		rcvctrl &amp;= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK;</div><div class='ctx'> 	}</div><div class='del'>-	if ((op &amp; HFI1_RCVCTRL_TAILUPD_ENB) &amp;&amp; rcd-&gt;rcvhdrtail_kvaddr)</div><div class='add'>+	if ((op &amp; HFI1_RCVCTRL_TAILUPD_ENB) &amp;&amp; hfi1_rcvhdrtail_kvaddr(rcd))</div><div class='ctx'> 		rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;</div><div class='ctx'> 	if (op &amp; HFI1_RCVCTRL_TAILUPD_DIS) {</div><div class='ctx'> 		/* See comment on RcvCtxtCtrl.TailUpd above */</div><div class='hunk'>@@ -12005,7 +12137,7 @@ void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op,</div><div class='ctx'> 		set_intr_bits(dd, IS_RCVURGENT_START + rcd-&gt;ctxt,</div><div class='ctx'> 			      IS_RCVURGENT_START + rcd-&gt;ctxt, false);</div><div class='ctx'> </div><div class='del'>-	hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx\n", ctxt, rcvctrl);</div><div class='add'>+	hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx", ctxt, rcvctrl);</div><div class='ctx'> 	write_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL, rcvctrl);</div><div class='ctx'> </div><div class='ctx'> 	/* work around sticky RcvCtxtStatus.BlockedRHQFull */</div><div class='hunk'>@@ -12075,10 +12207,10 @@ u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp)</div><div class='ctx'> 			hfi1_cdbg(CNTR, "reading %s", entry-&gt;name);</div><div class='ctx'> 			if (entry-&gt;flags &amp; CNTR_DISABLED) {</div><div class='ctx'> 				/* Nothing */</div><div class='del'>-				hfi1_cdbg(CNTR, "\tDisabled\n");</div><div class='add'>+				hfi1_cdbg(CNTR, "\tDisabled");</div><div class='ctx'> 			} else {</div><div class='ctx'> 				if (entry-&gt;flags &amp; CNTR_VL) {</div><div class='del'>-					hfi1_cdbg(CNTR, "\tPer VL\n");</div><div class='add'>+					hfi1_cdbg(CNTR, "\tPer VL");</div><div class='ctx'> 					for (j = 0; j &lt; C_VL_COUNT; j++) {</div><div class='ctx'> 						val = entry-&gt;rw_cntr(entry,</div><div class='ctx'> 								  dd, j,</div><div class='hunk'>@@ -12086,21 +12218,21 @@ u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp)</div><div class='ctx'> 								  0);</div><div class='ctx'> 						hfi1_cdbg(</div><div class='ctx'> 						   CNTR,</div><div class='del'>-						   "\t\tRead 0x%llx for %d\n",</div><div class='add'>+						   "\t\tRead 0x%llx for %d",</div><div class='ctx'> 						   val, j);</div><div class='ctx'> 						dd-&gt;cntrs[entry-&gt;offset + j] =</div><div class='ctx'> 									    val;</div><div class='ctx'> 					}</div><div class='ctx'> 				} else if (entry-&gt;flags &amp; CNTR_SDMA) {</div><div class='ctx'> 					hfi1_cdbg(CNTR,</div><div class='del'>-						  "\t Per SDMA Engine\n");</div><div class='add'>+						  "\t Per SDMA Engine");</div><div class='ctx'> 					for (j = 0; j &lt; chip_sdma_engines(dd);</div><div class='ctx'> 					     j++) {</div><div class='ctx'> 						val =</div><div class='ctx'> 						entry-&gt;rw_cntr(entry, dd, j,</div><div class='ctx'> 							       CNTR_MODE_R, 0);</div><div class='ctx'> 						hfi1_cdbg(CNTR,</div><div class='del'>-							  "\t\tRead 0x%llx for %d\n",</div><div class='add'>+							  "\t\tRead 0x%llx for %d",</div><div class='ctx'> 							  val, j);</div><div class='ctx'> 						dd-&gt;cntrs[entry-&gt;offset + j] =</div><div class='ctx'> 									val;</div><div class='hunk'>@@ -12141,7 +12273,7 @@ u32 hfi1_read_portcntrs(struct hfi1_pportdata *ppd, char **namep, u64 **cntrp)</div><div class='ctx'> 			hfi1_cdbg(CNTR, "reading %s", entry-&gt;name);</div><div class='ctx'> 			if (entry-&gt;flags &amp; CNTR_DISABLED) {</div><div class='ctx'> 				/* Nothing */</div><div class='del'>-				hfi1_cdbg(CNTR, "\tDisabled\n");</div><div class='add'>+				hfi1_cdbg(CNTR, "\tDisabled");</div><div class='ctx'> 				continue;</div><div class='ctx'> 			}</div><div class='ctx'> </div><div class='hunk'>@@ -12176,7 +12308,8 @@ static void free_cntrs(struct hfi1_devdata *dd)</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='ctx'> 	if (dd-&gt;synth_stats_timer.function)</div><div class='del'>-		del_timer_sync(&amp;dd-&gt;synth_stats_timer);</div><div class='add'>+		timer_delete_sync(&amp;dd-&gt;synth_stats_timer);</div><div class='add'>+	cancel_work_sync(&amp;dd-&gt;update_cntr_work);</div><div class='ctx'> 	ppd = (struct hfi1_pportdata *)(dd + 1);</div><div class='ctx'> 	for (i = 0; i &lt; dd-&gt;num_pports; i++, ppd++) {</div><div class='ctx'> 		kfree(ppd-&gt;cntrs);</div><div class='hunk'>@@ -12383,7 +12516,7 @@ static void do_update_synth_timer(struct work_struct *work)</div><div class='ctx'> </div><div class='ctx'> 	hfi1_cdbg(</div><div class='ctx'> 	    CNTR,</div><div class='del'>-	    "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx\n",</div><div class='add'>+	    "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx",</div><div class='ctx'> 	    dd-&gt;unit, cur_tx, cur_rx, dd-&gt;last_tx, dd-&gt;last_rx);</div><div class='ctx'> </div><div class='ctx'> 	if ((cur_tx &lt; dd-&gt;last_tx) || (cur_rx &lt; dd-&gt;last_rx)) {</div><div class='hunk'>@@ -12397,7 +12530,7 @@ static void do_update_synth_timer(struct work_struct *work)</div><div class='ctx'> 	} else {</div><div class='ctx'> 		total_flits = (cur_tx - dd-&gt;last_tx) + (cur_rx - dd-&gt;last_rx);</div><div class='ctx'> 		hfi1_cdbg(CNTR,</div><div class='del'>-			  "[%d] total flits 0x%llx limit 0x%llx\n", dd-&gt;unit,</div><div class='add'>+			  "[%d] total flits 0x%llx limit 0x%llx", dd-&gt;unit,</div><div class='ctx'> 			  total_flits, (u64)CNTR_32BIT_MAX);</div><div class='ctx'> 		if (total_flits &gt;= CNTR_32BIT_MAX) {</div><div class='ctx'> 			hfi1_cdbg(CNTR, "[%d] 32bit limit hit, updating",</div><div class='hunk'>@@ -12454,7 +12587,7 @@ static void do_update_synth_timer(struct work_struct *work)</div><div class='ctx'> </div><div class='ctx'> static void update_synth_timer(struct timer_list *t)</div><div class='ctx'> {</div><div class='del'>-	struct hfi1_devdata *dd = from_timer(dd, t, synth_stats_timer);</div><div class='add'>+	struct hfi1_devdata *dd = timer_container_of(dd, t, synth_stats_timer);</div><div class='ctx'> </div><div class='ctx'> 	queue_work(dd-&gt;update_cntr_wq, &amp;dd-&gt;update_cntr_work);</div><div class='ctx'> 	mod_timer(&amp;dd-&gt;synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);</div><div class='hunk'>@@ -12710,11 +12843,6 @@ bail:</div><div class='ctx'> static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate)</div><div class='ctx'> {</div><div class='ctx'> 	switch (chip_lstate) {</div><div class='del'>-	default:</div><div class='del'>-		dd_dev_err(dd,</div><div class='del'>-			   "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n",</div><div class='del'>-			   chip_lstate);</div><div class='del'>-		/* fall through */</div><div class='ctx'> 	case LSTATE_DOWN:</div><div class='ctx'> 		return IB_PORT_DOWN;</div><div class='ctx'> 	case LSTATE_INIT:</div><div class='hunk'>@@ -12723,6 +12851,11 @@ static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate)</div><div class='ctx'> 		return IB_PORT_ARMED;</div><div class='ctx'> 	case LSTATE_ACTIVE:</div><div class='ctx'> 		return IB_PORT_ACTIVE;</div><div class='add'>+	default:</div><div class='add'>+		dd_dev_err(dd,</div><div class='add'>+			   "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n",</div><div class='add'>+			   chip_lstate);</div><div class='add'>+		return IB_PORT_DOWN;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -12730,10 +12863,6 @@ u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate)</div><div class='ctx'> {</div><div class='ctx'> 	/* look at the HFI meta-states only */</div><div class='ctx'> 	switch (chip_pstate &amp; 0xf0) {</div><div class='del'>-	default:</div><div class='del'>-		dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n",</div><div class='del'>-			   chip_pstate);</div><div class='del'>-		/* fall through */</div><div class='ctx'> 	case PLS_DISABLED:</div><div class='ctx'> 		return IB_PORTPHYSSTATE_DISABLED;</div><div class='ctx'> 	case PLS_OFFLINE:</div><div class='hunk'>@@ -12746,25 +12875,13 @@ u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate)</div><div class='ctx'> 		return IB_PORTPHYSSTATE_LINKUP;</div><div class='ctx'> 	case PLS_PHYTEST:</div><div class='ctx'> 		return IB_PORTPHYSSTATE_PHY_TEST;</div><div class='add'>+	default:</div><div class='add'>+		dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n",</div><div class='add'>+			   chip_pstate);</div><div class='add'>+		return IB_PORTPHYSSTATE_DISABLED;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/* return the OPA port logical state name */</div><div class='del'>-const char *opa_lstate_name(u32 lstate)</div><div class='del'>-{</div><div class='del'>-	static const char * const port_logical_names[] = {</div><div class='del'>-		"PORT_NOP",</div><div class='del'>-		"PORT_DOWN",</div><div class='del'>-		"PORT_INIT",</div><div class='del'>-		"PORT_ARMED",</div><div class='del'>-		"PORT_ACTIVE",</div><div class='del'>-		"PORT_ACTIVE_DEFER",</div><div class='del'>-	};</div><div class='del'>-	if (lstate &lt; ARRAY_SIZE(port_logical_names))</div><div class='del'>-		return port_logical_names[lstate];</div><div class='del'>-	return "unknown";</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> /* return the OPA port physical state name */</div><div class='ctx'> const char *opa_pstate_name(u32 pstate)</div><div class='ctx'> {</div><div class='hunk'>@@ -12823,8 +12940,6 @@ static void update_statusp(struct hfi1_pportdata *ppd, u32 state)</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='del'>-	dd_dev_info(ppd-&gt;dd, "logical state changed to %s (0x%x)\n",</div><div class='del'>-		    opa_lstate_name(state), state);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='hunk'>@@ -13052,15 +13167,16 @@ static void read_mod_write(struct hfi1_devdata *dd, u16 src, u64 bits,</div><div class='ctx'> {</div><div class='ctx'> 	u64 reg;</div><div class='ctx'> 	u16 idx = src / BITS_PER_REGISTER;</div><div class='add'>+	unsigned long flags;</div><div class='ctx'> </div><div class='del'>-	spin_lock(&amp;dd-&gt;irq_src_lock);</div><div class='add'>+	spin_lock_irqsave(&amp;dd-&gt;irq_src_lock, flags);</div><div class='ctx'> 	reg = read_csr(dd, CCE_INT_MASK + (8 * idx));</div><div class='ctx'> 	if (set)</div><div class='ctx'> 		reg |= bits;</div><div class='ctx'> 	else</div><div class='ctx'> 		reg &amp;= ~bits;</div><div class='ctx'> 	write_csr(dd, CCE_INT_MASK + (8 * idx), reg);</div><div class='del'>-	spin_unlock(&amp;dd-&gt;irq_src_lock);</div><div class='add'>+	spin_unlock_irqrestore(&amp;dd-&gt;irq_src_lock, flags);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='hunk'>@@ -13101,7 +13217,7 @@ int set_intr_bits(struct hfi1_devdata *dd, u16 first, u16 last, bool set)</div><div class='ctx'> /*</div><div class='ctx'>  * Clear all interrupt sources on the chip.</div><div class='ctx'>  */</div><div class='del'>-void clear_all_interrupts(struct hfi1_devdata *dd)</div><div class='add'>+static void clear_all_interrupts(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='hunk'>@@ -13223,17 +13339,15 @@ static int set_up_interrupts(struct hfi1_devdata *dd)</div><div class='ctx'>  *                             in array of contexts</div><div class='ctx'>  *	freectxts  - number of free user contexts</div><div class='ctx'>  *	num_send_contexts - number of PIO send contexts being used</div><div class='del'>- *	num_vnic_contexts - number of contexts reserved for VNIC</div><div class='add'>+ *	num_netdev_contexts - number of contexts reserved for netdev</div><div class='ctx'>  */</div><div class='ctx'> static int set_up_context_variables(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long num_kernel_contexts;</div><div class='del'>-	u16 num_vnic_contexts = HFI1_NUM_VNIC_CTXT;</div><div class='del'>-	int total_contexts;</div><div class='add'>+	u16 num_netdev_contexts;</div><div class='ctx'> 	int ret;</div><div class='ctx'> 	unsigned ngroups;</div><div class='ctx'> 	int rmt_count;</div><div class='del'>-	int user_rmt_reduced;</div><div class='ctx'> 	u32 n_usr_ctxts;</div><div class='ctx'> 	u32 send_contexts = chip_send_contexts(dd);</div><div class='ctx'> 	u32 rcv_contexts = chip_rcv_contexts(dd);</div><div class='hunk'>@@ -13266,13 +13380,6 @@ static int set_up_context_variables(struct hfi1_devdata *dd)</div><div class='ctx'> 		num_kernel_contexts = send_contexts - num_vls - 1;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* Accommodate VNIC contexts if possible */</div><div class='del'>-	if ((num_kernel_contexts + num_vnic_contexts) &gt; rcv_contexts) {</div><div class='del'>-		dd_dev_err(dd, "No receive contexts available for VNIC\n");</div><div class='del'>-		num_vnic_contexts = 0;</div><div class='del'>-	}</div><div class='del'>-	total_contexts = num_kernel_contexts + num_vnic_contexts;</div><div class='del'>-</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * User contexts:</div><div class='ctx'> 	 *	- default to 1 user context per real (non-HT) CPU core if</div><div class='hunk'>@@ -13285,55 +13392,64 @@ static int set_up_context_variables(struct hfi1_devdata *dd)</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Adjust the counts given a global max.</div><div class='ctx'> 	 */</div><div class='del'>-	if (total_contexts + n_usr_ctxts &gt; rcv_contexts) {</div><div class='add'>+	if (num_kernel_contexts + n_usr_ctxts &gt; rcv_contexts) {</div><div class='ctx'> 		dd_dev_err(dd,</div><div class='del'>-			   "Reducing # user receive contexts to: %d, from %u\n",</div><div class='del'>-			   rcv_contexts - total_contexts,</div><div class='add'>+			   "Reducing # user receive contexts to: %u, from %u\n",</div><div class='add'>+			   (u32)(rcv_contexts - num_kernel_contexts),</div><div class='ctx'> 			   n_usr_ctxts);</div><div class='ctx'> 		/* recalculate */</div><div class='del'>-		n_usr_ctxts = rcv_contexts - total_contexts;</div><div class='add'>+		n_usr_ctxts = rcv_contexts - num_kernel_contexts;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	num_netdev_contexts =</div><div class='add'>+		hfi1_num_netdev_contexts(dd, rcv_contexts -</div><div class='add'>+					 (num_kernel_contexts + n_usr_ctxts),</div><div class='add'>+					 &amp;node_affinity.real_cpu_mask);</div><div class='ctx'> 	/*</div><div class='del'>-	 * The RMT entries are currently allocated as shown below:</div><div class='del'>-	 * 1. QOS (0 to 128 entries);</div><div class='del'>-	 * 2. FECN (num_kernel_context - 1 + num_user_contexts +</div><div class='del'>-	 *    num_vnic_contexts);</div><div class='del'>-	 * 3. VNIC (num_vnic_contexts).</div><div class='del'>-	 * It should be noted that FECN oversubscribe num_vnic_contexts</div><div class='del'>-	 * entries of RMT because both VNIC and PSM could allocate any receive</div><div class='del'>-	 * context between dd-&gt;first_dyn_alloc_text and dd-&gt;num_rcv_contexts,</div><div class='del'>-	 * and PSM FECN must reserve an RMT entry for each possible PSM receive</div><div class='del'>-	 * context.</div><div class='add'>+	 * RMT entries are allocated as follows:</div><div class='add'>+	 * 1. QOS (0 to 128 entries)</div><div class='add'>+	 * 2. FECN (num_kernel_context - 1 [a] + num_user_contexts +</div><div class='add'>+	 *          num_netdev_contexts [b])</div><div class='add'>+	 * 3. netdev (NUM_NETDEV_MAP_ENTRIES)</div><div class='add'>+	 *</div><div class='add'>+	 * Notes:</div><div class='add'>+	 * [a] Kernel contexts (except control) are included in FECN if kernel</div><div class='add'>+	 *     TID_RDMA is active.</div><div class='add'>+	 * [b] Netdev and user contexts are randomly allocated from the same</div><div class='add'>+	 *     context pool, so FECN must cover all contexts in the pool.</div><div class='ctx'> 	 */</div><div class='del'>-	rmt_count = qos_rmt_entries(dd, NULL, NULL) + (num_vnic_contexts * 2);</div><div class='del'>-	if (HFI1_CAP_IS_KSET(TID_RDMA))</div><div class='del'>-		rmt_count += num_kernel_contexts - 1;</div><div class='del'>-	if (rmt_count + n_usr_ctxts &gt; NUM_MAP_ENTRIES) {</div><div class='del'>-		user_rmt_reduced = NUM_MAP_ENTRIES - rmt_count;</div><div class='del'>-		dd_dev_err(dd,</div><div class='del'>-			   "RMT size is reducing the number of user receive contexts from %u to %d\n",</div><div class='del'>-			   n_usr_ctxts,</div><div class='del'>-			   user_rmt_reduced);</div><div class='del'>-		/* recalculate */</div><div class='del'>-		n_usr_ctxts = user_rmt_reduced;</div><div class='add'>+	rmt_count = qos_rmt_entries(num_kernel_contexts - 1, NULL, NULL)</div><div class='add'>+		    + (HFI1_CAP_IS_KSET(TID_RDMA) ? num_kernel_contexts - 1</div><div class='add'>+						  : 0)</div><div class='add'>+		    + n_usr_ctxts</div><div class='add'>+		    + num_netdev_contexts</div><div class='add'>+		    + NUM_NETDEV_MAP_ENTRIES;</div><div class='add'>+	if (rmt_count &gt; NUM_MAP_ENTRIES) {</div><div class='add'>+		int over = rmt_count - NUM_MAP_ENTRIES;</div><div class='add'>+		/* try to squish user contexts, minimum of 1 */</div><div class='add'>+		if (over &gt;= n_usr_ctxts) {</div><div class='add'>+			dd_dev_err(dd, "RMT overflow: reduce the requested number of contexts\n");</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		}</div><div class='add'>+		dd_dev_err(dd, "RMT overflow: reducing # user contexts from %u to %u\n",</div><div class='add'>+			   n_usr_ctxts, n_usr_ctxts - over);</div><div class='add'>+		n_usr_ctxts -= over;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	total_contexts += n_usr_ctxts;</div><div class='del'>-</div><div class='del'>-	/* the first N are kernel contexts, the rest are user/vnic contexts */</div><div class='del'>-	dd-&gt;num_rcv_contexts = total_contexts;</div><div class='add'>+	/* the first N are kernel contexts, the rest are user/netdev contexts */</div><div class='add'>+	dd-&gt;num_rcv_contexts =</div><div class='add'>+		num_kernel_contexts + n_usr_ctxts + num_netdev_contexts;</div><div class='ctx'> 	dd-&gt;n_krcv_queues = num_kernel_contexts;</div><div class='ctx'> 	dd-&gt;first_dyn_alloc_ctxt = num_kernel_contexts;</div><div class='del'>-	dd-&gt;num_vnic_contexts = num_vnic_contexts;</div><div class='add'>+	dd-&gt;num_netdev_contexts = num_netdev_contexts;</div><div class='ctx'> 	dd-&gt;num_user_contexts = n_usr_ctxts;</div><div class='ctx'> 	dd-&gt;freectxts = n_usr_ctxts;</div><div class='ctx'> 	dd_dev_info(dd,</div><div class='del'>-		    "rcv contexts: chip %d, used %d (kernel %d, vnic %u, user %u)\n",</div><div class='add'>+		    "rcv contexts: chip %d, used %d (kernel %d, netdev %u, user %u)\n",</div><div class='ctx'> 		    rcv_contexts,</div><div class='ctx'> 		    (int)dd-&gt;num_rcv_contexts,</div><div class='ctx'> 		    (int)dd-&gt;n_krcv_queues,</div><div class='del'>-		    dd-&gt;num_vnic_contexts,</div><div class='add'>+		    dd-&gt;num_netdev_contexts,</div><div class='ctx'> 		    dd-&gt;num_user_contexts);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='hunk'>@@ -14012,29 +14128,33 @@ static void init_early_variables(struct hfi1_devdata *dd)</div><div class='ctx'> </div><div class='ctx'> static void init_kdeth_qp(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='del'>-	/* user changed the KDETH_QP */</div><div class='del'>-	if (kdeth_qp != 0 &amp;&amp; kdeth_qp &gt;= 0xff) {</div><div class='del'>-		/* out of range or illegal value */</div><div class='del'>-		dd_dev_err(dd, "Invalid KDETH queue pair prefix, ignoring");</div><div class='del'>-		kdeth_qp = 0;</div><div class='del'>-	}</div><div class='del'>-	if (kdeth_qp == 0)	/* not set, or failed range check */</div><div class='del'>-		kdeth_qp = DEFAULT_KDETH_QP;</div><div class='del'>-</div><div class='ctx'> 	write_csr(dd, SEND_BTH_QP,</div><div class='del'>-		  (kdeth_qp &amp; SEND_BTH_QP_KDETH_QP_MASK) &lt;&lt;</div><div class='add'>+		  (RVT_KDETH_QP_PREFIX &amp; SEND_BTH_QP_KDETH_QP_MASK) &lt;&lt;</div><div class='ctx'> 		  SEND_BTH_QP_KDETH_QP_SHIFT);</div><div class='ctx'> </div><div class='ctx'> 	write_csr(dd, RCV_BTH_QP,</div><div class='del'>-		  (kdeth_qp &amp; RCV_BTH_QP_KDETH_QP_MASK) &lt;&lt;</div><div class='add'>+		  (RVT_KDETH_QP_PREFIX &amp; RCV_BTH_QP_KDETH_QP_MASK) &lt;&lt;</div><div class='ctx'> 		  RCV_BTH_QP_KDETH_QP_SHIFT);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * init_qpmap_table</div><div class='del'>- * @dd - device data</div><div class='del'>- * @first_ctxt - first context</div><div class='del'>- * @last_ctxt - first context</div><div class='add'>+ * hfi1_get_qp_map - get qp map</div><div class='add'>+ * @dd: device data</div><div class='add'>+ * @idx: index to read</div><div class='add'>+ */</div><div class='add'>+u8 hfi1_get_qp_map(struct hfi1_devdata *dd, u8 idx)</div><div class='add'>+{</div><div class='add'>+	u64 reg = read_csr(dd, RCV_QP_MAP_TABLE + (idx / 8) * 8);</div><div class='add'>+</div><div class='add'>+	reg &gt;&gt;= (idx % 8) * 8;</div><div class='add'>+	return reg;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * init_qpmap_table - init qp map</div><div class='add'>+ * @dd: device data</div><div class='add'>+ * @first_ctxt: first context</div><div class='add'>+ * @last_ctxt: first context</div><div class='ctx'>  *</div><div class='ctx'>  * This return sets the qpn mapping table that</div><div class='ctx'>  * is indexed by qpn[8:1].</div><div class='hunk'>@@ -14129,6 +14249,12 @@ static void complete_rsm_map_table(struct hfi1_devdata *dd,</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+/* Is a receive side mapping rule */</div><div class='add'>+static bool has_rsm_rule(struct hfi1_devdata *dd, u8 rule_index)</div><div class='add'>+{</div><div class='add'>+	return read_csr(dd, RCV_RSM_CFG + (8 * rule_index)) != 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /*</div><div class='ctx'>  * Add a receive side mapping rule.</div><div class='ctx'>  */</div><div class='hunk'>@@ -14164,15 +14290,15 @@ static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* return the number of RSM map table entries that will be used for QOS */</div><div class='del'>-static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,</div><div class='add'>+static int qos_rmt_entries(unsigned int n_krcv_queues, unsigned int *mp,</div><div class='ctx'> 			   unsigned int *np)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='ctx'> 	unsigned int m, n;</div><div class='del'>-	u8 max_by_vl = 0;</div><div class='add'>+	uint max_by_vl = 0;</div><div class='ctx'> </div><div class='ctx'> 	/* is QOS active at all? */</div><div class='del'>-	if (dd-&gt;n_krcv_queues &lt;= MIN_KERNEL_KCTXTS ||</div><div class='add'>+	if (n_krcv_queues &lt; MIN_KERNEL_KCTXTS ||</div><div class='ctx'> 	    num_vls == 1 ||</div><div class='ctx'> 	    krcvqsset &lt;= 1)</div><div class='ctx'> 		goto no_qos;</div><div class='hunk'>@@ -14209,8 +14335,8 @@ no_qos:</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * init_qos - init RX qos</div><div class='del'>- * @dd - device data</div><div class='del'>- * @rmt - RSM map table</div><div class='add'>+ * @dd: device data</div><div class='add'>+ * @rmt: RSM map table</div><div class='ctx'>  *</div><div class='ctx'>  * This routine initializes Rule 0 and the RSM map table to implement</div><div class='ctx'>  * quality of service (qos).</div><div class='hunk'>@@ -14230,7 +14356,7 @@ static void init_qos(struct hfi1_devdata *dd, struct rsm_map_table *rmt)</div><div class='ctx'> </div><div class='ctx'> 	if (!rmt)</div><div class='ctx'> 		goto bail;</div><div class='del'>-	rmt_entries = qos_rmt_entries(dd, &amp;m, &amp;n);</div><div class='add'>+	rmt_entries = qos_rmt_entries(dd-&gt;n_krcv_queues - 1, &amp;m, &amp;n);</div><div class='ctx'> 	if (rmt_entries == 0)</div><div class='ctx'> 		goto bail;</div><div class='ctx'> 	qpns_per_vl = 1 &lt;&lt; m;</div><div class='hunk'>@@ -14240,7 +14366,7 @@ static void init_qos(struct hfi1_devdata *dd, struct rsm_map_table *rmt)</div><div class='ctx'> 	if (rmt-&gt;used + rmt_entries &gt;= NUM_MAP_ENTRIES)</div><div class='ctx'> 		goto bail;</div><div class='ctx'> </div><div class='del'>-	/* add qos entries to the the RSM map table */</div><div class='add'>+	/* add qos entries to the RSM map table */</div><div class='ctx'> 	for (i = 0, ctxt = FIRST_KERNEL_KCTXT; i &lt; num_vls; i++) {</div><div class='ctx'> 		unsigned tctxt;</div><div class='ctx'> </div><div class='hunk'>@@ -14365,80 +14491,141 @@ static void init_fecn_handling(struct hfi1_devdata *dd,</div><div class='ctx'> 	rmt-&gt;used += total_cnt;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/* Initialize RSM for VNIC */</div><div class='del'>-void hfi1_init_vnic_rsm(struct hfi1_devdata *dd)</div><div class='add'>+static inline bool hfi1_is_rmt_full(int start, int spare)</div><div class='add'>+{</div><div class='add'>+	return (start + spare) &gt; NUM_MAP_ENTRIES;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool hfi1_netdev_update_rmt(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='ctx'> 	u8 i, j;</div><div class='ctx'> 	u8 ctx_id = 0;</div><div class='ctx'> 	u64 reg;</div><div class='ctx'> 	u32 regoff;</div><div class='del'>-	struct rsm_rule_data rrd;</div><div class='add'>+	int rmt_start = hfi1_netdev_get_free_rmt_idx(dd);</div><div class='add'>+	int ctxt_count = hfi1_netdev_ctxt_count(dd);</div><div class='ctx'> </div><div class='del'>-	if (hfi1_vnic_is_rsm_full(dd, NUM_VNIC_MAP_ENTRIES)) {</div><div class='del'>-		dd_dev_err(dd, "Vnic RSM disabled, rmt entries used = %d\n",</div><div class='del'>-			   dd-&gt;vnic.rmt_start);</div><div class='del'>-		return;</div><div class='add'>+	/* We already have contexts mapped in RMT */</div><div class='add'>+	if (has_rsm_rule(dd, RSM_INS_VNIC) || has_rsm_rule(dd, RSM_INS_AIP)) {</div><div class='add'>+		dd_dev_info(dd, "Contexts are already mapped in RMT\n");</div><div class='add'>+		return true;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	dev_dbg(&amp;(dd)-&gt;pcidev-&gt;dev, "Vnic rsm start = %d, end %d\n",</div><div class='del'>-		dd-&gt;vnic.rmt_start,</div><div class='del'>-		dd-&gt;vnic.rmt_start + NUM_VNIC_MAP_ENTRIES);</div><div class='add'>+	if (hfi1_is_rmt_full(rmt_start, NUM_NETDEV_MAP_ENTRIES)) {</div><div class='add'>+		dd_dev_err(dd, "Not enough RMT entries used = %d\n",</div><div class='add'>+			   rmt_start);</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dev_dbg(&amp;(dd)-&gt;pcidev-&gt;dev, "RMT start = %d, end %d\n",</div><div class='add'>+		rmt_start,</div><div class='add'>+		rmt_start + NUM_NETDEV_MAP_ENTRIES);</div><div class='ctx'> </div><div class='ctx'> 	/* Update RSM mapping table, 32 regs, 256 entries - 1 ctx per byte */</div><div class='del'>-	regoff = RCV_RSM_MAP_TABLE + (dd-&gt;vnic.rmt_start / 8) * 8;</div><div class='add'>+	regoff = RCV_RSM_MAP_TABLE + (rmt_start / 8) * 8;</div><div class='ctx'> 	reg = read_csr(dd, regoff);</div><div class='del'>-	for (i = 0; i &lt; NUM_VNIC_MAP_ENTRIES; i++) {</div><div class='del'>-		/* Update map register with vnic context */</div><div class='del'>-		j = (dd-&gt;vnic.rmt_start + i) % 8;</div><div class='add'>+	for (i = 0; i &lt; NUM_NETDEV_MAP_ENTRIES; i++) {</div><div class='add'>+		/* Update map register with netdev context */</div><div class='add'>+		j = (rmt_start + i) % 8;</div><div class='ctx'> 		reg &amp;= ~(0xffllu &lt;&lt; (j * 8));</div><div class='del'>-		reg |= (u64)dd-&gt;vnic.ctxt[ctx_id++]-&gt;ctxt &lt;&lt; (j * 8);</div><div class='del'>-		/* Wrap up vnic ctx index */</div><div class='del'>-		ctx_id %= dd-&gt;vnic.num_ctxt;</div><div class='add'>+		reg |= (u64)hfi1_netdev_get_ctxt(dd, ctx_id++)-&gt;ctxt &lt;&lt; (j * 8);</div><div class='add'>+		/* Wrap up netdev ctx index */</div><div class='add'>+		ctx_id %= ctxt_count;</div><div class='ctx'> 		/* Write back map register */</div><div class='del'>-		if (j == 7 || ((i + 1) == NUM_VNIC_MAP_ENTRIES)) {</div><div class='add'>+		if (j == 7 || ((i + 1) == NUM_NETDEV_MAP_ENTRIES)) {</div><div class='ctx'> 			dev_dbg(&amp;(dd)-&gt;pcidev-&gt;dev,</div><div class='del'>-				"Vnic rsm map reg[%d] =0x%llx\n",</div><div class='add'>+				"RMT[%d] =0x%llx\n",</div><div class='ctx'> 				regoff - RCV_RSM_MAP_TABLE, reg);</div><div class='ctx'> </div><div class='ctx'> 			write_csr(dd, regoff, reg);</div><div class='ctx'> 			regoff += 8;</div><div class='del'>-			if (i &lt; (NUM_VNIC_MAP_ENTRIES - 1))</div><div class='add'>+			if (i &lt; (NUM_NETDEV_MAP_ENTRIES - 1))</div><div class='ctx'> 				reg = read_csr(dd, regoff);</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* Add rule for vnic */</div><div class='del'>-	rrd.offset = dd-&gt;vnic.rmt_start;</div><div class='del'>-	rrd.pkt_type = 4;</div><div class='del'>-	/* Match 16B packets */</div><div class='del'>-	rrd.field1_off = L2_TYPE_MATCH_OFFSET;</div><div class='del'>-	rrd.mask1 = L2_TYPE_MASK;</div><div class='del'>-	rrd.value1 = L2_16B_VALUE;</div><div class='del'>-	/* Match ETH L4 packets */</div><div class='del'>-	rrd.field2_off = L4_TYPE_MATCH_OFFSET;</div><div class='del'>-	rrd.mask2 = L4_16B_TYPE_MASK;</div><div class='del'>-	rrd.value2 = L4_16B_ETH_VALUE;</div><div class='del'>-	/* Calc context from veswid and entropy */</div><div class='del'>-	rrd.index1_off = L4_16B_HDR_VESWID_OFFSET;</div><div class='del'>-	rrd.index1_width = ilog2(NUM_VNIC_MAP_ENTRIES);</div><div class='del'>-	rrd.index2_off = L2_16B_ENTROPY_OFFSET;</div><div class='del'>-	rrd.index2_width = ilog2(NUM_VNIC_MAP_ENTRIES);</div><div class='del'>-	add_rsm_rule(dd, RSM_INS_VNIC, &amp;rrd);</div><div class='del'>-</div><div class='del'>-	/* Enable RSM if not already enabled */</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hfi1_enable_rsm_rule(struct hfi1_devdata *dd,</div><div class='add'>+				 int rule, struct rsm_rule_data *rrd)</div><div class='add'>+{</div><div class='add'>+	if (!hfi1_netdev_update_rmt(dd)) {</div><div class='add'>+		dd_dev_err(dd, "Failed to update RMT for RSM%d rule\n", rule);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	add_rsm_rule(dd, rule, rrd);</div><div class='ctx'> 	add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+void hfi1_init_aip_rsm(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	/*</div><div class='add'>+	 * go through with the initialisation only if this rule actually doesn't</div><div class='add'>+	 * exist yet</div><div class='add'>+	 */</div><div class='add'>+	if (atomic_fetch_inc(&amp;dd-&gt;ipoib_rsm_usr_num) == 0) {</div><div class='add'>+		int rmt_start = hfi1_netdev_get_free_rmt_idx(dd);</div><div class='add'>+		struct rsm_rule_data rrd = {</div><div class='add'>+			.offset = rmt_start,</div><div class='add'>+			.pkt_type = IB_PACKET_TYPE,</div><div class='add'>+			.field1_off = LRH_BTH_MATCH_OFFSET,</div><div class='add'>+			.mask1 = LRH_BTH_MASK,</div><div class='add'>+			.value1 = LRH_BTH_VALUE,</div><div class='add'>+			.field2_off = BTH_DESTQP_MATCH_OFFSET,</div><div class='add'>+			.mask2 = BTH_DESTQP_MASK,</div><div class='add'>+			.value2 = BTH_DESTQP_VALUE,</div><div class='add'>+			.index1_off = DETH_AIP_SQPN_SELECT_OFFSET +</div><div class='add'>+					ilog2(NUM_NETDEV_MAP_ENTRIES),</div><div class='add'>+			.index1_width = ilog2(NUM_NETDEV_MAP_ENTRIES),</div><div class='add'>+			.index2_off = DETH_AIP_SQPN_SELECT_OFFSET,</div><div class='add'>+			.index2_width = ilog2(NUM_NETDEV_MAP_ENTRIES)</div><div class='add'>+		};</div><div class='add'>+</div><div class='add'>+		hfi1_enable_rsm_rule(dd, RSM_INS_AIP, &amp;rrd);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* Initialize RSM for VNIC */</div><div class='add'>+void hfi1_init_vnic_rsm(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	int rmt_start = hfi1_netdev_get_free_rmt_idx(dd);</div><div class='add'>+	struct rsm_rule_data rrd = {</div><div class='add'>+		/* Add rule for vnic */</div><div class='add'>+		.offset = rmt_start,</div><div class='add'>+		.pkt_type = 4,</div><div class='add'>+		/* Match 16B packets */</div><div class='add'>+		.field1_off = L2_TYPE_MATCH_OFFSET,</div><div class='add'>+		.mask1 = L2_TYPE_MASK,</div><div class='add'>+		.value1 = L2_16B_VALUE,</div><div class='add'>+		/* Match ETH L4 packets */</div><div class='add'>+		.field2_off = L4_TYPE_MATCH_OFFSET,</div><div class='add'>+		.mask2 = L4_16B_TYPE_MASK,</div><div class='add'>+		.value2 = L4_16B_ETH_VALUE,</div><div class='add'>+		/* Calc context from veswid and entropy */</div><div class='add'>+		.index1_off = L4_16B_HDR_VESWID_OFFSET,</div><div class='add'>+		.index1_width = ilog2(NUM_NETDEV_MAP_ENTRIES),</div><div class='add'>+		.index2_off = L2_16B_ENTROPY_OFFSET,</div><div class='add'>+		.index2_width = ilog2(NUM_NETDEV_MAP_ENTRIES)</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	hfi1_enable_rsm_rule(dd, RSM_INS_VNIC, &amp;rrd);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> void hfi1_deinit_vnic_rsm(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='ctx'> 	clear_rsm_rule(dd, RSM_INS_VNIC);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	/* Disable RSM if used only by vnic */</div><div class='del'>-	if (dd-&gt;vnic.rmt_start == 0)</div><div class='del'>-		clear_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);</div><div class='add'>+void hfi1_deinit_aip_rsm(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	/* only actually clear the rule if it's the last user asking to do so */</div><div class='add'>+	if (atomic_fetch_add_unless(&amp;dd-&gt;ipoib_rsm_usr_num, -1, 0) == 1)</div><div class='add'>+		clear_rsm_rule(dd, RSM_INS_AIP);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void init_rxe(struct hfi1_devdata *dd)</div><div class='add'>+static int init_rxe(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='ctx'> 	struct rsm_map_table *rmt;</div><div class='ctx'> 	u64 val;</div><div class='hunk'>@@ -14447,12 +14634,15 @@ static void init_rxe(struct hfi1_devdata *dd)</div><div class='ctx'> 	write_csr(dd, RCV_ERR_MASK, ~0ull);</div><div class='ctx'> </div><div class='ctx'> 	rmt = alloc_rsm_map_table(dd);</div><div class='add'>+	if (!rmt)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='ctx'> 	/* set up QOS, including the QPN map table */</div><div class='ctx'> 	init_qos(dd, rmt);</div><div class='ctx'> 	init_fecn_handling(dd, rmt);</div><div class='ctx'> 	complete_rsm_map_table(dd, rmt);</div><div class='del'>-	/* record number of used rsm map entries for vnic */</div><div class='del'>-	dd-&gt;vnic.rmt_start = rmt-&gt;used;</div><div class='add'>+	/* record number of used rsm map entries for netdev */</div><div class='add'>+	hfi1_netdev_set_free_rmt_idx(dd, rmt-&gt;used);</div><div class='ctx'> 	kfree(rmt);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='hunk'>@@ -14473,6 +14663,7 @@ static void init_rxe(struct hfi1_devdata *dd)</div><div class='ctx'> 	val |= ((4ull &amp; RCV_BYPASS_HDR_SIZE_MASK) &lt;&lt;</div><div class='ctx'> 		RCV_BYPASS_HDR_SIZE_SHIFT);</div><div class='ctx'> 	write_csr(dd, RCV_BYPASS, val);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void init_other(struct hfi1_devdata *dd)</div><div class='hunk'>@@ -14654,7 +14845,7 @@ int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *ctxt)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='del'>- * Start doing the clean up the the chip. Our clean up happens in multiple</div><div class='add'>+ * Start doing the clean up the chip. Our clean up happens in multiple</div><div class='ctx'>  * stages and this is just the first.</div><div class='ctx'>  */</div><div class='ctx'> void hfi1_start_cleanup(struct hfi1_devdata *dd)</div><div class='hunk'>@@ -14721,7 +14912,7 @@ static int obtain_boardname(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='ctx'> 	/* generic board description */</div><div class='ctx'> 	const char generic[] =</div><div class='del'>-		"Intel Omni-Path Host Fabric Interface Adapter 100 Series";</div><div class='add'>+		"Cornelis Omni-Path Host Fabric Interface Adapter 100 Series";</div><div class='ctx'> 	unsigned long size;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='hunk'>@@ -14783,8 +14974,7 @@ err_exit:</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * hfi1_init_dd() - Initialize most of the dd structure.</div><div class='del'>- * @dev: the pci_dev for hfi1_ib device</div><div class='del'>- * @ent: pci_device_id struct for this dev</div><div class='add'>+ * @dd: the dd device</div><div class='ctx'>  *</div><div class='ctx'>  * This is global, and is called directly at init to set up the</div><div class='ctx'>  * chip-specific function pointers for later use.</div><div class='hunk'>@@ -15005,12 +15195,20 @@ int hfi1_init_dd(struct hfi1_devdata *dd)</div><div class='ctx'> 		 (dd-&gt;revision &gt;&gt; CCE_REVISION_SW_SHIFT)</div><div class='ctx'> 		    &amp; CCE_REVISION_SW_MASK);</div><div class='ctx'> </div><div class='add'>+	/* alloc VNIC/AIP rx data */</div><div class='add'>+	ret = hfi1_alloc_rx(dd);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto bail_cleanup;</div><div class='add'>+</div><div class='ctx'> 	ret = set_up_context_variables(dd);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto bail_cleanup;</div><div class='ctx'> </div><div class='ctx'> 	/* set initial RXE CSRs */</div><div class='del'>-	init_rxe(dd);</div><div class='add'>+	ret = init_rxe(dd);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto bail_cleanup;</div><div class='add'>+</div><div class='ctx'> 	/* set initial TXE CSRs */</div><div class='ctx'> 	init_txe(dd);</div><div class='ctx'> 	/* set initial non-RXE, non-TXE CSRs */</div><div class='hunk'>@@ -15090,7 +15288,7 @@ int hfi1_init_dd(struct hfi1_devdata *dd)</div><div class='ctx'> 	init_completion(&amp;dd-&gt;user_comp);</div><div class='ctx'> </div><div class='ctx'> 	/* The user refcount starts with one to inidicate an active device */</div><div class='del'>-	atomic_set(&amp;dd-&gt;user_refcount, 1);</div><div class='add'>+	refcount_set(&amp;dd-&gt;user_refcount, 1);</div><div class='ctx'> </div><div class='ctx'> 	goto bail;</div><div class='ctx'> </div><div class='hunk'>@@ -15102,6 +15300,7 @@ bail_clear_intr:</div><div class='ctx'> 	hfi1_comp_vectors_clean_up(dd);</div><div class='ctx'> 	msix_clean_up_interrupts(dd);</div><div class='ctx'> bail_cleanup:</div><div class='add'>+	hfi1_free_rx(dd);</div><div class='ctx'> 	hfi1_pcie_ddcleanup(dd);</div><div class='ctx'> bail_free:</div><div class='ctx'> 	hfi1_free_devdata(dd);</div><div class='hunk'>@@ -15130,10 +15329,11 @@ static u16 delay_cycles(struct hfi1_pportdata *ppd, u32 desired_egress_rate,</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * create_pbc - build a pbc for transmission</div><div class='add'>+ * @ppd: info of physical Hfi port</div><div class='ctx'>  * @flags: special case flags or-ed in built pbc</div><div class='del'>- * @srate: static rate</div><div class='add'>+ * @srate_mbs: static rate</div><div class='ctx'>  * @vl: vl</div><div class='del'>- * @dwlen: dword length (header words + data words + pbc words)</div><div class='add'>+ * @dw_len: dword length (header words + data words + pbc words)</div><div class='ctx'>  *</div><div class='ctx'>  * Create a PBC with the given flags, rate, VL, and length.</div><div class='ctx'>  *</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/chip.h b/drivers/infiniband/hw/hfi1/chip.h<br/>index 4e6c3556ec48..6992f6d40255 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/chip.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/chip.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/chip.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/chip.h</a></div><div class='hunk'>@@ -1,52 +1,10 @@</div><div class='del'>-#ifndef _CHIP_H</div><div class='del'>-#define _CHIP_H</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2015 - 2020 Intel Corporation.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='add'>+#ifndef _CHIP_H</div><div class='add'>+#define _CHIP_H</div><div class='ctx'> /*</div><div class='ctx'>  * This file contains all of the defines that is specific to the HFI chip</div><div class='ctx'>  */</div><div class='hunk'>@@ -358,6 +316,8 @@</div><div class='ctx'> #define MAX_EAGER_BUFFER       (256 * 1024)</div><div class='ctx'> #define MAX_EAGER_BUFFER_TOTAL (64 * (1 &lt;&lt; 20)) /* max per ctxt 64MB */</div><div class='ctx'> #define MAX_EXPECTED_BUFFER    (2048 * 1024)</div><div class='add'>+#define HFI1_MIN_HDRQ_EGRBUF_CNT 32</div><div class='add'>+#define HFI1_MAX_HDRQ_EGRBUF_CNT 16352</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * Receive expected base and count and eager base and count increment -</div><div class='hunk'>@@ -699,6 +659,10 @@ static inline u32 chip_rcv_array_count(struct hfi1_devdata *dd)</div><div class='ctx'> 	return read_csr(dd, RCV_ARRAY_CNT);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+u8 encode_rcv_header_entry_size(u8 size);</div><div class='add'>+int hfi1_validate_rcvhdrcnt(struct hfi1_devdata *dd, uint thecnt);</div><div class='add'>+void set_hdrq_regs(struct hfi1_devdata *dd, u8 ctxt, u8 entsize, u16 hdrcnt);</div><div class='add'>+</div><div class='ctx'> u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl,</div><div class='ctx'> 	       u32 dw_len);</div><div class='ctx'> </div><div class='hunk'>@@ -807,7 +771,6 @@ int is_bx(struct hfi1_devdata *dd);</div><div class='ctx'> bool is_urg_masked(struct hfi1_ctxtdata *rcd);</div><div class='ctx'> u32 read_physical_state(struct hfi1_devdata *dd);</div><div class='ctx'> u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate);</div><div class='del'>-const char *opa_lstate_name(u32 lstate);</div><div class='ctx'> const char *opa_pstate_name(u32 pstate);</div><div class='ctx'> u32 driver_pstate(struct hfi1_pportdata *ppd);</div><div class='ctx'> u32 driver_lstate(struct hfi1_pportdata *ppd);</div><div class='hunk'>@@ -816,11 +779,6 @@ int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok);</div><div class='ctx'> int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok);</div><div class='ctx'> #define LCB_START DC_LCB_CSRS</div><div class='ctx'> #define LCB_END   DC_8051_CSRS /* next block is 8051 */</div><div class='del'>-static inline int is_lcb_offset(u32 offset)</div><div class='del'>-{</div><div class='del'>-	return (offset &gt;= LCB_START &amp;&amp; offset &lt; LCB_END);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> extern uint num_vls;</div><div class='ctx'> </div><div class='ctx'> extern uint disable_integrity;</div><div class='hunk'>@@ -859,6 +817,7 @@ static inline int idx_from_vl(int vl)</div><div class='ctx'> enum {</div><div class='ctx'> 	C_RCV_OVF = 0,</div><div class='ctx'> 	C_RX_LEN_ERR,</div><div class='add'>+	C_RX_SHORT_ERR,</div><div class='ctx'> 	C_RX_ICRC_ERR,</div><div class='ctx'> 	C_RX_EBP,</div><div class='ctx'> 	C_RX_TID_FULL,</div><div class='hunk'>@@ -926,6 +885,7 @@ enum {</div><div class='ctx'> 	C_DC_PG_STS_TX_MBE_CNT,</div><div class='ctx'> 	C_SW_CPU_INTR,</div><div class='ctx'> 	C_SW_CPU_RCV_LIM,</div><div class='add'>+	C_SW_CTX0_SEQ_DROP,</div><div class='ctx'> 	C_SW_VTX_WAIT,</div><div class='ctx'> 	C_SW_PIO_WAIT,</div><div class='ctx'> 	C_SW_PIO_DRAIN,</div><div class='hunk'>@@ -1245,6 +1205,7 @@ enum {</div><div class='ctx'> 	C_SW_IBP_RDMA_SEQ,</div><div class='ctx'> 	C_SW_IBP_UNALIGNED,</div><div class='ctx'> 	C_SW_IBP_SEQ_NAK,</div><div class='add'>+	C_SW_IBP_RC_CRWAITS,</div><div class='ctx'> 	C_SW_CPU_RC_ACKS,</div><div class='ctx'> 	C_SW_CPU_RC_QACKS,</div><div class='ctx'> 	C_SW_CPU_RC_DELAYED_COMP,</div><div class='hunk'>@@ -1438,13 +1399,16 @@ irqreturn_t general_interrupt(int irq, void *data);</div><div class='ctx'> irqreturn_t sdma_interrupt(int irq, void *data);</div><div class='ctx'> irqreturn_t receive_context_interrupt(int irq, void *data);</div><div class='ctx'> irqreturn_t receive_context_thread(int irq, void *data);</div><div class='add'>+irqreturn_t receive_context_interrupt_napi(int irq, void *data);</div><div class='ctx'> </div><div class='ctx'> int set_intr_bits(struct hfi1_devdata *dd, u16 first, u16 last, bool set);</div><div class='ctx'> void init_qsfp_int(struct hfi1_devdata *dd);</div><div class='del'>-void clear_all_interrupts(struct hfi1_devdata *dd);</div><div class='ctx'> void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr);</div><div class='ctx'> void remap_sdma_interrupts(struct hfi1_devdata *dd, int engine, int msix_intr);</div><div class='ctx'> void reset_interrupts(struct hfi1_devdata *dd);</div><div class='add'>+u8 hfi1_get_qp_map(struct hfi1_devdata *dd, u8 idx);</div><div class='add'>+void hfi1_init_aip_rsm(struct hfi1_devdata *dd);</div><div class='add'>+void hfi1_deinit_aip_rsm(struct hfi1_devdata *dd);</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * Interrupt source table.</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/chip_registers.h b/drivers/infiniband/hw/hfi1/chip_registers.h<br/>index ab3589d17aee..d79e25d20fb8 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/chip_registers.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/chip_registers.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/chip_registers.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/chip_registers.h</a></div><div class='hunk'>@@ -1,53 +1,11 @@</div><div class='del'>-#ifndef DEF_CHIP_REG</div><div class='del'>-#define DEF_CHIP_REG</div><div class='del'>-</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015, 2016 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='add'>+#ifndef DEF_CHIP_REG</div><div class='add'>+#define DEF_CHIP_REG</div><div class='add'>+</div><div class='ctx'> #define CORE		0x000000000000</div><div class='ctx'> #define CCE			(CORE + 0x000000000000)</div><div class='ctx'> #define ASIC		(CORE + 0x000000400000)</div><div class='hunk'>@@ -381,6 +339,7 @@</div><div class='ctx'> #define DC_LCB_STS_LINK_TRANSFER_ACTIVE (DC_LCB_CSRS + 0x000000000468)</div><div class='ctx'> #define DC_LCB_STS_ROUND_TRIP_LTP_CNT (DC_LCB_CSRS + 0x0000000004B0)</div><div class='ctx'> #define RCV_LENGTH_ERR_CNT 0</div><div class='add'>+#define RCV_SHORT_ERR_CNT 2</div><div class='ctx'> #define RCV_ICRC_ERR_CNT 6</div><div class='ctx'> #define RCV_EBP_CNT 9</div><div class='ctx'> #define RCV_BUF_OVFL_CNT 10</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/common.h b/drivers/infiniband/hw/hfi1/common.h<br/>index d47da7b0438f..8abc902b96f3 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/common.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/common.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/common.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/common.h</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2015 - 2020 Intel Corporation.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #ifndef _COMMON_H</div><div class='hunk'>@@ -72,13 +30,6 @@</div><div class='ctx'>  * compilation unit</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-/*</div><div class='del'>- * If a packet's QP[23:16] bits match this value, then it is</div><div class='del'>- * a PSM packet and the hardware will expect a KDETH header</div><div class='del'>- * following the BTH.</div><div class='del'>- */</div><div class='del'>-#define DEFAULT_KDETH_QP 0x80</div><div class='del'>-</div><div class='ctx'> /* driver/hw feature set bitmask */</div><div class='ctx'> #define HFI1_CAP_USER_SHIFT      24</div><div class='ctx'> #define HFI1_CAP_MASK            ((1UL &lt;&lt; HFI1_CAP_USER_SHIFT) - 1)</div><div class='hunk'>@@ -149,7 +100,8 @@</div><div class='ctx'> 				   HFI1_CAP_NO_INTEGRITY |		\</div><div class='ctx'> 				   HFI1_CAP_PKEY_CHECK |		\</div><div class='ctx'> 				   HFI1_CAP_TID_RDMA |			\</div><div class='del'>-				   HFI1_CAP_OPFN) &lt;&lt;			\</div><div class='add'>+				   HFI1_CAP_OPFN |			\</div><div class='add'>+				   HFI1_CAP_AIP) &lt;&lt;			\</div><div class='ctx'> 				  HFI1_CAP_USER_SHIFT)</div><div class='ctx'> /*</div><div class='ctx'>  * Set of capabilities that need to be enabled for kernel context in</div><div class='hunk'>@@ -166,6 +118,7 @@</div><div class='ctx'> 				 HFI1_CAP_PKEY_CHECK |			\</div><div class='ctx'> 				 HFI1_CAP_MULTI_PKT_EGR |		\</div><div class='ctx'> 				 HFI1_CAP_EXTENDED_PSN |		\</div><div class='add'>+				 HFI1_CAP_AIP |				\</div><div class='ctx'> 				 ((HFI1_CAP_HDRSUPP |			\</div><div class='ctx'> 				   HFI1_CAP_MULTI_PKT_EGR |		\</div><div class='ctx'> 				   HFI1_CAP_STATIC_RATE_CTRL |		\</div><div class='hunk'>@@ -184,61 +137,6 @@</div><div class='ctx'> #define HFI1_USER_SWVERSION ((HFI1_USER_SWMAJOR &lt;&lt; HFI1_SWMAJOR_SHIFT) | \</div><div class='ctx'> 			     HFI1_USER_SWMINOR)</div><div class='ctx'> </div><div class='del'>-#ifndef HFI1_KERN_TYPE</div><div class='del'>-#define HFI1_KERN_TYPE 0</div><div class='del'>-#endif</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Similarly, this is the kernel version going back to the user.  It's</div><div class='del'>- * slightly different, in that we want to tell if the driver was built as</div><div class='del'>- * part of a Intel release, or from the driver from openfabrics.org,</div><div class='del'>- * kernel.org, or a standard distribution, for support reasons.</div><div class='del'>- * The high bit is 0 for non-Intel and 1 for Intel-built/supplied.</div><div class='del'>- *</div><div class='del'>- * It's returned by the driver to the user code during initialization in the</div><div class='del'>- * spi_sw_version field of hfi1_base_info, so the user code can in turn</div><div class='del'>- * check for compatibility with the kernel.</div><div class='del'>-*/</div><div class='del'>-#define HFI1_KERN_SWVERSION ((HFI1_KERN_TYPE &lt;&lt; 31) | HFI1_USER_SWVERSION)</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Define the driver version number.  This is something that refers only</div><div class='del'>- * to the driver itself, not the software interfaces it supports.</div><div class='del'>- */</div><div class='del'>-#ifndef HFI1_DRIVER_VERSION_BASE</div><div class='del'>-#define HFI1_DRIVER_VERSION_BASE "0.9-294"</div><div class='del'>-#endif</div><div class='del'>-</div><div class='del'>-/* create the final driver version string */</div><div class='del'>-#ifdef HFI1_IDSTR</div><div class='del'>-#define HFI1_DRIVER_VERSION HFI1_DRIVER_VERSION_BASE " " HFI1_IDSTR</div><div class='del'>-#else</div><div class='del'>-#define HFI1_DRIVER_VERSION HFI1_DRIVER_VERSION_BASE</div><div class='del'>-#endif</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Diagnostics can send a packet by writing the following</div><div class='del'>- * struct to the diag packet special file.</div><div class='del'>- *</div><div class='del'>- * This allows a custom PBC qword, so that special modes and deliberate</div><div class='del'>- * changes to CRCs can be used.</div><div class='del'>- */</div><div class='del'>-#define _DIAG_PKT_VERS 1</div><div class='del'>-struct diag_pkt {</div><div class='del'>-	__u16 version;		/* structure version */</div><div class='del'>-	__u16 unit;		/* which device */</div><div class='del'>-	__u16 sw_index;		/* send sw index to use */</div><div class='del'>-	__u16 len;		/* data length, in bytes */</div><div class='del'>-	__u16 port;		/* port number */</div><div class='del'>-	__u16 unused;</div><div class='del'>-	__u32 flags;		/* call flags */</div><div class='del'>-	__u64 data;		/* user data pointer */</div><div class='del'>-	__u64 pbc;		/* PBC for the packet */</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* diag_pkt flags */</div><div class='del'>-#define F_DIAGPKT_WAIT 0x1	/* wait until packet is sent */</div><div class='del'>-</div><div class='ctx'> /*</div><div class='ctx'>  * The next set of defines are for packet headers, and chip register</div><div class='ctx'>  * and memory bits that are visible to and/or used by user-mode software.</div><div class='hunk'>@@ -323,6 +221,9 @@ struct diag_pkt {</div><div class='ctx'> /* RHF receive type error - bypass packet errors */</div><div class='ctx'> #define RHF_RTE_BYPASS_NO_ERR		0x0</div><div class='ctx'> </div><div class='add'>+/* MAX RcvSEQ */</div><div class='add'>+#define RHF_MAX_SEQ 13</div><div class='add'>+</div><div class='ctx'> /* IB - LRH header constants */</div><div class='ctx'> #define HFI1_LRH_GRH 0x0003      /* 1. word of IB LRH - next header: GRH */</div><div class='ctx'> #define HFI1_LRH_BTH 0x0002      /* 1. word of IB LRH - next header: BTH */</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/debugfs.c b/drivers/infiniband/hw/hfi1/debugfs.c<br/>index 15efb4a380b2..ac37ab7f8995 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/debugfs.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/debugfs.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/debugfs.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/debugfs.c</a></div><div class='hunk'>@@ -1,54 +1,12 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015-2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #include &lt;linux/debugfs.h&gt;</div><div class='ctx'> #include &lt;linux/seq_file.h&gt;</div><div class='ctx'> #include &lt;linux/kernel.h&gt;</div><div class='ctx'> #include &lt;linux/export.h&gt;</div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='ctx'> #include &lt;linux/string.h&gt;</div><div class='ctx'> #include &lt;linux/types.h&gt;</div><div class='ctx'> #include &lt;linux/ratelimit.h&gt;</div><div class='hunk'>@@ -64,34 +22,6 @@</div><div class='ctx'> </div><div class='ctx'> static struct dentry *hfi1_dbg_root;</div><div class='ctx'> </div><div class='del'>-/* wrappers to enforce srcu in seq file */</div><div class='del'>-ssize_t hfi1_seq_read(struct file *file, char __user *buf, size_t size,</div><div class='del'>-		      loff_t *ppos)</div><div class='del'>-{</div><div class='del'>-	struct dentry *d = file-&gt;f_path.dentry;</div><div class='del'>-	ssize_t r;</div><div class='del'>-</div><div class='del'>-	r = debugfs_file_get(d);</div><div class='del'>-	if (unlikely(r))</div><div class='del'>-		return r;</div><div class='del'>-	r = seq_read(file, buf, size, ppos);</div><div class='del'>-	debugfs_file_put(d);</div><div class='del'>-	return r;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-loff_t hfi1_seq_lseek(struct file *file, loff_t offset, int whence)</div><div class='del'>-{</div><div class='del'>-	struct dentry *d = file-&gt;f_path.dentry;</div><div class='del'>-	loff_t r;</div><div class='del'>-</div><div class='del'>-	r = debugfs_file_get(d);</div><div class='del'>-	if (unlikely(r))</div><div class='del'>-		return r;</div><div class='del'>-	r = seq_lseek(file, offset, whence);</div><div class='del'>-	debugfs_file_put(d);</div><div class='del'>-	return r;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> #define private2dd(file) (file_inode(file)-&gt;i_private)</div><div class='ctx'> #define private2ppd(file) (file_inode(file)-&gt;i_private)</div><div class='ctx'> </div><div class='hunk'>@@ -379,7 +309,7 @@ static void *_rcds_seq_next(struct seq_file *s, void *v, loff_t *pos)</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_dev(ibd);</div><div class='ctx'> </div><div class='ctx'> 	++*pos;</div><div class='del'>-	if (!dd-&gt;rcd || *pos &gt;= dd-&gt;n_krcv_queues)</div><div class='add'>+	if (!dd-&gt;rcd || *pos &gt;= dd-&gt;num_rcv_contexts)</div><div class='ctx'> 		return NULL;</div><div class='ctx'> 	return pos;</div><div class='ctx'> }</div><div class='hunk'>@@ -985,18 +915,10 @@ static ssize_t qsfp2_debugfs_read(struct file *file, char __user *buf,</div><div class='ctx'> static int __i2c_debugfs_open(struct inode *in, struct file *fp, u32 target)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_pportdata *ppd;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if (!try_module_get(THIS_MODULE))</div><div class='del'>-		return -ENODEV;</div><div class='ctx'> </div><div class='ctx'> 	ppd = private2ppd(fp);</div><div class='ctx'> </div><div class='del'>-	ret = acquire_chip_resource(ppd-&gt;dd, i2c_target(target), 0);</div><div class='del'>-	if (ret) /* failed - release the module */</div><div class='del'>-		module_put(THIS_MODULE);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='add'>+	return acquire_chip_resource(ppd-&gt;dd, i2c_target(target), 0);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int i2c1_debugfs_open(struct inode *in, struct file *fp)</div><div class='hunk'>@@ -1016,7 +938,6 @@ static int __i2c_debugfs_release(struct inode *in, struct file *fp, u32 target)</div><div class='ctx'> 	ppd = private2ppd(fp);</div><div class='ctx'> </div><div class='ctx'> 	release_chip_resource(ppd-&gt;dd, i2c_target(target));</div><div class='del'>-	module_put(THIS_MODULE);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -1034,18 +955,10 @@ static int i2c2_debugfs_release(struct inode *in, struct file *fp)</div><div class='ctx'> static int __qsfp_debugfs_open(struct inode *in, struct file *fp, u32 target)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_pportdata *ppd;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if (!try_module_get(THIS_MODULE))</div><div class='del'>-		return -ENODEV;</div><div class='ctx'> </div><div class='ctx'> 	ppd = private2ppd(fp);</div><div class='ctx'> </div><div class='del'>-	ret = acquire_chip_resource(ppd-&gt;dd, i2c_target(target), 0);</div><div class='del'>-	if (ret) /* failed - release the module */</div><div class='del'>-		module_put(THIS_MODULE);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='add'>+	return acquire_chip_resource(ppd-&gt;dd, i2c_target(target), 0);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int qsfp1_debugfs_open(struct inode *in, struct file *fp)</div><div class='hunk'>@@ -1065,7 +978,6 @@ static int __qsfp_debugfs_release(struct inode *in, struct file *fp, u32 target)</div><div class='ctx'> 	ppd = private2ppd(fp);</div><div class='ctx'> </div><div class='ctx'> 	release_chip_resource(ppd-&gt;dd, i2c_target(target));</div><div class='del'>-	module_put(THIS_MODULE);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -1155,6 +1067,7 @@ static int exprom_wp_debugfs_release(struct inode *in, struct file *fp)</div><div class='ctx'> { \</div><div class='ctx'> 	.name = nm, \</div><div class='ctx'> 	.ops = { \</div><div class='add'>+		.owner = THIS_MODULE, \</div><div class='ctx'> 		.read = readroutine, \</div><div class='ctx'> 		.write = writeroutine, \</div><div class='ctx'> 		.llseek = generic_file_llseek, \</div><div class='hunk'>@@ -1165,6 +1078,7 @@ static int exprom_wp_debugfs_release(struct inode *in, struct file *fp)</div><div class='ctx'> { \</div><div class='ctx'> 	.name = nm, \</div><div class='ctx'> 	.ops = { \</div><div class='add'>+		.owner = THIS_MODULE, \</div><div class='ctx'> 		.read = readf, \</div><div class='ctx'> 		.write = writef, \</div><div class='ctx'> 		.llseek = generic_file_llseek, \</div><div class='hunk'>@@ -1374,7 +1288,7 @@ static void _driver_stats_seq_stop(struct seq_file *s, void *v)</div><div class='ctx'> {</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static u64 hfi1_sps_ints(void)</div><div class='add'>+static void hfi1_sps_show_ints(struct seq_file *s)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long index, flags;</div><div class='ctx'> 	struct hfi1_devdata *dd;</div><div class='hunk'>@@ -1385,24 +1299,19 @@ static u64 hfi1_sps_ints(void)</div><div class='ctx'> 		sps_ints += get_all_cpu_total(dd-&gt;int_counter);</div><div class='ctx'> 	}</div><div class='ctx'> 	xa_unlock_irqrestore(&amp;hfi1_dev_table, flags);</div><div class='del'>-	return sps_ints;</div><div class='add'>+	seq_write(s, &amp;sps_ints, sizeof(u64));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int _driver_stats_seq_show(struct seq_file *s, void *v)</div><div class='ctx'> {</div><div class='ctx'> 	loff_t *spos = v;</div><div class='del'>-	char *buffer;</div><div class='ctx'> 	u64 *stats = (u64 *)&amp;hfi1_stats;</div><div class='del'>-	size_t sz = seq_get_buf(s, &amp;buffer);</div><div class='ctx'> </div><div class='del'>-	if (sz &lt; sizeof(u64))</div><div class='del'>-		return SEQ_SKIP;</div><div class='ctx'> 	/* special case for interrupts */</div><div class='ctx'> 	if (*spos == 0)</div><div class='del'>-		*(u64 *)buffer = hfi1_sps_ints();</div><div class='add'>+		hfi1_sps_show_ints(s);</div><div class='ctx'> 	else</div><div class='del'>-		*(u64 *)buffer = stats[*spos];</div><div class='del'>-	seq_commit(s,  sizeof(u64));</div><div class='add'>+		seq_write(s, stats + *spos, sizeof(u64));</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/debugfs.h b/drivers/infiniband/hw/hfi1/debugfs.h<br/>index 57e582caa5eb..65b48839abc6 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/debugfs.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/debugfs.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/debugfs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/debugfs.h</a></div><div class='hunk'>@@ -1,52 +1,11 @@</div><div class='del'>-#ifndef _HFI1_DEBUGFS_H</div><div class='del'>-#define _HFI1_DEBUGFS_H</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015, 2016, 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='add'>+#ifndef _HFI1_DEBUGFS_H</div><div class='add'>+#define _HFI1_DEBUGFS_H</div><div class='add'>+</div><div class='ctx'> struct hfi1_ibdev;</div><div class='ctx'> </div><div class='ctx'> #define DEBUGFS_SEQ_FILE_OPS(name) \</div><div class='hunk'>@@ -74,16 +33,11 @@ static int _##name##_open(struct inode *inode, struct file *s) \</div><div class='ctx'> static const struct file_operations _##name##_file_ops = { \</div><div class='ctx'> 	.owner   = THIS_MODULE, \</div><div class='ctx'> 	.open    = _##name##_open, \</div><div class='del'>-	.read    = hfi1_seq_read, \</div><div class='del'>-	.llseek  = hfi1_seq_lseek, \</div><div class='add'>+	.read    = seq_read, \</div><div class='add'>+	.llseek  = seq_lseek, \</div><div class='ctx'> 	.release = seq_release \</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-</div><div class='del'>-ssize_t hfi1_seq_read(struct file *file, char __user *buf, size_t size,</div><div class='del'>-		      loff_t *ppos);</div><div class='del'>-loff_t hfi1_seq_lseek(struct file *file, loff_t offset, int whence);</div><div class='del'>-</div><div class='ctx'> #ifdef CONFIG_DEBUG_FS</div><div class='ctx'> void hfi1_dbg_ibdev_init(struct hfi1_ibdev *ibd);</div><div class='ctx'> void hfi1_dbg_ibdev_exit(struct hfi1_ibdev *ibd);</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/device.c b/drivers/infiniband/hw/hfi1/device.c<br/>index bbb6069dec2a..a98a4175e53b 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/device.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/device.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/device.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/device.c</a></div><div class='hunk'>@@ -1,60 +1,38 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015, 2016 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/cdev.h&gt;</div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='ctx'> #include &lt;linux/device.h&gt;</div><div class='ctx'> #include &lt;linux/fs.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "hfi.h"</div><div class='ctx'> #include "device.h"</div><div class='ctx'> </div><div class='del'>-static struct class *class;</div><div class='del'>-static struct class *user_class;</div><div class='add'>+static char *hfi1_devnode(const struct device *dev, umode_t *mode)</div><div class='add'>+{</div><div class='add'>+	if (mode)</div><div class='add'>+		*mode = 0600;</div><div class='add'>+	return kasprintf(GFP_KERNEL, "%s", dev_name(dev));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct class class = {</div><div class='add'>+	.name = "hfi1",</div><div class='add'>+	.devnode = hfi1_devnode,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static char *hfi1_user_devnode(const struct device *dev, umode_t *mode)</div><div class='add'>+{</div><div class='add'>+	if (mode)</div><div class='add'>+		*mode = 0666;</div><div class='add'>+	return kasprintf(GFP_KERNEL, "%s", dev_name(dev));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct class user_class = {</div><div class='add'>+	.name = "hfi1_user",</div><div class='add'>+	.devnode = hfi1_user_devnode,</div><div class='add'>+};</div><div class='ctx'> static dev_t hfi1_dev;</div><div class='ctx'> </div><div class='ctx'> int hfi1_cdev_init(int minor, const char *name,</div><div class='hunk'>@@ -80,15 +58,15 @@ int hfi1_cdev_init(int minor, const char *name,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (user_accessible)</div><div class='del'>-		device = device_create(user_class, NULL, dev, NULL, "%s", name);</div><div class='add'>+		device = device_create(&amp;user_class, NULL, dev, NULL, "%s", name);</div><div class='ctx'> 	else</div><div class='del'>-		device = device_create(class, NULL, dev, NULL, "%s", name);</div><div class='add'>+		device = device_create(&amp;class, NULL, dev, NULL, "%s", name);</div><div class='ctx'> </div><div class='ctx'> 	if (IS_ERR(device)) {</div><div class='ctx'> 		ret = PTR_ERR(device);</div><div class='add'>+		pr_err("Could not create device for minor %d, %s (err %pe)\n",</div><div class='add'>+		       minor, name, device);</div><div class='ctx'> 		device = NULL;</div><div class='del'>-		pr_err("Could not create device for minor %d, %s (err %d)\n",</div><div class='del'>-			minor, name, -ret);</div><div class='ctx'> 		cdev_del(cdev);</div><div class='ctx'> 	}</div><div class='ctx'> done:</div><div class='hunk'>@@ -115,26 +93,6 @@ const char *class_name(void)</div><div class='ctx'> 	return hfi1_class_name;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static char *hfi1_devnode(struct device *dev, umode_t *mode)</div><div class='del'>-{</div><div class='del'>-	if (mode)</div><div class='del'>-		*mode = 0600;</div><div class='del'>-	return kasprintf(GFP_KERNEL, "%s", dev_name(dev));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static const char *hfi1_class_name_user = "hfi1_user";</div><div class='del'>-static const char *class_name_user(void)</div><div class='del'>-{</div><div class='del'>-	return hfi1_class_name_user;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static char *hfi1_user_devnode(struct device *dev, umode_t *mode)</div><div class='del'>-{</div><div class='del'>-	if (mode)</div><div class='del'>-		*mode = 0666;</div><div class='del'>-	return kasprintf(GFP_KERNEL, "%s", dev_name(dev));</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> int __init dev_init(void)</div><div class='ctx'> {</div><div class='ctx'> 	int ret;</div><div class='hunk'>@@ -145,27 +103,21 @@ int __init dev_init(void)</div><div class='ctx'> 		goto done;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	class = class_create(THIS_MODULE, class_name());</div><div class='del'>-	if (IS_ERR(class)) {</div><div class='del'>-		ret = PTR_ERR(class);</div><div class='add'>+	ret = class_register(&amp;class);</div><div class='add'>+	if (ret) {</div><div class='ctx'> 		pr_err("Could not create device class (err %d)\n", -ret);</div><div class='ctx'> 		unregister_chrdev_region(hfi1_dev, HFI1_NMINORS);</div><div class='ctx'> 		goto done;</div><div class='ctx'> 	}</div><div class='del'>-	class-&gt;devnode = hfi1_devnode;</div><div class='ctx'> </div><div class='del'>-	user_class = class_create(THIS_MODULE, class_name_user());</div><div class='del'>-	if (IS_ERR(user_class)) {</div><div class='del'>-		ret = PTR_ERR(user_class);</div><div class='add'>+	ret = class_register(&amp;user_class);</div><div class='add'>+	if (ret) {</div><div class='ctx'> 		pr_err("Could not create device class for user accessible files (err %d)\n",</div><div class='ctx'> 		       -ret);</div><div class='del'>-		class_destroy(class);</div><div class='del'>-		class = NULL;</div><div class='del'>-		user_class = NULL;</div><div class='add'>+		class_unregister(&amp;class);</div><div class='ctx'> 		unregister_chrdev_region(hfi1_dev, HFI1_NMINORS);</div><div class='ctx'> 		goto done;</div><div class='ctx'> 	}</div><div class='del'>-	user_class-&gt;devnode = hfi1_user_devnode;</div><div class='ctx'> </div><div class='ctx'> done:</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -173,11 +125,8 @@ done:</div><div class='ctx'> </div><div class='ctx'> void dev_cleanup(void)</div><div class='ctx'> {</div><div class='del'>-	class_destroy(class);</div><div class='del'>-	class = NULL;</div><div class='del'>-</div><div class='del'>-	class_destroy(user_class);</div><div class='del'>-	user_class = NULL;</div><div class='add'>+	class_unregister(&amp;class);</div><div class='add'>+	class_unregister(&amp;user_class);</div><div class='ctx'> </div><div class='ctx'> 	unregister_chrdev_region(hfi1_dev, HFI1_NMINORS);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/device.h b/drivers/infiniband/hw/hfi1/device.h<br/>index c3ec19cb0ac9..a91bea426ba5 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/device.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/device.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/device.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/device.h</a></div><div class='hunk'>@@ -1,52 +1,11 @@</div><div class='del'>-#ifndef _HFI1_DEVICE_H</div><div class='del'>-#define _HFI1_DEVICE_H</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015, 2016 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='add'>+#ifndef _HFI1_DEVICE_H</div><div class='add'>+#define _HFI1_DEVICE_H</div><div class='add'>+</div><div class='ctx'> int hfi1_cdev_init(int minor, const char *name,</div><div class='ctx'> 		   const struct file_operations *fops,</div><div class='ctx'> 		   struct cdev *cdev, struct device **devp,</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/driver.c b/drivers/infiniband/hw/hfi1/driver.c<br/>index 01aa1f132f55..06487e20f723 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/driver.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/driver.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/driver.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/driver.c</a></div><div class='hunk'>@@ -1,48 +1,7 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2015-2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2015-2020 Intel Corporation.</div><div class='add'>+ * Copyright(c) 2021 Cornelis Networks.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/spinlock.h&gt;</div><div class='hunk'>@@ -54,6 +13,7 @@</div><div class='ctx'> #include &lt;linux/module.h&gt;</div><div class='ctx'> #include &lt;linux/prefetch.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+#include &lt;linux/etherdevice.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "hfi.h"</div><div class='ctx'> #include "trace.h"</div><div class='hunk'>@@ -63,15 +23,12 @@</div><div class='ctx'> #include "vnic.h"</div><div class='ctx'> #include "fault.h"</div><div class='ctx'> </div><div class='add'>+#include "ipoib.h"</div><div class='add'>+#include "netdev.h"</div><div class='add'>+</div><div class='ctx'> #undef pr_fmt</div><div class='ctx'> #define pr_fmt(fmt) DRIVER_NAME ": " fmt</div><div class='ctx'> </div><div class='del'>-/*</div><div class='del'>- * The size has to be longer than this string, so we can append</div><div class='del'>- * board/chip information to it in the initialization code.</div><div class='del'>- */</div><div class='del'>-const char ib_hfi1_version[] = HFI1_DRIVER_VERSION "\n";</div><div class='del'>-</div><div class='ctx'> DEFINE_MUTEX(hfi1_mutex);	/* general driver use */</div><div class='ctx'> </div><div class='ctx'> unsigned int hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU;</div><div class='hunk'>@@ -94,7 +51,7 @@ module_param_cb(cap_mask, &amp;cap_ops, &amp;hfi1_cap_mask, S_IWUSR | S_IRUGO);</div><div class='ctx'> MODULE_PARM_DESC(cap_mask, "Bit mask of enabled/disabled HW features");</div><div class='ctx'> </div><div class='ctx'> MODULE_LICENSE("Dual BSD/GPL");</div><div class='del'>-MODULE_DESCRIPTION("Intel Omni-Path Architecture driver");</div><div class='add'>+MODULE_DESCRIPTION("Cornelis Omni-Path Express driver");</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * MAX_PKT_RCV is the max # if packets processed per receive interrupt.</div><div class='hunk'>@@ -155,7 +112,7 @@ static int hfi1_caps_get(char *buffer, const struct kernel_param *kp)</div><div class='ctx'> 	cap_mask &amp;= ~HFI1_CAP_LOCKED_SMASK;</div><div class='ctx'> 	cap_mask |= ((cap_mask &amp; HFI1_CAP_K2U) &lt;&lt; HFI1_CAP_USER_SHIFT);</div><div class='ctx'> </div><div class='del'>-	return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask);</div><div class='add'>+	return sysfs_emit(buffer, "0x%lx\n", cap_mask);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi)</div><div class='hunk'>@@ -411,14 +368,14 @@ drop:</div><div class='ctx'> static inline void init_packet(struct hfi1_ctxtdata *rcd,</div><div class='ctx'> 			       struct hfi1_packet *packet)</div><div class='ctx'> {</div><div class='del'>-	packet-&gt;rsize = rcd-&gt;rcvhdrqentsize; /* words */</div><div class='del'>-	packet-&gt;maxcnt = rcd-&gt;rcvhdrq_cnt * packet-&gt;rsize; /* words */</div><div class='add'>+	packet-&gt;rsize = get_hdrqentsize(rcd); /* words */</div><div class='add'>+	packet-&gt;maxcnt = get_hdrq_cnt(rcd) * packet-&gt;rsize; /* words */</div><div class='ctx'> 	packet-&gt;rcd = rcd;</div><div class='ctx'> 	packet-&gt;updegr = 0;</div><div class='ctx'> 	packet-&gt;etail = -1;</div><div class='ctx'> 	packet-&gt;rhf_addr = get_rhf_addr(rcd);</div><div class='ctx'> 	packet-&gt;rhf = rhf_to_cpu(packet-&gt;rhf_addr);</div><div class='del'>-	packet-&gt;rhqoff = rcd-&gt;head;</div><div class='add'>+	packet-&gt;rhqoff = hfi1_rcd_head(rcd);</div><div class='ctx'> 	packet-&gt;numpkt = 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -551,22 +508,22 @@ static inline void init_ps_mdata(struct ps_mdata *mdata,</div><div class='ctx'> 	mdata-&gt;maxcnt = packet-&gt;maxcnt;</div><div class='ctx'> 	mdata-&gt;ps_head = packet-&gt;rhqoff;</div><div class='ctx'> </div><div class='del'>-	if (HFI1_CAP_KGET_MASK(rcd-&gt;flags, DMA_RTAIL)) {</div><div class='add'>+	if (get_dma_rtail_setting(rcd)) {</div><div class='ctx'> 		mdata-&gt;ps_tail = get_rcvhdrtail(rcd);</div><div class='ctx'> 		if (rcd-&gt;ctxt == HFI1_CTRL_CTXT)</div><div class='del'>-			mdata-&gt;ps_seq = rcd-&gt;seq_cnt;</div><div class='add'>+			mdata-&gt;ps_seq = hfi1_seq_cnt(rcd);</div><div class='ctx'> 		else</div><div class='ctx'> 			mdata-&gt;ps_seq = 0; /* not used with DMA_RTAIL */</div><div class='ctx'> 	} else {</div><div class='ctx'> 		mdata-&gt;ps_tail = 0; /* used only with DMA_RTAIL*/</div><div class='del'>-		mdata-&gt;ps_seq = rcd-&gt;seq_cnt;</div><div class='add'>+		mdata-&gt;ps_seq = hfi1_seq_cnt(rcd);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static inline int ps_done(struct ps_mdata *mdata, u64 rhf,</div><div class='ctx'> 			  struct hfi1_ctxtdata *rcd)</div><div class='ctx'> {</div><div class='del'>-	if (HFI1_CAP_KGET_MASK(rcd-&gt;flags, DMA_RTAIL))</div><div class='add'>+	if (get_dma_rtail_setting(rcd))</div><div class='ctx'> 		return mdata-&gt;ps_head == mdata-&gt;ps_tail;</div><div class='ctx'> 	return mdata-&gt;ps_seq != rhf_rcv_seq(rhf);</div><div class='ctx'> }</div><div class='hunk'>@@ -592,11 +549,9 @@ static inline void update_ps_mdata(struct ps_mdata *mdata,</div><div class='ctx'> 		mdata-&gt;ps_head = 0;</div><div class='ctx'> </div><div class='ctx'> 	/* Control context must do seq counting */</div><div class='del'>-	if (!HFI1_CAP_KGET_MASK(rcd-&gt;flags, DMA_RTAIL) ||</div><div class='del'>-	    (rcd-&gt;ctxt == HFI1_CTRL_CTXT)) {</div><div class='del'>-		if (++mdata-&gt;ps_seq &gt; 13)</div><div class='del'>-			mdata-&gt;ps_seq = 1;</div><div class='del'>-	}</div><div class='add'>+	if (!get_dma_rtail_setting(rcd) ||</div><div class='add'>+	    rcd-&gt;ctxt == HFI1_CTRL_CTXT)</div><div class='add'>+		mdata-&gt;ps_seq = hfi1_seq_incr_wrap(mdata-&gt;ps_seq);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -734,6 +689,7 @@ static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread)</div><div class='ctx'> {</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	packet-&gt;rcd-&gt;dd-&gt;ctx0_seq_drop++;</div><div class='ctx'> 	/* Set up for the next packet */</div><div class='ctx'> 	packet-&gt;rhqoff += packet-&gt;rsize;</div><div class='ctx'> 	if (packet-&gt;rhqoff &gt;= packet-&gt;maxcnt)</div><div class='hunk'>@@ -749,6 +705,39 @@ static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread)</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void process_rcv_packet_napi(struct hfi1_packet *packet)</div><div class='add'>+{</div><div class='add'>+	packet-&gt;etype = rhf_rcv_type(packet-&gt;rhf);</div><div class='add'>+</div><div class='add'>+	/* total length */</div><div class='add'>+	packet-&gt;tlen = rhf_pkt_len(packet-&gt;rhf); /* in bytes */</div><div class='add'>+	/* retrieve eager buffer details */</div><div class='add'>+	packet-&gt;etail = rhf_egr_index(packet-&gt;rhf);</div><div class='add'>+	packet-&gt;ebuf = get_egrbuf(packet-&gt;rcd, packet-&gt;rhf,</div><div class='add'>+				  &amp;packet-&gt;updegr);</div><div class='add'>+	/*</div><div class='add'>+	 * Prefetch the contents of the eager buffer.  It is</div><div class='add'>+	 * OK to send a negative length to prefetch_range().</div><div class='add'>+	 * The +2 is the size of the RHF.</div><div class='add'>+	 */</div><div class='add'>+	prefetch_range(packet-&gt;ebuf,</div><div class='add'>+		       packet-&gt;tlen - ((packet-&gt;rcd-&gt;rcvhdrqentsize -</div><div class='add'>+				       (rhf_hdrq_offset(packet-&gt;rhf)</div><div class='add'>+					+ 2)) * 4));</div><div class='add'>+</div><div class='add'>+	packet-&gt;rcd-&gt;rhf_rcv_function_map[packet-&gt;etype](packet);</div><div class='add'>+	packet-&gt;numpkt++;</div><div class='add'>+</div><div class='add'>+	/* Set up for the next packet */</div><div class='add'>+	packet-&gt;rhqoff += packet-&gt;rsize;</div><div class='add'>+	if (packet-&gt;rhqoff &gt;= packet-&gt;maxcnt)</div><div class='add'>+		packet-&gt;rhqoff = 0;</div><div class='add'>+</div><div class='add'>+	packet-&gt;rhf_addr = (__le32 *)packet-&gt;rcd-&gt;rcvhdrq + packet-&gt;rhqoff +</div><div class='add'>+				      packet-&gt;rcd-&gt;rhf_offset;</div><div class='add'>+	packet-&gt;rhf = rhf_to_cpu(packet-&gt;rhf_addr);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static inline int process_rcv_packet(struct hfi1_packet *packet, int thread)</div><div class='ctx'> {</div><div class='ctx'> 	int ret;</div><div class='hunk'>@@ -769,7 +758,7 @@ static inline int process_rcv_packet(struct hfi1_packet *packet, int thread)</div><div class='ctx'> 		 * The +2 is the size of the RHF.</div><div class='ctx'> 		 */</div><div class='ctx'> 		prefetch_range(packet-&gt;ebuf,</div><div class='del'>-			       packet-&gt;tlen - ((packet-&gt;rcd-&gt;rcvhdrqentsize -</div><div class='add'>+			       packet-&gt;tlen - ((get_hdrqentsize(packet-&gt;rcd) -</div><div class='ctx'> 					       (rhf_hdrq_offset(packet-&gt;rhf)</div><div class='ctx'> 						+ 2)) * 4));</div><div class='ctx'> 	}</div><div class='hunk'>@@ -823,22 +812,50 @@ static inline void finish_packet(struct hfi1_packet *packet)</div><div class='ctx'> 	 * The only thing we need to do is a final update and call for an</div><div class='ctx'> 	 * interrupt</div><div class='ctx'> 	 */</div><div class='del'>-	update_usrhead(packet-&gt;rcd, packet-&gt;rcd-&gt;head, packet-&gt;updegr,</div><div class='add'>+	update_usrhead(packet-&gt;rcd, hfi1_rcd_head(packet-&gt;rcd), packet-&gt;updegr,</div><div class='ctx'> 		       packet-&gt;etail, rcv_intr_dynamic, packet-&gt;numpkt);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='add'>+ * handle_receive_interrupt_napi_fp - receive a packet</div><div class='add'>+ * @rcd: the context</div><div class='add'>+ * @budget: polling budget</div><div class='add'>+ *</div><div class='add'>+ * Called from interrupt handler for receive interrupt.</div><div class='add'>+ * This is the fast path interrupt handler</div><div class='add'>+ * when executing napi soft irq environment.</div><div class='add'>+ */</div><div class='add'>+int handle_receive_interrupt_napi_fp(struct hfi1_ctxtdata *rcd, int budget)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_packet packet;</div><div class='add'>+</div><div class='add'>+	init_packet(rcd, &amp;packet);</div><div class='add'>+	if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))</div><div class='add'>+		goto bail;</div><div class='add'>+</div><div class='add'>+	while (packet.numpkt &lt; budget) {</div><div class='add'>+		process_rcv_packet_napi(&amp;packet);</div><div class='add'>+		if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		process_rcv_update(0, &amp;packet);</div><div class='add'>+	}</div><div class='add'>+	hfi1_set_rcd_head(rcd, packet.rhqoff);</div><div class='add'>+bail:</div><div class='add'>+	finish_packet(&amp;packet);</div><div class='add'>+	return packet.numpkt;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='ctx'>  * Handle receive interrupts when using the no dma rtail option.</div><div class='ctx'>  */</div><div class='ctx'> int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread)</div><div class='ctx'> {</div><div class='del'>-	u32 seq;</div><div class='ctx'> 	int last = RCV_PKT_OK;</div><div class='ctx'> 	struct hfi1_packet packet;</div><div class='ctx'> </div><div class='ctx'> 	init_packet(rcd, &amp;packet);</div><div class='del'>-	seq = rhf_rcv_seq(packet.rhf);</div><div class='del'>-	if (seq != rcd-&gt;seq_cnt) {</div><div class='add'>+	if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) {</div><div class='ctx'> 		last = RCV_PKT_DONE;</div><div class='ctx'> 		goto bail;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -847,15 +864,12 @@ int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread)</div><div class='ctx'> </div><div class='ctx'> 	while (last == RCV_PKT_OK) {</div><div class='ctx'> 		last = process_rcv_packet(&amp;packet, thread);</div><div class='del'>-		seq = rhf_rcv_seq(packet.rhf);</div><div class='del'>-		if (++rcd-&gt;seq_cnt &gt; 13)</div><div class='del'>-			rcd-&gt;seq_cnt = 1;</div><div class='del'>-		if (seq != rcd-&gt;seq_cnt)</div><div class='add'>+		if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))</div><div class='ctx'> 			last = RCV_PKT_DONE;</div><div class='ctx'> 		process_rcv_update(last, &amp;packet);</div><div class='ctx'> 	}</div><div class='ctx'> 	process_rcv_qp_work(&amp;packet);</div><div class='del'>-	rcd-&gt;head = packet.rhqoff;</div><div class='add'>+	hfi1_set_rcd_head(rcd, packet.rhqoff);</div><div class='ctx'> bail:</div><div class='ctx'> 	finish_packet(&amp;packet);</div><div class='ctx'> 	return last;</div><div class='hunk'>@@ -884,15 +898,14 @@ int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread)</div><div class='ctx'> 		process_rcv_update(last, &amp;packet);</div><div class='ctx'> 	}</div><div class='ctx'> 	process_rcv_qp_work(&amp;packet);</div><div class='del'>-	rcd-&gt;head = packet.rhqoff;</div><div class='add'>+	hfi1_set_rcd_head(rcd, packet.rhqoff);</div><div class='ctx'> bail:</div><div class='ctx'> 	finish_packet(&amp;packet);</div><div class='ctx'> 	return last;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline void set_nodma_rtail(struct hfi1_devdata *dd, u16 ctxt)</div><div class='add'>+static void set_all_fastpath(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)</div><div class='ctx'> {</div><div class='del'>-	struct hfi1_ctxtdata *rcd;</div><div class='ctx'> 	u16 i;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='hunk'>@@ -900,50 +913,17 @@ static inline void set_nodma_rtail(struct hfi1_devdata *dd, u16 ctxt)</div><div class='ctx'> 	 * interrupt handler only for that context. Otherwise, switch</div><div class='ctx'> 	 * interrupt handler for all statically allocated kernel contexts.</div><div class='ctx'> 	 */</div><div class='del'>-	if (ctxt &gt;= dd-&gt;first_dyn_alloc_ctxt) {</div><div class='del'>-		rcd = hfi1_rcd_get_by_index_safe(dd, ctxt);</div><div class='del'>-		if (rcd) {</div><div class='del'>-			rcd-&gt;do_interrupt =</div><div class='del'>-				&amp;handle_receive_interrupt_nodma_rtail;</div><div class='del'>-			hfi1_rcd_put(rcd);</div><div class='del'>-		}</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	for (i = HFI1_CTRL_CTXT + 1; i &lt; dd-&gt;first_dyn_alloc_ctxt; i++) {</div><div class='del'>-		rcd = hfi1_rcd_get_by_index(dd, i);</div><div class='del'>-		if (rcd)</div><div class='del'>-			rcd-&gt;do_interrupt =</div><div class='del'>-				&amp;handle_receive_interrupt_nodma_rtail;</div><div class='add'>+	if (rcd-&gt;ctxt &gt;= dd-&gt;first_dyn_alloc_ctxt &amp;&amp; !rcd-&gt;is_vnic) {</div><div class='add'>+		hfi1_rcd_get(rcd);</div><div class='add'>+		hfi1_set_fast(rcd);</div><div class='ctx'> 		hfi1_rcd_put(rcd);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void set_dma_rtail(struct hfi1_devdata *dd, u16 ctxt)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_ctxtdata *rcd;</div><div class='del'>-	u16 i;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * For dynamically allocated kernel contexts (like vnic) switch</div><div class='del'>-	 * interrupt handler only for that context. Otherwise, switch</div><div class='del'>-	 * interrupt handler for all statically allocated kernel contexts.</div><div class='del'>-	 */</div><div class='del'>-	if (ctxt &gt;= dd-&gt;first_dyn_alloc_ctxt) {</div><div class='del'>-		rcd = hfi1_rcd_get_by_index_safe(dd, ctxt);</div><div class='del'>-		if (rcd) {</div><div class='del'>-			rcd-&gt;do_interrupt =</div><div class='del'>-				&amp;handle_receive_interrupt_dma_rtail;</div><div class='del'>-			hfi1_rcd_put(rcd);</div><div class='del'>-		}</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	for (i = HFI1_CTRL_CTXT + 1; i &lt; dd-&gt;first_dyn_alloc_ctxt; i++) {</div><div class='add'>+	for (i = HFI1_CTRL_CTXT + 1; i &lt; dd-&gt;num_rcv_contexts; i++) {</div><div class='ctx'> 		rcd = hfi1_rcd_get_by_index(dd, i);</div><div class='del'>-		if (rcd)</div><div class='del'>-			rcd-&gt;do_interrupt =</div><div class='del'>-				&amp;handle_receive_interrupt_dma_rtail;</div><div class='add'>+		if (rcd &amp;&amp; (i &lt; dd-&gt;first_dyn_alloc_ctxt || rcd-&gt;is_vnic))</div><div class='add'>+			hfi1_set_fast(rcd);</div><div class='ctx'> 		hfi1_rcd_put(rcd);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='hunk'>@@ -959,17 +939,14 @@ void set_all_slowpath(struct hfi1_devdata *dd)</div><div class='ctx'> 		if (!rcd)</div><div class='ctx'> 			continue;</div><div class='ctx'> 		if (i &lt; dd-&gt;first_dyn_alloc_ctxt || rcd-&gt;is_vnic)</div><div class='del'>-			rcd-&gt;do_interrupt = &amp;handle_receive_interrupt;</div><div class='add'>+			rcd-&gt;do_interrupt = rcd-&gt;slow_handler;</div><div class='ctx'> </div><div class='ctx'> 		hfi1_rcd_put(rcd);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd,</div><div class='del'>-				      struct hfi1_packet *packet,</div><div class='del'>-				      struct hfi1_devdata *dd)</div><div class='add'>+static bool __set_armed_to_active(struct hfi1_packet *packet)</div><div class='ctx'> {</div><div class='del'>-	struct work_struct *lsaw = &amp;rcd-&gt;ppd-&gt;linkstate_active_work;</div><div class='ctx'> 	u8 etype = rhf_rcv_type(packet-&gt;rhf);</div><div class='ctx'> 	u8 sc = SC15_PACKET;</div><div class='ctx'> </div><div class='hunk'>@@ -984,19 +961,34 @@ static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd,</div><div class='ctx'> 		sc = hfi1_16B_get_sc(hdr);</div><div class='ctx'> 	}</div><div class='ctx'> 	if (sc != SC15_PACKET) {</div><div class='del'>-		int hwstate = driver_lstate(rcd-&gt;ppd);</div><div class='add'>+		int hwstate = driver_lstate(packet-&gt;rcd-&gt;ppd);</div><div class='add'>+		struct work_struct *lsaw =</div><div class='add'>+				&amp;packet-&gt;rcd-&gt;ppd-&gt;linkstate_active_work;</div><div class='ctx'> </div><div class='ctx'> 		if (hwstate != IB_PORT_ACTIVE) {</div><div class='del'>-			dd_dev_info(dd,</div><div class='add'>+			dd_dev_info(packet-&gt;rcd-&gt;dd,</div><div class='ctx'> 				    "Unexpected link state %s\n",</div><div class='del'>-				    opa_lstate_name(hwstate));</div><div class='del'>-			return 0;</div><div class='add'>+				    ib_port_state_to_str(hwstate));</div><div class='add'>+			return false;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		queue_work(rcd-&gt;ppd-&gt;link_wq, lsaw);</div><div class='del'>-		return 1;</div><div class='add'>+		queue_work(packet-&gt;rcd-&gt;ppd-&gt;link_wq, lsaw);</div><div class='add'>+		return true;</div><div class='ctx'> 	}</div><div class='del'>-	return 0;</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * set_armed_to_active  - the fast path for armed to active</div><div class='add'>+ * @packet: the packet structure</div><div class='add'>+ *</div><div class='add'>+ * Return true if packet processing needs to bail.</div><div class='add'>+ */</div><div class='add'>+static bool set_armed_to_active(struct hfi1_packet *packet)</div><div class='add'>+{</div><div class='add'>+	if (likely(packet-&gt;rcd-&gt;ppd-&gt;host_link_state != HLS_UP_ARMED))</div><div class='add'>+		return false;</div><div class='add'>+	return __set_armed_to_active(packet);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -1014,15 +1006,15 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)</div><div class='ctx'> 	struct hfi1_packet packet;</div><div class='ctx'> 	int skip_pkt = 0;</div><div class='ctx'> </div><div class='add'>+	if (!rcd-&gt;rcvhdrq)</div><div class='add'>+		return RCV_PKT_OK;</div><div class='ctx'> 	/* Control context will always use the slow path interrupt handler */</div><div class='ctx'> 	needset = (rcd-&gt;ctxt == HFI1_CTRL_CTXT) ? 0 : 1;</div><div class='ctx'> </div><div class='ctx'> 	init_packet(rcd, &amp;packet);</div><div class='ctx'> </div><div class='del'>-	if (!HFI1_CAP_KGET_MASK(rcd-&gt;flags, DMA_RTAIL)) {</div><div class='del'>-		u32 seq = rhf_rcv_seq(packet.rhf);</div><div class='del'>-</div><div class='del'>-		if (seq != rcd-&gt;seq_cnt) {</div><div class='add'>+	if (!get_dma_rtail_setting(rcd)) {</div><div class='add'>+		if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) {</div><div class='ctx'> 			last = RCV_PKT_DONE;</div><div class='ctx'> 			goto bail;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -1039,22 +1031,15 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)</div><div class='ctx'> 		 * Control context can potentially receive an invalid</div><div class='ctx'> 		 * rhf. Drop such packets.</div><div class='ctx'> 		 */</div><div class='del'>-		if (rcd-&gt;ctxt == HFI1_CTRL_CTXT) {</div><div class='del'>-			u32 seq = rhf_rcv_seq(packet.rhf);</div><div class='del'>-</div><div class='del'>-			if (seq != rcd-&gt;seq_cnt)</div><div class='add'>+		if (rcd-&gt;ctxt == HFI1_CTRL_CTXT)</div><div class='add'>+			if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))</div><div class='ctx'> 				skip_pkt = 1;</div><div class='del'>-		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	prescan_rxq(rcd, &amp;packet);</div><div class='ctx'> </div><div class='ctx'> 	while (last == RCV_PKT_OK) {</div><div class='del'>-		if (unlikely(dd-&gt;do_drop &amp;&amp;</div><div class='del'>-			     atomic_xchg(&amp;dd-&gt;drop_packet, DROP_PACKET_OFF) ==</div><div class='del'>-			     DROP_PACKET_ON)) {</div><div class='del'>-			dd-&gt;do_drop = 0;</div><div class='del'>-</div><div class='add'>+		if (hfi1_need_drop(dd)) {</div><div class='ctx'> 			/* On to the next packet */</div><div class='ctx'> 			packet.rhqoff += packet.rsize;</div><div class='ctx'> 			packet.rhf_addr = (__le32 *)rcd-&gt;rcvhdrq +</div><div class='hunk'>@@ -1066,26 +1051,14 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)</div><div class='ctx'> 			last = skip_rcv_packet(&amp;packet, thread);</div><div class='ctx'> 			skip_pkt = 0;</div><div class='ctx'> 		} else {</div><div class='del'>-			/* Auto activate link on non-SC15 packet receive */</div><div class='del'>-			if (unlikely(rcd-&gt;ppd-&gt;host_link_state ==</div><div class='del'>-				     HLS_UP_ARMED) &amp;&amp;</div><div class='del'>-			    set_armed_to_active(rcd, &amp;packet, dd))</div><div class='add'>+			if (set_armed_to_active(&amp;packet))</div><div class='ctx'> 				goto bail;</div><div class='ctx'> 			last = process_rcv_packet(&amp;packet, thread);</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		if (!HFI1_CAP_KGET_MASK(rcd-&gt;flags, DMA_RTAIL)) {</div><div class='del'>-			u32 seq = rhf_rcv_seq(packet.rhf);</div><div class='del'>-</div><div class='del'>-			if (++rcd-&gt;seq_cnt &gt; 13)</div><div class='del'>-				rcd-&gt;seq_cnt = 1;</div><div class='del'>-			if (seq != rcd-&gt;seq_cnt)</div><div class='add'>+		if (!get_dma_rtail_setting(rcd)) {</div><div class='add'>+			if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))</div><div class='ctx'> 				last = RCV_PKT_DONE;</div><div class='del'>-			if (needset) {</div><div class='del'>-				dd_dev_info(dd, "Switching to NO_DMA_RTAIL\n");</div><div class='del'>-				set_nodma_rtail(dd, rcd-&gt;ctxt);</div><div class='del'>-				needset = 0;</div><div class='del'>-			}</div><div class='ctx'> 		} else {</div><div class='ctx'> 			if (packet.rhqoff == hdrqtail)</div><div class='ctx'> 				last = RCV_PKT_DONE;</div><div class='hunk'>@@ -1094,27 +1067,24 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)</div><div class='ctx'> 			 * rhf. Drop such packets.</div><div class='ctx'> 			 */</div><div class='ctx'> 			if (rcd-&gt;ctxt == HFI1_CTRL_CTXT) {</div><div class='del'>-				u32 seq = rhf_rcv_seq(packet.rhf);</div><div class='add'>+				bool lseq;</div><div class='ctx'> </div><div class='del'>-				if (++rcd-&gt;seq_cnt &gt; 13)</div><div class='del'>-					rcd-&gt;seq_cnt = 1;</div><div class='del'>-				if (!last &amp;&amp; (seq != rcd-&gt;seq_cnt))</div><div class='add'>+				lseq = hfi1_seq_incr(rcd,</div><div class='add'>+						     rhf_rcv_seq(packet.rhf));</div><div class='add'>+				if (!last &amp;&amp; lseq)</div><div class='ctx'> 					skip_pkt = 1;</div><div class='ctx'> 			}</div><div class='del'>-</div><div class='del'>-			if (needset) {</div><div class='del'>-				dd_dev_info(dd,</div><div class='del'>-					    "Switching to DMA_RTAIL\n");</div><div class='del'>-				set_dma_rtail(dd, rcd-&gt;ctxt);</div><div class='del'>-				needset = 0;</div><div class='del'>-			}</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='add'>+		if (needset) {</div><div class='add'>+			needset = false;</div><div class='add'>+			set_all_fastpath(dd, rcd);</div><div class='add'>+		}</div><div class='ctx'> 		process_rcv_update(last, &amp;packet);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	process_rcv_qp_work(&amp;packet);</div><div class='del'>-	rcd-&gt;head = packet.rhqoff;</div><div class='add'>+	hfi1_set_rcd_head(rcd, packet.rhqoff);</div><div class='ctx'> </div><div class='ctx'> bail:</div><div class='ctx'> 	/*</div><div class='hunk'>@@ -1126,6 +1096,63 @@ bail:</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='add'>+ * handle_receive_interrupt_napi_sp - receive a packet</div><div class='add'>+ * @rcd: the context</div><div class='add'>+ * @budget: polling budget</div><div class='add'>+ *</div><div class='add'>+ * Called from interrupt handler for errors or receive interrupt.</div><div class='add'>+ * This is the slow path interrupt handler</div><div class='add'>+ * when executing napi soft irq environment.</div><div class='add'>+ */</div><div class='add'>+int handle_receive_interrupt_napi_sp(struct hfi1_ctxtdata *rcd, int budget)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_devdata *dd = rcd-&gt;dd;</div><div class='add'>+	int last = RCV_PKT_OK;</div><div class='add'>+	bool needset = true;</div><div class='add'>+	struct hfi1_packet packet;</div><div class='add'>+</div><div class='add'>+	init_packet(rcd, &amp;packet);</div><div class='add'>+	if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))</div><div class='add'>+		goto bail;</div><div class='add'>+</div><div class='add'>+	while (last != RCV_PKT_DONE &amp;&amp; packet.numpkt &lt; budget) {</div><div class='add'>+		if (hfi1_need_drop(dd)) {</div><div class='add'>+			/* On to the next packet */</div><div class='add'>+			packet.rhqoff += packet.rsize;</div><div class='add'>+			packet.rhf_addr = (__le32 *)rcd-&gt;rcvhdrq +</div><div class='add'>+					  packet.rhqoff +</div><div class='add'>+					  rcd-&gt;rhf_offset;</div><div class='add'>+			packet.rhf = rhf_to_cpu(packet.rhf_addr);</div><div class='add'>+</div><div class='add'>+		} else {</div><div class='add'>+			if (set_armed_to_active(&amp;packet))</div><div class='add'>+				goto bail;</div><div class='add'>+			process_rcv_packet_napi(&amp;packet);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))</div><div class='add'>+			last = RCV_PKT_DONE;</div><div class='add'>+</div><div class='add'>+		if (needset) {</div><div class='add'>+			needset = false;</div><div class='add'>+			set_all_fastpath(dd, rcd);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		process_rcv_update(last, &amp;packet);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hfi1_set_rcd_head(rcd, packet.rhqoff);</div><div class='add'>+</div><div class='add'>+bail:</div><div class='add'>+	/*</div><div class='add'>+	 * Always write head at end, and setup rcv interrupt, even</div><div class='add'>+	 * if no packets were processed.</div><div class='add'>+	 */</div><div class='add'>+	finish_packet(&amp;packet);</div><div class='add'>+	return packet.numpkt;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='ctx'>  * We may discover in the interrupt that the hardware link state has</div><div class='ctx'>  * changed from ARMED to ACTIVE (due to the arrival of a non-SC15 packet),</div><div class='ctx'>  * and we need to update the driver's notion of the link state.  We cannot</div><div class='hunk'>@@ -1276,7 +1303,7 @@ void shutdown_led_override(struct hfi1_pportdata *ppd)</div><div class='ctx'> 	 */</div><div class='ctx'> 	smp_rmb();</div><div class='ctx'> 	if (atomic_read(&amp;ppd-&gt;led_override_timer_active)) {</div><div class='del'>-		del_timer_sync(&amp;ppd-&gt;led_override_timer);</div><div class='add'>+		timer_delete_sync(&amp;ppd-&gt;led_override_timer);</div><div class='ctx'> 		atomic_set(&amp;ppd-&gt;led_override_timer_active, 0);</div><div class='ctx'> 		/* Ensure the atomic_set is visible to all CPUs */</div><div class='ctx'> 		smp_wmb();</div><div class='hunk'>@@ -1288,7 +1315,8 @@ void shutdown_led_override(struct hfi1_pportdata *ppd)</div><div class='ctx'> </div><div class='ctx'> static void run_led_override(struct timer_list *t)</div><div class='ctx'> {</div><div class='del'>-	struct hfi1_pportdata *ppd = from_timer(ppd, t, led_override_timer);</div><div class='add'>+	struct hfi1_pportdata *ppd = timer_container_of(ppd, t,</div><div class='add'>+							led_override_timer);</div><div class='ctx'> 	struct hfi1_devdata *dd = ppd-&gt;dd;</div><div class='ctx'> 	unsigned long timeout;</div><div class='ctx'> 	int phase_idx;</div><div class='hunk'>@@ -1570,7 +1598,7 @@ static int hfi1_setup_bypass_packet(struct hfi1_packet *packet)</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> drop:</div><div class='del'>-	hfi1_cdbg(PKT, "%s: packet dropped\n", __func__);</div><div class='add'>+	hfi1_cdbg(PKT, "%s: packet dropped", __func__);</div><div class='ctx'> 	ibp-&gt;rvp.n_pkt_drops++;</div><div class='ctx'> 	return -EINVAL;</div><div class='ctx'> }</div><div class='hunk'>@@ -1602,59 +1630,114 @@ void handle_eflags(struct hfi1_packet *packet)</div><div class='ctx'> 		show_eflags_errs(packet);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void hfi1_ipoib_ib_rcv(struct hfi1_packet *packet)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ibport *ibp;</div><div class='add'>+	struct net_device *netdev;</div><div class='add'>+	struct hfi1_ctxtdata *rcd = packet-&gt;rcd;</div><div class='add'>+	struct napi_struct *napi = rcd-&gt;napi;</div><div class='add'>+	struct sk_buff *skb;</div><div class='add'>+	struct hfi1_netdev_rxq *rxq = container_of(napi,</div><div class='add'>+			struct hfi1_netdev_rxq, napi);</div><div class='add'>+	u32 extra_bytes;</div><div class='add'>+	u32 tlen, qpnum;</div><div class='add'>+	bool do_work, do_cnp;</div><div class='add'>+</div><div class='add'>+	trace_hfi1_rcvhdr(packet);</div><div class='add'>+</div><div class='add'>+	hfi1_setup_ib_header(packet);</div><div class='add'>+</div><div class='add'>+	packet-&gt;ohdr = &amp;((struct ib_header *)packet-&gt;hdr)-&gt;u.oth;</div><div class='add'>+	packet-&gt;grh = NULL;</div><div class='add'>+</div><div class='add'>+	if (unlikely(rhf_err_flags(packet-&gt;rhf))) {</div><div class='add'>+		handle_eflags(packet);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	qpnum = ib_bth_get_qpn(packet-&gt;ohdr);</div><div class='add'>+	netdev = hfi1_netdev_get_data(rcd-&gt;dd, qpnum);</div><div class='add'>+	if (!netdev)</div><div class='add'>+		goto drop_no_nd;</div><div class='add'>+</div><div class='add'>+	trace_input_ibhdr(rcd-&gt;dd, packet, !!(rhf_dc_info(packet-&gt;rhf)));</div><div class='add'>+	trace_ctxt_rsm_hist(rcd-&gt;ctxt);</div><div class='add'>+</div><div class='add'>+	/* handle congestion notifications */</div><div class='add'>+	do_work = hfi1_may_ecn(packet);</div><div class='add'>+	if (unlikely(do_work)) {</div><div class='add'>+		do_cnp = (packet-&gt;opcode != IB_OPCODE_CNP);</div><div class='add'>+		(void)hfi1_process_ecn_slowpath(hfi1_ipoib_priv(netdev)-&gt;qp,</div><div class='add'>+						 packet, do_cnp);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * We have split point after last byte of DETH</div><div class='add'>+	 * lets strip padding and CRC and ICRC.</div><div class='add'>+	 * tlen is whole packet len so we need to</div><div class='add'>+	 * subtract header size as well.</div><div class='add'>+	 */</div><div class='add'>+	tlen = packet-&gt;tlen;</div><div class='add'>+	extra_bytes = ib_bth_get_pad(packet-&gt;ohdr) + (SIZE_OF_CRC &lt;&lt; 2) +</div><div class='add'>+			packet-&gt;hlen;</div><div class='add'>+	if (unlikely(tlen &lt; extra_bytes))</div><div class='add'>+		goto drop;</div><div class='add'>+</div><div class='add'>+	tlen -= extra_bytes;</div><div class='add'>+</div><div class='add'>+	skb = hfi1_ipoib_prepare_skb(rxq, tlen, packet-&gt;ebuf);</div><div class='add'>+	if (unlikely(!skb))</div><div class='add'>+		goto drop;</div><div class='add'>+</div><div class='add'>+	dev_sw_netstats_rx_add(netdev, skb-&gt;len);</div><div class='add'>+</div><div class='add'>+	skb-&gt;dev = netdev;</div><div class='add'>+	skb-&gt;pkt_type = PACKET_HOST;</div><div class='add'>+	netif_receive_skb(skb);</div><div class='add'>+</div><div class='add'>+	return;</div><div class='add'>+</div><div class='add'>+drop:</div><div class='add'>+	++netdev-&gt;stats.rx_dropped;</div><div class='add'>+drop_no_nd:</div><div class='add'>+	ibp = rcd_to_iport(packet-&gt;rcd);</div><div class='add'>+	++ibp-&gt;rvp.n_pkt_drops;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /*</div><div class='ctx'>  * The following functions are called by the interrupt handler. They are type</div><div class='ctx'>  * specific handlers for each packet type.</div><div class='ctx'>  */</div><div class='del'>-static int process_receive_ib(struct hfi1_packet *packet)</div><div class='add'>+static void process_receive_ib(struct hfi1_packet *packet)</div><div class='ctx'> {</div><div class='ctx'> 	if (hfi1_setup_9B_packet(packet))</div><div class='del'>-		return RHF_RCV_CONTINUE;</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='ctx'> 	if (unlikely(hfi1_dbg_should_fault_rx(packet)))</div><div class='del'>-		return RHF_RCV_CONTINUE;</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='ctx'> 	trace_hfi1_rcvhdr(packet);</div><div class='ctx'> </div><div class='ctx'> 	if (unlikely(rhf_err_flags(packet-&gt;rhf))) {</div><div class='ctx'> 		handle_eflags(packet);</div><div class='del'>-		return RHF_RCV_CONTINUE;</div><div class='add'>+		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	hfi1_ib_rcv(packet);</div><div class='del'>-	return RHF_RCV_CONTINUE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline bool hfi1_is_vnic_packet(struct hfi1_packet *packet)</div><div class='del'>-{</div><div class='del'>-	/* Packet received in VNIC context via RSM */</div><div class='del'>-	if (packet-&gt;rcd-&gt;is_vnic)</div><div class='del'>-		return true;</div><div class='del'>-</div><div class='del'>-	if ((hfi1_16B_get_l2(packet-&gt;ebuf) == OPA_16B_L2_TYPE) &amp;&amp;</div><div class='del'>-	    (hfi1_16B_get_l4(packet-&gt;ebuf) == OPA_16B_L4_ETHR))</div><div class='del'>-		return true;</div><div class='del'>-</div><div class='del'>-	return false;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int process_receive_bypass(struct hfi1_packet *packet)</div><div class='add'>+static void process_receive_bypass(struct hfi1_packet *packet)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_devdata *dd = packet-&gt;rcd-&gt;dd;</div><div class='ctx'> </div><div class='del'>-	if (hfi1_is_vnic_packet(packet)) {</div><div class='del'>-		hfi1_vnic_bypass_rcv(packet);</div><div class='del'>-		return RHF_RCV_CONTINUE;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	if (hfi1_setup_bypass_packet(packet))</div><div class='del'>-		return RHF_RCV_CONTINUE;</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='ctx'> 	trace_hfi1_rcvhdr(packet);</div><div class='ctx'> </div><div class='ctx'> 	if (unlikely(rhf_err_flags(packet-&gt;rhf))) {</div><div class='ctx'> 		handle_eflags(packet);</div><div class='del'>-		return RHF_RCV_CONTINUE;</div><div class='add'>+		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (hfi1_16B_get_l2(packet-&gt;hdr) == 0x2) {</div><div class='hunk'>@@ -1677,17 +1760,16 @@ static int process_receive_bypass(struct hfi1_packet *packet)</div><div class='ctx'> 				(OPA_EI_STATUS_SMASK | BAD_L2_ERR);</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='del'>-	return RHF_RCV_CONTINUE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int process_receive_error(struct hfi1_packet *packet)</div><div class='add'>+static void process_receive_error(struct hfi1_packet *packet)</div><div class='ctx'> {</div><div class='ctx'> 	/* KHdrHCRCErr -- KDETH packet with a bad HCRC */</div><div class='ctx'> 	if (unlikely(</div><div class='ctx'> 		 hfi1_dbg_fault_suppress_err(&amp;packet-&gt;rcd-&gt;dd-&gt;verbs_dev) &amp;&amp;</div><div class='ctx'> 		 (rhf_rcv_type_err(packet-&gt;rhf) == RHF_RCV_TYPE_ERROR ||</div><div class='ctx'> 		  packet-&gt;rhf &amp; RHF_DC_ERR)))</div><div class='del'>-		return RHF_RCV_CONTINUE;</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='ctx'> 	hfi1_setup_ib_header(packet);</div><div class='ctx'> 	handle_eflags(packet);</div><div class='hunk'>@@ -1695,32 +1777,29 @@ static int process_receive_error(struct hfi1_packet *packet)</div><div class='ctx'> 	if (unlikely(rhf_err_flags(packet-&gt;rhf)))</div><div class='ctx'> 		dd_dev_err(packet-&gt;rcd-&gt;dd,</div><div class='ctx'> 			   "Unhandled error packet received. Dropping.\n");</div><div class='del'>-</div><div class='del'>-	return RHF_RCV_CONTINUE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int kdeth_process_expected(struct hfi1_packet *packet)</div><div class='add'>+static void kdeth_process_expected(struct hfi1_packet *packet)</div><div class='ctx'> {</div><div class='ctx'> 	hfi1_setup_9B_packet(packet);</div><div class='ctx'> 	if (unlikely(hfi1_dbg_should_fault_rx(packet)))</div><div class='del'>-		return RHF_RCV_CONTINUE;</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='ctx'> 	if (unlikely(rhf_err_flags(packet-&gt;rhf))) {</div><div class='ctx'> 		struct hfi1_ctxtdata *rcd = packet-&gt;rcd;</div><div class='ctx'> </div><div class='ctx'> 		if (hfi1_handle_kdeth_eflags(rcd, rcd-&gt;ppd, packet))</div><div class='del'>-			return RHF_RCV_CONTINUE;</div><div class='add'>+			return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	hfi1_kdeth_expected_rcv(packet);</div><div class='del'>-	return RHF_RCV_CONTINUE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int kdeth_process_eager(struct hfi1_packet *packet)</div><div class='add'>+static void kdeth_process_eager(struct hfi1_packet *packet)</div><div class='ctx'> {</div><div class='ctx'> 	hfi1_setup_9B_packet(packet);</div><div class='ctx'> 	if (unlikely(hfi1_dbg_should_fault_rx(packet)))</div><div class='del'>-		return RHF_RCV_CONTINUE;</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='ctx'> 	trace_hfi1_rcvhdr(packet);</div><div class='ctx'> 	if (unlikely(rhf_err_flags(packet-&gt;rhf))) {</div><div class='hunk'>@@ -1728,37 +1807,41 @@ static int kdeth_process_eager(struct hfi1_packet *packet)</div><div class='ctx'> </div><div class='ctx'> 		show_eflags_errs(packet);</div><div class='ctx'> 		if (hfi1_handle_kdeth_eflags(rcd, rcd-&gt;ppd, packet))</div><div class='del'>-			return RHF_RCV_CONTINUE;</div><div class='add'>+			return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	hfi1_kdeth_eager_rcv(packet);</div><div class='del'>-	return RHF_RCV_CONTINUE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int process_receive_invalid(struct hfi1_packet *packet)</div><div class='add'>+static void process_receive_invalid(struct hfi1_packet *packet)</div><div class='ctx'> {</div><div class='ctx'> 	dd_dev_err(packet-&gt;rcd-&gt;dd, "Invalid packet type %d. Dropping\n",</div><div class='ctx'> 		   rhf_rcv_type(packet-&gt;rhf));</div><div class='del'>-	return RHF_RCV_CONTINUE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+#define HFI1_RCVHDR_DUMP_MAX	5</div><div class='add'>+</div><div class='ctx'> void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_packet packet;</div><div class='ctx'> 	struct ps_mdata mdata;</div><div class='add'>+	int i;</div><div class='ctx'> </div><div class='del'>-	seq_printf(s, "Rcd %u: RcvHdr cnt %u entsize %u %s head %llu tail %llu\n",</div><div class='del'>-		   rcd-&gt;ctxt, rcd-&gt;rcvhdrq_cnt, rcd-&gt;rcvhdrqentsize,</div><div class='del'>-		   HFI1_CAP_KGET_MASK(rcd-&gt;flags, DMA_RTAIL) ?</div><div class='add'>+	seq_printf(s, "Rcd %u: RcvHdr cnt %u entsize %u %s ctrl 0x%08llx status 0x%08llx, head %llu tail %llu  sw head %u\n",</div><div class='add'>+		   rcd-&gt;ctxt, get_hdrq_cnt(rcd), get_hdrqentsize(rcd),</div><div class='add'>+		   get_dma_rtail_setting(rcd) ?</div><div class='ctx'> 		   "dma_rtail" : "nodma_rtail",</div><div class='add'>+		   read_kctxt_csr(rcd-&gt;dd, rcd-&gt;ctxt, RCV_CTXT_CTRL),</div><div class='add'>+		   read_kctxt_csr(rcd-&gt;dd, rcd-&gt;ctxt, RCV_CTXT_STATUS),</div><div class='ctx'> 		   read_uctxt_csr(rcd-&gt;dd, rcd-&gt;ctxt, RCV_HDR_HEAD) &amp;</div><div class='ctx'> 		   RCV_HDR_HEAD_HEAD_MASK,</div><div class='del'>-		   read_uctxt_csr(rcd-&gt;dd, rcd-&gt;ctxt, RCV_HDR_TAIL));</div><div class='add'>+		   read_uctxt_csr(rcd-&gt;dd, rcd-&gt;ctxt, RCV_HDR_TAIL),</div><div class='add'>+		   rcd-&gt;head);</div><div class='ctx'> </div><div class='ctx'> 	init_packet(rcd, &amp;packet);</div><div class='ctx'> 	init_ps_mdata(&amp;mdata, &amp;packet);</div><div class='ctx'> </div><div class='del'>-	while (1) {</div><div class='add'>+	for (i = 0; i &lt; HFI1_RCVHDR_DUMP_MAX; i++) {</div><div class='ctx'> 		__le32 *rhf_addr = (__le32 *)rcd-&gt;rcvhdrq + mdata.ps_head +</div><div class='ctx'> 					 rcd-&gt;rhf_offset;</div><div class='ctx'> 		struct ib_header *hdr;</div><div class='hunk'>@@ -1810,3 +1893,14 @@ const rhf_rcv_function_ptr normal_rhf_rcv_functions[] = {</div><div class='ctx'> 	[RHF_RCV_TYPE_INVALID6] = process_receive_invalid,</div><div class='ctx'> 	[RHF_RCV_TYPE_INVALID7] = process_receive_invalid,</div><div class='ctx'> };</div><div class='add'>+</div><div class='add'>+const rhf_rcv_function_ptr netdev_rhf_rcv_functions[] = {</div><div class='add'>+	[RHF_RCV_TYPE_EXPECTED] = process_receive_invalid,</div><div class='add'>+	[RHF_RCV_TYPE_EAGER] = process_receive_invalid,</div><div class='add'>+	[RHF_RCV_TYPE_IB] = hfi1_ipoib_ib_rcv,</div><div class='add'>+	[RHF_RCV_TYPE_ERROR] = process_receive_error,</div><div class='add'>+	[RHF_RCV_TYPE_BYPASS] = hfi1_vnic_bypass_rcv,</div><div class='add'>+	[RHF_RCV_TYPE_INVALID5] = process_receive_invalid,</div><div class='add'>+	[RHF_RCV_TYPE_INVALID6] = process_receive_invalid,</div><div class='add'>+	[RHF_RCV_TYPE_INVALID7] = process_receive_invalid,</div><div class='add'>+};</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/efivar.c b/drivers/infiniband/hw/hfi1/efivar.c<br/>index d106d23016ba..9ed05e10020e 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/efivar.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/efivar.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/efivar.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/efivar.c</a></div><div class='hunk'>@@ -1,51 +1,11 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015, 2016 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/ctype.h&gt;</div><div class='add'>+#include &lt;linux/string.h&gt;</div><div class='add'>+#include &lt;linux/string_helpers.h&gt;</div><div class='add'>+</div><div class='ctx'> #include "efivar.h"</div><div class='ctx'> </div><div class='ctx'> /* GUID for HFI1 variables in EFI */</div><div class='hunk'>@@ -78,7 +38,7 @@ static int read_efi_var(const char *name, unsigned long *size,</div><div class='ctx'> 	*size = 0;</div><div class='ctx'> 	*return_data = NULL;</div><div class='ctx'> </div><div class='del'>-	if (!efi_enabled(EFI_RUNTIME_SERVICES))</div><div class='add'>+	if (!efi_rt_services_supported(EFI_RT_SUPPORTED_GET_VARIABLE))</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	uni_name = kcalloc(strlen(name) + 1, sizeof(efi_char16_t), GFP_KERNEL);</div><div class='hunk'>@@ -112,7 +72,7 @@ static int read_efi_var(const char *name, unsigned long *size,</div><div class='ctx'> 	 * is in the EFIVAR_FS code and may not be compiled in.</div><div class='ctx'> 	 * However, even that is insufficient since it does not cover</div><div class='ctx'> 	 * EFI_BUFFER_TOO_SMALL which could be an important return.</div><div class='del'>-	 * For now, just split out succces or not found.</div><div class='add'>+	 * For now, just split out success or not found.</div><div class='ctx'> 	 */</div><div class='ctx'> 	ret = status == EFI_SUCCESS   ? 0 :</div><div class='ctx'> 	      status == EFI_NOT_FOUND ? -ENOENT :</div><div class='hunk'>@@ -152,9 +112,8 @@ int read_hfi1_efi_var(struct hfi1_devdata *dd, const char *kind,</div><div class='ctx'> 		      unsigned long *size, void **return_data)</div><div class='ctx'> {</div><div class='ctx'> 	char prefix_name[64];</div><div class='del'>-	char name[64];</div><div class='add'>+	char name[128];</div><div class='ctx'> 	int result;</div><div class='del'>-	int i;</div><div class='ctx'> </div><div class='ctx'> 	/* create a common prefix */</div><div class='ctx'> 	snprintf(prefix_name, sizeof(prefix_name), "%04x:%02x:%02x.%x",</div><div class='hunk'>@@ -170,10 +129,7 @@ int read_hfi1_efi_var(struct hfi1_devdata *dd, const char *kind,</div><div class='ctx'> 	 * variable.</div><div class='ctx'> 	 */</div><div class='ctx'> 	if (result) {</div><div class='del'>-		/* Converting to uppercase */</div><div class='del'>-		for (i = 0; prefix_name[i]; i++)</div><div class='del'>-			if (isalpha(prefix_name[i]))</div><div class='del'>-				prefix_name[i] = toupper(prefix_name[i]);</div><div class='add'>+		string_upper(prefix_name, prefix_name);</div><div class='ctx'> 		snprintf(name, sizeof(name), "%s-%s", prefix_name, kind);</div><div class='ctx'> 		result = read_efi_var(name, size, return_data);</div><div class='ctx'> 	}</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/efivar.h b/drivers/infiniband/hw/hfi1/efivar.h<br/>index 94e9e70de568..882240929a4b 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/efivar.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/efivar.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/efivar.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/efivar.h</a></div><div class='hunk'>@@ -1,49 +1,8 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015, 2016 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #ifndef _HFI1_EFIVAR_H</div><div class='ctx'> #define _HFI1_EFIVAR_H</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/eprom.c b/drivers/infiniband/hw/hfi1/eprom.c<br/>index 1613af1c58d9..f93a160d8d05 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/eprom.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/eprom.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/eprom.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/eprom.c</a></div><div class='hunk'>@@ -1,49 +1,8 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015, 2016 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #include &lt;linux/delay.h&gt;</div><div class='ctx'> #include "hfi.h"</div><div class='ctx'> #include "common.h"</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/eprom.h b/drivers/infiniband/hw/hfi1/eprom.h<br/>index e774184f1643..51648d1afcf1 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/eprom.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/eprom.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/eprom.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/eprom.h</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015, 2016 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> struct hfi1_devdata;</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/exp_rcv.c b/drivers/infiniband/hw/hfi1/exp_rcv.c<br/>index e9d5cc8b771a..879a66edbded 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/exp_rcv.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/exp_rcv.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/exp_rcv.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/exp_rcv.c</a></div><div class='hunk'>@@ -1,56 +1,14 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2017 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include "exp_rcv.h"</div><div class='ctx'> #include "trace.h"</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * exp_tid_group_init - initialize exp_tid_set</div><div class='del'>- * @set - the set</div><div class='add'>+ * hfi1_exp_tid_set_init - initialize exp_tid_set</div><div class='add'>+ * @set: the set</div><div class='ctx'>  */</div><div class='ctx'> static void hfi1_exp_tid_set_init(struct exp_tid_set *set)</div><div class='ctx'> {</div><div class='hunk'>@@ -60,7 +18,7 @@ static void hfi1_exp_tid_set_init(struct exp_tid_set *set)</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * hfi1_exp_tid_group_init - initialize rcd expected receive</div><div class='del'>- * @rcd - the rcd</div><div class='add'>+ * @rcd: the rcd</div><div class='ctx'>  */</div><div class='ctx'> void hfi1_exp_tid_group_init(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> {</div><div class='hunk'>@@ -70,8 +28,8 @@ void hfi1_exp_tid_group_init(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * alloc_ctxt_rcv_groups - initialize expected receive groups</div><div class='del'>- * @rcd - the context to add the groupings to</div><div class='add'>+ * hfi1_alloc_ctxt_rcv_groups - initialize expected receive groups</div><div class='add'>+ * @rcd: the context to add the groupings to</div><div class='ctx'>  */</div><div class='ctx'> int hfi1_alloc_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> {</div><div class='hunk'>@@ -100,8 +58,8 @@ int hfi1_alloc_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * free_ctxt_rcv_groups - free  expected receive groups</div><div class='del'>- * @rcd - the context to free</div><div class='add'>+ * hfi1_free_ctxt_rcv_groups - free  expected receive groups</div><div class='add'>+ * @rcd: the context to free</div><div class='ctx'>  *</div><div class='ctx'>  * The routine dismantles the expect receive linked</div><div class='ctx'>  * list and clears any tids associated with the receive</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/exp_rcv.h b/drivers/infiniband/hw/hfi1/exp_rcv.h<br/>index f25362015095..141413d9fbc7 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/exp_rcv.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/exp_rcv.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/exp_rcv.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/exp_rcv.h</a></div><div class='hunk'>@@ -1,52 +1,10 @@</div><div class='del'>-#ifndef _HFI1_EXP_RCV_H</div><div class='del'>-#define _HFI1_EXP_RCV_H</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2017 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='add'>+#ifndef _HFI1_EXP_RCV_H</div><div class='add'>+#define _HFI1_EXP_RCV_H</div><div class='ctx'> #include "hfi.h"</div><div class='ctx'> </div><div class='ctx'> #define EXP_TID_SET_EMPTY(set) (set.count == 0 &amp;&amp; list_empty(&amp;set.list))</div><div class='hunk'>@@ -175,12 +133,13 @@ static inline struct tid_group *tid_group_pop(struct exp_tid_set *set)</div><div class='ctx'> 	return grp;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline u32 rcventry2tidinfo(u32 rcventry)</div><div class='add'>+static inline u32 create_tid(u32 rcventry, u32 npages)</div><div class='ctx'> {</div><div class='ctx'> 	u32 pair = rcventry &amp; ~0x1;</div><div class='ctx'> </div><div class='ctx'> 	return EXP_TID_SET(IDX, pair &gt;&gt; 1) |</div><div class='del'>-		EXP_TID_SET(CTRL, 1 &lt;&lt; (rcventry - pair));</div><div class='add'>+		EXP_TID_SET(CTRL, 1 &lt;&lt; (rcventry - pair)) |</div><div class='add'>+		EXP_TID_SET(LEN, npages);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/fault.c b/drivers/infiniband/hw/hfi1/fault.c<br/>index 3fd3315d0fb0..a45cbffd52c7 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/fault.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/fault.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/fault.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/fault.c</a></div><div class='hunk'>@@ -1,53 +1,11 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #include &lt;linux/debugfs.h&gt;</div><div class='ctx'> #include &lt;linux/seq_file.h&gt;</div><div class='ctx'> #include &lt;linux/kernel.h&gt;</div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='ctx'> #include &lt;linux/types.h&gt;</div><div class='ctx'> #include &lt;linux/bitmap.h&gt;</div><div class='ctx'> </div><div class='hunk'>@@ -141,18 +99,18 @@ static ssize_t fault_opcodes_write(struct file *file, const char __user *buf,</div><div class='ctx'> 	if (!data)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> 	copy = min(len, datalen - 1);</div><div class='del'>-	if (copy_from_user(data, buf, copy))</div><div class='del'>-		return -EFAULT;</div><div class='add'>+	if (copy_from_user(data, buf, copy)) {</div><div class='add'>+		ret = -EFAULT;</div><div class='add'>+		goto free_data;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	ret = debugfs_file_get(file-&gt;f_path.dentry);</div><div class='del'>-	if (unlikely(ret))</div><div class='del'>-		return ret;</div><div class='ctx'> 	ptr = data;</div><div class='ctx'> 	token = ptr;</div><div class='ctx'> 	for (ptr = data; *ptr; ptr = end + 1, token = ptr) {</div><div class='ctx'> 		char *dash;</div><div class='ctx'> 		unsigned long range_start, range_end, i;</div><div class='ctx'> 		bool remove = false;</div><div class='add'>+		unsigned long bound = 1U &lt;&lt; BITS_PER_BYTE;</div><div class='ctx'> </div><div class='ctx'> 		end = strchr(ptr, ',');</div><div class='ctx'> 		if (end)</div><div class='hunk'>@@ -178,6 +136,10 @@ static ssize_t fault_opcodes_write(struct file *file, const char __user *buf,</div><div class='ctx'> 				    BITS_PER_BYTE);</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='add'>+		/* Check the inputs */</div><div class='add'>+		if (range_start &gt;= bound || range_end &gt;= bound)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='ctx'> 		for (i = range_start; i &lt;= range_end; i++) {</div><div class='ctx'> 			if (remove)</div><div class='ctx'> 				clear_bit(i, fault-&gt;opcodes);</div><div class='hunk'>@@ -189,7 +151,7 @@ static ssize_t fault_opcodes_write(struct file *file, const char __user *buf,</div><div class='ctx'> 	}</div><div class='ctx'> 	ret = len;</div><div class='ctx'> </div><div class='del'>-	debugfs_file_put(file-&gt;f_path.dentry);</div><div class='add'>+free_data:</div><div class='ctx'> 	kfree(data);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -207,23 +169,19 @@ static ssize_t fault_opcodes_read(struct file *file, char __user *buf,</div><div class='ctx'> 	data = kcalloc(datalen, sizeof(*data), GFP_KERNEL);</div><div class='ctx'> 	if (!data)</div><div class='ctx'> 		return -ENOMEM;</div><div class='del'>-	ret = debugfs_file_get(file-&gt;f_path.dentry);</div><div class='del'>-	if (unlikely(ret))</div><div class='del'>-		return ret;</div><div class='ctx'> 	bit = find_first_bit(fault-&gt;opcodes, bitsize);</div><div class='ctx'> 	while (bit &lt; bitsize) {</div><div class='ctx'> 		zero = find_next_zero_bit(fault-&gt;opcodes, bitsize, bit);</div><div class='ctx'> 		if (zero - 1 != bit)</div><div class='del'>-			size += snprintf(data + size,</div><div class='add'>+			size += scnprintf(data + size,</div><div class='ctx'> 					 datalen - size - 1,</div><div class='ctx'> 					 "0x%lx-0x%lx,", bit, zero - 1);</div><div class='ctx'> 		else</div><div class='del'>-			size += snprintf(data + size,</div><div class='add'>+			size += scnprintf(data + size,</div><div class='ctx'> 					 datalen - size - 1, "0x%lx,",</div><div class='ctx'> 					 bit);</div><div class='ctx'> 		bit = find_next_bit(fault-&gt;opcodes, bitsize, zero);</div><div class='ctx'> 	}</div><div class='del'>-	debugfs_file_put(file-&gt;f_path.dentry);</div><div class='ctx'> 	data[size - 1] = '\n';</div><div class='ctx'> 	data[size] = '\0';</div><div class='ctx'> 	ret = simple_read_from_buffer(buf, len, pos, data, size);</div><div class='hunk'>@@ -236,7 +194,6 @@ static const struct file_operations __fault_opcodes_fops = {</div><div class='ctx'> 	.open = fault_opcodes_open,</div><div class='ctx'> 	.read = fault_opcodes_read,</div><div class='ctx'> 	.write = fault_opcodes_write,</div><div class='del'>-	.llseek = no_llseek</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> void hfi1_fault_exit_debugfs(struct hfi1_ibdev *ibd)</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/fault.h b/drivers/infiniband/hw/hfi1/fault.h<br/>index a83382700a7c..51adafe240d7 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/fault.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/fault.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/fault.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/fault.h</a></div><div class='hunk'>@@ -1,51 +1,11 @@</div><div class='del'>-#ifndef _HFI1_FAULT_H</div><div class='del'>-#define _HFI1_FAULT_H</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='add'>+</div><div class='add'>+#ifndef _HFI1_FAULT_H</div><div class='add'>+#define _HFI1_FAULT_H</div><div class='add'>+</div><div class='ctx'> #include &lt;linux/fault-inject.h&gt;</div><div class='ctx'> #include &lt;linux/dcache.h&gt;</div><div class='ctx'> #include &lt;linux/bitops.h&gt;</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c<br/>index f9a7e9d29c8b..503abec709c9 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/file_ops.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/file_ops.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/file_ops.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/file_ops.c</a></div><div class='hunk'>@@ -1,49 +1,9 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2015-2017 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2020 Cornelis Networks, Inc.</div><div class='add'>+ * Copyright(c) 2015-2020 Intel Corporation.</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #include &lt;linux/poll.h&gt;</div><div class='ctx'> #include &lt;linux/cdev.h&gt;</div><div class='ctx'> #include &lt;linux/vmalloc.h&gt;</div><div class='hunk'>@@ -193,30 +153,28 @@ static int hfi1_file_open(struct inode *inode, struct file *fp)</div><div class='ctx'> 	if (!((dd-&gt;flags &amp; HFI1_PRESENT) &amp;&amp; dd-&gt;kregbase1))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (!atomic_inc_not_zero(&amp;dd-&gt;user_refcount))</div><div class='add'>+	if (!refcount_inc_not_zero(&amp;dd-&gt;user_refcount))</div><div class='ctx'> 		return -ENXIO;</div><div class='ctx'> </div><div class='ctx'> 	/* The real work is performed later in assign_ctxt() */</div><div class='ctx'> </div><div class='ctx'> 	fd = kzalloc(sizeof(*fd), GFP_KERNEL);</div><div class='ctx'> </div><div class='del'>-	if (fd) {</div><div class='del'>-		fd-&gt;rec_cpu_num = -1; /* no cpu affinity by default */</div><div class='del'>-		fd-&gt;mm = current-&gt;mm;</div><div class='del'>-		mmgrab(fd-&gt;mm);</div><div class='del'>-		fd-&gt;dd = dd;</div><div class='del'>-		kobject_get(&amp;fd-&gt;dd-&gt;kobj);</div><div class='del'>-		fp-&gt;private_data = fd;</div><div class='del'>-	} else {</div><div class='del'>-		fp-&gt;private_data = NULL;</div><div class='del'>-</div><div class='del'>-		if (atomic_dec_and_test(&amp;dd-&gt;user_refcount))</div><div class='del'>-			complete(&amp;dd-&gt;user_comp);</div><div class='del'>-</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-</div><div class='add'>+	if (!fd || init_srcu_struct(&amp;fd-&gt;pq_srcu))</div><div class='add'>+		goto nomem;</div><div class='add'>+	spin_lock_init(&amp;fd-&gt;pq_rcu_lock);</div><div class='add'>+	spin_lock_init(&amp;fd-&gt;tid_lock);</div><div class='add'>+	spin_lock_init(&amp;fd-&gt;invalid_lock);</div><div class='add'>+	fd-&gt;rec_cpu_num = -1; /* no cpu affinity by default */</div><div class='add'>+	fd-&gt;dd = dd;</div><div class='add'>+	fp-&gt;private_data = fd;</div><div class='ctx'> 	return 0;</div><div class='add'>+nomem:</div><div class='add'>+	kfree(fd);</div><div class='add'>+	fp-&gt;private_data = NULL;</div><div class='add'>+	if (refcount_dec_and_test(&amp;dd-&gt;user_refcount))</div><div class='add'>+		complete(&amp;dd-&gt;user_comp);</div><div class='add'>+	return -ENOMEM;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static long hfi1_file_ioctl(struct file *fp, unsigned int cmd,</div><div class='hunk'>@@ -301,28 +259,37 @@ static long hfi1_file_ioctl(struct file *fp, unsigned int cmd,</div><div class='ctx'> static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_filedata *fd = kiocb-&gt;ki_filp-&gt;private_data;</div><div class='del'>-	struct hfi1_user_sdma_pkt_q *pq = fd-&gt;pq;</div><div class='add'>+	struct hfi1_user_sdma_pkt_q *pq;</div><div class='ctx'> 	struct hfi1_user_sdma_comp_q *cq = fd-&gt;cq;</div><div class='ctx'> 	int done = 0, reqs = 0;</div><div class='ctx'> 	unsigned long dim = from-&gt;nr_segs;</div><div class='add'>+	int idx;</div><div class='ctx'> </div><div class='del'>-	if (!cq || !pq)</div><div class='del'>-		return -EIO;</div><div class='del'>-</div><div class='del'>-	if (!iter_is_iovec(from) || !dim)</div><div class='add'>+	if (!HFI1_CAP_IS_KSET(SDMA))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	if (!user_backed_iter(from))</div><div class='ctx'> 		return -EINVAL;</div><div class='add'>+	idx = srcu_read_lock(&amp;fd-&gt;pq_srcu);</div><div class='add'>+	pq = srcu_dereference(fd-&gt;pq, &amp;fd-&gt;pq_srcu);</div><div class='add'>+	if (!cq || !pq) {</div><div class='add'>+		srcu_read_unlock(&amp;fd-&gt;pq_srcu, idx);</div><div class='add'>+		return -EIO;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	trace_hfi1_sdma_request(fd-&gt;dd, fd-&gt;uctxt-&gt;ctxt, fd-&gt;subctxt, dim);</div><div class='ctx'> </div><div class='del'>-	if (atomic_read(&amp;pq-&gt;n_reqs) == pq-&gt;n_max_reqs)</div><div class='add'>+	if (atomic_read(&amp;pq-&gt;n_reqs) == pq-&gt;n_max_reqs) {</div><div class='add'>+		srcu_read_unlock(&amp;fd-&gt;pq_srcu, idx);</div><div class='ctx'> 		return -ENOSPC;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	while (dim) {</div><div class='add'>+		const struct iovec *iov = iter_iov(from);</div><div class='ctx'> 		int ret;</div><div class='ctx'> 		unsigned long count = 0;</div><div class='ctx'> </div><div class='ctx'> 		ret = hfi1_user_sdma_process_request(</div><div class='del'>-			fd, (struct iovec *)(from-&gt;iov + done),</div><div class='add'>+			fd, (struct iovec *)(iov + done),</div><div class='ctx'> 			dim, &amp;count);</div><div class='ctx'> 		if (ret) {</div><div class='ctx'> 			reqs = ret;</div><div class='hunk'>@@ -333,9 +300,21 @@ static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from)</div><div class='ctx'> 		reqs++;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	srcu_read_unlock(&amp;fd-&gt;pq_srcu, idx);</div><div class='ctx'> 	return reqs;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static inline void mmap_cdbg(u16 ctxt, u8 subctxt, u8 type, u8 mapio, u8 vmf,</div><div class='add'>+			     u64 memaddr, void *memvirt, dma_addr_t memdma,</div><div class='add'>+			     ssize_t memlen, struct vm_area_struct *vma)</div><div class='add'>+{</div><div class='add'>+	hfi1_cdbg(PROC,</div><div class='add'>+		  "%u:%u type:%u io/vf/dma:%d/%d/%d, addr:0x%llx, len:%lu(%lu), flags:0x%lx",</div><div class='add'>+		  ctxt, subctxt, type, mapio, vmf, !!memdma,</div><div class='add'>+		  memaddr ?: (u64)memvirt, memlen,</div><div class='add'>+		  vma-&gt;vm_end - vma-&gt;vm_start, vma-&gt;vm_flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_filedata *fd = fp-&gt;private_data;</div><div class='hunk'>@@ -345,6 +324,7 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)</div><div class='ctx'> 	u64 token = vma-&gt;vm_pgoff &lt;&lt; PAGE_SHIFT,</div><div class='ctx'> 		memaddr = 0;</div><div class='ctx'> 	void *memvirt = NULL;</div><div class='add'>+	dma_addr_t memdma = 0;</div><div class='ctx'> 	u8 subctxt, mapio = 0, vmf = 0, type;</div><div class='ctx'> 	ssize_t memlen = 0;</div><div class='ctx'> 	int ret = 0;</div><div class='hunk'>@@ -364,6 +344,11 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)</div><div class='ctx'> 		goto done;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	/*</div><div class='add'>+	 * vm_pgoff is used as a buffer selector cookie.  Always mmap from</div><div class='add'>+	 * the beginning.</div><div class='add'>+	 */ </div><div class='add'>+	vma-&gt;vm_pgoff = 0;</div><div class='ctx'> 	flags = vma-&gt;vm_flags;</div><div class='ctx'> </div><div class='ctx'> 	switch (type) {</div><div class='hunk'>@@ -385,7 +370,8 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)</div><div class='ctx'> 		vma-&gt;vm_page_prot = pgprot_writecombine(vma-&gt;vm_page_prot);</div><div class='ctx'> 		mapio = 1;</div><div class='ctx'> 		break;</div><div class='del'>-	case PIO_CRED:</div><div class='add'>+	case PIO_CRED: {</div><div class='add'>+		u64 cr_page_offset;</div><div class='ctx'> 		if (flags &amp; VM_WRITE) {</div><div class='ctx'> 			ret = -EPERM;</div><div class='ctx'> 			goto done;</div><div class='hunk'>@@ -395,10 +381,11 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)</div><div class='ctx'> 		 * second or third page allocated for credit returns (if number</div><div class='ctx'> 		 * of enabled contexts &gt; 64 and 128 respectively).</div><div class='ctx'> 		 */</div><div class='del'>-		memvirt = dd-&gt;cr_base[uctxt-&gt;numa_id].va;</div><div class='del'>-		memaddr = virt_to_phys(memvirt) +</div><div class='del'>-			(((u64)uctxt-&gt;sc-&gt;hw_free -</div><div class='del'>-			  (u64)dd-&gt;cr_base[uctxt-&gt;numa_id].va) &amp; PAGE_MASK);</div><div class='add'>+		cr_page_offset = ((u64)uctxt-&gt;sc-&gt;hw_free -</div><div class='add'>+			  	     (u64)dd-&gt;cr_base[uctxt-&gt;numa_id].va) &amp;</div><div class='add'>+				   PAGE_MASK;</div><div class='add'>+		memvirt = dd-&gt;cr_base[uctxt-&gt;numa_id].va + cr_page_offset;</div><div class='add'>+		memdma = dd-&gt;cr_base[uctxt-&gt;numa_id].dma + cr_page_offset;</div><div class='ctx'> 		memlen = PAGE_SIZE;</div><div class='ctx'> 		flags &amp;= ~VM_MAYWRITE;</div><div class='ctx'> 		flags |= VM_DONTCOPY | VM_DONTEXPAND;</div><div class='hunk'>@@ -408,14 +395,16 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)</div><div class='ctx'> 		 * memory been flagged as non-cached?</div><div class='ctx'> 		 */</div><div class='ctx'> 		/* vma-&gt;vm_page_prot = pgprot_noncached(vma-&gt;vm_page_prot); */</div><div class='del'>-		mapio = 1;</div><div class='ctx'> 		break;</div><div class='add'>+	}</div><div class='ctx'> 	case RCV_HDRQ:</div><div class='ctx'> 		memlen = rcvhdrq_size(uctxt);</div><div class='ctx'> 		memvirt = uctxt-&gt;rcvhdrq;</div><div class='add'>+		memdma = uctxt-&gt;rcvhdrq_dma;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case RCV_EGRBUF: {</div><div class='del'>-		unsigned long addr;</div><div class='add'>+		unsigned long vm_start_save;</div><div class='add'>+		unsigned long vm_end_save;</div><div class='ctx'> 		int i;</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * The RcvEgr buffer need to be handled differently</div><div class='hunk'>@@ -433,25 +422,35 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)</div><div class='ctx'> 			ret = -EPERM;</div><div class='ctx'> 			goto done;</div><div class='ctx'> 		}</div><div class='del'>-		vma-&gt;vm_flags &amp;= ~VM_MAYWRITE;</div><div class='del'>-		addr = vma-&gt;vm_start;</div><div class='add'>+		vm_flags_clear(vma, VM_MAYWRITE);</div><div class='add'>+		/*</div><div class='add'>+		 * Mmap multiple separate allocations into a single vma.  From</div><div class='add'>+		 * here, dma_mmap_coherent() calls dma_direct_mmap(), which</div><div class='add'>+		 * requires the mmap to exactly fill the vma starting at</div><div class='add'>+		 * vma_start.  Adjust the vma start and end for each eager</div><div class='add'>+		 * buffer segment mapped.  Restore the originals when done.</div><div class='add'>+		 */</div><div class='add'>+		vm_start_save = vma-&gt;vm_start;</div><div class='add'>+		vm_end_save = vma-&gt;vm_end;</div><div class='add'>+		vma-&gt;vm_end = vma-&gt;vm_start;</div><div class='ctx'> 		for (i = 0 ; i &lt; uctxt-&gt;egrbufs.numbufs; i++) {</div><div class='ctx'> 			memlen = uctxt-&gt;egrbufs.buffers[i].len;</div><div class='ctx'> 			memvirt = uctxt-&gt;egrbufs.buffers[i].addr;</div><div class='del'>-			ret = remap_pfn_range(</div><div class='del'>-				vma, addr,</div><div class='del'>-				/*</div><div class='del'>-				 * virt_to_pfn() does the same, but</div><div class='del'>-				 * it's not available on x86_64</div><div class='del'>-				 * when CONFIG_MMU is enabled.</div><div class='del'>-				 */</div><div class='del'>-				PFN_DOWN(__pa(memvirt)),</div><div class='del'>-				memlen,</div><div class='del'>-				vma-&gt;vm_page_prot);</div><div class='del'>-			if (ret &lt; 0)</div><div class='add'>+			memdma = uctxt-&gt;egrbufs.buffers[i].dma;</div><div class='add'>+			vma-&gt;vm_end += memlen;</div><div class='add'>+			mmap_cdbg(ctxt, subctxt, type, mapio, vmf, memaddr,</div><div class='add'>+				  memvirt, memdma, memlen, vma);</div><div class='add'>+			ret = dma_mmap_coherent(&amp;dd-&gt;pcidev-&gt;dev, vma,</div><div class='add'>+						memvirt, memdma, memlen);</div><div class='add'>+			if (ret &lt; 0) {</div><div class='add'>+				vma-&gt;vm_start = vm_start_save;</div><div class='add'>+				vma-&gt;vm_end = vm_end_save;</div><div class='ctx'> 				goto done;</div><div class='del'>-			addr += memlen;</div><div class='add'>+			}</div><div class='add'>+			vma-&gt;vm_start += memlen;</div><div class='ctx'> 		}</div><div class='add'>+		vma-&gt;vm_start = vm_start_save;</div><div class='add'>+		vma-&gt;vm_end = vm_end_save;</div><div class='ctx'> 		ret = 0;</div><div class='ctx'> 		goto done;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -505,12 +504,13 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)</div><div class='ctx'> 			ret = -EINVAL;</div><div class='ctx'> 			goto done;</div><div class='ctx'> 		}</div><div class='del'>-		if ((flags &amp; VM_WRITE) || !uctxt-&gt;rcvhdrtail_kvaddr) {</div><div class='add'>+		if ((flags &amp; VM_WRITE) || !hfi1_rcvhdrtail_kvaddr(uctxt)) {</div><div class='ctx'> 			ret = -EPERM;</div><div class='ctx'> 			goto done;</div><div class='ctx'> 		}</div><div class='ctx'> 		memlen = PAGE_SIZE;</div><div class='del'>-		memvirt = (void *)uctxt-&gt;rcvhdrtail_kvaddr;</div><div class='add'>+		memvirt = (void *)hfi1_rcvhdrtail_kvaddr(uctxt);</div><div class='add'>+		memdma = uctxt-&gt;rcvhdrqtailaddr_dma;</div><div class='ctx'> 		flags &amp;= ~VM_MAYWRITE;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case SUBCTXT_UREGS:</div><div class='hunk'>@@ -558,15 +558,16 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)</div><div class='ctx'> 		goto done;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	vma-&gt;vm_flags = flags;</div><div class='del'>-	hfi1_cdbg(PROC,</div><div class='del'>-		  "%u:%u type:%u io/vf:%d/%d, addr:0x%llx, len:%lu(%lu), flags:0x%lx\n",</div><div class='del'>-		    ctxt, subctxt, type, mapio, vmf, memaddr, memlen,</div><div class='del'>-		    vma-&gt;vm_end - vma-&gt;vm_start, vma-&gt;vm_flags);</div><div class='add'>+	vm_flags_reset(vma, flags);</div><div class='add'>+	mmap_cdbg(ctxt, subctxt, type, mapio, vmf, memaddr, memvirt, memdma, </div><div class='add'>+		  memlen, vma);</div><div class='ctx'> 	if (vmf) {</div><div class='ctx'> 		vma-&gt;vm_pgoff = PFN_DOWN(memaddr);</div><div class='ctx'> 		vma-&gt;vm_ops = &amp;vm_ops;</div><div class='ctx'> 		ret = 0;</div><div class='add'>+	} else if (memdma) {</div><div class='add'>+		ret = dma_mmap_coherent(&amp;dd-&gt;pcidev-&gt;dev, vma,</div><div class='add'>+					memvirt, memdma, memlen);</div><div class='ctx'> 	} else if (mapio) {</div><div class='ctx'> 		ret = io_remap_pfn_range(vma, vma-&gt;vm_start,</div><div class='ctx'> 					 PFN_DOWN(memaddr),</div><div class='hunk'>@@ -701,12 +702,11 @@ static int hfi1_file_close(struct inode *inode, struct file *fp)</div><div class='ctx'> </div><div class='ctx'> 	deallocate_ctxt(uctxt);</div><div class='ctx'> done:</div><div class='del'>-	mmdrop(fdata-&gt;mm);</div><div class='del'>-	kobject_put(&amp;dd-&gt;kobj);</div><div class='ctx'> </div><div class='del'>-	if (atomic_dec_and_test(&amp;dd-&gt;user_refcount))</div><div class='add'>+	if (refcount_dec_and_test(&amp;dd-&gt;user_refcount))</div><div class='ctx'> 		complete(&amp;dd-&gt;user_comp);</div><div class='ctx'> </div><div class='add'>+	cleanup_srcu_struct(&amp;fdata-&gt;pq_srcu);</div><div class='ctx'> 	kfree(fdata);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -728,7 +728,7 @@ static u64 kvirt_to_phys(void *addr)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * complete_subctxt</div><div class='add'>+ * complete_subctxt - complete sub-context info</div><div class='ctx'>  * @fd: valid filedata pointer</div><div class='ctx'>  *</div><div class='ctx'>  * Sub-context info can only be set up after the base context</div><div class='hunk'>@@ -833,7 +833,7 @@ static int assign_ctxt(struct hfi1_filedata *fd, unsigned long arg, u32 len)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * match_ctxt</div><div class='add'>+ * match_ctxt - match context</div><div class='ctx'>  * @fd: valid filedata pointer</div><div class='ctx'>  * @uinfo: user info to compare base context with</div><div class='ctx'>  * @uctxt: context to compare uinfo to.</div><div class='hunk'>@@ -890,7 +890,7 @@ static int match_ctxt(struct hfi1_filedata *fd,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * find_sub_ctxt</div><div class='add'>+ * find_sub_ctxt - fund sub-context</div><div class='ctx'>  * @fd: valid filedata pointer</div><div class='ctx'>  * @uinfo: matching info to use to find a possible context to share.</div><div class='ctx'>  *</div><div class='hunk'>@@ -975,7 +975,7 @@ static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd,</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='ctx'> 		goto ctxdata_free;</div><div class='ctx'> 	}</div><div class='del'>-	hfi1_cdbg(PROC, "allocated send context %u(%u)\n", uctxt-&gt;sc-&gt;sw_index,</div><div class='add'>+	hfi1_cdbg(PROC, "allocated send context %u(%u)", uctxt-&gt;sc-&gt;sw_index,</div><div class='ctx'> 		  uctxt-&gt;sc-&gt;hw_context);</div><div class='ctx'> 	ret = sc_enable(uctxt-&gt;sc);</div><div class='ctx'> 	if (ret)</div><div class='hunk'>@@ -996,7 +996,7 @@ static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd,</div><div class='ctx'> 	uctxt-&gt;userversion = uinfo-&gt;userversion;</div><div class='ctx'> 	uctxt-&gt;flags = hfi1_cap_mask; /* save current flag state */</div><div class='ctx'> 	init_waitqueue_head(&amp;uctxt-&gt;wait);</div><div class='del'>-	strlcpy(uctxt-&gt;comm, current-&gt;comm, sizeof(uctxt-&gt;comm));</div><div class='add'>+	strscpy(uctxt-&gt;comm, current-&gt;comm, sizeof(uctxt-&gt;comm));</div><div class='ctx'> 	memcpy(uctxt-&gt;uuid, uinfo-&gt;uuid, sizeof(uctxt-&gt;uuid));</div><div class='ctx'> 	uctxt-&gt;jkey = generate_jkey(current_uid());</div><div class='ctx'> 	hfi1_stats.sps_ctxts++;</div><div class='hunk'>@@ -1090,7 +1090,7 @@ static void user_init(struct hfi1_ctxtdata *uctxt)</div><div class='ctx'> 	 * don't have to wait to be sure the DMA update has happened</div><div class='ctx'> 	 * (chip resets head/tail to 0 on transition to enable).</div><div class='ctx'> 	 */</div><div class='del'>-	if (uctxt-&gt;rcvhdrtail_kvaddr)</div><div class='add'>+	if (hfi1_rcvhdrtail_kvaddr(uctxt))</div><div class='ctx'> 		clear_rcvhdrtail(uctxt);</div><div class='ctx'> </div><div class='ctx'> 	/* Setup J_KEY before enabling the context */</div><div class='hunk'>@@ -1138,7 +1138,7 @@ static int get_ctxt_info(struct hfi1_filedata *fd, unsigned long arg, u32 len)</div><div class='ctx'> 			HFI1_CAP_UGET_MASK(uctxt-&gt;flags, MASK) |</div><div class='ctx'> 			HFI1_CAP_KGET_MASK(uctxt-&gt;flags, K2U);</div><div class='ctx'> 	/* adjust flag if this fd is not able to cache */</div><div class='del'>-	if (!fd-&gt;handler)</div><div class='add'>+	if (!fd-&gt;use_mn)</div><div class='ctx'> 		cinfo.runtime_flags |= HFI1_CAP_TID_UNMAP; /* no caching */</div><div class='ctx'> </div><div class='ctx'> 	cinfo.num_active = hfi1_count_active_units();</div><div class='hunk'>@@ -1154,8 +1154,8 @@ static int get_ctxt_info(struct hfi1_filedata *fd, unsigned long arg, u32 len)</div><div class='ctx'> 	cinfo.send_ctxt = uctxt-&gt;sc-&gt;hw_context;</div><div class='ctx'> </div><div class='ctx'> 	cinfo.egrtids = uctxt-&gt;egrbufs.alloced;</div><div class='del'>-	cinfo.rcvhdrq_cnt = uctxt-&gt;rcvhdrq_cnt;</div><div class='del'>-	cinfo.rcvhdrq_entsize = uctxt-&gt;rcvhdrqentsize &lt;&lt; 2;</div><div class='add'>+	cinfo.rcvhdrq_cnt = get_hdrq_cnt(uctxt);</div><div class='add'>+	cinfo.rcvhdrq_entsize = get_hdrqentsize(uctxt) &lt;&lt; 2;</div><div class='ctx'> 	cinfo.sdma_ring_size = fd-&gt;cq-&gt;nentries;</div><div class='ctx'> 	cinfo.rcvegr_size = uctxt-&gt;egrbufs.rcvtid_size;</div><div class='ctx'> </div><div class='hunk'>@@ -1210,8 +1210,10 @@ static int setup_base_ctxt(struct hfi1_filedata *fd,</div><div class='ctx'> 		goto done;</div><div class='ctx'> </div><div class='ctx'> 	ret = init_user_ctxt(fd, uctxt);</div><div class='del'>-	if (ret)</div><div class='add'>+	if (ret) {</div><div class='add'>+		hfi1_free_ctxt_rcv_groups(uctxt);</div><div class='ctx'> 		goto done;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	user_init(uctxt);</div><div class='ctx'> </div><div class='hunk'>@@ -1253,8 +1255,8 @@ static int get_base_info(struct hfi1_filedata *fd, unsigned long arg, u32 len)</div><div class='ctx'> </div><div class='ctx'> 	memset(&amp;binfo, 0, sizeof(binfo));</div><div class='ctx'> 	binfo.hw_version = dd-&gt;revision;</div><div class='del'>-	binfo.sw_version = HFI1_KERN_SWVERSION;</div><div class='del'>-	binfo.bthqp = kdeth_qp;</div><div class='add'>+	binfo.sw_version = HFI1_USER_SWVERSION;</div><div class='add'>+	binfo.bthqp = RVT_KDETH_QP_PREFIX;</div><div class='ctx'> 	binfo.jkey = uctxt-&gt;jkey;</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * If more than 64 contexts are enabled the allocated credit</div><div class='hunk'>@@ -1347,12 +1349,15 @@ static int user_exp_rcv_setup(struct hfi1_filedata *fd, unsigned long arg,</div><div class='ctx'> 		addr = arg + offsetof(struct hfi1_tid_info, tidcnt);</div><div class='ctx'> 		if (copy_to_user((void __user *)addr, &amp;tinfo.tidcnt,</div><div class='ctx'> 				 sizeof(tinfo.tidcnt)))</div><div class='del'>-			return -EFAULT;</div><div class='add'>+			ret = -EFAULT;</div><div class='ctx'> </div><div class='ctx'> 		addr = arg + offsetof(struct hfi1_tid_info, length);</div><div class='del'>-		if (copy_to_user((void __user *)addr, &amp;tinfo.length,</div><div class='add'>+		if (!ret &amp;&amp; copy_to_user((void __user *)addr, &amp;tinfo.length,</div><div class='ctx'> 				 sizeof(tinfo.length)))</div><div class='ctx'> 			ret = -EFAULT;</div><div class='add'>+</div><div class='add'>+		if (ret)</div><div class='add'>+			hfi1_user_exp_rcv_invalid(fd, &amp;tinfo);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -1514,7 +1519,7 @@ int hfi1_set_uevent_bits(struct hfi1_pportdata *ppd, const int evtbit)</div><div class='ctx'>  * manage_rcvq - manage a context's receive queue</div><div class='ctx'>  * @uctxt: the context</div><div class='ctx'>  * @subctxt: the sub-context</div><div class='del'>- * @start_stop: action to carry out</div><div class='add'>+ * @arg: start/stop action to carry out</div><div class='ctx'>  *</div><div class='ctx'>  * start_stop == 0 disables receive on the context, for use in queue</div><div class='ctx'>  * overflow conditions.  start_stop==1 re-enables, to be used to</div><div class='hunk'>@@ -1543,7 +1548,7 @@ static int manage_rcvq(struct hfi1_ctxtdata *uctxt, u16 subctxt,</div><div class='ctx'> 		 * always resets it's tail register back to 0 on a</div><div class='ctx'> 		 * transition from disabled to enabled.</div><div class='ctx'> 		 */</div><div class='del'>-		if (uctxt-&gt;rcvhdrtail_kvaddr)</div><div class='add'>+		if (hfi1_rcvhdrtail_kvaddr(uctxt))</div><div class='ctx'> 			clear_rcvhdrtail(uctxt);</div><div class='ctx'> 		rcvctrl_op = HFI1_RCVCTRL_CTXT_ENB;</div><div class='ctx'> 	} else {</div><div class='hunk'>@@ -1684,7 +1689,7 @@ static int user_add(struct hfi1_devdata *dd)</div><div class='ctx'> 	snprintf(name, sizeof(name), "%s_%d", class_name(), dd-&gt;unit);</div><div class='ctx'> 	ret = hfi1_cdev_init(dd-&gt;unit, name, &amp;hfi1_file_ops,</div><div class='ctx'> 			     &amp;dd-&gt;user_cdev, &amp;dd-&gt;user_device,</div><div class='del'>-			     true, &amp;dd-&gt;kobj);</div><div class='add'>+			     true, &amp;dd-&gt;verbs_dev.rdi.ibdev.dev.kobj);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		user_remove(dd);</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/firmware.c b/drivers/infiniband/hw/hfi1/firmware.c<br/>index 2b57ba70ddd6..3c228aeaaf81 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/firmware.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/firmware.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/firmware.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/firmware.c</a></div><div class='hunk'>@@ -1,53 +1,10 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015 - 2017 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/firmware.h&gt;</div><div class='ctx'> #include &lt;linux/mutex.h&gt;</div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='ctx'> #include &lt;linux/delay.h&gt;</div><div class='ctx'> #include &lt;linux/crc32.h&gt;</div><div class='ctx'> </div><div class='hunk'>@@ -1157,7 +1114,7 @@ static void turn_off_spicos(struct hfi1_devdata *dd, int flags)</div><div class='ctx'>  * Reset all of the fabric serdes for this HFI in preparation to take the</div><div class='ctx'>  * link to Polling.</div><div class='ctx'>  *</div><div class='del'>- * To do a reset, we need to write to to the serdes registers.  Unfortunately,</div><div class='add'>+ * To do a reset, we need to write to the serdes registers.  Unfortunately,</div><div class='ctx'>  * the fabric serdes download to the other HFI on the ASIC will have turned</div><div class='ctx'>  * off the firmware validation on this HFI.  This means we can't write to the</div><div class='ctx'>  * registers to reset the serdes.  Work around this by performing a complete</div><div class='hunk'>@@ -1786,6 +1743,7 @@ int parse_platform_config(struct hfi1_devdata *dd)</div><div class='ctx'> </div><div class='ctx'> 	if (!dd-&gt;platform_config.data) {</div><div class='ctx'> 		dd_dev_err(dd, "%s: Missing config file\n", __func__);</div><div class='add'>+		ret = -EINVAL;</div><div class='ctx'> 		goto bail;</div><div class='ctx'> 	}</div><div class='ctx'> 	ptr = (u32 *)dd-&gt;platform_config.data;</div><div class='hunk'>@@ -1794,6 +1752,7 @@ int parse_platform_config(struct hfi1_devdata *dd)</div><div class='ctx'> 	ptr++;</div><div class='ctx'> 	if (magic_num != PLATFORM_CONFIG_MAGIC_NUM) {</div><div class='ctx'> 		dd_dev_err(dd, "%s: Bad config file\n", __func__);</div><div class='add'>+		ret = -EINVAL;</div><div class='ctx'> 		goto bail;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -1817,6 +1776,7 @@ int parse_platform_config(struct hfi1_devdata *dd)</div><div class='ctx'> 	if (file_length &gt; dd-&gt;platform_config.size) {</div><div class='ctx'> 		dd_dev_info(dd, "%s:File claims to be larger than read size\n",</div><div class='ctx'> 			    __func__);</div><div class='add'>+		ret = -EINVAL;</div><div class='ctx'> 		goto bail;</div><div class='ctx'> 	} else if (file_length &lt; dd-&gt;platform_config.size) {</div><div class='ctx'> 		dd_dev_info(dd,</div><div class='hunk'>@@ -1837,6 +1797,7 @@ int parse_platform_config(struct hfi1_devdata *dd)</div><div class='ctx'> 			dd_dev_err(dd, "%s: Failed validation at offset %ld\n",</div><div class='ctx'> 				   __func__, (ptr - (u32 *)</div><div class='ctx'> 					      dd-&gt;platform_config.data));</div><div class='add'>+			ret = -EINVAL;</div><div class='ctx'> 			goto bail;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='hunk'>@@ -1868,11 +1829,8 @@ int parse_platform_config(struct hfi1_devdata *dd)</div><div class='ctx'> 									2;</div><div class='ctx'> 				break;</div><div class='ctx'> 			case PLATFORM_CONFIG_RX_PRESET_TABLE:</div><div class='del'>-				/* fall through */</div><div class='ctx'> 			case PLATFORM_CONFIG_TX_PRESET_TABLE:</div><div class='del'>-				/* fall through */</div><div class='ctx'> 			case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:</div><div class='del'>-				/* fall through */</div><div class='ctx'> 			case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:</div><div class='ctx'> 				pcfgcache-&gt;config_tables[table_type].num_table =</div><div class='ctx'> 							table_length_dwords;</div><div class='hunk'>@@ -1883,6 +1841,7 @@ int parse_platform_config(struct hfi1_devdata *dd)</div><div class='ctx'> 					   __func__, table_type,</div><div class='ctx'> 					   (ptr - (u32 *)</div><div class='ctx'> 					    dd-&gt;platform_config.data));</div><div class='add'>+				ret = -EINVAL;</div><div class='ctx'> 				goto bail; /* We don't trust this file now */</div><div class='ctx'> 			}</div><div class='ctx'> 			pcfgcache-&gt;config_tables[table_type].table = ptr;</div><div class='hunk'>@@ -1890,15 +1849,10 @@ int parse_platform_config(struct hfi1_devdata *dd)</div><div class='ctx'> 			/* metadata table */</div><div class='ctx'> 			switch (table_type) {</div><div class='ctx'> 			case PLATFORM_CONFIG_SYSTEM_TABLE:</div><div class='del'>-				/* fall through */</div><div class='ctx'> 			case PLATFORM_CONFIG_PORT_TABLE:</div><div class='del'>-				/* fall through */</div><div class='ctx'> 			case PLATFORM_CONFIG_RX_PRESET_TABLE:</div><div class='del'>-				/* fall through */</div><div class='ctx'> 			case PLATFORM_CONFIG_TX_PRESET_TABLE:</div><div class='del'>-				/* fall through */</div><div class='ctx'> 			case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:</div><div class='del'>-				/* fall through */</div><div class='ctx'> 			case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:</div><div class='ctx'> 				break;</div><div class='ctx'> 			default:</div><div class='hunk'>@@ -1907,6 +1861,7 @@ int parse_platform_config(struct hfi1_devdata *dd)</div><div class='ctx'> 					   __func__, table_type,</div><div class='ctx'> 					   (ptr -</div><div class='ctx'> 					    (u32 *)dd-&gt;platform_config.data));</div><div class='add'>+				ret = -EINVAL;</div><div class='ctx'> 				goto bail; /* We don't trust this file now */</div><div class='ctx'> 			}</div><div class='ctx'> 			pcfgcache-&gt;config_tables[table_type].table_metadata =</div><div class='hunk'>@@ -1924,6 +1879,7 @@ int parse_platform_config(struct hfi1_devdata *dd)</div><div class='ctx'> 			dd_dev_err(dd, "%s: Failed CRC check at offset %ld\n",</div><div class='ctx'> 				   __func__, (ptr -</div><div class='ctx'> 				   (u32 *)dd-&gt;platform_config.data));</div><div class='add'>+			ret = -EINVAL;</div><div class='ctx'> 			goto bail;</div><div class='ctx'> 		}</div><div class='ctx'> 		/* Jump the CRC DWORD */</div><div class='hunk'>@@ -2027,15 +1983,10 @@ static int get_platform_fw_field_metadata(struct hfi1_devdata *dd, int table,</div><div class='ctx'> </div><div class='ctx'> 	switch (table) {</div><div class='ctx'> 	case PLATFORM_CONFIG_SYSTEM_TABLE:</div><div class='del'>-		/* fall through */</div><div class='ctx'> 	case PLATFORM_CONFIG_PORT_TABLE:</div><div class='del'>-		/* fall through */</div><div class='ctx'> 	case PLATFORM_CONFIG_RX_PRESET_TABLE:</div><div class='del'>-		/* fall through */</div><div class='ctx'> 	case PLATFORM_CONFIG_TX_PRESET_TABLE:</div><div class='del'>-		/* fall through */</div><div class='ctx'> 	case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:</div><div class='del'>-		/* fall through */</div><div class='ctx'> 	case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:</div><div class='ctx'> 		if (field &amp;&amp; field &lt; platform_config_table_limits[table])</div><div class='ctx'> 			src_ptr =</div><div class='hunk'>@@ -2138,11 +2089,8 @@ int get_platform_config_field(struct hfi1_devdata *dd,</div><div class='ctx'> 			pcfgcache-&gt;config_tables[table_type].table;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case PLATFORM_CONFIG_RX_PRESET_TABLE:</div><div class='del'>-		/* fall through */</div><div class='ctx'> 	case PLATFORM_CONFIG_TX_PRESET_TABLE:</div><div class='del'>-		/* fall through */</div><div class='ctx'> 	case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:</div><div class='del'>-		/* fall through */</div><div class='ctx'> 	case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:</div><div class='ctx'> 		src_ptr = pcfgcache-&gt;config_tables[table_type].table;</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h<br/>index b458c218842b..cb630551cf1a 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/hfi.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/hfi.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/hfi.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/hfi.h</a></div><div class='hunk'>@@ -1,52 +1,13 @@</div><div class='del'>-#ifndef _HFI1_KERNEL_H</div><div class='del'>-#define _HFI1_KERNEL_H</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2015-2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2020-2023 Cornelis Networks, Inc.</div><div class='add'>+ * Copyright(c) 2015-2020 Intel Corporation.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='add'>+#ifndef _HFI1_KERNEL_H</div><div class='add'>+#define _HFI1_KERNEL_H</div><div class='add'>+</div><div class='add'>+#include &lt;linux/refcount.h&gt;</div><div class='ctx'> #include &lt;linux/interrupt.h&gt;</div><div class='ctx'> #include &lt;linux/pci.h&gt;</div><div class='ctx'> #include &lt;linux/dma-mapping.h&gt;</div><div class='hunk'>@@ -68,7 +29,6 @@</div><div class='ctx'> #include &lt;rdma/ib_hdrs.h&gt;</div><div class='ctx'> #include &lt;rdma/opa_addr.h&gt;</div><div class='ctx'> #include &lt;linux/rhashtable.h&gt;</div><div class='del'>-#include &lt;linux/netdevice.h&gt;</div><div class='ctx'> #include &lt;rdma/rdma_vt.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "chip_registers.h"</div><div class='hunk'>@@ -197,7 +157,9 @@ struct exp_tid_set {</div><div class='ctx'> 	u32 count;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-typedef int (*rhf_rcv_function_ptr)(struct hfi1_packet *packet);</div><div class='add'>+struct hfi1_ctxtdata;</div><div class='add'>+typedef int (*intr_handler)(struct hfi1_ctxtdata *rcd, int data);</div><div class='add'>+typedef void (*rhf_rcv_function_ptr)(struct hfi1_packet *packet);</div><div class='ctx'> </div><div class='ctx'> struct tid_queue {</div><div class='ctx'> 	struct list_head queue_head;</div><div class='hunk'>@@ -226,7 +188,13 @@ struct hfi1_ctxtdata {</div><div class='ctx'> 	 * be valid. Worst case is we process an extra interrupt and up to 64</div><div class='ctx'> 	 * packets with the wrong interrupt handler.</div><div class='ctx'> 	 */</div><div class='del'>-	int (*do_interrupt)(struct hfi1_ctxtdata *rcd, int threaded);</div><div class='add'>+	intr_handler do_interrupt;</div><div class='add'>+	/** fast handler after autoactive */</div><div class='add'>+	intr_handler fast_handler;</div><div class='add'>+	/** slow handler */</div><div class='add'>+	intr_handler slow_handler;</div><div class='add'>+	/* napi pointer assiociated with netdev */</div><div class='add'>+	struct napi_struct *napi;</div><div class='ctx'> 	/* verbs rx_stats per rcd */</div><div class='ctx'> 	struct hfi1_opcode_stats_perctx *opstats;</div><div class='ctx'> 	/* clear interrupt mask */</div><div class='hunk'>@@ -377,11 +345,11 @@ struct hfi1_packet {</div><div class='ctx'> 	u32 rhqoff;</div><div class='ctx'> 	u32 dlid;</div><div class='ctx'> 	u32 slid;</div><div class='add'>+	int numpkt;</div><div class='ctx'> 	u16 tlen;</div><div class='ctx'> 	s16 etail;</div><div class='ctx'> 	u16 pkey;</div><div class='ctx'> 	u8 hlen;</div><div class='del'>-	u8 numpkt;</div><div class='ctx'> 	u8 rsize;</div><div class='ctx'> 	u8 updegr;</div><div class='ctx'> 	u8 etype;</div><div class='hunk'>@@ -539,6 +507,37 @@ static inline void hfi1_16B_set_qpn(struct opa_16b_mgmt *mgmt,</div><div class='ctx'> 	mgmt-&gt;src_qpn = cpu_to_be32(src_qp &amp; OPA_16B_MGMT_QPN_MASK);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+/**</div><div class='add'>+ * hfi1_get_rc_ohdr - get extended header</div><div class='add'>+ * @opah - the opaheader</div><div class='add'>+ */</div><div class='add'>+static inline struct ib_other_headers *</div><div class='add'>+hfi1_get_rc_ohdr(struct hfi1_opa_header *opah)</div><div class='add'>+{</div><div class='add'>+	struct ib_other_headers *ohdr;</div><div class='add'>+	struct ib_header *hdr = NULL;</div><div class='add'>+	struct hfi1_16b_header *hdr_16b = NULL;</div><div class='add'>+</div><div class='add'>+	/* Find out where the BTH is */</div><div class='add'>+	if (opah-&gt;hdr_type == HFI1_PKT_TYPE_9B) {</div><div class='add'>+		hdr = &amp;opah-&gt;ibh;</div><div class='add'>+		if (ib_get_lnh(hdr) == HFI1_LRH_BTH)</div><div class='add'>+			ohdr = &amp;hdr-&gt;u.oth;</div><div class='add'>+		else</div><div class='add'>+			ohdr = &amp;hdr-&gt;u.l.oth;</div><div class='add'>+	} else {</div><div class='add'>+		u8 l4;</div><div class='add'>+</div><div class='add'>+		hdr_16b = &amp;opah-&gt;opah;</div><div class='add'>+		l4  = hfi1_16B_get_l4(hdr_16b);</div><div class='add'>+		if (l4 == OPA_16B_L4_IB_LOCAL)</div><div class='add'>+			ohdr = &amp;hdr_16b-&gt;u.oth;</div><div class='add'>+		else</div><div class='add'>+			ohdr = &amp;hdr_16b-&gt;u.l.oth;</div><div class='add'>+	}</div><div class='add'>+	return ohdr;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> struct rvt_sge_state;</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -677,12 +676,6 @@ static inline void incr_cntr64(u64 *cntr)</div><div class='ctx'> 		(*cntr)++;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline void incr_cntr32(u32 *cntr)</div><div class='del'>-{</div><div class='del'>-	if (*cntr &lt; (u32)-1LL)</div><div class='del'>-		(*cntr)++;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> #define MAX_NAME_SIZE 64</div><div class='ctx'> struct hfi1_msix_entry {</div><div class='ctx'> 	enum irq_type type;</div><div class='hunk'>@@ -739,10 +732,6 @@ struct hfi1_pportdata {</div><div class='ctx'> 	struct hfi1_ibport ibport_data;</div><div class='ctx'> </div><div class='ctx'> 	struct hfi1_devdata *dd;</div><div class='del'>-	struct kobject pport_cc_kobj;</div><div class='del'>-	struct kobject sc2vl_kobj;</div><div class='del'>-	struct kobject sl2sc_kobj;</div><div class='del'>-	struct kobject vl2mtu_kobj;</div><div class='ctx'> </div><div class='ctx'> 	/* PHY support */</div><div class='ctx'> 	struct qsfp_data qsfp_info;</div><div class='hunk'>@@ -824,7 +813,7 @@ struct hfi1_pportdata {</div><div class='ctx'> 	u8 rx_pol_inv;</div><div class='ctx'> </div><div class='ctx'> 	u8 hw_pidx;     /* physical port index */</div><div class='del'>-	u8 port;        /* IB port number and index into dd-&gt;pports - 1 */</div><div class='add'>+	u32 port;        /* IB port number and index into dd-&gt;pports - 1 */</div><div class='ctx'> 	/* type of neighbor node */</div><div class='ctx'> 	u8 neighbor_type;</div><div class='ctx'> 	u8 neighbor_normal;</div><div class='hunk'>@@ -948,7 +937,7 @@ typedef void (*hfi1_make_req)(struct rvt_qp *qp,</div><div class='ctx'> 			      struct hfi1_pkt_state *ps,</div><div class='ctx'> 			      struct rvt_swqe *wqe);</div><div class='ctx'> extern const rhf_rcv_function_ptr normal_rhf_rcv_functions[];</div><div class='del'>-</div><div class='add'>+extern const rhf_rcv_function_ptr netdev_rhf_rcv_functions[];</div><div class='ctx'> </div><div class='ctx'> /* return values for the RHF receive functions */</div><div class='ctx'> #define RHF_RCV_CONTINUE  0	/* keep going */</div><div class='hunk'>@@ -1008,23 +997,10 @@ struct hfi1_asic_data {</div><div class='ctx'> #define NUM_MAP_ENTRIES	 256</div><div class='ctx'> #define NUM_MAP_REGS      32</div><div class='ctx'> </div><div class='del'>-/*</div><div class='del'>- * Number of VNIC contexts used. Ensure it is less than or equal to</div><div class='del'>- * max queues supported by VNIC (HFI1_VNIC_MAX_QUEUE).</div><div class='del'>- */</div><div class='del'>-#define HFI1_NUM_VNIC_CTXT   8</div><div class='del'>-</div><div class='del'>-/* Number of VNIC RSM entries */</div><div class='del'>-#define NUM_VNIC_MAP_ENTRIES 8</div><div class='del'>-</div><div class='ctx'> /* Virtual NIC information */</div><div class='ctx'> struct hfi1_vnic_data {</div><div class='del'>-	struct hfi1_ctxtdata *ctxt[HFI1_NUM_VNIC_CTXT];</div><div class='ctx'> 	struct kmem_cache *txreq_cache;</div><div class='del'>-	struct xarray vesws;</div><div class='ctx'> 	u8 num_vports;</div><div class='del'>-	u8 rmt_start;</div><div class='del'>-	u8 num_ctxt;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hfi1_vnic_vport_info;</div><div class='hunk'>@@ -1039,6 +1015,7 @@ struct sdma_vl_map;</div><div class='ctx'> #define SERIAL_MAX 16 /* length of the serial number */</div><div class='ctx'> </div><div class='ctx'> typedef int (*send_routine)(struct rvt_qp *, struct hfi1_pkt_state *, u64);</div><div class='add'>+struct hfi1_netdev_rx;</div><div class='ctx'> struct hfi1_devdata {</div><div class='ctx'> 	struct hfi1_ibdev verbs_dev;     /* must be first */</div><div class='ctx'> 	/* pointers to related structs for this device */</div><div class='hunk'>@@ -1122,14 +1099,16 @@ struct hfi1_devdata {</div><div class='ctx'> </div><div class='ctx'> 	char *boardname; /* human readable board info */</div><div class='ctx'> </div><div class='add'>+	u64 ctx0_seq_drop;</div><div class='add'>+</div><div class='ctx'> 	/* reset value */</div><div class='ctx'> 	u64 z_int_counter;</div><div class='ctx'> 	u64 z_rcv_limit;</div><div class='ctx'> 	u64 z_send_schedule;</div><div class='ctx'> </div><div class='ctx'> 	u64 __percpu *send_schedule;</div><div class='del'>-	/* number of reserved contexts for VNIC usage */</div><div class='del'>-	u16 num_vnic_contexts;</div><div class='add'>+	/* number of reserved contexts for netdev usage */</div><div class='add'>+	u16 num_netdev_contexts;</div><div class='ctx'> 	/* number of receive contexts in use by the driver */</div><div class='ctx'> 	u32 num_rcv_contexts;</div><div class='ctx'> 	/* number of pio send contexts in use by the driver */</div><div class='hunk'>@@ -1279,7 +1258,7 @@ struct hfi1_devdata {</div><div class='ctx'> 	struct err_info_constraint err_info_xmit_constraint;</div><div class='ctx'> </div><div class='ctx'> 	atomic_t drop_packet;</div><div class='del'>-	u8 do_drop;</div><div class='add'>+	bool do_drop;</div><div class='ctx'> 	u8 err_info_uncorrectable;</div><div class='ctx'> 	u8 err_info_fmconfig;</div><div class='ctx'> </div><div class='hunk'>@@ -1365,7 +1344,7 @@ struct hfi1_devdata {</div><div class='ctx'> 	/* Number of verbs contexts which have disabled ASPM */</div><div class='ctx'> 	atomic_t aspm_disabled_cnt;</div><div class='ctx'> 	/* Keeps track of user space clients */</div><div class='del'>-	atomic_t user_refcount;</div><div class='add'>+	refcount_t user_refcount;</div><div class='ctx'> 	/* Used to wait for outstanding user space clients before dev removal */</div><div class='ctx'> 	struct completion user_comp;</div><div class='ctx'> </div><div class='hunk'>@@ -1374,18 +1353,17 @@ struct hfi1_devdata {</div><div class='ctx'> 	bool aspm_enabled;	/* ASPM state: enabled/disabled */</div><div class='ctx'> 	struct rhashtable *sdma_rht;</div><div class='ctx'> </div><div class='del'>-	struct kobject kobj;</div><div class='del'>-</div><div class='ctx'> 	/* vnic data */</div><div class='ctx'> 	struct hfi1_vnic_data vnic;</div><div class='ctx'> 	/* Lock to protect IRQ SRC register access */</div><div class='ctx'> 	spinlock_t irq_src_lock;</div><div class='del'>-};</div><div class='add'>+	int vnic_num_vports;</div><div class='add'>+	struct hfi1_netdev_rx *netdev_rx;</div><div class='add'>+	struct hfi1_affinity_node *affinity_entry;</div><div class='ctx'> </div><div class='del'>-static inline bool hfi1_vnic_is_rsm_full(struct hfi1_devdata *dd, int spare)</div><div class='del'>-{</div><div class='del'>-	return (dd-&gt;vnic.rmt_start + spare) &gt; NUM_MAP_ENTRIES;</div><div class='del'>-}</div><div class='add'>+	/* Keeps track of IPoIB RSM rule users */</div><div class='add'>+	atomic_t ipoib_rsm_usr_num;</div><div class='add'>+};</div><div class='ctx'> </div><div class='ctx'> /* 8051 firmware version helper */</div><div class='ctx'> #define dc8051_ver(a, b, c) ((a) &lt;&lt; 16 | (b) &lt;&lt; 8 | (c))</div><div class='hunk'>@@ -1400,20 +1378,21 @@ static inline bool hfi1_vnic_is_rsm_full(struct hfi1_devdata *dd, int spare)</div><div class='ctx'> #define PT_INVALID        3</div><div class='ctx'> </div><div class='ctx'> struct tid_rb_node;</div><div class='del'>-struct mmu_rb_node;</div><div class='del'>-struct mmu_rb_handler;</div><div class='ctx'> </div><div class='ctx'> /* Private data for file operations */</div><div class='ctx'> struct hfi1_filedata {</div><div class='add'>+	struct srcu_struct pq_srcu;</div><div class='ctx'> 	struct hfi1_devdata *dd;</div><div class='ctx'> 	struct hfi1_ctxtdata *uctxt;</div><div class='ctx'> 	struct hfi1_user_sdma_comp_q *cq;</div><div class='del'>-	struct hfi1_user_sdma_pkt_q *pq;</div><div class='add'>+	/* update side lock for SRCU */</div><div class='add'>+	spinlock_t pq_rcu_lock;</div><div class='add'>+	struct hfi1_user_sdma_pkt_q __rcu *pq;</div><div class='ctx'> 	u16 subctxt;</div><div class='ctx'> 	/* for cpu affinity; -1 if none */</div><div class='ctx'> 	int rec_cpu_num;</div><div class='ctx'> 	u32 tid_n_pinned;</div><div class='del'>-	struct mmu_rb_handler *handler;</div><div class='add'>+	bool use_mn;</div><div class='ctx'> 	struct tid_rb_node **entry_to_rb;</div><div class='ctx'> 	spinlock_t tid_lock; /* protect tid_[limit,used] counters */</div><div class='ctx'> 	u32 tid_limit;</div><div class='hunk'>@@ -1422,7 +1401,6 @@ struct hfi1_filedata {</div><div class='ctx'> 	u32 invalid_tid_idx;</div><div class='ctx'> 	/* protect invalid_tids array and invalid_tid_idx */</div><div class='ctx'> 	spinlock_t invalid_lock;</div><div class='del'>-	struct mm_struct *mm;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> extern struct xarray hfi1_dev_table;</div><div class='hunk'>@@ -1450,7 +1428,7 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,</div><div class='ctx'> 			 struct hfi1_ctxtdata **rcd);</div><div class='ctx'> void hfi1_free_ctxt(struct hfi1_ctxtdata *rcd);</div><div class='ctx'> void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd,</div><div class='del'>-			 struct hfi1_devdata *dd, u8 hw_pidx, u8 port);</div><div class='add'>+			 struct hfi1_devdata *dd, u8 hw_pidx, u32 port);</div><div class='ctx'> void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd);</div><div class='ctx'> int hfi1_rcd_put(struct hfi1_ctxtdata *rcd);</div><div class='ctx'> int hfi1_rcd_get(struct hfi1_ctxtdata *rcd);</div><div class='hunk'>@@ -1460,6 +1438,8 @@ struct hfi1_ctxtdata *hfi1_rcd_get_by_index(struct hfi1_devdata *dd, u16 ctxt);</div><div class='ctx'> int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread);</div><div class='ctx'> int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread);</div><div class='ctx'> int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread);</div><div class='add'>+int handle_receive_interrupt_napi_fp(struct hfi1_ctxtdata *rcd, int budget);</div><div class='add'>+int handle_receive_interrupt_napi_sp(struct hfi1_ctxtdata *rcd, int budget);</div><div class='ctx'> void set_all_slowpath(struct hfi1_devdata *dd);</div><div class='ctx'> </div><div class='ctx'> extern const struct pci_device_id hfi1_pci_tbl[];</div><div class='hunk'>@@ -1476,12 +1456,148 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp,</div><div class='ctx'> #define RCV_PKT_LIMIT   0x1 /* stop, hit limit, start thread */</div><div class='ctx'> #define RCV_PKT_DONE    0x2 /* stop, no more packets detected */</div><div class='ctx'> </div><div class='add'>+/**</div><div class='add'>+ * hfi1_rcd_head - add accessor for rcd head</div><div class='add'>+ * @rcd: the context</div><div class='add'>+ */</div><div class='add'>+static inline u32 hfi1_rcd_head(struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	return rcd-&gt;head;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_set_rcd_head - add accessor for rcd head</div><div class='add'>+ * @rcd: the context</div><div class='add'>+ * @head: the new head</div><div class='add'>+ */</div><div class='add'>+static inline void hfi1_set_rcd_head(struct hfi1_ctxtdata *rcd, u32 head)</div><div class='add'>+{</div><div class='add'>+	rcd-&gt;head = head;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /* calculate the current RHF address */</div><div class='ctx'> static inline __le32 *get_rhf_addr(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> {</div><div class='ctx'> 	return (__le32 *)rcd-&gt;rcvhdrq + rcd-&gt;head + rcd-&gt;rhf_offset;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+/* return DMA_RTAIL configuration */</div><div class='add'>+static inline bool get_dma_rtail_setting(struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	return !!HFI1_CAP_KGET_MASK(rcd-&gt;flags, DMA_RTAIL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_seq_incr_wrap - wrapping increment for sequence</div><div class='add'>+ * @seq: the current sequence number</div><div class='add'>+ *</div><div class='add'>+ * Returns: the incremented seq</div><div class='add'>+ */</div><div class='add'>+static inline u8 hfi1_seq_incr_wrap(u8 seq)</div><div class='add'>+{</div><div class='add'>+	if (++seq &gt; RHF_MAX_SEQ)</div><div class='add'>+		seq = 1;</div><div class='add'>+	return seq;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_seq_cnt - return seq_cnt member</div><div class='add'>+ * @rcd: the receive context</div><div class='add'>+ *</div><div class='add'>+ * Return seq_cnt member</div><div class='add'>+ */</div><div class='add'>+static inline u8 hfi1_seq_cnt(struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	return rcd-&gt;seq_cnt;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_set_seq_cnt - return seq_cnt member</div><div class='add'>+ * @rcd: the receive context</div><div class='add'>+ *</div><div class='add'>+ * Return seq_cnt member</div><div class='add'>+ */</div><div class='add'>+static inline void hfi1_set_seq_cnt(struct hfi1_ctxtdata *rcd, u8 cnt)</div><div class='add'>+{</div><div class='add'>+	rcd-&gt;seq_cnt = cnt;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * last_rcv_seq - is last</div><div class='add'>+ * @rcd: the receive context</div><div class='add'>+ * @seq: sequence</div><div class='add'>+ *</div><div class='add'>+ * return true if last packet</div><div class='add'>+ */</div><div class='add'>+static inline bool last_rcv_seq(struct hfi1_ctxtdata *rcd, u32 seq)</div><div class='add'>+{</div><div class='add'>+	return seq != rcd-&gt;seq_cnt;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * rcd_seq_incr - increment context sequence number</div><div class='add'>+ * @rcd: the receive context</div><div class='add'>+ * @seq: the current sequence number</div><div class='add'>+ *</div><div class='add'>+ * Returns: true if the this was the last packet</div><div class='add'>+ */</div><div class='add'>+static inline bool hfi1_seq_incr(struct hfi1_ctxtdata *rcd, u32 seq)</div><div class='add'>+{</div><div class='add'>+	rcd-&gt;seq_cnt = hfi1_seq_incr_wrap(rcd-&gt;seq_cnt);</div><div class='add'>+	return last_rcv_seq(rcd, seq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * get_hdrqentsize - return hdrq entry size</div><div class='add'>+ * @rcd: the receive context</div><div class='add'>+ */</div><div class='add'>+static inline u8 get_hdrqentsize(struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	return rcd-&gt;rcvhdrqentsize;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * get_hdrq_cnt - return hdrq count</div><div class='add'>+ * @rcd: the receive context</div><div class='add'>+ */</div><div class='add'>+static inline u16 get_hdrq_cnt(struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	return rcd-&gt;rcvhdrq_cnt;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_is_slowpath - check if this context is slow path</div><div class='add'>+ * @rcd: the receive context</div><div class='add'>+ */</div><div class='add'>+static inline bool hfi1_is_slowpath(struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	return rcd-&gt;do_interrupt == rcd-&gt;slow_handler;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_is_fastpath - check if this context is fast path</div><div class='add'>+ * @rcd: the receive context</div><div class='add'>+ */</div><div class='add'>+static inline bool hfi1_is_fastpath(struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	if (rcd-&gt;ctxt == HFI1_CTRL_CTXT)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	return rcd-&gt;do_interrupt == rcd-&gt;fast_handler;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_set_fast - change to the fast handler</div><div class='add'>+ * @rcd: the receive context</div><div class='add'>+ */</div><div class='add'>+static inline void hfi1_set_fast(struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	if (unlikely(!rcd))</div><div class='add'>+		return;</div><div class='add'>+	if (unlikely(!hfi1_is_fastpath(rcd)))</div><div class='add'>+		rcd-&gt;do_interrupt = rcd-&gt;fast_handler;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int hfi1_reset_device(int);</div><div class='ctx'> </div><div class='ctx'> void receive_interrupt_work(struct work_struct *work);</div><div class='hunk'>@@ -1602,7 +1718,7 @@ static inline void pause_for_credit_return(struct hfi1_devdata *dd)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * sc_to_vlt() reverse lookup sc to vl</div><div class='add'>+ * sc_to_vlt() - reverse lookup sc to vl</div><div class='ctx'>  * @dd - devdata</div><div class='ctx'>  * @sc5 - 5 bit sc</div><div class='ctx'>  */</div><div class='hunk'>@@ -1808,10 +1924,10 @@ static inline struct hfi1_ibdev *dev_from_rdi(struct rvt_dev_info *rdi)</div><div class='ctx'> 	return container_of(rdi, struct hfi1_ibdev, rdi);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline struct hfi1_ibport *to_iport(struct ib_device *ibdev, u8 port)</div><div class='add'>+static inline struct hfi1_ibport *to_iport(struct ib_device *ibdev, u32 port)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='del'>-	unsigned pidx = port - 1; /* IB number port from 1, hdw from 0 */</div><div class='add'>+	u32 pidx = port - 1; /* IB number port from 1, hdw from 0 */</div><div class='ctx'> </div><div class='ctx'> 	WARN_ON(pidx &gt;= dd-&gt;num_pports);</div><div class='ctx'> 	return &amp;dd-&gt;pport[pidx].ibport_data;</div><div class='hunk'>@@ -1984,9 +2100,21 @@ int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr,</div><div class='ctx'> void hfi1_release_user_pages(struct mm_struct *mm, struct page **p,</div><div class='ctx'> 			     size_t npages, bool dirty);</div><div class='ctx'> </div><div class='add'>+/**</div><div class='add'>+ * hfi1_rcvhdrtail_kvaddr - return tail kvaddr</div><div class='add'>+ * @rcd - the receive context</div><div class='add'>+ */</div><div class='add'>+static inline __le64 *hfi1_rcvhdrtail_kvaddr(const struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	return (__le64 *)rcd-&gt;rcvhdrtail_kvaddr;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static inline void clear_rcvhdrtail(const struct hfi1_ctxtdata *rcd)</div><div class='ctx'> {</div><div class='del'>-	*((u64 *)rcd-&gt;rcvhdrtail_kvaddr) = 0ULL;</div><div class='add'>+	u64 *kv = (u64 *)hfi1_rcvhdrtail_kvaddr(rcd);</div><div class='add'>+</div><div class='add'>+	if (kv)</div><div class='add'>+		*kv = 0ULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static inline u32 get_rcvhdrtail(const struct hfi1_ctxtdata *rcd)</div><div class='hunk'>@@ -1995,7 +2123,17 @@ static inline u32 get_rcvhdrtail(const struct hfi1_ctxtdata *rcd)</div><div class='ctx'> 	 * volatile because it's a DMA target from the chip, routine is</div><div class='ctx'> 	 * inlined, and don't want register caching or reordering.</div><div class='ctx'> 	 */</div><div class='del'>-	return (u32)le64_to_cpu(*rcd-&gt;rcvhdrtail_kvaddr);</div><div class='add'>+	return (u32)le64_to_cpu(*hfi1_rcvhdrtail_kvaddr(rcd));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool hfi1_packet_present(struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	if (likely(!rcd-&gt;rcvhdrtail_kvaddr)) {</div><div class='add'>+		u32 seq = rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd)));</div><div class='add'>+</div><div class='add'>+		return !last_rcv_seq(rcd, seq);</div><div class='add'>+	}</div><div class='add'>+	return hfi1_rcd_head(rcd) != get_rcvhdrtail(rcd);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -2004,12 +2142,11 @@ static inline u32 get_rcvhdrtail(const struct hfi1_ctxtdata *rcd)</div><div class='ctx'> </div><div class='ctx'> extern const char ib_hfi1_version[];</div><div class='ctx'> extern const struct attribute_group ib_hfi1_attr_group;</div><div class='add'>+extern const struct attribute_group *hfi1_attr_port_groups[];</div><div class='ctx'> </div><div class='ctx'> int hfi1_device_create(struct hfi1_devdata *dd);</div><div class='ctx'> void hfi1_device_remove(struct hfi1_devdata *dd);</div><div class='ctx'> </div><div class='del'>-int hfi1_create_port_files(struct ib_device *ibdev, u8 port_num,</div><div class='del'>-			   struct kobject *kobj);</div><div class='ctx'> int hfi1_verbs_register_sysfs(struct hfi1_devdata *dd);</div><div class='ctx'> void hfi1_verbs_unregister_sysfs(struct hfi1_devdata *dd);</div><div class='ctx'> /* Hook for sysfs read of QSFP */</div><div class='hunk'>@@ -2052,7 +2189,6 @@ extern int num_user_contexts;</div><div class='ctx'> extern unsigned long n_krcvqs;</div><div class='ctx'> extern uint krcvqs[];</div><div class='ctx'> extern int krcvqsset;</div><div class='del'>-extern uint kdeth_qp;</div><div class='ctx'> extern uint loopback;</div><div class='ctx'> extern uint quick_linkup;</div><div class='ctx'> extern uint rcv_intr_timeout;</div><div class='hunk'>@@ -2203,20 +2339,6 @@ static inline u64 hfi1_pkt_base_sdma_integrity(struct hfi1_devdata *dd)</div><div class='ctx'> 	dev_err(&amp;(dd)-&gt;pcidev-&gt;dev, "%s: port %u: " fmt, \</div><div class='ctx'> 		rvt_get_ibdev_name(&amp;(dd)-&gt;verbs_dev.rdi), (port), ##__VA_ARGS__)</div><div class='ctx'> </div><div class='del'>-/*</div><div class='del'>- * this is used for formatting hw error messages...</div><div class='del'>- */</div><div class='del'>-struct hfi1_hwerror_msgs {</div><div class='del'>-	u64 mask;</div><div class='del'>-	const char *msg;</div><div class='del'>-	size_t sz;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* in intr.c... */</div><div class='del'>-void hfi1_format_hwerrors(u64 hwerrs,</div><div class='del'>-			  const struct hfi1_hwerror_msgs *hwerrmsgs,</div><div class='del'>-			  size_t nhwerrmsgs, char *msg, size_t lmsg);</div><div class='del'>-</div><div class='ctx'> #define USER_OPCODE_CHECK_VAL 0xC0</div><div class='ctx'> #define USER_OPCODE_CHECK_MASK 0xC0</div><div class='ctx'> #define OPCODE_CHECK_VAL_DISABLED 0x0</div><div class='hunk'>@@ -2267,10 +2389,29 @@ static inline bool is_integrated(struct hfi1_devdata *dd)</div><div class='ctx'> 	return dd-&gt;pcidev-&gt;device == PCI_DEVICE_ID_INTEL1;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+/**</div><div class='add'>+ * hfi1_need_drop - detect need for drop</div><div class='add'>+ * @dd: - the device</div><div class='add'>+ *</div><div class='add'>+ * In some cases, the first packet needs to be dropped.</div><div class='add'>+ *</div><div class='add'>+ * Return true is the current packet needs to be dropped and false otherwise.</div><div class='add'>+ */</div><div class='add'>+static inline bool hfi1_need_drop(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	if (unlikely(dd-&gt;do_drop &amp;&amp;</div><div class='add'>+		     atomic_xchg(&amp;dd-&gt;drop_packet, DROP_PACKET_OFF) ==</div><div class='add'>+		     DROP_PACKET_ON)) {</div><div class='add'>+		dd-&gt;do_drop = false;</div><div class='add'>+		return true;</div><div class='add'>+	}</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp);</div><div class='ctx'> </div><div class='ctx'> #define DD_DEV_ENTRY(dd)       __string(dev, dev_name(&amp;(dd)-&gt;pcidev-&gt;dev))</div><div class='del'>-#define DD_DEV_ASSIGN(dd)      __assign_str(dev, dev_name(&amp;(dd)-&gt;pcidev-&gt;dev))</div><div class='add'>+#define DD_DEV_ASSIGN(dd)      __assign_str(dev)</div><div class='ctx'> </div><div class='ctx'> static inline void hfi1_update_ah_attr(struct ib_device *ibdev,</div><div class='ctx'> 				       struct rdma_ah_attr *attr)</div><div class='hunk'>@@ -2404,7 +2545,7 @@ static inline bool hfi1_get_hdr_type(u32 lid, struct rdma_ah_attr *attr)</div><div class='ctx'> 			HFI1_PKT_TYPE_16B : HFI1_PKT_TYPE_9B;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='del'>-	 * Return a 16B header type if either the the destination</div><div class='add'>+	 * Return a 16B header type if either the destination</div><div class='ctx'> 	 * or source lid is extended.</div><div class='ctx'> 	 */</div><div class='ctx'> 	if (hfi1_get_packet_type(rdma_ah_get_dlid(attr)) == HFI1_PKT_TYPE_16B)</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c<br/>index 71cb9525c074..e4aef102dac0 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/init.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/init.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/init.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/init.c</a></div><div class='hunk'>@@ -1,48 +1,7 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2015 - 2020 Intel Corporation.</div><div class='add'>+ * Copyright(c) 2021 Cornelis Networks.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/pci.h&gt;</div><div class='hunk'>@@ -69,6 +28,7 @@</div><div class='ctx'> #include "affinity.h"</div><div class='ctx'> #include "vnic.h"</div><div class='ctx'> #include "exp_rcv.h"</div><div class='add'>+#include "netdev.h"</div><div class='ctx'> </div><div class='ctx'> #undef pr_fmt</div><div class='ctx'> #define pr_fmt(fmt) DRIVER_NAME ": " fmt</div><div class='hunk'>@@ -78,8 +38,6 @@</div><div class='ctx'>  */</div><div class='ctx'> #define HFI1_MIN_USER_CTXT_BUFCNT 7</div><div class='ctx'> </div><div class='del'>-#define HFI1_MIN_HDRQ_EGRBUF_CNT 2</div><div class='del'>-#define HFI1_MAX_HDRQ_EGRBUF_CNT 16352</div><div class='ctx'> #define HFI1_MIN_EAGER_BUFFER_SIZE (4 * 1024) /* 4KB */</div><div class='ctx'> #define HFI1_MAX_EAGER_BUFFER_SIZE (256 * 1024) /* 256KB */</div><div class='ctx'> </div><div class='hunk'>@@ -122,8 +80,6 @@ unsigned int user_credit_return_threshold = 33;	/* default is 33% */</div><div class='ctx'> module_param(user_credit_return_threshold, uint, S_IRUGO);</div><div class='ctx'> MODULE_PARM_DESC(user_credit_return_threshold, "Credit return threshold for user send contexts, return when unreturned credits passes this many blocks (in percent of allocated blocks, 0 is off)");</div><div class='ctx'> </div><div class='del'>-static inline u64 encode_rcv_header_entry_size(u16 size);</div><div class='del'>-</div><div class='ctx'> DEFINE_XARRAY_FLAGS(hfi1_dev_table, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);</div><div class='ctx'> </div><div class='ctx'> static int hfi1_create_kctxt(struct hfi1_devdata *dd,</div><div class='hunk'>@@ -154,7 +110,11 @@ static int hfi1_create_kctxt(struct hfi1_devdata *dd,</div><div class='ctx'> 	/* Control context must use DMA_RTAIL */</div><div class='ctx'> 	if (rcd-&gt;ctxt == HFI1_CTRL_CTXT)</div><div class='ctx'> 		rcd-&gt;flags |= HFI1_CAP_DMA_RTAIL;</div><div class='del'>-	rcd-&gt;seq_cnt = 1;</div><div class='add'>+	rcd-&gt;fast_handler = get_dma_rtail_setting(rcd) ?</div><div class='add'>+				handle_receive_interrupt_dma_rtail :</div><div class='add'>+				handle_receive_interrupt_nodma_rtail;</div><div class='add'>+</div><div class='add'>+	hfi1_set_seq_cnt(rcd, 1);</div><div class='ctx'> </div><div class='ctx'> 	rcd-&gt;sc = sc_alloc(dd, SC_ACK, rcd-&gt;rcvhdrqentsize, dd-&gt;node);</div><div class='ctx'> 	if (!rcd-&gt;sc) {</div><div class='hunk'>@@ -310,7 +270,7 @@ struct hfi1_ctxtdata *hfi1_rcd_get_by_index_safe(struct hfi1_devdata *dd,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * hfi1_rcd_get_by_index</div><div class='add'>+ * hfi1_rcd_get_by_index - get by index</div><div class='ctx'>  * @dd: pointer to a valid devdata structure</div><div class='ctx'>  * @ctxt: the index of an possilbe rcd</div><div class='ctx'>  *</div><div class='hunk'>@@ -373,13 +333,16 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,</div><div class='ctx'> 		rcd-&gt;numa_id = numa;</div><div class='ctx'> 		rcd-&gt;rcv_array_groups = dd-&gt;rcv_entries.ngroups;</div><div class='ctx'> 		rcd-&gt;rhf_rcv_function_map = normal_rhf_rcv_functions;</div><div class='add'>+		rcd-&gt;slow_handler = handle_receive_interrupt;</div><div class='add'>+		rcd-&gt;do_interrupt = rcd-&gt;slow_handler;</div><div class='add'>+		rcd-&gt;msix_intr = CCE_NUM_MSIX_VECTORS;</div><div class='ctx'> </div><div class='ctx'> 		mutex_init(&amp;rcd-&gt;exp_mutex);</div><div class='ctx'> 		spin_lock_init(&amp;rcd-&gt;exp_lock);</div><div class='ctx'> 		INIT_LIST_HEAD(&amp;rcd-&gt;flow_queue.queue_head);</div><div class='ctx'> 		INIT_LIST_HEAD(&amp;rcd-&gt;rarr_queue.queue_head);</div><div class='ctx'> </div><div class='del'>-		hfi1_cdbg(PROC, "setting up context %u\n", rcd-&gt;ctxt);</div><div class='add'>+		hfi1_cdbg(PROC, "setting up context %u", rcd-&gt;ctxt);</div><div class='ctx'> </div><div class='ctx'> 		/*</div><div class='ctx'> 		 * Calculate the context's RcvArray entry starting point.</div><div class='hunk'>@@ -437,7 +400,7 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,</div><div class='ctx'> 			rcd-&gt;egrbufs.count = MAX_EAGER_ENTRIES;</div><div class='ctx'> 		}</div><div class='ctx'> 		hfi1_cdbg(PROC,</div><div class='del'>-			  "ctxt%u: max Eager buffer RcvArray entries: %u\n",</div><div class='add'>+			  "ctxt%u: max Eager buffer RcvArray entries: %u",</div><div class='ctx'> 			  rcd-&gt;ctxt, rcd-&gt;egrbufs.count);</div><div class='ctx'> </div><div class='ctx'> 		/*</div><div class='hunk'>@@ -469,7 +432,7 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,</div><div class='ctx'> 		if (rcd-&gt;egrbufs.size &lt; hfi1_max_mtu) {</div><div class='ctx'> 			rcd-&gt;egrbufs.size = __roundup_pow_of_two(hfi1_max_mtu);</div><div class='ctx'> 			hfi1_cdbg(PROC,</div><div class='del'>-				  "ctxt%u: eager bufs size too small. Adjusting to %u\n",</div><div class='add'>+				  "ctxt%u: eager bufs size too small. Adjusting to %u",</div><div class='ctx'> 				    rcd-&gt;ctxt, rcd-&gt;egrbufs.size);</div><div class='ctx'> 		}</div><div class='ctx'> 		rcd-&gt;egrbufs.rcvtid_size = HFI1_MAX_EAGER_BUFFER_SIZE;</div><div class='hunk'>@@ -496,12 +459,12 @@ bail:</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * hfi1_free_ctxt</div><div class='add'>+ * hfi1_free_ctxt - free context</div><div class='ctx'>  * @rcd: pointer to an initialized rcd data structure</div><div class='ctx'>  *</div><div class='ctx'>  * This wrapper is the free function that matches hfi1_create_ctxtdata().</div><div class='ctx'>  * When a context is done being used (kernel or user), this function is called</div><div class='del'>- * for the "final" put to match the kref init from hf1i_create_ctxtdata().</div><div class='add'>+ * for the "final" put to match the kref init from hfi1_create_ctxtdata().</div><div class='ctx'>  * Other users of the context do a get/put sequence to make sure that the</div><div class='ctx'>  * structure isn't removed while in use.</div><div class='ctx'>  */</div><div class='hunk'>@@ -511,23 +474,6 @@ void hfi1_free_ctxt(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='del'>- * Convert a receive header entry size that to the encoding used in the CSR.</div><div class='del'>- *</div><div class='del'>- * Return a zero if the given size is invalid.</div><div class='del'>- */</div><div class='del'>-static inline u64 encode_rcv_header_entry_size(u16 size)</div><div class='del'>-{</div><div class='del'>-	/* there are only 3 valid receive header entry sizes */</div><div class='del'>-	if (size == 2)</div><div class='del'>-		return 1;</div><div class='del'>-	if (size == 16)</div><div class='del'>-		return 2;</div><div class='del'>-	else if (size == 32)</div><div class='del'>-		return 4;</div><div class='del'>-	return 0; /* invalid */</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='ctx'>  * Select the largest ccti value over all SLs to determine the intra-</div><div class='ctx'>  * packet gap for the link.</div><div class='ctx'>  *</div><div class='hunk'>@@ -543,7 +489,7 @@ void set_link_ipg(struct hfi1_pportdata *ppd)</div><div class='ctx'> 	u16 shift, mult;</div><div class='ctx'> 	u64 src;</div><div class='ctx'> 	u32 current_egress_rate; /* Mbits /sec */</div><div class='del'>-	u32 max_pkt_time;</div><div class='add'>+	u64 max_pkt_time;</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * max_pkt_time is the maximum packet egress time in units</div><div class='ctx'> 	 * of the fabric clock period 1/(805 MHz).</div><div class='hunk'>@@ -641,7 +587,7 @@ static enum hrtimer_restart cca_timer_fn(struct hrtimer *t)</div><div class='ctx'>  * Common code for initializing the physical port structure.</div><div class='ctx'>  */</div><div class='ctx'> void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd,</div><div class='del'>-			 struct hfi1_devdata *dd, u8 hw_pidx, u8 port)</div><div class='add'>+			 struct hfi1_devdata *dd, u8 hw_pidx, u32 port)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='ctx'> 	uint default_pkey_idx;</div><div class='hunk'>@@ -664,12 +610,7 @@ void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd,</div><div class='ctx'> </div><div class='ctx'> 	ppd-&gt;pkeys[default_pkey_idx] = DEFAULT_P_KEY;</div><div class='ctx'> 	ppd-&gt;part_enforce |= HFI1_PART_ENFORCE_IN;</div><div class='del'>-</div><div class='del'>-	if (loopback) {</div><div class='del'>-		dd_dev_err(dd, "Faking data partition 0x8001 in idx %u\n",</div><div class='del'>-			   !default_pkey_idx);</div><div class='del'>-		ppd-&gt;pkeys[!default_pkey_idx] = 0x8001;</div><div class='del'>-	}</div><div class='add'>+	ppd-&gt;pkeys[0] = 0x8001;</div><div class='ctx'> </div><div class='ctx'> 	INIT_WORK(&amp;ppd-&gt;link_vc_work, handle_verify_cap);</div><div class='ctx'> 	INIT_WORK(&amp;ppd-&gt;link_up_work, handle_link_up);</div><div class='hunk'>@@ -694,12 +635,11 @@ void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd,</div><div class='ctx'> 	spin_lock_init(&amp;ppd-&gt;cca_timer_lock);</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; OPA_MAX_SLS; i++) {</div><div class='del'>-		hrtimer_init(&amp;ppd-&gt;cca_timer[i].hrtimer, CLOCK_MONOTONIC,</div><div class='del'>-			     HRTIMER_MODE_REL);</div><div class='ctx'> 		ppd-&gt;cca_timer[i].ppd = ppd;</div><div class='ctx'> 		ppd-&gt;cca_timer[i].sl = i;</div><div class='ctx'> 		ppd-&gt;cca_timer[i].ccti = 0;</div><div class='del'>-		ppd-&gt;cca_timer[i].hrtimer.function = cca_timer_fn;</div><div class='add'>+		hrtimer_setup(&amp;ppd-&gt;cca_timer[i].hrtimer, cca_timer_fn, CLOCK_MONOTONIC,</div><div class='add'>+			      HRTIMER_MODE_REL);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ppd-&gt;cc_max_table_entries = IB_CC_TABLE_CAP_DEFAULT;</div><div class='hunk'>@@ -805,8 +745,8 @@ static int create_workqueues(struct hfi1_devdata *dd)</div><div class='ctx'> 			ppd-&gt;hfi1_wq =</div><div class='ctx'> 				alloc_workqueue(</div><div class='ctx'> 				    "hfi%d_%d",</div><div class='del'>-				    WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE |</div><div class='del'>-				    WQ_MEM_RECLAIM,</div><div class='add'>+				    WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM |</div><div class='add'>+				    WQ_PERCPU,</div><div class='ctx'> 				    HFI1_MAX_ACTIVE_WORKQUEUE_ENTRIES,</div><div class='ctx'> 				    dd-&gt;unit, pidx);</div><div class='ctx'> 			if (!ppd-&gt;hfi1_wq)</div><div class='hunk'>@@ -845,6 +785,29 @@ wq_error:</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='add'>+ * destroy_workqueues - destroy per port workqueues</div><div class='add'>+ * @dd: the hfi1_ib device</div><div class='add'>+ */</div><div class='add'>+static void destroy_workqueues(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	int pidx;</div><div class='add'>+	struct hfi1_pportdata *ppd;</div><div class='add'>+</div><div class='add'>+	for (pidx = 0; pidx &lt; dd-&gt;num_pports; ++pidx) {</div><div class='add'>+		ppd = dd-&gt;pport + pidx;</div><div class='add'>+</div><div class='add'>+		if (ppd-&gt;hfi1_wq) {</div><div class='add'>+			destroy_workqueue(ppd-&gt;hfi1_wq);</div><div class='add'>+			ppd-&gt;hfi1_wq = NULL;</div><div class='add'>+		}</div><div class='add'>+		if (ppd-&gt;link_wq) {</div><div class='add'>+			destroy_workqueue(ppd-&gt;link_wq);</div><div class='add'>+			ppd-&gt;link_wq = NULL;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='ctx'>  * enable_general_intr() - Enable the IRQs that will be handled by the</div><div class='ctx'>  * general interrupt handler.</div><div class='ctx'>  * @dd: valid devdata</div><div class='hunk'>@@ -892,10 +855,10 @@ int hfi1_init(struct hfi1_devdata *dd, int reinit)</div><div class='ctx'> </div><div class='ctx'> 	if (is_ax(dd)) {</div><div class='ctx'> 		atomic_set(&amp;dd-&gt;drop_packet, DROP_PACKET_ON);</div><div class='del'>-		dd-&gt;do_drop = 1;</div><div class='add'>+		dd-&gt;do_drop = true;</div><div class='ctx'> 	} else {</div><div class='ctx'> 		atomic_set(&amp;dd-&gt;drop_packet, DROP_PACKET_OFF);</div><div class='del'>-		dd-&gt;do_drop = 0;</div><div class='add'>+		dd-&gt;do_drop = false;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* make sure the link is not "up" */</div><div class='hunk'>@@ -911,18 +874,6 @@ int hfi1_init(struct hfi1_devdata *dd, int reinit)</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto done;</div><div class='ctx'> </div><div class='del'>-	/* allocate dummy tail memory for all receive contexts */</div><div class='del'>-	dd-&gt;rcvhdrtail_dummy_kvaddr = dma_alloc_coherent(&amp;dd-&gt;pcidev-&gt;dev,</div><div class='del'>-							 sizeof(u64),</div><div class='del'>-							 &amp;dd-&gt;rcvhdrtail_dummy_dma,</div><div class='del'>-							 GFP_KERNEL);</div><div class='del'>-</div><div class='del'>-	if (!dd-&gt;rcvhdrtail_dummy_kvaddr) {</div><div class='del'>-		dd_dev_err(dd, "cannot allocate dummy tail memory\n");</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto done;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	/* dd-&gt;rcd can be NULL if early initialization failed */</div><div class='ctx'> 	for (i = 0; dd-&gt;rcd &amp;&amp; i &lt; dd-&gt;first_dyn_alloc_ctxt; ++i) {</div><div class='ctx'> 		/*</div><div class='hunk'>@@ -935,8 +886,6 @@ int hfi1_init(struct hfi1_devdata *dd, int reinit)</div><div class='ctx'> 		if (!rcd)</div><div class='ctx'> 			continue;</div><div class='ctx'> </div><div class='del'>-		rcd-&gt;do_interrupt = &amp;handle_receive_interrupt;</div><div class='del'>-</div><div class='ctx'> 		lastfail = hfi1_create_rcvhdrq(dd, rcd);</div><div class='ctx'> 		if (!lastfail)</div><div class='ctx'> 			lastfail = hfi1_setup_eagerbufs(rcd);</div><div class='hunk'>@@ -1036,7 +985,7 @@ static void stop_timers(struct hfi1_devdata *dd)</div><div class='ctx'> 	for (pidx = 0; pidx &lt; dd-&gt;num_pports; ++pidx) {</div><div class='ctx'> 		ppd = dd-&gt;pport + pidx;</div><div class='ctx'> 		if (ppd-&gt;led_override_timer.function) {</div><div class='del'>-			del_timer_sync(&amp;ppd-&gt;led_override_timer);</div><div class='add'>+			timer_delete_sync(&amp;ppd-&gt;led_override_timer);</div><div class='ctx'> 			atomic_set(&amp;ppd-&gt;led_override_timer_active, 0);</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='hunk'>@@ -1077,7 +1026,6 @@ static void shutdown_device(struct hfi1_devdata *dd)</div><div class='ctx'> 	msix_clean_up_interrupts(dd);</div><div class='ctx'> </div><div class='ctx'> 	for (pidx = 0; pidx &lt; dd-&gt;num_pports; ++pidx) {</div><div class='del'>-		ppd = dd-&gt;pport + pidx;</div><div class='ctx'> 		for (i = 0; i &lt; dd-&gt;num_rcv_contexts; i++) {</div><div class='ctx'> 			rcd = hfi1_rcd_get_by_index(dd, i);</div><div class='ctx'> 			hfi1_rcvctrl(dd, HFI1_RCVCTRL_TAILUPD_DIS |</div><div class='hunk'>@@ -1117,15 +1065,10 @@ static void shutdown_device(struct hfi1_devdata *dd)</div><div class='ctx'> 		 * We can't count on interrupts since we are stopping.</div><div class='ctx'> 		 */</div><div class='ctx'> 		hfi1_quiet_serdes(ppd);</div><div class='del'>-</div><div class='del'>-		if (ppd-&gt;hfi1_wq) {</div><div class='del'>-			destroy_workqueue(ppd-&gt;hfi1_wq);</div><div class='del'>-			ppd-&gt;hfi1_wq = NULL;</div><div class='del'>-		}</div><div class='del'>-		if (ppd-&gt;link_wq) {</div><div class='del'>-			destroy_workqueue(ppd-&gt;link_wq);</div><div class='del'>-			ppd-&gt;link_wq = NULL;</div><div class='del'>-		}</div><div class='add'>+		if (ppd-&gt;hfi1_wq)</div><div class='add'>+			flush_workqueue(ppd-&gt;hfi1_wq);</div><div class='add'>+		if (ppd-&gt;link_wq)</div><div class='add'>+			flush_workqueue(ppd-&gt;link_wq);</div><div class='ctx'> 	}</div><div class='ctx'> 	sdma_exit(dd);</div><div class='ctx'> }</div><div class='hunk'>@@ -1149,9 +1092,9 @@ void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)</div><div class='ctx'> 		dma_free_coherent(&amp;dd-&gt;pcidev-&gt;dev, rcvhdrq_size(rcd),</div><div class='ctx'> 				  rcd-&gt;rcvhdrq, rcd-&gt;rcvhdrq_dma);</div><div class='ctx'> 		rcd-&gt;rcvhdrq = NULL;</div><div class='del'>-		if (rcd-&gt;rcvhdrtail_kvaddr) {</div><div class='add'>+		if (hfi1_rcvhdrtail_kvaddr(rcd)) {</div><div class='ctx'> 			dma_free_coherent(&amp;dd-&gt;pcidev-&gt;dev, PAGE_SIZE,</div><div class='del'>-					  (void *)rcd-&gt;rcvhdrtail_kvaddr,</div><div class='add'>+					  (void *)hfi1_rcvhdrtail_kvaddr(rcd),</div><div class='ctx'> 					  rcd-&gt;rcvhdrqtailaddr_dma);</div><div class='ctx'> 			rcd-&gt;rcvhdrtail_kvaddr = NULL;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -1162,7 +1105,7 @@ void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)</div><div class='ctx'> 	rcd-&gt;egrbufs.rcvtids = NULL;</div><div class='ctx'> </div><div class='ctx'> 	for (e = 0; e &lt; rcd-&gt;egrbufs.alloced; e++) {</div><div class='del'>-		if (rcd-&gt;egrbufs.buffers[e].dma)</div><div class='add'>+		if (rcd-&gt;egrbufs.buffers[e].addr)</div><div class='ctx'> 			dma_free_coherent(&amp;dd-&gt;pcidev-&gt;dev,</div><div class='ctx'> 					  rcd-&gt;egrbufs.buffers[e].len,</div><div class='ctx'> 					  rcd-&gt;egrbufs.buffers[e].addr,</div><div class='hunk'>@@ -1214,13 +1157,13 @@ static void finalize_asic_data(struct hfi1_devdata *dd,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * hfi1_clean_devdata - cleans up per-unit data structure</div><div class='add'>+ * hfi1_free_devdata - cleans up and frees per-unit data structure</div><div class='ctx'>  * @dd: pointer to a valid devdata structure</div><div class='ctx'>  *</div><div class='del'>- * It cleans up all data structures set up by</div><div class='add'>+ * It cleans up and frees all data structures set up by</div><div class='ctx'>  * by hfi1_alloc_devdata().</div><div class='ctx'>  */</div><div class='del'>-static void hfi1_clean_devdata(struct hfi1_devdata *dd)</div><div class='add'>+void hfi1_free_devdata(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_asic_data *ad;</div><div class='ctx'> 	unsigned long flags;</div><div class='hunk'>@@ -1243,27 +1186,15 @@ static void hfi1_clean_devdata(struct hfi1_devdata *dd)</div><div class='ctx'> 	dd-&gt;tx_opstats    = NULL;</div><div class='ctx'> 	kfree(dd-&gt;comp_vect);</div><div class='ctx'> 	dd-&gt;comp_vect = NULL;</div><div class='add'>+	if (dd-&gt;rcvhdrtail_dummy_kvaddr)</div><div class='add'>+		dma_free_coherent(&amp;dd-&gt;pcidev-&gt;dev, sizeof(u64),</div><div class='add'>+				  (void *)dd-&gt;rcvhdrtail_dummy_kvaddr,</div><div class='add'>+				  dd-&gt;rcvhdrtail_dummy_dma);</div><div class='add'>+	dd-&gt;rcvhdrtail_dummy_kvaddr = NULL;</div><div class='ctx'> 	sdma_clean(dd, dd-&gt;num_sdma);</div><div class='ctx'> 	rvt_dealloc_device(&amp;dd-&gt;verbs_dev.rdi);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void __hfi1_free_devdata(struct kobject *kobj)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_devdata *dd =</div><div class='del'>-		container_of(kobj, struct hfi1_devdata, kobj);</div><div class='del'>-</div><div class='del'>-	hfi1_clean_devdata(dd);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct kobj_type hfi1_devdata_type = {</div><div class='del'>-	.release = __hfi1_free_devdata,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-void hfi1_free_devdata(struct hfi1_devdata *dd)</div><div class='del'>-{</div><div class='del'>-	kobject_put(&amp;dd-&gt;kobj);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> /**</div><div class='ctx'>  * hfi1_alloc_devdata - Allocate our primary per-unit data structure.</div><div class='ctx'>  * @pdev: Valid PCI device</div><div class='hunk'>@@ -1290,7 +1221,6 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev,</div><div class='ctx'> 	dd-&gt;pport = (struct hfi1_pportdata *)(dd + 1);</div><div class='ctx'> 	dd-&gt;pcidev = pdev;</div><div class='ctx'> 	pci_set_drvdata(pdev, dd);</div><div class='del'>-	dd-&gt;node = NUMA_NO_NODE;</div><div class='ctx'> </div><div class='ctx'> 	ret = xa_alloc_irq(&amp;hfi1_dev_table, &amp;dd-&gt;unit, dd, xa_limit_32b,</div><div class='ctx'> 			GFP_KERNEL);</div><div class='hunk'>@@ -1300,6 +1230,15 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev,</div><div class='ctx'> 		goto bail;</div><div class='ctx'> 	}</div><div class='ctx'> 	rvt_set_ibdev_name(&amp;dd-&gt;verbs_dev.rdi, "%s_%d", class_name(), dd-&gt;unit);</div><div class='add'>+	/*</div><div class='add'>+	 * If the BIOS does not have the NUMA node information set, select</div><div class='add'>+	 * NUMA 0 so we get consistent performance.</div><div class='add'>+	 */</div><div class='add'>+	dd-&gt;node = pcibus_to_node(pdev-&gt;bus);</div><div class='add'>+	if (dd-&gt;node == NUMA_NO_NODE) {</div><div class='add'>+		dd_dev_err(dd, "Invalid PCI NUMA node. Performance may be affected\n");</div><div class='add'>+		dd-&gt;node = 0;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Initialize all locks for the device. This needs to be as early as</div><div class='hunk'>@@ -1349,11 +1288,20 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev,</div><div class='ctx'> 		goto bail;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	kobject_init(&amp;dd-&gt;kobj, &amp;hfi1_devdata_type);</div><div class='add'>+	/* allocate dummy tail memory for all receive contexts */</div><div class='add'>+	dd-&gt;rcvhdrtail_dummy_kvaddr =</div><div class='add'>+		dma_alloc_coherent(&amp;dd-&gt;pcidev-&gt;dev, sizeof(u64),</div><div class='add'>+				   &amp;dd-&gt;rcvhdrtail_dummy_dma, GFP_KERNEL);</div><div class='add'>+	if (!dd-&gt;rcvhdrtail_dummy_kvaddr) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto bail;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	atomic_set(&amp;dd-&gt;ipoib_rsm_usr_num, 0);</div><div class='ctx'> 	return dd;</div><div class='ctx'> </div><div class='ctx'> bail:</div><div class='del'>-	hfi1_clean_devdata(dd);</div><div class='add'>+	hfi1_free_devdata(dd);</div><div class='ctx'> 	return ERR_PTR(ret);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1394,7 +1342,7 @@ static void remove_one(struct pci_dev *);</div><div class='ctx'> static int init_one(struct pci_dev *, const struct pci_device_id *);</div><div class='ctx'> static void shutdown_one(struct pci_dev *);</div><div class='ctx'> </div><div class='del'>-#define DRIVER_LOAD_MSG "Intel " DRIVER_NAME " loaded: "</div><div class='add'>+#define DRIVER_LOAD_MSG "Cornelis " DRIVER_NAME " loaded: "</div><div class='ctx'> #define PFX DRIVER_NAME ": "</div><div class='ctx'> </div><div class='ctx'> const struct pci_device_id hfi1_pci_tbl[] = {</div><div class='hunk'>@@ -1489,7 +1437,6 @@ static int __init hfi1_mod_init(void)</div><div class='ctx'> 		goto bail_dev;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	hfi1_compute_tid_rdma_flow_wt();</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * These must be called before the driver is registered with</div><div class='ctx'> 	 * the PCI subsystem.</div><div class='hunk'>@@ -1557,13 +1504,6 @@ static void cleanup_device_data(struct hfi1_devdata *dd)</div><div class='ctx'> </div><div class='ctx'> 	free_credit_return(dd);</div><div class='ctx'> </div><div class='del'>-	if (dd-&gt;rcvhdrtail_dummy_kvaddr) {</div><div class='del'>-		dma_free_coherent(&amp;dd-&gt;pcidev-&gt;dev, sizeof(u64),</div><div class='del'>-				  (void *)dd-&gt;rcvhdrtail_dummy_kvaddr,</div><div class='del'>-				  dd-&gt;rcvhdrtail_dummy_dma);</div><div class='del'>-		dd-&gt;rcvhdrtail_dummy_kvaddr = NULL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Free any resources still in use (usually just kernel contexts)</div><div class='ctx'> 	 * at unload; we do for ctxtcnt, because that's what we allocate.</div><div class='hunk'>@@ -1612,29 +1552,6 @@ static void postinit_cleanup(struct hfi1_devdata *dd)</div><div class='ctx'> 	hfi1_free_devdata(dd);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int init_validate_rcvhdrcnt(struct hfi1_devdata *dd, uint thecnt)</div><div class='del'>-{</div><div class='del'>-	if (thecnt &lt;= HFI1_MIN_HDRQ_EGRBUF_CNT) {</div><div class='del'>-		dd_dev_err(dd, "Receive header queue count too small\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (thecnt &gt; HFI1_MAX_HDRQ_EGRBUF_CNT) {</div><div class='del'>-		dd_dev_err(dd,</div><div class='del'>-			   "Receive header queue count cannot be greater than %u\n",</div><div class='del'>-			   HFI1_MAX_HDRQ_EGRBUF_CNT);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (thecnt % HDRQ_INCREMENT) {</div><div class='del'>-		dd_dev_err(dd, "Receive header queue count %d must be divisible by %lu\n",</div><div class='del'>-			   thecnt, HDRQ_INCREMENT);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)</div><div class='ctx'> {</div><div class='ctx'> 	int ret = 0, j, pidx, initfail;</div><div class='hunk'>@@ -1662,7 +1579,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Validate some global module parameters */</div><div class='del'>-	ret = init_validate_rcvhdrcnt(dd, rcvhdrcnt);</div><div class='add'>+	ret = hfi1_validate_rcvhdrcnt(dd, rcvhdrcnt);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto bail;</div><div class='ctx'> </div><div class='hunk'>@@ -1721,9 +1638,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)</div><div class='ctx'> 	/* do the generic initialization */</div><div class='ctx'> 	initfail = hfi1_init(dd, 0);</div><div class='ctx'> </div><div class='del'>-	/* setup vnic */</div><div class='del'>-	hfi1_vnic_setup(dd);</div><div class='del'>-</div><div class='ctx'> 	ret = hfi1_register_ib_device(dd);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='hunk'>@@ -1762,7 +1676,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)</div><div class='ctx'> 			hfi1_device_remove(dd);</div><div class='ctx'> 		if (!ret)</div><div class='ctx'> 			hfi1_unregister_ib_device(dd);</div><div class='del'>-		hfi1_vnic_cleanup(dd);</div><div class='ctx'> 		postinit_cleanup(dd);</div><div class='ctx'> 		if (initfail)</div><div class='ctx'> 			ret = initfail;</div><div class='hunk'>@@ -1785,7 +1698,7 @@ static void wait_for_clients(struct hfi1_devdata *dd)</div><div class='ctx'> 	 * Remove the device init value and complete the device if there is</div><div class='ctx'> 	 * no clients or wait for active clients to finish.</div><div class='ctx'> 	 */</div><div class='del'>-	if (atomic_dec_and_test(&amp;dd-&gt;user_refcount))</div><div class='add'>+	if (refcount_dec_and_test(&amp;dd-&gt;user_refcount))</div><div class='ctx'> 		complete(&amp;dd-&gt;user_comp);</div><div class='ctx'> </div><div class='ctx'> 	wait_for_completion(&amp;dd-&gt;user_comp);</div><div class='hunk'>@@ -1807,14 +1720,15 @@ static void remove_one(struct pci_dev *pdev)</div><div class='ctx'> 	/* unregister from IB core */</div><div class='ctx'> 	hfi1_unregister_ib_device(dd);</div><div class='ctx'> </div><div class='del'>-	/* cleanup vnic */</div><div class='del'>-	hfi1_vnic_cleanup(dd);</div><div class='add'>+	/* free netdev data */</div><div class='add'>+	hfi1_free_rx(dd);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Disable the IB link, disable interrupts on the device,</div><div class='ctx'> 	 * clear dma engines, etc.</div><div class='ctx'> 	 */</div><div class='ctx'> 	shutdown_device(dd);</div><div class='add'>+	destroy_workqueues(dd);</div><div class='ctx'> </div><div class='ctx'> 	stop_timers(dd);</div><div class='ctx'> </div><div class='hunk'>@@ -1843,20 +1757,13 @@ static void shutdown_one(struct pci_dev *pdev)</div><div class='ctx'> int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned amt;</div><div class='del'>-	u64 reg;</div><div class='ctx'> </div><div class='ctx'> 	if (!rcd-&gt;rcvhdrq) {</div><div class='del'>-		gfp_t gfp_flags;</div><div class='del'>-</div><div class='ctx'> 		amt = rcvhdrq_size(rcd);</div><div class='ctx'> </div><div class='del'>-		if (rcd-&gt;ctxt &lt; dd-&gt;first_dyn_alloc_ctxt || rcd-&gt;is_vnic)</div><div class='del'>-			gfp_flags = GFP_KERNEL;</div><div class='del'>-		else</div><div class='del'>-			gfp_flags = GFP_USER;</div><div class='ctx'> 		rcd-&gt;rcvhdrq = dma_alloc_coherent(&amp;dd-&gt;pcidev-&gt;dev, amt,</div><div class='ctx'> 						  &amp;rcd-&gt;rcvhdrq_dma,</div><div class='del'>-						  gfp_flags | __GFP_COMP);</div><div class='add'>+						  GFP_KERNEL);</div><div class='ctx'> </div><div class='ctx'> 		if (!rcd-&gt;rcvhdrq) {</div><div class='ctx'> 			dd_dev_err(dd,</div><div class='hunk'>@@ -1870,35 +1777,14 @@ int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)</div><div class='ctx'> 			rcd-&gt;rcvhdrtail_kvaddr = dma_alloc_coherent(&amp;dd-&gt;pcidev-&gt;dev,</div><div class='ctx'> 								    PAGE_SIZE,</div><div class='ctx'> 								    &amp;rcd-&gt;rcvhdrqtailaddr_dma,</div><div class='del'>-								    gfp_flags);</div><div class='add'>+								    GFP_KERNEL);</div><div class='ctx'> 			if (!rcd-&gt;rcvhdrtail_kvaddr)</div><div class='ctx'> 				goto bail_free;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='del'>-	/*</div><div class='del'>-	 * These values are per-context:</div><div class='del'>-	 *	RcvHdrCnt</div><div class='del'>-	 *	RcvHdrEntSize</div><div class='del'>-	 *	RcvHdrSize</div><div class='del'>-	 */</div><div class='del'>-	reg = ((u64)(rcd-&gt;rcvhdrq_cnt &gt;&gt; HDRQ_SIZE_SHIFT)</div><div class='del'>-			&amp; RCV_HDR_CNT_CNT_MASK)</div><div class='del'>-		&lt;&lt; RCV_HDR_CNT_CNT_SHIFT;</div><div class='del'>-	write_kctxt_csr(dd, rcd-&gt;ctxt, RCV_HDR_CNT, reg);</div><div class='del'>-	reg = (encode_rcv_header_entry_size(rcd-&gt;rcvhdrqentsize)</div><div class='del'>-			&amp; RCV_HDR_ENT_SIZE_ENT_SIZE_MASK)</div><div class='del'>-		&lt;&lt; RCV_HDR_ENT_SIZE_ENT_SIZE_SHIFT;</div><div class='del'>-	write_kctxt_csr(dd, rcd-&gt;ctxt, RCV_HDR_ENT_SIZE, reg);</div><div class='del'>-	reg = ((u64)DEFAULT_RCVHDRSIZE &amp; RCV_HDR_SIZE_HDR_SIZE_MASK)</div><div class='del'>-		&lt;&lt; RCV_HDR_SIZE_HDR_SIZE_SHIFT;</div><div class='del'>-	write_kctxt_csr(dd, rcd-&gt;ctxt, RCV_HDR_SIZE, reg);</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * Program dummy tail address for every receive context</div><div class='del'>-	 * before enabling any receive context</div><div class='del'>-	 */</div><div class='del'>-	write_kctxt_csr(dd, rcd-&gt;ctxt, RCV_HDR_TAIL_ADDR,</div><div class='del'>-			dd-&gt;rcvhdrtail_dummy_dma);</div><div class='add'>+	set_hdrq_regs(rcd-&gt;dd, rcd-&gt;ctxt, rcd-&gt;rcvhdrqentsize,</div><div class='add'>+		      rcd-&gt;rcvhdrq_cnt);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='hunk'>@@ -1914,7 +1800,8 @@ bail:</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * allocate eager buffers, both kernel and user contexts.</div><div class='add'>+ * hfi1_setup_eagerbufs - llocate eager buffers, both kernel and user</div><div class='add'>+ * contexts.</div><div class='ctx'>  * @rcd: the context we are setting up.</div><div class='ctx'>  *</div><div class='ctx'>  * Allocate the eager TID buffers and program them into hip.</div><div class='hunk'>@@ -1926,20 +1813,11 @@ int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_devdata *dd = rcd-&gt;dd;</div><div class='ctx'> 	u32 max_entries, egrtop, alloced_bytes = 0;</div><div class='del'>-	gfp_t gfp_flags;</div><div class='ctx'> 	u16 order, idx = 0;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> 	u16 round_mtu = roundup_pow_of_two(hfi1_max_mtu);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='del'>-	 * GFP_USER, but without GFP_FS, so buffer cache can be</div><div class='del'>-	 * coalesced (we hope); otherwise, even at order 4,</div><div class='del'>-	 * heavy filesystem activity makes these fail, and we can</div><div class='del'>-	 * use compound pages.</div><div class='del'>-	 */</div><div class='del'>-	gfp_flags = __GFP_RECLAIM | __GFP_IO | __GFP_COMP;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='ctx'> 	 * The minimum size of the eager buffers is a groups of MTU-sized</div><div class='ctx'> 	 * buffers.</div><div class='ctx'> 	 * The global eager_buffer_size parameter is checked against the</div><div class='hunk'>@@ -1969,7 +1847,7 @@ int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> 			dma_alloc_coherent(&amp;dd-&gt;pcidev-&gt;dev,</div><div class='ctx'> 					   rcd-&gt;egrbufs.rcvtid_size,</div><div class='ctx'> 					   &amp;rcd-&gt;egrbufs.buffers[idx].dma,</div><div class='del'>-					   gfp_flags);</div><div class='add'>+					   GFP_KERNEL);</div><div class='ctx'> 		if (rcd-&gt;egrbufs.buffers[idx].addr) {</div><div class='ctx'> 			rcd-&gt;egrbufs.buffers[idx].len =</div><div class='ctx'> 				rcd-&gt;egrbufs.rcvtid_size;</div><div class='hunk'>@@ -2040,7 +1918,7 @@ int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> 	rcd-&gt;egrbufs.size = alloced_bytes;</div><div class='ctx'> </div><div class='ctx'> 	hfi1_cdbg(PROC,</div><div class='del'>-		  "ctxt%u: Alloced %u rcv tid entries @ %uKB, total %uKB\n",</div><div class='add'>+		  "ctxt%u: Alloced %u rcv tid entries @ %uKB, total %uKB",</div><div class='ctx'> 		  rcd-&gt;ctxt, rcd-&gt;egrbufs.alloced,</div><div class='ctx'> 		  rcd-&gt;egrbufs.rcvtid_size / 1024, rcd-&gt;egrbufs.size / 1024);</div><div class='ctx'> </div><div class='hunk'>@@ -2063,13 +1941,13 @@ int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> 		rcd-&gt;expected_count = MAX_TID_PAIR_ENTRIES * 2;</div><div class='ctx'> </div><div class='ctx'> 	rcd-&gt;expected_base = rcd-&gt;eager_base + egrtop;</div><div class='del'>-	hfi1_cdbg(PROC, "ctxt%u: eager:%u, exp:%u, egrbase:%u, expbase:%u\n",</div><div class='add'>+	hfi1_cdbg(PROC, "ctxt%u: eager:%u, exp:%u, egrbase:%u, expbase:%u",</div><div class='ctx'> 		  rcd-&gt;ctxt, rcd-&gt;egrbufs.alloced, rcd-&gt;expected_count,</div><div class='ctx'> 		  rcd-&gt;eager_base, rcd-&gt;expected_base);</div><div class='ctx'> </div><div class='ctx'> 	if (!hfi1_rcvbuf_validate(rcd-&gt;egrbufs.rcvtid_size, PT_EAGER, &amp;order)) {</div><div class='ctx'> 		hfi1_cdbg(PROC,</div><div class='del'>-			  "ctxt%u: current Eager buffer size is invalid %u\n",</div><div class='add'>+			  "ctxt%u: current Eager buffer size is invalid %u",</div><div class='ctx'> 			  rcd-&gt;ctxt, rcd-&gt;egrbufs.rcvtid_size);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		goto bail_rcvegrbuf_phys;</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/intr.c b/drivers/infiniband/hw/hfi1/intr.c<br/>index 387305b768e9..d8dd1a599631 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/intr.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/intr.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/intr.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/intr.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015, 2016 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/pci.h&gt;</div><div class='hunk'>@@ -89,37 +47,6 @@ static void add_full_mgmt_pkey(struct hfi1_pportdata *ppd)</div><div class='ctx'> 	hfi1_event_pkey_change(ppd-&gt;dd, ppd-&gt;port);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/**</div><div class='del'>- * format_hwmsg - format a single hwerror message</div><div class='del'>- * @msg message buffer</div><div class='del'>- * @msgl length of message buffer</div><div class='del'>- * @hwmsg message to add to message buffer</div><div class='del'>- */</div><div class='del'>-static void format_hwmsg(char *msg, size_t msgl, const char *hwmsg)</div><div class='del'>-{</div><div class='del'>-	strlcat(msg, "[", msgl);</div><div class='del'>-	strlcat(msg, hwmsg, msgl);</div><div class='del'>-	strlcat(msg, "]", msgl);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * hfi1_format_hwerrors - format hardware error messages for display</div><div class='del'>- * @hwerrs hardware errors bit vector</div><div class='del'>- * @hwerrmsgs hardware error descriptions</div><div class='del'>- * @nhwerrmsgs number of hwerrmsgs</div><div class='del'>- * @msg message buffer</div><div class='del'>- * @msgl message buffer length</div><div class='del'>- */</div><div class='del'>-void hfi1_format_hwerrors(u64 hwerrs, const struct hfi1_hwerror_msgs *hwerrmsgs,</div><div class='del'>-			  size_t nhwerrmsgs, char *msg, size_t msgl)</div><div class='del'>-{</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; nhwerrmsgs; i++)</div><div class='del'>-		if (hwerrs &amp; hwerrmsgs[i].mask)</div><div class='del'>-			format_hwmsg(msg, msgl, hwerrmsgs[i].msg);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static void signal_ib_event(struct hfi1_pportdata *ppd, enum ib_event_type ev)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_event event;</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/iowait.c b/drivers/infiniband/hw/hfi1/iowait.c<br/>index adb4a1ba921b..111489802614 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/iowait.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/iowait.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/iowait.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/iowait.c</a></div><div class='hunk'>@@ -26,7 +26,7 @@ inline void iowait_clear_flag(struct iowait *wait, u32 flag)</div><div class='ctx'> 	clear_bit(flag, &amp;wait-&gt;flags);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/**</div><div class='add'>+/*</div><div class='ctx'>  * iowait_init() - initialize wait structure</div><div class='ctx'>  * @wait: wait struct to initialize</div><div class='ctx'>  * @tx_limit: limit for overflow queuing</div><div class='hunk'>@@ -81,12 +81,14 @@ void iowait_init(struct iowait *wait, u32 tx_limit,</div><div class='ctx'> void iowait_cancel_work(struct iowait *w)</div><div class='ctx'> {</div><div class='ctx'> 	cancel_work_sync(&amp;iowait_get_ib_work(w)-&gt;iowork);</div><div class='del'>-	cancel_work_sync(&amp;iowait_get_tid_work(w)-&gt;iowork);</div><div class='add'>+	/* Make sure that the iowork for TID RDMA is used */</div><div class='add'>+	if (iowait_get_tid_work(w)-&gt;iowork.func)</div><div class='add'>+		cancel_work_sync(&amp;iowait_get_tid_work(w)-&gt;iowork);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * iowait_set_work_flag - set work flag based on leg</div><div class='del'>- * @w - the iowait work struct</div><div class='add'>+ * @w: the iowait work struct</div><div class='ctx'>  */</div><div class='ctx'> int iowait_set_work_flag(struct iowait_work *w)</div><div class='ctx'> {</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/iowait.h b/drivers/infiniband/hw/hfi1/iowait.h<br/>index 07847cb72169..7259f4f55700 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/iowait.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/iowait.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/iowait.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/iowait.h</a></div><div class='hunk'>@@ -1,52 +1,11 @@</div><div class='del'>-#ifndef _HFI1_IOWAIT_H</div><div class='del'>-#define _HFI1_IOWAIT_H</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='add'>+#ifndef _HFI1_IOWAIT_H</div><div class='add'>+#define _HFI1_IOWAIT_H</div><div class='add'>+</div><div class='ctx'> #include &lt;linux/list.h&gt;</div><div class='ctx'> #include &lt;linux/workqueue.h&gt;</div><div class='ctx'> #include &lt;linux/wait.h&gt;</div><div class='hunk'>@@ -133,7 +92,7 @@ struct iowait_work {</div><div class='ctx'>  *</div><div class='ctx'>  * The lock field is used by waiters to record</div><div class='ctx'>  * the seqlock_t that guards the list head.</div><div class='del'>- * Waiters explicity know that, but the destroy</div><div class='add'>+ * Waiters explicitly know that, but the destroy</div><div class='ctx'>  * code that unwaits QPs does not.</div><div class='ctx'>  */</div><div class='ctx'> struct iowait {</div><div class='hunk'>@@ -321,7 +280,7 @@ static inline void iowait_drain_wakeup(struct iowait *wait)</div><div class='ctx'> /**</div><div class='ctx'>  * iowait_get_txhead() - get packet off of iowait list</div><div class='ctx'>  *</div><div class='del'>- * @wait iowait_work struture</div><div class='add'>+ * @wait: iowait_work structure</div><div class='ctx'>  */</div><div class='ctx'> static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait)</div><div class='ctx'> {</div><div class='hunk'>@@ -399,7 +358,7 @@ static inline void iowait_get_priority(struct iowait *w)</div><div class='ctx'>  * @wait_head: the wait queue</div><div class='ctx'>  *</div><div class='ctx'>  * This function is called to insert an iowait struct into a</div><div class='del'>- * wait queue after a resource (eg, sdma decriptor or pio</div><div class='add'>+ * wait queue after a resource (eg, sdma descriptor or pio</div><div class='ctx'>  * buffer) is run out.</div><div class='ctx'>  */</div><div class='ctx'> static inline void iowait_queue(bool pkts_sent, struct iowait *w,</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/ipoib.h b/drivers/infiniband/hw/hfi1/ipoib.h<br/>new file mode 100644<br/>index 000000000000..aec60d4888eb<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/ipoib.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/ipoib.h</a></div><div class='hunk'>@@ -0,0 +1,171 @@</div><div class='add'>+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */</div><div class='add'>+/*</div><div class='add'>+ * Copyright(c) 2020 Intel Corporation.</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * This file contains HFI1 support for IPOIB functionality</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef HFI1_IPOIB_H</div><div class='add'>+#define HFI1_IPOIB_H</div><div class='add'>+</div><div class='add'>+#include &lt;linux/types.h&gt;</div><div class='add'>+#include &lt;linux/stddef.h&gt;</div><div class='add'>+#include &lt;linux/atomic.h&gt;</div><div class='add'>+#include &lt;linux/netdevice.h&gt;</div><div class='add'>+#include &lt;linux/slab.h&gt;</div><div class='add'>+#include &lt;linux/skbuff.h&gt;</div><div class='add'>+#include &lt;linux/list.h&gt;</div><div class='add'>+#include &lt;linux/if_infiniband.h&gt;</div><div class='add'>+</div><div class='add'>+#include "hfi.h"</div><div class='add'>+#include "iowait.h"</div><div class='add'>+#include "netdev.h"</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+</div><div class='add'>+#define HFI1_IPOIB_ENTROPY_SHIFT   24</div><div class='add'>+</div><div class='add'>+#define HFI1_IPOIB_TXREQ_NAME_LEN   32</div><div class='add'>+</div><div class='add'>+#define HFI1_IPOIB_PSEUDO_LEN 20</div><div class='add'>+#define HFI1_IPOIB_ENCAP_LEN 4</div><div class='add'>+</div><div class='add'>+struct hfi1_ipoib_dev_priv;</div><div class='add'>+</div><div class='add'>+union hfi1_ipoib_flow {</div><div class='add'>+	u16 as_int;</div><div class='add'>+	struct {</div><div class='add'>+		u8 tx_queue;</div><div class='add'>+		u8 sc5;</div><div class='add'>+	} __attribute__((__packed__));</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * struct ipoib_txreq - IPOIB transmit descriptor</div><div class='add'>+ * @txreq: sdma transmit request</div><div class='add'>+ * @sdma_hdr: 9b ib headers</div><div class='add'>+ * @sdma_status: status returned by sdma engine</div><div class='add'>+ * @complete: non-zero implies complete</div><div class='add'>+ * @priv: ipoib netdev private data</div><div class='add'>+ * @txq: txq on which skb was output</div><div class='add'>+ * @skb: skb to send</div><div class='add'>+ */</div><div class='add'>+struct ipoib_txreq {</div><div class='add'>+	struct sdma_txreq           txreq;</div><div class='add'>+	struct hfi1_sdma_header     *sdma_hdr;</div><div class='add'>+	int                         sdma_status;</div><div class='add'>+	int                         complete;</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv;</div><div class='add'>+	struct hfi1_ipoib_txq      *txq;</div><div class='add'>+	struct sk_buff             *skb;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * struct hfi1_ipoib_circ_buf - List of items to be processed</div><div class='add'>+ * @items: ring of items each a power of two size</div><div class='add'>+ * @max_items: max items + 1 that the ring can contain</div><div class='add'>+ * @shift: log2 of size for getting txreq</div><div class='add'>+ * @sent_txreqs: count of txreqs posted to sdma</div><div class='add'>+ * @tail: ring tail</div><div class='add'>+ * @stops: count of stops of queue</div><div class='add'>+ * @ring_full: ring has been filled</div><div class='add'>+ * @no_desc: descriptor shortage seen</div><div class='add'>+ * @complete_txreqs: count of txreqs completed by sdma</div><div class='add'>+ * @head: ring head</div><div class='add'>+ */</div><div class='add'>+struct hfi1_ipoib_circ_buf {</div><div class='add'>+	void *items;</div><div class='add'>+	u32 max_items;</div><div class='add'>+	u32 shift;</div><div class='add'>+	/* consumer cache line */</div><div class='add'>+	u64 ____cacheline_aligned_in_smp sent_txreqs;</div><div class='add'>+	u32 avail;</div><div class='add'>+	u32 tail;</div><div class='add'>+	atomic_t stops;</div><div class='add'>+	atomic_t ring_full;</div><div class='add'>+	atomic_t no_desc;</div><div class='add'>+	/* producer cache line */</div><div class='add'>+	u64 ____cacheline_aligned_in_smp complete_txreqs;</div><div class='add'>+	u32 head;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * struct hfi1_ipoib_txq - IPOIB per Tx queue information</div><div class='add'>+ * @priv: private pointer</div><div class='add'>+ * @sde: sdma engine</div><div class='add'>+ * @tx_list: tx request list</div><div class='add'>+ * @sent_txreqs: count of txreqs posted to sdma</div><div class='add'>+ * @flow: tracks when list needs to be flushed for a flow change</div><div class='add'>+ * @q_idx: ipoib Tx queue index</div><div class='add'>+ * @pkts_sent: indicator packets have been sent from this queue</div><div class='add'>+ * @wait: iowait structure</div><div class='add'>+ * @napi: pointer to tx napi interface</div><div class='add'>+ * @tx_ring: ring of ipoib txreqs to be reaped by napi callback</div><div class='add'>+ */</div><div class='add'>+struct hfi1_ipoib_txq {</div><div class='add'>+	struct napi_struct napi;</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv;</div><div class='add'>+	struct sdma_engine *sde;</div><div class='add'>+	struct list_head tx_list;</div><div class='add'>+	union hfi1_ipoib_flow flow;</div><div class='add'>+	u8 q_idx;</div><div class='add'>+	bool pkts_sent;</div><div class='add'>+	struct iowait wait;</div><div class='add'>+</div><div class='add'>+	struct hfi1_ipoib_circ_buf ____cacheline_aligned_in_smp tx_ring;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct hfi1_ipoib_dev_priv {</div><div class='add'>+	struct hfi1_devdata *dd;</div><div class='add'>+	struct net_device   *netdev;</div><div class='add'>+	struct ib_device    *device;</div><div class='add'>+	struct hfi1_ipoib_txq *txqs;</div><div class='add'>+	const struct net_device_ops *netdev_ops;</div><div class='add'>+	struct rvt_qp *qp;</div><div class='add'>+	u32 qkey;</div><div class='add'>+	u16 pkey;</div><div class='add'>+	u16 pkey_index;</div><div class='add'>+	u8 port_num;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* hfi1 ipoib rdma netdev's private data structure */</div><div class='add'>+struct hfi1_ipoib_rdma_netdev {</div><div class='add'>+	struct rdma_netdev rn;  /* keep this first */</div><div class='add'>+	/* followed by device private data */</div><div class='add'>+	struct hfi1_ipoib_dev_priv dev_priv;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline struct hfi1_ipoib_dev_priv *</div><div class='add'>+hfi1_ipoib_priv(const struct net_device *dev)</div><div class='add'>+{</div><div class='add'>+	return &amp;((struct hfi1_ipoib_rdma_netdev *)netdev_priv(dev))-&gt;dev_priv;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hfi1_ipoib_send(struct net_device *dev,</div><div class='add'>+		    struct sk_buff *skb,</div><div class='add'>+		    struct ib_ah *address,</div><div class='add'>+		    u32 dqpn);</div><div class='add'>+</div><div class='add'>+int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv);</div><div class='add'>+void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv);</div><div class='add'>+</div><div class='add'>+int hfi1_ipoib_rxq_init(struct net_device *dev);</div><div class='add'>+void hfi1_ipoib_rxq_deinit(struct net_device *dev);</div><div class='add'>+</div><div class='add'>+void hfi1_ipoib_napi_tx_enable(struct net_device *dev);</div><div class='add'>+void hfi1_ipoib_napi_tx_disable(struct net_device *dev);</div><div class='add'>+</div><div class='add'>+struct sk_buff *hfi1_ipoib_prepare_skb(struct hfi1_netdev_rxq *rxq,</div><div class='add'>+				       int size, void *data);</div><div class='add'>+</div><div class='add'>+int hfi1_ipoib_rn_get_params(struct ib_device *device,</div><div class='add'>+			     u32 port_num,</div><div class='add'>+			     enum rdma_netdev_t type,</div><div class='add'>+			     struct rdma_netdev_alloc_params *params);</div><div class='add'>+</div><div class='add'>+void hfi1_ipoib_tx_timeout(struct net_device *dev, unsigned int q);</div><div class='add'>+</div><div class='add'>+#endif /* _IPOIB_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/ipoib_main.c b/drivers/infiniband/hw/hfi1/ipoib_main.c<br/>new file mode 100644<br/>index 000000000000..7c9d5203002b<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/ipoib_main.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/ipoib_main.c</a></div><div class='hunk'>@@ -0,0 +1,250 @@</div><div class='add'>+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)</div><div class='add'>+/*</div><div class='add'>+ * Copyright(c) 2020 Intel Corporation.</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * This file contains HFI1 support for ipoib functionality</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include "ipoib.h"</div><div class='add'>+#include "hfi.h"</div><div class='add'>+</div><div class='add'>+static u32 qpn_from_mac(const u8 *mac_arr)</div><div class='add'>+{</div><div class='add'>+	return (u32)mac_arr[1] &lt;&lt; 16 | mac_arr[2] &lt;&lt; 8 | mac_arr[3];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hfi1_ipoib_dev_init(struct net_device *dev)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = priv-&gt;netdev_ops-&gt;ndo_init(dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = hfi1_netdev_add_data(priv-&gt;dd,</div><div class='add'>+				   qpn_from_mac(priv-&gt;netdev-&gt;dev_addr),</div><div class='add'>+				   dev);</div><div class='add'>+	if (ret &lt; 0) {</div><div class='add'>+		priv-&gt;netdev_ops-&gt;ndo_uninit(dev);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hfi1_ipoib_dev_uninit(struct net_device *dev)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);</div><div class='add'>+</div><div class='add'>+	hfi1_netdev_remove_data(priv-&gt;dd, qpn_from_mac(priv-&gt;netdev-&gt;dev_addr));</div><div class='add'>+</div><div class='add'>+	priv-&gt;netdev_ops-&gt;ndo_uninit(dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hfi1_ipoib_dev_open(struct net_device *dev)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = priv-&gt;netdev_ops-&gt;ndo_open(dev);</div><div class='add'>+	if (!ret) {</div><div class='add'>+		struct hfi1_ibport *ibp = to_iport(priv-&gt;device,</div><div class='add'>+						   priv-&gt;port_num);</div><div class='add'>+		struct rvt_qp *qp;</div><div class='add'>+		u32 qpn = qpn_from_mac(priv-&gt;netdev-&gt;dev_addr);</div><div class='add'>+</div><div class='add'>+		rcu_read_lock();</div><div class='add'>+		qp = rvt_lookup_qpn(ib_to_rvt(priv-&gt;device), &amp;ibp-&gt;rvp, qpn);</div><div class='add'>+		if (!qp) {</div><div class='add'>+			rcu_read_unlock();</div><div class='add'>+			priv-&gt;netdev_ops-&gt;ndo_stop(dev);</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		}</div><div class='add'>+		rvt_get_qp(qp);</div><div class='add'>+		priv-&gt;qp = qp;</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+</div><div class='add'>+		hfi1_netdev_enable_queues(priv-&gt;dd);</div><div class='add'>+		hfi1_ipoib_napi_tx_enable(dev);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hfi1_ipoib_dev_stop(struct net_device *dev)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);</div><div class='add'>+</div><div class='add'>+	if (!priv-&gt;qp)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	hfi1_ipoib_napi_tx_disable(dev);</div><div class='add'>+	hfi1_netdev_disable_queues(priv-&gt;dd);</div><div class='add'>+</div><div class='add'>+	rvt_put_qp(priv-&gt;qp);</div><div class='add'>+	priv-&gt;qp = NULL;</div><div class='add'>+</div><div class='add'>+	return priv-&gt;netdev_ops-&gt;ndo_stop(dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct net_device_ops hfi1_ipoib_netdev_ops = {</div><div class='add'>+	.ndo_init         = hfi1_ipoib_dev_init,</div><div class='add'>+	.ndo_uninit       = hfi1_ipoib_dev_uninit,</div><div class='add'>+	.ndo_open         = hfi1_ipoib_dev_open,</div><div class='add'>+	.ndo_stop         = hfi1_ipoib_dev_stop,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int hfi1_ipoib_mcast_attach(struct net_device *dev,</div><div class='add'>+				   struct ib_device *device,</div><div class='add'>+				   union ib_gid *mgid,</div><div class='add'>+				   u16 mlid,</div><div class='add'>+				   int set_qkey,</div><div class='add'>+				   u32 qkey)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);</div><div class='add'>+	u32 qpn = (u32)qpn_from_mac(priv-&gt;netdev-&gt;dev_addr);</div><div class='add'>+	struct hfi1_ibport *ibp = to_iport(priv-&gt;device, priv-&gt;port_num);</div><div class='add'>+	struct rvt_qp *qp;</div><div class='add'>+	int ret = -EINVAL;</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+</div><div class='add'>+	qp = rvt_lookup_qpn(ib_to_rvt(priv-&gt;device), &amp;ibp-&gt;rvp, qpn);</div><div class='add'>+	if (qp) {</div><div class='add'>+		rvt_get_qp(qp);</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+		if (set_qkey)</div><div class='add'>+			priv-&gt;qkey = qkey;</div><div class='add'>+</div><div class='add'>+		/* attach QP to multicast group */</div><div class='add'>+		ret = ib_attach_mcast(&amp;qp-&gt;ibqp, mgid, mlid);</div><div class='add'>+		rvt_put_qp(qp);</div><div class='add'>+	} else {</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hfi1_ipoib_mcast_detach(struct net_device *dev,</div><div class='add'>+				   struct ib_device *device,</div><div class='add'>+				   union ib_gid *mgid,</div><div class='add'>+				   u16 mlid)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);</div><div class='add'>+	u32 qpn = (u32)qpn_from_mac(priv-&gt;netdev-&gt;dev_addr);</div><div class='add'>+	struct hfi1_ibport *ibp = to_iport(priv-&gt;device, priv-&gt;port_num);</div><div class='add'>+	struct rvt_qp *qp;</div><div class='add'>+	int ret = -EINVAL;</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+</div><div class='add'>+	qp = rvt_lookup_qpn(ib_to_rvt(priv-&gt;device), &amp;ibp-&gt;rvp, qpn);</div><div class='add'>+	if (qp) {</div><div class='add'>+		rvt_get_qp(qp);</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+		ret = ib_detach_mcast(&amp;qp-&gt;ibqp, mgid, mlid);</div><div class='add'>+		rvt_put_qp(qp);</div><div class='add'>+	} else {</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+	}</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hfi1_ipoib_netdev_dtor(struct net_device *dev)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);</div><div class='add'>+</div><div class='add'>+	hfi1_ipoib_txreq_deinit(priv);</div><div class='add'>+	hfi1_ipoib_rxq_deinit(priv-&gt;netdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hfi1_ipoib_set_id(struct net_device *dev, int id)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);</div><div class='add'>+</div><div class='add'>+	priv-&gt;pkey_index = (u16)id;</div><div class='add'>+	ib_query_pkey(priv-&gt;device,</div><div class='add'>+		      priv-&gt;port_num,</div><div class='add'>+		      priv-&gt;pkey_index,</div><div class='add'>+		      &amp;priv-&gt;pkey);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hfi1_ipoib_setup_rn(struct ib_device *device,</div><div class='add'>+			       u32 port_num,</div><div class='add'>+			       struct net_device *netdev,</div><div class='add'>+			       void *param)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_devdata *dd = dd_from_ibdev(device);</div><div class='add'>+	struct rdma_netdev *rn = netdev_priv(netdev);</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rn-&gt;send = hfi1_ipoib_send;</div><div class='add'>+	rn-&gt;tx_timeout = hfi1_ipoib_tx_timeout;</div><div class='add'>+	rn-&gt;attach_mcast = hfi1_ipoib_mcast_attach;</div><div class='add'>+	rn-&gt;detach_mcast = hfi1_ipoib_mcast_detach;</div><div class='add'>+	rn-&gt;set_id = hfi1_ipoib_set_id;</div><div class='add'>+	rn-&gt;hca = device;</div><div class='add'>+	rn-&gt;port_num = port_num;</div><div class='add'>+	rn-&gt;mtu = netdev-&gt;mtu;</div><div class='add'>+</div><div class='add'>+	priv = hfi1_ipoib_priv(netdev);</div><div class='add'>+	priv-&gt;dd = dd;</div><div class='add'>+	priv-&gt;netdev = netdev;</div><div class='add'>+	priv-&gt;device = device;</div><div class='add'>+	priv-&gt;port_num = port_num;</div><div class='add'>+	priv-&gt;netdev_ops = netdev-&gt;netdev_ops;</div><div class='add'>+</div><div class='add'>+	ib_query_pkey(device, port_num, priv-&gt;pkey_index, &amp;priv-&gt;pkey);</div><div class='add'>+</div><div class='add'>+	rc = hfi1_ipoib_txreq_init(priv);</div><div class='add'>+	if (rc) {</div><div class='add'>+		dd_dev_err(dd, "IPoIB netdev TX init - failed(%d)\n", rc);</div><div class='add'>+		return rc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rc = hfi1_ipoib_rxq_init(netdev);</div><div class='add'>+	if (rc) {</div><div class='add'>+		dd_dev_err(dd, "IPoIB netdev RX init - failed(%d)\n", rc);</div><div class='add'>+		hfi1_ipoib_txreq_deinit(priv);</div><div class='add'>+		return rc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	netdev-&gt;netdev_ops = &amp;hfi1_ipoib_netdev_ops;</div><div class='add'>+</div><div class='add'>+	netdev-&gt;priv_destructor = hfi1_ipoib_netdev_dtor;</div><div class='add'>+	netdev-&gt;needs_free_netdev = true;</div><div class='add'>+	netdev-&gt;pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hfi1_ipoib_rn_get_params(struct ib_device *device,</div><div class='add'>+			     u32 port_num,</div><div class='add'>+			     enum rdma_netdev_t type,</div><div class='add'>+			     struct rdma_netdev_alloc_params *params)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_devdata *dd = dd_from_ibdev(device);</div><div class='add'>+</div><div class='add'>+	if (type != RDMA_NETDEV_IPOIB)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (!HFI1_CAP_IS_KSET(AIP) || !dd-&gt;num_netdev_contexts)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (!port_num || port_num &gt; dd-&gt;num_pports)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	params-&gt;sizeof_priv = sizeof(struct hfi1_ipoib_rdma_netdev);</div><div class='add'>+	params-&gt;txqs = dd-&gt;num_sdma;</div><div class='add'>+	params-&gt;rxqs = dd-&gt;num_netdev_contexts;</div><div class='add'>+	params-&gt;param = NULL;</div><div class='add'>+	params-&gt;initialize_rdma_netdev = hfi1_ipoib_setup_rn;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/ipoib_rx.c b/drivers/infiniband/hw/hfi1/ipoib_rx.c<br/>new file mode 100644<br/>index 000000000000..629691a572ef<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/ipoib_rx.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/ipoib_rx.c</a></div><div class='hunk'>@@ -0,0 +1,92 @@</div><div class='add'>+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)</div><div class='add'>+/*</div><div class='add'>+ * Copyright(c) 2020 Intel Corporation.</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include "netdev.h"</div><div class='add'>+#include "ipoib.h"</div><div class='add'>+</div><div class='add'>+#define HFI1_IPOIB_SKB_PAD ((NET_SKB_PAD) + (NET_IP_ALIGN))</div><div class='add'>+</div><div class='add'>+static void copy_ipoib_buf(struct sk_buff *skb, void *data, int size)</div><div class='add'>+{</div><div class='add'>+	skb_checksum_none_assert(skb);</div><div class='add'>+	skb-&gt;protocol = *((__be16 *)data);</div><div class='add'>+</div><div class='add'>+	skb_put_data(skb, data, size);</div><div class='add'>+	skb-&gt;mac_header = HFI1_IPOIB_PSEUDO_LEN;</div><div class='add'>+	skb_pull(skb, HFI1_IPOIB_ENCAP_LEN);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct sk_buff *prepare_frag_skb(struct napi_struct *napi, int size)</div><div class='add'>+{</div><div class='add'>+	struct sk_buff *skb;</div><div class='add'>+	int skb_size = SKB_DATA_ALIGN(size + HFI1_IPOIB_SKB_PAD);</div><div class='add'>+	void *frag;</div><div class='add'>+</div><div class='add'>+	skb_size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));</div><div class='add'>+	skb_size = SKB_DATA_ALIGN(skb_size);</div><div class='add'>+	frag = napi_alloc_frag(skb_size);</div><div class='add'>+</div><div class='add'>+	if (unlikely(!frag))</div><div class='add'>+		return napi_alloc_skb(napi, size);</div><div class='add'>+</div><div class='add'>+	skb = build_skb(frag, skb_size);</div><div class='add'>+</div><div class='add'>+	if (unlikely(!skb)) {</div><div class='add'>+		skb_free_frag(frag);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	skb_reserve(skb, HFI1_IPOIB_SKB_PAD);</div><div class='add'>+	return skb;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct sk_buff *hfi1_ipoib_prepare_skb(struct hfi1_netdev_rxq *rxq,</div><div class='add'>+				       int size, void *data)</div><div class='add'>+{</div><div class='add'>+	struct napi_struct *napi = &amp;rxq-&gt;napi;</div><div class='add'>+	int skb_size = size + HFI1_IPOIB_ENCAP_LEN;</div><div class='add'>+	struct sk_buff *skb;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * For smaller(4k + skb overhead) allocations we will go using</div><div class='add'>+	 * napi cache. Otherwise we will try to use napi frag cache.</div><div class='add'>+	 */</div><div class='add'>+	if (size &lt;= SKB_WITH_OVERHEAD(PAGE_SIZE))</div><div class='add'>+		skb = napi_alloc_skb(napi, skb_size);</div><div class='add'>+	else</div><div class='add'>+		skb = prepare_frag_skb(napi, skb_size);</div><div class='add'>+</div><div class='add'>+	if (unlikely(!skb))</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	copy_ipoib_buf(skb, data, size);</div><div class='add'>+</div><div class='add'>+	return skb;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hfi1_ipoib_rxq_init(struct net_device *netdev)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *ipoib_priv = hfi1_ipoib_priv(netdev);</div><div class='add'>+	struct hfi1_devdata *dd = ipoib_priv-&gt;dd;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = hfi1_netdev_rx_init(dd);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	hfi1_init_aip_rsm(dd);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hfi1_ipoib_rxq_deinit(struct net_device *netdev)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *ipoib_priv = hfi1_ipoib_priv(netdev);</div><div class='add'>+	struct hfi1_devdata *dd = ipoib_priv-&gt;dd;</div><div class='add'>+</div><div class='add'>+	hfi1_deinit_aip_rsm(dd);</div><div class='add'>+	hfi1_netdev_rx_destroy(dd);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/ipoib_tx.c b/drivers/infiniband/hw/hfi1/ipoib_tx.c<br/>new file mode 100644<br/>index 000000000000..8b9cc55db59d<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/ipoib_tx.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/ipoib_tx.c</a></div><div class='hunk'>@@ -0,0 +1,868 @@</div><div class='add'>+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)</div><div class='add'>+/*</div><div class='add'>+ * Copyright(c) 2020 Intel Corporation.</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * This file contains HFI1 support for IPOIB SDMA functionality</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/log2.h&gt;</div><div class='add'>+#include &lt;linux/circ_buf.h&gt;</div><div class='add'>+</div><div class='add'>+#include "sdma.h"</div><div class='add'>+#include "verbs.h"</div><div class='add'>+#include "trace_ibhdrs.h"</div><div class='add'>+#include "ipoib.h"</div><div class='add'>+#include "trace_tx.h"</div><div class='add'>+</div><div class='add'>+/* Add a convenience helper */</div><div class='add'>+#define CIRC_ADD(val, add, size) (((val) + (add)) &amp; ((size) - 1))</div><div class='add'>+#define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size)</div><div class='add'>+#define CIRC_PREV(val, size) CIRC_ADD(val, -1, size)</div><div class='add'>+</div><div class='add'>+struct ipoib_txparms {</div><div class='add'>+	struct hfi1_devdata        *dd;</div><div class='add'>+	struct rdma_ah_attr        *ah_attr;</div><div class='add'>+	struct hfi1_ibport         *ibp;</div><div class='add'>+	struct hfi1_ipoib_txq      *txq;</div><div class='add'>+	union hfi1_ipoib_flow       flow;</div><div class='add'>+	u32                         dqpn;</div><div class='add'>+	u8                          hdr_dwords;</div><div class='add'>+	u8                          entropy;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static struct ipoib_txreq *</div><div class='add'>+hfi1_txreq_from_idx(struct hfi1_ipoib_circ_buf *r, u32 idx)</div><div class='add'>+{</div><div class='add'>+	return (struct ipoib_txreq *)(r-&gt;items + (idx &lt;&lt; r-&gt;shift));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u32 hfi1_ipoib_txreqs(const u64 sent, const u64 completed)</div><div class='add'>+{</div><div class='add'>+	return sent - completed;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u64 hfi1_ipoib_used(struct hfi1_ipoib_txq *txq)</div><div class='add'>+{</div><div class='add'>+	return hfi1_ipoib_txreqs(txq-&gt;tx_ring.sent_txreqs,</div><div class='add'>+				 txq-&gt;tx_ring.complete_txreqs);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hfi1_ipoib_stop_txq(struct hfi1_ipoib_txq *txq)</div><div class='add'>+{</div><div class='add'>+	trace_hfi1_txq_stop(txq);</div><div class='add'>+	if (atomic_inc_return(&amp;txq-&gt;tx_ring.stops) == 1)</div><div class='add'>+		netif_stop_subqueue(txq-&gt;priv-&gt;netdev, txq-&gt;q_idx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hfi1_ipoib_wake_txq(struct hfi1_ipoib_txq *txq)</div><div class='add'>+{</div><div class='add'>+	trace_hfi1_txq_wake(txq);</div><div class='add'>+	if (atomic_dec_and_test(&amp;txq-&gt;tx_ring.stops))</div><div class='add'>+		netif_wake_subqueue(txq-&gt;priv-&gt;netdev, txq-&gt;q_idx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static uint hfi1_ipoib_ring_hwat(struct hfi1_ipoib_txq *txq)</div><div class='add'>+{</div><div class='add'>+	return min_t(uint, txq-&gt;priv-&gt;netdev-&gt;tx_queue_len,</div><div class='add'>+		     txq-&gt;tx_ring.max_items - 1);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static uint hfi1_ipoib_ring_lwat(struct hfi1_ipoib_txq *txq)</div><div class='add'>+{</div><div class='add'>+	return min_t(uint, txq-&gt;priv-&gt;netdev-&gt;tx_queue_len,</div><div class='add'>+		     txq-&gt;tx_ring.max_items) &gt;&gt; 1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hfi1_ipoib_check_queue_depth(struct hfi1_ipoib_txq *txq)</div><div class='add'>+{</div><div class='add'>+	++txq-&gt;tx_ring.sent_txreqs;</div><div class='add'>+	if (hfi1_ipoib_used(txq) &gt;= hfi1_ipoib_ring_hwat(txq) &amp;&amp;</div><div class='add'>+	    !atomic_xchg(&amp;txq-&gt;tx_ring.ring_full, 1)) {</div><div class='add'>+		trace_hfi1_txq_full(txq);</div><div class='add'>+		hfi1_ipoib_stop_txq(txq);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq)</div><div class='add'>+{</div><div class='add'>+	struct net_device *dev = txq-&gt;priv-&gt;netdev;</div><div class='add'>+</div><div class='add'>+	/* If shutting down just return as queue state is irrelevant */</div><div class='add'>+	if (unlikely(dev-&gt;reg_state != NETREG_REGISTERED))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * When the queue has been drained to less than half full it will be</div><div class='add'>+	 * restarted.</div><div class='add'>+	 * The size of the txreq ring is fixed at initialization.</div><div class='add'>+	 * The tx queue len can be adjusted upward while the interface is</div><div class='add'>+	 * running.</div><div class='add'>+	 * The tx queue len can be large enough to overflow the txreq_ring.</div><div class='add'>+	 * Use the minimum of the current tx_queue_len or the rings max txreqs</div><div class='add'>+	 * to protect against ring overflow.</div><div class='add'>+	 */</div><div class='add'>+	if (hfi1_ipoib_used(txq) &lt; hfi1_ipoib_ring_lwat(txq) &amp;&amp;</div><div class='add'>+	    atomic_xchg(&amp;txq-&gt;tx_ring.ring_full, 0)) {</div><div class='add'>+		trace_hfi1_txq_xmit_unstopped(txq);</div><div class='add'>+		hfi1_ipoib_wake_txq(txq);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hfi1_ipoib_free_tx(struct ipoib_txreq *tx, int budget)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv = tx-&gt;txq-&gt;priv;</div><div class='add'>+</div><div class='add'>+	if (likely(!tx-&gt;sdma_status)) {</div><div class='add'>+		dev_sw_netstats_tx_add(priv-&gt;netdev, 1, tx-&gt;skb-&gt;len);</div><div class='add'>+	} else {</div><div class='add'>+		++priv-&gt;netdev-&gt;stats.tx_errors;</div><div class='add'>+		dd_dev_warn(priv-&gt;dd,</div><div class='add'>+			    "%s: Status = 0x%x pbc 0x%llx txq = %d sde = %d\n",</div><div class='add'>+			    __func__, tx-&gt;sdma_status,</div><div class='add'>+			    le64_to_cpu(tx-&gt;sdma_hdr-&gt;pbc), tx-&gt;txq-&gt;q_idx,</div><div class='add'>+			    tx-&gt;txq-&gt;sde-&gt;this_idx);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	napi_consume_skb(tx-&gt;skb, budget);</div><div class='add'>+	tx-&gt;skb = NULL;</div><div class='add'>+	sdma_txclean(priv-&gt;dd, &amp;tx-&gt;txreq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hfi1_ipoib_drain_tx_ring(struct hfi1_ipoib_txq *txq)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_circ_buf *tx_ring = &amp;txq-&gt;tx_ring;</div><div class='add'>+	int i;</div><div class='add'>+	struct ipoib_txreq *tx;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; tx_ring-&gt;max_items; i++) {</div><div class='add'>+		tx = hfi1_txreq_from_idx(tx_ring, i);</div><div class='add'>+		tx-&gt;complete = 0;</div><div class='add'>+		dev_kfree_skb_any(tx-&gt;skb);</div><div class='add'>+		tx-&gt;skb = NULL;</div><div class='add'>+		sdma_txclean(txq-&gt;priv-&gt;dd, &amp;tx-&gt;txreq);</div><div class='add'>+	}</div><div class='add'>+	tx_ring-&gt;head = 0;</div><div class='add'>+	tx_ring-&gt;tail = 0;</div><div class='add'>+	tx_ring-&gt;complete_txreqs = 0;</div><div class='add'>+	tx_ring-&gt;sent_txreqs = 0;</div><div class='add'>+	tx_ring-&gt;avail = hfi1_ipoib_ring_hwat(txq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hfi1_ipoib_poll_tx_ring(struct napi_struct *napi, int budget)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_txq *txq =</div><div class='add'>+		container_of(napi, struct hfi1_ipoib_txq, napi);</div><div class='add'>+	struct hfi1_ipoib_circ_buf *tx_ring = &amp;txq-&gt;tx_ring;</div><div class='add'>+	u32 head = tx_ring-&gt;head;</div><div class='add'>+	u32 max_tx = tx_ring-&gt;max_items;</div><div class='add'>+	int work_done;</div><div class='add'>+	struct ipoib_txreq *tx =  hfi1_txreq_from_idx(tx_ring, head);</div><div class='add'>+</div><div class='add'>+	trace_hfi1_txq_poll(txq);</div><div class='add'>+	for (work_done = 0; work_done &lt; budget; work_done++) {</div><div class='add'>+		/* See hfi1_ipoib_sdma_complete() */</div><div class='add'>+		if (!smp_load_acquire(&amp;tx-&gt;complete))</div><div class='add'>+			break;</div><div class='add'>+		tx-&gt;complete = 0;</div><div class='add'>+		trace_hfi1_tx_produce(tx, head);</div><div class='add'>+		hfi1_ipoib_free_tx(tx, budget);</div><div class='add'>+		head = CIRC_NEXT(head, max_tx);</div><div class='add'>+		tx =  hfi1_txreq_from_idx(tx_ring, head);</div><div class='add'>+	}</div><div class='add'>+	tx_ring-&gt;complete_txreqs += work_done;</div><div class='add'>+</div><div class='add'>+	/* Finished freeing tx items so store the head value. */</div><div class='add'>+	smp_store_release(&amp;tx_ring-&gt;head, head);</div><div class='add'>+</div><div class='add'>+	hfi1_ipoib_check_queue_stopped(txq);</div><div class='add'>+</div><div class='add'>+	if (work_done &lt; budget)</div><div class='add'>+		napi_complete_done(napi, work_done);</div><div class='add'>+</div><div class='add'>+	return work_done;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hfi1_ipoib_sdma_complete(struct sdma_txreq *txreq, int status)</div><div class='add'>+{</div><div class='add'>+	struct ipoib_txreq *tx = container_of(txreq, struct ipoib_txreq, txreq);</div><div class='add'>+</div><div class='add'>+	trace_hfi1_txq_complete(tx-&gt;txq);</div><div class='add'>+	tx-&gt;sdma_status = status;</div><div class='add'>+	/* see hfi1_ipoib_poll_tx_ring */</div><div class='add'>+	smp_store_release(&amp;tx-&gt;complete, 1);</div><div class='add'>+	napi_schedule_irqoff(&amp;tx-&gt;txq-&gt;napi);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hfi1_ipoib_build_ulp_payload(struct ipoib_txreq *tx,</div><div class='add'>+					struct ipoib_txparms *txp)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_devdata *dd = txp-&gt;dd;</div><div class='add'>+	struct sdma_txreq *txreq = &amp;tx-&gt;txreq;</div><div class='add'>+	struct sk_buff *skb = tx-&gt;skb;</div><div class='add'>+	int ret = 0;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (skb_headlen(skb)) {</div><div class='add'>+		ret = sdma_txadd_kvaddr(dd, txreq, skb-&gt;data, skb_headlen(skb));</div><div class='add'>+		if (unlikely(ret))</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; skb_shinfo(skb)-&gt;nr_frags; i++) {</div><div class='add'>+		const skb_frag_t *frag = &amp;skb_shinfo(skb)-&gt;frags[i];</div><div class='add'>+</div><div class='add'>+		ret = sdma_txadd_page(dd,</div><div class='add'>+				      txreq,</div><div class='add'>+				      skb_frag_page(frag),</div><div class='add'>+				      skb_frag_off(frag),</div><div class='add'>+				      skb_frag_size(frag),</div><div class='add'>+				      NULL, NULL, NULL);</div><div class='add'>+		if (unlikely(ret))</div><div class='add'>+			break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hfi1_ipoib_build_tx_desc(struct ipoib_txreq *tx,</div><div class='add'>+				    struct ipoib_txparms *txp)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_devdata *dd = txp-&gt;dd;</div><div class='add'>+	struct sdma_txreq *txreq = &amp;tx-&gt;txreq;</div><div class='add'>+	struct hfi1_sdma_header *sdma_hdr = tx-&gt;sdma_hdr;</div><div class='add'>+	u16 pkt_bytes =</div><div class='add'>+		sizeof(sdma_hdr-&gt;pbc) + (txp-&gt;hdr_dwords &lt;&lt; 2) + tx-&gt;skb-&gt;len;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = sdma_txinit(txreq, 0, pkt_bytes, hfi1_ipoib_sdma_complete);</div><div class='add'>+	if (unlikely(ret))</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	/* add pbc + headers */</div><div class='add'>+	ret = sdma_txadd_kvaddr(dd,</div><div class='add'>+				txreq,</div><div class='add'>+				sdma_hdr,</div><div class='add'>+				sizeof(sdma_hdr-&gt;pbc) + (txp-&gt;hdr_dwords &lt;&lt; 2));</div><div class='add'>+	if (unlikely(ret))</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	/* add the ulp payload */</div><div class='add'>+	return hfi1_ipoib_build_ulp_payload(tx, txp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hfi1_ipoib_build_ib_tx_headers(struct ipoib_txreq *tx,</div><div class='add'>+					   struct ipoib_txparms *txp)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv = tx-&gt;txq-&gt;priv;</div><div class='add'>+	struct hfi1_sdma_header *sdma_hdr = tx-&gt;sdma_hdr;</div><div class='add'>+	struct sk_buff *skb = tx-&gt;skb;</div><div class='add'>+	struct hfi1_pportdata *ppd = ppd_from_ibp(txp-&gt;ibp);</div><div class='add'>+	struct rdma_ah_attr *ah_attr = txp-&gt;ah_attr;</div><div class='add'>+	struct ib_other_headers *ohdr;</div><div class='add'>+	struct ib_grh *grh;</div><div class='add'>+	u16 dwords;</div><div class='add'>+	u16 slid;</div><div class='add'>+	u16 dlid;</div><div class='add'>+	u16 lrh0;</div><div class='add'>+	u32 bth0;</div><div class='add'>+	u32 sqpn = (u32)(priv-&gt;netdev-&gt;dev_addr[1] &lt;&lt; 16 |</div><div class='add'>+			 priv-&gt;netdev-&gt;dev_addr[2] &lt;&lt; 8 |</div><div class='add'>+			 priv-&gt;netdev-&gt;dev_addr[3]);</div><div class='add'>+	u16 payload_dwords;</div><div class='add'>+	u8 pad_cnt;</div><div class='add'>+</div><div class='add'>+	pad_cnt = -skb-&gt;len &amp; 3;</div><div class='add'>+</div><div class='add'>+	/* Includes ICRC */</div><div class='add'>+	payload_dwords = ((skb-&gt;len + pad_cnt) &gt;&gt; 2) + SIZE_OF_CRC;</div><div class='add'>+</div><div class='add'>+	/* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */</div><div class='add'>+	txp-&gt;hdr_dwords = 7;</div><div class='add'>+</div><div class='add'>+	if (rdma_ah_get_ah_flags(ah_attr) &amp; IB_AH_GRH) {</div><div class='add'>+		grh = &amp;sdma_hdr-&gt;hdr.ibh.u.l.grh;</div><div class='add'>+		txp-&gt;hdr_dwords +=</div><div class='add'>+			hfi1_make_grh(txp-&gt;ibp,</div><div class='add'>+				      grh,</div><div class='add'>+				      rdma_ah_read_grh(ah_attr),</div><div class='add'>+				      txp-&gt;hdr_dwords - LRH_9B_DWORDS,</div><div class='add'>+				      payload_dwords);</div><div class='add'>+		lrh0 = HFI1_LRH_GRH;</div><div class='add'>+		ohdr = &amp;sdma_hdr-&gt;hdr.ibh.u.l.oth;</div><div class='add'>+	} else {</div><div class='add'>+		lrh0 = HFI1_LRH_BTH;</div><div class='add'>+		ohdr = &amp;sdma_hdr-&gt;hdr.ibh.u.oth;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	lrh0 |= (rdma_ah_get_sl(ah_attr) &amp; 0xf) &lt;&lt; 4;</div><div class='add'>+	lrh0 |= (txp-&gt;flow.sc5 &amp; 0xf) &lt;&lt; 12;</div><div class='add'>+</div><div class='add'>+	dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 9B);</div><div class='add'>+	if (dlid == be16_to_cpu(IB_LID_PERMISSIVE)) {</div><div class='add'>+		slid = be16_to_cpu(IB_LID_PERMISSIVE);</div><div class='add'>+	} else {</div><div class='add'>+		u16 lid = (u16)ppd-&gt;lid;</div><div class='add'>+</div><div class='add'>+		if (lid) {</div><div class='add'>+			lid |= rdma_ah_get_path_bits(ah_attr) &amp;</div><div class='add'>+				((1 &lt;&lt; ppd-&gt;lmc) - 1);</div><div class='add'>+			slid = lid;</div><div class='add'>+		} else {</div><div class='add'>+			slid = be16_to_cpu(IB_LID_PERMISSIVE);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Includes ICRC */</div><div class='add'>+	dwords = txp-&gt;hdr_dwords + payload_dwords;</div><div class='add'>+</div><div class='add'>+	/* Build the lrh */</div><div class='add'>+	sdma_hdr-&gt;hdr.hdr_type = HFI1_PKT_TYPE_9B;</div><div class='add'>+	hfi1_make_ib_hdr(&amp;sdma_hdr-&gt;hdr.ibh, lrh0, dwords, dlid, slid);</div><div class='add'>+</div><div class='add'>+	/* Build the bth */</div><div class='add'>+	bth0 = (IB_OPCODE_UD_SEND_ONLY &lt;&lt; 24) | (pad_cnt &lt;&lt; 20) | priv-&gt;pkey;</div><div class='add'>+</div><div class='add'>+	ohdr-&gt;bth[0] = cpu_to_be32(bth0);</div><div class='add'>+	ohdr-&gt;bth[1] = cpu_to_be32(txp-&gt;dqpn);</div><div class='add'>+	ohdr-&gt;bth[2] = cpu_to_be32(mask_psn((u32)txp-&gt;txq-&gt;tx_ring.sent_txreqs));</div><div class='add'>+</div><div class='add'>+	/* Build the deth */</div><div class='add'>+	ohdr-&gt;u.ud.deth[0] = cpu_to_be32(priv-&gt;qkey);</div><div class='add'>+	ohdr-&gt;u.ud.deth[1] = cpu_to_be32((txp-&gt;entropy &lt;&lt;</div><div class='add'>+					  HFI1_IPOIB_ENTROPY_SHIFT) | sqpn);</div><div class='add'>+</div><div class='add'>+	/* Construct the pbc. */</div><div class='add'>+	sdma_hdr-&gt;pbc =</div><div class='add'>+		cpu_to_le64(create_pbc(ppd,</div><div class='add'>+				       ib_is_sc5(txp-&gt;flow.sc5) &lt;&lt;</div><div class='add'>+							      PBC_DC_INFO_SHIFT,</div><div class='add'>+				       0,</div><div class='add'>+				       sc_to_vlt(priv-&gt;dd, txp-&gt;flow.sc5),</div><div class='add'>+				       dwords - SIZE_OF_CRC +</div><div class='add'>+						(sizeof(sdma_hdr-&gt;pbc) &gt;&gt; 2)));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct ipoib_txreq *hfi1_ipoib_send_dma_common(struct net_device *dev,</div><div class='add'>+						      struct sk_buff *skb,</div><div class='add'>+						      struct ipoib_txparms *txp)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);</div><div class='add'>+	struct hfi1_ipoib_txq *txq = txp-&gt;txq;</div><div class='add'>+	struct ipoib_txreq *tx;</div><div class='add'>+	struct hfi1_ipoib_circ_buf *tx_ring = &amp;txq-&gt;tx_ring;</div><div class='add'>+	u32 tail = tx_ring-&gt;tail;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (unlikely(!tx_ring-&gt;avail)) {</div><div class='add'>+		u32 head;</div><div class='add'>+</div><div class='add'>+		if (hfi1_ipoib_used(txq) &gt;= hfi1_ipoib_ring_hwat(txq))</div><div class='add'>+			/* This shouldn't happen with a stopped queue */</div><div class='add'>+			return ERR_PTR(-ENOMEM);</div><div class='add'>+		/* See hfi1_ipoib_poll_tx_ring() */</div><div class='add'>+		head = smp_load_acquire(&amp;tx_ring-&gt;head);</div><div class='add'>+		tx_ring-&gt;avail =</div><div class='add'>+			min_t(u32, hfi1_ipoib_ring_hwat(txq),</div><div class='add'>+			      CIRC_CNT(head, tail, tx_ring-&gt;max_items));</div><div class='add'>+	} else {</div><div class='add'>+		tx_ring-&gt;avail--;</div><div class='add'>+	}</div><div class='add'>+	tx = hfi1_txreq_from_idx(tx_ring, tail);</div><div class='add'>+	trace_hfi1_txq_alloc_tx(txq);</div><div class='add'>+</div><div class='add'>+	/* so that we can test if the sdma descriptors are there */</div><div class='add'>+	tx-&gt;txreq.num_desc = 0;</div><div class='add'>+	tx-&gt;txq = txq;</div><div class='add'>+	tx-&gt;skb = skb;</div><div class='add'>+	INIT_LIST_HEAD(&amp;tx-&gt;txreq.list);</div><div class='add'>+</div><div class='add'>+	hfi1_ipoib_build_ib_tx_headers(tx, txp);</div><div class='add'>+</div><div class='add'>+	ret = hfi1_ipoib_build_tx_desc(tx, txp);</div><div class='add'>+	if (likely(!ret)) {</div><div class='add'>+		if (txq-&gt;flow.as_int != txp-&gt;flow.as_int) {</div><div class='add'>+			txq-&gt;flow.tx_queue = txp-&gt;flow.tx_queue;</div><div class='add'>+			txq-&gt;flow.sc5 = txp-&gt;flow.sc5;</div><div class='add'>+			txq-&gt;sde =</div><div class='add'>+				sdma_select_engine_sc(priv-&gt;dd,</div><div class='add'>+						      txp-&gt;flow.tx_queue,</div><div class='add'>+						      txp-&gt;flow.sc5);</div><div class='add'>+			trace_hfi1_flow_switch(txq);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		return tx;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	sdma_txclean(priv-&gt;dd, &amp;tx-&gt;txreq);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(ret);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hfi1_ipoib_submit_tx_list(struct net_device *dev,</div><div class='add'>+				     struct hfi1_ipoib_txq *txq)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+	u16 count_out;</div><div class='add'>+</div><div class='add'>+	ret = sdma_send_txlist(txq-&gt;sde,</div><div class='add'>+			       iowait_get_ib_work(&amp;txq-&gt;wait),</div><div class='add'>+			       &amp;txq-&gt;tx_list,</div><div class='add'>+			       &amp;count_out);</div><div class='add'>+	if (likely(!ret) || ret == -EBUSY || ret == -ECOMM)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	dd_dev_warn(txq-&gt;priv-&gt;dd, "cannot send skb tx list, err %d.\n", ret);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hfi1_ipoib_flush_tx_list(struct net_device *dev,</div><div class='add'>+				    struct hfi1_ipoib_txq *txq)</div><div class='add'>+{</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	if (!list_empty(&amp;txq-&gt;tx_list)) {</div><div class='add'>+		/* Flush the current list */</div><div class='add'>+		ret = hfi1_ipoib_submit_tx_list(dev, txq);</div><div class='add'>+</div><div class='add'>+		if (unlikely(ret))</div><div class='add'>+			if (ret != -EBUSY)</div><div class='add'>+				++dev-&gt;stats.tx_carrier_errors;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hfi1_ipoib_submit_tx(struct hfi1_ipoib_txq *txq,</div><div class='add'>+				struct ipoib_txreq *tx)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = sdma_send_txreq(txq-&gt;sde,</div><div class='add'>+			      iowait_get_ib_work(&amp;txq-&gt;wait),</div><div class='add'>+			      &amp;tx-&gt;txreq,</div><div class='add'>+			      txq-&gt;pkts_sent);</div><div class='add'>+	if (likely(!ret)) {</div><div class='add'>+		txq-&gt;pkts_sent = true;</div><div class='add'>+		iowait_starve_clear(txq-&gt;pkts_sent, &amp;txq-&gt;wait);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hfi1_ipoib_send_dma_single(struct net_device *dev,</div><div class='add'>+				      struct sk_buff *skb,</div><div class='add'>+				      struct ipoib_txparms *txp)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_txq *txq = txp-&gt;txq;</div><div class='add'>+	struct hfi1_ipoib_circ_buf *tx_ring;</div><div class='add'>+	struct ipoib_txreq *tx;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	tx = hfi1_ipoib_send_dma_common(dev, skb, txp);</div><div class='add'>+	if (IS_ERR(tx)) {</div><div class='add'>+		int ret = PTR_ERR(tx);</div><div class='add'>+</div><div class='add'>+		dev_kfree_skb_any(skb);</div><div class='add'>+</div><div class='add'>+		if (ret == -ENOMEM)</div><div class='add'>+			++dev-&gt;stats.tx_errors;</div><div class='add'>+		else</div><div class='add'>+			++dev-&gt;stats.tx_carrier_errors;</div><div class='add'>+</div><div class='add'>+		return NETDEV_TX_OK;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	tx_ring = &amp;txq-&gt;tx_ring;</div><div class='add'>+	trace_hfi1_tx_consume(tx, tx_ring-&gt;tail);</div><div class='add'>+	/* consume tx */</div><div class='add'>+	smp_store_release(&amp;tx_ring-&gt;tail, CIRC_NEXT(tx_ring-&gt;tail, tx_ring-&gt;max_items));</div><div class='add'>+	ret = hfi1_ipoib_submit_tx(txq, tx);</div><div class='add'>+	if (likely(!ret)) {</div><div class='add'>+tx_ok:</div><div class='add'>+		trace_sdma_output_ibhdr(txq-&gt;priv-&gt;dd,</div><div class='add'>+					&amp;tx-&gt;sdma_hdr-&gt;hdr,</div><div class='add'>+					ib_is_sc5(txp-&gt;flow.sc5));</div><div class='add'>+		hfi1_ipoib_check_queue_depth(txq);</div><div class='add'>+		return NETDEV_TX_OK;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	txq-&gt;pkts_sent = false;</div><div class='add'>+</div><div class='add'>+	if (ret == -EBUSY || ret == -ECOMM)</div><div class='add'>+		goto tx_ok;</div><div class='add'>+</div><div class='add'>+	/* mark complete and kick napi tx */</div><div class='add'>+	smp_store_release(&amp;tx-&gt;complete, 1);</div><div class='add'>+	napi_schedule(&amp;tx-&gt;txq-&gt;napi);</div><div class='add'>+</div><div class='add'>+	++dev-&gt;stats.tx_carrier_errors;</div><div class='add'>+</div><div class='add'>+	return NETDEV_TX_OK;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hfi1_ipoib_send_dma_list(struct net_device *dev,</div><div class='add'>+				    struct sk_buff *skb,</div><div class='add'>+				    struct ipoib_txparms *txp)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_txq *txq = txp-&gt;txq;</div><div class='add'>+	struct hfi1_ipoib_circ_buf *tx_ring;</div><div class='add'>+	struct ipoib_txreq *tx;</div><div class='add'>+</div><div class='add'>+	/* Has the flow change ? */</div><div class='add'>+	if (txq-&gt;flow.as_int != txp-&gt;flow.as_int) {</div><div class='add'>+		int ret;</div><div class='add'>+</div><div class='add'>+		trace_hfi1_flow_flush(txq);</div><div class='add'>+		ret = hfi1_ipoib_flush_tx_list(dev, txq);</div><div class='add'>+		if (unlikely(ret)) {</div><div class='add'>+			if (ret == -EBUSY)</div><div class='add'>+				++dev-&gt;stats.tx_dropped;</div><div class='add'>+			dev_kfree_skb_any(skb);</div><div class='add'>+			return NETDEV_TX_OK;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	tx = hfi1_ipoib_send_dma_common(dev, skb, txp);</div><div class='add'>+	if (IS_ERR(tx)) {</div><div class='add'>+		int ret = PTR_ERR(tx);</div><div class='add'>+</div><div class='add'>+		dev_kfree_skb_any(skb);</div><div class='add'>+</div><div class='add'>+		if (ret == -ENOMEM)</div><div class='add'>+			++dev-&gt;stats.tx_errors;</div><div class='add'>+		else</div><div class='add'>+			++dev-&gt;stats.tx_carrier_errors;</div><div class='add'>+</div><div class='add'>+		return NETDEV_TX_OK;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	tx_ring = &amp;txq-&gt;tx_ring;</div><div class='add'>+	trace_hfi1_tx_consume(tx, tx_ring-&gt;tail);</div><div class='add'>+	/* consume tx */</div><div class='add'>+	smp_store_release(&amp;tx_ring-&gt;tail, CIRC_NEXT(tx_ring-&gt;tail, tx_ring-&gt;max_items));</div><div class='add'>+	list_add_tail(&amp;tx-&gt;txreq.list, &amp;txq-&gt;tx_list);</div><div class='add'>+</div><div class='add'>+	hfi1_ipoib_check_queue_depth(txq);</div><div class='add'>+</div><div class='add'>+	trace_sdma_output_ibhdr(txq-&gt;priv-&gt;dd,</div><div class='add'>+				&amp;tx-&gt;sdma_hdr-&gt;hdr,</div><div class='add'>+				ib_is_sc5(txp-&gt;flow.sc5));</div><div class='add'>+</div><div class='add'>+	if (!netdev_xmit_more())</div><div class='add'>+		(void)hfi1_ipoib_flush_tx_list(dev, txq);</div><div class='add'>+</div><div class='add'>+	return NETDEV_TX_OK;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u8 hfi1_ipoib_calc_entropy(struct sk_buff *skb)</div><div class='add'>+{</div><div class='add'>+	if (skb_transport_header_was_set(skb)) {</div><div class='add'>+		u8 *hdr = (u8 *)skb_transport_header(skb);</div><div class='add'>+</div><div class='add'>+		return (hdr[0] ^ hdr[1] ^ hdr[2] ^ hdr[3]);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return (u8)skb_get_queue_mapping(skb);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hfi1_ipoib_send(struct net_device *dev,</div><div class='add'>+		    struct sk_buff *skb,</div><div class='add'>+		    struct ib_ah *address,</div><div class='add'>+		    u32 dqpn)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);</div><div class='add'>+	struct ipoib_txparms txp;</div><div class='add'>+	struct rdma_netdev *rn = netdev_priv(dev);</div><div class='add'>+</div><div class='add'>+	if (unlikely(skb-&gt;len &gt; rn-&gt;mtu + HFI1_IPOIB_ENCAP_LEN)) {</div><div class='add'>+		dd_dev_warn(priv-&gt;dd, "packet len %d (&gt; %d) too long to send, dropping\n",</div><div class='add'>+			    skb-&gt;len,</div><div class='add'>+			    rn-&gt;mtu + HFI1_IPOIB_ENCAP_LEN);</div><div class='add'>+		++dev-&gt;stats.tx_dropped;</div><div class='add'>+		++dev-&gt;stats.tx_errors;</div><div class='add'>+		dev_kfree_skb_any(skb);</div><div class='add'>+		return NETDEV_TX_OK;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	txp.dd = priv-&gt;dd;</div><div class='add'>+	txp.ah_attr = &amp;ibah_to_rvtah(address)-&gt;attr;</div><div class='add'>+	txp.ibp = to_iport(priv-&gt;device, priv-&gt;port_num);</div><div class='add'>+	txp.txq = &amp;priv-&gt;txqs[skb_get_queue_mapping(skb)];</div><div class='add'>+	txp.dqpn = dqpn;</div><div class='add'>+	txp.flow.sc5 = txp.ibp-&gt;sl_to_sc[rdma_ah_get_sl(txp.ah_attr)];</div><div class='add'>+	txp.flow.tx_queue = (u8)skb_get_queue_mapping(skb);</div><div class='add'>+	txp.entropy = hfi1_ipoib_calc_entropy(skb);</div><div class='add'>+</div><div class='add'>+	if (netdev_xmit_more() || !list_empty(&amp;txp.txq-&gt;tx_list))</div><div class='add'>+		return hfi1_ipoib_send_dma_list(dev, skb, &amp;txp);</div><div class='add'>+</div><div class='add'>+	return hfi1_ipoib_send_dma_single(dev, skb,  &amp;txp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * hfi1_ipoib_sdma_sleep - ipoib sdma sleep function</div><div class='add'>+ *</div><div class='add'>+ * This function gets called from sdma_send_txreq() when there are not enough</div><div class='add'>+ * sdma descriptors available to send the packet. It adds Tx queue's wait</div><div class='add'>+ * structure to sdma engine's dmawait list to be woken up when descriptors</div><div class='add'>+ * become available.</div><div class='add'>+ */</div><div class='add'>+static int hfi1_ipoib_sdma_sleep(struct sdma_engine *sde,</div><div class='add'>+				 struct iowait_work *wait,</div><div class='add'>+				 struct sdma_txreq *txreq,</div><div class='add'>+				 uint seq,</div><div class='add'>+				 bool pkts_sent)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_txq *txq =</div><div class='add'>+		container_of(wait-&gt;iow, struct hfi1_ipoib_txq, wait);</div><div class='add'>+</div><div class='add'>+	write_seqlock(&amp;sde-&gt;waitlock);</div><div class='add'>+</div><div class='add'>+	if (likely(txq-&gt;priv-&gt;netdev-&gt;reg_state == NETREG_REGISTERED)) {</div><div class='add'>+		if (sdma_progress(sde, seq, txreq)) {</div><div class='add'>+			write_sequnlock(&amp;sde-&gt;waitlock);</div><div class='add'>+			return -EAGAIN;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (list_empty(&amp;txreq-&gt;list))</div><div class='add'>+			/* came from non-list submit */</div><div class='add'>+			list_add_tail(&amp;txreq-&gt;list, &amp;txq-&gt;tx_list);</div><div class='add'>+		if (list_empty(&amp;txq-&gt;wait.list)) {</div><div class='add'>+			struct hfi1_ibport *ibp = &amp;sde-&gt;ppd-&gt;ibport_data;</div><div class='add'>+</div><div class='add'>+			if (!atomic_xchg(&amp;txq-&gt;tx_ring.no_desc, 1)) {</div><div class='add'>+				trace_hfi1_txq_queued(txq);</div><div class='add'>+				hfi1_ipoib_stop_txq(txq);</div><div class='add'>+			}</div><div class='add'>+			ibp-&gt;rvp.n_dmawait++;</div><div class='add'>+			iowait_queue(pkts_sent, wait-&gt;iow, &amp;sde-&gt;dmawait);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		write_sequnlock(&amp;sde-&gt;waitlock);</div><div class='add'>+		return -EBUSY;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	write_sequnlock(&amp;sde-&gt;waitlock);</div><div class='add'>+	return -EINVAL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * hfi1_ipoib_sdma_wakeup - ipoib sdma wakeup function</div><div class='add'>+ *</div><div class='add'>+ * This function gets called when SDMA descriptors becomes available and Tx</div><div class='add'>+ * queue's wait structure was previously added to sdma engine's dmawait list.</div><div class='add'>+ */</div><div class='add'>+static void hfi1_ipoib_sdma_wakeup(struct iowait *wait, int reason)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_txq *txq =</div><div class='add'>+		container_of(wait, struct hfi1_ipoib_txq, wait);</div><div class='add'>+</div><div class='add'>+	trace_hfi1_txq_wakeup(txq);</div><div class='add'>+	if (likely(txq-&gt;priv-&gt;netdev-&gt;reg_state == NETREG_REGISTERED))</div><div class='add'>+		iowait_schedule(wait, system_highpri_wq, WORK_CPU_UNBOUND);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hfi1_ipoib_flush_txq(struct work_struct *work)</div><div class='add'>+{</div><div class='add'>+	struct iowait_work *ioww =</div><div class='add'>+		container_of(work, struct iowait_work, iowork);</div><div class='add'>+	struct iowait *wait = iowait_ioww_to_iow(ioww);</div><div class='add'>+	struct hfi1_ipoib_txq *txq =</div><div class='add'>+		container_of(wait, struct hfi1_ipoib_txq, wait);</div><div class='add'>+	struct net_device *dev = txq-&gt;priv-&gt;netdev;</div><div class='add'>+</div><div class='add'>+	if (likely(dev-&gt;reg_state == NETREG_REGISTERED) &amp;&amp;</div><div class='add'>+	    likely(!hfi1_ipoib_flush_tx_list(dev, txq)))</div><div class='add'>+		if (atomic_xchg(&amp;txq-&gt;tx_ring.no_desc, 0))</div><div class='add'>+			hfi1_ipoib_wake_txq(txq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv)</div><div class='add'>+{</div><div class='add'>+	struct net_device *dev = priv-&gt;netdev;</div><div class='add'>+	u32 tx_ring_size, tx_item_size;</div><div class='add'>+	struct hfi1_ipoib_circ_buf *tx_ring;</div><div class='add'>+	int i, j;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Ring holds 1 less than tx_ring_size</div><div class='add'>+	 * Round up to next power of 2 in order to hold at least tx_queue_len</div><div class='add'>+	 */</div><div class='add'>+	tx_ring_size = roundup_pow_of_two(dev-&gt;tx_queue_len + 1);</div><div class='add'>+	tx_item_size = roundup_pow_of_two(sizeof(struct ipoib_txreq));</div><div class='add'>+</div><div class='add'>+	priv-&gt;txqs = kcalloc_node(dev-&gt;num_tx_queues,</div><div class='add'>+				  sizeof(struct hfi1_ipoib_txq),</div><div class='add'>+				  GFP_KERNEL,</div><div class='add'>+				  priv-&gt;dd-&gt;node);</div><div class='add'>+	if (!priv-&gt;txqs)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; dev-&gt;num_tx_queues; i++) {</div><div class='add'>+		struct hfi1_ipoib_txq *txq = &amp;priv-&gt;txqs[i];</div><div class='add'>+		struct ipoib_txreq *tx;</div><div class='add'>+</div><div class='add'>+		tx_ring = &amp;txq-&gt;tx_ring;</div><div class='add'>+		iowait_init(&amp;txq-&gt;wait,</div><div class='add'>+			    0,</div><div class='add'>+			    hfi1_ipoib_flush_txq,</div><div class='add'>+			    NULL,</div><div class='add'>+			    hfi1_ipoib_sdma_sleep,</div><div class='add'>+			    hfi1_ipoib_sdma_wakeup,</div><div class='add'>+			    NULL,</div><div class='add'>+			    NULL);</div><div class='add'>+		txq-&gt;priv = priv;</div><div class='add'>+		txq-&gt;sde = NULL;</div><div class='add'>+		INIT_LIST_HEAD(&amp;txq-&gt;tx_list);</div><div class='add'>+		atomic_set(&amp;txq-&gt;tx_ring.stops, 0);</div><div class='add'>+		atomic_set(&amp;txq-&gt;tx_ring.ring_full, 0);</div><div class='add'>+		atomic_set(&amp;txq-&gt;tx_ring.no_desc, 0);</div><div class='add'>+		txq-&gt;q_idx = i;</div><div class='add'>+		txq-&gt;flow.tx_queue = 0xff;</div><div class='add'>+		txq-&gt;flow.sc5 = 0xff;</div><div class='add'>+		txq-&gt;pkts_sent = false;</div><div class='add'>+</div><div class='add'>+		netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),</div><div class='add'>+					     priv-&gt;dd-&gt;node);</div><div class='add'>+</div><div class='add'>+		txq-&gt;tx_ring.items =</div><div class='add'>+			kvzalloc_node(array_size(tx_ring_size, tx_item_size),</div><div class='add'>+				      GFP_KERNEL, priv-&gt;dd-&gt;node);</div><div class='add'>+		if (!txq-&gt;tx_ring.items)</div><div class='add'>+			goto free_txqs;</div><div class='add'>+</div><div class='add'>+		txq-&gt;tx_ring.max_items = tx_ring_size;</div><div class='add'>+		txq-&gt;tx_ring.shift = ilog2(tx_item_size);</div><div class='add'>+		txq-&gt;tx_ring.avail = hfi1_ipoib_ring_hwat(txq);</div><div class='add'>+		tx_ring = &amp;txq-&gt;tx_ring;</div><div class='add'>+		for (j = 0; j &lt; tx_ring_size; j++) {</div><div class='add'>+			hfi1_txreq_from_idx(tx_ring, j)-&gt;sdma_hdr =</div><div class='add'>+				kzalloc_node(sizeof(*tx-&gt;sdma_hdr),</div><div class='add'>+					     GFP_KERNEL, priv-&gt;dd-&gt;node);</div><div class='add'>+			if (!hfi1_txreq_from_idx(tx_ring, j)-&gt;sdma_hdr)</div><div class='add'>+				goto free_txqs;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		netif_napi_add_tx(dev, &amp;txq-&gt;napi, hfi1_ipoib_poll_tx_ring);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+free_txqs:</div><div class='add'>+	for (i--; i &gt;= 0; i--) {</div><div class='add'>+		struct hfi1_ipoib_txq *txq = &amp;priv-&gt;txqs[i];</div><div class='add'>+</div><div class='add'>+		netif_napi_del(&amp;txq-&gt;napi);</div><div class='add'>+		tx_ring = &amp;txq-&gt;tx_ring;</div><div class='add'>+		for (j = 0; j &lt; tx_ring_size; j++)</div><div class='add'>+			kfree(hfi1_txreq_from_idx(tx_ring, j)-&gt;sdma_hdr);</div><div class='add'>+		kvfree(tx_ring-&gt;items);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	kfree(priv-&gt;txqs);</div><div class='add'>+	priv-&gt;txqs = NULL;</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hfi1_ipoib_drain_tx_list(struct hfi1_ipoib_txq *txq)</div><div class='add'>+{</div><div class='add'>+	struct sdma_txreq *txreq;</div><div class='add'>+	struct sdma_txreq *txreq_tmp;</div><div class='add'>+</div><div class='add'>+	list_for_each_entry_safe(txreq, txreq_tmp, &amp;txq-&gt;tx_list, list) {</div><div class='add'>+		struct ipoib_txreq *tx =</div><div class='add'>+			container_of(txreq, struct ipoib_txreq, txreq);</div><div class='add'>+</div><div class='add'>+		list_del(&amp;txreq-&gt;list);</div><div class='add'>+		sdma_txclean(txq-&gt;priv-&gt;dd, &amp;tx-&gt;txreq);</div><div class='add'>+		dev_kfree_skb_any(tx-&gt;skb);</div><div class='add'>+		tx-&gt;skb = NULL;</div><div class='add'>+		txq-&gt;tx_ring.complete_txreqs++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (hfi1_ipoib_used(txq))</div><div class='add'>+		dd_dev_warn(txq-&gt;priv-&gt;dd,</div><div class='add'>+			    "txq %d not empty found %u requests\n",</div><div class='add'>+			    txq-&gt;q_idx,</div><div class='add'>+			    hfi1_ipoib_txreqs(txq-&gt;tx_ring.sent_txreqs,</div><div class='add'>+					      txq-&gt;tx_ring.complete_txreqs));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv)</div><div class='add'>+{</div><div class='add'>+	int i, j;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; priv-&gt;netdev-&gt;num_tx_queues; i++) {</div><div class='add'>+		struct hfi1_ipoib_txq *txq = &amp;priv-&gt;txqs[i];</div><div class='add'>+		struct hfi1_ipoib_circ_buf *tx_ring = &amp;txq-&gt;tx_ring;</div><div class='add'>+</div><div class='add'>+		iowait_cancel_work(&amp;txq-&gt;wait);</div><div class='add'>+		iowait_sdma_drain(&amp;txq-&gt;wait);</div><div class='add'>+		hfi1_ipoib_drain_tx_list(txq);</div><div class='add'>+		netif_napi_del(&amp;txq-&gt;napi);</div><div class='add'>+		hfi1_ipoib_drain_tx_ring(txq);</div><div class='add'>+		for (j = 0; j &lt; tx_ring-&gt;max_items; j++)</div><div class='add'>+			kfree(hfi1_txreq_from_idx(tx_ring, j)-&gt;sdma_hdr);</div><div class='add'>+		kvfree(tx_ring-&gt;items);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	kfree(priv-&gt;txqs);</div><div class='add'>+	priv-&gt;txqs = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hfi1_ipoib_napi_tx_enable(struct net_device *dev)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; dev-&gt;num_tx_queues; i++) {</div><div class='add'>+		struct hfi1_ipoib_txq *txq = &amp;priv-&gt;txqs[i];</div><div class='add'>+</div><div class='add'>+		napi_enable(&amp;txq-&gt;napi);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hfi1_ipoib_napi_tx_disable(struct net_device *dev)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; dev-&gt;num_tx_queues; i++) {</div><div class='add'>+		struct hfi1_ipoib_txq *txq = &amp;priv-&gt;txqs[i];</div><div class='add'>+</div><div class='add'>+		napi_disable(&amp;txq-&gt;napi);</div><div class='add'>+		hfi1_ipoib_drain_tx_ring(txq);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hfi1_ipoib_tx_timeout(struct net_device *dev, unsigned int q)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);</div><div class='add'>+	struct hfi1_ipoib_txq *txq = &amp;priv-&gt;txqs[q];</div><div class='add'>+</div><div class='add'>+	dd_dev_info(priv-&gt;dd, "timeout txq %p q %u stopped %u stops %d no_desc %d ring_full %d\n",</div><div class='add'>+		    txq, q,</div><div class='add'>+		    __netif_subqueue_stopped(dev, txq-&gt;q_idx),</div><div class='add'>+		    atomic_read(&amp;txq-&gt;tx_ring.stops),</div><div class='add'>+		    atomic_read(&amp;txq-&gt;tx_ring.no_desc),</div><div class='add'>+		    atomic_read(&amp;txq-&gt;tx_ring.ring_full));</div><div class='add'>+	dd_dev_info(priv-&gt;dd, "sde %p engine %u\n",</div><div class='add'>+		    txq-&gt;sde,</div><div class='add'>+		    txq-&gt;sde ? txq-&gt;sde-&gt;this_idx : 0);</div><div class='add'>+	dd_dev_info(priv-&gt;dd, "flow %x\n", txq-&gt;flow.as_int);</div><div class='add'>+	dd_dev_info(priv-&gt;dd, "sent %llu completed %llu used %llu\n",</div><div class='add'>+		    txq-&gt;tx_ring.sent_txreqs, txq-&gt;tx_ring.complete_txreqs,</div><div class='add'>+		    hfi1_ipoib_used(txq));</div><div class='add'>+	dd_dev_info(priv-&gt;dd, "tx_queue_len %u max_items %u\n",</div><div class='add'>+		    dev-&gt;tx_queue_len, txq-&gt;tx_ring.max_items);</div><div class='add'>+	dd_dev_info(priv-&gt;dd, "head %u tail %u\n",</div><div class='add'>+		    txq-&gt;tx_ring.head, txq-&gt;tx_ring.tail);</div><div class='add'>+	dd_dev_info(priv-&gt;dd, "wait queued %u\n",</div><div class='add'>+		    !list_empty(&amp;txq-&gt;wait.list));</div><div class='add'>+	dd_dev_info(priv-&gt;dd, "tx_list empty %u\n",</div><div class='add'>+		    list_empty(&amp;txq-&gt;tx_list));</div><div class='add'>+}</div><div class='add'>+</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c<br/>index 4228393e6c4c..961fa07116f0 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/mad.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/mad.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/mad.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/mad.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015-2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/net.h&gt;</div><div class='hunk'>@@ -108,7 +66,7 @@ static u16 hfi1_lookup_pkey_value(struct hfi1_ibport *ibp, int pkey_idx)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void hfi1_event_pkey_change(struct hfi1_devdata *dd, u8 port)</div><div class='add'>+void hfi1_event_pkey_change(struct hfi1_devdata *dd, u32 port)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_event event;</div><div class='ctx'> </div><div class='hunk'>@@ -297,7 +255,7 @@ static struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u32 dlid)</div><div class='ctx'> 	struct rvt_qp *qp0;</div><div class='ctx'> 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_ppd(ppd);</div><div class='del'>-	u8 port_num = ppd-&gt;port;</div><div class='add'>+	u32 port_num = ppd-&gt;port;</div><div class='ctx'> </div><div class='ctx'> 	memset(&amp;attr, 0, sizeof(attr));</div><div class='ctx'> 	attr.type = rdma_ah_find_type(&amp;dd-&gt;verbs_dev.rdi.ibdev, port_num);</div><div class='hunk'>@@ -411,7 +369,7 @@ static void send_trap(struct hfi1_ibport *ibp, struct trap_node *trap)</div><div class='ctx'> </div><div class='ctx'> void hfi1_handle_trap_timer(struct timer_list *t)</div><div class='ctx'> {</div><div class='del'>-	struct hfi1_ibport *ibp = from_timer(ibp, t, rvp.trap_timer);</div><div class='add'>+	struct hfi1_ibport *ibp = timer_container_of(ibp, t, rvp.trap_timer);</div><div class='ctx'> 	struct trap_node *trap = NULL;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int i;</div><div class='hunk'>@@ -515,7 +473,7 @@ static void bad_mkey(struct hfi1_ibport *ibp, struct ib_mad_hdr *mad,</div><div class='ctx'> /*</div><div class='ctx'>  * Send a Port Capability Mask Changed trap (ch. 14.3.11).</div><div class='ctx'>  */</div><div class='del'>-void hfi1_cap_mask_chg(struct rvt_dev_info *rdi, u8 port_num)</div><div class='add'>+void hfi1_cap_mask_chg(struct rvt_dev_info *rdi, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct trap_node *trap;</div><div class='ctx'> 	struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi);</div><div class='hunk'>@@ -581,7 +539,7 @@ void hfi1_node_desc_chg(struct hfi1_ibport *ibp)</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_nodedesc(struct opa_smp *smp, u32 am,</div><div class='ctx'> 				   u8 *data, struct ib_device *ibdev,</div><div class='del'>-				   u8 port, u32 *resp_len, u32 max_len)</div><div class='add'>+				   u32 port, u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct opa_node_description *nd;</div><div class='ctx'> </div><div class='hunk'>@@ -601,12 +559,12 @@ static int __subn_get_opa_nodedesc(struct opa_smp *smp, u32 am,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_nodeinfo(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				   struct ib_device *ibdev, u8 port,</div><div class='add'>+				   struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				   u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct opa_node_info *ni;</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='del'>-	unsigned pidx = port - 1; /* IB number port from 1, hw from 0 */</div><div class='add'>+	u32 pidx = port - 1; /* IB number port from 1, hw from 0 */</div><div class='ctx'> </div><div class='ctx'> 	ni = (struct opa_node_info *)data;</div><div class='ctx'> </div><div class='hunk'>@@ -641,11 +599,11 @@ static int __subn_get_opa_nodeinfo(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int subn_get_nodeinfo(struct ib_smp *smp, struct ib_device *ibdev,</div><div class='del'>-			     u8 port)</div><div class='add'>+			     u32 port)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_node_info *nip = (struct ib_node_info *)&amp;smp-&gt;data;</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='del'>-	unsigned pidx = port - 1; /* IB number port from 1, hw from 0 */</div><div class='add'>+	u32 pidx = port - 1; /* IB number port from 1, hw from 0 */</div><div class='ctx'> </div><div class='ctx'> 	/* GUID 0 is illegal */</div><div class='ctx'> 	if (smp-&gt;attr_mod || pidx &gt;= dd-&gt;num_pports ||</div><div class='hunk'>@@ -721,7 +679,7 @@ static int check_mkey(struct hfi1_ibport *ibp, struct ib_mad_hdr *mad,</div><div class='ctx'> 			/* Bad mkey not a violation below level 2 */</div><div class='ctx'> 			if (ibp-&gt;rvp.mkeyprot &lt; 2)</div><div class='ctx'> 				break;</div><div class='del'>-			/* fall through */</div><div class='add'>+			fallthrough;</div><div class='ctx'> 		case IB_MGMT_METHOD_SET:</div><div class='ctx'> 		case IB_MGMT_METHOD_TRAP_REPRESS:</div><div class='ctx'> 			if (ibp-&gt;rvp.mkey_violations != 0xFFFF)</div><div class='hunk'>@@ -794,7 +752,7 @@ void read_ltp_rtt(struct hfi1_devdata *dd)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				   struct ib_device *ibdev, u8 port,</div><div class='add'>+				   struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				   u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='hunk'>@@ -1009,7 +967,7 @@ static int __subn_get_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'>  * @port: the IB port number</div><div class='ctx'>  * @pkeys: the pkey table is placed here</div><div class='ctx'>  */</div><div class='del'>-static int get_pkeys(struct hfi1_devdata *dd, u8 port, u16 *pkeys)</div><div class='add'>+static int get_pkeys(struct hfi1_devdata *dd, u32 port, u16 *pkeys)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_pportdata *ppd = dd-&gt;pport + port - 1;</div><div class='ctx'> </div><div class='hunk'>@@ -1019,7 +977,7 @@ static int get_pkeys(struct hfi1_devdata *dd, u8 port, u16 *pkeys)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_pkeytable(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				    struct ib_device *ibdev, u8 port,</div><div class='add'>+				    struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				    u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='hunk'>@@ -1202,8 +1160,8 @@ static int port_states_transition_allowed(struct hfi1_pportdata *ppd,</div><div class='ctx'> 	if (ret == HFI_TRANSITION_DISALLOWED ||</div><div class='ctx'> 	    ret == HFI_TRANSITION_UNDEFINED) {</div><div class='ctx'> 		pr_warn("invalid logical state transition %s -&gt; %s\n",</div><div class='del'>-			opa_lstate_name(logical_old),</div><div class='del'>-			opa_lstate_name(logical_new));</div><div class='add'>+			ib_port_state_to_str(logical_old),</div><div class='add'>+			ib_port_state_to_str(logical_new));</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -1272,7 +1230,7 @@ static int set_port_states(struct hfi1_pportdata *ppd, struct opa_smp *smp,</div><div class='ctx'> 	case IB_PORT_NOP:</div><div class='ctx'> 		if (phys_state == IB_PORTPHYSSTATE_NOP)</div><div class='ctx'> 			break;</div><div class='del'>-		/* FALLTHROUGH */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case IB_PORT_DOWN:</div><div class='ctx'> 		if (phys_state == IB_PORTPHYSSTATE_NOP) {</div><div class='ctx'> 			link_state = HLS_DN_DOWNDEF;</div><div class='hunk'>@@ -1341,7 +1299,7 @@ static int set_port_states(struct hfi1_pportdata *ppd, struct opa_smp *smp,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/**</div><div class='add'>+/*</div><div class='ctx'>  * subn_set_opa_portinfo - set port information</div><div class='ctx'>  * @smp: the incoming SM packet</div><div class='ctx'>  * @ibdev: the infiniband device</div><div class='hunk'>@@ -1349,7 +1307,7 @@ static int set_port_states(struct hfi1_pportdata *ppd, struct opa_smp *smp,</div><div class='ctx'>  *</div><div class='ctx'>  */</div><div class='ctx'> static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				   struct ib_device *ibdev, u8 port,</div><div class='add'>+				   struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				   u32 *resp_len, u32 max_len, int local_mad)</div><div class='ctx'> {</div><div class='ctx'> 	struct opa_port_info *pi = (struct opa_port_info *)data;</div><div class='hunk'>@@ -1667,7 +1625,7 @@ get_only:</div><div class='ctx'>  * @port: the IB port number</div><div class='ctx'>  * @pkeys: the PKEY table</div><div class='ctx'>  */</div><div class='del'>-static int set_pkeys(struct hfi1_devdata *dd, u8 port, u16 *pkeys)</div><div class='add'>+static int set_pkeys(struct hfi1_devdata *dd, u32 port, u16 *pkeys)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_pportdata *ppd;</div><div class='ctx'> 	int i;</div><div class='hunk'>@@ -1718,7 +1676,7 @@ static int set_pkeys(struct hfi1_devdata *dd, u8 port, u16 *pkeys)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_set_opa_pkeytable(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				    struct ib_device *ibdev, u8 port,</div><div class='add'>+				    struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				    u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='hunk'>@@ -1732,7 +1690,7 @@ static int __subn_set_opa_pkeytable(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> 	u32 size = 0;</div><div class='ctx'> </div><div class='ctx'> 	if (n_blocks_sent == 0) {</div><div class='del'>-		pr_warn("OPA Get PKey AM Invalid : P = %d; B = 0x%x; N = 0x%x\n",</div><div class='add'>+		pr_warn("OPA Get PKey AM Invalid : P = %u; B = 0x%x; N = 0x%x\n",</div><div class='ctx'> 			port, start_block, n_blocks_sent);</div><div class='ctx'> 		smp-&gt;status |= IB_SMP_INVALID_FIELD;</div><div class='ctx'> 		return reply((struct ib_mad_hdr *)smp);</div><div class='hunk'>@@ -1825,7 +1783,7 @@ static int get_sc2vlt_tables(struct hfi1_devdata *dd, void *data)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_sl_to_sc(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				   struct ib_device *ibdev, u8 port,</div><div class='add'>+				   struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				   u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_ibport *ibp = to_iport(ibdev, port);</div><div class='hunk'>@@ -1848,7 +1806,7 @@ static int __subn_get_opa_sl_to_sc(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_set_opa_sl_to_sc(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				   struct ib_device *ibdev, u8 port,</div><div class='add'>+				   struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				   u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_ibport *ibp = to_iport(ibdev, port);</div><div class='hunk'>@@ -1877,7 +1835,7 @@ static int __subn_set_opa_sl_to_sc(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_sc_to_sl(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				   struct ib_device *ibdev, u8 port,</div><div class='add'>+				   struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				   u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_ibport *ibp = to_iport(ibdev, port);</div><div class='hunk'>@@ -1900,7 +1858,7 @@ static int __subn_get_opa_sc_to_sl(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_set_opa_sc_to_sl(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				   struct ib_device *ibdev, u8 port,</div><div class='add'>+				   struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				   u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_ibport *ibp = to_iport(ibdev, port);</div><div class='hunk'>@@ -1921,7 +1879,7 @@ static int __subn_set_opa_sc_to_sl(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_sc_to_vlt(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				    struct ib_device *ibdev, u8 port,</div><div class='add'>+				    struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				    u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	u32 n_blocks = OPA_AM_NBLK(am);</div><div class='hunk'>@@ -1943,7 +1901,7 @@ static int __subn_get_opa_sc_to_vlt(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_set_opa_sc_to_vlt(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				    struct ib_device *ibdev, u8 port,</div><div class='add'>+				    struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				    u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	u32 n_blocks = OPA_AM_NBLK(am);</div><div class='hunk'>@@ -1985,7 +1943,7 @@ static int __subn_set_opa_sc_to_vlt(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_sc_to_vlnt(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				     struct ib_device *ibdev, u8 port,</div><div class='add'>+				     struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				     u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	u32 n_blocks = OPA_AM_NPORT(am);</div><div class='hunk'>@@ -2010,7 +1968,7 @@ static int __subn_get_opa_sc_to_vlnt(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_set_opa_sc_to_vlnt(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				     struct ib_device *ibdev, u8 port,</div><div class='add'>+				     struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				     u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	u32 n_blocks = OPA_AM_NPORT(am);</div><div class='hunk'>@@ -2042,7 +2000,7 @@ static int __subn_set_opa_sc_to_vlnt(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_psi(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-			      struct ib_device *ibdev, u8 port,</div><div class='add'>+			      struct ib_device *ibdev, u32 port,</div><div class='ctx'> 			      u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	u32 nports = OPA_AM_NPORT(am);</div><div class='hunk'>@@ -2084,7 +2042,7 @@ static int __subn_get_opa_psi(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_set_opa_psi(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-			      struct ib_device *ibdev, u8 port,</div><div class='add'>+			      struct ib_device *ibdev, u32 port,</div><div class='ctx'> 			      u32 *resp_len, u32 max_len, int local_mad)</div><div class='ctx'> {</div><div class='ctx'> 	u32 nports = OPA_AM_NPORT(am);</div><div class='hunk'>@@ -2132,7 +2090,7 @@ static int __subn_set_opa_psi(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_cable_info(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				     struct ib_device *ibdev, u8 port,</div><div class='add'>+				     struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				     u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='hunk'>@@ -2184,7 +2142,7 @@ static int __subn_get_opa_cable_info(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_bct(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-			      struct ib_device *ibdev, u8 port, u32 *resp_len,</div><div class='add'>+			      struct ib_device *ibdev, u32 port, u32 *resp_len,</div><div class='ctx'> 			      u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	u32 num_ports = OPA_AM_NPORT(am);</div><div class='hunk'>@@ -2208,7 +2166,7 @@ static int __subn_get_opa_bct(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_set_opa_bct(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-			      struct ib_device *ibdev, u8 port, u32 *resp_len,</div><div class='add'>+			      struct ib_device *ibdev, u32 port, u32 *resp_len,</div><div class='ctx'> 			      u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	u32 num_ports = OPA_AM_NPORT(am);</div><div class='hunk'>@@ -2232,7 +2190,7 @@ static int __subn_set_opa_bct(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_vl_arb(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				 struct ib_device *ibdev, u8 port,</div><div class='add'>+				 struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				 u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_pportdata *ppd = ppd_from_ibp(to_iport(ibdev, port));</div><div class='hunk'>@@ -2274,7 +2232,7 @@ static int __subn_get_opa_vl_arb(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_set_opa_vl_arb(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				 struct ib_device *ibdev, u8 port,</div><div class='add'>+				 struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				 u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_pportdata *ppd = ppd_from_ibp(to_iport(ibdev, port));</div><div class='hunk'>@@ -2300,7 +2258,6 @@ static int __subn_set_opa_vl_arb(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> 	 * can be changed from the default values</div><div class='ctx'> 	 */</div><div class='ctx'> 	case OPA_VLARB_PREEMPT_ELEMENTS:</div><div class='del'>-		/* FALLTHROUGH */</div><div class='ctx'> 	case OPA_VLARB_PREEMPT_MATRIX:</div><div class='ctx'> 		smp-&gt;status |= IB_SMP_UNSUP_METH_ATTR;</div><div class='ctx'> 		break;</div><div class='hunk'>@@ -2326,7 +2283,7 @@ struct opa_port_status_req {</div><div class='ctx'> 	__be32 vl_select_mask;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define VL_MASK_ALL		0x000080ff</div><div class='add'>+#define VL_MASK_ALL		0x00000000000080ffUL</div><div class='ctx'> </div><div class='ctx'> struct opa_port_status_rsp {</div><div class='ctx'> 	__u8 port_num;</div><div class='hunk'>@@ -2381,7 +2338,7 @@ struct opa_port_status_rsp {</div><div class='ctx'> 		__be64 port_vl_rcv_bubble;</div><div class='ctx'> 		__be64 port_vl_mark_fecn;</div><div class='ctx'> 		__be64 port_vl_xmit_discards;</div><div class='del'>-	} vls[0]; /* real array size defined by # bits set in vl_select_mask */</div><div class='add'>+	} vls[]; /* real array size defined by # bits set in vl_select_mask */</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum counter_selects {</div><div class='hunk'>@@ -2423,7 +2380,7 @@ struct opa_aggregate {</div><div class='ctx'> 	__be16 attr_id;</div><div class='ctx'> 	__be16 err_reqlength;	/* 1 bit, 8 res, 7 bit */</div><div class='ctx'> 	__be32 attr_mod;</div><div class='del'>-	u8 data[0];</div><div class='add'>+	u8 data[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> #define MSK_LLI 0x000000f0</div><div class='hunk'>@@ -2480,9 +2437,9 @@ struct opa_port_data_counters_msg {</div><div class='ctx'> 			__be64 port_vl_xmit_wait_data;</div><div class='ctx'> 			__be64 port_vl_rcv_bubble;</div><div class='ctx'> 			__be64 port_vl_mark_fecn;</div><div class='del'>-		} vls[0];</div><div class='add'>+		} vls[];</div><div class='ctx'> 		/* array size defined by #bits set in vl_select_mask*/</div><div class='del'>-	} port[1]; /* array size defined by  #ports in attribute modifier */</div><div class='add'>+	} port;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct opa_port_error_counters64_msg {</div><div class='hunk'>@@ -2513,9 +2470,9 @@ struct opa_port_error_counters64_msg {</div><div class='ctx'> 		u8 reserved3[7];</div><div class='ctx'> 		struct _vls_ectrs {</div><div class='ctx'> 			__be64 port_vl_xmit_discards;</div><div class='del'>-		} vls[0];</div><div class='add'>+		} vls[];</div><div class='ctx'> 		/* array size defined by #bits set in vl_select_mask */</div><div class='del'>-	} port[1]; /* array size defined by #ports in attribute modifier */</div><div class='add'>+	} port;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct opa_port_error_info_msg {</div><div class='hunk'>@@ -2586,7 +2543,7 @@ struct opa_port_error_info_msg {</div><div class='ctx'> 			u8 error_info;</div><div class='ctx'> 		} __packed fm_config_ei;</div><div class='ctx'> 		__u32 reserved9;</div><div class='del'>-	} port[1]; /* actual array size defined by #ports in attr modifier */</div><div class='add'>+	} port;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /* opa_port_error_info_msg error_info_select_mask bit definitions */</div><div class='hunk'>@@ -2625,15 +2582,14 @@ static int pma_get_opa_classportinfo(struct opa_pma_mad *pmp,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void a0_portstatus(struct hfi1_pportdata *ppd,</div><div class='del'>-			  struct opa_port_status_rsp *rsp, u32 vl_select_mask)</div><div class='add'>+			  struct opa_port_status_rsp *rsp)</div><div class='ctx'> {</div><div class='ctx'> 	if (!is_bx(ppd-&gt;dd)) {</div><div class='ctx'> 		unsigned long vl;</div><div class='ctx'> 		u64 sum_vl_xmit_wait = 0;</div><div class='del'>-		u32 vl_all_mask = VL_MASK_ALL;</div><div class='add'>+		unsigned long vl_all_mask = VL_MASK_ALL;</div><div class='ctx'> </div><div class='del'>-		for_each_set_bit(vl, (unsigned long *)&amp;(vl_all_mask),</div><div class='del'>-				 8 * sizeof(vl_all_mask)) {</div><div class='add'>+		for_each_set_bit(vl, &amp;vl_all_mask, BITS_PER_LONG) {</div><div class='ctx'> 			u64 tmp = sum_vl_xmit_wait +</div><div class='ctx'> 				  read_port_cntr(ppd, C_TX_WAIT_VL,</div><div class='ctx'> 						 idx_from_vl(vl));</div><div class='hunk'>@@ -2724,18 +2680,18 @@ u64 get_xmit_wait_counters(struct hfi1_pportdata *ppd,</div><div class='ctx'> </div><div class='ctx'> static int pma_get_opa_portstatus(struct opa_pma_mad *pmp,</div><div class='ctx'> 				  struct ib_device *ibdev,</div><div class='del'>-				  u8 port, u32 *resp_len)</div><div class='add'>+				  u32 port, u32 *resp_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct opa_port_status_req *req =</div><div class='ctx'> 		(struct opa_port_status_req *)pmp-&gt;data;</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='ctx'> 	struct opa_port_status_rsp *rsp;</div><div class='del'>-	u32 vl_select_mask = be32_to_cpu(req-&gt;vl_select_mask);</div><div class='add'>+	unsigned long vl_select_mask = be32_to_cpu(req-&gt;vl_select_mask);</div><div class='ctx'> 	unsigned long vl;</div><div class='ctx'> 	size_t response_data_size;</div><div class='ctx'> 	u32 nports = be32_to_cpu(pmp-&gt;mad_hdr.attr_mod) &gt;&gt; 24;</div><div class='del'>-	u8 port_num = req-&gt;port_num;</div><div class='del'>-	u8 num_vls = hweight32(vl_select_mask);</div><div class='add'>+	u32 port_num = req-&gt;port_num;</div><div class='add'>+	u8 num_vls = hweight64(vl_select_mask);</div><div class='ctx'> 	struct _vls_pctrs *vlinfo;</div><div class='ctx'> 	struct hfi1_ibport *ibp = to_iport(ibdev, port);</div><div class='ctx'> 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);</div><div class='hunk'>@@ -2744,8 +2700,7 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp,</div><div class='ctx'> 	u16 link_width;</div><div class='ctx'> 	u16 link_speed;</div><div class='ctx'> </div><div class='del'>-	response_data_size = sizeof(struct opa_port_status_rsp) +</div><div class='del'>-				num_vls * sizeof(struct _vls_pctrs);</div><div class='add'>+	response_data_size = struct_size(rsp, vls, num_vls);</div><div class='ctx'> 	if (response_data_size &gt; sizeof(pmp-&gt;data)) {</div><div class='ctx'> 		pmp-&gt;mad_hdr.status |= OPA_PM_STATUS_REQUEST_TOO_LARGE;</div><div class='ctx'> 		return reply((struct ib_mad_hdr *)pmp);</div><div class='hunk'>@@ -2771,7 +2726,7 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp,</div><div class='ctx'> </div><div class='ctx'> 	hfi1_read_link_quality(dd, &amp;rsp-&gt;link_quality_indicator);</div><div class='ctx'> </div><div class='del'>-	rsp-&gt;vl_select_mask = cpu_to_be32(vl_select_mask);</div><div class='add'>+	rsp-&gt;vl_select_mask = cpu_to_be32((u32)vl_select_mask);</div><div class='ctx'> 	rsp-&gt;port_xmit_data = cpu_to_be64(read_dev_cntr(dd, C_DC_XMIT_FLITS,</div><div class='ctx'> 					  CNTR_INVALID_VL));</div><div class='ctx'> 	rsp-&gt;port_rcv_data = cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FLITS,</div><div class='hunk'>@@ -2842,8 +2797,7 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp,</div><div class='ctx'> 	 * So in the for_each_set_bit() loop below, we don't need</div><div class='ctx'> 	 * any additional checks for vl.</div><div class='ctx'> 	 */</div><div class='del'>-	for_each_set_bit(vl, (unsigned long *)&amp;(vl_select_mask),</div><div class='del'>-			 8 * sizeof(vl_select_mask)) {</div><div class='add'>+	for_each_set_bit(vl, &amp;vl_select_mask, BITS_PER_LONG) {</div><div class='ctx'> 		memset(vlinfo, 0, sizeof(*vlinfo));</div><div class='ctx'> </div><div class='ctx'> 		tmp = read_dev_cntr(dd, C_DC_RX_FLIT_VL, idx_from_vl(vl));</div><div class='hunk'>@@ -2884,7 +2838,7 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp,</div><div class='ctx'> 		vfi++;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	a0_portstatus(ppd, rsp, vl_select_mask);</div><div class='add'>+	a0_portstatus(ppd, rsp);</div><div class='ctx'> </div><div class='ctx'> 	if (resp_len)</div><div class='ctx'> 		*resp_len += response_data_size;</div><div class='hunk'>@@ -2892,7 +2846,7 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp,</div><div class='ctx'> 	return reply((struct ib_mad_hdr *)pmp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static u64 get_error_counter_summary(struct ib_device *ibdev, u8 port,</div><div class='add'>+static u64 get_error_counter_summary(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				     u8 res_lli, u8 res_ler)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='hunk'>@@ -2931,16 +2885,14 @@ static u64 get_error_counter_summary(struct ib_device *ibdev, u8 port,</div><div class='ctx'> 	return error_counter_summary;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void a0_datacounters(struct hfi1_pportdata *ppd, struct _port_dctrs *rsp,</div><div class='del'>-			    u32 vl_select_mask)</div><div class='add'>+static void a0_datacounters(struct hfi1_pportdata *ppd, struct _port_dctrs *rsp)</div><div class='ctx'> {</div><div class='ctx'> 	if (!is_bx(ppd-&gt;dd)) {</div><div class='ctx'> 		unsigned long vl;</div><div class='ctx'> 		u64 sum_vl_xmit_wait = 0;</div><div class='del'>-		u32 vl_all_mask = VL_MASK_ALL;</div><div class='add'>+		unsigned long vl_all_mask = VL_MASK_ALL;</div><div class='ctx'> </div><div class='del'>-		for_each_set_bit(vl, (unsigned long *)&amp;(vl_all_mask),</div><div class='del'>-				 8 * sizeof(vl_all_mask)) {</div><div class='add'>+		for_each_set_bit(vl, &amp;vl_all_mask, BITS_PER_LONG) {</div><div class='ctx'> 			u64 tmp = sum_vl_xmit_wait +</div><div class='ctx'> 				  read_port_cntr(ppd, C_TX_WAIT_VL,</div><div class='ctx'> 						 idx_from_vl(vl));</div><div class='hunk'>@@ -2979,7 +2931,7 @@ static void pma_get_opa_port_dctrs(struct ib_device *ibdev,</div><div class='ctx'> </div><div class='ctx'> static int pma_get_opa_datacounters(struct opa_pma_mad *pmp,</div><div class='ctx'> 				    struct ib_device *ibdev,</div><div class='del'>-				    u8 port, u32 *resp_len)</div><div class='add'>+				    u32 port, u32 *resp_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct opa_port_data_counters_msg *req =</div><div class='ctx'> 		(struct opa_port_data_counters_msg *)pmp-&gt;data;</div><div class='hunk'>@@ -2993,9 +2945,9 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp,</div><div class='ctx'> 	u8 lq, num_vls;</div><div class='ctx'> 	u8 res_lli, res_ler;</div><div class='ctx'> 	u64 port_mask;</div><div class='del'>-	u8 port_num;</div><div class='add'>+	u32 port_num;</div><div class='ctx'> 	unsigned long vl;</div><div class='del'>-	u32 vl_select_mask;</div><div class='add'>+	unsigned long vl_select_mask;</div><div class='ctx'> 	int vfi;</div><div class='ctx'> 	u16 link_width;</div><div class='ctx'> 	u16 link_speed;</div><div class='hunk'>@@ -3014,8 +2966,7 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Sanity check */</div><div class='del'>-	response_data_size = sizeof(struct opa_port_data_counters_msg) +</div><div class='del'>-				num_vls * sizeof(struct _vls_dctrs);</div><div class='add'>+	response_data_size = struct_size(req, port.vls, num_vls);</div><div class='ctx'> </div><div class='ctx'> 	if (response_data_size &gt; sizeof(pmp-&gt;data)) {</div><div class='ctx'> 		pmp-&gt;mad_hdr.status |= IB_SMP_INVALID_FIELD;</div><div class='hunk'>@@ -3035,7 +2986,7 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp,</div><div class='ctx'> 		return reply((struct ib_mad_hdr *)pmp);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	rsp = &amp;req-&gt;port[0];</div><div class='add'>+	rsp = &amp;req-&gt;port;</div><div class='ctx'> 	memset(rsp, 0, sizeof(*rsp));</div><div class='ctx'> </div><div class='ctx'> 	rsp-&gt;port_number = port;</div><div class='hunk'>@@ -3073,8 +3024,7 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp,</div><div class='ctx'> 	 * So in the for_each_set_bit() loop below, we don't need</div><div class='ctx'> 	 * any additional checks for vl.</div><div class='ctx'> 	 */</div><div class='del'>-	for_each_set_bit(vl, (unsigned long *)&amp;(vl_select_mask),</div><div class='del'>-			 8 * sizeof(req-&gt;vl_select_mask)) {</div><div class='add'>+	for_each_set_bit(vl, &amp;vl_select_mask, BITS_PER_LONG) {</div><div class='ctx'> 		memset(vlinfo, 0, sizeof(*vlinfo));</div><div class='ctx'> </div><div class='ctx'> 		rsp-&gt;vls[vfi].port_vl_xmit_data =</div><div class='hunk'>@@ -3122,7 +3072,7 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp,</div><div class='ctx'> 		vfi++;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	a0_datacounters(ppd, rsp, vl_select_mask);</div><div class='add'>+	a0_datacounters(ppd, rsp);</div><div class='ctx'> </div><div class='ctx'> 	if (resp_len)</div><div class='ctx'> 		*resp_len += response_data_size;</div><div class='hunk'>@@ -3131,7 +3081,7 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int pma_get_ib_portcounters_ext(struct ib_pma_mad *pmp,</div><div class='del'>-				       struct ib_device *ibdev, u8 port)</div><div class='add'>+				       struct ib_device *ibdev, u32 port)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_pma_portcounters_ext *p = (struct ib_pma_portcounters_ext *)</div><div class='ctx'> 						pmp-&gt;data;</div><div class='hunk'>@@ -3159,7 +3109,7 @@ bail:</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void pma_get_opa_port_ectrs(struct ib_device *ibdev,</div><div class='del'>-				   struct _port_ectrs *rsp, u8 port)</div><div class='add'>+				   struct _port_ectrs *rsp, u32 port)</div><div class='ctx'> {</div><div class='ctx'> 	u64 tmp, tmp2;</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='hunk'>@@ -3202,11 +3152,11 @@ static void pma_get_opa_port_ectrs(struct ib_device *ibdev,</div><div class='ctx'> </div><div class='ctx'> static int pma_get_opa_porterrors(struct opa_pma_mad *pmp,</div><div class='ctx'> 				  struct ib_device *ibdev,</div><div class='del'>-				  u8 port, u32 *resp_len)</div><div class='add'>+				  u32 port, u32 *resp_len)</div><div class='ctx'> {</div><div class='ctx'> 	size_t response_data_size;</div><div class='ctx'> 	struct _port_ectrs *rsp;</div><div class='del'>-	u8 port_num;</div><div class='add'>+	u32 port_num;</div><div class='ctx'> 	struct opa_port_error_counters64_msg *req;</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='ctx'> 	u32 num_ports;</div><div class='hunk'>@@ -3217,7 +3167,7 @@ static int pma_get_opa_porterrors(struct opa_pma_mad *pmp,</div><div class='ctx'> 	struct _vls_ectrs *vlinfo;</div><div class='ctx'> 	unsigned long vl;</div><div class='ctx'> 	u64 port_mask, tmp;</div><div class='del'>-	u32 vl_select_mask;</div><div class='add'>+	unsigned long vl_select_mask;</div><div class='ctx'> 	int vfi;</div><div class='ctx'> </div><div class='ctx'> 	req = (struct opa_port_error_counters64_msg *)pmp-&gt;data;</div><div class='hunk'>@@ -3232,8 +3182,7 @@ static int pma_get_opa_porterrors(struct opa_pma_mad *pmp,</div><div class='ctx'> 		return reply((struct ib_mad_hdr *)pmp);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	response_data_size = sizeof(struct opa_port_error_counters64_msg) +</div><div class='del'>-				num_vls * sizeof(struct _vls_ectrs);</div><div class='add'>+	response_data_size = struct_size(req, port.vls, num_vls);</div><div class='ctx'> </div><div class='ctx'> 	if (response_data_size &gt; sizeof(pmp-&gt;data)) {</div><div class='ctx'> 		pmp-&gt;mad_hdr.status |= IB_SMP_INVALID_FIELD;</div><div class='hunk'>@@ -3252,7 +3201,7 @@ static int pma_get_opa_porterrors(struct opa_pma_mad *pmp,</div><div class='ctx'> 		return reply((struct ib_mad_hdr *)pmp);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	rsp = &amp;req-&gt;port[0];</div><div class='add'>+	rsp = &amp;req-&gt;port;</div><div class='ctx'> </div><div class='ctx'> 	ibp = to_iport(ibdev, port_num);</div><div class='ctx'> 	ppd = ppd_from_ibp(ibp);</div><div class='hunk'>@@ -3276,8 +3225,7 @@ static int pma_get_opa_porterrors(struct opa_pma_mad *pmp,</div><div class='ctx'> 	vlinfo = &amp;rsp-&gt;vls[0];</div><div class='ctx'> 	vfi = 0;</div><div class='ctx'> 	vl_select_mask = be32_to_cpu(req-&gt;vl_select_mask);</div><div class='del'>-	for_each_set_bit(vl, (unsigned long *)&amp;(vl_select_mask),</div><div class='del'>-			 8 * sizeof(req-&gt;vl_select_mask)) {</div><div class='add'>+	for_each_set_bit(vl, &amp;vl_select_mask, BITS_PER_LONG) {</div><div class='ctx'> 		memset(vlinfo, 0, sizeof(*vlinfo));</div><div class='ctx'> 		rsp-&gt;vls[vfi].port_vl_xmit_discards =</div><div class='ctx'> 			cpu_to_be64(read_port_cntr(ppd, C_SW_XMIT_DSCD_VL,</div><div class='hunk'>@@ -3293,7 +3241,7 @@ static int pma_get_opa_porterrors(struct opa_pma_mad *pmp,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int pma_get_ib_portcounters(struct ib_pma_mad *pmp,</div><div class='del'>-				   struct ib_device *ibdev, u8 port)</div><div class='add'>+				   struct ib_device *ibdev, u32 port)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_pma_portcounters *p = (struct ib_pma_portcounters *)</div><div class='ctx'> 		pmp-&gt;data;</div><div class='hunk'>@@ -3379,7 +3327,7 @@ bail:</div><div class='ctx'> </div><div class='ctx'> static int pma_get_opa_errorinfo(struct opa_pma_mad *pmp,</div><div class='ctx'> 				 struct ib_device *ibdev,</div><div class='del'>-				 u8 port, u32 *resp_len)</div><div class='add'>+				 u32 port, u32 *resp_len)</div><div class='ctx'> {</div><div class='ctx'> 	size_t response_data_size;</div><div class='ctx'> 	struct _port_ei *rsp;</div><div class='hunk'>@@ -3387,12 +3335,12 @@ static int pma_get_opa_errorinfo(struct opa_pma_mad *pmp,</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='ctx'> 	u64 port_mask;</div><div class='ctx'> 	u32 num_ports;</div><div class='del'>-	u8 port_num;</div><div class='add'>+	u32 port_num;</div><div class='ctx'> 	u8 num_pslm;</div><div class='ctx'> 	u64 reg;</div><div class='ctx'> </div><div class='ctx'> 	req = (struct opa_port_error_info_msg *)pmp-&gt;data;</div><div class='del'>-	rsp = &amp;req-&gt;port[0];</div><div class='add'>+	rsp = &amp;req-&gt;port;</div><div class='ctx'> </div><div class='ctx'> 	num_ports = OPA_AM_NPORT(be32_to_cpu(pmp-&gt;mad_hdr.attr_mod));</div><div class='ctx'> 	num_pslm = hweight64(be64_to_cpu(req-&gt;port_select_mask[3]));</div><div class='hunk'>@@ -3478,7 +3426,7 @@ static int pma_get_opa_errorinfo(struct opa_pma_mad *pmp,</div><div class='ctx'> </div><div class='ctx'> static int pma_set_opa_portstatus(struct opa_pma_mad *pmp,</div><div class='ctx'> 				  struct ib_device *ibdev,</div><div class='del'>-				  u8 port, u32 *resp_len)</div><div class='add'>+				  u32 port, u32 *resp_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct opa_clear_port_status *req =</div><div class='ctx'> 		(struct opa_clear_port_status *)pmp-&gt;data;</div><div class='hunk'>@@ -3488,7 +3436,7 @@ static int pma_set_opa_portstatus(struct opa_pma_mad *pmp,</div><div class='ctx'> 	u32 nports = be32_to_cpu(pmp-&gt;mad_hdr.attr_mod) &gt;&gt; 24;</div><div class='ctx'> 	u64 portn = be64_to_cpu(req-&gt;port_select_mask[3]);</div><div class='ctx'> 	u32 counter_select = be32_to_cpu(req-&gt;counter_select_mask);</div><div class='del'>-	u32 vl_select_mask = VL_MASK_ALL; /* clear all per-vl cnts */</div><div class='add'>+	unsigned long vl_select_mask = VL_MASK_ALL; /* clear all per-vl cnts */</div><div class='ctx'> 	unsigned long vl;</div><div class='ctx'> </div><div class='ctx'> 	if ((nports != 1) || (portn != 1 &lt;&lt; port)) {</div><div class='hunk'>@@ -3582,8 +3530,7 @@ static int pma_set_opa_portstatus(struct opa_pma_mad *pmp,</div><div class='ctx'> 	if (counter_select &amp; CS_UNCORRECTABLE_ERRORS)</div><div class='ctx'> 		write_dev_cntr(dd, C_DC_UNC_ERR, CNTR_INVALID_VL, 0);</div><div class='ctx'> </div><div class='del'>-	for_each_set_bit(vl, (unsigned long *)&amp;(vl_select_mask),</div><div class='del'>-			 8 * sizeof(vl_select_mask)) {</div><div class='add'>+	for_each_set_bit(vl, &amp;vl_select_mask, BITS_PER_LONG) {</div><div class='ctx'> 		if (counter_select &amp; CS_PORT_XMIT_DATA)</div><div class='ctx'> 			write_port_cntr(ppd, C_TX_FLIT_VL, idx_from_vl(vl), 0);</div><div class='ctx'> </div><div class='hunk'>@@ -3631,19 +3578,19 @@ static int pma_set_opa_portstatus(struct opa_pma_mad *pmp,</div><div class='ctx'> </div><div class='ctx'> static int pma_set_opa_errorinfo(struct opa_pma_mad *pmp,</div><div class='ctx'> 				 struct ib_device *ibdev,</div><div class='del'>-				 u8 port, u32 *resp_len)</div><div class='add'>+				 u32 port, u32 *resp_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct _port_ei *rsp;</div><div class='ctx'> 	struct opa_port_error_info_msg *req;</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='ctx'> 	u64 port_mask;</div><div class='ctx'> 	u32 num_ports;</div><div class='del'>-	u8 port_num;</div><div class='add'>+	u32 port_num;</div><div class='ctx'> 	u8 num_pslm;</div><div class='ctx'> 	u32 error_info_select;</div><div class='ctx'> </div><div class='ctx'> 	req = (struct opa_port_error_info_msg *)pmp-&gt;data;</div><div class='del'>-	rsp = &amp;req-&gt;port[0];</div><div class='add'>+	rsp = &amp;req-&gt;port;</div><div class='ctx'> </div><div class='ctx'> 	num_ports = OPA_AM_NPORT(be32_to_cpu(pmp-&gt;mad_hdr.attr_mod));</div><div class='ctx'> 	num_pslm = hweight64(be64_to_cpu(req-&gt;port_select_mask[3]));</div><div class='hunk'>@@ -3713,7 +3660,7 @@ struct opa_congestion_info_attr {</div><div class='ctx'> } __packed;</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_cong_info(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				    struct ib_device *ibdev, u8 port,</div><div class='add'>+				    struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				    u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct opa_congestion_info_attr *p =</div><div class='hunk'>@@ -3738,7 +3685,7 @@ static int __subn_get_opa_cong_info(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_cong_setting(struct opa_smp *smp, u32 am,</div><div class='ctx'> 				       u8 *data, struct ib_device *ibdev,</div><div class='del'>-				       u8 port, u32 *resp_len, u32 max_len)</div><div class='add'>+				       u32 port, u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='ctx'> 	struct opa_congestion_setting_attr *p =</div><div class='hunk'>@@ -3830,7 +3777,7 @@ static void apply_cc_state(struct hfi1_pportdata *ppd)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_set_opa_cong_setting(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				       struct ib_device *ibdev, u8 port,</div><div class='add'>+				       struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				       u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct opa_congestion_setting_attr *p =</div><div class='hunk'>@@ -3871,7 +3818,7 @@ static int __subn_set_opa_cong_setting(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_hfi1_cong_log(struct opa_smp *smp, u32 am,</div><div class='ctx'> 					u8 *data, struct ib_device *ibdev,</div><div class='del'>-					u8 port, u32 *resp_len, u32 max_len)</div><div class='add'>+					u32 port, u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_ibport *ibp = to_iport(ibdev, port);</div><div class='ctx'> 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);</div><div class='hunk'>@@ -3936,7 +3883,7 @@ static int __subn_get_opa_hfi1_cong_log(struct opa_smp *smp, u32 am,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_cc_table(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				   struct ib_device *ibdev, u8 port,</div><div class='add'>+				   struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				   u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_cc_table_attr *cc_table_attr =</div><div class='hunk'>@@ -3988,7 +3935,7 @@ static int __subn_get_opa_cc_table(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_set_opa_cc_table(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				   struct ib_device *ibdev, u8 port,</div><div class='add'>+				   struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				   u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_cc_table_attr *p = (struct ib_cc_table_attr *)data;</div><div class='hunk'>@@ -4047,7 +3994,7 @@ struct opa_led_info {</div><div class='ctx'> #define OPA_LED_MASK	BIT(OPA_LED_SHIFT)</div><div class='ctx'> </div><div class='ctx'> static int __subn_get_opa_led_info(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				   struct ib_device *ibdev, u8 port,</div><div class='add'>+				   struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				   u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='hunk'>@@ -4077,7 +4024,7 @@ static int __subn_get_opa_led_info(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __subn_set_opa_led_info(struct opa_smp *smp, u32 am, u8 *data,</div><div class='del'>-				   struct ib_device *ibdev, u8 port,</div><div class='add'>+				   struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				   u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='hunk'>@@ -4100,7 +4047,7 @@ static int __subn_set_opa_led_info(struct opa_smp *smp, u32 am, u8 *data,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int subn_get_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am,</div><div class='del'>-			    u8 *data, struct ib_device *ibdev, u8 port,</div><div class='add'>+			    u8 *data, struct ib_device *ibdev, u32 port,</div><div class='ctx'> 			    u32 *resp_len, u32 max_len)</div><div class='ctx'> {</div><div class='ctx'> 	int ret;</div><div class='hunk'>@@ -4180,7 +4127,7 @@ static int subn_get_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am,</div><div class='ctx'> 			return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;</div><div class='ctx'> 		if (ibp-&gt;rvp.port_cap_flags &amp; IB_PORT_SM)</div><div class='ctx'> 			return IB_MAD_RESULT_SUCCESS;</div><div class='del'>-		/* FALLTHROUGH */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	default:</div><div class='ctx'> 		smp-&gt;status |= IB_SMP_UNSUP_METH_ATTR;</div><div class='ctx'> 		ret = reply((struct ib_mad_hdr *)smp);</div><div class='hunk'>@@ -4190,7 +4137,7 @@ static int subn_get_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int subn_set_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am,</div><div class='del'>-			    u8 *data, struct ib_device *ibdev, u8 port,</div><div class='add'>+			    u8 *data, struct ib_device *ibdev, u32 port,</div><div class='ctx'> 			    u32 *resp_len, u32 max_len, int local_mad)</div><div class='ctx'> {</div><div class='ctx'> 	int ret;</div><div class='hunk'>@@ -4250,7 +4197,7 @@ static int subn_set_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am,</div><div class='ctx'> 			return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;</div><div class='ctx'> 		if (ibp-&gt;rvp.port_cap_flags &amp; IB_PORT_SM)</div><div class='ctx'> 			return IB_MAD_RESULT_SUCCESS;</div><div class='del'>-		/* FALLTHROUGH */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	default:</div><div class='ctx'> 		smp-&gt;status |= IB_SMP_UNSUP_METH_ATTR;</div><div class='ctx'> 		ret = reply((struct ib_mad_hdr *)smp);</div><div class='hunk'>@@ -4265,7 +4212,7 @@ static inline void set_aggr_error(struct opa_aggregate *ag)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int subn_get_opa_aggregate(struct opa_smp *smp,</div><div class='del'>-				  struct ib_device *ibdev, u8 port,</div><div class='add'>+				  struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				  u32 *resp_len)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='hunk'>@@ -4314,7 +4261,7 @@ static int subn_get_opa_aggregate(struct opa_smp *smp,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int subn_set_opa_aggregate(struct opa_smp *smp,</div><div class='del'>-				  struct ib_device *ibdev, u8 port,</div><div class='add'>+				  struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				  u32 *resp_len, int local_mad)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='hunk'>@@ -4520,7 +4467,7 @@ static int hfi1_pkey_validation_pma(struct hfi1_ibport *ibp,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int process_subn_opa(struct ib_device *ibdev, int mad_flags,</div><div class='del'>-			    u8 port, const struct opa_mad *in_mad,</div><div class='add'>+			    u32 port, const struct opa_mad *in_mad,</div><div class='ctx'> 			    struct opa_mad *out_mad,</div><div class='ctx'> 			    u32 *resp_len, int local_mad)</div><div class='ctx'> {</div><div class='hunk'>@@ -4625,7 +4572,7 @@ static int process_subn_opa(struct ib_device *ibdev, int mad_flags,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int process_subn(struct ib_device *ibdev, int mad_flags,</div><div class='del'>-			u8 port, const struct ib_mad *in_mad,</div><div class='add'>+			u32 port, const struct ib_mad *in_mad,</div><div class='ctx'> 			struct ib_mad *out_mad)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_smp *smp = (struct ib_smp *)out_mad;</div><div class='hunk'>@@ -4683,7 +4630,7 @@ static int process_subn(struct ib_device *ibdev, int mad_flags,</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int process_perf(struct ib_device *ibdev, u8 port,</div><div class='add'>+static int process_perf(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 			const struct ib_mad *in_mad,</div><div class='ctx'> 			struct ib_mad *out_mad)</div><div class='ctx'> {</div><div class='hunk'>@@ -4745,7 +4692,7 @@ static int process_perf(struct ib_device *ibdev, u8 port,</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int process_perf_opa(struct ib_device *ibdev, u8 port,</div><div class='add'>+static int process_perf_opa(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 			    const struct opa_mad *in_mad,</div><div class='ctx'> 			    struct opa_mad *out_mad, u32 *resp_len)</div><div class='ctx'> {</div><div class='hunk'>@@ -4827,7 +4774,7 @@ static int process_perf_opa(struct ib_device *ibdev, u8 port,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hfi1_process_opa_mad(struct ib_device *ibdev, int mad_flags,</div><div class='del'>-				u8 port, const struct ib_wc *in_wc,</div><div class='add'>+				u32 port, const struct ib_wc *in_wc,</div><div class='ctx'> 				const struct ib_grh *in_grh,</div><div class='ctx'> 				const struct opa_mad *in_mad,</div><div class='ctx'> 				struct opa_mad *out_mad, size_t *out_mad_size,</div><div class='hunk'>@@ -4880,7 +4827,7 @@ bail:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hfi1_process_ib_mad(struct ib_device *ibdev, int mad_flags, u8 port,</div><div class='add'>+static int hfi1_process_ib_mad(struct ib_device *ibdev, int mad_flags, u32 port,</div><div class='ctx'> 			       const struct ib_wc *in_wc,</div><div class='ctx'> 			       const struct ib_grh *in_grh,</div><div class='ctx'> 			       const struct ib_mad *in_mad,</div><div class='hunk'>@@ -4913,6 +4860,8 @@ static int hfi1_process_ib_mad(struct ib_device *ibdev, int mad_flags, u8 port,</div><div class='ctx'>  * @in_grh: the global route header for this packet</div><div class='ctx'>  * @in_mad: the incoming MAD</div><div class='ctx'>  * @out_mad: any outgoing MAD reply</div><div class='add'>+ * @out_mad_size: size of the outgoing MAD reply</div><div class='add'>+ * @out_mad_pkey_index: used to apss back the packet key index</div><div class='ctx'>  *</div><div class='ctx'>  * Returns IB_MAD_RESULT_SUCCESS if this is a MAD that we are not</div><div class='ctx'>  * interested in processing.</div><div class='hunk'>@@ -4923,18 +4872,13 @@ static int hfi1_process_ib_mad(struct ib_device *ibdev, int mad_flags, u8 port,</div><div class='ctx'>  *</div><div class='ctx'>  * This is called by the ib_mad module.</div><div class='ctx'>  */</div><div class='del'>-int hfi1_process_mad(struct ib_device *ibdev, int mad_flags, u8 port,</div><div class='add'>+int hfi1_process_mad(struct ib_device *ibdev, int mad_flags, u32 port,</div><div class='ctx'> 		     const struct ib_wc *in_wc, const struct ib_grh *in_grh,</div><div class='del'>-		     const struct ib_mad_hdr *in_mad, size_t in_mad_size,</div><div class='del'>-		     struct ib_mad_hdr *out_mad, size_t *out_mad_size,</div><div class='del'>-		     u16 *out_mad_pkey_index)</div><div class='add'>+		     const struct ib_mad *in_mad, struct ib_mad *out_mad,</div><div class='add'>+		     size_t *out_mad_size, u16 *out_mad_pkey_index)</div><div class='ctx'> {</div><div class='del'>-	switch (in_mad-&gt;base_version) {</div><div class='add'>+	switch (in_mad-&gt;mad_hdr.base_version) {</div><div class='ctx'> 	case OPA_MGMT_BASE_VERSION:</div><div class='del'>-		if (unlikely(in_mad_size != sizeof(struct opa_mad))) {</div><div class='del'>-			dev_err(ibdev-&gt;dev.parent, "invalid in_mad_size\n");</div><div class='del'>-			return IB_MAD_RESULT_FAILURE;</div><div class='del'>-		}</div><div class='ctx'> 		return hfi1_process_opa_mad(ibdev, mad_flags, port,</div><div class='ctx'> 					    in_wc, in_grh,</div><div class='ctx'> 					    (struct opa_mad *)in_mad,</div><div class='hunk'>@@ -4942,10 +4886,8 @@ int hfi1_process_mad(struct ib_device *ibdev, int mad_flags, u8 port,</div><div class='ctx'> 					    out_mad_size,</div><div class='ctx'> 					    out_mad_pkey_index);</div><div class='ctx'> 	case IB_MGMT_BASE_VERSION:</div><div class='del'>-		return hfi1_process_ib_mad(ibdev, mad_flags, port,</div><div class='del'>-					  in_wc, in_grh,</div><div class='del'>-					  (const struct ib_mad *)in_mad,</div><div class='del'>-					  (struct ib_mad *)out_mad);</div><div class='add'>+		return hfi1_process_ib_mad(ibdev, mad_flags, port, in_wc,</div><div class='add'>+					   in_grh, in_mad, out_mad);</div><div class='ctx'> 	default:</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/mad.h b/drivers/infiniband/hw/hfi1/mad.h<br/>index 2f48e6953629..d6dde762921a 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/mad.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/mad.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/mad.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/mad.h</a></div><div class='hunk'>@@ -1,49 +1,8 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015 - 2017 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #ifndef _HFI1_MAD_H</div><div class='ctx'> #define _HFI1_MAD_H</div><div class='ctx'> </div><div class='hunk'>@@ -165,7 +124,6 @@ struct opa_mad_notice_attr {</div><div class='ctx'> 		} __packed ntc_2048;</div><div class='ctx'> </div><div class='ctx'> 	};</div><div class='del'>-	u8	class_data[0];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> #define IB_VLARB_LOWPRI_0_31    1</div><div class='hunk'>@@ -436,7 +394,7 @@ struct sc2vlnt {</div><div class='ctx'> 		    COUNTER_MASK(1, 3) | \</div><div class='ctx'> 		    COUNTER_MASK(1, 4))</div><div class='ctx'> </div><div class='del'>-void hfi1_event_pkey_change(struct hfi1_devdata *dd, u8 port);</div><div class='add'>+void hfi1_event_pkey_change(struct hfi1_devdata *dd, u32 port);</div><div class='ctx'> void hfi1_handle_trap_timer(struct timer_list *t);</div><div class='ctx'> u16 tx_link_width(u16 link_width);</div><div class='ctx'> u64 get_xmit_wait_counters(struct hfi1_pportdata *ppd, u16 link_width,</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/mmu_rb.c b/drivers/infiniband/hw/hfi1/mmu_rb.c<br/>index 14d2a90964c3..67a5c410fb5e 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/mmu_rb.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/mmu_rb.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/mmu_rb.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/mmu_rb.c</a></div><div class='hunk'>@@ -1,78 +1,25 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='add'>+ * Copyright(c) 2020 Cornelis Networks, Inc.</div><div class='ctx'>  * Copyright(c) 2016 - 2017 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #include &lt;linux/list.h&gt;</div><div class='ctx'> #include &lt;linux/rculist.h&gt;</div><div class='ctx'> #include &lt;linux/mmu_notifier.h&gt;</div><div class='ctx'> #include &lt;linux/interval_tree_generic.h&gt;</div><div class='add'>+#include &lt;linux/sched/mm.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "mmu_rb.h"</div><div class='ctx'> #include "trace.h"</div><div class='ctx'> </div><div class='del'>-struct mmu_rb_handler {</div><div class='del'>-	struct mmu_notifier mn;</div><div class='del'>-	struct rb_root_cached root;</div><div class='del'>-	void *ops_arg;</div><div class='del'>-	spinlock_t lock;        /* protect the RB tree */</div><div class='del'>-	struct mmu_rb_ops *ops;</div><div class='del'>-	struct mm_struct *mm;</div><div class='del'>-	struct list_head lru_list;</div><div class='del'>-	struct work_struct del_work;</div><div class='del'>-	struct list_head del_list;</div><div class='del'>-	struct workqueue_struct *wq;</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> static unsigned long mmu_node_start(struct mmu_rb_node *);</div><div class='ctx'> static unsigned long mmu_node_last(struct mmu_rb_node *);</div><div class='ctx'> static int mmu_notifier_range_start(struct mmu_notifier *,</div><div class='ctx'> 		const struct mmu_notifier_range *);</div><div class='ctx'> static struct mmu_rb_node *__mmu_rb_search(struct mmu_rb_handler *,</div><div class='ctx'> 					   unsigned long, unsigned long);</div><div class='del'>-static void do_remove(struct mmu_rb_handler *handler,</div><div class='del'>-		      struct list_head *del_list);</div><div class='add'>+static void release_immediate(struct kref *refcount);</div><div class='ctx'> static void handle_remove(struct work_struct *work);</div><div class='ctx'> </div><div class='ctx'> static const struct mmu_notifier_ops mn_opts = {</div><div class='hunk'>@@ -92,37 +39,39 @@ static unsigned long mmu_node_last(struct mmu_rb_node *node)</div><div class='ctx'> 	return PAGE_ALIGN(node-&gt;addr + node-&gt;len) - 1;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int hfi1_mmu_rb_register(void *ops_arg, struct mm_struct *mm,</div><div class='del'>-			 struct mmu_rb_ops *ops,</div><div class='add'>+int hfi1_mmu_rb_register(void *ops_arg,</div><div class='add'>+			 const struct mmu_rb_ops *ops,</div><div class='ctx'> 			 struct workqueue_struct *wq,</div><div class='ctx'> 			 struct mmu_rb_handler **handler)</div><div class='ctx'> {</div><div class='del'>-	struct mmu_rb_handler *handlr;</div><div class='add'>+	struct mmu_rb_handler *h;</div><div class='add'>+	void *free_ptr;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	handlr = kmalloc(sizeof(*handlr), GFP_KERNEL);</div><div class='del'>-	if (!handlr)</div><div class='add'>+	free_ptr = kzalloc(sizeof(*h) + cache_line_size() - 1, GFP_KERNEL);</div><div class='add'>+	if (!free_ptr)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	handlr-&gt;root = RB_ROOT_CACHED;</div><div class='del'>-	handlr-&gt;ops = ops;</div><div class='del'>-	handlr-&gt;ops_arg = ops_arg;</div><div class='del'>-	INIT_HLIST_NODE(&amp;handlr-&gt;mn.hlist);</div><div class='del'>-	spin_lock_init(&amp;handlr-&gt;lock);</div><div class='del'>-	handlr-&gt;mn.ops = &amp;mn_opts;</div><div class='del'>-	handlr-&gt;mm = mm;</div><div class='del'>-	INIT_WORK(&amp;handlr-&gt;del_work, handle_remove);</div><div class='del'>-	INIT_LIST_HEAD(&amp;handlr-&gt;del_list);</div><div class='del'>-	INIT_LIST_HEAD(&amp;handlr-&gt;lru_list);</div><div class='del'>-	handlr-&gt;wq = wq;</div><div class='del'>-</div><div class='del'>-	ret = mmu_notifier_register(&amp;handlr-&gt;mn, handlr-&gt;mm);</div><div class='add'>+	h = PTR_ALIGN(free_ptr, cache_line_size());</div><div class='add'>+	h-&gt;root = RB_ROOT_CACHED;</div><div class='add'>+	h-&gt;ops = ops;</div><div class='add'>+	h-&gt;ops_arg = ops_arg;</div><div class='add'>+	INIT_HLIST_NODE(&amp;h-&gt;mn.hlist);</div><div class='add'>+	spin_lock_init(&amp;h-&gt;lock);</div><div class='add'>+	h-&gt;mn.ops = &amp;mn_opts;</div><div class='add'>+	INIT_WORK(&amp;h-&gt;del_work, handle_remove);</div><div class='add'>+	INIT_LIST_HEAD(&amp;h-&gt;del_list);</div><div class='add'>+	INIT_LIST_HEAD(&amp;h-&gt;lru_list);</div><div class='add'>+	h-&gt;wq = wq;</div><div class='add'>+	h-&gt;free_ptr = free_ptr;</div><div class='add'>+</div><div class='add'>+	ret = mmu_notifier_register(&amp;h-&gt;mn, current-&gt;mm);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		kfree(handlr);</div><div class='add'>+		kfree(free_ptr);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	*handler = handlr;</div><div class='add'>+	*handler = h;</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -133,8 +82,11 @@ void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler)</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	struct list_head del_list;</div><div class='ctx'> </div><div class='add'>+	/* Prevent freeing of mm until we are completely finished. */</div><div class='add'>+	mmgrab(handler-&gt;mn.mm);</div><div class='add'>+</div><div class='ctx'> 	/* Unregister first so we don't get any more notifications. */</div><div class='del'>-	mmu_notifier_unregister(&amp;handler-&gt;mn, handler-&gt;mm);</div><div class='add'>+	mmu_notifier_unregister(&amp;handler-&gt;mn, handler-&gt;mn.mm);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Make sure the wq delete handler is finished running.  It will not</div><div class='hunk'>@@ -153,9 +105,16 @@ void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler)</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;handler-&gt;lock, flags);</div><div class='ctx'> </div><div class='del'>-	do_remove(handler, &amp;del_list);</div><div class='add'>+	while (!list_empty(&amp;del_list)) {</div><div class='add'>+		rbnode = list_first_entry(&amp;del_list, struct mmu_rb_node, list);</div><div class='add'>+		list_del(&amp;rbnode-&gt;list);</div><div class='add'>+		kref_put(&amp;rbnode-&gt;refcount, release_immediate);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Now the mm may be freed. */</div><div class='add'>+	mmdrop(handler-&gt;mn.mm);</div><div class='ctx'> </div><div class='del'>-	kfree(handler);</div><div class='add'>+	kfree(handler-&gt;free_ptr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int hfi1_mmu_rb_insert(struct mmu_rb_handler *handler,</div><div class='hunk'>@@ -165,27 +124,39 @@ int hfi1_mmu_rb_insert(struct mmu_rb_handler *handler,</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	trace_hfi1_mmu_rb_insert(mnode-&gt;addr, mnode-&gt;len);</div><div class='add'>+	trace_hfi1_mmu_rb_insert(mnode);</div><div class='add'>+</div><div class='add'>+	if (current-&gt;mm != handler-&gt;mn.mm)</div><div class='add'>+		return -EPERM;</div><div class='add'>+</div><div class='ctx'> 	spin_lock_irqsave(&amp;handler-&gt;lock, flags);</div><div class='ctx'> 	node = __mmu_rb_search(handler, mnode-&gt;addr, mnode-&gt;len);</div><div class='ctx'> 	if (node) {</div><div class='del'>-		ret = -EINVAL;</div><div class='add'>+		ret = -EEXIST;</div><div class='ctx'> 		goto unlock;</div><div class='ctx'> 	}</div><div class='ctx'> 	__mmu_int_rb_insert(mnode, &amp;handler-&gt;root);</div><div class='del'>-	list_add(&amp;mnode-&gt;list, &amp;handler-&gt;lru_list);</div><div class='del'>-</div><div class='del'>-	ret = handler-&gt;ops-&gt;insert(handler-&gt;ops_arg, mnode);</div><div class='del'>-	if (ret) {</div><div class='del'>-		__mmu_int_rb_remove(mnode, &amp;handler-&gt;root);</div><div class='del'>-		list_del(&amp;mnode-&gt;list); /* remove from LRU list */</div><div class='del'>-	}</div><div class='add'>+	list_add_tail(&amp;mnode-&gt;list, &amp;handler-&gt;lru_list);</div><div class='add'>+	mnode-&gt;handler = handler;</div><div class='ctx'> unlock:</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;handler-&gt;lock, flags);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* Caller must hold handler lock */</div><div class='add'>+struct mmu_rb_node *hfi1_mmu_rb_get_first(struct mmu_rb_handler *handler,</div><div class='add'>+					  unsigned long addr, unsigned long len)</div><div class='add'>+{</div><div class='add'>+	struct mmu_rb_node *node;</div><div class='add'>+</div><div class='add'>+	trace_hfi1_mmu_rb_search(addr, len);</div><div class='add'>+	node = __mmu_int_rb_iter_first(&amp;handler-&gt;root, addr, (addr + len) - 1);</div><div class='add'>+	if (node)</div><div class='add'>+		list_move_tail(&amp;node-&gt;list, &amp;handler-&gt;lru_list);</div><div class='add'>+	return node;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* Caller must hold handler lock */</div><div class='ctx'> static struct mmu_rb_node *__mmu_rb_search(struct mmu_rb_handler *handler,</div><div class='ctx'> 					   unsigned long addr,</div><div class='ctx'> 					   unsigned long len)</div><div class='hunk'>@@ -209,27 +180,47 @@ static struct mmu_rb_node *__mmu_rb_search(struct mmu_rb_handler *handler,</div><div class='ctx'> 	return node;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-bool hfi1_mmu_rb_remove_unless_exact(struct mmu_rb_handler *handler,</div><div class='del'>-				     unsigned long addr, unsigned long len,</div><div class='del'>-				     struct mmu_rb_node **rb_node)</div><div class='add'>+/*</div><div class='add'>+ * Must NOT call while holding mnode-&gt;handler-&gt;lock.</div><div class='add'>+ * mnode-&gt;handler-&gt;ops-&gt;remove() may sleep and mnode-&gt;handler-&gt;lock is a</div><div class='add'>+ * spinlock.</div><div class='add'>+ */</div><div class='add'>+static void release_immediate(struct kref *refcount)</div><div class='ctx'> {</div><div class='del'>-	struct mmu_rb_node *node;</div><div class='add'>+	struct mmu_rb_node *mnode =</div><div class='add'>+		container_of(refcount, struct mmu_rb_node, refcount);</div><div class='add'>+	trace_hfi1_mmu_release_node(mnode);</div><div class='add'>+	mnode-&gt;handler-&gt;ops-&gt;remove(mnode-&gt;handler-&gt;ops_arg, mnode);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* Caller must hold mnode-&gt;handler-&gt;lock */</div><div class='add'>+static void release_nolock(struct kref *refcount)</div><div class='add'>+{</div><div class='add'>+	struct mmu_rb_node *mnode =</div><div class='add'>+		container_of(refcount, struct mmu_rb_node, refcount);</div><div class='add'>+	list_move(&amp;mnode-&gt;list, &amp;mnode-&gt;handler-&gt;del_list);</div><div class='add'>+	queue_work(mnode-&gt;handler-&gt;wq, &amp;mnode-&gt;handler-&gt;del_work);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * struct mmu_rb_node-&gt;refcount kref_put() callback.</div><div class='add'>+ * Adds mmu_rb_node to mmu_rb_node-&gt;handler-&gt;del_list and queues</div><div class='add'>+ * handler-&gt;del_work on handler-&gt;wq.</div><div class='add'>+ * Does not remove mmu_rb_node from handler-&gt;lru_list or handler-&gt;rb_root.</div><div class='add'>+ * Acquires mmu_rb_node-&gt;handler-&gt;lock; do not call while already holding</div><div class='add'>+ * handler-&gt;lock.</div><div class='add'>+ */</div><div class='add'>+void hfi1_mmu_rb_release(struct kref *refcount)</div><div class='add'>+{</div><div class='add'>+	struct mmu_rb_node *mnode =</div><div class='add'>+		container_of(refcount, struct mmu_rb_node, refcount);</div><div class='add'>+	struct mmu_rb_handler *handler = mnode-&gt;handler;</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	bool ret = false;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;handler-&gt;lock, flags);</div><div class='del'>-	node = __mmu_rb_search(handler, addr, len);</div><div class='del'>-	if (node) {</div><div class='del'>-		if (node-&gt;addr == addr &amp;&amp; node-&gt;len == len)</div><div class='del'>-			goto unlock;</div><div class='del'>-		__mmu_int_rb_remove(node, &amp;handler-&gt;root);</div><div class='del'>-		list_del(&amp;node-&gt;list); /* remove from LRU list */</div><div class='del'>-		ret = true;</div><div class='del'>-	}</div><div class='del'>-unlock:</div><div class='add'>+	list_move(&amp;mnode-&gt;list, &amp;mnode-&gt;handler-&gt;del_list);</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;handler-&gt;lock, flags);</div><div class='del'>-	*rb_node = node;</div><div class='del'>-	return ret;</div><div class='add'>+	queue_work(handler-&gt;wq, &amp;handler-&gt;del_work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void hfi1_mmu_rb_evict(struct mmu_rb_handler *handler, void *evict_arg)</div><div class='hunk'>@@ -239,11 +230,17 @@ void hfi1_mmu_rb_evict(struct mmu_rb_handler *handler, void *evict_arg)</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	bool stop = false;</div><div class='ctx'> </div><div class='add'>+	if (current-&gt;mm != handler-&gt;mn.mm)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;del_list);</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;handler-&gt;lock, flags);</div><div class='del'>-	list_for_each_entry_safe_reverse(rbnode, ptr, &amp;handler-&gt;lru_list,</div><div class='del'>-					 list) {</div><div class='add'>+	list_for_each_entry_safe(rbnode, ptr, &amp;handler-&gt;lru_list, list) {</div><div class='add'>+		/* refcount == 1 implies mmu_rb_handler has only rbnode ref */</div><div class='add'>+		if (kref_read(&amp;rbnode-&gt;refcount) &gt; 1)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='ctx'> 		if (handler-&gt;ops-&gt;evict(handler-&gt;ops_arg, rbnode, evict_arg,</div><div class='ctx'> 					&amp;stop)) {</div><div class='ctx'> 			__mmu_int_rb_remove(rbnode, &amp;handler-&gt;root);</div><div class='hunk'>@@ -255,33 +252,12 @@ void hfi1_mmu_rb_evict(struct mmu_rb_handler *handler, void *evict_arg)</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;handler-&gt;lock, flags);</div><div class='ctx'> </div><div class='del'>-	while (!list_empty(&amp;del_list)) {</div><div class='del'>-		rbnode = list_first_entry(&amp;del_list, struct mmu_rb_node, list);</div><div class='del'>-		list_del(&amp;rbnode-&gt;list);</div><div class='del'>-		handler-&gt;ops-&gt;remove(handler-&gt;ops_arg, rbnode);</div><div class='add'>+	list_for_each_entry_safe(rbnode, ptr, &amp;del_list, list) {</div><div class='add'>+		trace_hfi1_mmu_rb_evict(rbnode);</div><div class='add'>+		kref_put(&amp;rbnode-&gt;refcount, release_immediate);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/*</div><div class='del'>- * It is up to the caller to ensure that this function does not race with the</div><div class='del'>- * mmu invalidate notifier which may be calling the users remove callback on</div><div class='del'>- * 'node'.</div><div class='del'>- */</div><div class='del'>-void hfi1_mmu_rb_remove(struct mmu_rb_handler *handler,</div><div class='del'>-			struct mmu_rb_node *node)</div><div class='del'>-{</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	/* Validity of handler and node pointers has been checked by caller. */</div><div class='del'>-	trace_hfi1_mmu_rb_remove(node-&gt;addr, node-&gt;len);</div><div class='del'>-	spin_lock_irqsave(&amp;handler-&gt;lock, flags);</div><div class='del'>-	__mmu_int_rb_remove(node, &amp;handler-&gt;root);</div><div class='del'>-	list_del(&amp;node-&gt;list); /* remove from LRU list */</div><div class='del'>-	spin_unlock_irqrestore(&amp;handler-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-	handler-&gt;ops-&gt;remove(handler-&gt;ops_arg, node);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int mmu_notifier_range_start(struct mmu_notifier *mn,</div><div class='ctx'> 		const struct mmu_notifier_range *range)</div><div class='ctx'> {</div><div class='hunk'>@@ -290,7 +266,6 @@ static int mmu_notifier_range_start(struct mmu_notifier *mn,</div><div class='ctx'> 	struct rb_root_cached *root = &amp;handler-&gt;root;</div><div class='ctx'> 	struct mmu_rb_node *node, *ptr = NULL;</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	bool added = false;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;handler-&gt;lock, flags);</div><div class='ctx'> 	for (node = __mmu_int_rb_iter_first(root, range-&gt;start, range-&gt;end-1);</div><div class='hunk'>@@ -298,42 +273,20 @@ static int mmu_notifier_range_start(struct mmu_notifier *mn,</div><div class='ctx'> 		/* Guard against node removal. */</div><div class='ctx'> 		ptr = __mmu_int_rb_iter_next(node, range-&gt;start,</div><div class='ctx'> 					     range-&gt;end - 1);</div><div class='del'>-		trace_hfi1_mmu_mem_invalidate(node-&gt;addr, node-&gt;len);</div><div class='del'>-		if (handler-&gt;ops-&gt;invalidate(handler-&gt;ops_arg, node)) {</div><div class='del'>-			__mmu_int_rb_remove(node, root);</div><div class='del'>-			/* move from LRU list to delete list */</div><div class='del'>-			list_move(&amp;node-&gt;list, &amp;handler-&gt;del_list);</div><div class='del'>-			added = true;</div><div class='del'>-		}</div><div class='add'>+		trace_hfi1_mmu_mem_invalidate(node);</div><div class='add'>+		/* Remove from rb tree and lru_list. */</div><div class='add'>+		__mmu_int_rb_remove(node, root);</div><div class='add'>+		list_del_init(&amp;node-&gt;list);</div><div class='add'>+		kref_put(&amp;node-&gt;refcount, release_nolock);</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;handler-&gt;lock, flags);</div><div class='ctx'> </div><div class='del'>-	if (added)</div><div class='del'>-		queue_work(handler-&gt;wq, &amp;handler-&gt;del_work);</div><div class='del'>-</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='del'>- * Call the remove function for the given handler and the list.  This</div><div class='del'>- * is expected to be called with a delete list extracted from handler.</div><div class='del'>- * The caller should not be holding the handler lock.</div><div class='del'>- */</div><div class='del'>-static void do_remove(struct mmu_rb_handler *handler,</div><div class='del'>-		      struct list_head *del_list)</div><div class='del'>-{</div><div class='del'>-	struct mmu_rb_node *node;</div><div class='del'>-</div><div class='del'>-	while (!list_empty(del_list)) {</div><div class='del'>-		node = list_first_entry(del_list, struct mmu_rb_node, list);</div><div class='del'>-		list_del(&amp;node-&gt;list);</div><div class='del'>-		handler-&gt;ops-&gt;remove(handler-&gt;ops_arg, node);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='ctx'>  * Work queue function to remove all nodes that have been queued up to</div><div class='del'>- * be removed.  The key feature is that mm-&gt;mmap_sem is not being held</div><div class='add'>+ * be removed.  The key feature is that mm-&gt;mmap_lock is not being held</div><div class='ctx'>  * and the remove callback can sleep while taking it, if needed.</div><div class='ctx'>  */</div><div class='ctx'> static void handle_remove(struct work_struct *work)</div><div class='hunk'>@@ -343,11 +296,17 @@ static void handle_remove(struct work_struct *work)</div><div class='ctx'> 						del_work);</div><div class='ctx'> 	struct list_head del_list;</div><div class='ctx'> 	unsigned long flags;</div><div class='add'>+	struct mmu_rb_node *node;</div><div class='ctx'> </div><div class='ctx'> 	/* remove anything that is queued to get removed */</div><div class='ctx'> 	spin_lock_irqsave(&amp;handler-&gt;lock, flags);</div><div class='ctx'> 	list_replace_init(&amp;handler-&gt;del_list, &amp;del_list);</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;handler-&gt;lock, flags);</div><div class='ctx'> </div><div class='del'>-	do_remove(handler, &amp;del_list);</div><div class='add'>+	while (!list_empty(&amp;del_list)) {</div><div class='add'>+		node = list_first_entry(&amp;del_list, struct mmu_rb_node, list);</div><div class='add'>+		list_del(&amp;node-&gt;list);</div><div class='add'>+		trace_hfi1_mmu_release_node(node);</div><div class='add'>+		handler-&gt;ops-&gt;remove(handler-&gt;ops_arg, node);</div><div class='add'>+	}</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/mmu_rb.h b/drivers/infiniband/hw/hfi1/mmu_rb.h<br/>index f04cec1e99d1..3fa50dd64db6 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/mmu_rb.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/mmu_rb.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/mmu_rb.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/mmu_rb.h</a></div><div class='hunk'>@@ -1,49 +1,9 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='add'>+ * Copyright(c) 2020 Cornelis Networks, Inc.</div><div class='ctx'>  * Copyright(c) 2016 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #ifndef _HFI1_MMU_RB_H</div><div class='ctx'> #define _HFI1_MMU_RB_H</div><div class='ctx'> </div><div class='hunk'>@@ -54,35 +14,54 @@ struct mmu_rb_node {</div><div class='ctx'> 	unsigned long len;</div><div class='ctx'> 	unsigned long __last;</div><div class='ctx'> 	struct rb_node node;</div><div class='add'>+	struct mmu_rb_handler *handler;</div><div class='ctx'> 	struct list_head list;</div><div class='add'>+	struct kref refcount;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/*</div><div class='del'>- * NOTE: filter, insert, invalidate, and evict must not sleep.  Only remove is</div><div class='del'>- * allowed to sleep.</div><div class='del'>- */</div><div class='add'>+/* filter and evict must not sleep. Only remove is allowed to sleep. */</div><div class='ctx'> struct mmu_rb_ops {</div><div class='ctx'> 	bool (*filter)(struct mmu_rb_node *node, unsigned long addr,</div><div class='ctx'> 		       unsigned long len);</div><div class='del'>-	int (*insert)(void *ops_arg, struct mmu_rb_node *mnode);</div><div class='ctx'> 	void (*remove)(void *ops_arg, struct mmu_rb_node *mnode);</div><div class='del'>-	int (*invalidate)(void *ops_arg, struct mmu_rb_node *node);</div><div class='ctx'> 	int (*evict)(void *ops_arg, struct mmu_rb_node *mnode,</div><div class='ctx'> 		     void *evict_arg, bool *stop);</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-int hfi1_mmu_rb_register(void *ops_arg, struct mm_struct *mm,</div><div class='del'>-			 struct mmu_rb_ops *ops,</div><div class='add'>+struct mmu_rb_handler {</div><div class='add'>+	/*</div><div class='add'>+	 * struct mmu_notifier is 56 bytes, and spinlock_t is 4 bytes, so</div><div class='add'>+	 * they fit together in one cache line.  mn is relatively rarely</div><div class='add'>+	 * accessed, so co-locating the spinlock with it achieves much of</div><div class='add'>+	 * the cacheline contention reduction of giving the spinlock its own</div><div class='add'>+	 * cacheline without the overhead of doing so.</div><div class='add'>+	 */</div><div class='add'>+	struct mmu_notifier mn;</div><div class='add'>+	spinlock_t lock;        /* protect the RB tree */</div><div class='add'>+</div><div class='add'>+	/* Begin on a new cachline boundary here */</div><div class='add'>+	struct rb_root_cached root ____cacheline_aligned_in_smp;</div><div class='add'>+	void *ops_arg;</div><div class='add'>+	const struct mmu_rb_ops *ops;</div><div class='add'>+	struct list_head lru_list;</div><div class='add'>+	struct work_struct del_work;</div><div class='add'>+	struct list_head del_list;</div><div class='add'>+	struct workqueue_struct *wq;</div><div class='add'>+	void *free_ptr;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+int hfi1_mmu_rb_register(void *ops_arg,</div><div class='add'>+			 const struct mmu_rb_ops *ops,</div><div class='ctx'> 			 struct workqueue_struct *wq,</div><div class='ctx'> 			 struct mmu_rb_handler **handler);</div><div class='ctx'> void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler);</div><div class='ctx'> int hfi1_mmu_rb_insert(struct mmu_rb_handler *handler,</div><div class='ctx'> 		       struct mmu_rb_node *mnode);</div><div class='add'>+void hfi1_mmu_rb_release(struct kref *refcount);</div><div class='add'>+</div><div class='ctx'> void hfi1_mmu_rb_evict(struct mmu_rb_handler *handler, void *evict_arg);</div><div class='del'>-void hfi1_mmu_rb_remove(struct mmu_rb_handler *handler,</div><div class='del'>-			struct mmu_rb_node *mnode);</div><div class='del'>-bool hfi1_mmu_rb_remove_unless_exact(struct mmu_rb_handler *handler,</div><div class='del'>-				     unsigned long addr, unsigned long len,</div><div class='del'>-				     struct mmu_rb_node **rb_node);</div><div class='add'>+struct mmu_rb_node *hfi1_mmu_rb_get_first(struct mmu_rb_handler *handler,</div><div class='add'>+					  unsigned long addr,</div><div class='add'>+					  unsigned long len);</div><div class='ctx'> </div><div class='ctx'> #endif /* _HFI1_MMU_RB_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/msix.c b/drivers/infiniband/hw/hfi1/msix.c<br/>index d920b165d696..77d2ece9a9cb 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/msix.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/msix.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/msix.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/msix.c</a></div><div class='hunk'>@@ -1,54 +1,12 @@</div><div class='ctx'> // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2018 - 2020 Intel Corporation.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include "hfi.h"</div><div class='ctx'> #include "affinity.h"</div><div class='ctx'> #include "sdma.h"</div><div class='add'>+#include "netdev.h"</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * msix_initialize() - Calculate, request and configure MSIx IRQs</div><div class='hunk'>@@ -69,7 +27,7 @@ int msix_initialize(struct hfi1_devdata *dd)</div><div class='ctx'> 	 *	one for each VNIC context</div><div class='ctx'> 	 *      ...any new IRQs should be added here.</div><div class='ctx'> 	 */</div><div class='del'>-	total = 1 + dd-&gt;num_sdma + dd-&gt;n_krcv_queues + dd-&gt;num_vnic_contexts;</div><div class='add'>+	total = 1 + dd-&gt;num_sdma + dd-&gt;n_krcv_queues + dd-&gt;num_netdev_contexts;</div><div class='ctx'> </div><div class='ctx'> 	if (total &gt;= CCE_NUM_MSIX_VECTORS)</div><div class='ctx'> 		return -EINVAL;</div><div class='hunk'>@@ -102,8 +60,8 @@ int msix_initialize(struct hfi1_devdata *dd)</div><div class='ctx'>  * @arg: context information for the IRQ</div><div class='ctx'>  * @handler: IRQ handler</div><div class='ctx'>  * @thread: IRQ thread handler (could be NULL)</div><div class='del'>- * @idx: zero base idx if multiple devices are needed</div><div class='ctx'>  * @type: affinty IRQ type</div><div class='add'>+ * @name: IRQ name</div><div class='ctx'>  *</div><div class='ctx'>  * Allocated an MSIx vector if available, and then create the appropriate</div><div class='ctx'>  * meta data needed to keep track of the pci IRQ request.</div><div class='hunk'>@@ -115,13 +73,11 @@ int msix_initialize(struct hfi1_devdata *dd)</div><div class='ctx'>  */</div><div class='ctx'> static int msix_request_irq(struct hfi1_devdata *dd, void *arg,</div><div class='ctx'> 			    irq_handler_t handler, irq_handler_t thread,</div><div class='del'>-			    u32 idx, enum irq_type type)</div><div class='add'>+			    enum irq_type type, const char *name)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned long nr;</div><div class='ctx'> 	int irq;</div><div class='ctx'> 	int ret;</div><div class='del'>-	const char *err_info;</div><div class='del'>-	char name[MAX_NAME_SIZE];</div><div class='ctx'> 	struct hfi1_msix_entry *me;</div><div class='ctx'> </div><div class='ctx'> 	/* Allocate an MSIx vector */</div><div class='hunk'>@@ -135,43 +91,15 @@ static int msix_request_irq(struct hfi1_devdata *dd, void *arg,</div><div class='ctx'> 	if (nr == dd-&gt;msix_info.max_requested)</div><div class='ctx'> 		return -ENOSPC;</div><div class='ctx'> </div><div class='del'>-	/* Specific verification and determine the name */</div><div class='del'>-	switch (type) {</div><div class='del'>-	case IRQ_GENERAL:</div><div class='del'>-		/* general interrupt must be MSIx vector 0 */</div><div class='del'>-		if (nr) {</div><div class='del'>-			spin_lock(&amp;dd-&gt;msix_info.msix_lock);</div><div class='del'>-			__clear_bit(nr, dd-&gt;msix_info.in_use_msix);</div><div class='del'>-			spin_unlock(&amp;dd-&gt;msix_info.msix_lock);</div><div class='del'>-			dd_dev_err(dd, "Invalid index %lu for GENERAL IRQ\n",</div><div class='del'>-				   nr);</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-		snprintf(name, sizeof(name), DRIVER_NAME "_%d", dd-&gt;unit);</div><div class='del'>-		err_info = "general";</div><div class='del'>-		break;</div><div class='del'>-	case IRQ_SDMA:</div><div class='del'>-		snprintf(name, sizeof(name), DRIVER_NAME "_%d sdma%d",</div><div class='del'>-			 dd-&gt;unit, idx);</div><div class='del'>-		err_info = "sdma";</div><div class='del'>-		break;</div><div class='del'>-	case IRQ_RCVCTXT:</div><div class='del'>-		snprintf(name, sizeof(name), DRIVER_NAME "_%d kctxt%d",</div><div class='del'>-			 dd-&gt;unit, idx);</div><div class='del'>-		err_info = "receive context";</div><div class='del'>-		break;</div><div class='del'>-	case IRQ_OTHER:</div><div class='del'>-	default:</div><div class='add'>+	if (type &lt; IRQ_SDMA || type &gt;= IRQ_OTHER)</div><div class='ctx'> 		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-	name[sizeof(name) - 1] = 0;</div><div class='ctx'> </div><div class='ctx'> 	irq = pci_irq_vector(dd-&gt;pcidev, nr);</div><div class='ctx'> 	ret = pci_request_irq(dd-&gt;pcidev, nr, handler, thread, arg, name);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		dd_dev_err(dd,</div><div class='del'>-			   "%s: request for IRQ %d failed, MSIx %d, err %d\n",</div><div class='del'>-			   err_info, irq, idx, ret);</div><div class='add'>+			   "%s: request for IRQ %d failed, MSIx %lx, err %d\n",</div><div class='add'>+			   name, irq, nr, ret);</div><div class='ctx'> 		spin_lock(&amp;dd-&gt;msix_info.msix_lock);</div><div class='ctx'> 		__clear_bit(nr, dd-&gt;msix_info.in_use_msix);</div><div class='ctx'> 		spin_unlock(&amp;dd-&gt;msix_info.msix_lock);</div><div class='hunk'>@@ -190,22 +118,19 @@ static int msix_request_irq(struct hfi1_devdata *dd, void *arg,</div><div class='ctx'> 	/* This is a request, so a failure is not fatal */</div><div class='ctx'> 	ret = hfi1_get_irq_affinity(dd, me);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		dd_dev_err(dd, "unable to pin IRQ %d\n", ret);</div><div class='add'>+		dd_dev_err(dd, "%s: unable to pin IRQ %d\n", name, ret);</div><div class='ctx'> </div><div class='ctx'> 	return nr;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/**</div><div class='del'>- * msix_request_rcd_irq() - Helper function for RCVAVAIL IRQs</div><div class='del'>- * @rcd: valid rcd context</div><div class='del'>- *</div><div class='del'>- */</div><div class='del'>-int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd)</div><div class='add'>+static int msix_request_rcd_irq_common(struct hfi1_ctxtdata *rcd,</div><div class='add'>+				       irq_handler_t handler,</div><div class='add'>+				       irq_handler_t thread,</div><div class='add'>+				       const char *name)</div><div class='ctx'> {</div><div class='del'>-	int nr;</div><div class='del'>-</div><div class='del'>-	nr = msix_request_irq(rcd-&gt;dd, rcd, receive_context_interrupt,</div><div class='del'>-			      receive_context_thread, rcd-&gt;ctxt, IRQ_RCVCTXT);</div><div class='add'>+	int nr = msix_request_irq(rcd-&gt;dd, rcd, handler, thread,</div><div class='add'>+				  rcd-&gt;is_vnic ? IRQ_NETDEVCTXT : IRQ_RCVCTXT,</div><div class='add'>+				  name);</div><div class='ctx'> 	if (nr &lt; 0)</div><div class='ctx'> 		return nr;</div><div class='ctx'> </div><div class='hunk'>@@ -222,16 +147,50 @@ int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * msix_request_smda_ira() - Helper for getting SDMA IRQ resources</div><div class='add'>+ * msix_request_rcd_irq() - Helper function for RCVAVAIL IRQs</div><div class='add'>+ * @rcd: valid rcd context</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	char name[MAX_NAME_SIZE];</div><div class='add'>+</div><div class='add'>+	snprintf(name, sizeof(name), DRIVER_NAME "_%d kctxt%d",</div><div class='add'>+		 rcd-&gt;dd-&gt;unit, rcd-&gt;ctxt);</div><div class='add'>+</div><div class='add'>+	return msix_request_rcd_irq_common(rcd, receive_context_interrupt,</div><div class='add'>+					   receive_context_thread, name);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * msix_netdev_request_rcd_irq  - Helper function for RCVAVAIL IRQs</div><div class='add'>+ * for netdev context</div><div class='add'>+ * @rcd: valid netdev contexti</div><div class='add'>+ */</div><div class='add'>+int msix_netdev_request_rcd_irq(struct hfi1_ctxtdata *rcd)</div><div class='add'>+{</div><div class='add'>+	char name[MAX_NAME_SIZE];</div><div class='add'>+</div><div class='add'>+	snprintf(name, sizeof(name), DRIVER_NAME "_%d nd kctxt%d",</div><div class='add'>+		 rcd-&gt;dd-&gt;unit, rcd-&gt;ctxt);</div><div class='add'>+	return msix_request_rcd_irq_common(rcd, receive_context_interrupt_napi,</div><div class='add'>+					   NULL, name);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * msix_request_sdma_irq  - Helper for getting SDMA IRQ resources</div><div class='ctx'>  * @sde: valid sdma engine</div><div class='ctx'>  *</div><div class='ctx'>  */</div><div class='ctx'> int msix_request_sdma_irq(struct sdma_engine *sde)</div><div class='ctx'> {</div><div class='ctx'> 	int nr;</div><div class='add'>+	char name[MAX_NAME_SIZE];</div><div class='ctx'> </div><div class='add'>+	snprintf(name, sizeof(name), DRIVER_NAME "_%d sdma%d",</div><div class='add'>+		 sde-&gt;dd-&gt;unit, sde-&gt;this_idx);</div><div class='ctx'> 	nr = msix_request_irq(sde-&gt;dd, sde, sdma_interrupt, NULL,</div><div class='del'>-			      sde-&gt;this_idx, IRQ_SDMA);</div><div class='add'>+			      IRQ_SDMA, name);</div><div class='ctx'> 	if (nr &lt; 0)</div><div class='ctx'> 		return nr;</div><div class='ctx'> 	sde-&gt;msix_intr = nr;</div><div class='hunk'>@@ -241,7 +200,33 @@ int msix_request_sdma_irq(struct sdma_engine *sde)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * enable_sdma_src() - Helper to enable SDMA IRQ srcs</div><div class='add'>+ * msix_request_general_irq - Helper for getting general IRQ</div><div class='add'>+ * resources</div><div class='add'>+ * @dd: valid device data</div><div class='add'>+ */</div><div class='add'>+int msix_request_general_irq(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	int nr;</div><div class='add'>+	char name[MAX_NAME_SIZE];</div><div class='add'>+</div><div class='add'>+	snprintf(name, sizeof(name), DRIVER_NAME "_%d", dd-&gt;unit);</div><div class='add'>+	nr = msix_request_irq(dd, dd, general_interrupt, NULL, IRQ_GENERAL,</div><div class='add'>+			      name);</div><div class='add'>+	if (nr &lt; 0)</div><div class='add'>+		return nr;</div><div class='add'>+</div><div class='add'>+	/* general interrupt must be MSIx vector 0 */</div><div class='add'>+	if (nr) {</div><div class='add'>+		msix_free_irq(dd, (u8)nr);</div><div class='add'>+		dd_dev_err(dd, "Invalid index %d for GENERAL IRQ\n", nr);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * enable_sdma_srcs - Helper to enable SDMA IRQ srcs</div><div class='ctx'>  * @dd: valid devdata structure</div><div class='ctx'>  * @i: index of SDMA engine</div><div class='ctx'>  */</div><div class='hunk'>@@ -265,10 +250,9 @@ static void enable_sdma_srcs(struct hfi1_devdata *dd, int i)</div><div class='ctx'> int msix_request_irqs(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='del'>-	int ret;</div><div class='add'>+	int ret = msix_request_general_irq(dd);</div><div class='ctx'> </div><div class='del'>-	ret = msix_request_irq(dd, dd, general_interrupt, NULL, 0, IRQ_GENERAL);</div><div class='del'>-	if (ret &lt; 0)</div><div class='add'>+	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; dd-&gt;num_sdma; i++) {</div><div class='hunk'>@@ -322,7 +306,7 @@ void msix_free_irq(struct hfi1_devdata *dd, u8 msix_intr)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * hfi1_clean_up_msix_interrupts() - Free all MSIx IRQ resources</div><div class='add'>+ * msix_clean_up_interrupts  - Free all MSIx IRQ resources</div><div class='ctx'>  * @dd: valid device data data structure</div><div class='ctx'>  *</div><div class='ctx'>  * Free the MSIx and associated PCI resources, if they have been allocated.</div><div class='hunk'>@@ -345,15 +329,16 @@ void msix_clean_up_interrupts(struct hfi1_devdata *dd)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * msix_vnic_syncrhonize_irq() - Vnic IRQ synchronize</div><div class='add'>+ * msix_netdev_synchronize_irq - netdev IRQ synchronize</div><div class='ctx'>  * @dd: valid devdata</div><div class='ctx'>  */</div><div class='del'>-void msix_vnic_synchronize_irq(struct hfi1_devdata *dd)</div><div class='add'>+void msix_netdev_synchronize_irq(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='add'>+	int ctxt_count = hfi1_netdev_ctxt_count(dd);</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; dd-&gt;vnic.num_ctxt; i++) {</div><div class='del'>-		struct hfi1_ctxtdata *rcd = dd-&gt;vnic.ctxt[i];</div><div class='add'>+	for (i = 0; i &lt; ctxt_count; i++) {</div><div class='add'>+		struct hfi1_ctxtdata *rcd = hfi1_netdev_get_ctxt(dd, i);</div><div class='ctx'> 		struct hfi1_msix_entry *me;</div><div class='ctx'> </div><div class='ctx'> 		me = &amp;dd-&gt;msix_info.msix_entries[rcd-&gt;msix_intr];</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/msix.h b/drivers/infiniband/hw/hfi1/msix.h<br/>index a514881632a4..9530ccb0a2ce 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/msix.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/msix.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/msix.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/msix.h</a></div><div class='hunk'>@@ -1,50 +1,8 @@</div><div class='ctx'> /* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2018 - 2020 Intel Corporation.</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #ifndef _HFI1_MSIX_H</div><div class='ctx'> #define _HFI1_MSIX_H</div><div class='ctx'> </div><div class='hunk'>@@ -54,11 +12,13 @@</div><div class='ctx'> int msix_initialize(struct hfi1_devdata *dd);</div><div class='ctx'> int msix_request_irqs(struct hfi1_devdata *dd);</div><div class='ctx'> void msix_clean_up_interrupts(struct hfi1_devdata *dd);</div><div class='add'>+int msix_request_general_irq(struct hfi1_devdata *dd);</div><div class='ctx'> int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd);</div><div class='ctx'> int msix_request_sdma_irq(struct sdma_engine *sde);</div><div class='ctx'> void msix_free_irq(struct hfi1_devdata *dd, u8 msix_intr);</div><div class='ctx'> </div><div class='del'>-/* VNIC interface */</div><div class='del'>-void msix_vnic_synchronize_irq(struct hfi1_devdata *dd);</div><div class='add'>+/* Netdev interface */</div><div class='add'>+void msix_netdev_synchronize_irq(struct hfi1_devdata *dd);</div><div class='add'>+int msix_netdev_request_rcd_irq(struct hfi1_ctxtdata *rcd);</div><div class='ctx'> </div><div class='ctx'> #endif</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/netdev.h b/drivers/infiniband/hw/hfi1/netdev.h<br/>new file mode 100644<br/>index 000000000000..07c8f77c9181<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/netdev.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/netdev.h</a></div><div class='hunk'>@@ -0,0 +1,105 @@</div><div class='add'>+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */</div><div class='add'>+/*</div><div class='add'>+ * Copyright(c) 2020 Intel Corporation.</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef HFI1_NETDEV_H</div><div class='add'>+#define HFI1_NETDEV_H</div><div class='add'>+</div><div class='add'>+#include "hfi.h"</div><div class='add'>+</div><div class='add'>+#include &lt;linux/netdevice.h&gt;</div><div class='add'>+#include &lt;linux/xarray.h&gt;</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * struct hfi1_netdev_rxq - Receive Queue for HFI</div><div class='add'>+ * Both IPoIB and VNIC netdevices will be working on the rx abstraction.</div><div class='add'>+ * @napi: napi object</div><div class='add'>+ * @rx: ptr to netdev_rx</div><div class='add'>+ * @rcd:  ptr to receive context data</div><div class='add'>+ */</div><div class='add'>+struct hfi1_netdev_rxq {</div><div class='add'>+	struct napi_struct napi;</div><div class='add'>+	struct hfi1_netdev_rx *rx;</div><div class='add'>+	struct hfi1_ctxtdata *rcd;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * Number of netdev contexts used. Ensure it is less than or equal to</div><div class='add'>+ * max queues supported by VNIC (HFI1_VNIC_MAX_QUEUE).</div><div class='add'>+ */</div><div class='add'>+#define HFI1_MAX_NETDEV_CTXTS   8</div><div class='add'>+</div><div class='add'>+/* Number of NETDEV RSM entries */</div><div class='add'>+#define NUM_NETDEV_MAP_ENTRIES HFI1_MAX_NETDEV_CTXTS</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * struct hfi1_netdev_rx: data required to setup and run HFI netdev.</div><div class='add'>+ * @rx_napi:	the dummy netdevice to support "polling" the receive contexts</div><div class='add'>+ * @dd:		hfi1_devdata</div><div class='add'>+ * @rxq:	pointer to dummy netdev receive queues.</div><div class='add'>+ * @num_rx_q:	number of receive queues</div><div class='add'>+ * @rmt_index:	first free index in RMT Array</div><div class='add'>+ * @msix_start: first free MSI-X interrupt vector.</div><div class='add'>+ * @dev_tbl:	netdev table for unique identifier VNIC and IPoIb VLANs.</div><div class='add'>+ * @enabled:	atomic counter of netdevs enabling receive queues.</div><div class='add'>+ *		When 0 NAPI will be disabled.</div><div class='add'>+ * @netdevs:	atomic counter of netdevs using dummy netdev.</div><div class='add'>+ *		When 0 receive queues will be freed.</div><div class='add'>+ */</div><div class='add'>+struct hfi1_netdev_rx {</div><div class='add'>+	struct net_device *rx_napi;</div><div class='add'>+	struct hfi1_devdata *dd;</div><div class='add'>+	struct hfi1_netdev_rxq *rxq;</div><div class='add'>+	int num_rx_q;</div><div class='add'>+	int rmt_start;</div><div class='add'>+	struct xarray dev_tbl;</div><div class='add'>+	/* count of enabled napi polls */</div><div class='add'>+	atomic_t enabled;</div><div class='add'>+	/* count of netdevs on top */</div><div class='add'>+	atomic_t netdevs;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline</div><div class='add'>+int hfi1_netdev_ctxt_count(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	return dd-&gt;netdev_rx-&gt;num_rx_q;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline</div><div class='add'>+struct hfi1_ctxtdata *hfi1_netdev_get_ctxt(struct hfi1_devdata *dd, int ctxt)</div><div class='add'>+{</div><div class='add'>+	return dd-&gt;netdev_rx-&gt;rxq[ctxt].rcd;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline</div><div class='add'>+int hfi1_netdev_get_free_rmt_idx(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	return dd-&gt;netdev_rx-&gt;rmt_start;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline</div><div class='add'>+void hfi1_netdev_set_free_rmt_idx(struct hfi1_devdata *dd, int rmt_idx)</div><div class='add'>+{</div><div class='add'>+	dd-&gt;netdev_rx-&gt;rmt_start = rmt_idx;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+u32 hfi1_num_netdev_contexts(struct hfi1_devdata *dd, u32 available_contexts,</div><div class='add'>+			     struct cpumask *cpu_mask);</div><div class='add'>+</div><div class='add'>+void hfi1_netdev_enable_queues(struct hfi1_devdata *dd);</div><div class='add'>+void hfi1_netdev_disable_queues(struct hfi1_devdata *dd);</div><div class='add'>+int hfi1_netdev_rx_init(struct hfi1_devdata *dd);</div><div class='add'>+int hfi1_netdev_rx_destroy(struct hfi1_devdata *dd);</div><div class='add'>+int hfi1_alloc_rx(struct hfi1_devdata *dd);</div><div class='add'>+void hfi1_free_rx(struct hfi1_devdata *dd);</div><div class='add'>+int hfi1_netdev_add_data(struct hfi1_devdata *dd, int id, void *data);</div><div class='add'>+void *hfi1_netdev_remove_data(struct hfi1_devdata *dd, int id);</div><div class='add'>+void *hfi1_netdev_get_data(struct hfi1_devdata *dd, int id);</div><div class='add'>+void *hfi1_netdev_get_first_data(struct hfi1_devdata *dd, int *start_id);</div><div class='add'>+</div><div class='add'>+/* chip.c  */</div><div class='add'>+int hfi1_netdev_rx_napi(struct napi_struct *napi, int budget);</div><div class='add'>+</div><div class='add'>+#endif /* HFI1_NETDEV_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/netdev_rx.c b/drivers/infiniband/hw/hfi1/netdev_rx.c<br/>new file mode 100644<br/>index 000000000000..8608044203bb<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/netdev_rx.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/netdev_rx.c</a></div><div class='hunk'>@@ -0,0 +1,487 @@</div><div class='add'>+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)</div><div class='add'>+/*</div><div class='add'>+ * Copyright(c) 2020 Intel Corporation.</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * This file contains HFI1 support for netdev RX functionality</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include "sdma.h"</div><div class='add'>+#include "verbs.h"</div><div class='add'>+#include "netdev.h"</div><div class='add'>+#include "hfi.h"</div><div class='add'>+</div><div class='add'>+#include &lt;linux/netdevice.h&gt;</div><div class='add'>+#include &lt;linux/etherdevice.h&gt;</div><div class='add'>+#include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+</div><div class='add'>+static int hfi1_netdev_setup_ctxt(struct hfi1_netdev_rx *rx,</div><div class='add'>+				  struct hfi1_ctxtdata *uctxt)</div><div class='add'>+{</div><div class='add'>+	unsigned int rcvctrl_ops;</div><div class='add'>+	struct hfi1_devdata *dd = rx-&gt;dd;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	uctxt-&gt;rhf_rcv_function_map = netdev_rhf_rcv_functions;</div><div class='add'>+	uctxt-&gt;do_interrupt = &amp;handle_receive_interrupt_napi_sp;</div><div class='add'>+</div><div class='add'>+	/* Now allocate the RcvHdr queue and eager buffers. */</div><div class='add'>+	ret = hfi1_create_rcvhdrq(dd, uctxt);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto done;</div><div class='add'>+</div><div class='add'>+	ret = hfi1_setup_eagerbufs(uctxt);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto done;</div><div class='add'>+</div><div class='add'>+	clear_rcvhdrtail(uctxt);</div><div class='add'>+</div><div class='add'>+	rcvctrl_ops = HFI1_RCVCTRL_CTXT_DIS;</div><div class='add'>+	rcvctrl_ops |= HFI1_RCVCTRL_INTRAVAIL_DIS;</div><div class='add'>+</div><div class='add'>+	if (!HFI1_CAP_KGET_MASK(uctxt-&gt;flags, MULTI_PKT_EGR))</div><div class='add'>+		rcvctrl_ops |= HFI1_RCVCTRL_ONE_PKT_EGR_ENB;</div><div class='add'>+	if (HFI1_CAP_KGET_MASK(uctxt-&gt;flags, NODROP_EGR_FULL))</div><div class='add'>+		rcvctrl_ops |= HFI1_RCVCTRL_NO_EGR_DROP_ENB;</div><div class='add'>+	if (HFI1_CAP_KGET_MASK(uctxt-&gt;flags, NODROP_RHQ_FULL))</div><div class='add'>+		rcvctrl_ops |= HFI1_RCVCTRL_NO_RHQ_DROP_ENB;</div><div class='add'>+	if (HFI1_CAP_KGET_MASK(uctxt-&gt;flags, DMA_RTAIL))</div><div class='add'>+		rcvctrl_ops |= HFI1_RCVCTRL_TAILUPD_ENB;</div><div class='add'>+</div><div class='add'>+	hfi1_rcvctrl(uctxt-&gt;dd, rcvctrl_ops, uctxt);</div><div class='add'>+done:</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hfi1_netdev_allocate_ctxt(struct hfi1_devdata *dd,</div><div class='add'>+				     struct hfi1_ctxtdata **ctxt)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ctxtdata *uctxt;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (dd-&gt;flags &amp; HFI1_FROZEN)</div><div class='add'>+		return -EIO;</div><div class='add'>+</div><div class='add'>+	ret = hfi1_create_ctxtdata(dd-&gt;pport, dd-&gt;node, &amp;uctxt);</div><div class='add'>+	if (ret &lt; 0) {</div><div class='add'>+		dd_dev_err(dd, "Unable to create ctxtdata, failing open\n");</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	uctxt-&gt;flags = HFI1_CAP_KGET(MULTI_PKT_EGR) |</div><div class='add'>+		HFI1_CAP_KGET(NODROP_RHQ_FULL) |</div><div class='add'>+		HFI1_CAP_KGET(NODROP_EGR_FULL) |</div><div class='add'>+		HFI1_CAP_KGET(DMA_RTAIL);</div><div class='add'>+	/* Netdev contexts are always NO_RDMA_RTAIL */</div><div class='add'>+	uctxt-&gt;fast_handler = handle_receive_interrupt_napi_fp;</div><div class='add'>+	uctxt-&gt;slow_handler = handle_receive_interrupt_napi_sp;</div><div class='add'>+	hfi1_set_seq_cnt(uctxt, 1);</div><div class='add'>+	uctxt-&gt;is_vnic = true;</div><div class='add'>+</div><div class='add'>+	hfi1_stats.sps_ctxts++;</div><div class='add'>+</div><div class='add'>+	dd_dev_info(dd, "created netdev context %d\n", uctxt-&gt;ctxt);</div><div class='add'>+	*ctxt = uctxt;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hfi1_netdev_deallocate_ctxt(struct hfi1_devdata *dd,</div><div class='add'>+					struct hfi1_ctxtdata *uctxt)</div><div class='add'>+{</div><div class='add'>+	flush_wc();</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Disable receive context and interrupt available, reset all</div><div class='add'>+	 * RcvCtxtCtrl bits to default values.</div><div class='add'>+	 */</div><div class='add'>+	hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS |</div><div class='add'>+		     HFI1_RCVCTRL_TIDFLOW_DIS |</div><div class='add'>+		     HFI1_RCVCTRL_INTRAVAIL_DIS |</div><div class='add'>+		     HFI1_RCVCTRL_ONE_PKT_EGR_DIS |</div><div class='add'>+		     HFI1_RCVCTRL_NO_RHQ_DROP_DIS |</div><div class='add'>+		     HFI1_RCVCTRL_NO_EGR_DROP_DIS, uctxt);</div><div class='add'>+</div><div class='add'>+	if (uctxt-&gt;msix_intr != CCE_NUM_MSIX_VECTORS)</div><div class='add'>+		msix_free_irq(dd, uctxt-&gt;msix_intr);</div><div class='add'>+</div><div class='add'>+	uctxt-&gt;msix_intr = CCE_NUM_MSIX_VECTORS;</div><div class='add'>+	uctxt-&gt;event_flags = 0;</div><div class='add'>+</div><div class='add'>+	hfi1_clear_tids(uctxt);</div><div class='add'>+	hfi1_clear_ctxt_pkey(dd, uctxt);</div><div class='add'>+</div><div class='add'>+	hfi1_stats.sps_ctxts--;</div><div class='add'>+</div><div class='add'>+	hfi1_free_ctxt(uctxt);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hfi1_netdev_allot_ctxt(struct hfi1_netdev_rx *rx,</div><div class='add'>+				  struct hfi1_ctxtdata **ctxt)</div><div class='add'>+{</div><div class='add'>+	int rc;</div><div class='add'>+	struct hfi1_devdata *dd = rx-&gt;dd;</div><div class='add'>+</div><div class='add'>+	rc = hfi1_netdev_allocate_ctxt(dd, ctxt);</div><div class='add'>+	if (rc) {</div><div class='add'>+		dd_dev_err(dd, "netdev ctxt alloc failed %d\n", rc);</div><div class='add'>+		return rc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rc = hfi1_netdev_setup_ctxt(rx, *ctxt);</div><div class='add'>+	if (rc) {</div><div class='add'>+		dd_dev_err(dd, "netdev ctxt setup failed %d\n", rc);</div><div class='add'>+		hfi1_netdev_deallocate_ctxt(dd, *ctxt);</div><div class='add'>+		*ctxt = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_num_netdev_contexts - Count of netdev recv contexts to use.</div><div class='add'>+ * @dd: device on which to allocate netdev contexts</div><div class='add'>+ * @available_contexts: count of available receive contexts</div><div class='add'>+ * @cpu_mask: mask of possible cpus to include for contexts</div><div class='add'>+ *</div><div class='add'>+ * Return: count of physical cores on a node or the remaining available recv</div><div class='add'>+ * contexts for netdev recv context usage up to the maximum of</div><div class='add'>+ * HFI1_MAX_NETDEV_CTXTS.</div><div class='add'>+ * A value of 0 can be returned when acceleration is explicitly turned off,</div><div class='add'>+ * a memory allocation error occurs or when there are no available contexts.</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+u32 hfi1_num_netdev_contexts(struct hfi1_devdata *dd, u32 available_contexts,</div><div class='add'>+			     struct cpumask *cpu_mask)</div><div class='add'>+{</div><div class='add'>+	cpumask_var_t node_cpu_mask;</div><div class='add'>+	unsigned int available_cpus;</div><div class='add'>+</div><div class='add'>+	if (!HFI1_CAP_IS_KSET(AIP))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	/* Always give user contexts priority over netdev contexts */</div><div class='add'>+	if (available_contexts == 0) {</div><div class='add'>+		dd_dev_info(dd, "No receive contexts available for netdevs.\n");</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!zalloc_cpumask_var(&amp;node_cpu_mask, GFP_KERNEL)) {</div><div class='add'>+		dd_dev_err(dd, "Unable to allocate cpu_mask for netdevs.\n");</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cpumask_and(node_cpu_mask, cpu_mask, cpumask_of_node(dd-&gt;node));</div><div class='add'>+</div><div class='add'>+	available_cpus = cpumask_weight(node_cpu_mask);</div><div class='add'>+</div><div class='add'>+	free_cpumask_var(node_cpu_mask);</div><div class='add'>+</div><div class='add'>+	return min3(available_cpus, available_contexts,</div><div class='add'>+		    (u32)HFI1_MAX_NETDEV_CTXTS);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hfi1_netdev_rxq_init(struct hfi1_netdev_rx *rx)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+	int rc;</div><div class='add'>+	struct hfi1_devdata *dd = rx-&gt;dd;</div><div class='add'>+	struct net_device *dev = rx-&gt;rx_napi;</div><div class='add'>+</div><div class='add'>+	rx-&gt;num_rx_q = dd-&gt;num_netdev_contexts;</div><div class='add'>+	rx-&gt;rxq = kcalloc_node(rx-&gt;num_rx_q, sizeof(*rx-&gt;rxq),</div><div class='add'>+			       GFP_KERNEL, dd-&gt;node);</div><div class='add'>+</div><div class='add'>+	if (!rx-&gt;rxq) {</div><div class='add'>+		dd_dev_err(dd, "Unable to allocate netdev queue data\n");</div><div class='add'>+		return (-ENOMEM);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; rx-&gt;num_rx_q; i++) {</div><div class='add'>+		struct hfi1_netdev_rxq *rxq = &amp;rx-&gt;rxq[i];</div><div class='add'>+</div><div class='add'>+		rc = hfi1_netdev_allot_ctxt(rx, &amp;rxq-&gt;rcd);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto bail_context_irq_failure;</div><div class='add'>+</div><div class='add'>+		hfi1_rcd_get(rxq-&gt;rcd);</div><div class='add'>+		rxq-&gt;rx = rx;</div><div class='add'>+		rxq-&gt;rcd-&gt;napi = &amp;rxq-&gt;napi;</div><div class='add'>+		dd_dev_info(dd, "Setting rcv queue %d napi to context %d\n",</div><div class='add'>+			    i, rxq-&gt;rcd-&gt;ctxt);</div><div class='add'>+		/*</div><div class='add'>+		 * Disable BUSY_POLL on this NAPI as this is not supported</div><div class='add'>+		 * right now.</div><div class='add'>+		 */</div><div class='add'>+		set_bit(NAPI_STATE_NO_BUSY_POLL, &amp;rxq-&gt;napi.state);</div><div class='add'>+		netif_napi_add(dev, &amp;rxq-&gt;napi, hfi1_netdev_rx_napi);</div><div class='add'>+		rc = msix_netdev_request_rcd_irq(rxq-&gt;rcd);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto bail_context_irq_failure;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+bail_context_irq_failure:</div><div class='add'>+	dd_dev_err(dd, "Unable to allot receive context\n");</div><div class='add'>+	for (; i &gt;= 0; i--) {</div><div class='add'>+		struct hfi1_netdev_rxq *rxq = &amp;rx-&gt;rxq[i];</div><div class='add'>+</div><div class='add'>+		if (rxq-&gt;rcd) {</div><div class='add'>+			hfi1_netdev_deallocate_ctxt(dd, rxq-&gt;rcd);</div><div class='add'>+			hfi1_rcd_put(rxq-&gt;rcd);</div><div class='add'>+			rxq-&gt;rcd = NULL;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	kfree(rx-&gt;rxq);</div><div class='add'>+	rx-&gt;rxq = NULL;</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hfi1_netdev_rxq_deinit(struct hfi1_netdev_rx *rx)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+	struct hfi1_devdata *dd = rx-&gt;dd;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; rx-&gt;num_rx_q; i++) {</div><div class='add'>+		struct hfi1_netdev_rxq *rxq = &amp;rx-&gt;rxq[i];</div><div class='add'>+</div><div class='add'>+		netif_napi_del(&amp;rxq-&gt;napi);</div><div class='add'>+		hfi1_netdev_deallocate_ctxt(dd, rxq-&gt;rcd);</div><div class='add'>+		hfi1_rcd_put(rxq-&gt;rcd);</div><div class='add'>+		rxq-&gt;rcd = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	kfree(rx-&gt;rxq);</div><div class='add'>+	rx-&gt;rxq = NULL;</div><div class='add'>+	rx-&gt;num_rx_q = 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void enable_queues(struct hfi1_netdev_rx *rx)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; rx-&gt;num_rx_q; i++) {</div><div class='add'>+		struct hfi1_netdev_rxq *rxq = &amp;rx-&gt;rxq[i];</div><div class='add'>+</div><div class='add'>+		dd_dev_info(rx-&gt;dd, "enabling queue %d on context %d\n", i,</div><div class='add'>+			    rxq-&gt;rcd-&gt;ctxt);</div><div class='add'>+		napi_enable(&amp;rxq-&gt;napi);</div><div class='add'>+		hfi1_rcvctrl(rx-&gt;dd,</div><div class='add'>+			     HFI1_RCVCTRL_CTXT_ENB | HFI1_RCVCTRL_INTRAVAIL_ENB,</div><div class='add'>+			     rxq-&gt;rcd);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void disable_queues(struct hfi1_netdev_rx *rx)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	msix_netdev_synchronize_irq(rx-&gt;dd);</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; rx-&gt;num_rx_q; i++) {</div><div class='add'>+		struct hfi1_netdev_rxq *rxq = &amp;rx-&gt;rxq[i];</div><div class='add'>+</div><div class='add'>+		dd_dev_info(rx-&gt;dd, "disabling queue %d on context %d\n", i,</div><div class='add'>+			    rxq-&gt;rcd-&gt;ctxt);</div><div class='add'>+</div><div class='add'>+		/* wait for napi if it was scheduled */</div><div class='add'>+		hfi1_rcvctrl(rx-&gt;dd,</div><div class='add'>+			     HFI1_RCVCTRL_CTXT_DIS | HFI1_RCVCTRL_INTRAVAIL_DIS,</div><div class='add'>+			     rxq-&gt;rcd);</div><div class='add'>+		napi_synchronize(&amp;rxq-&gt;napi);</div><div class='add'>+		napi_disable(&amp;rxq-&gt;napi);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_netdev_rx_init - Incrememnts netdevs counter. When called first time,</div><div class='add'>+ * it allocates receive queue data and calls netif_napi_add</div><div class='add'>+ * for each queue.</div><div class='add'>+ *</div><div class='add'>+ * @dd: hfi1 dev data</div><div class='add'>+ */</div><div class='add'>+int hfi1_netdev_rx_init(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_netdev_rx *rx = dd-&gt;netdev_rx;</div><div class='add'>+	int res;</div><div class='add'>+</div><div class='add'>+	if (atomic_fetch_inc(&amp;rx-&gt;netdevs))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;hfi1_mutex);</div><div class='add'>+	res = hfi1_netdev_rxq_init(rx);</div><div class='add'>+	mutex_unlock(&amp;hfi1_mutex);</div><div class='add'>+	return res;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_netdev_rx_destroy - Decrements netdevs counter, when it reaches 0</div><div class='add'>+ * napi is deleted and receive queses memory is freed.</div><div class='add'>+ *</div><div class='add'>+ * @dd: hfi1 dev data</div><div class='add'>+ */</div><div class='add'>+int hfi1_netdev_rx_destroy(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_netdev_rx *rx = dd-&gt;netdev_rx;</div><div class='add'>+</div><div class='add'>+	/* destroy the RX queues only if it is the last netdev going away */</div><div class='add'>+	if (atomic_fetch_add_unless(&amp;rx-&gt;netdevs, -1, 0) == 1) {</div><div class='add'>+		mutex_lock(&amp;hfi1_mutex);</div><div class='add'>+		hfi1_netdev_rxq_deinit(rx);</div><div class='add'>+		mutex_unlock(&amp;hfi1_mutex);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_alloc_rx - Allocates the rx support structure</div><div class='add'>+ * @dd: hfi1 dev data</div><div class='add'>+ *</div><div class='add'>+ * Allocate the rx structure to support gathering the receive</div><div class='add'>+ * resources and the dummy netdev.</div><div class='add'>+ *</div><div class='add'>+ * Updates dd struct pointer upon success.</div><div class='add'>+ *</div><div class='add'>+ * Return: 0 (success) -error on failure</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+int hfi1_alloc_rx(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_netdev_rx *rx;</div><div class='add'>+</div><div class='add'>+	dd_dev_info(dd, "allocating rx size %ld\n", sizeof(*rx));</div><div class='add'>+	rx = kzalloc_node(sizeof(*rx), GFP_KERNEL, dd-&gt;node);</div><div class='add'>+</div><div class='add'>+	if (!rx)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	rx-&gt;dd = dd;</div><div class='add'>+	rx-&gt;rx_napi = alloc_netdev_dummy(0);</div><div class='add'>+	if (!rx-&gt;rx_napi) {</div><div class='add'>+		kfree(rx);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	xa_init(&amp;rx-&gt;dev_tbl);</div><div class='add'>+	atomic_set(&amp;rx-&gt;enabled, 0);</div><div class='add'>+	atomic_set(&amp;rx-&gt;netdevs, 0);</div><div class='add'>+	dd-&gt;netdev_rx = rx;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hfi1_free_rx(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	if (dd-&gt;netdev_rx) {</div><div class='add'>+		dd_dev_info(dd, "hfi1 rx freed\n");</div><div class='add'>+		free_netdev(dd-&gt;netdev_rx-&gt;rx_napi);</div><div class='add'>+		kfree(dd-&gt;netdev_rx);</div><div class='add'>+		dd-&gt;netdev_rx = NULL;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_netdev_enable_queues - This is napi enable function.</div><div class='add'>+ * It enables napi objects associated with queues.</div><div class='add'>+ * When at least one device has called it it increments atomic counter.</div><div class='add'>+ * Disable function decrements counter and when it is 0,</div><div class='add'>+ * calls napi_disable for every queue.</div><div class='add'>+ *</div><div class='add'>+ * @dd: hfi1 dev data</div><div class='add'>+ */</div><div class='add'>+void hfi1_netdev_enable_queues(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_netdev_rx *rx;</div><div class='add'>+</div><div class='add'>+	if (!dd-&gt;netdev_rx)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	rx = dd-&gt;netdev_rx;</div><div class='add'>+	if (atomic_fetch_inc(&amp;rx-&gt;enabled))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;hfi1_mutex);</div><div class='add'>+	enable_queues(rx);</div><div class='add'>+	mutex_unlock(&amp;hfi1_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hfi1_netdev_disable_queues(struct hfi1_devdata *dd)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_netdev_rx *rx;</div><div class='add'>+</div><div class='add'>+	if (!dd-&gt;netdev_rx)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	rx = dd-&gt;netdev_rx;</div><div class='add'>+	if (atomic_dec_if_positive(&amp;rx-&gt;enabled))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;hfi1_mutex);</div><div class='add'>+	disable_queues(rx);</div><div class='add'>+	mutex_unlock(&amp;hfi1_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_netdev_add_data - Registers data with unique identifier</div><div class='add'>+ * to be requested later this is needed for VNIC and IPoIB VLANs</div><div class='add'>+ * implementations.</div><div class='add'>+ * This call is protected by mutex idr_lock.</div><div class='add'>+ *</div><div class='add'>+ * @dd: hfi1 dev data</div><div class='add'>+ * @id: requested integer id up to INT_MAX</div><div class='add'>+ * @data: data to be associated with index</div><div class='add'>+ */</div><div class='add'>+int hfi1_netdev_add_data(struct hfi1_devdata *dd, int id, void *data)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_netdev_rx *rx = dd-&gt;netdev_rx;</div><div class='add'>+</div><div class='add'>+	return xa_insert(&amp;rx-&gt;dev_tbl, id, data, GFP_NOWAIT);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_netdev_remove_data - Removes data with previously given id.</div><div class='add'>+ * Returns the reference to removed entry.</div><div class='add'>+ *</div><div class='add'>+ * @dd: hfi1 dev data</div><div class='add'>+ * @id: requested integer id up to INT_MAX</div><div class='add'>+ */</div><div class='add'>+void *hfi1_netdev_remove_data(struct hfi1_devdata *dd, int id)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_netdev_rx *rx = dd-&gt;netdev_rx;</div><div class='add'>+</div><div class='add'>+	return xa_erase(&amp;rx-&gt;dev_tbl, id);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_netdev_get_data - Gets data with given id</div><div class='add'>+ *</div><div class='add'>+ * @dd: hfi1 dev data</div><div class='add'>+ * @id: requested integer id up to INT_MAX</div><div class='add'>+ */</div><div class='add'>+void *hfi1_netdev_get_data(struct hfi1_devdata *dd, int id)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_netdev_rx *rx = dd-&gt;netdev_rx;</div><div class='add'>+</div><div class='add'>+	return xa_load(&amp;rx-&gt;dev_tbl, id);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hfi1_netdev_get_first_data - Gets first entry with greater or equal id.</div><div class='add'>+ *</div><div class='add'>+ * @dd: hfi1 dev data</div><div class='add'>+ * @start_id: requested integer id up to INT_MAX</div><div class='add'>+ */</div><div class='add'>+void *hfi1_netdev_get_first_data(struct hfi1_devdata *dd, int *start_id)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_netdev_rx *rx = dd-&gt;netdev_rx;</div><div class='add'>+	unsigned long index = *start_id;</div><div class='add'>+	void *ret;</div><div class='add'>+</div><div class='add'>+	ret = xa_find(&amp;rx-&gt;dev_tbl, &amp;index, UINT_MAX, XA_PRESENT);</div><div class='add'>+	*start_id = (int)index;</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/opa_compat.h b/drivers/infiniband/hw/hfi1/opa_compat.h<br/>index 774215b95df5..49f2da677b03 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/opa_compat.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/opa_compat.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/opa_compat.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/opa_compat.h</a></div><div class='hunk'>@@ -1,52 +1,10 @@</div><div class='del'>-#ifndef _LINUX_H</div><div class='del'>-#define _LINUX_H</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015, 2016 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='add'>+#ifndef _LINUX_H</div><div class='add'>+#define _LINUX_H</div><div class='ctx'> /*</div><div class='ctx'>  * This header file is for OPA-specific definitions which are</div><div class='ctx'>  * required by the HFI driver, and which aren't yet in the Linux</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/opfn.c b/drivers/infiniband/hw/hfi1/opfn.c<br/>index 370a5a8eaa71..6e0e3458d202 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/opfn.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/opfn.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/opfn.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/opfn.c</a></div><div class='hunk'>@@ -305,8 +305,8 @@ void opfn_trigger_conn_request(struct rvt_qp *qp, u32 bth1)</div><div class='ctx'> int opfn_init(void)</div><div class='ctx'> {</div><div class='ctx'> 	opfn_wq = alloc_workqueue("hfi_opfn",</div><div class='del'>-				  WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE |</div><div class='del'>-				  WQ_MEM_RECLAIM,</div><div class='add'>+				  WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM |</div><div class='add'>+				  WQ_PERCPU,</div><div class='ctx'> 				  HFI1_MAX_ACTIVE_WORKQUEUE_ENTRIES);</div><div class='ctx'> 	if (!opfn_wq)</div><div class='ctx'> 		return -ENOMEM;</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c<br/>index c96d193bb236..7133964749f8 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/pcie.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/pcie.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/pcie.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/pcie.c</a></div><div class='hunk'>@@ -1,55 +1,13 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2015 - 2019 Intel Corporation.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='add'>+#include &lt;linux/bitfield.h&gt;</div><div class='ctx'> #include &lt;linux/pci.h&gt;</div><div class='ctx'> #include &lt;linux/io.h&gt;</div><div class='ctx'> #include &lt;linux/delay.h&gt;</div><div class='ctx'> #include &lt;linux/vmalloc.h&gt;</div><div class='del'>-#include &lt;linux/aer.h&gt;</div><div class='ctx'> #include &lt;linux/module.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "hfi.h"</div><div class='hunk'>@@ -92,29 +50,21 @@ int hfi1_pcie_init(struct hfi1_devdata *dd)</div><div class='ctx'> 		goto bail;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));</div><div class='add'>+	ret = dma_set_mask_and_coherent(&amp;pdev-&gt;dev, DMA_BIT_MASK(64));</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * If the 64 bit setup fails, try 32 bit.  Some systems</div><div class='ctx'> 		 * do not setup 64 bit maps on systems with 2GB or less</div><div class='ctx'> 		 * memory installed.</div><div class='ctx'> 		 */</div><div class='del'>-		ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));</div><div class='add'>+		ret = dma_set_mask_and_coherent(&amp;pdev-&gt;dev, DMA_BIT_MASK(32));</div><div class='ctx'> 		if (ret) {</div><div class='ctx'> 			dd_dev_err(dd, "Unable to set DMA mask: %d\n", ret);</div><div class='ctx'> 			goto bail;</div><div class='ctx'> 		}</div><div class='del'>-		ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));</div><div class='del'>-	} else {</div><div class='del'>-		ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));</div><div class='del'>-	}</div><div class='del'>-	if (ret) {</div><div class='del'>-		dd_dev_err(dd, "Unable to set DMA consistent mask: %d\n", ret);</div><div class='del'>-		goto bail;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	pci_set_master(pdev);</div><div class='del'>-	(void)pci_enable_pcie_error_reporting(pdev);</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> bail:</div><div class='hunk'>@@ -161,7 +111,7 @@ int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	dd-&gt;kregbase1 = ioremap_nocache(addr, RCV_ARRAY);</div><div class='add'>+	dd-&gt;kregbase1 = ioremap(addr, RCV_ARRAY);</div><div class='ctx'> 	if (!dd-&gt;kregbase1) {</div><div class='ctx'> 		dd_dev_err(dd, "UC mapping of kregbase1 failed\n");</div><div class='ctx'> 		return -ENOMEM;</div><div class='hunk'>@@ -179,7 +129,7 @@ int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev)</div><div class='ctx'> 	dd_dev_info(dd, "RcvArray count: %u\n", rcv_array_count);</div><div class='ctx'> 	dd-&gt;base2_start  = RCV_ARRAY + rcv_array_count * 8;</div><div class='ctx'> </div><div class='del'>-	dd-&gt;kregbase2 = ioremap_nocache(</div><div class='add'>+	dd-&gt;kregbase2 = ioremap(</div><div class='ctx'> 		addr + dd-&gt;base2_start,</div><div class='ctx'> 		TXE_PIO_SEND - dd-&gt;base2_start);</div><div class='ctx'> 	if (!dd-&gt;kregbase2) {</div><div class='hunk'>@@ -261,12 +211,6 @@ static u32 extract_speed(u16 linkstat)</div><div class='ctx'> 	return speed;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/* return the PCIe link speed from the given link status */</div><div class='del'>-static u32 extract_width(u16 linkstat)</div><div class='del'>-{</div><div class='del'>-	return (linkstat &amp; PCI_EXP_LNKSTA_NLW) &gt;&gt; PCI_EXP_LNKSTA_NLW_SHIFT;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> /* read the link status and set dd-&gt;{lbus_width,lbus_speed,lbus_info} */</div><div class='ctx'> static void update_lbus_info(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='hunk'>@@ -279,7 +223,7 @@ static void update_lbus_info(struct hfi1_devdata *dd)</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	dd-&gt;lbus_width = extract_width(linkstat);</div><div class='add'>+	dd-&gt;lbus_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, linkstat);</div><div class='ctx'> 	dd-&gt;lbus_speed = extract_speed(linkstat);</div><div class='ctx'> 	snprintf(dd-&gt;lbus_info, sizeof(dd-&gt;lbus_info),</div><div class='ctx'> 		 "PCIe,%uMHz,x%u", dd-&gt;lbus_speed, dd-&gt;lbus_width);</div><div class='hunk'>@@ -306,7 +250,7 @@ int pcie_speeds(struct hfi1_devdata *dd)</div><div class='ctx'> 	ret = pcie_capability_read_dword(dd-&gt;pcidev, PCI_EXP_LNKCAP, &amp;linkcap);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		dd_dev_err(dd, "Unable to read from PCI config\n");</div><div class='del'>-		return ret;</div><div class='add'>+		return pcibios_err_to_errno(ret);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if ((linkcap &amp; PCI_EXP_LNKCAP_SLS) != PCI_EXP_LNKCAP_SLS_8_0GB) {</div><div class='hunk'>@@ -319,7 +263,9 @@ int pcie_speeds(struct hfi1_devdata *dd)</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * bus-&gt;max_bus_speed is set from the bridge's linkcap Max Link Speed</div><div class='ctx'> 	 */</div><div class='del'>-	if (parent &amp;&amp; dd-&gt;pcidev-&gt;bus-&gt;max_bus_speed != PCIE_SPEED_8_0GT) {</div><div class='add'>+	if (parent &amp;&amp;</div><div class='add'>+	    (dd-&gt;pcidev-&gt;bus-&gt;max_bus_speed == PCIE_SPEED_2_5GT ||</div><div class='add'>+	     dd-&gt;pcidev-&gt;bus-&gt;max_bus_speed == PCIE_SPEED_5_0GT)) {</div><div class='ctx'> 		dd_dev_info(dd, "Parent PCIe bridge does not support Gen3\n");</div><div class='ctx'> 		dd-&gt;link_gen3_capable = 0;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -332,10 +278,14 @@ int pcie_speeds(struct hfi1_devdata *dd)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/* restore command and BARs after a reset has wiped them out */</div><div class='add'>+/*</div><div class='add'>+ * Restore command and BARs after a reset has wiped them out</div><div class='add'>+ *</div><div class='add'>+ * Returns 0 on success, otherwise a negative error value</div><div class='add'>+ */</div><div class='ctx'> int restore_pci_variables(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='del'>-	int ret = 0;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	ret = pci_write_config_word(dd-&gt;pcidev, PCI_COMMAND, dd-&gt;pci_command);</div><div class='ctx'> 	if (ret)</div><div class='hunk'>@@ -384,13 +334,17 @@ int restore_pci_variables(struct hfi1_devdata *dd)</div><div class='ctx'> </div><div class='ctx'> error:</div><div class='ctx'> 	dd_dev_err(dd, "Unable to write to PCI config\n");</div><div class='del'>-	return ret;</div><div class='add'>+	return pcibios_err_to_errno(ret);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/* Save BARs and command to rewrite after device reset */</div><div class='add'>+/*</div><div class='add'>+ * Save BARs and command to rewrite after device reset</div><div class='add'>+ *</div><div class='add'>+ * Returns 0 on success, otherwise a negative error value</div><div class='add'>+ */</div><div class='ctx'> int save_pci_variables(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='del'>-	int ret = 0;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	ret = pci_read_config_dword(dd-&gt;pcidev, PCI_BASE_ADDRESS_0,</div><div class='ctx'> 				    &amp;dd-&gt;pcibar0);</div><div class='hunk'>@@ -439,7 +393,7 @@ int save_pci_variables(struct hfi1_devdata *dd)</div><div class='ctx'> </div><div class='ctx'> error:</div><div class='ctx'> 	dd_dev_err(dd, "Unable to read from PCI config\n");</div><div class='del'>-	return ret;</div><div class='add'>+	return pcibios_err_to_errno(ret);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -450,10 +404,6 @@ static int hfi1_pcie_caps;</div><div class='ctx'> module_param_named(pcie_caps, hfi1_pcie_caps, int, 0444);</div><div class='ctx'> MODULE_PARM_DESC(pcie_caps, "Max PCIe tuning: Payload (0..3), ReadReq (4..7)");</div><div class='ctx'> </div><div class='del'>-uint aspm_mode = ASPM_MODE_DISABLED;</div><div class='del'>-module_param_named(aspm, aspm_mode, uint, 0444);</div><div class='del'>-MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic");</div><div class='del'>-</div><div class='ctx'> /**</div><div class='ctx'>  * tune_pcie_caps() - Code to adjust PCIe capabilities.</div><div class='ctx'>  * @dd: Valid device data structure</div><div class='hunk'>@@ -1257,14 +1207,11 @@ retry:</div><div class='ctx'> 		    (u32)lnkctl2);</div><div class='ctx'> 	/* only write to parent if target is not as high as ours */</div><div class='ctx'> 	if ((lnkctl2 &amp; PCI_EXP_LNKCTL2_TLS) &lt; target_vector) {</div><div class='del'>-		lnkctl2 &amp;= ~PCI_EXP_LNKCTL2_TLS;</div><div class='del'>-		lnkctl2 |= target_vector;</div><div class='del'>-		dd_dev_info(dd, "%s: ..new link control2: 0x%x\n", __func__,</div><div class='del'>-			    (u32)lnkctl2);</div><div class='del'>-		ret = pcie_capability_write_word(parent,</div><div class='del'>-						 PCI_EXP_LNKCTL2, lnkctl2);</div><div class='add'>+		ret = pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL2,</div><div class='add'>+							 PCI_EXP_LNKCTL2_TLS,</div><div class='add'>+							 target_vector);</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			dd_dev_err(dd, "Unable to write to PCI config\n");</div><div class='add'>+			dd_dev_err(dd, "Unable to change parent PCI target speed\n");</div><div class='ctx'> 			return_error = 1;</div><div class='ctx'> 			goto done;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -1273,22 +1220,11 @@ retry:</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	dd_dev_info(dd, "%s: setting target link speed\n", __func__);</div><div class='del'>-	ret = pcie_capability_read_word(dd-&gt;pcidev, PCI_EXP_LNKCTL2, &amp;lnkctl2);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dd_dev_err(dd, "Unable to read from PCI config\n");</div><div class='del'>-		return_error = 1;</div><div class='del'>-		goto done;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	dd_dev_info(dd, "%s: ..old link control2: 0x%x\n", __func__,</div><div class='del'>-		    (u32)lnkctl2);</div><div class='del'>-	lnkctl2 &amp;= ~PCI_EXP_LNKCTL2_TLS;</div><div class='del'>-	lnkctl2 |= target_vector;</div><div class='del'>-	dd_dev_info(dd, "%s: ..new link control2: 0x%x\n", __func__,</div><div class='del'>-		    (u32)lnkctl2);</div><div class='del'>-	ret = pcie_capability_write_word(dd-&gt;pcidev, PCI_EXP_LNKCTL2, lnkctl2);</div><div class='add'>+	ret = pcie_capability_clear_and_set_word(dd-&gt;pcidev, PCI_EXP_LNKCTL2,</div><div class='add'>+						 PCI_EXP_LNKCTL2_TLS,</div><div class='add'>+						 target_vector);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dd_dev_err(dd, "Unable to write to PCI config\n");</div><div class='add'>+		dd_dev_err(dd, "Unable to change device PCI target speed\n");</div><div class='ctx'> 		return_error = 1;</div><div class='ctx'> 		goto done;</div><div class='ctx'> 	}</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/pin_system.c b/drivers/infiniband/hw/hfi1/pin_system.c<br/>new file mode 100644<br/>index 000000000000..cce56134519b<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/pin_system.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/pin_system.c</a></div><div class='hunk'>@@ -0,0 +1,474 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause</div><div class='add'>+/*</div><div class='add'>+ * Copyright(c) 2023 - Cornelis Networks, Inc.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/types.h&gt;</div><div class='add'>+</div><div class='add'>+#include "hfi.h"</div><div class='add'>+#include "common.h"</div><div class='add'>+#include "device.h"</div><div class='add'>+#include "pinning.h"</div><div class='add'>+#include "mmu_rb.h"</div><div class='add'>+#include "user_sdma.h"</div><div class='add'>+#include "trace.h"</div><div class='add'>+</div><div class='add'>+struct sdma_mmu_node {</div><div class='add'>+	struct mmu_rb_node rb;</div><div class='add'>+	struct hfi1_user_sdma_pkt_q *pq;</div><div class='add'>+	struct page **pages;</div><div class='add'>+	unsigned int npages;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr,</div><div class='add'>+			   unsigned long len);</div><div class='add'>+static int sdma_rb_evict(void *arg, struct mmu_rb_node *mnode, void *arg2,</div><div class='add'>+			 bool *stop);</div><div class='add'>+static void sdma_rb_remove(void *arg, struct mmu_rb_node *mnode);</div><div class='add'>+</div><div class='add'>+static const struct mmu_rb_ops sdma_rb_ops = {</div><div class='add'>+	.filter = sdma_rb_filter,</div><div class='add'>+	.evict = sdma_rb_evict,</div><div class='add'>+	.remove = sdma_rb_remove,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+int hfi1_init_system_pinning(struct hfi1_user_sdma_pkt_q *pq)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_devdata *dd = pq-&gt;dd;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = hfi1_mmu_rb_register(pq, &amp;sdma_rb_ops, dd-&gt;pport-&gt;hfi1_wq,</div><div class='add'>+				   &amp;pq-&gt;handler);</div><div class='add'>+	if (ret)</div><div class='add'>+		dd_dev_err(dd,</div><div class='add'>+			   "[%u:%u] Failed to register system memory DMA support with MMU: %d\n",</div><div class='add'>+			   pq-&gt;ctxt, pq-&gt;subctxt, ret);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hfi1_free_system_pinning(struct hfi1_user_sdma_pkt_q *pq)</div><div class='add'>+{</div><div class='add'>+	if (pq-&gt;handler)</div><div class='add'>+		hfi1_mmu_rb_unregister(pq-&gt;handler);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u32 sdma_cache_evict(struct hfi1_user_sdma_pkt_q *pq, u32 npages)</div><div class='add'>+{</div><div class='add'>+	struct evict_data evict_data;</div><div class='add'>+</div><div class='add'>+	evict_data.cleared = 0;</div><div class='add'>+	evict_data.target = npages;</div><div class='add'>+	hfi1_mmu_rb_evict(pq-&gt;handler, &amp;evict_data);</div><div class='add'>+	return evict_data.cleared;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void unpin_vector_pages(struct mm_struct *mm, struct page **pages,</div><div class='add'>+			       unsigned int start, unsigned int npages)</div><div class='add'>+{</div><div class='add'>+	hfi1_release_user_pages(mm, pages + start, npages, false);</div><div class='add'>+	kfree(pages);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct mm_struct *mm_from_sdma_node(struct sdma_mmu_node *node)</div><div class='add'>+{</div><div class='add'>+	return node-&gt;rb.handler-&gt;mn.mm;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void free_system_node(struct sdma_mmu_node *node)</div><div class='add'>+{</div><div class='add'>+	if (node-&gt;npages) {</div><div class='add'>+		unpin_vector_pages(mm_from_sdma_node(node), node-&gt;pages, 0,</div><div class='add'>+				   node-&gt;npages);</div><div class='add'>+		atomic_sub(node-&gt;npages, &amp;node-&gt;pq-&gt;n_locked);</div><div class='add'>+	}</div><div class='add'>+	kfree(node);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * kref_get()'s an additional kref on the returned rb_node to prevent rb_node</div><div class='add'>+ * from being released until after rb_node is assigned to an SDMA descriptor</div><div class='add'>+ * (struct sdma_desc) under add_system_iovec_to_sdma_packet(), even if the</div><div class='add'>+ * virtual address range for rb_node is invalidated between now and then.</div><div class='add'>+ */</div><div class='add'>+static struct sdma_mmu_node *find_system_node(struct mmu_rb_handler *handler,</div><div class='add'>+					      unsigned long start,</div><div class='add'>+					      unsigned long end)</div><div class='add'>+{</div><div class='add'>+	struct mmu_rb_node *rb_node;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;handler-&gt;lock, flags);</div><div class='add'>+	rb_node = hfi1_mmu_rb_get_first(handler, start, (end - start));</div><div class='add'>+	if (!rb_node) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;handler-&gt;lock, flags);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* "safety" kref to prevent release before add_system_iovec_to_sdma_packet() */</div><div class='add'>+	kref_get(&amp;rb_node-&gt;refcount);</div><div class='add'>+	spin_unlock_irqrestore(&amp;handler-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	return container_of(rb_node, struct sdma_mmu_node, rb);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int pin_system_pages(struct user_sdma_request *req,</div><div class='add'>+			    uintptr_t start_address, size_t length,</div><div class='add'>+			    struct sdma_mmu_node *node, int npages)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_user_sdma_pkt_q *pq = req-&gt;pq;</div><div class='add'>+	int pinned, cleared;</div><div class='add'>+	struct page **pages;</div><div class='add'>+</div><div class='add'>+	pages = kcalloc(npages, sizeof(*pages), GFP_KERNEL);</div><div class='add'>+	if (!pages)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+retry:</div><div class='add'>+	if (!hfi1_can_pin_pages(pq-&gt;dd, current-&gt;mm, atomic_read(&amp;pq-&gt;n_locked),</div><div class='add'>+				npages)) {</div><div class='add'>+		SDMA_DBG(req, "Evicting: nlocked %u npages %u",</div><div class='add'>+			 atomic_read(&amp;pq-&gt;n_locked), npages);</div><div class='add'>+		cleared = sdma_cache_evict(pq, npages);</div><div class='add'>+		if (cleared &gt;= npages)</div><div class='add'>+			goto retry;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	SDMA_DBG(req, "Acquire user pages start_address %lx node-&gt;npages %u npages %u",</div><div class='add'>+		 start_address, node-&gt;npages, npages);</div><div class='add'>+	pinned = hfi1_acquire_user_pages(current-&gt;mm, start_address, npages, 0,</div><div class='add'>+					 pages);</div><div class='add'>+</div><div class='add'>+	if (pinned &lt; 0) {</div><div class='add'>+		kfree(pages);</div><div class='add'>+		SDMA_DBG(req, "pinned %d", pinned);</div><div class='add'>+		return pinned;</div><div class='add'>+	}</div><div class='add'>+	if (pinned != npages) {</div><div class='add'>+		unpin_vector_pages(current-&gt;mm, pages, node-&gt;npages, pinned);</div><div class='add'>+		SDMA_DBG(req, "npages %u pinned %d", npages, pinned);</div><div class='add'>+		return -EFAULT;</div><div class='add'>+	}</div><div class='add'>+	node-&gt;rb.addr = start_address;</div><div class='add'>+	node-&gt;rb.len = length;</div><div class='add'>+	node-&gt;pages = pages;</div><div class='add'>+	node-&gt;npages = npages;</div><div class='add'>+	atomic_add(pinned, &amp;pq-&gt;n_locked);</div><div class='add'>+	SDMA_DBG(req, "done. pinned %d", pinned);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * kref refcount on *node_p will be 2 on successful addition: one kref from</div><div class='add'>+ * kref_init() for mmu_rb_handler and one kref to prevent *node_p from being</div><div class='add'>+ * released until after *node_p is assigned to an SDMA descriptor (struct</div><div class='add'>+ * sdma_desc) under add_system_iovec_to_sdma_packet(), even if the virtual</div><div class='add'>+ * address range for *node_p is invalidated between now and then.</div><div class='add'>+ */</div><div class='add'>+static int add_system_pinning(struct user_sdma_request *req,</div><div class='add'>+			      struct sdma_mmu_node **node_p,</div><div class='add'>+			      unsigned long start, unsigned long len)</div><div class='add'>+</div><div class='add'>+{</div><div class='add'>+	struct hfi1_user_sdma_pkt_q *pq = req-&gt;pq;</div><div class='add'>+	struct sdma_mmu_node *node;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	node = kzalloc(sizeof(*node), GFP_KERNEL);</div><div class='add'>+	if (!node)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	/* First kref "moves" to mmu_rb_handler */</div><div class='add'>+	kref_init(&amp;node-&gt;rb.refcount);</div><div class='add'>+</div><div class='add'>+	/* "safety" kref to prevent release before add_system_iovec_to_sdma_packet() */</div><div class='add'>+	kref_get(&amp;node-&gt;rb.refcount);</div><div class='add'>+</div><div class='add'>+	node-&gt;pq = pq;</div><div class='add'>+	ret = pin_system_pages(req, start, len, node, PFN_DOWN(len));</div><div class='add'>+	if (ret == 0) {</div><div class='add'>+		ret = hfi1_mmu_rb_insert(pq-&gt;handler, &amp;node-&gt;rb);</div><div class='add'>+		if (ret)</div><div class='add'>+			free_system_node(node);</div><div class='add'>+		else</div><div class='add'>+			*node_p = node;</div><div class='add'>+</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	kfree(node);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_system_cache_entry(struct user_sdma_request *req,</div><div class='add'>+				  struct sdma_mmu_node **node_p,</div><div class='add'>+				  size_t req_start, size_t req_len)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_user_sdma_pkt_q *pq = req-&gt;pq;</div><div class='add'>+	u64 start = ALIGN_DOWN(req_start, PAGE_SIZE);</div><div class='add'>+	u64 end = PFN_ALIGN(req_start + req_len);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if ((end - start) == 0) {</div><div class='add'>+		SDMA_DBG(req,</div><div class='add'>+			 "Request for empty cache entry req_start %lx req_len %lx start %llx end %llx",</div><div class='add'>+			 req_start, req_len, start, end);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	SDMA_DBG(req, "req_start %lx req_len %lu", req_start, req_len);</div><div class='add'>+</div><div class='add'>+	while (1) {</div><div class='add'>+		struct sdma_mmu_node *node =</div><div class='add'>+			find_system_node(pq-&gt;handler, start, end);</div><div class='add'>+		u64 prepend_len = 0;</div><div class='add'>+</div><div class='add'>+		SDMA_DBG(req, "node %p start %llx end %llu", node, start, end);</div><div class='add'>+		if (!node) {</div><div class='add'>+			ret = add_system_pinning(req, node_p, start,</div><div class='add'>+						 end - start);</div><div class='add'>+			if (ret == -EEXIST) {</div><div class='add'>+				/*</div><div class='add'>+				 * Another execution context has inserted a</div><div class='add'>+				 * conficting entry first.</div><div class='add'>+				 */</div><div class='add'>+				continue;</div><div class='add'>+			}</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (node-&gt;rb.addr &lt;= start) {</div><div class='add'>+			/*</div><div class='add'>+			 * This entry covers at least part of the region. If it doesn't extend</div><div class='add'>+			 * to the end, then this will be called again for the next segment.</div><div class='add'>+			 */</div><div class='add'>+			*node_p = node;</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		SDMA_DBG(req, "prepend: node-&gt;rb.addr %lx, node-&gt;rb.refcount %d",</div><div class='add'>+			 node-&gt;rb.addr, kref_read(&amp;node-&gt;rb.refcount));</div><div class='add'>+		prepend_len = node-&gt;rb.addr - start;</div><div class='add'>+</div><div class='add'>+		/*</div><div class='add'>+		 * This node will not be returned, instead a new node</div><div class='add'>+		 * will be. So release the reference.</div><div class='add'>+		 */</div><div class='add'>+		kref_put(&amp;node-&gt;rb.refcount, hfi1_mmu_rb_release);</div><div class='add'>+</div><div class='add'>+		/* Prepend a node to cover the beginning of the allocation */</div><div class='add'>+		ret = add_system_pinning(req, node_p, start, prepend_len);</div><div class='add'>+		if (ret == -EEXIST) {</div><div class='add'>+			/* Another execution context has inserted a conficting entry first. */</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void sdma_mmu_rb_node_get(void *ctx)</div><div class='add'>+{</div><div class='add'>+	struct mmu_rb_node *node = ctx;</div><div class='add'>+</div><div class='add'>+	kref_get(&amp;node-&gt;refcount);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void sdma_mmu_rb_node_put(void *ctx)</div><div class='add'>+{</div><div class='add'>+	struct sdma_mmu_node *node = ctx;</div><div class='add'>+</div><div class='add'>+	kref_put(&amp;node-&gt;rb.refcount, hfi1_mmu_rb_release);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int add_mapping_to_sdma_packet(struct user_sdma_request *req,</div><div class='add'>+				      struct user_sdma_txreq *tx,</div><div class='add'>+				      struct sdma_mmu_node *cache_entry,</div><div class='add'>+				      size_t start,</div><div class='add'>+				      size_t from_this_cache_entry)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_user_sdma_pkt_q *pq = req-&gt;pq;</div><div class='add'>+	unsigned int page_offset;</div><div class='add'>+	unsigned int from_this_page;</div><div class='add'>+	size_t page_index;</div><div class='add'>+	void *ctx;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Because the cache may be more fragmented than the memory that is being accessed,</div><div class='add'>+	 * it's not strictly necessary to have a descriptor per cache entry.</div><div class='add'>+	 */</div><div class='add'>+</div><div class='add'>+	while (from_this_cache_entry) {</div><div class='add'>+		page_index = PFN_DOWN(start - cache_entry-&gt;rb.addr);</div><div class='add'>+</div><div class='add'>+		if (page_index &gt;= cache_entry-&gt;npages) {</div><div class='add'>+			SDMA_DBG(req,</div><div class='add'>+				 "Request for page_index %zu &gt;= cache_entry-&gt;npages %u",</div><div class='add'>+				 page_index, cache_entry-&gt;npages);</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		page_offset = start - ALIGN_DOWN(start, PAGE_SIZE);</div><div class='add'>+		from_this_page = PAGE_SIZE - page_offset;</div><div class='add'>+</div><div class='add'>+		if (from_this_page &lt; from_this_cache_entry) {</div><div class='add'>+			ctx = NULL;</div><div class='add'>+		} else {</div><div class='add'>+			/*</div><div class='add'>+			 * In the case they are equal the next line has no practical effect,</div><div class='add'>+			 * but it's better to do a register to register copy than a conditional</div><div class='add'>+			 * branch.</div><div class='add'>+			 */</div><div class='add'>+			from_this_page = from_this_cache_entry;</div><div class='add'>+			ctx = cache_entry;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ret = sdma_txadd_page(pq-&gt;dd, &amp;tx-&gt;txreq,</div><div class='add'>+				      cache_entry-&gt;pages[page_index],</div><div class='add'>+				      page_offset, from_this_page,</div><div class='add'>+				      ctx,</div><div class='add'>+				      sdma_mmu_rb_node_get,</div><div class='add'>+				      sdma_mmu_rb_node_put);</div><div class='add'>+		if (ret) {</div><div class='add'>+			/*</div><div class='add'>+			 * When there's a failure, the entire request is freed by</div><div class='add'>+			 * user_sdma_send_pkts().</div><div class='add'>+			 */</div><div class='add'>+			SDMA_DBG(req,</div><div class='add'>+				 "sdma_txadd_page failed %d page_index %lu page_offset %u from_this_page %u",</div><div class='add'>+				 ret, page_index, page_offset, from_this_page);</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+		start += from_this_page;</div><div class='add'>+		from_this_cache_entry -= from_this_page;</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int add_system_iovec_to_sdma_packet(struct user_sdma_request *req,</div><div class='add'>+					   struct user_sdma_txreq *tx,</div><div class='add'>+					   struct user_sdma_iovec *iovec,</div><div class='add'>+					   size_t from_this_iovec)</div><div class='add'>+{</div><div class='add'>+	while (from_this_iovec &gt; 0) {</div><div class='add'>+		struct sdma_mmu_node *cache_entry;</div><div class='add'>+		size_t from_this_cache_entry;</div><div class='add'>+		size_t start;</div><div class='add'>+		int ret;</div><div class='add'>+</div><div class='add'>+		start = (uintptr_t)iovec-&gt;iov.iov_base + iovec-&gt;offset;</div><div class='add'>+		ret = get_system_cache_entry(req, &amp;cache_entry, start,</div><div class='add'>+					     from_this_iovec);</div><div class='add'>+		if (ret) {</div><div class='add'>+			SDMA_DBG(req, "pin system segment failed %d", ret);</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		from_this_cache_entry = cache_entry-&gt;rb.len - (start - cache_entry-&gt;rb.addr);</div><div class='add'>+		if (from_this_cache_entry &gt; from_this_iovec)</div><div class='add'>+			from_this_cache_entry = from_this_iovec;</div><div class='add'>+</div><div class='add'>+		ret = add_mapping_to_sdma_packet(req, tx, cache_entry, start,</div><div class='add'>+						 from_this_cache_entry);</div><div class='add'>+</div><div class='add'>+		/*</div><div class='add'>+		 * Done adding cache_entry to zero or more sdma_desc. Can</div><div class='add'>+		 * kref_put() the "safety" kref taken under</div><div class='add'>+		 * get_system_cache_entry().</div><div class='add'>+		 */</div><div class='add'>+		kref_put(&amp;cache_entry-&gt;rb.refcount, hfi1_mmu_rb_release);</div><div class='add'>+</div><div class='add'>+		if (ret) {</div><div class='add'>+			SDMA_DBG(req, "add system segment failed %d", ret);</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		iovec-&gt;offset += from_this_cache_entry;</div><div class='add'>+		from_this_iovec -= from_this_cache_entry;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * Add up to pkt_data_remaining bytes to the txreq, starting at the current</div><div class='add'>+ * offset in the given iovec entry and continuing until all data has been added</div><div class='add'>+ * to the iovec or the iovec entry type changes.</div><div class='add'>+ *</div><div class='add'>+ * On success, prior to returning, adjust pkt_data_remaining, req-&gt;iov_idx, and</div><div class='add'>+ * the offset value in req-&gt;iov[req-&gt;iov_idx] to reflect the data that has been</div><div class='add'>+ * consumed.</div><div class='add'>+ */</div><div class='add'>+int hfi1_add_pages_to_sdma_packet(struct user_sdma_request *req,</div><div class='add'>+				  struct user_sdma_txreq *tx,</div><div class='add'>+				  struct user_sdma_iovec *iovec,</div><div class='add'>+				  u32 *pkt_data_remaining)</div><div class='add'>+{</div><div class='add'>+	size_t remaining_to_add = *pkt_data_remaining;</div><div class='add'>+	/*</div><div class='add'>+	 * Walk through iovec entries, ensure the associated pages</div><div class='add'>+	 * are pinned and mapped, add data to the packet until no more</div><div class='add'>+	 * data remains to be added or the iovec entry type changes.</div><div class='add'>+	 */</div><div class='add'>+	while (remaining_to_add &gt; 0) {</div><div class='add'>+		struct user_sdma_iovec *cur_iovec;</div><div class='add'>+		size_t from_this_iovec;</div><div class='add'>+		int ret;</div><div class='add'>+</div><div class='add'>+		cur_iovec = iovec;</div><div class='add'>+		from_this_iovec = iovec-&gt;iov.iov_len - iovec-&gt;offset;</div><div class='add'>+</div><div class='add'>+		if (from_this_iovec &gt; remaining_to_add) {</div><div class='add'>+			from_this_iovec = remaining_to_add;</div><div class='add'>+		} else {</div><div class='add'>+			/* The current iovec entry will be consumed by this pass. */</div><div class='add'>+			req-&gt;iov_idx++;</div><div class='add'>+			iovec++;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ret = add_system_iovec_to_sdma_packet(req, tx, cur_iovec,</div><div class='add'>+						      from_this_iovec);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+</div><div class='add'>+		remaining_to_add -= from_this_iovec;</div><div class='add'>+	}</div><div class='add'>+	*pkt_data_remaining = remaining_to_add;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr,</div><div class='add'>+			   unsigned long len)</div><div class='add'>+{</div><div class='add'>+	return (bool)(node-&gt;addr == addr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * Return 1 to remove the node from the rb tree and call the remove op.</div><div class='add'>+ *</div><div class='add'>+ * Called with the rb tree lock held.</div><div class='add'>+ */</div><div class='add'>+static int sdma_rb_evict(void *arg, struct mmu_rb_node *mnode,</div><div class='add'>+			 void *evict_arg, bool *stop)</div><div class='add'>+{</div><div class='add'>+	struct sdma_mmu_node *node =</div><div class='add'>+		container_of(mnode, struct sdma_mmu_node, rb);</div><div class='add'>+	struct evict_data *evict_data = evict_arg;</div><div class='add'>+</div><div class='add'>+	/* this node will be evicted, add its pages to our count */</div><div class='add'>+	evict_data-&gt;cleared += node-&gt;npages;</div><div class='add'>+</div><div class='add'>+	/* have enough pages been cleared? */</div><div class='add'>+	if (evict_data-&gt;cleared &gt;= evict_data-&gt;target)</div><div class='add'>+		*stop = true;</div><div class='add'>+</div><div class='add'>+	return 1; /* remove this node */</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void sdma_rb_remove(void *arg, struct mmu_rb_node *mnode)</div><div class='add'>+{</div><div class='add'>+	struct sdma_mmu_node *node =</div><div class='add'>+		container_of(mnode, struct sdma_mmu_node, rb);</div><div class='add'>+</div><div class='add'>+	free_system_node(node);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/pinning.h b/drivers/infiniband/hw/hfi1/pinning.h<br/>new file mode 100644<br/>index 000000000000..a814a3aa9654<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/pinning.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/pinning.h</a></div><div class='hunk'>@@ -0,0 +1,20 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */</div><div class='add'>+/*</div><div class='add'>+ * Copyright(c) 2023 Cornelis Networks, Inc.</div><div class='add'>+ */</div><div class='add'>+#ifndef _HFI1_PINNING_H</div><div class='add'>+#define _HFI1_PINNING_H</div><div class='add'>+</div><div class='add'>+struct hfi1_user_sdma_pkt_q;</div><div class='add'>+struct user_sdma_request;</div><div class='add'>+struct user_sdma_txreq;</div><div class='add'>+struct user_sdma_iovec;</div><div class='add'>+</div><div class='add'>+int hfi1_init_system_pinning(struct hfi1_user_sdma_pkt_q *pq);</div><div class='add'>+void hfi1_free_system_pinning(struct hfi1_user_sdma_pkt_q *pq);</div><div class='add'>+int hfi1_add_pages_to_sdma_packet(struct user_sdma_request *req,</div><div class='add'>+				  struct user_sdma_txreq *tx,</div><div class='add'>+				  struct user_sdma_iovec *iovec,</div><div class='add'>+				  u32 *pkt_data_remaining);</div><div class='add'>+</div><div class='add'>+#endif /* _HFI1_PINNING_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/pio.c b/drivers/infiniband/hw/hfi1/pio.c<br/>index 16ba9d52e1b9..764286da2ce8 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/pio.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/pio.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/pio.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/pio.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015-2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/delay.h&gt;</div><div class='hunk'>@@ -86,7 +44,7 @@ void pio_send_control(struct hfi1_devdata *dd, int op)</div><div class='ctx'> 	switch (op) {</div><div class='ctx'> 	case PSC_GLOBAL_ENABLE:</div><div class='ctx'> 		reg |= SEND_CTRL_SEND_ENABLE_SMASK;</div><div class='del'>-	/* Fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case PSC_DATA_VL_ENABLE:</div><div class='ctx'> 		mask = 0;</div><div class='ctx'> 		for (i = 0; i &lt; ARRAY_SIZE(dd-&gt;vld); i++)</div><div class='hunk'>@@ -862,7 +820,7 @@ struct send_context *sc_alloc(struct hfi1_devdata *dd, int type,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	hfi1_cdbg(PIO,</div><div class='del'>-		  "Send context %u(%u) %s group %u credits %u credit_ctrl 0x%llx threshold %u\n",</div><div class='add'>+		  "Send context %u(%u) %s group %u credits %u credit_ctrl 0x%llx threshold %u",</div><div class='ctx'> 		  sw_index,</div><div class='ctx'> 		  hw_context,</div><div class='ctx'> 		  sc_type_name(type),</div><div class='hunk'>@@ -920,6 +878,7 @@ void sc_disable(struct send_context *sc)</div><div class='ctx'> {</div><div class='ctx'> 	u64 reg;</div><div class='ctx'> 	struct pio_buf *pbuf;</div><div class='add'>+	LIST_HEAD(wake_list);</div><div class='ctx'> </div><div class='ctx'> 	if (!sc)</div><div class='ctx'> 		return;</div><div class='hunk'>@@ -952,6 +911,23 @@ void sc_disable(struct send_context *sc)</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock(&amp;sc-&gt;release_lock);</div><div class='add'>+</div><div class='add'>+	write_seqlock(&amp;sc-&gt;waitlock);</div><div class='add'>+	list_splice_init(&amp;sc-&gt;piowait, &amp;wake_list);</div><div class='add'>+	write_sequnlock(&amp;sc-&gt;waitlock);</div><div class='add'>+	while (!list_empty(&amp;wake_list)) {</div><div class='add'>+		struct iowait *wait;</div><div class='add'>+		struct rvt_qp *qp;</div><div class='add'>+		struct hfi1_qp_priv *priv;</div><div class='add'>+</div><div class='add'>+		wait = list_first_entry(&amp;wake_list, struct iowait, list);</div><div class='add'>+		qp = iowait_to_qp(wait);</div><div class='add'>+		priv = qp-&gt;priv;</div><div class='add'>+		list_del_init(&amp;priv-&gt;s_iowait.list);</div><div class='add'>+		priv-&gt;s_iowait.lock = NULL;</div><div class='add'>+		hfi1_qp_wakeup(qp, RVT_S_WAIT_PIO | HFI1_S_WAIT_PIO_DRAIN);</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	spin_unlock_irq(&amp;sc-&gt;alloc_lock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -977,7 +953,7 @@ static bool is_sc_halted(struct hfi1_devdata *dd, u32 hw_context)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * sc_wait_for_packet_egress</div><div class='add'>+ * sc_wait_for_packet_egress - wait for packet</div><div class='ctx'>  * @sc: valid send context</div><div class='ctx'>  * @pause: wait for credit return</div><div class='ctx'>  *</div><div class='hunk'>@@ -1385,16 +1361,6 @@ void sc_flush(struct send_context *sc)</div><div class='ctx'> 	sc_wait_for_packet_egress(sc, 1);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/* drop all packets on the context, no waiting until they are sent */</div><div class='del'>-void sc_drop(struct send_context *sc)</div><div class='del'>-{</div><div class='del'>-	if (!sc)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	dd_dev_info(sc-&gt;dd, "%s: context %u(%u) - not implemented\n",</div><div class='del'>-		    __func__, sc-&gt;sw_index, sc-&gt;hw_context);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> /*</div><div class='ctx'>  * Start the software reaction to a context halt or SPC freeze:</div><div class='ctx'>  *	- mark the context as halted or frozen</div><div class='hunk'>@@ -1427,7 +1393,8 @@ void sc_stop(struct send_context *sc, int flag)</div><div class='ctx'>  * @cb: optional callback to call when the buffer is finished sending</div><div class='ctx'>  * @arg: argument for cb</div><div class='ctx'>  *</div><div class='del'>- * Return a pointer to a PIO buffer if successful, NULL if not enough room.</div><div class='add'>+ * Return a pointer to a PIO buffer, NULL if not enough room, -ECOMM</div><div class='add'>+ * when link is down.</div><div class='ctx'>  */</div><div class='ctx'> struct pio_buf *sc_buffer_alloc(struct send_context *sc, u32 dw_len,</div><div class='ctx'> 				pio_release_cb cb, void *arg)</div><div class='hunk'>@@ -1443,7 +1410,7 @@ struct pio_buf *sc_buffer_alloc(struct send_context *sc, u32 dw_len,</div><div class='ctx'> 	spin_lock_irqsave(&amp;sc-&gt;alloc_lock, flags);</div><div class='ctx'> 	if (!(sc-&gt;flags &amp; SCF_ENABLED)) {</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;sc-&gt;alloc_lock, flags);</div><div class='del'>-		goto done;</div><div class='add'>+		return ERR_PTR(-ECOMM);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> retry:</div><div class='hunk'>@@ -1577,9 +1544,8 @@ void hfi1_sc_wantpiobuf_intr(struct send_context *sc, u32 needint)</div><div class='ctx'> 	else</div><div class='ctx'> 		sc_del_credit_return_intr(sc);</div><div class='ctx'> 	trace_hfi1_wantpiointr(sc, needint, sc-&gt;credit_ctrl);</div><div class='del'>-	if (needint) {</div><div class='add'>+	if (needint)</div><div class='ctx'> 		sc_return_credits(sc);</div><div class='del'>-	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='hunk'>@@ -1917,9 +1883,7 @@ int pio_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls, u8 *vl_scontexts)</div><div class='ctx'> 			vl_scontexts[i] = sc_per_vl + (extra &gt; 0 ? 1 : 0);</div><div class='ctx'> 	}</div><div class='ctx'> 	/* build new map */</div><div class='del'>-	newmap = kzalloc(sizeof(*newmap) +</div><div class='del'>-			 roundup_pow_of_two(num_vls) *</div><div class='del'>-			 sizeof(struct pio_map_elem *),</div><div class='add'>+	newmap = kzalloc(struct_size(newmap, map, roundup_pow_of_two(num_vls)),</div><div class='ctx'> 			 GFP_KERNEL);</div><div class='ctx'> 	if (!newmap)</div><div class='ctx'> 		goto bail;</div><div class='hunk'>@@ -1934,9 +1898,8 @@ int pio_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls, u8 *vl_scontexts)</div><div class='ctx'> 			int sz = roundup_pow_of_two(vl_scontexts[i]);</div><div class='ctx'> </div><div class='ctx'> 			/* only allocate once */</div><div class='del'>-			newmap-&gt;map[i] = kzalloc(sizeof(*newmap-&gt;map[i]) +</div><div class='del'>-						 sz * sizeof(struct</div><div class='del'>-							     send_context *),</div><div class='add'>+			newmap-&gt;map[i] = kzalloc(struct_size(newmap-&gt;map[i],</div><div class='add'>+							     ksc, sz),</div><div class='ctx'> 						 GFP_KERNEL);</div><div class='ctx'> 			if (!newmap-&gt;map[i])</div><div class='ctx'> 				goto bail;</div><div class='hunk'>@@ -2113,7 +2076,7 @@ int init_credit_return(struct hfi1_devdata *dd)</div><div class='ctx'> 				   "Unable to allocate credit return DMA range for NUMA %d\n",</div><div class='ctx'> 				   i);</div><div class='ctx'> 			ret = -ENOMEM;</div><div class='del'>-			goto done;</div><div class='add'>+			goto free_cr_base;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> 	set_dev_node(&amp;dd-&gt;pcidev-&gt;dev, dd-&gt;node);</div><div class='hunk'>@@ -2121,6 +2084,10 @@ int init_credit_return(struct hfi1_devdata *dd)</div><div class='ctx'> 	ret = 0;</div><div class='ctx'> done:</div><div class='ctx'> 	return ret;</div><div class='add'>+</div><div class='add'>+free_cr_base:</div><div class='add'>+	free_credit_return(dd);</div><div class='add'>+	goto done;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void free_credit_return(struct hfi1_devdata *dd)</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/pio.h b/drivers/infiniband/hw/hfi1/pio.h<br/>index c9a58b642bdd..ab0f9a3a8d12 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/pio.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/pio.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/pio.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/pio.h</a></div><div class='hunk'>@@ -1,52 +1,10 @@</div><div class='del'>-#ifndef _PIO_H</div><div class='del'>-#define _PIO_H</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015-2017 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='add'>+#ifndef _PIO_H</div><div class='add'>+#define _PIO_H</div><div class='ctx'> /* send context types */</div><div class='ctx'> #define SC_KERNEL 0</div><div class='ctx'> #define SC_VL15   1</div><div class='hunk'>@@ -243,7 +201,7 @@ struct sc_config_sizes {</div><div class='ctx'>  */</div><div class='ctx'> struct pio_map_elem {</div><div class='ctx'> 	u32 mask;</div><div class='del'>-	struct send_context *ksc[0];</div><div class='add'>+	struct send_context *ksc[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -263,7 +221,7 @@ struct pio_vl_map {</div><div class='ctx'> 	u32 mask;</div><div class='ctx'> 	u8 actual_vls;</div><div class='ctx'> 	u8 vls;</div><div class='del'>-	struct pio_map_elem *map[0];</div><div class='add'>+	struct pio_map_elem *map[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> int pio_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls,</div><div class='hunk'>@@ -279,7 +237,6 @@ int init_credit_return(struct hfi1_devdata *dd);</div><div class='ctx'> void free_credit_return(struct hfi1_devdata *dd);</div><div class='ctx'> int init_sc_pools_and_sizes(struct hfi1_devdata *dd);</div><div class='ctx'> int init_send_contexts(struct hfi1_devdata *dd);</div><div class='del'>-int init_credit_return(struct hfi1_devdata *dd);</div><div class='ctx'> int init_pervl_scs(struct hfi1_devdata *dd);</div><div class='ctx'> struct send_context *sc_alloc(struct hfi1_devdata *dd, int type,</div><div class='ctx'> 			      uint hdrqentsize, int numa);</div><div class='hunk'>@@ -289,12 +246,10 @@ void sc_disable(struct send_context *sc);</div><div class='ctx'> int sc_restart(struct send_context *sc);</div><div class='ctx'> void sc_return_credits(struct send_context *sc);</div><div class='ctx'> void sc_flush(struct send_context *sc);</div><div class='del'>-void sc_drop(struct send_context *sc);</div><div class='ctx'> void sc_stop(struct send_context *sc, int bit);</div><div class='ctx'> struct pio_buf *sc_buffer_alloc(struct send_context *sc, u32 dw_len,</div><div class='ctx'> 				pio_release_cb cb, void *arg);</div><div class='ctx'> void sc_release_update(struct send_context *sc);</div><div class='del'>-void sc_return_credits(struct send_context *sc);</div><div class='ctx'> void sc_group_release_update(struct hfi1_devdata *dd, u32 hw_context);</div><div class='ctx'> void sc_add_credit_return_intr(struct send_context *sc);</div><div class='ctx'> void sc_del_credit_return_intr(struct send_context *sc);</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/pio_copy.c b/drivers/infiniband/hw/hfi1/pio_copy.c<br/>index 03024cec78dd..80fee812a930 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/pio_copy.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/pio_copy.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/pio_copy.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/pio_copy.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015, 2016 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include "hfi.h"</div><div class='hunk'>@@ -55,6 +13,7 @@</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * pio_copy - copy data block to MMIO space</div><div class='add'>+ * @dd: hfi1 dev data</div><div class='ctx'>  * @pbuf: a number of blocks allocated within a PIO send context</div><div class='ctx'>  * @pbc: PBC to send</div><div class='ctx'>  * @from: source, must be 8 byte aligned</div><div class='hunk'>@@ -191,30 +150,29 @@ static inline void jcopy(u8 *dest, const u8 *src, u32 n)</div><div class='ctx'> 	switch (n) {</div><div class='ctx'> 	case 7:</div><div class='ctx'> 		*dest++ = *src++;</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case 6:</div><div class='ctx'> 		*dest++ = *src++;</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case 5:</div><div class='ctx'> 		*dest++ = *src++;</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case 4:</div><div class='ctx'> 		*dest++ = *src++;</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case 3:</div><div class='ctx'> 		*dest++ = *src++;</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case 2:</div><div class='ctx'> 		*dest++ = *src++;</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case 1:</div><div class='ctx'> 		*dest++ = *src++;</div><div class='del'>-		/* fall through */</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='del'>- * Read nbytes from "from" and and place them in the low bytes</div><div class='add'>+ * Read nbytes from "from" and place them in the low bytes</div><div class='ctx'>  * of pbuf-&gt;carry.  Other bytes are left as-is.  Any previous</div><div class='ctx'>  * value in pbuf-&gt;carry is lost.</div><div class='ctx'>  *</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/platform.c b/drivers/infiniband/hw/hfi1/platform.c<br/>index cbf7faa5038c..7bd0e9b6cb50 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/platform.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/platform.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/platform.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/platform.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015, 2016 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/firmware.h&gt;</div><div class='hunk'>@@ -634,7 +592,7 @@ static void apply_tx_lanes(struct hfi1_pportdata *ppd, u8 field_id,</div><div class='ctx'> 			   u32 config_data, const char *message)</div><div class='ctx'> {</div><div class='ctx'> 	u8 i;</div><div class='del'>-	int ret = HCMD_SUCCESS;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; 4; i++) {</div><div class='ctx'> 		ret = load_8051_config(ppd-&gt;dd, field_id, i, config_data);</div><div class='hunk'>@@ -668,8 +626,8 @@ static u8 aoc_low_power_setting(struct hfi1_pportdata *ppd)</div><div class='ctx'> </div><div class='ctx'> 	/* active optical cables only */</div><div class='ctx'> 	switch ((cache[QSFP_MOD_TECH_OFFS] &amp; 0xF0) &gt;&gt; 4) {</div><div class='del'>-	case 0x0 ... 0x9: /* fallthrough */</div><div class='del'>-	case 0xC: /* fallthrough */</div><div class='add'>+	case 0x0 ... 0x9: fallthrough;</div><div class='add'>+	case 0xC: fallthrough;</div><div class='ctx'> 	case 0xE:</div><div class='ctx'> 		/* active AOC */</div><div class='ctx'> 		power_class = get_qsfp_power_class(cache[QSFP_MOD_PWR_OFFS]);</div><div class='hunk'>@@ -899,8 +857,8 @@ static int tune_qsfp(struct hfi1_pportdata *ppd,</div><div class='ctx'> </div><div class='ctx'> 		*ptr_tuning_method = OPA_PASSIVE_TUNING;</div><div class='ctx'> 		break;</div><div class='del'>-	case 0x0 ... 0x9: /* fallthrough */</div><div class='del'>-	case 0xC: /* fallthrough */</div><div class='add'>+	case 0x0 ... 0x9: fallthrough;</div><div class='add'>+	case 0xC: fallthrough;</div><div class='ctx'> 	case 0xE:</div><div class='ctx'> 		ret = tune_active_qsfp(ppd, ptr_tx_preset, ptr_rx_preset,</div><div class='ctx'> 				       ptr_total_atten);</div><div class='hunk'>@@ -909,7 +867,7 @@ static int tune_qsfp(struct hfi1_pportdata *ppd,</div><div class='ctx'> </div><div class='ctx'> 		*ptr_tuning_method = OPA_ACTIVE_TUNING;</div><div class='ctx'> 		break;</div><div class='del'>-	case 0xD: /* fallthrough */</div><div class='add'>+	case 0xD: fallthrough;</div><div class='ctx'> 	case 0xF:</div><div class='ctx'> 	default:</div><div class='ctx'> 		dd_dev_warn(ppd-&gt;dd, "%s: Unknown/unsupported cable\n",</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/platform.h b/drivers/infiniband/hw/hfi1/platform.h<br/>index eed0aa9124fa..0631f9bf3a89 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/platform.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/platform.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/platform.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/platform.h</a></div><div class='hunk'>@@ -1,49 +1,8 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015, 2016 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #ifndef __PLATFORM_H</div><div class='ctx'> #define __PLATFORM_H</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/qp.c b/drivers/infiniband/hw/hfi1/qp.c<br/>index 4e0e9fc0a777..f3d8c0c193ac 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/qp.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/qp.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/qp.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/qp.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2015 - 2020 Intel Corporation.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/err.h&gt;</div><div class='hunk'>@@ -186,31 +144,17 @@ static void flush_iowait(struct rvt_qp *qp)</div><div class='ctx'> 	write_sequnlock_irqrestore(lock, flags);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline int opa_mtu_enum_to_int(int mtu)</div><div class='del'>-{</div><div class='del'>-	switch (mtu) {</div><div class='del'>-	case OPA_MTU_8192:  return 8192;</div><div class='del'>-	case OPA_MTU_10240: return 10240;</div><div class='del'>-	default:            return -1;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='add'>+/*</div><div class='ctx'>  * This function is what we would push to the core layer if we wanted to be a</div><div class='ctx'>  * "first class citizen".  Instead we hide this here and rely on Verbs ULPs</div><div class='ctx'>  * to blindly pass the MTU enum value from the PathRecord to us.</div><div class='ctx'>  */</div><div class='ctx'> static inline int verbs_mtu_enum_to_int(struct ib_device *dev, enum ib_mtu mtu)</div><div class='ctx'> {</div><div class='del'>-	int val;</div><div class='del'>-</div><div class='ctx'> 	/* Constraining 10KB packets to 8KB packets */</div><div class='ctx'> 	if (mtu == (enum ib_mtu)OPA_MTU_10240)</div><div class='del'>-		mtu = OPA_MTU_8192;</div><div class='del'>-	val = opa_mtu_enum_to_int((int)mtu);</div><div class='del'>-	if (val &gt; 0)</div><div class='del'>-		return val;</div><div class='del'>-	return ib_mtu_enum_to_int(mtu);</div><div class='add'>+		mtu = (enum ib_mtu)OPA_MTU_8192;</div><div class='add'>+	return opa_mtu_enum_to_int((enum opa_mtu)mtu);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int hfi1_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr,</div><div class='hunk'>@@ -303,9 +247,9 @@ void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr,</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * hfi1_setup_wqe - set up the wqe</div><div class='del'>- * @qp - The qp</div><div class='del'>- * @wqe - The built wqe</div><div class='del'>- * @call_send - Determine if the send should be posted or scheduled.</div><div class='add'>+ * @qp: The qp</div><div class='add'>+ * @wqe: The built wqe</div><div class='add'>+ * @call_send: Determine if the send should be posted or scheduled.</div><div class='ctx'>  *</div><div class='ctx'>  * Perform setup of the wqe.  This is called</div><div class='ctx'>  * prior to inserting the wqe into the ring but after</div><div class='hunk'>@@ -326,7 +270,7 @@ int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, bool *call_send)</div><div class='ctx'> 	switch (qp-&gt;ibqp.qp_type) {</div><div class='ctx'> 	case IB_QPT_RC:</div><div class='ctx'> 		hfi1_setup_tid_rdma_wqe(qp, wqe);</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case IB_QPT_UC:</div><div class='ctx'> 		if (wqe-&gt;length &gt; 0x80000000U)</div><div class='ctx'> 			return -EINVAL;</div><div class='hunk'>@@ -348,11 +292,12 @@ int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, bool *call_send)</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_QPT_GSI:</div><div class='ctx'> 	case IB_QPT_UD:</div><div class='del'>-		ah = ibah_to_rvtah(wqe-&gt;ud_wr.ah);</div><div class='add'>+		ah = rvt_get_swqe_ah(wqe);</div><div class='ctx'> 		if (wqe-&gt;length &gt; (1 &lt;&lt; ah-&gt;log_pmtu))</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 		if (ibp-&gt;sl_to_sc[rdma_ah_get_sl(&amp;ah-&gt;attr)] == 0xf)</div><div class='ctx'> 			return -EINVAL;</div><div class='add'>+		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -381,7 +326,10 @@ bool _hfi1_schedule_send(struct rvt_qp *qp)</div><div class='ctx'> 	struct hfi1_ibport *ibp =</div><div class='ctx'> 		to_iport(qp-&gt;ibqp.device, qp-&gt;port_num);</div><div class='ctx'> 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);</div><div class='del'>-	struct hfi1_devdata *dd = dd_from_ibdev(qp-&gt;ibqp.device);</div><div class='add'>+	struct hfi1_devdata *dd = ppd-&gt;dd;</div><div class='add'>+</div><div class='add'>+	if (dd-&gt;flags &amp; HFI1_SHUTDOWN)</div><div class='add'>+		return true;</div><div class='ctx'> </div><div class='ctx'> 	return iowait_schedule(&amp;priv-&gt;s_iowait, ppd-&gt;hfi1_wq,</div><div class='ctx'> 			       priv-&gt;s_sde ?</div><div class='hunk'>@@ -605,7 +553,7 @@ struct sdma_engine *qp_to_sdma_engine(struct rvt_qp *qp, u8 sc5)</div><div class='ctx'> 	return sde;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/*</div><div class='add'>+/**</div><div class='ctx'>  * qp_to_send_context - map a qp to a send context</div><div class='ctx'>  * @qp: the QP</div><div class='ctx'>  * @sc5: the 5 bit sc</div><div class='hunk'>@@ -702,8 +650,8 @@ void qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter)</div><div class='ctx'> 		   sde ? sde-&gt;this_idx : 0,</div><div class='ctx'> 		   send_context,</div><div class='ctx'> 		   send_context ? send_context-&gt;sw_index : 0,</div><div class='del'>-		   ibcq_to_rvtcq(qp-&gt;ibqp.send_cq)-&gt;queue-&gt;head,</div><div class='del'>-		   ibcq_to_rvtcq(qp-&gt;ibqp.send_cq)-&gt;queue-&gt;tail,</div><div class='add'>+		   ib_cq_head(qp-&gt;ibqp.send_cq),</div><div class='add'>+		   ib_cq_tail(qp-&gt;ibqp.send_cq),</div><div class='ctx'> 		   qp-&gt;pid,</div><div class='ctx'> 		   qp-&gt;s_state,</div><div class='ctx'> 		   qp-&gt;s_ack_state,</div><div class='hunk'>@@ -922,8 +870,8 @@ void notify_error_qp(struct rvt_qp *qp)</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * hfi1_qp_iter_cb - callback for iterator</div><div class='del'>- * @qp - the qp</div><div class='del'>- * @v - the sl in low bits of v</div><div class='add'>+ * @qp: the qp</div><div class='add'>+ * @v: the sl in low bits of v</div><div class='ctx'>  *</div><div class='ctx'>  * This is called from the iterator callback to work</div><div class='ctx'>  * on an individual qp.</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/qp.h b/drivers/infiniband/hw/hfi1/qp.h<br/>index b670321365d3..870ff1a6e5c4 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/qp.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/qp.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/qp.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/qp.h</a></div><div class='hunk'>@@ -1,52 +1,10 @@</div><div class='del'>-#ifndef _QP_H</div><div class='del'>-#define _QP_H</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='add'>+#ifndef _QP_H</div><div class='add'>+#define _QP_H</div><div class='ctx'> #include &lt;linux/hash.h&gt;</div><div class='ctx'> #include &lt;rdma/rdmavt_qp.h&gt;</div><div class='ctx'> #include "verbs.h"</div><div class='hunk'>@@ -113,20 +71,6 @@ static inline void clear_ahg(struct rvt_qp *qp)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * hfi1_create_qp - create a queue pair for a device</div><div class='del'>- * @ibpd: the protection domain who's device we create the queue pair for</div><div class='del'>- * @init_attr: the attributes of the queue pair</div><div class='del'>- * @udata: user data for libibverbs.so</div><div class='del'>- *</div><div class='del'>- * Returns the queue pair on success, otherwise returns an errno.</div><div class='del'>- *</div><div class='del'>- * Called by the ib_create_qp() core verbs function.</div><div class='del'>- */</div><div class='del'>-struct ib_qp *hfi1_create_qp(struct ib_pd *ibpd,</div><div class='del'>-			     struct ib_qp_init_attr *init_attr,</div><div class='del'>-			     struct ib_udata *udata);</div><div class='del'>-</div><div class='del'>-/**</div><div class='ctx'>  * hfi1_qp_wakeup - wake up on the indicated event</div><div class='ctx'>  * @qp: the QP</div><div class='ctx'>  * @flag: flag the qp on which the qp is stalled</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/qsfp.c b/drivers/infiniband/hw/hfi1/qsfp.c<br/>index b5966991d647..3b7842a7f634 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/qsfp.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/qsfp.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/qsfp.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/qsfp.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015, 2016 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/delay.h&gt;</div><div class='hunk'>@@ -231,7 +189,7 @@ static int i2c_bus_write(struct hfi1_devdata *dd, struct hfi1_i2c_bus *i2c,</div><div class='ctx'> 		break;</div><div class='ctx'> 	case 2:</div><div class='ctx'> 		offset_bytes[1] = (offset &gt;&gt; 8) &amp; 0xff;</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case 1:</div><div class='ctx'> 		num_msgs = 2;</div><div class='ctx'> 		offset_bytes[0] = offset &amp; 0xff;</div><div class='hunk'>@@ -242,7 +200,7 @@ static int i2c_bus_write(struct hfi1_devdata *dd, struct hfi1_i2c_bus *i2c,</div><div class='ctx'> 		msgs[0].buf = offset_bytes;</div><div class='ctx'> </div><div class='ctx'> 		msgs[1].addr = slave_addr;</div><div class='del'>-		msgs[1].flags = I2C_M_NOSTART,</div><div class='add'>+		msgs[1].flags = I2C_M_NOSTART;</div><div class='ctx'> 		msgs[1].len = len;</div><div class='ctx'> 		msgs[1].buf = data;</div><div class='ctx'> 		break;</div><div class='hunk'>@@ -279,7 +237,7 @@ static int i2c_bus_read(struct hfi1_devdata *dd, struct hfi1_i2c_bus *bus,</div><div class='ctx'> 		break;</div><div class='ctx'> 	case 2:</div><div class='ctx'> 		offset_bytes[1] = (offset &gt;&gt; 8) &amp; 0xff;</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case 1:</div><div class='ctx'> 		num_msgs = 2;</div><div class='ctx'> 		offset_bytes[0] = offset &amp; 0xff;</div><div class='hunk'>@@ -290,7 +248,7 @@ static int i2c_bus_read(struct hfi1_devdata *dd, struct hfi1_i2c_bus *bus,</div><div class='ctx'> 		msgs[0].buf = offset_bytes;</div><div class='ctx'> </div><div class='ctx'> 		msgs[1].addr = slave_addr;</div><div class='del'>-		msgs[1].flags = I2C_M_RD,</div><div class='add'>+		msgs[1].flags = I2C_M_RD;</div><div class='ctx'> 		msgs[1].len = len;</div><div class='ctx'> 		msgs[1].buf = data;</div><div class='ctx'> 		break;</div><div class='hunk'>@@ -447,26 +405,6 @@ int qsfp_write(struct hfi1_pportdata *ppd, u32 target, int addr, void *bp,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='del'>- * Perform a stand-alone single QSFP write.  Acquire the resource, do the</div><div class='del'>- * write, then release the resource.</div><div class='del'>- */</div><div class='del'>-int one_qsfp_write(struct hfi1_pportdata *ppd, u32 target, int addr, void *bp,</div><div class='del'>-		   int len)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_devdata *dd = ppd-&gt;dd;</div><div class='del'>-	u32 resource = qsfp_resource(dd);</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	ret = acquire_chip_resource(dd, resource, QSFP_WAIT);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-	ret = qsfp_write(ppd, target, addr, bp, len);</div><div class='del'>-	release_chip_resource(dd, resource);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='ctx'>  * Access page n, offset m of QSFP memory as defined by SFF 8636</div><div class='ctx'>  * by reading @addr = ((256 * n) + m)</div><div class='ctx'>  *</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/qsfp.h b/drivers/infiniband/hw/hfi1/qsfp.h<br/>index 36cf52359848..5c59d53fcb63 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/qsfp.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/qsfp.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/qsfp.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/qsfp.h</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015, 2016 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> /* QSFP support common definitions, for hfi driver */</div><div class='ctx'> </div><div class='hunk'>@@ -237,8 +195,6 @@ int qsfp_write(struct hfi1_pportdata *ppd, u32 target, int addr, void *bp,</div><div class='ctx'> 	       int len);</div><div class='ctx'> int qsfp_read(struct hfi1_pportdata *ppd, u32 target, int addr, void *bp,</div><div class='ctx'> 	      int len);</div><div class='del'>-int one_qsfp_write(struct hfi1_pportdata *ppd, u32 target, int addr, void *bp,</div><div class='del'>-		   int len);</div><div class='ctx'> int one_qsfp_read(struct hfi1_pportdata *ppd, u32 target, int addr, void *bp,</div><div class='ctx'> 		  int len);</div><div class='ctx'> struct hfi1_asic_data;</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c<br/>index a922edcf23d6..b36242c9d42c 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/rc.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/rc.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/rc.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/rc.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/io.h&gt;</div><div class='hunk'>@@ -141,7 +99,7 @@ static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp,</div><div class='ctx'> 	case OP(RDMA_READ_RESPONSE_ONLY):</div><div class='ctx'> 		e = &amp;qp-&gt;s_ack_queue[qp-&gt;s_tail_ack_queue];</div><div class='ctx'> 		release_rdma_sge_mr(e);</div><div class='del'>-		/* FALLTHROUGH */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case OP(ATOMIC_ACKNOWLEDGE):</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * We can increment the tail pointer now that the last</div><div class='hunk'>@@ -160,7 +118,7 @@ static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp,</div><div class='ctx'> 			qp-&gt;s_acked_ack_queue = next;</div><div class='ctx'> 		qp-&gt;s_tail_ack_queue = next;</div><div class='ctx'> 		trace_hfi1_rsp_make_rc_ack(qp, e-&gt;psn);</div><div class='del'>-		/* FALLTHROUGH */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case OP(SEND_ONLY):</div><div class='ctx'> 	case OP(ACKNOWLEDGE):</div><div class='ctx'> 		/* Check for no next entry in the queue. */</div><div class='hunk'>@@ -267,7 +225,7 @@ static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp,</div><div class='ctx'> </div><div class='ctx'> 	case OP(RDMA_READ_RESPONSE_FIRST):</div><div class='ctx'> 		qp-&gt;s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE);</div><div class='del'>-		/* FALLTHROUGH */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case OP(RDMA_READ_RESPONSE_MIDDLE):</div><div class='ctx'> 		ps-&gt;s_txreq-&gt;ss = &amp;qp-&gt;s_ack_rdma_sge;</div><div class='ctx'> 		ps-&gt;s_txreq-&gt;mr = qp-&gt;s_ack_rdma_sge.sge.mr;</div><div class='hunk'>@@ -421,6 +379,7 @@ bail:</div><div class='ctx'> /**</div><div class='ctx'>  * hfi1_make_rc_req - construct a request packet (SEND, RDMA r/w, ATOMIC)</div><div class='ctx'>  * @qp: a pointer to the QP</div><div class='add'>+ * @ps: the current packet state</div><div class='ctx'>  *</div><div class='ctx'>  * Assumes s_lock is held.</div><div class='ctx'>  *</div><div class='hunk'>@@ -595,11 +554,8 @@ check_s_state:</div><div class='ctx'> 		case IB_WR_SEND_WITH_IMM:</div><div class='ctx'> 		case IB_WR_SEND_WITH_INV:</div><div class='ctx'> 			/* If no credit, return. */</div><div class='del'>-			if (!(qp-&gt;s_flags &amp; RVT_S_UNLIMITED_CREDIT) &amp;&amp;</div><div class='del'>-			    rvt_cmp_msn(wqe-&gt;ssn, qp-&gt;s_lsn + 1) &gt; 0) {</div><div class='del'>-				qp-&gt;s_flags |= RVT_S_WAIT_SSN_CREDIT;</div><div class='add'>+			if (!rvt_rc_credit_avail(qp, wqe))</div><div class='ctx'> 				goto bail;</div><div class='del'>-			}</div><div class='ctx'> 			if (len &gt; pmtu) {</div><div class='ctx'> 				qp-&gt;s_state = OP(SEND_FIRST);</div><div class='ctx'> 				len = pmtu;</div><div class='hunk'>@@ -632,11 +588,8 @@ check_s_state:</div><div class='ctx'> 			goto no_flow_control;</div><div class='ctx'> 		case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='ctx'> 			/* If no credit, return. */</div><div class='del'>-			if (!(qp-&gt;s_flags &amp; RVT_S_UNLIMITED_CREDIT) &amp;&amp;</div><div class='del'>-			    rvt_cmp_msn(wqe-&gt;ssn, qp-&gt;s_lsn + 1) &gt; 0) {</div><div class='del'>-				qp-&gt;s_flags |= RVT_S_WAIT_SSN_CREDIT;</div><div class='add'>+			if (!rvt_rc_credit_avail(qp, wqe))</div><div class='ctx'> 				goto bail;</div><div class='del'>-			}</div><div class='ctx'> no_flow_control:</div><div class='ctx'> 			put_ib_reth_vaddr(</div><div class='ctx'> 				wqe-&gt;rdma_wr.remote_addr,</div><div class='hunk'>@@ -887,8 +840,7 @@ no_flow_control:</div><div class='ctx'> 				goto bail;</div><div class='ctx'> 			}</div><div class='ctx'> 			qp-&gt;s_num_rd_atomic++;</div><div class='del'>-</div><div class='del'>-			/* FALLTHROUGH */</div><div class='add'>+			fallthrough;</div><div class='ctx'> 		case IB_WR_OPFN:</div><div class='ctx'> 			if (newreq &amp;&amp; !(qp-&gt;s_flags &amp; RVT_S_UNLIMITED_CREDIT))</div><div class='ctx'> 				qp-&gt;s_lsn++;</div><div class='hunk'>@@ -952,10 +904,10 @@ no_flow_control:</div><div class='ctx'> 		 * See restart_rc().</div><div class='ctx'> 		 */</div><div class='ctx'> 		qp-&gt;s_len = restart_sge(&amp;qp-&gt;s_sge, wqe, qp-&gt;s_psn, pmtu);</div><div class='del'>-		/* FALLTHROUGH */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case OP(SEND_FIRST):</div><div class='ctx'> 		qp-&gt;s_state = OP(SEND_MIDDLE);</div><div class='del'>-		/* FALLTHROUGH */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case OP(SEND_MIDDLE):</div><div class='ctx'> 		bth2 = mask_psn(qp-&gt;s_psn++);</div><div class='ctx'> 		ss = &amp;qp-&gt;s_sge;</div><div class='hunk'>@@ -997,10 +949,10 @@ no_flow_control:</div><div class='ctx'> 		 * See restart_rc().</div><div class='ctx'> 		 */</div><div class='ctx'> 		qp-&gt;s_len = restart_sge(&amp;qp-&gt;s_sge, wqe, qp-&gt;s_psn, pmtu);</div><div class='del'>-		/* FALLTHROUGH */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case OP(RDMA_WRITE_FIRST):</div><div class='ctx'> 		qp-&gt;s_state = OP(RDMA_WRITE_MIDDLE);</div><div class='del'>-		/* FALLTHROUGH */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case OP(RDMA_WRITE_MIDDLE):</div><div class='ctx'> 		bth2 = mask_psn(qp-&gt;s_psn++);</div><div class='ctx'> 		ss = &amp;qp-&gt;s_sge;</div><div class='hunk'>@@ -1382,9 +1334,8 @@ static const hfi1_make_rc_ack hfi1_make_rc_ack_tbl[2] = {</div><div class='ctx'> 	[HFI1_PKT_TYPE_16B] = &amp;hfi1_make_rc_ack_16B</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/**</div><div class='add'>+/*</div><div class='ctx'>  * hfi1_send_rc_ack - Construct an ACK packet and send it</div><div class='del'>- * @qp: a pointer to the QP</div><div class='ctx'>  *</div><div class='ctx'>  * This is called from hfi1_rc_rcv() and handle_receive_interrupt().</div><div class='ctx'>  * Note that RDMA reads and atomics are handled in the</div><div class='hunk'>@@ -1432,7 +1383,7 @@ void hfi1_send_rc_ack(struct hfi1_packet *packet, bool is_fecn)</div><div class='ctx'> 	pbc = create_pbc(ppd, pbc_flags, qp-&gt;srate_mbps,</div><div class='ctx'> 			 sc_to_vlt(ppd-&gt;dd, sc5), plen);</div><div class='ctx'> 	pbuf = sc_buffer_alloc(rcd-&gt;sc, plen, NULL, NULL);</div><div class='del'>-	if (!pbuf) {</div><div class='add'>+	if (IS_ERR_OR_NULL(pbuf)) {</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * We have no room to send at the moment.  Pass</div><div class='ctx'> 		 * responsibility for sending the ACK to the send engine</div><div class='hunk'>@@ -1483,6 +1434,11 @@ static void update_num_rd_atomic(struct rvt_qp *qp, u32 psn,</div><div class='ctx'> 			req-&gt;ack_pending = cur_seg - req-&gt;comp_seg;</div><div class='ctx'> 			priv-&gt;pending_tid_r_segs += req-&gt;ack_pending;</div><div class='ctx'> 			qp-&gt;s_num_rd_atomic += req-&gt;ack_pending;</div><div class='add'>+			trace_hfi1_tid_req_update_num_rd_atomic(qp, 0,</div><div class='add'>+								wqe-&gt;wr.opcode,</div><div class='add'>+								wqe-&gt;psn,</div><div class='add'>+								wqe-&gt;lpsn,</div><div class='add'>+								req);</div><div class='ctx'> 		} else {</div><div class='ctx'> 			priv-&gt;pending_tid_r_segs += req-&gt;total_segs;</div><div class='ctx'> 			qp-&gt;s_num_rd_atomic += req-&gt;total_segs;</div><div class='hunk'>@@ -1701,6 +1657,36 @@ static void reset_sending_psn(struct rvt_qp *qp, u32 psn)</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+/**</div><div class='add'>+ * hfi1_rc_verbs_aborted - handle abort status</div><div class='add'>+ * @qp: the QP</div><div class='add'>+ * @opah: the opa header</div><div class='add'>+ *</div><div class='add'>+ * This code modifies both ACK bit in BTH[2]</div><div class='add'>+ * and the s_flags to go into send one mode.</div><div class='add'>+ *</div><div class='add'>+ * This serves to throttle the send engine to only</div><div class='add'>+ * send a single packet in the likely case the</div><div class='add'>+ * a link has gone down.</div><div class='add'>+ */</div><div class='add'>+void hfi1_rc_verbs_aborted(struct rvt_qp *qp, struct hfi1_opa_header *opah)</div><div class='add'>+{</div><div class='add'>+	struct ib_other_headers *ohdr = hfi1_get_rc_ohdr(opah);</div><div class='add'>+	u8 opcode = ib_bth_get_opcode(ohdr);</div><div class='add'>+	u32 psn;</div><div class='add'>+</div><div class='add'>+	/* ignore responses */</div><div class='add'>+	if ((opcode &gt;= OP(RDMA_READ_RESPONSE_FIRST) &amp;&amp;</div><div class='add'>+	     opcode &lt;= OP(ATOMIC_ACKNOWLEDGE)) ||</div><div class='add'>+	    opcode == TID_OP(READ_RESP) ||</div><div class='add'>+	    opcode == TID_OP(WRITE_RESP))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	psn = ib_bth_get_psn(ohdr) | IB_BTH_REQ_ACK;</div><div class='add'>+	ohdr-&gt;bth[2] = cpu_to_be32(psn);</div><div class='add'>+	qp-&gt;s_flags |= RVT_S_SEND_ONE;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /*</div><div class='ctx'>  * This should be called with the QP s_lock held and interrupts disabled.</div><div class='ctx'>  */</div><div class='hunk'>@@ -1709,8 +1695,6 @@ void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah)</div><div class='ctx'> 	struct ib_other_headers *ohdr;</div><div class='ctx'> 	struct hfi1_qp_priv *priv = qp-&gt;priv;</div><div class='ctx'> 	struct rvt_swqe *wqe;</div><div class='del'>-	struct ib_header *hdr = NULL;</div><div class='del'>-	struct hfi1_16b_header *hdr_16b = NULL;</div><div class='ctx'> 	u32 opcode, head, tail;</div><div class='ctx'> 	u32 psn;</div><div class='ctx'> 	struct tid_rdma_request *req;</div><div class='hunk'>@@ -1719,24 +1703,7 @@ void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah)</div><div class='ctx'> 	if (!(ib_rvt_state_ops[qp-&gt;state] &amp; RVT_SEND_OR_FLUSH_OR_RECV_OK))</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='del'>-	/* Find out where the BTH is */</div><div class='del'>-	if (priv-&gt;hdr_type == HFI1_PKT_TYPE_9B) {</div><div class='del'>-		hdr = &amp;opah-&gt;ibh;</div><div class='del'>-		if (ib_get_lnh(hdr) == HFI1_LRH_BTH)</div><div class='del'>-			ohdr = &amp;hdr-&gt;u.oth;</div><div class='del'>-		else</div><div class='del'>-			ohdr = &amp;hdr-&gt;u.l.oth;</div><div class='del'>-	} else {</div><div class='del'>-		u8 l4;</div><div class='del'>-</div><div class='del'>-		hdr_16b = &amp;opah-&gt;opah;</div><div class='del'>-		l4  = hfi1_16B_get_l4(hdr_16b);</div><div class='del'>-		if (l4 == OPA_16B_L4_IB_LOCAL)</div><div class='del'>-			ohdr = &amp;hdr_16b-&gt;u.oth;</div><div class='del'>-		else</div><div class='del'>-			ohdr = &amp;hdr_16b-&gt;u.l.oth;</div><div class='del'>-	}</div><div class='del'>-</div><div class='add'>+	ohdr = hfi1_get_rc_ohdr(opah);</div><div class='ctx'> 	opcode = ib_bth_get_opcode(ohdr);</div><div class='ctx'> 	if ((opcode &gt;= OP(RDMA_READ_RESPONSE_FIRST) &amp;&amp;</div><div class='ctx'> 	     opcode &lt;= OP(ATOMIC_ACKNOWLEDGE)) ||</div><div class='hunk'>@@ -1819,23 +1786,13 @@ void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	while (qp-&gt;s_last != qp-&gt;s_acked) {</div><div class='del'>-		u32 s_last;</div><div class='del'>-</div><div class='ctx'> 		wqe = rvt_get_swqe_ptr(qp, qp-&gt;s_last);</div><div class='ctx'> 		if (cmp_psn(wqe-&gt;lpsn, qp-&gt;s_sending_psn) &gt;= 0 &amp;&amp;</div><div class='ctx'> 		    cmp_psn(qp-&gt;s_sending_psn, qp-&gt;s_sending_hpsn) &lt;= 0)</div><div class='ctx'> 			break;</div><div class='ctx'> 		trdma_clean_swqe(qp, wqe);</div><div class='del'>-		rvt_qp_wqe_unreserve(qp, wqe);</div><div class='del'>-		s_last = qp-&gt;s_last;</div><div class='del'>-		trace_hfi1_qp_send_completion(qp, wqe, s_last);</div><div class='del'>-		if (++s_last &gt;= qp-&gt;s_size)</div><div class='del'>-			s_last = 0;</div><div class='del'>-		qp-&gt;s_last = s_last;</div><div class='del'>-		/* see post_send() */</div><div class='del'>-		barrier();</div><div class='del'>-		rvt_put_qp_swqe(qp, wqe);</div><div class='del'>-		rvt_qp_swqe_complete(qp,</div><div class='add'>+		trace_hfi1_qp_send_completion(qp, wqe, qp-&gt;s_last);</div><div class='add'>+		rvt_qp_complete_swqe(qp,</div><div class='ctx'> 				     wqe,</div><div class='ctx'> 				     ib_hfi1_wc_opcode[wqe-&gt;wr.opcode],</div><div class='ctx'> 				     IB_WC_SUCCESS);</div><div class='hunk'>@@ -1879,19 +1836,9 @@ struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,</div><div class='ctx'> 	trace_hfi1_rc_completion(qp, wqe-&gt;lpsn);</div><div class='ctx'> 	if (cmp_psn(wqe-&gt;lpsn, qp-&gt;s_sending_psn) &lt; 0 ||</div><div class='ctx'> 	    cmp_psn(qp-&gt;s_sending_psn, qp-&gt;s_sending_hpsn) &gt; 0) {</div><div class='del'>-		u32 s_last;</div><div class='del'>-</div><div class='ctx'> 		trdma_clean_swqe(qp, wqe);</div><div class='del'>-		rvt_put_qp_swqe(qp, wqe);</div><div class='del'>-		rvt_qp_wqe_unreserve(qp, wqe);</div><div class='del'>-		s_last = qp-&gt;s_last;</div><div class='del'>-		trace_hfi1_qp_send_completion(qp, wqe, s_last);</div><div class='del'>-		if (++s_last &gt;= qp-&gt;s_size)</div><div class='del'>-			s_last = 0;</div><div class='del'>-		qp-&gt;s_last = s_last;</div><div class='del'>-		/* see post_send() */</div><div class='del'>-		barrier();</div><div class='del'>-		rvt_qp_swqe_complete(qp,</div><div class='add'>+		trace_hfi1_qp_send_completion(qp, wqe, qp-&gt;s_last);</div><div class='add'>+		rvt_qp_complete_swqe(qp,</div><div class='ctx'> 				     wqe,</div><div class='ctx'> 				     ib_hfi1_wc_opcode[wqe-&gt;wr.opcode],</div><div class='ctx'> 				     IB_WC_SUCCESS);</div><div class='hunk'>@@ -2003,7 +1950,7 @@ static void update_qp_retry_state(struct rvt_qp *qp, u32 psn, u32 spsn,</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/**</div><div class='add'>+/*</div><div class='ctx'>  * do_rc_ack - process an incoming RC ACK</div><div class='ctx'>  * @qp: the QP the ACK came in on</div><div class='ctx'>  * @psn: the packet sequence number of the ACK</div><div class='hunk'>@@ -2219,15 +2166,15 @@ int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode,</div><div class='ctx'> 		if (qp-&gt;s_flags &amp; RVT_S_WAIT_RNR)</div><div class='ctx'> 			goto bail_stop;</div><div class='ctx'> 		rdi = ib_to_rvt(qp-&gt;ibqp.device);</div><div class='del'>-		if (qp-&gt;s_rnr_retry == 0 &amp;&amp;</div><div class='del'>-		    !((rdi-&gt;post_parms[wqe-&gt;wr.opcode].flags &amp;</div><div class='del'>-		      RVT_OPERATION_IGN_RNR_CNT) &amp;&amp;</div><div class='del'>-		      qp-&gt;s_rnr_retry_cnt == 0)) {</div><div class='del'>-			status = IB_WC_RNR_RETRY_EXC_ERR;</div><div class='del'>-			goto class_b;</div><div class='add'>+		if (!(rdi-&gt;post_parms[wqe-&gt;wr.opcode].flags &amp;</div><div class='add'>+		       RVT_OPERATION_IGN_RNR_CNT)) {</div><div class='add'>+			if (qp-&gt;s_rnr_retry == 0) {</div><div class='add'>+				status = IB_WC_RNR_RETRY_EXC_ERR;</div><div class='add'>+				goto class_b;</div><div class='add'>+			}</div><div class='add'>+			if (qp-&gt;s_rnr_retry_cnt &lt; 7 &amp;&amp; qp-&gt;s_rnr_retry_cnt &gt; 0)</div><div class='add'>+				qp-&gt;s_rnr_retry--;</div><div class='ctx'> 		}</div><div class='del'>-		if (qp-&gt;s_rnr_retry_cnt &lt; 7 &amp;&amp; qp-&gt;s_rnr_retry_cnt &gt; 0)</div><div class='del'>-			qp-&gt;s_rnr_retry--;</div><div class='ctx'> </div><div class='ctx'> 		/*</div><div class='ctx'> 		 * The last valid PSN is the previous PSN. For TID RDMA WRITE</div><div class='hunk'>@@ -2552,6 +2499,7 @@ static inline void rc_cancel_ack(struct rvt_qp *qp)</div><div class='ctx'>  * @opcode: the opcode for this packet</div><div class='ctx'>  * @psn: the packet sequence number for this packet</div><div class='ctx'>  * @diff: the difference between the PSN and the expected PSN</div><div class='add'>+ * @rcd: the receive context</div><div class='ctx'>  *</div><div class='ctx'>  * This is called from hfi1_rc_rcv() to process an unexpected</div><div class='ctx'>  * incoming RC packet for the given QP.</div><div class='hunk'>@@ -2609,7 +2557,7 @@ static noinline int rc_rcv_error(struct ib_other_headers *ohdr, void *data,</div><div class='ctx'> 	 * to be sent before sending this one.</div><div class='ctx'> 	 */</div><div class='ctx'> 	e = NULL;</div><div class='del'>-	old_req = 1;</div><div class='add'>+	old_req = true;</div><div class='ctx'> 	ibp-&gt;rvp.n_rc_dupreq++;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;qp-&gt;s_lock, flags);</div><div class='hunk'>@@ -2911,7 +2859,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet)</div><div class='ctx'> 		if (!ret)</div><div class='ctx'> 			goto rnr_nak;</div><div class='ctx'> 		qp-&gt;r_rcv_len = 0;</div><div class='del'>-		/* FALLTHROUGH */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case OP(SEND_MIDDLE):</div><div class='ctx'> 	case OP(RDMA_WRITE_MIDDLE):</div><div class='ctx'> send_middle:</div><div class='hunk'>@@ -2951,7 +2899,7 @@ send_middle:</div><div class='ctx'> 			goto no_immediate_data;</div><div class='ctx'> 		if (opcode == OP(SEND_ONLY_WITH_INVALIDATE))</div><div class='ctx'> 			goto send_last_inv;</div><div class='del'>-		/* FALLTHROUGH -- for SEND_ONLY_WITH_IMMEDIATE */</div><div class='add'>+		fallthrough;	/* for SEND_ONLY_WITH_IMMEDIATE */</div><div class='ctx'> 	case OP(SEND_LAST_WITH_IMMEDIATE):</div><div class='ctx'> send_last_imm:</div><div class='ctx'> 		wc.ex.imm_data = ohdr-&gt;u.imm_data;</div><div class='hunk'>@@ -2967,7 +2915,7 @@ send_last_inv:</div><div class='ctx'> 		goto send_last;</div><div class='ctx'> 	case OP(RDMA_WRITE_LAST):</div><div class='ctx'> 		copy_last = rvt_is_user_qp(qp);</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case OP(SEND_LAST):</div><div class='ctx'> no_immediate_data:</div><div class='ctx'> 		wc.wc_flags = 0;</div><div class='hunk'>@@ -3015,13 +2963,12 @@ send_last:</div><div class='ctx'> 		wc.dlid_path_bits = 0;</div><div class='ctx'> 		wc.port_num = 0;</div><div class='ctx'> 		/* Signal completion event if the solicited bit is set. */</div><div class='del'>-		rvt_cq_enter(ibcq_to_rvtcq(qp-&gt;ibqp.recv_cq), &amp;wc,</div><div class='del'>-			     ib_bth_is_solicited(ohdr));</div><div class='add'>+		rvt_recv_cq(qp, &amp;wc, ib_bth_is_solicited(ohdr));</div><div class='ctx'> 		break;</div><div class='ctx'> </div><div class='ctx'> 	case OP(RDMA_WRITE_ONLY):</div><div class='ctx'> 		copy_last = rvt_is_user_qp(qp);</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case OP(RDMA_WRITE_FIRST):</div><div class='ctx'> 	case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE):</div><div class='ctx'> 		if (unlikely(!(qp-&gt;qp_access_flags &amp; IB_ACCESS_REMOTE_WRITE)))</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/ruc.c b/drivers/infiniband/hw/hfi1/ruc.c<br/>index 23ac6057b211..aafa4e03b179 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/ruc.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/ruc.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/ruc.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/ruc.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/spinlock.h&gt;</div><div class='hunk'>@@ -260,6 +218,7 @@ static inline void hfi1_make_ruc_bth(struct rvt_qp *qp,</div><div class='ctx'>  * @qp: the queue pair</div><div class='ctx'>  * @ohdr: a pointer to the destination header memory</div><div class='ctx'>  * @bth0: bth0 passed in from the RC/UC builder</div><div class='add'>+ * @bth1: bth1 passed in from the RC/UC builder</div><div class='ctx'>  * @bth2: bth2 passed in from the RC/UC builder</div><div class='ctx'>  * @middle: non zero implies indicates ahg "could" be used</div><div class='ctx'>  * @ps: the current packet state</div><div class='hunk'>@@ -348,6 +307,7 @@ static inline void hfi1_make_ruc_header_16B(struct rvt_qp *qp,</div><div class='ctx'>  * @qp: the queue pair</div><div class='ctx'>  * @ohdr: a pointer to the destination header memory</div><div class='ctx'>  * @bth0: bth0 passed in from the RC/UC builder</div><div class='add'>+ * @bth1: bth1 passed in from the RC/UC builder</div><div class='ctx'>  * @bth2: bth2 passed in from the RC/UC builder</div><div class='ctx'>  * @middle: non zero implies indicates ahg "could" be used</div><div class='ctx'>  * @ps: the current packet state</div><div class='hunk'>@@ -455,11 +415,10 @@ void hfi1_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr,</div><div class='ctx'> /**</div><div class='ctx'>  * hfi1_schedule_send_yield - test for a yield required for QP</div><div class='ctx'>  * send engine</div><div class='del'>- * @timeout: Final time for timeout slice for jiffies</div><div class='ctx'>  * @qp: a pointer to QP</div><div class='ctx'>  * @ps: a pointer to a structure with commonly lookup values for</div><div class='del'>- *      the the send engine progress</div><div class='del'>- * @tid - true if it is the tid leg</div><div class='add'>+ *      the send engine progress</div><div class='add'>+ * @tid: true if it is the tid leg</div><div class='ctx'>  *</div><div class='ctx'>  * This routine checks if the time slice for the QP has expired</div><div class='ctx'>  * for RC QPs, if so an additional work entry is queued. At this</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c<br/>index b0110728f541..5cfa4f8fbf3d 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/sdma.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/sdma.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/sdma.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/sdma.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/spinlock.h&gt;</div><div class='hunk'>@@ -65,6 +23,7 @@</div><div class='ctx'> #define SDMA_DESCQ_CNT 2048</div><div class='ctx'> #define SDMA_DESC_INTR 64</div><div class='ctx'> #define INVALID_TAIL 0xffff</div><div class='add'>+#define SDMA_PAD max_t(size_t, MAX_16B_PADDING, sizeof(u32))</div><div class='ctx'> </div><div class='ctx'> static uint sdma_descq_cnt = SDMA_DESCQ_CNT;</div><div class='ctx'> module_param(sdma_descq_cnt, uint, S_IRUGO);</div><div class='hunk'>@@ -231,11 +190,11 @@ static const struct sdma_set_state_action sdma_action_table[] = {</div><div class='ctx'> static void sdma_complete(struct kref *);</div><div class='ctx'> static void sdma_finalput(struct sdma_state *);</div><div class='ctx'> static void sdma_get(struct sdma_state *);</div><div class='del'>-static void sdma_hw_clean_up_task(unsigned long);</div><div class='add'>+static void sdma_hw_clean_up_task(struct tasklet_struct *);</div><div class='ctx'> static void sdma_put(struct sdma_state *);</div><div class='ctx'> static void sdma_set_state(struct sdma_engine *, enum sdma_states);</div><div class='ctx'> static void sdma_start_hw_clean_up(struct sdma_engine *);</div><div class='del'>-static void sdma_sw_clean_up_task(unsigned long);</div><div class='add'>+static void sdma_sw_clean_up_task(struct tasklet_struct *);</div><div class='ctx'> static void sdma_sendctrl(struct sdma_engine *, unsigned);</div><div class='ctx'> static void init_sdma_regs(struct sdma_engine *, u32, uint);</div><div class='ctx'> static void sdma_process_event(</div><div class='hunk'>@@ -405,19 +364,33 @@ static void sdma_flush(struct sdma_engine *sde)</div><div class='ctx'> 	struct sdma_txreq *txp, *txp_next;</div><div class='ctx'> 	LIST_HEAD(flushlist);</div><div class='ctx'> 	unsigned long flags;</div><div class='add'>+	uint seq;</div><div class='ctx'> </div><div class='ctx'> 	/* flush from head to tail */</div><div class='ctx'> 	sdma_flush_descq(sde);</div><div class='ctx'> 	spin_lock_irqsave(&amp;sde-&gt;flushlist_lock, flags);</div><div class='ctx'> 	/* copy flush list */</div><div class='del'>-	list_for_each_entry_safe(txp, txp_next, &amp;sde-&gt;flushlist, list) {</div><div class='del'>-		list_del_init(&amp;txp-&gt;list);</div><div class='del'>-		list_add_tail(&amp;txp-&gt;list, &amp;flushlist);</div><div class='del'>-	}</div><div class='add'>+	list_splice_init(&amp;sde-&gt;flushlist, &amp;flushlist);</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;sde-&gt;flushlist_lock, flags);</div><div class='ctx'> 	/* flush from flush list */</div><div class='ctx'> 	list_for_each_entry_safe(txp, txp_next, &amp;flushlist, list)</div><div class='ctx'> 		complete_tx(sde, txp, SDMA_TXREQ_S_ABORTED);</div><div class='add'>+	/* wakeup QPs orphaned on the dmawait list */</div><div class='add'>+	do {</div><div class='add'>+		struct iowait *w, *nw;</div><div class='add'>+</div><div class='add'>+		seq = read_seqbegin(&amp;sde-&gt;waitlock);</div><div class='add'>+		if (!list_empty(&amp;sde-&gt;dmawait)) {</div><div class='add'>+			write_seqlock(&amp;sde-&gt;waitlock);</div><div class='add'>+			list_for_each_entry_safe(w, nw, &amp;sde-&gt;dmawait, list) {</div><div class='add'>+				if (w-&gt;wakeup) {</div><div class='add'>+					w-&gt;wakeup(w, SDMA_AVAIL_REASON);</div><div class='add'>+					list_del_init(&amp;w-&gt;list);</div><div class='add'>+				}</div><div class='add'>+			}</div><div class='add'>+			write_sequnlock(&amp;sde-&gt;waitlock);</div><div class='add'>+		}</div><div class='add'>+	} while (read_seqretry(&amp;sde-&gt;waitlock, seq));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -494,7 +467,8 @@ static void sdma_err_progress_check_schedule(struct sdma_engine *sde)</div><div class='ctx'> static void sdma_err_progress_check(struct timer_list *t)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned index;</div><div class='del'>-	struct sdma_engine *sde = from_timer(sde, t, err_progress_check_timer);</div><div class='add'>+	struct sdma_engine *sde = timer_container_of(sde, t,</div><div class='add'>+						     err_progress_check_timer);</div><div class='ctx'> </div><div class='ctx'> 	dd_dev_err(sde-&gt;dd, "SDE progress check event\n");</div><div class='ctx'> 	for (index = 0; index &lt; sde-&gt;dd-&gt;num_sdma; index++) {</div><div class='hunk'>@@ -530,9 +504,10 @@ static void sdma_err_progress_check(struct timer_list *t)</div><div class='ctx'> 	schedule_work(&amp;sde-&gt;err_halt_worker);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void sdma_hw_clean_up_task(unsigned long opaque)</div><div class='add'>+static void sdma_hw_clean_up_task(struct tasklet_struct *t)</div><div class='ctx'> {</div><div class='del'>-	struct sdma_engine *sde = (struct sdma_engine *)opaque;</div><div class='add'>+	struct sdma_engine *sde = from_tasklet(sde, t,</div><div class='add'>+					       sdma_hw_clean_up_task);</div><div class='ctx'> 	u64 statuscsr;</div><div class='ctx'> </div><div class='ctx'> 	while (1) {</div><div class='hunk'>@@ -589,9 +564,9 @@ static void sdma_flush_descq(struct sdma_engine *sde)</div><div class='ctx'> 		sdma_desc_avail(sde, sdma_descq_freecnt(sde));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void sdma_sw_clean_up_task(unsigned long opaque)</div><div class='add'>+static void sdma_sw_clean_up_task(struct tasklet_struct *t)</div><div class='ctx'> {</div><div class='del'>-	struct sdma_engine *sde = (struct sdma_engine *)opaque;</div><div class='add'>+	struct sdma_engine *sde = from_tasklet(sde, t, sdma_sw_clean_up_task);</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;sde-&gt;tail_lock, flags);</div><div class='hunk'>@@ -818,7 +793,7 @@ struct sdma_engine *sdma_select_engine_sc(</div><div class='ctx'> struct sdma_rht_map_elem {</div><div class='ctx'> 	u32 mask;</div><div class='ctx'> 	u8 ctr;</div><div class='del'>-	struct sdma_engine *sde[0];</div><div class='add'>+	struct sdma_engine *sde[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct sdma_rht_node {</div><div class='hunk'>@@ -833,7 +808,7 @@ static const struct rhashtable_params sdma_rht_params = {</div><div class='ctx'> 	.nelem_hint = NR_CPUS_HINT,</div><div class='ctx'> 	.head_offset = offsetof(struct sdma_rht_node, node),</div><div class='ctx'> 	.key_offset = offsetof(struct sdma_rht_node, cpu_id),</div><div class='del'>-	.key_len = FIELD_SIZEOF(struct sdma_rht_node, cpu_id),</div><div class='add'>+	.key_len = sizeof_field(struct sdma_rht_node, cpu_id),</div><div class='ctx'> 	.max_size = NR_CPUS,</div><div class='ctx'> 	.min_size = 8,</div><div class='ctx'> 	.automatic_shrinking = true,</div><div class='hunk'>@@ -855,20 +830,19 @@ struct sdma_engine *sdma_select_user_engine(struct hfi1_devdata *dd,</div><div class='ctx'> {</div><div class='ctx'> 	struct sdma_rht_node *rht_node;</div><div class='ctx'> 	struct sdma_engine *sde = NULL;</div><div class='del'>-	const struct cpumask *current_mask = &amp;current-&gt;cpus_allowed;</div><div class='ctx'> 	unsigned long cpu_id;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * To ensure that always the same sdma engine(s) will be</div><div class='ctx'> 	 * selected make sure the process is pinned to this CPU only.</div><div class='ctx'> 	 */</div><div class='del'>-	if (cpumask_weight(current_mask) != 1)</div><div class='add'>+	if (current-&gt;nr_cpus_allowed != 1)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	cpu_id = smp_processor_id();</div><div class='ctx'> 	rcu_read_lock();</div><div class='del'>-	rht_node = rhashtable_lookup_fast(dd-&gt;sdma_rht, &amp;cpu_id,</div><div class='del'>-					  sdma_rht_params);</div><div class='add'>+	cpu_id = smp_processor_id();</div><div class='add'>+	rht_node = rhashtable_lookup(dd-&gt;sdma_rht, &amp;cpu_id,</div><div class='add'>+				     sdma_rht_params);</div><div class='ctx'> </div><div class='ctx'> 	if (rht_node &amp;&amp; rht_node-&gt;map[vl]) {</div><div class='ctx'> 		struct sdma_rht_map_elem *map = rht_node-&gt;map[vl];</div><div class='hunk'>@@ -1016,7 +990,7 @@ ssize_t sdma_set_cpu_to_sde_map(struct sdma_engine *sde, const char *buf,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Clean up old mappings */</div><div class='del'>-	for_each_cpu(cpu, cpu_online_mask) {</div><div class='add'>+	for_each_online_cpu(cpu) {</div><div class='ctx'> 		struct sdma_rht_node *rht_node;</div><div class='ctx'> </div><div class='ctx'> 		/* Don't cleanup sdes that are set in the new mask */</div><div class='hunk'>@@ -1270,7 +1244,7 @@ bail:</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * sdma_clean()  Clean up allocated memory</div><div class='add'>+ * sdma_clean - Clean up allocated memory</div><div class='ctx'>  * @dd:          struct hfi1_devdata</div><div class='ctx'>  * @num_engines: num sdma engines</div><div class='ctx'>  *</div><div class='hunk'>@@ -1283,7 +1257,7 @@ void sdma_clean(struct hfi1_devdata *dd, size_t num_engines)</div><div class='ctx'> 	struct sdma_engine *sde;</div><div class='ctx'> </div><div class='ctx'> 	if (dd-&gt;sdma_pad_dma) {</div><div class='del'>-		dma_free_coherent(&amp;dd-&gt;pcidev-&gt;dev, 4,</div><div class='add'>+		dma_free_coherent(&amp;dd-&gt;pcidev-&gt;dev, SDMA_PAD,</div><div class='ctx'> 				  (void *)dd-&gt;sdma_pad_dma,</div><div class='ctx'> 				  dd-&gt;sdma_pad_phys);</div><div class='ctx'> 		dd-&gt;sdma_pad_dma = NULL;</div><div class='hunk'>@@ -1315,11 +1289,13 @@ void sdma_clean(struct hfi1_devdata *dd, size_t num_engines)</div><div class='ctx'> 		kvfree(sde-&gt;tx_ring);</div><div class='ctx'> 		sde-&gt;tx_ring = NULL;</div><div class='ctx'> 	}</div><div class='del'>-	spin_lock_irq(&amp;dd-&gt;sde_map_lock);</div><div class='del'>-	sdma_map_free(rcu_access_pointer(dd-&gt;sdma_map));</div><div class='del'>-	RCU_INIT_POINTER(dd-&gt;sdma_map, NULL);</div><div class='del'>-	spin_unlock_irq(&amp;dd-&gt;sde_map_lock);</div><div class='del'>-	synchronize_rcu();</div><div class='add'>+	if (rcu_access_pointer(dd-&gt;sdma_map)) {</div><div class='add'>+		spin_lock_irq(&amp;dd-&gt;sde_map_lock);</div><div class='add'>+		sdma_map_free(rcu_access_pointer(dd-&gt;sdma_map));</div><div class='add'>+		RCU_INIT_POINTER(dd-&gt;sdma_map, NULL);</div><div class='add'>+		spin_unlock_irq(&amp;dd-&gt;sde_map_lock);</div><div class='add'>+		synchronize_rcu();</div><div class='add'>+	}</div><div class='ctx'> 	kfree(dd-&gt;per_sdma);</div><div class='ctx'> 	dd-&gt;per_sdma = NULL;</div><div class='ctx'> </div><div class='hunk'>@@ -1440,11 +1416,10 @@ int sdma_init(struct hfi1_devdata *dd, u8 port)</div><div class='ctx'> 		sde-&gt;tail_csr =</div><div class='ctx'> 			get_kctxt_csr_addr(dd, this_idx, SD(TAIL));</div><div class='ctx'> </div><div class='del'>-		tasklet_init(&amp;sde-&gt;sdma_hw_clean_up_task, sdma_hw_clean_up_task,</div><div class='del'>-			     (unsigned long)sde);</div><div class='del'>-</div><div class='del'>-		tasklet_init(&amp;sde-&gt;sdma_sw_clean_up_task, sdma_sw_clean_up_task,</div><div class='del'>-			     (unsigned long)sde);</div><div class='add'>+		tasklet_setup(&amp;sde-&gt;sdma_hw_clean_up_task,</div><div class='add'>+			      sdma_hw_clean_up_task);</div><div class='add'>+		tasklet_setup(&amp;sde-&gt;sdma_sw_clean_up_task,</div><div class='add'>+			      sdma_sw_clean_up_task);</div><div class='ctx'> 		INIT_WORK(&amp;sde-&gt;err_halt_worker, sdma_err_halt_wait);</div><div class='ctx'> 		INIT_WORK(&amp;sde-&gt;flush_worker, sdma_field_flush);</div><div class='ctx'> </div><div class='hunk'>@@ -1478,7 +1453,7 @@ int sdma_init(struct hfi1_devdata *dd, u8 port)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Allocate memory for pad */</div><div class='del'>-	dd-&gt;sdma_pad_dma = dma_alloc_coherent(&amp;dd-&gt;pcidev-&gt;dev, sizeof(u32),</div><div class='add'>+	dd-&gt;sdma_pad_dma = dma_alloc_coherent(&amp;dd-&gt;pcidev-&gt;dev, SDMA_PAD,</div><div class='ctx'> 					      &amp;dd-&gt;sdma_pad_phys, GFP_KERNEL);</div><div class='ctx'> 	if (!dd-&gt;sdma_pad_dma) {</div><div class='ctx'> 		dd_dev_err(dd, "failed to allocate SendDMA pad memory\n");</div><div class='hunk'>@@ -1513,8 +1488,11 @@ int sdma_init(struct hfi1_devdata *dd, u8 port)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ret = rhashtable_init(tmp_sdma_rht, &amp;sdma_rht_params);</div><div class='del'>-	if (ret &lt; 0)</div><div class='add'>+	if (ret &lt; 0) {</div><div class='add'>+		kfree(tmp_sdma_rht);</div><div class='ctx'> 		goto bail;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	dd-&gt;sdma_rht = tmp_sdma_rht;</div><div class='ctx'> </div><div class='ctx'> 	dd_dev_info(dd, "SDMA num_sdma: %u\n", dd-&gt;num_sdma);</div><div class='hunk'>@@ -1544,24 +1522,6 @@ void sdma_all_running(struct hfi1_devdata *dd)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * sdma_all_idle() - called when the link goes down</div><div class='del'>- * @dd: hfi1_devdata</div><div class='del'>- *</div><div class='del'>- * This routine moves all engines to the idle state.</div><div class='del'>- */</div><div class='del'>-void sdma_all_idle(struct hfi1_devdata *dd)</div><div class='del'>-{</div><div class='del'>-	struct sdma_engine *sde;</div><div class='del'>-	unsigned int i;</div><div class='del'>-</div><div class='del'>-	/* idle all engines */</div><div class='del'>-	for (i = 0; i &lt; dd-&gt;num_sdma; ++i) {</div><div class='del'>-		sde = &amp;dd-&gt;per_sdma[i];</div><div class='del'>-		sdma_process_event(sde, sdma_event_e70_go_idle);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='ctx'>  * sdma_start() - called to kick off state processing for all engines</div><div class='ctx'>  * @dd: hfi1_devdata</div><div class='ctx'>  *</div><div class='hunk'>@@ -1598,7 +1558,7 @@ void sdma_exit(struct hfi1_devdata *dd)</div><div class='ctx'> 				   sde-&gt;this_idx);</div><div class='ctx'> 		sdma_process_event(sde, sdma_event_e00_go_hw_down);</div><div class='ctx'> </div><div class='del'>-		del_timer_sync(&amp;sde-&gt;err_progress_check_timer);</div><div class='add'>+		timer_delete_sync(&amp;sde-&gt;err_progress_check_timer);</div><div class='ctx'> </div><div class='ctx'> 		/*</div><div class='ctx'> 		 * This waits for the state machine to exit so it is not</div><div class='hunk'>@@ -1618,20 +1578,18 @@ static inline void sdma_unmap_desc(</div><div class='ctx'> {</div><div class='ctx'> 	switch (sdma_mapping_type(descp)) {</div><div class='ctx'> 	case SDMA_MAP_SINGLE:</div><div class='del'>-		dma_unmap_single(</div><div class='del'>-			&amp;dd-&gt;pcidev-&gt;dev,</div><div class='del'>-			sdma_mapping_addr(descp),</div><div class='del'>-			sdma_mapping_len(descp),</div><div class='del'>-			DMA_TO_DEVICE);</div><div class='add'>+		dma_unmap_single(&amp;dd-&gt;pcidev-&gt;dev, sdma_mapping_addr(descp),</div><div class='add'>+				 sdma_mapping_len(descp), DMA_TO_DEVICE);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case SDMA_MAP_PAGE:</div><div class='del'>-		dma_unmap_page(</div><div class='del'>-			&amp;dd-&gt;pcidev-&gt;dev,</div><div class='del'>-			sdma_mapping_addr(descp),</div><div class='del'>-			sdma_mapping_len(descp),</div><div class='del'>-			DMA_TO_DEVICE);</div><div class='add'>+		dma_unmap_page(&amp;dd-&gt;pcidev-&gt;dev, sdma_mapping_addr(descp),</div><div class='add'>+			       sdma_mapping_len(descp), DMA_TO_DEVICE);</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	if (descp-&gt;pinning_ctx &amp;&amp; descp-&gt;ctx_put)</div><div class='add'>+		descp-&gt;ctx_put(descp-&gt;pinning_ctx);</div><div class='add'>+	descp-&gt;pinning_ctx = NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -1723,7 +1681,7 @@ retry:</div><div class='ctx'> 			sane = (hwhead == swhead);</div><div class='ctx'> </div><div class='ctx'> 		if (unlikely(!sane)) {</div><div class='del'>-			dd_dev_err(dd, "SDMA(%u) bad head (%s) hwhd=%hu swhd=%hu swtl=%hu cnt=%hu\n",</div><div class='add'>+			dd_dev_err(dd, "SDMA(%u) bad head (%s) hwhd=%u swhd=%u swtl=%u cnt=%u\n",</div><div class='ctx'> 				   sde-&gt;this_idx,</div><div class='ctx'> 				   use_dmahead ? "dma" : "kreg",</div><div class='ctx'> 				   hwhead, swhead, swtail, cnt);</div><div class='hunk'>@@ -1843,7 +1801,7 @@ retry:</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * The SDMA idle interrupt is not guaranteed to be ordered with respect</div><div class='del'>-	 * to updates to the the dma_head location in host memory. The head</div><div class='add'>+	 * to updates to the dma_head location in host memory. The head</div><div class='ctx'> 	 * value read might not be fully up to date. If there are pending</div><div class='ctx'> 	 * descriptors and the SDMA idle interrupt fired then read from the</div><div class='ctx'> 	 * CSR SDMA head instead to get the latest value from the hardware.</div><div class='hunk'>@@ -2413,7 +2371,7 @@ unlock_noconn:</div><div class='ctx'> 	list_add_tail(&amp;tx-&gt;list, &amp;sde-&gt;flushlist);</div><div class='ctx'> 	spin_unlock(&amp;sde-&gt;flushlist_lock);</div><div class='ctx'> 	iowait_inc_wait_count(wait, tx-&gt;num_desc);</div><div class='del'>-	schedule_work(&amp;sde-&gt;flush_worker);</div><div class='add'>+	queue_work_on(sde-&gt;cpu, system_highpri_wq, &amp;sde-&gt;flush_worker);</div><div class='ctx'> 	ret = -ECOMM;</div><div class='ctx'> 	goto unlock;</div><div class='ctx'> nodesc:</div><div class='hunk'>@@ -2431,11 +2389,11 @@ nodesc:</div><div class='ctx'>  * @sde: sdma engine to use</div><div class='ctx'>  * @wait: SE wait structure to use when full (may be NULL)</div><div class='ctx'>  * @tx_list: list of sdma_txreqs to submit</div><div class='del'>- * @count: pointer to a u16 which, after return will contain the total number of</div><div class='del'>- *         sdma_txreqs removed from the tx_list. This will include sdma_txreqs</div><div class='del'>- *         whose SDMA descriptors are submitted to the ring and the sdma_txreqs</div><div class='del'>- *         which are added to SDMA engine flush list if the SDMA engine state is</div><div class='del'>- *         not running.</div><div class='add'>+ * @count_out: pointer to a u16 which, after return will contain the total number of</div><div class='add'>+ *             sdma_txreqs removed from the tx_list. This will include sdma_txreqs</div><div class='add'>+ *             whose SDMA descriptors are submitted to the ring and the sdma_txreqs</div><div class='add'>+ *             which are added to SDMA engine flush list if the SDMA engine state is</div><div class='add'>+ *             not running.</div><div class='ctx'>  *</div><div class='ctx'>  * The call submits the list into the ring.</div><div class='ctx'>  *</div><div class='hunk'>@@ -2511,7 +2469,7 @@ unlock_noconn:</div><div class='ctx'> 		iowait_inc_wait_count(wait, tx-&gt;num_desc);</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock(&amp;sde-&gt;flushlist_lock);</div><div class='del'>-	schedule_work(&amp;sde-&gt;flush_worker);</div><div class='add'>+	queue_work_on(sde-&gt;cpu, system_highpri_wq, &amp;sde-&gt;flush_worker);</div><div class='ctx'> 	ret = -ECOMM;</div><div class='ctx'> 	goto update_tail;</div><div class='ctx'> nodesc:</div><div class='hunk'>@@ -2567,7 +2525,7 @@ static void __sdma_process_event(struct sdma_engine *sde,</div><div class='ctx'> 			 * 7220, e.g.</div><div class='ctx'> 			 */</div><div class='ctx'> 			ss-&gt;go_s99_running = 1;</div><div class='del'>-			/* fall through -- and start dma engine */</div><div class='add'>+			fallthrough;	/* and start dma engine */</div><div class='ctx'> 		case sdma_event_e10_go_hw_start:</div><div class='ctx'> 			/* This reference means the state machine is started */</div><div class='ctx'> 			sdma_get(&amp;sde-&gt;state);</div><div class='hunk'>@@ -2709,7 +2667,6 @@ static void __sdma_process_event(struct sdma_engine *sde,</div><div class='ctx'> 		case sdma_event_e70_go_idle:</div><div class='ctx'> 			break;</div><div class='ctx'> 		case sdma_event_e85_link_down:</div><div class='del'>-			/* fall through */</div><div class='ctx'> 		case sdma_event_e80_hw_freeze:</div><div class='ctx'> 			sdma_set_state(sde, sdma_state_s80_hw_freeze);</div><div class='ctx'> 			atomic_dec(&amp;sde-&gt;dd-&gt;sdma_unfreeze_count);</div><div class='hunk'>@@ -2990,7 +2947,7 @@ static void __sdma_process_event(struct sdma_engine *sde,</div><div class='ctx'> 		case sdma_event_e60_hw_halted:</div><div class='ctx'> 			need_progress = 1;</div><div class='ctx'> 			sdma_err_progress_check_schedule(sde);</div><div class='del'>-			/* fall through */</div><div class='add'>+			fallthrough;</div><div class='ctx'> 		case sdma_event_e90_sw_halted:</div><div class='ctx'> 			/*</div><div class='ctx'> 			* SW initiated halt does not perform engines</div><div class='hunk'>@@ -3004,7 +2961,7 @@ static void __sdma_process_event(struct sdma_engine *sde,</div><div class='ctx'> 			break;</div><div class='ctx'> 		case sdma_event_e85_link_down:</div><div class='ctx'> 			ss-&gt;go_s99_running = 0;</div><div class='del'>-			/* fall through */</div><div class='add'>+			fallthrough;</div><div class='ctx'> 		case sdma_event_e80_hw_freeze:</div><div class='ctx'> 			sdma_set_state(sde, sdma_state_s80_hw_freeze);</div><div class='ctx'> 			atomic_dec(&amp;sde-&gt;dd-&gt;sdma_unfreeze_count);</div><div class='hunk'>@@ -3039,6 +2996,7 @@ static void __sdma_process_event(struct sdma_engine *sde,</div><div class='ctx'> static int _extend_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='add'>+	struct sdma_desc *descp;</div><div class='ctx'> </div><div class='ctx'> 	/* Handle last descriptor */</div><div class='ctx'> 	if (unlikely((tx-&gt;num_desc == (MAX_DESC - 1)))) {</div><div class='hunk'>@@ -3059,12 +3017,10 @@ static int _extend_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx)</div><div class='ctx'> 	if (unlikely(tx-&gt;num_desc == MAX_DESC))</div><div class='ctx'> 		goto enomem;</div><div class='ctx'> </div><div class='del'>-	tx-&gt;descp = kmalloc_array(</div><div class='del'>-			MAX_DESC,</div><div class='del'>-			sizeof(struct sdma_desc),</div><div class='del'>-			GFP_ATOMIC);</div><div class='del'>-	if (!tx-&gt;descp)</div><div class='add'>+	descp = kmalloc_array(MAX_DESC, sizeof(struct sdma_desc), GFP_ATOMIC);</div><div class='add'>+	if (!descp)</div><div class='ctx'> 		goto enomem;</div><div class='add'>+	tx-&gt;descp = descp;</div><div class='ctx'> </div><div class='ctx'> 	/* reserve last descriptor for coalescing */</div><div class='ctx'> 	tx-&gt;desc_limit = MAX_DESC - 1;</div><div class='hunk'>@@ -3114,7 +3070,7 @@ int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='ctx'> 		if (type == SDMA_MAP_PAGE) {</div><div class='del'>-			kvaddr = kmap(page);</div><div class='add'>+			kvaddr = kmap_local_page(page);</div><div class='ctx'> 			kvaddr += offset;</div><div class='ctx'> 		} else if (WARN_ON(!kvaddr)) {</div><div class='ctx'> 			__sdma_txclean(dd, tx);</div><div class='hunk'>@@ -3124,7 +3080,7 @@ int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,</div><div class='ctx'> 		memcpy(tx-&gt;coalesce_buf + tx-&gt;coalesce_idx, kvaddr, len);</div><div class='ctx'> 		tx-&gt;coalesce_idx += len;</div><div class='ctx'> 		if (type == SDMA_MAP_PAGE)</div><div class='del'>-			kunmap(page);</div><div class='add'>+			kunmap_local(kvaddr);</div><div class='ctx'> </div><div class='ctx'> 		/* If there is more data, return */</div><div class='ctx'> 		if (tx-&gt;tlen - tx-&gt;coalesce_idx)</div><div class='hunk'>@@ -3154,7 +3110,7 @@ int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,</div><div class='ctx'> 		/* Add descriptor for coalesce buffer */</div><div class='ctx'> 		tx-&gt;desc_limit = MAX_DESC;</div><div class='ctx'> 		return _sdma_txadd_daddr(dd, SDMA_MAP_SINGLE, tx,</div><div class='del'>-					 addr, tx-&gt;tlen);</div><div class='add'>+					 addr, tx-&gt;tlen, NULL, NULL, NULL);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return 1;</div><div class='hunk'>@@ -3185,7 +3141,6 @@ int _pad_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx)</div><div class='ctx'> {</div><div class='ctx'> 	int rval = 0;</div><div class='ctx'> </div><div class='del'>-	tx-&gt;num_desc++;</div><div class='ctx'> 	if ((unlikely(tx-&gt;num_desc == tx-&gt;desc_limit))) {</div><div class='ctx'> 		rval = _extend_sdma_tx_descs(dd, tx);</div><div class='ctx'> 		if (rval) {</div><div class='hunk'>@@ -3193,12 +3148,15 @@ int _pad_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx)</div><div class='ctx'> 			return rval;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='ctx'> 	/* finish the one just added */</div><div class='ctx'> 	make_tx_sdma_desc(</div><div class='ctx'> 		tx,</div><div class='ctx'> 		SDMA_MAP_NONE,</div><div class='ctx'> 		dd-&gt;sdma_pad_phys,</div><div class='del'>-		sizeof(u32) - (tx-&gt;packet_len &amp; (sizeof(u32) - 1)));</div><div class='add'>+		sizeof(u32) - (tx-&gt;packet_len &amp; (sizeof(u32) - 1)),</div><div class='add'>+		NULL, NULL, NULL);</div><div class='add'>+	tx-&gt;num_desc++;</div><div class='ctx'> 	_sdma_close_tx(dd, tx);</div><div class='ctx'> 	return rval;</div><div class='ctx'> }</div><div class='hunk'>@@ -3235,7 +3193,7 @@ void _sdma_txreq_ahgadd(</div><div class='ctx'> 		tx-&gt;num_desc++;</div><div class='ctx'> 		tx-&gt;descs[2].qw[0] = 0;</div><div class='ctx'> 		tx-&gt;descs[2].qw[1] = 0;</div><div class='del'>-		/* FALLTHROUGH */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case SDMA_AHG_APPLY_UPDATE2:</div><div class='ctx'> 		tx-&gt;num_desc++;</div><div class='ctx'> 		tx-&gt;descs[1].qw[0] = 0;</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/sdma.h b/drivers/infiniband/hw/hfi1/sdma.h<br/>index 1e2e40f79cb2..91dfd5d0c419 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/sdma.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/sdma.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/sdma.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/sdma.h</a></div><div class='hunk'>@@ -1,52 +1,11 @@</div><div class='del'>-#ifndef _HFI1_SDMA_H</div><div class='del'>-#define _HFI1_SDMA_H</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='add'>+#ifndef _HFI1_SDMA_H</div><div class='add'>+#define _HFI1_SDMA_H</div><div class='add'>+</div><div class='ctx'> #include &lt;linux/types.h&gt;</div><div class='ctx'> #include &lt;linux/list.h&gt;</div><div class='ctx'> #include &lt;asm/byteorder.h&gt;</div><div class='hunk'>@@ -414,7 +373,6 @@ void sdma_start(struct hfi1_devdata *dd);</div><div class='ctx'> void sdma_exit(struct hfi1_devdata *dd);</div><div class='ctx'> void sdma_clean(struct hfi1_devdata *dd, size_t num_engines);</div><div class='ctx'> void sdma_all_running(struct hfi1_devdata *dd);</div><div class='del'>-void sdma_all_idle(struct hfi1_devdata *dd);</div><div class='ctx'> void sdma_freeze_notify(struct hfi1_devdata *dd, int go_idle);</div><div class='ctx'> void sdma_freeze(struct hfi1_devdata *dd);</div><div class='ctx'> void sdma_unfreeze(struct hfi1_devdata *dd);</div><div class='hunk'>@@ -636,7 +594,10 @@ static inline void make_tx_sdma_desc(</div><div class='ctx'> 	struct sdma_txreq *tx,</div><div class='ctx'> 	int type,</div><div class='ctx'> 	dma_addr_t addr,</div><div class='del'>-	size_t len)</div><div class='add'>+	size_t len,</div><div class='add'>+	void *pinning_ctx,</div><div class='add'>+	void (*ctx_get)(void *),</div><div class='add'>+	void (*ctx_put)(void *))</div><div class='ctx'> {</div><div class='ctx'> 	struct sdma_desc *desc = &amp;tx-&gt;descp[tx-&gt;num_desc];</div><div class='ctx'> </div><div class='hunk'>@@ -653,6 +614,11 @@ static inline void make_tx_sdma_desc(</div><div class='ctx'> 				&lt;&lt; SDMA_DESC0_PHY_ADDR_SHIFT) |</div><div class='ctx'> 			(((u64)len &amp; SDMA_DESC0_BYTE_COUNT_MASK)</div><div class='ctx'> 				&lt;&lt; SDMA_DESC0_BYTE_COUNT_SHIFT);</div><div class='add'>+</div><div class='add'>+	desc-&gt;pinning_ctx = pinning_ctx;</div><div class='add'>+	desc-&gt;ctx_put = ctx_put;</div><div class='add'>+	if (pinning_ctx &amp;&amp; ctx_get)</div><div class='add'>+		ctx_get(pinning_ctx);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* helper to extend txreq */</div><div class='hunk'>@@ -672,14 +638,13 @@ static inline void sdma_txclean(struct hfi1_devdata *dd, struct sdma_txreq *tx)</div><div class='ctx'> static inline void _sdma_close_tx(struct hfi1_devdata *dd,</div><div class='ctx'> 				  struct sdma_txreq *tx)</div><div class='ctx'> {</div><div class='del'>-	tx-&gt;descp[tx-&gt;num_desc].qw[0] |=</div><div class='del'>-		SDMA_DESC0_LAST_DESC_FLAG;</div><div class='del'>-	tx-&gt;descp[tx-&gt;num_desc].qw[1] |=</div><div class='del'>-		dd-&gt;default_desc1;</div><div class='add'>+	u16 last_desc = tx-&gt;num_desc - 1;</div><div class='add'>+</div><div class='add'>+	tx-&gt;descp[last_desc].qw[0] |= SDMA_DESC0_LAST_DESC_FLAG;</div><div class='add'>+	tx-&gt;descp[last_desc].qw[1] |= dd-&gt;default_desc1;</div><div class='ctx'> 	if (tx-&gt;flags &amp; SDMA_TXREQ_F_URGENT)</div><div class='del'>-		tx-&gt;descp[tx-&gt;num_desc].qw[1] |=</div><div class='del'>-			(SDMA_DESC1_HEAD_TO_HOST_FLAG |</div><div class='del'>-			 SDMA_DESC1_INT_REQ_FLAG);</div><div class='add'>+		tx-&gt;descp[last_desc].qw[1] |= (SDMA_DESC1_HEAD_TO_HOST_FLAG |</div><div class='add'>+					       SDMA_DESC1_INT_REQ_FLAG);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static inline int _sdma_txadd_daddr(</div><div class='hunk'>@@ -687,15 +652,20 @@ static inline int _sdma_txadd_daddr(</div><div class='ctx'> 	int type,</div><div class='ctx'> 	struct sdma_txreq *tx,</div><div class='ctx'> 	dma_addr_t addr,</div><div class='del'>-	u16 len)</div><div class='add'>+	u16 len,</div><div class='add'>+	void *pinning_ctx,</div><div class='add'>+	void (*ctx_get)(void *),</div><div class='add'>+	void (*ctx_put)(void *))</div><div class='ctx'> {</div><div class='ctx'> 	int rval = 0;</div><div class='ctx'> </div><div class='ctx'> 	make_tx_sdma_desc(</div><div class='ctx'> 		tx,</div><div class='ctx'> 		type,</div><div class='del'>-		addr, len);</div><div class='add'>+		addr, len,</div><div class='add'>+		pinning_ctx, ctx_get, ctx_put);</div><div class='ctx'> 	WARN_ON(len &gt; tx-&gt;tlen);</div><div class='add'>+	tx-&gt;num_desc++;</div><div class='ctx'> 	tx-&gt;tlen -= len;</div><div class='ctx'> 	/* special cases for last */</div><div class='ctx'> 	if (!tx-&gt;tlen) {</div><div class='hunk'>@@ -707,7 +677,6 @@ static inline int _sdma_txadd_daddr(</div><div class='ctx'> 			_sdma_close_tx(dd, tx);</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='del'>-	tx-&gt;num_desc++;</div><div class='ctx'> 	return rval;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -718,6 +687,14 @@ static inline int _sdma_txadd_daddr(</div><div class='ctx'>  * @page: page to map</div><div class='ctx'>  * @offset: offset within the page</div><div class='ctx'>  * @len: length in bytes</div><div class='add'>+ * @pinning_ctx: context to be stored on struct sdma_desc .pinning_ctx. Not</div><div class='add'>+ *               added if coalesce buffer is used. E.g. pointer to pinned-page</div><div class='add'>+ *               cache entry for the sdma_desc.</div><div class='add'>+ * @ctx_get: optional function to take reference to @pinning_ctx. Not called if</div><div class='add'>+ *           @pinning_ctx is NULL.</div><div class='add'>+ * @ctx_put: optional function to release reference to @pinning_ctx after</div><div class='add'>+ *           sdma_desc completes. May be called in interrupt context so must</div><div class='add'>+ *           not sleep. Not called if @pinning_ctx is NULL.</div><div class='ctx'>  *</div><div class='ctx'>  * This is used to add a page/offset/length descriptor.</div><div class='ctx'>  *</div><div class='hunk'>@@ -732,7 +709,10 @@ static inline int sdma_txadd_page(</div><div class='ctx'> 	struct sdma_txreq *tx,</div><div class='ctx'> 	struct page *page,</div><div class='ctx'> 	unsigned long offset,</div><div class='del'>-	u16 len)</div><div class='add'>+	u16 len,</div><div class='add'>+	void *pinning_ctx,</div><div class='add'>+	void (*ctx_get)(void *),</div><div class='add'>+	void (*ctx_put)(void *))</div><div class='ctx'> {</div><div class='ctx'> 	dma_addr_t addr;</div><div class='ctx'> 	int rval;</div><div class='hunk'>@@ -756,8 +736,8 @@ static inline int sdma_txadd_page(</div><div class='ctx'> 		return -ENOSPC;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return _sdma_txadd_daddr(</div><div class='del'>-			dd, SDMA_MAP_PAGE, tx, addr, len);</div><div class='add'>+	return _sdma_txadd_daddr(dd, SDMA_MAP_PAGE, tx, addr, len,</div><div class='add'>+				 pinning_ctx, ctx_get, ctx_put);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='hunk'>@@ -791,7 +771,8 @@ static inline int sdma_txadd_daddr(</div><div class='ctx'> 			return rval;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return _sdma_txadd_daddr(dd, SDMA_MAP_NONE, tx, addr, len);</div><div class='add'>+	return _sdma_txadd_daddr(dd, SDMA_MAP_NONE, tx, addr, len,</div><div class='add'>+				 NULL, NULL, NULL);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='hunk'>@@ -837,8 +818,8 @@ static inline int sdma_txadd_kvaddr(</div><div class='ctx'> 		return -ENOSPC;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return _sdma_txadd_daddr(</div><div class='del'>-			dd, SDMA_MAP_SINGLE, tx, addr, len);</div><div class='add'>+	return _sdma_txadd_daddr(dd, SDMA_MAP_SINGLE, tx, addr, len,</div><div class='add'>+				 NULL, NULL, NULL);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct iowait_work;</div><div class='hunk'>@@ -907,24 +888,6 @@ static inline unsigned sdma_progress(struct sdma_engine *sde, unsigned seq,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/**</div><div class='del'>- * sdma_iowait_schedule() - initialize wait structure</div><div class='del'>- * @sde: sdma_engine to schedule</div><div class='del'>- * @wait: wait struct to schedule</div><div class='del'>- *</div><div class='del'>- * This function initializes the iowait</div><div class='del'>- * structure embedded in the QP or PQ.</div><div class='del'>- *</div><div class='del'>- */</div><div class='del'>-static inline void sdma_iowait_schedule(</div><div class='del'>-	struct sdma_engine *sde,</div><div class='del'>-	struct iowait *wait)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_pportdata *ppd = sde-&gt;dd-&gt;pport;</div><div class='del'>-</div><div class='del'>-	iowait_schedule(wait, ppd-&gt;hfi1_wq, sde-&gt;cpu);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> /* for use by interrupt handling */</div><div class='ctx'> void sdma_engine_error(struct sdma_engine *sde, u64 status);</div><div class='ctx'> void sdma_engine_interrupt(struct sdma_engine *sde, u64 status);</div><div class='hunk'>@@ -1002,7 +965,7 @@ void sdma_engine_interrupt(struct sdma_engine *sde, u64 status);</div><div class='ctx'>  */</div><div class='ctx'> struct sdma_map_elem {</div><div class='ctx'> 	u32 mask;</div><div class='del'>-	struct sdma_engine *sde[0];</div><div class='add'>+	struct sdma_engine *sde[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='hunk'>@@ -1024,7 +987,7 @@ struct sdma_vl_map {</div><div class='ctx'> 	u32 mask;</div><div class='ctx'> 	u8 actual_vls;</div><div class='ctx'> 	u8 vls;</div><div class='del'>-	struct sdma_map_elem *map[0];</div><div class='add'>+	struct sdma_map_elem *map[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> int sdma_map_init(</div><div class='hunk'>@@ -1089,5 +1052,4 @@ u16 sdma_get_descq_cnt(void);</div><div class='ctx'> extern uint mod_num_sdma;</div><div class='ctx'> </div><div class='ctx'> void sdma_update_lmc(struct hfi1_devdata *dd, u64 mask, u32 lid);</div><div class='del'>-</div><div class='ctx'> #endif</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/sdma_txreq.h b/drivers/infiniband/hw/hfi1/sdma_txreq.h<br/>index 514a4784566b..5782166d984c 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/sdma_txreq.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/sdma_txreq.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/sdma_txreq.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/sdma_txreq.h</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2016 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #ifndef HFI1_SDMA_TXREQ_H</div><div class='hunk'>@@ -61,6 +19,9 @@</div><div class='ctx'> struct sdma_desc {</div><div class='ctx'> 	/* private:  don't use directly */</div><div class='ctx'> 	u64 qw[2];</div><div class='add'>+	void *pinning_ctx;</div><div class='add'>+	/* Release reference to @pinning_ctx. May be called in interrupt context. Must not sleep. */</div><div class='add'>+	void (*ctx_put)(void *ctx);</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/sysfs.c b/drivers/infiniband/hw/hfi1/sysfs.c<br/>index 90f62c4bddba..372cfd13dc61 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/sysfs.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/sysfs.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/sysfs.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/sysfs.c</a></div><div class='hunk'>@@ -1,55 +1,24 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015-2017 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #include &lt;linux/ctype.h&gt;</div><div class='add'>+#include &lt;rdma/ib_sysfs.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "hfi.h"</div><div class='ctx'> #include "mad.h"</div><div class='ctx'> #include "trace.h"</div><div class='ctx'> </div><div class='add'>+static struct hfi1_pportdata *hfi1_get_pportdata_kobj(struct kobject *kobj)</div><div class='add'>+{</div><div class='add'>+	u32 port_num;</div><div class='add'>+	struct ib_device *ibdev = ib_port_sysfs_get_ibdev_kobj(kobj, &amp;port_num);</div><div class='add'>+	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='add'>+</div><div class='add'>+	return &amp;dd-&gt;pport[port_num - 1];</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /*</div><div class='ctx'>  * Start of per-port congestion control structures and support code</div><div class='ctx'>  */</div><div class='hunk'>@@ -57,13 +26,12 @@</div><div class='ctx'> /*</div><div class='ctx'>  * Congestion control table size followed by table entries</div><div class='ctx'>  */</div><div class='del'>-static ssize_t read_cc_table_bin(struct file *filp, struct kobject *kobj,</div><div class='del'>-				 struct bin_attribute *bin_attr,</div><div class='add'>+static ssize_t cc_table_bin_read(struct file *filp, struct kobject *kobj,</div><div class='add'>+				 const struct bin_attribute *bin_attr,</div><div class='ctx'> 				 char *buf, loff_t pos, size_t count)</div><div class='ctx'> {</div><div class='ctx'> 	int ret;</div><div class='del'>-	struct hfi1_pportdata *ppd =</div><div class='del'>-		container_of(kobj, struct hfi1_pportdata, pport_cc_kobj);</div><div class='add'>+	struct hfi1_pportdata *ppd = hfi1_get_pportdata_kobj(kobj);</div><div class='ctx'> 	struct cc_state *cc_state;</div><div class='ctx'> </div><div class='ctx'> 	ret = ppd-&gt;total_cct_entry * sizeof(struct ib_cc_table_entry_shadow)</div><div class='hunk'>@@ -89,30 +57,19 @@ static ssize_t read_cc_table_bin(struct file *filp, struct kobject *kobj,</div><div class='ctx'> </div><div class='ctx'> 	return count;</div><div class='ctx'> }</div><div class='del'>-</div><div class='del'>-static void port_release(struct kobject *kobj)</div><div class='del'>-{</div><div class='del'>-	/* nothing to do since memory is freed by hfi1_free_devdata() */</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static const struct bin_attribute cc_table_bin_attr = {</div><div class='del'>-	.attr = {.name = "cc_table_bin", .mode = 0444},</div><div class='del'>-	.read = read_cc_table_bin,</div><div class='del'>-	.size = PAGE_SIZE,</div><div class='del'>-};</div><div class='add'>+static const BIN_ATTR_RO(cc_table_bin, PAGE_SIZE);</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * Congestion settings: port control, control map and an array of 16</div><div class='ctx'>  * entries for the congestion entries - increase, timer, event log</div><div class='ctx'>  * trigger threshold and the minimum injection rate delay.</div><div class='ctx'>  */</div><div class='del'>-static ssize_t read_cc_setting_bin(struct file *filp, struct kobject *kobj,</div><div class='del'>-				   struct bin_attribute *bin_attr,</div><div class='add'>+static ssize_t cc_setting_bin_read(struct file *filp, struct kobject *kobj,</div><div class='add'>+				   const struct bin_attribute *bin_attr,</div><div class='ctx'> 				   char *buf, loff_t pos, size_t count)</div><div class='ctx'> {</div><div class='add'>+	struct hfi1_pportdata *ppd = hfi1_get_pportdata_kobj(kobj);</div><div class='ctx'> 	int ret;</div><div class='del'>-	struct hfi1_pportdata *ppd =</div><div class='del'>-		container_of(kobj, struct hfi1_pportdata, pport_cc_kobj);</div><div class='ctx'> 	struct cc_state *cc_state;</div><div class='ctx'> </div><div class='ctx'> 	ret = sizeof(struct opa_congestion_setting_attr_shadow);</div><div class='hunk'>@@ -136,27 +93,30 @@ static ssize_t read_cc_setting_bin(struct file *filp, struct kobject *kobj,</div><div class='ctx'> </div><div class='ctx'> 	return count;</div><div class='ctx'> }</div><div class='add'>+static const BIN_ATTR_RO(cc_setting_bin, PAGE_SIZE);</div><div class='ctx'> </div><div class='del'>-static const struct bin_attribute cc_setting_bin_attr = {</div><div class='del'>-	.attr = {.name = "cc_settings_bin", .mode = 0444},</div><div class='del'>-	.read = read_cc_setting_bin,</div><div class='del'>-	.size = PAGE_SIZE,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct hfi1_port_attr {</div><div class='del'>-	struct attribute attr;</div><div class='del'>-	ssize_t	(*show)(struct hfi1_pportdata *, char *);</div><div class='del'>-	ssize_t	(*store)(struct hfi1_pportdata *, const char *, size_t);</div><div class='add'>+static const struct bin_attribute *const port_cc_bin_attributes[] = {</div><div class='add'>+	&amp;bin_attr_cc_setting_bin,</div><div class='add'>+	&amp;bin_attr_cc_table_bin,</div><div class='add'>+	NULL</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static ssize_t cc_prescan_show(struct hfi1_pportdata *ppd, char *buf)</div><div class='add'>+static ssize_t cc_prescan_show(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+			       struct ib_port_attribute *attr, char *buf)</div><div class='ctx'> {</div><div class='del'>-	return sprintf(buf, "%s\n", ppd-&gt;cc_prescan ? "on" : "off");</div><div class='add'>+	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='add'>+	struct hfi1_pportdata *ppd = &amp;dd-&gt;pport[port_num - 1];</div><div class='add'>+</div><div class='add'>+	return sysfs_emit(buf, "%s\n", ppd-&gt;cc_prescan ? "on" : "off");</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static ssize_t cc_prescan_store(struct hfi1_pportdata *ppd, const char *buf,</div><div class='add'>+static ssize_t cc_prescan_store(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+				struct ib_port_attribute *attr, const char *buf,</div><div class='ctx'> 				size_t count)</div><div class='ctx'> {</div><div class='add'>+	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='add'>+	struct hfi1_pportdata *ppd = &amp;dd-&gt;pport[port_num - 1];</div><div class='add'>+</div><div class='ctx'> 	if (!memcmp(buf, "on", 2))</div><div class='ctx'> 		ppd-&gt;cc_prescan = true;</div><div class='ctx'> 	else if (!memcmp(buf, "off", 3))</div><div class='hunk'>@@ -164,60 +124,41 @@ static ssize_t cc_prescan_store(struct hfi1_pportdata *ppd, const char *buf,</div><div class='ctx'> </div><div class='ctx'> 	return count;</div><div class='ctx'> }</div><div class='add'>+static IB_PORT_ATTR_ADMIN_RW(cc_prescan);</div><div class='ctx'> </div><div class='del'>-static struct hfi1_port_attr cc_prescan_attr =</div><div class='del'>-		__ATTR(cc_prescan, 0600, cc_prescan_show, cc_prescan_store);</div><div class='del'>-</div><div class='del'>-static ssize_t cc_attr_show(struct kobject *kobj, struct attribute *attr,</div><div class='del'>-			    char *buf)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_port_attr *port_attr =</div><div class='del'>-		container_of(attr, struct hfi1_port_attr, attr);</div><div class='del'>-	struct hfi1_pportdata *ppd =</div><div class='del'>-		container_of(kobj, struct hfi1_pportdata, pport_cc_kobj);</div><div class='del'>-</div><div class='del'>-	return port_attr-&gt;show(ppd, buf);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static ssize_t cc_attr_store(struct kobject *kobj, struct attribute *attr,</div><div class='del'>-			     const char *buf, size_t count)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_port_attr *port_attr =</div><div class='del'>-		container_of(attr, struct hfi1_port_attr, attr);</div><div class='del'>-	struct hfi1_pportdata *ppd =</div><div class='del'>-		container_of(kobj, struct hfi1_pportdata, pport_cc_kobj);</div><div class='del'>-</div><div class='del'>-	return port_attr-&gt;store(ppd, buf, count);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static const struct sysfs_ops port_cc_sysfs_ops = {</div><div class='del'>-	.show = cc_attr_show,</div><div class='del'>-	.store = cc_attr_store</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct attribute *port_cc_default_attributes[] = {</div><div class='del'>-	&amp;cc_prescan_attr.attr,</div><div class='add'>+static struct attribute *port_cc_attributes[] = {</div><div class='add'>+	&amp;ib_port_attr_cc_prescan.attr,</div><div class='ctx'> 	NULL</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static struct kobj_type port_cc_ktype = {</div><div class='del'>-	.release = port_release,</div><div class='del'>-	.sysfs_ops = &amp;port_cc_sysfs_ops,</div><div class='del'>-	.default_attrs = port_cc_default_attributes</div><div class='add'>+static const struct attribute_group port_cc_group = {</div><div class='add'>+	.name = "CCMgtA",</div><div class='add'>+	.attrs = port_cc_attributes,</div><div class='add'>+	.bin_attrs = port_cc_bin_attributes,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /* Start sc2vl */</div><div class='del'>-#define HFI1_SC2VL_ATTR(N)				    \</div><div class='del'>-	static struct hfi1_sc2vl_attr hfi1_sc2vl_attr_##N = { \</div><div class='del'>-		.attr = { .name = __stringify(N), .mode = 0444 }, \</div><div class='del'>-		.sc = N \</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> struct hfi1_sc2vl_attr {</div><div class='del'>-	struct attribute attr;</div><div class='add'>+	struct ib_port_attribute attr;</div><div class='ctx'> 	int sc;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static ssize_t sc2vl_attr_show(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+			       struct ib_port_attribute *attr, char *buf)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_sc2vl_attr *sattr =</div><div class='add'>+		container_of(attr, struct hfi1_sc2vl_attr, attr);</div><div class='add'>+	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='add'>+</div><div class='add'>+	return sysfs_emit(buf, "%u\n", *((u8 *)dd-&gt;sc2vl + sattr-&gt;sc));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define HFI1_SC2VL_ATTR(N)                                                     \</div><div class='add'>+	static struct hfi1_sc2vl_attr hfi1_sc2vl_attr_##N = {                  \</div><div class='add'>+		.attr = __ATTR(N, 0444, sc2vl_attr_show, NULL),                \</div><div class='add'>+		.sc = N,                                                       \</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> HFI1_SC2VL_ATTR(0);</div><div class='ctx'> HFI1_SC2VL_ATTR(1);</div><div class='ctx'> HFI1_SC2VL_ATTR(2);</div><div class='hunk'>@@ -251,78 +192,70 @@ HFI1_SC2VL_ATTR(29);</div><div class='ctx'> HFI1_SC2VL_ATTR(30);</div><div class='ctx'> HFI1_SC2VL_ATTR(31);</div><div class='ctx'> </div><div class='del'>-static struct attribute *sc2vl_default_attributes[] = {</div><div class='del'>-	&amp;hfi1_sc2vl_attr_0.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_1.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_2.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_3.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_4.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_5.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_6.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_7.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_8.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_9.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_10.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_11.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_12.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_13.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_14.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_15.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_16.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_17.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_18.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_19.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_20.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_21.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_22.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_23.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_24.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_25.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_26.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_27.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_28.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_29.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_30.attr,</div><div class='del'>-	&amp;hfi1_sc2vl_attr_31.attr,</div><div class='add'>+static struct attribute *port_sc2vl_attributes[] = {</div><div class='add'>+	&amp;hfi1_sc2vl_attr_0.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_1.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_2.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_3.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_4.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_5.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_6.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_7.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_8.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_9.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_10.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_11.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_12.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_13.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_14.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_15.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_16.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_17.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_18.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_19.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_20.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_21.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_22.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_23.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_24.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_25.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_26.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_27.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_28.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_29.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_30.attr.attr,</div><div class='add'>+	&amp;hfi1_sc2vl_attr_31.attr.attr,</div><div class='ctx'> 	NULL</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static ssize_t sc2vl_attr_show(struct kobject *kobj, struct attribute *attr,</div><div class='del'>-			       char *buf)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_sc2vl_attr *sattr =</div><div class='del'>-		container_of(attr, struct hfi1_sc2vl_attr, attr);</div><div class='del'>-	struct hfi1_pportdata *ppd =</div><div class='del'>-		container_of(kobj, struct hfi1_pportdata, sc2vl_kobj);</div><div class='del'>-	struct hfi1_devdata *dd = ppd-&gt;dd;</div><div class='del'>-</div><div class='del'>-	return sprintf(buf, "%u\n", *((u8 *)dd-&gt;sc2vl + sattr-&gt;sc));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static const struct sysfs_ops hfi1_sc2vl_ops = {</div><div class='del'>-	.show = sc2vl_attr_show,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct kobj_type hfi1_sc2vl_ktype = {</div><div class='del'>-	.release = port_release,</div><div class='del'>-	.sysfs_ops = &amp;hfi1_sc2vl_ops,</div><div class='del'>-	.default_attrs = sc2vl_default_attributes</div><div class='add'>+static const struct attribute_group port_sc2vl_group = {</div><div class='add'>+	.name = "sc2vl",</div><div class='add'>+	.attrs = port_sc2vl_attributes,</div><div class='ctx'> };</div><div class='del'>-</div><div class='ctx'> /* End sc2vl */</div><div class='ctx'> </div><div class='ctx'> /* Start sl2sc */</div><div class='del'>-#define HFI1_SL2SC_ATTR(N)				    \</div><div class='del'>-	static struct hfi1_sl2sc_attr hfi1_sl2sc_attr_##N = {	  \</div><div class='del'>-		.attr = { .name = __stringify(N), .mode = 0444 }, \</div><div class='del'>-		.sl = N						  \</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> struct hfi1_sl2sc_attr {</div><div class='del'>-	struct attribute attr;</div><div class='add'>+	struct ib_port_attribute attr;</div><div class='ctx'> 	int sl;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static ssize_t sl2sc_attr_show(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+			       struct ib_port_attribute *attr, char *buf)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_sl2sc_attr *sattr =</div><div class='add'>+		container_of(attr, struct hfi1_sl2sc_attr, attr);</div><div class='add'>+	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='add'>+	struct hfi1_ibport *ibp = &amp;dd-&gt;pport[port_num - 1].ibport_data;</div><div class='add'>+</div><div class='add'>+	return sysfs_emit(buf, "%u\n", ibp-&gt;sl_to_sc[sattr-&gt;sl]);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define HFI1_SL2SC_ATTR(N)                                                     \</div><div class='add'>+	static struct hfi1_sl2sc_attr hfi1_sl2sc_attr_##N = {                  \</div><div class='add'>+		.attr = __ATTR(N, 0444, sl2sc_attr_show, NULL), .sl = N        \</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> HFI1_SL2SC_ATTR(0);</div><div class='ctx'> HFI1_SL2SC_ATTR(1);</div><div class='ctx'> HFI1_SL2SC_ATTR(2);</div><div class='hunk'>@@ -356,79 +289,72 @@ HFI1_SL2SC_ATTR(29);</div><div class='ctx'> HFI1_SL2SC_ATTR(30);</div><div class='ctx'> HFI1_SL2SC_ATTR(31);</div><div class='ctx'> </div><div class='del'>-static struct attribute *sl2sc_default_attributes[] = {</div><div class='del'>-	&amp;hfi1_sl2sc_attr_0.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_1.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_2.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_3.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_4.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_5.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_6.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_7.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_8.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_9.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_10.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_11.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_12.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_13.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_14.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_15.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_16.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_17.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_18.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_19.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_20.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_21.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_22.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_23.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_24.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_25.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_26.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_27.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_28.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_29.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_30.attr,</div><div class='del'>-	&amp;hfi1_sl2sc_attr_31.attr,</div><div class='add'>+static struct attribute *port_sl2sc_attributes[] = {</div><div class='add'>+	&amp;hfi1_sl2sc_attr_0.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_1.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_2.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_3.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_4.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_5.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_6.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_7.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_8.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_9.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_10.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_11.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_12.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_13.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_14.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_15.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_16.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_17.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_18.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_19.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_20.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_21.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_22.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_23.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_24.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_25.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_26.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_27.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_28.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_29.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_30.attr.attr,</div><div class='add'>+	&amp;hfi1_sl2sc_attr_31.attr.attr,</div><div class='ctx'> 	NULL</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static ssize_t sl2sc_attr_show(struct kobject *kobj, struct attribute *attr,</div><div class='del'>-			       char *buf)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_sl2sc_attr *sattr =</div><div class='del'>-		container_of(attr, struct hfi1_sl2sc_attr, attr);</div><div class='del'>-	struct hfi1_pportdata *ppd =</div><div class='del'>-		container_of(kobj, struct hfi1_pportdata, sl2sc_kobj);</div><div class='del'>-	struct hfi1_ibport *ibp = &amp;ppd-&gt;ibport_data;</div><div class='del'>-</div><div class='del'>-	return sprintf(buf, "%u\n", ibp-&gt;sl_to_sc[sattr-&gt;sl]);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static const struct sysfs_ops hfi1_sl2sc_ops = {</div><div class='del'>-	.show = sl2sc_attr_show,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct kobj_type hfi1_sl2sc_ktype = {</div><div class='del'>-	.release = port_release,</div><div class='del'>-	.sysfs_ops = &amp;hfi1_sl2sc_ops,</div><div class='del'>-	.default_attrs = sl2sc_default_attributes</div><div class='add'>+static const struct attribute_group port_sl2sc_group = {</div><div class='add'>+	.name = "sl2sc",</div><div class='add'>+	.attrs = port_sl2sc_attributes,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /* End sl2sc */</div><div class='ctx'> </div><div class='ctx'> /* Start vl2mtu */</div><div class='ctx'> </div><div class='del'>-#define HFI1_VL2MTU_ATTR(N) \</div><div class='del'>-	static struct hfi1_vl2mtu_attr hfi1_vl2mtu_attr_##N = { \</div><div class='del'>-		.attr = { .name = __stringify(N), .mode = 0444 }, \</div><div class='del'>-		.vl = N						  \</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> struct hfi1_vl2mtu_attr {</div><div class='del'>-	struct attribute attr;</div><div class='add'>+	struct ib_port_attribute attr;</div><div class='ctx'> 	int vl;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static ssize_t vl2mtu_attr_show(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+				struct ib_port_attribute *attr, char *buf)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_vl2mtu_attr *vlattr =</div><div class='add'>+		container_of(attr, struct hfi1_vl2mtu_attr, attr);</div><div class='add'>+	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='add'>+</div><div class='add'>+	return sysfs_emit(buf, "%u\n", dd-&gt;vld[vlattr-&gt;vl].mtu);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define HFI1_VL2MTU_ATTR(N)                                                    \</div><div class='add'>+	static struct hfi1_vl2mtu_attr hfi1_vl2mtu_attr_##N = {                \</div><div class='add'>+		.attr = __ATTR(N, 0444, vl2mtu_attr_show, NULL),               \</div><div class='add'>+		.vl = N,                                                       \</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> HFI1_VL2MTU_ATTR(0);</div><div class='ctx'> HFI1_VL2MTU_ATTR(1);</div><div class='ctx'> HFI1_VL2MTU_ATTR(2);</div><div class='hunk'>@@ -446,46 +372,29 @@ HFI1_VL2MTU_ATTR(13);</div><div class='ctx'> HFI1_VL2MTU_ATTR(14);</div><div class='ctx'> HFI1_VL2MTU_ATTR(15);</div><div class='ctx'> </div><div class='del'>-static struct attribute *vl2mtu_default_attributes[] = {</div><div class='del'>-	&amp;hfi1_vl2mtu_attr_0.attr,</div><div class='del'>-	&amp;hfi1_vl2mtu_attr_1.attr,</div><div class='del'>-	&amp;hfi1_vl2mtu_attr_2.attr,</div><div class='del'>-	&amp;hfi1_vl2mtu_attr_3.attr,</div><div class='del'>-	&amp;hfi1_vl2mtu_attr_4.attr,</div><div class='del'>-	&amp;hfi1_vl2mtu_attr_5.attr,</div><div class='del'>-	&amp;hfi1_vl2mtu_attr_6.attr,</div><div class='del'>-	&amp;hfi1_vl2mtu_attr_7.attr,</div><div class='del'>-	&amp;hfi1_vl2mtu_attr_8.attr,</div><div class='del'>-	&amp;hfi1_vl2mtu_attr_9.attr,</div><div class='del'>-	&amp;hfi1_vl2mtu_attr_10.attr,</div><div class='del'>-	&amp;hfi1_vl2mtu_attr_11.attr,</div><div class='del'>-	&amp;hfi1_vl2mtu_attr_12.attr,</div><div class='del'>-	&amp;hfi1_vl2mtu_attr_13.attr,</div><div class='del'>-	&amp;hfi1_vl2mtu_attr_14.attr,</div><div class='del'>-	&amp;hfi1_vl2mtu_attr_15.attr,</div><div class='add'>+static struct attribute *port_vl2mtu_attributes[] = {</div><div class='add'>+	&amp;hfi1_vl2mtu_attr_0.attr.attr,</div><div class='add'>+	&amp;hfi1_vl2mtu_attr_1.attr.attr,</div><div class='add'>+	&amp;hfi1_vl2mtu_attr_2.attr.attr,</div><div class='add'>+	&amp;hfi1_vl2mtu_attr_3.attr.attr,</div><div class='add'>+	&amp;hfi1_vl2mtu_attr_4.attr.attr,</div><div class='add'>+	&amp;hfi1_vl2mtu_attr_5.attr.attr,</div><div class='add'>+	&amp;hfi1_vl2mtu_attr_6.attr.attr,</div><div class='add'>+	&amp;hfi1_vl2mtu_attr_7.attr.attr,</div><div class='add'>+	&amp;hfi1_vl2mtu_attr_8.attr.attr,</div><div class='add'>+	&amp;hfi1_vl2mtu_attr_9.attr.attr,</div><div class='add'>+	&amp;hfi1_vl2mtu_attr_10.attr.attr,</div><div class='add'>+	&amp;hfi1_vl2mtu_attr_11.attr.attr,</div><div class='add'>+	&amp;hfi1_vl2mtu_attr_12.attr.attr,</div><div class='add'>+	&amp;hfi1_vl2mtu_attr_13.attr.attr,</div><div class='add'>+	&amp;hfi1_vl2mtu_attr_14.attr.attr,</div><div class='add'>+	&amp;hfi1_vl2mtu_attr_15.attr.attr,</div><div class='ctx'> 	NULL</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static ssize_t vl2mtu_attr_show(struct kobject *kobj, struct attribute *attr,</div><div class='del'>-				char *buf)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_vl2mtu_attr *vlattr =</div><div class='del'>-		container_of(attr, struct hfi1_vl2mtu_attr, attr);</div><div class='del'>-	struct hfi1_pportdata *ppd =</div><div class='del'>-		container_of(kobj, struct hfi1_pportdata, vl2mtu_kobj);</div><div class='del'>-	struct hfi1_devdata *dd = ppd-&gt;dd;</div><div class='del'>-</div><div class='del'>-	return sprintf(buf, "%u\n", dd-&gt;vld[vlattr-&gt;vl].mtu);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static const struct sysfs_ops hfi1_vl2mtu_ops = {</div><div class='del'>-	.show = vl2mtu_attr_show,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct kobj_type hfi1_vl2mtu_ktype = {</div><div class='del'>-	.release = port_release,</div><div class='del'>-	.sysfs_ops = &amp;hfi1_vl2mtu_ops,</div><div class='del'>-	.default_attrs = vl2mtu_default_attributes</div><div class='add'>+static const struct attribute_group port_vl2mtu_group = {</div><div class='add'>+	.name = "vl2mtu",</div><div class='add'>+	.attrs = port_vl2mtu_attributes,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /* end of per-port file structures and support code */</div><div class='hunk'>@@ -500,7 +409,7 @@ static ssize_t hw_rev_show(struct device *device, struct device_attribute *attr,</div><div class='ctx'> 	struct hfi1_ibdev *dev =</div><div class='ctx'> 		rdma_device_to_drv_device(device, struct hfi1_ibdev, rdi.ibdev);</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%x\n", dd_from_dev(dev)-&gt;minrev);</div><div class='add'>+	return sysfs_emit(buf, "%x\n", dd_from_dev(dev)-&gt;minrev);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(hw_rev);</div><div class='ctx'> </div><div class='hunk'>@@ -510,13 +419,11 @@ static ssize_t board_id_show(struct device *device,</div><div class='ctx'> 	struct hfi1_ibdev *dev =</div><div class='ctx'> 		rdma_device_to_drv_device(device, struct hfi1_ibdev, rdi.ibdev);</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_dev(dev);</div><div class='del'>-	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (!dd-&gt;boardname)</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-	else</div><div class='del'>-		ret = scnprintf(buf, PAGE_SIZE, "%s\n", dd-&gt;boardname);</div><div class='del'>-	return ret;</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	return sysfs_emit(buf, "%s\n", dd-&gt;boardname);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(board_id);</div><div class='ctx'> </div><div class='hunk'>@@ -528,7 +435,7 @@ static ssize_t boardversion_show(struct device *device,</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_dev(dev);</div><div class='ctx'> </div><div class='ctx'> 	/* The string printed here is already newline-terminated. */</div><div class='del'>-	return scnprintf(buf, PAGE_SIZE, "%s", dd-&gt;boardversion);</div><div class='add'>+	return sysfs_emit(buf, "%s", dd-&gt;boardversion);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(boardversion);</div><div class='ctx'> </div><div class='hunk'>@@ -545,9 +452,9 @@ static ssize_t nctxts_show(struct device *device,</div><div class='ctx'> 	 * and a receive context, so returning the smaller of the two counts</div><div class='ctx'> 	 * give a more accurate picture of total contexts available.</div><div class='ctx'> 	 */</div><div class='del'>-	return scnprintf(buf, PAGE_SIZE, "%u\n",</div><div class='del'>-			 min(dd-&gt;num_user_contexts,</div><div class='del'>-			     (u32)dd-&gt;sc_sizes[SC_USER].count));</div><div class='add'>+	return sysfs_emit(buf, "%u\n",</div><div class='add'>+			  min(dd-&gt;num_user_contexts,</div><div class='add'>+			      (u32)dd-&gt;sc_sizes[SC_USER].count));</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(nctxts);</div><div class='ctx'> </div><div class='hunk'>@@ -559,7 +466,7 @@ static ssize_t nfreectxts_show(struct device *device,</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_dev(dev);</div><div class='ctx'> </div><div class='ctx'> 	/* Return the number of free user ports (contexts) available. */</div><div class='del'>-	return scnprintf(buf, PAGE_SIZE, "%u\n", dd-&gt;freectxts);</div><div class='add'>+	return sysfs_emit(buf, "%u\n", dd-&gt;freectxts);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(nfreectxts);</div><div class='ctx'> </div><div class='hunk'>@@ -570,7 +477,8 @@ static ssize_t serial_show(struct device *device,</div><div class='ctx'> 		rdma_device_to_drv_device(device, struct hfi1_ibdev, rdi.ibdev);</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_dev(dev);</div><div class='ctx'> </div><div class='del'>-	return scnprintf(buf, PAGE_SIZE, "%s", dd-&gt;serial);</div><div class='add'>+	/* dd-&gt;serial is already newline terminated in chip.c */</div><div class='add'>+	return sysfs_emit(buf, "%s", dd-&gt;serial);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(serial);</div><div class='ctx'> </div><div class='hunk'>@@ -598,9 +506,8 @@ static DEVICE_ATTR_WO(chip_reset);</div><div class='ctx'>  * Convert the reported temperature from an integer (reported in</div><div class='ctx'>  * units of 0.25C) to a floating point number.</div><div class='ctx'>  */</div><div class='del'>-#define temp2str(temp, buf, size, idx)					\</div><div class='del'>-	scnprintf((buf) + (idx), (size) - (idx), "%u.%02u ",		\</div><div class='del'>-			      ((temp) &gt;&gt; 2), ((temp) &amp; 0x3) * 25)</div><div class='add'>+#define temp_d(t) ((t) &gt;&gt; 2)</div><div class='add'>+#define temp_f(t) (((t)&amp;0x3) * 25u)</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * Dump tempsense values, in decimal, to ease shell-scripts.</div><div class='hunk'>@@ -615,19 +522,17 @@ static ssize_t tempsense_show(struct device *device,</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	ret = hfi1_tempsense_rd(dd, &amp;temp);</div><div class='del'>-	if (!ret) {</div><div class='del'>-		int idx = 0;</div><div class='del'>-</div><div class='del'>-		idx += temp2str(temp.curr, buf, PAGE_SIZE, idx);</div><div class='del'>-		idx += temp2str(temp.lo_lim, buf, PAGE_SIZE, idx);</div><div class='del'>-		idx += temp2str(temp.hi_lim, buf, PAGE_SIZE, idx);</div><div class='del'>-		idx += temp2str(temp.crit_lim, buf, PAGE_SIZE, idx);</div><div class='del'>-		idx += scnprintf(buf + idx, PAGE_SIZE - idx,</div><div class='del'>-				"%u %u %u\n", temp.triggers &amp; 0x1,</div><div class='del'>-				temp.triggers &amp; 0x2, temp.triggers &amp; 0x4);</div><div class='del'>-		ret = idx;</div><div class='del'>-	}</div><div class='del'>-	return ret;</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	return sysfs_emit(buf, "%u.%02u %u.%02u %u.%02u %u.%02u %u %u %u\n",</div><div class='add'>+			  temp_d(temp.curr), temp_f(temp.curr),</div><div class='add'>+			  temp_d(temp.lo_lim), temp_f(temp.lo_lim),</div><div class='add'>+			  temp_d(temp.hi_lim), temp_f(temp.hi_lim),</div><div class='add'>+			  temp_d(temp.crit_lim), temp_f(temp.crit_lim),</div><div class='add'>+			  temp.triggers &amp; 0x1,</div><div class='add'>+			  temp.triggers &amp; 0x2,</div><div class='add'>+			  temp.triggers &amp; 0x4);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(tempsense);</div><div class='ctx'> </div><div class='hunk'>@@ -653,98 +558,13 @@ const struct attribute_group ib_hfi1_attr_group = {</div><div class='ctx'> 	.attrs = hfi1_attributes,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-int hfi1_create_port_files(struct ib_device *ibdev, u8 port_num,</div><div class='del'>-			   struct kobject *kobj)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_pportdata *ppd;</div><div class='del'>-	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if (!port_num || port_num &gt; dd-&gt;num_pports) {</div><div class='del'>-		dd_dev_err(dd,</div><div class='del'>-			   "Skipping infiniband class with invalid port %u\n",</div><div class='del'>-			   port_num);</div><div class='del'>-		return -ENODEV;</div><div class='del'>-	}</div><div class='del'>-	ppd = &amp;dd-&gt;pport[port_num - 1];</div><div class='del'>-</div><div class='del'>-	ret = kobject_init_and_add(&amp;ppd-&gt;sc2vl_kobj, &amp;hfi1_sc2vl_ktype, kobj,</div><div class='del'>-				   "sc2vl");</div><div class='del'>-	if (ret) {</div><div class='del'>-		dd_dev_err(dd,</div><div class='del'>-			   "Skipping sc2vl sysfs info, (err %d) port %u\n",</div><div class='del'>-			   ret, port_num);</div><div class='del'>-		goto bail;</div><div class='del'>-	}</div><div class='del'>-	kobject_uevent(&amp;ppd-&gt;sc2vl_kobj, KOBJ_ADD);</div><div class='del'>-</div><div class='del'>-	ret = kobject_init_and_add(&amp;ppd-&gt;sl2sc_kobj, &amp;hfi1_sl2sc_ktype, kobj,</div><div class='del'>-				   "sl2sc");</div><div class='del'>-	if (ret) {</div><div class='del'>-		dd_dev_err(dd,</div><div class='del'>-			   "Skipping sl2sc sysfs info, (err %d) port %u\n",</div><div class='del'>-			   ret, port_num);</div><div class='del'>-		goto bail_sc2vl;</div><div class='del'>-	}</div><div class='del'>-	kobject_uevent(&amp;ppd-&gt;sl2sc_kobj, KOBJ_ADD);</div><div class='del'>-</div><div class='del'>-	ret = kobject_init_and_add(&amp;ppd-&gt;vl2mtu_kobj, &amp;hfi1_vl2mtu_ktype, kobj,</div><div class='del'>-				   "vl2mtu");</div><div class='del'>-	if (ret) {</div><div class='del'>-		dd_dev_err(dd,</div><div class='del'>-			   "Skipping vl2mtu sysfs info, (err %d) port %u\n",</div><div class='del'>-			   ret, port_num);</div><div class='del'>-		goto bail_sl2sc;</div><div class='del'>-	}</div><div class='del'>-	kobject_uevent(&amp;ppd-&gt;vl2mtu_kobj, KOBJ_ADD);</div><div class='del'>-</div><div class='del'>-	ret = kobject_init_and_add(&amp;ppd-&gt;pport_cc_kobj, &amp;port_cc_ktype,</div><div class='del'>-				   kobj, "CCMgtA");</div><div class='del'>-	if (ret) {</div><div class='del'>-		dd_dev_err(dd,</div><div class='del'>-			   "Skipping Congestion Control sysfs info, (err %d) port %u\n",</div><div class='del'>-			   ret, port_num);</div><div class='del'>-		goto bail_vl2mtu;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	kobject_uevent(&amp;ppd-&gt;pport_cc_kobj, KOBJ_ADD);</div><div class='del'>-</div><div class='del'>-	ret = sysfs_create_bin_file(&amp;ppd-&gt;pport_cc_kobj, &amp;cc_setting_bin_attr);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dd_dev_err(dd,</div><div class='del'>-			   "Skipping Congestion Control setting sysfs info, (err %d) port %u\n",</div><div class='del'>-			   ret, port_num);</div><div class='del'>-		goto bail_cc;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = sysfs_create_bin_file(&amp;ppd-&gt;pport_cc_kobj, &amp;cc_table_bin_attr);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dd_dev_err(dd,</div><div class='del'>-			   "Skipping Congestion Control table sysfs info, (err %d) port %u\n",</div><div class='del'>-			   ret, port_num);</div><div class='del'>-		goto bail_cc_entry_bin;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	dd_dev_info(dd,</div><div class='del'>-		    "Congestion Control Agent enabled for port %d\n",</div><div class='del'>-		    port_num);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-bail_cc_entry_bin:</div><div class='del'>-	sysfs_remove_bin_file(&amp;ppd-&gt;pport_cc_kobj,</div><div class='del'>-			      &amp;cc_setting_bin_attr);</div><div class='del'>-bail_cc:</div><div class='del'>-	kobject_put(&amp;ppd-&gt;pport_cc_kobj);</div><div class='del'>-bail_vl2mtu:</div><div class='del'>-	kobject_put(&amp;ppd-&gt;vl2mtu_kobj);</div><div class='del'>-bail_sl2sc:</div><div class='del'>-	kobject_put(&amp;ppd-&gt;sl2sc_kobj);</div><div class='del'>-bail_sc2vl:</div><div class='del'>-	kobject_put(&amp;ppd-&gt;sc2vl_kobj);</div><div class='del'>-bail:</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='add'>+const struct attribute_group *hfi1_attr_port_groups[] = {</div><div class='add'>+	&amp;port_cc_group,</div><div class='add'>+	&amp;port_sc2vl_group,</div><div class='add'>+	&amp;port_sl2sc_group,</div><div class='add'>+	&amp;port_vl2mtu_group,</div><div class='add'>+	NULL,</div><div class='add'>+};</div><div class='ctx'> </div><div class='ctx'> struct sde_attribute {</div><div class='ctx'> 	struct attribute attr;</div><div class='hunk'>@@ -814,7 +634,7 @@ static ssize_t sde_show_vl(struct sdma_engine *sde, char *buf)</div><div class='ctx'> 	if (vl &lt; 0)</div><div class='ctx'> 		return vl;</div><div class='ctx'> </div><div class='del'>-	return snprintf(buf, PAGE_SIZE, "%d\n", vl);</div><div class='add'>+	return sysfs_emit(buf, "%d\n", vl);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static SDE_ATTR(cpu_list, S_IWUSR | S_IRUGO,</div><div class='hunk'>@@ -853,8 +673,13 @@ int hfi1_verbs_register_sysfs(struct hfi1_devdata *dd)</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> bail:</div><div class='del'>-	for (i = 0; i &lt; dd-&gt;num_sdma; i++)</div><div class='del'>-		kobject_del(&amp;dd-&gt;per_sdma[i].kobj);</div><div class='add'>+	/*</div><div class='add'>+	 * The function kobject_put() will call kobject_del() if the kobject</div><div class='add'>+	 * has been added successfully. The sysfs files created under the</div><div class='add'>+	 * kobject directory will also be removed during the process.</div><div class='add'>+	 */</div><div class='add'>+	for (; i &gt;= 0; i--)</div><div class='add'>+		kobject_put(&amp;dd-&gt;per_sdma[i].kobj);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -864,19 +689,9 @@ bail:</div><div class='ctx'>  */</div><div class='ctx'> void hfi1_verbs_unregister_sysfs(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='del'>-	struct hfi1_pportdata *ppd;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; dd-&gt;num_pports; i++) {</div><div class='del'>-		ppd = &amp;dd-&gt;pport[i];</div><div class='del'>-</div><div class='del'>-		sysfs_remove_bin_file(&amp;ppd-&gt;pport_cc_kobj,</div><div class='del'>-				      &amp;cc_setting_bin_attr);</div><div class='del'>-		sysfs_remove_bin_file(&amp;ppd-&gt;pport_cc_kobj,</div><div class='del'>-				      &amp;cc_table_bin_attr);</div><div class='del'>-		kobject_put(&amp;ppd-&gt;pport_cc_kobj);</div><div class='del'>-		kobject_put(&amp;ppd-&gt;vl2mtu_kobj);</div><div class='del'>-		kobject_put(&amp;ppd-&gt;sl2sc_kobj);</div><div class='del'>-		kobject_put(&amp;ppd-&gt;sc2vl_kobj);</div><div class='del'>-	}</div><div class='add'>+	/* Unwind operations in hfi1_verbs_register_sysfs() */</div><div class='add'>+	for (i = 0; i &lt; dd-&gt;num_sdma; i++)</div><div class='add'>+		kobject_put(&amp;dd-&gt;per_sdma[i].kobj);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/tid_rdma.c b/drivers/infiniband/hw/hfi1/tid_rdma.c<br/>index 6fb93032fbef..eafd2f157e32 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/tid_rdma.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/tid_rdma.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/tid_rdma.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/tid_rdma.c</a></div><div class='hunk'>@@ -1,6 +1,6 @@</div><div class='ctx'> // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2018 Intel Corporation.</div><div class='add'>+ * Copyright(c) 2018 - 2020 Intel Corporation.</div><div class='ctx'>  *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='hunk'>@@ -107,8 +107,6 @@ static u32 mask_generation(u32 a)</div><div class='ctx'>  * C - Capcode</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-static u32 tid_rdma_flow_wt;</div><div class='del'>-</div><div class='ctx'> static void tid_rdma_trigger_resume(struct work_struct *work);</div><div class='ctx'> static void hfi1_kern_exp_rcv_free_flows(struct tid_rdma_request *req);</div><div class='ctx'> static int hfi1_kern_exp_rcv_alloc_flows(struct tid_rdma_request *req,</div><div class='hunk'>@@ -136,6 +134,26 @@ static void update_r_next_psn_fecn(struct hfi1_packet *packet,</div><div class='ctx'> 				   struct tid_rdma_flow *flow,</div><div class='ctx'> 				   bool fecn);</div><div class='ctx'> </div><div class='add'>+static void validate_r_tid_ack(struct hfi1_qp_priv *priv)</div><div class='add'>+{</div><div class='add'>+	if (priv-&gt;r_tid_ack == HFI1_QP_WQE_INVALID)</div><div class='add'>+		priv-&gt;r_tid_ack = priv-&gt;r_tid_tail;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void tid_rdma_schedule_ack(struct rvt_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_qp_priv *priv = qp-&gt;priv;</div><div class='add'>+</div><div class='add'>+	priv-&gt;s_flags |= RVT_S_ACK_PENDING;</div><div class='add'>+	hfi1_schedule_tid_send(qp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void tid_rdma_trigger_ack(struct rvt_qp *qp)</div><div class='add'>+{</div><div class='add'>+	validate_r_tid_ack(qp-&gt;priv);</div><div class='add'>+	tid_rdma_schedule_ack(qp);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static u64 tid_rdma_opfn_encode(struct tid_rdma_params *p)</div><div class='ctx'> {</div><div class='ctx'> 	return</div><div class='hunk'>@@ -176,7 +194,7 @@ void tid_rdma_opfn_init(struct rvt_qp *qp, struct tid_rdma_params *p)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_qp_priv *priv = qp-&gt;priv;</div><div class='ctx'> </div><div class='del'>-	p-&gt;qp = (kdeth_qp &lt;&lt; 16) | priv-&gt;rcd-&gt;ctxt;</div><div class='add'>+	p-&gt;qp = (RVT_KDETH_QP_PREFIX &lt;&lt; 16) | priv-&gt;rcd-&gt;ctxt;</div><div class='ctx'> 	p-&gt;max_len = TID_RDMA_MAX_SEGMENT_SIZE;</div><div class='ctx'> 	p-&gt;jkey = priv-&gt;rcd-&gt;jkey;</div><div class='ctx'> 	p-&gt;max_read = TID_RDMA_MAX_READ_SEGS_PER_REQ;</div><div class='hunk'>@@ -291,12 +309,13 @@ int hfi1_kern_exp_rcv_init(struct hfi1_ctxtdata *rcd, int reinit)</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * qp_to_rcd - determine the receive context used by a qp</div><div class='del'>- * @qp - the qp</div><div class='add'>+ * @rdi: rvt dev struct</div><div class='add'>+ * @qp: the qp</div><div class='ctx'>  *</div><div class='ctx'>  * This routine returns the receive context associated</div><div class='ctx'>  * with a a qp's qpn.</div><div class='ctx'>  *</div><div class='del'>- * Returns the context.</div><div class='add'>+ * Return: the context.</div><div class='ctx'>  */</div><div class='ctx'> static struct hfi1_ctxtdata *qp_to_rcd(struct rvt_dev_info *rdi,</div><div class='ctx'> 				       struct rvt_qp *qp)</div><div class='hunk'>@@ -312,9 +331,7 @@ static struct hfi1_ctxtdata *qp_to_rcd(struct rvt_dev_info *rdi,</div><div class='ctx'> 	if (qp-&gt;ibqp.qp_num == 0)</div><div class='ctx'> 		ctxt = 0;</div><div class='ctx'> 	else</div><div class='del'>-		ctxt = ((qp-&gt;ibqp.qp_num &gt;&gt; dd-&gt;qos_shift) %</div><div class='del'>-			(dd-&gt;n_krcv_queues - 1)) + 1;</div><div class='del'>-</div><div class='add'>+		ctxt = hfi1_get_qp_map(dd, qp-&gt;ibqp.qp_num &gt;&gt; dd-&gt;qos_shift);</div><div class='ctx'> 	return dd-&gt;rcd[ctxt];</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -468,6 +485,7 @@ static struct rvt_qp *first_qp(struct hfi1_ctxtdata *rcd,</div><div class='ctx'> /**</div><div class='ctx'>  * kernel_tid_waiters - determine rcd wait</div><div class='ctx'>  * @rcd: the receive context</div><div class='add'>+ * @queue: the queue to operate on</div><div class='ctx'>  * @qp: the head of the qp being processed</div><div class='ctx'>  *</div><div class='ctx'>  * This routine will return false IFF</div><div class='hunk'>@@ -477,7 +495,7 @@ static struct rvt_qp *first_qp(struct hfi1_ctxtdata *rcd,</div><div class='ctx'>  * Must hold the qp s_lock and the exp_lock.</div><div class='ctx'>  *</div><div class='ctx'>  * Return:</div><div class='del'>- * false if either of the conditions below are statisfied:</div><div class='add'>+ * false if either of the conditions below are satisfied:</div><div class='ctx'>  * 1. The list is empty or</div><div class='ctx'>  * 2. The indicated qp is at the head of the list and the</div><div class='ctx'>  *    HFI1_S_WAIT_TID_SPACE bit is set in qp-&gt;s_flags.</div><div class='hunk'>@@ -501,7 +519,9 @@ static bool kernel_tid_waiters(struct hfi1_ctxtdata *rcd,</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * dequeue_tid_waiter - dequeue the qp from the list</div><div class='del'>- * @qp - the qp to remove the wait list</div><div class='add'>+ * @rcd: the receive context</div><div class='add'>+ * @queue: the queue to operate on</div><div class='add'>+ * @qp: the qp to remove the wait list</div><div class='ctx'>  *</div><div class='ctx'>  * This routine removes the indicated qp from the</div><div class='ctx'>  * wait list if it is there.</div><div class='hunk'>@@ -533,6 +553,7 @@ static void dequeue_tid_waiter(struct hfi1_ctxtdata *rcd,</div><div class='ctx'> /**</div><div class='ctx'>  * queue_qp_for_tid_wait - suspend QP on tid space</div><div class='ctx'>  * @rcd: the receive context</div><div class='add'>+ * @queue: the queue to operate on</div><div class='ctx'>  * @qp: the qp</div><div class='ctx'>  *</div><div class='ctx'>  * The qp is inserted at the tail of the rcd</div><div class='hunk'>@@ -577,14 +598,14 @@ static void __trigger_tid_waiter(struct rvt_qp *qp)</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * tid_rdma_schedule_tid_wakeup - schedule wakeup for a qp</div><div class='del'>- * @qp - the qp</div><div class='add'>+ * @qp: the qp</div><div class='ctx'>  *</div><div class='ctx'>  * trigger a schedule or a waiting qp in a deadlock</div><div class='ctx'>  * safe manner.  The qp reference is held prior</div><div class='ctx'>  * to this call via first_qp().</div><div class='ctx'>  *</div><div class='ctx'>  * If the qp trigger was already scheduled (!rval)</div><div class='del'>- * the the reference is dropped, otherwise the resume</div><div class='add'>+ * the reference is dropped, otherwise the resume</div><div class='ctx'>  * or the destroy cancel will dispatch the reference.</div><div class='ctx'>  */</div><div class='ctx'> static void tid_rdma_schedule_tid_wakeup(struct rvt_qp *qp)</div><div class='hunk'>@@ -614,7 +635,7 @@ static void tid_rdma_schedule_tid_wakeup(struct rvt_qp *qp)</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * tid_rdma_trigger_resume - field a trigger work request</div><div class='del'>- * @work - the work item</div><div class='add'>+ * @work: the work item</div><div class='ctx'>  *</div><div class='ctx'>  * Complete the off qp trigger processing by directly</div><div class='ctx'>  * calling the progress routine.</div><div class='hunk'>@@ -638,7 +659,7 @@ static void tid_rdma_trigger_resume(struct work_struct *work)</div><div class='ctx'> 	rvt_put_qp(qp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/**</div><div class='add'>+/*</div><div class='ctx'>  * tid_rdma_flush_wait - unwind any tid space wait</div><div class='ctx'>  *</div><div class='ctx'>  * This is called when resetting a qp to</div><div class='hunk'>@@ -677,8 +698,8 @@ void hfi1_tid_rdma_flush_wait(struct rvt_qp *qp)</div><div class='ctx'> /* Flow functions */</div><div class='ctx'> /**</div><div class='ctx'>  * kern_reserve_flow - allocate a hardware flow</div><div class='del'>- * @rcd - the context to use for allocation</div><div class='del'>- * @last - the index of the preferred flow. Use RXE_NUM_TID_FLOWS to</div><div class='add'>+ * @rcd: the context to use for allocation</div><div class='add'>+ * @last: the index of the preferred flow. Use RXE_NUM_TID_FLOWS to</div><div class='ctx'>  *         signify "don't care".</div><div class='ctx'>  *</div><div class='ctx'>  * Use a bit mask based allocation to reserve a hardware</div><div class='hunk'>@@ -689,7 +710,7 @@ void hfi1_tid_rdma_flush_wait(struct rvt_qp *qp)</div><div class='ctx'>  * The exp_lock must be held.</div><div class='ctx'>  *</div><div class='ctx'>  * Return:</div><div class='del'>- * On success: a value postive value between 0 and RXE_NUM_TID_FLOWS - 1</div><div class='add'>+ * On success: a value positive value between 0 and RXE_NUM_TID_FLOWS - 1</div><div class='ctx'>  * On failure: -EAGAIN</div><div class='ctx'>  */</div><div class='ctx'> static int kern_reserve_flow(struct hfi1_ctxtdata *rcd, int last)</div><div class='hunk'>@@ -829,7 +850,7 @@ void hfi1_kern_init_ctxt_generations(struct hfi1_ctxtdata *rcd)</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; RXE_NUM_TID_FLOWS; i++) {</div><div class='del'>-		rcd-&gt;flows[i].generation = mask_generation(prandom_u32());</div><div class='add'>+		rcd-&gt;flows[i].generation = mask_generation(get_random_u32());</div><div class='ctx'> 		kern_set_hw_flow(rcd, KERN_GENERATION_RESERVED, i);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='hunk'>@@ -844,9 +865,10 @@ static u8 trdma_pset_order(struct tid_rdma_pageset *s)</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * tid_rdma_find_phys_blocks_4k - get groups base on mr info</div><div class='del'>- * @npages - number of pages</div><div class='del'>- * @pages - pointer to an array of page structs</div><div class='del'>- * @list - page set array to return</div><div class='add'>+ * @flow: overall info for a TID RDMA segment</div><div class='add'>+ * @pages: pointer to an array of page structs</div><div class='add'>+ * @npages: number of pages</div><div class='add'>+ * @list: page set array to return</div><div class='ctx'>  *</div><div class='ctx'>  * This routine returns the number of groups associated with</div><div class='ctx'>  * the current sge information.  This implementation is based</div><div class='hunk'>@@ -933,10 +955,10 @@ static u32 tid_rdma_find_phys_blocks_4k(struct tid_rdma_flow *flow,</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * tid_flush_pages - dump out pages into pagesets</div><div class='del'>- * @list - list of pagesets</div><div class='del'>- * @idx - pointer to current page index</div><div class='del'>- * @pages - number of pages to dump</div><div class='del'>- * @sets - current number of pagesset</div><div class='add'>+ * @list: list of pagesets</div><div class='add'>+ * @idx: pointer to current page index</div><div class='add'>+ * @pages: number of pages to dump</div><div class='add'>+ * @sets: current number of pagesset</div><div class='ctx'>  *</div><div class='ctx'>  * This routine flushes out accumuated pages.</div><div class='ctx'>  *</div><div class='hunk'>@@ -974,9 +996,10 @@ static u32 tid_flush_pages(struct tid_rdma_pageset *list,</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * tid_rdma_find_phys_blocks_8k - get groups base on mr info</div><div class='del'>- * @pages - pointer to an array of page structs</div><div class='del'>- * @npages - number of pages</div><div class='del'>- * @list - page set array to return</div><div class='add'>+ * @flow: overall info for a TID RDMA segment</div><div class='add'>+ * @pages: pointer to an array of page structs</div><div class='add'>+ * @npages: number of pages</div><div class='add'>+ * @list: page set array to return</div><div class='ctx'>  *</div><div class='ctx'>  * This routine parses an array of pages to compute pagesets</div><div class='ctx'>  * in an 8k compatible way.</div><div class='hunk'>@@ -984,7 +1007,7 @@ static u32 tid_flush_pages(struct tid_rdma_pageset *list,</div><div class='ctx'>  * pages are tested two at a time, i, i + 1 for contiguous</div><div class='ctx'>  * pages and i - 1 and i contiguous pages.</div><div class='ctx'>  *</div><div class='del'>- * If any condition is false, any accumlated pages are flushed and</div><div class='add'>+ * If any condition is false, any accumulated pages are flushed and</div><div class='ctx'>  * v0,v1 are emitted as separate PAGE_SIZE pagesets</div><div class='ctx'>  *</div><div class='ctx'>  * Otherwise, the current 8k is totaled for a future flush.</div><div class='hunk'>@@ -1048,7 +1071,7 @@ static u32 tid_rdma_find_phys_blocks_8k(struct tid_rdma_flow *flow,</div><div class='ctx'> 	return sets;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/**</div><div class='add'>+/*</div><div class='ctx'>  * Find pages for one segment of a sge array represented by @ss. The function</div><div class='ctx'>  * does not check the sge, the sge must have been checked for alignment with a</div><div class='ctx'>  * prior call to hfi1_kern_trdma_ok. Other sge checking is done as part of</div><div class='hunk'>@@ -1092,7 +1115,7 @@ static u32 kern_find_pages(struct tid_rdma_flow *flow,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	flow-&gt;length = flow-&gt;req-&gt;seg_len - length;</div><div class='del'>-	*last = req-&gt;isge == ss-&gt;num_sge ? false : true;</div><div class='add'>+	*last = req-&gt;isge != ss-&gt;num_sge;</div><div class='ctx'> 	return i;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1411,7 +1434,7 @@ static void kern_program_rcvarray(struct tid_rdma_flow *flow)</div><div class='ctx'>  * (5) computes a tidarray with formatted TID entries which can be sent</div><div class='ctx'>  *     to the sender</div><div class='ctx'>  * (6) Reserves and programs HW flows.</div><div class='del'>- * (7) It also manages queing the QP when TID/flow resources are not</div><div class='add'>+ * (7) It also manages queueing the QP when TID/flow resources are not</div><div class='ctx'>  *     available.</div><div class='ctx'>  *</div><div class='ctx'>  * @req points to struct tid_rdma_request of which the segments are a part. The</div><div class='hunk'>@@ -1581,8 +1604,8 @@ void hfi1_kern_exp_rcv_clear_all(struct tid_rdma_request *req)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='del'>- * hfi1_kern_exp_rcv_free_flows - free priviously allocated flow information</div><div class='del'>- * @req - the tid rdma request to be cleaned</div><div class='add'>+ * hfi1_kern_exp_rcv_free_flows - free previously allocated flow information</div><div class='add'>+ * @req: the tid rdma request to be cleaned</div><div class='ctx'>  */</div><div class='ctx'> static void hfi1_kern_exp_rcv_free_flows(struct tid_rdma_request *req)</div><div class='ctx'> {</div><div class='hunk'>@@ -1622,6 +1645,7 @@ static int hfi1_kern_exp_rcv_alloc_flows(struct tid_rdma_request *req,</div><div class='ctx'> 		flows[i].req = req;</div><div class='ctx'> 		flows[i].npagesets = 0;</div><div class='ctx'> 		flows[i].pagesets[0].mapped =  0;</div><div class='add'>+		flows[i].resync_npkts = 0;</div><div class='ctx'> 	}</div><div class='ctx'> 	req-&gt;flows = flows;</div><div class='ctx'> 	return 0;</div><div class='hunk'>@@ -1675,34 +1699,6 @@ static struct tid_rdma_flow *find_flow_ib(struct tid_rdma_request *req,</div><div class='ctx'> 	return NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct tid_rdma_flow *</div><div class='del'>-__find_flow_ranged(struct tid_rdma_request *req, u16 head, u16 tail,</div><div class='del'>-		   u32 psn, u16 *fidx)</div><div class='del'>-{</div><div class='del'>-	for ( ; CIRC_CNT(head, tail, MAX_FLOWS);</div><div class='del'>-	      tail = CIRC_NEXT(tail, MAX_FLOWS)) {</div><div class='del'>-		struct tid_rdma_flow *flow = &amp;req-&gt;flows[tail];</div><div class='del'>-		u32 spsn, lpsn;</div><div class='del'>-</div><div class='del'>-		spsn = full_flow_psn(flow, flow-&gt;flow_state.spsn);</div><div class='del'>-		lpsn = full_flow_psn(flow, flow-&gt;flow_state.lpsn);</div><div class='del'>-</div><div class='del'>-		if (cmp_psn(psn, spsn) &gt;= 0 &amp;&amp; cmp_psn(psn, lpsn) &lt;= 0) {</div><div class='del'>-			if (fidx)</div><div class='del'>-				*fidx = tail;</div><div class='del'>-			return flow;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct tid_rdma_flow *find_flow(struct tid_rdma_request *req,</div><div class='del'>-				       u32 psn, u16 *fidx)</div><div class='del'>-{</div><div class='del'>-	return __find_flow_ranged(req, req-&gt;setup_head, req-&gt;clear_tail, psn,</div><div class='del'>-				  fidx);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> /* TID RDMA READ functions */</div><div class='ctx'> u32 hfi1_build_tid_rdma_read_packet(struct rvt_swqe *wqe,</div><div class='ctx'> 				    struct ib_other_headers *ohdr, u32 *bth1,</div><div class='hunk'>@@ -2026,7 +2022,6 @@ static int tid_rdma_rcv_error(struct hfi1_packet *packet,</div><div class='ctx'> 	trace_hfi1_tid_req_rcv_err(qp, 0, e-&gt;opcode, e-&gt;psn, e-&gt;lpsn, req);</div><div class='ctx'> 	if (e-&gt;opcode == TID_OP(READ_REQ)) {</div><div class='ctx'> 		struct ib_reth *reth;</div><div class='del'>-		u32 offset;</div><div class='ctx'> 		u32 len;</div><div class='ctx'> 		u32 rkey;</div><div class='ctx'> 		u64 vaddr;</div><div class='hunk'>@@ -2038,7 +2033,6 @@ static int tid_rdma_rcv_error(struct hfi1_packet *packet,</div><div class='ctx'> 		 * The requester always restarts from the start of the original</div><div class='ctx'> 		 * request.</div><div class='ctx'> 		 */</div><div class='del'>-		offset = delta_psn(psn, e-&gt;psn) * qp-&gt;pmtu;</div><div class='ctx'> 		len = be32_to_cpu(reth-&gt;length);</div><div class='ctx'> 		if (psn != e-&gt;psn || len != req-&gt;total_len)</div><div class='ctx'> 			goto unlock;</div><div class='hunk'>@@ -2061,7 +2055,7 @@ static int tid_rdma_rcv_error(struct hfi1_packet *packet,</div><div class='ctx'> 		 * req-&gt;clear_tail is advanced). However, when an earlier</div><div class='ctx'> 		 * request is received, this request will not be complete any</div><div class='ctx'> 		 * more (qp-&gt;s_tail_ack_queue is moved back, see below).</div><div class='del'>-		 * Consequently, we need to update the TID flow info everytime</div><div class='add'>+		 * Consequently, we need to update the TID flow info every time</div><div class='ctx'> 		 * a duplicate request is received.</div><div class='ctx'> 		 */</div><div class='ctx'> 		bth0 = be32_to_cpu(ohdr-&gt;bth[0]);</div><div class='hunk'>@@ -2225,7 +2219,7 @@ void hfi1_rc_rcv_tid_rdma_read_req(struct hfi1_packet *packet)</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * 1. Verify TID RDMA READ REQ as per IB_OPCODE_RC_RDMA_READ</div><div class='ctx'> 	 *    (see hfi1_rc_rcv())</div><div class='del'>-	 * 2. Put TID RDMA READ REQ into the response queueu (s_ack_queue)</div><div class='add'>+	 * 2. Put TID RDMA READ REQ into the response queue (s_ack_queue)</div><div class='ctx'> 	 *     - Setup struct tid_rdma_req with request info</div><div class='ctx'> 	 *     - Initialize struct tid_rdma_flow info;</div><div class='ctx'> 	 *     - Copy TID entries;</div><div class='hunk'>@@ -2445,7 +2439,7 @@ find_tid_request(struct rvt_qp *qp, u32 psn, enum ib_wr_opcode opcode)</div><div class='ctx'> </div><div class='ctx'> void hfi1_rc_rcv_tid_rdma_read_resp(struct hfi1_packet *packet)</div><div class='ctx'> {</div><div class='del'>-	/* HANDLER FOR TID RDMA READ RESPONSE packet (Requestor side */</div><div class='add'>+	/* HANDLER FOR TID RDMA READ RESPONSE packet (Requester side) */</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * 1. Find matching SWQE</div><div class='hunk'>@@ -2605,18 +2599,9 @@ void hfi1_kern_read_tid_flow_free(struct rvt_qp *qp)</div><div class='ctx'> 	hfi1_kern_clear_hw_flow(priv-&gt;rcd, qp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static bool tid_rdma_tid_err(struct hfi1_ctxtdata *rcd,</div><div class='del'>-			     struct hfi1_packet *packet, u8 rcv_type,</div><div class='del'>-			     u8 opcode)</div><div class='add'>+static bool tid_rdma_tid_err(struct hfi1_packet *packet, u8 rcv_type)</div><div class='ctx'> {</div><div class='ctx'> 	struct rvt_qp *qp = packet-&gt;qp;</div><div class='del'>-	struct hfi1_qp_priv *qpriv = qp-&gt;priv;</div><div class='del'>-	u32 ipsn;</div><div class='del'>-	struct ib_other_headers *ohdr = packet-&gt;ohdr;</div><div class='del'>-	struct rvt_ack_entry *e;</div><div class='del'>-	struct tid_rdma_request *req;</div><div class='del'>-	struct rvt_dev_info *rdi = ib_to_rvt(qp-&gt;ibqp.device);</div><div class='del'>-	u32 i;</div><div class='ctx'> </div><div class='ctx'> 	if (rcv_type &gt;= RHF_RCV_TYPE_IB)</div><div class='ctx'> 		goto done;</div><div class='hunk'>@@ -2633,41 +2618,9 @@ static bool tid_rdma_tid_err(struct hfi1_ctxtdata *rcd,</div><div class='ctx'> 	if (rcv_type == RHF_RCV_TYPE_EAGER) {</div><div class='ctx'> 		hfi1_restart_rc(qp, qp-&gt;s_last_psn + 1, 1);</div><div class='ctx'> 		hfi1_schedule_send(qp);</div><div class='del'>-		goto done_unlock;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * For TID READ response, error out QP after freeing the tid</div><div class='del'>-	 * resources.</div><div class='del'>-	 */</div><div class='del'>-	if (opcode == TID_OP(READ_RESP)) {</div><div class='del'>-		ipsn = mask_psn(be32_to_cpu(ohdr-&gt;u.tid_rdma.r_rsp.verbs_psn));</div><div class='del'>-		if (cmp_psn(ipsn, qp-&gt;s_last_psn) &gt; 0 &amp;&amp;</div><div class='del'>-		    cmp_psn(ipsn, qp-&gt;s_psn) &lt; 0) {</div><div class='del'>-			hfi1_kern_read_tid_flow_free(qp);</div><div class='del'>-			spin_unlock(&amp;qp-&gt;s_lock);</div><div class='del'>-			rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);</div><div class='del'>-			goto done;</div><div class='del'>-		}</div><div class='del'>-		goto done_unlock;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Error out the qp for TID RDMA WRITE</div><div class='del'>-	 */</div><div class='del'>-	hfi1_kern_clear_hw_flow(qpriv-&gt;rcd, qp);</div><div class='del'>-	for (i = 0; i &lt; rvt_max_atomic(rdi); i++) {</div><div class='del'>-		e = &amp;qp-&gt;s_ack_queue[i];</div><div class='del'>-		if (e-&gt;opcode == TID_OP(WRITE_REQ)) {</div><div class='del'>-			req = ack_to_tid_req(e);</div><div class='del'>-			hfi1_kern_exp_rcv_clear_all(req);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	spin_unlock(&amp;qp-&gt;s_lock);</div><div class='del'>-	rvt_rc_error(qp, IB_WC_LOC_LEN_ERR);</div><div class='del'>-	goto done;</div><div class='del'>-</div><div class='del'>-done_unlock:</div><div class='add'>+	/* Since no payload is delivered, just drop the packet */</div><div class='ctx'> 	spin_unlock(&amp;qp-&gt;s_lock);</div><div class='ctx'> done:</div><div class='ctx'> 	return true;</div><div class='hunk'>@@ -2718,12 +2671,15 @@ static bool handle_read_kdeth_eflags(struct hfi1_ctxtdata *rcd,</div><div class='ctx'> 	u32 fpsn;</div><div class='ctx'> </div><div class='ctx'> 	lockdep_assert_held(&amp;qp-&gt;r_lock);</div><div class='add'>+	trace_hfi1_rsp_read_kdeth_eflags(qp, ibpsn);</div><div class='add'>+	trace_hfi1_sender_read_kdeth_eflags(qp);</div><div class='add'>+	trace_hfi1_tid_read_sender_kdeth_eflags(qp, 0);</div><div class='add'>+	spin_lock(&amp;qp-&gt;s_lock);</div><div class='ctx'> 	/* If the psn is out of valid range, drop the packet */</div><div class='ctx'> 	if (cmp_psn(ibpsn, qp-&gt;s_last_psn) &lt; 0 ||</div><div class='ctx'> 	    cmp_psn(ibpsn, qp-&gt;s_psn) &gt; 0)</div><div class='del'>-		return ret;</div><div class='add'>+		goto s_unlock;</div><div class='ctx'> </div><div class='del'>-	spin_lock(&amp;qp-&gt;s_lock);</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Note that NAKs implicitly ACK outstanding SEND and RDMA write</div><div class='ctx'> 	 * requests and implicitly NAK RDMA read and atomic requests issued</div><div class='hunk'>@@ -2771,14 +2727,19 @@ static bool handle_read_kdeth_eflags(struct hfi1_ctxtdata *rcd,</div><div class='ctx'> </div><div class='ctx'> 		wqe = do_rc_completion(qp, wqe, ibp);</div><div class='ctx'> 		if (qp-&gt;s_acked == qp-&gt;s_tail)</div><div class='del'>-			break;</div><div class='add'>+			goto s_unlock;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (qp-&gt;s_acked == qp-&gt;s_tail)</div><div class='add'>+		goto s_unlock;</div><div class='add'>+</div><div class='ctx'> 	/* Handle the eflags for the request */</div><div class='ctx'> 	if (wqe-&gt;wr.opcode != IB_WR_TID_RDMA_READ)</div><div class='ctx'> 		goto s_unlock;</div><div class='ctx'> </div><div class='ctx'> 	req = wqe_to_tid_req(wqe);</div><div class='add'>+	trace_hfi1_tid_req_read_kdeth_eflags(qp, 0, wqe-&gt;wr.opcode, wqe-&gt;psn,</div><div class='add'>+					     wqe-&gt;lpsn, req);</div><div class='ctx'> 	switch (rcv_type) {</div><div class='ctx'> 	case RHF_RCV_TYPE_EXPECTED:</div><div class='ctx'> 		switch (rte) {</div><div class='hunk'>@@ -2792,28 +2753,14 @@ static bool handle_read_kdeth_eflags(struct hfi1_ctxtdata *rcd,</div><div class='ctx'> 			 * to prevent continuous Flow Sequence errors for any</div><div class='ctx'> 			 * packets that could be still in the fabric.</div><div class='ctx'> 			 */</div><div class='del'>-			flow = find_flow(req, psn, NULL);</div><div class='del'>-			if (!flow) {</div><div class='del'>-				/*</div><div class='del'>-				 * We can't find the IB PSN matching the</div><div class='del'>-				 * received KDETH PSN. The only thing we can</div><div class='del'>-				 * do at this point is report the error to</div><div class='del'>-				 * the QP.</div><div class='del'>-				 */</div><div class='del'>-				hfi1_kern_read_tid_flow_free(qp);</div><div class='del'>-				spin_unlock(&amp;qp-&gt;s_lock);</div><div class='del'>-				rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);</div><div class='del'>-				return ret;</div><div class='del'>-			}</div><div class='add'>+			flow = &amp;req-&gt;flows[req-&gt;clear_tail];</div><div class='add'>+			trace_hfi1_tid_flow_read_kdeth_eflags(qp,</div><div class='add'>+							      req-&gt;clear_tail,</div><div class='add'>+							      flow);</div><div class='ctx'> 			if (priv-&gt;s_flags &amp; HFI1_R_TID_SW_PSN) {</div><div class='ctx'> 				diff = cmp_psn(psn,</div><div class='ctx'> 					       flow-&gt;flow_state.r_next_psn);</div><div class='ctx'> 				if (diff &gt; 0) {</div><div class='del'>-					if (!(qp-&gt;r_flags &amp; RVT_R_RDMAR_SEQ))</div><div class='del'>-						restart_tid_rdma_read_req(rcd,</div><div class='del'>-									  qp,</div><div class='del'>-									  wqe);</div><div class='del'>-</div><div class='ctx'> 					/* Drop the packet.*/</div><div class='ctx'> 					goto s_unlock;</div><div class='ctx'> 				} else if (diff &lt; 0) {</div><div class='hunk'>@@ -2886,6 +2833,7 @@ static bool handle_read_kdeth_eflags(struct hfi1_ctxtdata *rcd,</div><div class='ctx'> 		default:</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='add'>+		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -2965,7 +2913,7 @@ bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd,</div><div class='ctx'> 		if (lnh == HFI1_LRH_GRH)</div><div class='ctx'> 			goto r_unlock;</div><div class='ctx'> </div><div class='del'>-		if (tid_rdma_tid_err(rcd, packet, rcv_type, opcode))</div><div class='add'>+		if (tid_rdma_tid_err(packet, rcv_type))</div><div class='ctx'> 			goto r_unlock;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -2985,8 +2933,15 @@ bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd,</div><div class='ctx'> 	 */</div><div class='ctx'> 	spin_lock(&amp;qp-&gt;s_lock);</div><div class='ctx'> 	qpriv = qp-&gt;priv;</div><div class='add'>+	if (qpriv-&gt;r_tid_tail == HFI1_QP_WQE_INVALID ||</div><div class='add'>+	    qpriv-&gt;r_tid_tail == qpriv-&gt;r_tid_head)</div><div class='add'>+		goto unlock;</div><div class='ctx'> 	e = &amp;qp-&gt;s_ack_queue[qpriv-&gt;r_tid_tail];</div><div class='add'>+	if (e-&gt;opcode != TID_OP(WRITE_REQ))</div><div class='add'>+		goto unlock;</div><div class='ctx'> 	req = ack_to_tid_req(e);</div><div class='add'>+	if (req-&gt;comp_seg == req-&gt;cur_seg)</div><div class='add'>+		goto unlock;</div><div class='ctx'> 	flow = &amp;req-&gt;flows[req-&gt;clear_tail];</div><div class='ctx'> 	trace_hfi1_eflags_err_write(qp, rcv_type, rte, psn);</div><div class='ctx'> 	trace_hfi1_rsp_handle_kdeth_eflags(qp, psn);</div><div class='hunk'>@@ -3058,6 +3013,7 @@ bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd,</div><div class='ctx'> 		default:</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='add'>+		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -3076,10 +3032,7 @@ nak_psn:</div><div class='ctx'> 		qpriv-&gt;s_nak_state = IB_NAK_PSN_ERROR;</div><div class='ctx'> 		/* We are NAK'ing the next expected PSN */</div><div class='ctx'> 		qpriv-&gt;s_nak_psn = mask_psn(flow-&gt;flow_state.r_next_psn);</div><div class='del'>-		qpriv-&gt;s_flags |= RVT_S_ACK_PENDING;</div><div class='del'>-		if (qpriv-&gt;r_tid_ack == HFI1_QP_WQE_INVALID)</div><div class='del'>-			qpriv-&gt;r_tid_ack = qpriv-&gt;r_tid_tail;</div><div class='del'>-		hfi1_schedule_tid_send(qp);</div><div class='add'>+		tid_rdma_trigger_ack(qp);</div><div class='ctx'> 	}</div><div class='ctx'> 	goto unlock;</div><div class='ctx'> }</div><div class='hunk'>@@ -3271,11 +3224,13 @@ bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe)</div><div class='ctx'> 	case IB_WR_ATOMIC_CMP_AND_SWP:</div><div class='ctx'> 	case IB_WR_ATOMIC_FETCH_AND_ADD:</div><div class='ctx'> 	case IB_WR_RDMA_WRITE:</div><div class='add'>+	case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='ctx'> 		switch (prev-&gt;wr.opcode) {</div><div class='ctx'> 		case IB_WR_TID_RDMA_WRITE:</div><div class='ctx'> 			req = wqe_to_tid_req(prev);</div><div class='ctx'> 			if (req-&gt;ack_seg != req-&gt;total_segs)</div><div class='ctx'> 				goto interlock;</div><div class='add'>+			break;</div><div class='ctx'> 		default:</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -3283,7 +3238,7 @@ bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe)</div><div class='ctx'> 	case IB_WR_RDMA_READ:</div><div class='ctx'> 		if (prev-&gt;wr.opcode != IB_WR_TID_RDMA_WRITE)</div><div class='ctx'> 			break;</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case IB_WR_TID_RDMA_READ:</div><div class='ctx'> 		switch (prev-&gt;wr.opcode) {</div><div class='ctx'> 		case IB_WR_RDMA_READ:</div><div class='hunk'>@@ -3294,9 +3249,11 @@ bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe)</div><div class='ctx'> 			req = wqe_to_tid_req(prev);</div><div class='ctx'> 			if (req-&gt;ack_seg != req-&gt;total_segs)</div><div class='ctx'> 				goto interlock;</div><div class='add'>+			break;</div><div class='ctx'> 		default:</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='add'>+		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -3442,18 +3399,17 @@ u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe,</div><div class='ctx'> 	return sizeof(ohdr-&gt;u.tid_rdma.w_req) / sizeof(u32);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void hfi1_compute_tid_rdma_flow_wt(void)</div><div class='add'>+static u32 hfi1_compute_tid_rdma_flow_wt(struct rvt_qp *qp)</div><div class='ctx'> {</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Heuristic for computing the RNR timeout when waiting on the flow</div><div class='ctx'> 	 * queue. Rather than a computationaly expensive exact estimate of when</div><div class='ctx'> 	 * a flow will be available, we assume that if a QP is at position N in</div><div class='ctx'> 	 * the flow queue it has to wait approximately (N + 1) * (number of</div><div class='del'>-	 * segments between two sync points), assuming PMTU of 4K. The rationale</div><div class='del'>-	 * for this is that flows are released and recycled at each sync point.</div><div class='add'>+	 * segments between two sync points). The rationale for this is that</div><div class='add'>+	 * flows are released and recycled at each sync point.</div><div class='ctx'> 	 */</div><div class='del'>-	tid_rdma_flow_wt = MAX_TID_FLOW_PSN * enum_to_mtu(OPA_MTU_4096) /</div><div class='del'>-		TID_RDMA_MAX_SEGMENT_SIZE;</div><div class='add'>+	return (MAX_TID_FLOW_PSN * qp-&gt;pmtu) &gt;&gt; TID_RDMA_SEGMENT_SHIFT;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static u32 position_in_queue(struct hfi1_qp_priv *qpriv,</div><div class='hunk'>@@ -3486,7 +3442,7 @@ static u32 hfi1_compute_tid_rnr_timeout(struct rvt_qp *qp, u32 to_seg)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/**</div><div class='add'>+/*</div><div class='ctx'>  * Central place for resource allocation at TID write responder,</div><div class='ctx'>  * is called from write_req and write_data interrupt handlers as</div><div class='ctx'>  * well as the send thread when a queued QP is scheduled for</div><div class='hunk'>@@ -3576,7 +3532,7 @@ static void hfi1_tid_write_alloc_resources(struct rvt_qp *qp, bool intr_ctx)</div><div class='ctx'> 		if (qpriv-&gt;flow_state.index &gt;= RXE_NUM_TID_FLOWS) {</div><div class='ctx'> 			ret = hfi1_kern_setup_hw_flow(qpriv-&gt;rcd, qp);</div><div class='ctx'> 			if (ret) {</div><div class='del'>-				to_seg = tid_rdma_flow_wt *</div><div class='add'>+				to_seg = hfi1_compute_tid_rdma_flow_wt(qp) *</div><div class='ctx'> 					position_in_queue(qpriv,</div><div class='ctx'> 							  &amp;rcd-&gt;flow_queue);</div><div class='ctx'> 				break;</div><div class='hunk'>@@ -3597,7 +3553,7 @@ static void hfi1_tid_write_alloc_resources(struct rvt_qp *qp, bool intr_ctx)</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * If overtaking req-&gt;acked_tail, send an RNR NAK. Because the</div><div class='ctx'> 		 * QP is not queued in this case, and the issue can only be</div><div class='del'>-		 * caused due a delay in scheduling the second leg which we</div><div class='add'>+		 * caused by a delay in scheduling the second leg which we</div><div class='ctx'> 		 * cannot estimate, we use a rather arbitrary RNR timeout of</div><div class='ctx'> 		 * (MAX_FLOWS / 2) segments</div><div class='ctx'> 		 */</div><div class='hunk'>@@ -3605,8 +3561,7 @@ static void hfi1_tid_write_alloc_resources(struct rvt_qp *qp, bool intr_ctx)</div><div class='ctx'> 				MAX_FLOWS)) {</div><div class='ctx'> 			ret = -EAGAIN;</div><div class='ctx'> 			to_seg = MAX_FLOWS &gt;&gt; 1;</div><div class='del'>-			qpriv-&gt;s_flags |= RVT_S_ACK_PENDING;</div><div class='del'>-			hfi1_schedule_tid_send(qp);</div><div class='add'>+			tid_rdma_trigger_ack(qp);</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='hunk'>@@ -3694,7 +3649,7 @@ void hfi1_rc_rcv_tid_rdma_write_req(struct hfi1_packet *packet)</div><div class='ctx'> 	 * 1. Verify TID RDMA WRITE REQ as per IB_OPCODE_RC_RDMA_WRITE_FIRST</div><div class='ctx'> 	 *    (see hfi1_rc_rcv())</div><div class='ctx'> 	 *     - Don't allow 0-length requests.</div><div class='del'>-	 * 2. Put TID RDMA WRITE REQ into the response queueu (s_ack_queue)</div><div class='add'>+	 * 2. Put TID RDMA WRITE REQ into the response queue (s_ack_queue)</div><div class='ctx'> 	 *     - Setup struct tid_rdma_req with request info</div><div class='ctx'> 	 *     - Prepare struct tid_rdma_flow array?</div><div class='ctx'> 	 * 3. Set the qp-&gt;s_ack_state as state diagram in design doc.</div><div class='hunk'>@@ -4010,7 +3965,7 @@ static int hfi1_stop_tid_reap_timer(struct rvt_qp *qp)</div><div class='ctx'> </div><div class='ctx'> 	lockdep_assert_held(&amp;qp-&gt;s_lock);</div><div class='ctx'> 	if (qpriv-&gt;s_flags &amp; HFI1_R_TID_RSC_TIMER) {</div><div class='del'>-		rval = del_timer(&amp;qpriv-&gt;s_tid_timer);</div><div class='add'>+		rval = timer_delete(&amp;qpriv-&gt;s_tid_timer);</div><div class='ctx'> 		qpriv-&gt;s_flags &amp;= ~HFI1_R_TID_RSC_TIMER;</div><div class='ctx'> 	}</div><div class='ctx'> 	return rval;</div><div class='hunk'>@@ -4020,13 +3975,13 @@ void hfi1_del_tid_reap_timer(struct rvt_qp *qp)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_qp_priv *qpriv = qp-&gt;priv;</div><div class='ctx'> </div><div class='del'>-	del_timer_sync(&amp;qpriv-&gt;s_tid_timer);</div><div class='add'>+	timer_delete_sync(&amp;qpriv-&gt;s_tid_timer);</div><div class='ctx'> 	qpriv-&gt;s_flags &amp;= ~HFI1_R_TID_RSC_TIMER;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void hfi1_tid_timeout(struct timer_list *t)</div><div class='ctx'> {</div><div class='del'>-	struct hfi1_qp_priv *qpriv = from_timer(qpriv, t, s_tid_timer);</div><div class='add'>+	struct hfi1_qp_priv *qpriv = timer_container_of(qpriv, t, s_tid_timer);</div><div class='ctx'> 	struct rvt_qp *qp = qpriv-&gt;owner;</div><div class='ctx'> 	struct rvt_dev_info *rdi = ib_to_rvt(qp-&gt;ibqp.device);</div><div class='ctx'> 	unsigned long flags;</div><div class='hunk'>@@ -4071,7 +4026,7 @@ unlock_r_lock:</div><div class='ctx'> </div><div class='ctx'> void hfi1_rc_rcv_tid_rdma_write_resp(struct hfi1_packet *packet)</div><div class='ctx'> {</div><div class='del'>-	/* HANDLER FOR TID RDMA WRITE RESPONSE packet (Requestor side */</div><div class='add'>+	/* HANDLER FOR TID RDMA WRITE RESPONSE packet (Requester side) */</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * 1. Find matching SWQE</div><div class='hunk'>@@ -4406,8 +4361,7 @@ void hfi1_rc_rcv_tid_rdma_write_data(struct hfi1_packet *packet)</div><div class='ctx'> 	trace_hfi1_tid_req_rcv_write_data(qp, 0, e-&gt;opcode, e-&gt;psn, e-&gt;lpsn,</div><div class='ctx'> 					  req);</div><div class='ctx'> 	trace_hfi1_tid_write_rsp_rcv_data(qp);</div><div class='del'>-	if (priv-&gt;r_tid_ack == HFI1_QP_WQE_INVALID)</div><div class='del'>-		priv-&gt;r_tid_ack = priv-&gt;r_tid_tail;</div><div class='add'>+	validate_r_tid_ack(priv);</div><div class='ctx'> </div><div class='ctx'> 	if (opcode == TID_OP(WRITE_DATA_LAST)) {</div><div class='ctx'> 		release_rdma_sge_mr(e);</div><div class='hunk'>@@ -4446,8 +4400,7 @@ void hfi1_rc_rcv_tid_rdma_write_data(struct hfi1_packet *packet)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> done:</div><div class='del'>-	priv-&gt;s_flags |= RVT_S_ACK_PENDING;</div><div class='del'>-	hfi1_schedule_tid_send(qp);</div><div class='add'>+	tid_rdma_schedule_ack(qp);</div><div class='ctx'> exit:</div><div class='ctx'> 	priv-&gt;r_next_psn_kdeth = flow-&gt;flow_state.r_next_psn;</div><div class='ctx'> 	if (fecn)</div><div class='hunk'>@@ -4459,10 +4412,7 @@ send_nak:</div><div class='ctx'> 	if (!priv-&gt;s_nak_state) {</div><div class='ctx'> 		priv-&gt;s_nak_state = IB_NAK_PSN_ERROR;</div><div class='ctx'> 		priv-&gt;s_nak_psn = flow-&gt;flow_state.r_next_psn;</div><div class='del'>-		priv-&gt;s_flags |= RVT_S_ACK_PENDING;</div><div class='del'>-		if (priv-&gt;r_tid_ack == HFI1_QP_WQE_INVALID)</div><div class='del'>-			priv-&gt;r_tid_ack = priv-&gt;r_tid_tail;</div><div class='del'>-		hfi1_schedule_tid_send(qp);</div><div class='add'>+		tid_rdma_trigger_ack(qp);</div><div class='ctx'> 	}</div><div class='ctx'> 	goto done;</div><div class='ctx'> }</div><div class='hunk'>@@ -4552,7 +4502,7 @@ void hfi1_rc_rcv_tid_rdma_ack(struct hfi1_packet *packet)</div><div class='ctx'> 	struct rvt_swqe *wqe;</div><div class='ctx'> 	struct tid_rdma_request *req;</div><div class='ctx'> 	struct tid_rdma_flow *flow;</div><div class='del'>-	u32 aeth, psn, req_psn, ack_psn, fspsn, resync_psn, ack_kpsn;</div><div class='add'>+	u32 aeth, psn, req_psn, ack_psn, flpsn, resync_psn, ack_kpsn;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	u16 fidx;</div><div class='ctx'> </div><div class='hunk'>@@ -4581,6 +4531,9 @@ void hfi1_rc_rcv_tid_rdma_ack(struct hfi1_packet *packet)</div><div class='ctx'> 		ack_kpsn--;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (unlikely(qp-&gt;s_acked == qp-&gt;s_tail))</div><div class='add'>+		goto ack_op_err;</div><div class='add'>+</div><div class='ctx'> 	wqe = rvt_get_swqe_ptr(qp, qp-&gt;s_acked);</div><div class='ctx'> </div><div class='ctx'> 	if (wqe-&gt;wr.opcode != IB_WR_TID_RDMA_WRITE)</div><div class='hunk'>@@ -4593,7 +4546,8 @@ void hfi1_rc_rcv_tid_rdma_ack(struct hfi1_packet *packet)</div><div class='ctx'> 	trace_hfi1_tid_flow_rcv_tid_ack(qp, req-&gt;acked_tail, flow);</div><div class='ctx'> </div><div class='ctx'> 	/* Drop stale ACK/NAK */</div><div class='del'>-	if (cmp_psn(psn, full_flow_psn(flow, flow-&gt;flow_state.spsn)) &lt; 0)</div><div class='add'>+	if (cmp_psn(psn, full_flow_psn(flow, flow-&gt;flow_state.spsn)) &lt; 0 ||</div><div class='add'>+	    cmp_psn(req_psn, flow-&gt;flow_state.resp_ib_psn) &lt; 0)</div><div class='ctx'> 		goto ack_op_err;</div><div class='ctx'> </div><div class='ctx'> 	while (cmp_psn(ack_kpsn,</div><div class='hunk'>@@ -4692,6 +4646,15 @@ void hfi1_rc_rcv_tid_rdma_ack(struct hfi1_packet *packet)</div><div class='ctx'> 			 */</div><div class='ctx'> 			fpsn = full_flow_psn(flow, flow-&gt;flow_state.spsn);</div><div class='ctx'> 			req-&gt;r_ack_psn = psn;</div><div class='add'>+			/*</div><div class='add'>+			 * If resync_psn points to the last flow PSN for a</div><div class='add'>+			 * segment and the new segment (likely from a new</div><div class='add'>+			 * request) starts with a new generation number, we</div><div class='add'>+			 * need to adjust resync_psn accordingly.</div><div class='add'>+			 */</div><div class='add'>+			if (flow-&gt;flow_state.generation !=</div><div class='add'>+			    (resync_psn &gt;&gt; HFI1_KDETH_BTH_SEQ_SHIFT))</div><div class='add'>+				resync_psn = mask_psn(fpsn - 1);</div><div class='ctx'> 			flow-&gt;resync_npkts +=</div><div class='ctx'> 				delta_psn(mask_psn(resync_psn + 1), fpsn);</div><div class='ctx'> 			/*</div><div class='hunk'>@@ -4755,8 +4718,12 @@ done:</div><div class='ctx'> 		switch ((aeth &gt;&gt; IB_AETH_CREDIT_SHIFT) &amp;</div><div class='ctx'> 			IB_AETH_CREDIT_MASK) {</div><div class='ctx'> 		case 0: /* PSN sequence error */</div><div class='add'>+			if (!req-&gt;flows)</div><div class='add'>+				break;</div><div class='ctx'> 			flow = &amp;req-&gt;flows[req-&gt;acked_tail];</div><div class='del'>-			fspsn = full_flow_psn(flow, flow-&gt;flow_state.spsn);</div><div class='add'>+			flpsn = full_flow_psn(flow, flow-&gt;flow_state.lpsn);</div><div class='add'>+			if (cmp_psn(psn, flpsn) &gt; 0)</div><div class='add'>+				break;</div><div class='ctx'> 			trace_hfi1_tid_flow_rcv_tid_ack(qp, req-&gt;acked_tail,</div><div class='ctx'> 							flow);</div><div class='ctx'> 			req-&gt;r_ack_psn = mask_psn(be32_to_cpu(ohdr-&gt;bth[2]));</div><div class='hunk'>@@ -4814,7 +4781,7 @@ static int hfi1_stop_tid_retry_timer(struct rvt_qp *qp)</div><div class='ctx'> </div><div class='ctx'> 	lockdep_assert_held(&amp;qp-&gt;s_lock);</div><div class='ctx'> 	if (priv-&gt;s_flags &amp; HFI1_S_TID_RETRY_TIMER) {</div><div class='del'>-		rval = del_timer(&amp;priv-&gt;s_tid_retry_timer);</div><div class='add'>+		rval = timer_delete(&amp;priv-&gt;s_tid_retry_timer);</div><div class='ctx'> 		priv-&gt;s_flags &amp;= ~HFI1_S_TID_RETRY_TIMER;</div><div class='ctx'> 	}</div><div class='ctx'> 	return rval;</div><div class='hunk'>@@ -4824,13 +4791,14 @@ void hfi1_del_tid_retry_timer(struct rvt_qp *qp)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_qp_priv *priv = qp-&gt;priv;</div><div class='ctx'> </div><div class='del'>-	del_timer_sync(&amp;priv-&gt;s_tid_retry_timer);</div><div class='add'>+	timer_delete_sync(&amp;priv-&gt;s_tid_retry_timer);</div><div class='ctx'> 	priv-&gt;s_flags &amp;= ~HFI1_S_TID_RETRY_TIMER;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void hfi1_tid_retry_timeout(struct timer_list *t)</div><div class='ctx'> {</div><div class='del'>-	struct hfi1_qp_priv *priv = from_timer(priv, t, s_tid_retry_timer);</div><div class='add'>+	struct hfi1_qp_priv *priv = timer_container_of(priv, t,</div><div class='add'>+						       s_tid_retry_timer);</div><div class='ctx'> 	struct rvt_qp *qp = priv-&gt;owner;</div><div class='ctx'> 	struct rvt_swqe *wqe;</div><div class='ctx'> 	unsigned long flags;</div><div class='hunk'>@@ -5002,8 +4970,7 @@ void hfi1_rc_rcv_tid_rdma_resync(struct hfi1_packet *packet)</div><div class='ctx'> 	qpriv-&gt;resync = true;</div><div class='ctx'> 	/* RESYNC request always gets a TID RDMA ACK. */</div><div class='ctx'> 	qpriv-&gt;s_nak_state = 0;</div><div class='del'>-	qpriv-&gt;s_flags |= RVT_S_ACK_PENDING;</div><div class='del'>-	hfi1_schedule_tid_send(qp);</div><div class='add'>+	tid_rdma_trigger_ack(qp);</div><div class='ctx'> bail:</div><div class='ctx'> 	if (fecn)</div><div class='ctx'> 		qp-&gt;s_flags |= RVT_S_ECN;</div><div class='hunk'>@@ -5114,7 +5081,7 @@ int hfi1_make_tid_rdma_pkt(struct rvt_qp *qp, struct hfi1_pkt_state *ps)</div><div class='ctx'> 		if (priv-&gt;s_state == TID_OP(WRITE_REQ))</div><div class='ctx'> 			hfi1_tid_rdma_restart_req(qp, wqe, &amp;bth2);</div><div class='ctx'> 		priv-&gt;s_state = TID_OP(WRITE_DATA);</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> </div><div class='ctx'> 	case TID_OP(WRITE_DATA):</div><div class='ctx'> 		/*</div><div class='hunk'>@@ -5208,7 +5175,7 @@ bail_no_tx:</div><div class='ctx'> 	priv-&gt;s_flags &amp;= ~RVT_S_BUSY;</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * If we didn't get a txreq, the QP will be woken up later to try</div><div class='del'>-	 * again, set the flags to the the wake up which work item to wake</div><div class='add'>+	 * again, set the flags to the wake up which work item to wake</div><div class='ctx'> 	 * up.</div><div class='ctx'> 	 * (A better algorithm should be found to do this and generalize the</div><div class='ctx'> 	 * sleep/wakeup flags.)</div><div class='hunk'>@@ -5453,7 +5420,10 @@ static bool _hfi1_schedule_tid_send(struct rvt_qp *qp)</div><div class='ctx'> 	struct hfi1_ibport *ibp =</div><div class='ctx'> 		to_iport(qp-&gt;ibqp.device, qp-&gt;port_num);</div><div class='ctx'> 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);</div><div class='del'>-	struct hfi1_devdata *dd = dd_from_ibdev(qp-&gt;ibqp.device);</div><div class='add'>+	struct hfi1_devdata *dd = ppd-&gt;dd;</div><div class='add'>+</div><div class='add'>+	if ((dd-&gt;flags &amp; HFI1_SHUTDOWN))</div><div class='add'>+		return true;</div><div class='ctx'> </div><div class='ctx'> 	return iowait_tid_schedule(&amp;priv-&gt;s_iowait, ppd-&gt;hfi1_wq,</div><div class='ctx'> 				   priv-&gt;s_sde ?</div><div class='hunk'>@@ -5471,8 +5441,9 @@ static bool _hfi1_schedule_tid_send(struct rvt_qp *qp)</div><div class='ctx'>  * the two state machines can step on each other with respect to the</div><div class='ctx'>  * RVT_S_BUSY flag.</div><div class='ctx'>  * Therefore, a modified test is used.</div><div class='del'>- * @return true if the second leg is scheduled;</div><div class='del'>- *  false if the second leg is not scheduled.</div><div class='add'>+ *</div><div class='add'>+ * Return: %true if the second leg is scheduled;</div><div class='add'>+ *  %false if the second leg is not scheduled.</div><div class='ctx'>  */</div><div class='ctx'> bool hfi1_schedule_tid_send(struct rvt_qp *qp)</div><div class='ctx'> {</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/tid_rdma.h b/drivers/infiniband/hw/hfi1/tid_rdma.h<br/>index 1c536185261e..6e82df2190b7 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/tid_rdma.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/tid_rdma.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/tid_rdma.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/tid_rdma.h</a></div><div class='hunk'>@@ -17,6 +17,7 @@</div><div class='ctx'> #define TID_RDMA_MIN_SEGMENT_SIZE       BIT(18)   /* 256 KiB (for now) */</div><div class='ctx'> #define TID_RDMA_MAX_SEGMENT_SIZE       BIT(18)   /* 256 KiB (for now) */</div><div class='ctx'> #define TID_RDMA_MAX_PAGES              (BIT(18) &gt;&gt; PAGE_SHIFT)</div><div class='add'>+#define TID_RDMA_SEGMENT_SHIFT		18</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * Bit definitions for priv-&gt;s_flags.</div><div class='hunk'>@@ -274,8 +275,6 @@ u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe,</div><div class='ctx'> 				  struct ib_other_headers *ohdr,</div><div class='ctx'> 				  u32 *bth1, u32 *bth2, u32 *len);</div><div class='ctx'> </div><div class='del'>-void hfi1_compute_tid_rdma_flow_wt(void);</div><div class='del'>-</div><div class='ctx'> void hfi1_rc_rcv_tid_rdma_write_req(struct hfi1_packet *packet);</div><div class='ctx'> </div><div class='ctx'> u32 hfi1_build_tid_rdma_write_resp(struct rvt_qp *qp, struct rvt_ack_entry *e,</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/trace.c b/drivers/infiniband/hw/hfi1/trace.c<br/>index 9a3d236bcc88..10290ebf76b2 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/trace.c</a></div><div class='hunk'>@@ -1,52 +1,11 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2015 - 2020 Intel Corporation.</div><div class='ctx'>  */</div><div class='ctx'> #define CREATE_TRACE_POINTS</div><div class='ctx'> #include "trace.h"</div><div class='ctx'> #include "exp_rcv.h"</div><div class='add'>+#include "ipoib.h"</div><div class='ctx'> </div><div class='ctx'> static u8 __get_ib_hdr_len(struct ib_header *hdr)</div><div class='ctx'> {</div><div class='hunk'>@@ -126,6 +85,7 @@ const char *hfi1_trace_get_packet_l2_str(u8 l2)</div><div class='ctx'> #define RETH_PRN "reth vaddr:0x%.16llx rkey:0x%.8x dlen:0x%.8x"</div><div class='ctx'> #define AETH_PRN "aeth syn:0x%.2x %s msn:0x%.8x"</div><div class='ctx'> #define DETH_PRN "deth qkey:0x%.8x sqpn:0x%.6x"</div><div class='add'>+#define DETH_ENTROPY_PRN "deth qkey:0x%.8x sqpn:0x%.6x entropy:0x%.2x"</div><div class='ctx'> #define IETH_PRN "ieth rkey:0x%.8x"</div><div class='ctx'> #define ATOMICACKETH_PRN "origdata:%llx"</div><div class='ctx'> #define ATOMICETH_PRN "vaddr:0x%llx rkey:0x%.8x sdata:%llx cdata:%llx"</div><div class='hunk'>@@ -187,6 +147,11 @@ void hfi1_trace_parse_16b_bth(struct ib_other_headers *ohdr,</div><div class='ctx'> 	*qpn = ib_bth_get_qpn(ohdr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static u16 ib_get_len(const struct ib_header *hdr)</div><div class='add'>+{</div><div class='add'>+	return be16_to_cpu(hdr-&gt;lrh[2]);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> void hfi1_trace_parse_9b_hdr(struct ib_header *hdr, bool sc5,</div><div class='ctx'> 			     u8 *lnh, u8 *lver, u8 *sl, u8 *sc,</div><div class='ctx'> 			     u16 *len, u32 *dlid, u32 *slid)</div><div class='hunk'>@@ -444,6 +409,12 @@ const char *parse_everbs_hdrs(</div><div class='ctx'> 		break;</div><div class='ctx'> 	/* deth */</div><div class='ctx'> 	case OP(UD, SEND_ONLY):</div><div class='add'>+		trace_seq_printf(p, DETH_ENTROPY_PRN,</div><div class='add'>+				 be32_to_cpu(eh-&gt;ud.deth[0]),</div><div class='add'>+				 be32_to_cpu(eh-&gt;ud.deth[1]) &amp; RVT_QPN_MASK,</div><div class='add'>+				 be32_to_cpu(eh-&gt;ud.deth[1]) &gt;&gt;</div><div class='add'>+					     HFI1_IPOIB_ENTROPY_SHIFT);</div><div class='add'>+		break;</div><div class='ctx'> 	case OP(UD, SEND_ONLY_WITH_IMMEDIATE):</div><div class='ctx'> 		trace_seq_printf(p, DETH_PRN,</div><div class='ctx'> 				 be32_to_cpu(eh-&gt;ud.deth[0]),</div><div class='hunk'>@@ -512,6 +483,38 @@ u16 hfi1_trace_get_tid_idx(u32 ent)</div><div class='ctx'> 	return EXP_TID_GET(ent, IDX);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+struct hfi1_ctxt_hist {</div><div class='add'>+	atomic_t count;</div><div class='add'>+	atomic_t data[255];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static struct hfi1_ctxt_hist hist = {</div><div class='add'>+	.count = ATOMIC_INIT(0)</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+const char *hfi1_trace_print_rsm_hist(struct trace_seq *p, unsigned int ctxt)</div><div class='add'>+{</div><div class='add'>+	int i, len = ARRAY_SIZE(hist.data);</div><div class='add'>+	const char *ret = trace_seq_buffer_ptr(p);</div><div class='add'>+	unsigned long packet_count = atomic_fetch_inc(&amp;hist.count);</div><div class='add'>+</div><div class='add'>+	trace_seq_printf(p, "packet[%lu]", packet_count);</div><div class='add'>+	for (i = 0; i &lt; len; ++i) {</div><div class='add'>+		unsigned long val;</div><div class='add'>+		atomic_t *count = &amp;hist.data[i];</div><div class='add'>+</div><div class='add'>+		if (ctxt == i)</div><div class='add'>+			val = atomic_fetch_inc(count);</div><div class='add'>+		else</div><div class='add'>+			val = atomic_read(count);</div><div class='add'>+</div><div class='add'>+		if (val)</div><div class='add'>+			trace_seq_printf(p, "(%d:%lu)", i, val);</div><div class='add'>+	}</div><div class='add'>+	trace_seq_putc(p, 0);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> __hfi1_trace_fn(AFFINITY);</div><div class='ctx'> __hfi1_trace_fn(PKT);</div><div class='ctx'> __hfi1_trace_fn(PROC);</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/trace.h b/drivers/infiniband/hw/hfi1/trace.h<br/>index 1ce551864118..bb3cc006bacd 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/trace.h</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #define packettype_name(etype) { RHF_RCV_TYPE_##etype, #etype }</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/trace_ctxts.h b/drivers/infiniband/hw/hfi1/trace_ctxts.h<br/>index e00c8a7d559c..76c41bd79071 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_ctxts.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_ctxts.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_ctxts.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/trace_ctxts.h</a></div><div class='hunk'>@@ -1,49 +1,8 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='del'>-* Copyright(c) 2015, 2016 Intel Corporation.</div><div class='del'>-*</div><div class='del'>-* This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>-* redistributing this file, you may do so under either license.</div><div class='del'>-*</div><div class='del'>-* GPL LICENSE SUMMARY</div><div class='del'>-*</div><div class='del'>-* This program is free software; you can redistribute it and/or modify</div><div class='del'>-* it under the terms of version 2 of the GNU General Public License as</div><div class='del'>-* published by the Free Software Foundation.</div><div class='del'>-*</div><div class='del'>-* This program is distributed in the hope that it will be useful, but</div><div class='del'>-* WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>-* General Public License for more details.</div><div class='del'>-*</div><div class='del'>-* BSD LICENSE</div><div class='del'>-*</div><div class='del'>-* Redistribution and use in source and binary forms, with or without</div><div class='del'>-* modification, are permitted provided that the following conditions</div><div class='del'>-* are met:</div><div class='del'>-*</div><div class='del'>-*  - Redistributions of source code must retain the above copyright</div><div class='del'>-*    notice, this list of conditions and the following disclaimer.</div><div class='del'>-*  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>-*    notice, this list of conditions and the following disclaimer in</div><div class='del'>-*    the documentation and/or other materials provided with the</div><div class='del'>-*    distribution.</div><div class='del'>-*  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>-*    contributors may be used to endorse or promote products derived</div><div class='del'>-*    from this software without specific prior written permission.</div><div class='del'>-*</div><div class='del'>-* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>-* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>-* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>-* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>-* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>-* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>-* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>-* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>-* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>-* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>-* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>-*</div><div class='add'>+* Copyright(c) 2015 - 2020 Intel Corporation.</div><div class='ctx'> */</div><div class='add'>+</div><div class='ctx'> #if !defined(__HFI1_TRACE_CTXTS_H) || defined(TRACE_HEADER_MULTI_READ)</div><div class='ctx'> #define __HFI1_TRACE_CTXTS_H</div><div class='ctx'> </div><div class='hunk'>@@ -80,7 +39,7 @@ TRACE_EVENT(hfi1_uctxtdata,</div><div class='ctx'> 			   __entry-&gt;credits = uctxt-&gt;sc-&gt;credits;</div><div class='ctx'> 			   __entry-&gt;hw_free = le64_to_cpu(*uctxt-&gt;sc-&gt;hw_free);</div><div class='ctx'> 			   __entry-&gt;piobase = uctxt-&gt;sc-&gt;base_addr;</div><div class='del'>-			   __entry-&gt;rcvhdrq_cnt = uctxt-&gt;rcvhdrq_cnt;</div><div class='add'>+			   __entry-&gt;rcvhdrq_cnt = get_hdrq_cnt(uctxt);</div><div class='ctx'> 			   __entry-&gt;rcvhdrq_dma = uctxt-&gt;rcvhdrq_dma;</div><div class='ctx'> 			   __entry-&gt;eager_cnt = uctxt-&gt;egrbufs.alloced;</div><div class='ctx'> 			   __entry-&gt;rcvegr_dma = uctxt-&gt;egrbufs.rcvtids[0].dma;</div><div class='hunk'>@@ -138,6 +97,15 @@ TRACE_EVENT(hfi1_ctxt_info,</div><div class='ctx'> 		      )</div><div class='ctx'> );</div><div class='ctx'> </div><div class='add'>+const char *hfi1_trace_print_rsm_hist(struct trace_seq *p, unsigned int ctxt);</div><div class='add'>+TRACE_EVENT(ctxt_rsm_hist,</div><div class='add'>+	    TP_PROTO(unsigned int ctxt),</div><div class='add'>+	    TP_ARGS(ctxt),</div><div class='add'>+	    TP_STRUCT__entry(__field(unsigned int, ctxt)),</div><div class='add'>+	    TP_fast_assign(__entry-&gt;ctxt = ctxt;),</div><div class='add'>+	    TP_printk("%s", hfi1_trace_print_rsm_hist(p, __entry-&gt;ctxt))</div><div class='add'>+);</div><div class='add'>+</div><div class='ctx'> #endif /* __HFI1_TRACE_CTXTS_H */</div><div class='ctx'> </div><div class='ctx'> #undef TRACE_INCLUDE_PATH</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/trace_dbg.h b/drivers/infiniband/hw/hfi1/trace_dbg.h<br/>index de7a87392b8d..58304b91380f 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_dbg.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_dbg.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_dbg.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/trace_dbg.h</a></div><div class='hunk'>@@ -1,49 +1,8 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'> * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>-*</div><div class='del'>-* This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>-* redistributing this file, you may do so under either license.</div><div class='del'>-*</div><div class='del'>-* GPL LICENSE SUMMARY</div><div class='del'>-*</div><div class='del'>-* This program is free software; you can redistribute it and/or modify</div><div class='del'>-* it under the terms of version 2 of the GNU General Public License as</div><div class='del'>-* published by the Free Software Foundation.</div><div class='del'>-*</div><div class='del'>-* This program is distributed in the hope that it will be useful, but</div><div class='del'>-* WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>-* General Public License for more details.</div><div class='del'>-*</div><div class='del'>-* BSD LICENSE</div><div class='del'>-*</div><div class='del'>-* Redistribution and use in source and binary forms, with or without</div><div class='del'>-* modification, are permitted provided that the following conditions</div><div class='del'>-* are met:</div><div class='del'>-*</div><div class='del'>-*  - Redistributions of source code must retain the above copyright</div><div class='del'>-*    notice, this list of conditions and the following disclaimer.</div><div class='del'>-*  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>-*    notice, this list of conditions and the following disclaimer in</div><div class='del'>-*    the documentation and/or other materials provided with the</div><div class='del'>-*    distribution.</div><div class='del'>-*  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>-*    contributors may be used to endorse or promote products derived</div><div class='del'>-*    from this software without specific prior written permission.</div><div class='del'>-*</div><div class='del'>-* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>-* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>-* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>-* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>-* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>-* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>-* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>-* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>-* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>-* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>-* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>-*</div><div class='ctx'> */</div><div class='add'>+</div><div class='ctx'> #if !defined(__HFI1_TRACE_EXTRA_H) || defined(TRACE_HEADER_MULTI_READ)</div><div class='ctx'> #define __HFI1_TRACE_EXTRA_H</div><div class='ctx'> </div><div class='hunk'>@@ -63,24 +22,27 @@</div><div class='ctx'> </div><div class='ctx'> #define MAX_MSG_LEN 512</div><div class='ctx'> </div><div class='add'>+#pragma GCC diagnostic push</div><div class='add'>+#ifndef __clang__</div><div class='add'>+#pragma GCC diagnostic ignored "-Wsuggest-attribute=format"</div><div class='add'>+#endif</div><div class='add'>+</div><div class='ctx'> DECLARE_EVENT_CLASS(hfi1_trace_template,</div><div class='ctx'> 		    TP_PROTO(const char *function, struct va_format *vaf),</div><div class='ctx'> 		    TP_ARGS(function, vaf),</div><div class='ctx'> 		    TP_STRUCT__entry(__string(function, function)</div><div class='del'>-				     __dynamic_array(char, msg, MAX_MSG_LEN)</div><div class='add'>+				     __vstring(msg, vaf-&gt;fmt, vaf-&gt;va)</div><div class='ctx'> 				     ),</div><div class='del'>-		    TP_fast_assign(__assign_str(function, function);</div><div class='del'>-				   WARN_ON_ONCE(vsnprintf</div><div class='del'>-						(__get_dynamic_array(msg),</div><div class='del'>-						 MAX_MSG_LEN, vaf-&gt;fmt,</div><div class='del'>-						 *vaf-&gt;va) &gt;=</div><div class='del'>-						MAX_MSG_LEN);</div><div class='add'>+		    TP_fast_assign(__assign_str(function);</div><div class='add'>+				   __assign_vstr(msg, vaf-&gt;fmt, vaf-&gt;va);</div><div class='ctx'> 				   ),</div><div class='ctx'> 		    TP_printk("(%s) %s",</div><div class='ctx'> 			      __get_str(function),</div><div class='ctx'> 			      __get_str(msg))</div><div class='ctx'> );</div><div class='ctx'> </div><div class='add'>+#pragma GCC diagnostic pop</div><div class='add'>+</div><div class='ctx'> /*</div><div class='ctx'>  * It may be nice to macroize the __hfi1_trace but the va_* stuff requires an</div><div class='ctx'>  * actual function to work and can not be in a macro.</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h<br/>index d1372cc66de6..b21356abc9ec 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_ibhdrs.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_ibhdrs.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_ibhdrs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/trace_ibhdrs.h</a></div><div class='hunk'>@@ -1,49 +1,8 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015 - 2017 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #if !defined(__HFI1_TRACE_IBHDRS_H) || defined(TRACE_HEADER_MULTI_READ)</div><div class='ctx'> #define __HFI1_TRACE_IBHDRS_H</div><div class='ctx'> </div><div class='hunk'>@@ -79,6 +38,8 @@ __print_symbolic(opcode,                                   \</div><div class='ctx'> 	ib_opcode_name(RC_ATOMIC_ACKNOWLEDGE),             \</div><div class='ctx'> 	ib_opcode_name(RC_COMPARE_SWAP),                   \</div><div class='ctx'> 	ib_opcode_name(RC_FETCH_ADD),                      \</div><div class='add'>+	ib_opcode_name(RC_SEND_LAST_WITH_INVALIDATE),      \</div><div class='add'>+	ib_opcode_name(RC_SEND_ONLY_WITH_INVALIDATE),      \</div><div class='ctx'> 	ib_opcode_name(TID_RDMA_WRITE_REQ),	           \</div><div class='ctx'> 	ib_opcode_name(TID_RDMA_WRITE_RESP),	           \</div><div class='ctx'> 	ib_opcode_name(TID_RDMA_WRITE_DATA),	           \</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/trace_misc.h b/drivers/infiniband/hw/hfi1/trace_misc.h<br/>index 8db2253523ff..8dc46b6891df 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_misc.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_misc.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_misc.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/trace_misc.h</a></div><div class='hunk'>@@ -1,49 +1,8 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'> * Copyright(c) 2015, 2016 Intel Corporation.</div><div class='del'>-*</div><div class='del'>-* This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>-* redistributing this file, you may do so under either license.</div><div class='del'>-*</div><div class='del'>-* GPL LICENSE SUMMARY</div><div class='del'>-*</div><div class='del'>-* This program is free software; you can redistribute it and/or modify</div><div class='del'>-* it under the terms of version 2 of the GNU General Public License as</div><div class='del'>-* published by the Free Software Foundation.</div><div class='del'>-*</div><div class='del'>-* This program is distributed in the hope that it will be useful, but</div><div class='del'>-* WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>-* General Public License for more details.</div><div class='del'>-*</div><div class='del'>-* BSD LICENSE</div><div class='del'>-*</div><div class='del'>-* Redistribution and use in source and binary forms, with or without</div><div class='del'>-* modification, are permitted provided that the following conditions</div><div class='del'>-* are met:</div><div class='del'>-*</div><div class='del'>-*  - Redistributions of source code must retain the above copyright</div><div class='del'>-*    notice, this list of conditions and the following disclaimer.</div><div class='del'>-*  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>-*    notice, this list of conditions and the following disclaimer in</div><div class='del'>-*    the documentation and/or other materials provided with the</div><div class='del'>-*    distribution.</div><div class='del'>-*  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>-*    contributors may be used to endorse or promote products derived</div><div class='del'>-*    from this software without specific prior written permission.</div><div class='del'>-*</div><div class='del'>-* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>-* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>-* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>-* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>-* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>-* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>-* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>-* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>-* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>-* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>-* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>-*</div><div class='ctx'> */</div><div class='add'>+</div><div class='ctx'> #if !defined(__HFI1_TRACE_MISC_H) || defined(TRACE_HEADER_MULTI_READ)</div><div class='ctx'> #define __HFI1_TRACE_MISC_H</div><div class='ctx'> </div><div class='hunk'>@@ -63,7 +22,7 @@ TRACE_EVENT(hfi1_interrupt,</div><div class='ctx'> 			     __array(char, buf, 64)</div><div class='ctx'> 			     __field(int, src)</div><div class='ctx'> 			     ),</div><div class='del'>-	    TP_fast_assign(DD_DEV_ASSIGN(dd)</div><div class='add'>+	    TP_fast_assign(DD_DEV_ASSIGN(dd);</div><div class='ctx'> 			   is_entry-&gt;is_name(__entry-&gt;buf, 64,</div><div class='ctx'> 					     src - is_entry-&gt;start);</div><div class='ctx'> 			   __entry-&gt;src = src;</div><div class='hunk'>@@ -100,7 +59,7 @@ TRACE_EVENT(hfi1_fault_opcode,</div><div class='ctx'> 			     __field(u32, qpn)</div><div class='ctx'> 			     __field(u8, opcode)</div><div class='ctx'> 			     ),</div><div class='del'>-	    TP_fast_assign(DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device))</div><div class='add'>+	    TP_fast_assign(DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device));</div><div class='ctx'> 			   __entry-&gt;qpn = qp-&gt;ibqp.qp_num;</div><div class='ctx'> 			   __entry-&gt;opcode = opcode;</div><div class='ctx'> 			   ),</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/trace_mmu.h b/drivers/infiniband/hw/hfi1/trace_mmu.h<br/>index 3b7abbc382c2..5a9dfd85e7f5 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_mmu.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_mmu.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_mmu.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/trace_mmu.h</a></div><div class='hunk'>@@ -1,49 +1,8 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2017 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #if !defined(__HFI1_TRACE_MMU_H) || defined(TRACE_HEADER_MULTI_READ)</div><div class='ctx'> #define __HFI1_TRACE_MMU_H</div><div class='ctx'> </div><div class='hunk'>@@ -56,35 +15,53 @@</div><div class='ctx'> #define TRACE_SYSTEM hfi1_mmu</div><div class='ctx'> </div><div class='ctx'> DECLARE_EVENT_CLASS(hfi1_mmu_rb_template,</div><div class='del'>-		    TP_PROTO(unsigned long addr, unsigned long len),</div><div class='del'>-		    TP_ARGS(addr, len),</div><div class='add'>+		    TP_PROTO(struct mmu_rb_node *node),</div><div class='add'>+		    TP_ARGS(node),</div><div class='ctx'> 		    TP_STRUCT__entry(__field(unsigned long, addr)</div><div class='ctx'> 				     __field(unsigned long, len)</div><div class='add'>+				     __field(unsigned int, refcount)</div><div class='ctx'> 			    ),</div><div class='del'>-		    TP_fast_assign(__entry-&gt;addr = addr;</div><div class='del'>-				   __entry-&gt;len = len;</div><div class='add'>+		    TP_fast_assign(__entry-&gt;addr = node-&gt;addr;</div><div class='add'>+				   __entry-&gt;len = node-&gt;len;</div><div class='add'>+				   __entry-&gt;refcount = kref_read(&amp;node-&gt;refcount);</div><div class='ctx'> 			    ),</div><div class='del'>-		    TP_printk("MMU node addr 0x%lx, len %lu",</div><div class='add'>+		    TP_printk("MMU node addr 0x%lx, len %lu, refcount %u",</div><div class='ctx'> 			      __entry-&gt;addr,</div><div class='del'>-			      __entry-&gt;len</div><div class='add'>+			      __entry-&gt;len,</div><div class='add'>+			      __entry-&gt;refcount</div><div class='ctx'> 			    )</div><div class='ctx'> );</div><div class='ctx'> </div><div class='ctx'> DEFINE_EVENT(hfi1_mmu_rb_template, hfi1_mmu_rb_insert,</div><div class='del'>-	     TP_PROTO(unsigned long addr, unsigned long len),</div><div class='del'>-	     TP_ARGS(addr, len));</div><div class='del'>-</div><div class='del'>-DEFINE_EVENT(hfi1_mmu_rb_template, hfi1_mmu_rb_search,</div><div class='del'>-	     TP_PROTO(unsigned long addr, unsigned long len),</div><div class='del'>-	     TP_ARGS(addr, len));</div><div class='add'>+	     TP_PROTO(struct mmu_rb_node *node),</div><div class='add'>+	     TP_ARGS(node));</div><div class='ctx'> </div><div class='del'>-DEFINE_EVENT(hfi1_mmu_rb_template, hfi1_mmu_rb_remove,</div><div class='del'>-	     TP_PROTO(unsigned long addr, unsigned long len),</div><div class='del'>-	     TP_ARGS(addr, len));</div><div class='add'>+TRACE_EVENT(hfi1_mmu_rb_search,</div><div class='add'>+	    TP_PROTO(unsigned long addr, unsigned long len),</div><div class='add'>+	    TP_ARGS(addr, len),</div><div class='add'>+	    TP_STRUCT__entry(__field(unsigned long, addr)</div><div class='add'>+			     __field(unsigned long, len)</div><div class='add'>+		    ),</div><div class='add'>+	    TP_fast_assign(__entry-&gt;addr = addr;</div><div class='add'>+			   __entry-&gt;len = len;</div><div class='add'>+		    ),</div><div class='add'>+	    TP_printk("MMU node addr 0x%lx, len %lu",</div><div class='add'>+		      __entry-&gt;addr,</div><div class='add'>+		      __entry-&gt;len</div><div class='add'>+		    )</div><div class='add'>+);</div><div class='ctx'> </div><div class='ctx'> DEFINE_EVENT(hfi1_mmu_rb_template, hfi1_mmu_mem_invalidate,</div><div class='del'>-	     TP_PROTO(unsigned long addr, unsigned long len),</div><div class='del'>-	     TP_ARGS(addr, len));</div><div class='add'>+	     TP_PROTO(struct mmu_rb_node *node),</div><div class='add'>+	     TP_ARGS(node));</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(hfi1_mmu_rb_template, hfi1_mmu_rb_evict,</div><div class='add'>+	     TP_PROTO(struct mmu_rb_node *node),</div><div class='add'>+	     TP_ARGS(node));</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(hfi1_mmu_rb_template, hfi1_mmu_release_node,</div><div class='add'>+	     TP_PROTO(struct mmu_rb_node *node),</div><div class='add'>+	     TP_ARGS(node));</div><div class='ctx'> </div><div class='ctx'> #endif /* __HFI1_TRACE_RC_H */</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/trace_rc.h b/drivers/infiniband/hw/hfi1/trace_rc.h<br/>index 1ebca37862e0..fa254f9b9c42 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_rc.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_rc.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_rc.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/trace_rc.h</a></div><div class='hunk'>@@ -1,49 +1,8 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'> * Copyright(c) 2015, 2016, 2017 Intel Corporation.</div><div class='del'>-*</div><div class='del'>-* This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>-* redistributing this file, you may do so under either license.</div><div class='del'>-*</div><div class='del'>-* GPL LICENSE SUMMARY</div><div class='del'>-*</div><div class='del'>-* This program is free software; you can redistribute it and/or modify</div><div class='del'>-* it under the terms of version 2 of the GNU General Public License as</div><div class='del'>-* published by the Free Software Foundation.</div><div class='del'>-*</div><div class='del'>-* This program is distributed in the hope that it will be useful, but</div><div class='del'>-* WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>-* General Public License for more details.</div><div class='del'>-*</div><div class='del'>-* BSD LICENSE</div><div class='del'>-*</div><div class='del'>-* Redistribution and use in source and binary forms, with or without</div><div class='del'>-* modification, are permitted provided that the following conditions</div><div class='del'>-* are met:</div><div class='del'>-*</div><div class='del'>-*  - Redistributions of source code must retain the above copyright</div><div class='del'>-*    notice, this list of conditions and the following disclaimer.</div><div class='del'>-*  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>-*    notice, this list of conditions and the following disclaimer in</div><div class='del'>-*    the documentation and/or other materials provided with the</div><div class='del'>-*    distribution.</div><div class='del'>-*  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>-*    contributors may be used to endorse or promote products derived</div><div class='del'>-*    from this software without specific prior written permission.</div><div class='del'>-*</div><div class='del'>-* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>-* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>-* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>-* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>-* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>-* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>-* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>-* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>-* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>-* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>-* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>-*</div><div class='ctx'> */</div><div class='add'>+</div><div class='ctx'> #if !defined(__HFI1_TRACE_RC_H) || defined(TRACE_HEADER_MULTI_READ)</div><div class='ctx'> #define __HFI1_TRACE_RC_H</div><div class='ctx'> </div><div class='hunk'>@@ -70,7 +29,7 @@ DECLARE_EVENT_CLASS(hfi1_rc_template,</div><div class='ctx'> 			__field(u32, r_psn)</div><div class='ctx'> 			),</div><div class='ctx'> 		    TP_fast_assign(</div><div class='del'>-			DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device))</div><div class='add'>+			DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device));</div><div class='ctx'> 			__entry-&gt;qpn = qp-&gt;ibqp.qp_num;</div><div class='ctx'> 			__entry-&gt;s_flags = qp-&gt;s_flags;</div><div class='ctx'> 			__entry-&gt;psn = psn;</div><div class='hunk'>@@ -130,7 +89,7 @@ DECLARE_EVENT_CLASS(/* rc_ack */</div><div class='ctx'> 		__field(u32, lpsn)</div><div class='ctx'> 	),</div><div class='ctx'> 	TP_fast_assign(/* assign */</div><div class='del'>-		DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device))</div><div class='add'>+		DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device));</div><div class='ctx'> 		__entry-&gt;qpn = qp-&gt;ibqp.qp_num;</div><div class='ctx'> 		__entry-&gt;aeth = aeth;</div><div class='ctx'> 		__entry-&gt;psn = psn;</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/trace_rx.h b/drivers/infiniband/hw/hfi1/trace_rx.h<br/>index 3cec960e9674..8d5e12fe88a5 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_rx.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_rx.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_rx.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/trace_rx.h</a></div><div class='hunk'>@@ -1,49 +1,8 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #if !defined(__HFI1_TRACE_RX_H) || defined(TRACE_HEADER_MULTI_READ)</div><div class='ctx'> #define __HFI1_TRACE_RX_H</div><div class='ctx'> </div><div class='hunk'>@@ -106,19 +65,8 @@ TRACE_EVENT(hfi1_receive_interrupt,</div><div class='ctx'> 			     ),</div><div class='ctx'> 	    TP_fast_assign(DD_DEV_ASSIGN(dd);</div><div class='ctx'> 			__entry-&gt;ctxt = rcd-&gt;ctxt;</div><div class='del'>-			if (rcd-&gt;do_interrupt ==</div><div class='del'>-			    &amp;handle_receive_interrupt) {</div><div class='del'>-				__entry-&gt;slow_path = 1;</div><div class='del'>-				__entry-&gt;dma_rtail = 0xFF;</div><div class='del'>-			} else if (rcd-&gt;do_interrupt ==</div><div class='del'>-					&amp;handle_receive_interrupt_dma_rtail){</div><div class='del'>-				__entry-&gt;dma_rtail = 1;</div><div class='del'>-				__entry-&gt;slow_path = 0;</div><div class='del'>-			} else if (rcd-&gt;do_interrupt ==</div><div class='del'>-					&amp;handle_receive_interrupt_nodma_rtail) {</div><div class='del'>-				__entry-&gt;dma_rtail = 0;</div><div class='del'>-				__entry-&gt;slow_path = 0;</div><div class='del'>-			}</div><div class='add'>+			__entry-&gt;slow_path = hfi1_is_slowpath(rcd);</div><div class='add'>+			__entry-&gt;dma_rtail = get_dma_rtail_setting(rcd);</div><div class='ctx'> 			),</div><div class='ctx'> 	    TP_printk("[%s] ctxt %d SlowPath: %d DmaRtail: %d",</div><div class='ctx'> 		      __get_str(dev),</div><div class='hunk'>@@ -142,7 +90,7 @@ TRACE_EVENT(hfi1_mmu_invalidate,</div><div class='ctx'> 	    TP_fast_assign(</div><div class='ctx'> 			__entry-&gt;ctxt = ctxt;</div><div class='ctx'> 			__entry-&gt;subctxt = subctxt;</div><div class='del'>-			__assign_str(type, type);</div><div class='add'>+			__assign_str(type);</div><div class='ctx'> 			__entry-&gt;start = start;</div><div class='ctx'> 			__entry-&gt;end = end;</div><div class='ctx'> 	    ),</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/trace_tid.h b/drivers/infiniband/hw/hfi1/trace_tid.h<br/>index 4388b594ed1b..e358f5b885fa 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_tid.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_tid.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_tid.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/trace_tid.h</a></div><div class='hunk'>@@ -138,10 +138,10 @@ TRACE_EVENT(/* put_tid */</div><div class='ctx'> 	TP_ARGS(dd, index, type, pa, order),</div><div class='ctx'> 	TP_STRUCT__entry(/* entry */</div><div class='ctx'> 		DD_DEV_ENTRY(dd)</div><div class='del'>-		__field(unsigned long, pa);</div><div class='del'>-		__field(u32, index);</div><div class='del'>-		__field(u32, type);</div><div class='del'>-		__field(u16, order);</div><div class='add'>+		__field(unsigned long, pa)</div><div class='add'>+		__field(u32, index)</div><div class='add'>+		__field(u32, type)</div><div class='add'>+		__field(u16, order)</div><div class='ctx'> 	),</div><div class='ctx'> 	TP_fast_assign(/* assign */</div><div class='ctx'> 		DD_DEV_ASSIGN(dd);</div><div class='hunk'>@@ -358,7 +358,7 @@ DECLARE_EVENT_CLASS(/* msg */</div><div class='ctx'> 	),</div><div class='ctx'> 	TP_fast_assign(/* assign */</div><div class='ctx'> 		__entry-&gt;qpn = qp ? qp-&gt;ibqp.qp_num : 0;</div><div class='del'>-		__assign_str(msg, msg);</div><div class='add'>+		__assign_str(msg);</div><div class='ctx'> 		__entry-&gt;more = more;</div><div class='ctx'> 	),</div><div class='ctx'> 	TP_printk(/* print */</div><div class='hunk'>@@ -627,6 +627,12 @@ DEFINE_EVENT(/* event */</div><div class='ctx'> 	TP_ARGS(qp, index, flow)</div><div class='ctx'> );</div><div class='ctx'> </div><div class='add'>+DEFINE_EVENT(/* event */</div><div class='add'>+	hfi1_tid_flow_template, hfi1_tid_flow_read_kdeth_eflags,</div><div class='add'>+	TP_PROTO(struct rvt_qp *qp, int index, struct tid_rdma_flow *flow),</div><div class='add'>+	TP_ARGS(qp, index, flow)</div><div class='add'>+);</div><div class='add'>+</div><div class='ctx'> DECLARE_EVENT_CLASS(/* tid_node */</div><div class='ctx'> 	hfi1_tid_node_template,</div><div class='ctx'> 	TP_PROTO(struct rvt_qp *qp, const char *msg, u32 index, u32 base,</div><div class='hunk'>@@ -645,7 +651,7 @@ DECLARE_EVENT_CLASS(/* tid_node */</div><div class='ctx'> 	TP_fast_assign(/* assign */</div><div class='ctx'> 		DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device));</div><div class='ctx'> 		__entry-&gt;qpn = qp-&gt;ibqp.qp_num;</div><div class='del'>-		__assign_str(msg, msg);</div><div class='add'>+		__assign_str(msg);</div><div class='ctx'> 		__entry-&gt;index = index;</div><div class='ctx'> 		__entry-&gt;base = base;</div><div class='ctx'> 		__entry-&gt;map = map;</div><div class='hunk'>@@ -851,6 +857,12 @@ DEFINE_EVENT(/* event */</div><div class='ctx'> 	TP_ARGS(qp, psn)</div><div class='ctx'> );</div><div class='ctx'> </div><div class='add'>+DEFINE_EVENT(/* event */</div><div class='add'>+	hfi1_responder_info_template, hfi1_rsp_read_kdeth_eflags,</div><div class='add'>+	TP_PROTO(struct rvt_qp *qp, u32 psn),</div><div class='add'>+	TP_ARGS(qp, psn)</div><div class='add'>+);</div><div class='add'>+</div><div class='ctx'> DECLARE_EVENT_CLASS(/* sender_info */</div><div class='ctx'> 	hfi1_sender_info_template,</div><div class='ctx'> 	TP_PROTO(struct rvt_qp *qp),</div><div class='hunk'>@@ -874,7 +886,7 @@ DECLARE_EVENT_CLASS(/* sender_info */</div><div class='ctx'> 		__field(u8, s_retry)</div><div class='ctx'> 	),</div><div class='ctx'> 	TP_fast_assign(/* assign */</div><div class='del'>-		DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device))</div><div class='add'>+		DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device));</div><div class='ctx'> 		__entry-&gt;qpn = qp-&gt;ibqp.qp_num;</div><div class='ctx'> 		__entry-&gt;state = qp-&gt;state;</div><div class='ctx'> 		__entry-&gt;s_cur = qp-&gt;s_cur;</div><div class='hunk'>@@ -955,6 +967,12 @@ DEFINE_EVENT(/* event */</div><div class='ctx'> 	TP_ARGS(qp)</div><div class='ctx'> );</div><div class='ctx'> </div><div class='add'>+DEFINE_EVENT(/* event */</div><div class='add'>+	hfi1_sender_info_template, hfi1_sender_read_kdeth_eflags,</div><div class='add'>+	TP_PROTO(struct rvt_qp *qp),</div><div class='add'>+	TP_ARGS(qp)</div><div class='add'>+);</div><div class='add'>+</div><div class='ctx'> DECLARE_EVENT_CLASS(/* tid_read_sender */</div><div class='ctx'> 	hfi1_tid_read_sender_template,</div><div class='ctx'> 	TP_PROTO(struct rvt_qp *qp, char newreq),</div><div class='hunk'>@@ -1015,6 +1033,12 @@ DEFINE_EVENT(/* event */</div><div class='ctx'> 	TP_ARGS(qp, newreq)</div><div class='ctx'> );</div><div class='ctx'> </div><div class='add'>+DEFINE_EVENT(/* event */</div><div class='add'>+	hfi1_tid_read_sender_template, hfi1_tid_read_sender_kdeth_eflags,</div><div class='add'>+	TP_PROTO(struct rvt_qp *qp, char newreq),</div><div class='add'>+	TP_ARGS(qp, newreq)</div><div class='add'>+);</div><div class='add'>+</div><div class='ctx'> DECLARE_EVENT_CLASS(/* tid_rdma_request */</div><div class='ctx'> 	hfi1_tid_rdma_request_template,</div><div class='ctx'> 	TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn,</div><div class='hunk'>@@ -1216,6 +1240,13 @@ DEFINE_EVENT(/* event */</div><div class='ctx'> );</div><div class='ctx'> </div><div class='ctx'> DEFINE_EVENT(/* event */</div><div class='add'>+	hfi1_tid_rdma_request_template, hfi1_tid_req_read_kdeth_eflags,</div><div class='add'>+	TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn,</div><div class='add'>+		 struct tid_rdma_request *req),</div><div class='add'>+	TP_ARGS(qp, newreq, opcode, psn, lpsn, req)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(/* event */</div><div class='ctx'> 	hfi1_tid_rdma_request_template, hfi1_tid_req_make_rc_ack_write,</div><div class='ctx'> 	TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn,</div><div class='ctx'> 		 struct tid_rdma_request *req),</div><div class='hunk'>@@ -1229,6 +1260,13 @@ DEFINE_EVENT(/* event */</div><div class='ctx'> 	TP_ARGS(qp, newreq, opcode, psn, lpsn, req)</div><div class='ctx'> );</div><div class='ctx'> </div><div class='add'>+DEFINE_EVENT(/* event */</div><div class='add'>+	hfi1_tid_rdma_request_template, hfi1_tid_req_update_num_rd_atomic,</div><div class='add'>+	TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn,</div><div class='add'>+		 struct tid_rdma_request *req),</div><div class='add'>+	TP_ARGS(qp, newreq, opcode, psn, lpsn, req)</div><div class='add'>+);</div><div class='add'>+</div><div class='ctx'> DECLARE_EVENT_CLASS(/* rc_rcv_err */</div><div class='ctx'> 	hfi1_rc_rcv_err_template,</div><div class='ctx'> 	TP_PROTO(struct rvt_qp *qp, u32 opcode, u32 psn, int diff),</div><div class='hunk'>@@ -1247,7 +1285,7 @@ DECLARE_EVENT_CLASS(/* rc_rcv_err */</div><div class='ctx'> 		__field(int, diff)</div><div class='ctx'> 	),</div><div class='ctx'> 	TP_fast_assign(/* assign */</div><div class='del'>-		DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device))</div><div class='add'>+		DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device));</div><div class='ctx'> 		__entry-&gt;qpn = qp-&gt;ibqp.qp_num;</div><div class='ctx'> 		__entry-&gt;s_flags = qp-&gt;s_flags;</div><div class='ctx'> 		__entry-&gt;state = qp-&gt;state;</div><div class='hunk'>@@ -1536,7 +1574,7 @@ DECLARE_EVENT_CLASS(/* tid_ack */</div><div class='ctx'> 		__field(u32, resync_psn)</div><div class='ctx'> 	),</div><div class='ctx'> 	TP_fast_assign(/* assign */</div><div class='del'>-		DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device))</div><div class='add'>+		DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device));</div><div class='ctx'> 		__entry-&gt;qpn = qp-&gt;ibqp.qp_num;</div><div class='ctx'> 		__entry-&gt;aeth = aeth;</div><div class='ctx'> 		__entry-&gt;psn = psn;</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/trace_tx.h b/drivers/infiniband/hw/hfi1/trace_tx.h<br/>index 09eb0c9ada00..c0ba6b0a2c4e 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_tx.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/trace_tx.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/trace_tx.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/trace_tx.h</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015 - 2017 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> #if !defined(__HFI1_TRACE_TX_H) || defined(TRACE_HEADER_MULTI_READ)</div><div class='ctx'> #define __HFI1_TRACE_TX_H</div><div class='hunk'>@@ -53,6 +11,8 @@</div><div class='ctx'> #include "hfi.h"</div><div class='ctx'> #include "mad.h"</div><div class='ctx'> #include "sdma.h"</div><div class='add'>+#include "ipoib.h"</div><div class='add'>+#include "user_sdma.h"</div><div class='ctx'> </div><div class='ctx'> const char *parse_sdma_flags(struct trace_seq *p, u64 desc0, u64 desc1);</div><div class='ctx'> </div><div class='hunk'>@@ -118,7 +78,7 @@ DECLARE_EVENT_CLASS(hfi1_qpsleepwakeup_template,</div><div class='ctx'> 		    __field(unsigned long, iow_flags)</div><div class='ctx'> 		    ),</div><div class='ctx'> 		    TP_fast_assign(</div><div class='del'>-		    DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device))</div><div class='add'>+		    DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device));</div><div class='ctx'> 		    __entry-&gt;flags = flags;</div><div class='ctx'> 		    __entry-&gt;qpn = qp-&gt;ibqp.qp_num;</div><div class='ctx'> 		    __entry-&gt;s_flags = qp-&gt;s_flags;</div><div class='hunk'>@@ -588,7 +548,7 @@ TRACE_EVENT(hfi1_sdma_user_reqinfo,</div><div class='ctx'> 	    TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 *i),</div><div class='ctx'> 	    TP_ARGS(dd, ctxt, subctxt, i),</div><div class='ctx'> 	    TP_STRUCT__entry(</div><div class='del'>-		    DD_DEV_ENTRY(dd);</div><div class='add'>+		    DD_DEV_ENTRY(dd)</div><div class='ctx'> 		    __field(u16, ctxt)</div><div class='ctx'> 		    __field(u8, subctxt)</div><div class='ctx'> 		    __field(u8, ver_opcode)</div><div class='hunk'>@@ -653,6 +613,80 @@ TRACE_EVENT(hfi1_sdma_user_completion,</div><div class='ctx'> 		      __entry-&gt;code)</div><div class='ctx'> );</div><div class='ctx'> </div><div class='add'>+TRACE_EVENT(hfi1_usdma_defer,</div><div class='add'>+	    TP_PROTO(struct hfi1_user_sdma_pkt_q *pq,</div><div class='add'>+		     struct sdma_engine *sde,</div><div class='add'>+		     struct iowait *wait),</div><div class='add'>+	    TP_ARGS(pq, sde, wait),</div><div class='add'>+	    TP_STRUCT__entry(DD_DEV_ENTRY(pq-&gt;dd)</div><div class='add'>+			     __field(struct hfi1_user_sdma_pkt_q *, pq)</div><div class='add'>+			     __field(struct sdma_engine *, sde)</div><div class='add'>+			     __field(struct iowait *, wait)</div><div class='add'>+			     __field(int, engine)</div><div class='add'>+			     __field(int, empty)</div><div class='add'>+			     ),</div><div class='add'>+	     TP_fast_assign(DD_DEV_ASSIGN(pq-&gt;dd);</div><div class='add'>+			    __entry-&gt;pq = pq;</div><div class='add'>+			    __entry-&gt;sde = sde;</div><div class='add'>+			    __entry-&gt;wait = wait;</div><div class='add'>+			    __entry-&gt;engine = sde-&gt;this_idx;</div><div class='add'>+			    __entry-&gt;empty = list_empty(&amp;__entry-&gt;wait-&gt;list);</div><div class='add'>+			    ),</div><div class='add'>+	     TP_printk("[%s] pq %llx sde %llx wait %llx engine %d empty %d",</div><div class='add'>+		       __get_str(dev),</div><div class='add'>+		       (unsigned long long)__entry-&gt;pq,</div><div class='add'>+		       (unsigned long long)__entry-&gt;sde,</div><div class='add'>+		       (unsigned long long)__entry-&gt;wait,</div><div class='add'>+		       __entry-&gt;engine,</div><div class='add'>+		       __entry-&gt;empty</div><div class='add'>+		)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(hfi1_usdma_activate,</div><div class='add'>+	    TP_PROTO(struct hfi1_user_sdma_pkt_q *pq,</div><div class='add'>+		     struct iowait *wait,</div><div class='add'>+		     int reason),</div><div class='add'>+	    TP_ARGS(pq, wait, reason),</div><div class='add'>+	    TP_STRUCT__entry(DD_DEV_ENTRY(pq-&gt;dd)</div><div class='add'>+			     __field(struct hfi1_user_sdma_pkt_q *, pq)</div><div class='add'>+			     __field(struct iowait *, wait)</div><div class='add'>+			     __field(int, reason)</div><div class='add'>+			     ),</div><div class='add'>+	     TP_fast_assign(DD_DEV_ASSIGN(pq-&gt;dd);</div><div class='add'>+			    __entry-&gt;pq = pq;</div><div class='add'>+			    __entry-&gt;wait = wait;</div><div class='add'>+			    __entry-&gt;reason = reason;</div><div class='add'>+			    ),</div><div class='add'>+	     TP_printk("[%s] pq %llx wait %llx reason %d",</div><div class='add'>+		       __get_str(dev),</div><div class='add'>+		       (unsigned long long)__entry-&gt;pq,</div><div class='add'>+		       (unsigned long long)__entry-&gt;wait,</div><div class='add'>+		       __entry-&gt;reason</div><div class='add'>+		)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(hfi1_usdma_we,</div><div class='add'>+	    TP_PROTO(struct hfi1_user_sdma_pkt_q *pq,</div><div class='add'>+		     int we_ret),</div><div class='add'>+	    TP_ARGS(pq, we_ret),</div><div class='add'>+	    TP_STRUCT__entry(DD_DEV_ENTRY(pq-&gt;dd)</div><div class='add'>+			     __field(struct hfi1_user_sdma_pkt_q *, pq)</div><div class='add'>+			     __field(int, state)</div><div class='add'>+			     __field(int, we_ret)</div><div class='add'>+			     ),</div><div class='add'>+	     TP_fast_assign(DD_DEV_ASSIGN(pq-&gt;dd);</div><div class='add'>+			    __entry-&gt;pq = pq;</div><div class='add'>+			    __entry-&gt;state = pq-&gt;state;</div><div class='add'>+			    __entry-&gt;we_ret = we_ret;</div><div class='add'>+			    ),</div><div class='add'>+	     TP_printk("[%s] pq %llx state %d we_ret %d",</div><div class='add'>+		       __get_str(dev),</div><div class='add'>+		       (unsigned long long)__entry-&gt;pq,</div><div class='add'>+		       __entry-&gt;state,</div><div class='add'>+		       __entry-&gt;we_ret</div><div class='add'>+		)</div><div class='add'>+);</div><div class='add'>+</div><div class='ctx'> const char *print_u32_array(struct trace_seq *, u32 *, int);</div><div class='ctx'> #define __print_u32_hex(arr, len) print_u32_array(p, arr, len)</div><div class='ctx'> </div><div class='hunk'>@@ -706,8 +740,8 @@ TRACE_EVENT(hfi1_sdma_state,</div><div class='ctx'> 		__string(newstate, nstate)</div><div class='ctx'> 	    ),</div><div class='ctx'> 	    TP_fast_assign(DD_DEV_ASSIGN(sde-&gt;dd);</div><div class='del'>-		__assign_str(curstate, cstate);</div><div class='del'>-		__assign_str(newstate, nstate);</div><div class='add'>+		__assign_str(curstate);</div><div class='add'>+		__assign_str(newstate);</div><div class='ctx'> 	    ),</div><div class='ctx'> 	    TP_printk("[%s] current state %s new state %s",</div><div class='ctx'> 		      __get_str(dev),</div><div class='hunk'>@@ -792,7 +826,7 @@ TRACE_EVENT(</div><div class='ctx'> 		__field(int, send_flags)</div><div class='ctx'> 	),</div><div class='ctx'> 	TP_fast_assign(</div><div class='del'>-		DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device))</div><div class='add'>+		DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device));</div><div class='ctx'> 		__entry-&gt;wqe = wqe;</div><div class='ctx'> 		__entry-&gt;wr_id = wqe-&gt;wr.wr_id;</div><div class='ctx'> 		__entry-&gt;qpn = qp-&gt;ibqp.qp_num;</div><div class='hunk'>@@ -828,7 +862,7 @@ DECLARE_EVENT_CLASS(</div><div class='ctx'> 		__field(bool, flag)</div><div class='ctx'> 	),</div><div class='ctx'> 	TP_fast_assign(</div><div class='del'>-		DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device))</div><div class='add'>+		DD_DEV_ASSIGN(dd_from_ibdev(qp-&gt;ibqp.device));</div><div class='ctx'> 		__entry-&gt;qpn = qp-&gt;ibqp.qp_num;</div><div class='ctx'> 		__entry-&gt;flag = flag;</div><div class='ctx'> 	),</div><div class='hunk'>@@ -858,6 +892,170 @@ DEFINE_EVENT(</div><div class='ctx'> 	TP_ARGS(qp, flag)</div><div class='ctx'> );</div><div class='ctx'> </div><div class='add'>+DECLARE_EVENT_CLASS(/* AIP  */</div><div class='add'>+	hfi1_ipoib_txq_template,</div><div class='add'>+	TP_PROTO(struct hfi1_ipoib_txq *txq),</div><div class='add'>+	TP_ARGS(txq),</div><div class='add'>+	TP_STRUCT__entry(/* entry */</div><div class='add'>+		DD_DEV_ENTRY(txq-&gt;priv-&gt;dd)</div><div class='add'>+		__field(struct hfi1_ipoib_txq *, txq)</div><div class='add'>+		__field(struct sdma_engine *, sde)</div><div class='add'>+		__field(ulong, head)</div><div class='add'>+		__field(ulong, tail)</div><div class='add'>+		__field(uint, used)</div><div class='add'>+		__field(uint, flow)</div><div class='add'>+		__field(int, stops)</div><div class='add'>+		__field(int, no_desc)</div><div class='add'>+		__field(u8, idx)</div><div class='add'>+		__field(u8, stopped)</div><div class='add'>+	),</div><div class='add'>+	TP_fast_assign(/* assign */</div><div class='add'>+		DD_DEV_ASSIGN(txq-&gt;priv-&gt;dd);</div><div class='add'>+		__entry-&gt;txq = txq;</div><div class='add'>+		__entry-&gt;sde = txq-&gt;sde;</div><div class='add'>+		__entry-&gt;head = txq-&gt;tx_ring.head;</div><div class='add'>+		__entry-&gt;tail = txq-&gt;tx_ring.tail;</div><div class='add'>+		__entry-&gt;idx = txq-&gt;q_idx;</div><div class='add'>+		__entry-&gt;used =</div><div class='add'>+			txq-&gt;tx_ring.sent_txreqs -</div><div class='add'>+			txq-&gt;tx_ring.complete_txreqs;</div><div class='add'>+		__entry-&gt;flow = txq-&gt;flow.as_int;</div><div class='add'>+		__entry-&gt;stops = atomic_read(&amp;txq-&gt;tx_ring.stops);</div><div class='add'>+		__entry-&gt;no_desc = atomic_read(&amp;txq-&gt;tx_ring.no_desc);</div><div class='add'>+		__entry-&gt;stopped =</div><div class='add'>+		 __netif_subqueue_stopped(txq-&gt;priv-&gt;netdev, txq-&gt;q_idx);</div><div class='add'>+	),</div><div class='add'>+	TP_printk(/* print  */</div><div class='add'>+		"[%s] txq %llx idx %u sde %llx:%u cpu %d head %lx tail %lx flow %x used %u stops %d no_desc %d stopped %u",</div><div class='add'>+		__get_str(dev),</div><div class='add'>+		(unsigned long long)__entry-&gt;txq,</div><div class='add'>+		__entry-&gt;idx,</div><div class='add'>+		(unsigned long long)__entry-&gt;sde,</div><div class='add'>+		__entry-&gt;sde ? __entry-&gt;sde-&gt;this_idx : 0,</div><div class='add'>+		__entry-&gt;sde ? __entry-&gt;sde-&gt;cpu : 0,</div><div class='add'>+		__entry-&gt;head,</div><div class='add'>+		__entry-&gt;tail,</div><div class='add'>+		__entry-&gt;flow,</div><div class='add'>+		__entry-&gt;used,</div><div class='add'>+		__entry-&gt;stops,</div><div class='add'>+		__entry-&gt;no_desc,</div><div class='add'>+		__entry-&gt;stopped</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(/* queue stop */</div><div class='add'>+	hfi1_ipoib_txq_template, hfi1_txq_stop,</div><div class='add'>+	TP_PROTO(struct hfi1_ipoib_txq *txq),</div><div class='add'>+	TP_ARGS(txq)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(/* queue wake */</div><div class='add'>+	hfi1_ipoib_txq_template, hfi1_txq_wake,</div><div class='add'>+	TP_PROTO(struct hfi1_ipoib_txq *txq),</div><div class='add'>+	TP_ARGS(txq)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(/* flow flush */</div><div class='add'>+	hfi1_ipoib_txq_template, hfi1_flow_flush,</div><div class='add'>+	TP_PROTO(struct hfi1_ipoib_txq *txq),</div><div class='add'>+	TP_ARGS(txq)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(/* flow switch */</div><div class='add'>+	hfi1_ipoib_txq_template, hfi1_flow_switch,</div><div class='add'>+	TP_PROTO(struct hfi1_ipoib_txq *txq),</div><div class='add'>+	TP_ARGS(txq)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(/* wakeup */</div><div class='add'>+	hfi1_ipoib_txq_template, hfi1_txq_wakeup,</div><div class='add'>+	TP_PROTO(struct hfi1_ipoib_txq *txq),</div><div class='add'>+	TP_ARGS(txq)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(/* full */</div><div class='add'>+	hfi1_ipoib_txq_template, hfi1_txq_full,</div><div class='add'>+	TP_PROTO(struct hfi1_ipoib_txq *txq),</div><div class='add'>+	TP_ARGS(txq)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(/* queued */</div><div class='add'>+	hfi1_ipoib_txq_template, hfi1_txq_queued,</div><div class='add'>+	TP_PROTO(struct hfi1_ipoib_txq *txq),</div><div class='add'>+	TP_ARGS(txq)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(/* xmit_stopped */</div><div class='add'>+	hfi1_ipoib_txq_template, hfi1_txq_xmit_stopped,</div><div class='add'>+	TP_PROTO(struct hfi1_ipoib_txq *txq),</div><div class='add'>+	TP_ARGS(txq)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(/* xmit_unstopped */</div><div class='add'>+	hfi1_ipoib_txq_template, hfi1_txq_xmit_unstopped,</div><div class='add'>+	TP_PROTO(struct hfi1_ipoib_txq *txq),</div><div class='add'>+	TP_ARGS(txq)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DECLARE_EVENT_CLASS(/* AIP  */</div><div class='add'>+	hfi1_ipoib_tx_template,</div><div class='add'>+	TP_PROTO(struct ipoib_txreq *tx, u32 idx),</div><div class='add'>+	TP_ARGS(tx, idx),</div><div class='add'>+	TP_STRUCT__entry(/* entry */</div><div class='add'>+		DD_DEV_ENTRY(tx-&gt;txq-&gt;priv-&gt;dd)</div><div class='add'>+		__field(struct ipoib_txreq *, tx)</div><div class='add'>+		__field(struct hfi1_ipoib_txq *, txq)</div><div class='add'>+		__field(struct sk_buff *, skb)</div><div class='add'>+		__field(ulong, idx)</div><div class='add'>+	),</div><div class='add'>+	TP_fast_assign(/* assign */</div><div class='add'>+		DD_DEV_ASSIGN(tx-&gt;txq-&gt;priv-&gt;dd);</div><div class='add'>+		__entry-&gt;tx = tx;</div><div class='add'>+		__entry-&gt;skb = tx-&gt;skb;</div><div class='add'>+		__entry-&gt;txq = tx-&gt;txq;</div><div class='add'>+		__entry-&gt;idx = idx;</div><div class='add'>+	),</div><div class='add'>+	TP_printk(/* print  */</div><div class='add'>+		"[%s] tx %llx txq %llx,%u skb %llx idx %lu",</div><div class='add'>+		__get_str(dev),</div><div class='add'>+		(unsigned long long)__entry-&gt;tx,</div><div class='add'>+		(unsigned long long)__entry-&gt;txq,</div><div class='add'>+		__entry-&gt;txq ? __entry-&gt;txq-&gt;q_idx : 0,</div><div class='add'>+		(unsigned long long)__entry-&gt;skb,</div><div class='add'>+		__entry-&gt;idx</div><div class='add'>+	)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(/* produce */</div><div class='add'>+	hfi1_ipoib_tx_template, hfi1_tx_produce,</div><div class='add'>+	TP_PROTO(struct ipoib_txreq *tx, u32 idx),</div><div class='add'>+	TP_ARGS(tx, idx)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(/* consume */</div><div class='add'>+	hfi1_ipoib_tx_template, hfi1_tx_consume,</div><div class='add'>+	TP_PROTO(struct ipoib_txreq *tx, u32 idx),</div><div class='add'>+	TP_ARGS(tx, idx)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(/* alloc_tx */</div><div class='add'>+	hfi1_ipoib_txq_template, hfi1_txq_alloc_tx,</div><div class='add'>+	TP_PROTO(struct hfi1_ipoib_txq *txq),</div><div class='add'>+	TP_ARGS(txq)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(/* poll */</div><div class='add'>+	hfi1_ipoib_txq_template, hfi1_txq_poll,</div><div class='add'>+	TP_PROTO(struct hfi1_ipoib_txq *txq),</div><div class='add'>+	TP_ARGS(txq)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(/* complete */</div><div class='add'>+	hfi1_ipoib_txq_template, hfi1_txq_complete,</div><div class='add'>+	TP_PROTO(struct hfi1_ipoib_txq *txq),</div><div class='add'>+	TP_ARGS(txq)</div><div class='add'>+);</div><div class='add'>+</div><div class='ctx'> #endif /* __HFI1_TRACE_TX_H */</div><div class='ctx'> </div><div class='ctx'> #undef TRACE_INCLUDE_PATH</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/uc.c b/drivers/infiniband/hw/hfi1/uc.c<br/>index 4ed4fcfabd6c..33d2c2a218e2 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/uc.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/uc.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/uc.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/uc.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include "hfi.h"</div><div class='hunk'>@@ -55,6 +13,7 @@</div><div class='ctx'> /**</div><div class='ctx'>  * hfi1_make_uc_req - construct a request packet (SEND, RDMA write)</div><div class='ctx'>  * @qp: a pointer to the QP</div><div class='add'>+ * @ps: the current packet state</div><div class='ctx'>  *</div><div class='ctx'>  * Assume s_lock is held.</div><div class='ctx'>  *</div><div class='hunk'>@@ -216,7 +175,7 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)</div><div class='ctx'> </div><div class='ctx'> 	case OP(SEND_FIRST):</div><div class='ctx'> 		qp-&gt;s_state = OP(SEND_MIDDLE);</div><div class='del'>-		/* FALLTHROUGH */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case OP(SEND_MIDDLE):</div><div class='ctx'> 		len = qp-&gt;s_len;</div><div class='ctx'> 		if (len &gt; pmtu) {</div><div class='hunk'>@@ -241,7 +200,7 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)</div><div class='ctx'> </div><div class='ctx'> 	case OP(RDMA_WRITE_FIRST):</div><div class='ctx'> 		qp-&gt;s_state = OP(RDMA_WRITE_MIDDLE);</div><div class='del'>-		/* FALLTHROUGH */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case OP(RDMA_WRITE_MIDDLE):</div><div class='ctx'> 		len = qp-&gt;s_len;</div><div class='ctx'> 		if (len &gt; pmtu) {</div><div class='hunk'>@@ -291,12 +250,7 @@ bail_no_tx:</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * hfi1_uc_rcv - handle an incoming UC packet</div><div class='del'>- * @ibp: the port the packet came in on</div><div class='del'>- * @hdr: the header of the packet</div><div class='del'>- * @rcv_flags: flags relevant to rcv processing</div><div class='del'>- * @data: the packet data</div><div class='del'>- * @tlen: the length of the packet</div><div class='del'>- * @qp: the QP for this packet.</div><div class='add'>+ * @packet: the packet structure</div><div class='ctx'>  *</div><div class='ctx'>  * This is called from qp_rcv() to process an incoming UC packet</div><div class='ctx'>  * for the given QP.</div><div class='hunk'>@@ -414,7 +368,7 @@ send_first:</div><div class='ctx'> 			goto no_immediate_data;</div><div class='ctx'> 		else if (opcode == OP(SEND_ONLY_WITH_IMMEDIATE))</div><div class='ctx'> 			goto send_last_imm;</div><div class='del'>-		/* FALLTHROUGH */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case OP(SEND_MIDDLE):</div><div class='ctx'> 		/* Check for invalid length PMTU or posted rwqe len. */</div><div class='ctx'> 		/*</div><div class='hunk'>@@ -476,8 +430,7 @@ last_imm:</div><div class='ctx'> 		wc.dlid_path_bits = 0;</div><div class='ctx'> 		wc.port_num = 0;</div><div class='ctx'> 		/* Signal completion event if the solicited bit is set. */</div><div class='del'>-		rvt_cq_enter(ibcq_to_rvtcq(qp-&gt;ibqp.recv_cq), &amp;wc,</div><div class='del'>-			     ib_bth_is_solicited(ohdr));</div><div class='add'>+		rvt_recv_cq(qp, &amp;wc, ib_bth_is_solicited(ohdr));</div><div class='ctx'> 		break;</div><div class='ctx'> </div><div class='ctx'> 	case OP(RDMA_WRITE_FIRST):</div><div class='hunk'>@@ -516,7 +469,7 @@ rdma_first:</div><div class='ctx'> 			wc.ex.imm_data = ohdr-&gt;u.rc.imm_data;</div><div class='ctx'> 			goto rdma_last_imm;</div><div class='ctx'> 		}</div><div class='del'>-		/* FALLTHROUGH */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case OP(RDMA_WRITE_MIDDLE):</div><div class='ctx'> 		/* Check for invalid length PMTU or posted rwqe len. */</div><div class='ctx'> 		if (unlikely(tlen != (hdrsize + pmtu + 4)))</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/ud.c b/drivers/infiniband/hw/hfi1/ud.c<br/>index f88ad425664a..89d1bae8f824 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/ud.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/ud.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/ud.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/ud.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2015 - 2019 Intel Corporation.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/net.h&gt;</div><div class='hunk'>@@ -87,7 +45,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)</div><div class='ctx'> 	rcu_read_lock();</div><div class='ctx'> </div><div class='ctx'> 	qp = rvt_lookup_qpn(ib_to_rvt(sqp-&gt;ibqp.device), &amp;ibp-&gt;rvp,</div><div class='del'>-			    swqe-&gt;ud_wr.remote_qpn);</div><div class='add'>+			    rvt_get_swqe_remote_qpn(swqe));</div><div class='ctx'> 	if (!qp) {</div><div class='ctx'> 		ibp-&gt;rvp.n_pkt_drops++;</div><div class='ctx'> 		rcu_read_unlock();</div><div class='hunk'>@@ -105,7 +63,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)</div><div class='ctx'> 		goto drop;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ah_attr = &amp;ibah_to_rvtah(swqe-&gt;ud_wr.ah)-&gt;attr;</div><div class='add'>+	ah_attr = rvt_get_swqe_ah_attr(swqe);</div><div class='ctx'> 	ppd = ppd_from_ibp(ibp);</div><div class='ctx'> </div><div class='ctx'> 	if (qp-&gt;ibqp.qp_num &gt; 1) {</div><div class='hunk'>@@ -135,8 +93,8 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)</div><div class='ctx'> 	if (qp-&gt;ibqp.qp_num) {</div><div class='ctx'> 		u32 qkey;</div><div class='ctx'> </div><div class='del'>-		qkey = (int)swqe-&gt;ud_wr.remote_qkey &lt; 0 ?</div><div class='del'>-			sqp-&gt;qkey : swqe-&gt;ud_wr.remote_qkey;</div><div class='add'>+		qkey = (int)rvt_get_swqe_remote_qkey(swqe) &lt; 0 ?</div><div class='add'>+			sqp-&gt;qkey : rvt_get_swqe_remote_qkey(swqe);</div><div class='ctx'> 		if (unlikely(qkey != qp-&gt;qkey))</div><div class='ctx'> 			goto drop; /* silently drop per IBTA spec */</div><div class='ctx'> 	}</div><div class='hunk'>@@ -240,7 +198,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)</div><div class='ctx'> 	if (qp-&gt;ibqp.qp_type == IB_QPT_GSI || qp-&gt;ibqp.qp_type == IB_QPT_SMI) {</div><div class='ctx'> 		if (sqp-&gt;ibqp.qp_type == IB_QPT_GSI ||</div><div class='ctx'> 		    sqp-&gt;ibqp.qp_type == IB_QPT_SMI)</div><div class='del'>-			wc.pkey_index = swqe-&gt;ud_wr.pkey_index;</div><div class='add'>+			wc.pkey_index = rvt_get_swqe_pkey_index(swqe);</div><div class='ctx'> 		else</div><div class='ctx'> 			wc.pkey_index = sqp-&gt;s_pkey_index;</div><div class='ctx'> 	} else {</div><div class='hunk'>@@ -255,8 +213,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)</div><div class='ctx'> 	wc.dlid_path_bits = rdma_ah_get_dlid(ah_attr) &amp; ((1 &lt;&lt; ppd-&gt;lmc) - 1);</div><div class='ctx'> 	wc.port_num = qp-&gt;port_num;</div><div class='ctx'> 	/* Signal completion event if the solicited bit is set. */</div><div class='del'>-	rvt_cq_enter(ibcq_to_rvtcq(qp-&gt;ibqp.recv_cq), &amp;wc,</div><div class='del'>-		     swqe-&gt;wr.send_flags &amp; IB_SEND_SOLICITED);</div><div class='add'>+	rvt_recv_cq(qp, &amp;wc, swqe-&gt;wr.send_flags &amp; IB_SEND_SOLICITED);</div><div class='ctx'> 	ibp-&gt;rvp.n_loop_pkts++;</div><div class='ctx'> bail_unlock:</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;qp-&gt;r_lock, flags);</div><div class='hunk'>@@ -283,20 +240,21 @@ static void hfi1_make_bth_deth(struct rvt_qp *qp, struct rvt_swqe *wqe,</div><div class='ctx'> 		bth0 |= IB_BTH_SOLICITED;</div><div class='ctx'> 	bth0 |= extra_bytes &lt;&lt; 20;</div><div class='ctx'> 	if (qp-&gt;ibqp.qp_type == IB_QPT_GSI || qp-&gt;ibqp.qp_type == IB_QPT_SMI)</div><div class='del'>-		*pkey = hfi1_get_pkey(ibp, wqe-&gt;ud_wr.pkey_index);</div><div class='add'>+		*pkey = hfi1_get_pkey(ibp, rvt_get_swqe_pkey_index(wqe));</div><div class='ctx'> 	else</div><div class='ctx'> 		*pkey = hfi1_get_pkey(ibp, qp-&gt;s_pkey_index);</div><div class='ctx'> 	if (!bypass)</div><div class='ctx'> 		bth0 |= *pkey;</div><div class='ctx'> 	ohdr-&gt;bth[0] = cpu_to_be32(bth0);</div><div class='del'>-	ohdr-&gt;bth[1] = cpu_to_be32(wqe-&gt;ud_wr.remote_qpn);</div><div class='add'>+	ohdr-&gt;bth[1] = cpu_to_be32(rvt_get_swqe_remote_qpn(wqe));</div><div class='ctx'> 	ohdr-&gt;bth[2] = cpu_to_be32(mask_psn(wqe-&gt;psn));</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Qkeys with the high order bit set mean use the</div><div class='ctx'> 	 * qkey from the QP context instead of the WR (see 10.2.5).</div><div class='ctx'> 	 */</div><div class='del'>-	ohdr-&gt;u.ud.deth[0] = cpu_to_be32((int)wqe-&gt;ud_wr.remote_qkey &lt; 0 ?</div><div class='del'>-					 qp-&gt;qkey : wqe-&gt;ud_wr.remote_qkey);</div><div class='add'>+	ohdr-&gt;u.ud.deth[0] =</div><div class='add'>+		cpu_to_be32((int)rvt_get_swqe_remote_qkey(wqe) &lt; 0 ? qp-&gt;qkey :</div><div class='add'>+			    rvt_get_swqe_remote_qkey(wqe));</div><div class='ctx'> 	ohdr-&gt;u.ud.deth[1] = cpu_to_be32(qp-&gt;ibqp.qp_num);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -316,7 +274,7 @@ void hfi1_make_ud_req_9B(struct rvt_qp *qp, struct hfi1_pkt_state *ps,</div><div class='ctx'> </div><div class='ctx'> 	ibp = to_iport(qp-&gt;ibqp.device, qp-&gt;port_num);</div><div class='ctx'> 	ppd = ppd_from_ibp(ibp);</div><div class='del'>-	ah_attr = &amp;ibah_to_rvtah(wqe-&gt;ud_wr.ah)-&gt;attr;</div><div class='add'>+	ah_attr = rvt_get_swqe_ah_attr(wqe);</div><div class='ctx'> </div><div class='ctx'> 	extra_bytes = -wqe-&gt;length &amp; 3;</div><div class='ctx'> 	nwords = ((wqe-&gt;length + extra_bytes) &gt;&gt; 2) + SIZE_OF_CRC;</div><div class='hunk'>@@ -380,7 +338,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps,</div><div class='ctx'> 	struct hfi1_pportdata *ppd;</div><div class='ctx'> 	struct hfi1_ibport *ibp;</div><div class='ctx'> 	u32 dlid, slid, nwords, extra_bytes;</div><div class='del'>-	u32 dest_qp = wqe-&gt;ud_wr.remote_qpn;</div><div class='add'>+	u32 dest_qp = rvt_get_swqe_remote_qpn(wqe);</div><div class='ctx'> 	u32 src_qp = qp-&gt;ibqp.qp_num;</div><div class='ctx'> 	u16 len, pkey;</div><div class='ctx'> 	u8 l4, sc5;</div><div class='hunk'>@@ -388,7 +346,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps,</div><div class='ctx'> </div><div class='ctx'> 	ibp = to_iport(qp-&gt;ibqp.device, qp-&gt;port_num);</div><div class='ctx'> 	ppd = ppd_from_ibp(ibp);</div><div class='del'>-	ah_attr = &amp;ibah_to_rvtah(wqe-&gt;ud_wr.ah)-&gt;attr;</div><div class='add'>+	ah_attr = rvt_get_swqe_ah_attr(wqe);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Build 16B Management Packet if either the destination</div><div class='hunk'>@@ -450,7 +408,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps,</div><div class='ctx'> </div><div class='ctx'> 	if (is_mgmt) {</div><div class='ctx'> 		l4 = OPA_16B_L4_FM;</div><div class='del'>-		pkey = hfi1_get_pkey(ibp, wqe-&gt;ud_wr.pkey_index);</div><div class='add'>+		pkey = hfi1_get_pkey(ibp, rvt_get_swqe_pkey_index(wqe));</div><div class='ctx'> 		hfi1_16B_set_qpn(&amp;ps-&gt;s_txreq-&gt;phdr.hdr.opah.u.mgmt,</div><div class='ctx'> 				 dest_qp, src_qp);</div><div class='ctx'> 	} else {</div><div class='hunk'>@@ -468,6 +426,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps,</div><div class='ctx'> /**</div><div class='ctx'>  * hfi1_make_ud_req - construct a UD request packet</div><div class='ctx'>  * @qp: the QP</div><div class='add'>+ * @ps: the current packet state</div><div class='ctx'>  *</div><div class='ctx'>  * Assume s_lock is held.</div><div class='ctx'>  *</div><div class='hunk'>@@ -515,7 +474,7 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)</div><div class='ctx'> 	/* Construct the header. */</div><div class='ctx'> 	ibp = to_iport(qp-&gt;ibqp.device, qp-&gt;port_num);</div><div class='ctx'> 	ppd = ppd_from_ibp(ibp);</div><div class='del'>-	ah_attr = &amp;ibah_to_rvtah(wqe-&gt;ud_wr.ah)-&gt;attr;</div><div class='add'>+	ah_attr = rvt_get_swqe_ah_attr(wqe);</div><div class='ctx'> 	priv-&gt;hdr_type = hfi1_get_hdr_type(ppd-&gt;lid, ah_attr);</div><div class='ctx'> 	if ((!hfi1_check_mcast(rdma_ah_get_dlid(ah_attr))) ||</div><div class='ctx'> 	    (rdma_ah_get_dlid(ah_attr) == be32_to_cpu(OPA_LID_PERMISSIVE))) {</div><div class='hunk'>@@ -683,7 +642,7 @@ void return_cnp_16B(struct hfi1_ibport *ibp, struct rvt_qp *qp,</div><div class='ctx'> 	pbc = create_pbc(ppd, pbc_flags, qp-&gt;srate_mbps, vl, plen);</div><div class='ctx'> 	if (ctxt) {</div><div class='ctx'> 		pbuf = sc_buffer_alloc(ctxt, plen, NULL, NULL);</div><div class='del'>-		if (pbuf) {</div><div class='add'>+		if (!IS_ERR_OR_NULL(pbuf)) {</div><div class='ctx'> 			trace_pio_output_ibhdr(ppd-&gt;dd, &amp;hdr, sc5);</div><div class='ctx'> 			ppd-&gt;dd-&gt;pio_inline_send(ppd-&gt;dd, pbuf, pbc,</div><div class='ctx'> 						 &amp;hdr, hwords);</div><div class='hunk'>@@ -738,7 +697,7 @@ void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn,</div><div class='ctx'> 	pbc = create_pbc(ppd, pbc_flags, qp-&gt;srate_mbps, vl, plen);</div><div class='ctx'> 	if (ctxt) {</div><div class='ctx'> 		pbuf = sc_buffer_alloc(ctxt, plen, NULL, NULL);</div><div class='del'>-		if (pbuf) {</div><div class='add'>+		if (!IS_ERR_OR_NULL(pbuf)) {</div><div class='ctx'> 			trace_pio_output_ibhdr(ppd-&gt;dd, &amp;hdr, sc5);</div><div class='ctx'> 			ppd-&gt;dd-&gt;pio_inline_send(ppd-&gt;dd, pbuf, pbc,</div><div class='ctx'> 						 &amp;hdr, hwords);</div><div class='hunk'>@@ -840,12 +799,7 @@ static int opa_smp_check(struct hfi1_ibport *ibp, u16 pkey, u8 sc5,</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * hfi1_ud_rcv - receive an incoming UD packet</div><div class='del'>- * @ibp: the port the packet came in on</div><div class='del'>- * @hdr: the packet header</div><div class='del'>- * @rcv_flags: flags relevant to rcv processing</div><div class='del'>- * @data: the packet data</div><div class='del'>- * @tlen: the packet length</div><div class='del'>- * @qp: the QP the packet came on</div><div class='add'>+ * @packet: the packet structure</div><div class='ctx'>  *</div><div class='ctx'>  * This is called from qp_rcv() to process an incoming UD packet</div><div class='ctx'>  * for the given QP.</div><div class='hunk'>@@ -1061,7 +1015,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)</div><div class='ctx'> 		dlid &amp; ((1 &lt;&lt; ppd_from_ibp(ibp)-&gt;lmc) - 1);</div><div class='ctx'> 	wc.port_num = qp-&gt;port_num;</div><div class='ctx'> 	/* Signal completion event if the solicited bit is set. */</div><div class='del'>-	rvt_cq_enter(ibcq_to_rvtcq(qp-&gt;ibqp.recv_cq), &amp;wc, solicited);</div><div class='add'>+	rvt_recv_cq(qp, &amp;wc, solicited);</div><div class='ctx'> 	return;</div><div class='ctx'> </div><div class='ctx'> drop:</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/user_exp_rcv.c b/drivers/infiniband/hw/hfi1/user_exp_rcv.c<br/>index 0cd71ce7cc71..62b4f16dab27 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/user_exp_rcv.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/user_exp_rcv.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/user_exp_rcv.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/user_exp_rcv.c</a></div><div class='hunk'>@@ -1,48 +1,7 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='add'>+ * Copyright(c) 2020 Cornelis Networks, Inc.</div><div class='ctx'>  * Copyright(c) 2015-2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> #include &lt;asm/page.h&gt;</div><div class='ctx'> #include &lt;linux/string.h&gt;</div><div class='hunk'>@@ -59,24 +18,28 @@ static int set_rcvarray_entry(struct hfi1_filedata *fd,</div><div class='ctx'> 			      struct tid_user_buf *tbuf,</div><div class='ctx'> 			      u32 rcventry, struct tid_group *grp,</div><div class='ctx'> 			      u16 pageidx, unsigned int npages);</div><div class='del'>-static int tid_rb_insert(void *arg, struct mmu_rb_node *node);</div><div class='ctx'> static void cacheless_tid_rb_remove(struct hfi1_filedata *fdata,</div><div class='ctx'> 				    struct tid_rb_node *tnode);</div><div class='del'>-static void tid_rb_remove(void *arg, struct mmu_rb_node *node);</div><div class='del'>-static int tid_rb_invalidate(void *arg, struct mmu_rb_node *mnode);</div><div class='add'>+static bool tid_rb_invalidate(struct mmu_interval_notifier *mni,</div><div class='add'>+			      const struct mmu_notifier_range *range,</div><div class='add'>+			      unsigned long cur_seq);</div><div class='add'>+static bool tid_cover_invalidate(struct mmu_interval_notifier *mni,</div><div class='add'>+			         const struct mmu_notifier_range *range,</div><div class='add'>+			         unsigned long cur_seq);</div><div class='ctx'> static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *,</div><div class='del'>-			    struct tid_group *grp,</div><div class='del'>-			    unsigned int start, u16 count,</div><div class='add'>+			    struct tid_group *grp, u16 count,</div><div class='ctx'> 			    u32 *tidlist, unsigned int *tididx,</div><div class='ctx'> 			    unsigned int *pmapped);</div><div class='del'>-static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo,</div><div class='del'>-			      struct tid_group **grp);</div><div class='add'>+static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo);</div><div class='add'>+static void __clear_tid_node(struct hfi1_filedata *fd,</div><div class='add'>+			     struct tid_rb_node *node);</div><div class='ctx'> static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node);</div><div class='ctx'> </div><div class='del'>-static struct mmu_rb_ops tid_rb_ops = {</div><div class='del'>-	.insert = tid_rb_insert,</div><div class='del'>-	.remove = tid_rb_remove,</div><div class='del'>-	.invalidate = tid_rb_invalidate</div><div class='add'>+static const struct mmu_interval_notifier_ops tid_mn_ops = {</div><div class='add'>+	.invalidate = tid_rb_invalidate,</div><div class='add'>+};</div><div class='add'>+static const struct mmu_interval_notifier_ops tid_cover_ops = {</div><div class='add'>+	.invalidate = tid_cover_invalidate,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -87,14 +50,10 @@ static struct mmu_rb_ops tid_rb_ops = {</div><div class='ctx'> int hfi1_user_exp_rcv_init(struct hfi1_filedata *fd,</div><div class='ctx'> 			   struct hfi1_ctxtdata *uctxt)</div><div class='ctx'> {</div><div class='del'>-	struct hfi1_devdata *dd = uctxt-&gt;dd;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	spin_lock_init(&amp;fd-&gt;tid_lock);</div><div class='del'>-	spin_lock_init(&amp;fd-&gt;invalid_lock);</div><div class='del'>-</div><div class='ctx'> 	fd-&gt;entry_to_rb = kcalloc(uctxt-&gt;expected_count,</div><div class='del'>-				  sizeof(struct rb_node *),</div><div class='add'>+				  sizeof(*fd-&gt;entry_to_rb),</div><div class='ctx'> 				  GFP_KERNEL);</div><div class='ctx'> 	if (!fd-&gt;entry_to_rb)</div><div class='ctx'> 		return -ENOMEM;</div><div class='hunk'>@@ -109,20 +68,7 @@ int hfi1_user_exp_rcv_init(struct hfi1_filedata *fd,</div><div class='ctx'> 			fd-&gt;entry_to_rb = NULL;</div><div class='ctx'> 			return -ENOMEM;</div><div class='ctx'> 		}</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * Register MMU notifier callbacks. If the registration</div><div class='del'>-		 * fails, continue without TID caching for this context.</div><div class='del'>-		 */</div><div class='del'>-		ret = hfi1_mmu_rb_register(fd, fd-&gt;mm, &amp;tid_rb_ops,</div><div class='del'>-					   dd-&gt;pport-&gt;hfi1_wq,</div><div class='del'>-					   &amp;fd-&gt;handler);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dd_dev_info(dd,</div><div class='del'>-				    "Failed MMU notifier registration %d\n",</div><div class='del'>-				    ret);</div><div class='del'>-			ret = 0;</div><div class='del'>-		}</div><div class='add'>+		fd-&gt;use_mn = true;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='hunk'>@@ -139,7 +85,7 @@ int hfi1_user_exp_rcv_init(struct hfi1_filedata *fd,</div><div class='ctx'> 	 * init.</div><div class='ctx'> 	 */</div><div class='ctx'> 	spin_lock(&amp;fd-&gt;tid_lock);</div><div class='del'>-	if (uctxt-&gt;subctxt_cnt &amp;&amp; fd-&gt;handler) {</div><div class='add'>+	if (uctxt-&gt;subctxt_cnt &amp;&amp; fd-&gt;use_mn) {</div><div class='ctx'> 		u16 remainder;</div><div class='ctx'> </div><div class='ctx'> 		fd-&gt;tid_limit = uctxt-&gt;expected_count / uctxt-&gt;subctxt_cnt;</div><div class='hunk'>@@ -158,18 +104,12 @@ void hfi1_user_exp_rcv_free(struct hfi1_filedata *fd)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_ctxtdata *uctxt = fd-&gt;uctxt;</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * The notifier would have been removed when the process'es mm</div><div class='del'>-	 * was freed.</div><div class='del'>-	 */</div><div class='del'>-	if (fd-&gt;handler) {</div><div class='del'>-		hfi1_mmu_rb_unregister(fd-&gt;handler);</div><div class='del'>-	} else {</div><div class='del'>-		if (!EXP_TID_SET_EMPTY(uctxt-&gt;tid_full_list))</div><div class='del'>-			unlock_exp_tids(uctxt, &amp;uctxt-&gt;tid_full_list, fd);</div><div class='del'>-		if (!EXP_TID_SET_EMPTY(uctxt-&gt;tid_used_list))</div><div class='del'>-			unlock_exp_tids(uctxt, &amp;uctxt-&gt;tid_used_list, fd);</div><div class='del'>-	}</div><div class='add'>+	mutex_lock(&amp;uctxt-&gt;exp_mutex);</div><div class='add'>+	if (!EXP_TID_SET_EMPTY(uctxt-&gt;tid_full_list))</div><div class='add'>+		unlock_exp_tids(uctxt, &amp;uctxt-&gt;tid_full_list, fd);</div><div class='add'>+	if (!EXP_TID_SET_EMPTY(uctxt-&gt;tid_used_list))</div><div class='add'>+		unlock_exp_tids(uctxt, &amp;uctxt-&gt;tid_used_list, fd);</div><div class='add'>+	mutex_unlock(&amp;uctxt-&gt;exp_mutex);</div><div class='ctx'> </div><div class='ctx'> 	kfree(fd-&gt;invalid_tids);</div><div class='ctx'> 	fd-&gt;invalid_tids = NULL;</div><div class='hunk'>@@ -178,12 +118,12 @@ void hfi1_user_exp_rcv_free(struct hfi1_filedata *fd)</div><div class='ctx'> 	fd-&gt;entry_to_rb = NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/**</div><div class='add'>+/*</div><div class='ctx'>  * Release pinned receive buffer pages.</div><div class='ctx'>  *</div><div class='del'>- * @mapped - true if the pages have been DMA mapped. false otherwise.</div><div class='del'>- * @idx - Index of the first page to unpin.</div><div class='del'>- * @npages - No of pages to unpin.</div><div class='add'>+ * @mapped: true if the pages have been DMA mapped. false otherwise.</div><div class='add'>+ * @idx: Index of the first page to unpin.</div><div class='add'>+ * @npages: No of pages to unpin.</div><div class='ctx'>  *</div><div class='ctx'>  * If the pages have been DMA mapped (indicated by mapped parameter), their</div><div class='ctx'>  * info will be passed via a struct tid_rb_node. If they haven't been mapped,</div><div class='hunk'>@@ -198,46 +138,37 @@ static void unpin_rcv_pages(struct hfi1_filedata *fd,</div><div class='ctx'> {</div><div class='ctx'> 	struct page **pages;</div><div class='ctx'> 	struct hfi1_devdata *dd = fd-&gt;uctxt-&gt;dd;</div><div class='add'>+	struct mm_struct *mm;</div><div class='ctx'> </div><div class='ctx'> 	if (mapped) {</div><div class='del'>-		pci_unmap_single(dd-&gt;pcidev, node-&gt;dma_addr,</div><div class='del'>-				 node-&gt;mmu.len, PCI_DMA_FROMDEVICE);</div><div class='add'>+		dma_unmap_single(&amp;dd-&gt;pcidev-&gt;dev, node-&gt;dma_addr,</div><div class='add'>+				 node-&gt;npages * PAGE_SIZE, DMA_FROM_DEVICE);</div><div class='ctx'> 		pages = &amp;node-&gt;pages[idx];</div><div class='add'>+		mm = mm_from_tid_node(node);</div><div class='ctx'> 	} else {</div><div class='ctx'> 		pages = &amp;tidbuf-&gt;pages[idx];</div><div class='add'>+		mm = current-&gt;mm;</div><div class='ctx'> 	}</div><div class='del'>-	hfi1_release_user_pages(fd-&gt;mm, pages, npages, mapped);</div><div class='add'>+	hfi1_release_user_pages(mm, pages, npages, mapped);</div><div class='ctx'> 	fd-&gt;tid_n_pinned -= npages;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/**</div><div class='add'>+/*</div><div class='ctx'>  * Pin receive buffer pages.</div><div class='ctx'>  */</div><div class='ctx'> static int pin_rcv_pages(struct hfi1_filedata *fd, struct tid_user_buf *tidbuf)</div><div class='ctx'> {</div><div class='ctx'> 	int pinned;</div><div class='del'>-	unsigned int npages;</div><div class='add'>+	unsigned int npages = tidbuf-&gt;npages;</div><div class='ctx'> 	unsigned long vaddr = tidbuf-&gt;vaddr;</div><div class='ctx'> 	struct page **pages = NULL;</div><div class='ctx'> 	struct hfi1_devdata *dd = fd-&gt;uctxt-&gt;dd;</div><div class='ctx'> </div><div class='del'>-	/* Get the number of pages the user buffer spans */</div><div class='del'>-	npages = num_user_pages(vaddr, tidbuf-&gt;length);</div><div class='del'>-	if (!npages)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='ctx'> 	if (npages &gt; fd-&gt;uctxt-&gt;expected_count) {</div><div class='ctx'> 		dd_dev_err(dd, "Expected buffer too big\n");</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* Verify that access is OK for the user buffer */</div><div class='del'>-	if (!access_ok((void __user *)vaddr,</div><div class='del'>-		       npages * PAGE_SIZE)) {</div><div class='del'>-		dd_dev_err(dd, "Fail vaddr %p, %u pages, !access_ok\n",</div><div class='del'>-			   (void *)vaddr, npages);</div><div class='del'>-		return -EFAULT;</div><div class='del'>-	}</div><div class='ctx'> 	/* Allocate the array of struct page pointers needed for pinning */</div><div class='ctx'> 	pages = kcalloc(npages, sizeof(*pages), GFP_KERNEL);</div><div class='ctx'> 	if (!pages)</div><div class='hunk'>@@ -248,18 +179,17 @@ static int pin_rcv_pages(struct hfi1_filedata *fd, struct tid_user_buf *tidbuf)</div><div class='ctx'> 	 * pages, accept the amount pinned so far and program only that.</div><div class='ctx'> 	 * User space knows how to deal with partially programmed buffers.</div><div class='ctx'> 	 */</div><div class='del'>-	if (!hfi1_can_pin_pages(dd, fd-&gt;mm, fd-&gt;tid_n_pinned, npages)) {</div><div class='add'>+	if (!hfi1_can_pin_pages(dd, current-&gt;mm, fd-&gt;tid_n_pinned, npages)) {</div><div class='ctx'> 		kfree(pages);</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	pinned = hfi1_acquire_user_pages(fd-&gt;mm, vaddr, npages, true, pages);</div><div class='add'>+	pinned = hfi1_acquire_user_pages(current-&gt;mm, vaddr, npages, true, pages);</div><div class='ctx'> 	if (pinned &lt;= 0) {</div><div class='ctx'> 		kfree(pages);</div><div class='ctx'> 		return pinned;</div><div class='ctx'> 	}</div><div class='ctx'> 	tidbuf-&gt;pages = pages;</div><div class='del'>-	tidbuf-&gt;npages = npages;</div><div class='ctx'> 	fd-&gt;tid_n_pinned += pinned;</div><div class='ctx'> 	return pinned;</div><div class='ctx'> }</div><div class='hunk'>@@ -319,54 +249,70 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,</div><div class='ctx'> 	int ret = 0, need_group = 0, pinned;</div><div class='ctx'> 	struct hfi1_ctxtdata *uctxt = fd-&gt;uctxt;</div><div class='ctx'> 	struct hfi1_devdata *dd = uctxt-&gt;dd;</div><div class='del'>-	unsigned int ngroups, pageidx = 0, pageset_count,</div><div class='add'>+	unsigned int ngroups, pageset_count,</div><div class='ctx'> 		tididx = 0, mapped, mapped_pages = 0;</div><div class='ctx'> 	u32 *tidlist = NULL;</div><div class='ctx'> 	struct tid_user_buf *tidbuf;</div><div class='add'>+	unsigned long mmu_seq = 0;</div><div class='add'>+</div><div class='add'>+	if (!PAGE_ALIGNED(tinfo-&gt;vaddr))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	if (tinfo-&gt;length == 0)</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	tidbuf = kzalloc(sizeof(*tidbuf), GFP_KERNEL);</div><div class='ctx'> 	if (!tidbuf)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='add'>+	mutex_init(&amp;tidbuf-&gt;cover_mutex);</div><div class='ctx'> 	tidbuf-&gt;vaddr = tinfo-&gt;vaddr;</div><div class='ctx'> 	tidbuf-&gt;length = tinfo-&gt;length;</div><div class='add'>+	tidbuf-&gt;npages = num_user_pages(tidbuf-&gt;vaddr, tidbuf-&gt;length);</div><div class='ctx'> 	tidbuf-&gt;psets = kcalloc(uctxt-&gt;expected_count, sizeof(*tidbuf-&gt;psets),</div><div class='ctx'> 				GFP_KERNEL);</div><div class='ctx'> 	if (!tidbuf-&gt;psets) {</div><div class='del'>-		kfree(tidbuf);</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto fail_release_mem;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (fd-&gt;use_mn) {</div><div class='add'>+		ret = mmu_interval_notifier_insert(</div><div class='add'>+			&amp;tidbuf-&gt;notifier, current-&gt;mm,</div><div class='add'>+			tidbuf-&gt;vaddr, tidbuf-&gt;npages * PAGE_SIZE,</div><div class='add'>+			&amp;tid_cover_ops);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto fail_release_mem;</div><div class='add'>+		mmu_seq = mmu_interval_read_begin(&amp;tidbuf-&gt;notifier);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	pinned = pin_rcv_pages(fd, tidbuf);</div><div class='ctx'> 	if (pinned &lt;= 0) {</div><div class='del'>-		kfree(tidbuf-&gt;psets);</div><div class='del'>-		kfree(tidbuf);</div><div class='del'>-		return pinned;</div><div class='add'>+		ret = (pinned &lt; 0) ? pinned : -ENOSPC;</div><div class='add'>+		goto fail_unpin;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Find sets of physically contiguous pages */</div><div class='ctx'> 	tidbuf-&gt;n_psets = find_phys_blocks(tidbuf, pinned);</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * We don't need to access this under a lock since tid_used is per</div><div class='del'>-	 * process and the same process cannot be in hfi1_user_exp_rcv_clear()</div><div class='del'>-	 * and hfi1_user_exp_rcv_setup() at the same time.</div><div class='del'>-	 */</div><div class='add'>+	/* Reserve the number of expected tids to be used. */</div><div class='ctx'> 	spin_lock(&amp;fd-&gt;tid_lock);</div><div class='ctx'> 	if (fd-&gt;tid_used + tidbuf-&gt;n_psets &gt; fd-&gt;tid_limit)</div><div class='ctx'> 		pageset_count = fd-&gt;tid_limit - fd-&gt;tid_used;</div><div class='ctx'> 	else</div><div class='ctx'> 		pageset_count = tidbuf-&gt;n_psets;</div><div class='add'>+	fd-&gt;tid_used += pageset_count;</div><div class='ctx'> 	spin_unlock(&amp;fd-&gt;tid_lock);</div><div class='ctx'> </div><div class='del'>-	if (!pageset_count)</div><div class='del'>-		goto bail;</div><div class='add'>+	if (!pageset_count) {</div><div class='add'>+		ret = -ENOSPC;</div><div class='add'>+		goto fail_unreserve;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	ngroups = pageset_count / dd-&gt;rcv_entries.group_size;</div><div class='ctx'> 	tidlist = kcalloc(pageset_count, sizeof(*tidlist), GFP_KERNEL);</div><div class='ctx'> 	if (!tidlist) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='del'>-		goto nomem;</div><div class='add'>+		goto fail_unreserve;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	tididx = 0;</div><div class='hunk'>@@ -385,7 +331,7 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,</div><div class='ctx'> 			tid_group_pop(&amp;uctxt-&gt;tid_group_list);</div><div class='ctx'> </div><div class='ctx'> 		ret = program_rcvarray(fd, tidbuf, grp,</div><div class='del'>-				       pageidx, dd-&gt;rcv_entries.group_size,</div><div class='add'>+				       dd-&gt;rcv_entries.group_size,</div><div class='ctx'> 				       tidlist, &amp;tididx, &amp;mapped);</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * If there was a failure to program the RcvArray</div><div class='hunk'>@@ -401,11 +347,10 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,</div><div class='ctx'> </div><div class='ctx'> 		tid_group_add_tail(grp, &amp;uctxt-&gt;tid_full_list);</div><div class='ctx'> 		ngroups--;</div><div class='del'>-		pageidx += ret;</div><div class='ctx'> 		mapped_pages += mapped;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	while (pageidx &lt; pageset_count) {</div><div class='add'>+	while (tididx &lt; pageset_count) {</div><div class='ctx'> 		struct tid_group *grp, *ptr;</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * If we don't have any partially used tid groups, check</div><div class='hunk'>@@ -427,11 +372,11 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,</div><div class='ctx'> 		 */</div><div class='ctx'> 		list_for_each_entry_safe(grp, ptr, &amp;uctxt-&gt;tid_used_list.list,</div><div class='ctx'> 					 list) {</div><div class='del'>-			unsigned use = min_t(unsigned, pageset_count - pageidx,</div><div class='add'>+			unsigned use = min_t(unsigned, pageset_count - tididx,</div><div class='ctx'> 					     grp-&gt;size - grp-&gt;used);</div><div class='ctx'> </div><div class='ctx'> 			ret = program_rcvarray(fd, tidbuf, grp,</div><div class='del'>-					       pageidx, use, tidlist,</div><div class='add'>+					       use, tidlist,</div><div class='ctx'> 					       &amp;tididx, &amp;mapped);</div><div class='ctx'> 			if (ret &lt; 0) {</div><div class='ctx'> 				hfi1_cdbg(TID,</div><div class='hunk'>@@ -443,11 +388,10 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,</div><div class='ctx'> 					tid_group_move(grp,</div><div class='ctx'> 						       &amp;uctxt-&gt;tid_used_list,</div><div class='ctx'> 						       &amp;uctxt-&gt;tid_full_list);</div><div class='del'>-				pageidx += ret;</div><div class='ctx'> 				mapped_pages += mapped;</div><div class='ctx'> 				need_group = 0;</div><div class='ctx'> 				/* Check if we are done so we break out early */</div><div class='del'>-				if (pageidx &gt;= pageset_count)</div><div class='add'>+				if (tididx &gt;= pageset_count)</div><div class='ctx'> 					break;</div><div class='ctx'> 			} else if (WARN_ON(ret == 0)) {</div><div class='ctx'> 				/*</div><div class='hunk'>@@ -462,43 +406,78 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,</div><div class='ctx'> 	}</div><div class='ctx'> unlock:</div><div class='ctx'> 	mutex_unlock(&amp;uctxt-&gt;exp_mutex);</div><div class='del'>-nomem:</div><div class='ctx'> 	hfi1_cdbg(TID, "total mapped: tidpairs:%u pages:%u (%d)", tididx,</div><div class='ctx'> 		  mapped_pages, ret);</div><div class='del'>-	if (tididx) {</div><div class='del'>-		spin_lock(&amp;fd-&gt;tid_lock);</div><div class='del'>-		fd-&gt;tid_used += tididx;</div><div class='del'>-		spin_unlock(&amp;fd-&gt;tid_lock);</div><div class='del'>-		tinfo-&gt;tidcnt = tididx;</div><div class='del'>-		tinfo-&gt;length = mapped_pages * PAGE_SIZE;</div><div class='del'>-</div><div class='del'>-		if (copy_to_user(u64_to_user_ptr(tinfo-&gt;tidlist),</div><div class='del'>-				 tidlist, sizeof(tidlist[0]) * tididx)) {</div><div class='del'>-			/*</div><div class='del'>-			 * On failure to copy to the user level, we need to undo</div><div class='del'>-			 * everything done so far so we don't leak resources.</div><div class='del'>-			 */</div><div class='del'>-			tinfo-&gt;tidlist = (unsigned long)&amp;tidlist;</div><div class='del'>-			hfi1_user_exp_rcv_clear(fd, tinfo);</div><div class='del'>-			tinfo-&gt;tidlist = 0;</div><div class='del'>-			ret = -EFAULT;</div><div class='del'>-			goto bail;</div><div class='add'>+</div><div class='add'>+	/* fail if nothing was programmed, set error if none provided */</div><div class='add'>+	if (tididx == 0) {</div><div class='add'>+		if (ret &gt;= 0)</div><div class='add'>+			ret = -ENOSPC;</div><div class='add'>+		goto fail_unreserve;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* adjust reserved tid_used to actual count */</div><div class='add'>+	spin_lock(&amp;fd-&gt;tid_lock);</div><div class='add'>+	fd-&gt;tid_used -= pageset_count - tididx;</div><div class='add'>+	spin_unlock(&amp;fd-&gt;tid_lock);</div><div class='add'>+</div><div class='add'>+	/* unpin all pages not covered by a TID */</div><div class='add'>+	unpin_rcv_pages(fd, tidbuf, NULL, mapped_pages, pinned - mapped_pages,</div><div class='add'>+			false);</div><div class='add'>+</div><div class='add'>+	if (fd-&gt;use_mn) {</div><div class='add'>+		/* check for an invalidate during setup */</div><div class='add'>+		bool fail = false;</div><div class='add'>+</div><div class='add'>+		mutex_lock(&amp;tidbuf-&gt;cover_mutex);</div><div class='add'>+		fail = mmu_interval_read_retry(&amp;tidbuf-&gt;notifier, mmu_seq);</div><div class='add'>+		mutex_unlock(&amp;tidbuf-&gt;cover_mutex);</div><div class='add'>+</div><div class='add'>+		if (fail) {</div><div class='add'>+			ret = -EBUSY;</div><div class='add'>+			goto fail_unprogram;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * If not everything was mapped (due to insufficient RcvArray entries,</div><div class='del'>-	 * for example), unpin all unmapped pages so we can pin them nex time.</div><div class='del'>-	 */</div><div class='del'>-	if (mapped_pages != pinned)</div><div class='del'>-		unpin_rcv_pages(fd, tidbuf, NULL, mapped_pages,</div><div class='del'>-				(pinned - mapped_pages), false);</div><div class='del'>-bail:</div><div class='add'>+	tinfo-&gt;tidcnt = tididx;</div><div class='add'>+	tinfo-&gt;length = mapped_pages * PAGE_SIZE;</div><div class='add'>+</div><div class='add'>+	if (copy_to_user(u64_to_user_ptr(tinfo-&gt;tidlist),</div><div class='add'>+			 tidlist, sizeof(tidlist[0]) * tididx)) {</div><div class='add'>+		ret = -EFAULT;</div><div class='add'>+		goto fail_unprogram;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (fd-&gt;use_mn)</div><div class='add'>+		mmu_interval_notifier_remove(&amp;tidbuf-&gt;notifier);</div><div class='add'>+	kfree(tidbuf-&gt;pages);</div><div class='ctx'> 	kfree(tidbuf-&gt;psets);</div><div class='add'>+	kfree(tidbuf);</div><div class='ctx'> 	kfree(tidlist);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+fail_unprogram:</div><div class='add'>+	/* unprogram, unmap, and unpin all allocated TIDs */</div><div class='add'>+	tinfo-&gt;tidlist = (unsigned long)tidlist;</div><div class='add'>+	hfi1_user_exp_rcv_clear(fd, tinfo);</div><div class='add'>+	tinfo-&gt;tidlist = 0;</div><div class='add'>+	pinned = 0;		/* nothing left to unpin */</div><div class='add'>+	pageset_count = 0;	/* nothing left reserved */</div><div class='add'>+fail_unreserve:</div><div class='add'>+	spin_lock(&amp;fd-&gt;tid_lock);</div><div class='add'>+	fd-&gt;tid_used -= pageset_count;</div><div class='add'>+	spin_unlock(&amp;fd-&gt;tid_lock);</div><div class='add'>+fail_unpin:</div><div class='add'>+	if (fd-&gt;use_mn)</div><div class='add'>+		mmu_interval_notifier_remove(&amp;tidbuf-&gt;notifier);</div><div class='add'>+	if (pinned &gt; 0)</div><div class='add'>+		unpin_rcv_pages(fd, tidbuf, NULL, 0, pinned, false);</div><div class='add'>+fail_release_mem:</div><div class='ctx'> 	kfree(tidbuf-&gt;pages);</div><div class='add'>+	kfree(tidbuf-&gt;psets);</div><div class='ctx'> 	kfree(tidbuf);</div><div class='del'>-	return ret &gt; 0 ? 0 : ret;</div><div class='add'>+	kfree(tidlist);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int hfi1_user_exp_rcv_clear(struct hfi1_filedata *fd,</div><div class='hunk'>@@ -512,14 +491,14 @@ int hfi1_user_exp_rcv_clear(struct hfi1_filedata *fd,</div><div class='ctx'> 	if (unlikely(tinfo-&gt;tidcnt &gt; fd-&gt;tid_used))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	tidinfo = memdup_user(u64_to_user_ptr(tinfo-&gt;tidlist),</div><div class='del'>-			      sizeof(tidinfo[0]) * tinfo-&gt;tidcnt);</div><div class='add'>+	tidinfo = memdup_array_user(u64_to_user_ptr(tinfo-&gt;tidlist),</div><div class='add'>+				    tinfo-&gt;tidcnt, sizeof(tidinfo[0]));</div><div class='ctx'> 	if (IS_ERR(tidinfo))</div><div class='ctx'> 		return PTR_ERR(tidinfo);</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;uctxt-&gt;exp_mutex);</div><div class='ctx'> 	for (tididx = 0; tididx &lt; tinfo-&gt;tidcnt; tididx++) {</div><div class='del'>-		ret = unprogram_rcvarray(fd, tidinfo[tididx], NULL);</div><div class='add'>+		ret = unprogram_rcvarray(fd, tidinfo[tididx]);</div><div class='ctx'> 		if (ret) {</div><div class='ctx'> 			hfi1_cdbg(TID, "Failed to unprogram rcv array %d",</div><div class='ctx'> 				  ret);</div><div class='hunk'>@@ -656,7 +635,6 @@ static u32 find_phys_blocks(struct tid_user_buf *tidbuf, unsigned int npages)</div><div class='ctx'>  *	  struct tid_pageset holding information on physically contiguous</div><div class='ctx'>  *	  chunks from the user buffer), and other fields.</div><div class='ctx'>  * @grp: RcvArray group</div><div class='del'>- * @start: starting index into sets array</div><div class='ctx'>  * @count: number of struct tid_pageset's to program</div><div class='ctx'>  * @tidlist: the array of u32 elements when the information about the</div><div class='ctx'>  *           programmed RcvArray entries is to be encoded.</div><div class='hunk'>@@ -676,14 +654,14 @@ static u32 find_phys_blocks(struct tid_user_buf *tidbuf, unsigned int npages)</div><div class='ctx'>  * number of RcvArray entries programmed.</div><div class='ctx'>  */</div><div class='ctx'> static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *tbuf,</div><div class='del'>-			    struct tid_group *grp,</div><div class='del'>-			    unsigned int start, u16 count,</div><div class='add'>+			    struct tid_group *grp, u16 count,</div><div class='ctx'> 			    u32 *tidlist, unsigned int *tididx,</div><div class='ctx'> 			    unsigned int *pmapped)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_ctxtdata *uctxt = fd-&gt;uctxt;</div><div class='ctx'> 	struct hfi1_devdata *dd = uctxt-&gt;dd;</div><div class='ctx'> 	u16 idx;</div><div class='add'>+	unsigned int start = *tididx;</div><div class='ctx'> 	u32 tidinfo = 0, rcventry, useidx = 0;</div><div class='ctx'> 	int mapped = 0;</div><div class='ctx'> </div><div class='hunk'>@@ -728,8 +706,7 @@ static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *tbuf,</div><div class='ctx'> 			return ret;</div><div class='ctx'> 		mapped += npages;</div><div class='ctx'> </div><div class='del'>-		tidinfo = rcventry2tidinfo(rcventry - uctxt-&gt;expected_base) |</div><div class='del'>-			EXP_TID_SET(LEN, npages);</div><div class='add'>+		tidinfo = create_tid(rcventry - uctxt-&gt;expected_base, npages);</div><div class='ctx'> 		tidlist[(*tididx)++] = tidinfo;</div><div class='ctx'> 		grp-&gt;used++;</div><div class='ctx'> 		grp-&gt;map |= 1 &lt;&lt; useidx++;</div><div class='hunk'>@@ -759,14 +736,12 @@ static int set_rcvarray_entry(struct hfi1_filedata *fd,</div><div class='ctx'> 	 * Allocate the node first so we can handle a potential</div><div class='ctx'> 	 * failure before we've programmed anything.</div><div class='ctx'> 	 */</div><div class='del'>-	node = kzalloc(sizeof(*node) + (sizeof(struct page *) * npages),</div><div class='del'>-		       GFP_KERNEL);</div><div class='add'>+	node = kzalloc(struct_size(node, pages, npages), GFP_KERNEL);</div><div class='ctx'> 	if (!node)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	phys = pci_map_single(dd-&gt;pcidev,</div><div class='del'>-			      __va(page_to_phys(pages[0])),</div><div class='del'>-			      npages * PAGE_SIZE, PCI_DMA_FROMDEVICE);</div><div class='add'>+	phys = dma_map_single(&amp;dd-&gt;pcidev-&gt;dev, __va(page_to_phys(pages[0])),</div><div class='add'>+			      npages * PAGE_SIZE, DMA_FROM_DEVICE);</div><div class='ctx'> 	if (dma_mapping_error(&amp;dd-&gt;pcidev-&gt;dev, phys)) {</div><div class='ctx'> 		dd_dev_err(dd, "Failed to DMA map Exp Rcv pages 0x%llx\n",</div><div class='ctx'> 			   phys);</div><div class='hunk'>@@ -774,86 +749,100 @@ static int set_rcvarray_entry(struct hfi1_filedata *fd,</div><div class='ctx'> 		return -EFAULT;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	node-&gt;mmu.addr = tbuf-&gt;vaddr + (pageidx * PAGE_SIZE);</div><div class='del'>-	node-&gt;mmu.len = npages * PAGE_SIZE;</div><div class='add'>+	node-&gt;fdata = fd;</div><div class='add'>+	mutex_init(&amp;node-&gt;invalidate_mutex);</div><div class='ctx'> 	node-&gt;phys = page_to_phys(pages[0]);</div><div class='ctx'> 	node-&gt;npages = npages;</div><div class='ctx'> 	node-&gt;rcventry = rcventry;</div><div class='ctx'> 	node-&gt;dma_addr = phys;</div><div class='ctx'> 	node-&gt;grp = grp;</div><div class='ctx'> 	node-&gt;freed = false;</div><div class='del'>-	memcpy(node-&gt;pages, pages, sizeof(struct page *) * npages);</div><div class='add'>+	memcpy(node-&gt;pages, pages, flex_array_size(node, pages, npages));</div><div class='ctx'> </div><div class='del'>-	if (!fd-&gt;handler)</div><div class='del'>-		ret = tid_rb_insert(fd, &amp;node-&gt;mmu);</div><div class='del'>-	else</div><div class='del'>-		ret = hfi1_mmu_rb_insert(fd-&gt;handler, &amp;node-&gt;mmu);</div><div class='del'>-</div><div class='del'>-	if (ret) {</div><div class='del'>-		hfi1_cdbg(TID, "Failed to insert RB node %u 0x%lx, 0x%lx %d",</div><div class='del'>-			  node-&gt;rcventry, node-&gt;mmu.addr, node-&gt;phys, ret);</div><div class='del'>-		pci_unmap_single(dd-&gt;pcidev, phys, npages * PAGE_SIZE,</div><div class='del'>-				 PCI_DMA_FROMDEVICE);</div><div class='del'>-		kfree(node);</div><div class='del'>-		return -EFAULT;</div><div class='add'>+	if (fd-&gt;use_mn) {</div><div class='add'>+		ret = mmu_interval_notifier_insert(</div><div class='add'>+			&amp;node-&gt;notifier, current-&gt;mm,</div><div class='add'>+			tbuf-&gt;vaddr + (pageidx * PAGE_SIZE), npages * PAGE_SIZE,</div><div class='add'>+			&amp;tid_mn_ops);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto out_unmap;</div><div class='ctx'> 	}</div><div class='add'>+	fd-&gt;entry_to_rb[node-&gt;rcventry - uctxt-&gt;expected_base] = node;</div><div class='add'>+</div><div class='ctx'> 	hfi1_put_tid(dd, rcventry, PT_EXPECTED, phys, ilog2(npages) + 1);</div><div class='ctx'> 	trace_hfi1_exp_tid_reg(uctxt-&gt;ctxt, fd-&gt;subctxt, rcventry, npages,</div><div class='del'>-			       node-&gt;mmu.addr, node-&gt;phys, phys);</div><div class='add'>+			       node-&gt;notifier.interval_tree.start, node-&gt;phys,</div><div class='add'>+			       phys);</div><div class='ctx'> 	return 0;</div><div class='add'>+</div><div class='add'>+out_unmap:</div><div class='add'>+	hfi1_cdbg(TID, "Failed to insert RB node %u 0x%lx, 0x%lx %d",</div><div class='add'>+		  node-&gt;rcventry, node-&gt;notifier.interval_tree.start,</div><div class='add'>+		  node-&gt;phys, ret);</div><div class='add'>+	dma_unmap_single(&amp;dd-&gt;pcidev-&gt;dev, phys, npages * PAGE_SIZE,</div><div class='add'>+			 DMA_FROM_DEVICE);</div><div class='add'>+	kfree(node);</div><div class='add'>+	return -EFAULT;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo,</div><div class='del'>-			      struct tid_group **grp)</div><div class='add'>+static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_ctxtdata *uctxt = fd-&gt;uctxt;</div><div class='ctx'> 	struct hfi1_devdata *dd = uctxt-&gt;dd;</div><div class='ctx'> 	struct tid_rb_node *node;</div><div class='del'>-	u8 tidctrl = EXP_TID_GET(tidinfo, CTRL);</div><div class='add'>+	u32 tidctrl = EXP_TID_GET(tidinfo, CTRL);</div><div class='ctx'> 	u32 tididx = EXP_TID_GET(tidinfo, IDX) &lt;&lt; 1, rcventry;</div><div class='ctx'> </div><div class='del'>-	if (tididx &gt;= uctxt-&gt;expected_count) {</div><div class='del'>-		dd_dev_err(dd, "Invalid RcvArray entry (%u) index for ctxt %u\n",</div><div class='del'>-			   tididx, uctxt-&gt;ctxt);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (tidctrl == 0x3)</div><div class='add'>+	if (tidctrl == 0x3 || tidctrl == 0x0)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	rcventry = tididx + (tidctrl - 1);</div><div class='ctx'> </div><div class='add'>+	if (rcventry &gt;= uctxt-&gt;expected_count) {</div><div class='add'>+		dd_dev_err(dd, "Invalid RcvArray entry (%u) index for ctxt %u\n",</div><div class='add'>+			   rcventry, uctxt-&gt;ctxt);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	node = fd-&gt;entry_to_rb[rcventry];</div><div class='ctx'> 	if (!node || node-&gt;rcventry != (uctxt-&gt;expected_base + rcventry))</div><div class='ctx'> 		return -EBADF;</div><div class='ctx'> </div><div class='del'>-	if (grp)</div><div class='del'>-		*grp = node-&gt;grp;</div><div class='del'>-</div><div class='del'>-	if (!fd-&gt;handler)</div><div class='del'>-		cacheless_tid_rb_remove(fd, node);</div><div class='del'>-	else</div><div class='del'>-		hfi1_mmu_rb_remove(fd-&gt;handler, &amp;node-&gt;mmu);</div><div class='add'>+	if (fd-&gt;use_mn)</div><div class='add'>+		mmu_interval_notifier_remove(&amp;node-&gt;notifier);</div><div class='add'>+	cacheless_tid_rb_remove(fd, node);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node)</div><div class='add'>+static void __clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_ctxtdata *uctxt = fd-&gt;uctxt;</div><div class='ctx'> 	struct hfi1_devdata *dd = uctxt-&gt;dd;</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;node-&gt;invalidate_mutex);</div><div class='add'>+	if (node-&gt;freed)</div><div class='add'>+		goto done;</div><div class='add'>+	node-&gt;freed = true;</div><div class='add'>+</div><div class='ctx'> 	trace_hfi1_exp_tid_unreg(uctxt-&gt;ctxt, fd-&gt;subctxt, node-&gt;rcventry,</div><div class='del'>-				 node-&gt;npages, node-&gt;mmu.addr, node-&gt;phys,</div><div class='add'>+				 node-&gt;npages,</div><div class='add'>+				 node-&gt;notifier.interval_tree.start, node-&gt;phys,</div><div class='ctx'> 				 node-&gt;dma_addr);</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * Make sure device has seen the write before we unpin the</div><div class='del'>-	 * pages.</div><div class='del'>-	 */</div><div class='add'>+	/* Make sure device has seen the write before pages are unpinned */</div><div class='ctx'> 	hfi1_put_tid(dd, node-&gt;rcventry, PT_INVALID_FLUSH, 0, 0);</div><div class='ctx'> </div><div class='ctx'> 	unpin_rcv_pages(fd, NULL, node, 0, node-&gt;npages, true);</div><div class='add'>+done:</div><div class='add'>+	mutex_unlock(&amp;node-&gt;invalidate_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node)</div><div class='add'>+{</div><div class='add'>+	struct hfi1_ctxtdata *uctxt = fd-&gt;uctxt;</div><div class='add'>+</div><div class='add'>+	__clear_tid_node(fd, node);</div><div class='ctx'> </div><div class='ctx'> 	node-&gt;grp-&gt;used--;</div><div class='ctx'> 	node-&gt;grp-&gt;map &amp;= ~(1 &lt;&lt; (node-&gt;rcventry - node-&gt;grp-&gt;base));</div><div class='hunk'>@@ -891,39 +880,43 @@ static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt,</div><div class='ctx'> 				if (!node || node-&gt;rcventry != rcventry)</div><div class='ctx'> 					continue;</div><div class='ctx'> </div><div class='add'>+				if (fd-&gt;use_mn)</div><div class='add'>+					mmu_interval_notifier_remove(</div><div class='add'>+						&amp;node-&gt;notifier);</div><div class='ctx'> 				cacheless_tid_rb_remove(fd, node);</div><div class='ctx'> 			}</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/*</div><div class='del'>- * Always return 0 from this function.  A non-zero return indicates that the</div><div class='del'>- * remove operation will be called and that memory should be unpinned.</div><div class='del'>- * However, the driver cannot unpin out from under PSM.  Instead, retain the</div><div class='del'>- * memory (by returning 0) and inform PSM that the memory is going away.  PSM</div><div class='del'>- * will call back later when it has removed the memory from its list.</div><div class='del'>- */</div><div class='del'>-static int tid_rb_invalidate(void *arg, struct mmu_rb_node *mnode)</div><div class='add'>+static bool tid_rb_invalidate(struct mmu_interval_notifier *mni,</div><div class='add'>+			      const struct mmu_notifier_range *range,</div><div class='add'>+			      unsigned long cur_seq)</div><div class='ctx'> {</div><div class='del'>-	struct hfi1_filedata *fdata = arg;</div><div class='del'>-	struct hfi1_ctxtdata *uctxt = fdata-&gt;uctxt;</div><div class='ctx'> 	struct tid_rb_node *node =</div><div class='del'>-		container_of(mnode, struct tid_rb_node, mmu);</div><div class='add'>+		container_of(mni, struct tid_rb_node, notifier);</div><div class='add'>+	struct hfi1_filedata *fdata = node-&gt;fdata;</div><div class='add'>+	struct hfi1_ctxtdata *uctxt = fdata-&gt;uctxt;</div><div class='ctx'> </div><div class='ctx'> 	if (node-&gt;freed)</div><div class='del'>-		return 0;</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	/* take action only if unmapping */</div><div class='add'>+	if (range-&gt;event != MMU_NOTIFY_UNMAP)</div><div class='add'>+		return true;</div><div class='ctx'> </div><div class='del'>-	trace_hfi1_exp_tid_inval(uctxt-&gt;ctxt, fdata-&gt;subctxt, node-&gt;mmu.addr,</div><div class='add'>+	trace_hfi1_exp_tid_inval(uctxt-&gt;ctxt, fdata-&gt;subctxt,</div><div class='add'>+				 node-&gt;notifier.interval_tree.start,</div><div class='ctx'> 				 node-&gt;rcventry, node-&gt;npages, node-&gt;dma_addr);</div><div class='del'>-	node-&gt;freed = true;</div><div class='add'>+</div><div class='add'>+	/* clear the hardware rcvarray entry */</div><div class='add'>+	__clear_tid_node(fdata, node);</div><div class='ctx'> </div><div class='ctx'> 	spin_lock(&amp;fdata-&gt;invalid_lock);</div><div class='ctx'> 	if (fdata-&gt;invalid_tid_idx &lt; uctxt-&gt;expected_count) {</div><div class='ctx'> 		fdata-&gt;invalid_tids[fdata-&gt;invalid_tid_idx] =</div><div class='del'>-			rcventry2tidinfo(node-&gt;rcventry - uctxt-&gt;expected_base);</div><div class='del'>-		fdata-&gt;invalid_tids[fdata-&gt;invalid_tid_idx] |=</div><div class='del'>-			EXP_TID_SET(LEN, node-&gt;npages);</div><div class='add'>+			create_tid(node-&gt;rcventry - uctxt-&gt;expected_base,</div><div class='add'>+				   node-&gt;npages);</div><div class='ctx'> 		if (!fdata-&gt;invalid_tid_idx) {</div><div class='ctx'> 			unsigned long *ev;</div><div class='ctx'> </div><div class='hunk'>@@ -943,18 +936,24 @@ static int tid_rb_invalidate(void *arg, struct mmu_rb_node *mnode)</div><div class='ctx'> 		fdata-&gt;invalid_tid_idx++;</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock(&amp;fdata-&gt;invalid_lock);</div><div class='del'>-	return 0;</div><div class='add'>+	return true;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int tid_rb_insert(void *arg, struct mmu_rb_node *node)</div><div class='add'>+static bool tid_cover_invalidate(struct mmu_interval_notifier *mni,</div><div class='add'>+			         const struct mmu_notifier_range *range,</div><div class='add'>+			         unsigned long cur_seq)</div><div class='ctx'> {</div><div class='del'>-	struct hfi1_filedata *fdata = arg;</div><div class='del'>-	struct tid_rb_node *tnode =</div><div class='del'>-		container_of(node, struct tid_rb_node, mmu);</div><div class='del'>-	u32 base = fdata-&gt;uctxt-&gt;expected_base;</div><div class='add'>+	struct tid_user_buf *tidbuf =</div><div class='add'>+		container_of(mni, struct tid_user_buf, notifier);</div><div class='add'>+</div><div class='add'>+	/* take action only if unmapping */</div><div class='add'>+	if (range-&gt;event == MMU_NOTIFY_UNMAP) {</div><div class='add'>+		mutex_lock(&amp;tidbuf-&gt;cover_mutex);</div><div class='add'>+		mmu_interval_set_seq(mni, cur_seq);</div><div class='add'>+		mutex_unlock(&amp;tidbuf-&gt;cover_mutex);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	fdata-&gt;entry_to_rb[tnode-&gt;rcventry - base] = tnode;</div><div class='del'>-	return 0;</div><div class='add'>+	return true;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void cacheless_tid_rb_remove(struct hfi1_filedata *fdata,</div><div class='hunk'>@@ -965,12 +964,3 @@ static void cacheless_tid_rb_remove(struct hfi1_filedata *fdata,</div><div class='ctx'> 	fdata-&gt;entry_to_rb[tnode-&gt;rcventry - base] = NULL;</div><div class='ctx'> 	clear_tid_node(fdata, tnode);</div><div class='ctx'> }</div><div class='del'>-</div><div class='del'>-static void tid_rb_remove(void *arg, struct mmu_rb_node *node)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_filedata *fdata = arg;</div><div class='del'>-	struct tid_rb_node *tnode =</div><div class='del'>-		container_of(node, struct tid_rb_node, mmu);</div><div class='del'>-</div><div class='del'>-	cacheless_tid_rb_remove(fdata, tnode);</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/user_exp_rcv.h b/drivers/infiniband/hw/hfi1/user_exp_rcv.h<br/>index 43b105de1d54..055726f7c139 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/user_exp_rcv.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/user_exp_rcv.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/user_exp_rcv.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/user_exp_rcv.h</a></div><div class='hunk'>@@ -1,52 +1,12 @@</div><div class='del'>-#ifndef _HFI1_USER_EXP_RCV_H</div><div class='del'>-#define _HFI1_USER_EXP_RCV_H</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='add'>+ * Copyright(c) 2020 - Cornelis Networks, Inc.</div><div class='ctx'>  * Copyright(c) 2015 - 2017 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='add'>+#ifndef _HFI1_USER_EXP_RCV_H</div><div class='add'>+#define _HFI1_USER_EXP_RCV_H</div><div class='add'>+</div><div class='ctx'> #include "hfi.h"</div><div class='ctx'> #include "exp_rcv.h"</div><div class='ctx'> </div><div class='hunk'>@@ -56,6 +16,8 @@ struct tid_pageset {</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct tid_user_buf {</div><div class='add'>+	struct mmu_interval_notifier notifier;</div><div class='add'>+	struct mutex cover_mutex;</div><div class='ctx'> 	unsigned long vaddr;</div><div class='ctx'> 	unsigned long length;</div><div class='ctx'> 	unsigned int npages;</div><div class='hunk'>@@ -65,14 +27,16 @@ struct tid_user_buf {</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct tid_rb_node {</div><div class='del'>-	struct mmu_rb_node mmu;</div><div class='add'>+	struct mmu_interval_notifier notifier;</div><div class='add'>+	struct hfi1_filedata *fdata;</div><div class='add'>+	struct mutex invalidate_mutex; /* covers hw removal */</div><div class='ctx'> 	unsigned long phys;</div><div class='ctx'> 	struct tid_group *grp;</div><div class='ctx'> 	u32 rcventry;</div><div class='ctx'> 	dma_addr_t dma_addr;</div><div class='ctx'> 	bool freed;</div><div class='ctx'> 	unsigned int npages;</div><div class='del'>-	struct page *pages[0];</div><div class='add'>+	struct page *pages[] __counted_by(npages);</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static inline int num_user_pages(unsigned long addr,</div><div class='hunk'>@@ -94,4 +58,9 @@ int hfi1_user_exp_rcv_clear(struct hfi1_filedata *fd,</div><div class='ctx'> int hfi1_user_exp_rcv_invalid(struct hfi1_filedata *fd,</div><div class='ctx'> 			      struct hfi1_tid_info *tinfo);</div><div class='ctx'> </div><div class='add'>+static inline struct mm_struct *mm_from_tid_node(struct tid_rb_node *node)</div><div class='add'>+{</div><div class='add'>+	return node-&gt;notifier.mm;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> #endif /* _HFI1_USER_EXP_RCV_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/user_pages.c b/drivers/infiniband/hw/hfi1/user_pages.c<br/>index 02eee8eff1db..c77913a7920f 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/user_pages.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/user_pages.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/user_pages.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/user_pages.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015-2017 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/mm.h&gt;</div><div class='hunk'>@@ -71,33 +29,52 @@ MODULE_PARM_DESC(cache_size, "Send and receive side cache size limit (in MB)");</div><div class='ctx'> bool hfi1_can_pin_pages(struct hfi1_devdata *dd, struct mm_struct *mm,</div><div class='ctx'> 			u32 nlocked, u32 npages)</div><div class='ctx'> {</div><div class='del'>-	unsigned long ulimit = rlimit(RLIMIT_MEMLOCK), pinned, cache_limit,</div><div class='del'>-		size = (cache_size * (1UL &lt;&lt; 20)); /* convert to bytes */</div><div class='del'>-	unsigned int usr_ctxts =</div><div class='del'>-			dd-&gt;num_rcv_contexts - dd-&gt;first_dyn_alloc_ctxt;</div><div class='del'>-	bool can_lock = capable(CAP_IPC_LOCK);</div><div class='add'>+	unsigned long ulimit_pages;</div><div class='add'>+	unsigned long cache_limit_pages;</div><div class='add'>+	unsigned int usr_ctxts;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='del'>-	 * Calculate per-cache size. The calculation below uses only a quarter</div><div class='del'>-	 * of the available per-context limit. This leaves space for other</div><div class='del'>-	 * pinning. Should we worry about shared ctxts?</div><div class='add'>+	 * Perform RLIMIT_MEMLOCK based checks unless CAP_IPC_LOCK is present.</div><div class='ctx'> 	 */</div><div class='del'>-	cache_limit = (ulimit / usr_ctxts) / 4;</div><div class='del'>-</div><div class='del'>-	/* If ulimit isn't set to "unlimited" and is smaller than cache_size. */</div><div class='del'>-	if (ulimit != (-1UL) &amp;&amp; size &gt; cache_limit)</div><div class='del'>-		size = cache_limit;</div><div class='del'>-</div><div class='del'>-	/* Convert to number of pages */</div><div class='del'>-	size = DIV_ROUND_UP(size, PAGE_SIZE);</div><div class='del'>-</div><div class='del'>-	pinned = atomic64_read(&amp;mm-&gt;pinned_vm);</div><div class='add'>+	if (!capable(CAP_IPC_LOCK)) {</div><div class='add'>+		ulimit_pages =</div><div class='add'>+			DIV_ROUND_DOWN_ULL(rlimit(RLIMIT_MEMLOCK), PAGE_SIZE);</div><div class='add'>+</div><div class='add'>+		/*</div><div class='add'>+		 * Pinning these pages would exceed this process's locked memory</div><div class='add'>+		 * limit.</div><div class='add'>+		 */</div><div class='add'>+		if (atomic64_read(&amp;mm-&gt;pinned_vm) + npages &gt; ulimit_pages)</div><div class='add'>+			return false;</div><div class='add'>+</div><div class='add'>+		/*</div><div class='add'>+		 * Only allow 1/4 of the user's RLIMIT_MEMLOCK to be used for HFI</div><div class='add'>+		 * caches.  This fraction is then equally distributed among all</div><div class='add'>+		 * existing user contexts.  Note that if RLIMIT_MEMLOCK is</div><div class='add'>+		 * 'unlimited' (-1), the value of this limit will be &gt; 2^42 pages</div><div class='add'>+		 * (2^64 / 2^12 / 2^8 / 2^2).</div><div class='add'>+		 *</div><div class='add'>+		 * The effectiveness of this check may be reduced if I/O occurs on</div><div class='add'>+		 * some user contexts before all user contexts are created.  This</div><div class='add'>+		 * check assumes that this process is the only one using this</div><div class='add'>+		 * context (e.g., the corresponding fd was not passed to another</div><div class='add'>+		 * process for concurrent access) as there is no per-context,</div><div class='add'>+		 * per-process tracking of pinned pages.  It also assumes that each</div><div class='add'>+		 * user context has only one cache to limit.</div><div class='add'>+		 */</div><div class='add'>+		usr_ctxts = dd-&gt;num_rcv_contexts - dd-&gt;first_dyn_alloc_ctxt;</div><div class='add'>+		if (nlocked + npages &gt; (ulimit_pages / usr_ctxts / 4))</div><div class='add'>+			return false;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	/* First, check the absolute limit against all pinned pages. */</div><div class='del'>-	if (pinned + npages &gt;= ulimit &amp;&amp; !can_lock)</div><div class='add'>+	/*</div><div class='add'>+	 * Pinning these pages would exceed the size limit for this cache.</div><div class='add'>+	 */</div><div class='add'>+	cache_limit_pages = cache_size * (1024 * 1024) / PAGE_SIZE;</div><div class='add'>+	if (nlocked + npages &gt; cache_limit_pages)</div><div class='ctx'> 		return false;</div><div class='ctx'> </div><div class='del'>-	return ((nlocked + npages) &lt;= size) || can_lock;</div><div class='add'>+	return true;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t npages,</div><div class='hunk'>@@ -106,7 +83,7 @@ int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t np</div><div class='ctx'> 	int ret;</div><div class='ctx'> 	unsigned int gup_flags = FOLL_LONGTERM | (writable ? FOLL_WRITE : 0);</div><div class='ctx'> </div><div class='del'>-	ret = get_user_pages_fast(vaddr, npages, gup_flags, pages);</div><div class='add'>+	ret = pin_user_pages_fast(vaddr, npages, gup_flags, pages);</div><div class='ctx'> 	if (ret &lt; 0)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='hunk'>@@ -118,13 +95,7 @@ int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t np</div><div class='ctx'> void hfi1_release_user_pages(struct mm_struct *mm, struct page **p,</div><div class='ctx'> 			     size_t npages, bool dirty)</div><div class='ctx'> {</div><div class='del'>-	size_t i;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; npages; i++) {</div><div class='del'>-		if (dirty)</div><div class='del'>-			set_page_dirty_lock(p[i]);</div><div class='del'>-		put_page(p[i]);</div><div class='del'>-	}</div><div class='add'>+	unpin_user_pages_dirty_lock(p, npages, dirty);</div><div class='ctx'> </div><div class='ctx'> 	if (mm) { /* during close after signal, mm can be NULL */</div><div class='ctx'> 		atomic64_sub(npages, &amp;mm-&gt;pinned_vm);</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/user_sdma.c b/drivers/infiniband/hw/hfi1/user_sdma.c<br/>index 8bfbc6d7ea34..9b1aece1b080 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/user_sdma.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/user_sdma.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/user_sdma.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/user_sdma.c</a></div><div class='hunk'>@@ -1,49 +1,9 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='add'>+ * Copyright(c) 2020 - 2023 Cornelis Networks, Inc.</div><div class='ctx'>  * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='add'>+</div><div class='ctx'> #include &lt;linux/mm.h&gt;</div><div class='ctx'> #include &lt;linux/types.h&gt;</div><div class='ctx'> #include &lt;linux/device.h&gt;</div><div class='hunk'>@@ -64,7 +24,6 @@</div><div class='ctx'> </div><div class='ctx'> #include "hfi.h"</div><div class='ctx'> #include "sdma.h"</div><div class='del'>-#include "mmu_rb.h"</div><div class='ctx'> #include "user_sdma.h"</div><div class='ctx'> #include "verbs.h"  /* for the headers */</div><div class='ctx'> #include "common.h" /* for struct hfi1_tid_info */</div><div class='hunk'>@@ -79,11 +38,7 @@ static unsigned initial_pkt_count = 8;</div><div class='ctx'> static int user_sdma_send_pkts(struct user_sdma_request *req, u16 maxpkts);</div><div class='ctx'> static void user_sdma_txreq_cb(struct sdma_txreq *txreq, int status);</div><div class='ctx'> static inline void pq_update(struct hfi1_user_sdma_pkt_q *pq);</div><div class='del'>-static void user_sdma_free_request(struct user_sdma_request *req, bool unpin);</div><div class='del'>-static int pin_vector_pages(struct user_sdma_request *req,</div><div class='del'>-			    struct user_sdma_iovec *iovec);</div><div class='del'>-static void unpin_vector_pages(struct mm_struct *mm, struct page **pages,</div><div class='del'>-			       unsigned start, unsigned npages);</div><div class='add'>+static void user_sdma_free_request(struct user_sdma_request *req);</div><div class='ctx'> static int check_header_template(struct user_sdma_request *req,</div><div class='ctx'> 				 struct hfi1_pkt_header *hdr, u32 lrhlen,</div><div class='ctx'> 				 u32 datalen);</div><div class='hunk'>@@ -105,21 +60,6 @@ static int defer_packet_queue(</div><div class='ctx'> 	uint seq,</div><div class='ctx'> 	bool pkts_sent);</div><div class='ctx'> static void activate_packet_queue(struct iowait *wait, int reason);</div><div class='del'>-static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr,</div><div class='del'>-			   unsigned long len);</div><div class='del'>-static int sdma_rb_insert(void *arg, struct mmu_rb_node *mnode);</div><div class='del'>-static int sdma_rb_evict(void *arg, struct mmu_rb_node *mnode,</div><div class='del'>-			 void *arg2, bool *stop);</div><div class='del'>-static void sdma_rb_remove(void *arg, struct mmu_rb_node *mnode);</div><div class='del'>-static int sdma_rb_invalidate(void *arg, struct mmu_rb_node *mnode);</div><div class='del'>-</div><div class='del'>-static struct mmu_rb_ops sdma_rb_ops = {</div><div class='del'>-	.filter = sdma_rb_filter,</div><div class='del'>-	.insert = sdma_rb_insert,</div><div class='del'>-	.evict = sdma_rb_evict,</div><div class='del'>-	.remove = sdma_rb_remove,</div><div class='del'>-	.invalidate = sdma_rb_invalidate</div><div class='del'>-};</div><div class='ctx'> </div><div class='ctx'> static int defer_packet_queue(</div><div class='ctx'> 	struct sdma_engine *sde,</div><div class='hunk'>@@ -130,27 +70,26 @@ static int defer_packet_queue(</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_user_sdma_pkt_q *pq =</div><div class='ctx'> 		container_of(wait-&gt;iow, struct hfi1_user_sdma_pkt_q, busy);</div><div class='del'>-	struct user_sdma_txreq *tx =</div><div class='del'>-		container_of(txreq, struct user_sdma_txreq, txreq);</div><div class='ctx'> </div><div class='del'>-	if (sdma_progress(sde, seq, txreq)) {</div><div class='del'>-		if (tx-&gt;busycount++ &lt; MAX_DEFER_RETRY_COUNT)</div><div class='del'>-			goto eagain;</div><div class='del'>-	}</div><div class='add'>+	write_seqlock(&amp;sde-&gt;waitlock);</div><div class='add'>+	trace_hfi1_usdma_defer(pq, sde, &amp;pq-&gt;busy);</div><div class='add'>+	if (sdma_progress(sde, seq, txreq))</div><div class='add'>+		goto eagain;</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * We are assuming that if the list is enqueued somewhere, it</div><div class='ctx'> 	 * is to the dmawait list since that is the only place where</div><div class='ctx'> 	 * it is supposed to be enqueued.</div><div class='ctx'> 	 */</div><div class='ctx'> 	xchg(&amp;pq-&gt;state, SDMA_PKT_Q_DEFERRED);</div><div class='del'>-	write_seqlock(&amp;sde-&gt;waitlock);</div><div class='ctx'> 	if (list_empty(&amp;pq-&gt;busy.list)) {</div><div class='add'>+		pq-&gt;busy.lock = &amp;sde-&gt;waitlock;</div><div class='ctx'> 		iowait_get_priority(&amp;pq-&gt;busy);</div><div class='ctx'> 		iowait_queue(pkts_sent, &amp;pq-&gt;busy, &amp;sde-&gt;dmawait);</div><div class='ctx'> 	}</div><div class='ctx'> 	write_sequnlock(&amp;sde-&gt;waitlock);</div><div class='ctx'> 	return -EBUSY;</div><div class='ctx'> eagain:</div><div class='add'>+	write_sequnlock(&amp;sde-&gt;waitlock);</div><div class='ctx'> 	return -EAGAIN;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -158,6 +97,8 @@ static void activate_packet_queue(struct iowait *wait, int reason)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_user_sdma_pkt_q *pq =</div><div class='ctx'> 		container_of(wait, struct hfi1_user_sdma_pkt_q, busy);</div><div class='add'>+</div><div class='add'>+	trace_hfi1_usdma_activate(pq, wait, reason);</div><div class='ctx'> 	xchg(&amp;pq-&gt;state, SDMA_PKT_Q_ACTIVE);</div><div class='ctx'> 	wake_up(&amp;wait-&gt;wait_dma);</div><div class='ctx'> };</div><div class='hunk'>@@ -182,7 +123,6 @@ int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt,</div><div class='ctx'> 	pq = kzalloc(sizeof(*pq), GFP_KERNEL);</div><div class='ctx'> 	if (!pq)</div><div class='ctx'> 		return -ENOMEM;</div><div class='del'>-</div><div class='ctx'> 	pq-&gt;dd = dd;</div><div class='ctx'> 	pq-&gt;ctxt = uctxt-&gt;ctxt;</div><div class='ctx'> 	pq-&gt;subctxt = fd-&gt;subctxt;</div><div class='hunk'>@@ -190,7 +130,6 @@ int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt,</div><div class='ctx'> 	atomic_set(&amp;pq-&gt;n_reqs, 0);</div><div class='ctx'> 	init_waitqueue_head(&amp;pq-&gt;wait);</div><div class='ctx'> 	atomic_set(&amp;pq-&gt;n_locked, 0);</div><div class='del'>-	pq-&gt;mm = fd-&gt;mm;</div><div class='ctx'> </div><div class='ctx'> 	iowait_init(&amp;pq-&gt;busy, 0, NULL, NULL, defer_packet_queue,</div><div class='ctx'> 		    activate_packet_queue, NULL, NULL);</div><div class='hunk'>@@ -202,9 +141,7 @@ int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt,</div><div class='ctx'> 	if (!pq-&gt;reqs)</div><div class='ctx'> 		goto pq_reqs_nomem;</div><div class='ctx'> </div><div class='del'>-	pq-&gt;req_in_use = kcalloc(BITS_TO_LONGS(hfi1_sdma_comp_ring_size),</div><div class='del'>-				 sizeof(*pq-&gt;req_in_use),</div><div class='del'>-				 GFP_KERNEL);</div><div class='add'>+	pq-&gt;req_in_use = bitmap_zalloc(hfi1_sdma_comp_ring_size, GFP_KERNEL);</div><div class='ctx'> 	if (!pq-&gt;req_in_use)</div><div class='ctx'> 		goto pq_reqs_no_in_use;</div><div class='ctx'> </div><div class='hunk'>@@ -232,14 +169,11 @@ int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt,</div><div class='ctx'> </div><div class='ctx'> 	cq-&gt;nentries = hfi1_sdma_comp_ring_size;</div><div class='ctx'> </div><div class='del'>-	ret = hfi1_mmu_rb_register(pq, pq-&gt;mm, &amp;sdma_rb_ops, dd-&gt;pport-&gt;hfi1_wq,</div><div class='del'>-				   &amp;pq-&gt;handler);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dd_dev_err(dd, "Failed to register with MMU %d", ret);</div><div class='add'>+	ret = hfi1_init_system_pinning(pq);</div><div class='add'>+	if (ret)</div><div class='ctx'> 		goto pq_mmu_fail;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	fd-&gt;pq = pq;</div><div class='add'>+	rcu_assign_pointer(fd-&gt;pq, pq);</div><div class='ctx'> 	fd-&gt;cq = cq;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='hunk'>@@ -251,7 +185,7 @@ cq_comps_nomem:</div><div class='ctx'> cq_nomem:</div><div class='ctx'> 	kmem_cache_destroy(pq-&gt;txreq_cache);</div><div class='ctx'> pq_txreq_nomem:</div><div class='del'>-	kfree(pq-&gt;req_in_use);</div><div class='add'>+	bitmap_free(pq-&gt;req_in_use);</div><div class='ctx'> pq_reqs_no_in_use:</div><div class='ctx'> 	kfree(pq-&gt;reqs);</div><div class='ctx'> pq_reqs_nomem:</div><div class='hunk'>@@ -260,6 +194,21 @@ pq_reqs_nomem:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void flush_pq_iowait(struct hfi1_user_sdma_pkt_q *pq)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	seqlock_t *lock = pq-&gt;busy.lock;</div><div class='add'>+</div><div class='add'>+	if (!lock)</div><div class='add'>+		return;</div><div class='add'>+	write_seqlock_irqsave(lock, flags);</div><div class='add'>+	if (!list_empty(&amp;pq-&gt;busy.list)) {</div><div class='add'>+		list_del_init(&amp;pq-&gt;busy.list);</div><div class='add'>+		pq-&gt;busy.lock = NULL;</div><div class='add'>+	}</div><div class='add'>+	write_sequnlock_irqrestore(lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd,</div><div class='ctx'> 			       struct hfi1_ctxtdata *uctxt)</div><div class='ctx'> {</div><div class='hunk'>@@ -267,20 +216,27 @@ int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd,</div><div class='ctx'> </div><div class='ctx'> 	trace_hfi1_sdma_user_free_queues(uctxt-&gt;dd, uctxt-&gt;ctxt, fd-&gt;subctxt);</div><div class='ctx'> </div><div class='del'>-	pq = fd-&gt;pq;</div><div class='add'>+	spin_lock(&amp;fd-&gt;pq_rcu_lock);</div><div class='add'>+	pq = srcu_dereference_check(fd-&gt;pq, &amp;fd-&gt;pq_srcu,</div><div class='add'>+				    lockdep_is_held(&amp;fd-&gt;pq_rcu_lock));</div><div class='ctx'> 	if (pq) {</div><div class='del'>-		if (pq-&gt;handler)</div><div class='del'>-			hfi1_mmu_rb_unregister(pq-&gt;handler);</div><div class='add'>+		rcu_assign_pointer(fd-&gt;pq, NULL);</div><div class='add'>+		spin_unlock(&amp;fd-&gt;pq_rcu_lock);</div><div class='add'>+		synchronize_srcu(&amp;fd-&gt;pq_srcu);</div><div class='add'>+		/* at this point there can be no more new requests */</div><div class='ctx'> 		iowait_sdma_drain(&amp;pq-&gt;busy);</div><div class='ctx'> 		/* Wait until all requests have been freed. */</div><div class='ctx'> 		wait_event_interruptible(</div><div class='ctx'> 			pq-&gt;wait,</div><div class='ctx'> 			!atomic_read(&amp;pq-&gt;n_reqs));</div><div class='ctx'> 		kfree(pq-&gt;reqs);</div><div class='del'>-		kfree(pq-&gt;req_in_use);</div><div class='add'>+		hfi1_free_system_pinning(pq);</div><div class='add'>+		bitmap_free(pq-&gt;req_in_use);</div><div class='ctx'> 		kmem_cache_destroy(pq-&gt;txreq_cache);</div><div class='add'>+		flush_pq_iowait(pq);</div><div class='ctx'> 		kfree(pq);</div><div class='del'>-		fd-&gt;pq = NULL;</div><div class='add'>+	} else {</div><div class='add'>+		spin_unlock(&amp;fd-&gt;pq_rcu_lock);</div><div class='ctx'> 	}</div><div class='ctx'> 	if (fd-&gt;cq) {</div><div class='ctx'> 		vfree(fd-&gt;cq-&gt;comps);</div><div class='hunk'>@@ -324,7 +280,8 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,</div><div class='ctx'> {</div><div class='ctx'> 	int ret = 0, i;</div><div class='ctx'> 	struct hfi1_ctxtdata *uctxt = fd-&gt;uctxt;</div><div class='del'>-	struct hfi1_user_sdma_pkt_q *pq = fd-&gt;pq;</div><div class='add'>+	struct hfi1_user_sdma_pkt_q *pq =</div><div class='add'>+		srcu_dereference(fd-&gt;pq, &amp;fd-&gt;pq_srcu);</div><div class='ctx'> 	struct hfi1_user_sdma_comp_q *cq = fd-&gt;cq;</div><div class='ctx'> 	struct hfi1_devdata *dd = pq-&gt;dd;</div><div class='ctx'> 	unsigned long idx = 0;</div><div class='hunk'>@@ -429,6 +386,7 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		goto free_req;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='ctx'> 	/* Copy the header from the user buffer */</div><div class='ctx'> 	ret = copy_from_user(&amp;req-&gt;hdr, iovec[idx].iov_base + sizeof(info),</div><div class='ctx'> 			     sizeof(req-&gt;hdr));</div><div class='hunk'>@@ -503,9 +461,8 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,</div><div class='ctx'> 		memcpy(&amp;req-&gt;iovs[i].iov,</div><div class='ctx'> 		       iovec + idx++,</div><div class='ctx'> 		       sizeof(req-&gt;iovs[i].iov));</div><div class='del'>-		ret = pin_vector_pages(req, &amp;req-&gt;iovs[i]);</div><div class='del'>-		if (ret) {</div><div class='del'>-			req-&gt;data_iovs = i;</div><div class='add'>+		if (req-&gt;iovs[i].iov.iov_len == 0) {</div><div class='add'>+			ret = -EINVAL;</div><div class='ctx'> 			goto free_req;</div><div class='ctx'> 		}</div><div class='ctx'> 		req-&gt;data_len += req-&gt;iovs[i].iov.iov_len;</div><div class='hunk'>@@ -537,12 +494,12 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,</div><div class='ctx'> 		 * equal to the pkt count. However, there is no way to</div><div class='ctx'> 		 * tell at this point.</div><div class='ctx'> 		 */</div><div class='del'>-		tmp = memdup_user(iovec[idx].iov_base,</div><div class='del'>-				  ntids * sizeof(*req-&gt;tids));</div><div class='add'>+		tmp = memdup_array_user(iovec[idx].iov_base,</div><div class='add'>+					ntids, sizeof(*req-&gt;tids));</div><div class='ctx'> 		if (IS_ERR(tmp)) {</div><div class='ctx'> 			ret = PTR_ERR(tmp);</div><div class='del'>-			SDMA_DBG(req, "Failed to copy %d TIDs (%d)",</div><div class='del'>-				 ntids, ret);</div><div class='add'>+			SDMA_DBG(req, "Failed to copy %d TIDs (%pe)", ntids,</div><div class='add'>+				 tmp);</div><div class='ctx'> 			goto free_req;</div><div class='ctx'> 		}</div><div class='ctx'> 		req-&gt;tids = tmp;</div><div class='hunk'>@@ -567,10 +524,6 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,</div><div class='ctx'> </div><div class='ctx'> 	set_comp_state(pq, cq, info.comp_idx, QUEUED, 0);</div><div class='ctx'> 	pq-&gt;state = SDMA_PKT_Q_ACTIVE;</div><div class='del'>-	/* Send the first N packets in the request to buy us some time */</div><div class='del'>-	ret = user_sdma_send_pkts(req, pcount);</div><div class='del'>-	if (unlikely(ret &lt; 0 &amp;&amp; ret != -EBUSY))</div><div class='del'>-		goto free_req;</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * This is a somewhat blocking send implementation.</div><div class='hunk'>@@ -581,13 +534,18 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,</div><div class='ctx'> 	while (req-&gt;seqsubmitted != req-&gt;info.npkts) {</div><div class='ctx'> 		ret = user_sdma_send_pkts(req, pcount);</div><div class='ctx'> 		if (ret &lt; 0) {</div><div class='add'>+			int we_ret;</div><div class='add'>+</div><div class='ctx'> 			if (ret != -EBUSY)</div><div class='ctx'> 				goto free_req;</div><div class='del'>-			wait_event_interruptible_timeout(</div><div class='add'>+			we_ret = wait_event_interruptible_timeout(</div><div class='ctx'> 				pq-&gt;busy.wait_dma,</div><div class='del'>-				(pq-&gt;state == SDMA_PKT_Q_ACTIVE),</div><div class='add'>+				pq-&gt;state == SDMA_PKT_Q_ACTIVE,</div><div class='ctx'> 				msecs_to_jiffies(</div><div class='ctx'> 					SDMA_IOWAIT_TIMEOUT));</div><div class='add'>+			trace_hfi1_usdma_we(pq, we_ret);</div><div class='add'>+			if (we_ret &lt;= 0)</div><div class='add'>+				flush_pq_iowait(pq);</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> 	*count += idx;</div><div class='hunk'>@@ -602,7 +560,7 @@ free_req:</div><div class='ctx'> 		if (req-&gt;seqsubmitted)</div><div class='ctx'> 			wait_event(pq-&gt;busy.wait_dma,</div><div class='ctx'> 				   (req-&gt;seqcomp == req-&gt;seqsubmitted - 1));</div><div class='del'>-		user_sdma_free_request(req, true);</div><div class='add'>+		user_sdma_free_request(req);</div><div class='ctx'> 		pq_update(pq);</div><div class='ctx'> 		set_comp_state(pq, cq, info.comp_idx, ERROR, ret);</div><div class='ctx'> 	}</div><div class='hunk'>@@ -714,48 +672,6 @@ static int user_sdma_txadd_ahg(struct user_sdma_request *req,</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int user_sdma_txadd(struct user_sdma_request *req,</div><div class='del'>-			   struct user_sdma_txreq *tx,</div><div class='del'>-			   struct user_sdma_iovec *iovec, u32 datalen,</div><div class='del'>-			   u32 *queued_ptr, u32 *data_sent_ptr,</div><div class='del'>-			   u64 *iov_offset_ptr)</div><div class='del'>-{</div><div class='del'>-	int ret;</div><div class='del'>-	unsigned int pageidx, len;</div><div class='del'>-	unsigned long base, offset;</div><div class='del'>-	u64 iov_offset = *iov_offset_ptr;</div><div class='del'>-	u32 queued = *queued_ptr, data_sent = *data_sent_ptr;</div><div class='del'>-	struct hfi1_user_sdma_pkt_q *pq = req-&gt;pq;</div><div class='del'>-</div><div class='del'>-	base = (unsigned long)iovec-&gt;iov.iov_base;</div><div class='del'>-	offset = offset_in_page(base + iovec-&gt;offset + iov_offset);</div><div class='del'>-	pageidx = (((iovec-&gt;offset + iov_offset + base) - (base &amp; PAGE_MASK)) &gt;&gt;</div><div class='del'>-		   PAGE_SHIFT);</div><div class='del'>-	len = offset + req-&gt;info.fragsize &gt; PAGE_SIZE ?</div><div class='del'>-		PAGE_SIZE - offset : req-&gt;info.fragsize;</div><div class='del'>-	len = min((datalen - queued), len);</div><div class='del'>-	ret = sdma_txadd_page(pq-&gt;dd, &amp;tx-&gt;txreq, iovec-&gt;pages[pageidx],</div><div class='del'>-			      offset, len);</div><div class='del'>-	if (ret) {</div><div class='del'>-		SDMA_DBG(req, "SDMA txreq add page failed %d\n", ret);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-	iov_offset += len;</div><div class='del'>-	queued += len;</div><div class='del'>-	data_sent += len;</div><div class='del'>-	if (unlikely(queued &lt; datalen &amp;&amp; pageidx == iovec-&gt;npages &amp;&amp;</div><div class='del'>-		     req-&gt;iov_idx &lt; req-&gt;data_iovs - 1)) {</div><div class='del'>-		iovec-&gt;offset += iov_offset;</div><div class='del'>-		iovec = &amp;req-&gt;iovs[++req-&gt;iov_idx];</div><div class='del'>-		iov_offset = 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	*queued_ptr = queued;</div><div class='del'>-	*data_sent_ptr = data_sent;</div><div class='del'>-	*iov_offset_ptr = iov_offset;</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int user_sdma_send_pkts(struct user_sdma_request *req, u16 maxpkts)</div><div class='ctx'> {</div><div class='ctx'> 	int ret = 0;</div><div class='hunk'>@@ -787,8 +703,7 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, u16 maxpkts)</div><div class='ctx'> 		maxpkts = req-&gt;info.npkts - req-&gt;seqnum;</div><div class='ctx'> </div><div class='ctx'> 	while (npkts &lt; maxpkts) {</div><div class='del'>-		u32 datalen = 0, queued = 0, data_sent = 0;</div><div class='del'>-		u64 iov_offset = 0;</div><div class='add'>+		u32 datalen = 0;</div><div class='ctx'> </div><div class='ctx'> 		/*</div><div class='ctx'> 		 * Check whether any of the completions have come back</div><div class='hunk'>@@ -804,7 +719,6 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, u16 maxpkts)</div><div class='ctx'> </div><div class='ctx'> 		tx-&gt;flags = 0;</div><div class='ctx'> 		tx-&gt;req = req;</div><div class='del'>-		tx-&gt;busycount = 0;</div><div class='ctx'> 		INIT_LIST_HEAD(&amp;tx-&gt;list);</div><div class='ctx'> </div><div class='ctx'> 		/*</div><div class='hunk'>@@ -882,27 +796,17 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, u16 maxpkts)</div><div class='ctx'> 				goto free_txreq;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		/*</div><div class='del'>-		 * If the request contains any data vectors, add up to</div><div class='del'>-		 * fragsize bytes to the descriptor.</div><div class='del'>-		 */</div><div class='del'>-		while (queued &lt; datalen &amp;&amp;</div><div class='del'>-		       (req-&gt;sent + data_sent) &lt; req-&gt;data_len) {</div><div class='del'>-			ret = user_sdma_txadd(req, tx, iovec, datalen,</div><div class='del'>-					      &amp;queued, &amp;data_sent, &amp;iov_offset);</div><div class='del'>-			if (ret)</div><div class='del'>-				goto free_txreq;</div><div class='del'>-		}</div><div class='del'>-		/*</div><div class='del'>-		 * The txreq was submitted successfully so we can update</div><div class='del'>-		 * the counters.</div><div class='del'>-		 */</div><div class='ctx'> 		req-&gt;koffset += datalen;</div><div class='ctx'> 		if (req_opcode(req-&gt;info.ctrl) == EXPECTED)</div><div class='ctx'> 			req-&gt;tidoffset += datalen;</div><div class='del'>-		req-&gt;sent += data_sent;</div><div class='del'>-		if (req-&gt;data_len)</div><div class='del'>-			iovec-&gt;offset += iov_offset;</div><div class='add'>+		req-&gt;sent += datalen;</div><div class='add'>+		while (datalen) {</div><div class='add'>+			ret = hfi1_add_pages_to_sdma_packet(req, tx, iovec,</div><div class='add'>+							    &amp;datalen);</div><div class='add'>+			if (ret)</div><div class='add'>+				goto free_txreq;</div><div class='add'>+			iovec = &amp;req-&gt;iovs[req-&gt;iov_idx];</div><div class='add'>+		}</div><div class='ctx'> 		list_add_tail(&amp;tx-&gt;txreq.list, &amp;req-&gt;txps);</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * It is important to increment this here as it is used to</div><div class='hunk'>@@ -936,135 +840,6 @@ free_tx:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static u32 sdma_cache_evict(struct hfi1_user_sdma_pkt_q *pq, u32 npages)</div><div class='del'>-{</div><div class='del'>-	struct evict_data evict_data;</div><div class='del'>-</div><div class='del'>-	evict_data.cleared = 0;</div><div class='del'>-	evict_data.target = npages;</div><div class='del'>-	hfi1_mmu_rb_evict(pq-&gt;handler, &amp;evict_data);</div><div class='del'>-	return evict_data.cleared;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int pin_sdma_pages(struct user_sdma_request *req,</div><div class='del'>-			  struct user_sdma_iovec *iovec,</div><div class='del'>-			  struct sdma_mmu_node *node,</div><div class='del'>-			  int npages)</div><div class='del'>-{</div><div class='del'>-	int pinned, cleared;</div><div class='del'>-	struct page **pages;</div><div class='del'>-	struct hfi1_user_sdma_pkt_q *pq = req-&gt;pq;</div><div class='del'>-</div><div class='del'>-	pages = kcalloc(npages, sizeof(*pages), GFP_KERNEL);</div><div class='del'>-	if (!pages)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	memcpy(pages, node-&gt;pages, node-&gt;npages * sizeof(*pages));</div><div class='del'>-</div><div class='del'>-	npages -= node-&gt;npages;</div><div class='del'>-retry:</div><div class='del'>-	if (!hfi1_can_pin_pages(pq-&gt;dd, pq-&gt;mm,</div><div class='del'>-				atomic_read(&amp;pq-&gt;n_locked), npages)) {</div><div class='del'>-		cleared = sdma_cache_evict(pq, npages);</div><div class='del'>-		if (cleared &gt;= npages)</div><div class='del'>-			goto retry;</div><div class='del'>-	}</div><div class='del'>-	pinned = hfi1_acquire_user_pages(pq-&gt;mm,</div><div class='del'>-					 ((unsigned long)iovec-&gt;iov.iov_base +</div><div class='del'>-					 (node-&gt;npages * PAGE_SIZE)), npages, 0,</div><div class='del'>-					 pages + node-&gt;npages);</div><div class='del'>-	if (pinned &lt; 0) {</div><div class='del'>-		kfree(pages);</div><div class='del'>-		return pinned;</div><div class='del'>-	}</div><div class='del'>-	if (pinned != npages) {</div><div class='del'>-		unpin_vector_pages(pq-&gt;mm, pages, node-&gt;npages, pinned);</div><div class='del'>-		return -EFAULT;</div><div class='del'>-	}</div><div class='del'>-	kfree(node-&gt;pages);</div><div class='del'>-	node-&gt;rb.len = iovec-&gt;iov.iov_len;</div><div class='del'>-	node-&gt;pages = pages;</div><div class='del'>-	atomic_add(pinned, &amp;pq-&gt;n_locked);</div><div class='del'>-	return pinned;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void unpin_sdma_pages(struct sdma_mmu_node *node)</div><div class='del'>-{</div><div class='del'>-	if (node-&gt;npages) {</div><div class='del'>-		unpin_vector_pages(node-&gt;pq-&gt;mm, node-&gt;pages, 0, node-&gt;npages);</div><div class='del'>-		atomic_sub(node-&gt;npages, &amp;node-&gt;pq-&gt;n_locked);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int pin_vector_pages(struct user_sdma_request *req,</div><div class='del'>-			    struct user_sdma_iovec *iovec)</div><div class='del'>-{</div><div class='del'>-	int ret = 0, pinned, npages;</div><div class='del'>-	struct hfi1_user_sdma_pkt_q *pq = req-&gt;pq;</div><div class='del'>-	struct sdma_mmu_node *node = NULL;</div><div class='del'>-	struct mmu_rb_node *rb_node;</div><div class='del'>-	struct iovec *iov;</div><div class='del'>-	bool extracted;</div><div class='del'>-</div><div class='del'>-	extracted =</div><div class='del'>-		hfi1_mmu_rb_remove_unless_exact(pq-&gt;handler,</div><div class='del'>-						(unsigned long)</div><div class='del'>-						iovec-&gt;iov.iov_base,</div><div class='del'>-						iovec-&gt;iov.iov_len, &amp;rb_node);</div><div class='del'>-	if (rb_node) {</div><div class='del'>-		node = container_of(rb_node, struct sdma_mmu_node, rb);</div><div class='del'>-		if (!extracted) {</div><div class='del'>-			atomic_inc(&amp;node-&gt;refcount);</div><div class='del'>-			iovec-&gt;pages = node-&gt;pages;</div><div class='del'>-			iovec-&gt;npages = node-&gt;npages;</div><div class='del'>-			iovec-&gt;node = node;</div><div class='del'>-			return 0;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!node) {</div><div class='del'>-		node = kzalloc(sizeof(*node), GFP_KERNEL);</div><div class='del'>-		if (!node)</div><div class='del'>-			return -ENOMEM;</div><div class='del'>-</div><div class='del'>-		node-&gt;rb.addr = (unsigned long)iovec-&gt;iov.iov_base;</div><div class='del'>-		node-&gt;pq = pq;</div><div class='del'>-		atomic_set(&amp;node-&gt;refcount, 0);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	iov = &amp;iovec-&gt;iov;</div><div class='del'>-	npages = num_user_pages((unsigned long)iov-&gt;iov_base, iov-&gt;iov_len);</div><div class='del'>-	if (node-&gt;npages &lt; npages) {</div><div class='del'>-		pinned = pin_sdma_pages(req, iovec, node, npages);</div><div class='del'>-		if (pinned &lt; 0) {</div><div class='del'>-			ret = pinned;</div><div class='del'>-			goto bail;</div><div class='del'>-		}</div><div class='del'>-		node-&gt;npages += pinned;</div><div class='del'>-		npages = node-&gt;npages;</div><div class='del'>-	}</div><div class='del'>-	iovec-&gt;pages = node-&gt;pages;</div><div class='del'>-	iovec-&gt;npages = npages;</div><div class='del'>-	iovec-&gt;node = node;</div><div class='del'>-</div><div class='del'>-	ret = hfi1_mmu_rb_insert(req-&gt;pq-&gt;handler, &amp;node-&gt;rb);</div><div class='del'>-	if (ret) {</div><div class='del'>-		iovec-&gt;node = NULL;</div><div class='del'>-		goto bail;</div><div class='del'>-	}</div><div class='del'>-	return 0;</div><div class='del'>-bail:</div><div class='del'>-	unpin_sdma_pages(node);</div><div class='del'>-	kfree(node);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void unpin_vector_pages(struct mm_struct *mm, struct page **pages,</div><div class='del'>-			       unsigned start, unsigned npages)</div><div class='del'>-{</div><div class='del'>-	hfi1_release_user_pages(mm, pages + start, npages, false);</div><div class='del'>-	kfree(pages);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int check_header_template(struct user_sdma_request *req,</div><div class='ctx'> 				 struct hfi1_pkt_header *hdr, u32 lrhlen,</div><div class='ctx'> 				 u32 datalen)</div><div class='hunk'>@@ -1406,7 +1181,7 @@ static void user_sdma_txreq_cb(struct sdma_txreq *txreq, int status)</div><div class='ctx'> 	if (req-&gt;seqcomp != req-&gt;info.npkts - 1)</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='del'>-	user_sdma_free_request(req, false);</div><div class='add'>+	user_sdma_free_request(req);</div><div class='ctx'> 	set_comp_state(pq, cq, req-&gt;info.comp_idx, state, status);</div><div class='ctx'> 	pq_update(pq);</div><div class='ctx'> }</div><div class='hunk'>@@ -1417,10 +1192,8 @@ static inline void pq_update(struct hfi1_user_sdma_pkt_q *pq)</div><div class='ctx'> 		wake_up(&amp;pq-&gt;wait);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void user_sdma_free_request(struct user_sdma_request *req, bool unpin)</div><div class='add'>+static void user_sdma_free_request(struct user_sdma_request *req)</div><div class='ctx'> {</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='ctx'> 	if (!list_empty(&amp;req-&gt;txps)) {</div><div class='ctx'> 		struct sdma_txreq *t, *p;</div><div class='ctx'> </div><div class='hunk'>@@ -1433,21 +1206,6 @@ static void user_sdma_free_request(struct user_sdma_request *req, bool unpin)</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; req-&gt;data_iovs; i++) {</div><div class='del'>-		struct sdma_mmu_node *node = req-&gt;iovs[i].node;</div><div class='del'>-</div><div class='del'>-		if (!node)</div><div class='del'>-			continue;</div><div class='del'>-</div><div class='del'>-		req-&gt;iovs[i].node = NULL;</div><div class='del'>-</div><div class='del'>-		if (unpin)</div><div class='del'>-			hfi1_mmu_rb_remove(req-&gt;pq-&gt;handler,</div><div class='del'>-					   &amp;node-&gt;rb);</div><div class='del'>-		else</div><div class='del'>-			atomic_dec(&amp;node-&gt;refcount);</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	kfree(req-&gt;tids);</div><div class='ctx'> 	clear_bit(req-&gt;info.comp_idx, req-&gt;pq-&gt;req_in_use);</div><div class='ctx'> }</div><div class='hunk'>@@ -1464,63 +1222,3 @@ static inline void set_comp_state(struct hfi1_user_sdma_pkt_q *pq,</div><div class='ctx'> 	trace_hfi1_sdma_user_completion(pq-&gt;dd, pq-&gt;ctxt, pq-&gt;subctxt,</div><div class='ctx'> 					idx, state, ret);</div><div class='ctx'> }</div><div class='del'>-</div><div class='del'>-static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr,</div><div class='del'>-			   unsigned long len)</div><div class='del'>-{</div><div class='del'>-	return (bool)(node-&gt;addr == addr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int sdma_rb_insert(void *arg, struct mmu_rb_node *mnode)</div><div class='del'>-{</div><div class='del'>-	struct sdma_mmu_node *node =</div><div class='del'>-		container_of(mnode, struct sdma_mmu_node, rb);</div><div class='del'>-</div><div class='del'>-	atomic_inc(&amp;node-&gt;refcount);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Return 1 to remove the node from the rb tree and call the remove op.</div><div class='del'>- *</div><div class='del'>- * Called with the rb tree lock held.</div><div class='del'>- */</div><div class='del'>-static int sdma_rb_evict(void *arg, struct mmu_rb_node *mnode,</div><div class='del'>-			 void *evict_arg, bool *stop)</div><div class='del'>-{</div><div class='del'>-	struct sdma_mmu_node *node =</div><div class='del'>-		container_of(mnode, struct sdma_mmu_node, rb);</div><div class='del'>-	struct evict_data *evict_data = evict_arg;</div><div class='del'>-</div><div class='del'>-	/* is this node still being used? */</div><div class='del'>-	if (atomic_read(&amp;node-&gt;refcount))</div><div class='del'>-		return 0; /* keep this node */</div><div class='del'>-</div><div class='del'>-	/* this node will be evicted, add its pages to our count */</div><div class='del'>-	evict_data-&gt;cleared += node-&gt;npages;</div><div class='del'>-</div><div class='del'>-	/* have enough pages been cleared? */</div><div class='del'>-	if (evict_data-&gt;cleared &gt;= evict_data-&gt;target)</div><div class='del'>-		*stop = true;</div><div class='del'>-</div><div class='del'>-	return 1; /* remove this node */</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void sdma_rb_remove(void *arg, struct mmu_rb_node *mnode)</div><div class='del'>-{</div><div class='del'>-	struct sdma_mmu_node *node =</div><div class='del'>-		container_of(mnode, struct sdma_mmu_node, rb);</div><div class='del'>-</div><div class='del'>-	unpin_sdma_pages(node);</div><div class='del'>-	kfree(node);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int sdma_rb_invalidate(void *arg, struct mmu_rb_node *mnode)</div><div class='del'>-{</div><div class='del'>-	struct sdma_mmu_node *node =</div><div class='del'>-		container_of(mnode, struct sdma_mmu_node, rb);</div><div class='del'>-</div><div class='del'>-	if (!atomic_read(&amp;node-&gt;refcount))</div><div class='del'>-		return 1;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/user_sdma.h b/drivers/infiniband/hw/hfi1/user_sdma.h<br/>index 14dfd757dafd..8735524e3a9a 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/user_sdma.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/user_sdma.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/user_sdma.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/user_sdma.h</a></div><div class='hunk'>@@ -1,57 +1,20 @@</div><div class='del'>-#ifndef _HFI1_USER_SDMA_H</div><div class='del'>-#define _HFI1_USER_SDMA_H</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='add'>+ * Copyright(c) 2023 - Cornelis Networks, Inc.</div><div class='ctx'>  * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='add'>+#ifndef _HFI1_USER_SDMA_H</div><div class='add'>+#define _HFI1_USER_SDMA_H</div><div class='add'>+</div><div class='ctx'> #include &lt;linux/device.h&gt;</div><div class='ctx'> #include &lt;linux/wait.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "common.h"</div><div class='ctx'> #include "iowait.h"</div><div class='ctx'> #include "user_exp_rcv.h"</div><div class='add'>+#include "mmu_rb.h"</div><div class='add'>+#include "pinning.h"</div><div class='add'>+#include "sdma.h"</div><div class='ctx'> </div><div class='ctx'> /* The maximum number of Data io vectors per message/request */</div><div class='ctx'> #define MAX_VECTORS_PER_REQ 8</div><div class='hunk'>@@ -110,12 +73,6 @@ enum pkt_q_sdma_state {</div><div class='ctx'> 	SDMA_PKT_Q_DEFERRED,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/*</div><div class='del'>- * Maximum retry attempts to submit a TX request</div><div class='del'>- * before putting the process to sleep.</div><div class='del'>- */</div><div class='del'>-#define MAX_DEFER_RETRY_COUNT 1</div><div class='del'>-</div><div class='ctx'> #define SDMA_IOWAIT_TIMEOUT 1000 /* in milliseconds */</div><div class='ctx'> </div><div class='ctx'> #define SDMA_DBG(req, fmt, ...)				     \</div><div class='hunk'>@@ -139,7 +96,6 @@ struct hfi1_user_sdma_pkt_q {</div><div class='ctx'> 	unsigned long unpinned;</div><div class='ctx'> 	struct mmu_rb_handler *handler;</div><div class='ctx'> 	atomic_t n_locked;</div><div class='del'>-	struct mm_struct *mm;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hfi1_user_sdma_comp_q {</div><div class='hunk'>@@ -147,27 +103,14 @@ struct hfi1_user_sdma_comp_q {</div><div class='ctx'> 	struct hfi1_sdma_comp_entry *comps;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct sdma_mmu_node {</div><div class='del'>-	struct mmu_rb_node rb;</div><div class='del'>-	struct hfi1_user_sdma_pkt_q *pq;</div><div class='del'>-	atomic_t refcount;</div><div class='del'>-	struct page **pages;</div><div class='del'>-	unsigned int npages;</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> struct user_sdma_iovec {</div><div class='ctx'> 	struct list_head list;</div><div class='ctx'> 	struct iovec iov;</div><div class='del'>-	/* number of pages in this vector */</div><div class='del'>-	unsigned int npages;</div><div class='del'>-	/* array of pinned pages for this vector */</div><div class='del'>-	struct page **pages;</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * offset into the virtual address space of the vector at</div><div class='ctx'> 	 * which we last left off.</div><div class='ctx'> 	 */</div><div class='ctx'> 	u64 offset;</div><div class='del'>-	struct sdma_mmu_node *node;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /* evict operation argument */</div><div class='hunk'>@@ -245,7 +188,6 @@ struct user_sdma_txreq {</div><div class='ctx'> 	struct list_head list;</div><div class='ctx'> 	struct user_sdma_request *req;</div><div class='ctx'> 	u16 flags;</div><div class='del'>-	unsigned int busycount;</div><div class='ctx'> 	u16 seqnum;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -256,5 +198,4 @@ int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd,</div><div class='ctx'> int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,</div><div class='ctx'> 				   struct iovec *iovec, unsigned long dim,</div><div class='ctx'> 				   unsigned long *count);</div><div class='del'>-</div><div class='ctx'> #endif /* _HFI1_USER_SDMA_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c<br/>index 1eb4105b2d22..3cbbfccdd8cd 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/verbs.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/verbs.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/verbs.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/verbs.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2015 - 2020 Intel Corporation.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;rdma/ib_mad.h&gt;</div><div class='hunk'>@@ -54,6 +12,7 @@</div><div class='ctx'> #include &lt;linux/mm.h&gt;</div><div class='ctx'> #include &lt;linux/vmalloc.h&gt;</div><div class='ctx'> #include &lt;rdma/opa_addr.h&gt;</div><div class='add'>+#include &lt;linux/nospec.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "hfi.h"</div><div class='ctx'> #include "common.h"</div><div class='hunk'>@@ -65,6 +24,7 @@</div><div class='ctx'> #include "vnic.h"</div><div class='ctx'> #include "fault.h"</div><div class='ctx'> #include "affinity.h"</div><div class='add'>+#include "ipoib.h"</div><div class='ctx'> </div><div class='ctx'> static unsigned int hfi1_lkey_table_size = 16;</div><div class='ctx'> module_param_named(lkey_table_size, hfi1_lkey_table_size, uint,</div><div class='hunk'>@@ -146,9 +106,6 @@ static int pio_wait(struct rvt_qp *qp,</div><div class='ctx'> /* Length of buffer to create verbs txreq cache name */</div><div class='ctx'> #define TXREQ_NAME_LEN 24</div><div class='ctx'> </div><div class='del'>-/* 16B trailing buffer */</div><div class='del'>-static const u8 trail_buf[MAX_16B_PADDING];</div><div class='del'>-</div><div class='ctx'> static uint wss_threshold = 80;</div><div class='ctx'> module_param(wss_threshold, uint, S_IRUGO);</div><div class='ctx'> MODULE_PARM_DESC(wss_threshold, "Percentage (1-100) of LLC to use as a threshold for a cacheless copy");</div><div class='hunk'>@@ -517,10 +474,11 @@ static inline void hfi1_handle_packet(struct hfi1_packet *packet,</div><div class='ctx'> 				       opa_get_lid(packet-&gt;dlid, 9B));</div><div class='ctx'> 		if (!mcast)</div><div class='ctx'> 			goto drop;</div><div class='add'>+		rcu_read_lock();</div><div class='ctx'> 		list_for_each_entry_rcu(p, &amp;mcast-&gt;qp_list, list) {</div><div class='ctx'> 			packet-&gt;qp = p-&gt;qp;</div><div class='ctx'> 			if (hfi1_do_pkey_check(packet))</div><div class='del'>-				goto drop;</div><div class='add'>+				goto unlock_drop;</div><div class='ctx'> 			spin_lock_irqsave(&amp;packet-&gt;qp-&gt;r_lock, flags);</div><div class='ctx'> 			packet_handler = qp_ok(packet);</div><div class='ctx'> 			if (likely(packet_handler))</div><div class='hunk'>@@ -529,6 +487,7 @@ static inline void hfi1_handle_packet(struct hfi1_packet *packet,</div><div class='ctx'> 				ibp-&gt;rvp.n_pkt_drops++;</div><div class='ctx'> 			spin_unlock_irqrestore(&amp;packet-&gt;qp-&gt;r_lock, flags);</div><div class='ctx'> 		}</div><div class='add'>+		rcu_read_unlock();</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * Notify rvt_multicast_detach() if it is waiting for us</div><div class='ctx'> 		 * to finish.</div><div class='hunk'>@@ -595,7 +554,7 @@ void hfi1_16B_rcv(struct hfi1_packet *packet)</div><div class='ctx'>  */</div><div class='ctx'> static void mem_timer(struct timer_list *t)</div><div class='ctx'> {</div><div class='del'>-	struct hfi1_ibdev *dev = from_timer(dev, t, mem_timer);</div><div class='add'>+	struct hfi1_ibdev *dev = timer_container_of(dev, t, mem_timer);</div><div class='ctx'> 	struct list_head *list = &amp;dev-&gt;memwait;</div><div class='ctx'> 	struct rvt_qp *qp = NULL;</div><div class='ctx'> 	struct iowait *wait;</div><div class='hunk'>@@ -638,6 +597,8 @@ static void verbs_sdma_complete(</div><div class='ctx'> 		struct hfi1_opa_header *hdr;</div><div class='ctx'> </div><div class='ctx'> 		hdr = &amp;tx-&gt;phdr.hdr;</div><div class='add'>+		if (unlikely(status == SDMA_TXREQ_S_ABORTED))</div><div class='add'>+			hfi1_rc_verbs_aborted(qp, hdr);</div><div class='ctx'> 		hfi1_rc_send_complete(qp, hdr);</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock(&amp;qp-&gt;s_lock);</div><div class='hunk'>@@ -726,7 +687,7 @@ bail_txadd:</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * update_tx_opstats - record stats by opcode</div><div class='del'>- * @qp; the qp</div><div class='add'>+ * @qp: the qp</div><div class='ctx'>  * @ps: transmit packet state</div><div class='ctx'>  * @plen: the plen in dwords</div><div class='ctx'>  *</div><div class='hunk'>@@ -817,8 +778,8 @@ static int build_verbs_tx_desc(</div><div class='ctx'> </div><div class='ctx'> 	/* add icrc, lt byte, and padding to flit */</div><div class='ctx'> 	if (extra_bytes)</div><div class='del'>-		ret = sdma_txadd_kvaddr(sde-&gt;dd, &amp;tx-&gt;txreq,</div><div class='del'>-					(void *)trail_buf, extra_bytes);</div><div class='add'>+		ret = sdma_txadd_daddr(sde-&gt;dd, &amp;tx-&gt;txreq, sde-&gt;dd-&gt;sdma_pad_phys,</div><div class='add'>+				       extra_bytes);</div><div class='ctx'> </div><div class='ctx'> bail_txadd:</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -871,16 +832,17 @@ int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps,</div><div class='ctx'> 			else</div><div class='ctx'> 				pbc |= (ib_is_sc5(sc5) &lt;&lt; PBC_DC_INFO_SHIFT);</div><div class='ctx'> </div><div class='del'>-			if (unlikely(hfi1_dbg_should_fault_tx(qp, ps-&gt;opcode)))</div><div class='del'>-				pbc = hfi1_fault_tx(qp, ps-&gt;opcode, pbc);</div><div class='ctx'> 			pbc = create_pbc(ppd,</div><div class='ctx'> 					 pbc,</div><div class='ctx'> 					 qp-&gt;srate_mbps,</div><div class='ctx'> 					 vl,</div><div class='ctx'> 					 plen);</div><div class='ctx'> </div><div class='del'>-			/* Update HCRC based on packet opcode */</div><div class='del'>-			pbc = update_hcrc(ps-&gt;opcode, pbc);</div><div class='add'>+			if (unlikely(hfi1_dbg_should_fault_tx(qp, ps-&gt;opcode)))</div><div class='add'>+				pbc = hfi1_fault_tx(qp, ps-&gt;opcode, pbc);</div><div class='add'>+			else</div><div class='add'>+				/* Update HCRC based on packet opcode */</div><div class='add'>+				pbc = update_hcrc(ps-&gt;opcode, pbc);</div><div class='ctx'> 		}</div><div class='ctx'> 		tx-&gt;wqe = qp-&gt;s_wqe;</div><div class='ctx'> 		ret = build_verbs_tx_desc(tx-&gt;sde, len, tx, ahg_info, pbc);</div><div class='hunk'>@@ -1027,20 +989,20 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,</div><div class='ctx'> 		else</div><div class='ctx'> 			pbc |= (ib_is_sc5(sc5) &lt;&lt; PBC_DC_INFO_SHIFT);</div><div class='ctx'> </div><div class='add'>+		pbc = create_pbc(ppd, pbc, qp-&gt;srate_mbps, vl, plen);</div><div class='ctx'> 		if (unlikely(hfi1_dbg_should_fault_tx(qp, ps-&gt;opcode)))</div><div class='ctx'> 			pbc = hfi1_fault_tx(qp, ps-&gt;opcode, pbc);</div><div class='del'>-		pbc = create_pbc(ppd, pbc, qp-&gt;srate_mbps, vl, plen);</div><div class='del'>-</div><div class='del'>-		/* Update HCRC based on packet opcode */</div><div class='del'>-		pbc = update_hcrc(ps-&gt;opcode, pbc);</div><div class='add'>+		else</div><div class='add'>+			/* Update HCRC based on packet opcode */</div><div class='add'>+			pbc = update_hcrc(ps-&gt;opcode, pbc);</div><div class='ctx'> 	}</div><div class='ctx'> 	if (cb)</div><div class='ctx'> 		iowait_pio_inc(&amp;priv-&gt;s_iowait);</div><div class='ctx'> 	pbuf = sc_buffer_alloc(sc, plen, cb, qp);</div><div class='del'>-	if (unlikely(!pbuf)) {</div><div class='add'>+	if (IS_ERR_OR_NULL(pbuf)) {</div><div class='ctx'> 		if (cb)</div><div class='ctx'> 			verbs_pio_complete(qp, 0);</div><div class='del'>-		if (ppd-&gt;host_link_state != HLS_UP_ACTIVE) {</div><div class='add'>+		if (IS_ERR(pbuf)) {</div><div class='ctx'> 			/*</div><div class='ctx'> 			 * If we have filled the PIO buffers to capacity and are</div><div class='ctx'> 			 * not in an active state this request is not going to</div><div class='hunk'>@@ -1085,7 +1047,8 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,</div><div class='ctx'> 		}</div><div class='ctx'> 		/* add icrc, lt byte, and padding to flit */</div><div class='ctx'> 		if (extra_bytes)</div><div class='del'>-			seg_pio_copy_mid(pbuf, trail_buf, extra_bytes);</div><div class='add'>+			seg_pio_copy_mid(pbuf, ppd-&gt;dd-&gt;sdma_pad_dma,</div><div class='add'>+					 extra_bytes);</div><div class='ctx'> </div><div class='ctx'> 		seg_pio_copy_end(pbuf);</div><div class='ctx'> 	}</div><div class='hunk'>@@ -1095,15 +1058,15 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,</div><div class='ctx'> 			       &amp;ps-&gt;s_txreq-&gt;phdr.hdr, ib_is_sc5(sc5));</div><div class='ctx'> </div><div class='ctx'> pio_bail:</div><div class='add'>+	spin_lock_irqsave(&amp;qp-&gt;s_lock, flags);</div><div class='ctx'> 	if (qp-&gt;s_wqe) {</div><div class='del'>-		spin_lock_irqsave(&amp;qp-&gt;s_lock, flags);</div><div class='ctx'> 		rvt_send_complete(qp, qp-&gt;s_wqe, wc_status);</div><div class='del'>-		spin_unlock_irqrestore(&amp;qp-&gt;s_lock, flags);</div><div class='ctx'> 	} else if (qp-&gt;ibqp.qp_type == IB_QPT_RC) {</div><div class='del'>-		spin_lock_irqsave(&amp;qp-&gt;s_lock, flags);</div><div class='add'>+		if (unlikely(wc_status == IB_WC_GENERAL_ERR))</div><div class='add'>+			hfi1_rc_verbs_aborted(qp, &amp;ps-&gt;s_txreq-&gt;phdr.hdr);</div><div class='ctx'> 		hfi1_rc_send_complete(qp, &amp;ps-&gt;s_txreq-&gt;phdr.hdr);</div><div class='del'>-		spin_unlock_irqrestore(&amp;qp-&gt;s_lock, flags);</div><div class='ctx'> 	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;qp-&gt;s_lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	ret = 0;</div><div class='ctx'> </div><div class='hunk'>@@ -1140,7 +1103,7 @@ static inline int egress_pkey_matches_entry(u16 pkey, u16 ent)</div><div class='ctx'>  * egress_pkey_check - check P_KEY of a packet</div><div class='ctx'>  * @ppd:  Physical IB port data</div><div class='ctx'>  * @slid: SLID for packet</div><div class='del'>- * @bkey: PKEY for header</div><div class='add'>+ * @pkey: PKEY for header</div><div class='ctx'>  * @sc5:  SC for packet</div><div class='ctx'>  * @s_pkey_index: It will be used for look up optimization for kernel contexts</div><div class='ctx'>  * only. If it is negative value, then it means user contexts is calling this</div><div class='hunk'>@@ -1201,7 +1164,7 @@ bad:</div><div class='ctx'> 	return 1;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/**</div><div class='add'>+/*</div><div class='ctx'>  * get_send_routine - choose an egress routine</div><div class='ctx'>  *</div><div class='ctx'>  * Choose an egress routine based on QP type</div><div class='hunk'>@@ -1337,8 +1300,8 @@ static void hfi1_fill_device_attr(struct hfi1_devdata *dd)</div><div class='ctx'> 			IB_DEVICE_BAD_QKEY_CNTR | IB_DEVICE_SHUTDOWN_PORT |</div><div class='ctx'> 			IB_DEVICE_SYS_IMAGE_GUID | IB_DEVICE_RC_RNR_NAK_GEN |</div><div class='ctx'> 			IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_SRQ_RESIZE |</div><div class='del'>-			IB_DEVICE_MEM_MGT_EXTENSIONS |</div><div class='del'>-			IB_DEVICE_RDMA_NETDEV_OPA_VNIC;</div><div class='add'>+			IB_DEVICE_MEM_MGT_EXTENSIONS;</div><div class='add'>+	rdi-&gt;dparms.props.kernel_cap_flags = IBK_RDMA_NETDEV_OPA;</div><div class='ctx'> 	rdi-&gt;dparms.props.page_size_cap = PAGE_SIZE;</div><div class='ctx'> 	rdi-&gt;dparms.props.vendor_id = dd-&gt;oui1 &lt;&lt; 16 | dd-&gt;oui2 &lt;&lt; 8 | dd-&gt;oui3;</div><div class='ctx'> 	rdi-&gt;dparms.props.vendor_part_id = dd-&gt;pcidev-&gt;device;</div><div class='hunk'>@@ -1356,9 +1319,6 @@ static void hfi1_fill_device_attr(struct hfi1_devdata *dd)</div><div class='ctx'> 	rdi-&gt;dparms.props.max_cq = hfi1_max_cqs;</div><div class='ctx'> 	rdi-&gt;dparms.props.max_ah = hfi1_max_ahs;</div><div class='ctx'> 	rdi-&gt;dparms.props.max_cqe = hfi1_max_cqes;</div><div class='del'>-	rdi-&gt;dparms.props.max_mr = rdi-&gt;lkey_table.max;</div><div class='del'>-	rdi-&gt;dparms.props.max_fmr = rdi-&gt;lkey_table.max;</div><div class='del'>-	rdi-&gt;dparms.props.max_map_per_fmr = 32767;</div><div class='ctx'> 	rdi-&gt;dparms.props.max_pd = hfi1_max_pds;</div><div class='ctx'> 	rdi-&gt;dparms.props.max_qp_rd_atom = HFI1_MAX_RDMA_ATOMIC;</div><div class='ctx'> 	rdi-&gt;dparms.props.max_qp_init_rd_atom = 255;</div><div class='hunk'>@@ -1405,7 +1365,7 @@ static inline u16 opa_width_to_ib(u16 in)</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int query_port(struct rvt_dev_info *rdi, u8 port_num,</div><div class='add'>+static int query_port(struct rvt_dev_info *rdi, u32 port_num,</div><div class='ctx'> 		      struct ib_port_attr *props)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi);</div><div class='hunk'>@@ -1422,7 +1382,7 @@ static int query_port(struct rvt_dev_info *rdi, u8 port_num,</div><div class='ctx'> 	props-&gt;gid_tbl_len = HFI1_GUIDS_PER_PORT;</div><div class='ctx'> 	props-&gt;active_width = (u8)opa_width_to_ib(ppd-&gt;link_width_active);</div><div class='ctx'> 	/* see rate_show() in ib core/sysfs.c */</div><div class='del'>-	props-&gt;active_speed = (u8)opa_speed_to_ib(ppd-&gt;link_speed_active);</div><div class='add'>+	props-&gt;active_speed = opa_speed_to_ib(ppd-&gt;link_speed_active);</div><div class='ctx'> 	props-&gt;max_vl_num = ppd-&gt;vls_supported;</div><div class='ctx'> </div><div class='ctx'> 	/* Once we are a "first class" citizen and have added the OPA MTUs to</div><div class='hunk'>@@ -1437,6 +1397,7 @@ static int query_port(struct rvt_dev_info *rdi, u8 port_num,</div><div class='ctx'> 				      4096 : hfi1_max_mtu), IB_MTU_4096);</div><div class='ctx'> 	props-&gt;active_mtu = !valid_ib_mtu(ppd-&gt;ibmtu) ? props-&gt;max_mtu :</div><div class='ctx'> 		mtu_to_enum(ppd-&gt;ibmtu, IB_MTU_4096);</div><div class='add'>+	props-&gt;phys_mtu = hfi1_max_mtu;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -1481,17 +1442,15 @@ bail:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int shut_down_port(struct rvt_dev_info *rdi, u8 port_num)</div><div class='add'>+static int shut_down_port(struct rvt_dev_info *rdi, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi);</div><div class='ctx'> 	struct hfi1_devdata *dd = dd_from_dev(verbs_dev);</div><div class='ctx'> 	struct hfi1_pportdata *ppd = &amp;dd-&gt;pport[port_num - 1];</div><div class='del'>-	int ret;</div><div class='ctx'> </div><div class='ctx'> 	set_link_down_reason(ppd, OPA_LINKDOWN_REASON_UNKNOWN, 0,</div><div class='ctx'> 			     OPA_LINKDOWN_REASON_UNKNOWN);</div><div class='del'>-	ret = set_link_state(ppd, HLS_DN_DOWNDEF);</div><div class='del'>-	return ret;</div><div class='add'>+	return set_link_state(ppd, HLS_DN_DOWNDEF);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hfi1_get_guid_be(struct rvt_dev_info *rdi, struct rvt_ibport *rvp,</div><div class='hunk'>@@ -1536,6 +1495,7 @@ static int hfi1_check_ah(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr)</div><div class='ctx'> 	sl = rdma_ah_get_sl(ah_attr);</div><div class='ctx'> 	if (sl &gt;= ARRAY_SIZE(ibp-&gt;sl_to_sc))</div><div class='ctx'> 		return -EINVAL;</div><div class='add'>+	sl = array_index_nospec(sl, ARRAY_SIZE(ibp-&gt;sl_to_sc));</div><div class='ctx'> </div><div class='ctx'> 	sc5 = ibp-&gt;sl_to_sc[sl];</div><div class='ctx'> 	if (sc_to_vlt(dd, sc5) &gt; num_vls &amp;&amp; sc_to_vlt(dd, sc5) != 0xf)</div><div class='hunk'>@@ -1638,26 +1598,23 @@ static const char * const driver_cntr_names[] = {</div><div class='ctx'> 	"DRIVER_EgrHdrFull"</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static DEFINE_MUTEX(cntr_names_lock); /* protects the *_cntr_names bufers */</div><div class='del'>-static const char **dev_cntr_names;</div><div class='del'>-static const char **port_cntr_names;</div><div class='add'>+static struct rdma_stat_desc *dev_cntr_descs;</div><div class='add'>+static struct rdma_stat_desc *port_cntr_descs;</div><div class='ctx'> int num_driver_cntrs = ARRAY_SIZE(driver_cntr_names);</div><div class='ctx'> static int num_dev_cntrs;</div><div class='ctx'> static int num_port_cntrs;</div><div class='del'>-static int cntr_names_initialized;</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * Convert a list of names separated by '\n' into an array of NULL terminated</div><div class='ctx'>  * strings. Optionally some entries can be reserved in the array to hold extra</div><div class='ctx'>  * external strings.</div><div class='ctx'>  */</div><div class='del'>-static int init_cntr_names(const char *names_in,</div><div class='del'>-			   const size_t names_len,</div><div class='del'>-			   int num_extra_names,</div><div class='del'>-			   int *num_cntrs,</div><div class='del'>-			   const char ***cntr_names)</div><div class='add'>+static int init_cntr_names(const char *names_in, const size_t names_len,</div><div class='add'>+			   int num_extra_names, int *num_cntrs,</div><div class='add'>+			   struct rdma_stat_desc **cntr_descs)</div><div class='ctx'> {</div><div class='del'>-	char *names_out, *p, **q;</div><div class='add'>+	struct rdma_stat_desc *names_out;</div><div class='add'>+	char *p;</div><div class='ctx'> 	int i, n;</div><div class='ctx'> </div><div class='ctx'> 	n = 0;</div><div class='hunk'>@@ -1665,77 +1622,65 @@ static int init_cntr_names(const char *names_in,</div><div class='ctx'> 		if (names_in[i] == '\n')</div><div class='ctx'> 			n++;</div><div class='ctx'> </div><div class='del'>-	names_out = kmalloc((n + num_extra_names) * sizeof(char *) + names_len,</div><div class='add'>+	names_out = kzalloc((n + num_extra_names) * sizeof(*names_out)</div><div class='add'>+				+ names_len,</div><div class='ctx'> 			    GFP_KERNEL);</div><div class='ctx'> 	if (!names_out) {</div><div class='ctx'> 		*num_cntrs = 0;</div><div class='del'>-		*cntr_names = NULL;</div><div class='add'>+		*cntr_descs = NULL;</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	p = names_out + (n + num_extra_names) * sizeof(char *);</div><div class='add'>+	p = (char *)&amp;names_out[n + num_extra_names];</div><div class='ctx'> 	memcpy(p, names_in, names_len);</div><div class='ctx'> </div><div class='del'>-	q = (char **)names_out;</div><div class='ctx'> 	for (i = 0; i &lt; n; i++) {</div><div class='del'>-		q[i] = p;</div><div class='add'>+		names_out[i].name = p;</div><div class='ctx'> 		p = strchr(p, '\n');</div><div class='ctx'> 		*p++ = '\0';</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	*num_cntrs = n;</div><div class='del'>-	*cntr_names = (const char **)names_out;</div><div class='add'>+	*cntr_descs = names_out;</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct rdma_hw_stats *alloc_hw_stats(struct ib_device *ibdev,</div><div class='del'>-					    u8 port_num)</div><div class='add'>+static struct rdma_hw_stats *hfi1_alloc_hw_device_stats(struct ib_device *ibdev)</div><div class='ctx'> {</div><div class='del'>-	int i, err;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;cntr_names_lock);</div><div class='del'>-	if (!cntr_names_initialized) {</div><div class='add'>+	if (!dev_cntr_descs) {</div><div class='ctx'> 		struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='add'>+		int i, err;</div><div class='ctx'> </div><div class='del'>-		err = init_cntr_names(dd-&gt;cntrnames,</div><div class='del'>-				      dd-&gt;cntrnameslen,</div><div class='add'>+		err = init_cntr_names(dd-&gt;cntrnames, dd-&gt;cntrnameslen,</div><div class='ctx'> 				      num_driver_cntrs,</div><div class='del'>-				      &amp;num_dev_cntrs,</div><div class='del'>-				      &amp;dev_cntr_names);</div><div class='del'>-		if (err) {</div><div class='del'>-			mutex_unlock(&amp;cntr_names_lock);</div><div class='add'>+				      &amp;num_dev_cntrs, &amp;dev_cntr_descs);</div><div class='add'>+		if (err)</div><div class='ctx'> 			return NULL;</div><div class='del'>-		}</div><div class='ctx'> </div><div class='ctx'> 		for (i = 0; i &lt; num_driver_cntrs; i++)</div><div class='del'>-			dev_cntr_names[num_dev_cntrs + i] =</div><div class='del'>-				driver_cntr_names[i];</div><div class='add'>+			dev_cntr_descs[num_dev_cntrs + i].name =</div><div class='add'>+							driver_cntr_names[i];</div><div class='add'>+	}</div><div class='add'>+	return rdma_alloc_hw_stats_struct(dev_cntr_descs,</div><div class='add'>+					  num_dev_cntrs + num_driver_cntrs,</div><div class='add'>+					  RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct rdma_hw_stats *hfi_alloc_hw_port_stats(struct ib_device *ibdev,</div><div class='add'>+						     u32 port_num)</div><div class='add'>+{</div><div class='add'>+	if (!port_cntr_descs) {</div><div class='add'>+		struct hfi1_devdata *dd = dd_from_ibdev(ibdev);</div><div class='add'>+		int err;</div><div class='ctx'> </div><div class='del'>-		err = init_cntr_names(dd-&gt;portcntrnames,</div><div class='del'>-				      dd-&gt;portcntrnameslen,</div><div class='add'>+		err = init_cntr_names(dd-&gt;portcntrnames, dd-&gt;portcntrnameslen,</div><div class='ctx'> 				      0,</div><div class='del'>-				      &amp;num_port_cntrs,</div><div class='del'>-				      &amp;port_cntr_names);</div><div class='del'>-		if (err) {</div><div class='del'>-			kfree(dev_cntr_names);</div><div class='del'>-			dev_cntr_names = NULL;</div><div class='del'>-			mutex_unlock(&amp;cntr_names_lock);</div><div class='add'>+				      &amp;num_port_cntrs, &amp;port_cntr_descs);</div><div class='add'>+		if (err)</div><div class='ctx'> 			return NULL;</div><div class='del'>-		}</div><div class='del'>-		cntr_names_initialized = 1;</div><div class='ctx'> 	}</div><div class='del'>-	mutex_unlock(&amp;cntr_names_lock);</div><div class='del'>-</div><div class='del'>-	if (!port_num)</div><div class='del'>-		return rdma_alloc_hw_stats_struct(</div><div class='del'>-				dev_cntr_names,</div><div class='del'>-				num_dev_cntrs + num_driver_cntrs,</div><div class='del'>-				RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='del'>-	else</div><div class='del'>-		return rdma_alloc_hw_stats_struct(</div><div class='del'>-				port_cntr_names,</div><div class='del'>-				num_port_cntrs,</div><div class='del'>-				RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='add'>+	return rdma_alloc_hw_stats_struct(port_cntr_descs, num_port_cntrs,</div><div class='add'>+					  RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static u64 hfi1_sps_ints(void)</div><div class='hunk'>@@ -1753,7 +1698,7 @@ static u64 hfi1_sps_ints(void)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,</div><div class='del'>-			u8 port, int index)</div><div class='add'>+			u32 port, int index)</div><div class='ctx'> {</div><div class='ctx'> 	u64 *values;</div><div class='ctx'> 	int count;</div><div class='hunk'>@@ -1779,14 +1724,20 @@ static int get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static const struct ib_device_ops hfi1_dev_ops = {</div><div class='del'>-	.alloc_hw_stats = alloc_hw_stats,</div><div class='add'>+	.owner = THIS_MODULE,</div><div class='add'>+	.driver_id = RDMA_DRIVER_HFI1,</div><div class='add'>+</div><div class='add'>+	.alloc_hw_device_stats = hfi1_alloc_hw_device_stats,</div><div class='add'>+	.alloc_hw_port_stats = hfi_alloc_hw_port_stats,</div><div class='ctx'> 	.alloc_rdma_netdev = hfi1_vnic_alloc_rn,</div><div class='add'>+	.device_group = &amp;ib_hfi1_attr_group,</div><div class='ctx'> 	.get_dev_fw_str = hfi1_get_dev_fw_str,</div><div class='ctx'> 	.get_hw_stats = get_hw_stats,</div><div class='del'>-	.init_port = hfi1_create_port_files,</div><div class='ctx'> 	.modify_device = modify_device,</div><div class='add'>+	.port_groups = hfi1_attr_port_groups,</div><div class='ctx'> 	/* keep process mad in the driver */</div><div class='ctx'> 	.process_mad = hfi1_process_mad,</div><div class='add'>+	.rdma_netdev_get_params = hfi1_ipoib_rn_get_params,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='hunk'>@@ -1829,13 +1780,12 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)</div><div class='ctx'> 	 */</div><div class='ctx'> 	if (!ib_hfi1_sys_image_guid)</div><div class='ctx'> 		ib_hfi1_sys_image_guid = ibdev-&gt;node_guid;</div><div class='del'>-	ibdev-&gt;owner = THIS_MODULE;</div><div class='ctx'> 	ibdev-&gt;phys_port_cnt = dd-&gt;num_pports;</div><div class='ctx'> 	ibdev-&gt;dev.parent = &amp;dd-&gt;pcidev-&gt;dev;</div><div class='ctx'> </div><div class='ctx'> 	ib_set_device_ops(ibdev, &amp;hfi1_dev_ops);</div><div class='ctx'> </div><div class='del'>-	strlcpy(ibdev-&gt;node_desc, init_utsname()-&gt;nodename,</div><div class='add'>+	strscpy(ibdev-&gt;node_desc, init_utsname()-&gt;nodename,</div><div class='ctx'> 		sizeof(ibdev-&gt;node_desc));</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='hunk'>@@ -1858,9 +1808,8 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)</div><div class='ctx'> 	dd-&gt;verbs_dev.rdi.dparms.qpn_start = 0;</div><div class='ctx'> 	dd-&gt;verbs_dev.rdi.dparms.qpn_inc = 1;</div><div class='ctx'> 	dd-&gt;verbs_dev.rdi.dparms.qos_shift = dd-&gt;qos_shift;</div><div class='del'>-	dd-&gt;verbs_dev.rdi.dparms.qpn_res_start = kdeth_qp &lt;&lt; 16;</div><div class='del'>-	dd-&gt;verbs_dev.rdi.dparms.qpn_res_end =</div><div class='del'>-	dd-&gt;verbs_dev.rdi.dparms.qpn_res_start + 65535;</div><div class='add'>+	dd-&gt;verbs_dev.rdi.dparms.qpn_res_start = RVT_KDETH_QP_BASE;</div><div class='add'>+	dd-&gt;verbs_dev.rdi.dparms.qpn_res_end = RVT_AIP_QP_MAX;</div><div class='ctx'> 	dd-&gt;verbs_dev.rdi.dparms.max_rdma_atomic = HFI1_MAX_RDMA_ATOMIC;</div><div class='ctx'> 	dd-&gt;verbs_dev.rdi.dparms.psn_mask = PSN_MASK;</div><div class='ctx'> 	dd-&gt;verbs_dev.rdi.dparms.psn_shift = PSN_SHIFT;</div><div class='hunk'>@@ -1920,10 +1869,7 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)</div><div class='ctx'> 			      i,</div><div class='ctx'> 			      ppd-&gt;pkeys);</div><div class='ctx'> </div><div class='del'>-	rdma_set_device_sysfs_group(&amp;dd-&gt;verbs_dev.rdi.ibdev,</div><div class='del'>-				    &amp;ib_hfi1_attr_group);</div><div class='del'>-</div><div class='del'>-	ret = rvt_register_device(&amp;dd-&gt;verbs_dev.rdi, RDMA_DRIVER_HFI1);</div><div class='add'>+	ret = rvt_register_device(&amp;dd-&gt;verbs_dev.rdi);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err_verbs_txreq;</div><div class='ctx'> </div><div class='hunk'>@@ -1954,16 +1900,13 @@ void hfi1_unregister_ib_device(struct hfi1_devdata *dd)</div><div class='ctx'> 	if (!list_empty(&amp;dev-&gt;memwait))</div><div class='ctx'> 		dd_dev_err(dd, "memwait list not empty!\n");</div><div class='ctx'> </div><div class='del'>-	del_timer_sync(&amp;dev-&gt;mem_timer);</div><div class='add'>+	timer_delete_sync(&amp;dev-&gt;mem_timer);</div><div class='ctx'> 	verbs_txreq_exit(dev);</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;cntr_names_lock);</div><div class='del'>-	kfree(dev_cntr_names);</div><div class='del'>-	kfree(port_cntr_names);</div><div class='del'>-	dev_cntr_names = NULL;</div><div class='del'>-	port_cntr_names = NULL;</div><div class='del'>-	cntr_names_initialized = 0;</div><div class='del'>-	mutex_unlock(&amp;cntr_names_lock);</div><div class='add'>+	kfree(dev_cntr_descs);</div><div class='add'>+	kfree(port_cntr_descs);</div><div class='add'>+	dev_cntr_descs = NULL;</div><div class='add'>+	port_cntr_descs = NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void hfi1_cnp_rcv(struct hfi1_packet *packet)</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/verbs.h b/drivers/infiniband/hw/hfi1/verbs.h<br/>index 7ecb8ed4a1d9..070e4f0babe8 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/verbs.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/verbs.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/verbs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/verbs.h</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2015 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #ifndef HFI1_VERBS_H</div><div class='hunk'>@@ -107,9 +65,9 @@ enum {</div><div class='ctx'> 	HFI1_HAS_GRH = (1 &lt;&lt; 0),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define LRH_16B_BYTES (FIELD_SIZEOF(struct hfi1_16b_header, lrh))</div><div class='add'>+#define LRH_16B_BYTES (sizeof_field(struct hfi1_16b_header, lrh))</div><div class='ctx'> #define LRH_16B_DWORDS (LRH_16B_BYTES / sizeof(u32))</div><div class='del'>-#define LRH_9B_BYTES (FIELD_SIZEOF(struct ib_header, lrh))</div><div class='add'>+#define LRH_9B_BYTES (sizeof_field(struct ib_header, lrh))</div><div class='ctx'> #define LRH_9B_DWORDS (LRH_9B_BYTES / sizeof(u32))</div><div class='ctx'> </div><div class='ctx'> /* 24Bits for qpn, upper 8Bits reserved */</div><div class='hunk'>@@ -325,14 +283,13 @@ static inline struct rvt_qp *iowait_to_qp(struct iowait *s_iowait)</div><div class='ctx'>  */</div><div class='ctx'> void hfi1_bad_pkey(struct hfi1_ibport *ibp, u32 key, u32 sl,</div><div class='ctx'> 		   u32 qp1, u32 qp2, u32 lid1, u32 lid2);</div><div class='del'>-void hfi1_cap_mask_chg(struct rvt_dev_info *rdi, u8 port_num);</div><div class='add'>+void hfi1_cap_mask_chg(struct rvt_dev_info *rdi, u32 port_num);</div><div class='ctx'> void hfi1_sys_guid_chg(struct hfi1_ibport *ibp);</div><div class='ctx'> void hfi1_node_desc_chg(struct hfi1_ibport *ibp);</div><div class='del'>-int hfi1_process_mad(struct ib_device *ibdev, int mad_flags, u8 port,</div><div class='add'>+int hfi1_process_mad(struct ib_device *ibdev, int mad_flags, u32 port,</div><div class='ctx'> 		     const struct ib_wc *in_wc, const struct ib_grh *in_grh,</div><div class='del'>-		     const struct ib_mad_hdr *in_mad, size_t in_mad_size,</div><div class='del'>-		     struct ib_mad_hdr *out_mad, size_t *out_mad_size,</div><div class='del'>-		     u16 *out_mad_pkey_index);</div><div class='add'>+		     const struct ib_mad *in_mad, struct ib_mad *out_mad,</div><div class='add'>+		     size_t *out_mad_size, u16 *out_mad_pkey_index);</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * The PSN_MASK and PSN_SHIFT allow for</div><div class='hunk'>@@ -416,6 +373,7 @@ void hfi1_rc_hdrerr(</div><div class='ctx'> </div><div class='ctx'> u8 ah_to_sc(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr);</div><div class='ctx'> </div><div class='add'>+void hfi1_rc_verbs_aborted(struct rvt_qp *qp, struct hfi1_opa_header *opah);</div><div class='ctx'> void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah);</div><div class='ctx'> </div><div class='ctx'> void hfi1_ud_rcv(struct hfi1_packet *packet);</div><div class='hunk'>@@ -433,9 +391,6 @@ void hfi1_restart_rc(struct rvt_qp *qp, u32 psn, int wait);</div><div class='ctx'> int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe,</div><div class='ctx'> 		   bool *call_send);</div><div class='ctx'> </div><div class='del'>-extern const u32 rc_only_opcode;</div><div class='del'>-extern const u32 uc_only_opcode;</div><div class='del'>-</div><div class='ctx'> int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct hfi1_packet *packet);</div><div class='ctx'> </div><div class='ctx'> u32 hfi1_make_grh(struct hfi1_ibport *ibp, struct ib_grh *hdr,</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/verbs_txreq.c b/drivers/infiniband/hw/hfi1/verbs_txreq.c<br/>index c4ab2d5b4502..822f0d05bac8 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/verbs_txreq.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/verbs_txreq.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/verbs_txreq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/verbs_txreq.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2016 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include "hfi.h"</div><div class='hunk'>@@ -100,7 +58,7 @@ struct verbs_txreq *__get_txreq(struct hfi1_ibdev *dev,</div><div class='ctx'> 	if (ib_rvt_state_ops[qp-&gt;state] &amp; RVT_PROCESS_RECV_OK) {</div><div class='ctx'> 		struct hfi1_qp_priv *priv;</div><div class='ctx'> </div><div class='del'>-		tx = kmem_cache_alloc(dev-&gt;verbs_txreq_cache, GFP_ATOMIC);</div><div class='add'>+		tx = kmem_cache_alloc(dev-&gt;verbs_txreq_cache, VERBS_TXREQ_GFP);</div><div class='ctx'> 		if (tx)</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		priv = qp-&gt;priv;</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/verbs_txreq.h b/drivers/infiniband/hw/hfi1/verbs_txreq.h<br/>index b002e96eb335..56353c7676d0 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/verbs_txreq.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/verbs_txreq.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/verbs_txreq.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/verbs_txreq.h</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2016 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #ifndef HFI1_VERBS_TXREQ_H</div><div class='hunk'>@@ -72,6 +30,7 @@ struct hfi1_ibdev;</div><div class='ctx'> struct verbs_txreq *__get_txreq(struct hfi1_ibdev *dev,</div><div class='ctx'> 				struct rvt_qp *qp);</div><div class='ctx'> </div><div class='add'>+#define VERBS_TXREQ_GFP (GFP_ATOMIC | __GFP_NOWARN)</div><div class='ctx'> static inline struct verbs_txreq *get_txreq(struct hfi1_ibdev *dev,</div><div class='ctx'> 					    struct rvt_qp *qp)</div><div class='ctx'> 	__must_hold(&amp;qp-&gt;slock)</div><div class='hunk'>@@ -79,7 +38,7 @@ static inline struct verbs_txreq *get_txreq(struct hfi1_ibdev *dev,</div><div class='ctx'> 	struct verbs_txreq *tx;</div><div class='ctx'> 	struct hfi1_qp_priv *priv = qp-&gt;priv;</div><div class='ctx'> </div><div class='del'>-	tx = kmem_cache_alloc(dev-&gt;verbs_txreq_cache, GFP_ATOMIC);</div><div class='add'>+	tx = kmem_cache_alloc(dev-&gt;verbs_txreq_cache, VERBS_TXREQ_GFP);</div><div class='ctx'> 	if (unlikely(!tx)) {</div><div class='ctx'> 		/* call slow path to get the lock */</div><div class='ctx'> 		tx = __get_txreq(dev, qp);</div><div class='hunk'>@@ -90,7 +49,7 @@ static inline struct verbs_txreq *get_txreq(struct hfi1_ibdev *dev,</div><div class='ctx'> 	tx-&gt;mr = NULL;</div><div class='ctx'> 	tx-&gt;sde = priv-&gt;s_sde;</div><div class='ctx'> 	tx-&gt;psc = priv-&gt;s_sendcontext;</div><div class='del'>-	/* so that we can test if the sdma decriptors are there */</div><div class='add'>+	/* so that we can test if the sdma descriptors are there */</div><div class='ctx'> 	tx-&gt;txreq.num_desc = 0;</div><div class='ctx'> 	/* Set the header type */</div><div class='ctx'> 	tx-&gt;phdr.hdr.hdr_type = priv-&gt;hdr_type;</div><div class='hunk'>@@ -98,11 +57,6 @@ static inline struct verbs_txreq *get_txreq(struct hfi1_ibdev *dev,</div><div class='ctx'> 	return tx;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline struct sdma_txreq *get_sdma_txreq(struct verbs_txreq *tx)</div><div class='del'>-{</div><div class='del'>-	return &amp;tx-&gt;txreq;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static inline struct verbs_txreq *get_waiting_verbs_txreq(struct iowait_work *w)</div><div class='ctx'> {</div><div class='ctx'> 	struct sdma_txreq *stx;</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/vnic.h b/drivers/infiniband/hw/hfi1/vnic.h<br/>index 5ae781514e32..bbafeb5fc0ec 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/vnic.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/vnic.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/vnic.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/vnic.h</a></div><div class='hunk'>@@ -1,52 +1,10 @@</div><div class='del'>-#ifndef _HFI1_VNIC_H</div><div class='del'>-#define _HFI1_VNIC_H</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2017 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2017 - 2020 Intel Corporation.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='add'>+#ifndef _HFI1_VNIC_H</div><div class='add'>+#define _HFI1_VNIC_H</div><div class='ctx'> #include &lt;rdma/opa_vnic.h&gt;</div><div class='ctx'> #include "hfi.h"</div><div class='ctx'> #include "sdma.h"</div><div class='hunk'>@@ -69,6 +27,7 @@</div><div class='ctx'> #define HFI1_VNIC_SC_SHIFT      4</div><div class='ctx'> </div><div class='ctx'> #define HFI1_VNIC_MAX_QUEUE 16</div><div class='add'>+#define HFI1_NUM_VNIC_CTXT 8</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='ctx'>  * struct hfi1_vnic_sdma - VNIC per Tx ring SDMA information</div><div class='hunk'>@@ -104,7 +63,6 @@ struct hfi1_vnic_rx_queue {</div><div class='ctx'> 	struct hfi1_vnic_vport_info *vinfo;</div><div class='ctx'> 	struct net_device           *netdev;</div><div class='ctx'> 	struct napi_struct           napi;</div><div class='del'>-	struct sk_buff_head          skbq;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /**</div><div class='hunk'>@@ -146,7 +104,6 @@ struct hfi1_vnic_vport_info {</div><div class='ctx'> </div><div class='ctx'> /* vnic hfi1 internal functions */</div><div class='ctx'> void hfi1_vnic_setup(struct hfi1_devdata *dd);</div><div class='del'>-void hfi1_vnic_cleanup(struct hfi1_devdata *dd);</div><div class='ctx'> int hfi1_vnic_txreq_init(struct hfi1_devdata *dd);</div><div class='ctx'> void hfi1_vnic_txreq_deinit(struct hfi1_devdata *dd);</div><div class='ctx'> </div><div class='hunk'>@@ -157,7 +114,7 @@ bool hfi1_vnic_sdma_write_avail(struct hfi1_vnic_vport_info *vinfo,</div><div class='ctx'> </div><div class='ctx'> /* vnic rdma netdev operations */</div><div class='ctx'> struct net_device *hfi1_vnic_alloc_rn(struct ib_device *device,</div><div class='del'>-				      u8 port_num,</div><div class='add'>+				      u32 port_num,</div><div class='ctx'> 				      enum rdma_netdev_t type,</div><div class='ctx'> 				      const char *name,</div><div class='ctx'> 				      unsigned char name_assign_type,</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/vnic_main.c b/drivers/infiniband/hw/hfi1/vnic_main.c<br/>index b49e60e8397d..16a4c297a897 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/vnic_main.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/vnic_main.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/vnic_main.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/vnic_main.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='del'>- * Copyright(c) 2017 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='add'>+ * Copyright(c) 2017 - 2020 Intel Corporation.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -53,6 +11,7 @@</div><div class='ctx'> #include &lt;linux/if_vlan.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "vnic.h"</div><div class='add'>+#include "netdev.h"</div><div class='ctx'> </div><div class='ctx'> #define HFI_TX_TIMEOUT_MS 1000</div><div class='ctx'> </div><div class='hunk'>@@ -62,114 +21,6 @@</div><div class='ctx'> </div><div class='ctx'> static DEFINE_SPINLOCK(vport_cntr_lock);</div><div class='ctx'> </div><div class='del'>-static int setup_vnic_ctxt(struct hfi1_devdata *dd, struct hfi1_ctxtdata *uctxt)</div><div class='del'>-{</div><div class='del'>-	unsigned int rcvctrl_ops = 0;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	uctxt-&gt;do_interrupt = &amp;handle_receive_interrupt;</div><div class='del'>-</div><div class='del'>-	/* Now allocate the RcvHdr queue and eager buffers. */</div><div class='del'>-	ret = hfi1_create_rcvhdrq(dd, uctxt);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto done;</div><div class='del'>-</div><div class='del'>-	ret = hfi1_setup_eagerbufs(uctxt);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto done;</div><div class='del'>-</div><div class='del'>-	if (uctxt-&gt;rcvhdrtail_kvaddr)</div><div class='del'>-		clear_rcvhdrtail(uctxt);</div><div class='del'>-</div><div class='del'>-	rcvctrl_ops = HFI1_RCVCTRL_CTXT_ENB;</div><div class='del'>-	rcvctrl_ops |= HFI1_RCVCTRL_INTRAVAIL_ENB;</div><div class='del'>-</div><div class='del'>-	if (!HFI1_CAP_KGET_MASK(uctxt-&gt;flags, MULTI_PKT_EGR))</div><div class='del'>-		rcvctrl_ops |= HFI1_RCVCTRL_ONE_PKT_EGR_ENB;</div><div class='del'>-	if (HFI1_CAP_KGET_MASK(uctxt-&gt;flags, NODROP_EGR_FULL))</div><div class='del'>-		rcvctrl_ops |= HFI1_RCVCTRL_NO_EGR_DROP_ENB;</div><div class='del'>-	if (HFI1_CAP_KGET_MASK(uctxt-&gt;flags, NODROP_RHQ_FULL))</div><div class='del'>-		rcvctrl_ops |= HFI1_RCVCTRL_NO_RHQ_DROP_ENB;</div><div class='del'>-	if (HFI1_CAP_KGET_MASK(uctxt-&gt;flags, DMA_RTAIL))</div><div class='del'>-		rcvctrl_ops |= HFI1_RCVCTRL_TAILUPD_ENB;</div><div class='del'>-</div><div class='del'>-	hfi1_rcvctrl(uctxt-&gt;dd, rcvctrl_ops, uctxt);</div><div class='del'>-done:</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int allocate_vnic_ctxt(struct hfi1_devdata *dd,</div><div class='del'>-			      struct hfi1_ctxtdata **vnic_ctxt)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_ctxtdata *uctxt;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if (dd-&gt;flags &amp; HFI1_FROZEN)</div><div class='del'>-		return -EIO;</div><div class='del'>-</div><div class='del'>-	ret = hfi1_create_ctxtdata(dd-&gt;pport, dd-&gt;node, &amp;uctxt);</div><div class='del'>-	if (ret &lt; 0) {</div><div class='del'>-		dd_dev_err(dd, "Unable to create ctxtdata, failing open\n");</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	uctxt-&gt;flags = HFI1_CAP_KGET(MULTI_PKT_EGR) |</div><div class='del'>-			HFI1_CAP_KGET(NODROP_RHQ_FULL) |</div><div class='del'>-			HFI1_CAP_KGET(NODROP_EGR_FULL) |</div><div class='del'>-			HFI1_CAP_KGET(DMA_RTAIL);</div><div class='del'>-	uctxt-&gt;seq_cnt = 1;</div><div class='del'>-	uctxt-&gt;is_vnic = true;</div><div class='del'>-</div><div class='del'>-	msix_request_rcd_irq(uctxt);</div><div class='del'>-</div><div class='del'>-	hfi1_stats.sps_ctxts++;</div><div class='del'>-	dd_dev_dbg(dd, "created vnic context %d\n", uctxt-&gt;ctxt);</div><div class='del'>-	*vnic_ctxt = uctxt;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void deallocate_vnic_ctxt(struct hfi1_devdata *dd,</div><div class='del'>-				 struct hfi1_ctxtdata *uctxt)</div><div class='del'>-{</div><div class='del'>-	dd_dev_dbg(dd, "closing vnic context %d\n", uctxt-&gt;ctxt);</div><div class='del'>-	flush_wc();</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Disable receive context and interrupt available, reset all</div><div class='del'>-	 * RcvCtxtCtrl bits to default values.</div><div class='del'>-	 */</div><div class='del'>-	hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS |</div><div class='del'>-		     HFI1_RCVCTRL_TIDFLOW_DIS |</div><div class='del'>-		     HFI1_RCVCTRL_INTRAVAIL_DIS |</div><div class='del'>-		     HFI1_RCVCTRL_ONE_PKT_EGR_DIS |</div><div class='del'>-		     HFI1_RCVCTRL_NO_RHQ_DROP_DIS |</div><div class='del'>-		     HFI1_RCVCTRL_NO_EGR_DROP_DIS, uctxt);</div><div class='del'>-</div><div class='del'>-	/* msix_intr will always be &gt; 0, only clean up if this is true */</div><div class='del'>-	if (uctxt-&gt;msix_intr)</div><div class='del'>-		msix_free_irq(dd, uctxt-&gt;msix_intr);</div><div class='del'>-</div><div class='del'>-	uctxt-&gt;event_flags = 0;</div><div class='del'>-</div><div class='del'>-	hfi1_clear_tids(uctxt);</div><div class='del'>-	hfi1_clear_ctxt_pkey(dd, uctxt);</div><div class='del'>-</div><div class='del'>-	hfi1_stats.sps_ctxts--;</div><div class='del'>-</div><div class='del'>-	hfi1_free_ctxt(uctxt);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void hfi1_vnic_setup(struct hfi1_devdata *dd)</div><div class='del'>-{</div><div class='del'>-	xa_init(&amp;dd-&gt;vnic.vesws);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void hfi1_vnic_cleanup(struct hfi1_devdata *dd)</div><div class='del'>-{</div><div class='del'>-	WARN_ON(!xa_empty(&amp;dd-&gt;vnic.vesws));</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> #define SUM_GRP_COUNTERS(stats, qstats, x_grp) do {            \</div><div class='ctx'> 		u64 *src64, *dst64;                            \</div><div class='ctx'> 		for (src64 = &amp;qstats-&gt;x_grp.unicast,           \</div><div class='hunk'>@@ -179,6 +30,9 @@ void hfi1_vnic_cleanup(struct hfi1_devdata *dd)</div><div class='ctx'> 		}                                              \</div><div class='ctx'> 	} while (0)</div><div class='ctx'> </div><div class='add'>+#define VNIC_MASK (0xFF)</div><div class='add'>+#define VNIC_ID(val) ((1ull &lt;&lt; 24) | ((val) &amp; VNIC_MASK))</div><div class='add'>+</div><div class='ctx'> /* hfi1_vnic_update_stats - update statistics */</div><div class='ctx'> static void hfi1_vnic_update_stats(struct hfi1_vnic_vport_info *vinfo,</div><div class='ctx'> 				   struct opa_vnic_stats *stats)</div><div class='hunk'>@@ -454,71 +308,25 @@ static inline int hfi1_vnic_decap_skb(struct hfi1_vnic_rx_queue *rxq,</div><div class='ctx'> 	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline struct sk_buff *hfi1_vnic_get_skb(struct hfi1_vnic_rx_queue *rxq)</div><div class='add'>+static struct hfi1_vnic_vport_info *get_vnic_port(struct hfi1_devdata *dd,</div><div class='add'>+						  int vesw_id)</div><div class='ctx'> {</div><div class='del'>-	unsigned char *pad_info;</div><div class='del'>-	struct sk_buff *skb;</div><div class='add'>+	int vnic_id = VNIC_ID(vesw_id);</div><div class='ctx'> </div><div class='del'>-	skb = skb_dequeue(&amp;rxq-&gt;skbq);</div><div class='del'>-	if (unlikely(!skb))</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	/* remove tail padding and icrc */</div><div class='del'>-	pad_info = skb-&gt;data + skb-&gt;len - 1;</div><div class='del'>-	skb_trim(skb, (skb-&gt;len - OPA_VNIC_ICRC_TAIL_LEN -</div><div class='del'>-		       ((*pad_info) &amp; 0x7)));</div><div class='del'>-</div><div class='del'>-	return skb;</div><div class='add'>+	return hfi1_netdev_get_data(dd, vnic_id);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/* hfi1_vnic_handle_rx - handle skb receive */</div><div class='del'>-static void hfi1_vnic_handle_rx(struct hfi1_vnic_rx_queue *rxq,</div><div class='del'>-				int *work_done, int work_to_do)</div><div class='add'>+static struct hfi1_vnic_vport_info *get_first_vnic_port(struct hfi1_devdata *dd)</div><div class='ctx'> {</div><div class='del'>-	struct hfi1_vnic_vport_info *vinfo = rxq-&gt;vinfo;</div><div class='del'>-	struct sk_buff *skb;</div><div class='del'>-	int rc;</div><div class='del'>-</div><div class='del'>-	while (1) {</div><div class='del'>-		if (*work_done &gt;= work_to_do)</div><div class='del'>-			break;</div><div class='del'>-</div><div class='del'>-		skb = hfi1_vnic_get_skb(rxq);</div><div class='del'>-		if (unlikely(!skb))</div><div class='del'>-			break;</div><div class='del'>-</div><div class='del'>-		rc = hfi1_vnic_decap_skb(rxq, skb);</div><div class='del'>-		/* update rx counters */</div><div class='del'>-		hfi1_vnic_update_rx_counters(vinfo, rxq-&gt;idx, skb, rc);</div><div class='del'>-		if (unlikely(rc)) {</div><div class='del'>-			dev_kfree_skb_any(skb);</div><div class='del'>-			continue;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		skb_checksum_none_assert(skb);</div><div class='del'>-		skb-&gt;protocol = eth_type_trans(skb, rxq-&gt;netdev);</div><div class='del'>-</div><div class='del'>-		napi_gro_receive(&amp;rxq-&gt;napi, skb);</div><div class='del'>-		(*work_done)++;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* hfi1_vnic_napi - napi receive polling callback function */</div><div class='del'>-static int hfi1_vnic_napi(struct napi_struct *napi, int budget)</div><div class='del'>-{</div><div class='del'>-	struct hfi1_vnic_rx_queue *rxq = container_of(napi,</div><div class='del'>-					      struct hfi1_vnic_rx_queue, napi);</div><div class='del'>-	struct hfi1_vnic_vport_info *vinfo = rxq-&gt;vinfo;</div><div class='del'>-	int work_done = 0;</div><div class='add'>+	struct hfi1_vnic_vport_info *vinfo;</div><div class='add'>+	int next_id = VNIC_ID(0);</div><div class='ctx'> </div><div class='del'>-	v_dbg("napi %d budget %d\n", rxq-&gt;idx, budget);</div><div class='del'>-	hfi1_vnic_handle_rx(rxq, &amp;work_done, budget);</div><div class='add'>+	vinfo = hfi1_netdev_get_first_data(dd, &amp;next_id);</div><div class='ctx'> </div><div class='del'>-	v_dbg("napi %d work_done %d\n", rxq-&gt;idx, work_done);</div><div class='del'>-	if (work_done &lt; budget)</div><div class='del'>-		napi_complete(napi);</div><div class='add'>+	if (next_id &gt; VNIC_ID(VNIC_MASK))</div><div class='add'>+		return NULL;</div><div class='ctx'> </div><div class='del'>-	return work_done;</div><div class='add'>+	return vinfo;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void hfi1_vnic_bypass_rcv(struct hfi1_packet *packet)</div><div class='hunk'>@@ -527,13 +335,14 @@ void hfi1_vnic_bypass_rcv(struct hfi1_packet *packet)</div><div class='ctx'> 	struct hfi1_vnic_vport_info *vinfo = NULL;</div><div class='ctx'> 	struct hfi1_vnic_rx_queue *rxq;</div><div class='ctx'> 	struct sk_buff *skb;</div><div class='del'>-	int l4_type, vesw_id = -1;</div><div class='add'>+	int l4_type, vesw_id = -1, rc;</div><div class='ctx'> 	u8 q_idx;</div><div class='add'>+	unsigned char *pad_info;</div><div class='ctx'> </div><div class='ctx'> 	l4_type = hfi1_16B_get_l4(packet-&gt;ebuf);</div><div class='ctx'> 	if (likely(l4_type == OPA_16B_L4_ETHR)) {</div><div class='ctx'> 		vesw_id = HFI1_VNIC_GET_VESWID(packet-&gt;ebuf);</div><div class='del'>-		vinfo = xa_load(&amp;dd-&gt;vnic.vesws, vesw_id);</div><div class='add'>+		vinfo = get_vnic_port(dd, vesw_id);</div><div class='ctx'> </div><div class='ctx'> 		/*</div><div class='ctx'> 		 * In case of invalid vesw id, count the error on</div><div class='hunk'>@@ -541,10 +350,8 @@ void hfi1_vnic_bypass_rcv(struct hfi1_packet *packet)</div><div class='ctx'> 		 */</div><div class='ctx'> 		if (unlikely(!vinfo)) {</div><div class='ctx'> 			struct hfi1_vnic_vport_info *vinfo_tmp;</div><div class='del'>-			unsigned long index = 0;</div><div class='ctx'> </div><div class='del'>-			vinfo_tmp = xa_find(&amp;dd-&gt;vnic.vesws, &amp;index, ULONG_MAX,</div><div class='del'>-					XA_PRESENT);</div><div class='add'>+			vinfo_tmp = get_first_vnic_port(dd);</div><div class='ctx'> 			if (vinfo_tmp) {</div><div class='ctx'> 				spin_lock(&amp;vport_cntr_lock);</div><div class='ctx'> 				vinfo_tmp-&gt;stats[0].netstats.rx_nohandler++;</div><div class='hunk'>@@ -563,12 +370,6 @@ void hfi1_vnic_bypass_rcv(struct hfi1_packet *packet)</div><div class='ctx'> 	rxq = &amp;vinfo-&gt;rxq[q_idx];</div><div class='ctx'> 	if (unlikely(!netif_oper_up(vinfo-&gt;netdev))) {</div><div class='ctx'> 		vinfo-&gt;stats[q_idx].rx_drop_state++;</div><div class='del'>-		skb_queue_purge(&amp;rxq-&gt;skbq);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (unlikely(skb_queue_len(&amp;rxq-&gt;skbq) &gt; HFI1_VNIC_RCV_Q_SIZE)) {</div><div class='del'>-		vinfo-&gt;stats[q_idx].netstats.rx_fifo_errors++;</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -580,62 +381,65 @@ void hfi1_vnic_bypass_rcv(struct hfi1_packet *packet)</div><div class='ctx'> </div><div class='ctx'> 	memcpy(skb-&gt;data, packet-&gt;ebuf, packet-&gt;tlen);</div><div class='ctx'> 	skb_put(skb, packet-&gt;tlen);</div><div class='del'>-	skb_queue_tail(&amp;rxq-&gt;skbq, skb);</div><div class='ctx'> </div><div class='del'>-	if (napi_schedule_prep(&amp;rxq-&gt;napi)) {</div><div class='del'>-		v_dbg("napi %d scheduling\n", q_idx);</div><div class='del'>-		__napi_schedule(&amp;rxq-&gt;napi);</div><div class='add'>+	pad_info = skb-&gt;data + skb-&gt;len - 1;</div><div class='add'>+	skb_trim(skb, (skb-&gt;len - OPA_VNIC_ICRC_TAIL_LEN -</div><div class='add'>+		       ((*pad_info) &amp; 0x7)));</div><div class='add'>+</div><div class='add'>+	rc = hfi1_vnic_decap_skb(rxq, skb);</div><div class='add'>+</div><div class='add'>+	/* update rx counters */</div><div class='add'>+	hfi1_vnic_update_rx_counters(vinfo, rxq-&gt;idx, skb, rc);</div><div class='add'>+	if (unlikely(rc)) {</div><div class='add'>+		dev_kfree_skb_any(skb);</div><div class='add'>+		return;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	skb_checksum_none_assert(skb);</div><div class='add'>+	skb-&gt;protocol = eth_type_trans(skb, rxq-&gt;netdev);</div><div class='add'>+</div><div class='add'>+	napi_gro_receive(&amp;rxq-&gt;napi, skb);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hfi1_vnic_up(struct hfi1_vnic_vport_info *vinfo)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_devdata *dd = vinfo-&gt;dd;</div><div class='ctx'> 	struct net_device *netdev = vinfo-&gt;netdev;</div><div class='del'>-	int i, rc;</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='ctx'> 	/* ensure virtual eth switch id is valid */</div><div class='ctx'> 	if (!vinfo-&gt;vesw_id)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	rc = xa_insert(&amp;dd-&gt;vnic.vesws, vinfo-&gt;vesw_id, vinfo, GFP_KERNEL);</div><div class='add'>+	rc = hfi1_netdev_add_data(dd, VNIC_ID(vinfo-&gt;vesw_id), vinfo);</div><div class='ctx'> 	if (rc &lt; 0)</div><div class='ctx'> 		return rc;</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; vinfo-&gt;num_rx_q; i++) {</div><div class='del'>-		struct hfi1_vnic_rx_queue *rxq = &amp;vinfo-&gt;rxq[i];</div><div class='del'>-</div><div class='del'>-		skb_queue_head_init(&amp;rxq-&gt;skbq);</div><div class='del'>-		napi_enable(&amp;rxq-&gt;napi);</div><div class='del'>-	}</div><div class='add'>+	rc = hfi1_netdev_rx_init(dd);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_remove;</div><div class='ctx'> </div><div class='ctx'> 	netif_carrier_on(netdev);</div><div class='ctx'> 	netif_tx_start_all_queues(netdev);</div><div class='ctx'> 	set_bit(HFI1_VNIC_UP, &amp;vinfo-&gt;flags);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='add'>+</div><div class='add'>+err_remove:</div><div class='add'>+	hfi1_netdev_remove_data(dd, VNIC_ID(vinfo-&gt;vesw_id));</div><div class='add'>+	return rc;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void hfi1_vnic_down(struct hfi1_vnic_vport_info *vinfo)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_devdata *dd = vinfo-&gt;dd;</div><div class='del'>-	u8 i;</div><div class='ctx'> </div><div class='ctx'> 	clear_bit(HFI1_VNIC_UP, &amp;vinfo-&gt;flags);</div><div class='ctx'> 	netif_carrier_off(vinfo-&gt;netdev);</div><div class='ctx'> 	netif_tx_disable(vinfo-&gt;netdev);</div><div class='del'>-	xa_erase(&amp;dd-&gt;vnic.vesws, vinfo-&gt;vesw_id);</div><div class='del'>-</div><div class='del'>-	/* ensure irqs see the change */</div><div class='del'>-	msix_vnic_synchronize_irq(dd);</div><div class='add'>+	hfi1_netdev_remove_data(dd, VNIC_ID(vinfo-&gt;vesw_id));</div><div class='ctx'> </div><div class='del'>-	/* remove unread skbs */</div><div class='del'>-	for (i = 0; i &lt; vinfo-&gt;num_rx_q; i++) {</div><div class='del'>-		struct hfi1_vnic_rx_queue *rxq = &amp;vinfo-&gt;rxq[i];</div><div class='del'>-</div><div class='del'>-		napi_disable(&amp;rxq-&gt;napi);</div><div class='del'>-		skb_queue_purge(&amp;rxq-&gt;skbq);</div><div class='del'>-	}</div><div class='add'>+	hfi1_netdev_rx_destroy(dd);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hfi1_netdev_open(struct net_device *netdev)</div><div class='hunk'>@@ -660,70 +464,31 @@ static int hfi1_netdev_close(struct net_device *netdev)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hfi1_vnic_allot_ctxt(struct hfi1_devdata *dd,</div><div class='del'>-				struct hfi1_ctxtdata **vnic_ctxt)</div><div class='del'>-{</div><div class='del'>-	int rc;</div><div class='del'>-</div><div class='del'>-	rc = allocate_vnic_ctxt(dd, vnic_ctxt);</div><div class='del'>-	if (rc) {</div><div class='del'>-		dd_dev_err(dd, "vnic ctxt alloc failed %d\n", rc);</div><div class='del'>-		return rc;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	rc = setup_vnic_ctxt(dd, *vnic_ctxt);</div><div class='del'>-	if (rc) {</div><div class='del'>-		dd_dev_err(dd, "vnic ctxt setup failed %d\n", rc);</div><div class='del'>-		deallocate_vnic_ctxt(dd, *vnic_ctxt);</div><div class='del'>-		*vnic_ctxt = NULL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return rc;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int hfi1_vnic_init(struct hfi1_vnic_vport_info *vinfo)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_devdata *dd = vinfo-&gt;dd;</div><div class='del'>-	int i, rc = 0;</div><div class='add'>+	int rc = 0;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;hfi1_mutex);</div><div class='del'>-	if (!dd-&gt;vnic.num_vports) {</div><div class='add'>+	if (!dd-&gt;vnic_num_vports) {</div><div class='ctx'> 		rc = hfi1_vnic_txreq_init(dd);</div><div class='ctx'> 		if (rc)</div><div class='ctx'> 			goto txreq_fail;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	for (i = dd-&gt;vnic.num_ctxt; i &lt; vinfo-&gt;num_rx_q; i++) {</div><div class='del'>-		rc = hfi1_vnic_allot_ctxt(dd, &amp;dd-&gt;vnic.ctxt[i]);</div><div class='del'>-		if (rc)</div><div class='del'>-			break;</div><div class='del'>-		hfi1_rcd_get(dd-&gt;vnic.ctxt[i]);</div><div class='del'>-		dd-&gt;vnic.ctxt[i]-&gt;vnic_q_idx = i;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (i &lt; vinfo-&gt;num_rx_q) {</div><div class='del'>-		/*</div><div class='del'>-		 * If required amount of contexts is not</div><div class='del'>-		 * allocated successfully then remaining contexts</div><div class='del'>-		 * are released.</div><div class='del'>-		 */</div><div class='del'>-		while (i-- &gt; dd-&gt;vnic.num_ctxt) {</div><div class='del'>-			deallocate_vnic_ctxt(dd, dd-&gt;vnic.ctxt[i]);</div><div class='del'>-			hfi1_rcd_put(dd-&gt;vnic.ctxt[i]);</div><div class='del'>-			dd-&gt;vnic.ctxt[i] = NULL;</div><div class='del'>-		}</div><div class='add'>+	rc = hfi1_netdev_rx_init(dd);</div><div class='add'>+	if (rc) {</div><div class='add'>+		dd_dev_err(dd, "Unable to initialize netdev contexts\n");</div><div class='ctx'> 		goto alloc_fail;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (dd-&gt;vnic.num_ctxt != i) {</div><div class='del'>-		dd-&gt;vnic.num_ctxt = i;</div><div class='del'>-		hfi1_init_vnic_rsm(dd);</div><div class='del'>-	}</div><div class='add'>+	hfi1_init_vnic_rsm(dd);</div><div class='ctx'> </div><div class='del'>-	dd-&gt;vnic.num_vports++;</div><div class='add'>+	dd-&gt;vnic_num_vports++;</div><div class='ctx'> 	hfi1_vnic_sdma_init(vinfo);</div><div class='add'>+</div><div class='ctx'> alloc_fail:</div><div class='del'>-	if (!dd-&gt;vnic.num_vports)</div><div class='add'>+	if (!dd-&gt;vnic_num_vports)</div><div class='ctx'> 		hfi1_vnic_txreq_deinit(dd);</div><div class='ctx'> txreq_fail:</div><div class='ctx'> 	mutex_unlock(&amp;hfi1_mutex);</div><div class='hunk'>@@ -733,20 +498,14 @@ txreq_fail:</div><div class='ctx'> static void hfi1_vnic_deinit(struct hfi1_vnic_vport_info *vinfo)</div><div class='ctx'> {</div><div class='ctx'> 	struct hfi1_devdata *dd = vinfo-&gt;dd;</div><div class='del'>-	int i;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;hfi1_mutex);</div><div class='del'>-	if (--dd-&gt;vnic.num_vports == 0) {</div><div class='del'>-		for (i = 0; i &lt; dd-&gt;vnic.num_ctxt; i++) {</div><div class='del'>-			deallocate_vnic_ctxt(dd, dd-&gt;vnic.ctxt[i]);</div><div class='del'>-			hfi1_rcd_put(dd-&gt;vnic.ctxt[i]);</div><div class='del'>-			dd-&gt;vnic.ctxt[i] = NULL;</div><div class='del'>-		}</div><div class='add'>+	if (--dd-&gt;vnic_num_vports == 0) {</div><div class='ctx'> 		hfi1_deinit_vnic_rsm(dd);</div><div class='del'>-		dd-&gt;vnic.num_ctxt = 0;</div><div class='ctx'> 		hfi1_vnic_txreq_deinit(dd);</div><div class='ctx'> 	}</div><div class='ctx'> 	mutex_unlock(&amp;hfi1_mutex);</div><div class='add'>+	hfi1_netdev_rx_destroy(dd);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void hfi1_vnic_set_vesw_id(struct net_device *netdev, int id)</div><div class='hunk'>@@ -792,7 +551,7 @@ static void hfi1_vnic_free_rn(struct net_device *netdev)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct net_device *hfi1_vnic_alloc_rn(struct ib_device *device,</div><div class='del'>-				      u8 port_num,</div><div class='add'>+				      u32 port_num,</div><div class='ctx'> 				      enum rdma_netdev_t type,</div><div class='ctx'> 				      const char *name,</div><div class='ctx'> 				      unsigned char name_assign_type,</div><div class='hunk'>@@ -804,7 +563,7 @@ struct net_device *hfi1_vnic_alloc_rn(struct ib_device *device,</div><div class='ctx'> 	struct rdma_netdev *rn;</div><div class='ctx'> 	int i, size, rc;</div><div class='ctx'> </div><div class='del'>-	if (!dd-&gt;num_vnic_contexts)</div><div class='add'>+	if (!dd-&gt;num_netdev_contexts)</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='ctx'> 	if (!port_num || (port_num &gt; dd-&gt;num_pports))</div><div class='hunk'>@@ -815,15 +574,16 @@ struct net_device *hfi1_vnic_alloc_rn(struct ib_device *device,</div><div class='ctx'> </div><div class='ctx'> 	size = sizeof(struct opa_vnic_rdma_netdev) + sizeof(*vinfo);</div><div class='ctx'> 	netdev = alloc_netdev_mqs(size, name, name_assign_type, setup,</div><div class='del'>-				  dd-&gt;num_sdma, dd-&gt;num_vnic_contexts);</div><div class='add'>+				  chip_sdma_engines(dd),</div><div class='add'>+				  dd-&gt;num_netdev_contexts);</div><div class='ctx'> 	if (!netdev)</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='ctx'> 	rn = netdev_priv(netdev);</div><div class='ctx'> 	vinfo = opa_vnic_dev_priv(netdev);</div><div class='ctx'> 	vinfo-&gt;dd = dd;</div><div class='del'>-	vinfo-&gt;num_tx_q = dd-&gt;num_sdma;</div><div class='del'>-	vinfo-&gt;num_rx_q = dd-&gt;num_vnic_contexts;</div><div class='add'>+	vinfo-&gt;num_tx_q = chip_sdma_engines(dd);</div><div class='add'>+	vinfo-&gt;num_rx_q = dd-&gt;num_netdev_contexts;</div><div class='ctx'> 	vinfo-&gt;netdev = netdev;</div><div class='ctx'> 	rn-&gt;free_rdma_netdev = hfi1_vnic_free_rn;</div><div class='ctx'> 	rn-&gt;set_id = hfi1_vnic_set_vesw_id;</div><div class='hunk'>@@ -841,7 +601,6 @@ struct net_device *hfi1_vnic_alloc_rn(struct ib_device *device,</div><div class='ctx'> 		rxq-&gt;idx = i;</div><div class='ctx'> 		rxq-&gt;vinfo = vinfo;</div><div class='ctx'> 		rxq-&gt;netdev = netdev;</div><div class='del'>-		netif_napi_add(netdev, &amp;rxq-&gt;napi, hfi1_vnic_napi, 64);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	rc = hfi1_vnic_init(vinfo);</div><div class='head'>diff --git a/drivers/infiniband/hw/hfi1/vnic_sdma.c b/drivers/infiniband/hw/hfi1/vnic_sdma.c<br/>index af1b1ffcb38e..6caf01ba0bca 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/vnic_sdma.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hfi1/vnic_sdma.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hfi1/vnic_sdma.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hfi1/vnic_sdma.c</a></div><div class='hunk'>@@ -1,48 +1,6 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright(c) 2017 - 2018 Intel Corporation.</div><div class='del'>- *</div><div class='del'>- * This file is provided under a dual BSD/GPLv2 license.  When using or</div><div class='del'>- * redistributing this file, you may do so under either license.</div><div class='del'>- *</div><div class='del'>- * GPL LICENSE SUMMARY</div><div class='del'>- *</div><div class='del'>- * This program is free software; you can redistribute it and/or modify</div><div class='del'>- * it under the terms of version 2 of the GNU General Public License as</div><div class='del'>- * published by the Free Software Foundation.</div><div class='del'>- *</div><div class='del'>- * This program is distributed in the hope that it will be useful, but</div><div class='del'>- * WITHOUT ANY WARRANTY; without even the implied warranty of</div><div class='del'>- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</div><div class='del'>- * General Public License for more details.</div><div class='del'>- *</div><div class='del'>- * BSD LICENSE</div><div class='del'>- *</div><div class='del'>- * Redistribution and use in source and binary forms, with or without</div><div class='del'>- * modification, are permitted provided that the following conditions</div><div class='del'>- * are met:</div><div class='del'>- *</div><div class='del'>- *  - Redistributions of source code must retain the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer.</div><div class='del'>- *  - Redistributions in binary form must reproduce the above copyright</div><div class='del'>- *    notice, this list of conditions and the following disclaimer in</div><div class='del'>- *    the documentation and/or other materials provided with the</div><div class='del'>- *    distribution.</div><div class='del'>- *  - Neither the name of Intel Corporation nor the names of its</div><div class='del'>- *    contributors may be used to endorse or promote products derived</div><div class='del'>- *    from this software without specific prior written permission.</div><div class='del'>- *</div><div class='del'>- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</div><div class='del'>- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</div><div class='del'>- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</div><div class='del'>- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</div><div class='del'>- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</div><div class='del'>- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</div><div class='del'>- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</div><div class='del'>- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</div><div class='del'>- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</div><div class='del'>- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</div><div class='del'>- *</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -102,14 +60,15 @@ static noinline int build_vnic_ulp_payload(struct sdma_engine *sde,</div><div class='ctx'> 		goto bail_txadd;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; skb_shinfo(tx-&gt;skb)-&gt;nr_frags; i++) {</div><div class='del'>-		struct skb_frag_struct *frag = &amp;skb_shinfo(tx-&gt;skb)-&gt;frags[i];</div><div class='add'>+		skb_frag_t *frag = &amp;skb_shinfo(tx-&gt;skb)-&gt;frags[i];</div><div class='ctx'> </div><div class='ctx'> 		/* combine physically continuous fragments later? */</div><div class='ctx'> 		ret = sdma_txadd_page(sde-&gt;dd,</div><div class='ctx'> 				      &amp;tx-&gt;txreq,</div><div class='ctx'> 				      skb_frag_page(frag),</div><div class='del'>-				      frag-&gt;page_offset,</div><div class='del'>-				      skb_frag_size(frag));</div><div class='add'>+				      skb_frag_off(frag),</div><div class='add'>+				      skb_frag_size(frag),</div><div class='add'>+				      NULL, NULL, NULL);</div><div class='ctx'> 		if (unlikely(ret))</div><div class='ctx'> 			goto bail_txadd;</div><div class='ctx'> 	}</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/Kconfig b/drivers/infiniband/hw/hns/Kconfig<br/>index fddb5fdf92de..44cdb706fe27 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/Kconfig?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/Kconfig</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/Kconfig?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/Kconfig</a></div><div class='hunk'>@@ -1,31 +1,11 @@</div><div class='del'>-config INFINIBAND_HNS</div><div class='del'>-	tristate "HNS RoCE Driver"</div><div class='del'>-	depends on NET_VENDOR_HISILICON</div><div class='del'>-	depends on ARM64 || (COMPILE_TEST &amp;&amp; 64BIT)</div><div class='del'>-	---help---</div><div class='del'>-	  This is a RoCE/RDMA driver for the Hisilicon RoCE engine. The engine</div><div class='del'>-	  is used in Hisilicon Hip06 and more further ICT SoC based on</div><div class='del'>-	  platform device.</div><div class='del'>-</div><div class='del'>-	  To compile this driver as a module, choose M here: the module</div><div class='del'>-	  will be called hns-roce.</div><div class='del'>-</div><div class='del'>-config INFINIBAND_HNS_HIP06</div><div class='del'>-	tristate "Hisilicon Hip06 Family RoCE support"</div><div class='del'>-	depends on INFINIBAND_HNS &amp;&amp; HNS &amp;&amp; HNS_DSAF &amp;&amp; HNS_ENET</div><div class='del'>-	---help---</div><div class='del'>-	  RoCE driver support for Hisilicon RoCE engine in Hisilicon Hip06 and</div><div class='del'>-	  Hip07 SoC. These RoCE engines are platform devices.</div><div class='del'>-</div><div class='del'>-	  To compile this driver as a module, choose M here: the module</div><div class='del'>-	  will be called hns-roce-hw-v1.</div><div class='del'>-</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='ctx'> config INFINIBAND_HNS_HIP08</div><div class='ctx'> 	tristate "Hisilicon Hip08 Family RoCE support"</div><div class='del'>-	depends on INFINIBAND_HNS &amp;&amp; PCI &amp;&amp; HNS3</div><div class='del'>-	---help---</div><div class='add'>+	depends on ARM64 || (COMPILE_TEST &amp;&amp; 64BIT)</div><div class='add'>+	depends on PCI &amp;&amp; HNS3</div><div class='add'>+	help</div><div class='ctx'> 	  RoCE driver support for Hisilicon RoCE engine in Hisilicon Hip08 SoC.</div><div class='ctx'> 	  The RoCE engine is a PCI device.</div><div class='ctx'> </div><div class='del'>-	  To compile this driver as a module, choose M here: the module</div><div class='del'>-	  will be called hns-roce-hw-v2.</div><div class='add'>+	  To compile this driver, choose M here. This module will be called</div><div class='add'>+	  hns-roce-hw-v2.</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile<br/>index eee5205f936f..d07ef02c5231 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/Makefile?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/Makefile</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/Makefile?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/Makefile</a></div><div class='hunk'>@@ -1,14 +1,16 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='ctx'> #</div><div class='ctx'> # Makefile for the Hisilicon RoCE drivers.</div><div class='ctx'> #</div><div class='ctx'> </div><div class='ctx'> ccflags-y :=  -I $(srctree)/drivers/net/ethernet/hisilicon/hns3</div><div class='add'>+ccflags-y +=  -I $(srctree)/drivers/net/ethernet/hisilicon/hns3/hns3pf</div><div class='add'>+ccflags-y +=  -I $(srctree)/drivers/net/ethernet/hisilicon/hns3/hns3_common</div><div class='add'>+ccflags-y +=  -I $(src)</div><div class='ctx'> </div><div class='del'>-obj-$(CONFIG_INFINIBAND_HNS) += hns-roce.o</div><div class='del'>-hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \</div><div class='add'>+hns-roce-hw-v2-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \</div><div class='ctx'> 	hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \</div><div class='del'>-	hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o</div><div class='del'>-obj-$(CONFIG_INFINIBAND_HNS_HIP06) += hns-roce-hw-v1.o</div><div class='del'>-hns-roce-hw-v1-objs := hns_roce_hw_v1.o</div><div class='add'>+	hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o \</div><div class='add'>+	hns_roce_debugfs.o hns_roce_hw_v2.o hns_roce_bond.o</div><div class='add'>+</div><div class='ctx'> obj-$(CONFIG_INFINIBAND_HNS_HIP08) += hns-roce-hw-v2.o</div><div class='del'>-hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c<br/>index cdd2ac24fc2a..0c1c32d23c88 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_ah.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_ah.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_ah.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_ah.c</a></div><div class='hunk'>@@ -30,59 +30,98 @@</div><div class='ctx'>  * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/platform_device.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_addr.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_cache.h&gt;</div><div class='ctx'> #include "hns_roce_device.h"</div><div class='add'>+#include "hns_roce_hw_v2.h"</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_PORT_NUM_SHIFT		24</div><div class='del'>-#define HNS_ROCE_VLAN_SL_BIT_MASK	7</div><div class='del'>-#define HNS_ROCE_VLAN_SL_SHIFT		13</div><div class='add'>+static inline u16 get_ah_udp_sport(const struct rdma_ah_attr *ah_attr)</div><div class='add'>+{</div><div class='add'>+	u32 fl = ah_attr-&gt;grh.flow_label;</div><div class='add'>+	u16 sport;</div><div class='add'>+</div><div class='add'>+	if (!fl)</div><div class='add'>+		sport = get_random_u32_inclusive(IB_ROCE_UDP_ENCAP_VALID_PORT_MIN,</div><div class='add'>+						 IB_ROCE_UDP_ENCAP_VALID_PORT_MAX);</div><div class='add'>+	else</div><div class='add'>+		sport = rdma_flow_label_to_udp_sport(fl);</div><div class='add'>+</div><div class='add'>+	return sport;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr,</div><div class='del'>-		       u32 flags, struct ib_udata *udata)</div><div class='add'>+int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,</div><div class='add'>+		       struct ib_udata *udata)</div><div class='ctx'> {</div><div class='add'>+	struct rdma_ah_attr *ah_attr = init_attr-&gt;ah_attr;</div><div class='add'>+	const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ibah-&gt;device);</div><div class='del'>-	const struct ib_gid_attr *gid_attr;</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	struct hns_roce_ib_create_ah_resp resp = {};</div><div class='ctx'> 	struct hns_roce_ah *ah = to_hr_ah(ibah);</div><div class='del'>-	u16 vlan_tag = 0xffff;</div><div class='del'>-	const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);</div><div class='del'>-	bool vlan_en = false;</div><div class='add'>+	u8 tclass = get_tclass(grh);</div><div class='add'>+	u8 priority = 0;</div><div class='add'>+	u8 tc_mode = 0;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	gid_attr = ah_attr-&gt;grh.sgid_attr;</div><div class='del'>-	ret = rdma_read_gid_l2_fields(gid_attr, &amp;vlan_tag, NULL);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	/* Get mac address */</div><div class='del'>-	memcpy(ah-&gt;av.mac, ah_attr-&gt;roce.dmac, ETH_ALEN);</div><div class='del'>-</div><div class='del'>-	if (vlan_tag &lt; VLAN_CFI_MASK) {</div><div class='del'>-		vlan_en = true;</div><div class='del'>-		vlan_tag |= (rdma_ah_get_sl(ah_attr) &amp;</div><div class='del'>-			     HNS_ROCE_VLAN_SL_BIT_MASK) &lt;&lt;</div><div class='del'>-			     HNS_ROCE_VLAN_SL_SHIFT;</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08 &amp;&amp; udata) {</div><div class='add'>+		ret = -EOPNOTSUPP;</div><div class='add'>+		goto err_out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ah-&gt;av.port_pd = cpu_to_le32(to_hr_pd(ibah-&gt;pd)-&gt;pdn |</div><div class='del'>-				     (rdma_ah_get_port_num(ah_attr) &lt;&lt;</div><div class='del'>-				     HNS_ROCE_PORT_NUM_SHIFT));</div><div class='add'>+	ah-&gt;av.port = rdma_ah_get_port_num(ah_attr);</div><div class='ctx'> 	ah-&gt;av.gid_index = grh-&gt;sgid_index;</div><div class='del'>-	ah-&gt;av.vlan = cpu_to_le16(vlan_tag);</div><div class='del'>-	ah-&gt;av.vlan_en = vlan_en;</div><div class='del'>-	dev_dbg(dev, "gid_index = 0x%x,vlan = 0x%x\n", ah-&gt;av.gid_index,</div><div class='del'>-		ah-&gt;av.vlan);</div><div class='ctx'> </div><div class='ctx'> 	if (rdma_ah_get_static_rate(ah_attr))</div><div class='ctx'> 		ah-&gt;av.stat_rate = IB_RATE_10_GBPS;</div><div class='ctx'> </div><div class='add'>+	ah-&gt;av.hop_limit = grh-&gt;hop_limit;</div><div class='add'>+	ah-&gt;av.flowlabel = grh-&gt;flow_label;</div><div class='add'>+	ah-&gt;av.udp_sport = get_ah_udp_sport(ah_attr);</div><div class='add'>+	ah-&gt;av.tclass = tclass;</div><div class='add'>+</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;get_dscp(hr_dev, tclass, &amp;tc_mode, &amp;priority);</div><div class='add'>+	if (ret == -EOPNOTSUPP)</div><div class='add'>+		ret = 0;</div><div class='add'>+</div><div class='add'>+	if (ret &amp;&amp; grh-&gt;sgid_attr-&gt;gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)</div><div class='add'>+		goto err_out;</div><div class='add'>+</div><div class='add'>+	if (tc_mode == HNAE3_TC_MAP_MODE_DSCP &amp;&amp;</div><div class='add'>+	    grh-&gt;sgid_attr-&gt;gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)</div><div class='add'>+		ah-&gt;av.sl = priority;</div><div class='add'>+	else</div><div class='add'>+		ah-&gt;av.sl = rdma_ah_get_sl(ah_attr);</div><div class='add'>+</div><div class='add'>+	if (!check_sl_valid(hr_dev, ah-&gt;av.sl)) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err_out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	memcpy(ah-&gt;av.dgid, grh-&gt;dgid.raw, HNS_ROCE_GID_SIZE);</div><div class='del'>-	ah-&gt;av.sl_tclass_flowlabel = cpu_to_le32(rdma_ah_get_sl(ah_attr) &lt;&lt;</div><div class='del'>-						 HNS_ROCE_SL_SHIFT);</div><div class='add'>+	memcpy(ah-&gt;av.mac, ah_attr-&gt;roce.dmac, ETH_ALEN);</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	/* HIP08 needs to record vlan info in Address Vector */</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08) {</div><div class='add'>+		ret = rdma_read_gid_l2_fields(ah_attr-&gt;grh.sgid_attr,</div><div class='add'>+					      &amp;ah-&gt;av.vlan_id, NULL);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_out;</div><div class='add'>+</div><div class='add'>+		ah-&gt;av.vlan_en = ah-&gt;av.vlan_id &lt; VLAN_N_VID;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		resp.priority = ah-&gt;av.sl;</div><div class='add'>+		resp.tc_mode = tc_mode;</div><div class='add'>+		memcpy(resp.dmac, ah_attr-&gt;roce.dmac, ETH_ALEN);</div><div class='add'>+		ret = ib_copy_to_udata(udata, &amp;resp,</div><div class='add'>+				       min(udata-&gt;outlen, sizeof(resp)));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+err_out:</div><div class='add'>+	if (ret)</div><div class='add'>+		atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_AH_CREATE_ERR_CNT]);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int hns_roce_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)</div><div class='hunk'>@@ -91,23 +130,12 @@ int hns_roce_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)</div><div class='ctx'> </div><div class='ctx'> 	memset(ah_attr, 0, sizeof(*ah_attr));</div><div class='ctx'> </div><div class='del'>-	rdma_ah_set_sl(ah_attr, (le32_to_cpu(ah-&gt;av.sl_tclass_flowlabel) &gt;&gt;</div><div class='del'>-				 HNS_ROCE_SL_SHIFT));</div><div class='del'>-	rdma_ah_set_port_num(ah_attr, (le32_to_cpu(ah-&gt;av.port_pd) &gt;&gt;</div><div class='del'>-				       HNS_ROCE_PORT_NUM_SHIFT));</div><div class='add'>+	rdma_ah_set_sl(ah_attr, ah-&gt;av.sl);</div><div class='add'>+	rdma_ah_set_port_num(ah_attr, ah-&gt;av.port);</div><div class='ctx'> 	rdma_ah_set_static_rate(ah_attr, ah-&gt;av.stat_rate);</div><div class='del'>-	rdma_ah_set_grh(ah_attr, NULL,</div><div class='del'>-			(le32_to_cpu(ah-&gt;av.sl_tclass_flowlabel) &amp;</div><div class='del'>-			 HNS_ROCE_FLOW_LABEL_MASK), ah-&gt;av.gid_index,</div><div class='del'>-			ah-&gt;av.hop_limit,</div><div class='del'>-			(le32_to_cpu(ah-&gt;av.sl_tclass_flowlabel) &gt;&gt;</div><div class='del'>-			 HNS_ROCE_TCLASS_SHIFT));</div><div class='add'>+	rdma_ah_set_grh(ah_attr, NULL, ah-&gt;av.flowlabel,</div><div class='add'>+			ah-&gt;av.gid_index, ah-&gt;av.hop_limit, ah-&gt;av.tclass);</div><div class='ctx'> 	rdma_ah_set_dgid_raw(ah_attr, ah-&gt;av.dgid);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='del'>-</div><div class='del'>-void hns_roce_destroy_ah(struct ib_ah *ah, u32 flags)</div><div class='del'>-{</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c<br/>index dac058d3df53..6ee911f6885b 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_alloc.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_alloc.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_alloc.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_alloc.c</a></div><div class='hunk'>@@ -31,220 +31,157 @@</div><div class='ctx'>  * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/platform_device.h&gt;</div><div class='ctx'> #include &lt;linux/vmalloc.h&gt;</div><div class='add'>+#include &lt;rdma/ib_umem.h&gt;</div><div class='ctx'> #include "hns_roce_device.h"</div><div class='ctx'> </div><div class='del'>-int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj)</div><div class='add'>+void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf)</div><div class='ctx'> {</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	spin_lock(&amp;bitmap-&gt;lock);</div><div class='del'>-	*obj = find_next_zero_bit(bitmap-&gt;table, bitmap-&gt;max, bitmap-&gt;last);</div><div class='del'>-	if (*obj &gt;= bitmap-&gt;max) {</div><div class='del'>-		bitmap-&gt;top = (bitmap-&gt;top + bitmap-&gt;max + bitmap-&gt;reserved_top)</div><div class='del'>-			       &amp; bitmap-&gt;mask;</div><div class='del'>-		*obj = find_first_zero_bit(bitmap-&gt;table, bitmap-&gt;max);</div><div class='del'>-	}</div><div class='add'>+	struct hns_roce_buf_list *trunks;</div><div class='add'>+	u32 i;</div><div class='ctx'> </div><div class='del'>-	if (*obj &lt; bitmap-&gt;max) {</div><div class='del'>-		set_bit(*obj, bitmap-&gt;table);</div><div class='del'>-		bitmap-&gt;last = (*obj + 1);</div><div class='del'>-		if (bitmap-&gt;last == bitmap-&gt;max)</div><div class='del'>-			bitmap-&gt;last = 0;</div><div class='del'>-		*obj |= bitmap-&gt;top;</div><div class='del'>-	} else {</div><div class='del'>-		ret = -1;</div><div class='del'>-	}</div><div class='add'>+	if (!buf)</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-	spin_unlock(&amp;bitmap-&gt;lock);</div><div class='add'>+	trunks = buf-&gt;trunk_list;</div><div class='add'>+	if (trunks) {</div><div class='add'>+		buf-&gt;trunk_list = NULL;</div><div class='add'>+		for (i = 0; i &lt; buf-&gt;ntrunks; i++)</div><div class='add'>+			dma_free_coherent(hr_dev-&gt;dev, 1 &lt;&lt; buf-&gt;trunk_shift,</div><div class='add'>+					  trunks[i].buf, trunks[i].map);</div><div class='ctx'> </div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='add'>+		kfree(trunks);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj,</div><div class='del'>-			  int rr)</div><div class='del'>-{</div><div class='del'>-	hns_roce_bitmap_free_range(bitmap, obj, 1, rr);</div><div class='add'>+	kfree(buf);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_bitmap_free);</div><div class='ctx'> </div><div class='del'>-int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt,</div><div class='del'>-				int align, unsigned long *obj)</div><div class='add'>+/*</div><div class='add'>+ * Allocate the dma buffer for storing ROCEE table entries</div><div class='add'>+ *</div><div class='add'>+ * @size: required size</div><div class='add'>+ * @page_shift: the unit size in a continuous dma address range</div><div class='add'>+ * @flags: HNS_ROCE_BUF_ flags to control the allocation flow.</div><div class='add'>+ */</div><div class='add'>+struct hns_roce_buf *hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size,</div><div class='add'>+					u32 page_shift, u32 flags)</div><div class='ctx'> {</div><div class='del'>-	int ret = 0;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	if (likely(cnt == 1 &amp;&amp; align == 1))</div><div class='del'>-		return hns_roce_bitmap_alloc(bitmap, obj);</div><div class='del'>-</div><div class='del'>-	spin_lock(&amp;bitmap-&gt;lock);</div><div class='add'>+	u32 trunk_size, page_size, alloced_size;</div><div class='add'>+	struct hns_roce_buf_list *trunks;</div><div class='add'>+	struct hns_roce_buf *buf;</div><div class='add'>+	gfp_t gfp_flags;</div><div class='add'>+	u32 ntrunk, i;</div><div class='add'>+</div><div class='add'>+	/* The minimum shift of the page accessed by hw is HNS_HW_PAGE_SHIFT */</div><div class='add'>+	if (WARN_ON(page_shift &lt; HNS_HW_PAGE_SHIFT))</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	gfp_flags = (flags &amp; HNS_ROCE_BUF_NOSLEEP) ? GFP_ATOMIC : GFP_KERNEL;</div><div class='add'>+	buf = kzalloc(sizeof(*buf), gfp_flags);</div><div class='add'>+	if (!buf)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	buf-&gt;page_shift = page_shift;</div><div class='add'>+	page_size = 1 &lt;&lt; buf-&gt;page_shift;</div><div class='add'>+</div><div class='add'>+	/* Calc the trunk size and num by required size and page_shift */</div><div class='add'>+	if (flags &amp; HNS_ROCE_BUF_DIRECT) {</div><div class='add'>+		buf-&gt;trunk_shift = order_base_2(ALIGN(size, PAGE_SIZE));</div><div class='add'>+		ntrunk = 1;</div><div class='add'>+	} else {</div><div class='add'>+		buf-&gt;trunk_shift = order_base_2(ALIGN(page_size, PAGE_SIZE));</div><div class='add'>+		ntrunk = DIV_ROUND_UP(size, 1 &lt;&lt; buf-&gt;trunk_shift);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	*obj = bitmap_find_next_zero_area(bitmap-&gt;table, bitmap-&gt;max,</div><div class='del'>-					  bitmap-&gt;last, cnt, align - 1);</div><div class='del'>-	if (*obj &gt;= bitmap-&gt;max) {</div><div class='del'>-		bitmap-&gt;top = (bitmap-&gt;top + bitmap-&gt;max + bitmap-&gt;reserved_top)</div><div class='del'>-			       &amp; bitmap-&gt;mask;</div><div class='del'>-		*obj = bitmap_find_next_zero_area(bitmap-&gt;table, bitmap-&gt;max, 0,</div><div class='del'>-						  cnt, align - 1);</div><div class='add'>+	trunks = kcalloc(ntrunk, sizeof(*trunks), gfp_flags);</div><div class='add'>+	if (!trunks) {</div><div class='add'>+		kfree(buf);</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (*obj &lt; bitmap-&gt;max) {</div><div class='del'>-		for (i = 0; i &lt; cnt; i++)</div><div class='del'>-			set_bit(*obj + i, bitmap-&gt;table);</div><div class='add'>+	trunk_size = 1 &lt;&lt; buf-&gt;trunk_shift;</div><div class='add'>+	alloced_size = 0;</div><div class='add'>+	for (i = 0; i &lt; ntrunk; i++) {</div><div class='add'>+		trunks[i].buf = dma_alloc_coherent(hr_dev-&gt;dev, trunk_size,</div><div class='add'>+						   &amp;trunks[i].map, gfp_flags);</div><div class='add'>+		if (!trunks[i].buf)</div><div class='add'>+			break;</div><div class='ctx'> </div><div class='del'>-		if (*obj == bitmap-&gt;last) {</div><div class='del'>-			bitmap-&gt;last = (*obj + cnt);</div><div class='del'>-			if (bitmap-&gt;last &gt;= bitmap-&gt;max)</div><div class='del'>-				bitmap-&gt;last = 0;</div><div class='del'>-		}</div><div class='del'>-		*obj |= bitmap-&gt;top;</div><div class='del'>-	} else {</div><div class='del'>-		ret = -1;</div><div class='add'>+		alloced_size += trunk_size;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	spin_unlock(&amp;bitmap-&gt;lock);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='add'>+	buf-&gt;ntrunks = i;</div><div class='ctx'> </div><div class='del'>-void hns_roce_bitmap_free_range(struct hns_roce_bitmap *bitmap,</div><div class='del'>-				unsigned long obj, int cnt,</div><div class='del'>-				int rr)</div><div class='del'>-{</div><div class='del'>-	int i;</div><div class='add'>+	/* In nofail mode, it's only failed when the alloced size is 0 */</div><div class='add'>+	if ((flags &amp; HNS_ROCE_BUF_NOFAIL) ? i == 0 : i != ntrunk) {</div><div class='add'>+		for (i = 0; i &lt; buf-&gt;ntrunks; i++)</div><div class='add'>+			dma_free_coherent(hr_dev-&gt;dev, trunk_size,</div><div class='add'>+					  trunks[i].buf, trunks[i].map);</div><div class='ctx'> </div><div class='del'>-	obj &amp;= bitmap-&gt;max + bitmap-&gt;reserved_top - 1;</div><div class='add'>+		kfree(trunks);</div><div class='add'>+		kfree(buf);</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	spin_lock(&amp;bitmap-&gt;lock);</div><div class='del'>-	for (i = 0; i &lt; cnt; i++)</div><div class='del'>-		clear_bit(obj + i, bitmap-&gt;table);</div><div class='add'>+	buf-&gt;npages = DIV_ROUND_UP(alloced_size, page_size);</div><div class='add'>+	buf-&gt;trunk_list = trunks;</div><div class='ctx'> </div><div class='del'>-	if (!rr)</div><div class='del'>-		bitmap-&gt;last = min(bitmap-&gt;last, obj);</div><div class='del'>-	bitmap-&gt;top = (bitmap-&gt;top + bitmap-&gt;max + bitmap-&gt;reserved_top)</div><div class='del'>-		       &amp; bitmap-&gt;mask;</div><div class='del'>-	spin_unlock(&amp;bitmap-&gt;lock);</div><div class='add'>+	return buf;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int hns_roce_bitmap_init(struct hns_roce_bitmap *bitmap, u32 num, u32 mask,</div><div class='del'>-			 u32 reserved_bot, u32 reserved_top)</div><div class='add'>+int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,</div><div class='add'>+			   int buf_cnt, struct hns_roce_buf *buf,</div><div class='add'>+			   unsigned int page_shift)</div><div class='ctx'> {</div><div class='del'>-	u32 i;</div><div class='add'>+	unsigned int offset, max_size;</div><div class='add'>+	int total = 0;</div><div class='add'>+	int i;</div><div class='ctx'> </div><div class='del'>-	if (num != roundup_pow_of_two(num))</div><div class='add'>+	if (page_shift &gt; buf-&gt;trunk_shift) {</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "failed to check kmem buf shift %u &gt; %u\n",</div><div class='add'>+			page_shift, buf-&gt;trunk_shift);</div><div class='ctx'> 		return -EINVAL;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	bitmap-&gt;last = 0;</div><div class='del'>-	bitmap-&gt;top = 0;</div><div class='del'>-	bitmap-&gt;max = num - reserved_top;</div><div class='del'>-	bitmap-&gt;mask = mask;</div><div class='del'>-	bitmap-&gt;reserved_top = reserved_top;</div><div class='del'>-	spin_lock_init(&amp;bitmap-&gt;lock);</div><div class='del'>-	bitmap-&gt;table = kcalloc(BITS_TO_LONGS(bitmap-&gt;max), sizeof(long),</div><div class='del'>-				GFP_KERNEL);</div><div class='del'>-	if (!bitmap-&gt;table)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; reserved_bot; ++i)</div><div class='del'>-		set_bit(i, bitmap-&gt;table);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void hns_roce_bitmap_cleanup(struct hns_roce_bitmap *bitmap)</div><div class='del'>-{</div><div class='del'>-	kfree(bitmap-&gt;table);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size,</div><div class='del'>-		       struct hns_roce_buf *buf)</div><div class='del'>-{</div><div class='del'>-	int i;</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-</div><div class='del'>-	if (buf-&gt;nbufs == 1) {</div><div class='del'>-		dma_free_coherent(dev, size, buf-&gt;direct.buf, buf-&gt;direct.map);</div><div class='del'>-	} else {</div><div class='del'>-		for (i = 0; i &lt; buf-&gt;nbufs; ++i)</div><div class='del'>-			if (buf-&gt;page_list[i].buf)</div><div class='del'>-				dma_free_coherent(dev, 1 &lt;&lt; buf-&gt;page_shift,</div><div class='del'>-						  buf-&gt;page_list[i].buf,</div><div class='del'>-						  buf-&gt;page_list[i].map);</div><div class='del'>-		kfree(buf-&gt;page_list);</div><div class='add'>+	offset = 0;</div><div class='add'>+	max_size = buf-&gt;ntrunks &lt;&lt; buf-&gt;trunk_shift;</div><div class='add'>+	for (i = 0; i &lt; buf_cnt &amp;&amp; offset &lt; max_size; i++) {</div><div class='add'>+		bufs[total++] = hns_roce_buf_dma_addr(buf, offset);</div><div class='add'>+		offset += (1 &lt;&lt; page_shift);</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	return total;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_buf_free);</div><div class='ctx'> </div><div class='del'>-int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct,</div><div class='del'>-		       struct hns_roce_buf *buf, u32 page_shift)</div><div class='add'>+int hns_roce_get_umem_bufs(dma_addr_t *bufs, int buf_cnt, struct ib_umem *umem,</div><div class='add'>+			   unsigned int page_shift)</div><div class='ctx'> {</div><div class='del'>-	int i = 0;</div><div class='del'>-	dma_addr_t t;</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	u32 page_size = 1 &lt;&lt; page_shift;</div><div class='del'>-	u32 order;</div><div class='del'>-</div><div class='del'>-	/* SQ/RQ buf lease than one page, SQ + RQ = 8K */</div><div class='del'>-	if (size &lt;= max_direct) {</div><div class='del'>-		buf-&gt;nbufs = 1;</div><div class='del'>-		/* Npages calculated by page_size */</div><div class='del'>-		order = get_order(size);</div><div class='del'>-		if (order &lt;= page_shift - PAGE_SHIFT)</div><div class='del'>-			order = 0;</div><div class='del'>-		else</div><div class='del'>-			order -= page_shift - PAGE_SHIFT;</div><div class='del'>-		buf-&gt;npages = 1 &lt;&lt; order;</div><div class='del'>-		buf-&gt;page_shift = page_shift;</div><div class='del'>-		/* MTT PA must be recorded in 4k alignment, t is 4k aligned */</div><div class='del'>-		buf-&gt;direct.buf = dma_alloc_coherent(dev, size, &amp;t,</div><div class='del'>-						     GFP_KERNEL);</div><div class='del'>-		if (!buf-&gt;direct.buf)</div><div class='del'>-			return -ENOMEM;</div><div class='del'>-</div><div class='del'>-		buf-&gt;direct.map = t;</div><div class='del'>-</div><div class='del'>-		while (t &amp; ((1 &lt;&lt; buf-&gt;page_shift) - 1)) {</div><div class='del'>-			--buf-&gt;page_shift;</div><div class='del'>-			buf-&gt;npages *= 2;</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		buf-&gt;nbufs = (size + page_size - 1) / page_size;</div><div class='del'>-		buf-&gt;npages = buf-&gt;nbufs;</div><div class='del'>-		buf-&gt;page_shift = page_shift;</div><div class='del'>-		buf-&gt;page_list = kcalloc(buf-&gt;nbufs, sizeof(*buf-&gt;page_list),</div><div class='del'>-					 GFP_KERNEL);</div><div class='del'>-</div><div class='del'>-		if (!buf-&gt;page_list)</div><div class='del'>-			return -ENOMEM;</div><div class='del'>-</div><div class='del'>-		for (i = 0; i &lt; buf-&gt;nbufs; ++i) {</div><div class='del'>-			buf-&gt;page_list[i].buf = dma_alloc_coherent(dev,</div><div class='del'>-								   page_size,</div><div class='del'>-								   &amp;t,</div><div class='del'>-								   GFP_KERNEL);</div><div class='del'>-</div><div class='del'>-			if (!buf-&gt;page_list[i].buf)</div><div class='del'>-				goto err_free;</div><div class='del'>-</div><div class='del'>-			buf-&gt;page_list[i].map = t;</div><div class='del'>-		}</div><div class='add'>+	struct ib_block_iter biter;</div><div class='add'>+	int total = 0;</div><div class='add'>+</div><div class='add'>+	/* convert system page cnt to hw page cnt */</div><div class='add'>+	rdma_umem_for_each_dma_block(umem, &amp;biter, 1 &lt;&lt; page_shift) {</div><div class='add'>+		bufs[total++] = rdma_block_iter_dma_address(&amp;biter);</div><div class='add'>+		if (total &gt;= buf_cnt)</div><div class='add'>+			goto done;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err_free:</div><div class='del'>-	hns_roce_buf_free(hr_dev, size, buf);</div><div class='del'>-	return -ENOMEM;</div><div class='add'>+done:</div><div class='add'>+	return total;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_SRQ)</div><div class='del'>-		hns_roce_cleanup_srq_table(hr_dev);</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_XRC)</div><div class='add'>+		ida_destroy(&amp;hr_dev-&gt;xrcd_ida.ida);</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_SRQ) {</div><div class='add'>+		ida_destroy(&amp;hr_dev-&gt;srq_table.srq_ida.ida);</div><div class='add'>+		xa_destroy(&amp;hr_dev-&gt;srq_table.xa);</div><div class='add'>+	}</div><div class='ctx'> 	hns_roce_cleanup_qp_table(hr_dev);</div><div class='ctx'> 	hns_roce_cleanup_cq_table(hr_dev);</div><div class='del'>-	hns_roce_cleanup_mr_table(hr_dev);</div><div class='del'>-	hns_roce_cleanup_pd_table(hr_dev);</div><div class='del'>-	hns_roce_cleanup_uar_table(hr_dev);</div><div class='add'>+	ida_destroy(&amp;hr_dev-&gt;mr_table.mtpt_ida.ida);</div><div class='add'>+	ida_destroy(&amp;hr_dev-&gt;pd_ida.ida);</div><div class='add'>+	ida_destroy(&amp;hr_dev-&gt;uar_ida.ida);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c<br/>new file mode 100644<br/>index 000000000000..cc85f3ce1f3e<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_bond.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_bond.c</a></div><div class='hunk'>@@ -0,0 +1,1012 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0+</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2025 Hisilicon Limited.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;net/lag.h&gt;</div><div class='add'>+#include &lt;net/bonding.h&gt;</div><div class='add'>+#include "hns_roce_device.h"</div><div class='add'>+#include "hns_roce_hw_v2.h"</div><div class='add'>+#include "hns_roce_bond.h"</div><div class='add'>+</div><div class='add'>+static DEFINE_XARRAY(roce_bond_xa);</div><div class='add'>+</div><div class='add'>+static struct hns_roce_dev *hns_roce_get_hrdev_by_netdev(struct net_device *net_dev)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev =</div><div class='add'>+		ib_device_get_by_netdev(net_dev, RDMA_DRIVER_HNS);</div><div class='add'>+</div><div class='add'>+	if (!ibdev)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	return container_of(ibdev, struct hns_roce_dev, ib_dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct net_device *get_upper_dev_from_ndev(struct net_device *net_dev)</div><div class='add'>+{</div><div class='add'>+	struct net_device *upper_dev;</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	upper_dev = netdev_master_upper_dev_get_rcu(net_dev);</div><div class='add'>+	dev_hold(upper_dev);</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+</div><div class='add'>+	return upper_dev;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_netdev_bond_slave_id(struct net_device *net_dev,</div><div class='add'>+				    struct hns_roce_bond_group *bond_grp)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ROCE_BOND_FUNC_MAX; i++)</div><div class='add'>+		if (net_dev == bond_grp-&gt;bond_func_info[i].net_dev)</div><div class='add'>+			return i;</div><div class='add'>+</div><div class='add'>+	return -ENOENT;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct hns_roce_bond_group *hns_roce_get_bond_grp(struct net_device *net_dev,</div><div class='add'>+						  u8 bus_num)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_die_info *die_info = xa_load(&amp;roce_bond_xa, bus_num);</div><div class='add'>+	struct hns_roce_bond_group *bond_grp;</div><div class='add'>+	struct net_device *upper_dev = NULL;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (!die_info)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ROCE_BOND_NUM_MAX; i++) {</div><div class='add'>+		bond_grp = die_info-&gt;bgrps[i];</div><div class='add'>+		if (!bond_grp)</div><div class='add'>+			continue;</div><div class='add'>+		if (get_netdev_bond_slave_id(net_dev, bond_grp) &gt;= 0)</div><div class='add'>+			return bond_grp;</div><div class='add'>+		if (bond_grp-&gt;upper_dev) {</div><div class='add'>+			upper_dev = get_upper_dev_from_ndev(net_dev);</div><div class='add'>+			if (bond_grp-&gt;upper_dev == upper_dev) {</div><div class='add'>+				dev_put(upper_dev);</div><div class='add'>+				return bond_grp;</div><div class='add'>+			}</div><div class='add'>+			dev_put(upper_dev);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_set_bond_netdev(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+				    struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct net_device *active_dev;</div><div class='add'>+	struct net_device *old_dev;</div><div class='add'>+	int i, ret = 0;</div><div class='add'>+</div><div class='add'>+	if (bond_grp-&gt;tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {</div><div class='add'>+		rcu_read_lock();</div><div class='add'>+		active_dev =</div><div class='add'>+			bond_option_active_slave_get_rcu(netdev_priv(bond_grp-&gt;upper_dev));</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+	} else {</div><div class='add'>+		for (i = 0; i &lt; ROCE_BOND_FUNC_MAX; i++) {</div><div class='add'>+			active_dev = bond_grp-&gt;bond_func_info[i].net_dev;</div><div class='add'>+			if (active_dev &amp;&amp;</div><div class='add'>+			    ib_get_curr_port_state(active_dev) == IB_PORT_ACTIVE)</div><div class='add'>+				break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!active_dev || i == ROCE_BOND_FUNC_MAX)</div><div class='add'>+		active_dev = get_hr_netdev(hr_dev, 0);</div><div class='add'>+</div><div class='add'>+	old_dev = ib_device_get_netdev(&amp;hr_dev-&gt;ib_dev, 1);</div><div class='add'>+	if (old_dev == active_dev)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	ret = ib_device_set_netdev(&amp;hr_dev-&gt;ib_dev, active_dev, 1);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "failed to set netdev for bond.\n");</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (bond_grp-&gt;tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {</div><div class='add'>+		if (old_dev)</div><div class='add'>+			roce_del_all_netdev_gids(&amp;hr_dev-&gt;ib_dev, 1, old_dev);</div><div class='add'>+		rdma_roce_rescan_port(&amp;hr_dev-&gt;ib_dev, 1);</div><div class='add'>+	}</div><div class='add'>+out:</div><div class='add'>+	dev_put(old_dev);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct net_device *net_dev = get_hr_netdev(hr_dev, 0);</div><div class='add'>+	struct hns_roce_bond_group *bond_grp;</div><div class='add'>+	u8 bus_num = get_hr_bus_num(hr_dev);</div><div class='add'>+</div><div class='add'>+	bond_grp = hns_roce_get_bond_grp(net_dev, bus_num);</div><div class='add'>+	if (bond_grp &amp;&amp; bond_grp-&gt;bond_state != HNS_ROCE_BOND_NOT_BONDED &amp;&amp;</div><div class='add'>+	    bond_grp-&gt;bond_state != HNS_ROCE_BOND_NOT_ATTACHED)</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_bond_get_active_slave(struct hns_roce_bond_group *bond_grp)</div><div class='add'>+{</div><div class='add'>+	struct net_device *net_dev;</div><div class='add'>+	u32 active_slave_map = 0;</div><div class='add'>+	u8 active_slave_num = 0;</div><div class='add'>+	bool active;</div><div class='add'>+	u8 i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ROCE_BOND_FUNC_MAX; i++) {</div><div class='add'>+		net_dev = bond_grp-&gt;bond_func_info[i].net_dev;</div><div class='add'>+		if (!net_dev || !(bond_grp-&gt;slave_map &amp; (1U &lt;&lt; i)))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		active = (bond_grp-&gt;tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) ?</div><div class='add'>+			 net_lag_port_dev_txable(net_dev) :</div><div class='add'>+			 (ib_get_curr_port_state(net_dev) == IB_PORT_ACTIVE);</div><div class='add'>+		if (active) {</div><div class='add'>+			active_slave_num++;</div><div class='add'>+			active_slave_map |= (1U &lt;&lt; i);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	bond_grp-&gt;active_slave_num = active_slave_num;</div><div class='add'>+	bond_grp-&gt;active_slave_map = active_slave_map;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_recover_bond(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+				 struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	bond_grp-&gt;main_hr_dev = hr_dev;</div><div class='add'>+	hns_roce_bond_get_active_slave(bond_grp);</div><div class='add'>+</div><div class='add'>+	return hns_roce_cmd_bond(bond_grp, HNS_ROCE_SET_BOND);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_slave_uninit(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+				  u8 func_idx)</div><div class='add'>+{</div><div class='add'>+	struct hnae3_handle *handle;</div><div class='add'>+</div><div class='add'>+	handle = bond_grp-&gt;bond_func_info[func_idx].handle;</div><div class='add'>+	if (handle-&gt;priv)</div><div class='add'>+		hns_roce_bond_uninit_client(bond_grp, func_idx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct hns_roce_dev</div><div class='add'>+	*hns_roce_slave_init(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+			     u8 func_idx, bool need_switch);</div><div class='add'>+</div><div class='add'>+static int switch_main_dev(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+			   u8 main_func_idx)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev;</div><div class='add'>+	struct net_device *net_dev;</div><div class='add'>+	u8 i;</div><div class='add'>+</div><div class='add'>+	bond_grp-&gt;main_hr_dev = NULL;</div><div class='add'>+	hns_roce_bond_uninit_client(bond_grp, main_func_idx);</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ROCE_BOND_FUNC_MAX; i++) {</div><div class='add'>+		net_dev = bond_grp-&gt;bond_func_info[i].net_dev;</div><div class='add'>+		if ((bond_grp-&gt;slave_map &amp; (1U &lt;&lt; i)) &amp;&amp; net_dev) {</div><div class='add'>+			/* In case this slave is still being registered as</div><div class='add'>+			 * a non-bonded PF, uninit it first and then re-init</div><div class='add'>+			 * it as the main device.</div><div class='add'>+			 */</div><div class='add'>+			hns_roce_slave_uninit(bond_grp, i);</div><div class='add'>+			hr_dev = hns_roce_slave_init(bond_grp, i, false);</div><div class='add'>+			if (hr_dev) {</div><div class='add'>+				bond_grp-&gt;main_hr_dev = hr_dev;</div><div class='add'>+				break;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!bond_grp-&gt;main_hr_dev)</div><div class='add'>+		return -ENODEV;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct hns_roce_dev</div><div class='add'>+	*hns_roce_slave_init(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+			     u8 func_idx, bool need_switch)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = NULL;</div><div class='add'>+	struct hnae3_handle *handle;</div><div class='add'>+	u8 main_func_idx;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (need_switch) {</div><div class='add'>+		main_func_idx = PCI_FUNC(bond_grp-&gt;main_hr_dev-&gt;pci_dev-&gt;devfn);</div><div class='add'>+		if (func_idx == main_func_idx) {</div><div class='add'>+			ret = switch_main_dev(bond_grp, main_func_idx);</div><div class='add'>+			if (ret == -ENODEV)</div><div class='add'>+				return NULL;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	handle = bond_grp-&gt;bond_func_info[func_idx].handle;</div><div class='add'>+	if (handle) {</div><div class='add'>+		if (handle-&gt;priv)</div><div class='add'>+			return handle-&gt;priv;</div><div class='add'>+		/* Prevent this device from being initialized as a bond device */</div><div class='add'>+		if (need_switch)</div><div class='add'>+			bond_grp-&gt;bond_func_info[func_idx].net_dev = NULL;</div><div class='add'>+		hr_dev = hns_roce_bond_init_client(bond_grp, func_idx);</div><div class='add'>+		if (!hr_dev)</div><div class='add'>+			BOND_ERR_LOG("failed to init slave %u.\n", func_idx);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return hr_dev;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct hns_roce_die_info *alloc_die_info(int bus_num)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_die_info *die_info;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	die_info = kzalloc(sizeof(*die_info), GFP_KERNEL);</div><div class='add'>+	if (!die_info)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	ret = xa_err(xa_store(&amp;roce_bond_xa, bus_num, die_info, GFP_KERNEL));</div><div class='add'>+	if (ret) {</div><div class='add'>+		kfree(die_info);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_init(&amp;die_info-&gt;die_mutex);</div><div class='add'>+</div><div class='add'>+	return die_info;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void dealloc_die_info(struct hns_roce_die_info *die_info, u8 bus_num)</div><div class='add'>+{</div><div class='add'>+	mutex_destroy(&amp;die_info-&gt;die_mutex);</div><div class='add'>+	xa_erase(&amp;roce_bond_xa, bus_num);</div><div class='add'>+	kfree(die_info);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int alloc_bond_id(struct hns_roce_bond_group *bond_grp)</div><div class='add'>+{</div><div class='add'>+	u8 bus_num = bond_grp-&gt;bus_num;</div><div class='add'>+	struct hns_roce_die_info *die_info = xa_load(&amp;roce_bond_xa, bus_num);</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (!die_info) {</div><div class='add'>+		die_info = alloc_die_info(bus_num);</div><div class='add'>+		if (!die_info)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ROCE_BOND_NUM_MAX; i++) {</div><div class='add'>+		if (die_info-&gt;bond_id_mask &amp; BOND_ID(i))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		die_info-&gt;bond_id_mask |= BOND_ID(i);</div><div class='add'>+		die_info-&gt;bgrps[i] = bond_grp;</div><div class='add'>+		bond_grp-&gt;bond_id = i;</div><div class='add'>+</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return -ENOSPC;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int remove_bond_id(int bus_num, u8 bond_id)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_die_info *die_info = xa_load(&amp;roce_bond_xa, bus_num);</div><div class='add'>+</div><div class='add'>+	if (bond_id &gt;= ROCE_BOND_NUM_MAX)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!die_info)</div><div class='add'>+		return -ENODEV;</div><div class='add'>+</div><div class='add'>+	die_info-&gt;bond_id_mask &amp;= ~BOND_ID(bond_id);</div><div class='add'>+	die_info-&gt;bgrps[bond_id] = NULL;</div><div class='add'>+	if (!die_info-&gt;bond_id_mask)</div><div class='add'>+		dealloc_die_info(die_info, bus_num);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_set_bond(struct hns_roce_bond_group *bond_grp)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev;</div><div class='add'>+	int ret;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = ROCE_BOND_FUNC_MAX - 1; i &gt;= 0; i--) {</div><div class='add'>+		if (bond_grp-&gt;slave_map &amp; (1 &lt;&lt; i))</div><div class='add'>+			hns_roce_slave_uninit(bond_grp, i);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+	bond_grp-&gt;bond_state = HNS_ROCE_BOND_IS_BONDED;</div><div class='add'>+	mutex_unlock(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+	bond_grp-&gt;main_hr_dev = NULL;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ROCE_BOND_FUNC_MAX; i++) {</div><div class='add'>+		if (bond_grp-&gt;slave_map &amp; (1 &lt;&lt; i)) {</div><div class='add'>+			hr_dev = hns_roce_slave_init(bond_grp, i, false);</div><div class='add'>+			if (hr_dev) {</div><div class='add'>+				bond_grp-&gt;main_hr_dev = hr_dev;</div><div class='add'>+				break;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!bond_grp-&gt;main_hr_dev) {</div><div class='add'>+		ret = -ENODEV;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hns_roce_bond_get_active_slave(bond_grp);</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_SET_BOND);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	if (ret) {</div><div class='add'>+		BOND_ERR_LOG("failed to set RoCE bond, ret = %d.\n", ret);</div><div class='add'>+		hns_roce_cleanup_bond(bond_grp);</div><div class='add'>+	} else {</div><div class='add'>+		ibdev_info(&amp;bond_grp-&gt;main_hr_dev-&gt;ib_dev,</div><div class='add'>+			   "RoCE set bond finished!\n");</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_clear_bond(struct hns_roce_bond_group *bond_grp)</div><div class='add'>+{</div><div class='add'>+	u8 main_func_idx = PCI_FUNC(bond_grp-&gt;main_hr_dev-&gt;pci_dev-&gt;devfn);</div><div class='add'>+	struct hns_roce_dev *hr_dev;</div><div class='add'>+	u8 i;</div><div class='add'>+</div><div class='add'>+	if (bond_grp-&gt;bond_state == HNS_ROCE_BOND_NOT_BONDED)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	bond_grp-&gt;bond_state = HNS_ROCE_BOND_NOT_BONDED;</div><div class='add'>+	bond_grp-&gt;main_hr_dev = NULL;</div><div class='add'>+</div><div class='add'>+	hns_roce_slave_uninit(bond_grp, main_func_idx);</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ROCE_BOND_FUNC_MAX; i++) {</div><div class='add'>+		hr_dev = hns_roce_slave_init(bond_grp, i, false);</div><div class='add'>+		if (hr_dev)</div><div class='add'>+			bond_grp-&gt;main_hr_dev = hr_dev;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	hns_roce_cleanup_bond(bond_grp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_slave_changestate(struct hns_roce_bond_group *bond_grp)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	hns_roce_bond_get_active_slave(bond_grp);</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_CHANGE_BOND);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+	if (bond_grp-&gt;bond_state == HNS_ROCE_BOND_SLAVE_CHANGESTATE)</div><div class='add'>+		bond_grp-&gt;bond_state = HNS_ROCE_BOND_IS_BONDED;</div><div class='add'>+	mutex_unlock(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_err(&amp;bond_grp-&gt;main_hr_dev-&gt;ib_dev,</div><div class='add'>+			  "failed to change RoCE bond slave state, ret = %d.\n",</div><div class='add'>+			  ret);</div><div class='add'>+	else</div><div class='add'>+		ibdev_info(&amp;bond_grp-&gt;main_hr_dev-&gt;ib_dev,</div><div class='add'>+			   "RoCE slave changestate finished!\n");</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_slave_change_num(struct hns_roce_bond_group *bond_grp)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+	u8 i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ROCE_BOND_FUNC_MAX; i++) {</div><div class='add'>+		if (bond_grp-&gt;slave_map &amp; (1U &lt;&lt; i)) {</div><div class='add'>+			if (i == PCI_FUNC(bond_grp-&gt;main_hr_dev-&gt;pci_dev-&gt;devfn))</div><div class='add'>+				continue;</div><div class='add'>+			hns_roce_slave_uninit(bond_grp, i);</div><div class='add'>+		} else {</div><div class='add'>+			hns_roce_slave_init(bond_grp, i, true);</div><div class='add'>+			if (!bond_grp-&gt;main_hr_dev) {</div><div class='add'>+				ret = -ENODEV;</div><div class='add'>+				goto out;</div><div class='add'>+			}</div><div class='add'>+			bond_grp-&gt;bond_func_info[i].net_dev = NULL;</div><div class='add'>+			bond_grp-&gt;bond_func_info[i].handle = NULL;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hns_roce_bond_get_active_slave(bond_grp);</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_CHANGE_BOND);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	if (ret) {</div><div class='add'>+		BOND_ERR_LOG("failed to change RoCE bond slave num, ret = %d.\n", ret);</div><div class='add'>+		hns_roce_cleanup_bond(bond_grp);</div><div class='add'>+	} else {</div><div class='add'>+		mutex_lock(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+		if (bond_grp-&gt;bond_state == HNS_ROCE_BOND_SLAVE_CHANGE_NUM)</div><div class='add'>+			bond_grp-&gt;bond_state = HNS_ROCE_BOND_IS_BONDED;</div><div class='add'>+		mutex_unlock(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+		ibdev_info(&amp;bond_grp-&gt;main_hr_dev-&gt;ib_dev,</div><div class='add'>+			   "RoCE slave change num finished!\n");</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_bond_info_update_nolock(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+					     struct net_device *upper_dev)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_v2_priv *priv;</div><div class='add'>+	struct hns_roce_dev *hr_dev;</div><div class='add'>+	struct net_device *net_dev;</div><div class='add'>+	int func_idx;</div><div class='add'>+</div><div class='add'>+	bond_grp-&gt;slave_map = 0;</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	for_each_netdev_in_bond_rcu(upper_dev, net_dev) {</div><div class='add'>+		func_idx = get_netdev_bond_slave_id(net_dev, bond_grp);</div><div class='add'>+		if (func_idx &lt; 0) {</div><div class='add'>+			hr_dev = hns_roce_get_hrdev_by_netdev(net_dev);</div><div class='add'>+			if (!hr_dev)</div><div class='add'>+				continue;</div><div class='add'>+			func_idx = PCI_FUNC(hr_dev-&gt;pci_dev-&gt;devfn);</div><div class='add'>+			if (!bond_grp-&gt;bond_func_info[func_idx].net_dev) {</div><div class='add'>+				priv = hr_dev-&gt;priv;</div><div class='add'>+				bond_grp-&gt;bond_func_info[func_idx].net_dev =</div><div class='add'>+					net_dev;</div><div class='add'>+				bond_grp-&gt;bond_func_info[func_idx].handle =</div><div class='add'>+					priv-&gt;handle;</div><div class='add'>+			}</div><div class='add'>+			ib_device_put(&amp;hr_dev-&gt;ib_dev);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		bond_grp-&gt;slave_map |= (1 &lt;&lt; func_idx);</div><div class='add'>+	}</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool is_dev_bond_supported(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+				  struct net_device *net_dev)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = hns_roce_get_hrdev_by_netdev(net_dev);</div><div class='add'>+	bool ret = true;</div><div class='add'>+</div><div class='add'>+	if (!hr_dev) {</div><div class='add'>+		if (bond_grp &amp;&amp;</div><div class='add'>+		    get_netdev_bond_slave_id(net_dev, bond_grp) &gt;= 0)</div><div class='add'>+			return true;</div><div class='add'>+		else</div><div class='add'>+			return false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!(hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_BOND)) {</div><div class='add'>+		ret = false;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;is_vf || pci_num_vf(hr_dev-&gt;pci_dev) &gt; 0) {</div><div class='add'>+		ret = false;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (bond_grp-&gt;bus_num != get_hr_bus_num(hr_dev))</div><div class='add'>+		ret = false;</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	ib_device_put(&amp;hr_dev-&gt;ib_dev);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool check_slave_support(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+				struct net_device *upper_dev)</div><div class='add'>+{</div><div class='add'>+	struct net_device *net_dev;</div><div class='add'>+	u8 slave_num = 0;</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	for_each_netdev_in_bond_rcu(upper_dev, net_dev) {</div><div class='add'>+		if (is_dev_bond_supported(bond_grp, net_dev)) {</div><div class='add'>+			slave_num++;</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+</div><div class='add'>+	return (slave_num &gt; 1 &amp;&amp; slave_num &lt;= ROCE_BOND_FUNC_MAX);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_bond_work(struct work_struct *work)</div><div class='add'>+{</div><div class='add'>+	struct delayed_work *delayed_work = to_delayed_work(work);</div><div class='add'>+	struct hns_roce_bond_group *bond_grp =</div><div class='add'>+		container_of(delayed_work, struct hns_roce_bond_group,</div><div class='add'>+			     bond_work);</div><div class='add'>+	enum hns_roce_bond_state bond_state;</div><div class='add'>+	bool bond_ready;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+	bond_ready = check_slave_support(bond_grp, bond_grp-&gt;upper_dev);</div><div class='add'>+	hns_roce_bond_info_update_nolock(bond_grp, bond_grp-&gt;upper_dev);</div><div class='add'>+	bond_state = bond_grp-&gt;bond_state;</div><div class='add'>+	bond_grp-&gt;bond_ready = bond_ready;</div><div class='add'>+	mutex_unlock(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+</div><div class='add'>+	ibdev_info(&amp;bond_grp-&gt;main_hr_dev-&gt;ib_dev,</div><div class='add'>+		   "bond work: bond_ready - %d, bond_state - %d.\n",</div><div class='add'>+		   bond_ready, bond_state);</div><div class='add'>+</div><div class='add'>+	if (!bond_ready) {</div><div class='add'>+		hns_roce_clear_bond(bond_grp);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (bond_state) {</div><div class='add'>+	case HNS_ROCE_BOND_NOT_BONDED:</div><div class='add'>+		hns_roce_set_bond(bond_grp);</div><div class='add'>+		/* In set_bond flow, we don't need to set bond netdev here as</div><div class='add'>+		 * it has been done when bond_grp-&gt;main_hr_dev is registered.</div><div class='add'>+		 */</div><div class='add'>+		return;</div><div class='add'>+	case HNS_ROCE_BOND_SLAVE_CHANGESTATE:</div><div class='add'>+		hns_roce_slave_changestate(bond_grp);</div><div class='add'>+		break;</div><div class='add'>+	case HNS_ROCE_BOND_SLAVE_CHANGE_NUM:</div><div class='add'>+		hns_roce_slave_change_num(bond_grp);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+	hns_roce_set_bond_netdev(bond_grp, bond_grp-&gt;main_hr_dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_attach_bond_grp(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+				     struct hns_roce_dev *hr_dev,</div><div class='add'>+				     struct net_device *upper_dev)</div><div class='add'>+{</div><div class='add'>+	bond_grp-&gt;upper_dev = upper_dev;</div><div class='add'>+	bond_grp-&gt;main_hr_dev = hr_dev;</div><div class='add'>+	bond_grp-&gt;bond_state = HNS_ROCE_BOND_NOT_BONDED;</div><div class='add'>+	bond_grp-&gt;bond_ready = false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_detach_bond_grp(struct hns_roce_bond_group *bond_grp)</div><div class='add'>+{</div><div class='add'>+	mutex_lock(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+</div><div class='add'>+	cancel_delayed_work(&amp;bond_grp-&gt;bond_work);</div><div class='add'>+	bond_grp-&gt;upper_dev = NULL;</div><div class='add'>+	bond_grp-&gt;main_hr_dev = NULL;</div><div class='add'>+	bond_grp-&gt;bond_ready = false;</div><div class='add'>+	bond_grp-&gt;bond_state = HNS_ROCE_BOND_NOT_ATTACHED;</div><div class='add'>+	bond_grp-&gt;slave_map = 0;</div><div class='add'>+	memset(bond_grp-&gt;bond_func_info, 0, sizeof(bond_grp-&gt;bond_func_info));</div><div class='add'>+</div><div class='add'>+	mutex_unlock(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hns_roce_cleanup_bond(struct hns_roce_bond_group *bond_grp)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = bond_grp-&gt;main_hr_dev ?</div><div class='add'>+	      hns_roce_cmd_bond(bond_grp, HNS_ROCE_CLEAR_BOND) : -EIO;</div><div class='add'>+	if (ret)</div><div class='add'>+		BOND_ERR_LOG("failed to clear RoCE bond, ret = %d.\n", ret);</div><div class='add'>+	else</div><div class='add'>+		ibdev_info(&amp;bond_grp-&gt;main_hr_dev-&gt;ib_dev,</div><div class='add'>+			   "RoCE clear bond finished!\n");</div><div class='add'>+</div><div class='add'>+	hns_roce_detach_bond_grp(bond_grp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool lowerstate_event_filter(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+				    struct net_device *net_dev)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_bond_group *bond_grp_tmp;</div><div class='add'>+</div><div class='add'>+	bond_grp_tmp = hns_roce_get_bond_grp(net_dev, bond_grp-&gt;bus_num);</div><div class='add'>+	return bond_grp_tmp == bond_grp;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void lowerstate_event_setting(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+				     struct netdev_notifier_changelowerstate_info *info)</div><div class='add'>+{</div><div class='add'>+	mutex_lock(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+</div><div class='add'>+	if (bond_grp-&gt;bond_ready &amp;&amp;</div><div class='add'>+	    bond_grp-&gt;bond_state == HNS_ROCE_BOND_IS_BONDED)</div><div class='add'>+		bond_grp-&gt;bond_state = HNS_ROCE_BOND_SLAVE_CHANGESTATE;</div><div class='add'>+</div><div class='add'>+	mutex_unlock(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool hns_roce_bond_lowerstate_event(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+					   struct netdev_notifier_changelowerstate_info *info)</div><div class='add'>+{</div><div class='add'>+	struct net_device *net_dev =</div><div class='add'>+		netdev_notifier_info_to_dev((struct netdev_notifier_info *)info);</div><div class='add'>+</div><div class='add'>+	if (!netif_is_lag_port(net_dev))</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	if (!lowerstate_event_filter(bond_grp, net_dev))</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	lowerstate_event_setting(bond_grp, info);</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool is_bond_setting_supported(struct netdev_lag_upper_info *bond_info)</div><div class='add'>+{</div><div class='add'>+	if (!bond_info)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	if (bond_info-&gt;tx_type != NETDEV_LAG_TX_TYPE_ACTIVEBACKUP &amp;&amp;</div><div class='add'>+	    bond_info-&gt;tx_type != NETDEV_LAG_TX_TYPE_HASH)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	if (bond_info-&gt;tx_type == NETDEV_LAG_TX_TYPE_HASH &amp;&amp;</div><div class='add'>+	    bond_info-&gt;hash_type &gt; NETDEV_LAG_HASH_L23)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void upper_event_setting(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+				struct netdev_notifier_changeupper_info *info)</div><div class='add'>+{</div><div class='add'>+	struct netdev_lag_upper_info *bond_upper_info = NULL;</div><div class='add'>+	bool slave_inc = info-&gt;linking;</div><div class='add'>+</div><div class='add'>+	if (slave_inc)</div><div class='add'>+		bond_upper_info = info-&gt;upper_info;</div><div class='add'>+</div><div class='add'>+	if (bond_upper_info) {</div><div class='add'>+		bond_grp-&gt;tx_type = bond_upper_info-&gt;tx_type;</div><div class='add'>+		bond_grp-&gt;hash_type = bond_upper_info-&gt;hash_type;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool check_unlinking_bond_support(struct hns_roce_bond_group *bond_grp)</div><div class='add'>+{</div><div class='add'>+	struct net_device *net_dev;</div><div class='add'>+	u8 slave_num = 0;</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	for_each_netdev_in_bond_rcu(bond_grp-&gt;upper_dev, net_dev) {</div><div class='add'>+		if (get_netdev_bond_slave_id(net_dev, bond_grp) &gt;= 0)</div><div class='add'>+			slave_num++;</div><div class='add'>+	}</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+</div><div class='add'>+	return (slave_num &gt; 1);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool check_linking_bond_support(struct netdev_lag_upper_info *bond_info,</div><div class='add'>+				       struct hns_roce_bond_group *bond_grp,</div><div class='add'>+				       struct net_device *upper_dev)</div><div class='add'>+{</div><div class='add'>+	if (!is_bond_setting_supported(bond_info))</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	return check_slave_support(bond_grp, upper_dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static enum bond_support_type</div><div class='add'>+	check_bond_support(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+			   struct net_device *upper_dev,</div><div class='add'>+			   struct netdev_notifier_changeupper_info *info)</div><div class='add'>+{</div><div class='add'>+	bool bond_grp_exist = false;</div><div class='add'>+	bool support;</div><div class='add'>+</div><div class='add'>+	if (upper_dev == bond_grp-&gt;upper_dev)</div><div class='add'>+		bond_grp_exist = true;</div><div class='add'>+</div><div class='add'>+	if (!info-&gt;linking &amp;&amp; !bond_grp_exist)</div><div class='add'>+		return BOND_NOT_SUPPORT;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;linking)</div><div class='add'>+		support = check_linking_bond_support(info-&gt;upper_info, bond_grp,</div><div class='add'>+						     upper_dev);</div><div class='add'>+	else</div><div class='add'>+		support = check_unlinking_bond_support(bond_grp);</div><div class='add'>+</div><div class='add'>+	if (support)</div><div class='add'>+		return BOND_SUPPORT;</div><div class='add'>+</div><div class='add'>+	return bond_grp_exist ? BOND_EXISTING_NOT_SUPPORT : BOND_NOT_SUPPORT;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool upper_event_filter(struct netdev_notifier_changeupper_info *info,</div><div class='add'>+			       struct hns_roce_bond_group *bond_grp,</div><div class='add'>+			       struct net_device *net_dev)</div><div class='add'>+{</div><div class='add'>+	struct net_device *upper_dev = info-&gt;upper_dev;</div><div class='add'>+	struct hns_roce_bond_group *bond_grp_tmp;</div><div class='add'>+	struct hns_roce_dev *hr_dev;</div><div class='add'>+	bool ret = true;</div><div class='add'>+	u8 bus_num;</div><div class='add'>+</div><div class='add'>+	if (!info-&gt;linking ||</div><div class='add'>+	    bond_grp-&gt;bond_state != HNS_ROCE_BOND_NOT_ATTACHED)</div><div class='add'>+		return bond_grp-&gt;upper_dev == upper_dev;</div><div class='add'>+</div><div class='add'>+	hr_dev = hns_roce_get_hrdev_by_netdev(net_dev);</div><div class='add'>+	if (!hr_dev)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	bus_num = get_hr_bus_num(hr_dev);</div><div class='add'>+	if (bond_grp-&gt;bus_num != bus_num) {</div><div class='add'>+		ret = false;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	bond_grp_tmp = hns_roce_get_bond_grp(net_dev, bus_num);</div><div class='add'>+	if (bond_grp_tmp &amp;&amp; bond_grp_tmp != bond_grp)</div><div class='add'>+		ret = false;</div><div class='add'>+out:</div><div class='add'>+	ib_device_put(&amp;hr_dev-&gt;ib_dev);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool hns_roce_bond_upper_event(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+				      struct netdev_notifier_changeupper_info *info)</div><div class='add'>+{</div><div class='add'>+	struct net_device *net_dev =</div><div class='add'>+		netdev_notifier_info_to_dev((struct netdev_notifier_info *)info);</div><div class='add'>+	struct net_device *upper_dev = info-&gt;upper_dev;</div><div class='add'>+	enum bond_support_type support = BOND_SUPPORT;</div><div class='add'>+	struct hns_roce_dev *hr_dev;</div><div class='add'>+	int slave_id;</div><div class='add'>+</div><div class='add'>+	if (!upper_dev || !netif_is_lag_master(upper_dev))</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	if (!upper_event_filter(info, bond_grp, net_dev))</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+	support = check_bond_support(bond_grp, upper_dev, info);</div><div class='add'>+	if (support == BOND_NOT_SUPPORT) {</div><div class='add'>+		mutex_unlock(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (bond_grp-&gt;bond_state == HNS_ROCE_BOND_NOT_ATTACHED) {</div><div class='add'>+		hr_dev = hns_roce_get_hrdev_by_netdev(net_dev);</div><div class='add'>+		if (!hr_dev) {</div><div class='add'>+			mutex_unlock(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+			return false;</div><div class='add'>+		}</div><div class='add'>+		hns_roce_attach_bond_grp(bond_grp, hr_dev, upper_dev);</div><div class='add'>+		ib_device_put(&amp;hr_dev-&gt;ib_dev);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* In the case of netdev being unregistered, the roce</div><div class='add'>+	 * instance shouldn't be inited.</div><div class='add'>+	 */</div><div class='add'>+	if (net_dev-&gt;reg_state &gt;= NETREG_UNREGISTERING) {</div><div class='add'>+		slave_id = get_netdev_bond_slave_id(net_dev, bond_grp);</div><div class='add'>+		if (slave_id &gt;= 0) {</div><div class='add'>+			bond_grp-&gt;bond_func_info[slave_id].net_dev = NULL;</div><div class='add'>+			bond_grp-&gt;bond_func_info[slave_id].handle = NULL;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (support == BOND_SUPPORT) {</div><div class='add'>+		bond_grp-&gt;bond_ready = true;</div><div class='add'>+		if (bond_grp-&gt;bond_state != HNS_ROCE_BOND_NOT_BONDED)</div><div class='add'>+			bond_grp-&gt;bond_state = HNS_ROCE_BOND_SLAVE_CHANGE_NUM;</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+	if (support == BOND_SUPPORT)</div><div class='add'>+		upper_event_setting(bond_grp, info);</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_bond_event(struct notifier_block *self,</div><div class='add'>+			       unsigned long event, void *ptr)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_bond_group *bond_grp =</div><div class='add'>+		container_of(self, struct hns_roce_bond_group, bond_nb);</div><div class='add'>+	bool changed = false;</div><div class='add'>+</div><div class='add'>+	if (event == NETDEV_CHANGEUPPER)</div><div class='add'>+		changed = hns_roce_bond_upper_event(bond_grp, ptr);</div><div class='add'>+	if (event == NETDEV_CHANGELOWERSTATE)</div><div class='add'>+		changed = hns_roce_bond_lowerstate_event(bond_grp, ptr);</div><div class='add'>+</div><div class='add'>+	if (changed)</div><div class='add'>+		schedule_delayed_work(&amp;bond_grp-&gt;bond_work, HZ);</div><div class='add'>+</div><div class='add'>+	return NOTIFY_DONE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hns_roce_alloc_bond_grp(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_bond_group *bgrps[ROCE_BOND_NUM_MAX];</div><div class='add'>+	struct hns_roce_bond_group *bond_grp;</div><div class='add'>+	u8 bus_num = get_hr_bus_num(hr_dev);</div><div class='add'>+	int ret;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (xa_load(&amp;roce_bond_xa, bus_num))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ROCE_BOND_NUM_MAX; i++) {</div><div class='add'>+		bond_grp = kvzalloc(sizeof(*bond_grp), GFP_KERNEL);</div><div class='add'>+		if (!bond_grp) {</div><div class='add'>+			ret = -ENOMEM;</div><div class='add'>+			goto mem_err;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		mutex_init(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+		INIT_DELAYED_WORK(&amp;bond_grp-&gt;bond_work, hns_roce_bond_work);</div><div class='add'>+</div><div class='add'>+		bond_grp-&gt;bond_ready = false;</div><div class='add'>+		bond_grp-&gt;bond_state = HNS_ROCE_BOND_NOT_ATTACHED;</div><div class='add'>+		bond_grp-&gt;bus_num = bus_num;</div><div class='add'>+</div><div class='add'>+		ret = alloc_bond_id(bond_grp);</div><div class='add'>+		if (ret) {</div><div class='add'>+			dev_err(hr_dev-&gt;dev,</div><div class='add'>+				"failed to alloc bond ID, ret = %d.\n", ret);</div><div class='add'>+			goto alloc_id_err;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		bond_grp-&gt;bond_nb.notifier_call = hns_roce_bond_event;</div><div class='add'>+		ret = register_netdevice_notifier(&amp;bond_grp-&gt;bond_nb);</div><div class='add'>+		if (ret) {</div><div class='add'>+			ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+				  "failed to register bond nb, ret = %d.\n", ret);</div><div class='add'>+			goto register_nb_err;</div><div class='add'>+		}</div><div class='add'>+		bgrps[i] = bond_grp;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+register_nb_err:</div><div class='add'>+	remove_bond_id(bond_grp-&gt;bus_num, bond_grp-&gt;bond_id);</div><div class='add'>+alloc_id_err:</div><div class='add'>+	mutex_destroy(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+	kvfree(bond_grp);</div><div class='add'>+mem_err:</div><div class='add'>+	for (i--; i &gt;= 0; i--) {</div><div class='add'>+		unregister_netdevice_notifier(&amp;bgrps[i]-&gt;bond_nb);</div><div class='add'>+		cancel_delayed_work_sync(&amp;bgrps[i]-&gt;bond_work);</div><div class='add'>+		remove_bond_id(bgrps[i]-&gt;bus_num, bgrps[i]-&gt;bond_id);</div><div class='add'>+		mutex_destroy(&amp;bgrps[i]-&gt;bond_mutex);</div><div class='add'>+		kvfree(bgrps[i]);</div><div class='add'>+	}</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hns_roce_dealloc_bond_grp(void)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_bond_group *bond_grp;</div><div class='add'>+	struct hns_roce_die_info *die_info;</div><div class='add'>+	unsigned long id;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	xa_for_each(&amp;roce_bond_xa, id, die_info) {</div><div class='add'>+		for (i = 0; i &lt; ROCE_BOND_NUM_MAX; i++) {</div><div class='add'>+			bond_grp = die_info-&gt;bgrps[i];</div><div class='add'>+			if (!bond_grp)</div><div class='add'>+				continue;</div><div class='add'>+			unregister_netdevice_notifier(&amp;bond_grp-&gt;bond_nb);</div><div class='add'>+			cancel_delayed_work_sync(&amp;bond_grp-&gt;bond_work);</div><div class='add'>+			remove_bond_id(bond_grp-&gt;bus_num, bond_grp-&gt;bond_id);</div><div class='add'>+			mutex_destroy(&amp;bond_grp-&gt;bond_mutex);</div><div class='add'>+			kvfree(bond_grp);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hns_roce_bond_init(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct net_device *net_dev = get_hr_netdev(hr_dev, 0);</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	struct hns_roce_bond_group *bond_grp;</div><div class='add'>+	u8 bus_num = get_hr_bus_num(hr_dev);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	bond_grp = hns_roce_get_bond_grp(net_dev, bus_num);</div><div class='add'>+</div><div class='add'>+	if (priv-&gt;handle-&gt;rinfo.reset_state == HNS_ROCE_STATE_RST_INIT) {</div><div class='add'>+		ret = hns_roce_recover_bond(bond_grp, hr_dev);</div><div class='add'>+		if (ret) {</div><div class='add'>+			dev_err(hr_dev-&gt;dev,</div><div class='add'>+				"failed to recover RoCE bond, ret = %d.\n", ret);</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return hns_roce_set_bond_netdev(bond_grp, hr_dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hns_roce_bond_suspend(struct hnae3_handle *handle)</div><div class='add'>+{</div><div class='add'>+	u8 bus_num = handle-&gt;pdev-&gt;bus-&gt;number;</div><div class='add'>+	struct hns_roce_bond_group *bond_grp;</div><div class='add'>+	struct hns_roce_die_info *die_info;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	die_info = xa_load(&amp;roce_bond_xa, bus_num);</div><div class='add'>+	if (!die_info)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;die_info-&gt;die_mutex);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Avoid duplicated processing when calling this function</div><div class='add'>+	 * multiple times.</div><div class='add'>+	 */</div><div class='add'>+	if (die_info-&gt;suspend_cnt)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ROCE_BOND_NUM_MAX; i++) {</div><div class='add'>+		bond_grp = die_info-&gt;bgrps[i];</div><div class='add'>+		if (!bond_grp)</div><div class='add'>+			continue;</div><div class='add'>+		unregister_netdevice_notifier(&amp;bond_grp-&gt;bond_nb);</div><div class='add'>+		cancel_delayed_work_sync(&amp;bond_grp-&gt;bond_work);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	die_info-&gt;suspend_cnt++;</div><div class='add'>+	mutex_unlock(&amp;die_info-&gt;die_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hns_roce_bond_resume(struct hnae3_handle *handle)</div><div class='add'>+{</div><div class='add'>+	u8 bus_num = handle-&gt;pdev-&gt;bus-&gt;number;</div><div class='add'>+	struct hns_roce_bond_group *bond_grp;</div><div class='add'>+	struct hns_roce_die_info *die_info;</div><div class='add'>+	int i, ret;</div><div class='add'>+</div><div class='add'>+	die_info = xa_load(&amp;roce_bond_xa, bus_num);</div><div class='add'>+	if (!die_info)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;die_info-&gt;die_mutex);</div><div class='add'>+</div><div class='add'>+	die_info-&gt;suspend_cnt--;</div><div class='add'>+	if (die_info-&gt;suspend_cnt)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ROCE_BOND_NUM_MAX; i++) {</div><div class='add'>+		bond_grp = die_info-&gt;bgrps[i];</div><div class='add'>+		if (!bond_grp)</div><div class='add'>+			continue;</div><div class='add'>+		ret = register_netdevice_notifier(&amp;bond_grp-&gt;bond_nb);</div><div class='add'>+		if (ret)</div><div class='add'>+			dev_err(&amp;handle-&gt;pdev-&gt;dev,</div><div class='add'>+				"failed to resume bond notifier(bus_num = %u, id = %u), ret = %d.\n",</div><div class='add'>+				bus_num, bond_grp-&gt;bond_id, ret);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	mutex_unlock(&amp;die_info-&gt;die_mutex);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.h b/drivers/infiniband/hw/hns/hns_roce_bond.h<br/>new file mode 100644<br/>index 000000000000..98c295d78ca1<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_bond.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_bond.h</a></div><div class='hunk'>@@ -0,0 +1,95 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0+ */</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2025 Hisilicon Limited.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef _HNS_ROCE_BOND_H</div><div class='add'>+#define _HNS_ROCE_BOND_H</div><div class='add'>+</div><div class='add'>+#include &lt;linux/netdevice.h&gt;</div><div class='add'>+#include &lt;net/bonding.h&gt;</div><div class='add'>+</div><div class='add'>+#define ROCE_BOND_FUNC_MAX 4</div><div class='add'>+#define ROCE_BOND_NUM_MAX 2</div><div class='add'>+</div><div class='add'>+#define BOND_ID(id) BIT(id)</div><div class='add'>+</div><div class='add'>+#define BOND_ERR_LOG(fmt, ...)				\</div><div class='add'>+	pr_err("HNS RoCE Bonding: " fmt, ##__VA_ARGS__)</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	BOND_MODE_1,</div><div class='add'>+	BOND_MODE_2_4,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum hns_roce_bond_hashtype {</div><div class='add'>+	BOND_HASH_L2,</div><div class='add'>+	BOND_HASH_L34,</div><div class='add'>+	BOND_HASH_L23,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum bond_support_type {</div><div class='add'>+	BOND_NOT_SUPPORT,</div><div class='add'>+	/*</div><div class='add'>+	 * bond_grp already exists, but in the current</div><div class='add'>+	 * conditions it's no longer supported</div><div class='add'>+	 */</div><div class='add'>+	BOND_EXISTING_NOT_SUPPORT,</div><div class='add'>+	BOND_SUPPORT,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum hns_roce_bond_state {</div><div class='add'>+	HNS_ROCE_BOND_NOT_ATTACHED,</div><div class='add'>+	HNS_ROCE_BOND_NOT_BONDED,</div><div class='add'>+	HNS_ROCE_BOND_IS_BONDED,</div><div class='add'>+	HNS_ROCE_BOND_SLAVE_CHANGE_NUM,</div><div class='add'>+	HNS_ROCE_BOND_SLAVE_CHANGESTATE,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum hns_roce_bond_cmd_type {</div><div class='add'>+	HNS_ROCE_SET_BOND,</div><div class='add'>+	HNS_ROCE_CHANGE_BOND,</div><div class='add'>+	HNS_ROCE_CLEAR_BOND,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct hns_roce_func_info {</div><div class='add'>+	struct net_device *net_dev;</div><div class='add'>+	struct hnae3_handle *handle;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct hns_roce_bond_group {</div><div class='add'>+	struct net_device *upper_dev;</div><div class='add'>+	struct hns_roce_dev *main_hr_dev;</div><div class='add'>+	u8 active_slave_num;</div><div class='add'>+	u32 slave_map;</div><div class='add'>+	u32 active_slave_map;</div><div class='add'>+	u8 bond_id;</div><div class='add'>+	u8 bus_num;</div><div class='add'>+	struct hns_roce_func_info bond_func_info[ROCE_BOND_FUNC_MAX];</div><div class='add'>+	bool bond_ready;</div><div class='add'>+	enum hns_roce_bond_state bond_state;</div><div class='add'>+	enum netdev_lag_tx_type tx_type;</div><div class='add'>+	enum netdev_lag_hash hash_type;</div><div class='add'>+	struct mutex bond_mutex;</div><div class='add'>+	struct notifier_block bond_nb;</div><div class='add'>+	struct delayed_work bond_work;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct hns_roce_die_info {</div><div class='add'>+	u8 bond_id_mask;</div><div class='add'>+	struct hns_roce_bond_group *bgrps[ROCE_BOND_NUM_MAX];</div><div class='add'>+	struct mutex die_mutex;</div><div class='add'>+	u8 suspend_cnt;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct hns_roce_bond_group *hns_roce_get_bond_grp(struct net_device *net_dev,</div><div class='add'>+						  u8 bus_num);</div><div class='add'>+int hns_roce_alloc_bond_grp(struct hns_roce_dev *hr_dev);</div><div class='add'>+void hns_roce_dealloc_bond_grp(void);</div><div class='add'>+void hns_roce_cleanup_bond(struct hns_roce_bond_group *bond_grp);</div><div class='add'>+bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev);</div><div class='add'>+int hns_roce_bond_init(struct hns_roce_dev *hr_dev);</div><div class='add'>+void hns_roce_bond_suspend(struct hnae3_handle *handle);</div><div class='add'>+void hns_roce_bond_resume(struct hnae3_handle *handle);</div><div class='add'>+</div><div class='add'>+#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.c b/drivers/infiniband/hw/hns/hns_roce_cmd.c<br/>index 2acf946d02e5..873e8a69a1b9 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_cmd.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_cmd.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_cmd.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_cmd.c</a></div><div class='hunk'>@@ -31,60 +31,57 @@</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/dmapool.h&gt;</div><div class='del'>-#include &lt;linux/platform_device.h&gt;</div><div class='ctx'> #include "hns_roce_common.h"</div><div class='ctx'> #include "hns_roce_device.h"</div><div class='ctx'> #include "hns_roce_cmd.h"</div><div class='ctx'> </div><div class='del'>-#define CMD_POLL_TOKEN		0xffff</div><div class='del'>-#define CMD_MAX_NUM		32</div><div class='del'>-#define CMD_TOKEN_MASK		0x1f</div><div class='add'>+#define CMD_POLL_TOKEN 0xffff</div><div class='add'>+#define CMD_MAX_NUM 32</div><div class='ctx'> </div><div class='del'>-static int hns_roce_cmd_mbox_post_hw(struct hns_roce_dev *hr_dev, u64 in_param,</div><div class='del'>-				     u64 out_param, u32 in_modifier,</div><div class='del'>-				     u8 op_modifier, u16 op, u16 token,</div><div class='del'>-				     int event)</div><div class='add'>+static int hns_roce_cmd_mbox_post_hw(struct hns_roce_dev *hr_dev,</div><div class='add'>+				     struct hns_roce_mbox_msg *mbox_msg)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_cmdq *cmd = &amp;hr_dev-&gt;cmd;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;cmd-&gt;hcr_mutex);</div><div class='del'>-	ret = hr_dev-&gt;hw-&gt;post_mbox(hr_dev, in_param, out_param, in_modifier,</div><div class='del'>-				    op_modifier, op, token, event);</div><div class='del'>-	mutex_unlock(&amp;cmd-&gt;hcr_mutex);</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;post_mbox(hr_dev, mbox_msg);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_MBX_POSTED_CNT]);</div><div class='ctx'> </div><div class='del'>-	return ret;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* this should be called with "poll_sem" */</div><div class='del'>-static int __hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param,</div><div class='del'>-				    u64 out_param, unsigned long in_modifier,</div><div class='del'>-				    u8 op_modifier, u16 op,</div><div class='del'>-				    unsigned long timeout)</div><div class='add'>+static int __hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev,</div><div class='add'>+				    struct hns_roce_mbox_msg *mbox_msg)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_cmd_mbox_post_hw(hr_dev, in_param, out_param,</div><div class='del'>-					in_modifier, op_modifier, op,</div><div class='del'>-					CMD_POLL_TOKEN, 0);</div><div class='add'>+	ret = hns_roce_cmd_mbox_post_hw(hr_dev, mbox_msg);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "[cmd_poll]hns_roce_cmd_mbox_post_hw failed\n");</div><div class='add'>+		dev_err_ratelimited(hr_dev-&gt;dev,</div><div class='add'>+				    "failed to post mailbox 0x%x in poll mode, ret = %d.\n",</div><div class='add'>+				    mbox_msg-&gt;cmd, ret);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return hr_dev-&gt;hw-&gt;chk_mbox(hr_dev, timeout);</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;poll_mbox_done(hr_dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_MBX_POLLED_CNT]);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param,</div><div class='del'>-				  u64 out_param, unsigned long in_modifier,</div><div class='del'>-				  u8 op_modifier, u16 op, unsigned long timeout)</div><div class='add'>+static int hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev,</div><div class='add'>+				  struct hns_roce_mbox_msg *mbox_msg)</div><div class='ctx'> {</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	down(&amp;hr_dev-&gt;cmd.poll_sem);</div><div class='del'>-	ret = __hns_roce_cmd_mbox_poll(hr_dev, in_param, out_param, in_modifier,</div><div class='del'>-				       op_modifier, op, timeout);</div><div class='add'>+	ret = __hns_roce_cmd_mbox_poll(hr_dev, mbox_msg);</div><div class='ctx'> 	up(&amp;hr_dev-&gt;cmd.poll_sem);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -93,23 +90,24 @@ static int hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param,</div><div class='ctx'> void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status,</div><div class='ctx'> 			u64 out_param)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_cmd_context</div><div class='del'>-		*context = &amp;hr_dev-&gt;cmd.context[token &amp; hr_dev-&gt;cmd.token_mask];</div><div class='add'>+	struct hns_roce_cmd_context *context =</div><div class='add'>+		&amp;hr_dev-&gt;cmd.context[token % hr_dev-&gt;cmd.max_cmds];</div><div class='ctx'> </div><div class='del'>-	if (token != context-&gt;token)</div><div class='add'>+	if (unlikely(token != context-&gt;token)) {</div><div class='add'>+		dev_err_ratelimited(hr_dev-&gt;dev,</div><div class='add'>+				    "[cmd] invalid ae token 0x%x, context token is 0x%x.\n",</div><div class='add'>+				    token, context-&gt;token);</div><div class='ctx'> 		return;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	context-&gt;result = (status == HNS_ROCE_CMD_SUCCESS) ? 0 : (-EIO);</div><div class='ctx'> 	context-&gt;out_param = out_param;</div><div class='ctx'> 	complete(&amp;context-&gt;done);</div><div class='add'>+	atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_MBX_EVENT_CNT]);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_cmd_event);</div><div class='ctx'> </div><div class='del'>-/* this should be called with "use_events" */</div><div class='del'>-static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param,</div><div class='del'>-				    u64 out_param, unsigned long in_modifier,</div><div class='del'>-				    u8 op_modifier, u16 op,</div><div class='del'>-				    unsigned long timeout)</div><div class='add'>+static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev,</div><div class='add'>+				    struct hns_roce_mbox_msg *mbox_msg)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_cmdq *cmd = &amp;hr_dev-&gt;cmd;</div><div class='ctx'> 	struct hns_roce_cmd_context *context;</div><div class='hunk'>@@ -117,105 +115,91 @@ static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param,</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock(&amp;cmd-&gt;context_lock);</div><div class='del'>-	WARN_ON(cmd-&gt;free_head &lt; 0);</div><div class='del'>-	context = &amp;cmd-&gt;context[cmd-&gt;free_head];</div><div class='del'>-	context-&gt;token += cmd-&gt;token_mask + 1;</div><div class='del'>-	cmd-&gt;free_head = context-&gt;next;</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		context = &amp;cmd-&gt;context[cmd-&gt;free_head];</div><div class='add'>+		cmd-&gt;free_head = context-&gt;next;</div><div class='add'>+	} while (context-&gt;busy);</div><div class='add'>+</div><div class='add'>+	context-&gt;busy = 1;</div><div class='add'>+	context-&gt;token += cmd-&gt;max_cmds;</div><div class='add'>+</div><div class='ctx'> 	spin_unlock(&amp;cmd-&gt;context_lock);</div><div class='ctx'> </div><div class='del'>-	init_completion(&amp;context-&gt;done);</div><div class='add'>+	reinit_completion(&amp;context-&gt;done);</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_cmd_mbox_post_hw(hr_dev, in_param, out_param,</div><div class='del'>-					in_modifier, op_modifier, op,</div><div class='del'>-					context-&gt;token, 1);</div><div class='del'>-	if (ret)</div><div class='add'>+	mbox_msg-&gt;token = context-&gt;token;</div><div class='add'>+	ret = hns_roce_cmd_mbox_post_hw(hr_dev, mbox_msg);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err_ratelimited(dev,</div><div class='add'>+				    "failed to post mailbox 0x%x in event mode, ret = %d.\n",</div><div class='add'>+				    mbox_msg-&gt;cmd, ret);</div><div class='ctx'> 		goto out;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * It is timeout when wait_for_completion_timeout return 0</div><div class='del'>-	 * The return value is the time limit set in advance</div><div class='del'>-	 * how many seconds showing</div><div class='del'>-	 */</div><div class='ctx'> 	if (!wait_for_completion_timeout(&amp;context-&gt;done,</div><div class='del'>-					 msecs_to_jiffies(timeout))) {</div><div class='del'>-		dev_err(dev, "[cmd]wait_for_completion_timeout timeout\n");</div><div class='add'>+				msecs_to_jiffies(HNS_ROCE_CMD_TIMEOUT_MSECS))) {</div><div class='add'>+		dev_err_ratelimited(dev, "[cmd] token 0x%x mailbox 0x%x timeout.\n",</div><div class='add'>+				    context-&gt;token, mbox_msg-&gt;cmd);</div><div class='ctx'> 		ret = -EBUSY;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ret = context-&gt;result;</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "[cmd]event mod cmd process error!err=%d\n", ret);</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='add'>+	if (ret)</div><div class='add'>+		dev_err_ratelimited(dev, "[cmd] token 0x%x mailbox 0x%x error %d.\n",</div><div class='add'>+				    context-&gt;token, mbox_msg-&gt;cmd, ret);</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='del'>-	spin_lock(&amp;cmd-&gt;context_lock);</div><div class='del'>-	context-&gt;next = cmd-&gt;free_head;</div><div class='del'>-	cmd-&gt;free_head = context - cmd-&gt;context;</div><div class='del'>-	spin_unlock(&amp;cmd-&gt;context_lock);</div><div class='del'>-</div><div class='add'>+	context-&gt;busy = 0;</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param,</div><div class='del'>-				  u64 out_param, unsigned long in_modifier,</div><div class='del'>-				  u8 op_modifier, u16 op, unsigned long timeout)</div><div class='add'>+static int hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev,</div><div class='add'>+				  struct hns_roce_mbox_msg *mbox_msg)</div><div class='ctx'> {</div><div class='del'>-	int ret = 0;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	down(&amp;hr_dev-&gt;cmd.event_sem);</div><div class='del'>-	ret = __hns_roce_cmd_mbox_wait(hr_dev, in_param, out_param,</div><div class='del'>-				       in_modifier, op_modifier, op, timeout);</div><div class='add'>+	ret = __hns_roce_cmd_mbox_wait(hr_dev, mbox_msg);</div><div class='ctx'> 	up(&amp;hr_dev-&gt;cmd.event_sem);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param,</div><div class='del'>-		      unsigned long in_modifier, u8 op_modifier, u16 op,</div><div class='del'>-		      unsigned long timeout)</div><div class='add'>+		      u8 cmd, unsigned long tag)</div><div class='ctx'> {</div><div class='del'>-	int ret;</div><div class='add'>+	struct hns_roce_mbox_msg mbox_msg = {};</div><div class='add'>+	bool is_busy;</div><div class='ctx'> </div><div class='del'>-	if (hr_dev-&gt;hw-&gt;rst_prc_mbox) {</div><div class='del'>-		ret = hr_dev-&gt;hw-&gt;rst_prc_mbox(hr_dev);</div><div class='del'>-		if (ret == CMD_RST_PRC_SUCCESS)</div><div class='del'>-			return 0;</div><div class='del'>-		else if (ret == CMD_RST_PRC_EBUSY)</div><div class='del'>-			return -EBUSY;</div><div class='del'>-	}</div><div class='add'>+	if (hr_dev-&gt;hw-&gt;chk_mbox_avail)</div><div class='add'>+		if (!hr_dev-&gt;hw-&gt;chk_mbox_avail(hr_dev, &amp;is_busy))</div><div class='add'>+			return is_busy ? -EBUSY : 0;</div><div class='ctx'> </div><div class='del'>-	if (hr_dev-&gt;cmd.use_events)</div><div class='del'>-		ret = hns_roce_cmd_mbox_wait(hr_dev, in_param, out_param,</div><div class='del'>-					     in_modifier, op_modifier, op,</div><div class='del'>-					     timeout);</div><div class='del'>-	else</div><div class='del'>-		ret = hns_roce_cmd_mbox_poll(hr_dev, in_param, out_param,</div><div class='del'>-					     in_modifier, op_modifier, op,</div><div class='del'>-					     timeout);</div><div class='add'>+	mbox_msg.in_param = in_param;</div><div class='add'>+	mbox_msg.out_param = out_param;</div><div class='add'>+	mbox_msg.cmd = cmd;</div><div class='add'>+	mbox_msg.tag = tag;</div><div class='ctx'> </div><div class='del'>-	if (ret == CMD_RST_PRC_EBUSY)</div><div class='del'>-		return -EBUSY;</div><div class='add'>+	if (hr_dev-&gt;cmd.use_events) {</div><div class='add'>+		mbox_msg.event_en = 1;</div><div class='ctx'> </div><div class='del'>-	if (ret &amp;&amp; (hr_dev-&gt;hw-&gt;rst_prc_mbox &amp;&amp;</div><div class='del'>-		    hr_dev-&gt;hw-&gt;rst_prc_mbox(hr_dev) == CMD_RST_PRC_SUCCESS))</div><div class='del'>-		return 0;</div><div class='add'>+		return hns_roce_cmd_mbox_wait(hr_dev, &amp;mbox_msg);</div><div class='add'>+	} else {</div><div class='add'>+		mbox_msg.event_en = 0;</div><div class='add'>+		mbox_msg.token = CMD_POLL_TOKEN;</div><div class='ctx'> </div><div class='del'>-	return ret;</div><div class='add'>+		return hns_roce_cmd_mbox_poll(hr_dev, &amp;mbox_msg);</div><div class='add'>+	}</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_cmd_mbox);</div><div class='ctx'> </div><div class='ctx'> int hns_roce_cmd_init(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-</div><div class='del'>-	mutex_init(&amp;hr_dev-&gt;cmd.hcr_mutex);</div><div class='ctx'> 	sema_init(&amp;hr_dev-&gt;cmd.poll_sem, 1);</div><div class='ctx'> 	hr_dev-&gt;cmd.use_events = 0;</div><div class='del'>-	hr_dev-&gt;cmd.toggle = 1;</div><div class='ctx'> 	hr_dev-&gt;cmd.max_cmds = CMD_MAX_NUM;</div><div class='del'>-	hr_dev-&gt;cmd.pool = dma_pool_create("hns_roce_cmd", dev,</div><div class='add'>+	hr_dev-&gt;cmd.pool = dma_pool_create("hns_roce_cmd", hr_dev-&gt;dev,</div><div class='ctx'> 					   HNS_ROCE_MAILBOX_SIZE,</div><div class='ctx'> 					   HNS_ROCE_MAILBOX_SIZE, 0);</div><div class='ctx'> 	if (!hr_dev-&gt;cmd.pool)</div><div class='hunk'>@@ -234,47 +218,39 @@ int hns_roce_cmd_use_events(struct hns_roce_dev *hr_dev)</div><div class='ctx'> 	struct hns_roce_cmdq *hr_cmd = &amp;hr_dev-&gt;cmd;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	hr_cmd-&gt;context = kmalloc_array(hr_cmd-&gt;max_cmds,</div><div class='del'>-					sizeof(*hr_cmd-&gt;context),</div><div class='del'>-					GFP_KERNEL);</div><div class='del'>-	if (!hr_cmd-&gt;context)</div><div class='add'>+	hr_cmd-&gt;context =</div><div class='add'>+		kcalloc(hr_cmd-&gt;max_cmds, sizeof(*hr_cmd-&gt;context), GFP_KERNEL);</div><div class='add'>+	if (!hr_cmd-&gt;context) {</div><div class='add'>+		hr_dev-&gt;cmd_mod = 0;</div><div class='ctx'> 		return -ENOMEM;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; hr_cmd-&gt;max_cmds; ++i) {</div><div class='ctx'> 		hr_cmd-&gt;context[i].token = i;</div><div class='ctx'> 		hr_cmd-&gt;context[i].next = i + 1;</div><div class='add'>+		init_completion(&amp;hr_cmd-&gt;context[i].done);</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='del'>-	hr_cmd-&gt;context[hr_cmd-&gt;max_cmds - 1].next = -1;</div><div class='add'>+	hr_cmd-&gt;context[hr_cmd-&gt;max_cmds - 1].next = 0;</div><div class='ctx'> 	hr_cmd-&gt;free_head = 0;</div><div class='ctx'> </div><div class='ctx'> 	sema_init(&amp;hr_cmd-&gt;event_sem, hr_cmd-&gt;max_cmds);</div><div class='ctx'> 	spin_lock_init(&amp;hr_cmd-&gt;context_lock);</div><div class='ctx'> </div><div class='del'>-	hr_cmd-&gt;token_mask = CMD_TOKEN_MASK;</div><div class='ctx'> 	hr_cmd-&gt;use_events = 1;</div><div class='ctx'> </div><div class='del'>-	down(&amp;hr_cmd-&gt;poll_sem);</div><div class='del'>-</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_cmdq *hr_cmd = &amp;hr_dev-&gt;cmd;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	hr_cmd-&gt;use_events = 0;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; hr_cmd-&gt;max_cmds; ++i)</div><div class='del'>-		down(&amp;hr_cmd-&gt;event_sem);</div><div class='ctx'> </div><div class='ctx'> 	kfree(hr_cmd-&gt;context);</div><div class='del'>-	up(&amp;hr_cmd-&gt;poll_sem);</div><div class='add'>+	hr_cmd-&gt;use_events = 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct hns_roce_cmd_mailbox</div><div class='del'>-	*hns_roce_alloc_cmd_mailbox(struct hns_roce_dev *hr_dev)</div><div class='add'>+struct hns_roce_cmd_mailbox *</div><div class='add'>+hns_roce_alloc_cmd_mailbox(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_cmd_mailbox *mailbox;</div><div class='ctx'> </div><div class='hunk'>@@ -282,8 +258,8 @@ struct hns_roce_cmd_mailbox</div><div class='ctx'> 	if (!mailbox)</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='del'>-	mailbox-&gt;buf = dma_pool_alloc(hr_dev-&gt;cmd.pool, GFP_KERNEL,</div><div class='del'>-				      &amp;mailbox-&gt;dma);</div><div class='add'>+	mailbox-&gt;buf =</div><div class='add'>+		dma_pool_alloc(hr_dev-&gt;cmd.pool, GFP_KERNEL, &amp;mailbox-&gt;dma);</div><div class='ctx'> 	if (!mailbox-&gt;buf) {</div><div class='ctx'> 		kfree(mailbox);</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='hunk'>@@ -291,7 +267,6 @@ struct hns_roce_cmd_mailbox</div><div class='ctx'> </div><div class='ctx'> 	return mailbox;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_alloc_cmd_mailbox);</div><div class='ctx'> </div><div class='ctx'> void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 			       struct hns_roce_cmd_mailbox *mailbox)</div><div class='hunk'>@@ -302,4 +277,15 @@ void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 	dma_pool_free(hr_dev-&gt;cmd.pool, mailbox-&gt;buf, mailbox-&gt;dma);</div><div class='ctx'> 	kfree(mailbox);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_free_cmd_mailbox);</div><div class='add'>+</div><div class='add'>+int hns_roce_create_hw_ctx(struct hns_roce_dev *dev,</div><div class='add'>+			   struct hns_roce_cmd_mailbox *mailbox,</div><div class='add'>+			   u8 cmd, unsigned long idx)</div><div class='add'>+{</div><div class='add'>+	return hns_roce_cmd_mbox(dev, mailbox-&gt;dma, 0, cmd, idx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hns_roce_destroy_hw_ctx(struct hns_roce_dev *dev, u8 cmd, unsigned long idx)</div><div class='add'>+{</div><div class='add'>+	return hns_roce_cmd_mbox(dev, 0, 0, cmd, idx);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.h b/drivers/infiniband/hw/hns/hns_roce_cmd.h<br/>index 2b6ac646ca9a..11dbbabebdc9 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_cmd.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_cmd.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_cmd.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_cmd.h</a></div><div class='hunk'>@@ -108,6 +108,9 @@ enum {</div><div class='ctx'> 	HNS_ROCE_CMD_QUERY_CEQC		= 0x92,</div><div class='ctx'> 	HNS_ROCE_CMD_DESTROY_CEQC	= 0x93,</div><div class='ctx'> </div><div class='add'>+	/* SCC CTX commands */</div><div class='add'>+	HNS_ROCE_CMD_QUERY_SCCC		= 0xa2,</div><div class='add'>+</div><div class='ctx'> 	/* SCC CTX BT commands */</div><div class='ctx'> 	HNS_ROCE_CMD_READ_SCCC_BT0	= 0xa4,</div><div class='ctx'> 	HNS_ROCE_CMD_WRITE_SCCC_BT0	= 0xa5,</div><div class='hunk'>@@ -115,12 +118,12 @@ enum {</div><div class='ctx'> </div><div class='ctx'> enum {</div><div class='ctx'> 	/* TPT commands */</div><div class='del'>-	HNS_ROCE_CMD_SW2HW_MPT		= 0xd,</div><div class='del'>-	HNS_ROCE_CMD_HW2SW_MPT		= 0xf,</div><div class='add'>+	HNS_ROCE_CMD_CREATE_MPT		= 0xd,</div><div class='add'>+	HNS_ROCE_CMD_DESTROY_MPT	= 0xf,</div><div class='ctx'> </div><div class='ctx'> 	/* CQ commands */</div><div class='del'>-	HNS_ROCE_CMD_SW2HW_CQ		= 0x16,</div><div class='del'>-	HNS_ROCE_CMD_HW2SW_CQ		= 0x17,</div><div class='add'>+	HNS_ROCE_CMD_CREATE_CQC		= 0x16,</div><div class='add'>+	HNS_ROCE_CMD_DESTROY_CQC	= 0x17,</div><div class='ctx'> </div><div class='ctx'> 	/* QP/EE commands */</div><div class='ctx'> 	HNS_ROCE_CMD_RST2INIT_QP	= 0x19,</div><div class='hunk'>@@ -129,23 +132,27 @@ enum {</div><div class='ctx'> 	HNS_ROCE_CMD_RTS2RTS_QP		= 0x1c,</div><div class='ctx'> 	HNS_ROCE_CMD_2ERR_QP		= 0x1e,</div><div class='ctx'> 	HNS_ROCE_CMD_RTS2SQD_QP		= 0x1f,</div><div class='del'>-	HNS_ROCE_CMD_SQD2SQD_QP		= 0x38,</div><div class='ctx'> 	HNS_ROCE_CMD_SQD2RTS_QP		= 0x20,</div><div class='ctx'> 	HNS_ROCE_CMD_2RST_QP		= 0x21,</div><div class='ctx'> 	HNS_ROCE_CMD_QUERY_QP		= 0x22,</div><div class='del'>-	HNS_ROCE_CMD_SW2HW_SRQ		= 0x70,</div><div class='add'>+	HNS_ROCE_CMD_SQD2SQD_QP		= 0x38,</div><div class='add'>+	HNS_ROCE_CMD_CREATE_SRQ		= 0x70,</div><div class='ctx'> 	HNS_ROCE_CMD_MODIFY_SRQC	= 0x72,</div><div class='ctx'> 	HNS_ROCE_CMD_QUERY_SRQC		= 0x73,</div><div class='del'>-	HNS_ROCE_CMD_HW2SW_SRQ		= 0x74,</div><div class='add'>+	HNS_ROCE_CMD_DESTROY_SRQ	= 0x74,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param,</div><div class='del'>-		      unsigned long in_modifier, u8 op_modifier, u16 op,</div><div class='del'>-		      unsigned long timeout);</div><div class='add'>+		      u8 cmd, unsigned long tag);</div><div class='ctx'> </div><div class='del'>-struct hns_roce_cmd_mailbox</div><div class='del'>-	*hns_roce_alloc_cmd_mailbox(struct hns_roce_dev *hr_dev);</div><div class='add'>+struct hns_roce_cmd_mailbox *</div><div class='add'>+hns_roce_alloc_cmd_mailbox(struct hns_roce_dev *hr_dev);</div><div class='ctx'> void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 			       struct hns_roce_cmd_mailbox *mailbox);</div><div class='add'>+int hns_roce_create_hw_ctx(struct hns_roce_dev *dev,</div><div class='add'>+			   struct hns_roce_cmd_mailbox *mailbox,</div><div class='add'>+			   u8 cmd, unsigned long idx);</div><div class='add'>+int hns_roce_destroy_hw_ctx(struct hns_roce_dev *dev, u8 cmd,</div><div class='add'>+			    unsigned long idx);</div><div class='ctx'> </div><div class='ctx'> #endif /* _HNS_ROCE_CMD_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_common.h b/drivers/infiniband/hw/hns/hns_roce_common.h<br/>index 8e95a1aa1b4f..465d1f914b6c 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_common.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_common.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_common.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_common.h</a></div><div class='hunk'>@@ -32,232 +32,77 @@</div><div class='ctx'> </div><div class='ctx'> #ifndef _HNS_ROCE_COMMON_H</div><div class='ctx'> #define _HNS_ROCE_COMMON_H</div><div class='del'>-</div><div class='del'>-#ifndef assert</div><div class='del'>-#define assert(cond)</div><div class='del'>-#endif</div><div class='add'>+#include &lt;linux/bitfield.h&gt;</div><div class='ctx'> </div><div class='ctx'> #define roce_write(dev, reg, val)	writel((val), (dev)-&gt;reg_base + (reg))</div><div class='ctx'> #define roce_read(dev, reg)		readl((dev)-&gt;reg_base + (reg))</div><div class='ctx'> #define roce_raw_write(value, addr) \</div><div class='ctx'> 	__raw_writel((__force u32)cpu_to_le32(value), (addr))</div><div class='ctx'> </div><div class='del'>-#define roce_get_field(origin, mask, shift) \</div><div class='del'>-	(((le32_to_cpu(origin)) &amp; (mask)) &gt;&gt; (shift))</div><div class='add'>+#define roce_get_field(origin, mask, shift)                                    \</div><div class='add'>+	((le32_to_cpu(origin) &amp; (mask)) &gt;&gt; (u32)(shift))</div><div class='ctx'> </div><div class='ctx'> #define roce_get_bit(origin, shift) \</div><div class='ctx'> 	roce_get_field((origin), (1ul &lt;&lt; (shift)), (shift))</div><div class='ctx'> </div><div class='del'>-#define roce_set_field(origin, mask, shift, val) \</div><div class='del'>-	do { \</div><div class='del'>-		(origin) &amp;= ~cpu_to_le32(mask); \</div><div class='del'>-		(origin) |= cpu_to_le32(((u32)(val) &lt;&lt; (shift)) &amp; (mask)); \</div><div class='add'>+#define roce_set_field(origin, mask, shift, val)                               \</div><div class='add'>+	do {                                                                   \</div><div class='add'>+		(origin) &amp;= ~cpu_to_le32(mask);                                \</div><div class='add'>+		(origin) |=                                                    \</div><div class='add'>+			cpu_to_le32(((u32)(val) &lt;&lt; (u32)(shift)) &amp; (mask));    \</div><div class='ctx'> 	} while (0)</div><div class='ctx'> </div><div class='del'>-#define roce_set_bit(origin, shift, val) \</div><div class='add'>+#define roce_set_bit(origin, shift, val)                                       \</div><div class='ctx'> 	roce_set_field((origin), (1ul &lt;&lt; (shift)), (shift), (val))</div><div class='ctx'> </div><div class='del'>-#define ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S 3</div><div class='del'>-#define ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S 4</div><div class='del'>-</div><div class='del'>-#define ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S 5</div><div class='del'>-</div><div class='del'>-#define ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S 6</div><div class='del'>-</div><div class='del'>-#define ROCEE_GLB_CFG_ROCEE_PORT_ST_S 10</div><div class='del'>-#define ROCEE_GLB_CFG_ROCEE_PORT_ST_M  \</div><div class='del'>-	(((1UL &lt;&lt; 6) - 1) &lt;&lt; ROCEE_GLB_CFG_ROCEE_PORT_ST_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S 16</div><div class='del'>-</div><div class='del'>-#define ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S 0</div><div class='del'>-#define ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M  \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S 24</div><div class='del'>-#define ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M  \</div><div class='del'>-	(((1UL &lt;&lt; 4) - 1) &lt;&lt; ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S 0</div><div class='del'>-#define ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M   \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S 24</div><div class='del'>-#define ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M   \</div><div class='del'>-	(((1UL &lt;&lt; 4) - 1) &lt;&lt; ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S 0</div><div class='del'>-#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S 16</div><div class='del'>-#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S 0</div><div class='del'>-#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S 16</div><div class='del'>-#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_RAQ_WL_ROCEE_RAQ_WL_S 0</div><div class='del'>-#define ROCEE_RAQ_WL_ROCEE_RAQ_WL_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; ROCEE_RAQ_WL_ROCEE_RAQ_WL_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S 0</div><div class='del'>-#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; \</div><div class='del'>-	ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S 16</div><div class='del'>-#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M   \</div><div class='del'>-	(((1UL &lt;&lt; 4) - 1) &lt;&lt; \</div><div class='del'>-	ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S 20</div><div class='del'>-</div><div class='del'>-#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE 21</div><div class='del'>-</div><div class='del'>-#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S 0</div><div class='del'>-#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S 5</div><div class='del'>-#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S 0</div><div class='del'>-#define ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S 5</div><div class='del'>-#define ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S 0</div><div class='del'>-#define ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S 8</div><div class='del'>-#define ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S 0</div><div class='del'>-#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M   \</div><div class='del'>-	(((1UL &lt;&lt; 19) - 1) &lt;&lt; ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_S 19</div><div class='del'>-</div><div class='del'>-#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S 20</div><div class='del'>-#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M   \</div><div class='del'>-	(((1UL &lt;&lt; 2) - 1) &lt;&lt; ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S 22</div><div class='del'>-#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S 31</div><div class='del'>-</div><div class='del'>-#define ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S 0</div><div class='del'>-#define ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M   \</div><div class='del'>-	(((1UL &lt;&lt; 3) - 1) &lt;&lt; ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S 0</div><div class='del'>-#define ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_MB6_ROCEE_MB_CMD_S 0</div><div class='del'>-#define ROCEE_MB6_ROCEE_MB_CMD_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; ROCEE_MB6_ROCEE_MB_CMD_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_MB6_ROCEE_MB_CMD_MDF_S 8</div><div class='del'>-#define ROCEE_MB6_ROCEE_MB_CMD_MDF_M   \</div><div class='del'>-	(((1UL &lt;&lt; 4) - 1) &lt;&lt; ROCEE_MB6_ROCEE_MB_CMD_MDF_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_MB6_ROCEE_MB_EVENT_S 14</div><div class='del'>-</div><div class='del'>-#define ROCEE_MB6_ROCEE_MB_HW_RUN_S 15</div><div class='del'>-</div><div class='del'>-#define ROCEE_MB6_ROCEE_MB_TOKEN_S 16</div><div class='del'>-#define ROCEE_MB6_ROCEE_MB_TOKEN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; ROCEE_MB6_ROCEE_MB_TOKEN_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S 0</div><div class='del'>-#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S 24</div><div class='del'>-#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M   \</div><div class='del'>-	(((1UL &lt;&lt; 4) - 1) &lt;&lt; ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S 28</div><div class='del'>-#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M   \</div><div class='del'>-	(((1UL &lt;&lt; 3) - 1) &lt;&lt; ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S 31</div><div class='del'>-</div><div class='del'>-#define ROCEE_SMAC_H_ROCEE_SMAC_H_S 0</div><div class='del'>-#define ROCEE_SMAC_H_ROCEE_SMAC_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; ROCEE_SMAC_H_ROCEE_SMAC_H_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_SMAC_H_ROCEE_PORT_MTU_S 16</div><div class='del'>-#define ROCEE_SMAC_H_ROCEE_PORT_MTU_M   \</div><div class='del'>-	(((1UL &lt;&lt; 4) - 1) &lt;&lt; ROCEE_SMAC_H_ROCEE_PORT_MTU_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S 0</div><div class='del'>-#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M   \</div><div class='del'>-	(((1UL &lt;&lt; 2) - 1) &lt;&lt; ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S 8</div><div class='del'>-#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 4) - 1) &lt;&lt; ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQ_ALM_OVF_INT_ST_S 17</div><div class='del'>-</div><div class='del'>-#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S 0</div><div class='del'>-#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S 16</div><div class='del'>-#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S)</div><div class='del'>-</div><div class='del'>-#define ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S 0</div><div class='del'>-#define ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S)</div><div class='add'>+#define FIELD_LOC(field_type, field_h, field_l) field_type, field_h, field_l</div><div class='ctx'> </div><div class='del'>-#define ROCEE_CAEP_CEQC_SHIFT_CAEP_CEQ_ALM_OVF_INT_ST_S 16</div><div class='del'>-#define ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S 1</div><div class='del'>-#define ROCEE_CAEP_CEQ_ALM_OVF_CAEP_CEQ_ALM_OVF_S 0</div><div class='add'>+#define _hr_reg_enable(ptr, field_type, field_h, field_l)                      \</div><div class='add'>+	({                                                                     \</div><div class='add'>+		const field_type *_ptr = ptr;                                  \</div><div class='add'>+		*((__le32 *)_ptr + (field_h) / 32) |= cpu_to_le32(             \</div><div class='add'>+			BIT((field_l) % 32) +                                  \</div><div class='add'>+			BUILD_BUG_ON_ZERO((field_h) != (field_l)));            \</div><div class='add'>+	})</div><div class='ctx'> </div><div class='del'>-#define ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S 0</div><div class='del'>-#define ROCEE_CAEP_AE_MASK_CAEP_AE_IRQ_MASK_S 1</div><div class='add'>+#define hr_reg_enable(ptr, field) _hr_reg_enable(ptr, field)</div><div class='ctx'> </div><div class='del'>-#define ROCEE_CAEP_AE_ST_CAEP_AEQ_ALM_OVF_S 0</div><div class='add'>+#define _hr_reg_clear(ptr, field_type, field_h, field_l)                       \</div><div class='add'>+	({                                                                     \</div><div class='add'>+		const field_type *_ptr = ptr;                                  \</div><div class='add'>+		BUILD_BUG_ON(((field_h) / 32) != ((field_l) / 32));            \</div><div class='add'>+		*((__le32 *)_ptr + (field_h) / 32) &amp;=                          \</div><div class='add'>+			~cpu_to_le32(GENMASK((field_h) % 32, (field_l) % 32)); \</div><div class='add'>+	})</div><div class='ctx'> </div><div class='del'>-#define ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S 0</div><div class='del'>-#define ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_M   \</div><div class='del'>-	(((1UL &lt;&lt; 28) - 1) &lt;&lt; ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S)</div><div class='add'>+#define hr_reg_clear(ptr, field) _hr_reg_clear(ptr, field)</div><div class='ctx'> </div><div class='del'>-#define ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S 0</div><div class='del'>-#define ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M   \</div><div class='del'>-	(((1UL &lt;&lt; 28) - 1) &lt;&lt; ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S)</div><div class='add'>+#define _hr_reg_write_bool(ptr, field_type, field_h, field_l, val)             \</div><div class='add'>+	({                                                                     \</div><div class='add'>+		(val) ? _hr_reg_enable(ptr, field_type, field_h, field_l) :    \</div><div class='add'>+			_hr_reg_clear(ptr, field_type, field_h, field_l);      \</div><div class='add'>+	})</div><div class='ctx'> </div><div class='del'>-#define ROCEE_SDB_INV_CNT_SDB_INV_CNT_S 0</div><div class='del'>-#define ROCEE_SDB_INV_CNT_SDB_INV_CNT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; ROCEE_SDB_INV_CNT_SDB_INV_CNT_S)</div><div class='add'>+#define hr_reg_write_bool(ptr, field, val) _hr_reg_write_bool(ptr, field, val)</div><div class='ctx'> </div><div class='del'>-#define ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S	0</div><div class='del'>-#define ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M	\</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S)</div><div class='add'>+#define _hr_reg_write(ptr, field_type, field_h, field_l, val)                  \</div><div class='add'>+	({                                                                     \</div><div class='add'>+		_hr_reg_clear(ptr, field_type, field_h, field_l);              \</div><div class='add'>+		*((__le32 *)ptr + (field_h) / 32) |= cpu_to_le32(FIELD_PREP(   \</div><div class='add'>+			GENMASK((field_h) % 32, (field_l) % 32), val));        \</div><div class='add'>+	})</div><div class='ctx'> </div><div class='del'>-#define ROCEE_SDB_CNT_CMP_BITS 16</div><div class='add'>+#define hr_reg_write(ptr, field, val) _hr_reg_write(ptr, field, val)</div><div class='ctx'> </div><div class='del'>-#define ROCEE_TSP_BP_ST_QH_FIFO_ENTRY_S	20</div><div class='add'>+#define _hr_reg_read(ptr, field_type, field_h, field_l)                        \</div><div class='add'>+	({                                                                     \</div><div class='add'>+		const field_type *_ptr = ptr;                                  \</div><div class='add'>+		BUILD_BUG_ON(((field_h) / 32) != ((field_l) / 32));            \</div><div class='add'>+		FIELD_GET(GENMASK((field_h) % 32, (field_l) % 32),             \</div><div class='add'>+			  le32_to_cpu(*((__le32 *)_ptr + (field_h) / 32)));    \</div><div class='add'>+	})</div><div class='ctx'> </div><div class='del'>-#define ROCEE_CNT_CLR_CE_CNT_CLR_CE_S 0</div><div class='add'>+#define hr_reg_read(ptr, field) _hr_reg_read(ptr, field)</div><div class='ctx'> </div><div class='ctx'> /*************ROCEE_REG DEFINITION****************/</div><div class='ctx'> #define ROCEE_VENDOR_ID_REG			0x0</div><div class='hunk'>@@ -334,8 +179,8 @@</div><div class='ctx'> #define ROCEE_TX_CMQ_BASEADDR_L_REG		0x07000</div><div class='ctx'> #define ROCEE_TX_CMQ_BASEADDR_H_REG		0x07004</div><div class='ctx'> #define ROCEE_TX_CMQ_DEPTH_REG			0x07008</div><div class='del'>-#define ROCEE_TX_CMQ_TAIL_REG			0x07010</div><div class='del'>-#define ROCEE_TX_CMQ_HEAD_REG			0x07014</div><div class='add'>+#define ROCEE_TX_CMQ_PI_REG			0x07010</div><div class='add'>+#define ROCEE_TX_CMQ_CI_REG			0x07014</div><div class='ctx'> </div><div class='ctx'> #define ROCEE_RX_CMQ_BASEADDR_L_REG		0x07018</div><div class='ctx'> #define ROCEE_RX_CMQ_BASEADDR_H_REG		0x0701c</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c<br/>index 9caf35061721..6aa82fe9dd3d 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_cq.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_cq.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_cq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_cq.c</a></div><div class='hunk'>@@ -30,507 +30,565 @@</div><div class='ctx'>  * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/platform_device.h&gt;</div><div class='add'>+#include &lt;linux/pci.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_umem.h&gt;</div><div class='ctx'> #include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='ctx'> #include "hns_roce_device.h"</div><div class='ctx'> #include "hns_roce_cmd.h"</div><div class='ctx'> #include "hns_roce_hem.h"</div><div class='del'>-#include &lt;rdma/hns-abi.h&gt;</div><div class='ctx'> #include "hns_roce_common.h"</div><div class='ctx'> </div><div class='del'>-static void hns_roce_ib_cq_comp(struct hns_roce_cq *hr_cq)</div><div class='add'>+void hns_roce_put_cq_bankid_for_uctx(struct hns_roce_ucontext *uctx)</div><div class='ctx'> {</div><div class='del'>-	struct ib_cq *ibcq = &amp;hr_cq-&gt;ib_cq;</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(uctx-&gt;ibucontext.device);</div><div class='add'>+	struct hns_roce_cq_table *cq_table = &amp;hr_dev-&gt;cq_table;</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision &lt; PCI_REVISION_ID_HIP09)</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-	ibcq-&gt;comp_handler(ibcq, ibcq-&gt;cq_context);</div><div class='add'>+	mutex_lock(&amp;cq_table-&gt;bank_mutex);</div><div class='add'>+	cq_table-&gt;ctx_num[uctx-&gt;cq_bank_id]--;</div><div class='add'>+	mutex_unlock(&amp;cq_table-&gt;bank_mutex);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_ib_cq_event(struct hns_roce_cq *hr_cq,</div><div class='del'>-				 enum hns_roce_event event_type)</div><div class='add'>+void hns_roce_get_cq_bankid_for_uctx(struct hns_roce_ucontext *uctx)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_dev *hr_dev;</div><div class='del'>-	struct ib_event event;</div><div class='del'>-	struct ib_cq *ibcq;</div><div class='del'>-</div><div class='del'>-	ibcq = &amp;hr_cq-&gt;ib_cq;</div><div class='del'>-	hr_dev = to_hr_dev(ibcq-&gt;device);</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(uctx-&gt;ibucontext.device);</div><div class='add'>+	struct hns_roce_cq_table *cq_table = &amp;hr_dev-&gt;cq_table;</div><div class='add'>+	u32 least_load = cq_table-&gt;ctx_num[0];</div><div class='add'>+	u8 bankid = 0;</div><div class='add'>+	u8 i;</div><div class='ctx'> </div><div class='del'>-	if (event_type != HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID &amp;&amp;</div><div class='del'>-	    event_type != HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR &amp;&amp;</div><div class='del'>-	    event_type != HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev,</div><div class='del'>-			"hns_roce_ib: Unexpected event type 0x%x on CQ %06lx\n",</div><div class='del'>-			event_type, hr_cq-&gt;cqn);</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision &lt; PCI_REVISION_ID_HIP09)</div><div class='ctx'> 		return;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;cq_table-&gt;bank_mutex);</div><div class='add'>+	for (i = 1; i &lt; HNS_ROCE_CQ_BANK_NUM; i++) {</div><div class='add'>+		if (cq_table-&gt;ctx_num[i] &lt; least_load) {</div><div class='add'>+			least_load = cq_table-&gt;ctx_num[i];</div><div class='add'>+			bankid = i;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='add'>+	cq_table-&gt;ctx_num[bankid]++;</div><div class='add'>+	mutex_unlock(&amp;cq_table-&gt;bank_mutex);</div><div class='ctx'> </div><div class='del'>-	if (ibcq-&gt;event_handler) {</div><div class='del'>-		event.device = ibcq-&gt;device;</div><div class='del'>-		event.event = IB_EVENT_CQ_ERR;</div><div class='del'>-		event.element.cq = ibcq;</div><div class='del'>-		ibcq-&gt;event_handler(&amp;event, ibcq-&gt;cq_context);</div><div class='add'>+	uctx-&gt;cq_bank_id = bankid;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u8 get_least_load_bankid_for_cq(struct hns_roce_bank *bank)</div><div class='add'>+{</div><div class='add'>+	u32 least_load = bank[0].inuse;</div><div class='add'>+	u8 bankid = 0;</div><div class='add'>+	u32 bankcnt;</div><div class='add'>+	u8 i;</div><div class='add'>+</div><div class='add'>+	for (i = 1; i &lt; HNS_ROCE_CQ_BANK_NUM; i++) {</div><div class='add'>+		bankcnt = bank[i].inuse;</div><div class='add'>+		if (bankcnt &lt; least_load) {</div><div class='add'>+			least_load = bankcnt;</div><div class='add'>+			bankid = i;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	return bankid;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_sw2hw_cq(struct hns_roce_dev *dev,</div><div class='del'>-			     struct hns_roce_cmd_mailbox *mailbox,</div><div class='del'>-			     unsigned long cq_num)</div><div class='add'>+static u8 select_cq_bankid(struct hns_roce_dev *hr_dev,</div><div class='add'>+			   struct hns_roce_bank *bank, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	return hns_roce_cmd_mbox(dev, mailbox-&gt;dma, 0, cq_num, 0,</div><div class='del'>-			    HNS_ROCE_CMD_SW2HW_CQ, HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='add'>+	struct hns_roce_ucontext *uctx = udata ?</div><div class='add'>+		rdma_udata_to_drv_context(udata, struct hns_roce_ucontext,</div><div class='add'>+					  ibucontext) : NULL;</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09)</div><div class='add'>+		return uctx ? uctx-&gt;cq_bank_id : 0;</div><div class='add'>+</div><div class='add'>+	return get_least_load_bankid_for_cq(bank);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_cq_alloc(struct hns_roce_dev *hr_dev, int nent,</div><div class='del'>-			     struct hns_roce_mtt *hr_mtt,</div><div class='del'>-			     struct hns_roce_uar *hr_uar,</div><div class='del'>-			     struct hns_roce_cq *hr_cq, int vector)</div><div class='add'>+static int alloc_cqn(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,</div><div class='add'>+		     struct ib_udata *udata)</div><div class='ctx'> {</div><div class='add'>+	struct hns_roce_cq_table *cq_table = &amp;hr_dev-&gt;cq_table;</div><div class='add'>+	struct hns_roce_bank *bank;</div><div class='add'>+	u8 bankid;</div><div class='add'>+	int id;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;cq_table-&gt;bank_mutex);</div><div class='add'>+	bankid = select_cq_bankid(hr_dev, cq_table-&gt;bank, udata);</div><div class='add'>+	bank = &amp;cq_table-&gt;bank[bankid];</div><div class='add'>+</div><div class='add'>+	id = ida_alloc_range(&amp;bank-&gt;ida, bank-&gt;min, bank-&gt;max, GFP_KERNEL);</div><div class='add'>+	if (id &lt; 0) {</div><div class='add'>+		mutex_unlock(&amp;cq_table-&gt;bank_mutex);</div><div class='add'>+		return id;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* the lower 2 bits is bankid */</div><div class='add'>+	hr_cq-&gt;cqn = (id &lt;&lt; CQ_BANKID_SHIFT) | bankid;</div><div class='add'>+	bank-&gt;inuse++;</div><div class='add'>+	mutex_unlock(&amp;cq_table-&gt;bank_mutex);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u8 get_cq_bankid(unsigned long cqn)</div><div class='add'>+{</div><div class='add'>+	/* The lower 2 bits of CQN are used to hash to different banks */</div><div class='add'>+	return (u8)(cqn &amp; GENMASK(1, 0));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void free_cqn(struct hns_roce_dev *hr_dev, unsigned long cqn)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_cq_table *cq_table = &amp;hr_dev-&gt;cq_table;</div><div class='add'>+	struct hns_roce_bank *bank;</div><div class='add'>+</div><div class='add'>+	bank = &amp;cq_table-&gt;bank[get_cq_bankid(cqn)];</div><div class='add'>+</div><div class='add'>+	ida_free(&amp;bank-&gt;ida, cqn &gt;&gt; CQ_BANKID_SHIFT);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;cq_table-&gt;bank_mutex);</div><div class='add'>+	bank-&gt;inuse--;</div><div class='add'>+	mutex_unlock(&amp;cq_table-&gt;bank_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_create_cqc(struct hns_roce_dev *hr_dev,</div><div class='add'>+			       struct hns_roce_cq *hr_cq,</div><div class='add'>+			       u64 *mtts, dma_addr_t dma_handle)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='ctx'> 	struct hns_roce_cmd_mailbox *mailbox;</div><div class='del'>-	struct hns_roce_hem_table *mtt_table;</div><div class='del'>-	struct hns_roce_cq_table *cq_table;</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	dma_addr_t dma_handle;</div><div class='del'>-	u64 *mtts;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	cq_table = &amp;hr_dev-&gt;cq_table;</div><div class='add'>+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='add'>+	if (IS_ERR(mailbox)) {</div><div class='add'>+		ibdev_err(ibdev, "failed to alloc mailbox for CQC.\n");</div><div class='add'>+		return PTR_ERR(mailbox);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	/* Get the physical address of cq buf */</div><div class='del'>-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))</div><div class='del'>-		mtt_table = &amp;hr_dev-&gt;mr_table.mtt_cqe_table;</div><div class='del'>-	else</div><div class='del'>-		mtt_table = &amp;hr_dev-&gt;mr_table.mtt_table;</div><div class='add'>+	hr_dev-&gt;hw-&gt;write_cqc(hr_dev, hr_cq, mailbox-&gt;buf, mtts, dma_handle);</div><div class='ctx'> </div><div class='del'>-	mtts = hns_roce_table_find(hr_dev, mtt_table,</div><div class='del'>-				   hr_mtt-&gt;first_seg, &amp;dma_handle);</div><div class='del'>-	if (!mtts) {</div><div class='del'>-		dev_err(dev, "CQ alloc.Failed to find cq buf addr.\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='add'>+	ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_CQC,</div><div class='add'>+				     hr_cq-&gt;cqn);</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_err(ibdev,</div><div class='add'>+			  "failed to send create cmd for CQ(0x%lx), ret = %d.\n",</div><div class='add'>+			  hr_cq-&gt;cqn, ret);</div><div class='ctx'> </div><div class='del'>-	if (vector &gt;= hr_dev-&gt;caps.num_comp_vectors) {</div><div class='del'>-		dev_err(dev, "CQ alloc.Invalid vector.\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-	hr_cq-&gt;vector = vector;</div><div class='add'>+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_bitmap_alloc(&amp;cq_table-&gt;bitmap, &amp;hr_cq-&gt;cqn);</div><div class='del'>-	if (ret == -1) {</div><div class='del'>-		dev_err(dev, "CQ alloc.Failed to alloc index.\n");</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_cq_table *cq_table = &amp;hr_dev-&gt;cq_table;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	u64 mtts[MTT_MIN_COUNT] = {};</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_mtr_find(hr_dev, &amp;hr_cq-&gt;mtr, 0, mtts, ARRAY_SIZE(mtts));</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to find CQ mtr, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Get CQC memory HEM(Hardware Entry Memory) table */</div><div class='ctx'> 	ret = hns_roce_table_get(hr_dev, &amp;cq_table-&gt;table, hr_cq-&gt;cqn);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "CQ alloc.Failed to get context mem.\n");</div><div class='del'>-		goto err_out;</div><div class='add'>+		ibdev_err(ibdev, "failed to get CQ(0x%lx) context, ret = %d.\n",</div><div class='add'>+			  hr_cq-&gt;cqn, ret);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = xa_err(xa_store(&amp;cq_table-&gt;array, hr_cq-&gt;cqn, hr_cq, GFP_KERNEL));</div><div class='add'>+	ret = xa_err(xa_store_irq(&amp;cq_table-&gt;array, hr_cq-&gt;cqn, hr_cq, GFP_KERNEL));</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "CQ alloc failed xa_store.\n");</div><div class='add'>+		ibdev_err(ibdev, "failed to xa_store CQ, ret = %d.\n", ret);</div><div class='ctx'> 		goto err_put;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* Allocate mailbox memory */</div><div class='del'>-	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='del'>-	if (IS_ERR(mailbox)) {</div><div class='del'>-		ret = PTR_ERR(mailbox);</div><div class='del'>-		goto err_xa;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	hr_dev-&gt;hw-&gt;write_cqc(hr_dev, hr_cq, mailbox-&gt;buf, mtts, dma_handle,</div><div class='del'>-			      nent, vector);</div><div class='del'>-</div><div class='del'>-	/* Send mailbox to hw */</div><div class='del'>-	ret = hns_roce_sw2hw_cq(hr_dev, mailbox, hr_cq-&gt;cqn);</div><div class='del'>-	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "CQ alloc.Failed to cmd mailbox.\n");</div><div class='add'>+	ret = hns_roce_create_cqc(hr_dev, hr_cq, mtts,</div><div class='add'>+				  hns_roce_get_mtr_ba(&amp;hr_cq-&gt;mtr));</div><div class='add'>+	if (ret)</div><div class='ctx'> 		goto err_xa;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	hr_cq-&gt;cons_index = 0;</div><div class='del'>-	hr_cq-&gt;arm_sn = 1;</div><div class='del'>-	hr_cq-&gt;uar = hr_uar;</div><div class='del'>-</div><div class='del'>-	atomic_set(&amp;hr_cq-&gt;refcount, 1);</div><div class='del'>-	init_completion(&amp;hr_cq-&gt;free);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> err_xa:</div><div class='del'>-	xa_erase(&amp;cq_table-&gt;array, hr_cq-&gt;cqn);</div><div class='del'>-</div><div class='add'>+	xa_erase_irq(&amp;cq_table-&gt;array, hr_cq-&gt;cqn);</div><div class='ctx'> err_put:</div><div class='ctx'> 	hns_roce_table_put(hr_dev, &amp;cq_table-&gt;table, hr_cq-&gt;cqn);</div><div class='ctx'> </div><div class='del'>-err_out:</div><div class='del'>-	hns_roce_bitmap_free(&amp;cq_table-&gt;bitmap, hr_cq-&gt;cqn, BITMAP_NO_RR);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_hw2sw_cq(struct hns_roce_dev *dev,</div><div class='del'>-			     struct hns_roce_cmd_mailbox *mailbox,</div><div class='del'>-			     unsigned long cq_num)</div><div class='del'>-{</div><div class='del'>-	return hns_roce_cmd_mbox(dev, 0, mailbox ? mailbox-&gt;dma : 0, cq_num,</div><div class='del'>-				 mailbox ? 0 : 1, HNS_ROCE_CMD_HW2SW_CQ,</div><div class='del'>-				 HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)</div><div class='add'>+static void free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_cq_table *cq_table = &amp;hr_dev-&gt;cq_table;</div><div class='ctx'> 	struct device *dev = hr_dev-&gt;dev;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_hw2sw_cq(hr_dev, NULL, hr_cq-&gt;cqn);</div><div class='add'>+	ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_CQC,</div><div class='add'>+				      hr_cq-&gt;cqn);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		dev_err(dev, "HW2SW_CQ failed (%d) for CQN %06lx\n", ret,</div><div class='del'>-			hr_cq-&gt;cqn);</div><div class='add'>+		dev_err_ratelimited(dev, "DESTROY_CQ failed (%d) for CQN %06lx\n",</div><div class='add'>+				    ret, hr_cq-&gt;cqn);</div><div class='ctx'> </div><div class='del'>-	xa_erase(&amp;cq_table-&gt;array, hr_cq-&gt;cqn);</div><div class='add'>+	xa_erase_irq(&amp;cq_table-&gt;array, hr_cq-&gt;cqn);</div><div class='ctx'> </div><div class='ctx'> 	/* Waiting interrupt process procedure carried out */</div><div class='ctx'> 	synchronize_irq(hr_dev-&gt;eq_table.eq[hr_cq-&gt;vector].irq);</div><div class='ctx'> </div><div class='ctx'> 	/* wait for all interrupt processed */</div><div class='del'>-	if (atomic_dec_and_test(&amp;hr_cq-&gt;refcount))</div><div class='add'>+	if (refcount_dec_and_test(&amp;hr_cq-&gt;refcount))</div><div class='ctx'> 		complete(&amp;hr_cq-&gt;free);</div><div class='ctx'> 	wait_for_completion(&amp;hr_cq-&gt;free);</div><div class='ctx'> </div><div class='ctx'> 	hns_roce_table_put(hr_dev, &amp;cq_table-&gt;table, hr_cq-&gt;cqn);</div><div class='del'>-	hns_roce_bitmap_free(&amp;cq_table-&gt;bitmap, hr_cq-&gt;cqn, BITMAP_NO_RR);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_free_cq);</div><div class='ctx'> </div><div class='del'>-static int hns_roce_ib_get_cq_umem(struct hns_roce_dev *hr_dev,</div><div class='del'>-				   struct ib_udata *udata,</div><div class='del'>-				   struct hns_roce_cq_buf *buf,</div><div class='del'>-				   struct ib_umem **umem, u64 buf_addr, int cqe)</div><div class='add'>+static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,</div><div class='add'>+			struct ib_udata *udata, unsigned long addr)</div><div class='ctx'> {</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	struct hns_roce_buf_attr buf_attr = {};</div><div class='ctx'> 	int ret;</div><div class='del'>-	u32 page_shift;</div><div class='del'>-	u32 npages;</div><div class='del'>-</div><div class='del'>-	*umem = ib_umem_get(udata, buf_addr, cqe * hr_dev-&gt;caps.cq_entry_sz,</div><div class='del'>-			    IB_ACCESS_LOCAL_WRITE, 1);</div><div class='del'>-	if (IS_ERR(*umem))</div><div class='del'>-		return PTR_ERR(*umem);</div><div class='del'>-</div><div class='del'>-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))</div><div class='del'>-		buf-&gt;hr_mtt.mtt_type = MTT_TYPE_CQE;</div><div class='del'>-	else</div><div class='del'>-		buf-&gt;hr_mtt.mtt_type = MTT_TYPE_WQE;</div><div class='del'>-</div><div class='del'>-	if (hr_dev-&gt;caps.cqe_buf_pg_sz) {</div><div class='del'>-		npages = (ib_umem_page_count(*umem) +</div><div class='del'>-			(1 &lt;&lt; hr_dev-&gt;caps.cqe_buf_pg_sz) - 1) /</div><div class='del'>-			(1 &lt;&lt; hr_dev-&gt;caps.cqe_buf_pg_sz);</div><div class='del'>-		page_shift = PAGE_SHIFT + hr_dev-&gt;caps.cqe_buf_pg_sz;</div><div class='del'>-		ret = hns_roce_mtt_init(hr_dev, npages, page_shift,</div><div class='del'>-					&amp;buf-&gt;hr_mtt);</div><div class='del'>-	} else {</div><div class='del'>-		ret = hns_roce_mtt_init(hr_dev, ib_umem_page_count(*umem),</div><div class='del'>-				(*umem)-&gt;page_shift,</div><div class='del'>-				&amp;buf-&gt;hr_mtt);</div><div class='del'>-	}</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_buf;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_ib_umem_write_mtt(hr_dev, &amp;buf-&gt;hr_mtt, *umem);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_mtt;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='add'>+	buf_attr.page_shift = hr_dev-&gt;caps.cqe_buf_pg_sz + PAGE_SHIFT;</div><div class='add'>+	buf_attr.region[0].size = hr_cq-&gt;cq_depth * hr_cq-&gt;cqe_size;</div><div class='add'>+	buf_attr.region[0].hopnum = hr_dev-&gt;caps.cqe_hop_num;</div><div class='add'>+	buf_attr.region_count = 1;</div><div class='ctx'> </div><div class='del'>-err_mtt:</div><div class='del'>-	hns_roce_mtt_cleanup(hr_dev, &amp;buf-&gt;hr_mtt);</div><div class='add'>+	ret = hns_roce_mtr_create(hr_dev, &amp;hr_cq-&gt;mtr, &amp;buf_attr,</div><div class='add'>+				  hr_dev-&gt;caps.cqe_ba_pg_sz + PAGE_SHIFT,</div><div class='add'>+				  udata, addr);</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_err(ibdev, "failed to alloc CQ mtr, ret = %d.\n", ret);</div><div class='ctx'> </div><div class='del'>-err_buf:</div><div class='del'>-	ib_umem_release(*umem);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_ib_alloc_cq_buf(struct hns_roce_dev *hr_dev,</div><div class='del'>-				    struct hns_roce_cq_buf *buf, u32 nent)</div><div class='add'>+static void free_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)</div><div class='ctx'> {</div><div class='del'>-	int ret;</div><div class='del'>-	u32 page_shift = PAGE_SHIFT + hr_dev-&gt;caps.cqe_buf_pg_sz;</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_buf_alloc(hr_dev, nent * hr_dev-&gt;caps.cq_entry_sz,</div><div class='del'>-				 (1 &lt;&lt; page_shift) * 2, &amp;buf-&gt;hr_buf,</div><div class='del'>-				 page_shift);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto out;</div><div class='del'>-</div><div class='del'>-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))</div><div class='del'>-		buf-&gt;hr_mtt.mtt_type = MTT_TYPE_CQE;</div><div class='del'>-	else</div><div class='del'>-		buf-&gt;hr_mtt.mtt_type = MTT_TYPE_WQE;</div><div class='add'>+	hns_roce_mtr_destroy(hr_dev, &amp;hr_cq-&gt;mtr);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_mtt_init(hr_dev, buf-&gt;hr_buf.npages,</div><div class='del'>-				buf-&gt;hr_buf.page_shift, &amp;buf-&gt;hr_mtt);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_buf;</div><div class='add'>+static int alloc_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,</div><div class='add'>+		       struct ib_udata *udata, unsigned long addr,</div><div class='add'>+		       struct hns_roce_ib_create_cq_resp *resp)</div><div class='add'>+{</div><div class='add'>+	bool has_db = hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_CQ_RECORD_DB;</div><div class='add'>+	struct hns_roce_ucontext *uctx;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_buf_write_mtt(hr_dev, &amp;buf-&gt;hr_mtt, &amp;buf-&gt;hr_buf);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_mtt;</div><div class='add'>+	if (udata) {</div><div class='add'>+		if (has_db &amp;&amp;</div><div class='add'>+		    udata-&gt;outlen &gt;= offsetofend(typeof(*resp), cap_flags)) {</div><div class='add'>+			uctx = rdma_udata_to_drv_context(udata,</div><div class='add'>+					struct hns_roce_ucontext, ibucontext);</div><div class='add'>+			err = hns_roce_db_map_user(uctx, addr, &amp;hr_cq-&gt;db);</div><div class='add'>+			if (err)</div><div class='add'>+				return err;</div><div class='add'>+			hr_cq-&gt;flags |= HNS_ROCE_CQ_FLAG_RECORD_DB;</div><div class='add'>+			resp-&gt;cap_flags |= HNS_ROCE_CQ_FLAG_RECORD_DB;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		if (has_db) {</div><div class='add'>+			err = hns_roce_alloc_db(hr_dev, &amp;hr_cq-&gt;db, 1);</div><div class='add'>+			if (err)</div><div class='add'>+				return err;</div><div class='add'>+			hr_cq-&gt;set_ci_db = hr_cq-&gt;db.db_record;</div><div class='add'>+			*hr_cq-&gt;set_ci_db = 0;</div><div class='add'>+			hr_cq-&gt;flags |= HNS_ROCE_CQ_FLAG_RECORD_DB;</div><div class='add'>+		}</div><div class='add'>+		hr_cq-&gt;db_reg = hr_dev-&gt;reg_base + hr_dev-&gt;odb_offset +</div><div class='add'>+				DB_REG_OFFSET * hr_dev-&gt;priv_uar.index;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-err_mtt:</div><div class='del'>-	hns_roce_mtt_cleanup(hr_dev, &amp;buf-&gt;hr_mtt);</div><div class='add'>+static void free_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,</div><div class='add'>+		       struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_ucontext *uctx;</div><div class='ctx'> </div><div class='del'>-err_buf:</div><div class='del'>-	hns_roce_buf_free(hr_dev, nent * hr_dev-&gt;caps.cq_entry_sz,</div><div class='del'>-			  &amp;buf-&gt;hr_buf);</div><div class='del'>-out:</div><div class='del'>-	return ret;</div><div class='add'>+	if (!(hr_cq-&gt;flags &amp; HNS_ROCE_CQ_FLAG_RECORD_DB))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	hr_cq-&gt;flags &amp;= ~HNS_ROCE_CQ_FLAG_RECORD_DB;</div><div class='add'>+	if (udata) {</div><div class='add'>+		uctx = rdma_udata_to_drv_context(udata,</div><div class='add'>+						 struct hns_roce_ucontext,</div><div class='add'>+						 ibucontext);</div><div class='add'>+		hns_roce_db_unmap_user(uctx, &amp;hr_cq-&gt;db);</div><div class='add'>+	} else {</div><div class='add'>+		hns_roce_free_db(hr_dev, &amp;hr_cq-&gt;db);</div><div class='add'>+	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_ib_free_cq_buf(struct hns_roce_dev *hr_dev,</div><div class='del'>-				    struct hns_roce_cq_buf *buf, int cqe)</div><div class='add'>+static int verify_cq_create_attr(struct hns_roce_dev *hr_dev,</div><div class='add'>+				 const struct ib_cq_init_attr *attr)</div><div class='ctx'> {</div><div class='del'>-	hns_roce_buf_free(hr_dev, (cqe + 1) * hr_dev-&gt;caps.cq_entry_sz,</div><div class='del'>-			  &amp;buf-&gt;hr_buf);</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+</div><div class='add'>+	if (!attr-&gt;cqe || attr-&gt;cqe &gt; hr_dev-&gt;caps.max_cqes) {</div><div class='add'>+		ibdev_err(ibdev, "failed to check CQ count %u, max = %u.\n",</div><div class='add'>+			  attr-&gt;cqe, hr_dev-&gt;caps.max_cqes);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;comp_vector &gt;= hr_dev-&gt;caps.num_comp_vectors) {</div><div class='add'>+		ibdev_err(ibdev, "failed to check CQ vector = %u, max = %d.\n",</div><div class='add'>+			  attr-&gt;comp_vector, hr_dev-&gt;caps.num_comp_vectors);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,</div><div class='del'>-				    const struct ib_cq_init_attr *attr,</div><div class='del'>-				    struct ib_udata *udata)</div><div class='add'>+static int get_cq_ucmd(struct hns_roce_cq *hr_cq, struct ib_udata *udata,</div><div class='add'>+		       struct hns_roce_ib_create_cq *ucmd)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	struct hns_roce_ib_create_cq ucmd;</div><div class='del'>-	struct hns_roce_ib_create_cq_resp resp = {};</div><div class='del'>-	struct hns_roce_cq *hr_cq = NULL;</div><div class='del'>-	struct hns_roce_uar *uar = NULL;</div><div class='del'>-	int vector = attr-&gt;comp_vector;</div><div class='del'>-	int cq_entries = attr-&gt;cqe;</div><div class='add'>+	struct ib_device *ibdev = hr_cq-&gt;ib_cq.device;</div><div class='ctx'> 	int ret;</div><div class='del'>-	struct hns_roce_ucontext *context = rdma_udata_to_drv_context(</div><div class='del'>-		udata, struct hns_roce_ucontext, ibucontext);</div><div class='ctx'> </div><div class='del'>-	if (cq_entries &lt; 1 || cq_entries &gt; hr_dev-&gt;caps.max_cqes) {</div><div class='del'>-		dev_err(dev, "Creat CQ failed. entries=%d, max=%d\n",</div><div class='del'>-			cq_entries, hr_dev-&gt;caps.max_cqes);</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+	ret = ib_copy_from_udata(ucmd, udata, min(udata-&gt;inlen, sizeof(*ucmd)));</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to copy CQ udata, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	hr_cq = kzalloc(sizeof(*hr_cq), GFP_KERNEL);</div><div class='del'>-	if (!hr_cq)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void set_cq_param(struct hns_roce_cq *hr_cq, u32 cq_entries, int vector,</div><div class='add'>+			 struct hns_roce_ib_create_cq *ucmd)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq-&gt;ib_cq.device);</div><div class='ctx'> </div><div class='del'>-	if (hr_dev-&gt;caps.min_cqes)</div><div class='del'>-		cq_entries = max(cq_entries, hr_dev-&gt;caps.min_cqes);</div><div class='add'>+	cq_entries = max(cq_entries, hr_dev-&gt;caps.min_cqes);</div><div class='add'>+	cq_entries = roundup_pow_of_two(cq_entries);</div><div class='add'>+	hr_cq-&gt;ib_cq.cqe = cq_entries - 1; /* used as cqe index */</div><div class='add'>+	hr_cq-&gt;cq_depth = cq_entries;</div><div class='add'>+	hr_cq-&gt;vector = vector;</div><div class='ctx'> </div><div class='del'>-	cq_entries = roundup_pow_of_two((unsigned int)cq_entries);</div><div class='del'>-	hr_cq-&gt;ib_cq.cqe = cq_entries - 1;</div><div class='ctx'> 	spin_lock_init(&amp;hr_cq-&gt;lock);</div><div class='add'>+	INIT_LIST_HEAD(&amp;hr_cq-&gt;sq_list);</div><div class='add'>+	INIT_LIST_HEAD(&amp;hr_cq-&gt;rq_list);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (udata) {</div><div class='del'>-		if (ib_copy_from_udata(&amp;ucmd, udata, sizeof(ucmd))) {</div><div class='del'>-			dev_err(dev, "Failed to copy_from_udata.\n");</div><div class='del'>-			ret = -EFAULT;</div><div class='del'>-			goto err_cq;</div><div class='del'>-		}</div><div class='add'>+static int set_cqe_size(struct hns_roce_cq *hr_cq, struct ib_udata *udata,</div><div class='add'>+			struct hns_roce_ib_create_cq *ucmd)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq-&gt;ib_cq.device);</div><div class='ctx'> </div><div class='del'>-		/* Get user space address, write it into mtt table */</div><div class='del'>-		ret = hns_roce_ib_get_cq_umem(hr_dev, udata, &amp;hr_cq-&gt;hr_buf,</div><div class='del'>-					      &amp;hr_cq-&gt;umem, ucmd.buf_addr,</div><div class='del'>-					      cq_entries);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "Failed to get_cq_umem.\n");</div><div class='del'>-			goto err_cq;</div><div class='del'>-		}</div><div class='add'>+	if (!udata) {</div><div class='add'>+		hr_cq-&gt;cqe_size = hr_dev-&gt;caps.cqe_sz;</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		if ((hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RECORD_DB) &amp;&amp;</div><div class='del'>-		    (udata-&gt;outlen &gt;= sizeof(resp))) {</div><div class='del'>-			ret = hns_roce_db_map_user(context, udata, ucmd.db_addr,</div><div class='del'>-						   &amp;hr_cq-&gt;db);</div><div class='del'>-			if (ret) {</div><div class='del'>-				dev_err(dev, "cq record doorbell map failed!\n");</div><div class='del'>-				goto err_mtt;</div><div class='del'>-			}</div><div class='del'>-			hr_cq-&gt;db_en = 1;</div><div class='del'>-			resp.cap_flags |= HNS_ROCE_SUPPORT_CQ_RECORD_DB;</div><div class='add'>+	if (udata-&gt;inlen &gt;= offsetofend(typeof(*ucmd), cqe_size)) {</div><div class='add'>+		if (ucmd-&gt;cqe_size != HNS_ROCE_V2_CQE_SIZE &amp;&amp;</div><div class='add'>+		    ucmd-&gt;cqe_size != HNS_ROCE_V3_CQE_SIZE) {</div><div class='add'>+			ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+				  "invalid cqe size %u.\n", ucmd-&gt;cqe_size);</div><div class='add'>+			return -EINVAL;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		/* Get user space parameters */</div><div class='del'>-		uar = &amp;context-&gt;uar;</div><div class='add'>+		hr_cq-&gt;cqe_size = ucmd-&gt;cqe_size;</div><div class='ctx'> 	} else {</div><div class='del'>-		if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RECORD_DB) {</div><div class='del'>-			ret = hns_roce_alloc_db(hr_dev, &amp;hr_cq-&gt;db, 1);</div><div class='del'>-			if (ret)</div><div class='del'>-				goto err_cq;</div><div class='add'>+		hr_cq-&gt;cqe_size = HNS_ROCE_V2_CQE_SIZE;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-			hr_cq-&gt;set_ci_db = hr_cq-&gt;db.db_record;</div><div class='del'>-			*hr_cq-&gt;set_ci_db = 0;</div><div class='del'>-			hr_cq-&gt;db_en = 1;</div><div class='del'>-		}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		/* Init mmt table and write buff address to mtt table */</div><div class='del'>-		ret = hns_roce_ib_alloc_cq_buf(hr_dev, &amp;hr_cq-&gt;hr_buf,</div><div class='del'>-					       cq_entries);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "Failed to alloc_cq_buf.\n");</div><div class='del'>-			goto err_db;</div><div class='del'>-		}</div><div class='add'>+int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		       struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq-&gt;device);</div><div class='add'>+	struct ib_udata *udata = &amp;attrs-&gt;driver_udata;</div><div class='add'>+	struct hns_roce_ib_create_cq_resp resp = {};</div><div class='add'>+	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	struct hns_roce_ib_create_cq ucmd = {};</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;flags) {</div><div class='add'>+		ret = -EOPNOTSUPP;</div><div class='add'>+		goto err_out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = verify_cq_create_attr(hr_dev, attr);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_out;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		ret = get_cq_ucmd(hr_cq, udata, &amp;ucmd);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_out;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		uar = &amp;hr_dev-&gt;priv_uar;</div><div class='del'>-		hr_cq-&gt;cq_db_l = hr_dev-&gt;reg_base + hr_dev-&gt;odb_offset +</div><div class='del'>-				DB_REG_OFFSET * uar-&gt;index;</div><div class='add'>+	set_cq_param(hr_cq, attr-&gt;cqe, attr-&gt;comp_vector, &amp;ucmd);</div><div class='add'>+</div><div class='add'>+	ret = set_cqe_size(hr_cq, udata, &amp;ucmd);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_out;</div><div class='add'>+</div><div class='add'>+	ret = alloc_cq_buf(hr_dev, hr_cq, udata, ucmd.buf_addr);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to alloc CQ buf, ret = %d.\n", ret);</div><div class='add'>+		goto err_out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* Allocate cq index, fill cq_context */</div><div class='del'>-	ret = hns_roce_cq_alloc(hr_dev, cq_entries, &amp;hr_cq-&gt;hr_buf.hr_mtt, uar,</div><div class='del'>-				hr_cq, vector);</div><div class='add'>+	ret = alloc_cq_db(hr_dev, hr_cq, udata, ucmd.db_addr, &amp;resp);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "Creat CQ .Failed to cq_alloc.\n");</div><div class='del'>-		goto err_dbmap;</div><div class='add'>+		ibdev_err(ibdev, "failed to alloc CQ db, ret = %d.\n", ret);</div><div class='add'>+		goto err_cq_buf;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * For the QP created by kernel space, tptr value should be initialized</div><div class='del'>-	 * to zero; For the QP created by user space, it will cause synchronous</div><div class='del'>-	 * problems if tptr is set to zero here, so we initialze it in user</div><div class='del'>-	 * space.</div><div class='del'>-	 */</div><div class='del'>-	if (!udata &amp;&amp; hr_cq-&gt;tptr_addr)</div><div class='del'>-		*hr_cq-&gt;tptr_addr = 0;</div><div class='del'>-</div><div class='del'>-	/* Get created cq handler and carry out event */</div><div class='del'>-	hr_cq-&gt;comp = hns_roce_ib_cq_comp;</div><div class='del'>-	hr_cq-&gt;event = hns_roce_ib_cq_event;</div><div class='del'>-	hr_cq-&gt;cq_depth = cq_entries;</div><div class='add'>+	ret = alloc_cqn(hr_dev, hr_cq, udata);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to alloc CQN, ret = %d.\n", ret);</div><div class='add'>+		goto err_cq_db;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = alloc_cqc(hr_dev, hr_cq);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev,</div><div class='add'>+			  "failed to alloc CQ context, ret = %d.\n", ret);</div><div class='add'>+		goto err_cqn;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	if (udata) {</div><div class='ctx'> 		resp.cqn = hr_cq-&gt;cqn;</div><div class='del'>-		ret = ib_copy_to_udata(udata, &amp;resp, sizeof(resp));</div><div class='add'>+		ret = ib_copy_to_udata(udata, &amp;resp,</div><div class='add'>+				       min(udata-&gt;outlen, sizeof(resp)));</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto err_cqc;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return &amp;hr_cq-&gt;ib_cq;</div><div class='add'>+	hr_cq-&gt;cons_index = 0;</div><div class='add'>+	hr_cq-&gt;arm_sn = 1;</div><div class='add'>+	refcount_set(&amp;hr_cq-&gt;refcount, 1);</div><div class='add'>+	init_completion(&amp;hr_cq-&gt;free);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='ctx'> err_cqc:</div><div class='del'>-	hns_roce_free_cq(hr_dev, hr_cq);</div><div class='del'>-</div><div class='del'>-err_dbmap:</div><div class='del'>-	if (udata &amp;&amp; (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RECORD_DB) &amp;&amp;</div><div class='del'>-	    (udata-&gt;outlen &gt;= sizeof(resp)))</div><div class='del'>-		hns_roce_db_unmap_user(context, &amp;hr_cq-&gt;db);</div><div class='del'>-</div><div class='del'>-err_mtt:</div><div class='del'>-	hns_roce_mtt_cleanup(hr_dev, &amp;hr_cq-&gt;hr_buf.hr_mtt);</div><div class='del'>-	if (udata)</div><div class='del'>-		ib_umem_release(hr_cq-&gt;umem);</div><div class='del'>-	else</div><div class='del'>-		hns_roce_ib_free_cq_buf(hr_dev, &amp;hr_cq-&gt;hr_buf,</div><div class='del'>-					hr_cq-&gt;ib_cq.cqe);</div><div class='del'>-</div><div class='del'>-err_db:</div><div class='del'>-	if (!udata &amp;&amp; (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RECORD_DB))</div><div class='del'>-		hns_roce_free_db(hr_dev, &amp;hr_cq-&gt;db);</div><div class='add'>+	free_cqc(hr_dev, hr_cq);</div><div class='add'>+err_cqn:</div><div class='add'>+	free_cqn(hr_dev, hr_cq-&gt;cqn);</div><div class='add'>+err_cq_db:</div><div class='add'>+	free_cq_db(hr_dev, hr_cq, udata);</div><div class='add'>+err_cq_buf:</div><div class='add'>+	free_cq_buf(hr_dev, hr_cq);</div><div class='add'>+err_out:</div><div class='add'>+	atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_CQ_CREATE_ERR_CNT]);</div><div class='ctx'> </div><div class='del'>-err_cq:</div><div class='del'>-	kfree(hr_cq);</div><div class='del'>-	return ERR_PTR(ret);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_ib_create_cq);</div><div class='ctx'> </div><div class='del'>-int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)</div><div class='add'>+int hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq-&gt;device);</div><div class='ctx'> 	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	if (hr_dev-&gt;hw-&gt;destroy_cq) {</div><div class='del'>-		ret = hr_dev-&gt;hw-&gt;destroy_cq(ib_cq, udata);</div><div class='del'>-	} else {</div><div class='del'>-		hns_roce_free_cq(hr_dev, hr_cq);</div><div class='del'>-		hns_roce_mtt_cleanup(hr_dev, &amp;hr_cq-&gt;hr_buf.hr_mtt);</div><div class='del'>-</div><div class='del'>-		if (udata) {</div><div class='del'>-			ib_umem_release(hr_cq-&gt;umem);</div><div class='del'>-</div><div class='del'>-			if (hr_cq-&gt;db_en == 1)</div><div class='del'>-				hns_roce_db_unmap_user(</div><div class='del'>-					rdma_udata_to_drv_context(</div><div class='del'>-						udata,</div><div class='del'>-						struct hns_roce_ucontext,</div><div class='del'>-						ibucontext),</div><div class='del'>-					&amp;hr_cq-&gt;db);</div><div class='del'>-		} else {</div><div class='del'>-			/* Free the buff of stored cq */</div><div class='del'>-			hns_roce_ib_free_cq_buf(hr_dev, &amp;hr_cq-&gt;hr_buf,</div><div class='del'>-						ib_cq-&gt;cqe);</div><div class='del'>-			if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RECORD_DB)</div><div class='del'>-				hns_roce_free_db(hr_dev, &amp;hr_cq-&gt;db);</div><div class='del'>-		}</div><div class='ctx'> </div><div class='del'>-		kfree(hr_cq);</div><div class='del'>-	}</div><div class='add'>+	free_cqc(hr_dev, hr_cq);</div><div class='add'>+	free_cqn(hr_dev, hr_cq-&gt;cqn);</div><div class='add'>+	free_cq_db(hr_dev, hr_cq, udata);</div><div class='add'>+	free_cq_buf(hr_dev, hr_cq);</div><div class='ctx'> </div><div class='del'>-	return ret;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_ib_destroy_cq);</div><div class='ctx'> </div><div class='ctx'> void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	struct hns_roce_cq *cq;</div><div class='add'>+	struct hns_roce_cq *hr_cq;</div><div class='add'>+	struct ib_cq *ibcq;</div><div class='ctx'> </div><div class='del'>-	cq = xa_load(&amp;hr_dev-&gt;cq_table.array, cqn &amp; (hr_dev-&gt;caps.num_cqs - 1));</div><div class='del'>-	if (!cq) {</div><div class='del'>-		dev_warn(dev, "Completion event for bogus CQ 0x%08x\n", cqn);</div><div class='add'>+	hr_cq = xa_load(&amp;hr_dev-&gt;cq_table.array,</div><div class='add'>+			cqn &amp; (hr_dev-&gt;caps.num_cqs - 1));</div><div class='add'>+	if (!hr_cq) {</div><div class='add'>+		dev_warn(hr_dev-&gt;dev, "completion event for bogus CQ 0x%06x\n",</div><div class='add'>+			 cqn);</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	++cq-&gt;arm_sn;</div><div class='del'>-	cq-&gt;comp(cq);</div><div class='add'>+	++hr_cq-&gt;arm_sn;</div><div class='add'>+	ibcq = &amp;hr_cq-&gt;ib_cq;</div><div class='add'>+	if (ibcq-&gt;comp_handler)</div><div class='add'>+		ibcq-&gt;comp_handler(ibcq, ibcq-&gt;cq_context);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_cq_completion);</div><div class='ctx'> </div><div class='ctx'> void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_cq_table *cq_table = &amp;hr_dev-&gt;cq_table;</div><div class='ctx'> 	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	struct hns_roce_cq *cq;</div><div class='add'>+	struct hns_roce_cq *hr_cq;</div><div class='add'>+	struct ib_event event;</div><div class='add'>+	struct ib_cq *ibcq;</div><div class='ctx'> </div><div class='del'>-	cq = xa_load(&amp;cq_table-&gt;array, cqn &amp; (hr_dev-&gt;caps.num_cqs - 1));</div><div class='del'>-	if (cq)</div><div class='del'>-		atomic_inc(&amp;cq-&gt;refcount);</div><div class='add'>+	if (event_type != HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID &amp;&amp;</div><div class='add'>+	    event_type != HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR &amp;&amp;</div><div class='add'>+	    event_type != HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW) {</div><div class='add'>+		dev_err(dev, "unexpected event type 0x%x on CQ 0x%06x\n",</div><div class='add'>+			event_type, cqn);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (!cq) {</div><div class='del'>-		dev_warn(dev, "Async event for bogus CQ %08x\n", cqn);</div><div class='add'>+	xa_lock(&amp;hr_dev-&gt;cq_table.array);</div><div class='add'>+	hr_cq = xa_load(&amp;hr_dev-&gt;cq_table.array,</div><div class='add'>+			cqn &amp; (hr_dev-&gt;caps.num_cqs - 1));</div><div class='add'>+	if (hr_cq)</div><div class='add'>+		refcount_inc(&amp;hr_cq-&gt;refcount);</div><div class='add'>+	xa_unlock(&amp;hr_dev-&gt;cq_table.array);</div><div class='add'>+	if (!hr_cq) {</div><div class='add'>+		dev_warn(dev, "async event for bogus CQ 0x%06x\n", cqn);</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	cq-&gt;event(cq, (enum hns_roce_event)event_type);</div><div class='add'>+	ibcq = &amp;hr_cq-&gt;ib_cq;</div><div class='add'>+	if (ibcq-&gt;event_handler) {</div><div class='add'>+		event.device = ibcq-&gt;device;</div><div class='add'>+		event.element.cq = ibcq;</div><div class='add'>+		event.event = IB_EVENT_CQ_ERR;</div><div class='add'>+		ibcq-&gt;event_handler(&amp;event, ibcq-&gt;cq_context);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (atomic_dec_and_test(&amp;cq-&gt;refcount))</div><div class='del'>-		complete(&amp;cq-&gt;free);</div><div class='add'>+	if (refcount_dec_and_test(&amp;hr_cq-&gt;refcount))</div><div class='add'>+		complete(&amp;hr_cq-&gt;free);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_cq_event);</div><div class='ctx'> </div><div class='del'>-int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev)</div><div class='add'>+void hns_roce_init_cq_table(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_cq_table *cq_table = &amp;hr_dev-&gt;cq_table;</div><div class='add'>+	unsigned int reserved_from_bot;</div><div class='add'>+	unsigned int i;</div><div class='ctx'> </div><div class='add'>+	mutex_init(&amp;cq_table-&gt;bank_mutex);</div><div class='ctx'> 	xa_init(&amp;cq_table-&gt;array);</div><div class='ctx'> </div><div class='del'>-	return hns_roce_bitmap_init(&amp;cq_table-&gt;bitmap, hr_dev-&gt;caps.num_cqs,</div><div class='del'>-				    hr_dev-&gt;caps.num_cqs - 1,</div><div class='del'>-				    hr_dev-&gt;caps.reserved_cqs, 0);</div><div class='add'>+	reserved_from_bot = hr_dev-&gt;caps.reserved_cqs;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; reserved_from_bot; i++) {</div><div class='add'>+		cq_table-&gt;bank[get_cq_bankid(i)].inuse++;</div><div class='add'>+		cq_table-&gt;bank[get_cq_bankid(i)].min++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; HNS_ROCE_CQ_BANK_NUM; i++) {</div><div class='add'>+		ida_init(&amp;cq_table-&gt;bank[i].ida);</div><div class='add'>+		cq_table-&gt;bank[i].max = hr_dev-&gt;caps.num_cqs /</div><div class='add'>+					HNS_ROCE_CQ_BANK_NUM - 1;</div><div class='add'>+	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	hns_roce_bitmap_cleanup(&amp;hr_dev-&gt;cq_table.bitmap);</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; HNS_ROCE_CQ_BANK_NUM; i++)</div><div class='add'>+		ida_destroy(&amp;hr_dev-&gt;cq_table.bank[i].ida);</div><div class='add'>+	xa_destroy(&amp;hr_dev-&gt;cq_table.array);</div><div class='add'>+	mutex_destroy(&amp;hr_dev-&gt;cq_table.bank_mutex);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_db.c b/drivers/infiniband/hw/hns/hns_roce_db.c<br/>index 0c6c1fe87705..5c4c0480832b 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_db.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_db.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_db.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_db.c</a></div><div class='hunk'>@@ -4,21 +4,21 @@</div><div class='ctx'>  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/platform_device.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_umem.h&gt;</div><div class='ctx'> #include "hns_roce_device.h"</div><div class='ctx'> </div><div class='del'>-int hns_roce_db_map_user(struct hns_roce_ucontext *context,</div><div class='del'>-			 struct ib_udata *udata, unsigned long virt,</div><div class='add'>+int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt,</div><div class='ctx'> 			 struct hns_roce_db *db)</div><div class='ctx'> {</div><div class='add'>+	unsigned long page_addr = virt &amp; PAGE_MASK;</div><div class='ctx'> 	struct hns_roce_user_db_page *page;</div><div class='add'>+	unsigned int offset;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;context-&gt;page_mutex);</div><div class='ctx'> </div><div class='ctx'> 	list_for_each_entry(page, &amp;context-&gt;page_list, list)</div><div class='del'>-		if (page-&gt;user_virt == (virt &amp; PAGE_MASK))</div><div class='add'>+		if (page-&gt;user_virt == page_addr)</div><div class='ctx'> 			goto found;</div><div class='ctx'> </div><div class='ctx'> 	page = kmalloc(sizeof(*page), GFP_KERNEL);</div><div class='hunk'>@@ -28,8 +28,9 @@ int hns_roce_db_map_user(struct hns_roce_ucontext *context,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	refcount_set(&amp;page-&gt;refcount, 1);</div><div class='del'>-	page-&gt;user_virt = (virt &amp; PAGE_MASK);</div><div class='del'>-	page-&gt;umem = ib_umem_get(udata, virt &amp; PAGE_MASK, PAGE_SIZE, 0, 0);</div><div class='add'>+	page-&gt;user_virt = page_addr;</div><div class='add'>+	page-&gt;umem = ib_umem_get(context-&gt;ibucontext.device, page_addr,</div><div class='add'>+				 PAGE_SIZE, 0);</div><div class='ctx'> 	if (IS_ERR(page-&gt;umem)) {</div><div class='ctx'> 		ret = PTR_ERR(page-&gt;umem);</div><div class='ctx'> 		kfree(page);</div><div class='hunk'>@@ -39,10 +40,9 @@ int hns_roce_db_map_user(struct hns_roce_ucontext *context,</div><div class='ctx'> 	list_add(&amp;page-&gt;list, &amp;context-&gt;page_list);</div><div class='ctx'> </div><div class='ctx'> found:</div><div class='del'>-	db-&gt;dma = sg_dma_address(page-&gt;umem-&gt;sg_head.sgl) +</div><div class='del'>-		  (virt &amp; ~PAGE_MASK);</div><div class='del'>-	page-&gt;umem-&gt;sg_head.sgl-&gt;offset = virt &amp; ~PAGE_MASK;</div><div class='del'>-	db-&gt;virt_addr = sg_virt(page-&gt;umem-&gt;sg_head.sgl);</div><div class='add'>+	offset = virt - page_addr;</div><div class='add'>+	db-&gt;dma = sg_dma_address(page-&gt;umem-&gt;sgt_append.sgt.sgl) + offset;</div><div class='add'>+	db-&gt;virt_addr = sg_virt(page-&gt;umem-&gt;sgt_append.sgt.sgl) + offset;</div><div class='ctx'> 	db-&gt;u.user_page = page;</div><div class='ctx'> 	refcount_inc(&amp;page-&gt;refcount);</div><div class='ctx'> </div><div class='hunk'>@@ -51,7 +51,6 @@ out:</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(hns_roce_db_map_user);</div><div class='ctx'> </div><div class='ctx'> void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,</div><div class='ctx'> 			    struct hns_roce_db *db)</div><div class='hunk'>@@ -67,7 +66,6 @@ void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,</div><div class='ctx'> </div><div class='ctx'> 	mutex_unlock(&amp;context-&gt;page_mutex);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL(hns_roce_db_unmap_user);</div><div class='ctx'> </div><div class='ctx'> static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir(</div><div class='ctx'> 					struct device *dma_device)</div><div class='hunk'>@@ -78,7 +76,8 @@ static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir(</div><div class='ctx'> 	if (!pgdir)</div><div class='ctx'> 		return NULL;</div><div class='ctx'> </div><div class='del'>-	bitmap_fill(pgdir-&gt;order1, HNS_ROCE_DB_PER_PAGE / 2);</div><div class='add'>+	bitmap_fill(pgdir-&gt;order1,</div><div class='add'>+		    HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT);</div><div class='ctx'> 	pgdir-&gt;bits[0] = pgdir-&gt;order0;</div><div class='ctx'> 	pgdir-&gt;bits[1] = pgdir-&gt;order1;</div><div class='ctx'> 	pgdir-&gt;page = dma_alloc_coherent(dma_device, PAGE_SIZE,</div><div class='hunk'>@@ -94,8 +93,8 @@ static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir(</div><div class='ctx'> static int hns_roce_alloc_db_from_pgdir(struct hns_roce_db_pgdir *pgdir,</div><div class='ctx'> 					struct hns_roce_db *db, int order)</div><div class='ctx'> {</div><div class='del'>-	int o;</div><div class='del'>-	int i;</div><div class='add'>+	unsigned long o;</div><div class='add'>+	unsigned long i;</div><div class='ctx'> </div><div class='ctx'> 	for (o = order; o &lt;= 1; ++o) {</div><div class='ctx'> 		i = find_first_bit(pgdir-&gt;bits[o], HNS_ROCE_DB_PER_PAGE &gt;&gt; o);</div><div class='hunk'>@@ -116,7 +115,7 @@ found:</div><div class='ctx'> 	db-&gt;u.pgdir	= pgdir;</div><div class='ctx'> 	db-&gt;index	= i;</div><div class='ctx'> 	db-&gt;db_record	= pgdir-&gt;page + db-&gt;index;</div><div class='del'>-	db-&gt;dma		= pgdir-&gt;db_dma  + db-&gt;index * 4;</div><div class='add'>+	db-&gt;dma		= pgdir-&gt;db_dma  + db-&gt;index * HNS_ROCE_DB_UNIT_SIZE;</div><div class='ctx'> 	db-&gt;order	= order;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='hunk'>@@ -150,12 +149,11 @@ out:</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_alloc_db);</div><div class='ctx'> </div><div class='ctx'> void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db)</div><div class='ctx'> {</div><div class='del'>-	int o;</div><div class='del'>-	int i;</div><div class='add'>+	unsigned long o;</div><div class='add'>+	unsigned long i;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;hr_dev-&gt;pgdir_mutex);</div><div class='ctx'> </div><div class='hunk'>@@ -170,7 +168,8 @@ void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db)</div><div class='ctx'> 	i &gt;&gt;= o;</div><div class='ctx'> 	set_bit(i, db-&gt;u.pgdir-&gt;bits[o]);</div><div class='ctx'> </div><div class='del'>-	if (bitmap_full(db-&gt;u.pgdir-&gt;order1, HNS_ROCE_DB_PER_PAGE / 2)) {</div><div class='add'>+	if (bitmap_full(db-&gt;u.pgdir-&gt;order1,</div><div class='add'>+			HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT)) {</div><div class='ctx'> 		dma_free_coherent(hr_dev-&gt;dev, PAGE_SIZE, db-&gt;u.pgdir-&gt;page,</div><div class='ctx'> 				  db-&gt;u.pgdir-&gt;db_dma);</div><div class='ctx'> 		list_del(&amp;db-&gt;u.pgdir-&gt;list);</div><div class='hunk'>@@ -179,4 +178,3 @@ void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db)</div><div class='ctx'> </div><div class='ctx'> 	mutex_unlock(&amp;hr_dev-&gt;pgdir_mutex);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_free_db);</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.c b/drivers/infiniband/hw/hns/hns_roce_debugfs.c<br/>new file mode 100644<br/>index 000000000000..b869cdc54118<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_debugfs.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_debugfs.c</a></div><div class='hunk'>@@ -0,0 +1,111 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0+</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2023 Hisilicon Limited.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/debugfs.h&gt;</div><div class='add'>+#include &lt;linux/device.h&gt;</div><div class='add'>+#include &lt;linux/pci.h&gt;</div><div class='add'>+</div><div class='add'>+#include "hns_roce_device.h"</div><div class='add'>+</div><div class='add'>+static struct dentry *hns_roce_dbgfs_root;</div><div class='add'>+</div><div class='add'>+static int hns_debugfs_seqfile_open(struct inode *inode, struct file *f)</div><div class='add'>+{</div><div class='add'>+	struct hns_debugfs_seqfile *seqfile = inode-&gt;i_private;</div><div class='add'>+</div><div class='add'>+	return single_open(f, seqfile-&gt;read, seqfile-&gt;data);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct file_operations hns_debugfs_seqfile_fops = {</div><div class='add'>+	.owner = THIS_MODULE,</div><div class='add'>+	.open = hns_debugfs_seqfile_open,</div><div class='add'>+	.release = single_release,</div><div class='add'>+	.read = seq_read,</div><div class='add'>+	.llseek = seq_lseek</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static void init_debugfs_seqfile(struct hns_debugfs_seqfile *seq,</div><div class='add'>+				 const char *name, struct dentry *parent,</div><div class='add'>+				 int (*read_fn)(struct seq_file *, void *),</div><div class='add'>+				 void *data)</div><div class='add'>+{</div><div class='add'>+	debugfs_create_file(name, 0400, parent, seq, &amp;hns_debugfs_seqfile_fops);</div><div class='add'>+</div><div class='add'>+	seq-&gt;read = read_fn;</div><div class='add'>+	seq-&gt;data = data;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const char * const sw_stat_info[] = {</div><div class='add'>+	[HNS_ROCE_DFX_AEQE_CNT] = "aeqe",</div><div class='add'>+	[HNS_ROCE_DFX_CEQE_CNT] = "ceqe",</div><div class='add'>+	[HNS_ROCE_DFX_CMDS_CNT] = "cmds",</div><div class='add'>+	[HNS_ROCE_DFX_CMDS_ERR_CNT] = "cmds_err",</div><div class='add'>+	[HNS_ROCE_DFX_MBX_POSTED_CNT] = "posted_mbx",</div><div class='add'>+	[HNS_ROCE_DFX_MBX_POLLED_CNT] = "polled_mbx",</div><div class='add'>+	[HNS_ROCE_DFX_MBX_EVENT_CNT] = "mbx_event",</div><div class='add'>+	[HNS_ROCE_DFX_QP_CREATE_ERR_CNT] = "qp_create_err",</div><div class='add'>+	[HNS_ROCE_DFX_QP_MODIFY_ERR_CNT] = "qp_modify_err",</div><div class='add'>+	[HNS_ROCE_DFX_CQ_CREATE_ERR_CNT] = "cq_create_err",</div><div class='add'>+	[HNS_ROCE_DFX_CQ_MODIFY_ERR_CNT] = "cq_modify_err",</div><div class='add'>+	[HNS_ROCE_DFX_SRQ_CREATE_ERR_CNT] = "srq_create_err",</div><div class='add'>+	[HNS_ROCE_DFX_SRQ_MODIFY_ERR_CNT] = "srq_modify_err",</div><div class='add'>+	[HNS_ROCE_DFX_XRCD_ALLOC_ERR_CNT] = "xrcd_alloc_err",</div><div class='add'>+	[HNS_ROCE_DFX_MR_REG_ERR_CNT] = "mr_reg_err",</div><div class='add'>+	[HNS_ROCE_DFX_MR_REREG_ERR_CNT] = "mr_rereg_err",</div><div class='add'>+	[HNS_ROCE_DFX_AH_CREATE_ERR_CNT] = "ah_create_err",</div><div class='add'>+	[HNS_ROCE_DFX_MMAP_ERR_CNT] = "mmap_err",</div><div class='add'>+	[HNS_ROCE_DFX_UCTX_ALLOC_ERR_CNT] = "uctx_alloc_err",</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int sw_stat_debugfs_show(struct seq_file *file, void *offset)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = file-&gt;private;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; HNS_ROCE_DFX_CNT_TOTAL; i++)</div><div class='add'>+		seq_printf(file, "%-20s --- %lld\n", sw_stat_info[i],</div><div class='add'>+			   atomic64_read(&amp;hr_dev-&gt;dfx_cnt[i]));</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void create_sw_stat_debugfs(struct hns_roce_dev *hr_dev,</div><div class='add'>+				   struct dentry *parent)</div><div class='add'>+{</div><div class='add'>+	struct hns_sw_stat_debugfs *dbgfs = &amp;hr_dev-&gt;dbgfs.sw_stat_root;</div><div class='add'>+</div><div class='add'>+	dbgfs-&gt;root = debugfs_create_dir("sw_stat", parent);</div><div class='add'>+</div><div class='add'>+	init_debugfs_seqfile(&amp;dbgfs-&gt;sw_stat, "sw_stat", dbgfs-&gt;root,</div><div class='add'>+			     sw_stat_debugfs_show, hr_dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* debugfs for device */</div><div class='add'>+void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev_debugfs *dbgfs = &amp;hr_dev-&gt;dbgfs;</div><div class='add'>+</div><div class='add'>+	dbgfs-&gt;root = debugfs_create_dir(pci_name(hr_dev-&gt;pci_dev),</div><div class='add'>+					 hns_roce_dbgfs_root);</div><div class='add'>+</div><div class='add'>+	create_sw_stat_debugfs(hr_dev, dbgfs-&gt;root);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hns_roce_unregister_debugfs(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	debugfs_remove_recursive(hr_dev-&gt;dbgfs.root);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* debugfs for hns module */</div><div class='add'>+void hns_roce_init_debugfs(void)</div><div class='add'>+{</div><div class='add'>+	hns_roce_dbgfs_root = debugfs_create_dir("hns_roce", NULL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hns_roce_cleanup_debugfs(void)</div><div class='add'>+{</div><div class='add'>+	debugfs_remove_recursive(hns_roce_dbgfs_root);</div><div class='add'>+	hns_roce_dbgfs_root = NULL;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.h b/drivers/infiniband/hw/hns/hns_roce_debugfs.h<br/>new file mode 100644<br/>index 000000000000..98e87bd3161e<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_debugfs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_debugfs.h</a></div><div class='hunk'>@@ -0,0 +1,33 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0+ */</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2023 Hisilicon Limited.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef __HNS_ROCE_DEBUGFS_H</div><div class='add'>+#define __HNS_ROCE_DEBUGFS_H</div><div class='add'>+</div><div class='add'>+/* debugfs seqfile */</div><div class='add'>+struct hns_debugfs_seqfile {</div><div class='add'>+	int (*read)(struct seq_file *seq, void *data);</div><div class='add'>+	void *data;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct hns_sw_stat_debugfs {</div><div class='add'>+	struct dentry *root;</div><div class='add'>+	struct hns_debugfs_seqfile sw_stat;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* Debugfs for device */</div><div class='add'>+struct hns_roce_dev_debugfs {</div><div class='add'>+	struct dentry *root;</div><div class='add'>+	struct hns_sw_stat_debugfs sw_stat_root;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct hns_roce_dev;</div><div class='add'>+</div><div class='add'>+void hns_roce_init_debugfs(void);</div><div class='add'>+void hns_roce_cleanup_debugfs(void);</div><div class='add'>+void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev);</div><div class='add'>+void hns_roce_unregister_debugfs(struct hns_roce_dev *hr_dev);</div><div class='add'>+</div><div class='add'>+#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h<br/>index 563cf39df6d5..318f18cf37aa 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_device.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_device.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_device.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_device.h</a></div><div class='hunk'>@@ -33,104 +33,86 @@</div><div class='ctx'> #ifndef _HNS_ROCE_DEVICE_H</div><div class='ctx'> #define _HNS_ROCE_DEVICE_H</div><div class='ctx'> </div><div class='add'>+#include &lt;linux/pci.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+#include &lt;rdma/hns-abi.h&gt;</div><div class='add'>+#include "hns_roce_debugfs.h"</div><div class='ctx'> </div><div class='del'>-#define DRV_NAME "hns_roce"</div><div class='add'>+#define PCI_REVISION_ID_HIP08			0x21</div><div class='add'>+#define PCI_REVISION_ID_HIP09			0x30</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_HW_VER1	('h' &lt;&lt; 24 | 'i' &lt;&lt; 16 | '0' &lt;&lt; 8 | '6')</div><div class='del'>-</div><div class='del'>-#define MAC_ADDR_OCTET_NUM			6</div><div class='ctx'> #define HNS_ROCE_MAX_MSG_LEN			0x80000000</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_ALOGN_UP(a, b) ((((a) + (b) - 1) / (b)) * (b))</div><div class='del'>-</div><div class='ctx'> #define HNS_ROCE_IB_MIN_SQ_STRIDE		6</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_BA_SIZE			(32 * 4096)</div><div class='add'>+#define BA_BYTE_LEN				8</div><div class='ctx'> </div><div class='del'>-/* Hardware specification only for v1 engine */</div><div class='ctx'> #define HNS_ROCE_MIN_CQE_NUM			0x40</div><div class='del'>-#define HNS_ROCE_MIN_WQE_NUM			0x20</div><div class='del'>-</div><div class='del'>-/* Hardware specification only for v1 engine */</div><div class='del'>-#define HNS_ROCE_MAX_INNER_MTPT_NUM		0x7</div><div class='del'>-#define HNS_ROCE_MAX_MTPT_PBL_NUM		0x100000</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS	20</div><div class='del'>-#define HNS_ROCE_MAX_FREE_CQ_WAIT_CNT	\</div><div class='del'>-	(5000 / HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS)</div><div class='del'>-#define HNS_ROCE_CQE_WCMD_EMPTY_BIT		0x2</div><div class='del'>-#define HNS_ROCE_MIN_CQE_CNT			16</div><div class='add'>+#define HNS_ROCE_MIN_SRQ_WQE_NUM		1</div><div class='ctx'> </div><div class='ctx'> #define HNS_ROCE_MAX_IRQ_NUM			128</div><div class='ctx'> </div><div class='add'>+#define HNS_ROCE_SGE_IN_WQE			2</div><div class='add'>+#define HNS_ROCE_SGE_SHIFT			4</div><div class='add'>+</div><div class='ctx'> #define EQ_ENABLE				1</div><div class='ctx'> #define EQ_DISABLE				0</div><div class='ctx'> </div><div class='ctx'> #define HNS_ROCE_CEQ				0</div><div class='ctx'> #define HNS_ROCE_AEQ				1</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_CEQ_ENTRY_SIZE			0x4</div><div class='del'>-#define HNS_ROCE_AEQ_ENTRY_SIZE			0x10</div><div class='add'>+#define HNS_ROCE_CEQE_SIZE 0x4</div><div class='add'>+#define HNS_ROCE_AEQE_SIZE 0x10</div><div class='add'>+</div><div class='add'>+#define HNS_ROCE_V3_EQE_SIZE 0x40</div><div class='ctx'> </div><div class='del'>-/* 4G/4K = 1M */</div><div class='del'>-#define HNS_ROCE_SL_SHIFT			28</div><div class='del'>-#define HNS_ROCE_TCLASS_SHIFT			20</div><div class='del'>-#define HNS_ROCE_FLOW_LABEL_MASK		0xfffff</div><div class='add'>+#define HNS_ROCE_V2_CQE_SIZE 32</div><div class='add'>+#define HNS_ROCE_V3_CQE_SIZE 64</div><div class='add'>+</div><div class='add'>+#define HNS_ROCE_V2_QPC_SZ 256</div><div class='add'>+#define HNS_ROCE_V3_QPC_SZ 512</div><div class='ctx'> </div><div class='ctx'> #define HNS_ROCE_MAX_PORTS			6</div><div class='del'>-#define HNS_ROCE_MAX_GID_NUM			16</div><div class='ctx'> #define HNS_ROCE_GID_SIZE			16</div><div class='add'>+#define HNS_ROCE_SGE_SIZE			16</div><div class='add'>+#define HNS_ROCE_DWQE_SIZE			65536</div><div class='ctx'> </div><div class='ctx'> #define HNS_ROCE_HOP_NUM_0			0xff</div><div class='ctx'> </div><div class='del'>-#define BITMAP_NO_RR				0</div><div class='del'>-#define BITMAP_RR				1</div><div class='del'>-</div><div class='ctx'> #define MR_TYPE_MR				0x00</div><div class='ctx'> #define MR_TYPE_FRMR				0x01</div><div class='ctx'> #define MR_TYPE_DMA				0x03</div><div class='ctx'> </div><div class='ctx'> #define HNS_ROCE_FRMR_MAX_PA			512</div><div class='add'>+#define HNS_ROCE_FRMR_ALIGN_SIZE		128</div><div class='ctx'> </div><div class='ctx'> #define PKEY_ID					0xffff</div><div class='del'>-#define GUID_LEN				8</div><div class='ctx'> #define NODE_DESC_SIZE				64</div><div class='ctx'> #define DB_REG_OFFSET				0x1000</div><div class='ctx'> </div><div class='del'>-#define SERV_TYPE_RC				0</div><div class='del'>-#define SERV_TYPE_RD				1</div><div class='del'>-#define SERV_TYPE_UC				2</div><div class='del'>-#define SERV_TYPE_UD				3</div><div class='del'>-</div><div class='ctx'> /* Configure to HW for PAGE_SIZE larger than 4KB */</div><div class='ctx'> #define PG_SHIFT_OFFSET				(PAGE_SHIFT - 12)</div><div class='ctx'> </div><div class='del'>-#define PAGES_SHIFT_8				8</div><div class='del'>-#define PAGES_SHIFT_16				16</div><div class='del'>-#define PAGES_SHIFT_24				24</div><div class='del'>-#define PAGES_SHIFT_32				32</div><div class='add'>+#define ATOMIC_WR_LEN				8</div><div class='ctx'> </div><div class='ctx'> #define HNS_ROCE_IDX_QUE_ENTRY_SZ		4</div><div class='ctx'> #define SRQ_DB_REG				0x230</div><div class='ctx'> </div><div class='del'>-enum {</div><div class='del'>-	HNS_ROCE_SUPPORT_RQ_RECORD_DB = 1 &lt;&lt; 0,</div><div class='del'>-	HNS_ROCE_SUPPORT_SQ_RECORD_DB = 1 &lt;&lt; 1,</div><div class='del'>-};</div><div class='add'>+#define HNS_ROCE_QP_BANK_NUM 8</div><div class='add'>+#define HNS_ROCE_CQ_BANK_NUM 4</div><div class='ctx'> </div><div class='del'>-enum {</div><div class='del'>-	HNS_ROCE_SUPPORT_CQ_RECORD_DB = 1 &lt;&lt; 0,</div><div class='del'>-};</div><div class='add'>+#define CQ_BANKID_SHIFT 2</div><div class='add'>+#define CQ_BANKID_MASK GENMASK(1, 0)</div><div class='add'>+</div><div class='add'>+#define HNS_ROCE_MAX_CQ_COUNT 0xFFFF</div><div class='add'>+#define HNS_ROCE_MAX_CQ_PERIOD 0xFFFF</div><div class='ctx'> </div><div class='del'>-enum hns_roce_qp_state {</div><div class='del'>-	HNS_ROCE_QP_STATE_RST,</div><div class='del'>-	HNS_ROCE_QP_STATE_INIT,</div><div class='del'>-	HNS_ROCE_QP_STATE_RTR,</div><div class='del'>-	HNS_ROCE_QP_STATE_RTS,</div><div class='del'>-	HNS_ROCE_QP_STATE_SQD,</div><div class='del'>-	HNS_ROCE_QP_STATE_ERR,</div><div class='del'>-	HNS_ROCE_QP_NUM_STATE,</div><div class='add'>+enum {</div><div class='add'>+	SERV_TYPE_RC,</div><div class='add'>+	SERV_TYPE_UC,</div><div class='add'>+	SERV_TYPE_RD,</div><div class='add'>+	SERV_TYPE_UD,</div><div class='add'>+	SERV_TYPE_XRC = 5,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum hns_roce_event {</div><div class='hunk'>@@ -151,67 +133,33 @@ enum hns_roce_event {</div><div class='ctx'> 	/* 0x10 and 0x11 is unused in currently application case */</div><div class='ctx'> 	HNS_ROCE_EVENT_TYPE_DB_OVERFLOW               = 0x12,</div><div class='ctx'> 	HNS_ROCE_EVENT_TYPE_MB                        = 0x13,</div><div class='del'>-	HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW              = 0x14,</div><div class='ctx'> 	HNS_ROCE_EVENT_TYPE_FLR			      = 0x15,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Local Work Queue Catastrophic Error,SUBTYPE 0x5 */</div><div class='del'>-enum {</div><div class='del'>-	HNS_ROCE_LWQCE_QPC_ERROR		= 1,</div><div class='del'>-	HNS_ROCE_LWQCE_MTU_ERROR		= 2,</div><div class='del'>-	HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR	= 3,</div><div class='del'>-	HNS_ROCE_LWQCE_WQE_ADDR_ERROR		= 4,</div><div class='del'>-	HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR	= 5,</div><div class='del'>-	HNS_ROCE_LWQCE_SL_ERROR			= 6,</div><div class='del'>-	HNS_ROCE_LWQCE_PORT_ERROR		= 7,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* Local Access Violation Work Queue Error,SUBTYPE 0x7 */</div><div class='del'>-enum {</div><div class='del'>-	HNS_ROCE_LAVWQE_R_KEY_VIOLATION		= 1,</div><div class='del'>-	HNS_ROCE_LAVWQE_LENGTH_ERROR		= 2,</div><div class='del'>-	HNS_ROCE_LAVWQE_VA_ERROR		= 3,</div><div class='del'>-	HNS_ROCE_LAVWQE_PD_ERROR		= 4,</div><div class='del'>-	HNS_ROCE_LAVWQE_RW_ACC_ERROR		= 5,</div><div class='del'>-	HNS_ROCE_LAVWQE_KEY_STATE_ERROR		= 6,</div><div class='del'>-	HNS_ROCE_LAVWQE_MR_OPERATION_ERROR	= 7,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* DOORBELL overflow subtype */</div><div class='del'>-enum {</div><div class='del'>-	HNS_ROCE_DB_SUBTYPE_SDB_OVF		= 1,</div><div class='del'>-	HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF		= 2,</div><div class='del'>-	HNS_ROCE_DB_SUBTYPE_ODB_OVF		= 3,</div><div class='del'>-	HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF		= 4,</div><div class='del'>-	HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP		= 5,</div><div class='del'>-	HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP		= 6,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum {</div><div class='del'>-	/* RQ&amp;SRQ related operations */</div><div class='del'>-	HNS_ROCE_OPCODE_SEND_DATA_RECEIVE	= 0x06,</div><div class='del'>-	HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE	= 0x07,</div><div class='add'>+	HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION	      = 0x16,</div><div class='add'>+	HNS_ROCE_EVENT_TYPE_INVALID_XRCETH	      = 0x17,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum {</div><div class='ctx'> 	HNS_ROCE_CAP_FLAG_REREG_MR		= BIT(0),</div><div class='ctx'> 	HNS_ROCE_CAP_FLAG_ROCE_V1_V2		= BIT(1),</div><div class='ctx'> 	HNS_ROCE_CAP_FLAG_RQ_INLINE		= BIT(2),</div><div class='del'>-	HNS_ROCE_CAP_FLAG_RECORD_DB		= BIT(3),</div><div class='del'>-	HNS_ROCE_CAP_FLAG_SQ_RECORD_DB		= BIT(4),</div><div class='add'>+	HNS_ROCE_CAP_FLAG_CQ_RECORD_DB		= BIT(3),</div><div class='add'>+	HNS_ROCE_CAP_FLAG_QP_RECORD_DB		= BIT(4),</div><div class='ctx'> 	HNS_ROCE_CAP_FLAG_SRQ			= BIT(5),</div><div class='add'>+	HNS_ROCE_CAP_FLAG_XRC			= BIT(6),</div><div class='ctx'> 	HNS_ROCE_CAP_FLAG_MW			= BIT(7),</div><div class='ctx'> 	HNS_ROCE_CAP_FLAG_FRMR                  = BIT(8),</div><div class='ctx'> 	HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL		= BIT(9),</div><div class='ctx'> 	HNS_ROCE_CAP_FLAG_ATOMIC		= BIT(10),</div><div class='add'>+	HNS_ROCE_CAP_FLAG_DIRECT_WQE		= BIT(12),</div><div class='add'>+	HNS_ROCE_CAP_FLAG_SDI_MODE		= BIT(14),</div><div class='add'>+	HNS_ROCE_CAP_FLAG_STASH			= BIT(17),</div><div class='add'>+	HNS_ROCE_CAP_FLAG_CQE_INLINE		= BIT(19),</div><div class='add'>+	HNS_ROCE_CAP_FLAG_BOND                  = BIT(21),</div><div class='add'>+	HNS_ROCE_CAP_FLAG_SRQ_RECORD_DB         = BIT(22),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-enum hns_roce_mtt_type {</div><div class='del'>-	MTT_TYPE_WQE,</div><div class='del'>-	MTT_TYPE_CQE,</div><div class='del'>-	MTT_TYPE_SRQWQE,</div><div class='del'>-	MTT_TYPE_IDX</div><div class='del'>-};</div><div class='add'>+#define HNS_ROCE_DB_TYPE_COUNT			2</div><div class='add'>+#define HNS_ROCE_DB_UNIT_SIZE			4</div><div class='ctx'> </div><div class='ctx'> enum {</div><div class='ctx'> 	HNS_ROCE_DB_PER_PAGE = PAGE_SIZE / 4</div><div class='hunk'>@@ -231,26 +179,22 @@ enum hns_roce_instance_state {</div><div class='ctx'> 	HNS_ROCE_STATE_INIT,</div><div class='ctx'> 	HNS_ROCE_STATE_INITED,</div><div class='ctx'> 	HNS_ROCE_STATE_UNINIT,</div><div class='add'>+	HNS_ROCE_STATE_BOND_UNINIT,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum {</div><div class='ctx'> 	HNS_ROCE_RST_DIRECT_RETURN		= 0,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-enum {</div><div class='del'>-	CMD_RST_PRC_OTHERS,</div><div class='del'>-	CMD_RST_PRC_SUCCESS,</div><div class='del'>-	CMD_RST_PRC_EBUSY,</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> #define HNS_ROCE_CMD_SUCCESS			1</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_PORT_DOWN			0</div><div class='del'>-#define HNS_ROCE_PORT_UP			1</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_MTT_ENTRY_PER_SEG		8</div><div class='add'>+#define HNS_ROCE_MAX_HOP_NUM			3</div><div class='add'>+/* The minimum page size is 4K for hardware */</div><div class='add'>+#define HNS_HW_PAGE_SHIFT			12</div><div class='add'>+#define HNS_HW_PAGE_SIZE			(1 &lt;&lt; HNS_HW_PAGE_SHIFT)</div><div class='ctx'> </div><div class='del'>-#define PAGE_ADDR_SHIFT				12</div><div class='add'>+#define HNS_HW_MAX_PAGE_SHIFT			27</div><div class='add'>+#define HNS_HW_MAX_PAGE_SIZE			(1 &lt;&lt; HNS_HW_MAX_PAGE_SHIFT)</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_uar {</div><div class='ctx'> 	u64		pfn;</div><div class='hunk'>@@ -258,11 +202,25 @@ struct hns_roce_uar {</div><div class='ctx'> 	unsigned long	logic_idx;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+enum hns_roce_mmap_type {</div><div class='add'>+	HNS_ROCE_MMAP_TYPE_DB = 1,</div><div class='add'>+	HNS_ROCE_MMAP_TYPE_DWQE,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct hns_user_mmap_entry {</div><div class='add'>+	struct rdma_user_mmap_entry rdma_entry;</div><div class='add'>+	enum hns_roce_mmap_type mmap_type;</div><div class='add'>+	u64 address;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct hns_roce_ucontext {</div><div class='ctx'> 	struct ib_ucontext	ibucontext;</div><div class='ctx'> 	struct hns_roce_uar	uar;</div><div class='ctx'> 	struct list_head	page_list;</div><div class='ctx'> 	struct mutex		page_mutex;</div><div class='add'>+	struct hns_user_mmap_entry *db_mmap_entry;</div><div class='add'>+	u32			config;</div><div class='add'>+	u8 cq_bank_id;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_pd {</div><div class='hunk'>@@ -270,6 +228,11 @@ struct hns_roce_pd {</div><div class='ctx'> 	unsigned long		pdn;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct hns_roce_xrcd {</div><div class='add'>+	struct ib_xrcd ibxrcd;</div><div class='add'>+	u32 xrcdn;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct hns_roce_bitmap {</div><div class='ctx'> 	/* Bitmap Traversal last a bit which is 1 */</div><div class='ctx'> 	unsigned long		last;</div><div class='hunk'>@@ -281,20 +244,10 @@ struct hns_roce_bitmap {</div><div class='ctx'> 	unsigned long		*table;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* Order bitmap length -- bit num compute formula: 1 &lt;&lt; (max_order - order) */</div><div class='del'>-/* Order = 0: bitmap is biggest, order = max bitmap is least (only a bit) */</div><div class='del'>-/* Every bit repesent to a partner free/used status in bitmap */</div><div class='del'>-/*</div><div class='del'>- * Initial, bits of other bitmap are all 0 except that a bit of max_order is 1</div><div class='del'>- * Bit = 1 represent to idle and available; bit = 0: not available</div><div class='del'>- */</div><div class='del'>-struct hns_roce_buddy {</div><div class='del'>-	/* Members point to every order level bitmap */</div><div class='del'>-	unsigned long **bits;</div><div class='del'>-	/* Represent to avail bits of the order level bitmap */</div><div class='del'>-	u32            *num_free;</div><div class='del'>-	int             max_order;</div><div class='del'>-	spinlock_t      lock;</div><div class='add'>+struct hns_roce_ida {</div><div class='add'>+	struct ida ida;</div><div class='add'>+	u32 min; /* Lowest ID to allocate.  */</div><div class='add'>+	u32 max; /* Highest ID to allocate. */</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /* For Hardware Entry Memory */</div><div class='hunk'>@@ -303,12 +256,9 @@ struct hns_roce_hem_table {</div><div class='ctx'> 	u32		type;</div><div class='ctx'> 	/* HEM array elment num */</div><div class='ctx'> 	unsigned long	num_hem;</div><div class='del'>-	/* HEM entry record obj total num */</div><div class='del'>-	unsigned long	num_obj;</div><div class='del'>-	/*Single obj size */</div><div class='add'>+	/* Single obj size */</div><div class='ctx'> 	unsigned long	obj_size;</div><div class='ctx'> 	unsigned long	table_chunk_size;</div><div class='del'>-	int		lowmem;</div><div class='ctx'> 	struct mutex	mutex;</div><div class='ctx'> 	struct hns_roce_hem **hem;</div><div class='ctx'> 	u64		**bt_l1;</div><div class='hunk'>@@ -317,84 +267,97 @@ struct hns_roce_hem_table {</div><div class='ctx'> 	dma_addr_t	*bt_l0_dma_addr;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct hns_roce_mtt {</div><div class='del'>-	unsigned long		first_seg;</div><div class='del'>-	int			order;</div><div class='del'>-	int			page_shift;</div><div class='del'>-	enum hns_roce_mtt_type	mtt_type;</div><div class='add'>+struct hns_roce_buf_region {</div><div class='add'>+	u32 offset; /* page offset */</div><div class='add'>+	u32 count; /* page count */</div><div class='add'>+	int hopnum; /* addressing hop num */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define HNS_ROCE_MAX_BT_REGION	3</div><div class='add'>+#define HNS_ROCE_MAX_BT_LEVEL	3</div><div class='add'>+struct hns_roce_hem_list {</div><div class='add'>+	struct list_head root_bt;</div><div class='add'>+	/* link all bt dma mem by hop config */</div><div class='add'>+	struct list_head mid_bt[HNS_ROCE_MAX_BT_REGION][HNS_ROCE_MAX_BT_LEVEL];</div><div class='add'>+	struct list_head btm_bt; /* link all bottom bt in @mid_bt */</div><div class='add'>+	dma_addr_t root_ba; /* pointer to the root ba table */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum mtr_type {</div><div class='add'>+	MTR_DEFAULT = 0,</div><div class='add'>+	MTR_PBL,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct hns_roce_buf_attr {</div><div class='add'>+	struct {</div><div class='add'>+		size_t	size;  /* region size */</div><div class='add'>+		int	hopnum; /* multi-hop addressing hop num */</div><div class='add'>+	} region[HNS_ROCE_MAX_BT_REGION];</div><div class='add'>+	unsigned int region_count; /* valid region count */</div><div class='add'>+	unsigned int page_shift;  /* buffer page shift */</div><div class='add'>+	unsigned int user_access; /* umem access flag */</div><div class='add'>+	u64 iova;</div><div class='add'>+	enum mtr_type type;</div><div class='add'>+	bool mtt_only; /* only alloc buffer-required MTT memory */</div><div class='add'>+	bool adaptive; /* adaptive for page_shift and hopnum */</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct hns_roce_mw {</div><div class='del'>-	struct ib_mw		ibmw;</div><div class='del'>-	u32			pdn;</div><div class='del'>-	u32			rkey;</div><div class='del'>-	int			enabled; /* MW's active status */</div><div class='del'>-	u32			pbl_hop_num;</div><div class='del'>-	u32			pbl_ba_pg_sz;</div><div class='del'>-	u32			pbl_buf_pg_sz;</div><div class='add'>+struct hns_roce_hem_cfg {</div><div class='add'>+	dma_addr_t	root_ba; /* root BA table's address */</div><div class='add'>+	bool		is_direct; /* addressing without BA table */</div><div class='add'>+	unsigned int	ba_pg_shift; /* BA table page shift */</div><div class='add'>+	unsigned int	buf_pg_shift; /* buffer page shift */</div><div class='add'>+	unsigned int	buf_pg_count;  /* buffer page count */</div><div class='add'>+	struct hns_roce_buf_region region[HNS_ROCE_MAX_BT_REGION];</div><div class='add'>+	unsigned int	region_count;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-/* Only support 4K page size for mr register */</div><div class='del'>-#define MR_SIZE_4K 0</div><div class='add'>+/* memory translate region */</div><div class='add'>+struct hns_roce_mtr {</div><div class='add'>+	struct hns_roce_hem_list hem_list; /* multi-hop addressing resource */</div><div class='add'>+	struct ib_umem		*umem; /* user space buffer */</div><div class='add'>+	struct hns_roce_buf	*kmem; /* kernel space buffer */</div><div class='add'>+	struct hns_roce_hem_cfg  hem_cfg; /* config for hardware addressing */</div><div class='add'>+};</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_mr {</div><div class='ctx'> 	struct ib_mr		ibmr;</div><div class='del'>-	struct ib_umem		*umem;</div><div class='del'>-	u64			iova; /* MR's virtual orignal addr */</div><div class='add'>+	u64			iova; /* MR's virtual original addr */</div><div class='ctx'> 	u64			size; /* Address range of MR */</div><div class='ctx'> 	u32			key; /* Key of MR */</div><div class='ctx'> 	u32			pd;   /* PD num of MR */</div><div class='del'>-	u32			access;/* Access permission of MR */</div><div class='del'>-	u32			npages;</div><div class='add'>+	u32			access; /* Access permission of MR */</div><div class='ctx'> 	int			enabled; /* MR's active status */</div><div class='del'>-	int			type;	/* MR's register type */</div><div class='del'>-	u64			*pbl_buf;/* MR's PBL space */</div><div class='del'>-	dma_addr_t		pbl_dma_addr;	/* MR's PBL space PA */</div><div class='del'>-	u32			pbl_size;/* PA number in the PBL */</div><div class='del'>-	u64			pbl_ba;/* page table address */</div><div class='del'>-	u32			l0_chunk_last_num;/* L0 last number */</div><div class='del'>-	u32			l1_chunk_last_num;/* L1 last number */</div><div class='del'>-	u64			**pbl_bt_l2;/* PBL BT L2 */</div><div class='del'>-	u64			**pbl_bt_l1;/* PBL BT L1 */</div><div class='del'>-	u64			*pbl_bt_l0;/* PBL BT L0 */</div><div class='del'>-	dma_addr_t		*pbl_l2_dma_addr;/* PBL BT L2 dma addr */</div><div class='del'>-	dma_addr_t		*pbl_l1_dma_addr;/* PBL BT L1 dma addr */</div><div class='del'>-	dma_addr_t		pbl_l0_dma_addr;/* PBL BT L0 dma addr */</div><div class='del'>-	u32			pbl_ba_pg_sz;/* BT chunk page size */</div><div class='del'>-	u32			pbl_buf_pg_sz;/* buf chunk page size */</div><div class='del'>-	u32			pbl_hop_num;/* multi-hop number */</div><div class='add'>+	int			type; /* MR's register type */</div><div class='add'>+	u32			pbl_hop_num; /* multi-hop number */</div><div class='add'>+	struct hns_roce_mtr	pbl_mtr;</div><div class='add'>+	u32			npages;</div><div class='add'>+	dma_addr_t		*page_list;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_mr_table {</div><div class='del'>-	struct hns_roce_bitmap		mtpt_bitmap;</div><div class='del'>-	struct hns_roce_buddy		mtt_buddy;</div><div class='del'>-	struct hns_roce_hem_table	mtt_table;</div><div class='add'>+	struct hns_roce_ida mtpt_ida;</div><div class='ctx'> 	struct hns_roce_hem_table	mtpt_table;</div><div class='del'>-	struct hns_roce_buddy		mtt_cqe_buddy;</div><div class='del'>-	struct hns_roce_hem_table	mtt_cqe_table;</div><div class='del'>-	struct hns_roce_buddy		mtt_srqwqe_buddy;</div><div class='del'>-	struct hns_roce_hem_table	mtt_srqwqe_table;</div><div class='del'>-	struct hns_roce_buddy		mtt_idx_buddy;</div><div class='del'>-	struct hns_roce_hem_table	mtt_idx_table;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_wq {</div><div class='ctx'> 	u64		*wrid;     /* Work request ID */</div><div class='ctx'> 	spinlock_t	lock;</div><div class='del'>-	int		wqe_cnt;  /* WQE num */</div><div class='del'>-	u32		max_post;</div><div class='del'>-	int		max_gs;</div><div class='del'>-	int		offset;</div><div class='del'>-	int		wqe_shift;/* WQE size */</div><div class='add'>+	u32		wqe_cnt;  /* WQE num */</div><div class='add'>+	u32		max_gs;</div><div class='add'>+	u32		rsv_sge;</div><div class='add'>+	u32		offset;</div><div class='add'>+	u32		wqe_shift; /* WQE size */</div><div class='ctx'> 	u32		head;</div><div class='ctx'> 	u32		tail;</div><div class='del'>-	void __iomem	*db_reg_l;</div><div class='add'>+	void __iomem	*db_reg;</div><div class='add'>+	u32		ext_sge_cnt;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_sge {</div><div class='del'>-	int		sge_cnt;  /* SGE num */</div><div class='del'>-	int		offset;</div><div class='del'>-	int		sge_shift;/* SGE size */</div><div class='add'>+	unsigned int	sge_cnt; /* SGE num */</div><div class='add'>+	u32		offset;</div><div class='add'>+	u32		sge_shift; /* SGE size */</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_buf_list {</div><div class='hunk'>@@ -402,19 +365,34 @@ struct hns_roce_buf_list {</div><div class='ctx'> 	dma_addr_t	map;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+/*</div><div class='add'>+ * %HNS_ROCE_BUF_DIRECT indicates that the all memory must be in a continuous</div><div class='add'>+ * dma address range.</div><div class='add'>+ *</div><div class='add'>+ * %HNS_ROCE_BUF_NOSLEEP indicates that the caller cannot sleep.</div><div class='add'>+ *</div><div class='add'>+ * %HNS_ROCE_BUF_NOFAIL allocation only failed when allocated size is zero, even</div><div class='add'>+ * the allocated size is smaller than the required size.</div><div class='add'>+ */</div><div class='add'>+enum {</div><div class='add'>+	HNS_ROCE_BUF_DIRECT = BIT(0),</div><div class='add'>+	HNS_ROCE_BUF_NOSLEEP = BIT(1),</div><div class='add'>+	HNS_ROCE_BUF_NOFAIL = BIT(2),</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct hns_roce_buf {</div><div class='del'>-	struct hns_roce_buf_list	direct;</div><div class='del'>-	struct hns_roce_buf_list	*page_list;</div><div class='del'>-	int				nbufs;</div><div class='add'>+	struct hns_roce_buf_list	*trunk_list;</div><div class='add'>+	u32				ntrunks;</div><div class='ctx'> 	u32				npages;</div><div class='del'>-	int				page_shift;</div><div class='add'>+	unsigned int			trunk_shift;</div><div class='add'>+	unsigned int			page_shift;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_db_pgdir {</div><div class='ctx'> 	struct list_head	list;</div><div class='ctx'> 	DECLARE_BITMAP(order0, HNS_ROCE_DB_PER_PAGE);</div><div class='del'>-	DECLARE_BITMAP(order1, HNS_ROCE_DB_PER_PAGE / 2);</div><div class='del'>-	unsigned long		*bits[2];</div><div class='add'>+	DECLARE_BITMAP(order1, HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT);</div><div class='add'>+	unsigned long		*bits[HNS_ROCE_DB_TYPE_COUNT];</div><div class='ctx'> 	u32			*page;</div><div class='ctx'> 	dma_addr_t		db_dma;</div><div class='ctx'> };</div><div class='hunk'>@@ -434,110 +412,115 @@ struct hns_roce_db {</div><div class='ctx'> 	} u;</div><div class='ctx'> 	dma_addr_t	dma;</div><div class='ctx'> 	void		*virt_addr;</div><div class='del'>-	int		index;</div><div class='del'>-	int		order;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct hns_roce_cq_buf {</div><div class='del'>-	struct hns_roce_buf hr_buf;</div><div class='del'>-	struct hns_roce_mtt hr_mtt;</div><div class='add'>+	unsigned long	index;</div><div class='add'>+	unsigned long	order;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_cq {</div><div class='ctx'> 	struct ib_cq			ib_cq;</div><div class='del'>-	struct hns_roce_cq_buf		hr_buf;</div><div class='add'>+	struct hns_roce_mtr		mtr;</div><div class='ctx'> 	struct hns_roce_db		db;</div><div class='del'>-	u8				db_en;</div><div class='add'>+	u32				flags;</div><div class='ctx'> 	spinlock_t			lock;</div><div class='del'>-	struct ib_umem			*umem;</div><div class='del'>-	void (*comp)(struct hns_roce_cq *cq);</div><div class='del'>-	void (*event)(struct hns_roce_cq *cq, enum hns_roce_event event_type);</div><div class='del'>-</div><div class='del'>-	struct hns_roce_uar		*uar;</div><div class='ctx'> 	u32				cq_depth;</div><div class='ctx'> 	u32				cons_index;</div><div class='ctx'> 	u32				*set_ci_db;</div><div class='del'>-	void __iomem			*cq_db_l;</div><div class='del'>-	u16				*tptr_addr;</div><div class='add'>+	void __iomem			*db_reg;</div><div class='ctx'> 	int				arm_sn;</div><div class='add'>+	int				cqe_size;</div><div class='ctx'> 	unsigned long			cqn;</div><div class='ctx'> 	u32				vector;</div><div class='del'>-	atomic_t			refcount;</div><div class='add'>+	refcount_t			refcount;</div><div class='ctx'> 	struct completion		free;</div><div class='add'>+	struct list_head		sq_list; /* all qps on this send cq */</div><div class='add'>+	struct list_head		rq_list; /* all qps on this recv cq */</div><div class='add'>+	int				is_armed; /* cq is armed */</div><div class='add'>+	struct list_head		node; /* all armed cqs are on a list */</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_idx_que {</div><div class='del'>-	struct hns_roce_buf		idx_buf;</div><div class='del'>-	int				entry_sz;</div><div class='del'>-	u32				buf_size;</div><div class='del'>-	struct ib_umem			*umem;</div><div class='del'>-	struct hns_roce_mtt		mtt;</div><div class='del'>-	u64				*bitmap;</div><div class='add'>+	struct hns_roce_mtr		mtr;</div><div class='add'>+	u32				entry_shift;</div><div class='add'>+	unsigned long			*bitmap;</div><div class='add'>+	u32				head;</div><div class='add'>+	u32				tail;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_srq {</div><div class='ctx'> 	struct ib_srq		ibsrq;</div><div class='del'>-	void (*event)(struct hns_roce_srq *srq, enum hns_roce_event event);</div><div class='ctx'> 	unsigned long		srqn;</div><div class='del'>-	int			max;</div><div class='add'>+	u32			wqe_cnt;</div><div class='ctx'> 	int			max_gs;</div><div class='del'>-	int			wqe_shift;</div><div class='del'>-	void __iomem		*db_reg_l;</div><div class='add'>+	u32			rsv_sge;</div><div class='add'>+	u32			wqe_shift;</div><div class='add'>+	u32			cqn;</div><div class='add'>+	u32			xrcdn;</div><div class='add'>+	void __iomem		*db_reg;</div><div class='ctx'> </div><div class='del'>-	atomic_t		refcount;</div><div class='add'>+	refcount_t		refcount;</div><div class='ctx'> 	struct completion	free;</div><div class='ctx'> </div><div class='del'>-	struct hns_roce_buf	buf;</div><div class='add'>+	struct hns_roce_mtr	buf_mtr;</div><div class='add'>+</div><div class='ctx'> 	u64		       *wrid;</div><div class='del'>-	struct ib_umem	       *umem;</div><div class='del'>-	struct hns_roce_mtt	mtt;</div><div class='ctx'> 	struct hns_roce_idx_que idx_que;</div><div class='ctx'> 	spinlock_t		lock;</div><div class='del'>-	int			head;</div><div class='del'>-	int			tail;</div><div class='del'>-	u16			wqe_ctr;</div><div class='ctx'> 	struct mutex		mutex;</div><div class='add'>+	void (*event)(struct hns_roce_srq *srq, enum hns_roce_event event);</div><div class='add'>+	struct hns_roce_db	rdb;</div><div class='add'>+	u32			cap_flags;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_uar_table {</div><div class='ctx'> 	struct hns_roce_bitmap bitmap;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct hns_roce_bank {</div><div class='add'>+	struct ida ida;</div><div class='add'>+	u32 inuse; /* Number of IDs allocated */</div><div class='add'>+	u32 min; /* Lowest ID to allocate.  */</div><div class='add'>+	u32 max; /* Highest ID to allocate. */</div><div class='add'>+	u32 next; /* Next ID to allocate. */</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct hns_roce_qp_table {</div><div class='del'>-	struct hns_roce_bitmap		bitmap;</div><div class='ctx'> 	struct hns_roce_hem_table	qp_table;</div><div class='ctx'> 	struct hns_roce_hem_table	irrl_table;</div><div class='ctx'> 	struct hns_roce_hem_table	trrl_table;</div><div class='ctx'> 	struct hns_roce_hem_table	sccc_table;</div><div class='ctx'> 	struct mutex			scc_mutex;</div><div class='add'>+	struct hns_roce_bank bank[HNS_ROCE_QP_BANK_NUM];</div><div class='add'>+	struct mutex bank_mutex;</div><div class='add'>+	struct xarray			dip_xa;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_cq_table {</div><div class='del'>-	struct hns_roce_bitmap		bitmap;</div><div class='ctx'> 	struct xarray			array;</div><div class='ctx'> 	struct hns_roce_hem_table	table;</div><div class='add'>+	struct hns_roce_bank bank[HNS_ROCE_CQ_BANK_NUM];</div><div class='add'>+	struct mutex			bank_mutex;</div><div class='add'>+	u32 ctx_num[HNS_ROCE_CQ_BANK_NUM];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_srq_table {</div><div class='del'>-	struct hns_roce_bitmap		bitmap;</div><div class='add'>+	struct hns_roce_ida		srq_ida;</div><div class='ctx'> 	struct xarray			xa;</div><div class='ctx'> 	struct hns_roce_hem_table	table;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct hns_roce_raq_table {</div><div class='del'>-	struct hns_roce_buf_list	*e_raq_buf;</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> struct hns_roce_av {</div><div class='del'>-	__le32      port_pd;</div><div class='del'>-	u8          gid_index;</div><div class='del'>-	u8          stat_rate;</div><div class='del'>-	u8          hop_limit;</div><div class='del'>-	__le32      sl_tclass_flowlabel;</div><div class='del'>-	u8          dgid[HNS_ROCE_GID_SIZE];</div><div class='del'>-	u8          mac[6];</div><div class='del'>-	__le16      vlan;</div><div class='del'>-	bool	    vlan_en;</div><div class='add'>+	u8 port;</div><div class='add'>+	u8 gid_index;</div><div class='add'>+	u8 stat_rate;</div><div class='add'>+	u8 hop_limit;</div><div class='add'>+	u32 flowlabel;</div><div class='add'>+	u16 udp_sport;</div><div class='add'>+	u8 sl;</div><div class='add'>+	u8 tclass;</div><div class='add'>+	u8 dgid[HNS_ROCE_GID_SIZE];</div><div class='add'>+	u8 mac[ETH_ALEN];</div><div class='add'>+	u16 vlan_id;</div><div class='add'>+	u8 vlan_en;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_ah {</div><div class='hunk'>@@ -551,11 +534,16 @@ struct hns_roce_cmd_context {</div><div class='ctx'> 	int			next;</div><div class='ctx'> 	u64			out_param;</div><div class='ctx'> 	u16			token;</div><div class='add'>+	u16			busy;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum hns_roce_cmdq_state {</div><div class='add'>+	HNS_ROCE_CMDQ_STATE_NORMAL,</div><div class='add'>+	HNS_ROCE_CMDQ_STATE_FATAL_ERR,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_cmdq {</div><div class='ctx'> 	struct dma_pool		*pool;</div><div class='del'>-	struct mutex		hcr_mutex;</div><div class='ctx'> 	struct semaphore	poll_sem;</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Event mode: cmd register mutex protection,</div><div class='hunk'>@@ -567,18 +555,13 @@ struct hns_roce_cmdq {</div><div class='ctx'> 	int			free_head;</div><div class='ctx'> 	struct hns_roce_cmd_context *context;</div><div class='ctx'> 	/*</div><div class='del'>-	 * Result of get integer part</div><div class='del'>-	 * which max_comds compute according a power of 2</div><div class='del'>-	 */</div><div class='del'>-	u16			token_mask;</div><div class='del'>-	/*</div><div class='ctx'> 	 * Process whether use event mode, init default non-zero</div><div class='ctx'> 	 * After the event queue of cmd event ready,</div><div class='ctx'> 	 * can switch into event mode</div><div class='ctx'> 	 * close device, switch into poll mode(non event mode)</div><div class='ctx'> 	 */</div><div class='ctx'> 	u8			use_events;</div><div class='del'>-	u8			toggle;</div><div class='add'>+	enum hns_roce_cmdq_state state;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_cmd_mailbox {</div><div class='hunk'>@@ -586,40 +569,48 @@ struct hns_roce_cmd_mailbox {</div><div class='ctx'> 	dma_addr_t		dma;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct hns_roce_mbox_msg {</div><div class='add'>+	u64 in_param;</div><div class='add'>+	u64 out_param;</div><div class='add'>+	u8 cmd;</div><div class='add'>+	u32 tag;</div><div class='add'>+	u16 token;</div><div class='add'>+	u8 event_en;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct hns_roce_dev;</div><div class='ctx'> </div><div class='del'>-struct hns_roce_rinl_sge {</div><div class='del'>-	void			*addr;</div><div class='del'>-	u32			len;</div><div class='add'>+enum {</div><div class='add'>+	HNS_ROCE_FLUSH_FLAG = 0,</div><div class='add'>+	HNS_ROCE_STOP_FLUSH_FLAG = 1,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct hns_roce_rinl_wqe {</div><div class='del'>-	struct hns_roce_rinl_sge *sg_list;</div><div class='del'>-	u32			 sge_cnt;</div><div class='add'>+struct hns_roce_work {</div><div class='add'>+	struct hns_roce_dev *hr_dev;</div><div class='add'>+	struct work_struct work;</div><div class='add'>+	int event_type;</div><div class='add'>+	int sub_type;</div><div class='add'>+	u32 queue_num;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct hns_roce_rinl_buf {</div><div class='del'>-	struct hns_roce_rinl_wqe *wqe_list;</div><div class='del'>-	u32			 wqe_cnt;</div><div class='add'>+enum hns_roce_cong_type {</div><div class='add'>+	CONG_TYPE_DCQCN,</div><div class='add'>+	CONG_TYPE_LDCP,</div><div class='add'>+	CONG_TYPE_HC3,</div><div class='add'>+	CONG_TYPE_DIP,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_qp {</div><div class='ctx'> 	struct ib_qp		ibqp;</div><div class='del'>-	struct hns_roce_buf	hr_buf;</div><div class='ctx'> 	struct hns_roce_wq	rq;</div><div class='ctx'> 	struct hns_roce_db	rdb;</div><div class='ctx'> 	struct hns_roce_db	sdb;</div><div class='del'>-	u8			rdb_en;</div><div class='del'>-	u8			sdb_en;</div><div class='del'>-	u32			doorbell_qpn;</div><div class='del'>-	__le32			sq_signal_bits;</div><div class='del'>-	u32			sq_next_wqe;</div><div class='del'>-	int			sq_max_wqes_per_wr;</div><div class='del'>-	int			sq_spare_wqes;</div><div class='add'>+	unsigned long		en_flags;</div><div class='add'>+	enum ib_sig_type	sq_signal_bits;</div><div class='ctx'> 	struct hns_roce_wq	sq;</div><div class='ctx'> </div><div class='del'>-	struct ib_umem		*umem;</div><div class='del'>-	struct hns_roce_mtt	mtt;</div><div class='add'>+	struct hns_roce_mtr	mtr;</div><div class='add'>+</div><div class='ctx'> 	u32			buff_size;</div><div class='ctx'> 	struct mutex		mutex;</div><div class='ctx'> 	u8			port;</div><div class='hunk'>@@ -627,25 +618,36 @@ struct hns_roce_qp {</div><div class='ctx'> 	u8			sl;</div><div class='ctx'> 	u8			resp_depth;</div><div class='ctx'> 	u8			state;</div><div class='del'>-	u32			access_flags;</div><div class='ctx'> 	u32                     atomic_rd_en;</div><div class='del'>-	u32			pkey_index;</div><div class='ctx'> 	u32			qkey;</div><div class='ctx'> 	void			(*event)(struct hns_roce_qp *qp,</div><div class='ctx'> 					 enum hns_roce_event event_type);</div><div class='ctx'> 	unsigned long		qpn;</div><div class='ctx'> </div><div class='del'>-	atomic_t		refcount;</div><div class='add'>+	u32			xrcdn;</div><div class='add'>+</div><div class='add'>+	refcount_t		refcount;</div><div class='ctx'> 	struct completion	free;</div><div class='ctx'> </div><div class='ctx'> 	struct hns_roce_sge	sge;</div><div class='ctx'> 	u32			next_sge;</div><div class='del'>-</div><div class='del'>-	struct hns_roce_rinl_buf rq_inl_buf;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct hns_roce_sqp {</div><div class='del'>-	struct hns_roce_qp	hr_qp;</div><div class='add'>+	enum ib_mtu		path_mtu;</div><div class='add'>+	u32			max_inline_data;</div><div class='add'>+	u8			free_mr_en;</div><div class='add'>+</div><div class='add'>+	/* 0: flush needed, 1: unneeded */</div><div class='add'>+	unsigned long		flush_flag;</div><div class='add'>+	struct hns_roce_work	flush_work;</div><div class='add'>+	struct list_head	node; /* all qps are on a list */</div><div class='add'>+	struct list_head	rq_node; /* all recv qps are on a list */</div><div class='add'>+	struct list_head	sq_node; /* all send qps are on a list */</div><div class='add'>+	struct hns_user_mmap_entry *dwqe_mmap_entry;</div><div class='add'>+	u32			config;</div><div class='add'>+	enum hns_roce_cong_type	cong_type;</div><div class='add'>+	u8			tc_mode;</div><div class='add'>+	u8			priority;</div><div class='add'>+	spinlock_t flush_lock;</div><div class='add'>+	struct hns_roce_dip *dip;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_ib_iboe {</div><div class='hunk'>@@ -655,41 +657,24 @@ struct hns_roce_ib_iboe {</div><div class='ctx'> 	u8			phy_port[HNS_ROCE_MAX_PORTS];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-enum {</div><div class='del'>-	HNS_ROCE_EQ_STAT_INVALID  = 0,</div><div class='del'>-	HNS_ROCE_EQ_STAT_VALID    = 2,</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> struct hns_roce_ceqe {</div><div class='del'>-	u32			comp;</div><div class='add'>+	__le32	comp;</div><div class='add'>+	__le32	rsv[15];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+#define CEQE_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_ceqe, h, l)</div><div class='add'>+</div><div class='add'>+#define CEQE_CQN CEQE_FIELD_LOC(23, 0)</div><div class='add'>+#define CEQE_OWNER CEQE_FIELD_LOC(31, 31)</div><div class='add'>+</div><div class='ctx'> struct hns_roce_aeqe {</div><div class='ctx'> 	__le32 asyn;</div><div class='ctx'> 	union {</div><div class='ctx'> 		struct {</div><div class='del'>-			__le32 qp;</div><div class='del'>-			u32 rsv0;</div><div class='del'>-			u32 rsv1;</div><div class='del'>-		} qp_event;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			__le32 srq;</div><div class='del'>-			u32 rsv0;</div><div class='del'>-			u32 rsv1;</div><div class='del'>-		} srq_event;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			__le32 cq;</div><div class='add'>+			__le32 num;</div><div class='ctx'> 			u32 rsv0;</div><div class='ctx'> 			u32 rsv1;</div><div class='del'>-		} cq_event;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			__le32 ceqe;</div><div class='del'>-			u32 rsv0;</div><div class='del'>-			u32 rsv1;</div><div class='del'>-		} ce_event;</div><div class='add'>+		} queue_event;</div><div class='ctx'> </div><div class='ctx'> 		struct {</div><div class='ctx'> 			__le64  out_param;</div><div class='hunk'>@@ -698,48 +683,41 @@ struct hns_roce_aeqe {</div><div class='ctx'> 			u8	rsv0;</div><div class='ctx'> 		} __packed cmd;</div><div class='ctx'> 	 } event;</div><div class='add'>+	__le32 rsv[12];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+#define AEQE_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_aeqe, h, l)</div><div class='add'>+</div><div class='add'>+#define AEQE_EVENT_TYPE AEQE_FIELD_LOC(7, 0)</div><div class='add'>+#define AEQE_SUB_TYPE AEQE_FIELD_LOC(15, 8)</div><div class='add'>+#define AEQE_OWNER AEQE_FIELD_LOC(31, 31)</div><div class='add'>+#define AEQE_EVENT_QUEUE_NUM AEQE_FIELD_LOC(55, 32)</div><div class='add'>+</div><div class='ctx'> struct hns_roce_eq {</div><div class='ctx'> 	struct hns_roce_dev		*hr_dev;</div><div class='del'>-	void __iomem			*doorbell;</div><div class='add'>+	void __iomem			*db_reg;</div><div class='ctx'> </div><div class='del'>-	int				type_flag;/* Aeq:1 ceq:0 */</div><div class='add'>+	int				type_flag; /* Aeq:1 ceq:0 */</div><div class='ctx'> 	int				eqn;</div><div class='ctx'> 	u32				entries;</div><div class='del'>-	int				log_entries;</div><div class='ctx'> 	int				eqe_size;</div><div class='ctx'> 	int				irq;</div><div class='del'>-	int				log_page_size;</div><div class='del'>-	int				cons_index;</div><div class='del'>-	struct hns_roce_buf_list	*buf_list;</div><div class='add'>+	u32				cons_index;</div><div class='ctx'> 	int				over_ignore;</div><div class='ctx'> 	int				coalesce;</div><div class='ctx'> 	int				arm_st;</div><div class='del'>-	u64				eqe_ba;</div><div class='del'>-	int				eqe_ba_pg_sz;</div><div class='del'>-	int				eqe_buf_pg_sz;</div><div class='ctx'> 	int				hop_num;</div><div class='del'>-	u64				*bt_l0;	/* Base address table for L0 */</div><div class='del'>-	u64				**bt_l1; /* Base address table for L1 */</div><div class='del'>-	u64				**buf;</div><div class='del'>-	dma_addr_t			l0_dma;</div><div class='del'>-	dma_addr_t			*l1_dma;</div><div class='del'>-	dma_addr_t			*buf_dma;</div><div class='del'>-	u32				l0_last_num; /* L0 last chunk num */</div><div class='del'>-	u32				l1_last_num; /* L1 last chunk num */</div><div class='del'>-	int				eq_max_cnt;</div><div class='del'>-	int				eq_period;</div><div class='add'>+	struct hns_roce_mtr		mtr;</div><div class='add'>+	u16				eq_max_cnt;</div><div class='add'>+	u32				eq_period;</div><div class='ctx'> 	int				shift;</div><div class='del'>-	dma_addr_t			cur_eqe_ba;</div><div class='del'>-	dma_addr_t			nxt_eqe_ba;</div><div class='ctx'> 	int				event_type;</div><div class='ctx'> 	int				sub_type;</div><div class='add'>+	struct work_struct		work;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_eq_table {</div><div class='ctx'> 	struct hns_roce_eq	*eq;</div><div class='del'>-	void __iomem		**eqc_base; /* only for hw v1 */</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_caps {</div><div class='hunk'>@@ -750,54 +728,50 @@ struct hns_roce_caps {</div><div class='ctx'> 	int		local_ca_ack_delay;</div><div class='ctx'> 	int		num_uars;</div><div class='ctx'> 	u32		phy_num_uars;</div><div class='del'>-	u32		max_sq_sg;	/* 2 */</div><div class='del'>-	u32		max_sq_inline;	/* 32 */</div><div class='del'>-	u32		max_rq_sg;	/* 2 */</div><div class='del'>-	u32		max_extend_sg;</div><div class='del'>-	int		num_qps;	/* 256k */</div><div class='del'>-	int             reserved_qps;</div><div class='del'>-	int		num_qpc_timer;</div><div class='del'>-	int		num_cqc_timer;</div><div class='del'>-	u32		max_srq_sg;</div><div class='del'>-	int		num_srqs;</div><div class='del'>-	u32		max_wqes;	/* 16k */</div><div class='del'>-	u32		max_srqs;</div><div class='add'>+	u32		max_sq_sg;</div><div class='add'>+	u32		max_sq_inline;</div><div class='add'>+	u32		max_rq_sg;</div><div class='add'>+	u32		rsv0;</div><div class='add'>+	u32		num_qps;</div><div class='add'>+	u32		reserved_qps;</div><div class='add'>+	u32		num_srqs;</div><div class='add'>+	u32		max_wqes;</div><div class='ctx'> 	u32		max_srq_wrs;</div><div class='ctx'> 	u32		max_srq_sges;</div><div class='del'>-	u32		max_sq_desc_sz;	/* 64 */</div><div class='del'>-	u32		max_rq_desc_sz;	/* 64 */</div><div class='del'>-	u32		max_srq_desc_sz;</div><div class='add'>+	u32		max_sq_desc_sz;</div><div class='add'>+	u32		max_rq_desc_sz;</div><div class='add'>+	u32		rsv2;</div><div class='ctx'> 	int		max_qp_init_rdma;</div><div class='ctx'> 	int		max_qp_dest_rdma;</div><div class='del'>-	int		num_cqs;</div><div class='del'>-	int		max_cqes;</div><div class='del'>-	int		min_cqes;</div><div class='add'>+	u32		num_cqs;</div><div class='add'>+	u32		max_cqes;</div><div class='add'>+	u32		min_cqes;</div><div class='ctx'> 	u32		min_wqes;</div><div class='del'>-	int		reserved_cqs;</div><div class='del'>-	int		reserved_srqs;</div><div class='del'>-	u32		max_srqwqes;</div><div class='del'>-	int		num_aeq_vectors;	/* 1 */</div><div class='add'>+	u32		reserved_cqs;</div><div class='add'>+	u32		reserved_srqs;</div><div class='add'>+	int		num_aeq_vectors;</div><div class='ctx'> 	int		num_comp_vectors;</div><div class='ctx'> 	int		num_other_vectors;</div><div class='del'>-	int		num_mtpts;</div><div class='del'>-	u32		num_mtt_segs;</div><div class='del'>-	u32		num_cqe_segs;</div><div class='add'>+	u32		num_mtpts;</div><div class='add'>+	u32		rsv1;</div><div class='ctx'> 	u32		num_srqwqe_segs;</div><div class='ctx'> 	u32		num_idx_segs;</div><div class='ctx'> 	int		reserved_mrws;</div><div class='ctx'> 	int		reserved_uars;</div><div class='ctx'> 	int		num_pds;</div><div class='ctx'> 	int		reserved_pds;</div><div class='add'>+	u32		num_xrcds;</div><div class='add'>+	u32		reserved_xrcds;</div><div class='ctx'> 	u32		mtt_entry_sz;</div><div class='del'>-	u32		cq_entry_sz;</div><div class='add'>+	u32		cqe_sz;</div><div class='ctx'> 	u32		page_size_cap;</div><div class='ctx'> 	u32		reserved_lkey;</div><div class='ctx'> 	int		mtpt_entry_sz;</div><div class='del'>-	int		qpc_entry_sz;</div><div class='add'>+	int		qpc_sz;</div><div class='ctx'> 	int		irrl_entry_sz;</div><div class='ctx'> 	int		trrl_entry_sz;</div><div class='ctx'> 	int		cqc_entry_sz;</div><div class='del'>-	int		sccc_entry_sz;</div><div class='add'>+	int		sccc_sz;</div><div class='ctx'> 	int		qpc_timer_entry_sz;</div><div class='ctx'> 	int		cqc_timer_entry_sz;</div><div class='ctx'> 	int		srqc_entry_sz;</div><div class='hunk'>@@ -807,6 +781,8 @@ struct hns_roce_caps {</div><div class='ctx'> 	u32		pbl_hop_num;</div><div class='ctx'> 	int		aeqe_depth;</div><div class='ctx'> 	int		ceqe_depth;</div><div class='add'>+	u32		aeqe_size;</div><div class='add'>+	u32		ceqe_size;</div><div class='ctx'> 	enum ib_mtu	max_mtu;</div><div class='ctx'> 	u32		qpc_bt_num;</div><div class='ctx'> 	u32		qpc_timer_bt_num;</div><div class='hunk'>@@ -814,7 +790,11 @@ struct hns_roce_caps {</div><div class='ctx'> 	u32		cqc_bt_num;</div><div class='ctx'> 	u32		cqc_timer_bt_num;</div><div class='ctx'> 	u32		mpt_bt_num;</div><div class='add'>+	u32		eqc_bt_num;</div><div class='add'>+	u32		smac_bt_num;</div><div class='add'>+	u32		sgid_bt_num;</div><div class='ctx'> 	u32		sccc_bt_num;</div><div class='add'>+	u32		gmv_bt_num;</div><div class='ctx'> 	u32		qpc_ba_pg_sz;</div><div class='ctx'> 	u32		qpc_buf_pg_sz;</div><div class='ctx'> 	u32		qpc_hop_num;</div><div class='hunk'>@@ -830,6 +810,9 @@ struct hns_roce_caps {</div><div class='ctx'> 	u32		mtt_ba_pg_sz;</div><div class='ctx'> 	u32		mtt_buf_pg_sz;</div><div class='ctx'> 	u32		mtt_hop_num;</div><div class='add'>+	u32		wqe_sq_hop_num;</div><div class='add'>+	u32		wqe_sge_hop_num;</div><div class='add'>+	u32		wqe_rq_hop_num;</div><div class='ctx'> 	u32		sccc_ba_pg_sz;</div><div class='ctx'> 	u32		sccc_buf_pg_sz;</div><div class='ctx'> 	u32		sccc_hop_num;</div><div class='hunk'>@@ -839,7 +822,7 @@ struct hns_roce_caps {</div><div class='ctx'> 	u32		cqc_timer_ba_pg_sz;</div><div class='ctx'> 	u32		cqc_timer_buf_pg_sz;</div><div class='ctx'> 	u32		cqc_timer_hop_num;</div><div class='del'>-	u32		cqe_ba_pg_sz;</div><div class='add'>+	u32		cqe_ba_pg_sz; /* page_size = 4K*(2^cqe_ba_pg_sz) */</div><div class='ctx'> 	u32		cqe_buf_pg_sz;</div><div class='ctx'> 	u32		cqe_hop_num;</div><div class='ctx'> 	u32		srqwqe_ba_pg_sz;</div><div class='hunk'>@@ -851,118 +834,156 @@ struct hns_roce_caps {</div><div class='ctx'> 	u32		eqe_ba_pg_sz;</div><div class='ctx'> 	u32		eqe_buf_pg_sz;</div><div class='ctx'> 	u32		eqe_hop_num;</div><div class='add'>+	u32		gmv_entry_num;</div><div class='add'>+	u32		gmv_entry_sz;</div><div class='add'>+	u32		gmv_ba_pg_sz;</div><div class='add'>+	u32		gmv_buf_pg_sz;</div><div class='add'>+	u32		gmv_hop_num;</div><div class='ctx'> 	u32		sl_num;</div><div class='del'>-	u32		tsq_buf_pg_sz;</div><div class='del'>-	u32		tpq_buf_pg_sz;</div><div class='del'>-	u32		chunk_sz;	/* chunk size in non multihop mode*/</div><div class='add'>+	u32		llm_buf_pg_sz;</div><div class='add'>+	u32		chunk_sz; /* chunk size in non multihop mode */</div><div class='ctx'> 	u64		flags;</div><div class='add'>+	u16		default_ceq_max_cnt;</div><div class='add'>+	u16		default_ceq_period;</div><div class='add'>+	u16		default_aeq_max_cnt;</div><div class='add'>+	u16		default_aeq_period;</div><div class='add'>+	u16		default_aeq_arm_st;</div><div class='add'>+	u16		default_ceq_arm_st;</div><div class='add'>+	u8		cong_cap;</div><div class='add'>+	enum hns_roce_cong_type default_cong_type;</div><div class='add'>+	u32             max_ack_req_msg_len;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct hns_roce_work {</div><div class='del'>-	struct hns_roce_dev *hr_dev;</div><div class='del'>-	struct work_struct work;</div><div class='del'>-	u32 qpn;</div><div class='del'>-	u32 cqn;</div><div class='del'>-	int event_type;</div><div class='del'>-	int sub_type;</div><div class='add'>+enum hns_roce_device_state {</div><div class='add'>+	HNS_ROCE_DEVICE_STATE_INITED,</div><div class='add'>+	HNS_ROCE_DEVICE_STATE_RST_DOWN,</div><div class='add'>+	HNS_ROCE_DEVICE_STATE_UNINIT,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum hns_roce_hw_pkt_stat_index {</div><div class='add'>+	HNS_ROCE_HW_RX_RC_PKT_CNT,</div><div class='add'>+	HNS_ROCE_HW_RX_UC_PKT_CNT,</div><div class='add'>+	HNS_ROCE_HW_RX_UD_PKT_CNT,</div><div class='add'>+	HNS_ROCE_HW_RX_XRC_PKT_CNT,</div><div class='add'>+	HNS_ROCE_HW_RX_PKT_CNT,</div><div class='add'>+	HNS_ROCE_HW_RX_ERR_PKT_CNT,</div><div class='add'>+	HNS_ROCE_HW_RX_CNP_PKT_CNT,</div><div class='add'>+	HNS_ROCE_HW_TX_RC_PKT_CNT,</div><div class='add'>+	HNS_ROCE_HW_TX_UC_PKT_CNT,</div><div class='add'>+	HNS_ROCE_HW_TX_UD_PKT_CNT,</div><div class='add'>+	HNS_ROCE_HW_TX_XRC_PKT_CNT,</div><div class='add'>+	HNS_ROCE_HW_TX_PKT_CNT,</div><div class='add'>+	HNS_ROCE_HW_TX_ERR_PKT_CNT,</div><div class='add'>+	HNS_ROCE_HW_TX_CNP_PKT_CNT,</div><div class='add'>+	HNS_ROCE_HW_TRP_GET_MPT_ERR_PKT_CNT,</div><div class='add'>+	HNS_ROCE_HW_TRP_GET_IRRL_ERR_PKT_CNT,</div><div class='add'>+	HNS_ROCE_HW_ECN_DB_CNT,</div><div class='add'>+	HNS_ROCE_HW_RX_BUF_CNT,</div><div class='add'>+	HNS_ROCE_HW_TRP_RX_SOF_CNT,</div><div class='add'>+	HNS_ROCE_HW_CQ_CQE_CNT,</div><div class='add'>+	HNS_ROCE_HW_CQ_POE_CNT,</div><div class='add'>+	HNS_ROCE_HW_CQ_NOTIFY_CNT,</div><div class='add'>+	HNS_ROCE_HW_CNT_TOTAL</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct hns_roce_dfx_hw {</div><div class='del'>-	int (*query_cqc_info)(struct hns_roce_dev *hr_dev, u32 cqn,</div><div class='del'>-			      int *buffer);</div><div class='add'>+enum hns_roce_sw_dfx_stat_index {</div><div class='add'>+	HNS_ROCE_DFX_AEQE_CNT,</div><div class='add'>+	HNS_ROCE_DFX_CEQE_CNT,</div><div class='add'>+	HNS_ROCE_DFX_CMDS_CNT,</div><div class='add'>+	HNS_ROCE_DFX_CMDS_ERR_CNT,</div><div class='add'>+	HNS_ROCE_DFX_MBX_POSTED_CNT,</div><div class='add'>+	HNS_ROCE_DFX_MBX_POLLED_CNT,</div><div class='add'>+	HNS_ROCE_DFX_MBX_EVENT_CNT,</div><div class='add'>+	HNS_ROCE_DFX_QP_CREATE_ERR_CNT,</div><div class='add'>+	HNS_ROCE_DFX_QP_MODIFY_ERR_CNT,</div><div class='add'>+	HNS_ROCE_DFX_CQ_CREATE_ERR_CNT,</div><div class='add'>+	HNS_ROCE_DFX_CQ_MODIFY_ERR_CNT,</div><div class='add'>+	HNS_ROCE_DFX_SRQ_CREATE_ERR_CNT,</div><div class='add'>+	HNS_ROCE_DFX_SRQ_MODIFY_ERR_CNT,</div><div class='add'>+	HNS_ROCE_DFX_XRCD_ALLOC_ERR_CNT,</div><div class='add'>+	HNS_ROCE_DFX_MR_REG_ERR_CNT,</div><div class='add'>+	HNS_ROCE_DFX_MR_REREG_ERR_CNT,</div><div class='add'>+	HNS_ROCE_DFX_AH_CREATE_ERR_CNT,</div><div class='add'>+	HNS_ROCE_DFX_MMAP_ERR_CNT,</div><div class='add'>+	HNS_ROCE_DFX_UCTX_ALLOC_ERR_CNT,</div><div class='add'>+	HNS_ROCE_DFX_CNT_TOTAL</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_hw {</div><div class='del'>-	int (*reset)(struct hns_roce_dev *hr_dev, bool enable);</div><div class='ctx'> 	int (*cmq_init)(struct hns_roce_dev *hr_dev);</div><div class='ctx'> 	void (*cmq_exit)(struct hns_roce_dev *hr_dev);</div><div class='ctx'> 	int (*hw_profile)(struct hns_roce_dev *hr_dev);</div><div class='ctx'> 	int (*hw_init)(struct hns_roce_dev *hr_dev);</div><div class='ctx'> 	void (*hw_exit)(struct hns_roce_dev *hr_dev);</div><div class='del'>-	int (*post_mbox)(struct hns_roce_dev *hr_dev, u64 in_param,</div><div class='del'>-			 u64 out_param, u32 in_modifier, u8 op_modifier, u16 op,</div><div class='del'>-			 u16 token, int event);</div><div class='del'>-	int (*chk_mbox)(struct hns_roce_dev *hr_dev, unsigned long timeout);</div><div class='del'>-	int (*rst_prc_mbox)(struct hns_roce_dev *hr_dev);</div><div class='del'>-	int (*set_gid)(struct hns_roce_dev *hr_dev, u8 port, int gid_index,</div><div class='add'>+	int (*post_mbox)(struct hns_roce_dev *hr_dev,</div><div class='add'>+			 struct hns_roce_mbox_msg *mbox_msg);</div><div class='add'>+	int (*poll_mbox_done)(struct hns_roce_dev *hr_dev);</div><div class='add'>+	bool (*chk_mbox_avail)(struct hns_roce_dev *hr_dev, bool *is_busy);</div><div class='add'>+	int (*set_gid)(struct hns_roce_dev *hr_dev, int gid_index,</div><div class='ctx'> 		       const union ib_gid *gid, const struct ib_gid_attr *attr);</div><div class='del'>-	int (*set_mac)(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr);</div><div class='del'>-	void (*set_mtu)(struct hns_roce_dev *hr_dev, u8 phy_port,</div><div class='del'>-			enum ib_mtu mtu);</div><div class='del'>-	int (*write_mtpt)(void *mb_buf, struct hns_roce_mr *mr,</div><div class='del'>-			  unsigned long mtpt_idx);</div><div class='add'>+	int (*set_mac)(struct hns_roce_dev *hr_dev, u8 phy_port,</div><div class='add'>+		       const u8 *addr);</div><div class='add'>+	int (*write_mtpt)(struct hns_roce_dev *hr_dev, void *mb_buf,</div><div class='add'>+			  struct hns_roce_mr *mr);</div><div class='ctx'> 	int (*rereg_write_mtpt)(struct hns_roce_dev *hr_dev,</div><div class='del'>-				struct hns_roce_mr *mr, int flags, u32 pdn,</div><div class='del'>-				int mr_access_flags, u64 iova, u64 size,</div><div class='add'>+				struct hns_roce_mr *mr, int flags,</div><div class='ctx'> 				void *mb_buf);</div><div class='ctx'> 	int (*frmr_write_mtpt)(void *mb_buf, struct hns_roce_mr *mr);</div><div class='del'>-	int (*mw_write_mtpt)(void *mb_buf, struct hns_roce_mw *mw);</div><div class='ctx'> 	void (*write_cqc)(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 			  struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts,</div><div class='del'>-			  dma_addr_t dma_handle, int nent, u32 vector);</div><div class='add'>+			  dma_addr_t dma_handle);</div><div class='ctx'> 	int (*set_hem)(struct hns_roce_dev *hr_dev,</div><div class='del'>-		       struct hns_roce_hem_table *table, int obj, int step_idx);</div><div class='add'>+		       struct hns_roce_hem_table *table, int obj, u32 step_idx);</div><div class='ctx'> 	int (*clear_hem)(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 			 struct hns_roce_hem_table *table, int obj,</div><div class='del'>-			 int step_idx);</div><div class='del'>-	int (*query_qp)(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='del'>-			int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);</div><div class='add'>+			 u32 step_idx);</div><div class='ctx'> 	int (*modify_qp)(struct ib_qp *ibqp, const struct ib_qp_attr *attr,</div><div class='ctx'> 			 int attr_mask, enum ib_qp_state cur_state,</div><div class='del'>-			 enum ib_qp_state new_state);</div><div class='del'>-	int (*destroy_qp)(struct ib_qp *ibqp, struct ib_udata *udata);</div><div class='add'>+			 enum ib_qp_state new_state, struct ib_udata *udata);</div><div class='ctx'> 	int (*qp_flow_control_init)(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 			 struct hns_roce_qp *hr_qp);</div><div class='del'>-	int (*post_send)(struct ib_qp *ibqp, const struct ib_send_wr *wr,</div><div class='del'>-			 const struct ib_send_wr **bad_wr);</div><div class='del'>-	int (*post_recv)(struct ib_qp *qp, const struct ib_recv_wr *recv_wr,</div><div class='del'>-			 const struct ib_recv_wr **bad_recv_wr);</div><div class='del'>-	int (*req_notify_cq)(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);</div><div class='del'>-	int (*poll_cq)(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);</div><div class='del'>-	int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr,</div><div class='del'>-			struct ib_udata *udata);</div><div class='del'>-	int (*destroy_cq)(struct ib_cq *ibcq, struct ib_udata *udata);</div><div class='del'>-	int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);</div><div class='add'>+	void (*dereg_mr)(struct hns_roce_dev *hr_dev);</div><div class='ctx'> 	int (*init_eq)(struct hns_roce_dev *hr_dev);</div><div class='ctx'> 	void (*cleanup_eq)(struct hns_roce_dev *hr_dev);</div><div class='del'>-	void (*write_srqc)(struct hns_roce_dev *hr_dev,</div><div class='del'>-			   struct hns_roce_srq *srq, u32 pdn, u16 xrcd, u32 cqn,</div><div class='del'>-			   void *mb_buf, u64 *mtts_wqe, u64 *mtts_idx,</div><div class='del'>-			   dma_addr_t dma_handle_wqe,</div><div class='del'>-			   dma_addr_t dma_handle_idx);</div><div class='del'>-	int (*modify_srq)(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr,</div><div class='del'>-		       enum ib_srq_attr_mask srq_attr_mask,</div><div class='del'>-		       struct ib_udata *udata);</div><div class='del'>-	int (*query_srq)(struct ib_srq *ibsrq, struct ib_srq_attr *attr);</div><div class='del'>-	int (*post_srq_recv)(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,</div><div class='del'>-			     const struct ib_recv_wr **bad_wr);</div><div class='add'>+	int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf);</div><div class='add'>+	int (*query_cqc)(struct hns_roce_dev *hr_dev, u32 cqn, void *buffer);</div><div class='add'>+	int (*query_qpc)(struct hns_roce_dev *hr_dev, u32 qpn, void *buffer);</div><div class='add'>+	int (*query_mpt)(struct hns_roce_dev *hr_dev, u32 key, void *buffer);</div><div class='add'>+	int (*query_srqc)(struct hns_roce_dev *hr_dev, u32 srqn, void *buffer);</div><div class='add'>+	int (*query_sccc)(struct hns_roce_dev *hr_dev, u32 qpn, void *buffer);</div><div class='add'>+	int (*query_hw_counter)(struct hns_roce_dev *hr_dev,</div><div class='add'>+				u64 *stats, u32 port, int *hw_counters);</div><div class='add'>+	int (*get_dscp)(struct hns_roce_dev *hr_dev, u8 dscp,</div><div class='add'>+			u8 *tc_mode, u8 *priority);</div><div class='ctx'> 	const struct ib_device_ops *hns_roce_dev_ops;</div><div class='ctx'> 	const struct ib_device_ops *hns_roce_dev_srq_ops;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_dev {</div><div class='ctx'> 	struct ib_device	ib_dev;</div><div class='del'>-	struct platform_device  *pdev;</div><div class='ctx'> 	struct pci_dev		*pci_dev;</div><div class='ctx'> 	struct device		*dev;</div><div class='ctx'> 	struct hns_roce_uar     priv_uar;</div><div class='ctx'> 	const char		*irq_names[HNS_ROCE_MAX_IRQ_NUM];</div><div class='ctx'> 	spinlock_t		sm_lock;</div><div class='del'>-	spinlock_t		bt_cmd_lock;</div><div class='ctx'> 	bool			active;</div><div class='ctx'> 	bool			is_reset;</div><div class='ctx'> 	bool			dis_db;</div><div class='ctx'> 	unsigned long		reset_cnt;</div><div class='ctx'> 	struct hns_roce_ib_iboe iboe;</div><div class='add'>+	enum hns_roce_device_state state;</div><div class='add'>+	struct list_head	qp_list; /* list of all qps on this dev */</div><div class='add'>+	spinlock_t		qp_list_lock; /* protect qp_list */</div><div class='ctx'> </div><div class='ctx'> 	struct list_head        pgdir_list;</div><div class='ctx'> 	struct mutex            pgdir_mutex;</div><div class='ctx'> 	int			irq[HNS_ROCE_MAX_IRQ_NUM];</div><div class='ctx'> 	u8 __iomem		*reg_base;</div><div class='add'>+	void __iomem		*mem_base;</div><div class='ctx'> 	struct hns_roce_caps	caps;</div><div class='ctx'> 	struct xarray		qp_table_xa;</div><div class='ctx'> </div><div class='del'>-	unsigned char	dev_addr[HNS_ROCE_MAX_PORTS][MAC_ADDR_OCTET_NUM];</div><div class='add'>+	unsigned char	dev_addr[HNS_ROCE_MAX_PORTS][ETH_ALEN];</div><div class='ctx'> 	u64			sys_image_guid;</div><div class='ctx'> 	u32                     vendor_id;</div><div class='ctx'> 	u32                     vendor_part_id;</div><div class='hunk'>@@ -970,8 +991,9 @@ struct hns_roce_dev {</div><div class='ctx'> 	void __iomem            *priv_addr;</div><div class='ctx'> </div><div class='ctx'> 	struct hns_roce_cmdq	cmd;</div><div class='del'>-	struct hns_roce_bitmap    pd_bitmap;</div><div class='del'>-	struct hns_roce_uar_table uar_table;</div><div class='add'>+	struct hns_roce_ida pd_ida;</div><div class='add'>+	struct hns_roce_ida xrcd_ida;</div><div class='add'>+	struct hns_roce_ida uar_ida;</div><div class='ctx'> 	struct hns_roce_mr_table  mr_table;</div><div class='ctx'> 	struct hns_roce_cq_table  cq_table;</div><div class='ctx'> 	struct hns_roce_srq_table srq_table;</div><div class='hunk'>@@ -979,19 +1001,47 @@ struct hns_roce_dev {</div><div class='ctx'> 	struct hns_roce_eq_table  eq_table;</div><div class='ctx'> 	struct hns_roce_hem_table  qpc_timer_table;</div><div class='ctx'> 	struct hns_roce_hem_table  cqc_timer_table;</div><div class='add'>+	/* GMV is the memory area that the driver allocates for the hardware</div><div class='add'>+	 * to store SGID, SMAC and VLAN information.</div><div class='add'>+	 */</div><div class='add'>+	struct hns_roce_hem_table  gmv_table;</div><div class='ctx'> </div><div class='ctx'> 	int			cmd_mod;</div><div class='ctx'> 	int			loop_idc;</div><div class='ctx'> 	u32			sdb_offset;</div><div class='ctx'> 	u32			odb_offset;</div><div class='del'>-	dma_addr_t		tptr_dma_addr; /*only for hw v1*/</div><div class='del'>-	u32			tptr_size; /*only for hw v1*/</div><div class='ctx'> 	const struct hns_roce_hw *hw;</div><div class='ctx'> 	void			*priv;</div><div class='ctx'> 	struct workqueue_struct *irq_workq;</div><div class='del'>-	const struct hns_roce_dfx_hw *dfx;</div><div class='add'>+	struct work_struct ecc_work;</div><div class='add'>+	u32 func_num;</div><div class='add'>+	u32 is_vf;</div><div class='add'>+	u32 cong_algo_tmpl_id;</div><div class='add'>+	u64 dwqe_page;</div><div class='add'>+	struct hns_roce_dev_debugfs dbgfs;</div><div class='add'>+	atomic64_t *dfx_cnt;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+enum hns_roce_trace_type {</div><div class='add'>+	TRACE_SQ,</div><div class='add'>+	TRACE_RQ,</div><div class='add'>+	TRACE_SRQ,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline const char *trace_type_to_str(enum hns_roce_trace_type type)</div><div class='add'>+{</div><div class='add'>+	switch (type) {</div><div class='add'>+	case TRACE_SQ:</div><div class='add'>+		return "SQ";</div><div class='add'>+	case TRACE_RQ:</div><div class='add'>+		return "RQ";</div><div class='add'>+	case TRACE_SRQ:</div><div class='add'>+		return "SRQ";</div><div class='add'>+	default:</div><div class='add'>+		return "UNKNOWN";</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev)</div><div class='ctx'> {</div><div class='ctx'> 	return container_of(ib_dev, struct hns_roce_dev, ib_dev);</div><div class='hunk'>@@ -1008,6 +1058,11 @@ static inline struct hns_roce_pd *to_hr_pd(struct ib_pd *ibpd)</div><div class='ctx'> 	return container_of(ibpd, struct hns_roce_pd, ibpd);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static inline struct hns_roce_xrcd *to_hr_xrcd(struct ib_xrcd *ibxrcd)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibxrcd, struct hns_roce_xrcd, ibxrcd);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static inline struct hns_roce_ah *to_hr_ah(struct ib_ah *ibah)</div><div class='ctx'> {</div><div class='ctx'> 	return container_of(ibah, struct hns_roce_ah, ibah);</div><div class='hunk'>@@ -1018,11 +1073,6 @@ static inline struct hns_roce_mr *to_hr_mr(struct ib_mr *ibmr)</div><div class='ctx'> 	return container_of(ibmr, struct hns_roce_mr, ibmr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline struct hns_roce_mw *to_hr_mw(struct ib_mw *ibmw)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibmw, struct hns_roce_mw, ibmw);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static inline struct hns_roce_qp *to_hr_qp(struct ib_qp *ibqp)</div><div class='ctx'> {</div><div class='ctx'> 	return container_of(ibqp, struct hns_roce_qp, ibqp);</div><div class='hunk'>@@ -1038,37 +1088,101 @@ static inline struct hns_roce_srq *to_hr_srq(struct ib_srq *ibsrq)</div><div class='ctx'> 	return container_of(ibsrq, struct hns_roce_srq, ibsrq);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline struct hns_roce_sqp *hr_to_hr_sqp(struct hns_roce_qp *hr_qp)</div><div class='add'>+static inline struct hns_user_mmap_entry *</div><div class='add'>+to_hns_mmap(struct rdma_user_mmap_entry *rdma_entry)</div><div class='ctx'> {</div><div class='del'>-	return container_of(hr_qp, struct hns_roce_sqp, hr_qp);</div><div class='add'>+	return container_of(rdma_entry, struct hns_user_mmap_entry, rdma_entry);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static inline void hns_roce_write64_k(__le32 val[2], void __iomem *dest)</div><div class='ctx'> {</div><div class='del'>-	__raw_writeq(*(u64 *) val, dest);</div><div class='add'>+	writeq(*(u64 *)val, dest);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static inline struct hns_roce_qp</div><div class='ctx'> 	*__hns_roce_qp_lookup(struct hns_roce_dev *hr_dev, u32 qpn)</div><div class='ctx'> {</div><div class='del'>-	return xa_load(&amp;hr_dev-&gt;qp_table_xa, qpn &amp; (hr_dev-&gt;caps.num_qps - 1));</div><div class='add'>+	return xa_load(&amp;hr_dev-&gt;qp_table_xa, qpn);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline void *hns_roce_buf_offset(struct hns_roce_buf *buf, int offset)</div><div class='add'>+static inline void *hns_roce_buf_offset(struct hns_roce_buf *buf,</div><div class='add'>+					unsigned int offset)</div><div class='ctx'> {</div><div class='del'>-	u32 page_size = 1 &lt;&lt; buf-&gt;page_shift;</div><div class='add'>+	return (char *)(buf-&gt;trunk_list[offset &gt;&gt; buf-&gt;trunk_shift].buf) +</div><div class='add'>+			(offset &amp; ((1 &lt;&lt; buf-&gt;trunk_shift) - 1));</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (buf-&gt;nbufs == 1)</div><div class='del'>-		return (char *)(buf-&gt;direct.buf) + offset;</div><div class='del'>-	else</div><div class='del'>-		return (char *)(buf-&gt;page_list[offset &gt;&gt; buf-&gt;page_shift].buf) +</div><div class='del'>-		       (offset &amp; (page_size - 1));</div><div class='add'>+static inline dma_addr_t hns_roce_buf_dma_addr(struct hns_roce_buf *buf,</div><div class='add'>+					       unsigned int offset)</div><div class='add'>+{</div><div class='add'>+	return buf-&gt;trunk_list[offset &gt;&gt; buf-&gt;trunk_shift].map +</div><div class='add'>+			(offset &amp; ((1 &lt;&lt; buf-&gt;trunk_shift) - 1));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline dma_addr_t hns_roce_buf_page(struct hns_roce_buf *buf, u32 idx)</div><div class='add'>+{</div><div class='add'>+	return hns_roce_buf_dma_addr(buf, idx &lt;&lt; buf-&gt;page_shift);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int hns_roce_init_uar_table(struct hns_roce_dev *dev);</div><div class='add'>+#define hr_hw_page_align(x)		ALIGN(x, 1 &lt;&lt; HNS_HW_PAGE_SHIFT)</div><div class='add'>+</div><div class='add'>+static inline u64 to_hr_hw_page_addr(u64 addr)</div><div class='add'>+{</div><div class='add'>+	return addr &gt;&gt; HNS_HW_PAGE_SHIFT;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 to_hr_hw_page_shift(u32 page_shift)</div><div class='add'>+{</div><div class='add'>+	return page_shift - HNS_HW_PAGE_SHIFT;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 to_hr_hem_hopnum(u32 hopnum, u32 count)</div><div class='add'>+{</div><div class='add'>+	if (count &gt; 0)</div><div class='add'>+		return hopnum == HNS_ROCE_HOP_NUM_0 ? 0 : hopnum;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 to_hr_hem_entries_size(u32 count, u32 buf_shift)</div><div class='add'>+{</div><div class='add'>+	return hr_hw_page_align(count &lt;&lt; buf_shift);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 to_hr_hem_entries_count(u32 count, u32 buf_shift)</div><div class='add'>+{</div><div class='add'>+	return hr_hw_page_align(count &lt;&lt; buf_shift) &gt;&gt; buf_shift;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 to_hr_hem_entries_shift(u32 count, u32 buf_shift)</div><div class='add'>+{</div><div class='add'>+	if (!count)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	return ilog2(to_hr_hem_entries_count(count, buf_shift));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define DSCP_SHIFT 2</div><div class='add'>+</div><div class='add'>+static inline u8 get_tclass(const struct ib_global_route *grh)</div><div class='add'>+{</div><div class='add'>+	return grh-&gt;sgid_attr-&gt;gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP ?</div><div class='add'>+	       grh-&gt;traffic_class &gt;&gt; DSCP_SHIFT : grh-&gt;traffic_class;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct net_device *get_hr_netdev(struct hns_roce_dev *hr_dev,</div><div class='add'>+					       u8 port)</div><div class='add'>+{</div><div class='add'>+	return hr_dev-&gt;iboe.netdevs[port];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u8 get_hr_bus_num(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	return hr_dev-&gt;pci_dev-&gt;bus-&gt;number;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hns_roce_init_uar_table(struct hns_roce_dev *dev);</div><div class='ctx'> int hns_roce_uar_alloc(struct hns_roce_dev *dev, struct hns_roce_uar *uar);</div><div class='del'>-void hns_roce_uar_free(struct hns_roce_dev *dev, struct hns_roce_uar *uar);</div><div class='del'>-void hns_roce_cleanup_uar_table(struct hns_roce_dev *dev);</div><div class='ctx'> </div><div class='ctx'> int hns_roce_cmd_init(struct hns_roce_dev *hr_dev);</div><div class='ctx'> void hns_roce_cmd_cleanup(struct hns_roce_dev *hr_dev);</div><div class='hunk'>@@ -1077,116 +1191,107 @@ void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status,</div><div class='ctx'> int hns_roce_cmd_use_events(struct hns_roce_dev *hr_dev);</div><div class='ctx'> void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev);</div><div class='ctx'> </div><div class='del'>-int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift,</div><div class='del'>-		      struct hns_roce_mtt *mtt);</div><div class='del'>-void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev,</div><div class='del'>-			  struct hns_roce_mtt *mtt);</div><div class='del'>-int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev,</div><div class='del'>-			   struct hns_roce_mtt *mtt, struct hns_roce_buf *buf);</div><div class='del'>-</div><div class='del'>-int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev);</div><div class='del'>-int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev);</div><div class='del'>-int hns_roce_init_eq_table(struct hns_roce_dev *hr_dev);</div><div class='del'>-int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev);</div><div class='add'>+/* hns roce hw need current block and next block addr from mtt */</div><div class='add'>+#define MTT_MIN_COUNT	 2</div><div class='add'>+static inline dma_addr_t hns_roce_get_mtr_ba(struct hns_roce_mtr *mtr)</div><div class='add'>+{</div><div class='add'>+	return mtr-&gt;hem_cfg.root_ba;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,</div><div class='add'>+		      u32 offset, u64 *mtt_buf, int mtt_max);</div><div class='add'>+int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,</div><div class='add'>+			struct hns_roce_buf_attr *buf_attr,</div><div class='add'>+			unsigned int page_shift, struct ib_udata *udata,</div><div class='add'>+			unsigned long user_addr);</div><div class='add'>+void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev,</div><div class='add'>+			  struct hns_roce_mtr *mtr);</div><div class='add'>+int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,</div><div class='add'>+		     dma_addr_t *pages, unsigned int page_cnt);</div><div class='add'>+</div><div class='add'>+void hns_roce_init_pd_table(struct hns_roce_dev *hr_dev);</div><div class='add'>+void hns_roce_init_mr_table(struct hns_roce_dev *hr_dev);</div><div class='add'>+void hns_roce_init_cq_table(struct hns_roce_dev *hr_dev);</div><div class='ctx'> int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev);</div><div class='del'>-int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev);</div><div class='add'>+void hns_roce_init_srq_table(struct hns_roce_dev *hr_dev);</div><div class='add'>+void hns_roce_init_xrcd_table(struct hns_roce_dev *hr_dev);</div><div class='ctx'> </div><div class='del'>-void hns_roce_cleanup_pd_table(struct hns_roce_dev *hr_dev);</div><div class='del'>-void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev);</div><div class='del'>-void hns_roce_cleanup_eq_table(struct hns_roce_dev *hr_dev);</div><div class='ctx'> void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev);</div><div class='ctx'> void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev);</div><div class='del'>-void hns_roce_cleanup_srq_table(struct hns_roce_dev *hr_dev);</div><div class='del'>-</div><div class='del'>-int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj);</div><div class='del'>-void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj,</div><div class='del'>-			 int rr);</div><div class='del'>-int hns_roce_bitmap_init(struct hns_roce_bitmap *bitmap, u32 num, u32 mask,</div><div class='del'>-			 u32 reserved_bot, u32 resetrved_top);</div><div class='del'>-void hns_roce_bitmap_cleanup(struct hns_roce_bitmap *bitmap);</div><div class='add'>+</div><div class='ctx'> void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev);</div><div class='del'>-int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt,</div><div class='del'>-				int align, unsigned long *obj);</div><div class='del'>-void hns_roce_bitmap_free_range(struct hns_roce_bitmap *bitmap,</div><div class='del'>-				unsigned long obj, int cnt,</div><div class='del'>-				int rr);</div><div class='del'>-</div><div class='del'>-int hns_roce_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr,</div><div class='del'>-		       u32 flags, struct ib_udata *udata);</div><div class='add'>+</div><div class='add'>+int hns_roce_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr,</div><div class='add'>+		       struct ib_udata *udata);</div><div class='ctx'> int hns_roce_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);</div><div class='del'>-void hns_roce_destroy_ah(struct ib_ah *ah, u32 flags);</div><div class='add'>+static inline int hns_roce_destroy_ah(struct ib_ah *ah, u32 flags)</div><div class='add'>+{</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='ctx'> int hns_roce_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);</div><div class='del'>-void hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);</div><div class='add'>+int hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);</div><div class='ctx'> </div><div class='ctx'> struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc);</div><div class='ctx'> struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='ctx'> 				   u64 virt_addr, int access_flags,</div><div class='add'>+				   struct ib_dmah *dmah,</div><div class='ctx'> 				   struct ib_udata *udata);</div><div class='del'>-int hns_roce_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, u64 length,</div><div class='del'>-			   u64 virt_addr, int mr_access_flags, struct ib_pd *pd,</div><div class='del'>-			   struct ib_udata *udata);</div><div class='add'>+struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *mr, int flags, u64 start,</div><div class='add'>+				     u64 length, u64 virt_addr,</div><div class='add'>+				     int mr_access_flags, struct ib_pd *pd,</div><div class='add'>+				     struct ib_udata *udata);</div><div class='ctx'> struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,</div><div class='del'>-				u32 max_num_sg, struct ib_udata *udata);</div><div class='add'>+				u32 max_num_sg);</div><div class='ctx'> int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,</div><div class='ctx'> 		       unsigned int *sg_offset);</div><div class='ctx'> int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);</div><div class='del'>-int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev,</div><div class='del'>-		       struct hns_roce_cmd_mailbox *mailbox,</div><div class='del'>-		       unsigned long mpt_index);</div><div class='ctx'> unsigned long key_to_hw_index(u32 key);</div><div class='ctx'> </div><div class='del'>-struct ib_mw *hns_roce_alloc_mw(struct ib_pd *pd, enum ib_mw_type,</div><div class='del'>-				struct ib_udata *udata);</div><div class='del'>-int hns_roce_dealloc_mw(struct ib_mw *ibmw);</div><div class='add'>+void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf);</div><div class='add'>+struct hns_roce_buf *hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size,</div><div class='add'>+					u32 page_shift, u32 flags);</div><div class='ctx'> </div><div class='del'>-void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size,</div><div class='del'>-		       struct hns_roce_buf *buf);</div><div class='del'>-int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct,</div><div class='del'>-		       struct hns_roce_buf *buf, u32 page_shift);</div><div class='del'>-</div><div class='del'>-int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,</div><div class='del'>-			       struct hns_roce_mtt *mtt, struct ib_umem *umem);</div><div class='add'>+int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,</div><div class='add'>+			   int buf_cnt, struct hns_roce_buf *buf,</div><div class='add'>+			   unsigned int page_shift);</div><div class='add'>+int hns_roce_get_umem_bufs(dma_addr_t *bufs,</div><div class='add'>+			   int buf_cnt, struct ib_umem *umem,</div><div class='add'>+			   unsigned int page_shift);</div><div class='ctx'> </div><div class='ctx'> int hns_roce_create_srq(struct ib_srq *srq,</div><div class='ctx'> 			struct ib_srq_init_attr *srq_init_attr,</div><div class='ctx'> 			struct ib_udata *udata);</div><div class='del'>-int hns_roce_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr,</div><div class='del'>-			enum ib_srq_attr_mask srq_attr_mask,</div><div class='del'>-			struct ib_udata *udata);</div><div class='del'>-void hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata);</div><div class='add'>+int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata);</div><div class='add'>+</div><div class='add'>+int hns_roce_alloc_xrcd(struct ib_xrcd *ib_xrcd, struct ib_udata *udata);</div><div class='add'>+int hns_roce_dealloc_xrcd(struct ib_xrcd *ib_xrcd, struct ib_udata *udata);</div><div class='ctx'> </div><div class='del'>-struct ib_qp *hns_roce_create_qp(struct ib_pd *ib_pd,</div><div class='del'>-				 struct ib_qp_init_attr *init_attr,</div><div class='del'>-				 struct ib_udata *udata);</div><div class='add'>+int hns_roce_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *init_attr,</div><div class='add'>+		       struct ib_udata *udata);</div><div class='ctx'> int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> 		       int attr_mask, struct ib_udata *udata);</div><div class='del'>-void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n);</div><div class='del'>-void *get_send_wqe(struct hns_roce_qp *hr_qp, int n);</div><div class='del'>-void *get_send_extend_sge(struct hns_roce_qp *hr_qp, int n);</div><div class='del'>-bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,</div><div class='add'>+void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp);</div><div class='add'>+void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n);</div><div class='add'>+void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, unsigned int n);</div><div class='add'>+void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, unsigned int n);</div><div class='add'>+bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, u32 nreq,</div><div class='ctx'> 			  struct ib_cq *ib_cq);</div><div class='del'>-enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state);</div><div class='ctx'> void hns_roce_lock_cqs(struct hns_roce_cq *send_cq,</div><div class='ctx'> 		       struct hns_roce_cq *recv_cq);</div><div class='ctx'> void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,</div><div class='ctx'> 			 struct hns_roce_cq *recv_cq);</div><div class='ctx'> void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp);</div><div class='del'>-void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp);</div><div class='del'>-void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn,</div><div class='del'>-			       int cnt);</div><div class='add'>+void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,</div><div class='add'>+			 struct ib_udata *udata);</div><div class='ctx'> __be32 send_ieth(const struct ib_send_wr *wr);</div><div class='ctx'> int to_hr_qp_type(int qp_type);</div><div class='ctx'> </div><div class='del'>-struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,</div><div class='del'>-				    const struct ib_cq_init_attr *attr,</div><div class='del'>-				    struct ib_udata *udata);</div><div class='del'>-</div><div class='del'>-int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);</div><div class='del'>-void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq);</div><div class='add'>+int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		       struct uverbs_attr_bundle *attrs);</div><div class='ctx'> </div><div class='del'>-int hns_roce_db_map_user(struct hns_roce_ucontext *context,</div><div class='del'>-			 struct ib_udata *udata, unsigned long virt,</div><div class='add'>+int hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);</div><div class='add'>+int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt,</div><div class='ctx'> 			 struct hns_roce_db *db);</div><div class='ctx'> void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,</div><div class='ctx'> 			    struct hns_roce_db *db);</div><div class='hunk'>@@ -1196,12 +1301,27 @@ void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db);</div><div class='ctx'> </div><div class='ctx'> void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn);</div><div class='ctx'> void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type);</div><div class='add'>+void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp);</div><div class='ctx'> void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type);</div><div class='add'>+void hns_roce_flush_cqe(struct hns_roce_dev *hr_dev, u32 qpn);</div><div class='ctx'> void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type);</div><div class='del'>-int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index);</div><div class='add'>+void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev);</div><div class='ctx'> int hns_roce_init(struct hns_roce_dev *hr_dev);</div><div class='del'>-void hns_roce_exit(struct hns_roce_dev *hr_dev);</div><div class='add'>+void hns_roce_exit(struct hns_roce_dev *hr_dev, bool bond_cleanup);</div><div class='add'>+int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq);</div><div class='add'>+int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq);</div><div class='add'>+int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp);</div><div class='add'>+int hns_roce_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ib_qp);</div><div class='add'>+int hns_roce_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr);</div><div class='add'>+int hns_roce_fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ib_mr);</div><div class='add'>+int hns_roce_fill_res_srq_entry(struct sk_buff *msg, struct ib_srq *ib_srq);</div><div class='add'>+int hns_roce_fill_res_srq_entry_raw(struct sk_buff *msg, struct ib_srq *ib_srq);</div><div class='add'>+struct hns_user_mmap_entry *</div><div class='add'>+hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,</div><div class='add'>+				size_t length,</div><div class='add'>+				enum hns_roce_mmap_type mmap_type);</div><div class='add'>+bool check_sl_valid(struct hns_roce_dev *hr_dev, u8 sl);</div><div class='add'>+void hns_roce_put_cq_bankid_for_uctx(struct hns_roce_ucontext *uctx);</div><div class='add'>+void hns_roce_get_cq_bankid_for_uctx(struct hns_roce_ucontext *uctx);</div><div class='ctx'> </div><div class='del'>-int hns_roce_fill_res_entry(struct sk_buff *msg,</div><div class='del'>-			    struct rdma_restrack_entry *res);</div><div class='ctx'> #endif /* _HNS_ROCE_DEVICE_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c<br/>index 8e29dbb5b5fb..3d479c63b117 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_hem.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_hem.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_hem.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_hem.c</a></div><div class='hunk'>@@ -31,51 +31,77 @@</div><div class='ctx'>  * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/platform_device.h&gt;</div><div class='ctx'> #include "hns_roce_device.h"</div><div class='ctx'> #include "hns_roce_hem.h"</div><div class='ctx'> #include "hns_roce_common.h"</div><div class='ctx'> </div><div class='del'>-#define DMA_ADDR_T_SHIFT		12</div><div class='del'>-#define BT_BA_SHIFT			32</div><div class='add'>+#define HEM_INDEX_BUF			BIT(0)</div><div class='add'>+#define HEM_INDEX_L0			BIT(1)</div><div class='add'>+#define HEM_INDEX_L1			BIT(2)</div><div class='add'>+struct hns_roce_hem_index {</div><div class='add'>+	u64 buf;</div><div class='add'>+	u64 l0;</div><div class='add'>+	u64 l1;</div><div class='add'>+	u32 inited; /* indicate which index is available */</div><div class='add'>+};</div><div class='ctx'> </div><div class='ctx'> bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)</div><div class='ctx'> {</div><div class='del'>-	if ((hr_dev-&gt;caps.qpc_hop_num &amp;&amp; type == HEM_TYPE_QPC) ||</div><div class='del'>-	    (hr_dev-&gt;caps.mpt_hop_num &amp;&amp; type == HEM_TYPE_MTPT) ||</div><div class='del'>-	    (hr_dev-&gt;caps.cqc_hop_num &amp;&amp; type == HEM_TYPE_CQC) ||</div><div class='del'>-	    (hr_dev-&gt;caps.srqc_hop_num &amp;&amp; type == HEM_TYPE_SRQC) ||</div><div class='del'>-	    (hr_dev-&gt;caps.sccc_hop_num &amp;&amp; type == HEM_TYPE_SCCC) ||</div><div class='del'>-	    (hr_dev-&gt;caps.qpc_timer_hop_num &amp;&amp; type == HEM_TYPE_QPC_TIMER) ||</div><div class='del'>-	    (hr_dev-&gt;caps.cqc_timer_hop_num &amp;&amp; type == HEM_TYPE_CQC_TIMER) ||</div><div class='del'>-	    (hr_dev-&gt;caps.cqe_hop_num &amp;&amp; type == HEM_TYPE_CQE) ||</div><div class='del'>-	    (hr_dev-&gt;caps.mtt_hop_num &amp;&amp; type == HEM_TYPE_MTT) ||</div><div class='del'>-	    (hr_dev-&gt;caps.srqwqe_hop_num &amp;&amp; type == HEM_TYPE_SRQWQE) ||</div><div class='del'>-	    (hr_dev-&gt;caps.idx_hop_num &amp;&amp; type == HEM_TYPE_IDX))</div><div class='del'>-		return true;</div><div class='del'>-</div><div class='del'>-	return false;</div><div class='add'>+	int hop_num = 0;</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case HEM_TYPE_QPC:</div><div class='add'>+		hop_num = hr_dev-&gt;caps.qpc_hop_num;</div><div class='add'>+		break;</div><div class='add'>+	case HEM_TYPE_MTPT:</div><div class='add'>+		hop_num = hr_dev-&gt;caps.mpt_hop_num;</div><div class='add'>+		break;</div><div class='add'>+	case HEM_TYPE_CQC:</div><div class='add'>+		hop_num = hr_dev-&gt;caps.cqc_hop_num;</div><div class='add'>+		break;</div><div class='add'>+	case HEM_TYPE_SRQC:</div><div class='add'>+		hop_num = hr_dev-&gt;caps.srqc_hop_num;</div><div class='add'>+		break;</div><div class='add'>+	case HEM_TYPE_SCCC:</div><div class='add'>+		hop_num = hr_dev-&gt;caps.sccc_hop_num;</div><div class='add'>+		break;</div><div class='add'>+	case HEM_TYPE_QPC_TIMER:</div><div class='add'>+		hop_num = hr_dev-&gt;caps.qpc_timer_hop_num;</div><div class='add'>+		break;</div><div class='add'>+	case HEM_TYPE_CQC_TIMER:</div><div class='add'>+		hop_num = hr_dev-&gt;caps.cqc_timer_hop_num;</div><div class='add'>+		break;</div><div class='add'>+	case HEM_TYPE_GMV:</div><div class='add'>+		hop_num = hr_dev-&gt;caps.gmv_hop_num;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return hop_num;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop);</div><div class='ctx'> </div><div class='del'>-static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx,</div><div class='del'>-			    u32 bt_chunk_num)</div><div class='add'>+static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx,</div><div class='add'>+				    u32 bt_chunk_num, u64 hem_max_num)</div><div class='ctx'> {</div><div class='del'>-	int i;</div><div class='add'>+	u64 start_idx = round_down(hem_idx, bt_chunk_num);</div><div class='add'>+	u64 check_max_num = start_idx + bt_chunk_num;</div><div class='add'>+	u64 i;</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; bt_chunk_num; i++)</div><div class='del'>-		if (hem[start_idx + i])</div><div class='add'>+	for (i = start_idx; (i &lt; check_max_num) &amp;&amp; (i &lt; hem_max_num); i++)</div><div class='add'>+		if (i != hem_idx &amp;&amp; hem[i])</div><div class='ctx'> 			return false;</div><div class='ctx'> </div><div class='ctx'> 	return true;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num)</div><div class='add'>+static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num)</div><div class='ctx'> {</div><div class='add'>+	u64 start_idx = round_down(ba_idx, bt_chunk_num);</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; bt_chunk_num; i++)</div><div class='del'>-		if (bt[start_idx + i])</div><div class='add'>+		if (i != ba_idx &amp;&amp; bt[start_idx + i])</div><div class='ctx'> 			return false;</div><div class='ctx'> </div><div class='ctx'> 	return true;</div><div class='hunk'>@@ -93,17 +119,13 @@ static int hns_roce_get_bt_num(u32 table_type, u32 hop_num)</div><div class='ctx'> 		return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,</div><div class='del'>-			   struct hns_roce_hem_table *table, unsigned long *obj,</div><div class='del'>-			   struct hns_roce_hem_mhop *mhop)</div><div class='add'>+static int get_hem_table_config(struct hns_roce_dev *hr_dev,</div><div class='add'>+				struct hns_roce_hem_mhop *mhop,</div><div class='add'>+				u32 type)</div><div class='ctx'> {</div><div class='ctx'> 	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	u32 chunk_ba_num;</div><div class='del'>-	u32 table_idx;</div><div class='del'>-	u32 bt_num;</div><div class='del'>-	u32 chunk_size;</div><div class='ctx'> </div><div class='del'>-	switch (table-&gt;type) {</div><div class='add'>+	switch (type) {</div><div class='ctx'> 	case HEM_TYPE_QPC:</div><div class='ctx'> 		mhop-&gt;buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.qpc_buf_pg_sz</div><div class='ctx'> 					     + PAGE_SHIFT);</div><div class='hunk'>@@ -160,44 +182,36 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 		mhop-&gt;ba_l0_num = hr_dev-&gt;caps.srqc_bt_num;</div><div class='ctx'> 		mhop-&gt;hop_num = hr_dev-&gt;caps.srqc_hop_num;</div><div class='ctx'> 		break;</div><div class='del'>-	case HEM_TYPE_MTT:</div><div class='del'>-		mhop-&gt;buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.mtt_buf_pg_sz</div><div class='del'>-					     + PAGE_SHIFT);</div><div class='del'>-		mhop-&gt;bt_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.mtt_ba_pg_sz</div><div class='del'>-					     + PAGE_SHIFT);</div><div class='del'>-		mhop-&gt;ba_l0_num = mhop-&gt;bt_chunk_size / 8;</div><div class='del'>-		mhop-&gt;hop_num = hr_dev-&gt;caps.mtt_hop_num;</div><div class='del'>-		break;</div><div class='del'>-	case HEM_TYPE_CQE:</div><div class='del'>-		mhop-&gt;buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.cqe_buf_pg_sz</div><div class='del'>-					     + PAGE_SHIFT);</div><div class='del'>-		mhop-&gt;bt_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.cqe_ba_pg_sz</div><div class='del'>-					     + PAGE_SHIFT);</div><div class='del'>-		mhop-&gt;ba_l0_num = mhop-&gt;bt_chunk_size / 8;</div><div class='del'>-		mhop-&gt;hop_num = hr_dev-&gt;caps.cqe_hop_num;</div><div class='del'>-		break;</div><div class='del'>-	case HEM_TYPE_SRQWQE:</div><div class='del'>-		mhop-&gt;buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.srqwqe_buf_pg_sz</div><div class='del'>-					    + PAGE_SHIFT);</div><div class='del'>-		mhop-&gt;bt_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.srqwqe_ba_pg_sz</div><div class='del'>-					    + PAGE_SHIFT);</div><div class='del'>-		mhop-&gt;ba_l0_num = mhop-&gt;bt_chunk_size / 8;</div><div class='del'>-		mhop-&gt;hop_num = hr_dev-&gt;caps.srqwqe_hop_num;</div><div class='del'>-		break;</div><div class='del'>-	case HEM_TYPE_IDX:</div><div class='del'>-		mhop-&gt;buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.idx_buf_pg_sz</div><div class='del'>-				       + PAGE_SHIFT);</div><div class='del'>-		mhop-&gt;bt_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.idx_ba_pg_sz</div><div class='del'>-				       + PAGE_SHIFT);</div><div class='del'>-		mhop-&gt;ba_l0_num = mhop-&gt;bt_chunk_size / 8;</div><div class='del'>-		mhop-&gt;hop_num = hr_dev-&gt;caps.idx_hop_num;</div><div class='add'>+	case HEM_TYPE_GMV:</div><div class='add'>+		mhop-&gt;buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.gmv_buf_pg_sz +</div><div class='add'>+					     PAGE_SHIFT);</div><div class='add'>+		mhop-&gt;bt_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.gmv_ba_pg_sz +</div><div class='add'>+					    PAGE_SHIFT);</div><div class='add'>+		mhop-&gt;ba_l0_num = hr_dev-&gt;caps.gmv_bt_num;</div><div class='add'>+		mhop-&gt;hop_num = hr_dev-&gt;caps.gmv_hop_num;</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='del'>-		dev_err(dev, "Table %d not support multi-hop addressing!\n",</div><div class='del'>-			 table-&gt;type);</div><div class='add'>+		dev_err(dev, "table %u not support multi-hop addressing!\n",</div><div class='add'>+			type);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,</div><div class='add'>+			   struct hns_roce_hem_table *table, unsigned long *obj,</div><div class='add'>+			   struct hns_roce_hem_mhop *mhop)</div><div class='add'>+{</div><div class='add'>+	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	u32 chunk_ba_num;</div><div class='add'>+	u32 chunk_size;</div><div class='add'>+	u32 table_idx;</div><div class='add'>+	u32 bt_num;</div><div class='add'>+</div><div class='add'>+	if (get_hem_table_config(hr_dev, mhop, table-&gt;type))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='ctx'> 	if (!obj)</div><div class='ctx'> 		return 0;</div><div class='ctx'> </div><div class='hunk'>@@ -206,11 +220,10 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 	 * MTT/CQE alloc hem for bt pages.</div><div class='ctx'> 	 */</div><div class='ctx'> 	bt_num = hns_roce_get_bt_num(table-&gt;type, mhop-&gt;hop_num);</div><div class='del'>-	chunk_ba_num = mhop-&gt;bt_chunk_size / 8;</div><div class='add'>+	chunk_ba_num = mhop-&gt;bt_chunk_size / BA_BYTE_LEN;</div><div class='ctx'> 	chunk_size = table-&gt;type &lt; HEM_TYPE_MTT ? mhop-&gt;buf_chunk_size :</div><div class='ctx'> 			      mhop-&gt;bt_chunk_size;</div><div class='del'>-	table_idx = (*obj &amp; (table-&gt;num_obj - 1)) /</div><div class='del'>-		     (chunk_size / table-&gt;obj_size);</div><div class='add'>+	table_idx = *obj / (chunk_size / table-&gt;obj_size);</div><div class='ctx'> 	switch (bt_num) {</div><div class='ctx'> 	case 3:</div><div class='ctx'> 		mhop-&gt;l2_idx = table_idx &amp; (chunk_ba_num - 1);</div><div class='hunk'>@@ -225,8 +238,8 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 		mhop-&gt;l0_idx = table_idx;</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='del'>-		dev_err(dev, "Table %d not support hop_num = %d!\n",</div><div class='del'>-			     table-&gt;type, mhop-&gt;hop_num);</div><div class='add'>+		dev_err(dev, "table %u not support hop_num = %u!\n",</div><div class='add'>+			table-&gt;type, mhop-&gt;hop_num);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> 	if (mhop-&gt;l0_idx &gt;= mhop-&gt;ba_l0_num)</div><div class='hunk'>@@ -234,343 +247,275 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop);</div><div class='ctx'> </div><div class='ctx'> static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,</div><div class='del'>-					       int npages,</div><div class='del'>-					       unsigned long hem_alloc_size,</div><div class='del'>-					       gfp_t gfp_mask)</div><div class='add'>+					       unsigned long hem_alloc_size)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_hem_chunk *chunk = NULL;</div><div class='ctx'> 	struct hns_roce_hem *hem;</div><div class='del'>-	struct scatterlist *mem;</div><div class='ctx'> 	int order;</div><div class='ctx'> 	void *buf;</div><div class='ctx'> </div><div class='del'>-	WARN_ON(gfp_mask &amp; __GFP_HIGHMEM);</div><div class='add'>+	order = get_order(hem_alloc_size);</div><div class='add'>+	if (PAGE_SIZE &lt;&lt; order != hem_alloc_size) {</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "invalid hem_alloc_size: %lu!\n",</div><div class='add'>+			hem_alloc_size);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	hem = kmalloc(sizeof(*hem),</div><div class='del'>-		      gfp_mask &amp; ~(__GFP_HIGHMEM | __GFP_NOWARN));</div><div class='add'>+	hem = kmalloc(sizeof(*hem), GFP_KERNEL);</div><div class='ctx'> 	if (!hem)</div><div class='ctx'> 		return NULL;</div><div class='ctx'> </div><div class='del'>-	hem-&gt;refcount = 0;</div><div class='del'>-	INIT_LIST_HEAD(&amp;hem-&gt;chunk_list);</div><div class='del'>-</div><div class='del'>-	order = get_order(hem_alloc_size);</div><div class='del'>-</div><div class='del'>-	while (npages &gt; 0) {</div><div class='del'>-		if (!chunk) {</div><div class='del'>-			chunk = kmalloc(sizeof(*chunk),</div><div class='del'>-				gfp_mask &amp; ~(__GFP_HIGHMEM | __GFP_NOWARN));</div><div class='del'>-			if (!chunk)</div><div class='del'>-				goto fail;</div><div class='del'>-</div><div class='del'>-			sg_init_table(chunk-&gt;mem, HNS_ROCE_HEM_CHUNK_LEN);</div><div class='del'>-			chunk-&gt;npages = 0;</div><div class='del'>-			chunk-&gt;nsg = 0;</div><div class='del'>-			memset(chunk-&gt;buf, 0, sizeof(chunk-&gt;buf));</div><div class='del'>-			list_add_tail(&amp;chunk-&gt;list, &amp;hem-&gt;chunk_list);</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		while (1 &lt;&lt; order &gt; npages)</div><div class='del'>-			--order;</div><div class='add'>+	buf = dma_alloc_coherent(hr_dev-&gt;dev, hem_alloc_size,</div><div class='add'>+				 &amp;hem-&gt;dma, GFP_KERNEL);</div><div class='add'>+	if (!buf)</div><div class='add'>+		goto fail;</div><div class='ctx'> </div><div class='del'>-		/*</div><div class='del'>-		 * Alloc memory one time. If failed, don't alloc small block</div><div class='del'>-		 * memory, directly return fail.</div><div class='del'>-		 */</div><div class='del'>-		mem = &amp;chunk-&gt;mem[chunk-&gt;npages];</div><div class='del'>-		buf = dma_alloc_coherent(hr_dev-&gt;dev, PAGE_SIZE &lt;&lt; order,</div><div class='del'>-				&amp;sg_dma_address(mem), gfp_mask);</div><div class='del'>-		if (!buf)</div><div class='del'>-			goto fail;</div><div class='del'>-</div><div class='del'>-		chunk-&gt;buf[chunk-&gt;npages] = buf;</div><div class='del'>-		sg_dma_len(mem) = PAGE_SIZE &lt;&lt; order;</div><div class='del'>-</div><div class='del'>-		++chunk-&gt;npages;</div><div class='del'>-		++chunk-&gt;nsg;</div><div class='del'>-		npages -= 1 &lt;&lt; order;</div><div class='del'>-	}</div><div class='add'>+	hem-&gt;buf = buf;</div><div class='add'>+	hem-&gt;size = hem_alloc_size;</div><div class='ctx'> </div><div class='ctx'> 	return hem;</div><div class='ctx'> </div><div class='ctx'> fail:</div><div class='del'>-	hns_roce_free_hem(hr_dev, hem);</div><div class='add'>+	kfree(hem);</div><div class='ctx'> 	return NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_hem_chunk *chunk, *tmp;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='ctx'> 	if (!hem)</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='del'>-	list_for_each_entry_safe(chunk, tmp, &amp;hem-&gt;chunk_list, list) {</div><div class='del'>-		for (i = 0; i &lt; chunk-&gt;npages; ++i)</div><div class='del'>-			dma_free_coherent(hr_dev-&gt;dev,</div><div class='del'>-				   sg_dma_len(&amp;chunk-&gt;mem[i]),</div><div class='del'>-				   chunk-&gt;buf[i],</div><div class='del'>-				   sg_dma_address(&amp;chunk-&gt;mem[i]));</div><div class='del'>-		kfree(chunk);</div><div class='del'>-	}</div><div class='add'>+	dma_free_coherent(hr_dev-&gt;dev, hem-&gt;size, hem-&gt;buf, hem-&gt;dma);</div><div class='ctx'> </div><div class='ctx'> 	kfree(hem);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,</div><div class='del'>-			    struct hns_roce_hem_table *table, unsigned long obj)</div><div class='del'>-{</div><div class='del'>-	spinlock_t *lock = &amp;hr_dev-&gt;bt_cmd_lock;</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	unsigned long end = 0;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	struct hns_roce_hem_iter iter;</div><div class='del'>-	void __iomem *bt_cmd;</div><div class='del'>-	u32 bt_cmd_h_val = 0;</div><div class='del'>-	u32 bt_cmd_val[2];</div><div class='del'>-	u32 bt_cmd_l = 0;</div><div class='del'>-	u64 bt_ba = 0;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	/* Find the HEM(Hardware Entry Memory) entry */</div><div class='del'>-	unsigned long i = (obj &amp; (table-&gt;num_obj - 1)) /</div><div class='del'>-			  (table-&gt;table_chunk_size / table-&gt;obj_size);</div><div class='del'>-</div><div class='del'>-	switch (table-&gt;type) {</div><div class='del'>-	case HEM_TYPE_QPC:</div><div class='del'>-		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,</div><div class='del'>-			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);</div><div class='del'>-		break;</div><div class='del'>-	case HEM_TYPE_MTPT:</div><div class='del'>-		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,</div><div class='del'>-			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,</div><div class='del'>-			       HEM_TYPE_MTPT);</div><div class='del'>-		break;</div><div class='del'>-	case HEM_TYPE_CQC:</div><div class='del'>-		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,</div><div class='del'>-			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);</div><div class='del'>-		break;</div><div class='del'>-	case HEM_TYPE_SRQC:</div><div class='del'>-		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,</div><div class='del'>-			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,</div><div class='del'>-			       HEM_TYPE_SRQC);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-	roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,</div><div class='del'>-		       ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);</div><div class='del'>-	roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);</div><div class='del'>-	roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);</div><div class='del'>-</div><div class='del'>-	/* Currently iter only a chunk */</div><div class='del'>-	for (hns_roce_hem_first(table-&gt;hem[i], &amp;iter);</div><div class='del'>-	     !hns_roce_hem_last(&amp;iter); hns_roce_hem_next(&amp;iter)) {</div><div class='del'>-		bt_ba = hns_roce_hem_addr(&amp;iter) &gt;&gt; DMA_ADDR_T_SHIFT;</div><div class='del'>-</div><div class='del'>-		spin_lock_irqsave(lock, flags);</div><div class='del'>-</div><div class='del'>-		bt_cmd = hr_dev-&gt;reg_base + ROCEE_BT_CMD_H_REG;</div><div class='del'>-</div><div class='del'>-		end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;</div><div class='del'>-		while (1) {</div><div class='del'>-			if (readl(bt_cmd) &gt;&gt; BT_CMD_SYNC_SHIFT) {</div><div class='del'>-				if (!(time_before(jiffies, end))) {</div><div class='del'>-					dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");</div><div class='del'>-					spin_unlock_irqrestore(lock, flags);</div><div class='del'>-					return -EBUSY;</div><div class='del'>-				}</div><div class='del'>-			} else {</div><div class='del'>-				break;</div><div class='del'>-			}</div><div class='del'>-			mdelay(HW_SYNC_SLEEP_TIME_INTERVAL);</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		bt_cmd_l = (u32)bt_ba;</div><div class='del'>-		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,</div><div class='del'>-			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,</div><div class='del'>-			       bt_ba &gt;&gt; BT_BA_SHIFT);</div><div class='del'>-</div><div class='del'>-		bt_cmd_val[0] = bt_cmd_l;</div><div class='del'>-		bt_cmd_val[1] = bt_cmd_h_val;</div><div class='del'>-		hns_roce_write64_k(bt_cmd_val,</div><div class='del'>-				   hr_dev-&gt;reg_base + ROCEE_BT_CMD_L_REG);</div><div class='del'>-		spin_unlock_irqrestore(lock, flags);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,</div><div class='del'>-				   struct hns_roce_hem_table *table,</div><div class='del'>-				   unsigned long obj)</div><div class='add'>+static int calc_hem_config(struct hns_roce_dev *hr_dev,</div><div class='add'>+			   struct hns_roce_hem_table *table, unsigned long obj,</div><div class='add'>+			   struct hns_roce_hem_mhop *mhop,</div><div class='add'>+			   struct hns_roce_hem_index *index)</div><div class='ctx'> {</div><div class='ctx'> 	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	struct hns_roce_hem_mhop mhop;</div><div class='del'>-	struct hns_roce_hem_iter iter;</div><div class='del'>-	u32 buf_chunk_size;</div><div class='del'>-	u32 bt_chunk_size;</div><div class='add'>+	unsigned long mhop_obj = obj;</div><div class='add'>+	u32 l0_idx, l1_idx, l2_idx;</div><div class='ctx'> 	u32 chunk_ba_num;</div><div class='del'>-	u32 hop_num;</div><div class='del'>-	u32 size;</div><div class='ctx'> 	u32 bt_num;</div><div class='del'>-	u64 hem_idx;</div><div class='del'>-	u64 bt_l1_idx = 0;</div><div class='del'>-	u64 bt_l0_idx = 0;</div><div class='del'>-	u64 bt_ba;</div><div class='del'>-	unsigned long mhop_obj = obj;</div><div class='del'>-	int bt_l1_allocated = 0;</div><div class='del'>-	int bt_l0_allocated = 0;</div><div class='del'>-	int step_idx;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_calc_hem_mhop(hr_dev, table, &amp;mhop_obj, &amp;mhop);</div><div class='add'>+	ret = hns_roce_calc_hem_mhop(hr_dev, table, &amp;mhop_obj, mhop);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	buf_chunk_size = mhop.buf_chunk_size;</div><div class='del'>-	bt_chunk_size = mhop.bt_chunk_size;</div><div class='del'>-	hop_num = mhop.hop_num;</div><div class='del'>-	chunk_ba_num = bt_chunk_size / 8;</div><div class='del'>-</div><div class='del'>-	bt_num = hns_roce_get_bt_num(table-&gt;type, hop_num);</div><div class='add'>+	l0_idx = mhop-&gt;l0_idx;</div><div class='add'>+	l1_idx = mhop-&gt;l1_idx;</div><div class='add'>+	l2_idx = mhop-&gt;l2_idx;</div><div class='add'>+	chunk_ba_num = mhop-&gt;bt_chunk_size / BA_BYTE_LEN;</div><div class='add'>+	bt_num = hns_roce_get_bt_num(table-&gt;type, mhop-&gt;hop_num);</div><div class='ctx'> 	switch (bt_num) {</div><div class='ctx'> 	case 3:</div><div class='del'>-		hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +</div><div class='del'>-			  mhop.l1_idx * chunk_ba_num + mhop.l2_idx;</div><div class='del'>-		bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;</div><div class='del'>-		bt_l0_idx = mhop.l0_idx;</div><div class='add'>+		index-&gt;l1 = l0_idx * chunk_ba_num + l1_idx;</div><div class='add'>+		index-&gt;l0 = l0_idx;</div><div class='add'>+		index-&gt;buf = l0_idx * chunk_ba_num * chunk_ba_num +</div><div class='add'>+			     l1_idx * chunk_ba_num + l2_idx;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case 2:</div><div class='del'>-		hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;</div><div class='del'>-		bt_l0_idx = mhop.l0_idx;</div><div class='add'>+		index-&gt;l0 = l0_idx;</div><div class='add'>+		index-&gt;buf = l0_idx * chunk_ba_num + l1_idx;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case 1:</div><div class='del'>-		hem_idx = mhop.l0_idx;</div><div class='add'>+		index-&gt;buf = l0_idx;</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='del'>-		dev_err(dev, "Table %d not support hop_num = %d!\n",</div><div class='del'>-			     table-&gt;type, hop_num);</div><div class='add'>+		dev_err(dev, "table %u not support mhop.hop_num = %u!\n",</div><div class='add'>+			table-&gt;type, mhop-&gt;hop_num);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;table-&gt;mutex);</div><div class='add'>+	if (unlikely(index-&gt;buf &gt;= table-&gt;num_hem)) {</div><div class='add'>+		dev_err(dev, "table %u exceed hem limt idx %llu, max %lu!\n",</div><div class='add'>+			table-&gt;type, index-&gt;buf, table-&gt;num_hem);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (table-&gt;hem[hem_idx]) {</div><div class='del'>-		++table-&gt;hem[hem_idx]-&gt;refcount;</div><div class='del'>-		goto out;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void free_mhop_hem(struct hns_roce_dev *hr_dev,</div><div class='add'>+			  struct hns_roce_hem_table *table,</div><div class='add'>+			  struct hns_roce_hem_mhop *mhop,</div><div class='add'>+			  struct hns_roce_hem_index *index)</div><div class='add'>+{</div><div class='add'>+	u32 bt_size = mhop-&gt;bt_chunk_size;</div><div class='add'>+	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+</div><div class='add'>+	if (index-&gt;inited &amp; HEM_INDEX_BUF) {</div><div class='add'>+		hns_roce_free_hem(hr_dev, table-&gt;hem[index-&gt;buf]);</div><div class='add'>+		table-&gt;hem[index-&gt;buf] = NULL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (index-&gt;inited &amp; HEM_INDEX_L1) {</div><div class='add'>+		dma_free_coherent(dev, bt_size, table-&gt;bt_l1[index-&gt;l1],</div><div class='add'>+				  table-&gt;bt_l1_dma_addr[index-&gt;l1]);</div><div class='add'>+		table-&gt;bt_l1[index-&gt;l1] = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (index-&gt;inited &amp; HEM_INDEX_L0) {</div><div class='add'>+		dma_free_coherent(dev, bt_size, table-&gt;bt_l0[index-&gt;l0],</div><div class='add'>+				  table-&gt;bt_l0_dma_addr[index-&gt;l0]);</div><div class='add'>+		table-&gt;bt_l0[index-&gt;l0] = NULL;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int alloc_mhop_hem(struct hns_roce_dev *hr_dev,</div><div class='add'>+			  struct hns_roce_hem_table *table,</div><div class='add'>+			  struct hns_roce_hem_mhop *mhop,</div><div class='add'>+			  struct hns_roce_hem_index *index)</div><div class='add'>+{</div><div class='add'>+	u32 bt_size = mhop-&gt;bt_chunk_size;</div><div class='add'>+	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	u64 bt_ba;</div><div class='add'>+	u32 size;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='ctx'> 	/* alloc L1 BA's chunk */</div><div class='del'>-	if ((check_whether_bt_num_3(table-&gt;type, hop_num) ||</div><div class='del'>-		check_whether_bt_num_2(table-&gt;type, hop_num)) &amp;&amp;</div><div class='del'>-		!table-&gt;bt_l0[bt_l0_idx]) {</div><div class='del'>-		table-&gt;bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size,</div><div class='del'>-					    &amp;(table-&gt;bt_l0_dma_addr[bt_l0_idx]),</div><div class='add'>+	if ((check_whether_bt_num_3(table-&gt;type, mhop-&gt;hop_num) ||</div><div class='add'>+	     check_whether_bt_num_2(table-&gt;type, mhop-&gt;hop_num)) &amp;&amp;</div><div class='add'>+	     !table-&gt;bt_l0[index-&gt;l0]) {</div><div class='add'>+		table-&gt;bt_l0[index-&gt;l0] = dma_alloc_coherent(dev, bt_size,</div><div class='add'>+					    &amp;table-&gt;bt_l0_dma_addr[index-&gt;l0],</div><div class='ctx'> 					    GFP_KERNEL);</div><div class='del'>-		if (!table-&gt;bt_l0[bt_l0_idx]) {</div><div class='add'>+		if (!table-&gt;bt_l0[index-&gt;l0]) {</div><div class='ctx'> 			ret = -ENOMEM;</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		}</div><div class='del'>-		bt_l0_allocated = 1;</div><div class='del'>-</div><div class='del'>-		/* set base address to hardware */</div><div class='del'>-		if (table-&gt;type &lt; HEM_TYPE_MTT) {</div><div class='del'>-			step_idx = 0;</div><div class='del'>-			if (hr_dev-&gt;hw-&gt;set_hem(hr_dev, table, obj, step_idx)) {</div><div class='del'>-				ret = -ENODEV;</div><div class='del'>-				dev_err(dev, "set HEM base address to HW failed!\n");</div><div class='del'>-				goto err_dma_alloc_l1;</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='add'>+		index-&gt;inited |= HEM_INDEX_L0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* alloc L2 BA's chunk */</div><div class='del'>-	if (check_whether_bt_num_3(table-&gt;type, hop_num) &amp;&amp;</div><div class='del'>-	    !table-&gt;bt_l1[bt_l1_idx])  {</div><div class='del'>-		table-&gt;bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size,</div><div class='del'>-					    &amp;(table-&gt;bt_l1_dma_addr[bt_l1_idx]),</div><div class='add'>+	if (check_whether_bt_num_3(table-&gt;type, mhop-&gt;hop_num) &amp;&amp;</div><div class='add'>+	    !table-&gt;bt_l1[index-&gt;l1])  {</div><div class='add'>+		table-&gt;bt_l1[index-&gt;l1] = dma_alloc_coherent(dev, bt_size,</div><div class='add'>+					    &amp;table-&gt;bt_l1_dma_addr[index-&gt;l1],</div><div class='ctx'> 					    GFP_KERNEL);</div><div class='del'>-		if (!table-&gt;bt_l1[bt_l1_idx]) {</div><div class='add'>+		if (!table-&gt;bt_l1[index-&gt;l1]) {</div><div class='ctx'> 			ret = -ENOMEM;</div><div class='del'>-			goto err_dma_alloc_l1;</div><div class='del'>-		}</div><div class='del'>-		bt_l1_allocated = 1;</div><div class='del'>-		*(table-&gt;bt_l0[bt_l0_idx] + mhop.l1_idx) =</div><div class='del'>-					       table-&gt;bt_l1_dma_addr[bt_l1_idx];</div><div class='del'>-</div><div class='del'>-		/* set base address to hardware */</div><div class='del'>-		step_idx = 1;</div><div class='del'>-		if (hr_dev-&gt;hw-&gt;set_hem(hr_dev, table, obj, step_idx)) {</div><div class='del'>-			ret = -ENODEV;</div><div class='del'>-			dev_err(dev, "set HEM base address to HW failed!\n");</div><div class='del'>-			goto err_alloc_hem_buf;</div><div class='add'>+			goto err_alloc_hem;</div><div class='ctx'> 		}</div><div class='add'>+		index-&gt;inited |= HEM_INDEX_L1;</div><div class='add'>+		*(table-&gt;bt_l0[index-&gt;l0] + mhop-&gt;l1_idx) =</div><div class='add'>+					       table-&gt;bt_l1_dma_addr[index-&gt;l1];</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC.</div><div class='ctx'> 	 * alloc bt space chunk for MTT/CQE.</div><div class='ctx'> 	 */</div><div class='del'>-	size = table-&gt;type &lt; HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size;</div><div class='del'>-	table-&gt;hem[hem_idx] = hns_roce_alloc_hem(hr_dev,</div><div class='del'>-						size &gt;&gt; PAGE_SHIFT,</div><div class='del'>-						size,</div><div class='del'>-						(table-&gt;lowmem ? GFP_KERNEL :</div><div class='del'>-						GFP_HIGHUSER) | __GFP_NOWARN);</div><div class='del'>-	if (!table-&gt;hem[hem_idx]) {</div><div class='add'>+	size = table-&gt;type &lt; HEM_TYPE_MTT ? mhop-&gt;buf_chunk_size : bt_size;</div><div class='add'>+	table-&gt;hem[index-&gt;buf] = hns_roce_alloc_hem(hr_dev, size);</div><div class='add'>+	if (!table-&gt;hem[index-&gt;buf]) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='del'>-		goto err_alloc_hem_buf;</div><div class='add'>+		goto err_alloc_hem;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	hns_roce_hem_first(table-&gt;hem[hem_idx], &amp;iter);</div><div class='del'>-	bt_ba = hns_roce_hem_addr(&amp;iter);</div><div class='add'>+	index-&gt;inited |= HEM_INDEX_BUF;</div><div class='add'>+	bt_ba = table-&gt;hem[index-&gt;buf]-&gt;dma;</div><div class='ctx'> </div><div class='ctx'> 	if (table-&gt;type &lt; HEM_TYPE_MTT) {</div><div class='del'>-		if (hop_num == 2) {</div><div class='del'>-			*(table-&gt;bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba;</div><div class='del'>-			step_idx = 2;</div><div class='del'>-		} else if (hop_num == 1) {</div><div class='del'>-			*(table-&gt;bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;</div><div class='del'>-			step_idx = 1;</div><div class='del'>-		} else if (hop_num == HNS_ROCE_HOP_NUM_0) {</div><div class='del'>-			step_idx = 0;</div><div class='del'>-		} else {</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto err_dma_alloc_l1;</div><div class='add'>+		if (mhop-&gt;hop_num == 2)</div><div class='add'>+			*(table-&gt;bt_l1[index-&gt;l1] + mhop-&gt;l2_idx) = bt_ba;</div><div class='add'>+		else if (mhop-&gt;hop_num == 1)</div><div class='add'>+			*(table-&gt;bt_l0[index-&gt;l0] + mhop-&gt;l1_idx) = bt_ba;</div><div class='add'>+	} else if (mhop-&gt;hop_num == 2) {</div><div class='add'>+		*(table-&gt;bt_l0[index-&gt;l0] + mhop-&gt;l1_idx) = bt_ba;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+err_alloc_hem:</div><div class='add'>+	free_mhop_hem(hr_dev, table, mhop, index);</div><div class='add'>+out:</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int set_mhop_hem(struct hns_roce_dev *hr_dev,</div><div class='add'>+			struct hns_roce_hem_table *table, unsigned long obj,</div><div class='add'>+			struct hns_roce_hem_mhop *mhop,</div><div class='add'>+			struct hns_roce_hem_index *index)</div><div class='add'>+{</div><div class='add'>+	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	u32 step_idx;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	if (index-&gt;inited &amp; HEM_INDEX_L0) {</div><div class='add'>+		ret = hr_dev-&gt;hw-&gt;set_hem(hr_dev, table, obj, 0);</div><div class='add'>+		if (ret) {</div><div class='add'>+			dev_err(dev, "set HEM step 0 failed!\n");</div><div class='add'>+			goto out;</div><div class='ctx'> 		}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		/* set HEM base address to hardware */</div><div class='del'>-		if (hr_dev-&gt;hw-&gt;set_hem(hr_dev, table, obj, step_idx)) {</div><div class='del'>-			ret = -ENODEV;</div><div class='del'>-			dev_err(dev, "set HEM base address to HW failed!\n");</div><div class='del'>-			goto err_alloc_hem_buf;</div><div class='add'>+	if (index-&gt;inited &amp; HEM_INDEX_L1) {</div><div class='add'>+		ret = hr_dev-&gt;hw-&gt;set_hem(hr_dev, table, obj, 1);</div><div class='add'>+		if (ret) {</div><div class='add'>+			dev_err(dev, "set HEM step 1 failed!\n");</div><div class='add'>+			goto out;</div><div class='ctx'> 		}</div><div class='del'>-	} else if (hop_num == 2) {</div><div class='del'>-		*(table-&gt;bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	++table-&gt;hem[hem_idx]-&gt;refcount;</div><div class='del'>-	goto out;</div><div class='add'>+	if (index-&gt;inited &amp; HEM_INDEX_BUF) {</div><div class='add'>+		if (mhop-&gt;hop_num == HNS_ROCE_HOP_NUM_0)</div><div class='add'>+			step_idx = 0;</div><div class='add'>+		else</div><div class='add'>+			step_idx = mhop-&gt;hop_num;</div><div class='add'>+		ret = hr_dev-&gt;hw-&gt;set_hem(hr_dev, table, obj, step_idx);</div><div class='add'>+		if (ret)</div><div class='add'>+			dev_err(dev, "set HEM step last failed!\n");</div><div class='add'>+	}</div><div class='add'>+out:</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,</div><div class='add'>+				   struct hns_roce_hem_table *table,</div><div class='add'>+				   unsigned long obj)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_hem_index index = {};</div><div class='add'>+	struct hns_roce_hem_mhop mhop = {};</div><div class='add'>+	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = calc_hem_config(hr_dev, table, obj, &amp;mhop, &amp;index);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(dev, "calc hem config failed!\n");</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;table-&gt;mutex);</div><div class='add'>+	if (table-&gt;hem[index.buf]) {</div><div class='add'>+		refcount_inc(&amp;table-&gt;hem[index.buf]-&gt;refcount);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-err_alloc_hem_buf:</div><div class='del'>-	if (bt_l1_allocated) {</div><div class='del'>-		dma_free_coherent(dev, bt_chunk_size, table-&gt;bt_l1[bt_l1_idx],</div><div class='del'>-				  table-&gt;bt_l1_dma_addr[bt_l1_idx]);</div><div class='del'>-		table-&gt;bt_l1[bt_l1_idx] = NULL;</div><div class='add'>+	ret = alloc_mhop_hem(hr_dev, table, &amp;mhop, &amp;index);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(dev, "alloc mhop hem failed!\n");</div><div class='add'>+		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-err_dma_alloc_l1:</div><div class='del'>-	if (bt_l0_allocated) {</div><div class='del'>-		dma_free_coherent(dev, bt_chunk_size, table-&gt;bt_l0[bt_l0_idx],</div><div class='del'>-				  table-&gt;bt_l0_dma_addr[bt_l0_idx]);</div><div class='del'>-		table-&gt;bt_l0[bt_l0_idx] = NULL;</div><div class='add'>+	/* set HEM base address to hardware */</div><div class='add'>+	if (table-&gt;type &lt; HEM_TYPE_MTT) {</div><div class='add'>+		ret = set_mhop_hem(hr_dev, table, obj, &amp;mhop, &amp;index);</div><div class='add'>+		if (ret) {</div><div class='add'>+			dev_err(dev, "set HEM address to HW failed!\n");</div><div class='add'>+			goto err_alloc;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	refcount_set(&amp;table-&gt;hem[index.buf]-&gt;refcount, 1);</div><div class='add'>+	goto out;</div><div class='add'>+</div><div class='add'>+err_alloc:</div><div class='add'>+	free_mhop_hem(hr_dev, table, &amp;mhop, &amp;index);</div><div class='ctx'> out:</div><div class='ctx'> 	mutex_unlock(&amp;table-&gt;mutex);</div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -580,158 +525,121 @@ int hns_roce_table_get(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 		       struct hns_roce_hem_table *table, unsigned long obj)</div><div class='ctx'> {</div><div class='ctx'> 	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	int ret = 0;</div><div class='ctx'> 	unsigned long i;</div><div class='add'>+	int ret = 0;</div><div class='ctx'> </div><div class='ctx'> 	if (hns_roce_check_whether_mhop(hr_dev, table-&gt;type))</div><div class='ctx'> 		return hns_roce_table_mhop_get(hr_dev, table, obj);</div><div class='ctx'> </div><div class='del'>-	i = (obj &amp; (table-&gt;num_obj - 1)) / (table-&gt;table_chunk_size /</div><div class='del'>-	     table-&gt;obj_size);</div><div class='add'>+	i = obj / (table-&gt;table_chunk_size / table-&gt;obj_size);</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;table-&gt;mutex);</div><div class='ctx'> </div><div class='ctx'> 	if (table-&gt;hem[i]) {</div><div class='del'>-		++table-&gt;hem[i]-&gt;refcount;</div><div class='add'>+		refcount_inc(&amp;table-&gt;hem[i]-&gt;refcount);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	table-&gt;hem[i] = hns_roce_alloc_hem(hr_dev,</div><div class='del'>-				       table-&gt;table_chunk_size &gt;&gt; PAGE_SHIFT,</div><div class='del'>-				       table-&gt;table_chunk_size,</div><div class='del'>-				       (table-&gt;lowmem ? GFP_KERNEL :</div><div class='del'>-					GFP_HIGHUSER) | __GFP_NOWARN);</div><div class='add'>+	table-&gt;hem[i] = hns_roce_alloc_hem(hr_dev, table-&gt;table_chunk_size);</div><div class='ctx'> 	if (!table-&gt;hem[i]) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Set HEM base address(128K/page, pa) to Hardware */</div><div class='del'>-	if (hns_roce_set_hem(hr_dev, table, obj)) {</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT);</div><div class='add'>+	if (ret) {</div><div class='ctx'> 		hns_roce_free_hem(hr_dev, table-&gt;hem[i]);</div><div class='ctx'> 		table-&gt;hem[i] = NULL;</div><div class='del'>-		ret = -ENODEV;</div><div class='del'>-		dev_err(dev, "set HEM base address to HW failed.\n");</div><div class='add'>+		dev_err(dev, "set HEM base address to HW failed, ret = %d.\n",</div><div class='add'>+			ret);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	++table-&gt;hem[i]-&gt;refcount;</div><div class='add'>+	refcount_set(&amp;table-&gt;hem[i]-&gt;refcount, 1);</div><div class='ctx'> out:</div><div class='ctx'> 	mutex_unlock(&amp;table-&gt;mutex);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_table_get);</div><div class='ctx'> </div><div class='del'>-static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,</div><div class='del'>-				    struct hns_roce_hem_table *table,</div><div class='del'>-				    unsigned long obj,</div><div class='del'>-				    int check_refcount)</div><div class='add'>+static void clear_mhop_hem(struct hns_roce_dev *hr_dev,</div><div class='add'>+			   struct hns_roce_hem_table *table, unsigned long obj,</div><div class='add'>+			   struct hns_roce_hem_mhop *mhop,</div><div class='add'>+			   struct hns_roce_hem_index *index)</div><div class='ctx'> {</div><div class='ctx'> 	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	struct hns_roce_hem_mhop mhop;</div><div class='del'>-	unsigned long mhop_obj = obj;</div><div class='del'>-	u32 bt_chunk_size;</div><div class='add'>+	u32 hop_num = mhop-&gt;hop_num;</div><div class='ctx'> 	u32 chunk_ba_num;</div><div class='del'>-	u32 hop_num;</div><div class='del'>-	u32 start_idx;</div><div class='del'>-	u32 bt_num;</div><div class='del'>-	u64 hem_idx;</div><div class='del'>-	u64 bt_l1_idx = 0;</div><div class='add'>+	u32 step_idx;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_calc_hem_mhop(hr_dev, table, &amp;mhop_obj, &amp;mhop);</div><div class='del'>-	if (ret)</div><div class='del'>-		return;</div><div class='add'>+	index-&gt;inited = HEM_INDEX_BUF;</div><div class='add'>+	chunk_ba_num = mhop-&gt;bt_chunk_size / BA_BYTE_LEN;</div><div class='add'>+	if (check_whether_bt_num_2(table-&gt;type, hop_num)) {</div><div class='add'>+		if (hns_roce_check_hem_null(table-&gt;hem, index-&gt;buf,</div><div class='add'>+					    chunk_ba_num, table-&gt;num_hem))</div><div class='add'>+			index-&gt;inited |= HEM_INDEX_L0;</div><div class='add'>+	} else if (check_whether_bt_num_3(table-&gt;type, hop_num)) {</div><div class='add'>+		if (hns_roce_check_hem_null(table-&gt;hem, index-&gt;buf,</div><div class='add'>+					    chunk_ba_num, table-&gt;num_hem)) {</div><div class='add'>+			index-&gt;inited |= HEM_INDEX_L1;</div><div class='add'>+			if (hns_roce_check_bt_null(table-&gt;bt_l1, index-&gt;l1,</div><div class='add'>+						   chunk_ba_num))</div><div class='add'>+				index-&gt;inited |= HEM_INDEX_L0;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	bt_chunk_size = mhop.bt_chunk_size;</div><div class='del'>-	hop_num = mhop.hop_num;</div><div class='del'>-	chunk_ba_num = bt_chunk_size / 8;</div><div class='add'>+	if (table-&gt;type &lt; HEM_TYPE_MTT) {</div><div class='add'>+		if (hop_num == HNS_ROCE_HOP_NUM_0)</div><div class='add'>+			step_idx = 0;</div><div class='add'>+		else</div><div class='add'>+			step_idx = hop_num;</div><div class='ctx'> </div><div class='del'>-	bt_num = hns_roce_get_bt_num(table-&gt;type, hop_num);</div><div class='del'>-	switch (bt_num) {</div><div class='del'>-	case 3:</div><div class='del'>-		hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +</div><div class='del'>-			  mhop.l1_idx * chunk_ba_num + mhop.l2_idx;</div><div class='del'>-		bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;</div><div class='del'>-		break;</div><div class='del'>-	case 2:</div><div class='del'>-		hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;</div><div class='del'>-		break;</div><div class='del'>-	case 1:</div><div class='del'>-		hem_idx = mhop.l0_idx;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		dev_err(dev, "Table %d not support hop_num = %d!\n",</div><div class='del'>-			     table-&gt;type, hop_num);</div><div class='del'>-		return;</div><div class='add'>+		ret = hr_dev-&gt;hw-&gt;clear_hem(hr_dev, table, obj, step_idx);</div><div class='add'>+		if (ret)</div><div class='add'>+			dev_warn(dev, "failed to clear hop%u HEM, ret = %d.\n",</div><div class='add'>+				 hop_num, ret);</div><div class='add'>+</div><div class='add'>+		if (index-&gt;inited &amp; HEM_INDEX_L1) {</div><div class='add'>+			ret = hr_dev-&gt;hw-&gt;clear_hem(hr_dev, table, obj, 1);</div><div class='add'>+			if (ret)</div><div class='add'>+				dev_warn(dev, "failed to clear HEM step 1, ret = %d.\n",</div><div class='add'>+					 ret);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (index-&gt;inited &amp; HEM_INDEX_L0) {</div><div class='add'>+			ret = hr_dev-&gt;hw-&gt;clear_hem(hr_dev, table, obj, 0);</div><div class='add'>+			if (ret)</div><div class='add'>+				dev_warn(dev, "failed to clear HEM step 0, ret = %d.\n",</div><div class='add'>+					 ret);</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;table-&gt;mutex);</div><div class='add'>+static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,</div><div class='add'>+				    struct hns_roce_hem_table *table,</div><div class='add'>+				    unsigned long obj,</div><div class='add'>+				    int check_refcount)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_hem_index index = {};</div><div class='add'>+	struct hns_roce_hem_mhop mhop = {};</div><div class='add'>+	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	if (check_refcount &amp;&amp; (--table-&gt;hem[hem_idx]-&gt;refcount &gt; 0)) {</div><div class='del'>-		mutex_unlock(&amp;table-&gt;mutex);</div><div class='add'>+	ret = calc_hem_config(hr_dev, table, obj, &amp;mhop, &amp;index);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(dev, "calc hem config failed!\n");</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (table-&gt;type &lt; HEM_TYPE_MTT &amp;&amp; hop_num == 1) {</div><div class='del'>-		if (hr_dev-&gt;hw-&gt;clear_hem(hr_dev, table, obj, 1))</div><div class='del'>-			dev_warn(dev, "Clear HEM base address failed.\n");</div><div class='del'>-	} else if (table-&gt;type &lt; HEM_TYPE_MTT &amp;&amp; hop_num == 2) {</div><div class='del'>-		if (hr_dev-&gt;hw-&gt;clear_hem(hr_dev, table, obj, 2))</div><div class='del'>-			dev_warn(dev, "Clear HEM base address failed.\n");</div><div class='del'>-	} else if (table-&gt;type &lt; HEM_TYPE_MTT &amp;&amp;</div><div class='del'>-		   hop_num == HNS_ROCE_HOP_NUM_0) {</div><div class='del'>-		if (hr_dev-&gt;hw-&gt;clear_hem(hr_dev, table, obj, 0))</div><div class='del'>-			dev_warn(dev, "Clear HEM base address failed.\n");</div><div class='del'>-	}</div><div class='add'>+	if (!check_refcount)</div><div class='add'>+		mutex_lock(&amp;table-&gt;mutex);</div><div class='add'>+	else if (!refcount_dec_and_mutex_lock(&amp;table-&gt;hem[index.buf]-&gt;refcount,</div><div class='add'>+					      &amp;table-&gt;mutex))</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * free buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC.</div><div class='del'>-	 * free bt space chunk for MTT/CQE.</div><div class='del'>-	 */</div><div class='del'>-	hns_roce_free_hem(hr_dev, table-&gt;hem[hem_idx]);</div><div class='del'>-	table-&gt;hem[hem_idx] = NULL;</div><div class='del'>-</div><div class='del'>-	if (check_whether_bt_num_2(table-&gt;type, hop_num)) {</div><div class='del'>-		start_idx = mhop.l0_idx * chunk_ba_num;</div><div class='del'>-		if (hns_roce_check_hem_null(table-&gt;hem, start_idx,</div><div class='del'>-					    chunk_ba_num)) {</div><div class='del'>-			if (table-&gt;type &lt; HEM_TYPE_MTT &amp;&amp;</div><div class='del'>-			    hr_dev-&gt;hw-&gt;clear_hem(hr_dev, table, obj, 0))</div><div class='del'>-				dev_warn(dev, "Clear HEM base address failed.\n");</div><div class='del'>-</div><div class='del'>-			dma_free_coherent(dev, bt_chunk_size,</div><div class='del'>-					  table-&gt;bt_l0[mhop.l0_idx],</div><div class='del'>-					  table-&gt;bt_l0_dma_addr[mhop.l0_idx]);</div><div class='del'>-			table-&gt;bt_l0[mhop.l0_idx] = NULL;</div><div class='del'>-		}</div><div class='del'>-	} else if (check_whether_bt_num_3(table-&gt;type, hop_num)) {</div><div class='del'>-		start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +</div><div class='del'>-			    mhop.l1_idx * chunk_ba_num;</div><div class='del'>-		if (hns_roce_check_hem_null(table-&gt;hem, start_idx,</div><div class='del'>-					    chunk_ba_num)) {</div><div class='del'>-			if (hr_dev-&gt;hw-&gt;clear_hem(hr_dev, table, obj, 1))</div><div class='del'>-				dev_warn(dev, "Clear HEM base address failed.\n");</div><div class='del'>-</div><div class='del'>-			dma_free_coherent(dev, bt_chunk_size,</div><div class='del'>-					  table-&gt;bt_l1[bt_l1_idx],</div><div class='del'>-					  table-&gt;bt_l1_dma_addr[bt_l1_idx]);</div><div class='del'>-			table-&gt;bt_l1[bt_l1_idx] = NULL;</div><div class='del'>-</div><div class='del'>-			start_idx = mhop.l0_idx * chunk_ba_num;</div><div class='del'>-			if (hns_roce_check_bt_null(table-&gt;bt_l1, start_idx,</div><div class='del'>-						   chunk_ba_num)) {</div><div class='del'>-				if (hr_dev-&gt;hw-&gt;clear_hem(hr_dev, table, obj,</div><div class='del'>-							  0))</div><div class='del'>-					dev_warn(dev, "Clear HEM base address failed.\n");</div><div class='del'>-</div><div class='del'>-				dma_free_coherent(dev, bt_chunk_size,</div><div class='del'>-					    table-&gt;bt_l0[mhop.l0_idx],</div><div class='del'>-					    table-&gt;bt_l0_dma_addr[mhop.l0_idx]);</div><div class='del'>-				table-&gt;bt_l0[mhop.l0_idx] = NULL;</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='add'>+	clear_mhop_hem(hr_dev, table, obj, &amp;mhop, &amp;index);</div><div class='add'>+	free_mhop_hem(hr_dev, table, &amp;mhop, &amp;index);</div><div class='ctx'> </div><div class='ctx'> 	mutex_unlock(&amp;table-&gt;mutex);</div><div class='ctx'> }</div><div class='hunk'>@@ -741,72 +649,67 @@ void hns_roce_table_put(struct hns_roce_dev *hr_dev,</div><div class='ctx'> {</div><div class='ctx'> 	struct device *dev = hr_dev-&gt;dev;</div><div class='ctx'> 	unsigned long i;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (hns_roce_check_whether_mhop(hr_dev, table-&gt;type)) {</div><div class='ctx'> 		hns_roce_table_mhop_put(hr_dev, table, obj, 1);</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	i = (obj &amp; (table-&gt;num_obj - 1)) /</div><div class='del'>-	    (table-&gt;table_chunk_size / table-&gt;obj_size);</div><div class='add'>+	i = obj / (table-&gt;table_chunk_size / table-&gt;obj_size);</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;table-&gt;mutex);</div><div class='add'>+	if (!refcount_dec_and_mutex_lock(&amp;table-&gt;hem[i]-&gt;refcount,</div><div class='add'>+					 &amp;table-&gt;mutex))</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-	if (--table-&gt;hem[i]-&gt;refcount == 0) {</div><div class='del'>-		/* Clear HEM base address */</div><div class='del'>-		if (hr_dev-&gt;hw-&gt;clear_hem(hr_dev, table, obj, 0))</div><div class='del'>-			dev_warn(dev, "Clear HEM base address failed.\n");</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT);</div><div class='add'>+	if (ret)</div><div class='add'>+		dev_warn_ratelimited(dev, "failed to clear HEM base address, ret = %d.\n",</div><div class='add'>+				     ret);</div><div class='ctx'> </div><div class='del'>-		hns_roce_free_hem(hr_dev, table-&gt;hem[i]);</div><div class='del'>-		table-&gt;hem[i] = NULL;</div><div class='del'>-	}</div><div class='add'>+	hns_roce_free_hem(hr_dev, table-&gt;hem[i]);</div><div class='add'>+	table-&gt;hem[i] = NULL;</div><div class='ctx'> </div><div class='ctx'> 	mutex_unlock(&amp;table-&gt;mutex);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_table_put);</div><div class='ctx'> </div><div class='ctx'> void *hns_roce_table_find(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 			  struct hns_roce_hem_table *table,</div><div class='ctx'> 			  unsigned long obj, dma_addr_t *dma_handle)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_hem_chunk *chunk;</div><div class='ctx'> 	struct hns_roce_hem_mhop mhop;</div><div class='ctx'> 	struct hns_roce_hem *hem;</div><div class='del'>-	void *addr = NULL;</div><div class='ctx'> 	unsigned long mhop_obj = obj;</div><div class='ctx'> 	unsigned long obj_per_chunk;</div><div class='ctx'> 	unsigned long idx_offset;</div><div class='ctx'> 	int offset, dma_offset;</div><div class='del'>-	int length;</div><div class='del'>-	int i, j;</div><div class='add'>+	void *addr = NULL;</div><div class='ctx'> 	u32 hem_idx = 0;</div><div class='del'>-</div><div class='del'>-	if (!table-&gt;lowmem)</div><div class='del'>-		return NULL;</div><div class='add'>+	int i, j;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;table-&gt;mutex);</div><div class='ctx'> </div><div class='ctx'> 	if (!hns_roce_check_whether_mhop(hr_dev, table-&gt;type)) {</div><div class='ctx'> 		obj_per_chunk = table-&gt;table_chunk_size / table-&gt;obj_size;</div><div class='del'>-		hem = table-&gt;hem[(obj &amp; (table-&gt;num_obj - 1)) / obj_per_chunk];</div><div class='del'>-		idx_offset = (obj &amp; (table-&gt;num_obj - 1)) % obj_per_chunk;</div><div class='add'>+		hem = table-&gt;hem[obj / obj_per_chunk];</div><div class='add'>+		idx_offset = obj % obj_per_chunk;</div><div class='ctx'> 		dma_offset = offset = idx_offset * table-&gt;obj_size;</div><div class='ctx'> 	} else {</div><div class='ctx'> 		u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */</div><div class='ctx'> </div><div class='del'>-		hns_roce_calc_hem_mhop(hr_dev, table, &amp;mhop_obj, &amp;mhop);</div><div class='add'>+		if (hns_roce_calc_hem_mhop(hr_dev, table, &amp;mhop_obj, &amp;mhop))</div><div class='add'>+			goto out;</div><div class='ctx'> 		/* mtt mhop */</div><div class='ctx'> 		i = mhop.l0_idx;</div><div class='ctx'> 		j = mhop.l1_idx;</div><div class='ctx'> 		if (mhop.hop_num == 2)</div><div class='del'>-			hem_idx = i * (mhop.bt_chunk_size / 8) + j;</div><div class='add'>+			hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j;</div><div class='ctx'> 		else if (mhop.hop_num == 1 ||</div><div class='ctx'> 			 mhop.hop_num == HNS_ROCE_HOP_NUM_0)</div><div class='ctx'> 			hem_idx = i;</div><div class='ctx'> </div><div class='ctx'> 		hem = table-&gt;hem[hem_idx];</div><div class='del'>-		dma_offset = offset = (obj &amp; (table-&gt;num_obj - 1)) * seg_size %</div><div class='del'>-				       mhop.bt_chunk_size;</div><div class='add'>+		dma_offset = offset = obj * seg_size % mhop.bt_chunk_size;</div><div class='ctx'> 		if (mhop.hop_num == 2)</div><div class='ctx'> 			dma_offset = offset = 0;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -814,192 +717,49 @@ void *hns_roce_table_find(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 	if (!hem)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	list_for_each_entry(chunk, &amp;hem-&gt;chunk_list, list) {</div><div class='del'>-		for (i = 0; i &lt; chunk-&gt;npages; ++i) {</div><div class='del'>-			length = sg_dma_len(&amp;chunk-&gt;mem[i]);</div><div class='del'>-			if (dma_handle &amp;&amp; dma_offset &gt;= 0) {</div><div class='del'>-				if (length &gt; (u32)dma_offset)</div><div class='del'>-					*dma_handle = sg_dma_address(</div><div class='del'>-						&amp;chunk-&gt;mem[i]) + dma_offset;</div><div class='del'>-				dma_offset -= length;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			if (length &gt; (u32)offset) {</div><div class='del'>-				addr = chunk-&gt;buf[i] + offset;</div><div class='del'>-				goto out;</div><div class='del'>-			}</div><div class='del'>-			offset -= length;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='add'>+	*dma_handle = hem-&gt;dma + dma_offset;</div><div class='add'>+	addr = hem-&gt;buf + offset;</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='ctx'> 	mutex_unlock(&amp;table-&gt;mutex);</div><div class='ctx'> 	return addr;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_table_find);</div><div class='del'>-</div><div class='del'>-int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,</div><div class='del'>-			     struct hns_roce_hem_table *table,</div><div class='del'>-			     unsigned long start, unsigned long end)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_hem_mhop mhop;</div><div class='del'>-	unsigned long inc = table-&gt;table_chunk_size / table-&gt;obj_size;</div><div class='del'>-	unsigned long i;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if (hns_roce_check_whether_mhop(hr_dev, table-&gt;type)) {</div><div class='del'>-		hns_roce_calc_hem_mhop(hr_dev, table, NULL, &amp;mhop);</div><div class='del'>-		inc = mhop.bt_chunk_size / table-&gt;obj_size;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Allocate MTT entry memory according to chunk(128K) */</div><div class='del'>-	for (i = start; i &lt;= end; i += inc) {</div><div class='del'>-		ret = hns_roce_table_get(hr_dev, table, i);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto fail;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-fail:</div><div class='del'>-	while (i &gt; start) {</div><div class='del'>-		i -= inc;</div><div class='del'>-		hns_roce_table_put(hr_dev, table, i);</div><div class='del'>-	}</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,</div><div class='del'>-			      struct hns_roce_hem_table *table,</div><div class='del'>-			      unsigned long start, unsigned long end)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_hem_mhop mhop;</div><div class='del'>-	unsigned long inc = table-&gt;table_chunk_size / table-&gt;obj_size;</div><div class='del'>-	unsigned long i;</div><div class='del'>-</div><div class='del'>-	if (hns_roce_check_whether_mhop(hr_dev, table-&gt;type)) {</div><div class='del'>-		hns_roce_calc_hem_mhop(hr_dev, table, NULL, &amp;mhop);</div><div class='del'>-		inc = mhop.bt_chunk_size / table-&gt;obj_size;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	for (i = start; i &lt;= end; i += inc)</div><div class='del'>-		hns_roce_table_put(hr_dev, table, i);</div><div class='del'>-}</div><div class='ctx'> </div><div class='ctx'> int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 			    struct hns_roce_hem_table *table, u32 type,</div><div class='del'>-			    unsigned long obj_size, unsigned long nobj,</div><div class='del'>-			    int use_lowmem)</div><div class='add'>+			    unsigned long obj_size, unsigned long nobj)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='ctx'> 	unsigned long obj_per_chunk;</div><div class='ctx'> 	unsigned long num_hem;</div><div class='ctx'> </div><div class='ctx'> 	if (!hns_roce_check_whether_mhop(hr_dev, type)) {</div><div class='ctx'> 		table-&gt;table_chunk_size = hr_dev-&gt;caps.chunk_sz;</div><div class='ctx'> 		obj_per_chunk = table-&gt;table_chunk_size / obj_size;</div><div class='del'>-		num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;</div><div class='add'>+		num_hem = DIV_ROUND_UP(nobj, obj_per_chunk);</div><div class='ctx'> </div><div class='ctx'> 		table-&gt;hem = kcalloc(num_hem, sizeof(*table-&gt;hem), GFP_KERNEL);</div><div class='ctx'> 		if (!table-&gt;hem)</div><div class='ctx'> 			return -ENOMEM;</div><div class='ctx'> 	} else {</div><div class='add'>+		struct hns_roce_hem_mhop mhop = {};</div><div class='ctx'> 		unsigned long buf_chunk_size;</div><div class='ctx'> 		unsigned long bt_chunk_size;</div><div class='ctx'> 		unsigned long bt_chunk_num;</div><div class='del'>-		unsigned long num_bt_l0 = 0;</div><div class='add'>+		unsigned long num_bt_l0;</div><div class='ctx'> 		u32 hop_num;</div><div class='ctx'> </div><div class='del'>-		switch (type) {</div><div class='del'>-		case HEM_TYPE_QPC:</div><div class='del'>-			buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.qpc_buf_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			bt_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.qpc_ba_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			num_bt_l0 = hr_dev-&gt;caps.qpc_bt_num;</div><div class='del'>-			hop_num = hr_dev-&gt;caps.qpc_hop_num;</div><div class='del'>-			break;</div><div class='del'>-		case HEM_TYPE_MTPT:</div><div class='del'>-			buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.mpt_buf_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			bt_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.mpt_ba_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			num_bt_l0 = hr_dev-&gt;caps.mpt_bt_num;</div><div class='del'>-			hop_num = hr_dev-&gt;caps.mpt_hop_num;</div><div class='del'>-			break;</div><div class='del'>-		case HEM_TYPE_CQC:</div><div class='del'>-			buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.cqc_buf_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			bt_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.cqc_ba_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			num_bt_l0 = hr_dev-&gt;caps.cqc_bt_num;</div><div class='del'>-			hop_num = hr_dev-&gt;caps.cqc_hop_num;</div><div class='del'>-			break;</div><div class='del'>-		case HEM_TYPE_SCCC:</div><div class='del'>-			buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.sccc_buf_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			bt_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.sccc_ba_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			num_bt_l0 = hr_dev-&gt;caps.sccc_bt_num;</div><div class='del'>-			hop_num = hr_dev-&gt;caps.sccc_hop_num;</div><div class='del'>-			break;</div><div class='del'>-		case HEM_TYPE_QPC_TIMER:</div><div class='del'>-			buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.qpc_timer_buf_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			bt_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.qpc_timer_ba_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			num_bt_l0 = hr_dev-&gt;caps.qpc_timer_bt_num;</div><div class='del'>-			hop_num = hr_dev-&gt;caps.qpc_timer_hop_num;</div><div class='del'>-			break;</div><div class='del'>-		case HEM_TYPE_CQC_TIMER:</div><div class='del'>-			buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.cqc_timer_buf_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			bt_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.cqc_timer_ba_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			num_bt_l0 = hr_dev-&gt;caps.cqc_timer_bt_num;</div><div class='del'>-			hop_num = hr_dev-&gt;caps.cqc_timer_hop_num;</div><div class='del'>-			break;</div><div class='del'>-		case HEM_TYPE_SRQC:</div><div class='del'>-			buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.srqc_buf_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			bt_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.srqc_ba_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			num_bt_l0 = hr_dev-&gt;caps.srqc_bt_num;</div><div class='del'>-			hop_num = hr_dev-&gt;caps.srqc_hop_num;</div><div class='del'>-			break;</div><div class='del'>-		case HEM_TYPE_MTT:</div><div class='del'>-			buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.mtt_ba_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			bt_chunk_size = buf_chunk_size;</div><div class='del'>-			hop_num = hr_dev-&gt;caps.mtt_hop_num;</div><div class='del'>-			break;</div><div class='del'>-		case HEM_TYPE_CQE:</div><div class='del'>-			buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.cqe_ba_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			bt_chunk_size = buf_chunk_size;</div><div class='del'>-			hop_num = hr_dev-&gt;caps.cqe_hop_num;</div><div class='del'>-			break;</div><div class='del'>-		case HEM_TYPE_SRQWQE:</div><div class='del'>-			buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.srqwqe_ba_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			bt_chunk_size = buf_chunk_size;</div><div class='del'>-			hop_num = hr_dev-&gt;caps.srqwqe_hop_num;</div><div class='del'>-			break;</div><div class='del'>-		case HEM_TYPE_IDX:</div><div class='del'>-			buf_chunk_size = 1 &lt;&lt; (hr_dev-&gt;caps.idx_ba_pg_sz</div><div class='del'>-					+ PAGE_SHIFT);</div><div class='del'>-			bt_chunk_size = buf_chunk_size;</div><div class='del'>-			hop_num = hr_dev-&gt;caps.idx_hop_num;</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			dev_err(dev,</div><div class='del'>-			  "Table %d not support to init hem table here!\n",</div><div class='del'>-			  type);</div><div class='add'>+		if (get_hem_table_config(hr_dev, &amp;mhop, type))</div><div class='ctx'> 			return -EINVAL;</div><div class='del'>-		}</div><div class='add'>+</div><div class='add'>+		buf_chunk_size = mhop.buf_chunk_size;</div><div class='add'>+		bt_chunk_size = mhop.bt_chunk_size;</div><div class='add'>+		num_bt_l0 = mhop.ba_l0_num;</div><div class='add'>+		hop_num = mhop.hop_num;</div><div class='add'>+</div><div class='ctx'> 		obj_per_chunk = buf_chunk_size / obj_size;</div><div class='del'>-		num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;</div><div class='del'>-		bt_chunk_num = bt_chunk_size / 8;</div><div class='add'>+		num_hem = DIV_ROUND_UP(nobj, obj_per_chunk);</div><div class='add'>+		bt_chunk_num = bt_chunk_size / BA_BYTE_LEN;</div><div class='add'>+</div><div class='ctx'> 		if (type &gt;= HEM_TYPE_MTT)</div><div class='ctx'> 			num_bt_l0 = bt_chunk_num;</div><div class='ctx'> </div><div class='hunk'>@@ -1011,8 +771,7 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 		if (check_whether_bt_num_3(type, hop_num)) {</div><div class='ctx'> 			unsigned long num_bt_l1;</div><div class='ctx'> </div><div class='del'>-			num_bt_l1 = (num_hem + bt_chunk_num - 1) /</div><div class='del'>-					     bt_chunk_num;</div><div class='add'>+			num_bt_l1 = DIV_ROUND_UP(num_hem, bt_chunk_num);</div><div class='ctx'> 			table-&gt;bt_l1 = kcalloc(num_bt_l1,</div><div class='ctx'> 					       sizeof(*table-&gt;bt_l1),</div><div class='ctx'> 					       GFP_KERNEL);</div><div class='hunk'>@@ -1044,9 +803,7 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,</div><div class='ctx'> </div><div class='ctx'> 	table-&gt;type = type;</div><div class='ctx'> 	table-&gt;num_hem = num_hem;</div><div class='del'>-	table-&gt;num_obj = nobj;</div><div class='ctx'> 	table-&gt;obj_size = obj_size;</div><div class='del'>-	table-&gt;lowmem = use_lowmem;</div><div class='ctx'> 	mutex_init(&amp;table-&gt;mutex);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='hunk'>@@ -1076,10 +833,11 @@ static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_hem_mhop mhop;</div><div class='ctx'> 	u32 buf_chunk_size;</div><div class='del'>-	int i;</div><div class='ctx'> 	u64 obj;</div><div class='add'>+	int i;</div><div class='ctx'> </div><div class='del'>-	hns_roce_calc_hem_mhop(hr_dev, table, NULL, &amp;mhop);</div><div class='add'>+	if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &amp;mhop))</div><div class='add'>+		return;</div><div class='ctx'> 	buf_chunk_size = table-&gt;type &lt; HEM_TYPE_MTT ? mhop.buf_chunk_size :</div><div class='ctx'> 					mhop.bt_chunk_size;</div><div class='ctx'> </div><div class='hunk'>@@ -1106,33 +864,33 @@ void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,</div><div class='ctx'> {</div><div class='ctx'> 	struct device *dev = hr_dev-&gt;dev;</div><div class='ctx'> 	unsigned long i;</div><div class='add'>+	int obj;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (hns_roce_check_whether_mhop(hr_dev, table-&gt;type)) {</div><div class='ctx'> 		hns_roce_cleanup_mhop_hem_table(hr_dev, table);</div><div class='add'>+		mutex_destroy(&amp;table-&gt;mutex);</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; table-&gt;num_hem; ++i)</div><div class='ctx'> 		if (table-&gt;hem[i]) {</div><div class='del'>-			if (hr_dev-&gt;hw-&gt;clear_hem(hr_dev, table,</div><div class='del'>-			    i * table-&gt;table_chunk_size / table-&gt;obj_size, 0))</div><div class='del'>-				dev_err(dev, "Clear HEM base address failed.\n");</div><div class='add'>+			obj = i * table-&gt;table_chunk_size / table-&gt;obj_size;</div><div class='add'>+			ret = hr_dev-&gt;hw-&gt;clear_hem(hr_dev, table, obj, 0);</div><div class='add'>+			if (ret)</div><div class='add'>+				dev_err(dev, "clear HEM base address failed, ret = %d.\n",</div><div class='add'>+					ret);</div><div class='ctx'> </div><div class='ctx'> 			hns_roce_free_hem(hr_dev, table-&gt;hem[i]);</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='add'>+	mutex_destroy(&amp;table-&gt;mutex);</div><div class='ctx'> 	kfree(table-&gt;hem);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	if ((hr_dev-&gt;caps.num_idx_segs))</div><div class='del'>-		hns_roce_cleanup_hem_table(hr_dev,</div><div class='del'>-					   &amp;hr_dev-&gt;mr_table.mtt_idx_table);</div><div class='del'>-	if (hr_dev-&gt;caps.num_srqwqe_segs)</div><div class='del'>-		hns_roce_cleanup_hem_table(hr_dev,</div><div class='del'>-					   &amp;hr_dev-&gt;mr_table.mtt_srqwqe_table);</div><div class='del'>-	if (hr_dev-&gt;caps.srqc_entry_sz)</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_SRQ)</div><div class='ctx'> 		hns_roce_cleanup_hem_table(hr_dev,</div><div class='ctx'> 					   &amp;hr_dev-&gt;srq_table.table);</div><div class='ctx'> 	hns_roce_cleanup_hem_table(hr_dev, &amp;hr_dev-&gt;cq_table.table);</div><div class='hunk'>@@ -1142,17 +900,568 @@ void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)</div><div class='ctx'> 	if (hr_dev-&gt;caps.cqc_timer_entry_sz)</div><div class='ctx'> 		hns_roce_cleanup_hem_table(hr_dev,</div><div class='ctx'> 					   &amp;hr_dev-&gt;cqc_timer_table);</div><div class='del'>-	if (hr_dev-&gt;caps.sccc_entry_sz)</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)</div><div class='ctx'> 		hns_roce_cleanup_hem_table(hr_dev,</div><div class='ctx'> 					   &amp;hr_dev-&gt;qp_table.sccc_table);</div><div class='ctx'> 	if (hr_dev-&gt;caps.trrl_entry_sz)</div><div class='ctx'> 		hns_roce_cleanup_hem_table(hr_dev,</div><div class='ctx'> 					   &amp;hr_dev-&gt;qp_table.trrl_table);</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;caps.gmv_entry_sz)</div><div class='add'>+		hns_roce_cleanup_hem_table(hr_dev, &amp;hr_dev-&gt;gmv_table);</div><div class='add'>+</div><div class='ctx'> 	hns_roce_cleanup_hem_table(hr_dev, &amp;hr_dev-&gt;qp_table.irrl_table);</div><div class='ctx'> 	hns_roce_cleanup_hem_table(hr_dev, &amp;hr_dev-&gt;qp_table.qp_table);</div><div class='ctx'> 	hns_roce_cleanup_hem_table(hr_dev, &amp;hr_dev-&gt;mr_table.mtpt_table);</div><div class='del'>-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))</div><div class='del'>-		hns_roce_cleanup_hem_table(hr_dev,</div><div class='del'>-					   &amp;hr_dev-&gt;mr_table.mtt_cqe_table);</div><div class='del'>-	hns_roce_cleanup_hem_table(hr_dev, &amp;hr_dev-&gt;mr_table.mtt_table);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct hns_roce_hem_item {</div><div class='add'>+	struct list_head list; /* link all hems in the same bt level */</div><div class='add'>+	struct list_head sibling; /* link all hems in last hop for mtt */</div><div class='add'>+	void *addr;</div><div class='add'>+	dma_addr_t dma_addr;</div><div class='add'>+	size_t count; /* max ba numbers */</div><div class='add'>+	int start; /* start buf offset in this hem */</div><div class='add'>+	int end; /* end buf offset in this hem */</div><div class='add'>+	bool exist_bt;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* All HEM items are linked in a tree structure */</div><div class='add'>+struct hns_roce_hem_head {</div><div class='add'>+	struct list_head branch[HNS_ROCE_MAX_BT_REGION];</div><div class='add'>+	struct list_head root;</div><div class='add'>+	struct list_head leaf;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static struct hns_roce_hem_item *</div><div class='add'>+hem_list_alloc_item(struct hns_roce_dev *hr_dev, int start, int end, int count,</div><div class='add'>+		    bool exist_bt)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_hem_item *hem;</div><div class='add'>+</div><div class='add'>+	hem = kzalloc(sizeof(*hem), GFP_KERNEL);</div><div class='add'>+	if (!hem)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	if (exist_bt) {</div><div class='add'>+		hem-&gt;addr = dma_alloc_coherent(hr_dev-&gt;dev, count * BA_BYTE_LEN,</div><div class='add'>+					       &amp;hem-&gt;dma_addr, GFP_KERNEL);</div><div class='add'>+		if (!hem-&gt;addr) {</div><div class='add'>+			kfree(hem);</div><div class='add'>+			return NULL;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hem-&gt;exist_bt = exist_bt;</div><div class='add'>+	hem-&gt;count = count;</div><div class='add'>+	hem-&gt;start = start;</div><div class='add'>+	hem-&gt;end = end;</div><div class='add'>+	INIT_LIST_HEAD(&amp;hem-&gt;list);</div><div class='add'>+	INIT_LIST_HEAD(&amp;hem-&gt;sibling);</div><div class='add'>+</div><div class='add'>+	return hem;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hem_list_free_item(struct hns_roce_dev *hr_dev,</div><div class='add'>+			       struct hns_roce_hem_item *hem)</div><div class='add'>+{</div><div class='add'>+	if (hem-&gt;exist_bt)</div><div class='add'>+		dma_free_coherent(hr_dev-&gt;dev, hem-&gt;count * BA_BYTE_LEN,</div><div class='add'>+				  hem-&gt;addr, hem-&gt;dma_addr);</div><div class='add'>+	kfree(hem);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hem_list_free_all(struct hns_roce_dev *hr_dev,</div><div class='add'>+			      struct list_head *head)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_hem_item *hem, *temp_hem;</div><div class='add'>+</div><div class='add'>+	list_for_each_entry_safe(hem, temp_hem, head, list) {</div><div class='add'>+		list_del(&amp;hem-&gt;list);</div><div class='add'>+		hem_list_free_item(hr_dev, hem);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hem_list_link_bt(void *base_addr, u64 table_addr)</div><div class='add'>+{</div><div class='add'>+	*(u64 *)(base_addr) = table_addr;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* assign L0 table address to hem from root bt */</div><div class='add'>+static void hem_list_assign_bt(struct hns_roce_hem_item *hem, void *cpu_addr,</div><div class='add'>+			       u64 phy_addr)</div><div class='add'>+{</div><div class='add'>+	hem-&gt;addr = cpu_addr;</div><div class='add'>+	hem-&gt;dma_addr = (dma_addr_t)phy_addr;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool hem_list_page_is_in_range(struct hns_roce_hem_item *hem,</div><div class='add'>+					     int offset)</div><div class='add'>+{</div><div class='add'>+	return (hem-&gt;start &lt;= offset &amp;&amp; offset &lt;= hem-&gt;end);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct hns_roce_hem_item *hem_list_search_item(struct list_head *ba_list,</div><div class='add'>+						      int page_offset)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_hem_item *hem, *temp_hem;</div><div class='add'>+	struct hns_roce_hem_item *found = NULL;</div><div class='add'>+</div><div class='add'>+	list_for_each_entry_safe(hem, temp_hem, ba_list, list) {</div><div class='add'>+		if (hem_list_page_is_in_range(hem, page_offset)) {</div><div class='add'>+			found = hem;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return found;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool hem_list_is_bottom_bt(int hopnum, int bt_level)</div><div class='add'>+{</div><div class='add'>+	/*</div><div class='add'>+	 * hopnum    base address table levels</div><div class='add'>+	 * 0		L0(buf)</div><div class='add'>+	 * 1		L0 -&gt; buf</div><div class='add'>+	 * 2		L0 -&gt; L1 -&gt; buf</div><div class='add'>+	 * 3		L0 -&gt; L1 -&gt; L2 -&gt; buf</div><div class='add'>+	 */</div><div class='add'>+	return bt_level &gt;= (hopnum ? hopnum - 1 : hopnum);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * calc base address entries num</div><div class='add'>+ * @hopnum: num of mutihop addressing</div><div class='add'>+ * @bt_level: base address table level</div><div class='add'>+ * @unit: ba entries per bt page</div><div class='add'>+ */</div><div class='add'>+static u64 hem_list_calc_ba_range(int hopnum, int bt_level, int unit)</div><div class='add'>+{</div><div class='add'>+	u64 step;</div><div class='add'>+	int max;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (hopnum &lt;= bt_level)</div><div class='add'>+		return 0;</div><div class='add'>+	/*</div><div class='add'>+	 * hopnum  bt_level   range</div><div class='add'>+	 * 1	      0       unit</div><div class='add'>+	 * ------------</div><div class='add'>+	 * 2	      0       unit * unit</div><div class='add'>+	 * 2	      1       unit</div><div class='add'>+	 * ------------</div><div class='add'>+	 * 3	      0       unit * unit * unit</div><div class='add'>+	 * 3	      1       unit * unit</div><div class='add'>+	 * 3	      2       unit</div><div class='add'>+	 */</div><div class='add'>+	step = 1;</div><div class='add'>+	max = hopnum - bt_level;</div><div class='add'>+	for (i = 0; i &lt; max; i++)</div><div class='add'>+		step = step * unit;</div><div class='add'>+</div><div class='add'>+	return step;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * calc the root ba entries which could cover all regions</div><div class='add'>+ * @regions: buf region array</div><div class='add'>+ * @region_cnt: array size of @regions</div><div class='add'>+ * @unit: ba entries per bt page</div><div class='add'>+ */</div><div class='add'>+int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions,</div><div class='add'>+				   int region_cnt, int unit)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_buf_region *r;</div><div class='add'>+	int total = 0;</div><div class='add'>+	u64 step;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; region_cnt; i++) {</div><div class='add'>+		r = (struct hns_roce_buf_region *)&amp;regions[i];</div><div class='add'>+		/* when r-&gt;hopnum = 0, the region should not occupy root_ba. */</div><div class='add'>+		if (!r-&gt;hopnum)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		if (r-&gt;hopnum &gt; 1) {</div><div class='add'>+			step = hem_list_calc_ba_range(r-&gt;hopnum, 1, unit);</div><div class='add'>+			if (step &gt; 0)</div><div class='add'>+				total += (r-&gt;count + step - 1) / step;</div><div class='add'>+		} else {</div><div class='add'>+			total += r-&gt;count;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return total;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev,</div><div class='add'>+				 const struct hns_roce_buf_region *r, int unit,</div><div class='add'>+				 int offset, struct list_head *mid_bt,</div><div class='add'>+				 struct list_head *btm_bt)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL };</div><div class='add'>+	struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL];</div><div class='add'>+	struct hns_roce_hem_item *cur, *pre;</div><div class='add'>+	const int hopnum = r-&gt;hopnum;</div><div class='add'>+	int start_aligned;</div><div class='add'>+	int distance;</div><div class='add'>+	int ret = 0;</div><div class='add'>+	int max_ofs;</div><div class='add'>+	int level;</div><div class='add'>+	u64 step;</div><div class='add'>+	int end;</div><div class='add'>+</div><div class='add'>+	if (hopnum &lt;= 1)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (hopnum &gt; HNS_ROCE_MAX_BT_LEVEL) {</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "invalid hopnum %d!\n", hopnum);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (offset &lt; r-&gt;offset) {</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "invalid offset %d, min %u!\n",</div><div class='add'>+			offset, r-&gt;offset);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	distance = offset - r-&gt;offset;</div><div class='add'>+	max_ofs = r-&gt;offset + r-&gt;count - 1;</div><div class='add'>+	for (level = 0; level &lt; hopnum; level++)</div><div class='add'>+		INIT_LIST_HEAD(&amp;temp_list[level]);</div><div class='add'>+</div><div class='add'>+	/* config L1 bt to last bt and link them to corresponding parent */</div><div class='add'>+	for (level = 1; level &lt; hopnum; level++) {</div><div class='add'>+		if (!hem_list_is_bottom_bt(hopnum, level)) {</div><div class='add'>+			cur = hem_list_search_item(&amp;mid_bt[level], offset);</div><div class='add'>+			if (cur) {</div><div class='add'>+				hem_ptrs[level] = cur;</div><div class='add'>+				continue;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		step = hem_list_calc_ba_range(hopnum, level, unit);</div><div class='add'>+		if (step &lt; 1) {</div><div class='add'>+			ret = -EINVAL;</div><div class='add'>+			goto err_exit;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		start_aligned = (distance / step) * step + r-&gt;offset;</div><div class='add'>+		end = min_t(u64, start_aligned + step - 1, max_ofs);</div><div class='add'>+		cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit,</div><div class='add'>+					  true);</div><div class='add'>+		if (!cur) {</div><div class='add'>+			ret = -ENOMEM;</div><div class='add'>+			goto err_exit;</div><div class='add'>+		}</div><div class='add'>+		hem_ptrs[level] = cur;</div><div class='add'>+		list_add(&amp;cur-&gt;list, &amp;temp_list[level]);</div><div class='add'>+		if (hem_list_is_bottom_bt(hopnum, level))</div><div class='add'>+			list_add(&amp;cur-&gt;sibling, &amp;temp_list[0]);</div><div class='add'>+</div><div class='add'>+		/* link bt to parent bt */</div><div class='add'>+		if (level &gt; 1) {</div><div class='add'>+			pre = hem_ptrs[level - 1];</div><div class='add'>+			step = (cur-&gt;start - pre-&gt;start) / step * BA_BYTE_LEN;</div><div class='add'>+			hem_list_link_bt(pre-&gt;addr + step, cur-&gt;dma_addr);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	list_splice(&amp;temp_list[0], btm_bt);</div><div class='add'>+	for (level = 1; level &lt; hopnum; level++)</div><div class='add'>+		list_splice(&amp;temp_list[level], &amp;mid_bt[level]);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_exit:</div><div class='add'>+	for (level = 1; level &lt; hopnum; level++)</div><div class='add'>+		hem_list_free_all(hr_dev, &amp;temp_list[level]);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct hns_roce_hem_item *</div><div class='add'>+alloc_root_hem(struct hns_roce_dev *hr_dev, int unit, int *max_ba_num,</div><div class='add'>+	       const struct hns_roce_buf_region *regions, int region_cnt)</div><div class='add'>+{</div><div class='add'>+	const struct hns_roce_buf_region *r;</div><div class='add'>+	struct hns_roce_hem_item *hem;</div><div class='add'>+	int ba_num;</div><div class='add'>+	int offset;</div><div class='add'>+</div><div class='add'>+	ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit);</div><div class='add'>+	if (ba_num &lt; 1)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	if (ba_num &gt; unit)</div><div class='add'>+		return ERR_PTR(-ENOBUFS);</div><div class='add'>+</div><div class='add'>+	offset = regions[0].offset;</div><div class='add'>+	/* indicate to last region */</div><div class='add'>+	r = &amp;regions[region_cnt - 1];</div><div class='add'>+	hem = hem_list_alloc_item(hr_dev, offset, r-&gt;offset + r-&gt;count - 1,</div><div class='add'>+				  ba_num, true);</div><div class='add'>+	if (!hem)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	*max_ba_num = ba_num;</div><div class='add'>+</div><div class='add'>+	return hem;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int alloc_fake_root_bt(struct hns_roce_dev *hr_dev, void *cpu_base,</div><div class='add'>+			      u64 phy_base, const struct hns_roce_buf_region *r,</div><div class='add'>+			      struct list_head *branch_head,</div><div class='add'>+			      struct list_head *leaf_head)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_hem_item *hem;</div><div class='add'>+</div><div class='add'>+	/* This is on the has_mtt branch, if r-&gt;hopnum</div><div class='add'>+	 * is 0, there is no root_ba to reuse for the</div><div class='add'>+	 * region's fake hem, so a dma_alloc request is</div><div class='add'>+	 * necessary here.</div><div class='add'>+	 */</div><div class='add'>+	hem = hem_list_alloc_item(hr_dev, r-&gt;offset, r-&gt;offset + r-&gt;count - 1,</div><div class='add'>+				  r-&gt;count, !r-&gt;hopnum);</div><div class='add'>+	if (!hem)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	/* The root_ba can be reused only when r-&gt;hopnum &gt; 0. */</div><div class='add'>+	if (r-&gt;hopnum)</div><div class='add'>+		hem_list_assign_bt(hem, cpu_base, phy_base);</div><div class='add'>+	list_add(&amp;hem-&gt;list, branch_head);</div><div class='add'>+	list_add(&amp;hem-&gt;sibling, leaf_head);</div><div class='add'>+</div><div class='add'>+	/* If r-&gt;hopnum == 0, 0 is returned,</div><div class='add'>+	 * so that the root_bt entry is not occupied.</div><div class='add'>+	 */</div><div class='add'>+	return r-&gt;hopnum ? r-&gt;count : 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int setup_middle_bt(struct hns_roce_dev *hr_dev, void *cpu_base,</div><div class='add'>+			   int unit, const struct hns_roce_buf_region *r,</div><div class='add'>+			   const struct list_head *branch_head)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_hem_item *hem, *temp_hem;</div><div class='add'>+	int total = 0;</div><div class='add'>+	int offset;</div><div class='add'>+	u64 step;</div><div class='add'>+</div><div class='add'>+	step = hem_list_calc_ba_range(r-&gt;hopnum, 1, unit);</div><div class='add'>+	if (step &lt; 1)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	/* if exist mid bt, link L1 to L0 */</div><div class='add'>+	list_for_each_entry_safe(hem, temp_hem, branch_head, list) {</div><div class='add'>+		offset = (hem-&gt;start - r-&gt;offset) / step * BA_BYTE_LEN;</div><div class='add'>+		hem_list_link_bt(cpu_base + offset, hem-&gt;dma_addr);</div><div class='add'>+		total++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return total;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int</div><div class='add'>+setup_root_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list,</div><div class='add'>+	       int unit, int max_ba_num, struct hns_roce_hem_head *head,</div><div class='add'>+	       const struct hns_roce_buf_region *regions, int region_cnt)</div><div class='add'>+{</div><div class='add'>+	const struct hns_roce_buf_region *r;</div><div class='add'>+	struct hns_roce_hem_item *root_hem;</div><div class='add'>+	void *cpu_base;</div><div class='add'>+	u64 phy_base;</div><div class='add'>+	int i, total;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	root_hem = list_first_entry(&amp;head-&gt;root,</div><div class='add'>+				    struct hns_roce_hem_item, list);</div><div class='add'>+	if (!root_hem)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	total = 0;</div><div class='add'>+	for (i = 0; i &lt; region_cnt &amp;&amp; total &lt;= max_ba_num; i++) {</div><div class='add'>+		r = &amp;regions[i];</div><div class='add'>+		if (!r-&gt;count)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		/* all regions's mid[x][0] shared the root_bt's trunk */</div><div class='add'>+		cpu_base = root_hem-&gt;addr + total * BA_BYTE_LEN;</div><div class='add'>+		phy_base = root_hem-&gt;dma_addr + total * BA_BYTE_LEN;</div><div class='add'>+</div><div class='add'>+		/* if hopnum is 0 or 1, cut a new fake hem from the root bt</div><div class='add'>+		 * which's address share to all regions.</div><div class='add'>+		 */</div><div class='add'>+		if (hem_list_is_bottom_bt(r-&gt;hopnum, 0))</div><div class='add'>+			ret = alloc_fake_root_bt(hr_dev, cpu_base, phy_base, r,</div><div class='add'>+						 &amp;head-&gt;branch[i], &amp;head-&gt;leaf);</div><div class='add'>+		else</div><div class='add'>+			ret = setup_middle_bt(hr_dev, cpu_base, unit, r,</div><div class='add'>+					      &amp;hem_list-&gt;mid_bt[i][1]);</div><div class='add'>+</div><div class='add'>+		if (ret &lt; 0)</div><div class='add'>+			return ret;</div><div class='add'>+</div><div class='add'>+		total += ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	list_splice(&amp;head-&gt;leaf, &amp;hem_list-&gt;btm_bt);</div><div class='add'>+	list_splice(&amp;head-&gt;root, &amp;hem_list-&gt;root_bt);</div><div class='add'>+	for (i = 0; i &lt; region_cnt; i++)</div><div class='add'>+		list_splice(&amp;head-&gt;branch[i], &amp;hem_list-&gt;mid_bt[i][0]);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,</div><div class='add'>+				  struct hns_roce_hem_list *hem_list, int unit,</div><div class='add'>+				  const struct hns_roce_buf_region *regions,</div><div class='add'>+				  int region_cnt)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_hem_item *root_hem;</div><div class='add'>+	struct hns_roce_hem_head head;</div><div class='add'>+	int max_ba_num;</div><div class='add'>+	int ret;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	root_hem = hem_list_search_item(&amp;hem_list-&gt;root_bt, regions[0].offset);</div><div class='add'>+	if (root_hem)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	max_ba_num = 0;</div><div class='add'>+	root_hem = alloc_root_hem(hr_dev, unit, &amp;max_ba_num, regions,</div><div class='add'>+				  region_cnt);</div><div class='add'>+	if (IS_ERR(root_hem))</div><div class='add'>+		return PTR_ERR(root_hem);</div><div class='add'>+</div><div class='add'>+	/* List head for storing all allocated HEM items */</div><div class='add'>+	INIT_LIST_HEAD(&amp;head.root);</div><div class='add'>+	INIT_LIST_HEAD(&amp;head.leaf);</div><div class='add'>+	for (i = 0; i &lt; region_cnt; i++)</div><div class='add'>+		INIT_LIST_HEAD(&amp;head.branch[i]);</div><div class='add'>+</div><div class='add'>+	hem_list-&gt;root_ba = root_hem-&gt;dma_addr;</div><div class='add'>+	list_add(&amp;root_hem-&gt;list, &amp;head.root);</div><div class='add'>+	ret = setup_root_hem(hr_dev, hem_list, unit, max_ba_num, &amp;head, regions,</div><div class='add'>+			     region_cnt);</div><div class='add'>+	if (ret) {</div><div class='add'>+		for (i = 0; i &lt; region_cnt; i++)</div><div class='add'>+			hem_list_free_all(hr_dev, &amp;head.branch[i]);</div><div class='add'>+</div><div class='add'>+		hem_list_free_all(hr_dev, &amp;head.root);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* This is the bottom bt pages number of a 100G MR on 4K OS, assuming</div><div class='add'>+ * the bt page size is not expanded by cal_best_bt_pg_sz()</div><div class='add'>+ */</div><div class='add'>+#define RESCHED_LOOP_CNT_THRESHOLD_ON_4K 12800</div><div class='add'>+</div><div class='add'>+/* construct the base address table and link them by address hop config */</div><div class='add'>+int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,</div><div class='add'>+			      struct hns_roce_hem_list *hem_list,</div><div class='add'>+			      const struct hns_roce_buf_region *regions,</div><div class='add'>+			      int region_cnt, unsigned int bt_pg_shift)</div><div class='add'>+{</div><div class='add'>+	const struct hns_roce_buf_region *r;</div><div class='add'>+	int ofs, end;</div><div class='add'>+	int loop;</div><div class='add'>+	int unit;</div><div class='add'>+	int ret;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (region_cnt &gt; HNS_ROCE_MAX_BT_REGION) {</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "invalid region region_cnt %d!\n",</div><div class='add'>+			region_cnt);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	unit = (1 &lt;&lt; bt_pg_shift) / BA_BYTE_LEN;</div><div class='add'>+	for (i = 0; i &lt; region_cnt; i++) {</div><div class='add'>+		r = &amp;regions[i];</div><div class='add'>+		if (!r-&gt;count)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		end = r-&gt;offset + r-&gt;count;</div><div class='add'>+		for (ofs = r-&gt;offset, loop = 1; ofs &lt; end; ofs += unit, loop++) {</div><div class='add'>+			if (!(loop % RESCHED_LOOP_CNT_THRESHOLD_ON_4K))</div><div class='add'>+				cond_resched();</div><div class='add'>+</div><div class='add'>+			ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs,</div><div class='add'>+						    hem_list-&gt;mid_bt[i],</div><div class='add'>+						    &amp;hem_list-&gt;btm_bt);</div><div class='add'>+			if (ret) {</div><div class='add'>+				dev_err(hr_dev-&gt;dev,</div><div class='add'>+					"alloc hem trunk fail ret = %d!\n", ret);</div><div class='add'>+				goto err_alloc;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions,</div><div class='add'>+				     region_cnt);</div><div class='add'>+	if (ret)</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "alloc hem root fail ret = %d!\n", ret);</div><div class='add'>+	else</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+err_alloc:</div><div class='add'>+	hns_roce_hem_list_release(hr_dev, hem_list);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,</div><div class='add'>+			       struct hns_roce_hem_list *hem_list)</div><div class='add'>+{</div><div class='add'>+	int i, j;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; HNS_ROCE_MAX_BT_REGION; i++)</div><div class='add'>+		for (j = 0; j &lt; HNS_ROCE_MAX_BT_LEVEL; j++)</div><div class='add'>+			hem_list_free_all(hr_dev, &amp;hem_list-&gt;mid_bt[i][j]);</div><div class='add'>+</div><div class='add'>+	hem_list_free_all(hr_dev, &amp;hem_list-&gt;root_bt);</div><div class='add'>+	INIT_LIST_HEAD(&amp;hem_list-&gt;btm_bt);</div><div class='add'>+	hem_list-&gt;root_ba = 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list)</div><div class='add'>+{</div><div class='add'>+	int i, j;</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;hem_list-&gt;root_bt);</div><div class='add'>+	INIT_LIST_HEAD(&amp;hem_list-&gt;btm_bt);</div><div class='add'>+	for (i = 0; i &lt; HNS_ROCE_MAX_BT_REGION; i++)</div><div class='add'>+		for (j = 0; j &lt; HNS_ROCE_MAX_BT_LEVEL; j++)</div><div class='add'>+			INIT_LIST_HEAD(&amp;hem_list-&gt;mid_bt[i][j]);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,</div><div class='add'>+				 struct hns_roce_hem_list *hem_list,</div><div class='add'>+				 int offset, int *mtt_cnt)</div><div class='add'>+{</div><div class='add'>+	struct list_head *head = &amp;hem_list-&gt;btm_bt;</div><div class='add'>+	struct hns_roce_hem_item *hem, *temp_hem;</div><div class='add'>+	void *cpu_base = NULL;</div><div class='add'>+	int loop = 1;</div><div class='add'>+	int nr = 0;</div><div class='add'>+</div><div class='add'>+	list_for_each_entry_safe(hem, temp_hem, head, sibling) {</div><div class='add'>+		if (!(loop % RESCHED_LOOP_CNT_THRESHOLD_ON_4K))</div><div class='add'>+			cond_resched();</div><div class='add'>+		loop++;</div><div class='add'>+</div><div class='add'>+		if (hem_list_page_is_in_range(hem, offset)) {</div><div class='add'>+			nr = offset - hem-&gt;start;</div><div class='add'>+			cpu_base = hem-&gt;addr + nr * BA_BYTE_LEN;</div><div class='add'>+			nr = hem-&gt;end + 1 - offset;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (mtt_cnt)</div><div class='add'>+		*mtt_cnt = nr;</div><div class='add'>+</div><div class='add'>+	return cpu_base;</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.h b/drivers/infiniband/hw/hns/hns_roce_hem.h<br/>index d9d668992e49..9c415b2541af 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_hem.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_hem.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_hem.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_hem.h</a></div><div class='hunk'>@@ -34,9 +34,7 @@</div><div class='ctx'> #ifndef _HNS_ROCE_HEM_H</div><div class='ctx'> #define _HNS_ROCE_HEM_H</div><div class='ctx'> </div><div class='del'>-#define HW_SYNC_TIMEOUT_MSECS		500</div><div class='del'>-#define HW_SYNC_SLEEP_TIME_INTERVAL	20</div><div class='del'>-#define BT_CMD_SYNC_SHIFT		31</div><div class='add'>+#define HEM_HOP_STEP_DIRECT 0xff</div><div class='ctx'> </div><div class='ctx'> enum {</div><div class='ctx'> 	/* MAP HEM(Hardware Entry Memory) */</div><div class='hunk'>@@ -47,6 +45,7 @@ enum {</div><div class='ctx'> 	HEM_TYPE_SCCC,</div><div class='ctx'> 	HEM_TYPE_QPC_TIMER,</div><div class='ctx'> 	HEM_TYPE_CQC_TIMER,</div><div class='add'>+	HEM_TYPE_GMV,</div><div class='ctx'> </div><div class='ctx'> 	 /* UNMAP HEM */</div><div class='ctx'> 	HEM_TYPE_MTT,</div><div class='hunk'>@@ -57,44 +56,23 @@ enum {</div><div class='ctx'> 	HEM_TYPE_TRRL,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_HEM_CHUNK_LEN	\</div><div class='del'>-	 ((256 - sizeof(struct list_head) - 2 * sizeof(int)) /	 \</div><div class='del'>-	 (sizeof(struct scatterlist)))</div><div class='del'>-</div><div class='ctx'> #define check_whether_bt_num_3(type, hop_num) \</div><div class='del'>-	(type &lt; HEM_TYPE_MTT &amp;&amp; hop_num == 2)</div><div class='add'>+	((type) &lt; HEM_TYPE_MTT &amp;&amp; (hop_num) == 2)</div><div class='ctx'> </div><div class='ctx'> #define check_whether_bt_num_2(type, hop_num) \</div><div class='del'>-	((type &lt; HEM_TYPE_MTT &amp;&amp; hop_num == 1) || \</div><div class='del'>-	(type &gt;= HEM_TYPE_MTT &amp;&amp; hop_num == 2))</div><div class='add'>+	(((type) &lt; HEM_TYPE_MTT &amp;&amp; (hop_num) == 1) || \</div><div class='add'>+	((type) &gt;= HEM_TYPE_MTT &amp;&amp; (hop_num) == 2))</div><div class='ctx'> </div><div class='ctx'> #define check_whether_bt_num_1(type, hop_num) \</div><div class='del'>-	((type &lt; HEM_TYPE_MTT &amp;&amp; hop_num == HNS_ROCE_HOP_NUM_0) || \</div><div class='del'>-	(type &gt;= HEM_TYPE_MTT &amp;&amp; hop_num == 1) || \</div><div class='del'>-	(type &gt;= HEM_TYPE_MTT &amp;&amp; hop_num == HNS_ROCE_HOP_NUM_0))</div><div class='del'>-</div><div class='del'>-enum {</div><div class='del'>-	 HNS_ROCE_HEM_PAGE_SHIFT = 12,</div><div class='del'>-	 HNS_ROCE_HEM_PAGE_SIZE  = 1 &lt;&lt; HNS_ROCE_HEM_PAGE_SHIFT,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct hns_roce_hem_chunk {</div><div class='del'>-	struct list_head	 list;</div><div class='del'>-	int			 npages;</div><div class='del'>-	int			 nsg;</div><div class='del'>-	struct scatterlist	 mem[HNS_ROCE_HEM_CHUNK_LEN];</div><div class='del'>-	void			 *buf[HNS_ROCE_HEM_CHUNK_LEN];</div><div class='del'>-};</div><div class='add'>+	(((type) &lt; HEM_TYPE_MTT &amp;&amp; (hop_num) == HNS_ROCE_HOP_NUM_0) || \</div><div class='add'>+	((type) &gt;= HEM_TYPE_MTT &amp;&amp; (hop_num) == 1) || \</div><div class='add'>+	((type) &gt;= HEM_TYPE_MTT &amp;&amp; (hop_num) == HNS_ROCE_HOP_NUM_0))</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_hem {</div><div class='del'>-	struct list_head	 chunk_list;</div><div class='del'>-	int			 refcount;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct hns_roce_hem_iter {</div><div class='del'>-	struct hns_roce_hem		 *hem;</div><div class='del'>-	struct hns_roce_hem_chunk	 *chunk;</div><div class='del'>-	int				 page_idx;</div><div class='add'>+	void *buf;</div><div class='add'>+	dma_addr_t dma;</div><div class='add'>+	unsigned long size;</div><div class='add'>+	refcount_t refcount;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_hem_mhop {</div><div class='hunk'>@@ -102,9 +80,9 @@ struct hns_roce_hem_mhop {</div><div class='ctx'> 	u32	buf_chunk_size;</div><div class='ctx'> 	u32	bt_chunk_size;</div><div class='ctx'> 	u32	ba_l0_num;</div><div class='del'>-	u32	l0_idx;/* level 0 base address table index */</div><div class='del'>-	u32	l1_idx;/* level 1 base address table index */</div><div class='del'>-	u32	l2_idx;/* level 2 base address table index */</div><div class='add'>+	u32	l0_idx; /* level 0 base address table index */</div><div class='add'>+	u32	l1_idx; /* level 1 base address table index */</div><div class='add'>+	u32	l2_idx; /* level 2 base address table index */</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem);</div><div class='hunk'>@@ -115,16 +93,9 @@ void hns_roce_table_put(struct hns_roce_dev *hr_dev,</div><div class='ctx'> void *hns_roce_table_find(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 			  struct hns_roce_hem_table *table, unsigned long obj,</div><div class='ctx'> 			  dma_addr_t *dma_handle);</div><div class='del'>-int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,</div><div class='del'>-			     struct hns_roce_hem_table *table,</div><div class='del'>-			     unsigned long start, unsigned long end);</div><div class='del'>-void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,</div><div class='del'>-			      struct hns_roce_hem_table *table,</div><div class='del'>-			      unsigned long start, unsigned long end);</div><div class='ctx'> int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 			    struct hns_roce_hem_table *table, u32 type,</div><div class='del'>-			    unsigned long obj_size, unsigned long nobj,</div><div class='del'>-			    int use_lowmem);</div><div class='add'>+			    unsigned long obj_size, unsigned long nobj);</div><div class='ctx'> void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 				struct hns_roce_hem_table *table);</div><div class='ctx'> void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev);</div><div class='hunk'>@@ -133,38 +104,17 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 			   struct hns_roce_hem_mhop *mhop);</div><div class='ctx'> bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type);</div><div class='ctx'> </div><div class='del'>-static inline void hns_roce_hem_first(struct hns_roce_hem *hem,</div><div class='del'>-				      struct hns_roce_hem_iter *iter)</div><div class='del'>-{</div><div class='del'>-	iter-&gt;hem = hem;</div><div class='del'>-	iter-&gt;chunk = list_empty(&amp;hem-&gt;chunk_list) ? NULL :</div><div class='del'>-				 list_entry(hem-&gt;chunk_list.next,</div><div class='del'>-					    struct hns_roce_hem_chunk, list);</div><div class='del'>-	iter-&gt;page_idx = 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline int hns_roce_hem_last(struct hns_roce_hem_iter *iter)</div><div class='del'>-{</div><div class='del'>-	return !iter-&gt;chunk;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void hns_roce_hem_next(struct hns_roce_hem_iter *iter)</div><div class='del'>-{</div><div class='del'>-	if (++iter-&gt;page_idx &gt;= iter-&gt;chunk-&gt;nsg) {</div><div class='del'>-		if (iter-&gt;chunk-&gt;list.next == &amp;iter-&gt;hem-&gt;chunk_list) {</div><div class='del'>-			iter-&gt;chunk = NULL;</div><div class='del'>-			return;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		iter-&gt;chunk = list_entry(iter-&gt;chunk-&gt;list.next,</div><div class='del'>-					 struct hns_roce_hem_chunk, list);</div><div class='del'>-		iter-&gt;page_idx = 0;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline dma_addr_t hns_roce_hem_addr(struct hns_roce_hem_iter *iter)</div><div class='del'>-{</div><div class='del'>-	return sg_dma_address(&amp;iter-&gt;chunk-&gt;mem[iter-&gt;page_idx]);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-#endif /*_HNS_ROCE_HEM_H*/</div><div class='add'>+void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list);</div><div class='add'>+int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions,</div><div class='add'>+				   int region_cnt, int unit);</div><div class='add'>+int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,</div><div class='add'>+			      struct hns_roce_hem_list *hem_list,</div><div class='add'>+			      const struct hns_roce_buf_region *regions,</div><div class='add'>+			      int region_cnt, unsigned int bt_pg_shift);</div><div class='add'>+void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,</div><div class='add'>+			       struct hns_roce_hem_list *hem_list);</div><div class='add'>+void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,</div><div class='add'>+				 struct hns_roce_hem_list *hem_list,</div><div class='add'>+				 int offset, int *mtt_cnt);</div><div class='add'>+</div><div class='add'>+#endif /* _HNS_ROCE_HEM_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c<br/>deleted file mode 100644<br/>index 4c5d0f160c10..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_hw_v1.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_hw_v1.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,4734 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2016 Hisilicon Limited.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-</div><div class='del'>-#include &lt;linux/platform_device.h&gt;</div><div class='del'>-#include &lt;linux/acpi.h&gt;</div><div class='del'>-#include &lt;linux/etherdevice.h&gt;</div><div class='del'>-#include &lt;linux/interrupt.h&gt;</div><div class='del'>-#include &lt;linux/of.h&gt;</div><div class='del'>-#include &lt;linux/of_platform.h&gt;</div><div class='del'>-#include &lt;rdma/ib_umem.h&gt;</div><div class='del'>-#include "hns_roce_common.h"</div><div class='del'>-#include "hns_roce_device.h"</div><div class='del'>-#include "hns_roce_cmd.h"</div><div class='del'>-#include "hns_roce_hem.h"</div><div class='del'>-#include "hns_roce_hw_v1.h"</div><div class='del'>-</div><div class='del'>-static void set_data_seg(struct hns_roce_wqe_data_seg *dseg, struct ib_sge *sg)</div><div class='del'>-{</div><div class='del'>-	dseg-&gt;lkey = cpu_to_le32(sg-&gt;lkey);</div><div class='del'>-	dseg-&gt;addr = cpu_to_le64(sg-&gt;addr);</div><div class='del'>-	dseg-&gt;len  = cpu_to_le32(sg-&gt;length);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_raddr_seg(struct hns_roce_wqe_raddr_seg *rseg, u64 remote_addr,</div><div class='del'>-			  u32 rkey)</div><div class='del'>-{</div><div class='del'>-	rseg-&gt;raddr = cpu_to_le64(remote_addr);</div><div class='del'>-	rseg-&gt;rkey  = cpu_to_le32(rkey);</div><div class='del'>-	rseg-&gt;len   = 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_post_send(struct ib_qp *ibqp,</div><div class='del'>-				 const struct ib_send_wr *wr,</div><div class='del'>-				 const struct ib_send_wr **bad_wr)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='del'>-	struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)-&gt;ah);</div><div class='del'>-	struct hns_roce_ud_send_wqe *ud_sq_wqe = NULL;</div><div class='del'>-	struct hns_roce_wqe_ctrl_seg *ctrl = NULL;</div><div class='del'>-	struct hns_roce_wqe_data_seg *dseg = NULL;</div><div class='del'>-	struct hns_roce_qp *qp = to_hr_qp(ibqp);</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_sq_db sq_db;</div><div class='del'>-	int ps_opcode = 0, i = 0;</div><div class='del'>-	unsigned long flags = 0;</div><div class='del'>-	void *wqe = NULL;</div><div class='del'>-	u32 doorbell[2];</div><div class='del'>-	int nreq = 0;</div><div class='del'>-	u32 ind = 0;</div><div class='del'>-	int ret = 0;</div><div class='del'>-	u8 *smac;</div><div class='del'>-	int loopback;</div><div class='del'>-</div><div class='del'>-	if (unlikely(ibqp-&gt;qp_type != IB_QPT_GSI &amp;&amp;</div><div class='del'>-		ibqp-&gt;qp_type != IB_QPT_RC)) {</div><div class='del'>-		dev_err(dev, "un-supported QP type\n");</div><div class='del'>-		*bad_wr = NULL;</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;qp-&gt;sq.lock, flags);</div><div class='del'>-	ind = qp-&gt;sq_next_wqe;</div><div class='del'>-	for (nreq = 0; wr; ++nreq, wr = wr-&gt;next) {</div><div class='del'>-		if (hns_roce_wq_overflow(&amp;qp-&gt;sq, nreq, qp-&gt;ibqp.send_cq)) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			*bad_wr = wr;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (unlikely(wr-&gt;num_sge &gt; qp-&gt;sq.max_gs)) {</div><div class='del'>-			dev_err(dev, "num_sge=%d &gt; qp-&gt;sq.max_gs=%d\n",</div><div class='del'>-				wr-&gt;num_sge, qp-&gt;sq.max_gs);</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			*bad_wr = wr;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		wqe = get_send_wqe(qp, ind &amp; (qp-&gt;sq.wqe_cnt - 1));</div><div class='del'>-		qp-&gt;sq.wrid[(qp-&gt;sq.head + nreq) &amp; (qp-&gt;sq.wqe_cnt - 1)] =</div><div class='del'>-								      wr-&gt;wr_id;</div><div class='del'>-</div><div class='del'>-		/* Corresponding to the RC and RD type wqe process separately */</div><div class='del'>-		if (ibqp-&gt;qp_type == IB_QPT_GSI) {</div><div class='del'>-			ud_sq_wqe = wqe;</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;dmac_h,</div><div class='del'>-				       UD_SEND_WQE_U32_4_DMAC_0_M,</div><div class='del'>-				       UD_SEND_WQE_U32_4_DMAC_0_S,</div><div class='del'>-				       ah-&gt;av.mac[0]);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;dmac_h,</div><div class='del'>-				       UD_SEND_WQE_U32_4_DMAC_1_M,</div><div class='del'>-				       UD_SEND_WQE_U32_4_DMAC_1_S,</div><div class='del'>-				       ah-&gt;av.mac[1]);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;dmac_h,</div><div class='del'>-				       UD_SEND_WQE_U32_4_DMAC_2_M,</div><div class='del'>-				       UD_SEND_WQE_U32_4_DMAC_2_S,</div><div class='del'>-				       ah-&gt;av.mac[2]);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;dmac_h,</div><div class='del'>-				       UD_SEND_WQE_U32_4_DMAC_3_M,</div><div class='del'>-				       UD_SEND_WQE_U32_4_DMAC_3_S,</div><div class='del'>-				       ah-&gt;av.mac[3]);</div><div class='del'>-</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;u32_8,</div><div class='del'>-				       UD_SEND_WQE_U32_8_DMAC_4_M,</div><div class='del'>-				       UD_SEND_WQE_U32_8_DMAC_4_S,</div><div class='del'>-				       ah-&gt;av.mac[4]);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;u32_8,</div><div class='del'>-				       UD_SEND_WQE_U32_8_DMAC_5_M,</div><div class='del'>-				       UD_SEND_WQE_U32_8_DMAC_5_S,</div><div class='del'>-				       ah-&gt;av.mac[5]);</div><div class='del'>-</div><div class='del'>-			smac = (u8 *)hr_dev-&gt;dev_addr[qp-&gt;port];</div><div class='del'>-			loopback = ether_addr_equal_unaligned(ah-&gt;av.mac,</div><div class='del'>-							      smac) ? 1 : 0;</div><div class='del'>-			roce_set_bit(ud_sq_wqe-&gt;u32_8,</div><div class='del'>-				     UD_SEND_WQE_U32_8_LOOPBACK_INDICATOR_S,</div><div class='del'>-				     loopback);</div><div class='del'>-</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;u32_8,</div><div class='del'>-				       UD_SEND_WQE_U32_8_OPERATION_TYPE_M,</div><div class='del'>-				       UD_SEND_WQE_U32_8_OPERATION_TYPE_S,</div><div class='del'>-				       HNS_ROCE_WQE_OPCODE_SEND);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;u32_8,</div><div class='del'>-				       UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M,</div><div class='del'>-				       UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S,</div><div class='del'>-				       2);</div><div class='del'>-			roce_set_bit(ud_sq_wqe-&gt;u32_8,</div><div class='del'>-				UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S,</div><div class='del'>-				1);</div><div class='del'>-</div><div class='del'>-			ud_sq_wqe-&gt;u32_8 |= (wr-&gt;send_flags &amp; IB_SEND_SIGNALED ?</div><div class='del'>-				cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |</div><div class='del'>-				(wr-&gt;send_flags &amp; IB_SEND_SOLICITED ?</div><div class='del'>-				cpu_to_le32(HNS_ROCE_WQE_SE) : 0) |</div><div class='del'>-				((wr-&gt;opcode == IB_WR_SEND_WITH_IMM) ?</div><div class='del'>-				cpu_to_le32(HNS_ROCE_WQE_IMM) : 0);</div><div class='del'>-</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;u32_16,</div><div class='del'>-				       UD_SEND_WQE_U32_16_DEST_QP_M,</div><div class='del'>-				       UD_SEND_WQE_U32_16_DEST_QP_S,</div><div class='del'>-				       ud_wr(wr)-&gt;remote_qpn);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;u32_16,</div><div class='del'>-				       UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M,</div><div class='del'>-				       UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S,</div><div class='del'>-				       ah-&gt;av.stat_rate);</div><div class='del'>-</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;u32_36,</div><div class='del'>-				       UD_SEND_WQE_U32_36_FLOW_LABEL_M,</div><div class='del'>-				       UD_SEND_WQE_U32_36_FLOW_LABEL_S,</div><div class='del'>-				       ah-&gt;av.sl_tclass_flowlabel &amp;</div><div class='del'>-				       HNS_ROCE_FLOW_LABEL_MASK);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;u32_36,</div><div class='del'>-				      UD_SEND_WQE_U32_36_PRIORITY_M,</div><div class='del'>-				      UD_SEND_WQE_U32_36_PRIORITY_S,</div><div class='del'>-				      le32_to_cpu(ah-&gt;av.sl_tclass_flowlabel) &gt;&gt;</div><div class='del'>-				      HNS_ROCE_SL_SHIFT);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;u32_36,</div><div class='del'>-				       UD_SEND_WQE_U32_36_SGID_INDEX_M,</div><div class='del'>-				       UD_SEND_WQE_U32_36_SGID_INDEX_S,</div><div class='del'>-				       hns_get_gid_index(hr_dev, qp-&gt;phy_port,</div><div class='del'>-							 ah-&gt;av.gid_index));</div><div class='del'>-</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;u32_40,</div><div class='del'>-				       UD_SEND_WQE_U32_40_HOP_LIMIT_M,</div><div class='del'>-				       UD_SEND_WQE_U32_40_HOP_LIMIT_S,</div><div class='del'>-				       ah-&gt;av.hop_limit);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;u32_40,</div><div class='del'>-				       UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M,</div><div class='del'>-				       UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S,</div><div class='del'>-				       ah-&gt;av.sl_tclass_flowlabel &gt;&gt;</div><div class='del'>-				       HNS_ROCE_TCLASS_SHIFT);</div><div class='del'>-</div><div class='del'>-			memcpy(&amp;ud_sq_wqe-&gt;dgid[0], &amp;ah-&gt;av.dgid[0], GID_LEN);</div><div class='del'>-</div><div class='del'>-			ud_sq_wqe-&gt;va0_l =</div><div class='del'>-				       cpu_to_le32((u32)wr-&gt;sg_list[0].addr);</div><div class='del'>-			ud_sq_wqe-&gt;va0_h =</div><div class='del'>-				       cpu_to_le32((wr-&gt;sg_list[0].addr) &gt;&gt; 32);</div><div class='del'>-			ud_sq_wqe-&gt;l_key0 =</div><div class='del'>-				       cpu_to_le32(wr-&gt;sg_list[0].lkey);</div><div class='del'>-</div><div class='del'>-			ud_sq_wqe-&gt;va1_l =</div><div class='del'>-				       cpu_to_le32((u32)wr-&gt;sg_list[1].addr);</div><div class='del'>-			ud_sq_wqe-&gt;va1_h =</div><div class='del'>-				       cpu_to_le32((wr-&gt;sg_list[1].addr) &gt;&gt; 32);</div><div class='del'>-			ud_sq_wqe-&gt;l_key1 =</div><div class='del'>-				       cpu_to_le32(wr-&gt;sg_list[1].lkey);</div><div class='del'>-			ind++;</div><div class='del'>-		} else if (ibqp-&gt;qp_type == IB_QPT_RC) {</div><div class='del'>-			u32 tmp_len = 0;</div><div class='del'>-</div><div class='del'>-			ctrl = wqe;</div><div class='del'>-			memset(ctrl, 0, sizeof(struct hns_roce_wqe_ctrl_seg));</div><div class='del'>-			for (i = 0; i &lt; wr-&gt;num_sge; i++)</div><div class='del'>-				tmp_len += wr-&gt;sg_list[i].length;</div><div class='del'>-</div><div class='del'>-			ctrl-&gt;msg_length =</div><div class='del'>-			  cpu_to_le32(le32_to_cpu(ctrl-&gt;msg_length) + tmp_len);</div><div class='del'>-</div><div class='del'>-			ctrl-&gt;sgl_pa_h = 0;</div><div class='del'>-			ctrl-&gt;flag = 0;</div><div class='del'>-</div><div class='del'>-			switch (wr-&gt;opcode) {</div><div class='del'>-			case IB_WR_SEND_WITH_IMM:</div><div class='del'>-			case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='del'>-				ctrl-&gt;imm_data = wr-&gt;ex.imm_data;</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_SEND_WITH_INV:</div><div class='del'>-				ctrl-&gt;inv_key =</div><div class='del'>-					cpu_to_le32(wr-&gt;ex.invalidate_rkey);</div><div class='del'>-				break;</div><div class='del'>-			default:</div><div class='del'>-				ctrl-&gt;imm_data = 0;</div><div class='del'>-				break;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			/*Ctrl field, ctrl set type: sig, solic, imm, fence */</div><div class='del'>-			/* SO wait for conforming application scenarios */</div><div class='del'>-			ctrl-&gt;flag |= (wr-&gt;send_flags &amp; IB_SEND_SIGNALED ?</div><div class='del'>-				      cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |</div><div class='del'>-				      (wr-&gt;send_flags &amp; IB_SEND_SOLICITED ?</div><div class='del'>-				      cpu_to_le32(HNS_ROCE_WQE_SE) : 0) |</div><div class='del'>-				      ((wr-&gt;opcode == IB_WR_SEND_WITH_IMM ||</div><div class='del'>-				      wr-&gt;opcode == IB_WR_RDMA_WRITE_WITH_IMM) ?</div><div class='del'>-				      cpu_to_le32(HNS_ROCE_WQE_IMM) : 0) |</div><div class='del'>-				      (wr-&gt;send_flags &amp; IB_SEND_FENCE ?</div><div class='del'>-				      (cpu_to_le32(HNS_ROCE_WQE_FENCE)) : 0);</div><div class='del'>-</div><div class='del'>-			wqe += sizeof(struct hns_roce_wqe_ctrl_seg);</div><div class='del'>-</div><div class='del'>-			switch (wr-&gt;opcode) {</div><div class='del'>-			case IB_WR_RDMA_READ:</div><div class='del'>-				ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_READ;</div><div class='del'>-				set_raddr_seg(wqe,  rdma_wr(wr)-&gt;remote_addr,</div><div class='del'>-					       rdma_wr(wr)-&gt;rkey);</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_RDMA_WRITE:</div><div class='del'>-			case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='del'>-				ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_WRITE;</div><div class='del'>-				set_raddr_seg(wqe,  rdma_wr(wr)-&gt;remote_addr,</div><div class='del'>-					      rdma_wr(wr)-&gt;rkey);</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_SEND:</div><div class='del'>-			case IB_WR_SEND_WITH_INV:</div><div class='del'>-			case IB_WR_SEND_WITH_IMM:</div><div class='del'>-				ps_opcode = HNS_ROCE_WQE_OPCODE_SEND;</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_LOCAL_INV:</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_ATOMIC_CMP_AND_SWP:</div><div class='del'>-			case IB_WR_ATOMIC_FETCH_AND_ADD:</div><div class='del'>-			case IB_WR_LSO:</div><div class='del'>-			default:</div><div class='del'>-				ps_opcode = HNS_ROCE_WQE_OPCODE_MASK;</div><div class='del'>-				break;</div><div class='del'>-			}</div><div class='del'>-			ctrl-&gt;flag |= cpu_to_le32(ps_opcode);</div><div class='del'>-			wqe += sizeof(struct hns_roce_wqe_raddr_seg);</div><div class='del'>-</div><div class='del'>-			dseg = wqe;</div><div class='del'>-			if (wr-&gt;send_flags &amp; IB_SEND_INLINE &amp;&amp; wr-&gt;num_sge) {</div><div class='del'>-				if (le32_to_cpu(ctrl-&gt;msg_length) &gt;</div><div class='del'>-				    hr_dev-&gt;caps.max_sq_inline) {</div><div class='del'>-					ret = -EINVAL;</div><div class='del'>-					*bad_wr = wr;</div><div class='del'>-					dev_err(dev, "inline len(1-%d)=%d, illegal",</div><div class='del'>-						ctrl-&gt;msg_length,</div><div class='del'>-						hr_dev-&gt;caps.max_sq_inline);</div><div class='del'>-					goto out;</div><div class='del'>-				}</div><div class='del'>-				for (i = 0; i &lt; wr-&gt;num_sge; i++) {</div><div class='del'>-					memcpy(wqe, ((void *) (uintptr_t)</div><div class='del'>-					       wr-&gt;sg_list[i].addr),</div><div class='del'>-					       wr-&gt;sg_list[i].length);</div><div class='del'>-					wqe += wr-&gt;sg_list[i].length;</div><div class='del'>-				}</div><div class='del'>-				ctrl-&gt;flag |= cpu_to_le32(HNS_ROCE_WQE_INLINE);</div><div class='del'>-			} else {</div><div class='del'>-				/*sqe num is two */</div><div class='del'>-				for (i = 0; i &lt; wr-&gt;num_sge; i++)</div><div class='del'>-					set_data_seg(dseg + i, wr-&gt;sg_list + i);</div><div class='del'>-</div><div class='del'>-				ctrl-&gt;flag |= cpu_to_le32(wr-&gt;num_sge &lt;&lt;</div><div class='del'>-					      HNS_ROCE_WQE_SGE_NUM_BIT);</div><div class='del'>-			}</div><div class='del'>-			ind++;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-out:</div><div class='del'>-	/* Set DB return */</div><div class='del'>-	if (likely(nreq)) {</div><div class='del'>-		qp-&gt;sq.head += nreq;</div><div class='del'>-		/* Memory barrier */</div><div class='del'>-		wmb();</div><div class='del'>-</div><div class='del'>-		sq_db.u32_4 = 0;</div><div class='del'>-		sq_db.u32_8 = 0;</div><div class='del'>-		roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SQ_HEAD_M,</div><div class='del'>-			       SQ_DOORBELL_U32_4_SQ_HEAD_S,</div><div class='del'>-			      (qp-&gt;sq.head &amp; ((qp-&gt;sq.wqe_cnt &lt;&lt; 1) - 1)));</div><div class='del'>-		roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SL_M,</div><div class='del'>-			       SQ_DOORBELL_U32_4_SL_S, qp-&gt;sl);</div><div class='del'>-		roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_PORT_M,</div><div class='del'>-			       SQ_DOORBELL_U32_4_PORT_S, qp-&gt;phy_port);</div><div class='del'>-		roce_set_field(sq_db.u32_8, SQ_DOORBELL_U32_8_QPN_M,</div><div class='del'>-			       SQ_DOORBELL_U32_8_QPN_S, qp-&gt;doorbell_qpn);</div><div class='del'>-		roce_set_bit(sq_db.u32_8, SQ_DOORBELL_HW_SYNC_S, 1);</div><div class='del'>-</div><div class='del'>-		doorbell[0] = le32_to_cpu(sq_db.u32_4);</div><div class='del'>-		doorbell[1] = le32_to_cpu(sq_db.u32_8);</div><div class='del'>-</div><div class='del'>-		hns_roce_write64_k((__le32 *)doorbell, qp-&gt;sq.db_reg_l);</div><div class='del'>-		qp-&gt;sq_next_wqe = ind;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	spin_unlock_irqrestore(&amp;qp-&gt;sq.lock, flags);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_post_recv(struct ib_qp *ibqp,</div><div class='del'>-				 const struct ib_recv_wr *wr,</div><div class='del'>-				 const struct ib_recv_wr **bad_wr)</div><div class='del'>-{</div><div class='del'>-	int ret = 0;</div><div class='del'>-	int nreq = 0;</div><div class='del'>-	int ind = 0;</div><div class='del'>-	int i = 0;</div><div class='del'>-	u32 reg_val;</div><div class='del'>-	unsigned long flags = 0;</div><div class='del'>-	struct hns_roce_rq_wqe_ctrl *ctrl = NULL;</div><div class='del'>-	struct hns_roce_wqe_data_seg *scat = NULL;</div><div class='del'>-	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_rq_db rq_db;</div><div class='del'>-	uint32_t doorbell[2] = {0};</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;hr_qp-&gt;rq.lock, flags);</div><div class='del'>-	ind = hr_qp-&gt;rq.head &amp; (hr_qp-&gt;rq.wqe_cnt - 1);</div><div class='del'>-</div><div class='del'>-	for (nreq = 0; wr; ++nreq, wr = wr-&gt;next) {</div><div class='del'>-		if (hns_roce_wq_overflow(&amp;hr_qp-&gt;rq, nreq,</div><div class='del'>-			hr_qp-&gt;ibqp.recv_cq)) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			*bad_wr = wr;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (unlikely(wr-&gt;num_sge &gt; hr_qp-&gt;rq.max_gs)) {</div><div class='del'>-			dev_err(dev, "rq:num_sge=%d &gt; qp-&gt;sq.max_gs=%d\n",</div><div class='del'>-				wr-&gt;num_sge, hr_qp-&gt;rq.max_gs);</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			*bad_wr = wr;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		ctrl = get_recv_wqe(hr_qp, ind);</div><div class='del'>-</div><div class='del'>-		roce_set_field(ctrl-&gt;rwqe_byte_12,</div><div class='del'>-			       RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M,</div><div class='del'>-			       RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S,</div><div class='del'>-			       wr-&gt;num_sge);</div><div class='del'>-</div><div class='del'>-		scat = (struct hns_roce_wqe_data_seg *)(ctrl + 1);</div><div class='del'>-</div><div class='del'>-		for (i = 0; i &lt; wr-&gt;num_sge; i++)</div><div class='del'>-			set_data_seg(scat + i, wr-&gt;sg_list + i);</div><div class='del'>-</div><div class='del'>-		hr_qp-&gt;rq.wrid[ind] = wr-&gt;wr_id;</div><div class='del'>-</div><div class='del'>-		ind = (ind + 1) &amp; (hr_qp-&gt;rq.wqe_cnt - 1);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-out:</div><div class='del'>-	if (likely(nreq)) {</div><div class='del'>-		hr_qp-&gt;rq.head += nreq;</div><div class='del'>-		/* Memory barrier */</div><div class='del'>-		wmb();</div><div class='del'>-</div><div class='del'>-		if (ibqp-&gt;qp_type == IB_QPT_GSI) {</div><div class='del'>-			__le32 tmp;</div><div class='del'>-</div><div class='del'>-			/* SW update GSI rq header */</div><div class='del'>-			reg_val = roce_read(to_hr_dev(ibqp-&gt;device),</div><div class='del'>-					    ROCEE_QP1C_CFG3_0_REG +</div><div class='del'>-					    QP1C_CFGN_OFFSET * hr_qp-&gt;phy_port);</div><div class='del'>-			tmp = cpu_to_le32(reg_val);</div><div class='del'>-			roce_set_field(tmp,</div><div class='del'>-				       ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M,</div><div class='del'>-				       ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S,</div><div class='del'>-				       hr_qp-&gt;rq.head);</div><div class='del'>-			reg_val = le32_to_cpu(tmp);</div><div class='del'>-			roce_write(to_hr_dev(ibqp-&gt;device),</div><div class='del'>-				   ROCEE_QP1C_CFG3_0_REG +</div><div class='del'>-				   QP1C_CFGN_OFFSET * hr_qp-&gt;phy_port, reg_val);</div><div class='del'>-		} else {</div><div class='del'>-			rq_db.u32_4 = 0;</div><div class='del'>-			rq_db.u32_8 = 0;</div><div class='del'>-</div><div class='del'>-			roce_set_field(rq_db.u32_4, RQ_DOORBELL_U32_4_RQ_HEAD_M,</div><div class='del'>-				       RQ_DOORBELL_U32_4_RQ_HEAD_S,</div><div class='del'>-				       hr_qp-&gt;rq.head);</div><div class='del'>-			roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_QPN_M,</div><div class='del'>-				       RQ_DOORBELL_U32_8_QPN_S, hr_qp-&gt;qpn);</div><div class='del'>-			roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_CMD_M,</div><div class='del'>-				       RQ_DOORBELL_U32_8_CMD_S, 1);</div><div class='del'>-			roce_set_bit(rq_db.u32_8, RQ_DOORBELL_U32_8_HW_SYNC_S,</div><div class='del'>-				     1);</div><div class='del'>-</div><div class='del'>-			doorbell[0] = le32_to_cpu(rq_db.u32_4);</div><div class='del'>-			doorbell[1] = le32_to_cpu(rq_db.u32_8);</div><div class='del'>-</div><div class='del'>-			hns_roce_write64_k((__le32 *)doorbell,</div><div class='del'>-					   hr_qp-&gt;rq.db_reg_l);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;hr_qp-&gt;rq.lock, flags);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_set_db_event_mode(struct hns_roce_dev *hr_dev,</div><div class='del'>-				       int sdb_mode, int odb_mode)</div><div class='del'>-{</div><div class='del'>-	__le32 tmp;</div><div class='del'>-	u32 val;</div><div class='del'>-</div><div class='del'>-	val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);</div><div class='del'>-	tmp = cpu_to_le32(val);</div><div class='del'>-	roce_set_bit(tmp, ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S, sdb_mode);</div><div class='del'>-	roce_set_bit(tmp, ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S, odb_mode);</div><div class='del'>-	val = le32_to_cpu(tmp);</div><div class='del'>-	roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_set_db_ext_mode(struct hns_roce_dev *hr_dev, u32 sdb_mode,</div><div class='del'>-				     u32 odb_mode)</div><div class='del'>-{</div><div class='del'>-	__le32 tmp;</div><div class='del'>-	u32 val;</div><div class='del'>-</div><div class='del'>-	/* Configure SDB/ODB extend mode */</div><div class='del'>-	val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);</div><div class='del'>-	tmp = cpu_to_le32(val);</div><div class='del'>-	roce_set_bit(tmp, ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S, sdb_mode);</div><div class='del'>-	roce_set_bit(tmp, ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S, odb_mode);</div><div class='del'>-	val = le32_to_cpu(tmp);</div><div class='del'>-	roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_set_sdb(struct hns_roce_dev *hr_dev, u32 sdb_alept,</div><div class='del'>-			     u32 sdb_alful)</div><div class='del'>-{</div><div class='del'>-	__le32 tmp;</div><div class='del'>-	u32 val;</div><div class='del'>-</div><div class='del'>-	/* Configure SDB */</div><div class='del'>-	val = roce_read(hr_dev, ROCEE_DB_SQ_WL_REG);</div><div class='del'>-	tmp = cpu_to_le32(val);</div><div class='del'>-	roce_set_field(tmp, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M,</div><div class='del'>-		       ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S, sdb_alful);</div><div class='del'>-	roce_set_field(tmp, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M,</div><div class='del'>-		       ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S, sdb_alept);</div><div class='del'>-	val = le32_to_cpu(tmp);</div><div class='del'>-	roce_write(hr_dev, ROCEE_DB_SQ_WL_REG, val);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_set_odb(struct hns_roce_dev *hr_dev, u32 odb_alept,</div><div class='del'>-			     u32 odb_alful)</div><div class='del'>-{</div><div class='del'>-	__le32 tmp;</div><div class='del'>-	u32 val;</div><div class='del'>-</div><div class='del'>-	/* Configure ODB */</div><div class='del'>-	val = roce_read(hr_dev, ROCEE_DB_OTHERS_WL_REG);</div><div class='del'>-	tmp = cpu_to_le32(val);</div><div class='del'>-	roce_set_field(tmp, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M,</div><div class='del'>-		       ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S, odb_alful);</div><div class='del'>-	roce_set_field(tmp, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M,</div><div class='del'>-		       ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S, odb_alept);</div><div class='del'>-	val = le32_to_cpu(tmp);</div><div class='del'>-	roce_write(hr_dev, ROCEE_DB_OTHERS_WL_REG, val);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_set_sdb_ext(struct hns_roce_dev *hr_dev, u32 ext_sdb_alept,</div><div class='del'>-				 u32 ext_sdb_alful)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-	struct hns_roce_db_table *db;</div><div class='del'>-	dma_addr_t sdb_dma_addr;</div><div class='del'>-	__le32 tmp;</div><div class='del'>-	u32 val;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	db = &amp;priv-&gt;db_table;</div><div class='del'>-</div><div class='del'>-	/* Configure extend SDB threshold */</div><div class='del'>-	roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_EMPTY_REG, ext_sdb_alept);</div><div class='del'>-	roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_REG, ext_sdb_alful);</div><div class='del'>-</div><div class='del'>-	/* Configure extend SDB base addr */</div><div class='del'>-	sdb_dma_addr = db-&gt;ext_db-&gt;sdb_buf_list-&gt;map;</div><div class='del'>-	roce_write(hr_dev, ROCEE_EXT_DB_SQ_REG, (u32)(sdb_dma_addr &gt;&gt; 12));</div><div class='del'>-</div><div class='del'>-	/* Configure extend SDB depth */</div><div class='del'>-	val = roce_read(hr_dev, ROCEE_EXT_DB_SQ_H_REG);</div><div class='del'>-	tmp = cpu_to_le32(val);</div><div class='del'>-	roce_set_field(tmp, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M,</div><div class='del'>-		       ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S,</div><div class='del'>-		       db-&gt;ext_db-&gt;esdb_dep);</div><div class='del'>-	/*</div><div class='del'>-	 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of</div><div class='del'>-	 * using 4K page, and shift more 32 because of</div><div class='del'>-	 * caculating the high 32 bit value evaluated to hardware.</div><div class='del'>-	 */</div><div class='del'>-	roce_set_field(tmp, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M,</div><div class='del'>-		       ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S, sdb_dma_addr &gt;&gt; 44);</div><div class='del'>-	val = le32_to_cpu(tmp);</div><div class='del'>-	roce_write(hr_dev, ROCEE_EXT_DB_SQ_H_REG, val);</div><div class='del'>-</div><div class='del'>-	dev_dbg(dev, "ext SDB depth: 0x%x\n", db-&gt;ext_db-&gt;esdb_dep);</div><div class='del'>-	dev_dbg(dev, "ext SDB threshold: epmty: 0x%x, ful: 0x%x\n",</div><div class='del'>-		ext_sdb_alept, ext_sdb_alful);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_set_odb_ext(struct hns_roce_dev *hr_dev, u32 ext_odb_alept,</div><div class='del'>-				 u32 ext_odb_alful)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-	struct hns_roce_db_table *db;</div><div class='del'>-	dma_addr_t odb_dma_addr;</div><div class='del'>-	__le32 tmp;</div><div class='del'>-	u32 val;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	db = &amp;priv-&gt;db_table;</div><div class='del'>-</div><div class='del'>-	/* Configure extend ODB threshold */</div><div class='del'>-	roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG, ext_odb_alept);</div><div class='del'>-	roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_REG, ext_odb_alful);</div><div class='del'>-</div><div class='del'>-	/* Configure extend ODB base addr */</div><div class='del'>-	odb_dma_addr = db-&gt;ext_db-&gt;odb_buf_list-&gt;map;</div><div class='del'>-	roce_write(hr_dev, ROCEE_EXT_DB_OTH_REG, (u32)(odb_dma_addr &gt;&gt; 12));</div><div class='del'>-</div><div class='del'>-	/* Configure extend ODB depth */</div><div class='del'>-	val = roce_read(hr_dev, ROCEE_EXT_DB_OTH_H_REG);</div><div class='del'>-	tmp = cpu_to_le32(val);</div><div class='del'>-	roce_set_field(tmp, ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M,</div><div class='del'>-		       ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S,</div><div class='del'>-		       db-&gt;ext_db-&gt;eodb_dep);</div><div class='del'>-	roce_set_field(tmp, ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M,</div><div class='del'>-		       ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S,</div><div class='del'>-		       db-&gt;ext_db-&gt;eodb_dep);</div><div class='del'>-	val = le32_to_cpu(tmp);</div><div class='del'>-	roce_write(hr_dev, ROCEE_EXT_DB_OTH_H_REG, val);</div><div class='del'>-</div><div class='del'>-	dev_dbg(dev, "ext ODB depth: 0x%x\n", db-&gt;ext_db-&gt;eodb_dep);</div><div class='del'>-	dev_dbg(dev, "ext ODB threshold: empty: 0x%x, ful: 0x%x\n",</div><div class='del'>-		ext_odb_alept, ext_odb_alful);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_db_ext_init(struct hns_roce_dev *hr_dev, u32 sdb_ext_mod,</div><div class='del'>-				u32 odb_ext_mod)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-	struct hns_roce_db_table *db;</div><div class='del'>-	dma_addr_t sdb_dma_addr;</div><div class='del'>-	dma_addr_t odb_dma_addr;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	db = &amp;priv-&gt;db_table;</div><div class='del'>-</div><div class='del'>-	db-&gt;ext_db = kmalloc(sizeof(*db-&gt;ext_db), GFP_KERNEL);</div><div class='del'>-	if (!db-&gt;ext_db)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	if (sdb_ext_mod) {</div><div class='del'>-		db-&gt;ext_db-&gt;sdb_buf_list = kmalloc(</div><div class='del'>-				sizeof(*db-&gt;ext_db-&gt;sdb_buf_list), GFP_KERNEL);</div><div class='del'>-		if (!db-&gt;ext_db-&gt;sdb_buf_list) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto ext_sdb_buf_fail_out;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		db-&gt;ext_db-&gt;sdb_buf_list-&gt;buf = dma_alloc_coherent(dev,</div><div class='del'>-						     HNS_ROCE_V1_EXT_SDB_SIZE,</div><div class='del'>-						     &amp;sdb_dma_addr, GFP_KERNEL);</div><div class='del'>-		if (!db-&gt;ext_db-&gt;sdb_buf_list-&gt;buf) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto alloc_sq_db_buf_fail;</div><div class='del'>-		}</div><div class='del'>-		db-&gt;ext_db-&gt;sdb_buf_list-&gt;map = sdb_dma_addr;</div><div class='del'>-</div><div class='del'>-		db-&gt;ext_db-&gt;esdb_dep = ilog2(HNS_ROCE_V1_EXT_SDB_DEPTH);</div><div class='del'>-		hns_roce_set_sdb_ext(hr_dev, HNS_ROCE_V1_EXT_SDB_ALEPT,</div><div class='del'>-				     HNS_ROCE_V1_EXT_SDB_ALFUL);</div><div class='del'>-	} else</div><div class='del'>-		hns_roce_set_sdb(hr_dev, HNS_ROCE_V1_SDB_ALEPT,</div><div class='del'>-				 HNS_ROCE_V1_SDB_ALFUL);</div><div class='del'>-</div><div class='del'>-	if (odb_ext_mod) {</div><div class='del'>-		db-&gt;ext_db-&gt;odb_buf_list = kmalloc(</div><div class='del'>-				sizeof(*db-&gt;ext_db-&gt;odb_buf_list), GFP_KERNEL);</div><div class='del'>-		if (!db-&gt;ext_db-&gt;odb_buf_list) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto ext_odb_buf_fail_out;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		db-&gt;ext_db-&gt;odb_buf_list-&gt;buf = dma_alloc_coherent(dev,</div><div class='del'>-						     HNS_ROCE_V1_EXT_ODB_SIZE,</div><div class='del'>-						     &amp;odb_dma_addr, GFP_KERNEL);</div><div class='del'>-		if (!db-&gt;ext_db-&gt;odb_buf_list-&gt;buf) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto alloc_otr_db_buf_fail;</div><div class='del'>-		}</div><div class='del'>-		db-&gt;ext_db-&gt;odb_buf_list-&gt;map = odb_dma_addr;</div><div class='del'>-</div><div class='del'>-		db-&gt;ext_db-&gt;eodb_dep = ilog2(HNS_ROCE_V1_EXT_ODB_DEPTH);</div><div class='del'>-		hns_roce_set_odb_ext(hr_dev, HNS_ROCE_V1_EXT_ODB_ALEPT,</div><div class='del'>-				     HNS_ROCE_V1_EXT_ODB_ALFUL);</div><div class='del'>-	} else</div><div class='del'>-		hns_roce_set_odb(hr_dev, HNS_ROCE_V1_ODB_ALEPT,</div><div class='del'>-				 HNS_ROCE_V1_ODB_ALFUL);</div><div class='del'>-</div><div class='del'>-	hns_roce_set_db_ext_mode(hr_dev, sdb_ext_mod, odb_ext_mod);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-alloc_otr_db_buf_fail:</div><div class='del'>-	kfree(db-&gt;ext_db-&gt;odb_buf_list);</div><div class='del'>-</div><div class='del'>-ext_odb_buf_fail_out:</div><div class='del'>-	if (sdb_ext_mod) {</div><div class='del'>-		dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,</div><div class='del'>-				  db-&gt;ext_db-&gt;sdb_buf_list-&gt;buf,</div><div class='del'>-				  db-&gt;ext_db-&gt;sdb_buf_list-&gt;map);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-alloc_sq_db_buf_fail:</div><div class='del'>-	if (sdb_ext_mod)</div><div class='del'>-		kfree(db-&gt;ext_db-&gt;sdb_buf_list);</div><div class='del'>-</div><div class='del'>-ext_sdb_buf_fail_out:</div><div class='del'>-	kfree(db-&gt;ext_db);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct hns_roce_qp *hns_roce_v1_create_lp_qp(struct hns_roce_dev *hr_dev,</div><div class='del'>-						    struct ib_pd *pd)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct ib_qp_init_attr init_attr;</div><div class='del'>-	struct ib_qp *qp;</div><div class='del'>-</div><div class='del'>-	memset(&amp;init_attr, 0, sizeof(struct ib_qp_init_attr));</div><div class='del'>-	init_attr.qp_type		= IB_QPT_RC;</div><div class='del'>-	init_attr.sq_sig_type		= IB_SIGNAL_ALL_WR;</div><div class='del'>-	init_attr.cap.max_recv_wr	= HNS_ROCE_MIN_WQE_NUM;</div><div class='del'>-	init_attr.cap.max_send_wr	= HNS_ROCE_MIN_WQE_NUM;</div><div class='del'>-</div><div class='del'>-	qp = hns_roce_create_qp(pd, &amp;init_attr, NULL);</div><div class='del'>-	if (IS_ERR(qp)) {</div><div class='del'>-		dev_err(dev, "Create loop qp for mr free failed!");</div><div class='del'>-		return NULL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return to_hr_qp(qp);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_caps *caps = &amp;hr_dev-&gt;caps;</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct ib_cq_init_attr cq_init_attr;</div><div class='del'>-	struct hns_roce_free_mr *free_mr;</div><div class='del'>-	struct ib_qp_attr attr = { 0 };</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-	struct hns_roce_qp *hr_qp;</div><div class='del'>-	struct ib_device *ibdev;</div><div class='del'>-	struct ib_cq *cq;</div><div class='del'>-	struct ib_pd *pd;</div><div class='del'>-	union ib_gid dgid;</div><div class='del'>-	u64 subnet_prefix;</div><div class='del'>-	int attr_mask = 0;</div><div class='del'>-	int ret = -ENOMEM;</div><div class='del'>-	int i, j;</div><div class='del'>-	u8 queue_en[HNS_ROCE_V1_RESV_QP] = { 0 };</div><div class='del'>-	u8 phy_port;</div><div class='del'>-	u8 port = 0;</div><div class='del'>-	u8 sl;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	free_mr = &amp;priv-&gt;free_mr;</div><div class='del'>-</div><div class='del'>-	/* Reserved cq for loop qp */</div><div class='del'>-	cq_init_attr.cqe		= HNS_ROCE_MIN_WQE_NUM * 2;</div><div class='del'>-	cq_init_attr.comp_vector	= 0;</div><div class='del'>-	cq = hns_roce_ib_create_cq(&amp;hr_dev-&gt;ib_dev, &amp;cq_init_attr, NULL);</div><div class='del'>-	if (IS_ERR(cq)) {</div><div class='del'>-		dev_err(dev, "Create cq for reserved loop qp failed!");</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	free_mr-&gt;mr_free_cq = to_hr_cq(cq);</div><div class='del'>-	free_mr-&gt;mr_free_cq-&gt;ib_cq.device		= &amp;hr_dev-&gt;ib_dev;</div><div class='del'>-	free_mr-&gt;mr_free_cq-&gt;ib_cq.uobject		= NULL;</div><div class='del'>-	free_mr-&gt;mr_free_cq-&gt;ib_cq.comp_handler		= NULL;</div><div class='del'>-	free_mr-&gt;mr_free_cq-&gt;ib_cq.event_handler	= NULL;</div><div class='del'>-	free_mr-&gt;mr_free_cq-&gt;ib_cq.cq_context		= NULL;</div><div class='del'>-	atomic_set(&amp;free_mr-&gt;mr_free_cq-&gt;ib_cq.usecnt, 0);</div><div class='del'>-</div><div class='del'>-	ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='del'>-	pd = rdma_zalloc_drv_obj(ibdev, ib_pd);</div><div class='del'>-	if (!pd)</div><div class='del'>-		goto alloc_mem_failed;</div><div class='del'>-</div><div class='del'>-	pd-&gt;device  = ibdev;</div><div class='del'>-	ret = hns_roce_alloc_pd(pd, NULL);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto alloc_pd_failed;</div><div class='del'>-</div><div class='del'>-	free_mr-&gt;mr_free_pd = to_hr_pd(pd);</div><div class='del'>-	free_mr-&gt;mr_free_pd-&gt;ibpd.device  = &amp;hr_dev-&gt;ib_dev;</div><div class='del'>-	free_mr-&gt;mr_free_pd-&gt;ibpd.uobject = NULL;</div><div class='del'>-	free_mr-&gt;mr_free_pd-&gt;ibpd.__internal_mr = NULL;</div><div class='del'>-	atomic_set(&amp;free_mr-&gt;mr_free_pd-&gt;ibpd.usecnt, 0);</div><div class='del'>-</div><div class='del'>-	attr.qp_access_flags	= IB_ACCESS_REMOTE_WRITE;</div><div class='del'>-	attr.pkey_index		= 0;</div><div class='del'>-	attr.min_rnr_timer	= 0;</div><div class='del'>-	/* Disable read ability */</div><div class='del'>-	attr.max_dest_rd_atomic = 0;</div><div class='del'>-	attr.max_rd_atomic	= 0;</div><div class='del'>-	/* Use arbitrary values as rq_psn and sq_psn */</div><div class='del'>-	attr.rq_psn		= 0x0808;</div><div class='del'>-	attr.sq_psn		= 0x0808;</div><div class='del'>-	attr.retry_cnt		= 7;</div><div class='del'>-	attr.rnr_retry		= 7;</div><div class='del'>-	attr.timeout		= 0x12;</div><div class='del'>-	attr.path_mtu		= IB_MTU_256;</div><div class='del'>-	attr.ah_attr.type	= RDMA_AH_ATTR_TYPE_ROCE;</div><div class='del'>-	rdma_ah_set_grh(&amp;attr.ah_attr, NULL, 0, 0, 1, 0);</div><div class='del'>-	rdma_ah_set_static_rate(&amp;attr.ah_attr, 3);</div><div class='del'>-</div><div class='del'>-	subnet_prefix = cpu_to_be64(0xfe80000000000000LL);</div><div class='del'>-	for (i = 0; i &lt; HNS_ROCE_V1_RESV_QP; i++) {</div><div class='del'>-		phy_port = (i &gt;= HNS_ROCE_MAX_PORTS) ? (i - 2) :</div><div class='del'>-				(i % HNS_ROCE_MAX_PORTS);</div><div class='del'>-		sl = i / HNS_ROCE_MAX_PORTS;</div><div class='del'>-</div><div class='del'>-		for (j = 0; j &lt; caps-&gt;num_ports; j++) {</div><div class='del'>-			if (hr_dev-&gt;iboe.phy_port[j] == phy_port) {</div><div class='del'>-				queue_en[i] = 1;</div><div class='del'>-				port = j;</div><div class='del'>-				break;</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (!queue_en[i])</div><div class='del'>-			continue;</div><div class='del'>-</div><div class='del'>-		free_mr-&gt;mr_free_qp[i] = hns_roce_v1_create_lp_qp(hr_dev, pd);</div><div class='del'>-		if (!free_mr-&gt;mr_free_qp[i]) {</div><div class='del'>-			dev_err(dev, "Create loop qp failed!\n");</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto create_lp_qp_failed;</div><div class='del'>-		}</div><div class='del'>-		hr_qp = free_mr-&gt;mr_free_qp[i];</div><div class='del'>-</div><div class='del'>-		hr_qp-&gt;port		= port;</div><div class='del'>-		hr_qp-&gt;phy_port		= phy_port;</div><div class='del'>-		hr_qp-&gt;ibqp.qp_type	= IB_QPT_RC;</div><div class='del'>-		hr_qp-&gt;ibqp.device	= &amp;hr_dev-&gt;ib_dev;</div><div class='del'>-		hr_qp-&gt;ibqp.uobject	= NULL;</div><div class='del'>-		atomic_set(&amp;hr_qp-&gt;ibqp.usecnt, 0);</div><div class='del'>-		hr_qp-&gt;ibqp.pd		= pd;</div><div class='del'>-		hr_qp-&gt;ibqp.recv_cq	= cq;</div><div class='del'>-		hr_qp-&gt;ibqp.send_cq	= cq;</div><div class='del'>-</div><div class='del'>-		rdma_ah_set_port_num(&amp;attr.ah_attr, port + 1);</div><div class='del'>-		rdma_ah_set_sl(&amp;attr.ah_attr, sl);</div><div class='del'>-		attr.port_num		= port + 1;</div><div class='del'>-</div><div class='del'>-		attr.dest_qp_num	= hr_qp-&gt;qpn;</div><div class='del'>-		memcpy(rdma_ah_retrieve_dmac(&amp;attr.ah_attr),</div><div class='del'>-		       hr_dev-&gt;dev_addr[port],</div><div class='del'>-		       MAC_ADDR_OCTET_NUM);</div><div class='del'>-</div><div class='del'>-		memcpy(&amp;dgid.raw, &amp;subnet_prefix, sizeof(u64));</div><div class='del'>-		memcpy(&amp;dgid.raw[8], hr_dev-&gt;dev_addr[port], 3);</div><div class='del'>-		memcpy(&amp;dgid.raw[13], hr_dev-&gt;dev_addr[port] + 3, 3);</div><div class='del'>-		dgid.raw[11] = 0xff;</div><div class='del'>-		dgid.raw[12] = 0xfe;</div><div class='del'>-		dgid.raw[8] ^= 2;</div><div class='del'>-		rdma_ah_set_dgid_raw(&amp;attr.ah_attr, dgid.raw);</div><div class='del'>-</div><div class='del'>-		ret = hr_dev-&gt;hw-&gt;modify_qp(&amp;hr_qp-&gt;ibqp, &amp;attr, attr_mask,</div><div class='del'>-					    IB_QPS_RESET, IB_QPS_INIT);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "modify qp failed(%d)!\n", ret);</div><div class='del'>-			goto create_lp_qp_failed;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		ret = hr_dev-&gt;hw-&gt;modify_qp(&amp;hr_qp-&gt;ibqp, &amp;attr, IB_QP_DEST_QPN,</div><div class='del'>-					    IB_QPS_INIT, IB_QPS_RTR);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "modify qp failed(%d)!\n", ret);</div><div class='del'>-			goto create_lp_qp_failed;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		ret = hr_dev-&gt;hw-&gt;modify_qp(&amp;hr_qp-&gt;ibqp, &amp;attr, attr_mask,</div><div class='del'>-					    IB_QPS_RTR, IB_QPS_RTS);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "modify qp failed(%d)!\n", ret);</div><div class='del'>-			goto create_lp_qp_failed;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-create_lp_qp_failed:</div><div class='del'>-	for (i -= 1; i &gt;= 0; i--) {</div><div class='del'>-		hr_qp = free_mr-&gt;mr_free_qp[i];</div><div class='del'>-		if (hns_roce_v1_destroy_qp(&amp;hr_qp-&gt;ibqp, NULL))</div><div class='del'>-			dev_err(dev, "Destroy qp %d for mr free failed!\n", i);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	hns_roce_dealloc_pd(pd, NULL);</div><div class='del'>-</div><div class='del'>-alloc_pd_failed:</div><div class='del'>-	kfree(pd);</div><div class='del'>-</div><div class='del'>-alloc_mem_failed:</div><div class='del'>-	if (hns_roce_ib_destroy_cq(cq, NULL))</div><div class='del'>-		dev_err(dev, "Destroy cq for create_lp_qp failed!\n");</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_free_mr *free_mr;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-	struct hns_roce_qp *hr_qp;</div><div class='del'>-	int ret;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	free_mr = &amp;priv-&gt;free_mr;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; HNS_ROCE_V1_RESV_QP; i++) {</div><div class='del'>-		hr_qp = free_mr-&gt;mr_free_qp[i];</div><div class='del'>-		if (!hr_qp)</div><div class='del'>-			continue;</div><div class='del'>-</div><div class='del'>-		ret = hns_roce_v1_destroy_qp(&amp;hr_qp-&gt;ibqp, NULL);</div><div class='del'>-		if (ret)</div><div class='del'>-			dev_err(dev, "Destroy qp %d for mr free failed(%d)!\n",</div><div class='del'>-				i, ret);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_ib_destroy_cq(&amp;free_mr-&gt;mr_free_cq-&gt;ib_cq, NULL);</div><div class='del'>-	if (ret)</div><div class='del'>-		dev_err(dev, "Destroy cq for mr_free failed(%d)!\n", ret);</div><div class='del'>-</div><div class='del'>-	hns_roce_dealloc_pd(&amp;free_mr-&gt;mr_free_pd-&gt;ibpd, NULL);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_db_init(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-	struct hns_roce_db_table *db;</div><div class='del'>-	u32 sdb_ext_mod;</div><div class='del'>-	u32 odb_ext_mod;</div><div class='del'>-	u32 sdb_evt_mod;</div><div class='del'>-	u32 odb_evt_mod;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	db = &amp;priv-&gt;db_table;</div><div class='del'>-</div><div class='del'>-	memset(db, 0, sizeof(*db));</div><div class='del'>-</div><div class='del'>-	/* Default DB mode */</div><div class='del'>-	sdb_ext_mod = HNS_ROCE_SDB_EXTEND_MODE;</div><div class='del'>-	odb_ext_mod = HNS_ROCE_ODB_EXTEND_MODE;</div><div class='del'>-	sdb_evt_mod = HNS_ROCE_SDB_NORMAL_MODE;</div><div class='del'>-	odb_evt_mod = HNS_ROCE_ODB_POLL_MODE;</div><div class='del'>-</div><div class='del'>-	db-&gt;sdb_ext_mod = sdb_ext_mod;</div><div class='del'>-	db-&gt;odb_ext_mod = odb_ext_mod;</div><div class='del'>-</div><div class='del'>-	/* Init extend DB */</div><div class='del'>-	ret = hns_roce_db_ext_init(hr_dev, sdb_ext_mod, odb_ext_mod);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "Failed in extend DB configuration.\n");</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	hns_roce_set_db_event_mode(hr_dev, sdb_evt_mod, odb_evt_mod);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_recreate_lp_qp_work_fn(struct work_struct *work)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_recreate_lp_qp_work *lp_qp_work;</div><div class='del'>-	struct hns_roce_dev *hr_dev;</div><div class='del'>-</div><div class='del'>-	lp_qp_work = container_of(work, struct hns_roce_recreate_lp_qp_work,</div><div class='del'>-				  work);</div><div class='del'>-	hr_dev = to_hr_dev(lp_qp_work-&gt;ib_dev);</div><div class='del'>-</div><div class='del'>-	hns_roce_v1_release_lp_qp(hr_dev);</div><div class='del'>-</div><div class='del'>-	if (hns_roce_v1_rsv_lp_qp(hr_dev))</div><div class='del'>-		dev_err(&amp;hr_dev-&gt;pdev-&gt;dev, "create reserver qp failed\n");</div><div class='del'>-</div><div class='del'>-	if (lp_qp_work-&gt;comp_flag)</div><div class='del'>-		complete(lp_qp_work-&gt;comp);</div><div class='del'>-</div><div class='del'>-	kfree(lp_qp_work);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_recreate_lp_qp_work *lp_qp_work;</div><div class='del'>-	struct hns_roce_free_mr *free_mr;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-	struct completion comp;</div><div class='del'>-	unsigned long end =</div><div class='del'>-	  msecs_to_jiffies(HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS) + jiffies;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	free_mr = &amp;priv-&gt;free_mr;</div><div class='del'>-</div><div class='del'>-	lp_qp_work = kzalloc(sizeof(struct hns_roce_recreate_lp_qp_work),</div><div class='del'>-			     GFP_KERNEL);</div><div class='del'>-	if (!lp_qp_work)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	INIT_WORK(&amp;(lp_qp_work-&gt;work), hns_roce_v1_recreate_lp_qp_work_fn);</div><div class='del'>-</div><div class='del'>-	lp_qp_work-&gt;ib_dev = &amp;(hr_dev-&gt;ib_dev);</div><div class='del'>-	lp_qp_work-&gt;comp = &amp;comp;</div><div class='del'>-	lp_qp_work-&gt;comp_flag = 1;</div><div class='del'>-</div><div class='del'>-	init_completion(lp_qp_work-&gt;comp);</div><div class='del'>-</div><div class='del'>-	queue_work(free_mr-&gt;free_mr_wq, &amp;(lp_qp_work-&gt;work));</div><div class='del'>-</div><div class='del'>-	while (time_before_eq(jiffies, end)) {</div><div class='del'>-		if (try_wait_for_completion(&amp;comp))</div><div class='del'>-			return 0;</div><div class='del'>-		msleep(HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	lp_qp_work-&gt;comp_flag = 0;</div><div class='del'>-	if (try_wait_for_completion(&amp;comp))</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	dev_warn(dev, "recreate lp qp failed 20s timeout and return failed!\n");</div><div class='del'>-	return -ETIMEDOUT;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_send_lp_wqe(struct hns_roce_qp *hr_qp)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp-&gt;ibqp.device);</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct ib_send_wr send_wr;</div><div class='del'>-	const struct ib_send_wr *bad_wr;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	memset(&amp;send_wr, 0, sizeof(send_wr));</div><div class='del'>-	send_wr.next	= NULL;</div><div class='del'>-	send_wr.num_sge	= 0;</div><div class='del'>-	send_wr.send_flags = 0;</div><div class='del'>-	send_wr.sg_list	= NULL;</div><div class='del'>-	send_wr.wr_id	= (unsigned long long)&amp;send_wr;</div><div class='del'>-	send_wr.opcode	= IB_WR_RDMA_WRITE;</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_v1_post_send(&amp;hr_qp-&gt;ibqp, &amp;send_wr, &amp;bad_wr);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "Post write wqe for mr free failed(%d)!", ret);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_mr_free_work_fn(struct work_struct *work)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_mr_free_work *mr_work;</div><div class='del'>-	struct ib_wc wc[HNS_ROCE_V1_RESV_QP];</div><div class='del'>-	struct hns_roce_free_mr *free_mr;</div><div class='del'>-	struct hns_roce_cq *mr_free_cq;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-	struct hns_roce_dev *hr_dev;</div><div class='del'>-	struct hns_roce_mr *hr_mr;</div><div class='del'>-	struct hns_roce_qp *hr_qp;</div><div class='del'>-	struct device *dev;</div><div class='del'>-	unsigned long end =</div><div class='del'>-		msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies;</div><div class='del'>-	int i;</div><div class='del'>-	int ret;</div><div class='del'>-	int ne = 0;</div><div class='del'>-</div><div class='del'>-	mr_work = container_of(work, struct hns_roce_mr_free_work, work);</div><div class='del'>-	hr_mr = (struct hns_roce_mr *)mr_work-&gt;mr;</div><div class='del'>-	hr_dev = to_hr_dev(mr_work-&gt;ib_dev);</div><div class='del'>-	dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	free_mr = &amp;priv-&gt;free_mr;</div><div class='del'>-	mr_free_cq = free_mr-&gt;mr_free_cq;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; HNS_ROCE_V1_RESV_QP; i++) {</div><div class='del'>-		hr_qp = free_mr-&gt;mr_free_qp[i];</div><div class='del'>-		if (!hr_qp)</div><div class='del'>-			continue;</div><div class='del'>-		ne++;</div><div class='del'>-</div><div class='del'>-		ret = hns_roce_v1_send_lp_wqe(hr_qp);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev,</div><div class='del'>-			     "Send wqe (qp:0x%lx) for mr free failed(%d)!\n",</div><div class='del'>-			     hr_qp-&gt;qpn, ret);</div><div class='del'>-			goto free_work;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!ne) {</div><div class='del'>-		dev_err(dev, "Reserved loop qp is absent!\n");</div><div class='del'>-		goto free_work;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	do {</div><div class='del'>-		ret = hns_roce_v1_poll_cq(&amp;mr_free_cq-&gt;ib_cq, ne, wc);</div><div class='del'>-		if (ret &lt; 0 &amp;&amp; hr_qp) {</div><div class='del'>-			dev_err(dev,</div><div class='del'>-			   "(qp:0x%lx) starts, Poll cqe failed(%d) for mr 0x%x free! Remain %d cqe\n",</div><div class='del'>-			   hr_qp-&gt;qpn, ret, hr_mr-&gt;key, ne);</div><div class='del'>-			goto free_work;</div><div class='del'>-		}</div><div class='del'>-		ne -= ret;</div><div class='del'>-		usleep_range(HNS_ROCE_V1_FREE_MR_WAIT_VALUE * 1000,</div><div class='del'>-			     (1 + HNS_ROCE_V1_FREE_MR_WAIT_VALUE) * 1000);</div><div class='del'>-	} while (ne &amp;&amp; time_before_eq(jiffies, end));</div><div class='del'>-</div><div class='del'>-	if (ne != 0)</div><div class='del'>-		dev_err(dev,</div><div class='del'>-			"Poll cqe for mr 0x%x free timeout! Remain %d cqe\n",</div><div class='del'>-			hr_mr-&gt;key, ne);</div><div class='del'>-</div><div class='del'>-free_work:</div><div class='del'>-	if (mr_work-&gt;comp_flag)</div><div class='del'>-		complete(mr_work-&gt;comp);</div><div class='del'>-	kfree(mr_work);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev,</div><div class='del'>-				struct hns_roce_mr *mr, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_mr_free_work *mr_work;</div><div class='del'>-	struct hns_roce_free_mr *free_mr;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-	struct completion comp;</div><div class='del'>-	unsigned long end =</div><div class='del'>-		msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies;</div><div class='del'>-	unsigned long start = jiffies;</div><div class='del'>-	int npages;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	free_mr = &amp;priv-&gt;free_mr;</div><div class='del'>-</div><div class='del'>-	if (mr-&gt;enabled) {</div><div class='del'>-		if (hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mr-&gt;key)</div><div class='del'>-				       &amp; (hr_dev-&gt;caps.num_mtpts - 1)))</div><div class='del'>-			dev_warn(dev, "HW2SW_MPT failed!\n");</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mr_work = kzalloc(sizeof(*mr_work), GFP_KERNEL);</div><div class='del'>-	if (!mr_work) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto free_mr;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	INIT_WORK(&amp;(mr_work-&gt;work), hns_roce_v1_mr_free_work_fn);</div><div class='del'>-</div><div class='del'>-	mr_work-&gt;ib_dev = &amp;(hr_dev-&gt;ib_dev);</div><div class='del'>-	mr_work-&gt;comp = &amp;comp;</div><div class='del'>-	mr_work-&gt;comp_flag = 1;</div><div class='del'>-	mr_work-&gt;mr = (void *)mr;</div><div class='del'>-	init_completion(mr_work-&gt;comp);</div><div class='del'>-</div><div class='del'>-	queue_work(free_mr-&gt;free_mr_wq, &amp;(mr_work-&gt;work));</div><div class='del'>-</div><div class='del'>-	while (time_before_eq(jiffies, end)) {</div><div class='del'>-		if (try_wait_for_completion(&amp;comp))</div><div class='del'>-			goto free_mr;</div><div class='del'>-		msleep(HNS_ROCE_V1_FREE_MR_WAIT_VALUE);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mr_work-&gt;comp_flag = 0;</div><div class='del'>-	if (try_wait_for_completion(&amp;comp))</div><div class='del'>-		goto free_mr;</div><div class='del'>-</div><div class='del'>-	dev_warn(dev, "Free mr work 0x%x over 50s and failed!\n", mr-&gt;key);</div><div class='del'>-	ret = -ETIMEDOUT;</div><div class='del'>-</div><div class='del'>-free_mr:</div><div class='del'>-	dev_dbg(dev, "Free mr 0x%x use 0x%x us.\n",</div><div class='del'>-		mr-&gt;key, jiffies_to_usecs(jiffies) - jiffies_to_usecs(start));</div><div class='del'>-</div><div class='del'>-	if (mr-&gt;size != ~0ULL) {</div><div class='del'>-		npages = ib_umem_page_count(mr-&gt;umem);</div><div class='del'>-		dma_free_coherent(dev, npages * 8, mr-&gt;pbl_buf,</div><div class='del'>-				  mr-&gt;pbl_dma_addr);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	hns_roce_bitmap_free(&amp;hr_dev-&gt;mr_table.mtpt_bitmap,</div><div class='del'>-			     key_to_hw_index(mr-&gt;key), 0);</div><div class='del'>-</div><div class='del'>-	if (mr-&gt;umem)</div><div class='del'>-		ib_umem_release(mr-&gt;umem);</div><div class='del'>-</div><div class='del'>-	kfree(mr);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_db_free(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-	struct hns_roce_db_table *db;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	db = &amp;priv-&gt;db_table;</div><div class='del'>-</div><div class='del'>-	if (db-&gt;sdb_ext_mod) {</div><div class='del'>-		dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,</div><div class='del'>-				  db-&gt;ext_db-&gt;sdb_buf_list-&gt;buf,</div><div class='del'>-				  db-&gt;ext_db-&gt;sdb_buf_list-&gt;map);</div><div class='del'>-		kfree(db-&gt;ext_db-&gt;sdb_buf_list);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (db-&gt;odb_ext_mod) {</div><div class='del'>-		dma_free_coherent(dev, HNS_ROCE_V1_EXT_ODB_SIZE,</div><div class='del'>-				  db-&gt;ext_db-&gt;odb_buf_list-&gt;buf,</div><div class='del'>-				  db-&gt;ext_db-&gt;odb_buf_list-&gt;map);</div><div class='del'>-		kfree(db-&gt;ext_db-&gt;odb_buf_list);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	kfree(db-&gt;ext_db);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_raq_init(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	int ret;</div><div class='del'>-	u32 val;</div><div class='del'>-	__le32 tmp;</div><div class='del'>-	int raq_shift = 0;</div><div class='del'>-	dma_addr_t addr;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-	struct hns_roce_raq_table *raq;</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	raq = &amp;priv-&gt;raq_table;</div><div class='del'>-</div><div class='del'>-	raq-&gt;e_raq_buf = kzalloc(sizeof(*(raq-&gt;e_raq_buf)), GFP_KERNEL);</div><div class='del'>-	if (!raq-&gt;e_raq_buf)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	raq-&gt;e_raq_buf-&gt;buf = dma_alloc_coherent(dev, HNS_ROCE_V1_RAQ_SIZE,</div><div class='del'>-						 &amp;addr, GFP_KERNEL);</div><div class='del'>-	if (!raq-&gt;e_raq_buf-&gt;buf) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err_dma_alloc_raq;</div><div class='del'>-	}</div><div class='del'>-	raq-&gt;e_raq_buf-&gt;map = addr;</div><div class='del'>-</div><div class='del'>-	/* Configure raq extended address. 48bit 4K align*/</div><div class='del'>-	roce_write(hr_dev, ROCEE_EXT_RAQ_REG, raq-&gt;e_raq_buf-&gt;map &gt;&gt; 12);</div><div class='del'>-</div><div class='del'>-	/* Configure raq_shift */</div><div class='del'>-	raq_shift = ilog2(HNS_ROCE_V1_RAQ_SIZE / HNS_ROCE_V1_RAQ_ENTRY);</div><div class='del'>-	val = roce_read(hr_dev, ROCEE_EXT_RAQ_H_REG);</div><div class='del'>-	tmp = cpu_to_le32(val);</div><div class='del'>-	roce_set_field(tmp, ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M,</div><div class='del'>-		       ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S, raq_shift);</div><div class='del'>-	/*</div><div class='del'>-	 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of</div><div class='del'>-	 * using 4K page, and shift more 32 because of</div><div class='del'>-	 * caculating the high 32 bit value evaluated to hardware.</div><div class='del'>-	 */</div><div class='del'>-	roce_set_field(tmp, ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M,</div><div class='del'>-		       ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S,</div><div class='del'>-		       raq-&gt;e_raq_buf-&gt;map &gt;&gt; 44);</div><div class='del'>-	val = le32_to_cpu(tmp);</div><div class='del'>-	roce_write(hr_dev, ROCEE_EXT_RAQ_H_REG, val);</div><div class='del'>-	dev_dbg(dev, "Configure raq_shift 0x%x.\n", val);</div><div class='del'>-</div><div class='del'>-	/* Configure raq threshold */</div><div class='del'>-	val = roce_read(hr_dev, ROCEE_RAQ_WL_REG);</div><div class='del'>-	tmp = cpu_to_le32(val);</div><div class='del'>-	roce_set_field(tmp, ROCEE_RAQ_WL_ROCEE_RAQ_WL_M,</div><div class='del'>-		       ROCEE_RAQ_WL_ROCEE_RAQ_WL_S,</div><div class='del'>-		       HNS_ROCE_V1_EXT_RAQ_WF);</div><div class='del'>-	val = le32_to_cpu(tmp);</div><div class='del'>-	roce_write(hr_dev, ROCEE_RAQ_WL_REG, val);</div><div class='del'>-	dev_dbg(dev, "Configure raq_wl 0x%x.\n", val);</div><div class='del'>-</div><div class='del'>-	/* Enable extend raq */</div><div class='del'>-	val = roce_read(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG);</div><div class='del'>-	tmp = cpu_to_le32(val);</div><div class='del'>-	roce_set_field(tmp,</div><div class='del'>-		       ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M,</div><div class='del'>-		       ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S,</div><div class='del'>-		       POL_TIME_INTERVAL_VAL);</div><div class='del'>-	roce_set_bit(tmp, ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE, 1);</div><div class='del'>-	roce_set_field(tmp,</div><div class='del'>-		       ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M,</div><div class='del'>-		       ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S,</div><div class='del'>-		       2);</div><div class='del'>-	roce_set_bit(tmp,</div><div class='del'>-		     ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S, 1);</div><div class='del'>-	val = le32_to_cpu(tmp);</div><div class='del'>-	roce_write(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG, val);</div><div class='del'>-	dev_dbg(dev, "Configure WrmsPolTimeInterval 0x%x.\n", val);</div><div class='del'>-</div><div class='del'>-	/* Enable raq drop */</div><div class='del'>-	val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);</div><div class='del'>-	tmp = cpu_to_le32(val);</div><div class='del'>-	roce_set_bit(tmp, ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S, 1);</div><div class='del'>-	val = le32_to_cpu(tmp);</div><div class='del'>-	roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);</div><div class='del'>-	dev_dbg(dev, "Configure GlbCfg = 0x%x.\n", val);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err_dma_alloc_raq:</div><div class='del'>-	kfree(raq-&gt;e_raq_buf);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_raq_free(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-	struct hns_roce_raq_table *raq;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	raq = &amp;priv-&gt;raq_table;</div><div class='del'>-</div><div class='del'>-	dma_free_coherent(dev, HNS_ROCE_V1_RAQ_SIZE, raq-&gt;e_raq_buf-&gt;buf,</div><div class='del'>-			  raq-&gt;e_raq_buf-&gt;map);</div><div class='del'>-	kfree(raq-&gt;e_raq_buf);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_port_enable(struct hns_roce_dev *hr_dev, int enable_flag)</div><div class='del'>-{</div><div class='del'>-	__le32 tmp;</div><div class='del'>-	u32 val;</div><div class='del'>-</div><div class='del'>-	if (enable_flag) {</div><div class='del'>-		val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);</div><div class='del'>-		 /* Open all ports */</div><div class='del'>-		tmp = cpu_to_le32(val);</div><div class='del'>-		roce_set_field(tmp, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,</div><div class='del'>-			       ROCEE_GLB_CFG_ROCEE_PORT_ST_S,</div><div class='del'>-			       ALL_PORT_VAL_OPEN);</div><div class='del'>-		val = le32_to_cpu(tmp);</div><div class='del'>-		roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);</div><div class='del'>-	} else {</div><div class='del'>-		val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);</div><div class='del'>-		/* Close all ports */</div><div class='del'>-		tmp = cpu_to_le32(val);</div><div class='del'>-		roce_set_field(tmp, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,</div><div class='del'>-			       ROCEE_GLB_CFG_ROCEE_PORT_ST_S, 0x0);</div><div class='del'>-		val = le32_to_cpu(tmp);</div><div class='del'>-		roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_bt_init(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-</div><div class='del'>-	priv-&gt;bt_table.qpc_buf.buf = dma_alloc_coherent(dev,</div><div class='del'>-		HNS_ROCE_BT_RSV_BUF_SIZE, &amp;priv-&gt;bt_table.qpc_buf.map,</div><div class='del'>-		GFP_KERNEL);</div><div class='del'>-	if (!priv-&gt;bt_table.qpc_buf.buf)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	priv-&gt;bt_table.mtpt_buf.buf = dma_alloc_coherent(dev,</div><div class='del'>-		HNS_ROCE_BT_RSV_BUF_SIZE, &amp;priv-&gt;bt_table.mtpt_buf.map,</div><div class='del'>-		GFP_KERNEL);</div><div class='del'>-	if (!priv-&gt;bt_table.mtpt_buf.buf) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err_failed_alloc_mtpt_buf;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	priv-&gt;bt_table.cqc_buf.buf = dma_alloc_coherent(dev,</div><div class='del'>-		HNS_ROCE_BT_RSV_BUF_SIZE, &amp;priv-&gt;bt_table.cqc_buf.map,</div><div class='del'>-		GFP_KERNEL);</div><div class='del'>-	if (!priv-&gt;bt_table.cqc_buf.buf) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err_failed_alloc_cqc_buf;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err_failed_alloc_cqc_buf:</div><div class='del'>-	dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,</div><div class='del'>-		priv-&gt;bt_table.mtpt_buf.buf, priv-&gt;bt_table.mtpt_buf.map);</div><div class='del'>-</div><div class='del'>-err_failed_alloc_mtpt_buf:</div><div class='del'>-	dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,</div><div class='del'>-		priv-&gt;bt_table.qpc_buf.buf, priv-&gt;bt_table.qpc_buf.map);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_bt_free(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-</div><div class='del'>-	dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,</div><div class='del'>-		priv-&gt;bt_table.cqc_buf.buf, priv-&gt;bt_table.cqc_buf.map);</div><div class='del'>-</div><div class='del'>-	dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,</div><div class='del'>-		priv-&gt;bt_table.mtpt_buf.buf, priv-&gt;bt_table.mtpt_buf.map);</div><div class='del'>-</div><div class='del'>-	dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,</div><div class='del'>-		priv-&gt;bt_table.qpc_buf.buf, priv-&gt;bt_table.qpc_buf.map);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_tptr_init(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_buf_list *tptr_buf;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	tptr_buf = &amp;priv-&gt;tptr_table.tptr_buf;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * This buffer will be used for CQ's tptr(tail pointer), also</div><div class='del'>-	 * named ci(customer index). Every CQ will use 2 bytes to save</div><div class='del'>-	 * cqe ci in hip06. Hardware will read this area to get new ci</div><div class='del'>-	 * when the queue is almost full.</div><div class='del'>-	 */</div><div class='del'>-	tptr_buf-&gt;buf = dma_alloc_coherent(dev, HNS_ROCE_V1_TPTR_BUF_SIZE,</div><div class='del'>-					   &amp;tptr_buf-&gt;map, GFP_KERNEL);</div><div class='del'>-	if (!tptr_buf-&gt;buf)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	hr_dev-&gt;tptr_dma_addr = tptr_buf-&gt;map;</div><div class='del'>-	hr_dev-&gt;tptr_size = HNS_ROCE_V1_TPTR_BUF_SIZE;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_tptr_free(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_buf_list *tptr_buf;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	tptr_buf = &amp;priv-&gt;tptr_table.tptr_buf;</div><div class='del'>-</div><div class='del'>-	dma_free_coherent(dev, HNS_ROCE_V1_TPTR_BUF_SIZE,</div><div class='del'>-			  tptr_buf-&gt;buf, tptr_buf-&gt;map);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_free_mr_init(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_free_mr *free_mr;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	free_mr = &amp;priv-&gt;free_mr;</div><div class='del'>-</div><div class='del'>-	free_mr-&gt;free_mr_wq = create_singlethread_workqueue("hns_roce_free_mr");</div><div class='del'>-	if (!free_mr-&gt;free_mr_wq) {</div><div class='del'>-		dev_err(dev, "Create free mr workqueue failed!\n");</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_v1_rsv_lp_qp(hr_dev);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "Reserved loop qp failed(%d)!\n", ret);</div><div class='del'>-		flush_workqueue(free_mr-&gt;free_mr_wq);</div><div class='del'>-		destroy_workqueue(free_mr-&gt;free_mr_wq);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_free_mr_free(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_free_mr *free_mr;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	free_mr = &amp;priv-&gt;free_mr;</div><div class='del'>-</div><div class='del'>-	flush_workqueue(free_mr-&gt;free_mr_wq);</div><div class='del'>-	destroy_workqueue(free_mr-&gt;free_mr_wq);</div><div class='del'>-</div><div class='del'>-	hns_roce_v1_release_lp_qp(hr_dev);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * hns_roce_v1_reset - reset RoCE</div><div class='del'>- * @hr_dev: RoCE device struct pointer</div><div class='del'>- * @enable: true -- drop reset, false -- reset</div><div class='del'>- * return 0 - success , negative --fail</div><div class='del'>- */</div><div class='del'>-static int hns_roce_v1_reset(struct hns_roce_dev *hr_dev, bool dereset)</div><div class='del'>-{</div><div class='del'>-	struct device_node *dsaf_node;</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct device_node *np = dev-&gt;of_node;</div><div class='del'>-	struct fwnode_handle *fwnode;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	/* check if this is DT/ACPI case */</div><div class='del'>-	if (dev_of_node(dev)) {</div><div class='del'>-		dsaf_node = of_parse_phandle(np, "dsaf-handle", 0);</div><div class='del'>-		if (!dsaf_node) {</div><div class='del'>-			dev_err(dev, "could not find dsaf-handle\n");</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-		fwnode = &amp;dsaf_node-&gt;fwnode;</div><div class='del'>-	} else if (is_acpi_device_node(dev-&gt;fwnode)) {</div><div class='del'>-		struct fwnode_reference_args args;</div><div class='del'>-</div><div class='del'>-		ret = acpi_node_get_property_reference(dev-&gt;fwnode,</div><div class='del'>-						       "dsaf-handle", 0, &amp;args);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "could not find dsaf-handle\n");</div><div class='del'>-			return ret;</div><div class='del'>-		}</div><div class='del'>-		fwnode = args.fwnode;</div><div class='del'>-	} else {</div><div class='del'>-		dev_err(dev, "cannot read data from DT or ACPI\n");</div><div class='del'>-		return -ENXIO;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = hns_dsaf_roce_reset(fwnode, false);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	if (dereset) {</div><div class='del'>-		msleep(SLEEP_TIME_INTERVAL);</div><div class='del'>-		ret = hns_dsaf_roce_reset(fwnode, true);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_profile(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	int i = 0;</div><div class='del'>-	struct hns_roce_caps *caps = &amp;hr_dev-&gt;caps;</div><div class='del'>-</div><div class='del'>-	hr_dev-&gt;vendor_id = roce_read(hr_dev, ROCEE_VENDOR_ID_REG);</div><div class='del'>-	hr_dev-&gt;vendor_part_id = roce_read(hr_dev, ROCEE_VENDOR_PART_ID_REG);</div><div class='del'>-	hr_dev-&gt;sys_image_guid = roce_read(hr_dev, ROCEE_SYS_IMAGE_GUID_L_REG) |</div><div class='del'>-				((u64)roce_read(hr_dev,</div><div class='del'>-					    ROCEE_SYS_IMAGE_GUID_H_REG) &lt;&lt; 32);</div><div class='del'>-	hr_dev-&gt;hw_rev		= HNS_ROCE_HW_VER1;</div><div class='del'>-</div><div class='del'>-	caps-&gt;num_qps		= HNS_ROCE_V1_MAX_QP_NUM;</div><div class='del'>-	caps-&gt;max_wqes		= HNS_ROCE_V1_MAX_WQE_NUM;</div><div class='del'>-	caps-&gt;min_wqes		= HNS_ROCE_MIN_WQE_NUM;</div><div class='del'>-	caps-&gt;num_cqs		= HNS_ROCE_V1_MAX_CQ_NUM;</div><div class='del'>-	caps-&gt;min_cqes		= HNS_ROCE_MIN_CQE_NUM;</div><div class='del'>-	caps-&gt;max_cqes		= HNS_ROCE_V1_MAX_CQE_NUM;</div><div class='del'>-	caps-&gt;max_sq_sg		= HNS_ROCE_V1_SG_NUM;</div><div class='del'>-	caps-&gt;max_rq_sg		= HNS_ROCE_V1_SG_NUM;</div><div class='del'>-	caps-&gt;max_sq_inline	= HNS_ROCE_V1_INLINE_SIZE;</div><div class='del'>-	caps-&gt;num_uars		= HNS_ROCE_V1_UAR_NUM;</div><div class='del'>-	caps-&gt;phy_num_uars	= HNS_ROCE_V1_PHY_UAR_NUM;</div><div class='del'>-	caps-&gt;num_aeq_vectors	= HNS_ROCE_V1_AEQE_VEC_NUM;</div><div class='del'>-	caps-&gt;num_comp_vectors	= HNS_ROCE_V1_COMP_VEC_NUM;</div><div class='del'>-	caps-&gt;num_other_vectors	= HNS_ROCE_V1_ABNORMAL_VEC_NUM;</div><div class='del'>-	caps-&gt;num_mtpts		= HNS_ROCE_V1_MAX_MTPT_NUM;</div><div class='del'>-	caps-&gt;num_mtt_segs	= HNS_ROCE_V1_MAX_MTT_SEGS;</div><div class='del'>-	caps-&gt;num_pds		= HNS_ROCE_V1_MAX_PD_NUM;</div><div class='del'>-	caps-&gt;max_qp_init_rdma	= HNS_ROCE_V1_MAX_QP_INIT_RDMA;</div><div class='del'>-	caps-&gt;max_qp_dest_rdma	= HNS_ROCE_V1_MAX_QP_DEST_RDMA;</div><div class='del'>-	caps-&gt;max_sq_desc_sz	= HNS_ROCE_V1_MAX_SQ_DESC_SZ;</div><div class='del'>-	caps-&gt;max_rq_desc_sz	= HNS_ROCE_V1_MAX_RQ_DESC_SZ;</div><div class='del'>-	caps-&gt;qpc_entry_sz	= HNS_ROCE_V1_QPC_ENTRY_SIZE;</div><div class='del'>-	caps-&gt;irrl_entry_sz	= HNS_ROCE_V1_IRRL_ENTRY_SIZE;</div><div class='del'>-	caps-&gt;cqc_entry_sz	= HNS_ROCE_V1_CQC_ENTRY_SIZE;</div><div class='del'>-	caps-&gt;mtpt_entry_sz	= HNS_ROCE_V1_MTPT_ENTRY_SIZE;</div><div class='del'>-	caps-&gt;mtt_entry_sz	= HNS_ROCE_V1_MTT_ENTRY_SIZE;</div><div class='del'>-	caps-&gt;cq_entry_sz	= HNS_ROCE_V1_CQE_ENTRY_SIZE;</div><div class='del'>-	caps-&gt;page_size_cap	= HNS_ROCE_V1_PAGE_SIZE_SUPPORT;</div><div class='del'>-	caps-&gt;reserved_lkey	= 0;</div><div class='del'>-	caps-&gt;reserved_pds	= 0;</div><div class='del'>-	caps-&gt;reserved_mrws	= 1;</div><div class='del'>-	caps-&gt;reserved_uars	= 0;</div><div class='del'>-	caps-&gt;reserved_cqs	= 0;</div><div class='del'>-	caps-&gt;chunk_sz		= HNS_ROCE_V1_TABLE_CHUNK_SIZE;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; caps-&gt;num_ports; i++)</div><div class='del'>-		caps-&gt;pkey_table_len[i] = 1;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; caps-&gt;num_ports; i++) {</div><div class='del'>-		/* Six ports shared 16 GID in v1 engine */</div><div class='del'>-		if (i &gt;= (HNS_ROCE_V1_GID_NUM % caps-&gt;num_ports))</div><div class='del'>-			caps-&gt;gid_table_len[i] = HNS_ROCE_V1_GID_NUM /</div><div class='del'>-						 caps-&gt;num_ports;</div><div class='del'>-		else</div><div class='del'>-			caps-&gt;gid_table_len[i] = HNS_ROCE_V1_GID_NUM /</div><div class='del'>-						 caps-&gt;num_ports + 1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	caps-&gt;ceqe_depth = HNS_ROCE_V1_COMP_EQE_NUM;</div><div class='del'>-	caps-&gt;aeqe_depth = HNS_ROCE_V1_ASYNC_EQE_NUM;</div><div class='del'>-	caps-&gt;local_ca_ack_delay = roce_read(hr_dev, ROCEE_ACK_DELAY_REG);</div><div class='del'>-	caps-&gt;max_mtu = IB_MTU_2048;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_init(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	int ret;</div><div class='del'>-	u32 val;</div><div class='del'>-	__le32 tmp;</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-</div><div class='del'>-	/* DMAE user config */</div><div class='del'>-	val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG1_REG);</div><div class='del'>-	tmp = cpu_to_le32(val);</div><div class='del'>-	roce_set_field(tmp, ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M,</div><div class='del'>-		       ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S, 0xf);</div><div class='del'>-	roce_set_field(tmp, ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M,</div><div class='del'>-		       ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S,</div><div class='del'>-		       1 &lt;&lt; PAGES_SHIFT_16);</div><div class='del'>-	val = le32_to_cpu(tmp);</div><div class='del'>-	roce_write(hr_dev, ROCEE_DMAE_USER_CFG1_REG, val);</div><div class='del'>-</div><div class='del'>-	val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG2_REG);</div><div class='del'>-	tmp = cpu_to_le32(val);</div><div class='del'>-	roce_set_field(tmp, ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M,</div><div class='del'>-		       ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S, 0xf);</div><div class='del'>-	roce_set_field(tmp, ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M,</div><div class='del'>-		       ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S,</div><div class='del'>-		       1 &lt;&lt; PAGES_SHIFT_16);</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_db_init(hr_dev);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "doorbell init failed!\n");</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_raq_init(hr_dev);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "raq init failed!\n");</div><div class='del'>-		goto error_failed_raq_init;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_bt_init(hr_dev);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "bt init failed!\n");</div><div class='del'>-		goto error_failed_bt_init;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_tptr_init(hr_dev);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "tptr init failed!\n");</div><div class='del'>-		goto error_failed_tptr_init;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_free_mr_init(hr_dev);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "free mr init failed!\n");</div><div class='del'>-		goto error_failed_free_mr_init;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_UP);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-error_failed_free_mr_init:</div><div class='del'>-	hns_roce_tptr_free(hr_dev);</div><div class='del'>-</div><div class='del'>-error_failed_tptr_init:</div><div class='del'>-	hns_roce_bt_free(hr_dev);</div><div class='del'>-</div><div class='del'>-error_failed_bt_init:</div><div class='del'>-	hns_roce_raq_free(hr_dev);</div><div class='del'>-</div><div class='del'>-error_failed_raq_init:</div><div class='del'>-	hns_roce_db_free(hr_dev);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_exit(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN);</div><div class='del'>-	hns_roce_free_mr_free(hr_dev);</div><div class='del'>-	hns_roce_tptr_free(hr_dev);</div><div class='del'>-	hns_roce_bt_free(hr_dev);</div><div class='del'>-	hns_roce_raq_free(hr_dev);</div><div class='del'>-	hns_roce_db_free(hr_dev);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_cmd_pending(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	u32 status = readl(hr_dev-&gt;reg_base + ROCEE_MB6_REG);</div><div class='del'>-</div><div class='del'>-	return (!!(status &amp; (1 &lt;&lt; HCR_GO_BIT)));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,</div><div class='del'>-				 u64 out_param, u32 in_modifier, u8 op_modifier,</div><div class='del'>-				 u16 op, u16 token, int event)</div><div class='del'>-{</div><div class='del'>-	u32 __iomem *hcr = (u32 __iomem *)(hr_dev-&gt;reg_base + ROCEE_MB1_REG);</div><div class='del'>-	unsigned long end;</div><div class='del'>-	u32 val = 0;</div><div class='del'>-	__le32 tmp;</div><div class='del'>-</div><div class='del'>-	end = msecs_to_jiffies(GO_BIT_TIMEOUT_MSECS) + jiffies;</div><div class='del'>-	while (hns_roce_v1_cmd_pending(hr_dev)) {</div><div class='del'>-		if (time_after(jiffies, end)) {</div><div class='del'>-			dev_err(hr_dev-&gt;dev, "jiffies=%d end=%d\n",</div><div class='del'>-				(int)jiffies, (int)end);</div><div class='del'>-			return -EAGAIN;</div><div class='del'>-		}</div><div class='del'>-		cond_resched();</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	tmp = cpu_to_le32(val);</div><div class='del'>-	roce_set_field(tmp, ROCEE_MB6_ROCEE_MB_CMD_M, ROCEE_MB6_ROCEE_MB_CMD_S,</div><div class='del'>-		       op);</div><div class='del'>-	roce_set_field(tmp, ROCEE_MB6_ROCEE_MB_CMD_MDF_M,</div><div class='del'>-		       ROCEE_MB6_ROCEE_MB_CMD_MDF_S, op_modifier);</div><div class='del'>-	roce_set_bit(tmp, ROCEE_MB6_ROCEE_MB_EVENT_S, event);</div><div class='del'>-	roce_set_bit(tmp, ROCEE_MB6_ROCEE_MB_HW_RUN_S, 1);</div><div class='del'>-	roce_set_field(tmp, ROCEE_MB6_ROCEE_MB_TOKEN_M,</div><div class='del'>-		       ROCEE_MB6_ROCEE_MB_TOKEN_S, token);</div><div class='del'>-</div><div class='del'>-	val = le32_to_cpu(tmp);</div><div class='del'>-	writeq(in_param, hcr + 0);</div><div class='del'>-	writeq(out_param, hcr + 2);</div><div class='del'>-	writel(in_modifier, hcr + 4);</div><div class='del'>-	/* Memory barrier */</div><div class='del'>-	wmb();</div><div class='del'>-</div><div class='del'>-	writel(val, hcr + 5);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_chk_mbox(struct hns_roce_dev *hr_dev,</div><div class='del'>-				unsigned long timeout)</div><div class='del'>-{</div><div class='del'>-	u8 __iomem *hcr = hr_dev-&gt;reg_base + ROCEE_MB1_REG;</div><div class='del'>-	unsigned long end = 0;</div><div class='del'>-	u32 status = 0;</div><div class='del'>-</div><div class='del'>-	end = msecs_to_jiffies(timeout) + jiffies;</div><div class='del'>-	while (hns_roce_v1_cmd_pending(hr_dev) &amp;&amp; time_before(jiffies, end))</div><div class='del'>-		cond_resched();</div><div class='del'>-</div><div class='del'>-	if (hns_roce_v1_cmd_pending(hr_dev)) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "[cmd_poll]hw run cmd TIMEDOUT!\n");</div><div class='del'>-		return -ETIMEDOUT;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	status = le32_to_cpu((__force __le32)</div><div class='del'>-			      __raw_readl(hcr + HCR_STATUS_OFFSET));</div><div class='del'>-	if ((status &amp; STATUS_MASK) != 0x1) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "mailbox status 0x%x!\n", status);</div><div class='del'>-		return -EBUSY;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port,</div><div class='del'>-			       int gid_index, const union ib_gid *gid,</div><div class='del'>-			       const struct ib_gid_attr *attr)</div><div class='del'>-{</div><div class='del'>-	u32 *p = NULL;</div><div class='del'>-	u8 gid_idx = 0;</div><div class='del'>-</div><div class='del'>-	gid_idx = hns_get_gid_index(hr_dev, port, gid_index);</div><div class='del'>-</div><div class='del'>-	p = (u32 *)&amp;gid-&gt;raw[0];</div><div class='del'>-	roce_raw_write(*p, hr_dev-&gt;reg_base + ROCEE_PORT_GID_L_0_REG +</div><div class='del'>-		       (HNS_ROCE_V1_GID_NUM * gid_idx));</div><div class='del'>-</div><div class='del'>-	p = (u32 *)&amp;gid-&gt;raw[4];</div><div class='del'>-	roce_raw_write(*p, hr_dev-&gt;reg_base + ROCEE_PORT_GID_ML_0_REG +</div><div class='del'>-		       (HNS_ROCE_V1_GID_NUM * gid_idx));</div><div class='del'>-</div><div class='del'>-	p = (u32 *)&amp;gid-&gt;raw[8];</div><div class='del'>-	roce_raw_write(*p, hr_dev-&gt;reg_base + ROCEE_PORT_GID_MH_0_REG +</div><div class='del'>-		       (HNS_ROCE_V1_GID_NUM * gid_idx));</div><div class='del'>-</div><div class='del'>-	p = (u32 *)&amp;gid-&gt;raw[0xc];</div><div class='del'>-	roce_raw_write(*p, hr_dev-&gt;reg_base + ROCEE_PORT_GID_H_0_REG +</div><div class='del'>-		       (HNS_ROCE_V1_GID_NUM * gid_idx));</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,</div><div class='del'>-			       u8 *addr)</div><div class='del'>-{</div><div class='del'>-	u32 reg_smac_l;</div><div class='del'>-	u16 reg_smac_h;</div><div class='del'>-	__le32 tmp;</div><div class='del'>-	u16 *p_h;</div><div class='del'>-	u32 *p;</div><div class='del'>-	u32 val;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * When mac changed, loopback may fail</div><div class='del'>-	 * because of smac not equal to dmac.</div><div class='del'>-	 * We Need to release and create reserved qp again.</div><div class='del'>-	 */</div><div class='del'>-	if (hr_dev-&gt;hw-&gt;dereg_mr) {</div><div class='del'>-		int ret;</div><div class='del'>-</div><div class='del'>-		ret = hns_roce_v1_recreate_lp_qp(hr_dev);</div><div class='del'>-		if (ret &amp;&amp; ret != -ETIMEDOUT)</div><div class='del'>-			return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	p = (u32 *)(&amp;addr[0]);</div><div class='del'>-	reg_smac_l = *p;</div><div class='del'>-	roce_raw_write(reg_smac_l, hr_dev-&gt;reg_base + ROCEE_SMAC_L_0_REG +</div><div class='del'>-		       PHY_PORT_OFFSET * phy_port);</div><div class='del'>-</div><div class='del'>-	val = roce_read(hr_dev,</div><div class='del'>-			ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET);</div><div class='del'>-	tmp = cpu_to_le32(val);</div><div class='del'>-	p_h = (u16 *)(&amp;addr[4]);</div><div class='del'>-	reg_smac_h  = *p_h;</div><div class='del'>-	roce_set_field(tmp, ROCEE_SMAC_H_ROCEE_SMAC_H_M,</div><div class='del'>-		       ROCEE_SMAC_H_ROCEE_SMAC_H_S, reg_smac_h);</div><div class='del'>-	val = le32_to_cpu(tmp);</div><div class='del'>-	roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET,</div><div class='del'>-		   val);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_set_mtu(struct hns_roce_dev *hr_dev, u8 phy_port,</div><div class='del'>-				enum ib_mtu mtu)</div><div class='del'>-{</div><div class='del'>-	__le32 tmp;</div><div class='del'>-	u32 val;</div><div class='del'>-</div><div class='del'>-	val = roce_read(hr_dev,</div><div class='del'>-			ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET);</div><div class='del'>-	tmp = cpu_to_le32(val);</div><div class='del'>-	roce_set_field(tmp, ROCEE_SMAC_H_ROCEE_PORT_MTU_M,</div><div class='del'>-		       ROCEE_SMAC_H_ROCEE_PORT_MTU_S, mtu);</div><div class='del'>-	val = le32_to_cpu(tmp);</div><div class='del'>-	roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET,</div><div class='del'>-		   val);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,</div><div class='del'>-				  unsigned long mtpt_idx)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_v1_mpt_entry *mpt_entry;</div><div class='del'>-	struct sg_dma_page_iter sg_iter;</div><div class='del'>-	u64 *pages;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	/* MPT filled into mailbox buf */</div><div class='del'>-	mpt_entry = (struct hns_roce_v1_mpt_entry *)mb_buf;</div><div class='del'>-	memset(mpt_entry, 0, sizeof(*mpt_entry));</div><div class='del'>-</div><div class='del'>-	roce_set_field(mpt_entry-&gt;mpt_byte_4, MPT_BYTE_4_KEY_STATE_M,</div><div class='del'>-		       MPT_BYTE_4_KEY_STATE_S, KEY_VALID);</div><div class='del'>-	roce_set_field(mpt_entry-&gt;mpt_byte_4, MPT_BYTE_4_KEY_M,</div><div class='del'>-		       MPT_BYTE_4_KEY_S, mr-&gt;key);</div><div class='del'>-	roce_set_field(mpt_entry-&gt;mpt_byte_4, MPT_BYTE_4_PAGE_SIZE_M,</div><div class='del'>-		       MPT_BYTE_4_PAGE_SIZE_S, MR_SIZE_4K);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;mpt_byte_4, MPT_BYTE_4_MW_TYPE_S, 0);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;mpt_byte_4, MPT_BYTE_4_MW_BIND_ENABLE_S,</div><div class='del'>-		     (mr-&gt;access &amp; IB_ACCESS_MW_BIND ? 1 : 0));</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;mpt_byte_4, MPT_BYTE_4_OWN_S, 0);</div><div class='del'>-	roce_set_field(mpt_entry-&gt;mpt_byte_4, MPT_BYTE_4_MEMORY_LOCATION_TYPE_M,</div><div class='del'>-		       MPT_BYTE_4_MEMORY_LOCATION_TYPE_S, mr-&gt;type);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;mpt_byte_4, MPT_BYTE_4_REMOTE_ATOMIC_S, 0);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;mpt_byte_4, MPT_BYTE_4_LOCAL_WRITE_S,</div><div class='del'>-		     (mr-&gt;access &amp; IB_ACCESS_LOCAL_WRITE ? 1 : 0));</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;mpt_byte_4, MPT_BYTE_4_REMOTE_WRITE_S,</div><div class='del'>-		     (mr-&gt;access &amp; IB_ACCESS_REMOTE_WRITE ? 1 : 0));</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;mpt_byte_4, MPT_BYTE_4_REMOTE_READ_S,</div><div class='del'>-		     (mr-&gt;access &amp; IB_ACCESS_REMOTE_READ ? 1 : 0));</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;mpt_byte_4, MPT_BYTE_4_REMOTE_INVAL_ENABLE_S,</div><div class='del'>-		     0);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;mpt_byte_4, MPT_BYTE_4_ADDRESS_TYPE_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(mpt_entry-&gt;mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,</div><div class='del'>-		       MPT_BYTE_12_PBL_ADDR_H_S, 0);</div><div class='del'>-	roce_set_field(mpt_entry-&gt;mpt_byte_12, MPT_BYTE_12_MW_BIND_COUNTER_M,</div><div class='del'>-		       MPT_BYTE_12_MW_BIND_COUNTER_S, 0);</div><div class='del'>-</div><div class='del'>-	mpt_entry-&gt;virt_addr_l = cpu_to_le32((u32)mr-&gt;iova);</div><div class='del'>-	mpt_entry-&gt;virt_addr_h = cpu_to_le32((u32)(mr-&gt;iova &gt;&gt; 32));</div><div class='del'>-	mpt_entry-&gt;length = cpu_to_le32((u32)mr-&gt;size);</div><div class='del'>-</div><div class='del'>-	roce_set_field(mpt_entry-&gt;mpt_byte_28, MPT_BYTE_28_PD_M,</div><div class='del'>-		       MPT_BYTE_28_PD_S, mr-&gt;pd);</div><div class='del'>-	roce_set_field(mpt_entry-&gt;mpt_byte_28, MPT_BYTE_28_L_KEY_IDX_L_M,</div><div class='del'>-		       MPT_BYTE_28_L_KEY_IDX_L_S, mtpt_idx);</div><div class='del'>-	roce_set_field(mpt_entry-&gt;mpt_byte_64, MPT_BYTE_64_L_KEY_IDX_H_M,</div><div class='del'>-		       MPT_BYTE_64_L_KEY_IDX_H_S, mtpt_idx &gt;&gt; MTPT_IDX_SHIFT);</div><div class='del'>-</div><div class='del'>-	/* DMA memory register */</div><div class='del'>-	if (mr-&gt;type == MR_TYPE_DMA)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	pages = (u64 *) __get_free_page(GFP_KERNEL);</div><div class='del'>-	if (!pages)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	i = 0;</div><div class='del'>-	for_each_sg_dma_page(mr-&gt;umem-&gt;sg_head.sgl, &amp;sg_iter, mr-&gt;umem-&gt;nmap, 0) {</div><div class='del'>-		pages[i] = ((u64)sg_page_iter_dma_address(&amp;sg_iter)) &gt;&gt; 12;</div><div class='del'>-</div><div class='del'>-		/* Directly record to MTPT table firstly 7 entry */</div><div class='del'>-		if (i &gt;= HNS_ROCE_MAX_INNER_MTPT_NUM)</div><div class='del'>-			break;</div><div class='del'>-		i++;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Register user mr */</div><div class='del'>-	for (i = 0; i &lt; HNS_ROCE_MAX_INNER_MTPT_NUM; i++) {</div><div class='del'>-		switch (i) {</div><div class='del'>-		case 0:</div><div class='del'>-			mpt_entry-&gt;pa0_l = cpu_to_le32((u32)(pages[i]));</div><div class='del'>-			roce_set_field(mpt_entry-&gt;mpt_byte_36,</div><div class='del'>-				MPT_BYTE_36_PA0_H_M,</div><div class='del'>-				MPT_BYTE_36_PA0_H_S,</div><div class='del'>-				(u32)(pages[i] &gt;&gt; PAGES_SHIFT_32));</div><div class='del'>-			break;</div><div class='del'>-		case 1:</div><div class='del'>-			roce_set_field(mpt_entry-&gt;mpt_byte_36,</div><div class='del'>-				       MPT_BYTE_36_PA1_L_M,</div><div class='del'>-				       MPT_BYTE_36_PA1_L_S, (u32)(pages[i]));</div><div class='del'>-			roce_set_field(mpt_entry-&gt;mpt_byte_40,</div><div class='del'>-				MPT_BYTE_40_PA1_H_M,</div><div class='del'>-				MPT_BYTE_40_PA1_H_S,</div><div class='del'>-				(u32)(pages[i] &gt;&gt; PAGES_SHIFT_24));</div><div class='del'>-			break;</div><div class='del'>-		case 2:</div><div class='del'>-			roce_set_field(mpt_entry-&gt;mpt_byte_40,</div><div class='del'>-				       MPT_BYTE_40_PA2_L_M,</div><div class='del'>-				       MPT_BYTE_40_PA2_L_S, (u32)(pages[i]));</div><div class='del'>-			roce_set_field(mpt_entry-&gt;mpt_byte_44,</div><div class='del'>-				MPT_BYTE_44_PA2_H_M,</div><div class='del'>-				MPT_BYTE_44_PA2_H_S,</div><div class='del'>-				(u32)(pages[i] &gt;&gt; PAGES_SHIFT_16));</div><div class='del'>-			break;</div><div class='del'>-		case 3:</div><div class='del'>-			roce_set_field(mpt_entry-&gt;mpt_byte_44,</div><div class='del'>-				       MPT_BYTE_44_PA3_L_M,</div><div class='del'>-				       MPT_BYTE_44_PA3_L_S, (u32)(pages[i]));</div><div class='del'>-			roce_set_field(mpt_entry-&gt;mpt_byte_48,</div><div class='del'>-				MPT_BYTE_48_PA3_H_M,</div><div class='del'>-				MPT_BYTE_48_PA3_H_S,</div><div class='del'>-				(u32)(pages[i] &gt;&gt; PAGES_SHIFT_8));</div><div class='del'>-			break;</div><div class='del'>-		case 4:</div><div class='del'>-			mpt_entry-&gt;pa4_l = cpu_to_le32((u32)(pages[i]));</div><div class='del'>-			roce_set_field(mpt_entry-&gt;mpt_byte_56,</div><div class='del'>-				MPT_BYTE_56_PA4_H_M,</div><div class='del'>-				MPT_BYTE_56_PA4_H_S,</div><div class='del'>-				(u32)(pages[i] &gt;&gt; PAGES_SHIFT_32));</div><div class='del'>-			break;</div><div class='del'>-		case 5:</div><div class='del'>-			roce_set_field(mpt_entry-&gt;mpt_byte_56,</div><div class='del'>-				       MPT_BYTE_56_PA5_L_M,</div><div class='del'>-				       MPT_BYTE_56_PA5_L_S, (u32)(pages[i]));</div><div class='del'>-			roce_set_field(mpt_entry-&gt;mpt_byte_60,</div><div class='del'>-				MPT_BYTE_60_PA5_H_M,</div><div class='del'>-				MPT_BYTE_60_PA5_H_S,</div><div class='del'>-				(u32)(pages[i] &gt;&gt; PAGES_SHIFT_24));</div><div class='del'>-			break;</div><div class='del'>-		case 6:</div><div class='del'>-			roce_set_field(mpt_entry-&gt;mpt_byte_60,</div><div class='del'>-				       MPT_BYTE_60_PA6_L_M,</div><div class='del'>-				       MPT_BYTE_60_PA6_L_S, (u32)(pages[i]));</div><div class='del'>-			roce_set_field(mpt_entry-&gt;mpt_byte_64,</div><div class='del'>-				MPT_BYTE_64_PA6_H_M,</div><div class='del'>-				MPT_BYTE_64_PA6_H_S,</div><div class='del'>-				(u32)(pages[i] &gt;&gt; PAGES_SHIFT_16));</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	free_page((unsigned long) pages);</div><div class='del'>-</div><div class='del'>-	mpt_entry-&gt;pbl_addr_l = cpu_to_le32((u32)(mr-&gt;pbl_dma_addr));</div><div class='del'>-</div><div class='del'>-	roce_set_field(mpt_entry-&gt;mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,</div><div class='del'>-		       MPT_BYTE_12_PBL_ADDR_H_S,</div><div class='del'>-		       ((u32)(mr-&gt;pbl_dma_addr &gt;&gt; 32)));</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void *get_cqe(struct hns_roce_cq *hr_cq, int n)</div><div class='del'>-{</div><div class='del'>-	return hns_roce_buf_offset(&amp;hr_cq-&gt;hr_buf.hr_buf,</div><div class='del'>-				   n * HNS_ROCE_V1_CQE_ENTRY_SIZE);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void *get_sw_cqe(struct hns_roce_cq *hr_cq, int n)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_cqe *hr_cqe = get_cqe(hr_cq, n &amp; hr_cq-&gt;ib_cq.cqe);</div><div class='del'>-</div><div class='del'>-	/* Get cqe when Owner bit is Conversely with the MSB of cons_idx */</div><div class='del'>-	return (roce_get_bit(hr_cqe-&gt;cqe_byte_4, CQE_BYTE_4_OWNER_S) ^</div><div class='del'>-		!!(n &amp; (hr_cq-&gt;ib_cq.cqe + 1))) ? hr_cqe : NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct hns_roce_cqe *next_cqe_sw(struct hns_roce_cq *hr_cq)</div><div class='del'>-{</div><div class='del'>-	return get_sw_cqe(hr_cq, hr_cq-&gt;cons_index);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)</div><div class='del'>-{</div><div class='del'>-	__le32 doorbell[2];</div><div class='del'>-</div><div class='del'>-	doorbell[0] = cpu_to_le32(cons_index &amp; ((hr_cq-&gt;cq_depth &lt;&lt; 1) - 1));</div><div class='del'>-	doorbell[1] = 0;</div><div class='del'>-	roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);</div><div class='del'>-	roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,</div><div class='del'>-		       ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3);</div><div class='del'>-	roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M,</div><div class='del'>-		       ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 0);</div><div class='del'>-	roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M,</div><div class='del'>-		       ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S, hr_cq-&gt;cqn);</div><div class='del'>-</div><div class='del'>-	hns_roce_write64_k(doorbell, hr_cq-&gt;cq_db_l);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void __hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,</div><div class='del'>-				   struct hns_roce_srq *srq)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_cqe *cqe, *dest;</div><div class='del'>-	u32 prod_index;</div><div class='del'>-	int nfreed = 0;</div><div class='del'>-	u8 owner_bit;</div><div class='del'>-</div><div class='del'>-	for (prod_index = hr_cq-&gt;cons_index; get_sw_cqe(hr_cq, prod_index);</div><div class='del'>-	     ++prod_index) {</div><div class='del'>-		if (prod_index == hr_cq-&gt;cons_index + hr_cq-&gt;ib_cq.cqe)</div><div class='del'>-			break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Now backwards through the CQ, removing CQ entries</div><div class='del'>-	 * that match our QP by overwriting them with next entries.</div><div class='del'>-	 */</div><div class='del'>-	while ((int) --prod_index - (int) hr_cq-&gt;cons_index &gt;= 0) {</div><div class='del'>-		cqe = get_cqe(hr_cq, prod_index &amp; hr_cq-&gt;ib_cq.cqe);</div><div class='del'>-		if ((roce_get_field(cqe-&gt;cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,</div><div class='del'>-				     CQE_BYTE_16_LOCAL_QPN_S) &amp;</div><div class='del'>-				     HNS_ROCE_CQE_QPN_MASK) == qpn) {</div><div class='del'>-			/* In v1 engine, not support SRQ */</div><div class='del'>-			++nfreed;</div><div class='del'>-		} else if (nfreed) {</div><div class='del'>-			dest = get_cqe(hr_cq, (prod_index + nfreed) &amp;</div><div class='del'>-				       hr_cq-&gt;ib_cq.cqe);</div><div class='del'>-			owner_bit = roce_get_bit(dest-&gt;cqe_byte_4,</div><div class='del'>-						 CQE_BYTE_4_OWNER_S);</div><div class='del'>-			memcpy(dest, cqe, sizeof(*cqe));</div><div class='del'>-			roce_set_bit(dest-&gt;cqe_byte_4, CQE_BYTE_4_OWNER_S,</div><div class='del'>-				     owner_bit);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (nfreed) {</div><div class='del'>-		hr_cq-&gt;cons_index += nfreed;</div><div class='del'>-		/*</div><div class='del'>-		 * Make sure update of buffer contents is done before</div><div class='del'>-		 * updating consumer index.</div><div class='del'>-		 */</div><div class='del'>-		wmb();</div><div class='del'>-</div><div class='del'>-		hns_roce_v1_cq_set_ci(hr_cq, hr_cq-&gt;cons_index);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,</div><div class='del'>-				 struct hns_roce_srq *srq)</div><div class='del'>-{</div><div class='del'>-	spin_lock_irq(&amp;hr_cq-&gt;lock);</div><div class='del'>-	__hns_roce_v1_cq_clean(hr_cq, qpn, srq);</div><div class='del'>-	spin_unlock_irq(&amp;hr_cq-&gt;lock);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev,</div><div class='del'>-				  struct hns_roce_cq *hr_cq, void *mb_buf,</div><div class='del'>-				  u64 *mtts, dma_addr_t dma_handle, int nent,</div><div class='del'>-				  u32 vector)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_cq_context *cq_context = NULL;</div><div class='del'>-	struct hns_roce_buf_list *tptr_buf;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-	dma_addr_t tptr_dma_addr;</div><div class='del'>-	int offset;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-	tptr_buf = &amp;priv-&gt;tptr_table.tptr_buf;</div><div class='del'>-</div><div class='del'>-	cq_context = mb_buf;</div><div class='del'>-	memset(cq_context, 0, sizeof(*cq_context));</div><div class='del'>-</div><div class='del'>-	/* Get the tptr for this CQ. */</div><div class='del'>-	offset = hr_cq-&gt;cqn * HNS_ROCE_V1_TPTR_ENTRY_SIZE;</div><div class='del'>-	tptr_dma_addr = tptr_buf-&gt;map + offset;</div><div class='del'>-	hr_cq-&gt;tptr_addr = (u16 *)(tptr_buf-&gt;buf + offset);</div><div class='del'>-</div><div class='del'>-	/* Register cq_context members */</div><div class='del'>-	roce_set_field(cq_context-&gt;cqc_byte_4,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S, CQ_STATE_VALID);</div><div class='del'>-	roce_set_field(cq_context-&gt;cqc_byte_4, CQ_CONTEXT_CQC_BYTE_4_CQN_M,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_4_CQN_S, hr_cq-&gt;cqn);</div><div class='del'>-</div><div class='del'>-	cq_context-&gt;cq_bt_l = cpu_to_le32((u32)dma_handle);</div><div class='del'>-</div><div class='del'>-	roce_set_field(cq_context-&gt;cqc_byte_12,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S,</div><div class='del'>-		       ((u64)dma_handle &gt;&gt; 32));</div><div class='del'>-	roce_set_field(cq_context-&gt;cqc_byte_12,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S,</div><div class='del'>-		       ilog2((unsigned int)nent));</div><div class='del'>-	roce_set_field(cq_context-&gt;cqc_byte_12, CQ_CONTEXT_CQC_BYTE_12_CEQN_M,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_12_CEQN_S, vector);</div><div class='del'>-</div><div class='del'>-	cq_context-&gt;cur_cqe_ba0_l = cpu_to_le32((u32)(mtts[0]));</div><div class='del'>-</div><div class='del'>-	roce_set_field(cq_context-&gt;cqc_byte_20,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S, (mtts[0]) &gt;&gt; 32);</div><div class='del'>-	/* Dedicated hardware, directly set 0 */</div><div class='del'>-	roce_set_field(cq_context-&gt;cqc_byte_20,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S, 0);</div><div class='del'>-	/**</div><div class='del'>-	 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of</div><div class='del'>-	 * using 4K page, and shift more 32 because of</div><div class='del'>-	 * caculating the high 32 bit value evaluated to hardware.</div><div class='del'>-	 */</div><div class='del'>-	roce_set_field(cq_context-&gt;cqc_byte_20,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S,</div><div class='del'>-		       tptr_dma_addr &gt;&gt; 44);</div><div class='del'>-</div><div class='del'>-	cq_context-&gt;cqe_tptr_addr_l = cpu_to_le32((u32)(tptr_dma_addr &gt;&gt; 12));</div><div class='del'>-</div><div class='del'>-	roce_set_field(cq_context-&gt;cqc_byte_32,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S, 0);</div><div class='del'>-	roce_set_bit(cq_context-&gt;cqc_byte_32,</div><div class='del'>-		     CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S, 0);</div><div class='del'>-	roce_set_bit(cq_context-&gt;cqc_byte_32,</div><div class='del'>-		     CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S, 0);</div><div class='del'>-	roce_set_bit(cq_context-&gt;cqc_byte_32,</div><div class='del'>-		     CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S, 0);</div><div class='del'>-	roce_set_bit(cq_context-&gt;cqc_byte_32,</div><div class='del'>-		     CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S,</div><div class='del'>-		     0);</div><div class='del'>-	/* The initial value of cq's ci is 0 */</div><div class='del'>-	roce_set_field(cq_context-&gt;cqc_byte_32,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M,</div><div class='del'>-		       CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S, 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)</div><div class='del'>-{</div><div class='del'>-	return -EOPNOTSUPP;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq,</div><div class='del'>-				     enum ib_cq_notify_flags flags)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);</div><div class='del'>-	u32 notification_flag;</div><div class='del'>-	__le32 doorbell[2];</div><div class='del'>-</div><div class='del'>-	notification_flag = (flags &amp; IB_CQ_SOLICITED_MASK) ==</div><div class='del'>-			    IB_CQ_SOLICITED ? CQ_DB_REQ_NOT : CQ_DB_REQ_NOT_SOL;</div><div class='del'>-	/*</div><div class='del'>-	 * flags = 0; Notification Flag = 1, next</div><div class='del'>-	 * flags = 1; Notification Flag = 0, solocited</div><div class='del'>-	 */</div><div class='del'>-	doorbell[0] =</div><div class='del'>-		cpu_to_le32(hr_cq-&gt;cons_index &amp; ((hr_cq-&gt;cq_depth &lt;&lt; 1) - 1));</div><div class='del'>-	roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);</div><div class='del'>-	roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,</div><div class='del'>-		       ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3);</div><div class='del'>-	roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M,</div><div class='del'>-		       ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 1);</div><div class='del'>-	roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M,</div><div class='del'>-		       ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S,</div><div class='del'>-		       hr_cq-&gt;cqn | notification_flag);</div><div class='del'>-</div><div class='del'>-	hns_roce_write64_k(doorbell, hr_cq-&gt;cq_db_l);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq,</div><div class='del'>-				struct hns_roce_qp **cur_qp, struct ib_wc *wc)</div><div class='del'>-{</div><div class='del'>-	int qpn;</div><div class='del'>-	int is_send;</div><div class='del'>-	u16 wqe_ctr;</div><div class='del'>-	u32 status;</div><div class='del'>-	u32 opcode;</div><div class='del'>-	struct hns_roce_cqe *cqe;</div><div class='del'>-	struct hns_roce_qp *hr_qp;</div><div class='del'>-	struct hns_roce_wq *wq;</div><div class='del'>-	struct hns_roce_wqe_ctrl_seg *sq_wqe;</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq-&gt;ib_cq.device);</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-</div><div class='del'>-	/* Find cqe according consumer index */</div><div class='del'>-	cqe = next_cqe_sw(hr_cq);</div><div class='del'>-	if (!cqe)</div><div class='del'>-		return -EAGAIN;</div><div class='del'>-</div><div class='del'>-	++hr_cq-&gt;cons_index;</div><div class='del'>-	/* Memory barrier */</div><div class='del'>-	rmb();</div><div class='del'>-	/* 0-&gt;SQ, 1-&gt;RQ */</div><div class='del'>-	is_send  = !(roce_get_bit(cqe-&gt;cqe_byte_4, CQE_BYTE_4_SQ_RQ_FLAG_S));</div><div class='del'>-</div><div class='del'>-	/* Local_qpn in UD cqe is always 1, so it needs to compute new qpn */</div><div class='del'>-	if (roce_get_field(cqe-&gt;cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,</div><div class='del'>-			   CQE_BYTE_16_LOCAL_QPN_S) &lt;= 1) {</div><div class='del'>-		qpn = roce_get_field(cqe-&gt;cqe_byte_20, CQE_BYTE_20_PORT_NUM_M,</div><div class='del'>-				     CQE_BYTE_20_PORT_NUM_S) +</div><div class='del'>-		      roce_get_field(cqe-&gt;cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,</div><div class='del'>-				     CQE_BYTE_16_LOCAL_QPN_S) *</div><div class='del'>-				     HNS_ROCE_MAX_PORTS;</div><div class='del'>-	} else {</div><div class='del'>-		qpn = roce_get_field(cqe-&gt;cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,</div><div class='del'>-				     CQE_BYTE_16_LOCAL_QPN_S);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!*cur_qp || (qpn &amp; HNS_ROCE_CQE_QPN_MASK) != (*cur_qp)-&gt;qpn) {</div><div class='del'>-		hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);</div><div class='del'>-		if (unlikely(!hr_qp)) {</div><div class='del'>-			dev_err(dev, "CQ %06lx with entry for unknown QPN %06x\n",</div><div class='del'>-				hr_cq-&gt;cqn, (qpn &amp; HNS_ROCE_CQE_QPN_MASK));</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		*cur_qp = hr_qp;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	wc-&gt;qp = &amp;(*cur_qp)-&gt;ibqp;</div><div class='del'>-	wc-&gt;vendor_err = 0;</div><div class='del'>-</div><div class='del'>-	status = roce_get_field(cqe-&gt;cqe_byte_4,</div><div class='del'>-				CQE_BYTE_4_STATUS_OF_THE_OPERATION_M,</div><div class='del'>-				CQE_BYTE_4_STATUS_OF_THE_OPERATION_S) &amp;</div><div class='del'>-				HNS_ROCE_CQE_STATUS_MASK;</div><div class='del'>-	switch (status) {</div><div class='del'>-	case HNS_ROCE_CQE_SUCCESS:</div><div class='del'>-		wc-&gt;status = IB_WC_SUCCESS;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_LOC_LEN_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_LOC_QP_OP_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_LOC_PROT_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_WR_FLUSH_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_MW_BIND_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_BAD_RESP_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_LOC_ACCESS_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_REM_INV_REQ_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_REM_ACCESS_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_REM_OP_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_RETRY_EXC_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_RNR_RETRY_EXC_ERR;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		wc-&gt;status = IB_WC_GENERAL_ERR;</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* CQE status error, directly return */</div><div class='del'>-	if (wc-&gt;status != IB_WC_SUCCESS)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	if (is_send) {</div><div class='del'>-		/* SQ conrespond to CQE */</div><div class='del'>-		sq_wqe = get_send_wqe(*cur_qp, roce_get_field(cqe-&gt;cqe_byte_4,</div><div class='del'>-						CQE_BYTE_4_WQE_INDEX_M,</div><div class='del'>-						CQE_BYTE_4_WQE_INDEX_S)&amp;</div><div class='del'>-						((*cur_qp)-&gt;sq.wqe_cnt-1));</div><div class='del'>-		switch (le32_to_cpu(sq_wqe-&gt;flag) &amp; HNS_ROCE_WQE_OPCODE_MASK) {</div><div class='del'>-		case HNS_ROCE_WQE_OPCODE_SEND:</div><div class='del'>-			wc-&gt;opcode = IB_WC_SEND;</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_WQE_OPCODE_RDMA_READ:</div><div class='del'>-			wc-&gt;opcode = IB_WC_RDMA_READ;</div><div class='del'>-			wc-&gt;byte_len = le32_to_cpu(cqe-&gt;byte_cnt);</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_WQE_OPCODE_RDMA_WRITE:</div><div class='del'>-			wc-&gt;opcode = IB_WC_RDMA_WRITE;</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_WQE_OPCODE_LOCAL_INV:</div><div class='del'>-			wc-&gt;opcode = IB_WC_LOCAL_INV;</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_WQE_OPCODE_UD_SEND:</div><div class='del'>-			wc-&gt;opcode = IB_WC_SEND;</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			wc-&gt;status = IB_WC_GENERAL_ERR;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		wc-&gt;wc_flags = (le32_to_cpu(sq_wqe-&gt;flag) &amp; HNS_ROCE_WQE_IMM ?</div><div class='del'>-				IB_WC_WITH_IMM : 0);</div><div class='del'>-</div><div class='del'>-		wq = &amp;(*cur_qp)-&gt;sq;</div><div class='del'>-		if ((*cur_qp)-&gt;sq_signal_bits) {</div><div class='del'>-			/*</div><div class='del'>-			 * If sg_signal_bit is 1,</div><div class='del'>-			 * firstly tail pointer updated to wqe</div><div class='del'>-			 * which current cqe correspond to</div><div class='del'>-			 */</div><div class='del'>-			wqe_ctr = (u16)roce_get_field(cqe-&gt;cqe_byte_4,</div><div class='del'>-						      CQE_BYTE_4_WQE_INDEX_M,</div><div class='del'>-						      CQE_BYTE_4_WQE_INDEX_S);</div><div class='del'>-			wq-&gt;tail += (wqe_ctr - (u16)wq-&gt;tail) &amp;</div><div class='del'>-				    (wq-&gt;wqe_cnt - 1);</div><div class='del'>-		}</div><div class='del'>-		wc-&gt;wr_id = wq-&gt;wrid[wq-&gt;tail &amp; (wq-&gt;wqe_cnt - 1)];</div><div class='del'>-		++wq-&gt;tail;</div><div class='del'>-	} else {</div><div class='del'>-		/* RQ conrespond to CQE */</div><div class='del'>-		wc-&gt;byte_len = le32_to_cpu(cqe-&gt;byte_cnt);</div><div class='del'>-		opcode = roce_get_field(cqe-&gt;cqe_byte_4,</div><div class='del'>-					CQE_BYTE_4_OPERATION_TYPE_M,</div><div class='del'>-					CQE_BYTE_4_OPERATION_TYPE_S) &amp;</div><div class='del'>-					HNS_ROCE_CQE_OPCODE_MASK;</div><div class='del'>-		switch (opcode) {</div><div class='del'>-		case HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE:</div><div class='del'>-			wc-&gt;opcode = IB_WC_RECV_RDMA_WITH_IMM;</div><div class='del'>-			wc-&gt;wc_flags = IB_WC_WITH_IMM;</div><div class='del'>-			wc-&gt;ex.imm_data =</div><div class='del'>-				cpu_to_be32(le32_to_cpu(cqe-&gt;immediate_data));</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_OPCODE_SEND_DATA_RECEIVE:</div><div class='del'>-			if (roce_get_bit(cqe-&gt;cqe_byte_4,</div><div class='del'>-					 CQE_BYTE_4_IMM_INDICATOR_S)) {</div><div class='del'>-				wc-&gt;opcode = IB_WC_RECV;</div><div class='del'>-				wc-&gt;wc_flags = IB_WC_WITH_IMM;</div><div class='del'>-				wc-&gt;ex.imm_data = cpu_to_be32(</div><div class='del'>-					le32_to_cpu(cqe-&gt;immediate_data));</div><div class='del'>-			} else {</div><div class='del'>-				wc-&gt;opcode = IB_WC_RECV;</div><div class='del'>-				wc-&gt;wc_flags = 0;</div><div class='del'>-			}</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			wc-&gt;status = IB_WC_GENERAL_ERR;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		/* Update tail pointer, record wr_id */</div><div class='del'>-		wq = &amp;(*cur_qp)-&gt;rq;</div><div class='del'>-		wc-&gt;wr_id = wq-&gt;wrid[wq-&gt;tail &amp; (wq-&gt;wqe_cnt - 1)];</div><div class='del'>-		++wq-&gt;tail;</div><div class='del'>-		wc-&gt;sl = (u8)roce_get_field(cqe-&gt;cqe_byte_20, CQE_BYTE_20_SL_M,</div><div class='del'>-					    CQE_BYTE_20_SL_S);</div><div class='del'>-		wc-&gt;src_qp = (u8)roce_get_field(cqe-&gt;cqe_byte_20,</div><div class='del'>-						CQE_BYTE_20_REMOTE_QPN_M,</div><div class='del'>-						CQE_BYTE_20_REMOTE_QPN_S);</div><div class='del'>-		wc-&gt;wc_flags |= (roce_get_bit(cqe-&gt;cqe_byte_20,</div><div class='del'>-					      CQE_BYTE_20_GRH_PRESENT_S) ?</div><div class='del'>-					      IB_WC_GRH : 0);</div><div class='del'>-		wc-&gt;pkey_index = (u16)roce_get_field(cqe-&gt;cqe_byte_28,</div><div class='del'>-						     CQE_BYTE_28_P_KEY_IDX_M,</div><div class='del'>-						     CQE_BYTE_28_P_KEY_IDX_S);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);</div><div class='del'>-	struct hns_roce_qp *cur_qp = NULL;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int npolled;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;hr_cq-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-	for (npolled = 0; npolled &lt; num_entries; ++npolled) {</div><div class='del'>-		ret = hns_roce_v1_poll_one(hr_cq, &amp;cur_qp, wc + npolled);</div><div class='del'>-		if (ret)</div><div class='del'>-			break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (npolled) {</div><div class='del'>-		*hr_cq-&gt;tptr_addr = hr_cq-&gt;cons_index &amp;</div><div class='del'>-			((hr_cq-&gt;cq_depth &lt;&lt; 1) - 1);</div><div class='del'>-</div><div class='del'>-		/* Memroy barrier */</div><div class='del'>-		wmb();</div><div class='del'>-		hns_roce_v1_cq_set_ci(hr_cq, hr_cq-&gt;cons_index);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	spin_unlock_irqrestore(&amp;hr_cq-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-	if (ret == 0 || ret == -EAGAIN)</div><div class='del'>-		return npolled;</div><div class='del'>-	else</div><div class='del'>-		return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev,</div><div class='del'>-				 struct hns_roce_hem_table *table, int obj,</div><div class='del'>-				 int step_idx)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_v1_priv *priv;</div><div class='del'>-	unsigned long end = 0, flags = 0;</div><div class='del'>-	__le32 bt_cmd_val[2] = {0};</div><div class='del'>-	void __iomem *bt_cmd;</div><div class='del'>-	u64 bt_ba = 0;</div><div class='del'>-</div><div class='del'>-	priv = (struct hns_roce_v1_priv *)hr_dev-&gt;priv;</div><div class='del'>-</div><div class='del'>-	switch (table-&gt;type) {</div><div class='del'>-	case HEM_TYPE_QPC:</div><div class='del'>-		roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,</div><div class='del'>-			ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);</div><div class='del'>-		bt_ba = priv-&gt;bt_table.qpc_buf.map &gt;&gt; 12;</div><div class='del'>-		break;</div><div class='del'>-	case HEM_TYPE_MTPT:</div><div class='del'>-		roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,</div><div class='del'>-			ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_MTPT);</div><div class='del'>-		bt_ba = priv-&gt;bt_table.mtpt_buf.map &gt;&gt; 12;</div><div class='del'>-		break;</div><div class='del'>-	case HEM_TYPE_CQC:</div><div class='del'>-		roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,</div><div class='del'>-			ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);</div><div class='del'>-		bt_ba = priv-&gt;bt_table.cqc_buf.map &gt;&gt; 12;</div><div class='del'>-		break;</div><div class='del'>-	case HEM_TYPE_SRQC:</div><div class='del'>-		dev_dbg(dev, "HEM_TYPE_SRQC not support.\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	default:</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='del'>-	roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,</div><div class='del'>-		ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);</div><div class='del'>-	roce_set_bit(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);</div><div class='del'>-	roce_set_bit(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;hr_dev-&gt;bt_cmd_lock, flags);</div><div class='del'>-</div><div class='del'>-	bt_cmd = hr_dev-&gt;reg_base + ROCEE_BT_CMD_H_REG;</div><div class='del'>-</div><div class='del'>-	end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;</div><div class='del'>-	while (1) {</div><div class='del'>-		if (readl(bt_cmd) &gt;&gt; BT_CMD_SYNC_SHIFT) {</div><div class='del'>-			if (!(time_before(jiffies, end))) {</div><div class='del'>-				dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");</div><div class='del'>-				spin_unlock_irqrestore(&amp;hr_dev-&gt;bt_cmd_lock,</div><div class='del'>-					flags);</div><div class='del'>-				return -EBUSY;</div><div class='del'>-			}</div><div class='del'>-		} else {</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		msleep(HW_SYNC_SLEEP_TIME_INTERVAL);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	bt_cmd_val[0] = (__le32)bt_ba;</div><div class='del'>-	roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,</div><div class='del'>-		ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, bt_ba &gt;&gt; 32);</div><div class='del'>-	hns_roce_write64_k(bt_cmd_val, hr_dev-&gt;reg_base + ROCEE_BT_CMD_L_REG);</div><div class='del'>-</div><div class='del'>-	spin_unlock_irqrestore(&amp;hr_dev-&gt;bt_cmd_lock, flags);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_qp_modify(struct hns_roce_dev *hr_dev,</div><div class='del'>-				 struct hns_roce_mtt *mtt,</div><div class='del'>-				 enum hns_roce_qp_state cur_state,</div><div class='del'>-				 enum hns_roce_qp_state new_state,</div><div class='del'>-				 struct hns_roce_qp_context *context,</div><div class='del'>-				 struct hns_roce_qp *hr_qp)</div><div class='del'>-{</div><div class='del'>-	static const u16</div><div class='del'>-	op[HNS_ROCE_QP_NUM_STATE][HNS_ROCE_QP_NUM_STATE] = {</div><div class='del'>-		[HNS_ROCE_QP_STATE_RST] = {</div><div class='del'>-		[HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,</div><div class='del'>-		[HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,</div><div class='del'>-		[HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP,</div><div class='del'>-		},</div><div class='del'>-		[HNS_ROCE_QP_STATE_INIT] = {</div><div class='del'>-		[HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,</div><div class='del'>-		[HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,</div><div class='del'>-		/* Note: In v1 engine, HW doesn't support RST2INIT.</div><div class='del'>-		 * We use RST2INIT cmd instead of INIT2INIT.</div><div class='del'>-		 */</div><div class='del'>-		[HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP,</div><div class='del'>-		[HNS_ROCE_QP_STATE_RTR] = HNS_ROCE_CMD_INIT2RTR_QP,</div><div class='del'>-		},</div><div class='del'>-		[HNS_ROCE_QP_STATE_RTR] = {</div><div class='del'>-		[HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,</div><div class='del'>-		[HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,</div><div class='del'>-		[HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTR2RTS_QP,</div><div class='del'>-		},</div><div class='del'>-		[HNS_ROCE_QP_STATE_RTS] = {</div><div class='del'>-		[HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,</div><div class='del'>-		[HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,</div><div class='del'>-		[HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTS2RTS_QP,</div><div class='del'>-		[HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_RTS2SQD_QP,</div><div class='del'>-		},</div><div class='del'>-		[HNS_ROCE_QP_STATE_SQD] = {</div><div class='del'>-		[HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,</div><div class='del'>-		[HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,</div><div class='del'>-		[HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_SQD2RTS_QP,</div><div class='del'>-		[HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_SQD2SQD_QP,</div><div class='del'>-		},</div><div class='del'>-		[HNS_ROCE_QP_STATE_ERR] = {</div><div class='del'>-		[HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,</div><div class='del'>-		[HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,</div><div class='del'>-		}</div><div class='del'>-	};</div><div class='del'>-</div><div class='del'>-	struct hns_roce_cmd_mailbox *mailbox;</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	if (cur_state &gt;= HNS_ROCE_QP_NUM_STATE ||</div><div class='del'>-	    new_state &gt;= HNS_ROCE_QP_NUM_STATE ||</div><div class='del'>-	    !op[cur_state][new_state]) {</div><div class='del'>-		dev_err(dev, "[modify_qp]not support state %d to %d\n",</div><div class='del'>-			cur_state, new_state);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (op[cur_state][new_state] == HNS_ROCE_CMD_2RST_QP)</div><div class='del'>-		return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp-&gt;qpn, 2,</div><div class='del'>-					 HNS_ROCE_CMD_2RST_QP,</div><div class='del'>-					 HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='del'>-</div><div class='del'>-	if (op[cur_state][new_state] == HNS_ROCE_CMD_2ERR_QP)</div><div class='del'>-		return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp-&gt;qpn, 2,</div><div class='del'>-					 HNS_ROCE_CMD_2ERR_QP,</div><div class='del'>-					 HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='del'>-</div><div class='del'>-	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='del'>-	if (IS_ERR(mailbox))</div><div class='del'>-		return PTR_ERR(mailbox);</div><div class='del'>-</div><div class='del'>-	memcpy(mailbox-&gt;buf, context, sizeof(*context));</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_cmd_mbox(hr_dev, mailbox-&gt;dma, 0, hr_qp-&gt;qpn, 0,</div><div class='del'>-				op[cur_state][new_state],</div><div class='del'>-				HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='del'>-</div><div class='del'>-	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,</div><div class='del'>-			     int attr_mask, enum ib_qp_state cur_state,</div><div class='del'>-			     enum ib_qp_state new_state)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='del'>-	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='del'>-	struct hns_roce_sqp_context *context;</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	dma_addr_t dma_handle = 0;</div><div class='del'>-	u32 __iomem *addr;</div><div class='del'>-	int rq_pa_start;</div><div class='del'>-	__le32 tmp;</div><div class='del'>-	u32 reg_val;</div><div class='del'>-	u64 *mtts;</div><div class='del'>-</div><div class='del'>-	context = kzalloc(sizeof(*context), GFP_KERNEL);</div><div class='del'>-	if (!context)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	/* Search QP buf's MTTs */</div><div class='del'>-	mtts = hns_roce_table_find(hr_dev, &amp;hr_dev-&gt;mr_table.mtt_table,</div><div class='del'>-				   hr_qp-&gt;mtt.first_seg, &amp;dma_handle);</div><div class='del'>-	if (!mtts) {</div><div class='del'>-		dev_err(dev, "qp buf pa find failed\n");</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (cur_state == IB_QPS_RESET &amp;&amp; new_state == IB_QPS_INIT) {</div><div class='del'>-		roce_set_field(context-&gt;qp1c_bytes_4,</div><div class='del'>-			       QP1C_BYTES_4_SQ_WQE_SHIFT_M,</div><div class='del'>-			       QP1C_BYTES_4_SQ_WQE_SHIFT_S,</div><div class='del'>-			       ilog2((unsigned int)hr_qp-&gt;sq.wqe_cnt));</div><div class='del'>-		roce_set_field(context-&gt;qp1c_bytes_4,</div><div class='del'>-			       QP1C_BYTES_4_RQ_WQE_SHIFT_M,</div><div class='del'>-			       QP1C_BYTES_4_RQ_WQE_SHIFT_S,</div><div class='del'>-			       ilog2((unsigned int)hr_qp-&gt;rq.wqe_cnt));</div><div class='del'>-		roce_set_field(context-&gt;qp1c_bytes_4, QP1C_BYTES_4_PD_M,</div><div class='del'>-			       QP1C_BYTES_4_PD_S, to_hr_pd(ibqp-&gt;pd)-&gt;pdn);</div><div class='del'>-</div><div class='del'>-		context-&gt;sq_rq_bt_l = cpu_to_le32((u32)(dma_handle));</div><div class='del'>-		roce_set_field(context-&gt;qp1c_bytes_12,</div><div class='del'>-			       QP1C_BYTES_12_SQ_RQ_BT_H_M,</div><div class='del'>-			       QP1C_BYTES_12_SQ_RQ_BT_H_S,</div><div class='del'>-			       ((u32)(dma_handle &gt;&gt; 32)));</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qp1c_bytes_16, QP1C_BYTES_16_RQ_HEAD_M,</div><div class='del'>-			       QP1C_BYTES_16_RQ_HEAD_S, hr_qp-&gt;rq.head);</div><div class='del'>-		roce_set_field(context-&gt;qp1c_bytes_16, QP1C_BYTES_16_PORT_NUM_M,</div><div class='del'>-			       QP1C_BYTES_16_PORT_NUM_S, hr_qp-&gt;phy_port);</div><div class='del'>-		roce_set_bit(context-&gt;qp1c_bytes_16,</div><div class='del'>-			     QP1C_BYTES_16_SIGNALING_TYPE_S,</div><div class='del'>-			     le32_to_cpu(hr_qp-&gt;sq_signal_bits));</div><div class='del'>-		roce_set_bit(context-&gt;qp1c_bytes_16, QP1C_BYTES_16_RQ_BA_FLG_S,</div><div class='del'>-			     1);</div><div class='del'>-		roce_set_bit(context-&gt;qp1c_bytes_16, QP1C_BYTES_16_SQ_BA_FLG_S,</div><div class='del'>-			     1);</div><div class='del'>-		roce_set_bit(context-&gt;qp1c_bytes_16, QP1C_BYTES_16_QP1_ERR_S,</div><div class='del'>-			     0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qp1c_bytes_20, QP1C_BYTES_20_SQ_HEAD_M,</div><div class='del'>-			       QP1C_BYTES_20_SQ_HEAD_S, hr_qp-&gt;sq.head);</div><div class='del'>-		roce_set_field(context-&gt;qp1c_bytes_20, QP1C_BYTES_20_PKEY_IDX_M,</div><div class='del'>-			       QP1C_BYTES_20_PKEY_IDX_S, attr-&gt;pkey_index);</div><div class='del'>-</div><div class='del'>-		rq_pa_start = (u32)hr_qp-&gt;rq.offset / PAGE_SIZE;</div><div class='del'>-		context-&gt;cur_rq_wqe_ba_l =</div><div class='del'>-				cpu_to_le32((u32)(mtts[rq_pa_start]));</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qp1c_bytes_28,</div><div class='del'>-			       QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M,</div><div class='del'>-			       QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S,</div><div class='del'>-			       (mtts[rq_pa_start]) &gt;&gt; 32);</div><div class='del'>-		roce_set_field(context-&gt;qp1c_bytes_28,</div><div class='del'>-			       QP1C_BYTES_28_RQ_CUR_IDX_M,</div><div class='del'>-			       QP1C_BYTES_28_RQ_CUR_IDX_S, 0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qp1c_bytes_32,</div><div class='del'>-			       QP1C_BYTES_32_RX_CQ_NUM_M,</div><div class='del'>-			       QP1C_BYTES_32_RX_CQ_NUM_S,</div><div class='del'>-			       to_hr_cq(ibqp-&gt;recv_cq)-&gt;cqn);</div><div class='del'>-		roce_set_field(context-&gt;qp1c_bytes_32,</div><div class='del'>-			       QP1C_BYTES_32_TX_CQ_NUM_M,</div><div class='del'>-			       QP1C_BYTES_32_TX_CQ_NUM_S,</div><div class='del'>-			       to_hr_cq(ibqp-&gt;send_cq)-&gt;cqn);</div><div class='del'>-</div><div class='del'>-		context-&gt;cur_sq_wqe_ba_l  = cpu_to_le32((u32)mtts[0]);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qp1c_bytes_40,</div><div class='del'>-			       QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M,</div><div class='del'>-			       QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S,</div><div class='del'>-			       (mtts[0]) &gt;&gt; 32);</div><div class='del'>-		roce_set_field(context-&gt;qp1c_bytes_40,</div><div class='del'>-			       QP1C_BYTES_40_SQ_CUR_IDX_M,</div><div class='del'>-			       QP1C_BYTES_40_SQ_CUR_IDX_S, 0);</div><div class='del'>-</div><div class='del'>-		/* Copy context to QP1C register */</div><div class='del'>-		addr = (u32 __iomem *)(hr_dev-&gt;reg_base +</div><div class='del'>-				       ROCEE_QP1C_CFG0_0_REG +</div><div class='del'>-				       hr_qp-&gt;phy_port * sizeof(*context));</div><div class='del'>-</div><div class='del'>-		writel(le32_to_cpu(context-&gt;qp1c_bytes_4), addr);</div><div class='del'>-		writel(le32_to_cpu(context-&gt;sq_rq_bt_l), addr + 1);</div><div class='del'>-		writel(le32_to_cpu(context-&gt;qp1c_bytes_12), addr + 2);</div><div class='del'>-		writel(le32_to_cpu(context-&gt;qp1c_bytes_16), addr + 3);</div><div class='del'>-		writel(le32_to_cpu(context-&gt;qp1c_bytes_20), addr + 4);</div><div class='del'>-		writel(le32_to_cpu(context-&gt;cur_rq_wqe_ba_l), addr + 5);</div><div class='del'>-		writel(le32_to_cpu(context-&gt;qp1c_bytes_28), addr + 6);</div><div class='del'>-		writel(le32_to_cpu(context-&gt;qp1c_bytes_32), addr + 7);</div><div class='del'>-		writel(le32_to_cpu(context-&gt;cur_sq_wqe_ba_l), addr + 8);</div><div class='del'>-		writel(le32_to_cpu(context-&gt;qp1c_bytes_40), addr + 9);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Modify QP1C status */</div><div class='del'>-	reg_val = roce_read(hr_dev, ROCEE_QP1C_CFG0_0_REG +</div><div class='del'>-			    hr_qp-&gt;phy_port * sizeof(*context));</div><div class='del'>-	tmp = cpu_to_le32(reg_val);</div><div class='del'>-	roce_set_field(tmp, ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M,</div><div class='del'>-		       ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S, new_state);</div><div class='del'>-	reg_val = le32_to_cpu(tmp);</div><div class='del'>-	roce_write(hr_dev, ROCEE_QP1C_CFG0_0_REG +</div><div class='del'>-		    hr_qp-&gt;phy_port * sizeof(*context), reg_val);</div><div class='del'>-</div><div class='del'>-	hr_qp-&gt;state = new_state;</div><div class='del'>-	if (new_state == IB_QPS_RESET) {</div><div class='del'>-		hns_roce_v1_cq_clean(to_hr_cq(ibqp-&gt;recv_cq), hr_qp-&gt;qpn,</div><div class='del'>-				     ibqp-&gt;srq ? to_hr_srq(ibqp-&gt;srq) : NULL);</div><div class='del'>-		if (ibqp-&gt;send_cq != ibqp-&gt;recv_cq)</div><div class='del'>-			hns_roce_v1_cq_clean(to_hr_cq(ibqp-&gt;send_cq),</div><div class='del'>-					     hr_qp-&gt;qpn, NULL);</div><div class='del'>-</div><div class='del'>-		hr_qp-&gt;rq.head = 0;</div><div class='del'>-		hr_qp-&gt;rq.tail = 0;</div><div class='del'>-		hr_qp-&gt;sq.head = 0;</div><div class='del'>-		hr_qp-&gt;sq.tail = 0;</div><div class='del'>-		hr_qp-&gt;sq_next_wqe = 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	kfree(context);</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-out:</div><div class='del'>-	kfree(context);</div><div class='del'>-	return -EINVAL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,</div><div class='del'>-			    int attr_mask, enum ib_qp_state cur_state,</div><div class='del'>-			    enum ib_qp_state new_state)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='del'>-	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_qp_context *context;</div><div class='del'>-	const struct ib_global_route *grh = rdma_ah_read_grh(&amp;attr-&gt;ah_attr);</div><div class='del'>-	dma_addr_t dma_handle_2 = 0;</div><div class='del'>-	dma_addr_t dma_handle = 0;</div><div class='del'>-	__le32 doorbell[2] = {0};</div><div class='del'>-	int rq_pa_start = 0;</div><div class='del'>-	u64 *mtts_2 = NULL;</div><div class='del'>-	int ret = -EINVAL;</div><div class='del'>-	u64 *mtts = NULL;</div><div class='del'>-	int port;</div><div class='del'>-	u8 port_num;</div><div class='del'>-	u8 *dmac;</div><div class='del'>-	u8 *smac;</div><div class='del'>-</div><div class='del'>-	context = kzalloc(sizeof(*context), GFP_KERNEL);</div><div class='del'>-	if (!context)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	/* Search qp buf's mtts */</div><div class='del'>-	mtts = hns_roce_table_find(hr_dev, &amp;hr_dev-&gt;mr_table.mtt_table,</div><div class='del'>-				   hr_qp-&gt;mtt.first_seg, &amp;dma_handle);</div><div class='del'>-	if (mtts == NULL) {</div><div class='del'>-		dev_err(dev, "qp buf pa find failed\n");</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Search IRRL's mtts */</div><div class='del'>-	mtts_2 = hns_roce_table_find(hr_dev, &amp;hr_dev-&gt;qp_table.irrl_table,</div><div class='del'>-				     hr_qp-&gt;qpn, &amp;dma_handle_2);</div><div class='del'>-	if (mtts_2 == NULL) {</div><div class='del'>-		dev_err(dev, "qp irrl_table find failed\n");</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Reset to init</div><div class='del'>-	 *	Mandatory param:</div><div class='del'>-	 *	IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS</div><div class='del'>-	 *	Optional param: NA</div><div class='del'>-	 */</div><div class='del'>-	if (cur_state == IB_QPS_RESET &amp;&amp; new_state == IB_QPS_INIT) {</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_4,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S,</div><div class='del'>-			       to_hr_qp_type(hr_qp-&gt;ibqp.qp_type));</div><div class='del'>-</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_4,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0);</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_4,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,</div><div class='del'>-			     !!(attr-&gt;qp_access_flags &amp; IB_ACCESS_REMOTE_READ));</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_4,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,</div><div class='del'>-			     !!(attr-&gt;qp_access_flags &amp; IB_ACCESS_REMOTE_WRITE)</div><div class='del'>-			     );</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_4,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S,</div><div class='del'>-			     !!(attr-&gt;qp_access_flags &amp; IB_ACCESS_REMOTE_ATOMIC)</div><div class='del'>-			     );</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_4,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_4,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S,</div><div class='del'>-			       ilog2((unsigned int)hr_qp-&gt;sq.wqe_cnt));</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_4,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S,</div><div class='del'>-			       ilog2((unsigned int)hr_qp-&gt;rq.wqe_cnt));</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_4,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_4_PD_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_4_PD_S,</div><div class='del'>-			       to_hr_pd(ibqp-&gt;pd)-&gt;pdn);</div><div class='del'>-		hr_qp-&gt;access_flags = attr-&gt;qp_access_flags;</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_8,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S,</div><div class='del'>-			       to_hr_cq(ibqp-&gt;send_cq)-&gt;cqn);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_8,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S,</div><div class='del'>-			       to_hr_cq(ibqp-&gt;recv_cq)-&gt;cqn);</div><div class='del'>-</div><div class='del'>-		if (ibqp-&gt;srq)</div><div class='del'>-			roce_set_field(context-&gt;qpc_bytes_12,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S,</div><div class='del'>-				       to_hr_srq(ibqp-&gt;srq)-&gt;srqn);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_12,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,</div><div class='del'>-			       attr-&gt;pkey_index);</div><div class='del'>-		hr_qp-&gt;pkey_index = attr-&gt;pkey_index;</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_16,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_16_QP_NUM_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp-&gt;qpn);</div><div class='del'>-</div><div class='del'>-	} else if (cur_state == IB_QPS_INIT &amp;&amp; new_state == IB_QPS_INIT) {</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_4,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S,</div><div class='del'>-			       to_hr_qp_type(hr_qp-&gt;ibqp.qp_type));</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_4,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0);</div><div class='del'>-		if (attr_mask &amp; IB_QP_ACCESS_FLAGS) {</div><div class='del'>-			roce_set_bit(context-&gt;qpc_bytes_4,</div><div class='del'>-				     QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,</div><div class='del'>-				     !!(attr-&gt;qp_access_flags &amp;</div><div class='del'>-				     IB_ACCESS_REMOTE_READ));</div><div class='del'>-			roce_set_bit(context-&gt;qpc_bytes_4,</div><div class='del'>-				     QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,</div><div class='del'>-				     !!(attr-&gt;qp_access_flags &amp;</div><div class='del'>-				     IB_ACCESS_REMOTE_WRITE));</div><div class='del'>-		} else {</div><div class='del'>-			roce_set_bit(context-&gt;qpc_bytes_4,</div><div class='del'>-				     QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,</div><div class='del'>-				     !!(hr_qp-&gt;access_flags &amp;</div><div class='del'>-				     IB_ACCESS_REMOTE_READ));</div><div class='del'>-			roce_set_bit(context-&gt;qpc_bytes_4,</div><div class='del'>-				     QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,</div><div class='del'>-				     !!(hr_qp-&gt;access_flags &amp;</div><div class='del'>-				     IB_ACCESS_REMOTE_WRITE));</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_4,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_4,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S,</div><div class='del'>-			       ilog2((unsigned int)hr_qp-&gt;sq.wqe_cnt));</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_4,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S,</div><div class='del'>-			       ilog2((unsigned int)hr_qp-&gt;rq.wqe_cnt));</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_4,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_4_PD_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_4_PD_S,</div><div class='del'>-			       to_hr_pd(ibqp-&gt;pd)-&gt;pdn);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_8,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S,</div><div class='del'>-			       to_hr_cq(ibqp-&gt;send_cq)-&gt;cqn);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_8,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S,</div><div class='del'>-			       to_hr_cq(ibqp-&gt;recv_cq)-&gt;cqn);</div><div class='del'>-</div><div class='del'>-		if (ibqp-&gt;srq)</div><div class='del'>-			roce_set_field(context-&gt;qpc_bytes_12,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S,</div><div class='del'>-				       to_hr_srq(ibqp-&gt;srq)-&gt;srqn);</div><div class='del'>-		if (attr_mask &amp; IB_QP_PKEY_INDEX)</div><div class='del'>-			roce_set_field(context-&gt;qpc_bytes_12,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,</div><div class='del'>-				       attr-&gt;pkey_index);</div><div class='del'>-		else</div><div class='del'>-			roce_set_field(context-&gt;qpc_bytes_12,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,</div><div class='del'>-				       hr_qp-&gt;pkey_index);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_16,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_16_QP_NUM_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp-&gt;qpn);</div><div class='del'>-	} else if (cur_state == IB_QPS_INIT &amp;&amp; new_state == IB_QPS_RTR) {</div><div class='del'>-		if ((attr_mask &amp; IB_QP_ALT_PATH) ||</div><div class='del'>-		    (attr_mask &amp; IB_QP_ACCESS_FLAGS) ||</div><div class='del'>-		    (attr_mask &amp; IB_QP_PKEY_INDEX) ||</div><div class='del'>-		    (attr_mask &amp; IB_QP_QKEY)) {</div><div class='del'>-			dev_err(dev, "INIT2RTR attr_mask error\n");</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		dmac = (u8 *)attr-&gt;ah_attr.roce.dmac;</div><div class='del'>-</div><div class='del'>-		context-&gt;sq_rq_bt_l = cpu_to_le32((u32)(dma_handle));</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_24,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S,</div><div class='del'>-			       ((u32)(dma_handle &gt;&gt; 32)));</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_24,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S,</div><div class='del'>-			     1);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_24,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S,</div><div class='del'>-			       attr-&gt;min_rnr_timer);</div><div class='del'>-		context-&gt;irrl_ba_l = cpu_to_le32((u32)(dma_handle_2));</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_32,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S,</div><div class='del'>-			       ((u32)(dma_handle_2 &gt;&gt; 32)) &amp;</div><div class='del'>-				QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_32,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S, 0);</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_32,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S,</div><div class='del'>-			     1);</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_32,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S,</div><div class='del'>-			     le32_to_cpu(hr_qp-&gt;sq_signal_bits));</div><div class='del'>-</div><div class='del'>-		port = (attr_mask &amp; IB_QP_PORT) ? (attr-&gt;port_num - 1) :</div><div class='del'>-			hr_qp-&gt;port;</div><div class='del'>-		smac = (u8 *)hr_dev-&gt;dev_addr[port];</div><div class='del'>-		/* when dmac equals smac or loop_idc is 1, it should loopback */</div><div class='del'>-		if (ether_addr_equal_unaligned(dmac, smac) ||</div><div class='del'>-		    hr_dev-&gt;loop_idc == 0x1)</div><div class='del'>-			roce_set_bit(context-&gt;qpc_bytes_32,</div><div class='del'>-			      QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S, 1);</div><div class='del'>-</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_32,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S,</div><div class='del'>-			     rdma_ah_get_ah_flags(&amp;attr-&gt;ah_attr));</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_32,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S,</div><div class='del'>-			       ilog2((unsigned int)attr-&gt;max_dest_rd_atomic));</div><div class='del'>-</div><div class='del'>-		if (attr_mask &amp; IB_QP_DEST_QPN)</div><div class='del'>-			roce_set_field(context-&gt;qpc_bytes_36,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_36_DEST_QP_M,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_36_DEST_QP_S,</div><div class='del'>-				       attr-&gt;dest_qp_num);</div><div class='del'>-</div><div class='del'>-		/* Configure GID index */</div><div class='del'>-		port_num = rdma_ah_get_port_num(&amp;attr-&gt;ah_attr);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_36,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S,</div><div class='del'>-				hns_get_gid_index(hr_dev,</div><div class='del'>-						  port_num - 1,</div><div class='del'>-						  grh-&gt;sgid_index));</div><div class='del'>-</div><div class='del'>-		memcpy(&amp;(context-&gt;dmac_l), dmac, 4);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_44,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_44_DMAC_H_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_44_DMAC_H_S,</div><div class='del'>-			       *((u16 *)(&amp;dmac[4])));</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_44,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S,</div><div class='del'>-			       rdma_ah_get_static_rate(&amp;attr-&gt;ah_attr));</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_44,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_44_HOPLMT_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_44_HOPLMT_S,</div><div class='del'>-			       grh-&gt;hop_limit);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_48,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S,</div><div class='del'>-			       grh-&gt;flow_label);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_48,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_48_TCLASS_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_48_TCLASS_S,</div><div class='del'>-			       grh-&gt;traffic_class);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_48,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_48_MTU_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_48_MTU_S, attr-&gt;path_mtu);</div><div class='del'>-</div><div class='del'>-		memcpy(context-&gt;dgid, grh-&gt;dgid.raw,</div><div class='del'>-		       sizeof(grh-&gt;dgid.raw));</div><div class='del'>-</div><div class='del'>-		dev_dbg(dev, "dmac:%x :%lx\n", context-&gt;dmac_l,</div><div class='del'>-			roce_get_field(context-&gt;qpc_bytes_44,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_44_DMAC_H_M,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_44_DMAC_H_S));</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_68,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S,</div><div class='del'>-			       hr_qp-&gt;rq.head);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_68,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S, 0);</div><div class='del'>-</div><div class='del'>-		rq_pa_start = (u32)hr_qp-&gt;rq.offset / PAGE_SIZE;</div><div class='del'>-		context-&gt;cur_rq_wqe_ba_l =</div><div class='del'>-				cpu_to_le32((u32)(mtts[rq_pa_start]));</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_76,</div><div class='del'>-			QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M,</div><div class='del'>-			QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S,</div><div class='del'>-			mtts[rq_pa_start] &gt;&gt; 32);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_76,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S, 0);</div><div class='del'>-</div><div class='del'>-		context-&gt;rx_rnr_time = 0;</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_84,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S,</div><div class='del'>-			       attr-&gt;rq_psn - 1);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_84,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S, 0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_88,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S,</div><div class='del'>-			       attr-&gt;rq_psn);</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_88,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S, 0);</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_88,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S, 0);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_88,</div><div class='del'>-			QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M,</div><div class='del'>-			QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S,</div><div class='del'>-			0);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_88,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S,</div><div class='del'>-			       0);</div><div class='del'>-</div><div class='del'>-		context-&gt;dma_length = 0;</div><div class='del'>-		context-&gt;r_key = 0;</div><div class='del'>-		context-&gt;va_l = 0;</div><div class='del'>-		context-&gt;va_h = 0;</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_108,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S, 0);</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_108,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S, 0);</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_108,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S, 0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_112,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S, 0);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_112,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S, 0);</div><div class='del'>-</div><div class='del'>-		/* For chip resp ack */</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_156,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S,</div><div class='del'>-			       hr_qp-&gt;phy_port);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_156,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_156_SL_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_156_SL_S,</div><div class='del'>-			       rdma_ah_get_sl(&amp;attr-&gt;ah_attr));</div><div class='del'>-		hr_qp-&gt;sl = rdma_ah_get_sl(&amp;attr-&gt;ah_attr);</div><div class='del'>-	} else if (cur_state == IB_QPS_RTR &amp;&amp;</div><div class='del'>-		new_state == IB_QPS_RTS) {</div><div class='del'>-		/* If exist optional param, return error */</div><div class='del'>-		if ((attr_mask &amp; IB_QP_ALT_PATH) ||</div><div class='del'>-		    (attr_mask &amp; IB_QP_ACCESS_FLAGS) ||</div><div class='del'>-		    (attr_mask &amp; IB_QP_QKEY) ||</div><div class='del'>-		    (attr_mask &amp; IB_QP_PATH_MIG_STATE) ||</div><div class='del'>-		    (attr_mask &amp; IB_QP_CUR_STATE) ||</div><div class='del'>-		    (attr_mask &amp; IB_QP_MIN_RNR_TIMER)) {</div><div class='del'>-			dev_err(dev, "RTR2RTS attr_mask error\n");</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		context-&gt;rx_cur_sq_wqe_ba_l = cpu_to_le32((u32)(mtts[0]));</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_120,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S,</div><div class='del'>-			       (mtts[0]) &gt;&gt; 32);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_124,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S, 0);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_124,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S, 0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_128,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S,</div><div class='del'>-			       attr-&gt;sq_psn);</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_128,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S, 0);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_128,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S,</div><div class='del'>-			     0);</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_128,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S, 0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_132,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S, 0);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_132,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S, 0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_136,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S,</div><div class='del'>-			       attr-&gt;sq_psn);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_136,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S,</div><div class='del'>-			       attr-&gt;sq_psn);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_140,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S,</div><div class='del'>-			       (attr-&gt;sq_psn &gt;&gt; SQ_PSN_SHIFT));</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_140,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S, 0);</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_140,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S, 0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_148,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S, 0);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_148,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S,</div><div class='del'>-			       attr-&gt;retry_cnt);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_148,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S,</div><div class='del'>-			       attr-&gt;rnr_retry);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_148,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_148_LSN_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_148_LSN_S, 0x100);</div><div class='del'>-</div><div class='del'>-		context-&gt;rnr_retry = 0;</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_156,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S,</div><div class='del'>-			       attr-&gt;retry_cnt);</div><div class='del'>-		if (attr-&gt;timeout &lt; 0x12) {</div><div class='del'>-			dev_info(dev, "ack timeout value(0x%x) must bigger than 0x12.\n",</div><div class='del'>-				 attr-&gt;timeout);</div><div class='del'>-			roce_set_field(context-&gt;qpc_bytes_156,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S,</div><div class='del'>-				       0x12);</div><div class='del'>-		} else {</div><div class='del'>-			roce_set_field(context-&gt;qpc_bytes_156,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S,</div><div class='del'>-				       attr-&gt;timeout);</div><div class='del'>-		}</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_156,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S,</div><div class='del'>-			       attr-&gt;rnr_retry);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_156,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S,</div><div class='del'>-			       hr_qp-&gt;phy_port);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_156,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_156_SL_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_156_SL_S,</div><div class='del'>-			       rdma_ah_get_sl(&amp;attr-&gt;ah_attr));</div><div class='del'>-		hr_qp-&gt;sl = rdma_ah_get_sl(&amp;attr-&gt;ah_attr);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_156,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S,</div><div class='del'>-			       ilog2((unsigned int)attr-&gt;max_rd_atomic));</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_156,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S, 0);</div><div class='del'>-		context-&gt;pkt_use_len = 0;</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_164,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S, attr-&gt;sq_psn);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_164,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S, 0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_168,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S,</div><div class='del'>-			       attr-&gt;sq_psn);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_168,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S, 0);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_168,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S, 0);</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_168,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S, 0);</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_168,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S, 0);</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_168,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S, 0);</div><div class='del'>-		context-&gt;sge_use_len = 0;</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_176,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S, 0);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_176,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S,</div><div class='del'>-			       0);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_180,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S, 0);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_180,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S, 0);</div><div class='del'>-</div><div class='del'>-		context-&gt;tx_cur_sq_wqe_ba_l = cpu_to_le32((u32)(mtts[0]));</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_188,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S,</div><div class='del'>-			       (mtts[0]) &gt;&gt; 32);</div><div class='del'>-		roce_set_bit(context-&gt;qpc_bytes_188,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S, 0);</div><div class='del'>-		roce_set_field(context-&gt;qpc_bytes_188,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S,</div><div class='del'>-			       0);</div><div class='del'>-	} else if (!((cur_state == IB_QPS_INIT &amp;&amp; new_state == IB_QPS_RESET) ||</div><div class='del'>-		   (cur_state == IB_QPS_INIT &amp;&amp; new_state == IB_QPS_ERR) ||</div><div class='del'>-		   (cur_state == IB_QPS_RTR &amp;&amp; new_state == IB_QPS_RESET) ||</div><div class='del'>-		   (cur_state == IB_QPS_RTR &amp;&amp; new_state == IB_QPS_ERR) ||</div><div class='del'>-		   (cur_state == IB_QPS_RTS &amp;&amp; new_state == IB_QPS_RESET) ||</div><div class='del'>-		   (cur_state == IB_QPS_RTS &amp;&amp; new_state == IB_QPS_ERR) ||</div><div class='del'>-		   (cur_state == IB_QPS_ERR &amp;&amp; new_state == IB_QPS_RESET) ||</div><div class='del'>-		   (cur_state == IB_QPS_ERR &amp;&amp; new_state == IB_QPS_ERR))) {</div><div class='del'>-		dev_err(dev, "not support this status migration\n");</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Every status migrate must change state */</div><div class='del'>-	roce_set_field(context-&gt;qpc_bytes_144,</div><div class='del'>-		       QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,</div><div class='del'>-		       QP_CONTEXT_QPC_BYTES_144_QP_STATE_S, new_state);</div><div class='del'>-</div><div class='del'>-	/* SW pass context to HW */</div><div class='del'>-	ret = hns_roce_v1_qp_modify(hr_dev, &amp;hr_qp-&gt;mtt,</div><div class='del'>-				    to_hns_roce_state(cur_state),</div><div class='del'>-				    to_hns_roce_state(new_state), context,</div><div class='del'>-				    hr_qp);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "hns_roce_qp_modify failed\n");</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Use rst2init to instead of init2init with drv,</div><div class='del'>-	 * need to hw to flash RQ HEAD by DB again</div><div class='del'>-	 */</div><div class='del'>-	if (cur_state == IB_QPS_INIT &amp;&amp; new_state == IB_QPS_INIT) {</div><div class='del'>-		/* Memory barrier */</div><div class='del'>-		wmb();</div><div class='del'>-</div><div class='del'>-		roce_set_field(doorbell[0], RQ_DOORBELL_U32_4_RQ_HEAD_M,</div><div class='del'>-			       RQ_DOORBELL_U32_4_RQ_HEAD_S, hr_qp-&gt;rq.head);</div><div class='del'>-		roce_set_field(doorbell[1], RQ_DOORBELL_U32_8_QPN_M,</div><div class='del'>-			       RQ_DOORBELL_U32_8_QPN_S, hr_qp-&gt;qpn);</div><div class='del'>-		roce_set_field(doorbell[1], RQ_DOORBELL_U32_8_CMD_M,</div><div class='del'>-			       RQ_DOORBELL_U32_8_CMD_S, 1);</div><div class='del'>-		roce_set_bit(doorbell[1], RQ_DOORBELL_U32_8_HW_SYNC_S, 1);</div><div class='del'>-</div><div class='del'>-		if (ibqp-&gt;uobject) {</div><div class='del'>-			hr_qp-&gt;rq.db_reg_l = hr_dev-&gt;reg_base +</div><div class='del'>-				     hr_dev-&gt;odb_offset +</div><div class='del'>-				     DB_REG_OFFSET * hr_dev-&gt;priv_uar.index;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		hns_roce_write64_k(doorbell, hr_qp-&gt;rq.db_reg_l);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	hr_qp-&gt;state = new_state;</div><div class='del'>-</div><div class='del'>-	if (attr_mask &amp; IB_QP_MAX_DEST_RD_ATOMIC)</div><div class='del'>-		hr_qp-&gt;resp_depth = attr-&gt;max_dest_rd_atomic;</div><div class='del'>-	if (attr_mask &amp; IB_QP_PORT) {</div><div class='del'>-		hr_qp-&gt;port = attr-&gt;port_num - 1;</div><div class='del'>-		hr_qp-&gt;phy_port = hr_dev-&gt;iboe.phy_port[hr_qp-&gt;port];</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (new_state == IB_QPS_RESET &amp;&amp; !ibqp-&gt;uobject) {</div><div class='del'>-		hns_roce_v1_cq_clean(to_hr_cq(ibqp-&gt;recv_cq), hr_qp-&gt;qpn,</div><div class='del'>-				     ibqp-&gt;srq ? to_hr_srq(ibqp-&gt;srq) : NULL);</div><div class='del'>-		if (ibqp-&gt;send_cq != ibqp-&gt;recv_cq)</div><div class='del'>-			hns_roce_v1_cq_clean(to_hr_cq(ibqp-&gt;send_cq),</div><div class='del'>-					     hr_qp-&gt;qpn, NULL);</div><div class='del'>-</div><div class='del'>-		hr_qp-&gt;rq.head = 0;</div><div class='del'>-		hr_qp-&gt;rq.tail = 0;</div><div class='del'>-		hr_qp-&gt;sq.head = 0;</div><div class='del'>-		hr_qp-&gt;sq.tail = 0;</div><div class='del'>-		hr_qp-&gt;sq_next_wqe = 0;</div><div class='del'>-	}</div><div class='del'>-out:</div><div class='del'>-	kfree(context);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_modify_qp(struct ib_qp *ibqp,</div><div class='del'>-				 const struct ib_qp_attr *attr, int attr_mask,</div><div class='del'>-				 enum ib_qp_state cur_state,</div><div class='del'>-				 enum ib_qp_state new_state)</div><div class='del'>-{</div><div class='del'>-</div><div class='del'>-	if (ibqp-&gt;qp_type == IB_QPT_GSI || ibqp-&gt;qp_type == IB_QPT_SMI)</div><div class='del'>-		return hns_roce_v1_m_sqp(ibqp, attr, attr_mask, cur_state,</div><div class='del'>-					 new_state);</div><div class='del'>-	else</div><div class='del'>-		return hns_roce_v1_m_qp(ibqp, attr, attr_mask, cur_state,</div><div class='del'>-					new_state);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static enum ib_qp_state to_ib_qp_state(enum hns_roce_qp_state state)</div><div class='del'>-{</div><div class='del'>-	switch (state) {</div><div class='del'>-	case HNS_ROCE_QP_STATE_RST:</div><div class='del'>-		return IB_QPS_RESET;</div><div class='del'>-	case HNS_ROCE_QP_STATE_INIT:</div><div class='del'>-		return IB_QPS_INIT;</div><div class='del'>-	case HNS_ROCE_QP_STATE_RTR:</div><div class='del'>-		return IB_QPS_RTR;</div><div class='del'>-	case HNS_ROCE_QP_STATE_RTS:</div><div class='del'>-		return IB_QPS_RTS;</div><div class='del'>-	case HNS_ROCE_QP_STATE_SQD:</div><div class='del'>-		return IB_QPS_SQD;</div><div class='del'>-	case HNS_ROCE_QP_STATE_ERR:</div><div class='del'>-		return IB_QPS_ERR;</div><div class='del'>-	default:</div><div class='del'>-		return IB_QPS_ERR;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_query_qpc(struct hns_roce_dev *hr_dev,</div><div class='del'>-				 struct hns_roce_qp *hr_qp,</div><div class='del'>-				 struct hns_roce_qp_context *hr_context)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_cmd_mailbox *mailbox;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='del'>-	if (IS_ERR(mailbox))</div><div class='del'>-		return PTR_ERR(mailbox);</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox-&gt;dma, hr_qp-&gt;qpn, 0,</div><div class='del'>-				HNS_ROCE_CMD_QUERY_QP,</div><div class='del'>-				HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='del'>-	if (!ret)</div><div class='del'>-		memcpy(hr_context, mailbox-&gt;buf, sizeof(*hr_context));</div><div class='del'>-	else</div><div class='del'>-		dev_err(&amp;hr_dev-&gt;pdev-&gt;dev, "QUERY QP cmd process error\n");</div><div class='del'>-</div><div class='del'>-	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_q_sqp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='del'>-			     int qp_attr_mask,</div><div class='del'>-			     struct ib_qp_init_attr *qp_init_attr)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='del'>-	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='del'>-	struct hns_roce_sqp_context context;</div><div class='del'>-	u32 addr;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;hr_qp-&gt;mutex);</div><div class='del'>-</div><div class='del'>-	if (hr_qp-&gt;state == IB_QPS_RESET) {</div><div class='del'>-		qp_attr-&gt;qp_state = IB_QPS_RESET;</div><div class='del'>-		goto done;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	addr = ROCEE_QP1C_CFG0_0_REG +</div><div class='del'>-		hr_qp-&gt;port * sizeof(struct hns_roce_sqp_context);</div><div class='del'>-	context.qp1c_bytes_4 = cpu_to_le32(roce_read(hr_dev, addr));</div><div class='del'>-	context.sq_rq_bt_l = cpu_to_le32(roce_read(hr_dev, addr + 1));</div><div class='del'>-	context.qp1c_bytes_12 = cpu_to_le32(roce_read(hr_dev, addr + 2));</div><div class='del'>-	context.qp1c_bytes_16 = cpu_to_le32(roce_read(hr_dev, addr + 3));</div><div class='del'>-	context.qp1c_bytes_20 = cpu_to_le32(roce_read(hr_dev, addr + 4));</div><div class='del'>-	context.cur_rq_wqe_ba_l = cpu_to_le32(roce_read(hr_dev, addr + 5));</div><div class='del'>-	context.qp1c_bytes_28 = cpu_to_le32(roce_read(hr_dev, addr + 6));</div><div class='del'>-	context.qp1c_bytes_32 = cpu_to_le32(roce_read(hr_dev, addr + 7));</div><div class='del'>-	context.cur_sq_wqe_ba_l = cpu_to_le32(roce_read(hr_dev, addr + 8));</div><div class='del'>-	context.qp1c_bytes_40 = cpu_to_le32(roce_read(hr_dev, addr + 9));</div><div class='del'>-</div><div class='del'>-	hr_qp-&gt;state = roce_get_field(context.qp1c_bytes_4,</div><div class='del'>-				      QP1C_BYTES_4_QP_STATE_M,</div><div class='del'>-				      QP1C_BYTES_4_QP_STATE_S);</div><div class='del'>-	qp_attr-&gt;qp_state	= hr_qp-&gt;state;</div><div class='del'>-	qp_attr-&gt;path_mtu	= IB_MTU_256;</div><div class='del'>-	qp_attr-&gt;path_mig_state	= IB_MIG_ARMED;</div><div class='del'>-	qp_attr-&gt;qkey		= QKEY_VAL;</div><div class='del'>-	qp_attr-&gt;ah_attr.type   = RDMA_AH_ATTR_TYPE_ROCE;</div><div class='del'>-	qp_attr-&gt;rq_psn		= 0;</div><div class='del'>-	qp_attr-&gt;sq_psn		= 0;</div><div class='del'>-	qp_attr-&gt;dest_qp_num	= 1;</div><div class='del'>-	qp_attr-&gt;qp_access_flags = 6;</div><div class='del'>-</div><div class='del'>-	qp_attr-&gt;pkey_index = roce_get_field(context.qp1c_bytes_20,</div><div class='del'>-					     QP1C_BYTES_20_PKEY_IDX_M,</div><div class='del'>-					     QP1C_BYTES_20_PKEY_IDX_S);</div><div class='del'>-	qp_attr-&gt;port_num = hr_qp-&gt;port + 1;</div><div class='del'>-	qp_attr-&gt;sq_draining = 0;</div><div class='del'>-	qp_attr-&gt;max_rd_atomic = 0;</div><div class='del'>-	qp_attr-&gt;max_dest_rd_atomic = 0;</div><div class='del'>-	qp_attr-&gt;min_rnr_timer = 0;</div><div class='del'>-	qp_attr-&gt;timeout = 0;</div><div class='del'>-	qp_attr-&gt;retry_cnt = 0;</div><div class='del'>-	qp_attr-&gt;rnr_retry = 0;</div><div class='del'>-	qp_attr-&gt;alt_timeout = 0;</div><div class='del'>-</div><div class='del'>-done:</div><div class='del'>-	qp_attr-&gt;cur_qp_state = qp_attr-&gt;qp_state;</div><div class='del'>-	qp_attr-&gt;cap.max_recv_wr = hr_qp-&gt;rq.wqe_cnt;</div><div class='del'>-	qp_attr-&gt;cap.max_recv_sge = hr_qp-&gt;rq.max_gs;</div><div class='del'>-	qp_attr-&gt;cap.max_send_wr = hr_qp-&gt;sq.wqe_cnt;</div><div class='del'>-	qp_attr-&gt;cap.max_send_sge = hr_qp-&gt;sq.max_gs;</div><div class='del'>-	qp_attr-&gt;cap.max_inline_data = 0;</div><div class='del'>-	qp_init_attr-&gt;cap = qp_attr-&gt;cap;</div><div class='del'>-	qp_init_attr-&gt;create_flags = 0;</div><div class='del'>-</div><div class='del'>-	mutex_unlock(&amp;hr_qp-&gt;mutex);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_q_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='del'>-			    int qp_attr_mask,</div><div class='del'>-			    struct ib_qp_init_attr *qp_init_attr)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='del'>-	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_qp_context *context;</div><div class='del'>-	int tmp_qp_state = 0;</div><div class='del'>-	int ret = 0;</div><div class='del'>-	int state;</div><div class='del'>-</div><div class='del'>-	context = kzalloc(sizeof(*context), GFP_KERNEL);</div><div class='del'>-	if (!context)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	memset(qp_attr, 0, sizeof(*qp_attr));</div><div class='del'>-	memset(qp_init_attr, 0, sizeof(*qp_init_attr));</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;hr_qp-&gt;mutex);</div><div class='del'>-</div><div class='del'>-	if (hr_qp-&gt;state == IB_QPS_RESET) {</div><div class='del'>-		qp_attr-&gt;qp_state = IB_QPS_RESET;</div><div class='del'>-		goto done;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_v1_query_qpc(hr_dev, hr_qp, context);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "query qpc error\n");</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	state = roce_get_field(context-&gt;qpc_bytes_144,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,</div><div class='del'>-			       QP_CONTEXT_QPC_BYTES_144_QP_STATE_S);</div><div class='del'>-	tmp_qp_state = (int)to_ib_qp_state((enum hns_roce_qp_state)state);</div><div class='del'>-	if (tmp_qp_state == -1) {</div><div class='del'>-		dev_err(dev, "to_ib_qp_state error\n");</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-	hr_qp-&gt;state = (u8)tmp_qp_state;</div><div class='del'>-	qp_attr-&gt;qp_state = (enum ib_qp_state)hr_qp-&gt;state;</div><div class='del'>-	qp_attr-&gt;path_mtu = (enum ib_mtu)roce_get_field(context-&gt;qpc_bytes_48,</div><div class='del'>-					       QP_CONTEXT_QPC_BYTES_48_MTU_M,</div><div class='del'>-					       QP_CONTEXT_QPC_BYTES_48_MTU_S);</div><div class='del'>-	qp_attr-&gt;path_mig_state = IB_MIG_ARMED;</div><div class='del'>-	qp_attr-&gt;ah_attr.type   = RDMA_AH_ATTR_TYPE_ROCE;</div><div class='del'>-	if (hr_qp-&gt;ibqp.qp_type == IB_QPT_UD)</div><div class='del'>-		qp_attr-&gt;qkey = QKEY_VAL;</div><div class='del'>-</div><div class='del'>-	qp_attr-&gt;rq_psn = roce_get_field(context-&gt;qpc_bytes_88,</div><div class='del'>-					 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M,</div><div class='del'>-					 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S);</div><div class='del'>-	qp_attr-&gt;sq_psn = (u32)roce_get_field(context-&gt;qpc_bytes_164,</div><div class='del'>-					     QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M,</div><div class='del'>-					     QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S);</div><div class='del'>-	qp_attr-&gt;dest_qp_num = (u8)roce_get_field(context-&gt;qpc_bytes_36,</div><div class='del'>-					QP_CONTEXT_QPC_BYTES_36_DEST_QP_M,</div><div class='del'>-					QP_CONTEXT_QPC_BYTES_36_DEST_QP_S);</div><div class='del'>-	qp_attr-&gt;qp_access_flags = ((roce_get_bit(context-&gt;qpc_bytes_4,</div><div class='del'>-			QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S)) &lt;&lt; 2) |</div><div class='del'>-				   ((roce_get_bit(context-&gt;qpc_bytes_4,</div><div class='del'>-			QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S)) &lt;&lt; 1) |</div><div class='del'>-				   ((roce_get_bit(context-&gt;qpc_bytes_4,</div><div class='del'>-			QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S)) &lt;&lt; 3);</div><div class='del'>-</div><div class='del'>-	if (hr_qp-&gt;ibqp.qp_type == IB_QPT_RC ||</div><div class='del'>-	    hr_qp-&gt;ibqp.qp_type == IB_QPT_UC) {</div><div class='del'>-		struct ib_global_route *grh =</div><div class='del'>-			rdma_ah_retrieve_grh(&amp;qp_attr-&gt;ah_attr);</div><div class='del'>-</div><div class='del'>-		rdma_ah_set_sl(&amp;qp_attr-&gt;ah_attr,</div><div class='del'>-			       roce_get_field(context-&gt;qpc_bytes_156,</div><div class='del'>-					      QP_CONTEXT_QPC_BYTES_156_SL_M,</div><div class='del'>-					      QP_CONTEXT_QPC_BYTES_156_SL_S));</div><div class='del'>-		rdma_ah_set_ah_flags(&amp;qp_attr-&gt;ah_attr, IB_AH_GRH);</div><div class='del'>-		grh-&gt;flow_label =</div><div class='del'>-			roce_get_field(context-&gt;qpc_bytes_48,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S);</div><div class='del'>-		grh-&gt;sgid_index =</div><div class='del'>-			roce_get_field(context-&gt;qpc_bytes_36,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S);</div><div class='del'>-		grh-&gt;hop_limit =</div><div class='del'>-			roce_get_field(context-&gt;qpc_bytes_44,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_44_HOPLMT_M,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_44_HOPLMT_S);</div><div class='del'>-		grh-&gt;traffic_class =</div><div class='del'>-			roce_get_field(context-&gt;qpc_bytes_48,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_48_TCLASS_M,</div><div class='del'>-				       QP_CONTEXT_QPC_BYTES_48_TCLASS_S);</div><div class='del'>-</div><div class='del'>-		memcpy(grh-&gt;dgid.raw, context-&gt;dgid,</div><div class='del'>-		       sizeof(grh-&gt;dgid.raw));</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	qp_attr-&gt;pkey_index = roce_get_field(context-&gt;qpc_bytes_12,</div><div class='del'>-			      QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,</div><div class='del'>-			      QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S);</div><div class='del'>-	qp_attr-&gt;port_num = hr_qp-&gt;port + 1;</div><div class='del'>-	qp_attr-&gt;sq_draining = 0;</div><div class='del'>-	qp_attr-&gt;max_rd_atomic = 1 &lt;&lt; roce_get_field(context-&gt;qpc_bytes_156,</div><div class='del'>-				 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M,</div><div class='del'>-				 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S);</div><div class='del'>-	qp_attr-&gt;max_dest_rd_atomic = 1 &lt;&lt; roce_get_field(context-&gt;qpc_bytes_32,</div><div class='del'>-				 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M,</div><div class='del'>-				 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S);</div><div class='del'>-	qp_attr-&gt;min_rnr_timer = (u8)(roce_get_field(context-&gt;qpc_bytes_24,</div><div class='del'>-			QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M,</div><div class='del'>-			QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S));</div><div class='del'>-	qp_attr-&gt;timeout = (u8)(roce_get_field(context-&gt;qpc_bytes_156,</div><div class='del'>-			    QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,</div><div class='del'>-			    QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S));</div><div class='del'>-	qp_attr-&gt;retry_cnt = roce_get_field(context-&gt;qpc_bytes_148,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M,</div><div class='del'>-			     QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S);</div><div class='del'>-	qp_attr-&gt;rnr_retry = (u8)context-&gt;rnr_retry;</div><div class='del'>-</div><div class='del'>-done:</div><div class='del'>-	qp_attr-&gt;cur_qp_state = qp_attr-&gt;qp_state;</div><div class='del'>-	qp_attr-&gt;cap.max_recv_wr = hr_qp-&gt;rq.wqe_cnt;</div><div class='del'>-	qp_attr-&gt;cap.max_recv_sge = hr_qp-&gt;rq.max_gs;</div><div class='del'>-</div><div class='del'>-	if (!ibqp-&gt;uobject) {</div><div class='del'>-		qp_attr-&gt;cap.max_send_wr = hr_qp-&gt;sq.wqe_cnt;</div><div class='del'>-		qp_attr-&gt;cap.max_send_sge = hr_qp-&gt;sq.max_gs;</div><div class='del'>-	} else {</div><div class='del'>-		qp_attr-&gt;cap.max_send_wr = 0;</div><div class='del'>-		qp_attr-&gt;cap.max_send_sge = 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	qp_init_attr-&gt;cap = qp_attr-&gt;cap;</div><div class='del'>-</div><div class='del'>-out:</div><div class='del'>-	mutex_unlock(&amp;hr_qp-&gt;mutex);</div><div class='del'>-	kfree(context);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='del'>-				int qp_attr_mask,</div><div class='del'>-				struct ib_qp_init_attr *qp_init_attr)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='del'>-</div><div class='del'>-	return hr_qp-&gt;doorbell_qpn &lt;= 1 ?</div><div class='del'>-		hns_roce_v1_q_sqp(ibqp, qp_attr, qp_attr_mask, qp_init_attr) :</div><div class='del'>-		hns_roce_v1_q_qp(ibqp, qp_attr, qp_attr_mask, qp_init_attr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='del'>-	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='del'>-	struct hns_roce_cq *send_cq, *recv_cq;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_v1_modify_qp(ibqp, NULL, 0, hr_qp-&gt;state, IB_QPS_RESET);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	send_cq = to_hr_cq(hr_qp-&gt;ibqp.send_cq);</div><div class='del'>-	recv_cq = to_hr_cq(hr_qp-&gt;ibqp.recv_cq);</div><div class='del'>-</div><div class='del'>-	hns_roce_lock_cqs(send_cq, recv_cq);</div><div class='del'>-	if (!udata) {</div><div class='del'>-		__hns_roce_v1_cq_clean(recv_cq, hr_qp-&gt;qpn, hr_qp-&gt;ibqp.srq ?</div><div class='del'>-				       to_hr_srq(hr_qp-&gt;ibqp.srq) : NULL);</div><div class='del'>-		if (send_cq != recv_cq)</div><div class='del'>-			__hns_roce_v1_cq_clean(send_cq, hr_qp-&gt;qpn, NULL);</div><div class='del'>-	}</div><div class='del'>-	hns_roce_unlock_cqs(send_cq, recv_cq);</div><div class='del'>-</div><div class='del'>-	hns_roce_qp_remove(hr_dev, hr_qp);</div><div class='del'>-	hns_roce_qp_free(hr_dev, hr_qp);</div><div class='del'>-</div><div class='del'>-	/* RC QP, release QPN */</div><div class='del'>-	if (hr_qp-&gt;ibqp.qp_type == IB_QPT_RC)</div><div class='del'>-		hns_roce_release_range_qp(hr_dev, hr_qp-&gt;qpn, 1);</div><div class='del'>-</div><div class='del'>-	hns_roce_mtt_cleanup(hr_dev, &amp;hr_qp-&gt;mtt);</div><div class='del'>-</div><div class='del'>-	if (udata)</div><div class='del'>-		ib_umem_release(hr_qp-&gt;umem);</div><div class='del'>-	else {</div><div class='del'>-		kfree(hr_qp-&gt;sq.wrid);</div><div class='del'>-		kfree(hr_qp-&gt;rq.wrid);</div><div class='del'>-</div><div class='del'>-		hns_roce_buf_free(hr_dev, hr_qp-&gt;buff_size, &amp;hr_qp-&gt;hr_buf);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (hr_qp-&gt;ibqp.qp_type == IB_QPT_RC)</div><div class='del'>-		kfree(hr_qp);</div><div class='del'>-	else</div><div class='del'>-		kfree(hr_to_hr_sqp(hr_qp));</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ibcq-&gt;device);</div><div class='del'>-	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	u32 cqe_cnt_ori;</div><div class='del'>-	u32 cqe_cnt_cur;</div><div class='del'>-	u32 cq_buf_size;</div><div class='del'>-	int wait_time = 0;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	hns_roce_free_cq(hr_dev, hr_cq);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Before freeing cq buffer, we need to ensure that the outstanding CQE</div><div class='del'>-	 * have been written by checking the CQE counter.</div><div class='del'>-	 */</div><div class='del'>-	cqe_cnt_ori = roce_read(hr_dev, ROCEE_SCAEP_WR_CQE_CNT);</div><div class='del'>-	while (1) {</div><div class='del'>-		if (roce_read(hr_dev, ROCEE_CAEP_CQE_WCMD_EMPTY) &amp;</div><div class='del'>-		    HNS_ROCE_CQE_WCMD_EMPTY_BIT)</div><div class='del'>-			break;</div><div class='del'>-</div><div class='del'>-		cqe_cnt_cur = roce_read(hr_dev, ROCEE_SCAEP_WR_CQE_CNT);</div><div class='del'>-		if ((cqe_cnt_cur - cqe_cnt_ori) &gt;= HNS_ROCE_MIN_CQE_CNT)</div><div class='del'>-			break;</div><div class='del'>-</div><div class='del'>-		msleep(HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS);</div><div class='del'>-		if (wait_time &gt; HNS_ROCE_MAX_FREE_CQ_WAIT_CNT) {</div><div class='del'>-			dev_warn(dev, "Destroy cq 0x%lx timeout!\n",</div><div class='del'>-				hr_cq-&gt;cqn);</div><div class='del'>-			ret = -ETIMEDOUT;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		wait_time++;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	hns_roce_mtt_cleanup(hr_dev, &amp;hr_cq-&gt;hr_buf.hr_mtt);</div><div class='del'>-</div><div class='del'>-	if (ibcq-&gt;uobject)</div><div class='del'>-		ib_umem_release(hr_cq-&gt;umem);</div><div class='del'>-	else {</div><div class='del'>-		/* Free the buff of stored cq */</div><div class='del'>-		cq_buf_size = (ibcq-&gt;cqe + 1) * hr_dev-&gt;caps.cq_entry_sz;</div><div class='del'>-		hns_roce_buf_free(hr_dev, cq_buf_size, &amp;hr_cq-&gt;hr_buf.hr_buf);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	kfree(hr_cq);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_eq_cons_index_v1(struct hns_roce_eq *eq, int req_not)</div><div class='del'>-{</div><div class='del'>-	roce_raw_write((eq-&gt;cons_index &amp; HNS_ROCE_V1_CONS_IDX_M) |</div><div class='del'>-		      (req_not &lt;&lt; eq-&gt;log_entries), eq-&gt;doorbell);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_wq_catas_err_handle(struct hns_roce_dev *hr_dev,</div><div class='del'>-					    struct hns_roce_aeqe *aeqe, int qpn)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-</div><div class='del'>-	dev_warn(dev, "Local Work Queue Catastrophic Error.\n");</div><div class='del'>-	switch (roce_get_field(aeqe-&gt;asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,</div><div class='del'>-			       HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {</div><div class='del'>-	case HNS_ROCE_LWQCE_QPC_ERROR:</div><div class='del'>-		dev_warn(dev, "QP %d, QPC error.\n", qpn);</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_LWQCE_MTU_ERROR:</div><div class='del'>-		dev_warn(dev, "QP %d, MTU error.\n", qpn);</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR:</div><div class='del'>-		dev_warn(dev, "QP %d, WQE BA addr error.\n", qpn);</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_LWQCE_WQE_ADDR_ERROR:</div><div class='del'>-		dev_warn(dev, "QP %d, WQE addr error.\n", qpn);</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR:</div><div class='del'>-		dev_warn(dev, "QP %d, WQE shift error\n", qpn);</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_LWQCE_SL_ERROR:</div><div class='del'>-		dev_warn(dev, "QP %d, SL error.\n", qpn);</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_LWQCE_PORT_ERROR:</div><div class='del'>-		dev_warn(dev, "QP %d, port error.\n", qpn);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_local_wq_access_err_handle(struct hns_roce_dev *hr_dev,</div><div class='del'>-						   struct hns_roce_aeqe *aeqe,</div><div class='del'>-						   int qpn)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-</div><div class='del'>-	dev_warn(dev, "Local Access Violation Work Queue Error.\n");</div><div class='del'>-	switch (roce_get_field(aeqe-&gt;asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,</div><div class='del'>-			       HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {</div><div class='del'>-	case HNS_ROCE_LAVWQE_R_KEY_VIOLATION:</div><div class='del'>-		dev_warn(dev, "QP %d, R_key violation.\n", qpn);</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_LAVWQE_LENGTH_ERROR:</div><div class='del'>-		dev_warn(dev, "QP %d, length error.\n", qpn);</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_LAVWQE_VA_ERROR:</div><div class='del'>-		dev_warn(dev, "QP %d, VA error.\n", qpn);</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_LAVWQE_PD_ERROR:</div><div class='del'>-		dev_err(dev, "QP %d, PD error.\n", qpn);</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_LAVWQE_RW_ACC_ERROR:</div><div class='del'>-		dev_warn(dev, "QP %d, rw acc error.\n", qpn);</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_LAVWQE_KEY_STATE_ERROR:</div><div class='del'>-		dev_warn(dev, "QP %d, key state error.\n", qpn);</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_LAVWQE_MR_OPERATION_ERROR:</div><div class='del'>-		dev_warn(dev, "QP %d, MR operation error.\n", qpn);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_qp_err_handle(struct hns_roce_dev *hr_dev,</div><div class='del'>-				      struct hns_roce_aeqe *aeqe,</div><div class='del'>-				      int event_type)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	int phy_port;</div><div class='del'>-	int qpn;</div><div class='del'>-</div><div class='del'>-	qpn = roce_get_field(aeqe-&gt;event.qp_event.qp,</div><div class='del'>-			     HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M,</div><div class='del'>-			     HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S);</div><div class='del'>-	phy_port = roce_get_field(aeqe-&gt;event.qp_event.qp,</div><div class='del'>-				  HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M,</div><div class='del'>-				  HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S);</div><div class='del'>-	if (qpn &lt;= 1)</div><div class='del'>-		qpn = HNS_ROCE_MAX_PORTS * qpn + phy_port;</div><div class='del'>-</div><div class='del'>-	switch (event_type) {</div><div class='del'>-	case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:</div><div class='del'>-		dev_warn(dev, "Invalid Req Local Work Queue Error.\n"</div><div class='del'>-			 "QP %d, phy_port %d.\n", qpn, phy_port);</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:</div><div class='del'>-		hns_roce_v1_wq_catas_err_handle(hr_dev, aeqe, qpn);</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:</div><div class='del'>-		hns_roce_v1_local_wq_access_err_handle(hr_dev, aeqe, qpn);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	hns_roce_qp_event(hr_dev, qpn, event_type);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_cq_err_handle(struct hns_roce_dev *hr_dev,</div><div class='del'>-				      struct hns_roce_aeqe *aeqe,</div><div class='del'>-				      int event_type)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	u32 cqn;</div><div class='del'>-</div><div class='del'>-	cqn = roce_get_field(aeqe-&gt;event.cq_event.cq,</div><div class='del'>-			  HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M,</div><div class='del'>-			  HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S);</div><div class='del'>-</div><div class='del'>-	switch (event_type) {</div><div class='del'>-	case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:</div><div class='del'>-		dev_warn(dev, "CQ 0x%x access err.\n", cqn);</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:</div><div class='del'>-		dev_warn(dev, "CQ 0x%x overflow\n", cqn);</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID:</div><div class='del'>-		dev_warn(dev, "CQ 0x%x ID invalid.\n", cqn);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	hns_roce_cq_event(hr_dev, cqn, event_type);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_db_overflow_handle(struct hns_roce_dev *hr_dev,</div><div class='del'>-					   struct hns_roce_aeqe *aeqe)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-</div><div class='del'>-	switch (roce_get_field(aeqe-&gt;asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,</div><div class='del'>-			       HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {</div><div class='del'>-	case HNS_ROCE_DB_SUBTYPE_SDB_OVF:</div><div class='del'>-		dev_warn(dev, "SDB overflow.\n");</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF:</div><div class='del'>-		dev_warn(dev, "SDB almost overflow.\n");</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP:</div><div class='del'>-		dev_warn(dev, "SDB almost empty.\n");</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_DB_SUBTYPE_ODB_OVF:</div><div class='del'>-		dev_warn(dev, "ODB overflow.\n");</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF:</div><div class='del'>-		dev_warn(dev, "ODB almost overflow.\n");</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP:</div><div class='del'>-		dev_warn(dev, "SDB almost empty.\n");</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct hns_roce_aeqe *get_aeqe_v1(struct hns_roce_eq *eq, u32 entry)</div><div class='del'>-{</div><div class='del'>-	unsigned long off = (entry &amp; (eq-&gt;entries - 1)) *</div><div class='del'>-			     HNS_ROCE_AEQ_ENTRY_SIZE;</div><div class='del'>-</div><div class='del'>-	return (struct hns_roce_aeqe *)((u8 *)</div><div class='del'>-		(eq-&gt;buf_list[off / HNS_ROCE_BA_SIZE].buf) +</div><div class='del'>-		off % HNS_ROCE_BA_SIZE);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct hns_roce_aeqe *next_aeqe_sw_v1(struct hns_roce_eq *eq)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_aeqe *aeqe = get_aeqe_v1(eq, eq-&gt;cons_index);</div><div class='del'>-</div><div class='del'>-	return (roce_get_bit(aeqe-&gt;asyn, HNS_ROCE_AEQE_U32_4_OWNER_S) ^</div><div class='del'>-		!!(eq-&gt;cons_index &amp; eq-&gt;entries)) ? aeqe : NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_aeq_int(struct hns_roce_dev *hr_dev,</div><div class='del'>-			       struct hns_roce_eq *eq)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_aeqe *aeqe;</div><div class='del'>-	int aeqes_found = 0;</div><div class='del'>-	int event_type;</div><div class='del'>-</div><div class='del'>-	while ((aeqe = next_aeqe_sw_v1(eq))) {</div><div class='del'>-</div><div class='del'>-		/* Make sure we read the AEQ entry after we have checked the</div><div class='del'>-		 * ownership bit</div><div class='del'>-		 */</div><div class='del'>-		dma_rmb();</div><div class='del'>-</div><div class='del'>-		dev_dbg(dev, "aeqe = %p, aeqe-&gt;asyn.event_type = 0x%lx\n", aeqe,</div><div class='del'>-			roce_get_field(aeqe-&gt;asyn,</div><div class='del'>-				       HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,</div><div class='del'>-				       HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S));</div><div class='del'>-		event_type = roce_get_field(aeqe-&gt;asyn,</div><div class='del'>-					    HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,</div><div class='del'>-					    HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S);</div><div class='del'>-		switch (event_type) {</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_PATH_MIG:</div><div class='del'>-			dev_warn(dev, "PATH MIG not supported\n");</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_COMM_EST:</div><div class='del'>-			dev_warn(dev, "COMMUNICATION established\n");</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:</div><div class='del'>-			dev_warn(dev, "SQ DRAINED not supported\n");</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:</div><div class='del'>-			dev_warn(dev, "PATH MIG failed\n");</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:</div><div class='del'>-			hns_roce_v1_qp_err_handle(hr_dev, aeqe, event_type);</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:</div><div class='del'>-			dev_warn(dev, "SRQ not support!\n");</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID:</div><div class='del'>-			hns_roce_v1_cq_err_handle(hr_dev, aeqe, event_type);</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_PORT_CHANGE:</div><div class='del'>-			dev_warn(dev, "port change.\n");</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_MB:</div><div class='del'>-			hns_roce_cmd_event(hr_dev,</div><div class='del'>-					   le16_to_cpu(aeqe-&gt;event.cmd.token),</div><div class='del'>-					   aeqe-&gt;event.cmd.status,</div><div class='del'>-					   le64_to_cpu(aeqe-&gt;event.cmd.out_param</div><div class='del'>-					   ));</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:</div><div class='del'>-			hns_roce_v1_db_overflow_handle(hr_dev, aeqe);</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:</div><div class='del'>-			dev_warn(dev, "CEQ 0x%lx overflow.\n",</div><div class='del'>-			roce_get_field(aeqe-&gt;event.ce_event.ceqe,</div><div class='del'>-				     HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M,</div><div class='del'>-				     HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S));</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			dev_warn(dev, "Unhandled event %d on EQ %d at idx %u.\n",</div><div class='del'>-				 event_type, eq-&gt;eqn, eq-&gt;cons_index);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		eq-&gt;cons_index++;</div><div class='del'>-		aeqes_found = 1;</div><div class='del'>-</div><div class='del'>-		if (eq-&gt;cons_index &gt; 2 * hr_dev-&gt;caps.aeqe_depth - 1) {</div><div class='del'>-			dev_warn(dev, "cons_index overflow, set back to 0.\n");</div><div class='del'>-			eq-&gt;cons_index = 0;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	set_eq_cons_index_v1(eq, 0);</div><div class='del'>-</div><div class='del'>-	return aeqes_found;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct hns_roce_ceqe *get_ceqe_v1(struct hns_roce_eq *eq, u32 entry)</div><div class='del'>-{</div><div class='del'>-	unsigned long off = (entry &amp; (eq-&gt;entries - 1)) *</div><div class='del'>-			     HNS_ROCE_CEQ_ENTRY_SIZE;</div><div class='del'>-</div><div class='del'>-	return (struct hns_roce_ceqe *)((u8 *)</div><div class='del'>-			(eq-&gt;buf_list[off / HNS_ROCE_BA_SIZE].buf) +</div><div class='del'>-			off % HNS_ROCE_BA_SIZE);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct hns_roce_ceqe *next_ceqe_sw_v1(struct hns_roce_eq *eq)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_ceqe *ceqe = get_ceqe_v1(eq, eq-&gt;cons_index);</div><div class='del'>-</div><div class='del'>-	return (!!(roce_get_bit(ceqe-&gt;comp,</div><div class='del'>-		HNS_ROCE_CEQE_CEQE_COMP_OWNER_S))) ^</div><div class='del'>-		(!!(eq-&gt;cons_index &amp; eq-&gt;entries)) ? ceqe : NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_ceq_int(struct hns_roce_dev *hr_dev,</div><div class='del'>-			       struct hns_roce_eq *eq)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_ceqe *ceqe;</div><div class='del'>-	int ceqes_found = 0;</div><div class='del'>-	u32 cqn;</div><div class='del'>-</div><div class='del'>-	while ((ceqe = next_ceqe_sw_v1(eq))) {</div><div class='del'>-</div><div class='del'>-		/* Make sure we read CEQ entry after we have checked the</div><div class='del'>-		 * ownership bit</div><div class='del'>-		 */</div><div class='del'>-		dma_rmb();</div><div class='del'>-</div><div class='del'>-		cqn = roce_get_field(ceqe-&gt;comp,</div><div class='del'>-				     HNS_ROCE_CEQE_CEQE_COMP_CQN_M,</div><div class='del'>-				     HNS_ROCE_CEQE_CEQE_COMP_CQN_S);</div><div class='del'>-		hns_roce_cq_completion(hr_dev, cqn);</div><div class='del'>-</div><div class='del'>-		++eq-&gt;cons_index;</div><div class='del'>-		ceqes_found = 1;</div><div class='del'>-</div><div class='del'>-		if (eq-&gt;cons_index &gt; 2 * hr_dev-&gt;caps.ceqe_depth - 1) {</div><div class='del'>-			dev_warn(&amp;eq-&gt;hr_dev-&gt;pdev-&gt;dev,</div><div class='del'>-				"cons_index overflow, set back to 0.\n");</div><div class='del'>-			eq-&gt;cons_index = 0;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	set_eq_cons_index_v1(eq, 0);</div><div class='del'>-</div><div class='del'>-	return ceqes_found;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static irqreturn_t hns_roce_v1_msix_interrupt_eq(int irq, void *eq_ptr)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_eq  *eq  = eq_ptr;</div><div class='del'>-	struct hns_roce_dev *hr_dev = eq-&gt;hr_dev;</div><div class='del'>-	int int_work = 0;</div><div class='del'>-</div><div class='del'>-	if (eq-&gt;type_flag == HNS_ROCE_CEQ)</div><div class='del'>-		/* CEQ irq routine, CEQ is pulse irq, not clear */</div><div class='del'>-		int_work = hns_roce_v1_ceq_int(hr_dev, eq);</div><div class='del'>-	else</div><div class='del'>-		/* AEQ irq routine, AEQ is pulse irq, not clear */</div><div class='del'>-		int_work = hns_roce_v1_aeq_int(hr_dev, eq);</div><div class='del'>-</div><div class='del'>-	return IRQ_RETVAL(int_work);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static irqreturn_t hns_roce_v1_msix_interrupt_abn(int irq, void *dev_id)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_dev *hr_dev = dev_id;</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	int int_work = 0;</div><div class='del'>-	u32 caepaemask_val;</div><div class='del'>-	u32 cealmovf_val;</div><div class='del'>-	u32 caepaest_val;</div><div class='del'>-	u32 aeshift_val;</div><div class='del'>-	u32 ceshift_val;</div><div class='del'>-	u32 cemask_val;</div><div class='del'>-	__le32 tmp;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Abnormal interrupt:</div><div class='del'>-	 * AEQ overflow, ECC multi-bit err, CEQ overflow must clear</div><div class='del'>-	 * interrupt, mask irq, clear irq, cancel mask operation</div><div class='del'>-	 */</div><div class='del'>-	aeshift_val = roce_read(hr_dev, ROCEE_CAEP_AEQC_AEQE_SHIFT_REG);</div><div class='del'>-	tmp = cpu_to_le32(aeshift_val);</div><div class='del'>-</div><div class='del'>-	/* AEQE overflow */</div><div class='del'>-	if (roce_get_bit(tmp,</div><div class='del'>-		ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQ_ALM_OVF_INT_ST_S) == 1) {</div><div class='del'>-		dev_warn(dev, "AEQ overflow!\n");</div><div class='del'>-</div><div class='del'>-		/* Set mask */</div><div class='del'>-		caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);</div><div class='del'>-		tmp = cpu_to_le32(caepaemask_val);</div><div class='del'>-		roce_set_bit(tmp, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,</div><div class='del'>-			     HNS_ROCE_INT_MASK_ENABLE);</div><div class='del'>-		caepaemask_val = le32_to_cpu(tmp);</div><div class='del'>-		roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val);</div><div class='del'>-</div><div class='del'>-		/* Clear int state(INT_WC : write 1 clear) */</div><div class='del'>-		caepaest_val = roce_read(hr_dev, ROCEE_CAEP_AE_ST_REG);</div><div class='del'>-		tmp = cpu_to_le32(caepaest_val);</div><div class='del'>-		roce_set_bit(tmp, ROCEE_CAEP_AE_ST_CAEP_AEQ_ALM_OVF_S, 1);</div><div class='del'>-		caepaest_val = le32_to_cpu(tmp);</div><div class='del'>-		roce_write(hr_dev, ROCEE_CAEP_AE_ST_REG, caepaest_val);</div><div class='del'>-</div><div class='del'>-		/* Clear mask */</div><div class='del'>-		caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);</div><div class='del'>-		tmp = cpu_to_le32(caepaemask_val);</div><div class='del'>-		roce_set_bit(tmp, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,</div><div class='del'>-			     HNS_ROCE_INT_MASK_DISABLE);</div><div class='del'>-		caepaemask_val = le32_to_cpu(tmp);</div><div class='del'>-		roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* CEQ almost overflow */</div><div class='del'>-	for (i = 0; i &lt; hr_dev-&gt;caps.num_comp_vectors; i++) {</div><div class='del'>-		ceshift_val = roce_read(hr_dev, ROCEE_CAEP_CEQC_SHIFT_0_REG +</div><div class='del'>-					i * CEQ_REG_OFFSET);</div><div class='del'>-		tmp = cpu_to_le32(ceshift_val);</div><div class='del'>-</div><div class='del'>-		if (roce_get_bit(tmp,</div><div class='del'>-			ROCEE_CAEP_CEQC_SHIFT_CAEP_CEQ_ALM_OVF_INT_ST_S) == 1) {</div><div class='del'>-			dev_warn(dev, "CEQ[%d] almost overflow!\n", i);</div><div class='del'>-			int_work++;</div><div class='del'>-</div><div class='del'>-			/* Set mask */</div><div class='del'>-			cemask_val = roce_read(hr_dev,</div><div class='del'>-					       ROCEE_CAEP_CE_IRQ_MASK_0_REG +</div><div class='del'>-					       i * CEQ_REG_OFFSET);</div><div class='del'>-			tmp = cpu_to_le32(cemask_val);</div><div class='del'>-			roce_set_bit(tmp,</div><div class='del'>-				ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S,</div><div class='del'>-				HNS_ROCE_INT_MASK_ENABLE);</div><div class='del'>-			cemask_val = le32_to_cpu(tmp);</div><div class='del'>-			roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +</div><div class='del'>-				   i * CEQ_REG_OFFSET, cemask_val);</div><div class='del'>-</div><div class='del'>-			/* Clear int state(INT_WC : write 1 clear) */</div><div class='del'>-			cealmovf_val = roce_read(hr_dev,</div><div class='del'>-				       ROCEE_CAEP_CEQ_ALM_OVF_0_REG +</div><div class='del'>-				       i * CEQ_REG_OFFSET);</div><div class='del'>-			tmp = cpu_to_le32(cealmovf_val);</div><div class='del'>-			roce_set_bit(tmp,</div><div class='del'>-				     ROCEE_CAEP_CEQ_ALM_OVF_CAEP_CEQ_ALM_OVF_S,</div><div class='del'>-				     1);</div><div class='del'>-			cealmovf_val = le32_to_cpu(tmp);</div><div class='del'>-			roce_write(hr_dev, ROCEE_CAEP_CEQ_ALM_OVF_0_REG +</div><div class='del'>-				   i * CEQ_REG_OFFSET, cealmovf_val);</div><div class='del'>-</div><div class='del'>-			/* Clear mask */</div><div class='del'>-			cemask_val = roce_read(hr_dev,</div><div class='del'>-				     ROCEE_CAEP_CE_IRQ_MASK_0_REG +</div><div class='del'>-				     i * CEQ_REG_OFFSET);</div><div class='del'>-			tmp = cpu_to_le32(cemask_val);</div><div class='del'>-			roce_set_bit(tmp,</div><div class='del'>-			       ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S,</div><div class='del'>-			       HNS_ROCE_INT_MASK_DISABLE);</div><div class='del'>-			cemask_val = le32_to_cpu(tmp);</div><div class='del'>-			roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +</div><div class='del'>-				   i * CEQ_REG_OFFSET, cemask_val);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* ECC multi-bit error alarm */</div><div class='del'>-	dev_warn(dev, "ECC UCERR ALARM: 0x%x, 0x%x, 0x%x\n",</div><div class='del'>-		 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM0_REG),</div><div class='del'>-		 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM1_REG),</div><div class='del'>-		 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM2_REG));</div><div class='del'>-</div><div class='del'>-	dev_warn(dev, "ECC CERR ALARM: 0x%x, 0x%x, 0x%x\n",</div><div class='del'>-		 roce_read(hr_dev, ROCEE_ECC_CERR_ALM0_REG),</div><div class='del'>-		 roce_read(hr_dev, ROCEE_ECC_CERR_ALM1_REG),</div><div class='del'>-		 roce_read(hr_dev, ROCEE_ECC_CERR_ALM2_REG));</div><div class='del'>-</div><div class='del'>-	return IRQ_RETVAL(int_work);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_int_mask_enable(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	u32 aemask_val;</div><div class='del'>-	int masken = 0;</div><div class='del'>-	__le32 tmp;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	/* AEQ INT */</div><div class='del'>-	aemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);</div><div class='del'>-	tmp = cpu_to_le32(aemask_val);</div><div class='del'>-	roce_set_bit(tmp, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,</div><div class='del'>-		     masken);</div><div class='del'>-	roce_set_bit(tmp, ROCEE_CAEP_AE_MASK_CAEP_AE_IRQ_MASK_S, masken);</div><div class='del'>-	aemask_val = le32_to_cpu(tmp);</div><div class='del'>-	roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, aemask_val);</div><div class='del'>-</div><div class='del'>-	/* CEQ INT */</div><div class='del'>-	for (i = 0; i &lt; hr_dev-&gt;caps.num_comp_vectors; i++) {</div><div class='del'>-		/* IRQ mask */</div><div class='del'>-		roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +</div><div class='del'>-			   i * CEQ_REG_OFFSET, masken);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_free_eq(struct hns_roce_dev *hr_dev,</div><div class='del'>-				struct hns_roce_eq *eq)</div><div class='del'>-{</div><div class='del'>-	int npages = (PAGE_ALIGN(eq-&gt;eqe_size * eq-&gt;entries) +</div><div class='del'>-		      HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	if (!eq-&gt;buf_list)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; npages; ++i)</div><div class='del'>-		dma_free_coherent(&amp;hr_dev-&gt;pdev-&gt;dev, HNS_ROCE_BA_SIZE,</div><div class='del'>-				  eq-&gt;buf_list[i].buf, eq-&gt;buf_list[i].map);</div><div class='del'>-</div><div class='del'>-	kfree(eq-&gt;buf_list);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_enable_eq(struct hns_roce_dev *hr_dev, int eq_num,</div><div class='del'>-				  int enable_flag)</div><div class='del'>-{</div><div class='del'>-	void __iomem *eqc = hr_dev-&gt;eq_table.eqc_base[eq_num];</div><div class='del'>-	__le32 tmp;</div><div class='del'>-	u32 val;</div><div class='del'>-</div><div class='del'>-	val = readl(eqc);</div><div class='del'>-	tmp = cpu_to_le32(val);</div><div class='del'>-</div><div class='del'>-	if (enable_flag)</div><div class='del'>-		roce_set_field(tmp,</div><div class='del'>-			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,</div><div class='del'>-			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,</div><div class='del'>-			       HNS_ROCE_EQ_STAT_VALID);</div><div class='del'>-	else</div><div class='del'>-		roce_set_field(tmp,</div><div class='del'>-			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,</div><div class='del'>-			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,</div><div class='del'>-			       HNS_ROCE_EQ_STAT_INVALID);</div><div class='del'>-</div><div class='del'>-	val = le32_to_cpu(tmp);</div><div class='del'>-	writel(val, eqc);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_create_eq(struct hns_roce_dev *hr_dev,</div><div class='del'>-				 struct hns_roce_eq *eq)</div><div class='del'>-{</div><div class='del'>-	void __iomem *eqc = hr_dev-&gt;eq_table.eqc_base[eq-&gt;eqn];</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	dma_addr_t tmp_dma_addr;</div><div class='del'>-	u32 eqconsindx_val = 0;</div><div class='del'>-	u32 eqcuridx_val = 0;</div><div class='del'>-	u32 eqshift_val = 0;</div><div class='del'>-	__le32 tmp2 = 0;</div><div class='del'>-	__le32 tmp1 = 0;</div><div class='del'>-	__le32 tmp = 0;</div><div class='del'>-	int num_bas;</div><div class='del'>-	int ret;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	num_bas = (PAGE_ALIGN(eq-&gt;entries * eq-&gt;eqe_size) +</div><div class='del'>-		   HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE;</div><div class='del'>-</div><div class='del'>-	if ((eq-&gt;entries * eq-&gt;eqe_size) &gt; HNS_ROCE_BA_SIZE) {</div><div class='del'>-		dev_err(dev, "[error]eq buf %d gt ba size(%d) need bas=%d\n",</div><div class='del'>-			(eq-&gt;entries * eq-&gt;eqe_size), HNS_ROCE_BA_SIZE,</div><div class='del'>-			num_bas);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	eq-&gt;buf_list = kcalloc(num_bas, sizeof(*eq-&gt;buf_list), GFP_KERNEL);</div><div class='del'>-	if (!eq-&gt;buf_list)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; num_bas; ++i) {</div><div class='del'>-		eq-&gt;buf_list[i].buf = dma_alloc_coherent(dev, HNS_ROCE_BA_SIZE,</div><div class='del'>-							 &amp;tmp_dma_addr,</div><div class='del'>-							 GFP_KERNEL);</div><div class='del'>-		if (!eq-&gt;buf_list[i].buf) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto err_out_free_pages;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		eq-&gt;buf_list[i].map = tmp_dma_addr;</div><div class='del'>-		memset(eq-&gt;buf_list[i].buf, 0, HNS_ROCE_BA_SIZE);</div><div class='del'>-	}</div><div class='del'>-	eq-&gt;cons_index = 0;</div><div class='del'>-	roce_set_field(tmp, ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,</div><div class='del'>-		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,</div><div class='del'>-		       HNS_ROCE_EQ_STAT_INVALID);</div><div class='del'>-	roce_set_field(tmp, ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_M,</div><div class='del'>-		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S,</div><div class='del'>-		       eq-&gt;log_entries);</div><div class='del'>-	eqshift_val = le32_to_cpu(tmp);</div><div class='del'>-	writel(eqshift_val, eqc);</div><div class='del'>-</div><div class='del'>-	/* Configure eq extended address 12~44bit */</div><div class='del'>-	writel((u32)(eq-&gt;buf_list[0].map &gt;&gt; 12), eqc + 4);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Configure eq extended address 45~49 bit.</div><div class='del'>-	 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of</div><div class='del'>-	 * using 4K page, and shift more 32 because of</div><div class='del'>-	 * caculating the high 32 bit value evaluated to hardware.</div><div class='del'>-	 */</div><div class='del'>-	roce_set_field(tmp1, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M,</div><div class='del'>-		       ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S,</div><div class='del'>-		       eq-&gt;buf_list[0].map &gt;&gt; 44);</div><div class='del'>-	roce_set_field(tmp1, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M,</div><div class='del'>-		       ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S, 0);</div><div class='del'>-	eqcuridx_val = le32_to_cpu(tmp1);</div><div class='del'>-	writel(eqcuridx_val, eqc + 8);</div><div class='del'>-</div><div class='del'>-	/* Configure eq consumer index */</div><div class='del'>-	roce_set_field(tmp2, ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M,</div><div class='del'>-		       ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S, 0);</div><div class='del'>-	eqconsindx_val = le32_to_cpu(tmp2);</div><div class='del'>-	writel(eqconsindx_val, eqc + 0xc);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err_out_free_pages:</div><div class='del'>-	for (i -= 1; i &gt;= 0; i--)</div><div class='del'>-		dma_free_coherent(dev, HNS_ROCE_BA_SIZE, eq-&gt;buf_list[i].buf,</div><div class='del'>-				  eq-&gt;buf_list[i].map);</div><div class='del'>-</div><div class='del'>-	kfree(eq-&gt;buf_list);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_v1_init_eq_table(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_eq_table *eq_table = &amp;hr_dev-&gt;eq_table;</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct hns_roce_eq *eq;</div><div class='del'>-	int irq_num;</div><div class='del'>-	int eq_num;</div><div class='del'>-	int ret;</div><div class='del'>-	int i, j;</div><div class='del'>-</div><div class='del'>-	eq_num = hr_dev-&gt;caps.num_comp_vectors + hr_dev-&gt;caps.num_aeq_vectors;</div><div class='del'>-	irq_num = eq_num + hr_dev-&gt;caps.num_other_vectors;</div><div class='del'>-</div><div class='del'>-	eq_table-&gt;eq = kcalloc(eq_num, sizeof(*eq_table-&gt;eq), GFP_KERNEL);</div><div class='del'>-	if (!eq_table-&gt;eq)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	eq_table-&gt;eqc_base = kcalloc(eq_num, sizeof(*eq_table-&gt;eqc_base),</div><div class='del'>-				     GFP_KERNEL);</div><div class='del'>-	if (!eq_table-&gt;eqc_base) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err_eqc_base_alloc_fail;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; eq_num; i++) {</div><div class='del'>-		eq = &amp;eq_table-&gt;eq[i];</div><div class='del'>-		eq-&gt;hr_dev = hr_dev;</div><div class='del'>-		eq-&gt;eqn = i;</div><div class='del'>-		eq-&gt;irq = hr_dev-&gt;irq[i];</div><div class='del'>-		eq-&gt;log_page_size = PAGE_SHIFT;</div><div class='del'>-</div><div class='del'>-		if (i &lt; hr_dev-&gt;caps.num_comp_vectors) {</div><div class='del'>-			/* CEQ */</div><div class='del'>-			eq_table-&gt;eqc_base[i] = hr_dev-&gt;reg_base +</div><div class='del'>-						ROCEE_CAEP_CEQC_SHIFT_0_REG +</div><div class='del'>-						CEQ_REG_OFFSET * i;</div><div class='del'>-			eq-&gt;type_flag = HNS_ROCE_CEQ;</div><div class='del'>-			eq-&gt;doorbell = hr_dev-&gt;reg_base +</div><div class='del'>-				       ROCEE_CAEP_CEQC_CONS_IDX_0_REG +</div><div class='del'>-				       CEQ_REG_OFFSET * i;</div><div class='del'>-			eq-&gt;entries = hr_dev-&gt;caps.ceqe_depth;</div><div class='del'>-			eq-&gt;log_entries = ilog2(eq-&gt;entries);</div><div class='del'>-			eq-&gt;eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE;</div><div class='del'>-		} else {</div><div class='del'>-			/* AEQ */</div><div class='del'>-			eq_table-&gt;eqc_base[i] = hr_dev-&gt;reg_base +</div><div class='del'>-						ROCEE_CAEP_AEQC_AEQE_SHIFT_REG;</div><div class='del'>-			eq-&gt;type_flag = HNS_ROCE_AEQ;</div><div class='del'>-			eq-&gt;doorbell = hr_dev-&gt;reg_base +</div><div class='del'>-				       ROCEE_CAEP_AEQE_CONS_IDX_REG;</div><div class='del'>-			eq-&gt;entries = hr_dev-&gt;caps.aeqe_depth;</div><div class='del'>-			eq-&gt;log_entries = ilog2(eq-&gt;entries);</div><div class='del'>-			eq-&gt;eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Disable irq */</div><div class='del'>-	hns_roce_v1_int_mask_enable(hr_dev);</div><div class='del'>-</div><div class='del'>-	/* Configure ce int interval */</div><div class='del'>-	roce_write(hr_dev, ROCEE_CAEP_CE_INTERVAL_CFG_REG,</div><div class='del'>-		   HNS_ROCE_CEQ_DEFAULT_INTERVAL);</div><div class='del'>-</div><div class='del'>-	/* Configure ce int burst num */</div><div class='del'>-	roce_write(hr_dev, ROCEE_CAEP_CE_BURST_NUM_CFG_REG,</div><div class='del'>-		   HNS_ROCE_CEQ_DEFAULT_BURST_NUM);</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; eq_num; i++) {</div><div class='del'>-		ret = hns_roce_v1_create_eq(hr_dev, &amp;eq_table-&gt;eq[i]);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "eq create failed\n");</div><div class='del'>-			goto err_create_eq_fail;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	for (j = 0; j &lt; irq_num; j++) {</div><div class='del'>-		if (j &lt; eq_num)</div><div class='del'>-			ret = request_irq(hr_dev-&gt;irq[j],</div><div class='del'>-					  hns_roce_v1_msix_interrupt_eq, 0,</div><div class='del'>-					  hr_dev-&gt;irq_names[j],</div><div class='del'>-					  &amp;eq_table-&gt;eq[j]);</div><div class='del'>-		else</div><div class='del'>-			ret = request_irq(hr_dev-&gt;irq[j],</div><div class='del'>-					  hns_roce_v1_msix_interrupt_abn, 0,</div><div class='del'>-					  hr_dev-&gt;irq_names[j], hr_dev);</div><div class='del'>-</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "request irq error!\n");</div><div class='del'>-			goto err_request_irq_fail;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; eq_num; i++)</div><div class='del'>-		hns_roce_v1_enable_eq(hr_dev, i, EQ_ENABLE);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err_request_irq_fail:</div><div class='del'>-	for (j -= 1; j &gt;= 0; j--)</div><div class='del'>-		free_irq(hr_dev-&gt;irq[j], &amp;eq_table-&gt;eq[j]);</div><div class='del'>-</div><div class='del'>-err_create_eq_fail:</div><div class='del'>-	for (i -= 1; i &gt;= 0; i--)</div><div class='del'>-		hns_roce_v1_free_eq(hr_dev, &amp;eq_table-&gt;eq[i]);</div><div class='del'>-</div><div class='del'>-	kfree(eq_table-&gt;eqc_base);</div><div class='del'>-</div><div class='del'>-err_eqc_base_alloc_fail:</div><div class='del'>-	kfree(eq_table-&gt;eq);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v1_cleanup_eq_table(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_eq_table *eq_table = &amp;hr_dev-&gt;eq_table;</div><div class='del'>-	int irq_num;</div><div class='del'>-	int eq_num;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	eq_num = hr_dev-&gt;caps.num_comp_vectors + hr_dev-&gt;caps.num_aeq_vectors;</div><div class='del'>-	irq_num = eq_num + hr_dev-&gt;caps.num_other_vectors;</div><div class='del'>-	for (i = 0; i &lt; eq_num; i++) {</div><div class='del'>-		/* Disable EQ */</div><div class='del'>-		hns_roce_v1_enable_eq(hr_dev, i, EQ_DISABLE);</div><div class='del'>-</div><div class='del'>-		free_irq(hr_dev-&gt;irq[i], &amp;eq_table-&gt;eq[i]);</div><div class='del'>-</div><div class='del'>-		hns_roce_v1_free_eq(hr_dev, &amp;eq_table-&gt;eq[i]);</div><div class='del'>-	}</div><div class='del'>-	for (i = eq_num; i &lt; irq_num; i++)</div><div class='del'>-		free_irq(hr_dev-&gt;irq[i], hr_dev);</div><div class='del'>-</div><div class='del'>-	kfree(eq_table-&gt;eqc_base);</div><div class='del'>-	kfree(eq_table-&gt;eq);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static const struct ib_device_ops hns_roce_v1_dev_ops = {</div><div class='del'>-	.destroy_qp = hns_roce_v1_destroy_qp,</div><div class='del'>-	.modify_cq = hns_roce_v1_modify_cq,</div><div class='del'>-	.poll_cq = hns_roce_v1_poll_cq,</div><div class='del'>-	.post_recv = hns_roce_v1_post_recv,</div><div class='del'>-	.post_send = hns_roce_v1_post_send,</div><div class='del'>-	.query_qp = hns_roce_v1_query_qp,</div><div class='del'>-	.req_notify_cq = hns_roce_v1_req_notify_cq,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static const struct hns_roce_hw hns_roce_hw_v1 = {</div><div class='del'>-	.reset = hns_roce_v1_reset,</div><div class='del'>-	.hw_profile = hns_roce_v1_profile,</div><div class='del'>-	.hw_init = hns_roce_v1_init,</div><div class='del'>-	.hw_exit = hns_roce_v1_exit,</div><div class='del'>-	.post_mbox = hns_roce_v1_post_mbox,</div><div class='del'>-	.chk_mbox = hns_roce_v1_chk_mbox,</div><div class='del'>-	.set_gid = hns_roce_v1_set_gid,</div><div class='del'>-	.set_mac = hns_roce_v1_set_mac,</div><div class='del'>-	.set_mtu = hns_roce_v1_set_mtu,</div><div class='del'>-	.write_mtpt = hns_roce_v1_write_mtpt,</div><div class='del'>-	.write_cqc = hns_roce_v1_write_cqc,</div><div class='del'>-	.modify_cq = hns_roce_v1_modify_cq,</div><div class='del'>-	.clear_hem = hns_roce_v1_clear_hem,</div><div class='del'>-	.modify_qp = hns_roce_v1_modify_qp,</div><div class='del'>-	.query_qp = hns_roce_v1_query_qp,</div><div class='del'>-	.destroy_qp = hns_roce_v1_destroy_qp,</div><div class='del'>-	.post_send = hns_roce_v1_post_send,</div><div class='del'>-	.post_recv = hns_roce_v1_post_recv,</div><div class='del'>-	.req_notify_cq = hns_roce_v1_req_notify_cq,</div><div class='del'>-	.poll_cq = hns_roce_v1_poll_cq,</div><div class='del'>-	.dereg_mr = hns_roce_v1_dereg_mr,</div><div class='del'>-	.destroy_cq = hns_roce_v1_destroy_cq,</div><div class='del'>-	.init_eq = hns_roce_v1_init_eq_table,</div><div class='del'>-	.cleanup_eq = hns_roce_v1_cleanup_eq_table,</div><div class='del'>-	.hns_roce_dev_ops = &amp;hns_roce_v1_dev_ops,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static const struct of_device_id hns_roce_of_match[] = {</div><div class='del'>-	{ .compatible = "hisilicon,hns-roce-v1", .data = &amp;hns_roce_hw_v1, },</div><div class='del'>-	{},</div><div class='del'>-};</div><div class='del'>-MODULE_DEVICE_TABLE(of, hns_roce_of_match);</div><div class='del'>-</div><div class='del'>-static const struct acpi_device_id hns_roce_acpi_match[] = {</div><div class='del'>-	{ "HISI00D1", (kernel_ulong_t)&amp;hns_roce_hw_v1 },</div><div class='del'>-	{},</div><div class='del'>-};</div><div class='del'>-MODULE_DEVICE_TABLE(acpi, hns_roce_acpi_match);</div><div class='del'>-</div><div class='del'>-static int hns_roce_node_match(struct device *dev, void *fwnode)</div><div class='del'>-{</div><div class='del'>-	return dev-&gt;fwnode == fwnode;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct</div><div class='del'>-platform_device *hns_roce_find_pdev(struct fwnode_handle *fwnode)</div><div class='del'>-{</div><div class='del'>-	struct device *dev;</div><div class='del'>-</div><div class='del'>-	/* get the 'device' corresponding to the matching 'fwnode' */</div><div class='del'>-	dev = bus_find_device(&amp;platform_bus_type, NULL,</div><div class='del'>-			      fwnode, hns_roce_node_match);</div><div class='del'>-	/* get the platform device */</div><div class='del'>-	return dev ? to_platform_device(dev) : NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_get_cfg(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = &amp;hr_dev-&gt;pdev-&gt;dev;</div><div class='del'>-	struct platform_device *pdev = NULL;</div><div class='del'>-	struct net_device *netdev = NULL;</div><div class='del'>-	struct device_node *net_node;</div><div class='del'>-	struct resource *res;</div><div class='del'>-	int port_cnt = 0;</div><div class='del'>-	u8 phy_port;</div><div class='del'>-	int ret;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	/* check if we are compatible with the underlying SoC */</div><div class='del'>-	if (dev_of_node(dev)) {</div><div class='del'>-		const struct of_device_id *of_id;</div><div class='del'>-</div><div class='del'>-		of_id = of_match_node(hns_roce_of_match, dev-&gt;of_node);</div><div class='del'>-		if (!of_id) {</div><div class='del'>-			dev_err(dev, "device is not compatible!\n");</div><div class='del'>-			return -ENXIO;</div><div class='del'>-		}</div><div class='del'>-		hr_dev-&gt;hw = (const struct hns_roce_hw *)of_id-&gt;data;</div><div class='del'>-		if (!hr_dev-&gt;hw) {</div><div class='del'>-			dev_err(dev, "couldn't get H/W specific DT data!\n");</div><div class='del'>-			return -ENXIO;</div><div class='del'>-		}</div><div class='del'>-	} else if (is_acpi_device_node(dev-&gt;fwnode)) {</div><div class='del'>-		const struct acpi_device_id *acpi_id;</div><div class='del'>-</div><div class='del'>-		acpi_id = acpi_match_device(hns_roce_acpi_match, dev);</div><div class='del'>-		if (!acpi_id) {</div><div class='del'>-			dev_err(dev, "device is not compatible!\n");</div><div class='del'>-			return -ENXIO;</div><div class='del'>-		}</div><div class='del'>-		hr_dev-&gt;hw = (const struct hns_roce_hw *) acpi_id-&gt;driver_data;</div><div class='del'>-		if (!hr_dev-&gt;hw) {</div><div class='del'>-			dev_err(dev, "couldn't get H/W specific ACPI data!\n");</div><div class='del'>-			return -ENXIO;</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		dev_err(dev, "can't read compatibility data from DT or ACPI\n");</div><div class='del'>-		return -ENXIO;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* get the mapped register base address */</div><div class='del'>-	res = platform_get_resource(hr_dev-&gt;pdev, IORESOURCE_MEM, 0);</div><div class='del'>-	hr_dev-&gt;reg_base = devm_ioremap_resource(dev, res);</div><div class='del'>-	if (IS_ERR(hr_dev-&gt;reg_base))</div><div class='del'>-		return PTR_ERR(hr_dev-&gt;reg_base);</div><div class='del'>-</div><div class='del'>-	/* read the node_guid of IB device from the DT or ACPI */</div><div class='del'>-	ret = device_property_read_u8_array(dev, "node-guid",</div><div class='del'>-					    (u8 *)&amp;hr_dev-&gt;ib_dev.node_guid,</div><div class='del'>-					    GUID_LEN);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "couldn't get node_guid from DT or ACPI!\n");</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* get the RoCE associated ethernet ports or netdevices */</div><div class='del'>-	for (i = 0; i &lt; HNS_ROCE_MAX_PORTS; i++) {</div><div class='del'>-		if (dev_of_node(dev)) {</div><div class='del'>-			net_node = of_parse_phandle(dev-&gt;of_node, "eth-handle",</div><div class='del'>-						    i);</div><div class='del'>-			if (!net_node)</div><div class='del'>-				continue;</div><div class='del'>-			pdev = of_find_device_by_node(net_node);</div><div class='del'>-		} else if (is_acpi_device_node(dev-&gt;fwnode)) {</div><div class='del'>-			struct fwnode_reference_args args;</div><div class='del'>-</div><div class='del'>-			ret = acpi_node_get_property_reference(dev-&gt;fwnode,</div><div class='del'>-							       "eth-handle",</div><div class='del'>-							       i, &amp;args);</div><div class='del'>-			if (ret)</div><div class='del'>-				continue;</div><div class='del'>-			pdev = hns_roce_find_pdev(args.fwnode);</div><div class='del'>-		} else {</div><div class='del'>-			dev_err(dev, "cannot read data from DT or ACPI\n");</div><div class='del'>-			return -ENXIO;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (pdev) {</div><div class='del'>-			netdev = platform_get_drvdata(pdev);</div><div class='del'>-			phy_port = (u8)i;</div><div class='del'>-			if (netdev) {</div><div class='del'>-				hr_dev-&gt;iboe.netdevs[port_cnt] = netdev;</div><div class='del'>-				hr_dev-&gt;iboe.phy_port[port_cnt] = phy_port;</div><div class='del'>-			} else {</div><div class='del'>-				dev_err(dev, "no netdev found with pdev %s\n",</div><div class='del'>-					pdev-&gt;name);</div><div class='del'>-				return -ENODEV;</div><div class='del'>-			}</div><div class='del'>-			port_cnt++;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (port_cnt == 0) {</div><div class='del'>-		dev_err(dev, "unable to get eth-handle for available ports!\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	hr_dev-&gt;caps.num_ports = port_cnt;</div><div class='del'>-</div><div class='del'>-	/* cmd issue mode: 0 is poll, 1 is event */</div><div class='del'>-	hr_dev-&gt;cmd_mod = 1;</div><div class='del'>-	hr_dev-&gt;loop_idc = 0;</div><div class='del'>-	hr_dev-&gt;sdb_offset = ROCEE_DB_SQ_L_0_REG;</div><div class='del'>-	hr_dev-&gt;odb_offset = ROCEE_DB_OTHERS_L_0_REG;</div><div class='del'>-</div><div class='del'>-	/* read the interrupt names from the DT or ACPI */</div><div class='del'>-	ret = device_property_read_string_array(dev, "interrupt-names",</div><div class='del'>-						hr_dev-&gt;irq_names,</div><div class='del'>-						HNS_ROCE_V1_MAX_IRQ_NUM);</div><div class='del'>-	if (ret &lt; 0) {</div><div class='del'>-		dev_err(dev, "couldn't get interrupt names from DT or ACPI!\n");</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* fetch the interrupt numbers */</div><div class='del'>-	for (i = 0; i &lt; HNS_ROCE_V1_MAX_IRQ_NUM; i++) {</div><div class='del'>-		hr_dev-&gt;irq[i] = platform_get_irq(hr_dev-&gt;pdev, i);</div><div class='del'>-		if (hr_dev-&gt;irq[i] &lt;= 0) {</div><div class='del'>-			dev_err(dev, "platform get of irq[=%d] failed!\n", i);</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * hns_roce_probe - RoCE driver entrance</div><div class='del'>- * @pdev: pointer to platform device</div><div class='del'>- * Return : int</div><div class='del'>- *</div><div class='del'>- */</div><div class='del'>-static int hns_roce_probe(struct platform_device *pdev)</div><div class='del'>-{</div><div class='del'>-	int ret;</div><div class='del'>-	struct hns_roce_dev *hr_dev;</div><div class='del'>-	struct device *dev = &amp;pdev-&gt;dev;</div><div class='del'>-</div><div class='del'>-	hr_dev = ib_alloc_device(hns_roce_dev, ib_dev);</div><div class='del'>-	if (!hr_dev)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	hr_dev-&gt;priv = kzalloc(sizeof(struct hns_roce_v1_priv), GFP_KERNEL);</div><div class='del'>-	if (!hr_dev-&gt;priv) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto error_failed_kzalloc;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	hr_dev-&gt;pdev = pdev;</div><div class='del'>-	hr_dev-&gt;dev = dev;</div><div class='del'>-	platform_set_drvdata(pdev, hr_dev);</div><div class='del'>-</div><div class='del'>-	if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64ULL)) &amp;&amp;</div><div class='del'>-	    dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32ULL))) {</div><div class='del'>-		dev_err(dev, "Not usable DMA addressing mode\n");</div><div class='del'>-		ret = -EIO;</div><div class='del'>-		goto error_failed_get_cfg;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_get_cfg(hr_dev);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "Get Configuration failed!\n");</div><div class='del'>-		goto error_failed_get_cfg;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_init(hr_dev);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "RoCE engine init failed!\n");</div><div class='del'>-		goto error_failed_get_cfg;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-error_failed_get_cfg:</div><div class='del'>-	kfree(hr_dev-&gt;priv);</div><div class='del'>-</div><div class='del'>-error_failed_kzalloc:</div><div class='del'>-	ib_dealloc_device(&amp;hr_dev-&gt;ib_dev);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * hns_roce_remove - remove RoCE device</div><div class='del'>- * @pdev: pointer to platform device</div><div class='del'>- */</div><div class='del'>-static int hns_roce_remove(struct platform_device *pdev)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_dev *hr_dev = platform_get_drvdata(pdev);</div><div class='del'>-</div><div class='del'>-	hns_roce_exit(hr_dev);</div><div class='del'>-	kfree(hr_dev-&gt;priv);</div><div class='del'>-	ib_dealloc_device(&amp;hr_dev-&gt;ib_dev);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct platform_driver hns_roce_driver = {</div><div class='del'>-	.probe = hns_roce_probe,</div><div class='del'>-	.remove = hns_roce_remove,</div><div class='del'>-	.driver = {</div><div class='del'>-		.name = DRV_NAME,</div><div class='del'>-		.of_match_table = hns_roce_of_match,</div><div class='del'>-		.acpi_match_table = ACPI_PTR(hns_roce_acpi_match),</div><div class='del'>-	},</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-module_platform_driver(hns_roce_driver);</div><div class='del'>-</div><div class='del'>-MODULE_LICENSE("Dual BSD/GPL");</div><div class='del'>-MODULE_AUTHOR("Wei Hu &lt;xavier.huwei@huawei.com&gt;");</div><div class='del'>-MODULE_AUTHOR("Nenglong Zhao &lt;zhaonenglong@hisilicon.com&gt;");</div><div class='del'>-MODULE_AUTHOR("Lijun Ou &lt;oulijun@huawei.com&gt;");</div><div class='del'>-MODULE_DESCRIPTION("Hisilicon Hip06 Family RoCE Driver");</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h<br/>deleted file mode 100644<br/>index 52307b2c7100..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_hw_v1.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_hw_v1.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,1099 +0,0 @@</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2016 Hisilicon Limited.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='del'>- */</div><div class='del'>-</div><div class='del'>-#ifndef _HNS_ROCE_HW_V1_H</div><div class='del'>-#define _HNS_ROCE_HW_V1_H</div><div class='del'>-</div><div class='del'>-#define CQ_STATE_VALID					2</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_MAX_PD_NUM				0x8000</div><div class='del'>-#define HNS_ROCE_V1_MAX_CQ_NUM				0x10000</div><div class='del'>-#define HNS_ROCE_V1_MAX_CQE_NUM				0x8000</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_MAX_QP_NUM				0x40000</div><div class='del'>-#define HNS_ROCE_V1_MAX_WQE_NUM				0x4000</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_MAX_MTPT_NUM			0x80000</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_MAX_MTT_SEGS			0x100000</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_MAX_QP_INIT_RDMA			128</div><div class='del'>-#define HNS_ROCE_V1_MAX_QP_DEST_RDMA			128</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_MAX_SQ_DESC_SZ			64</div><div class='del'>-#define HNS_ROCE_V1_MAX_RQ_DESC_SZ			64</div><div class='del'>-#define HNS_ROCE_V1_SG_NUM				2</div><div class='del'>-#define HNS_ROCE_V1_INLINE_SIZE				32</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_UAR_NUM				256</div><div class='del'>-#define HNS_ROCE_V1_PHY_UAR_NUM				8</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_GID_NUM				16</div><div class='del'>-#define HNS_ROCE_V1_RESV_QP				8</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_MAX_IRQ_NUM				34</div><div class='del'>-#define HNS_ROCE_V1_COMP_VEC_NUM			32</div><div class='del'>-#define HNS_ROCE_V1_AEQE_VEC_NUM			1</div><div class='del'>-#define HNS_ROCE_V1_ABNORMAL_VEC_NUM			1</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_COMP_EQE_NUM			0x8000</div><div class='del'>-#define HNS_ROCE_V1_ASYNC_EQE_NUM			0x400</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_QPC_ENTRY_SIZE			256</div><div class='del'>-#define HNS_ROCE_V1_IRRL_ENTRY_SIZE			8</div><div class='del'>-#define HNS_ROCE_V1_CQC_ENTRY_SIZE			64</div><div class='del'>-#define HNS_ROCE_V1_MTPT_ENTRY_SIZE			64</div><div class='del'>-#define HNS_ROCE_V1_MTT_ENTRY_SIZE			64</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_CQE_ENTRY_SIZE			32</div><div class='del'>-#define HNS_ROCE_V1_PAGE_SIZE_SUPPORT			0xFFFFF000</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_TABLE_CHUNK_SIZE			(1 &lt;&lt; 17)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_EXT_RAQ_WF				8</div><div class='del'>-#define HNS_ROCE_V1_RAQ_ENTRY				64</div><div class='del'>-#define HNS_ROCE_V1_RAQ_DEPTH				32768</div><div class='del'>-#define HNS_ROCE_V1_RAQ_SIZE	(HNS_ROCE_V1_RAQ_ENTRY * HNS_ROCE_V1_RAQ_DEPTH)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_SDB_DEPTH				0x400</div><div class='del'>-#define HNS_ROCE_V1_ODB_DEPTH				0x400</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_DB_RSVD				0x80</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_SDB_ALEPT				HNS_ROCE_V1_DB_RSVD</div><div class='del'>-#define HNS_ROCE_V1_SDB_ALFUL	(HNS_ROCE_V1_SDB_DEPTH - HNS_ROCE_V1_DB_RSVD)</div><div class='del'>-#define HNS_ROCE_V1_ODB_ALEPT				HNS_ROCE_V1_DB_RSVD</div><div class='del'>-#define HNS_ROCE_V1_ODB_ALFUL	(HNS_ROCE_V1_ODB_DEPTH - HNS_ROCE_V1_DB_RSVD)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_EXT_SDB_DEPTH			0x4000</div><div class='del'>-#define HNS_ROCE_V1_EXT_ODB_DEPTH			0x4000</div><div class='del'>-#define HNS_ROCE_V1_EXT_SDB_ENTRY			16</div><div class='del'>-#define HNS_ROCE_V1_EXT_ODB_ENTRY			16</div><div class='del'>-#define HNS_ROCE_V1_EXT_SDB_SIZE  \</div><div class='del'>-	(HNS_ROCE_V1_EXT_SDB_DEPTH * HNS_ROCE_V1_EXT_SDB_ENTRY)</div><div class='del'>-#define HNS_ROCE_V1_EXT_ODB_SIZE  \</div><div class='del'>-	(HNS_ROCE_V1_EXT_ODB_DEPTH * HNS_ROCE_V1_EXT_ODB_ENTRY)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_EXT_SDB_ALEPT			HNS_ROCE_V1_DB_RSVD</div><div class='del'>-#define HNS_ROCE_V1_EXT_SDB_ALFUL  \</div><div class='del'>-	(HNS_ROCE_V1_EXT_SDB_DEPTH - HNS_ROCE_V1_DB_RSVD)</div><div class='del'>-#define HNS_ROCE_V1_EXT_ODB_ALEPT			HNS_ROCE_V1_DB_RSVD</div><div class='del'>-#define HNS_ROCE_V1_EXT_ODB_ALFUL	\</div><div class='del'>-	(HNS_ROCE_V1_EXT_ODB_DEPTH - HNS_ROCE_V1_DB_RSVD)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS		50000</div><div class='del'>-#define HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS	10000</div><div class='del'>-#define HNS_ROCE_V1_FREE_MR_WAIT_VALUE			5</div><div class='del'>-#define HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE		20</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_BT_RSV_BUF_SIZE			(1 &lt;&lt; 17)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_TPTR_ENTRY_SIZE			2</div><div class='del'>-#define HNS_ROCE_V1_TPTR_BUF_SIZE	\</div><div class='del'>-	(HNS_ROCE_V1_TPTR_ENTRY_SIZE * HNS_ROCE_V1_MAX_CQ_NUM)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_ODB_POLL_MODE				0</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_SDB_NORMAL_MODE			0</div><div class='del'>-#define HNS_ROCE_SDB_EXTEND_MODE			1</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_ODB_EXTEND_MODE			1</div><div class='del'>-</div><div class='del'>-#define KEY_VALID					0x02</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_CQE_QPN_MASK				0x3ffff</div><div class='del'>-#define HNS_ROCE_CQE_STATUS_MASK			0x1f</div><div class='del'>-#define HNS_ROCE_CQE_OPCODE_MASK			0xf</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_CQE_SUCCESS				0x00</div><div class='del'>-#define HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR		0x01</div><div class='del'>-#define HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR		0x02</div><div class='del'>-#define HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR		0x03</div><div class='del'>-#define HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR		0x04</div><div class='del'>-#define HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR	0x05</div><div class='del'>-#define HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR		0x06</div><div class='del'>-#define HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR		0x07</div><div class='del'>-#define HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR	0x08</div><div class='del'>-#define HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR		0x09</div><div class='del'>-#define HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR		0x0a</div><div class='del'>-#define HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR	0x0b</div><div class='del'>-#define HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR		0x0c</div><div class='del'>-</div><div class='del'>-#define QP1C_CFGN_OFFSET				0x28</div><div class='del'>-#define PHY_PORT_OFFSET					0x8</div><div class='del'>-#define MTPT_IDX_SHIFT					16</div><div class='del'>-#define ALL_PORT_VAL_OPEN				0x3f</div><div class='del'>-#define POL_TIME_INTERVAL_VAL				0x80</div><div class='del'>-#define SLEEP_TIME_INTERVAL				20</div><div class='del'>-#define SQ_PSN_SHIFT					8</div><div class='del'>-#define QKEY_VAL					0x80010000</div><div class='del'>-#define SDB_INV_CNT_OFFSET				8</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_CEQ_DEFAULT_INTERVAL			0x10</div><div class='del'>-#define HNS_ROCE_CEQ_DEFAULT_BURST_NUM			0x10</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_INT_MASK_DISABLE			0</div><div class='del'>-#define HNS_ROCE_INT_MASK_ENABLE			1</div><div class='del'>-</div><div class='del'>-#define CEQ_REG_OFFSET					0x18</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_CEQE_CEQE_COMP_OWNER_S	0</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V1_CONS_IDX_M GENMASK(15, 0)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_CEQE_CEQE_COMP_CQN_S 16</div><div class='del'>-#define HNS_ROCE_CEQE_CEQE_COMP_CQN_M GENMASK(31, 16)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S 16</div><div class='del'>-#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M GENMASK(23, 16)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S 24</div><div class='del'>-#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M GENMASK(30, 24)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_AEQE_U32_4_OWNER_S 31</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S 0</div><div class='del'>-#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S 25</div><div class='del'>-#define HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M GENMASK(27, 25)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S 0</div><div class='del'>-#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M GENMASK(15, 0)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S 0</div><div class='del'>-#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M GENMASK(4, 0)</div><div class='del'>-</div><div class='del'>-struct hns_roce_cq_context {</div><div class='del'>-	__le32 cqc_byte_4;</div><div class='del'>-	__le32 cq_bt_l;</div><div class='del'>-	__le32 cqc_byte_12;</div><div class='del'>-	__le32 cur_cqe_ba0_l;</div><div class='del'>-	__le32 cqc_byte_20;</div><div class='del'>-	__le32 cqe_tptr_addr_l;</div><div class='del'>-	__le32 cur_cqe_ba1_l;</div><div class='del'>-	__le32 cqc_byte_32;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S 0</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M   \</div><div class='del'>-	(((1UL &lt;&lt; 2) - 1) &lt;&lt; CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S)</div><div class='del'>-</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_4_CQN_S 16</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_4_CQN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; CQ_CONTEXT_CQC_BYTE_4_CQN_S)</div><div class='del'>-</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S 0</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 17) - 1) &lt;&lt; CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S)</div><div class='del'>-</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S 20</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 4) - 1) &lt;&lt; CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S)</div><div class='del'>-</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_12_CEQN_S 24</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_12_CEQN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; CQ_CONTEXT_CQC_BYTE_12_CEQN_S)</div><div class='del'>-</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S 0</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S)</div><div class='del'>-</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S 16</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S)</div><div class='del'>-</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S 8</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S)</div><div class='del'>-</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S 0</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S)</div><div class='del'>-</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S 9</div><div class='del'>-</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S 8</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S 14</div><div class='del'>-#define CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S 15</div><div class='del'>-</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S 16</div><div class='del'>-#define CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S)</div><div class='del'>-</div><div class='del'>-struct hns_roce_cqe {</div><div class='del'>-	__le32 cqe_byte_4;</div><div class='del'>-	union {</div><div class='del'>-		__le32 r_key;</div><div class='del'>-		__le32 immediate_data;</div><div class='del'>-	};</div><div class='del'>-	__le32 byte_cnt;</div><div class='del'>-	__le32 cqe_byte_16;</div><div class='del'>-	__le32 cqe_byte_20;</div><div class='del'>-	__le32 s_mac_l;</div><div class='del'>-	__le32 cqe_byte_28;</div><div class='del'>-	__le32 reserved;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define CQE_BYTE_4_OWNER_S 7</div><div class='del'>-#define CQE_BYTE_4_SQ_RQ_FLAG_S 14</div><div class='del'>-</div><div class='del'>-#define CQE_BYTE_4_STATUS_OF_THE_OPERATION_S 8</div><div class='del'>-#define CQE_BYTE_4_STATUS_OF_THE_OPERATION_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; CQE_BYTE_4_STATUS_OF_THE_OPERATION_S)</div><div class='del'>-</div><div class='del'>-#define CQE_BYTE_4_WQE_INDEX_S 16</div><div class='del'>-#define CQE_BYTE_4_WQE_INDEX_M	(((1UL &lt;&lt; 14) - 1) &lt;&lt; CQE_BYTE_4_WQE_INDEX_S)</div><div class='del'>-</div><div class='del'>-#define CQE_BYTE_4_OPERATION_TYPE_S 0</div><div class='del'>-#define CQE_BYTE_4_OPERATION_TYPE_M   \</div><div class='del'>-	(((1UL &lt;&lt; 4) - 1) &lt;&lt; CQE_BYTE_4_OPERATION_TYPE_S)</div><div class='del'>-</div><div class='del'>-#define CQE_BYTE_4_IMM_INDICATOR_S 15</div><div class='del'>-</div><div class='del'>-#define CQE_BYTE_16_LOCAL_QPN_S 0</div><div class='del'>-#define CQE_BYTE_16_LOCAL_QPN_M	(((1UL &lt;&lt; 24) - 1) &lt;&lt; CQE_BYTE_16_LOCAL_QPN_S)</div><div class='del'>-</div><div class='del'>-#define CQE_BYTE_20_PORT_NUM_S 26</div><div class='del'>-#define CQE_BYTE_20_PORT_NUM_M	(((1UL &lt;&lt; 3) - 1) &lt;&lt; CQE_BYTE_20_PORT_NUM_S)</div><div class='del'>-</div><div class='del'>-#define CQE_BYTE_20_SL_S 24</div><div class='del'>-#define CQE_BYTE_20_SL_M	(((1UL &lt;&lt; 2) - 1) &lt;&lt; CQE_BYTE_20_SL_S)</div><div class='del'>-</div><div class='del'>-#define CQE_BYTE_20_REMOTE_QPN_S 0</div><div class='del'>-#define CQE_BYTE_20_REMOTE_QPN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; CQE_BYTE_20_REMOTE_QPN_S)</div><div class='del'>-</div><div class='del'>-#define CQE_BYTE_20_GRH_PRESENT_S 29</div><div class='del'>-</div><div class='del'>-#define CQE_BYTE_28_P_KEY_IDX_S 16</div><div class='del'>-#define CQE_BYTE_28_P_KEY_IDX_M	(((1UL &lt;&lt; 16) - 1) &lt;&lt; CQE_BYTE_28_P_KEY_IDX_S)</div><div class='del'>-</div><div class='del'>-#define CQ_DB_REQ_NOT_SOL	0</div><div class='del'>-#define CQ_DB_REQ_NOT		(1 &lt;&lt; 16)</div><div class='del'>-</div><div class='del'>-struct hns_roce_v1_mpt_entry {</div><div class='del'>-	__le32  mpt_byte_4;</div><div class='del'>-	__le32  pbl_addr_l;</div><div class='del'>-	__le32  mpt_byte_12;</div><div class='del'>-	__le32  virt_addr_l;</div><div class='del'>-	__le32  virt_addr_h;</div><div class='del'>-	__le32  length;</div><div class='del'>-	__le32  mpt_byte_28;</div><div class='del'>-	__le32  pa0_l;</div><div class='del'>-	__le32  mpt_byte_36;</div><div class='del'>-	__le32  mpt_byte_40;</div><div class='del'>-	__le32  mpt_byte_44;</div><div class='del'>-	__le32  mpt_byte_48;</div><div class='del'>-	__le32  pa4_l;</div><div class='del'>-	__le32  mpt_byte_56;</div><div class='del'>-	__le32  mpt_byte_60;</div><div class='del'>-	__le32  mpt_byte_64;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_4_KEY_STATE_S 0</div><div class='del'>-#define MPT_BYTE_4_KEY_STATE_M	(((1UL &lt;&lt; 2) - 1) &lt;&lt; MPT_BYTE_4_KEY_STATE_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_4_KEY_S 8</div><div class='del'>-#define MPT_BYTE_4_KEY_M	(((1UL &lt;&lt; 8) - 1) &lt;&lt; MPT_BYTE_4_KEY_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_4_PAGE_SIZE_S 16</div><div class='del'>-#define MPT_BYTE_4_PAGE_SIZE_M	(((1UL &lt;&lt; 2) - 1) &lt;&lt; MPT_BYTE_4_PAGE_SIZE_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_4_MW_TYPE_S 20</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_4_MW_BIND_ENABLE_S 21</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_4_OWN_S 22</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_4_MEMORY_LOCATION_TYPE_S 24</div><div class='del'>-#define MPT_BYTE_4_MEMORY_LOCATION_TYPE_M   \</div><div class='del'>-	(((1UL &lt;&lt; 2) - 1) &lt;&lt; MPT_BYTE_4_MEMORY_LOCATION_TYPE_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_4_REMOTE_ATOMIC_S 26</div><div class='del'>-#define MPT_BYTE_4_LOCAL_WRITE_S 27</div><div class='del'>-#define MPT_BYTE_4_REMOTE_WRITE_S 28</div><div class='del'>-#define MPT_BYTE_4_REMOTE_READ_S 29</div><div class='del'>-#define MPT_BYTE_4_REMOTE_INVAL_ENABLE_S 30</div><div class='del'>-#define MPT_BYTE_4_ADDRESS_TYPE_S 31</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_12_PBL_ADDR_H_S 0</div><div class='del'>-#define MPT_BYTE_12_PBL_ADDR_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 17) - 1) &lt;&lt; MPT_BYTE_12_PBL_ADDR_H_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_12_MW_BIND_COUNTER_S 17</div><div class='del'>-#define MPT_BYTE_12_MW_BIND_COUNTER_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; MPT_BYTE_12_MW_BIND_COUNTER_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_28_PD_S 0</div><div class='del'>-#define MPT_BYTE_28_PD_M	(((1UL &lt;&lt; 16) - 1) &lt;&lt; MPT_BYTE_28_PD_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_28_L_KEY_IDX_L_S 16</div><div class='del'>-#define MPT_BYTE_28_L_KEY_IDX_L_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; MPT_BYTE_28_L_KEY_IDX_L_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_36_PA0_H_S 0</div><div class='del'>-#define MPT_BYTE_36_PA0_H_M	(((1UL &lt;&lt; 5) - 1) &lt;&lt; MPT_BYTE_36_PA0_H_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_36_PA1_L_S 8</div><div class='del'>-#define MPT_BYTE_36_PA1_L_M	(((1UL &lt;&lt; 24) - 1) &lt;&lt; MPT_BYTE_36_PA1_L_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_40_PA1_H_S 0</div><div class='del'>-#define MPT_BYTE_40_PA1_H_M	(((1UL &lt;&lt; 13) - 1) &lt;&lt; MPT_BYTE_40_PA1_H_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_40_PA2_L_S 16</div><div class='del'>-#define MPT_BYTE_40_PA2_L_M	(((1UL &lt;&lt; 16) - 1) &lt;&lt; MPT_BYTE_40_PA2_L_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_44_PA2_H_S 0</div><div class='del'>-#define MPT_BYTE_44_PA2_H_M	(((1UL &lt;&lt; 21) - 1) &lt;&lt; MPT_BYTE_44_PA2_H_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_44_PA3_L_S 24</div><div class='del'>-#define MPT_BYTE_44_PA3_L_M	(((1UL &lt;&lt; 8) - 1) &lt;&lt; MPT_BYTE_44_PA3_L_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_48_PA3_H_S 0</div><div class='del'>-#define MPT_BYTE_48_PA3_H_M	(((1UL &lt;&lt; 29) - 1) &lt;&lt; MPT_BYTE_48_PA3_H_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_56_PA4_H_S 0</div><div class='del'>-#define MPT_BYTE_56_PA4_H_M	(((1UL &lt;&lt; 5) - 1) &lt;&lt; MPT_BYTE_56_PA4_H_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_56_PA5_L_S 8</div><div class='del'>-#define MPT_BYTE_56_PA5_L_M	(((1UL &lt;&lt; 24) - 1) &lt;&lt; MPT_BYTE_56_PA5_L_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_60_PA5_H_S 0</div><div class='del'>-#define MPT_BYTE_60_PA5_H_M	(((1UL &lt;&lt; 13) - 1) &lt;&lt; MPT_BYTE_60_PA5_H_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_60_PA6_L_S 16</div><div class='del'>-#define MPT_BYTE_60_PA6_L_M	(((1UL &lt;&lt; 16) - 1) &lt;&lt; MPT_BYTE_60_PA6_L_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_64_PA6_H_S 0</div><div class='del'>-#define MPT_BYTE_64_PA6_H_M	(((1UL &lt;&lt; 21) - 1) &lt;&lt; MPT_BYTE_64_PA6_H_S)</div><div class='del'>-</div><div class='del'>-#define MPT_BYTE_64_L_KEY_IDX_H_S 24</div><div class='del'>-#define MPT_BYTE_64_L_KEY_IDX_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; MPT_BYTE_64_L_KEY_IDX_H_S)</div><div class='del'>-</div><div class='del'>-struct hns_roce_wqe_ctrl_seg {</div><div class='del'>-	__le32 sgl_pa_h;</div><div class='del'>-	__le32 flag;</div><div class='del'>-	union {</div><div class='del'>-		__be32 imm_data;</div><div class='del'>-		__le32 inv_key;</div><div class='del'>-	};</div><div class='del'>-	__le32 msg_length;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct hns_roce_wqe_data_seg {</div><div class='del'>-	__le64    addr;</div><div class='del'>-	__le32    lkey;</div><div class='del'>-	__le32    len;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct hns_roce_wqe_raddr_seg {</div><div class='del'>-	__le32 rkey;</div><div class='del'>-	__le32 len;/* reserved */</div><div class='del'>-	__le64 raddr;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct hns_roce_rq_wqe_ctrl {</div><div class='del'>-	__le32 rwqe_byte_4;</div><div class='del'>-	__le32 rocee_sgl_ba_l;</div><div class='del'>-	__le32 rwqe_byte_12;</div><div class='del'>-	__le32 reserved[5];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S 16</div><div class='del'>-#define RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M   \</div><div class='del'>-	(((1UL &lt;&lt; 6) - 1) &lt;&lt; RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_QP_DESTROY_TIMEOUT_MSECS	10000</div><div class='del'>-</div><div class='del'>-#define GID_LEN					16</div><div class='del'>-</div><div class='del'>-struct hns_roce_ud_send_wqe {</div><div class='del'>-	__le32 dmac_h;</div><div class='del'>-	__le32 u32_8;</div><div class='del'>-	__le32 immediate_data;</div><div class='del'>-</div><div class='del'>-	__le32 u32_16;</div><div class='del'>-	union {</div><div class='del'>-		unsigned char dgid[GID_LEN];</div><div class='del'>-		struct {</div><div class='del'>-			__le32 u32_20;</div><div class='del'>-			__le32 u32_24;</div><div class='del'>-			__le32 u32_28;</div><div class='del'>-			__le32 u32_32;</div><div class='del'>-		};</div><div class='del'>-	};</div><div class='del'>-</div><div class='del'>-	__le32 u32_36;</div><div class='del'>-	__le32 u32_40;</div><div class='del'>-</div><div class='del'>-	__le32 va0_l;</div><div class='del'>-	__le32 va0_h;</div><div class='del'>-	__le32 l_key0;</div><div class='del'>-</div><div class='del'>-	__le32 va1_l;</div><div class='del'>-	__le32 va1_h;</div><div class='del'>-	__le32 l_key1;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_4_DMAC_0_S 0</div><div class='del'>-#define UD_SEND_WQE_U32_4_DMAC_0_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; UD_SEND_WQE_U32_4_DMAC_0_S)</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_4_DMAC_1_S 8</div><div class='del'>-#define UD_SEND_WQE_U32_4_DMAC_1_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; UD_SEND_WQE_U32_4_DMAC_1_S)</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_4_DMAC_2_S 16</div><div class='del'>-#define UD_SEND_WQE_U32_4_DMAC_2_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; UD_SEND_WQE_U32_4_DMAC_2_S)</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_4_DMAC_3_S 24</div><div class='del'>-#define UD_SEND_WQE_U32_4_DMAC_3_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; UD_SEND_WQE_U32_4_DMAC_3_S)</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_8_DMAC_4_S 0</div><div class='del'>-#define UD_SEND_WQE_U32_8_DMAC_4_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; UD_SEND_WQE_U32_8_DMAC_4_S)</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_8_DMAC_5_S 8</div><div class='del'>-#define UD_SEND_WQE_U32_8_DMAC_5_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; UD_SEND_WQE_U32_8_DMAC_5_S)</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_8_LOOPBACK_INDICATOR_S 22</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_8_OPERATION_TYPE_S 16</div><div class='del'>-#define UD_SEND_WQE_U32_8_OPERATION_TYPE_M   \</div><div class='del'>-	(((1UL &lt;&lt; 4) - 1) &lt;&lt; UD_SEND_WQE_U32_8_OPERATION_TYPE_S)</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S 24</div><div class='del'>-#define UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M   \</div><div class='del'>-	(((1UL &lt;&lt; 6) - 1) &lt;&lt; UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S)</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S 31</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_16_DEST_QP_S 0</div><div class='del'>-#define UD_SEND_WQE_U32_16_DEST_QP_M   \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; UD_SEND_WQE_U32_16_DEST_QP_S)</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S 24</div><div class='del'>-#define UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S)</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_36_FLOW_LABEL_S 0</div><div class='del'>-#define UD_SEND_WQE_U32_36_FLOW_LABEL_M   \</div><div class='del'>-	(((1UL &lt;&lt; 20) - 1) &lt;&lt; UD_SEND_WQE_U32_36_FLOW_LABEL_S)</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_36_PRIORITY_S 20</div><div class='del'>-#define UD_SEND_WQE_U32_36_PRIORITY_M   \</div><div class='del'>-	(((1UL &lt;&lt; 4) - 1) &lt;&lt; UD_SEND_WQE_U32_36_PRIORITY_S)</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_36_SGID_INDEX_S 24</div><div class='del'>-#define UD_SEND_WQE_U32_36_SGID_INDEX_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; UD_SEND_WQE_U32_36_SGID_INDEX_S)</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_40_HOP_LIMIT_S 0</div><div class='del'>-#define UD_SEND_WQE_U32_40_HOP_LIMIT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; UD_SEND_WQE_U32_40_HOP_LIMIT_S)</div><div class='del'>-</div><div class='del'>-#define UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S 8</div><div class='del'>-#define UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S)</div><div class='del'>-</div><div class='del'>-struct hns_roce_sqp_context {</div><div class='del'>-	__le32 qp1c_bytes_4;</div><div class='del'>-	__le32 sq_rq_bt_l;</div><div class='del'>-	__le32 qp1c_bytes_12;</div><div class='del'>-	__le32 qp1c_bytes_16;</div><div class='del'>-	__le32 qp1c_bytes_20;</div><div class='del'>-	__le32 cur_rq_wqe_ba_l;</div><div class='del'>-	__le32 qp1c_bytes_28;</div><div class='del'>-	__le32 qp1c_bytes_32;</div><div class='del'>-	__le32 cur_sq_wqe_ba_l;</div><div class='del'>-	__le32 qp1c_bytes_40;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define QP1C_BYTES_4_QP_STATE_S 0</div><div class='del'>-#define QP1C_BYTES_4_QP_STATE_M   \</div><div class='del'>-	(((1UL &lt;&lt; 3) - 1) &lt;&lt; QP1C_BYTES_4_QP_STATE_S)</div><div class='del'>-</div><div class='del'>-#define QP1C_BYTES_4_SQ_WQE_SHIFT_S 8</div><div class='del'>-#define QP1C_BYTES_4_SQ_WQE_SHIFT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 4) - 1) &lt;&lt; QP1C_BYTES_4_SQ_WQE_SHIFT_S)</div><div class='del'>-</div><div class='del'>-#define QP1C_BYTES_4_RQ_WQE_SHIFT_S 12</div><div class='del'>-#define QP1C_BYTES_4_RQ_WQE_SHIFT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 4) - 1) &lt;&lt; QP1C_BYTES_4_RQ_WQE_SHIFT_S)</div><div class='del'>-</div><div class='del'>-#define QP1C_BYTES_4_PD_S 16</div><div class='del'>-#define QP1C_BYTES_4_PD_M	(((1UL &lt;&lt; 16) - 1) &lt;&lt; QP1C_BYTES_4_PD_S)</div><div class='del'>-</div><div class='del'>-#define QP1C_BYTES_12_SQ_RQ_BT_H_S 0</div><div class='del'>-#define QP1C_BYTES_12_SQ_RQ_BT_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 17) - 1) &lt;&lt; QP1C_BYTES_12_SQ_RQ_BT_H_S)</div><div class='del'>-</div><div class='del'>-#define QP1C_BYTES_16_RQ_HEAD_S 0</div><div class='del'>-#define QP1C_BYTES_16_RQ_HEAD_M	(((1UL &lt;&lt; 15) - 1) &lt;&lt; QP1C_BYTES_16_RQ_HEAD_S)</div><div class='del'>-</div><div class='del'>-#define QP1C_BYTES_16_PORT_NUM_S 16</div><div class='del'>-#define QP1C_BYTES_16_PORT_NUM_M   \</div><div class='del'>-	(((1UL &lt;&lt; 3) - 1) &lt;&lt; QP1C_BYTES_16_PORT_NUM_S)</div><div class='del'>-</div><div class='del'>-#define QP1C_BYTES_16_SIGNALING_TYPE_S 27</div><div class='del'>-#define QP1C_BYTES_16_LOCAL_ENABLE_E2E_CREDIT_S 28</div><div class='del'>-#define QP1C_BYTES_16_RQ_BA_FLG_S 29</div><div class='del'>-#define QP1C_BYTES_16_SQ_BA_FLG_S 30</div><div class='del'>-#define QP1C_BYTES_16_QP1_ERR_S 31</div><div class='del'>-</div><div class='del'>-#define QP1C_BYTES_20_SQ_HEAD_S 0</div><div class='del'>-#define QP1C_BYTES_20_SQ_HEAD_M	(((1UL &lt;&lt; 15) - 1) &lt;&lt; QP1C_BYTES_20_SQ_HEAD_S)</div><div class='del'>-</div><div class='del'>-#define QP1C_BYTES_20_PKEY_IDX_S 16</div><div class='del'>-#define QP1C_BYTES_20_PKEY_IDX_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; QP1C_BYTES_20_PKEY_IDX_S)</div><div class='del'>-</div><div class='del'>-#define QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S 0</div><div class='del'>-#define QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S)</div><div class='del'>-</div><div class='del'>-#define QP1C_BYTES_28_RQ_CUR_IDX_S 16</div><div class='del'>-#define QP1C_BYTES_28_RQ_CUR_IDX_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; QP1C_BYTES_28_RQ_CUR_IDX_S)</div><div class='del'>-</div><div class='del'>-#define QP1C_BYTES_32_TX_CQ_NUM_S 0</div><div class='del'>-#define QP1C_BYTES_32_TX_CQ_NUM_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; QP1C_BYTES_32_TX_CQ_NUM_S)</div><div class='del'>-</div><div class='del'>-#define QP1C_BYTES_32_RX_CQ_NUM_S 16</div><div class='del'>-#define QP1C_BYTES_32_RX_CQ_NUM_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; QP1C_BYTES_32_RX_CQ_NUM_S)</div><div class='del'>-</div><div class='del'>-#define QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S 0</div><div class='del'>-#define QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S)</div><div class='del'>-</div><div class='del'>-#define QP1C_BYTES_40_SQ_CUR_IDX_S 16</div><div class='del'>-#define QP1C_BYTES_40_SQ_CUR_IDX_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; QP1C_BYTES_40_SQ_CUR_IDX_S)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_WQE_INLINE		(1UL&lt;&lt;31)</div><div class='del'>-#define HNS_ROCE_WQE_SE			(1UL&lt;&lt;30)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_WQE_SGE_NUM_BIT	24</div><div class='del'>-#define HNS_ROCE_WQE_IMM		(1UL&lt;&lt;23)</div><div class='del'>-#define HNS_ROCE_WQE_FENCE		(1UL&lt;&lt;21)</div><div class='del'>-#define HNS_ROCE_WQE_CQ_NOTIFY		(1UL&lt;&lt;20)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_WQE_OPCODE_SEND	(0&lt;&lt;16)</div><div class='del'>-#define HNS_ROCE_WQE_OPCODE_RDMA_READ	(1&lt;&lt;16)</div><div class='del'>-#define HNS_ROCE_WQE_OPCODE_RDMA_WRITE	(2&lt;&lt;16)</div><div class='del'>-#define HNS_ROCE_WQE_OPCODE_LOCAL_INV	(4&lt;&lt;16)</div><div class='del'>-#define HNS_ROCE_WQE_OPCODE_UD_SEND	(7&lt;&lt;16)</div><div class='del'>-#define HNS_ROCE_WQE_OPCODE_MASK	(15&lt;&lt;16)</div><div class='del'>-</div><div class='del'>-struct hns_roce_qp_context {</div><div class='del'>-	__le32 qpc_bytes_4;</div><div class='del'>-	__le32 qpc_bytes_8;</div><div class='del'>-	__le32 qpc_bytes_12;</div><div class='del'>-	__le32 qpc_bytes_16;</div><div class='del'>-	__le32 sq_rq_bt_l;</div><div class='del'>-	__le32 qpc_bytes_24;</div><div class='del'>-	__le32 irrl_ba_l;</div><div class='del'>-	__le32 qpc_bytes_32;</div><div class='del'>-	__le32 qpc_bytes_36;</div><div class='del'>-	__le32 dmac_l;</div><div class='del'>-	__le32 qpc_bytes_44;</div><div class='del'>-	__le32 qpc_bytes_48;</div><div class='del'>-	u8     dgid[16];</div><div class='del'>-	__le32 qpc_bytes_68;</div><div class='del'>-	__le32 cur_rq_wqe_ba_l;</div><div class='del'>-	__le32 qpc_bytes_76;</div><div class='del'>-	__le32 rx_rnr_time;</div><div class='del'>-	__le32 qpc_bytes_84;</div><div class='del'>-	__le32 qpc_bytes_88;</div><div class='del'>-	union {</div><div class='del'>-		__le32 rx_sge_len;</div><div class='del'>-		__le32 dma_length;</div><div class='del'>-	};</div><div class='del'>-	union {</div><div class='del'>-		__le32 rx_sge_num;</div><div class='del'>-		__le32 rx_send_pktn;</div><div class='del'>-		__le32 r_key;</div><div class='del'>-	};</div><div class='del'>-	__le32 va_l;</div><div class='del'>-	__le32 va_h;</div><div class='del'>-	__le32 qpc_bytes_108;</div><div class='del'>-	__le32 qpc_bytes_112;</div><div class='del'>-	__le32 rx_cur_sq_wqe_ba_l;</div><div class='del'>-	__le32 qpc_bytes_120;</div><div class='del'>-	__le32 qpc_bytes_124;</div><div class='del'>-	__le32 qpc_bytes_128;</div><div class='del'>-	__le32 qpc_bytes_132;</div><div class='del'>-	__le32 qpc_bytes_136;</div><div class='del'>-	__le32 qpc_bytes_140;</div><div class='del'>-	__le32 qpc_bytes_144;</div><div class='del'>-	__le32 qpc_bytes_148;</div><div class='del'>-	union {</div><div class='del'>-		__le32 rnr_retry;</div><div class='del'>-		__le32 ack_time;</div><div class='del'>-	};</div><div class='del'>-	__le32 qpc_bytes_156;</div><div class='del'>-	__le32 pkt_use_len;</div><div class='del'>-	__le32 qpc_bytes_164;</div><div class='del'>-	__le32 qpc_bytes_168;</div><div class='del'>-	union {</div><div class='del'>-		__le32 sge_use_len;</div><div class='del'>-		__le32 pa_use_len;</div><div class='del'>-	};</div><div class='del'>-	__le32 qpc_bytes_176;</div><div class='del'>-	__le32 qpc_bytes_180;</div><div class='del'>-	__le32 tx_cur_sq_wqe_ba_l;</div><div class='del'>-	__le32 qpc_bytes_188;</div><div class='del'>-	__le32 rvd21;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M   \</div><div class='del'>-	(((1UL &lt;&lt; 3) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S 3</div><div class='del'>-#define QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S 4</div><div class='del'>-#define QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S 5</div><div class='del'>-#define QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S 6</div><div class='del'>-#define QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S 7</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S 8</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 4) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S 12</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 4) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_4_PD_S 16</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_4_PD_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_4_PD_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S 16</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S 16</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_16_QP_NUM_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_16_QP_NUM_M   \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_16_QP_NUM_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 17) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S 18</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S 23</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 17) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S 18</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M   \</div><div class='del'>-	(((1UL &lt;&lt; 2) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S 20</div><div class='del'>-#define QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S 21</div><div class='del'>-#define QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S 22</div><div class='del'>-#define QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S 23</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S 24</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_36_DEST_QP_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_36_DEST_QP_M   \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_36_DEST_QP_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S 24</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_44_DMAC_H_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_44_DMAC_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_44_DMAC_H_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S 16</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_44_HOPLMT_S 24</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_44_HOPLMT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_44_HOPLMT_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M   \</div><div class='del'>-	(((1UL &lt;&lt; 20) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_48_TCLASS_S 20</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_48_TCLASS_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_48_TCLASS_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_48_MTU_S 28</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_48_MTU_M   \</div><div class='del'>-	(((1UL &lt;&lt; 4) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_48_MTU_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S 16</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S 8</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S 24</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S 24</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S 25</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S 26</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M   \</div><div class='del'>-	(((1UL &lt;&lt; 2) - 1) &lt;&lt; \</div><div class='del'>-	QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S 29</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M   \</div><div class='del'>-	(((1UL &lt;&lt; 2) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S 24</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S 25</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S 24</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S 16</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S 24</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S 25</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M   \</div><div class='del'>-	(((1UL &lt;&lt; 2) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S 27</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S 24</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S 24</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S 16</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S 31</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_144_QP_STATE_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_144_QP_STATE_M   \</div><div class='del'>-	(((1UL &lt;&lt; 3) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_144_QP_STATE_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M   \</div><div class='del'>-	(((1UL &lt;&lt; 2) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S 2</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 3) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S 5</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 3) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_148_LSN_S 8</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_148_LSN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 16) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_148_LSN_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 3) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S 3</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S 8</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M   \</div><div class='del'>-	(((1UL &lt;&lt; 3) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S 11</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M   \</div><div class='del'>-	(((1UL &lt;&lt; 3) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_156_SL_S 14</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_156_SL_M   \</div><div class='del'>-	(((1UL &lt;&lt; 2) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_156_SL_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S 16</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S 24</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M   \</div><div class='del'>-	(((1UL &lt;&lt; 2) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S 24</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M   \</div><div class='del'>-	(((1UL &lt;&lt; 8) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M   \</div><div class='del'>-	(((1UL &lt;&lt; 24) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S 24</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M   \</div><div class='del'>-	(((1UL &lt;&lt; 2) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S 26</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M   \</div><div class='del'>-	(((1UL &lt;&lt; 2) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S 28</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S 29</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S 30</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S 16</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S 16</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S 0</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M   \</div><div class='del'>-	(((1UL &lt;&lt; 5) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S)</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S 8</div><div class='del'>-</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S 16</div><div class='del'>-#define QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S)</div><div class='del'>-</div><div class='del'>-#define STATUS_MASK		0xff</div><div class='del'>-#define GO_BIT_TIMEOUT_MSECS	10000</div><div class='del'>-#define HCR_STATUS_OFFSET	0x18</div><div class='del'>-#define HCR_GO_BIT		15</div><div class='del'>-</div><div class='del'>-struct hns_roce_rq_db {</div><div class='del'>-	__le32    u32_4;</div><div class='del'>-	__le32    u32_8;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define RQ_DOORBELL_U32_4_RQ_HEAD_S 0</div><div class='del'>-#define RQ_DOORBELL_U32_4_RQ_HEAD_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; RQ_DOORBELL_U32_4_RQ_HEAD_S)</div><div class='del'>-</div><div class='del'>-#define RQ_DOORBELL_U32_8_QPN_S 0</div><div class='del'>-#define RQ_DOORBELL_U32_8_QPN_M   (((1UL &lt;&lt; 24) - 1) &lt;&lt; RQ_DOORBELL_U32_8_QPN_S)</div><div class='del'>-</div><div class='del'>-#define RQ_DOORBELL_U32_8_CMD_S 28</div><div class='del'>-#define RQ_DOORBELL_U32_8_CMD_M   (((1UL &lt;&lt; 3) - 1) &lt;&lt; RQ_DOORBELL_U32_8_CMD_S)</div><div class='del'>-</div><div class='del'>-#define RQ_DOORBELL_U32_8_HW_SYNC_S 31</div><div class='del'>-</div><div class='del'>-struct hns_roce_sq_db {</div><div class='del'>-	__le32    u32_4;</div><div class='del'>-	__le32    u32_8;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define SQ_DOORBELL_U32_4_SQ_HEAD_S 0</div><div class='del'>-#define SQ_DOORBELL_U32_4_SQ_HEAD_M   \</div><div class='del'>-	(((1UL &lt;&lt; 15) - 1) &lt;&lt; SQ_DOORBELL_U32_4_SQ_HEAD_S)</div><div class='del'>-</div><div class='del'>-#define SQ_DOORBELL_U32_4_SL_S 16</div><div class='del'>-#define SQ_DOORBELL_U32_4_SL_M   \</div><div class='del'>-	(((1UL &lt;&lt; 2) - 1) &lt;&lt; SQ_DOORBELL_U32_4_SL_S)</div><div class='del'>-</div><div class='del'>-#define SQ_DOORBELL_U32_4_PORT_S 18</div><div class='del'>-#define SQ_DOORBELL_U32_4_PORT_M  (((1UL &lt;&lt; 3) - 1) &lt;&lt; SQ_DOORBELL_U32_4_PORT_S)</div><div class='del'>-</div><div class='del'>-#define SQ_DOORBELL_U32_8_QPN_S 0</div><div class='del'>-#define SQ_DOORBELL_U32_8_QPN_M   (((1UL &lt;&lt; 24) - 1) &lt;&lt; SQ_DOORBELL_U32_8_QPN_S)</div><div class='del'>-</div><div class='del'>-#define SQ_DOORBELL_HW_SYNC_S 31</div><div class='del'>-</div><div class='del'>-struct hns_roce_ext_db {</div><div class='del'>-	int esdb_dep;</div><div class='del'>-	int eodb_dep;</div><div class='del'>-	struct hns_roce_buf_list *sdb_buf_list;</div><div class='del'>-	struct hns_roce_buf_list *odb_buf_list;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct hns_roce_db_table {</div><div class='del'>-	int  sdb_ext_mod;</div><div class='del'>-	int  odb_ext_mod;</div><div class='del'>-	struct hns_roce_ext_db *ext_db;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct hns_roce_bt_table {</div><div class='del'>-	struct hns_roce_buf_list qpc_buf;</div><div class='del'>-	struct hns_roce_buf_list mtpt_buf;</div><div class='del'>-	struct hns_roce_buf_list cqc_buf;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct hns_roce_tptr_table {</div><div class='del'>-	struct hns_roce_buf_list tptr_buf;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct hns_roce_qp_work {</div><div class='del'>-	struct	work_struct work;</div><div class='del'>-	struct	ib_device *ib_dev;</div><div class='del'>-	struct	hns_roce_qp *qp;</div><div class='del'>-	u32	db_wait_stage;</div><div class='del'>-	u32	sdb_issue_ptr;</div><div class='del'>-	u32	sdb_inv_cnt;</div><div class='del'>-	u32	sche_cnt;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct hns_roce_mr_free_work {</div><div class='del'>-	struct	work_struct work;</div><div class='del'>-	struct	ib_device *ib_dev;</div><div class='del'>-	struct	completion *comp;</div><div class='del'>-	int	comp_flag;</div><div class='del'>-	void	*mr;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct hns_roce_recreate_lp_qp_work {</div><div class='del'>-	struct	work_struct work;</div><div class='del'>-	struct	ib_device *ib_dev;</div><div class='del'>-	struct	completion *comp;</div><div class='del'>-	int	comp_flag;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct hns_roce_free_mr {</div><div class='del'>-	struct workqueue_struct *free_mr_wq;</div><div class='del'>-	struct hns_roce_qp *mr_free_qp[HNS_ROCE_V1_RESV_QP];</div><div class='del'>-	struct hns_roce_cq *mr_free_cq;</div><div class='del'>-	struct hns_roce_pd *mr_free_pd;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct hns_roce_v1_priv {</div><div class='del'>-	struct hns_roce_db_table  db_table;</div><div class='del'>-	struct hns_roce_raq_table raq_table;</div><div class='del'>-	struct hns_roce_bt_table  bt_table;</div><div class='del'>-	struct hns_roce_tptr_table tptr_table;</div><div class='del'>-	struct hns_roce_free_mr free_mr;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-int hns_dsaf_roce_reset(struct fwnode_handle *dsaf_fwnode, bool dereset);</div><div class='del'>-int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);</div><div class='del'>-int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);</div><div class='del'>-</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c<br/>index b5392cb5b20f..2d6ae89e525b 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_hw_v2.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_hw_v2.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_hw_v2.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_hw_v2.c</a></div><div class='hunk'>@@ -33,244 +33,688 @@</div><div class='ctx'> #include &lt;linux/acpi.h&gt;</div><div class='ctx'> #include &lt;linux/etherdevice.h&gt;</div><div class='ctx'> #include &lt;linux/interrupt.h&gt;</div><div class='add'>+#include &lt;linux/iopoll.h&gt;</div><div class='ctx'> #include &lt;linux/kernel.h&gt;</div><div class='ctx'> #include &lt;linux/types.h&gt;</div><div class='add'>+#include &lt;linux/workqueue.h&gt;</div><div class='ctx'> #include &lt;net/addrconf.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_addr.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_cache.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_umem.h&gt;</div><div class='ctx'> #include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='ctx'> </div><div class='del'>-#include "hnae3.h"</div><div class='add'>+#include "hclge_main.h"</div><div class='ctx'> #include "hns_roce_common.h"</div><div class='ctx'> #include "hns_roce_device.h"</div><div class='ctx'> #include "hns_roce_cmd.h"</div><div class='ctx'> #include "hns_roce_hem.h"</div><div class='ctx'> #include "hns_roce_hw_v2.h"</div><div class='add'>+#include "hns_roce_bond.h"</div><div class='ctx'> </div><div class='del'>-static void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,</div><div class='del'>-			    struct ib_sge *sg)</div><div class='add'>+#define CREATE_TRACE_POINTS</div><div class='add'>+#include "hns_roce_trace.h"</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	CMD_RST_PRC_OTHERS,</div><div class='add'>+	CMD_RST_PRC_SUCCESS,</div><div class='add'>+	CMD_RST_PRC_EBUSY,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum ecc_resource_type {</div><div class='add'>+	ECC_RESOURCE_QPC,</div><div class='add'>+	ECC_RESOURCE_CQC,</div><div class='add'>+	ECC_RESOURCE_MPT,</div><div class='add'>+	ECC_RESOURCE_SRQC,</div><div class='add'>+	ECC_RESOURCE_GMV,</div><div class='add'>+	ECC_RESOURCE_QPC_TIMER,</div><div class='add'>+	ECC_RESOURCE_CQC_TIMER,</div><div class='add'>+	ECC_RESOURCE_SCCC,</div><div class='add'>+	ECC_RESOURCE_COUNT,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct {</div><div class='add'>+	const char *name;</div><div class='add'>+	u8 read_bt0_op;</div><div class='add'>+	u8 write_bt0_op;</div><div class='add'>+} fmea_ram_res[] = {</div><div class='add'>+	{ "ECC_RESOURCE_QPC",</div><div class='add'>+	  HNS_ROCE_CMD_READ_QPC_BT0, HNS_ROCE_CMD_WRITE_QPC_BT0 },</div><div class='add'>+	{ "ECC_RESOURCE_CQC",</div><div class='add'>+	  HNS_ROCE_CMD_READ_CQC_BT0, HNS_ROCE_CMD_WRITE_CQC_BT0 },</div><div class='add'>+	{ "ECC_RESOURCE_MPT",</div><div class='add'>+	  HNS_ROCE_CMD_READ_MPT_BT0, HNS_ROCE_CMD_WRITE_MPT_BT0 },</div><div class='add'>+	{ "ECC_RESOURCE_SRQC",</div><div class='add'>+	  HNS_ROCE_CMD_READ_SRQC_BT0, HNS_ROCE_CMD_WRITE_SRQC_BT0 },</div><div class='add'>+	/* ECC_RESOURCE_GMV is handled by cmdq, not mailbox */</div><div class='add'>+	{ "ECC_RESOURCE_GMV",</div><div class='add'>+	  0, 0 },</div><div class='add'>+	{ "ECC_RESOURCE_QPC_TIMER",</div><div class='add'>+	  HNS_ROCE_CMD_READ_QPC_TIMER_BT0, HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0 },</div><div class='add'>+	{ "ECC_RESOURCE_CQC_TIMER",</div><div class='add'>+	  HNS_ROCE_CMD_READ_CQC_TIMER_BT0, HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0 },</div><div class='add'>+	{ "ECC_RESOURCE_SCCC",</div><div class='add'>+	  HNS_ROCE_CMD_READ_SCCC_BT0, HNS_ROCE_CMD_WRITE_SCCC_BT0 },</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,</div><div class='add'>+				   struct ib_sge *sg)</div><div class='ctx'> {</div><div class='ctx'> 	dseg-&gt;lkey = cpu_to_le32(sg-&gt;lkey);</div><div class='ctx'> 	dseg-&gt;addr = cpu_to_le64(sg-&gt;addr);</div><div class='ctx'> 	dseg-&gt;len  = cpu_to_le32(sg-&gt;length);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+/*</div><div class='add'>+ * mapped-value = 1 + real-value</div><div class='add'>+ * The hns wr opcode real value is start from 0, In order to distinguish between</div><div class='add'>+ * initialized and uninitialized map values, we plus 1 to the actual value when</div><div class='add'>+ * defining the mapping, so that the validity can be identified by checking the</div><div class='add'>+ * mapped value is greater than 0.</div><div class='add'>+ */</div><div class='add'>+#define HR_OPC_MAP(ib_key, hr_key) \</div><div class='add'>+		[IB_WR_ ## ib_key] = 1 + HNS_ROCE_V2_WQE_OP_ ## hr_key</div><div class='add'>+</div><div class='add'>+static const u32 hns_roce_op_code[] = {</div><div class='add'>+	HR_OPC_MAP(RDMA_WRITE,			RDMA_WRITE),</div><div class='add'>+	HR_OPC_MAP(RDMA_WRITE_WITH_IMM,		RDMA_WRITE_WITH_IMM),</div><div class='add'>+	HR_OPC_MAP(SEND,			SEND),</div><div class='add'>+	HR_OPC_MAP(SEND_WITH_IMM,		SEND_WITH_IMM),</div><div class='add'>+	HR_OPC_MAP(RDMA_READ,			RDMA_READ),</div><div class='add'>+	HR_OPC_MAP(ATOMIC_CMP_AND_SWP,		ATOM_CMP_AND_SWAP),</div><div class='add'>+	HR_OPC_MAP(ATOMIC_FETCH_AND_ADD,	ATOM_FETCH_AND_ADD),</div><div class='add'>+	HR_OPC_MAP(SEND_WITH_INV,		SEND_WITH_INV),</div><div class='add'>+	HR_OPC_MAP(MASKED_ATOMIC_CMP_AND_SWP,	ATOM_MSK_CMP_AND_SWAP),</div><div class='add'>+	HR_OPC_MAP(MASKED_ATOMIC_FETCH_AND_ADD,	ATOM_MSK_FETCH_AND_ADD),</div><div class='add'>+	HR_OPC_MAP(REG_MR,			FAST_REG_PMR),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static u32 to_hr_opcode(u32 ib_opcode)</div><div class='add'>+{</div><div class='add'>+	if (ib_opcode &gt;= ARRAY_SIZE(hns_roce_op_code))</div><div class='add'>+		return HNS_ROCE_V2_WQE_OP_MASK;</div><div class='add'>+</div><div class='add'>+	return hns_roce_op_code[ib_opcode] ? hns_roce_op_code[ib_opcode] - 1 :</div><div class='add'>+					     HNS_ROCE_V2_WQE_OP_MASK;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,</div><div class='del'>-			 struct hns_roce_wqe_frmr_seg *fseg,</div><div class='ctx'> 			 const struct ib_reg_wr *wr)</div><div class='ctx'> {</div><div class='add'>+	struct hns_roce_wqe_frmr_seg *fseg =</div><div class='add'>+		(void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);</div><div class='ctx'> 	struct hns_roce_mr *mr = to_hr_mr(wr-&gt;mr);</div><div class='add'>+	u64 pbl_ba;</div><div class='ctx'> </div><div class='ctx'> 	/* use ib_access_flags */</div><div class='del'>-	roce_set_bit(rc_sq_wqe-&gt;byte_4,</div><div class='del'>-		     V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S,</div><div class='del'>-		     wr-&gt;access &amp; IB_ACCESS_MW_BIND ? 1 : 0);</div><div class='del'>-	roce_set_bit(rc_sq_wqe-&gt;byte_4,</div><div class='del'>-		     V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S,</div><div class='del'>-		     wr-&gt;access &amp; IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);</div><div class='del'>-	roce_set_bit(rc_sq_wqe-&gt;byte_4,</div><div class='del'>-		     V2_RC_FRMR_WQE_BYTE_4_RR_S,</div><div class='del'>-		     wr-&gt;access &amp; IB_ACCESS_REMOTE_READ ? 1 : 0);</div><div class='del'>-	roce_set_bit(rc_sq_wqe-&gt;byte_4,</div><div class='del'>-		     V2_RC_FRMR_WQE_BYTE_4_RW_S,</div><div class='del'>-		     wr-&gt;access &amp; IB_ACCESS_REMOTE_WRITE ? 1 : 0);</div><div class='del'>-	roce_set_bit(rc_sq_wqe-&gt;byte_4,</div><div class='del'>-		     V2_RC_FRMR_WQE_BYTE_4_LW_S,</div><div class='del'>-		     wr-&gt;access &amp; IB_ACCESS_LOCAL_WRITE ? 1 : 0);</div><div class='add'>+	hr_reg_write_bool(fseg, FRMR_BIND_EN, 0);</div><div class='add'>+	hr_reg_write_bool(fseg, FRMR_ATOMIC,</div><div class='add'>+			  wr-&gt;access &amp; IB_ACCESS_REMOTE_ATOMIC);</div><div class='add'>+	hr_reg_write_bool(fseg, FRMR_RR, wr-&gt;access &amp; IB_ACCESS_REMOTE_READ);</div><div class='add'>+	hr_reg_write_bool(fseg, FRMR_RW, wr-&gt;access &amp; IB_ACCESS_REMOTE_WRITE);</div><div class='add'>+	hr_reg_write_bool(fseg, FRMR_LW, wr-&gt;access &amp; IB_ACCESS_LOCAL_WRITE);</div><div class='ctx'> </div><div class='ctx'> 	/* Data structure reuse may lead to confusion */</div><div class='del'>-	rc_sq_wqe-&gt;msg_len = cpu_to_le32(mr-&gt;pbl_ba &amp; 0xffffffff);</div><div class='del'>-	rc_sq_wqe-&gt;inv_key = cpu_to_le32(mr-&gt;pbl_ba &gt;&gt; 32);</div><div class='add'>+	pbl_ba = mr-&gt;pbl_mtr.hem_cfg.root_ba;</div><div class='add'>+	rc_sq_wqe-&gt;msg_len = cpu_to_le32(lower_32_bits(pbl_ba));</div><div class='add'>+	rc_sq_wqe-&gt;inv_key = cpu_to_le32(upper_32_bits(pbl_ba));</div><div class='ctx'> </div><div class='ctx'> 	rc_sq_wqe-&gt;byte_16 = cpu_to_le32(wr-&gt;mr-&gt;length &amp; 0xffffffff);</div><div class='ctx'> 	rc_sq_wqe-&gt;byte_20 = cpu_to_le32(wr-&gt;mr-&gt;length &gt;&gt; 32);</div><div class='ctx'> 	rc_sq_wqe-&gt;rkey = cpu_to_le32(wr-&gt;key);</div><div class='ctx'> 	rc_sq_wqe-&gt;va = cpu_to_le64(wr-&gt;mr-&gt;iova);</div><div class='ctx'> </div><div class='del'>-	fseg-&gt;pbl_size = cpu_to_le32(mr-&gt;pbl_size);</div><div class='del'>-	roce_set_field(fseg-&gt;mode_buf_pg_sz,</div><div class='del'>-		       V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M,</div><div class='del'>-		       V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S,</div><div class='del'>-		       mr-&gt;pbl_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_bit(fseg-&gt;mode_buf_pg_sz,</div><div class='del'>-		     V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0);</div><div class='add'>+	hr_reg_write(fseg, FRMR_PBL_SIZE, mr-&gt;npages);</div><div class='add'>+	hr_reg_write(fseg, FRMR_PBL_BUF_PG_SZ,</div><div class='add'>+		     to_hr_hw_page_shift(mr-&gt;pbl_mtr.hem_cfg.buf_pg_shift));</div><div class='add'>+	hr_reg_clear(fseg, FRMR_BLK_MODE);</div><div class='add'>+	hr_reg_clear(fseg, FRMR_BLOCK_SIZE);</div><div class='add'>+	hr_reg_clear(fseg, FRMR_ZBVA);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void set_atomic_seg(struct hns_roce_wqe_atomic_seg *aseg,</div><div class='del'>-			   const struct ib_atomic_wr *wr)</div><div class='add'>+static void set_atomic_seg(const struct ib_send_wr *wr,</div><div class='add'>+			   struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,</div><div class='add'>+			   unsigned int valid_num_sge)</div><div class='ctx'> {</div><div class='del'>-	if (wr-&gt;wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {</div><div class='del'>-		aseg-&gt;fetchadd_swap_data = cpu_to_le64(wr-&gt;swap);</div><div class='del'>-		aseg-&gt;cmp_data  = cpu_to_le64(wr-&gt;compare_add);</div><div class='add'>+	struct hns_roce_v2_wqe_data_seg *dseg =</div><div class='add'>+		(void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);</div><div class='add'>+	struct hns_roce_wqe_atomic_seg *aseg =</div><div class='add'>+		(void *)dseg + sizeof(struct hns_roce_v2_wqe_data_seg);</div><div class='add'>+</div><div class='add'>+	set_data_seg_v2(dseg, wr-&gt;sg_list);</div><div class='add'>+</div><div class='add'>+	if (wr-&gt;opcode == IB_WR_ATOMIC_CMP_AND_SWP) {</div><div class='add'>+		aseg-&gt;fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)-&gt;swap);</div><div class='add'>+		aseg-&gt;cmp_data = cpu_to_le64(atomic_wr(wr)-&gt;compare_add);</div><div class='ctx'> 	} else {</div><div class='del'>-		aseg-&gt;fetchadd_swap_data = cpu_to_le64(wr-&gt;compare_add);</div><div class='del'>-		aseg-&gt;cmp_data  = 0;</div><div class='add'>+		aseg-&gt;fetchadd_swap_data =</div><div class='add'>+			cpu_to_le64(atomic_wr(wr)-&gt;compare_add);</div><div class='add'>+		aseg-&gt;cmp_data = 0;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, valid_num_sge);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr,</div><div class='del'>-			   unsigned int *sge_ind)</div><div class='add'>+static int fill_ext_sge_inl_data(struct hns_roce_qp *qp,</div><div class='add'>+				 const struct ib_send_wr *wr,</div><div class='add'>+				 unsigned int *sge_idx, u32 msg_len)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_v2_wqe_data_seg *dseg;</div><div class='del'>-	struct ib_sge *sg;</div><div class='del'>-	int num_in_wqe = 0;</div><div class='del'>-	int extend_sge_num;</div><div class='del'>-	int fi_sge_num;</div><div class='del'>-	int se_sge_num;</div><div class='del'>-	int shift;</div><div class='del'>-	int i;</div><div class='add'>+	struct ib_device *ibdev = &amp;(to_hr_dev(qp-&gt;ibqp.device))-&gt;ib_dev;</div><div class='add'>+	unsigned int left_len_in_pg;</div><div class='add'>+	unsigned int idx = *sge_idx;</div><div class='add'>+	unsigned int i = 0;</div><div class='add'>+	unsigned int len;</div><div class='add'>+	void *addr;</div><div class='add'>+	void *dseg;</div><div class='add'>+</div><div class='add'>+	if (msg_len &gt; qp-&gt;sq.ext_sge_cnt * HNS_ROCE_SGE_SIZE) {</div><div class='add'>+		ibdev_err(ibdev,</div><div class='add'>+			  "no enough extended sge space for inline data.\n");</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;ibqp.qp_type == IB_QPT_RC || qp-&gt;ibqp.qp_type == IB_QPT_UC)</div><div class='del'>-		num_in_wqe = HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE;</div><div class='del'>-	extend_sge_num = wr-&gt;num_sge - num_in_wqe;</div><div class='del'>-	sg = wr-&gt;sg_list + num_in_wqe;</div><div class='del'>-	shift = qp-&gt;hr_buf.page_shift;</div><div class='add'>+	dseg = hns_roce_get_extend_sge(qp, idx &amp; (qp-&gt;sge.sge_cnt - 1));</div><div class='add'>+	left_len_in_pg = hr_hw_page_align((uintptr_t)dseg) - (uintptr_t)dseg;</div><div class='add'>+	len = wr-&gt;sg_list[0].length;</div><div class='add'>+	addr = (void *)(unsigned long)(wr-&gt;sg_list[0].addr);</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * Check whether wr-&gt;num_sge sges are in the same page. If not, we</div><div class='del'>-	 * should calculate how many sges in the first page and the second</div><div class='del'>-	 * page.</div><div class='add'>+	/* When copying data to extended sge space, the left length in page may</div><div class='add'>+	 * not long enough for current user's sge. So the data should be</div><div class='add'>+	 * splited into several parts, one in the first page, and the others in</div><div class='add'>+	 * the subsequent pages.</div><div class='ctx'> 	 */</div><div class='del'>-	dseg = get_send_extend_sge(qp, (*sge_ind) &amp; (qp-&gt;sge.sge_cnt - 1));</div><div class='del'>-	fi_sge_num = (round_up((uintptr_t)dseg, 1 &lt;&lt; shift) -</div><div class='del'>-		      (uintptr_t)dseg) /</div><div class='del'>-		      sizeof(struct hns_roce_v2_wqe_data_seg);</div><div class='del'>-	if (extend_sge_num &gt; fi_sge_num) {</div><div class='del'>-		se_sge_num = extend_sge_num - fi_sge_num;</div><div class='del'>-		for (i = 0; i &lt; fi_sge_num; i++) {</div><div class='del'>-			set_data_seg_v2(dseg++, sg + i);</div><div class='del'>-			(*sge_ind)++;</div><div class='add'>+	while (1) {</div><div class='add'>+		if (len &lt;= left_len_in_pg) {</div><div class='add'>+			memcpy(dseg, addr, len);</div><div class='add'>+</div><div class='add'>+			idx += len / HNS_ROCE_SGE_SIZE;</div><div class='add'>+</div><div class='add'>+			i++;</div><div class='add'>+			if (i &gt;= wr-&gt;num_sge)</div><div class='add'>+				break;</div><div class='add'>+</div><div class='add'>+			left_len_in_pg -= len;</div><div class='add'>+			len = wr-&gt;sg_list[i].length;</div><div class='add'>+			addr = (void *)(unsigned long)(wr-&gt;sg_list[i].addr);</div><div class='add'>+			dseg += len;</div><div class='add'>+		} else {</div><div class='add'>+			memcpy(dseg, addr, left_len_in_pg);</div><div class='add'>+</div><div class='add'>+			len -= left_len_in_pg;</div><div class='add'>+			addr += left_len_in_pg;</div><div class='add'>+			idx += left_len_in_pg / HNS_ROCE_SGE_SIZE;</div><div class='add'>+			dseg = hns_roce_get_extend_sge(qp,</div><div class='add'>+						idx &amp; (qp-&gt;sge.sge_cnt - 1));</div><div class='add'>+			left_len_in_pg = 1 &lt;&lt; HNS_HW_PAGE_SHIFT;</div><div class='ctx'> 		}</div><div class='del'>-		dseg = get_send_extend_sge(qp,</div><div class='del'>-					   (*sge_ind) &amp; (qp-&gt;sge.sge_cnt - 1));</div><div class='del'>-		for (i = 0; i &lt; se_sge_num; i++) {</div><div class='del'>-			set_data_seg_v2(dseg++, sg + fi_sge_num + i);</div><div class='del'>-			(*sge_ind)++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*sge_idx = idx;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void set_extend_sge(struct hns_roce_qp *qp, struct ib_sge *sge,</div><div class='add'>+			   unsigned int *sge_ind, unsigned int cnt)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_v2_wqe_data_seg *dseg;</div><div class='add'>+	unsigned int idx = *sge_ind;</div><div class='add'>+</div><div class='add'>+	while (cnt &gt; 0) {</div><div class='add'>+		dseg = hns_roce_get_extend_sge(qp, idx &amp; (qp-&gt;sge.sge_cnt - 1));</div><div class='add'>+		if (likely(sge-&gt;length)) {</div><div class='add'>+			set_data_seg_v2(dseg, sge);</div><div class='add'>+			idx++;</div><div class='add'>+			cnt--;</div><div class='ctx'> 		}</div><div class='del'>-	} else {</div><div class='del'>-		for (i = 0; i &lt; extend_sge_num; i++) {</div><div class='del'>-			set_data_seg_v2(dseg++, sg + i);</div><div class='del'>-			(*sge_ind)++;</div><div class='add'>+		sge++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*sge_ind = idx;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool check_inl_data_len(struct hns_roce_qp *qp, unsigned int len)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(qp-&gt;ibqp.device);</div><div class='add'>+	int mtu = ib_mtu_enum_to_int(qp-&gt;path_mtu);</div><div class='add'>+</div><div class='add'>+	if (mtu &lt; 0 || len &gt; qp-&gt;max_inline_data || len &gt; mtu) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+			  "invalid length of data, data len = %u, max inline len = %u, path mtu = %d.\n",</div><div class='add'>+			  len, qp-&gt;max_inline_data, mtu);</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int set_rc_inl(struct hns_roce_qp *qp, const struct ib_send_wr *wr,</div><div class='add'>+		      struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,</div><div class='add'>+		      unsigned int *sge_idx)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(qp-&gt;ibqp.device);</div><div class='add'>+	u32 msg_len = le32_to_cpu(rc_sq_wqe-&gt;msg_len);</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	unsigned int curr_idx = *sge_idx;</div><div class='add'>+	void *dseg = rc_sq_wqe;</div><div class='add'>+	unsigned int i;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (unlikely(wr-&gt;opcode == IB_WR_RDMA_READ)) {</div><div class='add'>+		ibdev_err(ibdev, "invalid inline parameters!\n");</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!check_inl_data_len(qp, msg_len))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	dseg += sizeof(struct hns_roce_v2_rc_send_wqe);</div><div class='add'>+</div><div class='add'>+	if (msg_len &lt;= HNS_ROCE_V2_MAX_RC_INL_INN_SZ) {</div><div class='add'>+		hr_reg_clear(rc_sq_wqe, RC_SEND_WQE_INL_TYPE);</div><div class='add'>+</div><div class='add'>+		for (i = 0; i &lt; wr-&gt;num_sge; i++) {</div><div class='add'>+			memcpy(dseg, ((void *)wr-&gt;sg_list[i].addr),</div><div class='add'>+			       wr-&gt;sg_list[i].length);</div><div class='add'>+			dseg += wr-&gt;sg_list[i].length;</div><div class='ctx'> 		}</div><div class='add'>+	} else {</div><div class='add'>+		hr_reg_enable(rc_sq_wqe, RC_SEND_WQE_INL_TYPE);</div><div class='add'>+</div><div class='add'>+		ret = fill_ext_sge_inl_data(qp, wr, &amp;curr_idx, msg_len);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+</div><div class='add'>+		hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, curr_idx - *sge_idx);</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	*sge_idx = curr_idx;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,</div><div class='ctx'> 			     struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,</div><div class='del'>-			     void *wqe, unsigned int *sge_ind,</div><div class='del'>-			     const struct ib_send_wr **bad_wr)</div><div class='add'>+			     unsigned int *sge_ind,</div><div class='add'>+			     unsigned int valid_num_sge)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='del'>-	struct hns_roce_v2_wqe_data_seg *dseg = wqe;</div><div class='add'>+	struct hns_roce_v2_wqe_data_seg *dseg =</div><div class='add'>+		(void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);</div><div class='ctx'> 	struct hns_roce_qp *qp = to_hr_qp(ibqp);</div><div class='add'>+	int j = 0;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	if (wr-&gt;send_flags &amp; IB_SEND_INLINE &amp;&amp; wr-&gt;num_sge) {</div><div class='del'>-		if (le32_to_cpu(rc_sq_wqe-&gt;msg_len) &gt;</div><div class='del'>-		    hr_dev-&gt;caps.max_sq_inline) {</div><div class='del'>-			*bad_wr = wr;</div><div class='del'>-			dev_err(hr_dev-&gt;dev, "inline len(1-%d)=%d, illegal",</div><div class='del'>-				rc_sq_wqe-&gt;msg_len, hr_dev-&gt;caps.max_sq_inline);</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (wr-&gt;opcode == IB_WR_RDMA_READ) {</div><div class='del'>-			*bad_wr =  wr;</div><div class='del'>-			dev_err(hr_dev-&gt;dev, "Not support inline data!\n");</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='add'>+	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_INLINE,</div><div class='add'>+		     !!(wr-&gt;send_flags &amp; IB_SEND_INLINE));</div><div class='add'>+	if (wr-&gt;send_flags &amp; IB_SEND_INLINE)</div><div class='add'>+		return set_rc_inl(qp, wr, rc_sq_wqe, sge_ind);</div><div class='ctx'> </div><div class='add'>+	if (valid_num_sge &lt;= HNS_ROCE_SGE_IN_WQE) {</div><div class='ctx'> 		for (i = 0; i &lt; wr-&gt;num_sge; i++) {</div><div class='del'>-			memcpy(wqe, ((void *)wr-&gt;sg_list[i].addr),</div><div class='del'>-			       wr-&gt;sg_list[i].length);</div><div class='del'>-			wqe += wr-&gt;sg_list[i].length;</div><div class='add'>+			if (likely(wr-&gt;sg_list[i].length)) {</div><div class='add'>+				set_data_seg_v2(dseg, wr-&gt;sg_list + i);</div><div class='add'>+				dseg++;</div><div class='add'>+			}</div><div class='ctx'> 		}</div><div class='del'>-</div><div class='del'>-		roce_set_bit(rc_sq_wqe-&gt;byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S,</div><div class='del'>-			     1);</div><div class='ctx'> 	} else {</div><div class='del'>-		if (wr-&gt;num_sge &lt;= HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) {</div><div class='del'>-			for (i = 0; i &lt; wr-&gt;num_sge; i++) {</div><div class='del'>-				if (likely(wr-&gt;sg_list[i].length)) {</div><div class='del'>-					set_data_seg_v2(dseg, wr-&gt;sg_list + i);</div><div class='del'>-					dseg++;</div><div class='del'>-				}</div><div class='del'>-			}</div><div class='del'>-		} else {</div><div class='del'>-			roce_set_field(rc_sq_wqe-&gt;byte_20,</div><div class='del'>-				     V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,</div><div class='del'>-				     V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,</div><div class='del'>-				     (*sge_ind) &amp; (qp-&gt;sge.sge_cnt - 1));</div><div class='del'>-</div><div class='del'>-			for (i = 0; i &lt; HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; i++) {</div><div class='del'>-				if (likely(wr-&gt;sg_list[i].length)) {</div><div class='del'>-					set_data_seg_v2(dseg, wr-&gt;sg_list + i);</div><div class='del'>-					dseg++;</div><div class='del'>-				}</div><div class='add'>+		for (i = 0; i &lt; wr-&gt;num_sge &amp;&amp; j &lt; HNS_ROCE_SGE_IN_WQE; i++) {</div><div class='add'>+			if (likely(wr-&gt;sg_list[i].length)) {</div><div class='add'>+				set_data_seg_v2(dseg, wr-&gt;sg_list + i);</div><div class='add'>+				dseg++;</div><div class='add'>+				j++;</div><div class='ctx'> 			}</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		set_extend_sge(qp, wr-&gt;sg_list + i, sge_ind,</div><div class='add'>+			       valid_num_sge - HNS_ROCE_SGE_IN_WQE);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-			set_extend_sge(qp, wr, sge_ind);</div><div class='add'>+	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, valid_num_sge);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int check_send_valid(struct hns_roce_dev *hr_dev,</div><div class='add'>+			    struct hns_roce_qp *hr_qp)</div><div class='add'>+{</div><div class='add'>+	if (unlikely(hr_qp-&gt;state == IB_QPS_RESET ||</div><div class='add'>+		     hr_qp-&gt;state == IB_QPS_INIT ||</div><div class='add'>+		     hr_qp-&gt;state == IB_QPS_RTR))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	else if (unlikely(hr_dev-&gt;state &gt;= HNS_ROCE_DEVICE_STATE_RST_DOWN))</div><div class='add'>+		return -EIO;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static unsigned int calc_wr_sge_num(const struct ib_send_wr *wr,</div><div class='add'>+				    unsigned int *sge_len)</div><div class='add'>+{</div><div class='add'>+	unsigned int valid_num = 0;</div><div class='add'>+	unsigned int len = 0;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; wr-&gt;num_sge; i++) {</div><div class='add'>+		if (likely(wr-&gt;sg_list[i].length)) {</div><div class='add'>+			len += wr-&gt;sg_list[i].length;</div><div class='add'>+			valid_num++;</div><div class='ctx'> 		}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		roce_set_field(rc_sq_wqe-&gt;byte_16,</div><div class='del'>-			       V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,</div><div class='del'>-			       V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, wr-&gt;num_sge);</div><div class='add'>+	*sge_len = len;</div><div class='add'>+	return valid_num;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static __le32 get_immtdata(const struct ib_send_wr *wr)</div><div class='add'>+{</div><div class='add'>+	switch (wr-&gt;opcode) {</div><div class='add'>+	case IB_WR_SEND_WITH_IMM:</div><div class='add'>+	case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='add'>+		return cpu_to_le32(be32_to_cpu(wr-&gt;ex.imm_data));</div><div class='add'>+	default:</div><div class='add'>+		return 0;</div><div class='ctx'> 	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,</div><div class='add'>+			 const struct ib_send_wr *wr)</div><div class='add'>+{</div><div class='add'>+	u32 ib_op = wr-&gt;opcode;</div><div class='add'>+</div><div class='add'>+	if (ib_op != IB_WR_SEND &amp;&amp; ib_op != IB_WR_SEND_WITH_IMM)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ud_sq_wqe-&gt;immtdata = get_immtdata(wr);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_OPCODE, to_hr_opcode(ib_op));</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,</div><div class='del'>-				 const struct ib_qp_attr *attr,</div><div class='del'>-				 int attr_mask, enum ib_qp_state cur_state,</div><div class='del'>-				 enum ib_qp_state new_state);</div><div class='add'>+static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,</div><div class='add'>+		      struct hns_roce_ah *ah)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ib_dev = ah-&gt;ibah.device;</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_UDPSPN, ah-&gt;av.udp_sport);</div><div class='add'>+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_HOPLIMIT, ah-&gt;av.hop_limit);</div><div class='add'>+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_TCLASS, ah-&gt;av.tclass);</div><div class='add'>+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_FLOW_LABEL, ah-&gt;av.flowlabel);</div><div class='add'>+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, ah-&gt;av.sl);</div><div class='add'>+</div><div class='add'>+	ud_sq_wqe-&gt;sgid_index = ah-&gt;av.gid_index;</div><div class='add'>+</div><div class='add'>+	memcpy(ud_sq_wqe-&gt;dmac, ah-&gt;av.mac, ETH_ALEN);</div><div class='add'>+	memcpy(ud_sq_wqe-&gt;dgid, ah-&gt;av.dgid, GID_LEN_V2);</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_VLAN_EN, ah-&gt;av.vlan_en);</div><div class='add'>+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_VLAN, ah-&gt;av.vlan_id);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int set_ud_wqe(struct hns_roce_qp *qp,</div><div class='add'>+			     const struct ib_send_wr *wr,</div><div class='add'>+			     void *wqe, unsigned int *sge_idx,</div><div class='add'>+			     unsigned int owner_bit)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)-&gt;ah);</div><div class='add'>+	struct hns_roce_v2_ud_send_wqe *ud_sq_wqe = wqe;</div><div class='add'>+	unsigned int curr_idx = *sge_idx;</div><div class='add'>+	unsigned int valid_num_sge;</div><div class='add'>+	u32 msg_len = 0;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	valid_num_sge = calc_wr_sge_num(wr, &amp;msg_len);</div><div class='add'>+</div><div class='add'>+	ret = set_ud_opcode(ud_sq_wqe, wr);</div><div class='add'>+	if (WARN_ON_ONCE(ret))</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ud_sq_wqe-&gt;msg_len = cpu_to_le32(msg_len);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_CQE,</div><div class='add'>+		     !!(wr-&gt;send_flags &amp; IB_SEND_SIGNALED));</div><div class='add'>+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SE,</div><div class='add'>+		     !!(wr-&gt;send_flags &amp; IB_SEND_SOLICITED));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_PD, to_hr_pd(qp-&gt;ibqp.pd)-&gt;pdn);</div><div class='add'>+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SGE_NUM, valid_num_sge);</div><div class='add'>+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_MSG_START_SGE_IDX,</div><div class='add'>+		     curr_idx &amp; (qp-&gt;sge.sge_cnt - 1));</div><div class='add'>+</div><div class='add'>+	ud_sq_wqe-&gt;qkey = cpu_to_le32(ud_wr(wr)-&gt;remote_qkey &amp; 0x80000000 ?</div><div class='add'>+			  qp-&gt;qkey : ud_wr(wr)-&gt;remote_qkey);</div><div class='add'>+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_DQPN, ud_wr(wr)-&gt;remote_qpn);</div><div class='add'>+</div><div class='add'>+	ret = fill_ud_av(ud_sq_wqe, ah);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	qp-&gt;sl = to_hr_ah(ud_wr(wr)-&gt;ah)-&gt;av.sl;</div><div class='add'>+</div><div class='add'>+	set_extend_sge(qp, wr-&gt;sg_list, &amp;curr_idx, valid_num_sge);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * The pipeline can sequentially post all valid WQEs into WQ buffer,</div><div class='add'>+	 * including new WQEs waiting for the doorbell to update the PI again.</div><div class='add'>+	 * Therefore, the owner bit of WQE MUST be updated after all fields</div><div class='add'>+	 * and extSGEs have been written into DDR instead of cache.</div><div class='add'>+	 */</div><div class='add'>+	if (qp-&gt;en_flags &amp; HNS_ROCE_QP_CAP_OWNER_DB)</div><div class='add'>+		dma_wmb();</div><div class='add'>+</div><div class='add'>+	*sge_idx = curr_idx;</div><div class='add'>+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_OWNER, owner_bit);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int set_rc_opcode(struct hns_roce_dev *hr_dev,</div><div class='add'>+			 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,</div><div class='add'>+			 const struct ib_send_wr *wr)</div><div class='add'>+{</div><div class='add'>+	u32 ib_op = wr-&gt;opcode;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	rc_sq_wqe-&gt;immtdata = get_immtdata(wr);</div><div class='add'>+</div><div class='add'>+	switch (ib_op) {</div><div class='add'>+	case IB_WR_RDMA_READ:</div><div class='add'>+	case IB_WR_RDMA_WRITE:</div><div class='add'>+	case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='add'>+		rc_sq_wqe-&gt;rkey = cpu_to_le32(rdma_wr(wr)-&gt;rkey);</div><div class='add'>+		rc_sq_wqe-&gt;va = cpu_to_le64(rdma_wr(wr)-&gt;remote_addr);</div><div class='add'>+		break;</div><div class='add'>+	case IB_WR_SEND:</div><div class='add'>+	case IB_WR_SEND_WITH_IMM:</div><div class='add'>+		break;</div><div class='add'>+	case IB_WR_ATOMIC_CMP_AND_SWP:</div><div class='add'>+	case IB_WR_ATOMIC_FETCH_AND_ADD:</div><div class='add'>+		rc_sq_wqe-&gt;rkey = cpu_to_le32(atomic_wr(wr)-&gt;rkey);</div><div class='add'>+		rc_sq_wqe-&gt;va = cpu_to_le64(atomic_wr(wr)-&gt;remote_addr);</div><div class='add'>+		break;</div><div class='add'>+	case IB_WR_REG_MR:</div><div class='add'>+		if (hr_dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09)</div><div class='add'>+			set_frmr_seg(rc_sq_wqe, reg_wr(wr));</div><div class='add'>+		else</div><div class='add'>+			ret = -EOPNOTSUPP;</div><div class='add'>+		break;</div><div class='add'>+	case IB_WR_SEND_WITH_INV:</div><div class='add'>+		rc_sq_wqe-&gt;inv_key = cpu_to_le32(wr-&gt;ex.invalidate_rkey);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (unlikely(ret))</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_OPCODE, to_hr_opcode(ib_op));</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int set_rc_wqe(struct hns_roce_qp *qp,</div><div class='add'>+			     const struct ib_send_wr *wr,</div><div class='add'>+			     void *wqe, unsigned int *sge_idx,</div><div class='add'>+			     unsigned int owner_bit)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(qp-&gt;ibqp.device);</div><div class='add'>+	struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;</div><div class='add'>+	unsigned int curr_idx = *sge_idx;</div><div class='add'>+	unsigned int valid_num_sge;</div><div class='add'>+	u32 msg_len = 0;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	valid_num_sge = calc_wr_sge_num(wr, &amp;msg_len);</div><div class='add'>+</div><div class='add'>+	rc_sq_wqe-&gt;msg_len = cpu_to_le32(msg_len);</div><div class='add'>+</div><div class='add'>+	ret = set_rc_opcode(hr_dev, rc_sq_wqe, wr);</div><div class='add'>+	if (WARN_ON_ONCE(ret))</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SO,</div><div class='add'>+		     (wr-&gt;send_flags &amp; IB_SEND_FENCE) ? 1 : 0);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SE,</div><div class='add'>+		     (wr-&gt;send_flags &amp; IB_SEND_SOLICITED) ? 1 : 0);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_CQE,</div><div class='add'>+		     (wr-&gt;send_flags &amp; IB_SEND_SIGNALED) ? 1 : 0);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_MSG_START_SGE_IDX,</div><div class='add'>+		     curr_idx &amp; (qp-&gt;sge.sge_cnt - 1));</div><div class='add'>+</div><div class='add'>+	if (wr-&gt;opcode == IB_WR_ATOMIC_CMP_AND_SWP ||</div><div class='add'>+	    wr-&gt;opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {</div><div class='add'>+		if (msg_len != ATOMIC_WR_LEN)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		set_atomic_seg(wr, rc_sq_wqe, valid_num_sge);</div><div class='add'>+	} else if (wr-&gt;opcode != IB_WR_REG_MR) {</div><div class='add'>+		ret = set_rwqe_data_seg(&amp;qp-&gt;ibqp, wr, rc_sq_wqe,</div><div class='add'>+					&amp;curr_idx, valid_num_sge);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * The pipeline can sequentially post all valid WQEs into WQ buffer,</div><div class='add'>+	 * including new WQEs waiting for the doorbell to update the PI again.</div><div class='add'>+	 * Therefore, the owner bit of WQE MUST be updated after all fields</div><div class='add'>+	 * and extSGEs have been written into DDR instead of cache.</div><div class='add'>+	 */</div><div class='add'>+	if (qp-&gt;en_flags &amp; HNS_ROCE_QP_CAP_OWNER_DB)</div><div class='add'>+		dma_wmb();</div><div class='add'>+</div><div class='add'>+	*sge_idx = curr_idx;</div><div class='add'>+	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_OWNER, owner_bit);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void update_sq_db(struct hns_roce_dev *hr_dev,</div><div class='add'>+				struct hns_roce_qp *qp)</div><div class='add'>+{</div><div class='add'>+	if (unlikely(qp-&gt;state == IB_QPS_ERR)) {</div><div class='add'>+		flush_cqe(hr_dev, qp);</div><div class='add'>+	} else {</div><div class='add'>+		struct hns_roce_v2_db sq_db = {};</div><div class='add'>+</div><div class='add'>+		hr_reg_write(&amp;sq_db, DB_TAG, qp-&gt;qpn);</div><div class='add'>+		hr_reg_write(&amp;sq_db, DB_CMD, HNS_ROCE_V2_SQ_DB);</div><div class='add'>+		hr_reg_write(&amp;sq_db, DB_PI, qp-&gt;sq.head);</div><div class='add'>+		hr_reg_write(&amp;sq_db, DB_SL, qp-&gt;sl);</div><div class='add'>+</div><div class='add'>+		hns_roce_write64(hr_dev, (__le32 *)&amp;sq_db, qp-&gt;sq.db_reg);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void update_rq_db(struct hns_roce_dev *hr_dev,</div><div class='add'>+				struct hns_roce_qp *qp)</div><div class='add'>+{</div><div class='add'>+	if (unlikely(qp-&gt;state == IB_QPS_ERR)) {</div><div class='add'>+		flush_cqe(hr_dev, qp);</div><div class='add'>+	} else {</div><div class='add'>+		if (likely(qp-&gt;en_flags &amp; HNS_ROCE_QP_CAP_RQ_RECORD_DB)) {</div><div class='add'>+			*qp-&gt;rdb.db_record =</div><div class='add'>+					qp-&gt;rq.head &amp; V2_DB_PRODUCER_IDX_M;</div><div class='add'>+		} else {</div><div class='add'>+			struct hns_roce_v2_db rq_db = {};</div><div class='add'>+</div><div class='add'>+			hr_reg_write(&amp;rq_db, DB_TAG, qp-&gt;qpn);</div><div class='add'>+			hr_reg_write(&amp;rq_db, DB_CMD, HNS_ROCE_V2_RQ_DB);</div><div class='add'>+			hr_reg_write(&amp;rq_db, DB_PI, qp-&gt;rq.head);</div><div class='add'>+</div><div class='add'>+			hns_roce_write64(hr_dev, (__le32 *)&amp;rq_db,</div><div class='add'>+					 qp-&gt;rq.db_reg);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_write512(struct hns_roce_dev *hr_dev, u64 *val,</div><div class='add'>+			      u64 __iomem *dest)</div><div class='add'>+{</div><div class='add'>+#define HNS_ROCE_WRITE_TIMES 8</div><div class='add'>+	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev-&gt;priv;</div><div class='add'>+	struct hnae3_handle *handle = priv-&gt;handle;</div><div class='add'>+	const struct hnae3_ae_ops *ops = handle-&gt;ae_algo-&gt;ops;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (!hr_dev-&gt;dis_db &amp;&amp; !ops-&gt;get_hw_reset_stat(handle))</div><div class='add'>+		for (i = 0; i &lt; HNS_ROCE_WRITE_TIMES; i++)</div><div class='add'>+			writeq_relaxed(*(val + i), dest + i);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void write_dwqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp,</div><div class='add'>+		       void *wqe)</div><div class='add'>+{</div><div class='add'>+#define HNS_ROCE_SL_SHIFT 2</div><div class='add'>+	struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;</div><div class='add'>+</div><div class='add'>+	if (unlikely(qp-&gt;state == IB_QPS_ERR)) {</div><div class='add'>+		flush_cqe(hr_dev, qp);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+	/* All kinds of DirectWQE have the same header field layout */</div><div class='add'>+	hr_reg_enable(rc_sq_wqe, RC_SEND_WQE_FLAG);</div><div class='add'>+	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_DB_SL_L, qp-&gt;sl);</div><div class='add'>+	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_DB_SL_H,</div><div class='add'>+		     qp-&gt;sl &gt;&gt; HNS_ROCE_SL_SHIFT);</div><div class='add'>+	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_WQE_INDEX, qp-&gt;sq.head);</div><div class='add'>+</div><div class='add'>+	hns_roce_write512(hr_dev, wqe, qp-&gt;sq.db_reg);</div><div class='add'>+}</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_v2_post_send(struct ib_qp *ibqp,</div><div class='ctx'> 				 const struct ib_send_wr *wr,</div><div class='ctx'> 				 const struct ib_send_wr **bad_wr)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='del'>-	struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)-&gt;ah);</div><div class='del'>-	struct hns_roce_v2_ud_send_wqe *ud_sq_wqe;</div><div class='del'>-	struct hns_roce_v2_rc_send_wqe *rc_sq_wqe;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='ctx'> 	struct hns_roce_qp *qp = to_hr_qp(ibqp);</div><div class='del'>-	struct hns_roce_wqe_frmr_seg *fseg;</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	struct hns_roce_v2_db sq_db;</div><div class='del'>-	struct ib_qp_attr attr;</div><div class='del'>-	unsigned int sge_ind = 0;</div><div class='add'>+	unsigned long flags = 0;</div><div class='ctx'> 	unsigned int owner_bit;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	unsigned int ind;</div><div class='add'>+	unsigned int sge_idx;</div><div class='add'>+	unsigned int wqe_idx;</div><div class='ctx'> 	void *wqe = NULL;</div><div class='del'>-	bool loopback;</div><div class='del'>-	int attr_mask;</div><div class='del'>-	u32 tmp_len;</div><div class='del'>-	int ret = 0;</div><div class='del'>-	u32 hr_op;</div><div class='del'>-	u8 *smac;</div><div class='del'>-	int nreq;</div><div class='del'>-	int i;</div><div class='add'>+	u32 nreq;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	if (unlikely(ibqp-&gt;qp_type != IB_QPT_RC &amp;&amp;</div><div class='del'>-		     ibqp-&gt;qp_type != IB_QPT_GSI &amp;&amp;</div><div class='del'>-		     ibqp-&gt;qp_type != IB_QPT_UD)) {</div><div class='del'>-		dev_err(dev, "Not supported QP(0x%x)type!\n", ibqp-&gt;qp_type);</div><div class='del'>-		*bad_wr = wr;</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-	}</div><div class='add'>+	spin_lock_irqsave(&amp;qp-&gt;sq.lock, flags);</div><div class='ctx'> </div><div class='del'>-	if (unlikely(qp-&gt;state == IB_QPS_RESET || qp-&gt;state == IB_QPS_INIT ||</div><div class='del'>-		     qp-&gt;state == IB_QPS_RTR)) {</div><div class='del'>-		dev_err(dev, "Post WQE fail, QP state %d err!\n", qp-&gt;state);</div><div class='add'>+	ret = check_send_valid(hr_dev, qp);</div><div class='add'>+	if (unlikely(ret)) {</div><div class='ctx'> 		*bad_wr = wr;</div><div class='del'>-		return -EINVAL;</div><div class='add'>+		nreq = 0;</div><div class='add'>+		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	spin_lock_irqsave(&amp;qp-&gt;sq.lock, flags);</div><div class='del'>-	ind = qp-&gt;sq_next_wqe;</div><div class='del'>-	sge_ind = qp-&gt;next_sge;</div><div class='add'>+	sge_idx = qp-&gt;next_sge;</div><div class='ctx'> </div><div class='ctx'> 	for (nreq = 0; wr; ++nreq, wr = wr-&gt;next) {</div><div class='ctx'> 		if (hns_roce_wq_overflow(&amp;qp-&gt;sq, nreq, qp-&gt;ibqp.send_cq)) {</div><div class='hunk'>@@ -279,333 +723,48 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp,</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='add'>+		wqe_idx = (qp-&gt;sq.head + nreq) &amp; (qp-&gt;sq.wqe_cnt - 1);</div><div class='add'>+</div><div class='ctx'> 		if (unlikely(wr-&gt;num_sge &gt; qp-&gt;sq.max_gs)) {</div><div class='del'>-			dev_err(dev, "num_sge=%d &gt; qp-&gt;sq.max_gs=%d\n",</div><div class='del'>-				wr-&gt;num_sge, qp-&gt;sq.max_gs);</div><div class='add'>+			ibdev_err(ibdev, "num_sge = %d &gt; qp-&gt;sq.max_gs = %u.\n",</div><div class='add'>+				  wr-&gt;num_sge, qp-&gt;sq.max_gs);</div><div class='ctx'> 			ret = -EINVAL;</div><div class='ctx'> 			*bad_wr = wr;</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		wqe = get_send_wqe(qp, ind &amp; (qp-&gt;sq.wqe_cnt - 1));</div><div class='del'>-		qp-&gt;sq.wrid[(qp-&gt;sq.head + nreq) &amp; (qp-&gt;sq.wqe_cnt - 1)] =</div><div class='del'>-								      wr-&gt;wr_id;</div><div class='del'>-</div><div class='add'>+		wqe = hns_roce_get_send_wqe(qp, wqe_idx);</div><div class='add'>+		qp-&gt;sq.wrid[wqe_idx] = wr-&gt;wr_id;</div><div class='ctx'> 		owner_bit =</div><div class='ctx'> 		       ~(((qp-&gt;sq.head + nreq) &gt;&gt; ilog2(qp-&gt;sq.wqe_cnt)) &amp; 0x1);</div><div class='del'>-		tmp_len = 0;</div><div class='ctx'> </div><div class='del'>-		/* Corresponding to the QP type, wqe process separately */</div><div class='del'>-		if (ibqp-&gt;qp_type == IB_QPT_GSI) {</div><div class='del'>-			ud_sq_wqe = wqe;</div><div class='del'>-			memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe));</div><div class='del'>-</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;dmac, V2_UD_SEND_WQE_DMAC_0_M,</div><div class='del'>-				       V2_UD_SEND_WQE_DMAC_0_S, ah-&gt;av.mac[0]);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;dmac, V2_UD_SEND_WQE_DMAC_1_M,</div><div class='del'>-				       V2_UD_SEND_WQE_DMAC_1_S, ah-&gt;av.mac[1]);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;dmac, V2_UD_SEND_WQE_DMAC_2_M,</div><div class='del'>-				       V2_UD_SEND_WQE_DMAC_2_S, ah-&gt;av.mac[2]);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;dmac, V2_UD_SEND_WQE_DMAC_3_M,</div><div class='del'>-				       V2_UD_SEND_WQE_DMAC_3_S, ah-&gt;av.mac[3]);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;byte_48,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_48_DMAC_4_S,</div><div class='del'>-				       ah-&gt;av.mac[4]);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;byte_48,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_48_DMAC_5_S,</div><div class='del'>-				       ah-&gt;av.mac[5]);</div><div class='del'>-</div><div class='del'>-			/* MAC loopback */</div><div class='del'>-			smac = (u8 *)hr_dev-&gt;dev_addr[qp-&gt;port];</div><div class='del'>-			loopback = ether_addr_equal_unaligned(ah-&gt;av.mac,</div><div class='del'>-							      smac) ? 1 : 0;</div><div class='del'>-</div><div class='del'>-			roce_set_bit(ud_sq_wqe-&gt;byte_40,</div><div class='del'>-				     V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback);</div><div class='del'>-</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;byte_4,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_4_OPCODE_M,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_4_OPCODE_S,</div><div class='del'>-				       HNS_ROCE_V2_WQE_OP_SEND);</div><div class='del'>-</div><div class='del'>-			for (i = 0; i &lt; wr-&gt;num_sge; i++)</div><div class='del'>-				tmp_len += wr-&gt;sg_list[i].length;</div><div class='del'>-</div><div class='del'>-			ud_sq_wqe-&gt;msg_len =</div><div class='del'>-			 cpu_to_le32(le32_to_cpu(ud_sq_wqe-&gt;msg_len) + tmp_len);</div><div class='del'>-</div><div class='del'>-			switch (wr-&gt;opcode) {</div><div class='del'>-			case IB_WR_SEND_WITH_IMM:</div><div class='del'>-			case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='del'>-				ud_sq_wqe-&gt;immtdata =</div><div class='del'>-				      cpu_to_le32(be32_to_cpu(wr-&gt;ex.imm_data));</div><div class='del'>-				break;</div><div class='del'>-			default:</div><div class='del'>-				ud_sq_wqe-&gt;immtdata = 0;</div><div class='del'>-				break;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			/* Set sig attr */</div><div class='del'>-			roce_set_bit(ud_sq_wqe-&gt;byte_4,</div><div class='del'>-				   V2_UD_SEND_WQE_BYTE_4_CQE_S,</div><div class='del'>-				   (wr-&gt;send_flags &amp; IB_SEND_SIGNALED) ? 1 : 0);</div><div class='del'>-</div><div class='del'>-			/* Set se attr */</div><div class='del'>-			roce_set_bit(ud_sq_wqe-&gt;byte_4,</div><div class='del'>-				  V2_UD_SEND_WQE_BYTE_4_SE_S,</div><div class='del'>-				  (wr-&gt;send_flags &amp; IB_SEND_SOLICITED) ? 1 : 0);</div><div class='del'>-</div><div class='del'>-			roce_set_bit(ud_sq_wqe-&gt;byte_4,</div><div class='del'>-				     V2_UD_SEND_WQE_BYTE_4_OWNER_S, owner_bit);</div><div class='del'>-</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;byte_16,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_16_PD_M,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_16_PD_S,</div><div class='del'>-				       to_hr_pd(ibqp-&gt;pd)-&gt;pdn);</div><div class='del'>-</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;byte_16,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S,</div><div class='del'>-				       wr-&gt;num_sge);</div><div class='del'>-</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;byte_20,</div><div class='del'>-				     V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,</div><div class='del'>-				     V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,</div><div class='del'>-				     sge_ind &amp; (qp-&gt;sge.sge_cnt - 1));</div><div class='del'>-</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;byte_24,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0);</div><div class='del'>-			ud_sq_wqe-&gt;qkey =</div><div class='del'>-			     cpu_to_le32(ud_wr(wr)-&gt;remote_qkey &amp; 0x80000000 ?</div><div class='del'>-			     qp-&gt;qkey : ud_wr(wr)-&gt;remote_qkey);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;byte_32,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_32_DQPN_M,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_32_DQPN_S,</div><div class='del'>-				       ud_wr(wr)-&gt;remote_qpn);</div><div class='del'>-</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;byte_36,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_36_VLAN_M,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_36_VLAN_S,</div><div class='del'>-				       le16_to_cpu(ah-&gt;av.vlan));</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;byte_36,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S,</div><div class='del'>-				       ah-&gt;av.hop_limit);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;byte_36,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_36_TCLASS_M,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_36_TCLASS_S,</div><div class='del'>-				       ah-&gt;av.sl_tclass_flowlabel &gt;&gt;</div><div class='del'>-				       HNS_ROCE_TCLASS_SHIFT);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;byte_40,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S,</div><div class='del'>-				       ah-&gt;av.sl_tclass_flowlabel &amp;</div><div class='del'>-				       HNS_ROCE_FLOW_LABEL_MASK);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;byte_40,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_40_SL_M,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_40_SL_S,</div><div class='del'>-				      le32_to_cpu(ah-&gt;av.sl_tclass_flowlabel) &gt;&gt;</div><div class='del'>-				      HNS_ROCE_SL_SHIFT);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;byte_40,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_40_PORTN_M,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_40_PORTN_S,</div><div class='del'>-				       qp-&gt;port);</div><div class='del'>-</div><div class='del'>-			roce_set_bit(ud_sq_wqe-&gt;byte_40,</div><div class='del'>-				     V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,</div><div class='del'>-				     ah-&gt;av.vlan_en ? 1 : 0);</div><div class='del'>-			roce_set_field(ud_sq_wqe-&gt;byte_48,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,</div><div class='del'>-				       V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S,</div><div class='del'>-				       hns_get_gid_index(hr_dev, qp-&gt;phy_port,</div><div class='del'>-							 ah-&gt;av.gid_index));</div><div class='del'>-</div><div class='del'>-			memcpy(&amp;ud_sq_wqe-&gt;dgid[0], &amp;ah-&gt;av.dgid[0],</div><div class='del'>-			       GID_LEN_V2);</div><div class='del'>-</div><div class='del'>-			set_extend_sge(qp, wr, &amp;sge_ind);</div><div class='del'>-			ind++;</div><div class='del'>-		} else if (ibqp-&gt;qp_type == IB_QPT_RC) {</div><div class='del'>-			rc_sq_wqe = wqe;</div><div class='del'>-			memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe));</div><div class='del'>-			for (i = 0; i &lt; wr-&gt;num_sge; i++)</div><div class='del'>-				tmp_len += wr-&gt;sg_list[i].length;</div><div class='del'>-</div><div class='del'>-			rc_sq_wqe-&gt;msg_len =</div><div class='del'>-			 cpu_to_le32(le32_to_cpu(rc_sq_wqe-&gt;msg_len) + tmp_len);</div><div class='del'>-</div><div class='del'>-			switch (wr-&gt;opcode) {</div><div class='del'>-			case IB_WR_SEND_WITH_IMM:</div><div class='del'>-			case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='del'>-				rc_sq_wqe-&gt;immtdata =</div><div class='del'>-				      cpu_to_le32(be32_to_cpu(wr-&gt;ex.imm_data));</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_SEND_WITH_INV:</div><div class='del'>-				rc_sq_wqe-&gt;inv_key =</div><div class='del'>-					cpu_to_le32(wr-&gt;ex.invalidate_rkey);</div><div class='del'>-				break;</div><div class='del'>-			default:</div><div class='del'>-				rc_sq_wqe-&gt;immtdata = 0;</div><div class='del'>-				break;</div><div class='del'>-			}</div><div class='add'>+		/* RC and UD share the same DirectWQE field layout */</div><div class='add'>+		((struct hns_roce_v2_rc_send_wqe *)wqe)-&gt;byte_4 = 0;</div><div class='ctx'> </div><div class='del'>-			roce_set_bit(rc_sq_wqe-&gt;byte_4,</div><div class='del'>-				     V2_RC_SEND_WQE_BYTE_4_FENCE_S,</div><div class='del'>-				     (wr-&gt;send_flags &amp; IB_SEND_FENCE) ? 1 : 0);</div><div class='del'>-</div><div class='del'>-			roce_set_bit(rc_sq_wqe-&gt;byte_4,</div><div class='del'>-				  V2_RC_SEND_WQE_BYTE_4_SE_S,</div><div class='del'>-				  (wr-&gt;send_flags &amp; IB_SEND_SOLICITED) ? 1 : 0);</div><div class='del'>-</div><div class='del'>-			roce_set_bit(rc_sq_wqe-&gt;byte_4,</div><div class='del'>-				   V2_RC_SEND_WQE_BYTE_4_CQE_S,</div><div class='del'>-				   (wr-&gt;send_flags &amp; IB_SEND_SIGNALED) ? 1 : 0);</div><div class='del'>-</div><div class='del'>-			roce_set_bit(rc_sq_wqe-&gt;byte_4,</div><div class='del'>-				     V2_RC_SEND_WQE_BYTE_4_OWNER_S, owner_bit);</div><div class='del'>-</div><div class='del'>-			wqe += sizeof(struct hns_roce_v2_rc_send_wqe);</div><div class='del'>-			switch (wr-&gt;opcode) {</div><div class='del'>-			case IB_WR_RDMA_READ:</div><div class='del'>-				hr_op = HNS_ROCE_V2_WQE_OP_RDMA_READ;</div><div class='del'>-				rc_sq_wqe-&gt;rkey =</div><div class='del'>-					cpu_to_le32(rdma_wr(wr)-&gt;rkey);</div><div class='del'>-				rc_sq_wqe-&gt;va =</div><div class='del'>-					cpu_to_le64(rdma_wr(wr)-&gt;remote_addr);</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_RDMA_WRITE:</div><div class='del'>-				hr_op = HNS_ROCE_V2_WQE_OP_RDMA_WRITE;</div><div class='del'>-				rc_sq_wqe-&gt;rkey =</div><div class='del'>-					cpu_to_le32(rdma_wr(wr)-&gt;rkey);</div><div class='del'>-				rc_sq_wqe-&gt;va =</div><div class='del'>-					cpu_to_le64(rdma_wr(wr)-&gt;remote_addr);</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='del'>-				hr_op = HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM;</div><div class='del'>-				rc_sq_wqe-&gt;rkey =</div><div class='del'>-					cpu_to_le32(rdma_wr(wr)-&gt;rkey);</div><div class='del'>-				rc_sq_wqe-&gt;va =</div><div class='del'>-					cpu_to_le64(rdma_wr(wr)-&gt;remote_addr);</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_SEND:</div><div class='del'>-				hr_op = HNS_ROCE_V2_WQE_OP_SEND;</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_SEND_WITH_INV:</div><div class='del'>-				hr_op = HNS_ROCE_V2_WQE_OP_SEND_WITH_INV;</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_SEND_WITH_IMM:</div><div class='del'>-				hr_op = HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM;</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_LOCAL_INV:</div><div class='del'>-				hr_op = HNS_ROCE_V2_WQE_OP_LOCAL_INV;</div><div class='del'>-				roce_set_bit(rc_sq_wqe-&gt;byte_4,</div><div class='del'>-					       V2_RC_SEND_WQE_BYTE_4_SO_S, 1);</div><div class='del'>-				rc_sq_wqe-&gt;inv_key =</div><div class='del'>-					    cpu_to_le32(wr-&gt;ex.invalidate_rkey);</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_REG_MR:</div><div class='del'>-				hr_op = HNS_ROCE_V2_WQE_OP_FAST_REG_PMR;</div><div class='del'>-				fseg = wqe;</div><div class='del'>-				set_frmr_seg(rc_sq_wqe, fseg, reg_wr(wr));</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_ATOMIC_CMP_AND_SWP:</div><div class='del'>-				hr_op = HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP;</div><div class='del'>-				rc_sq_wqe-&gt;rkey =</div><div class='del'>-					cpu_to_le32(atomic_wr(wr)-&gt;rkey);</div><div class='del'>-				rc_sq_wqe-&gt;va =</div><div class='del'>-					cpu_to_le64(atomic_wr(wr)-&gt;remote_addr);</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_ATOMIC_FETCH_AND_ADD:</div><div class='del'>-				hr_op = HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD;</div><div class='del'>-				rc_sq_wqe-&gt;rkey =</div><div class='del'>-					cpu_to_le32(atomic_wr(wr)-&gt;rkey);</div><div class='del'>-				rc_sq_wqe-&gt;va =</div><div class='del'>-					cpu_to_le64(atomic_wr(wr)-&gt;remote_addr);</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:</div><div class='del'>-				hr_op =</div><div class='del'>-				       HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP;</div><div class='del'>-				break;</div><div class='del'>-			case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:</div><div class='del'>-				hr_op =</div><div class='del'>-				      HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD;</div><div class='del'>-				break;</div><div class='del'>-			default:</div><div class='del'>-				hr_op = HNS_ROCE_V2_WQE_OP_MASK;</div><div class='del'>-				break;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			roce_set_field(rc_sq_wqe-&gt;byte_4,</div><div class='del'>-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,</div><div class='del'>-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_S, hr_op);</div><div class='del'>-</div><div class='del'>-			if (wr-&gt;opcode == IB_WR_ATOMIC_CMP_AND_SWP ||</div><div class='del'>-			    wr-&gt;opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {</div><div class='del'>-				struct hns_roce_v2_wqe_data_seg *dseg;</div><div class='del'>-</div><div class='del'>-				dseg = wqe;</div><div class='del'>-				set_data_seg_v2(dseg, wr-&gt;sg_list);</div><div class='del'>-				wqe += sizeof(struct hns_roce_v2_wqe_data_seg);</div><div class='del'>-				set_atomic_seg(wqe, atomic_wr(wr));</div><div class='del'>-				roce_set_field(rc_sq_wqe-&gt;byte_16,</div><div class='del'>-					       V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,</div><div class='del'>-					       V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S,</div><div class='del'>-					       wr-&gt;num_sge);</div><div class='del'>-			} else if (wr-&gt;opcode != IB_WR_REG_MR) {</div><div class='del'>-				ret = set_rwqe_data_seg(ibqp, wr, rc_sq_wqe,</div><div class='del'>-							wqe, &amp;sge_ind, bad_wr);</div><div class='del'>-				if (ret)</div><div class='del'>-					goto out;</div><div class='del'>-			}</div><div class='add'>+		/* Corresponding to the QP type, wqe process separately */</div><div class='add'>+		if (ibqp-&gt;qp_type == IB_QPT_RC)</div><div class='add'>+			ret = set_rc_wqe(qp, wr, wqe, &amp;sge_idx, owner_bit);</div><div class='add'>+		else</div><div class='add'>+			ret = set_ud_wqe(qp, wr, wqe, &amp;sge_idx, owner_bit);</div><div class='ctx'> </div><div class='del'>-			ind++;</div><div class='del'>-		} else {</div><div class='del'>-			dev_err(dev, "Illegal qp_type(0x%x)\n", ibqp-&gt;qp_type);</div><div class='del'>-			spin_unlock_irqrestore(&amp;qp-&gt;sq.lock, flags);</div><div class='add'>+		trace_hns_sq_wqe(qp-&gt;qpn, wqe_idx, wqe, 1 &lt;&lt; qp-&gt;sq.wqe_shift,</div><div class='add'>+				 wr-&gt;wr_id, TRACE_SQ);</div><div class='add'>+		if (unlikely(ret)) {</div><div class='ctx'> 			*bad_wr = wr;</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='add'>+			goto out;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='ctx'> 	if (likely(nreq)) {</div><div class='ctx'> 		qp-&gt;sq.head += nreq;</div><div class='del'>-		/* Memory barrier */</div><div class='del'>-		wmb();</div><div class='del'>-</div><div class='del'>-		sq_db.byte_4 = 0;</div><div class='del'>-		sq_db.parameter = 0;</div><div class='del'>-</div><div class='del'>-		roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M,</div><div class='del'>-			       V2_DB_BYTE_4_TAG_S, qp-&gt;doorbell_qpn);</div><div class='del'>-		roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,</div><div class='del'>-			       V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);</div><div class='del'>-		roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M,</div><div class='del'>-			       V2_DB_PARAMETER_IDX_S,</div><div class='del'>-			       qp-&gt;sq.head &amp; ((qp-&gt;sq.wqe_cnt &lt;&lt; 1) - 1));</div><div class='del'>-		roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,</div><div class='del'>-			       V2_DB_PARAMETER_SL_S, qp-&gt;sl);</div><div class='del'>-</div><div class='del'>-		hns_roce_write64(hr_dev, (__le32 *)&amp;sq_db, qp-&gt;sq.db_reg_l);</div><div class='del'>-</div><div class='del'>-		qp-&gt;sq_next_wqe = ind;</div><div class='del'>-		qp-&gt;next_sge = sge_ind;</div><div class='del'>-</div><div class='del'>-		if (qp-&gt;state == IB_QPS_ERR) {</div><div class='del'>-			attr_mask = IB_QP_STATE;</div><div class='del'>-			attr.qp_state = IB_QPS_ERR;</div><div class='del'>-</div><div class='del'>-			ret = hns_roce_v2_modify_qp(&amp;qp-&gt;ibqp, &amp;attr, attr_mask,</div><div class='del'>-						    qp-&gt;state, IB_QPS_ERR);</div><div class='del'>-			if (ret) {</div><div class='del'>-				spin_unlock_irqrestore(&amp;qp-&gt;sq.lock, flags);</div><div class='del'>-				*bad_wr = wr;</div><div class='del'>-				return ret;</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='add'>+		qp-&gt;next_sge = sge_idx;</div><div class='add'>+</div><div class='add'>+		if (nreq == 1 &amp;&amp; !ret &amp;&amp;</div><div class='add'>+		    (qp-&gt;en_flags &amp; HNS_ROCE_QP_CAP_DIRECT_WQE))</div><div class='add'>+			write_dwqe(hr_dev, qp, wqe);</div><div class='add'>+		else</div><div class='add'>+			update_sq_db(hr_dev, qp);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	spin_unlock_irqrestore(&amp;qp-&gt;sq.lock, flags);</div><div class='hunk'>@@ -613,108 +772,251 @@ out:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int check_recv_valid(struct hns_roce_dev *hr_dev,</div><div class='add'>+			    struct hns_roce_qp *hr_qp)</div><div class='add'>+{</div><div class='add'>+	if (unlikely(hr_dev-&gt;state &gt;= HNS_ROCE_DEVICE_STATE_RST_DOWN))</div><div class='add'>+		return -EIO;</div><div class='add'>+</div><div class='add'>+	if (hr_qp-&gt;state == IB_QPS_RESET)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void fill_recv_sge_to_wqe(const struct ib_recv_wr *wr, void *wqe,</div><div class='add'>+				 u32 max_sge, bool rsv)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_v2_wqe_data_seg *dseg = wqe;</div><div class='add'>+	u32 i, cnt;</div><div class='add'>+</div><div class='add'>+	for (i = 0, cnt = 0; i &lt; wr-&gt;num_sge; i++) {</div><div class='add'>+		/* Skip zero-length sge */</div><div class='add'>+		if (!wr-&gt;sg_list[i].length)</div><div class='add'>+			continue;</div><div class='add'>+		set_data_seg_v2(dseg + cnt, wr-&gt;sg_list + i);</div><div class='add'>+		cnt++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Fill a reserved sge to make hw stop reading remaining segments */</div><div class='add'>+	if (rsv) {</div><div class='add'>+		dseg[cnt].lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);</div><div class='add'>+		dseg[cnt].addr = 0;</div><div class='add'>+		dseg[cnt].len = cpu_to_le32(HNS_ROCE_INVALID_SGE_LENGTH);</div><div class='add'>+	} else {</div><div class='add'>+		/* Clear remaining segments to make ROCEE ignore sges */</div><div class='add'>+		if (cnt &lt; max_sge)</div><div class='add'>+			memset(dseg + cnt, 0,</div><div class='add'>+			       (max_sge - cnt) * HNS_ROCE_SGE_SIZE);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void fill_rq_wqe(struct hns_roce_qp *hr_qp, const struct ib_recv_wr *wr,</div><div class='add'>+			u32 wqe_idx, u32 max_sge)</div><div class='add'>+{</div><div class='add'>+	void *wqe = NULL;</div><div class='add'>+</div><div class='add'>+	wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx);</div><div class='add'>+	fill_recv_sge_to_wqe(wr, wqe, max_sge, hr_qp-&gt;rq.rsv_sge);</div><div class='add'>+</div><div class='add'>+	trace_hns_rq_wqe(hr_qp-&gt;qpn, wqe_idx, wqe, 1 &lt;&lt; hr_qp-&gt;rq.wqe_shift,</div><div class='add'>+			 wr-&gt;wr_id, TRACE_RQ);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int hns_roce_v2_post_recv(struct ib_qp *ibqp,</div><div class='ctx'> 				 const struct ib_recv_wr *wr,</div><div class='ctx'> 				 const struct ib_recv_wr **bad_wr)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='ctx'> 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='del'>-	struct hns_roce_v2_wqe_data_seg *dseg;</div><div class='del'>-	struct hns_roce_rinl_sge *sge_list;</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	struct ib_qp_attr attr;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	u32 wqe_idx, nreq, max_sge;</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	void *wqe = NULL;</div><div class='del'>-	int attr_mask;</div><div class='del'>-	int ret = 0;</div><div class='del'>-	int nreq;</div><div class='del'>-	int ind;</div><div class='del'>-	int i;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;hr_qp-&gt;rq.lock, flags);</div><div class='del'>-	ind = hr_qp-&gt;rq.head &amp; (hr_qp-&gt;rq.wqe_cnt - 1);</div><div class='ctx'> </div><div class='del'>-	if (hr_qp-&gt;state == IB_QPS_RESET) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;hr_qp-&gt;rq.lock, flags);</div><div class='add'>+	ret = check_recv_valid(hr_dev, hr_qp);</div><div class='add'>+	if (unlikely(ret)) {</div><div class='ctx'> 		*bad_wr = wr;</div><div class='del'>-		return -EINVAL;</div><div class='add'>+		nreq = 0;</div><div class='add'>+		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	max_sge = hr_qp-&gt;rq.max_gs - hr_qp-&gt;rq.rsv_sge;</div><div class='ctx'> 	for (nreq = 0; wr; ++nreq, wr = wr-&gt;next) {</div><div class='del'>-		if (hns_roce_wq_overflow(&amp;hr_qp-&gt;rq, nreq,</div><div class='del'>-			hr_qp-&gt;ibqp.recv_cq)) {</div><div class='add'>+		if (unlikely(hns_roce_wq_overflow(&amp;hr_qp-&gt;rq, nreq,</div><div class='add'>+						  hr_qp-&gt;ibqp.recv_cq))) {</div><div class='ctx'> 			ret = -ENOMEM;</div><div class='ctx'> 			*bad_wr = wr;</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		if (unlikely(wr-&gt;num_sge &gt; hr_qp-&gt;rq.max_gs)) {</div><div class='del'>-			dev_err(dev, "rq:num_sge=%d &gt; qp-&gt;sq.max_gs=%d\n",</div><div class='del'>-				wr-&gt;num_sge, hr_qp-&gt;rq.max_gs);</div><div class='add'>+		if (unlikely(wr-&gt;num_sge &gt; max_sge)) {</div><div class='add'>+			ibdev_err(ibdev, "num_sge = %d &gt;= max_sge = %u.\n",</div><div class='add'>+				  wr-&gt;num_sge, max_sge);</div><div class='ctx'> 			ret = -EINVAL;</div><div class='ctx'> 			*bad_wr = wr;</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		wqe = get_recv_wqe(hr_qp, ind);</div><div class='del'>-		dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;</div><div class='del'>-		for (i = 0; i &lt; wr-&gt;num_sge; i++) {</div><div class='del'>-			if (!wr-&gt;sg_list[i].length)</div><div class='del'>-				continue;</div><div class='del'>-			set_data_seg_v2(dseg, wr-&gt;sg_list + i);</div><div class='del'>-			dseg++;</div><div class='del'>-		}</div><div class='add'>+		wqe_idx = (hr_qp-&gt;rq.head + nreq) &amp; (hr_qp-&gt;rq.wqe_cnt - 1);</div><div class='add'>+		fill_rq_wqe(hr_qp, wr, wqe_idx, max_sge);</div><div class='add'>+		hr_qp-&gt;rq.wrid[wqe_idx] = wr-&gt;wr_id;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	if (likely(nreq)) {</div><div class='add'>+		hr_qp-&gt;rq.head += nreq;</div><div class='add'>+</div><div class='add'>+		update_rq_db(hr_dev, hr_qp);</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;hr_qp-&gt;rq.lock, flags);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void *get_srq_wqe_buf(struct hns_roce_srq *srq, u32 n)</div><div class='add'>+{</div><div class='add'>+	return hns_roce_buf_offset(srq-&gt;buf_mtr.kmem, n &lt;&lt; srq-&gt;wqe_shift);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void *get_idx_buf(struct hns_roce_idx_que *idx_que, u32 n)</div><div class='add'>+{</div><div class='add'>+	return hns_roce_buf_offset(idx_que-&gt;mtr.kmem,</div><div class='add'>+				   n &lt;&lt; idx_que-&gt;entry_shift);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, u32 wqe_index)</div><div class='add'>+{</div><div class='add'>+	/* always called with interrupts disabled. */</div><div class='add'>+	spin_lock(&amp;srq-&gt;lock);</div><div class='ctx'> </div><div class='del'>-		if (i &lt; hr_qp-&gt;rq.max_gs) {</div><div class='del'>-			dseg-&gt;lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);</div><div class='del'>-			dseg-&gt;addr = 0;</div><div class='add'>+	bitmap_clear(srq-&gt;idx_que.bitmap, wqe_index, 1);</div><div class='add'>+	srq-&gt;idx_que.tail++;</div><div class='add'>+</div><div class='add'>+	spin_unlock(&amp;srq-&gt;lock);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_srqwq_overflow(struct hns_roce_srq *srq)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_idx_que *idx_que = &amp;srq-&gt;idx_que;</div><div class='add'>+</div><div class='add'>+	return idx_que-&gt;head - idx_que-&gt;tail &gt;= srq-&gt;wqe_cnt;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int check_post_srq_valid(struct hns_roce_srq *srq, u32 max_sge,</div><div class='add'>+				const struct ib_recv_wr *wr)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ib_dev = srq-&gt;ibsrq.device;</div><div class='add'>+</div><div class='add'>+	if (unlikely(wr-&gt;num_sge &gt; max_sge)) {</div><div class='add'>+		ibdev_err(ib_dev,</div><div class='add'>+			  "failed to check sge, wr-&gt;num_sge = %d, max_sge = %u.\n",</div><div class='add'>+			  wr-&gt;num_sge, max_sge);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (unlikely(hns_roce_srqwq_overflow(srq))) {</div><div class='add'>+		ibdev_err(ib_dev,</div><div class='add'>+			  "failed to check srqwq status, srqwq is full.\n");</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_srq_wqe_idx(struct hns_roce_srq *srq, u32 *wqe_idx)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_idx_que *idx_que = &amp;srq-&gt;idx_que;</div><div class='add'>+	u32 pos;</div><div class='add'>+</div><div class='add'>+	pos = find_first_zero_bit(idx_que-&gt;bitmap, srq-&gt;wqe_cnt);</div><div class='add'>+	if (unlikely(pos == srq-&gt;wqe_cnt))</div><div class='add'>+		return -ENOSPC;</div><div class='add'>+</div><div class='add'>+	bitmap_set(idx_que-&gt;bitmap, pos, 1);</div><div class='add'>+	*wqe_idx = pos;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void fill_wqe_idx(struct hns_roce_srq *srq, unsigned int wqe_idx)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_idx_que *idx_que = &amp;srq-&gt;idx_que;</div><div class='add'>+	unsigned int head;</div><div class='add'>+	__le32 *buf;</div><div class='add'>+</div><div class='add'>+	head = idx_que-&gt;head &amp; (srq-&gt;wqe_cnt - 1);</div><div class='add'>+</div><div class='add'>+	buf = get_idx_buf(idx_que, head);</div><div class='add'>+	*buf = cpu_to_le32(wqe_idx);</div><div class='add'>+</div><div class='add'>+	idx_que-&gt;head++;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void update_srq_db(struct hns_roce_srq *srq)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(srq-&gt;ibsrq.device);</div><div class='add'>+	struct hns_roce_v2_db db = {};</div><div class='add'>+</div><div class='add'>+	hr_reg_write(&amp;db, DB_TAG, srq-&gt;srqn);</div><div class='add'>+	hr_reg_write(&amp;db, DB_CMD, HNS_ROCE_V2_SRQ_DB);</div><div class='add'>+	hr_reg_write(&amp;db, DB_PI, srq-&gt;idx_que.head);</div><div class='add'>+</div><div class='add'>+	hns_roce_write64(hr_dev, (__le32 *)&amp;db, srq-&gt;db_reg);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,</div><div class='add'>+				     const struct ib_recv_wr *wr,</div><div class='add'>+				     const struct ib_recv_wr **bad_wr)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_srq *srq = to_hr_srq(ibsrq);</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int ret = 0;</div><div class='add'>+	u32 max_sge;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+	void *wqe;</div><div class='add'>+	u32 nreq;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;srq-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	max_sge = srq-&gt;max_gs - srq-&gt;rsv_sge;</div><div class='add'>+	for (nreq = 0; wr; ++nreq, wr = wr-&gt;next) {</div><div class='add'>+		ret = check_post_srq_valid(srq, max_sge, wr);</div><div class='add'>+		if (ret) {</div><div class='add'>+			*bad_wr = wr;</div><div class='add'>+			break;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		/* rq support inline data */</div><div class='del'>-		if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RQ_INLINE) {</div><div class='del'>-			sge_list = hr_qp-&gt;rq_inl_buf.wqe_list[ind].sg_list;</div><div class='del'>-			hr_qp-&gt;rq_inl_buf.wqe_list[ind].sge_cnt =</div><div class='del'>-							       (u32)wr-&gt;num_sge;</div><div class='del'>-			for (i = 0; i &lt; wr-&gt;num_sge; i++) {</div><div class='del'>-				sge_list[i].addr =</div><div class='del'>-					       (void *)(u64)wr-&gt;sg_list[i].addr;</div><div class='del'>-				sge_list[i].len = wr-&gt;sg_list[i].length;</div><div class='del'>-			}</div><div class='add'>+		ret = get_srq_wqe_idx(srq, &amp;wqe_idx);</div><div class='add'>+		if (unlikely(ret)) {</div><div class='add'>+			*bad_wr = wr;</div><div class='add'>+			break;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		hr_qp-&gt;rq.wrid[ind] = wr-&gt;wr_id;</div><div class='add'>+		wqe = get_srq_wqe_buf(srq, wqe_idx);</div><div class='add'>+		fill_recv_sge_to_wqe(wr, wqe, max_sge, srq-&gt;rsv_sge);</div><div class='add'>+		fill_wqe_idx(srq, wqe_idx);</div><div class='add'>+		srq-&gt;wrid[wqe_idx] = wr-&gt;wr_id;</div><div class='ctx'> </div><div class='del'>-		ind = (ind + 1) &amp; (hr_qp-&gt;rq.wqe_cnt - 1);</div><div class='add'>+		trace_hns_srq_wqe(srq-&gt;srqn, wqe_idx, wqe, 1 &lt;&lt; srq-&gt;wqe_shift,</div><div class='add'>+				  wr-&gt;wr_id, TRACE_SRQ);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-out:</div><div class='ctx'> 	if (likely(nreq)) {</div><div class='del'>-		hr_qp-&gt;rq.head += nreq;</div><div class='del'>-		/* Memory barrier */</div><div class='del'>-		wmb();</div><div class='del'>-</div><div class='del'>-		*hr_qp-&gt;rdb.db_record = hr_qp-&gt;rq.head &amp; 0xffff;</div><div class='del'>-</div><div class='del'>-		if (hr_qp-&gt;state == IB_QPS_ERR) {</div><div class='del'>-			attr_mask = IB_QP_STATE;</div><div class='del'>-			attr.qp_state = IB_QPS_ERR;</div><div class='del'>-</div><div class='del'>-			ret = hns_roce_v2_modify_qp(&amp;hr_qp-&gt;ibqp, &amp;attr,</div><div class='del'>-						    attr_mask, hr_qp-&gt;state,</div><div class='del'>-						    IB_QPS_ERR);</div><div class='del'>-			if (ret) {</div><div class='del'>-				spin_unlock_irqrestore(&amp;hr_qp-&gt;rq.lock, flags);</div><div class='del'>-				*bad_wr = wr;</div><div class='del'>-				return ret;</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='add'>+		if (srq-&gt;cap_flags &amp; HNS_ROCE_SRQ_CAP_RECORD_DB)</div><div class='add'>+			*srq-&gt;rdb.db_record = srq-&gt;idx_que.head &amp;</div><div class='add'>+					      V2_DB_PRODUCER_IDX_M;</div><div class='add'>+		else</div><div class='add'>+			update_srq_db(srq);</div><div class='ctx'> 	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;hr_qp-&gt;rq.lock, flags);</div><div class='add'>+</div><div class='add'>+	spin_unlock_irqrestore(&amp;srq-&gt;lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,</div><div class='add'>+static u32 hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 				      unsigned long instance_stage,</div><div class='ctx'> 				      unsigned long reset_stage)</div><div class='ctx'> {</div><div class='hunk'>@@ -737,13 +1039,18 @@ static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 	return CMD_RST_PRC_SUCCESS;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,</div><div class='add'>+static u32 hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 					unsigned long instance_stage,</div><div class='ctx'> 					unsigned long reset_stage)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev-&gt;priv;</div><div class='add'>+#define HW_RESET_TIMEOUT_US 1000000</div><div class='add'>+#define HW_RESET_SLEEP_US 1000</div><div class='add'>+</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='ctx'> 	struct hnae3_handle *handle = priv-&gt;handle;</div><div class='ctx'> 	const struct hnae3_ae_ops *ops = handle-&gt;ae_algo-&gt;ops;</div><div class='add'>+	unsigned long val;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	/* When hardware reset is detected, we should stop sending mailbox&amp;cmq&amp;</div><div class='ctx'> 	 * doorbell to hardware. If now in .init_instance() function, we should</div><div class='hunk'>@@ -755,7 +1062,11 @@ static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 	 * again.</div><div class='ctx'> 	 */</div><div class='ctx'> 	hr_dev-&gt;dis_db = true;</div><div class='del'>-	if (!ops-&gt;get_hw_reset_stat(handle))</div><div class='add'>+</div><div class='add'>+	ret = read_poll_timeout(ops-&gt;ae_dev_reset_cnt, val,</div><div class='add'>+				val &gt; hr_dev-&gt;reset_cnt, HW_RESET_SLEEP_US,</div><div class='add'>+				HW_RESET_TIMEOUT_US, false, handle);</div><div class='add'>+	if (!ret)</div><div class='ctx'> 		hr_dev-&gt;is_reset = true;</div><div class='ctx'> </div><div class='ctx'> 	if (!hr_dev-&gt;is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT ||</div><div class='hunk'>@@ -765,9 +1076,9 @@ static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 	return CMD_RST_PRC_SUCCESS;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)</div><div class='add'>+static u32 hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev-&gt;priv;</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='ctx'> 	struct hnae3_handle *handle = priv-&gt;handle;</div><div class='ctx'> 	const struct hnae3_ae_ops *ops = handle-&gt;ae_algo-&gt;ops;</div><div class='ctx'> </div><div class='hunk'>@@ -782,20 +1093,16 @@ static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)</div><div class='ctx'> 	return CMD_RST_PRC_EBUSY;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev)</div><div class='add'>+static u32 check_aedev_reset_status(struct hns_roce_dev *hr_dev,</div><div class='add'>+				    struct hnae3_handle *handle)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev-&gt;priv;</div><div class='del'>-	struct hnae3_handle *handle = priv-&gt;handle;</div><div class='ctx'> 	const struct hnae3_ae_ops *ops = handle-&gt;ae_algo-&gt;ops;</div><div class='del'>-	unsigned long instance_stage;	/* the current instance stage */</div><div class='del'>-	unsigned long reset_stage;	/* the current reset stage */</div><div class='add'>+	unsigned long instance_stage; /* the current instance stage */</div><div class='add'>+	unsigned long reset_stage; /* the current reset stage */</div><div class='ctx'> 	unsigned long reset_cnt;</div><div class='ctx'> 	bool sw_resetting;</div><div class='ctx'> 	bool hw_resetting;</div><div class='ctx'> </div><div class='del'>-	if (hr_dev-&gt;is_reset)</div><div class='del'>-		return CMD_RST_PRC_SUCCESS;</div><div class='del'>-</div><div class='ctx'> 	/* Get information about reset from NIC driver or RoCE driver itself,</div><div class='ctx'> 	 * the meaning of the following variables from NIC driver are described</div><div class='ctx'> 	 * as below:</div><div class='hunk'>@@ -806,28 +1113,53 @@ static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev)</div><div class='ctx'> 	instance_stage = handle-&gt;rinfo.instance_state;</div><div class='ctx'> 	reset_stage = handle-&gt;rinfo.reset_state;</div><div class='ctx'> 	reset_cnt = ops-&gt;ae_dev_reset_cnt(handle);</div><div class='del'>-	hw_resetting = ops-&gt;get_hw_reset_stat(handle);</div><div class='del'>-	sw_resetting = ops-&gt;ae_dev_resetting(handle);</div><div class='del'>-</div><div class='ctx'> 	if (reset_cnt != hr_dev-&gt;reset_cnt)</div><div class='ctx'> 		return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage,</div><div class='ctx'> 						  reset_stage);</div><div class='del'>-	else if (hw_resetting)</div><div class='add'>+</div><div class='add'>+	hw_resetting = ops-&gt;get_cmdq_stat(handle);</div><div class='add'>+	if (hw_resetting)</div><div class='ctx'> 		return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage,</div><div class='ctx'> 						    reset_stage);</div><div class='del'>-	else if (sw_resetting &amp;&amp; instance_stage == HNS_ROCE_STATE_INIT)</div><div class='add'>+</div><div class='add'>+	sw_resetting = ops-&gt;ae_dev_resetting(handle);</div><div class='add'>+	if (sw_resetting &amp;&amp; instance_stage == HNS_ROCE_STATE_INIT)</div><div class='ctx'> 		return hns_roce_v2_cmd_sw_resetting(hr_dev);</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	return CMD_RST_PRC_OTHERS;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring *ring)</div><div class='add'>+static bool check_device_is_in_reset(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	int ntu = ring-&gt;next_to_use;</div><div class='del'>-	int ntc = ring-&gt;next_to_clean;</div><div class='del'>-	int used = (ntu - ntc + ring-&gt;desc_num) % ring-&gt;desc_num;</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	struct hnae3_handle *handle = priv-&gt;handle;</div><div class='add'>+	const struct hnae3_ae_ops *ops = handle-&gt;ae_algo-&gt;ops;</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;reset_cnt != ops-&gt;ae_dev_reset_cnt(handle))</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	if (ops-&gt;get_hw_reset_stat(handle))</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	if (ops-&gt;ae_dev_resetting(handle))</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool v2_chk_mbox_is_avail(struct hns_roce_dev *hr_dev, bool *busy)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	u32 status;</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;is_reset)</div><div class='add'>+		status = CMD_RST_PRC_SUCCESS;</div><div class='add'>+	else</div><div class='add'>+		status = check_aedev_reset_status(hr_dev, priv-&gt;handle);</div><div class='add'>+</div><div class='add'>+	*busy = (status == CMD_RST_PRC_EBUSY);</div><div class='ctx'> </div><div class='del'>-	return ring-&gt;desc_num - used - 1;</div><div class='add'>+	return status == CMD_RST_PRC_OTHERS;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev,</div><div class='hunk'>@@ -835,124 +1167,71 @@ static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev,</div><div class='ctx'> {</div><div class='ctx'> 	int size = ring-&gt;desc_num * sizeof(struct hns_roce_cmq_desc);</div><div class='ctx'> </div><div class='del'>-	ring-&gt;desc = kzalloc(size, GFP_KERNEL);</div><div class='add'>+	ring-&gt;desc = dma_alloc_coherent(hr_dev-&gt;dev, size,</div><div class='add'>+					&amp;ring-&gt;desc_dma_addr, GFP_KERNEL);</div><div class='ctx'> 	if (!ring-&gt;desc)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	ring-&gt;desc_dma_addr = dma_map_single(hr_dev-&gt;dev, ring-&gt;desc, size,</div><div class='del'>-					     DMA_BIDIRECTIONAL);</div><div class='del'>-	if (dma_mapping_error(hr_dev-&gt;dev, ring-&gt;desc_dma_addr)) {</div><div class='del'>-		ring-&gt;desc_dma_addr = 0;</div><div class='del'>-		kfree(ring-&gt;desc);</div><div class='del'>-		ring-&gt;desc = NULL;</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 				   struct hns_roce_v2_cmq_ring *ring)</div><div class='ctx'> {</div><div class='del'>-	dma_unmap_single(hr_dev-&gt;dev, ring-&gt;desc_dma_addr,</div><div class='del'>-			 ring-&gt;desc_num * sizeof(struct hns_roce_cmq_desc),</div><div class='del'>-			 DMA_BIDIRECTIONAL);</div><div class='add'>+	dma_free_coherent(hr_dev-&gt;dev,</div><div class='add'>+			  ring-&gt;desc_num * sizeof(struct hns_roce_cmq_desc),</div><div class='add'>+			  ring-&gt;desc, ring-&gt;desc_dma_addr);</div><div class='ctx'> </div><div class='ctx'> 	ring-&gt;desc_dma_addr = 0;</div><div class='del'>-	kfree(ring-&gt;desc);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type)</div><div class='add'>+static int init_csq(struct hns_roce_dev *hr_dev,</div><div class='add'>+		    struct hns_roce_v2_cmq_ring *csq)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev-&gt;priv;</div><div class='del'>-	struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?</div><div class='del'>-					    &amp;priv-&gt;cmq.csq : &amp;priv-&gt;cmq.crq;</div><div class='add'>+	dma_addr_t dma;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	ring-&gt;flag = ring_type;</div><div class='del'>-	ring-&gt;next_to_clean = 0;</div><div class='del'>-	ring-&gt;next_to_use = 0;</div><div class='add'>+	csq-&gt;desc_num = CMD_CSQ_DESC_NUM;</div><div class='add'>+	spin_lock_init(&amp;csq-&gt;lock);</div><div class='add'>+	csq-&gt;flag = TYPE_CSQ;</div><div class='add'>+	csq-&gt;head = 0;</div><div class='ctx'> </div><div class='del'>-	return hns_roce_alloc_cmq_desc(hr_dev, ring);</div><div class='del'>-}</div><div class='add'>+	ret = hns_roce_alloc_cmq_desc(hr_dev, csq);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='del'>-static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev-&gt;priv;</div><div class='del'>-	struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?</div><div class='del'>-					    &amp;priv-&gt;cmq.csq : &amp;priv-&gt;cmq.crq;</div><div class='del'>-	dma_addr_t dma = ring-&gt;desc_dma_addr;</div><div class='del'>-</div><div class='del'>-	if (ring_type == TYPE_CSQ) {</div><div class='del'>-		roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, (u32)dma);</div><div class='del'>-		roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG,</div><div class='del'>-			   upper_32_bits(dma));</div><div class='del'>-		roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,</div><div class='del'>-			  (ring-&gt;desc_num &gt;&gt; HNS_ROCE_CMQ_DESC_NUM_S) |</div><div class='del'>-			   HNS_ROCE_CMQ_ENABLE);</div><div class='del'>-		roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0);</div><div class='del'>-		roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0);</div><div class='del'>-	} else {</div><div class='del'>-		roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_L_REG, (u32)dma);</div><div class='del'>-		roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG,</div><div class='del'>-			   upper_32_bits(dma));</div><div class='del'>-		roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG,</div><div class='del'>-			  (ring-&gt;desc_num &gt;&gt; HNS_ROCE_CMQ_DESC_NUM_S) |</div><div class='del'>-			   HNS_ROCE_CMQ_ENABLE);</div><div class='del'>-		roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0);</div><div class='del'>-		roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0);</div><div class='del'>-	}</div><div class='add'>+	dma = csq-&gt;desc_dma_addr;</div><div class='add'>+	roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, lower_32_bits(dma));</div><div class='add'>+	roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG, upper_32_bits(dma));</div><div class='add'>+	roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,</div><div class='add'>+		   (u32)csq-&gt;desc_num &gt;&gt; HNS_ROCE_CMQ_DESC_NUM_S);</div><div class='add'>+</div><div class='add'>+	/* Make sure to write CI first and then PI */</div><div class='add'>+	roce_write(hr_dev, ROCEE_TX_CMQ_CI_REG, 0);</div><div class='add'>+	roce_write(hr_dev, ROCEE_TX_CMQ_PI_REG, 0);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev-&gt;priv;</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	/* Setup the queue entries for command queue */</div><div class='del'>-	priv-&gt;cmq.csq.desc_num = CMD_CSQ_DESC_NUM;</div><div class='del'>-	priv-&gt;cmq.crq.desc_num = CMD_CRQ_DESC_NUM;</div><div class='del'>-</div><div class='del'>-	/* Setup the lock for command queue */</div><div class='del'>-	spin_lock_init(&amp;priv-&gt;cmq.csq.lock);</div><div class='del'>-	spin_lock_init(&amp;priv-&gt;cmq.crq.lock);</div><div class='del'>-</div><div class='del'>-	/* Setup Tx write back timeout */</div><div class='ctx'> 	priv-&gt;cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT;</div><div class='ctx'> </div><div class='del'>-	/* Init CSQ */</div><div class='del'>-	ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "Init CSQ error, ret = %d.\n", ret);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Init CRQ */</div><div class='del'>-	ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "Init CRQ error, ret = %d.\n", ret);</div><div class='del'>-		goto err_crq;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Init CSQ REG */</div><div class='del'>-	hns_roce_cmq_init_regs(hr_dev, TYPE_CSQ);</div><div class='del'>-</div><div class='del'>-	/* Init CRQ REG */</div><div class='del'>-	hns_roce_cmq_init_regs(hr_dev, TYPE_CRQ);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err_crq:</div><div class='del'>-	hns_roce_free_cmq_desc(hr_dev, &amp;priv-&gt;cmq.csq);</div><div class='add'>+	ret = init_csq(hr_dev, &amp;priv-&gt;cmq.csq);</div><div class='add'>+	if (ret)</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "failed to init CSQ, ret = %d.\n", ret);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev-&gt;priv;</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='ctx'> </div><div class='ctx'> 	hns_roce_free_cmq_desc(hr_dev, &amp;priv-&gt;cmq.csq);</div><div class='del'>-	hns_roce_free_cmq_desc(hr_dev, &amp;priv-&gt;cmq.crq);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,</div><div class='hunk'>@@ -961,8 +1240,7 @@ static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,</div><div class='ctx'> {</div><div class='ctx'> 	memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));</div><div class='ctx'> 	desc-&gt;opcode = cpu_to_le16(opcode);</div><div class='del'>-	desc-&gt;flag =</div><div class='del'>-		cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);</div><div class='add'>+	desc-&gt;flag = cpu_to_le16(HNS_ROCE_CMD_FLAG_IN);</div><div class='ctx'> 	if (is_read)</div><div class='ctx'> 		desc-&gt;flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR);</div><div class='ctx'> 	else</div><div class='hunk'>@@ -971,119 +1249,168 @@ static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev-&gt;priv;</div><div class='del'>-	u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);</div><div class='add'>+	u32 tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG);</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='ctx'> </div><div class='del'>-	return head == priv-&gt;cmq.csq.next_to_use;</div><div class='add'>+	return tail == priv-&gt;cmq.csq.head;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev)</div><div class='add'>+static void update_cmdq_status(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev-&gt;priv;</div><div class='del'>-	struct hns_roce_v2_cmq_ring *csq = &amp;priv-&gt;cmq.csq;</div><div class='del'>-	struct hns_roce_cmq_desc *desc;</div><div class='del'>-	u16 ntc = csq-&gt;next_to_clean;</div><div class='del'>-	u32 head;</div><div class='del'>-	int clean = 0;</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	struct hnae3_handle *handle = priv-&gt;handle;</div><div class='ctx'> </div><div class='del'>-	desc = &amp;csq-&gt;desc[ntc];</div><div class='del'>-	head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);</div><div class='del'>-	while (head != ntc) {</div><div class='del'>-		memset(desc, 0, sizeof(*desc));</div><div class='del'>-		ntc++;</div><div class='del'>-		if (ntc == csq-&gt;desc_num)</div><div class='del'>-			ntc = 0;</div><div class='del'>-		desc = &amp;csq-&gt;desc[ntc];</div><div class='del'>-		clean++;</div><div class='del'>-	}</div><div class='del'>-	csq-&gt;next_to_clean = ntc;</div><div class='add'>+	if (handle-&gt;rinfo.reset_state == HNS_ROCE_STATE_RST_INIT ||</div><div class='add'>+	    handle-&gt;rinfo.instance_state == HNS_ROCE_STATE_INIT)</div><div class='add'>+		hr_dev-&gt;cmd.state = HNS_ROCE_CMDQ_STATE_FATAL_ERR;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	return clean;</div><div class='add'>+static int hns_roce_cmd_err_convert_errno(u16 desc_ret)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_cmd_errcode errcode_table[] = {</div><div class='add'>+		{CMD_EXEC_SUCCESS, 0},</div><div class='add'>+		{CMD_NO_AUTH, -EPERM},</div><div class='add'>+		{CMD_NOT_EXIST, -EOPNOTSUPP},</div><div class='add'>+		{CMD_CRQ_FULL, -EXFULL},</div><div class='add'>+		{CMD_NEXT_ERR, -ENOSR},</div><div class='add'>+		{CMD_NOT_EXEC, -ENOTBLK},</div><div class='add'>+		{CMD_PARA_ERR, -EINVAL},</div><div class='add'>+		{CMD_RESULT_ERR, -ERANGE},</div><div class='add'>+		{CMD_TIMEOUT, -ETIME},</div><div class='add'>+		{CMD_HILINK_ERR, -ENOLINK},</div><div class='add'>+		{CMD_INFO_ILLEGAL, -ENXIO},</div><div class='add'>+		{CMD_INVALID, -EBADR},</div><div class='add'>+	};</div><div class='add'>+	u16 i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ARRAY_SIZE(errcode_table); i++)</div><div class='add'>+		if (desc_ret == errcode_table[i].return_status)</div><div class='add'>+			return errcode_table[i].errno;</div><div class='add'>+	return -EIO;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,</div><div class='del'>-			       struct hns_roce_cmq_desc *desc, int num)</div><div class='add'>+static u32 hns_roce_cmdq_tx_timeout(u16 opcode, u32 tx_timeout)</div><div class='add'>+{</div><div class='add'>+	static const struct hns_roce_cmdq_tx_timeout_map cmdq_tx_timeout[] = {</div><div class='add'>+		{HNS_ROCE_OPC_POST_MB, HNS_ROCE_OPC_POST_MB_TIMEOUT},</div><div class='add'>+	};</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ARRAY_SIZE(cmdq_tx_timeout); i++)</div><div class='add'>+		if (cmdq_tx_timeout[i].opcode == opcode)</div><div class='add'>+			return cmdq_tx_timeout[i].tx_timeout;</div><div class='add'>+</div><div class='add'>+	return tx_timeout;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_wait_csq_done(struct hns_roce_dev *hr_dev, u32 tx_timeout)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev-&gt;priv;</div><div class='del'>-	struct hns_roce_v2_cmq_ring *csq = &amp;priv-&gt;cmq.csq;</div><div class='del'>-	struct hns_roce_cmq_desc *desc_to_use;</div><div class='del'>-	bool complete = false;</div><div class='ctx'> 	u32 timeout = 0;</div><div class='del'>-	int handle = 0;</div><div class='del'>-	u16 desc_ret;</div><div class='del'>-	int ret = 0;</div><div class='del'>-	int ntc;</div><div class='ctx'> </div><div class='del'>-	spin_lock_bh(&amp;csq-&gt;lock);</div><div class='add'>+	do {</div><div class='add'>+		if (hns_roce_cmq_csq_done(hr_dev))</div><div class='add'>+			break;</div><div class='add'>+		udelay(1);</div><div class='add'>+	} while (++timeout &lt; tx_timeout);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (num &gt; hns_roce_cmq_space(csq)) {</div><div class='del'>-		spin_unlock_bh(&amp;csq-&gt;lock);</div><div class='del'>-		return -EBUSY;</div><div class='del'>-	}</div><div class='add'>+static int __hns_roce_cmq_send_one(struct hns_roce_dev *hr_dev,</div><div class='add'>+				   struct hns_roce_cmq_desc *desc,</div><div class='add'>+				   int num, u32 tx_timeout)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	struct hns_roce_v2_cmq_ring *csq = &amp;priv-&gt;cmq.csq;</div><div class='add'>+	u16 desc_ret;</div><div class='add'>+	u32 tail;</div><div class='add'>+	int ret;</div><div class='add'>+	int i;</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * Record the location of desc in the cmq for this time</div><div class='del'>-	 * which will be use for hardware to write back</div><div class='del'>-	 */</div><div class='del'>-	ntc = csq-&gt;next_to_use;</div><div class='add'>+	tail = csq-&gt;head;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; num; i++) {</div><div class='add'>+		trace_hns_cmdq_req(hr_dev, &amp;desc[i]);</div><div class='ctx'> </div><div class='del'>-	while (handle &lt; num) {</div><div class='del'>-		desc_to_use = &amp;csq-&gt;desc[csq-&gt;next_to_use];</div><div class='del'>-		*desc_to_use = desc[handle];</div><div class='del'>-		dev_dbg(hr_dev-&gt;dev, "set cmq desc:\n");</div><div class='del'>-		csq-&gt;next_to_use++;</div><div class='del'>-		if (csq-&gt;next_to_use == csq-&gt;desc_num)</div><div class='del'>-			csq-&gt;next_to_use = 0;</div><div class='del'>-		handle++;</div><div class='add'>+		csq-&gt;desc[csq-&gt;head++] = desc[i];</div><div class='add'>+		if (csq-&gt;head == csq-&gt;desc_num)</div><div class='add'>+			csq-&gt;head = 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Write to hardware */</div><div class='del'>-	roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, csq-&gt;next_to_use);</div><div class='add'>+	roce_write(hr_dev, ROCEE_TX_CMQ_PI_REG, csq-&gt;head);</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * If the command is sync, wait for the firmware to write back,</div><div class='del'>-	 * if multi descriptors to be sent, use the first one to check</div><div class='del'>-	 */</div><div class='del'>-	if ((desc-&gt;flag) &amp; HNS_ROCE_CMD_FLAG_NO_INTR) {</div><div class='del'>-		do {</div><div class='del'>-			if (hns_roce_cmq_csq_done(hr_dev))</div><div class='del'>-				break;</div><div class='del'>-			udelay(1);</div><div class='del'>-			timeout++;</div><div class='del'>-		} while (timeout &lt; priv-&gt;cmq.tx_timeout);</div><div class='del'>-	}</div><div class='add'>+	atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_CMDS_CNT]);</div><div class='ctx'> </div><div class='add'>+	hns_roce_wait_csq_done(hr_dev, tx_timeout);</div><div class='ctx'> 	if (hns_roce_cmq_csq_done(hr_dev)) {</div><div class='del'>-		complete = true;</div><div class='del'>-		handle = 0;</div><div class='del'>-		while (handle &lt; num) {</div><div class='del'>-			/* get the result of hardware write back */</div><div class='del'>-			desc_to_use = &amp;csq-&gt;desc[ntc];</div><div class='del'>-			desc[handle] = *desc_to_use;</div><div class='del'>-			dev_dbg(hr_dev-&gt;dev, "Get cmq desc:\n");</div><div class='del'>-			desc_ret = desc[handle].retval;</div><div class='del'>-			if (desc_ret == CMD_EXEC_SUCCESS)</div><div class='del'>-				ret = 0;</div><div class='del'>-			else</div><div class='del'>-				ret = -EIO;</div><div class='del'>-			priv-&gt;cmq.last_status = desc_ret;</div><div class='del'>-			ntc++;</div><div class='del'>-			handle++;</div><div class='del'>-			if (ntc == csq-&gt;desc_num)</div><div class='del'>-				ntc = 0;</div><div class='add'>+		ret = 0;</div><div class='add'>+		for (i = 0; i &lt; num; i++) {</div><div class='add'>+			trace_hns_cmdq_resp(hr_dev, &amp;csq-&gt;desc[tail]);</div><div class='add'>+</div><div class='add'>+			/* check the result of hardware write back */</div><div class='add'>+			desc_ret = le16_to_cpu(csq-&gt;desc[tail++].retval);</div><div class='add'>+			if (tail == csq-&gt;desc_num)</div><div class='add'>+				tail = 0;</div><div class='add'>+			if (likely(desc_ret == CMD_EXEC_SUCCESS))</div><div class='add'>+				continue;</div><div class='add'>+</div><div class='add'>+			ret = hns_roce_cmd_err_convert_errno(desc_ret);</div><div class='ctx'> 		}</div><div class='del'>-	}</div><div class='add'>+	} else {</div><div class='add'>+		/* FW/HW reset or incorrect number of desc */</div><div class='add'>+		tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG);</div><div class='add'>+		dev_warn(hr_dev-&gt;dev, "CMDQ move tail from %u to %u.\n",</div><div class='add'>+			 csq-&gt;head, tail);</div><div class='add'>+		csq-&gt;head = tail;</div><div class='add'>+</div><div class='add'>+		update_cmdq_status(hr_dev);</div><div class='ctx'> </div><div class='del'>-	if (!complete)</div><div class='ctx'> 		ret = -EAGAIN;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ret)</div><div class='add'>+		atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_CMDS_ERR_CNT]);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,</div><div class='add'>+			       struct hns_roce_cmq_desc *desc, int num)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	struct hns_roce_v2_cmq_ring *csq = &amp;priv-&gt;cmq.csq;</div><div class='add'>+	u16 opcode = le16_to_cpu(desc-&gt;opcode);</div><div class='add'>+	u32 tx_timeout = hns_roce_cmdq_tx_timeout(opcode, priv-&gt;cmq.tx_timeout);</div><div class='add'>+	u8 try_cnt = HNS_ROCE_OPC_POST_MB_TRY_CNT;</div><div class='add'>+	u32 rsv_tail;</div><div class='add'>+	int ret;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	while (try_cnt) {</div><div class='add'>+		try_cnt--;</div><div class='add'>+</div><div class='add'>+		spin_lock_bh(&amp;csq-&gt;lock);</div><div class='add'>+		rsv_tail = csq-&gt;head;</div><div class='add'>+		ret = __hns_roce_cmq_send_one(hr_dev, desc, num, tx_timeout);</div><div class='add'>+		if (opcode == HNS_ROCE_OPC_POST_MB &amp;&amp; ret == -ETIME &amp;&amp;</div><div class='add'>+		    try_cnt) {</div><div class='add'>+			spin_unlock_bh(&amp;csq-&gt;lock);</div><div class='add'>+			mdelay(HNS_ROCE_OPC_POST_MB_RETRY_GAP_MSEC);</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='del'>-	/* clean the command send queue */</div><div class='del'>-	handle = hns_roce_cmq_csq_clean(hr_dev);</div><div class='del'>-	if (handle != num)</div><div class='del'>-		dev_warn(hr_dev-&gt;dev, "Cleaned %d, need to clean %d\n",</div><div class='del'>-			 handle, num);</div><div class='add'>+		for (i = 0; i &lt; num; i++) {</div><div class='add'>+			desc[i] = csq-&gt;desc[rsv_tail++];</div><div class='add'>+			if (rsv_tail == csq-&gt;desc_num)</div><div class='add'>+				rsv_tail = 0;</div><div class='add'>+		}</div><div class='add'>+		spin_unlock_bh(&amp;csq-&gt;lock);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	spin_unlock_bh(&amp;csq-&gt;lock);</div><div class='add'>+	if (ret)</div><div class='add'>+		dev_err_ratelimited(hr_dev-&gt;dev,</div><div class='add'>+				    "Cmdq IO error, opcode = 0x%x, return = %d.\n",</div><div class='add'>+				    opcode, ret);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -1091,27 +1418,112 @@ static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,</div><div class='ctx'> static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 			     struct hns_roce_cmq_desc *desc, int num)</div><div class='ctx'> {</div><div class='del'>-	int retval;</div><div class='add'>+	bool busy;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_v2_rst_process_cmd(hr_dev);</div><div class='del'>-	if (ret == CMD_RST_PRC_SUCCESS)</div><div class='del'>-		return 0;</div><div class='del'>-	if (ret == CMD_RST_PRC_EBUSY)</div><div class='del'>-		return ret;</div><div class='add'>+	if (hr_dev-&gt;cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR)</div><div class='add'>+		return -EIO;</div><div class='add'>+</div><div class='add'>+	if (!v2_chk_mbox_is_avail(hr_dev, &amp;busy))</div><div class='add'>+		return busy ? -EBUSY : 0;</div><div class='ctx'> </div><div class='ctx'> 	ret = __hns_roce_cmq_send(hr_dev, desc, num);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		retval = hns_roce_v2_rst_process_cmd(hr_dev);</div><div class='del'>-		if (retval == CMD_RST_PRC_SUCCESS)</div><div class='del'>-			return 0;</div><div class='del'>-		else if (retval == CMD_RST_PRC_EBUSY)</div><div class='del'>-			return retval;</div><div class='add'>+		if (!v2_chk_mbox_is_avail(hr_dev, &amp;busy))</div><div class='add'>+			return busy ? -EBUSY : 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static enum hns_roce_opcode_type</div><div class='add'>+	get_bond_opcode(enum hns_roce_bond_cmd_type bond_type)</div><div class='add'>+{</div><div class='add'>+	switch (bond_type) {</div><div class='add'>+	case HNS_ROCE_SET_BOND:</div><div class='add'>+		return HNS_ROCE_OPC_SET_BOND_INFO;</div><div class='add'>+	case HNS_ROCE_CHANGE_BOND:</div><div class='add'>+		return HNS_ROCE_OPC_CHANGE_ACTIVE_PORT;</div><div class='add'>+	case HNS_ROCE_CLEAR_BOND:</div><div class='add'>+		return HNS_ROCE_OPC_CLEAR_BOND_INFO;</div><div class='add'>+	default:</div><div class='add'>+		WARN(true, "Invalid bond type %d!\n", bond_type);</div><div class='add'>+		return HNS_ROCE_OPC_SET_BOND_INFO;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static enum hns_roce_bond_hashtype</div><div class='add'>+	get_bond_hashtype(enum netdev_lag_hash netdev_hashtype)</div><div class='add'>+{</div><div class='add'>+	switch (netdev_hashtype) {</div><div class='add'>+	case NETDEV_LAG_HASH_L2:</div><div class='add'>+		return BOND_HASH_L2;</div><div class='add'>+	case NETDEV_LAG_HASH_L34:</div><div class='add'>+		return BOND_HASH_L34;</div><div class='add'>+	case NETDEV_LAG_HASH_L23:</div><div class='add'>+		return BOND_HASH_L23;</div><div class='add'>+	default:</div><div class='add'>+		WARN(true, "Invalid hash type %d!\n", netdev_hashtype);</div><div class='add'>+		return BOND_HASH_L2;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hns_roce_cmd_bond(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+		      enum hns_roce_bond_cmd_type bond_type)</div><div class='add'>+{</div><div class='add'>+	enum hns_roce_opcode_type opcode = get_bond_opcode(bond_type);</div><div class='add'>+	struct hns_roce_bond_info *slave_info;</div><div class='add'>+	struct hns_roce_cmq_desc desc = {};</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	slave_info = (struct hns_roce_bond_info *)desc.data;</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc, opcode, false);</div><div class='add'>+</div><div class='add'>+	slave_info-&gt;bond_id = cpu_to_le32(bond_grp-&gt;bond_id);</div><div class='add'>+	if (bond_type == HNS_ROCE_CLEAR_BOND)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	if (bond_grp-&gt;tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {</div><div class='add'>+		slave_info-&gt;bond_mode = cpu_to_le32(BOND_MODE_1);</div><div class='add'>+		if (bond_grp-&gt;active_slave_num != 1)</div><div class='add'>+			ibdev_warn(&amp;bond_grp-&gt;main_hr_dev-&gt;ib_dev,</div><div class='add'>+				   "active slave cnt(%u) in Mode 1 is invalid.\n",</div><div class='add'>+				   bond_grp-&gt;active_slave_num);</div><div class='add'>+	} else {</div><div class='add'>+		slave_info-&gt;bond_mode = cpu_to_le32(BOND_MODE_2_4);</div><div class='add'>+		slave_info-&gt;hash_policy =</div><div class='add'>+			cpu_to_le32(get_bond_hashtype(bond_grp-&gt;hash_type));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	slave_info-&gt;active_slave_cnt = cpu_to_le32(bond_grp-&gt;active_slave_num);</div><div class='add'>+	slave_info-&gt;active_slave_mask = cpu_to_le32(bond_grp-&gt;active_slave_map);</div><div class='add'>+	slave_info-&gt;slave_mask = cpu_to_le32(bond_grp-&gt;slave_map);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	ret = hns_roce_cmq_send(bond_grp-&gt;main_hr_dev, &amp;desc, 1);</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_err(&amp;bond_grp-&gt;main_hr_dev-&gt;ib_dev,</div><div class='add'>+			  "cmq bond type(%d) failed, ret = %d.\n",</div><div class='add'>+			  bond_type, ret);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int config_hem_ba_to_hw(struct hns_roce_dev *hr_dev,</div><div class='add'>+			       dma_addr_t base_addr, u8 cmd, unsigned long tag)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_cmd_mailbox *mbox;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	mbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='add'>+	if (IS_ERR(mbox))</div><div class='add'>+		return PTR_ERR(mbox);</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_cmd_mbox(hr_dev, base_addr, mbox-&gt;dma, cmd, tag);</div><div class='add'>+	hns_roce_free_cmd_mailbox(hr_dev, mbox);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_query_version *resp;</div><div class='hunk'>@@ -1124,12 +1536,198 @@ static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='ctx'> 	resp = (struct hns_roce_query_version *)desc.data;</div><div class='del'>-	hr_dev-&gt;hw_rev = le32_to_cpu(resp-&gt;rocee_hw_version);</div><div class='add'>+	hr_dev-&gt;hw_rev = le16_to_cpu(resp-&gt;rocee_hw_version);</div><div class='ctx'> 	hr_dev-&gt;vendor_id = hr_dev-&gt;pci_dev-&gt;vendor;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void func_clr_hw_resetting_state(struct hns_roce_dev *hr_dev,</div><div class='add'>+					struct hnae3_handle *handle)</div><div class='add'>+{</div><div class='add'>+	const struct hnae3_ae_ops *ops = handle-&gt;ae_algo-&gt;ops;</div><div class='add'>+	unsigned long end;</div><div class='add'>+</div><div class='add'>+	hr_dev-&gt;dis_db = true;</div><div class='add'>+</div><div class='add'>+	dev_warn(hr_dev-&gt;dev,</div><div class='add'>+		 "func clear is pending, device in resetting state.\n");</div><div class='add'>+	end = HNS_ROCE_V2_HW_RST_TIMEOUT;</div><div class='add'>+	while (end) {</div><div class='add'>+		if (!ops-&gt;get_hw_reset_stat(handle)) {</div><div class='add'>+			hr_dev-&gt;is_reset = true;</div><div class='add'>+			dev_info(hr_dev-&gt;dev,</div><div class='add'>+				 "func clear success after reset.\n");</div><div class='add'>+			return;</div><div class='add'>+		}</div><div class='add'>+		msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);</div><div class='add'>+		end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dev_warn(hr_dev-&gt;dev, "func clear failed.\n");</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void func_clr_sw_resetting_state(struct hns_roce_dev *hr_dev,</div><div class='add'>+					struct hnae3_handle *handle)</div><div class='add'>+{</div><div class='add'>+	const struct hnae3_ae_ops *ops = handle-&gt;ae_algo-&gt;ops;</div><div class='add'>+	unsigned long end;</div><div class='add'>+</div><div class='add'>+	hr_dev-&gt;dis_db = true;</div><div class='add'>+</div><div class='add'>+	dev_warn(hr_dev-&gt;dev,</div><div class='add'>+		 "func clear is pending, device in resetting state.\n");</div><div class='add'>+	end = HNS_ROCE_V2_HW_RST_TIMEOUT;</div><div class='add'>+	while (end) {</div><div class='add'>+		if (ops-&gt;ae_dev_reset_cnt(handle) !=</div><div class='add'>+		    hr_dev-&gt;reset_cnt) {</div><div class='add'>+			hr_dev-&gt;is_reset = true;</div><div class='add'>+			dev_info(hr_dev-&gt;dev,</div><div class='add'>+				 "func clear success after sw reset\n");</div><div class='add'>+			return;</div><div class='add'>+		}</div><div class='add'>+		msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);</div><div class='add'>+		end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dev_warn(hr_dev-&gt;dev, "func clear failed because of unfinished sw reset\n");</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_func_clr_rst_proc(struct hns_roce_dev *hr_dev, int retval,</div><div class='add'>+				       int flag)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	struct hnae3_handle *handle = priv-&gt;handle;</div><div class='add'>+	const struct hnae3_ae_ops *ops = handle-&gt;ae_algo-&gt;ops;</div><div class='add'>+</div><div class='add'>+	if (ops-&gt;ae_dev_reset_cnt(handle) != hr_dev-&gt;reset_cnt) {</div><div class='add'>+		hr_dev-&gt;dis_db = true;</div><div class='add'>+		hr_dev-&gt;is_reset = true;</div><div class='add'>+		dev_info(hr_dev-&gt;dev, "func clear success after reset.\n");</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ops-&gt;get_hw_reset_stat(handle)) {</div><div class='add'>+		func_clr_hw_resetting_state(hr_dev, handle);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ops-&gt;ae_dev_resetting(handle) &amp;&amp;</div><div class='add'>+	    handle-&gt;rinfo.instance_state == HNS_ROCE_STATE_INIT) {</div><div class='add'>+		func_clr_sw_resetting_state(hr_dev, handle);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (retval &amp;&amp; !flag)</div><div class='add'>+		dev_warn(hr_dev-&gt;dev,</div><div class='add'>+			 "func clear read failed, ret = %d.\n", retval);</div><div class='add'>+</div><div class='add'>+	dev_warn(hr_dev-&gt;dev, "func clear failed.\n");</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void __hns_roce_function_clear(struct hns_roce_dev *hr_dev, int vf_id)</div><div class='add'>+{</div><div class='add'>+	bool fclr_write_fail_flag = false;</div><div class='add'>+	struct hns_roce_func_clear *resp;</div><div class='add'>+	struct hns_roce_cmq_desc desc;</div><div class='add'>+	unsigned long end;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	if (check_device_is_in_reset(hr_dev))</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_OPC_FUNC_CLEAR, false);</div><div class='add'>+	resp = (struct hns_roce_func_clear *)desc.data;</div><div class='add'>+	resp-&gt;rst_funcid_en = cpu_to_le32(vf_id);</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='add'>+	if (ret) {</div><div class='add'>+		fclr_write_fail_flag = true;</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "func clear write failed, ret = %d.\n",</div><div class='add'>+			 ret);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL);</div><div class='add'>+	end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS;</div><div class='add'>+	while (end) {</div><div class='add'>+		if (check_device_is_in_reset(hr_dev))</div><div class='add'>+			goto out;</div><div class='add'>+		msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT);</div><div class='add'>+		end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT;</div><div class='add'>+</div><div class='add'>+		hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_OPC_FUNC_CLEAR,</div><div class='add'>+					      true);</div><div class='add'>+</div><div class='add'>+		resp-&gt;rst_funcid_en = cpu_to_le32(vf_id);</div><div class='add'>+		ret = hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='add'>+		if (ret)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		if (hr_reg_read(resp, FUNC_CLEAR_RST_FUN_DONE)) {</div><div class='add'>+			if (vf_id == 0)</div><div class='add'>+				hr_dev-&gt;is_reset = true;</div><div class='add'>+			return;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	hns_roce_func_clr_rst_proc(hr_dev, ret, fclr_write_fail_flag);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_free_vf_resource(struct hns_roce_dev *hr_dev, int vf_id)</div><div class='add'>+{</div><div class='add'>+	enum hns_roce_opcode_type opcode = HNS_ROCE_OPC_ALLOC_VF_RES;</div><div class='add'>+	struct hns_roce_cmq_desc desc[2];</div><div class='add'>+	struct hns_roce_cmq_req *req_a;</div><div class='add'>+</div><div class='add'>+	req_a = (struct hns_roce_cmq_req *)desc[0].data;</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc[0], opcode, false);</div><div class='add'>+	desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc[1], opcode, false);</div><div class='add'>+	hr_reg_write(req_a, FUNC_RES_A_VF_ID, vf_id);</div><div class='add'>+</div><div class='add'>+	return hns_roce_cmq_send(hr_dev, desc, 2);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_function_clear(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	for (i = hr_dev-&gt;func_num - 1; i &gt;= 0; i--) {</div><div class='add'>+		__hns_roce_function_clear(hr_dev, i);</div><div class='add'>+</div><div class='add'>+		if (i == 0)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		ret = hns_roce_free_vf_resource(hr_dev, i);</div><div class='add'>+		if (ret)</div><div class='add'>+			ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+				  "failed to free vf resource, vf_id = %d, ret = %d.\n",</div><div class='add'>+				  i, ret);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_clear_extdb_list_info(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_cmq_desc desc;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_OPC_CLEAR_EXTDB_LIST_INFO,</div><div class='add'>+				      false);</div><div class='add'>+	ret = hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+			  "failed to clear extended doorbell info, ret = %d.\n",</div><div class='add'>+			  ret);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_query_fw_info *resp;</div><div class='hunk'>@@ -1147,707 +1745,1279 @@ static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int hns_roce_query_func_info(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_cmq_desc desc;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08) {</div><div class='add'>+		hr_dev-&gt;func_num = 1;</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_OPC_QUERY_FUNC_INFO,</div><div class='add'>+				      true);</div><div class='add'>+	ret = hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='add'>+	if (ret) {</div><div class='add'>+		hr_dev-&gt;func_num = 1;</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hr_dev-&gt;func_num = le32_to_cpu(desc.func_info.own_func_num);</div><div class='add'>+	hr_dev-&gt;cong_algo_tmpl_id = le32_to_cpu(desc.func_info.own_mac_id);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_hw_v2_query_counter(struct hns_roce_dev *hr_dev,</div><div class='add'>+					u64 *stats, u32 port, int *num_counters)</div><div class='add'>+{</div><div class='add'>+#define CNT_PER_DESC 3</div><div class='add'>+	struct hns_roce_cmq_desc *desc;</div><div class='add'>+	int bd_idx, cnt_idx;</div><div class='add'>+	__le64 *cnt_data;</div><div class='add'>+	int desc_num;</div><div class='add'>+	int ret;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (port &gt; hr_dev-&gt;caps.num_ports)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	desc_num = DIV_ROUND_UP(HNS_ROCE_HW_CNT_TOTAL, CNT_PER_DESC);</div><div class='add'>+	desc = kcalloc(desc_num, sizeof(*desc), GFP_KERNEL);</div><div class='add'>+	if (!desc)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; desc_num; i++) {</div><div class='add'>+		hns_roce_cmq_setup_basic_desc(&amp;desc[i],</div><div class='add'>+					      HNS_ROCE_OPC_QUERY_COUNTER, true);</div><div class='add'>+		if (i != desc_num - 1)</div><div class='add'>+			desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_cmq_send(hr_dev, desc, desc_num);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+			  "failed to get counter, ret = %d.\n", ret);</div><div class='add'>+		goto err_out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; HNS_ROCE_HW_CNT_TOTAL &amp;&amp; i &lt; *num_counters; i++) {</div><div class='add'>+		bd_idx = i / CNT_PER_DESC;</div><div class='add'>+		if (bd_idx != HNS_ROCE_HW_CNT_TOTAL / CNT_PER_DESC &amp;&amp;</div><div class='add'>+		    !(desc[bd_idx].flag &amp; cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT)))</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		cnt_data = (__le64 *)&amp;desc[bd_idx].data[0];</div><div class='add'>+		cnt_idx = i % CNT_PER_DESC;</div><div class='add'>+		stats[i] = le64_to_cpu(cnt_data[cnt_idx]);</div><div class='add'>+	}</div><div class='add'>+	*num_counters = i;</div><div class='add'>+</div><div class='add'>+err_out:</div><div class='add'>+	kfree(desc);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_cfg_global_param *req;</div><div class='ctx'> 	struct hns_roce_cmq_desc desc;</div><div class='add'>+	struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;</div><div class='add'>+	u32 clock_cycles_of_1us;</div><div class='ctx'> </div><div class='ctx'> 	hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM,</div><div class='ctx'> 				      false);</div><div class='ctx'> </div><div class='del'>-	req = (struct hns_roce_cfg_global_param *)desc.data;</div><div class='del'>-	memset(req, 0, sizeof(*req));</div><div class='del'>-	roce_set_field(req-&gt;time_cfg_udp_port,</div><div class='del'>-		       CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M,</div><div class='del'>-		       CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S, 0x3e8);</div><div class='del'>-	roce_set_field(req-&gt;time_cfg_udp_port,</div><div class='del'>-		       CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M,</div><div class='del'>-		       CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S, 0x12b7);</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08)</div><div class='add'>+		clock_cycles_of_1us = HNS_ROCE_1NS_CFG;</div><div class='add'>+	else</div><div class='add'>+		clock_cycles_of_1us = HNS_ROCE_1US_CFG;</div><div class='add'>+</div><div class='add'>+	hr_reg_write(req, CFG_GLOBAL_PARAM_1US_CYCLES, clock_cycles_of_1us);</div><div class='add'>+	hr_reg_write(req, CFG_GLOBAL_PARAM_UDP_PORT, ROCE_V2_UDP_DPORT);</div><div class='ctx'> </div><div class='ctx'> 	return hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)</div><div class='add'>+static int load_func_res_caps(struct hns_roce_dev *hr_dev, bool is_vf)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_cmq_desc desc[2];</div><div class='del'>-	struct hns_roce_pf_res_a *req_a;</div><div class='del'>-	struct hns_roce_pf_res_b *req_b;</div><div class='add'>+	struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data;</div><div class='add'>+	struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data;</div><div class='add'>+	struct hns_roce_caps *caps = &amp;hr_dev-&gt;caps;</div><div class='add'>+	enum hns_roce_opcode_type opcode;</div><div class='add'>+	u32 func_num;</div><div class='ctx'> 	int ret;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; 2; i++) {</div><div class='del'>-		hns_roce_cmq_setup_basic_desc(&amp;desc[i],</div><div class='del'>-					      HNS_ROCE_OPC_QUERY_PF_RES, true);</div><div class='ctx'> </div><div class='del'>-		if (i == 0)</div><div class='del'>-			desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);</div><div class='del'>-		else</div><div class='del'>-			desc[i].flag &amp;= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);</div><div class='add'>+	if (is_vf) {</div><div class='add'>+		opcode = HNS_ROCE_OPC_QUERY_VF_RES;</div><div class='add'>+		func_num = 1;</div><div class='add'>+	} else {</div><div class='add'>+		opcode = HNS_ROCE_OPC_QUERY_PF_RES;</div><div class='add'>+		func_num = hr_dev-&gt;func_num;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc[0], opcode, true);</div><div class='add'>+	desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc[1], opcode, true);</div><div class='add'>+</div><div class='ctx'> 	ret = hns_roce_cmq_send(hr_dev, desc, 2);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	req_a = (struct hns_roce_pf_res_a *)desc[0].data;</div><div class='del'>-	req_b = (struct hns_roce_pf_res_b *)desc[1].data;</div><div class='del'>-</div><div class='del'>-	hr_dev-&gt;caps.qpc_bt_num = roce_get_field(req_a-&gt;qpc_bt_idx_num,</div><div class='del'>-						 PF_RES_DATA_1_PF_QPC_BT_NUM_M,</div><div class='del'>-						 PF_RES_DATA_1_PF_QPC_BT_NUM_S);</div><div class='del'>-	hr_dev-&gt;caps.srqc_bt_num = roce_get_field(req_a-&gt;srqc_bt_idx_num,</div><div class='del'>-						PF_RES_DATA_2_PF_SRQC_BT_NUM_M,</div><div class='del'>-						PF_RES_DATA_2_PF_SRQC_BT_NUM_S);</div><div class='del'>-	hr_dev-&gt;caps.cqc_bt_num = roce_get_field(req_a-&gt;cqc_bt_idx_num,</div><div class='del'>-						 PF_RES_DATA_3_PF_CQC_BT_NUM_M,</div><div class='del'>-						 PF_RES_DATA_3_PF_CQC_BT_NUM_S);</div><div class='del'>-	hr_dev-&gt;caps.mpt_bt_num = roce_get_field(req_a-&gt;mpt_bt_idx_num,</div><div class='del'>-						 PF_RES_DATA_4_PF_MPT_BT_NUM_M,</div><div class='del'>-						 PF_RES_DATA_4_PF_MPT_BT_NUM_S);</div><div class='del'>-</div><div class='del'>-	hr_dev-&gt;caps.sl_num = roce_get_field(req_b-&gt;qid_idx_sl_num,</div><div class='del'>-					     PF_RES_DATA_3_PF_SL_NUM_M,</div><div class='del'>-					     PF_RES_DATA_3_PF_SL_NUM_S);</div><div class='del'>-	hr_dev-&gt;caps.sccc_bt_num = roce_get_field(req_b-&gt;sccc_bt_idx_num,</div><div class='del'>-					     PF_RES_DATA_4_PF_SCCC_BT_NUM_M,</div><div class='del'>-					     PF_RES_DATA_4_PF_SCCC_BT_NUM_S);</div><div class='add'>+	caps-&gt;qpc_bt_num = hr_reg_read(r_a, FUNC_RES_A_QPC_BT_NUM) / func_num;</div><div class='add'>+	caps-&gt;srqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_SRQC_BT_NUM) / func_num;</div><div class='add'>+	caps-&gt;cqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_CQC_BT_NUM) / func_num;</div><div class='add'>+	caps-&gt;mpt_bt_num = hr_reg_read(r_a, FUNC_RES_A_MPT_BT_NUM) / func_num;</div><div class='add'>+	caps-&gt;eqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_EQC_BT_NUM) / func_num;</div><div class='add'>+	caps-&gt;smac_bt_num = hr_reg_read(r_b, FUNC_RES_B_SMAC_NUM) / func_num;</div><div class='add'>+	caps-&gt;sgid_bt_num = hr_reg_read(r_b, FUNC_RES_B_SGID_NUM) / func_num;</div><div class='add'>+	caps-&gt;sccc_bt_num = hr_reg_read(r_b, FUNC_RES_B_SCCC_BT_NUM) / func_num;</div><div class='add'>+</div><div class='add'>+	if (is_vf) {</div><div class='add'>+		caps-&gt;sl_num = hr_reg_read(r_b, FUNC_RES_V_QID_NUM) / func_num;</div><div class='add'>+		caps-&gt;gmv_bt_num = hr_reg_read(r_b, FUNC_RES_V_GMV_BT_NUM) /</div><div class='add'>+					       func_num;</div><div class='add'>+	} else {</div><div class='add'>+		caps-&gt;sl_num = hr_reg_read(r_b, FUNC_RES_B_QID_NUM) / func_num;</div><div class='add'>+		caps-&gt;gmv_bt_num = hr_reg_read(r_b, FUNC_RES_B_GMV_BT_NUM) /</div><div class='add'>+					       func_num;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev)</div><div class='add'>+static int load_pf_timer_res_caps(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_pf_timer_res_a *req_a;</div><div class='del'>-	struct hns_roce_cmq_desc desc[2];</div><div class='del'>-	int ret, i;</div><div class='add'>+	struct hns_roce_cmq_desc desc;</div><div class='add'>+	struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;</div><div class='add'>+	struct hns_roce_caps *caps = &amp;hr_dev-&gt;caps;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; 2; i++) {</div><div class='del'>-		hns_roce_cmq_setup_basic_desc(&amp;desc[i],</div><div class='del'>-					      HNS_ROCE_OPC_QUERY_PF_TIMER_RES,</div><div class='del'>-					      true);</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_OPC_QUERY_PF_TIMER_RES,</div><div class='add'>+				      true);</div><div class='ctx'> </div><div class='del'>-		if (i == 0)</div><div class='del'>-			desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);</div><div class='del'>-		else</div><div class='del'>-			desc[i].flag &amp;= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);</div><div class='add'>+	ret = hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	caps-&gt;qpc_timer_bt_num = hr_reg_read(req, PF_TIMER_RES_QPC_ITEM_NUM);</div><div class='add'>+	caps-&gt;cqc_timer_bt_num = hr_reg_read(req, PF_TIMER_RES_CQC_ITEM_NUM);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = load_func_res_caps(hr_dev, false);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(dev, "failed to load pf res caps, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_cmq_send(hr_dev, desc, 2);</div><div class='add'>+	ret = load_pf_timer_res_caps(hr_dev);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		return ret;</div><div class='add'>+		dev_err(dev, "failed to load pf timer resource, ret = %d.\n",</div><div class='add'>+			ret);</div><div class='ctx'> </div><div class='del'>-	req_a = (struct hns_roce_pf_timer_res_a *)desc[0].data;</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	hr_dev-&gt;caps.qpc_timer_bt_num =</div><div class='del'>-				roce_get_field(req_a-&gt;qpc_timer_bt_idx_num,</div><div class='del'>-					PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M,</div><div class='del'>-					PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S);</div><div class='del'>-	hr_dev-&gt;caps.cqc_timer_bt_num =</div><div class='del'>-				roce_get_field(req_a-&gt;cqc_timer_bt_idx_num,</div><div class='del'>-					PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M,</div><div class='del'>-					PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S);</div><div class='add'>+static int hns_roce_query_vf_resource(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	ret = load_func_res_caps(hr_dev, true);</div><div class='add'>+	if (ret)</div><div class='add'>+		dev_err(dev, "failed to load vf res caps, ret = %d.\n", ret);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev,</div><div class='del'>-						  int vf_id)</div><div class='add'>+static int __hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev,</div><div class='add'>+					  u32 vf_id)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_cmq_desc desc;</div><div class='ctx'> 	struct hns_roce_vf_switch *swt;</div><div class='add'>+	struct hns_roce_cmq_desc desc;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	swt = (struct hns_roce_vf_switch *)desc.data;</div><div class='ctx'> 	hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_SWITCH_PARAMETER_CFG, true);</div><div class='del'>-	swt-&gt;rocee_sel |= cpu_to_le16(HNS_ICL_SWITCH_CMD_ROCEE_SEL);</div><div class='del'>-	roce_set_field(swt-&gt;fun_id,</div><div class='del'>-			VF_SWITCH_DATA_FUN_ID_VF_ID_M,</div><div class='del'>-			VF_SWITCH_DATA_FUN_ID_VF_ID_S,</div><div class='del'>-			vf_id);</div><div class='add'>+	swt-&gt;rocee_sel |= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL);</div><div class='add'>+	hr_reg_write(swt, VF_SWITCH_VF_ID, vf_id);</div><div class='ctx'> 	ret = hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='del'>-	desc.flag =</div><div class='del'>-		cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);</div><div class='add'>+</div><div class='add'>+	desc.flag = cpu_to_le16(HNS_ROCE_CMD_FLAG_IN);</div><div class='ctx'> 	desc.flag &amp;= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);</div><div class='del'>-	roce_set_bit(swt-&gt;cfg, VF_SWITCH_DATA_CFG_ALW_LPBK_S, 1);</div><div class='del'>-	roce_set_bit(swt-&gt;cfg, VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S, 1);</div><div class='del'>-	roce_set_bit(swt-&gt;cfg, VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S, 1);</div><div class='add'>+	hr_reg_enable(swt, VF_SWITCH_ALW_LPBK);</div><div class='add'>+	hr_reg_clear(swt, VF_SWITCH_ALW_LCL_LPBK);</div><div class='add'>+	hr_reg_enable(swt, VF_SWITCH_ALW_DST_OVRD);</div><div class='ctx'> </div><div class='ctx'> 	return hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)</div><div class='add'>+static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	u32 vf_id;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	for (vf_id = 0; vf_id &lt; hr_dev-&gt;func_num; vf_id++) {</div><div class='add'>+		ret = __hns_roce_set_vf_switch_param(hr_dev, vf_id);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int config_vf_hem_resource(struct hns_roce_dev *hr_dev, int vf_id)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_cmq_desc desc[2];</div><div class='del'>-	struct hns_roce_vf_res_a *req_a;</div><div class='del'>-	struct hns_roce_vf_res_b *req_b;</div><div class='del'>-	int i;</div><div class='add'>+	struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data;</div><div class='add'>+	struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data;</div><div class='add'>+	enum hns_roce_opcode_type opcode = HNS_ROCE_OPC_ALLOC_VF_RES;</div><div class='add'>+	struct hns_roce_caps *caps = &amp;hr_dev-&gt;caps;</div><div class='ctx'> </div><div class='del'>-	req_a = (struct hns_roce_vf_res_a *)desc[0].data;</div><div class='del'>-	req_b = (struct hns_roce_vf_res_b *)desc[1].data;</div><div class='del'>-	memset(req_a, 0, sizeof(*req_a));</div><div class='del'>-	memset(req_b, 0, sizeof(*req_b));</div><div class='del'>-	for (i = 0; i &lt; 2; i++) {</div><div class='del'>-		hns_roce_cmq_setup_basic_desc(&amp;desc[i],</div><div class='del'>-					      HNS_ROCE_OPC_ALLOC_VF_RES, false);</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc[0], opcode, false);</div><div class='add'>+	desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc[1], opcode, false);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(r_a, FUNC_RES_A_VF_ID, vf_id);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(r_a, FUNC_RES_A_QPC_BT_NUM, caps-&gt;qpc_bt_num);</div><div class='add'>+	hr_reg_write(r_a, FUNC_RES_A_QPC_BT_IDX, vf_id * caps-&gt;qpc_bt_num);</div><div class='add'>+	hr_reg_write(r_a, FUNC_RES_A_SRQC_BT_NUM, caps-&gt;srqc_bt_num);</div><div class='add'>+	hr_reg_write(r_a, FUNC_RES_A_SRQC_BT_IDX, vf_id * caps-&gt;srqc_bt_num);</div><div class='add'>+	hr_reg_write(r_a, FUNC_RES_A_CQC_BT_NUM, caps-&gt;cqc_bt_num);</div><div class='add'>+	hr_reg_write(r_a, FUNC_RES_A_CQC_BT_IDX, vf_id * caps-&gt;cqc_bt_num);</div><div class='add'>+	hr_reg_write(r_a, FUNC_RES_A_MPT_BT_NUM, caps-&gt;mpt_bt_num);</div><div class='add'>+	hr_reg_write(r_a, FUNC_RES_A_MPT_BT_IDX, vf_id * caps-&gt;mpt_bt_num);</div><div class='add'>+	hr_reg_write(r_a, FUNC_RES_A_EQC_BT_NUM, caps-&gt;eqc_bt_num);</div><div class='add'>+	hr_reg_write(r_a, FUNC_RES_A_EQC_BT_IDX, vf_id * caps-&gt;eqc_bt_num);</div><div class='add'>+	hr_reg_write(r_b, FUNC_RES_V_QID_NUM, caps-&gt;sl_num);</div><div class='add'>+	hr_reg_write(r_b, FUNC_RES_B_QID_IDX, vf_id * caps-&gt;sl_num);</div><div class='add'>+	hr_reg_write(r_b, FUNC_RES_B_SCCC_BT_NUM, caps-&gt;sccc_bt_num);</div><div class='add'>+	hr_reg_write(r_b, FUNC_RES_B_SCCC_BT_IDX, vf_id * caps-&gt;sccc_bt_num);</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09) {</div><div class='add'>+		hr_reg_write(r_b, FUNC_RES_V_GMV_BT_NUM, caps-&gt;gmv_bt_num);</div><div class='add'>+		hr_reg_write(r_b, FUNC_RES_B_GMV_BT_IDX,</div><div class='add'>+			     vf_id * caps-&gt;gmv_bt_num);</div><div class='add'>+	} else {</div><div class='add'>+		hr_reg_write(r_b, FUNC_RES_B_SGID_NUM, caps-&gt;sgid_bt_num);</div><div class='add'>+		hr_reg_write(r_b, FUNC_RES_B_SGID_IDX,</div><div class='add'>+			     vf_id * caps-&gt;sgid_bt_num);</div><div class='add'>+		hr_reg_write(r_b, FUNC_RES_B_SMAC_NUM, caps-&gt;smac_bt_num);</div><div class='add'>+		hr_reg_write(r_b, FUNC_RES_B_SMAC_IDX,</div><div class='add'>+			     vf_id * caps-&gt;smac_bt_num);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		if (i == 0)</div><div class='del'>-			desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);</div><div class='del'>-		else</div><div class='del'>-			desc[i].flag &amp;= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);</div><div class='del'>-</div><div class='del'>-		if (i == 0) {</div><div class='del'>-			roce_set_field(req_a-&gt;vf_qpc_bt_idx_num,</div><div class='del'>-				       VF_RES_A_DATA_1_VF_QPC_BT_IDX_M,</div><div class='del'>-				       VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0);</div><div class='del'>-			roce_set_field(req_a-&gt;vf_qpc_bt_idx_num,</div><div class='del'>-				       VF_RES_A_DATA_1_VF_QPC_BT_NUM_M,</div><div class='del'>-				       VF_RES_A_DATA_1_VF_QPC_BT_NUM_S,</div><div class='del'>-				       HNS_ROCE_VF_QPC_BT_NUM);</div><div class='del'>-</div><div class='del'>-			roce_set_field(req_a-&gt;vf_srqc_bt_idx_num,</div><div class='del'>-				       VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M,</div><div class='del'>-				       VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0);</div><div class='del'>-			roce_set_field(req_a-&gt;vf_srqc_bt_idx_num,</div><div class='del'>-				       VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M,</div><div class='del'>-				       VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S,</div><div class='del'>-				       HNS_ROCE_VF_SRQC_BT_NUM);</div><div class='del'>-</div><div class='del'>-			roce_set_field(req_a-&gt;vf_cqc_bt_idx_num,</div><div class='del'>-				       VF_RES_A_DATA_3_VF_CQC_BT_IDX_M,</div><div class='del'>-				       VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0);</div><div class='del'>-			roce_set_field(req_a-&gt;vf_cqc_bt_idx_num,</div><div class='del'>-				       VF_RES_A_DATA_3_VF_CQC_BT_NUM_M,</div><div class='del'>-				       VF_RES_A_DATA_3_VF_CQC_BT_NUM_S,</div><div class='del'>-				       HNS_ROCE_VF_CQC_BT_NUM);</div><div class='del'>-</div><div class='del'>-			roce_set_field(req_a-&gt;vf_mpt_bt_idx_num,</div><div class='del'>-				       VF_RES_A_DATA_4_VF_MPT_BT_IDX_M,</div><div class='del'>-				       VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0);</div><div class='del'>-			roce_set_field(req_a-&gt;vf_mpt_bt_idx_num,</div><div class='del'>-				       VF_RES_A_DATA_4_VF_MPT_BT_NUM_M,</div><div class='del'>-				       VF_RES_A_DATA_4_VF_MPT_BT_NUM_S,</div><div class='del'>-				       HNS_ROCE_VF_MPT_BT_NUM);</div><div class='del'>-</div><div class='del'>-			roce_set_field(req_a-&gt;vf_eqc_bt_idx_num,</div><div class='del'>-				       VF_RES_A_DATA_5_VF_EQC_IDX_M,</div><div class='del'>-				       VF_RES_A_DATA_5_VF_EQC_IDX_S, 0);</div><div class='del'>-			roce_set_field(req_a-&gt;vf_eqc_bt_idx_num,</div><div class='del'>-				       VF_RES_A_DATA_5_VF_EQC_NUM_M,</div><div class='del'>-				       VF_RES_A_DATA_5_VF_EQC_NUM_S,</div><div class='del'>-				       HNS_ROCE_VF_EQC_NUM);</div><div class='del'>-		} else {</div><div class='del'>-			roce_set_field(req_b-&gt;vf_smac_idx_num,</div><div class='del'>-				       VF_RES_B_DATA_1_VF_SMAC_IDX_M,</div><div class='del'>-				       VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0);</div><div class='del'>-			roce_set_field(req_b-&gt;vf_smac_idx_num,</div><div class='del'>-				       VF_RES_B_DATA_1_VF_SMAC_NUM_M,</div><div class='del'>-				       VF_RES_B_DATA_1_VF_SMAC_NUM_S,</div><div class='del'>-				       HNS_ROCE_VF_SMAC_NUM);</div><div class='del'>-</div><div class='del'>-			roce_set_field(req_b-&gt;vf_sgid_idx_num,</div><div class='del'>-				       VF_RES_B_DATA_2_VF_SGID_IDX_M,</div><div class='del'>-				       VF_RES_B_DATA_2_VF_SGID_IDX_S, 0);</div><div class='del'>-			roce_set_field(req_b-&gt;vf_sgid_idx_num,</div><div class='del'>-				       VF_RES_B_DATA_2_VF_SGID_NUM_M,</div><div class='del'>-				       VF_RES_B_DATA_2_VF_SGID_NUM_S,</div><div class='del'>-				       HNS_ROCE_VF_SGID_NUM);</div><div class='del'>-</div><div class='del'>-			roce_set_field(req_b-&gt;vf_qid_idx_sl_num,</div><div class='del'>-				       VF_RES_B_DATA_3_VF_QID_IDX_M,</div><div class='del'>-				       VF_RES_B_DATA_3_VF_QID_IDX_S, 0);</div><div class='del'>-			roce_set_field(req_b-&gt;vf_qid_idx_sl_num,</div><div class='del'>-				       VF_RES_B_DATA_3_VF_SL_NUM_M,</div><div class='del'>-				       VF_RES_B_DATA_3_VF_SL_NUM_S,</div><div class='del'>-				       HNS_ROCE_VF_SL_NUM);</div><div class='del'>-</div><div class='del'>-			roce_set_field(req_b-&gt;vf_sccc_idx_num,</div><div class='del'>-				       VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M,</div><div class='del'>-				       VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S, 0);</div><div class='del'>-			roce_set_field(req_b-&gt;vf_sccc_idx_num,</div><div class='del'>-				       VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M,</div><div class='del'>-				       VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S,</div><div class='del'>-				       HNS_ROCE_VF_SCCC_BT_NUM);</div><div class='add'>+	return hns_roce_cmq_send(hr_dev, desc, 2);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	u32 func_num = max_t(u32, 1, hr_dev-&gt;func_num);</div><div class='add'>+	u32 vf_id;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	for (vf_id = 0; vf_id &lt; func_num; vf_id++) {</div><div class='add'>+		ret = config_vf_hem_resource(hr_dev, vf_id);</div><div class='add'>+		if (ret) {</div><div class='add'>+			dev_err(hr_dev-&gt;dev,</div><div class='add'>+				"failed to config vf-%u hem res, ret = %d.\n",</div><div class='add'>+				vf_id, ret);</div><div class='add'>+			return ret;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return hns_roce_cmq_send(hr_dev, desc, 2);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	u8 srqc_hop_num = hr_dev-&gt;caps.srqc_hop_num;</div><div class='del'>-	u8 qpc_hop_num = hr_dev-&gt;caps.qpc_hop_num;</div><div class='del'>-	u8 cqc_hop_num = hr_dev-&gt;caps.cqc_hop_num;</div><div class='del'>-	u8 mpt_hop_num = hr_dev-&gt;caps.mpt_hop_num;</div><div class='del'>-	u8 sccc_hop_num = hr_dev-&gt;caps.sccc_hop_num;</div><div class='del'>-	struct hns_roce_cfg_bt_attr *req;</div><div class='ctx'> 	struct hns_roce_cmq_desc desc;</div><div class='add'>+	struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;</div><div class='add'>+	struct hns_roce_caps *caps = &amp;hr_dev-&gt;caps;</div><div class='ctx'> </div><div class='ctx'> 	hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_OPC_CFG_BT_ATTR, false);</div><div class='del'>-	req = (struct hns_roce_cfg_bt_attr *)desc.data;</div><div class='del'>-	memset(req, 0, sizeof(*req));</div><div class='del'>-</div><div class='del'>-	roce_set_field(req-&gt;vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M,</div><div class='del'>-		       CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.qpc_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_field(req-&gt;vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M,</div><div class='del'>-		       CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.qpc_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_field(req-&gt;vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M,</div><div class='del'>-		       CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S,</div><div class='del'>-		       qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num);</div><div class='del'>-</div><div class='del'>-	roce_set_field(req-&gt;vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M,</div><div class='del'>-		       CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.srqc_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_field(req-&gt;vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M,</div><div class='del'>-		       CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.srqc_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_field(req-&gt;vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M,</div><div class='del'>-		       CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S,</div><div class='del'>-		       srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num);</div><div class='del'>-</div><div class='del'>-	roce_set_field(req-&gt;vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M,</div><div class='del'>-		       CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.cqc_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_field(req-&gt;vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M,</div><div class='del'>-		       CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.cqc_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_field(req-&gt;vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M,</div><div class='del'>-		       CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S,</div><div class='del'>-		       cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num);</div><div class='del'>-</div><div class='del'>-	roce_set_field(req-&gt;vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M,</div><div class='del'>-		       CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.mpt_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_field(req-&gt;vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M,</div><div class='del'>-		       CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.mpt_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_field(req-&gt;vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M,</div><div class='del'>-		       CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S,</div><div class='del'>-		       mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num);</div><div class='del'>-</div><div class='del'>-	roce_set_field(req-&gt;vf_sccc_cfg,</div><div class='del'>-		       CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M,</div><div class='del'>-		       CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.sccc_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_field(req-&gt;vf_sccc_cfg,</div><div class='del'>-		       CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M,</div><div class='del'>-		       CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.sccc_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_field(req-&gt;vf_sccc_cfg,</div><div class='del'>-		       CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M,</div><div class='del'>-		       CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S,</div><div class='del'>-		       sccc_hop_num ==</div><div class='del'>-			      HNS_ROCE_HOP_NUM_0 ? 0 : sccc_hop_num);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(req, CFG_BT_ATTR_QPC_BA_PGSZ,</div><div class='add'>+		     caps-&gt;qpc_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='add'>+	hr_reg_write(req, CFG_BT_ATTR_QPC_BUF_PGSZ,</div><div class='add'>+		     caps-&gt;qpc_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='add'>+	hr_reg_write(req, CFG_BT_ATTR_QPC_HOPNUM,</div><div class='add'>+		     to_hr_hem_hopnum(caps-&gt;qpc_hop_num, caps-&gt;num_qps));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(req, CFG_BT_ATTR_SRQC_BA_PGSZ,</div><div class='add'>+		     caps-&gt;srqc_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='add'>+	hr_reg_write(req, CFG_BT_ATTR_SRQC_BUF_PGSZ,</div><div class='add'>+		     caps-&gt;srqc_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='add'>+	hr_reg_write(req, CFG_BT_ATTR_SRQC_HOPNUM,</div><div class='add'>+		     to_hr_hem_hopnum(caps-&gt;srqc_hop_num, caps-&gt;num_srqs));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(req, CFG_BT_ATTR_CQC_BA_PGSZ,</div><div class='add'>+		     caps-&gt;cqc_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='add'>+	hr_reg_write(req, CFG_BT_ATTR_CQC_BUF_PGSZ,</div><div class='add'>+		     caps-&gt;cqc_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='add'>+	hr_reg_write(req, CFG_BT_ATTR_CQC_HOPNUM,</div><div class='add'>+		     to_hr_hem_hopnum(caps-&gt;cqc_hop_num, caps-&gt;num_cqs));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(req, CFG_BT_ATTR_MPT_BA_PGSZ,</div><div class='add'>+		     caps-&gt;mpt_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='add'>+	hr_reg_write(req, CFG_BT_ATTR_MPT_BUF_PGSZ,</div><div class='add'>+		     caps-&gt;mpt_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='add'>+	hr_reg_write(req, CFG_BT_ATTR_MPT_HOPNUM,</div><div class='add'>+		     to_hr_hem_hopnum(caps-&gt;mpt_hop_num, caps-&gt;num_mtpts));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(req, CFG_BT_ATTR_SCCC_BA_PGSZ,</div><div class='add'>+		     caps-&gt;sccc_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='add'>+	hr_reg_write(req, CFG_BT_ATTR_SCCC_BUF_PGSZ,</div><div class='add'>+		     caps-&gt;sccc_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='add'>+	hr_reg_write(req, CFG_BT_ATTR_SCCC_HOPNUM,</div><div class='add'>+		     to_hr_hem_hopnum(caps-&gt;sccc_hop_num, caps-&gt;num_qps));</div><div class='ctx'> </div><div class='ctx'> 	return hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)</div><div class='add'>+static void calc_pg_sz(u32 obj_num, u32 obj_size, u32 hop_num, u32 ctx_bt_num,</div><div class='add'>+		       u32 *buf_page_size, u32 *bt_page_size, u32 hem_type)</div><div class='add'>+{</div><div class='add'>+	u64 obj_per_chunk;</div><div class='add'>+	u64 bt_chunk_size = PAGE_SIZE;</div><div class='add'>+	u64 buf_chunk_size = PAGE_SIZE;</div><div class='add'>+	u64 obj_per_chunk_default = buf_chunk_size / obj_size;</div><div class='add'>+</div><div class='add'>+	*buf_page_size = 0;</div><div class='add'>+	*bt_page_size = 0;</div><div class='add'>+</div><div class='add'>+	switch (hop_num) {</div><div class='add'>+	case 3:</div><div class='add'>+		obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *</div><div class='add'>+				(bt_chunk_size / BA_BYTE_LEN) *</div><div class='add'>+				(bt_chunk_size / BA_BYTE_LEN) *</div><div class='add'>+				 obj_per_chunk_default;</div><div class='add'>+		break;</div><div class='add'>+	case 2:</div><div class='add'>+		obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *</div><div class='add'>+				(bt_chunk_size / BA_BYTE_LEN) *</div><div class='add'>+				 obj_per_chunk_default;</div><div class='add'>+		break;</div><div class='add'>+	case 1:</div><div class='add'>+		obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *</div><div class='add'>+				obj_per_chunk_default;</div><div class='add'>+		break;</div><div class='add'>+	case HNS_ROCE_HOP_NUM_0:</div><div class='add'>+		obj_per_chunk = ctx_bt_num * obj_per_chunk_default;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		pr_err("table %u not support hop_num = %u!\n", hem_type,</div><div class='add'>+		       hop_num);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (hem_type &gt;= HEM_TYPE_MTT)</div><div class='add'>+		*bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));</div><div class='add'>+	else</div><div class='add'>+		*buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void set_hem_page_size(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_caps *caps = &amp;hr_dev-&gt;caps;</div><div class='add'>+</div><div class='add'>+	/* EQ */</div><div class='add'>+	caps-&gt;eqe_ba_pg_sz = 0;</div><div class='add'>+	caps-&gt;eqe_buf_pg_sz = 0;</div><div class='add'>+</div><div class='add'>+	/* Link Table */</div><div class='add'>+	caps-&gt;llm_buf_pg_sz = 0;</div><div class='add'>+</div><div class='add'>+	/* MR */</div><div class='add'>+	caps-&gt;mpt_ba_pg_sz = 0;</div><div class='add'>+	caps-&gt;mpt_buf_pg_sz = 0;</div><div class='add'>+	caps-&gt;pbl_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K;</div><div class='add'>+	caps-&gt;pbl_buf_pg_sz = 0;</div><div class='add'>+	calc_pg_sz(caps-&gt;num_mtpts, caps-&gt;mtpt_entry_sz, caps-&gt;mpt_hop_num,</div><div class='add'>+		   caps-&gt;mpt_bt_num, &amp;caps-&gt;mpt_buf_pg_sz, &amp;caps-&gt;mpt_ba_pg_sz,</div><div class='add'>+		   HEM_TYPE_MTPT);</div><div class='add'>+</div><div class='add'>+	/* QP */</div><div class='add'>+	caps-&gt;qpc_ba_pg_sz = 0;</div><div class='add'>+	caps-&gt;qpc_buf_pg_sz = 0;</div><div class='add'>+	caps-&gt;qpc_timer_ba_pg_sz = 0;</div><div class='add'>+	caps-&gt;qpc_timer_buf_pg_sz = 0;</div><div class='add'>+	caps-&gt;sccc_ba_pg_sz = 0;</div><div class='add'>+	caps-&gt;sccc_buf_pg_sz = 0;</div><div class='add'>+	caps-&gt;mtt_ba_pg_sz = 0;</div><div class='add'>+	caps-&gt;mtt_buf_pg_sz = 0;</div><div class='add'>+	calc_pg_sz(caps-&gt;num_qps, caps-&gt;qpc_sz, caps-&gt;qpc_hop_num,</div><div class='add'>+		   caps-&gt;qpc_bt_num, &amp;caps-&gt;qpc_buf_pg_sz, &amp;caps-&gt;qpc_ba_pg_sz,</div><div class='add'>+		   HEM_TYPE_QPC);</div><div class='add'>+</div><div class='add'>+	if (caps-&gt;flags &amp; HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)</div><div class='add'>+		calc_pg_sz(caps-&gt;num_qps, caps-&gt;sccc_sz, caps-&gt;sccc_hop_num,</div><div class='add'>+			   caps-&gt;sccc_bt_num, &amp;caps-&gt;sccc_buf_pg_sz,</div><div class='add'>+			   &amp;caps-&gt;sccc_ba_pg_sz, HEM_TYPE_SCCC);</div><div class='add'>+</div><div class='add'>+	/* CQ */</div><div class='add'>+	caps-&gt;cqc_ba_pg_sz = 0;</div><div class='add'>+	caps-&gt;cqc_buf_pg_sz = 0;</div><div class='add'>+	caps-&gt;cqc_timer_ba_pg_sz = 0;</div><div class='add'>+	caps-&gt;cqc_timer_buf_pg_sz = 0;</div><div class='add'>+	caps-&gt;cqe_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_256K;</div><div class='add'>+	caps-&gt;cqe_buf_pg_sz = 0;</div><div class='add'>+	calc_pg_sz(caps-&gt;num_cqs, caps-&gt;cqc_entry_sz, caps-&gt;cqc_hop_num,</div><div class='add'>+		   caps-&gt;cqc_bt_num, &amp;caps-&gt;cqc_buf_pg_sz, &amp;caps-&gt;cqc_ba_pg_sz,</div><div class='add'>+		   HEM_TYPE_CQC);</div><div class='add'>+	calc_pg_sz(caps-&gt;max_cqes, caps-&gt;cqe_sz, caps-&gt;cqe_hop_num,</div><div class='add'>+		   1, &amp;caps-&gt;cqe_buf_pg_sz, &amp;caps-&gt;cqe_ba_pg_sz, HEM_TYPE_CQE);</div><div class='add'>+</div><div class='add'>+	/* SRQ */</div><div class='add'>+	if (caps-&gt;flags &amp; HNS_ROCE_CAP_FLAG_SRQ) {</div><div class='add'>+		caps-&gt;srqc_ba_pg_sz = 0;</div><div class='add'>+		caps-&gt;srqc_buf_pg_sz = 0;</div><div class='add'>+		caps-&gt;srqwqe_ba_pg_sz = 0;</div><div class='add'>+		caps-&gt;srqwqe_buf_pg_sz = 0;</div><div class='add'>+		caps-&gt;idx_ba_pg_sz = 0;</div><div class='add'>+		caps-&gt;idx_buf_pg_sz = 0;</div><div class='add'>+		calc_pg_sz(caps-&gt;num_srqs, caps-&gt;srqc_entry_sz,</div><div class='add'>+			   caps-&gt;srqc_hop_num, caps-&gt;srqc_bt_num,</div><div class='add'>+			   &amp;caps-&gt;srqc_buf_pg_sz, &amp;caps-&gt;srqc_ba_pg_sz,</div><div class='add'>+			   HEM_TYPE_SRQC);</div><div class='add'>+		calc_pg_sz(caps-&gt;num_srqwqe_segs, caps-&gt;mtt_entry_sz,</div><div class='add'>+			   caps-&gt;srqwqe_hop_num, 1, &amp;caps-&gt;srqwqe_buf_pg_sz,</div><div class='add'>+			   &amp;caps-&gt;srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE);</div><div class='add'>+		calc_pg_sz(caps-&gt;num_idx_segs, caps-&gt;idx_entry_sz,</div><div class='add'>+			   caps-&gt;idx_hop_num, 1, &amp;caps-&gt;idx_buf_pg_sz,</div><div class='add'>+			   &amp;caps-&gt;idx_ba_pg_sz, HEM_TYPE_IDX);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* GMV */</div><div class='add'>+	caps-&gt;gmv_ba_pg_sz = 0;</div><div class='add'>+	caps-&gt;gmv_buf_pg_sz = 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* Apply all loaded caps before setting to hardware */</div><div class='add'>+static void apply_func_caps(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+#define MAX_GID_TBL_LEN 256</div><div class='add'>+	struct hns_roce_caps *caps = &amp;hr_dev-&gt;caps;</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+</div><div class='add'>+	/* The following configurations don't need to be got from firmware. */</div><div class='add'>+	caps-&gt;qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;</div><div class='add'>+	caps-&gt;cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;</div><div class='add'>+	caps-&gt;mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;</div><div class='add'>+</div><div class='add'>+	caps-&gt;pbl_hop_num = HNS_ROCE_PBL_HOP_NUM;</div><div class='add'>+	caps-&gt;qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;</div><div class='add'>+	caps-&gt;cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;</div><div class='add'>+</div><div class='add'>+	caps-&gt;num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS;</div><div class='add'>+	caps-&gt;num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS;</div><div class='add'>+</div><div class='add'>+	if (!caps-&gt;num_comp_vectors)</div><div class='add'>+		caps-&gt;num_comp_vectors =</div><div class='add'>+			min_t(u32, caps-&gt;eqc_bt_num - HNS_ROCE_V2_AEQE_VEC_NUM,</div><div class='add'>+				(u32)priv-&gt;handle-&gt;rinfo.num_vectors -</div><div class='add'>+		(HNS_ROCE_V2_AEQE_VEC_NUM + HNS_ROCE_V2_ABNORMAL_VEC_NUM));</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09) {</div><div class='add'>+		caps-&gt;eqe_hop_num = HNS_ROCE_V3_EQE_HOP_NUM;</div><div class='add'>+		caps-&gt;ceqe_size = HNS_ROCE_V3_EQE_SIZE;</div><div class='add'>+		caps-&gt;aeqe_size = HNS_ROCE_V3_EQE_SIZE;</div><div class='add'>+</div><div class='add'>+		/* The following configurations will be overwritten */</div><div class='add'>+		caps-&gt;qpc_sz = HNS_ROCE_V3_QPC_SZ;</div><div class='add'>+		caps-&gt;cqe_sz = HNS_ROCE_V3_CQE_SIZE;</div><div class='add'>+		caps-&gt;sccc_sz = HNS_ROCE_V3_SCCC_SZ;</div><div class='add'>+</div><div class='add'>+		/* The following configurations are not got from firmware */</div><div class='add'>+		caps-&gt;gmv_entry_sz = HNS_ROCE_V3_GMV_ENTRY_SZ;</div><div class='add'>+</div><div class='add'>+		caps-&gt;gmv_hop_num = HNS_ROCE_HOP_NUM_0;</div><div class='add'>+</div><div class='add'>+		/* It's meaningless to support excessively large gid_table_len,</div><div class='add'>+		 * as the type of sgid_index in kernel struct ib_global_route</div><div class='add'>+		 * and userspace struct ibv_global_route are u8/uint8_t (0-255).</div><div class='add'>+		 */</div><div class='add'>+		caps-&gt;gid_table_len[0] = min_t(u32, MAX_GID_TBL_LEN,</div><div class='add'>+					 caps-&gt;gmv_bt_num *</div><div class='add'>+					 (HNS_HW_PAGE_SIZE / caps-&gt;gmv_entry_sz));</div><div class='add'>+</div><div class='add'>+		caps-&gt;gmv_entry_num = caps-&gt;gmv_bt_num * (HNS_HW_PAGE_SIZE /</div><div class='add'>+							  caps-&gt;gmv_entry_sz);</div><div class='add'>+	} else {</div><div class='add'>+		u32 func_num = max_t(u32, 1, hr_dev-&gt;func_num);</div><div class='add'>+</div><div class='add'>+		caps-&gt;eqe_hop_num = HNS_ROCE_V2_EQE_HOP_NUM;</div><div class='add'>+		caps-&gt;ceqe_size = HNS_ROCE_CEQE_SIZE;</div><div class='add'>+		caps-&gt;aeqe_size = HNS_ROCE_AEQE_SIZE;</div><div class='add'>+		caps-&gt;gid_table_len[0] /= func_num;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;is_vf) {</div><div class='add'>+		caps-&gt;default_aeq_arm_st = 0x3;</div><div class='add'>+		caps-&gt;default_ceq_arm_st = 0x3;</div><div class='add'>+		caps-&gt;default_ceq_max_cnt = 0x1;</div><div class='add'>+		caps-&gt;default_ceq_period = 0x10;</div><div class='add'>+		caps-&gt;default_aeq_max_cnt = 0x1;</div><div class='add'>+		caps-&gt;default_aeq_period = 0x10;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	set_hem_page_size(hr_dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_query_caps(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM] = {};</div><div class='add'>+	struct hns_roce_caps *caps = &amp;hr_dev-&gt;caps;</div><div class='add'>+	struct hns_roce_query_pf_caps_a *resp_a;</div><div class='add'>+	struct hns_roce_query_pf_caps_b *resp_b;</div><div class='add'>+	struct hns_roce_query_pf_caps_c *resp_c;</div><div class='add'>+	struct hns_roce_query_pf_caps_d *resp_d;</div><div class='add'>+	struct hns_roce_query_pf_caps_e *resp_e;</div><div class='add'>+	struct hns_roce_query_pf_caps_f *resp_f;</div><div class='add'>+	enum hns_roce_opcode_type cmd;</div><div class='add'>+	int ctx_hop_num;</div><div class='add'>+	int pbl_hop_num;</div><div class='add'>+	int cmd_num;</div><div class='ctx'> 	int ret;</div><div class='add'>+	int i;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_cmq_query_hw_info(hr_dev);</div><div class='add'>+	cmd = hr_dev-&gt;is_vf ? HNS_ROCE_OPC_QUERY_VF_CAPS_NUM :</div><div class='add'>+	      HNS_ROCE_OPC_QUERY_PF_CAPS_NUM;</div><div class='add'>+	cmd_num = hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08 ?</div><div class='add'>+		  HNS_ROCE_QUERY_PF_CAPS_CMD_NUM_HIP08 :</div><div class='add'>+		  HNS_ROCE_QUERY_PF_CAPS_CMD_NUM;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; cmd_num - 1; i++) {</div><div class='add'>+		hns_roce_cmq_setup_basic_desc(&amp;desc[i], cmd, true);</div><div class='add'>+		desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc[cmd_num - 1], cmd, true);</div><div class='add'>+	desc[cmd_num - 1].flag &amp;= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_cmq_send(hr_dev, desc, cmd_num);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	resp_a = (struct hns_roce_query_pf_caps_a *)desc[0].data;</div><div class='add'>+	resp_b = (struct hns_roce_query_pf_caps_b *)desc[1].data;</div><div class='add'>+	resp_c = (struct hns_roce_query_pf_caps_c *)desc[2].data;</div><div class='add'>+	resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data;</div><div class='add'>+	resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data;</div><div class='add'>+	resp_f = (struct hns_roce_query_pf_caps_f *)desc[5].data;</div><div class='add'>+</div><div class='add'>+	caps-&gt;local_ca_ack_delay = resp_a-&gt;local_ca_ack_delay;</div><div class='add'>+	caps-&gt;max_sq_sg = le16_to_cpu(resp_a-&gt;max_sq_sg);</div><div class='add'>+	caps-&gt;max_sq_inline = le16_to_cpu(resp_a-&gt;max_sq_inline);</div><div class='add'>+	caps-&gt;max_rq_sg = le16_to_cpu(resp_a-&gt;max_rq_sg);</div><div class='add'>+	caps-&gt;max_rq_sg = roundup_pow_of_two(caps-&gt;max_rq_sg);</div><div class='add'>+	caps-&gt;max_srq_sges = le16_to_cpu(resp_a-&gt;max_srq_sges);</div><div class='add'>+	caps-&gt;max_srq_sges = roundup_pow_of_two(caps-&gt;max_srq_sges);</div><div class='add'>+	caps-&gt;num_aeq_vectors = resp_a-&gt;num_aeq_vectors;</div><div class='add'>+	caps-&gt;num_other_vectors = resp_a-&gt;num_other_vectors;</div><div class='add'>+	caps-&gt;max_sq_desc_sz = resp_a-&gt;max_sq_desc_sz;</div><div class='add'>+	caps-&gt;max_rq_desc_sz = resp_a-&gt;max_rq_desc_sz;</div><div class='add'>+</div><div class='add'>+	caps-&gt;mtpt_entry_sz = resp_b-&gt;mtpt_entry_sz;</div><div class='add'>+	caps-&gt;irrl_entry_sz = resp_b-&gt;irrl_entry_sz;</div><div class='add'>+	caps-&gt;trrl_entry_sz = resp_b-&gt;trrl_entry_sz;</div><div class='add'>+	caps-&gt;cqc_entry_sz = resp_b-&gt;cqc_entry_sz;</div><div class='add'>+	caps-&gt;srqc_entry_sz = resp_b-&gt;srqc_entry_sz;</div><div class='add'>+	caps-&gt;idx_entry_sz = resp_b-&gt;idx_entry_sz;</div><div class='add'>+	caps-&gt;sccc_sz = resp_b-&gt;sccc_sz;</div><div class='add'>+	caps-&gt;max_mtu = resp_b-&gt;max_mtu;</div><div class='add'>+	caps-&gt;min_cqes = resp_b-&gt;min_cqes;</div><div class='add'>+	caps-&gt;min_wqes = resp_b-&gt;min_wqes;</div><div class='add'>+	caps-&gt;page_size_cap = le32_to_cpu(resp_b-&gt;page_size_cap);</div><div class='add'>+	caps-&gt;pkey_table_len[0] = resp_b-&gt;pkey_table_len;</div><div class='add'>+	caps-&gt;phy_num_uars = resp_b-&gt;phy_num_uars;</div><div class='add'>+	ctx_hop_num = resp_b-&gt;ctx_hop_num;</div><div class='add'>+	pbl_hop_num = resp_b-&gt;pbl_hop_num;</div><div class='add'>+</div><div class='add'>+	caps-&gt;num_pds = 1 &lt;&lt; hr_reg_read(resp_c, PF_CAPS_C_NUM_PDS);</div><div class='add'>+</div><div class='add'>+	caps-&gt;flags = hr_reg_read(resp_c, PF_CAPS_C_CAP_FLAGS);</div><div class='add'>+	caps-&gt;flags |= le16_to_cpu(resp_d-&gt;cap_flags_ex) &lt;&lt;</div><div class='add'>+		       HNS_ROCE_CAP_FLAGS_EX_SHIFT;</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;is_vf)</div><div class='add'>+		caps-&gt;flags &amp;= ~HNS_ROCE_CAP_FLAG_BOND;</div><div class='add'>+</div><div class='add'>+	caps-&gt;num_cqs = 1 &lt;&lt; hr_reg_read(resp_c, PF_CAPS_C_NUM_CQS);</div><div class='add'>+	caps-&gt;gid_table_len[0] = hr_reg_read(resp_c, PF_CAPS_C_MAX_GID);</div><div class='add'>+	caps-&gt;max_cqes = 1 &lt;&lt; hr_reg_read(resp_c, PF_CAPS_C_CQ_DEPTH);</div><div class='add'>+	caps-&gt;num_xrcds = 1 &lt;&lt; hr_reg_read(resp_c, PF_CAPS_C_NUM_XRCDS);</div><div class='add'>+	caps-&gt;num_mtpts = 1 &lt;&lt; hr_reg_read(resp_c, PF_CAPS_C_NUM_MRWS);</div><div class='add'>+	caps-&gt;num_qps = 1 &lt;&lt; hr_reg_read(resp_c, PF_CAPS_C_NUM_QPS);</div><div class='add'>+	caps-&gt;max_qp_init_rdma = hr_reg_read(resp_c, PF_CAPS_C_MAX_ORD);</div><div class='add'>+	caps-&gt;max_qp_dest_rdma = caps-&gt;max_qp_init_rdma;</div><div class='add'>+	caps-&gt;max_wqes = 1 &lt;&lt; le16_to_cpu(resp_c-&gt;sq_depth);</div><div class='add'>+</div><div class='add'>+	caps-&gt;num_srqs = 1 &lt;&lt; hr_reg_read(resp_d, PF_CAPS_D_NUM_SRQS);</div><div class='add'>+	caps-&gt;cong_cap = hr_reg_read(resp_d, PF_CAPS_D_CONG_CAP);</div><div class='add'>+	caps-&gt;max_srq_wrs = 1 &lt;&lt; le16_to_cpu(resp_d-&gt;srq_depth);</div><div class='add'>+	caps-&gt;ceqe_depth = 1 &lt;&lt; hr_reg_read(resp_d, PF_CAPS_D_CEQ_DEPTH);</div><div class='add'>+	caps-&gt;num_comp_vectors = hr_reg_read(resp_d, PF_CAPS_D_NUM_CEQS);</div><div class='add'>+	caps-&gt;aeqe_depth = 1 &lt;&lt; hr_reg_read(resp_d, PF_CAPS_D_AEQ_DEPTH);</div><div class='add'>+	caps-&gt;default_cong_type = hr_reg_read(resp_d, PF_CAPS_D_DEFAULT_ALG);</div><div class='add'>+	caps-&gt;reserved_pds = hr_reg_read(resp_d, PF_CAPS_D_RSV_PDS);</div><div class='add'>+	caps-&gt;num_uars = 1 &lt;&lt; hr_reg_read(resp_d, PF_CAPS_D_NUM_UARS);</div><div class='add'>+	caps-&gt;reserved_qps = hr_reg_read(resp_d, PF_CAPS_D_RSV_QPS);</div><div class='add'>+	caps-&gt;reserved_uars = hr_reg_read(resp_d, PF_CAPS_D_RSV_UARS);</div><div class='add'>+</div><div class='add'>+	caps-&gt;reserved_mrws = hr_reg_read(resp_e, PF_CAPS_E_RSV_MRWS);</div><div class='add'>+	caps-&gt;chunk_sz = 1 &lt;&lt; hr_reg_read(resp_e, PF_CAPS_E_CHUNK_SIZE_SHIFT);</div><div class='add'>+	caps-&gt;reserved_cqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_CQS);</div><div class='add'>+	caps-&gt;reserved_xrcds = hr_reg_read(resp_e, PF_CAPS_E_RSV_XRCDS);</div><div class='add'>+	caps-&gt;reserved_srqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_SRQS);</div><div class='add'>+	caps-&gt;reserved_lkey = hr_reg_read(resp_e, PF_CAPS_E_RSV_LKEYS);</div><div class='add'>+</div><div class='add'>+	caps-&gt;max_ack_req_msg_len = le32_to_cpu(resp_f-&gt;max_ack_req_msg_len);</div><div class='add'>+</div><div class='add'>+	caps-&gt;qpc_hop_num = ctx_hop_num;</div><div class='add'>+	caps-&gt;sccc_hop_num = ctx_hop_num;</div><div class='add'>+	caps-&gt;srqc_hop_num = ctx_hop_num;</div><div class='add'>+	caps-&gt;cqc_hop_num = ctx_hop_num;</div><div class='add'>+	caps-&gt;mpt_hop_num = ctx_hop_num;</div><div class='add'>+	caps-&gt;mtt_hop_num = pbl_hop_num;</div><div class='add'>+	caps-&gt;cqe_hop_num = pbl_hop_num;</div><div class='add'>+	caps-&gt;srqwqe_hop_num = pbl_hop_num;</div><div class='add'>+	caps-&gt;idx_hop_num = pbl_hop_num;</div><div class='add'>+	caps-&gt;wqe_sq_hop_num = hr_reg_read(resp_d, PF_CAPS_D_SQWQE_HOP_NUM);</div><div class='add'>+	caps-&gt;wqe_sge_hop_num = hr_reg_read(resp_d, PF_CAPS_D_EX_SGE_HOP_NUM);</div><div class='add'>+	caps-&gt;wqe_rq_hop_num = hr_reg_read(resp_d, PF_CAPS_D_RQWQE_HOP_NUM);</div><div class='add'>+</div><div class='add'>+	if (!(caps-&gt;page_size_cap &amp; PAGE_SIZE))</div><div class='add'>+		caps-&gt;page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;</div><div class='add'>+</div><div class='add'>+	if (!hr_dev-&gt;is_vf) {</div><div class='add'>+		caps-&gt;cqe_sz = resp_a-&gt;cqe_sz;</div><div class='add'>+		caps-&gt;qpc_sz = le16_to_cpu(resp_b-&gt;qpc_sz);</div><div class='add'>+		caps-&gt;default_aeq_arm_st =</div><div class='add'>+				hr_reg_read(resp_d, PF_CAPS_D_AEQ_ARM_ST);</div><div class='add'>+		caps-&gt;default_ceq_arm_st =</div><div class='add'>+				hr_reg_read(resp_d, PF_CAPS_D_CEQ_ARM_ST);</div><div class='add'>+		caps-&gt;default_ceq_max_cnt = le16_to_cpu(resp_e-&gt;ceq_max_cnt);</div><div class='add'>+		caps-&gt;default_ceq_period = le16_to_cpu(resp_e-&gt;ceq_period);</div><div class='add'>+		caps-&gt;default_aeq_max_cnt = le16_to_cpu(resp_e-&gt;aeq_max_cnt);</div><div class='add'>+		caps-&gt;default_aeq_period = le16_to_cpu(resp_e-&gt;aeq_period);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int config_hem_entry_size(struct hns_roce_dev *hr_dev, u32 type, u32 val)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_cmq_desc desc;</div><div class='add'>+	struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;</div><div class='add'>+</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE,</div><div class='add'>+				      false);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(req, CFG_HEM_ENTRY_SIZE_TYPE, type);</div><div class='add'>+	hr_reg_write(req, CFG_HEM_ENTRY_SIZE_VALUE, val);</div><div class='add'>+</div><div class='add'>+	return hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_config_entry_size(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_caps *caps = &amp;hr_dev-&gt;caps;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_QPC_SIZE,</div><div class='add'>+				    caps-&gt;qpc_sz);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "Query hardware version fail, ret = %d.\n",</div><div class='del'>-			ret);</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "failed to cfg qpc sz, ret = %d.\n", ret);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_query_fw_ver(hr_dev);</div><div class='add'>+	ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_SCCC_SIZE,</div><div class='add'>+				    caps-&gt;sccc_sz);</div><div class='add'>+	if (ret)</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "failed to cfg sccc sz, ret = %d.\n", ret);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_v2_vf_profile(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	hr_dev-&gt;func_num = 1;</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_query_caps(hr_dev);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "Query firmware version fail, ret = %d.\n",</div><div class='del'>-			ret);</div><div class='add'>+		dev_err(dev, "failed to query VF caps, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_query_vf_resource(hr_dev);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(dev, "failed to query VF resource, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	apply_func_caps(hr_dev);</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_v2_set_bt(hr_dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		dev_err(dev, "failed to config VF BA table, ret = %d.\n", ret);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_v2_pf_profile(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_query_func_info(hr_dev);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(dev, "failed to query func info, ret = %d.\n", ret);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ret = hns_roce_config_global_param(hr_dev);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "Configure global param fail, ret = %d.\n",</div><div class='del'>-			ret);</div><div class='add'>+		dev_err(dev, "failed to config global param, ret = %d.\n", ret);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* Get pf resource owned by every pf */</div><div class='del'>-	ret = hns_roce_query_pf_resource(hr_dev);</div><div class='add'>+	ret = hns_roce_set_vf_switch_param(hr_dev);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "Query pf resource fail, ret = %d.\n",</div><div class='del'>-			ret);</div><div class='add'>+		dev_err(dev, "failed to set switch param, ret = %d.\n", ret);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (hr_dev-&gt;pci_dev-&gt;revision == 0x21) {</div><div class='del'>-		ret = hns_roce_query_pf_timer_resource(hr_dev);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(hr_dev-&gt;dev,</div><div class='del'>-				"Query pf timer resource fail, ret = %d.\n",</div><div class='del'>-				ret);</div><div class='del'>-			return ret;</div><div class='del'>-		}</div><div class='add'>+	ret = hns_roce_query_caps(hr_dev);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(dev, "failed to query PF caps, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	ret = hns_roce_query_pf_resource(hr_dev);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(dev, "failed to query pf resource, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	apply_func_caps(hr_dev);</div><div class='add'>+</div><div class='ctx'> 	ret = hns_roce_alloc_vf_resource(hr_dev);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "Allocate vf resource fail, ret = %d.\n",</div><div class='del'>-			ret);</div><div class='add'>+		dev_err(dev, "failed to alloc vf resource, ret = %d.\n", ret);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (hr_dev-&gt;pci_dev-&gt;revision == 0x21) {</div><div class='del'>-		ret = hns_roce_set_vf_switch_param(hr_dev, 0);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(hr_dev-&gt;dev,</div><div class='del'>-				"Set function switch param fail, ret = %d.\n",</div><div class='del'>-				ret);</div><div class='del'>-			return ret;</div><div class='del'>-		}</div><div class='add'>+	ret = hns_roce_v2_set_bt(hr_dev);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(dev, "failed to config BA table, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Configure the size of QPC, SCCC, etc. */</div><div class='add'>+	return hns_roce_config_entry_size(hr_dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_cmq_query_hw_info(hr_dev);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(dev, "failed to query hardware info, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_query_fw_ver(hr_dev);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(dev, "failed to query firmware info, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	hr_dev-&gt;vendor_part_id = hr_dev-&gt;pci_dev-&gt;device;</div><div class='ctx'> 	hr_dev-&gt;sys_image_guid = be64_to_cpu(hr_dev-&gt;ib_dev.node_guid);</div><div class='ctx'> </div><div class='del'>-	caps-&gt;num_qps		= HNS_ROCE_V2_MAX_QP_NUM;</div><div class='del'>-	caps-&gt;max_wqes		= HNS_ROCE_V2_MAX_WQE_NUM;</div><div class='del'>-	caps-&gt;num_cqs		= HNS_ROCE_V2_MAX_CQ_NUM;</div><div class='del'>-	caps-&gt;num_srqs		= HNS_ROCE_V2_MAX_SRQ_NUM;</div><div class='del'>-	caps-&gt;min_cqes		= HNS_ROCE_MIN_CQE_NUM;</div><div class='del'>-	caps-&gt;max_cqes		= HNS_ROCE_V2_MAX_CQE_NUM;</div><div class='del'>-	caps-&gt;max_srqwqes	= HNS_ROCE_V2_MAX_SRQWQE_NUM;</div><div class='del'>-	caps-&gt;max_sq_sg		= HNS_ROCE_V2_MAX_SQ_SGE_NUM;</div><div class='del'>-	caps-&gt;max_extend_sg	= HNS_ROCE_V2_MAX_EXTEND_SGE_NUM;</div><div class='del'>-	caps-&gt;max_rq_sg		= HNS_ROCE_V2_MAX_RQ_SGE_NUM;</div><div class='del'>-	caps-&gt;max_sq_inline	= HNS_ROCE_V2_MAX_SQ_INLINE;</div><div class='del'>-	caps-&gt;max_srq_sg	= HNS_ROCE_V2_MAX_SRQ_SGE_NUM;</div><div class='del'>-	caps-&gt;num_uars		= HNS_ROCE_V2_UAR_NUM;</div><div class='del'>-	caps-&gt;phy_num_uars	= HNS_ROCE_V2_PHY_UAR_NUM;</div><div class='del'>-	caps-&gt;num_aeq_vectors	= HNS_ROCE_V2_AEQE_VEC_NUM;</div><div class='del'>-	caps-&gt;num_comp_vectors	= HNS_ROCE_V2_COMP_VEC_NUM;</div><div class='del'>-	caps-&gt;num_other_vectors	= HNS_ROCE_V2_ABNORMAL_VEC_NUM;</div><div class='del'>-	caps-&gt;num_mtpts		= HNS_ROCE_V2_MAX_MTPT_NUM;</div><div class='del'>-	caps-&gt;num_mtt_segs	= HNS_ROCE_V2_MAX_MTT_SEGS;</div><div class='del'>-	caps-&gt;num_cqe_segs	= HNS_ROCE_V2_MAX_CQE_SEGS;</div><div class='del'>-	caps-&gt;num_srqwqe_segs	= HNS_ROCE_V2_MAX_SRQWQE_SEGS;</div><div class='del'>-	caps-&gt;num_idx_segs	= HNS_ROCE_V2_MAX_IDX_SEGS;</div><div class='del'>-	caps-&gt;num_pds		= HNS_ROCE_V2_MAX_PD_NUM;</div><div class='del'>-	caps-&gt;max_qp_init_rdma	= HNS_ROCE_V2_MAX_QP_INIT_RDMA;</div><div class='del'>-	caps-&gt;max_qp_dest_rdma	= HNS_ROCE_V2_MAX_QP_DEST_RDMA;</div><div class='del'>-	caps-&gt;max_sq_desc_sz	= HNS_ROCE_V2_MAX_SQ_DESC_SZ;</div><div class='del'>-	caps-&gt;max_rq_desc_sz	= HNS_ROCE_V2_MAX_RQ_DESC_SZ;</div><div class='del'>-	caps-&gt;max_srq_desc_sz	= HNS_ROCE_V2_MAX_SRQ_DESC_SZ;</div><div class='del'>-	caps-&gt;qpc_entry_sz	= HNS_ROCE_V2_QPC_ENTRY_SZ;</div><div class='del'>-	caps-&gt;irrl_entry_sz	= HNS_ROCE_V2_IRRL_ENTRY_SZ;</div><div class='del'>-	caps-&gt;trrl_entry_sz	= HNS_ROCE_V2_TRRL_ENTRY_SZ;</div><div class='del'>-	caps-&gt;cqc_entry_sz	= HNS_ROCE_V2_CQC_ENTRY_SZ;</div><div class='del'>-	caps-&gt;srqc_entry_sz	= HNS_ROCE_V2_SRQC_ENTRY_SZ;</div><div class='del'>-	caps-&gt;mtpt_entry_sz	= HNS_ROCE_V2_MTPT_ENTRY_SZ;</div><div class='del'>-	caps-&gt;mtt_entry_sz	= HNS_ROCE_V2_MTT_ENTRY_SZ;</div><div class='del'>-	caps-&gt;idx_entry_sz	= 4;</div><div class='del'>-	caps-&gt;cq_entry_sz	= HNS_ROCE_V2_CQE_ENTRY_SIZE;</div><div class='del'>-	caps-&gt;page_size_cap	= HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;</div><div class='del'>-	caps-&gt;reserved_lkey	= 0;</div><div class='del'>-	caps-&gt;reserved_pds	= 0;</div><div class='del'>-	caps-&gt;reserved_mrws	= 1;</div><div class='del'>-	caps-&gt;reserved_uars	= 0;</div><div class='del'>-	caps-&gt;reserved_cqs	= 0;</div><div class='del'>-	caps-&gt;reserved_srqs	= 0;</div><div class='del'>-	caps-&gt;reserved_qps	= HNS_ROCE_V2_RSV_QPS;</div><div class='del'>-</div><div class='del'>-	caps-&gt;qpc_ba_pg_sz	= 0;</div><div class='del'>-	caps-&gt;qpc_buf_pg_sz	= 0;</div><div class='del'>-	caps-&gt;qpc_hop_num	= HNS_ROCE_CONTEXT_HOP_NUM;</div><div class='del'>-	caps-&gt;srqc_ba_pg_sz	= 0;</div><div class='del'>-	caps-&gt;srqc_buf_pg_sz	= 0;</div><div class='del'>-	caps-&gt;srqc_hop_num	= HNS_ROCE_CONTEXT_HOP_NUM;</div><div class='del'>-	caps-&gt;cqc_ba_pg_sz	= 0;</div><div class='del'>-	caps-&gt;cqc_buf_pg_sz	= 0;</div><div class='del'>-	caps-&gt;cqc_hop_num	= HNS_ROCE_CONTEXT_HOP_NUM;</div><div class='del'>-	caps-&gt;mpt_ba_pg_sz	= 0;</div><div class='del'>-	caps-&gt;mpt_buf_pg_sz	= 0;</div><div class='del'>-	caps-&gt;mpt_hop_num	= HNS_ROCE_CONTEXT_HOP_NUM;</div><div class='del'>-	caps-&gt;pbl_ba_pg_sz	= 2;</div><div class='del'>-	caps-&gt;pbl_buf_pg_sz	= 0;</div><div class='del'>-	caps-&gt;pbl_hop_num	= HNS_ROCE_PBL_HOP_NUM;</div><div class='del'>-	caps-&gt;mtt_ba_pg_sz	= 0;</div><div class='del'>-	caps-&gt;mtt_buf_pg_sz	= 0;</div><div class='del'>-	caps-&gt;mtt_hop_num	= HNS_ROCE_MTT_HOP_NUM;</div><div class='del'>-	caps-&gt;cqe_ba_pg_sz	= 0;</div><div class='del'>-	caps-&gt;cqe_buf_pg_sz	= 0;</div><div class='del'>-	caps-&gt;cqe_hop_num	= HNS_ROCE_CQE_HOP_NUM;</div><div class='del'>-	caps-&gt;srqwqe_ba_pg_sz	= 0;</div><div class='del'>-	caps-&gt;srqwqe_buf_pg_sz	= 0;</div><div class='del'>-	caps-&gt;srqwqe_hop_num	= HNS_ROCE_SRQWQE_HOP_NUM;</div><div class='del'>-	caps-&gt;idx_ba_pg_sz	= 0;</div><div class='del'>-	caps-&gt;idx_buf_pg_sz	= 0;</div><div class='del'>-	caps-&gt;idx_hop_num	= HNS_ROCE_IDX_HOP_NUM;</div><div class='del'>-	caps-&gt;eqe_ba_pg_sz	= 0;</div><div class='del'>-	caps-&gt;eqe_buf_pg_sz	= 0;</div><div class='del'>-	caps-&gt;eqe_hop_num	= HNS_ROCE_EQE_HOP_NUM;</div><div class='del'>-	caps-&gt;tsq_buf_pg_sz	= 0;</div><div class='del'>-	caps-&gt;chunk_sz		= HNS_ROCE_V2_TABLE_CHUNK_SIZE;</div><div class='del'>-</div><div class='del'>-	caps-&gt;flags		= HNS_ROCE_CAP_FLAG_REREG_MR |</div><div class='del'>-				  HNS_ROCE_CAP_FLAG_ROCE_V1_V2 |</div><div class='del'>-				  HNS_ROCE_CAP_FLAG_RQ_INLINE |</div><div class='del'>-				  HNS_ROCE_CAP_FLAG_RECORD_DB |</div><div class='del'>-				  HNS_ROCE_CAP_FLAG_SQ_RECORD_DB;</div><div class='del'>-</div><div class='del'>-	if (hr_dev-&gt;pci_dev-&gt;revision == 0x21)</div><div class='del'>-		caps-&gt;flags |= HNS_ROCE_CAP_FLAG_MW |</div><div class='del'>-			       HNS_ROCE_CAP_FLAG_FRMR;</div><div class='del'>-</div><div class='del'>-	caps-&gt;pkey_table_len[0] = 1;</div><div class='del'>-	caps-&gt;gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;</div><div class='del'>-	caps-&gt;ceqe_depth	= HNS_ROCE_V2_COMP_EQE_NUM;</div><div class='del'>-	caps-&gt;aeqe_depth	= HNS_ROCE_V2_ASYNC_EQE_NUM;</div><div class='del'>-	caps-&gt;local_ca_ack_delay = 0;</div><div class='del'>-	caps-&gt;max_mtu = IB_MTU_4096;</div><div class='del'>-</div><div class='del'>-	caps-&gt;max_srqs		= HNS_ROCE_V2_MAX_SRQ;</div><div class='del'>-	caps-&gt;max_srq_wrs	= HNS_ROCE_V2_MAX_SRQ_WR;</div><div class='del'>-	caps-&gt;max_srq_sges	= HNS_ROCE_V2_MAX_SRQ_SGE;</div><div class='del'>-</div><div class='del'>-	if (hr_dev-&gt;pci_dev-&gt;revision == 0x21) {</div><div class='del'>-		caps-&gt;flags |= HNS_ROCE_CAP_FLAG_ATOMIC |</div><div class='del'>-			       HNS_ROCE_CAP_FLAG_SRQ |</div><div class='del'>-			       HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;</div><div class='del'>-</div><div class='del'>-		caps-&gt;num_qpc_timer	  = HNS_ROCE_V2_MAX_QPC_TIMER_NUM;</div><div class='del'>-		caps-&gt;qpc_timer_entry_sz  = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;</div><div class='del'>-		caps-&gt;qpc_timer_ba_pg_sz  = 0;</div><div class='del'>-		caps-&gt;qpc_timer_buf_pg_sz = 0;</div><div class='del'>-		caps-&gt;qpc_timer_hop_num   = HNS_ROCE_HOP_NUM_0;</div><div class='del'>-		caps-&gt;num_cqc_timer	  = HNS_ROCE_V2_MAX_CQC_TIMER_NUM;</div><div class='del'>-		caps-&gt;cqc_timer_entry_sz  = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;</div><div class='del'>-		caps-&gt;cqc_timer_ba_pg_sz  = 0;</div><div class='del'>-		caps-&gt;cqc_timer_buf_pg_sz = 0;</div><div class='del'>-		caps-&gt;cqc_timer_hop_num   = HNS_ROCE_HOP_NUM_0;</div><div class='del'>-</div><div class='del'>-		caps-&gt;sccc_entry_sz	= HNS_ROCE_V2_SCCC_ENTRY_SZ;</div><div class='del'>-		caps-&gt;sccc_ba_pg_sz	= 0;</div><div class='del'>-		caps-&gt;sccc_buf_pg_sz    = 0;</div><div class='del'>-		caps-&gt;sccc_hop_num	= HNS_ROCE_SCCC_HOP_NUM;</div><div class='add'>+	if (hr_dev-&gt;is_vf)</div><div class='add'>+		return hns_roce_v2_vf_profile(hr_dev);</div><div class='add'>+	else</div><div class='add'>+		return hns_roce_v2_pf_profile(hr_dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void config_llm_table(struct hns_roce_buf *data_buf, void *cfg_buf)</div><div class='add'>+{</div><div class='add'>+	u32 i, next_ptr, page_num;</div><div class='add'>+	__le64 *entry = cfg_buf;</div><div class='add'>+	dma_addr_t addr;</div><div class='add'>+	u64 val;</div><div class='add'>+</div><div class='add'>+	page_num = data_buf-&gt;npages;</div><div class='add'>+	for (i = 0; i &lt; page_num; i++) {</div><div class='add'>+		addr = hns_roce_buf_page(data_buf, i);</div><div class='add'>+		if (i == (page_num - 1))</div><div class='add'>+			next_ptr = 0;</div><div class='add'>+		else</div><div class='add'>+			next_ptr = i + 1;</div><div class='add'>+</div><div class='add'>+		val = HNS_ROCE_EXT_LLM_ENTRY(addr, (u64)next_ptr);</div><div class='add'>+		entry[i] = cpu_to_le64(val);</div><div class='ctx'> 	}</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_v2_set_bt(hr_dev);</div><div class='add'>+static int set_llm_cfg_to_hw(struct hns_roce_dev *hr_dev,</div><div class='add'>+			     struct hns_roce_link_table *table)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_cmq_desc desc[2];</div><div class='add'>+	struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data;</div><div class='add'>+	struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data;</div><div class='add'>+	struct hns_roce_buf *buf = table-&gt;buf;</div><div class='add'>+	enum hns_roce_opcode_type opcode;</div><div class='add'>+	dma_addr_t addr;</div><div class='add'>+</div><div class='add'>+	opcode = HNS_ROCE_OPC_CFG_EXT_LLM;</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc[0], opcode, false);</div><div class='add'>+	desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc[1], opcode, false);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(r_a, CFG_LLM_A_BA_L, lower_32_bits(table-&gt;table.map));</div><div class='add'>+	hr_reg_write(r_a, CFG_LLM_A_BA_H, upper_32_bits(table-&gt;table.map));</div><div class='add'>+	hr_reg_write(r_a, CFG_LLM_A_DEPTH, buf-&gt;npages);</div><div class='add'>+	hr_reg_write(r_a, CFG_LLM_A_PGSZ, to_hr_hw_page_shift(buf-&gt;page_shift));</div><div class='add'>+	hr_reg_enable(r_a, CFG_LLM_A_INIT_EN);</div><div class='add'>+</div><div class='add'>+	addr = to_hr_hw_page_addr(hns_roce_buf_page(buf, 0));</div><div class='add'>+	hr_reg_write(r_a, CFG_LLM_A_HEAD_BA_L, lower_32_bits(addr));</div><div class='add'>+	hr_reg_write(r_a, CFG_LLM_A_HEAD_BA_H, upper_32_bits(addr));</div><div class='add'>+	hr_reg_write(r_a, CFG_LLM_A_HEAD_NXTPTR, 1);</div><div class='add'>+	hr_reg_write(r_a, CFG_LLM_A_HEAD_PTR, 0);</div><div class='add'>+</div><div class='add'>+	addr = to_hr_hw_page_addr(hns_roce_buf_page(buf, buf-&gt;npages - 1));</div><div class='add'>+	hr_reg_write(r_b, CFG_LLM_B_TAIL_BA_L, lower_32_bits(addr));</div><div class='add'>+	hr_reg_write(r_b, CFG_LLM_B_TAIL_BA_H, upper_32_bits(addr));</div><div class='add'>+	hr_reg_write(r_b, CFG_LLM_B_TAIL_PTR, buf-&gt;npages - 1);</div><div class='add'>+</div><div class='add'>+	return hns_roce_cmq_send(hr_dev, desc, 2);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct hns_roce_link_table *</div><div class='add'>+alloc_link_table_buf(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	u16 total_sl = hr_dev-&gt;caps.sl_num * hr_dev-&gt;func_num;</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	struct hns_roce_link_table *link_tbl;</div><div class='add'>+	u32 pg_shift, size, min_size;</div><div class='add'>+</div><div class='add'>+	link_tbl = &amp;priv-&gt;ext_llm;</div><div class='add'>+	pg_shift = hr_dev-&gt;caps.llm_buf_pg_sz + PAGE_SHIFT;</div><div class='add'>+	size = hr_dev-&gt;caps.num_qps * hr_dev-&gt;func_num *</div><div class='add'>+	       HNS_ROCE_V2_EXT_LLM_ENTRY_SZ;</div><div class='add'>+	min_size = HNS_ROCE_EXT_LLM_MIN_PAGES(total_sl) &lt;&lt; pg_shift;</div><div class='add'>+</div><div class='add'>+	/* Alloc data table */</div><div class='add'>+	size = max(size, min_size);</div><div class='add'>+	link_tbl-&gt;buf = hns_roce_buf_alloc(hr_dev, size, pg_shift, 0);</div><div class='add'>+	if (IS_ERR(link_tbl-&gt;buf))</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	/* Alloc config table */</div><div class='add'>+	size = link_tbl-&gt;buf-&gt;npages * sizeof(u64);</div><div class='add'>+	link_tbl-&gt;table.buf = dma_alloc_coherent(hr_dev-&gt;dev, size,</div><div class='add'>+						 &amp;link_tbl-&gt;table.map,</div><div class='add'>+						 GFP_KERNEL);</div><div class='add'>+	if (!link_tbl-&gt;table.buf) {</div><div class='add'>+		hns_roce_buf_free(hr_dev, link_tbl-&gt;buf);</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return link_tbl;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void free_link_table_buf(struct hns_roce_dev *hr_dev,</div><div class='add'>+				struct hns_roce_link_table *tbl)</div><div class='add'>+{</div><div class='add'>+	if (tbl-&gt;buf) {</div><div class='add'>+		u32 size = tbl-&gt;buf-&gt;npages * sizeof(u64);</div><div class='add'>+</div><div class='add'>+		dma_free_coherent(hr_dev-&gt;dev, size, tbl-&gt;table.buf,</div><div class='add'>+				  tbl-&gt;table.map);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hns_roce_buf_free(hr_dev, tbl-&gt;buf);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_link_table *link_tbl;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	link_tbl = alloc_link_table_buf(hr_dev);</div><div class='add'>+	if (IS_ERR(link_tbl))</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	if (WARN_ON(link_tbl-&gt;buf-&gt;npages &gt; HNS_ROCE_V2_EXT_LLM_MAX_DEPTH)) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err_alloc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	config_llm_table(link_tbl-&gt;buf, link_tbl-&gt;table.buf);</div><div class='add'>+	ret = set_llm_cfg_to_hw(hr_dev, link_tbl);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "Configure bt attribute fail, ret = %d.\n",</div><div class='del'>-			ret);</div><div class='add'>+		goto err_alloc;</div><div class='ctx'> </div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_alloc:</div><div class='add'>+	free_link_table_buf(hr_dev, link_tbl);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_config_link_table(struct hns_roce_dev *hr_dev,</div><div class='del'>-				      enum hns_roce_link_table_type type)</div><div class='add'>+static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_cmq_desc desc[2];</div><div class='del'>-	struct hns_roce_cfg_llm_a *req_a =</div><div class='del'>-				(struct hns_roce_cfg_llm_a *)desc[0].data;</div><div class='del'>-	struct hns_roce_cfg_llm_b *req_b =</div><div class='del'>-				(struct hns_roce_cfg_llm_b *)desc[1].data;</div><div class='ctx'> 	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='del'>-	struct hns_roce_link_table *link_tbl;</div><div class='del'>-	struct hns_roce_link_table_entry *entry;</div><div class='del'>-	enum hns_roce_opcode_type opcode;</div><div class='del'>-	u32 page_num;</div><div class='del'>-	int i;</div><div class='ctx'> </div><div class='del'>-	switch (type) {</div><div class='del'>-	case TSQ_LINK_TABLE:</div><div class='del'>-		link_tbl = &amp;priv-&gt;tsq;</div><div class='del'>-		opcode = HNS_ROCE_OPC_CFG_EXT_LLM;</div><div class='del'>-		break;</div><div class='del'>-	case TPQ_LINK_TABLE:</div><div class='del'>-		link_tbl = &amp;priv-&gt;tpq;</div><div class='del'>-		opcode = HNS_ROCE_OPC_CFG_TMOUT_LLM;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	free_link_table_buf(hr_dev, &amp;priv-&gt;ext_llm);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void free_dip_entry(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dip *hr_dip;</div><div class='add'>+	unsigned long idx;</div><div class='add'>+</div><div class='add'>+	xa_lock(&amp;hr_dev-&gt;qp_table.dip_xa);</div><div class='add'>+</div><div class='add'>+	xa_for_each(&amp;hr_dev-&gt;qp_table.dip_xa, idx, hr_dip) {</div><div class='add'>+		__xa_erase(&amp;hr_dev-&gt;qp_table.dip_xa, hr_dip-&gt;dip_idx);</div><div class='add'>+		kfree(hr_dip);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	page_num = link_tbl-&gt;npages;</div><div class='del'>-	entry = link_tbl-&gt;table.buf;</div><div class='del'>-	memset(req_a, 0, sizeof(*req_a));</div><div class='del'>-	memset(req_b, 0, sizeof(*req_b));</div><div class='add'>+	xa_unlock(&amp;hr_dev-&gt;qp_table.dip_xa);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; 2; i++) {</div><div class='del'>-		hns_roce_cmq_setup_basic_desc(&amp;desc[i], opcode, false);</div><div class='add'>+static struct ib_pd *free_mr_init_pd(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	struct hns_roce_v2_free_mr *free_mr = &amp;priv-&gt;free_mr;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	struct hns_roce_pd *hr_pd;</div><div class='add'>+	struct ib_pd *pd;</div><div class='add'>+</div><div class='add'>+	hr_pd = kzalloc(sizeof(*hr_pd), GFP_KERNEL);</div><div class='add'>+	if (!hr_pd)</div><div class='add'>+		return NULL;</div><div class='add'>+	pd = &amp;hr_pd-&gt;ibpd;</div><div class='add'>+	pd-&gt;device = ibdev;</div><div class='add'>+</div><div class='add'>+	if (hns_roce_alloc_pd(pd, NULL)) {</div><div class='add'>+		ibdev_err(ibdev, "failed to create pd for free mr.\n");</div><div class='add'>+		kfree(hr_pd);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+	free_mr-&gt;rsv_pd = to_hr_pd(pd);</div><div class='add'>+	free_mr-&gt;rsv_pd-&gt;ibpd.device = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	free_mr-&gt;rsv_pd-&gt;ibpd.uobject = NULL;</div><div class='add'>+	free_mr-&gt;rsv_pd-&gt;ibpd.__internal_mr = NULL;</div><div class='add'>+	atomic_set(&amp;free_mr-&gt;rsv_pd-&gt;ibpd.usecnt, 0);</div><div class='add'>+</div><div class='add'>+	return pd;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		if (i == 0)</div><div class='del'>-			desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);</div><div class='del'>-		else</div><div class='del'>-			desc[i].flag &amp;= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);</div><div class='del'>-</div><div class='del'>-		if (i == 0) {</div><div class='del'>-			req_a-&gt;base_addr_l = link_tbl-&gt;table.map &amp; 0xffffffff;</div><div class='del'>-			req_a-&gt;base_addr_h = (link_tbl-&gt;table.map &gt;&gt; 32) &amp;</div><div class='del'>-					     0xffffffff;</div><div class='del'>-			roce_set_field(req_a-&gt;depth_pgsz_init_en,</div><div class='del'>-				       CFG_LLM_QUE_DEPTH_M,</div><div class='del'>-				       CFG_LLM_QUE_DEPTH_S,</div><div class='del'>-				       link_tbl-&gt;npages);</div><div class='del'>-			roce_set_field(req_a-&gt;depth_pgsz_init_en,</div><div class='del'>-				       CFG_LLM_QUE_PGSZ_M,</div><div class='del'>-				       CFG_LLM_QUE_PGSZ_S,</div><div class='del'>-				       link_tbl-&gt;pg_sz);</div><div class='del'>-			req_a-&gt;head_ba_l = entry[0].blk_ba0;</div><div class='del'>-			req_a-&gt;head_ba_h_nxtptr = entry[0].blk_ba1_nxt_ptr;</div><div class='del'>-			roce_set_field(req_a-&gt;head_ptr,</div><div class='del'>-				       CFG_LLM_HEAD_PTR_M,</div><div class='del'>-				       CFG_LLM_HEAD_PTR_S, 0);</div><div class='del'>-		} else {</div><div class='del'>-			req_b-&gt;tail_ba_l = entry[page_num - 1].blk_ba0;</div><div class='del'>-			roce_set_field(req_b-&gt;tail_ba_h,</div><div class='del'>-				       CFG_LLM_TAIL_BA_H_M,</div><div class='del'>-				       CFG_LLM_TAIL_BA_H_S,</div><div class='del'>-				       entry[page_num - 1].blk_ba1_nxt_ptr &amp;</div><div class='del'>-				       HNS_ROCE_LINK_TABLE_BA1_M);</div><div class='del'>-			roce_set_field(req_b-&gt;tail_ptr,</div><div class='del'>-				       CFG_LLM_TAIL_PTR_M,</div><div class='del'>-				       CFG_LLM_TAIL_PTR_S,</div><div class='del'>-				       (entry[page_num - 2].blk_ba1_nxt_ptr &amp;</div><div class='del'>-				       HNS_ROCE_LINK_TABLE_NXT_PTR_M) &gt;&gt;</div><div class='del'>-				       HNS_ROCE_LINK_TABLE_NXT_PTR_S);</div><div class='add'>+static struct ib_cq *free_mr_init_cq(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	struct hns_roce_v2_free_mr *free_mr = &amp;priv-&gt;free_mr;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	struct ib_cq_init_attr cq_init_attr = {};</div><div class='add'>+	struct hns_roce_cq *hr_cq;</div><div class='add'>+	struct ib_cq *cq;</div><div class='add'>+</div><div class='add'>+	cq_init_attr.cqe = HNS_ROCE_FREE_MR_USED_CQE_NUM;</div><div class='add'>+</div><div class='add'>+	hr_cq = kzalloc(sizeof(*hr_cq), GFP_KERNEL);</div><div class='add'>+	if (!hr_cq)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	cq = &amp;hr_cq-&gt;ib_cq;</div><div class='add'>+	cq-&gt;device = ibdev;</div><div class='add'>+</div><div class='add'>+	if (hns_roce_create_cq(cq, &amp;cq_init_attr, NULL)) {</div><div class='add'>+		ibdev_err(ibdev, "failed to create cq for free mr.\n");</div><div class='add'>+		kfree(hr_cq);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+	free_mr-&gt;rsv_cq = to_hr_cq(cq);</div><div class='add'>+	free_mr-&gt;rsv_cq-&gt;ib_cq.device = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	free_mr-&gt;rsv_cq-&gt;ib_cq.uobject = NULL;</div><div class='add'>+	free_mr-&gt;rsv_cq-&gt;ib_cq.comp_handler = NULL;</div><div class='add'>+	free_mr-&gt;rsv_cq-&gt;ib_cq.event_handler = NULL;</div><div class='add'>+	free_mr-&gt;rsv_cq-&gt;ib_cq.cq_context = NULL;</div><div class='add'>+	atomic_set(&amp;free_mr-&gt;rsv_cq-&gt;ib_cq.usecnt, 0);</div><div class='add'>+</div><div class='add'>+	return cq;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int free_mr_init_qp(struct hns_roce_dev *hr_dev, struct ib_cq *cq,</div><div class='add'>+			   struct ib_qp_init_attr *init_attr, int i)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	struct hns_roce_v2_free_mr *free_mr = &amp;priv-&gt;free_mr;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	struct hns_roce_qp *hr_qp;</div><div class='add'>+	struct ib_qp *qp;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL);</div><div class='add'>+	if (!hr_qp)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	qp = &amp;hr_qp-&gt;ibqp;</div><div class='add'>+	qp-&gt;device = ibdev;</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_create_qp(qp, init_attr, NULL);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to create qp for free mr.\n");</div><div class='add'>+		kfree(hr_qp);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	free_mr-&gt;rsv_qp[i] = hr_qp;</div><div class='add'>+	free_mr-&gt;rsv_qp[i]-&gt;ibqp.recv_cq = cq;</div><div class='add'>+	free_mr-&gt;rsv_qp[i]-&gt;ibqp.send_cq = cq;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void free_mr_exit(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	struct hns_roce_v2_free_mr *free_mr = &amp;priv-&gt;free_mr;</div><div class='add'>+	struct ib_qp *qp;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ARRAY_SIZE(free_mr-&gt;rsv_qp); i++) {</div><div class='add'>+		if (free_mr-&gt;rsv_qp[i]) {</div><div class='add'>+			qp = &amp;free_mr-&gt;rsv_qp[i]-&gt;ibqp;</div><div class='add'>+			hns_roce_v2_destroy_qp(qp, NULL);</div><div class='add'>+			kfree(free_mr-&gt;rsv_qp[i]);</div><div class='add'>+			free_mr-&gt;rsv_qp[i] = NULL;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='del'>-	roce_set_field(req_a-&gt;depth_pgsz_init_en,</div><div class='del'>-		       CFG_LLM_INIT_EN_M, CFG_LLM_INIT_EN_S, 1);</div><div class='ctx'> </div><div class='del'>-	return hns_roce_cmq_send(hr_dev, desc, 2);</div><div class='add'>+	if (free_mr-&gt;rsv_cq) {</div><div class='add'>+		hns_roce_destroy_cq(&amp;free_mr-&gt;rsv_cq-&gt;ib_cq, NULL);</div><div class='add'>+		kfree(free_mr-&gt;rsv_cq);</div><div class='add'>+		free_mr-&gt;rsv_cq = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (free_mr-&gt;rsv_pd) {</div><div class='add'>+		hns_roce_dealloc_pd(&amp;free_mr-&gt;rsv_pd-&gt;ibpd, NULL);</div><div class='add'>+		kfree(free_mr-&gt;rsv_pd);</div><div class='add'>+		free_mr-&gt;rsv_pd = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_destroy(&amp;free_mr-&gt;mutex);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev,</div><div class='del'>-				    enum hns_roce_link_table_type type)</div><div class='add'>+static int free_mr_alloc_res(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='del'>-	struct hns_roce_link_table *link_tbl;</div><div class='del'>-	struct hns_roce_link_table_entry *entry;</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	u32 buf_chk_sz;</div><div class='del'>-	dma_addr_t t;</div><div class='del'>-	int func_num = 1;</div><div class='del'>-	int pg_num_a;</div><div class='del'>-	int pg_num_b;</div><div class='del'>-	int pg_num;</div><div class='del'>-	int size;</div><div class='add'>+	struct hns_roce_v2_free_mr *free_mr = &amp;priv-&gt;free_mr;</div><div class='add'>+	struct ib_qp_init_attr qp_init_attr = {};</div><div class='add'>+	struct ib_pd *pd;</div><div class='add'>+	struct ib_cq *cq;</div><div class='add'>+	int ret;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	switch (type) {</div><div class='del'>-	case TSQ_LINK_TABLE:</div><div class='del'>-		link_tbl = &amp;priv-&gt;tsq;</div><div class='del'>-		buf_chk_sz = 1 &lt;&lt; (hr_dev-&gt;caps.tsq_buf_pg_sz + PAGE_SHIFT);</div><div class='del'>-		pg_num_a = hr_dev-&gt;caps.num_qps * 8 / buf_chk_sz;</div><div class='del'>-		pg_num_b = hr_dev-&gt;caps.sl_num * 4 + 2;</div><div class='del'>-		break;</div><div class='del'>-	case TPQ_LINK_TABLE:</div><div class='del'>-		link_tbl = &amp;priv-&gt;tpq;</div><div class='del'>-		buf_chk_sz = 1 &lt;&lt; (hr_dev-&gt;caps.tpq_buf_pg_sz +	PAGE_SHIFT);</div><div class='del'>-		pg_num_a = hr_dev-&gt;caps.num_cqs * 4 / buf_chk_sz;</div><div class='del'>-		pg_num_b = 2 * 4 * func_num + 2;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	pd = free_mr_init_pd(hr_dev);</div><div class='add'>+	if (!pd)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cq = free_mr_init_cq(hr_dev);</div><div class='add'>+	if (!cq) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto create_failed_cq;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	pg_num = max(pg_num_a, pg_num_b);</div><div class='del'>-	size = pg_num * sizeof(struct hns_roce_link_table_entry);</div><div class='add'>+	qp_init_attr.qp_type = IB_QPT_RC;</div><div class='add'>+	qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;</div><div class='add'>+	qp_init_attr.send_cq = cq;</div><div class='add'>+	qp_init_attr.recv_cq = cq;</div><div class='add'>+	for (i = 0; i &lt; ARRAY_SIZE(free_mr-&gt;rsv_qp); i++) {</div><div class='add'>+		qp_init_attr.cap.max_send_wr = HNS_ROCE_FREE_MR_USED_SQWQE_NUM;</div><div class='add'>+		qp_init_attr.cap.max_send_sge = HNS_ROCE_FREE_MR_USED_SQSGE_NUM;</div><div class='add'>+		qp_init_attr.cap.max_recv_wr = HNS_ROCE_FREE_MR_USED_RQWQE_NUM;</div><div class='add'>+		qp_init_attr.cap.max_recv_sge = HNS_ROCE_FREE_MR_USED_RQSGE_NUM;</div><div class='ctx'> </div><div class='del'>-	link_tbl-&gt;table.buf = dma_alloc_coherent(dev, size,</div><div class='del'>-						 &amp;link_tbl-&gt;table.map,</div><div class='del'>-						 GFP_KERNEL);</div><div class='del'>-	if (!link_tbl-&gt;table.buf)</div><div class='del'>-		goto out;</div><div class='add'>+		ret = free_mr_init_qp(hr_dev, cq, &amp;qp_init_attr, i);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto create_failed_qp;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	link_tbl-&gt;pg_list = kcalloc(pg_num, sizeof(*link_tbl-&gt;pg_list),</div><div class='del'>-				    GFP_KERNEL);</div><div class='del'>-	if (!link_tbl-&gt;pg_list)</div><div class='del'>-		goto err_kcalloc_failed;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-	entry = link_tbl-&gt;table.buf;</div><div class='del'>-	for (i = 0; i &lt; pg_num; ++i) {</div><div class='del'>-		link_tbl-&gt;pg_list[i].buf = dma_alloc_coherent(dev, buf_chk_sz,</div><div class='del'>-							      &amp;t, GFP_KERNEL);</div><div class='del'>-		if (!link_tbl-&gt;pg_list[i].buf)</div><div class='del'>-			goto err_alloc_buf_failed;</div><div class='add'>+create_failed_qp:</div><div class='add'>+	for (i--; i &gt;= 0; i--) {</div><div class='add'>+		hns_roce_v2_destroy_qp(&amp;free_mr-&gt;rsv_qp[i]-&gt;ibqp, NULL);</div><div class='add'>+		kfree(free_mr-&gt;rsv_qp[i]);</div><div class='add'>+	}</div><div class='add'>+	hns_roce_destroy_cq(cq, NULL);</div><div class='add'>+	kfree(cq);</div><div class='ctx'> </div><div class='del'>-		link_tbl-&gt;pg_list[i].map = t;</div><div class='del'>-		memset(link_tbl-&gt;pg_list[i].buf, 0, buf_chk_sz);</div><div class='add'>+create_failed_cq:</div><div class='add'>+	hns_roce_dealloc_pd(pd, NULL);</div><div class='add'>+	kfree(pd);</div><div class='ctx'> </div><div class='del'>-		entry[i].blk_ba0 = (t &gt;&gt; 12) &amp; 0xffffffff;</div><div class='del'>-		roce_set_field(entry[i].blk_ba1_nxt_ptr,</div><div class='del'>-			       HNS_ROCE_LINK_TABLE_BA1_M,</div><div class='del'>-			       HNS_ROCE_LINK_TABLE_BA1_S,</div><div class='del'>-			       t &gt;&gt; 44);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		if (i &lt; (pg_num - 1))</div><div class='del'>-			roce_set_field(entry[i].blk_ba1_nxt_ptr,</div><div class='del'>-				       HNS_ROCE_LINK_TABLE_NXT_PTR_M,</div><div class='del'>-				       HNS_ROCE_LINK_TABLE_NXT_PTR_S,</div><div class='del'>-				       i + 1);</div><div class='add'>+static int free_mr_modify_rsv_qp(struct hns_roce_dev *hr_dev,</div><div class='add'>+				 struct ib_qp_attr *attr, int sl_num)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	struct hns_roce_v2_free_mr *free_mr = &amp;priv-&gt;free_mr;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	struct hns_roce_qp *hr_qp;</div><div class='add'>+	int loopback;</div><div class='add'>+	int mask;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	hr_qp = to_hr_qp(&amp;free_mr-&gt;rsv_qp[sl_num]-&gt;ibqp);</div><div class='add'>+	hr_qp-&gt;free_mr_en = 1;</div><div class='add'>+	hr_qp-&gt;ibqp.device = ibdev;</div><div class='add'>+	hr_qp-&gt;ibqp.qp_type = IB_QPT_RC;</div><div class='add'>+</div><div class='add'>+	mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS;</div><div class='add'>+	attr-&gt;qp_state = IB_QPS_INIT;</div><div class='add'>+	attr-&gt;port_num = 1;</div><div class='add'>+	attr-&gt;qp_access_flags = IB_ACCESS_REMOTE_WRITE;</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;modify_qp(&amp;hr_qp-&gt;ibqp, attr, mask, IB_QPS_INIT,</div><div class='add'>+				    IB_QPS_INIT, NULL);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err_ratelimited(ibdev, "failed to modify qp to init, ret = %d.\n",</div><div class='add'>+				      ret);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='del'>-	link_tbl-&gt;npages = pg_num;</div><div class='del'>-	link_tbl-&gt;pg_sz = buf_chk_sz;</div><div class='ctx'> </div><div class='del'>-	return hns_roce_config_link_table(hr_dev, type);</div><div class='add'>+	loopback = hr_dev-&gt;loop_idc;</div><div class='add'>+	/* Set qpc lbi = 1 incidate loopback IO */</div><div class='add'>+	hr_dev-&gt;loop_idc = 1;</div><div class='ctx'> </div><div class='del'>-err_alloc_buf_failed:</div><div class='del'>-	for (i -= 1; i &gt;= 0; i--)</div><div class='del'>-		dma_free_coherent(dev, buf_chk_sz,</div><div class='del'>-				  link_tbl-&gt;pg_list[i].buf,</div><div class='del'>-				  link_tbl-&gt;pg_list[i].map);</div><div class='del'>-	kfree(link_tbl-&gt;pg_list);</div><div class='add'>+	mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN |</div><div class='add'>+	       IB_QP_RQ_PSN | IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER;</div><div class='add'>+	attr-&gt;qp_state = IB_QPS_RTR;</div><div class='add'>+	attr-&gt;ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;</div><div class='add'>+	attr-&gt;path_mtu = IB_MTU_256;</div><div class='add'>+	attr-&gt;dest_qp_num = hr_qp-&gt;qpn;</div><div class='add'>+	attr-&gt;rq_psn = HNS_ROCE_FREE_MR_USED_PSN;</div><div class='ctx'> </div><div class='del'>-err_kcalloc_failed:</div><div class='del'>-	dma_free_coherent(dev, size, link_tbl-&gt;table.buf,</div><div class='del'>-			  link_tbl-&gt;table.map);</div><div class='add'>+	rdma_ah_set_sl(&amp;attr-&gt;ah_attr, (u8)sl_num);</div><div class='ctx'> </div><div class='del'>-out:</div><div class='del'>-	return -ENOMEM;</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;modify_qp(&amp;hr_qp-&gt;ibqp, attr, mask, IB_QPS_INIT,</div><div class='add'>+				    IB_QPS_RTR, NULL);</div><div class='add'>+	hr_dev-&gt;loop_idc = loopback;</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to modify qp to rtr, ret = %d.\n",</div><div class='add'>+			  ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mask = IB_QP_STATE | IB_QP_SQ_PSN | IB_QP_RETRY_CNT | IB_QP_TIMEOUT |</div><div class='add'>+	       IB_QP_RNR_RETRY | IB_QP_MAX_QP_RD_ATOMIC;</div><div class='add'>+	attr-&gt;qp_state = IB_QPS_RTS;</div><div class='add'>+	attr-&gt;sq_psn = HNS_ROCE_FREE_MR_USED_PSN;</div><div class='add'>+	attr-&gt;retry_cnt = HNS_ROCE_FREE_MR_USED_QP_RETRY_CNT;</div><div class='add'>+	attr-&gt;timeout = HNS_ROCE_FREE_MR_USED_QP_TIMEOUT;</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;modify_qp(&amp;hr_qp-&gt;ibqp, attr, mask, IB_QPS_RTR,</div><div class='add'>+				    IB_QPS_RTS, NULL);</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_err(ibdev, "failed to modify qp to rts, ret = %d.\n",</div><div class='add'>+			  ret);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev,</div><div class='del'>-				     struct hns_roce_link_table *link_tbl)</div><div class='add'>+static int free_mr_modify_qp(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	int size;</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	struct hns_roce_v2_free_mr *free_mr = &amp;priv-&gt;free_mr;</div><div class='add'>+	struct ib_qp_attr attr = {};</div><div class='add'>+	int ret;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	size = link_tbl-&gt;npages * sizeof(struct hns_roce_link_table_entry);</div><div class='add'>+	rdma_ah_set_grh(&amp;attr.ah_attr, NULL, 0, 0, 1, 0);</div><div class='add'>+	rdma_ah_set_static_rate(&amp;attr.ah_attr, 3);</div><div class='add'>+	rdma_ah_set_port_num(&amp;attr.ah_attr, 1);</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; link_tbl-&gt;npages; ++i)</div><div class='del'>-		if (link_tbl-&gt;pg_list[i].buf)</div><div class='del'>-			dma_free_coherent(dev, link_tbl-&gt;pg_sz,</div><div class='del'>-					  link_tbl-&gt;pg_list[i].buf,</div><div class='del'>-					  link_tbl-&gt;pg_list[i].map);</div><div class='del'>-	kfree(link_tbl-&gt;pg_list);</div><div class='add'>+	for (i = 0; i &lt; ARRAY_SIZE(free_mr-&gt;rsv_qp); i++) {</div><div class='add'>+		ret = free_mr_modify_rsv_qp(hr_dev, &amp;attr, i);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	dma_free_coherent(dev, size, link_tbl-&gt;table.buf,</div><div class='del'>-			  link_tbl-&gt;table.map);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)</div><div class='add'>+static int free_mr_init(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='del'>-	int qpc_count, cqc_count;</div><div class='del'>-	int ret, i;</div><div class='add'>+	struct hns_roce_v2_free_mr *free_mr = &amp;priv-&gt;free_mr;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	mutex_init(&amp;free_mr-&gt;mutex);</div><div class='ctx'> </div><div class='del'>-	/* TSQ includes SQ doorbell and ack doorbell */</div><div class='del'>-	ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE);</div><div class='add'>+	ret = free_mr_alloc_res(hr_dev);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "TSQ init failed, ret = %d.\n", ret);</div><div class='add'>+		mutex_destroy(&amp;free_mr-&gt;mutex);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "TPQ init failed, ret = %d.\n", ret);</div><div class='del'>-		goto err_tpq_init_failed;</div><div class='add'>+	ret = free_mr_modify_qp(hr_dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_modify_qp;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_modify_qp:</div><div class='add'>+	free_mr_exit(hr_dev);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_hem_table(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	unsigned int qpc_count;</div><div class='add'>+	unsigned int cqc_count;</div><div class='add'>+	unsigned int gmv_count;</div><div class='add'>+	int ret;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	/* Alloc memory for source address table buffer space chunk */</div><div class='add'>+	for (gmv_count = 0; gmv_count &lt; hr_dev-&gt;caps.gmv_entry_num;</div><div class='add'>+	     gmv_count++) {</div><div class='add'>+		ret = hns_roce_table_get(hr_dev, &amp;hr_dev-&gt;gmv_table, gmv_count);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_gmv_failed;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* Alloc memory for QPC Timer buffer space chunk*/</div><div class='add'>+	if (hr_dev-&gt;is_vf)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	/* Alloc memory for QPC Timer buffer space chunk */</div><div class='ctx'> 	for (qpc_count = 0; qpc_count &lt; hr_dev-&gt;caps.qpc_timer_bt_num;</div><div class='ctx'> 	     qpc_count++) {</div><div class='ctx'> 		ret = hns_roce_table_get(hr_dev, &amp;hr_dev-&gt;qpc_timer_table,</div><div class='hunk'>@@ -1858,7 +3028,7 @@ static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* Alloc memory for CQC Timer buffer space chunk*/</div><div class='add'>+	/* Alloc memory for CQC Timer buffer space chunk */</div><div class='ctx'> 	for (cqc_count = 0; cqc_count &lt; hr_dev-&gt;caps.cqc_timer_bt_num;</div><div class='ctx'> 	     cqc_count++) {</div><div class='ctx'> 		ret = hns_roce_table_get(hr_dev, &amp;hr_dev-&gt;cqc_timer_table,</div><div class='hunk'>@@ -1879,187 +3049,305 @@ err_qpc_timer_failed:</div><div class='ctx'> 	for (i = 0; i &lt; qpc_count; i++)</div><div class='ctx'> 		hns_roce_table_put(hr_dev, &amp;hr_dev-&gt;qpc_timer_table, i);</div><div class='ctx'> </div><div class='del'>-	hns_roce_free_link_table(hr_dev, &amp;priv-&gt;tpq);</div><div class='del'>-</div><div class='del'>-err_tpq_init_failed:</div><div class='del'>-	hns_roce_free_link_table(hr_dev, &amp;priv-&gt;tsq);</div><div class='add'>+err_gmv_failed:</div><div class='add'>+	for (i = 0; i &lt; gmv_count; i++)</div><div class='add'>+		hns_roce_table_put(hr_dev, &amp;hr_dev-&gt;gmv_table, i);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)</div><div class='add'>+static void put_hem_table(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; hr_dev-&gt;caps.gmv_entry_num; i++)</div><div class='add'>+		hns_roce_table_put(hr_dev, &amp;hr_dev-&gt;gmv_table, i);</div><div class='ctx'> </div><div class='del'>-	hns_roce_free_link_table(hr_dev, &amp;priv-&gt;tpq);</div><div class='del'>-	hns_roce_free_link_table(hr_dev, &amp;priv-&gt;tsq);</div><div class='add'>+	if (hr_dev-&gt;is_vf)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; hr_dev-&gt;caps.qpc_timer_bt_num; i++)</div><div class='add'>+		hns_roce_table_put(hr_dev, &amp;hr_dev-&gt;qpc_timer_table, i);</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; hr_dev-&gt;caps.cqc_timer_bt_num; i++)</div><div class='add'>+		hns_roce_table_put(hr_dev, &amp;hr_dev-&gt;cqc_timer_table, i);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_query_mbox_status(struct hns_roce_dev *hr_dev)</div><div class='add'>+static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_cmq_desc desc;</div><div class='del'>-	struct hns_roce_mbox_status *mb_st =</div><div class='del'>-				       (struct hns_roce_mbox_status *)desc.data;</div><div class='del'>-	enum hns_roce_cmd_return_status status;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_OPC_QUERY_MB_ST, true);</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08) {</div><div class='add'>+		ret = free_mr_init(hr_dev);</div><div class='add'>+		if (ret) {</div><div class='add'>+			dev_err(hr_dev-&gt;dev, "failed to init free mr!\n");</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	status = hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='del'>-	if (status)</div><div class='del'>-		return status;</div><div class='add'>+	/* The hns ROCEE requires the extdb info to be cleared before using */</div><div class='add'>+	ret = hns_roce_clear_extdb_list_info(hr_dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_clear_extdb_failed;</div><div class='ctx'> </div><div class='del'>-	return cpu_to_le32(mb_st-&gt;mb_status_hw_run);</div><div class='del'>-}</div><div class='add'>+	ret = get_hem_table(hr_dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_get_hem_table_failed;</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	u32 status = hns_roce_query_mbox_status(hr_dev);</div><div class='add'>+	if (hr_dev-&gt;is_vf)</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-	return status &gt;&gt; HNS_ROCE_HW_RUN_BIT_SHIFT;</div><div class='add'>+	ret = hns_roce_init_link_table(hr_dev);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "failed to init llm, ret = %d.\n", ret);</div><div class='add'>+		goto err_llm_init_failed;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_llm_init_failed:</div><div class='add'>+	put_hem_table(hr_dev);</div><div class='add'>+err_get_hem_table_failed:</div><div class='add'>+	hns_roce_function_clear(hr_dev);</div><div class='add'>+err_clear_extdb_failed:</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08)</div><div class='add'>+		free_mr_exit(hr_dev);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev)</div><div class='add'>+static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	u32 status = hns_roce_query_mbox_status(hr_dev);</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08)</div><div class='add'>+		free_mr_exit(hr_dev);</div><div class='add'>+</div><div class='add'>+	hns_roce_function_clear(hr_dev);</div><div class='ctx'> </div><div class='del'>-	return status &amp; HNS_ROCE_HW_MB_STATUS_MASK;</div><div class='add'>+	if (!hr_dev-&gt;is_vf)</div><div class='add'>+		hns_roce_free_link_table(hr_dev);</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09)</div><div class='add'>+		free_dip_entry(hr_dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param,</div><div class='del'>-			      u64 out_param, u32 in_modifier, u8 op_modifier,</div><div class='del'>-			      u16 op, u16 token, int event)</div><div class='add'>+static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev,</div><div class='add'>+			      struct hns_roce_mbox_msg *mbox_msg)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_cmq_desc desc;</div><div class='ctx'> 	struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data;</div><div class='ctx'> </div><div class='ctx'> 	hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_OPC_POST_MB, false);</div><div class='ctx'> </div><div class='del'>-	mb-&gt;in_param_l = cpu_to_le64(in_param);</div><div class='del'>-	mb-&gt;in_param_h = cpu_to_le64(in_param) &gt;&gt; 32;</div><div class='del'>-	mb-&gt;out_param_l = cpu_to_le64(out_param);</div><div class='del'>-	mb-&gt;out_param_h = cpu_to_le64(out_param) &gt;&gt; 32;</div><div class='del'>-	mb-&gt;cmd_tag = cpu_to_le32(in_modifier &lt;&lt; 8 | op);</div><div class='del'>-	mb-&gt;token_event_en = cpu_to_le32(event &lt;&lt; 16 | token);</div><div class='add'>+	mb-&gt;in_param_l = cpu_to_le32(mbox_msg-&gt;in_param);</div><div class='add'>+	mb-&gt;in_param_h = cpu_to_le32(mbox_msg-&gt;in_param &gt;&gt; 32);</div><div class='add'>+	mb-&gt;out_param_l = cpu_to_le32(mbox_msg-&gt;out_param);</div><div class='add'>+	mb-&gt;out_param_h = cpu_to_le32(mbox_msg-&gt;out_param &gt;&gt; 32);</div><div class='add'>+	mb-&gt;cmd_tag = cpu_to_le32(mbox_msg-&gt;tag &lt;&lt; 8 | mbox_msg-&gt;cmd);</div><div class='add'>+	mb-&gt;token_event_en = cpu_to_le32(mbox_msg-&gt;event_en &lt;&lt; 16 |</div><div class='add'>+					 mbox_msg-&gt;token);</div><div class='ctx'> </div><div class='ctx'> 	return hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,</div><div class='del'>-				 u64 out_param, u32 in_modifier, u8 op_modifier,</div><div class='del'>-				 u16 op, u16 token, int event)</div><div class='add'>+static int v2_wait_mbox_complete(struct hns_roce_dev *hr_dev, u32 timeout,</div><div class='add'>+				 u8 *complete_status)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	struct hns_roce_mbox_status *mb_st;</div><div class='add'>+	struct hns_roce_cmq_desc desc;</div><div class='ctx'> 	unsigned long end;</div><div class='del'>-	int ret;</div><div class='add'>+	int ret = -EBUSY;</div><div class='add'>+	u32 status;</div><div class='add'>+	bool busy;</div><div class='add'>+</div><div class='add'>+	mb_st = (struct hns_roce_mbox_status *)desc.data;</div><div class='add'>+	end = msecs_to_jiffies(timeout) + jiffies;</div><div class='add'>+	while (v2_chk_mbox_is_avail(hr_dev, &amp;busy)) {</div><div class='add'>+		if (hr_dev-&gt;cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR)</div><div class='add'>+			return -EIO;</div><div class='add'>+</div><div class='add'>+		status = 0;</div><div class='add'>+		hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_OPC_QUERY_MB_ST,</div><div class='add'>+					      true);</div><div class='add'>+		ret = __hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='add'>+		if (!ret) {</div><div class='add'>+			status = le32_to_cpu(mb_st-&gt;mb_status_hw_run);</div><div class='add'>+			/* No pending message exists in ROCEE mbox. */</div><div class='add'>+			if (!(status &amp; MB_ST_HW_RUN_M))</div><div class='add'>+				break;</div><div class='add'>+		} else if (!v2_chk_mbox_is_avail(hr_dev, &amp;busy)) {</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='del'>-	end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies;</div><div class='del'>-	while (hns_roce_v2_cmd_pending(hr_dev)) {</div><div class='ctx'> 		if (time_after(jiffies, end)) {</div><div class='del'>-			dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies,</div><div class='del'>-				(int)end);</div><div class='del'>-			return -EAGAIN;</div><div class='add'>+			dev_err_ratelimited(hr_dev-&gt;dev,</div><div class='add'>+					    "failed to wait mbox status 0x%x\n",</div><div class='add'>+					    status);</div><div class='add'>+			return -ETIMEDOUT;</div><div class='ctx'> 		}</div><div class='add'>+</div><div class='ctx'> 		cond_resched();</div><div class='add'>+		ret = -EBUSY;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier,</div><div class='del'>-				 op_modifier, op, token, event);</div><div class='del'>-	if (ret)</div><div class='del'>-		dev_err(dev, "Post mailbox fail(%d)\n", ret);</div><div class='add'>+	if (!ret) {</div><div class='add'>+		*complete_status = (u8)(status &amp; MB_ST_COMPLETE_M);</div><div class='add'>+	} else if (!v2_chk_mbox_is_avail(hr_dev, &amp;busy)) {</div><div class='add'>+		/* Ignore all errors if the mbox is unavailable. */</div><div class='add'>+		ret = 0;</div><div class='add'>+		*complete_status = MB_ST_COMPLETE_M;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev,</div><div class='del'>-				unsigned long timeout)</div><div class='add'>+static int v2_post_mbox(struct hns_roce_dev *hr_dev,</div><div class='add'>+			struct hns_roce_mbox_msg *mbox_msg)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	unsigned long end = 0;</div><div class='del'>-	u32 status;</div><div class='del'>-</div><div class='del'>-	end = msecs_to_jiffies(timeout) + jiffies;</div><div class='del'>-	while (hns_roce_v2_cmd_pending(hr_dev) &amp;&amp; time_before(jiffies, end))</div><div class='del'>-		cond_resched();</div><div class='add'>+	u8 status = 0;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	if (hns_roce_v2_cmd_pending(hr_dev)) {</div><div class='del'>-		dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n");</div><div class='del'>-		return -ETIMEDOUT;</div><div class='add'>+	/* Waiting for the mbox to be idle */</div><div class='add'>+	ret = v2_wait_mbox_complete(hr_dev, HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS,</div><div class='add'>+				    &amp;status);</div><div class='add'>+	if (unlikely(ret)) {</div><div class='add'>+		dev_err_ratelimited(hr_dev-&gt;dev,</div><div class='add'>+				    "failed to check post mbox status = 0x%x, ret = %d.\n",</div><div class='add'>+				    status, ret);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	status = hns_roce_v2_cmd_complete(hr_dev);</div><div class='del'>-	if (status != 0x1) {</div><div class='del'>-		if (status == CMD_RST_PRC_EBUSY)</div><div class='del'>-			return status;</div><div class='add'>+	/* Post new message to mbox */</div><div class='add'>+	ret = hns_roce_mbox_post(hr_dev, mbox_msg);</div><div class='add'>+	if (ret)</div><div class='add'>+		dev_err_ratelimited(hr_dev-&gt;dev,</div><div class='add'>+				    "failed to post mailbox, ret = %d.\n", ret);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int v2_poll_mbox_done(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	u8 status = 0;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-		dev_err(dev, "mailbox status 0x%x!\n", status);</div><div class='del'>-		return -EBUSY;</div><div class='add'>+	ret = v2_wait_mbox_complete(hr_dev, HNS_ROCE_CMD_TIMEOUT_MSECS,</div><div class='add'>+				    &amp;status);</div><div class='add'>+	if (!ret) {</div><div class='add'>+		if (status != MB_ST_COMPLETE_SUCC)</div><div class='add'>+			return -EBUSY;</div><div class='add'>+	} else {</div><div class='add'>+		dev_err_ratelimited(hr_dev-&gt;dev,</div><div class='add'>+				    "failed to check mbox status = 0x%x, ret = %d.\n",</div><div class='add'>+				    status, ret);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_config_sgid_table(struct hns_roce_dev *hr_dev,</div><div class='del'>-				      int gid_index, const union ib_gid *gid,</div><div class='del'>-				      enum hns_roce_sgid_type sgid_type)</div><div class='add'>+static void copy_gid(void *dest, const union ib_gid *gid)</div><div class='add'>+{</div><div class='add'>+#define GID_SIZE 4</div><div class='add'>+	const union ib_gid *src = gid;</div><div class='add'>+	__le32 (*p)[GID_SIZE] = dest;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (!gid)</div><div class='add'>+		src = &amp;zgid;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; GID_SIZE; i++)</div><div class='add'>+		(*p)[i] = cpu_to_le32(*(u32 *)&amp;src-&gt;raw[i * sizeof(u32)]);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int config_sgid_table(struct hns_roce_dev *hr_dev,</div><div class='add'>+			     int gid_index, const union ib_gid *gid,</div><div class='add'>+			     enum hns_roce_sgid_type sgid_type)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_cmq_desc desc;</div><div class='ctx'> 	struct hns_roce_cfg_sgid_tb *sgid_tb =</div><div class='ctx'> 				    (struct hns_roce_cfg_sgid_tb *)desc.data;</div><div class='del'>-	u32 *p;</div><div class='ctx'> </div><div class='ctx'> 	hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_OPC_CFG_SGID_TB, false);</div><div class='ctx'> </div><div class='del'>-	roce_set_field(sgid_tb-&gt;table_idx_rsv,</div><div class='del'>-		       CFG_SGID_TB_TABLE_IDX_M,</div><div class='del'>-		       CFG_SGID_TB_TABLE_IDX_S, gid_index);</div><div class='del'>-	roce_set_field(sgid_tb-&gt;vf_sgid_type_rsv,</div><div class='del'>-		       CFG_SGID_TB_VF_SGID_TYPE_M,</div><div class='del'>-		       CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type);</div><div class='add'>+	hr_reg_write(sgid_tb, CFG_SGID_TB_TABLE_IDX, gid_index);</div><div class='add'>+	hr_reg_write(sgid_tb, CFG_SGID_TB_VF_SGID_TYPE, sgid_type);</div><div class='ctx'> </div><div class='del'>-	p = (u32 *)&amp;gid-&gt;raw[0];</div><div class='del'>-	sgid_tb-&gt;vf_sgid_l = cpu_to_le32(*p);</div><div class='add'>+	copy_gid(&amp;sgid_tb-&gt;vf_sgid_l, gid);</div><div class='ctx'> </div><div class='del'>-	p = (u32 *)&amp;gid-&gt;raw[4];</div><div class='del'>-	sgid_tb-&gt;vf_sgid_ml = cpu_to_le32(*p);</div><div class='add'>+	return hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	p = (u32 *)&amp;gid-&gt;raw[8];</div><div class='del'>-	sgid_tb-&gt;vf_sgid_mh = cpu_to_le32(*p);</div><div class='add'>+static int config_gmv_table(struct hns_roce_dev *hr_dev,</div><div class='add'>+			    int gid_index, const union ib_gid *gid,</div><div class='add'>+			    enum hns_roce_sgid_type sgid_type,</div><div class='add'>+			    const struct ib_gid_attr *attr)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_cmq_desc desc[2];</div><div class='add'>+	struct hns_roce_cfg_gmv_tb_a *tb_a =</div><div class='add'>+				(struct hns_roce_cfg_gmv_tb_a *)desc[0].data;</div><div class='add'>+	struct hns_roce_cfg_gmv_tb_b *tb_b =</div><div class='add'>+				(struct hns_roce_cfg_gmv_tb_b *)desc[1].data;</div><div class='ctx'> </div><div class='del'>-	p = (u32 *)&amp;gid-&gt;raw[0xc];</div><div class='del'>-	sgid_tb-&gt;vf_sgid_h = cpu_to_le32(*p);</div><div class='add'>+	u16 vlan_id = VLAN_CFI_MASK;</div><div class='add'>+	u8 mac[ETH_ALEN] = {};</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	return hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='add'>+	if (gid) {</div><div class='add'>+		ret = rdma_read_gid_l2_fields(attr, &amp;vlan_id, mac);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc[0], HNS_ROCE_OPC_CFG_GMV_TBL, false);</div><div class='add'>+	desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);</div><div class='add'>+</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc[1], HNS_ROCE_OPC_CFG_GMV_TBL, false);</div><div class='add'>+</div><div class='add'>+	copy_gid(&amp;tb_a-&gt;vf_sgid_l, gid);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(tb_a, GMV_TB_A_VF_SGID_TYPE, sgid_type);</div><div class='add'>+	hr_reg_write(tb_a, GMV_TB_A_VF_VLAN_EN, vlan_id &lt; VLAN_CFI_MASK);</div><div class='add'>+	hr_reg_write(tb_a, GMV_TB_A_VF_VLAN_ID, vlan_id);</div><div class='add'>+</div><div class='add'>+	tb_b-&gt;vf_smac_l = cpu_to_le32(*(u32 *)mac);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(tb_b, GMV_TB_B_SMAC_H, *(u16 *)&amp;mac[4]);</div><div class='add'>+	hr_reg_write(tb_b, GMV_TB_B_SGID_IDX, gid_index);</div><div class='add'>+</div><div class='add'>+	return hns_roce_cmq_send(hr_dev, desc, 2);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port,</div><div class='del'>-			       int gid_index, const union ib_gid *gid,</div><div class='add'>+static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, int gid_index,</div><div class='add'>+			       const union ib_gid *gid,</div><div class='ctx'> 			       const struct ib_gid_attr *attr)</div><div class='ctx'> {</div><div class='ctx'> 	enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	if (!gid || !attr)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	if (attr-&gt;gid_type == IB_GID_TYPE_ROCE)</div><div class='del'>-		sgid_type = GID_TYPE_FLAG_ROCE_V1;</div><div class='del'>-</div><div class='del'>-	if (attr-&gt;gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {</div><div class='del'>-		if (ipv6_addr_v4mapped((void *)gid))</div><div class='del'>-			sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;</div><div class='del'>-		else</div><div class='del'>-			sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;</div><div class='add'>+	if (gid) {</div><div class='add'>+		if (attr-&gt;gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {</div><div class='add'>+			if (ipv6_addr_v4mapped((void *)gid))</div><div class='add'>+				sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;</div><div class='add'>+			else</div><div class='add'>+				sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;</div><div class='add'>+		} else if (attr-&gt;gid_type == IB_GID_TYPE_ROCE) {</div><div class='add'>+			sgid_type = GID_TYPE_FLAG_ROCE_V1;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_config_sgid_table(hr_dev, gid_index, gid, sgid_type);</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09)</div><div class='add'>+		ret = config_gmv_table(hr_dev, gid_index, gid, sgid_type, attr);</div><div class='add'>+	else</div><div class='add'>+		ret = config_sgid_table(hr_dev, gid_index, gid, sgid_type);</div><div class='add'>+</div><div class='ctx'> 	if (ret)</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "Configure sgid table failed(%d)!\n", ret);</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev, "failed to set gid, ret = %d!\n",</div><div class='add'>+			  ret);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,</div><div class='del'>-			       u8 *addr)</div><div class='add'>+			       const u8 *addr)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_cmq_desc desc;</div><div class='ctx'> 	struct hns_roce_cfg_smac_tb *smac_tb =</div><div class='hunk'>@@ -2072,103 +3360,71 @@ static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,</div><div class='ctx'> 	reg_smac_l = *(u32 *)(&amp;addr[0]);</div><div class='ctx'> 	reg_smac_h = *(u16 *)(&amp;addr[4]);</div><div class='ctx'> </div><div class='del'>-	memset(smac_tb, 0, sizeof(*smac_tb));</div><div class='del'>-	roce_set_field(smac_tb-&gt;tb_idx_rsv,</div><div class='del'>-		       CFG_SMAC_TB_IDX_M,</div><div class='del'>-		       CFG_SMAC_TB_IDX_S, phy_port);</div><div class='del'>-	roce_set_field(smac_tb-&gt;vf_smac_h_rsv,</div><div class='del'>-		       CFG_SMAC_TB_VF_SMAC_H_M,</div><div class='del'>-		       CFG_SMAC_TB_VF_SMAC_H_S, reg_smac_h);</div><div class='del'>-	smac_tb-&gt;vf_smac_l = reg_smac_l;</div><div class='add'>+	hr_reg_write(smac_tb, CFG_SMAC_TB_IDX, phy_port);</div><div class='add'>+	hr_reg_write(smac_tb, CFG_SMAC_TB_VF_SMAC_H, reg_smac_h);</div><div class='add'>+	smac_tb-&gt;vf_smac_l = cpu_to_le32(reg_smac_l);</div><div class='ctx'> </div><div class='ctx'> 	return hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int set_mtpt_pbl(struct hns_roce_v2_mpt_entry *mpt_entry,</div><div class='add'>+static int set_mtpt_pbl(struct hns_roce_dev *hr_dev,</div><div class='add'>+			struct hns_roce_v2_mpt_entry *mpt_entry,</div><div class='ctx'> 			struct hns_roce_mr *mr)</div><div class='ctx'> {</div><div class='del'>-	struct sg_dma_page_iter sg_iter;</div><div class='del'>-	u64 page_addr;</div><div class='del'>-	u64 *pages;</div><div class='add'>+	u64 pages[HNS_ROCE_V2_MAX_INNER_MTPT_NUM] = { 0 };</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	dma_addr_t pbl_ba;</div><div class='add'>+	int ret;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	mpt_entry-&gt;pbl_size = cpu_to_le32(mr-&gt;pbl_size);</div><div class='del'>-	mpt_entry-&gt;pbl_ba_l = cpu_to_le32(lower_32_bits(mr-&gt;pbl_ba &gt;&gt; 3));</div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_48_mode_ba,</div><div class='del'>-		       V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S,</div><div class='del'>-		       upper_32_bits(mr-&gt;pbl_ba &gt;&gt; 3));</div><div class='add'>+	ret = hns_roce_mtr_find(hr_dev, &amp;mr-&gt;pbl_mtr, 0, pages,</div><div class='add'>+				min_t(int, ARRAY_SIZE(pages), mr-&gt;npages));</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to find PBL mtr, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Aligned to the hardware address access unit */</div><div class='add'>+	for (i = 0; i &lt; ARRAY_SIZE(pages); i++)</div><div class='add'>+		pages[i] &gt;&gt;= MPT_PBL_BUF_ADDR_S;</div><div class='ctx'> </div><div class='del'>-	pages = (u64 *)__get_free_page(GFP_KERNEL);</div><div class='del'>-	if (!pages)</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+	pbl_ba = hns_roce_get_mtr_ba(&amp;mr-&gt;pbl_mtr);</div><div class='ctx'> </div><div class='del'>-	i = 0;</div><div class='del'>-	for_each_sg_dma_page(mr-&gt;umem-&gt;sg_head.sgl, &amp;sg_iter, mr-&gt;umem-&gt;nmap, 0) {</div><div class='del'>-		page_addr = sg_page_iter_dma_address(&amp;sg_iter);</div><div class='del'>-		pages[i] = page_addr &gt;&gt; 6;</div><div class='add'>+	mpt_entry-&gt;pbl_size = cpu_to_le32(mr-&gt;npages);</div><div class='add'>+	mpt_entry-&gt;pbl_ba_l = cpu_to_le32(pbl_ba &gt;&gt; MPT_PBL_BA_ADDR_S);</div><div class='add'>+	hr_reg_write(mpt_entry, MPT_PBL_BA_H,</div><div class='add'>+		     upper_32_bits(pbl_ba &gt;&gt; MPT_PBL_BA_ADDR_S));</div><div class='ctx'> </div><div class='del'>-		/* Record the first 2 entry directly to MTPT table */</div><div class='del'>-		if (i &gt;= HNS_ROCE_V2_MAX_INNER_MTPT_NUM - 1)</div><div class='del'>-			goto found;</div><div class='del'>-		i++;</div><div class='del'>-	}</div><div class='del'>-found:</div><div class='ctx'> 	mpt_entry-&gt;pa0_l = cpu_to_le32(lower_32_bits(pages[0]));</div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M,</div><div class='del'>-		       V2_MPT_BYTE_56_PA0_H_S, upper_32_bits(pages[0]));</div><div class='add'>+	hr_reg_write(mpt_entry, MPT_PA0_H, upper_32_bits(pages[0]));</div><div class='ctx'> </div><div class='ctx'> 	mpt_entry-&gt;pa1_l = cpu_to_le32(lower_32_bits(pages[1]));</div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_64_buf_pa1, V2_MPT_BYTE_64_PA1_H_M,</div><div class='del'>-		       V2_MPT_BYTE_64_PA1_H_S, upper_32_bits(pages[1]));</div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_64_buf_pa1,</div><div class='del'>-		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,</div><div class='del'>-		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,</div><div class='del'>-		       mr-&gt;pbl_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-</div><div class='del'>-	free_page((unsigned long)pages);</div><div class='add'>+	hr_reg_write(mpt_entry, MPT_PA1_H, upper_32_bits(pages[1]));</div><div class='add'>+	hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ,</div><div class='add'>+		     to_hr_hw_page_shift(mr-&gt;pbl_mtr.hem_cfg.buf_pg_shift));</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,</div><div class='del'>-				  unsigned long mtpt_idx)</div><div class='add'>+static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev,</div><div class='add'>+				  void *mb_buf, struct hns_roce_mr *mr)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_v2_mpt_entry *mpt_entry;</div><div class='del'>-	int ret;</div><div class='ctx'> </div><div class='ctx'> 	mpt_entry = mb_buf;</div><div class='ctx'> 	memset(mpt_entry, 0, sizeof(*mpt_entry));</div><div class='ctx'> </div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,</div><div class='del'>-		       V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);</div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,</div><div class='del'>-		       V2_MPT_BYTE_4_PBL_HOP_NUM_S, mr-&gt;pbl_hop_num ==</div><div class='del'>-		       HNS_ROCE_HOP_NUM_0 ? 0 : mr-&gt;pbl_hop_num);</div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_4_pd_hop_st,</div><div class='del'>-		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,</div><div class='del'>-		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,</div><div class='del'>-		       mr-&gt;pbl_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,</div><div class='del'>-		       V2_MPT_BYTE_4_PD_S, mr-&gt;pd);</div><div class='del'>-</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 0);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S,</div><div class='del'>-		     (mr-&gt;access &amp; IB_ACCESS_MW_BIND ? 1 : 0));</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S,</div><div class='del'>-		     mr-&gt;access &amp; IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,</div><div class='del'>-		     (mr-&gt;access &amp; IB_ACCESS_REMOTE_READ ? 1 : 0));</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,</div><div class='del'>-		     (mr-&gt;access &amp; IB_ACCESS_REMOTE_WRITE ? 1 : 0));</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,</div><div class='del'>-		     (mr-&gt;access &amp; IB_ACCESS_LOCAL_WRITE ? 1 : 0));</div><div class='del'>-</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_12_mw_pa, V2_MPT_BYTE_12_PA_S,</div><div class='del'>-		     mr-&gt;type == MR_TYPE_MR ? 0 : 1);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_12_mw_pa, V2_MPT_BYTE_12_INNER_PA_VLD_S,</div><div class='del'>-		     1);</div><div class='add'>+	hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_VALID);</div><div class='add'>+	hr_reg_write(mpt_entry, MPT_PD, mr-&gt;pd);</div><div class='add'>+</div><div class='add'>+	hr_reg_write_bool(mpt_entry, MPT_ATOMIC_EN,</div><div class='add'>+			  mr-&gt;access &amp; IB_ACCESS_REMOTE_ATOMIC);</div><div class='add'>+	hr_reg_write_bool(mpt_entry, MPT_RR_EN,</div><div class='add'>+			  mr-&gt;access &amp; IB_ACCESS_REMOTE_READ);</div><div class='add'>+	hr_reg_write_bool(mpt_entry, MPT_RW_EN,</div><div class='add'>+			  mr-&gt;access &amp; IB_ACCESS_REMOTE_WRITE);</div><div class='add'>+	hr_reg_write_bool(mpt_entry, MPT_LW_EN,</div><div class='add'>+			  mr-&gt;access &amp; IB_ACCESS_LOCAL_WRITE);</div><div class='ctx'> </div><div class='ctx'> 	mpt_entry-&gt;len_l = cpu_to_le32(lower_32_bits(mr-&gt;size));</div><div class='ctx'> 	mpt_entry-&gt;len_h = cpu_to_le32(upper_32_bits(mr-&gt;size));</div><div class='hunk'>@@ -2176,56 +3432,51 @@ static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,</div><div class='ctx'> 	mpt_entry-&gt;va_l = cpu_to_le32(lower_32_bits(mr-&gt;iova));</div><div class='ctx'> 	mpt_entry-&gt;va_h = cpu_to_le32(upper_32_bits(mr-&gt;iova));</div><div class='ctx'> </div><div class='add'>+	if (mr-&gt;type != MR_TYPE_MR)</div><div class='add'>+		hr_reg_enable(mpt_entry, MPT_PA);</div><div class='add'>+</div><div class='ctx'> 	if (mr-&gt;type == MR_TYPE_DMA)</div><div class='ctx'> 		return 0;</div><div class='ctx'> </div><div class='del'>-	ret = set_mtpt_pbl(mpt_entry, mr);</div><div class='add'>+	if (mr-&gt;pbl_hop_num != HNS_ROCE_HOP_NUM_0)</div><div class='add'>+		hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM, mr-&gt;pbl_hop_num);</div><div class='ctx'> </div><div class='del'>-	return ret;</div><div class='add'>+	hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ,</div><div class='add'>+		     to_hr_hw_page_shift(mr-&gt;pbl_mtr.hem_cfg.ba_pg_shift));</div><div class='add'>+	hr_reg_enable(mpt_entry, MPT_INNER_PA_VLD);</div><div class='add'>+</div><div class='add'>+	return set_mtpt_pbl(hr_dev, mpt_entry, mr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 					struct hns_roce_mr *mr, int flags,</div><div class='del'>-					u32 pdn, int mr_access_flags, u64 iova,</div><div class='del'>-					u64 size, void *mb_buf)</div><div class='add'>+					void *mb_buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf;</div><div class='add'>+	u32 mr_access_flags = mr-&gt;access;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,</div><div class='del'>-		       V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);</div><div class='del'>-</div><div class='del'>-	if (flags &amp; IB_MR_REREG_PD) {</div><div class='del'>-		roce_set_field(mpt_entry-&gt;byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,</div><div class='del'>-			       V2_MPT_BYTE_4_PD_S, pdn);</div><div class='del'>-		mr-&gt;pd = pdn;</div><div class='del'>-	}</div><div class='add'>+	hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_VALID);</div><div class='add'>+	hr_reg_write(mpt_entry, MPT_PD, mr-&gt;pd);</div><div class='ctx'> </div><div class='ctx'> 	if (flags &amp; IB_MR_REREG_ACCESS) {</div><div class='del'>-		roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en,</div><div class='del'>-			     V2_MPT_BYTE_8_BIND_EN_S,</div><div class='del'>-			     (mr_access_flags &amp; IB_ACCESS_MW_BIND ? 1 : 0));</div><div class='del'>-		roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en,</div><div class='del'>-			     V2_MPT_BYTE_8_ATOMIC_EN_S,</div><div class='add'>+		hr_reg_write(mpt_entry, MPT_ATOMIC_EN,</div><div class='ctx'> 			     mr_access_flags &amp; IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);</div><div class='del'>-		roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,</div><div class='add'>+		hr_reg_write(mpt_entry, MPT_RR_EN,</div><div class='ctx'> 			     mr_access_flags &amp; IB_ACCESS_REMOTE_READ ? 1 : 0);</div><div class='del'>-		roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,</div><div class='add'>+		hr_reg_write(mpt_entry, MPT_RW_EN,</div><div class='ctx'> 			     mr_access_flags &amp; IB_ACCESS_REMOTE_WRITE ? 1 : 0);</div><div class='del'>-		roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,</div><div class='add'>+		hr_reg_write(mpt_entry, MPT_LW_EN,</div><div class='ctx'> 			     mr_access_flags &amp; IB_ACCESS_LOCAL_WRITE ? 1 : 0);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (flags &amp; IB_MR_REREG_TRANS) {</div><div class='del'>-		mpt_entry-&gt;va_l = cpu_to_le32(lower_32_bits(iova));</div><div class='del'>-		mpt_entry-&gt;va_h = cpu_to_le32(upper_32_bits(iova));</div><div class='del'>-		mpt_entry-&gt;len_l = cpu_to_le32(lower_32_bits(size));</div><div class='del'>-		mpt_entry-&gt;len_h = cpu_to_le32(upper_32_bits(size));</div><div class='add'>+		mpt_entry-&gt;va_l = cpu_to_le32(lower_32_bits(mr-&gt;iova));</div><div class='add'>+		mpt_entry-&gt;va_h = cpu_to_le32(upper_32_bits(mr-&gt;iova));</div><div class='add'>+		mpt_entry-&gt;len_l = cpu_to_le32(lower_32_bits(mr-&gt;size));</div><div class='add'>+		mpt_entry-&gt;len_h = cpu_to_le32(upper_32_bits(mr-&gt;size));</div><div class='ctx'> </div><div class='del'>-		mr-&gt;iova = iova;</div><div class='del'>-		mr-&gt;size = size;</div><div class='del'>-</div><div class='del'>-		ret = set_mtpt_pbl(mpt_entry, mr);</div><div class='add'>+		ret = set_mtpt_pbl(hr_dev, mpt_entry, mr);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -2233,135 +3484,165 @@ static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr)</div><div class='ctx'> {</div><div class='add'>+	dma_addr_t pbl_ba = hns_roce_get_mtr_ba(&amp;mr-&gt;pbl_mtr);</div><div class='ctx'> 	struct hns_roce_v2_mpt_entry *mpt_entry;</div><div class='ctx'> </div><div class='ctx'> 	mpt_entry = mb_buf;</div><div class='ctx'> 	memset(mpt_entry, 0, sizeof(*mpt_entry));</div><div class='ctx'> </div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,</div><div class='del'>-		       V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);</div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,</div><div class='del'>-		       V2_MPT_BYTE_4_PBL_HOP_NUM_S, 1);</div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_4_pd_hop_st,</div><div class='del'>-		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,</div><div class='del'>-		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,</div><div class='del'>-		       mr-&gt;pbl_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,</div><div class='del'>-		       V2_MPT_BYTE_4_PD_S, mr-&gt;pd);</div><div class='del'>-</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 1);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);</div><div class='del'>-</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_12_mw_pa, V2_MPT_BYTE_12_FRE_S, 1);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 0);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);</div><div class='del'>-</div><div class='del'>-	mpt_entry-&gt;pbl_size = cpu_to_le32(mr-&gt;pbl_size);</div><div class='del'>-</div><div class='del'>-	mpt_entry-&gt;pbl_ba_l = cpu_to_le32(lower_32_bits(mr-&gt;pbl_ba &gt;&gt; 3));</div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M,</div><div class='del'>-		       V2_MPT_BYTE_48_PBL_BA_H_S,</div><div class='del'>-		       upper_32_bits(mr-&gt;pbl_ba &gt;&gt; 3));</div><div class='del'>-</div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_64_buf_pa1,</div><div class='del'>-		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,</div><div class='del'>-		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,</div><div class='del'>-		       mr-&gt;pbl_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='add'>+	hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_FREE);</div><div class='add'>+	hr_reg_write(mpt_entry, MPT_PD, mr-&gt;pd);</div><div class='add'>+</div><div class='add'>+	hr_reg_enable(mpt_entry, MPT_RA_EN);</div><div class='add'>+	hr_reg_enable(mpt_entry, MPT_R_INV_EN);</div><div class='add'>+</div><div class='add'>+	hr_reg_enable(mpt_entry, MPT_FRE);</div><div class='add'>+	hr_reg_enable(mpt_entry, MPT_BPD);</div><div class='add'>+	hr_reg_clear(mpt_entry, MPT_PA);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM, 1);</div><div class='add'>+	hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ,</div><div class='add'>+		     to_hr_hw_page_shift(mr-&gt;pbl_mtr.hem_cfg.ba_pg_shift));</div><div class='add'>+	hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ,</div><div class='add'>+		     to_hr_hw_page_shift(mr-&gt;pbl_mtr.hem_cfg.buf_pg_shift));</div><div class='add'>+</div><div class='add'>+	mpt_entry-&gt;pbl_size = cpu_to_le32(mr-&gt;npages);</div><div class='add'>+</div><div class='add'>+	mpt_entry-&gt;pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba &gt;&gt;</div><div class='add'>+							MPT_PBL_BA_ADDR_S));</div><div class='add'>+	hr_reg_write(mpt_entry, MPT_PBL_BA_H,</div><div class='add'>+		     upper_32_bits(pbl_ba &gt;&gt; MPT_PBL_BA_ADDR_S));</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw)</div><div class='add'>+static int free_mr_post_send_lp_wqe(struct hns_roce_qp *hr_qp)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_v2_mpt_entry *mpt_entry;</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp-&gt;ibqp.device);</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	const struct ib_send_wr *bad_wr;</div><div class='add'>+	struct ib_rdma_wr rdma_wr = {};</div><div class='add'>+	struct ib_send_wr *send_wr;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	mpt_entry = mb_buf;</div><div class='del'>-	memset(mpt_entry, 0, sizeof(*mpt_entry));</div><div class='add'>+	send_wr = &amp;rdma_wr.wr;</div><div class='add'>+	send_wr-&gt;opcode = IB_WR_RDMA_WRITE;</div><div class='ctx'> </div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,</div><div class='del'>-		       V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);</div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,</div><div class='del'>-		       V2_MPT_BYTE_4_PD_S, mw-&gt;pdn);</div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_4_pd_hop_st,</div><div class='del'>-		       V2_MPT_BYTE_4_PBL_HOP_NUM_M,</div><div class='del'>-		       V2_MPT_BYTE_4_PBL_HOP_NUM_S,</div><div class='del'>-		       mw-&gt;pbl_hop_num == HNS_ROCE_HOP_NUM_0 ?</div><div class='del'>-		       0 : mw-&gt;pbl_hop_num);</div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_4_pd_hop_st,</div><div class='del'>-		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,</div><div class='del'>-		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,</div><div class='del'>-		       mw-&gt;pbl_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);</div><div class='del'>-</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 1);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);</div><div class='del'>-	roce_set_bit(mpt_entry-&gt;byte_12_mw_pa, V2_MPT_BYTE_12_BQP_S,</div><div class='del'>-		     mw-&gt;ibmw.type == IB_MW_TYPE_1 ? 0 : 1);</div><div class='del'>-</div><div class='del'>-	roce_set_field(mpt_entry-&gt;byte_64_buf_pa1,</div><div class='del'>-		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,</div><div class='del'>-		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,</div><div class='del'>-		       mw-&gt;pbl_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-</div><div class='del'>-	mpt_entry-&gt;lkey = cpu_to_le32(mw-&gt;rkey);</div><div class='add'>+	ret = hns_roce_v2_post_send(&amp;hr_qp-&gt;ibqp, send_wr, &amp;bad_wr);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err_ratelimited(ibdev, "failed to post wqe for free mr, ret = %d.\n",</div><div class='add'>+				      ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)</div><div class='del'>-{</div><div class='del'>-	return hns_roce_buf_offset(&amp;hr_cq-&gt;hr_buf.hr_buf,</div><div class='del'>-				   n * HNS_ROCE_V2_CQE_ENTRY_SIZE);</div><div class='del'>-}</div><div class='add'>+static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,</div><div class='add'>+			       struct ib_wc *wc);</div><div class='ctx'> </div><div class='del'>-static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n)</div><div class='add'>+static void free_mr_send_cmd_to_hw(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n &amp; hr_cq-&gt;ib_cq.cqe);</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	struct hns_roce_v2_free_mr *free_mr = &amp;priv-&gt;free_mr;</div><div class='add'>+	struct ib_wc wc[ARRAY_SIZE(free_mr-&gt;rsv_qp)];</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	struct hns_roce_qp *hr_qp;</div><div class='add'>+	unsigned long end;</div><div class='add'>+	int cqe_cnt = 0;</div><div class='add'>+	int npolled;</div><div class='add'>+	int ret;</div><div class='add'>+	int i;</div><div class='ctx'> </div><div class='del'>-	/* Get cqe when Owner bit is Conversely with the MSB of cons_idx */</div><div class='del'>-	return (roce_get_bit(cqe-&gt;byte_4, V2_CQE_BYTE_4_OWNER_S) ^</div><div class='del'>-		!!(n &amp; (hr_cq-&gt;ib_cq.cqe + 1))) ? cqe : NULL;</div><div class='add'>+	/*</div><div class='add'>+	 * If the device initialization is not complete or in the uninstall</div><div class='add'>+	 * process, then there is no need to execute free mr.</div><div class='add'>+	 */</div><div class='add'>+	if (priv-&gt;handle-&gt;rinfo.reset_state == HNS_ROCE_STATE_RST_INIT ||</div><div class='add'>+	    priv-&gt;handle-&gt;rinfo.instance_state == HNS_ROCE_STATE_INIT ||</div><div class='add'>+	    hr_dev-&gt;state == HNS_ROCE_DEVICE_STATE_UNINIT)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;free_mr-&gt;mutex);</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ARRAY_SIZE(free_mr-&gt;rsv_qp); i++) {</div><div class='add'>+		hr_qp = free_mr-&gt;rsv_qp[i];</div><div class='add'>+</div><div class='add'>+		ret = free_mr_post_send_lp_wqe(hr_qp);</div><div class='add'>+		if (ret) {</div><div class='add'>+			ibdev_err_ratelimited(ibdev,</div><div class='add'>+					      "failed to send wqe (qp:0x%lx) for free mr, ret = %d.\n",</div><div class='add'>+					      hr_qp-&gt;qpn, ret);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		cqe_cnt++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	end = msecs_to_jiffies(HNS_ROCE_V2_FREE_MR_TIMEOUT) + jiffies;</div><div class='add'>+	while (cqe_cnt) {</div><div class='add'>+		npolled = hns_roce_v2_poll_cq(&amp;free_mr-&gt;rsv_cq-&gt;ib_cq, cqe_cnt, wc);</div><div class='add'>+		if (npolled &lt; 0) {</div><div class='add'>+			ibdev_err_ratelimited(ibdev,</div><div class='add'>+					      "failed to poll cqe for free mr, remain %d cqe.\n",</div><div class='add'>+					      cqe_cnt);</div><div class='add'>+			goto out;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (time_after(jiffies, end)) {</div><div class='add'>+			ibdev_err_ratelimited(ibdev,</div><div class='add'>+					      "failed to poll cqe for free mr and timeout, remain %d cqe.\n",</div><div class='add'>+					      cqe_cnt);</div><div class='add'>+			goto out;</div><div class='add'>+		}</div><div class='add'>+		cqe_cnt -= npolled;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	mutex_unlock(&amp;free_mr-&gt;mutex);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *hr_cq)</div><div class='add'>+static void hns_roce_v2_dereg_mr(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	return get_sw_cqe_v2(hr_cq, hr_cq-&gt;cons_index);</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08)</div><div class='add'>+		free_mr_send_cmd_to_hw(hr_dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void *get_srq_wqe(struct hns_roce_srq *srq, int n)</div><div class='add'>+static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)</div><div class='ctx'> {</div><div class='del'>-	return hns_roce_buf_offset(&amp;srq-&gt;buf, n &lt;&lt; srq-&gt;wqe_shift);</div><div class='add'>+	return hns_roce_buf_offset(hr_cq-&gt;mtr.kmem, n * hr_cq-&gt;cqe_size);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index)</div><div class='add'>+static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, unsigned int n)</div><div class='ctx'> {</div><div class='del'>-	u32 bitmap_num;</div><div class='del'>-	int bit_num;</div><div class='del'>-</div><div class='del'>-	/* always called with interrupts disabled. */</div><div class='del'>-	spin_lock(&amp;srq-&gt;lock);</div><div class='del'>-</div><div class='del'>-	bitmap_num = wqe_index / (sizeof(u64) * 8);</div><div class='del'>-	bit_num = wqe_index % (sizeof(u64) * 8);</div><div class='del'>-	srq-&gt;idx_que.bitmap[bitmap_num] |= (1ULL &lt;&lt; bit_num);</div><div class='del'>-	srq-&gt;tail++;</div><div class='add'>+	struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n &amp; hr_cq-&gt;ib_cq.cqe);</div><div class='ctx'> </div><div class='del'>-	spin_unlock(&amp;srq-&gt;lock);</div><div class='add'>+	/* Get cqe when Owner bit is Conversely with the MSB of cons_idx */</div><div class='add'>+	return (hr_reg_read(cqe, CQE_OWNER) ^ !!(n &amp; hr_cq-&gt;cq_depth)) ? cqe :</div><div class='add'>+									 NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)</div><div class='add'>+static inline void update_cq_db(struct hns_roce_dev *hr_dev,</div><div class='add'>+				struct hns_roce_cq *hr_cq)</div><div class='ctx'> {</div><div class='del'>-	*hr_cq-&gt;set_ci_db = cons_index &amp; 0xffffff;</div><div class='add'>+	if (likely(hr_cq-&gt;flags &amp; HNS_ROCE_CQ_FLAG_RECORD_DB)) {</div><div class='add'>+		*hr_cq-&gt;set_ci_db = hr_cq-&gt;cons_index &amp; V2_CQ_DB_CONS_IDX_M;</div><div class='add'>+	} else {</div><div class='add'>+		struct hns_roce_v2_db cq_db = {};</div><div class='add'>+</div><div class='add'>+		hr_reg_write(&amp;cq_db, DB_TAG, hr_cq-&gt;cqn);</div><div class='add'>+		hr_reg_write(&amp;cq_db, DB_CMD, HNS_ROCE_V2_CQ_DB);</div><div class='add'>+		hr_reg_write(&amp;cq_db, DB_CQ_CI, hr_cq-&gt;cons_index);</div><div class='add'>+		hr_reg_write(&amp;cq_db, DB_CQ_CMD_SN, 1);</div><div class='add'>+</div><div class='add'>+		hns_roce_write64(hr_dev, (__le32 *)&amp;cq_db, hr_cq-&gt;db_reg);</div><div class='add'>+	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,</div><div class='ctx'> 				   struct hns_roce_srq *srq)</div><div class='ctx'> {</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq-&gt;ib_cq.device);</div><div class='ctx'> 	struct hns_roce_v2_cqe *cqe, *dest;</div><div class='ctx'> 	u32 prod_index;</div><div class='ctx'> 	int nfreed = 0;</div><div class='hunk'>@@ -2370,7 +3651,7 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,</div><div class='ctx'> </div><div class='ctx'> 	for (prod_index = hr_cq-&gt;cons_index; get_sw_cqe_v2(hr_cq, prod_index);</div><div class='ctx'> 	     ++prod_index) {</div><div class='del'>-		if (prod_index == hr_cq-&gt;cons_index + hr_cq-&gt;ib_cq.cqe)</div><div class='add'>+		if (prod_index &gt; hr_cq-&gt;cons_index + hr_cq-&gt;ib_cq.cqe)</div><div class='ctx'> 			break;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -2380,36 +3661,24 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,</div><div class='ctx'> 	 */</div><div class='ctx'> 	while ((int) --prod_index - (int) hr_cq-&gt;cons_index &gt;= 0) {</div><div class='ctx'> 		cqe = get_cqe_v2(hr_cq, prod_index &amp; hr_cq-&gt;ib_cq.cqe);</div><div class='del'>-		if ((roce_get_field(cqe-&gt;byte_16, V2_CQE_BYTE_16_LCL_QPN_M,</div><div class='del'>-				    V2_CQE_BYTE_16_LCL_QPN_S) &amp;</div><div class='del'>-				    HNS_ROCE_V2_CQE_QPN_MASK) == qpn) {</div><div class='del'>-			if (srq &amp;&amp;</div><div class='del'>-			    roce_get_bit(cqe-&gt;byte_4, V2_CQE_BYTE_4_S_R_S)) {</div><div class='del'>-				wqe_index = roce_get_field(cqe-&gt;byte_4,</div><div class='del'>-						     V2_CQE_BYTE_4_WQE_INDX_M,</div><div class='del'>-						     V2_CQE_BYTE_4_WQE_INDX_S);</div><div class='add'>+		if (hr_reg_read(cqe, CQE_LCL_QPN) == qpn) {</div><div class='add'>+			if (srq &amp;&amp; hr_reg_read(cqe, CQE_S_R)) {</div><div class='add'>+				wqe_index = hr_reg_read(cqe, CQE_WQE_IDX);</div><div class='ctx'> 				hns_roce_free_srq_wqe(srq, wqe_index);</div><div class='ctx'> 			}</div><div class='ctx'> 			++nfreed;</div><div class='ctx'> 		} else if (nfreed) {</div><div class='ctx'> 			dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &amp;</div><div class='ctx'> 					  hr_cq-&gt;ib_cq.cqe);</div><div class='del'>-			owner_bit = roce_get_bit(dest-&gt;byte_4,</div><div class='del'>-						 V2_CQE_BYTE_4_OWNER_S);</div><div class='del'>-			memcpy(dest, cqe, sizeof(*cqe));</div><div class='del'>-			roce_set_bit(dest-&gt;byte_4, V2_CQE_BYTE_4_OWNER_S,</div><div class='del'>-				     owner_bit);</div><div class='add'>+			owner_bit = hr_reg_read(dest, CQE_OWNER);</div><div class='add'>+			memcpy(dest, cqe, hr_cq-&gt;cqe_size);</div><div class='add'>+			hr_reg_write(dest, CQE_OWNER, owner_bit);</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (nfreed) {</div><div class='ctx'> 		hr_cq-&gt;cons_index += nfreed;</div><div class='del'>-		/*</div><div class='del'>-		 * Make sure update of buffer contents is done before</div><div class='del'>-		 * updating consumer index.</div><div class='del'>-		 */</div><div class='del'>-		wmb();</div><div class='del'>-		hns_roce_v2_cq_set_ci(hr_cq, hr_cq-&gt;cons_index);</div><div class='add'>+		update_cq_db(hr_dev, hr_cq);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -2423,77 +3692,51 @@ static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,</div><div class='ctx'> </div><div class='ctx'> static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 				  struct hns_roce_cq *hr_cq, void *mb_buf,</div><div class='del'>-				  u64 *mtts, dma_addr_t dma_handle, int nent,</div><div class='del'>-				  u32 vector)</div><div class='add'>+				  u64 *mtts, dma_addr_t dma_handle)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_v2_cq_context *cq_context;</div><div class='ctx'> </div><div class='ctx'> 	cq_context = mb_buf;</div><div class='ctx'> 	memset(cq_context, 0, sizeof(*cq_context));</div><div class='ctx'> </div><div class='del'>-	roce_set_field(cq_context-&gt;byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M,</div><div class='del'>-		       V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID);</div><div class='del'>-	roce_set_field(cq_context-&gt;byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M,</div><div class='del'>-		       V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE);</div><div class='del'>-	roce_set_field(cq_context-&gt;byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M,</div><div class='del'>-		       V2_CQC_BYTE_4_SHIFT_S, ilog2((unsigned int)nent));</div><div class='del'>-	roce_set_field(cq_context-&gt;byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M,</div><div class='del'>-		       V2_CQC_BYTE_4_CEQN_S, vector);</div><div class='del'>-	cq_context-&gt;byte_4_pg_ceqn = cpu_to_le32(cq_context-&gt;byte_4_pg_ceqn);</div><div class='del'>-</div><div class='del'>-	roce_set_field(cq_context-&gt;byte_8_cqn, V2_CQC_BYTE_8_CQN_M,</div><div class='del'>-		       V2_CQC_BYTE_8_CQN_S, hr_cq-&gt;cqn);</div><div class='del'>-</div><div class='del'>-	cq_context-&gt;cqe_cur_blk_addr = (u32)(mtts[0] &gt;&gt; PAGE_ADDR_SHIFT);</div><div class='del'>-	cq_context-&gt;cqe_cur_blk_addr =</div><div class='del'>-				cpu_to_le32(cq_context-&gt;cqe_cur_blk_addr);</div><div class='del'>-</div><div class='del'>-	roce_set_field(cq_context-&gt;byte_16_hop_addr,</div><div class='del'>-		       V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M,</div><div class='del'>-		       V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S,</div><div class='del'>-		       cpu_to_le32((mtts[0]) &gt;&gt; (32 + PAGE_ADDR_SHIFT)));</div><div class='del'>-	roce_set_field(cq_context-&gt;byte_16_hop_addr,</div><div class='del'>-		       V2_CQC_BYTE_16_CQE_HOP_NUM_M,</div><div class='del'>-		       V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev-&gt;caps.cqe_hop_num ==</div><div class='del'>-		       HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev-&gt;caps.cqe_hop_num);</div><div class='del'>-</div><div class='del'>-	cq_context-&gt;cqe_nxt_blk_addr = (u32)(mtts[1] &gt;&gt; PAGE_ADDR_SHIFT);</div><div class='del'>-	roce_set_field(cq_context-&gt;byte_24_pgsz_addr,</div><div class='del'>-		       V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M,</div><div class='del'>-		       V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S,</div><div class='del'>-		       cpu_to_le32((mtts[1]) &gt;&gt; (32 + PAGE_ADDR_SHIFT)));</div><div class='del'>-	roce_set_field(cq_context-&gt;byte_24_pgsz_addr,</div><div class='del'>-		       V2_CQC_BYTE_24_CQE_BA_PG_SZ_M,</div><div class='del'>-		       V2_CQC_BYTE_24_CQE_BA_PG_SZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.cqe_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_field(cq_context-&gt;byte_24_pgsz_addr,</div><div class='del'>-		       V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M,</div><div class='del'>-		       V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.cqe_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-</div><div class='del'>-	cq_context-&gt;cqe_ba = (u32)(dma_handle &gt;&gt; 3);</div><div class='del'>-</div><div class='del'>-	roce_set_field(cq_context-&gt;byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M,</div><div class='del'>-		       V2_CQC_BYTE_40_CQE_BA_S, (dma_handle &gt;&gt; (32 + 3)));</div><div class='del'>-</div><div class='del'>-	if (hr_cq-&gt;db_en)</div><div class='del'>-		roce_set_bit(cq_context-&gt;byte_44_db_record,</div><div class='del'>-			     V2_CQC_BYTE_44_DB_RECORD_EN_S, 1);</div><div class='del'>-</div><div class='del'>-	roce_set_field(cq_context-&gt;byte_44_db_record,</div><div class='del'>-		       V2_CQC_BYTE_44_DB_RECORD_ADDR_M,</div><div class='del'>-		       V2_CQC_BYTE_44_DB_RECORD_ADDR_S,</div><div class='del'>-		       ((u32)hr_cq-&gt;db.dma) &gt;&gt; 1);</div><div class='del'>-	cq_context-&gt;db_record_addr = hr_cq-&gt;db.dma &gt;&gt; 32;</div><div class='del'>-</div><div class='del'>-	roce_set_field(cq_context-&gt;byte_56_cqe_period_maxcnt,</div><div class='del'>-		       V2_CQC_BYTE_56_CQ_MAX_CNT_M,</div><div class='del'>-		       V2_CQC_BYTE_56_CQ_MAX_CNT_S,</div><div class='del'>-		       HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);</div><div class='del'>-	roce_set_field(cq_context-&gt;byte_56_cqe_period_maxcnt,</div><div class='del'>-		       V2_CQC_BYTE_56_CQ_PERIOD_M,</div><div class='del'>-		       V2_CQC_BYTE_56_CQ_PERIOD_S,</div><div class='del'>-		       HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQ_ST, V2_CQ_STATE_VALID);</div><div class='add'>+	hr_reg_write(cq_context, CQC_ARM_ST, NO_ARMED);</div><div class='add'>+	hr_reg_write(cq_context, CQC_SHIFT, ilog2(hr_cq-&gt;cq_depth));</div><div class='add'>+	hr_reg_write(cq_context, CQC_CEQN, hr_cq-&gt;vector);</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQN, hr_cq-&gt;cqn);</div><div class='add'>+</div><div class='add'>+	if (hr_cq-&gt;cqe_size == HNS_ROCE_V3_CQE_SIZE)</div><div class='add'>+		hr_reg_write(cq_context, CQC_CQE_SIZE, CQE_SIZE_64B);</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_STASH)</div><div class='add'>+		hr_reg_enable(cq_context, CQC_STASH);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQE_CUR_BLK_ADDR_L,</div><div class='add'>+		     to_hr_hw_page_addr(mtts[0]));</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQE_CUR_BLK_ADDR_H,</div><div class='add'>+		     upper_32_bits(to_hr_hw_page_addr(mtts[0])));</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQE_HOP_NUM, hr_dev-&gt;caps.cqe_hop_num ==</div><div class='add'>+		     HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev-&gt;caps.cqe_hop_num);</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQE_NEX_BLK_ADDR_L,</div><div class='add'>+		     to_hr_hw_page_addr(mtts[1]));</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQE_NEX_BLK_ADDR_H,</div><div class='add'>+		     upper_32_bits(to_hr_hw_page_addr(mtts[1])));</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQE_BAR_PG_SZ,</div><div class='add'>+		     to_hr_hw_page_shift(hr_cq-&gt;mtr.hem_cfg.ba_pg_shift));</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQE_BUF_PG_SZ,</div><div class='add'>+		     to_hr_hw_page_shift(hr_cq-&gt;mtr.hem_cfg.buf_pg_shift));</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQE_BA_L, dma_handle &gt;&gt; CQC_CQE_BA_L_S);</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQE_BA_H, dma_handle &gt;&gt; CQC_CQE_BA_H_S);</div><div class='add'>+	hr_reg_write_bool(cq_context, CQC_DB_RECORD_EN,</div><div class='add'>+			  hr_cq-&gt;flags &amp; HNS_ROCE_CQ_FLAG_RECORD_DB);</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_L,</div><div class='add'>+		     ((u32)hr_cq-&gt;db.dma) &gt;&gt; 1);</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_H,</div><div class='add'>+		     hr_cq-&gt;db.dma &gt;&gt; CQC_CQE_DB_RECORD_ADDR_H_S);</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQ_MAX_CNT,</div><div class='add'>+		     HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQ_PERIOD,</div><div class='add'>+		     HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,</div><div class='hunk'>@@ -2501,348 +3744,354 @@ static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ibcq-&gt;device);</div><div class='ctx'> 	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);</div><div class='del'>-	u32 notification_flag;</div><div class='del'>-	u32 doorbell[2];</div><div class='add'>+	struct hns_roce_v2_db cq_db = {};</div><div class='add'>+	u32 notify_flag;</div><div class='ctx'> </div><div class='del'>-	doorbell[0] = 0;</div><div class='del'>-	doorbell[1] = 0;</div><div class='del'>-</div><div class='del'>-	notification_flag = (flags &amp; IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?</div><div class='del'>-			     V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL;</div><div class='ctx'> 	/*</div><div class='del'>-	 * flags = 0; Notification Flag = 1, next</div><div class='del'>-	 * flags = 1; Notification Flag = 0, solocited</div><div class='add'>+	 * flags = 0, then notify_flag : next</div><div class='add'>+	 * flags = 1, then notify flag : solocited</div><div class='ctx'> 	 */</div><div class='del'>-	roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_TAG_M, V2_DB_BYTE_4_TAG_S,</div><div class='del'>-		       hr_cq-&gt;cqn);</div><div class='del'>-	roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S,</div><div class='del'>-		       HNS_ROCE_V2_CQ_DB_NTR);</div><div class='del'>-	roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M,</div><div class='del'>-		       V2_CQ_DB_PARAMETER_CONS_IDX_S,</div><div class='del'>-		       hr_cq-&gt;cons_index &amp; ((hr_cq-&gt;cq_depth &lt;&lt; 1) - 1));</div><div class='del'>-	roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M,</div><div class='del'>-		       V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq-&gt;arm_sn &amp; 0x3);</div><div class='del'>-	roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S,</div><div class='del'>-		     notification_flag);</div><div class='del'>-</div><div class='del'>-	hns_roce_write64(hr_dev, doorbell, hr_cq-&gt;cq_db_l);</div><div class='add'>+	notify_flag = (flags &amp; IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?</div><div class='add'>+		      V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL;</div><div class='add'>+</div><div class='add'>+	hr_reg_write(&amp;cq_db, DB_TAG, hr_cq-&gt;cqn);</div><div class='add'>+	hr_reg_write(&amp;cq_db, DB_CMD, HNS_ROCE_V2_CQ_DB_NOTIFY);</div><div class='add'>+	hr_reg_write(&amp;cq_db, DB_CQ_CI, hr_cq-&gt;cons_index);</div><div class='add'>+	hr_reg_write(&amp;cq_db, DB_CQ_CMD_SN, hr_cq-&gt;arm_sn);</div><div class='add'>+	hr_reg_write(&amp;cq_db, DB_CQ_NOTIFY, notify_flag);</div><div class='add'>+</div><div class='add'>+	hns_roce_write64(hr_dev, (__le32 *)&amp;cq_db, hr_cq-&gt;db_reg);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,</div><div class='del'>-						    struct hns_roce_qp **cur_qp,</div><div class='del'>-						    struct ib_wc *wc)</div><div class='add'>+static int sw_comp(struct hns_roce_qp *hr_qp, struct hns_roce_wq *wq,</div><div class='add'>+		   int num_entries, struct ib_wc *wc)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_rinl_sge *sge_list;</div><div class='del'>-	u32 wr_num, wr_cnt, sge_num;</div><div class='del'>-	u32 sge_cnt, data_len, size;</div><div class='del'>-	void *wqe_buf;</div><div class='add'>+	unsigned int left;</div><div class='add'>+	int npolled = 0;</div><div class='ctx'> </div><div class='del'>-	wr_num = roce_get_field(cqe-&gt;byte_4, V2_CQE_BYTE_4_WQE_INDX_M,</div><div class='del'>-				V2_CQE_BYTE_4_WQE_INDX_S) &amp; 0xffff;</div><div class='del'>-	wr_cnt = wr_num &amp; ((*cur_qp)-&gt;rq.wqe_cnt - 1);</div><div class='add'>+	left = wq-&gt;head - wq-&gt;tail;</div><div class='add'>+	if (left == 0)</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-	sge_list = (*cur_qp)-&gt;rq_inl_buf.wqe_list[wr_cnt].sg_list;</div><div class='del'>-	sge_num = (*cur_qp)-&gt;rq_inl_buf.wqe_list[wr_cnt].sge_cnt;</div><div class='del'>-	wqe_buf = get_recv_wqe(*cur_qp, wr_cnt);</div><div class='del'>-	data_len = wc-&gt;byte_len;</div><div class='add'>+	left = min_t(unsigned int, (unsigned int)num_entries, left);</div><div class='add'>+	while (npolled &lt; left) {</div><div class='add'>+		wc-&gt;wr_id = wq-&gt;wrid[wq-&gt;tail &amp; (wq-&gt;wqe_cnt - 1)];</div><div class='add'>+		wc-&gt;status = IB_WC_WR_FLUSH_ERR;</div><div class='add'>+		wc-&gt;vendor_err = 0;</div><div class='add'>+		wc-&gt;qp = &amp;hr_qp-&gt;ibqp;</div><div class='add'>+</div><div class='add'>+		wq-&gt;tail++;</div><div class='add'>+		wc++;</div><div class='add'>+		npolled++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return npolled;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	for (sge_cnt = 0; (sge_cnt &lt; sge_num) &amp;&amp; (data_len); sge_cnt++) {</div><div class='del'>-		size = min(sge_list[sge_cnt].len, data_len);</div><div class='del'>-		memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size);</div><div class='add'>+static int hns_roce_v2_sw_poll_cq(struct hns_roce_cq *hr_cq, int num_entries,</div><div class='add'>+				  struct ib_wc *wc)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_qp *hr_qp;</div><div class='add'>+	int npolled = 0;</div><div class='ctx'> </div><div class='del'>-		data_len -= size;</div><div class='del'>-		wqe_buf += size;</div><div class='add'>+	list_for_each_entry(hr_qp, &amp;hr_cq-&gt;sq_list, sq_node) {</div><div class='add'>+		npolled += sw_comp(hr_qp, &amp;hr_qp-&gt;sq,</div><div class='add'>+				   num_entries - npolled, wc + npolled);</div><div class='add'>+		if (npolled &gt;= num_entries)</div><div class='add'>+			goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (data_len) {</div><div class='del'>-		wc-&gt;status = IB_WC_LOC_LEN_ERR;</div><div class='del'>-		return -EAGAIN;</div><div class='add'>+	list_for_each_entry(hr_qp, &amp;hr_cq-&gt;rq_list, rq_node) {</div><div class='add'>+		npolled += sw_comp(hr_qp, &amp;hr_qp-&gt;rq,</div><div class='add'>+				   num_entries - npolled, wc + npolled);</div><div class='add'>+		if (npolled &gt;= num_entries)</div><div class='add'>+			goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+out:</div><div class='add'>+	return npolled;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,</div><div class='del'>-				struct hns_roce_qp **cur_qp, struct ib_wc *wc)</div><div class='add'>+static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp,</div><div class='add'>+			   struct hns_roce_cq *cq, struct hns_roce_v2_cqe *cqe,</div><div class='add'>+			   struct ib_wc *wc)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_srq *srq = NULL;</div><div class='del'>-	struct hns_roce_dev *hr_dev;</div><div class='del'>-	struct hns_roce_v2_cqe *cqe;</div><div class='del'>-	struct hns_roce_qp *hr_qp;</div><div class='del'>-	struct hns_roce_wq *wq;</div><div class='del'>-	struct ib_qp_attr attr;</div><div class='del'>-	int attr_mask;</div><div class='del'>-	int is_send;</div><div class='del'>-	u16 wqe_ctr;</div><div class='del'>-	u32 opcode;</div><div class='del'>-	u32 status;</div><div class='del'>-	int qpn;</div><div class='del'>-	int ret;</div><div class='add'>+	static const struct {</div><div class='add'>+		u32 cqe_status;</div><div class='add'>+		enum ib_wc_status wc_status;</div><div class='add'>+	} map[] = {</div><div class='add'>+		{ HNS_ROCE_CQE_V2_SUCCESS, IB_WC_SUCCESS },</div><div class='add'>+		{ HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR, IB_WC_LOC_LEN_ERR },</div><div class='add'>+		{ HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR, IB_WC_LOC_QP_OP_ERR },</div><div class='add'>+		{ HNS_ROCE_CQE_V2_LOCAL_PROT_ERR, IB_WC_LOC_PROT_ERR },</div><div class='add'>+		{ HNS_ROCE_CQE_V2_WR_FLUSH_ERR, IB_WC_WR_FLUSH_ERR },</div><div class='add'>+		{ HNS_ROCE_CQE_V2_MW_BIND_ERR, IB_WC_MW_BIND_ERR },</div><div class='add'>+		{ HNS_ROCE_CQE_V2_BAD_RESP_ERR, IB_WC_BAD_RESP_ERR },</div><div class='add'>+		{ HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR, IB_WC_LOC_ACCESS_ERR },</div><div class='add'>+		{ HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR, IB_WC_REM_INV_REQ_ERR },</div><div class='add'>+		{ HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR, IB_WC_REM_ACCESS_ERR },</div><div class='add'>+		{ HNS_ROCE_CQE_V2_REMOTE_OP_ERR, IB_WC_REM_OP_ERR },</div><div class='add'>+		{ HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR,</div><div class='add'>+		  IB_WC_RETRY_EXC_ERR },</div><div class='add'>+		{ HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR },</div><div class='add'>+		{ HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR, IB_WC_REM_ABORT_ERR },</div><div class='add'>+		{ HNS_ROCE_CQE_V2_GENERAL_ERR, IB_WC_GENERAL_ERR}</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	u32 cqe_status = hr_reg_read(cqe, CQE_STATUS);</div><div class='add'>+	int i;</div><div class='ctx'> </div><div class='del'>-	/* Find cqe according to consumer index */</div><div class='del'>-	cqe = next_cqe_sw_v2(hr_cq);</div><div class='del'>-	if (!cqe)</div><div class='del'>-		return -EAGAIN;</div><div class='add'>+	wc-&gt;status = IB_WC_GENERAL_ERR;</div><div class='add'>+	for (i = 0; i &lt; ARRAY_SIZE(map); i++)</div><div class='add'>+		if (cqe_status == map[i].cqe_status) {</div><div class='add'>+			wc-&gt;status = map[i].wc_status;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='del'>-	++hr_cq-&gt;cons_index;</div><div class='del'>-	/* Memory barrier */</div><div class='del'>-	rmb();</div><div class='add'>+	if (likely(wc-&gt;status == IB_WC_SUCCESS ||</div><div class='add'>+		   wc-&gt;status == IB_WC_WR_FLUSH_ERR))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	ibdev_err_ratelimited(&amp;hr_dev-&gt;ib_dev, "error cqe status 0x%x:\n",</div><div class='add'>+			      cqe_status);</div><div class='add'>+	print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_NONE, 16, 4, cqe,</div><div class='add'>+		       cq-&gt;cqe_size, false);</div><div class='add'>+	wc-&gt;vendor_err = hr_reg_read(cqe, CQE_SUB_STATUS);</div><div class='ctx'> </div><div class='del'>-	/* 0-&gt;SQ, 1-&gt;RQ */</div><div class='del'>-	is_send = !roce_get_bit(cqe-&gt;byte_4, V2_CQE_BYTE_4_S_R_S);</div><div class='add'>+	/*</div><div class='add'>+	 * For hns ROCEE, GENERAL_ERR is an error type that is not defined in</div><div class='add'>+	 * the standard protocol, the driver must ignore it and needn't to set</div><div class='add'>+	 * the QP to an error state.</div><div class='add'>+	 */</div><div class='add'>+	if (cqe_status == HNS_ROCE_CQE_V2_GENERAL_ERR)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	flush_cqe(hr_dev, qp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_cur_qp(struct hns_roce_cq *hr_cq, struct hns_roce_v2_cqe *cqe,</div><div class='add'>+		      struct hns_roce_qp **cur_qp)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq-&gt;ib_cq.device);</div><div class='add'>+	struct hns_roce_qp *hr_qp = *cur_qp;</div><div class='add'>+	u32 qpn;</div><div class='ctx'> </div><div class='del'>-	qpn = roce_get_field(cqe-&gt;byte_16, V2_CQE_BYTE_16_LCL_QPN_M,</div><div class='del'>-				V2_CQE_BYTE_16_LCL_QPN_S);</div><div class='add'>+	qpn = hr_reg_read(cqe, CQE_LCL_QPN);</div><div class='ctx'> </div><div class='del'>-	if (!*cur_qp || (qpn &amp; HNS_ROCE_V2_CQE_QPN_MASK) != (*cur_qp)-&gt;qpn) {</div><div class='del'>-		hr_dev = to_hr_dev(hr_cq-&gt;ib_cq.device);</div><div class='add'>+	if (!hr_qp || qpn != hr_qp-&gt;qpn) {</div><div class='ctx'> 		hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);</div><div class='ctx'> 		if (unlikely(!hr_qp)) {</div><div class='del'>-			dev_err(hr_dev-&gt;dev, "CQ %06lx with entry for unknown QPN %06x\n",</div><div class='del'>-				hr_cq-&gt;cqn, (qpn &amp; HNS_ROCE_V2_CQE_QPN_MASK));</div><div class='add'>+			ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+				  "CQ %06lx with entry for unknown QPN %06x\n",</div><div class='add'>+				  hr_cq-&gt;cqn, qpn);</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 		}</div><div class='ctx'> 		*cur_qp = hr_qp;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	wc-&gt;qp = &amp;(*cur_qp)-&gt;ibqp;</div><div class='del'>-	wc-&gt;vendor_err = 0;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (is_send) {</div><div class='del'>-		wq = &amp;(*cur_qp)-&gt;sq;</div><div class='del'>-		if ((*cur_qp)-&gt;sq_signal_bits) {</div><div class='del'>-			/*</div><div class='del'>-			 * If sg_signal_bit is 1,</div><div class='del'>-			 * firstly tail pointer updated to wqe</div><div class='del'>-			 * which current cqe correspond to</div><div class='del'>-			 */</div><div class='del'>-			wqe_ctr = (u16)roce_get_field(cqe-&gt;byte_4,</div><div class='del'>-						      V2_CQE_BYTE_4_WQE_INDX_M,</div><div class='del'>-						      V2_CQE_BYTE_4_WQE_INDX_S);</div><div class='del'>-			wq-&gt;tail += (wqe_ctr - (u16)wq-&gt;tail) &amp;</div><div class='del'>-				    (wq-&gt;wqe_cnt - 1);</div><div class='del'>-		}</div><div class='add'>+/*</div><div class='add'>+ * mapped-value = 1 + real-value</div><div class='add'>+ * The ib wc opcode's real value is start from 0, In order to distinguish</div><div class='add'>+ * between initialized and uninitialized map values, we plus 1 to the actual</div><div class='add'>+ * value when defining the mapping, so that the validity can be identified by</div><div class='add'>+ * checking whether the mapped value is greater than 0.</div><div class='add'>+ */</div><div class='add'>+#define HR_WC_OP_MAP(hr_key, ib_key) \</div><div class='add'>+		[HNS_ROCE_V2_WQE_OP_ ## hr_key] = 1 + IB_WC_ ## ib_key</div><div class='add'>+</div><div class='add'>+static const u32 wc_send_op_map[] = {</div><div class='add'>+	HR_WC_OP_MAP(SEND,			SEND),</div><div class='add'>+	HR_WC_OP_MAP(SEND_WITH_INV,		SEND),</div><div class='add'>+	HR_WC_OP_MAP(SEND_WITH_IMM,		SEND),</div><div class='add'>+	HR_WC_OP_MAP(RDMA_READ,			RDMA_READ),</div><div class='add'>+	HR_WC_OP_MAP(RDMA_WRITE,		RDMA_WRITE),</div><div class='add'>+	HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM,	RDMA_WRITE),</div><div class='add'>+	HR_WC_OP_MAP(ATOM_CMP_AND_SWAP,		COMP_SWAP),</div><div class='add'>+	HR_WC_OP_MAP(ATOM_FETCH_AND_ADD,	FETCH_ADD),</div><div class='add'>+	HR_WC_OP_MAP(ATOM_MSK_CMP_AND_SWAP,	MASKED_COMP_SWAP),</div><div class='add'>+	HR_WC_OP_MAP(ATOM_MSK_FETCH_AND_ADD,	MASKED_FETCH_ADD),</div><div class='add'>+	HR_WC_OP_MAP(FAST_REG_PMR,		REG_MR),</div><div class='add'>+};</div><div class='ctx'> </div><div class='del'>-		wc-&gt;wr_id = wq-&gt;wrid[wq-&gt;tail &amp; (wq-&gt;wqe_cnt - 1)];</div><div class='del'>-		++wq-&gt;tail;</div><div class='del'>-	} else if ((*cur_qp)-&gt;ibqp.srq) {</div><div class='del'>-		srq = to_hr_srq((*cur_qp)-&gt;ibqp.srq);</div><div class='del'>-		wqe_ctr = le16_to_cpu(roce_get_field(cqe-&gt;byte_4,</div><div class='del'>-						     V2_CQE_BYTE_4_WQE_INDX_M,</div><div class='del'>-						     V2_CQE_BYTE_4_WQE_INDX_S));</div><div class='del'>-		wc-&gt;wr_id = srq-&gt;wrid[wqe_ctr];</div><div class='del'>-		hns_roce_free_srq_wqe(srq, wqe_ctr);</div><div class='del'>-	} else {</div><div class='del'>-		/* Update tail pointer, record wr_id */</div><div class='del'>-		wq = &amp;(*cur_qp)-&gt;rq;</div><div class='del'>-		wc-&gt;wr_id = wq-&gt;wrid[wq-&gt;tail &amp; (wq-&gt;wqe_cnt - 1)];</div><div class='del'>-		++wq-&gt;tail;</div><div class='del'>-	}</div><div class='add'>+static int to_ib_wc_send_op(u32 hr_opcode)</div><div class='add'>+{</div><div class='add'>+	if (hr_opcode &gt;= ARRAY_SIZE(wc_send_op_map))</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	status = roce_get_field(cqe-&gt;byte_4, V2_CQE_BYTE_4_STATUS_M,</div><div class='del'>-				V2_CQE_BYTE_4_STATUS_S);</div><div class='del'>-	switch (status &amp; HNS_ROCE_V2_CQE_STATUS_MASK) {</div><div class='del'>-	case HNS_ROCE_CQE_V2_SUCCESS:</div><div class='del'>-		wc-&gt;status = IB_WC_SUCCESS;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_LOC_LEN_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_LOC_QP_OP_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_V2_LOCAL_PROT_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_LOC_PROT_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_V2_WR_FLUSH_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_WR_FLUSH_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_V2_MW_BIND_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_MW_BIND_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_V2_BAD_RESP_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_BAD_RESP_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_LOC_ACCESS_ERR;</div><div class='del'>-		break;</div><div class='del'>-	case HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_REM_INV_REQ_ERR;</div><div class='add'>+	return wc_send_op_map[hr_opcode] ? wc_send_op_map[hr_opcode] - 1 :</div><div class='add'>+					   -EINVAL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const u32 wc_recv_op_map[] = {</div><div class='add'>+	HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM,		WITH_IMM),</div><div class='add'>+	HR_WC_OP_MAP(SEND,				RECV),</div><div class='add'>+	HR_WC_OP_MAP(SEND_WITH_IMM,			WITH_IMM),</div><div class='add'>+	HR_WC_OP_MAP(SEND_WITH_INV,			RECV),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int to_ib_wc_recv_op(u32 hr_opcode)</div><div class='add'>+{</div><div class='add'>+	if (hr_opcode &gt;= ARRAY_SIZE(wc_recv_op_map))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	return wc_recv_op_map[hr_opcode] ? wc_recv_op_map[hr_opcode] - 1 :</div><div class='add'>+					   -EINVAL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void fill_send_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe)</div><div class='add'>+{</div><div class='add'>+	u32 hr_opcode;</div><div class='add'>+	int ib_opcode;</div><div class='add'>+</div><div class='add'>+	wc-&gt;wc_flags = 0;</div><div class='add'>+</div><div class='add'>+	hr_opcode = hr_reg_read(cqe, CQE_OPCODE);</div><div class='add'>+	switch (hr_opcode) {</div><div class='add'>+	case HNS_ROCE_V2_WQE_OP_RDMA_READ:</div><div class='add'>+		wc-&gt;byte_len = le32_to_cpu(cqe-&gt;byte_cnt);</div><div class='ctx'> 		break;</div><div class='del'>-	case HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_REM_ACCESS_ERR;</div><div class='add'>+	case HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM:</div><div class='add'>+	case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM:</div><div class='add'>+		wc-&gt;wc_flags |= IB_WC_WITH_IMM;</div><div class='ctx'> 		break;</div><div class='del'>-	case HNS_ROCE_CQE_V2_REMOTE_OP_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_REM_OP_ERR;</div><div class='add'>+	case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP:</div><div class='add'>+	case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD:</div><div class='add'>+	case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP:</div><div class='add'>+	case HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD:</div><div class='add'>+		wc-&gt;byte_len  = 8;</div><div class='ctx'> 		break;</div><div class='del'>-	case HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_RETRY_EXC_ERR;</div><div class='add'>+	default:</div><div class='ctx'> 		break;</div><div class='del'>-	case HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_RNR_RETRY_EXC_ERR;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ib_opcode = to_ib_wc_send_op(hr_opcode);</div><div class='add'>+	if (ib_opcode &lt; 0)</div><div class='add'>+		wc-&gt;status = IB_WC_GENERAL_ERR;</div><div class='add'>+	else</div><div class='add'>+		wc-&gt;opcode = ib_opcode;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_recv_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe)</div><div class='add'>+{</div><div class='add'>+	u32 hr_opcode;</div><div class='add'>+	int ib_opcode;</div><div class='add'>+</div><div class='add'>+	wc-&gt;byte_len = le32_to_cpu(cqe-&gt;byte_cnt);</div><div class='add'>+</div><div class='add'>+	hr_opcode = hr_reg_read(cqe, CQE_OPCODE);</div><div class='add'>+	switch (hr_opcode) {</div><div class='add'>+	case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM:</div><div class='add'>+	case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM:</div><div class='add'>+		wc-&gt;wc_flags = IB_WC_WITH_IMM;</div><div class='add'>+		wc-&gt;ex.imm_data = cpu_to_be32(le32_to_cpu(cqe-&gt;immtdata));</div><div class='ctx'> 		break;</div><div class='del'>-	case HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR:</div><div class='del'>-		wc-&gt;status = IB_WC_REM_ABORT_ERR;</div><div class='add'>+	case HNS_ROCE_V2_OPCODE_SEND_WITH_INV:</div><div class='add'>+		wc-&gt;wc_flags = IB_WC_WITH_INVALIDATE;</div><div class='add'>+		wc-&gt;ex.invalidate_rkey = le32_to_cpu(cqe-&gt;rkey);</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='del'>-		wc-&gt;status = IB_WC_GENERAL_ERR;</div><div class='del'>-		break;</div><div class='add'>+		wc-&gt;wc_flags = 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* flush cqe if wc status is error, excluding flush error */</div><div class='del'>-	if ((wc-&gt;status != IB_WC_SUCCESS) &amp;&amp;</div><div class='del'>-	    (wc-&gt;status != IB_WC_WR_FLUSH_ERR)) {</div><div class='del'>-		attr_mask = IB_QP_STATE;</div><div class='del'>-		attr.qp_state = IB_QPS_ERR;</div><div class='del'>-		return hns_roce_v2_modify_qp(&amp;(*cur_qp)-&gt;ibqp,</div><div class='del'>-					     &amp;attr, attr_mask,</div><div class='del'>-					     (*cur_qp)-&gt;state, IB_QPS_ERR);</div><div class='add'>+	ib_opcode = to_ib_wc_recv_op(hr_opcode);</div><div class='add'>+	if (ib_opcode &lt; 0)</div><div class='add'>+		wc-&gt;status = IB_WC_GENERAL_ERR;</div><div class='add'>+	else</div><div class='add'>+		wc-&gt;opcode = ib_opcode;</div><div class='add'>+</div><div class='add'>+	wc-&gt;sl = hr_reg_read(cqe, CQE_SL);</div><div class='add'>+	wc-&gt;src_qp = hr_reg_read(cqe, CQE_RMT_QPN);</div><div class='add'>+	wc-&gt;slid = 0;</div><div class='add'>+	wc-&gt;wc_flags |= hr_reg_read(cqe, CQE_GRH) ? IB_WC_GRH : 0;</div><div class='add'>+	wc-&gt;port_num = hr_reg_read(cqe, CQE_PORTN);</div><div class='add'>+	wc-&gt;pkey_index = 0;</div><div class='add'>+</div><div class='add'>+	if (hr_reg_read(cqe, CQE_VID_VLD)) {</div><div class='add'>+		wc-&gt;vlan_id = hr_reg_read(cqe, CQE_VID);</div><div class='add'>+		wc-&gt;wc_flags |= IB_WC_WITH_VLAN;</div><div class='add'>+	} else {</div><div class='add'>+		wc-&gt;vlan_id = 0xffff;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (wc-&gt;status == IB_WC_WR_FLUSH_ERR)</div><div class='del'>-		return 0;</div><div class='add'>+	wc-&gt;network_hdr_type = hr_reg_read(cqe, CQE_PORT_TYPE);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,</div><div class='add'>+				struct hns_roce_qp **cur_qp, struct ib_wc *wc)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq-&gt;ib_cq.device);</div><div class='add'>+	struct hns_roce_qp *qp = *cur_qp;</div><div class='add'>+	struct hns_roce_srq *srq = NULL;</div><div class='add'>+	struct hns_roce_v2_cqe *cqe;</div><div class='add'>+	struct hns_roce_wq *wq;</div><div class='add'>+	int is_send;</div><div class='add'>+	u16 wqe_idx;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='add'>+	cqe = get_sw_cqe_v2(hr_cq, hr_cq-&gt;cons_index);</div><div class='add'>+	if (!cqe)</div><div class='add'>+		return -EAGAIN;</div><div class='add'>+</div><div class='add'>+	++hr_cq-&gt;cons_index;</div><div class='add'>+	/* Memory barrier */</div><div class='add'>+	rmb();</div><div class='add'>+</div><div class='add'>+	ret = get_cur_qp(hr_cq, cqe, &amp;qp);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	wc-&gt;qp = &amp;qp-&gt;ibqp;</div><div class='add'>+	wc-&gt;vendor_err = 0;</div><div class='add'>+</div><div class='add'>+	wqe_idx = hr_reg_read(cqe, CQE_WQE_IDX);</div><div class='add'>+</div><div class='add'>+	is_send = !hr_reg_read(cqe, CQE_S_R);</div><div class='ctx'> 	if (is_send) {</div><div class='del'>-		wc-&gt;wc_flags = 0;</div><div class='del'>-		/* SQ corresponding to CQE */</div><div class='del'>-		switch (roce_get_field(cqe-&gt;byte_4, V2_CQE_BYTE_4_OPCODE_M,</div><div class='del'>-				       V2_CQE_BYTE_4_OPCODE_S) &amp; 0x1f) {</div><div class='del'>-		case HNS_ROCE_SQ_OPCODE_SEND:</div><div class='del'>-			wc-&gt;opcode = IB_WC_SEND;</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV:</div><div class='del'>-			wc-&gt;opcode = IB_WC_SEND;</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM:</div><div class='del'>-			wc-&gt;opcode = IB_WC_SEND;</div><div class='del'>-			wc-&gt;wc_flags |= IB_WC_WITH_IMM;</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_SQ_OPCODE_RDMA_READ:</div><div class='del'>-			wc-&gt;opcode = IB_WC_RDMA_READ;</div><div class='del'>-			wc-&gt;byte_len = le32_to_cpu(cqe-&gt;byte_cnt);</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_SQ_OPCODE_RDMA_WRITE:</div><div class='del'>-			wc-&gt;opcode = IB_WC_RDMA_WRITE;</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM:</div><div class='del'>-			wc-&gt;opcode = IB_WC_RDMA_WRITE;</div><div class='del'>-			wc-&gt;wc_flags |= IB_WC_WITH_IMM;</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_SQ_OPCODE_LOCAL_INV:</div><div class='del'>-			wc-&gt;opcode = IB_WC_LOCAL_INV;</div><div class='del'>-			wc-&gt;wc_flags |= IB_WC_WITH_INVALIDATE;</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP:</div><div class='del'>-			wc-&gt;opcode = IB_WC_COMP_SWAP;</div><div class='del'>-			wc-&gt;byte_len  = 8;</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD:</div><div class='del'>-			wc-&gt;opcode = IB_WC_FETCH_ADD;</div><div class='del'>-			wc-&gt;byte_len  = 8;</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP:</div><div class='del'>-			wc-&gt;opcode = IB_WC_MASKED_COMP_SWAP;</div><div class='del'>-			wc-&gt;byte_len  = 8;</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD:</div><div class='del'>-			wc-&gt;opcode = IB_WC_MASKED_FETCH_ADD;</div><div class='del'>-			wc-&gt;byte_len  = 8;</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_SQ_OPCODE_FAST_REG_WR:</div><div class='del'>-			wc-&gt;opcode = IB_WC_REG_MR;</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_SQ_OPCODE_BIND_MW:</div><div class='del'>-			wc-&gt;opcode = IB_WC_REG_MR;</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			wc-&gt;status = IB_WC_GENERAL_ERR;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		/* RQ correspond to CQE */</div><div class='del'>-		wc-&gt;byte_len = le32_to_cpu(cqe-&gt;byte_cnt);</div><div class='add'>+		wq = &amp;qp-&gt;sq;</div><div class='ctx'> </div><div class='del'>-		opcode = roce_get_field(cqe-&gt;byte_4, V2_CQE_BYTE_4_OPCODE_M,</div><div class='del'>-					V2_CQE_BYTE_4_OPCODE_S);</div><div class='del'>-		switch (opcode &amp; 0x1f) {</div><div class='del'>-		case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM:</div><div class='del'>-			wc-&gt;opcode = IB_WC_RECV_RDMA_WITH_IMM;</div><div class='del'>-			wc-&gt;wc_flags = IB_WC_WITH_IMM;</div><div class='del'>-			wc-&gt;ex.imm_data =</div><div class='del'>-				cpu_to_be32(le32_to_cpu(cqe-&gt;immtdata));</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_V2_OPCODE_SEND:</div><div class='del'>-			wc-&gt;opcode = IB_WC_RECV;</div><div class='del'>-			wc-&gt;wc_flags = 0;</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM:</div><div class='del'>-			wc-&gt;opcode = IB_WC_RECV;</div><div class='del'>-			wc-&gt;wc_flags = IB_WC_WITH_IMM;</div><div class='del'>-			wc-&gt;ex.imm_data =</div><div class='del'>-				cpu_to_be32(le32_to_cpu(cqe-&gt;immtdata));</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_V2_OPCODE_SEND_WITH_INV:</div><div class='del'>-			wc-&gt;opcode = IB_WC_RECV;</div><div class='del'>-			wc-&gt;wc_flags = IB_WC_WITH_INVALIDATE;</div><div class='del'>-			wc-&gt;ex.invalidate_rkey = le32_to_cpu(cqe-&gt;rkey);</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			wc-&gt;status = IB_WC_GENERAL_ERR;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='add'>+		/* If sg_signal_bit is set, tail pointer will be updated to</div><div class='add'>+		 * the WQE corresponding to the current CQE.</div><div class='add'>+		 */</div><div class='add'>+		if (qp-&gt;sq_signal_bits)</div><div class='add'>+			wq-&gt;tail += (wqe_idx - (u16)wq-&gt;tail) &amp;</div><div class='add'>+				    (wq-&gt;wqe_cnt - 1);</div><div class='ctx'> </div><div class='del'>-		if ((wc-&gt;qp-&gt;qp_type == IB_QPT_RC ||</div><div class='del'>-		     wc-&gt;qp-&gt;qp_type == IB_QPT_UC) &amp;&amp;</div><div class='del'>-		    (opcode == HNS_ROCE_V2_OPCODE_SEND ||</div><div class='del'>-		    opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM ||</div><div class='del'>-		    opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &amp;&amp;</div><div class='del'>-		    (roce_get_bit(cqe-&gt;byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) {</div><div class='del'>-			ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc);</div><div class='del'>-			if (ret)</div><div class='del'>-				return -EAGAIN;</div><div class='del'>-		}</div><div class='add'>+		wc-&gt;wr_id = wq-&gt;wrid[wq-&gt;tail &amp; (wq-&gt;wqe_cnt - 1)];</div><div class='add'>+		++wq-&gt;tail;</div><div class='ctx'> </div><div class='del'>-		wc-&gt;sl = (u8)roce_get_field(cqe-&gt;byte_32, V2_CQE_BYTE_32_SL_M,</div><div class='del'>-					    V2_CQE_BYTE_32_SL_S);</div><div class='del'>-		wc-&gt;src_qp = (u8)roce_get_field(cqe-&gt;byte_32,</div><div class='del'>-						V2_CQE_BYTE_32_RMT_QPN_M,</div><div class='del'>-						V2_CQE_BYTE_32_RMT_QPN_S);</div><div class='del'>-		wc-&gt;slid = 0;</div><div class='del'>-		wc-&gt;wc_flags |= (roce_get_bit(cqe-&gt;byte_32,</div><div class='del'>-					      V2_CQE_BYTE_32_GRH_S) ?</div><div class='del'>-					      IB_WC_GRH : 0);</div><div class='del'>-		wc-&gt;port_num = roce_get_field(cqe-&gt;byte_32,</div><div class='del'>-				V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S);</div><div class='del'>-		wc-&gt;pkey_index = 0;</div><div class='del'>-		memcpy(wc-&gt;smac, cqe-&gt;smac, 4);</div><div class='del'>-		wc-&gt;smac[4] = roce_get_field(cqe-&gt;byte_28,</div><div class='del'>-					     V2_CQE_BYTE_28_SMAC_4_M,</div><div class='del'>-					     V2_CQE_BYTE_28_SMAC_4_S);</div><div class='del'>-		wc-&gt;smac[5] = roce_get_field(cqe-&gt;byte_28,</div><div class='del'>-					     V2_CQE_BYTE_28_SMAC_5_M,</div><div class='del'>-					     V2_CQE_BYTE_28_SMAC_5_S);</div><div class='del'>-		if (roce_get_bit(cqe-&gt;byte_28, V2_CQE_BYTE_28_VID_VLD_S)) {</div><div class='del'>-			wc-&gt;vlan_id = (u16)roce_get_field(cqe-&gt;byte_28,</div><div class='del'>-							  V2_CQE_BYTE_28_VID_M,</div><div class='del'>-							  V2_CQE_BYTE_28_VID_S);</div><div class='add'>+		fill_send_wc(wc, cqe);</div><div class='add'>+	} else {</div><div class='add'>+		if (qp-&gt;ibqp.srq) {</div><div class='add'>+			srq = to_hr_srq(qp-&gt;ibqp.srq);</div><div class='add'>+			wc-&gt;wr_id = srq-&gt;wrid[wqe_idx];</div><div class='add'>+			hns_roce_free_srq_wqe(srq, wqe_idx);</div><div class='ctx'> 		} else {</div><div class='del'>-			wc-&gt;vlan_id = 0xffff;</div><div class='add'>+			wq = &amp;qp-&gt;rq;</div><div class='add'>+			wc-&gt;wr_id = wq-&gt;wrid[wq-&gt;tail &amp; (wq-&gt;wqe_cnt - 1)];</div><div class='add'>+			++wq-&gt;tail;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		wc-&gt;wc_flags |= (IB_WC_WITH_VLAN | IB_WC_WITH_SMAC);</div><div class='del'>-		wc-&gt;network_hdr_type = roce_get_field(cqe-&gt;byte_28,</div><div class='del'>-						    V2_CQE_BYTE_28_PORT_TYPE_M,</div><div class='del'>-						    V2_CQE_BYTE_28_PORT_TYPE_S);</div><div class='add'>+		ret = fill_recv_wc(wc, cqe);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	get_cqe_status(hr_dev, qp, hr_cq, cqe, wc);</div><div class='add'>+	if (unlikely(wc-&gt;status != IB_WC_SUCCESS))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,</div><div class='ctx'> 			       struct ib_wc *wc)</div><div class='ctx'> {</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ibcq-&gt;device);</div><div class='ctx'> 	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);</div><div class='ctx'> 	struct hns_roce_qp *cur_qp = NULL;</div><div class='ctx'> 	unsigned long flags;</div><div class='hunk'>@@ -2850,29 +4099,109 @@ static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;hr_cq-&gt;lock, flags);</div><div class='ctx'> </div><div class='add'>+	/*</div><div class='add'>+	 * When the device starts to reset, the state is RST_DOWN. At this time,</div><div class='add'>+	 * there may still be some valid CQEs in the hardware that are not</div><div class='add'>+	 * polled. Therefore, it is not allowed to switch to the software mode</div><div class='add'>+	 * immediately. When the state changes to UNINIT, CQE no longer exists</div><div class='add'>+	 * in the hardware, and then switch to software mode.</div><div class='add'>+	 */</div><div class='add'>+	if (hr_dev-&gt;state == HNS_ROCE_DEVICE_STATE_UNINIT) {</div><div class='add'>+		npolled = hns_roce_v2_sw_poll_cq(hr_cq, num_entries, wc);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	for (npolled = 0; npolled &lt; num_entries; ++npolled) {</div><div class='ctx'> 		if (hns_roce_v2_poll_one(hr_cq, &amp;cur_qp, wc + npolled))</div><div class='ctx'> 			break;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (npolled) {</div><div class='del'>-		/* Memory barrier */</div><div class='del'>-		wmb();</div><div class='del'>-		hns_roce_v2_cq_set_ci(hr_cq, hr_cq-&gt;cons_index);</div><div class='del'>-	}</div><div class='add'>+	if (npolled)</div><div class='add'>+		update_cq_db(hr_dev, hr_cq);</div><div class='ctx'> </div><div class='add'>+out:</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;hr_cq-&gt;lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	return npolled;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type,</div><div class='add'>+			      u32 step_idx, u8 *mbox_cmd)</div><div class='add'>+{</div><div class='add'>+	u8 cmd;</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case HEM_TYPE_QPC:</div><div class='add'>+		cmd = HNS_ROCE_CMD_WRITE_QPC_BT0;</div><div class='add'>+		break;</div><div class='add'>+	case HEM_TYPE_MTPT:</div><div class='add'>+		cmd = HNS_ROCE_CMD_WRITE_MPT_BT0;</div><div class='add'>+		break;</div><div class='add'>+	case HEM_TYPE_CQC:</div><div class='add'>+		cmd = HNS_ROCE_CMD_WRITE_CQC_BT0;</div><div class='add'>+		break;</div><div class='add'>+	case HEM_TYPE_SRQC:</div><div class='add'>+		cmd = HNS_ROCE_CMD_WRITE_SRQC_BT0;</div><div class='add'>+		break;</div><div class='add'>+	case HEM_TYPE_SCCC:</div><div class='add'>+		cmd = HNS_ROCE_CMD_WRITE_SCCC_BT0;</div><div class='add'>+		break;</div><div class='add'>+	case HEM_TYPE_QPC_TIMER:</div><div class='add'>+		cmd = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0;</div><div class='add'>+		break;</div><div class='add'>+	case HEM_TYPE_CQC_TIMER:</div><div class='add'>+		cmd = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		dev_warn(hr_dev-&gt;dev, "failed to check hem type %u.\n", type);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*mbox_cmd = cmd + step_idx;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int config_gmv_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj,</div><div class='add'>+			       dma_addr_t base_addr)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_cmq_desc desc;</div><div class='add'>+	struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;</div><div class='add'>+	u32 idx = obj / (HNS_HW_PAGE_SIZE / hr_dev-&gt;caps.gmv_entry_sz);</div><div class='add'>+	u64 addr = to_hr_hw_page_addr(base_addr);</div><div class='add'>+</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_OPC_CFG_GMV_BT, false);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr));</div><div class='add'>+	hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr));</div><div class='add'>+	hr_reg_write(req, CFG_GMV_BT_IDX, idx);</div><div class='add'>+</div><div class='add'>+	return hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj,</div><div class='add'>+			 dma_addr_t base_addr, u32 hem_type, u32 step_idx)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+	u8 cmd;</div><div class='add'>+</div><div class='add'>+	if (unlikely(hem_type == HEM_TYPE_GMV))</div><div class='add'>+		return config_gmv_ba_to_hw(hr_dev, obj, base_addr);</div><div class='add'>+</div><div class='add'>+	if (unlikely(hem_type == HEM_TYPE_SCCC &amp;&amp; step_idx))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	ret = get_op_for_set_hem(hr_dev, hem_type, step_idx, &amp;cmd);</div><div class='add'>+	if (ret &lt; 0)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	return config_hem_ba_to_hw(hr_dev, base_addr, cmd, obj);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 			       struct hns_roce_hem_table *table, int obj,</div><div class='del'>-			       int step_idx)</div><div class='add'>+			       u32 step_idx)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	struct hns_roce_cmd_mailbox *mailbox;</div><div class='del'>-	struct hns_roce_hem_iter iter;</div><div class='ctx'> 	struct hns_roce_hem_mhop mhop;</div><div class='ctx'> 	struct hns_roce_hem *hem;</div><div class='ctx'> 	unsigned long mhop_obj = obj;</div><div class='hunk'>@@ -2883,7 +4212,6 @@ static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 	u64 bt_ba = 0;</div><div class='ctx'> 	u32 chunk_ba_num;</div><div class='ctx'> 	u32 hop_num;</div><div class='del'>-	u16 op = 0xff;</div><div class='ctx'> </div><div class='ctx'> 	if (!hns_roce_check_whether_mhop(hr_dev, table-&gt;type))</div><div class='ctx'> 		return 0;</div><div class='hunk'>@@ -2905,145 +4233,93 @@ static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 		hem_idx = i;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	switch (table-&gt;type) {</div><div class='del'>-	case HEM_TYPE_QPC:</div><div class='del'>-		op = HNS_ROCE_CMD_WRITE_QPC_BT0;</div><div class='del'>-		break;</div><div class='del'>-	case HEM_TYPE_MTPT:</div><div class='del'>-		op = HNS_ROCE_CMD_WRITE_MPT_BT0;</div><div class='del'>-		break;</div><div class='del'>-	case HEM_TYPE_CQC:</div><div class='del'>-		op = HNS_ROCE_CMD_WRITE_CQC_BT0;</div><div class='del'>-		break;</div><div class='del'>-	case HEM_TYPE_SRQC:</div><div class='del'>-		op = HNS_ROCE_CMD_WRITE_SRQC_BT0;</div><div class='del'>-		break;</div><div class='del'>-	case HEM_TYPE_SCCC:</div><div class='del'>-		op = HNS_ROCE_CMD_WRITE_SCCC_BT0;</div><div class='del'>-		break;</div><div class='del'>-	case HEM_TYPE_QPC_TIMER:</div><div class='del'>-		op = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0;</div><div class='del'>-		break;</div><div class='del'>-	case HEM_TYPE_CQC_TIMER:</div><div class='del'>-		op = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		dev_warn(dev, "Table %d not to be written by mailbox!\n",</div><div class='del'>-			 table-&gt;type);</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (table-&gt;type == HEM_TYPE_SCCC &amp;&amp; step_idx)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	op += step_idx;</div><div class='del'>-</div><div class='del'>-	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='del'>-	if (IS_ERR(mailbox))</div><div class='del'>-		return PTR_ERR(mailbox);</div><div class='del'>-</div><div class='ctx'> 	if (table-&gt;type == HEM_TYPE_SCCC)</div><div class='ctx'> 		obj = mhop.l0_idx;</div><div class='ctx'> </div><div class='ctx'> 	if (check_whether_last_step(hop_num, step_idx)) {</div><div class='ctx'> 		hem = table-&gt;hem[hem_idx];</div><div class='del'>-		for (hns_roce_hem_first(hem, &amp;iter);</div><div class='del'>-		     !hns_roce_hem_last(&amp;iter); hns_roce_hem_next(&amp;iter)) {</div><div class='del'>-			bt_ba = hns_roce_hem_addr(&amp;iter);</div><div class='del'>-</div><div class='del'>-			/* configure the ba, tag, and op */</div><div class='del'>-			ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox-&gt;dma,</div><div class='del'>-						obj, 0, op,</div><div class='del'>-						HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='del'>-		}</div><div class='add'>+</div><div class='add'>+		ret = set_hem_to_hw(hr_dev, obj, hem-&gt;dma, table-&gt;type, step_idx);</div><div class='ctx'> 	} else {</div><div class='ctx'> 		if (step_idx == 0)</div><div class='ctx'> 			bt_ba = table-&gt;bt_l0_dma_addr[i];</div><div class='ctx'> 		else if (step_idx == 1 &amp;&amp; hop_num == 2)</div><div class='ctx'> 			bt_ba = table-&gt;bt_l1_dma_addr[l1_idx];</div><div class='ctx'> </div><div class='del'>-		/* configure the ba, tag, and op */</div><div class='del'>-		ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox-&gt;dma, obj,</div><div class='del'>-					0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='add'>+		ret = set_hem_to_hw(hr_dev, obj, bt_ba, table-&gt;type, step_idx);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,</div><div class='del'>-				 struct hns_roce_hem_table *table, int obj,</div><div class='del'>-				 int step_idx)</div><div class='add'>+				 struct hns_roce_hem_table *table,</div><div class='add'>+				 int tag, u32 step_idx)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='ctx'> 	struct hns_roce_cmd_mailbox *mailbox;</div><div class='del'>-	int ret = 0;</div><div class='del'>-	u16 op = 0xff;</div><div class='add'>+	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	u8 cmd = 0xff;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (!hns_roce_check_whether_mhop(hr_dev, table-&gt;type))</div><div class='ctx'> 		return 0;</div><div class='ctx'> </div><div class='ctx'> 	switch (table-&gt;type) {</div><div class='ctx'> 	case HEM_TYPE_QPC:</div><div class='del'>-		op = HNS_ROCE_CMD_DESTROY_QPC_BT0;</div><div class='add'>+		cmd = HNS_ROCE_CMD_DESTROY_QPC_BT0;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case HEM_TYPE_MTPT:</div><div class='del'>-		op = HNS_ROCE_CMD_DESTROY_MPT_BT0;</div><div class='add'>+		cmd = HNS_ROCE_CMD_DESTROY_MPT_BT0;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case HEM_TYPE_CQC:</div><div class='del'>-		op = HNS_ROCE_CMD_DESTROY_CQC_BT0;</div><div class='add'>+		cmd = HNS_ROCE_CMD_DESTROY_CQC_BT0;</div><div class='add'>+		break;</div><div class='add'>+	case HEM_TYPE_SRQC:</div><div class='add'>+		cmd = HNS_ROCE_CMD_DESTROY_SRQC_BT0;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case HEM_TYPE_SCCC:</div><div class='ctx'> 	case HEM_TYPE_QPC_TIMER:</div><div class='ctx'> 	case HEM_TYPE_CQC_TIMER:</div><div class='del'>-		break;</div><div class='del'>-	case HEM_TYPE_SRQC:</div><div class='del'>-		op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;</div><div class='del'>-		break;</div><div class='add'>+	case HEM_TYPE_GMV:</div><div class='add'>+		return 0;</div><div class='ctx'> 	default:</div><div class='del'>-		dev_warn(dev, "Table %d not to be destroyed by mailbox!\n",</div><div class='add'>+		dev_warn(dev, "table %u not to be destroyed by mailbox!\n",</div><div class='ctx'> 			 table-&gt;type);</div><div class='ctx'> 		return 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (table-&gt;type == HEM_TYPE_SCCC ||</div><div class='del'>-	    table-&gt;type == HEM_TYPE_QPC_TIMER ||</div><div class='del'>-	    table-&gt;type == HEM_TYPE_CQC_TIMER)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	op += step_idx;</div><div class='add'>+	cmd += step_idx;</div><div class='ctx'> </div><div class='ctx'> 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='ctx'> 	if (IS_ERR(mailbox))</div><div class='ctx'> 		return PTR_ERR(mailbox);</div><div class='ctx'> </div><div class='del'>-	/* configure the tag and op */</div><div class='del'>-	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox-&gt;dma, obj, 0, op,</div><div class='del'>-				HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='add'>+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox-&gt;dma, cmd, tag);</div><div class='ctx'> </div><div class='ctx'> 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,</div><div class='del'>-				 struct hns_roce_mtt *mtt,</div><div class='del'>-				 enum ib_qp_state cur_state,</div><div class='del'>-				 enum ib_qp_state new_state,</div><div class='ctx'> 				 struct hns_roce_v2_qp_context *context,</div><div class='add'>+				 struct hns_roce_v2_qp_context *qpc_mask,</div><div class='ctx'> 				 struct hns_roce_qp *hr_qp)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_cmd_mailbox *mailbox;</div><div class='add'>+	int qpc_size;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='ctx'> 	if (IS_ERR(mailbox))</div><div class='ctx'> 		return PTR_ERR(mailbox);</div><div class='ctx'> </div><div class='del'>-	memcpy(mailbox-&gt;buf, context, sizeof(*context) * 2);</div><div class='add'>+	/* The qpc size of HIP08 is only 256B, which is half of HIP09 */</div><div class='add'>+	qpc_size = hr_dev-&gt;caps.qpc_sz;</div><div class='add'>+	memcpy(mailbox-&gt;buf, context, qpc_size);</div><div class='add'>+	memcpy(mailbox-&gt;buf + qpc_size, qpc_mask, qpc_size);</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_cmd_mbox(hr_dev, mailbox-&gt;dma, 0, hr_qp-&gt;qpn, 0,</div><div class='del'>-				HNS_ROCE_CMD_MODIFY_QPC,</div><div class='del'>-				HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='add'>+	ret = hns_roce_cmd_mbox(hr_dev, mailbox-&gt;dma, 0,</div><div class='add'>+				HNS_ROCE_CMD_MODIFY_QPC, hr_qp-&gt;qpn);</div><div class='ctx'> </div><div class='ctx'> 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='ctx'> </div><div class='hunk'>@@ -3067,24 +4343,46 @@ static void set_access_flags(struct hns_roce_qp *hr_qp,</div><div class='ctx'> 	if (!dest_rd_atomic)</div><div class='ctx'> 		access_flags &amp;= IB_ACCESS_REMOTE_WRITE;</div><div class='ctx'> </div><div class='del'>-	roce_set_bit(context-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,</div><div class='del'>-		     !!(access_flags &amp; IB_ACCESS_REMOTE_READ));</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 0);</div><div class='add'>+	hr_reg_write_bool(context, QPC_RRE,</div><div class='add'>+			  access_flags &amp; IB_ACCESS_REMOTE_READ);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_RRE);</div><div class='ctx'> </div><div class='del'>-	roce_set_bit(context-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,</div><div class='del'>-		     !!(access_flags &amp; IB_ACCESS_REMOTE_WRITE));</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 0);</div><div class='add'>+	hr_reg_write_bool(context, QPC_RWE,</div><div class='add'>+			  access_flags &amp; IB_ACCESS_REMOTE_WRITE);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_RWE);</div><div class='ctx'> </div><div class='del'>-	roce_set_bit(context-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,</div><div class='del'>-		     !!(access_flags &amp; IB_ACCESS_REMOTE_ATOMIC));</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0);</div><div class='add'>+	hr_reg_write_bool(context, QPC_ATE,</div><div class='add'>+			  access_flags &amp; IB_ACCESS_REMOTE_ATOMIC);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_ATE);</div><div class='add'>+	hr_reg_write_bool(context, QPC_EXT_ATE,</div><div class='add'>+			  access_flags &amp; IB_ACCESS_REMOTE_ATOMIC);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_EXT_ATE);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp,</div><div class='add'>+			    struct hns_roce_v2_qp_context *context)</div><div class='add'>+{</div><div class='add'>+	hr_reg_write(context, QPC_SGE_SHIFT,</div><div class='add'>+		     to_hr_hem_entries_shift(hr_qp-&gt;sge.sge_cnt,</div><div class='add'>+					     hr_qp-&gt;sge.sge_shift));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_SQ_SHIFT, ilog2(hr_qp-&gt;sq.wqe_cnt));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_RQ_SHIFT, ilog2(hr_qp-&gt;rq.wqe_cnt));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int get_cqn(struct ib_cq *ib_cq)</div><div class='add'>+{</div><div class='add'>+	return ib_cq ? to_hr_cq(ib_cq)-&gt;cqn : 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int get_pdn(struct ib_pd *ib_pd)</div><div class='add'>+{</div><div class='add'>+	return ib_pd ? to_hr_pd(ib_pd)-&gt;pdn : 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void modify_qp_reset_to_init(struct ib_qp *ibqp,</div><div class='del'>-				    const struct ib_qp_attr *attr,</div><div class='del'>-				    int attr_mask,</div><div class='del'>-				    struct hns_roce_v2_qp_context *context,</div><div class='del'>-				    struct hns_roce_v2_qp_context *qpc_mask)</div><div class='add'>+				    struct hns_roce_v2_qp_context *context)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='ctx'> 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='hunk'>@@ -3095,1149 +4393,975 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp,</div><div class='ctx'> 	 * we should set all bits of the relevant fields in context mask to</div><div class='ctx'> 	 * 0 at the same time, else set them to 0x1.</div><div class='ctx'> 	 */</div><div class='del'>-	roce_set_field(context-&gt;byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,</div><div class='del'>-		       V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp-&gt;ibqp.qp_type));</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,</div><div class='del'>-		       V2_QPC_BYTE_4_TST_S, 0);</div><div class='del'>-</div><div class='del'>-	if (ibqp-&gt;qp_type == IB_QPT_GSI)</div><div class='del'>-		roce_set_field(context-&gt;byte_4_sqpn_tst,</div><div class='del'>-			       V2_QPC_BYTE_4_SGE_SHIFT_M,</div><div class='del'>-			       V2_QPC_BYTE_4_SGE_SHIFT_S,</div><div class='del'>-			       ilog2((unsigned int)hr_qp-&gt;sge.sge_cnt));</div><div class='del'>-	else</div><div class='del'>-		roce_set_field(context-&gt;byte_4_sqpn_tst,</div><div class='del'>-			       V2_QPC_BYTE_4_SGE_SHIFT_M,</div><div class='del'>-			       V2_QPC_BYTE_4_SGE_SHIFT_S,</div><div class='del'>-			       hr_qp-&gt;sq.max_gs &gt; 2 ?</div><div class='del'>-			       ilog2((unsigned int)hr_qp-&gt;sge.sge_cnt) : 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,</div><div class='del'>-		       V2_QPC_BYTE_4_SGE_SHIFT_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,</div><div class='del'>-		       V2_QPC_BYTE_4_SQPN_S, hr_qp-&gt;qpn);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,</div><div class='del'>-		       V2_QPC_BYTE_4_SQPN_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,</div><div class='del'>-		       V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp-&gt;pd)-&gt;pdn);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,</div><div class='del'>-		       V2_QPC_BYTE_16_PD_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,</div><div class='del'>-		       V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp-&gt;rq.max_gs));</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,</div><div class='del'>-		       V2_QPC_BYTE_20_RQWS_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-		       V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,</div><div class='del'>-		       ilog2((unsigned int)hr_qp-&gt;sq.wqe_cnt));</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-		       V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-		       V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,</div><div class='del'>-		       (hr_qp-&gt;ibqp.qp_type == IB_QPT_XRC_INI ||</div><div class='del'>-		       hr_qp-&gt;ibqp.qp_type == IB_QPT_XRC_TGT || ibqp-&gt;srq) ? 0 :</div><div class='del'>-		       ilog2((unsigned int)hr_qp-&gt;rq.wqe_cnt));</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-		       V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);</div><div class='add'>+	hr_reg_write(context, QPC_TST, to_hr_qp_type(ibqp-&gt;qp_type));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_PD, get_pdn(ibqp-&gt;pd));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_RQWS, ilog2(hr_qp-&gt;rq.max_gs));</div><div class='add'>+</div><div class='add'>+	set_qpc_wqe_cnt(hr_qp, context);</div><div class='ctx'> </div><div class='ctx'> 	/* No VLAN need to set 0xFFF */</div><div class='del'>-	roce_set_field(context-&gt;byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,</div><div class='del'>-		       V2_QPC_BYTE_24_VLAN_ID_S, 0xfff);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,</div><div class='del'>-		       V2_QPC_BYTE_24_VLAN_ID_S, 0);</div><div class='add'>+	hr_reg_write(context, QPC_VLAN_ID, 0xfff);</div><div class='add'>+</div><div class='add'>+	if (ibqp-&gt;qp_type == IB_QPT_XRC_TGT) {</div><div class='add'>+		context-&gt;qkey_xrcd = cpu_to_le32(hr_qp-&gt;xrcdn);</div><div class='add'>+</div><div class='add'>+		hr_reg_enable(context, QPC_XRC_QP_TYPE);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (hr_qp-&gt;en_flags &amp; HNS_ROCE_QP_CAP_RQ_RECORD_DB)</div><div class='add'>+		hr_reg_enable(context, QPC_RQ_RECORD_EN);</div><div class='add'>+</div><div class='add'>+	if (hr_qp-&gt;en_flags &amp; HNS_ROCE_QP_CAP_OWNER_DB)</div><div class='add'>+		hr_reg_enable(context, QPC_OWNER_MODE);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_L,</div><div class='add'>+		     lower_32_bits(hr_qp-&gt;rdb.dma) &gt;&gt; 1);</div><div class='add'>+	hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_H,</div><div class='add'>+		     upper_32_bits(hr_qp-&gt;rdb.dma));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp-&gt;recv_cq));</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * Set some fields in context to zero, Because the default values</div><div class='del'>-	 * of all fields in context are zero, we need not set them to 0 again.</div><div class='del'>-	 * but we should set the relevant fields of context mask to 0.</div><div class='del'>-	 */</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_TX_ERR_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_RX_ERR_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_TX_ERR_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_RX_ERR_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_60_qpst_tempid, V2_QPC_BYTE_60_TEMPID_M,</div><div class='del'>-		       V2_QPC_BYTE_60_TEMPID_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_60_qpst_tempid,</div><div class='del'>-		       V2_QPC_BYTE_60_SCC_TOKEN_M, V2_QPC_BYTE_60_SCC_TOKEN_S,</div><div class='del'>-		       0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_60_qpst_tempid,</div><div class='del'>-		     V2_QPC_BYTE_60_SQ_DB_DOING_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_60_qpst_tempid,</div><div class='del'>-		     V2_QPC_BYTE_60_RQ_DB_DOING_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_28_at_fl, V2_QPC_BYTE_28_CNP_TX_FLAG_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_28_at_fl, V2_QPC_BYTE_28_CE_FLAG_S, 0);</div><div class='del'>-</div><div class='del'>-	if (hr_qp-&gt;rdb_en) {</div><div class='del'>-		roce_set_bit(context-&gt;byte_68_rq_db,</div><div class='del'>-			     V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1);</div><div class='del'>-		roce_set_bit(qpc_mask-&gt;byte_68_rq_db,</div><div class='del'>-			     V2_QPC_BYTE_68_RQ_RECORD_EN_S, 0);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_68_rq_db,</div><div class='del'>-		       V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,</div><div class='del'>-		       V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S,</div><div class='del'>-		       ((u32)hr_qp-&gt;rdb.dma) &gt;&gt; 1);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_68_rq_db,</div><div class='del'>-		       V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,</div><div class='del'>-		       V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S, 0);</div><div class='del'>-	context-&gt;rq_db_record_addr = hr_qp-&gt;rdb.dma &gt;&gt; 32;</div><div class='del'>-	qpc_mask-&gt;rq_db_record_addr = 0;</div><div class='del'>-</div><div class='del'>-	roce_set_bit(context-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S,</div><div class='del'>-		    (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RQ_INLINE) ? 1 : 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,</div><div class='del'>-		       V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp-&gt;recv_cq)-&gt;cqn);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,</div><div class='del'>-		       V2_QPC_BYTE_80_RX_CQN_S, 0);</div><div class='ctx'> 	if (ibqp-&gt;srq) {</div><div class='del'>-		roce_set_field(context-&gt;byte_76_srqn_op_en,</div><div class='del'>-			       V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,</div><div class='del'>-			       to_hr_srq(ibqp-&gt;srq)-&gt;srqn);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_76_srqn_op_en,</div><div class='del'>-			       V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);</div><div class='del'>-		roce_set_bit(context-&gt;byte_76_srqn_op_en,</div><div class='del'>-			     V2_QPC_BYTE_76_SRQ_EN_S, 1);</div><div class='del'>-		roce_set_bit(qpc_mask-&gt;byte_76_srqn_op_en,</div><div class='del'>-			     V2_QPC_BYTE_76_SRQ_EN_S, 0);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_84_rq_ci_pi,</div><div class='del'>-		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,</div><div class='del'>-		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_84_rq_ci_pi,</div><div class='del'>-		       V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,</div><div class='del'>-		       V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_92_srq_info, V2_QPC_BYTE_92_SRQ_INFO_M,</div><div class='del'>-		       V2_QPC_BYTE_92_SRQ_INFO_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,</div><div class='del'>-		       V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_104_rq_sge,</div><div class='del'>-		       V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M,</div><div class='del'>-		       V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_108_rx_reqepsn,</div><div class='del'>-		     V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_108_rx_reqepsn,</div><div class='del'>-		       V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,</div><div class='del'>-		       V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_108_rx_reqepsn,</div><div class='del'>-		     V2_QPC_BYTE_108_RX_REQ_RNR_S, 0);</div><div class='add'>+		hr_reg_enable(context, QPC_SRQ_EN);</div><div class='add'>+		hr_reg_write(context, QPC_SRQN, to_hr_srq(ibqp-&gt;srq)-&gt;srqn);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	qpc_mask-&gt;rq_rnr_timer = 0;</div><div class='del'>-	qpc_mask-&gt;rx_msg_len = 0;</div><div class='del'>-	qpc_mask-&gt;rx_rkey_pkt_info = 0;</div><div class='del'>-	qpc_mask-&gt;rx_va = 0;</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,</div><div class='del'>-		       V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,</div><div class='del'>-		       V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_140_raq, V2_QPC_BYTE_140_RQ_RTY_WAIT_DO_S,</div><div class='del'>-		     0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M,</div><div class='del'>-		       V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M,</div><div class='del'>-		       V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_144_raq,</div><div class='del'>-		       V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M,</div><div class='del'>-		       V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_144_raq, V2_QPC_BYTE_144_RAQ_CREDIT_M,</div><div class='del'>-		       V2_QPC_BYTE_144_RAQ_CREDIT_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_144_raq, V2_QPC_BYTE_144_RESP_RTY_FLG_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_148_raq, V2_QPC_BYTE_148_RQ_MSN_M,</div><div class='del'>-		       V2_QPC_BYTE_148_RQ_MSN_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_148_raq, V2_QPC_BYTE_148_RAQ_SYNDROME_M,</div><div class='del'>-		       V2_QPC_BYTE_148_RAQ_SYNDROME_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,</div><div class='del'>-		       V2_QPC_BYTE_152_RAQ_PSN_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_152_raq,</div><div class='del'>-		       V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M,</div><div class='del'>-		       V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_156_raq, V2_QPC_BYTE_156_RAQ_USE_PKTN_M,</div><div class='del'>-		       V2_QPC_BYTE_156_RAQ_USE_PKTN_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_160_sq_ci_pi,</div><div class='del'>-		       V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,</div><div class='del'>-		       V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_160_sq_ci_pi,</div><div class='del'>-		       V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M,</div><div class='del'>-		       V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_168_irrl_idx,</div><div class='del'>-		     V2_QPC_BYTE_168_POLL_DB_WAIT_DO_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_168_irrl_idx,</div><div class='del'>-		     V2_QPC_BYTE_168_SCC_TOKEN_FORBID_SQ_DEQ_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_168_irrl_idx,</div><div class='del'>-		     V2_QPC_BYTE_168_WAIT_ACK_TIMEOUT_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_168_irrl_idx,</div><div class='del'>-		     V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_168_irrl_idx,</div><div class='del'>-		     V2_QPC_BYTE_168_SQ_INVLD_FLG_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_168_irrl_idx,</div><div class='del'>-		       V2_QPC_BYTE_168_IRRL_IDX_LSB_M,</div><div class='del'>-		       V2_QPC_BYTE_168_IRRL_IDX_LSB_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M,</div><div class='del'>-		       V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 4);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_172_sq_psn,</div><div class='del'>-		       V2_QPC_BYTE_172_ACK_REQ_FREQ_M,</div><div class='del'>-		       V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_172_sq_psn, V2_QPC_BYTE_172_MSG_RNR_FLG_S,</div><div class='del'>-		     0);</div><div class='del'>-</div><div class='del'>-	roce_set_bit(context-&gt;byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 1);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_176_msg_pktn,</div><div class='del'>-		       V2_QPC_BYTE_176_MSG_USE_PKTN_M,</div><div class='del'>-		       V2_QPC_BYTE_176_MSG_USE_PKTN_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_176_msg_pktn,</div><div class='del'>-		       V2_QPC_BYTE_176_IRRL_HEAD_PRE_M,</div><div class='del'>-		       V2_QPC_BYTE_176_IRRL_HEAD_PRE_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_184_irrl_idx,</div><div class='del'>-		       V2_QPC_BYTE_184_IRRL_IDX_MSB_M,</div><div class='del'>-		       V2_QPC_BYTE_184_IRRL_IDX_MSB_S, 0);</div><div class='del'>-</div><div class='del'>-	qpc_mask-&gt;cur_sge_offset = 0;</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_192_ext_sge,</div><div class='del'>-		       V2_QPC_BYTE_192_CUR_SGE_IDX_M,</div><div class='del'>-		       V2_QPC_BYTE_192_CUR_SGE_IDX_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_192_ext_sge,</div><div class='del'>-		       V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M,</div><div class='del'>-		       V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,</div><div class='del'>-		       V2_QPC_BYTE_196_IRRL_HEAD_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_200_sq_max, V2_QPC_BYTE_200_SQ_MAX_IDX_M,</div><div class='del'>-		       V2_QPC_BYTE_200_SQ_MAX_IDX_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_200_sq_max,</div><div class='del'>-		       V2_QPC_BYTE_200_LCL_OPERATED_CNT_M,</div><div class='del'>-		       V2_QPC_BYTE_200_LCL_OPERATED_CNT_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_208_irrl, V2_QPC_BYTE_208_PKT_RNR_FLG_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_208_irrl, V2_QPC_BYTE_208_PKT_RTY_FLG_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,</div><div class='del'>-		       V2_QPC_BYTE_212_CHECK_FLG_S, 0);</div><div class='del'>-</div><div class='del'>-	qpc_mask-&gt;sq_timer = 0;</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_220_retry_psn_msn,</div><div class='del'>-		       V2_QPC_BYTE_220_RETRY_MSG_MSN_M,</div><div class='del'>-		       V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_232_irrl_sge,</div><div class='del'>-		       V2_QPC_BYTE_232_IRRL_SGE_IDX_M,</div><div class='del'>-		       V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_232_irrl_sge, V2_QPC_BYTE_232_SO_LP_VLD_S,</div><div class='del'>-		     0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_232_irrl_sge,</div><div class='del'>-		     V2_QPC_BYTE_232_FENCE_LP_VLD_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_232_irrl_sge, V2_QPC_BYTE_232_IRRL_LP_VLD_S,</div><div class='del'>-		     0);</div><div class='del'>-</div><div class='del'>-	qpc_mask-&gt;irrl_cur_sge_offset = 0;</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_240_irrl_tail,</div><div class='del'>-		       V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,</div><div class='del'>-		       V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_240_irrl_tail,</div><div class='del'>-		       V2_QPC_BYTE_240_IRRL_TAIL_RD_M,</div><div class='del'>-		       V2_QPC_BYTE_240_IRRL_TAIL_RD_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_240_irrl_tail,</div><div class='del'>-		       V2_QPC_BYTE_240_RX_ACK_MSN_M,</div><div class='del'>-		       V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_M,</div><div class='del'>-		       V2_QPC_BYTE_248_IRRL_PSN_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_248_ack_psn, V2_QPC_BYTE_248_ACK_PSN_ERR_S,</div><div class='del'>-		     0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_248_ack_psn,</div><div class='del'>-		       V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,</div><div class='del'>-		       V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_VLD_S,</div><div class='del'>-		     0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_248_ack_psn,</div><div class='del'>-		     V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_248_ack_psn, V2_QPC_BYTE_248_CQ_ERR_IND_S,</div><div class='del'>-		     0);</div><div class='del'>-</div><div class='del'>-	hr_qp-&gt;access_flags = attr-&gt;qp_access_flags;</div><div class='del'>-	roce_set_field(context-&gt;byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,</div><div class='del'>-		       V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp-&gt;send_cq)-&gt;cqn);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,</div><div class='del'>-		       V2_QPC_BYTE_252_TX_CQN_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_252_err_txcqn, V2_QPC_BYTE_252_ERR_TYPE_M,</div><div class='del'>-		       V2_QPC_BYTE_252_ERR_TYPE_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_256_sqflush_rqcqe,</div><div class='del'>-		       V2_QPC_BYTE_256_RQ_CQE_IDX_M,</div><div class='del'>-		       V2_QPC_BYTE_256_RQ_CQE_IDX_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_256_sqflush_rqcqe,</div><div class='del'>-		       V2_QPC_BYTE_256_SQ_FLUSH_IDX_M,</div><div class='del'>-		       V2_QPC_BYTE_256_SQ_FLUSH_IDX_S, 0);</div><div class='add'>+	hr_reg_enable(context, QPC_FRE);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_TX_CQN, get_cqn(ibqp-&gt;send_cq));</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;caps.qpc_sz &lt; HNS_ROCE_V3_QPC_SZ)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_STASH)</div><div class='add'>+		hr_reg_enable(&amp;context-&gt;ext, QPCEX_STASH);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void modify_qp_init_to_init(struct ib_qp *ibqp,</div><div class='del'>-				   const struct ib_qp_attr *attr, int attr_mask,</div><div class='ctx'> 				   struct hns_roce_v2_qp_context *context,</div><div class='ctx'> 				   struct hns_roce_v2_qp_context *qpc_mask)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='del'>-</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * In v2 engine, software pass context and context mask to hardware</div><div class='ctx'> 	 * when modifying qp. If software need modify some fields in context,</div><div class='ctx'> 	 * we should set all bits of the relevant fields in context mask to</div><div class='ctx'> 	 * 0 at the same time, else set them to 0x1.</div><div class='ctx'> 	 */</div><div class='del'>-	roce_set_field(context-&gt;byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,</div><div class='del'>-		       V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp-&gt;ibqp.qp_type));</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,</div><div class='del'>-		       V2_QPC_BYTE_4_TST_S, 0);</div><div class='del'>-</div><div class='del'>-	if (ibqp-&gt;qp_type == IB_QPT_GSI)</div><div class='del'>-		roce_set_field(context-&gt;byte_4_sqpn_tst,</div><div class='del'>-			       V2_QPC_BYTE_4_SGE_SHIFT_M,</div><div class='del'>-			       V2_QPC_BYTE_4_SGE_SHIFT_S,</div><div class='del'>-			       ilog2((unsigned int)hr_qp-&gt;sge.sge_cnt));</div><div class='del'>-	else</div><div class='del'>-		roce_set_field(context-&gt;byte_4_sqpn_tst,</div><div class='del'>-			       V2_QPC_BYTE_4_SGE_SHIFT_M,</div><div class='del'>-			       V2_QPC_BYTE_4_SGE_SHIFT_S, hr_qp-&gt;sq.max_gs &gt; 2 ?</div><div class='del'>-			       ilog2((unsigned int)hr_qp-&gt;sge.sge_cnt) : 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,</div><div class='del'>-		       V2_QPC_BYTE_4_SGE_SHIFT_S, 0);</div><div class='del'>-</div><div class='del'>-	if (attr_mask &amp; IB_QP_ACCESS_FLAGS) {</div><div class='del'>-		roce_set_bit(context-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,</div><div class='del'>-			     !!(attr-&gt;qp_access_flags &amp; IB_ACCESS_REMOTE_READ));</div><div class='del'>-		roce_set_bit(qpc_mask-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,</div><div class='del'>-			     0);</div><div class='del'>-</div><div class='del'>-		roce_set_bit(context-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,</div><div class='del'>-			     !!(attr-&gt;qp_access_flags &amp;</div><div class='del'>-			     IB_ACCESS_REMOTE_WRITE));</div><div class='del'>-		roce_set_bit(qpc_mask-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,</div><div class='del'>-			     0);</div><div class='del'>-</div><div class='del'>-		roce_set_bit(context-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,</div><div class='del'>-			     !!(attr-&gt;qp_access_flags &amp;</div><div class='del'>-			     IB_ACCESS_REMOTE_ATOMIC));</div><div class='del'>-		roce_set_bit(qpc_mask-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,</div><div class='del'>-			     0);</div><div class='del'>-	} else {</div><div class='del'>-		roce_set_bit(context-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,</div><div class='del'>-			     !!(hr_qp-&gt;access_flags &amp; IB_ACCESS_REMOTE_READ));</div><div class='del'>-		roce_set_bit(qpc_mask-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,</div><div class='del'>-			     0);</div><div class='del'>-</div><div class='del'>-		roce_set_bit(context-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,</div><div class='del'>-			     !!(hr_qp-&gt;access_flags &amp; IB_ACCESS_REMOTE_WRITE));</div><div class='del'>-		roce_set_bit(qpc_mask-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,</div><div class='del'>-			     0);</div><div class='del'>-</div><div class='del'>-		roce_set_bit(context-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,</div><div class='del'>-			     !!(hr_qp-&gt;access_flags &amp; IB_ACCESS_REMOTE_ATOMIC));</div><div class='del'>-		roce_set_bit(qpc_mask-&gt;byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,</div><div class='del'>-			     0);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-		       V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,</div><div class='del'>-		       ilog2((unsigned int)hr_qp-&gt;sq.wqe_cnt));</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-		       V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-		       V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,</div><div class='del'>-		       (hr_qp-&gt;ibqp.qp_type == IB_QPT_XRC_INI ||</div><div class='del'>-		       hr_qp-&gt;ibqp.qp_type == IB_QPT_XRC_TGT || ibqp-&gt;srq) ? 0 :</div><div class='del'>-		       ilog2((unsigned int)hr_qp-&gt;rq.wqe_cnt));</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-		       V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,</div><div class='del'>-		       V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp-&gt;pd)-&gt;pdn);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,</div><div class='del'>-		       V2_QPC_BYTE_16_PD_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,</div><div class='del'>-		       V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp-&gt;recv_cq)-&gt;cqn);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,</div><div class='del'>-		       V2_QPC_BYTE_80_RX_CQN_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,</div><div class='del'>-		       V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp-&gt;send_cq)-&gt;cqn);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,</div><div class='del'>-		       V2_QPC_BYTE_252_TX_CQN_S, 0);</div><div class='add'>+	hr_reg_write(context, QPC_TST, to_hr_qp_type(ibqp-&gt;qp_type));</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_TST);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_PD, get_pdn(ibqp-&gt;pd));</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_PD);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp-&gt;recv_cq));</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_RX_CQN);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_TX_CQN, get_cqn(ibqp-&gt;send_cq));</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_TX_CQN);</div><div class='ctx'> </div><div class='ctx'> 	if (ibqp-&gt;srq) {</div><div class='del'>-		roce_set_bit(context-&gt;byte_76_srqn_op_en,</div><div class='del'>-			     V2_QPC_BYTE_76_SRQ_EN_S, 1);</div><div class='del'>-		roce_set_bit(qpc_mask-&gt;byte_76_srqn_op_en,</div><div class='del'>-			     V2_QPC_BYTE_76_SRQ_EN_S, 0);</div><div class='del'>-		roce_set_field(context-&gt;byte_76_srqn_op_en,</div><div class='del'>-			       V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,</div><div class='del'>-			       to_hr_srq(ibqp-&gt;srq)-&gt;srqn);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_76_srqn_op_en,</div><div class='del'>-			       V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,</div><div class='del'>-		       V2_QPC_BYTE_4_SQPN_S, hr_qp-&gt;qpn);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,</div><div class='del'>-		       V2_QPC_BYTE_4_SQPN_S, 0);</div><div class='add'>+		hr_reg_enable(context, QPC_SRQ_EN);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_SRQ_EN);</div><div class='add'>+		hr_reg_write(context, QPC_SRQN, to_hr_srq(ibqp-&gt;srq)-&gt;srqn);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_SRQN);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (attr_mask &amp; IB_QP_DEST_QPN) {</div><div class='del'>-		roce_set_field(context-&gt;byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,</div><div class='del'>-			       V2_QPC_BYTE_56_DQPN_S, hr_qp-&gt;qpn);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_56_dqpn_err,</div><div class='del'>-			       V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);</div><div class='add'>+static int config_qp_rq_buf(struct hns_roce_dev *hr_dev,</div><div class='add'>+			    struct hns_roce_qp *hr_qp,</div><div class='add'>+			    struct hns_roce_v2_qp_context *context,</div><div class='add'>+			    struct hns_roce_v2_qp_context *qpc_mask)</div><div class='add'>+{</div><div class='add'>+	u64 mtts[MTT_MIN_COUNT] = { 0 };</div><div class='add'>+	u64 wqe_sge_ba;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	/* Search qp buf's mtts */</div><div class='add'>+	ret = hns_roce_mtr_find(hr_dev, &amp;hr_qp-&gt;mtr, hr_qp-&gt;rq.offset, mtts,</div><div class='add'>+				MTT_MIN_COUNT);</div><div class='add'>+	if (hr_qp-&gt;rq.wqe_cnt &amp;&amp; ret) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+			  "failed to find QP(0x%lx) RQ WQE buf, ret = %d.\n",</div><div class='add'>+			  hr_qp-&gt;qpn, ret);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	wqe_sge_ba = hns_roce_get_mtr_ba(&amp;hr_qp-&gt;mtr);</div><div class='add'>+</div><div class='add'>+	context-&gt;wqe_sge_ba = cpu_to_le32(wqe_sge_ba &gt;&gt; 3);</div><div class='add'>+	qpc_mask-&gt;wqe_sge_ba = 0;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * In v2 engine, software pass context and context mask to hardware</div><div class='add'>+	 * when modifying qp. If software need modify some fields in context,</div><div class='add'>+	 * we should set all bits of the relevant fields in context mask to</div><div class='add'>+	 * 0 at the same time, else set them to 0x1.</div><div class='add'>+	 */</div><div class='add'>+	hr_reg_write(context, QPC_WQE_SGE_BA_H, wqe_sge_ba &gt;&gt; (32 + 3));</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_WQE_SGE_BA_H);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_SQ_HOP_NUM,</div><div class='add'>+		     to_hr_hem_hopnum(hr_dev-&gt;caps.wqe_sq_hop_num,</div><div class='add'>+				      hr_qp-&gt;sq.wqe_cnt));</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_SQ_HOP_NUM);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_SGE_HOP_NUM,</div><div class='add'>+		     to_hr_hem_hopnum(hr_dev-&gt;caps.wqe_sge_hop_num,</div><div class='add'>+				      hr_qp-&gt;sge.sge_cnt));</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_SGE_HOP_NUM);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_RQ_HOP_NUM,</div><div class='add'>+		     to_hr_hem_hopnum(hr_dev-&gt;caps.wqe_rq_hop_num,</div><div class='add'>+				      hr_qp-&gt;rq.wqe_cnt));</div><div class='add'>+</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_RQ_HOP_NUM);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_WQE_SGE_BA_PG_SZ,</div><div class='add'>+		     to_hr_hw_page_shift(hr_qp-&gt;mtr.hem_cfg.ba_pg_shift));</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_WQE_SGE_BA_PG_SZ);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_WQE_SGE_BUF_PG_SZ,</div><div class='add'>+		     to_hr_hw_page_shift(hr_qp-&gt;mtr.hem_cfg.buf_pg_shift));</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_WQE_SGE_BUF_PG_SZ);</div><div class='add'>+</div><div class='add'>+	context-&gt;rq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0]));</div><div class='add'>+	qpc_mask-&gt;rq_cur_blk_addr = 0;</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_RQ_CUR_BLK_ADDR_H,</div><div class='add'>+		     upper_32_bits(to_hr_hw_page_addr(mtts[0])));</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_RQ_CUR_BLK_ADDR_H);</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08) {</div><div class='add'>+		context-&gt;rq_nxt_blk_addr =</div><div class='add'>+				cpu_to_le32(to_hr_hw_page_addr(mtts[1]));</div><div class='add'>+		qpc_mask-&gt;rq_nxt_blk_addr = 0;</div><div class='add'>+		hr_reg_write(context, QPC_RQ_NXT_BLK_ADDR_H,</div><div class='add'>+			     upper_32_bits(to_hr_hw_page_addr(mtts[1])));</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_RQ_NXT_BLK_ADDR_H);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int config_qp_sq_buf(struct hns_roce_dev *hr_dev,</div><div class='add'>+			    struct hns_roce_qp *hr_qp,</div><div class='add'>+			    struct hns_roce_v2_qp_context *context,</div><div class='add'>+			    struct hns_roce_v2_qp_context *qpc_mask)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	u64 sge_cur_blk = 0;</div><div class='add'>+	u64 sq_cur_blk = 0;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	/* search qp buf's mtts */</div><div class='add'>+	ret = hns_roce_mtr_find(hr_dev, &amp;hr_qp-&gt;mtr, hr_qp-&gt;sq.offset,</div><div class='add'>+				&amp;sq_cur_blk, 1);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to find QP(0x%lx) SQ WQE buf, ret = %d.\n",</div><div class='add'>+			  hr_qp-&gt;qpn, ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+	if (hr_qp-&gt;sge.sge_cnt &gt; 0) {</div><div class='add'>+		ret = hns_roce_mtr_find(hr_dev, &amp;hr_qp-&gt;mtr,</div><div class='add'>+					hr_qp-&gt;sge.offset, &amp;sge_cur_blk, 1);</div><div class='add'>+		if (ret) {</div><div class='add'>+			ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf, ret = %d.\n",</div><div class='add'>+				  hr_qp-&gt;qpn, ret);</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * In v2 engine, software pass context and context mask to hardware</div><div class='add'>+	 * when modifying qp. If software need modify some fields in context,</div><div class='add'>+	 * we should set all bits of the relevant fields in context mask to</div><div class='add'>+	 * 0 at the same time, else set them to 0x1.</div><div class='add'>+	 */</div><div class='add'>+	hr_reg_write(context, QPC_SQ_CUR_BLK_ADDR_L,</div><div class='add'>+		     lower_32_bits(to_hr_hw_page_addr(sq_cur_blk)));</div><div class='add'>+	hr_reg_write(context, QPC_SQ_CUR_BLK_ADDR_H,</div><div class='add'>+		     upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_SQ_CUR_BLK_ADDR_L);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_SQ_CUR_BLK_ADDR_H);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_L,</div><div class='add'>+		     lower_32_bits(to_hr_hw_page_addr(sge_cur_blk)));</div><div class='add'>+	hr_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_H,</div><div class='add'>+		     upper_32_bits(to_hr_hw_page_addr(sge_cur_blk)));</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_SQ_CUR_SGE_BLK_ADDR_L);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_SQ_CUR_SGE_BLK_ADDR_H);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_L,</div><div class='add'>+		     lower_32_bits(to_hr_hw_page_addr(sq_cur_blk)));</div><div class='add'>+	hr_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_H,</div><div class='add'>+		     upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_RX_SQ_CUR_BLK_ADDR_L);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_RX_SQ_CUR_BLK_ADDR_H);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline enum ib_mtu get_mtu(struct ib_qp *ibqp,</div><div class='add'>+				  const struct ib_qp_attr *attr)</div><div class='add'>+{</div><div class='add'>+	if (ibqp-&gt;qp_type == IB_QPT_GSI || ibqp-&gt;qp_type == IB_QPT_UD)</div><div class='add'>+		return IB_MTU_4096;</div><div class='add'>+</div><div class='add'>+	return attr-&gt;path_mtu;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int modify_qp_init_to_rtr(struct ib_qp *ibqp,</div><div class='ctx'> 				 const struct ib_qp_attr *attr, int attr_mask,</div><div class='ctx'> 				 struct hns_roce_v2_qp_context *context,</div><div class='del'>-				 struct hns_roce_v2_qp_context *qpc_mask)</div><div class='add'>+				 struct hns_roce_v2_qp_context *qpc_mask,</div><div class='add'>+				 struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	const struct ib_global_route *grh = rdma_ah_read_grh(&amp;attr-&gt;ah_attr);</div><div class='add'>+	struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata,</div><div class='add'>+					  struct hns_roce_ucontext, ibucontext);</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='ctx'> 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	dma_addr_t dma_handle_3;</div><div class='del'>-	dma_addr_t dma_handle_2;</div><div class='del'>-	dma_addr_t dma_handle;</div><div class='del'>-	u32 page_size;</div><div class='del'>-	u8 port_num;</div><div class='del'>-	u64 *mtts_3;</div><div class='del'>-	u64 *mtts_2;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	dma_addr_t trrl_ba;</div><div class='add'>+	dma_addr_t irrl_ba;</div><div class='add'>+	enum ib_mtu ib_mtu;</div><div class='add'>+	u8 ack_req_freq;</div><div class='add'>+	const u8 *smac;</div><div class='add'>+	int lp_msg_len;</div><div class='add'>+	u8 lp_pktn_ini;</div><div class='ctx'> 	u64 *mtts;</div><div class='ctx'> 	u8 *dmac;</div><div class='del'>-	u8 *smac;</div><div class='del'>-	int port;</div><div class='add'>+	u32 port;</div><div class='add'>+	int mtu;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	/* Search qp buf's mtts */</div><div class='del'>-	mtts = hns_roce_table_find(hr_dev, &amp;hr_dev-&gt;mr_table.mtt_table,</div><div class='del'>-				   hr_qp-&gt;mtt.first_seg, &amp;dma_handle);</div><div class='del'>-	if (!mtts) {</div><div class='del'>-		dev_err(dev, "qp buf pa find failed\n");</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	ret = config_qp_rq_buf(hr_dev, hr_qp, context, qpc_mask);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to config rq buf, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Search IRRL's mtts */</div><div class='del'>-	mtts_2 = hns_roce_table_find(hr_dev, &amp;hr_dev-&gt;qp_table.irrl_table,</div><div class='del'>-				     hr_qp-&gt;qpn, &amp;dma_handle_2);</div><div class='del'>-	if (!mtts_2) {</div><div class='del'>-		dev_err(dev, "qp irrl_table find failed\n");</div><div class='add'>+	mtts = hns_roce_table_find(hr_dev, &amp;hr_dev-&gt;qp_table.irrl_table,</div><div class='add'>+				   hr_qp-&gt;qpn, &amp;irrl_ba);</div><div class='add'>+	if (!mtts) {</div><div class='add'>+		ibdev_err(ibdev, "failed to find qp irrl_table.\n");</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Search TRRL's mtts */</div><div class='del'>-	mtts_3 = hns_roce_table_find(hr_dev, &amp;hr_dev-&gt;qp_table.trrl_table,</div><div class='del'>-				     hr_qp-&gt;qpn, &amp;dma_handle_3);</div><div class='del'>-	if (!mtts_3) {</div><div class='del'>-		dev_err(dev, "qp trrl_table find failed\n");</div><div class='add'>+	mtts = hns_roce_table_find(hr_dev, &amp;hr_dev-&gt;qp_table.trrl_table,</div><div class='add'>+				   hr_qp-&gt;qpn, &amp;trrl_ba);</div><div class='add'>+	if (!mtts) {</div><div class='add'>+		ibdev_err(ibdev, "failed to find qp trrl_table.\n");</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_ALT_PATH) {</div><div class='del'>-		dev_err(dev, "INIT2RTR attr_mask (0x%x) error\n", attr_mask);</div><div class='add'>+		ibdev_err(ibdev, "INIT2RTR attr_mask (0x%x) error.\n",</div><div class='add'>+			  attr_mask);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	dmac = (u8 *)attr-&gt;ah_attr.roce.dmac;</div><div class='del'>-	context-&gt;wqe_sge_ba = (u32)(dma_handle &gt;&gt; 3);</div><div class='del'>-	qpc_mask-&gt;wqe_sge_ba = 0;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * In v2 engine, software pass context and context mask to hardware</div><div class='del'>-	 * when modifying qp. If software need modify some fields in context,</div><div class='del'>-	 * we should set all bits of the relevant fields in context mask to</div><div class='del'>-	 * 0 at the same time, else set them to 0x1.</div><div class='del'>-	 */</div><div class='del'>-	roce_set_field(context-&gt;byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,</div><div class='del'>-		       V2_QPC_BYTE_12_WQE_SGE_BA_S, dma_handle &gt;&gt; (32 + 3));</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,</div><div class='del'>-		       V2_QPC_BYTE_12_WQE_SGE_BA_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,</div><div class='del'>-		       V2_QPC_BYTE_12_SQ_HOP_NUM_S,</div><div class='del'>-		       hr_dev-&gt;caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ?</div><div class='del'>-		       0 : hr_dev-&gt;caps.mtt_hop_num);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,</div><div class='del'>-		       V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-		       V2_QPC_BYTE_20_SGE_HOP_NUM_M,</div><div class='del'>-		       V2_QPC_BYTE_20_SGE_HOP_NUM_S,</div><div class='del'>-		       ((ibqp-&gt;qp_type == IB_QPT_GSI) || hr_qp-&gt;sq.max_gs &gt; 2) ?</div><div class='del'>-		       hr_dev-&gt;caps.mtt_hop_num : 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-		       V2_QPC_BYTE_20_SGE_HOP_NUM_M,</div><div class='del'>-		       V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-		       V2_QPC_BYTE_20_RQ_HOP_NUM_M,</div><div class='del'>-		       V2_QPC_BYTE_20_RQ_HOP_NUM_S,</div><div class='del'>-		       hr_dev-&gt;caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ?</div><div class='del'>-		       0 : hr_dev-&gt;caps.mtt_hop_num);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-		       V2_QPC_BYTE_20_RQ_HOP_NUM_M,</div><div class='del'>-		       V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_16_buf_ba_pg_sz,</div><div class='del'>-		       V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,</div><div class='del'>-		       V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.mtt_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_16_buf_ba_pg_sz,</div><div class='del'>-		       V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,</div><div class='del'>-		       V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_16_buf_ba_pg_sz,</div><div class='del'>-		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,</div><div class='del'>-		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.mtt_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_16_buf_ba_pg_sz,</div><div class='del'>-		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,</div><div class='del'>-		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);</div><div class='del'>-</div><div class='del'>-	page_size = 1 &lt;&lt; (hr_dev-&gt;caps.mtt_buf_pg_sz + PAGE_SHIFT);</div><div class='del'>-	context-&gt;rq_cur_blk_addr = (u32)(mtts[hr_qp-&gt;rq.offset / page_size]</div><div class='del'>-				    &gt;&gt; PAGE_ADDR_SHIFT);</div><div class='del'>-	qpc_mask-&gt;rq_cur_blk_addr = 0;</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_92_srq_info,</div><div class='del'>-		       V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,</div><div class='del'>-		       V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S,</div><div class='del'>-		       mtts[hr_qp-&gt;rq.offset / page_size]</div><div class='del'>-		       &gt;&gt; (32 + PAGE_ADDR_SHIFT));</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_92_srq_info,</div><div class='del'>-		       V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,</div><div class='del'>-		       V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0);</div><div class='del'>-</div><div class='del'>-	context-&gt;rq_nxt_blk_addr = (u32)(mtts[hr_qp-&gt;rq.offset / page_size + 1]</div><div class='del'>-				    &gt;&gt; PAGE_ADDR_SHIFT);</div><div class='del'>-	qpc_mask-&gt;rq_nxt_blk_addr = 0;</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_104_rq_sge,</div><div class='del'>-		       V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,</div><div class='del'>-		       V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S,</div><div class='del'>-		       mtts[hr_qp-&gt;rq.offset / page_size + 1]</div><div class='del'>-		       &gt;&gt; (32 + PAGE_ADDR_SHIFT));</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_104_rq_sge,</div><div class='del'>-		       V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,</div><div class='del'>-		       V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,</div><div class='del'>-		       V2_QPC_BYTE_132_TRRL_BA_S, dma_handle_3 &gt;&gt; 4);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,</div><div class='del'>-		       V2_QPC_BYTE_132_TRRL_BA_S, 0);</div><div class='del'>-	context-&gt;trrl_ba = (u32)(dma_handle_3 &gt;&gt; (16 + 4));</div><div class='add'>+	hr_reg_write(context, QPC_TRRL_BA_L, trrl_ba &gt;&gt; QPC_TRRL_BA_L_S);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_TRRL_BA_L);</div><div class='add'>+	context-&gt;trrl_ba = cpu_to_le32(trrl_ba &gt;&gt; QPC_TRRL_BA_M_S);</div><div class='ctx'> 	qpc_mask-&gt;trrl_ba = 0;</div><div class='del'>-	roce_set_field(context-&gt;byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,</div><div class='del'>-		       V2_QPC_BYTE_140_TRRL_BA_S,</div><div class='del'>-		       (u32)(dma_handle_3 &gt;&gt; (32 + 16 + 4)));</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,</div><div class='del'>-		       V2_QPC_BYTE_140_TRRL_BA_S, 0);</div><div class='add'>+	hr_reg_write(context, QPC_TRRL_BA_H, trrl_ba &gt;&gt; QPC_TRRL_BA_H_S);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_TRRL_BA_H);</div><div class='ctx'> </div><div class='del'>-	context-&gt;irrl_ba = (u32)(dma_handle_2 &gt;&gt; 6);</div><div class='add'>+	context-&gt;irrl_ba = cpu_to_le32(irrl_ba &gt;&gt; QPC_IRRL_BA_L_S);</div><div class='ctx'> 	qpc_mask-&gt;irrl_ba = 0;</div><div class='del'>-	roce_set_field(context-&gt;byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,</div><div class='del'>-		       V2_QPC_BYTE_208_IRRL_BA_S,</div><div class='del'>-		       dma_handle_2 &gt;&gt; (32 + 6));</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,</div><div class='del'>-		       V2_QPC_BYTE_208_IRRL_BA_S, 0);</div><div class='add'>+	hr_reg_write(context, QPC_IRRL_BA_H, irrl_ba &gt;&gt; QPC_IRRL_BA_H_S);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_IRRL_BA_H);</div><div class='ctx'> </div><div class='del'>-	roce_set_bit(context-&gt;byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 1);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 0);</div><div class='add'>+	hr_reg_enable(context, QPC_RMT_E2E);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_RMT_E2E);</div><div class='ctx'> </div><div class='del'>-	roce_set_bit(context-&gt;byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,</div><div class='del'>-		     hr_qp-&gt;sq_signal_bits);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,</div><div class='del'>-		     0);</div><div class='add'>+	hr_reg_write(context, QPC_SIG_TYPE, hr_qp-&gt;sq_signal_bits);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_SIG_TYPE);</div><div class='ctx'> </div><div class='ctx'> 	port = (attr_mask &amp; IB_QP_PORT) ? (attr-&gt;port_num - 1) : hr_qp-&gt;port;</div><div class='ctx'> </div><div class='del'>-	smac = (u8 *)hr_dev-&gt;dev_addr[port];</div><div class='add'>+	smac = (const u8 *)hr_dev-&gt;dev_addr[port];</div><div class='add'>+	dmac = (u8 *)attr-&gt;ah_attr.roce.dmac;</div><div class='ctx'> 	/* when dmac equals smac or loop_idc is 1, it should loopback */</div><div class='ctx'> 	if (ether_addr_equal_unaligned(dmac, smac) ||</div><div class='ctx'> 	    hr_dev-&gt;loop_idc == 0x1) {</div><div class='del'>-		roce_set_bit(context-&gt;byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 1);</div><div class='del'>-		roce_set_bit(qpc_mask-&gt;byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0);</div><div class='add'>+		hr_reg_write(context, QPC_LBI, hr_dev-&gt;loop_idc);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_LBI);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_DEST_QPN) {</div><div class='del'>-		roce_set_field(context-&gt;byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,</div><div class='del'>-			       V2_QPC_BYTE_56_DQPN_S, attr-&gt;dest_qp_num);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_56_dqpn_err,</div><div class='del'>-			       V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Configure GID index */</div><div class='del'>-	port_num = rdma_ah_get_port_num(&amp;attr-&gt;ah_attr);</div><div class='del'>-	roce_set_field(context-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-		       V2_QPC_BYTE_20_SGID_IDX_M,</div><div class='del'>-		       V2_QPC_BYTE_20_SGID_IDX_S,</div><div class='del'>-		       hns_get_gid_index(hr_dev, port_num - 1,</div><div class='del'>-					 grh-&gt;sgid_index));</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-		       V2_QPC_BYTE_20_SGID_IDX_M,</div><div class='del'>-		       V2_QPC_BYTE_20_SGID_IDX_S, 0);</div><div class='del'>-	memcpy(&amp;(context-&gt;dmac), dmac, 4);</div><div class='del'>-	roce_set_field(context-&gt;byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,</div><div class='del'>-		       V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&amp;dmac[4])));</div><div class='add'>+		hr_reg_write(context, QPC_DQPN, attr-&gt;dest_qp_num);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_DQPN);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	memcpy(&amp;context-&gt;dmac, dmac, sizeof(u32));</div><div class='add'>+	hr_reg_write(context, QPC_DMAC_H, *((u16 *)(&amp;dmac[4])));</div><div class='ctx'> 	qpc_mask-&gt;dmac = 0;</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,</div><div class='del'>-		       V2_QPC_BYTE_52_DMAC_S, 0);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_DMAC_H);</div><div class='ctx'> </div><div class='del'>-	roce_set_field(context-&gt;byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,</div><div class='del'>-		       V2_QPC_BYTE_56_LP_PKTN_INI_S, 4);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,</div><div class='del'>-		       V2_QPC_BYTE_56_LP_PKTN_INI_S, 0);</div><div class='add'>+	ib_mtu = get_mtu(ibqp, attr);</div><div class='add'>+	hr_qp-&gt;path_mtu = ib_mtu;</div><div class='ctx'> </div><div class='del'>-	if (ibqp-&gt;qp_type == IB_QPT_GSI || ibqp-&gt;qp_type == IB_QPT_UD)</div><div class='del'>-		roce_set_field(context-&gt;byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,</div><div class='del'>-			       V2_QPC_BYTE_24_MTU_S, IB_MTU_4096);</div><div class='del'>-	else if (attr_mask &amp; IB_QP_PATH_MTU)</div><div class='del'>-		roce_set_field(context-&gt;byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,</div><div class='del'>-			       V2_QPC_BYTE_24_MTU_S, attr-&gt;path_mtu);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,</div><div class='del'>-		       V2_QPC_BYTE_24_MTU_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(context-&gt;byte_84_rq_ci_pi,</div><div class='del'>-		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,</div><div class='del'>-		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp-&gt;rq.head);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_84_rq_ci_pi,</div><div class='del'>-		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,</div><div class='del'>-		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_84_rq_ci_pi,</div><div class='del'>-		       V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,</div><div class='del'>-		       V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_108_rx_reqepsn,</div><div class='del'>-		     V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,</div><div class='del'>-		       V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_108_rx_reqepsn,</div><div class='del'>-		       V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,</div><div class='del'>-		       V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);</div><div class='add'>+	mtu = ib_mtu_enum_to_int(ib_mtu);</div><div class='add'>+	if (WARN_ON(mtu &lt;= 0))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+#define MIN_LP_MSG_LEN 1024</div><div class='add'>+	/* mtu * (2 ^ lp_pktn_ini) should be in the range of 1024 to mtu */</div><div class='add'>+	lp_msg_len = max(mtu, MIN_LP_MSG_LEN);</div><div class='add'>+	lp_pktn_ini = ilog2(lp_msg_len / mtu);</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_PATH_MTU) {</div><div class='add'>+		hr_reg_write(context, QPC_MTU, ib_mtu);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_MTU);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_LP_PKTN_INI, lp_pktn_ini);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_LP_PKTN_INI);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * There are several constraints for ACK_REQ_FREQ:</div><div class='add'>+	 * 1. mtu * (2 ^ ACK_REQ_FREQ) should not be too large, otherwise</div><div class='add'>+	 *    it may cause some unexpected retries when sending large</div><div class='add'>+	 *    payload.</div><div class='add'>+	 * 2. ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI.</div><div class='add'>+	 * 3. ACK_REQ_FREQ must be equal to LP_PKTN_INI when using LDCP</div><div class='add'>+	 *    or HC3 congestion control algorithm.</div><div class='add'>+	 */</div><div class='add'>+	if (hr_qp-&gt;cong_type == CONG_TYPE_LDCP ||</div><div class='add'>+	    hr_qp-&gt;cong_type == CONG_TYPE_HC3 ||</div><div class='add'>+	    hr_dev-&gt;caps.max_ack_req_msg_len &lt; lp_msg_len)</div><div class='add'>+		ack_req_freq = lp_pktn_ini;</div><div class='add'>+	else</div><div class='add'>+		ack_req_freq = ilog2(hr_dev-&gt;caps.max_ack_req_msg_len / mtu);</div><div class='add'>+	hr_reg_write(context, QPC_ACK_REQ_FREQ, ack_req_freq);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_ACK_REQ_FREQ);</div><div class='add'>+</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_RX_REQ_PSN_ERR);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_RX_REQ_MSN);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_RX_REQ_LAST_OPTYPE);</div><div class='ctx'> </div><div class='ctx'> 	context-&gt;rq_rnr_timer = 0;</div><div class='ctx'> 	qpc_mask-&gt;rq_rnr_timer = 0;</div><div class='ctx'> </div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,</div><div class='del'>-		       V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,</div><div class='del'>-		       V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_TRRL_HEAD_MAX);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_TRRL_TAIL_MAX);</div><div class='add'>+</div><div class='add'>+#define MAX_LP_SGEN 3</div><div class='add'>+	/* rocee send 2^lp_sgen_ini segs every time */</div><div class='add'>+	hr_reg_write(context, QPC_LP_SGEN_INI, MAX_LP_SGEN);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_LP_SGEN_INI);</div><div class='ctx'> </div><div class='del'>-	roce_set_field(context-&gt;byte_168_irrl_idx,</div><div class='del'>-		       V2_QPC_BYTE_168_LP_SGEN_INI_M,</div><div class='del'>-		       V2_QPC_BYTE_168_LP_SGEN_INI_S, 3);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_168_irrl_idx,</div><div class='del'>-		       V2_QPC_BYTE_168_LP_SGEN_INI_M,</div><div class='del'>-		       V2_QPC_BYTE_168_LP_SGEN_INI_S, 0);</div><div class='add'>+	if (udata &amp;&amp; ibqp-&gt;qp_type == IB_QPT_RC &amp;&amp;</div><div class='add'>+	    (uctx-&gt;config &amp; HNS_ROCE_RQ_INLINE_FLAGS)) {</div><div class='add'>+		hr_reg_write_bool(context, QPC_RQIE,</div><div class='add'>+				  hr_dev-&gt;caps.flags &amp;</div><div class='add'>+				  HNS_ROCE_CAP_FLAG_RQ_INLINE);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_RQIE);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (udata &amp;&amp;</div><div class='add'>+	    (ibqp-&gt;qp_type == IB_QPT_RC || ibqp-&gt;qp_type == IB_QPT_XRC_TGT) &amp;&amp;</div><div class='add'>+	    (uctx-&gt;config &amp; HNS_ROCE_CQE_INLINE_FLAGS)) {</div><div class='add'>+		hr_reg_write_bool(context, QPC_CQEIE,</div><div class='add'>+				  hr_dev-&gt;caps.flags &amp;</div><div class='add'>+				  HNS_ROCE_CAP_FLAG_CQE_INLINE);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_CQEIE);</div><div class='add'>+</div><div class='add'>+		hr_reg_write(context, QPC_CQEIS, 0);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_CQEIS);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,</div><div class='del'>-				const struct ib_qp_attr *attr, int attr_mask,</div><div class='add'>+static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, int attr_mask,</div><div class='ctx'> 				struct hns_roce_v2_qp_context *context,</div><div class='ctx'> 				struct hns_roce_v2_qp_context *qpc_mask)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='ctx'> 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	dma_addr_t dma_handle;</div><div class='del'>-	u32 page_size;</div><div class='del'>-	u64 *mtts;</div><div class='del'>-</div><div class='del'>-	/* Search qp buf's mtts */</div><div class='del'>-	mtts = hns_roce_table_find(hr_dev, &amp;hr_dev-&gt;mr_table.mtt_table,</div><div class='del'>-				   hr_qp-&gt;mtt.first_seg, &amp;dma_handle);</div><div class='del'>-	if (!mtts) {</div><div class='del'>-		dev_err(dev, "qp buf pa find failed\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	/* Not support alternate path and path migration */</div><div class='del'>-	if ((attr_mask &amp; IB_QP_ALT_PATH) ||</div><div class='del'>-	    (attr_mask &amp; IB_QP_PATH_MIG_STATE)) {</div><div class='del'>-		dev_err(dev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);</div><div class='add'>+	if (attr_mask &amp; (IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE)) {</div><div class='add'>+		ibdev_err(ibdev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * In v2 engine, software pass context and context mask to hardware</div><div class='del'>-	 * when modifying qp. If software need modify some fields in context,</div><div class='del'>-	 * we should set all bits of the relevant fields in context mask to</div><div class='del'>-	 * 0 at the same time, else set them to 0x1.</div><div class='del'>-	 */</div><div class='del'>-	context-&gt;sq_cur_blk_addr = (u32)(mtts[0] &gt;&gt; PAGE_ADDR_SHIFT);</div><div class='del'>-	roce_set_field(context-&gt;byte_168_irrl_idx,</div><div class='del'>-		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,</div><div class='del'>-		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S,</div><div class='del'>-		       mtts[0] &gt;&gt; (32 + PAGE_ADDR_SHIFT));</div><div class='del'>-	qpc_mask-&gt;sq_cur_blk_addr = 0;</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_168_irrl_idx,</div><div class='del'>-		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,</div><div class='del'>-		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0);</div><div class='del'>-</div><div class='del'>-	page_size = 1 &lt;&lt; (hr_dev-&gt;caps.mtt_buf_pg_sz + PAGE_SHIFT);</div><div class='del'>-	context-&gt;sq_cur_sge_blk_addr =</div><div class='del'>-		       ((ibqp-&gt;qp_type == IB_QPT_GSI) || hr_qp-&gt;sq.max_gs &gt; 2) ?</div><div class='del'>-				      ((u32)(mtts[hr_qp-&gt;sge.offset / page_size]</div><div class='del'>-				      &gt;&gt; PAGE_ADDR_SHIFT)) : 0;</div><div class='del'>-	roce_set_field(context-&gt;byte_184_irrl_idx,</div><div class='del'>-		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,</div><div class='del'>-		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S,</div><div class='del'>-		       ((ibqp-&gt;qp_type == IB_QPT_GSI) || hr_qp-&gt;sq.max_gs &gt; 2) ?</div><div class='del'>-		       (mtts[hr_qp-&gt;sge.offset / page_size] &gt;&gt;</div><div class='del'>-		       (32 + PAGE_ADDR_SHIFT)) : 0);</div><div class='del'>-	qpc_mask-&gt;sq_cur_sge_blk_addr = 0;</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_184_irrl_idx,</div><div class='del'>-		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,</div><div class='del'>-		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0);</div><div class='del'>-</div><div class='del'>-	context-&gt;rx_sq_cur_blk_addr = (u32)(mtts[0] &gt;&gt; PAGE_ADDR_SHIFT);</div><div class='del'>-	roce_set_field(context-&gt;byte_232_irrl_sge,</div><div class='del'>-		       V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,</div><div class='del'>-		       V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S,</div><div class='del'>-		       mtts[0] &gt;&gt; (32 + PAGE_ADDR_SHIFT));</div><div class='del'>-	qpc_mask-&gt;rx_sq_cur_blk_addr = 0;</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_232_irrl_sge,</div><div class='del'>-		       V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,</div><div class='del'>-		       V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0);</div><div class='add'>+	ret = config_qp_sq_buf(hr_dev, hr_qp, context, qpc_mask);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to config sq buf, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Set some fields in context to zero, Because the default values</div><div class='ctx'> 	 * of all fields in context are zero, we need not set them to 0 again.</div><div class='ctx'> 	 * but we should set the relevant fields of context mask to 0.</div><div class='ctx'> 	 */</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_232_irrl_sge,</div><div class='del'>-		       V2_QPC_BYTE_232_IRRL_SGE_IDX_M,</div><div class='del'>-		       V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_IRRL_SGE_IDX);</div><div class='add'>+</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_RX_ACK_MSN);</div><div class='add'>+</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_ACK_LAST_OPTYPE);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_IRRL_PSN_VLD);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_IRRL_PSN);</div><div class='add'>+</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_IRRL_TAIL_REAL);</div><div class='add'>+</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_RETRY_MSG_MSN);</div><div class='add'>+</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_RNR_RETRY_FLAG);</div><div class='ctx'> </div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_240_irrl_tail,</div><div class='del'>-		       V2_QPC_BYTE_240_RX_ACK_MSN_M,</div><div class='del'>-		       V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_CHECK_FLG);</div><div class='ctx'> </div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_248_ack_psn,</div><div class='del'>-		       V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,</div><div class='del'>-		       V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_248_ack_psn,</div><div class='del'>-		     V2_QPC_BYTE_248_IRRL_PSN_VLD_S, 0);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_248_ack_psn,</div><div class='del'>-		       V2_QPC_BYTE_248_IRRL_PSN_M,</div><div class='del'>-		       V2_QPC_BYTE_248_IRRL_PSN_S, 0);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_V2_IRRL_HEAD);</div><div class='ctx'> </div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_240_irrl_tail,</div><div class='del'>-		       V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,</div><div class='del'>-		       V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_220_retry_psn_msn,</div><div class='del'>-		       V2_QPC_BYTE_220_RETRY_MSG_MSN_M,</div><div class='del'>-		       V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);</div><div class='add'>+static int alloc_dip_entry(struct xarray *dip_xa, u32 qpn)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dip *hr_dip;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_248_ack_psn,</div><div class='del'>-		     V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);</div><div class='add'>+	hr_dip = xa_load(dip_xa, qpn);</div><div class='add'>+	if (hr_dip)</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,</div><div class='del'>-		       V2_QPC_BYTE_212_CHECK_FLG_S, 0);</div><div class='add'>+	hr_dip = kzalloc(sizeof(*hr_dip), GFP_KERNEL);</div><div class='add'>+	if (!hr_dip)</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	roce_set_field(context-&gt;byte_212_lsn, V2_QPC_BYTE_212_LSN_M,</div><div class='del'>-		       V2_QPC_BYTE_212_LSN_S, 0x100);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_212_lsn, V2_QPC_BYTE_212_LSN_M,</div><div class='del'>-		       V2_QPC_BYTE_212_LSN_S, 0);</div><div class='add'>+	ret = xa_err(xa_store(dip_xa, qpn, hr_dip, GFP_KERNEL));</div><div class='add'>+	if (ret)</div><div class='add'>+		kfree(hr_dip);</div><div class='ctx'> </div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,</div><div class='del'>-		       V2_QPC_BYTE_196_IRRL_HEAD_S, 0);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr,</div><div class='add'>+			   u32 *dip_idx)</div><div class='add'>+{</div><div class='add'>+	const struct ib_global_route *grh = rdma_ah_read_grh(&amp;attr-&gt;ah_attr);</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='add'>+	struct xarray *dip_xa = &amp;hr_dev-&gt;qp_table.dip_xa;</div><div class='add'>+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='add'>+	struct hns_roce_dip *hr_dip;</div><div class='add'>+	unsigned long idx;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	ret = alloc_dip_entry(dip_xa, ibqp-&gt;qp_num);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	xa_lock(dip_xa);</div><div class='add'>+</div><div class='add'>+	xa_for_each(dip_xa, idx, hr_dip) {</div><div class='add'>+		if (hr_dip-&gt;qp_cnt &amp;&amp;</div><div class='add'>+		    !memcmp(grh-&gt;dgid.raw, hr_dip-&gt;dgid, GID_LEN_V2)) {</div><div class='add'>+			*dip_idx = hr_dip-&gt;dip_idx;</div><div class='add'>+			hr_dip-&gt;qp_cnt++;</div><div class='add'>+			hr_qp-&gt;dip = hr_dip;</div><div class='add'>+			goto out;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* If no dgid is found, a new dip and a mapping between dgid and</div><div class='add'>+	 * dip_idx will be created.</div><div class='add'>+	 */</div><div class='add'>+	xa_for_each(dip_xa, idx, hr_dip) {</div><div class='add'>+		if (hr_dip-&gt;qp_cnt)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		*dip_idx = idx;</div><div class='add'>+		memcpy(hr_dip-&gt;dgid, grh-&gt;dgid.raw, sizeof(grh-&gt;dgid.raw));</div><div class='add'>+		hr_dip-&gt;dip_idx = idx;</div><div class='add'>+		hr_dip-&gt;qp_cnt++;</div><div class='add'>+		hr_qp-&gt;dip = hr_dip;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* This should never happen. */</div><div class='add'>+	if (WARN_ON_ONCE(!hr_qp-&gt;dip))</div><div class='add'>+		ret = -ENOSPC;</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	xa_unlock(dip_xa);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	CONG_DCQCN,</div><div class='add'>+	CONG_WINDOW,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	UNSUPPORT_CONG_LEVEL,</div><div class='add'>+	SUPPORT_CONG_LEVEL,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	CONG_LDCP,</div><div class='add'>+	CONG_HC3,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	DIP_INVALID,</div><div class='add'>+	DIP_VALID,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	WND_LIMIT,</div><div class='add'>+	WND_UNLIMIT,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int check_cong_type(struct ib_qp *ibqp,</div><div class='add'>+			   struct hns_roce_congestion_algorithm *cong_alg)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='add'>+</div><div class='add'>+	/* different congestion types match different configurations */</div><div class='add'>+	switch (hr_qp-&gt;cong_type) {</div><div class='add'>+	case CONG_TYPE_DCQCN:</div><div class='add'>+		cong_alg-&gt;alg_sel = CONG_DCQCN;</div><div class='add'>+		cong_alg-&gt;alg_sub_sel = UNSUPPORT_CONG_LEVEL;</div><div class='add'>+		cong_alg-&gt;dip_vld = DIP_INVALID;</div><div class='add'>+		cong_alg-&gt;wnd_mode_sel = WND_LIMIT;</div><div class='add'>+		break;</div><div class='add'>+	case CONG_TYPE_LDCP:</div><div class='add'>+		cong_alg-&gt;alg_sel = CONG_WINDOW;</div><div class='add'>+		cong_alg-&gt;alg_sub_sel = CONG_LDCP;</div><div class='add'>+		cong_alg-&gt;dip_vld = DIP_INVALID;</div><div class='add'>+		cong_alg-&gt;wnd_mode_sel = WND_UNLIMIT;</div><div class='add'>+		break;</div><div class='add'>+	case CONG_TYPE_HC3:</div><div class='add'>+		cong_alg-&gt;alg_sel = CONG_WINDOW;</div><div class='add'>+		cong_alg-&gt;alg_sub_sel = CONG_HC3;</div><div class='add'>+		cong_alg-&gt;dip_vld = DIP_INVALID;</div><div class='add'>+		cong_alg-&gt;wnd_mode_sel = WND_LIMIT;</div><div class='add'>+		break;</div><div class='add'>+	case CONG_TYPE_DIP:</div><div class='add'>+		cong_alg-&gt;alg_sel = CONG_DCQCN;</div><div class='add'>+		cong_alg-&gt;alg_sub_sel = UNSUPPORT_CONG_LEVEL;</div><div class='add'>+		cong_alg-&gt;dip_vld = DIP_VALID;</div><div class='add'>+		cong_alg-&gt;wnd_mode_sel = WND_LIMIT;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		hr_qp-&gt;cong_type = CONG_TYPE_DCQCN;</div><div class='add'>+		cong_alg-&gt;alg_sel = CONG_DCQCN;</div><div class='add'>+		cong_alg-&gt;alg_sub_sel = UNSUPPORT_CONG_LEVEL;</div><div class='add'>+		cong_alg-&gt;dip_vld = DIP_INVALID;</div><div class='add'>+		cong_alg-&gt;wnd_mode_sel = WND_LIMIT;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline bool hns_roce_v2_check_qp_stat(enum ib_qp_state cur_state,</div><div class='del'>-					     enum ib_qp_state new_state)</div><div class='add'>+static int fill_cong_field(struct ib_qp *ibqp, const struct ib_qp_attr *attr,</div><div class='add'>+			   struct hns_roce_v2_qp_context *context,</div><div class='add'>+			   struct hns_roce_v2_qp_context *qpc_mask)</div><div class='ctx'> {</div><div class='add'>+	const struct ib_global_route *grh = rdma_ah_read_grh(&amp;attr-&gt;ah_attr);</div><div class='add'>+	struct hns_roce_congestion_algorithm cong_field;</div><div class='add'>+	struct ib_device *ibdev = ibqp-&gt;device;</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);</div><div class='add'>+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='add'>+	u32 dip_idx = 0;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	if ((cur_state != IB_QPS_RESET &amp;&amp;</div><div class='del'>-	    (new_state == IB_QPS_ERR || new_state == IB_QPS_RESET)) ||</div><div class='del'>-	    ((cur_state == IB_QPS_RTS || cur_state == IB_QPS_SQD) &amp;&amp;</div><div class='del'>-	    (new_state == IB_QPS_RTS || new_state == IB_QPS_SQD)) ||</div><div class='del'>-	    (cur_state == IB_QPS_SQE &amp;&amp; new_state == IB_QPS_RTS))</div><div class='del'>-		return true;</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08 ||</div><div class='add'>+	    grh-&gt;sgid_attr-&gt;gid_type == IB_GID_TYPE_ROCE)</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-	return false;</div><div class='add'>+	ret = check_cong_type(ibqp, &amp;cong_field);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='add'>+	hr_reg_write(context, QPC_CONG_ALGO_TMPL_ID, hr_dev-&gt;cong_algo_tmpl_id +</div><div class='add'>+		     hr_qp-&gt;cong_type * HNS_ROCE_CONG_SIZE);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_CONG_ALGO_TMPL_ID);</div><div class='add'>+	hr_reg_write(&amp;context-&gt;ext, QPCEX_CONG_ALG_SEL, cong_field.alg_sel);</div><div class='add'>+	hr_reg_clear(&amp;qpc_mask-&gt;ext, QPCEX_CONG_ALG_SEL);</div><div class='add'>+	hr_reg_write(&amp;context-&gt;ext, QPCEX_CONG_ALG_SUB_SEL,</div><div class='add'>+		     cong_field.alg_sub_sel);</div><div class='add'>+	hr_reg_clear(&amp;qpc_mask-&gt;ext, QPCEX_CONG_ALG_SUB_SEL);</div><div class='add'>+	hr_reg_write(&amp;context-&gt;ext, QPCEX_DIP_CTX_IDX_VLD, cong_field.dip_vld);</div><div class='add'>+	hr_reg_clear(&amp;qpc_mask-&gt;ext, QPCEX_DIP_CTX_IDX_VLD);</div><div class='add'>+	hr_reg_write(&amp;context-&gt;ext, QPCEX_SQ_RQ_NOT_FORBID_EN,</div><div class='add'>+		     cong_field.wnd_mode_sel);</div><div class='add'>+	hr_reg_clear(&amp;qpc_mask-&gt;ext, QPCEX_SQ_RQ_NOT_FORBID_EN);</div><div class='add'>+</div><div class='add'>+	/* if dip is disabled, there is no need to set dip idx */</div><div class='add'>+	if (cong_field.dip_vld == 0)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	ret = get_dip_ctx_idx(ibqp, attr, &amp;dip_idx);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to fill cong field, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hr_reg_write(&amp;context-&gt;ext, QPCEX_DIP_CTX_IDX, dip_idx);</div><div class='add'>+	hr_reg_write(&amp;qpc_mask-&gt;ext, QPCEX_DIP_CTX_IDX, 0);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,</div><div class='del'>-				 const struct ib_qp_attr *attr,</div><div class='del'>-				 int attr_mask, enum ib_qp_state cur_state,</div><div class='del'>-				 enum ib_qp_state new_state)</div><div class='add'>+static int hns_roce_hw_v2_get_dscp(struct hns_roce_dev *hr_dev, u8 dscp,</div><div class='add'>+				   u8 *tc_mode, u8 *priority)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	struct hnae3_handle *handle = priv-&gt;handle;</div><div class='add'>+	const struct hnae3_ae_ops *ops = handle-&gt;ae_algo-&gt;ops;</div><div class='add'>+</div><div class='add'>+	if (!ops-&gt;get_dscp_prio)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	return ops-&gt;get_dscp_prio(handle, dscp, tc_mode, priority);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+bool check_sl_valid(struct hns_roce_dev *hr_dev, u8 sl)</div><div class='add'>+{</div><div class='add'>+	u32 max_sl;</div><div class='add'>+</div><div class='add'>+	max_sl = min_t(u32, MAX_SERVICE_LEVEL, hr_dev-&gt;caps.sl_num - 1);</div><div class='add'>+	if (unlikely(sl &gt; max_sl)) {</div><div class='add'>+		ibdev_err_ratelimited(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+				      "failed to set SL(%u). Shouldn't be larger than %u.\n",</div><div class='add'>+				      sl, max_sl);</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_set_sl(struct ib_qp *ibqp,</div><div class='add'>+			   const struct ib_qp_attr *attr,</div><div class='add'>+			   struct hns_roce_v2_qp_context *context,</div><div class='add'>+			   struct hns_roce_v2_qp_context *qpc_mask)</div><div class='ctx'> {</div><div class='add'>+	const struct ib_global_route *grh = rdma_ah_read_grh(&amp;attr-&gt;ah_attr);</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='ctx'> 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='del'>-	struct hns_roce_v2_qp_context *context;</div><div class='del'>-	struct hns_roce_v2_qp_context *qpc_mask;</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	int ret = -EINVAL;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	context = kcalloc(2, sizeof(*context), GFP_ATOMIC);</div><div class='del'>-	if (!context)</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+	ret = hns_roce_hw_v2_get_dscp(hr_dev, get_tclass(&amp;attr-&gt;ah_attr.grh),</div><div class='add'>+				      &amp;hr_qp-&gt;tc_mode, &amp;hr_qp-&gt;priority);</div><div class='add'>+	if (ret &amp;&amp; ret != -EOPNOTSUPP &amp;&amp;</div><div class='add'>+	    grh-&gt;sgid_attr-&gt;gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {</div><div class='add'>+		ibdev_err_ratelimited(ibdev,</div><div class='add'>+				      "failed to get dscp, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (hr_qp-&gt;tc_mode == HNAE3_TC_MAP_MODE_DSCP &amp;&amp;</div><div class='add'>+	    grh-&gt;sgid_attr-&gt;gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)</div><div class='add'>+		hr_qp-&gt;sl = hr_qp-&gt;priority;</div><div class='add'>+	else</div><div class='add'>+		hr_qp-&gt;sl = rdma_ah_get_sl(&amp;attr-&gt;ah_attr);</div><div class='add'>+</div><div class='add'>+	if (!check_sl_valid(hr_dev, hr_qp-&gt;sl))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_SL, hr_qp-&gt;sl);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_SL);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_v2_set_path(struct ib_qp *ibqp,</div><div class='add'>+				const struct ib_qp_attr *attr,</div><div class='add'>+				int attr_mask,</div><div class='add'>+				struct hns_roce_v2_qp_context *context,</div><div class='add'>+				struct hns_roce_v2_qp_context *qpc_mask)</div><div class='add'>+{</div><div class='add'>+	const struct ib_global_route *grh = rdma_ah_read_grh(&amp;attr-&gt;ah_attr);</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='add'>+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	const struct ib_gid_attr *gid_attr = NULL;</div><div class='add'>+	u8 sl = rdma_ah_get_sl(&amp;attr-&gt;ah_attr);</div><div class='add'>+	int is_roce_protocol;</div><div class='add'>+	u16 vlan_id = 0xffff;</div><div class='add'>+	bool is_udp = false;</div><div class='add'>+	u8 ib_port;</div><div class='add'>+	u8 hr_port;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	qpc_mask = context + 1;</div><div class='ctx'> 	/*</div><div class='del'>-	 * In v2 engine, software pass context and context mask to hardware</div><div class='del'>-	 * when modifying qp. If software need modify some fields in context,</div><div class='del'>-	 * we should set all bits of the relevant fields in context mask to</div><div class='del'>-	 * 0 at the same time, else set them to 0x1.</div><div class='add'>+	 * If free_mr_en of qp is set, it means that this qp comes from</div><div class='add'>+	 * free mr. This qp will perform the loopback operation.</div><div class='add'>+	 * In the loopback scenario, only sl needs to be set.</div><div class='ctx'> 	 */</div><div class='del'>-	memset(qpc_mask, 0xff, sizeof(*qpc_mask));</div><div class='add'>+	if (hr_qp-&gt;free_mr_en) {</div><div class='add'>+		if (!check_sl_valid(hr_dev, sl))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		hr_reg_write(context, QPC_SL, sl);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_SL);</div><div class='add'>+		hr_qp-&gt;sl = sl;</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ib_port = (attr_mask &amp; IB_QP_PORT) ? attr-&gt;port_num : hr_qp-&gt;port + 1;</div><div class='add'>+	hr_port = ib_port - 1;</div><div class='add'>+	is_roce_protocol = rdma_cap_eth_ah(&amp;hr_dev-&gt;ib_dev, ib_port) &amp;&amp;</div><div class='add'>+			   rdma_ah_get_ah_flags(&amp;attr-&gt;ah_attr) &amp; IB_AH_GRH;</div><div class='add'>+</div><div class='add'>+	if (is_roce_protocol) {</div><div class='add'>+		gid_attr = attr-&gt;ah_attr.grh.sgid_attr;</div><div class='add'>+		ret = rdma_read_gid_l2_fields(gid_attr, &amp;vlan_id, NULL);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+</div><div class='add'>+		is_udp = (gid_attr-&gt;gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Only HIP08 needs to set the vlan_en bits in QPC */</div><div class='add'>+	if (vlan_id &lt; VLAN_N_VID &amp;&amp;</div><div class='add'>+	    hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08) {</div><div class='add'>+		hr_reg_enable(context, QPC_RQ_VLAN_EN);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_RQ_VLAN_EN);</div><div class='add'>+		hr_reg_enable(context, QPC_SQ_VLAN_EN);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_SQ_VLAN_EN);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_VLAN_ID, vlan_id);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_VLAN_ID);</div><div class='add'>+</div><div class='add'>+	if (grh-&gt;sgid_index &gt;= hr_dev-&gt;caps.gid_table_len[hr_port]) {</div><div class='add'>+		ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n",</div><div class='add'>+			  grh-&gt;sgid_index, hr_dev-&gt;caps.gid_table_len[hr_port]);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {</div><div class='add'>+		ibdev_err(ibdev, "ah attr is not RDMA roce type\n");</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_UDPSPN,</div><div class='add'>+		     is_udp ? rdma_get_udp_sport(grh-&gt;flow_label, ibqp-&gt;qp_num,</div><div class='add'>+						 attr-&gt;dest_qp_num) :</div><div class='add'>+				    0);</div><div class='add'>+</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_UDPSPN);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_GMV_IDX, grh-&gt;sgid_index);</div><div class='add'>+</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_GMV_IDX);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_HOPLIMIT, grh-&gt;hop_limit);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_HOPLIMIT);</div><div class='add'>+</div><div class='add'>+	ret = fill_cong_field(ibqp, attr, context, qpc_mask);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_TC, get_tclass(&amp;attr-&gt;ah_attr.grh));</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_TC);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(context, QPC_FL, grh-&gt;flow_label);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_FL);</div><div class='add'>+	memcpy(context-&gt;dgid, grh-&gt;dgid.raw, sizeof(grh-&gt;dgid.raw));</div><div class='add'>+	memset(qpc_mask-&gt;dgid, 0, sizeof(grh-&gt;dgid.raw));</div><div class='add'>+</div><div class='add'>+	return  hns_roce_set_sl(ibqp, attr, context, qpc_mask);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool check_qp_state(enum ib_qp_state cur_state,</div><div class='add'>+			   enum ib_qp_state new_state)</div><div class='add'>+{</div><div class='add'>+	static const bool sm[][IB_QPS_ERR + 1] = {</div><div class='add'>+		[IB_QPS_RESET] = { [IB_QPS_RESET] = true,</div><div class='add'>+				   [IB_QPS_INIT] = true },</div><div class='add'>+		[IB_QPS_INIT] = { [IB_QPS_RESET] = true,</div><div class='add'>+				  [IB_QPS_INIT] = true,</div><div class='add'>+				  [IB_QPS_RTR] = true,</div><div class='add'>+				  [IB_QPS_ERR] = true },</div><div class='add'>+		[IB_QPS_RTR] = { [IB_QPS_RESET] = true,</div><div class='add'>+				 [IB_QPS_RTS] = true,</div><div class='add'>+				 [IB_QPS_ERR] = true },</div><div class='add'>+		[IB_QPS_RTS] = { [IB_QPS_RESET] = true,</div><div class='add'>+				 [IB_QPS_RTS] = true,</div><div class='add'>+				 [IB_QPS_ERR] = true },</div><div class='add'>+		[IB_QPS_SQD] = {},</div><div class='add'>+		[IB_QPS_SQE] = {},</div><div class='add'>+		[IB_QPS_ERR] = { [IB_QPS_RESET] = true,</div><div class='add'>+				 [IB_QPS_ERR] = true }</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	return sm[cur_state][new_state];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp,</div><div class='add'>+				      const struct ib_qp_attr *attr,</div><div class='add'>+				      int attr_mask,</div><div class='add'>+				      enum ib_qp_state cur_state,</div><div class='add'>+				      enum ib_qp_state new_state,</div><div class='add'>+				      struct hns_roce_v2_qp_context *context,</div><div class='add'>+				      struct hns_roce_v2_qp_context *qpc_mask,</div><div class='add'>+				      struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	if (!check_qp_state(cur_state, new_state))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='ctx'> 	if (cur_state == IB_QPS_RESET &amp;&amp; new_state == IB_QPS_INIT) {</div><div class='del'>-		memset(qpc_mask, 0, sizeof(*qpc_mask));</div><div class='del'>-		modify_qp_reset_to_init(ibqp, attr, attr_mask, context,</div><div class='del'>-					qpc_mask);</div><div class='add'>+		memset(qpc_mask, 0, hr_dev-&gt;caps.qpc_sz);</div><div class='add'>+		modify_qp_reset_to_init(ibqp, context);</div><div class='ctx'> 	} else if (cur_state == IB_QPS_INIT &amp;&amp; new_state == IB_QPS_INIT) {</div><div class='del'>-		modify_qp_init_to_init(ibqp, attr, attr_mask, context,</div><div class='del'>-				       qpc_mask);</div><div class='add'>+		modify_qp_init_to_init(ibqp, context, qpc_mask);</div><div class='ctx'> 	} else if (cur_state == IB_QPS_INIT &amp;&amp; new_state == IB_QPS_RTR) {</div><div class='ctx'> 		ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,</div><div class='del'>-					    qpc_mask);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto out;</div><div class='add'>+					    qpc_mask, udata);</div><div class='ctx'> 	} else if (cur_state == IB_QPS_RTR &amp;&amp; new_state == IB_QPS_RTS) {</div><div class='del'>-		ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context,</div><div class='del'>-					   qpc_mask);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto out;</div><div class='del'>-	} else if (hns_roce_v2_check_qp_stat(cur_state, new_state)) {</div><div class='del'>-		/* Nothing */</div><div class='del'>-		;</div><div class='del'>-	} else {</div><div class='del'>-		dev_err(dev, "Illegal state for QP!\n");</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-		goto out;</div><div class='add'>+		ret = modify_qp_rtr_to_rts(ibqp, attr_mask, context, qpc_mask);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* When QP state is err, SQ and RQ WQE should be flushed */</div><div class='del'>-	if (new_state == IB_QPS_ERR) {</div><div class='del'>-		roce_set_field(context-&gt;byte_160_sq_ci_pi,</div><div class='del'>-			       V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,</div><div class='del'>-			       V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S,</div><div class='del'>-			       hr_qp-&gt;sq.head);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_160_sq_ci_pi,</div><div class='del'>-			       V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,</div><div class='del'>-			       V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);</div><div class='del'>-</div><div class='del'>-		if (!ibqp-&gt;srq) {</div><div class='del'>-			roce_set_field(context-&gt;byte_84_rq_ci_pi,</div><div class='del'>-			       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,</div><div class='del'>-			       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S,</div><div class='del'>-			       hr_qp-&gt;rq.head);</div><div class='del'>-			roce_set_field(qpc_mask-&gt;byte_84_rq_ci_pi,</div><div class='del'>-			       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,</div><div class='del'>-			       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (attr_mask &amp; IB_QP_AV) {</div><div class='del'>-		const struct ib_global_route *grh =</div><div class='del'>-					    rdma_ah_read_grh(&amp;attr-&gt;ah_attr);</div><div class='del'>-		const struct ib_gid_attr *gid_attr = NULL;</div><div class='del'>-		int is_roce_protocol;</div><div class='del'>-		u16 vlan = 0xffff;</div><div class='del'>-		u8 ib_port;</div><div class='del'>-		u8 hr_port;</div><div class='del'>-</div><div class='del'>-		ib_port = (attr_mask &amp; IB_QP_PORT) ? attr-&gt;port_num :</div><div class='del'>-			   hr_qp-&gt;port + 1;</div><div class='del'>-		hr_port = ib_port - 1;</div><div class='del'>-		is_roce_protocol = rdma_cap_eth_ah(&amp;hr_dev-&gt;ib_dev, ib_port) &amp;&amp;</div><div class='del'>-			       rdma_ah_get_ah_flags(&amp;attr-&gt;ah_attr) &amp; IB_AH_GRH;</div><div class='del'>-</div><div class='del'>-		if (is_roce_protocol) {</div><div class='del'>-			gid_attr = attr-&gt;ah_attr.grh.sgid_attr;</div><div class='del'>-			ret = rdma_read_gid_l2_fields(gid_attr, &amp;vlan, NULL);</div><div class='del'>-			if (ret)</div><div class='del'>-				goto out;</div><div class='add'>+static bool check_qp_timeout_cfg_range(struct hns_roce_dev *hr_dev, u8 *timeout)</div><div class='add'>+{</div><div class='add'>+#define QP_ACK_TIMEOUT_MAX_HIP08 20</div><div class='add'>+#define QP_ACK_TIMEOUT_MAX 31</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08) {</div><div class='add'>+		if (*timeout &gt; QP_ACK_TIMEOUT_MAX_HIP08) {</div><div class='add'>+			ibdev_warn(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+				   "local ACK timeout shall be 0 to 20.\n");</div><div class='add'>+			return false;</div><div class='ctx'> 		}</div><div class='del'>-</div><div class='del'>-		if (vlan &lt; VLAN_CFI_MASK) {</div><div class='del'>-			roce_set_bit(context-&gt;byte_76_srqn_op_en,</div><div class='del'>-				     V2_QPC_BYTE_76_RQ_VLAN_EN_S, 1);</div><div class='del'>-			roce_set_bit(qpc_mask-&gt;byte_76_srqn_op_en,</div><div class='del'>-				     V2_QPC_BYTE_76_RQ_VLAN_EN_S, 0);</div><div class='del'>-			roce_set_bit(context-&gt;byte_168_irrl_idx,</div><div class='del'>-				     V2_QPC_BYTE_168_SQ_VLAN_EN_S, 1);</div><div class='del'>-			roce_set_bit(qpc_mask-&gt;byte_168_irrl_idx,</div><div class='del'>-				     V2_QPC_BYTE_168_SQ_VLAN_EN_S, 0);</div><div class='add'>+		*timeout += HNS_ROCE_V2_QP_ACK_TIMEOUT_OFS_HIP08;</div><div class='add'>+	} else if (hr_dev-&gt;pci_dev-&gt;revision &gt; PCI_REVISION_ID_HIP08) {</div><div class='add'>+		if (*timeout &gt; QP_ACK_TIMEOUT_MAX) {</div><div class='add'>+			ibdev_warn(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+				   "local ACK timeout shall be 0 to 31.\n");</div><div class='add'>+			return false;</div><div class='ctx'> 		}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		roce_set_field(context-&gt;byte_24_mtu_tc,</div><div class='del'>-			       V2_QPC_BYTE_24_VLAN_ID_M,</div><div class='del'>-			       V2_QPC_BYTE_24_VLAN_ID_S, vlan);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_24_mtu_tc,</div><div class='del'>-			       V2_QPC_BYTE_24_VLAN_ID_M,</div><div class='del'>-			       V2_QPC_BYTE_24_VLAN_ID_S, 0);</div><div class='del'>-</div><div class='del'>-		if (grh-&gt;sgid_index &gt;= hr_dev-&gt;caps.gid_table_len[hr_port]) {</div><div class='del'>-			dev_err(hr_dev-&gt;dev,</div><div class='del'>-				"sgid_index(%u) too large. max is %d\n",</div><div class='del'>-				grh-&gt;sgid_index,</div><div class='del'>-				hr_dev-&gt;caps.gid_table_len[hr_port]);</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		if (attr-&gt;ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {</div><div class='del'>-			dev_err(hr_dev-&gt;dev, "ah attr is not RDMA roce type\n");</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='add'>+static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp,</div><div class='add'>+				      const struct ib_qp_attr *attr,</div><div class='add'>+				      int attr_mask,</div><div class='add'>+				      struct hns_roce_v2_qp_context *context,</div><div class='add'>+				      struct hns_roce_v2_qp_context *qpc_mask)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='add'>+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='add'>+	int ret = 0;</div><div class='add'>+	u8 timeout;</div><div class='ctx'> </div><div class='del'>-		roce_set_field(context-&gt;byte_52_udpspn_dmac,</div><div class='del'>-			   V2_QPC_BYTE_52_UDPSPN_M, V2_QPC_BYTE_52_UDPSPN_S,</div><div class='del'>-			   (gid_attr-&gt;gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP) ?</div><div class='del'>-			   0 : 0x12b7);</div><div class='del'>-</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_52_udpspn_dmac,</div><div class='del'>-			       V2_QPC_BYTE_52_UDPSPN_M,</div><div class='del'>-			       V2_QPC_BYTE_52_UDPSPN_S, 0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-			       V2_QPC_BYTE_20_SGID_IDX_M,</div><div class='del'>-			       V2_QPC_BYTE_20_SGID_IDX_S, grh-&gt;sgid_index);</div><div class='del'>-</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-			       V2_QPC_BYTE_20_SGID_IDX_M,</div><div class='del'>-			       V2_QPC_BYTE_20_SGID_IDX_S, 0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;byte_24_mtu_tc,</div><div class='del'>-			       V2_QPC_BYTE_24_HOP_LIMIT_M,</div><div class='del'>-			       V2_QPC_BYTE_24_HOP_LIMIT_S, grh-&gt;hop_limit);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_24_mtu_tc,</div><div class='del'>-			       V2_QPC_BYTE_24_HOP_LIMIT_M,</div><div class='del'>-			       V2_QPC_BYTE_24_HOP_LIMIT_S, 0);</div><div class='del'>-</div><div class='del'>-		if (hr_dev-&gt;pci_dev-&gt;revision == 0x21 &amp;&amp;</div><div class='del'>-		    gid_attr-&gt;gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)</div><div class='del'>-			roce_set_field(context-&gt;byte_24_mtu_tc,</div><div class='del'>-				       V2_QPC_BYTE_24_TC_M, V2_QPC_BYTE_24_TC_S,</div><div class='del'>-				       grh-&gt;traffic_class &gt;&gt; 2);</div><div class='del'>-		else</div><div class='del'>-			roce_set_field(context-&gt;byte_24_mtu_tc,</div><div class='del'>-				       V2_QPC_BYTE_24_TC_M, V2_QPC_BYTE_24_TC_S,</div><div class='del'>-				       grh-&gt;traffic_class);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,</div><div class='del'>-			       V2_QPC_BYTE_24_TC_S, 0);</div><div class='del'>-		roce_set_field(context-&gt;byte_28_at_fl, V2_QPC_BYTE_28_FL_M,</div><div class='del'>-			       V2_QPC_BYTE_28_FL_S, grh-&gt;flow_label);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_28_at_fl, V2_QPC_BYTE_28_FL_M,</div><div class='del'>-			       V2_QPC_BYTE_28_FL_S, 0);</div><div class='del'>-		memcpy(context-&gt;dgid, grh-&gt;dgid.raw, sizeof(grh-&gt;dgid.raw));</div><div class='del'>-		memset(qpc_mask-&gt;dgid, 0, sizeof(grh-&gt;dgid.raw));</div><div class='del'>-		roce_set_field(context-&gt;byte_28_at_fl, V2_QPC_BYTE_28_SL_M,</div><div class='del'>-			       V2_QPC_BYTE_28_SL_S,</div><div class='del'>-			       rdma_ah_get_sl(&amp;attr-&gt;ah_attr));</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_28_at_fl, V2_QPC_BYTE_28_SL_M,</div><div class='del'>-			       V2_QPC_BYTE_28_SL_S, 0);</div><div class='del'>-		hr_qp-&gt;sl = rdma_ah_get_sl(&amp;attr-&gt;ah_attr);</div><div class='add'>+	if (attr_mask &amp; IB_QP_AV) {</div><div class='add'>+		ret = hns_roce_v2_set_path(ibqp, attr, attr_mask, context,</div><div class='add'>+					   qpc_mask);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_TIMEOUT) {</div><div class='del'>-		if (attr-&gt;timeout &lt; 31) {</div><div class='del'>-			roce_set_field(context-&gt;byte_28_at_fl,</div><div class='del'>-				       V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,</div><div class='del'>-				       attr-&gt;timeout);</div><div class='del'>-			roce_set_field(qpc_mask-&gt;byte_28_at_fl,</div><div class='del'>-				       V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,</div><div class='del'>-				       0);</div><div class='del'>-		} else {</div><div class='del'>-			dev_warn(dev, "Local ACK timeout shall be 0 to 30.\n");</div><div class='add'>+		timeout = attr-&gt;timeout;</div><div class='add'>+		if (check_qp_timeout_cfg_range(hr_dev, &amp;timeout)) {</div><div class='add'>+			hr_reg_write(context, QPC_AT, timeout);</div><div class='add'>+			hr_reg_clear(qpc_mask, QPC_AT);</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_RETRY_CNT) {</div><div class='del'>-		roce_set_field(context-&gt;byte_212_lsn,</div><div class='del'>-			       V2_QPC_BYTE_212_RETRY_NUM_INIT_M,</div><div class='del'>-			       V2_QPC_BYTE_212_RETRY_NUM_INIT_S,</div><div class='del'>-			       attr-&gt;retry_cnt);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_212_lsn,</div><div class='del'>-			       V2_QPC_BYTE_212_RETRY_NUM_INIT_M,</div><div class='del'>-			       V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;byte_212_lsn,</div><div class='del'>-			       V2_QPC_BYTE_212_RETRY_CNT_M,</div><div class='del'>-			       V2_QPC_BYTE_212_RETRY_CNT_S,</div><div class='del'>-			       attr-&gt;retry_cnt);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_212_lsn,</div><div class='del'>-			       V2_QPC_BYTE_212_RETRY_CNT_M,</div><div class='del'>-			       V2_QPC_BYTE_212_RETRY_CNT_S, 0);</div><div class='add'>+		hr_reg_write(context, QPC_RETRY_NUM_INIT, attr-&gt;retry_cnt);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_RETRY_NUM_INIT);</div><div class='add'>+</div><div class='add'>+		hr_reg_write(context, QPC_RETRY_CNT, attr-&gt;retry_cnt);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_RETRY_CNT);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_RNR_RETRY) {</div><div class='del'>-		roce_set_field(context-&gt;byte_244_rnr_rxack,</div><div class='del'>-			       V2_QPC_BYTE_244_RNR_NUM_INIT_M,</div><div class='del'>-			       V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr-&gt;rnr_retry);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_244_rnr_rxack,</div><div class='del'>-			       V2_QPC_BYTE_244_RNR_NUM_INIT_M,</div><div class='del'>-			       V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0);</div><div class='add'>+		hr_reg_write(context, QPC_RNR_NUM_INIT, attr-&gt;rnr_retry);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_RNR_NUM_INIT);</div><div class='ctx'> </div><div class='del'>-		roce_set_field(context-&gt;byte_244_rnr_rxack,</div><div class='del'>-			       V2_QPC_BYTE_244_RNR_CNT_M,</div><div class='del'>-			       V2_QPC_BYTE_244_RNR_CNT_S, attr-&gt;rnr_retry);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_244_rnr_rxack,</div><div class='del'>-			       V2_QPC_BYTE_244_RNR_CNT_M,</div><div class='del'>-			       V2_QPC_BYTE_244_RNR_CNT_S, 0);</div><div class='add'>+		hr_reg_write(context, QPC_RNR_CNT, attr-&gt;rnr_retry);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_RNR_CNT);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_SQ_PSN) {</div><div class='del'>-		roce_set_field(context-&gt;byte_172_sq_psn,</div><div class='del'>-			       V2_QPC_BYTE_172_SQ_CUR_PSN_M,</div><div class='del'>-			       V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr-&gt;sq_psn);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_172_sq_psn,</div><div class='del'>-			       V2_QPC_BYTE_172_SQ_CUR_PSN_M,</div><div class='del'>-			       V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;byte_196_sq_psn,</div><div class='del'>-			       V2_QPC_BYTE_196_SQ_MAX_PSN_M,</div><div class='del'>-			       V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr-&gt;sq_psn);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_196_sq_psn,</div><div class='del'>-			       V2_QPC_BYTE_196_SQ_MAX_PSN_M,</div><div class='del'>-			       V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;byte_220_retry_psn_msn,</div><div class='del'>-			       V2_QPC_BYTE_220_RETRY_MSG_PSN_M,</div><div class='del'>-			       V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr-&gt;sq_psn);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_220_retry_psn_msn,</div><div class='del'>-			       V2_QPC_BYTE_220_RETRY_MSG_PSN_M,</div><div class='del'>-			       V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;byte_224_retry_msg,</div><div class='del'>-			       V2_QPC_BYTE_224_RETRY_MSG_PSN_M,</div><div class='del'>-			       V2_QPC_BYTE_224_RETRY_MSG_PSN_S,</div><div class='del'>-			       attr-&gt;sq_psn &gt;&gt; 16);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_224_retry_msg,</div><div class='del'>-			       V2_QPC_BYTE_224_RETRY_MSG_PSN_M,</div><div class='del'>-			       V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;byte_224_retry_msg,</div><div class='del'>-			       V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,</div><div class='del'>-			       V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S,</div><div class='del'>-			       attr-&gt;sq_psn);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_224_retry_msg,</div><div class='del'>-			       V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,</div><div class='del'>-			       V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0);</div><div class='del'>-</div><div class='del'>-		roce_set_field(context-&gt;byte_244_rnr_rxack,</div><div class='del'>-			       V2_QPC_BYTE_244_RX_ACK_EPSN_M,</div><div class='del'>-			       V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr-&gt;sq_psn);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_244_rnr_rxack,</div><div class='del'>-			       V2_QPC_BYTE_244_RX_ACK_EPSN_M,</div><div class='del'>-			       V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0);</div><div class='add'>+		hr_reg_write(context, QPC_SQ_CUR_PSN, attr-&gt;sq_psn);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_SQ_CUR_PSN);</div><div class='add'>+</div><div class='add'>+		hr_reg_write(context, QPC_SQ_MAX_PSN, attr-&gt;sq_psn);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_SQ_MAX_PSN);</div><div class='add'>+</div><div class='add'>+		hr_reg_write(context, QPC_RETRY_MSG_PSN_L, attr-&gt;sq_psn);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_RETRY_MSG_PSN_L);</div><div class='add'>+</div><div class='add'>+		hr_reg_write(context, QPC_RETRY_MSG_PSN_H,</div><div class='add'>+			     attr-&gt;sq_psn &gt;&gt; RETRY_MSG_PSN_SHIFT);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_RETRY_MSG_PSN_H);</div><div class='add'>+</div><div class='add'>+		hr_reg_write(context, QPC_RETRY_MSG_FPKT_PSN, attr-&gt;sq_psn);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_RETRY_MSG_FPKT_PSN);</div><div class='add'>+</div><div class='add'>+		hr_reg_write(context, QPC_RX_ACK_EPSN, attr-&gt;sq_psn);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_RX_ACK_EPSN);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if ((attr_mask &amp; IB_QP_MAX_DEST_RD_ATOMIC) &amp;&amp;</div><div class='ctx'> 	     attr-&gt;max_dest_rd_atomic) {</div><div class='del'>-		roce_set_field(context-&gt;byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,</div><div class='del'>-			       V2_QPC_BYTE_140_RR_MAX_S,</div><div class='del'>-			       fls(attr-&gt;max_dest_rd_atomic - 1));</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,</div><div class='del'>-			       V2_QPC_BYTE_140_RR_MAX_S, 0);</div><div class='add'>+		hr_reg_write(context, QPC_RR_MAX,</div><div class='add'>+			     fls(attr-&gt;max_dest_rd_atomic - 1));</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_RR_MAX);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if ((attr_mask &amp; IB_QP_MAX_QP_RD_ATOMIC) &amp;&amp; attr-&gt;max_rd_atomic) {</div><div class='del'>-		roce_set_field(context-&gt;byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M,</div><div class='del'>-			       V2_QPC_BYTE_208_SR_MAX_S,</div><div class='del'>-			       fls(attr-&gt;max_rd_atomic - 1));</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_208_irrl,</div><div class='del'>-			       V2_QPC_BYTE_208_SR_MAX_M,</div><div class='del'>-			       V2_QPC_BYTE_208_SR_MAX_S, 0);</div><div class='add'>+		hr_reg_write(context, QPC_SR_MAX, fls(attr-&gt;max_rd_atomic - 1));</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_SR_MAX);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))</div><div class='ctx'> 		set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_MIN_RNR_TIMER) {</div><div class='del'>-		roce_set_field(context-&gt;byte_80_rnr_rx_cqn,</div><div class='del'>-			       V2_QPC_BYTE_80_MIN_RNR_TIME_M,</div><div class='del'>-			       V2_QPC_BYTE_80_MIN_RNR_TIME_S,</div><div class='del'>-			       attr-&gt;min_rnr_timer);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_80_rnr_rx_cqn,</div><div class='del'>-			       V2_QPC_BYTE_80_MIN_RNR_TIME_M,</div><div class='del'>-			       V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0);</div><div class='add'>+		hr_reg_write(context, QPC_MIN_RNR_TIME,</div><div class='add'>+			    hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08 ?</div><div class='add'>+			    HNS_ROCE_RNR_TIMER_10NS : attr-&gt;min_rnr_timer);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_MIN_RNR_TIME);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* RC&amp;UC required attr */</div><div class='ctx'> 	if (attr_mask &amp; IB_QP_RQ_PSN) {</div><div class='del'>-		roce_set_field(context-&gt;byte_108_rx_reqepsn,</div><div class='del'>-			       V2_QPC_BYTE_108_RX_REQ_EPSN_M,</div><div class='del'>-			       V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr-&gt;rq_psn);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_108_rx_reqepsn,</div><div class='del'>-			       V2_QPC_BYTE_108_RX_REQ_EPSN_M,</div><div class='del'>-			       V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0);</div><div class='add'>+		hr_reg_write(context, QPC_RX_REQ_EPSN, attr-&gt;rq_psn);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_RX_REQ_EPSN);</div><div class='ctx'> </div><div class='del'>-		roce_set_field(context-&gt;byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,</div><div class='del'>-			       V2_QPC_BYTE_152_RAQ_PSN_S, attr-&gt;rq_psn - 1);</div><div class='del'>-		roce_set_field(qpc_mask-&gt;byte_152_raq,</div><div class='del'>-			       V2_QPC_BYTE_152_RAQ_PSN_M,</div><div class='del'>-			       V2_QPC_BYTE_152_RAQ_PSN_S, 0);</div><div class='add'>+		hr_reg_write(context, QPC_RAQ_PSN, attr-&gt;rq_psn - 1);</div><div class='add'>+		hr_reg_clear(qpc_mask, QPC_RAQ_PSN);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_QKEY) {</div><div class='del'>-		context-&gt;qkey_xrcd = attr-&gt;qkey;</div><div class='add'>+		context-&gt;qkey_xrcd = cpu_to_le32(attr-&gt;qkey);</div><div class='ctx'> 		qpc_mask-&gt;qkey_xrcd = 0;</div><div class='ctx'> 		hr_qp-&gt;qkey = attr-&gt;qkey;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	roce_set_bit(context-&gt;byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S,</div><div class='del'>-		     ibqp-&gt;srq ? 1 : 0);</div><div class='del'>-	roce_set_bit(qpc_mask-&gt;byte_108_rx_reqepsn,</div><div class='del'>-		     V2_QPC_BYTE_108_INV_CREDIT_S, 0);</div><div class='del'>-</div><div class='del'>-	/* Every status migrate must change state */</div><div class='del'>-	roce_set_field(context-&gt;byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,</div><div class='del'>-		       V2_QPC_BYTE_60_QP_ST_S, new_state);</div><div class='del'>-	roce_set_field(qpc_mask-&gt;byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,</div><div class='del'>-		       V2_QPC_BYTE_60_QP_ST_S, 0);</div><div class='del'>-</div><div class='del'>-	/* SW pass context to HW */</div><div class='del'>-	ret = hns_roce_v2_qp_modify(hr_dev, &amp;hr_qp-&gt;mtt, cur_state, new_state,</div><div class='del'>-				    context, hr_qp);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "hns_roce_qp_modify failed(%d)\n", ret);</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	hr_qp-&gt;state = new_state;</div><div class='add'>+static void hns_roce_v2_record_opt_fields(struct ib_qp *ibqp,</div><div class='add'>+					  const struct ib_qp_attr *attr,</div><div class='add'>+					  int attr_mask)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='add'>+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_ACCESS_FLAGS)</div><div class='ctx'> 		hr_qp-&gt;atomic_rd_en = attr-&gt;qp_access_flags;</div><div class='hunk'>@@ -4248,47 +5372,149 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,</div><div class='ctx'> 		hr_qp-&gt;port = attr-&gt;port_num - 1;</div><div class='ctx'> 		hr_qp-&gt;phy_port = hr_dev-&gt;iboe.phy_port[hr_qp-&gt;port];</div><div class='ctx'> 	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void clear_qp(struct hns_roce_qp *hr_qp)</div><div class='add'>+{</div><div class='add'>+	struct ib_qp *ibqp = &amp;hr_qp-&gt;ibqp;</div><div class='ctx'> </div><div class='del'>-	if (new_state == IB_QPS_RESET &amp;&amp; !ibqp-&gt;uobject) {</div><div class='del'>-		hns_roce_v2_cq_clean(to_hr_cq(ibqp-&gt;recv_cq), hr_qp-&gt;qpn,</div><div class='del'>-				     ibqp-&gt;srq ? to_hr_srq(ibqp-&gt;srq) : NULL);</div><div class='del'>-		if (ibqp-&gt;send_cq != ibqp-&gt;recv_cq)</div><div class='del'>-			hns_roce_v2_cq_clean(to_hr_cq(ibqp-&gt;send_cq),</div><div class='del'>-					     hr_qp-&gt;qpn, NULL);</div><div class='add'>+	if (ibqp-&gt;send_cq)</div><div class='add'>+		hns_roce_v2_cq_clean(to_hr_cq(ibqp-&gt;send_cq),</div><div class='add'>+				     hr_qp-&gt;qpn, NULL);</div><div class='ctx'> </div><div class='del'>-		hr_qp-&gt;rq.head = 0;</div><div class='del'>-		hr_qp-&gt;rq.tail = 0;</div><div class='del'>-		hr_qp-&gt;sq.head = 0;</div><div class='del'>-		hr_qp-&gt;sq.tail = 0;</div><div class='del'>-		hr_qp-&gt;sq_next_wqe = 0;</div><div class='del'>-		hr_qp-&gt;next_sge = 0;</div><div class='del'>-		if (hr_qp-&gt;rq.wqe_cnt)</div><div class='del'>-			*hr_qp-&gt;rdb.db_record = 0;</div><div class='add'>+	if (ibqp-&gt;recv_cq  &amp;&amp; ibqp-&gt;recv_cq != ibqp-&gt;send_cq)</div><div class='add'>+		hns_roce_v2_cq_clean(to_hr_cq(ibqp-&gt;recv_cq),</div><div class='add'>+				     hr_qp-&gt;qpn, ibqp-&gt;srq ?</div><div class='add'>+				     to_hr_srq(ibqp-&gt;srq) : NULL);</div><div class='add'>+</div><div class='add'>+	if (hr_qp-&gt;en_flags &amp; HNS_ROCE_QP_CAP_RQ_RECORD_DB)</div><div class='add'>+		*hr_qp-&gt;rdb.db_record = 0;</div><div class='add'>+</div><div class='add'>+	hr_qp-&gt;rq.head = 0;</div><div class='add'>+	hr_qp-&gt;rq.tail = 0;</div><div class='add'>+	hr_qp-&gt;sq.head = 0;</div><div class='add'>+	hr_qp-&gt;sq.tail = 0;</div><div class='add'>+	hr_qp-&gt;next_sge = 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void v2_set_flushed_fields(struct ib_qp *ibqp,</div><div class='add'>+				  struct hns_roce_v2_qp_context *context,</div><div class='add'>+				  struct hns_roce_v2_qp_context *qpc_mask)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='add'>+	unsigned long sq_flag = 0;</div><div class='add'>+	unsigned long rq_flag = 0;</div><div class='add'>+</div><div class='add'>+	if (ibqp-&gt;qp_type == IB_QPT_XRC_TGT)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;hr_qp-&gt;sq.lock, sq_flag);</div><div class='add'>+	trace_hns_sq_flush_cqe(hr_qp-&gt;qpn, hr_qp-&gt;sq.head, TRACE_SQ);</div><div class='add'>+	hr_reg_write(context, QPC_SQ_PRODUCER_IDX, hr_qp-&gt;sq.head);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_SQ_PRODUCER_IDX);</div><div class='add'>+	hr_qp-&gt;state = IB_QPS_ERR;</div><div class='add'>+	spin_unlock_irqrestore(&amp;hr_qp-&gt;sq.lock, sq_flag);</div><div class='add'>+</div><div class='add'>+	if (ibqp-&gt;srq || ibqp-&gt;qp_type == IB_QPT_XRC_INI) /* no RQ */</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;hr_qp-&gt;rq.lock, rq_flag);</div><div class='add'>+	trace_hns_rq_flush_cqe(hr_qp-&gt;qpn, hr_qp-&gt;rq.head, TRACE_RQ);</div><div class='add'>+	hr_reg_write(context, QPC_RQ_PRODUCER_IDX, hr_qp-&gt;rq.head);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_RQ_PRODUCER_IDX);</div><div class='add'>+	spin_unlock_irqrestore(&amp;hr_qp-&gt;rq.lock, rq_flag);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,</div><div class='add'>+				 const struct ib_qp_attr *attr,</div><div class='add'>+				 int attr_mask, enum ib_qp_state cur_state,</div><div class='add'>+				 enum ib_qp_state new_state, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='add'>+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='add'>+	struct hns_roce_v2_qp_context *context;</div><div class='add'>+	struct hns_roce_v2_qp_context *qpc_mask;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	int ret = -ENOMEM;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; ~IB_QP_ATTR_STANDARD_BITS)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * In v2 engine, software pass context and context mask to hardware</div><div class='add'>+	 * when modifying qp. If software need modify some fields in context,</div><div class='add'>+	 * we should set all bits of the relevant fields in context mask to</div><div class='add'>+	 * 0 at the same time, else set them to 0x1.</div><div class='add'>+	 */</div><div class='add'>+	context = kvzalloc(sizeof(*context), GFP_KERNEL);</div><div class='add'>+	qpc_mask = kvzalloc(sizeof(*qpc_mask), GFP_KERNEL);</div><div class='add'>+	if (!context || !qpc_mask)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	memset(qpc_mask, 0xff, hr_dev-&gt;caps.qpc_sz);</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state,</div><div class='add'>+					 new_state, context, qpc_mask, udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	/* When QP state is err, SQ and RQ WQE should be flushed */</div><div class='add'>+	if (new_state == IB_QPS_ERR)</div><div class='add'>+		v2_set_flushed_fields(ibqp, context, qpc_mask);</div><div class='add'>+</div><div class='add'>+	/* Configure the optional fields */</div><div class='add'>+	ret = hns_roce_v2_set_opt_fields(ibqp, attr, attr_mask, context,</div><div class='add'>+					 qpc_mask);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	hr_reg_write_bool(context, QPC_INV_CREDIT,</div><div class='add'>+			  to_hr_qp_type(hr_qp-&gt;ibqp.qp_type) == SERV_TYPE_XRC ||</div><div class='add'>+			  ibqp-&gt;srq);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_INV_CREDIT);</div><div class='add'>+</div><div class='add'>+	/* Every status migrate must change state */</div><div class='add'>+	hr_reg_write(context, QPC_QP_ST, new_state);</div><div class='add'>+	hr_reg_clear(qpc_mask, QPC_QP_ST);</div><div class='add'>+</div><div class='add'>+	/* SW pass context to HW */</div><div class='add'>+	ret = hns_roce_v2_qp_modify(hr_dev, context, qpc_mask, hr_qp);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err_ratelimited(ibdev, "failed to modify QP, ret = %d.\n", ret);</div><div class='add'>+		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	hr_qp-&gt;state = new_state;</div><div class='add'>+</div><div class='add'>+	hns_roce_v2_record_opt_fields(ibqp, attr, attr_mask);</div><div class='add'>+</div><div class='add'>+	if (new_state == IB_QPS_RESET &amp;&amp; !ibqp-&gt;uobject)</div><div class='add'>+		clear_qp(hr_qp);</div><div class='add'>+</div><div class='ctx'> out:</div><div class='del'>-	kfree(context);</div><div class='add'>+	kvfree(qpc_mask);</div><div class='add'>+	kvfree(context);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline enum ib_qp_state to_ib_qp_st(enum hns_roce_v2_qp_state state)</div><div class='add'>+static int to_ib_qp_st(enum hns_roce_v2_qp_state state)</div><div class='ctx'> {</div><div class='del'>-	switch (state) {</div><div class='del'>-	case HNS_ROCE_QP_ST_RST:	return IB_QPS_RESET;</div><div class='del'>-	case HNS_ROCE_QP_ST_INIT:	return IB_QPS_INIT;</div><div class='del'>-	case HNS_ROCE_QP_ST_RTR:	return IB_QPS_RTR;</div><div class='del'>-	case HNS_ROCE_QP_ST_RTS:	return IB_QPS_RTS;</div><div class='del'>-	case HNS_ROCE_QP_ST_SQ_DRAINING:</div><div class='del'>-	case HNS_ROCE_QP_ST_SQD:	return IB_QPS_SQD;</div><div class='del'>-	case HNS_ROCE_QP_ST_SQER:	return IB_QPS_SQE;</div><div class='del'>-	case HNS_ROCE_QP_ST_ERR:	return IB_QPS_ERR;</div><div class='del'>-	default:			return -1;</div><div class='del'>-	}</div><div class='add'>+	static const enum ib_qp_state map[] = {</div><div class='add'>+		[HNS_ROCE_QP_ST_RST] = IB_QPS_RESET,</div><div class='add'>+		[HNS_ROCE_QP_ST_INIT] = IB_QPS_INIT,</div><div class='add'>+		[HNS_ROCE_QP_ST_RTR] = IB_QPS_RTR,</div><div class='add'>+		[HNS_ROCE_QP_ST_RTS] = IB_QPS_RTS,</div><div class='add'>+		[HNS_ROCE_QP_ST_SQD] = IB_QPS_SQD,</div><div class='add'>+		[HNS_ROCE_QP_ST_SQER] = IB_QPS_SQE,</div><div class='add'>+		[HNS_ROCE_QP_ST_ERR] = IB_QPS_ERR,</div><div class='add'>+		[HNS_ROCE_QP_ST_SQ_DRAINING] = IB_QPS_SQD</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	return (state &lt; ARRAY_SIZE(map)) ? map[state] : -1;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,</div><div class='del'>-				 struct hns_roce_qp *hr_qp,</div><div class='del'>-				 struct hns_roce_v2_qp_context *hr_context)</div><div class='add'>+static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, u32 qpn,</div><div class='add'>+				 void *buffer)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_cmd_mailbox *mailbox;</div><div class='ctx'> 	int ret;</div><div class='hunk'>@@ -4297,37 +5523,90 @@ static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 	if (IS_ERR(mailbox))</div><div class='ctx'> 		return PTR_ERR(mailbox);</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox-&gt;dma, hr_qp-&gt;qpn, 0,</div><div class='del'>-				HNS_ROCE_CMD_QUERY_QPC,</div><div class='del'>-				HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "QUERY QP cmd process error\n");</div><div class='add'>+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox-&gt;dma, HNS_ROCE_CMD_QUERY_QPC,</div><div class='add'>+				qpn);</div><div class='add'>+	if (ret)</div><div class='ctx'> 		goto out;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	memcpy(hr_context, mailbox-&gt;buf, sizeof(*hr_context));</div><div class='add'>+	memcpy(buffer, mailbox-&gt;buf, hr_dev-&gt;caps.qpc_sz);</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='ctx'> 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int hns_roce_v2_query_srqc(struct hns_roce_dev *hr_dev, u32 srqn,</div><div class='add'>+				 void *buffer)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_srq_context *context;</div><div class='add'>+	struct hns_roce_cmd_mailbox *mailbox;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='add'>+	if (IS_ERR(mailbox))</div><div class='add'>+		return PTR_ERR(mailbox);</div><div class='add'>+</div><div class='add'>+	context = mailbox-&gt;buf;</div><div class='add'>+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox-&gt;dma, HNS_ROCE_CMD_QUERY_SRQC,</div><div class='add'>+				srqn);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	memcpy(buffer, context, sizeof(*context));</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_v2_query_sccc(struct hns_roce_dev *hr_dev, u32 sccn,</div><div class='add'>+				  void *buffer)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_v2_scc_context *context;</div><div class='add'>+	struct hns_roce_cmd_mailbox *mailbox;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='add'>+	if (IS_ERR(mailbox))</div><div class='add'>+		return PTR_ERR(mailbox);</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox-&gt;dma, HNS_ROCE_CMD_QUERY_SCCC,</div><div class='add'>+				sccn);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	context = mailbox-&gt;buf;</div><div class='add'>+	memcpy(buffer, context, sizeof(*context));</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u8 get_qp_timeout_attr(struct hns_roce_dev *hr_dev,</div><div class='add'>+			      struct hns_roce_v2_qp_context *context)</div><div class='add'>+{</div><div class='add'>+	u8 timeout;</div><div class='add'>+</div><div class='add'>+	timeout = (u8)hr_reg_read(context, QPC_AT);</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08)</div><div class='add'>+		timeout -= HNS_ROCE_V2_QP_ACK_TIMEOUT_OFS_HIP08;</div><div class='add'>+</div><div class='add'>+	return timeout;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 				int qp_attr_mask,</div><div class='ctx'> 				struct ib_qp_init_attr *qp_init_attr)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='ctx'> 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='del'>-	struct hns_roce_v2_qp_context *context;</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	struct hns_roce_v2_qp_context context = {};</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='ctx'> 	int tmp_qp_state;</div><div class='ctx'> 	int state;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	context = kzalloc(sizeof(*context), GFP_KERNEL);</div><div class='del'>-	if (!context)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='ctx'> 	memset(qp_attr, 0, sizeof(*qp_attr));</div><div class='ctx'> 	memset(qp_init_attr, 0, sizeof(*qp_init_attr));</div><div class='ctx'> </div><div class='hunk'>@@ -4339,221 +5618,204 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 		goto done;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, context);</div><div class='add'>+	ret = hns_roce_v2_query_qpc(hr_dev, hr_qp-&gt;qpn, &amp;context);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "query qpc error\n");</div><div class='add'>+		ibdev_err_ratelimited(ibdev,</div><div class='add'>+				      "failed to query QPC, ret = %d.\n",</div><div class='add'>+				      ret);</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	state = roce_get_field(context-&gt;byte_60_qpst_tempid,</div><div class='del'>-			       V2_QPC_BYTE_60_QP_ST_M, V2_QPC_BYTE_60_QP_ST_S);</div><div class='add'>+	state = hr_reg_read(&amp;context, QPC_QP_ST);</div><div class='ctx'> 	tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state);</div><div class='ctx'> 	if (tmp_qp_state == -1) {</div><div class='del'>-		dev_err(dev, "Illegal ib_qp_state\n");</div><div class='add'>+		ibdev_err_ratelimited(ibdev, "Illegal ib_qp_state\n");</div><div class='ctx'> 		ret = -EINVAL;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> 	hr_qp-&gt;state = (u8)tmp_qp_state;</div><div class='ctx'> 	qp_attr-&gt;qp_state = (enum ib_qp_state)hr_qp-&gt;state;</div><div class='del'>-	qp_attr-&gt;path_mtu = (enum ib_mtu)roce_get_field(context-&gt;byte_24_mtu_tc,</div><div class='del'>-							V2_QPC_BYTE_24_MTU_M,</div><div class='del'>-							V2_QPC_BYTE_24_MTU_S);</div><div class='add'>+	qp_attr-&gt;path_mtu = (enum ib_mtu)hr_reg_read(&amp;context, QPC_MTU);</div><div class='ctx'> 	qp_attr-&gt;path_mig_state = IB_MIG_ARMED;</div><div class='del'>-	qp_attr-&gt;ah_attr.type   = RDMA_AH_ATTR_TYPE_ROCE;</div><div class='add'>+	qp_attr-&gt;ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;</div><div class='ctx'> 	if (hr_qp-&gt;ibqp.qp_type == IB_QPT_UD)</div><div class='del'>-		qp_attr-&gt;qkey = V2_QKEY_VAL;</div><div class='del'>-</div><div class='del'>-	qp_attr-&gt;rq_psn = roce_get_field(context-&gt;byte_108_rx_reqepsn,</div><div class='del'>-					 V2_QPC_BYTE_108_RX_REQ_EPSN_M,</div><div class='del'>-					 V2_QPC_BYTE_108_RX_REQ_EPSN_S);</div><div class='del'>-	qp_attr-&gt;sq_psn = (u32)roce_get_field(context-&gt;byte_172_sq_psn,</div><div class='del'>-					      V2_QPC_BYTE_172_SQ_CUR_PSN_M,</div><div class='del'>-					      V2_QPC_BYTE_172_SQ_CUR_PSN_S);</div><div class='del'>-	qp_attr-&gt;dest_qp_num = (u8)roce_get_field(context-&gt;byte_56_dqpn_err,</div><div class='del'>-						  V2_QPC_BYTE_56_DQPN_M,</div><div class='del'>-						  V2_QPC_BYTE_56_DQPN_S);</div><div class='del'>-	qp_attr-&gt;qp_access_flags = ((roce_get_bit(context-&gt;byte_76_srqn_op_en,</div><div class='del'>-						  V2_QPC_BYTE_76_RRE_S)) &lt;&lt; 2) |</div><div class='del'>-				   ((roce_get_bit(context-&gt;byte_76_srqn_op_en,</div><div class='del'>-						  V2_QPC_BYTE_76_RWE_S)) &lt;&lt; 1) |</div><div class='del'>-				   ((roce_get_bit(context-&gt;byte_76_srqn_op_en,</div><div class='del'>-						  V2_QPC_BYTE_76_ATE_S)) &lt;&lt; 3);</div><div class='add'>+		qp_attr-&gt;qkey = le32_to_cpu(context.qkey_xrcd);</div><div class='add'>+</div><div class='add'>+	qp_attr-&gt;rq_psn = hr_reg_read(&amp;context, QPC_RX_REQ_EPSN);</div><div class='add'>+	qp_attr-&gt;sq_psn = (u32)hr_reg_read(&amp;context, QPC_SQ_CUR_PSN);</div><div class='add'>+	qp_attr-&gt;dest_qp_num = hr_reg_read(&amp;context, QPC_DQPN);</div><div class='add'>+	qp_attr-&gt;qp_access_flags =</div><div class='add'>+		((hr_reg_read(&amp;context, QPC_RRE)) &lt;&lt; V2_QP_RRE_S) |</div><div class='add'>+		((hr_reg_read(&amp;context, QPC_RWE)) &lt;&lt; V2_QP_RWE_S) |</div><div class='add'>+		((hr_reg_read(&amp;context, QPC_ATE)) &lt;&lt; V2_QP_ATE_S);</div><div class='add'>+</div><div class='ctx'> 	if (hr_qp-&gt;ibqp.qp_type == IB_QPT_RC ||</div><div class='del'>-	    hr_qp-&gt;ibqp.qp_type == IB_QPT_UC) {</div><div class='add'>+	    hr_qp-&gt;ibqp.qp_type == IB_QPT_XRC_INI ||</div><div class='add'>+	    hr_qp-&gt;ibqp.qp_type == IB_QPT_XRC_TGT) {</div><div class='ctx'> 		struct ib_global_route *grh =</div><div class='del'>-				rdma_ah_retrieve_grh(&amp;qp_attr-&gt;ah_attr);</div><div class='add'>+			rdma_ah_retrieve_grh(&amp;qp_attr-&gt;ah_attr);</div><div class='ctx'> </div><div class='ctx'> 		rdma_ah_set_sl(&amp;qp_attr-&gt;ah_attr,</div><div class='del'>-			       roce_get_field(context-&gt;byte_28_at_fl,</div><div class='del'>-					      V2_QPC_BYTE_28_SL_M,</div><div class='del'>-					      V2_QPC_BYTE_28_SL_S));</div><div class='del'>-		grh-&gt;flow_label = roce_get_field(context-&gt;byte_28_at_fl,</div><div class='del'>-						 V2_QPC_BYTE_28_FL_M,</div><div class='del'>-						 V2_QPC_BYTE_28_FL_S);</div><div class='del'>-		grh-&gt;sgid_index = roce_get_field(context-&gt;byte_20_smac_sgid_idx,</div><div class='del'>-						 V2_QPC_BYTE_20_SGID_IDX_M,</div><div class='del'>-						 V2_QPC_BYTE_20_SGID_IDX_S);</div><div class='del'>-		grh-&gt;hop_limit = roce_get_field(context-&gt;byte_24_mtu_tc,</div><div class='del'>-						V2_QPC_BYTE_24_HOP_LIMIT_M,</div><div class='del'>-						V2_QPC_BYTE_24_HOP_LIMIT_S);</div><div class='del'>-		grh-&gt;traffic_class = roce_get_field(context-&gt;byte_24_mtu_tc,</div><div class='del'>-						    V2_QPC_BYTE_24_TC_M,</div><div class='del'>-						    V2_QPC_BYTE_24_TC_S);</div><div class='del'>-</div><div class='del'>-		memcpy(grh-&gt;dgid.raw, context-&gt;dgid, sizeof(grh-&gt;dgid.raw));</div><div class='add'>+			       hr_reg_read(&amp;context, QPC_SL));</div><div class='add'>+		rdma_ah_set_port_num(&amp;qp_attr-&gt;ah_attr, hr_qp-&gt;port + 1);</div><div class='add'>+		rdma_ah_set_ah_flags(&amp;qp_attr-&gt;ah_attr, IB_AH_GRH);</div><div class='add'>+		grh-&gt;flow_label = hr_reg_read(&amp;context, QPC_FL);</div><div class='add'>+		grh-&gt;sgid_index = hr_reg_read(&amp;context, QPC_GMV_IDX);</div><div class='add'>+		grh-&gt;hop_limit = hr_reg_read(&amp;context, QPC_HOPLIMIT);</div><div class='add'>+		grh-&gt;traffic_class = hr_reg_read(&amp;context, QPC_TC);</div><div class='add'>+</div><div class='add'>+		memcpy(grh-&gt;dgid.raw, context.dgid, sizeof(grh-&gt;dgid.raw));</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	qp_attr-&gt;port_num = hr_qp-&gt;port + 1;</div><div class='ctx'> 	qp_attr-&gt;sq_draining = 0;</div><div class='del'>-	qp_attr-&gt;max_rd_atomic = 1 &lt;&lt; roce_get_field(context-&gt;byte_208_irrl,</div><div class='del'>-						     V2_QPC_BYTE_208_SR_MAX_M,</div><div class='del'>-						     V2_QPC_BYTE_208_SR_MAX_S);</div><div class='del'>-	qp_attr-&gt;max_dest_rd_atomic = 1 &lt;&lt; roce_get_field(context-&gt;byte_140_raq,</div><div class='del'>-						     V2_QPC_BYTE_140_RR_MAX_M,</div><div class='del'>-						     V2_QPC_BYTE_140_RR_MAX_S);</div><div class='del'>-	qp_attr-&gt;min_rnr_timer = (u8)roce_get_field(context-&gt;byte_80_rnr_rx_cqn,</div><div class='del'>-						 V2_QPC_BYTE_80_MIN_RNR_TIME_M,</div><div class='del'>-						 V2_QPC_BYTE_80_MIN_RNR_TIME_S);</div><div class='del'>-	qp_attr-&gt;timeout = (u8)roce_get_field(context-&gt;byte_28_at_fl,</div><div class='del'>-					      V2_QPC_BYTE_28_AT_M,</div><div class='del'>-					      V2_QPC_BYTE_28_AT_S);</div><div class='del'>-	qp_attr-&gt;retry_cnt = roce_get_field(context-&gt;byte_212_lsn,</div><div class='del'>-					    V2_QPC_BYTE_212_RETRY_CNT_M,</div><div class='del'>-					    V2_QPC_BYTE_212_RETRY_CNT_S);</div><div class='del'>-	qp_attr-&gt;rnr_retry = context-&gt;rq_rnr_timer;</div><div class='add'>+	qp_attr-&gt;max_rd_atomic = 1 &lt;&lt; hr_reg_read(&amp;context, QPC_SR_MAX);</div><div class='add'>+	qp_attr-&gt;max_dest_rd_atomic = 1 &lt;&lt; hr_reg_read(&amp;context, QPC_RR_MAX);</div><div class='add'>+</div><div class='add'>+	qp_attr-&gt;min_rnr_timer = (u8)hr_reg_read(&amp;context, QPC_MIN_RNR_TIME);</div><div class='add'>+	qp_attr-&gt;timeout = get_qp_timeout_attr(hr_dev, &amp;context);</div><div class='add'>+	qp_attr-&gt;retry_cnt = hr_reg_read(&amp;context, QPC_RETRY_NUM_INIT);</div><div class='add'>+	qp_attr-&gt;rnr_retry = hr_reg_read(&amp;context, QPC_RNR_NUM_INIT);</div><div class='ctx'> </div><div class='ctx'> done:</div><div class='ctx'> 	qp_attr-&gt;cur_qp_state = qp_attr-&gt;qp_state;</div><div class='ctx'> 	qp_attr-&gt;cap.max_recv_wr = hr_qp-&gt;rq.wqe_cnt;</div><div class='del'>-	qp_attr-&gt;cap.max_recv_sge = hr_qp-&gt;rq.max_gs;</div><div class='add'>+	qp_attr-&gt;cap.max_recv_sge = hr_qp-&gt;rq.max_gs - hr_qp-&gt;rq.rsv_sge;</div><div class='add'>+	qp_attr-&gt;cap.max_inline_data = hr_qp-&gt;max_inline_data;</div><div class='ctx'> </div><div class='del'>-	if (!ibqp-&gt;uobject) {</div><div class='del'>-		qp_attr-&gt;cap.max_send_wr = hr_qp-&gt;sq.wqe_cnt;</div><div class='del'>-		qp_attr-&gt;cap.max_send_sge = hr_qp-&gt;sq.max_gs;</div><div class='del'>-	} else {</div><div class='del'>-		qp_attr-&gt;cap.max_send_wr = 0;</div><div class='del'>-		qp_attr-&gt;cap.max_send_sge = 0;</div><div class='del'>-	}</div><div class='add'>+	qp_attr-&gt;cap.max_send_wr = hr_qp-&gt;sq.wqe_cnt;</div><div class='add'>+	qp_attr-&gt;cap.max_send_sge = hr_qp-&gt;sq.max_gs;</div><div class='ctx'> </div><div class='add'>+	qp_init_attr-&gt;qp_context = ibqp-&gt;qp_context;</div><div class='add'>+	qp_init_attr-&gt;qp_type = ibqp-&gt;qp_type;</div><div class='add'>+	qp_init_attr-&gt;recv_cq = ibqp-&gt;recv_cq;</div><div class='add'>+	qp_init_attr-&gt;send_cq = ibqp-&gt;send_cq;</div><div class='add'>+	qp_init_attr-&gt;srq = ibqp-&gt;srq;</div><div class='ctx'> 	qp_init_attr-&gt;cap = qp_attr-&gt;cap;</div><div class='add'>+	qp_init_attr-&gt;sq_sig_type = hr_qp-&gt;sq_signal_bits;</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='ctx'> 	mutex_unlock(&amp;hr_qp-&gt;mutex);</div><div class='del'>-	kfree(context);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static inline int modify_qp_is_ok(struct hns_roce_qp *hr_qp)</div><div class='add'>+{</div><div class='add'>+	return ((hr_qp-&gt;ibqp.qp_type == IB_QPT_RC ||</div><div class='add'>+		 hr_qp-&gt;ibqp.qp_type == IB_QPT_UD ||</div><div class='add'>+		 hr_qp-&gt;ibqp.qp_type == IB_QPT_XRC_INI ||</div><div class='add'>+		 hr_qp-&gt;ibqp.qp_type == IB_QPT_XRC_TGT) &amp;&amp;</div><div class='add'>+		hr_qp-&gt;state != IB_QPS_RESET);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 					 struct hns_roce_qp *hr_qp,</div><div class='ctx'> 					 struct ib_udata *udata)</div><div class='ctx'> {</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='ctx'> 	struct hns_roce_cq *send_cq, *recv_cq;</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	int ret;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	if (hr_qp-&gt;ibqp.qp_type == IB_QPT_RC &amp;&amp; hr_qp-&gt;state != IB_QPS_RESET) {</div><div class='add'>+	if (modify_qp_is_ok(hr_qp)) {</div><div class='ctx'> 		/* Modify qp to reset before destroying qp */</div><div class='ctx'> 		ret = hns_roce_v2_modify_qp(&amp;hr_qp-&gt;ibqp, NULL, 0,</div><div class='del'>-					    hr_qp-&gt;state, IB_QPS_RESET);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "modify QP %06lx to ERR failed.\n",</div><div class='del'>-				hr_qp-&gt;qpn);</div><div class='del'>-			return ret;</div><div class='del'>-		}</div><div class='add'>+					    hr_qp-&gt;state, IB_QPS_RESET, udata);</div><div class='add'>+		if (ret)</div><div class='add'>+			ibdev_err_ratelimited(ibdev,</div><div class='add'>+					      "failed to modify QP to RST, ret = %d.\n",</div><div class='add'>+					      ret);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	send_cq = to_hr_cq(hr_qp-&gt;ibqp.send_cq);</div><div class='del'>-	recv_cq = to_hr_cq(hr_qp-&gt;ibqp.recv_cq);</div><div class='add'>+	send_cq = hr_qp-&gt;ibqp.send_cq ? to_hr_cq(hr_qp-&gt;ibqp.send_cq) : NULL;</div><div class='add'>+	recv_cq = hr_qp-&gt;ibqp.recv_cq ? to_hr_cq(hr_qp-&gt;ibqp.recv_cq) : NULL;</div><div class='ctx'> </div><div class='add'>+	spin_lock_irqsave(&amp;hr_dev-&gt;qp_list_lock, flags);</div><div class='ctx'> 	hns_roce_lock_cqs(send_cq, recv_cq);</div><div class='ctx'> </div><div class='ctx'> 	if (!udata) {</div><div class='del'>-		__hns_roce_v2_cq_clean(recv_cq, hr_qp-&gt;qpn, hr_qp-&gt;ibqp.srq ?</div><div class='del'>-				       to_hr_srq(hr_qp-&gt;ibqp.srq) : NULL);</div><div class='del'>-		if (send_cq != recv_cq)</div><div class='add'>+		if (recv_cq)</div><div class='add'>+			__hns_roce_v2_cq_clean(recv_cq, hr_qp-&gt;qpn,</div><div class='add'>+					       (hr_qp-&gt;ibqp.srq ?</div><div class='add'>+						to_hr_srq(hr_qp-&gt;ibqp.srq) :</div><div class='add'>+						NULL));</div><div class='add'>+</div><div class='add'>+		if (send_cq &amp;&amp; send_cq != recv_cq)</div><div class='ctx'> 			__hns_roce_v2_cq_clean(send_cq, hr_qp-&gt;qpn, NULL);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	hns_roce_qp_remove(hr_dev, hr_qp);</div><div class='ctx'> </div><div class='ctx'> 	hns_roce_unlock_cqs(send_cq, recv_cq);</div><div class='add'>+	spin_unlock_irqrestore(&amp;hr_dev-&gt;qp_list_lock, flags);</div><div class='ctx'> </div><div class='del'>-	hns_roce_qp_free(hr_dev, hr_qp);</div><div class='del'>-</div><div class='del'>-	/* Not special_QP, free their QPN */</div><div class='del'>-	if ((hr_qp-&gt;ibqp.qp_type == IB_QPT_RC) ||</div><div class='del'>-	    (hr_qp-&gt;ibqp.qp_type == IB_QPT_UC) ||</div><div class='del'>-	    (hr_qp-&gt;ibqp.qp_type == IB_QPT_UD))</div><div class='del'>-		hns_roce_release_range_qp(hr_dev, hr_qp-&gt;qpn, 1);</div><div class='del'>-</div><div class='del'>-	hns_roce_mtt_cleanup(hr_dev, &amp;hr_qp-&gt;mtt);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (udata) {</div><div class='del'>-		struct hns_roce_ucontext *context =</div><div class='del'>-			rdma_udata_to_drv_context(</div><div class='del'>-				udata,</div><div class='del'>-				struct hns_roce_ucontext,</div><div class='del'>-				ibucontext);</div><div class='add'>+static void put_dip_ctx_idx(struct hns_roce_dev *hr_dev,</div><div class='add'>+			    struct hns_roce_qp *hr_qp)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dip *hr_dip = hr_qp-&gt;dip;</div><div class='ctx'> </div><div class='del'>-		if (hr_qp-&gt;sq.wqe_cnt &amp;&amp; (hr_qp-&gt;sdb_en == 1))</div><div class='del'>-			hns_roce_db_unmap_user(context, &amp;hr_qp-&gt;sdb);</div><div class='add'>+	if (!hr_dip)</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-		if (hr_qp-&gt;rq.wqe_cnt &amp;&amp; (hr_qp-&gt;rdb_en == 1))</div><div class='del'>-			hns_roce_db_unmap_user(context, &amp;hr_qp-&gt;rdb);</div><div class='del'>-		ib_umem_release(hr_qp-&gt;umem);</div><div class='del'>-	} else {</div><div class='del'>-		kfree(hr_qp-&gt;sq.wrid);</div><div class='del'>-		kfree(hr_qp-&gt;rq.wrid);</div><div class='del'>-		hns_roce_buf_free(hr_dev, hr_qp-&gt;buff_size, &amp;hr_qp-&gt;hr_buf);</div><div class='del'>-		if (hr_qp-&gt;rq.wqe_cnt)</div><div class='del'>-			hns_roce_free_db(hr_dev, &amp;hr_qp-&gt;rdb);</div><div class='del'>-	}</div><div class='add'>+	xa_lock(&amp;hr_dev-&gt;qp_table.dip_xa);</div><div class='ctx'> </div><div class='del'>-	if ((hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RQ_INLINE) &amp;&amp;</div><div class='del'>-	     hr_qp-&gt;rq.wqe_cnt) {</div><div class='del'>-		kfree(hr_qp-&gt;rq_inl_buf.wqe_list[0].sg_list);</div><div class='del'>-		kfree(hr_qp-&gt;rq_inl_buf.wqe_list);</div><div class='del'>-	}</div><div class='add'>+	hr_dip-&gt;qp_cnt--;</div><div class='add'>+	if (!hr_dip-&gt;qp_cnt)</div><div class='add'>+		memset(hr_dip-&gt;dgid, 0, GID_LEN_V2);</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	xa_unlock(&amp;hr_dev-&gt;qp_table.dip_xa);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)</div><div class='add'>+int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='ctx'> 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='add'>+	unsigned long flags;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	/* Make sure flush_cqe() is completed */</div><div class='add'>+	spin_lock_irqsave(&amp;hr_qp-&gt;flush_lock, flags);</div><div class='add'>+	set_bit(HNS_ROCE_STOP_FLUSH_FLAG, &amp;hr_qp-&gt;flush_flag);</div><div class='add'>+	spin_unlock_irqrestore(&amp;hr_qp-&gt;flush_lock, flags);</div><div class='add'>+	flush_work(&amp;hr_qp-&gt;flush_work.work);</div><div class='add'>+</div><div class='add'>+	if (hr_qp-&gt;cong_type == CONG_TYPE_DIP)</div><div class='add'>+		put_dip_ctx_idx(hr_dev, hr_qp);</div><div class='add'>+</div><div class='ctx'> 	ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "Destroy qp failed(%d)\n", ret);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_err_ratelimited(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+				      "failed to destroy QP, QPN = 0x%06lx, ret = %d.\n",</div><div class='add'>+				      hr_qp-&gt;qpn, ret);</div><div class='ctx'> </div><div class='del'>-	if (hr_qp-&gt;ibqp.qp_type == IB_QPT_GSI)</div><div class='del'>-		kfree(hr_to_hr_sqp(hr_qp));</div><div class='del'>-	else</div><div class='del'>-		kfree(hr_qp);</div><div class='add'>+	hns_roce_qp_destroy(hr_dev, hr_qp, udata);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,</div><div class='del'>-						struct hns_roce_qp *hr_qp)</div><div class='add'>+					    struct hns_roce_qp *hr_qp)</div><div class='ctx'> {</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='ctx'> 	struct hns_roce_sccc_clr_done *resp;</div><div class='ctx'> 	struct hns_roce_sccc_clr *clr;</div><div class='ctx'> 	struct hns_roce_cmq_desc desc;</div><div class='ctx'> 	int ret, i;</div><div class='ctx'> </div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='ctx'> 	mutex_lock(&amp;hr_dev-&gt;qp_table.scc_mutex);</div><div class='ctx'> </div><div class='ctx'> 	/* set scc ctx clear done flag */</div><div class='ctx'> 	hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_OPC_RESET_SCCC, false);</div><div class='ctx'> 	ret =  hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "Reset SCC ctx  failed(%d)\n", ret);</div><div class='add'>+		ibdev_err(ibdev, "failed to reset SCC ctx, ret = %d.\n", ret);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -4563,7 +5825,7 @@ static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 	clr-&gt;qpn = cpu_to_le32(hr_qp-&gt;qpn);</div><div class='ctx'> 	ret =  hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "Clear SCC ctx failed(%d)\n", ret);</div><div class='add'>+		ibdev_err(ibdev, "failed to clear SCC ctx, ret = %d.\n", ret);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -4574,7 +5836,8 @@ static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 					      HNS_ROCE_OPC_QUERY_SCCC, true);</div><div class='ctx'> 		ret = hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			dev_err(hr_dev-&gt;dev, "Query clr cmq failed(%d)\n", ret);</div><div class='add'>+			ibdev_err(ibdev, "failed to query clr cmq, ret = %d\n",</div><div class='add'>+				  ret);</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='hunk'>@@ -4584,7 +5847,7 @@ static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 		msleep(20);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	dev_err(hr_dev-&gt;dev, "Query SCC clr done flag overtime.\n");</div><div class='add'>+	ibdev_err(ibdev, "query SCC clr done flag overtime.\n");</div><div class='ctx'> 	ret = -ETIMEDOUT;</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='hunk'>@@ -4592,6 +5855,197 @@ out:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+#define DMA_IDX_SHIFT 3</div><div class='add'>+#define DMA_WQE_SHIFT 3</div><div class='add'>+</div><div class='add'>+static int hns_roce_v2_write_srqc_index_queue(struct hns_roce_srq *srq,</div><div class='add'>+					      struct hns_roce_srq_context *ctx)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_idx_que *idx_que = &amp;srq-&gt;idx_que;</div><div class='add'>+	struct ib_device *ibdev = srq-&gt;ibsrq.device;</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);</div><div class='add'>+	u64 mtts_idx[MTT_MIN_COUNT] = {};</div><div class='add'>+	dma_addr_t dma_handle_idx;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	/* Get physical address of idx que buf */</div><div class='add'>+	ret = hns_roce_mtr_find(hr_dev, &amp;idx_que-&gt;mtr, 0, mtts_idx,</div><div class='add'>+				ARRAY_SIZE(mtts_idx));</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to find mtr for SRQ idx, ret = %d.\n",</div><div class='add'>+			  ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dma_handle_idx = hns_roce_get_mtr_ba(&amp;idx_que-&gt;mtr);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(ctx, SRQC_IDX_HOP_NUM,</div><div class='add'>+		     to_hr_hem_hopnum(hr_dev-&gt;caps.idx_hop_num, srq-&gt;wqe_cnt));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(ctx, SRQC_IDX_BT_BA_L, dma_handle_idx &gt;&gt; DMA_IDX_SHIFT);</div><div class='add'>+	hr_reg_write(ctx, SRQC_IDX_BT_BA_H,</div><div class='add'>+		     upper_32_bits(dma_handle_idx &gt;&gt; DMA_IDX_SHIFT));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(ctx, SRQC_IDX_BA_PG_SZ,</div><div class='add'>+		     to_hr_hw_page_shift(idx_que-&gt;mtr.hem_cfg.ba_pg_shift));</div><div class='add'>+	hr_reg_write(ctx, SRQC_IDX_BUF_PG_SZ,</div><div class='add'>+		     to_hr_hw_page_shift(idx_que-&gt;mtr.hem_cfg.buf_pg_shift));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_L,</div><div class='add'>+		     to_hr_hw_page_addr(mtts_idx[0]));</div><div class='add'>+	hr_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_H,</div><div class='add'>+		     upper_32_bits(to_hr_hw_page_addr(mtts_idx[0])));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_L,</div><div class='add'>+		     to_hr_hw_page_addr(mtts_idx[1]));</div><div class='add'>+	hr_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_H,</div><div class='add'>+		     upper_32_bits(to_hr_hw_page_addr(mtts_idx[1])));</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_v2_write_srqc(struct hns_roce_srq *srq, void *mb_buf)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = srq-&gt;ibsrq.device;</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);</div><div class='add'>+	struct hns_roce_srq_context *ctx = mb_buf;</div><div class='add'>+	u64 mtts_wqe[MTT_MIN_COUNT] = {};</div><div class='add'>+	dma_addr_t dma_handle_wqe;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	memset(ctx, 0, sizeof(*ctx));</div><div class='add'>+</div><div class='add'>+	/* Get the physical address of srq buf */</div><div class='add'>+	ret = hns_roce_mtr_find(hr_dev, &amp;srq-&gt;buf_mtr, 0, mtts_wqe,</div><div class='add'>+				ARRAY_SIZE(mtts_wqe));</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to find mtr for SRQ WQE, ret = %d.\n",</div><div class='add'>+			  ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dma_handle_wqe = hns_roce_get_mtr_ba(&amp;srq-&gt;buf_mtr);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(ctx, SRQC_SRQ_ST, 1);</div><div class='add'>+	hr_reg_write_bool(ctx, SRQC_SRQ_TYPE,</div><div class='add'>+			  srq-&gt;ibsrq.srq_type == IB_SRQT_XRC);</div><div class='add'>+	hr_reg_write(ctx, SRQC_PD, to_hr_pd(srq-&gt;ibsrq.pd)-&gt;pdn);</div><div class='add'>+	hr_reg_write(ctx, SRQC_SRQN, srq-&gt;srqn);</div><div class='add'>+	hr_reg_write(ctx, SRQC_XRCD, srq-&gt;xrcdn);</div><div class='add'>+	hr_reg_write(ctx, SRQC_XRC_CQN, srq-&gt;cqn);</div><div class='add'>+	hr_reg_write(ctx, SRQC_SHIFT, ilog2(srq-&gt;wqe_cnt));</div><div class='add'>+	hr_reg_write(ctx, SRQC_RQWS,</div><div class='add'>+		     srq-&gt;max_gs &lt;= 0 ? 0 : fls(srq-&gt;max_gs - 1));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(ctx, SRQC_WQE_HOP_NUM,</div><div class='add'>+		     to_hr_hem_hopnum(hr_dev-&gt;caps.srqwqe_hop_num,</div><div class='add'>+				      srq-&gt;wqe_cnt));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(ctx, SRQC_WQE_BT_BA_L, dma_handle_wqe &gt;&gt; DMA_WQE_SHIFT);</div><div class='add'>+	hr_reg_write(ctx, SRQC_WQE_BT_BA_H,</div><div class='add'>+		     upper_32_bits(dma_handle_wqe &gt;&gt; DMA_WQE_SHIFT));</div><div class='add'>+</div><div class='add'>+	hr_reg_write(ctx, SRQC_WQE_BA_PG_SZ,</div><div class='add'>+		     to_hr_hw_page_shift(srq-&gt;buf_mtr.hem_cfg.ba_pg_shift));</div><div class='add'>+	hr_reg_write(ctx, SRQC_WQE_BUF_PG_SZ,</div><div class='add'>+		     to_hr_hw_page_shift(srq-&gt;buf_mtr.hem_cfg.buf_pg_shift));</div><div class='add'>+</div><div class='add'>+	if (srq-&gt;cap_flags &amp; HNS_ROCE_SRQ_CAP_RECORD_DB) {</div><div class='add'>+		hr_reg_enable(ctx, SRQC_DB_RECORD_EN);</div><div class='add'>+		hr_reg_write(ctx, SRQC_DB_RECORD_ADDR_L,</div><div class='add'>+			     lower_32_bits(srq-&gt;rdb.dma) &gt;&gt; 1);</div><div class='add'>+		hr_reg_write(ctx, SRQC_DB_RECORD_ADDR_H,</div><div class='add'>+			     upper_32_bits(srq-&gt;rdb.dma));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return hns_roce_v2_write_srqc_index_queue(srq, ctx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,</div><div class='add'>+				  struct ib_srq_attr *srq_attr,</div><div class='add'>+				  enum ib_srq_attr_mask srq_attr_mask,</div><div class='add'>+				  struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq-&gt;device);</div><div class='add'>+	struct hns_roce_srq *srq = to_hr_srq(ibsrq);</div><div class='add'>+	struct hns_roce_srq_context *srq_context;</div><div class='add'>+	struct hns_roce_srq_context *srqc_mask;</div><div class='add'>+	struct hns_roce_cmd_mailbox *mailbox;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	/* Resizing SRQs is not supported yet */</div><div class='add'>+	if (srq_attr_mask &amp; IB_SRQ_MAX_WR) {</div><div class='add'>+		ret = -EOPNOTSUPP;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (srq_attr_mask &amp; IB_SRQ_LIMIT) {</div><div class='add'>+		if (srq_attr-&gt;srq_limit &gt; srq-&gt;wqe_cnt) {</div><div class='add'>+			ret = -EINVAL;</div><div class='add'>+			goto out;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='add'>+		if (IS_ERR(mailbox)) {</div><div class='add'>+			ret = PTR_ERR(mailbox);</div><div class='add'>+			goto out;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		srq_context = mailbox-&gt;buf;</div><div class='add'>+		srqc_mask = (struct hns_roce_srq_context *)mailbox-&gt;buf + 1;</div><div class='add'>+</div><div class='add'>+		memset(srqc_mask, 0xff, sizeof(*srqc_mask));</div><div class='add'>+</div><div class='add'>+		hr_reg_write(srq_context, SRQC_LIMIT_WL, srq_attr-&gt;srq_limit);</div><div class='add'>+		hr_reg_clear(srqc_mask, SRQC_LIMIT_WL);</div><div class='add'>+</div><div class='add'>+		ret = hns_roce_cmd_mbox(hr_dev, mailbox-&gt;dma, 0,</div><div class='add'>+					HNS_ROCE_CMD_MODIFY_SRQC, srq-&gt;srqn);</div><div class='add'>+		hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='add'>+		if (ret)</div><div class='add'>+			ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+				  "failed to handle cmd of modifying SRQ, ret = %d.\n",</div><div class='add'>+				  ret);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	if (ret)</div><div class='add'>+		atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_SRQ_MODIFY_ERR_CNT]);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq-&gt;device);</div><div class='add'>+	struct hns_roce_srq *srq = to_hr_srq(ibsrq);</div><div class='add'>+	struct hns_roce_srq_context *srq_context;</div><div class='add'>+	struct hns_roce_cmd_mailbox *mailbox;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='add'>+	if (IS_ERR(mailbox))</div><div class='add'>+		return PTR_ERR(mailbox);</div><div class='add'>+</div><div class='add'>+	srq_context = mailbox-&gt;buf;</div><div class='add'>+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox-&gt;dma,</div><div class='add'>+				HNS_ROCE_CMD_QUERY_SRQC, srq-&gt;srqn);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+			  "failed to process cmd of querying SRQ, ret = %d.\n",</div><div class='add'>+			  ret);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	attr-&gt;srq_limit = hr_reg_read(srq_context, SRQC_LIMIT_WL);</div><div class='add'>+	attr-&gt;max_wr = srq-&gt;wqe_cnt;</div><div class='add'>+	attr-&gt;max_sge = srq-&gt;max_gs - srq-&gt;rsv_sge;</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(cq-&gt;device);</div><div class='hunk'>@@ -4602,137 +6056,208 @@ static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='del'>-	if (IS_ERR(mailbox))</div><div class='del'>-		return PTR_ERR(mailbox);</div><div class='add'>+	ret = PTR_ERR_OR_ZERO(mailbox);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_out;</div><div class='ctx'> </div><div class='ctx'> 	cq_context = mailbox-&gt;buf;</div><div class='ctx'> 	cqc_mask = (struct hns_roce_v2_cq_context *)mailbox-&gt;buf + 1;</div><div class='ctx'> </div><div class='ctx'> 	memset(cqc_mask, 0xff, sizeof(*cqc_mask));</div><div class='ctx'> </div><div class='del'>-	roce_set_field(cq_context-&gt;byte_56_cqe_period_maxcnt,</div><div class='del'>-		       V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S,</div><div class='del'>-		       cq_count);</div><div class='del'>-	roce_set_field(cqc_mask-&gt;byte_56_cqe_period_maxcnt,</div><div class='del'>-		       V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S,</div><div class='del'>-		       0);</div><div class='del'>-	roce_set_field(cq_context-&gt;byte_56_cqe_period_maxcnt,</div><div class='del'>-		       V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S,</div><div class='del'>-		       cq_period);</div><div class='del'>-	roce_set_field(cqc_mask-&gt;byte_56_cqe_period_maxcnt,</div><div class='del'>-		       V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S,</div><div class='del'>-		       0);</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_cmd_mbox(hr_dev, mailbox-&gt;dma, 0, hr_cq-&gt;cqn, 1,</div><div class='del'>-				HNS_ROCE_CMD_MODIFY_CQC,</div><div class='del'>-				HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQ_MAX_CNT, cq_count);</div><div class='add'>+	hr_reg_clear(cqc_mask, CQC_CQ_MAX_CNT);</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08) {</div><div class='add'>+		if (cq_period * HNS_ROCE_CLOCK_ADJUST &gt; USHRT_MAX) {</div><div class='add'>+			dev_info(hr_dev-&gt;dev,</div><div class='add'>+				 "cq_period(%u) reached the upper limit, adjusted to 65.\n",</div><div class='add'>+				 cq_period);</div><div class='add'>+			cq_period = HNS_ROCE_MAX_CQ_PERIOD_HIP08;</div><div class='add'>+		}</div><div class='add'>+		cq_period *= HNS_ROCE_CLOCK_ADJUST;</div><div class='add'>+	}</div><div class='add'>+	hr_reg_write(cq_context, CQC_CQ_PERIOD, cq_period);</div><div class='add'>+	hr_reg_clear(cqc_mask, CQC_CQ_PERIOD);</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_cmd_mbox(hr_dev, mailbox-&gt;dma, 0,</div><div class='add'>+				HNS_ROCE_CMD_MODIFY_CQC, hr_cq-&gt;cqn);</div><div class='ctx'> 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "MODIFY CQ Failed to cmd mailbox.\n");</div><div class='add'>+		ibdev_err_ratelimited(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+				      "failed to process cmd when modifying CQ, ret = %d.\n",</div><div class='add'>+				      ret);</div><div class='add'>+</div><div class='add'>+err_out:</div><div class='add'>+	if (ret)</div><div class='add'>+		atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_CQ_MODIFY_ERR_CNT]);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_set_qps_to_err(struct hns_roce_dev *hr_dev, u32 qpn)</div><div class='add'>+static int hns_roce_v2_query_cqc(struct hns_roce_dev *hr_dev, u32 cqn,</div><div class='add'>+				 void *buffer)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_qp *hr_qp;</div><div class='del'>-	struct ib_qp_attr attr;</div><div class='del'>-	int attr_mask;</div><div class='add'>+	struct hns_roce_v2_cq_context *context;</div><div class='add'>+	struct hns_roce_cmd_mailbox *mailbox;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);</div><div class='del'>-	if (!hr_qp) {</div><div class='del'>-		dev_warn(hr_dev-&gt;dev, "no hr_qp can be found!\n");</div><div class='del'>-		return;</div><div class='add'>+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='add'>+	if (IS_ERR(mailbox))</div><div class='add'>+		return PTR_ERR(mailbox);</div><div class='add'>+</div><div class='add'>+	context = mailbox-&gt;buf;</div><div class='add'>+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox-&gt;dma,</div><div class='add'>+				HNS_ROCE_CMD_QUERY_CQC, cqn);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err_ratelimited(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+				      "failed to process cmd when querying CQ, ret = %d.\n",</div><div class='add'>+				      ret);</div><div class='add'>+		goto err_mailbox;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (hr_qp-&gt;ibqp.uobject) {</div><div class='del'>-		if (hr_qp-&gt;sdb_en == 1) {</div><div class='del'>-			hr_qp-&gt;sq.head = *(int *)(hr_qp-&gt;sdb.virt_addr);</div><div class='del'>-			if (hr_qp-&gt;rdb_en == 1)</div><div class='del'>-				hr_qp-&gt;rq.head = *(int *)(hr_qp-&gt;rdb.virt_addr);</div><div class='del'>-		} else {</div><div class='del'>-			dev_warn(hr_dev-&gt;dev, "flush cqe is unsupported in userspace!\n");</div><div class='del'>-			return;</div><div class='del'>-		}</div><div class='add'>+	memcpy(buffer, context, sizeof(*context));</div><div class='add'>+</div><div class='add'>+err_mailbox:</div><div class='add'>+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_v2_query_mpt(struct hns_roce_dev *hr_dev, u32 key,</div><div class='add'>+				 void *buffer)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_v2_mpt_entry *context;</div><div class='add'>+	struct hns_roce_cmd_mailbox *mailbox;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='add'>+	if (IS_ERR(mailbox))</div><div class='add'>+		return PTR_ERR(mailbox);</div><div class='add'>+</div><div class='add'>+	context = mailbox-&gt;buf;</div><div class='add'>+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox-&gt;dma, HNS_ROCE_CMD_QUERY_MPT,</div><div class='add'>+				key_to_hw_index(key));</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+			  "failed to process cmd when querying MPT, ret = %d.\n",</div><div class='add'>+			  ret);</div><div class='add'>+		goto err_mailbox;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	attr_mask = IB_QP_STATE;</div><div class='del'>-	attr.qp_state = IB_QPS_ERR;</div><div class='del'>-	ret = hns_roce_v2_modify_qp(&amp;hr_qp-&gt;ibqp, &amp;attr, attr_mask,</div><div class='del'>-				    hr_qp-&gt;state, IB_QPS_ERR);</div><div class='del'>-	if (ret)</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "failed to modify qp %d to err state.\n",</div><div class='del'>-			qpn);</div><div class='add'>+	memcpy(buffer, context, sizeof(*context));</div><div class='add'>+</div><div class='add'>+err_mailbox:</div><div class='add'>+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_irq_work_handle(struct work_struct *work)</div><div class='add'>+static void dump_aeqe_log(struct hns_roce_work *irq_work)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_work *irq_work =</div><div class='del'>-				container_of(work, struct hns_roce_work, work);</div><div class='del'>-	struct device *dev = irq_work-&gt;hr_dev-&gt;dev;</div><div class='del'>-	u32 qpn = irq_work-&gt;qpn;</div><div class='del'>-	u32 cqn = irq_work-&gt;cqn;</div><div class='add'>+	struct hns_roce_dev *hr_dev = irq_work-&gt;hr_dev;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='ctx'> </div><div class='ctx'> 	switch (irq_work-&gt;event_type) {</div><div class='ctx'> 	case HNS_ROCE_EVENT_TYPE_PATH_MIG:</div><div class='del'>-		dev_info(dev, "Path migrated succeeded.\n");</div><div class='add'>+		ibdev_info(ibdev, "path migrated succeeded.\n");</div><div class='ctx'> 		break;</div><div class='ctx'> 	case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:</div><div class='del'>-		dev_warn(dev, "Path migration failed.\n");</div><div class='add'>+		ibdev_warn(ibdev, "path migration failed.\n");</div><div class='ctx'> 		break;</div><div class='ctx'> 	case HNS_ROCE_EVENT_TYPE_COMM_EST:</div><div class='del'>-		dev_info(dev, "Communication established.\n");</div><div class='ctx'> 		break;</div><div class='ctx'> 	case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:</div><div class='del'>-		dev_warn(dev, "Send queue drained.\n");</div><div class='add'>+		ibdev_dbg(ibdev, "send queue drained.\n");</div><div class='ctx'> 		break;</div><div class='ctx'> 	case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:</div><div class='del'>-		dev_err(dev, "Local work queue 0x%x catas error, sub_type:%d\n",</div><div class='del'>-			qpn, irq_work-&gt;sub_type);</div><div class='del'>-		hns_roce_set_qps_to_err(irq_work-&gt;hr_dev, qpn);</div><div class='add'>+		ibdev_err(ibdev, "local work queue 0x%x catast error, sub_event type is: %d\n",</div><div class='add'>+			  irq_work-&gt;queue_num, irq_work-&gt;sub_type);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:</div><div class='del'>-		dev_err(dev, "Invalid request local work queue 0x%x error.\n",</div><div class='del'>-			qpn);</div><div class='del'>-		hns_roce_set_qps_to_err(irq_work-&gt;hr_dev, qpn);</div><div class='add'>+		ibdev_err(ibdev, "invalid request local work queue 0x%x error.\n",</div><div class='add'>+			  irq_work-&gt;queue_num);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:</div><div class='del'>-		dev_err(dev, "Local access violation work queue 0x%x error, sub_type:%d\n",</div><div class='del'>-			qpn, irq_work-&gt;sub_type);</div><div class='del'>-		hns_roce_set_qps_to_err(irq_work-&gt;hr_dev, qpn);</div><div class='add'>+		ibdev_err(ibdev, "local access violation work queue 0x%x error, sub_event type is: %d\n",</div><div class='add'>+			  irq_work-&gt;queue_num, irq_work-&gt;sub_type);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:</div><div class='del'>-		dev_warn(dev, "SRQ limit reach.\n");</div><div class='add'>+		ibdev_dbg(ibdev, "SRQ limit reach.\n");</div><div class='ctx'> 		break;</div><div class='ctx'> 	case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:</div><div class='del'>-		dev_warn(dev, "SRQ last wqe reach.\n");</div><div class='add'>+		ibdev_dbg(ibdev, "SRQ last wqe reach.\n");</div><div class='ctx'> 		break;</div><div class='ctx'> 	case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:</div><div class='del'>-		dev_err(dev, "SRQ catas error.\n");</div><div class='add'>+		ibdev_err(ibdev, "SRQ catas error.\n");</div><div class='ctx'> 		break;</div><div class='ctx'> 	case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:</div><div class='del'>-		dev_err(dev, "CQ 0x%x access err.\n", cqn);</div><div class='add'>+		ibdev_err(ibdev, "CQ 0x%x access err.\n", irq_work-&gt;queue_num);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:</div><div class='del'>-		dev_warn(dev, "CQ 0x%x overflow\n", cqn);</div><div class='add'>+		ibdev_warn(ibdev, "CQ 0x%x overflow\n", irq_work-&gt;queue_num);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:</div><div class='del'>-		dev_warn(dev, "DB overflow.\n");</div><div class='add'>+		ibdev_warn(ibdev, "DB overflow.\n");</div><div class='add'>+		break;</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_MB:</div><div class='ctx'> 		break;</div><div class='ctx'> 	case HNS_ROCE_EVENT_TYPE_FLR:</div><div class='del'>-		dev_warn(dev, "Function level reset.\n");</div><div class='add'>+		ibdev_warn(ibdev, "function level reset.\n");</div><div class='add'>+		break;</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:</div><div class='add'>+		ibdev_err(ibdev, "xrc domain violation error.\n");</div><div class='add'>+		break;</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:</div><div class='add'>+		ibdev_err(ibdev, "invalid xrceth error.\n");</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ibdev_info(ibdev, "Undefined event %d.\n",</div><div class='add'>+			   irq_work-&gt;event_type);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_irq_work_handle(struct work_struct *work)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_work *irq_work =</div><div class='add'>+				container_of(work, struct hns_roce_work, work);</div><div class='add'>+	struct hns_roce_dev *hr_dev = irq_work-&gt;hr_dev;</div><div class='add'>+	int event_type = irq_work-&gt;event_type;</div><div class='add'>+	u32 queue_num = irq_work-&gt;queue_num;</div><div class='add'>+</div><div class='add'>+	switch (event_type) {</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_PATH_MIG:</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_COMM_EST:</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:</div><div class='add'>+		hns_roce_qp_event(hr_dev, queue_num, event_type);</div><div class='add'>+		break;</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:</div><div class='add'>+		hns_roce_srq_event(hr_dev, queue_num, event_type);</div><div class='add'>+		break;</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:</div><div class='add'>+	case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:</div><div class='add'>+		hns_roce_cq_event(hr_dev, queue_num, event_type);</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	dump_aeqe_log(irq_work);</div><div class='add'>+</div><div class='ctx'> 	kfree(irq_work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev,</div><div class='del'>-				      struct hns_roce_eq *eq,</div><div class='del'>-				      u32 qpn, u32 cqn)</div><div class='add'>+				      struct hns_roce_eq *eq, u32 queue_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_work *irq_work;</div><div class='ctx'> </div><div class='hunk'>@@ -4740,144 +6265,77 @@ static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 	if (!irq_work)</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='del'>-	INIT_WORK(&amp;(irq_work-&gt;work), hns_roce_irq_work_handle);</div><div class='add'>+	INIT_WORK(&amp;irq_work-&gt;work, hns_roce_irq_work_handle);</div><div class='ctx'> 	irq_work-&gt;hr_dev = hr_dev;</div><div class='del'>-	irq_work-&gt;qpn = qpn;</div><div class='del'>-	irq_work-&gt;cqn = cqn;</div><div class='ctx'> 	irq_work-&gt;event_type = eq-&gt;event_type;</div><div class='ctx'> 	irq_work-&gt;sub_type = eq-&gt;sub_type;</div><div class='del'>-	queue_work(hr_dev-&gt;irq_workq, &amp;(irq_work-&gt;work));</div><div class='add'>+	irq_work-&gt;queue_num = queue_num;</div><div class='add'>+	queue_work(hr_dev-&gt;irq_workq, &amp;irq_work-&gt;work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void set_eq_cons_index_v2(struct hns_roce_eq *eq)</div><div class='add'>+static void update_eq_db(struct hns_roce_eq *eq)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = eq-&gt;hr_dev;</div><div class='del'>-	u32 doorbell[2];</div><div class='del'>-</div><div class='del'>-	doorbell[0] = 0;</div><div class='del'>-	doorbell[1] = 0;</div><div class='add'>+	struct hns_roce_v2_db eq_db = {};</div><div class='ctx'> </div><div class='ctx'> 	if (eq-&gt;type_flag == HNS_ROCE_AEQ) {</div><div class='del'>-		roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,</div><div class='del'>-			       HNS_ROCE_V2_EQ_DB_CMD_S,</div><div class='del'>-			       eq-&gt;arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?</div><div class='del'>-			       HNS_ROCE_EQ_DB_CMD_AEQ :</div><div class='del'>-			       HNS_ROCE_EQ_DB_CMD_AEQ_ARMED);</div><div class='add'>+		hr_reg_write(&amp;eq_db, EQ_DB_CMD,</div><div class='add'>+			     eq-&gt;arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?</div><div class='add'>+			     HNS_ROCE_EQ_DB_CMD_AEQ :</div><div class='add'>+			     HNS_ROCE_EQ_DB_CMD_AEQ_ARMED);</div><div class='ctx'> 	} else {</div><div class='del'>-		roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_TAG_M,</div><div class='del'>-			       HNS_ROCE_V2_EQ_DB_TAG_S, eq-&gt;eqn);</div><div class='add'>+		hr_reg_write(&amp;eq_db, EQ_DB_TAG, eq-&gt;eqn);</div><div class='ctx'> </div><div class='del'>-		roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,</div><div class='del'>-			       HNS_ROCE_V2_EQ_DB_CMD_S,</div><div class='del'>-			       eq-&gt;arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?</div><div class='del'>-			       HNS_ROCE_EQ_DB_CMD_CEQ :</div><div class='del'>-			       HNS_ROCE_EQ_DB_CMD_CEQ_ARMED);</div><div class='add'>+		hr_reg_write(&amp;eq_db, EQ_DB_CMD,</div><div class='add'>+			     eq-&gt;arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?</div><div class='add'>+			     HNS_ROCE_EQ_DB_CMD_CEQ :</div><div class='add'>+			     HNS_ROCE_EQ_DB_CMD_CEQ_ARMED);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	roce_set_field(doorbell[1], HNS_ROCE_V2_EQ_DB_PARA_M,</div><div class='del'>-		       HNS_ROCE_V2_EQ_DB_PARA_S,</div><div class='del'>-		       (eq-&gt;cons_index &amp; HNS_ROCE_V2_CONS_IDX_M));</div><div class='del'>-</div><div class='del'>-	hns_roce_write64(hr_dev, doorbell, eq-&gt;doorbell);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct hns_roce_aeqe *get_aeqe_v2(struct hns_roce_eq *eq, u32 entry)</div><div class='del'>-{</div><div class='del'>-	u32 buf_chk_sz;</div><div class='del'>-	unsigned long off;</div><div class='add'>+	hr_reg_write(&amp;eq_db, EQ_DB_CI, eq-&gt;cons_index);</div><div class='ctx'> </div><div class='del'>-	buf_chk_sz = 1 &lt;&lt; (eq-&gt;eqe_buf_pg_sz + PAGE_SHIFT);</div><div class='del'>-	off = (entry &amp; (eq-&gt;entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;</div><div class='del'>-</div><div class='del'>-	return (struct hns_roce_aeqe *)((char *)(eq-&gt;buf_list-&gt;buf) +</div><div class='del'>-		off % buf_chk_sz);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct hns_roce_aeqe *mhop_get_aeqe(struct hns_roce_eq *eq, u32 entry)</div><div class='del'>-{</div><div class='del'>-	u32 buf_chk_sz;</div><div class='del'>-	unsigned long off;</div><div class='del'>-</div><div class='del'>-	buf_chk_sz = 1 &lt;&lt; (eq-&gt;eqe_buf_pg_sz + PAGE_SHIFT);</div><div class='del'>-</div><div class='del'>-	off = (entry &amp; (eq-&gt;entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;</div><div class='del'>-</div><div class='del'>-	if (eq-&gt;hop_num == HNS_ROCE_HOP_NUM_0)</div><div class='del'>-		return (struct hns_roce_aeqe *)((u8 *)(eq-&gt;bt_l0) +</div><div class='del'>-			off % buf_chk_sz);</div><div class='del'>-	else</div><div class='del'>-		return (struct hns_roce_aeqe *)((u8 *)</div><div class='del'>-			(eq-&gt;buf[off / buf_chk_sz]) + off % buf_chk_sz);</div><div class='add'>+	hns_roce_write64(hr_dev, (__le32 *)&amp;eq_db, eq-&gt;db_reg);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_aeqe *aeqe;</div><div class='ctx'> </div><div class='del'>-	if (!eq-&gt;hop_num)</div><div class='del'>-		aeqe = get_aeqe_v2(eq, eq-&gt;cons_index);</div><div class='del'>-	else</div><div class='del'>-		aeqe = mhop_get_aeqe(eq, eq-&gt;cons_index);</div><div class='add'>+	aeqe = hns_roce_buf_offset(eq-&gt;mtr.kmem,</div><div class='add'>+				   (eq-&gt;cons_index &amp; (eq-&gt;entries - 1)) *</div><div class='add'>+				   eq-&gt;eqe_size);</div><div class='ctx'> </div><div class='del'>-	return (roce_get_bit(aeqe-&gt;asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^</div><div class='add'>+	return (hr_reg_read(aeqe, AEQE_OWNER) ^</div><div class='ctx'> 		!!(eq-&gt;cons_index &amp; eq-&gt;entries)) ? aeqe : NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,</div><div class='del'>-			       struct hns_roce_eq *eq)</div><div class='add'>+static irqreturn_t hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,</div><div class='add'>+				       struct hns_roce_eq *eq)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	struct hns_roce_aeqe *aeqe;</div><div class='del'>-	int aeqe_found = 0;</div><div class='add'>+	struct hns_roce_aeqe *aeqe = next_aeqe_sw_v2(eq);</div><div class='add'>+	irqreturn_t aeqe_found = IRQ_NONE;</div><div class='add'>+	int num_aeqes = 0;</div><div class='ctx'> 	int event_type;</div><div class='add'>+	u32 queue_num;</div><div class='ctx'> 	int sub_type;</div><div class='del'>-	u32 srqn;</div><div class='del'>-	u32 qpn;</div><div class='del'>-	u32 cqn;</div><div class='del'>-</div><div class='del'>-	while ((aeqe = next_aeqe_sw_v2(eq))) {</div><div class='ctx'> </div><div class='add'>+	while (aeqe &amp;&amp; num_aeqes &lt; HNS_AEQ_POLLING_BUDGET) {</div><div class='ctx'> 		/* Make sure we read AEQ entry after we have checked the</div><div class='ctx'> 		 * ownership bit</div><div class='ctx'> 		 */</div><div class='ctx'> 		dma_rmb();</div><div class='ctx'> </div><div class='del'>-		event_type = roce_get_field(aeqe-&gt;asyn,</div><div class='del'>-					    HNS_ROCE_V2_AEQE_EVENT_TYPE_M,</div><div class='del'>-					    HNS_ROCE_V2_AEQE_EVENT_TYPE_S);</div><div class='del'>-		sub_type = roce_get_field(aeqe-&gt;asyn,</div><div class='del'>-					  HNS_ROCE_V2_AEQE_SUB_TYPE_M,</div><div class='del'>-					  HNS_ROCE_V2_AEQE_SUB_TYPE_S);</div><div class='del'>-		qpn = roce_get_field(aeqe-&gt;event.qp_event.qp,</div><div class='del'>-				     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,</div><div class='del'>-				     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);</div><div class='del'>-		cqn = roce_get_field(aeqe-&gt;event.cq_event.cq,</div><div class='del'>-				     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,</div><div class='del'>-				     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);</div><div class='del'>-		srqn = roce_get_field(aeqe-&gt;event.srq_event.srq,</div><div class='del'>-				     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,</div><div class='del'>-				     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);</div><div class='add'>+		event_type = hr_reg_read(aeqe, AEQE_EVENT_TYPE);</div><div class='add'>+		sub_type = hr_reg_read(aeqe, AEQE_SUB_TYPE);</div><div class='add'>+		queue_num = hr_reg_read(aeqe, AEQE_EVENT_QUEUE_NUM);</div><div class='ctx'> </div><div class='ctx'> 		switch (event_type) {</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_PATH_MIG:</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_COMM_EST:</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:</div><div class='ctx'> 		case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:</div><div class='ctx'> 		case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:</div><div class='ctx'> 		case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:</div><div class='del'>-			hns_roce_qp_event(hr_dev, qpn, event_type);</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:</div><div class='del'>-			hns_roce_srq_event(hr_dev, srqn, event_type);</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:</div><div class='del'>-			hns_roce_cq_event(hr_dev, cqn, event_type);</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:</div><div class='add'>+		case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:</div><div class='add'>+		case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:</div><div class='add'>+			hns_roce_flush_cqe(hr_dev, queue_num);</div><div class='ctx'> 			break;</div><div class='ctx'> 		case HNS_ROCE_EVENT_TYPE_MB:</div><div class='ctx'> 			hns_roce_cmd_event(hr_dev,</div><div class='hunk'>@@ -4885,681 +6343,397 @@ static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 					aeqe-&gt;event.cmd.status,</div><div class='ctx'> 					le64_to_cpu(aeqe-&gt;event.cmd.out_param));</div><div class='ctx'> 			break;</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:</div><div class='del'>-			break;</div><div class='del'>-		case HNS_ROCE_EVENT_TYPE_FLR:</div><div class='del'>-			break;</div><div class='ctx'> 		default:</div><div class='del'>-			dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n",</div><div class='del'>-				event_type, eq-&gt;eqn, eq-&gt;cons_index);</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='ctx'> 		eq-&gt;event_type = event_type;</div><div class='ctx'> 		eq-&gt;sub_type = sub_type;</div><div class='ctx'> 		++eq-&gt;cons_index;</div><div class='del'>-		aeqe_found = 1;</div><div class='add'>+		aeqe_found = IRQ_HANDLED;</div><div class='add'>+		trace_hns_ae_info(event_type, aeqe, eq-&gt;eqe_size);</div><div class='ctx'> </div><div class='del'>-		if (eq-&gt;cons_index &gt; (2 * eq-&gt;entries - 1)) {</div><div class='del'>-			dev_warn(dev, "cons_index overflow, set back to 0.\n");</div><div class='del'>-			eq-&gt;cons_index = 0;</div><div class='del'>-		}</div><div class='del'>-		hns_roce_v2_init_irq_work(hr_dev, eq, qpn, cqn);</div><div class='del'>-	}</div><div class='add'>+		atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_AEQE_CNT]);</div><div class='ctx'> </div><div class='del'>-	set_eq_cons_index_v2(eq);</div><div class='del'>-	return aeqe_found;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct hns_roce_ceqe *get_ceqe_v2(struct hns_roce_eq *eq, u32 entry)</div><div class='del'>-{</div><div class='del'>-	u32 buf_chk_sz;</div><div class='del'>-	unsigned long off;</div><div class='add'>+		hns_roce_v2_init_irq_work(hr_dev, eq, queue_num);</div><div class='ctx'> </div><div class='del'>-	buf_chk_sz = 1 &lt;&lt; (eq-&gt;eqe_buf_pg_sz + PAGE_SHIFT);</div><div class='del'>-	off = (entry &amp; (eq-&gt;entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;</div><div class='del'>-</div><div class='del'>-	return (struct hns_roce_ceqe *)((char *)(eq-&gt;buf_list-&gt;buf) +</div><div class='del'>-		off % buf_chk_sz);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct hns_roce_ceqe *mhop_get_ceqe(struct hns_roce_eq *eq, u32 entry)</div><div class='del'>-{</div><div class='del'>-	u32 buf_chk_sz;</div><div class='del'>-	unsigned long off;</div><div class='del'>-</div><div class='del'>-	buf_chk_sz = 1 &lt;&lt; (eq-&gt;eqe_buf_pg_sz + PAGE_SHIFT);</div><div class='add'>+		aeqe = next_aeqe_sw_v2(eq);</div><div class='add'>+		++num_aeqes;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	off = (entry &amp; (eq-&gt;entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;</div><div class='add'>+	update_eq_db(eq);</div><div class='ctx'> </div><div class='del'>-	if (eq-&gt;hop_num == HNS_ROCE_HOP_NUM_0)</div><div class='del'>-		return (struct hns_roce_ceqe *)((u8 *)(eq-&gt;bt_l0) +</div><div class='del'>-			off % buf_chk_sz);</div><div class='del'>-	else</div><div class='del'>-		return (struct hns_roce_ceqe *)((u8 *)(eq-&gt;buf[off /</div><div class='del'>-			buf_chk_sz]) + off % buf_chk_sz);</div><div class='add'>+	return IRQ_RETVAL(aeqe_found);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_ceqe *ceqe;</div><div class='ctx'> </div><div class='del'>-	if (!eq-&gt;hop_num)</div><div class='del'>-		ceqe = get_ceqe_v2(eq, eq-&gt;cons_index);</div><div class='del'>-	else</div><div class='del'>-		ceqe = mhop_get_ceqe(eq, eq-&gt;cons_index);</div><div class='add'>+	ceqe = hns_roce_buf_offset(eq-&gt;mtr.kmem,</div><div class='add'>+				   (eq-&gt;cons_index &amp; (eq-&gt;entries - 1)) *</div><div class='add'>+				   eq-&gt;eqe_size);</div><div class='ctx'> </div><div class='del'>-	return (!!(roce_get_bit(ceqe-&gt;comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^</div><div class='del'>-		(!!(eq-&gt;cons_index &amp; eq-&gt;entries)) ? ceqe : NULL;</div><div class='add'>+	return (hr_reg_read(ceqe, CEQE_OWNER) ^</div><div class='add'>+		!!(eq-&gt;cons_index &amp; eq-&gt;entries)) ? ceqe : NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev,</div><div class='del'>-			       struct hns_roce_eq *eq)</div><div class='add'>+static irqreturn_t hns_roce_v2_ceq_int(struct hns_roce_eq *eq)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	struct hns_roce_ceqe *ceqe;</div><div class='del'>-	int ceqe_found = 0;</div><div class='del'>-	u32 cqn;</div><div class='add'>+	queue_work(system_bh_wq, &amp;eq-&gt;work);</div><div class='ctx'> </div><div class='del'>-	while ((ceqe = next_ceqe_sw_v2(eq))) {</div><div class='del'>-</div><div class='del'>-		/* Make sure we read CEQ entry after we have checked the</div><div class='del'>-		 * ownership bit</div><div class='del'>-		 */</div><div class='del'>-		dma_rmb();</div><div class='del'>-</div><div class='del'>-		cqn = roce_get_field(ceqe-&gt;comp,</div><div class='del'>-				     HNS_ROCE_V2_CEQE_COMP_CQN_M,</div><div class='del'>-				     HNS_ROCE_V2_CEQE_COMP_CQN_S);</div><div class='del'>-</div><div class='del'>-		hns_roce_cq_completion(hr_dev, cqn);</div><div class='del'>-</div><div class='del'>-		++eq-&gt;cons_index;</div><div class='del'>-		ceqe_found = 1;</div><div class='del'>-</div><div class='del'>-		if (eq-&gt;cons_index &gt; (2 * eq-&gt;entries - 1)) {</div><div class='del'>-			dev_warn(dev, "cons_index overflow, set back to 0.\n");</div><div class='del'>-			eq-&gt;cons_index = 0;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	set_eq_cons_index_v2(eq);</div><div class='del'>-</div><div class='del'>-	return ceqe_found;</div><div class='add'>+	return IRQ_HANDLED;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_eq *eq = eq_ptr;</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = eq-&gt;hr_dev;</div><div class='del'>-	int int_work = 0;</div><div class='add'>+	irqreturn_t int_work;</div><div class='ctx'> </div><div class='ctx'> 	if (eq-&gt;type_flag == HNS_ROCE_CEQ)</div><div class='ctx'> 		/* Completion event interrupt */</div><div class='del'>-		int_work = hns_roce_v2_ceq_int(hr_dev, eq);</div><div class='add'>+		int_work = hns_roce_v2_ceq_int(eq);</div><div class='ctx'> 	else</div><div class='del'>-		/* Asychronous event interrupt */</div><div class='add'>+		/* Asynchronous event interrupt */</div><div class='ctx'> 		int_work = hns_roce_v2_aeq_int(hr_dev, eq);</div><div class='ctx'> </div><div class='ctx'> 	return IRQ_RETVAL(int_work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id)</div><div class='add'>+static irqreturn_t abnormal_interrupt_basic(struct hns_roce_dev *hr_dev,</div><div class='add'>+					    u32 int_st)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_dev *hr_dev = dev_id;</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	int int_work = 0;</div><div class='del'>-	u32 int_st;</div><div class='add'>+	struct pci_dev *pdev = hr_dev-&gt;pci_dev;</div><div class='add'>+	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);</div><div class='add'>+	const struct hnae3_ae_ops *ops = ae_dev-&gt;ops;</div><div class='add'>+	enum hnae3_reset_type reset_type;</div><div class='add'>+	irqreturn_t int_work = IRQ_NONE;</div><div class='ctx'> 	u32 int_en;</div><div class='ctx'> </div><div class='del'>-	/* Abnormal interrupt */</div><div class='del'>-	int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG);</div><div class='ctx'> 	int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG);</div><div class='ctx'> </div><div class='del'>-	if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {</div><div class='del'>-		struct pci_dev *pdev = hr_dev-&gt;pci_dev;</div><div class='del'>-		struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);</div><div class='del'>-		const struct hnae3_ae_ops *ops = ae_dev-&gt;ops;</div><div class='add'>+	if (int_st &amp; BIT(HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "AEQ overflow!\n");</div><div class='ctx'> </div><div class='del'>-		dev_err(dev, "AEQ overflow!\n");</div><div class='add'>+		roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG,</div><div class='add'>+			   1 &lt;&lt; HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S);</div><div class='ctx'> </div><div class='del'>-		roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S, 1);</div><div class='del'>-		roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);</div><div class='add'>+		reset_type = hr_dev-&gt;is_vf ?</div><div class='add'>+			     HNAE3_VF_FUNC_RESET : HNAE3_FUNC_RESET;</div><div class='ctx'> </div><div class='ctx'> 		/* Set reset level for reset_event() */</div><div class='ctx'> 		if (ops-&gt;set_default_reset_request)</div><div class='del'>-			ops-&gt;set_default_reset_request(ae_dev,</div><div class='del'>-						       HNAE3_FUNC_RESET);</div><div class='add'>+			ops-&gt;set_default_reset_request(ae_dev, reset_type);</div><div class='ctx'> 		if (ops-&gt;reset_event)</div><div class='ctx'> 			ops-&gt;reset_event(pdev, NULL);</div><div class='ctx'> </div><div class='del'>-		roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);</div><div class='add'>+		int_en |= 1 &lt;&lt; HNS_ROCE_V2_VF_ABN_INT_EN_S;</div><div class='ctx'> 		roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);</div><div class='ctx'> </div><div class='del'>-		int_work = 1;</div><div class='del'>-	} else if (roce_get_bit(int_st,	HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) {</div><div class='del'>-		dev_err(dev, "BUS ERR!\n");</div><div class='add'>+		int_work = IRQ_HANDLED;</div><div class='add'>+	} else {</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "there is no basic abn irq found.\n");</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S, 1);</div><div class='del'>-		roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);</div><div class='add'>+	return IRQ_RETVAL(int_work);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);</div><div class='del'>-		roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);</div><div class='add'>+static int fmea_ram_ecc_query(struct hns_roce_dev *hr_dev,</div><div class='add'>+			       struct fmea_ram_ecc *ecc_info)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_cmq_desc desc;</div><div class='add'>+	struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_QUERY_RAM_ECC, true);</div><div class='add'>+	ret = hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='del'>-		int_work = 1;</div><div class='del'>-	} else if (roce_get_bit(int_st,	HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) {</div><div class='del'>-		dev_err(dev, "OTHER ERR!\n");</div><div class='add'>+	ecc_info-&gt;is_ecc_err = hr_reg_read(req, QUERY_RAM_ECC_1BIT_ERR);</div><div class='add'>+	ecc_info-&gt;res_type = hr_reg_read(req, QUERY_RAM_ECC_RES_TYPE);</div><div class='add'>+	ecc_info-&gt;index = hr_reg_read(req, QUERY_RAM_ECC_TAG);</div><div class='ctx'> </div><div class='del'>-		roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S, 1);</div><div class='del'>-		roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);</div><div class='del'>-		roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);</div><div class='add'>+static int fmea_recover_gmv(struct hns_roce_dev *hr_dev, u32 idx)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_cmq_desc desc;</div><div class='add'>+	struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;</div><div class='add'>+	u32 addr_upper;</div><div class='add'>+	u32 addr_low;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-		int_work = 1;</div><div class='del'>-	} else</div><div class='del'>-		dev_err(dev, "There is no abnormal irq found!\n");</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_OPC_CFG_GMV_BT, true);</div><div class='add'>+	hr_reg_write(req, CFG_GMV_BT_IDX, idx);</div><div class='ctx'> </div><div class='del'>-	return IRQ_RETVAL(int_work);</div><div class='add'>+	ret = hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(hr_dev-&gt;dev,</div><div class='add'>+			"failed to execute cmd to read gmv, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	addr_low =  hr_reg_read(req, CFG_GMV_BT_BA_L);</div><div class='add'>+	addr_upper = hr_reg_read(req, CFG_GMV_BT_BA_H);</div><div class='add'>+</div><div class='add'>+	hns_roce_cmq_setup_basic_desc(&amp;desc, HNS_ROCE_OPC_CFG_GMV_BT, false);</div><div class='add'>+	hr_reg_write(req, CFG_GMV_BT_BA_L, addr_low);</div><div class='add'>+	hr_reg_write(req, CFG_GMV_BT_BA_H, addr_upper);</div><div class='add'>+	hr_reg_write(req, CFG_GMV_BT_IDX, idx);</div><div class='add'>+</div><div class='add'>+	return hns_roce_cmq_send(hr_dev, &amp;desc, 1);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev,</div><div class='del'>-					int eq_num, int enable_flag)</div><div class='add'>+static u64 fmea_get_ram_res_addr(u32 res_type, __le64 *data)</div><div class='ctx'> {</div><div class='del'>-	int i;</div><div class='add'>+	if (res_type == ECC_RESOURCE_QPC_TIMER ||</div><div class='add'>+	    res_type == ECC_RESOURCE_CQC_TIMER ||</div><div class='add'>+	    res_type == ECC_RESOURCE_SCCC)</div><div class='add'>+		return le64_to_cpu(*data);</div><div class='ctx'> </div><div class='del'>-	if (enable_flag == EQ_ENABLE) {</div><div class='del'>-		for (i = 0; i &lt; eq_num; i++)</div><div class='del'>-			roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +</div><div class='del'>-				   i * EQ_REG_OFFSET,</div><div class='del'>-				   HNS_ROCE_V2_VF_EVENT_INT_EN_M);</div><div class='add'>+	return le64_to_cpu(*data) &lt;&lt; HNS_HW_PAGE_SHIFT;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,</div><div class='del'>-			   HNS_ROCE_V2_VF_ABN_INT_EN_M);</div><div class='del'>-		roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,</div><div class='del'>-			   HNS_ROCE_V2_VF_ABN_INT_CFG_M);</div><div class='del'>-	} else {</div><div class='del'>-		for (i = 0; i &lt; eq_num; i++)</div><div class='del'>-			roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +</div><div class='del'>-				   i * EQ_REG_OFFSET,</div><div class='del'>-				   HNS_ROCE_V2_VF_EVENT_INT_EN_M &amp; 0x0);</div><div class='add'>+static int fmea_recover_others(struct hns_roce_dev *hr_dev, u32 res_type,</div><div class='add'>+			       u32 index)</div><div class='add'>+{</div><div class='add'>+	u8 write_bt0_op = fmea_ram_res[res_type].write_bt0_op;</div><div class='add'>+	u8 read_bt0_op = fmea_ram_res[res_type].read_bt0_op;</div><div class='add'>+	struct hns_roce_cmd_mailbox *mailbox;</div><div class='add'>+	u64 addr;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='add'>+	if (IS_ERR(mailbox))</div><div class='add'>+		return PTR_ERR(mailbox);</div><div class='ctx'> </div><div class='del'>-		roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,</div><div class='del'>-			   HNS_ROCE_V2_VF_ABN_INT_EN_M &amp; 0x0);</div><div class='del'>-		roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,</div><div class='del'>-			   HNS_ROCE_V2_VF_ABN_INT_CFG_M &amp; 0x0);</div><div class='add'>+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox-&gt;dma, read_bt0_op, index);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(hr_dev-&gt;dev,</div><div class='add'>+			"failed to execute cmd to read fmea ram, ret = %d.\n",</div><div class='add'>+			ret);</div><div class='add'>+		goto out;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	addr = fmea_get_ram_res_addr(res_type, mailbox-&gt;buf);</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_cmd_mbox(hr_dev, addr, 0, write_bt0_op, index);</div><div class='add'>+	if (ret)</div><div class='add'>+		dev_err(hr_dev-&gt;dev,</div><div class='add'>+			"failed to execute cmd to write fmea ram, ret = %d.\n",</div><div class='add'>+			ret);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn)</div><div class='add'>+static void fmea_ram_ecc_recover(struct hns_roce_dev *hr_dev,</div><div class='add'>+				 struct fmea_ram_ecc *ecc_info)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	u32 res_type = ecc_info-&gt;res_type;</div><div class='add'>+	u32 index = ecc_info-&gt;index;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	if (eqn &lt; hr_dev-&gt;caps.num_comp_vectors)</div><div class='del'>-		ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn &amp; HNS_ROCE_V2_EQN_M,</div><div class='del'>-					0, HNS_ROCE_CMD_DESTROY_CEQC,</div><div class='del'>-					HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='add'>+	BUILD_BUG_ON(ARRAY_SIZE(fmea_ram_res) != ECC_RESOURCE_COUNT);</div><div class='add'>+</div><div class='add'>+	if (res_type &gt;= ECC_RESOURCE_COUNT) {</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "unsupported fmea ram ecc type %u.\n",</div><div class='add'>+			res_type);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (res_type == ECC_RESOURCE_GMV)</div><div class='add'>+		ret = fmea_recover_gmv(hr_dev, index);</div><div class='ctx'> 	else</div><div class='del'>-		ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn &amp; HNS_ROCE_V2_EQN_M,</div><div class='del'>-					0, HNS_ROCE_CMD_DESTROY_AEQC,</div><div class='del'>-					HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='add'>+		ret = fmea_recover_others(hr_dev, res_type, index);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);</div><div class='add'>+		dev_err(hr_dev-&gt;dev,</div><div class='add'>+			"failed to recover %s, index = %u, ret = %d.\n",</div><div class='add'>+			fmea_ram_res[res_type].name, index, ret);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_mhop_free_eq(struct hns_roce_dev *hr_dev,</div><div class='del'>-				  struct hns_roce_eq *eq)</div><div class='add'>+static void fmea_ram_ecc_work(struct work_struct *ecc_work)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	u64 idx;</div><div class='del'>-	u64 size;</div><div class='del'>-	u32 buf_chk_sz;</div><div class='del'>-	u32 bt_chk_sz;</div><div class='del'>-	u32 mhop_num;</div><div class='del'>-	int eqe_alloc;</div><div class='del'>-	int i = 0;</div><div class='del'>-	int j = 0;</div><div class='del'>-</div><div class='del'>-	mhop_num = hr_dev-&gt;caps.eqe_hop_num;</div><div class='del'>-	buf_chk_sz = 1 &lt;&lt; (hr_dev-&gt;caps.eqe_buf_pg_sz + PAGE_SHIFT);</div><div class='del'>-	bt_chk_sz = 1 &lt;&lt; (hr_dev-&gt;caps.eqe_ba_pg_sz + PAGE_SHIFT);</div><div class='add'>+	struct hns_roce_dev *hr_dev =</div><div class='add'>+		container_of(ecc_work, struct hns_roce_dev, ecc_work);</div><div class='add'>+	struct fmea_ram_ecc ecc_info = {};</div><div class='ctx'> </div><div class='del'>-	/* hop_num = 0 */</div><div class='del'>-	if (mhop_num == HNS_ROCE_HOP_NUM_0) {</div><div class='del'>-		dma_free_coherent(dev, (unsigned int)(eq-&gt;entries *</div><div class='del'>-				  eq-&gt;eqe_size), eq-&gt;bt_l0, eq-&gt;l0_dma);</div><div class='add'>+	if (fmea_ram_ecc_query(hr_dev, &amp;ecc_info)) {</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "failed to query fmea ram ecc.\n");</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* hop_num = 1 or hop = 2 */</div><div class='del'>-	dma_free_coherent(dev, bt_chk_sz, eq-&gt;bt_l0, eq-&gt;l0_dma);</div><div class='del'>-	if (mhop_num == 1) {</div><div class='del'>-		for (i = 0; i &lt; eq-&gt;l0_last_num; i++) {</div><div class='del'>-			if (i == eq-&gt;l0_last_num - 1) {</div><div class='del'>-				eqe_alloc = i * (buf_chk_sz / eq-&gt;eqe_size);</div><div class='del'>-				size = (eq-&gt;entries - eqe_alloc) * eq-&gt;eqe_size;</div><div class='del'>-				dma_free_coherent(dev, size, eq-&gt;buf[i],</div><div class='del'>-						  eq-&gt;buf_dma[i]);</div><div class='del'>-				break;</div><div class='del'>-			}</div><div class='del'>-			dma_free_coherent(dev, buf_chk_sz, eq-&gt;buf[i],</div><div class='del'>-					  eq-&gt;buf_dma[i]);</div><div class='del'>-		}</div><div class='del'>-	} else if (mhop_num == 2) {</div><div class='del'>-		for (i = 0; i &lt; eq-&gt;l0_last_num; i++) {</div><div class='del'>-			dma_free_coherent(dev, bt_chk_sz, eq-&gt;bt_l1[i],</div><div class='del'>-					  eq-&gt;l1_dma[i]);</div><div class='del'>-</div><div class='del'>-			for (j = 0; j &lt; bt_chk_sz / 8; j++) {</div><div class='del'>-				idx = i * (bt_chk_sz / 8) + j;</div><div class='del'>-				if ((i == eq-&gt;l0_last_num - 1)</div><div class='del'>-				     &amp;&amp; j == eq-&gt;l1_last_num - 1) {</div><div class='del'>-					eqe_alloc = (buf_chk_sz / eq-&gt;eqe_size)</div><div class='del'>-						    * idx;</div><div class='del'>-					size = (eq-&gt;entries - eqe_alloc)</div><div class='del'>-						* eq-&gt;eqe_size;</div><div class='del'>-					dma_free_coherent(dev, size,</div><div class='del'>-							  eq-&gt;buf[idx],</div><div class='del'>-							  eq-&gt;buf_dma[idx]);</div><div class='del'>-					break;</div><div class='del'>-				}</div><div class='del'>-				dma_free_coherent(dev, buf_chk_sz, eq-&gt;buf[idx],</div><div class='del'>-						  eq-&gt;buf_dma[idx]);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='add'>+	if (!ecc_info.is_ecc_err) {</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "there is no fmea ram ecc err found.\n");</div><div class='add'>+		return;</div><div class='ctx'> 	}</div><div class='del'>-	kfree(eq-&gt;buf_dma);</div><div class='del'>-	kfree(eq-&gt;buf);</div><div class='del'>-	kfree(eq-&gt;l1_dma);</div><div class='del'>-	kfree(eq-&gt;bt_l1);</div><div class='del'>-	eq-&gt;buf_dma = NULL;</div><div class='del'>-	eq-&gt;buf = NULL;</div><div class='del'>-	eq-&gt;l1_dma = NULL;</div><div class='del'>-	eq-&gt;bt_l1 = NULL;</div><div class='add'>+</div><div class='add'>+	fmea_ram_ecc_recover(hr_dev, &amp;ecc_info);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_v2_free_eq(struct hns_roce_dev *hr_dev,</div><div class='del'>-				struct hns_roce_eq *eq)</div><div class='add'>+static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id)</div><div class='ctx'> {</div><div class='del'>-	u32 buf_chk_sz;</div><div class='add'>+	struct hns_roce_dev *hr_dev = dev_id;</div><div class='add'>+	irqreturn_t int_work = IRQ_NONE;</div><div class='add'>+	u32 int_st;</div><div class='ctx'> </div><div class='del'>-	buf_chk_sz = 1 &lt;&lt; (eq-&gt;eqe_buf_pg_sz + PAGE_SHIFT);</div><div class='add'>+	int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG);</div><div class='ctx'> </div><div class='del'>-	if (hr_dev-&gt;caps.eqe_hop_num) {</div><div class='del'>-		hns_roce_mhop_free_eq(hr_dev, eq);</div><div class='del'>-		return;</div><div class='add'>+	if (int_st) {</div><div class='add'>+		int_work = abnormal_interrupt_basic(hr_dev, int_st);</div><div class='add'>+	} else if (hr_dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09) {</div><div class='add'>+		queue_work(hr_dev-&gt;irq_workq, &amp;hr_dev-&gt;ecc_work);</div><div class='add'>+		int_work = IRQ_HANDLED;</div><div class='add'>+	} else {</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "there is no abnormal irq found.\n");</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (eq-&gt;buf_list)</div><div class='del'>-		dma_free_coherent(hr_dev-&gt;dev, buf_chk_sz,</div><div class='del'>-				  eq-&gt;buf_list-&gt;buf, eq-&gt;buf_list-&gt;map);</div><div class='add'>+	return IRQ_RETVAL(int_work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,</div><div class='del'>-				struct hns_roce_eq *eq,</div><div class='del'>-				void *mb_buf)</div><div class='add'>+static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev,</div><div class='add'>+					int eq_num, u32 enable_flag)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_eq_context *eqc;</div><div class='add'>+	int i;</div><div class='ctx'> </div><div class='del'>-	eqc = mb_buf;</div><div class='del'>-	memset(eqc, 0, sizeof(struct hns_roce_eq_context));</div><div class='add'>+	for (i = 0; i &lt; eq_num; i++)</div><div class='add'>+		roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +</div><div class='add'>+			   i * EQ_REG_OFFSET, enable_flag);</div><div class='ctx'> </div><div class='del'>-	/* init eqc */</div><div class='del'>-	eq-&gt;doorbell = hr_dev-&gt;reg_base + ROCEE_VF_EQ_DB_CFG0_REG;</div><div class='del'>-	eq-&gt;hop_num = hr_dev-&gt;caps.eqe_hop_num;</div><div class='del'>-	eq-&gt;cons_index = 0;</div><div class='del'>-	eq-&gt;over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;</div><div class='del'>-	eq-&gt;coalesce = HNS_ROCE_V2_EQ_COALESCE_0;</div><div class='del'>-	eq-&gt;arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;</div><div class='del'>-	eq-&gt;eqe_ba_pg_sz = hr_dev-&gt;caps.eqe_ba_pg_sz;</div><div class='del'>-	eq-&gt;eqe_buf_pg_sz = hr_dev-&gt;caps.eqe_buf_pg_sz;</div><div class='del'>-	eq-&gt;shift = ilog2((unsigned int)eq-&gt;entries);</div><div class='add'>+	roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, enable_flag);</div><div class='add'>+	roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, enable_flag);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (!eq-&gt;hop_num)</div><div class='del'>-		eq-&gt;eqe_ba = eq-&gt;buf_list-&gt;map;</div><div class='del'>-	else</div><div class='del'>-		eq-&gt;eqe_ba = eq-&gt;l0_dma;</div><div class='del'>-</div><div class='del'>-	/* set eqc state */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_4,</div><div class='del'>-		       HNS_ROCE_EQC_EQ_ST_M,</div><div class='del'>-		       HNS_ROCE_EQC_EQ_ST_S,</div><div class='del'>-		       HNS_ROCE_V2_EQ_STATE_VALID);</div><div class='del'>-</div><div class='del'>-	/* set eqe hop num */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_4,</div><div class='del'>-		       HNS_ROCE_EQC_HOP_NUM_M,</div><div class='del'>-		       HNS_ROCE_EQC_HOP_NUM_S, eq-&gt;hop_num);</div><div class='del'>-</div><div class='del'>-	/* set eqc over_ignore */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_4,</div><div class='del'>-		       HNS_ROCE_EQC_OVER_IGNORE_M,</div><div class='del'>-		       HNS_ROCE_EQC_OVER_IGNORE_S, eq-&gt;over_ignore);</div><div class='del'>-</div><div class='del'>-	/* set eqc coalesce */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_4,</div><div class='del'>-		       HNS_ROCE_EQC_COALESCE_M,</div><div class='del'>-		       HNS_ROCE_EQC_COALESCE_S, eq-&gt;coalesce);</div><div class='del'>-</div><div class='del'>-	/* set eqc arm_state */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_4,</div><div class='del'>-		       HNS_ROCE_EQC_ARM_ST_M,</div><div class='del'>-		       HNS_ROCE_EQC_ARM_ST_S, eq-&gt;arm_st);</div><div class='del'>-</div><div class='del'>-	/* set eqn */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_4,</div><div class='del'>-		       HNS_ROCE_EQC_EQN_M,</div><div class='del'>-		       HNS_ROCE_EQC_EQN_S, eq-&gt;eqn);</div><div class='del'>-</div><div class='del'>-	/* set eqe_cnt */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_4,</div><div class='del'>-		       HNS_ROCE_EQC_EQE_CNT_M,</div><div class='del'>-		       HNS_ROCE_EQC_EQE_CNT_S,</div><div class='del'>-		       HNS_ROCE_EQ_INIT_EQE_CNT);</div><div class='del'>-</div><div class='del'>-	/* set eqe_ba_pg_sz */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_8,</div><div class='del'>-		       HNS_ROCE_EQC_BA_PG_SZ_M,</div><div class='del'>-		       HNS_ROCE_EQC_BA_PG_SZ_S,</div><div class='del'>-		       eq-&gt;eqe_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-</div><div class='del'>-	/* set eqe_buf_pg_sz */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_8,</div><div class='del'>-		       HNS_ROCE_EQC_BUF_PG_SZ_M,</div><div class='del'>-		       HNS_ROCE_EQC_BUF_PG_SZ_S,</div><div class='del'>-		       eq-&gt;eqe_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-</div><div class='del'>-	/* set eq_producer_idx */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_8,</div><div class='del'>-		       HNS_ROCE_EQC_PROD_INDX_M,</div><div class='del'>-		       HNS_ROCE_EQC_PROD_INDX_S,</div><div class='del'>-		       HNS_ROCE_EQ_INIT_PROD_IDX);</div><div class='del'>-</div><div class='del'>-	/* set eq_max_cnt */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_12,</div><div class='del'>-		       HNS_ROCE_EQC_MAX_CNT_M,</div><div class='del'>-		       HNS_ROCE_EQC_MAX_CNT_S, eq-&gt;eq_max_cnt);</div><div class='del'>-</div><div class='del'>-	/* set eq_period */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_12,</div><div class='del'>-		       HNS_ROCE_EQC_PERIOD_M,</div><div class='del'>-		       HNS_ROCE_EQC_PERIOD_S, eq-&gt;eq_period);</div><div class='del'>-</div><div class='del'>-	/* set eqe_report_timer */</div><div class='del'>-	roce_set_field(eqc-&gt;eqe_report_timer,</div><div class='del'>-		       HNS_ROCE_EQC_REPORT_TIMER_M,</div><div class='del'>-		       HNS_ROCE_EQC_REPORT_TIMER_S,</div><div class='del'>-		       HNS_ROCE_EQ_INIT_REPORT_TIMER);</div><div class='del'>-</div><div class='del'>-	/* set eqe_ba [34:3] */</div><div class='del'>-	roce_set_field(eqc-&gt;eqe_ba0,</div><div class='del'>-		       HNS_ROCE_EQC_EQE_BA_L_M,</div><div class='del'>-		       HNS_ROCE_EQC_EQE_BA_L_S, eq-&gt;eqe_ba &gt;&gt; 3);</div><div class='del'>-</div><div class='del'>-	/* set eqe_ba [64:35] */</div><div class='del'>-	roce_set_field(eqc-&gt;eqe_ba1,</div><div class='del'>-		       HNS_ROCE_EQC_EQE_BA_H_M,</div><div class='del'>-		       HNS_ROCE_EQC_EQE_BA_H_S, eq-&gt;eqe_ba &gt;&gt; 35);</div><div class='del'>-</div><div class='del'>-	/* set eq shift */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_28,</div><div class='del'>-		       HNS_ROCE_EQC_SHIFT_M,</div><div class='del'>-		       HNS_ROCE_EQC_SHIFT_S, eq-&gt;shift);</div><div class='del'>-</div><div class='del'>-	/* set eq MSI_IDX */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_28,</div><div class='del'>-		       HNS_ROCE_EQC_MSI_INDX_M,</div><div class='del'>-		       HNS_ROCE_EQC_MSI_INDX_S,</div><div class='del'>-		       HNS_ROCE_EQ_INIT_MSI_IDX);</div><div class='del'>-</div><div class='del'>-	/* set cur_eqe_ba [27:12] */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_28,</div><div class='del'>-		       HNS_ROCE_EQC_CUR_EQE_BA_L_M,</div><div class='del'>-		       HNS_ROCE_EQC_CUR_EQE_BA_L_S, eq-&gt;cur_eqe_ba &gt;&gt; 12);</div><div class='del'>-</div><div class='del'>-	/* set cur_eqe_ba [59:28] */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_32,</div><div class='del'>-		       HNS_ROCE_EQC_CUR_EQE_BA_M_M,</div><div class='del'>-		       HNS_ROCE_EQC_CUR_EQE_BA_M_S, eq-&gt;cur_eqe_ba &gt;&gt; 28);</div><div class='del'>-</div><div class='del'>-	/* set cur_eqe_ba [63:60] */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_36,</div><div class='del'>-		       HNS_ROCE_EQC_CUR_EQE_BA_H_M,</div><div class='del'>-		       HNS_ROCE_EQC_CUR_EQE_BA_H_S, eq-&gt;cur_eqe_ba &gt;&gt; 60);</div><div class='del'>-</div><div class='del'>-	/* set eq consumer idx */</div><div class='del'>-	roce_set_field(eqc-&gt;byte_36,</div><div class='del'>-		       HNS_ROCE_EQC_CONS_INDX_M,</div><div class='del'>-		       HNS_ROCE_EQC_CONS_INDX_S,</div><div class='del'>-		       HNS_ROCE_EQ_INIT_CONS_IDX);</div><div class='del'>-</div><div class='del'>-	/* set nex_eqe_ba[43:12] */</div><div class='del'>-	roce_set_field(eqc-&gt;nxt_eqe_ba0,</div><div class='del'>-		       HNS_ROCE_EQC_NXT_EQE_BA_L_M,</div><div class='del'>-		       HNS_ROCE_EQC_NXT_EQE_BA_L_S, eq-&gt;nxt_eqe_ba &gt;&gt; 12);</div><div class='del'>-</div><div class='del'>-	/* set nex_eqe_ba[63:44] */</div><div class='del'>-	roce_set_field(eqc-&gt;nxt_eqe_ba1,</div><div class='del'>-		       HNS_ROCE_EQC_NXT_EQE_BA_H_M,</div><div class='del'>-		       HNS_ROCE_EQC_NXT_EQE_BA_H_S, eq-&gt;nxt_eqe_ba &gt;&gt; 44);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev,</div><div class='del'>-				  struct hns_roce_eq *eq)</div><div class='add'>+static void free_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	int eq_alloc_done = 0;</div><div class='del'>-	int eq_buf_cnt = 0;</div><div class='del'>-	int eqe_alloc;</div><div class='del'>-	u32 buf_chk_sz;</div><div class='del'>-	u32 bt_chk_sz;</div><div class='del'>-	u32 mhop_num;</div><div class='del'>-	u64 size;</div><div class='del'>-	u64 idx;</div><div class='del'>-	int ba_num;</div><div class='del'>-	int bt_num;</div><div class='del'>-	int record_i;</div><div class='del'>-	int record_j;</div><div class='del'>-	int i = 0;</div><div class='del'>-	int j = 0;</div><div class='add'>+	hns_roce_mtr_destroy(hr_dev, &amp;eq-&gt;mtr);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	mhop_num = hr_dev-&gt;caps.eqe_hop_num;</div><div class='del'>-	buf_chk_sz = 1 &lt;&lt; (hr_dev-&gt;caps.eqe_buf_pg_sz + PAGE_SHIFT);</div><div class='del'>-	bt_chk_sz = 1 &lt;&lt; (hr_dev-&gt;caps.eqe_ba_pg_sz + PAGE_SHIFT);</div><div class='add'>+static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev,</div><div class='add'>+				    struct hns_roce_eq *eq)</div><div class='add'>+{</div><div class='add'>+	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	int eqn = eq-&gt;eqn;</div><div class='add'>+	int ret;</div><div class='add'>+	u8 cmd;</div><div class='ctx'> </div><div class='del'>-	ba_num = (PAGE_ALIGN(eq-&gt;entries * eq-&gt;eqe_size) + buf_chk_sz - 1)</div><div class='del'>-		  / buf_chk_sz;</div><div class='del'>-	bt_num = (ba_num + bt_chk_sz / 8 - 1) / (bt_chk_sz / 8);</div><div class='add'>+	if (eqn &lt; hr_dev-&gt;caps.num_comp_vectors)</div><div class='add'>+		cmd = HNS_ROCE_CMD_DESTROY_CEQC;</div><div class='add'>+	else</div><div class='add'>+		cmd = HNS_ROCE_CMD_DESTROY_AEQC;</div><div class='ctx'> </div><div class='del'>-	/* hop_num = 0 */</div><div class='del'>-	if (mhop_num == HNS_ROCE_HOP_NUM_0) {</div><div class='del'>-		if (eq-&gt;entries &gt; buf_chk_sz / eq-&gt;eqe_size) {</div><div class='del'>-			dev_err(dev, "eq entries %d is larger than buf_pg_sz!",</div><div class='del'>-				eq-&gt;entries);</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-		eq-&gt;bt_l0 = dma_alloc_coherent(dev, eq-&gt;entries * eq-&gt;eqe_size,</div><div class='del'>-					       &amp;(eq-&gt;l0_dma), GFP_KERNEL);</div><div class='del'>-		if (!eq-&gt;bt_l0)</div><div class='del'>-			return -ENOMEM;</div><div class='add'>+	ret = hns_roce_destroy_hw_ctx(hr_dev, cmd, eqn &amp; HNS_ROCE_V2_EQN_M);</div><div class='add'>+	if (ret)</div><div class='add'>+		dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);</div><div class='ctx'> </div><div class='del'>-		eq-&gt;cur_eqe_ba = eq-&gt;l0_dma;</div><div class='del'>-		eq-&gt;nxt_eqe_ba = 0;</div><div class='add'>+	free_eq_buf(hr_dev, eq);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		memset(eq-&gt;bt_l0, 0, eq-&gt;entries * eq-&gt;eqe_size);</div><div class='add'>+static void init_eq_config(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)</div><div class='add'>+{</div><div class='add'>+	eq-&gt;db_reg = hr_dev-&gt;reg_base + ROCEE_VF_EQ_DB_CFG0_REG;</div><div class='add'>+	eq-&gt;cons_index = 0;</div><div class='add'>+	eq-&gt;over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;</div><div class='add'>+	eq-&gt;coalesce = HNS_ROCE_V2_EQ_COALESCE_0;</div><div class='add'>+	eq-&gt;arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;</div><div class='add'>+	eq-&gt;shift = ilog2((unsigned int)eq-&gt;entries);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='add'>+static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq,</div><div class='add'>+		      void *mb_buf)</div><div class='add'>+{</div><div class='add'>+	u64 eqe_ba[MTT_MIN_COUNT] = { 0 };</div><div class='add'>+	struct hns_roce_eq_context *eqc;</div><div class='add'>+	u64 bt_ba = 0;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	eq-&gt;buf_dma = kcalloc(ba_num, sizeof(*eq-&gt;buf_dma), GFP_KERNEL);</div><div class='del'>-	if (!eq-&gt;buf_dma)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	eq-&gt;buf = kcalloc(ba_num, sizeof(*eq-&gt;buf), GFP_KERNEL);</div><div class='del'>-	if (!eq-&gt;buf)</div><div class='del'>-		goto err_kcalloc_buf;</div><div class='del'>-</div><div class='del'>-	if (mhop_num == 2) {</div><div class='del'>-		eq-&gt;l1_dma = kcalloc(bt_num, sizeof(*eq-&gt;l1_dma), GFP_KERNEL);</div><div class='del'>-		if (!eq-&gt;l1_dma)</div><div class='del'>-			goto err_kcalloc_l1_dma;</div><div class='del'>-</div><div class='del'>-		eq-&gt;bt_l1 = kcalloc(bt_num, sizeof(*eq-&gt;bt_l1), GFP_KERNEL);</div><div class='del'>-		if (!eq-&gt;bt_l1)</div><div class='del'>-			goto err_kcalloc_bt_l1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* alloc L0 BT */</div><div class='del'>-	eq-&gt;bt_l0 = dma_alloc_coherent(dev, bt_chk_sz, &amp;eq-&gt;l0_dma, GFP_KERNEL);</div><div class='del'>-	if (!eq-&gt;bt_l0)</div><div class='del'>-		goto err_dma_alloc_l0;</div><div class='del'>-</div><div class='del'>-	if (mhop_num == 1) {</div><div class='del'>-		if (ba_num &gt; (bt_chk_sz / 8))</div><div class='del'>-			dev_err(dev, "ba_num %d is too large for 1 hop\n",</div><div class='del'>-				ba_num);</div><div class='del'>-</div><div class='del'>-		/* alloc buf */</div><div class='del'>-		for (i = 0; i &lt; bt_chk_sz / 8; i++) {</div><div class='del'>-			if (eq_buf_cnt + 1 &lt; ba_num) {</div><div class='del'>-				size = buf_chk_sz;</div><div class='del'>-			} else {</div><div class='del'>-				eqe_alloc = i * (buf_chk_sz / eq-&gt;eqe_size);</div><div class='del'>-				size = (eq-&gt;entries - eqe_alloc) * eq-&gt;eqe_size;</div><div class='del'>-			}</div><div class='del'>-			eq-&gt;buf[i] = dma_alloc_coherent(dev, size,</div><div class='del'>-							&amp;(eq-&gt;buf_dma[i]),</div><div class='del'>-							GFP_KERNEL);</div><div class='del'>-			if (!eq-&gt;buf[i])</div><div class='del'>-				goto err_dma_alloc_buf;</div><div class='add'>+	eqc = mb_buf;</div><div class='add'>+	memset(eqc, 0, sizeof(struct hns_roce_eq_context));</div><div class='ctx'> </div><div class='del'>-			*(eq-&gt;bt_l0 + i) = eq-&gt;buf_dma[i];</div><div class='add'>+	init_eq_config(hr_dev, eq);</div><div class='ctx'> </div><div class='del'>-			eq_buf_cnt++;</div><div class='del'>-			if (eq_buf_cnt &gt;= ba_num)</div><div class='del'>-				break;</div><div class='del'>-		}</div><div class='del'>-		eq-&gt;cur_eqe_ba = eq-&gt;buf_dma[0];</div><div class='del'>-		eq-&gt;nxt_eqe_ba = eq-&gt;buf_dma[1];</div><div class='del'>-</div><div class='del'>-	} else if (mhop_num == 2) {</div><div class='del'>-		/* alloc L1 BT and buf */</div><div class='del'>-		for (i = 0; i &lt; bt_chk_sz / 8; i++) {</div><div class='del'>-			eq-&gt;bt_l1[i] = dma_alloc_coherent(dev, bt_chk_sz,</div><div class='del'>-							  &amp;(eq-&gt;l1_dma[i]),</div><div class='del'>-							  GFP_KERNEL);</div><div class='del'>-			if (!eq-&gt;bt_l1[i])</div><div class='del'>-				goto err_dma_alloc_l1;</div><div class='del'>-			*(eq-&gt;bt_l0 + i) = eq-&gt;l1_dma[i];</div><div class='del'>-</div><div class='del'>-			for (j = 0; j &lt; bt_chk_sz / 8; j++) {</div><div class='del'>-				idx = i * bt_chk_sz / 8 + j;</div><div class='del'>-				if (eq_buf_cnt + 1 &lt; ba_num) {</div><div class='del'>-					size = buf_chk_sz;</div><div class='del'>-				} else {</div><div class='del'>-					eqe_alloc = (buf_chk_sz / eq-&gt;eqe_size)</div><div class='del'>-						    * idx;</div><div class='del'>-					size = (eq-&gt;entries - eqe_alloc)</div><div class='del'>-						* eq-&gt;eqe_size;</div><div class='del'>-				}</div><div class='del'>-				eq-&gt;buf[idx] = dma_alloc_coherent(dev, size,</div><div class='del'>-								  &amp;(eq-&gt;buf_dma[idx]),</div><div class='del'>-								  GFP_KERNEL);</div><div class='del'>-				if (!eq-&gt;buf[idx])</div><div class='del'>-					goto err_dma_alloc_buf;</div><div class='del'>-</div><div class='del'>-				*(eq-&gt;bt_l1[i] + j) = eq-&gt;buf_dma[idx];</div><div class='del'>-</div><div class='del'>-				eq_buf_cnt++;</div><div class='del'>-				if (eq_buf_cnt &gt;= ba_num) {</div><div class='del'>-					eq_alloc_done = 1;</div><div class='del'>-					break;</div><div class='del'>-				}</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			if (eq_alloc_done)</div><div class='del'>-				break;</div><div class='del'>-		}</div><div class='del'>-		eq-&gt;cur_eqe_ba = eq-&gt;buf_dma[0];</div><div class='del'>-		eq-&gt;nxt_eqe_ba = eq-&gt;buf_dma[1];</div><div class='add'>+	/* if not multi-hop, eqe buffer only use one trunk */</div><div class='add'>+	ret = hns_roce_mtr_find(hr_dev, &amp;eq-&gt;mtr, 0, eqe_ba,</div><div class='add'>+				ARRAY_SIZE(eqe_ba));</div><div class='add'>+	if (ret) {</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "failed to find EQE mtr, ret = %d\n", ret);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	eq-&gt;l0_last_num = i + 1;</div><div class='del'>-	if (mhop_num == 2)</div><div class='del'>-		eq-&gt;l1_last_num = j + 1;</div><div class='add'>+	bt_ba = hns_roce_get_mtr_ba(&amp;eq-&gt;mtr);</div><div class='add'>+</div><div class='add'>+	hr_reg_write(eqc, EQC_EQ_ST, HNS_ROCE_V2_EQ_STATE_VALID);</div><div class='add'>+	hr_reg_write(eqc, EQC_EQE_HOP_NUM, eq-&gt;hop_num);</div><div class='add'>+	hr_reg_write(eqc, EQC_OVER_IGNORE, eq-&gt;over_ignore);</div><div class='add'>+	hr_reg_write(eqc, EQC_COALESCE, eq-&gt;coalesce);</div><div class='add'>+	hr_reg_write(eqc, EQC_ARM_ST, eq-&gt;arm_st);</div><div class='add'>+	hr_reg_write(eqc, EQC_EQN, eq-&gt;eqn);</div><div class='add'>+	hr_reg_write(eqc, EQC_EQE_CNT, HNS_ROCE_EQ_INIT_EQE_CNT);</div><div class='add'>+	hr_reg_write(eqc, EQC_EQE_BA_PG_SZ,</div><div class='add'>+		     to_hr_hw_page_shift(eq-&gt;mtr.hem_cfg.ba_pg_shift));</div><div class='add'>+	hr_reg_write(eqc, EQC_EQE_BUF_PG_SZ,</div><div class='add'>+		     to_hr_hw_page_shift(eq-&gt;mtr.hem_cfg.buf_pg_shift));</div><div class='add'>+	hr_reg_write(eqc, EQC_EQ_PROD_INDX, HNS_ROCE_EQ_INIT_PROD_IDX);</div><div class='add'>+	hr_reg_write(eqc, EQC_EQ_MAX_CNT, eq-&gt;eq_max_cnt);</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08) {</div><div class='add'>+		if (eq-&gt;eq_period * HNS_ROCE_CLOCK_ADJUST &gt; USHRT_MAX) {</div><div class='add'>+			dev_info(hr_dev-&gt;dev, "eq_period(%u) reached the upper limit, adjusted to 65.\n",</div><div class='add'>+				 eq-&gt;eq_period);</div><div class='add'>+			eq-&gt;eq_period = HNS_ROCE_MAX_EQ_PERIOD;</div><div class='add'>+		}</div><div class='add'>+		eq-&gt;eq_period *= HNS_ROCE_CLOCK_ADJUST;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hr_reg_write(eqc, EQC_EQ_PERIOD, eq-&gt;eq_period);</div><div class='add'>+	hr_reg_write(eqc, EQC_EQE_REPORT_TIMER, HNS_ROCE_EQ_INIT_REPORT_TIMER);</div><div class='add'>+	hr_reg_write(eqc, EQC_EQE_BA_L, bt_ba &gt;&gt; 3);</div><div class='add'>+	hr_reg_write(eqc, EQC_EQE_BA_H, bt_ba &gt;&gt; 35);</div><div class='add'>+	hr_reg_write(eqc, EQC_SHIFT, eq-&gt;shift);</div><div class='add'>+	hr_reg_write(eqc, EQC_MSI_INDX, HNS_ROCE_EQ_INIT_MSI_IDX);</div><div class='add'>+	hr_reg_write(eqc, EQC_CUR_EQE_BA_L, eqe_ba[0] &gt;&gt; 12);</div><div class='add'>+	hr_reg_write(eqc, EQC_CUR_EQE_BA_M, eqe_ba[0] &gt;&gt; 28);</div><div class='add'>+	hr_reg_write(eqc, EQC_CUR_EQE_BA_H, eqe_ba[0] &gt;&gt; 60);</div><div class='add'>+	hr_reg_write(eqc, EQC_EQ_CONS_INDX, HNS_ROCE_EQ_INIT_CONS_IDX);</div><div class='add'>+	hr_reg_write(eqc, EQC_NEX_EQE_BA_L, eqe_ba[1] &gt;&gt; 12);</div><div class='add'>+	hr_reg_write(eqc, EQC_NEX_EQE_BA_H, eqe_ba[1] &gt;&gt; 44);</div><div class='add'>+	hr_reg_write(eqc, EQC_EQE_SIZE, eq-&gt;eqe_size == HNS_ROCE_V3_EQE_SIZE);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-err_dma_alloc_l1:</div><div class='del'>-	dma_free_coherent(dev, bt_chk_sz, eq-&gt;bt_l0, eq-&gt;l0_dma);</div><div class='del'>-	eq-&gt;bt_l0 = NULL;</div><div class='del'>-	eq-&gt;l0_dma = 0;</div><div class='del'>-	for (i -= 1; i &gt;= 0; i--) {</div><div class='del'>-		dma_free_coherent(dev, bt_chk_sz, eq-&gt;bt_l1[i],</div><div class='del'>-				  eq-&gt;l1_dma[i]);</div><div class='del'>-</div><div class='del'>-		for (j = 0; j &lt; bt_chk_sz / 8; j++) {</div><div class='del'>-			idx = i * bt_chk_sz / 8 + j;</div><div class='del'>-			dma_free_coherent(dev, buf_chk_sz, eq-&gt;buf[idx],</div><div class='del'>-					  eq-&gt;buf_dma[idx]);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	goto err_dma_alloc_l0;</div><div class='del'>-</div><div class='del'>-err_dma_alloc_buf:</div><div class='del'>-	dma_free_coherent(dev, bt_chk_sz, eq-&gt;bt_l0, eq-&gt;l0_dma);</div><div class='del'>-	eq-&gt;bt_l0 = NULL;</div><div class='del'>-	eq-&gt;l0_dma = 0;</div><div class='del'>-</div><div class='del'>-	if (mhop_num == 1)</div><div class='del'>-		for (i -= 1; i &gt;= 0; i--)</div><div class='del'>-			dma_free_coherent(dev, buf_chk_sz, eq-&gt;buf[i],</div><div class='del'>-					  eq-&gt;buf_dma[i]);</div><div class='del'>-	else if (mhop_num == 2) {</div><div class='del'>-		record_i = i;</div><div class='del'>-		record_j = j;</div><div class='del'>-		for (; i &gt;= 0; i--) {</div><div class='del'>-			dma_free_coherent(dev, bt_chk_sz, eq-&gt;bt_l1[i],</div><div class='del'>-					  eq-&gt;l1_dma[i]);</div><div class='del'>-</div><div class='del'>-			for (j = 0; j &lt; bt_chk_sz / 8; j++) {</div><div class='del'>-				if (i == record_i &amp;&amp; j &gt;= record_j)</div><div class='del'>-					break;</div><div class='del'>-</div><div class='del'>-				idx = i * bt_chk_sz / 8 + j;</div><div class='del'>-				dma_free_coherent(dev, buf_chk_sz,</div><div class='del'>-						  eq-&gt;buf[idx],</div><div class='del'>-						  eq-&gt;buf_dma[idx]);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-err_dma_alloc_l0:</div><div class='del'>-	kfree(eq-&gt;bt_l1);</div><div class='del'>-	eq-&gt;bt_l1 = NULL;</div><div class='add'>+static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_buf_attr buf_attr = {};</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-err_kcalloc_bt_l1:</div><div class='del'>-	kfree(eq-&gt;l1_dma);</div><div class='del'>-	eq-&gt;l1_dma = NULL;</div><div class='add'>+	if (hr_dev-&gt;caps.eqe_hop_num == HNS_ROCE_HOP_NUM_0)</div><div class='add'>+		eq-&gt;hop_num = 0;</div><div class='add'>+	else</div><div class='add'>+		eq-&gt;hop_num = hr_dev-&gt;caps.eqe_hop_num;</div><div class='ctx'> </div><div class='del'>-err_kcalloc_l1_dma:</div><div class='del'>-	kfree(eq-&gt;buf);</div><div class='del'>-	eq-&gt;buf = NULL;</div><div class='add'>+	buf_attr.page_shift = hr_dev-&gt;caps.eqe_buf_pg_sz + PAGE_SHIFT;</div><div class='add'>+	buf_attr.region[0].size = eq-&gt;entries * eq-&gt;eqe_size;</div><div class='add'>+	buf_attr.region[0].hopnum = eq-&gt;hop_num;</div><div class='add'>+	buf_attr.region_count = 1;</div><div class='ctx'> </div><div class='del'>-err_kcalloc_buf:</div><div class='del'>-	kfree(eq-&gt;buf_dma);</div><div class='del'>-	eq-&gt;buf_dma = NULL;</div><div class='add'>+	err = hns_roce_mtr_create(hr_dev, &amp;eq-&gt;mtr, &amp;buf_attr,</div><div class='add'>+				  hr_dev-&gt;caps.eqe_ba_pg_sz + PAGE_SHIFT, NULL,</div><div class='add'>+				  0);</div><div class='add'>+	if (err)</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "failed to alloc EQE mtr, err %d\n", err);</div><div class='ctx'> </div><div class='del'>-	return -ENOMEM;</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,</div><div class='del'>-				 struct hns_roce_eq *eq,</div><div class='del'>-				 unsigned int eq_cmd)</div><div class='add'>+				 struct hns_roce_eq *eq, u8 eq_cmd)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='ctx'> 	struct hns_roce_cmd_mailbox *mailbox;</div><div class='del'>-	u32 buf_chk_sz = 0;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	/* Allocate mailbox memory */</div><div class='hunk'>@@ -5567,38 +6741,17 @@ static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 	if (IS_ERR(mailbox))</div><div class='ctx'> 		return PTR_ERR(mailbox);</div><div class='ctx'> </div><div class='del'>-	if (!hr_dev-&gt;caps.eqe_hop_num) {</div><div class='del'>-		buf_chk_sz = 1 &lt;&lt; (hr_dev-&gt;caps.eqe_buf_pg_sz + PAGE_SHIFT);</div><div class='del'>-</div><div class='del'>-		eq-&gt;buf_list = kzalloc(sizeof(struct hns_roce_buf_list),</div><div class='del'>-				       GFP_KERNEL);</div><div class='del'>-		if (!eq-&gt;buf_list) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto free_cmd_mbox;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		eq-&gt;buf_list-&gt;buf = dma_alloc_coherent(dev, buf_chk_sz,</div><div class='del'>-						       &amp;(eq-&gt;buf_list-&gt;map),</div><div class='del'>-						       GFP_KERNEL);</div><div class='del'>-		if (!eq-&gt;buf_list-&gt;buf) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto err_alloc_buf;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-	} else {</div><div class='del'>-		ret = hns_roce_mhop_alloc_eq(hr_dev, eq);</div><div class='del'>-		if (ret) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto free_cmd_mbox;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='add'>+	ret = alloc_eq_buf(hr_dev, eq);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto free_cmd_mbox;</div><div class='ctx'> </div><div class='del'>-	hns_roce_config_eqc(hr_dev, eq, mailbox-&gt;buf);</div><div class='add'>+	ret = config_eqc(hr_dev, eq, mailbox-&gt;buf);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_cmd_mbox;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_cmd_mbox(hr_dev, mailbox-&gt;dma, 0, eq-&gt;eqn, 0,</div><div class='del'>-				eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='add'>+	ret = hns_roce_create_hw_ctx(hr_dev, mailbox, eq_cmd, eq-&gt;eqn);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "[mailbox cmd] create eqc failed.\n");</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "[mailbox cmd] create eqc failed.\n");</div><div class='ctx'> 		goto err_cmd_mbox;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -5607,16 +6760,7 @@ static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> err_cmd_mbox:</div><div class='del'>-	if (!hr_dev-&gt;caps.eqe_hop_num)</div><div class='del'>-		dma_free_coherent(dev, buf_chk_sz, eq-&gt;buf_list-&gt;buf,</div><div class='del'>-				  eq-&gt;buf_list-&gt;map);</div><div class='del'>-	else {</div><div class='del'>-		hns_roce_mhop_free_eq(hr_dev, eq);</div><div class='del'>-		goto free_cmd_mbox;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-err_alloc_buf:</div><div class='del'>-	kfree(eq-&gt;buf_list);</div><div class='add'>+	free_eq_buf(hr_dev, eq);</div><div class='ctx'> </div><div class='ctx'> free_cmd_mbox:</div><div class='ctx'> 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='hunk'>@@ -5624,144 +6768,112 @@ free_cmd_mbox:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)</div><div class='add'>+static void hns_roce_ceq_work(struct work_struct *work)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_eq_table *eq_table = &amp;hr_dev-&gt;eq_table;</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	struct hns_roce_eq *eq;</div><div class='del'>-	unsigned int eq_cmd;</div><div class='del'>-	int irq_num;</div><div class='del'>-	int eq_num;</div><div class='del'>-	int other_num;</div><div class='del'>-	int comp_num;</div><div class='del'>-	int aeq_num;</div><div class='del'>-	int i, j, k;</div><div class='del'>-	int ret;</div><div class='add'>+	struct hns_roce_eq *eq = from_work(eq, work, work);</div><div class='add'>+	struct hns_roce_ceqe *ceqe = next_ceqe_sw_v2(eq);</div><div class='add'>+	struct hns_roce_dev *hr_dev = eq-&gt;hr_dev;</div><div class='add'>+	int ceqe_num = 0;</div><div class='add'>+	u32 cqn;</div><div class='ctx'> </div><div class='del'>-	other_num = hr_dev-&gt;caps.num_other_vectors;</div><div class='del'>-	comp_num = hr_dev-&gt;caps.num_comp_vectors;</div><div class='del'>-	aeq_num = hr_dev-&gt;caps.num_aeq_vectors;</div><div class='add'>+	while (ceqe &amp;&amp; ceqe_num &lt; hr_dev-&gt;caps.ceqe_depth) {</div><div class='add'>+		/* Make sure we read CEQ entry after we have checked the</div><div class='add'>+		 * ownership bit</div><div class='add'>+		 */</div><div class='add'>+		dma_rmb();</div><div class='ctx'> </div><div class='del'>-	eq_num = comp_num + aeq_num;</div><div class='del'>-	irq_num = eq_num + other_num;</div><div class='add'>+		cqn = hr_reg_read(ceqe, CEQE_CQN);</div><div class='ctx'> </div><div class='del'>-	eq_table-&gt;eq = kcalloc(eq_num, sizeof(*eq_table-&gt;eq), GFP_KERNEL);</div><div class='del'>-	if (!eq_table-&gt;eq)</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+		hns_roce_cq_completion(hr_dev, cqn);</div><div class='add'>+</div><div class='add'>+		++eq-&gt;cons_index;</div><div class='add'>+		++ceqe_num;</div><div class='add'>+		atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_CEQE_CNT]);</div><div class='add'>+</div><div class='add'>+		ceqe = next_ceqe_sw_v2(eq);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	update_eq_db(eq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num,</div><div class='add'>+				  int comp_num, int aeq_num, int other_num)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_eq_table *eq_table = &amp;hr_dev-&gt;eq_table;</div><div class='add'>+	int i, j;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; irq_num; i++) {</div><div class='ctx'> 		hr_dev-&gt;irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,</div><div class='ctx'> 					       GFP_KERNEL);</div><div class='ctx'> 		if (!hr_dev-&gt;irq_names[i]) {</div><div class='ctx'> 			ret = -ENOMEM;</div><div class='del'>-			goto err_failed_kzalloc;</div><div class='add'>+			goto err_kzalloc_failed;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* create eq */</div><div class='del'>-	for (j = 0; j &lt; eq_num; j++) {</div><div class='del'>-		eq = &amp;eq_table-&gt;eq[j];</div><div class='del'>-		eq-&gt;hr_dev = hr_dev;</div><div class='del'>-		eq-&gt;eqn = j;</div><div class='del'>-		if (j &lt; comp_num) {</div><div class='del'>-			/* CEQ */</div><div class='del'>-			eq_cmd = HNS_ROCE_CMD_CREATE_CEQC;</div><div class='del'>-			eq-&gt;type_flag = HNS_ROCE_CEQ;</div><div class='del'>-			eq-&gt;entries = hr_dev-&gt;caps.ceqe_depth;</div><div class='del'>-			eq-&gt;eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE;</div><div class='del'>-			eq-&gt;irq = hr_dev-&gt;irq[j + other_num + aeq_num];</div><div class='del'>-			eq-&gt;eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM;</div><div class='del'>-			eq-&gt;eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL;</div><div class='del'>-		} else {</div><div class='del'>-			/* AEQ */</div><div class='del'>-			eq_cmd = HNS_ROCE_CMD_CREATE_AEQC;</div><div class='del'>-			eq-&gt;type_flag = HNS_ROCE_AEQ;</div><div class='del'>-			eq-&gt;entries = hr_dev-&gt;caps.aeqe_depth;</div><div class='del'>-			eq-&gt;eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE;</div><div class='del'>-			eq-&gt;irq = hr_dev-&gt;irq[j - comp_num + other_num];</div><div class='del'>-			eq-&gt;eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM;</div><div class='del'>-			eq-&gt;eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL;</div><div class='del'>-		}</div><div class='add'>+	/* irq contains: abnormal + AEQ + CEQ */</div><div class='add'>+	for (j = 0; j &lt; other_num; j++)</div><div class='add'>+		snprintf((char *)hr_dev-&gt;irq_names[j], HNS_ROCE_INT_NAME_LEN,</div><div class='add'>+			 "hns-%s-abn-%d", pci_name(hr_dev-&gt;pci_dev), j);</div><div class='ctx'> </div><div class='del'>-		ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "eq create failed.\n");</div><div class='del'>-			goto err_create_eq_fail;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='add'>+	for (j = other_num; j &lt; (other_num + aeq_num); j++)</div><div class='add'>+		snprintf((char *)hr_dev-&gt;irq_names[j], HNS_ROCE_INT_NAME_LEN,</div><div class='add'>+			 "hns-%s-aeq-%d", pci_name(hr_dev-&gt;pci_dev), j - other_num);</div><div class='ctx'> </div><div class='del'>-	/* enable irq */</div><div class='del'>-	hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE);</div><div class='add'>+	for (j = (other_num + aeq_num); j &lt; irq_num; j++)</div><div class='add'>+		snprintf((char *)hr_dev-&gt;irq_names[j], HNS_ROCE_INT_NAME_LEN,</div><div class='add'>+			 "hns-%s-ceq-%d", pci_name(hr_dev-&gt;pci_dev),</div><div class='add'>+			 j - other_num - aeq_num);</div><div class='ctx'> </div><div class='del'>-	/* irq contains: abnormal + AEQ + CEQ*/</div><div class='del'>-	for (k = 0; k &lt; irq_num; k++)</div><div class='del'>-		if (k &lt; other_num)</div><div class='del'>-			snprintf((char *)hr_dev-&gt;irq_names[k],</div><div class='del'>-				 HNS_ROCE_INT_NAME_LEN, "hns-abn-%d", k);</div><div class='del'>-		else if (k &lt; (other_num + aeq_num))</div><div class='del'>-			snprintf((char *)hr_dev-&gt;irq_names[k],</div><div class='del'>-				 HNS_ROCE_INT_NAME_LEN, "hns-aeq-%d",</div><div class='del'>-				 k - other_num);</div><div class='del'>-		else</div><div class='del'>-			snprintf((char *)hr_dev-&gt;irq_names[k],</div><div class='del'>-				 HNS_ROCE_INT_NAME_LEN, "hns-ceq-%d",</div><div class='del'>-				 k - other_num - aeq_num);</div><div class='del'>-</div><div class='del'>-	for (k = 0; k &lt; irq_num; k++) {</div><div class='del'>-		if (k &lt; other_num)</div><div class='del'>-			ret = request_irq(hr_dev-&gt;irq[k],</div><div class='add'>+	for (j = 0; j &lt; irq_num; j++) {</div><div class='add'>+		if (j &lt; other_num) {</div><div class='add'>+			ret = request_irq(hr_dev-&gt;irq[j],</div><div class='ctx'> 					  hns_roce_v2_msix_interrupt_abn,</div><div class='del'>-					  0, hr_dev-&gt;irq_names[k], hr_dev);</div><div class='del'>-</div><div class='del'>-		else if (k &lt; (other_num + comp_num))</div><div class='del'>-			ret = request_irq(eq_table-&gt;eq[k - other_num].irq,</div><div class='add'>+					  0, hr_dev-&gt;irq_names[j], hr_dev);</div><div class='add'>+		} else if (j &lt; (other_num + comp_num)) {</div><div class='add'>+			INIT_WORK(&amp;eq_table-&gt;eq[j - other_num].work,</div><div class='add'>+				  hns_roce_ceq_work);</div><div class='add'>+			ret = request_irq(eq_table-&gt;eq[j - other_num].irq,</div><div class='ctx'> 					  hns_roce_v2_msix_interrupt_eq,</div><div class='del'>-					  0, hr_dev-&gt;irq_names[k + aeq_num],</div><div class='del'>-					  &amp;eq_table-&gt;eq[k - other_num]);</div><div class='del'>-		else</div><div class='del'>-			ret = request_irq(eq_table-&gt;eq[k - other_num].irq,</div><div class='add'>+					  0, hr_dev-&gt;irq_names[j + aeq_num],</div><div class='add'>+					  &amp;eq_table-&gt;eq[j - other_num]);</div><div class='add'>+		} else {</div><div class='add'>+			ret = request_irq(eq_table-&gt;eq[j - other_num].irq,</div><div class='ctx'> 					  hns_roce_v2_msix_interrupt_eq,</div><div class='del'>-					  0, hr_dev-&gt;irq_names[k - comp_num],</div><div class='del'>-					  &amp;eq_table-&gt;eq[k - other_num]);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "Request irq error!\n");</div><div class='del'>-			goto err_request_irq_fail;</div><div class='add'>+					  0, hr_dev-&gt;irq_names[j - comp_num],</div><div class='add'>+					  &amp;eq_table-&gt;eq[j - other_num]);</div><div class='ctx'> 		}</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	hr_dev-&gt;irq_workq =</div><div class='del'>-		create_singlethread_workqueue("hns_roce_irq_workqueue");</div><div class='del'>-	if (!hr_dev-&gt;irq_workq) {</div><div class='del'>-		dev_err(dev, "Create irq workqueue failed!\n");</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err_request_irq_fail;</div><div class='add'>+		if (ret) {</div><div class='add'>+			dev_err(hr_dev-&gt;dev, "request irq error!\n");</div><div class='add'>+			goto err_request_failed;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-err_request_irq_fail:</div><div class='del'>-	for (k -= 1; k &gt;= 0; k--)</div><div class='del'>-		if (k &lt; other_num)</div><div class='del'>-			free_irq(hr_dev-&gt;irq[k], hr_dev);</div><div class='del'>-		else</div><div class='del'>-			free_irq(eq_table-&gt;eq[k - other_num].irq,</div><div class='del'>-				 &amp;eq_table-&gt;eq[k - other_num]);</div><div class='del'>-</div><div class='del'>-err_create_eq_fail:</div><div class='del'>-	for (j -= 1; j &gt;= 0; j--)</div><div class='del'>-		hns_roce_v2_free_eq(hr_dev, &amp;eq_table-&gt;eq[j]);</div><div class='add'>+err_request_failed:</div><div class='add'>+	for (j -= 1; j &gt;= 0; j--) {</div><div class='add'>+		if (j &lt; other_num) {</div><div class='add'>+			free_irq(hr_dev-&gt;irq[j], hr_dev);</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+		free_irq(eq_table-&gt;eq[j - other_num].irq,</div><div class='add'>+			 &amp;eq_table-&gt;eq[j - other_num]);</div><div class='add'>+		if (j &lt; other_num + comp_num)</div><div class='add'>+			cancel_work_sync(&amp;eq_table-&gt;eq[j - other_num].work);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-err_failed_kzalloc:</div><div class='add'>+err_kzalloc_failed:</div><div class='ctx'> 	for (i -= 1; i &gt;= 0; i--)</div><div class='ctx'> 		kfree(hr_dev-&gt;irq_names[i]);</div><div class='del'>-	kfree(eq_table-&gt;eq);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)</div><div class='add'>+static void __hns_roce_free_irq(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_eq_table *eq_table = &amp;hr_dev-&gt;eq_table;</div><div class='ctx'> 	int irq_num;</div><div class='ctx'> 	int eq_num;</div><div class='ctx'> 	int i;</div><div class='hunk'>@@ -5769,312 +6881,131 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)</div><div class='ctx'> 	eq_num = hr_dev-&gt;caps.num_comp_vectors + hr_dev-&gt;caps.num_aeq_vectors;</div><div class='ctx'> 	irq_num = eq_num + hr_dev-&gt;caps.num_other_vectors;</div><div class='ctx'> </div><div class='del'>-	/* Disable irq */</div><div class='del'>-	hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);</div><div class='del'>-</div><div class='ctx'> 	for (i = 0; i &lt; hr_dev-&gt;caps.num_other_vectors; i++)</div><div class='ctx'> 		free_irq(hr_dev-&gt;irq[i], hr_dev);</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; eq_num; i++) {</div><div class='del'>-		hns_roce_v2_destroy_eqc(hr_dev, i);</div><div class='del'>-</div><div class='del'>-		free_irq(eq_table-&gt;eq[i].irq, &amp;eq_table-&gt;eq[i]);</div><div class='del'>-</div><div class='del'>-		hns_roce_v2_free_eq(hr_dev, &amp;eq_table-&gt;eq[i]);</div><div class='add'>+		free_irq(hr_dev-&gt;eq_table.eq[i].irq, &amp;hr_dev-&gt;eq_table.eq[i]);</div><div class='add'>+		if (i &lt; hr_dev-&gt;caps.num_comp_vectors)</div><div class='add'>+			cancel_work_sync(&amp;hr_dev-&gt;eq_table.eq[i].work);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; irq_num; i++)</div><div class='ctx'> 		kfree(hr_dev-&gt;irq_names[i]);</div><div class='del'>-</div><div class='del'>-	kfree(eq_table-&gt;eq);</div><div class='del'>-</div><div class='del'>-	flush_workqueue(hr_dev-&gt;irq_workq);</div><div class='del'>-	destroy_workqueue(hr_dev-&gt;irq_workq);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_v2_write_srqc(struct hns_roce_dev *hr_dev,</div><div class='del'>-				   struct hns_roce_srq *srq, u32 pdn, u16 xrcd,</div><div class='del'>-				   u32 cqn, void *mb_buf, u64 *mtts_wqe,</div><div class='del'>-				   u64 *mtts_idx, dma_addr_t dma_handle_wqe,</div><div class='del'>-				   dma_addr_t dma_handle_idx)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_srq_context *srq_context;</div><div class='del'>-</div><div class='del'>-	srq_context = mb_buf;</div><div class='del'>-	memset(srq_context, 0, sizeof(*srq_context));</div><div class='del'>-</div><div class='del'>-	roce_set_field(srq_context-&gt;byte_4_srqn_srqst, SRQC_BYTE_4_SRQ_ST_M,</div><div class='del'>-		       SRQC_BYTE_4_SRQ_ST_S, 1);</div><div class='del'>-</div><div class='del'>-	roce_set_field(srq_context-&gt;byte_4_srqn_srqst,</div><div class='del'>-		       SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M,</div><div class='del'>-		       SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S,</div><div class='del'>-		       (hr_dev-&gt;caps.srqwqe_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :</div><div class='del'>-		       hr_dev-&gt;caps.srqwqe_hop_num));</div><div class='del'>-	roce_set_field(srq_context-&gt;byte_4_srqn_srqst,</div><div class='del'>-		       SRQC_BYTE_4_SRQ_SHIFT_M, SRQC_BYTE_4_SRQ_SHIFT_S,</div><div class='del'>-		       ilog2(srq-&gt;max));</div><div class='del'>-</div><div class='del'>-	roce_set_field(srq_context-&gt;byte_4_srqn_srqst, SRQC_BYTE_4_SRQN_M,</div><div class='del'>-		       SRQC_BYTE_4_SRQN_S, srq-&gt;srqn);</div><div class='del'>-</div><div class='del'>-	roce_set_field(srq_context-&gt;byte_8_limit_wl, SRQC_BYTE_8_SRQ_LIMIT_WL_M,</div><div class='del'>-		       SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);</div><div class='del'>-</div><div class='del'>-	roce_set_field(srq_context-&gt;byte_12_xrcd, SRQC_BYTE_12_SRQ_XRCD_M,</div><div class='del'>-		       SRQC_BYTE_12_SRQ_XRCD_S, xrcd);</div><div class='del'>-</div><div class='del'>-	srq_context-&gt;wqe_bt_ba = cpu_to_le32((u32)(dma_handle_wqe &gt;&gt; 3));</div><div class='del'>-</div><div class='del'>-	roce_set_field(srq_context-&gt;byte_24_wqe_bt_ba,</div><div class='del'>-		       SRQC_BYTE_24_SRQ_WQE_BT_BA_M,</div><div class='del'>-		       SRQC_BYTE_24_SRQ_WQE_BT_BA_S,</div><div class='del'>-		       cpu_to_le32(dma_handle_wqe &gt;&gt; 35));</div><div class='del'>-</div><div class='del'>-	roce_set_field(srq_context-&gt;byte_28_rqws_pd, SRQC_BYTE_28_PD_M,</div><div class='del'>-		       SRQC_BYTE_28_PD_S, pdn);</div><div class='del'>-	roce_set_field(srq_context-&gt;byte_28_rqws_pd, SRQC_BYTE_28_RQWS_M,</div><div class='del'>-		       SRQC_BYTE_28_RQWS_S, srq-&gt;max_gs &lt;= 0 ? 0 :</div><div class='del'>-		       fls(srq-&gt;max_gs - 1));</div><div class='del'>-</div><div class='del'>-	srq_context-&gt;idx_bt_ba = (u32)(dma_handle_idx &gt;&gt; 3);</div><div class='del'>-	srq_context-&gt;idx_bt_ba = cpu_to_le32(srq_context-&gt;idx_bt_ba);</div><div class='del'>-	roce_set_field(srq_context-&gt;rsv_idx_bt_ba,</div><div class='del'>-		       SRQC_BYTE_36_SRQ_IDX_BT_BA_M,</div><div class='del'>-		       SRQC_BYTE_36_SRQ_IDX_BT_BA_S,</div><div class='del'>-		       cpu_to_le32(dma_handle_idx &gt;&gt; 35));</div><div class='del'>-</div><div class='del'>-	srq_context-&gt;idx_cur_blk_addr = (u32)(mtts_idx[0] &gt;&gt; PAGE_ADDR_SHIFT);</div><div class='del'>-	srq_context-&gt;idx_cur_blk_addr =</div><div class='del'>-				     cpu_to_le32(srq_context-&gt;idx_cur_blk_addr);</div><div class='del'>-	roce_set_field(srq_context-&gt;byte_44_idxbufpgsz_addr,</div><div class='del'>-		       SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M,</div><div class='del'>-		       SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S,</div><div class='del'>-		       cpu_to_le32((mtts_idx[0]) &gt;&gt; (32 + PAGE_ADDR_SHIFT)));</div><div class='del'>-	roce_set_field(srq_context-&gt;byte_44_idxbufpgsz_addr,</div><div class='del'>-		       SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M,</div><div class='del'>-		       SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S,</div><div class='del'>-		       hr_dev-&gt;caps.idx_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :</div><div class='del'>-		       hr_dev-&gt;caps.idx_hop_num);</div><div class='del'>-</div><div class='del'>-	roce_set_field(srq_context-&gt;byte_44_idxbufpgsz_addr,</div><div class='del'>-		       SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M,</div><div class='del'>-		       SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.idx_ba_pg_sz);</div><div class='del'>-	roce_set_field(srq_context-&gt;byte_44_idxbufpgsz_addr,</div><div class='del'>-		       SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M,</div><div class='del'>-		       SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.idx_buf_pg_sz);</div><div class='del'>-</div><div class='del'>-	srq_context-&gt;idx_nxt_blk_addr = (u32)(mtts_idx[1] &gt;&gt; PAGE_ADDR_SHIFT);</div><div class='del'>-	srq_context-&gt;idx_nxt_blk_addr =</div><div class='del'>-				   cpu_to_le32(srq_context-&gt;idx_nxt_blk_addr);</div><div class='del'>-	roce_set_field(srq_context-&gt;rsv_idxnxtblkaddr,</div><div class='del'>-		       SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M,</div><div class='del'>-		       SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S,</div><div class='del'>-		       cpu_to_le32((mtts_idx[1]) &gt;&gt; (32 + PAGE_ADDR_SHIFT)));</div><div class='del'>-	roce_set_field(srq_context-&gt;byte_56_xrc_cqn,</div><div class='del'>-		       SRQC_BYTE_56_SRQ_XRC_CQN_M, SRQC_BYTE_56_SRQ_XRC_CQN_S,</div><div class='del'>-		       cqn);</div><div class='del'>-	roce_set_field(srq_context-&gt;byte_56_xrc_cqn,</div><div class='del'>-		       SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M,</div><div class='del'>-		       SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.srqwqe_ba_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-	roce_set_field(srq_context-&gt;byte_56_xrc_cqn,</div><div class='del'>-		       SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M,</div><div class='del'>-		       SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S,</div><div class='del'>-		       hr_dev-&gt;caps.srqwqe_buf_pg_sz + PG_SHIFT_OFFSET);</div><div class='del'>-</div><div class='del'>-	roce_set_bit(srq_context-&gt;db_record_addr_record_en,</div><div class='del'>-		     SRQC_BYTE_60_SRQ_RECORD_EN_S, 0);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,</div><div class='del'>-				  struct ib_srq_attr *srq_attr,</div><div class='del'>-				  enum ib_srq_attr_mask srq_attr_mask,</div><div class='del'>-				  struct ib_udata *udata)</div><div class='add'>+static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq-&gt;device);</div><div class='del'>-	struct hns_roce_srq *srq = to_hr_srq(ibsrq);</div><div class='del'>-	struct hns_roce_srq_context *srq_context;</div><div class='del'>-	struct hns_roce_srq_context *srqc_mask;</div><div class='del'>-	struct hns_roce_cmd_mailbox *mailbox;</div><div class='add'>+	struct hns_roce_eq_table *eq_table = &amp;hr_dev-&gt;eq_table;</div><div class='add'>+	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	struct hns_roce_eq *eq;</div><div class='add'>+	int other_num;</div><div class='add'>+	int comp_num;</div><div class='add'>+	int aeq_num;</div><div class='add'>+	int irq_num;</div><div class='add'>+	int eq_num;</div><div class='add'>+	u8 eq_cmd;</div><div class='ctx'> 	int ret;</div><div class='add'>+	int i;</div><div class='ctx'> </div><div class='del'>-	if (srq_attr_mask &amp; IB_SRQ_LIMIT) {</div><div class='del'>-		if (srq_attr-&gt;srq_limit &gt;= srq-&gt;max)</div><div class='del'>-			return -EINVAL;</div><div class='add'>+	if (hr_dev-&gt;caps.aeqe_depth &lt; HNS_AEQ_POLLING_BUDGET)</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-		mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='del'>-		if (IS_ERR(mailbox))</div><div class='del'>-			return PTR_ERR(mailbox);</div><div class='add'>+	other_num = hr_dev-&gt;caps.num_other_vectors;</div><div class='add'>+	comp_num = hr_dev-&gt;caps.num_comp_vectors;</div><div class='add'>+	aeq_num = hr_dev-&gt;caps.num_aeq_vectors;</div><div class='ctx'> </div><div class='del'>-		srq_context = mailbox-&gt;buf;</div><div class='del'>-		srqc_mask = (struct hns_roce_srq_context *)mailbox-&gt;buf + 1;</div><div class='add'>+	eq_num = comp_num + aeq_num;</div><div class='add'>+	irq_num = eq_num + other_num;</div><div class='ctx'> </div><div class='del'>-		memset(srqc_mask, 0xff, sizeof(*srqc_mask));</div><div class='add'>+	eq_table-&gt;eq = kcalloc(eq_num, sizeof(*eq_table-&gt;eq), GFP_KERNEL);</div><div class='add'>+	if (!eq_table-&gt;eq)</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-		roce_set_field(srq_context-&gt;byte_8_limit_wl,</div><div class='del'>-			       SRQC_BYTE_8_SRQ_LIMIT_WL_M,</div><div class='del'>-			       SRQC_BYTE_8_SRQ_LIMIT_WL_S, srq_attr-&gt;srq_limit);</div><div class='del'>-		roce_set_field(srqc_mask-&gt;byte_8_limit_wl,</div><div class='del'>-			       SRQC_BYTE_8_SRQ_LIMIT_WL_M,</div><div class='del'>-			       SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);</div><div class='add'>+	/* create eq */</div><div class='add'>+	for (i = 0; i &lt; eq_num; i++) {</div><div class='add'>+		eq = &amp;eq_table-&gt;eq[i];</div><div class='add'>+		eq-&gt;hr_dev = hr_dev;</div><div class='add'>+		eq-&gt;eqn = i;</div><div class='add'>+		if (i &lt; comp_num) {</div><div class='add'>+			/* CEQ */</div><div class='add'>+			eq_cmd = HNS_ROCE_CMD_CREATE_CEQC;</div><div class='add'>+			eq-&gt;type_flag = HNS_ROCE_CEQ;</div><div class='add'>+			eq-&gt;entries = hr_dev-&gt;caps.ceqe_depth;</div><div class='add'>+			eq-&gt;eqe_size = hr_dev-&gt;caps.ceqe_size;</div><div class='add'>+			eq-&gt;irq = hr_dev-&gt;irq[i + other_num + aeq_num];</div><div class='add'>+			eq-&gt;eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM;</div><div class='add'>+			eq-&gt;eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL;</div><div class='add'>+		} else {</div><div class='add'>+			/* AEQ */</div><div class='add'>+			eq_cmd = HNS_ROCE_CMD_CREATE_AEQC;</div><div class='add'>+			eq-&gt;type_flag = HNS_ROCE_AEQ;</div><div class='add'>+			eq-&gt;entries = hr_dev-&gt;caps.aeqe_depth;</div><div class='add'>+			eq-&gt;eqe_size = hr_dev-&gt;caps.aeqe_size;</div><div class='add'>+			eq-&gt;irq = hr_dev-&gt;irq[i - comp_num + other_num];</div><div class='add'>+			eq-&gt;eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM;</div><div class='add'>+			eq-&gt;eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='del'>-		ret = hns_roce_cmd_mbox(hr_dev, mailbox-&gt;dma, 0, srq-&gt;srqn, 0,</div><div class='del'>-					HNS_ROCE_CMD_MODIFY_SRQC,</div><div class='del'>-					HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='del'>-		hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='add'>+		ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd);</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			dev_err(hr_dev-&gt;dev,</div><div class='del'>-				"MODIFY SRQ Failed to cmd mailbox.\n");</div><div class='del'>-			return ret;</div><div class='add'>+			dev_err(dev, "failed to create eq.\n");</div><div class='add'>+			goto err_create_eq_fail;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='add'>+	INIT_WORK(&amp;hr_dev-&gt;ecc_work, fmea_ram_ecc_work);</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq-&gt;device);</div><div class='del'>-	struct hns_roce_srq *srq = to_hr_srq(ibsrq);</div><div class='del'>-	struct hns_roce_srq_context *srq_context;</div><div class='del'>-	struct hns_roce_cmd_mailbox *mailbox;</div><div class='del'>-	int limit_wl;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='del'>-	if (IS_ERR(mailbox))</div><div class='del'>-		return PTR_ERR(mailbox);</div><div class='add'>+	hr_dev-&gt;irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", 0);</div><div class='add'>+	if (!hr_dev-&gt;irq_workq) {</div><div class='add'>+		dev_err(dev, "failed to create irq workqueue.\n");</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err_create_eq_fail;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	srq_context = mailbox-&gt;buf;</div><div class='del'>-	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox-&gt;dma, srq-&gt;srqn, 0,</div><div class='del'>-				HNS_ROCE_CMD_QUERY_SRQC,</div><div class='del'>-				HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='add'>+	ret = __hns_roce_request_irq(hr_dev, irq_num, comp_num, aeq_num,</div><div class='add'>+				     other_num);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "QUERY SRQ cmd process error\n");</div><div class='del'>-		goto out;</div><div class='add'>+		dev_err(dev, "failed to request irq.\n");</div><div class='add'>+		goto err_request_irq_fail;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	limit_wl = roce_get_field(srq_context-&gt;byte_8_limit_wl,</div><div class='del'>-				  SRQC_BYTE_8_SRQ_LIMIT_WL_M,</div><div class='del'>-				  SRQC_BYTE_8_SRQ_LIMIT_WL_S);</div><div class='del'>-</div><div class='del'>-	attr-&gt;srq_limit = limit_wl;</div><div class='del'>-	attr-&gt;max_wr    = srq-&gt;max - 1;</div><div class='del'>-	attr-&gt;max_sge   = srq-&gt;max_gs;</div><div class='del'>-</div><div class='del'>-	memcpy(srq_context, mailbox-&gt;buf, sizeof(*srq_context));</div><div class='del'>-</div><div class='del'>-out:</div><div class='del'>-	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int find_empty_entry(struct hns_roce_idx_que *idx_que)</div><div class='del'>-{</div><div class='del'>-	int bit_num;</div><div class='del'>-	int i;</div><div class='add'>+	/* enable irq */</div><div class='add'>+	hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE);</div><div class='ctx'> </div><div class='del'>-	/* bitmap[i] is set zero if all bits are allocated */</div><div class='del'>-	for (i = 0; idx_que-&gt;bitmap[i] == 0; ++i)</div><div class='del'>-		;</div><div class='del'>-	bit_num = ffs(idx_que-&gt;bitmap[i]);</div><div class='del'>-	idx_que-&gt;bitmap[i] &amp;= ~(1ULL &lt;&lt; (bit_num - 1));</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-	return i * sizeof(u64) * 8 + (bit_num - 1);</div><div class='del'>-}</div><div class='add'>+err_request_irq_fail:</div><div class='add'>+	destroy_workqueue(hr_dev-&gt;irq_workq);</div><div class='ctx'> </div><div class='del'>-static void fill_idx_queue(struct hns_roce_idx_que *idx_que,</div><div class='del'>-			   int cur_idx, int wqe_idx)</div><div class='del'>-{</div><div class='del'>-	unsigned int *addr;</div><div class='add'>+err_create_eq_fail:</div><div class='add'>+	for (i -= 1; i &gt;= 0; i--)</div><div class='add'>+		hns_roce_v2_destroy_eqc(hr_dev, &amp;eq_table-&gt;eq[i]);</div><div class='add'>+	kfree(eq_table-&gt;eq);</div><div class='ctx'> </div><div class='del'>-	addr = (unsigned int *)hns_roce_buf_offset(&amp;idx_que-&gt;idx_buf,</div><div class='del'>-						   cur_idx * idx_que-&gt;entry_sz);</div><div class='del'>-	*addr = wqe_idx;</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,</div><div class='del'>-				     const struct ib_recv_wr *wr,</div><div class='del'>-				     const struct ib_recv_wr **bad_wr)</div><div class='add'>+static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq-&gt;device);</div><div class='del'>-	struct hns_roce_srq *srq = to_hr_srq(ibsrq);</div><div class='del'>-	struct hns_roce_v2_wqe_data_seg *dseg;</div><div class='del'>-	struct hns_roce_v2_db srq_db;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int ret = 0;</div><div class='del'>-	int wqe_idx;</div><div class='del'>-	void *wqe;</div><div class='del'>-	int nreq;</div><div class='del'>-	int ind;</div><div class='add'>+	struct hns_roce_eq_table *eq_table = &amp;hr_dev-&gt;eq_table;</div><div class='add'>+	int eq_num;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	spin_lock_irqsave(&amp;srq-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-	ind = srq-&gt;head &amp; (srq-&gt;max - 1);</div><div class='del'>-</div><div class='del'>-	for (nreq = 0; wr; ++nreq, wr = wr-&gt;next) {</div><div class='del'>-		if (unlikely(wr-&gt;num_sge &gt; srq-&gt;max_gs)) {</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			*bad_wr = wr;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (unlikely(srq-&gt;head == srq-&gt;tail)) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			*bad_wr = wr;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		wqe_idx = find_empty_entry(&amp;srq-&gt;idx_que);</div><div class='del'>-		fill_idx_queue(&amp;srq-&gt;idx_que, ind, wqe_idx);</div><div class='del'>-		wqe = get_srq_wqe(srq, wqe_idx);</div><div class='del'>-		dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;</div><div class='del'>-</div><div class='del'>-		for (i = 0; i &lt; wr-&gt;num_sge; ++i) {</div><div class='del'>-			dseg[i].len = cpu_to_le32(wr-&gt;sg_list[i].length);</div><div class='del'>-			dseg[i].lkey = cpu_to_le32(wr-&gt;sg_list[i].lkey);</div><div class='del'>-			dseg[i].addr = cpu_to_le64(wr-&gt;sg_list[i].addr);</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (i &lt; srq-&gt;max_gs) {</div><div class='del'>-			dseg-&gt;len = 0;</div><div class='del'>-			dseg-&gt;lkey = cpu_to_le32(0x100);</div><div class='del'>-			dseg-&gt;addr = 0;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		srq-&gt;wrid[wqe_idx] = wr-&gt;wr_id;</div><div class='del'>-		ind = (ind + 1) &amp; (srq-&gt;max - 1);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (likely(nreq)) {</div><div class='del'>-		srq-&gt;head += nreq;</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * Make sure that descriptors are written before</div><div class='del'>-		 * doorbell record.</div><div class='del'>-		 */</div><div class='del'>-		wmb();</div><div class='del'>-</div><div class='del'>-		srq_db.byte_4 = HNS_ROCE_V2_SRQ_DB &lt;&lt; 24 | srq-&gt;srqn;</div><div class='del'>-		srq_db.parameter = srq-&gt;head;</div><div class='add'>+	eq_num = hr_dev-&gt;caps.num_comp_vectors + hr_dev-&gt;caps.num_aeq_vectors;</div><div class='ctx'> </div><div class='del'>-		hns_roce_write64(hr_dev, (__le32 *)&amp;srq_db, srq-&gt;db_reg_l);</div><div class='add'>+	/* Disable irq */</div><div class='add'>+	hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);</div><div class='ctx'> </div><div class='del'>-	}</div><div class='add'>+	__hns_roce_free_irq(hr_dev);</div><div class='add'>+	destroy_workqueue(hr_dev-&gt;irq_workq);</div><div class='ctx'> </div><div class='del'>-	spin_unlock_irqrestore(&amp;srq-&gt;lock, flags);</div><div class='add'>+	for (i = 0; i &lt; eq_num; i++)</div><div class='add'>+		hns_roce_v2_destroy_eqc(hr_dev, &amp;eq_table-&gt;eq[i]);</div><div class='ctx'> </div><div class='del'>-	return ret;</div><div class='add'>+	kfree(eq_table-&gt;eq);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {</div><div class='del'>-	.query_cqc_info = hns_roce_v2_query_cqc_info,</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> static const struct ib_device_ops hns_roce_v2_dev_ops = {</div><div class='ctx'> 	.destroy_qp = hns_roce_v2_destroy_qp,</div><div class='ctx'> 	.modify_cq = hns_roce_v2_modify_cq,</div><div class='hunk'>@@ -6097,33 +7028,30 @@ static const struct hns_roce_hw hns_roce_hw_v2 = {</div><div class='ctx'> 	.hw_profile = hns_roce_v2_profile,</div><div class='ctx'> 	.hw_init = hns_roce_v2_init,</div><div class='ctx'> 	.hw_exit = hns_roce_v2_exit,</div><div class='del'>-	.post_mbox = hns_roce_v2_post_mbox,</div><div class='del'>-	.chk_mbox = hns_roce_v2_chk_mbox,</div><div class='del'>-	.rst_prc_mbox = hns_roce_v2_rst_process_cmd,</div><div class='add'>+	.post_mbox = v2_post_mbox,</div><div class='add'>+	.poll_mbox_done = v2_poll_mbox_done,</div><div class='add'>+	.chk_mbox_avail = v2_chk_mbox_is_avail,</div><div class='ctx'> 	.set_gid = hns_roce_v2_set_gid,</div><div class='ctx'> 	.set_mac = hns_roce_v2_set_mac,</div><div class='ctx'> 	.write_mtpt = hns_roce_v2_write_mtpt,</div><div class='ctx'> 	.rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt,</div><div class='ctx'> 	.frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt,</div><div class='del'>-	.mw_write_mtpt = hns_roce_v2_mw_write_mtpt,</div><div class='ctx'> 	.write_cqc = hns_roce_v2_write_cqc,</div><div class='ctx'> 	.set_hem = hns_roce_v2_set_hem,</div><div class='ctx'> 	.clear_hem = hns_roce_v2_clear_hem,</div><div class='ctx'> 	.modify_qp = hns_roce_v2_modify_qp,</div><div class='del'>-	.query_qp = hns_roce_v2_query_qp,</div><div class='del'>-	.destroy_qp = hns_roce_v2_destroy_qp,</div><div class='add'>+	.dereg_mr = hns_roce_v2_dereg_mr,</div><div class='ctx'> 	.qp_flow_control_init = hns_roce_v2_qp_flow_control_init,</div><div class='del'>-	.modify_cq = hns_roce_v2_modify_cq,</div><div class='del'>-	.post_send = hns_roce_v2_post_send,</div><div class='del'>-	.post_recv = hns_roce_v2_post_recv,</div><div class='del'>-	.req_notify_cq = hns_roce_v2_req_notify_cq,</div><div class='del'>-	.poll_cq = hns_roce_v2_poll_cq,</div><div class='ctx'> 	.init_eq = hns_roce_v2_init_eq_table,</div><div class='ctx'> 	.cleanup_eq = hns_roce_v2_cleanup_eq_table,</div><div class='ctx'> 	.write_srqc = hns_roce_v2_write_srqc,</div><div class='del'>-	.modify_srq = hns_roce_v2_modify_srq,</div><div class='del'>-	.query_srq = hns_roce_v2_query_srq,</div><div class='del'>-	.post_srq_recv = hns_roce_v2_post_srq_recv,</div><div class='add'>+	.query_cqc = hns_roce_v2_query_cqc,</div><div class='add'>+	.query_qpc = hns_roce_v2_query_qpc,</div><div class='add'>+	.query_mpt = hns_roce_v2_query_mpt,</div><div class='add'>+	.query_srqc = hns_roce_v2_query_srqc,</div><div class='add'>+	.query_sccc = hns_roce_v2_query_sccc,</div><div class='add'>+	.query_hw_counter = hns_roce_hw_v2_query_counter,</div><div class='add'>+	.get_dscp = hns_roce_hw_v2_get_dscp,</div><div class='ctx'> 	.hns_roce_dev_ops = &amp;hns_roce_v2_dev_ops,</div><div class='ctx'> 	.hns_roce_dev_srq_ops = &amp;hns_roce_v2_dev_srq_ops,</div><div class='ctx'> };</div><div class='hunk'>@@ -6134,25 +7062,33 @@ static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {</div><div class='ctx'> 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},</div><div class='ctx'> 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},</div><div class='ctx'> 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},</div><div class='add'>+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 0},</div><div class='add'>+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),</div><div class='add'>+	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},</div><div class='ctx'> 	/* required last entry */</div><div class='ctx'> 	{0, }</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl);</div><div class='ctx'> </div><div class='del'>-static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,</div><div class='add'>+static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 				  struct hnae3_handle *handle)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='add'>+	const struct pci_device_id *id;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='add'>+	hr_dev-&gt;pci_dev = handle-&gt;pdev;</div><div class='add'>+	id = pci_match_id(hns_roce_hw_v2_pci_tbl, hr_dev-&gt;pci_dev);</div><div class='add'>+	hr_dev-&gt;is_vf = id-&gt;driver_data;</div><div class='add'>+	hr_dev-&gt;dev = &amp;handle-&gt;pdev-&gt;dev;</div><div class='ctx'> 	hr_dev-&gt;hw = &amp;hns_roce_hw_v2;</div><div class='del'>-	hr_dev-&gt;dfx = &amp;hns_roce_dfx_hw_v2;</div><div class='ctx'> 	hr_dev-&gt;sdb_offset = ROCEE_DB_SQ_L_0_REG;</div><div class='ctx'> 	hr_dev-&gt;odb_offset = hr_dev-&gt;sdb_offset;</div><div class='ctx'> </div><div class='ctx'> 	/* Get info from NIC driver. */</div><div class='ctx'> 	hr_dev-&gt;reg_base = handle-&gt;rinfo.roce_io_base;</div><div class='add'>+	hr_dev-&gt;mem_base = handle-&gt;rinfo.roce_mem_base;</div><div class='ctx'> 	hr_dev-&gt;caps.num_ports = 1;</div><div class='ctx'> 	hr_dev-&gt;iboe.netdevs[0] = handle-&gt;rinfo.netdev;</div><div class='ctx'> 	hr_dev-&gt;iboe.phy_port[0] = 0;</div><div class='hunk'>@@ -6160,7 +7096,7 @@ static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 	addrconf_addr_eui48((u8 *)&amp;hr_dev-&gt;ib_dev.node_guid,</div><div class='ctx'> 			    hr_dev-&gt;iboe.netdevs[0]-&gt;dev_addr);</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; HNS_ROCE_V2_MAX_IRQ_NUM; i++)</div><div class='add'>+	for (i = 0; i &lt; handle-&gt;rinfo.num_vectors; i++)</div><div class='ctx'> 		hr_dev-&gt;irq[i] = pci_irq_vector(handle-&gt;pdev,</div><div class='ctx'> 						i + handle-&gt;rinfo.base_vector);</div><div class='ctx'> </div><div class='hunk'>@@ -6170,8 +7106,6 @@ static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,</div><div class='ctx'> </div><div class='ctx'> 	hr_dev-&gt;reset_cnt = handle-&gt;ae_algo-&gt;ops-&gt;ae_dev_reset_cnt(handle);</div><div class='ctx'> 	priv-&gt;handle = handle;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)</div><div class='hunk'>@@ -6189,26 +7123,19 @@ static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)</div><div class='ctx'> 		goto error_failed_kzalloc;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	hr_dev-&gt;pci_dev = handle-&gt;pdev;</div><div class='del'>-	hr_dev-&gt;dev = &amp;handle-&gt;pdev-&gt;dev;</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_hw_v2_get_cfg(hr_dev, handle);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "Get Configuration failed!\n");</div><div class='del'>-		goto error_failed_get_cfg;</div><div class='del'>-	}</div><div class='add'>+	hns_roce_hw_v2_get_cfg(hr_dev, handle);</div><div class='ctx'> </div><div class='ctx'> 	ret = hns_roce_init(hr_dev);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		dev_err(hr_dev-&gt;dev, "RoCE Engine init failed!\n");</div><div class='del'>-		goto error_failed_get_cfg;</div><div class='add'>+		goto error_failed_roce_init;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	handle-&gt;priv = hr_dev;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-error_failed_get_cfg:</div><div class='add'>+error_failed_roce_init:</div><div class='ctx'> 	kfree(hr_dev-&gt;priv);</div><div class='ctx'> </div><div class='ctx'> error_failed_kzalloc:</div><div class='hunk'>@@ -6218,15 +7145,19 @@ error_failed_kzalloc:</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,</div><div class='del'>-					   bool reset)</div><div class='add'>+					   bool reset, bool bond_cleanup)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)handle-&gt;priv;</div><div class='add'>+	struct hns_roce_dev *hr_dev = handle-&gt;priv;</div><div class='ctx'> </div><div class='ctx'> 	if (!hr_dev)</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='ctx'> 	handle-&gt;priv = NULL;</div><div class='del'>-	hns_roce_exit(hr_dev);</div><div class='add'>+</div><div class='add'>+	hr_dev-&gt;state = HNS_ROCE_DEVICE_STATE_UNINIT;</div><div class='add'>+	hns_roce_handle_device_err(hr_dev);</div><div class='add'>+</div><div class='add'>+	hns_roce_exit(hr_dev, bond_cleanup);</div><div class='ctx'> 	kfree(hr_dev-&gt;priv);</div><div class='ctx'> 	ib_dealloc_device(&amp;hr_dev-&gt;ib_dev);</div><div class='ctx'> }</div><div class='hunk'>@@ -6249,6 +7180,9 @@ static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)</div><div class='ctx'> 	if (!id)</div><div class='ctx'> 		return 0;</div><div class='ctx'> </div><div class='add'>+	if (id-&gt;driver_data &amp;&amp; handle-&gt;pdev-&gt;revision == PCI_REVISION_ID_HIP08)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='ctx'> 	ret = __hns_roce_hw_v2_init_instance(handle);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		handle-&gt;rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;</div><div class='hunk'>@@ -6262,7 +7196,6 @@ static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)</div><div class='ctx'> </div><div class='ctx'> 	handle-&gt;rinfo.instance_state = HNS_ROCE_STATE_INITED;</div><div class='ctx'> </div><div class='del'>-</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> reset_chk_err:</div><div class='hunk'>@@ -6275,19 +7208,61 @@ reset_chk_err:</div><div class='ctx'> static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,</div><div class='ctx'> 					   bool reset)</div><div class='ctx'> {</div><div class='add'>+	/* Suspend bond to avoid concurrency */</div><div class='add'>+	hns_roce_bond_suspend(handle);</div><div class='add'>+</div><div class='ctx'> 	if (handle-&gt;rinfo.instance_state != HNS_ROCE_STATE_INITED)</div><div class='del'>-		return;</div><div class='add'>+		goto out;</div><div class='ctx'> </div><div class='ctx'> 	handle-&gt;rinfo.instance_state = HNS_ROCE_STATE_UNINIT;</div><div class='ctx'> </div><div class='del'>-	__hns_roce_hw_v2_uninit_instance(handle, reset);</div><div class='add'>+	__hns_roce_hw_v2_uninit_instance(handle, reset, true);</div><div class='add'>+</div><div class='add'>+	handle-&gt;rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	hns_roce_bond_resume(handle);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct hns_roce_dev</div><div class='add'>+	*hns_roce_bond_init_client(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+				   int func_idx)</div><div class='add'>+{</div><div class='add'>+	struct hnae3_handle *handle;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	handle = bond_grp-&gt;bond_func_info[func_idx].handle;</div><div class='add'>+	if (!handle || !handle-&gt;client)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_hw_v2_init_instance(handle);</div><div class='add'>+	if (ret)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	return handle-&gt;priv;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hns_roce_bond_uninit_client(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+				 int func_idx)</div><div class='add'>+{</div><div class='add'>+	struct hnae3_handle *handle = bond_grp-&gt;bond_func_info[func_idx].handle;</div><div class='add'>+</div><div class='add'>+	if (handle-&gt;rinfo.instance_state != HNS_ROCE_STATE_INITED)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	handle-&gt;rinfo.instance_state = HNS_ROCE_STATE_BOND_UNINIT;</div><div class='add'>+</div><div class='add'>+	__hns_roce_hw_v2_uninit_instance(handle, false, false);</div><div class='ctx'> </div><div class='ctx'> 	handle-&gt;rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;</div><div class='ctx'> }</div><div class='add'>+</div><div class='ctx'> static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev;</div><div class='del'>-	struct ib_event event;</div><div class='add'>+</div><div class='add'>+	/* Suspend bond to avoid concurrency */</div><div class='add'>+	hns_roce_bond_suspend(handle);</div><div class='ctx'> </div><div class='ctx'> 	if (handle-&gt;rinfo.instance_state != HNS_ROCE_STATE_INITED) {</div><div class='ctx'> 		set_bit(HNS_ROCE_RST_DIRECT_RETURN, &amp;handle-&gt;rinfo.state);</div><div class='hunk'>@@ -6297,17 +7272,16 @@ static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)</div><div class='ctx'> 	handle-&gt;rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN;</div><div class='ctx'> 	clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &amp;handle-&gt;rinfo.state);</div><div class='ctx'> </div><div class='del'>-	hr_dev = (struct hns_roce_dev *)handle-&gt;priv;</div><div class='add'>+	hr_dev = handle-&gt;priv;</div><div class='ctx'> 	if (!hr_dev)</div><div class='ctx'> 		return 0;</div><div class='ctx'> </div><div class='ctx'> 	hr_dev-&gt;active = false;</div><div class='ctx'> 	hr_dev-&gt;dis_db = true;</div><div class='ctx'> </div><div class='del'>-	event.event = IB_EVENT_DEVICE_FATAL;</div><div class='del'>-	event.device = &amp;hr_dev-&gt;ib_dev;</div><div class='del'>-	event.element.port_num = 1;</div><div class='del'>-	ib_dispatch_event(&amp;event);</div><div class='add'>+	rdma_user_mmap_disassociate(&amp;hr_dev-&gt;ib_dev);</div><div class='add'>+</div><div class='add'>+	hr_dev-&gt;state = HNS_ROCE_DEVICE_STATE_RST_DOWN;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -6320,6 +7294,7 @@ static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle)</div><div class='ctx'> 	if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN,</div><div class='ctx'> 			       &amp;handle-&gt;rinfo.state)) {</div><div class='ctx'> 		handle-&gt;rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;</div><div class='add'>+		hns_roce_bond_resume(handle);</div><div class='ctx'> 		return 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -6336,9 +7311,10 @@ static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle)</div><div class='ctx'> 		dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret);</div><div class='ctx'> 	} else {</div><div class='ctx'> 		handle-&gt;rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;</div><div class='del'>-		dev_info(dev, "Reset done, RoCE client reinit finished.\n");</div><div class='add'>+		dev_info(dev, "reset done, RoCE client reinit finished.\n");</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	hns_roce_bond_resume(handle);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -6349,8 +7325,8 @@ static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle)</div><div class='ctx'> </div><div class='ctx'> 	handle-&gt;rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT;</div><div class='ctx'> 	dev_info(&amp;handle-&gt;pdev-&gt;dev, "In reset process RoCE client uninit.\n");</div><div class='del'>-	msleep(100);</div><div class='del'>-	__hns_roce_hw_v2_uninit_instance(handle, false);</div><div class='add'>+	msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY);</div><div class='add'>+	__hns_roce_hw_v2_uninit_instance(handle, false, false);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -6377,9 +7353,30 @@ static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle,</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void hns_roce_hw_v2_link_status_change(struct hnae3_handle *handle,</div><div class='add'>+					      bool linkup)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)handle-&gt;priv;</div><div class='add'>+	struct net_device *netdev = handle-&gt;rinfo.netdev;</div><div class='add'>+</div><div class='add'>+	if (linkup || !hr_dev)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	/* For bond device, the link status depends on the upper netdev,</div><div class='add'>+	 * and the upper device's link status depends on all the slaves'</div><div class='add'>+	 * netdev but not only one. So bond device cannot get a correct</div><div class='add'>+	 * link status from this path.</div><div class='add'>+	 */</div><div class='add'>+	if (hns_roce_get_bond_grp(netdev, get_hr_bus_num(hr_dev)))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	ib_dispatch_port_state_event(&amp;hr_dev-&gt;ib_dev, netdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static const struct hnae3_client_ops hns_roce_hw_v2_ops = {</div><div class='ctx'> 	.init_instance = hns_roce_hw_v2_init_instance,</div><div class='ctx'> 	.uninit_instance = hns_roce_hw_v2_uninit_instance,</div><div class='add'>+	.link_status_change = hns_roce_hw_v2_link_status_change,</div><div class='ctx'> 	.reset_notify = hns_roce_hw_v2_reset_notify,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -6391,12 +7388,15 @@ static struct hnae3_client hns_roce_hw_v2_client = {</div><div class='ctx'> </div><div class='ctx'> static int __init hns_roce_hw_v2_init(void)</div><div class='ctx'> {</div><div class='add'>+	hns_roce_init_debugfs();</div><div class='ctx'> 	return hnae3_register_client(&amp;hns_roce_hw_v2_client);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void __exit hns_roce_hw_v2_exit(void)</div><div class='ctx'> {</div><div class='add'>+	hns_roce_dealloc_bond_grp();</div><div class='ctx'> 	hnae3_unregister_client(&amp;hns_roce_hw_v2_client);</div><div class='add'>+	hns_roce_cleanup_debugfs();</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> module_init(hns_roce_hw_v2_init);</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h<br/>index edfdbe2ce0db..285fe0875fac 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_hw_v2.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_hw_v2.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_hw_v2.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_hw_v2.h</a></div><div class='hunk'>@@ -34,69 +34,38 @@</div><div class='ctx'> #define _HNS_ROCE_HW_V2_H</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/bitops.h&gt;</div><div class='add'>+#include "hnae3.h"</div><div class='add'>+#include "hns_roce_bond.h"</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_VF_QPC_BT_NUM			256</div><div class='del'>-#define HNS_ROCE_VF_SCCC_BT_NUM			64</div><div class='del'>-#define HNS_ROCE_VF_SRQC_BT_NUM			64</div><div class='del'>-#define HNS_ROCE_VF_CQC_BT_NUM			64</div><div class='del'>-#define HNS_ROCE_VF_MPT_BT_NUM			64</div><div class='del'>-#define HNS_ROCE_VF_EQC_NUM			64</div><div class='del'>-#define HNS_ROCE_VF_SMAC_NUM			32</div><div class='del'>-#define HNS_ROCE_VF_SGID_NUM			32</div><div class='del'>-#define HNS_ROCE_VF_SL_NUM			8</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V2_MAX_QP_NUM			0x100000</div><div class='del'>-#define HNS_ROCE_V2_MAX_QPC_TIMER_NUM		0x200</div><div class='del'>-#define HNS_ROCE_V2_MAX_WQE_NUM			0x8000</div><div class='del'>-#define	HNS_ROCE_V2_MAX_SRQ			0x100000</div><div class='del'>-#define HNS_ROCE_V2_MAX_SRQ_WR			0x8000</div><div class='del'>-#define HNS_ROCE_V2_MAX_SRQ_SGE			0x100</div><div class='del'>-#define HNS_ROCE_V2_MAX_CQ_NUM			0x100000</div><div class='del'>-#define HNS_ROCE_V2_MAX_CQC_TIMER_NUM		0x100</div><div class='del'>-#define HNS_ROCE_V2_MAX_SRQ_NUM			0x100000</div><div class='del'>-#define HNS_ROCE_V2_MAX_CQE_NUM			0x10000</div><div class='del'>-#define HNS_ROCE_V2_MAX_SRQWQE_NUM		0x8000</div><div class='del'>-#define HNS_ROCE_V2_MAX_RQ_SGE_NUM		0x100</div><div class='del'>-#define HNS_ROCE_V2_MAX_SQ_SGE_NUM		0xff</div><div class='del'>-#define HNS_ROCE_V2_MAX_SRQ_SGE_NUM		0x100</div><div class='del'>-#define HNS_ROCE_V2_MAX_EXTEND_SGE_NUM		0x200000</div><div class='del'>-#define HNS_ROCE_V2_MAX_SQ_INLINE		0x20</div><div class='del'>-#define HNS_ROCE_V2_UAR_NUM			256</div><div class='del'>-#define HNS_ROCE_V2_PHY_UAR_NUM			1</div><div class='del'>-#define HNS_ROCE_V2_MAX_IRQ_NUM			65</div><div class='del'>-#define HNS_ROCE_V2_COMP_VEC_NUM		63</div><div class='add'>+#define HNS_ROCE_V2_MAX_RC_INL_INN_SZ		32</div><div class='add'>+#define HNS_ROCE_V2_MTT_ENTRY_SZ		64</div><div class='ctx'> #define HNS_ROCE_V2_AEQE_VEC_NUM		1</div><div class='ctx'> #define HNS_ROCE_V2_ABNORMAL_VEC_NUM		1</div><div class='del'>-#define HNS_ROCE_V2_MAX_MTPT_NUM		0x100000</div><div class='del'>-#define HNS_ROCE_V2_MAX_MTT_SEGS		0x1000000</div><div class='del'>-#define HNS_ROCE_V2_MAX_CQE_SEGS		0x1000000</div><div class='ctx'> #define HNS_ROCE_V2_MAX_SRQWQE_SEGS		0x1000000</div><div class='ctx'> #define HNS_ROCE_V2_MAX_IDX_SEGS		0x1000000</div><div class='del'>-#define HNS_ROCE_V2_MAX_PD_NUM			0x1000000</div><div class='del'>-#define HNS_ROCE_V2_MAX_QP_INIT_RDMA		128</div><div class='del'>-#define HNS_ROCE_V2_MAX_QP_DEST_RDMA		128</div><div class='del'>-#define HNS_ROCE_V2_MAX_SQ_DESC_SZ		64</div><div class='del'>-#define HNS_ROCE_V2_MAX_RQ_DESC_SZ		16</div><div class='del'>-#define HNS_ROCE_V2_MAX_SRQ_DESC_SZ		64</div><div class='del'>-#define HNS_ROCE_V2_QPC_ENTRY_SZ		256</div><div class='del'>-#define HNS_ROCE_V2_IRRL_ENTRY_SZ		64</div><div class='del'>-#define HNS_ROCE_V2_TRRL_ENTRY_SZ		48</div><div class='del'>-#define HNS_ROCE_V2_CQC_ENTRY_SZ		64</div><div class='del'>-#define HNS_ROCE_V2_SRQC_ENTRY_SZ		64</div><div class='del'>-#define HNS_ROCE_V2_MTPT_ENTRY_SZ		64</div><div class='del'>-#define HNS_ROCE_V2_MTT_ENTRY_SZ		64</div><div class='del'>-#define HNS_ROCE_V2_CQE_ENTRY_SIZE		32</div><div class='del'>-#define HNS_ROCE_V2_SCCC_ENTRY_SZ		32</div><div class='del'>-#define HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ		4096</div><div class='del'>-#define HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ		4096</div><div class='del'>-#define HNS_ROCE_V2_PAGE_SIZE_SUPPORTED		0xFFFFF000</div><div class='add'>+#define HNS_ROCE_V2_MAX_XRCD_NUM		0x1000000</div><div class='add'>+</div><div class='add'>+#define HNS_ROCE_V2_QP_ACK_TIMEOUT_OFS_HIP08    10</div><div class='add'>+</div><div class='add'>+#define HNS_ROCE_V3_SCCC_SZ			64</div><div class='add'>+#define HNS_ROCE_V3_GMV_ENTRY_SZ		32</div><div class='add'>+</div><div class='add'>+#define HNS_ROCE_V2_EXT_LLM_ENTRY_SZ		8</div><div class='add'>+#define HNS_ROCE_V2_EXT_LLM_MAX_DEPTH		4096</div><div class='add'>+</div><div class='add'>+#define HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ		PAGE_SIZE</div><div class='add'>+#define HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ		PAGE_SIZE</div><div class='add'>+#define HNS_ROCE_V2_PAGE_SIZE_SUPPORTED		0xFFFF000</div><div class='ctx'> #define HNS_ROCE_V2_MAX_INNER_MTPT_NUM		2</div><div class='del'>-#define HNS_ROCE_INVALID_LKEY			0x100</div><div class='add'>+#define HNS_ROCE_INVALID_LKEY			0x0</div><div class='add'>+#define HNS_ROCE_INVALID_SGE_LENGTH		0x80000000</div><div class='ctx'> #define HNS_ROCE_CMQ_TX_TIMEOUT			30000</div><div class='del'>-#define HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE	2</div><div class='ctx'> #define HNS_ROCE_V2_RSV_QPS			8</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_V2_HW_RST_TIMEOUT             1000</div><div class='add'>+#define HNS_ROCE_V2_HW_RST_TIMEOUT		1000</div><div class='add'>+#define HNS_ROCE_V2_HW_RST_UNINT_DELAY		100</div><div class='add'>+</div><div class='add'>+#define HNS_ROCE_V2_HW_RST_COMPLETION_WAIT	20</div><div class='ctx'> </div><div class='ctx'> #define HNS_ROCE_CONTEXT_HOP_NUM		1</div><div class='ctx'> #define HNS_ROCE_SCCC_HOP_NUM			1</div><div class='hunk'>@@ -104,33 +73,39 @@</div><div class='ctx'> #define HNS_ROCE_CQE_HOP_NUM			1</div><div class='ctx'> #define HNS_ROCE_SRQWQE_HOP_NUM			1</div><div class='ctx'> #define HNS_ROCE_PBL_HOP_NUM			2</div><div class='del'>-#define HNS_ROCE_EQE_HOP_NUM			2</div><div class='ctx'> #define HNS_ROCE_IDX_HOP_NUM			1</div><div class='add'>+#define HNS_ROCE_SQWQE_HOP_NUM			2</div><div class='add'>+#define HNS_ROCE_EXT_SGE_HOP_NUM		1</div><div class='add'>+#define HNS_ROCE_RQWQE_HOP_NUM			2</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_V2_GID_INDEX_NUM		256</div><div class='add'>+#define HNS_ROCE_V2_EQE_HOP_NUM			2</div><div class='add'>+#define HNS_ROCE_V3_EQE_HOP_NUM			1</div><div class='add'>+</div><div class='add'>+#define HNS_ROCE_BA_PG_SZ_SUPPORTED_256K	6</div><div class='add'>+#define HNS_ROCE_BA_PG_SZ_SUPPORTED_16K		2</div><div class='add'>+#define HNS_ROCE_V2_GID_INDEX_NUM		16</div><div class='ctx'> </div><div class='ctx'> #define HNS_ROCE_V2_TABLE_CHUNK_SIZE		(1 &lt;&lt; 18)</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_CMD_FLAG_IN_VALID_SHIFT	0</div><div class='del'>-#define HNS_ROCE_CMD_FLAG_OUT_VALID_SHIFT	1</div><div class='del'>-#define HNS_ROCE_CMD_FLAG_NEXT_SHIFT		2</div><div class='del'>-#define HNS_ROCE_CMD_FLAG_WR_OR_RD_SHIFT	3</div><div class='del'>-#define HNS_ROCE_CMD_FLAG_NO_INTR_SHIFT		4</div><div class='del'>-#define HNS_ROCE_CMD_FLAG_ERR_INTR_SHIFT	5</div><div class='add'>+/* budget must be smaller than aeqe_depth to guarantee that we update</div><div class='add'>+ * the ci before we polled all the entries in the EQ.</div><div class='add'>+ */</div><div class='add'>+#define HNS_AEQ_POLLING_BUDGET 64</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_CMD_FLAG_IN		BIT(HNS_ROCE_CMD_FLAG_IN_VALID_SHIFT)</div><div class='del'>-#define HNS_ROCE_CMD_FLAG_OUT		BIT(HNS_ROCE_CMD_FLAG_OUT_VALID_SHIFT)</div><div class='del'>-#define HNS_ROCE_CMD_FLAG_NEXT		BIT(HNS_ROCE_CMD_FLAG_NEXT_SHIFT)</div><div class='del'>-#define HNS_ROCE_CMD_FLAG_WR		BIT(HNS_ROCE_CMD_FLAG_WR_OR_RD_SHIFT)</div><div class='del'>-#define HNS_ROCE_CMD_FLAG_NO_INTR	BIT(HNS_ROCE_CMD_FLAG_NO_INTR_SHIFT)</div><div class='del'>-#define HNS_ROCE_CMD_FLAG_ERR_INTR	BIT(HNS_ROCE_CMD_FLAG_ERR_INTR_SHIFT)</div><div class='add'>+enum {</div><div class='add'>+	HNS_ROCE_CMD_FLAG_IN = BIT(0),</div><div class='add'>+	HNS_ROCE_CMD_FLAG_OUT = BIT(1),</div><div class='add'>+	HNS_ROCE_CMD_FLAG_NEXT = BIT(2),</div><div class='add'>+	HNS_ROCE_CMD_FLAG_WR = BIT(3),</div><div class='add'>+	HNS_ROCE_CMD_FLAG_ERR_INTR = BIT(5),</div><div class='add'>+};</div><div class='ctx'> </div><div class='ctx'> #define HNS_ROCE_CMQ_DESC_NUM_S		3</div><div class='del'>-#define HNS_ROCE_CMQ_EN_B		16</div><div class='del'>-#define HNS_ROCE_CMQ_ENABLE		BIT(HNS_ROCE_CMQ_EN_B)</div><div class='ctx'> </div><div class='ctx'> #define HNS_ROCE_CMQ_SCC_CLR_DONE_CNT		5</div><div class='ctx'> </div><div class='add'>+#define HNS_ROCE_CONG_SIZE 64</div><div class='add'>+</div><div class='ctx'> #define check_whether_last_step(hop_num, step_idx) \</div><div class='ctx'> 	((step_idx == 0 &amp;&amp; hop_num == HNS_ROCE_HOP_NUM_0) || \</div><div class='ctx'> 	(step_idx == 1 &amp;&amp; hop_num == 1) || \</div><div class='hunk'>@@ -141,12 +116,29 @@</div><div class='ctx'> #define CMD_CSQ_DESC_NUM		1024</div><div class='ctx'> #define CMD_CRQ_DESC_NUM		1024</div><div class='ctx'> </div><div class='add'>+/* Free mr used parameters */</div><div class='add'>+#define HNS_ROCE_FREE_MR_USED_CQE_NUM		128</div><div class='add'>+#define HNS_ROCE_FREE_MR_USED_QP_NUM		0x8</div><div class='add'>+#define HNS_ROCE_FREE_MR_USED_PSN		0x0808</div><div class='add'>+#define HNS_ROCE_FREE_MR_USED_QP_RETRY_CNT	0x7</div><div class='add'>+#define HNS_ROCE_FREE_MR_USED_QP_TIMEOUT	0x12</div><div class='add'>+#define HNS_ROCE_FREE_MR_USED_SQWQE_NUM		128</div><div class='add'>+#define HNS_ROCE_FREE_MR_USED_SQSGE_NUM		0x2</div><div class='add'>+#define HNS_ROCE_FREE_MR_USED_RQWQE_NUM		128</div><div class='add'>+#define HNS_ROCE_FREE_MR_USED_RQSGE_NUM		0x2</div><div class='add'>+#define HNS_ROCE_V2_FREE_MR_TIMEOUT		4500</div><div class='add'>+</div><div class='ctx'> enum {</div><div class='ctx'> 	NO_ARMED = 0x0,</div><div class='ctx'> 	REG_NXT_CEQE = 0x2,</div><div class='ctx'> 	REG_NXT_SE_CEQE = 0x3</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+enum {</div><div class='add'>+	CQE_SIZE_32B = 0x0,</div><div class='add'>+	CQE_SIZE_64B = 0x1</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> #define V2_CQ_DB_REQ_NOT_SOL			0</div><div class='ctx'> #define V2_CQ_DB_REQ_NOT			1</div><div class='ctx'> </div><div class='hunk'>@@ -155,8 +147,6 @@ enum {</div><div class='ctx'> </div><div class='ctx'> #define	GID_LEN_V2				16</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_V2_CQE_QPN_MASK		0x3ffff</div><div class='del'>-</div><div class='ctx'> enum {</div><div class='ctx'> 	HNS_ROCE_V2_WQE_OP_SEND				= 0x0,</div><div class='ctx'> 	HNS_ROCE_V2_WQE_OP_SEND_WITH_INV		= 0x1,</div><div class='hunk'>@@ -169,28 +159,11 @@ enum {</div><div class='ctx'> 	HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP	= 0x8,</div><div class='ctx'> 	HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD	= 0x9,</div><div class='ctx'> 	HNS_ROCE_V2_WQE_OP_FAST_REG_PMR			= 0xa,</div><div class='del'>-	HNS_ROCE_V2_WQE_OP_LOCAL_INV			= 0xb,</div><div class='del'>-	HNS_ROCE_V2_WQE_OP_BIND_MW_TYPE			= 0xc,</div><div class='add'>+	HNS_ROCE_V2_WQE_OP_BIND_MW			= 0xc,</div><div class='ctx'> 	HNS_ROCE_V2_WQE_OP_MASK				= 0x1f,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum {</div><div class='del'>-	HNS_ROCE_SQ_OPCODE_SEND = 0x0,</div><div class='del'>-	HNS_ROCE_SQ_OPCODE_SEND_WITH_INV = 0x1,</div><div class='del'>-	HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM = 0x2,</div><div class='del'>-	HNS_ROCE_SQ_OPCODE_RDMA_WRITE = 0x3,</div><div class='del'>-	HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM = 0x4,</div><div class='del'>-	HNS_ROCE_SQ_OPCODE_RDMA_READ = 0x5,</div><div class='del'>-	HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP = 0x6,</div><div class='del'>-	HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD = 0x7,</div><div class='del'>-	HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP = 0x8,</div><div class='del'>-	HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD = 0x9,</div><div class='del'>-	HNS_ROCE_SQ_OPCODE_FAST_REG_WR = 0xa,</div><div class='del'>-	HNS_ROCE_SQ_OPCODE_LOCAL_INV = 0xb,</div><div class='del'>-	HNS_ROCE_SQ_OPCODE_BIND_MW = 0xc,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum {</div><div class='ctx'> 	/* rq operations */</div><div class='ctx'> 	HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM = 0x0,</div><div class='ctx'> 	HNS_ROCE_V2_OPCODE_SEND = 0x1,</div><div class='hunk'>@@ -199,11 +172,11 @@ enum {</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum {</div><div class='del'>-	HNS_ROCE_V2_SQ_DB	= 0x0,</div><div class='del'>-	HNS_ROCE_V2_RQ_DB	= 0x1,</div><div class='del'>-	HNS_ROCE_V2_SRQ_DB	= 0x2,</div><div class='del'>-	HNS_ROCE_V2_CQ_DB_PTR	= 0x3,</div><div class='del'>-	HNS_ROCE_V2_CQ_DB_NTR	= 0x4,</div><div class='add'>+	HNS_ROCE_V2_SQ_DB,</div><div class='add'>+	HNS_ROCE_V2_RQ_DB,</div><div class='add'>+	HNS_ROCE_V2_SRQ_DB,</div><div class='add'>+	HNS_ROCE_V2_CQ_DB,</div><div class='add'>+	HNS_ROCE_V2_CQ_DB_NOTIFY</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum {</div><div class='hunk'>@@ -221,6 +194,7 @@ enum {</div><div class='ctx'> 	HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR		= 0x15,</div><div class='ctx'> 	HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR		= 0x16,</div><div class='ctx'> 	HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR		= 0x22,</div><div class='add'>+	HNS_ROCE_CQE_V2_GENERAL_ERR			= 0x23,</div><div class='ctx'> </div><div class='ctx'> 	HNS_ROCE_V2_CQE_STATUS_MASK			= 0xff,</div><div class='ctx'> };</div><div class='hunk'>@@ -231,20 +205,41 @@ enum hns_roce_opcode_type {</div><div class='ctx'> 	HNS_ROCE_OPC_QUERY_HW_VER			= 0x8000,</div><div class='ctx'> 	HNS_ROCE_OPC_CFG_GLOBAL_PARAM			= 0x8001,</div><div class='ctx'> 	HNS_ROCE_OPC_ALLOC_PF_RES			= 0x8004,</div><div class='add'>+	HNS_ROCE_OPC_QUERY_COUNTER			= 0x8206,</div><div class='ctx'> 	HNS_ROCE_OPC_QUERY_PF_RES			= 0x8400,</div><div class='ctx'> 	HNS_ROCE_OPC_ALLOC_VF_RES			= 0x8401,</div><div class='ctx'> 	HNS_ROCE_OPC_CFG_EXT_LLM			= 0x8403,</div><div class='del'>-	HNS_ROCE_OPC_CFG_TMOUT_LLM			= 0x8404,</div><div class='ctx'> 	HNS_ROCE_OPC_QUERY_PF_TIMER_RES			= 0x8406,</div><div class='add'>+	HNS_ROCE_OPC_QUERY_FUNC_INFO			= 0x8407,</div><div class='add'>+	HNS_ROCE_OPC_QUERY_PF_CAPS_NUM                  = 0x8408,</div><div class='add'>+	HNS_ROCE_OPC_CFG_ENTRY_SIZE			= 0x8409,</div><div class='add'>+	HNS_ROCE_OPC_QUERY_VF_CAPS_NUM			= 0x8410,</div><div class='ctx'> 	HNS_ROCE_OPC_CFG_SGID_TB			= 0x8500,</div><div class='ctx'> 	HNS_ROCE_OPC_CFG_SMAC_TB			= 0x8501,</div><div class='ctx'> 	HNS_ROCE_OPC_POST_MB				= 0x8504,</div><div class='ctx'> 	HNS_ROCE_OPC_QUERY_MB_ST			= 0x8505,</div><div class='ctx'> 	HNS_ROCE_OPC_CFG_BT_ATTR			= 0x8506,</div><div class='add'>+	HNS_ROCE_OPC_FUNC_CLEAR				= 0x8508,</div><div class='ctx'> 	HNS_ROCE_OPC_CLR_SCCC				= 0x8509,</div><div class='ctx'> 	HNS_ROCE_OPC_QUERY_SCCC				= 0x850a,</div><div class='ctx'> 	HNS_ROCE_OPC_RESET_SCCC				= 0x850b,</div><div class='add'>+	HNS_ROCE_OPC_CLEAR_EXTDB_LIST_INFO		= 0x850d,</div><div class='add'>+	HNS_ROCE_OPC_QUERY_VF_RES			= 0x850e,</div><div class='add'>+	HNS_ROCE_OPC_CFG_GMV_TBL			= 0x850f,</div><div class='add'>+	HNS_ROCE_OPC_CFG_GMV_BT				= 0x8510,</div><div class='add'>+	HNS_ROCE_QUERY_RAM_ECC				= 0x8513,</div><div class='ctx'> 	HNS_SWITCH_PARAMETER_CFG			= 0x1033,</div><div class='add'>+	HNS_ROCE_OPC_SET_BOND_INFO                      = 0x8601,</div><div class='add'>+	HNS_ROCE_OPC_CLEAR_BOND_INFO                    = 0x8602,</div><div class='add'>+	HNS_ROCE_OPC_CHANGE_ACTIVE_PORT                 = 0x8603,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define HNS_ROCE_OPC_POST_MB_TIMEOUT 35000</div><div class='add'>+#define HNS_ROCE_OPC_POST_MB_TRY_CNT 8</div><div class='add'>+#define HNS_ROCE_OPC_POST_MB_RETRY_GAP_MSEC 5</div><div class='add'>+struct hns_roce_cmdq_tx_timeout_map {</div><div class='add'>+	u16 opcode;</div><div class='add'>+	u32 tx_timeout;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum {</div><div class='hunk'>@@ -253,10 +248,25 @@ enum {</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum hns_roce_cmd_return_status {</div><div class='del'>-	CMD_EXEC_SUCCESS	= 0,</div><div class='del'>-	CMD_NO_AUTH		= 1,</div><div class='del'>-	CMD_NOT_EXEC		= 2,</div><div class='del'>-	CMD_QUEUE_FULL		= 3,</div><div class='add'>+	CMD_EXEC_SUCCESS,</div><div class='add'>+	CMD_NO_AUTH,</div><div class='add'>+	CMD_NOT_EXIST,</div><div class='add'>+	CMD_CRQ_FULL,</div><div class='add'>+	CMD_NEXT_ERR,</div><div class='add'>+	CMD_NOT_EXEC,</div><div class='add'>+	CMD_PARA_ERR,</div><div class='add'>+	CMD_RESULT_ERR,</div><div class='add'>+	CMD_TIMEOUT,</div><div class='add'>+	CMD_HILINK_ERR,</div><div class='add'>+	CMD_INFO_ILLEGAL,</div><div class='add'>+	CMD_INVALID,</div><div class='add'>+	CMD_ROH_CHECK_FAIL,</div><div class='add'>+	CMD_OTHER_ERR = 0xff</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct hns_roce_cmd_errcode {</div><div class='add'>+	enum hns_roce_cmd_return_status return_status;</div><div class='add'>+	int errno;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum hns_roce_sgid_type {</div><div class='hunk'>@@ -266,181 +276,114 @@ enum hns_roce_sgid_type {</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_v2_cq_context {</div><div class='del'>-	__le32	byte_4_pg_ceqn;</div><div class='del'>-	__le32	byte_8_cqn;</div><div class='del'>-	__le32	cqe_cur_blk_addr;</div><div class='del'>-	__le32	byte_16_hop_addr;</div><div class='del'>-	__le32	cqe_nxt_blk_addr;</div><div class='del'>-	__le32	byte_24_pgsz_addr;</div><div class='del'>-	__le32	byte_28_cq_pi;</div><div class='del'>-	__le32	byte_32_cq_ci;</div><div class='del'>-	__le32	cqe_ba;</div><div class='del'>-	__le32	byte_40_cqe_ba;</div><div class='del'>-	__le32	byte_44_db_record;</div><div class='del'>-	__le32	db_record_addr;</div><div class='del'>-	__le32	byte_52_cqe_cnt;</div><div class='del'>-	__le32	byte_56_cqe_period_maxcnt;</div><div class='del'>-	__le32	cqe_report_timer;</div><div class='del'>-	__le32	byte_64_se_cqe_idx;</div><div class='add'>+	__le32 byte_4_pg_ceqn;</div><div class='add'>+	__le32 byte_8_cqn;</div><div class='add'>+	__le32 cqe_cur_blk_addr;</div><div class='add'>+	__le32 byte_16_hop_addr;</div><div class='add'>+	__le32 cqe_nxt_blk_addr;</div><div class='add'>+	__le32 byte_24_pgsz_addr;</div><div class='add'>+	__le32 byte_28_cq_pi;</div><div class='add'>+	__le32 byte_32_cq_ci;</div><div class='add'>+	__le32 cqe_ba;</div><div class='add'>+	__le32 byte_40_cqe_ba;</div><div class='add'>+	__le32 byte_44_db_record;</div><div class='add'>+	__le32 db_record_addr;</div><div class='add'>+	__le32 byte_52_cqe_cnt;</div><div class='add'>+	__le32 byte_56_cqe_period_maxcnt;</div><div class='add'>+	__le32 cqe_report_timer;</div><div class='add'>+	__le32 byte_64_se_cqe_idx;</div><div class='ctx'> };</div><div class='del'>-#define HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM 0x0</div><div class='del'>-#define HNS_ROCE_V2_CQ_DEFAULT_INTERVAL	0x0</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_4_CQ_ST_S 0</div><div class='del'>-#define V2_CQC_BYTE_4_CQ_ST_M GENMASK(1, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_4_POLL_S 2</div><div class='ctx'> </div><div class='del'>-#define	V2_CQC_BYTE_4_SE_S 3</div><div class='add'>+#define CQC_CQE_BA_L_S 3</div><div class='add'>+#define CQC_CQE_BA_H_S (32 + CQC_CQE_BA_L_S)</div><div class='add'>+#define CQC_CQE_DB_RECORD_ADDR_H_S 32</div><div class='ctx'> </div><div class='del'>-#define	V2_CQC_BYTE_4_OVER_IGNORE_S 4</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_4_COALESCE_S 5</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_4_ARM_ST_S 6</div><div class='del'>-#define V2_CQC_BYTE_4_ARM_ST_M GENMASK(7, 6)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_4_SHIFT_S 8</div><div class='del'>-#define V2_CQC_BYTE_4_SHIFT_M GENMASK(12, 8)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_4_CMD_SN_S 13</div><div class='del'>-#define V2_CQC_BYTE_4_CMD_SN_M GENMASK(14, 13)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_4_CEQN_S 15</div><div class='del'>-#define V2_CQC_BYTE_4_CEQN_M GENMASK(23, 15)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_4_PAGE_OFFSET_S 24</div><div class='del'>-#define V2_CQC_BYTE_4_PAGE_OFFSET_M GENMASK(31, 24)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_8_CQN_S 0</div><div class='del'>-#define V2_CQC_BYTE_8_CQN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S 0</div><div class='del'>-#define V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M GENMASK(19, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_16_CQE_HOP_NUM_S 30</div><div class='del'>-#define V2_CQC_BYTE_16_CQE_HOP_NUM_M GENMASK(31, 30)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S 0</div><div class='del'>-#define V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M GENMASK(19, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_24_CQE_BA_PG_SZ_S 24</div><div class='del'>-#define V2_CQC_BYTE_24_CQE_BA_PG_SZ_M GENMASK(27, 24)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S 28</div><div class='del'>-#define V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M GENMASK(31, 28)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_28_CQ_PRODUCER_IDX_S 0</div><div class='del'>-#define V2_CQC_BYTE_28_CQ_PRODUCER_IDX_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_32_CQ_CONSUMER_IDX_S 0</div><div class='del'>-#define V2_CQC_BYTE_32_CQ_CONSUMER_IDX_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_40_CQE_BA_S 0</div><div class='del'>-#define V2_CQC_BYTE_40_CQE_BA_M GENMASK(28, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_44_DB_RECORD_EN_S 0</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_44_DB_RECORD_ADDR_S 1</div><div class='del'>-#define	V2_CQC_BYTE_44_DB_RECORD_ADDR_M GENMASK(31, 1)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_52_CQE_CNT_S 0</div><div class='del'>-#define	V2_CQC_BYTE_52_CQE_CNT_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_56_CQ_MAX_CNT_S 0</div><div class='del'>-#define V2_CQC_BYTE_56_CQ_MAX_CNT_M GENMASK(15, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_CQC_BYTE_56_CQ_PERIOD_S 16</div><div class='del'>-#define V2_CQC_BYTE_56_CQ_PERIOD_M GENMASK(31, 16)</div><div class='add'>+#define HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM 0x0</div><div class='add'>+#define HNS_ROCE_V2_CQ_DEFAULT_INTERVAL	0x0</div><div class='ctx'> </div><div class='del'>-#define	V2_CQC_BYTE_64_SE_CQE_IDX_S 0</div><div class='del'>-#define	V2_CQC_BYTE_64_SE_CQE_IDX_M GENMASK(23, 0)</div><div class='add'>+#define CQC_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_cq_context, h, l)</div><div class='add'>+</div><div class='add'>+#define CQC_CQ_ST CQC_FIELD_LOC(1, 0)</div><div class='add'>+#define CQC_POLL CQC_FIELD_LOC(2, 2)</div><div class='add'>+#define CQC_SE CQC_FIELD_LOC(3, 3)</div><div class='add'>+#define CQC_OVER_IGNORE CQC_FIELD_LOC(4, 4)</div><div class='add'>+#define CQC_ARM_ST CQC_FIELD_LOC(7, 6)</div><div class='add'>+#define CQC_SHIFT CQC_FIELD_LOC(12, 8)</div><div class='add'>+#define CQC_CMD_SN CQC_FIELD_LOC(14, 13)</div><div class='add'>+#define CQC_CEQN CQC_FIELD_LOC(23, 15)</div><div class='add'>+#define CQC_CQN CQC_FIELD_LOC(55, 32)</div><div class='add'>+#define CQC_POE_EN CQC_FIELD_LOC(56, 56)</div><div class='add'>+#define CQC_POE_NUM CQC_FIELD_LOC(58, 57)</div><div class='add'>+#define CQC_CQE_SIZE CQC_FIELD_LOC(60, 59)</div><div class='add'>+#define CQC_CQ_CNT_MODE CQC_FIELD_LOC(61, 61)</div><div class='add'>+#define CQC_STASH CQC_FIELD_LOC(63, 63)</div><div class='add'>+#define CQC_CQE_CUR_BLK_ADDR_L CQC_FIELD_LOC(95, 64)</div><div class='add'>+#define CQC_CQE_CUR_BLK_ADDR_H CQC_FIELD_LOC(115, 96)</div><div class='add'>+#define CQC_POE_QID CQC_FIELD_LOC(125, 116)</div><div class='add'>+#define CQC_CQE_HOP_NUM CQC_FIELD_LOC(127, 126)</div><div class='add'>+#define CQC_CQE_NEX_BLK_ADDR_L CQC_FIELD_LOC(159, 128)</div><div class='add'>+#define CQC_CQE_NEX_BLK_ADDR_H CQC_FIELD_LOC(179, 160)</div><div class='add'>+#define CQC_CQE_BAR_PG_SZ CQC_FIELD_LOC(187, 184)</div><div class='add'>+#define CQC_CQE_BUF_PG_SZ CQC_FIELD_LOC(191, 188)</div><div class='add'>+#define CQC_CQ_PRODUCER_IDX CQC_FIELD_LOC(215, 192)</div><div class='add'>+#define CQC_CQ_CONSUMER_IDX CQC_FIELD_LOC(247, 224)</div><div class='add'>+#define CQC_CQE_BA_L CQC_FIELD_LOC(287, 256)</div><div class='add'>+#define CQC_CQE_BA_H CQC_FIELD_LOC(316, 288)</div><div class='add'>+#define CQC_POE_QID_H_0 CQC_FIELD_LOC(319, 317)</div><div class='add'>+#define CQC_DB_RECORD_EN CQC_FIELD_LOC(320, 320)</div><div class='add'>+#define CQC_CQE_DB_RECORD_ADDR_L CQC_FIELD_LOC(351, 321)</div><div class='add'>+#define CQC_CQE_DB_RECORD_ADDR_H CQC_FIELD_LOC(383, 352)</div><div class='add'>+#define CQC_CQE_CNT CQC_FIELD_LOC(407, 384)</div><div class='add'>+#define CQC_CQ_MAX_CNT CQC_FIELD_LOC(431, 416)</div><div class='add'>+#define CQC_CQ_PERIOD CQC_FIELD_LOC(447, 432)</div><div class='add'>+#define CQC_CQE_REPORT_TIMER CQC_FIELD_LOC(471, 448)</div><div class='add'>+#define CQC_WR_CQE_IDX CQC_FIELD_LOC(479, 472)</div><div class='add'>+#define CQC_SE_CQE_IDX CQC_FIELD_LOC(503, 480)</div><div class='add'>+#define CQC_POE_QID_H_1 CQC_FIELD_LOC(511, 511)</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_srq_context {</div><div class='del'>-	__le32	byte_4_srqn_srqst;</div><div class='del'>-	__le32	byte_8_limit_wl;</div><div class='del'>-	__le32	byte_12_xrcd;</div><div class='del'>-	__le32	byte_16_pi_ci;</div><div class='del'>-	__le32	wqe_bt_ba;</div><div class='del'>-	__le32	byte_24_wqe_bt_ba;</div><div class='del'>-	__le32	byte_28_rqws_pd;</div><div class='del'>-	__le32	idx_bt_ba;</div><div class='del'>-	__le32	rsv_idx_bt_ba;</div><div class='del'>-	__le32	idx_cur_blk_addr;</div><div class='del'>-	__le32	byte_44_idxbufpgsz_addr;</div><div class='del'>-	__le32	idx_nxt_blk_addr;</div><div class='del'>-	__le32	rsv_idxnxtblkaddr;</div><div class='del'>-	__le32	byte_56_xrc_cqn;</div><div class='del'>-	__le32	db_record_addr_record_en;</div><div class='del'>-	__le32	db_record_addr;</div><div class='add'>+	__le32 data[16];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define SRQC_BYTE_4_SRQ_ST_S 0</div><div class='del'>-#define SRQC_BYTE_4_SRQ_ST_M GENMASK(1, 0)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S 2</div><div class='del'>-#define SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M GENMASK(3, 2)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_4_SRQ_SHIFT_S 4</div><div class='del'>-#define SRQC_BYTE_4_SRQ_SHIFT_M GENMASK(7, 4)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_4_SRQN_S 8</div><div class='del'>-#define SRQC_BYTE_4_SRQN_M GENMASK(31, 8)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_8_SRQ_LIMIT_WL_S 0</div><div class='del'>-#define SRQC_BYTE_8_SRQ_LIMIT_WL_M GENMASK(15, 0)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_12_SRQ_XRCD_S 0</div><div class='del'>-#define SRQC_BYTE_12_SRQ_XRCD_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_16_SRQ_PRODUCER_IDX_S 0</div><div class='del'>-#define SRQC_BYTE_16_SRQ_PRODUCER_IDX_M GENMASK(15, 0)</div><div class='add'>+#define SRQC_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_srq_context, h, l)</div><div class='add'>+</div><div class='add'>+#define SRQC_SRQ_ST SRQC_FIELD_LOC(1, 0)</div><div class='add'>+#define SRQC_WQE_HOP_NUM SRQC_FIELD_LOC(3, 2)</div><div class='add'>+#define SRQC_SHIFT SRQC_FIELD_LOC(7, 4)</div><div class='add'>+#define SRQC_SRQN SRQC_FIELD_LOC(31, 8)</div><div class='add'>+#define SRQC_LIMIT_WL SRQC_FIELD_LOC(47, 32)</div><div class='add'>+#define SRQC_RSV0 SRQC_FIELD_LOC(63, 48)</div><div class='add'>+#define SRQC_XRCD SRQC_FIELD_LOC(87, 64)</div><div class='add'>+#define SRQC_RSV1 SRQC_FIELD_LOC(95, 88)</div><div class='add'>+#define SRQC_PRODUCER_IDX SRQC_FIELD_LOC(111, 96)</div><div class='add'>+#define SRQC_CONSUMER_IDX SRQC_FIELD_LOC(127, 112)</div><div class='add'>+#define SRQC_WQE_BT_BA_L SRQC_FIELD_LOC(159, 128)</div><div class='add'>+#define SRQC_WQE_BT_BA_H SRQC_FIELD_LOC(188, 160)</div><div class='add'>+#define SRQC_RSV2 SRQC_FIELD_LOC(190, 189)</div><div class='add'>+#define SRQC_SRQ_TYPE SRQC_FIELD_LOC(191, 191)</div><div class='add'>+#define SRQC_PD SRQC_FIELD_LOC(215, 192)</div><div class='add'>+#define SRQC_RQWS SRQC_FIELD_LOC(219, 216)</div><div class='add'>+#define SRQC_RSV3 SRQC_FIELD_LOC(223, 220)</div><div class='add'>+#define SRQC_IDX_BT_BA_L SRQC_FIELD_LOC(255, 224)</div><div class='add'>+#define SRQC_IDX_BT_BA_H SRQC_FIELD_LOC(284, 256)</div><div class='add'>+#define SRQC_RSV4 SRQC_FIELD_LOC(287, 285)</div><div class='add'>+#define SRQC_IDX_CUR_BLK_ADDR_L SRQC_FIELD_LOC(319, 288)</div><div class='add'>+#define SRQC_IDX_CUR_BLK_ADDR_H SRQC_FIELD_LOC(339, 320)</div><div class='add'>+#define SRQC_RSV5 SRQC_FIELD_LOC(341, 340)</div><div class='add'>+#define SRQC_IDX_HOP_NUM SRQC_FIELD_LOC(343, 342)</div><div class='add'>+#define SRQC_IDX_BA_PG_SZ SRQC_FIELD_LOC(347, 344)</div><div class='add'>+#define SRQC_IDX_BUF_PG_SZ SRQC_FIELD_LOC(351, 348)</div><div class='add'>+#define SRQC_IDX_NXT_BLK_ADDR_L SRQC_FIELD_LOC(383, 352)</div><div class='add'>+#define SRQC_IDX_NXT_BLK_ADDR_H SRQC_FIELD_LOC(403, 384)</div><div class='add'>+#define SRQC_RSV6 SRQC_FIELD_LOC(415, 404)</div><div class='add'>+#define SRQC_XRC_CQN SRQC_FIELD_LOC(439, 416)</div><div class='add'>+#define SRQC_WQE_BA_PG_SZ SRQC_FIELD_LOC(443, 440)</div><div class='add'>+#define SRQC_WQE_BUF_PG_SZ SRQC_FIELD_LOC(447, 444)</div><div class='add'>+#define SRQC_DB_RECORD_EN SRQC_FIELD_LOC(448, 448)</div><div class='add'>+#define SRQC_DB_RECORD_ADDR_L SRQC_FIELD_LOC(479, 449)</div><div class='add'>+#define SRQC_DB_RECORD_ADDR_H SRQC_FIELD_LOC(511, 480)</div><div class='ctx'> </div><div class='del'>-#define SRQC_BYTE_16_SRQ_CONSUMER_IDX_S 0</div><div class='del'>-#define SRQC_BYTE_16_SRQ_CONSUMER_IDX_M GENMASK(31, 16)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_24_SRQ_WQE_BT_BA_S 0</div><div class='del'>-#define SRQC_BYTE_24_SRQ_WQE_BT_BA_M GENMASK(28, 0)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_28_PD_S 0</div><div class='del'>-#define SRQC_BYTE_28_PD_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_28_RQWS_S 24</div><div class='del'>-#define SRQC_BYTE_28_RQWS_M GENMASK(27, 24)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_36_SRQ_IDX_BT_BA_S 0</div><div class='del'>-#define SRQC_BYTE_36_SRQ_IDX_BT_BA_M GENMASK(28, 0)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S 0</div><div class='del'>-#define SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M GENMASK(19, 0)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S 22</div><div class='del'>-#define SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M GENMASK(23, 22)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S 24</div><div class='del'>-#define SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M GENMASK(27, 24)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S 28</div><div class='del'>-#define SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M GENMASK(31, 28)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S 0</div><div class='del'>-#define SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M GENMASK(19, 0)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_56_SRQ_XRC_CQN_S 0</div><div class='del'>-#define SRQC_BYTE_56_SRQ_XRC_CQN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S 24</div><div class='del'>-#define SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M GENMASK(27, 24)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S 28</div><div class='del'>-#define SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M GENMASK(31, 28)</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_60_SRQ_RECORD_EN_S 0</div><div class='del'>-</div><div class='del'>-#define SRQC_BYTE_60_SRQ_DB_RECORD_ADDR_S 1</div><div class='del'>-#define SRQC_BYTE_60_SRQ_DB_RECORD_ADDR_M GENMASK(31, 1)</div><div class='del'>-</div><div class='del'>-enum{</div><div class='add'>+enum {</div><div class='ctx'> 	V2_MPT_ST_VALID = 0x1,</div><div class='ctx'> 	V2_MPT_ST_FREE	= 0x2,</div><div class='ctx'> };</div><div class='hunk'>@@ -450,441 +393,296 @@ enum hns_roce_v2_qp_state {</div><div class='ctx'> 	HNS_ROCE_QP_ST_INIT,</div><div class='ctx'> 	HNS_ROCE_QP_ST_RTR,</div><div class='ctx'> 	HNS_ROCE_QP_ST_RTS,</div><div class='del'>-	HNS_ROCE_QP_ST_SQER,</div><div class='ctx'> 	HNS_ROCE_QP_ST_SQD,</div><div class='add'>+	HNS_ROCE_QP_ST_SQER,</div><div class='ctx'> 	HNS_ROCE_QP_ST_ERR,</div><div class='ctx'> 	HNS_ROCE_QP_ST_SQ_DRAINING,</div><div class='ctx'> 	HNS_ROCE_QP_NUM_ST</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct hns_roce_v2_qp_context {</div><div class='del'>-	__le32	byte_4_sqpn_tst;</div><div class='del'>-	__le32	wqe_sge_ba;</div><div class='del'>-	__le32	byte_12_sq_hop;</div><div class='del'>-	__le32	byte_16_buf_ba_pg_sz;</div><div class='del'>-	__le32	byte_20_smac_sgid_idx;</div><div class='del'>-	__le32	byte_24_mtu_tc;</div><div class='del'>-	__le32	byte_28_at_fl;</div><div class='del'>-	u8	dgid[GID_LEN_V2];</div><div class='del'>-	__le32	dmac;</div><div class='del'>-	__le32	byte_52_udpspn_dmac;</div><div class='del'>-	__le32	byte_56_dqpn_err;</div><div class='del'>-	__le32	byte_60_qpst_tempid;</div><div class='del'>-	__le32	qkey_xrcd;</div><div class='del'>-	__le32	byte_68_rq_db;</div><div class='del'>-	__le32	rq_db_record_addr;</div><div class='del'>-	__le32	byte_76_srqn_op_en;</div><div class='del'>-	__le32	byte_80_rnr_rx_cqn;</div><div class='del'>-	__le32	byte_84_rq_ci_pi;</div><div class='del'>-	__le32	rq_cur_blk_addr;</div><div class='del'>-	__le32	byte_92_srq_info;</div><div class='del'>-	__le32	byte_96_rx_reqmsn;</div><div class='del'>-	__le32	rq_nxt_blk_addr;</div><div class='del'>-	__le32	byte_104_rq_sge;</div><div class='del'>-	__le32	byte_108_rx_reqepsn;</div><div class='del'>-	__le32	rq_rnr_timer;</div><div class='del'>-	__le32	rx_msg_len;</div><div class='del'>-	__le32	rx_rkey_pkt_info;</div><div class='del'>-	__le64	rx_va;</div><div class='del'>-	__le32	byte_132_trrl;</div><div class='del'>-	__le32	trrl_ba;</div><div class='del'>-	__le32	byte_140_raq;</div><div class='del'>-	__le32	byte_144_raq;</div><div class='del'>-	__le32	byte_148_raq;</div><div class='del'>-	__le32	byte_152_raq;</div><div class='del'>-	__le32	byte_156_raq;</div><div class='del'>-	__le32	byte_160_sq_ci_pi;</div><div class='del'>-	__le32	sq_cur_blk_addr;</div><div class='del'>-	__le32	byte_168_irrl_idx;</div><div class='del'>-	__le32	byte_172_sq_psn;</div><div class='del'>-	__le32	byte_176_msg_pktn;</div><div class='del'>-	__le32	sq_cur_sge_blk_addr;</div><div class='del'>-	__le32	byte_184_irrl_idx;</div><div class='del'>-	__le32	cur_sge_offset;</div><div class='del'>-	__le32	byte_192_ext_sge;</div><div class='del'>-	__le32	byte_196_sq_psn;</div><div class='del'>-	__le32	byte_200_sq_max;</div><div class='del'>-	__le32	irrl_ba;</div><div class='del'>-	__le32	byte_208_irrl;</div><div class='del'>-	__le32	byte_212_lsn;</div><div class='del'>-	__le32	sq_timer;</div><div class='del'>-	__le32	byte_220_retry_psn_msn;</div><div class='del'>-	__le32	byte_224_retry_msg;</div><div class='del'>-	__le32	rx_sq_cur_blk_addr;</div><div class='del'>-	__le32	byte_232_irrl_sge;</div><div class='del'>-	__le32	irrl_cur_sge_offset;</div><div class='del'>-	__le32	byte_240_irrl_tail;</div><div class='del'>-	__le32	byte_244_rnr_rxack;</div><div class='del'>-	__le32	byte_248_ack_psn;</div><div class='del'>-	__le32	byte_252_err_txcqn;</div><div class='del'>-	__le32	byte_256_sqflush_rqcqe;</div><div class='add'>+struct hns_roce_v2_qp_context_ex {</div><div class='add'>+	__le32 data[64];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define	V2_QPC_BYTE_4_TST_S 0</div><div class='del'>-#define V2_QPC_BYTE_4_TST_M GENMASK(2, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_4_SGE_SHIFT_S 3</div><div class='del'>-#define V2_QPC_BYTE_4_SGE_SHIFT_M GENMASK(7, 3)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_4_SQPN_S 8</div><div class='del'>-#define V2_QPC_BYTE_4_SQPN_M  GENMASK(31, 8)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_12_WQE_SGE_BA_S 0</div><div class='del'>-#define V2_QPC_BYTE_12_WQE_SGE_BA_M GENMASK(28, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_12_SQ_HOP_NUM_S 29</div><div class='del'>-#define V2_QPC_BYTE_12_SQ_HOP_NUM_M GENMASK(30, 29)</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_12_RSVD_LKEY_EN_S 31</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S 0</div><div class='del'>-#define V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M GENMASK(3, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S 4</div><div class='del'>-#define V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M GENMASK(7, 4)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_16_PD_S 8</div><div class='del'>-#define V2_QPC_BYTE_16_PD_M GENMASK(31, 8)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_20_RQ_HOP_NUM_S 0</div><div class='del'>-#define V2_QPC_BYTE_20_RQ_HOP_NUM_M GENMASK(1, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_20_SGE_HOP_NUM_S 2</div><div class='del'>-#define V2_QPC_BYTE_20_SGE_HOP_NUM_M GENMASK(3, 2)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_20_RQWS_S 4</div><div class='del'>-#define V2_QPC_BYTE_20_RQWS_M GENMASK(7, 4)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_20_SQ_SHIFT_S 8</div><div class='del'>-#define V2_QPC_BYTE_20_SQ_SHIFT_M GENMASK(11, 8)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_20_RQ_SHIFT_S 12</div><div class='del'>-#define V2_QPC_BYTE_20_RQ_SHIFT_M GENMASK(15, 12)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_20_SGID_IDX_S 16</div><div class='del'>-#define V2_QPC_BYTE_20_SGID_IDX_M GENMASK(23, 16)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_20_SMAC_IDX_S 24</div><div class='del'>-#define V2_QPC_BYTE_20_SMAC_IDX_M GENMASK(31, 24)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_24_HOP_LIMIT_S 0</div><div class='del'>-#define V2_QPC_BYTE_24_HOP_LIMIT_M GENMASK(7, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_24_TC_S 8</div><div class='del'>-#define V2_QPC_BYTE_24_TC_M GENMASK(15, 8)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_24_VLAN_ID_S 16</div><div class='del'>-#define V2_QPC_BYTE_24_VLAN_ID_M GENMASK(27, 16)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_24_MTU_S 28</div><div class='del'>-#define V2_QPC_BYTE_24_MTU_M GENMASK(31, 28)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_28_FL_S 0</div><div class='del'>-#define V2_QPC_BYTE_28_FL_M GENMASK(19, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_28_SL_S 20</div><div class='del'>-#define V2_QPC_BYTE_28_SL_M GENMASK(23, 20)</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_28_CNP_TX_FLAG_S 24</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_28_CE_FLAG_S 25</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_28_LBI_S 26</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_28_AT_S 27</div><div class='del'>-#define V2_QPC_BYTE_28_AT_M GENMASK(31, 27)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_52_DMAC_S 0</div><div class='del'>-#define V2_QPC_BYTE_52_DMAC_M GENMASK(15, 0)</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_52_UDPSPN_S 16</div><div class='del'>-#define V2_QPC_BYTE_52_UDPSPN_M GENMASK(31, 16)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_56_DQPN_S 0</div><div class='del'>-#define V2_QPC_BYTE_56_DQPN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_56_SQ_TX_ERR_S 24</div><div class='del'>-#define	V2_QPC_BYTE_56_SQ_RX_ERR_S 25</div><div class='del'>-#define	V2_QPC_BYTE_56_RQ_TX_ERR_S 26</div><div class='del'>-#define	V2_QPC_BYTE_56_RQ_RX_ERR_S 27</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_56_LP_PKTN_INI_S 28</div><div class='del'>-#define V2_QPC_BYTE_56_LP_PKTN_INI_M GENMASK(31, 28)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_60_TEMPID_S 0</div><div class='del'>-#define V2_QPC_BYTE_60_TEMPID_M GENMASK(7, 0)</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_60_SCC_TOKEN_S 8</div><div class='del'>-#define V2_QPC_BYTE_60_SCC_TOKEN_M GENMASK(26, 8)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_60_SQ_DB_DOING_S 27</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_60_RQ_DB_DOING_S 28</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_60_QP_ST_S 29</div><div class='del'>-#define V2_QPC_BYTE_60_QP_ST_M GENMASK(31, 29)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_68_RQ_RECORD_EN_S 0</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S 1</div><div class='del'>-#define V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M GENMASK(31, 1)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_76_SRQN_S 0</div><div class='del'>-#define V2_QPC_BYTE_76_SRQN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_76_SRQ_EN_S 24</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_76_RRE_S 25</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_76_RWE_S 26</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_76_ATE_S 27</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_76_RQIE_S 28</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_76_RQ_VLAN_EN_S 30</div><div class='del'>-#define	V2_QPC_BYTE_80_RX_CQN_S 0</div><div class='del'>-#define V2_QPC_BYTE_80_RX_CQN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_80_MIN_RNR_TIME_S 27</div><div class='del'>-#define V2_QPC_BYTE_80_MIN_RNR_TIME_M GENMASK(31, 27)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S 0</div><div class='del'>-#define V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M GENMASK(15, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S 16</div><div class='del'>-#define V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M GENMASK(31, 16)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S 0</div><div class='del'>-#define V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M GENMASK(19, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_92_SRQ_INFO_S 20</div><div class='del'>-#define V2_QPC_BYTE_92_SRQ_INFO_M GENMASK(31, 20)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_96_RX_REQ_MSN_S 0</div><div class='del'>-#define V2_QPC_BYTE_96_RX_REQ_MSN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S 0</div><div class='del'>-#define V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M GENMASK(19, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S 24</div><div class='del'>-#define V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M GENMASK(31, 24)</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_108_INV_CREDIT_S 0</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S 3</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S 4</div><div class='del'>-#define V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M GENMASK(6, 4)</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_108_RX_REQ_RNR_S 7</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_108_RX_REQ_EPSN_S 8</div><div class='del'>-#define V2_QPC_BYTE_108_RX_REQ_EPSN_M GENMASK(31, 8)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_132_TRRL_HEAD_MAX_S 0</div><div class='del'>-#define V2_QPC_BYTE_132_TRRL_HEAD_MAX_M GENMASK(7, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_132_TRRL_TAIL_MAX_S 8</div><div class='del'>-#define V2_QPC_BYTE_132_TRRL_TAIL_MAX_M GENMASK(15, 8)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_132_TRRL_BA_S 16</div><div class='del'>-#define V2_QPC_BYTE_132_TRRL_BA_M GENMASK(31, 16)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_140_TRRL_BA_S 0</div><div class='del'>-#define V2_QPC_BYTE_140_TRRL_BA_M GENMASK(11, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_140_RR_MAX_S 12</div><div class='del'>-#define V2_QPC_BYTE_140_RR_MAX_M GENMASK(14, 12)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_140_RQ_RTY_WAIT_DO_S 15</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S 16</div><div class='del'>-#define V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M GENMASK(23, 16)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S 24</div><div class='del'>-#define V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M GENMASK(31, 24)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S 0</div><div class='del'>-#define V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_144_RAQ_CREDIT_S 25</div><div class='del'>-#define V2_QPC_BYTE_144_RAQ_CREDIT_M GENMASK(29, 25)</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_144_RESP_RTY_FLG_S 31</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_148_RQ_MSN_S 0</div><div class='del'>-#define V2_QPC_BYTE_148_RQ_MSN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_148_RAQ_SYNDROME_S 24</div><div class='del'>-#define V2_QPC_BYTE_148_RAQ_SYNDROME_M GENMASK(31, 24)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_152_RAQ_PSN_S 0</div><div class='del'>-#define V2_QPC_BYTE_152_RAQ_PSN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S 24</div><div class='del'>-#define V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M GENMASK(31, 24)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_156_RAQ_USE_PKTN_S 0</div><div class='del'>-#define V2_QPC_BYTE_156_RAQ_USE_PKTN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S 0</div><div class='del'>-#define V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M GENMASK(15, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S 16</div><div class='del'>-#define V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M GENMASK(31, 16)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S 0</div><div class='del'>-#define V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M GENMASK(19, 0)</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S 20</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_168_SQ_INVLD_FLG_S 21</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_168_LP_SGEN_INI_S 22</div><div class='del'>-#define V2_QPC_BYTE_168_LP_SGEN_INI_M GENMASK(23, 22)</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_168_SQ_VLAN_EN_S 24</div><div class='del'>-#define V2_QPC_BYTE_168_POLL_DB_WAIT_DO_S 25</div><div class='del'>-#define V2_QPC_BYTE_168_SCC_TOKEN_FORBID_SQ_DEQ_S 26</div><div class='del'>-#define V2_QPC_BYTE_168_WAIT_ACK_TIMEOUT_S 27</div><div class='del'>-#define	V2_QPC_BYTE_168_IRRL_IDX_LSB_S 28</div><div class='del'>-#define V2_QPC_BYTE_168_IRRL_IDX_LSB_M GENMASK(31, 28)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_172_ACK_REQ_FREQ_S 0</div><div class='del'>-#define V2_QPC_BYTE_172_ACK_REQ_FREQ_M GENMASK(5, 0)</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_172_MSG_RNR_FLG_S 6</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_172_FRE_S 7</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_172_SQ_CUR_PSN_S 8</div><div class='del'>-#define V2_QPC_BYTE_172_SQ_CUR_PSN_M GENMASK(31, 8)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_176_MSG_USE_PKTN_S 0</div><div class='del'>-#define V2_QPC_BYTE_176_MSG_USE_PKTN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_176_IRRL_HEAD_PRE_S 24</div><div class='del'>-#define V2_QPC_BYTE_176_IRRL_HEAD_PRE_M GENMASK(31, 24)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S 0</div><div class='del'>-#define V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M GENMASK(19, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_184_IRRL_IDX_MSB_S 20</div><div class='del'>-#define V2_QPC_BYTE_184_IRRL_IDX_MSB_M GENMASK(31, 20)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_192_CUR_SGE_IDX_S 0</div><div class='del'>-#define V2_QPC_BYTE_192_CUR_SGE_IDX_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S 24</div><div class='del'>-#define V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M GENMASK(31, 24)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_196_IRRL_HEAD_S 0</div><div class='del'>-#define V2_QPC_BYTE_196_IRRL_HEAD_M GENMASK(7, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_196_SQ_MAX_PSN_S 8</div><div class='del'>-#define V2_QPC_BYTE_196_SQ_MAX_PSN_M GENMASK(31, 8)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_200_SQ_MAX_IDX_S 0</div><div class='del'>-#define V2_QPC_BYTE_200_SQ_MAX_IDX_M GENMASK(15, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_200_LCL_OPERATED_CNT_S 16</div><div class='del'>-#define V2_QPC_BYTE_200_LCL_OPERATED_CNT_M GENMASK(31, 16)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_208_IRRL_BA_S 0</div><div class='del'>-#define V2_QPC_BYTE_208_IRRL_BA_M GENMASK(25, 0)</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_208_PKT_RNR_FLG_S 26</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_208_PKT_RTY_FLG_S 27</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_208_RMT_E2E_S 28</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_208_SR_MAX_S 29</div><div class='del'>-#define V2_QPC_BYTE_208_SR_MAX_M GENMASK(31, 29)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_212_LSN_S 0</div><div class='del'>-#define V2_QPC_BYTE_212_LSN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_212_RETRY_NUM_INIT_S 24</div><div class='del'>-#define V2_QPC_BYTE_212_RETRY_NUM_INIT_M GENMASK(26, 24)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_212_CHECK_FLG_S 27</div><div class='del'>-#define V2_QPC_BYTE_212_CHECK_FLG_M GENMASK(28, 27)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_212_RETRY_CNT_S 29</div><div class='del'>-#define V2_QPC_BYTE_212_RETRY_CNT_M GENMASK(31, 29)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_220_RETRY_MSG_MSN_S 0</div><div class='del'>-#define V2_QPC_BYTE_220_RETRY_MSG_MSN_M GENMASK(15, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_220_RETRY_MSG_PSN_S 16</div><div class='del'>-#define V2_QPC_BYTE_220_RETRY_MSG_PSN_M GENMASK(31, 16)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_224_RETRY_MSG_PSN_S 0</div><div class='del'>-#define V2_QPC_BYTE_224_RETRY_MSG_PSN_M GENMASK(7, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S 8</div><div class='del'>-#define V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M GENMASK(31, 8)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S 0</div><div class='del'>-#define V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M GENMASK(19, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_232_IRRL_SGE_IDX_S 20</div><div class='del'>-#define V2_QPC_BYTE_232_IRRL_SGE_IDX_M GENMASK(28, 20)</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_232_SO_LP_VLD_S 29</div><div class='del'>-#define V2_QPC_BYTE_232_FENCE_LP_VLD_S 30</div><div class='del'>-#define V2_QPC_BYTE_232_IRRL_LP_VLD_S 31</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_240_IRRL_TAIL_REAL_S 0</div><div class='del'>-#define V2_QPC_BYTE_240_IRRL_TAIL_REAL_M GENMASK(7, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_240_IRRL_TAIL_RD_S 8</div><div class='del'>-#define V2_QPC_BYTE_240_IRRL_TAIL_RD_M GENMASK(15, 8)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_240_RX_ACK_MSN_S 16</div><div class='del'>-#define V2_QPC_BYTE_240_RX_ACK_MSN_M GENMASK(31, 16)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_244_RX_ACK_EPSN_S 0</div><div class='del'>-#define V2_QPC_BYTE_244_RX_ACK_EPSN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_244_RNR_NUM_INIT_S 24</div><div class='del'>-#define V2_QPC_BYTE_244_RNR_NUM_INIT_M GENMASK(26, 24)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_244_RNR_CNT_S 27</div><div class='del'>-#define V2_QPC_BYTE_244_RNR_CNT_M GENMASK(29, 27)</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_244_LCL_OP_FLG_S 30</div><div class='del'>-#define V2_QPC_BYTE_244_IRRL_RD_FLG_S 31</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_248_IRRL_PSN_S 0</div><div class='del'>-#define V2_QPC_BYTE_248_IRRL_PSN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_248_ACK_PSN_ERR_S 24</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S 25</div><div class='del'>-#define V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M GENMASK(26, 25)</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_248_IRRL_PSN_VLD_S 27</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_248_RNR_RETRY_FLAG_S 28</div><div class='del'>-</div><div class='del'>-#define V2_QPC_BYTE_248_CQ_ERR_IND_S 31</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_252_TX_CQN_S 0</div><div class='del'>-#define V2_QPC_BYTE_252_TX_CQN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_252_SIG_TYPE_S 24</div><div class='del'>-</div><div class='del'>-#define	V2_QPC_BYTE_252_ERR_TYPE_S 25</div><div class='del'>-#define V2_QPC_BYTE_252_ERR_TYPE_M GENMASK(31, 25)</div><div class='add'>+struct hns_roce_v2_qp_context {</div><div class='add'>+	__le32 byte_4_sqpn_tst;</div><div class='add'>+	__le32 wqe_sge_ba;</div><div class='add'>+	__le32 byte_12_sq_hop;</div><div class='add'>+	__le32 byte_16_buf_ba_pg_sz;</div><div class='add'>+	__le32 byte_20_smac_sgid_idx;</div><div class='add'>+	__le32 byte_24_mtu_tc;</div><div class='add'>+	__le32 byte_28_at_fl;</div><div class='add'>+	u8 dgid[GID_LEN_V2];</div><div class='add'>+	__le32 dmac;</div><div class='add'>+	__le32 byte_52_udpspn_dmac;</div><div class='add'>+	__le32 byte_56_dqpn_err;</div><div class='add'>+	__le32 byte_60_qpst_tempid;</div><div class='add'>+	__le32 qkey_xrcd;</div><div class='add'>+	__le32 byte_68_rq_db;</div><div class='add'>+	__le32 rq_db_record_addr;</div><div class='add'>+	__le32 byte_76_srqn_op_en;</div><div class='add'>+	__le32 byte_80_rnr_rx_cqn;</div><div class='add'>+	__le32 byte_84_rq_ci_pi;</div><div class='add'>+	__le32 rq_cur_blk_addr;</div><div class='add'>+	__le32 byte_92_srq_info;</div><div class='add'>+	__le32 byte_96_rx_reqmsn;</div><div class='add'>+	__le32 rq_nxt_blk_addr;</div><div class='add'>+	__le32 byte_104_rq_sge;</div><div class='add'>+	__le32 byte_108_rx_reqepsn;</div><div class='add'>+	__le32 rq_rnr_timer;</div><div class='add'>+	__le32 rx_msg_len;</div><div class='add'>+	__le32 rx_rkey_pkt_info;</div><div class='add'>+	__le64 rx_va;</div><div class='add'>+	__le32 byte_132_trrl;</div><div class='add'>+	__le32 trrl_ba;</div><div class='add'>+	__le32 byte_140_raq;</div><div class='add'>+	__le32 byte_144_raq;</div><div class='add'>+	__le32 byte_148_raq;</div><div class='add'>+	__le32 byte_152_raq;</div><div class='add'>+	__le32 byte_156_raq;</div><div class='add'>+	__le32 byte_160_sq_ci_pi;</div><div class='add'>+	__le32 sq_cur_blk_addr;</div><div class='add'>+	__le32 byte_168_irrl_idx;</div><div class='add'>+	__le32 byte_172_sq_psn;</div><div class='add'>+	__le32 byte_176_msg_pktn;</div><div class='add'>+	__le32 sq_cur_sge_blk_addr;</div><div class='add'>+	__le32 byte_184_irrl_idx;</div><div class='add'>+	__le32 cur_sge_offset;</div><div class='add'>+	__le32 byte_192_ext_sge;</div><div class='add'>+	__le32 byte_196_sq_psn;</div><div class='add'>+	__le32 byte_200_sq_max;</div><div class='add'>+	__le32 irrl_ba;</div><div class='add'>+	__le32 byte_208_irrl;</div><div class='add'>+	__le32 byte_212_lsn;</div><div class='add'>+	__le32 sq_timer;</div><div class='add'>+	__le32 byte_220_retry_psn_msn;</div><div class='add'>+	__le32 byte_224_retry_msg;</div><div class='add'>+	__le32 rx_sq_cur_blk_addr;</div><div class='add'>+	__le32 byte_232_irrl_sge;</div><div class='add'>+	__le32 irrl_cur_sge_offset;</div><div class='add'>+	__le32 byte_240_irrl_tail;</div><div class='add'>+	__le32 byte_244_rnr_rxack;</div><div class='add'>+	__le32 byte_248_ack_psn;</div><div class='add'>+	__le32 byte_252_err_txcqn;</div><div class='add'>+	__le32 byte_256_sqflush_rqcqe;</div><div class='add'>+</div><div class='add'>+	struct hns_roce_v2_qp_context_ex ext;</div><div class='add'>+};</div><div class='ctx'> </div><div class='del'>-#define	V2_QPC_BYTE_256_RQ_CQE_IDX_S 0</div><div class='del'>-#define V2_QPC_BYTE_256_RQ_CQE_IDX_M GENMASK(15, 0)</div><div class='add'>+#define QPC_TRRL_BA_L_S 4</div><div class='add'>+#define QPC_TRRL_BA_M_S (16 + QPC_TRRL_BA_L_S)</div><div class='add'>+#define QPC_TRRL_BA_H_S (32 + QPC_TRRL_BA_M_S)</div><div class='add'>+#define QPC_IRRL_BA_L_S 6</div><div class='add'>+#define QPC_IRRL_BA_H_S (32 + QPC_IRRL_BA_L_S)</div><div class='add'>+</div><div class='add'>+#define QPC_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_qp_context, h, l)</div><div class='add'>+</div><div class='add'>+#define QPC_TST QPC_FIELD_LOC(2, 0)</div><div class='add'>+#define QPC_SGE_SHIFT QPC_FIELD_LOC(7, 3)</div><div class='add'>+#define QPC_CNP_TIMER QPC_FIELD_LOC(31, 8)</div><div class='add'>+#define QPC_WQE_SGE_BA_L QPC_FIELD_LOC(63, 32)</div><div class='add'>+#define QPC_WQE_SGE_BA_H QPC_FIELD_LOC(92, 64)</div><div class='add'>+#define QPC_SQ_HOP_NUM QPC_FIELD_LOC(94, 93)</div><div class='add'>+#define QPC_CIRE_EN QPC_FIELD_LOC(95, 95)</div><div class='add'>+#define QPC_WQE_SGE_BA_PG_SZ QPC_FIELD_LOC(99, 96)</div><div class='add'>+#define QPC_WQE_SGE_BUF_PG_SZ QPC_FIELD_LOC(103, 100)</div><div class='add'>+#define QPC_PD QPC_FIELD_LOC(127, 104)</div><div class='add'>+#define QPC_RQ_HOP_NUM QPC_FIELD_LOC(129, 128)</div><div class='add'>+#define QPC_SGE_HOP_NUM QPC_FIELD_LOC(131, 130)</div><div class='add'>+#define QPC_RQWS QPC_FIELD_LOC(135, 132)</div><div class='add'>+#define QPC_SQ_SHIFT QPC_FIELD_LOC(139, 136)</div><div class='add'>+#define QPC_RQ_SHIFT QPC_FIELD_LOC(143, 140)</div><div class='add'>+#define QPC_GMV_IDX QPC_FIELD_LOC(159, 144)</div><div class='add'>+#define QPC_HOPLIMIT QPC_FIELD_LOC(167, 160)</div><div class='add'>+#define QPC_TC QPC_FIELD_LOC(175, 168)</div><div class='add'>+#define QPC_VLAN_ID QPC_FIELD_LOC(187, 176)</div><div class='add'>+#define QPC_MTU QPC_FIELD_LOC(191, 188)</div><div class='add'>+#define QPC_FL QPC_FIELD_LOC(211, 192)</div><div class='add'>+#define QPC_SL QPC_FIELD_LOC(215, 212)</div><div class='add'>+#define QPC_CNP_TX_FLAG QPC_FIELD_LOC(216, 216)</div><div class='add'>+#define QPC_CE_FLAG QPC_FIELD_LOC(217, 217)</div><div class='add'>+#define QPC_LBI QPC_FIELD_LOC(218, 218)</div><div class='add'>+#define QPC_AT QPC_FIELD_LOC(223, 219)</div><div class='add'>+#define QPC_DGID QPC_FIELD_LOC(351, 224)</div><div class='add'>+#define QPC_DMAC_L QPC_FIELD_LOC(383, 352)</div><div class='add'>+#define QPC_DMAC_H QPC_FIELD_LOC(399, 384)</div><div class='add'>+#define QPC_UDPSPN QPC_FIELD_LOC(415, 400)</div><div class='add'>+#define QPC_DQPN QPC_FIELD_LOC(439, 416)</div><div class='add'>+#define QPC_SQ_TX_ERR QPC_FIELD_LOC(440, 440)</div><div class='add'>+#define QPC_SQ_RX_ERR QPC_FIELD_LOC(441, 441)</div><div class='add'>+#define QPC_RQ_TX_ERR QPC_FIELD_LOC(442, 442)</div><div class='add'>+#define QPC_RQ_RX_ERR QPC_FIELD_LOC(443, 443)</div><div class='add'>+#define QPC_LP_PKTN_INI QPC_FIELD_LOC(447, 444)</div><div class='add'>+#define QPC_CONG_ALGO_TMPL_ID QPC_FIELD_LOC(455, 448)</div><div class='add'>+#define QPC_SCC_TOKEN QPC_FIELD_LOC(474, 456)</div><div class='add'>+#define QPC_SQ_DB_DOING QPC_FIELD_LOC(475, 475)</div><div class='add'>+#define QPC_RQ_DB_DOING QPC_FIELD_LOC(476, 476)</div><div class='add'>+#define QPC_QP_ST QPC_FIELD_LOC(479, 477)</div><div class='add'>+#define QPC_QKEY_XRCD QPC_FIELD_LOC(511, 480)</div><div class='add'>+#define QPC_RQ_RECORD_EN QPC_FIELD_LOC(512, 512)</div><div class='add'>+#define QPC_RQ_DB_RECORD_ADDR_L QPC_FIELD_LOC(543, 513)</div><div class='add'>+#define QPC_RQ_DB_RECORD_ADDR_H QPC_FIELD_LOC(575, 544)</div><div class='add'>+#define QPC_SRQN QPC_FIELD_LOC(599, 576)</div><div class='add'>+#define QPC_SRQ_EN QPC_FIELD_LOC(600, 600)</div><div class='add'>+#define QPC_RRE QPC_FIELD_LOC(601, 601)</div><div class='add'>+#define QPC_RWE QPC_FIELD_LOC(602, 602)</div><div class='add'>+#define QPC_ATE QPC_FIELD_LOC(603, 603)</div><div class='add'>+#define QPC_RQIE QPC_FIELD_LOC(604, 604)</div><div class='add'>+#define QPC_EXT_ATE QPC_FIELD_LOC(605, 605)</div><div class='add'>+#define QPC_RQ_VLAN_EN QPC_FIELD_LOC(606, 606)</div><div class='add'>+#define QPC_RQ_RTY_TX_ERR QPC_FIELD_LOC(607, 607)</div><div class='add'>+#define QPC_RX_CQN QPC_FIELD_LOC(631, 608)</div><div class='add'>+#define QPC_XRC_QP_TYPE QPC_FIELD_LOC(632, 632)</div><div class='add'>+#define QPC_CQEIE QPC_FIELD_LOC(633, 633)</div><div class='add'>+#define QPC_CQEIS QPC_FIELD_LOC(634, 634)</div><div class='add'>+#define QPC_MIN_RNR_TIME QPC_FIELD_LOC(639, 635)</div><div class='add'>+#define QPC_RQ_PRODUCER_IDX QPC_FIELD_LOC(655, 640)</div><div class='add'>+#define QPC_RQ_CONSUMER_IDX QPC_FIELD_LOC(671, 656)</div><div class='add'>+#define QPC_RQ_CUR_BLK_ADDR_L QPC_FIELD_LOC(703, 672)</div><div class='add'>+#define QPC_RQ_CUR_BLK_ADDR_H QPC_FIELD_LOC(723, 704)</div><div class='add'>+#define QPC_SRQ_INFO QPC_FIELD_LOC(735, 724)</div><div class='add'>+#define QPC_RX_REQ_MSN QPC_FIELD_LOC(759, 736)</div><div class='add'>+#define QPC_REDUCE_CODE QPC_FIELD_LOC(766, 760)</div><div class='add'>+#define QPC_RX_XRC_PKT_CQE_FLG QPC_FIELD_LOC(767, 767)</div><div class='add'>+#define QPC_RQ_NXT_BLK_ADDR_L QPC_FIELD_LOC(799, 768)</div><div class='add'>+#define QPC_RQ_NXT_BLK_ADDR_H QPC_FIELD_LOC(819, 800)</div><div class='add'>+#define QPC_REDUCE_EN QPC_FIELD_LOC(820, 820)</div><div class='add'>+#define QPC_FLUSH_EN QPC_FIELD_LOC(821, 821)</div><div class='add'>+#define QPC_AW_EN QPC_FIELD_LOC(822, 822)</div><div class='add'>+#define QPC_WN_EN QPC_FIELD_LOC(823, 823)</div><div class='add'>+#define QPC_RQ_CUR_WQE_SGE_NUM QPC_FIELD_LOC(831, 824)</div><div class='add'>+#define QPC_INV_CREDIT QPC_FIELD_LOC(832, 832)</div><div class='add'>+#define QPC_LAST_WRITE_TYPE QPC_FIELD_LOC(834, 833)</div><div class='add'>+#define QPC_RX_REQ_PSN_ERR QPC_FIELD_LOC(835, 835)</div><div class='add'>+#define QPC_RX_REQ_LAST_OPTYPE QPC_FIELD_LOC(838, 836)</div><div class='add'>+#define QPC_RX_REQ_RNR QPC_FIELD_LOC(839, 839)</div><div class='add'>+#define QPC_RX_REQ_EPSN QPC_FIELD_LOC(863, 840)</div><div class='add'>+#define QPC_RQ_RNR_TIMER QPC_FIELD_LOC(895, 864)</div><div class='add'>+#define QPC_RX_MSG_LEN QPC_FIELD_LOC(927, 896)</div><div class='add'>+#define QPC_RX_RKEY_PKT_INFO QPC_FIELD_LOC(959, 928)</div><div class='add'>+#define QPC_RX_VA QPC_FIELD_LOC(1023, 960)</div><div class='add'>+#define QPC_TRRL_HEAD_MAX QPC_FIELD_LOC(1031, 1024)</div><div class='add'>+#define QPC_TRRL_TAIL_MAX QPC_FIELD_LOC(1039, 1032)</div><div class='add'>+#define QPC_TRRL_BA_L QPC_FIELD_LOC(1055, 1040)</div><div class='add'>+#define QPC_TRRL_BA_M QPC_FIELD_LOC(1087, 1056)</div><div class='add'>+#define QPC_TRRL_BA_H QPC_FIELD_LOC(1099, 1088)</div><div class='add'>+#define QPC_RR_MAX QPC_FIELD_LOC(1102, 1100)</div><div class='add'>+#define QPC_RQ_RTY_WAIT_DO QPC_FIELD_LOC(1103, 1103)</div><div class='add'>+#define QPC_RAQ_TRRL_HEAD QPC_FIELD_LOC(1111, 1104)</div><div class='add'>+#define QPC_RAQ_TRRL_TAIL QPC_FIELD_LOC(1119, 1112)</div><div class='add'>+#define QPC_RAQ_RTY_INI_PSN QPC_FIELD_LOC(1143, 1120)</div><div class='add'>+#define QPC_CIRE_SLV_RQ_EN QPC_FIELD_LOC(1144, 1144)</div><div class='add'>+#define QPC_RAQ_CREDIT QPC_FIELD_LOC(1149, 1145)</div><div class='add'>+#define QPC_RQ_DB_IN_EXT QPC_FIELD_LOC(1150, 1150)</div><div class='add'>+#define QPC_RESP_RTY_FLG QPC_FIELD_LOC(1151, 1151)</div><div class='add'>+#define QPC_RAQ_MSN QPC_FIELD_LOC(1175, 1152)</div><div class='add'>+#define QPC_RAQ_SYNDROME QPC_FIELD_LOC(1183, 1176)</div><div class='add'>+#define QPC_RAQ_PSN QPC_FIELD_LOC(1207, 1184)</div><div class='add'>+#define QPC_RAQ_TRRL_RTY_HEAD QPC_FIELD_LOC(1215, 1208)</div><div class='add'>+#define QPC_RAQ_USE_PKTN QPC_FIELD_LOC(1239, 1216)</div><div class='add'>+#define QPC_RQ_SCC_TOKEN QPC_FIELD_LOC(1245, 1240)</div><div class='add'>+#define QPC_RVD10 QPC_FIELD_LOC(1247, 1246)</div><div class='add'>+#define QPC_SQ_PRODUCER_IDX QPC_FIELD_LOC(1263, 1248)</div><div class='add'>+#define QPC_SQ_CONSUMER_IDX QPC_FIELD_LOC(1279, 1264)</div><div class='add'>+#define QPC_SQ_CUR_BLK_ADDR_L QPC_FIELD_LOC(1311, 1280)</div><div class='add'>+#define QPC_SQ_CUR_BLK_ADDR_H QPC_FIELD_LOC(1331, 1312)</div><div class='add'>+#define QPC_MSG_RTY_LP_FLG QPC_FIELD_LOC(1332, 1332)</div><div class='add'>+#define QPC_SQ_INVLD_FLG QPC_FIELD_LOC(1333, 1333)</div><div class='add'>+#define QPC_LP_SGEN_INI QPC_FIELD_LOC(1335, 1334)</div><div class='add'>+#define QPC_SQ_VLAN_EN QPC_FIELD_LOC(1336, 1336)</div><div class='add'>+#define QPC_POLL_DB_WAIT_DO QPC_FIELD_LOC(1337, 1337)</div><div class='add'>+#define QPC_SCC_TOKEN_FORBID_SQ_DEQ QPC_FIELD_LOC(1338, 1338)</div><div class='add'>+#define QPC_WAIT_ACK_TIMEOUT QPC_FIELD_LOC(1339, 1339)</div><div class='add'>+#define QPC_IRRL_IDX_LSB QPC_FIELD_LOC(1343, 1340)</div><div class='add'>+#define QPC_ACK_REQ_FREQ QPC_FIELD_LOC(1349, 1344)</div><div class='add'>+#define QPC_MSG_RNR_FLG QPC_FIELD_LOC(1350, 1350)</div><div class='add'>+#define QPC_FRE QPC_FIELD_LOC(1351, 1351)</div><div class='add'>+#define QPC_SQ_CUR_PSN QPC_FIELD_LOC(1375, 1352)</div><div class='add'>+#define QPC_MSG_USE_PKTN QPC_FIELD_LOC(1399, 1376)</div><div class='add'>+#define QPC_IRRL_HEAD_PRE QPC_FIELD_LOC(1407, 1400)</div><div class='add'>+#define QPC_SQ_CUR_SGE_BLK_ADDR_L QPC_FIELD_LOC(1439, 1408)</div><div class='add'>+#define QPC_SQ_CUR_SGE_BLK_ADDR_H QPC_FIELD_LOC(1459, 1440)</div><div class='add'>+#define QPC_IRRL_IDX_MSB QPC_FIELD_LOC(1471, 1460)</div><div class='add'>+#define QPC_CUR_SGE_OFFSET QPC_FIELD_LOC(1503, 1472)</div><div class='add'>+#define QPC_CUR_SGE_IDX QPC_FIELD_LOC(1527, 1504)</div><div class='add'>+#define QPC_EXT_SGE_NUM_LEFT QPC_FIELD_LOC(1535, 1528)</div><div class='add'>+#define QPC_OWNER_MODE QPC_FIELD_LOC(1536, 1536)</div><div class='add'>+#define QPC_CIRE_SLV_SQ_EN QPC_FIELD_LOC(1537, 1537)</div><div class='add'>+#define QPC_CIRE_DOING QPC_FIELD_LOC(1538, 1538)</div><div class='add'>+#define QPC_CIRE_RESULT QPC_FIELD_LOC(1539, 1539)</div><div class='add'>+#define QPC_OWNER_DB_WAIT_DO QPC_FIELD_LOC(1540, 1540)</div><div class='add'>+#define QPC_SQ_WQE_INVLD QPC_FIELD_LOC(1541, 1541)</div><div class='add'>+#define QPC_DCA_MODE QPC_FIELD_LOC(1542, 1542)</div><div class='add'>+#define QPC_RTY_OWNER_NOCHK QPC_FIELD_LOC(1543, 1543)</div><div class='add'>+#define QPC_V2_IRRL_HEAD QPC_FIELD_LOC(1543, 1536)</div><div class='add'>+#define QPC_SQ_MAX_PSN QPC_FIELD_LOC(1567, 1544)</div><div class='add'>+#define QPC_SQ_MAX_IDX QPC_FIELD_LOC(1583, 1568)</div><div class='add'>+#define QPC_LCL_OPERATED_CNT QPC_FIELD_LOC(1599, 1584)</div><div class='add'>+#define QPC_IRRL_BA_L QPC_FIELD_LOC(1631, 1600)</div><div class='add'>+#define QPC_IRRL_BA_H QPC_FIELD_LOC(1657, 1632)</div><div class='add'>+#define QPC_PKT_RNR_FLG QPC_FIELD_LOC(1658, 1658)</div><div class='add'>+#define QPC_PKT_RTY_FLG QPC_FIELD_LOC(1659, 1659)</div><div class='add'>+#define QPC_RMT_E2E QPC_FIELD_LOC(1660, 1660)</div><div class='add'>+#define QPC_SR_MAX QPC_FIELD_LOC(1663, 1661)</div><div class='add'>+#define QPC_LSN QPC_FIELD_LOC(1687, 1664)</div><div class='add'>+#define QPC_RETRY_NUM_INIT QPC_FIELD_LOC(1690, 1688)</div><div class='add'>+#define QPC_CHECK_FLG QPC_FIELD_LOC(1692, 1691)</div><div class='add'>+#define QPC_RETRY_CNT QPC_FIELD_LOC(1695, 1693)</div><div class='add'>+#define QPC_SQ_TIMER QPC_FIELD_LOC(1727, 1696)</div><div class='add'>+#define QPC_RETRY_MSG_MSN QPC_FIELD_LOC(1743, 1728)</div><div class='add'>+#define QPC_RETRY_MSG_PSN_L QPC_FIELD_LOC(1759, 1744)</div><div class='add'>+#define QPC_RETRY_MSG_PSN_H QPC_FIELD_LOC(1767, 1760)</div><div class='add'>+#define QPC_RETRY_MSG_FPKT_PSN QPC_FIELD_LOC(1791, 1768)</div><div class='add'>+#define QPC_RX_SQ_CUR_BLK_ADDR_L QPC_FIELD_LOC(1823, 1792)</div><div class='add'>+#define QPC_RX_SQ_CUR_BLK_ADDR_H QPC_FIELD_LOC(1843, 1824)</div><div class='add'>+#define QPC_IRRL_SGE_IDX QPC_FIELD_LOC(1851, 1844)</div><div class='add'>+#define QPC_LSAN_EN QPC_FIELD_LOC(1852, 1852)</div><div class='add'>+#define QPC_SO_LP_VLD QPC_FIELD_LOC(1853, 1853)</div><div class='add'>+#define QPC_FENCE_LP_VLD QPC_FIELD_LOC(1854, 1854)</div><div class='add'>+#define QPC_IRRL_LP_VLD QPC_FIELD_LOC(1855, 1855)</div><div class='add'>+#define QPC_IRRL_CUR_SGE_OFFSET QPC_FIELD_LOC(1887, 1856)</div><div class='add'>+#define QPC_IRRL_TAIL_REAL QPC_FIELD_LOC(1895, 1888)</div><div class='add'>+#define QPC_IRRL_TAIL_RD QPC_FIELD_LOC(1903, 1896)</div><div class='add'>+#define QPC_RX_ACK_MSN QPC_FIELD_LOC(1919, 1904)</div><div class='add'>+#define QPC_RX_ACK_EPSN QPC_FIELD_LOC(1943, 1920)</div><div class='add'>+#define QPC_RNR_NUM_INIT QPC_FIELD_LOC(1946, 1944)</div><div class='add'>+#define QPC_RNR_CNT QPC_FIELD_LOC(1949, 1947)</div><div class='add'>+#define QPC_LCL_OP_FLG QPC_FIELD_LOC(1950, 1950)</div><div class='add'>+#define QPC_IRRL_RD_FLG QPC_FIELD_LOC(1951, 1951)</div><div class='add'>+#define QPC_IRRL_PSN QPC_FIELD_LOC(1975, 1952)</div><div class='add'>+#define QPC_ACK_PSN_ERR QPC_FIELD_LOC(1976, 1976)</div><div class='add'>+#define QPC_ACK_LAST_OPTYPE QPC_FIELD_LOC(1978, 1977)</div><div class='add'>+#define QPC_IRRL_PSN_VLD QPC_FIELD_LOC(1979, 1979)</div><div class='add'>+#define QPC_RNR_RETRY_FLAG QPC_FIELD_LOC(1980, 1980)</div><div class='add'>+#define QPC_SQ_RTY_TX_ERR QPC_FIELD_LOC(1981, 1981)</div><div class='add'>+#define QPC_LAST_IND QPC_FIELD_LOC(1982, 1982)</div><div class='add'>+#define QPC_CQ_ERR_IND QPC_FIELD_LOC(1983, 1983)</div><div class='add'>+#define QPC_TX_CQN QPC_FIELD_LOC(2007, 1984)</div><div class='add'>+#define QPC_SIG_TYPE QPC_FIELD_LOC(2008, 2008)</div><div class='add'>+#define QPC_ERR_TYPE QPC_FIELD_LOC(2015, 2009)</div><div class='add'>+#define QPC_RQ_CQE_IDX QPC_FIELD_LOC(2031, 2016)</div><div class='add'>+#define QPC_SQ_FLUSH_IDX QPC_FIELD_LOC(2047, 2032)</div><div class='add'>+</div><div class='add'>+#define RETRY_MSG_PSN_SHIFT 16</div><div class='add'>+</div><div class='add'>+#define QPCEX_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_qp_context_ex, h, l)</div><div class='add'>+</div><div class='add'>+#define QPCEX_CONG_ALG_SEL QPCEX_FIELD_LOC(0, 0)</div><div class='add'>+#define QPCEX_CONG_ALG_SUB_SEL QPCEX_FIELD_LOC(1, 1)</div><div class='add'>+#define QPCEX_DIP_CTX_IDX_VLD QPCEX_FIELD_LOC(2, 2)</div><div class='add'>+#define QPCEX_DIP_CTX_IDX QPCEX_FIELD_LOC(22, 3)</div><div class='add'>+#define QPCEX_SQ_RQ_NOT_FORBID_EN QPCEX_FIELD_LOC(23, 23)</div><div class='add'>+#define QPCEX_STASH QPCEX_FIELD_LOC(82, 82)</div><div class='add'>+</div><div class='add'>+#define SCC_CONTEXT_SIZE 16</div><div class='add'>+</div><div class='add'>+struct hns_roce_v2_scc_context {</div><div class='add'>+	__le32 data[SCC_CONTEXT_SIZE];</div><div class='add'>+};</div><div class='ctx'> </div><div class='del'>-#define	V2_QPC_BYTE_256_SQ_FLUSH_IDX_S 16</div><div class='del'>-#define V2_QPC_BYTE_256_SQ_FLUSH_IDX_M GENMASK(31, 16)</div><div class='add'>+#define	V2_QP_RWE_S 1 /* rdma write enable */</div><div class='add'>+#define	V2_QP_RRE_S 2 /* rdma read enable */</div><div class='add'>+#define	V2_QP_ATE_S 3 /* rdma atomic enable */</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_v2_cqe {</div><div class='ctx'> 	__le32	byte_4;</div><div class='hunk'>@@ -898,58 +696,34 @@ struct hns_roce_v2_cqe {</div><div class='ctx'> 	u8	smac[4];</div><div class='ctx'> 	__le32	byte_28;</div><div class='ctx'> 	__le32	byte_32;</div><div class='add'>+	__le32	rsv[8];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define	V2_CQE_BYTE_4_OPCODE_S 0</div><div class='del'>-#define V2_CQE_BYTE_4_OPCODE_M GENMASK(4, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_CQE_BYTE_4_RQ_INLINE_S 5</div><div class='del'>-</div><div class='del'>-#define	V2_CQE_BYTE_4_S_R_S 6</div><div class='del'>-</div><div class='del'>-#define	V2_CQE_BYTE_4_OWNER_S 7</div><div class='del'>-</div><div class='del'>-#define	V2_CQE_BYTE_4_STATUS_S 8</div><div class='del'>-#define V2_CQE_BYTE_4_STATUS_M GENMASK(15, 8)</div><div class='del'>-</div><div class='del'>-#define	V2_CQE_BYTE_4_WQE_INDX_S 16</div><div class='del'>-#define V2_CQE_BYTE_4_WQE_INDX_M GENMASK(31, 16)</div><div class='del'>-</div><div class='del'>-#define	V2_CQE_BYTE_12_XRC_SRQN_S 0</div><div class='del'>-#define V2_CQE_BYTE_12_XRC_SRQN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_CQE_BYTE_16_LCL_QPN_S 0</div><div class='del'>-#define V2_CQE_BYTE_16_LCL_QPN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_CQE_BYTE_16_SUB_STATUS_S 24</div><div class='del'>-#define V2_CQE_BYTE_16_SUB_STATUS_M GENMASK(31, 24)</div><div class='del'>-</div><div class='del'>-#define	V2_CQE_BYTE_28_SMAC_4_S 0</div><div class='del'>-#define V2_CQE_BYTE_28_SMAC_4_M	GENMASK(7, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_CQE_BYTE_28_SMAC_5_S 8</div><div class='del'>-#define V2_CQE_BYTE_28_SMAC_5_M	GENMASK(15, 8)</div><div class='del'>-</div><div class='del'>-#define	V2_CQE_BYTE_28_PORT_TYPE_S 16</div><div class='del'>-#define V2_CQE_BYTE_28_PORT_TYPE_M GENMASK(17, 16)</div><div class='del'>-</div><div class='del'>-#define V2_CQE_BYTE_28_VID_S 18</div><div class='del'>-#define V2_CQE_BYTE_28_VID_M GENMASK(29, 18)</div><div class='del'>-</div><div class='del'>-#define V2_CQE_BYTE_28_VID_VLD_S 30</div><div class='del'>-</div><div class='del'>-#define	V2_CQE_BYTE_32_RMT_QPN_S 0</div><div class='del'>-#define V2_CQE_BYTE_32_RMT_QPN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_CQE_BYTE_32_SL_S 24</div><div class='del'>-#define V2_CQE_BYTE_32_SL_M GENMASK(26, 24)</div><div class='del'>-</div><div class='del'>-#define	V2_CQE_BYTE_32_PORTN_S 27</div><div class='del'>-#define V2_CQE_BYTE_32_PORTN_M GENMASK(29, 27)</div><div class='del'>-</div><div class='del'>-#define	V2_CQE_BYTE_32_GRH_S 30</div><div class='del'>-</div><div class='del'>-#define	V2_CQE_BYTE_32_LPK_S 31</div><div class='add'>+#define CQE_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_cqe, h, l)</div><div class='add'>+</div><div class='add'>+#define CQE_OPCODE CQE_FIELD_LOC(4, 0)</div><div class='add'>+#define CQE_RQ_INLINE CQE_FIELD_LOC(5, 5)</div><div class='add'>+#define CQE_S_R CQE_FIELD_LOC(6, 6)</div><div class='add'>+#define CQE_OWNER CQE_FIELD_LOC(7, 7)</div><div class='add'>+#define CQE_STATUS CQE_FIELD_LOC(15, 8)</div><div class='add'>+#define CQE_WQE_IDX CQE_FIELD_LOC(31, 16)</div><div class='add'>+#define CQE_RKEY_IMMTDATA CQE_FIELD_LOC(63, 32)</div><div class='add'>+#define CQE_XRC_SRQN CQE_FIELD_LOC(87, 64)</div><div class='add'>+#define CQE_RSV0 CQE_FIELD_LOC(95, 88)</div><div class='add'>+#define CQE_LCL_QPN CQE_FIELD_LOC(119, 96)</div><div class='add'>+#define CQE_SUB_STATUS CQE_FIELD_LOC(127, 120)</div><div class='add'>+#define CQE_BYTE_CNT CQE_FIELD_LOC(159, 128)</div><div class='add'>+#define CQE_SMAC CQE_FIELD_LOC(207, 160)</div><div class='add'>+#define CQE_PORT_TYPE CQE_FIELD_LOC(209, 208)</div><div class='add'>+#define CQE_VID CQE_FIELD_LOC(221, 210)</div><div class='add'>+#define CQE_VID_VLD CQE_FIELD_LOC(222, 222)</div><div class='add'>+#define CQE_RSV2 CQE_FIELD_LOC(223, 223)</div><div class='add'>+#define CQE_RMT_QPN CQE_FIELD_LOC(247, 224)</div><div class='add'>+#define CQE_SL CQE_FIELD_LOC(250, 248)</div><div class='add'>+#define CQE_PORTN CQE_FIELD_LOC(253, 251)</div><div class='add'>+#define CQE_GRH CQE_FIELD_LOC(254, 254)</div><div class='add'>+#define CQE_LPK CQE_FIELD_LOC(255, 255)</div><div class='add'>+#define CQE_RSV3 CQE_FIELD_LOC(511, 256)</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_v2_mpt_entry {</div><div class='ctx'> 	__le32	byte_4_pd_hop_st;</div><div class='hunk'>@@ -970,6 +744,52 @@ struct hns_roce_v2_mpt_entry {</div><div class='ctx'> 	__le32	byte_64_buf_pa1;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+#define MPT_PBL_BUF_ADDR_S 6</div><div class='add'>+#define MPT_PBL_BA_ADDR_S 3</div><div class='add'>+</div><div class='add'>+#define MPT_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_mpt_entry, h, l)</div><div class='add'>+</div><div class='add'>+#define MPT_ST MPT_FIELD_LOC(1, 0)</div><div class='add'>+#define MPT_PBL_HOP_NUM MPT_FIELD_LOC(3, 2)</div><div class='add'>+#define MPT_PBL_BA_PG_SZ MPT_FIELD_LOC(7, 4)</div><div class='add'>+#define MPT_PD MPT_FIELD_LOC(31, 8)</div><div class='add'>+#define MPT_RA_EN MPT_FIELD_LOC(32, 32)</div><div class='add'>+#define MPT_R_INV_EN MPT_FIELD_LOC(33, 33)</div><div class='add'>+#define MPT_L_INV_EN MPT_FIELD_LOC(34, 34)</div><div class='add'>+#define MPT_BIND_EN MPT_FIELD_LOC(35, 35)</div><div class='add'>+#define MPT_ATOMIC_EN MPT_FIELD_LOC(36, 36)</div><div class='add'>+#define MPT_RR_EN MPT_FIELD_LOC(37, 37)</div><div class='add'>+#define MPT_RW_EN MPT_FIELD_LOC(38, 38)</div><div class='add'>+#define MPT_LW_EN MPT_FIELD_LOC(39, 39)</div><div class='add'>+#define MPT_MW_CNT MPT_FIELD_LOC(63, 40)</div><div class='add'>+#define MPT_FRE MPT_FIELD_LOC(64, 64)</div><div class='add'>+#define MPT_PA MPT_FIELD_LOC(65, 65)</div><div class='add'>+#define MPT_ZBVA MPT_FIELD_LOC(66, 66)</div><div class='add'>+#define MPT_SHARE MPT_FIELD_LOC(67, 67)</div><div class='add'>+#define MPT_MR_MW MPT_FIELD_LOC(68, 68)</div><div class='add'>+#define MPT_BPD MPT_FIELD_LOC(69, 69)</div><div class='add'>+#define MPT_BQP MPT_FIELD_LOC(70, 70)</div><div class='add'>+#define MPT_INNER_PA_VLD MPT_FIELD_LOC(71, 71)</div><div class='add'>+#define MPT_MW_BIND_QPN MPT_FIELD_LOC(95, 72)</div><div class='add'>+#define MPT_BOUND_LKEY MPT_FIELD_LOC(127, 96)</div><div class='add'>+#define MPT_LEN_L MPT_FIELD_LOC(159, 128)</div><div class='add'>+#define MPT_LEN_H MPT_FIELD_LOC(191, 160)</div><div class='add'>+#define MPT_LKEY MPT_FIELD_LOC(223, 192)</div><div class='add'>+#define MPT_VA MPT_FIELD_LOC(287, 224)</div><div class='add'>+#define MPT_PBL_SIZE MPT_FIELD_LOC(319, 288)</div><div class='add'>+#define MPT_PBL_BA_L MPT_FIELD_LOC(351, 320)</div><div class='add'>+#define MPT_PBL_BA_H MPT_FIELD_LOC(380, 352)</div><div class='add'>+#define MPT_BLK_MODE MPT_FIELD_LOC(381, 381)</div><div class='add'>+#define MPT_RSV0 MPT_FIELD_LOC(383, 382)</div><div class='add'>+#define MPT_PA0_L MPT_FIELD_LOC(415, 384)</div><div class='add'>+#define MPT_PA0_H MPT_FIELD_LOC(441, 416)</div><div class='add'>+#define MPT_BOUND_VA MPT_FIELD_LOC(447, 442)</div><div class='add'>+#define MPT_PA1_L MPT_FIELD_LOC(479, 448)</div><div class='add'>+#define MPT_PA1_H MPT_FIELD_LOC(505, 480)</div><div class='add'>+#define MPT_PERSIST_EN MPT_FIELD_LOC(506, 506)</div><div class='add'>+#define MPT_RSV2 MPT_FIELD_LOC(507, 507)</div><div class='add'>+#define MPT_PBL_BUF_PG_SZ MPT_FIELD_LOC(511, 508)</div><div class='add'>+</div><div class='ctx'> #define V2_MPT_BYTE_4_MPT_ST_S 0</div><div class='ctx'> #define V2_MPT_BYTE_4_MPT_ST_M GENMASK(1, 0)</div><div class='ctx'> </div><div class='hunk'>@@ -998,24 +818,16 @@ struct hns_roce_v2_mpt_entry {</div><div class='ctx'> </div><div class='ctx'> #define V2_MPT_BYTE_8_LW_EN_S 7</div><div class='ctx'> </div><div class='del'>-#define V2_MPT_BYTE_8_MW_CNT_S 8</div><div class='del'>-#define V2_MPT_BYTE_8_MW_CNT_M GENMASK(31, 8)</div><div class='del'>-</div><div class='ctx'> #define V2_MPT_BYTE_12_FRE_S 0</div><div class='ctx'> </div><div class='ctx'> #define V2_MPT_BYTE_12_PA_S 1</div><div class='ctx'> </div><div class='del'>-#define V2_MPT_BYTE_12_MR_MW_S 4</div><div class='del'>-</div><div class='ctx'> #define V2_MPT_BYTE_12_BPD_S 5</div><div class='ctx'> </div><div class='ctx'> #define V2_MPT_BYTE_12_BQP_S 6</div><div class='ctx'> </div><div class='ctx'> #define V2_MPT_BYTE_12_INNER_PA_VLD_S 7</div><div class='ctx'> </div><div class='del'>-#define V2_MPT_BYTE_12_MW_BIND_QPN_S 8</div><div class='del'>-#define V2_MPT_BYTE_12_MW_BIND_QPN_M GENMASK(31, 8)</div><div class='del'>-</div><div class='ctx'> #define V2_MPT_BYTE_48_PBL_BA_H_S 0</div><div class='ctx'> #define V2_MPT_BYTE_48_PBL_BA_H_M GENMASK(28, 0)</div><div class='ctx'> </div><div class='hunk'>@@ -1030,36 +842,29 @@ struct hns_roce_v2_mpt_entry {</div><div class='ctx'> #define V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S 28</div><div class='ctx'> #define V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M GENMASK(31, 28)</div><div class='ctx'> </div><div class='del'>-#define	V2_DB_BYTE_4_TAG_S 0</div><div class='del'>-#define V2_DB_BYTE_4_TAG_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_DB_BYTE_4_CMD_S 24</div><div class='del'>-#define V2_DB_BYTE_4_CMD_M GENMASK(27, 24)</div><div class='del'>-</div><div class='del'>-#define V2_DB_PARAMETER_IDX_S 0</div><div class='del'>-#define V2_DB_PARAMETER_IDX_M GENMASK(15, 0)</div><div class='del'>-</div><div class='del'>-#define V2_DB_PARAMETER_SL_S 16</div><div class='del'>-#define V2_DB_PARAMETER_SL_M GENMASK(18, 16)</div><div class='del'>-</div><div class='del'>-struct hns_roce_v2_cq_db {</div><div class='del'>-	__le32	byte_4;</div><div class='del'>-	__le32	parameter;</div><div class='add'>+struct hns_roce_v2_db {</div><div class='add'>+	__le32	data[2];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define	V2_CQ_DB_BYTE_4_TAG_S 0</div><div class='del'>-#define V2_CQ_DB_BYTE_4_TAG_M GENMASK(23, 0)</div><div class='add'>+#define DB_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_db, h, l)</div><div class='ctx'> </div><div class='del'>-#define	V2_CQ_DB_BYTE_4_CMD_S 24</div><div class='del'>-#define V2_CQ_DB_BYTE_4_CMD_M GENMASK(27, 24)</div><div class='add'>+#define DB_TAG DB_FIELD_LOC(23, 0)</div><div class='add'>+#define DB_CMD DB_FIELD_LOC(27, 24)</div><div class='add'>+#define DB_FLAG DB_FIELD_LOC(31, 31)</div><div class='add'>+#define DB_PI DB_FIELD_LOC(47, 32)</div><div class='add'>+#define DB_SL DB_FIELD_LOC(50, 48)</div><div class='add'>+#define DB_CQ_CI DB_FIELD_LOC(55, 32)</div><div class='add'>+#define DB_CQ_NOTIFY DB_FIELD_LOC(56, 56)</div><div class='add'>+#define DB_CQ_CMD_SN DB_FIELD_LOC(58, 57)</div><div class='add'>+#define EQ_DB_TAG DB_FIELD_LOC(7, 0)</div><div class='add'>+#define EQ_DB_CMD DB_FIELD_LOC(17, 16)</div><div class='add'>+#define EQ_DB_CI DB_FIELD_LOC(55, 32)</div><div class='ctx'> </div><div class='del'>-#define V2_CQ_DB_PARAMETER_CONS_IDX_S 0</div><div class='del'>-#define V2_CQ_DB_PARAMETER_CONS_IDX_M GENMASK(23, 0)</div><div class='add'>+#define V2_DB_PRODUCER_IDX_S 0</div><div class='add'>+#define V2_DB_PRODUCER_IDX_M GENMASK(15, 0)</div><div class='ctx'> </div><div class='del'>-#define V2_CQ_DB_PARAMETER_CMD_SN_S 25</div><div class='del'>-#define V2_CQ_DB_PARAMETER_CMD_SN_M GENMASK(26, 25)</div><div class='del'>-</div><div class='del'>-#define V2_CQ_DB_PARAMETER_NOTIFY_S 24</div><div class='add'>+#define V2_CQ_DB_CONS_IDX_S 0</div><div class='add'>+#define V2_CQ_DB_CONS_IDX_M GENMASK(23, 0)</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_v2_ud_send_wqe {</div><div class='ctx'> 	__le32	byte_4;</div><div class='hunk'>@@ -1072,80 +877,30 @@ struct hns_roce_v2_ud_send_wqe {</div><div class='ctx'> 	__le32	byte_32;</div><div class='ctx'> 	__le32	byte_36;</div><div class='ctx'> 	__le32	byte_40;</div><div class='del'>-	__le32	dmac;</div><div class='del'>-	__le32	byte_48;</div><div class='add'>+	u8	dmac[ETH_ALEN];</div><div class='add'>+	u8	sgid_index;</div><div class='add'>+	u8	smac_index;</div><div class='ctx'> 	u8	dgid[GID_LEN_V2];</div><div class='del'>-</div><div class='ctx'> };</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_4_OPCODE_S 0</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_4_OPCODE_M GENMASK(4, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_4_OWNER_S 7</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_4_CQE_S 8</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_4_SE_S 11</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_16_PD_S 0</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_16_PD_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S 24</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M GENMASK(31, 24)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S 0</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_24_UDPSPN_S 16</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_24_UDPSPN_M GENMASK(31, 16)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_32_DQPN_S 0</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_32_DQPN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_36_VLAN_S 0</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_36_VLAN_M GENMASK(15, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S 16</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M GENMASK(23, 16)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_36_TCLASS_S 24</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_36_TCLASS_M GENMASK(31, 24)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S 0</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M GENMASK(19, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_40_SL_S 20</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_40_SL_M GENMASK(23, 20)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_40_PORTN_S 24</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_40_PORTN_M GENMASK(26, 24)</div><div class='del'>-</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S 30</div><div class='ctx'> </div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_40_LBI_S 31</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_DMAC_0_S 0</div><div class='del'>-#define V2_UD_SEND_WQE_DMAC_0_M GENMASK(7, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_DMAC_1_S 8</div><div class='del'>-#define V2_UD_SEND_WQE_DMAC_1_M GENMASK(15, 8)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_DMAC_2_S 16</div><div class='del'>-#define V2_UD_SEND_WQE_DMAC_2_M GENMASK(23, 16)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_DMAC_3_S 24</div><div class='del'>-#define V2_UD_SEND_WQE_DMAC_3_M GENMASK(31, 24)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_48_DMAC_4_S 0</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_48_DMAC_4_M GENMASK(7, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_48_DMAC_5_S 8</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_48_DMAC_5_M GENMASK(15, 8)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S 16</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M GENMASK(23, 16)</div><div class='del'>-</div><div class='del'>-#define	V2_UD_SEND_WQE_BYTE_48_SMAC_INDX_S 24</div><div class='del'>-#define V2_UD_SEND_WQE_BYTE_48_SMAC_INDX_M GENMASK(31, 24)</div><div class='add'>+#define UD_SEND_WQE_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_ud_send_wqe, h, l)</div><div class='add'>+</div><div class='add'>+#define UD_SEND_WQE_OPCODE UD_SEND_WQE_FIELD_LOC(4, 0)</div><div class='add'>+#define UD_SEND_WQE_OWNER UD_SEND_WQE_FIELD_LOC(7, 7)</div><div class='add'>+#define UD_SEND_WQE_CQE UD_SEND_WQE_FIELD_LOC(8, 8)</div><div class='add'>+#define UD_SEND_WQE_SE UD_SEND_WQE_FIELD_LOC(11, 11)</div><div class='add'>+#define UD_SEND_WQE_PD UD_SEND_WQE_FIELD_LOC(119, 96)</div><div class='add'>+#define UD_SEND_WQE_SGE_NUM UD_SEND_WQE_FIELD_LOC(127, 120)</div><div class='add'>+#define UD_SEND_WQE_MSG_START_SGE_IDX UD_SEND_WQE_FIELD_LOC(151, 128)</div><div class='add'>+#define UD_SEND_WQE_UDPSPN UD_SEND_WQE_FIELD_LOC(191, 176)</div><div class='add'>+#define UD_SEND_WQE_DQPN UD_SEND_WQE_FIELD_LOC(247, 224)</div><div class='add'>+#define UD_SEND_WQE_VLAN UD_SEND_WQE_FIELD_LOC(271, 256)</div><div class='add'>+#define UD_SEND_WQE_HOPLIMIT UD_SEND_WQE_FIELD_LOC(279, 272)</div><div class='add'>+#define UD_SEND_WQE_TCLASS UD_SEND_WQE_FIELD_LOC(287, 280)</div><div class='add'>+#define UD_SEND_WQE_FLOW_LABEL UD_SEND_WQE_FIELD_LOC(307, 288)</div><div class='add'>+#define UD_SEND_WQE_SL UD_SEND_WQE_FIELD_LOC(311, 308)</div><div class='add'>+#define UD_SEND_WQE_VLAN_EN UD_SEND_WQE_FIELD_LOC(318, 318)</div><div class='add'>+#define UD_SEND_WQE_LBI UD_SEND_WQE_FIELD_LOC(319, 319)</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_v2_rc_send_wqe {</div><div class='ctx'> 	__le32		byte_4;</div><div class='hunk'>@@ -1160,49 +915,41 @@ struct hns_roce_v2_rc_send_wqe {</div><div class='ctx'> 	__le64		va;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define	V2_RC_SEND_WQE_BYTE_4_OPCODE_S 0</div><div class='del'>-#define V2_RC_SEND_WQE_BYTE_4_OPCODE_M GENMASK(4, 0)</div><div class='del'>-</div><div class='del'>-#define V2_RC_SEND_WQE_BYTE_4_OWNER_S 7</div><div class='del'>-</div><div class='del'>-#define V2_RC_SEND_WQE_BYTE_4_CQE_S 8</div><div class='del'>-</div><div class='del'>-#define V2_RC_SEND_WQE_BYTE_4_FENCE_S 9</div><div class='del'>-</div><div class='del'>-#define V2_RC_SEND_WQE_BYTE_4_SO_S 10</div><div class='del'>-</div><div class='del'>-#define V2_RC_SEND_WQE_BYTE_4_SE_S 11</div><div class='del'>-</div><div class='del'>-#define V2_RC_SEND_WQE_BYTE_4_INLINE_S 12</div><div class='del'>-</div><div class='del'>-#define V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S 19</div><div class='del'>-</div><div class='del'>-#define V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S 20</div><div class='del'>-</div><div class='del'>-#define V2_RC_FRMR_WQE_BYTE_4_RR_S 21</div><div class='del'>-</div><div class='del'>-#define V2_RC_FRMR_WQE_BYTE_4_RW_S 22</div><div class='del'>-</div><div class='del'>-#define V2_RC_FRMR_WQE_BYTE_4_LW_S 23</div><div class='del'>-</div><div class='del'>-#define	V2_RC_SEND_WQE_BYTE_16_XRC_SRQN_S 0</div><div class='del'>-#define V2_RC_SEND_WQE_BYTE_16_XRC_SRQN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define	V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S 24</div><div class='del'>-#define V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M GENMASK(31, 24)</div><div class='del'>-</div><div class='del'>-#define V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S 0</div><div class='del'>-#define V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M GENMASK(23, 0)</div><div class='add'>+#define RC_SEND_WQE_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_rc_send_wqe, h, l)</div><div class='add'>+</div><div class='add'>+#define RC_SEND_WQE_OPCODE RC_SEND_WQE_FIELD_LOC(4, 0)</div><div class='add'>+#define RC_SEND_WQE_DB_SL_L RC_SEND_WQE_FIELD_LOC(6, 5)</div><div class='add'>+#define RC_SEND_WQE_DB_SL_H RC_SEND_WQE_FIELD_LOC(14, 13)</div><div class='add'>+#define RC_SEND_WQE_OWNER RC_SEND_WQE_FIELD_LOC(7, 7)</div><div class='add'>+#define RC_SEND_WQE_CQE RC_SEND_WQE_FIELD_LOC(8, 8)</div><div class='add'>+#define RC_SEND_WQE_FENCE RC_SEND_WQE_FIELD_LOC(9, 9)</div><div class='add'>+#define RC_SEND_WQE_SO RC_SEND_WQE_FIELD_LOC(10, 10)</div><div class='add'>+#define RC_SEND_WQE_SE RC_SEND_WQE_FIELD_LOC(11, 11)</div><div class='add'>+#define RC_SEND_WQE_INLINE RC_SEND_WQE_FIELD_LOC(12, 12)</div><div class='add'>+#define RC_SEND_WQE_WQE_INDEX RC_SEND_WQE_FIELD_LOC(30, 15)</div><div class='add'>+#define RC_SEND_WQE_FLAG RC_SEND_WQE_FIELD_LOC(31, 31)</div><div class='add'>+#define RC_SEND_WQE_XRC_SRQN RC_SEND_WQE_FIELD_LOC(119, 96)</div><div class='add'>+#define RC_SEND_WQE_SGE_NUM RC_SEND_WQE_FIELD_LOC(127, 120)</div><div class='add'>+#define RC_SEND_WQE_MSG_START_SGE_IDX RC_SEND_WQE_FIELD_LOC(151, 128)</div><div class='add'>+#define RC_SEND_WQE_INL_TYPE RC_SEND_WQE_FIELD_LOC(159, 159)</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_wqe_frmr_seg {</div><div class='ctx'> 	__le32	pbl_size;</div><div class='del'>-	__le32	mode_buf_pg_sz;</div><div class='add'>+	__le32	byte_40;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S	4</div><div class='del'>-#define V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M	GENMASK(7, 4)</div><div class='add'>+#define FRMR_WQE_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_wqe_frmr_seg, h, l)</div><div class='ctx'> </div><div class='del'>-#define V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S 8</div><div class='add'>+#define FRMR_PBL_SIZE FRMR_WQE_FIELD_LOC(31, 0)</div><div class='add'>+#define FRMR_BLOCK_SIZE FRMR_WQE_FIELD_LOC(35, 32)</div><div class='add'>+#define FRMR_PBL_BUF_PG_SZ FRMR_WQE_FIELD_LOC(39, 36)</div><div class='add'>+#define FRMR_BLK_MODE FRMR_WQE_FIELD_LOC(40, 40)</div><div class='add'>+#define FRMR_ZBVA FRMR_WQE_FIELD_LOC(41, 41)</div><div class='add'>+#define FRMR_BIND_EN FRMR_WQE_FIELD_LOC(42, 42)</div><div class='add'>+#define FRMR_ATOMIC FRMR_WQE_FIELD_LOC(43, 43)</div><div class='add'>+#define FRMR_RR FRMR_WQE_FIELD_LOC(44, 44)</div><div class='add'>+#define FRMR_RW FRMR_WQE_FIELD_LOC(45, 45)</div><div class='add'>+#define FRMR_LW FRMR_WQE_FIELD_LOC(46, 46)</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_v2_wqe_data_seg {</div><div class='ctx'> 	__le32    len;</div><div class='hunk'>@@ -1210,11 +957,6 @@ struct hns_roce_v2_wqe_data_seg {</div><div class='ctx'> 	__le64    addr;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct hns_roce_v2_db {</div><div class='del'>-	__le32	byte_4;</div><div class='del'>-	__le32	parameter;</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> struct hns_roce_query_version {</div><div class='ctx'> 	__le16 rocee_vendor_id;</div><div class='ctx'> 	__le16 rocee_hw_version;</div><div class='hunk'>@@ -1226,216 +968,75 @@ struct hns_roce_query_fw_info {</div><div class='ctx'> 	__le32 rsv[5];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct hns_roce_cfg_llm_a {</div><div class='del'>-	__le32 base_addr_l;</div><div class='del'>-	__le32 base_addr_h;</div><div class='del'>-	__le32 depth_pgsz_init_en;</div><div class='del'>-	__le32 head_ba_l;</div><div class='del'>-	__le32 head_ba_h_nxtptr;</div><div class='del'>-	__le32 head_ptr;</div><div class='add'>+struct hns_roce_func_clear {</div><div class='add'>+	__le32 rst_funcid_en;</div><div class='add'>+	__le32 func_done;</div><div class='add'>+	__le32 rsv[4];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define CFG_LLM_QUE_DEPTH_S 0</div><div class='del'>-#define CFG_LLM_QUE_DEPTH_M GENMASK(12, 0)</div><div class='del'>-</div><div class='del'>-#define CFG_LLM_QUE_PGSZ_S 16</div><div class='del'>-#define CFG_LLM_QUE_PGSZ_M GENMASK(19, 16)</div><div class='del'>-</div><div class='del'>-#define CFG_LLM_INIT_EN_S 20</div><div class='del'>-#define CFG_LLM_INIT_EN_M GENMASK(20, 20)</div><div class='del'>-</div><div class='del'>-#define CFG_LLM_HEAD_PTR_S 0</div><div class='del'>-#define CFG_LLM_HEAD_PTR_M GENMASK(11, 0)</div><div class='del'>-</div><div class='del'>-struct hns_roce_cfg_llm_b {</div><div class='del'>-	__le32 tail_ba_l;</div><div class='del'>-	__le32 tail_ba_h;</div><div class='del'>-	__le32 tail_ptr;</div><div class='del'>-	__le32 rsv[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define CFG_LLM_TAIL_BA_H_S 0</div><div class='del'>-#define CFG_LLM_TAIL_BA_H_M GENMASK(19, 0)</div><div class='del'>-</div><div class='del'>-#define CFG_LLM_TAIL_PTR_S 0</div><div class='del'>-#define CFG_LLM_TAIL_PTR_M GENMASK(11, 0)</div><div class='del'>-</div><div class='del'>-struct hns_roce_cfg_global_param {</div><div class='del'>-	__le32 time_cfg_udp_port;</div><div class='del'>-	__le32 rsv[5];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S 0</div><div class='del'>-#define CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M GENMASK(9, 0)</div><div class='del'>-</div><div class='del'>-#define CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S 16</div><div class='del'>-#define CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M GENMASK(31, 16)</div><div class='del'>-</div><div class='del'>-struct hns_roce_pf_res_a {</div><div class='del'>-	__le32	rsv;</div><div class='del'>-	__le32	qpc_bt_idx_num;</div><div class='del'>-	__le32	srqc_bt_idx_num;</div><div class='del'>-	__le32	cqc_bt_idx_num;</div><div class='del'>-	__le32	mpt_bt_idx_num;</div><div class='del'>-	__le32	eqc_bt_idx_num;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_1_PF_QPC_BT_IDX_S 0</div><div class='del'>-#define PF_RES_DATA_1_PF_QPC_BT_IDX_M GENMASK(10, 0)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_1_PF_QPC_BT_NUM_S 16</div><div class='del'>-#define PF_RES_DATA_1_PF_QPC_BT_NUM_M GENMASK(27, 16)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_2_PF_SRQC_BT_IDX_S 0</div><div class='del'>-#define PF_RES_DATA_2_PF_SRQC_BT_IDX_M GENMASK(8, 0)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_2_PF_SRQC_BT_NUM_S 16</div><div class='del'>-#define PF_RES_DATA_2_PF_SRQC_BT_NUM_M GENMASK(25, 16)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_3_PF_CQC_BT_IDX_S 0</div><div class='del'>-#define PF_RES_DATA_3_PF_CQC_BT_IDX_M GENMASK(8, 0)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_3_PF_CQC_BT_NUM_S 16</div><div class='del'>-#define PF_RES_DATA_3_PF_CQC_BT_NUM_M GENMASK(25, 16)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_4_PF_MPT_BT_IDX_S 0</div><div class='del'>-#define PF_RES_DATA_4_PF_MPT_BT_IDX_M GENMASK(8, 0)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_4_PF_MPT_BT_NUM_S 16</div><div class='del'>-#define PF_RES_DATA_4_PF_MPT_BT_NUM_M GENMASK(25, 16)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_5_PF_EQC_BT_IDX_S 0</div><div class='del'>-#define PF_RES_DATA_5_PF_EQC_BT_IDX_M GENMASK(8, 0)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_5_PF_EQC_BT_NUM_S 16</div><div class='del'>-#define PF_RES_DATA_5_PF_EQC_BT_NUM_M GENMASK(25, 16)</div><div class='del'>-</div><div class='del'>-struct hns_roce_pf_res_b {</div><div class='del'>-	__le32	rsv0;</div><div class='del'>-	__le32	smac_idx_num;</div><div class='del'>-	__le32	sgid_idx_num;</div><div class='del'>-	__le32	qid_idx_sl_num;</div><div class='del'>-	__le32	sccc_bt_idx_num;</div><div class='del'>-	__le32	rsv;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_1_PF_SMAC_IDX_S 0</div><div class='del'>-#define PF_RES_DATA_1_PF_SMAC_IDX_M GENMASK(7, 0)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_1_PF_SMAC_NUM_S 8</div><div class='del'>-#define PF_RES_DATA_1_PF_SMAC_NUM_M GENMASK(16, 8)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_2_PF_SGID_IDX_S 0</div><div class='del'>-#define PF_RES_DATA_2_PF_SGID_IDX_M GENMASK(7, 0)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_2_PF_SGID_NUM_S 8</div><div class='del'>-#define PF_RES_DATA_2_PF_SGID_NUM_M GENMASK(16, 8)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_3_PF_QID_IDX_S 0</div><div class='del'>-#define PF_RES_DATA_3_PF_QID_IDX_M GENMASK(9, 0)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_3_PF_SL_NUM_S 16</div><div class='del'>-#define PF_RES_DATA_3_PF_SL_NUM_M GENMASK(26, 16)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_4_PF_SCCC_BT_IDX_S 0</div><div class='del'>-#define PF_RES_DATA_4_PF_SCCC_BT_IDX_M GENMASK(8, 0)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_4_PF_SCCC_BT_NUM_S 9</div><div class='del'>-#define PF_RES_DATA_4_PF_SCCC_BT_NUM_M GENMASK(17, 9)</div><div class='del'>-</div><div class='del'>-struct hns_roce_pf_timer_res_a {</div><div class='del'>-	__le32	rsv0;</div><div class='del'>-	__le32	qpc_timer_bt_idx_num;</div><div class='del'>-	__le32	cqc_timer_bt_idx_num;</div><div class='del'>-	__le32	rsv[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_1_PF_QPC_TIMER_BT_IDX_S 0</div><div class='del'>-#define PF_RES_DATA_1_PF_QPC_TIMER_BT_IDX_M GENMASK(11, 0)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S 16</div><div class='del'>-#define PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M GENMASK(28, 16)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_2_PF_CQC_TIMER_BT_IDX_S 0</div><div class='del'>-#define PF_RES_DATA_2_PF_CQC_TIMER_BT_IDX_M GENMASK(10, 0)</div><div class='del'>-</div><div class='del'>-#define PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S 16</div><div class='del'>-#define PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M GENMASK(27, 16)</div><div class='del'>-</div><div class='del'>-struct hns_roce_vf_res_a {</div><div class='del'>-	__le32 vf_id;</div><div class='del'>-	__le32 vf_qpc_bt_idx_num;</div><div class='del'>-	__le32 vf_srqc_bt_idx_num;</div><div class='del'>-	__le32 vf_cqc_bt_idx_num;</div><div class='del'>-	__le32 vf_mpt_bt_idx_num;</div><div class='del'>-	__le32 vf_eqc_bt_idx_num;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define VF_RES_A_DATA_1_VF_QPC_BT_IDX_S 0</div><div class='del'>-#define VF_RES_A_DATA_1_VF_QPC_BT_IDX_M GENMASK(10, 0)</div><div class='del'>-</div><div class='del'>-#define VF_RES_A_DATA_1_VF_QPC_BT_NUM_S 16</div><div class='del'>-#define VF_RES_A_DATA_1_VF_QPC_BT_NUM_M GENMASK(27, 16)</div><div class='del'>-</div><div class='del'>-#define VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S 0</div><div class='del'>-#define VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M GENMASK(8, 0)</div><div class='del'>-</div><div class='del'>-#define VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S 16</div><div class='del'>-#define VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M GENMASK(25, 16)</div><div class='del'>-</div><div class='del'>-#define VF_RES_A_DATA_3_VF_CQC_BT_IDX_S 0</div><div class='del'>-#define VF_RES_A_DATA_3_VF_CQC_BT_IDX_M GENMASK(8, 0)</div><div class='del'>-</div><div class='del'>-#define VF_RES_A_DATA_3_VF_CQC_BT_NUM_S 16</div><div class='del'>-#define VF_RES_A_DATA_3_VF_CQC_BT_NUM_M GENMASK(25, 16)</div><div class='del'>-</div><div class='del'>-#define VF_RES_A_DATA_4_VF_MPT_BT_IDX_S 0</div><div class='del'>-#define VF_RES_A_DATA_4_VF_MPT_BT_IDX_M GENMASK(8, 0)</div><div class='del'>-</div><div class='del'>-#define VF_RES_A_DATA_4_VF_MPT_BT_NUM_S 16</div><div class='del'>-#define VF_RES_A_DATA_4_VF_MPT_BT_NUM_M GENMASK(25, 16)</div><div class='del'>-</div><div class='del'>-#define VF_RES_A_DATA_5_VF_EQC_IDX_S 0</div><div class='del'>-#define VF_RES_A_DATA_5_VF_EQC_IDX_M GENMASK(8, 0)</div><div class='del'>-</div><div class='del'>-#define VF_RES_A_DATA_5_VF_EQC_NUM_S 16</div><div class='del'>-#define VF_RES_A_DATA_5_VF_EQC_NUM_M GENMASK(25, 16)</div><div class='del'>-</div><div class='del'>-struct hns_roce_vf_res_b {</div><div class='del'>-	__le32 rsv0;</div><div class='del'>-	__le32 vf_smac_idx_num;</div><div class='del'>-	__le32 vf_sgid_idx_num;</div><div class='del'>-	__le32 vf_qid_idx_sl_num;</div><div class='del'>-	__le32 vf_sccc_idx_num;</div><div class='del'>-	__le32 rsv1;</div><div class='del'>-};</div><div class='add'>+#define FUNC_CLEAR_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_func_clear, h, l)</div><div class='ctx'> </div><div class='del'>-#define VF_RES_B_DATA_0_VF_ID_S 0</div><div class='del'>-#define VF_RES_B_DATA_0_VF_ID_M GENMASK(7, 0)</div><div class='add'>+#define FUNC_CLEAR_RST_FUN_DONE FUNC_CLEAR_FIELD_LOC(32, 32)</div><div class='ctx'> </div><div class='del'>-#define VF_RES_B_DATA_1_VF_SMAC_IDX_S 0</div><div class='del'>-#define VF_RES_B_DATA_1_VF_SMAC_IDX_M GENMASK(7, 0)</div><div class='del'>-</div><div class='del'>-#define VF_RES_B_DATA_1_VF_SMAC_NUM_S 8</div><div class='del'>-#define VF_RES_B_DATA_1_VF_SMAC_NUM_M GENMASK(16, 8)</div><div class='del'>-</div><div class='del'>-#define VF_RES_B_DATA_2_VF_SGID_IDX_S 0</div><div class='del'>-#define VF_RES_B_DATA_2_VF_SGID_IDX_M GENMASK(7, 0)</div><div class='del'>-</div><div class='del'>-#define VF_RES_B_DATA_2_VF_SGID_NUM_S 8</div><div class='del'>-#define VF_RES_B_DATA_2_VF_SGID_NUM_M GENMASK(16, 8)</div><div class='del'>-</div><div class='del'>-#define VF_RES_B_DATA_3_VF_QID_IDX_S 0</div><div class='del'>-#define VF_RES_B_DATA_3_VF_QID_IDX_M GENMASK(9, 0)</div><div class='del'>-</div><div class='del'>-#define VF_RES_B_DATA_3_VF_SL_NUM_S 16</div><div class='del'>-#define VF_RES_B_DATA_3_VF_SL_NUM_M GENMASK(19, 16)</div><div class='del'>-</div><div class='del'>-#define VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S 0</div><div class='del'>-#define VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M GENMASK(8, 0)</div><div class='add'>+/* Each physical function manages up to 248 virtual functions, it takes up to</div><div class='add'>+ * 100ms for each function to execute clear. If an abnormal reset occurs, it is</div><div class='add'>+ * executed twice at most, so it takes up to 249 * 2 * 100ms.</div><div class='add'>+ */</div><div class='add'>+#define HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS	(249 * 2 * 100)</div><div class='add'>+#define HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL	40</div><div class='add'>+#define HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT	20</div><div class='add'>+</div><div class='add'>+#define CFG_LLM_A_BA_L CMQ_REQ_FIELD_LOC(31, 0)</div><div class='add'>+#define CFG_LLM_A_BA_H CMQ_REQ_FIELD_LOC(63, 32)</div><div class='add'>+#define CFG_LLM_A_DEPTH CMQ_REQ_FIELD_LOC(76, 64)</div><div class='add'>+#define CFG_LLM_A_PGSZ CMQ_REQ_FIELD_LOC(83, 80)</div><div class='add'>+#define CFG_LLM_A_INIT_EN CMQ_REQ_FIELD_LOC(84, 84)</div><div class='add'>+#define CFG_LLM_A_HEAD_BA_L CMQ_REQ_FIELD_LOC(127, 96)</div><div class='add'>+#define CFG_LLM_A_HEAD_BA_H CMQ_REQ_FIELD_LOC(147, 128)</div><div class='add'>+#define CFG_LLM_A_HEAD_NXTPTR CMQ_REQ_FIELD_LOC(159, 148)</div><div class='add'>+#define CFG_LLM_A_HEAD_PTR CMQ_REQ_FIELD_LOC(171, 160)</div><div class='add'>+#define CFG_LLM_B_TAIL_BA_L CMQ_REQ_FIELD_LOC(31, 0)</div><div class='add'>+#define CFG_LLM_B_TAIL_BA_H CMQ_REQ_FIELD_LOC(63, 32)</div><div class='add'>+#define CFG_LLM_B_TAIL_PTR CMQ_REQ_FIELD_LOC(75, 64)</div><div class='add'>+</div><div class='add'>+/* Fields of HNS_ROCE_OPC_CFG_GLOBAL_PARAM */</div><div class='add'>+#define CFG_GLOBAL_PARAM_1US_CYCLES CMQ_REQ_FIELD_LOC(9, 0)</div><div class='add'>+#define CFG_GLOBAL_PARAM_UDP_PORT CMQ_REQ_FIELD_LOC(31, 16)</div><div class='ctx'> </div><div class='del'>-#define VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S 9</div><div class='del'>-#define VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M GENMASK(17, 9)</div><div class='add'>+/*</div><div class='add'>+ * Fields of HNS_ROCE_OPC_QUERY_PF_RES, HNS_ROCE_OPC_QUERY_VF_RES</div><div class='add'>+ * and HNS_ROCE_OPC_ALLOC_VF_RES</div><div class='add'>+ */</div><div class='add'>+#define FUNC_RES_A_VF_ID CMQ_REQ_FIELD_LOC(7, 0)</div><div class='add'>+#define FUNC_RES_A_QPC_BT_IDX CMQ_REQ_FIELD_LOC(42, 32)</div><div class='add'>+#define FUNC_RES_A_QPC_BT_NUM CMQ_REQ_FIELD_LOC(59, 48)</div><div class='add'>+#define FUNC_RES_A_SRQC_BT_IDX CMQ_REQ_FIELD_LOC(72, 64)</div><div class='add'>+#define FUNC_RES_A_SRQC_BT_NUM CMQ_REQ_FIELD_LOC(89, 80)</div><div class='add'>+#define FUNC_RES_A_CQC_BT_IDX CMQ_REQ_FIELD_LOC(104, 96)</div><div class='add'>+#define FUNC_RES_A_CQC_BT_NUM CMQ_REQ_FIELD_LOC(121, 112)</div><div class='add'>+#define FUNC_RES_A_MPT_BT_IDX CMQ_REQ_FIELD_LOC(136, 128)</div><div class='add'>+#define FUNC_RES_A_MPT_BT_NUM CMQ_REQ_FIELD_LOC(153, 144)</div><div class='add'>+#define FUNC_RES_A_EQC_BT_IDX CMQ_REQ_FIELD_LOC(168, 160)</div><div class='add'>+#define FUNC_RES_A_EQC_BT_NUM CMQ_REQ_FIELD_LOC(185, 176)</div><div class='add'>+#define FUNC_RES_B_SMAC_IDX CMQ_REQ_FIELD_LOC(39, 32)</div><div class='add'>+#define FUNC_RES_B_SMAC_NUM CMQ_REQ_FIELD_LOC(48, 40)</div><div class='add'>+#define FUNC_RES_B_SGID_IDX CMQ_REQ_FIELD_LOC(71, 64)</div><div class='add'>+#define FUNC_RES_B_SGID_NUM CMQ_REQ_FIELD_LOC(80, 72)</div><div class='add'>+#define FUNC_RES_B_QID_IDX CMQ_REQ_FIELD_LOC(105, 96)</div><div class='add'>+#define FUNC_RES_B_QID_NUM CMQ_REQ_FIELD_LOC(122, 112)</div><div class='add'>+#define FUNC_RES_V_QID_NUM CMQ_REQ_FIELD_LOC(115, 112)</div><div class='add'>+</div><div class='add'>+#define FUNC_RES_B_SCCC_BT_IDX CMQ_REQ_FIELD_LOC(136, 128)</div><div class='add'>+#define FUNC_RES_B_SCCC_BT_NUM CMQ_REQ_FIELD_LOC(145, 137)</div><div class='add'>+#define FUNC_RES_B_GMV_BT_IDX CMQ_REQ_FIELD_LOC(167, 160)</div><div class='add'>+#define FUNC_RES_B_GMV_BT_NUM CMQ_REQ_FIELD_LOC(176, 168)</div><div class='add'>+#define FUNC_RES_V_GMV_BT_NUM CMQ_REQ_FIELD_LOC(184, 176)</div><div class='add'>+</div><div class='add'>+/* Fields of HNS_ROCE_OPC_QUERY_PF_TIMER_RES */</div><div class='add'>+#define PF_TIMER_RES_QPC_ITEM_IDX CMQ_REQ_FIELD_LOC(43, 32)</div><div class='add'>+#define PF_TIMER_RES_QPC_ITEM_NUM CMQ_REQ_FIELD_LOC(60, 48)</div><div class='add'>+#define PF_TIMER_RES_CQC_ITEM_IDX CMQ_REQ_FIELD_LOC(74, 64)</div><div class='add'>+#define PF_TIMER_RES_CQC_ITEM_NUM CMQ_REQ_FIELD_LOC(91, 80)</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_vf_switch {</div><div class='ctx'> 	__le32 rocee_sel;</div><div class='hunk'>@@ -1446,12 +1047,12 @@ struct hns_roce_vf_switch {</div><div class='ctx'> 	__le32 resv3;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define VF_SWITCH_DATA_FUN_ID_VF_ID_S 3</div><div class='del'>-#define VF_SWITCH_DATA_FUN_ID_VF_ID_M GENMASK(10, 3)</div><div class='add'>+#define VF_SWITCH_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_vf_switch, h, l)</div><div class='ctx'> </div><div class='del'>-#define VF_SWITCH_DATA_CFG_ALW_LPBK_S 1</div><div class='del'>-#define VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S 2</div><div class='del'>-#define VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S 3</div><div class='add'>+#define VF_SWITCH_VF_ID VF_SWITCH_FIELD_LOC(42, 35)</div><div class='add'>+#define VF_SWITCH_ALW_LPBK VF_SWITCH_FIELD_LOC(65, 65)</div><div class='add'>+#define VF_SWITCH_ALW_LCL_LPBK VF_SWITCH_FIELD_LOC(66, 66)</div><div class='add'>+#define VF_SWITCH_ALW_DST_OVRD VF_SWITCH_FIELD_LOC(67, 67)</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_post_mbox {</div><div class='ctx'> 	__le32	in_param_l;</div><div class='hunk'>@@ -1467,59 +1068,48 @@ struct hns_roce_mbox_status {</div><div class='ctx'> 	__le32	rsv[5];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct hns_roce_cfg_bt_attr {</div><div class='del'>-	__le32 vf_qpc_cfg;</div><div class='del'>-	__le32 vf_srqc_cfg;</div><div class='del'>-	__le32 vf_cqc_cfg;</div><div class='del'>-	__le32 vf_mpt_cfg;</div><div class='del'>-	__le32 vf_sccc_cfg;</div><div class='del'>-	__le32 rsv;</div><div class='add'>+#define HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS 10000</div><div class='add'>+</div><div class='add'>+#define MB_ST_HW_RUN_M BIT(31)</div><div class='add'>+#define MB_ST_COMPLETE_M GENMASK(7, 0)</div><div class='add'>+</div><div class='add'>+#define MB_ST_COMPLETE_SUCC 1</div><div class='add'>+</div><div class='add'>+/* Fields of HNS_ROCE_OPC_CFG_BT_ATTR */</div><div class='add'>+#define CFG_BT_ATTR_QPC_BA_PGSZ CMQ_REQ_FIELD_LOC(3, 0)</div><div class='add'>+#define CFG_BT_ATTR_QPC_BUF_PGSZ CMQ_REQ_FIELD_LOC(7, 4)</div><div class='add'>+#define CFG_BT_ATTR_QPC_HOPNUM CMQ_REQ_FIELD_LOC(9, 8)</div><div class='add'>+#define CFG_BT_ATTR_SRQC_BA_PGSZ CMQ_REQ_FIELD_LOC(35, 32)</div><div class='add'>+#define CFG_BT_ATTR_SRQC_BUF_PGSZ CMQ_REQ_FIELD_LOC(39, 36)</div><div class='add'>+#define CFG_BT_ATTR_SRQC_HOPNUM CMQ_REQ_FIELD_LOC(41, 40)</div><div class='add'>+#define CFG_BT_ATTR_CQC_BA_PGSZ CMQ_REQ_FIELD_LOC(67, 64)</div><div class='add'>+#define CFG_BT_ATTR_CQC_BUF_PGSZ CMQ_REQ_FIELD_LOC(71, 68)</div><div class='add'>+#define CFG_BT_ATTR_CQC_HOPNUM CMQ_REQ_FIELD_LOC(73, 72)</div><div class='add'>+#define CFG_BT_ATTR_MPT_BA_PGSZ CMQ_REQ_FIELD_LOC(99, 96)</div><div class='add'>+#define CFG_BT_ATTR_MPT_BUF_PGSZ CMQ_REQ_FIELD_LOC(103, 100)</div><div class='add'>+#define CFG_BT_ATTR_MPT_HOPNUM CMQ_REQ_FIELD_LOC(105, 104)</div><div class='add'>+#define CFG_BT_ATTR_SCCC_BA_PGSZ CMQ_REQ_FIELD_LOC(131, 128)</div><div class='add'>+#define CFG_BT_ATTR_SCCC_BUF_PGSZ CMQ_REQ_FIELD_LOC(135, 132)</div><div class='add'>+#define CFG_BT_ATTR_SCCC_HOPNUM CMQ_REQ_FIELD_LOC(137, 136)</div><div class='add'>+</div><div class='add'>+/* Fields of HNS_ROCE_OPC_CFG_ENTRY_SIZE */</div><div class='add'>+#define CFG_HEM_ENTRY_SIZE_TYPE CMQ_REQ_FIELD_LOC(31, 0)</div><div class='add'>+enum {</div><div class='add'>+	HNS_ROCE_CFG_QPC_SIZE = BIT(0),</div><div class='add'>+	HNS_ROCE_CFG_SCCC_SIZE = BIT(1),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S 0</div><div class='del'>-#define CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M GENMASK(3, 0)</div><div class='del'>-</div><div class='del'>-#define CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S 4</div><div class='del'>-#define CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M GENMASK(7, 4)</div><div class='del'>-</div><div class='del'>-#define CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S 8</div><div class='del'>-#define CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M GENMASK(9, 8)</div><div class='del'>-</div><div class='del'>-#define CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S 0</div><div class='del'>-#define CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M GENMASK(3, 0)</div><div class='del'>-</div><div class='del'>-#define CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S 4</div><div class='del'>-#define CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M GENMASK(7, 4)</div><div class='add'>+#define CFG_HEM_ENTRY_SIZE_VALUE CMQ_REQ_FIELD_LOC(191, 160)</div><div class='ctx'> </div><div class='del'>-#define CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S 8</div><div class='del'>-#define CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M GENMASK(9, 8)</div><div class='add'>+/* Fields of HNS_ROCE_OPC_CFG_GMV_BT */</div><div class='add'>+#define CFG_GMV_BT_BA_L CMQ_REQ_FIELD_LOC(31, 0)</div><div class='add'>+#define CFG_GMV_BT_BA_H CMQ_REQ_FIELD_LOC(51, 32)</div><div class='add'>+#define CFG_GMV_BT_IDX CMQ_REQ_FIELD_LOC(95, 64)</div><div class='ctx'> </div><div class='del'>-#define CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S 0</div><div class='del'>-#define CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M GENMASK(3, 0)</div><div class='del'>-</div><div class='del'>-#define CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S 4</div><div class='del'>-#define CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M GENMASK(7, 4)</div><div class='del'>-</div><div class='del'>-#define CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S 8</div><div class='del'>-#define CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M GENMASK(9, 8)</div><div class='del'>-</div><div class='del'>-#define CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S 0</div><div class='del'>-#define CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M GENMASK(3, 0)</div><div class='del'>-</div><div class='del'>-#define CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S 4</div><div class='del'>-#define CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M GENMASK(7, 4)</div><div class='del'>-</div><div class='del'>-#define CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S 8</div><div class='del'>-#define CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M GENMASK(9, 8)</div><div class='del'>-</div><div class='del'>-#define CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S 0</div><div class='del'>-#define CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M GENMASK(3, 0)</div><div class='del'>-</div><div class='del'>-#define CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S 4</div><div class='del'>-#define CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M GENMASK(7, 4)</div><div class='del'>-</div><div class='del'>-#define CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S 8</div><div class='del'>-#define CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M GENMASK(9, 8)</div><div class='add'>+/* Fields of HNS_ROCE_QUERY_RAM_ECC */</div><div class='add'>+#define QUERY_RAM_ECC_1BIT_ERR CMQ_REQ_FIELD_LOC(31, 0)</div><div class='add'>+#define QUERY_RAM_ECC_RES_TYPE CMQ_REQ_FIELD_LOC(63, 32)</div><div class='add'>+#define QUERY_RAM_ECC_TAG CMQ_REQ_FIELD_LOC(95, 64)</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_cfg_sgid_tb {</div><div class='ctx'> 	__le32	table_idx_rsv;</div><div class='hunk'>@@ -1529,11 +1119,11 @@ struct hns_roce_cfg_sgid_tb {</div><div class='ctx'> 	__le32	vf_sgid_h;</div><div class='ctx'> 	__le32	vf_sgid_type_rsv;</div><div class='ctx'> };</div><div class='del'>-#define CFG_SGID_TB_TABLE_IDX_S 0</div><div class='del'>-#define CFG_SGID_TB_TABLE_IDX_M GENMASK(7, 0)</div><div class='ctx'> </div><div class='del'>-#define CFG_SGID_TB_VF_SGID_TYPE_S 0</div><div class='del'>-#define CFG_SGID_TB_VF_SGID_TYPE_M GENMASK(1, 0)</div><div class='add'>+#define SGID_TB_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_cfg_sgid_tb, h, l)</div><div class='add'>+</div><div class='add'>+#define CFG_SGID_TB_TABLE_IDX SGID_TB_FIELD_LOC(7, 0)</div><div class='add'>+#define CFG_SGID_TB_VF_SGID_TYPE SGID_TB_FIELD_LOC(161, 160)</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_cfg_smac_tb {</div><div class='ctx'> 	__le32	tb_idx_rsv;</div><div class='hunk'>@@ -1541,90 +1131,245 @@ struct hns_roce_cfg_smac_tb {</div><div class='ctx'> 	__le32	vf_smac_h_rsv;</div><div class='ctx'> 	__le32	rsv[3];</div><div class='ctx'> };</div><div class='del'>-#define CFG_SMAC_TB_IDX_S 0</div><div class='del'>-#define CFG_SMAC_TB_IDX_M GENMASK(7, 0)</div><div class='ctx'> </div><div class='del'>-#define CFG_SMAC_TB_VF_SMAC_H_S 0</div><div class='del'>-#define CFG_SMAC_TB_VF_SMAC_H_M GENMASK(15, 0)</div><div class='add'>+#define SMAC_TB_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_cfg_smac_tb, h, l)</div><div class='add'>+</div><div class='add'>+#define CFG_SMAC_TB_IDX SMAC_TB_FIELD_LOC(7, 0)</div><div class='add'>+#define CFG_SMAC_TB_VF_SMAC_H SMAC_TB_FIELD_LOC(79, 64)</div><div class='add'>+</div><div class='add'>+struct hns_roce_cfg_gmv_tb_a {</div><div class='add'>+	__le32 vf_sgid_l;</div><div class='add'>+	__le32 vf_sgid_ml;</div><div class='add'>+	__le32 vf_sgid_mh;</div><div class='add'>+	__le32 vf_sgid_h;</div><div class='add'>+	__le32 vf_sgid_type_vlan;</div><div class='add'>+	__le32 resv;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define GMV_TB_A_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_cfg_gmv_tb_a, h, l)</div><div class='add'>+</div><div class='add'>+#define GMV_TB_A_VF_SGID_TYPE GMV_TB_A_FIELD_LOC(129, 128)</div><div class='add'>+#define GMV_TB_A_VF_VLAN_EN GMV_TB_A_FIELD_LOC(130, 130)</div><div class='add'>+#define GMV_TB_A_VF_VLAN_ID GMV_TB_A_FIELD_LOC(155, 144)</div><div class='add'>+</div><div class='add'>+struct hns_roce_cfg_gmv_tb_b {</div><div class='add'>+	__le32	vf_smac_l;</div><div class='add'>+	__le32	vf_smac_h;</div><div class='add'>+	__le32	table_idx_rsv;</div><div class='add'>+	__le32	resv[3];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define GMV_TB_B_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_cfg_gmv_tb_b, h, l)</div><div class='add'>+</div><div class='add'>+#define GMV_TB_B_SMAC_H GMV_TB_B_FIELD_LOC(47, 32)</div><div class='add'>+#define GMV_TB_B_SGID_IDX GMV_TB_B_FIELD_LOC(71, 64)</div><div class='add'>+</div><div class='add'>+#define HNS_ROCE_QUERY_PF_CAPS_CMD_NUM_HIP08 5</div><div class='add'>+#define HNS_ROCE_QUERY_PF_CAPS_CMD_NUM 6</div><div class='add'>+struct hns_roce_query_pf_caps_a {</div><div class='add'>+	u8 number_ports;</div><div class='add'>+	u8 local_ca_ack_delay;</div><div class='add'>+	__le16 max_sq_sg;</div><div class='add'>+	__le16 max_sq_inline;</div><div class='add'>+	__le16 max_rq_sg;</div><div class='add'>+	__le32 rsv0;</div><div class='add'>+	__le16 num_qpc_timer;</div><div class='add'>+	__le16 num_cqc_timer;</div><div class='add'>+	__le16 max_srq_sges;</div><div class='add'>+	u8 num_aeq_vectors;</div><div class='add'>+	u8 num_other_vectors;</div><div class='add'>+	u8 max_sq_desc_sz;</div><div class='add'>+	u8 max_rq_desc_sz;</div><div class='add'>+	u8 rsv1;</div><div class='add'>+	u8 cqe_sz;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct hns_roce_query_pf_caps_b {</div><div class='add'>+	u8 mtpt_entry_sz;</div><div class='add'>+	u8 irrl_entry_sz;</div><div class='add'>+	u8 trrl_entry_sz;</div><div class='add'>+	u8 cqc_entry_sz;</div><div class='add'>+	u8 srqc_entry_sz;</div><div class='add'>+	u8 idx_entry_sz;</div><div class='add'>+	u8 sccc_sz;</div><div class='add'>+	u8 max_mtu;</div><div class='add'>+	__le16 qpc_sz;</div><div class='add'>+	__le16 qpc_timer_entry_sz;</div><div class='add'>+	__le16 cqc_timer_entry_sz;</div><div class='add'>+	u8 min_cqes;</div><div class='add'>+	u8 min_wqes;</div><div class='add'>+	__le32 page_size_cap;</div><div class='add'>+	u8 pkey_table_len;</div><div class='add'>+	u8 phy_num_uars;</div><div class='add'>+	u8 ctx_hop_num;</div><div class='add'>+	u8 pbl_hop_num;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct hns_roce_query_pf_caps_c {</div><div class='add'>+	__le32 cap_flags_num_pds;</div><div class='add'>+	__le32 max_gid_num_cqs;</div><div class='add'>+	__le32 cq_depth;</div><div class='add'>+	__le32 num_mrws;</div><div class='add'>+	__le32 ord_num_qps;</div><div class='add'>+	__le16 sq_depth;</div><div class='add'>+	__le16 rq_depth;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define PF_CAPS_C_FIELD_LOC(h, l) \</div><div class='add'>+	FIELD_LOC(struct hns_roce_query_pf_caps_c, h, l)</div><div class='add'>+</div><div class='add'>+#define PF_CAPS_C_NUM_PDS PF_CAPS_C_FIELD_LOC(19, 0)</div><div class='add'>+#define PF_CAPS_C_CAP_FLAGS PF_CAPS_C_FIELD_LOC(31, 20)</div><div class='add'>+#define PF_CAPS_C_NUM_CQS PF_CAPS_C_FIELD_LOC(51, 32)</div><div class='add'>+#define PF_CAPS_C_MAX_GID PF_CAPS_C_FIELD_LOC(60, 52)</div><div class='add'>+#define PF_CAPS_C_CQ_DEPTH PF_CAPS_C_FIELD_LOC(86, 64)</div><div class='add'>+#define PF_CAPS_C_NUM_XRCDS PF_CAPS_C_FIELD_LOC(91, 87)</div><div class='add'>+#define PF_CAPS_C_NUM_MRWS PF_CAPS_C_FIELD_LOC(115, 96)</div><div class='add'>+#define PF_CAPS_C_NUM_QPS PF_CAPS_C_FIELD_LOC(147, 128)</div><div class='add'>+#define PF_CAPS_C_MAX_ORD PF_CAPS_C_FIELD_LOC(155, 148)</div><div class='add'>+</div><div class='add'>+struct hns_roce_query_pf_caps_d {</div><div class='add'>+	__le32 wq_hop_num_max_srqs;</div><div class='add'>+	__le16 srq_depth;</div><div class='add'>+	__le16 cap_flags_ex;</div><div class='add'>+	__le32 num_ceqs_ceq_depth;</div><div class='add'>+	__le32 arm_st_aeq_depth;</div><div class='add'>+	__le32 num_uars_rsv_pds;</div><div class='add'>+	__le32 rsv_uars_rsv_qps;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define PF_CAPS_D_FIELD_LOC(h, l) \</div><div class='add'>+	FIELD_LOC(struct hns_roce_query_pf_caps_d, h, l)</div><div class='add'>+</div><div class='add'>+#define PF_CAPS_D_NUM_SRQS PF_CAPS_D_FIELD_LOC(19, 0)</div><div class='add'>+#define PF_CAPS_D_RQWQE_HOP_NUM PF_CAPS_D_FIELD_LOC(21, 20)</div><div class='add'>+#define PF_CAPS_D_EX_SGE_HOP_NUM PF_CAPS_D_FIELD_LOC(23, 22)</div><div class='add'>+#define PF_CAPS_D_SQWQE_HOP_NUM PF_CAPS_D_FIELD_LOC(25, 24)</div><div class='add'>+#define PF_CAPS_D_CONG_CAP PF_CAPS_D_FIELD_LOC(29, 26)</div><div class='add'>+#define PF_CAPS_D_CEQ_DEPTH PF_CAPS_D_FIELD_LOC(85, 64)</div><div class='add'>+#define PF_CAPS_D_NUM_CEQS PF_CAPS_D_FIELD_LOC(95, 86)</div><div class='add'>+#define PF_CAPS_D_AEQ_DEPTH PF_CAPS_D_FIELD_LOC(117, 96)</div><div class='add'>+#define PF_CAPS_D_AEQ_ARM_ST PF_CAPS_D_FIELD_LOC(119, 118)</div><div class='add'>+#define PF_CAPS_D_CEQ_ARM_ST PF_CAPS_D_FIELD_LOC(121, 120)</div><div class='add'>+#define PF_CAPS_D_DEFAULT_ALG PF_CAPS_D_FIELD_LOC(127, 122)</div><div class='add'>+#define PF_CAPS_D_RSV_PDS PF_CAPS_D_FIELD_LOC(147, 128)</div><div class='add'>+#define PF_CAPS_D_NUM_UARS PF_CAPS_D_FIELD_LOC(155, 148)</div><div class='add'>+#define PF_CAPS_D_RSV_QPS PF_CAPS_D_FIELD_LOC(179, 160)</div><div class='add'>+#define PF_CAPS_D_RSV_UARS PF_CAPS_D_FIELD_LOC(187, 180)</div><div class='add'>+</div><div class='add'>+#define HNS_ROCE_CAP_FLAGS_EX_SHIFT 12</div><div class='add'>+</div><div class='add'>+struct hns_roce_congestion_algorithm {</div><div class='add'>+	u8 alg_sel;</div><div class='add'>+	u8 alg_sub_sel;</div><div class='add'>+	u8 dip_vld;</div><div class='add'>+	u8 wnd_mode_sel;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct hns_roce_query_pf_caps_e {</div><div class='add'>+	__le32 chunk_size_shift_rsv_mrws;</div><div class='add'>+	__le32 rsv_cqs;</div><div class='add'>+	__le32 rsv_srqs;</div><div class='add'>+	__le32 rsv_lkey;</div><div class='add'>+	__le16 ceq_max_cnt;</div><div class='add'>+	__le16 ceq_period;</div><div class='add'>+	__le16 aeq_max_cnt;</div><div class='add'>+	__le16 aeq_period;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct hns_roce_query_pf_caps_f {</div><div class='add'>+	__le32 max_ack_req_msg_len;</div><div class='add'>+	__le32 rsv[5];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define PF_CAPS_E_FIELD_LOC(h, l) \</div><div class='add'>+	FIELD_LOC(struct hns_roce_query_pf_caps_e, h, l)</div><div class='add'>+</div><div class='add'>+#define PF_CAPS_E_RSV_MRWS PF_CAPS_E_FIELD_LOC(19, 0)</div><div class='add'>+#define PF_CAPS_E_CHUNK_SIZE_SHIFT PF_CAPS_E_FIELD_LOC(31, 20)</div><div class='add'>+#define PF_CAPS_E_RSV_CQS PF_CAPS_E_FIELD_LOC(51, 32)</div><div class='add'>+#define PF_CAPS_E_RSV_XRCDS PF_CAPS_E_FIELD_LOC(63, 52)</div><div class='add'>+#define PF_CAPS_E_RSV_SRQS PF_CAPS_E_FIELD_LOC(83, 64)</div><div class='add'>+#define PF_CAPS_E_RSV_LKEYS PF_CAPS_E_FIELD_LOC(115, 96)</div><div class='add'>+</div><div class='add'>+struct hns_roce_cmq_req {</div><div class='add'>+	__le32 data[6];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define CMQ_REQ_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_cmq_req, h, l)</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_cmq_desc {</div><div class='ctx'> 	__le16 opcode;</div><div class='ctx'> 	__le16 flag;</div><div class='ctx'> 	__le16 retval;</div><div class='ctx'> 	__le16 rsv;</div><div class='del'>-	__le32 data[6];</div><div class='add'>+	union {</div><div class='add'>+		__le32 data[6];</div><div class='add'>+		struct {</div><div class='add'>+			__le32 own_func_num;</div><div class='add'>+			__le32 own_mac_id;</div><div class='add'>+			__le32 rsv[4];</div><div class='add'>+		} func_info;</div><div class='add'>+	};</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS	10000</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_HW_RUN_BIT_SHIFT	31</div><div class='del'>-#define HNS_ROCE_HW_MB_STATUS_MASK	0xFF</div><div class='del'>-</div><div class='ctx'> struct hns_roce_v2_cmq_ring {</div><div class='ctx'> 	dma_addr_t desc_dma_addr;</div><div class='ctx'> 	struct hns_roce_cmq_desc *desc;</div><div class='ctx'> 	u32 head;</div><div class='del'>-	u32 tail;</div><div class='del'>-</div><div class='ctx'> 	u16 buf_size;</div><div class='ctx'> 	u16 desc_num;</div><div class='del'>-	int next_to_use;</div><div class='del'>-	int next_to_clean;</div><div class='ctx'> 	u8 flag;</div><div class='ctx'> 	spinlock_t lock; /* command queue lock */</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_v2_cmq {</div><div class='ctx'> 	struct hns_roce_v2_cmq_ring csq;</div><div class='del'>-	struct hns_roce_v2_cmq_ring crq;</div><div class='ctx'> 	u16 tx_timeout;</div><div class='del'>-	u16 last_status;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum hns_roce_link_table_type {</div><div class='del'>-	TSQ_LINK_TABLE,</div><div class='del'>-	TPQ_LINK_TABLE,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_link_table {</div><div class='ctx'> 	struct hns_roce_buf_list table;</div><div class='del'>-	struct hns_roce_buf_list *pg_list;</div><div class='del'>-	u32 npages;</div><div class='del'>-	u32 pg_sz;</div><div class='add'>+	struct hns_roce_buf *buf;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct hns_roce_link_table_entry {</div><div class='del'>-	u32 blk_ba0;</div><div class='del'>-	u32 blk_ba1_nxt_ptr;</div><div class='del'>-};</div><div class='del'>-#define HNS_ROCE_LINK_TABLE_BA1_S 0</div><div class='del'>-#define HNS_ROCE_LINK_TABLE_BA1_M GENMASK(19, 0)</div><div class='add'>+#define HNS_ROCE_EXT_LLM_ENTRY(addr, id) (((id) &lt;&lt; (64 - 12)) | ((addr) &gt;&gt; 12))</div><div class='add'>+#define HNS_ROCE_EXT_LLM_MIN_PAGES(que_num) ((que_num) * 4 + 2)</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_LINK_TABLE_NXT_PTR_S 20</div><div class='del'>-#define HNS_ROCE_LINK_TABLE_NXT_PTR_M GENMASK(31, 20)</div><div class='add'>+struct hns_roce_v2_free_mr {</div><div class='add'>+	struct hns_roce_qp *rsv_qp[HNS_ROCE_FREE_MR_USED_QP_NUM];</div><div class='add'>+	struct hns_roce_cq *rsv_cq;</div><div class='add'>+	struct hns_roce_pd *rsv_pd;</div><div class='add'>+	struct mutex mutex;</div><div class='add'>+};</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_v2_priv {</div><div class='ctx'> 	struct hnae3_handle *handle;</div><div class='ctx'> 	struct hns_roce_v2_cmq cmq;</div><div class='del'>-	struct hns_roce_link_table tsq;</div><div class='del'>-	struct hns_roce_link_table tpq;</div><div class='add'>+	struct hns_roce_link_table ext_llm;</div><div class='add'>+	struct hns_roce_v2_free_mr free_mr;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct hns_roce_eq_context {</div><div class='del'>-	__le32	byte_4;</div><div class='del'>-	__le32	byte_8;</div><div class='del'>-	__le32	byte_12;</div><div class='del'>-	__le32	eqe_report_timer;</div><div class='del'>-	__le32	eqe_ba0;</div><div class='del'>-	__le32	eqe_ba1;</div><div class='del'>-	__le32	byte_28;</div><div class='del'>-	__le32	byte_32;</div><div class='del'>-	__le32	byte_36;</div><div class='del'>-	__le32	nxt_eqe_ba0;</div><div class='del'>-	__le32	nxt_eqe_ba1;</div><div class='del'>-	__le32	rsv[5];</div><div class='add'>+struct hns_roce_dip {</div><div class='add'>+	u8 dgid[GID_LEN_V2];</div><div class='add'>+	u32 dip_idx;</div><div class='add'>+	u32 qp_cnt;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct fmea_ram_ecc {</div><div class='add'>+	u32	is_ecc_err;</div><div class='add'>+	u32	res_type;</div><div class='add'>+	u32	index;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* only for RNR timeout issue of HIP08 */</div><div class='add'>+#define HNS_ROCE_CLOCK_ADJUST 1000</div><div class='add'>+#define HNS_ROCE_MAX_CQ_PERIOD_HIP08 65</div><div class='add'>+#define HNS_ROCE_MAX_EQ_PERIOD 65</div><div class='add'>+#define HNS_ROCE_RNR_TIMER_10NS 1</div><div class='add'>+#define HNS_ROCE_1US_CFG 999</div><div class='add'>+#define HNS_ROCE_1NS_CFG 0</div><div class='add'>+</div><div class='ctx'> #define HNS_ROCE_AEQ_DEFAULT_BURST_NUM	0x0</div><div class='ctx'> #define HNS_ROCE_AEQ_DEFAULT_INTERVAL	0x0</div><div class='ctx'> #define HNS_ROCE_CEQ_DEFAULT_BURST_NUM	0x0</div><div class='hunk'>@@ -1652,15 +1397,10 @@ struct hns_roce_eq_context {</div><div class='ctx'> #define HNS_ROCE_EQ_INIT_CONS_IDX		0</div><div class='ctx'> #define HNS_ROCE_EQ_INIT_NXT_EQE_BA		0</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_V2_CEQ_CEQE_OWNER_S		31</div><div class='del'>-#define HNS_ROCE_V2_AEQ_AEQE_OWNER_S		31</div><div class='del'>-</div><div class='ctx'> #define HNS_ROCE_V2_COMP_EQE_NUM		0x1000</div><div class='ctx'> #define HNS_ROCE_V2_ASYNC_EQE_NUM		0x1000</div><div class='ctx'> </div><div class='ctx'> #define HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S	0</div><div class='del'>-#define HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S		1</div><div class='del'>-#define HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S	2</div><div class='ctx'> </div><div class='ctx'> #define HNS_ROCE_EQ_DB_CMD_AEQ			0x0</div><div class='ctx'> #define HNS_ROCE_EQ_DB_CMD_AEQ_ARMED		0x1</div><div class='hunk'>@@ -1675,114 +1415,44 @@ struct hns_roce_eq_context {</div><div class='ctx'> #define HNS_ROCE_INT_NAME_LEN			32</div><div class='ctx'> #define HNS_ROCE_V2_EQN_M GENMASK(23, 0)</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_V2_CONS_IDX_M GENMASK(23, 0)</div><div class='del'>-</div><div class='ctx'> #define HNS_ROCE_V2_VF_ABN_INT_EN_S 0</div><div class='ctx'> #define HNS_ROCE_V2_VF_ABN_INT_EN_M GENMASK(0, 0)</div><div class='ctx'> #define HNS_ROCE_V2_VF_ABN_INT_ST_M GENMASK(2, 0)</div><div class='ctx'> #define HNS_ROCE_V2_VF_ABN_INT_CFG_M GENMASK(2, 0)</div><div class='ctx'> #define HNS_ROCE_V2_VF_EVENT_INT_EN_M GENMASK(0, 0)</div><div class='ctx'> </div><div class='del'>-/* WORD0 */</div><div class='del'>-#define HNS_ROCE_EQC_EQ_ST_S 0</div><div class='del'>-#define HNS_ROCE_EQC_EQ_ST_M GENMASK(1, 0)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_EQC_HOP_NUM_S 2</div><div class='del'>-#define HNS_ROCE_EQC_HOP_NUM_M GENMASK(3, 2)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_EQC_OVER_IGNORE_S 4</div><div class='del'>-#define HNS_ROCE_EQC_OVER_IGNORE_M GENMASK(4, 4)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_EQC_COALESCE_S 5</div><div class='del'>-#define HNS_ROCE_EQC_COALESCE_M GENMASK(5, 5)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_EQC_ARM_ST_S 6</div><div class='del'>-#define HNS_ROCE_EQC_ARM_ST_M GENMASK(7, 6)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_EQC_EQN_S 8</div><div class='del'>-#define HNS_ROCE_EQC_EQN_M GENMASK(15, 8)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_EQC_EQE_CNT_S 16</div><div class='del'>-#define HNS_ROCE_EQC_EQE_CNT_M GENMASK(31, 16)</div><div class='del'>-</div><div class='del'>-/* WORD1 */</div><div class='del'>-#define HNS_ROCE_EQC_BA_PG_SZ_S 0</div><div class='del'>-#define HNS_ROCE_EQC_BA_PG_SZ_M GENMASK(3, 0)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_EQC_BUF_PG_SZ_S 4</div><div class='del'>-#define HNS_ROCE_EQC_BUF_PG_SZ_M GENMASK(7, 4)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_EQC_PROD_INDX_S 8</div><div class='del'>-#define HNS_ROCE_EQC_PROD_INDX_M GENMASK(31, 8)</div><div class='del'>-</div><div class='del'>-/* WORD2 */</div><div class='del'>-#define HNS_ROCE_EQC_MAX_CNT_S 0</div><div class='del'>-#define HNS_ROCE_EQC_MAX_CNT_M GENMASK(15, 0)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_EQC_PERIOD_S 16</div><div class='del'>-#define HNS_ROCE_EQC_PERIOD_M GENMASK(31, 16)</div><div class='del'>-</div><div class='del'>-/* WORD3 */</div><div class='del'>-#define HNS_ROCE_EQC_REPORT_TIMER_S 0</div><div class='del'>-#define HNS_ROCE_EQC_REPORT_TIMER_M GENMASK(31, 0)</div><div class='del'>-</div><div class='del'>-/* WORD4 */</div><div class='del'>-#define HNS_ROCE_EQC_EQE_BA_L_S 0</div><div class='del'>-#define HNS_ROCE_EQC_EQE_BA_L_M GENMASK(31, 0)</div><div class='del'>-</div><div class='del'>-/* WORD5 */</div><div class='del'>-#define HNS_ROCE_EQC_EQE_BA_H_S 0</div><div class='del'>-#define HNS_ROCE_EQC_EQE_BA_H_M GENMASK(28, 0)</div><div class='del'>-</div><div class='del'>-/* WORD6 */</div><div class='del'>-#define HNS_ROCE_EQC_SHIFT_S 0</div><div class='del'>-#define HNS_ROCE_EQC_SHIFT_M GENMASK(7, 0)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_EQC_MSI_INDX_S 8</div><div class='del'>-#define HNS_ROCE_EQC_MSI_INDX_M GENMASK(15, 8)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_EQC_CUR_EQE_BA_L_S 16</div><div class='del'>-#define HNS_ROCE_EQC_CUR_EQE_BA_L_M GENMASK(31, 16)</div><div class='del'>-</div><div class='del'>-/* WORD7 */</div><div class='del'>-#define HNS_ROCE_EQC_CUR_EQE_BA_M_S 0</div><div class='del'>-#define HNS_ROCE_EQC_CUR_EQE_BA_M_M GENMASK(31, 0)</div><div class='del'>-</div><div class='del'>-/* WORD8 */</div><div class='del'>-#define HNS_ROCE_EQC_CUR_EQE_BA_H_S 0</div><div class='del'>-#define HNS_ROCE_EQC_CUR_EQE_BA_H_M GENMASK(3, 0)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_EQC_CONS_INDX_S 8</div><div class='del'>-#define HNS_ROCE_EQC_CONS_INDX_M GENMASK(31, 8)</div><div class='del'>-</div><div class='del'>-/* WORD9 */</div><div class='del'>-#define HNS_ROCE_EQC_NXT_EQE_BA_L_S 0</div><div class='del'>-#define HNS_ROCE_EQC_NXT_EQE_BA_L_M GENMASK(31, 0)</div><div class='del'>-</div><div class='del'>-/* WORD10 */</div><div class='del'>-#define HNS_ROCE_EQC_NXT_EQE_BA_H_S 0</div><div class='del'>-#define HNS_ROCE_EQC_NXT_EQE_BA_H_M GENMASK(19, 0)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V2_CEQE_COMP_CQN_S 0</div><div class='del'>-#define HNS_ROCE_V2_CEQE_COMP_CQN_M GENMASK(23, 0)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V2_AEQE_EVENT_TYPE_S 0</div><div class='del'>-#define HNS_ROCE_V2_AEQE_EVENT_TYPE_M GENMASK(7, 0)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V2_AEQE_SUB_TYPE_S 8</div><div class='del'>-#define HNS_ROCE_V2_AEQE_SUB_TYPE_M GENMASK(15, 8)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V2_EQ_DB_CMD_S	16</div><div class='del'>-#define HNS_ROCE_V2_EQ_DB_CMD_M	GENMASK(17, 16)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V2_EQ_DB_TAG_S	0</div><div class='del'>-#define HNS_ROCE_V2_EQ_DB_TAG_M	GENMASK(7, 0)</div><div class='del'>-</div><div class='del'>-#define HNS_ROCE_V2_EQ_DB_PARA_S 0</div><div class='del'>-#define HNS_ROCE_V2_EQ_DB_PARA_M GENMASK(23, 0)</div><div class='add'>+struct hns_roce_eq_context {</div><div class='add'>+	__le32	data[16];</div><div class='add'>+};</div><div class='ctx'> </div><div class='del'>-#define HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S 0</div><div class='del'>-#define HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M GENMASK(23, 0)</div><div class='add'>+#define EQC_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_eq_context, h, l)</div><div class='add'>+</div><div class='add'>+#define EQC_EQ_ST EQC_FIELD_LOC(1, 0)</div><div class='add'>+#define EQC_EQE_HOP_NUM EQC_FIELD_LOC(3, 2)</div><div class='add'>+#define EQC_OVER_IGNORE EQC_FIELD_LOC(4, 4)</div><div class='add'>+#define EQC_COALESCE EQC_FIELD_LOC(5, 5)</div><div class='add'>+#define EQC_ARM_ST EQC_FIELD_LOC(7, 6)</div><div class='add'>+#define EQC_EQN EQC_FIELD_LOC(15, 8)</div><div class='add'>+#define EQC_EQE_CNT EQC_FIELD_LOC(31, 16)</div><div class='add'>+#define EQC_EQE_BA_PG_SZ EQC_FIELD_LOC(35, 32)</div><div class='add'>+#define EQC_EQE_BUF_PG_SZ EQC_FIELD_LOC(39, 36)</div><div class='add'>+#define EQC_EQ_PROD_INDX EQC_FIELD_LOC(63, 40)</div><div class='add'>+#define EQC_EQ_MAX_CNT EQC_FIELD_LOC(79, 64)</div><div class='add'>+#define EQC_EQ_PERIOD EQC_FIELD_LOC(95, 80)</div><div class='add'>+#define EQC_EQE_REPORT_TIMER EQC_FIELD_LOC(127, 96)</div><div class='add'>+#define EQC_EQE_BA_L EQC_FIELD_LOC(159, 128)</div><div class='add'>+#define EQC_EQE_BA_H EQC_FIELD_LOC(188, 160)</div><div class='add'>+#define EQC_SHIFT EQC_FIELD_LOC(199, 192)</div><div class='add'>+#define EQC_MSI_INDX EQC_FIELD_LOC(207, 200)</div><div class='add'>+#define EQC_CUR_EQE_BA_L EQC_FIELD_LOC(223, 208)</div><div class='add'>+#define EQC_CUR_EQE_BA_M EQC_FIELD_LOC(255, 224)</div><div class='add'>+#define EQC_CUR_EQE_BA_H EQC_FIELD_LOC(259, 256)</div><div class='add'>+#define EQC_EQ_CONS_INDX EQC_FIELD_LOC(287, 264)</div><div class='add'>+#define EQC_NEX_EQE_BA_L EQC_FIELD_LOC(319, 288)</div><div class='add'>+#define EQC_NEX_EQE_BA_H EQC_FIELD_LOC(339, 320)</div><div class='add'>+#define EQC_EQE_SIZE EQC_FIELD_LOC(341, 340)</div><div class='add'>+</div><div class='add'>+#define MAX_SERVICE_LEVEL 0x7</div><div class='ctx'> </div><div class='ctx'> struct hns_roce_wqe_atomic_seg {</div><div class='ctx'> 	__le64          fetchadd_swap_data;</div><div class='hunk'>@@ -1799,13 +1469,28 @@ struct hns_roce_sccc_clr_done {</div><div class='ctx'> 	__le32 rsv[5];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,</div><div class='del'>-			       int *buffer);</div><div class='add'>+struct hns_roce_bond_info {</div><div class='add'>+	__le32 bond_id;</div><div class='add'>+	__le32 bond_mode;</div><div class='add'>+	__le32 active_slave_cnt;</div><div class='add'>+	__le32 active_slave_mask;</div><div class='add'>+	__le32 slave_mask;</div><div class='add'>+	__le32 hash_policy;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct hns_roce_dev</div><div class='add'>+	*hns_roce_bond_init_client(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+				   int func_idx);</div><div class='add'>+void hns_roce_bond_uninit_client(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+				 int func_idx);</div><div class='add'>+int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);</div><div class='add'>+int hns_roce_cmd_bond(struct hns_roce_bond_group *bond_grp,</div><div class='add'>+		      enum hns_roce_bond_cmd_type bond_type);</div><div class='ctx'> </div><div class='ctx'> static inline void hns_roce_write64(struct hns_roce_dev *hr_dev, __le32 val[2],</div><div class='ctx'> 				    void __iomem *dest)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev-&gt;priv;</div><div class='add'>+	struct hns_roce_v2_priv *priv = hr_dev-&gt;priv;</div><div class='ctx'> 	struct hnae3_handle *handle = priv-&gt;handle;</div><div class='ctx'> 	const struct hnae3_ae_ops *ops = handle-&gt;ae_algo-&gt;ops;</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c<br/>deleted file mode 100644<br/>index 5a97b5a0b7be..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,35 +0,0 @@</div><div class='del'>-// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)</div><div class='del'>-// Copyright (c) 2019 Hisilicon Limited.</div><div class='del'>-</div><div class='del'>-#include "hnae3.h"</div><div class='del'>-#include "hns_roce_device.h"</div><div class='del'>-#include "hns_roce_cmd.h"</div><div class='del'>-#include "hns_roce_hw_v2.h"</div><div class='del'>-</div><div class='del'>-int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,</div><div class='del'>-			       int *buffer)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_v2_cq_context *cq_context;</div><div class='del'>-	struct hns_roce_cmd_mailbox *mailbox;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='del'>-	if (IS_ERR(mailbox))</div><div class='del'>-		return PTR_ERR(mailbox);</div><div class='del'>-</div><div class='del'>-	cq_context = mailbox-&gt;buf;</div><div class='del'>-	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox-&gt;dma, cqn, 0,</div><div class='del'>-				HNS_ROCE_CMD_QUERY_CQC,</div><div class='del'>-				HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "QUERY cqc cmd process error\n");</div><div class='del'>-		goto err_mailbox;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	memcpy(buffer, cq_context, sizeof(*cq_context));</div><div class='del'>-</div><div class='del'>-err_mailbox:</div><div class='del'>-	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c<br/>index 8da5f18bf820..2f4864ab7d4e 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_main.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_main.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_main.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_main.c</a></div><div class='hunk'>@@ -31,7 +31,6 @@</div><div class='ctx'>  * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> #include &lt;linux/acpi.h&gt;</div><div class='del'>-#include &lt;linux/of_platform.h&gt;</div><div class='ctx'> #include &lt;linux/module.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_addr.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_smi.h&gt;</div><div class='hunk'>@@ -39,35 +38,23 @@</div><div class='ctx'> #include &lt;rdma/ib_cache.h&gt;</div><div class='ctx'> #include "hns_roce_common.h"</div><div class='ctx'> #include "hns_roce_device.h"</div><div class='del'>-#include &lt;rdma/hns-abi.h&gt;</div><div class='ctx'> #include "hns_roce_hem.h"</div><div class='add'>+#include "hns_roce_hw_v2.h"</div><div class='add'>+#include "hns_roce_bond.h"</div><div class='ctx'> </div><div class='del'>-/**</div><div class='del'>- * hns_get_gid_index - Get gid index.</div><div class='del'>- * @hr_dev: pointer to structure hns_roce_dev.</div><div class='del'>- * @port:  port, value range: 0 ~ MAX</div><div class='del'>- * @gid_index:  gid_index, value range: 0 ~ MAX</div><div class='del'>- * Description:</div><div class='del'>- *    N ports shared gids, allocation method as follow:</div><div class='del'>- *		GID[0][0], GID[1][0],.....GID[N - 1][0],</div><div class='del'>- *		GID[0][0], GID[1][0],.....GID[N - 1][0],</div><div class='del'>- *		And so on</div><div class='del'>- */</div><div class='del'>-int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index)</div><div class='del'>-{</div><div class='del'>-	return gid_index * hr_dev-&gt;caps.num_ports + port;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_get_gid_index);</div><div class='del'>-</div><div class='del'>-static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr)</div><div class='add'>+static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u32 port,</div><div class='add'>+			    const u8 *addr)</div><div class='ctx'> {</div><div class='ctx'> 	u8 phy_port;</div><div class='del'>-	u32 i = 0;</div><div class='add'>+	u32 i;</div><div class='ctx'> </div><div class='del'>-	if (!memcmp(hr_dev-&gt;dev_addr[port], addr, MAC_ADDR_OCTET_NUM))</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09)</div><div class='ctx'> 		return 0;</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; MAC_ADDR_OCTET_NUM; i++)</div><div class='add'>+	if (!memcmp(hr_dev-&gt;dev_addr[port], addr, ETH_ALEN))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ETH_ALEN; i++)</div><div class='ctx'> 		hr_dev-&gt;dev_addr[port][i] = addr[i];</div><div class='ctx'> </div><div class='ctx'> 	phy_port = hr_dev-&gt;iboe.phy_port[port];</div><div class='hunk'>@@ -77,18 +64,13 @@ static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr)</div><div class='ctx'> static int hns_roce_add_gid(const struct ib_gid_attr *attr, void **context)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(attr-&gt;device);</div><div class='del'>-	u8 port = attr-&gt;port_num - 1;</div><div class='del'>-	unsigned long flags;</div><div class='add'>+	u32 port = attr-&gt;port_num - 1;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (port &gt;= hr_dev-&gt;caps.num_ports)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	spin_lock_irqsave(&amp;hr_dev-&gt;iboe.lock, flags);</div><div class='del'>-</div><div class='del'>-	ret = hr_dev-&gt;hw-&gt;set_gid(hr_dev, port, attr-&gt;index, &amp;attr-&gt;gid, attr);</div><div class='del'>-</div><div class='del'>-	spin_unlock_irqrestore(&amp;hr_dev-&gt;iboe.lock, flags);</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;set_gid(hr_dev, attr-&gt;index, &amp;attr-&gt;gid, attr);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -96,47 +78,86 @@ static int hns_roce_add_gid(const struct ib_gid_attr *attr, void **context)</div><div class='ctx'> static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(attr-&gt;device);</div><div class='del'>-	struct ib_gid_attr zattr = { };</div><div class='del'>-	u8 port = attr-&gt;port_num - 1;</div><div class='del'>-	unsigned long flags;</div><div class='add'>+	u32 port = attr-&gt;port_num - 1;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (port &gt;= hr_dev-&gt;caps.num_ports)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	spin_lock_irqsave(&amp;hr_dev-&gt;iboe.lock, flags);</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;set_gid(hr_dev, attr-&gt;index, NULL, NULL);</div><div class='ctx'> </div><div class='del'>-	ret = hr_dev-&gt;hw-&gt;set_gid(hr_dev, port, attr-&gt;index, &amp;zgid, &amp;zattr);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	spin_unlock_irqrestore(&amp;hr_dev-&gt;iboe.lock, flags);</div><div class='add'>+static int hns_roce_get_port_state(struct hns_roce_dev *hr_dev, u32 port_num,</div><div class='add'>+				   enum ib_port_state *state)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_bond_group *bond_grp;</div><div class='add'>+	u8 bus_num = get_hr_bus_num(hr_dev);</div><div class='add'>+	struct net_device *net_dev;</div><div class='ctx'> </div><div class='del'>-	return ret;</div><div class='add'>+	net_dev = ib_device_get_netdev(&amp;hr_dev-&gt;ib_dev, port_num);</div><div class='add'>+	if (!net_dev)</div><div class='add'>+		return -ENODEV;</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_BOND) {</div><div class='add'>+		bond_grp = hns_roce_get_bond_grp(net_dev, bus_num);</div><div class='add'>+		if (bond_grp) {</div><div class='add'>+			*state = ib_get_curr_port_state(bond_grp-&gt;upper_dev);</div><div class='add'>+			goto out;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*state = ib_get_curr_port_state(net_dev);</div><div class='add'>+out:</div><div class='add'>+	dev_put(net_dev);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port,</div><div class='del'>-			   unsigned long event)</div><div class='add'>+static int handle_en_event(struct net_device *netdev,</div><div class='add'>+			   struct hns_roce_dev *hr_dev,</div><div class='add'>+			   u32 port, unsigned long event)</div><div class='ctx'> {</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='ctx'> 	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	struct net_device *netdev;</div><div class='add'>+	enum ib_port_state curr_state;</div><div class='add'>+	struct ib_event ibevent;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	netdev = hr_dev-&gt;iboe.netdevs[port];</div><div class='ctx'> 	if (!netdev) {</div><div class='del'>-		dev_err(dev, "port(%d) can't find netdev\n", port);</div><div class='add'>+		dev_err(dev, "can't find netdev on port(%u)!\n", port);</div><div class='ctx'> 		return -ENODEV;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	switch (event) {</div><div class='del'>-	case NETDEV_UP:</div><div class='del'>-	case NETDEV_CHANGE:</div><div class='ctx'> 	case NETDEV_REGISTER:</div><div class='ctx'> 	case NETDEV_CHANGEADDR:</div><div class='ctx'> 		ret = hns_roce_set_mac(hr_dev, port, netdev-&gt;dev_addr);</div><div class='ctx'> 		break;</div><div class='add'>+	case NETDEV_UP:</div><div class='add'>+	case NETDEV_CHANGE:</div><div class='add'>+		ret = hns_roce_set_mac(hr_dev, port, netdev-&gt;dev_addr);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case NETDEV_DOWN:</div><div class='del'>-		/*</div><div class='del'>-		 * In v1 engine, only support all ports closed together.</div><div class='del'>-		 */</div><div class='add'>+		if (!netif_is_lag_master(netdev))</div><div class='add'>+			break;</div><div class='add'>+		curr_state = ib_get_curr_port_state(netdev);</div><div class='add'>+</div><div class='add'>+		write_lock_irq(&amp;ibdev-&gt;cache_lock);</div><div class='add'>+		if (ibdev-&gt;port_data[port].cache.last_port_state == curr_state) {</div><div class='add'>+			write_unlock_irq(&amp;ibdev-&gt;cache_lock);</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+		ibdev-&gt;port_data[port].cache.last_port_state = curr_state;</div><div class='add'>+		write_unlock_irq(&amp;ibdev-&gt;cache_lock);</div><div class='add'>+</div><div class='add'>+		ibevent.event = (curr_state == IB_PORT_DOWN) ?</div><div class='add'>+				IB_EVENT_PORT_ERR : IB_EVENT_PORT_ACTIVE;</div><div class='add'>+		ibevent.device = ibdev;</div><div class='add'>+		ibevent.element.port_num = port + 1;</div><div class='add'>+		ib_dispatch_event(&amp;ibevent);</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		dev_dbg(dev, "NETDEV event = 0x%x!\n", (u32)(event));</div><div class='hunk'>@@ -150,17 +171,25 @@ static int hns_roce_netdev_event(struct notifier_block *self,</div><div class='ctx'> 				 unsigned long event, void *ptr)</div><div class='ctx'> {</div><div class='ctx'> 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);</div><div class='add'>+	struct hns_roce_bond_group *bond_grp;</div><div class='ctx'> 	struct hns_roce_ib_iboe *iboe = NULL;</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = NULL;</div><div class='del'>-	u8 port = 0;</div><div class='del'>-	int ret = 0;</div><div class='add'>+	struct net_device *upper = NULL;</div><div class='add'>+	int ret;</div><div class='add'>+	u32 port;</div><div class='ctx'> </div><div class='ctx'> 	hr_dev = container_of(self, struct hns_roce_dev, iboe.nb);</div><div class='ctx'> 	iboe = &amp;hr_dev-&gt;iboe;</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_BOND) {</div><div class='add'>+		bond_grp = hns_roce_get_bond_grp(get_hr_netdev(hr_dev, 0),</div><div class='add'>+						 get_hr_bus_num(hr_dev));</div><div class='add'>+		upper = bond_grp ? bond_grp-&gt;upper_dev : NULL;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	for (port = 0; port &lt; hr_dev-&gt;caps.num_ports; port++) {</div><div class='del'>-		if (dev == iboe-&gt;netdevs[port]) {</div><div class='del'>-			ret = handle_en_event(hr_dev, port, event);</div><div class='add'>+		if ((!upper &amp;&amp; dev == iboe-&gt;netdevs[port]) ||</div><div class='add'>+		    (upper &amp;&amp; dev == upper)) {</div><div class='add'>+			ret = handle_en_event(dev, hr_dev, port, event);</div><div class='ctx'> 			if (ret)</div><div class='ctx'> 				return NOTIFY_DONE;</div><div class='ctx'> 			break;</div><div class='hunk'>@@ -172,15 +201,13 @@ static int hns_roce_netdev_event(struct notifier_block *self,</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_setup_mtu_mac(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='add'>+	struct net_device *net_dev;</div><div class='ctx'> 	int ret;</div><div class='ctx'> 	u8 i;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; hr_dev-&gt;caps.num_ports; i++) {</div><div class='del'>-		if (hr_dev-&gt;hw-&gt;set_mtu)</div><div class='del'>-			hr_dev-&gt;hw-&gt;set_mtu(hr_dev, hr_dev-&gt;iboe.phy_port[i],</div><div class='del'>-					    hr_dev-&gt;caps.max_mtu);</div><div class='del'>-		ret = hns_roce_set_mac(hr_dev, i,</div><div class='del'>-				       hr_dev-&gt;iboe.netdevs[i]-&gt;dev_addr);</div><div class='add'>+		net_dev = get_hr_netdev(hr_dev, i);</div><div class='add'>+		ret = hns_roce_set_mac(hr_dev, i, net_dev-&gt;dev_addr);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			return ret;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -209,7 +236,7 @@ static int hns_roce_query_device(struct ib_device *ib_dev,</div><div class='ctx'> 				  IB_DEVICE_RC_RNR_NAK_GEN;</div><div class='ctx'> 	props-&gt;max_send_sge = hr_dev-&gt;caps.max_sq_sg;</div><div class='ctx'> 	props-&gt;max_recv_sge = hr_dev-&gt;caps.max_rq_sg;</div><div class='del'>-	props-&gt;max_sge_rd = 1;</div><div class='add'>+	props-&gt;max_sge_rd = hr_dev-&gt;caps.max_sq_sg;</div><div class='ctx'> 	props-&gt;max_cq = hr_dev-&gt;caps.num_cqs;</div><div class='ctx'> 	props-&gt;max_cqe = hr_dev-&gt;caps.max_cqes;</div><div class='ctx'> 	props-&gt;max_mr = hr_dev-&gt;caps.num_mtpts;</div><div class='hunk'>@@ -220,31 +247,39 @@ static int hns_roce_query_device(struct ib_device *ib_dev,</div><div class='ctx'> 			    IB_ATOMIC_HCA : IB_ATOMIC_NONE;</div><div class='ctx'> 	props-&gt;max_pkeys = 1;</div><div class='ctx'> 	props-&gt;local_ca_ack_delay = hr_dev-&gt;caps.local_ca_ack_delay;</div><div class='add'>+	props-&gt;max_ah = INT_MAX;</div><div class='add'>+	props-&gt;cq_caps.max_cq_moderation_period = HNS_ROCE_MAX_CQ_PERIOD;</div><div class='add'>+	props-&gt;cq_caps.max_cq_moderation_count = HNS_ROCE_MAX_CQ_COUNT;</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08)</div><div class='add'>+		props-&gt;cq_caps.max_cq_moderation_period = HNS_ROCE_MAX_CQ_PERIOD_HIP08;</div><div class='add'>+</div><div class='ctx'> 	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_SRQ) {</div><div class='del'>-		props-&gt;max_srq = hr_dev-&gt;caps.max_srqs;</div><div class='add'>+		props-&gt;max_srq = hr_dev-&gt;caps.num_srqs;</div><div class='ctx'> 		props-&gt;max_srq_wr = hr_dev-&gt;caps.max_srq_wrs;</div><div class='ctx'> 		props-&gt;max_srq_sge = hr_dev-&gt;caps.max_srq_sges;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_FRMR) {</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_FRMR &amp;&amp;</div><div class='add'>+	    hr_dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09) {</div><div class='ctx'> 		props-&gt;device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;</div><div class='ctx'> 		props-&gt;max_fast_reg_page_list_len = HNS_ROCE_FRMR_MAX_PA;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_XRC)</div><div class='add'>+		props-&gt;device_cap_flags |= IB_DEVICE_XRC;</div><div class='add'>+</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num,</div><div class='add'>+static int hns_roce_query_port(struct ib_device *ib_dev, u32 port_num,</div><div class='ctx'> 			       struct ib_port_attr *props)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='ctx'> 	struct net_device *net_dev;</div><div class='del'>-	unsigned long flags;</div><div class='ctx'> 	enum ib_mtu mtu;</div><div class='del'>-	u8 port;</div><div class='add'>+	u32 port;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	assert(port_num &gt; 0);</div><div class='ctx'> 	port = port_num - 1;</div><div class='ctx'> </div><div class='ctx'> 	/* props being zeroed by the caller, avoid zeroing it here */</div><div class='hunk'>@@ -256,38 +291,46 @@ static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num,</div><div class='ctx'> 				IB_PORT_BOOT_MGMT_SUP;</div><div class='ctx'> 	props-&gt;max_msg_sz = HNS_ROCE_MAX_MSG_LEN;</div><div class='ctx'> 	props-&gt;pkey_tbl_len = 1;</div><div class='del'>-	props-&gt;active_width = IB_WIDTH_4X;</div><div class='del'>-	props-&gt;active_speed = 1;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;hr_dev-&gt;iboe.lock, flags);</div><div class='add'>+	ret = ib_get_eth_speed(ib_dev, port_num, &amp;props-&gt;active_speed,</div><div class='add'>+			       &amp;props-&gt;active_width);</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_warn(ib_dev, "failed to get speed, ret = %d.\n", ret);</div><div class='ctx'> </div><div class='del'>-	net_dev = hr_dev-&gt;iboe.netdevs[port];</div><div class='add'>+	net_dev = ib_device_get_netdev(ib_dev, port_num);</div><div class='ctx'> 	if (!net_dev) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;hr_dev-&gt;iboe.lock, flags);</div><div class='del'>-		dev_err(dev, "find netdev %d failed!\r\n", port);</div><div class='add'>+		ibdev_err(ib_dev, "find netdev %u failed!\n", port);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	mtu = iboe_get_mtu(net_dev-&gt;mtu);</div><div class='ctx'> 	props-&gt;active_mtu = mtu ? min(props-&gt;max_mtu, mtu) : IB_MTU_256;</div><div class='del'>-	props-&gt;state = (netif_running(net_dev) &amp;&amp; netif_carrier_ok(net_dev)) ?</div><div class='del'>-			IB_PORT_ACTIVE : IB_PORT_DOWN;</div><div class='del'>-	props-&gt;phys_state = (props-&gt;state == IB_PORT_ACTIVE) ? 5 : 3;</div><div class='ctx'> </div><div class='del'>-	spin_unlock_irqrestore(&amp;hr_dev-&gt;iboe.lock, flags);</div><div class='add'>+	dev_put(net_dev);</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_get_port_state(hr_dev, port_num, &amp;props-&gt;state);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ib_dev, "failed to get port state.\n");</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='add'>+	props-&gt;phys_state = props-&gt;state == IB_PORT_ACTIVE ?</div><div class='add'>+				    IB_PORT_PHYS_STATE_LINK_UP :</div><div class='add'>+				    IB_PORT_PHYS_STATE_DISABLED;</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static enum rdma_link_layer hns_roce_get_link_layer(struct ib_device *device,</div><div class='del'>-						    u8 port_num)</div><div class='add'>+						    u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	return IB_LINK_LAYER_ETHERNET;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_query_pkey(struct ib_device *ib_dev, u8 port, u16 index,</div><div class='add'>+static int hns_roce_query_pkey(struct ib_device *ib_dev, u32 port, u16 index,</div><div class='ctx'> 			       u16 *pkey)</div><div class='ctx'> {</div><div class='add'>+	if (index &gt; 0)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='ctx'> 	*pkey = PKEY_ID;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='hunk'>@@ -310,85 +353,217 @@ static int hns_roce_modify_device(struct ib_device *ib_dev, int mask,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_modify_port(struct ib_device *ib_dev, u8 port_num, int mask,</div><div class='del'>-				struct ib_port_modify *props)</div><div class='add'>+struct hns_user_mmap_entry *</div><div class='add'>+hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,</div><div class='add'>+				size_t length,</div><div class='add'>+				enum hns_roce_mmap_type mmap_type)</div><div class='add'>+{</div><div class='add'>+	struct hns_user_mmap_entry *entry;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	entry = kzalloc(sizeof(*entry), GFP_KERNEL);</div><div class='add'>+	if (!entry)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	entry-&gt;address = address;</div><div class='add'>+	entry-&gt;mmap_type = mmap_type;</div><div class='add'>+</div><div class='add'>+	switch (mmap_type) {</div><div class='add'>+	/* pgoff 0 must be used by DB for compatibility */</div><div class='add'>+	case HNS_ROCE_MMAP_TYPE_DB:</div><div class='add'>+		ret = rdma_user_mmap_entry_insert_exact(</div><div class='add'>+				ucontext, &amp;entry-&gt;rdma_entry, length, 0);</div><div class='add'>+		break;</div><div class='add'>+	case HNS_ROCE_MMAP_TYPE_DWQE:</div><div class='add'>+		ret = rdma_user_mmap_entry_insert_range(</div><div class='add'>+				ucontext, &amp;entry-&gt;rdma_entry, length, 1,</div><div class='add'>+				U32_MAX);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ret) {</div><div class='add'>+		kfree(entry);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return entry;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_dealloc_uar_entry(struct hns_roce_ucontext *context)</div><div class='ctx'> {</div><div class='add'>+	if (context-&gt;db_mmap_entry)</div><div class='add'>+		rdma_user_mmap_entry_remove(</div><div class='add'>+			&amp;context-&gt;db_mmap_entry-&gt;rdma_entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_alloc_uar_entry(struct ib_ucontext *uctx)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_ucontext *context = to_hr_ucontext(uctx);</div><div class='add'>+	u64 address;</div><div class='add'>+</div><div class='add'>+	address = context-&gt;uar.pfn &lt;&lt; PAGE_SHIFT;</div><div class='add'>+	context-&gt;db_mmap_entry = hns_roce_user_mmap_entry_insert(</div><div class='add'>+		uctx, address, PAGE_SIZE, HNS_ROCE_MMAP_TYPE_DB);</div><div class='add'>+	if (!context-&gt;db_mmap_entry)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx,</div><div class='ctx'> 				   struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	int ret = 0;</div><div class='ctx'> 	struct hns_roce_ucontext *context = to_hr_ucontext(uctx);</div><div class='del'>-	struct hns_roce_ib_alloc_ucontext_resp resp = {};</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(uctx-&gt;device);</div><div class='add'>+	struct hns_roce_ib_alloc_ucontext_resp resp = {};</div><div class='add'>+	struct hns_roce_ib_alloc_ucontext ucmd = {};</div><div class='add'>+	int ret = -EAGAIN;</div><div class='ctx'> </div><div class='ctx'> 	if (!hr_dev-&gt;active)</div><div class='del'>-		return -EAGAIN;</div><div class='add'>+		goto error_out;</div><div class='ctx'> </div><div class='ctx'> 	resp.qp_tab_size = hr_dev-&gt;caps.num_qps;</div><div class='add'>+	resp.srq_tab_size = hr_dev-&gt;caps.num_srqs;</div><div class='add'>+</div><div class='add'>+	ret = ib_copy_from_udata(&amp;ucmd, udata,</div><div class='add'>+				 min(udata-&gt;inlen, sizeof(ucmd)));</div><div class='add'>+	if (ret)</div><div class='add'>+		goto error_out;</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09)</div><div class='add'>+		context-&gt;config = ucmd.config &amp; HNS_ROCE_EXSGE_FLAGS;</div><div class='add'>+</div><div class='add'>+	if (context-&gt;config &amp; HNS_ROCE_EXSGE_FLAGS) {</div><div class='add'>+		resp.config |= HNS_ROCE_RSP_EXSGE_FLAGS;</div><div class='add'>+		resp.max_inline_data = hr_dev-&gt;caps.max_sq_inline;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RQ_INLINE) {</div><div class='add'>+		context-&gt;config |= ucmd.config &amp; HNS_ROCE_RQ_INLINE_FLAGS;</div><div class='add'>+		if (context-&gt;config &amp; HNS_ROCE_RQ_INLINE_FLAGS)</div><div class='add'>+			resp.config |= HNS_ROCE_RSP_RQ_INLINE_FLAGS;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_CQE_INLINE) {</div><div class='add'>+		context-&gt;config |= ucmd.config &amp; HNS_ROCE_CQE_INLINE_FLAGS;</div><div class='add'>+		if (context-&gt;config &amp; HNS_ROCE_CQE_INLINE_FLAGS)</div><div class='add'>+			resp.config |= HNS_ROCE_RSP_CQE_INLINE_FLAGS;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09)</div><div class='add'>+		resp.congest_type = hr_dev-&gt;caps.cong_cap;</div><div class='ctx'> </div><div class='ctx'> 	ret = hns_roce_uar_alloc(hr_dev, &amp;context-&gt;uar);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto error_fail_uar_alloc;</div><div class='add'>+		goto error_out;</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_alloc_uar_entry(uctx);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto error_fail_uar_entry;</div><div class='ctx'> </div><div class='del'>-	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RECORD_DB) {</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_CQ_RECORD_DB ||</div><div class='add'>+	    hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_QP_RECORD_DB) {</div><div class='ctx'> 		INIT_LIST_HEAD(&amp;context-&gt;page_list);</div><div class='ctx'> 		mutex_init(&amp;context-&gt;page_mutex);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = ib_copy_to_udata(udata, &amp;resp, sizeof(resp));</div><div class='add'>+	resp.cqe_size = hr_dev-&gt;caps.cqe_sz;</div><div class='add'>+</div><div class='add'>+	ret = ib_copy_to_udata(udata, &amp;resp,</div><div class='add'>+			       min(udata-&gt;outlen, sizeof(resp)));</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto error_fail_copy_to_udata;</div><div class='ctx'> </div><div class='add'>+	hns_roce_get_cq_bankid_for_uctx(context);</div><div class='add'>+</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> error_fail_copy_to_udata:</div><div class='del'>-	hns_roce_uar_free(hr_dev, &amp;context-&gt;uar);</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_CQ_RECORD_DB ||</div><div class='add'>+	    hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_QP_RECORD_DB)</div><div class='add'>+		mutex_destroy(&amp;context-&gt;page_mutex);</div><div class='add'>+	hns_roce_dealloc_uar_entry(context);</div><div class='add'>+</div><div class='add'>+error_fail_uar_entry:</div><div class='add'>+	ida_free(&amp;hr_dev-&gt;uar_ida.ida, (int)context-&gt;uar.logic_idx);</div><div class='add'>+</div><div class='add'>+error_out:</div><div class='add'>+	atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_UCTX_ALLOC_ERR_CNT]);</div><div class='ctx'> </div><div class='del'>-error_fail_uar_alloc:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void hns_roce_dealloc_ucontext(struct ib_ucontext *ibcontext)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_ucontext *context = to_hr_ucontext(ibcontext);</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ibcontext-&gt;device);</div><div class='add'>+</div><div class='add'>+	hns_roce_put_cq_bankid_for_uctx(context);</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_CQ_RECORD_DB ||</div><div class='add'>+	    hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_QP_RECORD_DB)</div><div class='add'>+		mutex_destroy(&amp;context-&gt;page_mutex);</div><div class='ctx'> </div><div class='del'>-	hns_roce_uar_free(to_hr_dev(ibcontext-&gt;device), &amp;context-&gt;uar);</div><div class='add'>+	hns_roce_dealloc_uar_entry(context);</div><div class='add'>+</div><div class='add'>+	ida_free(&amp;hr_dev-&gt;uar_ida.ida, (int)context-&gt;uar.logic_idx);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_mmap(struct ib_ucontext *context,</div><div class='del'>-			 struct vm_area_struct *vma)</div><div class='add'>+static int hns_roce_mmap(struct ib_ucontext *uctx, struct vm_area_struct *vma)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(context-&gt;device);</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(uctx-&gt;device);</div><div class='add'>+	struct rdma_user_mmap_entry *rdma_entry;</div><div class='add'>+	struct hns_user_mmap_entry *entry;</div><div class='add'>+	phys_addr_t pfn;</div><div class='add'>+	pgprot_t prot;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	switch (vma-&gt;vm_pgoff) {</div><div class='del'>-	case 0:</div><div class='del'>-		return rdma_user_mmap_io(context, vma,</div><div class='del'>-					 to_hr_ucontext(context)-&gt;uar.pfn,</div><div class='del'>-					 PAGE_SIZE,</div><div class='del'>-					 pgprot_noncached(vma-&gt;vm_page_prot));</div><div class='add'>+	if (hr_dev-&gt;dis_db) {</div><div class='add'>+		atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_MMAP_ERR_CNT]);</div><div class='add'>+		return -EPERM;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	/* vm_pgoff: 1 -- TPTR */</div><div class='del'>-	case 1:</div><div class='del'>-		if (!hr_dev-&gt;tptr_dma_addr || !hr_dev-&gt;tptr_size)</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		/*</div><div class='del'>-		 * FIXME: using io_remap_pfn_range on the dma address returned</div><div class='del'>-		 * by dma_alloc_coherent is totally wrong.</div><div class='del'>-		 */</div><div class='del'>-		return rdma_user_mmap_io(context, vma,</div><div class='del'>-					 hr_dev-&gt;tptr_dma_addr &gt;&gt; PAGE_SHIFT,</div><div class='del'>-					 hr_dev-&gt;tptr_size,</div><div class='del'>-					 vma-&gt;vm_page_prot);</div><div class='add'>+	rdma_entry = rdma_user_mmap_entry_get_pgoff(uctx, vma-&gt;vm_pgoff);</div><div class='add'>+	if (!rdma_entry) {</div><div class='add'>+		atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_MMAP_ERR_CNT]);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	entry = to_hns_mmap(rdma_entry);</div><div class='add'>+	pfn = entry-&gt;address &gt;&gt; PAGE_SHIFT;</div><div class='ctx'> </div><div class='add'>+	switch (entry-&gt;mmap_type) {</div><div class='add'>+	case HNS_ROCE_MMAP_TYPE_DB:</div><div class='add'>+	case HNS_ROCE_MMAP_TYPE_DWQE:</div><div class='add'>+		prot = pgprot_device(vma-&gt;vm_page_prot);</div><div class='add'>+		break;</div><div class='ctx'> 	default:</div><div class='del'>-		return -EINVAL;</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto out;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	ret = rdma_user_mmap_io(uctx, vma, pfn, rdma_entry-&gt;npages * PAGE_SIZE,</div><div class='add'>+				prot, rdma_entry);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	rdma_user_mmap_entry_put(rdma_entry);</div><div class='add'>+	if (ret)</div><div class='add'>+		atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_MMAP_ERR_CNT]);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_free_mmap(struct rdma_user_mmap_entry *rdma_entry)</div><div class='add'>+{</div><div class='add'>+	struct hns_user_mmap_entry *entry = to_hns_mmap(rdma_entry);</div><div class='add'>+</div><div class='add'>+	kfree(entry);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_port_immutable(struct ib_device *ib_dev, u8 port_num,</div><div class='add'>+static int hns_roce_port_immutable(struct ib_device *ib_dev, u32 port_num,</div><div class='ctx'> 				   struct ib_port_immutable *immutable)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr attr;</div><div class='hunk'>@@ -413,9 +588,121 @@ static void hns_roce_disassociate_ucontext(struct ib_ucontext *ibcontext)</div><div class='ctx'> {</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev)</div><div class='add'>+static void hns_roce_get_fw_ver(struct ib_device *device, char *str)</div><div class='ctx'> {</div><div class='add'>+	u64 fw_ver = to_hr_dev(device)-&gt;caps.fw_ver;</div><div class='add'>+	unsigned int major, minor, sub_minor;</div><div class='add'>+</div><div class='add'>+	major = upper_32_bits(fw_ver);</div><div class='add'>+	minor = high_16_bits(lower_32_bits(fw_ver));</div><div class='add'>+	sub_minor = low_16_bits(fw_ver);</div><div class='add'>+</div><div class='add'>+	snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u.%04u", major, minor,</div><div class='add'>+		 sub_minor);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define HNS_ROCE_HW_CNT(ename, cname) \</div><div class='add'>+	[HNS_ROCE_HW_##ename##_CNT].name = cname</div><div class='add'>+</div><div class='add'>+static const struct rdma_stat_desc hns_roce_port_stats_descs[] = {</div><div class='add'>+	HNS_ROCE_HW_CNT(RX_RC_PKT, "rx_rc_pkt"),</div><div class='add'>+	HNS_ROCE_HW_CNT(RX_UC_PKT, "rx_uc_pkt"),</div><div class='add'>+	HNS_ROCE_HW_CNT(RX_UD_PKT, "rx_ud_pkt"),</div><div class='add'>+	HNS_ROCE_HW_CNT(RX_XRC_PKT, "rx_xrc_pkt"),</div><div class='add'>+	HNS_ROCE_HW_CNT(RX_PKT, "rx_pkt"),</div><div class='add'>+	HNS_ROCE_HW_CNT(RX_ERR_PKT, "rx_err_pkt"),</div><div class='add'>+	HNS_ROCE_HW_CNT(RX_CNP_PKT, "rx_cnp_pkt"),</div><div class='add'>+	HNS_ROCE_HW_CNT(TX_RC_PKT, "tx_rc_pkt"),</div><div class='add'>+	HNS_ROCE_HW_CNT(TX_UC_PKT, "tx_uc_pkt"),</div><div class='add'>+	HNS_ROCE_HW_CNT(TX_UD_PKT, "tx_ud_pkt"),</div><div class='add'>+	HNS_ROCE_HW_CNT(TX_XRC_PKT, "tx_xrc_pkt"),</div><div class='add'>+	HNS_ROCE_HW_CNT(TX_PKT, "tx_pkt"),</div><div class='add'>+	HNS_ROCE_HW_CNT(TX_ERR_PKT, "tx_err_pkt"),</div><div class='add'>+	HNS_ROCE_HW_CNT(TX_CNP_PKT, "tx_cnp_pkt"),</div><div class='add'>+	HNS_ROCE_HW_CNT(TRP_GET_MPT_ERR_PKT, "trp_get_mpt_err_pkt"),</div><div class='add'>+	HNS_ROCE_HW_CNT(TRP_GET_IRRL_ERR_PKT, "trp_get_irrl_err_pkt"),</div><div class='add'>+	HNS_ROCE_HW_CNT(ECN_DB, "ecn_doorbell"),</div><div class='add'>+	HNS_ROCE_HW_CNT(RX_BUF, "rx_buffer"),</div><div class='add'>+	HNS_ROCE_HW_CNT(TRP_RX_SOF, "trp_rx_sof"),</div><div class='add'>+	HNS_ROCE_HW_CNT(CQ_CQE, "cq_cqe"),</div><div class='add'>+	HNS_ROCE_HW_CNT(CQ_POE, "cq_poe"),</div><div class='add'>+	HNS_ROCE_HW_CNT(CQ_NOTIFY, "cq_notify"),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static struct rdma_hw_stats *hns_roce_alloc_hw_port_stats(</div><div class='add'>+				struct ib_device *device, u32 port_num)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(device);</div><div class='add'>+</div><div class='add'>+	if (port_num &gt; hr_dev-&gt;caps.num_ports) {</div><div class='add'>+		ibdev_err(device, "invalid port num.\n");</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return rdma_alloc_hw_stats_struct(hns_roce_port_stats_descs,</div><div class='add'>+					  ARRAY_SIZE(hns_roce_port_stats_descs),</div><div class='add'>+					  RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_get_hw_stats(struct ib_device *device,</div><div class='add'>+				 struct rdma_hw_stats *stats,</div><div class='add'>+				 u32 port, int index)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(device);</div><div class='add'>+	int num_counters = HNS_ROCE_HW_CNT_TOTAL;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (port == 0)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (port &gt; hr_dev-&gt;caps.num_ports)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;query_hw_counter(hr_dev, stats-&gt;value, port,</div><div class='add'>+					   &amp;num_counters);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(device, "failed to query hw counter, ret = %d\n",</div><div class='add'>+			  ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return num_counters;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void</div><div class='add'>+	hns_roce_unregister_bond_cleanup(struct hns_roce_dev *hr_dev,</div><div class='add'>+					 struct hns_roce_bond_group *bond_grp)</div><div class='add'>+{</div><div class='add'>+	struct net_device *net_dev;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	/* To avoid the loss of other slave devices when main_hr_dev</div><div class='add'>+	 * is unregistered, re-initialize the remaining slaves before</div><div class='add'>+	 * the bond resources cleanup.</div><div class='add'>+	 */</div><div class='add'>+	bond_grp-&gt;bond_state = HNS_ROCE_BOND_NOT_BONDED;</div><div class='add'>+	for (i = 0; i &lt; ROCE_BOND_FUNC_MAX; i++) {</div><div class='add'>+		net_dev = bond_grp-&gt;bond_func_info[i].net_dev;</div><div class='add'>+		if (net_dev &amp;&amp; net_dev != get_hr_netdev(hr_dev, 0))</div><div class='add'>+			hns_roce_bond_init_client(bond_grp, i);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hns_roce_cleanup_bond(bond_grp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev,</div><div class='add'>+				       bool bond_cleanup)</div><div class='add'>+{</div><div class='add'>+	struct net_device *net_dev = get_hr_netdev(hr_dev, 0);</div><div class='ctx'> 	struct hns_roce_ib_iboe *iboe = &amp;hr_dev-&gt;iboe;</div><div class='add'>+	struct hns_roce_bond_group *bond_grp;</div><div class='add'>+	u8 bus_num = get_hr_bus_num(hr_dev);</div><div class='add'>+</div><div class='add'>+	if (bond_cleanup &amp;&amp; hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_BOND) {</div><div class='add'>+		bond_grp = hns_roce_get_bond_grp(net_dev, bus_num);</div><div class='add'>+		if (bond_grp)</div><div class='add'>+			hns_roce_unregister_bond_cleanup(hr_dev, bond_grp);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	hr_dev-&gt;active = false;</div><div class='ctx'> 	unregister_netdevice_notifier(&amp;iboe-&gt;nb);</div><div class='hunk'>@@ -423,26 +710,32 @@ static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static const struct ib_device_ops hns_roce_dev_ops = {</div><div class='add'>+	.owner = THIS_MODULE,</div><div class='add'>+	.driver_id = RDMA_DRIVER_HNS,</div><div class='add'>+	.uverbs_abi_ver = 1,</div><div class='add'>+	.uverbs_no_driver_id_binding = 1,</div><div class='add'>+</div><div class='add'>+	.get_dev_fw_str = hns_roce_get_fw_ver,</div><div class='ctx'> 	.add_gid = hns_roce_add_gid,</div><div class='ctx'> 	.alloc_pd = hns_roce_alloc_pd,</div><div class='ctx'> 	.alloc_ucontext = hns_roce_alloc_ucontext,</div><div class='ctx'> 	.create_ah = hns_roce_create_ah,</div><div class='del'>-	.create_cq = hns_roce_ib_create_cq,</div><div class='add'>+	.create_user_ah = hns_roce_create_ah,</div><div class='add'>+	.create_cq = hns_roce_create_cq,</div><div class='ctx'> 	.create_qp = hns_roce_create_qp,</div><div class='ctx'> 	.dealloc_pd = hns_roce_dealloc_pd,</div><div class='ctx'> 	.dealloc_ucontext = hns_roce_dealloc_ucontext,</div><div class='ctx'> 	.del_gid = hns_roce_del_gid,</div><div class='ctx'> 	.dereg_mr = hns_roce_dereg_mr,</div><div class='ctx'> 	.destroy_ah = hns_roce_destroy_ah,</div><div class='del'>-	.destroy_cq = hns_roce_ib_destroy_cq,</div><div class='add'>+	.destroy_cq = hns_roce_destroy_cq,</div><div class='ctx'> 	.disassociate_ucontext = hns_roce_disassociate_ucontext,</div><div class='del'>-	.fill_res_entry = hns_roce_fill_res_entry,</div><div class='ctx'> 	.get_dma_mr = hns_roce_get_dma_mr,</div><div class='ctx'> 	.get_link_layer = hns_roce_get_link_layer,</div><div class='ctx'> 	.get_port_immutable = hns_roce_port_immutable,</div><div class='ctx'> 	.mmap = hns_roce_mmap,</div><div class='add'>+	.mmap_free = hns_roce_free_mmap,</div><div class='ctx'> 	.modify_device = hns_roce_modify_device,</div><div class='del'>-	.modify_port = hns_roce_modify_port,</div><div class='ctx'> 	.modify_qp = hns_roce_modify_qp,</div><div class='ctx'> 	.query_ah = hns_roce_query_ah,</div><div class='ctx'> 	.query_device = hns_roce_query_device,</div><div class='hunk'>@@ -451,17 +744,19 @@ static const struct ib_device_ops hns_roce_dev_ops = {</div><div class='ctx'> 	.reg_user_mr = hns_roce_reg_user_mr,</div><div class='ctx'> </div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_ah, hns_roce_ah, ibah),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_cq, hns_roce_cq, ib_cq),</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_pd, hns_roce_pd, ibpd),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_qp, hns_roce_qp, ibqp),</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_ucontext, hns_roce_ucontext, ibucontext),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static const struct ib_device_ops hns_roce_dev_mr_ops = {</div><div class='del'>-	.rereg_user_mr = hns_roce_rereg_user_mr,</div><div class='add'>+static const struct ib_device_ops hns_roce_dev_hw_stats_ops = {</div><div class='add'>+	.alloc_hw_port_stats = hns_roce_alloc_hw_port_stats,</div><div class='add'>+	.get_hw_stats = hns_roce_get_hw_stats,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static const struct ib_device_ops hns_roce_dev_mw_ops = {</div><div class='del'>-	.alloc_mw = hns_roce_alloc_mw,</div><div class='del'>-	.dealloc_mw = hns_roce_dealloc_mw,</div><div class='add'>+static const struct ib_device_ops hns_roce_dev_mr_ops = {</div><div class='add'>+	.rereg_user_mr = hns_roce_rereg_user_mr,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static const struct ib_device_ops hns_roce_dev_frmr_ops = {</div><div class='hunk'>@@ -476,88 +771,98 @@ static const struct ib_device_ops hns_roce_dev_srq_ops = {</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_srq, hns_roce_srq, ibsrq),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static const struct ib_device_ops hns_roce_dev_xrcd_ops = {</div><div class='add'>+	.alloc_xrcd = hns_roce_alloc_xrcd,</div><div class='add'>+	.dealloc_xrcd = hns_roce_dealloc_xrcd,</div><div class='add'>+</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_xrcd, hns_roce_xrcd, ibxrcd),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops hns_roce_dev_restrack_ops = {</div><div class='add'>+	.fill_res_cq_entry = hns_roce_fill_res_cq_entry,</div><div class='add'>+	.fill_res_cq_entry_raw = hns_roce_fill_res_cq_entry_raw,</div><div class='add'>+	.fill_res_qp_entry = hns_roce_fill_res_qp_entry,</div><div class='add'>+	.fill_res_qp_entry_raw = hns_roce_fill_res_qp_entry_raw,</div><div class='add'>+	.fill_res_mr_entry = hns_roce_fill_res_mr_entry,</div><div class='add'>+	.fill_res_mr_entry_raw = hns_roce_fill_res_mr_entry_raw,</div><div class='add'>+	.fill_res_srq_entry = hns_roce_fill_res_srq_entry,</div><div class='add'>+	.fill_res_srq_entry_raw = hns_roce_fill_res_srq_entry_raw,</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> static int hns_roce_register_device(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	int ret;</div><div class='ctx'> 	struct hns_roce_ib_iboe *iboe = NULL;</div><div class='del'>-	struct ib_device *ib_dev = NULL;</div><div class='ctx'> 	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	struct ib_device *ib_dev = NULL;</div><div class='add'>+	struct net_device *net_dev;</div><div class='ctx'> 	unsigned int i;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	iboe = &amp;hr_dev-&gt;iboe;</div><div class='ctx'> 	spin_lock_init(&amp;iboe-&gt;lock);</div><div class='ctx'> </div><div class='ctx'> 	ib_dev = &amp;hr_dev-&gt;ib_dev;</div><div class='ctx'> </div><div class='del'>-	ib_dev-&gt;owner			= THIS_MODULE;</div><div class='del'>-	ib_dev-&gt;node_type		= RDMA_NODE_IB_CA;</div><div class='del'>-	ib_dev-&gt;dev.parent		= dev;</div><div class='del'>-</div><div class='del'>-	ib_dev-&gt;phys_port_cnt		= hr_dev-&gt;caps.num_ports;</div><div class='del'>-	ib_dev-&gt;local_dma_lkey		= hr_dev-&gt;caps.reserved_lkey;</div><div class='del'>-	ib_dev-&gt;num_comp_vectors	= hr_dev-&gt;caps.num_comp_vectors;</div><div class='del'>-	ib_dev-&gt;uverbs_abi_ver		= 1;</div><div class='del'>-	ib_dev-&gt;uverbs_cmd_mask		=</div><div class='del'>-		(1ULL &lt;&lt; IB_USER_VERBS_CMD_GET_CONTEXT) |</div><div class='del'>-		(1ULL &lt;&lt; IB_USER_VERBS_CMD_QUERY_DEVICE) |</div><div class='del'>-		(1ULL &lt;&lt; IB_USER_VERBS_CMD_QUERY_PORT) |</div><div class='del'>-		(1ULL &lt;&lt; IB_USER_VERBS_CMD_ALLOC_PD) |</div><div class='del'>-		(1ULL &lt;&lt; IB_USER_VERBS_CMD_DEALLOC_PD) |</div><div class='del'>-		(1ULL &lt;&lt; IB_USER_VERBS_CMD_REG_MR) |</div><div class='del'>-		(1ULL &lt;&lt; IB_USER_VERBS_CMD_DEREG_MR) |</div><div class='del'>-		(1ULL &lt;&lt; IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |</div><div class='del'>-		(1ULL &lt;&lt; IB_USER_VERBS_CMD_CREATE_CQ) |</div><div class='del'>-		(1ULL &lt;&lt; IB_USER_VERBS_CMD_DESTROY_CQ) |</div><div class='del'>-		(1ULL &lt;&lt; IB_USER_VERBS_CMD_CREATE_QP) |</div><div class='del'>-		(1ULL &lt;&lt; IB_USER_VERBS_CMD_MODIFY_QP) |</div><div class='del'>-		(1ULL &lt;&lt; IB_USER_VERBS_CMD_QUERY_QP) |</div><div class='del'>-		(1ULL &lt;&lt; IB_USER_VERBS_CMD_DESTROY_QP);</div><div class='del'>-</div><div class='del'>-	ib_dev-&gt;uverbs_ex_cmd_mask |=</div><div class='del'>-		(1ULL &lt;&lt; IB_USER_VERBS_EX_CMD_MODIFY_CQ);</div><div class='del'>-</div><div class='del'>-	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_REREG_MR) {</div><div class='del'>-		ib_dev-&gt;uverbs_cmd_mask |= (1ULL &lt;&lt; IB_USER_VERBS_CMD_REREG_MR);</div><div class='del'>-		ib_set_device_ops(ib_dev, &amp;hns_roce_dev_mr_ops);</div><div class='del'>-	}</div><div class='add'>+	ib_dev-&gt;node_type = RDMA_NODE_IB_CA;</div><div class='add'>+	ib_dev-&gt;dev.parent = dev;</div><div class='ctx'> </div><div class='del'>-	/* MW */</div><div class='del'>-	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_MW) {</div><div class='del'>-		ib_dev-&gt;uverbs_cmd_mask |=</div><div class='del'>-					(1ULL &lt;&lt; IB_USER_VERBS_CMD_ALLOC_MW) |</div><div class='del'>-					(1ULL &lt;&lt; IB_USER_VERBS_CMD_DEALLOC_MW);</div><div class='del'>-		ib_set_device_ops(ib_dev, &amp;hns_roce_dev_mw_ops);</div><div class='del'>-	}</div><div class='add'>+	ib_dev-&gt;phys_port_cnt = hr_dev-&gt;caps.num_ports;</div><div class='add'>+	ib_dev-&gt;local_dma_lkey = hr_dev-&gt;caps.reserved_lkey;</div><div class='add'>+	ib_dev-&gt;num_comp_vectors = hr_dev-&gt;caps.num_comp_vectors;</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_REREG_MR)</div><div class='add'>+		ib_set_device_ops(ib_dev, &amp;hns_roce_dev_mr_ops);</div><div class='ctx'> </div><div class='del'>-	/* FRMR */</div><div class='ctx'> 	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_FRMR)</div><div class='ctx'> 		ib_set_device_ops(ib_dev, &amp;hns_roce_dev_frmr_ops);</div><div class='ctx'> </div><div class='del'>-	/* SRQ */</div><div class='ctx'> 	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_SRQ) {</div><div class='del'>-		ib_dev-&gt;uverbs_cmd_mask |=</div><div class='del'>-				(1ULL &lt;&lt; IB_USER_VERBS_CMD_CREATE_SRQ) |</div><div class='del'>-				(1ULL &lt;&lt; IB_USER_VERBS_CMD_MODIFY_SRQ) |</div><div class='del'>-				(1ULL &lt;&lt; IB_USER_VERBS_CMD_QUERY_SRQ) |</div><div class='del'>-				(1ULL &lt;&lt; IB_USER_VERBS_CMD_DESTROY_SRQ) |</div><div class='del'>-				(1ULL &lt;&lt; IB_USER_VERBS_CMD_POST_SRQ_RECV);</div><div class='ctx'> 		ib_set_device_ops(ib_dev, &amp;hns_roce_dev_srq_ops);</div><div class='ctx'> 		ib_set_device_ops(ib_dev, hr_dev-&gt;hw-&gt;hns_roce_dev_srq_ops);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ib_dev-&gt;driver_id = RDMA_DRIVER_HNS;</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_XRC)</div><div class='add'>+		ib_set_device_ops(ib_dev, &amp;hns_roce_dev_xrcd_ops);</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09 &amp;&amp;</div><div class='add'>+	    !hr_dev-&gt;is_vf)</div><div class='add'>+		ib_set_device_ops(ib_dev, &amp;hns_roce_dev_hw_stats_ops);</div><div class='add'>+</div><div class='ctx'> 	ib_set_device_ops(ib_dev, hr_dev-&gt;hw-&gt;hns_roce_dev_ops);</div><div class='ctx'> 	ib_set_device_ops(ib_dev, &amp;hns_roce_dev_ops);</div><div class='del'>-	for (i = 0; i &lt; hr_dev-&gt;caps.num_ports; i++) {</div><div class='del'>-		if (!hr_dev-&gt;iboe.netdevs[i])</div><div class='del'>-			continue;</div><div class='add'>+	ib_set_device_ops(ib_dev, &amp;hns_roce_dev_restrack_ops);</div><div class='ctx'> </div><div class='del'>-		ret = ib_device_set_netdev(ib_dev, hr_dev-&gt;iboe.netdevs[i],</div><div class='del'>-					   i + 1);</div><div class='del'>-		if (ret)</div><div class='add'>+	dma_set_max_seg_size(dev, SZ_2G);</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_BOND) {</div><div class='add'>+		ret = hns_roce_alloc_bond_grp(hr_dev);</div><div class='add'>+		if (ret) {</div><div class='add'>+			dev_err(dev, "failed to alloc bond_grp for bus %u, ret = %d\n",</div><div class='add'>+				get_hr_bus_num(hr_dev), ret);</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_BOND &amp;&amp;</div><div class='add'>+	    hns_roce_bond_is_active(hr_dev)) {</div><div class='add'>+		ret = hns_roce_bond_init(hr_dev);</div><div class='add'>+		if (ret) {</div><div class='add'>+			dev_err(dev, "failed to init bond!\n");</div><div class='ctx'> 			return ret;</div><div class='add'>+		}</div><div class='add'>+		ret = ib_register_device(ib_dev, "hns_bond_%d", dev);</div><div class='add'>+	} else {</div><div class='add'>+		for (i = 0; i &lt; hr_dev-&gt;caps.num_ports; i++) {</div><div class='add'>+			net_dev = get_hr_netdev(hr_dev, i);</div><div class='add'>+			if (!net_dev)</div><div class='add'>+				continue;</div><div class='add'>+</div><div class='add'>+			ret = ib_device_set_netdev(ib_dev, net_dev, i + 1);</div><div class='add'>+			if (ret)</div><div class='add'>+				return ret;</div><div class='add'>+		}</div><div class='add'>+		ret = ib_register_device(ib_dev, "hns_%d", dev);</div><div class='ctx'> 	}</div><div class='del'>-	ret = ib_register_device(ib_dev, "hns_%d");</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		dev_err(dev, "ib_register_device failed!\n");</div><div class='ctx'> 		return ret;</div><div class='hunk'>@@ -587,41 +892,22 @@ error_failed_setup_mtu_mac:</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_init_hem(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	int ret;</div><div class='ctx'> 	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_init_hem_table(hr_dev, &amp;hr_dev-&gt;mr_table.mtt_table,</div><div class='del'>-				      HEM_TYPE_MTT, hr_dev-&gt;caps.mtt_entry_sz,</div><div class='del'>-				      hr_dev-&gt;caps.num_mtt_segs, 1);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "Failed to init MTT context memory, aborting.\n");</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) {</div><div class='del'>-		ret = hns_roce_init_hem_table(hr_dev,</div><div class='del'>-				      &amp;hr_dev-&gt;mr_table.mtt_cqe_table,</div><div class='del'>-				      HEM_TYPE_CQE, hr_dev-&gt;caps.mtt_entry_sz,</div><div class='del'>-				      hr_dev-&gt;caps.num_cqe_segs, 1);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "Failed to init MTT CQE context memory, aborting.\n");</div><div class='del'>-			goto err_unmap_cqe;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	ret = hns_roce_init_hem_table(hr_dev, &amp;hr_dev-&gt;mr_table.mtpt_table,</div><div class='ctx'> 				      HEM_TYPE_MTPT, hr_dev-&gt;caps.mtpt_entry_sz,</div><div class='del'>-				      hr_dev-&gt;caps.num_mtpts, 1);</div><div class='add'>+				      hr_dev-&gt;caps.num_mtpts);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "Failed to init MTPT context memory, aborting.\n");</div><div class='del'>-		goto err_unmap_mtt;</div><div class='add'>+		dev_err(dev, "failed to init MTPT context memory, aborting.\n");</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ret = hns_roce_init_hem_table(hr_dev, &amp;hr_dev-&gt;qp_table.qp_table,</div><div class='del'>-				      HEM_TYPE_QPC, hr_dev-&gt;caps.qpc_entry_sz,</div><div class='del'>-				      hr_dev-&gt;caps.num_qps, 1);</div><div class='add'>+				      HEM_TYPE_QPC, hr_dev-&gt;caps.qpc_sz,</div><div class='add'>+				      hr_dev-&gt;caps.num_qps);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "Failed to init QP context memory, aborting.\n");</div><div class='add'>+		dev_err(dev, "failed to init QP context memory, aborting.\n");</div><div class='ctx'> 		goto err_unmap_dmpt;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -629,9 +915,9 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev)</div><div class='ctx'> 				      HEM_TYPE_IRRL,</div><div class='ctx'> 				      hr_dev-&gt;caps.irrl_entry_sz *</div><div class='ctx'> 				      hr_dev-&gt;caps.max_qp_init_rdma,</div><div class='del'>-				      hr_dev-&gt;caps.num_qps, 1);</div><div class='add'>+				      hr_dev-&gt;caps.num_qps);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "Failed to init irrl_table memory, aborting.\n");</div><div class='add'>+		dev_err(dev, "failed to init irrl_table memory, aborting.\n");</div><div class='ctx'> 		goto err_unmap_qp;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -641,123 +927,100 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev)</div><div class='ctx'> 					      HEM_TYPE_TRRL,</div><div class='ctx'> 					      hr_dev-&gt;caps.trrl_entry_sz *</div><div class='ctx'> 					      hr_dev-&gt;caps.max_qp_dest_rdma,</div><div class='del'>-					      hr_dev-&gt;caps.num_qps, 1);</div><div class='add'>+					      hr_dev-&gt;caps.num_qps);</div><div class='ctx'> 		if (ret) {</div><div class='ctx'> 			dev_err(dev,</div><div class='del'>-			       "Failed to init trrl_table memory, aborting.\n");</div><div class='add'>+				"failed to init trrl_table memory, aborting.\n");</div><div class='ctx'> 			goto err_unmap_irrl;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ret = hns_roce_init_hem_table(hr_dev, &amp;hr_dev-&gt;cq_table.table,</div><div class='ctx'> 				      HEM_TYPE_CQC, hr_dev-&gt;caps.cqc_entry_sz,</div><div class='del'>-				      hr_dev-&gt;caps.num_cqs, 1);</div><div class='add'>+				      hr_dev-&gt;caps.num_cqs);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "Failed to init CQ context memory, aborting.\n");</div><div class='add'>+		dev_err(dev, "failed to init CQ context memory, aborting.\n");</div><div class='ctx'> 		goto err_unmap_trrl;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (hr_dev-&gt;caps.srqc_entry_sz) {</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_SRQ) {</div><div class='ctx'> 		ret = hns_roce_init_hem_table(hr_dev, &amp;hr_dev-&gt;srq_table.table,</div><div class='ctx'> 					      HEM_TYPE_SRQC,</div><div class='ctx'> 					      hr_dev-&gt;caps.srqc_entry_sz,</div><div class='del'>-					      hr_dev-&gt;caps.num_srqs, 1);</div><div class='add'>+					      hr_dev-&gt;caps.num_srqs);</div><div class='ctx'> 		if (ret) {</div><div class='ctx'> 			dev_err(dev,</div><div class='del'>-			      "Failed to init SRQ context memory, aborting.\n");</div><div class='add'>+				"failed to init SRQ context memory, aborting.\n");</div><div class='ctx'> 			goto err_unmap_cq;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (hr_dev-&gt;caps.num_srqwqe_segs) {</div><div class='del'>-		ret = hns_roce_init_hem_table(hr_dev,</div><div class='del'>-					     &amp;hr_dev-&gt;mr_table.mtt_srqwqe_table,</div><div class='del'>-					     HEM_TYPE_SRQWQE,</div><div class='del'>-					     hr_dev-&gt;caps.mtt_entry_sz,</div><div class='del'>-					     hr_dev-&gt;caps.num_srqwqe_segs, 1);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev,</div><div class='del'>-				"Failed to init MTT srqwqe memory, aborting.\n");</div><div class='del'>-			goto err_unmap_srq;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (hr_dev-&gt;caps.num_idx_segs) {</div><div class='del'>-		ret = hns_roce_init_hem_table(hr_dev,</div><div class='del'>-					      &amp;hr_dev-&gt;mr_table.mtt_idx_table,</div><div class='del'>-					      HEM_TYPE_IDX,</div><div class='del'>-					      hr_dev-&gt;caps.idx_entry_sz,</div><div class='del'>-					      hr_dev-&gt;caps.num_idx_segs, 1);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev,</div><div class='del'>-				"Failed to init MTT idx memory, aborting.\n");</div><div class='del'>-			goto err_unmap_srqwqe;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (hr_dev-&gt;caps.sccc_entry_sz) {</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {</div><div class='ctx'> 		ret = hns_roce_init_hem_table(hr_dev,</div><div class='ctx'> 					      &amp;hr_dev-&gt;qp_table.sccc_table,</div><div class='ctx'> 					      HEM_TYPE_SCCC,</div><div class='del'>-					      hr_dev-&gt;caps.sccc_entry_sz,</div><div class='del'>-					      hr_dev-&gt;caps.num_qps, 1);</div><div class='add'>+					      hr_dev-&gt;caps.sccc_sz,</div><div class='add'>+					      hr_dev-&gt;caps.num_qps);</div><div class='ctx'> 		if (ret) {</div><div class='ctx'> 			dev_err(dev,</div><div class='del'>-			      "Failed to init SCC context memory, aborting.\n");</div><div class='del'>-			goto err_unmap_idx;</div><div class='add'>+				"failed to init SCC context memory, aborting.\n");</div><div class='add'>+			goto err_unmap_srq;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (hr_dev-&gt;caps.qpc_timer_entry_sz) {</div><div class='del'>-		ret = hns_roce_init_hem_table(hr_dev,</div><div class='del'>-					      &amp;hr_dev-&gt;qpc_timer_table,</div><div class='add'>+		ret = hns_roce_init_hem_table(hr_dev, &amp;hr_dev-&gt;qpc_timer_table,</div><div class='ctx'> 					      HEM_TYPE_QPC_TIMER,</div><div class='ctx'> 					      hr_dev-&gt;caps.qpc_timer_entry_sz,</div><div class='del'>-					      hr_dev-&gt;caps.num_qpc_timer, 1);</div><div class='add'>+					      hr_dev-&gt;caps.qpc_timer_bt_num);</div><div class='ctx'> 		if (ret) {</div><div class='ctx'> 			dev_err(dev,</div><div class='del'>-			      "Failed to init QPC timer memory, aborting.\n");</div><div class='add'>+				"failed to init QPC timer memory, aborting.\n");</div><div class='ctx'> 			goto err_unmap_ctx;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (hr_dev-&gt;caps.cqc_timer_entry_sz) {</div><div class='del'>-		ret = hns_roce_init_hem_table(hr_dev,</div><div class='del'>-					      &amp;hr_dev-&gt;cqc_timer_table,</div><div class='add'>+		ret = hns_roce_init_hem_table(hr_dev, &amp;hr_dev-&gt;cqc_timer_table,</div><div class='ctx'> 					      HEM_TYPE_CQC_TIMER,</div><div class='ctx'> 					      hr_dev-&gt;caps.cqc_timer_entry_sz,</div><div class='del'>-					      hr_dev-&gt;caps.num_cqc_timer, 1);</div><div class='add'>+					      hr_dev-&gt;caps.cqc_timer_bt_num);</div><div class='ctx'> 		if (ret) {</div><div class='ctx'> 			dev_err(dev,</div><div class='del'>-			      "Failed to init CQC timer memory, aborting.\n");</div><div class='add'>+				"failed to init CQC timer memory, aborting.\n");</div><div class='ctx'> 			goto err_unmap_qpc_timer;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (hr_dev-&gt;caps.gmv_entry_sz) {</div><div class='add'>+		ret = hns_roce_init_hem_table(hr_dev, &amp;hr_dev-&gt;gmv_table,</div><div class='add'>+					      HEM_TYPE_GMV,</div><div class='add'>+					      hr_dev-&gt;caps.gmv_entry_sz,</div><div class='add'>+					      hr_dev-&gt;caps.gmv_entry_num);</div><div class='add'>+		if (ret) {</div><div class='add'>+			dev_err(dev,</div><div class='add'>+				"failed to init gmv table memory, ret = %d\n",</div><div class='add'>+				ret);</div><div class='add'>+			goto err_unmap_cqc_timer;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='add'>+err_unmap_cqc_timer:</div><div class='add'>+	if (hr_dev-&gt;caps.cqc_timer_entry_sz)</div><div class='add'>+		hns_roce_cleanup_hem_table(hr_dev, &amp;hr_dev-&gt;cqc_timer_table);</div><div class='add'>+</div><div class='ctx'> err_unmap_qpc_timer:</div><div class='ctx'> 	if (hr_dev-&gt;caps.qpc_timer_entry_sz)</div><div class='del'>-		hns_roce_cleanup_hem_table(hr_dev,</div><div class='del'>-					   &amp;hr_dev-&gt;qpc_timer_table);</div><div class='add'>+		hns_roce_cleanup_hem_table(hr_dev, &amp;hr_dev-&gt;qpc_timer_table);</div><div class='ctx'> </div><div class='ctx'> err_unmap_ctx:</div><div class='del'>-	if (hr_dev-&gt;caps.sccc_entry_sz)</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)</div><div class='ctx'> 		hns_roce_cleanup_hem_table(hr_dev,</div><div class='ctx'> 					   &amp;hr_dev-&gt;qp_table.sccc_table);</div><div class='del'>-</div><div class='del'>-err_unmap_idx:</div><div class='del'>-	if (hr_dev-&gt;caps.num_idx_segs)</div><div class='del'>-		hns_roce_cleanup_hem_table(hr_dev,</div><div class='del'>-					   &amp;hr_dev-&gt;mr_table.mtt_idx_table);</div><div class='del'>-</div><div class='del'>-err_unmap_srqwqe:</div><div class='del'>-	if (hr_dev-&gt;caps.num_srqwqe_segs)</div><div class='del'>-		hns_roce_cleanup_hem_table(hr_dev,</div><div class='del'>-					   &amp;hr_dev-&gt;mr_table.mtt_srqwqe_table);</div><div class='del'>-</div><div class='ctx'> err_unmap_srq:</div><div class='del'>-	if (hr_dev-&gt;caps.srqc_entry_sz)</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_SRQ)</div><div class='ctx'> 		hns_roce_cleanup_hem_table(hr_dev, &amp;hr_dev-&gt;srq_table.table);</div><div class='ctx'> </div><div class='ctx'> err_unmap_cq:</div><div class='hunk'>@@ -777,17 +1040,15 @@ err_unmap_qp:</div><div class='ctx'> err_unmap_dmpt:</div><div class='ctx'> 	hns_roce_cleanup_hem_table(hr_dev, &amp;hr_dev-&gt;mr_table.mtpt_table);</div><div class='ctx'> </div><div class='del'>-err_unmap_mtt:</div><div class='del'>-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))</div><div class='del'>-		hns_roce_cleanup_hem_table(hr_dev,</div><div class='del'>-					   &amp;hr_dev-&gt;mr_table.mtt_cqe_table);</div><div class='del'>-</div><div class='del'>-err_unmap_cqe:</div><div class='del'>-	hns_roce_cleanup_hem_table(hr_dev, &amp;hr_dev-&gt;mr_table.mtt_table);</div><div class='del'>-</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void hns_roce_teardown_hca(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	hns_roce_cleanup_bitmap(hr_dev);</div><div class='add'>+	mutex_destroy(&amp;hr_dev-&gt;pgdir_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> /**</div><div class='ctx'>  * hns_roce_setup_hca - setup host channel adapter</div><div class='ctx'>  * @hr_dev: pointer to hns roce device</div><div class='hunk'>@@ -795,110 +1056,133 @@ err_unmap_cqe:</div><div class='ctx'>  */</div><div class='ctx'> static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	int ret;</div><div class='ctx'> 	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_init(&amp;hr_dev-&gt;sm_lock);</div><div class='del'>-	spin_lock_init(&amp;hr_dev-&gt;bt_cmd_lock);</div><div class='ctx'> </div><div class='del'>-	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RECORD_DB) {</div><div class='del'>-		INIT_LIST_HEAD(&amp;hr_dev-&gt;pgdir_list);</div><div class='del'>-		mutex_init(&amp;hr_dev-&gt;pgdir_mutex);</div><div class='del'>-	}</div><div class='add'>+	INIT_LIST_HEAD(&amp;hr_dev-&gt;qp_list);</div><div class='add'>+	spin_lock_init(&amp;hr_dev-&gt;qp_list_lock);</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_init_uar_table(hr_dev);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "Failed to initialize uar table. aborting\n");</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='add'>+	INIT_LIST_HEAD(&amp;hr_dev-&gt;pgdir_list);</div><div class='add'>+	mutex_init(&amp;hr_dev-&gt;pgdir_mutex);</div><div class='add'>+</div><div class='add'>+	hns_roce_init_uar_table(hr_dev);</div><div class='ctx'> </div><div class='ctx'> 	ret = hns_roce_uar_alloc(hr_dev, &amp;hr_dev-&gt;priv_uar);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "Failed to allocate priv_uar.\n");</div><div class='add'>+		dev_err(dev, "failed to allocate priv_uar.\n");</div><div class='ctx'> 		goto err_uar_table_free;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_init_pd_table(hr_dev);</div><div class='add'>+	ret = hns_roce_init_qp_table(hr_dev);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "Failed to init protected domain table.\n");</div><div class='del'>-		goto err_uar_alloc_free;</div><div class='add'>+		dev_err(dev, "failed to init qp_table.\n");</div><div class='add'>+		goto err_uar_table_free;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_init_mr_table(hr_dev);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "Failed to init memory region table.\n");</div><div class='del'>-		goto err_pd_table_free;</div><div class='del'>-	}</div><div class='add'>+	hns_roce_init_pd_table(hr_dev);</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_init_cq_table(hr_dev);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "Failed to init completion queue table.\n");</div><div class='del'>-		goto err_mr_table_free;</div><div class='del'>-	}</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_XRC)</div><div class='add'>+		hns_roce_init_xrcd_table(hr_dev);</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_init_qp_table(hr_dev);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "Failed to init queue pair table.\n");</div><div class='del'>-		goto err_cq_table_free;</div><div class='del'>-	}</div><div class='add'>+	hns_roce_init_mr_table(hr_dev);</div><div class='ctx'> </div><div class='del'>-	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_SRQ) {</div><div class='del'>-		ret = hns_roce_init_srq_table(hr_dev);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev,</div><div class='del'>-				"Failed to init share receive queue table.\n");</div><div class='del'>-			goto err_qp_table_free;</div><div class='add'>+	hns_roce_init_cq_table(hr_dev);</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_SRQ)</div><div class='add'>+		hns_roce_init_srq_table(hr_dev);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_uar_table_free:</div><div class='add'>+	ida_destroy(&amp;hr_dev-&gt;uar_ida.ida);</div><div class='add'>+	mutex_destroy(&amp;hr_dev-&gt;pgdir_mutex);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void check_and_get_armed_cq(struct list_head *cq_list, struct ib_cq *cq)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_cq *hr_cq = to_hr_cq(cq);</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;hr_cq-&gt;lock, flags);</div><div class='add'>+	if (cq-&gt;comp_handler) {</div><div class='add'>+		if (!hr_cq-&gt;is_armed) {</div><div class='add'>+			hr_cq-&gt;is_armed = 1;</div><div class='add'>+			list_add_tail(&amp;hr_cq-&gt;node, cq_list);</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;hr_cq-&gt;lock, flags);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_qp *hr_qp;</div><div class='add'>+	struct hns_roce_cq *hr_cq;</div><div class='add'>+	struct list_head cq_list;</div><div class='add'>+	unsigned long flags_qp;</div><div class='add'>+	unsigned long flags;</div><div class='ctx'> </div><div class='del'>-err_qp_table_free:</div><div class='del'>-	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_SRQ)</div><div class='del'>-		hns_roce_cleanup_qp_table(hr_dev);</div><div class='add'>+	INIT_LIST_HEAD(&amp;cq_list);</div><div class='ctx'> </div><div class='del'>-err_cq_table_free:</div><div class='del'>-	hns_roce_cleanup_cq_table(hr_dev);</div><div class='add'>+	spin_lock_irqsave(&amp;hr_dev-&gt;qp_list_lock, flags);</div><div class='add'>+	list_for_each_entry(hr_qp, &amp;hr_dev-&gt;qp_list, node) {</div><div class='add'>+		spin_lock_irqsave(&amp;hr_qp-&gt;sq.lock, flags_qp);</div><div class='add'>+		if (hr_qp-&gt;sq.tail != hr_qp-&gt;sq.head)</div><div class='add'>+			check_and_get_armed_cq(&amp;cq_list, hr_qp-&gt;ibqp.send_cq);</div><div class='add'>+		spin_unlock_irqrestore(&amp;hr_qp-&gt;sq.lock, flags_qp);</div><div class='ctx'> </div><div class='del'>-err_mr_table_free:</div><div class='del'>-	hns_roce_cleanup_mr_table(hr_dev);</div><div class='add'>+		spin_lock_irqsave(&amp;hr_qp-&gt;rq.lock, flags_qp);</div><div class='add'>+		if ((!hr_qp-&gt;ibqp.srq) &amp;&amp; (hr_qp-&gt;rq.tail != hr_qp-&gt;rq.head))</div><div class='add'>+			check_and_get_armed_cq(&amp;cq_list, hr_qp-&gt;ibqp.recv_cq);</div><div class='add'>+		spin_unlock_irqrestore(&amp;hr_qp-&gt;rq.lock, flags_qp);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-err_pd_table_free:</div><div class='del'>-	hns_roce_cleanup_pd_table(hr_dev);</div><div class='add'>+	list_for_each_entry(hr_cq, &amp;cq_list, node)</div><div class='add'>+		hns_roce_cq_completion(hr_dev, hr_cq-&gt;cqn);</div><div class='ctx'> </div><div class='del'>-err_uar_alloc_free:</div><div class='del'>-	hns_roce_uar_free(hr_dev, &amp;hr_dev-&gt;priv_uar);</div><div class='add'>+	spin_unlock_irqrestore(&amp;hr_dev-&gt;qp_list_lock, flags);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-err_uar_table_free:</div><div class='del'>-	hns_roce_cleanup_uar_table(hr_dev);</div><div class='del'>-	return ret;</div><div class='add'>+static int hns_roce_alloc_dfx_cnt(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	hr_dev-&gt;dfx_cnt = kvcalloc(HNS_ROCE_DFX_CNT_TOTAL, sizeof(atomic64_t),</div><div class='add'>+				   GFP_KERNEL);</div><div class='add'>+	if (!hr_dev-&gt;dfx_cnt)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void hns_roce_dealloc_dfx_cnt(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	kvfree(hr_dev-&gt;dfx_cnt);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int hns_roce_init(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	int ret;</div><div class='ctx'> 	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	if (hr_dev-&gt;hw-&gt;reset) {</div><div class='del'>-		ret = hr_dev-&gt;hw-&gt;reset(hr_dev, true);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "Reset RoCE engine failed!\n");</div><div class='del'>-			return ret;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='ctx'> 	hr_dev-&gt;is_reset = false;</div><div class='ctx'> </div><div class='add'>+	ret = hns_roce_alloc_dfx_cnt(hr_dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='ctx'> 	if (hr_dev-&gt;hw-&gt;cmq_init) {</div><div class='ctx'> 		ret = hr_dev-&gt;hw-&gt;cmq_init(hr_dev);</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			dev_err(dev, "Init RoCE Command Queue failed!\n");</div><div class='del'>-			goto error_failed_cmq_init;</div><div class='add'>+			dev_err(dev, "init RoCE Command Queue failed!\n");</div><div class='add'>+			goto error_failed_alloc_dfx_cnt;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ret = hr_dev-&gt;hw-&gt;hw_profile(hr_dev);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "Get RoCE engine profile failed!\n");</div><div class='add'>+		dev_err(dev, "get RoCE engine profile failed!\n");</div><div class='ctx'> 		goto error_failed_cmd_init;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -908,6 +1192,7 @@ int hns_roce_init(struct hns_roce_dev *hr_dev)</div><div class='ctx'> 		goto error_failed_cmd_init;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	/* EQ depends on poll mode, event mode depends on EQ */</div><div class='ctx'> 	ret = hr_dev-&gt;hw-&gt;init_eq(hr_dev);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		dev_err(dev, "eq init failed!\n");</div><div class='hunk'>@@ -916,10 +1201,9 @@ int hns_roce_init(struct hns_roce_dev *hr_dev)</div><div class='ctx'> </div><div class='ctx'> 	if (hr_dev-&gt;cmd_mod) {</div><div class='ctx'> 		ret = hns_roce_cmd_use_events(hr_dev);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "Switch to event-driven cmd failed!\n");</div><div class='del'>-			goto error_failed_use_event;</div><div class='del'>-		}</div><div class='add'>+		if (ret)</div><div class='add'>+			dev_warn(dev,</div><div class='add'>+				 "Cmd event  mode failed, set back to poll!\n");</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ret = hns_roce_init_hem(hr_dev);</div><div class='hunk'>@@ -946,6 +1230,8 @@ int hns_roce_init(struct hns_roce_dev *hr_dev)</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto error_failed_register_device;</div><div class='ctx'> </div><div class='add'>+	hns_roce_register_debugfs(hr_dev);</div><div class='add'>+</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> error_failed_register_device:</div><div class='hunk'>@@ -953,7 +1239,7 @@ error_failed_register_device:</div><div class='ctx'> 		hr_dev-&gt;hw-&gt;hw_exit(hr_dev);</div><div class='ctx'> </div><div class='ctx'> error_failed_engine_init:</div><div class='del'>-	hns_roce_cleanup_bitmap(hr_dev);</div><div class='add'>+	hns_roce_teardown_hca(hr_dev);</div><div class='ctx'> </div><div class='ctx'> error_failed_setup_hca:</div><div class='ctx'> 	hns_roce_cleanup_hem(hr_dev);</div><div class='hunk'>@@ -961,8 +1247,6 @@ error_failed_setup_hca:</div><div class='ctx'> error_failed_init_hem:</div><div class='ctx'> 	if (hr_dev-&gt;cmd_mod)</div><div class='ctx'> 		hns_roce_cmd_use_polling(hr_dev);</div><div class='del'>-</div><div class='del'>-error_failed_use_event:</div><div class='ctx'> 	hr_dev-&gt;hw-&gt;cleanup_eq(hr_dev);</div><div class='ctx'> </div><div class='ctx'> error_failed_eq_table:</div><div class='hunk'>@@ -972,23 +1256,20 @@ error_failed_cmd_init:</div><div class='ctx'> 	if (hr_dev-&gt;hw-&gt;cmq_exit)</div><div class='ctx'> 		hr_dev-&gt;hw-&gt;cmq_exit(hr_dev);</div><div class='ctx'> </div><div class='del'>-error_failed_cmq_init:</div><div class='del'>-	if (hr_dev-&gt;hw-&gt;reset) {</div><div class='del'>-		if (hr_dev-&gt;hw-&gt;reset(hr_dev, false))</div><div class='del'>-			dev_err(dev, "Dereset RoCE engine failed!\n");</div><div class='del'>-	}</div><div class='add'>+error_failed_alloc_dfx_cnt:</div><div class='add'>+	hns_roce_dealloc_dfx_cnt(hr_dev);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_init);</div><div class='ctx'> </div><div class='del'>-void hns_roce_exit(struct hns_roce_dev *hr_dev)</div><div class='add'>+void hns_roce_exit(struct hns_roce_dev *hr_dev, bool bond_cleanup)</div><div class='ctx'> {</div><div class='del'>-	hns_roce_unregister_device(hr_dev);</div><div class='add'>+	hns_roce_unregister_debugfs(hr_dev);</div><div class='add'>+	hns_roce_unregister_device(hr_dev, bond_cleanup);</div><div class='ctx'> </div><div class='ctx'> 	if (hr_dev-&gt;hw-&gt;hw_exit)</div><div class='ctx'> 		hr_dev-&gt;hw-&gt;hw_exit(hr_dev);</div><div class='del'>-	hns_roce_cleanup_bitmap(hr_dev);</div><div class='add'>+	hns_roce_teardown_hca(hr_dev);</div><div class='ctx'> 	hns_roce_cleanup_hem(hr_dev);</div><div class='ctx'> </div><div class='ctx'> 	if (hr_dev-&gt;cmd_mod)</div><div class='hunk'>@@ -998,10 +1279,8 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev)</div><div class='ctx'> 	hns_roce_cmd_cleanup(hr_dev);</div><div class='ctx'> 	if (hr_dev-&gt;hw-&gt;cmq_exit)</div><div class='ctx'> 		hr_dev-&gt;hw-&gt;cmq_exit(hr_dev);</div><div class='del'>-	if (hr_dev-&gt;hw-&gt;reset)</div><div class='del'>-		hr_dev-&gt;hw-&gt;reset(hr_dev, false);</div><div class='add'>+	hns_roce_dealloc_dfx_cnt(hr_dev);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_exit);</div><div class='ctx'> </div><div class='ctx'> MODULE_LICENSE("Dual BSD/GPL");</div><div class='ctx'> MODULE_AUTHOR("Wei Hu &lt;xavier.huwei@huawei.com&gt;");</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c<br/>index 6110ec408626..31cb8699e198 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_mr.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_mr.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_mr.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_mr.c</a></div><div class='hunk'>@@ -31,1290 +31,361 @@</div><div class='ctx'>  * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/platform_device.h&gt;</div><div class='ctx'> #include &lt;linux/vmalloc.h&gt;</div><div class='add'>+#include &lt;linux/count_zeros.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_umem.h&gt;</div><div class='add'>+#include &lt;linux/math.h&gt;</div><div class='ctx'> #include "hns_roce_device.h"</div><div class='ctx'> #include "hns_roce_cmd.h"</div><div class='ctx'> #include "hns_roce_hem.h"</div><div class='add'>+#include "hns_roce_trace.h"</div><div class='ctx'> </div><div class='del'>-static u32 hw_index_to_key(unsigned long ind)</div><div class='add'>+static u32 hw_index_to_key(int ind)</div><div class='ctx'> {</div><div class='del'>-	return (u32)(ind &gt;&gt; 24) | (ind &lt;&lt; 8);</div><div class='add'>+	return ((u32)ind &gt;&gt; 24) | ((u32)ind &lt;&lt; 8);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> unsigned long key_to_hw_index(u32 key)</div><div class='ctx'> {</div><div class='ctx'> 	return (key &lt;&lt; 24) | (key &gt;&gt; 8);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(key_to_hw_index);</div><div class='ctx'> </div><div class='del'>-static int hns_roce_sw2hw_mpt(struct hns_roce_dev *hr_dev,</div><div class='del'>-			      struct hns_roce_cmd_mailbox *mailbox,</div><div class='del'>-			      unsigned long mpt_index)</div><div class='add'>+static int alloc_mr_key(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)</div><div class='ctx'> {</div><div class='del'>-	return hns_roce_cmd_mbox(hr_dev, mailbox-&gt;dma, 0, mpt_index, 0,</div><div class='del'>-				 HNS_ROCE_CMD_SW2HW_MPT,</div><div class='del'>-				 HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev,</div><div class='del'>-			      struct hns_roce_cmd_mailbox *mailbox,</div><div class='del'>-			      unsigned long mpt_index)</div><div class='del'>-{</div><div class='del'>-	return hns_roce_cmd_mbox(hr_dev, 0, mailbox ? mailbox-&gt;dma : 0,</div><div class='del'>-				 mpt_index, !mailbox, HNS_ROCE_CMD_HW2SW_MPT,</div><div class='del'>-				 HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_hw2sw_mpt);</div><div class='del'>-</div><div class='del'>-static int hns_roce_buddy_alloc(struct hns_roce_buddy *buddy, int order,</div><div class='del'>-				unsigned long *seg)</div><div class='del'>-{</div><div class='del'>-	int o;</div><div class='del'>-	u32 m;</div><div class='del'>-</div><div class='del'>-	spin_lock(&amp;buddy-&gt;lock);</div><div class='del'>-</div><div class='del'>-	for (o = order; o &lt;= buddy-&gt;max_order; ++o) {</div><div class='del'>-		if (buddy-&gt;num_free[o]) {</div><div class='del'>-			m = 1 &lt;&lt; (buddy-&gt;max_order - o);</div><div class='del'>-			*seg = find_first_bit(buddy-&gt;bits[o], m);</div><div class='del'>-			if (*seg &lt; m)</div><div class='del'>-				goto found;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	spin_unlock(&amp;buddy-&gt;lock);</div><div class='del'>-	return -1;</div><div class='del'>-</div><div class='del'>- found:</div><div class='del'>-	clear_bit(*seg, buddy-&gt;bits[o]);</div><div class='del'>-	--buddy-&gt;num_free[o];</div><div class='del'>-</div><div class='del'>-	while (o &gt; order) {</div><div class='del'>-		--o;</div><div class='del'>-		*seg &lt;&lt;= 1;</div><div class='del'>-		set_bit(*seg ^ 1, buddy-&gt;bits[o]);</div><div class='del'>-		++buddy-&gt;num_free[o];</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	spin_unlock(&amp;buddy-&gt;lock);</div><div class='del'>-</div><div class='del'>-	*seg &lt;&lt;= order;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_buddy_free(struct hns_roce_buddy *buddy, unsigned long seg,</div><div class='del'>-				int order)</div><div class='del'>-{</div><div class='del'>-	seg &gt;&gt;= order;</div><div class='add'>+	struct hns_roce_ida *mtpt_ida = &amp;hr_dev-&gt;mr_table.mtpt_ida;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	int err;</div><div class='add'>+	int id;</div><div class='ctx'> </div><div class='del'>-	spin_lock(&amp;buddy-&gt;lock);</div><div class='del'>-</div><div class='del'>-	while (test_bit(seg ^ 1, buddy-&gt;bits[order])) {</div><div class='del'>-		clear_bit(seg ^ 1, buddy-&gt;bits[order]);</div><div class='del'>-		--buddy-&gt;num_free[order];</div><div class='del'>-		seg &gt;&gt;= 1;</div><div class='del'>-		++order;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	set_bit(seg, buddy-&gt;bits[order]);</div><div class='del'>-	++buddy-&gt;num_free[order];</div><div class='del'>-</div><div class='del'>-	spin_unlock(&amp;buddy-&gt;lock);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_buddy_init(struct hns_roce_buddy *buddy, int max_order)</div><div class='del'>-{</div><div class='del'>-	int i, s;</div><div class='del'>-</div><div class='del'>-	buddy-&gt;max_order = max_order;</div><div class='del'>-	spin_lock_init(&amp;buddy-&gt;lock);</div><div class='del'>-	buddy-&gt;bits = kcalloc(buddy-&gt;max_order + 1,</div><div class='del'>-			      sizeof(*buddy-&gt;bits),</div><div class='del'>-			      GFP_KERNEL);</div><div class='del'>-	buddy-&gt;num_free = kcalloc(buddy-&gt;max_order + 1,</div><div class='del'>-				  sizeof(*buddy-&gt;num_free),</div><div class='del'>-				  GFP_KERNEL);</div><div class='del'>-	if (!buddy-&gt;bits || !buddy-&gt;num_free)</div><div class='del'>-		goto err_out;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt;= buddy-&gt;max_order; ++i) {</div><div class='del'>-		s = BITS_TO_LONGS(1 &lt;&lt; (buddy-&gt;max_order - i));</div><div class='del'>-		buddy-&gt;bits[i] = kcalloc(s, sizeof(long), GFP_KERNEL |</div><div class='del'>-					 __GFP_NOWARN);</div><div class='del'>-		if (!buddy-&gt;bits[i]) {</div><div class='del'>-			buddy-&gt;bits[i] = vzalloc(array_size(s, sizeof(long)));</div><div class='del'>-			if (!buddy-&gt;bits[i])</div><div class='del'>-				goto err_out_free;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	set_bit(0, buddy-&gt;bits[buddy-&gt;max_order]);</div><div class='del'>-	buddy-&gt;num_free[buddy-&gt;max_order] = 1;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err_out_free:</div><div class='del'>-	for (i = 0; i &lt;= buddy-&gt;max_order; ++i)</div><div class='del'>-		kvfree(buddy-&gt;bits[i]);</div><div class='del'>-</div><div class='del'>-err_out:</div><div class='del'>-	kfree(buddy-&gt;bits);</div><div class='del'>-	kfree(buddy-&gt;num_free);</div><div class='del'>-	return -ENOMEM;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_buddy_cleanup(struct hns_roce_buddy *buddy)</div><div class='del'>-{</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt;= buddy-&gt;max_order; ++i)</div><div class='del'>-		kvfree(buddy-&gt;bits[i]);</div><div class='del'>-</div><div class='del'>-	kfree(buddy-&gt;bits);</div><div class='del'>-	kfree(buddy-&gt;num_free);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_alloc_mtt_range(struct hns_roce_dev *hr_dev, int order,</div><div class='del'>-				    unsigned long *seg, u32 mtt_type)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_mr_table *mr_table = &amp;hr_dev-&gt;mr_table;</div><div class='del'>-	struct hns_roce_hem_table *table;</div><div class='del'>-	struct hns_roce_buddy *buddy;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	switch (mtt_type) {</div><div class='del'>-	case MTT_TYPE_WQE:</div><div class='del'>-		buddy = &amp;mr_table-&gt;mtt_buddy;</div><div class='del'>-		table = &amp;mr_table-&gt;mtt_table;</div><div class='del'>-		break;</div><div class='del'>-	case MTT_TYPE_CQE:</div><div class='del'>-		buddy = &amp;mr_table-&gt;mtt_cqe_buddy;</div><div class='del'>-		table = &amp;mr_table-&gt;mtt_cqe_table;</div><div class='del'>-		break;</div><div class='del'>-	case MTT_TYPE_SRQWQE:</div><div class='del'>-		buddy = &amp;mr_table-&gt;mtt_srqwqe_buddy;</div><div class='del'>-		table = &amp;mr_table-&gt;mtt_srqwqe_table;</div><div class='del'>-		break;</div><div class='del'>-	case MTT_TYPE_IDX:</div><div class='del'>-		buddy = &amp;mr_table-&gt;mtt_idx_buddy;</div><div class='del'>-		table = &amp;mr_table-&gt;mtt_idx_table;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "Unsupport MTT table type: %d\n",</div><div class='del'>-			mtt_type);</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	/* Allocate a key for mr from mr_table */</div><div class='add'>+	id = ida_alloc_range(&amp;mtpt_ida-&gt;ida, mtpt_ida-&gt;min, mtpt_ida-&gt;max,</div><div class='add'>+			     GFP_KERNEL);</div><div class='add'>+	if (id &lt; 0) {</div><div class='add'>+		ibdev_err(ibdev, "failed to alloc id for MR key, id(%d)\n", id);</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_buddy_alloc(buddy, order, seg);</div><div class='del'>-	if (ret == -1)</div><div class='del'>-		return -1;</div><div class='add'>+	mr-&gt;key = hw_index_to_key(id); /* MR key */</div><div class='ctx'> </div><div class='del'>-	if (hns_roce_table_get_range(hr_dev, table, *seg,</div><div class='del'>-				     *seg + (1 &lt;&lt; order) - 1)) {</div><div class='del'>-		hns_roce_buddy_free(buddy, *seg, order);</div><div class='del'>-		return -1;</div><div class='add'>+	err = hns_roce_table_get(hr_dev, &amp;hr_dev-&gt;mr_table.mtpt_table,</div><div class='add'>+				 (unsigned long)id);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(ibdev, "failed to alloc mtpt, ret = %d.\n", err);</div><div class='add'>+		goto err_free_bitmap;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='add'>+err_free_bitmap:</div><div class='add'>+	ida_free(&amp;mtpt_ida-&gt;ida, id);</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift,</div><div class='del'>-		      struct hns_roce_mtt *mtt)</div><div class='add'>+static void free_mr_key(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)</div><div class='ctx'> {</div><div class='del'>-	int ret;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	/* Page num is zero, correspond to DMA memory register */</div><div class='del'>-	if (!npages) {</div><div class='del'>-		mtt-&gt;order = -1;</div><div class='del'>-		mtt-&gt;page_shift = HNS_ROCE_HEM_PAGE_SHIFT;</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Note: if page_shift is zero, FAST memory register */</div><div class='del'>-	mtt-&gt;page_shift = page_shift;</div><div class='del'>-</div><div class='del'>-	/* Compute MTT entry necessary */</div><div class='del'>-	for (mtt-&gt;order = 0, i = HNS_ROCE_MTT_ENTRY_PER_SEG; i &lt; npages;</div><div class='del'>-	     i &lt;&lt;= 1)</div><div class='del'>-		++mtt-&gt;order;</div><div class='del'>-</div><div class='del'>-	/* Allocate MTT entry */</div><div class='del'>-	ret = hns_roce_alloc_mtt_range(hr_dev, mtt-&gt;order, &amp;mtt-&gt;first_seg,</div><div class='del'>-				       mtt-&gt;mtt_type);</div><div class='del'>-	if (ret == -1)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_mr_table *mr_table = &amp;hr_dev-&gt;mr_table;</div><div class='del'>-</div><div class='del'>-	if (mtt-&gt;order &lt; 0)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	switch (mtt-&gt;mtt_type) {</div><div class='del'>-	case MTT_TYPE_WQE:</div><div class='del'>-		hns_roce_buddy_free(&amp;mr_table-&gt;mtt_buddy, mtt-&gt;first_seg,</div><div class='del'>-				    mtt-&gt;order);</div><div class='del'>-		hns_roce_table_put_range(hr_dev, &amp;mr_table-&gt;mtt_table,</div><div class='del'>-					mtt-&gt;first_seg,</div><div class='del'>-					mtt-&gt;first_seg + (1 &lt;&lt; mtt-&gt;order) - 1);</div><div class='del'>-		break;</div><div class='del'>-	case MTT_TYPE_CQE:</div><div class='del'>-		hns_roce_buddy_free(&amp;mr_table-&gt;mtt_cqe_buddy, mtt-&gt;first_seg,</div><div class='del'>-				    mtt-&gt;order);</div><div class='del'>-		hns_roce_table_put_range(hr_dev, &amp;mr_table-&gt;mtt_cqe_table,</div><div class='del'>-					mtt-&gt;first_seg,</div><div class='del'>-					mtt-&gt;first_seg + (1 &lt;&lt; mtt-&gt;order) - 1);</div><div class='del'>-		break;</div><div class='del'>-	case MTT_TYPE_SRQWQE:</div><div class='del'>-		hns_roce_buddy_free(&amp;mr_table-&gt;mtt_srqwqe_buddy, mtt-&gt;first_seg,</div><div class='del'>-				    mtt-&gt;order);</div><div class='del'>-		hns_roce_table_put_range(hr_dev, &amp;mr_table-&gt;mtt_srqwqe_table,</div><div class='del'>-					mtt-&gt;first_seg,</div><div class='del'>-					mtt-&gt;first_seg + (1 &lt;&lt; mtt-&gt;order) - 1);</div><div class='del'>-		break;</div><div class='del'>-	case MTT_TYPE_IDX:</div><div class='del'>-		hns_roce_buddy_free(&amp;mr_table-&gt;mtt_idx_buddy, mtt-&gt;first_seg,</div><div class='del'>-				    mtt-&gt;order);</div><div class='del'>-		hns_roce_table_put_range(hr_dev, &amp;mr_table-&gt;mtt_idx_table,</div><div class='del'>-					mtt-&gt;first_seg,</div><div class='del'>-					mtt-&gt;first_seg + (1 &lt;&lt; mtt-&gt;order) - 1);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		dev_err(hr_dev-&gt;dev,</div><div class='del'>-			"Unsupport mtt type %d, clean mtt failed\n",</div><div class='del'>-			mtt-&gt;mtt_type);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_mtt_cleanup);</div><div class='del'>-</div><div class='del'>-static void hns_roce_loop_free(struct hns_roce_dev *hr_dev,</div><div class='del'>-			       struct hns_roce_mr *mr, int err_loop_index,</div><div class='del'>-			       int loop_i, int loop_j)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	u32 mhop_num;</div><div class='del'>-	u32 pbl_bt_sz;</div><div class='del'>-	u64 bt_idx;</div><div class='del'>-	int i, j;</div><div class='del'>-</div><div class='del'>-	pbl_bt_sz = 1 &lt;&lt; (hr_dev-&gt;caps.pbl_ba_pg_sz + PAGE_SHIFT);</div><div class='del'>-	mhop_num = hr_dev-&gt;caps.pbl_hop_num;</div><div class='del'>-</div><div class='del'>-	i = loop_i;</div><div class='del'>-	if (mhop_num == 3 &amp;&amp; err_loop_index == 2) {</div><div class='del'>-		for (; i &gt;= 0; i--) {</div><div class='del'>-			dma_free_coherent(dev, pbl_bt_sz, mr-&gt;pbl_bt_l1[i],</div><div class='del'>-					  mr-&gt;pbl_l1_dma_addr[i]);</div><div class='del'>-</div><div class='del'>-			for (j = 0; j &lt; pbl_bt_sz / 8; j++) {</div><div class='del'>-				if (i == loop_i &amp;&amp; j &gt;= loop_j)</div><div class='del'>-					break;</div><div class='del'>-</div><div class='del'>-				bt_idx = i * pbl_bt_sz / 8 + j;</div><div class='del'>-				dma_free_coherent(dev, pbl_bt_sz,</div><div class='del'>-						  mr-&gt;pbl_bt_l2[bt_idx],</div><div class='del'>-						  mr-&gt;pbl_l2_dma_addr[bt_idx]);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	} else if (mhop_num == 3 &amp;&amp; err_loop_index == 1) {</div><div class='del'>-		for (i -= 1; i &gt;= 0; i--) {</div><div class='del'>-			dma_free_coherent(dev, pbl_bt_sz, mr-&gt;pbl_bt_l1[i],</div><div class='del'>-					  mr-&gt;pbl_l1_dma_addr[i]);</div><div class='del'>-</div><div class='del'>-			for (j = 0; j &lt; pbl_bt_sz / 8; j++) {</div><div class='del'>-				bt_idx = i * pbl_bt_sz / 8 + j;</div><div class='del'>-				dma_free_coherent(dev, pbl_bt_sz,</div><div class='del'>-						  mr-&gt;pbl_bt_l2[bt_idx],</div><div class='del'>-						  mr-&gt;pbl_l2_dma_addr[bt_idx]);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	} else if (mhop_num == 2 &amp;&amp; err_loop_index == 1) {</div><div class='del'>-		for (i -= 1; i &gt;= 0; i--)</div><div class='del'>-			dma_free_coherent(dev, pbl_bt_sz, mr-&gt;pbl_bt_l1[i],</div><div class='del'>-					  mr-&gt;pbl_l1_dma_addr[i]);</div><div class='del'>-	} else {</div><div class='del'>-		dev_warn(dev, "not support: mhop_num=%d, err_loop_index=%d.",</div><div class='del'>-			 mhop_num, err_loop_index);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='add'>+	unsigned long obj = key_to_hw_index(mr-&gt;key);</div><div class='ctx'> </div><div class='del'>-	dma_free_coherent(dev, pbl_bt_sz, mr-&gt;pbl_bt_l0, mr-&gt;pbl_l0_dma_addr);</div><div class='del'>-	mr-&gt;pbl_bt_l0 = NULL;</div><div class='del'>-	mr-&gt;pbl_l0_dma_addr = 0;</div><div class='add'>+	hns_roce_table_put(hr_dev, &amp;hr_dev-&gt;mr_table.mtpt_table, obj);</div><div class='add'>+	ida_free(&amp;hr_dev-&gt;mr_table.mtpt_ida.ida, (int)obj);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/* PBL multi hop addressing */</div><div class='del'>-static int hns_roce_mhop_alloc(struct hns_roce_dev *hr_dev, int npages,</div><div class='del'>-			       struct hns_roce_mr *mr)</div><div class='add'>+static int alloc_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr,</div><div class='add'>+			struct ib_udata *udata, u64 start)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	int mr_alloc_done = 0;</div><div class='del'>-	int npages_allocated;</div><div class='del'>-	int i = 0, j = 0;</div><div class='del'>-	u32 pbl_bt_sz;</div><div class='del'>-	u32 mhop_num;</div><div class='del'>-	u64 pbl_last_bt_num;</div><div class='del'>-	u64 pbl_bt_cnt = 0;</div><div class='del'>-	u64 bt_idx;</div><div class='del'>-	u64 size;</div><div class='del'>-</div><div class='del'>-	mhop_num = (mr-&gt;type == MR_TYPE_FRMR ? 1 : hr_dev-&gt;caps.pbl_hop_num);</div><div class='del'>-	pbl_bt_sz = 1 &lt;&lt; (hr_dev-&gt;caps.pbl_ba_pg_sz + PAGE_SHIFT);</div><div class='del'>-	pbl_last_bt_num = (npages + pbl_bt_sz / 8 - 1) / (pbl_bt_sz / 8);</div><div class='del'>-</div><div class='del'>-	if (mhop_num == HNS_ROCE_HOP_NUM_0)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	/* hop_num = 1 */</div><div class='del'>-	if (mhop_num == 1) {</div><div class='del'>-		if (npages &gt; pbl_bt_sz / 8) {</div><div class='del'>-			dev_err(dev, "npages %d is larger than buf_pg_sz!",</div><div class='del'>-				npages);</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-		mr-&gt;pbl_buf = dma_alloc_coherent(dev, npages * 8,</div><div class='del'>-						 &amp;(mr-&gt;pbl_dma_addr),</div><div class='del'>-						 GFP_KERNEL);</div><div class='del'>-		if (!mr-&gt;pbl_buf)</div><div class='del'>-			return -ENOMEM;</div><div class='del'>-</div><div class='del'>-		mr-&gt;pbl_size = npages;</div><div class='del'>-		mr-&gt;pbl_ba = mr-&gt;pbl_dma_addr;</div><div class='del'>-		mr-&gt;pbl_hop_num = mhop_num;</div><div class='del'>-		mr-&gt;pbl_ba_pg_sz = hr_dev-&gt;caps.pbl_ba_pg_sz;</div><div class='del'>-		mr-&gt;pbl_buf_pg_sz = hr_dev-&gt;caps.pbl_buf_pg_sz;</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mr-&gt;pbl_l1_dma_addr = kcalloc(pbl_bt_sz / 8,</div><div class='del'>-				      sizeof(*mr-&gt;pbl_l1_dma_addr),</div><div class='del'>-				      GFP_KERNEL);</div><div class='del'>-	if (!mr-&gt;pbl_l1_dma_addr)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	mr-&gt;pbl_bt_l1 = kcalloc(pbl_bt_sz / 8, sizeof(*mr-&gt;pbl_bt_l1),</div><div class='del'>-				GFP_KERNEL);</div><div class='del'>-	if (!mr-&gt;pbl_bt_l1)</div><div class='del'>-		goto err_kcalloc_bt_l1;</div><div class='del'>-</div><div class='del'>-	if (mhop_num == 3) {</div><div class='del'>-		mr-&gt;pbl_l2_dma_addr = kcalloc(pbl_last_bt_num,</div><div class='del'>-					      sizeof(*mr-&gt;pbl_l2_dma_addr),</div><div class='del'>-					      GFP_KERNEL);</div><div class='del'>-		if (!mr-&gt;pbl_l2_dma_addr)</div><div class='del'>-			goto err_kcalloc_l2_dma;</div><div class='del'>-</div><div class='del'>-		mr-&gt;pbl_bt_l2 = kcalloc(pbl_last_bt_num,</div><div class='del'>-					sizeof(*mr-&gt;pbl_bt_l2),</div><div class='del'>-					GFP_KERNEL);</div><div class='del'>-		if (!mr-&gt;pbl_bt_l2)</div><div class='del'>-			goto err_kcalloc_bt_l2;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* alloc L0 BT */</div><div class='del'>-	mr-&gt;pbl_bt_l0 = dma_alloc_coherent(dev, pbl_bt_sz,</div><div class='del'>-					   &amp;(mr-&gt;pbl_l0_dma_addr),</div><div class='del'>-					   GFP_KERNEL);</div><div class='del'>-	if (!mr-&gt;pbl_bt_l0)</div><div class='del'>-		goto err_dma_alloc_l0;</div><div class='del'>-</div><div class='del'>-	if (mhop_num == 2) {</div><div class='del'>-		/* alloc L1 BT */</div><div class='del'>-		for (i = 0; i &lt; pbl_bt_sz / 8; i++) {</div><div class='del'>-			if (pbl_bt_cnt + 1 &lt; pbl_last_bt_num) {</div><div class='del'>-				size = pbl_bt_sz;</div><div class='del'>-			} else {</div><div class='del'>-				npages_allocated = i * (pbl_bt_sz / 8);</div><div class='del'>-				size = (npages - npages_allocated) * 8;</div><div class='del'>-			}</div><div class='del'>-			mr-&gt;pbl_bt_l1[i] = dma_alloc_coherent(dev, size,</div><div class='del'>-						    &amp;(mr-&gt;pbl_l1_dma_addr[i]),</div><div class='del'>-						    GFP_KERNEL);</div><div class='del'>-			if (!mr-&gt;pbl_bt_l1[i]) {</div><div class='del'>-				hns_roce_loop_free(hr_dev, mr, 1, i, 0);</div><div class='del'>-				goto err_dma_alloc_l0;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			*(mr-&gt;pbl_bt_l0 + i) = mr-&gt;pbl_l1_dma_addr[i];</div><div class='del'>-</div><div class='del'>-			pbl_bt_cnt++;</div><div class='del'>-			if (pbl_bt_cnt &gt;= pbl_last_bt_num)</div><div class='del'>-				break;</div><div class='del'>-		}</div><div class='del'>-	} else if (mhop_num == 3) {</div><div class='del'>-		/* alloc L1, L2 BT */</div><div class='del'>-		for (i = 0; i &lt; pbl_bt_sz / 8; i++) {</div><div class='del'>-			mr-&gt;pbl_bt_l1[i] = dma_alloc_coherent(dev, pbl_bt_sz,</div><div class='del'>-						    &amp;(mr-&gt;pbl_l1_dma_addr[i]),</div><div class='del'>-						    GFP_KERNEL);</div><div class='del'>-			if (!mr-&gt;pbl_bt_l1[i]) {</div><div class='del'>-				hns_roce_loop_free(hr_dev, mr, 1, i, 0);</div><div class='del'>-				goto err_dma_alloc_l0;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			*(mr-&gt;pbl_bt_l0 + i) = mr-&gt;pbl_l1_dma_addr[i];</div><div class='del'>-</div><div class='del'>-			for (j = 0; j &lt; pbl_bt_sz / 8; j++) {</div><div class='del'>-				bt_idx = i * pbl_bt_sz / 8 + j;</div><div class='del'>-</div><div class='del'>-				if (pbl_bt_cnt + 1 &lt; pbl_last_bt_num) {</div><div class='del'>-					size = pbl_bt_sz;</div><div class='del'>-				} else {</div><div class='del'>-					npages_allocated = bt_idx *</div><div class='del'>-							   (pbl_bt_sz / 8);</div><div class='del'>-					size = (npages - npages_allocated) * 8;</div><div class='del'>-				}</div><div class='del'>-				mr-&gt;pbl_bt_l2[bt_idx] = dma_alloc_coherent(</div><div class='del'>-					      dev, size,</div><div class='del'>-					      &amp;(mr-&gt;pbl_l2_dma_addr[bt_idx]),</div><div class='del'>-					      GFP_KERNEL);</div><div class='del'>-				if (!mr-&gt;pbl_bt_l2[bt_idx]) {</div><div class='del'>-					hns_roce_loop_free(hr_dev, mr, 2, i, j);</div><div class='del'>-					goto err_dma_alloc_l0;</div><div class='del'>-				}</div><div class='del'>-</div><div class='del'>-				*(mr-&gt;pbl_bt_l1[i] + j) =</div><div class='del'>-						mr-&gt;pbl_l2_dma_addr[bt_idx];</div><div class='del'>-</div><div class='del'>-				pbl_bt_cnt++;</div><div class='del'>-				if (pbl_bt_cnt &gt;= pbl_last_bt_num) {</div><div class='del'>-					mr_alloc_done = 1;</div><div class='del'>-					break;</div><div class='del'>-				}</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			if (mr_alloc_done)</div><div class='del'>-				break;</div><div class='del'>-		}</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	bool is_fast = mr-&gt;type == MR_TYPE_FRMR;</div><div class='add'>+	struct hns_roce_buf_attr buf_attr = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mr-&gt;pbl_hop_num = is_fast ? 1 : hr_dev-&gt;caps.pbl_hop_num;</div><div class='add'>+	buf_attr.page_shift = is_fast ? PAGE_SHIFT :</div><div class='add'>+			      hr_dev-&gt;caps.pbl_buf_pg_sz + PAGE_SHIFT;</div><div class='add'>+	buf_attr.region[0].size = mr-&gt;size;</div><div class='add'>+	buf_attr.region[0].hopnum = mr-&gt;pbl_hop_num;</div><div class='add'>+	buf_attr.region_count = 1;</div><div class='add'>+	buf_attr.user_access = mr-&gt;access;</div><div class='add'>+	/* fast MR's buffer is alloced before mapping, not at creation */</div><div class='add'>+	buf_attr.mtt_only = is_fast;</div><div class='add'>+	buf_attr.iova = mr-&gt;iova;</div><div class='add'>+	/* pagesize and hopnum is fixed for fast MR */</div><div class='add'>+	buf_attr.adaptive = !is_fast;</div><div class='add'>+	buf_attr.type = MTR_PBL;</div><div class='add'>+</div><div class='add'>+	err = hns_roce_mtr_create(hr_dev, &amp;mr-&gt;pbl_mtr, &amp;buf_attr,</div><div class='add'>+				  hr_dev-&gt;caps.pbl_ba_pg_sz + PAGE_SHIFT,</div><div class='add'>+				  udata, start);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(ibdev, "failed to alloc pbl mtr, ret = %d.\n", err);</div><div class='add'>+		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	mr-&gt;l0_chunk_last_num = i + 1;</div><div class='del'>-	if (mhop_num == 3)</div><div class='del'>-		mr-&gt;l1_chunk_last_num = j + 1;</div><div class='del'>-</div><div class='del'>-	mr-&gt;pbl_size = npages;</div><div class='del'>-	mr-&gt;pbl_ba = mr-&gt;pbl_l0_dma_addr;</div><div class='del'>-	mr-&gt;pbl_hop_num = hr_dev-&gt;caps.pbl_hop_num;</div><div class='del'>-	mr-&gt;pbl_ba_pg_sz = hr_dev-&gt;caps.pbl_ba_pg_sz;</div><div class='del'>-	mr-&gt;pbl_buf_pg_sz = hr_dev-&gt;caps.pbl_buf_pg_sz;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err_dma_alloc_l0:</div><div class='del'>-	kfree(mr-&gt;pbl_bt_l2);</div><div class='del'>-	mr-&gt;pbl_bt_l2 = NULL;</div><div class='del'>-</div><div class='del'>-err_kcalloc_bt_l2:</div><div class='del'>-	kfree(mr-&gt;pbl_l2_dma_addr);</div><div class='del'>-	mr-&gt;pbl_l2_dma_addr = NULL;</div><div class='del'>-</div><div class='del'>-err_kcalloc_l2_dma:</div><div class='del'>-	kfree(mr-&gt;pbl_bt_l1);</div><div class='del'>-	mr-&gt;pbl_bt_l1 = NULL;</div><div class='del'>-</div><div class='del'>-err_kcalloc_bt_l1:</div><div class='del'>-	kfree(mr-&gt;pbl_l1_dma_addr);</div><div class='del'>-	mr-&gt;pbl_l1_dma_addr = NULL;</div><div class='add'>+	mr-&gt;npages = mr-&gt;pbl_mtr.hem_cfg.buf_pg_count;</div><div class='add'>+	mr-&gt;pbl_hop_num = buf_attr.region[0].hopnum;</div><div class='ctx'> </div><div class='del'>-	return -ENOMEM;</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova,</div><div class='del'>-			     u64 size, u32 access, int npages,</div><div class='del'>-			     struct hns_roce_mr *mr)</div><div class='add'>+static void free_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	unsigned long index = 0;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	/* Allocate a key for mr from mr_table */</div><div class='del'>-	ret = hns_roce_bitmap_alloc(&amp;hr_dev-&gt;mr_table.mtpt_bitmap, &amp;index);</div><div class='del'>-	if (ret == -1)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	mr-&gt;iova = iova;			/* MR va starting addr */</div><div class='del'>-	mr-&gt;size = size;			/* MR addr range */</div><div class='del'>-	mr-&gt;pd = pd;				/* MR num */</div><div class='del'>-	mr-&gt;access = access;			/* MR access permit */</div><div class='del'>-	mr-&gt;enabled = 0;			/* MR active status */</div><div class='del'>-	mr-&gt;key = hw_index_to_key(index);	/* MR key */</div><div class='del'>-</div><div class='del'>-	if (size == ~0ull) {</div><div class='del'>-		mr-&gt;pbl_buf = NULL;</div><div class='del'>-		mr-&gt;pbl_dma_addr = 0;</div><div class='del'>-		/* PBL multi-hop addressing parameters */</div><div class='del'>-		mr-&gt;pbl_bt_l2 = NULL;</div><div class='del'>-		mr-&gt;pbl_bt_l1 = NULL;</div><div class='del'>-		mr-&gt;pbl_bt_l0 = NULL;</div><div class='del'>-		mr-&gt;pbl_l2_dma_addr = NULL;</div><div class='del'>-		mr-&gt;pbl_l1_dma_addr = NULL;</div><div class='del'>-		mr-&gt;pbl_l0_dma_addr = 0;</div><div class='del'>-	} else {</div><div class='del'>-		if (!hr_dev-&gt;caps.pbl_hop_num) {</div><div class='del'>-			mr-&gt;pbl_buf = dma_alloc_coherent(dev, npages * 8,</div><div class='del'>-							 &amp;(mr-&gt;pbl_dma_addr),</div><div class='del'>-							 GFP_KERNEL);</div><div class='del'>-			if (!mr-&gt;pbl_buf)</div><div class='del'>-				return -ENOMEM;</div><div class='del'>-		} else {</div><div class='del'>-			ret = hns_roce_mhop_alloc(hr_dev, npages, mr);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='add'>+	hns_roce_mtr_destroy(hr_dev, &amp;mr-&gt;pbl_mtr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev,</div><div class='del'>-			       struct hns_roce_mr *mr)</div><div class='add'>+static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	int npages_allocated;</div><div class='del'>-	int npages;</div><div class='del'>-	int i, j;</div><div class='del'>-	u32 pbl_bt_sz;</div><div class='del'>-	u32 mhop_num;</div><div class='del'>-	u64 bt_idx;</div><div class='del'>-</div><div class='del'>-	npages = mr-&gt;pbl_size;</div><div class='del'>-	pbl_bt_sz = 1 &lt;&lt; (hr_dev-&gt;caps.pbl_ba_pg_sz + PAGE_SHIFT);</div><div class='del'>-	mhop_num = (mr-&gt;type == MR_TYPE_FRMR) ? 1 : hr_dev-&gt;caps.pbl_hop_num;</div><div class='del'>-</div><div class='del'>-	if (mhop_num == HNS_ROCE_HOP_NUM_0)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	/* hop_num = 1 */</div><div class='del'>-	if (mhop_num == 1) {</div><div class='del'>-		dma_free_coherent(dev, (unsigned int)(npages * 8),</div><div class='del'>-				  mr-&gt;pbl_buf, mr-&gt;pbl_dma_addr);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	dma_free_coherent(dev, pbl_bt_sz, mr-&gt;pbl_bt_l0,</div><div class='del'>-			  mr-&gt;pbl_l0_dma_addr);</div><div class='del'>-</div><div class='del'>-	if (mhop_num == 2) {</div><div class='del'>-		for (i = 0; i &lt; mr-&gt;l0_chunk_last_num; i++) {</div><div class='del'>-			if (i == mr-&gt;l0_chunk_last_num - 1) {</div><div class='del'>-				npages_allocated = i * (pbl_bt_sz / 8);</div><div class='del'>-</div><div class='del'>-				dma_free_coherent(dev,</div><div class='del'>-					      (npages - npages_allocated) * 8,</div><div class='del'>-					      mr-&gt;pbl_bt_l1[i],</div><div class='del'>-					      mr-&gt;pbl_l1_dma_addr[i]);</div><div class='del'>-</div><div class='del'>-				break;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			dma_free_coherent(dev, pbl_bt_sz, mr-&gt;pbl_bt_l1[i],</div><div class='del'>-					  mr-&gt;pbl_l1_dma_addr[i]);</div><div class='del'>-		}</div><div class='del'>-	} else if (mhop_num == 3) {</div><div class='del'>-		for (i = 0; i &lt; mr-&gt;l0_chunk_last_num; i++) {</div><div class='del'>-			dma_free_coherent(dev, pbl_bt_sz, mr-&gt;pbl_bt_l1[i],</div><div class='del'>-					  mr-&gt;pbl_l1_dma_addr[i]);</div><div class='del'>-</div><div class='del'>-			for (j = 0; j &lt; pbl_bt_sz / 8; j++) {</div><div class='del'>-				bt_idx = i * (pbl_bt_sz / 8) + j;</div><div class='del'>-</div><div class='del'>-				if ((i == mr-&gt;l0_chunk_last_num - 1)</div><div class='del'>-				    &amp;&amp; j == mr-&gt;l1_chunk_last_num - 1) {</div><div class='del'>-					npages_allocated = bt_idx *</div><div class='del'>-							   (pbl_bt_sz / 8);</div><div class='del'>-</div><div class='del'>-					dma_free_coherent(dev,</div><div class='del'>-					      (npages - npages_allocated) * 8,</div><div class='del'>-					      mr-&gt;pbl_bt_l2[bt_idx],</div><div class='del'>-					      mr-&gt;pbl_l2_dma_addr[bt_idx]);</div><div class='del'>-</div><div class='del'>-					break;</div><div class='del'>-				}</div><div class='del'>-</div><div class='del'>-				dma_free_coherent(dev, pbl_bt_sz,</div><div class='del'>-						mr-&gt;pbl_bt_l2[bt_idx],</div><div class='del'>-						mr-&gt;pbl_l2_dma_addr[bt_idx]);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	kfree(mr-&gt;pbl_bt_l1);</div><div class='del'>-	kfree(mr-&gt;pbl_l1_dma_addr);</div><div class='del'>-	mr-&gt;pbl_bt_l1 = NULL;</div><div class='del'>-	mr-&gt;pbl_l1_dma_addr = NULL;</div><div class='del'>-	if (mhop_num == 3) {</div><div class='del'>-		kfree(mr-&gt;pbl_bt_l2);</div><div class='del'>-		kfree(mr-&gt;pbl_l2_dma_addr);</div><div class='del'>-		mr-&gt;pbl_bt_l2 = NULL;</div><div class='del'>-		mr-&gt;pbl_l2_dma_addr = NULL;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_mr_free(struct hns_roce_dev *hr_dev,</div><div class='del'>-			     struct hns_roce_mr *mr)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	int npages = 0;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (mr-&gt;enabled) {</div><div class='del'>-		ret = hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mr-&gt;key)</div><div class='del'>-					 &amp; (hr_dev-&gt;caps.num_mtpts - 1));</div><div class='add'>+		ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_MPT,</div><div class='add'>+					      key_to_hw_index(mr-&gt;key) &amp;</div><div class='add'>+					      (hr_dev-&gt;caps.num_mtpts - 1));</div><div class='ctx'> 		if (ret)</div><div class='del'>-			dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (mr-&gt;size != ~0ULL) {</div><div class='del'>-		if (mr-&gt;type == MR_TYPE_MR)</div><div class='del'>-			npages = ib_umem_page_count(mr-&gt;umem);</div><div class='del'>-</div><div class='del'>-		if (!hr_dev-&gt;caps.pbl_hop_num)</div><div class='del'>-			dma_free_coherent(dev, (unsigned int)(npages * 8),</div><div class='del'>-					  mr-&gt;pbl_buf, mr-&gt;pbl_dma_addr);</div><div class='del'>-		else</div><div class='del'>-			hns_roce_mhop_free(hr_dev, mr);</div><div class='add'>+			ibdev_warn_ratelimited(ibdev, "failed to destroy mpt, ret = %d.\n",</div><div class='add'>+					       ret);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (mr-&gt;enabled)</div><div class='del'>-		hns_roce_table_put(hr_dev, &amp;hr_dev-&gt;mr_table.mtpt_table,</div><div class='del'>-				   key_to_hw_index(mr-&gt;key));</div><div class='del'>-</div><div class='del'>-	hns_roce_bitmap_free(&amp;hr_dev-&gt;mr_table.mtpt_bitmap,</div><div class='del'>-			     key_to_hw_index(mr-&gt;key), BITMAP_NO_RR);</div><div class='add'>+	free_mr_pbl(hr_dev, mr);</div><div class='add'>+	free_mr_key(hr_dev, mr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 			      struct hns_roce_mr *mr)</div><div class='ctx'> {</div><div class='del'>-	int ret;</div><div class='ctx'> 	unsigned long mtpt_idx = key_to_hw_index(mr-&gt;key);</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='ctx'> 	struct hns_roce_cmd_mailbox *mailbox;</div><div class='del'>-	struct hns_roce_mr_table *mr_table = &amp;hr_dev-&gt;mr_table;</div><div class='del'>-</div><div class='del'>-	/* Prepare HEM entry memory */</div><div class='del'>-	ret = hns_roce_table_get(hr_dev, &amp;mr_table-&gt;mtpt_table, mtpt_idx);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='add'>+	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	/* Allocate mailbox memory */</div><div class='ctx'> 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='del'>-	if (IS_ERR(mailbox)) {</div><div class='del'>-		ret = PTR_ERR(mailbox);</div><div class='del'>-		goto err_table;</div><div class='del'>-	}</div><div class='add'>+	if (IS_ERR(mailbox))</div><div class='add'>+		return PTR_ERR(mailbox);</div><div class='ctx'> </div><div class='add'>+	trace_hns_mr(mr);</div><div class='ctx'> 	if (mr-&gt;type != MR_TYPE_FRMR)</div><div class='del'>-		ret = hr_dev-&gt;hw-&gt;write_mtpt(mailbox-&gt;buf, mr, mtpt_idx);</div><div class='add'>+		ret = hr_dev-&gt;hw-&gt;write_mtpt(hr_dev, mailbox-&gt;buf, mr);</div><div class='ctx'> 	else</div><div class='ctx'> 		ret = hr_dev-&gt;hw-&gt;frmr_write_mtpt(mailbox-&gt;buf, mr);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "Write mtpt fail!\n");</div><div class='add'>+		dev_err(dev, "failed to write mtpt, ret = %d.\n", ret);</div><div class='ctx'> 		goto err_page;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_sw2hw_mpt(hr_dev, mailbox,</div><div class='del'>-				 mtpt_idx &amp; (hr_dev-&gt;caps.num_mtpts - 1));</div><div class='add'>+	ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT,</div><div class='add'>+				     mtpt_idx &amp; (hr_dev-&gt;caps.num_mtpts - 1));</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "SW2HW_MPT failed (%d)\n", ret);</div><div class='add'>+		dev_err(dev, "failed to create mpt, ret = %d.\n", ret);</div><div class='ctx'> 		goto err_page;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;enabled = 1;</div><div class='del'>-	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='ctx'> </div><div class='ctx'> err_page:</div><div class='ctx'> 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='ctx'> </div><div class='del'>-err_table:</div><div class='del'>-	hns_roce_table_put(hr_dev, &amp;mr_table-&gt;mtpt_table, mtpt_idx);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_write_mtt_chunk(struct hns_roce_dev *hr_dev,</div><div class='del'>-				    struct hns_roce_mtt *mtt, u32 start_index,</div><div class='del'>-				    u32 npages, u64 *page_list)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_hem_table *table;</div><div class='del'>-	dma_addr_t dma_handle;</div><div class='del'>-	__le64 *mtts;</div><div class='del'>-	u32 bt_page_size;</div><div class='del'>-	u32 i;</div><div class='del'>-</div><div class='del'>-	switch (mtt-&gt;mtt_type) {</div><div class='del'>-	case MTT_TYPE_WQE:</div><div class='del'>-		table = &amp;hr_dev-&gt;mr_table.mtt_table;</div><div class='del'>-		bt_page_size = 1 &lt;&lt; (hr_dev-&gt;caps.mtt_ba_pg_sz + PAGE_SHIFT);</div><div class='del'>-		break;</div><div class='del'>-	case MTT_TYPE_CQE:</div><div class='del'>-		table = &amp;hr_dev-&gt;mr_table.mtt_cqe_table;</div><div class='del'>-		bt_page_size = 1 &lt;&lt; (hr_dev-&gt;caps.cqe_ba_pg_sz + PAGE_SHIFT);</div><div class='del'>-		break;</div><div class='del'>-	case MTT_TYPE_SRQWQE:</div><div class='del'>-		table = &amp;hr_dev-&gt;mr_table.mtt_srqwqe_table;</div><div class='del'>-		bt_page_size = 1 &lt;&lt; (hr_dev-&gt;caps.srqwqe_ba_pg_sz + PAGE_SHIFT);</div><div class='del'>-		break;</div><div class='del'>-	case MTT_TYPE_IDX:</div><div class='del'>-		table = &amp;hr_dev-&gt;mr_table.mtt_idx_table;</div><div class='del'>-		bt_page_size = 1 &lt;&lt; (hr_dev-&gt;caps.idx_ba_pg_sz + PAGE_SHIFT);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* All MTTs must fit in the same page */</div><div class='del'>-	if (start_index / (bt_page_size / sizeof(u64)) !=</div><div class='del'>-		(start_index + npages - 1) / (bt_page_size / sizeof(u64)))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	if (start_index &amp; (HNS_ROCE_MTT_ENTRY_PER_SEG - 1))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	mtts = hns_roce_table_find(hr_dev, table,</div><div class='del'>-				mtt-&gt;first_seg +</div><div class='del'>-				start_index / HNS_ROCE_MTT_ENTRY_PER_SEG,</div><div class='del'>-				&amp;dma_handle);</div><div class='del'>-	if (!mtts)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	/* Save page addr, low 12 bits : 0 */</div><div class='del'>-	for (i = 0; i &lt; npages; ++i) {</div><div class='del'>-		if (!hr_dev-&gt;caps.mtt_hop_num)</div><div class='del'>-			mtts[i] = cpu_to_le64(page_list[i] &gt;&gt; PAGE_ADDR_SHIFT);</div><div class='del'>-		else</div><div class='del'>-			mtts[i] = cpu_to_le64(page_list[i]);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_write_mtt(struct hns_roce_dev *hr_dev,</div><div class='del'>-			      struct hns_roce_mtt *mtt, u32 start_index,</div><div class='del'>-			      u32 npages, u64 *page_list)</div><div class='add'>+void hns_roce_init_mr_table(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	int chunk;</div><div class='del'>-	int ret;</div><div class='del'>-	u32 bt_page_size;</div><div class='del'>-</div><div class='del'>-	if (mtt-&gt;order &lt; 0)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	switch (mtt-&gt;mtt_type) {</div><div class='del'>-	case MTT_TYPE_WQE:</div><div class='del'>-		bt_page_size = 1 &lt;&lt; (hr_dev-&gt;caps.mtt_ba_pg_sz + PAGE_SHIFT);</div><div class='del'>-		break;</div><div class='del'>-	case MTT_TYPE_CQE:</div><div class='del'>-		bt_page_size = 1 &lt;&lt; (hr_dev-&gt;caps.cqe_ba_pg_sz + PAGE_SHIFT);</div><div class='del'>-		break;</div><div class='del'>-	case MTT_TYPE_SRQWQE:</div><div class='del'>-		bt_page_size = 1 &lt;&lt; (hr_dev-&gt;caps.srqwqe_ba_pg_sz + PAGE_SHIFT);</div><div class='del'>-		break;</div><div class='del'>-	case MTT_TYPE_IDX:</div><div class='del'>-		bt_page_size = 1 &lt;&lt; (hr_dev-&gt;caps.idx_ba_pg_sz + PAGE_SHIFT);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		dev_err(hr_dev-&gt;dev,</div><div class='del'>-			"Unsupport mtt type %d, write mtt failed\n",</div><div class='del'>-			mtt-&gt;mtt_type);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='add'>+	struct hns_roce_ida *mtpt_ida = &amp;hr_dev-&gt;mr_table.mtpt_ida;</div><div class='ctx'> </div><div class='del'>-	while (npages &gt; 0) {</div><div class='del'>-		chunk = min_t(int, bt_page_size / sizeof(u64), npages);</div><div class='del'>-</div><div class='del'>-		ret = hns_roce_write_mtt_chunk(hr_dev, mtt, start_index, chunk,</div><div class='del'>-					       page_list);</div><div class='del'>-		if (ret)</div><div class='del'>-			return ret;</div><div class='del'>-</div><div class='del'>-		npages -= chunk;</div><div class='del'>-		start_index += chunk;</div><div class='del'>-		page_list += chunk;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev,</div><div class='del'>-			   struct hns_roce_mtt *mtt, struct hns_roce_buf *buf)</div><div class='del'>-{</div><div class='del'>-	u64 *page_list;</div><div class='del'>-	int ret;</div><div class='del'>-	u32 i;</div><div class='del'>-</div><div class='del'>-	page_list = kmalloc_array(buf-&gt;npages, sizeof(*page_list), GFP_KERNEL);</div><div class='del'>-	if (!page_list)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; buf-&gt;npages; ++i) {</div><div class='del'>-		if (buf-&gt;nbufs == 1)</div><div class='del'>-			page_list[i] = buf-&gt;direct.map + (i &lt;&lt; buf-&gt;page_shift);</div><div class='del'>-		else</div><div class='del'>-			page_list[i] = buf-&gt;page_list[i].map;</div><div class='del'>-</div><div class='del'>-	}</div><div class='del'>-	ret = hns_roce_write_mtt(hr_dev, mtt, 0, buf-&gt;npages, page_list);</div><div class='del'>-</div><div class='del'>-	kfree(page_list);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_mr_table *mr_table = &amp;hr_dev-&gt;mr_table;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_bitmap_init(&amp;mr_table-&gt;mtpt_bitmap,</div><div class='del'>-				   hr_dev-&gt;caps.num_mtpts,</div><div class='del'>-				   hr_dev-&gt;caps.num_mtpts - 1,</div><div class='del'>-				   hr_dev-&gt;caps.reserved_mrws, 0);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	ret = hns_roce_buddy_init(&amp;mr_table-&gt;mtt_buddy,</div><div class='del'>-				  ilog2(hr_dev-&gt;caps.num_mtt_segs));</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_buddy;</div><div class='del'>-</div><div class='del'>-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) {</div><div class='del'>-		ret = hns_roce_buddy_init(&amp;mr_table-&gt;mtt_cqe_buddy,</div><div class='del'>-					  ilog2(hr_dev-&gt;caps.num_cqe_segs));</div><div class='del'>-		if (ret)</div><div class='del'>-			goto err_buddy_cqe;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (hr_dev-&gt;caps.num_srqwqe_segs) {</div><div class='del'>-		ret = hns_roce_buddy_init(&amp;mr_table-&gt;mtt_srqwqe_buddy,</div><div class='del'>-					  ilog2(hr_dev-&gt;caps.num_srqwqe_segs));</div><div class='del'>-		if (ret)</div><div class='del'>-			goto err_buddy_srqwqe;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (hr_dev-&gt;caps.num_idx_segs) {</div><div class='del'>-		ret = hns_roce_buddy_init(&amp;mr_table-&gt;mtt_idx_buddy,</div><div class='del'>-					  ilog2(hr_dev-&gt;caps.num_idx_segs));</div><div class='del'>-		if (ret)</div><div class='del'>-			goto err_buddy_idx;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err_buddy_idx:</div><div class='del'>-	if (hr_dev-&gt;caps.num_srqwqe_segs)</div><div class='del'>-		hns_roce_buddy_cleanup(&amp;mr_table-&gt;mtt_srqwqe_buddy);</div><div class='del'>-</div><div class='del'>-err_buddy_srqwqe:</div><div class='del'>-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))</div><div class='del'>-		hns_roce_buddy_cleanup(&amp;mr_table-&gt;mtt_cqe_buddy);</div><div class='del'>-</div><div class='del'>-err_buddy_cqe:</div><div class='del'>-	hns_roce_buddy_cleanup(&amp;mr_table-&gt;mtt_buddy);</div><div class='del'>-</div><div class='del'>-err_buddy:</div><div class='del'>-	hns_roce_bitmap_cleanup(&amp;mr_table-&gt;mtpt_bitmap);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	struct hns_roce_mr_table *mr_table = &amp;hr_dev-&gt;mr_table;</div><div class='del'>-</div><div class='del'>-	if (hr_dev-&gt;caps.num_idx_segs)</div><div class='del'>-		hns_roce_buddy_cleanup(&amp;mr_table-&gt;mtt_idx_buddy);</div><div class='del'>-	if (hr_dev-&gt;caps.num_srqwqe_segs)</div><div class='del'>-		hns_roce_buddy_cleanup(&amp;mr_table-&gt;mtt_srqwqe_buddy);</div><div class='del'>-	hns_roce_buddy_cleanup(&amp;mr_table-&gt;mtt_buddy);</div><div class='del'>-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))</div><div class='del'>-		hns_roce_buddy_cleanup(&amp;mr_table-&gt;mtt_cqe_buddy);</div><div class='del'>-	hns_roce_bitmap_cleanup(&amp;mr_table-&gt;mtpt_bitmap);</div><div class='add'>+	ida_init(&amp;mtpt_ida-&gt;ida);</div><div class='add'>+	mtpt_ida-&gt;max = hr_dev-&gt;caps.num_mtpts - 1;</div><div class='add'>+	mtpt_ida-&gt;min = hr_dev-&gt;caps.reserved_mrws;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc)</div><div class='ctx'> {</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(pd-&gt;device);</div><div class='ctx'> 	struct hns_roce_mr *mr;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	mr = kmalloc(sizeof(*mr), GFP_KERNEL);</div><div class='del'>-	if (mr == NULL)</div><div class='add'>+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='add'>+	if (!mr)</div><div class='ctx'> 		return  ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;type = MR_TYPE_DMA;</div><div class='add'>+	mr-&gt;pd = to_hr_pd(pd)-&gt;pdn;</div><div class='add'>+	mr-&gt;access = acc;</div><div class='ctx'> </div><div class='ctx'> 	/* Allocate memory region key */</div><div class='del'>-	ret = hns_roce_mr_alloc(to_hr_dev(pd-&gt;device), to_hr_pd(pd)-&gt;pdn, 0,</div><div class='del'>-				~0ULL, acc, 0, mr);</div><div class='add'>+	hns_roce_hem_list_init(&amp;mr-&gt;pbl_mtr.hem_list);</div><div class='add'>+	ret = alloc_mr_key(hr_dev, mr);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err_free;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_mr_enable(to_hr_dev(pd-&gt;device), mr);</div><div class='add'>+	ret = hns_roce_mr_enable(hr_dev, mr);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err_mr;</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;ibmr.rkey = mr-&gt;ibmr.lkey = mr-&gt;key;</div><div class='del'>-	mr-&gt;umem = NULL;</div><div class='ctx'> </div><div class='ctx'> 	return &amp;mr-&gt;ibmr;</div><div class='del'>-</div><div class='ctx'> err_mr:</div><div class='del'>-	hns_roce_mr_free(to_hr_dev(pd-&gt;device), mr);</div><div class='add'>+	free_mr_key(hr_dev, mr);</div><div class='ctx'> </div><div class='ctx'> err_free:</div><div class='ctx'> 	kfree(mr);</div><div class='ctx'> 	return ERR_PTR(ret);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,</div><div class='del'>-			       struct hns_roce_mtt *mtt, struct ib_umem *umem)</div><div class='del'>-{</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	struct sg_dma_page_iter sg_iter;</div><div class='del'>-	unsigned int order;</div><div class='del'>-	int npage = 0;</div><div class='del'>-	int ret = 0;</div><div class='del'>-	int i;</div><div class='del'>-	u64 page_addr;</div><div class='del'>-	u64 *pages;</div><div class='del'>-	u32 bt_page_size;</div><div class='del'>-	u32 n;</div><div class='del'>-</div><div class='del'>-	switch (mtt-&gt;mtt_type) {</div><div class='del'>-	case MTT_TYPE_WQE:</div><div class='del'>-		order = hr_dev-&gt;caps.mtt_ba_pg_sz;</div><div class='del'>-		break;</div><div class='del'>-	case MTT_TYPE_CQE:</div><div class='del'>-		order = hr_dev-&gt;caps.cqe_ba_pg_sz;</div><div class='del'>-		break;</div><div class='del'>-	case MTT_TYPE_SRQWQE:</div><div class='del'>-		order = hr_dev-&gt;caps.srqwqe_ba_pg_sz;</div><div class='del'>-		break;</div><div class='del'>-	case MTT_TYPE_IDX:</div><div class='del'>-		order = hr_dev-&gt;caps.idx_ba_pg_sz;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		dev_err(dev, "Unsupport mtt type %d, write mtt failed\n",</div><div class='del'>-			mtt-&gt;mtt_type);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	bt_page_size = 1 &lt;&lt; (order + PAGE_SHIFT);</div><div class='del'>-</div><div class='del'>-	pages = (u64 *) __get_free_pages(GFP_KERNEL, order);</div><div class='del'>-	if (!pages)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	i = n = 0;</div><div class='del'>-</div><div class='del'>-	for_each_sg_dma_page(umem-&gt;sg_head.sgl, &amp;sg_iter, umem-&gt;nmap, 0) {</div><div class='del'>-		page_addr = sg_page_iter_dma_address(&amp;sg_iter);</div><div class='del'>-		if (!(npage % (1 &lt;&lt; (mtt-&gt;page_shift - PAGE_SHIFT)))) {</div><div class='del'>-			if (page_addr &amp; ((1 &lt;&lt; mtt-&gt;page_shift) - 1)) {</div><div class='del'>-				dev_err(dev,</div><div class='del'>-					"page_addr 0x%llx is not page_shift %d alignment!\n",</div><div class='del'>-					page_addr, mtt-&gt;page_shift);</div><div class='del'>-				ret = -EINVAL;</div><div class='del'>-				goto out;</div><div class='del'>-			}</div><div class='del'>-			pages[i++] = page_addr;</div><div class='del'>-		}</div><div class='del'>-		npage++;</div><div class='del'>-		if (i == bt_page_size / sizeof(u64)) {</div><div class='del'>-			ret = hns_roce_write_mtt(hr_dev, mtt, n, i, pages);</div><div class='del'>-			if (ret)</div><div class='del'>-				goto out;</div><div class='del'>-			n += i;</div><div class='del'>-			i = 0;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (i)</div><div class='del'>-		ret = hns_roce_write_mtt(hr_dev, mtt, n, i, pages);</div><div class='del'>-</div><div class='del'>-out:</div><div class='del'>-	free_pages((unsigned long) pages, order);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int hns_roce_ib_umem_write_mr(struct hns_roce_dev *hr_dev,</div><div class='del'>-				     struct hns_roce_mr *mr,</div><div class='del'>-				     struct ib_umem *umem)</div><div class='del'>-{</div><div class='del'>-	struct sg_dma_page_iter sg_iter;</div><div class='del'>-	int i = 0, j = 0;</div><div class='del'>-	u64 page_addr;</div><div class='del'>-	u32 pbl_bt_sz;</div><div class='del'>-</div><div class='del'>-	if (hr_dev-&gt;caps.pbl_hop_num == HNS_ROCE_HOP_NUM_0)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	pbl_bt_sz = 1 &lt;&lt; (hr_dev-&gt;caps.pbl_ba_pg_sz + PAGE_SHIFT);</div><div class='del'>-	for_each_sg_dma_page(umem-&gt;sg_head.sgl, &amp;sg_iter, umem-&gt;nmap, 0) {</div><div class='del'>-		page_addr = sg_page_iter_dma_address(&amp;sg_iter);</div><div class='del'>-		if (!hr_dev-&gt;caps.pbl_hop_num) {</div><div class='del'>-			mr-&gt;pbl_buf[i++] = page_addr &gt;&gt; 12;</div><div class='del'>-		} else if (hr_dev-&gt;caps.pbl_hop_num == 1) {</div><div class='del'>-			mr-&gt;pbl_buf[i++] = page_addr;</div><div class='del'>-		} else {</div><div class='del'>-			if (hr_dev-&gt;caps.pbl_hop_num == 2)</div><div class='del'>-				mr-&gt;pbl_bt_l1[i][j] = page_addr;</div><div class='del'>-			else if (hr_dev-&gt;caps.pbl_hop_num == 3)</div><div class='del'>-				mr-&gt;pbl_bt_l2[i][j] = page_addr;</div><div class='del'>-</div><div class='del'>-			j++;</div><div class='del'>-			if (j &gt;= (pbl_bt_sz / 8)) {</div><div class='del'>-				i++;</div><div class='del'>-				j = 0;</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Memory barrier */</div><div class='del'>-	mb();</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='ctx'> 				   u64 virt_addr, int access_flags,</div><div class='add'>+				   struct ib_dmah *dmah,</div><div class='ctx'> 				   struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(pd-&gt;device);</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='ctx'> 	struct hns_roce_mr *mr;</div><div class='del'>-	int bt_size;</div><div class='ctx'> 	int ret;</div><div class='del'>-	int n;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	mr = kmalloc(sizeof(*mr), GFP_KERNEL);</div><div class='del'>-	if (!mr)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='del'>-	mr-&gt;umem = ib_umem_get(udata, start, length, access_flags, 0);</div><div class='del'>-	if (IS_ERR(mr-&gt;umem)) {</div><div class='del'>-		ret = PTR_ERR(mr-&gt;umem);</div><div class='del'>-		goto err_free;</div><div class='add'>+	if (dmah) {</div><div class='add'>+		ret = -EOPNOTSUPP;</div><div class='add'>+		goto err_out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	n = ib_umem_page_count(mr-&gt;umem);</div><div class='del'>-</div><div class='del'>-	if (!hr_dev-&gt;caps.pbl_hop_num) {</div><div class='del'>-		if (n &gt; HNS_ROCE_MAX_MTPT_PBL_NUM) {</div><div class='del'>-			dev_err(dev,</div><div class='del'>-			     " MR len %lld err. MR is limited to 4G at most!\n",</div><div class='del'>-			     length);</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto err_umem;</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		u64 pbl_size = 1;</div><div class='del'>-</div><div class='del'>-		bt_size = (1 &lt;&lt; (hr_dev-&gt;caps.pbl_ba_pg_sz + PAGE_SHIFT)) / 8;</div><div class='del'>-		for (i = 0; i &lt; hr_dev-&gt;caps.pbl_hop_num; i++)</div><div class='del'>-			pbl_size *= bt_size;</div><div class='del'>-		if (n &gt; pbl_size) {</div><div class='del'>-			dev_err(dev,</div><div class='del'>-			    " MR len %lld err. MR page num is limited to %lld!\n",</div><div class='del'>-			    length, pbl_size);</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto err_umem;</div><div class='del'>-		}</div><div class='add'>+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='add'>+	if (!mr) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err_out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	mr-&gt;iova = virt_addr;</div><div class='add'>+	mr-&gt;size = length;</div><div class='add'>+	mr-&gt;pd = to_hr_pd(pd)-&gt;pdn;</div><div class='add'>+	mr-&gt;access = access_flags;</div><div class='ctx'> 	mr-&gt;type = MR_TYPE_MR;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)-&gt;pdn, virt_addr, length,</div><div class='del'>-				access_flags, n, mr);</div><div class='add'>+	ret = alloc_mr_key(hr_dev, mr);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err_umem;</div><div class='add'>+		goto err_alloc_mr;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr-&gt;umem);</div><div class='add'>+	ret = alloc_mr_pbl(hr_dev, mr, udata, start);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err_mr;</div><div class='add'>+		goto err_alloc_key;</div><div class='ctx'> </div><div class='ctx'> 	ret = hns_roce_mr_enable(hr_dev, mr);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err_mr;</div><div class='add'>+		goto err_alloc_pbl;</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;ibmr.rkey = mr-&gt;ibmr.lkey = mr-&gt;key;</div><div class='ctx'> </div><div class='ctx'> 	return &amp;mr-&gt;ibmr;</div><div class='ctx'> </div><div class='del'>-err_mr:</div><div class='del'>-	hns_roce_mr_free(hr_dev, mr);</div><div class='del'>-</div><div class='del'>-err_umem:</div><div class='del'>-	ib_umem_release(mr-&gt;umem);</div><div class='del'>-</div><div class='del'>-err_free:</div><div class='add'>+err_alloc_pbl:</div><div class='add'>+	free_mr_pbl(hr_dev, mr);</div><div class='add'>+err_alloc_key:</div><div class='add'>+	free_mr_key(hr_dev, mr);</div><div class='add'>+err_alloc_mr:</div><div class='ctx'> 	kfree(mr);</div><div class='add'>+err_out:</div><div class='add'>+	atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_MR_REG_ERR_CNT]);</div><div class='add'>+</div><div class='ctx'> 	return ERR_PTR(ret);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, u64 length,</div><div class='del'>-			   u64 virt_addr, int mr_access_flags, struct ib_pd *pd,</div><div class='del'>-			   struct ib_udata *udata)</div><div class='add'>+struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start,</div><div class='add'>+				     u64 length, u64 virt_addr,</div><div class='add'>+				     int mr_access_flags, struct ib_pd *pd,</div><div class='add'>+				     struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ibmr-&gt;device);</div><div class='add'>+	struct ib_device *ib_dev = &amp;hr_dev-&gt;ib_dev;</div><div class='ctx'> 	struct hns_roce_mr *mr = to_hr_mr(ibmr);</div><div class='ctx'> 	struct hns_roce_cmd_mailbox *mailbox;</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='ctx'> 	unsigned long mtpt_idx;</div><div class='del'>-	u32 pdn = 0;</div><div class='del'>-	int npages;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	if (!mr-&gt;enabled)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	if (!mr-&gt;enabled) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err_out;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='del'>-	if (IS_ERR(mailbox))</div><div class='del'>-		return PTR_ERR(mailbox);</div><div class='add'>+	ret = PTR_ERR_OR_ZERO(mailbox);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_out;</div><div class='ctx'> </div><div class='ctx'> 	mtpt_idx = key_to_hw_index(mr-&gt;key) &amp; (hr_dev-&gt;caps.num_mtpts - 1);</div><div class='del'>-	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox-&gt;dma, mtpt_idx, 0,</div><div class='del'>-				HNS_ROCE_CMD_QUERY_MPT,</div><div class='del'>-				HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox-&gt;dma, HNS_ROCE_CMD_QUERY_MPT,</div><div class='add'>+				mtpt_idx);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto free_cmd_mbox;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_hw2sw_mpt(hr_dev, NULL, mtpt_idx);</div><div class='add'>+	ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_MPT,</div><div class='add'>+				      mtpt_idx);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret);</div><div class='add'>+		ibdev_warn(ib_dev, "failed to destroy MPT, ret = %d.\n", ret);</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;enabled = 0;</div><div class='add'>+	mr-&gt;iova = virt_addr;</div><div class='add'>+	mr-&gt;size = length;</div><div class='ctx'> </div><div class='ctx'> 	if (flags &amp; IB_MR_REREG_PD)</div><div class='del'>-		pdn = to_hr_pd(pd)-&gt;pdn;</div><div class='add'>+		mr-&gt;pd = to_hr_pd(pd)-&gt;pdn;</div><div class='ctx'> </div><div class='del'>-	if (flags &amp; IB_MR_REREG_TRANS) {</div><div class='del'>-		if (mr-&gt;size != ~0ULL) {</div><div class='del'>-			npages = ib_umem_page_count(mr-&gt;umem);</div><div class='del'>-</div><div class='del'>-			if (hr_dev-&gt;caps.pbl_hop_num)</div><div class='del'>-				hns_roce_mhop_free(hr_dev, mr);</div><div class='del'>-			else</div><div class='del'>-				dma_free_coherent(dev, npages * 8, mr-&gt;pbl_buf,</div><div class='del'>-						  mr-&gt;pbl_dma_addr);</div><div class='del'>-		}</div><div class='del'>-		ib_umem_release(mr-&gt;umem);</div><div class='add'>+	if (flags &amp; IB_MR_REREG_ACCESS)</div><div class='add'>+		mr-&gt;access = mr_access_flags;</div><div class='ctx'> </div><div class='del'>-		mr-&gt;umem =</div><div class='del'>-			ib_umem_get(udata, start, length, mr_access_flags, 0);</div><div class='del'>-		if (IS_ERR(mr-&gt;umem)) {</div><div class='del'>-			ret = PTR_ERR(mr-&gt;umem);</div><div class='del'>-			mr-&gt;umem = NULL;</div><div class='add'>+	if (flags &amp; IB_MR_REREG_TRANS) {</div><div class='add'>+		free_mr_pbl(hr_dev, mr);</div><div class='add'>+		ret = alloc_mr_pbl(hr_dev, mr, udata, start);</div><div class='add'>+		if (ret) {</div><div class='add'>+			ibdev_err(ib_dev, "failed to alloc mr PBL, ret = %d.\n",</div><div class='add'>+				  ret);</div><div class='ctx'> 			goto free_cmd_mbox;</div><div class='ctx'> 		}</div><div class='del'>-		npages = ib_umem_page_count(mr-&gt;umem);</div><div class='del'>-</div><div class='del'>-		if (hr_dev-&gt;caps.pbl_hop_num) {</div><div class='del'>-			ret = hns_roce_mhop_alloc(hr_dev, npages, mr);</div><div class='del'>-			if (ret)</div><div class='del'>-				goto release_umem;</div><div class='del'>-		} else {</div><div class='del'>-			mr-&gt;pbl_buf = dma_alloc_coherent(dev, npages * 8,</div><div class='del'>-							 &amp;(mr-&gt;pbl_dma_addr),</div><div class='del'>-							 GFP_KERNEL);</div><div class='del'>-			if (!mr-&gt;pbl_buf) {</div><div class='del'>-				ret = -ENOMEM;</div><div class='del'>-				goto release_umem;</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hr_dev-&gt;hw-&gt;rereg_write_mtpt(hr_dev, mr, flags, pdn,</div><div class='del'>-					   mr_access_flags, virt_addr,</div><div class='del'>-					   length, mailbox-&gt;buf);</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;rereg_write_mtpt(hr_dev, mr, flags, mailbox-&gt;buf);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		if (flags &amp; IB_MR_REREG_TRANS)</div><div class='del'>-			goto release_umem;</div><div class='del'>-		else</div><div class='del'>-			goto free_cmd_mbox;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (flags &amp; IB_MR_REREG_TRANS) {</div><div class='del'>-		ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr-&gt;umem);</div><div class='del'>-		if (ret) {</div><div class='del'>-			if (mr-&gt;size != ~0ULL) {</div><div class='del'>-				npages = ib_umem_page_count(mr-&gt;umem);</div><div class='del'>-</div><div class='del'>-				if (hr_dev-&gt;caps.pbl_hop_num)</div><div class='del'>-					hns_roce_mhop_free(hr_dev, mr);</div><div class='del'>-				else</div><div class='del'>-					dma_free_coherent(dev, npages * 8,</div><div class='del'>-							  mr-&gt;pbl_buf,</div><div class='del'>-							  mr-&gt;pbl_dma_addr);</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			goto release_umem;</div><div class='del'>-		}</div><div class='add'>+		ibdev_err(ib_dev, "failed to write mtpt, ret = %d.\n", ret);</div><div class='add'>+		goto free_cmd_mbox;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_sw2hw_mpt(hr_dev, mailbox, mtpt_idx);</div><div class='add'>+	ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT,</div><div class='add'>+				     mtpt_idx);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "SW2HW_MPT failed (%d)\n", ret);</div><div class='del'>-		goto release_umem;</div><div class='add'>+		ibdev_err(ib_dev, "failed to create MPT, ret = %d.\n", ret);</div><div class='add'>+		goto free_cmd_mbox;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;enabled = 1;</div><div class='del'>-	if (flags &amp; IB_MR_REREG_ACCESS)</div><div class='del'>-		mr-&gt;access = mr_access_flags;</div><div class='del'>-</div><div class='del'>-	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-release_umem:</div><div class='del'>-	ib_umem_release(mr-&gt;umem);</div><div class='ctx'> </div><div class='ctx'> free_cmd_mbox:</div><div class='ctx'> 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='ctx'> </div><div class='del'>-	return ret;</div><div class='add'>+err_out:</div><div class='add'>+	if (ret) {</div><div class='add'>+		atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_MR_REREG_ERR_CNT]);</div><div class='add'>+		return ERR_PTR(ret);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ibmr-&gt;device);</div><div class='ctx'> 	struct hns_roce_mr *mr = to_hr_mr(ibmr);</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	if (hr_dev-&gt;hw-&gt;dereg_mr) {</div><div class='del'>-		ret = hr_dev-&gt;hw-&gt;dereg_mr(hr_dev, mr, udata);</div><div class='del'>-	} else {</div><div class='del'>-		hns_roce_mr_free(hr_dev, mr);</div><div class='ctx'> </div><div class='del'>-		if (mr-&gt;umem)</div><div class='del'>-			ib_umem_release(mr-&gt;umem);</div><div class='add'>+	if (hr_dev-&gt;hw-&gt;dereg_mr)</div><div class='add'>+		hr_dev-&gt;hw-&gt;dereg_mr(hr_dev);</div><div class='ctx'> </div><div class='del'>-		kfree(mr);</div><div class='del'>-	}</div><div class='add'>+	hns_roce_mr_free(hr_dev, mr);</div><div class='add'>+	kfree(mr);</div><div class='ctx'> </div><div class='del'>-	return ret;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,</div><div class='del'>-				u32 max_num_sg, struct ib_udata *udata)</div><div class='add'>+				u32 max_num_sg)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(pd-&gt;device);</div><div class='ctx'> 	struct device *dev = hr_dev-&gt;dev;</div><div class='ctx'> 	struct hns_roce_mr *mr;</div><div class='del'>-	u64 length;</div><div class='del'>-	u32 page_size;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	page_size = 1 &lt;&lt; (hr_dev-&gt;caps.pbl_buf_pg_sz + PAGE_SHIFT);</div><div class='del'>-	length = max_num_sg * page_size;</div><div class='del'>-</div><div class='ctx'> 	if (mr_type != IB_MR_TYPE_MEM_REG)</div><div class='ctx'> 		return ERR_PTR(-EINVAL);</div><div class='ctx'> </div><div class='hunk'>@@ -1329,25 +400,31 @@ struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;type = MR_TYPE_FRMR;</div><div class='add'>+	mr-&gt;pd = to_hr_pd(pd)-&gt;pdn;</div><div class='add'>+	mr-&gt;size = max_num_sg * (1 &lt;&lt; PAGE_SHIFT);</div><div class='ctx'> </div><div class='ctx'> 	/* Allocate memory region key */</div><div class='del'>-	ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)-&gt;pdn, 0, length,</div><div class='del'>-				0, max_num_sg, mr);</div><div class='add'>+	ret = alloc_mr_key(hr_dev, mr);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err_free;</div><div class='ctx'> </div><div class='add'>+	ret = alloc_mr_pbl(hr_dev, mr, NULL, 0);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_key;</div><div class='add'>+</div><div class='ctx'> 	ret = hns_roce_mr_enable(hr_dev, mr);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err_mr;</div><div class='add'>+		goto err_pbl;</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;ibmr.rkey = mr-&gt;ibmr.lkey = mr-&gt;key;</div><div class='del'>-	mr-&gt;umem = NULL;</div><div class='add'>+	mr-&gt;ibmr.length = mr-&gt;size;</div><div class='ctx'> </div><div class='ctx'> 	return &amp;mr-&gt;ibmr;</div><div class='ctx'> </div><div class='del'>-err_mr:</div><div class='del'>-	hns_roce_mr_free(to_hr_dev(pd-&gt;device), mr);</div><div class='del'>-</div><div class='add'>+err_pbl:</div><div class='add'>+	free_mr_pbl(hr_dev, mr);</div><div class='add'>+err_key:</div><div class='add'>+	free_mr_key(hr_dev, mr);</div><div class='ctx'> err_free:</div><div class='ctx'> 	kfree(mr);</div><div class='ctx'> 	return ERR_PTR(ret);</div><div class='hunk'>@@ -1357,137 +434,671 @@ static int hns_roce_set_page(struct ib_mr *ibmr, u64 addr)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_mr *mr = to_hr_mr(ibmr);</div><div class='ctx'> </div><div class='del'>-	mr-&gt;pbl_buf[mr-&gt;npages++] = cpu_to_le64(addr);</div><div class='add'>+	if (likely(mr-&gt;npages &lt; mr-&gt;pbl_mtr.hem_cfg.buf_pg_count)) {</div><div class='add'>+		mr-&gt;page_list[mr-&gt;npages++] = addr;</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	return -ENOBUFS;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,</div><div class='del'>-		       unsigned int *sg_offset)</div><div class='add'>+		       unsigned int *sg_offset_p)</div><div class='ctx'> {</div><div class='add'>+	unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ibmr-&gt;device);</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='ctx'> 	struct hns_roce_mr *mr = to_hr_mr(ibmr);</div><div class='add'>+	struct hns_roce_mtr *mtr = &amp;mr-&gt;pbl_mtr;</div><div class='add'>+	int ret, sg_num = 0;</div><div class='add'>+</div><div class='add'>+	if (!IS_ALIGNED(sg_offset, HNS_ROCE_FRMR_ALIGN_SIZE) ||</div><div class='add'>+	    ibmr-&gt;page_size &lt; HNS_HW_PAGE_SIZE ||</div><div class='add'>+	    ibmr-&gt;page_size &gt; HNS_HW_MAX_PAGE_SIZE)</div><div class='add'>+		return sg_num;</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;npages = 0;</div><div class='add'>+	mr-&gt;page_list = kvcalloc(mr-&gt;pbl_mtr.hem_cfg.buf_pg_count,</div><div class='add'>+				 sizeof(dma_addr_t), GFP_KERNEL);</div><div class='add'>+	if (!mr-&gt;page_list)</div><div class='add'>+		return sg_num;</div><div class='add'>+</div><div class='add'>+	sg_num = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset_p, hns_roce_set_page);</div><div class='add'>+	if (sg_num &lt; 1) {</div><div class='add'>+		ibdev_err(ibdev, "failed to store sg pages %u %u, cnt = %d.\n",</div><div class='add'>+			  mr-&gt;npages, mr-&gt;pbl_mtr.hem_cfg.buf_pg_count, sg_num);</div><div class='add'>+		goto err_page_list;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mtr-&gt;hem_cfg.region[0].offset = 0;</div><div class='add'>+	mtr-&gt;hem_cfg.region[0].count = mr-&gt;npages;</div><div class='add'>+	mtr-&gt;hem_cfg.region[0].hopnum = mr-&gt;pbl_hop_num;</div><div class='add'>+	mtr-&gt;hem_cfg.region_count = 1;</div><div class='add'>+	ret = hns_roce_mtr_map(hr_dev, mtr, mr-&gt;page_list, mr-&gt;npages);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to map sg mtr, ret = %d.\n", ret);</div><div class='add'>+		sg_num = 0;</div><div class='add'>+	} else {</div><div class='add'>+		mr-&gt;pbl_mtr.hem_cfg.buf_pg_shift = (u32)ilog2(ibmr-&gt;page_size);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+err_page_list:</div><div class='add'>+	kvfree(mr-&gt;page_list);</div><div class='add'>+	mr-&gt;page_list = NULL;</div><div class='ctx'> </div><div class='del'>-	return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, hns_roce_set_page);</div><div class='add'>+	return sg_num;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_mw_free(struct hns_roce_dev *hr_dev,</div><div class='del'>-			     struct hns_roce_mw *mw)</div><div class='add'>+static int mtr_map_region(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,</div><div class='add'>+			  struct hns_roce_buf_region *region, dma_addr_t *pages,</div><div class='add'>+			  int max_count)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	int ret;</div><div class='add'>+	int count, npage;</div><div class='add'>+	int offset, end;</div><div class='add'>+	__le64 *mtts;</div><div class='add'>+	u64 addr;</div><div class='add'>+	int i;</div><div class='ctx'> </div><div class='del'>-	if (mw-&gt;enabled) {</div><div class='del'>-		ret = hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mw-&gt;rkey)</div><div class='del'>-					 &amp; (hr_dev-&gt;caps.num_mtpts - 1));</div><div class='del'>-		if (ret)</div><div class='del'>-			dev_warn(dev, "MW HW2SW_MPT failed (%d)\n", ret);</div><div class='add'>+	offset = region-&gt;offset;</div><div class='add'>+	end = offset + region-&gt;count;</div><div class='add'>+	npage = 0;</div><div class='add'>+	while (offset &lt; end &amp;&amp; npage &lt; max_count) {</div><div class='add'>+		count = 0;</div><div class='add'>+		mtts = hns_roce_hem_list_find_mtt(hr_dev, &amp;mtr-&gt;hem_list,</div><div class='add'>+						  offset, &amp;count);</div><div class='add'>+		if (!mtts)</div><div class='add'>+			return -ENOBUFS;</div><div class='add'>+</div><div class='add'>+		for (i = 0; i &lt; count &amp;&amp; npage &lt; max_count; i++) {</div><div class='add'>+			addr = pages[npage];</div><div class='add'>+</div><div class='add'>+			mtts[i] = cpu_to_le64(addr);</div><div class='add'>+			npage++;</div><div class='add'>+		}</div><div class='add'>+		offset += count;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return npage;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool mtr_has_mtt(struct hns_roce_buf_attr *attr)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; attr-&gt;region_count; i++)</div><div class='add'>+		if (attr-&gt;region[i].hopnum != HNS_ROCE_HOP_NUM_0 &amp;&amp;</div><div class='add'>+		    attr-&gt;region[i].hopnum &gt; 0)</div><div class='add'>+			return true;</div><div class='add'>+</div><div class='add'>+	/* because the mtr only one root base address, when hopnum is 0 means</div><div class='add'>+	 * root base address equals the first buffer address, thus all alloced</div><div class='add'>+	 * memory must in a continuous space accessed by direct mode.</div><div class='add'>+	 */</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		hns_roce_table_put(hr_dev, &amp;hr_dev-&gt;mr_table.mtpt_table,</div><div class='del'>-				   key_to_hw_index(mw-&gt;rkey));</div><div class='add'>+static inline size_t mtr_bufs_size(struct hns_roce_buf_attr *attr)</div><div class='add'>+{</div><div class='add'>+	size_t size = 0;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; attr-&gt;region_count; i++)</div><div class='add'>+		size += attr-&gt;region[i].size;</div><div class='add'>+</div><div class='add'>+	return size;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * check the given pages in continuous address space</div><div class='add'>+ * Returns 0 on success, or the error page num.</div><div class='add'>+ */</div><div class='add'>+static inline int mtr_check_direct_pages(dma_addr_t *pages, int page_count,</div><div class='add'>+					 unsigned int page_shift)</div><div class='add'>+{</div><div class='add'>+	size_t page_size = 1 &lt;&lt; page_shift;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 1; i &lt; page_count; i++)</div><div class='add'>+		if (pages[i] - pages[i - 1] != page_size)</div><div class='add'>+			return i;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mtr_free_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)</div><div class='add'>+{</div><div class='add'>+	/* release user buffers */</div><div class='add'>+	if (mtr-&gt;umem) {</div><div class='add'>+		ib_umem_release(mtr-&gt;umem);</div><div class='add'>+		mtr-&gt;umem = NULL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	hns_roce_bitmap_free(&amp;hr_dev-&gt;mr_table.mtpt_bitmap,</div><div class='del'>-			     key_to_hw_index(mw-&gt;rkey), BITMAP_NO_RR);</div><div class='add'>+	/* release kernel buffers */</div><div class='add'>+	if (mtr-&gt;kmem) {</div><div class='add'>+		hns_roce_buf_free(hr_dev, mtr-&gt;kmem);</div><div class='add'>+		mtr-&gt;kmem = NULL;</div><div class='add'>+	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_mw_enable(struct hns_roce_dev *hr_dev,</div><div class='del'>-			      struct hns_roce_mw *mw)</div><div class='add'>+static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,</div><div class='add'>+			  struct hns_roce_buf_attr *buf_attr,</div><div class='add'>+			  struct ib_udata *udata, unsigned long user_addr)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_mr_table *mr_table = &amp;hr_dev-&gt;mr_table;</div><div class='del'>-	struct hns_roce_cmd_mailbox *mailbox;</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	unsigned long mtpt_idx = key_to_hw_index(mw-&gt;rkey);</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	size_t total_size;</div><div class='add'>+</div><div class='add'>+	total_size = mtr_bufs_size(buf_attr);</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		mtr-&gt;kmem = NULL;</div><div class='add'>+		mtr-&gt;umem = ib_umem_get(ibdev, user_addr, total_size,</div><div class='add'>+					buf_attr-&gt;user_access);</div><div class='add'>+		if (IS_ERR(mtr-&gt;umem)) {</div><div class='add'>+			ibdev_err(ibdev, "failed to get umem, ret = %pe.\n",</div><div class='add'>+				  mtr-&gt;umem);</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		mtr-&gt;umem = NULL;</div><div class='add'>+		mtr-&gt;kmem = hns_roce_buf_alloc(hr_dev, total_size,</div><div class='add'>+					       buf_attr-&gt;page_shift,</div><div class='add'>+					       !mtr_has_mtt(buf_attr) ?</div><div class='add'>+					       HNS_ROCE_BUF_DIRECT : 0);</div><div class='add'>+		if (IS_ERR(mtr-&gt;kmem)) {</div><div class='add'>+			ibdev_err(ibdev, "failed to alloc kmem, ret = %pe.\n",</div><div class='add'>+				  mtr-&gt;kmem);</div><div class='add'>+			return PTR_ERR(mtr-&gt;kmem);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int cal_mtr_pg_cnt(struct hns_roce_mtr *mtr)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_buf_region *region;</div><div class='add'>+	int page_cnt = 0;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; mtr-&gt;hem_cfg.region_count; i++) {</div><div class='add'>+		region = &amp;mtr-&gt;hem_cfg.region[i];</div><div class='add'>+		page_cnt += region-&gt;count;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return page_cnt;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool need_split_huge_page(struct hns_roce_mtr *mtr)</div><div class='add'>+{</div><div class='add'>+	/* When HEM buffer uses 0-level addressing, the page size is</div><div class='add'>+	 * equal to the whole buffer size. If the current MTR has multiple</div><div class='add'>+	 * regions, we split the buffer into small pages(4k, required by hns</div><div class='add'>+	 * ROCEE). These pages will be used in multiple regions.</div><div class='add'>+	 */</div><div class='add'>+	return mtr-&gt;hem_cfg.is_direct &amp;&amp; mtr-&gt;hem_cfg.region_count &gt; 1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mtr_map_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	int page_count = cal_mtr_pg_cnt(mtr);</div><div class='add'>+	unsigned int page_shift;</div><div class='add'>+	dma_addr_t *pages;</div><div class='add'>+	int npage;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	/* prepare HEM entry memory */</div><div class='del'>-	ret = hns_roce_table_get(hr_dev, &amp;mr_table-&gt;mtpt_table, mtpt_idx);</div><div class='add'>+	page_shift = need_split_huge_page(mtr) ? HNS_HW_PAGE_SHIFT :</div><div class='add'>+						 mtr-&gt;hem_cfg.buf_pg_shift;</div><div class='add'>+	/* alloc a tmp array to store buffer's dma address */</div><div class='add'>+	pages = kvcalloc(page_count, sizeof(dma_addr_t), GFP_KERNEL);</div><div class='add'>+	if (!pages)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	if (mtr-&gt;umem)</div><div class='add'>+		npage = hns_roce_get_umem_bufs(pages, page_count,</div><div class='add'>+					       mtr-&gt;umem, page_shift);</div><div class='add'>+	else</div><div class='add'>+		npage = hns_roce_get_kmem_bufs(hr_dev, pages, page_count,</div><div class='add'>+					       mtr-&gt;kmem, page_shift);</div><div class='add'>+</div><div class='add'>+	if (npage != page_count) {</div><div class='add'>+		ibdev_err(ibdev, "failed to get mtr page %d != %d.\n", npage,</div><div class='add'>+			  page_count);</div><div class='add'>+		ret = -ENOBUFS;</div><div class='add'>+		goto err_alloc_list;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (need_split_huge_page(mtr) &amp;&amp; npage &gt; 1) {</div><div class='add'>+		ret = mtr_check_direct_pages(pages, npage, page_shift);</div><div class='add'>+		if (ret) {</div><div class='add'>+			ibdev_err(ibdev, "failed to check %s page: %d / %d.\n",</div><div class='add'>+				  mtr-&gt;umem ? "umtr" : "kmtr", ret, npage);</div><div class='add'>+			ret = -ENOBUFS;</div><div class='add'>+			goto err_alloc_list;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_mtr_map(hr_dev, mtr, pages, page_count);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		return ret;</div><div class='add'>+		ibdev_err(ibdev, "failed to map mtr pages, ret = %d.\n", ret);</div><div class='ctx'> </div><div class='del'>-	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='del'>-	if (IS_ERR(mailbox)) {</div><div class='del'>-		ret = PTR_ERR(mailbox);</div><div class='del'>-		goto err_table;</div><div class='add'>+err_alloc_list:</div><div class='add'>+	kvfree(pages);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,</div><div class='add'>+		     dma_addr_t *pages, unsigned int page_cnt)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	struct hns_roce_buf_region *r;</div><div class='add'>+	unsigned int i, mapped_cnt;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Only use the first page address as root ba when hopnum is 0, this</div><div class='add'>+	 * is because the addresses of all pages are consecutive in this case.</div><div class='add'>+	 */</div><div class='add'>+	if (mtr-&gt;hem_cfg.is_direct) {</div><div class='add'>+		mtr-&gt;hem_cfg.root_ba = pages[0];</div><div class='add'>+		return 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hr_dev-&gt;hw-&gt;mw_write_mtpt(mailbox-&gt;buf, mw);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "MW write mtpt fail!\n");</div><div class='del'>-		goto err_page;</div><div class='add'>+	for (i = 0, mapped_cnt = 0; i &lt; mtr-&gt;hem_cfg.region_count &amp;&amp;</div><div class='add'>+	     mapped_cnt &lt; page_cnt; i++) {</div><div class='add'>+		r = &amp;mtr-&gt;hem_cfg.region[i];</div><div class='add'>+</div><div class='add'>+		if (r-&gt;offset + r-&gt;count &gt; page_cnt) {</div><div class='add'>+			ret = -EINVAL;</div><div class='add'>+			ibdev_err(ibdev,</div><div class='add'>+				  "failed to check mtr%u count %u + %u &gt; %u.\n",</div><div class='add'>+				  i, r-&gt;offset, r-&gt;count, page_cnt);</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ret = mtr_map_region(hr_dev, mtr, r, &amp;pages[r-&gt;offset],</div><div class='add'>+				     page_cnt - mapped_cnt);</div><div class='add'>+		if (ret &lt; 0) {</div><div class='add'>+			ibdev_err(ibdev,</div><div class='add'>+				  "failed to map mtr%u offset %u, ret = %d.\n",</div><div class='add'>+				  i, r-&gt;offset, ret);</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+		mapped_cnt += ret;</div><div class='add'>+		ret = 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_sw2hw_mpt(hr_dev, mailbox,</div><div class='del'>-				 mtpt_idx &amp; (hr_dev-&gt;caps.num_mtpts - 1));</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "MW sw2hw_mpt failed (%d)\n", ret);</div><div class='del'>-		goto err_page;</div><div class='add'>+	if (mapped_cnt &lt; page_cnt) {</div><div class='add'>+		ret = -ENOBUFS;</div><div class='add'>+		ibdev_err(ibdev, "failed to map mtr pages count: %u &lt; %u.\n",</div><div class='add'>+			  mapped_cnt, page_cnt);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	mw-&gt;enabled = 1;</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_get_direct_addr_mtt(struct hns_roce_hem_cfg *cfg,</div><div class='add'>+					u32 start_index, u64 *mtt_buf,</div><div class='add'>+					int mtt_cnt)</div><div class='add'>+{</div><div class='add'>+	int mtt_count;</div><div class='add'>+	int total = 0;</div><div class='add'>+	u32 npage;</div><div class='add'>+	u64 addr;</div><div class='ctx'> </div><div class='del'>-	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='add'>+	if (mtt_cnt &gt; cfg-&gt;region_count)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	for (mtt_count = 0; mtt_count &lt; cfg-&gt;region_count &amp;&amp; total &lt; mtt_cnt;</div><div class='add'>+	     mtt_count++) {</div><div class='add'>+		npage = cfg-&gt;region[mtt_count].offset;</div><div class='add'>+		if (npage &lt; start_index)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		addr = cfg-&gt;root_ba + (npage &lt;&lt; HNS_HW_PAGE_SHIFT);</div><div class='add'>+		mtt_buf[total] = addr;</div><div class='add'>+</div><div class='add'>+		total++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!total)</div><div class='add'>+		return -ENOENT;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-err_page:</div><div class='del'>-	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='add'>+static int hns_roce_get_mhop_mtt(struct hns_roce_dev *hr_dev,</div><div class='add'>+				 struct hns_roce_mtr *mtr, u32 start_index,</div><div class='add'>+				 u64 *mtt_buf, int mtt_cnt)</div><div class='add'>+{</div><div class='add'>+	int left = mtt_cnt;</div><div class='add'>+	int total = 0;</div><div class='add'>+	int mtt_count;</div><div class='add'>+	__le64 *mtts;</div><div class='add'>+	u32 npage;</div><div class='add'>+</div><div class='add'>+	while (left &gt; 0) {</div><div class='add'>+		mtt_count = 0;</div><div class='add'>+		mtts = hns_roce_hem_list_find_mtt(hr_dev, &amp;mtr-&gt;hem_list,</div><div class='add'>+						  start_index + total,</div><div class='add'>+						  &amp;mtt_count);</div><div class='add'>+		if (!mtts || !mtt_count)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		npage = min(mtt_count, left);</div><div class='add'>+		left -= npage;</div><div class='add'>+		for (mtt_count = 0; mtt_count &lt; npage; mtt_count++)</div><div class='add'>+			mtt_buf[total++] = le64_to_cpu(mtts[mtt_count]);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-err_table:</div><div class='del'>-	hns_roce_table_put(hr_dev, &amp;mr_table-&gt;mtpt_table, mtpt_idx);</div><div class='add'>+	if (!total)</div><div class='add'>+		return -ENOENT;</div><div class='ctx'> </div><div class='del'>-	return ret;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_mw *hns_roce_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,</div><div class='del'>-				struct ib_udata *udata)</div><div class='add'>+int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,</div><div class='add'>+		      u32 offset, u64 *mtt_buf, int mtt_max)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ib_pd-&gt;device);</div><div class='del'>-	struct hns_roce_mw *mw;</div><div class='del'>-	unsigned long index = 0;</div><div class='add'>+	struct hns_roce_hem_cfg *cfg = &amp;mtr-&gt;hem_cfg;</div><div class='add'>+	u32 start_index;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	mw = kmalloc(sizeof(*mw), GFP_KERNEL);</div><div class='del'>-	if (!mw)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='add'>+	if (!mtt_buf || mtt_max &lt; 1)</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	/* Allocate a key for mw from bitmap */</div><div class='del'>-	ret = hns_roce_bitmap_alloc(&amp;hr_dev-&gt;mr_table.mtpt_bitmap, &amp;index);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_bitmap;</div><div class='add'>+	/* no mtt memory in direct mode, so just return the buffer address */</div><div class='add'>+	if (cfg-&gt;is_direct) {</div><div class='add'>+		start_index = offset &gt;&gt; HNS_HW_PAGE_SHIFT;</div><div class='add'>+		ret = hns_roce_get_direct_addr_mtt(cfg, start_index,</div><div class='add'>+						   mtt_buf, mtt_max);</div><div class='add'>+	} else {</div><div class='add'>+		start_index = offset &gt;&gt; cfg-&gt;buf_pg_shift;</div><div class='add'>+		ret = hns_roce_get_mhop_mtt(hr_dev, mtr, start_index,</div><div class='add'>+					    mtt_buf, mtt_max);</div><div class='add'>+	}</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	mw-&gt;rkey = hw_index_to_key(index);</div><div class='add'>+static int get_best_page_shift(struct hns_roce_dev *hr_dev,</div><div class='add'>+			       struct hns_roce_mtr *mtr,</div><div class='add'>+			       struct hns_roce_buf_attr *buf_attr)</div><div class='add'>+{</div><div class='add'>+	unsigned int page_sz;</div><div class='ctx'> </div><div class='del'>-	mw-&gt;ibmw.rkey = mw-&gt;rkey;</div><div class='del'>-	mw-&gt;ibmw.type = type;</div><div class='del'>-	mw-&gt;pdn = to_hr_pd(ib_pd)-&gt;pdn;</div><div class='del'>-	mw-&gt;pbl_hop_num = hr_dev-&gt;caps.pbl_hop_num;</div><div class='del'>-	mw-&gt;pbl_ba_pg_sz = hr_dev-&gt;caps.pbl_ba_pg_sz;</div><div class='del'>-	mw-&gt;pbl_buf_pg_sz = hr_dev-&gt;caps.pbl_buf_pg_sz;</div><div class='add'>+	if (!buf_attr-&gt;adaptive || buf_attr-&gt;type != MTR_PBL || !mtr-&gt;umem)</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_mw_enable(hr_dev, mw);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_mw;</div><div class='add'>+	page_sz = ib_umem_find_best_pgsz(mtr-&gt;umem,</div><div class='add'>+					 hr_dev-&gt;caps.page_size_cap,</div><div class='add'>+					 buf_attr-&gt;iova);</div><div class='add'>+	if (!page_sz)</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	return &amp;mw-&gt;ibmw;</div><div class='add'>+	buf_attr-&gt;page_shift = order_base_2(page_sz);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_best_hop_num(struct hns_roce_dev *hr_dev,</div><div class='add'>+			    struct hns_roce_mtr *mtr,</div><div class='add'>+			    struct hns_roce_buf_attr *buf_attr,</div><div class='add'>+			    unsigned int ba_pg_shift)</div><div class='add'>+{</div><div class='add'>+#define INVALID_HOPNUM -1</div><div class='add'>+#define MIN_BA_CNT 1</div><div class='add'>+	size_t buf_pg_sz = 1 &lt;&lt; buf_attr-&gt;page_shift;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	size_t ba_pg_sz = 1 &lt;&lt; ba_pg_shift;</div><div class='add'>+	int hop_num = INVALID_HOPNUM;</div><div class='add'>+	size_t unit = MIN_BA_CNT;</div><div class='add'>+	size_t ba_cnt;</div><div class='add'>+	int j;</div><div class='add'>+</div><div class='add'>+	if (!buf_attr-&gt;adaptive || buf_attr-&gt;type != MTR_PBL)</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-err_mw:</div><div class='del'>-	hns_roce_mw_free(hr_dev, mw);</div><div class='add'>+	/* Caculating the number of buf pages, each buf page need a BA */</div><div class='add'>+	if (mtr-&gt;umem)</div><div class='add'>+		ba_cnt = ib_umem_num_dma_blocks(mtr-&gt;umem, buf_pg_sz);</div><div class='add'>+	else</div><div class='add'>+		ba_cnt = DIV_ROUND_UP(buf_attr-&gt;region[0].size, buf_pg_sz);</div><div class='ctx'> </div><div class='del'>-err_bitmap:</div><div class='del'>-	kfree(mw);</div><div class='add'>+	for (j = 0; j &lt;= HNS_ROCE_MAX_HOP_NUM; j++) {</div><div class='add'>+		if (ba_cnt &lt;= unit) {</div><div class='add'>+			hop_num = j;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		/* Number of BAs can be represented at per hop */</div><div class='add'>+		unit *= ba_pg_sz / BA_BYTE_LEN;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	return ERR_PTR(ret);</div><div class='add'>+	if (hop_num &lt; 0) {</div><div class='add'>+		ibdev_err(ibdev,</div><div class='add'>+			  "failed to calculate a valid hopnum.\n");</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	buf_attr-&gt;region[0].hopnum = hop_num;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool is_buf_attr_valid(struct hns_roce_dev *hr_dev,</div><div class='add'>+			      struct hns_roce_buf_attr *attr)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;region_count &gt; ARRAY_SIZE(attr-&gt;region) ||</div><div class='add'>+	    attr-&gt;region_count &lt; 1 || attr-&gt;page_shift &lt; HNS_HW_PAGE_SHIFT) {</div><div class='add'>+		ibdev_err(ibdev,</div><div class='add'>+			  "invalid buf attr, region count %u, page shift %u.\n",</div><div class='add'>+			  attr-&gt;region_count, attr-&gt;page_shift);</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int hns_roce_dealloc_mw(struct ib_mw *ibmw)</div><div class='add'>+static int mtr_init_buf_cfg(struct hns_roce_dev *hr_dev,</div><div class='add'>+			    struct hns_roce_mtr *mtr,</div><div class='add'>+			    struct hns_roce_buf_attr *attr)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ibmw-&gt;device);</div><div class='del'>-	struct hns_roce_mw *mw = to_hr_mw(ibmw);</div><div class='add'>+	struct hns_roce_hem_cfg *cfg = &amp;mtr-&gt;hem_cfg;</div><div class='add'>+	struct hns_roce_buf_region *r;</div><div class='add'>+	size_t buf_pg_sz;</div><div class='add'>+	size_t buf_size;</div><div class='add'>+	int page_cnt, i;</div><div class='add'>+	u64 pgoff = 0;</div><div class='add'>+</div><div class='add'>+	if (!is_buf_attr_valid(hr_dev, attr))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	/* If mtt is disabled, all pages must be within a continuous range */</div><div class='add'>+	cfg-&gt;is_direct = !mtr_has_mtt(attr);</div><div class='add'>+	cfg-&gt;region_count = attr-&gt;region_count;</div><div class='add'>+	buf_size = mtr_bufs_size(attr);</div><div class='add'>+	if (need_split_huge_page(mtr)) {</div><div class='add'>+		buf_pg_sz = HNS_HW_PAGE_SIZE;</div><div class='add'>+		cfg-&gt;buf_pg_count = 1;</div><div class='add'>+		/* The ROCEE requires the page size to be 4K * 2 ^ N. */</div><div class='add'>+		cfg-&gt;buf_pg_shift = HNS_HW_PAGE_SHIFT +</div><div class='add'>+			order_base_2(DIV_ROUND_UP(buf_size, HNS_HW_PAGE_SIZE));</div><div class='add'>+	} else {</div><div class='add'>+		buf_pg_sz = 1 &lt;&lt; attr-&gt;page_shift;</div><div class='add'>+		cfg-&gt;buf_pg_count = mtr-&gt;umem ?</div><div class='add'>+			ib_umem_num_dma_blocks(mtr-&gt;umem, buf_pg_sz) :</div><div class='add'>+			DIV_ROUND_UP(buf_size, buf_pg_sz);</div><div class='add'>+		cfg-&gt;buf_pg_shift = attr-&gt;page_shift;</div><div class='add'>+		pgoff = mtr-&gt;umem ? mtr-&gt;umem-&gt;address &amp; ~PAGE_MASK : 0;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	hns_roce_mw_free(hr_dev, mw);</div><div class='del'>-	kfree(mw);</div><div class='add'>+	/* Convert buffer size to page index and page count for each region and</div><div class='add'>+	 * the buffer's offset needs to be appended to the first region.</div><div class='add'>+	 */</div><div class='add'>+	for (page_cnt = 0, i = 0; i &lt; attr-&gt;region_count; i++) {</div><div class='add'>+		r = &amp;cfg-&gt;region[i];</div><div class='add'>+		r-&gt;offset = page_cnt;</div><div class='add'>+		buf_size = hr_hw_page_align(attr-&gt;region[i].size + pgoff);</div><div class='add'>+		if (attr-&gt;type == MTR_PBL &amp;&amp; mtr-&gt;umem)</div><div class='add'>+			r-&gt;count = ib_umem_num_dma_blocks(mtr-&gt;umem, buf_pg_sz);</div><div class='add'>+		else</div><div class='add'>+			r-&gt;count = DIV_ROUND_UP(buf_size, buf_pg_sz);</div><div class='add'>+</div><div class='add'>+		pgoff = 0;</div><div class='add'>+		page_cnt += r-&gt;count;</div><div class='add'>+		r-&gt;hopnum = to_hr_hem_hopnum(attr-&gt;region[i].hopnum, r-&gt;count);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u64 cal_pages_per_l1ba(unsigned int ba_per_bt, unsigned int hopnum)</div><div class='add'>+{</div><div class='add'>+	return int_pow(ba_per_bt, hopnum - 1);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static unsigned int cal_best_bt_pg_sz(struct hns_roce_dev *hr_dev,</div><div class='add'>+				      struct hns_roce_mtr *mtr,</div><div class='add'>+				      unsigned int pg_shift)</div><div class='add'>+{</div><div class='add'>+	unsigned long cap = hr_dev-&gt;caps.page_size_cap;</div><div class='add'>+	struct hns_roce_buf_region *re;</div><div class='add'>+	unsigned int pgs_per_l1ba;</div><div class='add'>+	unsigned int ba_per_bt;</div><div class='add'>+	unsigned int ba_num;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for_each_set_bit_from(pg_shift, &amp;cap, sizeof(cap) * BITS_PER_BYTE) {</div><div class='add'>+		if (!(BIT(pg_shift) &amp; cap))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		ba_per_bt = BIT(pg_shift) / BA_BYTE_LEN;</div><div class='add'>+		ba_num = 0;</div><div class='add'>+		for (i = 0; i &lt; mtr-&gt;hem_cfg.region_count; i++) {</div><div class='add'>+			re = &amp;mtr-&gt;hem_cfg.region[i];</div><div class='add'>+			if (re-&gt;hopnum == 0)</div><div class='add'>+				continue;</div><div class='add'>+</div><div class='add'>+			pgs_per_l1ba = cal_pages_per_l1ba(ba_per_bt, re-&gt;hopnum);</div><div class='add'>+			ba_num += DIV_ROUND_UP(re-&gt;count, pgs_per_l1ba);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (ba_num &lt;= ba_per_bt)</div><div class='add'>+			return pg_shift;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mtr_alloc_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,</div><div class='add'>+			 unsigned int ba_page_shift)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_hem_cfg *cfg = &amp;mtr-&gt;hem_cfg;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	hns_roce_hem_list_init(&amp;mtr-&gt;hem_list);</div><div class='add'>+	if (!cfg-&gt;is_direct) {</div><div class='add'>+		ba_page_shift = cal_best_bt_pg_sz(hr_dev, mtr, ba_page_shift);</div><div class='add'>+		if (!ba_page_shift)</div><div class='add'>+			return -ERANGE;</div><div class='add'>+</div><div class='add'>+		ret = hns_roce_hem_list_request(hr_dev, &amp;mtr-&gt;hem_list,</div><div class='add'>+						cfg-&gt;region, cfg-&gt;region_count,</div><div class='add'>+						ba_page_shift);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+		cfg-&gt;root_ba = mtr-&gt;hem_list.root_ba;</div><div class='add'>+		cfg-&gt;ba_pg_shift = ba_page_shift;</div><div class='add'>+	} else {</div><div class='add'>+		cfg-&gt;ba_pg_shift = cfg-&gt;buf_pg_shift;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mtr_free_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)</div><div class='add'>+{</div><div class='add'>+	hns_roce_hem_list_release(hr_dev, &amp;mtr-&gt;hem_list);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * hns_roce_mtr_create - Create hns memory translate region.</div><div class='add'>+ *</div><div class='add'>+ * @hr_dev: RoCE device struct pointer</div><div class='add'>+ * @mtr: memory translate region</div><div class='add'>+ * @buf_attr: buffer attribute for creating mtr</div><div class='add'>+ * @ba_page_shift: page shift for multi-hop base address table</div><div class='add'>+ * @udata: user space context, if it's NULL, means kernel space</div><div class='add'>+ * @user_addr: userspace virtual address to start at</div><div class='add'>+ */</div><div class='add'>+int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,</div><div class='add'>+			struct hns_roce_buf_attr *buf_attr,</div><div class='add'>+			unsigned int ba_page_shift, struct ib_udata *udata,</div><div class='add'>+			unsigned long user_addr)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	trace_hns_buf_attr(buf_attr);</div><div class='add'>+	/* The caller has its own buffer list and invokes the hns_roce_mtr_map()</div><div class='add'>+	 * to finish the MTT configuration.</div><div class='add'>+	 */</div><div class='add'>+	if (buf_attr-&gt;mtt_only) {</div><div class='add'>+		mtr-&gt;umem = NULL;</div><div class='add'>+		mtr-&gt;kmem = NULL;</div><div class='add'>+	} else {</div><div class='add'>+		ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, udata, user_addr);</div><div class='add'>+		if (ret) {</div><div class='add'>+			ibdev_err(ibdev,</div><div class='add'>+				  "failed to alloc mtr bufs, ret = %d.\n", ret);</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ret = get_best_page_shift(hr_dev, mtr, buf_attr);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_init_buf;</div><div class='add'>+</div><div class='add'>+		ret = get_best_hop_num(hr_dev, mtr, buf_attr, ba_page_shift);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_init_buf;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = mtr_init_buf_cfg(hr_dev, mtr, buf_attr);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_init_buf;</div><div class='add'>+</div><div class='add'>+	ret = mtr_alloc_mtt(hr_dev, mtr, ba_page_shift);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to alloc mtr mtt, ret = %d.\n", ret);</div><div class='add'>+		goto err_init_buf;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (buf_attr-&gt;mtt_only)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	/* Write buffer's dma address to MTT */</div><div class='add'>+	ret = mtr_map_bufs(hr_dev, mtr);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to map mtr bufs, ret = %d.\n", ret);</div><div class='add'>+		goto err_alloc_mtt;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='add'>+</div><div class='add'>+err_alloc_mtt:</div><div class='add'>+	mtr_free_mtt(hr_dev, mtr);</div><div class='add'>+err_init_buf:</div><div class='add'>+	mtr_free_bufs(hr_dev, mtr);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)</div><div class='add'>+{</div><div class='add'>+	/* release multi-hop addressing resource */</div><div class='add'>+	hns_roce_hem_list_release(hr_dev, &amp;mtr-&gt;hem_list);</div><div class='add'>+</div><div class='add'>+	/* free buffers */</div><div class='add'>+	mtr_free_bufs(hr_dev, mtr);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c<br/>index 813401384d78..225c3e328e0e 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_pd.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_pd.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_pd.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_pd.c</a></div><div class='hunk'>@@ -30,76 +30,70 @@</div><div class='ctx'>  * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/platform_device.h&gt;</div><div class='del'>-#include &lt;linux/pci.h&gt;</div><div class='del'>-#include &lt;uapi/rdma/hns-abi.h&gt;</div><div class='ctx'> #include "hns_roce_device.h"</div><div class='ctx'> </div><div class='del'>-static int hns_roce_pd_alloc(struct hns_roce_dev *hr_dev, unsigned long *pdn)</div><div class='add'>+void hns_roce_init_pd_table(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	return hns_roce_bitmap_alloc(&amp;hr_dev-&gt;pd_bitmap, pdn) ? -ENOMEM : 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void hns_roce_pd_free(struct hns_roce_dev *hr_dev, unsigned long pdn)</div><div class='del'>-{</div><div class='del'>-	hns_roce_bitmap_free(&amp;hr_dev-&gt;pd_bitmap, pdn, BITMAP_NO_RR);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	return hns_roce_bitmap_init(&amp;hr_dev-&gt;pd_bitmap, hr_dev-&gt;caps.num_pds,</div><div class='del'>-				    hr_dev-&gt;caps.num_pds - 1,</div><div class='del'>-				    hr_dev-&gt;caps.reserved_pds, 0);</div><div class='del'>-}</div><div class='add'>+	struct hns_roce_ida *pd_ida = &amp;hr_dev-&gt;pd_ida;</div><div class='ctx'> </div><div class='del'>-void hns_roce_cleanup_pd_table(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	hns_roce_bitmap_cleanup(&amp;hr_dev-&gt;pd_bitmap);</div><div class='add'>+	ida_init(&amp;pd_ida-&gt;ida);</div><div class='add'>+	pd_ida-&gt;max = hr_dev-&gt;caps.num_pds - 1;</div><div class='add'>+	pd_ida-&gt;min = hr_dev-&gt;caps.reserved_pds;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *ib_dev = ibpd-&gt;device;</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='add'>+	struct hns_roce_ida *pd_ida = &amp;hr_dev-&gt;pd_ida;</div><div class='ctx'> 	struct hns_roce_pd *pd = to_hr_pd(ibpd);</div><div class='del'>-	int ret;</div><div class='add'>+	int ret = 0;</div><div class='add'>+	int id;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_pd_alloc(to_hr_dev(ib_dev), &amp;pd-&gt;pdn);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "[alloc_pd]hns_roce_pd_alloc failed!\n");</div><div class='del'>-		return ret;</div><div class='add'>+	id = ida_alloc_range(&amp;pd_ida-&gt;ida, pd_ida-&gt;min, pd_ida-&gt;max,</div><div class='add'>+			     GFP_KERNEL);</div><div class='add'>+	if (id &lt; 0) {</div><div class='add'>+		ibdev_err(ib_dev, "failed to alloc pd, id = %d.\n", id);</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='add'>+	pd-&gt;pdn = (unsigned long)id;</div><div class='ctx'> </div><div class='ctx'> 	if (udata) {</div><div class='del'>-		struct hns_roce_ib_alloc_pd_resp uresp = {.pdn = pd-&gt;pdn};</div><div class='add'>+		struct hns_roce_ib_alloc_pd_resp resp = {.pdn = pd-&gt;pdn};</div><div class='ctx'> </div><div class='del'>-		if (ib_copy_to_udata(udata, &amp;uresp, sizeof(uresp))) {</div><div class='del'>-			hns_roce_pd_free(to_hr_dev(ib_dev), pd-&gt;pdn);</div><div class='del'>-			dev_err(dev, "[alloc_pd]ib_copy_to_udata failed!\n");</div><div class='del'>-			return -EFAULT;</div><div class='add'>+		ret = ib_copy_to_udata(udata, &amp;resp,</div><div class='add'>+				       min(udata-&gt;outlen, sizeof(resp)));</div><div class='add'>+		if (ret) {</div><div class='add'>+			ida_free(&amp;pd_ida-&gt;ida, id);</div><div class='add'>+			ibdev_err(ib_dev, "failed to copy to udata, ret = %d\n", ret);</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_alloc_pd);</div><div class='ctx'> </div><div class='del'>-void hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)</div><div class='add'>+int hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	hns_roce_pd_free(to_hr_dev(pd-&gt;device), to_hr_pd(pd)-&gt;pdn);</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(pd-&gt;device);</div><div class='add'>+</div><div class='add'>+	ida_free(&amp;hr_dev-&gt;pd_ida.ida, (int)to_hr_pd(pd)-&gt;pdn);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_dealloc_pd);</div><div class='ctx'> </div><div class='ctx'> int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar)</div><div class='ctx'> {</div><div class='del'>-	struct resource *res;</div><div class='del'>-	int ret = 0;</div><div class='add'>+	struct hns_roce_ida *uar_ida = &amp;hr_dev-&gt;uar_ida;</div><div class='add'>+	int id;</div><div class='ctx'> </div><div class='ctx'> 	/* Using bitmap to manager UAR index */</div><div class='del'>-	ret = hns_roce_bitmap_alloc(&amp;hr_dev-&gt;uar_table.bitmap, &amp;uar-&gt;logic_idx);</div><div class='del'>-	if (ret == -1)</div><div class='add'>+	id = ida_alloc_range(&amp;uar_ida-&gt;ida, uar_ida-&gt;min, uar_ida-&gt;max,</div><div class='add'>+			     GFP_KERNEL);</div><div class='add'>+	if (id &lt; 0) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev, "failed to alloc uar id(%d).\n", id);</div><div class='ctx'> 		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+	uar-&gt;logic_idx = (unsigned long)id;</div><div class='ctx'> </div><div class='ctx'> 	if (uar-&gt;logic_idx &gt; 0 &amp;&amp; hr_dev-&gt;caps.phy_num_uars &gt; 1)</div><div class='ctx'> 		uar-&gt;index = (uar-&gt;logic_idx - 1) %</div><div class='hunk'>@@ -107,36 +101,73 @@ int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar)</div><div class='ctx'> 	else</div><div class='ctx'> 		uar-&gt;index = 0;</div><div class='ctx'> </div><div class='del'>-	if (!dev_is_pci(hr_dev-&gt;dev)) {</div><div class='del'>-		res = platform_get_resource(hr_dev-&gt;pdev, IORESOURCE_MEM, 0);</div><div class='del'>-		if (!res) {</div><div class='del'>-			dev_err(&amp;hr_dev-&gt;pdev-&gt;dev, "memory resource not found!\n");</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-		uar-&gt;pfn = ((res-&gt;start) &gt;&gt; PAGE_SHIFT) + uar-&gt;index;</div><div class='del'>-	} else {</div><div class='del'>-		uar-&gt;pfn = ((pci_resource_start(hr_dev-&gt;pci_dev, 2))</div><div class='del'>-			   &gt;&gt; PAGE_SHIFT);</div><div class='add'>+	uar-&gt;pfn = ((pci_resource_start(hr_dev-&gt;pci_dev, 2)) &gt;&gt; PAGE_SHIFT);</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_DIRECT_WQE)</div><div class='add'>+		hr_dev-&gt;dwqe_page = pci_resource_start(hr_dev-&gt;pci_dev, 4);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hns_roce_init_uar_table(struct hns_roce_dev *hr_dev)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_ida *uar_ida = &amp;hr_dev-&gt;uar_ida;</div><div class='add'>+</div><div class='add'>+	ida_init(&amp;uar_ida-&gt;ida);</div><div class='add'>+	uar_ida-&gt;max = hr_dev-&gt;caps.num_uars - 1;</div><div class='add'>+	uar_ida-&gt;min = hr_dev-&gt;caps.reserved_uars;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_xrcd_alloc(struct hns_roce_dev *hr_dev, u32 *xrcdn)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_ida *xrcd_ida = &amp;hr_dev-&gt;xrcd_ida;</div><div class='add'>+	int id;</div><div class='add'>+</div><div class='add'>+	id = ida_alloc_range(&amp;xrcd_ida-&gt;ida, xrcd_ida-&gt;min, xrcd_ida-&gt;max,</div><div class='add'>+			     GFP_KERNEL);</div><div class='add'>+	if (id &lt; 0) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev, "failed to alloc xrcdn(%d).\n", id);</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='add'>+	*xrcdn = (u32)id;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void hns_roce_uar_free(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar)</div><div class='add'>+void hns_roce_init_xrcd_table(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	hns_roce_bitmap_free(&amp;hr_dev-&gt;uar_table.bitmap, uar-&gt;logic_idx,</div><div class='del'>-			     BITMAP_NO_RR);</div><div class='add'>+	struct hns_roce_ida *xrcd_ida = &amp;hr_dev-&gt;xrcd_ida;</div><div class='add'>+</div><div class='add'>+	ida_init(&amp;xrcd_ida-&gt;ida);</div><div class='add'>+	xrcd_ida-&gt;max = hr_dev-&gt;caps.num_xrcds - 1;</div><div class='add'>+	xrcd_ida-&gt;min = hr_dev-&gt;caps.reserved_xrcds;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int hns_roce_init_uar_table(struct hns_roce_dev *hr_dev)</div><div class='add'>+int hns_roce_alloc_xrcd(struct ib_xrcd *ib_xrcd, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	return hns_roce_bitmap_init(&amp;hr_dev-&gt;uar_table.bitmap,</div><div class='del'>-				    hr_dev-&gt;caps.num_uars,</div><div class='del'>-				    hr_dev-&gt;caps.num_uars - 1,</div><div class='del'>-				    hr_dev-&gt;caps.reserved_uars, 0);</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_xrcd-&gt;device);</div><div class='add'>+	struct hns_roce_xrcd *xrcd = to_hr_xrcd(ib_xrcd);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!(hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_XRC)) {</div><div class='add'>+		ret = -EOPNOTSUPP;</div><div class='add'>+		goto err_out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_xrcd_alloc(hr_dev, &amp;xrcd-&gt;xrcdn);</div><div class='add'>+</div><div class='add'>+err_out:</div><div class='add'>+	if (ret)</div><div class='add'>+		atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_XRCD_ALLOC_ERR_CNT]);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void hns_roce_cleanup_uar_table(struct hns_roce_dev *hr_dev)</div><div class='add'>+int hns_roce_dealloc_xrcd(struct ib_xrcd *ib_xrcd, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	hns_roce_bitmap_cleanup(&amp;hr_dev-&gt;uar_table.bitmap);</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_xrcd-&gt;device);</div><div class='add'>+	u32 xrcdn = to_hr_xrcd(ib_xrcd)-&gt;xrcdn;</div><div class='add'>+</div><div class='add'>+	ida_free(&amp;hr_dev-&gt;xrcd_ida.ida, (int)xrcdn);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c<br/>index 8db2817a249e..d1640c5fbaab 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_qp.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_qp.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_qp.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_qp.c</a></div><div class='hunk'>@@ -31,46 +31,127 @@</div><div class='ctx'>  * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/pci.h&gt;</div><div class='del'>-#include &lt;linux/platform_device.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_addr.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_umem.h&gt;</div><div class='ctx'> #include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='ctx'> #include "hns_roce_common.h"</div><div class='ctx'> #include "hns_roce_device.h"</div><div class='ctx'> #include "hns_roce_hem.h"</div><div class='del'>-#include &lt;rdma/hns-abi.h&gt;</div><div class='ctx'> </div><div class='del'>-#define SQP_NUM				(2 * HNS_ROCE_MAX_PORTS)</div><div class='del'>-</div><div class='del'>-void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type)</div><div class='add'>+static struct hns_roce_qp *hns_roce_qp_lookup(struct hns_roce_dev *hr_dev,</div><div class='add'>+					      u32 qpn)</div><div class='ctx'> {</div><div class='ctx'> 	struct device *dev = hr_dev-&gt;dev;</div><div class='ctx'> 	struct hns_roce_qp *qp;</div><div class='add'>+	unsigned long flags;</div><div class='ctx'> </div><div class='del'>-	xa_lock(&amp;hr_dev-&gt;qp_table_xa);</div><div class='add'>+	xa_lock_irqsave(&amp;hr_dev-&gt;qp_table_xa, flags);</div><div class='ctx'> 	qp = __hns_roce_qp_lookup(hr_dev, qpn);</div><div class='ctx'> 	if (qp)</div><div class='del'>-		atomic_inc(&amp;qp-&gt;refcount);</div><div class='del'>-	xa_unlock(&amp;hr_dev-&gt;qp_table_xa);</div><div class='add'>+		refcount_inc(&amp;qp-&gt;refcount);</div><div class='add'>+	xa_unlock_irqrestore(&amp;hr_dev-&gt;qp_table_xa, flags);</div><div class='add'>+</div><div class='add'>+	if (!qp)</div><div class='add'>+		dev_warn(dev, "async event for bogus QP %08x\n", qpn);</div><div class='add'>+</div><div class='add'>+	return qp;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void flush_work_handle(struct work_struct *work)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_work *flush_work = container_of(work,</div><div class='add'>+					struct hns_roce_work, work);</div><div class='add'>+	struct hns_roce_qp *hr_qp = container_of(flush_work,</div><div class='add'>+					struct hns_roce_qp, flush_work);</div><div class='add'>+	struct device *dev = flush_work-&gt;hr_dev-&gt;dev;</div><div class='add'>+	struct ib_qp_attr attr;</div><div class='add'>+	int attr_mask;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	if (!qp) {</div><div class='del'>-		dev_warn(dev, "Async event for bogus QP %08x\n", qpn);</div><div class='add'>+	attr_mask = IB_QP_STATE;</div><div class='add'>+	attr.qp_state = IB_QPS_ERR;</div><div class='add'>+</div><div class='add'>+	if (test_and_clear_bit(HNS_ROCE_FLUSH_FLAG, &amp;hr_qp-&gt;flush_flag)) {</div><div class='add'>+		ret = hns_roce_modify_qp(&amp;hr_qp-&gt;ibqp, &amp;attr, attr_mask, NULL);</div><div class='add'>+		if (ret)</div><div class='add'>+			dev_err(dev, "modify QP to error state failed(%d) during CQE flush\n",</div><div class='add'>+				ret);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * make sure we signal QP destroy leg that flush QP was completed</div><div class='add'>+	 * so that it can safely proceed ahead now and destroy QP</div><div class='add'>+	 */</div><div class='add'>+	if (refcount_dec_and_test(&amp;hr_qp-&gt;refcount))</div><div class='add'>+		complete(&amp;hr_qp-&gt;free);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_work *flush_work = &amp;hr_qp-&gt;flush_work;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;hr_qp-&gt;flush_lock, flags);</div><div class='add'>+	/* Exit directly after destroy_qp() */</div><div class='add'>+	if (test_bit(HNS_ROCE_STOP_FLUSH_FLAG, &amp;hr_qp-&gt;flush_flag)) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;hr_qp-&gt;flush_lock, flags);</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	refcount_inc(&amp;hr_qp-&gt;refcount);</div><div class='add'>+	queue_work(hr_dev-&gt;irq_workq, &amp;flush_work-&gt;work);</div><div class='add'>+	spin_unlock_irqrestore(&amp;hr_qp-&gt;flush_lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp)</div><div class='add'>+{</div><div class='add'>+	/*</div><div class='add'>+	 * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state</div><div class='add'>+	 * gets into errored mode. Hence, as a workaround to this</div><div class='add'>+	 * hardware limitation, driver needs to assist in flushing. But</div><div class='add'>+	 * the flushing operation uses mailbox to convey the QP state to</div><div class='add'>+	 * the hardware and which can sleep due to the mutex protection</div><div class='add'>+	 * around the mailbox calls. Hence, use the deferred flush for</div><div class='add'>+	 * now.</div><div class='add'>+	 */</div><div class='add'>+	if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &amp;qp-&gt;flush_flag))</div><div class='add'>+		init_flush_work(dev, qp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_qp *qp;</div><div class='add'>+</div><div class='add'>+	qp = hns_roce_qp_lookup(hr_dev, qpn);</div><div class='add'>+	if (!qp)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='ctx'> 	qp-&gt;event(qp, (enum hns_roce_event)event_type);</div><div class='ctx'> </div><div class='del'>-	if (atomic_dec_and_test(&amp;qp-&gt;refcount))</div><div class='add'>+	if (refcount_dec_and_test(&amp;qp-&gt;refcount))</div><div class='add'>+		complete(&amp;qp-&gt;free);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void hns_roce_flush_cqe(struct hns_roce_dev *hr_dev, u32 qpn)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_qp *qp;</div><div class='add'>+</div><div class='add'>+	qp = hns_roce_qp_lookup(hr_dev, qpn);</div><div class='add'>+	if (!qp)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	qp-&gt;state = IB_QPS_ERR;</div><div class='add'>+	flush_cqe(hr_dev, qp);</div><div class='add'>+</div><div class='add'>+	if (refcount_dec_and_test(&amp;qp-&gt;refcount))</div><div class='ctx'> 		complete(&amp;qp-&gt;free);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_qp_event);</div><div class='ctx'> </div><div class='ctx'> static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp,</div><div class='ctx'> 				 enum hns_roce_event type)</div><div class='ctx'> {</div><div class='del'>-	struct ib_event event;</div><div class='ctx'> 	struct ib_qp *ibqp = &amp;hr_qp-&gt;ibqp;</div><div class='add'>+	struct ib_event event;</div><div class='ctx'> </div><div class='ctx'> 	if (ibqp-&gt;event_handler) {</div><div class='ctx'> 		event.device = ibqp-&gt;device;</div><div class='hunk'>@@ -98,6 +179,8 @@ static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp,</div><div class='ctx'> 			event.event = IB_EVENT_QP_REQ_ERR;</div><div class='ctx'> 			break;</div><div class='ctx'> 		case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:</div><div class='add'>+		case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:</div><div class='add'>+		case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:</div><div class='ctx'> 			event.event = IB_EVENT_QP_ACCESS_ERR;</div><div class='ctx'> 			break;</div><div class='ctx'> 		default:</div><div class='hunk'>@@ -109,82 +192,156 @@ static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp,</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_reserve_range_qp(struct hns_roce_dev *hr_dev, int cnt,</div><div class='del'>-				     int align, unsigned long *base)</div><div class='add'>+static u8 get_affinity_cq_bank(u8 qp_bank)</div><div class='add'>+{</div><div class='add'>+	return (qp_bank &gt;&gt; 1) &amp; CQ_BANKID_MASK;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u8 get_least_load_bankid_for_qp(struct ib_qp_init_attr *init_attr,</div><div class='add'>+					struct hns_roce_bank *bank)</div><div class='add'>+{</div><div class='add'>+#define INVALID_LOAD_QPNUM 0xFFFFFFFF</div><div class='add'>+	struct ib_cq *scq = init_attr-&gt;send_cq;</div><div class='add'>+	u32 least_load = INVALID_LOAD_QPNUM;</div><div class='add'>+	unsigned long cqn = 0;</div><div class='add'>+	u8 bankid = 0;</div><div class='add'>+	u32 bankcnt;</div><div class='add'>+	u8 i;</div><div class='add'>+</div><div class='add'>+	if (scq)</div><div class='add'>+		cqn = to_hr_cq(scq)-&gt;cqn;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; HNS_ROCE_QP_BANK_NUM; i++) {</div><div class='add'>+		if (scq &amp;&amp; (get_affinity_cq_bank(i) != (cqn &amp; CQ_BANKID_MASK)))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		bankcnt = bank[i].inuse;</div><div class='add'>+		if (bankcnt &lt; least_load) {</div><div class='add'>+			least_load = bankcnt;</div><div class='add'>+			bankid = i;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return bankid;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int alloc_qpn_with_bankid(struct hns_roce_bank *bank, u8 bankid,</div><div class='add'>+				 unsigned long *qpn)</div><div class='add'>+{</div><div class='add'>+	int id;</div><div class='add'>+</div><div class='add'>+	id = ida_alloc_range(&amp;bank-&gt;ida, bank-&gt;next, bank-&gt;max, GFP_KERNEL);</div><div class='add'>+	if (id &lt; 0) {</div><div class='add'>+		id = ida_alloc_range(&amp;bank-&gt;ida, bank-&gt;min, bank-&gt;max,</div><div class='add'>+				     GFP_KERNEL);</div><div class='add'>+		if (id &lt; 0)</div><div class='add'>+			return id;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* the QPN should keep increasing until the max value is reached. */</div><div class='add'>+	bank-&gt;next = (id + 1) &gt; bank-&gt;max ? bank-&gt;min : id + 1;</div><div class='add'>+</div><div class='add'>+	/* the lower 3 bits is bankid */</div><div class='add'>+	*qpn = (id &lt;&lt; 3) | bankid;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,</div><div class='add'>+		     struct ib_qp_init_attr *init_attr)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_qp_table *qp_table = &amp;hr_dev-&gt;qp_table;</div><div class='add'>+	unsigned long num = 0;</div><div class='add'>+	u8 bankid;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	return hns_roce_bitmap_alloc_range(&amp;qp_table-&gt;bitmap, cnt, align,</div><div class='del'>-					   base) ?</div><div class='del'>-		       -ENOMEM :</div><div class='del'>-		       0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state)</div><div class='del'>-{</div><div class='del'>-	switch (state) {</div><div class='del'>-	case IB_QPS_RESET:</div><div class='del'>-		return HNS_ROCE_QP_STATE_RST;</div><div class='del'>-	case IB_QPS_INIT:</div><div class='del'>-		return HNS_ROCE_QP_STATE_INIT;</div><div class='del'>-	case IB_QPS_RTR:</div><div class='del'>-		return HNS_ROCE_QP_STATE_RTR;</div><div class='del'>-	case IB_QPS_RTS:</div><div class='del'>-		return HNS_ROCE_QP_STATE_RTS;</div><div class='del'>-	case IB_QPS_SQD:</div><div class='del'>-		return HNS_ROCE_QP_STATE_SQD;</div><div class='del'>-	case IB_QPS_ERR:</div><div class='del'>-		return HNS_ROCE_QP_STATE_ERR;</div><div class='del'>-	default:</div><div class='del'>-		return HNS_ROCE_QP_NUM_STATE;</div><div class='add'>+	if (hr_qp-&gt;ibqp.qp_type == IB_QPT_GSI) {</div><div class='add'>+		num = 1;</div><div class='add'>+	} else {</div><div class='add'>+		mutex_lock(&amp;qp_table-&gt;bank_mutex);</div><div class='add'>+		bankid = get_least_load_bankid_for_qp(init_attr, qp_table-&gt;bank);</div><div class='add'>+</div><div class='add'>+		ret = alloc_qpn_with_bankid(&amp;qp_table-&gt;bank[bankid], bankid,</div><div class='add'>+					    &amp;num);</div><div class='add'>+		if (ret) {</div><div class='add'>+			ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+				  "failed to alloc QPN, ret = %d\n", ret);</div><div class='add'>+			mutex_unlock(&amp;qp_table-&gt;bank_mutex);</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		qp_table-&gt;bank[bankid].inuse++;</div><div class='add'>+		mutex_unlock(&amp;qp_table-&gt;bank_mutex);</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	hr_qp-&gt;qpn = num;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(to_hns_roce_state);</div><div class='ctx'> </div><div class='del'>-static int hns_roce_gsi_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn,</div><div class='del'>-				 struct hns_roce_qp *hr_qp)</div><div class='add'>+static void add_qp_to_list(struct hns_roce_dev *hr_dev,</div><div class='add'>+			   struct hns_roce_qp *hr_qp,</div><div class='add'>+			   struct ib_cq *send_cq, struct ib_cq *recv_cq)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_cq *hr_send_cq, *hr_recv_cq;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	hr_send_cq = send_cq ? to_hr_cq(send_cq) : NULL;</div><div class='add'>+	hr_recv_cq = recv_cq ? to_hr_cq(recv_cq) : NULL;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;hr_dev-&gt;qp_list_lock, flags);</div><div class='add'>+	hns_roce_lock_cqs(hr_send_cq, hr_recv_cq);</div><div class='add'>+</div><div class='add'>+	list_add_tail(&amp;hr_qp-&gt;node, &amp;hr_dev-&gt;qp_list);</div><div class='add'>+	if (hr_send_cq)</div><div class='add'>+		list_add_tail(&amp;hr_qp-&gt;sq_node, &amp;hr_send_cq-&gt;sq_list);</div><div class='add'>+	if (hr_recv_cq)</div><div class='add'>+		list_add_tail(&amp;hr_qp-&gt;rq_node, &amp;hr_recv_cq-&gt;rq_list);</div><div class='add'>+</div><div class='add'>+	hns_roce_unlock_cqs(hr_send_cq, hr_recv_cq);</div><div class='add'>+	spin_unlock_irqrestore(&amp;hr_dev-&gt;qp_list_lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_qp_store(struct hns_roce_dev *hr_dev,</div><div class='add'>+			     struct hns_roce_qp *hr_qp,</div><div class='add'>+			     struct ib_qp_init_attr *init_attr)</div><div class='ctx'> {</div><div class='ctx'> 	struct xarray *xa = &amp;hr_dev-&gt;qp_table_xa;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	if (!qpn)</div><div class='add'>+	if (!hr_qp-&gt;qpn)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	hr_qp-&gt;qpn = qpn;</div><div class='del'>-	atomic_set(&amp;hr_qp-&gt;refcount, 1);</div><div class='del'>-	init_completion(&amp;hr_qp-&gt;free);</div><div class='del'>-</div><div class='del'>-	ret = xa_err(xa_store_irq(xa, hr_qp-&gt;qpn &amp; (hr_dev-&gt;caps.num_qps - 1),</div><div class='del'>-				hr_qp, GFP_KERNEL));</div><div class='add'>+	ret = xa_err(xa_store_irq(xa, hr_qp-&gt;qpn, hr_qp, GFP_KERNEL));</div><div class='ctx'> 	if (ret)</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "QPC xa_store failed\n");</div><div class='add'>+		dev_err(hr_dev-&gt;dev, "failed to xa store for QPC\n");</div><div class='add'>+	else</div><div class='add'>+		/* add QP to device's QP list for softwc */</div><div class='add'>+		add_qp_to_list(hr_dev, hr_qp, init_attr-&gt;send_cq,</div><div class='add'>+			       init_attr-&gt;recv_cq);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn,</div><div class='del'>-			     struct hns_roce_qp *hr_qp)</div><div class='add'>+static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_qp_table *qp_table = &amp;hr_dev-&gt;qp_table;</div><div class='ctx'> 	struct device *dev = hr_dev-&gt;dev;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	if (!qpn)</div><div class='add'>+	if (!hr_qp-&gt;qpn)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	hr_qp-&gt;qpn = qpn;</div><div class='del'>-</div><div class='ctx'> 	/* Alloc memory for QPC */</div><div class='ctx'> 	ret = hns_roce_table_get(hr_dev, &amp;qp_table-&gt;qp_table, hr_qp-&gt;qpn);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "QPC table get failed\n");</div><div class='add'>+		dev_err(dev, "failed to get QPC table\n");</div><div class='ctx'> 		goto err_out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Alloc memory for IRRL */</div><div class='ctx'> 	ret = hns_roce_table_get(hr_dev, &amp;qp_table-&gt;irrl_table, hr_qp-&gt;qpn);</div><div class='ctx'> 	if (ret) {</div><div class='del'>-		dev_err(dev, "IRRL table get failed\n");</div><div class='add'>+		dev_err(dev, "failed to get IRRL table\n");</div><div class='ctx'> 		goto err_put_qp;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -193,32 +350,23 @@ static int hns_roce_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn,</div><div class='ctx'> 		ret = hns_roce_table_get(hr_dev, &amp;qp_table-&gt;trrl_table,</div><div class='ctx'> 					 hr_qp-&gt;qpn);</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			dev_err(dev, "TRRL table get failed\n");</div><div class='add'>+			dev_err(dev, "failed to get TRRL table\n");</div><div class='ctx'> 			goto err_put_irrl;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (hr_dev-&gt;caps.sccc_entry_sz) {</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {</div><div class='ctx'> 		/* Alloc memory for SCC CTX */</div><div class='ctx'> 		ret = hns_roce_table_get(hr_dev, &amp;qp_table-&gt;sccc_table,</div><div class='ctx'> 					 hr_qp-&gt;qpn);</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			dev_err(dev, "SCC CTX table get failed\n");</div><div class='add'>+			dev_err(dev, "failed to get SCC CTX table\n");</div><div class='ctx'> 			goto err_put_trrl;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_put_sccc;</div><div class='del'>-</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-err_put_sccc:</div><div class='del'>-	if (hr_dev-&gt;caps.sccc_entry_sz)</div><div class='del'>-		hns_roce_table_put(hr_dev, &amp;qp_table-&gt;sccc_table,</div><div class='del'>-				   hr_qp-&gt;qpn);</div><div class='del'>-</div><div class='ctx'> err_put_trrl:</div><div class='ctx'> 	if (hr_dev-&gt;caps.trrl_entry_sz)</div><div class='ctx'> 		hns_roce_table_put(hr_dev, &amp;qp_table-&gt;trrl_table, hr_qp-&gt;qpn);</div><div class='hunk'>@@ -233,61 +381,92 @@ err_out:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void qp_user_mmap_entry_remove(struct hns_roce_qp *hr_qp)</div><div class='add'>+{</div><div class='add'>+	rdma_user_mmap_entry_remove(&amp;hr_qp-&gt;dwqe_mmap_entry-&gt;rdma_entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)</div><div class='ctx'> {</div><div class='ctx'> 	struct xarray *xa = &amp;hr_dev-&gt;qp_table_xa;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> </div><div class='add'>+	list_del(&amp;hr_qp-&gt;node);</div><div class='add'>+</div><div class='add'>+	if (hr_qp-&gt;ibqp.qp_type != IB_QPT_XRC_TGT)</div><div class='add'>+		list_del(&amp;hr_qp-&gt;sq_node);</div><div class='add'>+</div><div class='add'>+	if (hr_qp-&gt;ibqp.qp_type != IB_QPT_XRC_INI &amp;&amp;</div><div class='add'>+	    hr_qp-&gt;ibqp.qp_type != IB_QPT_XRC_TGT)</div><div class='add'>+		list_del(&amp;hr_qp-&gt;rq_node);</div><div class='add'>+</div><div class='ctx'> 	xa_lock_irqsave(xa, flags);</div><div class='del'>-	__xa_erase(xa, hr_qp-&gt;qpn &amp; (hr_dev-&gt;caps.num_qps - 1));</div><div class='add'>+	__xa_erase(xa, hr_qp-&gt;qpn);</div><div class='ctx'> 	xa_unlock_irqrestore(xa, flags);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_qp_remove);</div><div class='ctx'> </div><div class='del'>-void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)</div><div class='add'>+static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_qp_table *qp_table = &amp;hr_dev-&gt;qp_table;</div><div class='ctx'> </div><div class='del'>-	if (atomic_dec_and_test(&amp;hr_qp-&gt;refcount))</div><div class='del'>-		complete(&amp;hr_qp-&gt;free);</div><div class='del'>-	wait_for_completion(&amp;hr_qp-&gt;free);</div><div class='add'>+	if (hr_dev-&gt;caps.trrl_entry_sz)</div><div class='add'>+		hns_roce_table_put(hr_dev, &amp;qp_table-&gt;trrl_table, hr_qp-&gt;qpn);</div><div class='add'>+	hns_roce_table_put(hr_dev, &amp;qp_table-&gt;irrl_table, hr_qp-&gt;qpn);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if ((hr_qp-&gt;ibqp.qp_type) != IB_QPT_GSI) {</div><div class='del'>-		if (hr_dev-&gt;caps.trrl_entry_sz)</div><div class='del'>-			hns_roce_table_put(hr_dev, &amp;qp_table-&gt;trrl_table,</div><div class='del'>-					   hr_qp-&gt;qpn);</div><div class='del'>-		hns_roce_table_put(hr_dev, &amp;qp_table-&gt;irrl_table, hr_qp-&gt;qpn);</div><div class='del'>-		hns_roce_table_put(hr_dev, &amp;qp_table-&gt;qp_table, hr_qp-&gt;qpn);</div><div class='del'>-	}</div><div class='add'>+static inline u8 get_qp_bankid(unsigned long qpn)</div><div class='add'>+{</div><div class='add'>+	/* The lower 3 bits of QPN are used to hash to different banks */</div><div class='add'>+	return (u8)(qpn &amp; GENMASK(2, 0));</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_qp_free);</div><div class='ctx'> </div><div class='del'>-void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn,</div><div class='del'>-			       int cnt)</div><div class='add'>+static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_qp_table *qp_table = &amp;hr_dev-&gt;qp_table;</div><div class='add'>+	u8 bankid;</div><div class='add'>+</div><div class='add'>+	if (hr_qp-&gt;ibqp.qp_type == IB_QPT_GSI)</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-	if (base_qpn &lt; SQP_NUM)</div><div class='add'>+	if (hr_qp-&gt;qpn &lt; hr_dev-&gt;caps.reserved_qps)</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='del'>-	hns_roce_bitmap_free_range(&amp;qp_table-&gt;bitmap, base_qpn, cnt, BITMAP_RR);</div><div class='add'>+	bankid = get_qp_bankid(hr_qp-&gt;qpn);</div><div class='add'>+</div><div class='add'>+	ida_free(&amp;hr_dev-&gt;qp_table.bank[bankid].ida,</div><div class='add'>+		 hr_qp-&gt;qpn / HNS_ROCE_QP_BANK_NUM);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;hr_dev-&gt;qp_table.bank_mutex);</div><div class='add'>+	hr_dev-&gt;qp_table.bank[bankid].inuse--;</div><div class='add'>+	mutex_unlock(&amp;hr_dev-&gt;qp_table.bank_mutex);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_release_range_qp);</div><div class='ctx'> </div><div class='del'>-static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev,</div><div class='del'>-				struct ib_qp_cap *cap, bool is_user, int has_rq,</div><div class='del'>-				struct hns_roce_qp *hr_qp)</div><div class='add'>+static u32 proc_rq_sge(struct hns_roce_dev *dev, struct hns_roce_qp *hr_qp,</div><div class='add'>+		       bool user)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	u32 max_cnt;</div><div class='add'>+	u32 max_sge = dev-&gt;caps.max_rq_sg;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09)</div><div class='add'>+		return max_sge;</div><div class='add'>+</div><div class='add'>+	/* Reserve SGEs only for HIP08 in kernel; The userspace driver will</div><div class='add'>+	 * calculate number of max_sge with reserved SGEs when allocating wqe</div><div class='add'>+	 * buf, so there is no need to do this again in kernel. But the number</div><div class='add'>+	 * may exceed the capacity of SGEs recorded in the firmware, so the</div><div class='add'>+	 * kernel driver should just adapt the value accordingly.</div><div class='add'>+	 */</div><div class='add'>+	if (user)</div><div class='add'>+		max_sge = roundup_pow_of_two(max_sge + 1);</div><div class='add'>+	else</div><div class='add'>+		hr_qp-&gt;rq.rsv_sge = 1;</div><div class='ctx'> </div><div class='del'>-	/* Check the validity of QP support capacity */</div><div class='del'>-	if (cap-&gt;max_recv_wr &gt; hr_dev-&gt;caps.max_wqes ||</div><div class='del'>-	    cap-&gt;max_recv_sge &gt; hr_dev-&gt;caps.max_rq_sg) {</div><div class='del'>-		dev_err(dev, "RQ WR or sge error!max_recv_wr=%d max_recv_sge=%d\n",</div><div class='del'>-			cap-&gt;max_recv_wr, cap-&gt;max_recv_sge);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='add'>+	return max_sge;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap,</div><div class='add'>+		       struct hns_roce_qp *hr_qp, int has_rq, bool user)</div><div class='add'>+{</div><div class='add'>+	u32 max_sge = proc_rq_sge(hr_dev, hr_qp, user);</div><div class='add'>+	u32 cnt;</div><div class='ctx'> </div><div class='ctx'> 	/* If srq exist, set zero for relative number of rq */</div><div class='ctx'> 	if (!has_rq) {</div><div class='hunk'>@@ -295,217 +474,274 @@ static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev,</div><div class='ctx'> 		hr_qp-&gt;rq.max_gs = 0;</div><div class='ctx'> 		cap-&gt;max_recv_wr = 0;</div><div class='ctx'> 		cap-&gt;max_recv_sge = 0;</div><div class='del'>-	} else {</div><div class='del'>-		if (is_user &amp;&amp; (!cap-&gt;max_recv_wr || !cap-&gt;max_recv_sge)) {</div><div class='del'>-			dev_err(dev, "user space no need config max_recv_wr max_recv_sge\n");</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (hr_dev-&gt;caps.min_wqes)</div><div class='del'>-			max_cnt = max(cap-&gt;max_recv_wr, hr_dev-&gt;caps.min_wqes);</div><div class='del'>-		else</div><div class='del'>-			max_cnt = cap-&gt;max_recv_wr;</div><div class='ctx'> </div><div class='del'>-		hr_qp-&gt;rq.wqe_cnt = roundup_pow_of_two(max_cnt);</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		if ((u32)hr_qp-&gt;rq.wqe_cnt &gt; hr_dev-&gt;caps.max_wqes) {</div><div class='del'>-			dev_err(dev, "while setting rq size, rq.wqe_cnt too large\n");</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='add'>+	/* Check the validity of QP support capacity */</div><div class='add'>+	if (!cap-&gt;max_recv_wr || cap-&gt;max_recv_wr &gt; hr_dev-&gt;caps.max_wqes ||</div><div class='add'>+	    cap-&gt;max_recv_sge &gt; max_sge) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+			  "RQ config error, depth = %u, sge = %u\n",</div><div class='add'>+			  cap-&gt;max_recv_wr, cap-&gt;max_recv_sge);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		max_cnt = max(1U, cap-&gt;max_recv_sge);</div><div class='del'>-		hr_qp-&gt;rq.max_gs = roundup_pow_of_two(max_cnt);</div><div class='del'>-		if (hr_dev-&gt;caps.max_rq_sg &lt;= 2)</div><div class='del'>-			hr_qp-&gt;rq.wqe_shift =</div><div class='del'>-					ilog2(hr_dev-&gt;caps.max_rq_desc_sz);</div><div class='del'>-		else</div><div class='del'>-			hr_qp-&gt;rq.wqe_shift =</div><div class='del'>-					ilog2(hr_dev-&gt;caps.max_rq_desc_sz</div><div class='del'>-					      * hr_qp-&gt;rq.max_gs);</div><div class='add'>+	cnt = roundup_pow_of_two(max(cap-&gt;max_recv_wr, hr_dev-&gt;caps.min_wqes));</div><div class='add'>+	if (cnt &gt; hr_dev-&gt;caps.max_wqes) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev, "rq depth %u too large\n",</div><div class='add'>+			  cap-&gt;max_recv_wr);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	cap-&gt;max_recv_wr = hr_qp-&gt;rq.max_post = hr_qp-&gt;rq.wqe_cnt;</div><div class='del'>-	cap-&gt;max_recv_sge = hr_qp-&gt;rq.max_gs;</div><div class='add'>+	hr_qp-&gt;rq.max_gs = roundup_pow_of_two(max(1U, cap-&gt;max_recv_sge) +</div><div class='add'>+					      hr_qp-&gt;rq.rsv_sge);</div><div class='add'>+</div><div class='add'>+	hr_qp-&gt;rq.wqe_shift = ilog2(hr_dev-&gt;caps.max_rq_desc_sz *</div><div class='add'>+				    hr_qp-&gt;rq.max_gs);</div><div class='add'>+</div><div class='add'>+	hr_qp-&gt;rq.wqe_cnt = cnt;</div><div class='add'>+</div><div class='add'>+	cap-&gt;max_recv_wr = cnt;</div><div class='add'>+	cap-&gt;max_recv_sge = hr_qp-&gt;rq.max_gs - hr_qp-&gt;rq.rsv_sge;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_set_user_sq_size(struct hns_roce_dev *hr_dev,</div><div class='del'>-				     struct ib_qp_cap *cap,</div><div class='del'>-				     struct hns_roce_qp *hr_qp,</div><div class='del'>-				     struct hns_roce_ib_create_qp *ucmd)</div><div class='add'>+static u32 get_max_inline_data(struct hns_roce_dev *hr_dev,</div><div class='add'>+			       struct ib_qp_cap *cap)</div><div class='add'>+{</div><div class='add'>+	if (cap-&gt;max_inline_data) {</div><div class='add'>+		cap-&gt;max_inline_data = roundup_pow_of_two(cap-&gt;max_inline_data);</div><div class='add'>+		return min(cap-&gt;max_inline_data,</div><div class='add'>+			   hr_dev-&gt;caps.max_sq_inline);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void update_inline_data(struct hns_roce_qp *hr_qp,</div><div class='add'>+			       struct ib_qp_cap *cap)</div><div class='add'>+{</div><div class='add'>+	u32 sge_num = hr_qp-&gt;sq.ext_sge_cnt;</div><div class='add'>+</div><div class='add'>+	if (hr_qp-&gt;config &amp; HNS_ROCE_EXSGE_FLAGS) {</div><div class='add'>+		if (!(hr_qp-&gt;ibqp.qp_type == IB_QPT_GSI ||</div><div class='add'>+		      hr_qp-&gt;ibqp.qp_type == IB_QPT_UD))</div><div class='add'>+			sge_num = max((u32)HNS_ROCE_SGE_IN_WQE, sge_num);</div><div class='add'>+</div><div class='add'>+		cap-&gt;max_inline_data = max(cap-&gt;max_inline_data,</div><div class='add'>+					   sge_num * HNS_ROCE_SGE_SIZE);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hr_qp-&gt;max_inline_data = cap-&gt;max_inline_data;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u32 get_sge_num_from_max_send_sge(bool is_ud_or_gsi,</div><div class='add'>+					 u32 max_send_sge)</div><div class='add'>+{</div><div class='add'>+	unsigned int std_sge_num;</div><div class='add'>+	unsigned int min_sge;</div><div class='add'>+</div><div class='add'>+	std_sge_num = is_ud_or_gsi ? 0 : HNS_ROCE_SGE_IN_WQE;</div><div class='add'>+	min_sge = is_ud_or_gsi ? 1 : 0;</div><div class='add'>+	return max_send_sge &gt; std_sge_num ? (max_send_sge - std_sge_num) :</div><div class='add'>+				min_sge;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static unsigned int get_sge_num_from_max_inl_data(bool is_ud_or_gsi,</div><div class='add'>+						  u32 max_inline_data)</div><div class='add'>+{</div><div class='add'>+	unsigned int inline_sge;</div><div class='add'>+</div><div class='add'>+	if (!max_inline_data)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * if max_inline_data less than</div><div class='add'>+	 * HNS_ROCE_SGE_IN_WQE * HNS_ROCE_SGE_SIZE,</div><div class='add'>+	 * In addition to ud's mode, no need to extend sge.</div><div class='add'>+	 */</div><div class='add'>+	inline_sge = roundup_pow_of_two(max_inline_data) / HNS_ROCE_SGE_SIZE;</div><div class='add'>+	if (!is_ud_or_gsi &amp;&amp; inline_sge &lt;= HNS_ROCE_SGE_IN_WQE)</div><div class='add'>+		inline_sge = 0;</div><div class='add'>+</div><div class='add'>+	return inline_sge;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt,</div><div class='add'>+			      struct hns_roce_qp *hr_qp, struct ib_qp_cap *cap)</div><div class='add'>+{</div><div class='add'>+	bool is_ud_or_gsi = (hr_qp-&gt;ibqp.qp_type == IB_QPT_GSI ||</div><div class='add'>+				hr_qp-&gt;ibqp.qp_type == IB_QPT_UD);</div><div class='add'>+	unsigned int std_sge_num;</div><div class='add'>+	u32 inline_ext_sge = 0;</div><div class='add'>+	u32 ext_wqe_sge_cnt;</div><div class='add'>+	u32 total_sge_cnt;</div><div class='add'>+</div><div class='add'>+	cap-&gt;max_inline_data = get_max_inline_data(hr_dev, cap);</div><div class='add'>+</div><div class='add'>+	hr_qp-&gt;sge.sge_shift = HNS_ROCE_SGE_SHIFT;</div><div class='add'>+	std_sge_num = is_ud_or_gsi ? 0 : HNS_ROCE_SGE_IN_WQE;</div><div class='add'>+	ext_wqe_sge_cnt = get_sge_num_from_max_send_sge(is_ud_or_gsi,</div><div class='add'>+							cap-&gt;max_send_sge);</div><div class='add'>+</div><div class='add'>+	if (hr_qp-&gt;config &amp; HNS_ROCE_EXSGE_FLAGS) {</div><div class='add'>+		inline_ext_sge = max(ext_wqe_sge_cnt,</div><div class='add'>+				     get_sge_num_from_max_inl_data(is_ud_or_gsi,</div><div class='add'>+							 cap-&gt;max_inline_data));</div><div class='add'>+		hr_qp-&gt;sq.ext_sge_cnt = inline_ext_sge ?</div><div class='add'>+					roundup_pow_of_two(inline_ext_sge) : 0;</div><div class='add'>+</div><div class='add'>+		hr_qp-&gt;sq.max_gs = max(1U, (hr_qp-&gt;sq.ext_sge_cnt + std_sge_num));</div><div class='add'>+		hr_qp-&gt;sq.max_gs = min(hr_qp-&gt;sq.max_gs, hr_dev-&gt;caps.max_sq_sg);</div><div class='add'>+</div><div class='add'>+		ext_wqe_sge_cnt = hr_qp-&gt;sq.ext_sge_cnt;</div><div class='add'>+	} else {</div><div class='add'>+		hr_qp-&gt;sq.max_gs = max(1U, cap-&gt;max_send_sge);</div><div class='add'>+		hr_qp-&gt;sq.max_gs = min(hr_qp-&gt;sq.max_gs, hr_dev-&gt;caps.max_sq_sg);</div><div class='add'>+		hr_qp-&gt;sq.ext_sge_cnt = hr_qp-&gt;sq.max_gs;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* If the number of extended sge is not zero, they MUST use the</div><div class='add'>+	 * space of HNS_HW_PAGE_SIZE at least.</div><div class='add'>+	 */</div><div class='add'>+	if (ext_wqe_sge_cnt) {</div><div class='add'>+		total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * ext_wqe_sge_cnt);</div><div class='add'>+		hr_qp-&gt;sge.sge_cnt = max(total_sge_cnt,</div><div class='add'>+				(u32)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	update_inline_data(hr_qp, cap);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev,</div><div class='add'>+					struct ib_qp_cap *cap,</div><div class='add'>+					struct hns_roce_ib_create_qp *ucmd)</div><div class='ctx'> {</div><div class='ctx'> 	u32 roundup_sq_stride = roundup_pow_of_two(hr_dev-&gt;caps.max_sq_desc_sz);</div><div class='ctx'> 	u8 max_sq_stride = ilog2(roundup_sq_stride);</div><div class='del'>-	u32 ex_sge_num;</div><div class='del'>-	u32 page_size;</div><div class='del'>-	u32 max_cnt;</div><div class='ctx'> </div><div class='ctx'> 	/* Sanity check SQ size before proceeding */</div><div class='del'>-	if ((u32)(1 &lt;&lt; ucmd-&gt;log_sq_bb_count) &gt; hr_dev-&gt;caps.max_wqes ||</div><div class='del'>-	     ucmd-&gt;log_sq_stride &gt; max_sq_stride ||</div><div class='del'>-	     ucmd-&gt;log_sq_stride &lt; HNS_ROCE_IB_MIN_SQ_STRIDE) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "check SQ size error!\n");</div><div class='add'>+	if (ucmd-&gt;log_sq_stride &gt; max_sq_stride ||</div><div class='add'>+	    ucmd-&gt;log_sq_stride &lt; HNS_ROCE_IB_MIN_SQ_STRIDE) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev, "failed to check SQ stride size.\n");</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (cap-&gt;max_send_sge &gt; hr_dev-&gt;caps.max_sq_sg) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "SQ sge error! max_send_sge=%d\n",</div><div class='del'>-			cap-&gt;max_send_sge);</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev, "failed to check SQ SGE size %u.\n",</div><div class='add'>+			  cap-&gt;max_send_sge);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	hr_qp-&gt;sq.wqe_cnt = 1 &lt;&lt; ucmd-&gt;log_sq_bb_count;</div><div class='del'>-	hr_qp-&gt;sq.wqe_shift = ucmd-&gt;log_sq_stride;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	max_cnt = max(1U, cap-&gt;max_send_sge);</div><div class='del'>-	if (hr_dev-&gt;caps.max_sq_sg &lt;= 2)</div><div class='del'>-		hr_qp-&gt;sq.max_gs = roundup_pow_of_two(max_cnt);</div><div class='del'>-	else</div><div class='del'>-		hr_qp-&gt;sq.max_gs = max_cnt;</div><div class='add'>+static int set_user_sq_size(struct hns_roce_dev *hr_dev,</div><div class='add'>+			    struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp,</div><div class='add'>+			    struct hns_roce_ib_create_qp *ucmd)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	u32 cnt = 0;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	if (hr_qp-&gt;sq.max_gs &gt; 2)</div><div class='del'>-		hr_qp-&gt;sge.sge_cnt = roundup_pow_of_two(hr_qp-&gt;sq.wqe_cnt *</div><div class='del'>-							(hr_qp-&gt;sq.max_gs - 2));</div><div class='add'>+	if (check_shl_overflow(1, ucmd-&gt;log_sq_bb_count, &amp;cnt) ||</div><div class='add'>+	    cnt &gt; hr_dev-&gt;caps.max_wqes)</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if ((hr_qp-&gt;sq.max_gs &gt; 2) &amp;&amp; (hr_dev-&gt;pci_dev-&gt;revision == 0x20)) {</div><div class='del'>-		if (hr_qp-&gt;sge.sge_cnt &gt; hr_dev-&gt;caps.max_extend_sg) {</div><div class='del'>-			dev_err(hr_dev-&gt;dev,</div><div class='del'>-				"The extended sge cnt error! sge_cnt=%d\n",</div><div class='del'>-				hr_qp-&gt;sge.sge_cnt);</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='add'>+	ret = check_sq_size_with_integrity(hr_dev, cap, ucmd);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to check user SQ size, ret = %d.\n",</div><div class='add'>+			  ret);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	hr_qp-&gt;sge.sge_shift = 4;</div><div class='del'>-	ex_sge_num = hr_qp-&gt;sge.sge_cnt;</div><div class='add'>+	set_ext_sge_param(hr_dev, cnt, hr_qp, cap);</div><div class='ctx'> </div><div class='del'>-	/* Get buf size, SQ and RQ  are aligned to page_szie */</div><div class='del'>-	if (hr_dev-&gt;caps.max_sq_sg &lt;= 2) {</div><div class='del'>-		hr_qp-&gt;buff_size = HNS_ROCE_ALOGN_UP((hr_qp-&gt;rq.wqe_cnt &lt;&lt;</div><div class='del'>-					     hr_qp-&gt;rq.wqe_shift), PAGE_SIZE) +</div><div class='del'>-				   HNS_ROCE_ALOGN_UP((hr_qp-&gt;sq.wqe_cnt &lt;&lt;</div><div class='del'>-					     hr_qp-&gt;sq.wqe_shift), PAGE_SIZE);</div><div class='del'>-</div><div class='del'>-		hr_qp-&gt;sq.offset = 0;</div><div class='del'>-		hr_qp-&gt;rq.offset = HNS_ROCE_ALOGN_UP((hr_qp-&gt;sq.wqe_cnt &lt;&lt;</div><div class='del'>-					     hr_qp-&gt;sq.wqe_shift), PAGE_SIZE);</div><div class='del'>-	} else {</div><div class='del'>-		page_size = 1 &lt;&lt; (hr_dev-&gt;caps.mtt_buf_pg_sz + PAGE_SHIFT);</div><div class='del'>-		hr_qp-&gt;sge.sge_cnt =</div><div class='del'>-		       max(page_size / (1 &lt;&lt; hr_qp-&gt;sge.sge_shift), ex_sge_num);</div><div class='del'>-		hr_qp-&gt;buff_size = HNS_ROCE_ALOGN_UP((hr_qp-&gt;rq.wqe_cnt &lt;&lt;</div><div class='del'>-					     hr_qp-&gt;rq.wqe_shift), page_size) +</div><div class='del'>-				   HNS_ROCE_ALOGN_UP((hr_qp-&gt;sge.sge_cnt &lt;&lt;</div><div class='del'>-					     hr_qp-&gt;sge.sge_shift), page_size) +</div><div class='del'>-				   HNS_ROCE_ALOGN_UP((hr_qp-&gt;sq.wqe_cnt &lt;&lt;</div><div class='del'>-					     hr_qp-&gt;sq.wqe_shift), page_size);</div><div class='del'>-</div><div class='del'>-		hr_qp-&gt;sq.offset = 0;</div><div class='del'>-		if (ex_sge_num) {</div><div class='del'>-			hr_qp-&gt;sge.offset = HNS_ROCE_ALOGN_UP(</div><div class='del'>-							(hr_qp-&gt;sq.wqe_cnt &lt;&lt;</div><div class='del'>-							hr_qp-&gt;sq.wqe_shift),</div><div class='del'>-							page_size);</div><div class='del'>-			hr_qp-&gt;rq.offset = hr_qp-&gt;sge.offset +</div><div class='del'>-					HNS_ROCE_ALOGN_UP((hr_qp-&gt;sge.sge_cnt &lt;&lt;</div><div class='del'>-						hr_qp-&gt;sge.sge_shift),</div><div class='del'>-						page_size);</div><div class='del'>-		} else {</div><div class='del'>-			hr_qp-&gt;rq.offset = HNS_ROCE_ALOGN_UP(</div><div class='del'>-							(hr_qp-&gt;sq.wqe_cnt &lt;&lt;</div><div class='del'>-							hr_qp-&gt;sq.wqe_shift),</div><div class='del'>-							page_size);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='add'>+	hr_qp-&gt;sq.wqe_shift = ucmd-&gt;log_sq_stride;</div><div class='add'>+	hr_qp-&gt;sq.wqe_cnt = cnt;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev,</div><div class='del'>-				       struct ib_qp_cap *cap,</div><div class='del'>-				       struct hns_roce_qp *hr_qp)</div><div class='add'>+static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev,</div><div class='add'>+			    struct hns_roce_qp *hr_qp,</div><div class='add'>+			    struct hns_roce_buf_attr *buf_attr)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	u32 page_size;</div><div class='del'>-	u32 max_cnt;</div><div class='del'>-	int size;</div><div class='del'>-</div><div class='del'>-	if (cap-&gt;max_send_wr  &gt; hr_dev-&gt;caps.max_wqes  ||</div><div class='del'>-	    cap-&gt;max_send_sge &gt; hr_dev-&gt;caps.max_sq_sg ||</div><div class='del'>-	    cap-&gt;max_inline_data &gt; hr_dev-&gt;caps.max_sq_inline) {</div><div class='del'>-		dev_err(dev, "SQ WR or sge or inline data error!\n");</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	int buf_size;</div><div class='add'>+	int idx = 0;</div><div class='add'>+</div><div class='add'>+	hr_qp-&gt;buff_size = 0;</div><div class='add'>+</div><div class='add'>+	/* SQ WQE */</div><div class='add'>+	hr_qp-&gt;sq.offset = 0;</div><div class='add'>+	buf_size = to_hr_hem_entries_size(hr_qp-&gt;sq.wqe_cnt,</div><div class='add'>+					  hr_qp-&gt;sq.wqe_shift);</div><div class='add'>+	if (buf_size &gt; 0 &amp;&amp; idx &lt; ARRAY_SIZE(buf_attr-&gt;region)) {</div><div class='add'>+		buf_attr-&gt;region[idx].size = buf_size;</div><div class='add'>+		buf_attr-&gt;region[idx].hopnum = hr_dev-&gt;caps.wqe_sq_hop_num;</div><div class='add'>+		idx++;</div><div class='add'>+		hr_qp-&gt;buff_size += buf_size;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	hr_qp-&gt;sq.wqe_shift = ilog2(hr_dev-&gt;caps.max_sq_desc_sz);</div><div class='del'>-	hr_qp-&gt;sq_max_wqes_per_wr = 1;</div><div class='del'>-	hr_qp-&gt;sq_spare_wqes = 0;</div><div class='add'>+	/* extend SGE WQE in SQ */</div><div class='add'>+	hr_qp-&gt;sge.offset = hr_qp-&gt;buff_size;</div><div class='add'>+	buf_size = to_hr_hem_entries_size(hr_qp-&gt;sge.sge_cnt,</div><div class='add'>+					  hr_qp-&gt;sge.sge_shift);</div><div class='add'>+	if (buf_size &gt; 0 &amp;&amp; idx &lt; ARRAY_SIZE(buf_attr-&gt;region)) {</div><div class='add'>+		buf_attr-&gt;region[idx].size = buf_size;</div><div class='add'>+		buf_attr-&gt;region[idx].hopnum = hr_dev-&gt;caps.wqe_sge_hop_num;</div><div class='add'>+		idx++;</div><div class='add'>+		hr_qp-&gt;buff_size += buf_size;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (hr_dev-&gt;caps.min_wqes)</div><div class='del'>-		max_cnt = max(cap-&gt;max_send_wr, hr_dev-&gt;caps.min_wqes);</div><div class='del'>-	else</div><div class='del'>-		max_cnt = cap-&gt;max_send_wr;</div><div class='add'>+	/* RQ WQE */</div><div class='add'>+	hr_qp-&gt;rq.offset = hr_qp-&gt;buff_size;</div><div class='add'>+	buf_size = to_hr_hem_entries_size(hr_qp-&gt;rq.wqe_cnt,</div><div class='add'>+					  hr_qp-&gt;rq.wqe_shift);</div><div class='add'>+	if (buf_size &gt; 0 &amp;&amp; idx &lt; ARRAY_SIZE(buf_attr-&gt;region)) {</div><div class='add'>+		buf_attr-&gt;region[idx].size = buf_size;</div><div class='add'>+		buf_attr-&gt;region[idx].hopnum = hr_dev-&gt;caps.wqe_rq_hop_num;</div><div class='add'>+		idx++;</div><div class='add'>+		hr_qp-&gt;buff_size += buf_size;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	hr_qp-&gt;sq.wqe_cnt = roundup_pow_of_two(max_cnt);</div><div class='del'>-	if ((u32)hr_qp-&gt;sq.wqe_cnt &gt; hr_dev-&gt;caps.max_wqes) {</div><div class='del'>-		dev_err(dev, "while setting kernel sq size, sq.wqe_cnt too large\n");</div><div class='add'>+	if (hr_qp-&gt;buff_size &lt; 1)</div><div class='ctx'> 		return -EINVAL;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	/* Get data_seg numbers */</div><div class='del'>-	max_cnt = max(1U, cap-&gt;max_send_sge);</div><div class='del'>-	if (hr_dev-&gt;caps.max_sq_sg &lt;= 2)</div><div class='del'>-		hr_qp-&gt;sq.max_gs = roundup_pow_of_two(max_cnt);</div><div class='del'>-	else</div><div class='del'>-		hr_qp-&gt;sq.max_gs = max_cnt;</div><div class='add'>+	buf_attr-&gt;page_shift = HNS_HW_PAGE_SHIFT + hr_dev-&gt;caps.mtt_buf_pg_sz;</div><div class='add'>+	buf_attr-&gt;region_count = idx;</div><div class='ctx'> </div><div class='del'>-	if (hr_qp-&gt;sq.max_gs &gt; 2) {</div><div class='del'>-		hr_qp-&gt;sge.sge_cnt = roundup_pow_of_two(hr_qp-&gt;sq.wqe_cnt *</div><div class='del'>-				     (hr_qp-&gt;sq.max_gs - 2));</div><div class='del'>-		hr_qp-&gt;sge.sge_shift = 4;</div><div class='del'>-	}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	/* ud sqwqe's sge use extend sge */</div><div class='del'>-	if (hr_dev-&gt;caps.max_sq_sg &gt; 2 &amp;&amp; hr_qp-&gt;ibqp.qp_type == IB_QPT_GSI) {</div><div class='del'>-		hr_qp-&gt;sge.sge_cnt = roundup_pow_of_two(hr_qp-&gt;sq.wqe_cnt *</div><div class='del'>-				     hr_qp-&gt;sq.max_gs);</div><div class='del'>-		hr_qp-&gt;sge.sge_shift = 4;</div><div class='del'>-	}</div><div class='add'>+static int set_kernel_sq_size(struct hns_roce_dev *hr_dev,</div><div class='add'>+			      struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	u32 cnt;</div><div class='ctx'> </div><div class='del'>-	if ((hr_qp-&gt;sq.max_gs &gt; 2) &amp;&amp; hr_dev-&gt;pci_dev-&gt;revision == 0x20) {</div><div class='del'>-		if (hr_qp-&gt;sge.sge_cnt &gt; hr_dev-&gt;caps.max_extend_sg) {</div><div class='del'>-			dev_err(dev, "The extended sge cnt error! sge_cnt=%d\n",</div><div class='del'>-				hr_qp-&gt;sge.sge_cnt);</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='add'>+	if (!cap-&gt;max_send_wr || cap-&gt;max_send_wr &gt; hr_dev-&gt;caps.max_wqes ||</div><div class='add'>+	    cap-&gt;max_send_sge &gt; hr_dev-&gt;caps.max_sq_sg) {</div><div class='add'>+		ibdev_err(ibdev, "failed to check SQ WR or SGE num.\n");</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	/* Get buf size, SQ and RQ are aligned to PAGE_SIZE */</div><div class='del'>-	page_size = 1 &lt;&lt; (hr_dev-&gt;caps.mtt_buf_pg_sz + PAGE_SHIFT);</div><div class='del'>-	hr_qp-&gt;sq.offset = 0;</div><div class='del'>-	size = HNS_ROCE_ALOGN_UP(hr_qp-&gt;sq.wqe_cnt &lt;&lt; hr_qp-&gt;sq.wqe_shift,</div><div class='del'>-				 page_size);</div><div class='del'>-</div><div class='del'>-	if (hr_dev-&gt;caps.max_sq_sg &gt; 2 &amp;&amp; hr_qp-&gt;sge.sge_cnt) {</div><div class='del'>-		hr_qp-&gt;sge.sge_cnt = max(page_size/(1 &lt;&lt; hr_qp-&gt;sge.sge_shift),</div><div class='del'>-					(u32)hr_qp-&gt;sge.sge_cnt);</div><div class='del'>-		hr_qp-&gt;sge.offset = size;</div><div class='del'>-		size += HNS_ROCE_ALOGN_UP(hr_qp-&gt;sge.sge_cnt &lt;&lt;</div><div class='del'>-					  hr_qp-&gt;sge.sge_shift, page_size);</div><div class='add'>+	cnt = roundup_pow_of_two(max(cap-&gt;max_send_wr, hr_dev-&gt;caps.min_wqes));</div><div class='add'>+	if (cnt &gt; hr_dev-&gt;caps.max_wqes) {</div><div class='add'>+		ibdev_err(ibdev, "failed to check WQE num, WQE num = %u.\n",</div><div class='add'>+			  cnt);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	hr_qp-&gt;rq.offset = size;</div><div class='del'>-	size += HNS_ROCE_ALOGN_UP((hr_qp-&gt;rq.wqe_cnt &lt;&lt; hr_qp-&gt;rq.wqe_shift),</div><div class='del'>-				  page_size);</div><div class='del'>-	hr_qp-&gt;buff_size = size;</div><div class='add'>+	hr_qp-&gt;sq.wqe_shift = ilog2(hr_dev-&gt;caps.max_sq_desc_sz);</div><div class='add'>+	hr_qp-&gt;sq.wqe_cnt = cnt;</div><div class='ctx'> </div><div class='del'>-	/* Get wr and sge number which send */</div><div class='del'>-	cap-&gt;max_send_wr = hr_qp-&gt;sq.max_post = hr_qp-&gt;sq.wqe_cnt;</div><div class='del'>-	cap-&gt;max_send_sge = hr_qp-&gt;sq.max_gs;</div><div class='add'>+	set_ext_sge_param(hr_dev, cnt, hr_qp, cap);</div><div class='ctx'> </div><div class='del'>-	/* We don't support inline sends for kernel QPs (yet) */</div><div class='del'>-	cap-&gt;max_inline_data = 0;</div><div class='add'>+	/* sync the parameters of kernel QP to user's configuration */</div><div class='add'>+	cap-&gt;max_send_wr = cnt;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -528,522 +764,718 @@ static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr)</div><div class='ctx'> 	return 1;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,</div><div class='del'>-				     struct ib_pd *ib_pd,</div><div class='del'>-				     struct ib_qp_init_attr *init_attr,</div><div class='del'>-				     struct ib_udata *udata, unsigned long sqpn,</div><div class='del'>-				     struct hns_roce_qp *hr_qp)</div><div class='add'>+static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,</div><div class='add'>+			struct ib_qp_init_attr *init_attr,</div><div class='add'>+			struct ib_udata *udata, unsigned long addr)</div><div class='ctx'> {</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	struct hns_roce_ib_create_qp ucmd;</div><div class='del'>-	struct hns_roce_ib_create_qp_resp resp = {};</div><div class='del'>-	struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(</div><div class='del'>-		udata, struct hns_roce_ucontext, ibucontext);</div><div class='del'>-	unsigned long qpn = 0;</div><div class='del'>-	int ret = 0;</div><div class='del'>-	u32 page_shift;</div><div class='del'>-	u32 npages;</div><div class='del'>-	int i;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	struct hns_roce_buf_attr buf_attr = {};</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	mutex_init(&amp;hr_qp-&gt;mutex);</div><div class='del'>-	spin_lock_init(&amp;hr_qp-&gt;sq.lock);</div><div class='del'>-	spin_lock_init(&amp;hr_qp-&gt;rq.lock);</div><div class='add'>+	ret = set_wqe_buf_attr(hr_dev, hr_qp, &amp;buf_attr);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret);</div><div class='add'>+		goto err_inline;</div><div class='add'>+	}</div><div class='add'>+	ret = hns_roce_mtr_create(hr_dev, &amp;hr_qp-&gt;mtr, &amp;buf_attr,</div><div class='add'>+				  PAGE_SHIFT + hr_dev-&gt;caps.mtt_ba_pg_sz,</div><div class='add'>+				  udata, addr);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret);</div><div class='add'>+		goto err_inline;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	hr_qp-&gt;state = IB_QPS_RESET;</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_DIRECT_WQE)</div><div class='add'>+		hr_qp-&gt;en_flags |= HNS_ROCE_QP_CAP_DIRECT_WQE;</div><div class='ctx'> </div><div class='del'>-	hr_qp-&gt;ibqp.qp_type = init_attr-&gt;qp_type;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;sq_sig_type == IB_SIGNAL_ALL_WR)</div><div class='del'>-		hr_qp-&gt;sq_signal_bits = cpu_to_le32(IB_SIGNAL_ALL_WR);</div><div class='del'>-	else</div><div class='del'>-		hr_qp-&gt;sq_signal_bits = cpu_to_le32(IB_SIGNAL_REQ_WR);</div><div class='add'>+err_inline:</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_set_rq_size(hr_dev, &amp;init_attr-&gt;cap, udata,</div><div class='del'>-				   hns_roce_qp_has_rq(init_attr), hr_qp);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(dev, "hns_roce_set_rq_size failed\n");</div><div class='del'>-		goto err_out;</div><div class='del'>-	}</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if ((hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RQ_INLINE) &amp;&amp;</div><div class='del'>-	    hns_roce_qp_has_rq(init_attr)) {</div><div class='del'>-		/* allocate recv inline buf */</div><div class='del'>-		hr_qp-&gt;rq_inl_buf.wqe_list = kcalloc(hr_qp-&gt;rq.wqe_cnt,</div><div class='del'>-					       sizeof(struct hns_roce_rinl_wqe),</div><div class='del'>-					       GFP_KERNEL);</div><div class='del'>-		if (!hr_qp-&gt;rq_inl_buf.wqe_list) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto err_out;</div><div class='del'>-		}</div><div class='add'>+static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)</div><div class='add'>+{</div><div class='add'>+	hns_roce_mtr_destroy(hr_dev, &amp;hr_qp-&gt;mtr);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		hr_qp-&gt;rq_inl_buf.wqe_cnt = hr_qp-&gt;rq.wqe_cnt;</div><div class='add'>+static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev,</div><div class='add'>+				   struct ib_qp_init_attr *init_attr,</div><div class='add'>+				   struct ib_udata *udata,</div><div class='add'>+				   struct hns_roce_ib_create_qp_resp *resp,</div><div class='add'>+				   struct hns_roce_ib_create_qp *ucmd)</div><div class='add'>+{</div><div class='add'>+	return ((hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_QP_RECORD_DB) &amp;&amp;</div><div class='add'>+		udata-&gt;outlen &gt;= offsetofend(typeof(*resp), cap_flags) &amp;&amp;</div><div class='add'>+		hns_roce_qp_has_sq(init_attr) &amp;&amp;</div><div class='add'>+		udata-&gt;inlen &gt;= offsetofend(typeof(*ucmd), sdb_addr));</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		/* Firstly, allocate a list of sge space buffer */</div><div class='del'>-		hr_qp-&gt;rq_inl_buf.wqe_list[0].sg_list =</div><div class='del'>-					kcalloc(hr_qp-&gt;rq_inl_buf.wqe_cnt,</div><div class='del'>-					       init_attr-&gt;cap.max_recv_sge *</div><div class='del'>-					       sizeof(struct hns_roce_rinl_sge),</div><div class='del'>-					       GFP_KERNEL);</div><div class='del'>-		if (!hr_qp-&gt;rq_inl_buf.wqe_list[0].sg_list) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto err_wqe_list;</div><div class='del'>-		}</div><div class='add'>+static inline bool user_qp_has_rdb(struct hns_roce_dev *hr_dev,</div><div class='add'>+				   struct ib_qp_init_attr *init_attr,</div><div class='add'>+				   struct ib_udata *udata,</div><div class='add'>+				   struct hns_roce_ib_create_qp_resp *resp)</div><div class='add'>+{</div><div class='add'>+	return ((hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_QP_RECORD_DB) &amp;&amp;</div><div class='add'>+		udata-&gt;outlen &gt;= offsetofend(typeof(*resp), cap_flags) &amp;&amp;</div><div class='add'>+		hns_roce_qp_has_rq(init_attr));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev,</div><div class='add'>+				     struct ib_qp_init_attr *init_attr)</div><div class='add'>+{</div><div class='add'>+	return ((hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_QP_RECORD_DB) &amp;&amp;</div><div class='add'>+		hns_roce_qp_has_rq(init_attr));</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		for (i = 1; i &lt; hr_qp-&gt;rq_inl_buf.wqe_cnt; i++)</div><div class='del'>-			/* Secondly, reallocate the buffer */</div><div class='del'>-			hr_qp-&gt;rq_inl_buf.wqe_list[i].sg_list =</div><div class='del'>-				&amp;hr_qp-&gt;rq_inl_buf.wqe_list[0].sg_list[i *</div><div class='del'>-				init_attr-&gt;cap.max_recv_sge];</div><div class='add'>+static int qp_mmap_entry(struct hns_roce_qp *hr_qp,</div><div class='add'>+			 struct hns_roce_dev *hr_dev,</div><div class='add'>+			 struct ib_udata *udata,</div><div class='add'>+			 struct hns_roce_ib_create_qp_resp *resp)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_ucontext *uctx =</div><div class='add'>+		rdma_udata_to_drv_context(udata,</div><div class='add'>+			struct hns_roce_ucontext, ibucontext);</div><div class='add'>+	struct rdma_user_mmap_entry *rdma_entry;</div><div class='add'>+	u64 address;</div><div class='add'>+</div><div class='add'>+	address = hr_dev-&gt;dwqe_page + hr_qp-&gt;qpn * HNS_ROCE_DWQE_SIZE;</div><div class='add'>+</div><div class='add'>+	hr_qp-&gt;dwqe_mmap_entry =</div><div class='add'>+		hns_roce_user_mmap_entry_insert(&amp;uctx-&gt;ibucontext, address,</div><div class='add'>+						HNS_ROCE_DWQE_SIZE,</div><div class='add'>+						HNS_ROCE_MMAP_TYPE_DWQE);</div><div class='add'>+</div><div class='add'>+	if (!hr_qp-&gt;dwqe_mmap_entry) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev, "failed to get dwqe mmap entry.\n");</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (udata) {</div><div class='del'>-		if (ib_copy_from_udata(&amp;ucmd, udata, sizeof(ucmd))) {</div><div class='del'>-			dev_err(dev, "ib_copy_from_udata error for create qp\n");</div><div class='del'>-			ret = -EFAULT;</div><div class='del'>-			goto err_rq_sge_list;</div><div class='del'>-		}</div><div class='add'>+	rdma_entry = &amp;hr_qp-&gt;dwqe_mmap_entry-&gt;rdma_entry;</div><div class='add'>+	resp-&gt;dwqe_mmap_key = rdma_user_mmap_get_offset(rdma_entry);</div><div class='ctx'> </div><div class='del'>-		ret = hns_roce_set_user_sq_size(hr_dev, &amp;init_attr-&gt;cap, hr_qp,</div><div class='del'>-						&amp;ucmd);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "hns_roce_set_user_sq_size error for create qp\n");</div><div class='del'>-			goto err_rq_sge_list;</div><div class='del'>-		}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		hr_qp-&gt;umem = ib_umem_get(udata, ucmd.buf_addr,</div><div class='del'>-					  hr_qp-&gt;buff_size, 0, 0);</div><div class='del'>-		if (IS_ERR(hr_qp-&gt;umem)) {</div><div class='del'>-			dev_err(dev, "ib_umem_get error for create qp\n");</div><div class='del'>-			ret = PTR_ERR(hr_qp-&gt;umem);</div><div class='del'>-			goto err_rq_sge_list;</div><div class='del'>-		}</div><div class='add'>+static int alloc_user_qp_db(struct hns_roce_dev *hr_dev,</div><div class='add'>+			    struct hns_roce_qp *hr_qp,</div><div class='add'>+			    struct ib_qp_init_attr *init_attr,</div><div class='add'>+			    struct ib_udata *udata,</div><div class='add'>+			    struct hns_roce_ib_create_qp *ucmd,</div><div class='add'>+			    struct hns_roce_ib_create_qp_resp *resp)</div><div class='add'>+{</div><div class='add'>+	bool has_sdb = user_qp_has_sdb(hr_dev, init_attr, udata, resp, ucmd);</div><div class='add'>+	struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata,</div><div class='add'>+		struct hns_roce_ucontext, ibucontext);</div><div class='add'>+	bool has_rdb = user_qp_has_rdb(hr_dev, init_attr, udata, resp);</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-		hr_qp-&gt;mtt.mtt_type = MTT_TYPE_WQE;</div><div class='del'>-		page_shift = PAGE_SHIFT;</div><div class='del'>-		if (hr_dev-&gt;caps.mtt_buf_pg_sz) {</div><div class='del'>-			npages = (ib_umem_page_count(hr_qp-&gt;umem) +</div><div class='del'>-				  (1 &lt;&lt; hr_dev-&gt;caps.mtt_buf_pg_sz) - 1) /</div><div class='del'>-				 (1 &lt;&lt; hr_dev-&gt;caps.mtt_buf_pg_sz);</div><div class='del'>-			page_shift += hr_dev-&gt;caps.mtt_buf_pg_sz;</div><div class='del'>-			ret = hns_roce_mtt_init(hr_dev, npages,</div><div class='del'>-				    page_shift,</div><div class='del'>-				    &amp;hr_qp-&gt;mtt);</div><div class='del'>-		} else {</div><div class='del'>-			ret = hns_roce_mtt_init(hr_dev,</div><div class='del'>-						ib_umem_page_count(hr_qp-&gt;umem),</div><div class='del'>-						page_shift, &amp;hr_qp-&gt;mtt);</div><div class='del'>-		}</div><div class='add'>+	if (has_sdb) {</div><div class='add'>+		ret = hns_roce_db_map_user(uctx, ucmd-&gt;sdb_addr, &amp;hr_qp-&gt;sdb);</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			dev_err(dev, "hns_roce_mtt_init error for create qp\n");</div><div class='del'>-			goto err_buf;</div><div class='add'>+			ibdev_err(ibdev,</div><div class='add'>+				  "failed to map user SQ doorbell, ret = %d.\n",</div><div class='add'>+				  ret);</div><div class='add'>+			goto err_out;</div><div class='ctx'> 		}</div><div class='add'>+		hr_qp-&gt;en_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		ret = hns_roce_ib_umem_write_mtt(hr_dev, &amp;hr_qp-&gt;mtt,</div><div class='del'>-						 hr_qp-&gt;umem);</div><div class='add'>+	if (has_rdb) {</div><div class='add'>+		ret = hns_roce_db_map_user(uctx, ucmd-&gt;db_addr, &amp;hr_qp-&gt;rdb);</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			dev_err(dev, "hns_roce_ib_umem_write_mtt error for create qp\n");</div><div class='del'>-			goto err_mtt;</div><div class='add'>+			ibdev_err(ibdev,</div><div class='add'>+				  "failed to map user RQ doorbell, ret = %d.\n",</div><div class='add'>+				  ret);</div><div class='add'>+			goto err_sdb;</div><div class='ctx'> 		}</div><div class='add'>+		hr_qp-&gt;en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		if ((hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) &amp;&amp;</div><div class='del'>-		    (udata-&gt;inlen &gt;= sizeof(ucmd)) &amp;&amp;</div><div class='del'>-		    (udata-&gt;outlen &gt;= sizeof(resp)) &amp;&amp;</div><div class='del'>-		    hns_roce_qp_has_sq(init_attr)) {</div><div class='del'>-			ret = hns_roce_db_map_user(uctx, udata, ucmd.sdb_addr,</div><div class='del'>-						   &amp;hr_qp-&gt;sdb);</div><div class='del'>-			if (ret) {</div><div class='del'>-				dev_err(dev, "sq record doorbell map failed!\n");</div><div class='del'>-				goto err_mtt;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			/* indicate kernel supports sq record db */</div><div class='del'>-			resp.cap_flags |= HNS_ROCE_SUPPORT_SQ_RECORD_DB;</div><div class='del'>-			hr_qp-&gt;sdb_en = 1;</div><div class='del'>-		}</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-		if ((hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RECORD_DB) &amp;&amp;</div><div class='del'>-		    (udata-&gt;outlen &gt;= sizeof(resp)) &amp;&amp;</div><div class='del'>-		    hns_roce_qp_has_rq(init_attr)) {</div><div class='del'>-			ret = hns_roce_db_map_user(uctx, udata, ucmd.db_addr,</div><div class='del'>-						   &amp;hr_qp-&gt;rdb);</div><div class='del'>-			if (ret) {</div><div class='del'>-				dev_err(dev, "rq record doorbell map failed!\n");</div><div class='del'>-				goto err_sq_dbmap;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			/* indicate kernel supports rq record db */</div><div class='del'>-			resp.cap_flags |= HNS_ROCE_SUPPORT_RQ_RECORD_DB;</div><div class='del'>-			hr_qp-&gt;rdb_en = 1;</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		if (init_attr-&gt;create_flags &amp;</div><div class='del'>-		    IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {</div><div class='del'>-			dev_err(dev, "init_attr-&gt;create_flags error!\n");</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto err_rq_sge_list;</div><div class='del'>-		}</div><div class='add'>+err_sdb:</div><div class='add'>+	if (has_sdb)</div><div class='add'>+		hns_roce_db_unmap_user(uctx, &amp;hr_qp-&gt;sdb);</div><div class='add'>+err_out:</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		if (init_attr-&gt;create_flags &amp; IB_QP_CREATE_IPOIB_UD_LSO) {</div><div class='del'>-			dev_err(dev, "init_attr-&gt;create_flags error!\n");</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto err_rq_sge_list;</div><div class='del'>-		}</div><div class='add'>+static int alloc_kernel_qp_db(struct hns_roce_dev *hr_dev,</div><div class='add'>+			      struct hns_roce_qp *hr_qp,</div><div class='add'>+			      struct ib_qp_init_attr *init_attr)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (hr_dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09)</div><div class='add'>+		hr_qp-&gt;sq.db_reg = hr_dev-&gt;mem_base +</div><div class='add'>+				   HNS_ROCE_DWQE_SIZE * hr_qp-&gt;qpn;</div><div class='add'>+	else</div><div class='add'>+		hr_qp-&gt;sq.db_reg = hr_dev-&gt;reg_base + hr_dev-&gt;sdb_offset +</div><div class='add'>+				   DB_REG_OFFSET * hr_dev-&gt;priv_uar.index;</div><div class='add'>+</div><div class='add'>+	hr_qp-&gt;rq.db_reg = hr_dev-&gt;reg_base + hr_dev-&gt;odb_offset +</div><div class='add'>+			   DB_REG_OFFSET * hr_dev-&gt;priv_uar.index;</div><div class='ctx'> </div><div class='del'>-		/* Set SQ size */</div><div class='del'>-		ret = hns_roce_set_kernel_sq_size(hr_dev, &amp;init_attr-&gt;cap,</div><div class='del'>-						  hr_qp);</div><div class='add'>+	if (kernel_qp_has_rdb(hr_dev, init_attr)) {</div><div class='add'>+		ret = hns_roce_alloc_db(hr_dev, &amp;hr_qp-&gt;rdb, 0);</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			dev_err(dev, "hns_roce_set_kernel_sq_size error!\n");</div><div class='del'>-			goto err_rq_sge_list;</div><div class='add'>+			ibdev_err(ibdev,</div><div class='add'>+				  "failed to alloc kernel RQ doorbell, ret = %d.\n",</div><div class='add'>+				  ret);</div><div class='add'>+			return ret;</div><div class='ctx'> 		}</div><div class='add'>+		*hr_qp-&gt;rdb.db_record = 0;</div><div class='add'>+		hr_qp-&gt;en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		/* QP doorbell register address */</div><div class='del'>-		hr_qp-&gt;sq.db_reg_l = hr_dev-&gt;reg_base + hr_dev-&gt;sdb_offset +</div><div class='del'>-				     DB_REG_OFFSET * hr_dev-&gt;priv_uar.index;</div><div class='del'>-		hr_qp-&gt;rq.db_reg_l = hr_dev-&gt;reg_base + hr_dev-&gt;odb_offset +</div><div class='del'>-				     DB_REG_OFFSET * hr_dev-&gt;priv_uar.index;</div><div class='del'>-</div><div class='del'>-		if ((hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RECORD_DB) &amp;&amp;</div><div class='del'>-		    hns_roce_qp_has_rq(init_attr)) {</div><div class='del'>-			ret = hns_roce_alloc_db(hr_dev, &amp;hr_qp-&gt;rdb, 0);</div><div class='del'>-			if (ret) {</div><div class='del'>-				dev_err(dev, "rq record doorbell alloc failed!\n");</div><div class='del'>-				goto err_rq_sge_list;</div><div class='del'>-			}</div><div class='del'>-			*hr_qp-&gt;rdb.db_record = 0;</div><div class='del'>-			hr_qp-&gt;rdb_en = 1;</div><div class='del'>-		}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		/* Allocate QP buf */</div><div class='del'>-		page_shift = PAGE_SHIFT + hr_dev-&gt;caps.mtt_buf_pg_sz;</div><div class='del'>-		if (hns_roce_buf_alloc(hr_dev, hr_qp-&gt;buff_size,</div><div class='del'>-				       (1 &lt;&lt; page_shift) * 2,</div><div class='del'>-				       &amp;hr_qp-&gt;hr_buf, page_shift)) {</div><div class='del'>-			dev_err(dev, "hns_roce_buf_alloc error!\n");</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto err_db;</div><div class='del'>-		}</div><div class='add'>+static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,</div><div class='add'>+		       struct ib_qp_init_attr *init_attr,</div><div class='add'>+		       struct ib_udata *udata,</div><div class='add'>+		       struct hns_roce_ib_create_qp *ucmd,</div><div class='add'>+		       struct hns_roce_ib_create_qp_resp *resp)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-		hr_qp-&gt;mtt.mtt_type = MTT_TYPE_WQE;</div><div class='del'>-		/* Write MTT */</div><div class='del'>-		ret = hns_roce_mtt_init(hr_dev, hr_qp-&gt;hr_buf.npages,</div><div class='del'>-					hr_qp-&gt;hr_buf.page_shift, &amp;hr_qp-&gt;mtt);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "hns_roce_mtt_init error for kernel create qp\n");</div><div class='del'>-			goto err_buf;</div><div class='del'>-		}</div><div class='add'>+	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_SDI_MODE)</div><div class='add'>+		hr_qp-&gt;en_flags |= HNS_ROCE_QP_CAP_OWNER_DB;</div><div class='ctx'> </div><div class='del'>-		ret = hns_roce_buf_write_mtt(hr_dev, &amp;hr_qp-&gt;mtt,</div><div class='del'>-					     &amp;hr_qp-&gt;hr_buf);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "hns_roce_buf_write_mtt error for kernel create qp\n");</div><div class='del'>-			goto err_mtt;</div><div class='add'>+	if (udata) {</div><div class='add'>+		if (hr_qp-&gt;en_flags &amp; HNS_ROCE_QP_CAP_DIRECT_WQE) {</div><div class='add'>+			ret = qp_mmap_entry(hr_qp, hr_dev, udata, resp);</div><div class='add'>+			if (ret)</div><div class='add'>+				return ret;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		hr_qp-&gt;sq.wrid = kcalloc(hr_qp-&gt;sq.wqe_cnt, sizeof(u64),</div><div class='del'>-					 GFP_KERNEL);</div><div class='del'>-		hr_qp-&gt;rq.wrid = kcalloc(hr_qp-&gt;rq.wqe_cnt, sizeof(u64),</div><div class='del'>-					 GFP_KERNEL);</div><div class='del'>-		if (!hr_qp-&gt;sq.wrid || !hr_qp-&gt;rq.wrid) {</div><div class='add'>+		ret = alloc_user_qp_db(hr_dev, hr_qp, init_attr, udata, ucmd,</div><div class='add'>+				       resp);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_remove_qp;</div><div class='add'>+	} else {</div><div class='add'>+		ret = alloc_kernel_qp_db(hr_dev, hr_qp, init_attr);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_remove_qp:</div><div class='add'>+	if (hr_qp-&gt;en_flags &amp; HNS_ROCE_QP_CAP_DIRECT_WQE)</div><div class='add'>+		qp_user_mmap_entry_remove(hr_qp);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,</div><div class='add'>+		       struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(</div><div class='add'>+		udata, struct hns_roce_ucontext, ibucontext);</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		if (hr_qp-&gt;en_flags &amp; HNS_ROCE_QP_CAP_RQ_RECORD_DB)</div><div class='add'>+			hns_roce_db_unmap_user(uctx, &amp;hr_qp-&gt;rdb);</div><div class='add'>+		if (hr_qp-&gt;en_flags &amp; HNS_ROCE_QP_CAP_SQ_RECORD_DB)</div><div class='add'>+			hns_roce_db_unmap_user(uctx, &amp;hr_qp-&gt;sdb);</div><div class='add'>+		if (hr_qp-&gt;en_flags &amp; HNS_ROCE_QP_CAP_DIRECT_WQE)</div><div class='add'>+			qp_user_mmap_entry_remove(hr_qp);</div><div class='add'>+	} else {</div><div class='add'>+		if (hr_qp-&gt;en_flags &amp; HNS_ROCE_QP_CAP_RQ_RECORD_DB)</div><div class='add'>+			hns_roce_free_db(hr_dev, &amp;hr_qp-&gt;rdb);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int alloc_kernel_wrid(struct hns_roce_dev *hr_dev,</div><div class='add'>+			     struct hns_roce_qp *hr_qp)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	u64 *sq_wrid = NULL;</div><div class='add'>+	u64 *rq_wrid = NULL;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	sq_wrid = kcalloc(hr_qp-&gt;sq.wqe_cnt, sizeof(u64), GFP_KERNEL);</div><div class='add'>+	if (!sq_wrid) {</div><div class='add'>+		ibdev_err(ibdev, "failed to alloc SQ wrid.\n");</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (hr_qp-&gt;rq.wqe_cnt) {</div><div class='add'>+		rq_wrid = kcalloc(hr_qp-&gt;rq.wqe_cnt, sizeof(u64), GFP_KERNEL);</div><div class='add'>+		if (!rq_wrid) {</div><div class='add'>+			ibdev_err(ibdev, "failed to alloc RQ wrid.\n");</div><div class='ctx'> 			ret = -ENOMEM;</div><div class='del'>-			goto err_wrid;</div><div class='add'>+			goto err_sq;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (sqpn) {</div><div class='del'>-		qpn = sqpn;</div><div class='del'>-	} else {</div><div class='del'>-		/* Get QPN */</div><div class='del'>-		ret = hns_roce_reserve_range_qp(hr_dev, 1, 1, &amp;qpn);</div><div class='add'>+	hr_qp-&gt;sq.wrid = sq_wrid;</div><div class='add'>+	hr_qp-&gt;rq.wrid = rq_wrid;</div><div class='add'>+	return 0;</div><div class='add'>+err_sq:</div><div class='add'>+	kfree(sq_wrid);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void free_kernel_wrid(struct hns_roce_qp *hr_qp)</div><div class='add'>+{</div><div class='add'>+	kfree(hr_qp-&gt;rq.wrid);</div><div class='add'>+	kfree(hr_qp-&gt;sq.wrid);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void default_congest_type(struct hns_roce_dev *hr_dev,</div><div class='add'>+				 struct hns_roce_qp *hr_qp)</div><div class='add'>+{</div><div class='add'>+	if (hr_qp-&gt;ibqp.qp_type == IB_QPT_UD ||</div><div class='add'>+	    hr_qp-&gt;ibqp.qp_type == IB_QPT_GSI)</div><div class='add'>+		hr_qp-&gt;cong_type = CONG_TYPE_DCQCN;</div><div class='add'>+	else</div><div class='add'>+		hr_qp-&gt;cong_type = hr_dev-&gt;caps.default_cong_type;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int set_congest_type(struct hns_roce_qp *hr_qp,</div><div class='add'>+			    struct hns_roce_ib_create_qp *ucmd)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp-&gt;ibqp.device);</div><div class='add'>+</div><div class='add'>+	switch (ucmd-&gt;cong_type_flags) {</div><div class='add'>+	case HNS_ROCE_CREATE_QP_FLAGS_DCQCN:</div><div class='add'>+		hr_qp-&gt;cong_type = CONG_TYPE_DCQCN;</div><div class='add'>+		break;</div><div class='add'>+	case HNS_ROCE_CREATE_QP_FLAGS_LDCP:</div><div class='add'>+		hr_qp-&gt;cong_type = CONG_TYPE_LDCP;</div><div class='add'>+		break;</div><div class='add'>+	case HNS_ROCE_CREATE_QP_FLAGS_HC3:</div><div class='add'>+		hr_qp-&gt;cong_type = CONG_TYPE_HC3;</div><div class='add'>+		break;</div><div class='add'>+	case HNS_ROCE_CREATE_QP_FLAGS_DIP:</div><div class='add'>+		hr_qp-&gt;cong_type = CONG_TYPE_DIP;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!test_bit(hr_qp-&gt;cong_type, (unsigned long *)&amp;hr_dev-&gt;caps.cong_cap))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (hr_qp-&gt;ibqp.qp_type == IB_QPT_UD &amp;&amp;</div><div class='add'>+	    hr_qp-&gt;cong_type != CONG_TYPE_DCQCN)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int set_congest_param(struct hns_roce_dev *hr_dev,</div><div class='add'>+			     struct hns_roce_qp *hr_qp,</div><div class='add'>+			     struct hns_roce_ib_create_qp *ucmd)</div><div class='add'>+{</div><div class='add'>+	if (ucmd-&gt;comp_mask &amp; HNS_ROCE_CREATE_QP_MASK_CONGEST_TYPE)</div><div class='add'>+		return set_congest_type(hr_qp, ucmd);</div><div class='add'>+</div><div class='add'>+	default_congest_type(hr_dev, hr_qp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,</div><div class='add'>+			struct ib_qp_init_attr *init_attr,</div><div class='add'>+			struct ib_udata *udata,</div><div class='add'>+			struct hns_roce_ib_create_qp *ucmd)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	struct hns_roce_ucontext *uctx;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (init_attr-&gt;sq_sig_type == IB_SIGNAL_ALL_WR)</div><div class='add'>+		hr_qp-&gt;sq_signal_bits = IB_SIGNAL_ALL_WR;</div><div class='add'>+	else</div><div class='add'>+		hr_qp-&gt;sq_signal_bits = IB_SIGNAL_REQ_WR;</div><div class='add'>+</div><div class='add'>+	ret = set_rq_size(hr_dev, &amp;init_attr-&gt;cap, hr_qp,</div><div class='add'>+			  hns_roce_qp_has_rq(init_attr), !!udata);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to set user RQ size, ret = %d.\n",</div><div class='add'>+			  ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		ret = ib_copy_from_udata(ucmd, udata,</div><div class='add'>+					 min(udata-&gt;inlen, sizeof(*ucmd)));</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			dev_err(dev, "hns_roce_reserve_range_qp alloc qpn error\n");</div><div class='del'>-			goto err_wrid;</div><div class='add'>+			ibdev_err(ibdev,</div><div class='add'>+				  "failed to copy QP ucmd, ret = %d\n", ret);</div><div class='add'>+			return ret;</div><div class='ctx'> 		}</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;qp_type == IB_QPT_GSI &amp;&amp;</div><div class='del'>-	    hr_dev-&gt;hw_rev == HNS_ROCE_HW_VER1) {</div><div class='del'>-		/* In v1 engine, GSI QP context in RoCE engine's register */</div><div class='del'>-		ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp);</div><div class='add'>+		uctx = rdma_udata_to_drv_context(udata, struct hns_roce_ucontext,</div><div class='add'>+						 ibucontext);</div><div class='add'>+		hr_qp-&gt;config = uctx-&gt;config;</div><div class='add'>+		ret = set_user_sq_size(hr_dev, &amp;init_attr-&gt;cap, hr_qp, ucmd);</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			dev_err(dev, "hns_roce_qp_alloc failed!\n");</div><div class='del'>-			goto err_qpn;</div><div class='add'>+			ibdev_err(ibdev,</div><div class='add'>+				  "failed to set user SQ size, ret = %d.\n",</div><div class='add'>+				  ret);</div><div class='add'>+			return ret;</div><div class='ctx'> 		}</div><div class='add'>+</div><div class='add'>+		ret = set_congest_param(hr_dev, hr_qp, ucmd);</div><div class='ctx'> 	} else {</div><div class='del'>-		ret = hns_roce_qp_alloc(hr_dev, qpn, hr_qp);</div><div class='add'>+		if (hr_dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09)</div><div class='add'>+			hr_qp-&gt;config = HNS_ROCE_EXSGE_FLAGS;</div><div class='add'>+		default_congest_type(hr_dev, hr_qp);</div><div class='add'>+		ret = set_kernel_sq_size(hr_dev, &amp;init_attr-&gt;cap, hr_qp);</div><div class='add'>+		if (ret)</div><div class='add'>+			ibdev_err(ibdev,</div><div class='add'>+				  "failed to set kernel SQ size, ret = %d.\n",</div><div class='add'>+				  ret);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,</div><div class='add'>+				     struct ib_qp_init_attr *init_attr,</div><div class='add'>+				     struct ib_udata *udata,</div><div class='add'>+				     struct hns_roce_qp *hr_qp)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_work *flush_work = &amp;hr_qp-&gt;flush_work;</div><div class='add'>+	struct hns_roce_ib_create_qp_resp resp = {};</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	struct hns_roce_ib_create_qp ucmd = {};</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	mutex_init(&amp;hr_qp-&gt;mutex);</div><div class='add'>+	spin_lock_init(&amp;hr_qp-&gt;sq.lock);</div><div class='add'>+	spin_lock_init(&amp;hr_qp-&gt;rq.lock);</div><div class='add'>+	spin_lock_init(&amp;hr_qp-&gt;flush_lock);</div><div class='add'>+</div><div class='add'>+	hr_qp-&gt;state = IB_QPS_RESET;</div><div class='add'>+	hr_qp-&gt;flush_flag = 0;</div><div class='add'>+	flush_work-&gt;hr_dev = hr_dev;</div><div class='add'>+	INIT_WORK(&amp;flush_work-&gt;work, flush_work_handle);</div><div class='add'>+</div><div class='add'>+	if (init_attr-&gt;create_flags)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	ret = set_qp_param(hr_dev, hr_qp, init_attr, udata, &amp;ucmd);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to set QP param, ret = %d.\n", ret);</div><div class='add'>+		goto err_out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!udata) {</div><div class='add'>+		ret = alloc_kernel_wrid(hr_dev, hr_qp);</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			dev_err(dev, "hns_roce_qp_alloc failed!\n");</div><div class='del'>-			goto err_qpn;</div><div class='add'>+			ibdev_err(ibdev, "failed to alloc wrid, ret = %d.\n",</div><div class='add'>+				  ret);</div><div class='add'>+			goto err_out;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (sqpn)</div><div class='del'>-		hr_qp-&gt;doorbell_qpn = 1;</div><div class='del'>-	else</div><div class='del'>-		hr_qp-&gt;doorbell_qpn = cpu_to_le64(hr_qp-&gt;qpn);</div><div class='add'>+	ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to alloc QP buffer, ret = %d.\n", ret);</div><div class='add'>+		goto err_buf;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = alloc_qpn(hr_dev, hr_qp, init_attr);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to alloc QPN, ret = %d.\n", ret);</div><div class='add'>+		goto err_qpn;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &amp;ucmd, &amp;resp);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n",</div><div class='add'>+			  ret);</div><div class='add'>+		goto err_db;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = alloc_qpc(hr_dev, hr_qp);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to alloc QP context, ret = %d.\n",</div><div class='add'>+			  ret);</div><div class='add'>+		goto err_qpc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_qp_store(hr_dev, hr_qp, init_attr);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to store QP, ret = %d.\n", ret);</div><div class='add'>+		goto err_store;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	if (udata) {</div><div class='add'>+		resp.cap_flags = hr_qp-&gt;en_flags;</div><div class='ctx'> 		ret = ib_copy_to_udata(udata, &amp;resp,</div><div class='ctx'> 				       min(udata-&gt;outlen, sizeof(resp)));</div><div class='del'>-		if (ret)</div><div class='del'>-			goto err_qp;</div><div class='add'>+		if (ret) {</div><div class='add'>+			ibdev_err(ibdev, "copy qp resp failed!\n");</div><div class='add'>+			goto err_flow_ctrl;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {</div><div class='ctx'> 		ret = hr_dev-&gt;hw-&gt;qp_flow_control_init(hr_dev, hr_qp);</div><div class='ctx'> 		if (ret)</div><div class='del'>-			goto err_qp;</div><div class='add'>+			goto err_flow_ctrl;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	hr_qp-&gt;ibqp.qp_num = hr_qp-&gt;qpn;</div><div class='ctx'> 	hr_qp-&gt;event = hns_roce_ib_qp_event;</div><div class='add'>+	refcount_set(&amp;hr_qp-&gt;refcount, 1);</div><div class='add'>+	init_completion(&amp;hr_qp-&gt;free);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-err_qp:</div><div class='del'>-	if (init_attr-&gt;qp_type == IB_QPT_GSI &amp;&amp;</div><div class='del'>-		hr_dev-&gt;hw_rev == HNS_ROCE_HW_VER1)</div><div class='del'>-		hns_roce_qp_remove(hr_dev, hr_qp);</div><div class='del'>-	else</div><div class='del'>-		hns_roce_qp_free(hr_dev, hr_qp);</div><div class='del'>-</div><div class='add'>+err_flow_ctrl:</div><div class='add'>+	hns_roce_qp_remove(hr_dev, hr_qp);</div><div class='add'>+err_store:</div><div class='add'>+	free_qpc(hr_dev, hr_qp);</div><div class='add'>+err_qpc:</div><div class='add'>+	free_qp_db(hr_dev, hr_qp, udata);</div><div class='add'>+err_db:</div><div class='add'>+	free_qpn(hr_dev, hr_qp);</div><div class='ctx'> err_qpn:</div><div class='del'>-	if (!sqpn)</div><div class='del'>-		hns_roce_release_range_qp(hr_dev, qpn, 1);</div><div class='add'>+	free_qp_buf(hr_dev, hr_qp);</div><div class='add'>+err_buf:</div><div class='add'>+	free_kernel_wrid(hr_qp);</div><div class='add'>+err_out:</div><div class='add'>+	mutex_destroy(&amp;hr_qp-&gt;mutex);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-err_wrid:</div><div class='del'>-	if (udata) {</div><div class='del'>-		if ((hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RECORD_DB) &amp;&amp;</div><div class='del'>-		    (udata-&gt;outlen &gt;= sizeof(resp)) &amp;&amp;</div><div class='del'>-		    hns_roce_qp_has_rq(init_attr))</div><div class='del'>-			hns_roce_db_unmap_user(uctx, &amp;hr_qp-&gt;rdb);</div><div class='del'>-	} else {</div><div class='del'>-		kfree(hr_qp-&gt;sq.wrid);</div><div class='del'>-		kfree(hr_qp-&gt;rq.wrid);</div><div class='add'>+void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,</div><div class='add'>+			 struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	if (refcount_dec_and_test(&amp;hr_qp-&gt;refcount))</div><div class='add'>+		complete(&amp;hr_qp-&gt;free);</div><div class='add'>+	wait_for_completion(&amp;hr_qp-&gt;free);</div><div class='add'>+</div><div class='add'>+	free_qpc(hr_dev, hr_qp);</div><div class='add'>+	free_qpn(hr_dev, hr_qp);</div><div class='add'>+	free_qp_buf(hr_dev, hr_qp);</div><div class='add'>+	free_kernel_wrid(hr_qp);</div><div class='add'>+	free_qp_db(hr_dev, hr_qp, udata);</div><div class='add'>+	mutex_destroy(&amp;hr_qp-&gt;mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type,</div><div class='add'>+			 bool is_user)</div><div class='add'>+{</div><div class='add'>+	switch (type) {</div><div class='add'>+	case IB_QPT_XRC_INI:</div><div class='add'>+	case IB_QPT_XRC_TGT:</div><div class='add'>+		if (!(hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_XRC))</div><div class='add'>+			goto out;</div><div class='add'>+		break;</div><div class='add'>+	case IB_QPT_UD:</div><div class='add'>+		if (hr_dev-&gt;pci_dev-&gt;revision == PCI_REVISION_ID_HIP08 &amp;&amp;</div><div class='add'>+		    is_user)</div><div class='add'>+			goto out;</div><div class='add'>+		break;</div><div class='add'>+	case IB_QPT_RC:</div><div class='add'>+	case IB_QPT_GSI:</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-err_sq_dbmap:</div><div class='del'>-	if (udata)</div><div class='del'>-		if ((hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) &amp;&amp;</div><div class='del'>-		    (udata-&gt;inlen &gt;= sizeof(ucmd)) &amp;&amp;</div><div class='del'>-		    (udata-&gt;outlen &gt;= sizeof(resp)) &amp;&amp;</div><div class='del'>-		    hns_roce_qp_has_sq(init_attr))</div><div class='del'>-			hns_roce_db_unmap_user(uctx, &amp;hr_qp-&gt;sdb);</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-err_mtt:</div><div class='del'>-	hns_roce_mtt_cleanup(hr_dev, &amp;hr_qp-&gt;mtt);</div><div class='add'>+out:</div><div class='add'>+	ibdev_err(&amp;hr_dev-&gt;ib_dev, "not support QP type %d\n", type);</div><div class='ctx'> </div><div class='del'>-err_buf:</div><div class='del'>-	if (hr_qp-&gt;umem)</div><div class='del'>-		ib_umem_release(hr_qp-&gt;umem);</div><div class='del'>-	else</div><div class='del'>-		hns_roce_buf_free(hr_dev, hr_qp-&gt;buff_size, &amp;hr_qp-&gt;hr_buf);</div><div class='add'>+	return -EOPNOTSUPP;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-err_db:</div><div class='del'>-	if (!udata &amp;&amp; hns_roce_qp_has_rq(init_attr) &amp;&amp;</div><div class='del'>-	    (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RECORD_DB))</div><div class='del'>-		hns_roce_free_db(hr_dev, &amp;hr_qp-&gt;rdb);</div><div class='add'>+int hns_roce_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr,</div><div class='add'>+		       struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = qp-&gt;device;</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);</div><div class='add'>+	struct hns_roce_qp *hr_qp = to_hr_qp(qp);</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-err_rq_sge_list:</div><div class='del'>-	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RQ_INLINE)</div><div class='del'>-		kfree(hr_qp-&gt;rq_inl_buf.wqe_list[0].sg_list);</div><div class='add'>+	ret = check_qp_type(hr_dev, init_attr-&gt;qp_type, !!udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_out;</div><div class='ctx'> </div><div class='del'>-err_wqe_list:</div><div class='del'>-	if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_RQ_INLINE)</div><div class='del'>-		kfree(hr_qp-&gt;rq_inl_buf.wqe_list);</div><div class='add'>+	if (init_attr-&gt;qp_type == IB_QPT_XRC_TGT)</div><div class='add'>+		hr_qp-&gt;xrcdn = to_hr_xrcd(init_attr-&gt;xrcd)-&gt;xrcdn;</div><div class='add'>+</div><div class='add'>+	if (init_attr-&gt;qp_type == IB_QPT_GSI) {</div><div class='add'>+		hr_qp-&gt;port = init_attr-&gt;port_num - 1;</div><div class='add'>+		hr_qp-&gt;phy_port = hr_dev-&gt;iboe.phy_port[hr_qp-&gt;port];</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_create_qp_common(hr_dev, init_attr, udata, hr_qp);</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_err(ibdev, "create QP type %d failed(%d)\n",</div><div class='add'>+			  init_attr-&gt;qp_type, ret);</div><div class='ctx'> </div><div class='ctx'> err_out:</div><div class='add'>+	if (ret)</div><div class='add'>+		atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_QP_CREATE_ERR_CNT]);</div><div class='add'>+</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_qp *hns_roce_create_qp(struct ib_pd *pd,</div><div class='del'>-				 struct ib_qp_init_attr *init_attr,</div><div class='del'>-				 struct ib_udata *udata)</div><div class='add'>+int to_hr_qp_type(int qp_type)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(pd-&gt;device);</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='del'>-	struct hns_roce_sqp *hr_sqp;</div><div class='del'>-	struct hns_roce_qp *hr_qp;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	switch (init_attr-&gt;qp_type) {</div><div class='del'>-	case IB_QPT_RC: {</div><div class='del'>-		hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL);</div><div class='del'>-		if (!hr_qp)</div><div class='del'>-			return ERR_PTR(-ENOMEM);</div><div class='add'>+	switch (qp_type) {</div><div class='add'>+	case IB_QPT_RC:</div><div class='add'>+		return SERV_TYPE_RC;</div><div class='add'>+	case IB_QPT_UD:</div><div class='add'>+	case IB_QPT_GSI:</div><div class='add'>+		return SERV_TYPE_UD;</div><div class='add'>+	case IB_QPT_XRC_INI:</div><div class='add'>+	case IB_QPT_XRC_TGT:</div><div class='add'>+		return SERV_TYPE_XRC;</div><div class='add'>+	default:</div><div class='add'>+		return -1;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, 0,</div><div class='del'>-						hr_qp);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "Create RC QP failed\n");</div><div class='del'>-			kfree(hr_qp);</div><div class='del'>-			return ERR_PTR(ret);</div><div class='del'>-		}</div><div class='add'>+static int check_mtu_validate(struct hns_roce_dev *hr_dev,</div><div class='add'>+			      struct hns_roce_qp *hr_qp,</div><div class='add'>+			      struct ib_qp_attr *attr, int attr_mask)</div><div class='add'>+{</div><div class='add'>+	struct net_device *net_dev;</div><div class='add'>+	enum ib_mtu active_mtu;</div><div class='add'>+	int p;</div><div class='ctx'> </div><div class='del'>-		hr_qp-&gt;ibqp.qp_num = hr_qp-&gt;qpn;</div><div class='add'>+	p = attr_mask &amp; IB_QP_PORT ? (attr-&gt;port_num - 1) : hr_qp-&gt;port;</div><div class='add'>+	net_dev = get_hr_netdev(hr_dev, p);</div><div class='add'>+	active_mtu = iboe_get_mtu(net_dev-&gt;mtu);</div><div class='ctx'> </div><div class='del'>-		break;</div><div class='add'>+	if ((hr_dev-&gt;caps.max_mtu &gt;= IB_MTU_2048 &amp;&amp;</div><div class='add'>+	    attr-&gt;path_mtu &gt; hr_dev-&gt;caps.max_mtu) ||</div><div class='add'>+	    attr-&gt;path_mtu &lt; IB_MTU_256 || attr-&gt;path_mtu &gt; active_mtu) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+			"attr path_mtu(%d)invalid while modify qp",</div><div class='add'>+			attr-&gt;path_mtu);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	}</div><div class='del'>-	case IB_QPT_GSI: {</div><div class='del'>-		/* Userspace is not allowed to create special QPs: */</div><div class='del'>-		if (udata) {</div><div class='del'>-			dev_err(dev, "not support usr space GSI\n");</div><div class='del'>-			return ERR_PTR(-EINVAL);</div><div class='del'>-		}</div><div class='ctx'> </div><div class='del'>-		hr_sqp = kzalloc(sizeof(*hr_sqp), GFP_KERNEL);</div><div class='del'>-		if (!hr_sqp)</div><div class='del'>-			return ERR_PTR(-ENOMEM);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		hr_qp = &amp;hr_sqp-&gt;hr_qp;</div><div class='del'>-		hr_qp-&gt;port = init_attr-&gt;port_num - 1;</div><div class='del'>-		hr_qp-&gt;phy_port = hr_dev-&gt;iboe.phy_port[hr_qp-&gt;port];</div><div class='add'>+static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='add'>+				  int attr_mask)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='add'>+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='add'>+	int p;</div><div class='ctx'> </div><div class='del'>-		/* when hw version is v1, the sqpn is allocated */</div><div class='del'>-		if (hr_dev-&gt;caps.max_sq_sg &lt;= 2)</div><div class='del'>-			hr_qp-&gt;ibqp.qp_num = HNS_ROCE_MAX_PORTS +</div><div class='del'>-					     hr_dev-&gt;iboe.phy_port[hr_qp-&gt;port];</div><div class='del'>-		else</div><div class='del'>-			hr_qp-&gt;ibqp.qp_num = 1;</div><div class='add'>+	if ((attr_mask &amp; IB_QP_PORT) &amp;&amp;</div><div class='add'>+	    (attr-&gt;port_num == 0 || attr-&gt;port_num &gt; hr_dev-&gt;caps.num_ports)) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev, "invalid attr, port_num = %u.\n",</div><div class='add'>+			  attr-&gt;port_num);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata,</div><div class='del'>-						hr_qp-&gt;ibqp.qp_num, hr_qp);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(dev, "Create GSI QP failed!\n");</div><div class='del'>-			kfree(hr_sqp);</div><div class='del'>-			return ERR_PTR(ret);</div><div class='add'>+	if (attr_mask &amp; IB_QP_PKEY_INDEX) {</div><div class='add'>+		p = attr_mask &amp; IB_QP_PORT ? (attr-&gt;port_num - 1) : hr_qp-&gt;port;</div><div class='add'>+		if (attr-&gt;pkey_index &gt;= hr_dev-&gt;caps.pkey_table_len[p]) {</div><div class='add'>+			ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+				  "invalid attr, pkey_index = %u.\n",</div><div class='add'>+				  attr-&gt;pkey_index);</div><div class='add'>+			return -EINVAL;</div><div class='ctx'> 		}</div><div class='del'>-</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	default:{</div><div class='del'>-		dev_err(dev, "not support QP type %d\n", init_attr-&gt;qp_type);</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-	}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return &amp;hr_qp-&gt;ibqp;</div><div class='del'>-}</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_create_qp);</div><div class='add'>+	if (attr_mask &amp; IB_QP_MAX_QP_RD_ATOMIC &amp;&amp;</div><div class='add'>+	    attr-&gt;max_rd_atomic &gt; hr_dev-&gt;caps.max_qp_init_rdma) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+			  "invalid attr, max_rd_atomic = %u.\n",</div><div class='add'>+			  attr-&gt;max_rd_atomic);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-int to_hr_qp_type(int qp_type)</div><div class='del'>-{</div><div class='del'>-	int transport_type;</div><div class='add'>+	if (attr_mask &amp; IB_QP_MAX_DEST_RD_ATOMIC &amp;&amp;</div><div class='add'>+	    attr-&gt;max_dest_rd_atomic &gt; hr_dev-&gt;caps.max_qp_dest_rdma) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+			  "invalid attr, max_dest_rd_atomic = %u.\n",</div><div class='add'>+			  attr-&gt;max_dest_rd_atomic);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (qp_type == IB_QPT_RC)</div><div class='del'>-		transport_type = SERV_TYPE_RC;</div><div class='del'>-	else if (qp_type == IB_QPT_UC)</div><div class='del'>-		transport_type = SERV_TYPE_UC;</div><div class='del'>-	else if (qp_type == IB_QPT_UD)</div><div class='del'>-		transport_type = SERV_TYPE_UD;</div><div class='del'>-	else if (qp_type == IB_QPT_GSI)</div><div class='del'>-		transport_type = SERV_TYPE_UD;</div><div class='del'>-	else</div><div class='del'>-		transport_type = -1;</div><div class='add'>+	if (attr_mask &amp; IB_QP_PATH_MTU)</div><div class='add'>+		return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask);</div><div class='ctx'> </div><div class='del'>-	return transport_type;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(to_hr_qp_type);</div><div class='ctx'> </div><div class='ctx'> int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> 		       int attr_mask, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp-&gt;device);</div><div class='add'>+	struct hns_roce_ib_modify_qp_resp resp = {};</div><div class='ctx'> 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);</div><div class='ctx'> 	enum ib_qp_state cur_state, new_state;</div><div class='del'>-	struct device *dev = hr_dev-&gt;dev;</div><div class='ctx'> 	int ret = -EINVAL;</div><div class='del'>-	int p;</div><div class='del'>-	enum ib_mtu active_mtu;</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;hr_qp-&gt;mutex);</div><div class='ctx'> </div><div class='del'>-	cur_state = attr_mask &amp; IB_QP_CUR_STATE ?</div><div class='del'>-		    attr-&gt;cur_qp_state : (enum ib_qp_state)hr_qp-&gt;state;</div><div class='del'>-	new_state = attr_mask &amp; IB_QP_STATE ?</div><div class='del'>-		    attr-&gt;qp_state : cur_state;</div><div class='add'>+	if (attr_mask &amp; IB_QP_CUR_STATE &amp;&amp; attr-&gt;cur_qp_state != hr_qp-&gt;state)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	cur_state = hr_qp-&gt;state;</div><div class='add'>+	new_state = attr_mask &amp; IB_QP_STATE ? attr-&gt;qp_state : cur_state;</div><div class='ctx'> </div><div class='ctx'> 	if (ibqp-&gt;uobject &amp;&amp;</div><div class='ctx'> 	    (attr_mask &amp; IB_QP_STATE) &amp;&amp; new_state == IB_QPS_ERR) {</div><div class='del'>-		if (hr_qp-&gt;sdb_en == 1) {</div><div class='add'>+		if (hr_qp-&gt;en_flags &amp; HNS_ROCE_QP_CAP_SQ_RECORD_DB) {</div><div class='ctx'> 			hr_qp-&gt;sq.head = *(int *)(hr_qp-&gt;sdb.virt_addr);</div><div class='ctx'> </div><div class='del'>-			if (hr_qp-&gt;rdb_en == 1)</div><div class='add'>+			if (hr_qp-&gt;en_flags &amp; HNS_ROCE_QP_CAP_RQ_RECORD_DB)</div><div class='ctx'> 				hr_qp-&gt;rq.head = *(int *)(hr_qp-&gt;rdb.virt_addr);</div><div class='ctx'> 		} else {</div><div class='del'>-			dev_warn(dev, "flush cqe is not supported in userspace!\n");</div><div class='add'>+			ibdev_warn(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+				  "flush cqe is not supported in userspace!\n");</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp-&gt;qp_type,</div><div class='ctx'> 				attr_mask)) {</div><div class='del'>-		dev_err(dev, "ib_modify_qp_is_ok failed\n");</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev, "ib_modify_qp_is_ok failed\n");</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if ((attr_mask &amp; IB_QP_PORT) &amp;&amp;</div><div class='del'>-	    (attr-&gt;port_num == 0 || attr-&gt;port_num &gt; hr_dev-&gt;caps.num_ports)) {</div><div class='del'>-		dev_err(dev, "attr port_num invalid.attr-&gt;port_num=%d\n",</div><div class='del'>-			attr-&gt;port_num);</div><div class='add'>+	ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask);</div><div class='add'>+	if (ret)</div><div class='ctx'> 		goto out;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	if (attr_mask &amp; IB_QP_PKEY_INDEX) {</div><div class='del'>-		p = attr_mask &amp; IB_QP_PORT ? (attr-&gt;port_num - 1) : hr_qp-&gt;port;</div><div class='del'>-		if (attr-&gt;pkey_index &gt;= hr_dev-&gt;caps.pkey_table_len[p]) {</div><div class='del'>-			dev_err(dev, "attr pkey_index invalid.attr-&gt;pkey_index=%d\n",</div><div class='del'>-				attr-&gt;pkey_index);</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (attr_mask &amp; IB_QP_PATH_MTU) {</div><div class='del'>-		p = attr_mask &amp; IB_QP_PORT ? (attr-&gt;port_num - 1) : hr_qp-&gt;port;</div><div class='del'>-		active_mtu = iboe_get_mtu(hr_dev-&gt;iboe.netdevs[p]-&gt;mtu);</div><div class='del'>-</div><div class='del'>-		if ((hr_dev-&gt;caps.max_mtu == IB_MTU_4096 &amp;&amp;</div><div class='del'>-		    attr-&gt;path_mtu &gt; IB_MTU_4096) ||</div><div class='del'>-		    (hr_dev-&gt;caps.max_mtu == IB_MTU_2048 &amp;&amp;</div><div class='del'>-		    attr-&gt;path_mtu &gt; IB_MTU_2048) ||</div><div class='del'>-		    attr-&gt;path_mtu &lt; IB_MTU_256 ||</div><div class='del'>-		    attr-&gt;path_mtu &gt; active_mtu) {</div><div class='del'>-			dev_err(dev, "attr path_mtu(%d)invalid while modify qp",</div><div class='del'>-				attr-&gt;path_mtu);</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (attr_mask &amp; IB_QP_MAX_QP_RD_ATOMIC &amp;&amp;</div><div class='del'>-	    attr-&gt;max_rd_atomic &gt; hr_dev-&gt;caps.max_qp_init_rdma) {</div><div class='del'>-		dev_err(dev, "attr max_rd_atomic invalid.attr-&gt;max_rd_atomic=%d\n",</div><div class='del'>-			attr-&gt;max_rd_atomic);</div><div class='add'>+	if (cur_state == new_state &amp;&amp; cur_state == IB_QPS_RESET)</div><div class='ctx'> 		goto out;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	if (attr_mask &amp; IB_QP_MAX_DEST_RD_ATOMIC &amp;&amp;</div><div class='del'>-	    attr-&gt;max_dest_rd_atomic &gt; hr_dev-&gt;caps.max_qp_dest_rdma) {</div><div class='del'>-		dev_err(dev, "attr max_dest_rd_atomic invalid.attr-&gt;max_dest_rd_atomic=%d\n",</div><div class='del'>-			attr-&gt;max_dest_rd_atomic);</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;modify_qp(ibqp, attr, attr_mask, cur_state,</div><div class='add'>+				    new_state, udata);</div><div class='add'>+	if (ret)</div><div class='ctx'> 		goto out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (cur_state == new_state &amp;&amp; cur_state == IB_QPS_RESET) {</div><div class='del'>-		if (hr_dev-&gt;caps.min_wqes) {</div><div class='del'>-			ret = -EPERM;</div><div class='del'>-			dev_err(dev, "cur_state=%d new_state=%d\n", cur_state,</div><div class='del'>-				new_state);</div><div class='del'>-		} else {</div><div class='del'>-			ret = 0;</div><div class='del'>-		}</div><div class='ctx'> </div><div class='del'>-		goto out;</div><div class='add'>+	if (udata &amp;&amp; udata-&gt;outlen) {</div><div class='add'>+		resp.tc_mode = hr_qp-&gt;tc_mode;</div><div class='add'>+		resp.priority = hr_qp-&gt;sl;</div><div class='add'>+		ret = ib_copy_to_udata(udata, &amp;resp,</div><div class='add'>+				       min(udata-&gt;outlen, sizeof(resp)));</div><div class='add'>+		if (ret)</div><div class='add'>+			ibdev_err_ratelimited(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+					      "failed to copy modify qp resp.\n");</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hr_dev-&gt;hw-&gt;modify_qp(ibqp, attr, attr_mask, cur_state,</div><div class='del'>-				    new_state);</div><div class='del'>-</div><div class='ctx'> out:</div><div class='ctx'> 	mutex_unlock(&amp;hr_qp-&gt;mutex);</div><div class='add'>+	if (ret)</div><div class='add'>+		atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_QP_MODIFY_ERR_CNT]);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='hunk'>@@ -1051,69 +1483,80 @@ out:</div><div class='ctx'> void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq)</div><div class='ctx'> 		       __acquires(&amp;send_cq-&gt;lock) __acquires(&amp;recv_cq-&gt;lock)</div><div class='ctx'> {</div><div class='del'>-	if (send_cq == recv_cq) {</div><div class='del'>-		spin_lock_irq(&amp;send_cq-&gt;lock);</div><div class='add'>+	if (unlikely(send_cq == NULL &amp;&amp; recv_cq == NULL)) {</div><div class='add'>+		__acquire(&amp;send_cq-&gt;lock);</div><div class='add'>+		__acquire(&amp;recv_cq-&gt;lock);</div><div class='add'>+	} else if (unlikely(send_cq != NULL &amp;&amp; recv_cq == NULL)) {</div><div class='add'>+		spin_lock(&amp;send_cq-&gt;lock);</div><div class='add'>+		__acquire(&amp;recv_cq-&gt;lock);</div><div class='add'>+	} else if (unlikely(send_cq == NULL &amp;&amp; recv_cq != NULL)) {</div><div class='add'>+		spin_lock(&amp;recv_cq-&gt;lock);</div><div class='add'>+		__acquire(&amp;send_cq-&gt;lock);</div><div class='add'>+	} else if (send_cq == recv_cq) {</div><div class='add'>+		spin_lock(&amp;send_cq-&gt;lock);</div><div class='ctx'> 		__acquire(&amp;recv_cq-&gt;lock);</div><div class='ctx'> 	} else if (send_cq-&gt;cqn &lt; recv_cq-&gt;cqn) {</div><div class='del'>-		spin_lock_irq(&amp;send_cq-&gt;lock);</div><div class='add'>+		spin_lock(&amp;send_cq-&gt;lock);</div><div class='ctx'> 		spin_lock_nested(&amp;recv_cq-&gt;lock, SINGLE_DEPTH_NESTING);</div><div class='ctx'> 	} else {</div><div class='del'>-		spin_lock_irq(&amp;recv_cq-&gt;lock);</div><div class='add'>+		spin_lock(&amp;recv_cq-&gt;lock);</div><div class='ctx'> 		spin_lock_nested(&amp;send_cq-&gt;lock, SINGLE_DEPTH_NESTING);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_lock_cqs);</div><div class='ctx'> </div><div class='ctx'> void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,</div><div class='ctx'> 			 struct hns_roce_cq *recv_cq) __releases(&amp;send_cq-&gt;lock)</div><div class='ctx'> 			 __releases(&amp;recv_cq-&gt;lock)</div><div class='ctx'> {</div><div class='del'>-	if (send_cq == recv_cq) {</div><div class='add'>+	if (unlikely(send_cq == NULL &amp;&amp; recv_cq == NULL)) {</div><div class='add'>+		__release(&amp;recv_cq-&gt;lock);</div><div class='add'>+		__release(&amp;send_cq-&gt;lock);</div><div class='add'>+	} else if (unlikely(send_cq != NULL &amp;&amp; recv_cq == NULL)) {</div><div class='add'>+		__release(&amp;recv_cq-&gt;lock);</div><div class='add'>+		spin_unlock(&amp;send_cq-&gt;lock);</div><div class='add'>+	} else if (unlikely(send_cq == NULL &amp;&amp; recv_cq != NULL)) {</div><div class='add'>+		__release(&amp;send_cq-&gt;lock);</div><div class='add'>+		spin_unlock(&amp;recv_cq-&gt;lock);</div><div class='add'>+	} else if (send_cq == recv_cq) {</div><div class='ctx'> 		__release(&amp;recv_cq-&gt;lock);</div><div class='del'>-		spin_unlock_irq(&amp;send_cq-&gt;lock);</div><div class='add'>+		spin_unlock(&amp;send_cq-&gt;lock);</div><div class='ctx'> 	} else if (send_cq-&gt;cqn &lt; recv_cq-&gt;cqn) {</div><div class='ctx'> 		spin_unlock(&amp;recv_cq-&gt;lock);</div><div class='del'>-		spin_unlock_irq(&amp;send_cq-&gt;lock);</div><div class='add'>+		spin_unlock(&amp;send_cq-&gt;lock);</div><div class='ctx'> 	} else {</div><div class='ctx'> 		spin_unlock(&amp;send_cq-&gt;lock);</div><div class='del'>-		spin_unlock_irq(&amp;recv_cq-&gt;lock);</div><div class='add'>+		spin_unlock(&amp;recv_cq-&gt;lock);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_unlock_cqs);</div><div class='ctx'> </div><div class='del'>-static void *get_wqe(struct hns_roce_qp *hr_qp, int offset)</div><div class='add'>+static inline void *get_wqe(struct hns_roce_qp *hr_qp, u32 offset)</div><div class='ctx'> {</div><div class='del'>-</div><div class='del'>-	return hns_roce_buf_offset(&amp;hr_qp-&gt;hr_buf, offset);</div><div class='add'>+	return hns_roce_buf_offset(hr_qp-&gt;mtr.kmem, offset);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n)</div><div class='add'>+void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n)</div><div class='ctx'> {</div><div class='ctx'> 	return get_wqe(hr_qp, hr_qp-&gt;rq.offset + (n &lt;&lt; hr_qp-&gt;rq.wqe_shift));</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(get_recv_wqe);</div><div class='ctx'> </div><div class='del'>-void *get_send_wqe(struct hns_roce_qp *hr_qp, int n)</div><div class='add'>+void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, unsigned int n)</div><div class='ctx'> {</div><div class='ctx'> 	return get_wqe(hr_qp, hr_qp-&gt;sq.offset + (n &lt;&lt; hr_qp-&gt;sq.wqe_shift));</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(get_send_wqe);</div><div class='ctx'> </div><div class='del'>-void *get_send_extend_sge(struct hns_roce_qp *hr_qp, int n)</div><div class='add'>+void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, unsigned int n)</div><div class='ctx'> {</div><div class='del'>-	return hns_roce_buf_offset(&amp;hr_qp-&gt;hr_buf, hr_qp-&gt;sge.offset +</div><div class='del'>-					(n &lt;&lt; hr_qp-&gt;sge.sge_shift));</div><div class='add'>+	return get_wqe(hr_qp, hr_qp-&gt;sge.offset + (n &lt;&lt; hr_qp-&gt;sge.sge_shift));</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(get_send_extend_sge);</div><div class='ctx'> </div><div class='del'>-bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,</div><div class='add'>+bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, u32 nreq,</div><div class='ctx'> 			  struct ib_cq *ib_cq)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_cq *hr_cq;</div><div class='ctx'> 	u32 cur;</div><div class='ctx'> </div><div class='ctx'> 	cur = hr_wq-&gt;head - hr_wq-&gt;tail;</div><div class='del'>-	if (likely(cur + nreq &lt; hr_wq-&gt;max_post))</div><div class='add'>+	if (likely(cur + nreq &lt; hr_wq-&gt;wqe_cnt))</div><div class='ctx'> 		return false;</div><div class='ctx'> </div><div class='ctx'> 	hr_cq = to_hr_cq(ib_cq);</div><div class='hunk'>@@ -1121,33 +1564,32 @@ bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,</div><div class='ctx'> 	cur = hr_wq-&gt;head - hr_wq-&gt;tail;</div><div class='ctx'> 	spin_unlock(&amp;hr_cq-&gt;lock);</div><div class='ctx'> </div><div class='del'>-	return cur + nreq &gt;= hr_wq-&gt;max_post;</div><div class='add'>+	return cur + nreq &gt;= hr_wq-&gt;wqe_cnt;</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_wq_overflow);</div><div class='ctx'> </div><div class='ctx'> int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_qp_table *qp_table = &amp;hr_dev-&gt;qp_table;</div><div class='del'>-	int reserved_from_top = 0;</div><div class='del'>-	int reserved_from_bot;</div><div class='del'>-	int ret;</div><div class='add'>+	unsigned int reserved_from_bot;</div><div class='add'>+	unsigned int i;</div><div class='ctx'> </div><div class='ctx'> 	mutex_init(&amp;qp_table-&gt;scc_mutex);</div><div class='add'>+	mutex_init(&amp;qp_table-&gt;bank_mutex);</div><div class='ctx'> 	xa_init(&amp;hr_dev-&gt;qp_table_xa);</div><div class='add'>+	xa_init(&amp;qp_table-&gt;dip_xa);</div><div class='ctx'> </div><div class='del'>-	/* In hw v1, a port include two SQP, six ports total 12 */</div><div class='del'>-	if (hr_dev-&gt;caps.max_sq_sg &lt;= 2)</div><div class='del'>-		reserved_from_bot = SQP_NUM;</div><div class='del'>-	else</div><div class='del'>-		reserved_from_bot = hr_dev-&gt;caps.reserved_qps;</div><div class='add'>+	reserved_from_bot = hr_dev-&gt;caps.reserved_qps;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_bitmap_init(&amp;qp_table-&gt;bitmap, hr_dev-&gt;caps.num_qps,</div><div class='del'>-				   hr_dev-&gt;caps.num_qps - 1, reserved_from_bot,</div><div class='del'>-				   reserved_from_top);</div><div class='del'>-	if (ret) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "qp bitmap init failed!error=%d\n",</div><div class='del'>-			ret);</div><div class='del'>-		return ret;</div><div class='add'>+	for (i = 0; i &lt; reserved_from_bot; i++) {</div><div class='add'>+		hr_dev-&gt;qp_table.bank[get_qp_bankid(i)].inuse++;</div><div class='add'>+		hr_dev-&gt;qp_table.bank[get_qp_bankid(i)].min++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; HNS_ROCE_QP_BANK_NUM; i++) {</div><div class='add'>+		ida_init(&amp;hr_dev-&gt;qp_table.bank[i].ida);</div><div class='add'>+		hr_dev-&gt;qp_table.bank[i].max = hr_dev-&gt;caps.num_qps /</div><div class='add'>+					       HNS_ROCE_QP_BANK_NUM - 1;</div><div class='add'>+		hr_dev-&gt;qp_table.bank[i].next = hr_dev-&gt;qp_table.bank[i].min;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='hunk'>@@ -1155,5 +1597,12 @@ int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev)</div><div class='ctx'> </div><div class='ctx'> void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='del'>-	hns_roce_bitmap_cleanup(&amp;hr_dev-&gt;qp_table.bitmap);</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; HNS_ROCE_QP_BANK_NUM; i++)</div><div class='add'>+		ida_destroy(&amp;hr_dev-&gt;qp_table.bank[i].ida);</div><div class='add'>+	xa_destroy(&amp;hr_dev-&gt;qp_table.dip_xa);</div><div class='add'>+	xa_destroy(&amp;hr_dev-&gt;qp_table_xa);</div><div class='add'>+	mutex_destroy(&amp;hr_dev-&gt;qp_table.bank_mutex);</div><div class='add'>+	mutex_destroy(&amp;hr_dev-&gt;qp_table.scc_mutex);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c<br/>index 0a31d0a3d657..230187dda6a0 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_restrack.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_restrack.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_restrack.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_restrack.c</a></div><div class='hunk'>@@ -4,123 +4,231 @@</div><div class='ctx'> #include &lt;rdma/rdma_cm.h&gt;</div><div class='ctx'> #include &lt;rdma/restrack.h&gt;</div><div class='ctx'> #include &lt;uapi/rdma/rdma_netlink.h&gt;</div><div class='del'>-#include "hnae3.h"</div><div class='ctx'> #include "hns_roce_common.h"</div><div class='ctx'> #include "hns_roce_device.h"</div><div class='ctx'> #include "hns_roce_hw_v2.h"</div><div class='ctx'> </div><div class='del'>-static int hns_roce_fill_cq(struct sk_buff *msg,</div><div class='del'>-			    struct hns_roce_v2_cq_context *context)</div><div class='add'>+int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq)</div><div class='ctx'> {</div><div class='del'>-	if (rdma_nl_put_driver_u32(msg, "state",</div><div class='del'>-				   roce_get_field(context-&gt;byte_4_pg_ceqn,</div><div class='del'>-						  V2_CQC_BYTE_4_ARM_ST_M,</div><div class='del'>-						  V2_CQC_BYTE_4_ARM_ST_S)))</div><div class='add'>+	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);</div><div class='add'>+	struct nlattr *table_attr;</div><div class='add'>+</div><div class='add'>+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);</div><div class='add'>+	if (!table_attr)</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "cq_depth", hr_cq-&gt;cq_depth))</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='del'>-	if (rdma_nl_put_driver_u32(msg, "ceqn",</div><div class='del'>-				   roce_get_field(context-&gt;byte_4_pg_ceqn,</div><div class='del'>-						  V2_CQC_BYTE_4_CEQN_M,</div><div class='del'>-						  V2_CQC_BYTE_4_CEQN_S)))</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "cons_index", hr_cq-&gt;cons_index))</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='del'>-	if (rdma_nl_put_driver_u32(msg, "cqn",</div><div class='del'>-				   roce_get_field(context-&gt;byte_8_cqn,</div><div class='del'>-						  V2_CQC_BYTE_8_CQN_M,</div><div class='del'>-						  V2_CQC_BYTE_8_CQN_S)))</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "cqe_size", hr_cq-&gt;cqe_size))</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='del'>-	if (rdma_nl_put_driver_u32(msg, "hopnum",</div><div class='del'>-				   roce_get_field(context-&gt;byte_16_hop_addr,</div><div class='del'>-						  V2_CQC_BYTE_16_CQE_HOP_NUM_M,</div><div class='del'>-						  V2_CQC_BYTE_16_CQE_HOP_NUM_S)))</div><div class='add'>+	if (rdma_nl_put_driver_u32(msg, "arm_sn", hr_cq-&gt;arm_sn))</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='del'>-	if (rdma_nl_put_driver_u32(</div><div class='del'>-		    msg, "pi",</div><div class='del'>-		    roce_get_field(context-&gt;byte_28_cq_pi,</div><div class='del'>-				   V2_CQC_BYTE_28_CQ_PRODUCER_IDX_M,</div><div class='del'>-				   V2_CQC_BYTE_28_CQ_PRODUCER_IDX_S)))</div><div class='add'>+	nla_nest_end(msg, table_attr);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	nla_nest_cancel(msg, table_attr);</div><div class='add'>+</div><div class='add'>+	return -EMSGSIZE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq-&gt;device);</div><div class='add'>+	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);</div><div class='add'>+	struct hns_roce_v2_cq_context context;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!hr_dev-&gt;hw-&gt;query_cqc)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;query_cqc(hr_dev, hr_cq-&gt;cqn, &amp;context);</div><div class='add'>+	if (ret)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ret = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, sizeof(context), &amp;context);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_qp *hr_qp = to_hr_qp(ib_qp);</div><div class='add'>+	struct nlattr *table_attr;</div><div class='add'>+</div><div class='add'>+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);</div><div class='add'>+	if (!table_attr)</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	if (rdma_nl_put_driver_u32_hex(msg, "sq_wqe_cnt", hr_qp-&gt;sq.wqe_cnt))</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='del'>-	if (rdma_nl_put_driver_u32(</div><div class='del'>-		    msg, "ci",</div><div class='del'>-		    roce_get_field(context-&gt;byte_32_cq_ci,</div><div class='del'>-				   V2_CQC_BYTE_32_CQ_CONSUMER_IDX_M,</div><div class='del'>-				   V2_CQC_BYTE_32_CQ_CONSUMER_IDX_S)))</div><div class='add'>+	if (rdma_nl_put_driver_u32_hex(msg, "sq_max_gs", hr_qp-&gt;sq.max_gs))</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='del'>-	if (rdma_nl_put_driver_u32(</div><div class='del'>-		    msg, "coalesce",</div><div class='del'>-		    roce_get_field(context-&gt;byte_56_cqe_period_maxcnt,</div><div class='del'>-				   V2_CQC_BYTE_56_CQ_MAX_CNT_M,</div><div class='del'>-				   V2_CQC_BYTE_56_CQ_MAX_CNT_S)))</div><div class='add'>+	if (rdma_nl_put_driver_u32_hex(msg, "rq_wqe_cnt", hr_qp-&gt;rq.wqe_cnt))</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='del'>-	if (rdma_nl_put_driver_u32(</div><div class='del'>-		    msg, "period",</div><div class='del'>-		    roce_get_field(context-&gt;byte_56_cqe_period_maxcnt,</div><div class='del'>-				   V2_CQC_BYTE_56_CQ_PERIOD_M,</div><div class='del'>-				   V2_CQC_BYTE_56_CQ_PERIOD_S)))</div><div class='add'>+	if (rdma_nl_put_driver_u32_hex(msg, "rq_max_gs", hr_qp-&gt;rq.max_gs))</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='del'>-	if (rdma_nl_put_driver_u32(msg, "cnt",</div><div class='del'>-				   roce_get_field(context-&gt;byte_52_cqe_cnt,</div><div class='del'>-						  V2_CQC_BYTE_52_CQE_CNT_M,</div><div class='del'>-						  V2_CQC_BYTE_52_CQE_CNT_S)))</div><div class='add'>+	if (rdma_nl_put_driver_u32_hex(msg, "ext_sge_sge_cnt", hr_qp-&gt;sge.sge_cnt))</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='add'>+	nla_nest_end(msg, table_attr);</div><div class='add'>+</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> err:</div><div class='add'>+	nla_nest_cancel(msg, table_attr);</div><div class='add'>+</div><div class='ctx'> 	return -EMSGSIZE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_fill_res_cq_entry(struct sk_buff *msg,</div><div class='del'>-				      struct rdma_restrack_entry *res)</div><div class='add'>+int hns_roce_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ib_qp)</div><div class='ctx'> {</div><div class='del'>-	struct ib_cq *ib_cq = container_of(res, struct ib_cq, res);</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq-&gt;device);</div><div class='del'>-	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);</div><div class='del'>-	struct hns_roce_v2_cq_context *context;</div><div class='del'>-	struct nlattr *table_attr;</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_qp-&gt;device);</div><div class='add'>+	struct hns_roce_qp *hr_qp = to_hr_qp(ib_qp);</div><div class='add'>+	struct hns_roce_full_qp_ctx {</div><div class='add'>+		struct hns_roce_v2_qp_context qpc;</div><div class='add'>+		struct hns_roce_v2_scc_context sccc;</div><div class='add'>+	} context = {};</div><div class='add'>+	u32 sccn = hr_qp-&gt;qpn;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	if (!hr_dev-&gt;dfx-&gt;query_cqc_info)</div><div class='add'>+	if (!hr_dev-&gt;hw-&gt;query_qpc)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	context = kzalloc(sizeof(struct hns_roce_v2_cq_context), GFP_KERNEL);</div><div class='del'>-	if (!context)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	ret = hr_dev-&gt;dfx-&gt;query_cqc_info(hr_dev, hr_cq-&gt;cqn, (int *)context);</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;query_qpc(hr_dev, hr_qp-&gt;qpn, &amp;context.qpc);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err;</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	/* If SCC is disabled or the query fails, the queried SCCC will</div><div class='add'>+	 * be all 0.</div><div class='add'>+	 */</div><div class='add'>+	if (!(hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) ||</div><div class='add'>+	    !hr_dev-&gt;hw-&gt;query_sccc)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	if (hr_qp-&gt;cong_type == CONG_TYPE_DIP) {</div><div class='add'>+		if (!hr_qp-&gt;dip)</div><div class='add'>+			goto out;</div><div class='add'>+		sccn = hr_qp-&gt;dip-&gt;dip_idx;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;query_sccc(hr_dev, sccn, &amp;context.sccc);</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_warn_ratelimited(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+				       "failed to query SCCC, ret = %d.\n",</div><div class='add'>+				       ret);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	ret = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, sizeof(context), &amp;context);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hns_roce_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_mr *hr_mr = to_hr_mr(ib_mr);</div><div class='add'>+	struct nlattr *table_attr;</div><div class='ctx'> </div><div class='ctx'> 	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);</div><div class='ctx'> 	if (!table_attr)</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	if (rdma_nl_put_driver_u32_hex(msg, "pbl_hop_num", hr_mr-&gt;pbl_hop_num))</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='del'>-	if (hns_roce_fill_cq(msg, context))</div><div class='del'>-		goto err_cancel_table;</div><div class='add'>+	if (rdma_nl_put_driver_u32_hex(msg, "ba_pg_shift",</div><div class='add'>+				       hr_mr-&gt;pbl_mtr.hem_cfg.ba_pg_shift))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	if (rdma_nl_put_driver_u32_hex(msg, "buf_pg_shift",</div><div class='add'>+				       hr_mr-&gt;pbl_mtr.hem_cfg.buf_pg_shift))</div><div class='add'>+		goto err;</div><div class='ctx'> </div><div class='ctx'> 	nla_nest_end(msg, table_attr);</div><div class='del'>-	kfree(context);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-err_cancel_table:</div><div class='del'>-	nla_nest_cancel(msg, table_attr);</div><div class='ctx'> err:</div><div class='del'>-	kfree(context);</div><div class='add'>+	nla_nest_cancel(msg, table_attr);</div><div class='add'>+</div><div class='ctx'> 	return -EMSGSIZE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int hns_roce_fill_res_entry(struct sk_buff *msg,</div><div class='del'>-			    struct rdma_restrack_entry *res)</div><div class='add'>+int hns_roce_fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ib_mr)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_mr-&gt;device);</div><div class='add'>+	struct hns_roce_mr *hr_mr = to_hr_mr(ib_mr);</div><div class='add'>+	struct hns_roce_v2_mpt_entry context;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!hr_dev-&gt;hw-&gt;query_mpt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;query_mpt(hr_dev, hr_mr-&gt;key, &amp;context);</div><div class='add'>+	if (ret)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ret = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, sizeof(context), &amp;context);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hns_roce_fill_res_srq_entry(struct sk_buff *msg, struct ib_srq *ib_srq)</div><div class='ctx'> {</div><div class='del'>-	if (res-&gt;type == RDMA_RESTRACK_CQ)</div><div class='del'>-		return hns_roce_fill_res_cq_entry(msg, res);</div><div class='add'>+	struct hns_roce_srq *hr_srq = to_hr_srq(ib_srq);</div><div class='add'>+	struct nlattr *table_attr;</div><div class='add'>+</div><div class='add'>+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);</div><div class='add'>+	if (!table_attr)</div><div class='add'>+		return -EMSGSIZE;</div><div class='add'>+</div><div class='add'>+	if (rdma_nl_put_driver_u32_hex(msg, "srqn", hr_srq-&gt;srqn))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	if (rdma_nl_put_driver_u32_hex(msg, "wqe_cnt", hr_srq-&gt;wqe_cnt))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	if (rdma_nl_put_driver_u32_hex(msg, "max_gs", hr_srq-&gt;max_gs))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	if (rdma_nl_put_driver_u32_hex(msg, "xrcdn", hr_srq-&gt;xrcdn))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	nla_nest_end(msg, table_attr);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	nla_nest_cancel(msg, table_attr);</div><div class='add'>+	return -EMSGSIZE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int hns_roce_fill_res_srq_entry_raw(struct sk_buff *msg, struct ib_srq *ib_srq)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_srq-&gt;device);</div><div class='add'>+	struct hns_roce_srq *hr_srq = to_hr_srq(ib_srq);</div><div class='add'>+	struct hns_roce_srq_context context;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!hr_dev-&gt;hw-&gt;query_srqc)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;query_srqc(hr_dev, hr_srq-&gt;srqn, &amp;context);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, sizeof(context), &amp;context);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c<br/>index b3421b1f21e0..8a6efb6b9c9e 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_srq.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/hns/hns_roce_srq.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_srq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_srq.c</a></div><div class='hunk'>@@ -4,7 +4,7 @@</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;rdma/ib_umem.h&gt;</div><div class='del'>-#include &lt;rdma/hns-abi.h&gt;</div><div class='add'>+#include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='ctx'> #include "hns_roce_device.h"</div><div class='ctx'> #include "hns_roce_cmd.h"</div><div class='ctx'> #include "hns_roce_hem.h"</div><div class='hunk'>@@ -17,7 +17,7 @@ void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type)</div><div class='ctx'> 	xa_lock(&amp;srq_table-&gt;xa);</div><div class='ctx'> 	srq = xa_load(&amp;srq_table-&gt;xa, srqn &amp; (hr_dev-&gt;caps.num_srqs - 1));</div><div class='ctx'> 	if (srq)</div><div class='del'>-		atomic_inc(&amp;srq-&gt;refcount);</div><div class='add'>+		refcount_inc(&amp;srq-&gt;refcount);</div><div class='ctx'> 	xa_unlock(&amp;srq_table-&gt;xa);</div><div class='ctx'> </div><div class='ctx'> 	if (!srq) {</div><div class='hunk'>@@ -27,10 +27,9 @@ void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type)</div><div class='ctx'> </div><div class='ctx'> 	srq-&gt;event(srq, event_type);</div><div class='ctx'> </div><div class='del'>-	if (atomic_dec_and_test(&amp;srq-&gt;refcount))</div><div class='add'>+	if (refcount_dec_and_test(&amp;srq-&gt;refcount))</div><div class='ctx'> 		complete(&amp;srq-&gt;free);</div><div class='ctx'> }</div><div class='del'>-EXPORT_SYMBOL_GPL(hns_roce_srq_event);</div><div class='ctx'> </div><div class='ctx'> static void hns_roce_ib_srq_event(struct hns_roce_srq *srq,</div><div class='ctx'> 				  enum hns_roce_event event_type)</div><div class='hunk'>@@ -51,7 +50,7 @@ static void hns_roce_ib_srq_event(struct hns_roce_srq *srq,</div><div class='ctx'> 			break;</div><div class='ctx'> 		default:</div><div class='ctx'> 			dev_err(hr_dev-&gt;dev,</div><div class='del'>-			   "hns_roce:Unexpected event type 0x%x on SRQ %06lx\n",</div><div class='add'>+			   "hns_roce:Unexpected event type %d on SRQ %06lx\n",</div><div class='ctx'> 			   event_type, srq-&gt;srqn);</div><div class='ctx'> 			return;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -60,388 +59,491 @@ static void hns_roce_ib_srq_event(struct hns_roce_srq *srq,</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_sw2hw_srq(struct hns_roce_dev *dev,</div><div class='del'>-			      struct hns_roce_cmd_mailbox *mailbox,</div><div class='del'>-			      unsigned long srq_num)</div><div class='add'>+static int alloc_srqn(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)</div><div class='ctx'> {</div><div class='del'>-	return hns_roce_cmd_mbox(dev, mailbox-&gt;dma, 0, srq_num, 0,</div><div class='del'>-				 HNS_ROCE_CMD_SW2HW_SRQ,</div><div class='del'>-				 HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='add'>+	struct hns_roce_ida *srq_ida = &amp;hr_dev-&gt;srq_table.srq_ida;</div><div class='add'>+	int id;</div><div class='add'>+</div><div class='add'>+	id = ida_alloc_range(&amp;srq_ida-&gt;ida, srq_ida-&gt;min, srq_ida-&gt;max,</div><div class='add'>+			     GFP_KERNEL);</div><div class='add'>+	if (id &lt; 0) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev, "failed to alloc srq(%d).\n", id);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	srq-&gt;srqn = id;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_hw2sw_srq(struct hns_roce_dev *dev,</div><div class='del'>-			     struct hns_roce_cmd_mailbox *mailbox,</div><div class='del'>-			     unsigned long srq_num)</div><div class='add'>+static void free_srqn(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)</div><div class='ctx'> {</div><div class='del'>-	return hns_roce_cmd_mbox(dev, 0, mailbox ? mailbox-&gt;dma : 0, srq_num,</div><div class='del'>-				 mailbox ? 0 : 1, HNS_ROCE_CMD_HW2SW_SRQ,</div><div class='del'>-				 HNS_ROCE_CMD_TIMEOUT_MSECS);</div><div class='add'>+	ida_free(&amp;hr_dev-&gt;srq_table.srq_ida.ida, (int)srq-&gt;srqn);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_srq_alloc(struct hns_roce_dev *hr_dev, u32 pdn, u32 cqn,</div><div class='del'>-			      u16 xrcd, struct hns_roce_mtt *hr_mtt,</div><div class='del'>-			      u64 db_rec_addr, struct hns_roce_srq *srq)</div><div class='add'>+static int hns_roce_create_srqc(struct hns_roce_dev *hr_dev,</div><div class='add'>+				struct hns_roce_srq *srq)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_srq_table *srq_table = &amp;hr_dev-&gt;srq_table;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='ctx'> 	struct hns_roce_cmd_mailbox *mailbox;</div><div class='del'>-	dma_addr_t dma_handle_wqe;</div><div class='del'>-	dma_addr_t dma_handle_idx;</div><div class='del'>-	u64 *mtts_wqe;</div><div class='del'>-	u64 *mtts_idx;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	/* Get the physical address of srq buf */</div><div class='del'>-	mtts_wqe = hns_roce_table_find(hr_dev,</div><div class='del'>-				       &amp;hr_dev-&gt;mr_table.mtt_srqwqe_table,</div><div class='del'>-				       srq-&gt;mtt.first_seg,</div><div class='del'>-				       &amp;dma_handle_wqe);</div><div class='del'>-	if (!mtts_wqe) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev,</div><div class='del'>-			"SRQ alloc.Failed to find srq buf addr.\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Get physical address of idx que buf */</div><div class='del'>-	mtts_idx = hns_roce_table_find(hr_dev, &amp;hr_dev-&gt;mr_table.mtt_idx_table,</div><div class='del'>-				       srq-&gt;idx_que.mtt.first_seg,</div><div class='del'>-				       &amp;dma_handle_idx);</div><div class='del'>-	if (!mtts_idx) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev,</div><div class='del'>-			"SRQ alloc.Failed to find idx que buf addr.\n");</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='add'>+	if (IS_ERR(mailbox)) {</div><div class='add'>+		ibdev_err(ibdev, "failed to alloc mailbox for SRQC.\n");</div><div class='add'>+		return PTR_ERR(mailbox);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_bitmap_alloc(&amp;srq_table-&gt;bitmap, &amp;srq-&gt;srqn);</div><div class='del'>-	if (ret == -1) {</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "SRQ alloc.Failed to alloc index.\n");</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+	ret = hr_dev-&gt;hw-&gt;write_srqc(srq, mailbox-&gt;buf);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to write SRQC.\n");</div><div class='add'>+		goto err_mbox;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_table_get(hr_dev, &amp;srq_table-&gt;table, srq-&gt;srqn);</div><div class='add'>+	ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_SRQ,</div><div class='add'>+				     srq-&gt;srqn);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		goto err_out;</div><div class='add'>+		ibdev_err(ibdev, "failed to config SRQC, ret = %d.\n", ret);</div><div class='ctx'> </div><div class='del'>-	ret = xa_err(xa_store(&amp;srq_table-&gt;xa, srq-&gt;srqn, srq, GFP_KERNEL));</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_put;</div><div class='add'>+err_mbox:</div><div class='add'>+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);</div><div class='del'>-	if (IS_ERR(mailbox)) {</div><div class='del'>-		ret = PTR_ERR(mailbox);</div><div class='del'>-		goto err_xa;</div><div class='add'>+static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_srq_table *srq_table = &amp;hr_dev-&gt;srq_table;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_table_get(hr_dev, &amp;srq_table-&gt;table, srq-&gt;srqn);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to get SRQC table, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	hr_dev-&gt;hw-&gt;write_srqc(hr_dev, srq, pdn, xrcd, cqn, mailbox-&gt;buf,</div><div class='del'>-			       mtts_wqe, mtts_idx, dma_handle_wqe,</div><div class='del'>-			       dma_handle_idx);</div><div class='add'>+	ret = xa_err(xa_store_irq(&amp;srq_table-&gt;xa, srq-&gt;srqn, srq, GFP_KERNEL));</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to store SRQC, ret = %d.\n", ret);</div><div class='add'>+		goto err_put;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_sw2hw_srq(hr_dev, mailbox, srq-&gt;srqn);</div><div class='del'>-	hns_roce_free_cmd_mailbox(hr_dev, mailbox);</div><div class='add'>+	ret = hns_roce_create_srqc(hr_dev, srq);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err_xa;</div><div class='ctx'> </div><div class='del'>-	atomic_set(&amp;srq-&gt;refcount, 1);</div><div class='del'>-	init_completion(&amp;srq-&gt;free);</div><div class='del'>-	return ret;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='ctx'> err_xa:</div><div class='del'>-	xa_erase(&amp;srq_table-&gt;xa, srq-&gt;srqn);</div><div class='del'>-</div><div class='add'>+	xa_erase_irq(&amp;srq_table-&gt;xa, srq-&gt;srqn);</div><div class='ctx'> err_put:</div><div class='ctx'> 	hns_roce_table_put(hr_dev, &amp;srq_table-&gt;table, srq-&gt;srqn);</div><div class='ctx'> </div><div class='del'>-err_out:</div><div class='del'>-	hns_roce_bitmap_free(&amp;srq_table-&gt;bitmap, srq-&gt;srqn, BITMAP_NO_RR);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void hns_roce_srq_free(struct hns_roce_dev *hr_dev,</div><div class='del'>-			      struct hns_roce_srq *srq)</div><div class='add'>+static void free_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_srq_table *srq_table = &amp;hr_dev-&gt;srq_table;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_hw2sw_srq(hr_dev, NULL, srq-&gt;srqn);</div><div class='add'>+	ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_SRQ,</div><div class='add'>+				      srq-&gt;srqn);</div><div class='ctx'> 	if (ret)</div><div class='del'>-		dev_err(hr_dev-&gt;dev, "HW2SW_SRQ failed (%d) for CQN %06lx\n",</div><div class='del'>-			ret, srq-&gt;srqn);</div><div class='add'>+		dev_err_ratelimited(hr_dev-&gt;dev, "DESTROY_SRQ failed (%d) for SRQN %06lx\n",</div><div class='add'>+				    ret, srq-&gt;srqn);</div><div class='ctx'> </div><div class='del'>-	xa_erase(&amp;srq_table-&gt;xa, srq-&gt;srqn);</div><div class='add'>+	xa_erase_irq(&amp;srq_table-&gt;xa, srq-&gt;srqn);</div><div class='ctx'> </div><div class='del'>-	if (atomic_dec_and_test(&amp;srq-&gt;refcount))</div><div class='add'>+	if (refcount_dec_and_test(&amp;srq-&gt;refcount))</div><div class='ctx'> 		complete(&amp;srq-&gt;free);</div><div class='ctx'> 	wait_for_completion(&amp;srq-&gt;free);</div><div class='ctx'> </div><div class='ctx'> 	hns_roce_table_put(hr_dev, &amp;srq_table-&gt;table, srq-&gt;srqn);</div><div class='del'>-	hns_roce_bitmap_free(&amp;srq_table-&gt;bitmap, srq-&gt;srqn, BITMAP_NO_RR);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int hns_roce_create_idx_que(struct ib_pd *pd, struct hns_roce_srq *srq,</div><div class='del'>-				   u32 page_shift)</div><div class='add'>+static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,</div><div class='add'>+			 struct ib_udata *udata, unsigned long addr)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(pd-&gt;device);</div><div class='ctx'> 	struct hns_roce_idx_que *idx_que = &amp;srq-&gt;idx_que;</div><div class='del'>-	u32 bitmap_num;</div><div class='del'>-	int i;</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	struct hns_roce_buf_attr buf_attr = {};</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	bitmap_num = HNS_ROCE_ALOGN_UP(srq-&gt;max, 8 * sizeof(u64));</div><div class='add'>+	srq-&gt;idx_que.entry_shift = ilog2(HNS_ROCE_IDX_QUE_ENTRY_SZ);</div><div class='add'>+</div><div class='add'>+	buf_attr.page_shift = hr_dev-&gt;caps.idx_buf_pg_sz + PAGE_SHIFT;</div><div class='add'>+	buf_attr.region[0].size = to_hr_hem_entries_size(srq-&gt;wqe_cnt,</div><div class='add'>+					srq-&gt;idx_que.entry_shift);</div><div class='add'>+	buf_attr.region[0].hopnum = hr_dev-&gt;caps.idx_hop_num;</div><div class='add'>+	buf_attr.region_count = 1;</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_mtr_create(hr_dev, &amp;idx_que-&gt;mtr, &amp;buf_attr,</div><div class='add'>+				  hr_dev-&gt;caps.idx_ba_pg_sz + PAGE_SHIFT,</div><div class='add'>+				  udata, addr);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev,</div><div class='add'>+			  "failed to alloc SRQ idx mtr, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	idx_que-&gt;bitmap = kcalloc(1, bitmap_num / 8, GFP_KERNEL);</div><div class='del'>-	if (!idx_que-&gt;bitmap)</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+	if (!udata) {</div><div class='add'>+		idx_que-&gt;bitmap = bitmap_zalloc(srq-&gt;wqe_cnt, GFP_KERNEL);</div><div class='add'>+		if (!idx_que-&gt;bitmap) {</div><div class='add'>+			ibdev_err(ibdev, "failed to alloc SRQ idx bitmap.\n");</div><div class='add'>+			ret = -ENOMEM;</div><div class='add'>+			goto err_idx_mtr;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	bitmap_num = bitmap_num / (8 * sizeof(u64));</div><div class='add'>+	idx_que-&gt;head = 0;</div><div class='add'>+	idx_que-&gt;tail = 0;</div><div class='ctx'> </div><div class='del'>-	idx_que-&gt;buf_size = srq-&gt;idx_que.buf_size;</div><div class='add'>+	return 0;</div><div class='add'>+err_idx_mtr:</div><div class='add'>+	hns_roce_mtr_destroy(hr_dev, &amp;idx_que-&gt;mtr);</div><div class='ctx'> </div><div class='del'>-	if (hns_roce_buf_alloc(hr_dev, idx_que-&gt;buf_size, (1 &lt;&lt; page_shift) * 2,</div><div class='del'>-			       &amp;idx_que-&gt;idx_buf, page_shift)) {</div><div class='del'>-		kfree(idx_que-&gt;bitmap);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; bitmap_num; i++)</div><div class='del'>-		idx_que-&gt;bitmap[i] = ~(0UL);</div><div class='add'>+static void free_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_idx_que *idx_que = &amp;srq-&gt;idx_que;</div><div class='add'>+</div><div class='add'>+	bitmap_free(idx_que-&gt;bitmap);</div><div class='add'>+	idx_que-&gt;bitmap = NULL;</div><div class='add'>+	hns_roce_mtr_destroy(hr_dev, &amp;idx_que-&gt;mtr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int alloc_srq_wqe_buf(struct hns_roce_dev *hr_dev,</div><div class='add'>+			     struct hns_roce_srq *srq,</div><div class='add'>+			     struct ib_udata *udata, unsigned long addr)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = &amp;hr_dev-&gt;ib_dev;</div><div class='add'>+	struct hns_roce_buf_attr buf_attr = {};</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	srq-&gt;wqe_shift = ilog2(roundup_pow_of_two(max(HNS_ROCE_SGE_SIZE,</div><div class='add'>+						      HNS_ROCE_SGE_SIZE *</div><div class='add'>+						      srq-&gt;max_gs)));</div><div class='add'>+</div><div class='add'>+	buf_attr.page_shift = hr_dev-&gt;caps.srqwqe_buf_pg_sz + PAGE_SHIFT;</div><div class='add'>+	buf_attr.region[0].size = to_hr_hem_entries_size(srq-&gt;wqe_cnt,</div><div class='add'>+							 srq-&gt;wqe_shift);</div><div class='add'>+	buf_attr.region[0].hopnum = hr_dev-&gt;caps.srqwqe_hop_num;</div><div class='add'>+	buf_attr.region_count = 1;</div><div class='add'>+</div><div class='add'>+	ret = hns_roce_mtr_create(hr_dev, &amp;srq-&gt;buf_mtr, &amp;buf_attr,</div><div class='add'>+				  hr_dev-&gt;caps.srqwqe_ba_pg_sz + PAGE_SHIFT,</div><div class='add'>+				  udata, addr);</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_err(ibdev,</div><div class='add'>+			  "failed to alloc SRQ buf mtr, ret = %d.\n", ret);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void free_srq_wqe_buf(struct hns_roce_dev *hr_dev,</div><div class='add'>+			     struct hns_roce_srq *srq)</div><div class='add'>+{</div><div class='add'>+	hns_roce_mtr_destroy(hr_dev, &amp;srq-&gt;buf_mtr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int alloc_srq_wrid(struct hns_roce_srq *srq)</div><div class='add'>+{</div><div class='add'>+	srq-&gt;wrid = kvmalloc_array(srq-&gt;wqe_cnt, sizeof(u64), GFP_KERNEL);</div><div class='add'>+	if (!srq-&gt;wrid)</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int hns_roce_create_srq(struct ib_srq *ib_srq,</div><div class='del'>-			struct ib_srq_init_attr *srq_init_attr,</div><div class='del'>-			struct ib_udata *udata)</div><div class='add'>+static void free_srq_wrid(struct hns_roce_srq *srq)</div><div class='ctx'> {</div><div class='del'>-	struct hns_roce_dev *hr_dev = to_hr_dev(ib_srq-&gt;device);</div><div class='del'>-	struct hns_roce_ib_create_srq_resp resp = {};</div><div class='del'>-	struct hns_roce_srq *srq = to_hr_srq(ib_srq);</div><div class='del'>-	int srq_desc_size;</div><div class='del'>-	int srq_buf_size;</div><div class='del'>-	u32 page_shift;</div><div class='del'>-	int ret = 0;</div><div class='del'>-	u32 npages;</div><div class='del'>-	u32 cqn;</div><div class='del'>-</div><div class='del'>-	/* Check the actual SRQ wqe and SRQ sge num */</div><div class='del'>-	if (srq_init_attr-&gt;attr.max_wr &gt;= hr_dev-&gt;caps.max_srq_wrs ||</div><div class='del'>-	    srq_init_attr-&gt;attr.max_sge &gt; hr_dev-&gt;caps.max_srq_sges)</div><div class='add'>+	kvfree(srq-&gt;wrid);</div><div class='add'>+	srq-&gt;wrid = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u32 proc_srq_sge(struct hns_roce_dev *dev, struct hns_roce_srq *hr_srq,</div><div class='add'>+			bool user)</div><div class='add'>+{</div><div class='add'>+	u32 max_sge = dev-&gt;caps.max_srq_sges;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;pci_dev-&gt;revision &gt;= PCI_REVISION_ID_HIP09)</div><div class='add'>+		return max_sge;</div><div class='add'>+</div><div class='add'>+	/* Reserve SGEs only for HIP08 in kernel; The userspace driver will</div><div class='add'>+	 * calculate number of max_sge with reserved SGEs when allocating wqe</div><div class='add'>+	 * buf, so there is no need to do this again in kernel. But the number</div><div class='add'>+	 * may exceed the capacity of SGEs recorded in the firmware, so the</div><div class='add'>+	 * kernel driver should just adapt the value accordingly.</div><div class='add'>+	 */</div><div class='add'>+	if (user)</div><div class='add'>+		max_sge = roundup_pow_of_two(max_sge + 1);</div><div class='add'>+	else</div><div class='add'>+		hr_srq-&gt;rsv_sge = 1;</div><div class='add'>+</div><div class='add'>+	return max_sge;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int set_srq_basic_param(struct hns_roce_srq *srq,</div><div class='add'>+			       struct ib_srq_init_attr *init_attr,</div><div class='add'>+			       struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(srq-&gt;ibsrq.device);</div><div class='add'>+	struct ib_srq_attr *attr = &amp;init_attr-&gt;attr;</div><div class='add'>+	u32 max_sge;</div><div class='add'>+</div><div class='add'>+	max_sge = proc_srq_sge(hr_dev, srq, !!udata);</div><div class='add'>+	if (attr-&gt;max_wr &gt; hr_dev-&gt;caps.max_srq_wrs ||</div><div class='add'>+	    attr-&gt;max_sge &gt; max_sge || !attr-&gt;max_sge) {</div><div class='add'>+		ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+			  "invalid SRQ attr, depth = %u, sge = %u.\n",</div><div class='add'>+			  attr-&gt;max_wr, attr-&gt;max_sge);</div><div class='ctx'> 		return -EINVAL;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	mutex_init(&amp;srq-&gt;mutex);</div><div class='del'>-	spin_lock_init(&amp;srq-&gt;lock);</div><div class='add'>+	attr-&gt;max_wr = max_t(u32, attr-&gt;max_wr, HNS_ROCE_MIN_SRQ_WQE_NUM);</div><div class='add'>+	srq-&gt;wqe_cnt = roundup_pow_of_two(attr-&gt;max_wr);</div><div class='add'>+	srq-&gt;max_gs = roundup_pow_of_two(attr-&gt;max_sge + srq-&gt;rsv_sge);</div><div class='ctx'> </div><div class='del'>-	srq-&gt;max = roundup_pow_of_two(srq_init_attr-&gt;attr.max_wr + 1);</div><div class='del'>-	srq-&gt;max_gs = srq_init_attr-&gt;attr.max_sge;</div><div class='add'>+	attr-&gt;max_wr = srq-&gt;wqe_cnt;</div><div class='add'>+	attr-&gt;max_sge = srq-&gt;max_gs - srq-&gt;rsv_sge;</div><div class='add'>+	attr-&gt;srq_limit = 0;</div><div class='ctx'> </div><div class='del'>-	srq_desc_size = max(16, 16 * srq-&gt;max_gs);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	srq-&gt;wqe_shift = ilog2(srq_desc_size);</div><div class='add'>+static void set_srq_ext_param(struct hns_roce_srq *srq,</div><div class='add'>+			      struct ib_srq_init_attr *init_attr)</div><div class='add'>+{</div><div class='add'>+	srq-&gt;cqn = ib_srq_has_cq(init_attr-&gt;srq_type) ?</div><div class='add'>+		   to_hr_cq(init_attr-&gt;ext.cq)-&gt;cqn : 0;</div><div class='ctx'> </div><div class='del'>-	srq_buf_size = srq-&gt;max * srq_desc_size;</div><div class='add'>+	srq-&gt;xrcdn = (init_attr-&gt;srq_type == IB_SRQT_XRC) ?</div><div class='add'>+		     to_hr_xrcd(init_attr-&gt;ext.xrc.xrcd)-&gt;xrcdn : 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	srq-&gt;idx_que.entry_sz = HNS_ROCE_IDX_QUE_ENTRY_SZ;</div><div class='del'>-	srq-&gt;idx_que.buf_size = srq-&gt;max * srq-&gt;idx_que.entry_sz;</div><div class='del'>-	srq-&gt;mtt.mtt_type = MTT_TYPE_SRQWQE;</div><div class='del'>-	srq-&gt;idx_que.mtt.mtt_type = MTT_TYPE_IDX;</div><div class='add'>+static int set_srq_param(struct hns_roce_srq *srq,</div><div class='add'>+			 struct ib_srq_init_attr *init_attr,</div><div class='add'>+			 struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	if (udata) {</div><div class='del'>-		struct hns_roce_ib_create_srq  ucmd;</div><div class='del'>-</div><div class='del'>-		if (ib_copy_from_udata(&amp;ucmd, udata, sizeof(ucmd)))</div><div class='del'>-			return -EFAULT;</div><div class='del'>-</div><div class='del'>-		srq-&gt;umem =</div><div class='del'>-			ib_umem_get(udata, ucmd.buf_addr, srq_buf_size, 0, 0);</div><div class='del'>-		if (IS_ERR(srq-&gt;umem))</div><div class='del'>-			return PTR_ERR(srq-&gt;umem);</div><div class='del'>-</div><div class='del'>-		if (hr_dev-&gt;caps.srqwqe_buf_pg_sz) {</div><div class='del'>-			npages = (ib_umem_page_count(srq-&gt;umem) +</div><div class='del'>-				  (1 &lt;&lt; hr_dev-&gt;caps.srqwqe_buf_pg_sz) - 1) /</div><div class='del'>-				  (1 &lt;&lt; hr_dev-&gt;caps.srqwqe_buf_pg_sz);</div><div class='del'>-			page_shift = PAGE_SHIFT + hr_dev-&gt;caps.srqwqe_buf_pg_sz;</div><div class='del'>-			ret = hns_roce_mtt_init(hr_dev, npages,</div><div class='del'>-						page_shift,</div><div class='del'>-						&amp;srq-&gt;mtt);</div><div class='del'>-		} else</div><div class='del'>-			ret = hns_roce_mtt_init(hr_dev,</div><div class='del'>-						ib_umem_page_count(srq-&gt;umem),</div><div class='del'>-						srq-&gt;umem-&gt;page_shift,</div><div class='del'>-						&amp;srq-&gt;mtt);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto err_buf;</div><div class='add'>+	ret = set_srq_basic_param(srq, init_attr, udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='del'>-		ret = hns_roce_ib_umem_write_mtt(hr_dev, &amp;srq-&gt;mtt, srq-&gt;umem);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto err_srq_mtt;</div><div class='add'>+	set_srq_ext_param(srq, init_attr);</div><div class='ctx'> </div><div class='del'>-		/* config index queue BA */</div><div class='del'>-		srq-&gt;idx_que.umem = ib_umem_get(udata, ucmd.que_addr,</div><div class='del'>-						srq-&gt;idx_que.buf_size, 0, 0);</div><div class='del'>-		if (IS_ERR(srq-&gt;idx_que.umem)) {</div><div class='del'>-			dev_err(hr_dev-&gt;dev,</div><div class='del'>-				"ib_umem_get error for index queue\n");</div><div class='del'>-			ret = PTR_ERR(srq-&gt;idx_que.umem);</div><div class='del'>-			goto err_srq_mtt;</div><div class='del'>-		}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		if (hr_dev-&gt;caps.idx_buf_pg_sz) {</div><div class='del'>-			npages = (ib_umem_page_count(srq-&gt;idx_que.umem) +</div><div class='del'>-				  (1 &lt;&lt; hr_dev-&gt;caps.idx_buf_pg_sz) - 1) /</div><div class='del'>-				  (1 &lt;&lt; hr_dev-&gt;caps.idx_buf_pg_sz);</div><div class='del'>-			page_shift = PAGE_SHIFT + hr_dev-&gt;caps.idx_buf_pg_sz;</div><div class='del'>-			ret = hns_roce_mtt_init(hr_dev, npages,</div><div class='del'>-						page_shift, &amp;srq-&gt;idx_que.mtt);</div><div class='del'>-		} else {</div><div class='del'>-			ret = hns_roce_mtt_init(hr_dev,</div><div class='del'>-				       ib_umem_page_count(srq-&gt;idx_que.umem),</div><div class='del'>-				       srq-&gt;idx_que.umem-&gt;page_shift,</div><div class='del'>-				       &amp;srq-&gt;idx_que.mtt);</div><div class='del'>-		}</div><div class='add'>+static int alloc_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,</div><div class='add'>+			 struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_ib_create_srq ucmd = {};</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='add'>+	if (udata) {</div><div class='add'>+		ret = ib_copy_from_udata(&amp;ucmd, udata,</div><div class='add'>+					 min(udata-&gt;inlen, sizeof(ucmd)));</div><div class='ctx'> 		if (ret) {</div><div class='del'>-			dev_err(hr_dev-&gt;dev,</div><div class='del'>-				"hns_roce_mtt_init error for idx que\n");</div><div class='del'>-			goto err_idx_mtt;</div><div class='add'>+			ibdev_err(&amp;hr_dev-&gt;ib_dev,</div><div class='add'>+				  "failed to copy SRQ udata, ret = %d.\n",</div><div class='add'>+				  ret);</div><div class='add'>+			return ret;</div><div class='ctx'> 		}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		ret = hns_roce_ib_umem_write_mtt(hr_dev, &amp;srq-&gt;idx_que.mtt,</div><div class='del'>-						 srq-&gt;idx_que.umem);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(hr_dev-&gt;dev,</div><div class='del'>-			      "hns_roce_ib_umem_write_mtt error for idx que\n");</div><div class='del'>-			goto err_idx_buf;</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		page_shift = PAGE_SHIFT + hr_dev-&gt;caps.srqwqe_buf_pg_sz;</div><div class='del'>-		if (hns_roce_buf_alloc(hr_dev, srq_buf_size,</div><div class='del'>-				       (1 &lt;&lt; page_shift) * 2, &amp;srq-&gt;buf,</div><div class='del'>-				       page_shift))</div><div class='del'>-			return -ENOMEM;</div><div class='add'>+	ret = alloc_srq_idx(hr_dev, srq, udata, ucmd.que_addr);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='del'>-		srq-&gt;head = 0;</div><div class='del'>-		srq-&gt;tail = srq-&gt;max - 1;</div><div class='add'>+	ret = alloc_srq_wqe_buf(hr_dev, srq, udata, ucmd.buf_addr);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_idx;</div><div class='ctx'> </div><div class='del'>-		ret = hns_roce_mtt_init(hr_dev, srq-&gt;buf.npages,</div><div class='del'>-					srq-&gt;buf.page_shift, &amp;srq-&gt;mtt);</div><div class='add'>+	if (!udata) {</div><div class='add'>+		ret = alloc_srq_wrid(srq);</div><div class='ctx'> 		if (ret)</div><div class='del'>-			goto err_buf;</div><div class='add'>+			goto err_wqe_buf;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		ret = hns_roce_buf_write_mtt(hr_dev, &amp;srq-&gt;mtt, &amp;srq-&gt;buf);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto err_srq_mtt;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-		page_shift = PAGE_SHIFT + hr_dev-&gt;caps.idx_buf_pg_sz;</div><div class='del'>-		ret = hns_roce_create_idx_que(ib_srq-&gt;pd, srq, page_shift);</div><div class='del'>-		if (ret) {</div><div class='del'>-			dev_err(hr_dev-&gt;dev, "Create idx queue fail(%d)!\n",</div><div class='del'>-				ret);</div><div class='del'>-			goto err_srq_mtt;</div><div class='del'>-		}</div><div class='add'>+err_wqe_buf:</div><div class='add'>+	free_srq_wqe_buf(hr_dev, srq);</div><div class='add'>+err_idx:</div><div class='add'>+	free_srq_idx(hr_dev, srq);</div><div class='ctx'> </div><div class='del'>-		/* Init mtt table for idx_que */</div><div class='del'>-		ret = hns_roce_mtt_init(hr_dev, srq-&gt;idx_que.idx_buf.npages,</div><div class='del'>-					srq-&gt;idx_que.idx_buf.page_shift,</div><div class='del'>-					&amp;srq-&gt;idx_que.mtt);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto err_create_idx;</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		/* Write buffer address into the mtt table */</div><div class='del'>-		ret = hns_roce_buf_write_mtt(hr_dev, &amp;srq-&gt;idx_que.mtt,</div><div class='del'>-					     &amp;srq-&gt;idx_que.idx_buf);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto err_idx_buf;</div><div class='add'>+static void free_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)</div><div class='add'>+{</div><div class='add'>+	free_srq_wrid(srq);</div><div class='add'>+	free_srq_wqe_buf(hr_dev, srq);</div><div class='add'>+	free_srq_idx(hr_dev, srq);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		srq-&gt;wrid = kvmalloc_array(srq-&gt;max, sizeof(u64), GFP_KERNEL);</div><div class='del'>-		if (!srq-&gt;wrid) {</div><div class='del'>-			ret = -ENOMEM;</div><div class='del'>-			goto err_idx_buf;</div><div class='del'>-		}</div><div class='add'>+static int get_srq_ucmd(struct hns_roce_srq *srq, struct ib_udata *udata,</div><div class='add'>+			struct hns_roce_ib_create_srq *ucmd)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = srq-&gt;ibsrq.device;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = ib_copy_from_udata(ucmd, udata, min(udata-&gt;inlen, sizeof(*ucmd)));</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_err(ibdev, "failed to copy SRQ udata, ret = %d.\n", ret);</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	cqn = ib_srq_has_cq(srq_init_attr-&gt;srq_type) ?</div><div class='del'>-	      to_hr_cq(srq_init_attr-&gt;ext.cq)-&gt;cqn : 0;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	srq-&gt;db_reg_l = hr_dev-&gt;reg_base + SRQ_DB_REG;</div><div class='add'>+static void free_srq_db(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,</div><div class='add'>+			struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_ucontext *uctx;</div><div class='ctx'> </div><div class='del'>-	ret = hns_roce_srq_alloc(hr_dev, to_hr_pd(ib_srq-&gt;pd)-&gt;pdn, cqn, 0,</div><div class='del'>-				 &amp;srq-&gt;mtt, 0, srq);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_wrid;</div><div class='add'>+	if (!(srq-&gt;cap_flags &amp; HNS_ROCE_SRQ_CAP_RECORD_DB))</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-	srq-&gt;event = hns_roce_ib_srq_event;</div><div class='del'>-	srq-&gt;ibsrq.ext.xrc.srq_num = srq-&gt;srqn;</div><div class='del'>-	resp.srqn = srq-&gt;srqn;</div><div class='add'>+	srq-&gt;cap_flags &amp;= ~HNS_ROCE_SRQ_CAP_RECORD_DB;</div><div class='add'>+	if (udata) {</div><div class='add'>+		uctx = rdma_udata_to_drv_context(udata,</div><div class='add'>+						 struct hns_roce_ucontext,</div><div class='add'>+						 ibucontext);</div><div class='add'>+		hns_roce_db_unmap_user(uctx, &amp;srq-&gt;rdb);</div><div class='add'>+	} else {</div><div class='add'>+		hns_roce_free_db(hr_dev, &amp;srq-&gt;rdb);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int alloc_srq_db(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,</div><div class='add'>+			struct ib_udata *udata,</div><div class='add'>+			struct hns_roce_ib_create_srq_resp *resp)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_ib_create_srq ucmd = {};</div><div class='add'>+	struct hns_roce_ucontext *uctx;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='ctx'> 	if (udata) {</div><div class='del'>-		if (ib_copy_to_udata(udata, &amp;resp,</div><div class='del'>-				     min(udata-&gt;outlen, sizeof(resp)))) {</div><div class='del'>-			ret = -EFAULT;</div><div class='del'>-			goto err_srqc_alloc;</div><div class='add'>+		ret = get_srq_ucmd(srq, udata, &amp;ucmd);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+</div><div class='add'>+		if ((hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_SRQ_RECORD_DB) &amp;&amp;</div><div class='add'>+		    (ucmd.req_cap_flags &amp; HNS_ROCE_SRQ_CAP_RECORD_DB)) {</div><div class='add'>+			uctx = rdma_udata_to_drv_context(udata,</div><div class='add'>+					struct hns_roce_ucontext, ibucontext);</div><div class='add'>+			ret = hns_roce_db_map_user(uctx, ucmd.db_addr,</div><div class='add'>+						   &amp;srq-&gt;rdb);</div><div class='add'>+			if (ret)</div><div class='add'>+				return ret;</div><div class='add'>+</div><div class='add'>+			srq-&gt;cap_flags |= HNS_ROCE_RSP_SRQ_CAP_RECORD_DB;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		if (hr_dev-&gt;caps.flags &amp; HNS_ROCE_CAP_FLAG_SRQ_RECORD_DB) {</div><div class='add'>+			ret = hns_roce_alloc_db(hr_dev, &amp;srq-&gt;rdb, 1);</div><div class='add'>+			if (ret)</div><div class='add'>+				return ret;</div><div class='add'>+</div><div class='add'>+			*srq-&gt;rdb.db_record = 0;</div><div class='add'>+			srq-&gt;cap_flags |= HNS_ROCE_RSP_SRQ_CAP_RECORD_DB;</div><div class='ctx'> 		}</div><div class='add'>+		srq-&gt;db_reg = hr_dev-&gt;reg_base + SRQ_DB_REG;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-err_srqc_alloc:</div><div class='del'>-	hns_roce_srq_free(hr_dev, srq);</div><div class='add'>+int hns_roce_create_srq(struct ib_srq *ib_srq,</div><div class='add'>+			struct ib_srq_init_attr *init_attr,</div><div class='add'>+			struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_srq-&gt;device);</div><div class='add'>+	struct hns_roce_ib_create_srq_resp resp = {};</div><div class='add'>+	struct hns_roce_srq *srq = to_hr_srq(ib_srq);</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-err_wrid:</div><div class='del'>-	kvfree(srq-&gt;wrid);</div><div class='add'>+	mutex_init(&amp;srq-&gt;mutex);</div><div class='add'>+	spin_lock_init(&amp;srq-&gt;lock);</div><div class='ctx'> </div><div class='del'>-err_idx_buf:</div><div class='del'>-	hns_roce_mtt_cleanup(hr_dev, &amp;srq-&gt;idx_que.mtt);</div><div class='add'>+	ret = set_srq_param(srq, init_attr, udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_out;</div><div class='ctx'> </div><div class='del'>-err_idx_mtt:</div><div class='del'>-	if (udata)</div><div class='del'>-		ib_umem_release(srq-&gt;idx_que.umem);</div><div class='add'>+	ret = alloc_srq_buf(hr_dev, srq, udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_out;</div><div class='ctx'> </div><div class='del'>-err_create_idx:</div><div class='del'>-	hns_roce_buf_free(hr_dev, srq-&gt;idx_que.buf_size,</div><div class='del'>-			  &amp;srq-&gt;idx_que.idx_buf);</div><div class='del'>-	kfree(srq-&gt;idx_que.bitmap);</div><div class='add'>+	ret = alloc_srq_db(hr_dev, srq, udata, &amp;resp);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_srq_buf;</div><div class='ctx'> </div><div class='del'>-err_srq_mtt:</div><div class='del'>-	hns_roce_mtt_cleanup(hr_dev, &amp;srq-&gt;mtt);</div><div class='add'>+	ret = alloc_srqn(hr_dev, srq);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_srq_db;</div><div class='ctx'> </div><div class='del'>-err_buf:</div><div class='del'>-	if (udata)</div><div class='del'>-		ib_umem_release(srq-&gt;umem);</div><div class='del'>-	else</div><div class='del'>-		hns_roce_buf_free(hr_dev, srq_buf_size, &amp;srq-&gt;buf);</div><div class='add'>+	ret = alloc_srqc(hr_dev, srq);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_srqn;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		resp.cap_flags = srq-&gt;cap_flags;</div><div class='add'>+		resp.srqn = srq-&gt;srqn;</div><div class='add'>+		if (ib_copy_to_udata(udata, &amp;resp,</div><div class='add'>+				     min(udata-&gt;outlen, sizeof(resp)))) {</div><div class='add'>+			ret = -EFAULT;</div><div class='add'>+			goto err_srqc;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	srq-&gt;event = hns_roce_ib_srq_event;</div><div class='add'>+	refcount_set(&amp;srq-&gt;refcount, 1);</div><div class='add'>+	init_completion(&amp;srq-&gt;free);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_srqc:</div><div class='add'>+	free_srqc(hr_dev, srq);</div><div class='add'>+err_srqn:</div><div class='add'>+	free_srqn(hr_dev, srq);</div><div class='add'>+err_srq_db:</div><div class='add'>+	free_srq_db(hr_dev, srq, udata);</div><div class='add'>+err_srq_buf:</div><div class='add'>+	free_srq_buf(hr_dev, srq);</div><div class='add'>+err_out:</div><div class='add'>+	mutex_destroy(&amp;srq-&gt;mutex);</div><div class='add'>+	atomic64_inc(&amp;hr_dev-&gt;dfx_cnt[HNS_ROCE_DFX_SRQ_CREATE_ERR_CNT]);</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)</div><div class='add'>+int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq-&gt;device);</div><div class='ctx'> 	struct hns_roce_srq *srq = to_hr_srq(ibsrq);</div><div class='ctx'> </div><div class='del'>-	hns_roce_srq_free(hr_dev, srq);</div><div class='del'>-	hns_roce_mtt_cleanup(hr_dev, &amp;srq-&gt;mtt);</div><div class='del'>-</div><div class='del'>-	if (ibsrq-&gt;uobject) {</div><div class='del'>-		hns_roce_mtt_cleanup(hr_dev, &amp;srq-&gt;idx_que.mtt);</div><div class='del'>-		ib_umem_release(srq-&gt;idx_que.umem);</div><div class='del'>-		ib_umem_release(srq-&gt;umem);</div><div class='del'>-	} else {</div><div class='del'>-		kvfree(srq-&gt;wrid);</div><div class='del'>-		hns_roce_buf_free(hr_dev, srq-&gt;max &lt;&lt; srq-&gt;wqe_shift,</div><div class='del'>-				  &amp;srq-&gt;buf);</div><div class='del'>-	}</div><div class='add'>+	free_srqc(hr_dev, srq);</div><div class='add'>+	free_srqn(hr_dev, srq);</div><div class='add'>+	free_srq_db(hr_dev, srq, udata);</div><div class='add'>+	free_srq_buf(hr_dev, srq);</div><div class='add'>+	mutex_destroy(&amp;srq-&gt;mutex);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev)</div><div class='add'>+void hns_roce_init_srq_table(struct hns_roce_dev *hr_dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct hns_roce_srq_table *srq_table = &amp;hr_dev-&gt;srq_table;</div><div class='add'>+	struct hns_roce_ida *srq_ida = &amp;srq_table-&gt;srq_ida;</div><div class='ctx'> </div><div class='ctx'> 	xa_init(&amp;srq_table-&gt;xa);</div><div class='ctx'> </div><div class='del'>-	return hns_roce_bitmap_init(&amp;srq_table-&gt;bitmap, hr_dev-&gt;caps.num_srqs,</div><div class='del'>-				    hr_dev-&gt;caps.num_srqs - 1,</div><div class='del'>-				    hr_dev-&gt;caps.reserved_srqs, 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void hns_roce_cleanup_srq_table(struct hns_roce_dev *hr_dev)</div><div class='del'>-{</div><div class='del'>-	hns_roce_bitmap_cleanup(&amp;hr_dev-&gt;srq_table.bitmap);</div><div class='add'>+	ida_init(&amp;srq_ida-&gt;ida);</div><div class='add'>+	srq_ida-&gt;max = hr_dev-&gt;caps.num_srqs - 1;</div><div class='add'>+	srq_ida-&gt;min = hr_dev-&gt;caps.reserved_srqs;</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/hns/hns_roce_trace.h b/drivers/infiniband/hw/hns/hns_roce_trace.h<br/>new file mode 100644<br/>index 000000000000..59ceb591b3a1<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/hns/hns_roce_trace.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/hns/hns_roce_trace.h</a></div><div class='hunk'>@@ -0,0 +1,216 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0+ */</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2025 Hisilicon Limited.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#undef TRACE_SYSTEM</div><div class='add'>+#define TRACE_SYSTEM hns_roce</div><div class='add'>+</div><div class='add'>+#if !defined(__HNS_ROCE_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)</div><div class='add'>+#define __HNS_ROCE_TRACE_H</div><div class='add'>+</div><div class='add'>+#include &lt;linux/tracepoint.h&gt;</div><div class='add'>+#include &lt;linux/string_choices.h&gt;</div><div class='add'>+#include "hns_roce_device.h"</div><div class='add'>+#include "hns_roce_hw_v2.h"</div><div class='add'>+</div><div class='add'>+DECLARE_EVENT_CLASS(flush_head_template,</div><div class='add'>+		    TP_PROTO(unsigned long qpn, u32 pi,</div><div class='add'>+			     enum hns_roce_trace_type type),</div><div class='add'>+		    TP_ARGS(qpn, pi, type),</div><div class='add'>+</div><div class='add'>+		    TP_STRUCT__entry(__field(unsigned long, qpn)</div><div class='add'>+				     __field(u32, pi)</div><div class='add'>+				     __field(enum hns_roce_trace_type, type)</div><div class='add'>+		    ),</div><div class='add'>+</div><div class='add'>+		    TP_fast_assign(__entry-&gt;qpn = qpn;</div><div class='add'>+				   __entry-&gt;pi = pi;</div><div class='add'>+				   __entry-&gt;type = type;</div><div class='add'>+		    ),</div><div class='add'>+</div><div class='add'>+		    TP_printk("%s 0x%lx flush head 0x%x.",</div><div class='add'>+			      trace_type_to_str(__entry-&gt;type),</div><div class='add'>+			      __entry-&gt;qpn, __entry-&gt;pi)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(flush_head_template, hns_sq_flush_cqe,</div><div class='add'>+	     TP_PROTO(unsigned long qpn, u32 pi,</div><div class='add'>+		      enum hns_roce_trace_type type),</div><div class='add'>+	     TP_ARGS(qpn, pi, type));</div><div class='add'>+DEFINE_EVENT(flush_head_template, hns_rq_flush_cqe,</div><div class='add'>+	     TP_PROTO(unsigned long qpn, u32 pi,</div><div class='add'>+		      enum hns_roce_trace_type type),</div><div class='add'>+	     TP_ARGS(qpn, pi, type));</div><div class='add'>+</div><div class='add'>+#define MAX_SGE_PER_WQE 64</div><div class='add'>+#define MAX_WQE_SIZE (MAX_SGE_PER_WQE * HNS_ROCE_SGE_SIZE)</div><div class='add'>+DECLARE_EVENT_CLASS(wqe_template,</div><div class='add'>+		    TP_PROTO(unsigned long qpn, u32 idx, void *wqe, u32 len,</div><div class='add'>+			     u64 id, enum hns_roce_trace_type type),</div><div class='add'>+		    TP_ARGS(qpn, idx, wqe, len, id, type),</div><div class='add'>+</div><div class='add'>+		    TP_STRUCT__entry(__field(unsigned long, qpn)</div><div class='add'>+				     __field(u32, idx)</div><div class='add'>+				     __array(u32, wqe,</div><div class='add'>+					     MAX_WQE_SIZE / sizeof(__le32))</div><div class='add'>+				     __field(u32, len)</div><div class='add'>+				     __field(u64, id)</div><div class='add'>+				     __field(enum hns_roce_trace_type, type)</div><div class='add'>+				     ),</div><div class='add'>+</div><div class='add'>+		    TP_fast_assign(__entry-&gt;qpn = qpn;</div><div class='add'>+				   __entry-&gt;idx = idx;</div><div class='add'>+				   __entry-&gt;id = id;</div><div class='add'>+				   __entry-&gt;len = len / sizeof(__le32);</div><div class='add'>+				   __entry-&gt;type = type;</div><div class='add'>+				   for (int i = 0; i &lt; __entry-&gt;len; i++)</div><div class='add'>+					__entry-&gt;wqe[i] = le32_to_cpu(((__le32 *)wqe)[i]);</div><div class='add'>+				   ),</div><div class='add'>+</div><div class='add'>+		    TP_printk("%s 0x%lx wqe(0x%x/0x%llx): %s",</div><div class='add'>+			      trace_type_to_str(__entry-&gt;type),</div><div class='add'>+			      __entry-&gt;qpn, __entry-&gt;idx, __entry-&gt;id,</div><div class='add'>+			      __print_array(__entry-&gt;wqe, __entry-&gt;len,</div><div class='add'>+					    sizeof(__le32)))</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(wqe_template, hns_sq_wqe,</div><div class='add'>+	     TP_PROTO(unsigned long qpn, u32 idx, void *wqe, u32 len, u64 id,</div><div class='add'>+		      enum hns_roce_trace_type type),</div><div class='add'>+	     TP_ARGS(qpn, idx, wqe, len, id, type));</div><div class='add'>+DEFINE_EVENT(wqe_template, hns_rq_wqe,</div><div class='add'>+	     TP_PROTO(unsigned long qpn, u32 idx, void *wqe, u32 len, u64 id,</div><div class='add'>+		      enum hns_roce_trace_type type),</div><div class='add'>+	     TP_ARGS(qpn, idx, wqe, len, id, type));</div><div class='add'>+DEFINE_EVENT(wqe_template, hns_srq_wqe,</div><div class='add'>+	     TP_PROTO(unsigned long qpn, u32 idx, void *wqe, u32 len, u64 id,</div><div class='add'>+		      enum hns_roce_trace_type type),</div><div class='add'>+	     TP_ARGS(qpn, idx, wqe, len, id, type));</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(hns_ae_info,</div><div class='add'>+	    TP_PROTO(int event_type, void *aeqe, unsigned int len),</div><div class='add'>+	    TP_ARGS(event_type, aeqe, len),</div><div class='add'>+</div><div class='add'>+	    TP_STRUCT__entry(__field(int, event_type)</div><div class='add'>+			     __array(u32, aeqe,</div><div class='add'>+				     HNS_ROCE_V3_EQE_SIZE / sizeof(__le32))</div><div class='add'>+			     __field(u32, len)</div><div class='add'>+	    ),</div><div class='add'>+</div><div class='add'>+	    TP_fast_assign(__entry-&gt;event_type = event_type;</div><div class='add'>+			   __entry-&gt;len = len / sizeof(__le32);</div><div class='add'>+			   for (int i = 0; i &lt; __entry-&gt;len; i++)</div><div class='add'>+				__entry-&gt;aeqe[i] = le32_to_cpu(((__le32 *)aeqe)[i]);</div><div class='add'>+	    ),</div><div class='add'>+</div><div class='add'>+	    TP_printk("event %2d aeqe: %s", __entry-&gt;event_type,</div><div class='add'>+		      __print_array(__entry-&gt;aeqe, __entry-&gt;len, sizeof(__le32)))</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(hns_mr,</div><div class='add'>+	    TP_PROTO(struct hns_roce_mr *mr),</div><div class='add'>+	    TP_ARGS(mr),</div><div class='add'>+</div><div class='add'>+	    TP_STRUCT__entry(__field(u64, iova)</div><div class='add'>+			     __field(u64, size)</div><div class='add'>+			     __field(u32, key)</div><div class='add'>+			     __field(u32, pd)</div><div class='add'>+			     __field(u32, pbl_hop_num)</div><div class='add'>+			     __field(u32, npages)</div><div class='add'>+			     __field(int, type)</div><div class='add'>+			     __field(int, enabled)</div><div class='add'>+	    ),</div><div class='add'>+</div><div class='add'>+	    TP_fast_assign(__entry-&gt;iova = mr-&gt;iova;</div><div class='add'>+			   __entry-&gt;size = mr-&gt;size;</div><div class='add'>+			   __entry-&gt;key = mr-&gt;key;</div><div class='add'>+			   __entry-&gt;pd = mr-&gt;pd;</div><div class='add'>+			   __entry-&gt;pbl_hop_num = mr-&gt;pbl_hop_num;</div><div class='add'>+			   __entry-&gt;npages = mr-&gt;npages;</div><div class='add'>+			   __entry-&gt;type = mr-&gt;type;</div><div class='add'>+			   __entry-&gt;enabled = mr-&gt;enabled;</div><div class='add'>+	    ),</div><div class='add'>+</div><div class='add'>+	    TP_printk("iova:0x%llx, size:%llu, key:%u, pd:%u, pbl_hop:%u, npages:%u, type:%d, status:%d",</div><div class='add'>+		      __entry-&gt;iova, __entry-&gt;size, __entry-&gt;key,</div><div class='add'>+		      __entry-&gt;pd, __entry-&gt;pbl_hop_num, __entry-&gt;npages,</div><div class='add'>+		      __entry-&gt;type, __entry-&gt;enabled)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(hns_buf_attr,</div><div class='add'>+	    TP_PROTO(struct hns_roce_buf_attr *attr),</div><div class='add'>+	    TP_ARGS(attr),</div><div class='add'>+</div><div class='add'>+	    TP_STRUCT__entry(__field(unsigned int, region_count)</div><div class='add'>+			     __field(unsigned int, region0_size)</div><div class='add'>+			     __field(int, region0_hopnum)</div><div class='add'>+			     __field(unsigned int, region1_size)</div><div class='add'>+			     __field(int, region1_hopnum)</div><div class='add'>+			     __field(unsigned int, region2_size)</div><div class='add'>+			     __field(int, region2_hopnum)</div><div class='add'>+			     __field(unsigned int, page_shift)</div><div class='add'>+			     __field(bool, mtt_only)</div><div class='add'>+	    ),</div><div class='add'>+</div><div class='add'>+	    TP_fast_assign(__entry-&gt;region_count = attr-&gt;region_count;</div><div class='add'>+			   __entry-&gt;region0_size = attr-&gt;region[0].size;</div><div class='add'>+			   __entry-&gt;region0_hopnum = attr-&gt;region[0].hopnum;</div><div class='add'>+			   __entry-&gt;region1_size = attr-&gt;region[1].size;</div><div class='add'>+			   __entry-&gt;region1_hopnum = attr-&gt;region[1].hopnum;</div><div class='add'>+			   __entry-&gt;region2_size = attr-&gt;region[2].size;</div><div class='add'>+			   __entry-&gt;region2_hopnum = attr-&gt;region[2].hopnum;</div><div class='add'>+			   __entry-&gt;page_shift = attr-&gt;page_shift;</div><div class='add'>+			   __entry-&gt;mtt_only = attr-&gt;mtt_only;</div><div class='add'>+	    ),</div><div class='add'>+</div><div class='add'>+	    TP_printk("rg cnt:%u, pg_sft:0x%x, mtt_only:%s, rg 0 (sz:%u, hop:%u), rg 1 (sz:%u, hop:%u), rg 2 (sz:%u, hop:%u)\n",</div><div class='add'>+		      __entry-&gt;region_count, __entry-&gt;page_shift,</div><div class='add'>+		      str_yes_no(__entry-&gt;mtt_only),</div><div class='add'>+		      __entry-&gt;region0_size, __entry-&gt;region0_hopnum,</div><div class='add'>+		      __entry-&gt;region1_size, __entry-&gt;region1_hopnum,</div><div class='add'>+		      __entry-&gt;region2_size, __entry-&gt;region2_hopnum)</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DECLARE_EVENT_CLASS(cmdq,</div><div class='add'>+		    TP_PROTO(struct hns_roce_dev *hr_dev,</div><div class='add'>+			     struct hns_roce_cmq_desc *desc),</div><div class='add'>+		    TP_ARGS(hr_dev, desc),</div><div class='add'>+</div><div class='add'>+		    TP_STRUCT__entry(__string(dev_name, dev_name(hr_dev-&gt;dev))</div><div class='add'>+				     __field(u16, opcode)</div><div class='add'>+				     __field(u16, flag)</div><div class='add'>+				     __field(u16, retval)</div><div class='add'>+				     __array(u32, data, 6)</div><div class='add'>+		    ),</div><div class='add'>+</div><div class='add'>+		    TP_fast_assign(__assign_str(dev_name);</div><div class='add'>+				   __entry-&gt;opcode = le16_to_cpu(desc-&gt;opcode);</div><div class='add'>+				   __entry-&gt;flag = le16_to_cpu(desc-&gt;flag);</div><div class='add'>+				   __entry-&gt;retval = le16_to_cpu(desc-&gt;retval);</div><div class='add'>+				   for (int i = 0; i &lt; 6; i++)</div><div class='add'>+					__entry-&gt;data[i] = le32_to_cpu(desc-&gt;data[i]);</div><div class='add'>+		    ),</div><div class='add'>+</div><div class='add'>+		    TP_printk("%s cmdq opcode:0x%x, flag:0x%x, retval:0x%x, data:%s\n",</div><div class='add'>+			      __get_str(dev_name), __entry-&gt;opcode,</div><div class='add'>+			      __entry-&gt;flag, __entry-&gt;retval,</div><div class='add'>+			      __print_array(__entry-&gt;data, 6, sizeof(__le32)))</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(cmdq, hns_cmdq_req,</div><div class='add'>+	     TP_PROTO(struct hns_roce_dev *hr_dev,</div><div class='add'>+		      struct hns_roce_cmq_desc *desc),</div><div class='add'>+	     TP_ARGS(hr_dev, desc));</div><div class='add'>+DEFINE_EVENT(cmdq, hns_cmdq_resp,</div><div class='add'>+	     TP_PROTO(struct hns_roce_dev *hr_dev,</div><div class='add'>+		      struct hns_roce_cmq_desc *desc),</div><div class='add'>+	     TP_ARGS(hr_dev, desc));</div><div class='add'>+</div><div class='add'>+#endif /* __HNS_ROCE_TRACE_H */</div><div class='add'>+</div><div class='add'>+#undef TRACE_INCLUDE_FILE</div><div class='add'>+#define TRACE_INCLUDE_FILE hns_roce_trace</div><div class='add'>+#undef TRACE_INCLUDE_PATH</div><div class='add'>+#define TRACE_INCLUDE_PATH .</div><div class='add'>+#include &lt;trace/define_trace.h&gt;</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/Kconfig b/drivers/infiniband/hw/i40iw/Kconfig<br/>deleted file mode 100644<br/>index d867ef1ac72a..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/Kconfig?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/Kconfig</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,8 +0,0 @@</div><div class='del'>-config INFINIBAND_I40IW</div><div class='del'>-	tristate "Intel(R) Ethernet X722 iWARP Driver"</div><div class='del'>-	depends on INET &amp;&amp; I40E</div><div class='del'>-	depends on IPV6 || !IPV6</div><div class='del'>-	depends on PCI</div><div class='del'>-	select GENERIC_ALLOCATOR</div><div class='del'>-	---help---</div><div class='del'>-	Intel(R) Ethernet X722 iWARP Driver</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/Makefile b/drivers/infiniband/hw/i40iw/Makefile<br/>deleted file mode 100644<br/>index 8942f8229945..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/Makefile?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/Makefile</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,10 +0,0 @@</div><div class='del'>-# SPDX-License-Identifier: GPL-2.0</div><div class='del'>-ccflags-y :=  -I $(srctree)/drivers/net/ethernet/intel/i40e</div><div class='del'>-</div><div class='del'>-obj-$(CONFIG_INFINIBAND_I40IW) += i40iw.o</div><div class='del'>-</div><div class='del'>-i40iw-objs :=\</div><div class='del'>-               i40iw_cm.o i40iw_ctrl.o \</div><div class='del'>-               i40iw_hmc.o i40iw_hw.o i40iw_main.o  \</div><div class='del'>-               i40iw_pble.o i40iw_puda.o i40iw_uk.o i40iw_utils.o \</div><div class='del'>-               i40iw_verbs.o i40iw_virtchnl.o i40iw_vf.o</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw.h b/drivers/infiniband/hw/i40iw/i40iw.h<br/>deleted file mode 100644<br/>index 8feec35f95a7..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,602 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#ifndef I40IW_IW_H</div><div class='del'>-#define I40IW_IW_H</div><div class='del'>-#include &lt;linux/netdevice.h&gt;</div><div class='del'>-#include &lt;linux/inetdevice.h&gt;</div><div class='del'>-#include &lt;linux/spinlock.h&gt;</div><div class='del'>-#include &lt;linux/kernel.h&gt;</div><div class='del'>-#include &lt;linux/delay.h&gt;</div><div class='del'>-#include &lt;linux/pci.h&gt;</div><div class='del'>-#include &lt;linux/dma-mapping.h&gt;</div><div class='del'>-#include &lt;linux/workqueue.h&gt;</div><div class='del'>-#include &lt;linux/slab.h&gt;</div><div class='del'>-#include &lt;linux/io.h&gt;</div><div class='del'>-#include &lt;linux/crc32c.h&gt;</div><div class='del'>-#include &lt;rdma/ib_smi.h&gt;</div><div class='del'>-#include &lt;rdma/ib_verbs.h&gt;</div><div class='del'>-#include &lt;rdma/ib_pack.h&gt;</div><div class='del'>-#include &lt;rdma/rdma_cm.h&gt;</div><div class='del'>-#include &lt;rdma/iw_cm.h&gt;</div><div class='del'>-#include &lt;crypto/hash.h&gt;</div><div class='del'>-</div><div class='del'>-#include "i40iw_status.h"</div><div class='del'>-#include "i40iw_osdep.h"</div><div class='del'>-#include "i40iw_d.h"</div><div class='del'>-#include "i40iw_hmc.h"</div><div class='del'>-</div><div class='del'>-#include &lt;i40e_client.h&gt;</div><div class='del'>-#include "i40iw_type.h"</div><div class='del'>-#include "i40iw_p.h"</div><div class='del'>-#include &lt;rdma/i40iw-abi.h&gt;</div><div class='del'>-#include "i40iw_pble.h"</div><div class='del'>-#include "i40iw_verbs.h"</div><div class='del'>-#include "i40iw_cm.h"</div><div class='del'>-#include "i40iw_user.h"</div><div class='del'>-#include "i40iw_puda.h"</div><div class='del'>-</div><div class='del'>-#define I40IW_FW_VERSION  2</div><div class='del'>-#define I40IW_HW_VERSION  2</div><div class='del'>-</div><div class='del'>-#define I40IW_ARP_ADD     1</div><div class='del'>-#define I40IW_ARP_DELETE  2</div><div class='del'>-#define I40IW_ARP_RESOLVE 3</div><div class='del'>-</div><div class='del'>-#define I40IW_MACIP_ADD     1</div><div class='del'>-#define I40IW_MACIP_DELETE  2</div><div class='del'>-</div><div class='del'>-#define IW_CCQ_SIZE         (I40IW_CQP_SW_SQSIZE_2048 + 1)</div><div class='del'>-#define IW_CEQ_SIZE         2048</div><div class='del'>-#define IW_AEQ_SIZE         2048</div><div class='del'>-</div><div class='del'>-#define RX_BUF_SIZE            (1536 + 8)</div><div class='del'>-#define IW_REG0_SIZE           (4 * 1024)</div><div class='del'>-#define IW_TX_TIMEOUT          (6 * HZ)</div><div class='del'>-#define IW_FIRST_QPN           1</div><div class='del'>-#define IW_SW_CONTEXT_ALIGN    1024</div><div class='del'>-</div><div class='del'>-#define MAX_DPC_ITERATIONS		128</div><div class='del'>-</div><div class='del'>-#define I40IW_EVENT_TIMEOUT		100000</div><div class='del'>-#define I40IW_VCHNL_EVENT_TIMEOUT	100000</div><div class='del'>-</div><div class='del'>-#define	I40IW_NO_VLAN			0xffff</div><div class='del'>-#define	I40IW_NO_QSET			0xffff</div><div class='del'>-</div><div class='del'>-/* access to mcast filter list */</div><div class='del'>-#define IW_ADD_MCAST false</div><div class='del'>-#define IW_DEL_MCAST true</div><div class='del'>-</div><div class='del'>-#define I40IW_DRV_OPT_ENABLE_MPA_VER_0     0x00000001</div><div class='del'>-#define I40IW_DRV_OPT_DISABLE_MPA_CRC      0x00000002</div><div class='del'>-#define I40IW_DRV_OPT_DISABLE_FIRST_WRITE  0x00000004</div><div class='del'>-#define I40IW_DRV_OPT_DISABLE_INTF         0x00000008</div><div class='del'>-#define I40IW_DRV_OPT_ENABLE_MSI           0x00000010</div><div class='del'>-#define I40IW_DRV_OPT_DUAL_LOGICAL_PORT    0x00000020</div><div class='del'>-#define I40IW_DRV_OPT_NO_INLINE_DATA       0x00000080</div><div class='del'>-#define I40IW_DRV_OPT_DISABLE_INT_MOD      0x00000100</div><div class='del'>-#define I40IW_DRV_OPT_DISABLE_VIRT_WQ      0x00000200</div><div class='del'>-#define I40IW_DRV_OPT_ENABLE_PAU           0x00000400</div><div class='del'>-#define I40IW_DRV_OPT_MCAST_LOGPORT_MAP    0x00000800</div><div class='del'>-</div><div class='del'>-#define IW_HMC_OBJ_TYPE_NUM ARRAY_SIZE(iw_hmc_obj_types)</div><div class='del'>-#define IW_CFG_FPM_QP_COUNT               32768</div><div class='del'>-#define I40IW_MAX_PAGES_PER_FMR           512</div><div class='del'>-#define I40IW_MIN_PAGES_PER_FMR           1</div><div class='del'>-#define I40IW_CQP_COMPL_RQ_WQE_FLUSHED    2</div><div class='del'>-#define I40IW_CQP_COMPL_SQ_WQE_FLUSHED    3</div><div class='del'>-#define I40IW_CQP_COMPL_RQ_SQ_WQE_FLUSHED 4</div><div class='del'>-</div><div class='del'>-struct i40iw_cqp_compl_info {</div><div class='del'>-	u32 op_ret_val;</div><div class='del'>-	u16 maj_err_code;</div><div class='del'>-	u16 min_err_code;</div><div class='del'>-	bool error;</div><div class='del'>-	u8 op_code;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define i40iw_pr_err(fmt, args ...) pr_err("%s: "fmt, __func__, ## args)</div><div class='del'>-</div><div class='del'>-#define i40iw_pr_info(fmt, args ...) pr_info("%s: " fmt, __func__, ## args)</div><div class='del'>-</div><div class='del'>-#define i40iw_pr_warn(fmt, args ...) pr_warn("%s: " fmt, __func__, ## args)</div><div class='del'>-</div><div class='del'>-struct i40iw_cqp_request {</div><div class='del'>-	struct cqp_commands_info info;</div><div class='del'>-	wait_queue_head_t waitq;</div><div class='del'>-	struct list_head list;</div><div class='del'>-	atomic_t refcount;</div><div class='del'>-	void (*callback_fcn)(struct i40iw_cqp_request*, u32);</div><div class='del'>-	void *param;</div><div class='del'>-	struct i40iw_cqp_compl_info compl_info;</div><div class='del'>-	bool waiting;</div><div class='del'>-	bool request_done;</div><div class='del'>-	bool dynamic;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cqp {</div><div class='del'>-	struct i40iw_sc_cqp sc_cqp;</div><div class='del'>-	spinlock_t req_lock; /*cqp request list */</div><div class='del'>-	wait_queue_head_t waitq;</div><div class='del'>-	struct i40iw_dma_mem sq;</div><div class='del'>-	struct i40iw_dma_mem host_ctx;</div><div class='del'>-	u64 *scratch_array;</div><div class='del'>-	struct i40iw_cqp_request *cqp_requests;</div><div class='del'>-	struct list_head cqp_avail_reqs;</div><div class='del'>-	struct list_head cqp_pending_reqs;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_device;</div><div class='del'>-</div><div class='del'>-struct i40iw_ccq {</div><div class='del'>-	struct i40iw_sc_cq sc_cq;</div><div class='del'>-	spinlock_t lock; /* ccq control */</div><div class='del'>-	wait_queue_head_t waitq;</div><div class='del'>-	struct i40iw_dma_mem mem_cq;</div><div class='del'>-	struct i40iw_dma_mem shadow_area;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_ceq {</div><div class='del'>-	struct i40iw_sc_ceq sc_ceq;</div><div class='del'>-	struct i40iw_dma_mem mem;</div><div class='del'>-	u32 irq;</div><div class='del'>-	u32 msix_idx;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct tasklet_struct dpc_tasklet;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_aeq {</div><div class='del'>-	struct i40iw_sc_aeq sc_aeq;</div><div class='del'>-	struct i40iw_dma_mem mem;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_arp_entry {</div><div class='del'>-	u32 ip_addr[4];</div><div class='del'>-	u8 mac_addr[ETH_ALEN];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum init_completion_state {</div><div class='del'>-	INVALID_STATE = 0,</div><div class='del'>-	INITIAL_STATE,</div><div class='del'>-	CQP_CREATED,</div><div class='del'>-	HMC_OBJS_CREATED,</div><div class='del'>-	PBLE_CHUNK_MEM,</div><div class='del'>-	CCQ_CREATED,</div><div class='del'>-	AEQ_CREATED,</div><div class='del'>-	CEQ_CREATED,</div><div class='del'>-	ILQ_CREATED,</div><div class='del'>-	IEQ_CREATED,</div><div class='del'>-	IP_ADDR_REGISTERED,</div><div class='del'>-	RDMA_DEV_REGISTERED</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_msix_vector {</div><div class='del'>-	u32 idx;</div><div class='del'>-	u32 irq;</div><div class='del'>-	u32 cpu_affinity;</div><div class='del'>-	u32 ceq_id;</div><div class='del'>-	cpumask_t mask;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct l2params_work {</div><div class='del'>-	struct work_struct work;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_l2params l2params;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define I40IW_MSIX_TABLE_SIZE   65</div><div class='del'>-</div><div class='del'>-struct virtchnl_work {</div><div class='del'>-	struct work_struct work;</div><div class='del'>-	union {</div><div class='del'>-		struct i40iw_cqp_request *cqp_request;</div><div class='del'>-		struct i40iw_virtchnl_work_info work_info;</div><div class='del'>-	};</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40e_qvlist_info;</div><div class='del'>-</div><div class='del'>-struct i40iw_device {</div><div class='del'>-	struct i40iw_ib_device *iwibdev;</div><div class='del'>-	struct net_device *netdev;</div><div class='del'>-	wait_queue_head_t vchnl_waitq;</div><div class='del'>-	struct i40iw_sc_dev sc_dev;</div><div class='del'>-	struct i40iw_sc_vsi vsi;</div><div class='del'>-	struct i40iw_handler *hdl;</div><div class='del'>-	struct i40e_info *ldev;</div><div class='del'>-	struct i40e_client *client;</div><div class='del'>-	struct i40iw_hw hw;</div><div class='del'>-	struct i40iw_cm_core cm_core;</div><div class='del'>-	u8 *mem_resources;</div><div class='del'>-	unsigned long *allocated_qps;</div><div class='del'>-	unsigned long *allocated_cqs;</div><div class='del'>-	unsigned long *allocated_mrs;</div><div class='del'>-	unsigned long *allocated_pds;</div><div class='del'>-	unsigned long *allocated_arps;</div><div class='del'>-	struct i40iw_qp **qp_table;</div><div class='del'>-	bool msix_shared;</div><div class='del'>-	u32 msix_count;</div><div class='del'>-	struct i40iw_msix_vector *iw_msixtbl;</div><div class='del'>-	struct i40e_qvlist_info *iw_qvlist;</div><div class='del'>-</div><div class='del'>-	struct i40iw_hmc_pble_rsrc *pble_rsrc;</div><div class='del'>-	struct i40iw_arp_entry *arp_table;</div><div class='del'>-	struct i40iw_cqp cqp;</div><div class='del'>-	struct i40iw_ccq ccq;</div><div class='del'>-	u32 ceqs_count;</div><div class='del'>-	struct i40iw_ceq *ceqlist;</div><div class='del'>-	struct i40iw_aeq aeq;</div><div class='del'>-	u32 arp_table_size;</div><div class='del'>-	u32 next_arp_index;</div><div class='del'>-	spinlock_t resource_lock; /* hw resource access */</div><div class='del'>-	spinlock_t qptable_lock;</div><div class='del'>-	u32 vendor_id;</div><div class='del'>-	u32 vendor_part_id;</div><div class='del'>-	u32 of_device_registered;</div><div class='del'>-</div><div class='del'>-	u32 device_cap_flags;</div><div class='del'>-	unsigned long db_start;</div><div class='del'>-	u8 resource_profile;</div><div class='del'>-	u8 max_rdma_vfs;</div><div class='del'>-	u8 max_enabled_vfs;</div><div class='del'>-	u8 max_sge;</div><div class='del'>-	u8 iw_status;</div><div class='del'>-	u8 send_term_ok;</div><div class='del'>-	bool push_mode;		/* Initialized from parameter passed to driver */</div><div class='del'>-</div><div class='del'>-	/* x710 specific */</div><div class='del'>-	struct mutex pbl_mutex;</div><div class='del'>-	struct tasklet_struct dpc_tasklet;</div><div class='del'>-	struct workqueue_struct *virtchnl_wq;</div><div class='del'>-	struct virtchnl_work virtchnl_w[I40IW_MAX_PE_ENABLED_VF_COUNT];</div><div class='del'>-	struct i40iw_dma_mem obj_mem;</div><div class='del'>-	struct i40iw_dma_mem obj_next;</div><div class='del'>-	u8 *hmc_info_mem;</div><div class='del'>-	u32 sd_type;</div><div class='del'>-	struct workqueue_struct *param_wq;</div><div class='del'>-	atomic_t params_busy;</div><div class='del'>-	enum init_completion_state init_state;</div><div class='del'>-	u16 mac_ip_table_idx;</div><div class='del'>-	atomic_t vchnl_msgs;</div><div class='del'>-	u32 max_mr;</div><div class='del'>-	u32 max_qp;</div><div class='del'>-	u32 max_cq;</div><div class='del'>-	u32 max_pd;</div><div class='del'>-	u32 next_qp;</div><div class='del'>-	u32 next_cq;</div><div class='del'>-	u32 next_pd;</div><div class='del'>-	u32 max_mr_size;</div><div class='del'>-	u32 max_qp_wr;</div><div class='del'>-	u32 max_cqe;</div><div class='del'>-	u32 mr_stagmask;</div><div class='del'>-	u32 mpa_version;</div><div class='del'>-	bool dcb;</div><div class='del'>-	bool closing;</div><div class='del'>-	bool reset;</div><div class='del'>-	u32 used_pds;</div><div class='del'>-	u32 used_cqs;</div><div class='del'>-	u32 used_mrs;</div><div class='del'>-	u32 used_qps;</div><div class='del'>-	wait_queue_head_t close_wq;</div><div class='del'>-	atomic64_t use_count;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_ib_device {</div><div class='del'>-	struct ib_device ibdev;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_handler {</div><div class='del'>-	struct list_head list;</div><div class='del'>-	struct i40e_client *client;</div><div class='del'>-	struct i40iw_device device;</div><div class='del'>-	struct i40e_info ldev;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * to_iwdev - get device</div><div class='del'>- * @ibdev: ib device</div><div class='del'>- **/</div><div class='del'>-static inline struct i40iw_device *to_iwdev(struct ib_device *ibdev)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibdev, struct i40iw_ib_device, ibdev)-&gt;iwdev;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * to_ucontext - get user context</div><div class='del'>- * @ibucontext: ib user context</div><div class='del'>- **/</div><div class='del'>-static inline struct i40iw_ucontext *to_ucontext(struct ib_ucontext *ibucontext)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibucontext, struct i40iw_ucontext, ibucontext);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * to_iwpd - get protection domain</div><div class='del'>- * @ibpd: ib pd</div><div class='del'>- **/</div><div class='del'>-static inline struct i40iw_pd *to_iwpd(struct ib_pd *ibpd)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibpd, struct i40iw_pd, ibpd);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * to_iwmr - get device memory region</div><div class='del'>- * @ibdev: ib memory region</div><div class='del'>- **/</div><div class='del'>-static inline struct i40iw_mr *to_iwmr(struct ib_mr *ibmr)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibmr, struct i40iw_mr, ibmr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * to_iwmr_from_ibfmr - get device memory region</div><div class='del'>- * @ibfmr: ib fmr</div><div class='del'>- **/</div><div class='del'>-static inline struct i40iw_mr *to_iwmr_from_ibfmr(struct ib_fmr *ibfmr)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibfmr, struct i40iw_mr, ibfmr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * to_iwmw - get device memory window</div><div class='del'>- * @ibmw: ib memory window</div><div class='del'>- **/</div><div class='del'>-static inline struct i40iw_mr *to_iwmw(struct ib_mw *ibmw)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibmw, struct i40iw_mr, ibmw);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * to_iwcq - get completion queue</div><div class='del'>- * @ibcq: ib cqdevice</div><div class='del'>- **/</div><div class='del'>-static inline struct i40iw_cq *to_iwcq(struct ib_cq *ibcq)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibcq, struct i40iw_cq, ibcq);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * to_iwqp - get device qp</div><div class='del'>- * @ibqp: ib qp</div><div class='del'>- **/</div><div class='del'>-static inline struct i40iw_qp *to_iwqp(struct ib_qp *ibqp)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibqp, struct i40iw_qp, ibqp);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* i40iw.c */</div><div class='del'>-void i40iw_add_ref(struct ib_qp *);</div><div class='del'>-void i40iw_rem_ref(struct ib_qp *);</div><div class='del'>-struct ib_qp *i40iw_get_qp(struct ib_device *, int);</div><div class='del'>-</div><div class='del'>-void i40iw_flush_wqes(struct i40iw_device *iwdev,</div><div class='del'>-		      struct i40iw_qp *qp);</div><div class='del'>-</div><div class='del'>-void i40iw_manage_arp_cache(struct i40iw_device *iwdev,</div><div class='del'>-			    unsigned char *mac_addr,</div><div class='del'>-			    u32 *ip_addr,</div><div class='del'>-			    bool ipv4,</div><div class='del'>-			    u32 action);</div><div class='del'>-</div><div class='del'>-int i40iw_manage_apbvt(struct i40iw_device *iwdev,</div><div class='del'>-		       u16 accel_local_port,</div><div class='del'>-		       bool add_port);</div><div class='del'>-</div><div class='del'>-struct i40iw_cqp_request *i40iw_get_cqp_request(struct i40iw_cqp *cqp, bool wait);</div><div class='del'>-void i40iw_free_cqp_request(struct i40iw_cqp *cqp, struct i40iw_cqp_request *cqp_request);</div><div class='del'>-void i40iw_put_cqp_request(struct i40iw_cqp *cqp, struct i40iw_cqp_request *cqp_request);</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_alloc_resource - allocate a resource</div><div class='del'>- * @iwdev: device pointer</div><div class='del'>- * @resource_array: resource bit array:</div><div class='del'>- * @max_resources: maximum resource number</div><div class='del'>- * @req_resources_num: Allocated resource number</div><div class='del'>- * @next: next free id</div><div class='del'>- **/</div><div class='del'>-static inline int i40iw_alloc_resource(struct i40iw_device *iwdev,</div><div class='del'>-				       unsigned long *resource_array,</div><div class='del'>-				       u32 max_resources,</div><div class='del'>-				       u32 *req_resource_num,</div><div class='del'>-				       u32 *next)</div><div class='del'>-{</div><div class='del'>-	u32 resource_num;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;iwdev-&gt;resource_lock, flags);</div><div class='del'>-	resource_num = find_next_zero_bit(resource_array, max_resources, *next);</div><div class='del'>-	if (resource_num &gt;= max_resources) {</div><div class='del'>-		resource_num = find_first_zero_bit(resource_array, max_resources);</div><div class='del'>-		if (resource_num &gt;= max_resources) {</div><div class='del'>-			spin_unlock_irqrestore(&amp;iwdev-&gt;resource_lock, flags);</div><div class='del'>-			return -EOVERFLOW;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	set_bit(resource_num, resource_array);</div><div class='del'>-	*next = resource_num + 1;</div><div class='del'>-	if (*next == max_resources)</div><div class='del'>-		*next = 0;</div><div class='del'>-	*req_resource_num = resource_num;</div><div class='del'>-	spin_unlock_irqrestore(&amp;iwdev-&gt;resource_lock, flags);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_is_resource_allocated - detrmine if resource is</div><div class='del'>- * allocated</div><div class='del'>- * @iwdev: device pointer</div><div class='del'>- * @resource_array: resource array for the resource_num</div><div class='del'>- * @resource_num: resource number to check</div><div class='del'>- **/</div><div class='del'>-static inline bool i40iw_is_resource_allocated(struct i40iw_device *iwdev,</div><div class='del'>-					       unsigned long *resource_array,</div><div class='del'>-					       u32 resource_num)</div><div class='del'>-{</div><div class='del'>-	bool bit_is_set;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;iwdev-&gt;resource_lock, flags);</div><div class='del'>-</div><div class='del'>-	bit_is_set = test_bit(resource_num, resource_array);</div><div class='del'>-	spin_unlock_irqrestore(&amp;iwdev-&gt;resource_lock, flags);</div><div class='del'>-</div><div class='del'>-	return bit_is_set;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_free_resource - free a resource</div><div class='del'>- * @iwdev: device pointer</div><div class='del'>- * @resource_array: resource array for the resource_num</div><div class='del'>- * @resource_num: resource number to free</div><div class='del'>- **/</div><div class='del'>-static inline void i40iw_free_resource(struct i40iw_device *iwdev,</div><div class='del'>-				       unsigned long *resource_array,</div><div class='del'>-				       u32 resource_num)</div><div class='del'>-{</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;iwdev-&gt;resource_lock, flags);</div><div class='del'>-	clear_bit(resource_num, resource_array);</div><div class='del'>-	spin_unlock_irqrestore(&amp;iwdev-&gt;resource_lock, flags);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * to_iwhdl - Get the handler from the device pointer</div><div class='del'>- * @iwdev: device pointer</div><div class='del'>- **/</div><div class='del'>-static inline struct i40iw_handler *to_iwhdl(struct i40iw_device *iw_dev)</div><div class='del'>-{</div><div class='del'>-	return container_of(iw_dev, struct i40iw_handler, device);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct i40iw_handler *i40iw_find_netdev(struct net_device *netdev);</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * iw_init_resources -</div><div class='del'>- */</div><div class='del'>-u32 i40iw_initialize_hw_resources(struct i40iw_device *iwdev);</div><div class='del'>-</div><div class='del'>-int i40iw_register_rdma_device(struct i40iw_device *iwdev);</div><div class='del'>-void i40iw_port_ibevent(struct i40iw_device *iwdev);</div><div class='del'>-void i40iw_cm_disconn(struct i40iw_qp *iwqp);</div><div class='del'>-void i40iw_cm_disconn_worker(void *);</div><div class='del'>-int mini_cm_recv_pkt(struct i40iw_cm_core *, struct i40iw_device *,</div><div class='del'>-		     struct sk_buff *);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_handle_cqp_op(struct i40iw_device *iwdev,</div><div class='del'>-					   struct i40iw_cqp_request *cqp_request);</div><div class='del'>-enum i40iw_status_code i40iw_add_mac_addr(struct i40iw_device *iwdev,</div><div class='del'>-					  u8 *mac_addr, u8 *mac_index);</div><div class='del'>-int i40iw_modify_qp(struct ib_qp *, struct ib_qp_attr *, int, struct ib_udata *);</div><div class='del'>-void i40iw_cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq);</div><div class='del'>-</div><div class='del'>-void i40iw_cleanup_pending_cqp_op(struct i40iw_device *iwdev);</div><div class='del'>-void i40iw_rem_pdusecount(struct i40iw_pd *iwpd, struct i40iw_device *iwdev);</div><div class='del'>-void i40iw_add_pdusecount(struct i40iw_pd *iwpd);</div><div class='del'>-void i40iw_rem_devusecount(struct i40iw_device *iwdev);</div><div class='del'>-void i40iw_add_devusecount(struct i40iw_device *iwdev);</div><div class='del'>-void i40iw_hw_modify_qp(struct i40iw_device *iwdev, struct i40iw_qp *iwqp,</div><div class='del'>-			struct i40iw_modify_qp_info *info, bool wait);</div><div class='del'>-</div><div class='del'>-void i40iw_qp_suspend_resume(struct i40iw_sc_dev *dev,</div><div class='del'>-			     struct i40iw_sc_qp *qp,</div><div class='del'>-			     bool suspend);</div><div class='del'>-enum i40iw_status_code i40iw_manage_qhash(struct i40iw_device *iwdev,</div><div class='del'>-					  struct i40iw_cm_info *cminfo,</div><div class='del'>-					  enum i40iw_quad_entry_type etype,</div><div class='del'>-					  enum i40iw_quad_hash_manage_type mtype,</div><div class='del'>-					  void *cmnode,</div><div class='del'>-					  bool wait);</div><div class='del'>-void i40iw_receive_ilq(struct i40iw_sc_vsi *vsi, struct i40iw_puda_buf *rbuf);</div><div class='del'>-void i40iw_free_sqbuf(struct i40iw_sc_vsi *vsi, void *bufp);</div><div class='del'>-void i40iw_free_qp_resources(struct i40iw_device *iwdev,</div><div class='del'>-			     struct i40iw_qp *iwqp,</div><div class='del'>-			     u32 qp_num);</div><div class='del'>-enum i40iw_status_code i40iw_obj_aligned_mem(struct i40iw_device *iwdev,</div><div class='del'>-					     struct i40iw_dma_mem *memptr,</div><div class='del'>-					     u32 size, u32 mask);</div><div class='del'>-</div><div class='del'>-void i40iw_request_reset(struct i40iw_device *iwdev);</div><div class='del'>-void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev);</div><div class='del'>-int i40iw_setup_cm_core(struct i40iw_device *iwdev);</div><div class='del'>-void i40iw_cleanup_cm_core(struct i40iw_cm_core *cm_core);</div><div class='del'>-void i40iw_process_ceq(struct i40iw_device *, struct i40iw_ceq *iwceq);</div><div class='del'>-void i40iw_process_aeq(struct i40iw_device *);</div><div class='del'>-void i40iw_next_iw_state(struct i40iw_qp *iwqp,</div><div class='del'>-			 u8 state, u8 del_hash,</div><div class='del'>-			 u8 term, u8 term_len);</div><div class='del'>-int i40iw_send_syn(struct i40iw_cm_node *cm_node, u32 sendack);</div><div class='del'>-int i40iw_send_reset(struct i40iw_cm_node *cm_node);</div><div class='del'>-struct i40iw_cm_node *i40iw_find_node(struct i40iw_cm_core *cm_core,</div><div class='del'>-				      u16 rem_port,</div><div class='del'>-				      u32 *rem_addr,</div><div class='del'>-				      u16 loc_port,</div><div class='del'>-				      u32 *loc_addr,</div><div class='del'>-				      bool add_refcnt,</div><div class='del'>-				      bool accelerated_list);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_hw_flush_wqes(struct i40iw_device *iwdev,</div><div class='del'>-					   struct i40iw_sc_qp *qp,</div><div class='del'>-					   struct i40iw_qp_flush_info *info,</div><div class='del'>-					   bool wait);</div><div class='del'>-</div><div class='del'>-void i40iw_gen_ae(struct i40iw_device *iwdev,</div><div class='del'>-		  struct i40iw_sc_qp *qp,</div><div class='del'>-		  struct i40iw_gen_ae_info *info,</div><div class='del'>-		  bool wait);</div><div class='del'>-</div><div class='del'>-void i40iw_copy_ip_ntohl(u32 *dst, __be32 *src);</div><div class='del'>-struct ib_mr *i40iw_reg_phys_mr(struct ib_pd *ib_pd,</div><div class='del'>-				u64 addr,</div><div class='del'>-				u64 size,</div><div class='del'>-				int acc,</div><div class='del'>-				u64 *iova_start);</div><div class='del'>-</div><div class='del'>-int i40iw_inetaddr_event(struct notifier_block *notifier,</div><div class='del'>-			 unsigned long event,</div><div class='del'>-			 void *ptr);</div><div class='del'>-int i40iw_inet6addr_event(struct notifier_block *notifier,</div><div class='del'>-			  unsigned long event,</div><div class='del'>-			  void *ptr);</div><div class='del'>-int i40iw_net_event(struct notifier_block *notifier,</div><div class='del'>-		    unsigned long event,</div><div class='del'>-		    void *ptr);</div><div class='del'>-int i40iw_netdevice_event(struct notifier_block *notifier,</div><div class='del'>-			  unsigned long event,</div><div class='del'>-			  void *ptr);</div><div class='del'>-</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.c b/drivers/infiniband/hw/i40iw/i40iw_cm.c<br/>deleted file mode 100644<br/>index 8233f5a4e623..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_cm.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_cm.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,4419 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#include &lt;linux/atomic.h&gt;</div><div class='del'>-#include &lt;linux/ip.h&gt;</div><div class='del'>-#include &lt;linux/tcp.h&gt;</div><div class='del'>-#include &lt;linux/init.h&gt;</div><div class='del'>-#include &lt;linux/if_arp.h&gt;</div><div class='del'>-#include &lt;linux/if_vlan.h&gt;</div><div class='del'>-#include &lt;linux/notifier.h&gt;</div><div class='del'>-#include &lt;linux/net.h&gt;</div><div class='del'>-#include &lt;linux/types.h&gt;</div><div class='del'>-#include &lt;linux/timer.h&gt;</div><div class='del'>-#include &lt;linux/time.h&gt;</div><div class='del'>-#include &lt;linux/delay.h&gt;</div><div class='del'>-#include &lt;linux/etherdevice.h&gt;</div><div class='del'>-#include &lt;linux/netdevice.h&gt;</div><div class='del'>-#include &lt;linux/random.h&gt;</div><div class='del'>-#include &lt;linux/list.h&gt;</div><div class='del'>-#include &lt;linux/threads.h&gt;</div><div class='del'>-#include &lt;linux/highmem.h&gt;</div><div class='del'>-#include &lt;net/arp.h&gt;</div><div class='del'>-#include &lt;net/ndisc.h&gt;</div><div class='del'>-#include &lt;net/neighbour.h&gt;</div><div class='del'>-#include &lt;net/route.h&gt;</div><div class='del'>-#include &lt;net/addrconf.h&gt;</div><div class='del'>-#include &lt;net/ip6_route.h&gt;</div><div class='del'>-#include &lt;net/ip_fib.h&gt;</div><div class='del'>-#include &lt;net/secure_seq.h&gt;</div><div class='del'>-#include &lt;net/tcp.h&gt;</div><div class='del'>-#include &lt;asm/checksum.h&gt;</div><div class='del'>-</div><div class='del'>-#include "i40iw.h"</div><div class='del'>-</div><div class='del'>-static void i40iw_rem_ref_cm_node(struct i40iw_cm_node *);</div><div class='del'>-static void i40iw_cm_post_event(struct i40iw_cm_event *event);</div><div class='del'>-static void i40iw_disconnect_worker(struct work_struct *work);</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_free_sqbuf - put back puda buffer if refcount = 0</div><div class='del'>- * @vsi: pointer to vsi structure</div><div class='del'>- * @buf: puda buffer to free</div><div class='del'>- */</div><div class='del'>-void i40iw_free_sqbuf(struct i40iw_sc_vsi *vsi, void *bufp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_buf *buf = (struct i40iw_puda_buf *)bufp;</div><div class='del'>-	struct i40iw_puda_rsrc *ilq = vsi-&gt;ilq;</div><div class='del'>-</div><div class='del'>-	if (!atomic_dec_return(&amp;buf-&gt;refcount))</div><div class='del'>-		i40iw_puda_ret_bufpool(ilq, buf);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_derive_hw_ird_setting - Calculate IRD</div><div class='del'>- *</div><div class='del'>- * @cm_ird: IRD of connection's node</div><div class='del'>- *</div><div class='del'>- * The ird from the connection is rounded to a supported HW</div><div class='del'>- * setting (2,8,32,64) and then encoded for ird_size field of</div><div class='del'>- * qp_ctx</div><div class='del'>- */</div><div class='del'>-static u8 i40iw_derive_hw_ird_setting(u16 cm_ird)</div><div class='del'>-{</div><div class='del'>-	u8 encoded_ird_size;</div><div class='del'>-</div><div class='del'>-	/* ird_size field is encoded in qp_ctx */</div><div class='del'>-	switch (cm_ird ? roundup_pow_of_two(cm_ird) : 0) {</div><div class='del'>-	case I40IW_HW_IRD_SETTING_64:</div><div class='del'>-		encoded_ird_size = 3;</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_HW_IRD_SETTING_32:</div><div class='del'>-	case I40IW_HW_IRD_SETTING_16:</div><div class='del'>-		encoded_ird_size = 2;</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_HW_IRD_SETTING_8:</div><div class='del'>-	case I40IW_HW_IRD_SETTING_4:</div><div class='del'>-		encoded_ird_size = 1;</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_HW_IRD_SETTING_2:</div><div class='del'>-	default:</div><div class='del'>-		encoded_ird_size = 0;</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	return encoded_ird_size;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_record_ird_ord - Record IRD/ORD passed in</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @conn_ird: connection IRD</div><div class='del'>- * @conn_ord: connection ORD</div><div class='del'>- */</div><div class='del'>-static void i40iw_record_ird_ord(struct i40iw_cm_node *cm_node, u32 conn_ird,</div><div class='del'>-				 u32 conn_ord)</div><div class='del'>-{</div><div class='del'>-	if (conn_ird &gt; I40IW_MAX_IRD_SIZE)</div><div class='del'>-		conn_ird = I40IW_MAX_IRD_SIZE;</div><div class='del'>-</div><div class='del'>-	if (conn_ord &gt; I40IW_MAX_ORD_SIZE)</div><div class='del'>-		conn_ord = I40IW_MAX_ORD_SIZE;</div><div class='del'>-	else if (!conn_ord &amp;&amp; cm_node-&gt;send_rdma0_op == SEND_RDMA_READ_ZERO)</div><div class='del'>-		conn_ord = 1;</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;ird_size = conn_ird;</div><div class='del'>-	cm_node-&gt;ord_size = conn_ord;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_copy_ip_ntohl - change network to host ip</div><div class='del'>- * @dst: host ip</div><div class='del'>- * @src: big endian</div><div class='del'>- */</div><div class='del'>-void i40iw_copy_ip_ntohl(u32 *dst, __be32 *src)</div><div class='del'>-{</div><div class='del'>-	*dst++ = ntohl(*src++);</div><div class='del'>-	*dst++ = ntohl(*src++);</div><div class='del'>-	*dst++ = ntohl(*src++);</div><div class='del'>-	*dst = ntohl(*src);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_copy_ip_htonl - change host addr to network ip</div><div class='del'>- * @dst: host ip</div><div class='del'>- * @src: little endian</div><div class='del'>- */</div><div class='del'>-static inline void i40iw_copy_ip_htonl(__be32 *dst, u32 *src)</div><div class='del'>-{</div><div class='del'>-	*dst++ = htonl(*src++);</div><div class='del'>-	*dst++ = htonl(*src++);</div><div class='del'>-	*dst++ = htonl(*src++);</div><div class='del'>-	*dst = htonl(*src);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_fill_sockaddr4 - get addr info for passive connection</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @event: upper layer's cm event</div><div class='del'>- */</div><div class='del'>-static inline void i40iw_fill_sockaddr4(struct i40iw_cm_node *cm_node,</div><div class='del'>-					struct iw_cm_event *event)</div><div class='del'>-{</div><div class='del'>-	struct sockaddr_in *laddr = (struct sockaddr_in *)&amp;event-&gt;local_addr;</div><div class='del'>-	struct sockaddr_in *raddr = (struct sockaddr_in *)&amp;event-&gt;remote_addr;</div><div class='del'>-</div><div class='del'>-	laddr-&gt;sin_family = AF_INET;</div><div class='del'>-	raddr-&gt;sin_family = AF_INET;</div><div class='del'>-</div><div class='del'>-	laddr-&gt;sin_port = htons(cm_node-&gt;loc_port);</div><div class='del'>-	raddr-&gt;sin_port = htons(cm_node-&gt;rem_port);</div><div class='del'>-</div><div class='del'>-	laddr-&gt;sin_addr.s_addr = htonl(cm_node-&gt;loc_addr[0]);</div><div class='del'>-	raddr-&gt;sin_addr.s_addr = htonl(cm_node-&gt;rem_addr[0]);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_fill_sockaddr6 - get ipv6 addr info for passive side</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @event: upper layer's cm event</div><div class='del'>- */</div><div class='del'>-static inline void i40iw_fill_sockaddr6(struct i40iw_cm_node *cm_node,</div><div class='del'>-					struct iw_cm_event *event)</div><div class='del'>-{</div><div class='del'>-	struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)&amp;event-&gt;local_addr;</div><div class='del'>-	struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *)&amp;event-&gt;remote_addr;</div><div class='del'>-</div><div class='del'>-	laddr6-&gt;sin6_family = AF_INET6;</div><div class='del'>-	raddr6-&gt;sin6_family = AF_INET6;</div><div class='del'>-</div><div class='del'>-	laddr6-&gt;sin6_port = htons(cm_node-&gt;loc_port);</div><div class='del'>-	raddr6-&gt;sin6_port = htons(cm_node-&gt;rem_port);</div><div class='del'>-</div><div class='del'>-	i40iw_copy_ip_htonl(laddr6-&gt;sin6_addr.in6_u.u6_addr32,</div><div class='del'>-			    cm_node-&gt;loc_addr);</div><div class='del'>-	i40iw_copy_ip_htonl(raddr6-&gt;sin6_addr.in6_u.u6_addr32,</div><div class='del'>-			    cm_node-&gt;rem_addr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_addr_info</div><div class='del'>- * @cm_node: contains ip/tcp info</div><div class='del'>- * @cm_info: to get a copy of the cm_node ip/tcp info</div><div class='del'>-*/</div><div class='del'>-static void i40iw_get_addr_info(struct i40iw_cm_node *cm_node,</div><div class='del'>-				struct i40iw_cm_info *cm_info)</div><div class='del'>-{</div><div class='del'>-	cm_info-&gt;ipv4 = cm_node-&gt;ipv4;</div><div class='del'>-	cm_info-&gt;vlan_id = cm_node-&gt;vlan_id;</div><div class='del'>-	memcpy(cm_info-&gt;loc_addr, cm_node-&gt;loc_addr, sizeof(cm_info-&gt;loc_addr));</div><div class='del'>-	memcpy(cm_info-&gt;rem_addr, cm_node-&gt;rem_addr, sizeof(cm_info-&gt;rem_addr));</div><div class='del'>-	cm_info-&gt;loc_port = cm_node-&gt;loc_port;</div><div class='del'>-	cm_info-&gt;rem_port = cm_node-&gt;rem_port;</div><div class='del'>-	cm_info-&gt;user_pri = cm_node-&gt;user_pri;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_cmevent_info - for cm event upcall</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @cm_id: upper layers cm struct for the event</div><div class='del'>- * @event: upper layer's cm event</div><div class='del'>- */</div><div class='del'>-static inline void i40iw_get_cmevent_info(struct i40iw_cm_node *cm_node,</div><div class='del'>-					  struct iw_cm_id *cm_id,</div><div class='del'>-					  struct iw_cm_event *event)</div><div class='del'>-{</div><div class='del'>-	memcpy(&amp;event-&gt;local_addr, &amp;cm_id-&gt;m_local_addr,</div><div class='del'>-	       sizeof(event-&gt;local_addr));</div><div class='del'>-	memcpy(&amp;event-&gt;remote_addr, &amp;cm_id-&gt;m_remote_addr,</div><div class='del'>-	       sizeof(event-&gt;remote_addr));</div><div class='del'>-	if (cm_node) {</div><div class='del'>-		event-&gt;private_data = (void *)cm_node-&gt;pdata_buf;</div><div class='del'>-		event-&gt;private_data_len = (u8)cm_node-&gt;pdata.size;</div><div class='del'>-		event-&gt;ird = cm_node-&gt;ird_size;</div><div class='del'>-		event-&gt;ord = cm_node-&gt;ord_size;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_send_cm_event - upcall cm's event handler</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @cm_id: upper layer's cm info struct</div><div class='del'>- * @type: Event type to indicate</div><div class='del'>- * @status: status for the event type</div><div class='del'>- */</div><div class='del'>-static int i40iw_send_cm_event(struct i40iw_cm_node *cm_node,</div><div class='del'>-			       struct iw_cm_id *cm_id,</div><div class='del'>-			       enum iw_cm_event_type type,</div><div class='del'>-			       int status)</div><div class='del'>-{</div><div class='del'>-	struct iw_cm_event event;</div><div class='del'>-</div><div class='del'>-	memset(&amp;event, 0, sizeof(event));</div><div class='del'>-	event.event = type;</div><div class='del'>-	event.status = status;</div><div class='del'>-	switch (type) {</div><div class='del'>-	case IW_CM_EVENT_CONNECT_REQUEST:</div><div class='del'>-		if (cm_node-&gt;ipv4)</div><div class='del'>-			i40iw_fill_sockaddr4(cm_node, &amp;event);</div><div class='del'>-		else</div><div class='del'>-			i40iw_fill_sockaddr6(cm_node, &amp;event);</div><div class='del'>-		event.provider_data = (void *)cm_node;</div><div class='del'>-		event.private_data = (void *)cm_node-&gt;pdata_buf;</div><div class='del'>-		event.private_data_len = (u8)cm_node-&gt;pdata.size;</div><div class='del'>-		event.ird = cm_node-&gt;ird_size;</div><div class='del'>-		break;</div><div class='del'>-	case IW_CM_EVENT_CONNECT_REPLY:</div><div class='del'>-		i40iw_get_cmevent_info(cm_node, cm_id, &amp;event);</div><div class='del'>-		break;</div><div class='del'>-	case IW_CM_EVENT_ESTABLISHED:</div><div class='del'>-		event.ird = cm_node-&gt;ird_size;</div><div class='del'>-		event.ord = cm_node-&gt;ord_size;</div><div class='del'>-		break;</div><div class='del'>-	case IW_CM_EVENT_DISCONNECT:</div><div class='del'>-		break;</div><div class='del'>-	case IW_CM_EVENT_CLOSE:</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		i40iw_pr_err("event type received type = %d\n", type);</div><div class='del'>-		return -1;</div><div class='del'>-	}</div><div class='del'>-	return cm_id-&gt;event_handler(cm_id, &amp;event);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_create_event - create cm event</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @type: Event type to generate</div><div class='del'>- */</div><div class='del'>-static struct i40iw_cm_event *i40iw_create_event(struct i40iw_cm_node *cm_node,</div><div class='del'>-						 enum i40iw_cm_event_type type)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_event *event;</div><div class='del'>-</div><div class='del'>-	if (!cm_node-&gt;cm_id)</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	event = kzalloc(sizeof(*event), GFP_ATOMIC);</div><div class='del'>-</div><div class='del'>-	if (!event)</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	event-&gt;type = type;</div><div class='del'>-	event-&gt;cm_node = cm_node;</div><div class='del'>-	memcpy(event-&gt;cm_info.rem_addr, cm_node-&gt;rem_addr, sizeof(event-&gt;cm_info.rem_addr));</div><div class='del'>-	memcpy(event-&gt;cm_info.loc_addr, cm_node-&gt;loc_addr, sizeof(event-&gt;cm_info.loc_addr));</div><div class='del'>-	event-&gt;cm_info.rem_port = cm_node-&gt;rem_port;</div><div class='del'>-	event-&gt;cm_info.loc_port = cm_node-&gt;loc_port;</div><div class='del'>-	event-&gt;cm_info.cm_id = cm_node-&gt;cm_id;</div><div class='del'>-</div><div class='del'>-	i40iw_debug(cm_node-&gt;dev,</div><div class='del'>-		    I40IW_DEBUG_CM,</div><div class='del'>-		    "node=%p event=%p type=%u dst=%pI4 src=%pI4\n",</div><div class='del'>-		    cm_node,</div><div class='del'>-		    event,</div><div class='del'>-		    type,</div><div class='del'>-		    event-&gt;cm_info.loc_addr,</div><div class='del'>-		    event-&gt;cm_info.rem_addr);</div><div class='del'>-</div><div class='del'>-	i40iw_cm_post_event(event);</div><div class='del'>-	return event;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_free_retrans_entry - free send entry</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- */</div><div class='del'>-static void i40iw_free_retrans_entry(struct i40iw_cm_node *cm_node)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = cm_node-&gt;iwdev;</div><div class='del'>-	struct i40iw_timer_entry *send_entry;</div><div class='del'>-</div><div class='del'>-	send_entry = cm_node-&gt;send_entry;</div><div class='del'>-	if (send_entry) {</div><div class='del'>-		cm_node-&gt;send_entry = NULL;</div><div class='del'>-		i40iw_free_sqbuf(&amp;iwdev-&gt;vsi, (void *)send_entry-&gt;sqbuf);</div><div class='del'>-		kfree(send_entry);</div><div class='del'>-		atomic_dec(&amp;cm_node-&gt;ref_count);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cleanup_retrans_entry - free send entry with lock</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- */</div><div class='del'>-static void i40iw_cleanup_retrans_entry(struct i40iw_cm_node *cm_node)</div><div class='del'>-{</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='del'>-	i40iw_free_retrans_entry(cm_node);</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_form_cm_frame - get a free packet and build frame</div><div class='del'>- * @cm_node: connection's node ionfo to use in frame</div><div class='del'>- * @options: pointer to options info</div><div class='del'>- * @hdr: pointer mpa header</div><div class='del'>- * @pdata: pointer to private data</div><div class='del'>- * @flags:  indicates FIN or ACK</div><div class='del'>- */</div><div class='del'>-static struct i40iw_puda_buf *i40iw_form_cm_frame(struct i40iw_cm_node *cm_node,</div><div class='del'>-						  struct i40iw_kmem_info *options,</div><div class='del'>-						  struct i40iw_kmem_info *hdr,</div><div class='del'>-						  struct i40iw_kmem_info *pdata,</div><div class='del'>-						  u8 flags)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_buf *sqbuf;</div><div class='del'>-	struct i40iw_sc_vsi *vsi = &amp;cm_node-&gt;iwdev-&gt;vsi;</div><div class='del'>-	u8 *buf;</div><div class='del'>-</div><div class='del'>-	struct tcphdr *tcph;</div><div class='del'>-	struct iphdr *iph;</div><div class='del'>-	struct ipv6hdr *ip6h;</div><div class='del'>-	struct ethhdr *ethh;</div><div class='del'>-	u16 packetsize;</div><div class='del'>-	u16 eth_hlen = ETH_HLEN;</div><div class='del'>-	u32 opts_len = 0;</div><div class='del'>-	u32 pd_len = 0;</div><div class='del'>-	u32 hdr_len = 0;</div><div class='del'>-	u16 vtag;</div><div class='del'>-</div><div class='del'>-	sqbuf = i40iw_puda_get_bufpool(vsi-&gt;ilq);</div><div class='del'>-	if (!sqbuf)</div><div class='del'>-		return NULL;</div><div class='del'>-	buf = sqbuf-&gt;mem.va;</div><div class='del'>-</div><div class='del'>-	if (options)</div><div class='del'>-		opts_len = (u32)options-&gt;size;</div><div class='del'>-</div><div class='del'>-	if (hdr)</div><div class='del'>-		hdr_len = hdr-&gt;size;</div><div class='del'>-</div><div class='del'>-	if (pdata)</div><div class='del'>-		pd_len = pdata-&gt;size;</div><div class='del'>-</div><div class='del'>-	if (cm_node-&gt;vlan_id &lt;= VLAN_VID_MASK)</div><div class='del'>-		eth_hlen += 4;</div><div class='del'>-</div><div class='del'>-	if (cm_node-&gt;ipv4)</div><div class='del'>-		packetsize = sizeof(*iph) + sizeof(*tcph);</div><div class='del'>-	else</div><div class='del'>-		packetsize = sizeof(*ip6h) + sizeof(*tcph);</div><div class='del'>-	packetsize += opts_len + hdr_len + pd_len;</div><div class='del'>-</div><div class='del'>-	memset(buf, 0x00, eth_hlen + packetsize);</div><div class='del'>-</div><div class='del'>-	sqbuf-&gt;totallen = packetsize + eth_hlen;</div><div class='del'>-	sqbuf-&gt;maclen = eth_hlen;</div><div class='del'>-	sqbuf-&gt;tcphlen = sizeof(*tcph) + opts_len;</div><div class='del'>-	sqbuf-&gt;scratch = (void *)cm_node;</div><div class='del'>-</div><div class='del'>-	ethh = (struct ethhdr *)buf;</div><div class='del'>-	buf += eth_hlen;</div><div class='del'>-</div><div class='del'>-	if (cm_node-&gt;ipv4) {</div><div class='del'>-		sqbuf-&gt;ipv4 = true;</div><div class='del'>-</div><div class='del'>-		iph = (struct iphdr *)buf;</div><div class='del'>-		buf += sizeof(*iph);</div><div class='del'>-		tcph = (struct tcphdr *)buf;</div><div class='del'>-		buf += sizeof(*tcph);</div><div class='del'>-</div><div class='del'>-		ether_addr_copy(ethh-&gt;h_dest, cm_node-&gt;rem_mac);</div><div class='del'>-		ether_addr_copy(ethh-&gt;h_source, cm_node-&gt;loc_mac);</div><div class='del'>-		if (cm_node-&gt;vlan_id &lt;= VLAN_VID_MASK) {</div><div class='del'>-			((struct vlan_ethhdr *)ethh)-&gt;h_vlan_proto = htons(ETH_P_8021Q);</div><div class='del'>-			vtag = (cm_node-&gt;user_pri &lt;&lt; VLAN_PRIO_SHIFT) | cm_node-&gt;vlan_id;</div><div class='del'>-			((struct vlan_ethhdr *)ethh)-&gt;h_vlan_TCI = htons(vtag);</div><div class='del'>-</div><div class='del'>-			((struct vlan_ethhdr *)ethh)-&gt;h_vlan_encapsulated_proto = htons(ETH_P_IP);</div><div class='del'>-		} else {</div><div class='del'>-			ethh-&gt;h_proto = htons(ETH_P_IP);</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		iph-&gt;version = IPVERSION;</div><div class='del'>-		iph-&gt;ihl = 5;	/* 5 * 4Byte words, IP headr len */</div><div class='del'>-		iph-&gt;tos = cm_node-&gt;tos;</div><div class='del'>-		iph-&gt;tot_len = htons(packetsize);</div><div class='del'>-		iph-&gt;id = htons(++cm_node-&gt;tcp_cntxt.loc_id);</div><div class='del'>-</div><div class='del'>-		iph-&gt;frag_off = htons(0x4000);</div><div class='del'>-		iph-&gt;ttl = 0x40;</div><div class='del'>-		iph-&gt;protocol = IPPROTO_TCP;</div><div class='del'>-		iph-&gt;saddr = htonl(cm_node-&gt;loc_addr[0]);</div><div class='del'>-		iph-&gt;daddr = htonl(cm_node-&gt;rem_addr[0]);</div><div class='del'>-	} else {</div><div class='del'>-		sqbuf-&gt;ipv4 = false;</div><div class='del'>-		ip6h = (struct ipv6hdr *)buf;</div><div class='del'>-		buf += sizeof(*ip6h);</div><div class='del'>-		tcph = (struct tcphdr *)buf;</div><div class='del'>-		buf += sizeof(*tcph);</div><div class='del'>-</div><div class='del'>-		ether_addr_copy(ethh-&gt;h_dest, cm_node-&gt;rem_mac);</div><div class='del'>-		ether_addr_copy(ethh-&gt;h_source, cm_node-&gt;loc_mac);</div><div class='del'>-		if (cm_node-&gt;vlan_id &lt;= VLAN_VID_MASK) {</div><div class='del'>-			((struct vlan_ethhdr *)ethh)-&gt;h_vlan_proto = htons(ETH_P_8021Q);</div><div class='del'>-			vtag = (cm_node-&gt;user_pri &lt;&lt; VLAN_PRIO_SHIFT) | cm_node-&gt;vlan_id;</div><div class='del'>-			((struct vlan_ethhdr *)ethh)-&gt;h_vlan_TCI = htons(vtag);</div><div class='del'>-			((struct vlan_ethhdr *)ethh)-&gt;h_vlan_encapsulated_proto = htons(ETH_P_IPV6);</div><div class='del'>-		} else {</div><div class='del'>-			ethh-&gt;h_proto = htons(ETH_P_IPV6);</div><div class='del'>-		}</div><div class='del'>-		ip6h-&gt;version = 6;</div><div class='del'>-		ip6h-&gt;priority = cm_node-&gt;tos &gt;&gt; 4;</div><div class='del'>-		ip6h-&gt;flow_lbl[0] = cm_node-&gt;tos &lt;&lt; 4;</div><div class='del'>-		ip6h-&gt;flow_lbl[1] = 0;</div><div class='del'>-		ip6h-&gt;flow_lbl[2] = 0;</div><div class='del'>-		ip6h-&gt;payload_len = htons(packetsize - sizeof(*ip6h));</div><div class='del'>-		ip6h-&gt;nexthdr = 6;</div><div class='del'>-		ip6h-&gt;hop_limit = 128;</div><div class='del'>-		i40iw_copy_ip_htonl(ip6h-&gt;saddr.in6_u.u6_addr32,</div><div class='del'>-				    cm_node-&gt;loc_addr);</div><div class='del'>-		i40iw_copy_ip_htonl(ip6h-&gt;daddr.in6_u.u6_addr32,</div><div class='del'>-				    cm_node-&gt;rem_addr);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	tcph-&gt;source = htons(cm_node-&gt;loc_port);</div><div class='del'>-	tcph-&gt;dest = htons(cm_node-&gt;rem_port);</div><div class='del'>-</div><div class='del'>-	tcph-&gt;seq = htonl(cm_node-&gt;tcp_cntxt.loc_seq_num);</div><div class='del'>-</div><div class='del'>-	if (flags &amp; SET_ACK) {</div><div class='del'>-		cm_node-&gt;tcp_cntxt.loc_ack_num = cm_node-&gt;tcp_cntxt.rcv_nxt;</div><div class='del'>-		tcph-&gt;ack_seq = htonl(cm_node-&gt;tcp_cntxt.loc_ack_num);</div><div class='del'>-		tcph-&gt;ack = 1;</div><div class='del'>-	} else {</div><div class='del'>-		tcph-&gt;ack_seq = 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (flags &amp; SET_SYN) {</div><div class='del'>-		cm_node-&gt;tcp_cntxt.loc_seq_num++;</div><div class='del'>-		tcph-&gt;syn = 1;</div><div class='del'>-	} else {</div><div class='del'>-		cm_node-&gt;tcp_cntxt.loc_seq_num += hdr_len + pd_len;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (flags &amp; SET_FIN) {</div><div class='del'>-		cm_node-&gt;tcp_cntxt.loc_seq_num++;</div><div class='del'>-		tcph-&gt;fin = 1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (flags &amp; SET_RST)</div><div class='del'>-		tcph-&gt;rst = 1;</div><div class='del'>-</div><div class='del'>-	tcph-&gt;doff = (u16)((sizeof(*tcph) + opts_len + 3) &gt;&gt; 2);</div><div class='del'>-	sqbuf-&gt;tcphlen = tcph-&gt;doff &lt;&lt; 2;</div><div class='del'>-	tcph-&gt;window = htons(cm_node-&gt;tcp_cntxt.rcv_wnd);</div><div class='del'>-	tcph-&gt;urg_ptr = 0;</div><div class='del'>-</div><div class='del'>-	if (opts_len) {</div><div class='del'>-		memcpy(buf, options-&gt;addr, opts_len);</div><div class='del'>-		buf += opts_len;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (hdr_len) {</div><div class='del'>-		memcpy(buf, hdr-&gt;addr, hdr_len);</div><div class='del'>-		buf += hdr_len;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (pdata &amp;&amp; pdata-&gt;addr)</div><div class='del'>-		memcpy(buf, pdata-&gt;addr, pdata-&gt;size);</div><div class='del'>-</div><div class='del'>-	atomic_set(&amp;sqbuf-&gt;refcount, 1);</div><div class='del'>-</div><div class='del'>-	return sqbuf;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_send_reset - Send RST packet</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- */</div><div class='del'>-int i40iw_send_reset(struct i40iw_cm_node *cm_node)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_buf *sqbuf;</div><div class='del'>-	int flags = SET_RST | SET_ACK;</div><div class='del'>-</div><div class='del'>-	sqbuf = i40iw_form_cm_frame(cm_node, NULL, NULL, NULL, flags);</div><div class='del'>-	if (!sqbuf) {</div><div class='del'>-		i40iw_pr_err("no sqbuf\n");</div><div class='del'>-		return -1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 0, 1);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_active_open_err - send event for active side cm error</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @reset: Flag to send reset or not</div><div class='del'>- */</div><div class='del'>-static void i40iw_active_open_err(struct i40iw_cm_node *cm_node, bool reset)</div><div class='del'>-{</div><div class='del'>-	i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-	cm_node-&gt;cm_core-&gt;stats_connect_errs++;</div><div class='del'>-	if (reset) {</div><div class='del'>-		i40iw_debug(cm_node-&gt;dev,</div><div class='del'>-			    I40IW_DEBUG_CM,</div><div class='del'>-			    "%s cm_node=%p state=%d\n",</div><div class='del'>-			    __func__,</div><div class='del'>-			    cm_node,</div><div class='del'>-			    cm_node-&gt;state);</div><div class='del'>-		atomic_inc(&amp;cm_node-&gt;ref_count);</div><div class='del'>-		i40iw_send_reset(cm_node);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;state = I40IW_CM_STATE_CLOSED;</div><div class='del'>-	i40iw_create_event(cm_node, I40IW_CM_EVENT_ABORTED);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_passive_open_err - handle passive side cm error</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @reset: send reset or just free cm_node</div><div class='del'>- */</div><div class='del'>-static void i40iw_passive_open_err(struct i40iw_cm_node *cm_node, bool reset)</div><div class='del'>-{</div><div class='del'>-	i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-	cm_node-&gt;cm_core-&gt;stats_passive_errs++;</div><div class='del'>-	cm_node-&gt;state = I40IW_CM_STATE_CLOSED;</div><div class='del'>-	i40iw_debug(cm_node-&gt;dev,</div><div class='del'>-		    I40IW_DEBUG_CM,</div><div class='del'>-		    "%s cm_node=%p state =%d\n",</div><div class='del'>-		    __func__,</div><div class='del'>-		    cm_node,</div><div class='del'>-		    cm_node-&gt;state);</div><div class='del'>-	if (reset)</div><div class='del'>-		i40iw_send_reset(cm_node);</div><div class='del'>-	else</div><div class='del'>-		i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_event_connect_error - to create connect error event</div><div class='del'>- * @event: cm information for connect event</div><div class='del'>- */</div><div class='del'>-static void i40iw_event_connect_error(struct i40iw_cm_event *event)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-	struct iw_cm_id *cm_id;</div><div class='del'>-</div><div class='del'>-	cm_id = event-&gt;cm_node-&gt;cm_id;</div><div class='del'>-	if (!cm_id)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	iwqp = cm_id-&gt;provider_data;</div><div class='del'>-</div><div class='del'>-	if (!iwqp || !iwqp-&gt;iwdev)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	iwqp-&gt;cm_id = NULL;</div><div class='del'>-	cm_id-&gt;provider_data = NULL;</div><div class='del'>-	i40iw_send_cm_event(event-&gt;cm_node, cm_id,</div><div class='del'>-			    IW_CM_EVENT_CONNECT_REPLY,</div><div class='del'>-			    -ECONNRESET);</div><div class='del'>-	cm_id-&gt;rem_ref(cm_id);</div><div class='del'>-	i40iw_rem_ref_cm_node(event-&gt;cm_node);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_process_options</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @optionsloc: point to start of options</div><div class='del'>- * @optionsize: size of all options</div><div class='del'>- * @syn_packet: flag if syn packet</div><div class='del'>- */</div><div class='del'>-static int i40iw_process_options(struct i40iw_cm_node *cm_node,</div><div class='del'>-				 u8 *optionsloc,</div><div class='del'>-				 u32 optionsize,</div><div class='del'>-				 u32 syn_packet)</div><div class='del'>-{</div><div class='del'>-	u32 tmp;</div><div class='del'>-	u32 offset = 0;</div><div class='del'>-	union all_known_options *all_options;</div><div class='del'>-	char got_mss_option = 0;</div><div class='del'>-</div><div class='del'>-	while (offset &lt; optionsize) {</div><div class='del'>-		all_options = (union all_known_options *)(optionsloc + offset);</div><div class='del'>-		switch (all_options-&gt;as_base.optionnum) {</div><div class='del'>-		case OPTION_NUMBER_END:</div><div class='del'>-			offset = optionsize;</div><div class='del'>-			break;</div><div class='del'>-		case OPTION_NUMBER_NONE:</div><div class='del'>-			offset += 1;</div><div class='del'>-			continue;</div><div class='del'>-		case OPTION_NUMBER_MSS:</div><div class='del'>-			i40iw_debug(cm_node-&gt;dev,</div><div class='del'>-				    I40IW_DEBUG_CM,</div><div class='del'>-				    "%s: MSS Length: %d Offset: %d Size: %d\n",</div><div class='del'>-				    __func__,</div><div class='del'>-				    all_options-&gt;as_mss.length,</div><div class='del'>-				    offset,</div><div class='del'>-				    optionsize);</div><div class='del'>-			got_mss_option = 1;</div><div class='del'>-			if (all_options-&gt;as_mss.length != 4)</div><div class='del'>-				return -1;</div><div class='del'>-			tmp = ntohs(all_options-&gt;as_mss.mss);</div><div class='del'>-			if (tmp &gt; 0 &amp;&amp; tmp &lt; cm_node-&gt;tcp_cntxt.mss)</div><div class='del'>-				cm_node-&gt;tcp_cntxt.mss = tmp;</div><div class='del'>-			break;</div><div class='del'>-		case OPTION_NUMBER_WINDOW_SCALE:</div><div class='del'>-			cm_node-&gt;tcp_cntxt.snd_wscale =</div><div class='del'>-			    all_options-&gt;as_windowscale.shiftcount;</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			i40iw_debug(cm_node-&gt;dev,</div><div class='del'>-				    I40IW_DEBUG_CM,</div><div class='del'>-				    "TCP Option not understood: %x\n",</div><div class='del'>-				    all_options-&gt;as_base.optionnum);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		offset += all_options-&gt;as_base.length;</div><div class='del'>-	}</div><div class='del'>-	if (!got_mss_option &amp;&amp; syn_packet)</div><div class='del'>-		cm_node-&gt;tcp_cntxt.mss = I40IW_CM_DEFAULT_MSS;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_handle_tcp_options -</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @tcph: pointer tcp header</div><div class='del'>- * @optionsize: size of options rcvd</div><div class='del'>- * @passive: active or passive flag</div><div class='del'>- */</div><div class='del'>-static int i40iw_handle_tcp_options(struct i40iw_cm_node *cm_node,</div><div class='del'>-				    struct tcphdr *tcph,</div><div class='del'>-				    int optionsize,</div><div class='del'>-				    int passive)</div><div class='del'>-{</div><div class='del'>-	u8 *optionsloc = (u8 *)&amp;tcph[1];</div><div class='del'>-</div><div class='del'>-	if (optionsize) {</div><div class='del'>-		if (i40iw_process_options(cm_node,</div><div class='del'>-					  optionsloc,</div><div class='del'>-					  optionsize,</div><div class='del'>-					  (u32)tcph-&gt;syn)) {</div><div class='del'>-			i40iw_debug(cm_node-&gt;dev,</div><div class='del'>-				    I40IW_DEBUG_CM,</div><div class='del'>-				    "%s: Node %p, Sending RESET\n",</div><div class='del'>-				    __func__,</div><div class='del'>-				    cm_node);</div><div class='del'>-			if (passive)</div><div class='del'>-				i40iw_passive_open_err(cm_node, true);</div><div class='del'>-			else</div><div class='del'>-				i40iw_active_open_err(cm_node, true);</div><div class='del'>-			return -1;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;tcp_cntxt.snd_wnd = ntohs(tcph-&gt;window) &lt;&lt;</div><div class='del'>-	    cm_node-&gt;tcp_cntxt.snd_wscale;</div><div class='del'>-</div><div class='del'>-	if (cm_node-&gt;tcp_cntxt.snd_wnd &gt; cm_node-&gt;tcp_cntxt.max_snd_wnd)</div><div class='del'>-		cm_node-&gt;tcp_cntxt.max_snd_wnd = cm_node-&gt;tcp_cntxt.snd_wnd;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_build_mpa_v1 - build a MPA V1 frame</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @mpa_key: to do read0 or write0</div><div class='del'>- */</div><div class='del'>-static void i40iw_build_mpa_v1(struct i40iw_cm_node *cm_node,</div><div class='del'>-			       void *start_addr,</div><div class='del'>-			       u8 mpa_key)</div><div class='del'>-{</div><div class='del'>-	struct ietf_mpa_v1 *mpa_frame = (struct ietf_mpa_v1 *)start_addr;</div><div class='del'>-</div><div class='del'>-	switch (mpa_key) {</div><div class='del'>-	case MPA_KEY_REQUEST:</div><div class='del'>-		memcpy(mpa_frame-&gt;key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE);</div><div class='del'>-		break;</div><div class='del'>-	case MPA_KEY_REPLY:</div><div class='del'>-		memcpy(mpa_frame-&gt;key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	mpa_frame-&gt;flags = IETF_MPA_FLAGS_CRC;</div><div class='del'>-	mpa_frame-&gt;rev = cm_node-&gt;mpa_frame_rev;</div><div class='del'>-	mpa_frame-&gt;priv_data_len = htons(cm_node-&gt;pdata.size);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_build_mpa_v2 - build a MPA V2 frame</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @start_addr: buffer start address</div><div class='del'>- * @mpa_key: to do read0 or write0</div><div class='del'>- */</div><div class='del'>-static void i40iw_build_mpa_v2(struct i40iw_cm_node *cm_node,</div><div class='del'>-			       void *start_addr,</div><div class='del'>-			       u8 mpa_key)</div><div class='del'>-{</div><div class='del'>-	struct ietf_mpa_v2 *mpa_frame = (struct ietf_mpa_v2 *)start_addr;</div><div class='del'>-	struct ietf_rtr_msg *rtr_msg = &amp;mpa_frame-&gt;rtr_msg;</div><div class='del'>-	u16 ctrl_ird, ctrl_ord;</div><div class='del'>-</div><div class='del'>-	/* initialize the upper 5 bytes of the frame */</div><div class='del'>-	i40iw_build_mpa_v1(cm_node, start_addr, mpa_key);</div><div class='del'>-	mpa_frame-&gt;flags |= IETF_MPA_V2_FLAG;</div><div class='del'>-	mpa_frame-&gt;priv_data_len += htons(IETF_RTR_MSG_SIZE);</div><div class='del'>-</div><div class='del'>-	/* initialize RTR msg */</div><div class='del'>-	if (cm_node-&gt;mpav2_ird_ord == IETF_NO_IRD_ORD) {</div><div class='del'>-		ctrl_ird = IETF_NO_IRD_ORD;</div><div class='del'>-		ctrl_ord = IETF_NO_IRD_ORD;</div><div class='del'>-	} else {</div><div class='del'>-		ctrl_ird = (cm_node-&gt;ird_size &gt; IETF_NO_IRD_ORD) ?</div><div class='del'>-			IETF_NO_IRD_ORD : cm_node-&gt;ird_size;</div><div class='del'>-		ctrl_ord = (cm_node-&gt;ord_size &gt; IETF_NO_IRD_ORD) ?</div><div class='del'>-			IETF_NO_IRD_ORD : cm_node-&gt;ord_size;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ctrl_ird |= IETF_PEER_TO_PEER;</div><div class='del'>-</div><div class='del'>-	switch (mpa_key) {</div><div class='del'>-	case MPA_KEY_REQUEST:</div><div class='del'>-		ctrl_ord |= IETF_RDMA0_WRITE;</div><div class='del'>-		ctrl_ord |= IETF_RDMA0_READ;</div><div class='del'>-		break;</div><div class='del'>-	case MPA_KEY_REPLY:</div><div class='del'>-		switch (cm_node-&gt;send_rdma0_op) {</div><div class='del'>-		case SEND_RDMA_WRITE_ZERO:</div><div class='del'>-			ctrl_ord |= IETF_RDMA0_WRITE;</div><div class='del'>-			break;</div><div class='del'>-		case SEND_RDMA_READ_ZERO:</div><div class='del'>-			ctrl_ord |= IETF_RDMA0_READ;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	rtr_msg-&gt;ctrl_ird = htons(ctrl_ird);</div><div class='del'>-	rtr_msg-&gt;ctrl_ord = htons(ctrl_ord);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cm_build_mpa_frame - build mpa frame for mpa version 1 or version 2</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @mpa: mpa: data buffer</div><div class='del'>- * @mpa_key: to do read0 or write0</div><div class='del'>- */</div><div class='del'>-static int i40iw_cm_build_mpa_frame(struct i40iw_cm_node *cm_node,</div><div class='del'>-				    struct i40iw_kmem_info *mpa,</div><div class='del'>-				    u8 mpa_key)</div><div class='del'>-{</div><div class='del'>-	int hdr_len = 0;</div><div class='del'>-</div><div class='del'>-	switch (cm_node-&gt;mpa_frame_rev) {</div><div class='del'>-	case IETF_MPA_V1:</div><div class='del'>-		hdr_len = sizeof(struct ietf_mpa_v1);</div><div class='del'>-		i40iw_build_mpa_v1(cm_node, mpa-&gt;addr, mpa_key);</div><div class='del'>-		break;</div><div class='del'>-	case IETF_MPA_V2:</div><div class='del'>-		hdr_len = sizeof(struct ietf_mpa_v2);</div><div class='del'>-		i40iw_build_mpa_v2(cm_node, mpa-&gt;addr, mpa_key);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return hdr_len;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_send_mpa_request - active node send mpa request to passive node</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- */</div><div class='del'>-static int i40iw_send_mpa_request(struct i40iw_cm_node *cm_node)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_buf *sqbuf;</div><div class='del'>-</div><div class='del'>-	if (!cm_node) {</div><div class='del'>-		i40iw_pr_err("cm_node == NULL\n");</div><div class='del'>-		return -1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;mpa_hdr.addr = &amp;cm_node-&gt;mpa_frame;</div><div class='del'>-	cm_node-&gt;mpa_hdr.size = i40iw_cm_build_mpa_frame(cm_node,</div><div class='del'>-							 &amp;cm_node-&gt;mpa_hdr,</div><div class='del'>-							 MPA_KEY_REQUEST);</div><div class='del'>-	if (!cm_node-&gt;mpa_hdr.size) {</div><div class='del'>-		i40iw_pr_err("mpa size = %d\n", cm_node-&gt;mpa_hdr.size);</div><div class='del'>-		return -1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	sqbuf = i40iw_form_cm_frame(cm_node,</div><div class='del'>-				    NULL,</div><div class='del'>-				    &amp;cm_node-&gt;mpa_hdr,</div><div class='del'>-				    &amp;cm_node-&gt;pdata,</div><div class='del'>-				    SET_ACK);</div><div class='del'>-	if (!sqbuf) {</div><div class='del'>-		i40iw_pr_err("sq_buf == NULL\n");</div><div class='del'>-		return -1;</div><div class='del'>-	}</div><div class='del'>-	return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 1, 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_send_mpa_reject -</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @pdata: reject data for connection</div><div class='del'>- * @plen: length of reject data</div><div class='del'>- */</div><div class='del'>-static int i40iw_send_mpa_reject(struct i40iw_cm_node *cm_node,</div><div class='del'>-				 const void *pdata,</div><div class='del'>-				 u8 plen)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_buf *sqbuf;</div><div class='del'>-	struct i40iw_kmem_info priv_info;</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;mpa_hdr.addr = &amp;cm_node-&gt;mpa_frame;</div><div class='del'>-	cm_node-&gt;mpa_hdr.size = i40iw_cm_build_mpa_frame(cm_node,</div><div class='del'>-							 &amp;cm_node-&gt;mpa_hdr,</div><div class='del'>-							 MPA_KEY_REPLY);</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;mpa_frame.flags |= IETF_MPA_FLAGS_REJECT;</div><div class='del'>-	priv_info.addr = (void *)pdata;</div><div class='del'>-	priv_info.size = plen;</div><div class='del'>-</div><div class='del'>-	sqbuf = i40iw_form_cm_frame(cm_node,</div><div class='del'>-				    NULL,</div><div class='del'>-				    &amp;cm_node-&gt;mpa_hdr,</div><div class='del'>-				    &amp;priv_info,</div><div class='del'>-				    SET_ACK | SET_FIN);</div><div class='del'>-	if (!sqbuf) {</div><div class='del'>-		i40iw_pr_err("no sqbuf\n");</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	cm_node-&gt;state = I40IW_CM_STATE_FIN_WAIT1;</div><div class='del'>-	return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 1, 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * recv_mpa - process an IETF MPA frame</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @buffer: Data pointer</div><div class='del'>- * @type: to return accept or reject</div><div class='del'>- * @len: Len of mpa buffer</div><div class='del'>- */</div><div class='del'>-static int i40iw_parse_mpa(struct i40iw_cm_node *cm_node, u8 *buffer, u32 *type, u32 len)</div><div class='del'>-{</div><div class='del'>-	struct ietf_mpa_v1 *mpa_frame;</div><div class='del'>-	struct ietf_mpa_v2 *mpa_v2_frame;</div><div class='del'>-	struct ietf_rtr_msg *rtr_msg;</div><div class='del'>-	int mpa_hdr_len;</div><div class='del'>-	int priv_data_len;</div><div class='del'>-</div><div class='del'>-	*type = I40IW_MPA_REQUEST_ACCEPT;</div><div class='del'>-</div><div class='del'>-	if (len &lt; sizeof(struct ietf_mpa_v1)) {</div><div class='del'>-		i40iw_pr_err("ietf buffer small (%x)\n", len);</div><div class='del'>-		return -1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mpa_frame = (struct ietf_mpa_v1 *)buffer;</div><div class='del'>-	mpa_hdr_len = sizeof(struct ietf_mpa_v1);</div><div class='del'>-	priv_data_len = ntohs(mpa_frame-&gt;priv_data_len);</div><div class='del'>-</div><div class='del'>-	if (priv_data_len &gt; IETF_MAX_PRIV_DATA_LEN) {</div><div class='del'>-		i40iw_pr_err("large pri_data %d\n", priv_data_len);</div><div class='del'>-		return -1;</div><div class='del'>-	}</div><div class='del'>-	if (mpa_frame-&gt;rev != IETF_MPA_V1 &amp;&amp; mpa_frame-&gt;rev != IETF_MPA_V2) {</div><div class='del'>-		i40iw_pr_err("unsupported mpa rev = %d\n", mpa_frame-&gt;rev);</div><div class='del'>-		return -1;</div><div class='del'>-	}</div><div class='del'>-	if (mpa_frame-&gt;rev &gt; cm_node-&gt;mpa_frame_rev) {</div><div class='del'>-		i40iw_pr_err("rev %d\n", mpa_frame-&gt;rev);</div><div class='del'>-		return -1;</div><div class='del'>-	}</div><div class='del'>-	cm_node-&gt;mpa_frame_rev = mpa_frame-&gt;rev;</div><div class='del'>-</div><div class='del'>-	if (cm_node-&gt;state != I40IW_CM_STATE_MPAREQ_SENT) {</div><div class='del'>-		if (memcmp(mpa_frame-&gt;key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE)) {</div><div class='del'>-			i40iw_pr_err("Unexpected MPA Key received\n");</div><div class='del'>-			return -1;</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		if (memcmp(mpa_frame-&gt;key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE)) {</div><div class='del'>-			i40iw_pr_err("Unexpected MPA Key received\n");</div><div class='del'>-			return -1;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (priv_data_len + mpa_hdr_len &gt; len) {</div><div class='del'>-		i40iw_pr_err("ietf buffer len(%x + %x != %x)\n",</div><div class='del'>-			     priv_data_len, mpa_hdr_len, len);</div><div class='del'>-		return -1;</div><div class='del'>-	}</div><div class='del'>-	if (len &gt; MAX_CM_BUFFER) {</div><div class='del'>-		i40iw_pr_err("ietf buffer large len = %d\n", len);</div><div class='del'>-		return -1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	switch (mpa_frame-&gt;rev) {</div><div class='del'>-	case IETF_MPA_V2:{</div><div class='del'>-			u16 ird_size;</div><div class='del'>-			u16 ord_size;</div><div class='del'>-			u16 ctrl_ord;</div><div class='del'>-			u16 ctrl_ird;</div><div class='del'>-</div><div class='del'>-			mpa_v2_frame = (struct ietf_mpa_v2 *)buffer;</div><div class='del'>-			mpa_hdr_len += IETF_RTR_MSG_SIZE;</div><div class='del'>-			rtr_msg = &amp;mpa_v2_frame-&gt;rtr_msg;</div><div class='del'>-</div><div class='del'>-			/* parse rtr message */</div><div class='del'>-			ctrl_ord = ntohs(rtr_msg-&gt;ctrl_ord);</div><div class='del'>-			ctrl_ird = ntohs(rtr_msg-&gt;ctrl_ird);</div><div class='del'>-			ird_size = ctrl_ird &amp; IETF_NO_IRD_ORD;</div><div class='del'>-			ord_size = ctrl_ord &amp; IETF_NO_IRD_ORD;</div><div class='del'>-</div><div class='del'>-			if (!(ctrl_ird &amp; IETF_PEER_TO_PEER))</div><div class='del'>-				return -1;</div><div class='del'>-</div><div class='del'>-			if (ird_size == IETF_NO_IRD_ORD || ord_size == IETF_NO_IRD_ORD) {</div><div class='del'>-				cm_node-&gt;mpav2_ird_ord = IETF_NO_IRD_ORD;</div><div class='del'>-				goto negotiate_done;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			if (cm_node-&gt;state != I40IW_CM_STATE_MPAREQ_SENT) {</div><div class='del'>-				/* responder */</div><div class='del'>-				if (!ord_size &amp;&amp; (ctrl_ord &amp; IETF_RDMA0_READ))</div><div class='del'>-					cm_node-&gt;ird_size = 1;</div><div class='del'>-				if (cm_node-&gt;ord_size &gt; ird_size)</div><div class='del'>-					cm_node-&gt;ord_size = ird_size;</div><div class='del'>-			} else {</div><div class='del'>-				/* initiator */</div><div class='del'>-				if (!ird_size &amp;&amp; (ctrl_ord &amp; IETF_RDMA0_READ))</div><div class='del'>-					return -1;</div><div class='del'>-				if (cm_node-&gt;ord_size &gt; ird_size)</div><div class='del'>-					cm_node-&gt;ord_size = ird_size;</div><div class='del'>-</div><div class='del'>-				if (cm_node-&gt;ird_size &lt; ord_size)</div><div class='del'>-					/* no resources available */</div><div class='del'>-					return -1;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-negotiate_done:</div><div class='del'>-			if (ctrl_ord &amp; IETF_RDMA0_READ)</div><div class='del'>-				cm_node-&gt;send_rdma0_op = SEND_RDMA_READ_ZERO;</div><div class='del'>-			else if (ctrl_ord &amp; IETF_RDMA0_WRITE)</div><div class='del'>-				cm_node-&gt;send_rdma0_op = SEND_RDMA_WRITE_ZERO;</div><div class='del'>-			else	/* Not supported RDMA0 operation */</div><div class='del'>-				return -1;</div><div class='del'>-			i40iw_debug(cm_node-&gt;dev, I40IW_DEBUG_CM,</div><div class='del'>-				    "MPAV2: Negotiated ORD: %d, IRD: %d\n",</div><div class='del'>-				    cm_node-&gt;ord_size, cm_node-&gt;ird_size);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case IETF_MPA_V1:</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	memcpy(cm_node-&gt;pdata_buf, buffer + mpa_hdr_len, priv_data_len);</div><div class='del'>-	cm_node-&gt;pdata.size = priv_data_len;</div><div class='del'>-</div><div class='del'>-	if (mpa_frame-&gt;flags &amp; IETF_MPA_FLAGS_REJECT)</div><div class='del'>-		*type = I40IW_MPA_REQUEST_REJECT;</div><div class='del'>-</div><div class='del'>-	if (mpa_frame-&gt;flags &amp; IETF_MPA_FLAGS_MARKERS)</div><div class='del'>-		cm_node-&gt;snd_mark_en = true;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_schedule_cm_timer</div><div class='del'>- * @@cm_node: connection's node</div><div class='del'>- * @sqbuf: buffer to send</div><div class='del'>- * @type: if it is send or close</div><div class='del'>- * @send_retrans: if rexmits to be done</div><div class='del'>- * @close_when_complete: is cm_node to be removed</div><div class='del'>- *</div><div class='del'>- * note - cm_node needs to be protected before calling this. Encase in:</div><div class='del'>- *		i40iw_rem_ref_cm_node(cm_core, cm_node);</div><div class='del'>- *		i40iw_schedule_cm_timer(...)</div><div class='del'>- *		atomic_inc(&amp;cm_node-&gt;ref_count);</div><div class='del'>- */</div><div class='del'>-int i40iw_schedule_cm_timer(struct i40iw_cm_node *cm_node,</div><div class='del'>-			    struct i40iw_puda_buf *sqbuf,</div><div class='del'>-			    enum i40iw_timer_type type,</div><div class='del'>-			    int send_retrans,</div><div class='del'>-			    int close_when_complete)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_vsi *vsi = &amp;cm_node-&gt;iwdev-&gt;vsi;</div><div class='del'>-	struct i40iw_cm_core *cm_core = cm_node-&gt;cm_core;</div><div class='del'>-	struct i40iw_timer_entry *new_send;</div><div class='del'>-	int ret = 0;</div><div class='del'>-	u32 was_timer_set;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC);</div><div class='del'>-	if (!new_send) {</div><div class='del'>-		if (type != I40IW_TIMER_TYPE_CLOSE)</div><div class='del'>-			i40iw_free_sqbuf(vsi, (void *)sqbuf);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-	new_send-&gt;retrycount = I40IW_DEFAULT_RETRYS;</div><div class='del'>-	new_send-&gt;retranscount = I40IW_DEFAULT_RETRANS;</div><div class='del'>-	new_send-&gt;sqbuf = sqbuf;</div><div class='del'>-	new_send-&gt;timetosend = jiffies;</div><div class='del'>-	new_send-&gt;type = type;</div><div class='del'>-	new_send-&gt;send_retrans = send_retrans;</div><div class='del'>-	new_send-&gt;close_when_complete = close_when_complete;</div><div class='del'>-</div><div class='del'>-	if (type == I40IW_TIMER_TYPE_CLOSE) {</div><div class='del'>-		new_send-&gt;timetosend += (HZ / 10);</div><div class='del'>-		if (cm_node-&gt;close_entry) {</div><div class='del'>-			kfree(new_send);</div><div class='del'>-			i40iw_pr_err("already close entry\n");</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-		cm_node-&gt;close_entry = new_send;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (type == I40IW_TIMER_TYPE_SEND) {</div><div class='del'>-		spin_lock_irqsave(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='del'>-		cm_node-&gt;send_entry = new_send;</div><div class='del'>-		atomic_inc(&amp;cm_node-&gt;ref_count);</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='del'>-		new_send-&gt;timetosend = jiffies + I40IW_RETRY_TIMEOUT;</div><div class='del'>-</div><div class='del'>-		atomic_inc(&amp;sqbuf-&gt;refcount);</div><div class='del'>-		i40iw_puda_send_buf(vsi-&gt;ilq, sqbuf);</div><div class='del'>-		if (!send_retrans) {</div><div class='del'>-			i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-			if (close_when_complete)</div><div class='del'>-				i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-			return ret;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-	was_timer_set = timer_pending(&amp;cm_core-&gt;tcp_timer);</div><div class='del'>-</div><div class='del'>-	if (!was_timer_set) {</div><div class='del'>-		cm_core-&gt;tcp_timer.expires = new_send-&gt;timetosend;</div><div class='del'>-		add_timer(&amp;cm_core-&gt;tcp_timer);</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_retrans_expired - Could not rexmit the packet</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- */</div><div class='del'>-static void i40iw_retrans_expired(struct i40iw_cm_node *cm_node)</div><div class='del'>-{</div><div class='del'>-	struct iw_cm_id *cm_id = cm_node-&gt;cm_id;</div><div class='del'>-	enum i40iw_cm_node_state state = cm_node-&gt;state;</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;state = I40IW_CM_STATE_CLOSED;</div><div class='del'>-	switch (state) {</div><div class='del'>-	case I40IW_CM_STATE_SYN_RCVD:</div><div class='del'>-	case I40IW_CM_STATE_CLOSING:</div><div class='del'>-		i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_FIN_WAIT1:</div><div class='del'>-	case I40IW_CM_STATE_LAST_ACK:</div><div class='del'>-		if (cm_node-&gt;cm_id)</div><div class='del'>-			cm_id-&gt;rem_ref(cm_id);</div><div class='del'>-		i40iw_send_reset(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		atomic_inc(&amp;cm_node-&gt;ref_count);</div><div class='del'>-		i40iw_send_reset(cm_node);</div><div class='del'>-		i40iw_create_event(cm_node, I40IW_CM_EVENT_ABORTED);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_handle_close_entry - for handling retry/timeouts</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @rem_node: flag for remove cm_node</div><div class='del'>- */</div><div class='del'>-static void i40iw_handle_close_entry(struct i40iw_cm_node *cm_node, u32 rem_node)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_timer_entry *close_entry = cm_node-&gt;close_entry;</div><div class='del'>-	struct iw_cm_id *cm_id = cm_node-&gt;cm_id;</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	if (!close_entry)</div><div class='del'>-		return;</div><div class='del'>-	iwqp = (struct i40iw_qp *)close_entry-&gt;sqbuf;</div><div class='del'>-	if (iwqp) {</div><div class='del'>-		spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-		if (iwqp-&gt;cm_id) {</div><div class='del'>-			iwqp-&gt;hw_tcp_state = I40IW_TCP_STATE_CLOSED;</div><div class='del'>-			iwqp-&gt;hw_iwarp_state = I40IW_QP_STATE_ERROR;</div><div class='del'>-			iwqp-&gt;last_aeq = I40IW_AE_RESET_SENT;</div><div class='del'>-			iwqp-&gt;ibqp_state = IB_QPS_ERR;</div><div class='del'>-			spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-			i40iw_cm_disconn(iwqp);</div><div class='del'>-		} else {</div><div class='del'>-			spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-		}</div><div class='del'>-	} else if (rem_node) {</div><div class='del'>-		/* TIME_WAIT state */</div><div class='del'>-		i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-	}</div><div class='del'>-	if (cm_id)</div><div class='del'>-		cm_id-&gt;rem_ref(cm_id);</div><div class='del'>-	kfree(close_entry);</div><div class='del'>-	cm_node-&gt;close_entry = NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_build_timer_list - Add cm_nodes to timer list</div><div class='del'>- * @timer_list: ptr to timer list</div><div class='del'>- * @hte: ptr to accelerated or non-accelerated list</div><div class='del'>- */</div><div class='del'>-static void i40iw_build_timer_list(struct list_head *timer_list,</div><div class='del'>-				   struct list_head *hte)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_node *cm_node;</div><div class='del'>-	struct list_head *list_core_temp, *list_node;</div><div class='del'>-</div><div class='del'>-	list_for_each_safe(list_node, list_core_temp, hte) {</div><div class='del'>-		cm_node = container_of(list_node, struct i40iw_cm_node, list);</div><div class='del'>-		if (cm_node-&gt;close_entry || cm_node-&gt;send_entry) {</div><div class='del'>-			atomic_inc(&amp;cm_node-&gt;ref_count);</div><div class='del'>-			list_add(&amp;cm_node-&gt;timer_entry, timer_list);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cm_timer_tick - system's timer expired callback</div><div class='del'>- * @pass: Pointing to cm_core</div><div class='del'>- */</div><div class='del'>-static void i40iw_cm_timer_tick(struct timer_list *t)</div><div class='del'>-{</div><div class='del'>-	unsigned long nexttimeout = jiffies + I40IW_LONG_TIME;</div><div class='del'>-	struct i40iw_cm_node *cm_node;</div><div class='del'>-	struct i40iw_timer_entry *send_entry, *close_entry;</div><div class='del'>-	struct list_head *list_core_temp;</div><div class='del'>-	struct i40iw_sc_vsi *vsi;</div><div class='del'>-	struct list_head *list_node;</div><div class='del'>-	struct i40iw_cm_core *cm_core = from_timer(cm_core, t, tcp_timer);</div><div class='del'>-	u32 settimer = 0;</div><div class='del'>-	unsigned long timetosend;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	struct list_head timer_list;</div><div class='del'>-</div><div class='del'>-	INIT_LIST_HEAD(&amp;timer_list);</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-	i40iw_build_timer_list(&amp;timer_list, &amp;cm_core-&gt;non_accelerated_list);</div><div class='del'>-	i40iw_build_timer_list(&amp;timer_list, &amp;cm_core-&gt;accelerated_list);</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-</div><div class='del'>-	list_for_each_safe(list_node, list_core_temp, &amp;timer_list) {</div><div class='del'>-		cm_node = container_of(list_node,</div><div class='del'>-				       struct i40iw_cm_node,</div><div class='del'>-				       timer_entry);</div><div class='del'>-		close_entry = cm_node-&gt;close_entry;</div><div class='del'>-</div><div class='del'>-		if (close_entry) {</div><div class='del'>-			if (time_after(close_entry-&gt;timetosend, jiffies)) {</div><div class='del'>-				if (nexttimeout &gt; close_entry-&gt;timetosend ||</div><div class='del'>-				    !settimer) {</div><div class='del'>-					nexttimeout = close_entry-&gt;timetosend;</div><div class='del'>-					settimer = 1;</div><div class='del'>-				}</div><div class='del'>-			} else {</div><div class='del'>-				i40iw_handle_close_entry(cm_node, 1);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		spin_lock_irqsave(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='del'>-</div><div class='del'>-		send_entry = cm_node-&gt;send_entry;</div><div class='del'>-		if (!send_entry)</div><div class='del'>-			goto done;</div><div class='del'>-		if (time_after(send_entry-&gt;timetosend, jiffies)) {</div><div class='del'>-			if (cm_node-&gt;state != I40IW_CM_STATE_OFFLOADED) {</div><div class='del'>-				if ((nexttimeout &gt; send_entry-&gt;timetosend) ||</div><div class='del'>-				    !settimer) {</div><div class='del'>-					nexttimeout = send_entry-&gt;timetosend;</div><div class='del'>-					settimer = 1;</div><div class='del'>-				}</div><div class='del'>-			} else {</div><div class='del'>-				i40iw_free_retrans_entry(cm_node);</div><div class='del'>-			}</div><div class='del'>-			goto done;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if ((cm_node-&gt;state == I40IW_CM_STATE_OFFLOADED) ||</div><div class='del'>-		    (cm_node-&gt;state == I40IW_CM_STATE_CLOSED)) {</div><div class='del'>-			i40iw_free_retrans_entry(cm_node);</div><div class='del'>-			goto done;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (!send_entry-&gt;retranscount || !send_entry-&gt;retrycount) {</div><div class='del'>-			i40iw_free_retrans_entry(cm_node);</div><div class='del'>-</div><div class='del'>-			spin_unlock_irqrestore(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='del'>-			i40iw_retrans_expired(cm_node);</div><div class='del'>-			cm_node-&gt;state = I40IW_CM_STATE_CLOSED;</div><div class='del'>-			spin_lock_irqsave(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='del'>-			goto done;</div><div class='del'>-		}</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='del'>-</div><div class='del'>-		vsi = &amp;cm_node-&gt;iwdev-&gt;vsi;</div><div class='del'>-</div><div class='del'>-		if (!cm_node-&gt;ack_rcvd) {</div><div class='del'>-			atomic_inc(&amp;send_entry-&gt;sqbuf-&gt;refcount);</div><div class='del'>-			i40iw_puda_send_buf(vsi-&gt;ilq, send_entry-&gt;sqbuf);</div><div class='del'>-			cm_node-&gt;cm_core-&gt;stats_pkt_retrans++;</div><div class='del'>-		}</div><div class='del'>-		spin_lock_irqsave(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='del'>-		if (send_entry-&gt;send_retrans) {</div><div class='del'>-			send_entry-&gt;retranscount--;</div><div class='del'>-			timetosend = (I40IW_RETRY_TIMEOUT &lt;&lt;</div><div class='del'>-				      (I40IW_DEFAULT_RETRANS -</div><div class='del'>-				       send_entry-&gt;retranscount));</div><div class='del'>-</div><div class='del'>-			send_entry-&gt;timetosend = jiffies +</div><div class='del'>-			    min(timetosend, I40IW_MAX_TIMEOUT);</div><div class='del'>-			if (nexttimeout &gt; send_entry-&gt;timetosend || !settimer) {</div><div class='del'>-				nexttimeout = send_entry-&gt;timetosend;</div><div class='del'>-				settimer = 1;</div><div class='del'>-			}</div><div class='del'>-		} else {</div><div class='del'>-			int close_when_complete;</div><div class='del'>-</div><div class='del'>-			close_when_complete = send_entry-&gt;close_when_complete;</div><div class='del'>-			i40iw_debug(cm_node-&gt;dev,</div><div class='del'>-				    I40IW_DEBUG_CM,</div><div class='del'>-				    "cm_node=%p state=%d\n",</div><div class='del'>-				    cm_node,</div><div class='del'>-				    cm_node-&gt;state);</div><div class='del'>-			i40iw_free_retrans_entry(cm_node);</div><div class='del'>-			if (close_when_complete)</div><div class='del'>-				i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-		}</div><div class='del'>-done:</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='del'>-		i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (settimer) {</div><div class='del'>-		spin_lock_irqsave(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-		if (!timer_pending(&amp;cm_core-&gt;tcp_timer)) {</div><div class='del'>-			cm_core-&gt;tcp_timer.expires = nexttimeout;</div><div class='del'>-			add_timer(&amp;cm_core-&gt;tcp_timer);</div><div class='del'>-		}</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_send_syn - send SYN packet</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @sendack: flag to set ACK bit or not</div><div class='del'>- */</div><div class='del'>-int i40iw_send_syn(struct i40iw_cm_node *cm_node, u32 sendack)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_buf *sqbuf;</div><div class='del'>-	int flags = SET_SYN;</div><div class='del'>-	char optionsbuffer[sizeof(struct option_mss) +</div><div class='del'>-			   sizeof(struct option_windowscale) +</div><div class='del'>-			   sizeof(struct option_base) + TCP_OPTIONS_PADDING];</div><div class='del'>-	struct i40iw_kmem_info opts;</div><div class='del'>-</div><div class='del'>-	int optionssize = 0;</div><div class='del'>-	/* Sending MSS option */</div><div class='del'>-	union all_known_options *options;</div><div class='del'>-</div><div class='del'>-	opts.addr = optionsbuffer;</div><div class='del'>-	if (!cm_node) {</div><div class='del'>-		i40iw_pr_err("no cm_node\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	options = (union all_known_options *)&amp;optionsbuffer[optionssize];</div><div class='del'>-	options-&gt;as_mss.optionnum = OPTION_NUMBER_MSS;</div><div class='del'>-	options-&gt;as_mss.length = sizeof(struct option_mss);</div><div class='del'>-	options-&gt;as_mss.mss = htons(cm_node-&gt;tcp_cntxt.mss);</div><div class='del'>-	optionssize += sizeof(struct option_mss);</div><div class='del'>-</div><div class='del'>-	options = (union all_known_options *)&amp;optionsbuffer[optionssize];</div><div class='del'>-	options-&gt;as_windowscale.optionnum = OPTION_NUMBER_WINDOW_SCALE;</div><div class='del'>-	options-&gt;as_windowscale.length = sizeof(struct option_windowscale);</div><div class='del'>-	options-&gt;as_windowscale.shiftcount = cm_node-&gt;tcp_cntxt.rcv_wscale;</div><div class='del'>-	optionssize += sizeof(struct option_windowscale);</div><div class='del'>-	options = (union all_known_options *)&amp;optionsbuffer[optionssize];</div><div class='del'>-	options-&gt;as_end = OPTION_NUMBER_END;</div><div class='del'>-	optionssize += 1;</div><div class='del'>-</div><div class='del'>-	if (sendack)</div><div class='del'>-		flags |= SET_ACK;</div><div class='del'>-</div><div class='del'>-	opts.size = optionssize;</div><div class='del'>-</div><div class='del'>-	sqbuf = i40iw_form_cm_frame(cm_node, &amp;opts, NULL, NULL, flags);</div><div class='del'>-	if (!sqbuf) {</div><div class='del'>-		i40iw_pr_err("no sqbuf\n");</div><div class='del'>-		return -1;</div><div class='del'>-	}</div><div class='del'>-	return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 1, 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_send_ack - Send ACK packet</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- */</div><div class='del'>-static void i40iw_send_ack(struct i40iw_cm_node *cm_node)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_buf *sqbuf;</div><div class='del'>-	struct i40iw_sc_vsi *vsi = &amp;cm_node-&gt;iwdev-&gt;vsi;</div><div class='del'>-</div><div class='del'>-	sqbuf = i40iw_form_cm_frame(cm_node, NULL, NULL, NULL, SET_ACK);</div><div class='del'>-	if (sqbuf)</div><div class='del'>-		i40iw_puda_send_buf(vsi-&gt;ilq, sqbuf);</div><div class='del'>-	else</div><div class='del'>-		i40iw_pr_err("no sqbuf\n");</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_send_fin - Send FIN pkt</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- */</div><div class='del'>-static int i40iw_send_fin(struct i40iw_cm_node *cm_node)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_buf *sqbuf;</div><div class='del'>-</div><div class='del'>-	sqbuf = i40iw_form_cm_frame(cm_node, NULL, NULL, NULL, SET_ACK | SET_FIN);</div><div class='del'>-	if (!sqbuf) {</div><div class='del'>-		i40iw_pr_err("no sqbuf\n");</div><div class='del'>-		return -1;</div><div class='del'>-	}</div><div class='del'>-	return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 1, 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_find_node - find a cm node that matches the reference cm node</div><div class='del'>- * @cm_core: cm's core</div><div class='del'>- * @rem_port: remote tcp port num</div><div class='del'>- * @rem_addr: remote ip addr</div><div class='del'>- * @loc_port: local tcp port num</div><div class='del'>- * @loc_addr: loc ip addr</div><div class='del'>- * @add_refcnt: flag to increment refcount of cm_node</div><div class='del'>- * @accelerated_list: flag for accelerated vs non-accelerated list to search</div><div class='del'>- */</div><div class='del'>-struct i40iw_cm_node *i40iw_find_node(struct i40iw_cm_core *cm_core,</div><div class='del'>-				      u16 rem_port,</div><div class='del'>-				      u32 *rem_addr,</div><div class='del'>-				      u16 loc_port,</div><div class='del'>-				      u32 *loc_addr,</div><div class='del'>-				      bool add_refcnt,</div><div class='del'>-				      bool accelerated_list)</div><div class='del'>-{</div><div class='del'>-	struct list_head *hte;</div><div class='del'>-	struct i40iw_cm_node *cm_node;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	hte = accelerated_list ?</div><div class='del'>-	      &amp;cm_core-&gt;accelerated_list : &amp;cm_core-&gt;non_accelerated_list;</div><div class='del'>-</div><div class='del'>-	/* walk list and find cm_node associated with this session ID */</div><div class='del'>-	spin_lock_irqsave(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-	list_for_each_entry(cm_node, hte, list) {</div><div class='del'>-		if (!memcmp(cm_node-&gt;loc_addr, loc_addr, sizeof(cm_node-&gt;loc_addr)) &amp;&amp;</div><div class='del'>-		    (cm_node-&gt;loc_port == loc_port) &amp;&amp;</div><div class='del'>-		    !memcmp(cm_node-&gt;rem_addr, rem_addr, sizeof(cm_node-&gt;rem_addr)) &amp;&amp;</div><div class='del'>-		    (cm_node-&gt;rem_port == rem_port)) {</div><div class='del'>-			if (add_refcnt)</div><div class='del'>-				atomic_inc(&amp;cm_node-&gt;ref_count);</div><div class='del'>-			spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-			return cm_node;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-</div><div class='del'>-	/* no owner node */</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_find_listener - find a cm node listening on this addr-port pair</div><div class='del'>- * @cm_core: cm's core</div><div class='del'>- * @dst_port: listener tcp port num</div><div class='del'>- * @dst_addr: listener ip addr</div><div class='del'>- * @listener_state: state to match with listen node's</div><div class='del'>- */</div><div class='del'>-static struct i40iw_cm_listener *i40iw_find_listener(</div><div class='del'>-						     struct i40iw_cm_core *cm_core,</div><div class='del'>-						     u32 *dst_addr,</div><div class='del'>-						     u16 dst_port,</div><div class='del'>-						     u16 vlan_id,</div><div class='del'>-						     enum i40iw_cm_listener_state</div><div class='del'>-						     listener_state)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_listener *listen_node;</div><div class='del'>-	static const u32 ip_zero[4] = { 0, 0, 0, 0 };</div><div class='del'>-	u32 listen_addr[4];</div><div class='del'>-	u16 listen_port;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	/* walk list and find cm_node associated with this session ID */</div><div class='del'>-	spin_lock_irqsave(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='del'>-	list_for_each_entry(listen_node, &amp;cm_core-&gt;listen_nodes, list) {</div><div class='del'>-		memcpy(listen_addr, listen_node-&gt;loc_addr, sizeof(listen_addr));</div><div class='del'>-		listen_port = listen_node-&gt;loc_port;</div><div class='del'>-		/* compare node pair, return node handle if a match */</div><div class='del'>-		if ((!memcmp(listen_addr, dst_addr, sizeof(listen_addr)) ||</div><div class='del'>-		     !memcmp(listen_addr, ip_zero, sizeof(listen_addr))) &amp;&amp;</div><div class='del'>-		     (listen_port == dst_port) &amp;&amp;</div><div class='del'>-		     (listener_state &amp; listen_node-&gt;listener_state)) {</div><div class='del'>-			atomic_inc(&amp;listen_node-&gt;ref_count);</div><div class='del'>-			spin_unlock_irqrestore(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='del'>-			return listen_node;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_add_hte_node - add a cm node to the hash table</div><div class='del'>- * @cm_core: cm's core</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- */</div><div class='del'>-static void i40iw_add_hte_node(struct i40iw_cm_core *cm_core,</div><div class='del'>-			       struct i40iw_cm_node *cm_node)</div><div class='del'>-{</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	if (!cm_node || !cm_core) {</div><div class='del'>-		i40iw_pr_err("cm_node or cm_core == NULL\n");</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-	list_add_tail(&amp;cm_node-&gt;list, &amp;cm_core-&gt;non_accelerated_list);</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_find_port - find port that matches reference port</div><div class='del'>- * @hte: ptr to accelerated or non-accelerated list</div><div class='del'>- * @accelerated_list: flag for accelerated vs non-accelerated list</div><div class='del'>- */</div><div class='del'>-static bool i40iw_find_port(struct list_head *hte, u16 port)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_node *cm_node;</div><div class='del'>-</div><div class='del'>-	list_for_each_entry(cm_node, hte, list) {</div><div class='del'>-		if (cm_node-&gt;loc_port == port)</div><div class='del'>-			return true;</div><div class='del'>-	}</div><div class='del'>-	return false;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_port_in_use - determine if port is in use</div><div class='del'>- * @cm_core: cm's core</div><div class='del'>- * @port: port number</div><div class='del'>- */</div><div class='del'>-bool i40iw_port_in_use(struct i40iw_cm_core *cm_core, u16 port)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_listener *listen_node;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-	if (i40iw_find_port(&amp;cm_core-&gt;accelerated_list, port) ||</div><div class='del'>-	    i40iw_find_port(&amp;cm_core-&gt;non_accelerated_list, port)) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-		return true;</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='del'>-	list_for_each_entry(listen_node, &amp;cm_core-&gt;listen_nodes, list) {</div><div class='del'>-		if (listen_node-&gt;loc_port == port) {</div><div class='del'>-			spin_unlock_irqrestore(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='del'>-			return true;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='del'>-</div><div class='del'>-	return false;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_del_multiple_qhash - Remove qhash and child listens</div><div class='del'>- * @iwdev: iWarp device</div><div class='del'>- * @cm_info: CM info for parent listen node</div><div class='del'>- * @cm_parent_listen_node: The parent listen node</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_del_multiple_qhash(</div><div class='del'>-						       struct i40iw_device *iwdev,</div><div class='del'>-						       struct i40iw_cm_info *cm_info,</div><div class='del'>-						       struct i40iw_cm_listener *cm_parent_listen_node)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_listener *child_listen_node;</div><div class='del'>-	enum i40iw_status_code ret = I40IW_ERR_CONFIG;</div><div class='del'>-	struct list_head *pos, *tpos;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;iwdev-&gt;cm_core.listen_list_lock, flags);</div><div class='del'>-	list_for_each_safe(pos, tpos, &amp;cm_parent_listen_node-&gt;child_listen_list) {</div><div class='del'>-		child_listen_node = list_entry(pos, struct i40iw_cm_listener, child_listen_list);</div><div class='del'>-		if (child_listen_node-&gt;ipv4)</div><div class='del'>-			i40iw_debug(&amp;iwdev-&gt;sc_dev,</div><div class='del'>-				    I40IW_DEBUG_CM,</div><div class='del'>-				    "removing child listen for IP=%pI4, port=%d, vlan=%d\n",</div><div class='del'>-				    child_listen_node-&gt;loc_addr,</div><div class='del'>-				    child_listen_node-&gt;loc_port,</div><div class='del'>-				    child_listen_node-&gt;vlan_id);</div><div class='del'>-		else</div><div class='del'>-			i40iw_debug(&amp;iwdev-&gt;sc_dev, I40IW_DEBUG_CM,</div><div class='del'>-				    "removing child listen for IP=%pI6, port=%d, vlan=%d\n",</div><div class='del'>-				    child_listen_node-&gt;loc_addr,</div><div class='del'>-				    child_listen_node-&gt;loc_port,</div><div class='del'>-				    child_listen_node-&gt;vlan_id);</div><div class='del'>-		list_del(pos);</div><div class='del'>-		memcpy(cm_info-&gt;loc_addr, child_listen_node-&gt;loc_addr,</div><div class='del'>-		       sizeof(cm_info-&gt;loc_addr));</div><div class='del'>-		cm_info-&gt;vlan_id = child_listen_node-&gt;vlan_id;</div><div class='del'>-		if (child_listen_node-&gt;qhash_set) {</div><div class='del'>-			ret = i40iw_manage_qhash(iwdev, cm_info,</div><div class='del'>-						 I40IW_QHASH_TYPE_TCP_SYN,</div><div class='del'>-						 I40IW_QHASH_MANAGE_TYPE_DELETE,</div><div class='del'>-						 NULL, false);</div><div class='del'>-			child_listen_node-&gt;qhash_set = false;</div><div class='del'>-		} else {</div><div class='del'>-			ret = I40IW_SUCCESS;</div><div class='del'>-		}</div><div class='del'>-		i40iw_debug(&amp;iwdev-&gt;sc_dev,</div><div class='del'>-			    I40IW_DEBUG_CM,</div><div class='del'>-			    "freed pointer = %p\n",</div><div class='del'>-			    child_listen_node);</div><div class='del'>-		kfree(child_listen_node);</div><div class='del'>-		cm_parent_listen_node-&gt;cm_core-&gt;stats_listen_nodes_destroyed++;</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;iwdev-&gt;cm_core.listen_list_lock, flags);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_netdev_vlan_ipv6 - Gets the netdev and vlan</div><div class='del'>- * @addr: local IPv6 address</div><div class='del'>- * @vlan_id: vlan id for the given IPv6 address</div><div class='del'>- *</div><div class='del'>- * Returns the net_device of the IPv6 address and also sets the</div><div class='del'>- * vlan id for that address.</div><div class='del'>- */</div><div class='del'>-static struct net_device *i40iw_netdev_vlan_ipv6(u32 *addr, u16 *vlan_id)</div><div class='del'>-{</div><div class='del'>-	struct net_device *ip_dev = NULL;</div><div class='del'>-	struct in6_addr laddr6;</div><div class='del'>-</div><div class='del'>-	if (!IS_ENABLED(CONFIG_IPV6))</div><div class='del'>-		return NULL;</div><div class='del'>-	i40iw_copy_ip_htonl(laddr6.in6_u.u6_addr32, addr);</div><div class='del'>-	if (vlan_id)</div><div class='del'>-		*vlan_id = I40IW_NO_VLAN;</div><div class='del'>-	rcu_read_lock();</div><div class='del'>-	for_each_netdev_rcu(&amp;init_net, ip_dev) {</div><div class='del'>-		if (ipv6_chk_addr(&amp;init_net, &amp;laddr6, ip_dev, 1)) {</div><div class='del'>-			if (vlan_id)</div><div class='del'>-				*vlan_id = rdma_vlan_dev_vlan_id(ip_dev);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	rcu_read_unlock();</div><div class='del'>-	return ip_dev;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_vlan_ipv4 - Returns the vlan_id for IPv4 address</div><div class='del'>- * @addr: local IPv4 address</div><div class='del'>- */</div><div class='del'>-static u16 i40iw_get_vlan_ipv4(u32 *addr)</div><div class='del'>-{</div><div class='del'>-	struct net_device *netdev;</div><div class='del'>-	u16 vlan_id = I40IW_NO_VLAN;</div><div class='del'>-</div><div class='del'>-	netdev = ip_dev_find(&amp;init_net, htonl(addr[0]));</div><div class='del'>-	if (netdev) {</div><div class='del'>-		vlan_id = rdma_vlan_dev_vlan_id(netdev);</div><div class='del'>-		dev_put(netdev);</div><div class='del'>-	}</div><div class='del'>-	return vlan_id;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_add_mqh_6 - Adds multiple qhashes for IPv6</div><div class='del'>- * @iwdev: iWarp device</div><div class='del'>- * @cm_info: CM info for parent listen node</div><div class='del'>- * @cm_parent_listen_node: The parent listen node</div><div class='del'>- *</div><div class='del'>- * Adds a qhash and a child listen node for every IPv6 address</div><div class='del'>- * on the adapter and adds the associated qhash filter</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_add_mqh_6(struct i40iw_device *iwdev,</div><div class='del'>-					      struct i40iw_cm_info *cm_info,</div><div class='del'>-					      struct i40iw_cm_listener *cm_parent_listen_node)</div><div class='del'>-{</div><div class='del'>-	struct net_device *ip_dev;</div><div class='del'>-	struct inet6_dev *idev;</div><div class='del'>-	struct inet6_ifaddr *ifp, *tmp;</div><div class='del'>-	enum i40iw_status_code ret = 0;</div><div class='del'>-	struct i40iw_cm_listener *child_listen_node;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	rtnl_lock();</div><div class='del'>-	for_each_netdev(&amp;init_net, ip_dev) {</div><div class='del'>-		if ((((rdma_vlan_dev_vlan_id(ip_dev) &lt; I40IW_NO_VLAN) &amp;&amp;</div><div class='del'>-		      (rdma_vlan_dev_real_dev(ip_dev) == iwdev-&gt;netdev)) ||</div><div class='del'>-		     (ip_dev == iwdev-&gt;netdev)) &amp;&amp; (ip_dev-&gt;flags &amp; IFF_UP)) {</div><div class='del'>-			idev = __in6_dev_get(ip_dev);</div><div class='del'>-			if (!idev) {</div><div class='del'>-				i40iw_pr_err("idev == NULL\n");</div><div class='del'>-				break;</div><div class='del'>-			}</div><div class='del'>-			list_for_each_entry_safe(ifp, tmp, &amp;idev-&gt;addr_list, if_list) {</div><div class='del'>-				i40iw_debug(&amp;iwdev-&gt;sc_dev,</div><div class='del'>-					    I40IW_DEBUG_CM,</div><div class='del'>-					    "IP=%pI6, vlan_id=%d, MAC=%pM\n",</div><div class='del'>-					    &amp;ifp-&gt;addr,</div><div class='del'>-					    rdma_vlan_dev_vlan_id(ip_dev),</div><div class='del'>-					    ip_dev-&gt;dev_addr);</div><div class='del'>-				child_listen_node =</div><div class='del'>-					kzalloc(sizeof(*child_listen_node), GFP_ATOMIC);</div><div class='del'>-				i40iw_debug(&amp;iwdev-&gt;sc_dev,</div><div class='del'>-					    I40IW_DEBUG_CM,</div><div class='del'>-					    "Allocating child listener %p\n",</div><div class='del'>-					    child_listen_node);</div><div class='del'>-				if (!child_listen_node) {</div><div class='del'>-					ret = I40IW_ERR_NO_MEMORY;</div><div class='del'>-					goto exit;</div><div class='del'>-				}</div><div class='del'>-				cm_info-&gt;vlan_id = rdma_vlan_dev_vlan_id(ip_dev);</div><div class='del'>-				cm_parent_listen_node-&gt;vlan_id = cm_info-&gt;vlan_id;</div><div class='del'>-</div><div class='del'>-				memcpy(child_listen_node, cm_parent_listen_node,</div><div class='del'>-				       sizeof(*child_listen_node));</div><div class='del'>-</div><div class='del'>-				i40iw_copy_ip_ntohl(child_listen_node-&gt;loc_addr,</div><div class='del'>-						    ifp-&gt;addr.in6_u.u6_addr32);</div><div class='del'>-				memcpy(cm_info-&gt;loc_addr, child_listen_node-&gt;loc_addr,</div><div class='del'>-				       sizeof(cm_info-&gt;loc_addr));</div><div class='del'>-</div><div class='del'>-				ret = i40iw_manage_qhash(iwdev, cm_info,</div><div class='del'>-							 I40IW_QHASH_TYPE_TCP_SYN,</div><div class='del'>-							 I40IW_QHASH_MANAGE_TYPE_ADD,</div><div class='del'>-							 NULL, true);</div><div class='del'>-				if (!ret) {</div><div class='del'>-					child_listen_node-&gt;qhash_set = true;</div><div class='del'>-					spin_lock_irqsave(&amp;iwdev-&gt;cm_core.listen_list_lock, flags);</div><div class='del'>-					list_add(&amp;child_listen_node-&gt;child_listen_list,</div><div class='del'>-						 &amp;cm_parent_listen_node-&gt;child_listen_list);</div><div class='del'>-					spin_unlock_irqrestore(&amp;iwdev-&gt;cm_core.listen_list_lock, flags);</div><div class='del'>-					cm_parent_listen_node-&gt;cm_core-&gt;stats_listen_nodes_created++;</div><div class='del'>-				} else {</div><div class='del'>-					kfree(child_listen_node);</div><div class='del'>-				}</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-exit:</div><div class='del'>-	rtnl_unlock();</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_add_mqh_4 - Adds multiple qhashes for IPv4</div><div class='del'>- * @iwdev: iWarp device</div><div class='del'>- * @cm_info: CM info for parent listen node</div><div class='del'>- * @cm_parent_listen_node: The parent listen node</div><div class='del'>- *</div><div class='del'>- * Adds a qhash and a child listen node for every IPv4 address</div><div class='del'>- * on the adapter and adds the associated qhash filter</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_add_mqh_4(</div><div class='del'>-				struct i40iw_device *iwdev,</div><div class='del'>-				struct i40iw_cm_info *cm_info,</div><div class='del'>-				struct i40iw_cm_listener *cm_parent_listen_node)</div><div class='del'>-{</div><div class='del'>-	struct net_device *dev;</div><div class='del'>-	struct in_device *idev;</div><div class='del'>-	struct i40iw_cm_listener *child_listen_node;</div><div class='del'>-	enum i40iw_status_code ret = 0;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	rtnl_lock();</div><div class='del'>-	for_each_netdev(&amp;init_net, dev) {</div><div class='del'>-		if ((((rdma_vlan_dev_vlan_id(dev) &lt; I40IW_NO_VLAN) &amp;&amp;</div><div class='del'>-		      (rdma_vlan_dev_real_dev(dev) == iwdev-&gt;netdev)) ||</div><div class='del'>-		    (dev == iwdev-&gt;netdev)) &amp;&amp; (dev-&gt;flags &amp; IFF_UP)) {</div><div class='del'>-			idev = in_dev_get(dev);</div><div class='del'>-			for_ifa(idev) {</div><div class='del'>-				i40iw_debug(&amp;iwdev-&gt;sc_dev,</div><div class='del'>-					    I40IW_DEBUG_CM,</div><div class='del'>-					    "Allocating child CM Listener forIP=%pI4, vlan_id=%d, MAC=%pM\n",</div><div class='del'>-					    &amp;ifa-&gt;ifa_address,</div><div class='del'>-					    rdma_vlan_dev_vlan_id(dev),</div><div class='del'>-					    dev-&gt;dev_addr);</div><div class='del'>-				child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL);</div><div class='del'>-				cm_parent_listen_node-&gt;cm_core-&gt;stats_listen_nodes_created++;</div><div class='del'>-				i40iw_debug(&amp;iwdev-&gt;sc_dev,</div><div class='del'>-					    I40IW_DEBUG_CM,</div><div class='del'>-					    "Allocating child listener %p\n",</div><div class='del'>-					    child_listen_node);</div><div class='del'>-				if (!child_listen_node) {</div><div class='del'>-					in_dev_put(idev);</div><div class='del'>-					ret = I40IW_ERR_NO_MEMORY;</div><div class='del'>-					goto exit;</div><div class='del'>-				}</div><div class='del'>-				cm_info-&gt;vlan_id = rdma_vlan_dev_vlan_id(dev);</div><div class='del'>-				cm_parent_listen_node-&gt;vlan_id = cm_info-&gt;vlan_id;</div><div class='del'>-				memcpy(child_listen_node,</div><div class='del'>-				       cm_parent_listen_node,</div><div class='del'>-				       sizeof(*child_listen_node));</div><div class='del'>-</div><div class='del'>-				child_listen_node-&gt;loc_addr[0] = ntohl(ifa-&gt;ifa_address);</div><div class='del'>-				memcpy(cm_info-&gt;loc_addr, child_listen_node-&gt;loc_addr,</div><div class='del'>-				       sizeof(cm_info-&gt;loc_addr));</div><div class='del'>-</div><div class='del'>-				ret = i40iw_manage_qhash(iwdev,</div><div class='del'>-							 cm_info,</div><div class='del'>-							 I40IW_QHASH_TYPE_TCP_SYN,</div><div class='del'>-							 I40IW_QHASH_MANAGE_TYPE_ADD,</div><div class='del'>-							 NULL,</div><div class='del'>-							 true);</div><div class='del'>-				if (!ret) {</div><div class='del'>-					child_listen_node-&gt;qhash_set = true;</div><div class='del'>-					spin_lock_irqsave(&amp;iwdev-&gt;cm_core.listen_list_lock, flags);</div><div class='del'>-					list_add(&amp;child_listen_node-&gt;child_listen_list,</div><div class='del'>-						 &amp;cm_parent_listen_node-&gt;child_listen_list);</div><div class='del'>-					spin_unlock_irqrestore(&amp;iwdev-&gt;cm_core.listen_list_lock, flags);</div><div class='del'>-				} else {</div><div class='del'>-					kfree(child_listen_node);</div><div class='del'>-					cm_parent_listen_node-&gt;cm_core-&gt;stats_listen_nodes_created--;</div><div class='del'>-				}</div><div class='del'>-			}</div><div class='del'>-			endfor_ifa(idev);</div><div class='del'>-			in_dev_put(idev);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-exit:</div><div class='del'>-	rtnl_unlock();</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_dec_refcnt_listen - delete listener and associated cm nodes</div><div class='del'>- * @cm_core: cm's core</div><div class='del'>- * @free_hanging_nodes: to free associated cm_nodes</div><div class='del'>- * @apbvt_del: flag to delete the apbvt</div><div class='del'>- */</div><div class='del'>-static int i40iw_dec_refcnt_listen(struct i40iw_cm_core *cm_core,</div><div class='del'>-				   struct i40iw_cm_listener *listener,</div><div class='del'>-				   int free_hanging_nodes, bool apbvt_del)</div><div class='del'>-{</div><div class='del'>-	int ret = -EINVAL;</div><div class='del'>-	int err = 0;</div><div class='del'>-	struct list_head *list_pos;</div><div class='del'>-	struct list_head *list_temp;</div><div class='del'>-	struct i40iw_cm_node *cm_node;</div><div class='del'>-	struct list_head reset_list;</div><div class='del'>-	struct i40iw_cm_info nfo;</div><div class='del'>-	struct i40iw_cm_node *loopback;</div><div class='del'>-	enum i40iw_cm_node_state old_state;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	/* free non-accelerated child nodes for this listener */</div><div class='del'>-	INIT_LIST_HEAD(&amp;reset_list);</div><div class='del'>-	if (free_hanging_nodes) {</div><div class='del'>-		spin_lock_irqsave(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-		list_for_each_safe(list_pos,</div><div class='del'>-				   list_temp, &amp;cm_core-&gt;non_accelerated_list) {</div><div class='del'>-			cm_node = container_of(list_pos, struct i40iw_cm_node, list);</div><div class='del'>-			if ((cm_node-&gt;listener == listener) &amp;&amp;</div><div class='del'>-			    !cm_node-&gt;accelerated) {</div><div class='del'>-				atomic_inc(&amp;cm_node-&gt;ref_count);</div><div class='del'>-				list_add(&amp;cm_node-&gt;reset_entry, &amp;reset_list);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	list_for_each_safe(list_pos, list_temp, &amp;reset_list) {</div><div class='del'>-		cm_node = container_of(list_pos, struct i40iw_cm_node, reset_entry);</div><div class='del'>-		loopback = cm_node-&gt;loopbackpartner;</div><div class='del'>-		if (cm_node-&gt;state &gt;= I40IW_CM_STATE_FIN_WAIT1) {</div><div class='del'>-			i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-		} else {</div><div class='del'>-			if (!loopback) {</div><div class='del'>-				i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-				err = i40iw_send_reset(cm_node);</div><div class='del'>-				if (err) {</div><div class='del'>-					cm_node-&gt;state = I40IW_CM_STATE_CLOSED;</div><div class='del'>-					i40iw_pr_err("send reset\n");</div><div class='del'>-				} else {</div><div class='del'>-					old_state = cm_node-&gt;state;</div><div class='del'>-					cm_node-&gt;state = I40IW_CM_STATE_LISTENER_DESTROYED;</div><div class='del'>-					if (old_state != I40IW_CM_STATE_MPAREQ_RCVD)</div><div class='del'>-						i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-				}</div><div class='del'>-			} else {</div><div class='del'>-				struct i40iw_cm_event event;</div><div class='del'>-</div><div class='del'>-				event.cm_node = loopback;</div><div class='del'>-				memcpy(event.cm_info.rem_addr,</div><div class='del'>-				       loopback-&gt;rem_addr, sizeof(event.cm_info.rem_addr));</div><div class='del'>-				memcpy(event.cm_info.loc_addr,</div><div class='del'>-				       loopback-&gt;loc_addr, sizeof(event.cm_info.loc_addr));</div><div class='del'>-				event.cm_info.rem_port = loopback-&gt;rem_port;</div><div class='del'>-				event.cm_info.loc_port = loopback-&gt;loc_port;</div><div class='del'>-				event.cm_info.cm_id = loopback-&gt;cm_id;</div><div class='del'>-				event.cm_info.ipv4 = loopback-&gt;ipv4;</div><div class='del'>-				atomic_inc(&amp;loopback-&gt;ref_count);</div><div class='del'>-				loopback-&gt;state = I40IW_CM_STATE_CLOSED;</div><div class='del'>-				i40iw_event_connect_error(&amp;event);</div><div class='del'>-				cm_node-&gt;state = I40IW_CM_STATE_LISTENER_DESTROYED;</div><div class='del'>-				i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!atomic_dec_return(&amp;listener-&gt;ref_count)) {</div><div class='del'>-		spin_lock_irqsave(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='del'>-		list_del(&amp;listener-&gt;list);</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='del'>-</div><div class='del'>-		if (listener-&gt;iwdev) {</div><div class='del'>-			if (apbvt_del)</div><div class='del'>-				i40iw_manage_apbvt(listener-&gt;iwdev,</div><div class='del'>-						   listener-&gt;loc_port,</div><div class='del'>-						   I40IW_MANAGE_APBVT_DEL);</div><div class='del'>-</div><div class='del'>-			memcpy(nfo.loc_addr, listener-&gt;loc_addr, sizeof(nfo.loc_addr));</div><div class='del'>-			nfo.loc_port = listener-&gt;loc_port;</div><div class='del'>-			nfo.ipv4 = listener-&gt;ipv4;</div><div class='del'>-			nfo.vlan_id = listener-&gt;vlan_id;</div><div class='del'>-			nfo.user_pri = listener-&gt;user_pri;</div><div class='del'>-</div><div class='del'>-			if (!list_empty(&amp;listener-&gt;child_listen_list)) {</div><div class='del'>-				i40iw_del_multiple_qhash(listener-&gt;iwdev, &amp;nfo, listener);</div><div class='del'>-			} else {</div><div class='del'>-				if (listener-&gt;qhash_set)</div><div class='del'>-					i40iw_manage_qhash(listener-&gt;iwdev,</div><div class='del'>-							   &amp;nfo,</div><div class='del'>-							   I40IW_QHASH_TYPE_TCP_SYN,</div><div class='del'>-							   I40IW_QHASH_MANAGE_TYPE_DELETE,</div><div class='del'>-							   NULL,</div><div class='del'>-							   false);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		cm_core-&gt;stats_listen_destroyed++;</div><div class='del'>-		kfree(listener);</div><div class='del'>-		cm_core-&gt;stats_listen_nodes_destroyed++;</div><div class='del'>-		listener = NULL;</div><div class='del'>-		ret = 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (listener) {</div><div class='del'>-		if (atomic_read(&amp;listener-&gt;pend_accepts_cnt) &gt; 0)</div><div class='del'>-			i40iw_debug(cm_core-&gt;dev,</div><div class='del'>-				    I40IW_DEBUG_CM,</div><div class='del'>-				    "%s: listener (%p) pending accepts=%u\n",</div><div class='del'>-				    __func__,</div><div class='del'>-				    listener,</div><div class='del'>-				    atomic_read(&amp;listener-&gt;pend_accepts_cnt));</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cm_del_listen - delete a linstener</div><div class='del'>- * @cm_core: cm's core</div><div class='del'>-  * @listener: passive connection's listener</div><div class='del'>- * @apbvt_del: flag to delete apbvt</div><div class='del'>- */</div><div class='del'>-static int i40iw_cm_del_listen(struct i40iw_cm_core *cm_core,</div><div class='del'>-			       struct i40iw_cm_listener *listener,</div><div class='del'>-			       bool apbvt_del)</div><div class='del'>-{</div><div class='del'>-	listener-&gt;listener_state = I40IW_CM_LISTENER_PASSIVE_STATE;</div><div class='del'>-	listener-&gt;cm_id = NULL;	/* going to be destroyed pretty soon */</div><div class='del'>-	return i40iw_dec_refcnt_listen(cm_core, listener, 1, apbvt_del);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_addr_resolve_neigh - resolve neighbor address</div><div class='del'>- * @iwdev: iwarp device structure</div><div class='del'>- * @src_ip: local ip address</div><div class='del'>- * @dst_ip: remote ip address</div><div class='del'>- * @arpindex: if there is an arp entry</div><div class='del'>- */</div><div class='del'>-static int i40iw_addr_resolve_neigh(struct i40iw_device *iwdev,</div><div class='del'>-				    u32 src_ip,</div><div class='del'>-				    u32 dst_ip,</div><div class='del'>-				    int arpindex)</div><div class='del'>-{</div><div class='del'>-	struct rtable *rt;</div><div class='del'>-	struct neighbour *neigh;</div><div class='del'>-	int rc = arpindex;</div><div class='del'>-	struct net_device *netdev = iwdev-&gt;netdev;</div><div class='del'>-	__be32 dst_ipaddr = htonl(dst_ip);</div><div class='del'>-	__be32 src_ipaddr = htonl(src_ip);</div><div class='del'>-</div><div class='del'>-	rt = ip_route_output(&amp;init_net, dst_ipaddr, src_ipaddr, 0, 0);</div><div class='del'>-	if (IS_ERR(rt)) {</div><div class='del'>-		i40iw_pr_err("ip_route_output\n");</div><div class='del'>-		return rc;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (netif_is_bond_slave(netdev))</div><div class='del'>-		netdev = netdev_master_upper_dev_get(netdev);</div><div class='del'>-</div><div class='del'>-	neigh = dst_neigh_lookup(&amp;rt-&gt;dst, &amp;dst_ipaddr);</div><div class='del'>-</div><div class='del'>-	rcu_read_lock();</div><div class='del'>-	if (neigh) {</div><div class='del'>-		if (neigh-&gt;nud_state &amp; NUD_VALID) {</div><div class='del'>-			if (arpindex &gt;= 0) {</div><div class='del'>-				if (ether_addr_equal(iwdev-&gt;arp_table[arpindex].mac_addr,</div><div class='del'>-						     neigh-&gt;ha))</div><div class='del'>-					/* Mac address same as arp table */</div><div class='del'>-					goto resolve_neigh_exit;</div><div class='del'>-				i40iw_manage_arp_cache(iwdev,</div><div class='del'>-						       iwdev-&gt;arp_table[arpindex].mac_addr,</div><div class='del'>-						       &amp;dst_ip,</div><div class='del'>-						       true,</div><div class='del'>-						       I40IW_ARP_DELETE);</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			i40iw_manage_arp_cache(iwdev, neigh-&gt;ha, &amp;dst_ip, true, I40IW_ARP_ADD);</div><div class='del'>-			rc = i40iw_arp_table(iwdev, &amp;dst_ip, true, NULL, I40IW_ARP_RESOLVE);</div><div class='del'>-		} else {</div><div class='del'>-			neigh_event_send(neigh, NULL);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>- resolve_neigh_exit:</div><div class='del'>-</div><div class='del'>-	rcu_read_unlock();</div><div class='del'>-	if (neigh)</div><div class='del'>-		neigh_release(neigh);</div><div class='del'>-</div><div class='del'>-	ip_rt_put(rt);</div><div class='del'>-	return rc;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_dst_ipv6</div><div class='del'>- */</div><div class='del'>-static struct dst_entry *i40iw_get_dst_ipv6(struct sockaddr_in6 *src_addr,</div><div class='del'>-					    struct sockaddr_in6 *dst_addr)</div><div class='del'>-{</div><div class='del'>-	struct dst_entry *dst;</div><div class='del'>-	struct flowi6 fl6;</div><div class='del'>-</div><div class='del'>-	memset(&amp;fl6, 0, sizeof(fl6));</div><div class='del'>-	fl6.daddr = dst_addr-&gt;sin6_addr;</div><div class='del'>-	fl6.saddr = src_addr-&gt;sin6_addr;</div><div class='del'>-	if (ipv6_addr_type(&amp;fl6.daddr) &amp; IPV6_ADDR_LINKLOCAL)</div><div class='del'>-		fl6.flowi6_oif = dst_addr-&gt;sin6_scope_id;</div><div class='del'>-</div><div class='del'>-	dst = ip6_route_output(&amp;init_net, NULL, &amp;fl6);</div><div class='del'>-	return dst;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_addr_resolve_neigh_ipv6 - resolve neighbor ipv6 address</div><div class='del'>- * @iwdev: iwarp device structure</div><div class='del'>- * @dst_ip: remote ip address</div><div class='del'>- * @arpindex: if there is an arp entry</div><div class='del'>- */</div><div class='del'>-static int i40iw_addr_resolve_neigh_ipv6(struct i40iw_device *iwdev,</div><div class='del'>-					 u32 *src,</div><div class='del'>-					 u32 *dest,</div><div class='del'>-					 int arpindex)</div><div class='del'>-{</div><div class='del'>-	struct neighbour *neigh;</div><div class='del'>-	int rc = arpindex;</div><div class='del'>-	struct net_device *netdev = iwdev-&gt;netdev;</div><div class='del'>-	struct dst_entry *dst;</div><div class='del'>-	struct sockaddr_in6 dst_addr;</div><div class='del'>-	struct sockaddr_in6 src_addr;</div><div class='del'>-</div><div class='del'>-	memset(&amp;dst_addr, 0, sizeof(dst_addr));</div><div class='del'>-	dst_addr.sin6_family = AF_INET6;</div><div class='del'>-	i40iw_copy_ip_htonl(dst_addr.sin6_addr.in6_u.u6_addr32, dest);</div><div class='del'>-	memset(&amp;src_addr, 0, sizeof(src_addr));</div><div class='del'>-	src_addr.sin6_family = AF_INET6;</div><div class='del'>-	i40iw_copy_ip_htonl(src_addr.sin6_addr.in6_u.u6_addr32, src);</div><div class='del'>-	dst = i40iw_get_dst_ipv6(&amp;src_addr, &amp;dst_addr);</div><div class='del'>-	if (!dst || dst-&gt;error) {</div><div class='del'>-		if (dst) {</div><div class='del'>-			dst_release(dst);</div><div class='del'>-			i40iw_pr_err("ip6_route_output returned dst-&gt;error = %d\n",</div><div class='del'>-				     dst-&gt;error);</div><div class='del'>-		}</div><div class='del'>-		return rc;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (netif_is_bond_slave(netdev))</div><div class='del'>-		netdev = netdev_master_upper_dev_get(netdev);</div><div class='del'>-</div><div class='del'>-	neigh = dst_neigh_lookup(dst, dst_addr.sin6_addr.in6_u.u6_addr32);</div><div class='del'>-</div><div class='del'>-	rcu_read_lock();</div><div class='del'>-	if (neigh) {</div><div class='del'>-		i40iw_debug(&amp;iwdev-&gt;sc_dev, I40IW_DEBUG_CM, "dst_neigh_lookup MAC=%pM\n", neigh-&gt;ha);</div><div class='del'>-		if (neigh-&gt;nud_state &amp; NUD_VALID) {</div><div class='del'>-			if (arpindex &gt;= 0) {</div><div class='del'>-				if (ether_addr_equal</div><div class='del'>-				    (iwdev-&gt;arp_table[arpindex].mac_addr,</div><div class='del'>-				     neigh-&gt;ha)) {</div><div class='del'>-					/* Mac address same as in arp table */</div><div class='del'>-					goto resolve_neigh_exit6;</div><div class='del'>-				}</div><div class='del'>-				i40iw_manage_arp_cache(iwdev,</div><div class='del'>-						       iwdev-&gt;arp_table[arpindex].mac_addr,</div><div class='del'>-						       dest,</div><div class='del'>-						       false,</div><div class='del'>-						       I40IW_ARP_DELETE);</div><div class='del'>-			}</div><div class='del'>-			i40iw_manage_arp_cache(iwdev,</div><div class='del'>-					       neigh-&gt;ha,</div><div class='del'>-					       dest,</div><div class='del'>-					       false,</div><div class='del'>-					       I40IW_ARP_ADD);</div><div class='del'>-			rc = i40iw_arp_table(iwdev,</div><div class='del'>-					     dest,</div><div class='del'>-					     false,</div><div class='del'>-					     NULL,</div><div class='del'>-					     I40IW_ARP_RESOLVE);</div><div class='del'>-		} else {</div><div class='del'>-			neigh_event_send(neigh, NULL);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>- resolve_neigh_exit6:</div><div class='del'>-	rcu_read_unlock();</div><div class='del'>-	if (neigh)</div><div class='del'>-		neigh_release(neigh);</div><div class='del'>-	dst_release(dst);</div><div class='del'>-	return rc;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ipv4_is_loopback - check if loopback</div><div class='del'>- * @loc_addr: local addr to compare</div><div class='del'>- * @rem_addr: remote address</div><div class='del'>- */</div><div class='del'>-static bool i40iw_ipv4_is_loopback(u32 loc_addr, u32 rem_addr)</div><div class='del'>-{</div><div class='del'>-	return ipv4_is_loopback(htonl(rem_addr)) || (loc_addr == rem_addr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ipv6_is_loopback - check if loopback</div><div class='del'>- * @loc_addr: local addr to compare</div><div class='del'>- * @rem_addr: remote address</div><div class='del'>- */</div><div class='del'>-static bool i40iw_ipv6_is_loopback(u32 *loc_addr, u32 *rem_addr)</div><div class='del'>-{</div><div class='del'>-	struct in6_addr raddr6;</div><div class='del'>-</div><div class='del'>-	i40iw_copy_ip_htonl(raddr6.in6_u.u6_addr32, rem_addr);</div><div class='del'>-	return !memcmp(loc_addr, rem_addr, 16) || ipv6_addr_loopback(&amp;raddr6);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_make_cm_node - create a new instance of a cm node</div><div class='del'>- * @cm_core: cm's core</div><div class='del'>- * @iwdev: iwarp device structure</div><div class='del'>- * @cm_info: quad info for connection</div><div class='del'>- * @listener: passive connection's listener</div><div class='del'>- */</div><div class='del'>-static struct i40iw_cm_node *i40iw_make_cm_node(</div><div class='del'>-				   struct i40iw_cm_core *cm_core,</div><div class='del'>-				   struct i40iw_device *iwdev,</div><div class='del'>-				   struct i40iw_cm_info *cm_info,</div><div class='del'>-				   struct i40iw_cm_listener *listener)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_node *cm_node;</div><div class='del'>-	int oldarpindex;</div><div class='del'>-	int arpindex;</div><div class='del'>-	struct net_device *netdev = iwdev-&gt;netdev;</div><div class='del'>-</div><div class='del'>-	/* create an hte and cm_node for this instance */</div><div class='del'>-	cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC);</div><div class='del'>-	if (!cm_node)</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	/* set our node specific transport info */</div><div class='del'>-	cm_node-&gt;ipv4 = cm_info-&gt;ipv4;</div><div class='del'>-	cm_node-&gt;vlan_id = cm_info-&gt;vlan_id;</div><div class='del'>-	if ((cm_node-&gt;vlan_id == I40IW_NO_VLAN) &amp;&amp; iwdev-&gt;dcb)</div><div class='del'>-		cm_node-&gt;vlan_id = 0;</div><div class='del'>-	cm_node-&gt;tos = cm_info-&gt;tos;</div><div class='del'>-	cm_node-&gt;user_pri = cm_info-&gt;user_pri;</div><div class='del'>-	if (listener) {</div><div class='del'>-		if (listener-&gt;tos != cm_info-&gt;tos)</div><div class='del'>-			i40iw_debug(&amp;iwdev-&gt;sc_dev, I40IW_DEBUG_DCB,</div><div class='del'>-				    "application TOS[%d] and remote client TOS[%d] mismatch\n",</div><div class='del'>-				     listener-&gt;tos, cm_info-&gt;tos);</div><div class='del'>-		cm_node-&gt;tos = max(listener-&gt;tos, cm_info-&gt;tos);</div><div class='del'>-		cm_node-&gt;user_pri = rt_tos2priority(cm_node-&gt;tos);</div><div class='del'>-		i40iw_debug(&amp;iwdev-&gt;sc_dev, I40IW_DEBUG_DCB, "listener: TOS:[%d] UP:[%d]\n",</div><div class='del'>-			    cm_node-&gt;tos, cm_node-&gt;user_pri);</div><div class='del'>-	}</div><div class='del'>-	memcpy(cm_node-&gt;loc_addr, cm_info-&gt;loc_addr, sizeof(cm_node-&gt;loc_addr));</div><div class='del'>-	memcpy(cm_node-&gt;rem_addr, cm_info-&gt;rem_addr, sizeof(cm_node-&gt;rem_addr));</div><div class='del'>-	cm_node-&gt;loc_port = cm_info-&gt;loc_port;</div><div class='del'>-	cm_node-&gt;rem_port = cm_info-&gt;rem_port;</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;mpa_frame_rev = iwdev-&gt;mpa_version;</div><div class='del'>-	cm_node-&gt;send_rdma0_op = SEND_RDMA_READ_ZERO;</div><div class='del'>-	cm_node-&gt;ird_size = I40IW_MAX_IRD_SIZE;</div><div class='del'>-	cm_node-&gt;ord_size = I40IW_MAX_ORD_SIZE;</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;listener = listener;</div><div class='del'>-	cm_node-&gt;cm_id = cm_info-&gt;cm_id;</div><div class='del'>-	ether_addr_copy(cm_node-&gt;loc_mac, netdev-&gt;dev_addr);</div><div class='del'>-	spin_lock_init(&amp;cm_node-&gt;retrans_list_lock);</div><div class='del'>-	cm_node-&gt;ack_rcvd = false;</div><div class='del'>-</div><div class='del'>-	atomic_set(&amp;cm_node-&gt;ref_count, 1);</div><div class='del'>-	/* associate our parent CM core */</div><div class='del'>-	cm_node-&gt;cm_core = cm_core;</div><div class='del'>-	cm_node-&gt;tcp_cntxt.loc_id = I40IW_CM_DEF_LOCAL_ID;</div><div class='del'>-	cm_node-&gt;tcp_cntxt.rcv_wscale = I40IW_CM_DEFAULT_RCV_WND_SCALE;</div><div class='del'>-	cm_node-&gt;tcp_cntxt.rcv_wnd =</div><div class='del'>-			I40IW_CM_DEFAULT_RCV_WND_SCALED &gt;&gt; I40IW_CM_DEFAULT_RCV_WND_SCALE;</div><div class='del'>-	if (cm_node-&gt;ipv4) {</div><div class='del'>-		cm_node-&gt;tcp_cntxt.loc_seq_num = secure_tcp_seq(htonl(cm_node-&gt;loc_addr[0]),</div><div class='del'>-							htonl(cm_node-&gt;rem_addr[0]),</div><div class='del'>-							htons(cm_node-&gt;loc_port),</div><div class='del'>-							htons(cm_node-&gt;rem_port));</div><div class='del'>-		cm_node-&gt;tcp_cntxt.mss = iwdev-&gt;vsi.mtu - I40IW_MTU_TO_MSS_IPV4;</div><div class='del'>-	} else if (IS_ENABLED(CONFIG_IPV6)) {</div><div class='del'>-		__be32 loc[4] = {</div><div class='del'>-			htonl(cm_node-&gt;loc_addr[0]), htonl(cm_node-&gt;loc_addr[1]),</div><div class='del'>-			htonl(cm_node-&gt;loc_addr[2]), htonl(cm_node-&gt;loc_addr[3])</div><div class='del'>-		};</div><div class='del'>-		__be32 rem[4] = {</div><div class='del'>-			htonl(cm_node-&gt;rem_addr[0]), htonl(cm_node-&gt;rem_addr[1]),</div><div class='del'>-			htonl(cm_node-&gt;rem_addr[2]), htonl(cm_node-&gt;rem_addr[3])</div><div class='del'>-		};</div><div class='del'>-		cm_node-&gt;tcp_cntxt.loc_seq_num = secure_tcpv6_seq(loc, rem,</div><div class='del'>-							htons(cm_node-&gt;loc_port),</div><div class='del'>-							htons(cm_node-&gt;rem_port));</div><div class='del'>-		cm_node-&gt;tcp_cntxt.mss = iwdev-&gt;vsi.mtu - I40IW_MTU_TO_MSS_IPV6;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;iwdev = iwdev;</div><div class='del'>-	cm_node-&gt;dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-</div><div class='del'>-	if ((cm_node-&gt;ipv4 &amp;&amp;</div><div class='del'>-	     i40iw_ipv4_is_loopback(cm_node-&gt;loc_addr[0], cm_node-&gt;rem_addr[0])) ||</div><div class='del'>-	     (!cm_node-&gt;ipv4 &amp;&amp; i40iw_ipv6_is_loopback(cm_node-&gt;loc_addr,</div><div class='del'>-						       cm_node-&gt;rem_addr))) {</div><div class='del'>-		arpindex = i40iw_arp_table(iwdev,</div><div class='del'>-					   cm_node-&gt;rem_addr,</div><div class='del'>-					   false,</div><div class='del'>-					   NULL,</div><div class='del'>-					   I40IW_ARP_RESOLVE);</div><div class='del'>-	} else {</div><div class='del'>-		oldarpindex = i40iw_arp_table(iwdev,</div><div class='del'>-					      cm_node-&gt;rem_addr,</div><div class='del'>-					      false,</div><div class='del'>-					      NULL,</div><div class='del'>-					      I40IW_ARP_RESOLVE);</div><div class='del'>-		if (cm_node-&gt;ipv4)</div><div class='del'>-			arpindex = i40iw_addr_resolve_neigh(iwdev,</div><div class='del'>-							    cm_info-&gt;loc_addr[0],</div><div class='del'>-							    cm_info-&gt;rem_addr[0],</div><div class='del'>-							    oldarpindex);</div><div class='del'>-		else if (IS_ENABLED(CONFIG_IPV6))</div><div class='del'>-			arpindex = i40iw_addr_resolve_neigh_ipv6(iwdev,</div><div class='del'>-								 cm_info-&gt;loc_addr,</div><div class='del'>-								 cm_info-&gt;rem_addr,</div><div class='del'>-								 oldarpindex);</div><div class='del'>-		else</div><div class='del'>-			arpindex = -EINVAL;</div><div class='del'>-	}</div><div class='del'>-	if (arpindex &lt; 0) {</div><div class='del'>-		i40iw_pr_err("cm_node arpindex\n");</div><div class='del'>-		kfree(cm_node);</div><div class='del'>-		return NULL;</div><div class='del'>-	}</div><div class='del'>-	ether_addr_copy(cm_node-&gt;rem_mac, iwdev-&gt;arp_table[arpindex].mac_addr);</div><div class='del'>-	i40iw_add_hte_node(cm_core, cm_node);</div><div class='del'>-	cm_core-&gt;stats_nodes_created++;</div><div class='del'>-	return cm_node;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_rem_ref_cm_node - destroy an instance of a cm node</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- */</div><div class='del'>-static void i40iw_rem_ref_cm_node(struct i40iw_cm_node *cm_node)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_core *cm_core = cm_node-&gt;cm_core;</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-	struct i40iw_cm_info nfo;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;cm_node-&gt;cm_core-&gt;ht_lock, flags);</div><div class='del'>-	if (atomic_dec_return(&amp;cm_node-&gt;ref_count)) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_node-&gt;cm_core-&gt;ht_lock, flags);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-	list_del(&amp;cm_node-&gt;list);</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm_node-&gt;cm_core-&gt;ht_lock, flags);</div><div class='del'>-</div><div class='del'>-	/* if the node is destroyed before connection was accelerated */</div><div class='del'>-	if (!cm_node-&gt;accelerated &amp;&amp; cm_node-&gt;accept_pend) {</div><div class='del'>-		pr_err("node destroyed before established\n");</div><div class='del'>-		atomic_dec(&amp;cm_node-&gt;listener-&gt;pend_accepts_cnt);</div><div class='del'>-	}</div><div class='del'>-	if (cm_node-&gt;close_entry)</div><div class='del'>-		i40iw_handle_close_entry(cm_node, 0);</div><div class='del'>-	if (cm_node-&gt;listener) {</div><div class='del'>-		i40iw_dec_refcnt_listen(cm_core, cm_node-&gt;listener, 0, true);</div><div class='del'>-	} else {</div><div class='del'>-		if (cm_node-&gt;apbvt_set) {</div><div class='del'>-			i40iw_manage_apbvt(cm_node-&gt;iwdev,</div><div class='del'>-					   cm_node-&gt;loc_port,</div><div class='del'>-					   I40IW_MANAGE_APBVT_DEL);</div><div class='del'>-			cm_node-&gt;apbvt_set = 0;</div><div class='del'>-		}</div><div class='del'>-		i40iw_get_addr_info(cm_node, &amp;nfo);</div><div class='del'>-		if (cm_node-&gt;qhash_set) {</div><div class='del'>-			i40iw_manage_qhash(cm_node-&gt;iwdev,</div><div class='del'>-					   &amp;nfo,</div><div class='del'>-					   I40IW_QHASH_TYPE_TCP_ESTABLISHED,</div><div class='del'>-					   I40IW_QHASH_MANAGE_TYPE_DELETE,</div><div class='del'>-					   NULL,</div><div class='del'>-					   false);</div><div class='del'>-			cm_node-&gt;qhash_set = 0;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	iwqp = cm_node-&gt;iwqp;</div><div class='del'>-	if (iwqp) {</div><div class='del'>-		iwqp-&gt;cm_node = NULL;</div><div class='del'>-		i40iw_rem_ref(&amp;iwqp-&gt;ibqp);</div><div class='del'>-		cm_node-&gt;iwqp = NULL;</div><div class='del'>-	} else if (cm_node-&gt;qhash_set) {</div><div class='del'>-		i40iw_get_addr_info(cm_node, &amp;nfo);</div><div class='del'>-		i40iw_manage_qhash(cm_node-&gt;iwdev,</div><div class='del'>-				   &amp;nfo,</div><div class='del'>-				   I40IW_QHASH_TYPE_TCP_ESTABLISHED,</div><div class='del'>-				   I40IW_QHASH_MANAGE_TYPE_DELETE,</div><div class='del'>-				   NULL,</div><div class='del'>-				   false);</div><div class='del'>-		cm_node-&gt;qhash_set = 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;cm_core-&gt;stats_nodes_destroyed++;</div><div class='del'>-	kfree(cm_node);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_handle_fin_pkt - FIN packet received</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- */</div><div class='del'>-static void i40iw_handle_fin_pkt(struct i40iw_cm_node *cm_node)</div><div class='del'>-{</div><div class='del'>-	u32 ret;</div><div class='del'>-</div><div class='del'>-	switch (cm_node-&gt;state) {</div><div class='del'>-	case I40IW_CM_STATE_SYN_RCVD:</div><div class='del'>-	case I40IW_CM_STATE_SYN_SENT:</div><div class='del'>-	case I40IW_CM_STATE_ESTABLISHED:</div><div class='del'>-	case I40IW_CM_STATE_MPAREJ_RCVD:</div><div class='del'>-		cm_node-&gt;tcp_cntxt.rcv_nxt++;</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		cm_node-&gt;state = I40IW_CM_STATE_LAST_ACK;</div><div class='del'>-		i40iw_send_fin(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_MPAREQ_SENT:</div><div class='del'>-		i40iw_create_event(cm_node, I40IW_CM_EVENT_ABORTED);</div><div class='del'>-		cm_node-&gt;tcp_cntxt.rcv_nxt++;</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		cm_node-&gt;state = I40IW_CM_STATE_CLOSED;</div><div class='del'>-		atomic_inc(&amp;cm_node-&gt;ref_count);</div><div class='del'>-		i40iw_send_reset(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_FIN_WAIT1:</div><div class='del'>-		cm_node-&gt;tcp_cntxt.rcv_nxt++;</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		cm_node-&gt;state = I40IW_CM_STATE_CLOSING;</div><div class='del'>-		i40iw_send_ack(cm_node);</div><div class='del'>-		/*</div><div class='del'>-		 * Wait for ACK as this is simultaneous close.</div><div class='del'>-		 * After we receive ACK, do not send anything.</div><div class='del'>-		 * Just rm the node.</div><div class='del'>-		 */</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_FIN_WAIT2:</div><div class='del'>-		cm_node-&gt;tcp_cntxt.rcv_nxt++;</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		cm_node-&gt;state = I40IW_CM_STATE_TIME_WAIT;</div><div class='del'>-		i40iw_send_ack(cm_node);</div><div class='del'>-		ret =</div><div class='del'>-		    i40iw_schedule_cm_timer(cm_node, NULL, I40IW_TIMER_TYPE_CLOSE, 1, 0);</div><div class='del'>-		if (ret)</div><div class='del'>-			i40iw_pr_err("node %p state = %d\n", cm_node, cm_node-&gt;state);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_TIME_WAIT:</div><div class='del'>-		cm_node-&gt;tcp_cntxt.rcv_nxt++;</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		cm_node-&gt;state = I40IW_CM_STATE_CLOSED;</div><div class='del'>-		i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_OFFLOADED:</div><div class='del'>-	default:</div><div class='del'>-		i40iw_pr_err("bad state node %p state = %d\n", cm_node, cm_node-&gt;state);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_handle_rst_pkt - process received RST packet</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @rbuf: receive buffer</div><div class='del'>- */</div><div class='del'>-static void i40iw_handle_rst_pkt(struct i40iw_cm_node *cm_node,</div><div class='del'>-				 struct i40iw_puda_buf *rbuf)</div><div class='del'>-{</div><div class='del'>-	i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-	switch (cm_node-&gt;state) {</div><div class='del'>-	case I40IW_CM_STATE_SYN_SENT:</div><div class='del'>-	case I40IW_CM_STATE_MPAREQ_SENT:</div><div class='del'>-		switch (cm_node-&gt;mpa_frame_rev) {</div><div class='del'>-		case IETF_MPA_V2:</div><div class='del'>-			cm_node-&gt;mpa_frame_rev = IETF_MPA_V1;</div><div class='del'>-			/* send a syn and goto syn sent state */</div><div class='del'>-			cm_node-&gt;state = I40IW_CM_STATE_SYN_SENT;</div><div class='del'>-			if (i40iw_send_syn(cm_node, 0))</div><div class='del'>-				i40iw_active_open_err(cm_node, false);</div><div class='del'>-			break;</div><div class='del'>-		case IETF_MPA_V1:</div><div class='del'>-		default:</div><div class='del'>-			i40iw_active_open_err(cm_node, false);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_MPAREQ_RCVD:</div><div class='del'>-		atomic_add_return(1, &amp;cm_node-&gt;passive_state);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_ESTABLISHED:</div><div class='del'>-	case I40IW_CM_STATE_SYN_RCVD:</div><div class='del'>-	case I40IW_CM_STATE_LISTENING:</div><div class='del'>-		i40iw_pr_err("Bad state state = %d\n", cm_node-&gt;state);</div><div class='del'>-		i40iw_passive_open_err(cm_node, false);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_OFFLOADED:</div><div class='del'>-		i40iw_active_open_err(cm_node, false);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_CLOSED:</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_FIN_WAIT2:</div><div class='del'>-	case I40IW_CM_STATE_FIN_WAIT1:</div><div class='del'>-	case I40IW_CM_STATE_LAST_ACK:</div><div class='del'>-		cm_node-&gt;cm_id-&gt;rem_ref(cm_node-&gt;cm_id);</div><div class='del'>-		/* fall through */</div><div class='del'>-	case I40IW_CM_STATE_TIME_WAIT:</div><div class='del'>-		cm_node-&gt;state = I40IW_CM_STATE_CLOSED;</div><div class='del'>-		i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_handle_rcv_mpa - Process a recv'd mpa buffer</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @rbuf: receive buffer</div><div class='del'>- */</div><div class='del'>-static void i40iw_handle_rcv_mpa(struct i40iw_cm_node *cm_node,</div><div class='del'>-				 struct i40iw_puda_buf *rbuf)</div><div class='del'>-{</div><div class='del'>-	int ret;</div><div class='del'>-	int datasize = rbuf-&gt;datalen;</div><div class='del'>-	u8 *dataloc = rbuf-&gt;data;</div><div class='del'>-</div><div class='del'>-	enum i40iw_cm_event_type type = I40IW_CM_EVENT_UNKNOWN;</div><div class='del'>-	u32 res_type;</div><div class='del'>-</div><div class='del'>-	ret = i40iw_parse_mpa(cm_node, dataloc, &amp;res_type, datasize);</div><div class='del'>-	if (ret) {</div><div class='del'>-		if (cm_node-&gt;state == I40IW_CM_STATE_MPAREQ_SENT)</div><div class='del'>-			i40iw_active_open_err(cm_node, true);</div><div class='del'>-		else</div><div class='del'>-			i40iw_passive_open_err(cm_node, true);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	switch (cm_node-&gt;state) {</div><div class='del'>-	case I40IW_CM_STATE_ESTABLISHED:</div><div class='del'>-		if (res_type == I40IW_MPA_REQUEST_REJECT)</div><div class='del'>-			i40iw_pr_err("state for reject\n");</div><div class='del'>-		cm_node-&gt;state = I40IW_CM_STATE_MPAREQ_RCVD;</div><div class='del'>-		type = I40IW_CM_EVENT_MPA_REQ;</div><div class='del'>-		i40iw_send_ack(cm_node);	/* ACK received MPA request */</div><div class='del'>-		atomic_set(&amp;cm_node-&gt;passive_state,</div><div class='del'>-			   I40IW_PASSIVE_STATE_INDICATED);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_MPAREQ_SENT:</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		if (res_type == I40IW_MPA_REQUEST_REJECT) {</div><div class='del'>-			type = I40IW_CM_EVENT_MPA_REJECT;</div><div class='del'>-			cm_node-&gt;state = I40IW_CM_STATE_MPAREJ_RCVD;</div><div class='del'>-		} else {</div><div class='del'>-			type = I40IW_CM_EVENT_CONNECTED;</div><div class='del'>-			cm_node-&gt;state = I40IW_CM_STATE_OFFLOADED;</div><div class='del'>-		}</div><div class='del'>-		i40iw_send_ack(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		pr_err("%s wrong cm_node state =%d\n", __func__, cm_node-&gt;state);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	i40iw_create_event(cm_node, type);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_indicate_pkt_err - Send up err event to cm</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- */</div><div class='del'>-static void i40iw_indicate_pkt_err(struct i40iw_cm_node *cm_node)</div><div class='del'>-{</div><div class='del'>-	switch (cm_node-&gt;state) {</div><div class='del'>-	case I40IW_CM_STATE_SYN_SENT:</div><div class='del'>-	case I40IW_CM_STATE_MPAREQ_SENT:</div><div class='del'>-		i40iw_active_open_err(cm_node, true);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_ESTABLISHED:</div><div class='del'>-	case I40IW_CM_STATE_SYN_RCVD:</div><div class='del'>-		i40iw_passive_open_err(cm_node, true);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_OFFLOADED:</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_check_syn - Check for error on received syn ack</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @tcph: pointer tcp header</div><div class='del'>- */</div><div class='del'>-static int i40iw_check_syn(struct i40iw_cm_node *cm_node, struct tcphdr *tcph)</div><div class='del'>-{</div><div class='del'>-	int err = 0;</div><div class='del'>-</div><div class='del'>-	if (ntohl(tcph-&gt;ack_seq) != cm_node-&gt;tcp_cntxt.loc_seq_num) {</div><div class='del'>-		err = 1;</div><div class='del'>-		i40iw_active_open_err(cm_node, true);</div><div class='del'>-	}</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_check_seq - check seq numbers if OK</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @tcph: pointer tcp header</div><div class='del'>- */</div><div class='del'>-static int i40iw_check_seq(struct i40iw_cm_node *cm_node, struct tcphdr *tcph)</div><div class='del'>-{</div><div class='del'>-	int err = 0;</div><div class='del'>-	u32 seq;</div><div class='del'>-	u32 ack_seq;</div><div class='del'>-	u32 loc_seq_num = cm_node-&gt;tcp_cntxt.loc_seq_num;</div><div class='del'>-	u32 rcv_nxt = cm_node-&gt;tcp_cntxt.rcv_nxt;</div><div class='del'>-	u32 rcv_wnd;</div><div class='del'>-</div><div class='del'>-	seq = ntohl(tcph-&gt;seq);</div><div class='del'>-	ack_seq = ntohl(tcph-&gt;ack_seq);</div><div class='del'>-	rcv_wnd = cm_node-&gt;tcp_cntxt.rcv_wnd;</div><div class='del'>-	if (ack_seq != loc_seq_num)</div><div class='del'>-		err = -1;</div><div class='del'>-	else if (!between(seq, rcv_nxt, (rcv_nxt + rcv_wnd)))</div><div class='del'>-		err = -1;</div><div class='del'>-	if (err) {</div><div class='del'>-		i40iw_pr_err("seq number\n");</div><div class='del'>-		i40iw_indicate_pkt_err(cm_node);</div><div class='del'>-	}</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_handle_syn_pkt - is for Passive node</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @rbuf: receive buffer</div><div class='del'>- */</div><div class='del'>-static void i40iw_handle_syn_pkt(struct i40iw_cm_node *cm_node,</div><div class='del'>-				 struct i40iw_puda_buf *rbuf)</div><div class='del'>-{</div><div class='del'>-	struct tcphdr *tcph = (struct tcphdr *)rbuf-&gt;tcph;</div><div class='del'>-	int ret;</div><div class='del'>-	u32 inc_sequence;</div><div class='del'>-	int optionsize;</div><div class='del'>-	struct i40iw_cm_info nfo;</div><div class='del'>-</div><div class='del'>-	optionsize = (tcph-&gt;doff &lt;&lt; 2) - sizeof(struct tcphdr);</div><div class='del'>-	inc_sequence = ntohl(tcph-&gt;seq);</div><div class='del'>-</div><div class='del'>-	switch (cm_node-&gt;state) {</div><div class='del'>-	case I40IW_CM_STATE_SYN_SENT:</div><div class='del'>-	case I40IW_CM_STATE_MPAREQ_SENT:</div><div class='del'>-		/* Rcvd syn on active open connection */</div><div class='del'>-		i40iw_active_open_err(cm_node, 1);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_LISTENING:</div><div class='del'>-		/* Passive OPEN */</div><div class='del'>-		if (atomic_read(&amp;cm_node-&gt;listener-&gt;pend_accepts_cnt) &gt;</div><div class='del'>-		    cm_node-&gt;listener-&gt;backlog) {</div><div class='del'>-			cm_node-&gt;cm_core-&gt;stats_backlog_drops++;</div><div class='del'>-			i40iw_passive_open_err(cm_node, false);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		ret = i40iw_handle_tcp_options(cm_node, tcph, optionsize, 1);</div><div class='del'>-		if (ret) {</div><div class='del'>-			i40iw_passive_open_err(cm_node, false);</div><div class='del'>-			/* drop pkt */</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		cm_node-&gt;tcp_cntxt.rcv_nxt = inc_sequence + 1;</div><div class='del'>-		cm_node-&gt;accept_pend = 1;</div><div class='del'>-		atomic_inc(&amp;cm_node-&gt;listener-&gt;pend_accepts_cnt);</div><div class='del'>-</div><div class='del'>-		cm_node-&gt;state = I40IW_CM_STATE_SYN_RCVD;</div><div class='del'>-		i40iw_get_addr_info(cm_node, &amp;nfo);</div><div class='del'>-		ret = i40iw_manage_qhash(cm_node-&gt;iwdev,</div><div class='del'>-					 &amp;nfo,</div><div class='del'>-					 I40IW_QHASH_TYPE_TCP_ESTABLISHED,</div><div class='del'>-					 I40IW_QHASH_MANAGE_TYPE_ADD,</div><div class='del'>-					 (void *)cm_node,</div><div class='del'>-					 false);</div><div class='del'>-		cm_node-&gt;qhash_set = true;</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_CLOSED:</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		atomic_inc(&amp;cm_node-&gt;ref_count);</div><div class='del'>-		i40iw_send_reset(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_OFFLOADED:</div><div class='del'>-	case I40IW_CM_STATE_ESTABLISHED:</div><div class='del'>-	case I40IW_CM_STATE_FIN_WAIT1:</div><div class='del'>-	case I40IW_CM_STATE_FIN_WAIT2:</div><div class='del'>-	case I40IW_CM_STATE_MPAREQ_RCVD:</div><div class='del'>-	case I40IW_CM_STATE_LAST_ACK:</div><div class='del'>-	case I40IW_CM_STATE_CLOSING:</div><div class='del'>-	case I40IW_CM_STATE_UNKNOWN:</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_handle_synack_pkt - Process SYN+ACK packet (active side)</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @rbuf: receive buffer</div><div class='del'>- */</div><div class='del'>-static void i40iw_handle_synack_pkt(struct i40iw_cm_node *cm_node,</div><div class='del'>-				    struct i40iw_puda_buf *rbuf)</div><div class='del'>-{</div><div class='del'>-	struct tcphdr *tcph = (struct tcphdr *)rbuf-&gt;tcph;</div><div class='del'>-	int ret;</div><div class='del'>-	u32 inc_sequence;</div><div class='del'>-	int optionsize;</div><div class='del'>-</div><div class='del'>-	optionsize = (tcph-&gt;doff &lt;&lt; 2) - sizeof(struct tcphdr);</div><div class='del'>-	inc_sequence = ntohl(tcph-&gt;seq);</div><div class='del'>-	switch (cm_node-&gt;state) {</div><div class='del'>-	case I40IW_CM_STATE_SYN_SENT:</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		/* active open */</div><div class='del'>-		if (i40iw_check_syn(cm_node, tcph)) {</div><div class='del'>-			i40iw_pr_err("check syn fail\n");</div><div class='del'>-			return;</div><div class='del'>-		}</div><div class='del'>-		cm_node-&gt;tcp_cntxt.rem_ack_num = ntohl(tcph-&gt;ack_seq);</div><div class='del'>-		/* setup options */</div><div class='del'>-		ret = i40iw_handle_tcp_options(cm_node, tcph, optionsize, 0);</div><div class='del'>-		if (ret) {</div><div class='del'>-			i40iw_debug(cm_node-&gt;dev,</div><div class='del'>-				    I40IW_DEBUG_CM,</div><div class='del'>-				    "cm_node=%p tcp_options failed\n",</div><div class='del'>-				    cm_node);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		cm_node-&gt;tcp_cntxt.rcv_nxt = inc_sequence + 1;</div><div class='del'>-		i40iw_send_ack(cm_node);	/* ACK  for the syn_ack */</div><div class='del'>-		ret = i40iw_send_mpa_request(cm_node);</div><div class='del'>-		if (ret) {</div><div class='del'>-			i40iw_debug(cm_node-&gt;dev,</div><div class='del'>-				    I40IW_DEBUG_CM,</div><div class='del'>-				    "cm_node=%p i40iw_send_mpa_request failed\n",</div><div class='del'>-				    cm_node);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		cm_node-&gt;state = I40IW_CM_STATE_MPAREQ_SENT;</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_MPAREQ_RCVD:</div><div class='del'>-		i40iw_passive_open_err(cm_node, true);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_LISTENING:</div><div class='del'>-		cm_node-&gt;tcp_cntxt.loc_seq_num = ntohl(tcph-&gt;ack_seq);</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		cm_node-&gt;state = I40IW_CM_STATE_CLOSED;</div><div class='del'>-		i40iw_send_reset(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_CLOSED:</div><div class='del'>-		cm_node-&gt;tcp_cntxt.loc_seq_num = ntohl(tcph-&gt;ack_seq);</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		atomic_inc(&amp;cm_node-&gt;ref_count);</div><div class='del'>-		i40iw_send_reset(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_ESTABLISHED:</div><div class='del'>-	case I40IW_CM_STATE_FIN_WAIT1:</div><div class='del'>-	case I40IW_CM_STATE_FIN_WAIT2:</div><div class='del'>-	case I40IW_CM_STATE_LAST_ACK:</div><div class='del'>-	case I40IW_CM_STATE_OFFLOADED:</div><div class='del'>-	case I40IW_CM_STATE_CLOSING:</div><div class='del'>-	case I40IW_CM_STATE_UNKNOWN:</div><div class='del'>-	case I40IW_CM_STATE_MPAREQ_SENT:</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_handle_ack_pkt - process packet with ACK</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @rbuf: receive buffer</div><div class='del'>- */</div><div class='del'>-static int i40iw_handle_ack_pkt(struct i40iw_cm_node *cm_node,</div><div class='del'>-				struct i40iw_puda_buf *rbuf)</div><div class='del'>-{</div><div class='del'>-	struct tcphdr *tcph = (struct tcphdr *)rbuf-&gt;tcph;</div><div class='del'>-	u32 inc_sequence;</div><div class='del'>-	int ret = 0;</div><div class='del'>-	int optionsize;</div><div class='del'>-	u32 datasize = rbuf-&gt;datalen;</div><div class='del'>-</div><div class='del'>-	optionsize = (tcph-&gt;doff &lt;&lt; 2) - sizeof(struct tcphdr);</div><div class='del'>-</div><div class='del'>-	if (i40iw_check_seq(cm_node, tcph))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	inc_sequence = ntohl(tcph-&gt;seq);</div><div class='del'>-	switch (cm_node-&gt;state) {</div><div class='del'>-	case I40IW_CM_STATE_SYN_RCVD:</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		ret = i40iw_handle_tcp_options(cm_node, tcph, optionsize, 1);</div><div class='del'>-		if (ret)</div><div class='del'>-			break;</div><div class='del'>-		cm_node-&gt;tcp_cntxt.rem_ack_num = ntohl(tcph-&gt;ack_seq);</div><div class='del'>-		cm_node-&gt;state = I40IW_CM_STATE_ESTABLISHED;</div><div class='del'>-		if (datasize) {</div><div class='del'>-			cm_node-&gt;tcp_cntxt.rcv_nxt = inc_sequence + datasize;</div><div class='del'>-			i40iw_handle_rcv_mpa(cm_node, rbuf);</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_ESTABLISHED:</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		if (datasize) {</div><div class='del'>-			cm_node-&gt;tcp_cntxt.rcv_nxt = inc_sequence + datasize;</div><div class='del'>-			i40iw_handle_rcv_mpa(cm_node, rbuf);</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_MPAREQ_SENT:</div><div class='del'>-		cm_node-&gt;tcp_cntxt.rem_ack_num = ntohl(tcph-&gt;ack_seq);</div><div class='del'>-		if (datasize) {</div><div class='del'>-			cm_node-&gt;tcp_cntxt.rcv_nxt = inc_sequence + datasize;</div><div class='del'>-			cm_node-&gt;ack_rcvd = false;</div><div class='del'>-			i40iw_handle_rcv_mpa(cm_node, rbuf);</div><div class='del'>-		} else {</div><div class='del'>-			cm_node-&gt;ack_rcvd = true;</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_LISTENING:</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		cm_node-&gt;state = I40IW_CM_STATE_CLOSED;</div><div class='del'>-		i40iw_send_reset(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_CLOSED:</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		atomic_inc(&amp;cm_node-&gt;ref_count);</div><div class='del'>-		i40iw_send_reset(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_LAST_ACK:</div><div class='del'>-	case I40IW_CM_STATE_CLOSING:</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		cm_node-&gt;state = I40IW_CM_STATE_CLOSED;</div><div class='del'>-		if (!cm_node-&gt;accept_pend)</div><div class='del'>-			cm_node-&gt;cm_id-&gt;rem_ref(cm_node-&gt;cm_id);</div><div class='del'>-		i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_FIN_WAIT1:</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		cm_node-&gt;state = I40IW_CM_STATE_FIN_WAIT2;</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_SYN_SENT:</div><div class='del'>-	case I40IW_CM_STATE_FIN_WAIT2:</div><div class='del'>-	case I40IW_CM_STATE_OFFLOADED:</div><div class='del'>-	case I40IW_CM_STATE_MPAREQ_RCVD:</div><div class='del'>-	case I40IW_CM_STATE_UNKNOWN:</div><div class='del'>-	default:</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_process_packet - process cm packet</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @rbuf: receive buffer</div><div class='del'>- */</div><div class='del'>-static void i40iw_process_packet(struct i40iw_cm_node *cm_node,</div><div class='del'>-				 struct i40iw_puda_buf *rbuf)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_tcpip_pkt_type pkt_type = I40IW_PKT_TYPE_UNKNOWN;</div><div class='del'>-	struct tcphdr *tcph = (struct tcphdr *)rbuf-&gt;tcph;</div><div class='del'>-	u32 fin_set = 0;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if (tcph-&gt;rst) {</div><div class='del'>-		pkt_type = I40IW_PKT_TYPE_RST;</div><div class='del'>-	} else if (tcph-&gt;syn) {</div><div class='del'>-		pkt_type = I40IW_PKT_TYPE_SYN;</div><div class='del'>-		if (tcph-&gt;ack)</div><div class='del'>-			pkt_type = I40IW_PKT_TYPE_SYNACK;</div><div class='del'>-	} else if (tcph-&gt;ack) {</div><div class='del'>-		pkt_type = I40IW_PKT_TYPE_ACK;</div><div class='del'>-	}</div><div class='del'>-	if (tcph-&gt;fin)</div><div class='del'>-		fin_set = 1;</div><div class='del'>-</div><div class='del'>-	switch (pkt_type) {</div><div class='del'>-	case I40IW_PKT_TYPE_SYN:</div><div class='del'>-		i40iw_handle_syn_pkt(cm_node, rbuf);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_PKT_TYPE_SYNACK:</div><div class='del'>-		i40iw_handle_synack_pkt(cm_node, rbuf);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_PKT_TYPE_ACK:</div><div class='del'>-		ret = i40iw_handle_ack_pkt(cm_node, rbuf);</div><div class='del'>-		if (fin_set &amp;&amp; !ret)</div><div class='del'>-			i40iw_handle_fin_pkt(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_PKT_TYPE_RST:</div><div class='del'>-		i40iw_handle_rst_pkt(cm_node, rbuf);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		if (fin_set &amp;&amp;</div><div class='del'>-		    (!i40iw_check_seq(cm_node, (struct tcphdr *)rbuf-&gt;tcph)))</div><div class='del'>-			i40iw_handle_fin_pkt(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_make_listen_node - create a listen node with params</div><div class='del'>- * @cm_core: cm's core</div><div class='del'>- * @iwdev: iwarp device structure</div><div class='del'>- * @cm_info: quad info for connection</div><div class='del'>- */</div><div class='del'>-static struct i40iw_cm_listener *i40iw_make_listen_node(</div><div class='del'>-					struct i40iw_cm_core *cm_core,</div><div class='del'>-					struct i40iw_device *iwdev,</div><div class='del'>-					struct i40iw_cm_info *cm_info)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_listener *listener;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	/* cannot have multiple matching listeners */</div><div class='del'>-	listener = i40iw_find_listener(cm_core, cm_info-&gt;loc_addr,</div><div class='del'>-				       cm_info-&gt;loc_port,</div><div class='del'>-				       cm_info-&gt;vlan_id,</div><div class='del'>-				       I40IW_CM_LISTENER_EITHER_STATE);</div><div class='del'>-	if (listener &amp;&amp;</div><div class='del'>-	    (listener-&gt;listener_state == I40IW_CM_LISTENER_ACTIVE_STATE)) {</div><div class='del'>-		atomic_dec(&amp;listener-&gt;ref_count);</div><div class='del'>-		i40iw_debug(cm_core-&gt;dev,</div><div class='del'>-			    I40IW_DEBUG_CM,</div><div class='del'>-			    "Not creating listener since it already exists\n");</div><div class='del'>-		return NULL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!listener) {</div><div class='del'>-		/* create a CM listen node (1/2 node to compare incoming traffic to) */</div><div class='del'>-		listener = kzalloc(sizeof(*listener), GFP_KERNEL);</div><div class='del'>-		if (!listener)</div><div class='del'>-			return NULL;</div><div class='del'>-		cm_core-&gt;stats_listen_nodes_created++;</div><div class='del'>-		memcpy(listener-&gt;loc_addr, cm_info-&gt;loc_addr, sizeof(listener-&gt;loc_addr));</div><div class='del'>-		listener-&gt;loc_port = cm_info-&gt;loc_port;</div><div class='del'>-</div><div class='del'>-		INIT_LIST_HEAD(&amp;listener-&gt;child_listen_list);</div><div class='del'>-</div><div class='del'>-		atomic_set(&amp;listener-&gt;ref_count, 1);</div><div class='del'>-	} else {</div><div class='del'>-		listener-&gt;reused_node = 1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	listener-&gt;cm_id = cm_info-&gt;cm_id;</div><div class='del'>-	listener-&gt;ipv4 = cm_info-&gt;ipv4;</div><div class='del'>-	listener-&gt;vlan_id = cm_info-&gt;vlan_id;</div><div class='del'>-	atomic_set(&amp;listener-&gt;pend_accepts_cnt, 0);</div><div class='del'>-	listener-&gt;cm_core = cm_core;</div><div class='del'>-	listener-&gt;iwdev = iwdev;</div><div class='del'>-</div><div class='del'>-	listener-&gt;backlog = cm_info-&gt;backlog;</div><div class='del'>-	listener-&gt;listener_state = I40IW_CM_LISTENER_ACTIVE_STATE;</div><div class='del'>-</div><div class='del'>-	if (!listener-&gt;reused_node) {</div><div class='del'>-		spin_lock_irqsave(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='del'>-		list_add(&amp;listener-&gt;list, &amp;cm_core-&gt;listen_nodes);</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return listener;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_create_cm_node - make a connection node with params</div><div class='del'>- * @cm_core: cm's core</div><div class='del'>- * @iwdev: iwarp device structure</div><div class='del'>- * @conn_param: upper layer connection parameters</div><div class='del'>- * @cm_info: quad info for connection</div><div class='del'>- */</div><div class='del'>-static struct i40iw_cm_node *i40iw_create_cm_node(</div><div class='del'>-					struct i40iw_cm_core *cm_core,</div><div class='del'>-					struct i40iw_device *iwdev,</div><div class='del'>-					struct iw_cm_conn_param *conn_param,</div><div class='del'>-					struct i40iw_cm_info *cm_info)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_node *cm_node;</div><div class='del'>-	struct i40iw_cm_listener *loopback_remotelistener;</div><div class='del'>-	struct i40iw_cm_node *loopback_remotenode;</div><div class='del'>-	struct i40iw_cm_info loopback_cm_info;</div><div class='del'>-</div><div class='del'>-	u16 private_data_len = conn_param-&gt;private_data_len;</div><div class='del'>-	const void *private_data = conn_param-&gt;private_data;</div><div class='del'>-</div><div class='del'>-	/* create a CM connection node */</div><div class='del'>-	cm_node = i40iw_make_cm_node(cm_core, iwdev, cm_info, NULL);</div><div class='del'>-	if (!cm_node)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-	/* set our node side to client (active) side */</div><div class='del'>-	cm_node-&gt;tcp_cntxt.client = 1;</div><div class='del'>-	cm_node-&gt;tcp_cntxt.rcv_wscale = I40IW_CM_DEFAULT_RCV_WND_SCALE;</div><div class='del'>-</div><div class='del'>-	i40iw_record_ird_ord(cm_node, conn_param-&gt;ird, conn_param-&gt;ord);</div><div class='del'>-</div><div class='del'>-	if (!memcmp(cm_info-&gt;loc_addr, cm_info-&gt;rem_addr, sizeof(cm_info-&gt;loc_addr))) {</div><div class='del'>-		loopback_remotelistener = i40iw_find_listener(</div><div class='del'>-						cm_core,</div><div class='del'>-						cm_info-&gt;rem_addr,</div><div class='del'>-						cm_node-&gt;rem_port,</div><div class='del'>-						cm_node-&gt;vlan_id,</div><div class='del'>-						I40IW_CM_LISTENER_ACTIVE_STATE);</div><div class='del'>-		if (!loopback_remotelistener) {</div><div class='del'>-			i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-			return ERR_PTR(-ECONNREFUSED);</div><div class='del'>-		} else {</div><div class='del'>-			loopback_cm_info = *cm_info;</div><div class='del'>-			loopback_cm_info.loc_port = cm_info-&gt;rem_port;</div><div class='del'>-			loopback_cm_info.rem_port = cm_info-&gt;loc_port;</div><div class='del'>-			loopback_cm_info.cm_id = loopback_remotelistener-&gt;cm_id;</div><div class='del'>-			loopback_cm_info.ipv4 = cm_info-&gt;ipv4;</div><div class='del'>-			loopback_remotenode = i40iw_make_cm_node(cm_core,</div><div class='del'>-								 iwdev,</div><div class='del'>-								 &amp;loopback_cm_info,</div><div class='del'>-								 loopback_remotelistener);</div><div class='del'>-			if (!loopback_remotenode) {</div><div class='del'>-				i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-				return ERR_PTR(-ENOMEM);</div><div class='del'>-			}</div><div class='del'>-			cm_core-&gt;stats_loopbacks++;</div><div class='del'>-			loopback_remotenode-&gt;loopbackpartner = cm_node;</div><div class='del'>-			loopback_remotenode-&gt;tcp_cntxt.rcv_wscale =</div><div class='del'>-				I40IW_CM_DEFAULT_RCV_WND_SCALE;</div><div class='del'>-			cm_node-&gt;loopbackpartner = loopback_remotenode;</div><div class='del'>-			memcpy(loopback_remotenode-&gt;pdata_buf, private_data,</div><div class='del'>-			       private_data_len);</div><div class='del'>-			loopback_remotenode-&gt;pdata.size = private_data_len;</div><div class='del'>-</div><div class='del'>-			if (loopback_remotenode-&gt;ord_size &gt; cm_node-&gt;ird_size)</div><div class='del'>-				loopback_remotenode-&gt;ord_size =</div><div class='del'>-					cm_node-&gt;ird_size;</div><div class='del'>-</div><div class='del'>-			cm_node-&gt;state = I40IW_CM_STATE_OFFLOADED;</div><div class='del'>-			cm_node-&gt;tcp_cntxt.rcv_nxt =</div><div class='del'>-				loopback_remotenode-&gt;tcp_cntxt.loc_seq_num;</div><div class='del'>-			loopback_remotenode-&gt;tcp_cntxt.rcv_nxt =</div><div class='del'>-				cm_node-&gt;tcp_cntxt.loc_seq_num;</div><div class='del'>-			cm_node-&gt;tcp_cntxt.max_snd_wnd =</div><div class='del'>-				loopback_remotenode-&gt;tcp_cntxt.rcv_wnd;</div><div class='del'>-			loopback_remotenode-&gt;tcp_cntxt.max_snd_wnd = cm_node-&gt;tcp_cntxt.rcv_wnd;</div><div class='del'>-			cm_node-&gt;tcp_cntxt.snd_wnd = loopback_remotenode-&gt;tcp_cntxt.rcv_wnd;</div><div class='del'>-			loopback_remotenode-&gt;tcp_cntxt.snd_wnd = cm_node-&gt;tcp_cntxt.rcv_wnd;</div><div class='del'>-			cm_node-&gt;tcp_cntxt.snd_wscale = loopback_remotenode-&gt;tcp_cntxt.rcv_wscale;</div><div class='del'>-			loopback_remotenode-&gt;tcp_cntxt.snd_wscale = cm_node-&gt;tcp_cntxt.rcv_wscale;</div><div class='del'>-		}</div><div class='del'>-		return cm_node;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;pdata.size = private_data_len;</div><div class='del'>-	cm_node-&gt;pdata.addr = cm_node-&gt;pdata_buf;</div><div class='del'>-</div><div class='del'>-	memcpy(cm_node-&gt;pdata_buf, private_data, private_data_len);</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;state = I40IW_CM_STATE_SYN_SENT;</div><div class='del'>-	return cm_node;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cm_reject - reject and teardown a connection</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @pdate: ptr to private data for reject</div><div class='del'>- * @plen: size of private data</div><div class='del'>- */</div><div class='del'>-static int i40iw_cm_reject(struct i40iw_cm_node *cm_node, const void *pdata, u8 plen)</div><div class='del'>-{</div><div class='del'>-	int ret = 0;</div><div class='del'>-	int err;</div><div class='del'>-	int passive_state;</div><div class='del'>-	struct iw_cm_id *cm_id = cm_node-&gt;cm_id;</div><div class='del'>-	struct i40iw_cm_node *loopback = cm_node-&gt;loopbackpartner;</div><div class='del'>-</div><div class='del'>-	if (cm_node-&gt;tcp_cntxt.client)</div><div class='del'>-		return ret;</div><div class='del'>-	i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-</div><div class='del'>-	if (!loopback) {</div><div class='del'>-		passive_state = atomic_add_return(1, &amp;cm_node-&gt;passive_state);</div><div class='del'>-		if (passive_state == I40IW_SEND_RESET_EVENT) {</div><div class='del'>-			cm_node-&gt;state = I40IW_CM_STATE_CLOSED;</div><div class='del'>-			i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-		} else {</div><div class='del'>-			if (cm_node-&gt;state == I40IW_CM_STATE_LISTENER_DESTROYED) {</div><div class='del'>-				i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-			} else {</div><div class='del'>-				ret = i40iw_send_mpa_reject(cm_node, pdata, plen);</div><div class='del'>-				if (ret) {</div><div class='del'>-					cm_node-&gt;state = I40IW_CM_STATE_CLOSED;</div><div class='del'>-					err = i40iw_send_reset(cm_node);</div><div class='del'>-					if (err)</div><div class='del'>-						i40iw_pr_err("send reset failed\n");</div><div class='del'>-				} else {</div><div class='del'>-					cm_id-&gt;add_ref(cm_id);</div><div class='del'>-				}</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		cm_node-&gt;cm_id = NULL;</div><div class='del'>-		if (cm_node-&gt;state == I40IW_CM_STATE_LISTENER_DESTROYED) {</div><div class='del'>-			i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-			i40iw_rem_ref_cm_node(loopback);</div><div class='del'>-		} else {</div><div class='del'>-			ret = i40iw_send_cm_event(loopback,</div><div class='del'>-						  loopback-&gt;cm_id,</div><div class='del'>-						  IW_CM_EVENT_CONNECT_REPLY,</div><div class='del'>-						  -ECONNREFUSED);</div><div class='del'>-			i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-			loopback-&gt;state = I40IW_CM_STATE_CLOSING;</div><div class='del'>-</div><div class='del'>-			cm_id = loopback-&gt;cm_id;</div><div class='del'>-			i40iw_rem_ref_cm_node(loopback);</div><div class='del'>-			cm_id-&gt;rem_ref(cm_id);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cm_close - close of cm connection</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- */</div><div class='del'>-static int i40iw_cm_close(struct i40iw_cm_node *cm_node)</div><div class='del'>-{</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	if (!cm_node)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	switch (cm_node-&gt;state) {</div><div class='del'>-	case I40IW_CM_STATE_SYN_RCVD:</div><div class='del'>-	case I40IW_CM_STATE_SYN_SENT:</div><div class='del'>-	case I40IW_CM_STATE_ONE_SIDE_ESTABLISHED:</div><div class='del'>-	case I40IW_CM_STATE_ESTABLISHED:</div><div class='del'>-	case I40IW_CM_STATE_ACCEPTING:</div><div class='del'>-	case I40IW_CM_STATE_MPAREQ_SENT:</div><div class='del'>-	case I40IW_CM_STATE_MPAREQ_RCVD:</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		i40iw_send_reset(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_CLOSE_WAIT:</div><div class='del'>-		cm_node-&gt;state = I40IW_CM_STATE_LAST_ACK;</div><div class='del'>-		i40iw_send_fin(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_FIN_WAIT1:</div><div class='del'>-	case I40IW_CM_STATE_FIN_WAIT2:</div><div class='del'>-	case I40IW_CM_STATE_LAST_ACK:</div><div class='del'>-	case I40IW_CM_STATE_TIME_WAIT:</div><div class='del'>-	case I40IW_CM_STATE_CLOSING:</div><div class='del'>-		ret = -1;</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_LISTENING:</div><div class='del'>-		i40iw_cleanup_retrans_entry(cm_node);</div><div class='del'>-		i40iw_send_reset(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_MPAREJ_RCVD:</div><div class='del'>-	case I40IW_CM_STATE_UNKNOWN:</div><div class='del'>-	case I40IW_CM_STATE_INITED:</div><div class='del'>-	case I40IW_CM_STATE_CLOSED:</div><div class='del'>-	case I40IW_CM_STATE_LISTENER_DESTROYED:</div><div class='del'>-		i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_STATE_OFFLOADED:</div><div class='del'>-		if (cm_node-&gt;send_entry)</div><div class='del'>-			i40iw_pr_err("send_entry\n");</div><div class='del'>-		i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_receive_ilq - recv an ETHERNET packet, and process it</div><div class='del'>- * through CM</div><div class='del'>- * @vsi: pointer to the vsi structure</div><div class='del'>- * @rbuf: receive buffer</div><div class='del'>- */</div><div class='del'>-void i40iw_receive_ilq(struct i40iw_sc_vsi *vsi, struct i40iw_puda_buf *rbuf)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_node *cm_node;</div><div class='del'>-	struct i40iw_cm_listener *listener;</div><div class='del'>-	struct iphdr *iph;</div><div class='del'>-	struct ipv6hdr *ip6h;</div><div class='del'>-	struct tcphdr *tcph;</div><div class='del'>-	struct i40iw_cm_info cm_info;</div><div class='del'>-	struct i40iw_sc_dev *dev = vsi-&gt;dev;</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)dev-&gt;back_dev;</div><div class='del'>-	struct i40iw_cm_core *cm_core = &amp;iwdev-&gt;cm_core;</div><div class='del'>-	struct vlan_ethhdr *ethh;</div><div class='del'>-	u16 vtag;</div><div class='del'>-</div><div class='del'>-	/* if vlan, then maclen = 18 else 14 */</div><div class='del'>-	iph = (struct iphdr *)rbuf-&gt;iph;</div><div class='del'>-	memset(&amp;cm_info, 0, sizeof(cm_info));</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(dev,</div><div class='del'>-			I40IW_DEBUG_ILQ,</div><div class='del'>-			"RECEIVE ILQ BUFFER",</div><div class='del'>-			rbuf-&gt;mem.va,</div><div class='del'>-			rbuf-&gt;totallen);</div><div class='del'>-	ethh = (struct vlan_ethhdr *)rbuf-&gt;mem.va;</div><div class='del'>-</div><div class='del'>-	if (ethh-&gt;h_vlan_proto == htons(ETH_P_8021Q)) {</div><div class='del'>-		vtag = ntohs(ethh-&gt;h_vlan_TCI);</div><div class='del'>-		cm_info.user_pri = (vtag &amp; VLAN_PRIO_MASK) &gt;&gt; VLAN_PRIO_SHIFT;</div><div class='del'>-		cm_info.vlan_id = vtag &amp; VLAN_VID_MASK;</div><div class='del'>-		i40iw_debug(cm_core-&gt;dev,</div><div class='del'>-			    I40IW_DEBUG_CM,</div><div class='del'>-			    "%s vlan_id=%d\n",</div><div class='del'>-			    __func__,</div><div class='del'>-			    cm_info.vlan_id);</div><div class='del'>-	} else {</div><div class='del'>-		cm_info.vlan_id = I40IW_NO_VLAN;</div><div class='del'>-	}</div><div class='del'>-	tcph = (struct tcphdr *)rbuf-&gt;tcph;</div><div class='del'>-</div><div class='del'>-	if (rbuf-&gt;ipv4) {</div><div class='del'>-		cm_info.loc_addr[0] = ntohl(iph-&gt;daddr);</div><div class='del'>-		cm_info.rem_addr[0] = ntohl(iph-&gt;saddr);</div><div class='del'>-		cm_info.ipv4 = true;</div><div class='del'>-		cm_info.tos = iph-&gt;tos;</div><div class='del'>-	} else {</div><div class='del'>-		ip6h = (struct ipv6hdr *)rbuf-&gt;iph;</div><div class='del'>-		i40iw_copy_ip_ntohl(cm_info.loc_addr,</div><div class='del'>-				    ip6h-&gt;daddr.in6_u.u6_addr32);</div><div class='del'>-		i40iw_copy_ip_ntohl(cm_info.rem_addr,</div><div class='del'>-				    ip6h-&gt;saddr.in6_u.u6_addr32);</div><div class='del'>-		cm_info.ipv4 = false;</div><div class='del'>-		cm_info.tos = (ip6h-&gt;priority &lt;&lt; 4) | (ip6h-&gt;flow_lbl[0] &gt;&gt; 4);</div><div class='del'>-	}</div><div class='del'>-	cm_info.loc_port = ntohs(tcph-&gt;dest);</div><div class='del'>-	cm_info.rem_port = ntohs(tcph-&gt;source);</div><div class='del'>-	cm_node = i40iw_find_node(cm_core,</div><div class='del'>-				  cm_info.rem_port,</div><div class='del'>-				  cm_info.rem_addr,</div><div class='del'>-				  cm_info.loc_port,</div><div class='del'>-				  cm_info.loc_addr,</div><div class='del'>-				  true,</div><div class='del'>-				  false);</div><div class='del'>-</div><div class='del'>-	if (!cm_node) {</div><div class='del'>-		/* Only type of packet accepted are for */</div><div class='del'>-		/* the PASSIVE open (syn only) */</div><div class='del'>-		if (!tcph-&gt;syn || tcph-&gt;ack)</div><div class='del'>-			return;</div><div class='del'>-		listener =</div><div class='del'>-		    i40iw_find_listener(cm_core,</div><div class='del'>-					cm_info.loc_addr,</div><div class='del'>-					cm_info.loc_port,</div><div class='del'>-					cm_info.vlan_id,</div><div class='del'>-					I40IW_CM_LISTENER_ACTIVE_STATE);</div><div class='del'>-		if (!listener) {</div><div class='del'>-			cm_info.cm_id = NULL;</div><div class='del'>-			i40iw_debug(cm_core-&gt;dev,</div><div class='del'>-				    I40IW_DEBUG_CM,</div><div class='del'>-				    "%s no listener found\n",</div><div class='del'>-				    __func__);</div><div class='del'>-			return;</div><div class='del'>-		}</div><div class='del'>-		cm_info.cm_id = listener-&gt;cm_id;</div><div class='del'>-		cm_node = i40iw_make_cm_node(cm_core, iwdev, &amp;cm_info, listener);</div><div class='del'>-		if (!cm_node) {</div><div class='del'>-			i40iw_debug(cm_core-&gt;dev,</div><div class='del'>-				    I40IW_DEBUG_CM,</div><div class='del'>-				    "%s allocate node failed\n",</div><div class='del'>-				    __func__);</div><div class='del'>-			atomic_dec(&amp;listener-&gt;ref_count);</div><div class='del'>-			return;</div><div class='del'>-		}</div><div class='del'>-		if (!tcph-&gt;rst &amp;&amp; !tcph-&gt;fin) {</div><div class='del'>-			cm_node-&gt;state = I40IW_CM_STATE_LISTENING;</div><div class='del'>-		} else {</div><div class='del'>-			i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-			return;</div><div class='del'>-		}</div><div class='del'>-		atomic_inc(&amp;cm_node-&gt;ref_count);</div><div class='del'>-	} else if (cm_node-&gt;state == I40IW_CM_STATE_OFFLOADED) {</div><div class='del'>-		i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-	i40iw_process_packet(cm_node, rbuf);</div><div class='del'>-	i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_setup_cm_core - allocate a top level instance of a cm</div><div class='del'>- * core</div><div class='del'>- * @iwdev: iwarp device structure</div><div class='del'>- */</div><div class='del'>-int i40iw_setup_cm_core(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_core *cm_core = &amp;iwdev-&gt;cm_core;</div><div class='del'>-</div><div class='del'>-	cm_core-&gt;iwdev = iwdev;</div><div class='del'>-	cm_core-&gt;dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-</div><div class='del'>-	INIT_LIST_HEAD(&amp;cm_core-&gt;accelerated_list);</div><div class='del'>-	INIT_LIST_HEAD(&amp;cm_core-&gt;non_accelerated_list);</div><div class='del'>-	INIT_LIST_HEAD(&amp;cm_core-&gt;listen_nodes);</div><div class='del'>-</div><div class='del'>-	timer_setup(&amp;cm_core-&gt;tcp_timer, i40iw_cm_timer_tick, 0);</div><div class='del'>-</div><div class='del'>-	spin_lock_init(&amp;cm_core-&gt;ht_lock);</div><div class='del'>-	spin_lock_init(&amp;cm_core-&gt;listen_list_lock);</div><div class='del'>-	spin_lock_init(&amp;cm_core-&gt;apbvt_lock);</div><div class='del'>-</div><div class='del'>-	cm_core-&gt;event_wq = alloc_ordered_workqueue("iwewq",</div><div class='del'>-						    WQ_MEM_RECLAIM);</div><div class='del'>-	if (!cm_core-&gt;event_wq)</div><div class='del'>-		goto error;</div><div class='del'>-</div><div class='del'>-	cm_core-&gt;disconn_wq = alloc_ordered_workqueue("iwdwq",</div><div class='del'>-						      WQ_MEM_RECLAIM);</div><div class='del'>-	if (!cm_core-&gt;disconn_wq)</div><div class='del'>-		goto error;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-error:</div><div class='del'>-	i40iw_cleanup_cm_core(&amp;iwdev-&gt;cm_core);</div><div class='del'>-</div><div class='del'>-	return -ENOMEM;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cleanup_cm_core - deallocate a top level instance of a</div><div class='del'>- * cm core</div><div class='del'>- * @cm_core: cm's core</div><div class='del'>- */</div><div class='del'>-void i40iw_cleanup_cm_core(struct i40iw_cm_core *cm_core)</div><div class='del'>-{</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	if (!cm_core)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-	if (timer_pending(&amp;cm_core-&gt;tcp_timer))</div><div class='del'>-		del_timer_sync(&amp;cm_core-&gt;tcp_timer);</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-</div><div class='del'>-	if (cm_core-&gt;event_wq)</div><div class='del'>-		destroy_workqueue(cm_core-&gt;event_wq);</div><div class='del'>-	if (cm_core-&gt;disconn_wq)</div><div class='del'>-		destroy_workqueue(cm_core-&gt;disconn_wq);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_init_tcp_ctx - setup qp context</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- * @tcp_info: offload info for tcp</div><div class='del'>- * @iwqp: associate qp for the connection</div><div class='del'>- */</div><div class='del'>-static void i40iw_init_tcp_ctx(struct i40iw_cm_node *cm_node,</div><div class='del'>-			       struct i40iw_tcp_offload_info *tcp_info,</div><div class='del'>-			       struct i40iw_qp *iwqp)</div><div class='del'>-{</div><div class='del'>-	tcp_info-&gt;ipv4 = cm_node-&gt;ipv4;</div><div class='del'>-	tcp_info-&gt;drop_ooo_seg = true;</div><div class='del'>-	tcp_info-&gt;wscale = true;</div><div class='del'>-	tcp_info-&gt;ignore_tcp_opt = true;</div><div class='del'>-	tcp_info-&gt;ignore_tcp_uns_opt = true;</div><div class='del'>-	tcp_info-&gt;no_nagle = false;</div><div class='del'>-</div><div class='del'>-	tcp_info-&gt;ttl = I40IW_DEFAULT_TTL;</div><div class='del'>-	tcp_info-&gt;rtt_var = cpu_to_le32(I40IW_DEFAULT_RTT_VAR);</div><div class='del'>-	tcp_info-&gt;ss_thresh = cpu_to_le32(I40IW_DEFAULT_SS_THRESH);</div><div class='del'>-	tcp_info-&gt;rexmit_thresh = I40IW_DEFAULT_REXMIT_THRESH;</div><div class='del'>-</div><div class='del'>-	tcp_info-&gt;tcp_state = I40IW_TCP_STATE_ESTABLISHED;</div><div class='del'>-	tcp_info-&gt;snd_wscale = cm_node-&gt;tcp_cntxt.snd_wscale;</div><div class='del'>-	tcp_info-&gt;rcv_wscale = cm_node-&gt;tcp_cntxt.rcv_wscale;</div><div class='del'>-</div><div class='del'>-	tcp_info-&gt;snd_nxt = cpu_to_le32(cm_node-&gt;tcp_cntxt.loc_seq_num);</div><div class='del'>-	tcp_info-&gt;snd_wnd = cpu_to_le32(cm_node-&gt;tcp_cntxt.snd_wnd);</div><div class='del'>-	tcp_info-&gt;rcv_nxt = cpu_to_le32(cm_node-&gt;tcp_cntxt.rcv_nxt);</div><div class='del'>-	tcp_info-&gt;snd_max = cpu_to_le32(cm_node-&gt;tcp_cntxt.loc_seq_num);</div><div class='del'>-</div><div class='del'>-	tcp_info-&gt;snd_una = cpu_to_le32(cm_node-&gt;tcp_cntxt.loc_seq_num);</div><div class='del'>-	tcp_info-&gt;cwnd = cpu_to_le32(2 * cm_node-&gt;tcp_cntxt.mss);</div><div class='del'>-	tcp_info-&gt;snd_wl1 = cpu_to_le32(cm_node-&gt;tcp_cntxt.rcv_nxt);</div><div class='del'>-	tcp_info-&gt;snd_wl2 = cpu_to_le32(cm_node-&gt;tcp_cntxt.loc_seq_num);</div><div class='del'>-	tcp_info-&gt;max_snd_window = cpu_to_le32(cm_node-&gt;tcp_cntxt.max_snd_wnd);</div><div class='del'>-	tcp_info-&gt;rcv_wnd = cpu_to_le32(cm_node-&gt;tcp_cntxt.rcv_wnd &lt;&lt;</div><div class='del'>-					cm_node-&gt;tcp_cntxt.rcv_wscale);</div><div class='del'>-</div><div class='del'>-	tcp_info-&gt;flow_label = 0;</div><div class='del'>-	tcp_info-&gt;snd_mss = cpu_to_le32(((u32)cm_node-&gt;tcp_cntxt.mss));</div><div class='del'>-	if (cm_node-&gt;vlan_id &lt;= VLAN_VID_MASK) {</div><div class='del'>-		tcp_info-&gt;insert_vlan_tag = true;</div><div class='del'>-		tcp_info-&gt;vlan_tag = cpu_to_le16(((u16)cm_node-&gt;user_pri &lt;&lt; I40IW_VLAN_PRIO_SHIFT) |</div><div class='del'>-						  cm_node-&gt;vlan_id);</div><div class='del'>-	}</div><div class='del'>-	if (cm_node-&gt;ipv4) {</div><div class='del'>-		tcp_info-&gt;src_port = cpu_to_le16(cm_node-&gt;loc_port);</div><div class='del'>-		tcp_info-&gt;dst_port = cpu_to_le16(cm_node-&gt;rem_port);</div><div class='del'>-</div><div class='del'>-		tcp_info-&gt;dest_ip_addr3 = cpu_to_le32(cm_node-&gt;rem_addr[0]);</div><div class='del'>-		tcp_info-&gt;local_ipaddr3 = cpu_to_le32(cm_node-&gt;loc_addr[0]);</div><div class='del'>-		tcp_info-&gt;arp_idx =</div><div class='del'>-			cpu_to_le16((u16)i40iw_arp_table(</div><div class='del'>-							 iwqp-&gt;iwdev,</div><div class='del'>-							 &amp;tcp_info-&gt;dest_ip_addr3,</div><div class='del'>-							 true,</div><div class='del'>-							 NULL,</div><div class='del'>-							 I40IW_ARP_RESOLVE));</div><div class='del'>-	} else {</div><div class='del'>-		tcp_info-&gt;src_port = cpu_to_le16(cm_node-&gt;loc_port);</div><div class='del'>-		tcp_info-&gt;dst_port = cpu_to_le16(cm_node-&gt;rem_port);</div><div class='del'>-		tcp_info-&gt;dest_ip_addr0 = cpu_to_le32(cm_node-&gt;rem_addr[0]);</div><div class='del'>-		tcp_info-&gt;dest_ip_addr1 = cpu_to_le32(cm_node-&gt;rem_addr[1]);</div><div class='del'>-		tcp_info-&gt;dest_ip_addr2 = cpu_to_le32(cm_node-&gt;rem_addr[2]);</div><div class='del'>-		tcp_info-&gt;dest_ip_addr3 = cpu_to_le32(cm_node-&gt;rem_addr[3]);</div><div class='del'>-		tcp_info-&gt;local_ipaddr0 = cpu_to_le32(cm_node-&gt;loc_addr[0]);</div><div class='del'>-		tcp_info-&gt;local_ipaddr1 = cpu_to_le32(cm_node-&gt;loc_addr[1]);</div><div class='del'>-		tcp_info-&gt;local_ipaddr2 = cpu_to_le32(cm_node-&gt;loc_addr[2]);</div><div class='del'>-		tcp_info-&gt;local_ipaddr3 = cpu_to_le32(cm_node-&gt;loc_addr[3]);</div><div class='del'>-		tcp_info-&gt;arp_idx =</div><div class='del'>-			cpu_to_le16((u16)i40iw_arp_table(</div><div class='del'>-							 iwqp-&gt;iwdev,</div><div class='del'>-							 &amp;tcp_info-&gt;dest_ip_addr0,</div><div class='del'>-							 false,</div><div class='del'>-							 NULL,</div><div class='del'>-							 I40IW_ARP_RESOLVE));</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cm_init_tsa_conn - setup qp for RTS</div><div class='del'>- * @iwqp: associate qp for the connection</div><div class='del'>- * @cm_node: connection's node</div><div class='del'>- */</div><div class='del'>-static void i40iw_cm_init_tsa_conn(struct i40iw_qp *iwqp,</div><div class='del'>-				   struct i40iw_cm_node *cm_node)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_tcp_offload_info tcp_info;</div><div class='del'>-	struct i40iwarp_offload_info *iwarp_info;</div><div class='del'>-	struct i40iw_qp_host_ctx_info *ctx_info;</div><div class='del'>-	struct i40iw_device *iwdev = iwqp-&gt;iwdev;</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwqp-&gt;iwdev-&gt;sc_dev;</div><div class='del'>-</div><div class='del'>-	memset(&amp;tcp_info, 0x00, sizeof(struct i40iw_tcp_offload_info));</div><div class='del'>-	iwarp_info = &amp;iwqp-&gt;iwarp_info;</div><div class='del'>-	ctx_info = &amp;iwqp-&gt;ctx_info;</div><div class='del'>-</div><div class='del'>-	ctx_info-&gt;tcp_info = &amp;tcp_info;</div><div class='del'>-	ctx_info-&gt;send_cq_num = iwqp-&gt;iwscq-&gt;sc_cq.cq_uk.cq_id;</div><div class='del'>-	ctx_info-&gt;rcv_cq_num = iwqp-&gt;iwrcq-&gt;sc_cq.cq_uk.cq_id;</div><div class='del'>-</div><div class='del'>-	iwarp_info-&gt;ord_size = cm_node-&gt;ord_size;</div><div class='del'>-	iwarp_info-&gt;ird_size = i40iw_derive_hw_ird_setting(cm_node-&gt;ird_size);</div><div class='del'>-</div><div class='del'>-	if (iwarp_info-&gt;ord_size == 1)</div><div class='del'>-		iwarp_info-&gt;ord_size = 2;</div><div class='del'>-</div><div class='del'>-	iwarp_info-&gt;rd_enable = true;</div><div class='del'>-	iwarp_info-&gt;rdmap_ver = 1;</div><div class='del'>-	iwarp_info-&gt;ddp_ver = 1;</div><div class='del'>-</div><div class='del'>-	iwarp_info-&gt;pd_id = iwqp-&gt;iwpd-&gt;sc_pd.pd_id;</div><div class='del'>-</div><div class='del'>-	ctx_info-&gt;tcp_info_valid = true;</div><div class='del'>-	ctx_info-&gt;iwarp_info_valid = true;</div><div class='del'>-	ctx_info-&gt;add_to_qoslist = true;</div><div class='del'>-	ctx_info-&gt;user_pri = cm_node-&gt;user_pri;</div><div class='del'>-</div><div class='del'>-	i40iw_init_tcp_ctx(cm_node, &amp;tcp_info, iwqp);</div><div class='del'>-	if (cm_node-&gt;snd_mark_en) {</div><div class='del'>-		iwarp_info-&gt;snd_mark_en = true;</div><div class='del'>-		iwarp_info-&gt;snd_mark_offset = (tcp_info.snd_nxt &amp;</div><div class='del'>-				SNDMARKER_SEQNMASK) + cm_node-&gt;lsmm_size;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;state = I40IW_CM_STATE_OFFLOADED;</div><div class='del'>-	tcp_info.tcp_state = I40IW_TCP_STATE_ESTABLISHED;</div><div class='del'>-	tcp_info.src_mac_addr_idx = iwdev-&gt;mac_ip_table_idx;</div><div class='del'>-	tcp_info.tos = cm_node-&gt;tos;</div><div class='del'>-</div><div class='del'>-	dev-&gt;iw_priv_qp_ops-&gt;qp_setctx(&amp;iwqp-&gt;sc_qp, (u64 *)(iwqp-&gt;host_ctx.va), ctx_info);</div><div class='del'>-</div><div class='del'>-	/* once tcp_info is set, no need to do it again */</div><div class='del'>-	ctx_info-&gt;tcp_info_valid = false;</div><div class='del'>-	ctx_info-&gt;iwarp_info_valid = false;</div><div class='del'>-	ctx_info-&gt;add_to_qoslist = false;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cm_disconn - when a connection is being closed</div><div class='del'>- * @iwqp: associate qp for the connection</div><div class='del'>- */</div><div class='del'>-void i40iw_cm_disconn(struct i40iw_qp *iwqp)</div><div class='del'>-{</div><div class='del'>-	struct disconn_work *work;</div><div class='del'>-	struct i40iw_device *iwdev = iwqp-&gt;iwdev;</div><div class='del'>-	struct i40iw_cm_core *cm_core = &amp;iwdev-&gt;cm_core;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	work = kzalloc(sizeof(*work), GFP_ATOMIC);</div><div class='del'>-	if (!work)</div><div class='del'>-		return;	/* Timer will clean up */</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;iwdev-&gt;qptable_lock, flags);</div><div class='del'>-	if (!iwdev-&gt;qp_table[iwqp-&gt;ibqp.qp_num]) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;iwdev-&gt;qptable_lock, flags);</div><div class='del'>-		i40iw_debug(&amp;iwdev-&gt;sc_dev, I40IW_DEBUG_CM,</div><div class='del'>-			    "%s qp_id %d is already freed\n",</div><div class='del'>-			     __func__, iwqp-&gt;ibqp.qp_num);</div><div class='del'>-		kfree(work);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-	i40iw_add_ref(&amp;iwqp-&gt;ibqp);</div><div class='del'>-	spin_unlock_irqrestore(&amp;iwdev-&gt;qptable_lock, flags);</div><div class='del'>-</div><div class='del'>-	work-&gt;iwqp = iwqp;</div><div class='del'>-	INIT_WORK(&amp;work-&gt;work, i40iw_disconnect_worker);</div><div class='del'>-	queue_work(cm_core-&gt;disconn_wq, &amp;work-&gt;work);</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_qp_disconnect - free qp and close cm</div><div class='del'>- * @iwqp: associate qp for the connection</div><div class='del'>- */</div><div class='del'>-static void i40iw_qp_disconnect(struct i40iw_qp *iwqp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_ib_device *iwibdev;</div><div class='del'>-</div><div class='del'>-	iwdev = to_iwdev(iwqp-&gt;ibqp.device);</div><div class='del'>-	if (!iwdev) {</div><div class='del'>-		i40iw_pr_err("iwdev == NULL\n");</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	iwibdev = iwdev-&gt;iwibdev;</div><div class='del'>-</div><div class='del'>-	if (iwqp-&gt;active_conn) {</div><div class='del'>-		/* indicate this connection is NOT active */</div><div class='del'>-		iwqp-&gt;active_conn = 0;</div><div class='del'>-	} else {</div><div class='del'>-		/* Need to free the Last Streaming Mode Message */</div><div class='del'>-		if (iwqp-&gt;ietf_mem.va) {</div><div class='del'>-			if (iwqp-&gt;lsmm_mr)</div><div class='del'>-				iwibdev-&gt;ibdev.ops.dereg_mr(iwqp-&gt;lsmm_mr,</div><div class='del'>-							    NULL);</div><div class='del'>-			i40iw_free_dma_mem(iwdev-&gt;sc_dev.hw, &amp;iwqp-&gt;ietf_mem);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* close the CM node down if it is still active */</div><div class='del'>-	if (iwqp-&gt;cm_node) {</div><div class='del'>-		i40iw_debug(&amp;iwdev-&gt;sc_dev, I40IW_DEBUG_CM, "%s Call close API\n", __func__);</div><div class='del'>-		i40iw_cm_close(iwqp-&gt;cm_node);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cm_disconn_true - called by worker thread to disconnect qp</div><div class='del'>- * @iwqp: associate qp for the connection</div><div class='del'>- */</div><div class='del'>-static void i40iw_cm_disconn_true(struct i40iw_qp *iwqp)</div><div class='del'>-{</div><div class='del'>-	struct iw_cm_id *cm_id;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_sc_qp *qp = &amp;iwqp-&gt;sc_qp;</div><div class='del'>-	u16 last_ae;</div><div class='del'>-	u8 original_hw_tcp_state;</div><div class='del'>-	u8 original_ibqp_state;</div><div class='del'>-	int disconn_status = 0;</div><div class='del'>-	int issue_disconn = 0;</div><div class='del'>-	int issue_close = 0;</div><div class='del'>-	int issue_flush = 0;</div><div class='del'>-	struct ib_event ibevent;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if (!iwqp) {</div><div class='del'>-		i40iw_pr_err("iwqp == NULL\n");</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-	cm_id = iwqp-&gt;cm_id;</div><div class='del'>-	/* make sure we havent already closed this connection */</div><div class='del'>-	if (!cm_id) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	iwdev = to_iwdev(iwqp-&gt;ibqp.device);</div><div class='del'>-</div><div class='del'>-	original_hw_tcp_state = iwqp-&gt;hw_tcp_state;</div><div class='del'>-	original_ibqp_state = iwqp-&gt;ibqp_state;</div><div class='del'>-	last_ae = iwqp-&gt;last_aeq;</div><div class='del'>-</div><div class='del'>-	if (qp-&gt;term_flags) {</div><div class='del'>-		issue_disconn = 1;</div><div class='del'>-		issue_close = 1;</div><div class='del'>-		iwqp-&gt;cm_id = NULL;</div><div class='del'>-		/*When term timer expires after cm_timer, don't want</div><div class='del'>-		 *terminate-handler to issue cm_disconn which can re-free</div><div class='del'>-		 *a QP even after its refcnt=0.</div><div class='del'>-		 */</div><div class='del'>-		i40iw_terminate_del_timer(qp);</div><div class='del'>-		if (!iwqp-&gt;flush_issued) {</div><div class='del'>-			iwqp-&gt;flush_issued = 1;</div><div class='del'>-			issue_flush = 1;</div><div class='del'>-		}</div><div class='del'>-	} else if ((original_hw_tcp_state == I40IW_TCP_STATE_CLOSE_WAIT) ||</div><div class='del'>-		   ((original_ibqp_state == IB_QPS_RTS) &amp;&amp;</div><div class='del'>-		    (last_ae == I40IW_AE_LLP_CONNECTION_RESET))) {</div><div class='del'>-		issue_disconn = 1;</div><div class='del'>-		if (last_ae == I40IW_AE_LLP_CONNECTION_RESET)</div><div class='del'>-			disconn_status = -ECONNRESET;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (((original_hw_tcp_state == I40IW_TCP_STATE_CLOSED) ||</div><div class='del'>-	     (original_hw_tcp_state == I40IW_TCP_STATE_TIME_WAIT) ||</div><div class='del'>-	     (last_ae == I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE) ||</div><div class='del'>-	     (last_ae == I40IW_AE_LLP_CONNECTION_RESET) ||</div><div class='del'>-	      iwdev-&gt;reset)) {</div><div class='del'>-		issue_close = 1;</div><div class='del'>-		iwqp-&gt;cm_id = NULL;</div><div class='del'>-		if (!iwqp-&gt;flush_issued) {</div><div class='del'>-			iwqp-&gt;flush_issued = 1;</div><div class='del'>-			issue_flush = 1;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-	if (issue_flush &amp;&amp; !iwqp-&gt;destroyed) {</div><div class='del'>-		/* Flush the queues */</div><div class='del'>-		i40iw_flush_wqes(iwdev, iwqp);</div><div class='del'>-</div><div class='del'>-		if (qp-&gt;term_flags &amp;&amp; iwqp-&gt;ibqp.event_handler) {</div><div class='del'>-			ibevent.device = iwqp-&gt;ibqp.device;</div><div class='del'>-			ibevent.event = (qp-&gt;eventtype == TERM_EVENT_QP_FATAL) ?</div><div class='del'>-					IB_EVENT_QP_FATAL : IB_EVENT_QP_ACCESS_ERR;</div><div class='del'>-			ibevent.element.qp = &amp;iwqp-&gt;ibqp;</div><div class='del'>-			iwqp-&gt;ibqp.event_handler(&amp;ibevent, iwqp-&gt;ibqp.qp_context);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (cm_id &amp;&amp; cm_id-&gt;event_handler) {</div><div class='del'>-		if (issue_disconn) {</div><div class='del'>-			ret = i40iw_send_cm_event(NULL,</div><div class='del'>-						  cm_id,</div><div class='del'>-						  IW_CM_EVENT_DISCONNECT,</div><div class='del'>-						  disconn_status);</div><div class='del'>-</div><div class='del'>-			if (ret)</div><div class='del'>-				i40iw_debug(&amp;iwdev-&gt;sc_dev,</div><div class='del'>-					    I40IW_DEBUG_CM,</div><div class='del'>-					    "disconnect event failed %s: - cm_id = %p\n",</div><div class='del'>-					    __func__, cm_id);</div><div class='del'>-		}</div><div class='del'>-		if (issue_close) {</div><div class='del'>-			i40iw_qp_disconnect(iwqp);</div><div class='del'>-			cm_id-&gt;provider_data = iwqp;</div><div class='del'>-			ret = i40iw_send_cm_event(NULL, cm_id, IW_CM_EVENT_CLOSE, 0);</div><div class='del'>-			if (ret)</div><div class='del'>-				i40iw_debug(&amp;iwdev-&gt;sc_dev,</div><div class='del'>-					    I40IW_DEBUG_CM,</div><div class='del'>-					    "close event failed %s: - cm_id = %p\n",</div><div class='del'>-					    __func__, cm_id);</div><div class='del'>-			cm_id-&gt;rem_ref(cm_id);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_disconnect_worker - worker for connection close</div><div class='del'>- * @work: points or disconn structure</div><div class='del'>- */</div><div class='del'>-static void i40iw_disconnect_worker(struct work_struct *work)</div><div class='del'>-{</div><div class='del'>-	struct disconn_work *dwork = container_of(work, struct disconn_work, work);</div><div class='del'>-	struct i40iw_qp *iwqp = dwork-&gt;iwqp;</div><div class='del'>-</div><div class='del'>-	kfree(dwork);</div><div class='del'>-	i40iw_cm_disconn_true(iwqp);</div><div class='del'>-	i40iw_rem_ref(&amp;iwqp-&gt;ibqp);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_accept - registered call for connection to be accepted</div><div class='del'>- * @cm_id: cm information for passive connection</div><div class='del'>- * @conn_param: accpet parameters</div><div class='del'>- */</div><div class='del'>-int i40iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)</div><div class='del'>-{</div><div class='del'>-	struct ib_qp *ibqp;</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	struct i40iw_cm_core *cm_core;</div><div class='del'>-	struct i40iw_cm_node *cm_node;</div><div class='del'>-	struct ib_qp_attr attr;</div><div class='del'>-	int passive_state;</div><div class='del'>-	struct ib_mr *ibmr;</div><div class='del'>-	struct i40iw_pd *iwpd;</div><div class='del'>-	u16 buf_len = 0;</div><div class='del'>-	struct i40iw_kmem_info accept;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	u64 tagged_offset;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	memset(&amp;attr, 0, sizeof(attr));</div><div class='del'>-	ibqp = i40iw_get_qp(cm_id-&gt;device, conn_param-&gt;qpn);</div><div class='del'>-	if (!ibqp)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	iwqp = to_iwqp(ibqp);</div><div class='del'>-	iwdev = iwqp-&gt;iwdev;</div><div class='del'>-	dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	cm_core = &amp;iwdev-&gt;cm_core;</div><div class='del'>-	cm_node = (struct i40iw_cm_node *)cm_id-&gt;provider_data;</div><div class='del'>-</div><div class='del'>-	if (((struct sockaddr_in *)&amp;cm_id-&gt;local_addr)-&gt;sin_family == AF_INET) {</div><div class='del'>-		cm_node-&gt;ipv4 = true;</div><div class='del'>-		cm_node-&gt;vlan_id = i40iw_get_vlan_ipv4(cm_node-&gt;loc_addr);</div><div class='del'>-	} else {</div><div class='del'>-		cm_node-&gt;ipv4 = false;</div><div class='del'>-		i40iw_netdev_vlan_ipv6(cm_node-&gt;loc_addr, &amp;cm_node-&gt;vlan_id);</div><div class='del'>-	}</div><div class='del'>-	i40iw_debug(cm_node-&gt;dev,</div><div class='del'>-		    I40IW_DEBUG_CM,</div><div class='del'>-		    "Accept vlan_id=%d\n",</div><div class='del'>-		    cm_node-&gt;vlan_id);</div><div class='del'>-	if (cm_node-&gt;state == I40IW_CM_STATE_LISTENER_DESTROYED) {</div><div class='del'>-		if (cm_node-&gt;loopbackpartner)</div><div class='del'>-			i40iw_rem_ref_cm_node(cm_node-&gt;loopbackpartner);</div><div class='del'>-		i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	passive_state = atomic_add_return(1, &amp;cm_node-&gt;passive_state);</div><div class='del'>-	if (passive_state == I40IW_SEND_RESET_EVENT) {</div><div class='del'>-		i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-		return -ECONNRESET;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;cm_core-&gt;stats_accepts++;</div><div class='del'>-	iwqp-&gt;cm_node = (void *)cm_node;</div><div class='del'>-	cm_node-&gt;iwqp = iwqp;</div><div class='del'>-</div><div class='del'>-	buf_len = conn_param-&gt;private_data_len + I40IW_MAX_IETF_SIZE;</div><div class='del'>-</div><div class='del'>-	status = i40iw_allocate_dma_mem(dev-&gt;hw, &amp;iwqp-&gt;ietf_mem, buf_len, 1);</div><div class='del'>-</div><div class='del'>-	if (status)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	cm_node-&gt;pdata.size = conn_param-&gt;private_data_len;</div><div class='del'>-	accept.addr = iwqp-&gt;ietf_mem.va;</div><div class='del'>-	accept.size = i40iw_cm_build_mpa_frame(cm_node, &amp;accept, MPA_KEY_REPLY);</div><div class='del'>-	memcpy(accept.addr + accept.size, conn_param-&gt;private_data,</div><div class='del'>-	       conn_param-&gt;private_data_len);</div><div class='del'>-</div><div class='del'>-	/* setup our first outgoing iWarp send WQE (the IETF frame response) */</div><div class='del'>-	if ((cm_node-&gt;ipv4 &amp;&amp;</div><div class='del'>-	     !i40iw_ipv4_is_loopback(cm_node-&gt;loc_addr[0], cm_node-&gt;rem_addr[0])) ||</div><div class='del'>-	    (!cm_node-&gt;ipv4 &amp;&amp;</div><div class='del'>-	     !i40iw_ipv6_is_loopback(cm_node-&gt;loc_addr, cm_node-&gt;rem_addr))) {</div><div class='del'>-		iwpd = iwqp-&gt;iwpd;</div><div class='del'>-		tagged_offset = (uintptr_t)iwqp-&gt;ietf_mem.va;</div><div class='del'>-		ibmr = i40iw_reg_phys_mr(&amp;iwpd-&gt;ibpd,</div><div class='del'>-					 iwqp-&gt;ietf_mem.pa,</div><div class='del'>-					 buf_len,</div><div class='del'>-					 IB_ACCESS_LOCAL_WRITE,</div><div class='del'>-					 &amp;tagged_offset);</div><div class='del'>-		if (IS_ERR(ibmr)) {</div><div class='del'>-			i40iw_free_dma_mem(dev-&gt;hw, &amp;iwqp-&gt;ietf_mem);</div><div class='del'>-			return -ENOMEM;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		ibmr-&gt;pd = &amp;iwpd-&gt;ibpd;</div><div class='del'>-		ibmr-&gt;device = iwpd-&gt;ibpd.device;</div><div class='del'>-		iwqp-&gt;lsmm_mr = ibmr;</div><div class='del'>-		if (iwqp-&gt;page)</div><div class='del'>-			iwqp-&gt;sc_qp.qp_uk.sq_base = kmap(iwqp-&gt;page);</div><div class='del'>-		dev-&gt;iw_priv_qp_ops-&gt;qp_send_lsmm(&amp;iwqp-&gt;sc_qp,</div><div class='del'>-							iwqp-&gt;ietf_mem.va,</div><div class='del'>-							(accept.size + conn_param-&gt;private_data_len),</div><div class='del'>-							ibmr-&gt;lkey);</div><div class='del'>-</div><div class='del'>-	} else {</div><div class='del'>-		if (iwqp-&gt;page)</div><div class='del'>-			iwqp-&gt;sc_qp.qp_uk.sq_base = kmap(iwqp-&gt;page);</div><div class='del'>-		dev-&gt;iw_priv_qp_ops-&gt;qp_send_lsmm(&amp;iwqp-&gt;sc_qp, NULL, 0, 0);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (iwqp-&gt;page)</div><div class='del'>-		kunmap(iwqp-&gt;page);</div><div class='del'>-</div><div class='del'>-	iwqp-&gt;cm_id = cm_id;</div><div class='del'>-	cm_node-&gt;cm_id = cm_id;</div><div class='del'>-</div><div class='del'>-	cm_id-&gt;provider_data = (void *)iwqp;</div><div class='del'>-	iwqp-&gt;active_conn = 0;</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;lsmm_size = accept.size + conn_param-&gt;private_data_len;</div><div class='del'>-	i40iw_cm_init_tsa_conn(iwqp, cm_node);</div><div class='del'>-	cm_id-&gt;add_ref(cm_id);</div><div class='del'>-	i40iw_add_ref(&amp;iwqp-&gt;ibqp);</div><div class='del'>-</div><div class='del'>-	attr.qp_state = IB_QPS_RTS;</div><div class='del'>-	cm_node-&gt;qhash_set = false;</div><div class='del'>-	i40iw_modify_qp(&amp;iwqp-&gt;ibqp, &amp;attr, IB_QP_STATE, NULL);</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;accelerated = true;</div><div class='del'>-	spin_lock_irqsave(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-	list_move_tail(&amp;cm_node-&gt;list, &amp;cm_core-&gt;accelerated_list);</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-</div><div class='del'>-	status =</div><div class='del'>-		i40iw_send_cm_event(cm_node, cm_id, IW_CM_EVENT_ESTABLISHED, 0);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_CM, "error sending cm event - ESTABLISHED\n");</div><div class='del'>-</div><div class='del'>-	if (cm_node-&gt;loopbackpartner) {</div><div class='del'>-		cm_node-&gt;loopbackpartner-&gt;pdata.size = conn_param-&gt;private_data_len;</div><div class='del'>-</div><div class='del'>-		/* copy entire MPA frame to our cm_node's frame */</div><div class='del'>-		memcpy(cm_node-&gt;loopbackpartner-&gt;pdata_buf,</div><div class='del'>-		       conn_param-&gt;private_data,</div><div class='del'>-		       conn_param-&gt;private_data_len);</div><div class='del'>-		i40iw_create_event(cm_node-&gt;loopbackpartner, I40IW_CM_EVENT_CONNECTED);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (cm_node-&gt;accept_pend) {</div><div class='del'>-		atomic_dec(&amp;cm_node-&gt;listener-&gt;pend_accepts_cnt);</div><div class='del'>-		cm_node-&gt;accept_pend = 0;</div><div class='del'>-	}</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_reject - registered call for connection to be rejected</div><div class='del'>- * @cm_id: cm information for passive connection</div><div class='del'>- * @pdata: private data to be sent</div><div class='del'>- * @pdata_len: private data length</div><div class='del'>- */</div><div class='del'>-int i40iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_cm_node *cm_node;</div><div class='del'>-	struct i40iw_cm_node *loopback;</div><div class='del'>-</div><div class='del'>-	cm_node = (struct i40iw_cm_node *)cm_id-&gt;provider_data;</div><div class='del'>-	loopback = cm_node-&gt;loopbackpartner;</div><div class='del'>-	cm_node-&gt;cm_id = cm_id;</div><div class='del'>-	cm_node-&gt;pdata.size = pdata_len;</div><div class='del'>-</div><div class='del'>-	iwdev = to_iwdev(cm_id-&gt;device);</div><div class='del'>-	if (!iwdev)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	cm_node-&gt;cm_core-&gt;stats_rejects++;</div><div class='del'>-</div><div class='del'>-	if (pdata_len + sizeof(struct ietf_mpa_v2) &gt; MAX_CM_BUFFER)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	if (loopback) {</div><div class='del'>-		memcpy(&amp;loopback-&gt;pdata_buf, pdata, pdata_len);</div><div class='del'>-		loopback-&gt;pdata.size = pdata_len;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return i40iw_cm_reject(cm_node, pdata, pdata_len);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_connect - registered call for connection to be established</div><div class='del'>- * @cm_id: cm information for passive connection</div><div class='del'>- * @conn_param: Information about the connection</div><div class='del'>- */</div><div class='del'>-int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)</div><div class='del'>-{</div><div class='del'>-	struct ib_qp *ibqp;</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_cm_node *cm_node;</div><div class='del'>-	struct i40iw_cm_info cm_info;</div><div class='del'>-	struct sockaddr_in *laddr;</div><div class='del'>-	struct sockaddr_in *raddr;</div><div class='del'>-	struct sockaddr_in6 *laddr6;</div><div class='del'>-	struct sockaddr_in6 *raddr6;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	ibqp = i40iw_get_qp(cm_id-&gt;device, conn_param-&gt;qpn);</div><div class='del'>-	if (!ibqp)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	iwqp = to_iwqp(ibqp);</div><div class='del'>-	if (!iwqp)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	iwdev = to_iwdev(iwqp-&gt;ibqp.device);</div><div class='del'>-	if (!iwdev)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	laddr = (struct sockaddr_in *)&amp;cm_id-&gt;m_local_addr;</div><div class='del'>-	raddr = (struct sockaddr_in *)&amp;cm_id-&gt;m_remote_addr;</div><div class='del'>-	laddr6 = (struct sockaddr_in6 *)&amp;cm_id-&gt;m_local_addr;</div><div class='del'>-	raddr6 = (struct sockaddr_in6 *)&amp;cm_id-&gt;m_remote_addr;</div><div class='del'>-</div><div class='del'>-	if (!(laddr-&gt;sin_port) || !(raddr-&gt;sin_port))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	iwqp-&gt;active_conn = 1;</div><div class='del'>-	iwqp-&gt;cm_id = NULL;</div><div class='del'>-	cm_id-&gt;provider_data = iwqp;</div><div class='del'>-</div><div class='del'>-	/* set up the connection params for the node */</div><div class='del'>-	if (cm_id-&gt;remote_addr.ss_family == AF_INET) {</div><div class='del'>-		cm_info.ipv4 = true;</div><div class='del'>-		memset(cm_info.loc_addr, 0, sizeof(cm_info.loc_addr));</div><div class='del'>-		memset(cm_info.rem_addr, 0, sizeof(cm_info.rem_addr));</div><div class='del'>-		cm_info.loc_addr[0] = ntohl(laddr-&gt;sin_addr.s_addr);</div><div class='del'>-		cm_info.rem_addr[0] = ntohl(raddr-&gt;sin_addr.s_addr);</div><div class='del'>-		cm_info.loc_port = ntohs(laddr-&gt;sin_port);</div><div class='del'>-		cm_info.rem_port = ntohs(raddr-&gt;sin_port);</div><div class='del'>-		cm_info.vlan_id = i40iw_get_vlan_ipv4(cm_info.loc_addr);</div><div class='del'>-	} else {</div><div class='del'>-		cm_info.ipv4 = false;</div><div class='del'>-		i40iw_copy_ip_ntohl(cm_info.loc_addr,</div><div class='del'>-				    laddr6-&gt;sin6_addr.in6_u.u6_addr32);</div><div class='del'>-		i40iw_copy_ip_ntohl(cm_info.rem_addr,</div><div class='del'>-				    raddr6-&gt;sin6_addr.in6_u.u6_addr32);</div><div class='del'>-		cm_info.loc_port = ntohs(laddr6-&gt;sin6_port);</div><div class='del'>-		cm_info.rem_port = ntohs(raddr6-&gt;sin6_port);</div><div class='del'>-		i40iw_netdev_vlan_ipv6(cm_info.loc_addr, &amp;cm_info.vlan_id);</div><div class='del'>-	}</div><div class='del'>-	cm_info.cm_id = cm_id;</div><div class='del'>-	cm_info.tos = cm_id-&gt;tos;</div><div class='del'>-	cm_info.user_pri = rt_tos2priority(cm_id-&gt;tos);</div><div class='del'>-	i40iw_debug(&amp;iwdev-&gt;sc_dev, I40IW_DEBUG_DCB, "%s TOS:[%d] UP:[%d]\n",</div><div class='del'>-		    __func__, cm_id-&gt;tos, cm_info.user_pri);</div><div class='del'>-	cm_id-&gt;add_ref(cm_id);</div><div class='del'>-	cm_node = i40iw_create_cm_node(&amp;iwdev-&gt;cm_core, iwdev,</div><div class='del'>-				       conn_param, &amp;cm_info);</div><div class='del'>-</div><div class='del'>-	if (IS_ERR(cm_node)) {</div><div class='del'>-		ret = PTR_ERR(cm_node);</div><div class='del'>-		cm_id-&gt;rem_ref(cm_id);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if ((cm_info.ipv4 &amp;&amp; (laddr-&gt;sin_addr.s_addr != raddr-&gt;sin_addr.s_addr)) ||</div><div class='del'>-	    (!cm_info.ipv4 &amp;&amp; memcmp(laddr6-&gt;sin6_addr.in6_u.u6_addr32,</div><div class='del'>-				     raddr6-&gt;sin6_addr.in6_u.u6_addr32,</div><div class='del'>-				     sizeof(laddr6-&gt;sin6_addr.in6_u.u6_addr32)))) {</div><div class='del'>-		if (i40iw_manage_qhash(iwdev, &amp;cm_info, I40IW_QHASH_TYPE_TCP_ESTABLISHED,</div><div class='del'>-				       I40IW_QHASH_MANAGE_TYPE_ADD, NULL, true)) {</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto err;</div><div class='del'>-		}</div><div class='del'>-		cm_node-&gt;qhash_set = true;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (i40iw_manage_apbvt(iwdev, cm_info.loc_port,</div><div class='del'>-			       I40IW_MANAGE_APBVT_ADD)) {</div><div class='del'>-		ret =  -EINVAL;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;apbvt_set = true;</div><div class='del'>-	iwqp-&gt;cm_node = cm_node;</div><div class='del'>-	cm_node-&gt;iwqp = iwqp;</div><div class='del'>-	iwqp-&gt;cm_id = cm_id;</div><div class='del'>-	i40iw_add_ref(&amp;iwqp-&gt;ibqp);</div><div class='del'>-</div><div class='del'>-	if (cm_node-&gt;state != I40IW_CM_STATE_OFFLOADED) {</div><div class='del'>-		cm_node-&gt;state = I40IW_CM_STATE_SYN_SENT;</div><div class='del'>-		ret = i40iw_send_syn(cm_node, 0);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (cm_node-&gt;loopbackpartner) {</div><div class='del'>-		cm_node-&gt;loopbackpartner-&gt;state = I40IW_CM_STATE_MPAREQ_RCVD;</div><div class='del'>-		i40iw_create_event(cm_node-&gt;loopbackpartner,</div><div class='del'>-				   I40IW_CM_EVENT_MPA_REQ);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i40iw_debug(cm_node-&gt;dev,</div><div class='del'>-		    I40IW_DEBUG_CM,</div><div class='del'>-		    "Api - connect(): port=0x%04x, cm_node=%p, cm_id = %p.\n",</div><div class='del'>-		    cm_node-&gt;rem_port,</div><div class='del'>-		    cm_node,</div><div class='del'>-		    cm_node-&gt;cm_id);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err:</div><div class='del'>-	if (cm_info.ipv4)</div><div class='del'>-		i40iw_debug(&amp;iwdev-&gt;sc_dev,</div><div class='del'>-			    I40IW_DEBUG_CM,</div><div class='del'>-			    "Api - connect() FAILED: dest addr=%pI4",</div><div class='del'>-			    cm_info.rem_addr);</div><div class='del'>-	else</div><div class='del'>-		i40iw_debug(&amp;iwdev-&gt;sc_dev,</div><div class='del'>-			    I40IW_DEBUG_CM,</div><div class='del'>-			    "Api - connect() FAILED: dest addr=%pI6",</div><div class='del'>-			    cm_info.rem_addr);</div><div class='del'>-</div><div class='del'>-	i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-	cm_id-&gt;rem_ref(cm_id);</div><div class='del'>-	iwdev-&gt;cm_core.stats_connect_errs++;</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_create_listen - registered call creating listener</div><div class='del'>- * @cm_id: cm information for passive connection</div><div class='del'>- * @backlog: to max accept pending count</div><div class='del'>- */</div><div class='del'>-int i40iw_create_listen(struct iw_cm_id *cm_id, int backlog)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_cm_listener *cm_listen_node;</div><div class='del'>-	struct i40iw_cm_info cm_info;</div><div class='del'>-	enum i40iw_status_code ret;</div><div class='del'>-	struct sockaddr_in *laddr;</div><div class='del'>-	struct sockaddr_in6 *laddr6;</div><div class='del'>-	bool wildcard = false;</div><div class='del'>-</div><div class='del'>-	iwdev = to_iwdev(cm_id-&gt;device);</div><div class='del'>-	if (!iwdev)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	laddr = (struct sockaddr_in *)&amp;cm_id-&gt;m_local_addr;</div><div class='del'>-	laddr6 = (struct sockaddr_in6 *)&amp;cm_id-&gt;m_local_addr;</div><div class='del'>-	memset(&amp;cm_info, 0, sizeof(cm_info));</div><div class='del'>-	if (laddr-&gt;sin_family == AF_INET) {</div><div class='del'>-		cm_info.ipv4 = true;</div><div class='del'>-		cm_info.loc_addr[0] = ntohl(laddr-&gt;sin_addr.s_addr);</div><div class='del'>-		cm_info.loc_port = ntohs(laddr-&gt;sin_port);</div><div class='del'>-</div><div class='del'>-		if (laddr-&gt;sin_addr.s_addr != INADDR_ANY)</div><div class='del'>-			cm_info.vlan_id = i40iw_get_vlan_ipv4(cm_info.loc_addr);</div><div class='del'>-		else</div><div class='del'>-			wildcard = true;</div><div class='del'>-</div><div class='del'>-	} else {</div><div class='del'>-		cm_info.ipv4 = false;</div><div class='del'>-		i40iw_copy_ip_ntohl(cm_info.loc_addr,</div><div class='del'>-				    laddr6-&gt;sin6_addr.in6_u.u6_addr32);</div><div class='del'>-		cm_info.loc_port = ntohs(laddr6-&gt;sin6_port);</div><div class='del'>-		if (ipv6_addr_type(&amp;laddr6-&gt;sin6_addr) != IPV6_ADDR_ANY)</div><div class='del'>-			i40iw_netdev_vlan_ipv6(cm_info.loc_addr,</div><div class='del'>-					       &amp;cm_info.vlan_id);</div><div class='del'>-		else</div><div class='del'>-			wildcard = true;</div><div class='del'>-	}</div><div class='del'>-	cm_info.backlog = backlog;</div><div class='del'>-	cm_info.cm_id = cm_id;</div><div class='del'>-</div><div class='del'>-	cm_listen_node = i40iw_make_listen_node(&amp;iwdev-&gt;cm_core, iwdev, &amp;cm_info);</div><div class='del'>-	if (!cm_listen_node) {</div><div class='del'>-		i40iw_pr_err("cm_listen_node == NULL\n");</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cm_id-&gt;provider_data = cm_listen_node;</div><div class='del'>-</div><div class='del'>-	cm_listen_node-&gt;tos = cm_id-&gt;tos;</div><div class='del'>-	cm_listen_node-&gt;user_pri = rt_tos2priority(cm_id-&gt;tos);</div><div class='del'>-	cm_info.user_pri = cm_listen_node-&gt;user_pri;</div><div class='del'>-</div><div class='del'>-	if (!cm_listen_node-&gt;reused_node) {</div><div class='del'>-		if (wildcard) {</div><div class='del'>-			if (cm_info.ipv4)</div><div class='del'>-				ret = i40iw_add_mqh_4(iwdev,</div><div class='del'>-						      &amp;cm_info,</div><div class='del'>-						      cm_listen_node);</div><div class='del'>-			else</div><div class='del'>-				ret = i40iw_add_mqh_6(iwdev,</div><div class='del'>-						      &amp;cm_info,</div><div class='del'>-						      cm_listen_node);</div><div class='del'>-			if (ret)</div><div class='del'>-				goto error;</div><div class='del'>-</div><div class='del'>-			ret = i40iw_manage_apbvt(iwdev,</div><div class='del'>-						 cm_info.loc_port,</div><div class='del'>-						 I40IW_MANAGE_APBVT_ADD);</div><div class='del'>-</div><div class='del'>-			if (ret)</div><div class='del'>-				goto error;</div><div class='del'>-		} else {</div><div class='del'>-			ret = i40iw_manage_qhash(iwdev,</div><div class='del'>-						 &amp;cm_info,</div><div class='del'>-						 I40IW_QHASH_TYPE_TCP_SYN,</div><div class='del'>-						 I40IW_QHASH_MANAGE_TYPE_ADD,</div><div class='del'>-						 NULL,</div><div class='del'>-						 true);</div><div class='del'>-			if (ret)</div><div class='del'>-				goto error;</div><div class='del'>-			cm_listen_node-&gt;qhash_set = true;</div><div class='del'>-			ret = i40iw_manage_apbvt(iwdev,</div><div class='del'>-						 cm_info.loc_port,</div><div class='del'>-						 I40IW_MANAGE_APBVT_ADD);</div><div class='del'>-			if (ret)</div><div class='del'>-				goto error;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	cm_id-&gt;add_ref(cm_id);</div><div class='del'>-	cm_listen_node-&gt;cm_core-&gt;stats_listen_created++;</div><div class='del'>-	return 0;</div><div class='del'>- error:</div><div class='del'>-	i40iw_cm_del_listen(&amp;iwdev-&gt;cm_core, (void *)cm_listen_node, false);</div><div class='del'>-	return -EINVAL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_destroy_listen - registered call to destroy listener</div><div class='del'>- * @cm_id: cm information for passive connection</div><div class='del'>- */</div><div class='del'>-int i40iw_destroy_listen(struct iw_cm_id *cm_id)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-</div><div class='del'>-	iwdev = to_iwdev(cm_id-&gt;device);</div><div class='del'>-	if (cm_id-&gt;provider_data)</div><div class='del'>-		i40iw_cm_del_listen(&amp;iwdev-&gt;cm_core, cm_id-&gt;provider_data, true);</div><div class='del'>-	else</div><div class='del'>-		i40iw_pr_err("cm_id-&gt;provider_data was NULL\n");</div><div class='del'>-</div><div class='del'>-	cm_id-&gt;rem_ref(cm_id);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cm_event_connected - handle connected active node</div><div class='del'>- * @event: the info for cm_node of connection</div><div class='del'>- */</div><div class='del'>-static void i40iw_cm_event_connected(struct i40iw_cm_event *event)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_cm_core *cm_core;</div><div class='del'>-	struct i40iw_cm_node *cm_node;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	struct ib_qp_attr attr;</div><div class='del'>-	struct iw_cm_id *cm_id;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int status;</div><div class='del'>-	bool read0;</div><div class='del'>-</div><div class='del'>-	cm_node = event-&gt;cm_node;</div><div class='del'>-	cm_id = cm_node-&gt;cm_id;</div><div class='del'>-	iwqp = (struct i40iw_qp *)cm_id-&gt;provider_data;</div><div class='del'>-	iwdev = to_iwdev(iwqp-&gt;ibqp.device);</div><div class='del'>-	dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	cm_core = &amp;iwdev-&gt;cm_core;</div><div class='del'>-</div><div class='del'>-	if (iwqp-&gt;destroyed) {</div><div class='del'>-		status = -ETIMEDOUT;</div><div class='del'>-		goto error;</div><div class='del'>-	}</div><div class='del'>-	i40iw_cm_init_tsa_conn(iwqp, cm_node);</div><div class='del'>-	read0 = (cm_node-&gt;send_rdma0_op == SEND_RDMA_READ_ZERO);</div><div class='del'>-	if (iwqp-&gt;page)</div><div class='del'>-		iwqp-&gt;sc_qp.qp_uk.sq_base = kmap(iwqp-&gt;page);</div><div class='del'>-	dev-&gt;iw_priv_qp_ops-&gt;qp_send_rtt(&amp;iwqp-&gt;sc_qp, read0);</div><div class='del'>-	if (iwqp-&gt;page)</div><div class='del'>-		kunmap(iwqp-&gt;page);</div><div class='del'>-</div><div class='del'>-	memset(&amp;attr, 0, sizeof(attr));</div><div class='del'>-	attr.qp_state = IB_QPS_RTS;</div><div class='del'>-	cm_node-&gt;qhash_set = false;</div><div class='del'>-	i40iw_modify_qp(&amp;iwqp-&gt;ibqp, &amp;attr, IB_QP_STATE, NULL);</div><div class='del'>-</div><div class='del'>-	cm_node-&gt;accelerated = true;</div><div class='del'>-	spin_lock_irqsave(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-	list_move_tail(&amp;cm_node-&gt;list, &amp;cm_core-&gt;accelerated_list);</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-	status = i40iw_send_cm_event(cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY,</div><div class='del'>-				     0);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_CM, "error sending cm event - CONNECT_REPLY\n");</div><div class='del'>-</div><div class='del'>-	return;</div><div class='del'>-</div><div class='del'>-error:</div><div class='del'>-	iwqp-&gt;cm_id = NULL;</div><div class='del'>-	cm_id-&gt;provider_data = NULL;</div><div class='del'>-	i40iw_send_cm_event(event-&gt;cm_node,</div><div class='del'>-			    cm_id,</div><div class='del'>-			    IW_CM_EVENT_CONNECT_REPLY,</div><div class='del'>-			    status);</div><div class='del'>-	cm_id-&gt;rem_ref(cm_id);</div><div class='del'>-	i40iw_rem_ref_cm_node(event-&gt;cm_node);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cm_event_reset - handle reset</div><div class='del'>- * @event: the info for cm_node of connection</div><div class='del'>- */</div><div class='del'>-static void i40iw_cm_event_reset(struct i40iw_cm_event *event)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_node *cm_node = event-&gt;cm_node;</div><div class='del'>-	struct iw_cm_id   *cm_id = cm_node-&gt;cm_id;</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-</div><div class='del'>-	if (!cm_id)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	iwqp = cm_id-&gt;provider_data;</div><div class='del'>-	if (!iwqp)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	i40iw_debug(cm_node-&gt;dev,</div><div class='del'>-		    I40IW_DEBUG_CM,</div><div class='del'>-		    "reset event %p - cm_id = %p\n",</div><div class='del'>-		     event-&gt;cm_node, cm_id);</div><div class='del'>-	iwqp-&gt;cm_id = NULL;</div><div class='del'>-</div><div class='del'>-	i40iw_send_cm_event(cm_node, cm_node-&gt;cm_id, IW_CM_EVENT_DISCONNECT, -ECONNRESET);</div><div class='del'>-	i40iw_send_cm_event(cm_node, cm_node-&gt;cm_id, IW_CM_EVENT_CLOSE, 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cm_event_handler - worker thread callback to send event to cm upper layer</div><div class='del'>- * @work: pointer of cm event info.</div><div class='del'>- */</div><div class='del'>-static void i40iw_cm_event_handler(struct work_struct *work)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_event *event = container_of(work,</div><div class='del'>-						    struct i40iw_cm_event,</div><div class='del'>-						    event_work);</div><div class='del'>-	struct i40iw_cm_node *cm_node;</div><div class='del'>-</div><div class='del'>-	if (!event || !event-&gt;cm_node || !event-&gt;cm_node-&gt;cm_core)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	cm_node = event-&gt;cm_node;</div><div class='del'>-</div><div class='del'>-	switch (event-&gt;type) {</div><div class='del'>-	case I40IW_CM_EVENT_MPA_REQ:</div><div class='del'>-		i40iw_send_cm_event(cm_node,</div><div class='del'>-				    cm_node-&gt;cm_id,</div><div class='del'>-				    IW_CM_EVENT_CONNECT_REQUEST,</div><div class='del'>-				    0);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_EVENT_RESET:</div><div class='del'>-		i40iw_cm_event_reset(event);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_EVENT_CONNECTED:</div><div class='del'>-		if (!event-&gt;cm_node-&gt;cm_id ||</div><div class='del'>-		    (event-&gt;cm_node-&gt;state != I40IW_CM_STATE_OFFLOADED))</div><div class='del'>-			break;</div><div class='del'>-		i40iw_cm_event_connected(event);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_EVENT_MPA_REJECT:</div><div class='del'>-		if (!event-&gt;cm_node-&gt;cm_id ||</div><div class='del'>-		    (cm_node-&gt;state == I40IW_CM_STATE_OFFLOADED))</div><div class='del'>-			break;</div><div class='del'>-		i40iw_send_cm_event(cm_node,</div><div class='del'>-				    cm_node-&gt;cm_id,</div><div class='del'>-				    IW_CM_EVENT_CONNECT_REPLY,</div><div class='del'>-				    -ECONNREFUSED);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_CM_EVENT_ABORTED:</div><div class='del'>-		if (!event-&gt;cm_node-&gt;cm_id ||</div><div class='del'>-		    (event-&gt;cm_node-&gt;state == I40IW_CM_STATE_OFFLOADED))</div><div class='del'>-			break;</div><div class='del'>-		i40iw_event_connect_error(event);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		i40iw_pr_err("event type = %d\n", event-&gt;type);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	event-&gt;cm_info.cm_id-&gt;rem_ref(event-&gt;cm_info.cm_id);</div><div class='del'>-	i40iw_rem_ref_cm_node(event-&gt;cm_node);</div><div class='del'>-	kfree(event);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cm_post_event - queue event request for worker thread</div><div class='del'>- * @event: cm node's info for up event call</div><div class='del'>- */</div><div class='del'>-static void i40iw_cm_post_event(struct i40iw_cm_event *event)</div><div class='del'>-{</div><div class='del'>-	atomic_inc(&amp;event-&gt;cm_node-&gt;ref_count);</div><div class='del'>-	event-&gt;cm_info.cm_id-&gt;add_ref(event-&gt;cm_info.cm_id);</div><div class='del'>-	INIT_WORK(&amp;event-&gt;event_work, i40iw_cm_event_handler);</div><div class='del'>-</div><div class='del'>-	queue_work(event-&gt;cm_node-&gt;cm_core-&gt;event_wq, &amp;event-&gt;event_work);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_qhash_ctrl - enable/disable qhash for list</div><div class='del'>- * @iwdev: device pointer</div><div class='del'>- * @parent_listen_node: parent listen node</div><div class='del'>- * @nfo: cm info node</div><div class='del'>- * @ipaddr: Pointer to IPv4 or IPv6 address</div><div class='del'>- * @ipv4: flag indicating IPv4 when true</div><div class='del'>- * @ifup: flag indicating interface up when true</div><div class='del'>- *</div><div class='del'>- * Enables or disables the qhash for the node in the child</div><div class='del'>- * listen list that matches ipaddr. If no matching IP was found</div><div class='del'>- * it will allocate and add a new child listen node to the</div><div class='del'>- * parent listen node. The listen_list_lock is assumed to be</div><div class='del'>- * held when called.</div><div class='del'>- */</div><div class='del'>-static void i40iw_qhash_ctrl(struct i40iw_device *iwdev,</div><div class='del'>-			     struct i40iw_cm_listener *parent_listen_node,</div><div class='del'>-			     struct i40iw_cm_info *nfo,</div><div class='del'>-			     u32 *ipaddr, bool ipv4, bool ifup)</div><div class='del'>-{</div><div class='del'>-	struct list_head *child_listen_list = &amp;parent_listen_node-&gt;child_listen_list;</div><div class='del'>-	struct i40iw_cm_listener *child_listen_node;</div><div class='del'>-	struct list_head *pos, *tpos;</div><div class='del'>-	enum i40iw_status_code ret;</div><div class='del'>-	bool node_allocated = false;</div><div class='del'>-	enum i40iw_quad_hash_manage_type op =</div><div class='del'>-		ifup ? I40IW_QHASH_MANAGE_TYPE_ADD : I40IW_QHASH_MANAGE_TYPE_DELETE;</div><div class='del'>-</div><div class='del'>-	list_for_each_safe(pos, tpos, child_listen_list) {</div><div class='del'>-		child_listen_node =</div><div class='del'>-			list_entry(pos,</div><div class='del'>-				   struct i40iw_cm_listener,</div><div class='del'>-				   child_listen_list);</div><div class='del'>-		if (!memcmp(child_listen_node-&gt;loc_addr, ipaddr, ipv4 ? 4 : 16))</div><div class='del'>-			goto set_qhash;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* if not found then add a child listener if interface is going up */</div><div class='del'>-	if (!ifup)</div><div class='del'>-		return;</div><div class='del'>-	child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_ATOMIC);</div><div class='del'>-	if (!child_listen_node)</div><div class='del'>-		return;</div><div class='del'>-	node_allocated = true;</div><div class='del'>-	memcpy(child_listen_node, parent_listen_node, sizeof(*child_listen_node));</div><div class='del'>-</div><div class='del'>-	memcpy(child_listen_node-&gt;loc_addr, ipaddr,  ipv4 ? 4 : 16);</div><div class='del'>-</div><div class='del'>-set_qhash:</div><div class='del'>-	memcpy(nfo-&gt;loc_addr,</div><div class='del'>-	       child_listen_node-&gt;loc_addr,</div><div class='del'>-	       sizeof(nfo-&gt;loc_addr));</div><div class='del'>-	nfo-&gt;vlan_id = child_listen_node-&gt;vlan_id;</div><div class='del'>-	ret = i40iw_manage_qhash(iwdev, nfo,</div><div class='del'>-				 I40IW_QHASH_TYPE_TCP_SYN,</div><div class='del'>-				 op,</div><div class='del'>-				 NULL, false);</div><div class='del'>-	if (!ret) {</div><div class='del'>-		child_listen_node-&gt;qhash_set = ifup;</div><div class='del'>-		if (node_allocated)</div><div class='del'>-			list_add(&amp;child_listen_node-&gt;child_listen_list,</div><div class='del'>-				 &amp;parent_listen_node-&gt;child_listen_list);</div><div class='del'>-	} else if (node_allocated) {</div><div class='del'>-		kfree(child_listen_node);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cm_teardown_connections - teardown QPs</div><div class='del'>- * @iwdev: device pointer</div><div class='del'>- * @ipaddr: Pointer to IPv4 or IPv6 address</div><div class='del'>- * @ipv4: flag indicating IPv4 when true</div><div class='del'>- * @disconnect_all: flag indicating disconnect all QPs</div><div class='del'>- * teardown QPs where source or destination addr matches ip addr</div><div class='del'>- */</div><div class='del'>-void i40iw_cm_teardown_connections(struct i40iw_device *iwdev, u32 *ipaddr,</div><div class='del'>-				   struct i40iw_cm_info *nfo,</div><div class='del'>-				   bool disconnect_all)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_core *cm_core = &amp;iwdev-&gt;cm_core;</div><div class='del'>-	struct list_head *list_core_temp;</div><div class='del'>-	struct list_head *list_node;</div><div class='del'>-	struct i40iw_cm_node *cm_node;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	struct list_head teardown_list;</div><div class='del'>-	struct ib_qp_attr attr;</div><div class='del'>-</div><div class='del'>-	INIT_LIST_HEAD(&amp;teardown_list);</div><div class='del'>-	spin_lock_irqsave(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-	list_for_each_safe(list_node, list_core_temp,</div><div class='del'>-			   &amp;cm_core-&gt;accelerated_list) {</div><div class='del'>-		cm_node = container_of(list_node, struct i40iw_cm_node, list);</div><div class='del'>-		if (disconnect_all ||</div><div class='del'>-		    (nfo-&gt;vlan_id == cm_node-&gt;vlan_id &amp;&amp;</div><div class='del'>-		    (!memcmp(cm_node-&gt;loc_addr, ipaddr, nfo-&gt;ipv4 ? 4 : 16) ||</div><div class='del'>-		     !memcmp(cm_node-&gt;rem_addr, ipaddr, nfo-&gt;ipv4 ? 4 : 16)))) {</div><div class='del'>-			atomic_inc(&amp;cm_node-&gt;ref_count);</div><div class='del'>-			list_add(&amp;cm_node-&gt;teardown_entry, &amp;teardown_list);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	list_for_each_safe(list_node, list_core_temp,</div><div class='del'>-			   &amp;cm_core-&gt;non_accelerated_list) {</div><div class='del'>-		cm_node = container_of(list_node, struct i40iw_cm_node, list);</div><div class='del'>-		if (disconnect_all ||</div><div class='del'>-		    (nfo-&gt;vlan_id == cm_node-&gt;vlan_id &amp;&amp;</div><div class='del'>-		    (!memcmp(cm_node-&gt;loc_addr, ipaddr, nfo-&gt;ipv4 ? 4 : 16) ||</div><div class='del'>-		     !memcmp(cm_node-&gt;rem_addr, ipaddr, nfo-&gt;ipv4 ? 4 : 16)))) {</div><div class='del'>-			atomic_inc(&amp;cm_node-&gt;ref_count);</div><div class='del'>-			list_add(&amp;cm_node-&gt;teardown_entry, &amp;teardown_list);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='del'>-</div><div class='del'>-	list_for_each_safe(list_node, list_core_temp, &amp;teardown_list) {</div><div class='del'>-		cm_node = container_of(list_node, struct i40iw_cm_node,</div><div class='del'>-				       teardown_entry);</div><div class='del'>-		attr.qp_state = IB_QPS_ERR;</div><div class='del'>-		i40iw_modify_qp(&amp;cm_node-&gt;iwqp-&gt;ibqp, &amp;attr, IB_QP_STATE, NULL);</div><div class='del'>-		if (iwdev-&gt;reset)</div><div class='del'>-			i40iw_cm_disconn(cm_node-&gt;iwqp);</div><div class='del'>-		i40iw_rem_ref_cm_node(cm_node);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ifdown_notify - process an ifdown on an interface</div><div class='del'>- * @iwdev: device pointer</div><div class='del'>- * @ipaddr: Pointer to IPv4 or IPv6 address</div><div class='del'>- * @ipv4: flag indicating IPv4 when true</div><div class='del'>- * @ifup: flag indicating interface up when true</div><div class='del'>- */</div><div class='del'>-void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev,</div><div class='del'>-		     u32 *ipaddr, bool ipv4, bool ifup)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_core *cm_core = &amp;iwdev-&gt;cm_core;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	struct i40iw_cm_listener *listen_node;</div><div class='del'>-	static const u32 ip_zero[4] = { 0, 0, 0, 0 };</div><div class='del'>-	struct i40iw_cm_info nfo;</div><div class='del'>-	u16 vlan_id = rdma_vlan_dev_vlan_id(netdev);</div><div class='del'>-	enum i40iw_status_code ret;</div><div class='del'>-	enum i40iw_quad_hash_manage_type op =</div><div class='del'>-		ifup ? I40IW_QHASH_MANAGE_TYPE_ADD : I40IW_QHASH_MANAGE_TYPE_DELETE;</div><div class='del'>-</div><div class='del'>-	nfo.vlan_id = vlan_id;</div><div class='del'>-	nfo.ipv4 = ipv4;</div><div class='del'>-</div><div class='del'>-	/* Disable or enable qhash for listeners */</div><div class='del'>-	spin_lock_irqsave(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='del'>-	list_for_each_entry(listen_node, &amp;cm_core-&gt;listen_nodes, list) {</div><div class='del'>-		if (vlan_id == listen_node-&gt;vlan_id &amp;&amp;</div><div class='del'>-		    (!memcmp(listen_node-&gt;loc_addr, ipaddr, ipv4 ? 4 : 16) ||</div><div class='del'>-		    !memcmp(listen_node-&gt;loc_addr, ip_zero, ipv4 ? 4 : 16))) {</div><div class='del'>-			memcpy(nfo.loc_addr, listen_node-&gt;loc_addr,</div><div class='del'>-			       sizeof(nfo.loc_addr));</div><div class='del'>-			nfo.loc_port = listen_node-&gt;loc_port;</div><div class='del'>-			nfo.user_pri = listen_node-&gt;user_pri;</div><div class='del'>-			if (!list_empty(&amp;listen_node-&gt;child_listen_list)) {</div><div class='del'>-				i40iw_qhash_ctrl(iwdev,</div><div class='del'>-						 listen_node,</div><div class='del'>-						 &amp;nfo,</div><div class='del'>-						 ipaddr, ipv4, ifup);</div><div class='del'>-			} else if (memcmp(listen_node-&gt;loc_addr, ip_zero,</div><div class='del'>-					  ipv4 ? 4 : 16)) {</div><div class='del'>-				ret = i40iw_manage_qhash(iwdev,</div><div class='del'>-							 &amp;nfo,</div><div class='del'>-							 I40IW_QHASH_TYPE_TCP_SYN,</div><div class='del'>-							 op,</div><div class='del'>-							 NULL,</div><div class='del'>-							 false);</div><div class='del'>-				if (!ret)</div><div class='del'>-					listen_node-&gt;qhash_set = ifup;</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='del'>-</div><div class='del'>-	/* teardown connected qp's on ifdown */</div><div class='del'>-	if (!ifup)</div><div class='del'>-		i40iw_cm_teardown_connections(iwdev, ipaddr, &amp;nfo, false);</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.h b/drivers/infiniband/hw/i40iw/i40iw_cm.h<br/>deleted file mode 100644<br/>index 66dc1ba03389..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_cm.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_cm.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,462 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#ifndef I40IW_CM_H</div><div class='del'>-#define I40IW_CM_H</div><div class='del'>-</div><div class='del'>-#define QUEUE_EVENTS</div><div class='del'>-</div><div class='del'>-#define I40IW_MANAGE_APBVT_DEL 0</div><div class='del'>-#define I40IW_MANAGE_APBVT_ADD 1</div><div class='del'>-</div><div class='del'>-#define I40IW_MPA_REQUEST_ACCEPT  1</div><div class='del'>-#define I40IW_MPA_REQUEST_REJECT  2</div><div class='del'>-</div><div class='del'>-/* IETF MPA -- defines, enums, structs */</div><div class='del'>-#define IEFT_MPA_KEY_REQ  "MPA ID Req Frame"</div><div class='del'>-#define IEFT_MPA_KEY_REP  "MPA ID Rep Frame"</div><div class='del'>-#define IETF_MPA_KEY_SIZE 16</div><div class='del'>-#define IETF_MPA_VERSION  1</div><div class='del'>-#define IETF_MAX_PRIV_DATA_LEN 512</div><div class='del'>-#define IETF_MPA_FRAME_SIZE    20</div><div class='del'>-#define IETF_RTR_MSG_SIZE      4</div><div class='del'>-#define IETF_MPA_V2_FLAG       0x10</div><div class='del'>-#define SNDMARKER_SEQNMASK     0x000001FF</div><div class='del'>-</div><div class='del'>-#define I40IW_MAX_IETF_SIZE      32</div><div class='del'>-</div><div class='del'>-/* IETF RTR MSG Fields               */</div><div class='del'>-#define IETF_PEER_TO_PEER       0x8000</div><div class='del'>-#define IETF_FLPDU_ZERO_LEN     0x4000</div><div class='del'>-#define IETF_RDMA0_WRITE        0x8000</div><div class='del'>-#define IETF_RDMA0_READ         0x4000</div><div class='del'>-#define IETF_NO_IRD_ORD         0x3FFF</div><div class='del'>-</div><div class='del'>-/* HW-supported IRD sizes*/</div><div class='del'>-#define	I40IW_HW_IRD_SETTING_2	2</div><div class='del'>-#define	I40IW_HW_IRD_SETTING_4	4</div><div class='del'>-#define	I40IW_HW_IRD_SETTING_8	8</div><div class='del'>-#define	I40IW_HW_IRD_SETTING_16	16</div><div class='del'>-#define	I40IW_HW_IRD_SETTING_32	32</div><div class='del'>-#define	I40IW_HW_IRD_SETTING_64	64</div><div class='del'>-</div><div class='del'>-#define MAX_PORTS		65536</div><div class='del'>-#define I40IW_VLAN_PRIO_SHIFT   13</div><div class='del'>-</div><div class='del'>-enum ietf_mpa_flags {</div><div class='del'>-	IETF_MPA_FLAGS_MARKERS = 0x80,	/* receive Markers */</div><div class='del'>-	IETF_MPA_FLAGS_CRC = 0x40,	/* receive Markers */</div><div class='del'>-	IETF_MPA_FLAGS_REJECT = 0x20,	/* Reject */</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct ietf_mpa_v1 {</div><div class='del'>-	u8 key[IETF_MPA_KEY_SIZE];</div><div class='del'>-	u8 flags;</div><div class='del'>-	u8 rev;</div><div class='del'>-	__be16 priv_data_len;</div><div class='del'>-	u8 priv_data[0];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define ietf_mpa_req_resp_frame ietf_mpa_frame</div><div class='del'>-</div><div class='del'>-struct ietf_rtr_msg {</div><div class='del'>-	__be16 ctrl_ird;</div><div class='del'>-	__be16 ctrl_ord;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct ietf_mpa_v2 {</div><div class='del'>-	u8 key[IETF_MPA_KEY_SIZE];</div><div class='del'>-	u8 flags;</div><div class='del'>-	u8 rev;</div><div class='del'>-	__be16 priv_data_len;</div><div class='del'>-	struct ietf_rtr_msg rtr_msg;</div><div class='del'>-	u8 priv_data[0];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cm_node;</div><div class='del'>-enum i40iw_timer_type {</div><div class='del'>-	I40IW_TIMER_TYPE_SEND,</div><div class='del'>-	I40IW_TIMER_TYPE_RECV,</div><div class='del'>-	I40IW_TIMER_NODE_CLEANUP,</div><div class='del'>-	I40IW_TIMER_TYPE_CLOSE,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define I40IW_PASSIVE_STATE_INDICATED    0</div><div class='del'>-#define I40IW_DO_NOT_SEND_RESET_EVENT    1</div><div class='del'>-#define I40IW_SEND_RESET_EVENT           2</div><div class='del'>-</div><div class='del'>-#define MAX_I40IW_IFS 4</div><div class='del'>-</div><div class='del'>-#define SET_ACK 0x1</div><div class='del'>-#define SET_SYN 0x2</div><div class='del'>-#define SET_FIN 0x4</div><div class='del'>-#define SET_RST 0x8</div><div class='del'>-</div><div class='del'>-#define TCP_OPTIONS_PADDING     3</div><div class='del'>-</div><div class='del'>-struct option_base {</div><div class='del'>-	u8 optionnum;</div><div class='del'>-	u8 length;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum option_numbers {</div><div class='del'>-	OPTION_NUMBER_END,</div><div class='del'>-	OPTION_NUMBER_NONE,</div><div class='del'>-	OPTION_NUMBER_MSS,</div><div class='del'>-	OPTION_NUMBER_WINDOW_SCALE,</div><div class='del'>-	OPTION_NUMBER_SACK_PERM,</div><div class='del'>-	OPTION_NUMBER_SACK,</div><div class='del'>-	OPTION_NUMBER_WRITE0 = 0xbc</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct option_mss {</div><div class='del'>-	u8 optionnum;</div><div class='del'>-	u8 length;</div><div class='del'>-	__be16 mss;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct option_windowscale {</div><div class='del'>-	u8 optionnum;</div><div class='del'>-	u8 length;</div><div class='del'>-	u8 shiftcount;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-union all_known_options {</div><div class='del'>-	char as_end;</div><div class='del'>-	struct option_base as_base;</div><div class='del'>-	struct option_mss as_mss;</div><div class='del'>-	struct option_windowscale as_windowscale;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_timer_entry {</div><div class='del'>-	struct list_head list;</div><div class='del'>-	unsigned long timetosend;	/* jiffies */</div><div class='del'>-	struct i40iw_puda_buf *sqbuf;</div><div class='del'>-	u32 type;</div><div class='del'>-	u32 retrycount;</div><div class='del'>-	u32 retranscount;</div><div class='del'>-	u32 context;</div><div class='del'>-	u32 send_retrans;</div><div class='del'>-	int close_when_complete;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define I40IW_DEFAULT_RETRYS	64</div><div class='del'>-#define I40IW_DEFAULT_RETRANS	8</div><div class='del'>-#define I40IW_DEFAULT_TTL	0x40</div><div class='del'>-#define I40IW_DEFAULT_RTT_VAR	0x6</div><div class='del'>-#define I40IW_DEFAULT_SS_THRESH 0x3FFFFFFF</div><div class='del'>-#define I40IW_DEFAULT_REXMIT_THRESH 8</div><div class='del'>-</div><div class='del'>-#define I40IW_RETRY_TIMEOUT   HZ</div><div class='del'>-#define I40IW_SHORT_TIME      10</div><div class='del'>-#define I40IW_LONG_TIME       (2 * HZ)</div><div class='del'>-#define I40IW_MAX_TIMEOUT     ((unsigned long)(12 * HZ))</div><div class='del'>-</div><div class='del'>-#define I40IW_CM_HASHTABLE_SIZE         1024</div><div class='del'>-#define I40IW_CM_TCP_TIMER_INTERVAL     3000</div><div class='del'>-#define I40IW_CM_DEFAULT_MTU            1540</div><div class='del'>-#define I40IW_CM_DEFAULT_FRAME_CNT      10</div><div class='del'>-#define I40IW_CM_THREAD_STACK_SIZE      256</div><div class='del'>-#define I40IW_CM_DEFAULT_RCV_WND        64240</div><div class='del'>-#define I40IW_CM_DEFAULT_RCV_WND_SCALED 0x3fffc</div><div class='del'>-#define I40IW_CM_DEFAULT_RCV_WND_SCALE  2</div><div class='del'>-#define I40IW_CM_DEFAULT_FREE_PKTS      0x000A</div><div class='del'>-#define I40IW_CM_FREE_PKT_LO_WATERMARK  2</div><div class='del'>-</div><div class='del'>-#define I40IW_CM_DEFAULT_MSS   536</div><div class='del'>-</div><div class='del'>-#define I40IW_CM_DEF_SEQ       0x159bf75f</div><div class='del'>-#define I40IW_CM_DEF_LOCAL_ID  0x3b47</div><div class='del'>-</div><div class='del'>-#define I40IW_CM_DEF_SEQ2      0x18ed5740</div><div class='del'>-#define I40IW_CM_DEF_LOCAL_ID2 0xb807</div><div class='del'>-#define MAX_CM_BUFFER   (I40IW_MAX_IETF_SIZE + IETF_MAX_PRIV_DATA_LEN)</div><div class='del'>-</div><div class='del'>-typedef u32 i40iw_addr_t;</div><div class='del'>-</div><div class='del'>-#define i40iw_cm_tsa_context i40iw_qp_context</div><div class='del'>-</div><div class='del'>-struct i40iw_qp;</div><div class='del'>-</div><div class='del'>-/* cm node transition states */</div><div class='del'>-enum i40iw_cm_node_state {</div><div class='del'>-	I40IW_CM_STATE_UNKNOWN,</div><div class='del'>-	I40IW_CM_STATE_INITED,</div><div class='del'>-	I40IW_CM_STATE_LISTENING,</div><div class='del'>-	I40IW_CM_STATE_SYN_RCVD,</div><div class='del'>-	I40IW_CM_STATE_SYN_SENT,</div><div class='del'>-	I40IW_CM_STATE_ONE_SIDE_ESTABLISHED,</div><div class='del'>-	I40IW_CM_STATE_ESTABLISHED,</div><div class='del'>-	I40IW_CM_STATE_ACCEPTING,</div><div class='del'>-	I40IW_CM_STATE_MPAREQ_SENT,</div><div class='del'>-	I40IW_CM_STATE_MPAREQ_RCVD,</div><div class='del'>-	I40IW_CM_STATE_MPAREJ_RCVD,</div><div class='del'>-	I40IW_CM_STATE_OFFLOADED,</div><div class='del'>-	I40IW_CM_STATE_FIN_WAIT1,</div><div class='del'>-	I40IW_CM_STATE_FIN_WAIT2,</div><div class='del'>-	I40IW_CM_STATE_CLOSE_WAIT,</div><div class='del'>-	I40IW_CM_STATE_TIME_WAIT,</div><div class='del'>-	I40IW_CM_STATE_LAST_ACK,</div><div class='del'>-	I40IW_CM_STATE_CLOSING,</div><div class='del'>-	I40IW_CM_STATE_LISTENER_DESTROYED,</div><div class='del'>-	I40IW_CM_STATE_CLOSED</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum mpa_frame_version {</div><div class='del'>-	IETF_MPA_V1 = 1,</div><div class='del'>-	IETF_MPA_V2 = 2</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum mpa_frame_key {</div><div class='del'>-	MPA_KEY_REQUEST,</div><div class='del'>-	MPA_KEY_REPLY</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum send_rdma0 {</div><div class='del'>-	SEND_RDMA_READ_ZERO = 1,</div><div class='del'>-	SEND_RDMA_WRITE_ZERO = 2</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_tcpip_pkt_type {</div><div class='del'>-	I40IW_PKT_TYPE_UNKNOWN,</div><div class='del'>-	I40IW_PKT_TYPE_SYN,</div><div class='del'>-	I40IW_PKT_TYPE_SYNACK,</div><div class='del'>-	I40IW_PKT_TYPE_ACK,</div><div class='del'>-	I40IW_PKT_TYPE_FIN,</div><div class='del'>-	I40IW_PKT_TYPE_RST</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* CM context params */</div><div class='del'>-struct i40iw_cm_tcp_context {</div><div class='del'>-	u8 client;</div><div class='del'>-</div><div class='del'>-	u32 loc_seq_num;</div><div class='del'>-	u32 loc_ack_num;</div><div class='del'>-	u32 rem_ack_num;</div><div class='del'>-	u32 rcv_nxt;</div><div class='del'>-</div><div class='del'>-	u32 loc_id;</div><div class='del'>-	u32 rem_id;</div><div class='del'>-</div><div class='del'>-	u32 snd_wnd;</div><div class='del'>-	u32 max_snd_wnd;</div><div class='del'>-</div><div class='del'>-	u32 rcv_wnd;</div><div class='del'>-	u32 mss;</div><div class='del'>-	u8 snd_wscale;</div><div class='del'>-	u8 rcv_wscale;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_cm_listener_state {</div><div class='del'>-	I40IW_CM_LISTENER_PASSIVE_STATE = 1,</div><div class='del'>-	I40IW_CM_LISTENER_ACTIVE_STATE = 2,</div><div class='del'>-	I40IW_CM_LISTENER_EITHER_STATE = 3</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cm_listener {</div><div class='del'>-	struct list_head list;</div><div class='del'>-	struct i40iw_cm_core *cm_core;</div><div class='del'>-	u8 loc_mac[ETH_ALEN];</div><div class='del'>-	u32 loc_addr[4];</div><div class='del'>-	u16 loc_port;</div><div class='del'>-	struct iw_cm_id *cm_id;</div><div class='del'>-	atomic_t ref_count;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	atomic_t pend_accepts_cnt;</div><div class='del'>-	int backlog;</div><div class='del'>-	enum i40iw_cm_listener_state listener_state;</div><div class='del'>-	u32 reused_node;</div><div class='del'>-	u8 user_pri;</div><div class='del'>-	u8 tos;</div><div class='del'>-	u16 vlan_id;</div><div class='del'>-	bool qhash_set;</div><div class='del'>-	bool ipv4;</div><div class='del'>-	struct list_head child_listen_list;</div><div class='del'>-</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_kmem_info {</div><div class='del'>-	void *addr;</div><div class='del'>-	u32 size;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* per connection node and node state information */</div><div class='del'>-struct i40iw_cm_node {</div><div class='del'>-	u32 loc_addr[4], rem_addr[4];</div><div class='del'>-	u16 loc_port, rem_port;</div><div class='del'>-	u16 vlan_id;</div><div class='del'>-	enum i40iw_cm_node_state state;</div><div class='del'>-	u8 loc_mac[ETH_ALEN];</div><div class='del'>-	u8 rem_mac[ETH_ALEN];</div><div class='del'>-	atomic_t ref_count;</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	struct i40iw_cm_tcp_context tcp_cntxt;</div><div class='del'>-	struct i40iw_cm_core *cm_core;</div><div class='del'>-	struct i40iw_cm_node *loopbackpartner;</div><div class='del'>-	struct i40iw_timer_entry *send_entry;</div><div class='del'>-	struct i40iw_timer_entry *close_entry;</div><div class='del'>-	spinlock_t retrans_list_lock; /* cm transmit packet */</div><div class='del'>-	enum send_rdma0 send_rdma0_op;</div><div class='del'>-	u16 ird_size;</div><div class='del'>-	u16 ord_size;</div><div class='del'>-	u16     mpav2_ird_ord;</div><div class='del'>-	struct iw_cm_id *cm_id;</div><div class='del'>-	struct list_head list;</div><div class='del'>-	bool accelerated;</div><div class='del'>-	struct i40iw_cm_listener *listener;</div><div class='del'>-	int apbvt_set;</div><div class='del'>-	int accept_pend;</div><div class='del'>-	struct list_head timer_entry;</div><div class='del'>-	struct list_head reset_entry;</div><div class='del'>-	struct list_head teardown_entry;</div><div class='del'>-	atomic_t passive_state;</div><div class='del'>-	bool qhash_set;</div><div class='del'>-	u8 user_pri;</div><div class='del'>-	u8 tos;</div><div class='del'>-	bool ipv4;</div><div class='del'>-	bool snd_mark_en;</div><div class='del'>-	u16 lsmm_size;</div><div class='del'>-	enum mpa_frame_version mpa_frame_rev;</div><div class='del'>-	struct i40iw_kmem_info pdata;</div><div class='del'>-	union {</div><div class='del'>-		struct ietf_mpa_v1 mpa_frame;</div><div class='del'>-		struct ietf_mpa_v2 mpa_v2_frame;</div><div class='del'>-	};</div><div class='del'>-</div><div class='del'>-	u8 pdata_buf[IETF_MAX_PRIV_DATA_LEN];</div><div class='del'>-	struct i40iw_kmem_info mpa_hdr;</div><div class='del'>-	bool ack_rcvd;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* structure for client or CM to fill when making CM api calls. */</div><div class='del'>-/*	- only need to set relevant data, based on op. */</div><div class='del'>-struct i40iw_cm_info {</div><div class='del'>-	struct iw_cm_id *cm_id;</div><div class='del'>-	u16 loc_port;</div><div class='del'>-	u16 rem_port;</div><div class='del'>-	u32 loc_addr[4];</div><div class='del'>-	u32 rem_addr[4];</div><div class='del'>-	u16 vlan_id;</div><div class='del'>-	int backlog;</div><div class='del'>-	u8 user_pri;</div><div class='del'>-	u8 tos;</div><div class='del'>-	bool ipv4;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* CM event codes */</div><div class='del'>-enum i40iw_cm_event_type {</div><div class='del'>-	I40IW_CM_EVENT_UNKNOWN,</div><div class='del'>-	I40IW_CM_EVENT_ESTABLISHED,</div><div class='del'>-	I40IW_CM_EVENT_MPA_REQ,</div><div class='del'>-	I40IW_CM_EVENT_MPA_CONNECT,</div><div class='del'>-	I40IW_CM_EVENT_MPA_ACCEPT,</div><div class='del'>-	I40IW_CM_EVENT_MPA_REJECT,</div><div class='del'>-	I40IW_CM_EVENT_MPA_ESTABLISHED,</div><div class='del'>-	I40IW_CM_EVENT_CONNECTED,</div><div class='del'>-	I40IW_CM_EVENT_RESET,</div><div class='del'>-	I40IW_CM_EVENT_ABORTED</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* event to post to CM event handler */</div><div class='del'>-struct i40iw_cm_event {</div><div class='del'>-	enum i40iw_cm_event_type type;</div><div class='del'>-	struct i40iw_cm_info cm_info;</div><div class='del'>-	struct work_struct event_work;</div><div class='del'>-	struct i40iw_cm_node *cm_node;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cm_core {</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-</div><div class='del'>-	struct list_head listen_nodes;</div><div class='del'>-	struct list_head accelerated_list;</div><div class='del'>-	struct list_head non_accelerated_list;</div><div class='del'>-</div><div class='del'>-	struct timer_list tcp_timer;</div><div class='del'>-</div><div class='del'>-	struct workqueue_struct *event_wq;</div><div class='del'>-	struct workqueue_struct *disconn_wq;</div><div class='del'>-</div><div class='del'>-	spinlock_t ht_lock; /* manage hash table */</div><div class='del'>-	spinlock_t listen_list_lock; /* listen list */</div><div class='del'>-	spinlock_t apbvt_lock; /*manage apbvt entries*/</div><div class='del'>-</div><div class='del'>-	unsigned long ports_in_use[BITS_TO_LONGS(MAX_PORTS)];</div><div class='del'>-</div><div class='del'>-	u64	stats_nodes_created;</div><div class='del'>-	u64	stats_nodes_destroyed;</div><div class='del'>-	u64	stats_listen_created;</div><div class='del'>-	u64	stats_listen_destroyed;</div><div class='del'>-	u64	stats_listen_nodes_created;</div><div class='del'>-	u64	stats_listen_nodes_destroyed;</div><div class='del'>-	u64	stats_loopbacks;</div><div class='del'>-	u64	stats_accepts;</div><div class='del'>-	u64	stats_rejects;</div><div class='del'>-	u64	stats_connect_errs;</div><div class='del'>-	u64	stats_passive_errs;</div><div class='del'>-	u64	stats_pkt_retrans;</div><div class='del'>-	u64	stats_backlog_drops;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-int i40iw_schedule_cm_timer(struct i40iw_cm_node *cm_node,</div><div class='del'>-			    struct i40iw_puda_buf *sqbuf,</div><div class='del'>-			    enum i40iw_timer_type type,</div><div class='del'>-			    int send_retrans,</div><div class='del'>-			    int close_when_complete);</div><div class='del'>-</div><div class='del'>-int i40iw_accept(struct iw_cm_id *, struct iw_cm_conn_param *);</div><div class='del'>-int i40iw_reject(struct iw_cm_id *, const void *, u8);</div><div class='del'>-int i40iw_connect(struct iw_cm_id *, struct iw_cm_conn_param *);</div><div class='del'>-int i40iw_create_listen(struct iw_cm_id *, int);</div><div class='del'>-int i40iw_destroy_listen(struct iw_cm_id *);</div><div class='del'>-</div><div class='del'>-int i40iw_cm_start(struct i40iw_device *);</div><div class='del'>-int i40iw_cm_stop(struct i40iw_device *);</div><div class='del'>-</div><div class='del'>-int i40iw_arp_table(struct i40iw_device *iwdev,</div><div class='del'>-		    u32 *ip_addr,</div><div class='del'>-		    bool ipv4,</div><div class='del'>-		    u8 *mac_addr,</div><div class='del'>-		    u32 action);</div><div class='del'>-</div><div class='del'>-void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev,</div><div class='del'>-		     u32 *ipaddr, bool ipv4, bool ifup);</div><div class='del'>-void i40iw_cm_teardown_connections(struct i40iw_device *iwdev, u32 *ipaddr,</div><div class='del'>-				   struct i40iw_cm_info *nfo,</div><div class='del'>-				   bool disconnect_all);</div><div class='del'>-bool i40iw_port_in_use(struct i40iw_cm_core *cm_core, u16 port);</div><div class='del'>-#endif /* I40IW_CM_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c<br/>deleted file mode 100644<br/>index 4d841a3c68f3..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_ctrl.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_ctrl.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,5198 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#include "i40iw_osdep.h"</div><div class='del'>-#include "i40iw_register.h"</div><div class='del'>-#include "i40iw_status.h"</div><div class='del'>-#include "i40iw_hmc.h"</div><div class='del'>-</div><div class='del'>-#include "i40iw_d.h"</div><div class='del'>-#include "i40iw_type.h"</div><div class='del'>-#include "i40iw_p.h"</div><div class='del'>-#include "i40iw_vf.h"</div><div class='del'>-#include "i40iw_virtchnl.h"</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_insert_wqe_hdr - write wqe header</div><div class='del'>- * @wqe: cqp wqe for header</div><div class='del'>- * @header: header for the cqp wqe</div><div class='del'>- */</div><div class='del'>-void i40iw_insert_wqe_hdr(u64 *wqe, u64 header)</div><div class='del'>-{</div><div class='del'>-	wmb();            /* make sure WQE is populated before polarity is set */</div><div class='del'>-	set_64bit_val(wqe, 24, header);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void i40iw_check_cqp_progress(struct i40iw_cqp_timeout *cqp_timeout, struct i40iw_sc_dev *dev)</div><div class='del'>-{</div><div class='del'>-	if (cqp_timeout-&gt;compl_cqp_cmds != dev-&gt;cqp_cmd_stats[OP_COMPLETED_COMMANDS]) {</div><div class='del'>-		cqp_timeout-&gt;compl_cqp_cmds = dev-&gt;cqp_cmd_stats[OP_COMPLETED_COMMANDS];</div><div class='del'>-		cqp_timeout-&gt;count = 0;</div><div class='del'>-	} else {</div><div class='del'>-		if (dev-&gt;cqp_cmd_stats[OP_REQUESTED_COMMANDS] != cqp_timeout-&gt;compl_cqp_cmds)</div><div class='del'>-			cqp_timeout-&gt;count++;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_cqp_reg_info - get head and tail for cqp using registers</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @val: cqp tail register value</div><div class='del'>- * @tail:wqtail register value</div><div class='del'>- * @error: cqp processing err</div><div class='del'>- */</div><div class='del'>-static inline void i40iw_get_cqp_reg_info(struct i40iw_sc_cqp *cqp,</div><div class='del'>-					  u32 *val,</div><div class='del'>-					  u32 *tail,</div><div class='del'>-					  u32 *error)</div><div class='del'>-{</div><div class='del'>-	if (cqp-&gt;dev-&gt;is_pf) {</div><div class='del'>-		*val = i40iw_rd32(cqp-&gt;dev-&gt;hw, I40E_PFPE_CQPTAIL);</div><div class='del'>-		*tail = RS_32(*val, I40E_PFPE_CQPTAIL_WQTAIL);</div><div class='del'>-		*error = RS_32(*val, I40E_PFPE_CQPTAIL_CQP_OP_ERR);</div><div class='del'>-	} else {</div><div class='del'>-		*val = i40iw_rd32(cqp-&gt;dev-&gt;hw, I40E_VFPE_CQPTAIL1);</div><div class='del'>-		*tail = RS_32(*val, I40E_VFPE_CQPTAIL_WQTAIL);</div><div class='del'>-		*error = RS_32(*val, I40E_VFPE_CQPTAIL_CQP_OP_ERR);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cqp_poll_registers - poll cqp registers</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @tail:wqtail register value</div><div class='del'>- * @count: how many times to try for completion</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_cqp_poll_registers(</div><div class='del'>-						struct i40iw_sc_cqp *cqp,</div><div class='del'>-						u32 tail,</div><div class='del'>-						u32 count)</div><div class='del'>-{</div><div class='del'>-	u32 i = 0;</div><div class='del'>-	u32 newtail, error, val;</div><div class='del'>-</div><div class='del'>-	while (i &lt; count) {</div><div class='del'>-		i++;</div><div class='del'>-		i40iw_get_cqp_reg_info(cqp, &amp;val, &amp;newtail, &amp;error);</div><div class='del'>-		if (error) {</div><div class='del'>-			error = (cqp-&gt;dev-&gt;is_pf) ?</div><div class='del'>-				 i40iw_rd32(cqp-&gt;dev-&gt;hw, I40E_PFPE_CQPERRCODES) :</div><div class='del'>-				 i40iw_rd32(cqp-&gt;dev-&gt;hw, I40E_VFPE_CQPERRCODES1);</div><div class='del'>-			return I40IW_ERR_CQP_COMPL_ERROR;</div><div class='del'>-		}</div><div class='del'>-		if (newtail != tail) {</div><div class='del'>-			/* SUCCESS */</div><div class='del'>-			I40IW_RING_MOVE_TAIL(cqp-&gt;sq_ring);</div><div class='del'>-			cqp-&gt;dev-&gt;cqp_cmd_stats[OP_COMPLETED_COMMANDS]++;</div><div class='del'>-			return 0;</div><div class='del'>-		}</div><div class='del'>-		udelay(I40IW_SLEEP_COUNT);</div><div class='del'>-	}</div><div class='del'>-	return I40IW_ERR_TIMEOUT;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_parse_fpm_commit_buf - parse fpm commit buffer</div><div class='del'>- * @buf: ptr to fpm commit buffer</div><div class='del'>- * @info: ptr to i40iw_hmc_obj_info struct</div><div class='del'>- * @sd: number of SDs for HMC objects</div><div class='del'>- *</div><div class='del'>- * parses fpm commit info and copy base value</div><div class='del'>- * of hmc objects in hmc_info</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_parse_fpm_commit_buf(</div><div class='del'>-				u64 *buf,</div><div class='del'>-				struct i40iw_hmc_obj_info *info,</div><div class='del'>-				u32 *sd)</div><div class='del'>-{</div><div class='del'>-	u64 temp;</div><div class='del'>-	u64 size;</div><div class='del'>-	u64 base = 0;</div><div class='del'>-	u32 i, j;</div><div class='del'>-	u32 k = 0;</div><div class='del'>-</div><div class='del'>-	/* copy base values in obj_info */</div><div class='del'>-	for (i = I40IW_HMC_IW_QP, j = 0; i &lt;= I40IW_HMC_IW_PBLE; i++, j += 8) {</div><div class='del'>-		if ((i == I40IW_HMC_IW_SRQ) ||</div><div class='del'>-			(i == I40IW_HMC_IW_FSIMC) ||</div><div class='del'>-			(i == I40IW_HMC_IW_FSIAV)) {</div><div class='del'>-			info[i].base = 0;</div><div class='del'>-			info[i].cnt = 0;</div><div class='del'>-			continue;</div><div class='del'>-		}</div><div class='del'>-		get_64bit_val(buf, j, &amp;temp);</div><div class='del'>-		info[i].base = RS_64_1(temp, 32) * 512;</div><div class='del'>-		if (info[i].base &gt; base) {</div><div class='del'>-			base = info[i].base;</div><div class='del'>-			k = i;</div><div class='del'>-		}</div><div class='del'>-		if (i == I40IW_HMC_IW_APBVT_ENTRY) {</div><div class='del'>-			info[i].cnt = 1;</div><div class='del'>-			continue;</div><div class='del'>-		}</div><div class='del'>-		if (i == I40IW_HMC_IW_QP)</div><div class='del'>-			info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);</div><div class='del'>-		else if (i == I40IW_HMC_IW_CQ)</div><div class='del'>-			info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);</div><div class='del'>-		else</div><div class='del'>-			info[i].cnt = (u32)(temp);</div><div class='del'>-	}</div><div class='del'>-	size = info[k].cnt * info[k].size + info[k].base;</div><div class='del'>-	if (size &amp; 0x1FFFFF)</div><div class='del'>-		*sd = (u32)((size &gt;&gt; 21) + 1); /* add 1 for remainder */</div><div class='del'>-	else</div><div class='del'>-		*sd = (u32)(size &gt;&gt; 21);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_decode_fpm_query() - Decode a 64 bit value into max count and size</div><div class='del'>- * @buf: ptr to fpm query buffer</div><div class='del'>- * @buf_idx: index into buf</div><div class='del'>- * @info: ptr to i40iw_hmc_obj_info struct</div><div class='del'>- * @rsrc_idx: resource index into info</div><div class='del'>- *</div><div class='del'>- * Decode a 64 bit value from fpm query buffer into max count and size</div><div class='del'>- */</div><div class='del'>-static u64 i40iw_sc_decode_fpm_query(u64 *buf,</div><div class='del'>-					    u32 buf_idx,</div><div class='del'>-					    struct i40iw_hmc_obj_info *obj_info,</div><div class='del'>-					    u32 rsrc_idx)</div><div class='del'>-{</div><div class='del'>-	u64 temp;</div><div class='del'>-	u32 size;</div><div class='del'>-</div><div class='del'>-	get_64bit_val(buf, buf_idx, &amp;temp);</div><div class='del'>-	obj_info[rsrc_idx].max_cnt = (u32)temp;</div><div class='del'>-	size = (u32)RS_64_1(temp, 32);</div><div class='del'>-	obj_info[rsrc_idx].size = LS_64_1(1, size);</div><div class='del'>-</div><div class='del'>-	return temp;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_parse_fpm_query_buf() - parses fpm query buffer</div><div class='del'>- * @buf: ptr to fpm query buffer</div><div class='del'>- * @info: ptr to i40iw_hmc_obj_info struct</div><div class='del'>- * @hmc_fpm_misc: ptr to fpm data</div><div class='del'>- *</div><div class='del'>- * parses fpm query buffer and copy max_cnt and</div><div class='del'>- * size value of hmc objects in hmc_info</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_parse_fpm_query_buf(</div><div class='del'>-				u64 *buf,</div><div class='del'>-				struct i40iw_hmc_info *hmc_info,</div><div class='del'>-				struct i40iw_hmc_fpm_misc *hmc_fpm_misc)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_hmc_obj_info *obj_info;</div><div class='del'>-	u64 temp;</div><div class='del'>-	u32 size;</div><div class='del'>-	u16 max_pe_sds;</div><div class='del'>-</div><div class='del'>-	obj_info = hmc_info-&gt;hmc_obj;</div><div class='del'>-</div><div class='del'>-	get_64bit_val(buf, 0, &amp;temp);</div><div class='del'>-	hmc_info-&gt;first_sd_index = (u16)RS_64(temp, I40IW_QUERY_FPM_FIRST_PE_SD_INDEX);</div><div class='del'>-	max_pe_sds = (u16)RS_64(temp, I40IW_QUERY_FPM_MAX_PE_SDS);</div><div class='del'>-</div><div class='del'>-	/* Reduce SD count for VFs by 1 to account for PBLE backing page rounding */</div><div class='del'>-	if (hmc_info-&gt;hmc_fn_id &gt;= I40IW_FIRST_VF_FPM_ID)</div><div class='del'>-		max_pe_sds--;</div><div class='del'>-	hmc_fpm_misc-&gt;max_sds = max_pe_sds;</div><div class='del'>-	hmc_info-&gt;sd_table.sd_cnt = max_pe_sds + hmc_info-&gt;first_sd_index;</div><div class='del'>-</div><div class='del'>-	get_64bit_val(buf, 8, &amp;temp);</div><div class='del'>-	obj_info[I40IW_HMC_IW_QP].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);</div><div class='del'>-	size = (u32)RS_64_1(temp, 32);</div><div class='del'>-	obj_info[I40IW_HMC_IW_QP].size = LS_64_1(1, size);</div><div class='del'>-</div><div class='del'>-	get_64bit_val(buf, 16, &amp;temp);</div><div class='del'>-	obj_info[I40IW_HMC_IW_CQ].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);</div><div class='del'>-	size = (u32)RS_64_1(temp, 32);</div><div class='del'>-	obj_info[I40IW_HMC_IW_CQ].size = LS_64_1(1, size);</div><div class='del'>-</div><div class='del'>-	i40iw_sc_decode_fpm_query(buf, 32, obj_info, I40IW_HMC_IW_HTE);</div><div class='del'>-	i40iw_sc_decode_fpm_query(buf, 40, obj_info, I40IW_HMC_IW_ARP);</div><div class='del'>-</div><div class='del'>-	obj_info[I40IW_HMC_IW_APBVT_ENTRY].size = 8192;</div><div class='del'>-	obj_info[I40IW_HMC_IW_APBVT_ENTRY].max_cnt = 1;</div><div class='del'>-</div><div class='del'>-	i40iw_sc_decode_fpm_query(buf, 48, obj_info, I40IW_HMC_IW_MR);</div><div class='del'>-	i40iw_sc_decode_fpm_query(buf, 56, obj_info, I40IW_HMC_IW_XF);</div><div class='del'>-</div><div class='del'>-	get_64bit_val(buf, 64, &amp;temp);</div><div class='del'>-	obj_info[I40IW_HMC_IW_XFFL].max_cnt = (u32)temp;</div><div class='del'>-	obj_info[I40IW_HMC_IW_XFFL].size = 4;</div><div class='del'>-	hmc_fpm_misc-&gt;xf_block_size = RS_64(temp, I40IW_QUERY_FPM_XFBLOCKSIZE);</div><div class='del'>-	if (!hmc_fpm_misc-&gt;xf_block_size)</div><div class='del'>-		return I40IW_ERR_INVALID_SIZE;</div><div class='del'>-</div><div class='del'>-	i40iw_sc_decode_fpm_query(buf, 72, obj_info, I40IW_HMC_IW_Q1);</div><div class='del'>-</div><div class='del'>-	get_64bit_val(buf, 80, &amp;temp);</div><div class='del'>-	obj_info[I40IW_HMC_IW_Q1FL].max_cnt = (u32)temp;</div><div class='del'>-	obj_info[I40IW_HMC_IW_Q1FL].size = 4;</div><div class='del'>-	hmc_fpm_misc-&gt;q1_block_size = RS_64(temp, I40IW_QUERY_FPM_Q1BLOCKSIZE);</div><div class='del'>-	if (!hmc_fpm_misc-&gt;q1_block_size)</div><div class='del'>-		return I40IW_ERR_INVALID_SIZE;</div><div class='del'>-</div><div class='del'>-	i40iw_sc_decode_fpm_query(buf, 88, obj_info, I40IW_HMC_IW_TIMER);</div><div class='del'>-</div><div class='del'>-	get_64bit_val(buf, 112, &amp;temp);</div><div class='del'>-	obj_info[I40IW_HMC_IW_PBLE].max_cnt = (u32)temp;</div><div class='del'>-	obj_info[I40IW_HMC_IW_PBLE].size = 8;</div><div class='del'>-</div><div class='del'>-	get_64bit_val(buf, 120, &amp;temp);</div><div class='del'>-	hmc_fpm_misc-&gt;max_ceqs = (u8)RS_64(temp, I40IW_QUERY_FPM_MAX_CEQS);</div><div class='del'>-	hmc_fpm_misc-&gt;ht_multiplier = RS_64(temp, I40IW_QUERY_FPM_HTMULTIPLIER);</div><div class='del'>-	hmc_fpm_misc-&gt;timer_bucket = RS_64(temp, I40IW_QUERY_FPM_TIMERBUCKET);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_fill_qos_list - Change all unknown qs handles to available ones</div><div class='del'>- * @qs_list: list of qs_handles to be fixed with valid qs_handles</div><div class='del'>- */</div><div class='del'>-static void i40iw_fill_qos_list(u16 *qs_list)</div><div class='del'>-{</div><div class='del'>-	u16 qshandle = qs_list[0];</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; I40IW_MAX_USER_PRIORITY; i++) {</div><div class='del'>-		if (qs_list[i] == QS_HANDLE_UNKNOWN)</div><div class='del'>-			qs_list[i] = qshandle;</div><div class='del'>-		else</div><div class='del'>-			qshandle = qs_list[i];</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_qp_from_entry - Given entry, get to the qp structure</div><div class='del'>- * @entry: Points to list of qp structure</div><div class='del'>- */</div><div class='del'>-static struct i40iw_sc_qp *i40iw_qp_from_entry(struct list_head *entry)</div><div class='del'>-{</div><div class='del'>-	if (!entry)</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	return (struct i40iw_sc_qp *)((char *)entry - offsetof(struct i40iw_sc_qp, list));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_qp - get the next qp from the list given current qp</div><div class='del'>- * @head: Listhead of qp's</div><div class='del'>- * @qp: current qp</div><div class='del'>- */</div><div class='del'>-static struct i40iw_sc_qp *i40iw_get_qp(struct list_head *head, struct i40iw_sc_qp *qp)</div><div class='del'>-{</div><div class='del'>-	struct list_head *entry = NULL;</div><div class='del'>-	struct list_head *lastentry;</div><div class='del'>-</div><div class='del'>-	if (list_empty(head))</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	if (!qp) {</div><div class='del'>-		entry = head-&gt;next;</div><div class='del'>-	} else {</div><div class='del'>-		lastentry = &amp;qp-&gt;list;</div><div class='del'>-		entry = (lastentry != head) ? lastentry-&gt;next : NULL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return i40iw_qp_from_entry(entry);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_change_l2params - given the new l2 parameters, change all qp</div><div class='del'>- * @vsi: pointer to the vsi structure</div><div class='del'>- * @l2params: New paramaters from l2</div><div class='del'>- */</div><div class='del'>-void i40iw_change_l2params(struct i40iw_sc_vsi *vsi, struct i40iw_l2params *l2params)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_dev *dev = vsi-&gt;dev;</div><div class='del'>-	struct i40iw_sc_qp *qp = NULL;</div><div class='del'>-	bool qs_handle_change = false;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	u16 qs_handle;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	if (vsi-&gt;mtu != l2params-&gt;mtu) {</div><div class='del'>-		vsi-&gt;mtu = l2params-&gt;mtu;</div><div class='del'>-		i40iw_reinitialize_ieq(dev);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i40iw_fill_qos_list(l2params-&gt;qs_handle_list);</div><div class='del'>-	for (i = 0; i &lt; I40IW_MAX_USER_PRIORITY; i++) {</div><div class='del'>-		qs_handle = l2params-&gt;qs_handle_list[i];</div><div class='del'>-		if (vsi-&gt;qos[i].qs_handle != qs_handle)</div><div class='del'>-			qs_handle_change = true;</div><div class='del'>-		spin_lock_irqsave(&amp;vsi-&gt;qos[i].lock, flags);</div><div class='del'>-		qp = i40iw_get_qp(&amp;vsi-&gt;qos[i].qplist, qp);</div><div class='del'>-		while (qp) {</div><div class='del'>-			if (qs_handle_change) {</div><div class='del'>-				qp-&gt;qs_handle = qs_handle;</div><div class='del'>-				/* issue cqp suspend command */</div><div class='del'>-				i40iw_qp_suspend_resume(dev, qp, true);</div><div class='del'>-			}</div><div class='del'>-			qp = i40iw_get_qp(&amp;vsi-&gt;qos[i].qplist, qp);</div><div class='del'>-		}</div><div class='del'>-		spin_unlock_irqrestore(&amp;vsi-&gt;qos[i].lock, flags);</div><div class='del'>-		vsi-&gt;qos[i].qs_handle = qs_handle;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_qp_rem_qos - remove qp from qos lists during destroy qp</div><div class='del'>- * @qp: qp to be removed from qos</div><div class='del'>- */</div><div class='del'>-void i40iw_qp_rem_qos(struct i40iw_sc_qp *qp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_vsi *vsi = qp-&gt;vsi;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	if (!qp-&gt;on_qoslist)</div><div class='del'>-		return;</div><div class='del'>-	spin_lock_irqsave(&amp;vsi-&gt;qos[qp-&gt;user_pri].lock, flags);</div><div class='del'>-	list_del(&amp;qp-&gt;list);</div><div class='del'>-	spin_unlock_irqrestore(&amp;vsi-&gt;qos[qp-&gt;user_pri].lock, flags);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_qp_add_qos - called during setctx fot qp to be added to qos</div><div class='del'>- * @qp: qp to be added to qos</div><div class='del'>- */</div><div class='del'>-void i40iw_qp_add_qos(struct i40iw_sc_qp *qp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_vsi *vsi = qp-&gt;vsi;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	if (qp-&gt;on_qoslist)</div><div class='del'>-		return;</div><div class='del'>-	spin_lock_irqsave(&amp;vsi-&gt;qos[qp-&gt;user_pri].lock, flags);</div><div class='del'>-	qp-&gt;qs_handle = vsi-&gt;qos[qp-&gt;user_pri].qs_handle;</div><div class='del'>-	list_add(&amp;qp-&gt;list, &amp;vsi-&gt;qos[qp-&gt;user_pri].qplist);</div><div class='del'>-	qp-&gt;on_qoslist = true;</div><div class='del'>-	spin_unlock_irqrestore(&amp;vsi-&gt;qos[qp-&gt;user_pri].lock, flags);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_pd_init - initialize sc pd struct</div><div class='del'>- * @dev: sc device struct</div><div class='del'>- * @pd: sc pd ptr</div><div class='del'>- * @pd_id: pd_id for allocated pd</div><div class='del'>- * @abi_ver: ABI version from user context, -1 if not valid</div><div class='del'>- */</div><div class='del'>-static void i40iw_sc_pd_init(struct i40iw_sc_dev *dev,</div><div class='del'>-			     struct i40iw_sc_pd *pd,</div><div class='del'>-			     u16 pd_id,</div><div class='del'>-			     int abi_ver)</div><div class='del'>-{</div><div class='del'>-	pd-&gt;size = sizeof(*pd);</div><div class='del'>-	pd-&gt;pd_id = pd_id;</div><div class='del'>-	pd-&gt;abi_ver = abi_ver;</div><div class='del'>-	pd-&gt;dev = dev;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_encoded_wqe_size - given wq size, returns hardware encoded size</div><div class='del'>- * @wqsize: size of the wq (sq, rq, srq) to encoded_size</div><div class='del'>- * @cqpsq: encoded size for sq for cqp as its encoded size is 1+ other wq's</div><div class='del'>- */</div><div class='del'>-u8 i40iw_get_encoded_wqe_size(u32 wqsize, bool cqpsq)</div><div class='del'>-{</div><div class='del'>-	u8 encoded_size = 0;</div><div class='del'>-</div><div class='del'>-	/* cqp sq's hw coded value starts from 1 for size of 4</div><div class='del'>-	 * while it starts from 0 for qp' wq's.</div><div class='del'>-	 */</div><div class='del'>-	if (cqpsq)</div><div class='del'>-		encoded_size = 1;</div><div class='del'>-	wqsize &gt;&gt;= 2;</div><div class='del'>-	while (wqsize &gt;&gt;= 1)</div><div class='del'>-		encoded_size++;</div><div class='del'>-	return encoded_size;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_cqp_init - Initialize buffers for a control Queue Pair</div><div class='del'>- * @cqp: IWARP control queue pair pointer</div><div class='del'>- * @info: IWARP control queue pair init info pointer</div><div class='del'>- *</div><div class='del'>- * Initializes the object and context buffers for a control Queue Pair.</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_cqp_init(struct i40iw_sc_cqp *cqp,</div><div class='del'>-						struct i40iw_cqp_init_info *info)</div><div class='del'>-{</div><div class='del'>-	u8 hw_sq_size;</div><div class='del'>-</div><div class='del'>-	if ((info-&gt;sq_size &gt; I40IW_CQP_SW_SQSIZE_2048) ||</div><div class='del'>-	    (info-&gt;sq_size &lt; I40IW_CQP_SW_SQSIZE_4) ||</div><div class='del'>-	    ((info-&gt;sq_size &amp; (info-&gt;sq_size - 1))))</div><div class='del'>-		return I40IW_ERR_INVALID_SIZE;</div><div class='del'>-</div><div class='del'>-	hw_sq_size = i40iw_get_encoded_wqe_size(info-&gt;sq_size, true);</div><div class='del'>-	cqp-&gt;size = sizeof(*cqp);</div><div class='del'>-	cqp-&gt;sq_size = info-&gt;sq_size;</div><div class='del'>-	cqp-&gt;hw_sq_size = hw_sq_size;</div><div class='del'>-	cqp-&gt;sq_base = info-&gt;sq;</div><div class='del'>-	cqp-&gt;host_ctx = info-&gt;host_ctx;</div><div class='del'>-	cqp-&gt;sq_pa = info-&gt;sq_pa;</div><div class='del'>-	cqp-&gt;host_ctx_pa = info-&gt;host_ctx_pa;</div><div class='del'>-	cqp-&gt;dev = info-&gt;dev;</div><div class='del'>-	cqp-&gt;struct_ver = info-&gt;struct_ver;</div><div class='del'>-	cqp-&gt;scratch_array = info-&gt;scratch_array;</div><div class='del'>-	cqp-&gt;polarity = 0;</div><div class='del'>-	cqp-&gt;en_datacenter_tcp = info-&gt;en_datacenter_tcp;</div><div class='del'>-	cqp-&gt;enabled_vf_count = info-&gt;enabled_vf_count;</div><div class='del'>-	cqp-&gt;hmc_profile = info-&gt;hmc_profile;</div><div class='del'>-	info-&gt;dev-&gt;cqp = cqp;</div><div class='del'>-</div><div class='del'>-	I40IW_RING_INIT(cqp-&gt;sq_ring, cqp-&gt;sq_size);</div><div class='del'>-	cqp-&gt;dev-&gt;cqp_cmd_stats[OP_REQUESTED_COMMANDS] = 0;</div><div class='del'>-	cqp-&gt;dev-&gt;cqp_cmd_stats[OP_COMPLETED_COMMANDS] = 0;</div><div class='del'>-	INIT_LIST_HEAD(&amp;cqp-&gt;dev-&gt;cqp_cmd_head);               /* for the cqp commands backlog. */</div><div class='del'>-</div><div class='del'>-	i40iw_wr32(cqp-&gt;dev-&gt;hw, I40E_PFPE_CQPTAIL, 0);</div><div class='del'>-	i40iw_wr32(cqp-&gt;dev-&gt;hw, I40E_PFPE_CQPDB, 0);</div><div class='del'>-</div><div class='del'>-	i40iw_debug(cqp-&gt;dev, I40IW_DEBUG_WQE,</div><div class='del'>-		    "%s: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%llxh] cqp[%p] polarity[x%04X]\n",</div><div class='del'>-		    __func__, cqp-&gt;sq_size, cqp-&gt;hw_sq_size,</div><div class='del'>-		    cqp-&gt;sq_base, cqp-&gt;sq_pa, cqp, cqp-&gt;polarity);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_cqp_create - create cqp during bringup</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @maj_err: If error, major err number</div><div class='del'>- * @min_err: If error, minor err number</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_cqp_create(struct i40iw_sc_cqp *cqp,</div><div class='del'>-						  u16 *maj_err,</div><div class='del'>-						  u16 *min_err)</div><div class='del'>-{</div><div class='del'>-	u64 temp;</div><div class='del'>-	u32 cnt = 0, p1, p2, val = 0, err_code;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-</div><div class='del'>-	*maj_err = 0;</div><div class='del'>-	*min_err = 0;</div><div class='del'>-</div><div class='del'>-	ret_code = i40iw_allocate_dma_mem(cqp-&gt;dev-&gt;hw,</div><div class='del'>-					  &amp;cqp-&gt;sdbuf,</div><div class='del'>-					  I40IW_UPDATE_SD_BUF_SIZE * cqp-&gt;sq_size,</div><div class='del'>-					  I40IW_SD_BUF_ALIGNMENT);</div><div class='del'>-</div><div class='del'>-	if (ret_code)</div><div class='del'>-		goto exit;</div><div class='del'>-</div><div class='del'>-	temp = LS_64(cqp-&gt;hw_sq_size, I40IW_CQPHC_SQSIZE) |</div><div class='del'>-	       LS_64(cqp-&gt;struct_ver, I40IW_CQPHC_SVER);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(cqp-&gt;host_ctx, 0, temp);</div><div class='del'>-	set_64bit_val(cqp-&gt;host_ctx, 8, cqp-&gt;sq_pa);</div><div class='del'>-	temp = LS_64(cqp-&gt;enabled_vf_count, I40IW_CQPHC_ENABLED_VFS) |</div><div class='del'>-	       LS_64(cqp-&gt;hmc_profile, I40IW_CQPHC_HMC_PROFILE);</div><div class='del'>-	set_64bit_val(cqp-&gt;host_ctx, 16, temp);</div><div class='del'>-	set_64bit_val(cqp-&gt;host_ctx, 24, (uintptr_t)cqp);</div><div class='del'>-	set_64bit_val(cqp-&gt;host_ctx, 32, 0);</div><div class='del'>-	set_64bit_val(cqp-&gt;host_ctx, 40, 0);</div><div class='del'>-	set_64bit_val(cqp-&gt;host_ctx, 48, 0);</div><div class='del'>-	set_64bit_val(cqp-&gt;host_ctx, 56, 0);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "CQP_HOST_CTX",</div><div class='del'>-			cqp-&gt;host_ctx, I40IW_CQP_CTX_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	p1 = RS_32_1(cqp-&gt;host_ctx_pa, 32);</div><div class='del'>-	p2 = (u32)cqp-&gt;host_ctx_pa;</div><div class='del'>-</div><div class='del'>-	if (cqp-&gt;dev-&gt;is_pf) {</div><div class='del'>-		i40iw_wr32(cqp-&gt;dev-&gt;hw, I40E_PFPE_CCQPHIGH, p1);</div><div class='del'>-		i40iw_wr32(cqp-&gt;dev-&gt;hw, I40E_PFPE_CCQPLOW, p2);</div><div class='del'>-	} else {</div><div class='del'>-		i40iw_wr32(cqp-&gt;dev-&gt;hw, I40E_VFPE_CCQPHIGH1, p1);</div><div class='del'>-		i40iw_wr32(cqp-&gt;dev-&gt;hw, I40E_VFPE_CCQPLOW1, p2);</div><div class='del'>-	}</div><div class='del'>-	do {</div><div class='del'>-		if (cnt++ &gt; I40IW_DONE_COUNT) {</div><div class='del'>-			i40iw_free_dma_mem(cqp-&gt;dev-&gt;hw, &amp;cqp-&gt;sdbuf);</div><div class='del'>-			ret_code = I40IW_ERR_TIMEOUT;</div><div class='del'>-			/*</div><div class='del'>-			 * read PFPE_CQPERRORCODES register to get the minor</div><div class='del'>-			 * and major error code</div><div class='del'>-			 */</div><div class='del'>-			if (cqp-&gt;dev-&gt;is_pf)</div><div class='del'>-				err_code = i40iw_rd32(cqp-&gt;dev-&gt;hw, I40E_PFPE_CQPERRCODES);</div><div class='del'>-			else</div><div class='del'>-				err_code = i40iw_rd32(cqp-&gt;dev-&gt;hw, I40E_VFPE_CQPERRCODES1);</div><div class='del'>-			*min_err = RS_32(err_code, I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE);</div><div class='del'>-			*maj_err = RS_32(err_code, I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE);</div><div class='del'>-			goto exit;</div><div class='del'>-		}</div><div class='del'>-		udelay(I40IW_SLEEP_COUNT);</div><div class='del'>-		if (cqp-&gt;dev-&gt;is_pf)</div><div class='del'>-			val = i40iw_rd32(cqp-&gt;dev-&gt;hw, I40E_PFPE_CCQPSTATUS);</div><div class='del'>-		else</div><div class='del'>-			val = i40iw_rd32(cqp-&gt;dev-&gt;hw, I40E_VFPE_CCQPSTATUS1);</div><div class='del'>-	} while (!val);</div><div class='del'>-</div><div class='del'>-exit:</div><div class='del'>-	if (!ret_code)</div><div class='del'>-		cqp-&gt;process_cqp_sds = i40iw_update_sds_noccq;</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_cqp_post_sq - post of cqp's sq</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- */</div><div class='del'>-void i40iw_sc_cqp_post_sq(struct i40iw_sc_cqp *cqp)</div><div class='del'>-{</div><div class='del'>-	if (cqp-&gt;dev-&gt;is_pf)</div><div class='del'>-		i40iw_wr32(cqp-&gt;dev-&gt;hw, I40E_PFPE_CQPDB, I40IW_RING_GETCURRENT_HEAD(cqp-&gt;sq_ring));</div><div class='del'>-	else</div><div class='del'>-		i40iw_wr32(cqp-&gt;dev-&gt;hw, I40E_VFPE_CQPDB1, I40IW_RING_GETCURRENT_HEAD(cqp-&gt;sq_ring));</div><div class='del'>-</div><div class='del'>-	i40iw_debug(cqp-&gt;dev,</div><div class='del'>-		    I40IW_DEBUG_WQE,</div><div class='del'>-		    "%s: HEAD_TAIL[%04d,%04d,%04d]\n",</div><div class='del'>-		    __func__,</div><div class='del'>-		    cqp-&gt;sq_ring.head,</div><div class='del'>-		    cqp-&gt;sq_ring.tail,</div><div class='del'>-		    cqp-&gt;sq_ring.size);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_cqp_get_next_send_wqe_idx - get next WQE on CQP SQ and pass back the index</div><div class='del'>- * @cqp: pointer to CQP structure</div><div class='del'>- * @scratch: private data for CQP WQE</div><div class='del'>- * @wqe_idx: WQE index for next WQE on CQP SQ</div><div class='del'>- */</div><div class='del'>-static u64 *i40iw_sc_cqp_get_next_send_wqe_idx(struct i40iw_sc_cqp *cqp,</div><div class='del'>-					       u64 scratch, u32 *wqe_idx)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe = NULL;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-</div><div class='del'>-	if (I40IW_RING_FULL_ERR(cqp-&gt;sq_ring)) {</div><div class='del'>-		i40iw_debug(cqp-&gt;dev,</div><div class='del'>-			    I40IW_DEBUG_WQE,</div><div class='del'>-			    "%s: ring is full head %x tail %x size %x\n",</div><div class='del'>-			    __func__,</div><div class='del'>-			    cqp-&gt;sq_ring.head,</div><div class='del'>-			    cqp-&gt;sq_ring.tail,</div><div class='del'>-			    cqp-&gt;sq_ring.size);</div><div class='del'>-		return NULL;</div><div class='del'>-	}</div><div class='del'>-	I40IW_ATOMIC_RING_MOVE_HEAD(cqp-&gt;sq_ring, *wqe_idx, ret_code);</div><div class='del'>-	cqp-&gt;dev-&gt;cqp_cmd_stats[OP_REQUESTED_COMMANDS]++;</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return NULL;</div><div class='del'>-	if (!*wqe_idx)</div><div class='del'>-		cqp-&gt;polarity = !cqp-&gt;polarity;</div><div class='del'>-</div><div class='del'>-	wqe = cqp-&gt;sq_base[*wqe_idx].elem;</div><div class='del'>-	cqp-&gt;scratch_array[*wqe_idx] = scratch;</div><div class='del'>-	I40IW_CQP_INIT_WQE(wqe);</div><div class='del'>-</div><div class='del'>-	return wqe;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_cqp_get_next_send_wqe - get next wqe on cqp sq</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @scratch: private data for CQP WQE</div><div class='del'>- */</div><div class='del'>-u64 *i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp *cqp, u64 scratch)</div><div class='del'>-{</div><div class='del'>-	u32 wqe_idx;</div><div class='del'>-</div><div class='del'>-	return i40iw_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &amp;wqe_idx);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_cqp_destroy - destroy cqp during close</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_cqp_destroy(struct i40iw_sc_cqp *cqp)</div><div class='del'>-{</div><div class='del'>-	u32 cnt = 0, val = 1;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-	u32 cqpstat_addr;</div><div class='del'>-</div><div class='del'>-	if (cqp-&gt;dev-&gt;is_pf) {</div><div class='del'>-		i40iw_wr32(cqp-&gt;dev-&gt;hw, I40E_PFPE_CCQPHIGH, 0);</div><div class='del'>-		i40iw_wr32(cqp-&gt;dev-&gt;hw, I40E_PFPE_CCQPLOW, 0);</div><div class='del'>-		cqpstat_addr = I40E_PFPE_CCQPSTATUS;</div><div class='del'>-	} else {</div><div class='del'>-		i40iw_wr32(cqp-&gt;dev-&gt;hw, I40E_VFPE_CCQPHIGH1, 0);</div><div class='del'>-		i40iw_wr32(cqp-&gt;dev-&gt;hw, I40E_VFPE_CCQPLOW1, 0);</div><div class='del'>-		cqpstat_addr = I40E_VFPE_CCQPSTATUS1;</div><div class='del'>-	}</div><div class='del'>-	do {</div><div class='del'>-		if (cnt++ &gt; I40IW_DONE_COUNT) {</div><div class='del'>-			ret_code = I40IW_ERR_TIMEOUT;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		udelay(I40IW_SLEEP_COUNT);</div><div class='del'>-		val = i40iw_rd32(cqp-&gt;dev-&gt;hw, cqpstat_addr);</div><div class='del'>-	} while (val);</div><div class='del'>-</div><div class='del'>-	i40iw_free_dma_mem(cqp-&gt;dev-&gt;hw, &amp;cqp-&gt;sdbuf);</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_ccq_arm - enable intr for control cq</div><div class='del'>- * @ccq: ccq sc struct</div><div class='del'>- */</div><div class='del'>-static void i40iw_sc_ccq_arm(struct i40iw_sc_cq *ccq)</div><div class='del'>-{</div><div class='del'>-	u64 temp_val;</div><div class='del'>-	u16 sw_cq_sel;</div><div class='del'>-	u8 arm_next_se;</div><div class='del'>-	u8 arm_seq_num;</div><div class='del'>-</div><div class='del'>-	/* write to cq doorbell shadow area */</div><div class='del'>-	/* arm next se should always be zero */</div><div class='del'>-	get_64bit_val(ccq-&gt;cq_uk.shadow_area, 32, &amp;temp_val);</div><div class='del'>-</div><div class='del'>-	sw_cq_sel = (u16)RS_64(temp_val, I40IW_CQ_DBSA_SW_CQ_SELECT);</div><div class='del'>-	arm_next_se = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_NEXT_SE);</div><div class='del'>-</div><div class='del'>-	arm_seq_num = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_SEQ_NUM);</div><div class='del'>-	arm_seq_num++;</div><div class='del'>-</div><div class='del'>-	temp_val = LS_64(arm_seq_num, I40IW_CQ_DBSA_ARM_SEQ_NUM) |</div><div class='del'>-		   LS_64(sw_cq_sel, I40IW_CQ_DBSA_SW_CQ_SELECT) |</div><div class='del'>-		   LS_64(arm_next_se, I40IW_CQ_DBSA_ARM_NEXT_SE) |</div><div class='del'>-		   LS_64(1, I40IW_CQ_DBSA_ARM_NEXT);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(ccq-&gt;cq_uk.shadow_area, 32, temp_val);</div><div class='del'>-</div><div class='del'>-	wmb();       /* make sure shadow area is updated before arming */</div><div class='del'>-</div><div class='del'>-	if (ccq-&gt;dev-&gt;is_pf)</div><div class='del'>-		i40iw_wr32(ccq-&gt;dev-&gt;hw, I40E_PFPE_CQARM, ccq-&gt;cq_uk.cq_id);</div><div class='del'>-	else</div><div class='del'>-		i40iw_wr32(ccq-&gt;dev-&gt;hw, I40E_VFPE_CQARM1, ccq-&gt;cq_uk.cq_id);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_ccq_get_cqe_info - get ccq's cq entry</div><div class='del'>- * @ccq: ccq sc struct</div><div class='del'>- * @info: completion q entry to return</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_ccq_get_cqe_info(</div><div class='del'>-					struct i40iw_sc_cq *ccq,</div><div class='del'>-					struct i40iw_ccq_cqe_info *info)</div><div class='del'>-{</div><div class='del'>-	u64 qp_ctx, temp, temp1;</div><div class='del'>-	u64 *cqe;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u32 wqe_idx;</div><div class='del'>-	u8 polarity;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-</div><div class='del'>-	if (ccq-&gt;cq_uk.avoid_mem_cflct)</div><div class='del'>-		cqe = (u64 *)I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(&amp;ccq-&gt;cq_uk);</div><div class='del'>-	else</div><div class='del'>-		cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(&amp;ccq-&gt;cq_uk);</div><div class='del'>-</div><div class='del'>-	get_64bit_val(cqe, 24, &amp;temp);</div><div class='del'>-	polarity = (u8)RS_64(temp, I40IW_CQ_VALID);</div><div class='del'>-	if (polarity != ccq-&gt;cq_uk.polarity)</div><div class='del'>-		return I40IW_ERR_QUEUE_EMPTY;</div><div class='del'>-</div><div class='del'>-	get_64bit_val(cqe, 8, &amp;qp_ctx);</div><div class='del'>-	cqp = (struct i40iw_sc_cqp *)(unsigned long)qp_ctx;</div><div class='del'>-	info-&gt;error = (bool)RS_64(temp, I40IW_CQ_ERROR);</div><div class='del'>-	info-&gt;min_err_code = (u16)RS_64(temp, I40IW_CQ_MINERR);</div><div class='del'>-	if (info-&gt;error) {</div><div class='del'>-		info-&gt;maj_err_code = (u16)RS_64(temp, I40IW_CQ_MAJERR);</div><div class='del'>-		info-&gt;min_err_code = (u16)RS_64(temp, I40IW_CQ_MINERR);</div><div class='del'>-	}</div><div class='del'>-	wqe_idx = (u32)RS_64(temp, I40IW_CQ_WQEIDX);</div><div class='del'>-	info-&gt;scratch = cqp-&gt;scratch_array[wqe_idx];</div><div class='del'>-</div><div class='del'>-	get_64bit_val(cqe, 16, &amp;temp1);</div><div class='del'>-	info-&gt;op_ret_val = (u32)RS_64(temp1, I40IW_CCQ_OPRETVAL);</div><div class='del'>-	get_64bit_val(cqp-&gt;sq_base[wqe_idx].elem, 24, &amp;temp1);</div><div class='del'>-	info-&gt;op_code = (u8)RS_64(temp1, I40IW_CQPSQ_OPCODE);</div><div class='del'>-	info-&gt;cqp = cqp;</div><div class='del'>-</div><div class='del'>-	/*  move the head for cq */</div><div class='del'>-	I40IW_RING_MOVE_HEAD(ccq-&gt;cq_uk.cq_ring, ret_code);</div><div class='del'>-	if (I40IW_RING_GETCURRENT_HEAD(ccq-&gt;cq_uk.cq_ring) == 0)</div><div class='del'>-		ccq-&gt;cq_uk.polarity ^= 1;</div><div class='del'>-</div><div class='del'>-	/* update cq tail in cq shadow memory also */</div><div class='del'>-	I40IW_RING_MOVE_TAIL(ccq-&gt;cq_uk.cq_ring);</div><div class='del'>-	set_64bit_val(ccq-&gt;cq_uk.shadow_area,</div><div class='del'>-		      0,</div><div class='del'>-		      I40IW_RING_GETCURRENT_HEAD(ccq-&gt;cq_uk.cq_ring));</div><div class='del'>-	wmb(); /* write shadow area before tail */</div><div class='del'>-	I40IW_RING_MOVE_TAIL(cqp-&gt;sq_ring);</div><div class='del'>-	ccq-&gt;dev-&gt;cqp_cmd_stats[OP_COMPLETED_COMMANDS]++;</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @op_code: cqp opcode for completion</div><div class='del'>- * @info: completion q entry to return</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_poll_for_cqp_op_done(</div><div class='del'>-					struct i40iw_sc_cqp *cqp,</div><div class='del'>-					u8 op_code,</div><div class='del'>-					struct i40iw_ccq_cqe_info *compl_info)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_ccq_cqe_info info;</div><div class='del'>-	struct i40iw_sc_cq *ccq;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-	u32 cnt = 0;</div><div class='del'>-</div><div class='del'>-	memset(&amp;info, 0, sizeof(info));</div><div class='del'>-	ccq = cqp-&gt;dev-&gt;ccq;</div><div class='del'>-	while (1) {</div><div class='del'>-		if (cnt++ &gt; I40IW_DONE_COUNT)</div><div class='del'>-			return I40IW_ERR_TIMEOUT;</div><div class='del'>-</div><div class='del'>-		if (i40iw_sc_ccq_get_cqe_info(ccq, &amp;info)) {</div><div class='del'>-			udelay(I40IW_SLEEP_COUNT);</div><div class='del'>-			continue;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (info.error) {</div><div class='del'>-			ret_code = I40IW_ERR_CQP_COMPL_ERROR;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		/* check if opcode is cq create */</div><div class='del'>-		if (op_code != info.op_code) {</div><div class='del'>-			i40iw_debug(cqp-&gt;dev, I40IW_DEBUG_WQE,</div><div class='del'>-				    "%s: opcode mismatch for my op code 0x%x, returned opcode %x\n",</div><div class='del'>-				    __func__, op_code, info.op_code);</div><div class='del'>-		}</div><div class='del'>-		/* success, exit out of the loop */</div><div class='del'>-		if (op_code == info.op_code)</div><div class='del'>-			break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (compl_info)</div><div class='del'>-		memcpy(compl_info, &amp;info, sizeof(*compl_info));</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_manage_push_page - Handle push page</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @info: push page info</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_manage_push_page(</div><div class='del'>-				struct i40iw_sc_cqp *cqp,</div><div class='del'>-				struct i40iw_cqp_manage_push_page_info *info,</div><div class='del'>-				u64 scratch,</div><div class='del'>-				bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;push_idx &gt;= I40IW_MAX_PUSH_PAGE_COUNT)</div><div class='del'>-		return I40IW_ERR_INVALID_PUSH_PAGE_INDEX;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 16, info-&gt;qs_handle);</div><div class='del'>-</div><div class='del'>-	header = LS_64(info-&gt;push_idx, I40IW_CQPSQ_MPP_PPIDX) |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_MANAGE_PUSH_PAGES, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID) |</div><div class='del'>-		 LS_64(info-&gt;free_page, I40IW_CQPSQ_MPP_FREE_PAGE);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "MANAGE_PUSH_PAGES WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_manage_hmc_pm_func_table - manage of function table</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @vf_index: vf index for cqp</div><div class='del'>- * @free_pm_fcn: function number</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_manage_hmc_pm_func_table(</div><div class='del'>-				struct i40iw_sc_cqp *cqp,</div><div class='del'>-				u64 scratch,</div><div class='del'>-				u8 vf_index,</div><div class='del'>-				bool free_pm_fcn,</div><div class='del'>-				bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	if (vf_index &gt;= I40IW_MAX_VF_PER_PF)</div><div class='del'>-		return I40IW_ERR_INVALID_VF_ID;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	header = LS_64(vf_index, I40IW_CQPSQ_MHMC_VFIDX) |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(free_pm_fcn, I40IW_CQPSQ_MHMC_FREEPMFN) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "MANAGE_HMC_PM_FUNC_TABLE WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_set_hmc_resource_profile - cqp wqe for hmc profile</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @hmc_profile_type: type of profile to set</div><div class='del'>- * @vf_num: vf number for profile</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- * @poll_registers: flag to poll register for cqp completion</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_set_hmc_resource_profile(</div><div class='del'>-				struct i40iw_sc_cqp *cqp,</div><div class='del'>-				u64 scratch,</div><div class='del'>-				u8 hmc_profile_type,</div><div class='del'>-				u8 vf_num, bool post_sq,</div><div class='del'>-				bool poll_registers)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-	u32 val, tail, error;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 16,</div><div class='del'>-		      (LS_64(hmc_profile_type, I40IW_CQPSQ_SHMCRP_HMC_PROFILE) |</div><div class='del'>-				LS_64(vf_num, I40IW_CQPSQ_SHMCRP_VFNUM)));</div><div class='del'>-</div><div class='del'>-	header = LS_64(I40IW_CQP_OP_SET_HMC_RESOURCE_PROFILE, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		       LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "MANAGE_HMC_PM_FUNC_TABLE WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	i40iw_get_cqp_reg_info(cqp, &amp;val, &amp;tail, &amp;error);</div><div class='del'>-	if (error)</div><div class='del'>-		return I40IW_ERR_CQP_COMPL_ERROR;</div><div class='del'>-</div><div class='del'>-	if (post_sq) {</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-		if (poll_registers)</div><div class='del'>-			ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000000);</div><div class='del'>-		else</div><div class='del'>-			ret_code = i40iw_sc_poll_for_cqp_op_done(cqp,</div><div class='del'>-								 I40IW_CQP_OP_SHMC_PAGES_ALLOCATED,</div><div class='del'>-								 NULL);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_manage_hmc_pm_func_table_done - wait for cqp wqe completion for function table</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_manage_hmc_pm_func_table_done(struct i40iw_sc_cqp *cqp)</div><div class='del'>-{</div><div class='del'>-	return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, NULL);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_commit_fpm_values_done - wait for cqp eqe completion for fpm commit</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_commit_fpm_values_done(struct i40iw_sc_cqp *cqp)</div><div class='del'>-{</div><div class='del'>-	return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_COMMIT_FPM_VALUES, NULL);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_commit_fpm_values - cqp wqe for commit fpm values</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @hmc_fn_id: hmc function id</div><div class='del'>- * @commit_fpm_mem; Memory for fpm values</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- * @wait_type: poll ccq or cqp registers for cqp completion</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_commit_fpm_values(</div><div class='del'>-					struct i40iw_sc_cqp *cqp,</div><div class='del'>-					u64 scratch,</div><div class='del'>-					u8 hmc_fn_id,</div><div class='del'>-					struct i40iw_dma_mem *commit_fpm_mem,</div><div class='del'>-					bool post_sq,</div><div class='del'>-					u8 wait_type)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-	u32 tail, val, error;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 16, hmc_fn_id);</div><div class='del'>-	set_64bit_val(wqe, 32, commit_fpm_mem-&gt;pa);</div><div class='del'>-</div><div class='del'>-	header = LS_64(I40IW_CQP_OP_COMMIT_FPM_VALUES, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "COMMIT_FPM_VALUES WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	i40iw_get_cqp_reg_info(cqp, &amp;val, &amp;tail, &amp;error);</div><div class='del'>-	if (error)</div><div class='del'>-		return I40IW_ERR_CQP_COMPL_ERROR;</div><div class='del'>-</div><div class='del'>-	if (post_sq) {</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-</div><div class='del'>-		if (wait_type == I40IW_CQP_WAIT_POLL_REGS)</div><div class='del'>-			ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);</div><div class='del'>-		else if (wait_type == I40IW_CQP_WAIT_POLL_CQ)</div><div class='del'>-			ret_code = i40iw_sc_commit_fpm_values_done(cqp);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_query_fpm_values_done - poll for cqp wqe completion for query fpm</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_query_fpm_values_done(struct i40iw_sc_cqp *cqp)</div><div class='del'>-{</div><div class='del'>-	return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_QUERY_FPM_VALUES, NULL);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_query_fpm_values - cqp wqe query fpm values</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @hmc_fn_id: hmc function id</div><div class='del'>- * @query_fpm_mem: memory for return fpm values</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- * @wait_type: poll ccq or cqp registers for cqp completion</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_query_fpm_values(</div><div class='del'>-					struct i40iw_sc_cqp *cqp,</div><div class='del'>-					u64 scratch,</div><div class='del'>-					u8 hmc_fn_id,</div><div class='del'>-					struct i40iw_dma_mem *query_fpm_mem,</div><div class='del'>-					bool post_sq,</div><div class='del'>-					u8 wait_type)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-	u32 tail, val, error;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 16, hmc_fn_id);</div><div class='del'>-	set_64bit_val(wqe, 32, query_fpm_mem-&gt;pa);</div><div class='del'>-</div><div class='del'>-	header = LS_64(I40IW_CQP_OP_QUERY_FPM_VALUES, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "QUERY_FPM WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	/* read the tail from CQP_TAIL register */</div><div class='del'>-	i40iw_get_cqp_reg_info(cqp, &amp;val, &amp;tail, &amp;error);</div><div class='del'>-</div><div class='del'>-	if (error)</div><div class='del'>-		return I40IW_ERR_CQP_COMPL_ERROR;</div><div class='del'>-</div><div class='del'>-	if (post_sq) {</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-		if (wait_type == I40IW_CQP_WAIT_POLL_REGS)</div><div class='del'>-			ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);</div><div class='del'>-		else if (wait_type == I40IW_CQP_WAIT_POLL_CQ)</div><div class='del'>-			ret_code = i40iw_sc_query_fpm_values_done(cqp);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_add_arp_cache_entry - cqp wqe add arp cache entry</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @info: arp entry information</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_add_arp_cache_entry(</div><div class='del'>-				struct i40iw_sc_cqp *cqp,</div><div class='del'>-				struct i40iw_add_arp_cache_entry_info *info,</div><div class='del'>-				u64 scratch,</div><div class='del'>-				bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 temp, header;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	set_64bit_val(wqe, 8, info-&gt;reach_max);</div><div class='del'>-</div><div class='del'>-	temp = info-&gt;mac_addr[5] |</div><div class='del'>-	       LS_64_1(info-&gt;mac_addr[4], 8) |</div><div class='del'>-	       LS_64_1(info-&gt;mac_addr[3], 16) |</div><div class='del'>-	       LS_64_1(info-&gt;mac_addr[2], 24) |</div><div class='del'>-	       LS_64_1(info-&gt;mac_addr[1], 32) |</div><div class='del'>-	       LS_64_1(info-&gt;mac_addr[0], 40);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 16, temp);</div><div class='del'>-</div><div class='del'>-	header = info-&gt;arp_index |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64((info-&gt;permanent ? 1 : 0), I40IW_CQPSQ_MAT_PERMANENT) |</div><div class='del'>-		 LS_64(1, I40IW_CQPSQ_MAT_ENTRYVALID) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "ARP_CACHE_ENTRY WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_del_arp_cache_entry - dele arp cache entry</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @arp_index: arp index to delete arp entry</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_del_arp_cache_entry(</div><div class='del'>-					struct i40iw_sc_cqp *cqp,</div><div class='del'>-					u64 scratch,</div><div class='del'>-					u16 arp_index,</div><div class='del'>-					bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	header = arp_index |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "ARP_CACHE_DEL_ENTRY WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_query_arp_cache_entry - cqp wqe to query arp and arp index</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @arp_index: arp index to delete arp entry</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_query_arp_cache_entry(</div><div class='del'>-				struct i40iw_sc_cqp *cqp,</div><div class='del'>-				u64 scratch,</div><div class='del'>-				u16 arp_index,</div><div class='del'>-				bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	header = arp_index |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(1, I40IW_CQPSQ_MAT_QUERY) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "QUERY_ARP_CACHE_ENTRY WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_manage_apbvt_entry - for adding and deleting apbvt entries</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @info: info for apbvt entry to add or delete</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_manage_apbvt_entry(</div><div class='del'>-				struct i40iw_sc_cqp *cqp,</div><div class='del'>-				struct i40iw_apbvt_info *info,</div><div class='del'>-				u64 scratch,</div><div class='del'>-				bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 16, info-&gt;port);</div><div class='del'>-</div><div class='del'>-	header = LS_64(I40IW_CQP_OP_MANAGE_APBVT, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(info-&gt;add, I40IW_CQPSQ_MAPT_ADDPORT) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "MANAGE_APBVT WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_manage_qhash_table_entry - manage quad hash entries</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @info: info for quad hash to manage</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- *</div><div class='del'>- * This is called before connection establishment is started. For passive connections, when</div><div class='del'>- * listener is created, it will call with entry type of  I40IW_QHASH_TYPE_TCP_SYN with local</div><div class='del'>- * ip address and tcp port. When SYN is received (passive connections) or</div><div class='del'>- * sent (active connections), this routine is called with entry type of</div><div class='del'>- * I40IW_QHASH_TYPE_TCP_ESTABLISHED and quad is passed in info.</div><div class='del'>- *</div><div class='del'>- * When iwarp connection is done and its state moves to RTS, the quad hash entry in</div><div class='del'>- * the hardware will point to iwarp's qp number and requires no calls from the driver.</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_manage_qhash_table_entry(</div><div class='del'>-					struct i40iw_sc_cqp *cqp,</div><div class='del'>-					struct i40iw_qhash_table_info *info,</div><div class='del'>-					u64 scratch,</div><div class='del'>-					bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 qw1 = 0;</div><div class='del'>-	u64 qw2 = 0;</div><div class='del'>-	u64 temp;</div><div class='del'>-	struct i40iw_sc_vsi *vsi = info-&gt;vsi;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	temp = info-&gt;mac_addr[5] |</div><div class='del'>-		LS_64_1(info-&gt;mac_addr[4], 8) |</div><div class='del'>-		LS_64_1(info-&gt;mac_addr[3], 16) |</div><div class='del'>-		LS_64_1(info-&gt;mac_addr[2], 24) |</div><div class='del'>-		LS_64_1(info-&gt;mac_addr[1], 32) |</div><div class='del'>-		LS_64_1(info-&gt;mac_addr[0], 40);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 0, temp);</div><div class='del'>-</div><div class='del'>-	qw1 = LS_64(info-&gt;qp_num, I40IW_CQPSQ_QHASH_QPN) |</div><div class='del'>-	      LS_64(info-&gt;dest_port, I40IW_CQPSQ_QHASH_DEST_PORT);</div><div class='del'>-	if (info-&gt;ipv4_valid) {</div><div class='del'>-		set_64bit_val(wqe,</div><div class='del'>-			      48,</div><div class='del'>-			      LS_64(info-&gt;dest_ip[0], I40IW_CQPSQ_QHASH_ADDR3));</div><div class='del'>-	} else {</div><div class='del'>-		set_64bit_val(wqe,</div><div class='del'>-			      56,</div><div class='del'>-			      LS_64(info-&gt;dest_ip[0], I40IW_CQPSQ_QHASH_ADDR0) |</div><div class='del'>-			      LS_64(info-&gt;dest_ip[1], I40IW_CQPSQ_QHASH_ADDR1));</div><div class='del'>-</div><div class='del'>-		set_64bit_val(wqe,</div><div class='del'>-			      48,</div><div class='del'>-			      LS_64(info-&gt;dest_ip[2], I40IW_CQPSQ_QHASH_ADDR2) |</div><div class='del'>-			      LS_64(info-&gt;dest_ip[3], I40IW_CQPSQ_QHASH_ADDR3));</div><div class='del'>-	}</div><div class='del'>-	qw2 = LS_64(vsi-&gt;qos[info-&gt;user_pri].qs_handle, I40IW_CQPSQ_QHASH_QS_HANDLE);</div><div class='del'>-	if (info-&gt;vlan_valid)</div><div class='del'>-		qw2 |= LS_64(info-&gt;vlan_id, I40IW_CQPSQ_QHASH_VLANID);</div><div class='del'>-	set_64bit_val(wqe, 16, qw2);</div><div class='del'>-	if (info-&gt;entry_type == I40IW_QHASH_TYPE_TCP_ESTABLISHED) {</div><div class='del'>-		qw1 |= LS_64(info-&gt;src_port, I40IW_CQPSQ_QHASH_SRC_PORT);</div><div class='del'>-		if (!info-&gt;ipv4_valid) {</div><div class='del'>-			set_64bit_val(wqe,</div><div class='del'>-				      40,</div><div class='del'>-				      LS_64(info-&gt;src_ip[0], I40IW_CQPSQ_QHASH_ADDR0) |</div><div class='del'>-				      LS_64(info-&gt;src_ip[1], I40IW_CQPSQ_QHASH_ADDR1));</div><div class='del'>-			set_64bit_val(wqe,</div><div class='del'>-				      32,</div><div class='del'>-				      LS_64(info-&gt;src_ip[2], I40IW_CQPSQ_QHASH_ADDR2) |</div><div class='del'>-				      LS_64(info-&gt;src_ip[3], I40IW_CQPSQ_QHASH_ADDR3));</div><div class='del'>-		} else {</div><div class='del'>-			set_64bit_val(wqe,</div><div class='del'>-				      32,</div><div class='del'>-				      LS_64(info-&gt;src_ip[0], I40IW_CQPSQ_QHASH_ADDR3));</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 8, qw1);</div><div class='del'>-	temp = LS_64(cqp-&gt;polarity, I40IW_CQPSQ_QHASH_WQEVALID) |</div><div class='del'>-	       LS_64(I40IW_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY, I40IW_CQPSQ_QHASH_OPCODE) |</div><div class='del'>-	       LS_64(info-&gt;manage, I40IW_CQPSQ_QHASH_MANAGE) |</div><div class='del'>-	       LS_64(info-&gt;ipv4_valid, I40IW_CQPSQ_QHASH_IPV4VALID) |</div><div class='del'>-	       LS_64(info-&gt;vlan_valid, I40IW_CQPSQ_QHASH_VLANVALID) |</div><div class='del'>-	       LS_64(info-&gt;entry_type, I40IW_CQPSQ_QHASH_ENTRYTYPE);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, temp);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "MANAGE_QHASH WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_alloc_local_mac_ipaddr_entry - cqp wqe for loc mac entry</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_alloc_local_mac_ipaddr_entry(</div><div class='del'>-					struct i40iw_sc_cqp *cqp,</div><div class='del'>-					u64 scratch,</div><div class='del'>-					bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	header = LS_64(I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "ALLOCATE_LOCAL_MAC_IPADDR WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_add_local_mac_ipaddr_entry - add mac enry</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @info:mac addr info</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_add_local_mac_ipaddr_entry(</div><div class='del'>-				struct i40iw_sc_cqp *cqp,</div><div class='del'>-				struct i40iw_local_mac_ipaddr_entry_info *info,</div><div class='del'>-				u64 scratch,</div><div class='del'>-				bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 temp, header;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	temp = info-&gt;mac_addr[5] |</div><div class='del'>-		LS_64_1(info-&gt;mac_addr[4], 8) |</div><div class='del'>-		LS_64_1(info-&gt;mac_addr[3], 16) |</div><div class='del'>-		LS_64_1(info-&gt;mac_addr[2], 24) |</div><div class='del'>-		LS_64_1(info-&gt;mac_addr[1], 32) |</div><div class='del'>-		LS_64_1(info-&gt;mac_addr[0], 40);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 32, temp);</div><div class='del'>-</div><div class='del'>-	header = LS_64(info-&gt;entry_idx, I40IW_CQPSQ_MLIPA_IPTABLEIDX) |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "ADD_LOCAL_MAC_IPADDR WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_del_local_mac_ipaddr_entry - cqp wqe to dele local mac</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @entry_idx: index of mac entry</div><div class='del'>- * @ ignore_ref_count: to force mac adde delete</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_del_local_mac_ipaddr_entry(</div><div class='del'>-				struct i40iw_sc_cqp *cqp,</div><div class='del'>-				u64 scratch,</div><div class='del'>-				u8 entry_idx,</div><div class='del'>-				u8 ignore_ref_count,</div><div class='del'>-				bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	header = LS_64(entry_idx, I40IW_CQPSQ_MLIPA_IPTABLEIDX) |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(1, I40IW_CQPSQ_MLIPA_FREEENTRY) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID) |</div><div class='del'>-		 LS_64(ignore_ref_count, I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "DEL_LOCAL_MAC_IPADDR WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_cqp_nop - send a nop wqe</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_cqp_nop(struct i40iw_sc_cqp *cqp,</div><div class='del'>-					       u64 scratch,</div><div class='del'>-					       bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	header = LS_64(I40IW_CQP_OP_NOP, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "NOP WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_ceq_init - initialize ceq</div><div class='del'>- * @ceq: ceq sc structure</div><div class='del'>- * @info: ceq initialization info</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_ceq_init(struct i40iw_sc_ceq *ceq,</div><div class='del'>-						struct i40iw_ceq_init_info *info)</div><div class='del'>-{</div><div class='del'>-	u32 pble_obj_cnt;</div><div class='del'>-</div><div class='del'>-	if ((info-&gt;elem_cnt &lt; I40IW_MIN_CEQ_ENTRIES) ||</div><div class='del'>-	    (info-&gt;elem_cnt &gt; I40IW_MAX_CEQ_ENTRIES))</div><div class='del'>-		return I40IW_ERR_INVALID_SIZE;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;ceq_id &gt;= I40IW_MAX_CEQID)</div><div class='del'>-		return I40IW_ERR_INVALID_CEQ_ID;</div><div class='del'>-</div><div class='del'>-	pble_obj_cnt = info-&gt;dev-&gt;hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].cnt;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;virtual_map &amp;&amp; (info-&gt;first_pm_pbl_idx &gt;= pble_obj_cnt))</div><div class='del'>-		return I40IW_ERR_INVALID_PBLE_INDEX;</div><div class='del'>-</div><div class='del'>-	ceq-&gt;size = sizeof(*ceq);</div><div class='del'>-	ceq-&gt;ceqe_base = (struct i40iw_ceqe *)info-&gt;ceqe_base;</div><div class='del'>-	ceq-&gt;ceq_id = info-&gt;ceq_id;</div><div class='del'>-	ceq-&gt;dev = info-&gt;dev;</div><div class='del'>-	ceq-&gt;elem_cnt = info-&gt;elem_cnt;</div><div class='del'>-	ceq-&gt;ceq_elem_pa = info-&gt;ceqe_pa;</div><div class='del'>-	ceq-&gt;virtual_map = info-&gt;virtual_map;</div><div class='del'>-</div><div class='del'>-	ceq-&gt;pbl_chunk_size = (ceq-&gt;virtual_map ? info-&gt;pbl_chunk_size : 0);</div><div class='del'>-	ceq-&gt;first_pm_pbl_idx = (ceq-&gt;virtual_map ? info-&gt;first_pm_pbl_idx : 0);</div><div class='del'>-	ceq-&gt;pbl_list = (ceq-&gt;virtual_map ? info-&gt;pbl_list : NULL);</div><div class='del'>-</div><div class='del'>-	ceq-&gt;tph_en = info-&gt;tph_en;</div><div class='del'>-	ceq-&gt;tph_val = info-&gt;tph_val;</div><div class='del'>-	ceq-&gt;polarity = 1;</div><div class='del'>-	I40IW_RING_INIT(ceq-&gt;ceq_ring, ceq-&gt;elem_cnt);</div><div class='del'>-	ceq-&gt;dev-&gt;ceq[info-&gt;ceq_id] = ceq;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_ceq_create - create ceq wqe</div><div class='del'>- * @ceq: ceq sc structure</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_ceq_create(struct i40iw_sc_ceq *ceq,</div><div class='del'>-						  u64 scratch,</div><div class='del'>-						  bool post_sq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	cqp = ceq-&gt;dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	set_64bit_val(wqe, 16, ceq-&gt;elem_cnt);</div><div class='del'>-	set_64bit_val(wqe, 32, (ceq-&gt;virtual_map ? 0 : ceq-&gt;ceq_elem_pa));</div><div class='del'>-	set_64bit_val(wqe, 48, (ceq-&gt;virtual_map ? ceq-&gt;first_pm_pbl_idx : 0));</div><div class='del'>-	set_64bit_val(wqe, 56, LS_64(ceq-&gt;tph_val, I40IW_CQPSQ_TPHVAL));</div><div class='del'>-</div><div class='del'>-	header = ceq-&gt;ceq_id |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_CREATE_CEQ, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(ceq-&gt;pbl_chunk_size, I40IW_CQPSQ_CEQ_LPBLSIZE) |</div><div class='del'>-		 LS_64(ceq-&gt;virtual_map, I40IW_CQPSQ_CEQ_VMAP) |</div><div class='del'>-		 LS_64(ceq-&gt;tph_en, I40IW_CQPSQ_TPHEN) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "CEQ_CREATE WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_cceq_create_done - poll for control ceq wqe to complete</div><div class='del'>- * @ceq: ceq sc structure</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_cceq_create_done(struct i40iw_sc_ceq *ceq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-</div><div class='del'>-	cqp = ceq-&gt;dev-&gt;cqp;</div><div class='del'>-	return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_CEQ, NULL);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_cceq_destroy_done - poll for destroy cceq to complete</div><div class='del'>- * @ceq: ceq sc structure</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_cceq_destroy_done(struct i40iw_sc_ceq *ceq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-</div><div class='del'>-	cqp = ceq-&gt;dev-&gt;cqp;</div><div class='del'>-	cqp-&gt;process_cqp_sds = i40iw_update_sds_noccq;</div><div class='del'>-	return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_DESTROY_CEQ, NULL);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_cceq_create - create cceq</div><div class='del'>- * @ceq: ceq sc structure</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_cceq_create(struct i40iw_sc_ceq *ceq, u64 scratch)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-</div><div class='del'>-	ret_code = i40iw_sc_ceq_create(ceq, scratch, true);</div><div class='del'>-	if (!ret_code)</div><div class='del'>-		ret_code = i40iw_sc_cceq_create_done(ceq);</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_ceq_destroy - destroy ceq</div><div class='del'>- * @ceq: ceq sc structure</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_ceq_destroy(struct i40iw_sc_ceq *ceq,</div><div class='del'>-						   u64 scratch,</div><div class='del'>-						   bool post_sq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	cqp = ceq-&gt;dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	set_64bit_val(wqe, 16, ceq-&gt;elem_cnt);</div><div class='del'>-	set_64bit_val(wqe, 48, ceq-&gt;first_pm_pbl_idx);</div><div class='del'>-	header = ceq-&gt;ceq_id |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_DESTROY_CEQ, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(ceq-&gt;pbl_chunk_size, I40IW_CQPSQ_CEQ_LPBLSIZE) |</div><div class='del'>-		 LS_64(ceq-&gt;virtual_map, I40IW_CQPSQ_CEQ_VMAP) |</div><div class='del'>-		 LS_64(ceq-&gt;tph_en, I40IW_CQPSQ_TPHEN) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "CEQ_DESTROY WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_process_ceq - process ceq</div><div class='del'>- * @dev: sc device struct</div><div class='del'>- * @ceq: ceq sc structure</div><div class='del'>- */</div><div class='del'>-static void *i40iw_sc_process_ceq(struct i40iw_sc_dev *dev, struct i40iw_sc_ceq *ceq)</div><div class='del'>-{</div><div class='del'>-	u64 temp;</div><div class='del'>-	u64 *ceqe;</div><div class='del'>-	struct i40iw_sc_cq *cq = NULL;</div><div class='del'>-	u8 polarity;</div><div class='del'>-</div><div class='del'>-	ceqe = (u64 *)I40IW_GET_CURRENT_CEQ_ELEMENT(ceq);</div><div class='del'>-	get_64bit_val(ceqe, 0, &amp;temp);</div><div class='del'>-	polarity = (u8)RS_64(temp, I40IW_CEQE_VALID);</div><div class='del'>-	if (polarity != ceq-&gt;polarity)</div><div class='del'>-		return cq;</div><div class='del'>-</div><div class='del'>-	cq = (struct i40iw_sc_cq *)(unsigned long)LS_64_1(temp, 1);</div><div class='del'>-</div><div class='del'>-	I40IW_RING_MOVE_TAIL(ceq-&gt;ceq_ring);</div><div class='del'>-	if (I40IW_RING_GETCURRENT_TAIL(ceq-&gt;ceq_ring) == 0)</div><div class='del'>-		ceq-&gt;polarity ^= 1;</div><div class='del'>-</div><div class='del'>-	if (dev-&gt;is_pf)</div><div class='del'>-		i40iw_wr32(dev-&gt;hw, I40E_PFPE_CQACK, cq-&gt;cq_uk.cq_id);</div><div class='del'>-	else</div><div class='del'>-		i40iw_wr32(dev-&gt;hw, I40E_VFPE_CQACK1, cq-&gt;cq_uk.cq_id);</div><div class='del'>-</div><div class='del'>-	return cq;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_aeq_init - initialize aeq</div><div class='del'>- * @aeq: aeq structure ptr</div><div class='del'>- * @info: aeq initialization info</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_aeq_init(struct i40iw_sc_aeq *aeq,</div><div class='del'>-						struct i40iw_aeq_init_info *info)</div><div class='del'>-{</div><div class='del'>-	u32 pble_obj_cnt;</div><div class='del'>-</div><div class='del'>-	if ((info-&gt;elem_cnt &lt; I40IW_MIN_AEQ_ENTRIES) ||</div><div class='del'>-	    (info-&gt;elem_cnt &gt; I40IW_MAX_AEQ_ENTRIES))</div><div class='del'>-		return I40IW_ERR_INVALID_SIZE;</div><div class='del'>-	pble_obj_cnt = info-&gt;dev-&gt;hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].cnt;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;virtual_map &amp;&amp; (info-&gt;first_pm_pbl_idx &gt;= pble_obj_cnt))</div><div class='del'>-		return I40IW_ERR_INVALID_PBLE_INDEX;</div><div class='del'>-</div><div class='del'>-	aeq-&gt;size = sizeof(*aeq);</div><div class='del'>-	aeq-&gt;polarity = 1;</div><div class='del'>-	aeq-&gt;aeqe_base = (struct i40iw_sc_aeqe *)info-&gt;aeqe_base;</div><div class='del'>-	aeq-&gt;dev = info-&gt;dev;</div><div class='del'>-	aeq-&gt;elem_cnt = info-&gt;elem_cnt;</div><div class='del'>-</div><div class='del'>-	aeq-&gt;aeq_elem_pa = info-&gt;aeq_elem_pa;</div><div class='del'>-	I40IW_RING_INIT(aeq-&gt;aeq_ring, aeq-&gt;elem_cnt);</div><div class='del'>-	info-&gt;dev-&gt;aeq = aeq;</div><div class='del'>-</div><div class='del'>-	aeq-&gt;virtual_map = info-&gt;virtual_map;</div><div class='del'>-	aeq-&gt;pbl_list = (aeq-&gt;virtual_map ? info-&gt;pbl_list : NULL);</div><div class='del'>-	aeq-&gt;pbl_chunk_size = (aeq-&gt;virtual_map ? info-&gt;pbl_chunk_size : 0);</div><div class='del'>-	aeq-&gt;first_pm_pbl_idx = (aeq-&gt;virtual_map ? info-&gt;first_pm_pbl_idx : 0);</div><div class='del'>-	info-&gt;dev-&gt;aeq = aeq;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_aeq_create - create aeq</div><div class='del'>- * @aeq: aeq structure ptr</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_aeq_create(struct i40iw_sc_aeq *aeq,</div><div class='del'>-						  u64 scratch,</div><div class='del'>-						  bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	cqp = aeq-&gt;dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	set_64bit_val(wqe, 16, aeq-&gt;elem_cnt);</div><div class='del'>-	set_64bit_val(wqe, 32,</div><div class='del'>-		      (aeq-&gt;virtual_map ? 0 : aeq-&gt;aeq_elem_pa));</div><div class='del'>-	set_64bit_val(wqe, 48,</div><div class='del'>-		      (aeq-&gt;virtual_map ? aeq-&gt;first_pm_pbl_idx : 0));</div><div class='del'>-</div><div class='del'>-	header = LS_64(I40IW_CQP_OP_CREATE_AEQ, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(aeq-&gt;pbl_chunk_size, I40IW_CQPSQ_AEQ_LPBLSIZE) |</div><div class='del'>-		 LS_64(aeq-&gt;virtual_map, I40IW_CQPSQ_AEQ_VMAP) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "AEQ_CREATE WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_aeq_destroy - destroy aeq during close</div><div class='del'>- * @aeq: aeq structure ptr</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_aeq_destroy(struct i40iw_sc_aeq *aeq,</div><div class='del'>-						   u64 scratch,</div><div class='del'>-						   bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	cqp = aeq-&gt;dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	set_64bit_val(wqe, 16, aeq-&gt;elem_cnt);</div><div class='del'>-	set_64bit_val(wqe, 48, aeq-&gt;first_pm_pbl_idx);</div><div class='del'>-	header = LS_64(I40IW_CQP_OP_DESTROY_AEQ, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(aeq-&gt;pbl_chunk_size, I40IW_CQPSQ_AEQ_LPBLSIZE) |</div><div class='del'>-		 LS_64(aeq-&gt;virtual_map, I40IW_CQPSQ_AEQ_VMAP) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "AEQ_DESTROY WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_get_next_aeqe - get next aeq entry</div><div class='del'>- * @aeq: aeq structure ptr</div><div class='del'>- * @info: aeqe info to be returned</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_get_next_aeqe(struct i40iw_sc_aeq *aeq,</div><div class='del'>-						     struct i40iw_aeqe_info *info)</div><div class='del'>-{</div><div class='del'>-	u64 temp, compl_ctx;</div><div class='del'>-	u64 *aeqe;</div><div class='del'>-	u16 wqe_idx;</div><div class='del'>-	u8 ae_src;</div><div class='del'>-	u8 polarity;</div><div class='del'>-</div><div class='del'>-	aeqe = (u64 *)I40IW_GET_CURRENT_AEQ_ELEMENT(aeq);</div><div class='del'>-	get_64bit_val(aeqe, 0, &amp;compl_ctx);</div><div class='del'>-	get_64bit_val(aeqe, 8, &amp;temp);</div><div class='del'>-	polarity = (u8)RS_64(temp, I40IW_AEQE_VALID);</div><div class='del'>-</div><div class='del'>-	if (aeq-&gt;polarity != polarity)</div><div class='del'>-		return I40IW_ERR_QUEUE_EMPTY;</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(aeq-&gt;dev, I40IW_DEBUG_WQE, "AEQ_ENTRY", aeqe, 16);</div><div class='del'>-</div><div class='del'>-	ae_src = (u8)RS_64(temp, I40IW_AEQE_AESRC);</div><div class='del'>-	wqe_idx = (u16)RS_64(temp, I40IW_AEQE_WQDESCIDX);</div><div class='del'>-	info-&gt;qp_cq_id = (u32)RS_64(temp, I40IW_AEQE_QPCQID);</div><div class='del'>-	info-&gt;ae_id = (u16)RS_64(temp, I40IW_AEQE_AECODE);</div><div class='del'>-	info-&gt;tcp_state = (u8)RS_64(temp, I40IW_AEQE_TCPSTATE);</div><div class='del'>-	info-&gt;iwarp_state = (u8)RS_64(temp, I40IW_AEQE_IWSTATE);</div><div class='del'>-	info-&gt;q2_data_written = (u8)RS_64(temp, I40IW_AEQE_Q2DATA);</div><div class='del'>-	info-&gt;aeqe_overflow = (bool)RS_64(temp, I40IW_AEQE_OVERFLOW);</div><div class='del'>-</div><div class='del'>-	switch (info-&gt;ae_id) {</div><div class='del'>-	case I40IW_AE_PRIV_OPERATION_DENIED:</div><div class='del'>-	case I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG:</div><div class='del'>-	case I40IW_AE_UDA_XMIT_DGRAM_TOO_SHORT:</div><div class='del'>-	case I40IW_AE_BAD_CLOSE:</div><div class='del'>-	case I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE:</div><div class='del'>-	case I40IW_AE_RDMA_READ_WHILE_ORD_ZERO:</div><div class='del'>-	case I40IW_AE_STAG_ZERO_INVALID:</div><div class='del'>-	case I40IW_AE_IB_RREQ_AND_Q1_FULL:</div><div class='del'>-	case I40IW_AE_WQE_UNEXPECTED_OPCODE:</div><div class='del'>-	case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION:</div><div class='del'>-	case I40IW_AE_DDP_UBE_INVALID_MO:</div><div class='del'>-	case I40IW_AE_DDP_UBE_INVALID_QN:</div><div class='del'>-	case I40IW_AE_DDP_NO_L_BIT:</div><div class='del'>-	case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:</div><div class='del'>-	case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE:</div><div class='del'>-	case I40IW_AE_ROE_INVALID_RDMA_READ_REQUEST:</div><div class='del'>-	case I40IW_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:</div><div class='del'>-	case I40IW_AE_INVALID_ARP_ENTRY:</div><div class='del'>-	case I40IW_AE_INVALID_TCP_OPTION_RCVD:</div><div class='del'>-	case I40IW_AE_STALE_ARP_ENTRY:</div><div class='del'>-	case I40IW_AE_LLP_CLOSE_COMPLETE:</div><div class='del'>-	case I40IW_AE_LLP_CONNECTION_RESET:</div><div class='del'>-	case I40IW_AE_LLP_FIN_RECEIVED:</div><div class='del'>-	case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR:</div><div class='del'>-	case I40IW_AE_LLP_SEGMENT_TOO_SMALL:</div><div class='del'>-	case I40IW_AE_LLP_SYN_RECEIVED:</div><div class='del'>-	case I40IW_AE_LLP_TERMINATE_RECEIVED:</div><div class='del'>-	case I40IW_AE_LLP_TOO_MANY_RETRIES:</div><div class='del'>-	case I40IW_AE_LLP_DOUBT_REACHABILITY:</div><div class='del'>-	case I40IW_AE_RESET_SENT:</div><div class='del'>-	case I40IW_AE_TERMINATE_SENT:</div><div class='del'>-	case I40IW_AE_RESET_NOT_SENT:</div><div class='del'>-	case I40IW_AE_LCE_QP_CATASTROPHIC:</div><div class='del'>-	case I40IW_AE_QP_SUSPEND_COMPLETE:</div><div class='del'>-		info-&gt;qp = true;</div><div class='del'>-		info-&gt;compl_ctx = compl_ctx;</div><div class='del'>-		ae_src = I40IW_AE_SOURCE_RSVD;</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_LCE_CQ_CATASTROPHIC:</div><div class='del'>-		info-&gt;cq = true;</div><div class='del'>-		info-&gt;compl_ctx = LS_64_1(compl_ctx, 1);</div><div class='del'>-		ae_src = I40IW_AE_SOURCE_RSVD;</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	switch (ae_src) {</div><div class='del'>-	case I40IW_AE_SOURCE_RQ:</div><div class='del'>-	case I40IW_AE_SOURCE_RQ_0011:</div><div class='del'>-		info-&gt;qp = true;</div><div class='del'>-		info-&gt;wqe_idx = wqe_idx;</div><div class='del'>-		info-&gt;compl_ctx = compl_ctx;</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_SOURCE_CQ:</div><div class='del'>-	case I40IW_AE_SOURCE_CQ_0110:</div><div class='del'>-	case I40IW_AE_SOURCE_CQ_1010:</div><div class='del'>-	case I40IW_AE_SOURCE_CQ_1110:</div><div class='del'>-		info-&gt;cq = true;</div><div class='del'>-		info-&gt;compl_ctx = LS_64_1(compl_ctx, 1);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_SOURCE_SQ:</div><div class='del'>-	case I40IW_AE_SOURCE_SQ_0111:</div><div class='del'>-		info-&gt;qp = true;</div><div class='del'>-		info-&gt;sq = true;</div><div class='del'>-		info-&gt;wqe_idx = wqe_idx;</div><div class='del'>-		info-&gt;compl_ctx = compl_ctx;</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_SOURCE_IN_RR_WR:</div><div class='del'>-	case I40IW_AE_SOURCE_IN_RR_WR_1011:</div><div class='del'>-		info-&gt;qp = true;</div><div class='del'>-		info-&gt;compl_ctx = compl_ctx;</div><div class='del'>-		info-&gt;in_rdrsp_wr = true;</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_SOURCE_OUT_RR:</div><div class='del'>-	case I40IW_AE_SOURCE_OUT_RR_1111:</div><div class='del'>-		info-&gt;qp = true;</div><div class='del'>-		info-&gt;compl_ctx = compl_ctx;</div><div class='del'>-		info-&gt;out_rdrsp = true;</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_SOURCE_RSVD:</div><div class='del'>-		/* fallthrough */</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	I40IW_RING_MOVE_TAIL(aeq-&gt;aeq_ring);</div><div class='del'>-	if (I40IW_RING_GETCURRENT_TAIL(aeq-&gt;aeq_ring) == 0)</div><div class='del'>-		aeq-&gt;polarity ^= 1;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_repost_aeq_entries - repost completed aeq entries</div><div class='del'>- * @dev: sc device struct</div><div class='del'>- * @count: allocate count</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_repost_aeq_entries(struct i40iw_sc_dev *dev,</div><div class='del'>-							  u32 count)</div><div class='del'>-{</div><div class='del'>-</div><div class='del'>-	if (dev-&gt;is_pf)</div><div class='del'>-		i40iw_wr32(dev-&gt;hw, I40E_PFPE_AEQALLOC, count);</div><div class='del'>-	else</div><div class='del'>-		i40iw_wr32(dev-&gt;hw, I40E_VFPE_AEQALLOC1, count);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_aeq_create_done - create aeq</div><div class='del'>- * @aeq: aeq structure ptr</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_aeq_create_done(struct i40iw_sc_aeq *aeq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-</div><div class='del'>-	cqp = aeq-&gt;dev-&gt;cqp;</div><div class='del'>-	return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_AEQ, NULL);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_aeq_destroy_done - destroy of aeq during close</div><div class='del'>- * @aeq: aeq structure ptr</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_aeq_destroy_done(struct i40iw_sc_aeq *aeq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-</div><div class='del'>-	cqp = aeq-&gt;dev-&gt;cqp;</div><div class='del'>-	return  i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_DESTROY_AEQ, NULL);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_ccq_init - initialize control cq</div><div class='del'>- * @cq: sc's cq ctruct</div><div class='del'>- * @info: info for control cq initialization</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_ccq_init(struct i40iw_sc_cq *cq,</div><div class='del'>-						struct i40iw_ccq_init_info *info)</div><div class='del'>-{</div><div class='del'>-	u32 pble_obj_cnt;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;num_elem &lt; I40IW_MIN_CQ_SIZE || info-&gt;num_elem &gt; I40IW_MAX_CQ_SIZE)</div><div class='del'>-		return I40IW_ERR_INVALID_SIZE;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;ceq_id &gt; I40IW_MAX_CEQID)</div><div class='del'>-		return I40IW_ERR_INVALID_CEQ_ID;</div><div class='del'>-</div><div class='del'>-	pble_obj_cnt = info-&gt;dev-&gt;hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].cnt;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;virtual_map &amp;&amp; (info-&gt;first_pm_pbl_idx &gt;= pble_obj_cnt))</div><div class='del'>-		return I40IW_ERR_INVALID_PBLE_INDEX;</div><div class='del'>-</div><div class='del'>-	cq-&gt;cq_pa = info-&gt;cq_pa;</div><div class='del'>-	cq-&gt;cq_uk.cq_base = info-&gt;cq_base;</div><div class='del'>-	cq-&gt;shadow_area_pa = info-&gt;shadow_area_pa;</div><div class='del'>-	cq-&gt;cq_uk.shadow_area = info-&gt;shadow_area;</div><div class='del'>-	cq-&gt;shadow_read_threshold = info-&gt;shadow_read_threshold;</div><div class='del'>-	cq-&gt;dev = info-&gt;dev;</div><div class='del'>-	cq-&gt;ceq_id = info-&gt;ceq_id;</div><div class='del'>-	cq-&gt;cq_uk.cq_size = info-&gt;num_elem;</div><div class='del'>-	cq-&gt;cq_type = I40IW_CQ_TYPE_CQP;</div><div class='del'>-	cq-&gt;ceqe_mask = info-&gt;ceqe_mask;</div><div class='del'>-	I40IW_RING_INIT(cq-&gt;cq_uk.cq_ring, info-&gt;num_elem);</div><div class='del'>-</div><div class='del'>-	cq-&gt;cq_uk.cq_id = 0;    /* control cq is id 0 always */</div><div class='del'>-	cq-&gt;ceq_id_valid = info-&gt;ceq_id_valid;</div><div class='del'>-	cq-&gt;tph_en = info-&gt;tph_en;</div><div class='del'>-	cq-&gt;tph_val = info-&gt;tph_val;</div><div class='del'>-	cq-&gt;cq_uk.avoid_mem_cflct = info-&gt;avoid_mem_cflct;</div><div class='del'>-</div><div class='del'>-	cq-&gt;pbl_list = info-&gt;pbl_list;</div><div class='del'>-	cq-&gt;virtual_map = info-&gt;virtual_map;</div><div class='del'>-	cq-&gt;pbl_chunk_size = info-&gt;pbl_chunk_size;</div><div class='del'>-	cq-&gt;first_pm_pbl_idx = info-&gt;first_pm_pbl_idx;</div><div class='del'>-	cq-&gt;cq_uk.polarity = true;</div><div class='del'>-</div><div class='del'>-	/* following are only for iw cqs so initialize them to zero */</div><div class='del'>-	cq-&gt;cq_uk.cqe_alloc_reg = NULL;</div><div class='del'>-	info-&gt;dev-&gt;ccq = cq;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_ccq_create_done - poll cqp for ccq create</div><div class='del'>- * @ccq: ccq sc struct</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_ccq_create_done(struct i40iw_sc_cq *ccq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-</div><div class='del'>-	cqp = ccq-&gt;dev-&gt;cqp;</div><div class='del'>-	return	i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_CQ, NULL);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_ccq_create - create control cq</div><div class='del'>- * @ccq: ccq sc struct</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @check_overflow: overlow flag for ccq</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_ccq_create(struct i40iw_sc_cq *ccq,</div><div class='del'>-						  u64 scratch,</div><div class='del'>-						  bool check_overflow,</div><div class='del'>-						  bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 header;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-</div><div class='del'>-	cqp = ccq-&gt;dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	set_64bit_val(wqe, 0, ccq-&gt;cq_uk.cq_size);</div><div class='del'>-	set_64bit_val(wqe, 8, RS_64_1(ccq, 1));</div><div class='del'>-	set_64bit_val(wqe, 16,</div><div class='del'>-		      LS_64(ccq-&gt;shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));</div><div class='del'>-	set_64bit_val(wqe, 32, (ccq-&gt;virtual_map ? 0 : ccq-&gt;cq_pa));</div><div class='del'>-	set_64bit_val(wqe, 40, ccq-&gt;shadow_area_pa);</div><div class='del'>-	set_64bit_val(wqe, 48,</div><div class='del'>-		      (ccq-&gt;virtual_map ? ccq-&gt;first_pm_pbl_idx : 0));</div><div class='del'>-	set_64bit_val(wqe, 56,</div><div class='del'>-		      LS_64(ccq-&gt;tph_val, I40IW_CQPSQ_TPHVAL));</div><div class='del'>-</div><div class='del'>-	header = ccq-&gt;cq_uk.cq_id |</div><div class='del'>-		 LS_64((ccq-&gt;ceq_id_valid ? ccq-&gt;ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(ccq-&gt;pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |</div><div class='del'>-		 LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |</div><div class='del'>-		 LS_64(ccq-&gt;virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |</div><div class='del'>-		 LS_64(ccq-&gt;ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |</div><div class='del'>-		 LS_64(ccq-&gt;ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |</div><div class='del'>-		 LS_64(ccq-&gt;tph_en, I40IW_CQPSQ_TPHEN) |</div><div class='del'>-		 LS_64(ccq-&gt;cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "CCQ_CREATE WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq) {</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-		ret_code = i40iw_sc_ccq_create_done(ccq);</div><div class='del'>-		if (ret_code)</div><div class='del'>-			return ret_code;</div><div class='del'>-	}</div><div class='del'>-	cqp-&gt;process_cqp_sds = i40iw_cqp_sds_cmd;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_ccq_destroy - destroy ccq during close</div><div class='del'>- * @ccq: ccq sc struct</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_ccq_destroy(struct i40iw_sc_cq *ccq,</div><div class='del'>-						   u64 scratch,</div><div class='del'>-						   bool post_sq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-	u32 tail, val, error;</div><div class='del'>-</div><div class='del'>-	cqp = ccq-&gt;dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	set_64bit_val(wqe, 0, ccq-&gt;cq_uk.cq_size);</div><div class='del'>-	set_64bit_val(wqe, 8, RS_64_1(ccq, 1));</div><div class='del'>-	set_64bit_val(wqe, 40, ccq-&gt;shadow_area_pa);</div><div class='del'>-</div><div class='del'>-	header = ccq-&gt;cq_uk.cq_id |</div><div class='del'>-		 LS_64((ccq-&gt;ceq_id_valid ? ccq-&gt;ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_DESTROY_CQ, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(ccq-&gt;ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |</div><div class='del'>-		 LS_64(ccq-&gt;ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |</div><div class='del'>-		 LS_64(ccq-&gt;tph_en, I40IW_CQPSQ_TPHEN) |</div><div class='del'>-		 LS_64(ccq-&gt;cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "CCQ_DESTROY WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	i40iw_get_cqp_reg_info(cqp, &amp;val, &amp;tail, &amp;error);</div><div class='del'>-	if (error)</div><div class='del'>-		return I40IW_ERR_CQP_COMPL_ERROR;</div><div class='del'>-</div><div class='del'>-	if (post_sq) {</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-		ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cqp-&gt;process_cqp_sds = i40iw_update_sds_noccq;</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_cq_init - initialize completion q</div><div class='del'>- * @cq: cq struct</div><div class='del'>- * @info: cq initialization info</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_cq_init(struct i40iw_sc_cq *cq,</div><div class='del'>-					       struct i40iw_cq_init_info *info)</div><div class='del'>-{</div><div class='del'>-	u32 __iomem *cqe_alloc_reg = NULL;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-	u32 pble_obj_cnt;</div><div class='del'>-	u32 arm_offset;</div><div class='del'>-</div><div class='del'>-	pble_obj_cnt = info-&gt;dev-&gt;hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].cnt;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;virtual_map &amp;&amp; (info-&gt;first_pm_pbl_idx &gt;= pble_obj_cnt))</div><div class='del'>-		return I40IW_ERR_INVALID_PBLE_INDEX;</div><div class='del'>-</div><div class='del'>-	cq-&gt;cq_pa = info-&gt;cq_base_pa;</div><div class='del'>-	cq-&gt;dev = info-&gt;dev;</div><div class='del'>-	cq-&gt;ceq_id = info-&gt;ceq_id;</div><div class='del'>-	arm_offset = (info-&gt;dev-&gt;is_pf) ? I40E_PFPE_CQARM : I40E_VFPE_CQARM1;</div><div class='del'>-	if (i40iw_get_hw_addr(cq-&gt;dev))</div><div class='del'>-		cqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(cq-&gt;dev) +</div><div class='del'>-					      arm_offset);</div><div class='del'>-	info-&gt;cq_uk_init_info.cqe_alloc_reg = cqe_alloc_reg;</div><div class='del'>-	ret_code = i40iw_cq_uk_init(&amp;cq-&gt;cq_uk, &amp;info-&gt;cq_uk_init_info);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return ret_code;</div><div class='del'>-	cq-&gt;virtual_map = info-&gt;virtual_map;</div><div class='del'>-	cq-&gt;pbl_chunk_size = info-&gt;pbl_chunk_size;</div><div class='del'>-	cq-&gt;ceqe_mask = info-&gt;ceqe_mask;</div><div class='del'>-	cq-&gt;cq_type = (info-&gt;type) ? info-&gt;type : I40IW_CQ_TYPE_IWARP;</div><div class='del'>-</div><div class='del'>-	cq-&gt;shadow_area_pa = info-&gt;shadow_area_pa;</div><div class='del'>-	cq-&gt;shadow_read_threshold = info-&gt;shadow_read_threshold;</div><div class='del'>-</div><div class='del'>-	cq-&gt;ceq_id_valid = info-&gt;ceq_id_valid;</div><div class='del'>-	cq-&gt;tph_en = info-&gt;tph_en;</div><div class='del'>-	cq-&gt;tph_val = info-&gt;tph_val;</div><div class='del'>-</div><div class='del'>-	cq-&gt;first_pm_pbl_idx = info-&gt;first_pm_pbl_idx;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_cq_create - create completion q</div><div class='del'>- * @cq: cq struct</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @check_overflow: flag for overflow check</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_cq_create(struct i40iw_sc_cq *cq,</div><div class='del'>-						 u64 scratch,</div><div class='del'>-						 bool check_overflow,</div><div class='del'>-						 bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	if (cq-&gt;cq_uk.cq_id &gt; I40IW_MAX_CQID)</div><div class='del'>-		return I40IW_ERR_INVALID_CQ_ID;</div><div class='del'>-</div><div class='del'>-	if (cq-&gt;ceq_id &gt; I40IW_MAX_CEQID)</div><div class='del'>-		return I40IW_ERR_INVALID_CEQ_ID;</div><div class='del'>-</div><div class='del'>-	cqp = cq-&gt;dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 0, cq-&gt;cq_uk.cq_size);</div><div class='del'>-	set_64bit_val(wqe, 8, RS_64_1(cq, 1));</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      16,</div><div class='del'>-		      LS_64(cq-&gt;shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 32, (cq-&gt;virtual_map ? 0 : cq-&gt;cq_pa));</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 40, cq-&gt;shadow_area_pa);</div><div class='del'>-	set_64bit_val(wqe, 48, (cq-&gt;virtual_map ? cq-&gt;first_pm_pbl_idx : 0));</div><div class='del'>-	set_64bit_val(wqe, 56, LS_64(cq-&gt;tph_val, I40IW_CQPSQ_TPHVAL));</div><div class='del'>-</div><div class='del'>-	header = cq-&gt;cq_uk.cq_id |</div><div class='del'>-		 LS_64((cq-&gt;ceq_id_valid ? cq-&gt;ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(cq-&gt;pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |</div><div class='del'>-		 LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |</div><div class='del'>-		 LS_64(cq-&gt;virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |</div><div class='del'>-		 LS_64(cq-&gt;ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |</div><div class='del'>-		 LS_64(cq-&gt;ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |</div><div class='del'>-		 LS_64(cq-&gt;tph_en, I40IW_CQPSQ_TPHEN) |</div><div class='del'>-		 LS_64(cq-&gt;cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "CQ_CREATE WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_cq_destroy - destroy completion q</div><div class='del'>- * @cq: cq struct</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_cq_destroy(struct i40iw_sc_cq *cq,</div><div class='del'>-						  u64 scratch,</div><div class='del'>-						  bool post_sq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	cqp = cq-&gt;dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	set_64bit_val(wqe, 0, cq-&gt;cq_uk.cq_size);</div><div class='del'>-	set_64bit_val(wqe, 8, RS_64_1(cq, 1));</div><div class='del'>-	set_64bit_val(wqe, 40, cq-&gt;shadow_area_pa);</div><div class='del'>-	set_64bit_val(wqe, 48, (cq-&gt;virtual_map ? cq-&gt;first_pm_pbl_idx : 0));</div><div class='del'>-</div><div class='del'>-	header = cq-&gt;cq_uk.cq_id |</div><div class='del'>-		 LS_64((cq-&gt;ceq_id_valid ? cq-&gt;ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_DESTROY_CQ, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(cq-&gt;pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |</div><div class='del'>-		 LS_64(cq-&gt;virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |</div><div class='del'>-		 LS_64(cq-&gt;ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |</div><div class='del'>-		 LS_64(cq-&gt;ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |</div><div class='del'>-		 LS_64(cq-&gt;tph_en, I40IW_CQPSQ_TPHEN) |</div><div class='del'>-		 LS_64(cq-&gt;cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "CQ_DESTROY WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_cq_modify - modify a Completion Queue</div><div class='del'>- * @cq: cq struct</div><div class='del'>- * @info: modification info struct</div><div class='del'>- * @scratch:</div><div class='del'>- * @post_sq: flag to post to sq</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_cq_modify(struct i40iw_sc_cq *cq,</div><div class='del'>-						 struct i40iw_modify_cq_info *info,</div><div class='del'>-						 u64 scratch,</div><div class='del'>-						 bool post_sq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-	u32 cq_size, ceq_id, first_pm_pbl_idx;</div><div class='del'>-	u8 pbl_chunk_size;</div><div class='del'>-	bool virtual_map, ceq_id_valid, check_overflow;</div><div class='del'>-	u32 pble_obj_cnt;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;ceq_valid &amp;&amp; (info-&gt;ceq_id &gt; I40IW_MAX_CEQID))</div><div class='del'>-		return I40IW_ERR_INVALID_CEQ_ID;</div><div class='del'>-</div><div class='del'>-	pble_obj_cnt = cq-&gt;dev-&gt;hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].cnt;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;cq_resize &amp;&amp; info-&gt;virtual_map &amp;&amp;</div><div class='del'>-	    (info-&gt;first_pm_pbl_idx &gt;= pble_obj_cnt))</div><div class='del'>-		return I40IW_ERR_INVALID_PBLE_INDEX;</div><div class='del'>-</div><div class='del'>-	cqp = cq-&gt;dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	cq-&gt;pbl_list = info-&gt;pbl_list;</div><div class='del'>-	cq-&gt;cq_pa = info-&gt;cq_pa;</div><div class='del'>-	cq-&gt;first_pm_pbl_idx = info-&gt;first_pm_pbl_idx;</div><div class='del'>-</div><div class='del'>-	cq_size = info-&gt;cq_resize ? info-&gt;cq_size : cq-&gt;cq_uk.cq_size;</div><div class='del'>-	if (info-&gt;ceq_change) {</div><div class='del'>-		ceq_id_valid = true;</div><div class='del'>-		ceq_id = info-&gt;ceq_id;</div><div class='del'>-	} else {</div><div class='del'>-		ceq_id_valid = cq-&gt;ceq_id_valid;</div><div class='del'>-		ceq_id = ceq_id_valid ? cq-&gt;ceq_id : 0;</div><div class='del'>-	}</div><div class='del'>-	virtual_map = info-&gt;cq_resize ? info-&gt;virtual_map : cq-&gt;virtual_map;</div><div class='del'>-	first_pm_pbl_idx = (info-&gt;cq_resize ?</div><div class='del'>-			    (info-&gt;virtual_map ? info-&gt;first_pm_pbl_idx : 0) :</div><div class='del'>-			    (cq-&gt;virtual_map ? cq-&gt;first_pm_pbl_idx : 0));</div><div class='del'>-	pbl_chunk_size = (info-&gt;cq_resize ?</div><div class='del'>-			  (info-&gt;virtual_map ? info-&gt;pbl_chunk_size : 0) :</div><div class='del'>-			  (cq-&gt;virtual_map ? cq-&gt;pbl_chunk_size : 0));</div><div class='del'>-	check_overflow = info-&gt;check_overflow_change ? info-&gt;check_overflow :</div><div class='del'>-			 cq-&gt;check_overflow;</div><div class='del'>-	cq-&gt;cq_uk.cq_size = cq_size;</div><div class='del'>-	cq-&gt;ceq_id_valid = ceq_id_valid;</div><div class='del'>-	cq-&gt;ceq_id = ceq_id;</div><div class='del'>-	cq-&gt;virtual_map = virtual_map;</div><div class='del'>-	cq-&gt;first_pm_pbl_idx = first_pm_pbl_idx;</div><div class='del'>-	cq-&gt;pbl_chunk_size = pbl_chunk_size;</div><div class='del'>-	cq-&gt;check_overflow = check_overflow;</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 0, cq_size);</div><div class='del'>-	set_64bit_val(wqe, 8, RS_64_1(cq, 1));</div><div class='del'>-	set_64bit_val(wqe, 16,</div><div class='del'>-		      LS_64(info-&gt;shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));</div><div class='del'>-	set_64bit_val(wqe, 32, (cq-&gt;virtual_map ? 0 : cq-&gt;cq_pa));</div><div class='del'>-	set_64bit_val(wqe, 40, cq-&gt;shadow_area_pa);</div><div class='del'>-	set_64bit_val(wqe, 48, (cq-&gt;virtual_map ? first_pm_pbl_idx : 0));</div><div class='del'>-	set_64bit_val(wqe, 56, LS_64(cq-&gt;tph_val, I40IW_CQPSQ_TPHVAL));</div><div class='del'>-</div><div class='del'>-	header = cq-&gt;cq_uk.cq_id |</div><div class='del'>-		 LS_64(ceq_id, I40IW_CQPSQ_CQ_CEQID) |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_MODIFY_CQ, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(info-&gt;cq_resize, I40IW_CQPSQ_CQ_CQRESIZE) |</div><div class='del'>-		 LS_64(pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |</div><div class='del'>-		 LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |</div><div class='del'>-		 LS_64(virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |</div><div class='del'>-		 LS_64(cq-&gt;ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |</div><div class='del'>-		 LS_64(ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |</div><div class='del'>-		 LS_64(cq-&gt;tph_en, I40IW_CQPSQ_TPHEN) |</div><div class='del'>-		 LS_64(cq-&gt;cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "CQ_MODIFY WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_qp_init - initialize qp</div><div class='del'>- * @qp: sc qp</div><div class='del'>- * @info: initialization qp info</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_qp_init(struct i40iw_sc_qp *qp,</div><div class='del'>-					       struct i40iw_qp_init_info *info)</div><div class='del'>-{</div><div class='del'>-	u32 __iomem *wqe_alloc_reg = NULL;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-	u32 pble_obj_cnt;</div><div class='del'>-	u8 wqe_size;</div><div class='del'>-	u32 offset;</div><div class='del'>-</div><div class='del'>-	qp-&gt;dev = info-&gt;pd-&gt;dev;</div><div class='del'>-	qp-&gt;vsi = info-&gt;vsi;</div><div class='del'>-	qp-&gt;sq_pa = info-&gt;sq_pa;</div><div class='del'>-	qp-&gt;rq_pa = info-&gt;rq_pa;</div><div class='del'>-	qp-&gt;hw_host_ctx_pa = info-&gt;host_ctx_pa;</div><div class='del'>-	qp-&gt;q2_pa = info-&gt;q2_pa;</div><div class='del'>-	qp-&gt;shadow_area_pa = info-&gt;shadow_area_pa;</div><div class='del'>-</div><div class='del'>-	qp-&gt;q2_buf = info-&gt;q2;</div><div class='del'>-	qp-&gt;pd = info-&gt;pd;</div><div class='del'>-	qp-&gt;hw_host_ctx = info-&gt;host_ctx;</div><div class='del'>-	offset = (qp-&gt;pd-&gt;dev-&gt;is_pf) ? I40E_PFPE_WQEALLOC : I40E_VFPE_WQEALLOC1;</div><div class='del'>-	if (i40iw_get_hw_addr(qp-&gt;pd-&gt;dev))</div><div class='del'>-		wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp-&gt;pd-&gt;dev) +</div><div class='del'>-					      offset);</div><div class='del'>-</div><div class='del'>-	info-&gt;qp_uk_init_info.wqe_alloc_reg = wqe_alloc_reg;</div><div class='del'>-	info-&gt;qp_uk_init_info.abi_ver = qp-&gt;pd-&gt;abi_ver;</div><div class='del'>-	ret_code = i40iw_qp_uk_init(&amp;qp-&gt;qp_uk, &amp;info-&gt;qp_uk_init_info);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return ret_code;</div><div class='del'>-	qp-&gt;virtual_map = info-&gt;virtual_map;</div><div class='del'>-</div><div class='del'>-	pble_obj_cnt = info-&gt;pd-&gt;dev-&gt;hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].cnt;</div><div class='del'>-</div><div class='del'>-	if ((info-&gt;virtual_map &amp;&amp; (info-&gt;sq_pa &gt;= pble_obj_cnt)) ||</div><div class='del'>-	    (info-&gt;virtual_map &amp;&amp; (info-&gt;rq_pa &gt;= pble_obj_cnt)))</div><div class='del'>-		return I40IW_ERR_INVALID_PBLE_INDEX;</div><div class='del'>-</div><div class='del'>-	qp-&gt;llp_stream_handle = (void *)(-1);</div><div class='del'>-	qp-&gt;qp_type = (info-&gt;type) ? info-&gt;type : I40IW_QP_TYPE_IWARP;</div><div class='del'>-</div><div class='del'>-	qp-&gt;hw_sq_size = i40iw_get_encoded_wqe_size(qp-&gt;qp_uk.sq_ring.size,</div><div class='del'>-						    false);</div><div class='del'>-	i40iw_debug(qp-&gt;dev, I40IW_DEBUG_WQE, "%s: hw_sq_size[%04d] sq_ring.size[%04d]\n",</div><div class='del'>-		    __func__, qp-&gt;hw_sq_size, qp-&gt;qp_uk.sq_ring.size);</div><div class='del'>-</div><div class='del'>-	switch (qp-&gt;pd-&gt;abi_ver) {</div><div class='del'>-	case 4:</div><div class='del'>-		ret_code = i40iw_fragcnt_to_wqesize_rq(qp-&gt;qp_uk.max_rq_frag_cnt,</div><div class='del'>-						       &amp;wqe_size);</div><div class='del'>-		if (ret_code)</div><div class='del'>-			return ret_code;</div><div class='del'>-		break;</div><div class='del'>-	case 5: /* fallthrough until next ABI version */</div><div class='del'>-	default:</div><div class='del'>-		if (qp-&gt;qp_uk.max_rq_frag_cnt &gt; I40IW_MAX_WQ_FRAGMENT_COUNT)</div><div class='del'>-			return I40IW_ERR_INVALID_FRAG_COUNT;</div><div class='del'>-		wqe_size = I40IW_MAX_WQE_SIZE_RQ;</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	qp-&gt;hw_rq_size = i40iw_get_encoded_wqe_size(qp-&gt;qp_uk.rq_size *</div><div class='del'>-				(wqe_size / I40IW_QP_WQE_MIN_SIZE), false);</div><div class='del'>-	i40iw_debug(qp-&gt;dev, I40IW_DEBUG_WQE,</div><div class='del'>-		    "%s: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n",</div><div class='del'>-		    __func__, qp-&gt;hw_rq_size, qp-&gt;qp_uk.rq_size, wqe_size);</div><div class='del'>-	qp-&gt;sq_tph_val = info-&gt;sq_tph_val;</div><div class='del'>-	qp-&gt;rq_tph_val = info-&gt;rq_tph_val;</div><div class='del'>-	qp-&gt;sq_tph_en = info-&gt;sq_tph_en;</div><div class='del'>-	qp-&gt;rq_tph_en = info-&gt;rq_tph_en;</div><div class='del'>-	qp-&gt;rcv_tph_en = info-&gt;rcv_tph_en;</div><div class='del'>-	qp-&gt;xmit_tph_en = info-&gt;xmit_tph_en;</div><div class='del'>-	qp-&gt;qs_handle = qp-&gt;vsi-&gt;qos[qp-&gt;user_pri].qs_handle;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_qp_create - create qp</div><div class='del'>- * @qp: sc qp</div><div class='del'>- * @info: qp create info</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_qp_create(</div><div class='del'>-				struct i40iw_sc_qp *qp,</div><div class='del'>-				struct i40iw_create_qp_info *info,</div><div class='del'>-				u64 scratch,</div><div class='del'>-				bool post_sq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	if ((qp-&gt;qp_uk.qp_id &lt; I40IW_MIN_IW_QP_ID) ||</div><div class='del'>-	    (qp-&gt;qp_uk.qp_id &gt; I40IW_MAX_IW_QP_ID))</div><div class='del'>-		return I40IW_ERR_INVALID_QP_ID;</div><div class='del'>-</div><div class='del'>-	cqp = qp-&gt;pd-&gt;dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 16, qp-&gt;hw_host_ctx_pa);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 40, qp-&gt;shadow_area_pa);</div><div class='del'>-</div><div class='del'>-	header = qp-&gt;qp_uk.qp_id |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_CREATE_QP, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64((info-&gt;ord_valid ? 1 : 0), I40IW_CQPSQ_QP_ORDVALID) |</div><div class='del'>-		 LS_64(info-&gt;tcp_ctx_valid, I40IW_CQPSQ_QP_TOECTXVALID) |</div><div class='del'>-		 LS_64(qp-&gt;qp_type, I40IW_CQPSQ_QP_QPTYPE) |</div><div class='del'>-		 LS_64(qp-&gt;virtual_map, I40IW_CQPSQ_QP_VQ) |</div><div class='del'>-		 LS_64(info-&gt;cq_num_valid, I40IW_CQPSQ_QP_CQNUMVALID) |</div><div class='del'>-		 LS_64(info-&gt;arp_cache_idx_valid, I40IW_CQPSQ_QP_ARPTABIDXVALID) |</div><div class='del'>-		 LS_64(info-&gt;next_iwarp_state, I40IW_CQPSQ_QP_NEXTIWSTATE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "QP_CREATE WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_qp_modify - modify qp cqp wqe</div><div class='del'>- * @qp: sc qp</div><div class='del'>- * @info: modify qp info</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_qp_modify(</div><div class='del'>-				struct i40iw_sc_qp *qp,</div><div class='del'>-				struct i40iw_modify_qp_info *info,</div><div class='del'>-				u64 scratch,</div><div class='del'>-				bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 header;</div><div class='del'>-	u8 term_actions = 0;</div><div class='del'>-	u8 term_len = 0;</div><div class='del'>-</div><div class='del'>-	cqp = qp-&gt;pd-&gt;dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	if (info-&gt;next_iwarp_state == I40IW_QP_STATE_TERMINATE) {</div><div class='del'>-		if (info-&gt;dont_send_fin)</div><div class='del'>-			term_actions += I40IWQP_TERM_SEND_TERM_ONLY;</div><div class='del'>-		if (info-&gt;dont_send_term)</div><div class='del'>-			term_actions += I40IWQP_TERM_SEND_FIN_ONLY;</div><div class='del'>-		if ((term_actions == I40IWQP_TERM_SEND_TERM_AND_FIN) ||</div><div class='del'>-		    (term_actions == I40IWQP_TERM_SEND_TERM_ONLY))</div><div class='del'>-			term_len = info-&gt;termlen;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      8,</div><div class='del'>-		      LS_64(term_len, I40IW_CQPSQ_QP_TERMLEN));</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 16, qp-&gt;hw_host_ctx_pa);</div><div class='del'>-	set_64bit_val(wqe, 40, qp-&gt;shadow_area_pa);</div><div class='del'>-</div><div class='del'>-	header = qp-&gt;qp_uk.qp_id |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_MODIFY_QP, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(info-&gt;ord_valid, I40IW_CQPSQ_QP_ORDVALID) |</div><div class='del'>-		 LS_64(info-&gt;tcp_ctx_valid, I40IW_CQPSQ_QP_TOECTXVALID) |</div><div class='del'>-		 LS_64(info-&gt;cached_var_valid, I40IW_CQPSQ_QP_CACHEDVARVALID) |</div><div class='del'>-		 LS_64(qp-&gt;virtual_map, I40IW_CQPSQ_QP_VQ) |</div><div class='del'>-		 LS_64(info-&gt;cq_num_valid, I40IW_CQPSQ_QP_CQNUMVALID) |</div><div class='del'>-		 LS_64(info-&gt;force_loopback, I40IW_CQPSQ_QP_FORCELOOPBACK) |</div><div class='del'>-		 LS_64(qp-&gt;qp_type, I40IW_CQPSQ_QP_QPTYPE) |</div><div class='del'>-		 LS_64(info-&gt;remove_hash_idx, I40IW_CQPSQ_QP_REMOVEHASHENTRY) |</div><div class='del'>-		 LS_64(term_actions, I40IW_CQPSQ_QP_TERMACT) |</div><div class='del'>-		 LS_64(info-&gt;reset_tcp_conn, I40IW_CQPSQ_QP_RESETCON) |</div><div class='del'>-		 LS_64(info-&gt;arp_cache_idx_valid, I40IW_CQPSQ_QP_ARPTABIDXVALID) |</div><div class='del'>-		 LS_64(info-&gt;next_iwarp_state, I40IW_CQPSQ_QP_NEXTIWSTATE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "QP_MODIFY WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_qp_destroy - cqp destroy qp</div><div class='del'>- * @qp: sc qp</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @remove_hash_idx: flag if to remove hash idx</div><div class='del'>- * @ignore_mw_bnd: memory window bind flag</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_qp_destroy(</div><div class='del'>-					struct i40iw_sc_qp *qp,</div><div class='del'>-					u64 scratch,</div><div class='del'>-					bool remove_hash_idx,</div><div class='del'>-					bool ignore_mw_bnd,</div><div class='del'>-					bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	i40iw_qp_rem_qos(qp);</div><div class='del'>-	cqp = qp-&gt;pd-&gt;dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	set_64bit_val(wqe, 16, qp-&gt;hw_host_ctx_pa);</div><div class='del'>-	set_64bit_val(wqe, 40, qp-&gt;shadow_area_pa);</div><div class='del'>-</div><div class='del'>-	header = qp-&gt;qp_uk.qp_id |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_DESTROY_QP, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(qp-&gt;qp_type, I40IW_CQPSQ_QP_QPTYPE) |</div><div class='del'>-		 LS_64(ignore_mw_bnd, I40IW_CQPSQ_QP_IGNOREMWBOUND) |</div><div class='del'>-		 LS_64(remove_hash_idx, I40IW_CQPSQ_QP_REMOVEHASHENTRY) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "QP_DESTROY WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_qp_flush_wqes - flush qp's wqe</div><div class='del'>- * @qp: sc qp</div><div class='del'>- * @info: dlush information</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_qp_flush_wqes(</div><div class='del'>-				struct i40iw_sc_qp *qp,</div><div class='del'>-				struct i40iw_qp_flush_info *info,</div><div class='del'>-				u64 scratch,</div><div class='del'>-				bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 temp = 0;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 header;</div><div class='del'>-	bool flush_sq = false, flush_rq = false;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;rq &amp;&amp; !qp-&gt;flush_rq)</div><div class='del'>-		flush_rq = true;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;sq &amp;&amp; !qp-&gt;flush_sq)</div><div class='del'>-		flush_sq = true;</div><div class='del'>-</div><div class='del'>-	qp-&gt;flush_sq |= flush_sq;</div><div class='del'>-	qp-&gt;flush_rq |= flush_rq;</div><div class='del'>-	if (!flush_sq &amp;&amp; !flush_rq)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	cqp = qp-&gt;pd-&gt;dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	if (info-&gt;userflushcode) {</div><div class='del'>-		if (flush_rq) {</div><div class='del'>-			temp |= LS_64(info-&gt;rq_minor_code, I40IW_CQPSQ_FWQE_RQMNERR) |</div><div class='del'>-				LS_64(info-&gt;rq_major_code, I40IW_CQPSQ_FWQE_RQMJERR);</div><div class='del'>-		}</div><div class='del'>-		if (flush_sq) {</div><div class='del'>-			temp |= LS_64(info-&gt;sq_minor_code, I40IW_CQPSQ_FWQE_SQMNERR) |</div><div class='del'>-				LS_64(info-&gt;sq_major_code, I40IW_CQPSQ_FWQE_SQMJERR);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	set_64bit_val(wqe, 16, temp);</div><div class='del'>-</div><div class='del'>-	temp = (info-&gt;generate_ae) ?</div><div class='del'>-		info-&gt;ae_code | LS_64(info-&gt;ae_source, I40IW_CQPSQ_FWQE_AESOURCE) : 0;</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 8, temp);</div><div class='del'>-</div><div class='del'>-	header = qp-&gt;qp_uk.qp_id |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_FLUSH_WQES, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(info-&gt;generate_ae, I40IW_CQPSQ_FWQE_GENERATE_AE) |</div><div class='del'>-		 LS_64(info-&gt;userflushcode, I40IW_CQPSQ_FWQE_USERFLCODE) |</div><div class='del'>-		 LS_64(flush_sq, I40IW_CQPSQ_FWQE_FLUSHSQ) |</div><div class='del'>-		 LS_64(flush_rq, I40IW_CQPSQ_FWQE_FLUSHRQ) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "QP_FLUSH WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_gen_ae - generate AE, currently uses flush WQE CQP OP</div><div class='del'>- * @qp: sc qp</div><div class='del'>- * @info: gen ae information</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_gen_ae(</div><div class='del'>-				struct i40iw_sc_qp *qp,</div><div class='del'>-				struct i40iw_gen_ae_info *info,</div><div class='del'>-				u64 scratch,</div><div class='del'>-				bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 temp;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	cqp = qp-&gt;pd-&gt;dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	temp = info-&gt;ae_code |</div><div class='del'>-	       LS_64(info-&gt;ae_source, I40IW_CQPSQ_FWQE_AESOURCE);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 8, temp);</div><div class='del'>-</div><div class='del'>-	header = qp-&gt;qp_uk.qp_id |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_GEN_AE, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(1, I40IW_CQPSQ_FWQE_GENERATE_AE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "GEN_AE WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_qp_upload_context - upload qp's context</div><div class='del'>- * @dev: sc device struct</div><div class='del'>- * @info: upload context info ptr for return</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_qp_upload_context(</div><div class='del'>-					struct i40iw_sc_dev *dev,</div><div class='del'>-					struct i40iw_upload_context_info *info,</div><div class='del'>-					u64 scratch,</div><div class='del'>-					bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 header;</div><div class='del'>-</div><div class='del'>-	cqp = dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	set_64bit_val(wqe, 16, info-&gt;buf_pa);</div><div class='del'>-</div><div class='del'>-	header = LS_64(info-&gt;qp_id, I40IW_CQPSQ_UCTX_QPID) |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_UPLOAD_CONTEXT, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(info-&gt;qp_type, I40IW_CQPSQ_UCTX_QPTYPE) |</div><div class='del'>-		 LS_64(info-&gt;raw_format, I40IW_CQPSQ_UCTX_RAWFORMAT) |</div><div class='del'>-		 LS_64(info-&gt;freeze_qp, I40IW_CQPSQ_UCTX_FREEZEQP) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "QP_UPLOAD_CTX WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_qp_setctx - set qp's context</div><div class='del'>- * @qp: sc qp</div><div class='del'>- * @qp_ctx: context ptr</div><div class='del'>- * @info: ctx info</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_qp_setctx(</div><div class='del'>-				struct i40iw_sc_qp *qp,</div><div class='del'>-				u64 *qp_ctx,</div><div class='del'>-				struct i40iw_qp_host_ctx_info *info)</div><div class='del'>-{</div><div class='del'>-	struct i40iwarp_offload_info *iw;</div><div class='del'>-	struct i40iw_tcp_offload_info *tcp;</div><div class='del'>-	struct i40iw_sc_vsi *vsi;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	u64 qw0, qw3, qw7 = 0;</div><div class='del'>-</div><div class='del'>-	iw = info-&gt;iwarp_info;</div><div class='del'>-	tcp = info-&gt;tcp_info;</div><div class='del'>-	vsi = qp-&gt;vsi;</div><div class='del'>-	dev = qp-&gt;dev;</div><div class='del'>-	if (info-&gt;add_to_qoslist) {</div><div class='del'>-		qp-&gt;user_pri = info-&gt;user_pri;</div><div class='del'>-		i40iw_qp_add_qos(qp);</div><div class='del'>-		i40iw_debug(qp-&gt;dev, I40IW_DEBUG_DCB, "%s qp[%d] UP[%d] qset[%d]\n",</div><div class='del'>-			    __func__, qp-&gt;qp_uk.qp_id, qp-&gt;user_pri, qp-&gt;qs_handle);</div><div class='del'>-	}</div><div class='del'>-	qw0 = LS_64(qp-&gt;qp_uk.rq_wqe_size, I40IWQPC_RQWQESIZE) |</div><div class='del'>-	      LS_64(info-&gt;err_rq_idx_valid, I40IWQPC_ERR_RQ_IDX_VALID) |</div><div class='del'>-	      LS_64(qp-&gt;rcv_tph_en, I40IWQPC_RCVTPHEN) |</div><div class='del'>-	      LS_64(qp-&gt;xmit_tph_en, I40IWQPC_XMITTPHEN) |</div><div class='del'>-	      LS_64(qp-&gt;rq_tph_en, I40IWQPC_RQTPHEN) |</div><div class='del'>-	      LS_64(qp-&gt;sq_tph_en, I40IWQPC_SQTPHEN) |</div><div class='del'>-	      LS_64(info-&gt;push_idx, I40IWQPC_PPIDX) |</div><div class='del'>-	      LS_64(info-&gt;push_mode_en, I40IWQPC_PMENA);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(qp_ctx, 8, qp-&gt;sq_pa);</div><div class='del'>-	set_64bit_val(qp_ctx, 16, qp-&gt;rq_pa);</div><div class='del'>-</div><div class='del'>-	qw3 = LS_64(qp-&gt;src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) |</div><div class='del'>-	      LS_64(qp-&gt;hw_rq_size, I40IWQPC_RQSIZE) |</div><div class='del'>-	      LS_64(qp-&gt;hw_sq_size, I40IWQPC_SQSIZE);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(qp_ctx,</div><div class='del'>-		      128,</div><div class='del'>-		      LS_64(info-&gt;err_rq_idx, I40IWQPC_ERR_RQ_IDX));</div><div class='del'>-</div><div class='del'>-	set_64bit_val(qp_ctx,</div><div class='del'>-		      136,</div><div class='del'>-		      LS_64(info-&gt;send_cq_num, I40IWQPC_TXCQNUM) |</div><div class='del'>-		      LS_64(info-&gt;rcv_cq_num, I40IWQPC_RXCQNUM));</div><div class='del'>-</div><div class='del'>-	set_64bit_val(qp_ctx,</div><div class='del'>-		      168,</div><div class='del'>-		      LS_64(info-&gt;qp_compl_ctx, I40IWQPC_QPCOMPCTX));</div><div class='del'>-	set_64bit_val(qp_ctx,</div><div class='del'>-		      176,</div><div class='del'>-		      LS_64(qp-&gt;sq_tph_val, I40IWQPC_SQTPHVAL) |</div><div class='del'>-		      LS_64(qp-&gt;rq_tph_val, I40IWQPC_RQTPHVAL) |</div><div class='del'>-		      LS_64(qp-&gt;qs_handle, I40IWQPC_QSHANDLE) |</div><div class='del'>-		      LS_64(vsi-&gt;exception_lan_queue, I40IWQPC_EXCEPTION_LAN_QUEUE));</div><div class='del'>-</div><div class='del'>-	if (info-&gt;iwarp_info_valid) {</div><div class='del'>-		qw0 |= LS_64(iw-&gt;ddp_ver, I40IWQPC_DDP_VER) |</div><div class='del'>-		       LS_64(iw-&gt;rdmap_ver, I40IWQPC_RDMAP_VER);</div><div class='del'>-</div><div class='del'>-		qw7 |= LS_64(iw-&gt;pd_id, I40IWQPC_PDIDX);</div><div class='del'>-		set_64bit_val(qp_ctx,</div><div class='del'>-			      144,</div><div class='del'>-			      LS_64(qp-&gt;q2_pa, I40IWQPC_Q2ADDR) |</div><div class='del'>-			      LS_64(vsi-&gt;fcn_id, I40IWQPC_STAT_INDEX));</div><div class='del'>-		set_64bit_val(qp_ctx,</div><div class='del'>-			      152,</div><div class='del'>-			      LS_64(iw-&gt;last_byte_sent, I40IWQPC_LASTBYTESENT));</div><div class='del'>-</div><div class='del'>-		set_64bit_val(qp_ctx,</div><div class='del'>-			      160,</div><div class='del'>-			      LS_64(iw-&gt;ord_size, I40IWQPC_ORDSIZE) |</div><div class='del'>-			      LS_64(iw-&gt;ird_size, I40IWQPC_IRDSIZE) |</div><div class='del'>-			      LS_64(iw-&gt;wr_rdresp_en, I40IWQPC_WRRDRSPOK) |</div><div class='del'>-			      LS_64(iw-&gt;rd_enable, I40IWQPC_RDOK) |</div><div class='del'>-			      LS_64(iw-&gt;snd_mark_en, I40IWQPC_SNDMARKERS) |</div><div class='del'>-			      LS_64(iw-&gt;bind_en, I40IWQPC_BINDEN) |</div><div class='del'>-			      LS_64(iw-&gt;fast_reg_en, I40IWQPC_FASTREGEN) |</div><div class='del'>-			      LS_64(iw-&gt;priv_mode_en, I40IWQPC_PRIVEN) |</div><div class='del'>-			      LS_64((((vsi-&gt;stats_fcn_id_alloc) &amp;&amp;</div><div class='del'>-				      (dev-&gt;is_pf) &amp;&amp; (vsi-&gt;fcn_id &gt;= I40IW_FIRST_NON_PF_STAT)) ? 1 : 0),</div><div class='del'>-				    I40IWQPC_USESTATSINSTANCE) |</div><div class='del'>-			      LS_64(1, I40IWQPC_IWARPMODE) |</div><div class='del'>-			      LS_64(iw-&gt;rcv_mark_en, I40IWQPC_RCVMARKERS) |</div><div class='del'>-			      LS_64(iw-&gt;align_hdrs, I40IWQPC_ALIGNHDRS) |</div><div class='del'>-			      LS_64(iw-&gt;rcv_no_mpa_crc, I40IWQPC_RCVNOMPACRC) |</div><div class='del'>-			      LS_64(iw-&gt;rcv_mark_offset, I40IWQPC_RCVMARKOFFSET) |</div><div class='del'>-			      LS_64(iw-&gt;snd_mark_offset, I40IWQPC_SNDMARKOFFSET));</div><div class='del'>-	}</div><div class='del'>-	if (info-&gt;tcp_info_valid) {</div><div class='del'>-		qw0 |= LS_64(tcp-&gt;ipv4, I40IWQPC_IPV4) |</div><div class='del'>-		       LS_64(tcp-&gt;no_nagle, I40IWQPC_NONAGLE) |</div><div class='del'>-		       LS_64(tcp-&gt;insert_vlan_tag, I40IWQPC_INSERTVLANTAG) |</div><div class='del'>-		       LS_64(tcp-&gt;time_stamp, I40IWQPC_TIMESTAMP) |</div><div class='del'>-		       LS_64(tcp-&gt;cwnd_inc_limit, I40IWQPC_LIMIT) |</div><div class='del'>-		       LS_64(tcp-&gt;drop_ooo_seg, I40IWQPC_DROPOOOSEG) |</div><div class='del'>-		       LS_64(tcp-&gt;dup_ack_thresh, I40IWQPC_DUPACK_THRESH);</div><div class='del'>-</div><div class='del'>-		qw3 |= LS_64(tcp-&gt;ttl, I40IWQPC_TTL) |</div><div class='del'>-		       LS_64(tcp-&gt;src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) |</div><div class='del'>-		       LS_64(tcp-&gt;avoid_stretch_ack, I40IWQPC_AVOIDSTRETCHACK) |</div><div class='del'>-		       LS_64(tcp-&gt;tos, I40IWQPC_TOS) |</div><div class='del'>-		       LS_64(tcp-&gt;src_port, I40IWQPC_SRCPORTNUM) |</div><div class='del'>-		       LS_64(tcp-&gt;dst_port, I40IWQPC_DESTPORTNUM);</div><div class='del'>-</div><div class='del'>-		qp-&gt;src_mac_addr_idx = tcp-&gt;src_mac_addr_idx;</div><div class='del'>-		set_64bit_val(qp_ctx,</div><div class='del'>-			      32,</div><div class='del'>-			      LS_64(tcp-&gt;dest_ip_addr2, I40IWQPC_DESTIPADDR2) |</div><div class='del'>-			      LS_64(tcp-&gt;dest_ip_addr3, I40IWQPC_DESTIPADDR3));</div><div class='del'>-</div><div class='del'>-		set_64bit_val(qp_ctx,</div><div class='del'>-			      40,</div><div class='del'>-			      LS_64(tcp-&gt;dest_ip_addr0, I40IWQPC_DESTIPADDR0) |</div><div class='del'>-			      LS_64(tcp-&gt;dest_ip_addr1, I40IWQPC_DESTIPADDR1));</div><div class='del'>-</div><div class='del'>-		set_64bit_val(qp_ctx,</div><div class='del'>-			      48,</div><div class='del'>-			      LS_64(tcp-&gt;snd_mss, I40IWQPC_SNDMSS) |</div><div class='del'>-				LS_64(tcp-&gt;vlan_tag, I40IWQPC_VLANTAG) |</div><div class='del'>-				LS_64(tcp-&gt;arp_idx, I40IWQPC_ARPIDX));</div><div class='del'>-</div><div class='del'>-		qw7 |= LS_64(tcp-&gt;flow_label, I40IWQPC_FLOWLABEL) |</div><div class='del'>-		       LS_64(tcp-&gt;wscale, I40IWQPC_WSCALE) |</div><div class='del'>-		       LS_64(tcp-&gt;ignore_tcp_opt, I40IWQPC_IGNORE_TCP_OPT) |</div><div class='del'>-		       LS_64(tcp-&gt;ignore_tcp_uns_opt, I40IWQPC_IGNORE_TCP_UNS_OPT) |</div><div class='del'>-		       LS_64(tcp-&gt;tcp_state, I40IWQPC_TCPSTATE) |</div><div class='del'>-		       LS_64(tcp-&gt;rcv_wscale, I40IWQPC_RCVSCALE) |</div><div class='del'>-		       LS_64(tcp-&gt;snd_wscale, I40IWQPC_SNDSCALE);</div><div class='del'>-</div><div class='del'>-		set_64bit_val(qp_ctx,</div><div class='del'>-			      72,</div><div class='del'>-			      LS_64(tcp-&gt;time_stamp_recent, I40IWQPC_TIMESTAMP_RECENT) |</div><div class='del'>-			      LS_64(tcp-&gt;time_stamp_age, I40IWQPC_TIMESTAMP_AGE));</div><div class='del'>-		set_64bit_val(qp_ctx,</div><div class='del'>-			      80,</div><div class='del'>-			      LS_64(tcp-&gt;snd_nxt, I40IWQPC_SNDNXT) |</div><div class='del'>-			      LS_64(tcp-&gt;snd_wnd, I40IWQPC_SNDWND));</div><div class='del'>-</div><div class='del'>-		set_64bit_val(qp_ctx,</div><div class='del'>-			      88,</div><div class='del'>-			      LS_64(tcp-&gt;rcv_nxt, I40IWQPC_RCVNXT) |</div><div class='del'>-			      LS_64(tcp-&gt;rcv_wnd, I40IWQPC_RCVWND));</div><div class='del'>-		set_64bit_val(qp_ctx,</div><div class='del'>-			      96,</div><div class='del'>-			      LS_64(tcp-&gt;snd_max, I40IWQPC_SNDMAX) |</div><div class='del'>-			      LS_64(tcp-&gt;snd_una, I40IWQPC_SNDUNA));</div><div class='del'>-		set_64bit_val(qp_ctx,</div><div class='del'>-			      104,</div><div class='del'>-			      LS_64(tcp-&gt;srtt, I40IWQPC_SRTT) |</div><div class='del'>-			      LS_64(tcp-&gt;rtt_var, I40IWQPC_RTTVAR));</div><div class='del'>-		set_64bit_val(qp_ctx,</div><div class='del'>-			      112,</div><div class='del'>-			      LS_64(tcp-&gt;ss_thresh, I40IWQPC_SSTHRESH) |</div><div class='del'>-			      LS_64(tcp-&gt;cwnd, I40IWQPC_CWND));</div><div class='del'>-		set_64bit_val(qp_ctx,</div><div class='del'>-			      120,</div><div class='del'>-			      LS_64(tcp-&gt;snd_wl1, I40IWQPC_SNDWL1) |</div><div class='del'>-			      LS_64(tcp-&gt;snd_wl2, I40IWQPC_SNDWL2));</div><div class='del'>-		set_64bit_val(qp_ctx,</div><div class='del'>-			      128,</div><div class='del'>-			      LS_64(tcp-&gt;max_snd_window, I40IWQPC_MAXSNDWND) |</div><div class='del'>-			      LS_64(tcp-&gt;rexmit_thresh, I40IWQPC_REXMIT_THRESH));</div><div class='del'>-		set_64bit_val(qp_ctx,</div><div class='del'>-			      184,</div><div class='del'>-			      LS_64(tcp-&gt;local_ipaddr3, I40IWQPC_LOCAL_IPADDR3) |</div><div class='del'>-			      LS_64(tcp-&gt;local_ipaddr2, I40IWQPC_LOCAL_IPADDR2));</div><div class='del'>-		set_64bit_val(qp_ctx,</div><div class='del'>-			      192,</div><div class='del'>-			      LS_64(tcp-&gt;local_ipaddr1, I40IWQPC_LOCAL_IPADDR1) |</div><div class='del'>-			      LS_64(tcp-&gt;local_ipaddr0, I40IWQPC_LOCAL_IPADDR0));</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	set_64bit_val(qp_ctx, 0, qw0);</div><div class='del'>-	set_64bit_val(qp_ctx, 24, qw3);</div><div class='del'>-	set_64bit_val(qp_ctx, 56, qw7);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(qp-&gt;dev, I40IW_DEBUG_WQE, "QP_HOST)CTX WQE",</div><div class='del'>-			qp_ctx, I40IW_QP_CTX_SIZE);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_alloc_stag - mr stag alloc</div><div class='del'>- * @dev: sc device struct</div><div class='del'>- * @info: stag info</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_alloc_stag(</div><div class='del'>-				struct i40iw_sc_dev *dev,</div><div class='del'>-				struct i40iw_allocate_stag_info *info,</div><div class='del'>-				u64 scratch,</div><div class='del'>-				bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 header;</div><div class='del'>-	enum i40iw_page_size page_size;</div><div class='del'>-</div><div class='del'>-	page_size = (info-&gt;page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;</div><div class='del'>-	cqp = dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      8,</div><div class='del'>-		      LS_64(info-&gt;pd_id, I40IW_CQPSQ_STAG_PDID) |</div><div class='del'>-		      LS_64(info-&gt;total_len, I40IW_CQPSQ_STAG_STAGLEN));</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      16,</div><div class='del'>-		      LS_64(info-&gt;stag_idx, I40IW_CQPSQ_STAG_IDX));</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      40,</div><div class='del'>-		      LS_64(info-&gt;hmc_fcn_index, I40IW_CQPSQ_STAG_HMCFNIDX));</div><div class='del'>-</div><div class='del'>-	header = LS_64(I40IW_CQP_OP_ALLOC_STAG, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(1, I40IW_CQPSQ_STAG_MR) |</div><div class='del'>-		 LS_64(info-&gt;access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |</div><div class='del'>-		 LS_64(info-&gt;chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) |</div><div class='del'>-		 LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) |</div><div class='del'>-		 LS_64(info-&gt;remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |</div><div class='del'>-		 LS_64(info-&gt;use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) |</div><div class='del'>-		 LS_64(info-&gt;use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "ALLOC_STAG WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_mr_reg_non_shared - non-shared mr registration</div><div class='del'>- * @dev: sc device struct</div><div class='del'>- * @info: mr info</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_mr_reg_non_shared(</div><div class='del'>-				struct i40iw_sc_dev *dev,</div><div class='del'>-				struct i40iw_reg_ns_stag_info *info,</div><div class='del'>-				u64 scratch,</div><div class='del'>-				bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 temp;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 header;</div><div class='del'>-	u32 pble_obj_cnt;</div><div class='del'>-	bool remote_access;</div><div class='del'>-	u8 addr_type;</div><div class='del'>-	enum i40iw_page_size page_size;</div><div class='del'>-</div><div class='del'>-	page_size = (info-&gt;page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;</div><div class='del'>-	if (info-&gt;access_rights &amp; (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY |</div><div class='del'>-				   I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY))</div><div class='del'>-		remote_access = true;</div><div class='del'>-	else</div><div class='del'>-		remote_access = false;</div><div class='del'>-</div><div class='del'>-	pble_obj_cnt = dev-&gt;hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].cnt;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;chunk_size &amp;&amp; (info-&gt;first_pm_pbl_index &gt;= pble_obj_cnt))</div><div class='del'>-		return I40IW_ERR_INVALID_PBLE_INDEX;</div><div class='del'>-</div><div class='del'>-	cqp = dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	temp = (info-&gt;addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info-&gt;va : info-&gt;fbo;</div><div class='del'>-	set_64bit_val(wqe, 0, temp);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      8,</div><div class='del'>-		      LS_64(info-&gt;total_len, I40IW_CQPSQ_STAG_STAGLEN) |</div><div class='del'>-		      LS_64(info-&gt;pd_id, I40IW_CQPSQ_STAG_PDID));</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      16,</div><div class='del'>-		      LS_64(info-&gt;stag_key, I40IW_CQPSQ_STAG_KEY) |</div><div class='del'>-		      LS_64(info-&gt;stag_idx, I40IW_CQPSQ_STAG_IDX));</div><div class='del'>-	if (!info-&gt;chunk_size) {</div><div class='del'>-		set_64bit_val(wqe, 32, info-&gt;reg_addr_pa);</div><div class='del'>-		set_64bit_val(wqe, 48, 0);</div><div class='del'>-	} else {</div><div class='del'>-		set_64bit_val(wqe, 32, 0);</div><div class='del'>-		set_64bit_val(wqe, 48, info-&gt;first_pm_pbl_index);</div><div class='del'>-	}</div><div class='del'>-	set_64bit_val(wqe, 40, info-&gt;hmc_fcn_index);</div><div class='del'>-	set_64bit_val(wqe, 56, 0);</div><div class='del'>-</div><div class='del'>-	addr_type = (info-&gt;addr_type == I40IW_ADDR_TYPE_VA_BASED) ? 1 : 0;</div><div class='del'>-	header = LS_64(I40IW_CQP_OP_REG_MR, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(1, I40IW_CQPSQ_STAG_MR) |</div><div class='del'>-		 LS_64(info-&gt;chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) |</div><div class='del'>-		 LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) |</div><div class='del'>-		 LS_64(info-&gt;access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |</div><div class='del'>-		 LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |</div><div class='del'>-		 LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) |</div><div class='del'>-		 LS_64(info-&gt;use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) |</div><div class='del'>-		 LS_64(info-&gt;use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MR_REG_NS WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_mr_reg_shared - registered shared memory region</div><div class='del'>- * @dev: sc device struct</div><div class='del'>- * @info: info for shared memory registeration</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_mr_reg_shared(</div><div class='del'>-					struct i40iw_sc_dev *dev,</div><div class='del'>-					struct i40iw_register_shared_stag *info,</div><div class='del'>-					u64 scratch,</div><div class='del'>-					bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 temp, va64, fbo, header;</div><div class='del'>-	u32 va32;</div><div class='del'>-	bool remote_access;</div><div class='del'>-	u8 addr_type;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;access_rights &amp; (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY |</div><div class='del'>-				   I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY))</div><div class='del'>-		remote_access = true;</div><div class='del'>-	else</div><div class='del'>-		remote_access = false;</div><div class='del'>-	cqp = dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	va64 = (uintptr_t)(info-&gt;va);</div><div class='del'>-	va32 = (u32)(va64 &amp; 0x00000000FFFFFFFF);</div><div class='del'>-	fbo = (u64)(va32 &amp; (4096 - 1));</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      0,</div><div class='del'>-		      (info-&gt;addr_type == I40IW_ADDR_TYPE_VA_BASED ? (uintptr_t)info-&gt;va : fbo));</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      8,</div><div class='del'>-		      LS_64(info-&gt;pd_id, I40IW_CQPSQ_STAG_PDID));</div><div class='del'>-	temp = LS_64(info-&gt;new_stag_key, I40IW_CQPSQ_STAG_KEY) |</div><div class='del'>-	       LS_64(info-&gt;new_stag_idx, I40IW_CQPSQ_STAG_IDX) |</div><div class='del'>-	       LS_64(info-&gt;parent_stag_idx, I40IW_CQPSQ_STAG_PARENTSTAGIDX);</div><div class='del'>-	set_64bit_val(wqe, 16, temp);</div><div class='del'>-</div><div class='del'>-	addr_type = (info-&gt;addr_type == I40IW_ADDR_TYPE_VA_BASED) ? 1 : 0;</div><div class='del'>-	header = LS_64(I40IW_CQP_OP_REG_SMR, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(1, I40IW_CQPSQ_STAG_MR) |</div><div class='del'>-		 LS_64(info-&gt;access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |</div><div class='del'>-		 LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |</div><div class='del'>-		 LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MR_REG_SHARED WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_dealloc_stag - deallocate stag</div><div class='del'>- * @dev: sc device struct</div><div class='del'>- * @info: dealloc stag info</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_dealloc_stag(</div><div class='del'>-					struct i40iw_sc_dev *dev,</div><div class='del'>-					struct i40iw_dealloc_stag_info *info,</div><div class='del'>-					u64 scratch,</div><div class='del'>-					bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 header;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-</div><div class='del'>-	cqp = dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      8,</div><div class='del'>-		      LS_64(info-&gt;pd_id, I40IW_CQPSQ_STAG_PDID));</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      16,</div><div class='del'>-		      LS_64(info-&gt;stag_idx, I40IW_CQPSQ_STAG_IDX));</div><div class='del'>-</div><div class='del'>-	header = LS_64(I40IW_CQP_OP_DEALLOC_STAG, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(info-&gt;mr, I40IW_CQPSQ_STAG_MR) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "DEALLOC_STAG WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_query_stag - query hardware for stag</div><div class='del'>- * @dev: sc device struct</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @stag_index: stag index for query</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_query_stag(struct i40iw_sc_dev *dev,</div><div class='del'>-						  u64 scratch,</div><div class='del'>-						  u32 stag_index,</div><div class='del'>-						  bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 header;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-</div><div class='del'>-	cqp = dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      16,</div><div class='del'>-		      LS_64(stag_index, I40IW_CQPSQ_QUERYSTAG_IDX));</div><div class='del'>-</div><div class='del'>-	header = LS_64(I40IW_CQP_OP_QUERY_STAG, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "QUERY_STAG WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_mw_alloc - mw allocate</div><div class='del'>- * @dev: sc device struct</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @mw_stag_index:stag index</div><div class='del'>- * @pd_id: pd is for this mw</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_mw_alloc(</div><div class='del'>-					struct i40iw_sc_dev *dev,</div><div class='del'>-					u64 scratch,</div><div class='del'>-					u32 mw_stag_index,</div><div class='del'>-					u16 pd_id,</div><div class='del'>-					bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 header;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-</div><div class='del'>-	cqp = dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	set_64bit_val(wqe, 8, LS_64(pd_id, I40IW_CQPSQ_STAG_PDID));</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      16,</div><div class='del'>-		      LS_64(mw_stag_index, I40IW_CQPSQ_STAG_IDX));</div><div class='del'>-</div><div class='del'>-	header = LS_64(I40IW_CQP_OP_ALLOC_STAG, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MW_ALLOC WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp</div><div class='del'>- * @qp: sc qp struct</div><div class='del'>- * @info: fast mr info</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_sc_mr_fast_register(</div><div class='del'>-				struct i40iw_sc_qp *qp,</div><div class='del'>-				struct i40iw_fast_reg_stag_info *info,</div><div class='del'>-				bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 temp, header;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u32 wqe_idx;</div><div class='del'>-	enum i40iw_page_size page_size;</div><div class='del'>-</div><div class='del'>-	page_size = (info-&gt;page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;</div><div class='del'>-	wqe = i40iw_qp_get_next_send_wqe(&amp;qp-&gt;qp_uk, &amp;wqe_idx, I40IW_QP_WQE_MIN_SIZE,</div><div class='del'>-					 0, info-&gt;wr_id);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_QP_TOOMANY_WRS_POSTED;</div><div class='del'>-</div><div class='del'>-	i40iw_debug(qp-&gt;dev, I40IW_DEBUG_MR, "%s: wr_id[%llxh] wqe_idx[%04d] location[%p]\n",</div><div class='del'>-		    __func__, info-&gt;wr_id, wqe_idx,</div><div class='del'>-		    &amp;qp-&gt;qp_uk.sq_wrtrk_array[wqe_idx].wrid);</div><div class='del'>-	temp = (info-&gt;addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info-&gt;va : info-&gt;fbo;</div><div class='del'>-	set_64bit_val(wqe, 0, temp);</div><div class='del'>-</div><div class='del'>-	temp = RS_64(info-&gt;first_pm_pbl_index &gt;&gt; 16, I40IWQPSQ_FIRSTPMPBLIDXHI);</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      8,</div><div class='del'>-		      LS_64(temp, I40IWQPSQ_FIRSTPMPBLIDXHI) |</div><div class='del'>-		      LS_64(info-&gt;reg_addr_pa &gt;&gt; I40IWQPSQ_PBLADDR_SHIFT, I40IWQPSQ_PBLADDR));</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      16,</div><div class='del'>-		      info-&gt;total_len |</div><div class='del'>-		      LS_64(info-&gt;first_pm_pbl_index, I40IWQPSQ_FIRSTPMPBLIDXLO));</div><div class='del'>-</div><div class='del'>-	header = LS_64(info-&gt;stag_key, I40IWQPSQ_STAGKEY) |</div><div class='del'>-		 LS_64(info-&gt;stag_idx, I40IWQPSQ_STAGINDEX) |</div><div class='del'>-		 LS_64(I40IWQP_OP_FAST_REGISTER, I40IWQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(info-&gt;chunk_size, I40IWQPSQ_LPBLSIZE) |</div><div class='del'>-		 LS_64(page_size, I40IWQPSQ_HPAGESIZE) |</div><div class='del'>-		 LS_64(info-&gt;access_rights, I40IWQPSQ_STAGRIGHTS) |</div><div class='del'>-		 LS_64(info-&gt;addr_type, I40IWQPSQ_VABASEDTO) |</div><div class='del'>-		 LS_64(info-&gt;read_fence, I40IWQPSQ_READFENCE) |</div><div class='del'>-		 LS_64(info-&gt;local_fence, I40IWQPSQ_LOCALFENCE) |</div><div class='del'>-		 LS_64(info-&gt;signaled, I40IWQPSQ_SIGCOMPL) |</div><div class='del'>-		 LS_64(qp-&gt;qp_uk.swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(qp-&gt;dev, I40IW_DEBUG_WQE, "FAST_REG WQE",</div><div class='del'>-			wqe, I40IW_QP_WQE_MIN_SIZE);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_qp_post_wr(&amp;qp-&gt;qp_uk);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_send_lsmm - send last streaming mode message</div><div class='del'>- * @qp: sc qp struct</div><div class='del'>- * @lsmm_buf: buffer with lsmm message</div><div class='del'>- * @size: size of lsmm buffer</div><div class='del'>- * @stag: stag of lsmm buffer</div><div class='del'>- */</div><div class='del'>-static void i40iw_sc_send_lsmm(struct i40iw_sc_qp *qp,</div><div class='del'>-			       void *lsmm_buf,</div><div class='del'>-			       u32 size,</div><div class='del'>-			       i40iw_stag stag)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-	struct i40iw_qp_uk *qp_uk;</div><div class='del'>-</div><div class='del'>-	qp_uk = &amp;qp-&gt;qp_uk;</div><div class='del'>-	wqe = qp_uk-&gt;sq_base-&gt;elem;</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 8, (size | LS_64(stag, I40IWQPSQ_FRAG_STAG)));</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 16, 0);</div><div class='del'>-</div><div class='del'>-	header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(1, I40IWQPSQ_STREAMMODE) |</div><div class='del'>-		 LS_64(1, I40IWQPSQ_WAITFORRCVPDU) |</div><div class='del'>-		 LS_64(qp-&gt;qp_uk.swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(qp-&gt;dev, I40IW_DEBUG_QP, "SEND_LSMM WQE",</div><div class='del'>-			wqe, I40IW_QP_WQE_MIN_SIZE);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_send_lsmm_nostag - for privilege qp</div><div class='del'>- * @qp: sc qp struct</div><div class='del'>- * @lsmm_buf: buffer with lsmm message</div><div class='del'>- * @size: size of lsmm buffer</div><div class='del'>- */</div><div class='del'>-static void i40iw_sc_send_lsmm_nostag(struct i40iw_sc_qp *qp,</div><div class='del'>-				      void *lsmm_buf,</div><div class='del'>-				      u32 size)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-	struct i40iw_qp_uk *qp_uk;</div><div class='del'>-</div><div class='del'>-	qp_uk = &amp;qp-&gt;qp_uk;</div><div class='del'>-	wqe = qp_uk-&gt;sq_base-&gt;elem;</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 8, size);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 16, 0);</div><div class='del'>-</div><div class='del'>-	header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(1, I40IWQPSQ_STREAMMODE) |</div><div class='del'>-		 LS_64(1, I40IWQPSQ_WAITFORRCVPDU) |</div><div class='del'>-		 LS_64(qp-&gt;qp_uk.swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(qp-&gt;dev, I40IW_DEBUG_WQE, "SEND_LSMM_NOSTAG WQE",</div><div class='del'>-			wqe, I40IW_QP_WQE_MIN_SIZE);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_send_rtt - send last read0 or write0</div><div class='del'>- * @qp: sc qp struct</div><div class='del'>- * @read: Do read0 or write0</div><div class='del'>- */</div><div class='del'>-static void i40iw_sc_send_rtt(struct i40iw_sc_qp *qp, bool read)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-	struct i40iw_qp_uk *qp_uk;</div><div class='del'>-</div><div class='del'>-	qp_uk = &amp;qp-&gt;qp_uk;</div><div class='del'>-	wqe = qp_uk-&gt;sq_base-&gt;elem;</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 0, 0);</div><div class='del'>-	set_64bit_val(wqe, 8, 0);</div><div class='del'>-	set_64bit_val(wqe, 16, 0);</div><div class='del'>-	if (read) {</div><div class='del'>-		header = LS_64(0x1234, I40IWQPSQ_REMSTAG) |</div><div class='del'>-			 LS_64(I40IWQP_OP_RDMA_READ, I40IWQPSQ_OPCODE) |</div><div class='del'>-			 LS_64(qp-&gt;qp_uk.swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-		set_64bit_val(wqe, 8, ((u64)0xabcd &lt;&lt; 32));</div><div class='del'>-	} else {</div><div class='del'>-		header = LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) |</div><div class='del'>-			 LS_64(qp-&gt;qp_uk.swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(qp-&gt;dev, I40IW_DEBUG_WQE, "RTR WQE",</div><div class='del'>-			wqe, I40IW_QP_WQE_MIN_SIZE);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_post_wqe0 - send wqe with opcode</div><div class='del'>- * @qp: sc qp struct</div><div class='del'>- * @opcode: opcode to use for wqe0</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_post_wqe0(struct i40iw_sc_qp *qp, u8 opcode)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-	struct i40iw_qp_uk *qp_uk;</div><div class='del'>-</div><div class='del'>-	qp_uk = &amp;qp-&gt;qp_uk;</div><div class='del'>-	wqe = qp_uk-&gt;sq_base-&gt;elem;</div><div class='del'>-</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_QP_TOOMANY_WRS_POSTED;</div><div class='del'>-	switch (opcode) {</div><div class='del'>-	case I40IWQP_OP_NOP:</div><div class='del'>-		set_64bit_val(wqe, 0, 0);</div><div class='del'>-		set_64bit_val(wqe, 8, 0);</div><div class='del'>-		set_64bit_val(wqe, 16, 0);</div><div class='del'>-		header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) |</div><div class='del'>-			 LS_64(qp-&gt;qp_uk.swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-</div><div class='del'>-		i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-		break;</div><div class='del'>-	case I40IWQP_OP_RDMA_SEND:</div><div class='del'>-		set_64bit_val(wqe, 0, 0);</div><div class='del'>-		set_64bit_val(wqe, 8, 0);</div><div class='del'>-		set_64bit_val(wqe, 16, 0);</div><div class='del'>-		header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |</div><div class='del'>-			 LS_64(qp-&gt;qp_uk.swqe_polarity, I40IWQPSQ_VALID) |</div><div class='del'>-			 LS_64(1, I40IWQPSQ_STREAMMODE) |</div><div class='del'>-			 LS_64(1, I40IWQPSQ_WAITFORRCVPDU);</div><div class='del'>-</div><div class='del'>-		i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		i40iw_debug(qp-&gt;dev, I40IW_DEBUG_QP, "%s: Invalid WQE zero opcode\n",</div><div class='del'>-			    __func__);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info</div><div class='del'>- * @dev : ptr to i40iw_dev struct</div><div class='del'>- * @hmc_fn_id: hmc function id</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_sc_init_iw_hmc(struct i40iw_sc_dev *dev, u8 hmc_fn_id)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_hmc_info *hmc_info;</div><div class='del'>-	struct i40iw_dma_mem query_fpm_mem;</div><div class='del'>-	struct i40iw_virt_mem virt_mem;</div><div class='del'>-	struct i40iw_vfdev *vf_dev = NULL;</div><div class='del'>-	u32 mem_size;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-	bool poll_registers = true;</div><div class='del'>-	u16 iw_vf_idx;</div><div class='del'>-	u8 wait_type;</div><div class='del'>-</div><div class='del'>-	if (hmc_fn_id &gt;= I40IW_MAX_VF_FPM_ID ||</div><div class='del'>-	    (dev-&gt;hmc_fn_id != hmc_fn_id &amp;&amp; hmc_fn_id &lt; I40IW_FIRST_VF_FPM_ID))</div><div class='del'>-		return I40IW_ERR_INVALID_HMCFN_ID;</div><div class='del'>-</div><div class='del'>-	i40iw_debug(dev, I40IW_DEBUG_HMC, "hmc_fn_id %u, dev-&gt;hmc_fn_id %u\n", hmc_fn_id,</div><div class='del'>-		    dev-&gt;hmc_fn_id);</div><div class='del'>-	if (hmc_fn_id == dev-&gt;hmc_fn_id) {</div><div class='del'>-		hmc_info = dev-&gt;hmc_info;</div><div class='del'>-		query_fpm_mem.pa = dev-&gt;fpm_query_buf_pa;</div><div class='del'>-		query_fpm_mem.va = dev-&gt;fpm_query_buf;</div><div class='del'>-	} else {</div><div class='del'>-		vf_dev = i40iw_vfdev_from_fpm(dev, hmc_fn_id);</div><div class='del'>-		if (!vf_dev)</div><div class='del'>-			return I40IW_ERR_INVALID_VF_ID;</div><div class='del'>-</div><div class='del'>-		hmc_info = &amp;vf_dev-&gt;hmc_info;</div><div class='del'>-		iw_vf_idx = vf_dev-&gt;iw_vf_idx;</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_HMC, "vf_dev %p, hmc_info %p, hmc_obj %p\n", vf_dev,</div><div class='del'>-			    hmc_info, hmc_info-&gt;hmc_obj);</div><div class='del'>-		if (!vf_dev-&gt;fpm_query_buf) {</div><div class='del'>-			if (!dev-&gt;vf_fpm_query_buf[iw_vf_idx].va) {</div><div class='del'>-				ret_code = i40iw_alloc_query_fpm_buf(dev,</div><div class='del'>-								     &amp;dev-&gt;vf_fpm_query_buf[iw_vf_idx]);</div><div class='del'>-				if (ret_code)</div><div class='del'>-					return ret_code;</div><div class='del'>-			}</div><div class='del'>-			vf_dev-&gt;fpm_query_buf = dev-&gt;vf_fpm_query_buf[iw_vf_idx].va;</div><div class='del'>-			vf_dev-&gt;fpm_query_buf_pa = dev-&gt;vf_fpm_query_buf[iw_vf_idx].pa;</div><div class='del'>-		}</div><div class='del'>-		query_fpm_mem.pa = vf_dev-&gt;fpm_query_buf_pa;</div><div class='del'>-		query_fpm_mem.va = vf_dev-&gt;fpm_query_buf;</div><div class='del'>-		/**</div><div class='del'>-		 * It is HARDWARE specific:</div><div class='del'>-		 * this call is done by PF for VF and</div><div class='del'>-		 * i40iw_sc_query_fpm_values needs ccq poll</div><div class='del'>-		 * because PF ccq is already created.</div><div class='del'>-		 */</div><div class='del'>-		poll_registers = false;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	hmc_info-&gt;hmc_fn_id = hmc_fn_id;</div><div class='del'>-</div><div class='del'>-	if (hmc_fn_id != dev-&gt;hmc_fn_id) {</div><div class='del'>-		ret_code =</div><div class='del'>-			i40iw_cqp_query_fpm_values_cmd(dev, &amp;query_fpm_mem, hmc_fn_id);</div><div class='del'>-	} else {</div><div class='del'>-		wait_type = poll_registers ? (u8)I40IW_CQP_WAIT_POLL_REGS :</div><div class='del'>-			    (u8)I40IW_CQP_WAIT_POLL_CQ;</div><div class='del'>-</div><div class='del'>-		ret_code = i40iw_sc_query_fpm_values(</div><div class='del'>-					dev-&gt;cqp,</div><div class='del'>-					0,</div><div class='del'>-					hmc_info-&gt;hmc_fn_id,</div><div class='del'>-					&amp;query_fpm_mem,</div><div class='del'>-					true,</div><div class='del'>-					wait_type);</div><div class='del'>-	}</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return ret_code;</div><div class='del'>-</div><div class='del'>-	/* parse the fpm_query_buf and fill hmc obj info */</div><div class='del'>-	ret_code =</div><div class='del'>-		i40iw_sc_parse_fpm_query_buf((u64 *)query_fpm_mem.va,</div><div class='del'>-					     hmc_info,</div><div class='del'>-					     &amp;dev-&gt;hmc_fpm_misc);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return ret_code;</div><div class='del'>-	i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "QUERY FPM BUFFER",</div><div class='del'>-			query_fpm_mem.va, I40IW_QUERY_FPM_BUF_SIZE);</div><div class='del'>-</div><div class='del'>-	if (hmc_fn_id != dev-&gt;hmc_fn_id) {</div><div class='del'>-		i40iw_cqp_commit_fpm_values_cmd(dev, &amp;query_fpm_mem, hmc_fn_id);</div><div class='del'>-</div><div class='del'>-		/* parse the fpm_commit_buf and fill hmc obj info */</div><div class='del'>-		i40iw_sc_parse_fpm_commit_buf((u64 *)query_fpm_mem.va, hmc_info-&gt;hmc_obj, &amp;hmc_info-&gt;sd_table.sd_cnt);</div><div class='del'>-		mem_size = sizeof(struct i40iw_hmc_sd_entry) *</div><div class='del'>-			   (hmc_info-&gt;sd_table.sd_cnt + hmc_info-&gt;first_sd_index);</div><div class='del'>-		ret_code = i40iw_allocate_virt_mem(dev-&gt;hw, &amp;virt_mem, mem_size);</div><div class='del'>-		if (ret_code)</div><div class='del'>-			return ret_code;</div><div class='del'>-		hmc_info-&gt;sd_table.sd_entry = virt_mem.va;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_configure_iw_fpm() - commits hmc obj cnt values using cqp command and</div><div class='del'>- * populates fpm base address in hmc_info</div><div class='del'>- * @dev : ptr to i40iw_dev struct</div><div class='del'>- * @hmc_fn_id: hmc function id</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_sc_configure_iw_fpm(struct i40iw_sc_dev *dev,</div><div class='del'>-							u8 hmc_fn_id)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_hmc_info *hmc_info;</div><div class='del'>-	struct i40iw_hmc_obj_info *obj_info;</div><div class='del'>-	u64 *buf;</div><div class='del'>-	struct i40iw_dma_mem commit_fpm_mem;</div><div class='del'>-	u32 i, j;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-	bool poll_registers = true;</div><div class='del'>-	u8 wait_type;</div><div class='del'>-</div><div class='del'>-	if (hmc_fn_id &gt;= I40IW_MAX_VF_FPM_ID ||</div><div class='del'>-	    (dev-&gt;hmc_fn_id != hmc_fn_id &amp;&amp; hmc_fn_id &lt; I40IW_FIRST_VF_FPM_ID))</div><div class='del'>-		return I40IW_ERR_INVALID_HMCFN_ID;</div><div class='del'>-</div><div class='del'>-	if (hmc_fn_id == dev-&gt;hmc_fn_id) {</div><div class='del'>-		hmc_info = dev-&gt;hmc_info;</div><div class='del'>-	} else {</div><div class='del'>-		hmc_info = i40iw_vf_hmcinfo_from_fpm(dev, hmc_fn_id);</div><div class='del'>-		poll_registers = false;</div><div class='del'>-	}</div><div class='del'>-	if (!hmc_info)</div><div class='del'>-		return I40IW_ERR_BAD_PTR;</div><div class='del'>-</div><div class='del'>-	obj_info = hmc_info-&gt;hmc_obj;</div><div class='del'>-	buf = dev-&gt;fpm_commit_buf;</div><div class='del'>-</div><div class='del'>-	/* copy cnt values in commit buf */</div><div class='del'>-	for (i = I40IW_HMC_IW_QP, j = 0; i &lt;= I40IW_HMC_IW_PBLE;</div><div class='del'>-	     i++, j += 8)</div><div class='del'>-		set_64bit_val(buf, j, (u64)obj_info[i].cnt);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(buf, 40, 0);   /* APBVT rsvd */</div><div class='del'>-</div><div class='del'>-	commit_fpm_mem.pa = dev-&gt;fpm_commit_buf_pa;</div><div class='del'>-	commit_fpm_mem.va = dev-&gt;fpm_commit_buf;</div><div class='del'>-	wait_type = poll_registers ? (u8)I40IW_CQP_WAIT_POLL_REGS :</div><div class='del'>-			(u8)I40IW_CQP_WAIT_POLL_CQ;</div><div class='del'>-	ret_code = i40iw_sc_commit_fpm_values(</div><div class='del'>-					dev-&gt;cqp,</div><div class='del'>-					0,</div><div class='del'>-					hmc_info-&gt;hmc_fn_id,</div><div class='del'>-					&amp;commit_fpm_mem,</div><div class='del'>-					true,</div><div class='del'>-					wait_type);</div><div class='del'>-</div><div class='del'>-	/* parse the fpm_commit_buf and fill hmc obj info */</div><div class='del'>-	if (!ret_code)</div><div class='del'>-		ret_code = i40iw_sc_parse_fpm_commit_buf(dev-&gt;fpm_commit_buf,</div><div class='del'>-							 hmc_info-&gt;hmc_obj,</div><div class='del'>-							 &amp;hmc_info-&gt;sd_table.sd_cnt);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "COMMIT FPM BUFFER",</div><div class='del'>-			commit_fpm_mem.va, I40IW_COMMIT_FPM_BUF_SIZE);</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * cqp_sds_wqe_fill - fill cqp wqe doe sd</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @info; sd info for wqe</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code cqp_sds_wqe_fill(struct i40iw_sc_cqp *cqp,</div><div class='del'>-					       struct i40iw_update_sds_info *info,</div><div class='del'>-					       u64 scratch)</div><div class='del'>-{</div><div class='del'>-	u64 data;</div><div class='del'>-	u64 header;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	int mem_entries, wqe_entries;</div><div class='del'>-	struct i40iw_dma_mem *sdbuf = &amp;cqp-&gt;sdbuf;</div><div class='del'>-	u64 offset;</div><div class='del'>-	u32 wqe_idx;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &amp;wqe_idx);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	I40IW_CQP_INIT_WQE(wqe);</div><div class='del'>-	wqe_entries = (info-&gt;cnt &gt; 3) ? 3 : info-&gt;cnt;</div><div class='del'>-	mem_entries = info-&gt;cnt - wqe_entries;</div><div class='del'>-</div><div class='del'>-	header = LS_64(I40IW_CQP_OP_UPDATE_PE_SDS, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID) |</div><div class='del'>-		 LS_64(mem_entries, I40IW_CQPSQ_UPESD_ENTRY_COUNT);</div><div class='del'>-</div><div class='del'>-	if (mem_entries) {</div><div class='del'>-		offset = wqe_idx * I40IW_UPDATE_SD_BUF_SIZE;</div><div class='del'>-		memcpy((char *)sdbuf-&gt;va + offset, &amp;info-&gt;entry[3],</div><div class='del'>-		       mem_entries &lt;&lt; 4);</div><div class='del'>-		data = (u64)sdbuf-&gt;pa + offset;</div><div class='del'>-	} else {</div><div class='del'>-		data = 0;</div><div class='del'>-	}</div><div class='del'>-	data |= LS_64(info-&gt;hmc_fn_id, I40IW_CQPSQ_UPESD_HMCFNID);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 16, data);</div><div class='del'>-</div><div class='del'>-	switch (wqe_entries) {</div><div class='del'>-	case 3:</div><div class='del'>-		set_64bit_val(wqe, 48,</div><div class='del'>-			      (LS_64(info-&gt;entry[2].cmd, I40IW_CQPSQ_UPESD_SDCMD) |</div><div class='del'>-					LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID)));</div><div class='del'>-</div><div class='del'>-		set_64bit_val(wqe, 56, info-&gt;entry[2].data);</div><div class='del'>-		/* fallthrough */</div><div class='del'>-	case 2:</div><div class='del'>-		set_64bit_val(wqe, 32,</div><div class='del'>-			      (LS_64(info-&gt;entry[1].cmd, I40IW_CQPSQ_UPESD_SDCMD) |</div><div class='del'>-					LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID)));</div><div class='del'>-</div><div class='del'>-		set_64bit_val(wqe, 40, info-&gt;entry[1].data);</div><div class='del'>-		/* fallthrough */</div><div class='del'>-	case 1:</div><div class='del'>-		set_64bit_val(wqe, 0,</div><div class='del'>-			      LS_64(info-&gt;entry[0].cmd, I40IW_CQPSQ_UPESD_SDCMD));</div><div class='del'>-</div><div class='del'>-		set_64bit_val(wqe, 8, info-&gt;entry[0].data);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "UPDATE_PE_SDS WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_update_pe_sds - cqp wqe for sd</div><div class='del'>- * @dev: ptr to i40iw_dev struct</div><div class='del'>- * @info: sd info for sd's</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_update_pe_sds(struct i40iw_sc_dev *dev,</div><div class='del'>-						  struct i40iw_update_sds_info *info,</div><div class='del'>-						  u64 scratch)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_cqp *cqp = dev-&gt;cqp;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-</div><div class='del'>-	ret_code = cqp_sds_wqe_fill(cqp, info, scratch);</div><div class='del'>-	if (!ret_code)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_update_sds_noccq - update sd before ccq created</div><div class='del'>- * @dev: sc device struct</div><div class='del'>- * @info: sd info for sd's</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_update_sds_noccq(struct i40iw_sc_dev *dev,</div><div class='del'>-					      struct i40iw_update_sds_info *info)</div><div class='del'>-{</div><div class='del'>-	u32 error, val, tail;</div><div class='del'>-	struct i40iw_sc_cqp *cqp = dev-&gt;cqp;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-</div><div class='del'>-	ret_code = cqp_sds_wqe_fill(cqp, info, 0);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return ret_code;</div><div class='del'>-	i40iw_get_cqp_reg_info(cqp, &amp;val, &amp;tail, &amp;error);</div><div class='del'>-	if (error)</div><div class='del'>-		return I40IW_ERR_CQP_COMPL_ERROR;</div><div class='del'>-</div><div class='del'>-	i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_suspend_qp - suspend qp for param change</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @qp: sc qp struct</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_sc_suspend_qp(struct i40iw_sc_cqp *cqp,</div><div class='del'>-					   struct i40iw_sc_qp *qp,</div><div class='del'>-					   u64 scratch)</div><div class='del'>-{</div><div class='del'>-	u64 header;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	header = LS_64(qp-&gt;qp_uk.qp_id, I40IW_CQPSQ_SUSPENDQP_QPID) |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_SUSPEND_QP, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "SUSPEND_QP WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_resume_qp - resume qp after suspend</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @qp: sc qp struct</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_sc_resume_qp(struct i40iw_sc_cqp *cqp,</div><div class='del'>-					  struct i40iw_sc_qp *qp,</div><div class='del'>-					  u64 scratch)</div><div class='del'>-{</div><div class='del'>-	u64 header;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      16,</div><div class='del'>-			LS_64(qp-&gt;qs_handle, I40IW_CQPSQ_RESUMEQP_QSHANDLE));</div><div class='del'>-</div><div class='del'>-	header = LS_64(qp-&gt;qp_uk.qp_id, I40IW_CQPSQ_RESUMEQP_QPID) |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_RESUME_QP, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "RESUME_QP WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- * @scratch: u64 saved to be used during cqp completion</div><div class='del'>- * @hmc_fn_id: hmc function id</div><div class='del'>- * @post_sq: flag for cqp db to ring</div><div class='del'>- * @poll_registers: flag to poll register for cqp completion</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_sc_static_hmc_pages_allocated(</div><div class='del'>-					struct i40iw_sc_cqp *cqp,</div><div class='del'>-					u64 scratch,</div><div class='del'>-					u8 hmc_fn_id,</div><div class='del'>-					bool post_sq,</div><div class='del'>-					bool poll_registers)</div><div class='del'>-{</div><div class='del'>-	u64 header;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u32 tail, val, error;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-	set_64bit_val(wqe,</div><div class='del'>-		      16,</div><div class='del'>-		      LS_64(hmc_fn_id, I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID));</div><div class='del'>-</div><div class='del'>-	header = LS_64(I40IW_CQP_OP_SHMC_PAGES_ALLOCATED, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "SHMC_PAGES_ALLOCATED WQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-	i40iw_get_cqp_reg_info(cqp, &amp;val, &amp;tail, &amp;error);</div><div class='del'>-	if (error) {</div><div class='del'>-		ret_code = I40IW_ERR_CQP_COMPL_ERROR;</div><div class='del'>-		return ret_code;</div><div class='del'>-	}</div><div class='del'>-	if (post_sq) {</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-		if (poll_registers)</div><div class='del'>-			/* check for cqp sq tail update */</div><div class='del'>-			ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000);</div><div class='del'>-		else</div><div class='del'>-			ret_code = i40iw_sc_poll_for_cqp_op_done(cqp,</div><div class='del'>-								 I40IW_CQP_OP_SHMC_PAGES_ALLOCATED,</div><div class='del'>-								 NULL);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ring_full - check if cqp ring is full</div><div class='del'>- * @cqp: struct for cqp hw</div><div class='del'>- */</div><div class='del'>-static bool i40iw_ring_full(struct i40iw_sc_cqp *cqp)</div><div class='del'>-{</div><div class='del'>-	return I40IW_RING_FULL_ERR(cqp-&gt;sq_ring);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_est_sd - returns approximate number of SDs for HMC</div><div class='del'>- * @dev: sc device struct</div><div class='del'>- * @hmc_info: hmc structure, size and count for HMC objects</div><div class='del'>- */</div><div class='del'>-static u64 i40iw_est_sd(struct i40iw_sc_dev *dev, struct i40iw_hmc_info *hmc_info)</div><div class='del'>-{</div><div class='del'>-	int i;</div><div class='del'>-	u64 size = 0;</div><div class='del'>-	u64 sd;</div><div class='del'>-</div><div class='del'>-	for (i = I40IW_HMC_IW_QP; i &lt; I40IW_HMC_IW_PBLE; i++)</div><div class='del'>-		size += hmc_info-&gt;hmc_obj[i].cnt * hmc_info-&gt;hmc_obj[i].size;</div><div class='del'>-</div><div class='del'>-	if (dev-&gt;is_pf)</div><div class='del'>-		size += hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].size;</div><div class='del'>-</div><div class='del'>-	if (size &amp; 0x1FFFFF)</div><div class='del'>-		sd = (size &gt;&gt; 21) + 1; /* add 1 for remainder */</div><div class='del'>-	else</div><div class='del'>-		sd = size &gt;&gt; 21;</div><div class='del'>-</div><div class='del'>-	if (!dev-&gt;is_pf) {</div><div class='del'>-		/* 2MB alignment for VF PBLE HMC */</div><div class='del'>-		size = hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].size;</div><div class='del'>-		if (size &amp; 0x1FFFFF)</div><div class='del'>-			sd += (size &gt;&gt; 21) + 1; /* add 1 for remainder */</div><div class='del'>-		else</div><div class='del'>-			sd += size &gt;&gt; 21;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return sd;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_config_fpm_values - configure HMC objects</div><div class='del'>- * @dev: sc device struct</div><div class='del'>- * @qp_count: desired qp count</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_count)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_virt_mem virt_mem;</div><div class='del'>-	u32 i, mem_size;</div><div class='del'>-	u32 qpwantedoriginal, qpwanted, mrwanted, pblewanted;</div><div class='del'>-	u64 sd_needed;</div><div class='del'>-	u32 loop_count = 0;</div><div class='del'>-</div><div class='del'>-	struct i40iw_hmc_info *hmc_info;</div><div class='del'>-	struct i40iw_hmc_fpm_misc *hmc_fpm_misc;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-</div><div class='del'>-	hmc_info = dev-&gt;hmc_info;</div><div class='del'>-	hmc_fpm_misc = &amp;dev-&gt;hmc_fpm_misc;</div><div class='del'>-</div><div class='del'>-	ret_code = i40iw_sc_init_iw_hmc(dev, dev-&gt;hmc_fn_id);</div><div class='del'>-	if (ret_code) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_HMC,</div><div class='del'>-			    "i40iw_sc_init_iw_hmc returned error_code = %d\n",</div><div class='del'>-			    ret_code);</div><div class='del'>-		return ret_code;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	for (i = I40IW_HMC_IW_QP; i &lt; I40IW_HMC_IW_MAX; i++)</div><div class='del'>-		hmc_info-&gt;hmc_obj[i].cnt = hmc_info-&gt;hmc_obj[i].max_cnt;</div><div class='del'>-	sd_needed = i40iw_est_sd(dev, hmc_info);</div><div class='del'>-	i40iw_debug(dev, I40IW_DEBUG_HMC,</div><div class='del'>-		    "%s: FW initial max sd_count[%08lld] first_sd_index[%04d]\n",</div><div class='del'>-		    __func__, sd_needed, hmc_info-&gt;first_sd_index);</div><div class='del'>-	i40iw_debug(dev, I40IW_DEBUG_HMC,</div><div class='del'>-		    "%s: sd count %d where max sd is %d\n",</div><div class='del'>-		    __func__, hmc_info-&gt;sd_table.sd_cnt,</div><div class='del'>-		    hmc_fpm_misc-&gt;max_sds);</div><div class='del'>-</div><div class='del'>-	qpwanted = min(qp_count, hmc_info-&gt;hmc_obj[I40IW_HMC_IW_QP].max_cnt);</div><div class='del'>-	qpwantedoriginal = qpwanted;</div><div class='del'>-	mrwanted = hmc_info-&gt;hmc_obj[I40IW_HMC_IW_MR].max_cnt;</div><div class='del'>-	pblewanted = hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].max_cnt;</div><div class='del'>-</div><div class='del'>-	i40iw_debug(dev, I40IW_DEBUG_HMC,</div><div class='del'>-		    "req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d\n",</div><div class='del'>-		    qp_count, hmc_fpm_misc-&gt;max_sds,</div><div class='del'>-		    hmc_info-&gt;hmc_obj[I40IW_HMC_IW_QP].max_cnt,</div><div class='del'>-		    hmc_info-&gt;hmc_obj[I40IW_HMC_IW_CQ].max_cnt,</div><div class='del'>-		    hmc_info-&gt;hmc_obj[I40IW_HMC_IW_MR].max_cnt,</div><div class='del'>-		    hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].max_cnt);</div><div class='del'>-</div><div class='del'>-	do {</div><div class='del'>-		++loop_count;</div><div class='del'>-		hmc_info-&gt;hmc_obj[I40IW_HMC_IW_QP].cnt = qpwanted;</div><div class='del'>-		hmc_info-&gt;hmc_obj[I40IW_HMC_IW_CQ].cnt =</div><div class='del'>-			min(2 * qpwanted, hmc_info-&gt;hmc_obj[I40IW_HMC_IW_CQ].cnt);</div><div class='del'>-		hmc_info-&gt;hmc_obj[I40IW_HMC_IW_SRQ].cnt = 0x00; /* Reserved */</div><div class='del'>-		hmc_info-&gt;hmc_obj[I40IW_HMC_IW_HTE].cnt =</div><div class='del'>-					qpwanted * hmc_fpm_misc-&gt;ht_multiplier;</div><div class='del'>-		hmc_info-&gt;hmc_obj[I40IW_HMC_IW_ARP].cnt =</div><div class='del'>-			hmc_info-&gt;hmc_obj[I40IW_HMC_IW_ARP].max_cnt;</div><div class='del'>-		hmc_info-&gt;hmc_obj[I40IW_HMC_IW_APBVT_ENTRY].cnt = 1;</div><div class='del'>-		hmc_info-&gt;hmc_obj[I40IW_HMC_IW_MR].cnt = mrwanted;</div><div class='del'>-</div><div class='del'>-		hmc_info-&gt;hmc_obj[I40IW_HMC_IW_XF].cnt =</div><div class='del'>-			roundup_pow_of_two(I40IW_MAX_WQ_ENTRIES * qpwanted);</div><div class='del'>-		hmc_info-&gt;hmc_obj[I40IW_HMC_IW_Q1].cnt =</div><div class='del'>-			roundup_pow_of_two(2 * I40IW_MAX_IRD_SIZE * qpwanted);</div><div class='del'>-		hmc_info-&gt;hmc_obj[I40IW_HMC_IW_XFFL].cnt =</div><div class='del'>-			hmc_info-&gt;hmc_obj[I40IW_HMC_IW_XF].cnt / hmc_fpm_misc-&gt;xf_block_size;</div><div class='del'>-		hmc_info-&gt;hmc_obj[I40IW_HMC_IW_Q1FL].cnt =</div><div class='del'>-			hmc_info-&gt;hmc_obj[I40IW_HMC_IW_Q1].cnt / hmc_fpm_misc-&gt;q1_block_size;</div><div class='del'>-		hmc_info-&gt;hmc_obj[I40IW_HMC_IW_TIMER].cnt =</div><div class='del'>-			((qpwanted) / 512 + 1) * hmc_fpm_misc-&gt;timer_bucket;</div><div class='del'>-		hmc_info-&gt;hmc_obj[I40IW_HMC_IW_FSIMC].cnt = 0x00;</div><div class='del'>-		hmc_info-&gt;hmc_obj[I40IW_HMC_IW_FSIAV].cnt = 0x00;</div><div class='del'>-		hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].cnt = pblewanted;</div><div class='del'>-</div><div class='del'>-		/* How much memory is needed for all the objects. */</div><div class='del'>-		sd_needed = i40iw_est_sd(dev, hmc_info);</div><div class='del'>-		if ((loop_count &gt; 1000) ||</div><div class='del'>-		    ((!(loop_count % 10)) &amp;&amp;</div><div class='del'>-		    (qpwanted &gt; qpwantedoriginal * 2 / 3))) {</div><div class='del'>-			if (qpwanted &gt; FPM_MULTIPLIER)</div><div class='del'>-				qpwanted = roundup_pow_of_two(qpwanted -</div><div class='del'>-							      FPM_MULTIPLIER);</div><div class='del'>-			qpwanted &gt;&gt;= 1;</div><div class='del'>-		}</div><div class='del'>-		if (mrwanted &gt; FPM_MULTIPLIER * 10)</div><div class='del'>-			mrwanted -= FPM_MULTIPLIER * 10;</div><div class='del'>-		if (pblewanted &gt; FPM_MULTIPLIER * 1000)</div><div class='del'>-			pblewanted -= FPM_MULTIPLIER * 1000;</div><div class='del'>-	} while (sd_needed &gt; hmc_fpm_misc-&gt;max_sds &amp;&amp; loop_count &lt; 2000);</div><div class='del'>-</div><div class='del'>-	i40iw_debug(dev, I40IW_DEBUG_HMC,</div><div class='del'>-		    "loop_cnt=%d, sd_needed=%lld, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d\n",</div><div class='del'>-		    loop_count, sd_needed,</div><div class='del'>-		    hmc_info-&gt;hmc_obj[I40IW_HMC_IW_QP].cnt,</div><div class='del'>-		    hmc_info-&gt;hmc_obj[I40IW_HMC_IW_CQ].cnt,</div><div class='del'>-		    hmc_info-&gt;hmc_obj[I40IW_HMC_IW_MR].cnt,</div><div class='del'>-		    hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].cnt);</div><div class='del'>-</div><div class='del'>-	ret_code = i40iw_sc_configure_iw_fpm(dev, dev-&gt;hmc_fn_id);</div><div class='del'>-	if (ret_code) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_HMC,</div><div class='del'>-			    "configure_iw_fpm returned error_code[x%08X]\n",</div><div class='del'>-			    i40iw_rd32(dev-&gt;hw, dev-&gt;is_pf ? I40E_PFPE_CQPERRCODES : I40E_VFPE_CQPERRCODES1));</div><div class='del'>-		return ret_code;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mem_size = sizeof(struct i40iw_hmc_sd_entry) *</div><div class='del'>-		   (hmc_info-&gt;sd_table.sd_cnt + hmc_info-&gt;first_sd_index + 1);</div><div class='del'>-	ret_code = i40iw_allocate_virt_mem(dev-&gt;hw, &amp;virt_mem, mem_size);</div><div class='del'>-	if (ret_code) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_HMC,</div><div class='del'>-			    "%s: failed to allocate memory for sd_entry buffer\n",</div><div class='del'>-			    __func__);</div><div class='del'>-		return ret_code;</div><div class='del'>-	}</div><div class='del'>-	hmc_info-&gt;sd_table.sd_entry = virt_mem.va;</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_exec_cqp_cmd - execute cqp cmd when wqe are available</div><div class='del'>- * @dev: rdma device</div><div class='del'>- * @pcmdinfo: cqp command info</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_exec_cqp_cmd(struct i40iw_sc_dev *dev,</div><div class='del'>-						 struct cqp_commands_info *pcmdinfo)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_dma_mem values_mem;</div><div class='del'>-</div><div class='del'>-	dev-&gt;cqp_cmd_stats[pcmdinfo-&gt;cqp_cmd]++;</div><div class='del'>-	switch (pcmdinfo-&gt;cqp_cmd) {</div><div class='del'>-	case OP_DELETE_LOCAL_MAC_IPADDR_ENTRY:</div><div class='del'>-		status = i40iw_sc_del_local_mac_ipaddr_entry(</div><div class='del'>-				pcmdinfo-&gt;in.u.del_local_mac_ipaddr_entry.cqp,</div><div class='del'>-				pcmdinfo-&gt;in.u.del_local_mac_ipaddr_entry.scratch,</div><div class='del'>-				pcmdinfo-&gt;in.u.del_local_mac_ipaddr_entry.entry_idx,</div><div class='del'>-				pcmdinfo-&gt;in.u.del_local_mac_ipaddr_entry.ignore_ref_count,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-		break;</div><div class='del'>-	case OP_CEQ_DESTROY:</div><div class='del'>-		status = i40iw_sc_ceq_destroy(pcmdinfo-&gt;in.u.ceq_destroy.ceq,</div><div class='del'>-					      pcmdinfo-&gt;in.u.ceq_destroy.scratch,</div><div class='del'>-					      pcmdinfo-&gt;post_sq);</div><div class='del'>-		break;</div><div class='del'>-	case OP_AEQ_DESTROY:</div><div class='del'>-		status = i40iw_sc_aeq_destroy(pcmdinfo-&gt;in.u.aeq_destroy.aeq,</div><div class='del'>-					      pcmdinfo-&gt;in.u.aeq_destroy.scratch,</div><div class='del'>-					      pcmdinfo-&gt;post_sq);</div><div class='del'>-</div><div class='del'>-		break;</div><div class='del'>-	case OP_DELETE_ARP_CACHE_ENTRY:</div><div class='del'>-		status = i40iw_sc_del_arp_cache_entry(</div><div class='del'>-				pcmdinfo-&gt;in.u.del_arp_cache_entry.cqp,</div><div class='del'>-				pcmdinfo-&gt;in.u.del_arp_cache_entry.scratch,</div><div class='del'>-				pcmdinfo-&gt;in.u.del_arp_cache_entry.arp_index,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-		break;</div><div class='del'>-	case OP_MANAGE_APBVT_ENTRY:</div><div class='del'>-		status = i40iw_sc_manage_apbvt_entry(</div><div class='del'>-				pcmdinfo-&gt;in.u.manage_apbvt_entry.cqp,</div><div class='del'>-				&amp;pcmdinfo-&gt;in.u.manage_apbvt_entry.info,</div><div class='del'>-				pcmdinfo-&gt;in.u.manage_apbvt_entry.scratch,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-		break;</div><div class='del'>-	case OP_CEQ_CREATE:</div><div class='del'>-		status = i40iw_sc_ceq_create(pcmdinfo-&gt;in.u.ceq_create.ceq,</div><div class='del'>-					     pcmdinfo-&gt;in.u.ceq_create.scratch,</div><div class='del'>-					     pcmdinfo-&gt;post_sq);</div><div class='del'>-		break;</div><div class='del'>-	case OP_AEQ_CREATE:</div><div class='del'>-		status = i40iw_sc_aeq_create(pcmdinfo-&gt;in.u.aeq_create.aeq,</div><div class='del'>-					     pcmdinfo-&gt;in.u.aeq_create.scratch,</div><div class='del'>-					     pcmdinfo-&gt;post_sq);</div><div class='del'>-		break;</div><div class='del'>-	case OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY:</div><div class='del'>-		status = i40iw_sc_alloc_local_mac_ipaddr_entry(</div><div class='del'>-				pcmdinfo-&gt;in.u.alloc_local_mac_ipaddr_entry.cqp,</div><div class='del'>-				pcmdinfo-&gt;in.u.alloc_local_mac_ipaddr_entry.scratch,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-		break;</div><div class='del'>-	case OP_ADD_LOCAL_MAC_IPADDR_ENTRY:</div><div class='del'>-		status = i40iw_sc_add_local_mac_ipaddr_entry(</div><div class='del'>-				pcmdinfo-&gt;in.u.add_local_mac_ipaddr_entry.cqp,</div><div class='del'>-				&amp;pcmdinfo-&gt;in.u.add_local_mac_ipaddr_entry.info,</div><div class='del'>-				pcmdinfo-&gt;in.u.add_local_mac_ipaddr_entry.scratch,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-		break;</div><div class='del'>-	case OP_MANAGE_QHASH_TABLE_ENTRY:</div><div class='del'>-		status = i40iw_sc_manage_qhash_table_entry(</div><div class='del'>-				pcmdinfo-&gt;in.u.manage_qhash_table_entry.cqp,</div><div class='del'>-				&amp;pcmdinfo-&gt;in.u.manage_qhash_table_entry.info,</div><div class='del'>-				pcmdinfo-&gt;in.u.manage_qhash_table_entry.scratch,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-</div><div class='del'>-		break;</div><div class='del'>-	case OP_QP_MODIFY:</div><div class='del'>-		status = i40iw_sc_qp_modify(</div><div class='del'>-				pcmdinfo-&gt;in.u.qp_modify.qp,</div><div class='del'>-				&amp;pcmdinfo-&gt;in.u.qp_modify.info,</div><div class='del'>-				pcmdinfo-&gt;in.u.qp_modify.scratch,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-</div><div class='del'>-		break;</div><div class='del'>-	case OP_QP_UPLOAD_CONTEXT:</div><div class='del'>-		status = i40iw_sc_qp_upload_context(</div><div class='del'>-				pcmdinfo-&gt;in.u.qp_upload_context.dev,</div><div class='del'>-				&amp;pcmdinfo-&gt;in.u.qp_upload_context.info,</div><div class='del'>-				pcmdinfo-&gt;in.u.qp_upload_context.scratch,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-</div><div class='del'>-		break;</div><div class='del'>-	case OP_CQ_CREATE:</div><div class='del'>-		status = i40iw_sc_cq_create(</div><div class='del'>-				pcmdinfo-&gt;in.u.cq_create.cq,</div><div class='del'>-				pcmdinfo-&gt;in.u.cq_create.scratch,</div><div class='del'>-				pcmdinfo-&gt;in.u.cq_create.check_overflow,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-		break;</div><div class='del'>-	case OP_CQ_DESTROY:</div><div class='del'>-		status = i40iw_sc_cq_destroy(</div><div class='del'>-				pcmdinfo-&gt;in.u.cq_destroy.cq,</div><div class='del'>-				pcmdinfo-&gt;in.u.cq_destroy.scratch,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-</div><div class='del'>-		break;</div><div class='del'>-	case OP_QP_CREATE:</div><div class='del'>-		status = i40iw_sc_qp_create(</div><div class='del'>-				pcmdinfo-&gt;in.u.qp_create.qp,</div><div class='del'>-				&amp;pcmdinfo-&gt;in.u.qp_create.info,</div><div class='del'>-				pcmdinfo-&gt;in.u.qp_create.scratch,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-		break;</div><div class='del'>-	case OP_QP_DESTROY:</div><div class='del'>-		status = i40iw_sc_qp_destroy(</div><div class='del'>-				pcmdinfo-&gt;in.u.qp_destroy.qp,</div><div class='del'>-				pcmdinfo-&gt;in.u.qp_destroy.scratch,</div><div class='del'>-				pcmdinfo-&gt;in.u.qp_destroy.remove_hash_idx,</div><div class='del'>-				pcmdinfo-&gt;in.u.qp_destroy.</div><div class='del'>-				ignore_mw_bnd,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-</div><div class='del'>-		break;</div><div class='del'>-	case OP_ALLOC_STAG:</div><div class='del'>-		status = i40iw_sc_alloc_stag(</div><div class='del'>-				pcmdinfo-&gt;in.u.alloc_stag.dev,</div><div class='del'>-				&amp;pcmdinfo-&gt;in.u.alloc_stag.info,</div><div class='del'>-				pcmdinfo-&gt;in.u.alloc_stag.scratch,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-		break;</div><div class='del'>-	case OP_MR_REG_NON_SHARED:</div><div class='del'>-		status = i40iw_sc_mr_reg_non_shared(</div><div class='del'>-				pcmdinfo-&gt;in.u.mr_reg_non_shared.dev,</div><div class='del'>-				&amp;pcmdinfo-&gt;in.u.mr_reg_non_shared.info,</div><div class='del'>-				pcmdinfo-&gt;in.u.mr_reg_non_shared.scratch,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-</div><div class='del'>-		break;</div><div class='del'>-	case OP_DEALLOC_STAG:</div><div class='del'>-		status = i40iw_sc_dealloc_stag(</div><div class='del'>-				pcmdinfo-&gt;in.u.dealloc_stag.dev,</div><div class='del'>-				&amp;pcmdinfo-&gt;in.u.dealloc_stag.info,</div><div class='del'>-				pcmdinfo-&gt;in.u.dealloc_stag.scratch,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-</div><div class='del'>-		break;</div><div class='del'>-	case OP_MW_ALLOC:</div><div class='del'>-		status = i40iw_sc_mw_alloc(</div><div class='del'>-				pcmdinfo-&gt;in.u.mw_alloc.dev,</div><div class='del'>-				pcmdinfo-&gt;in.u.mw_alloc.scratch,</div><div class='del'>-				pcmdinfo-&gt;in.u.mw_alloc.mw_stag_index,</div><div class='del'>-				pcmdinfo-&gt;in.u.mw_alloc.pd_id,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-</div><div class='del'>-		break;</div><div class='del'>-	case OP_QP_FLUSH_WQES:</div><div class='del'>-		status = i40iw_sc_qp_flush_wqes(</div><div class='del'>-				pcmdinfo-&gt;in.u.qp_flush_wqes.qp,</div><div class='del'>-				&amp;pcmdinfo-&gt;in.u.qp_flush_wqes.info,</div><div class='del'>-				pcmdinfo-&gt;in.u.qp_flush_wqes.</div><div class='del'>-				scratch, pcmdinfo-&gt;post_sq);</div><div class='del'>-		break;</div><div class='del'>-	case OP_GEN_AE:</div><div class='del'>-		status = i40iw_sc_gen_ae(</div><div class='del'>-				pcmdinfo-&gt;in.u.gen_ae.qp,</div><div class='del'>-				&amp;pcmdinfo-&gt;in.u.gen_ae.info,</div><div class='del'>-				pcmdinfo-&gt;in.u.gen_ae.scratch,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-		break;</div><div class='del'>-	case OP_ADD_ARP_CACHE_ENTRY:</div><div class='del'>-		status = i40iw_sc_add_arp_cache_entry(</div><div class='del'>-				pcmdinfo-&gt;in.u.add_arp_cache_entry.cqp,</div><div class='del'>-				&amp;pcmdinfo-&gt;in.u.add_arp_cache_entry.info,</div><div class='del'>-				pcmdinfo-&gt;in.u.add_arp_cache_entry.scratch,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-		break;</div><div class='del'>-	case OP_MANAGE_PUSH_PAGE:</div><div class='del'>-		status = i40iw_sc_manage_push_page(</div><div class='del'>-				pcmdinfo-&gt;in.u.manage_push_page.cqp,</div><div class='del'>-				&amp;pcmdinfo-&gt;in.u.manage_push_page.info,</div><div class='del'>-				pcmdinfo-&gt;in.u.manage_push_page.scratch,</div><div class='del'>-				pcmdinfo-&gt;post_sq);</div><div class='del'>-		break;</div><div class='del'>-	case OP_UPDATE_PE_SDS:</div><div class='del'>-		/* case I40IW_CQP_OP_UPDATE_PE_SDS */</div><div class='del'>-		status = i40iw_update_pe_sds(</div><div class='del'>-				pcmdinfo-&gt;in.u.update_pe_sds.dev,</div><div class='del'>-				&amp;pcmdinfo-&gt;in.u.update_pe_sds.info,</div><div class='del'>-				pcmdinfo-&gt;in.u.update_pe_sds.</div><div class='del'>-				scratch);</div><div class='del'>-</div><div class='del'>-		break;</div><div class='del'>-	case OP_MANAGE_HMC_PM_FUNC_TABLE:</div><div class='del'>-		status = i40iw_sc_manage_hmc_pm_func_table(</div><div class='del'>-				pcmdinfo-&gt;in.u.manage_hmc_pm.dev-&gt;cqp,</div><div class='del'>-				pcmdinfo-&gt;in.u.manage_hmc_pm.scratch,</div><div class='del'>-				(u8)pcmdinfo-&gt;in.u.manage_hmc_pm.info.vf_id,</div><div class='del'>-				pcmdinfo-&gt;in.u.manage_hmc_pm.info.free_fcn,</div><div class='del'>-				true);</div><div class='del'>-		break;</div><div class='del'>-	case OP_SUSPEND:</div><div class='del'>-		status = i40iw_sc_suspend_qp(</div><div class='del'>-				pcmdinfo-&gt;in.u.suspend_resume.cqp,</div><div class='del'>-				pcmdinfo-&gt;in.u.suspend_resume.qp,</div><div class='del'>-				pcmdinfo-&gt;in.u.suspend_resume.scratch);</div><div class='del'>-		break;</div><div class='del'>-	case OP_RESUME:</div><div class='del'>-		status = i40iw_sc_resume_qp(</div><div class='del'>-				pcmdinfo-&gt;in.u.suspend_resume.cqp,</div><div class='del'>-				pcmdinfo-&gt;in.u.suspend_resume.qp,</div><div class='del'>-				pcmdinfo-&gt;in.u.suspend_resume.scratch);</div><div class='del'>-		break;</div><div class='del'>-	case OP_MANAGE_VF_PBLE_BP:</div><div class='del'>-		status = i40iw_manage_vf_pble_bp(</div><div class='del'>-				pcmdinfo-&gt;in.u.manage_vf_pble_bp.cqp,</div><div class='del'>-				&amp;pcmdinfo-&gt;in.u.manage_vf_pble_bp.info,</div><div class='del'>-				pcmdinfo-&gt;in.u.manage_vf_pble_bp.scratch, true);</div><div class='del'>-		break;</div><div class='del'>-	case OP_QUERY_FPM_VALUES:</div><div class='del'>-		values_mem.pa = pcmdinfo-&gt;in.u.query_fpm_values.fpm_values_pa;</div><div class='del'>-		values_mem.va = pcmdinfo-&gt;in.u.query_fpm_values.fpm_values_va;</div><div class='del'>-		status = i40iw_sc_query_fpm_values(</div><div class='del'>-				pcmdinfo-&gt;in.u.query_fpm_values.cqp,</div><div class='del'>-				pcmdinfo-&gt;in.u.query_fpm_values.scratch,</div><div class='del'>-				pcmdinfo-&gt;in.u.query_fpm_values.hmc_fn_id,</div><div class='del'>-				&amp;values_mem, true, I40IW_CQP_WAIT_EVENT);</div><div class='del'>-		break;</div><div class='del'>-	case OP_COMMIT_FPM_VALUES:</div><div class='del'>-		values_mem.pa = pcmdinfo-&gt;in.u.commit_fpm_values.fpm_values_pa;</div><div class='del'>-		values_mem.va = pcmdinfo-&gt;in.u.commit_fpm_values.fpm_values_va;</div><div class='del'>-		status = i40iw_sc_commit_fpm_values(</div><div class='del'>-				pcmdinfo-&gt;in.u.commit_fpm_values.cqp,</div><div class='del'>-				pcmdinfo-&gt;in.u.commit_fpm_values.scratch,</div><div class='del'>-				pcmdinfo-&gt;in.u.commit_fpm_values.hmc_fn_id,</div><div class='del'>-				&amp;values_mem,</div><div class='del'>-				true,</div><div class='del'>-				I40IW_CQP_WAIT_EVENT);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		status = I40IW_NOT_SUPPORTED;</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_process_cqp_cmd - process all cqp commands</div><div class='del'>- * @dev: sc device struct</div><div class='del'>- * @pcmdinfo: cqp command info</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_process_cqp_cmd(struct i40iw_sc_dev *dev,</div><div class='del'>-					     struct cqp_commands_info *pcmdinfo)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status = 0;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;dev-&gt;cqp_lock, flags);</div><div class='del'>-	if (list_empty(&amp;dev-&gt;cqp_cmd_head) &amp;&amp; !i40iw_ring_full(dev-&gt;cqp))</div><div class='del'>-		status = i40iw_exec_cqp_cmd(dev, pcmdinfo);</div><div class='del'>-	else</div><div class='del'>-		list_add_tail(&amp;pcmdinfo-&gt;cqp_cmd_entry, &amp;dev-&gt;cqp_cmd_head);</div><div class='del'>-	spin_unlock_irqrestore(&amp;dev-&gt;cqp_lock, flags);</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_process_bh - called from tasklet for cqp list</div><div class='del'>- * @dev: sc device struct</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_process_bh(struct i40iw_sc_dev *dev)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status = 0;</div><div class='del'>-	struct cqp_commands_info *pcmdinfo;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;dev-&gt;cqp_lock, flags);</div><div class='del'>-	while (!list_empty(&amp;dev-&gt;cqp_cmd_head) &amp;&amp; !i40iw_ring_full(dev-&gt;cqp)) {</div><div class='del'>-		pcmdinfo = (struct cqp_commands_info *)i40iw_remove_head(&amp;dev-&gt;cqp_cmd_head);</div><div class='del'>-</div><div class='del'>-		status = i40iw_exec_cqp_cmd(dev, pcmdinfo);</div><div class='del'>-		if (status)</div><div class='del'>-			break;</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;dev-&gt;cqp_lock, flags);</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_iwarp_opcode - determine if incoming is rdma layer</div><div class='del'>- * @info: aeq info for the packet</div><div class='del'>- * @pkt: packet for error</div><div class='del'>- */</div><div class='del'>-static u32 i40iw_iwarp_opcode(struct i40iw_aeqe_info *info, u8 *pkt)</div><div class='del'>-{</div><div class='del'>-	__be16 *mpa;</div><div class='del'>-	u32 opcode = 0xffffffff;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;q2_data_written) {</div><div class='del'>-		mpa = (__be16 *)pkt;</div><div class='del'>-		opcode = ntohs(mpa[1]) &amp; 0xf;</div><div class='del'>-	}</div><div class='del'>-	return opcode;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_locate_mpa - return pointer to mpa in the pkt</div><div class='del'>- * @pkt: packet with data</div><div class='del'>- */</div><div class='del'>-static u8 *i40iw_locate_mpa(u8 *pkt)</div><div class='del'>-{</div><div class='del'>-	/* skip over ethernet header */</div><div class='del'>-	pkt += I40IW_MAC_HLEN;</div><div class='del'>-</div><div class='del'>-	/* Skip over IP and TCP headers */</div><div class='del'>-	pkt += 4 * (pkt[0] &amp; 0x0f);</div><div class='del'>-	pkt += 4 * ((pkt[12] &gt;&gt; 4) &amp; 0x0f);</div><div class='del'>-	return pkt;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_setup_termhdr - termhdr for terminate pkt</div><div class='del'>- * @qp: sc qp ptr for pkt</div><div class='del'>- * @hdr: term hdr</div><div class='del'>- * @opcode: flush opcode for termhdr</div><div class='del'>- * @layer_etype: error layer + error type</div><div class='del'>- * @err: error cod ein the header</div><div class='del'>- */</div><div class='del'>-static void i40iw_setup_termhdr(struct i40iw_sc_qp *qp,</div><div class='del'>-				struct i40iw_terminate_hdr *hdr,</div><div class='del'>-				enum i40iw_flush_opcode opcode,</div><div class='del'>-				u8 layer_etype,</div><div class='del'>-				u8 err)</div><div class='del'>-{</div><div class='del'>-	qp-&gt;flush_code = opcode;</div><div class='del'>-	hdr-&gt;layer_etype = layer_etype;</div><div class='del'>-	hdr-&gt;error_code = err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_bld_terminate_hdr - build terminate message header</div><div class='del'>- * @qp: qp associated with received terminate AE</div><div class='del'>- * @info: the struct contiaing AE information</div><div class='del'>- */</div><div class='del'>-static int i40iw_bld_terminate_hdr(struct i40iw_sc_qp *qp,</div><div class='del'>-				   struct i40iw_aeqe_info *info)</div><div class='del'>-{</div><div class='del'>-	u8 *pkt = qp-&gt;q2_buf + Q2_BAD_FRAME_OFFSET;</div><div class='del'>-	u16 ddp_seg_len;</div><div class='del'>-	int copy_len = 0;</div><div class='del'>-	u8 is_tagged = 0;</div><div class='del'>-	u32 opcode;</div><div class='del'>-	struct i40iw_terminate_hdr *termhdr;</div><div class='del'>-</div><div class='del'>-	termhdr = (struct i40iw_terminate_hdr *)qp-&gt;q2_buf;</div><div class='del'>-	memset(termhdr, 0, Q2_BAD_FRAME_OFFSET);</div><div class='del'>-</div><div class='del'>-	if (info-&gt;q2_data_written) {</div><div class='del'>-		/* Use data from offending packet to fill in ddp &amp; rdma hdrs */</div><div class='del'>-		pkt = i40iw_locate_mpa(pkt);</div><div class='del'>-		ddp_seg_len = ntohs(*(__be16 *)pkt);</div><div class='del'>-		if (ddp_seg_len) {</div><div class='del'>-			copy_len = 2;</div><div class='del'>-			termhdr-&gt;hdrct = DDP_LEN_FLAG;</div><div class='del'>-			if (pkt[2] &amp; 0x80) {</div><div class='del'>-				is_tagged = 1;</div><div class='del'>-				if (ddp_seg_len &gt;= TERM_DDP_LEN_TAGGED) {</div><div class='del'>-					copy_len += TERM_DDP_LEN_TAGGED;</div><div class='del'>-					termhdr-&gt;hdrct |= DDP_HDR_FLAG;</div><div class='del'>-				}</div><div class='del'>-			} else {</div><div class='del'>-				if (ddp_seg_len &gt;= TERM_DDP_LEN_UNTAGGED) {</div><div class='del'>-					copy_len += TERM_DDP_LEN_UNTAGGED;</div><div class='del'>-					termhdr-&gt;hdrct |= DDP_HDR_FLAG;</div><div class='del'>-				}</div><div class='del'>-</div><div class='del'>-				if (ddp_seg_len &gt;= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {</div><div class='del'>-					if ((pkt[3] &amp; RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {</div><div class='del'>-						copy_len += TERM_RDMA_LEN;</div><div class='del'>-						termhdr-&gt;hdrct |= RDMA_HDR_FLAG;</div><div class='del'>-					}</div><div class='del'>-				}</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	opcode = i40iw_iwarp_opcode(info, pkt);</div><div class='del'>-</div><div class='del'>-	switch (info-&gt;ae_id) {</div><div class='del'>-	case I40IW_AE_AMP_UNALLOCATED_STAG:</div><div class='del'>-		qp-&gt;eventtype = TERM_EVENT_QP_ACCESS_ERR;</div><div class='del'>-		if (opcode == I40IW_OP_TYPE_RDMA_WRITE)</div><div class='del'>-			i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,</div><div class='del'>-					    (LAYER_DDP &lt;&lt; 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_INV_STAG);</div><div class='del'>-		else</div><div class='del'>-			i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,</div><div class='del'>-					    (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_PROT, RDMAP_INV_STAG);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_AMP_BOUNDS_VIOLATION:</div><div class='del'>-		qp-&gt;eventtype = TERM_EVENT_QP_ACCESS_ERR;</div><div class='del'>-		if (info-&gt;q2_data_written)</div><div class='del'>-			i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,</div><div class='del'>-					    (LAYER_DDP &lt;&lt; 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_BOUNDS);</div><div class='del'>-		else</div><div class='del'>-			i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,</div><div class='del'>-					    (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_PROT, RDMAP_INV_BOUNDS);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_AMP_BAD_PD:</div><div class='del'>-		switch (opcode) {</div><div class='del'>-		case I40IW_OP_TYPE_RDMA_WRITE:</div><div class='del'>-			i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,</div><div class='del'>-					    (LAYER_DDP &lt;&lt; 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_UNASSOC_STAG);</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_OP_TYPE_SEND_INV:</div><div class='del'>-		case I40IW_OP_TYPE_SEND_SOL_INV:</div><div class='del'>-			i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,</div><div class='del'>-					    (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_PROT, RDMAP_CANT_INV_STAG);</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,</div><div class='del'>-					    (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_PROT, RDMAP_UNASSOC_STAG);</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_AMP_INVALID_STAG:</div><div class='del'>-		qp-&gt;eventtype = TERM_EVENT_QP_ACCESS_ERR;</div><div class='del'>-		i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,</div><div class='del'>-				    (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_PROT, RDMAP_INV_STAG);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_AMP_BAD_QP:</div><div class='del'>-		i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR,</div><div class='del'>-				    (LAYER_DDP &lt;&lt; 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_QN);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_AMP_BAD_STAG_KEY:</div><div class='del'>-	case I40IW_AE_AMP_BAD_STAG_INDEX:</div><div class='del'>-		qp-&gt;eventtype = TERM_EVENT_QP_ACCESS_ERR;</div><div class='del'>-		switch (opcode) {</div><div class='del'>-		case I40IW_OP_TYPE_SEND_INV:</div><div class='del'>-		case I40IW_OP_TYPE_SEND_SOL_INV:</div><div class='del'>-			i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR,</div><div class='del'>-					    (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_OP, RDMAP_CANT_INV_STAG);</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,</div><div class='del'>-					    (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_OP, RDMAP_INV_STAG);</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_AMP_RIGHTS_VIOLATION:</div><div class='del'>-	case I40IW_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:</div><div class='del'>-	case I40IW_AE_PRIV_OPERATION_DENIED:</div><div class='del'>-		qp-&gt;eventtype = TERM_EVENT_QP_ACCESS_ERR;</div><div class='del'>-		i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,</div><div class='del'>-				    (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_PROT, RDMAP_ACCESS);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_AMP_TO_WRAP:</div><div class='del'>-		qp-&gt;eventtype = TERM_EVENT_QP_ACCESS_ERR;</div><div class='del'>-		i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,</div><div class='del'>-				    (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_PROT, RDMAP_TO_WRAP);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR:</div><div class='del'>-		i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,</div><div class='del'>-				    (LAYER_MPA &lt;&lt; 4) | DDP_LLP, MPA_CRC);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_LLP_SEGMENT_TOO_LARGE:</div><div class='del'>-	case I40IW_AE_LLP_SEGMENT_TOO_SMALL:</div><div class='del'>-		i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR,</div><div class='del'>-				    (LAYER_DDP &lt;&lt; 4) | DDP_CATASTROPHIC, DDP_CATASTROPHIC_LOCAL);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_LCE_QP_CATASTROPHIC:</div><div class='del'>-	case I40IW_AE_DDP_NO_L_BIT:</div><div class='del'>-		i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR,</div><div class='del'>-				    (LAYER_DDP &lt;&lt; 4) | DDP_CATASTROPHIC, DDP_CATASTROPHIC_LOCAL);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN:</div><div class='del'>-		i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,</div><div class='del'>-				    (LAYER_DDP &lt;&lt; 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MSN_RANGE);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:</div><div class='del'>-		qp-&gt;eventtype = TERM_EVENT_QP_ACCESS_ERR;</div><div class='del'>-		i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR,</div><div class='del'>-				    (LAYER_DDP &lt;&lt; 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_TOO_LONG);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION:</div><div class='del'>-		if (is_tagged)</div><div class='del'>-			i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,</div><div class='del'>-					    (LAYER_DDP &lt;&lt; 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_INV_DDP_VER);</div><div class='del'>-		else</div><div class='del'>-			i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,</div><div class='del'>-					    (LAYER_DDP &lt;&lt; 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_DDP_VER);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_DDP_UBE_INVALID_MO:</div><div class='del'>-		i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,</div><div class='del'>-				    (LAYER_DDP &lt;&lt; 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MO);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:</div><div class='del'>-		i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR,</div><div class='del'>-				    (LAYER_DDP &lt;&lt; 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MSN_NO_BUF);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_DDP_UBE_INVALID_QN:</div><div class='del'>-		i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,</div><div class='del'>-				    (LAYER_DDP &lt;&lt; 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_QN);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:</div><div class='del'>-		i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,</div><div class='del'>-				    (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_OP, RDMAP_INV_RDMAP_VER);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE:</div><div class='del'>-		i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR,</div><div class='del'>-				    (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_OP, RDMAP_UNEXPECTED_OP);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR,</div><div class='del'>-				    (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_OP, RDMAP_UNSPECIFIED);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (copy_len)</div><div class='del'>-		memcpy(termhdr + 1, pkt, copy_len);</div><div class='del'>-</div><div class='del'>-	return sizeof(struct i40iw_terminate_hdr) + copy_len;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_terminate_send_fin() - Send fin for terminate message</div><div class='del'>- * @qp: qp associated with received terminate AE</div><div class='del'>- */</div><div class='del'>-void i40iw_terminate_send_fin(struct i40iw_sc_qp *qp)</div><div class='del'>-{</div><div class='del'>-	/* Send the fin only */</div><div class='del'>-	i40iw_term_modify_qp(qp,</div><div class='del'>-			     I40IW_QP_STATE_TERMINATE,</div><div class='del'>-			     I40IWQP_TERM_SEND_FIN_ONLY,</div><div class='del'>-			     0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_terminate_connection() - Bad AE and send terminate to remote QP</div><div class='del'>- * @qp: qp associated with received terminate AE</div><div class='del'>- * @info: the struct contiaing AE information</div><div class='del'>- */</div><div class='del'>-void i40iw_terminate_connection(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info)</div><div class='del'>-{</div><div class='del'>-	u8 termlen = 0;</div><div class='del'>-</div><div class='del'>-	if (qp-&gt;term_flags &amp; I40IW_TERM_SENT)</div><div class='del'>-		return;         /* Sanity check */</div><div class='del'>-</div><div class='del'>-	/* Eventtype can change from bld_terminate_hdr */</div><div class='del'>-	qp-&gt;eventtype = TERM_EVENT_QP_FATAL;</div><div class='del'>-	termlen = i40iw_bld_terminate_hdr(qp, info);</div><div class='del'>-	i40iw_terminate_start_timer(qp);</div><div class='del'>-	qp-&gt;term_flags |= I40IW_TERM_SENT;</div><div class='del'>-	i40iw_term_modify_qp(qp, I40IW_QP_STATE_TERMINATE,</div><div class='del'>-			     I40IWQP_TERM_SEND_TERM_ONLY, termlen);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_terminate_received - handle terminate received AE</div><div class='del'>- * @qp: qp associated with received terminate AE</div><div class='del'>- * @info: the struct contiaing AE information</div><div class='del'>- */</div><div class='del'>-void i40iw_terminate_received(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info)</div><div class='del'>-{</div><div class='del'>-	u8 *pkt = qp-&gt;q2_buf + Q2_BAD_FRAME_OFFSET;</div><div class='del'>-	__be32 *mpa;</div><div class='del'>-	u8 ddp_ctl;</div><div class='del'>-	u8 rdma_ctl;</div><div class='del'>-	u16 aeq_id = 0;</div><div class='del'>-	struct i40iw_terminate_hdr *termhdr;</div><div class='del'>-</div><div class='del'>-	mpa = (__be32 *)i40iw_locate_mpa(pkt);</div><div class='del'>-	if (info-&gt;q2_data_written) {</div><div class='del'>-		/* did not validate the frame - do it now */</div><div class='del'>-		ddp_ctl = (ntohl(mpa[0]) &gt;&gt; 8) &amp; 0xff;</div><div class='del'>-		rdma_ctl = ntohl(mpa[0]) &amp; 0xff;</div><div class='del'>-		if ((ddp_ctl &amp; 0xc0) != 0x40)</div><div class='del'>-			aeq_id = I40IW_AE_LCE_QP_CATASTROPHIC;</div><div class='del'>-		else if ((ddp_ctl &amp; 0x03) != 1)</div><div class='del'>-			aeq_id = I40IW_AE_DDP_UBE_INVALID_DDP_VERSION;</div><div class='del'>-		else if (ntohl(mpa[2]) != 2)</div><div class='del'>-			aeq_id = I40IW_AE_DDP_UBE_INVALID_QN;</div><div class='del'>-		else if (ntohl(mpa[3]) != 1)</div><div class='del'>-			aeq_id = I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN;</div><div class='del'>-		else if (ntohl(mpa[4]) != 0)</div><div class='del'>-			aeq_id = I40IW_AE_DDP_UBE_INVALID_MO;</div><div class='del'>-		else if ((rdma_ctl &amp; 0xc0) != 0x40)</div><div class='del'>-			aeq_id = I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION;</div><div class='del'>-</div><div class='del'>-		info-&gt;ae_id = aeq_id;</div><div class='del'>-		if (info-&gt;ae_id) {</div><div class='del'>-			/* Bad terminate recvd - send back a terminate */</div><div class='del'>-			i40iw_terminate_connection(qp, info);</div><div class='del'>-			return;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	qp-&gt;term_flags |= I40IW_TERM_RCVD;</div><div class='del'>-	qp-&gt;eventtype = TERM_EVENT_QP_FATAL;</div><div class='del'>-	termhdr = (struct i40iw_terminate_hdr *)&amp;mpa[5];</div><div class='del'>-	if (termhdr-&gt;layer_etype == RDMAP_REMOTE_PROT ||</div><div class='del'>-	    termhdr-&gt;layer_etype == RDMAP_REMOTE_OP) {</div><div class='del'>-		i40iw_terminate_done(qp, 0);</div><div class='del'>-	} else {</div><div class='del'>-		i40iw_terminate_start_timer(qp);</div><div class='del'>-		i40iw_terminate_send_fin(qp);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_sc_vsi_init - Initialize virtual device</div><div class='del'>- * @vsi: pointer to the vsi structure</div><div class='del'>- * @info: parameters to initialize vsi</div><div class='del'>- **/</div><div class='del'>-void i40iw_sc_vsi_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_init_info *info)</div><div class='del'>-{</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	vsi-&gt;dev = info-&gt;dev;</div><div class='del'>-	vsi-&gt;back_vsi = info-&gt;back_vsi;</div><div class='del'>-	vsi-&gt;mtu = info-&gt;params-&gt;mtu;</div><div class='del'>-	vsi-&gt;exception_lan_queue = info-&gt;exception_lan_queue;</div><div class='del'>-	i40iw_fill_qos_list(info-&gt;params-&gt;qs_handle_list);</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; I40IW_MAX_USER_PRIORITY; i++) {</div><div class='del'>-		vsi-&gt;qos[i].qs_handle = info-&gt;params-&gt;qs_handle_list[i];</div><div class='del'>-		i40iw_debug(vsi-&gt;dev, I40IW_DEBUG_DCB, "qset[%d]: %d\n", i,</div><div class='del'>-			    vsi-&gt;qos[i].qs_handle);</div><div class='del'>-		spin_lock_init(&amp;vsi-&gt;qos[i].lock);</div><div class='del'>-		INIT_LIST_HEAD(&amp;vsi-&gt;qos[i].qplist);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hw_stats_init - Initiliaze HW stats table</div><div class='del'>- * @stats: pestat struct</div><div class='del'>- * @fcn_idx: PCI fn id</div><div class='del'>- * @is_pf: Is it a PF?</div><div class='del'>- *</div><div class='del'>- * Populate the HW stats table with register offset addr for each</div><div class='del'>- * stats. And start the perioidic stats timer.</div><div class='del'>- */</div><div class='del'>-void i40iw_hw_stats_init(struct i40iw_vsi_pestat *stats, u8 fcn_idx, bool is_pf)</div><div class='del'>-{</div><div class='del'>-	u32 stats_reg_offset;</div><div class='del'>-	u32 stats_index;</div><div class='del'>-	struct i40iw_dev_hw_stats_offsets *stats_table =</div><div class='del'>-		&amp;stats-&gt;hw_stats_offsets;</div><div class='del'>-	struct i40iw_dev_hw_stats *last_rd_stats = &amp;stats-&gt;last_read_hw_stats;</div><div class='del'>-</div><div class='del'>-	if (is_pf) {</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =</div><div class='del'>-				I40E_GLPES_PFIP4RXDISCARD(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =</div><div class='del'>-				I40E_GLPES_PFIP4RXTRUNC(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =</div><div class='del'>-				I40E_GLPES_PFIP4TXNOROUTE(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =</div><div class='del'>-				I40E_GLPES_PFIP6RXDISCARD(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =</div><div class='del'>-				I40E_GLPES_PFIP6RXTRUNC(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =</div><div class='del'>-				I40E_GLPES_PFIP6TXNOROUTE(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =</div><div class='del'>-				I40E_GLPES_PFTCPRTXSEG(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =</div><div class='del'>-				I40E_GLPES_PFTCPRXOPTERR(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =</div><div class='del'>-				I40E_GLPES_PFTCPRXPROTOERR(fcn_idx);</div><div class='del'>-</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =</div><div class='del'>-				I40E_GLPES_PFIP4RXOCTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =</div><div class='del'>-				I40E_GLPES_PFIP4RXPKTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =</div><div class='del'>-				I40E_GLPES_PFIP4RXFRAGSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =</div><div class='del'>-				I40E_GLPES_PFIP4RXMCPKTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =</div><div class='del'>-				I40E_GLPES_PFIP4TXOCTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =</div><div class='del'>-				I40E_GLPES_PFIP4TXPKTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =</div><div class='del'>-				I40E_GLPES_PFIP4TXFRAGSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =</div><div class='del'>-				I40E_GLPES_PFIP4TXMCPKTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =</div><div class='del'>-				I40E_GLPES_PFIP6RXOCTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =</div><div class='del'>-				I40E_GLPES_PFIP6RXPKTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =</div><div class='del'>-				I40E_GLPES_PFIP6RXFRAGSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =</div><div class='del'>-				I40E_GLPES_PFIP6RXMCPKTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =</div><div class='del'>-				I40E_GLPES_PFIP6TXOCTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =</div><div class='del'>-				I40E_GLPES_PFIP6TXPKTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =</div><div class='del'>-				I40E_GLPES_PFIP6TXPKTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =</div><div class='del'>-				I40E_GLPES_PFIP6TXFRAGSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =</div><div class='del'>-				I40E_GLPES_PFTCPRXSEGSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =</div><div class='del'>-				I40E_GLPES_PFTCPTXSEGLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =</div><div class='del'>-				I40E_GLPES_PFRDMARXRDSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =</div><div class='del'>-				I40E_GLPES_PFRDMARXSNDSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =</div><div class='del'>-				I40E_GLPES_PFRDMARXWRSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =</div><div class='del'>-				I40E_GLPES_PFRDMATXRDSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =</div><div class='del'>-				I40E_GLPES_PFRDMATXSNDSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =</div><div class='del'>-				I40E_GLPES_PFRDMATXWRSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =</div><div class='del'>-				I40E_GLPES_PFRDMAVBNDLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =</div><div class='del'>-				I40E_GLPES_PFRDMAVINVLO(fcn_idx);</div><div class='del'>-	} else {</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =</div><div class='del'>-				I40E_GLPES_VFIP4RXDISCARD(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =</div><div class='del'>-				I40E_GLPES_VFIP4RXTRUNC(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =</div><div class='del'>-				I40E_GLPES_VFIP4TXNOROUTE(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =</div><div class='del'>-				I40E_GLPES_VFIP6RXDISCARD(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =</div><div class='del'>-				I40E_GLPES_VFIP6RXTRUNC(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =</div><div class='del'>-				I40E_GLPES_VFIP6TXNOROUTE(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =</div><div class='del'>-				I40E_GLPES_VFTCPRTXSEG(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =</div><div class='del'>-				I40E_GLPES_VFTCPRXOPTERR(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =</div><div class='del'>-				I40E_GLPES_VFTCPRXPROTOERR(fcn_idx);</div><div class='del'>-</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =</div><div class='del'>-				I40E_GLPES_VFIP4RXOCTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =</div><div class='del'>-				I40E_GLPES_VFIP4RXPKTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =</div><div class='del'>-				I40E_GLPES_VFIP4RXFRAGSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =</div><div class='del'>-				I40E_GLPES_VFIP4RXMCPKTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =</div><div class='del'>-				I40E_GLPES_VFIP4TXOCTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =</div><div class='del'>-				I40E_GLPES_VFIP4TXPKTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =</div><div class='del'>-				I40E_GLPES_VFIP4TXFRAGSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =</div><div class='del'>-				I40E_GLPES_VFIP4TXMCPKTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =</div><div class='del'>-				I40E_GLPES_VFIP6RXOCTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =</div><div class='del'>-				I40E_GLPES_VFIP6RXPKTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =</div><div class='del'>-				I40E_GLPES_VFIP6RXFRAGSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =</div><div class='del'>-				I40E_GLPES_VFIP6RXMCPKTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =</div><div class='del'>-				I40E_GLPES_VFIP6TXOCTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =</div><div class='del'>-				I40E_GLPES_VFIP6TXPKTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =</div><div class='del'>-				I40E_GLPES_VFIP6TXPKTSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =</div><div class='del'>-				I40E_GLPES_VFIP6TXFRAGSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =</div><div class='del'>-				I40E_GLPES_VFTCPRXSEGSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =</div><div class='del'>-				I40E_GLPES_VFTCPTXSEGLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =</div><div class='del'>-				I40E_GLPES_VFRDMARXRDSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =</div><div class='del'>-				I40E_GLPES_VFRDMARXSNDSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =</div><div class='del'>-				I40E_GLPES_VFRDMARXWRSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =</div><div class='del'>-				I40E_GLPES_VFRDMATXRDSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =</div><div class='del'>-				I40E_GLPES_VFRDMATXSNDSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =</div><div class='del'>-				I40E_GLPES_VFRDMATXWRSLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =</div><div class='del'>-				I40E_GLPES_VFRDMAVBNDLO(fcn_idx);</div><div class='del'>-		stats_table-&gt;stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =</div><div class='del'>-				I40E_GLPES_VFRDMAVINVLO(fcn_idx);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	for (stats_index = 0; stats_index &lt; I40IW_HW_STAT_INDEX_MAX_64;</div><div class='del'>-	     stats_index++) {</div><div class='del'>-		stats_reg_offset = stats_table-&gt;stats_offset_64[stats_index];</div><div class='del'>-		last_rd_stats-&gt;stats_value_64[stats_index] =</div><div class='del'>-			readq(stats-&gt;hw-&gt;hw_addr + stats_reg_offset);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	for (stats_index = 0; stats_index &lt; I40IW_HW_STAT_INDEX_MAX_32;</div><div class='del'>-	     stats_index++) {</div><div class='del'>-		stats_reg_offset = stats_table-&gt;stats_offset_32[stats_index];</div><div class='del'>-		last_rd_stats-&gt;stats_value_32[stats_index] =</div><div class='del'>-			i40iw_rd32(stats-&gt;hw, stats_reg_offset);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hw_stats_read_32 - Read 32-bit HW stats counters and accommodates for roll-overs.</div><div class='del'>- * @stat: pestat struct</div><div class='del'>- * @index: index in HW stats table which contains offset reg-addr</div><div class='del'>- * @value: hw stats value</div><div class='del'>- */</div><div class='del'>-void i40iw_hw_stats_read_32(struct i40iw_vsi_pestat *stats,</div><div class='del'>-			    enum i40iw_hw_stats_index_32b index,</div><div class='del'>-			    u64 *value)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_dev_hw_stats_offsets *stats_table =</div><div class='del'>-		&amp;stats-&gt;hw_stats_offsets;</div><div class='del'>-	struct i40iw_dev_hw_stats *last_rd_stats = &amp;stats-&gt;last_read_hw_stats;</div><div class='del'>-	struct i40iw_dev_hw_stats *hw_stats = &amp;stats-&gt;hw_stats;</div><div class='del'>-	u64 new_stats_value = 0;</div><div class='del'>-	u32 stats_reg_offset = stats_table-&gt;stats_offset_32[index];</div><div class='del'>-</div><div class='del'>-	new_stats_value = i40iw_rd32(stats-&gt;hw, stats_reg_offset);</div><div class='del'>-	/*roll-over case */</div><div class='del'>-	if (new_stats_value &lt; last_rd_stats-&gt;stats_value_32[index])</div><div class='del'>-		hw_stats-&gt;stats_value_32[index] += new_stats_value;</div><div class='del'>-	else</div><div class='del'>-		hw_stats-&gt;stats_value_32[index] +=</div><div class='del'>-			new_stats_value - last_rd_stats-&gt;stats_value_32[index];</div><div class='del'>-	last_rd_stats-&gt;stats_value_32[index] = new_stats_value;</div><div class='del'>-	*value = hw_stats-&gt;stats_value_32[index];</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hw_stats_read_64 - Read HW stats counters (greater than 32-bit) and accommodates for roll-overs.</div><div class='del'>- * @stats: pestat struct</div><div class='del'>- * @index: index in HW stats table which contains offset reg-addr</div><div class='del'>- * @value: hw stats value</div><div class='del'>- */</div><div class='del'>-void i40iw_hw_stats_read_64(struct i40iw_vsi_pestat *stats,</div><div class='del'>-			    enum i40iw_hw_stats_index_64b index,</div><div class='del'>-			    u64 *value)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_dev_hw_stats_offsets *stats_table =</div><div class='del'>-		&amp;stats-&gt;hw_stats_offsets;</div><div class='del'>-	struct i40iw_dev_hw_stats *last_rd_stats = &amp;stats-&gt;last_read_hw_stats;</div><div class='del'>-	struct i40iw_dev_hw_stats *hw_stats = &amp;stats-&gt;hw_stats;</div><div class='del'>-	u64 new_stats_value = 0;</div><div class='del'>-	u32 stats_reg_offset = stats_table-&gt;stats_offset_64[index];</div><div class='del'>-</div><div class='del'>-	new_stats_value = readq(stats-&gt;hw-&gt;hw_addr + stats_reg_offset);</div><div class='del'>-	/*roll-over case */</div><div class='del'>-	if (new_stats_value &lt; last_rd_stats-&gt;stats_value_64[index])</div><div class='del'>-		hw_stats-&gt;stats_value_64[index] += new_stats_value;</div><div class='del'>-	else</div><div class='del'>-		hw_stats-&gt;stats_value_64[index] +=</div><div class='del'>-			new_stats_value - last_rd_stats-&gt;stats_value_64[index];</div><div class='del'>-	last_rd_stats-&gt;stats_value_64[index] = new_stats_value;</div><div class='del'>-	*value = hw_stats-&gt;stats_value_64[index];</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hw_stats_read_all - read all HW stat counters</div><div class='del'>- * @stats: pestat struct</div><div class='del'>- * @stats_values: hw stats structure</div><div class='del'>- *</div><div class='del'>- * Read all the HW stat counters and populates hw_stats structure</div><div class='del'>- * of passed-in vsi's pestat as well as copy created in stat_values.</div><div class='del'>- */</div><div class='del'>-void i40iw_hw_stats_read_all(struct i40iw_vsi_pestat *stats,</div><div class='del'>-			     struct i40iw_dev_hw_stats *stats_values)</div><div class='del'>-{</div><div class='del'>-	u32 stats_index;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;stats-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-	for (stats_index = 0; stats_index &lt; I40IW_HW_STAT_INDEX_MAX_32;</div><div class='del'>-	     stats_index++)</div><div class='del'>-		i40iw_hw_stats_read_32(stats, stats_index,</div><div class='del'>-				       &amp;stats_values-&gt;stats_value_32[stats_index]);</div><div class='del'>-	for (stats_index = 0; stats_index &lt; I40IW_HW_STAT_INDEX_MAX_64;</div><div class='del'>-	     stats_index++)</div><div class='del'>-		i40iw_hw_stats_read_64(stats, stats_index,</div><div class='del'>-				       &amp;stats_values-&gt;stats_value_64[stats_index]);</div><div class='del'>-	spin_unlock_irqrestore(&amp;stats-&gt;lock, flags);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hw_stats_refresh_all - Update all HW stats structs</div><div class='del'>- * @stats: pestat struct</div><div class='del'>- *</div><div class='del'>- * Read all the HW stats counters to refresh values in hw_stats structure</div><div class='del'>- * of passed-in dev's pestat</div><div class='del'>- */</div><div class='del'>-void i40iw_hw_stats_refresh_all(struct i40iw_vsi_pestat *stats)</div><div class='del'>-{</div><div class='del'>-	u64 stats_value;</div><div class='del'>-	u32 stats_index;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;stats-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-	for (stats_index = 0; stats_index &lt; I40IW_HW_STAT_INDEX_MAX_32;</div><div class='del'>-	     stats_index++)</div><div class='del'>-		i40iw_hw_stats_read_32(stats, stats_index, &amp;stats_value);</div><div class='del'>-	for (stats_index = 0; stats_index &lt; I40IW_HW_STAT_INDEX_MAX_64;</div><div class='del'>-	     stats_index++)</div><div class='del'>-		i40iw_hw_stats_read_64(stats, stats_index, &amp;stats_value);</div><div class='del'>-	spin_unlock_irqrestore(&amp;stats-&gt;lock, flags);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_fcn_id - Return the function id</div><div class='del'>- * @dev: pointer to the device</div><div class='del'>- */</div><div class='del'>-static u8 i40iw_get_fcn_id(struct i40iw_sc_dev *dev)</div><div class='del'>-{</div><div class='del'>-	u8 fcn_id = I40IW_INVALID_FCN_ID;</div><div class='del'>-	u8 i;</div><div class='del'>-</div><div class='del'>-	for (i = I40IW_FIRST_NON_PF_STAT; i &lt; I40IW_MAX_STATS_COUNT; i++)</div><div class='del'>-		if (!dev-&gt;fcn_id_array[i]) {</div><div class='del'>-			fcn_id = i;</div><div class='del'>-			dev-&gt;fcn_id_array[i] = true;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-	return fcn_id;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vsi_stats_init - Initialize the vsi statistics</div><div class='del'>- * @vsi: pointer to the vsi structure</div><div class='del'>- * @info: The info structure used for initialization</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_vsi_stats_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_stats_info *info)</div><div class='del'>-{</div><div class='del'>-	u8 fcn_id = info-&gt;fcn_id;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;alloc_fcn_id)</div><div class='del'>-		fcn_id = i40iw_get_fcn_id(vsi-&gt;dev);</div><div class='del'>-</div><div class='del'>-	if (fcn_id == I40IW_INVALID_FCN_ID)</div><div class='del'>-		return I40IW_ERR_NOT_READY;</div><div class='del'>-</div><div class='del'>-	vsi-&gt;pestat = info-&gt;pestat;</div><div class='del'>-	vsi-&gt;pestat-&gt;hw = vsi-&gt;dev-&gt;hw;</div><div class='del'>-	vsi-&gt;pestat-&gt;vsi = vsi;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;stats_initialize) {</div><div class='del'>-		i40iw_hw_stats_init(vsi-&gt;pestat, fcn_id, true);</div><div class='del'>-		spin_lock_init(&amp;vsi-&gt;pestat-&gt;lock);</div><div class='del'>-		i40iw_hw_stats_start_timer(vsi);</div><div class='del'>-	}</div><div class='del'>-	vsi-&gt;stats_fcn_id_alloc = info-&gt;alloc_fcn_id;</div><div class='del'>-	vsi-&gt;fcn_id = fcn_id;</div><div class='del'>-	return I40IW_SUCCESS;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vsi_stats_free - Free the vsi stats</div><div class='del'>- * @vsi: pointer to the vsi structure</div><div class='del'>- */</div><div class='del'>-void i40iw_vsi_stats_free(struct i40iw_sc_vsi *vsi)</div><div class='del'>-{</div><div class='del'>-	u8 fcn_id = vsi-&gt;fcn_id;</div><div class='del'>-</div><div class='del'>-	if (vsi-&gt;stats_fcn_id_alloc &amp;&amp; fcn_id &lt; I40IW_MAX_STATS_COUNT)</div><div class='del'>-		vsi-&gt;dev-&gt;fcn_id_array[fcn_id] = false;</div><div class='del'>-	i40iw_hw_stats_stop_timer(vsi);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct i40iw_cqp_ops iw_cqp_ops = {</div><div class='del'>-	.cqp_init = i40iw_sc_cqp_init,</div><div class='del'>-	.cqp_create = i40iw_sc_cqp_create,</div><div class='del'>-	.cqp_post_sq = i40iw_sc_cqp_post_sq,</div><div class='del'>-	.cqp_get_next_send_wqe = i40iw_sc_cqp_get_next_send_wqe,</div><div class='del'>-	.cqp_destroy = i40iw_sc_cqp_destroy,</div><div class='del'>-	.poll_for_cqp_op_done = i40iw_sc_poll_for_cqp_op_done</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct i40iw_ccq_ops iw_ccq_ops = {</div><div class='del'>-	.ccq_init = i40iw_sc_ccq_init,</div><div class='del'>-	.ccq_create = i40iw_sc_ccq_create,</div><div class='del'>-	.ccq_destroy = i40iw_sc_ccq_destroy,</div><div class='del'>-	.ccq_create_done = i40iw_sc_ccq_create_done,</div><div class='del'>-	.ccq_get_cqe_info = i40iw_sc_ccq_get_cqe_info,</div><div class='del'>-	.ccq_arm = i40iw_sc_ccq_arm</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct i40iw_ceq_ops iw_ceq_ops = {</div><div class='del'>-	.ceq_init = i40iw_sc_ceq_init,</div><div class='del'>-	.ceq_create = i40iw_sc_ceq_create,</div><div class='del'>-	.cceq_create_done = i40iw_sc_cceq_create_done,</div><div class='del'>-	.cceq_destroy_done = i40iw_sc_cceq_destroy_done,</div><div class='del'>-	.cceq_create = i40iw_sc_cceq_create,</div><div class='del'>-	.ceq_destroy = i40iw_sc_ceq_destroy,</div><div class='del'>-	.process_ceq = i40iw_sc_process_ceq</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct i40iw_aeq_ops iw_aeq_ops = {</div><div class='del'>-	.aeq_init = i40iw_sc_aeq_init,</div><div class='del'>-	.aeq_create = i40iw_sc_aeq_create,</div><div class='del'>-	.aeq_destroy = i40iw_sc_aeq_destroy,</div><div class='del'>-	.get_next_aeqe = i40iw_sc_get_next_aeqe,</div><div class='del'>-	.repost_aeq_entries = i40iw_sc_repost_aeq_entries,</div><div class='del'>-	.aeq_create_done = i40iw_sc_aeq_create_done,</div><div class='del'>-	.aeq_destroy_done = i40iw_sc_aeq_destroy_done</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/* iwarp pd ops */</div><div class='del'>-static struct i40iw_pd_ops iw_pd_ops = {</div><div class='del'>-	.pd_init = i40iw_sc_pd_init,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct i40iw_priv_qp_ops iw_priv_qp_ops = {</div><div class='del'>-	.qp_init = i40iw_sc_qp_init,</div><div class='del'>-	.qp_create = i40iw_sc_qp_create,</div><div class='del'>-	.qp_modify = i40iw_sc_qp_modify,</div><div class='del'>-	.qp_destroy = i40iw_sc_qp_destroy,</div><div class='del'>-	.qp_flush_wqes = i40iw_sc_qp_flush_wqes,</div><div class='del'>-	.qp_upload_context = i40iw_sc_qp_upload_context,</div><div class='del'>-	.qp_setctx = i40iw_sc_qp_setctx,</div><div class='del'>-	.qp_send_lsmm = i40iw_sc_send_lsmm,</div><div class='del'>-	.qp_send_lsmm_nostag = i40iw_sc_send_lsmm_nostag,</div><div class='del'>-	.qp_send_rtt = i40iw_sc_send_rtt,</div><div class='del'>-	.qp_post_wqe0 = i40iw_sc_post_wqe0,</div><div class='del'>-	.iw_mr_fast_register = i40iw_sc_mr_fast_register</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct i40iw_priv_cq_ops iw_priv_cq_ops = {</div><div class='del'>-	.cq_init = i40iw_sc_cq_init,</div><div class='del'>-	.cq_create = i40iw_sc_cq_create,</div><div class='del'>-	.cq_destroy = i40iw_sc_cq_destroy,</div><div class='del'>-	.cq_modify = i40iw_sc_cq_modify,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct i40iw_mr_ops iw_mr_ops = {</div><div class='del'>-	.alloc_stag = i40iw_sc_alloc_stag,</div><div class='del'>-	.mr_reg_non_shared = i40iw_sc_mr_reg_non_shared,</div><div class='del'>-	.mr_reg_shared = i40iw_sc_mr_reg_shared,</div><div class='del'>-	.dealloc_stag = i40iw_sc_dealloc_stag,</div><div class='del'>-	.query_stag = i40iw_sc_query_stag,</div><div class='del'>-	.mw_alloc = i40iw_sc_mw_alloc</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct i40iw_cqp_misc_ops iw_cqp_misc_ops = {</div><div class='del'>-	.manage_push_page = i40iw_sc_manage_push_page,</div><div class='del'>-	.manage_hmc_pm_func_table = i40iw_sc_manage_hmc_pm_func_table,</div><div class='del'>-	.set_hmc_resource_profile = i40iw_sc_set_hmc_resource_profile,</div><div class='del'>-	.commit_fpm_values = i40iw_sc_commit_fpm_values,</div><div class='del'>-	.query_fpm_values = i40iw_sc_query_fpm_values,</div><div class='del'>-	.static_hmc_pages_allocated = i40iw_sc_static_hmc_pages_allocated,</div><div class='del'>-	.add_arp_cache_entry = i40iw_sc_add_arp_cache_entry,</div><div class='del'>-	.del_arp_cache_entry = i40iw_sc_del_arp_cache_entry,</div><div class='del'>-	.query_arp_cache_entry = i40iw_sc_query_arp_cache_entry,</div><div class='del'>-	.manage_apbvt_entry = i40iw_sc_manage_apbvt_entry,</div><div class='del'>-	.manage_qhash_table_entry = i40iw_sc_manage_qhash_table_entry,</div><div class='del'>-	.alloc_local_mac_ipaddr_table_entry = i40iw_sc_alloc_local_mac_ipaddr_entry,</div><div class='del'>-	.add_local_mac_ipaddr_entry = i40iw_sc_add_local_mac_ipaddr_entry,</div><div class='del'>-	.del_local_mac_ipaddr_entry = i40iw_sc_del_local_mac_ipaddr_entry,</div><div class='del'>-	.cqp_nop = i40iw_sc_cqp_nop,</div><div class='del'>-	.commit_fpm_values_done = i40iw_sc_commit_fpm_values_done,</div><div class='del'>-	.query_fpm_values_done = i40iw_sc_query_fpm_values_done,</div><div class='del'>-	.manage_hmc_pm_func_table_done = i40iw_sc_manage_hmc_pm_func_table_done,</div><div class='del'>-	.update_suspend_qp = i40iw_sc_suspend_qp,</div><div class='del'>-	.update_resume_qp = i40iw_sc_resume_qp</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct i40iw_hmc_ops iw_hmc_ops = {</div><div class='del'>-	.init_iw_hmc = i40iw_sc_init_iw_hmc,</div><div class='del'>-	.parse_fpm_query_buf = i40iw_sc_parse_fpm_query_buf,</div><div class='del'>-	.configure_iw_fpm = i40iw_sc_configure_iw_fpm,</div><div class='del'>-	.parse_fpm_commit_buf = i40iw_sc_parse_fpm_commit_buf,</div><div class='del'>-	.create_hmc_object = i40iw_sc_create_hmc_obj,</div><div class='del'>-	.del_hmc_object = i40iw_sc_del_hmc_obj</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_device_init - Initialize IWARP device</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @info: IWARP init info</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_device_init(struct i40iw_sc_dev *dev,</div><div class='del'>-					 struct i40iw_device_init_info *info)</div><div class='del'>-{</div><div class='del'>-	u32 val;</div><div class='del'>-	u32 vchnl_ver = 0;</div><div class='del'>-	u16 hmc_fcn = 0;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-	u8 db_size;</div><div class='del'>-</div><div class='del'>-	spin_lock_init(&amp;dev-&gt;cqp_lock);</div><div class='del'>-</div><div class='del'>-	i40iw_device_init_uk(&amp;dev-&gt;dev_uk);</div><div class='del'>-</div><div class='del'>-	dev-&gt;debug_mask = info-&gt;debug_mask;</div><div class='del'>-</div><div class='del'>-	dev-&gt;hmc_fn_id = info-&gt;hmc_fn_id;</div><div class='del'>-	dev-&gt;is_pf = info-&gt;is_pf;</div><div class='del'>-</div><div class='del'>-	dev-&gt;fpm_query_buf_pa = info-&gt;fpm_query_buf_pa;</div><div class='del'>-	dev-&gt;fpm_query_buf = info-&gt;fpm_query_buf;</div><div class='del'>-</div><div class='del'>-	dev-&gt;fpm_commit_buf_pa = info-&gt;fpm_commit_buf_pa;</div><div class='del'>-	dev-&gt;fpm_commit_buf = info-&gt;fpm_commit_buf;</div><div class='del'>-</div><div class='del'>-	dev-&gt;hw = info-&gt;hw;</div><div class='del'>-	dev-&gt;hw-&gt;hw_addr = info-&gt;bar0;</div><div class='del'>-</div><div class='del'>-	if (dev-&gt;is_pf) {</div><div class='del'>-		val = i40iw_rd32(dev-&gt;hw, I40E_GLPCI_DREVID);</div><div class='del'>-		dev-&gt;hw_rev = (u8)RS_32(val, I40E_GLPCI_DREVID_DEFAULT_REVID);</div><div class='del'>-</div><div class='del'>-		val = i40iw_rd32(dev-&gt;hw, I40E_GLPCI_LBARCTRL);</div><div class='del'>-		db_size = (u8)RS_32(val, I40E_GLPCI_LBARCTRL_PE_DB_SIZE);</div><div class='del'>-		if ((db_size != I40IW_PE_DB_SIZE_4M) &amp;&amp;</div><div class='del'>-		    (db_size != I40IW_PE_DB_SIZE_8M)) {</div><div class='del'>-			i40iw_debug(dev, I40IW_DEBUG_DEV,</div><div class='del'>-				    "%s: PE doorbell is not enabled in CSR val 0x%x\n",</div><div class='del'>-				    __func__, val);</div><div class='del'>-			ret_code = I40IW_ERR_PE_DOORBELL_NOT_ENABLED;</div><div class='del'>-			return ret_code;</div><div class='del'>-		}</div><div class='del'>-		dev-&gt;db_addr = dev-&gt;hw-&gt;hw_addr + I40IW_DB_ADDR_OFFSET;</div><div class='del'>-		dev-&gt;vchnl_if.vchnl_recv = i40iw_vchnl_recv_pf;</div><div class='del'>-	} else {</div><div class='del'>-		dev-&gt;db_addr = dev-&gt;hw-&gt;hw_addr + I40IW_VF_DB_ADDR_OFFSET;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	dev-&gt;cqp_ops = &amp;iw_cqp_ops;</div><div class='del'>-	dev-&gt;ccq_ops = &amp;iw_ccq_ops;</div><div class='del'>-	dev-&gt;ceq_ops = &amp;iw_ceq_ops;</div><div class='del'>-	dev-&gt;aeq_ops = &amp;iw_aeq_ops;</div><div class='del'>-	dev-&gt;cqp_misc_ops = &amp;iw_cqp_misc_ops;</div><div class='del'>-	dev-&gt;iw_pd_ops = &amp;iw_pd_ops;</div><div class='del'>-	dev-&gt;iw_priv_qp_ops = &amp;iw_priv_qp_ops;</div><div class='del'>-	dev-&gt;iw_priv_cq_ops = &amp;iw_priv_cq_ops;</div><div class='del'>-	dev-&gt;mr_ops = &amp;iw_mr_ops;</div><div class='del'>-	dev-&gt;hmc_ops = &amp;iw_hmc_ops;</div><div class='del'>-	dev-&gt;vchnl_if.vchnl_send = info-&gt;vchnl_send;</div><div class='del'>-	if (dev-&gt;vchnl_if.vchnl_send)</div><div class='del'>-		dev-&gt;vchnl_up = true;</div><div class='del'>-	else</div><div class='del'>-		dev-&gt;vchnl_up = false;</div><div class='del'>-	if (!dev-&gt;is_pf) {</div><div class='del'>-		dev-&gt;vchnl_if.vchnl_recv = i40iw_vchnl_recv_vf;</div><div class='del'>-		ret_code = i40iw_vchnl_vf_get_ver(dev, &amp;vchnl_ver);</div><div class='del'>-		if (!ret_code) {</div><div class='del'>-			i40iw_debug(dev, I40IW_DEBUG_DEV,</div><div class='del'>-				    "%s: Get Channel version rc = 0x%0x, version is %u\n",</div><div class='del'>-				__func__, ret_code, vchnl_ver);</div><div class='del'>-			ret_code = i40iw_vchnl_vf_get_hmc_fcn(dev, &amp;hmc_fcn);</div><div class='del'>-			if (!ret_code) {</div><div class='del'>-				i40iw_debug(dev, I40IW_DEBUG_DEV,</div><div class='del'>-					    "%s Get HMC function rc = 0x%0x, hmc fcn is %u\n",</div><div class='del'>-					    __func__, ret_code, hmc_fcn);</div><div class='del'>-				dev-&gt;hmc_fn_id = (u8)hmc_fcn;</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	dev-&gt;iw_vf_cqp_ops = &amp;iw_vf_cqp_ops;</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_d.h b/drivers/infiniband/hw/i40iw/i40iw_d.h<br/>deleted file mode 100644<br/>index 6ddaeec87d2f..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_d.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_d.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,1737 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#ifndef I40IW_D_H</div><div class='del'>-#define I40IW_D_H</div><div class='del'>-</div><div class='del'>-#define I40IW_FIRST_USER_QP_ID  2</div><div class='del'>-</div><div class='del'>-#define I40IW_DB_ADDR_OFFSET    (4 * 1024 * 1024 - 64 * 1024)</div><div class='del'>-#define I40IW_VF_DB_ADDR_OFFSET (64 * 1024)</div><div class='del'>-</div><div class='del'>-#define I40IW_PUSH_OFFSET       (4 * 1024 * 1024)</div><div class='del'>-#define I40IW_PF_FIRST_PUSH_PAGE_INDEX 16</div><div class='del'>-#define I40IW_VF_PUSH_OFFSET    ((8 + 64) * 1024)</div><div class='del'>-#define I40IW_VF_FIRST_PUSH_PAGE_INDEX 2</div><div class='del'>-</div><div class='del'>-#define I40IW_PE_DB_SIZE_4M     1</div><div class='del'>-#define I40IW_PE_DB_SIZE_8M     2</div><div class='del'>-</div><div class='del'>-#define I40IW_DDP_VER 1</div><div class='del'>-#define I40IW_RDMAP_VER 1</div><div class='del'>-</div><div class='del'>-#define I40IW_RDMA_MODE_RDMAC 0</div><div class='del'>-#define I40IW_RDMA_MODE_IETF  1</div><div class='del'>-</div><div class='del'>-#define I40IW_QP_STATE_INVALID 0</div><div class='del'>-#define I40IW_QP_STATE_IDLE 1</div><div class='del'>-#define I40IW_QP_STATE_RTS 2</div><div class='del'>-#define I40IW_QP_STATE_CLOSING 3</div><div class='del'>-#define I40IW_QP_STATE_RESERVED 4</div><div class='del'>-#define I40IW_QP_STATE_TERMINATE 5</div><div class='del'>-#define I40IW_QP_STATE_ERROR 6</div><div class='del'>-</div><div class='del'>-#define I40IW_STAG_STATE_INVALID 0</div><div class='del'>-#define I40IW_STAG_STATE_VALID 1</div><div class='del'>-</div><div class='del'>-#define I40IW_STAG_TYPE_SHARED 0</div><div class='del'>-#define I40IW_STAG_TYPE_NONSHARED 1</div><div class='del'>-</div><div class='del'>-#define I40IW_MAX_USER_PRIORITY 8</div><div class='del'>-#define I40IW_MAX_STATS_COUNT 16</div><div class='del'>-#define I40IW_FIRST_NON_PF_STAT	4</div><div class='del'>-</div><div class='del'>-</div><div class='del'>-#define I40IW_MTU_TO_MSS_IPV4		40</div><div class='del'>-#define I40IW_MTU_TO_MSS_IPV6		60</div><div class='del'>-#define I40IW_DEFAULT_MTU		1500</div><div class='del'>-</div><div class='del'>-#define LS_64_1(val, bits)      ((u64)(uintptr_t)val &lt;&lt; bits)</div><div class='del'>-#define RS_64_1(val, bits)      ((u64)(uintptr_t)val &gt;&gt; bits)</div><div class='del'>-#define LS_32_1(val, bits)      (u32)(val &lt;&lt; bits)</div><div class='del'>-#define RS_32_1(val, bits)      (u32)(val &gt;&gt; bits)</div><div class='del'>-#define I40E_HI_DWORD(x)        ((u32)((((x) &gt;&gt; 16) &gt;&gt; 16) &amp; 0xFFFFFFFF))</div><div class='del'>-</div><div class='del'>-#define QS_HANDLE_UNKNOWN       0xffff</div><div class='del'>-</div><div class='del'>-#define LS_64(val, field) (((u64)val &lt;&lt; field ## _SHIFT) &amp; (field ## _MASK))</div><div class='del'>-</div><div class='del'>-#define RS_64(val, field) ((u64)(val &amp; field ## _MASK) &gt;&gt; field ## _SHIFT)</div><div class='del'>-#define LS_32(val, field) ((val &lt;&lt; field ## _SHIFT) &amp; (field ## _MASK))</div><div class='del'>-#define RS_32(val, field) ((val &amp; field ## _MASK) &gt;&gt; field ## _SHIFT)</div><div class='del'>-</div><div class='del'>-#define TERM_DDP_LEN_TAGGED     14</div><div class='del'>-#define TERM_DDP_LEN_UNTAGGED   18</div><div class='del'>-#define TERM_RDMA_LEN           28</div><div class='del'>-#define RDMA_OPCODE_MASK        0x0f</div><div class='del'>-#define RDMA_READ_REQ_OPCODE    1</div><div class='del'>-#define Q2_BAD_FRAME_OFFSET     72</div><div class='del'>-#define Q2_FPSN_OFFSET          64</div><div class='del'>-#define CQE_MAJOR_DRV           0x8000</div><div class='del'>-</div><div class='del'>-#define I40IW_TERM_SENT 0x01</div><div class='del'>-#define I40IW_TERM_RCVD 0x02</div><div class='del'>-#define I40IW_TERM_DONE 0x04</div><div class='del'>-#define I40IW_MAC_HLEN  14</div><div class='del'>-</div><div class='del'>-#define I40IW_INVALID_WQE_INDEX 0xffffffff</div><div class='del'>-</div><div class='del'>-#define I40IW_CQP_WAIT_POLL_REGS 1</div><div class='del'>-#define I40IW_CQP_WAIT_POLL_CQ 2</div><div class='del'>-#define I40IW_CQP_WAIT_EVENT 3</div><div class='del'>-</div><div class='del'>-#define I40IW_CQP_INIT_WQE(wqe) memset(wqe, 0, 64)</div><div class='del'>-</div><div class='del'>-#define I40IW_GET_CURRENT_CQ_ELEMENT(_cq) \</div><div class='del'>-	( \</div><div class='del'>-		&amp;((_cq)-&gt;cq_base[I40IW_RING_GETCURRENT_HEAD((_cq)-&gt;cq_ring)])  \</div><div class='del'>-	)</div><div class='del'>-#define I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(_cq) \</div><div class='del'>-	( \</div><div class='del'>-		&amp;(((struct i40iw_extended_cqe *)        \</div><div class='del'>-		   ((_cq)-&gt;cq_base))[I40IW_RING_GETCURRENT_HEAD((_cq)-&gt;cq_ring)]) \</div><div class='del'>-	)</div><div class='del'>-</div><div class='del'>-#define I40IW_GET_CURRENT_AEQ_ELEMENT(_aeq) \</div><div class='del'>-	( \</div><div class='del'>-		&amp;_aeq-&gt;aeqe_base[I40IW_RING_GETCURRENT_TAIL(_aeq-&gt;aeq_ring)]   \</div><div class='del'>-	)</div><div class='del'>-</div><div class='del'>-#define I40IW_GET_CURRENT_CEQ_ELEMENT(_ceq) \</div><div class='del'>-	( \</div><div class='del'>-		&amp;_ceq-&gt;ceqe_base[I40IW_RING_GETCURRENT_TAIL(_ceq-&gt;ceq_ring)]   \</div><div class='del'>-	)</div><div class='del'>-</div><div class='del'>-#define I40IW_AE_SOURCE_RSVD            0x0</div><div class='del'>-#define I40IW_AE_SOURCE_RQ              0x1</div><div class='del'>-#define I40IW_AE_SOURCE_RQ_0011         0x3</div><div class='del'>-</div><div class='del'>-#define I40IW_AE_SOURCE_CQ              0x2</div><div class='del'>-#define I40IW_AE_SOURCE_CQ_0110         0x6</div><div class='del'>-#define I40IW_AE_SOURCE_CQ_1010         0xA</div><div class='del'>-#define I40IW_AE_SOURCE_CQ_1110         0xE</div><div class='del'>-</div><div class='del'>-#define I40IW_AE_SOURCE_SQ              0x5</div><div class='del'>-#define I40IW_AE_SOURCE_SQ_0111         0x7</div><div class='del'>-</div><div class='del'>-#define I40IW_AE_SOURCE_IN_RR_WR        0x9</div><div class='del'>-#define I40IW_AE_SOURCE_IN_RR_WR_1011   0xB</div><div class='del'>-#define I40IW_AE_SOURCE_OUT_RR          0xD</div><div class='del'>-#define I40IW_AE_SOURCE_OUT_RR_1111     0xF</div><div class='del'>-</div><div class='del'>-#define I40IW_TCP_STATE_NON_EXISTENT 0</div><div class='del'>-#define I40IW_TCP_STATE_CLOSED 1</div><div class='del'>-#define I40IW_TCP_STATE_LISTEN 2</div><div class='del'>-#define I40IW_STATE_SYN_SEND 3</div><div class='del'>-#define I40IW_TCP_STATE_SYN_RECEIVED 4</div><div class='del'>-#define I40IW_TCP_STATE_ESTABLISHED 5</div><div class='del'>-#define I40IW_TCP_STATE_CLOSE_WAIT 6</div><div class='del'>-#define I40IW_TCP_STATE_FIN_WAIT_1 7</div><div class='del'>-#define I40IW_TCP_STATE_CLOSING  8</div><div class='del'>-#define I40IW_TCP_STATE_LAST_ACK 9</div><div class='del'>-#define I40IW_TCP_STATE_FIN_WAIT_2 10</div><div class='del'>-#define I40IW_TCP_STATE_TIME_WAIT 11</div><div class='del'>-#define I40IW_TCP_STATE_RESERVED_1 12</div><div class='del'>-#define I40IW_TCP_STATE_RESERVED_2 13</div><div class='del'>-#define I40IW_TCP_STATE_RESERVED_3 14</div><div class='del'>-#define I40IW_TCP_STATE_RESERVED_4 15</div><div class='del'>-</div><div class='del'>-/* ILQ CQP hash table fields */</div><div class='del'>-#define I40IW_CQPSQ_QHASH_VLANID_SHIFT 32</div><div class='del'>-#define I40IW_CQPSQ_QHASH_VLANID_MASK \</div><div class='del'>-	((u64)0xfff &lt;&lt; I40IW_CQPSQ_QHASH_VLANID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QHASH_QPN_SHIFT 32</div><div class='del'>-#define I40IW_CQPSQ_QHASH_QPN_MASK \</div><div class='del'>-	((u64)0x3ffff &lt;&lt; I40IW_CQPSQ_QHASH_QPN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QHASH_QS_HANDLE_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_QHASH_QS_HANDLE_MASK ((u64)0x3ff &lt;&lt; I40IW_CQPSQ_QHASH_QS_HANDLE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QHASH_SRC_PORT_SHIFT 16</div><div class='del'>-#define I40IW_CQPSQ_QHASH_SRC_PORT_MASK \</div><div class='del'>-	((u64)0xffff &lt;&lt; I40IW_CQPSQ_QHASH_SRC_PORT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QHASH_DEST_PORT_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_QHASH_DEST_PORT_MASK \</div><div class='del'>-	((u64)0xffff &lt;&lt; I40IW_CQPSQ_QHASH_DEST_PORT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QHASH_ADDR0_SHIFT 32</div><div class='del'>-#define I40IW_CQPSQ_QHASH_ADDR0_MASK \</div><div class='del'>-	((u64)0xffffffff &lt;&lt; I40IW_CQPSQ_QHASH_ADDR0_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QHASH_ADDR1_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_QHASH_ADDR1_MASK \</div><div class='del'>-	((u64)0xffffffff &lt;&lt; I40IW_CQPSQ_QHASH_ADDR1_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QHASH_ADDR2_SHIFT 32</div><div class='del'>-#define I40IW_CQPSQ_QHASH_ADDR2_MASK \</div><div class='del'>-	((u64)0xffffffff &lt;&lt; I40IW_CQPSQ_QHASH_ADDR2_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QHASH_ADDR3_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_QHASH_ADDR3_MASK \</div><div class='del'>-	((u64)0xffffffff &lt;&lt; I40IW_CQPSQ_QHASH_ADDR3_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QHASH_WQEVALID_SHIFT 63</div><div class='del'>-#define I40IW_CQPSQ_QHASH_WQEVALID_MASK \</div><div class='del'>-	((u64)0x1 &lt;&lt; I40IW_CQPSQ_QHASH_WQEVALID_SHIFT)</div><div class='del'>-#define I40IW_CQPSQ_QHASH_OPCODE_SHIFT 32</div><div class='del'>-#define I40IW_CQPSQ_QHASH_OPCODE_MASK \</div><div class='del'>-	((u64)0x3f &lt;&lt; I40IW_CQPSQ_QHASH_OPCODE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QHASH_MANAGE_SHIFT 61</div><div class='del'>-#define I40IW_CQPSQ_QHASH_MANAGE_MASK \</div><div class='del'>-	((u64)0x3 &lt;&lt; I40IW_CQPSQ_QHASH_MANAGE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QHASH_IPV4VALID_SHIFT 60</div><div class='del'>-#define I40IW_CQPSQ_QHASH_IPV4VALID_MASK \</div><div class='del'>-	((u64)0x1 &lt;&lt; I40IW_CQPSQ_QHASH_IPV4VALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QHASH_VLANVALID_SHIFT 59</div><div class='del'>-#define I40IW_CQPSQ_QHASH_VLANVALID_MASK \</div><div class='del'>-	((u64)0x1 &lt;&lt; I40IW_CQPSQ_QHASH_VLANVALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QHASH_ENTRYTYPE_SHIFT 42</div><div class='del'>-#define I40IW_CQPSQ_QHASH_ENTRYTYPE_MASK \</div><div class='del'>-	((u64)0x7 &lt;&lt; I40IW_CQPSQ_QHASH_ENTRYTYPE_SHIFT)</div><div class='del'>-/* CQP Host Context */</div><div class='del'>-#define I40IW_CQPHC_EN_DC_TCP_SHIFT 0</div><div class='del'>-#define I40IW_CQPHC_EN_DC_TCP_MASK (1UL &lt;&lt; I40IW_CQPHC_EN_DC_TCP_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPHC_SQSIZE_SHIFT 8</div><div class='del'>-#define I40IW_CQPHC_SQSIZE_MASK (0xfUL &lt;&lt; I40IW_CQPHC_SQSIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPHC_DISABLE_PFPDUS_SHIFT 1</div><div class='del'>-#define I40IW_CQPHC_DISABLE_PFPDUS_MASK (0x1UL &lt;&lt; I40IW_CQPHC_DISABLE_PFPDUS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPHC_ENABLED_VFS_SHIFT 32</div><div class='del'>-#define I40IW_CQPHC_ENABLED_VFS_MASK (0x3fULL &lt;&lt; I40IW_CQPHC_ENABLED_VFS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPHC_HMC_PROFILE_SHIFT 0</div><div class='del'>-#define I40IW_CQPHC_HMC_PROFILE_MASK (0x7ULL &lt;&lt; I40IW_CQPHC_HMC_PROFILE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPHC_SVER_SHIFT 24</div><div class='del'>-#define I40IW_CQPHC_SVER_MASK (0xffUL &lt;&lt; I40IW_CQPHC_SVER_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPHC_SQBASE_SHIFT 9</div><div class='del'>-#define I40IW_CQPHC_SQBASE_MASK \</div><div class='del'>-	(0xfffffffffffffeULL &lt;&lt; I40IW_CQPHC_SQBASE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPHC_QPCTX_SHIFT 0</div><div class='del'>-#define I40IW_CQPHC_QPCTX_MASK  \</div><div class='del'>-	(0xffffffffffffffffULL &lt;&lt; I40IW_CQPHC_QPCTX_SHIFT)</div><div class='del'>-#define I40IW_CQPHC_SVER        1</div><div class='del'>-</div><div class='del'>-#define I40IW_CQP_SW_SQSIZE_4 4</div><div class='del'>-#define I40IW_CQP_SW_SQSIZE_2048 2048</div><div class='del'>-</div><div class='del'>-/* iWARP QP Doorbell shadow area */</div><div class='del'>-#define I40IW_QP_DBSA_HW_SQ_TAIL_SHIFT 0</div><div class='del'>-#define I40IW_QP_DBSA_HW_SQ_TAIL_MASK \</div><div class='del'>-	(0x3fffUL &lt;&lt; I40IW_QP_DBSA_HW_SQ_TAIL_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Completion Queue Doorbell shadow area */</div><div class='del'>-#define I40IW_CQ_DBSA_CQEIDX_SHIFT 0</div><div class='del'>-#define I40IW_CQ_DBSA_CQEIDX_MASK (0xfffffUL &lt;&lt; I40IW_CQ_DBSA_CQEIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQ_DBSA_SW_CQ_SELECT_SHIFT 0</div><div class='del'>-#define I40IW_CQ_DBSA_SW_CQ_SELECT_MASK \</div><div class='del'>-	(0x3fffUL &lt;&lt; I40IW_CQ_DBSA_SW_CQ_SELECT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQ_DBSA_ARM_NEXT_SHIFT 14</div><div class='del'>-#define I40IW_CQ_DBSA_ARM_NEXT_MASK (1UL &lt;&lt; I40IW_CQ_DBSA_ARM_NEXT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQ_DBSA_ARM_NEXT_SE_SHIFT 15</div><div class='del'>-#define I40IW_CQ_DBSA_ARM_NEXT_SE_MASK (1UL &lt;&lt; I40IW_CQ_DBSA_ARM_NEXT_SE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQ_DBSA_ARM_SEQ_NUM_SHIFT 16</div><div class='del'>-#define I40IW_CQ_DBSA_ARM_SEQ_NUM_MASK \</div><div class='del'>-	(0x3UL &lt;&lt; I40IW_CQ_DBSA_ARM_SEQ_NUM_SHIFT)</div><div class='del'>-</div><div class='del'>-/* CQP and iWARP Completion Queue */</div><div class='del'>-#define I40IW_CQ_QPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IW_CQ_QPCTX_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-#define I40IW_CCQ_OPRETVAL_SHIFT 0</div><div class='del'>-#define I40IW_CCQ_OPRETVAL_MASK (0xffffffffUL &lt;&lt; I40IW_CCQ_OPRETVAL_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQ_MINERR_SHIFT 0</div><div class='del'>-#define I40IW_CQ_MINERR_MASK (0xffffUL &lt;&lt; I40IW_CQ_MINERR_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQ_MAJERR_SHIFT 16</div><div class='del'>-#define I40IW_CQ_MAJERR_MASK (0xffffUL &lt;&lt; I40IW_CQ_MAJERR_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQ_WQEIDX_SHIFT 32</div><div class='del'>-#define I40IW_CQ_WQEIDX_MASK (0x3fffULL &lt;&lt; I40IW_CQ_WQEIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQ_ERROR_SHIFT 55</div><div class='del'>-#define I40IW_CQ_ERROR_MASK (1ULL &lt;&lt; I40IW_CQ_ERROR_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQ_SQ_SHIFT 62</div><div class='del'>-#define I40IW_CQ_SQ_MASK (1ULL &lt;&lt; I40IW_CQ_SQ_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQ_VALID_SHIFT 63</div><div class='del'>-#define I40IW_CQ_VALID_MASK (1ULL &lt;&lt; I40IW_CQ_VALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWCQ_PAYLDLEN_SHIFT 0</div><div class='del'>-#define I40IWCQ_PAYLDLEN_MASK (0xffffffffUL &lt;&lt; I40IWCQ_PAYLDLEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWCQ_TCPSEQNUM_SHIFT 32</div><div class='del'>-#define I40IWCQ_TCPSEQNUM_MASK (0xffffffffULL &lt;&lt; I40IWCQ_TCPSEQNUM_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWCQ_INVSTAG_SHIFT 0</div><div class='del'>-#define I40IWCQ_INVSTAG_MASK (0xffffffffUL &lt;&lt; I40IWCQ_INVSTAG_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWCQ_QPID_SHIFT 32</div><div class='del'>-#define I40IWCQ_QPID_MASK (0x3ffffULL &lt;&lt; I40IWCQ_QPID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWCQ_PSHDROP_SHIFT 51</div><div class='del'>-#define I40IWCQ_PSHDROP_MASK (1ULL &lt;&lt; I40IWCQ_PSHDROP_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWCQ_SRQ_SHIFT 52</div><div class='del'>-#define I40IWCQ_SRQ_MASK (1ULL &lt;&lt; I40IWCQ_SRQ_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWCQ_STAG_SHIFT 53</div><div class='del'>-#define I40IWCQ_STAG_MASK (1ULL &lt;&lt; I40IWCQ_STAG_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWCQ_SOEVENT_SHIFT 54</div><div class='del'>-#define I40IWCQ_SOEVENT_MASK (1ULL &lt;&lt; I40IWCQ_SOEVENT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWCQ_OP_SHIFT 56</div><div class='del'>-#define I40IWCQ_OP_MASK (0x3fULL &lt;&lt; I40IWCQ_OP_SHIFT)</div><div class='del'>-</div><div class='del'>-/* CEQE format */</div><div class='del'>-#define I40IW_CEQE_CQCTX_SHIFT 0</div><div class='del'>-#define I40IW_CEQE_CQCTX_MASK   \</div><div class='del'>-	(0x7fffffffffffffffULL &lt;&lt; I40IW_CEQE_CQCTX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CEQE_VALID_SHIFT 63</div><div class='del'>-#define I40IW_CEQE_VALID_MASK (1ULL &lt;&lt; I40IW_CEQE_VALID_SHIFT)</div><div class='del'>-</div><div class='del'>-/* AEQE format */</div><div class='del'>-#define I40IW_AEQE_COMPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IW_AEQE_COMPCTX_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-#define I40IW_AEQE_QPCQID_SHIFT 0</div><div class='del'>-#define I40IW_AEQE_QPCQID_MASK (0x3ffffUL &lt;&lt; I40IW_AEQE_QPCQID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_AEQE_WQDESCIDX_SHIFT 18</div><div class='del'>-#define I40IW_AEQE_WQDESCIDX_MASK (0x3fffULL &lt;&lt; I40IW_AEQE_WQDESCIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_AEQE_OVERFLOW_SHIFT 33</div><div class='del'>-#define I40IW_AEQE_OVERFLOW_MASK (1ULL &lt;&lt; I40IW_AEQE_OVERFLOW_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_AEQE_AECODE_SHIFT 34</div><div class='del'>-#define I40IW_AEQE_AECODE_MASK (0xffffULL &lt;&lt; I40IW_AEQE_AECODE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_AEQE_AESRC_SHIFT 50</div><div class='del'>-#define I40IW_AEQE_AESRC_MASK (0xfULL &lt;&lt; I40IW_AEQE_AESRC_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_AEQE_IWSTATE_SHIFT 54</div><div class='del'>-#define I40IW_AEQE_IWSTATE_MASK (0x7ULL &lt;&lt; I40IW_AEQE_IWSTATE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_AEQE_TCPSTATE_SHIFT 57</div><div class='del'>-#define I40IW_AEQE_TCPSTATE_MASK (0xfULL &lt;&lt; I40IW_AEQE_TCPSTATE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_AEQE_Q2DATA_SHIFT 61</div><div class='del'>-#define I40IW_AEQE_Q2DATA_MASK (0x3ULL &lt;&lt; I40IW_AEQE_Q2DATA_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_AEQE_VALID_SHIFT 63</div><div class='del'>-#define I40IW_AEQE_VALID_MASK (1ULL &lt;&lt; I40IW_AEQE_VALID_SHIFT)</div><div class='del'>-</div><div class='del'>-/* CQP SQ WQES */</div><div class='del'>-#define I40IW_QP_TYPE_IWARP     1</div><div class='del'>-#define I40IW_QP_TYPE_UDA       2</div><div class='del'>-#define I40IW_QP_TYPE_CQP       4</div><div class='del'>-</div><div class='del'>-#define I40IW_CQ_TYPE_IWARP     1</div><div class='del'>-#define I40IW_CQ_TYPE_ILQ       2</div><div class='del'>-#define I40IW_CQ_TYPE_IEQ       3</div><div class='del'>-#define I40IW_CQ_TYPE_CQP       4</div><div class='del'>-</div><div class='del'>-#define I40IWQP_TERM_SEND_TERM_AND_FIN          0</div><div class='del'>-#define I40IWQP_TERM_SEND_TERM_ONLY             1</div><div class='del'>-#define I40IWQP_TERM_SEND_FIN_ONLY              2</div><div class='del'>-#define I40IWQP_TERM_DONOT_SEND_TERM_OR_FIN     3</div><div class='del'>-</div><div class='del'>-#define I40IW_CQP_OP_CREATE_QP                  0</div><div class='del'>-#define I40IW_CQP_OP_MODIFY_QP                  0x1</div><div class='del'>-#define I40IW_CQP_OP_DESTROY_QP                 0x02</div><div class='del'>-#define I40IW_CQP_OP_CREATE_CQ                  0x03</div><div class='del'>-#define I40IW_CQP_OP_MODIFY_CQ                  0x04</div><div class='del'>-#define I40IW_CQP_OP_DESTROY_CQ                 0x05</div><div class='del'>-#define I40IW_CQP_OP_CREATE_SRQ                 0x06</div><div class='del'>-#define I40IW_CQP_OP_MODIFY_SRQ                 0x07</div><div class='del'>-#define I40IW_CQP_OP_DESTROY_SRQ                0x08</div><div class='del'>-#define I40IW_CQP_OP_ALLOC_STAG                 0x09</div><div class='del'>-#define I40IW_CQP_OP_REG_MR                     0x0a</div><div class='del'>-#define I40IW_CQP_OP_QUERY_STAG                 0x0b</div><div class='del'>-#define I40IW_CQP_OP_REG_SMR                    0x0c</div><div class='del'>-#define I40IW_CQP_OP_DEALLOC_STAG               0x0d</div><div class='del'>-#define I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE    0x0e</div><div class='del'>-#define I40IW_CQP_OP_MANAGE_ARP                 0x0f</div><div class='del'>-#define I40IW_CQP_OP_MANAGE_VF_PBLE_BP          0x10</div><div class='del'>-#define I40IW_CQP_OP_MANAGE_PUSH_PAGES          0x11</div><div class='del'>-#define I40IW_CQP_OP_MANAGE_PE_TEAM             0x12</div><div class='del'>-#define I40IW_CQP_OP_UPLOAD_CONTEXT             0x13</div><div class='del'>-#define I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY 0x14</div><div class='del'>-#define I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE   0x15</div><div class='del'>-#define I40IW_CQP_OP_CREATE_CEQ                 0x16</div><div class='del'>-#define I40IW_CQP_OP_DESTROY_CEQ                0x18</div><div class='del'>-#define I40IW_CQP_OP_CREATE_AEQ                 0x19</div><div class='del'>-#define I40IW_CQP_OP_DESTROY_AEQ                0x1b</div><div class='del'>-#define I40IW_CQP_OP_CREATE_ADDR_VECT           0x1c</div><div class='del'>-#define I40IW_CQP_OP_MODIFY_ADDR_VECT           0x1d</div><div class='del'>-#define I40IW_CQP_OP_DESTROY_ADDR_VECT          0x1e</div><div class='del'>-#define I40IW_CQP_OP_UPDATE_PE_SDS              0x1f</div><div class='del'>-#define I40IW_CQP_OP_QUERY_FPM_VALUES           0x20</div><div class='del'>-#define I40IW_CQP_OP_COMMIT_FPM_VALUES          0x21</div><div class='del'>-#define I40IW_CQP_OP_FLUSH_WQES                 0x22</div><div class='del'>-/* I40IW_CQP_OP_GEN_AE is the same value as I40IW_CQP_OP_FLUSH_WQES */</div><div class='del'>-#define I40IW_CQP_OP_GEN_AE                     0x22</div><div class='del'>-#define I40IW_CQP_OP_MANAGE_APBVT               0x23</div><div class='del'>-#define I40IW_CQP_OP_NOP                        0x24</div><div class='del'>-#define I40IW_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY 0x25</div><div class='del'>-#define I40IW_CQP_OP_CREATE_UDA_MCAST_GROUP     0x26</div><div class='del'>-#define I40IW_CQP_OP_MODIFY_UDA_MCAST_GROUP     0x27</div><div class='del'>-#define I40IW_CQP_OP_DESTROY_UDA_MCAST_GROUP    0x28</div><div class='del'>-#define I40IW_CQP_OP_SUSPEND_QP                 0x29</div><div class='del'>-#define I40IW_CQP_OP_RESUME_QP                  0x2a</div><div class='del'>-#define I40IW_CQP_OP_SHMC_PAGES_ALLOCATED       0x2b</div><div class='del'>-#define I40IW_CQP_OP_SET_HMC_RESOURCE_PROFILE   0x2d</div><div class='del'>-</div><div class='del'>-#define I40IW_UDA_QPSQ_NEXT_HEADER_SHIFT 16</div><div class='del'>-#define I40IW_UDA_QPSQ_NEXT_HEADER_MASK ((u64)0xff &lt;&lt; I40IW_UDA_QPSQ_NEXT_HEADER_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_UDA_QPSQ_OPCODE_SHIFT 32</div><div class='del'>-#define I40IW_UDA_QPSQ_OPCODE_MASK ((u64)0x3f &lt;&lt; I40IW_UDA_QPSQ_OPCODE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_UDA_QPSQ_MACLEN_SHIFT 56</div><div class='del'>-#define I40IW_UDA_QPSQ_MACLEN_MASK \</div><div class='del'>-	((u64)0x7f &lt;&lt; I40IW_UDA_QPSQ_MACLEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_UDA_QPSQ_IPLEN_SHIFT 48</div><div class='del'>-#define I40IW_UDA_QPSQ_IPLEN_MASK \</div><div class='del'>-	((u64)0x7f &lt;&lt; I40IW_UDA_QPSQ_IPLEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_UDA_QPSQ_L4T_SHIFT 30</div><div class='del'>-#define I40IW_UDA_QPSQ_L4T_MASK \</div><div class='del'>-	((u64)0x3 &lt;&lt; I40IW_UDA_QPSQ_L4T_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_UDA_QPSQ_IIPT_SHIFT 28</div><div class='del'>-#define I40IW_UDA_QPSQ_IIPT_MASK \</div><div class='del'>-	((u64)0x3 &lt;&lt; I40IW_UDA_QPSQ_IIPT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_UDA_QPSQ_L4LEN_SHIFT 24</div><div class='del'>-#define I40IW_UDA_QPSQ_L4LEN_MASK ((u64)0xf &lt;&lt; I40IW_UDA_QPSQ_L4LEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_UDA_QPSQ_AVIDX_SHIFT 0</div><div class='del'>-#define I40IW_UDA_QPSQ_AVIDX_MASK ((u64)0xffff &lt;&lt; I40IW_UDA_QPSQ_AVIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_UDA_QPSQ_VALID_SHIFT 63</div><div class='del'>-#define I40IW_UDA_QPSQ_VALID_MASK \</div><div class='del'>-	((u64)0x1 &lt;&lt; I40IW_UDA_QPSQ_VALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_UDA_QPSQ_SIGCOMPL_SHIFT 62</div><div class='del'>-#define I40IW_UDA_QPSQ_SIGCOMPL_MASK ((u64)0x1 &lt;&lt; I40IW_UDA_QPSQ_SIGCOMPL_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_UDA_PAYLOADLEN_SHIFT 0</div><div class='del'>-#define I40IW_UDA_PAYLOADLEN_MASK ((u64)0x3fff &lt;&lt; I40IW_UDA_PAYLOADLEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_UDA_HDRLEN_SHIFT 16</div><div class='del'>-#define I40IW_UDA_HDRLEN_MASK ((u64)0x1ff &lt;&lt; I40IW_UDA_HDRLEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_VLAN_TAG_VALID_SHIFT 50</div><div class='del'>-#define I40IW_VLAN_TAG_VALID_MASK ((u64)0x1 &lt;&lt; I40IW_VLAN_TAG_VALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_UDA_L3PROTO_SHIFT 0</div><div class='del'>-#define I40IW_UDA_L3PROTO_MASK ((u64)0x3 &lt;&lt; I40IW_UDA_L3PROTO_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_UDA_L4PROTO_SHIFT 16</div><div class='del'>-#define I40IW_UDA_L4PROTO_MASK ((u64)0x3 &lt;&lt; I40IW_UDA_L4PROTO_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_UDA_QPSQ_DOLOOPBACK_SHIFT 44</div><div class='del'>-#define I40IW_UDA_QPSQ_DOLOOPBACK_MASK \</div><div class='del'>-	((u64)0x1 &lt;&lt; I40IW_UDA_QPSQ_DOLOOPBACK_SHIFT)</div><div class='del'>-</div><div class='del'>-/* CQP SQ WQE common fields */</div><div class='del'>-#define I40IW_CQPSQ_OPCODE_SHIFT 32</div><div class='del'>-#define I40IW_CQPSQ_OPCODE_MASK (0x3fULL &lt;&lt; I40IW_CQPSQ_OPCODE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_WQEVALID_SHIFT 63</div><div class='del'>-#define I40IW_CQPSQ_WQEVALID_MASK (1ULL &lt;&lt; I40IW_CQPSQ_WQEVALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_TPHVAL_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_TPHVAL_MASK (0xffUL &lt;&lt; I40IW_CQPSQ_TPHVAL_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_TPHEN_SHIFT 60</div><div class='del'>-#define I40IW_CQPSQ_TPHEN_MASK (1ULL &lt;&lt; I40IW_CQPSQ_TPHEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_PBUFADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IW_CQPSQ_PBUFADDR_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-/* Create/Modify/Destroy QP */</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_NEWMSS_SHIFT 32</div><div class='del'>-#define I40IW_CQPSQ_QP_NEWMSS_MASK (0x3fffULL &lt;&lt; I40IW_CQPSQ_QP_NEWMSS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_TERMLEN_SHIFT 48</div><div class='del'>-#define I40IW_CQPSQ_QP_TERMLEN_MASK (0xfULL &lt;&lt; I40IW_CQPSQ_QP_TERMLEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_QPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IW_CQPSQ_QP_QPCTX_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_QPID_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_QP_QPID_MASK (0x3FFFFUL)</div><div class='del'>-/* I40IWCQ_QPID_MASK */</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_OP_SHIFT 32</div><div class='del'>-#define I40IW_CQPSQ_QP_OP_MASK I40IWCQ_OP_MASK</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_ORDVALID_SHIFT 42</div><div class='del'>-#define I40IW_CQPSQ_QP_ORDVALID_MASK (1ULL &lt;&lt; I40IW_CQPSQ_QP_ORDVALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_TOECTXVALID_SHIFT 43</div><div class='del'>-#define I40IW_CQPSQ_QP_TOECTXVALID_MASK \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_QP_TOECTXVALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_CACHEDVARVALID_SHIFT 44</div><div class='del'>-#define I40IW_CQPSQ_QP_CACHEDVARVALID_MASK      \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_QP_CACHEDVARVALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_VQ_SHIFT 45</div><div class='del'>-#define I40IW_CQPSQ_QP_VQ_MASK (1ULL &lt;&lt; I40IW_CQPSQ_QP_VQ_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_FORCELOOPBACK_SHIFT 46</div><div class='del'>-#define I40IW_CQPSQ_QP_FORCELOOPBACK_MASK       \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_QP_FORCELOOPBACK_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_CQNUMVALID_SHIFT 47</div><div class='del'>-#define I40IW_CQPSQ_QP_CQNUMVALID_MASK  \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_QP_CQNUMVALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_QPTYPE_SHIFT 48</div><div class='del'>-#define I40IW_CQPSQ_QP_QPTYPE_MASK (0x3ULL &lt;&lt; I40IW_CQPSQ_QP_QPTYPE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_MSSCHANGE_SHIFT 52</div><div class='del'>-#define I40IW_CQPSQ_QP_MSSCHANGE_MASK (1ULL &lt;&lt; I40IW_CQPSQ_QP_MSSCHANGE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_IGNOREMWBOUND_SHIFT 54</div><div class='del'>-#define I40IW_CQPSQ_QP_IGNOREMWBOUND_MASK       \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_QP_IGNOREMWBOUND_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_REMOVEHASHENTRY_SHIFT 55</div><div class='del'>-#define I40IW_CQPSQ_QP_REMOVEHASHENTRY_MASK     \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_QP_REMOVEHASHENTRY_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_TERMACT_SHIFT 56</div><div class='del'>-#define I40IW_CQPSQ_QP_TERMACT_MASK (0x3ULL &lt;&lt; I40IW_CQPSQ_QP_TERMACT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_RESETCON_SHIFT 58</div><div class='del'>-#define I40IW_CQPSQ_QP_RESETCON_MASK (1ULL &lt;&lt; I40IW_CQPSQ_QP_RESETCON_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_ARPTABIDXVALID_SHIFT 59</div><div class='del'>-#define I40IW_CQPSQ_QP_ARPTABIDXVALID_MASK      \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_QP_ARPTABIDXVALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_NEXTIWSTATE_SHIFT 60</div><div class='del'>-#define I40IW_CQPSQ_QP_NEXTIWSTATE_MASK \</div><div class='del'>-	(0x7ULL &lt;&lt; I40IW_CQPSQ_QP_NEXTIWSTATE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_QP_DBSHADOWADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IW_CQPSQ_QP_DBSHADOWADDR_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-/* Create/Modify/Destroy CQ */</div><div class='del'>-#define I40IW_CQPSQ_CQ_CQSIZE_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_CQ_CQSIZE_MASK (0x3ffffUL &lt;&lt; I40IW_CQPSQ_CQ_CQSIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_CQ_CQCTX_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_CQ_CQCTX_MASK       \</div><div class='del'>-	(0x7fffffffffffffffULL &lt;&lt; I40IW_CQPSQ_CQ_CQCTX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_CQ_CQCTX_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_CQ_CQCTX_MASK       \</div><div class='del'>-	(0x7fffffffffffffffULL &lt;&lt; I40IW_CQPSQ_CQ_CQCTX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD_MASK       \</div><div class='del'>-	(0x3ffff &lt;&lt; I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_CQ_CEQID_SHIFT 24</div><div class='del'>-#define I40IW_CQPSQ_CQ_CEQID_MASK (0x7fUL &lt;&lt; I40IW_CQPSQ_CQ_CEQID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_CQ_OP_SHIFT 32</div><div class='del'>-#define I40IW_CQPSQ_CQ_OP_MASK (0x3fULL &lt;&lt; I40IW_CQPSQ_CQ_OP_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_CQ_CQRESIZE_SHIFT 43</div><div class='del'>-#define I40IW_CQPSQ_CQ_CQRESIZE_MASK (1ULL &lt;&lt; I40IW_CQPSQ_CQ_CQRESIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT 44</div><div class='del'>-#define I40IW_CQPSQ_CQ_LPBLSIZE_MASK (3ULL &lt;&lt; I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_CQ_CHKOVERFLOW_SHIFT 46</div><div class='del'>-#define I40IW_CQPSQ_CQ_CHKOVERFLOW_MASK         \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_CQ_CHKOVERFLOW_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_CQ_VIRTMAP_SHIFT 47</div><div class='del'>-#define I40IW_CQPSQ_CQ_VIRTMAP_MASK (1ULL &lt;&lt; I40IW_CQPSQ_CQ_VIRTMAP_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_CQ_ENCEQEMASK_SHIFT 48</div><div class='del'>-#define I40IW_CQPSQ_CQ_ENCEQEMASK_MASK  \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_CQ_ENCEQEMASK_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_CQ_CEQIDVALID_SHIFT 49</div><div class='del'>-#define I40IW_CQPSQ_CQ_CEQIDVALID_MASK  \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_CQ_CEQIDVALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT_SHIFT 61</div><div class='del'>-#define I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT_MASK      \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Create/Modify/Destroy Shared Receive Queue */</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_SRQ_RQSIZE_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_SRQ_RQSIZE_MASK (0xfUL &lt;&lt; I40IW_CQPSQ_SRQ_RQSIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_SRQ_RQWQESIZE_SHIFT 4</div><div class='del'>-#define I40IW_CQPSQ_SRQ_RQWQESIZE_MASK \</div><div class='del'>-	(0x7UL &lt;&lt; I40IW_CQPSQ_SRQ_RQWQESIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_SRQ_SRQLIMIT_SHIFT 32</div><div class='del'>-#define I40IW_CQPSQ_SRQ_SRQLIMIT_MASK   \</div><div class='del'>-	(0xfffULL &lt;&lt; I40IW_CQPSQ_SRQ_SRQLIMIT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_SRQ_SRQCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IW_CQPSQ_SRQ_SRQCTX_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_SRQ_PDID_SHIFT 16</div><div class='del'>-#define I40IW_CQPSQ_SRQ_PDID_MASK       \</div><div class='del'>-	(0x7fffULL &lt;&lt; I40IW_CQPSQ_SRQ_PDID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_SRQ_SRQID_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_SRQ_SRQID_MASK (0x7fffUL &lt;&lt; I40IW_CQPSQ_SRQ_SRQID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_SRQ_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT</div><div class='del'>-#define I40IW_CQPSQ_SRQ_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_SRQ_VIRTMAP_SHIFT I40IW_CQPSQ_CQ_VIRTMAP_SHIFT</div><div class='del'>-#define I40IW_CQPSQ_SRQ_VIRTMAP_MASK I40IW_CQPSQ_CQ_VIRTMAP_MASK</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_SRQ_TPHEN_SHIFT I40IW_CQPSQ_TPHEN_SHIFT</div><div class='del'>-#define I40IW_CQPSQ_SRQ_TPHEN_MASK I40IW_CQPSQ_TPHEN_MASK</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_SRQ_ARMLIMITEVENT_SHIFT 61</div><div class='del'>-#define I40IW_CQPSQ_SRQ_ARMLIMITEVENT_MASK      \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_SRQ_ARMLIMITEVENT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_SRQ_DBSHADOWAREA_SHIFT 6</div><div class='del'>-#define I40IW_CQPSQ_SRQ_DBSHADOWAREA_MASK       \</div><div class='del'>-	(0x3ffffffffffffffULL &lt;&lt; I40IW_CQPSQ_SRQ_DBSHADOWAREA_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_SRQ_FIRSTPMPBLIDX_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_SRQ_FIRSTPMPBLIDX_MASK      \</div><div class='del'>-	(0xfffffffUL &lt;&lt; I40IW_CQPSQ_SRQ_FIRSTPMPBLIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Allocate/Register/Register Shared/Deallocate Stag */</div><div class='del'>-#define I40IW_CQPSQ_STAG_VA_FBO_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IW_CQPSQ_STAG_VA_FBO_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_STAG_STAGLEN_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_STAG_STAGLEN_MASK   \</div><div class='del'>-	(0x3fffffffffffULL &lt;&lt; I40IW_CQPSQ_STAG_STAGLEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_STAG_PDID_SHIFT 48</div><div class='del'>-#define I40IW_CQPSQ_STAG_PDID_MASK (0x7fffULL &lt;&lt; I40IW_CQPSQ_STAG_PDID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_STAG_KEY_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_STAG_KEY_MASK (0xffUL &lt;&lt; I40IW_CQPSQ_STAG_KEY_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_STAG_IDX_SHIFT 8</div><div class='del'>-#define I40IW_CQPSQ_STAG_IDX_MASK (0xffffffUL &lt;&lt; I40IW_CQPSQ_STAG_IDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_STAG_PARENTSTAGIDX_SHIFT 32</div><div class='del'>-#define I40IW_CQPSQ_STAG_PARENTSTAGIDX_MASK     \</div><div class='del'>-	(0xffffffULL &lt;&lt; I40IW_CQPSQ_STAG_PARENTSTAGIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_STAG_MR_SHIFT 43</div><div class='del'>-#define I40IW_CQPSQ_STAG_MR_MASK (1ULL &lt;&lt; I40IW_CQPSQ_STAG_MR_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_STAG_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT</div><div class='del'>-#define I40IW_CQPSQ_STAG_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_STAG_HPAGESIZE_SHIFT 46</div><div class='del'>-#define I40IW_CQPSQ_STAG_HPAGESIZE_MASK \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_STAG_HPAGESIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_STAG_ARIGHTS_SHIFT 48</div><div class='del'>-#define I40IW_CQPSQ_STAG_ARIGHTS_MASK   \</div><div class='del'>-	(0x1fULL &lt;&lt; I40IW_CQPSQ_STAG_ARIGHTS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_STAG_REMACCENABLED_SHIFT 53</div><div class='del'>-#define I40IW_CQPSQ_STAG_REMACCENABLED_MASK     \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_STAG_REMACCENABLED_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_STAG_VABASEDTO_SHIFT 59</div><div class='del'>-#define I40IW_CQPSQ_STAG_VABASEDTO_MASK \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_STAG_VABASEDTO_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_STAG_USEHMCFNIDX_SHIFT 60</div><div class='del'>-#define I40IW_CQPSQ_STAG_USEHMCFNIDX_MASK       \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_STAG_USEHMCFNIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_STAG_USEPFRID_SHIFT 61</div><div class='del'>-#define I40IW_CQPSQ_STAG_USEPFRID_MASK  \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_STAG_USEPFRID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_STAG_PBA_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IW_CQPSQ_STAG_PBA_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_STAG_HMCFNIDX_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_STAG_HMCFNIDX_MASK \</div><div class='del'>-	(0x3fUL &lt;&lt; I40IW_CQPSQ_STAG_HMCFNIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_STAG_FIRSTPMPBLIDX_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_STAG_FIRSTPMPBLIDX_MASK     \</div><div class='del'>-	(0xfffffffUL &lt;&lt; I40IW_CQPSQ_STAG_FIRSTPMPBLIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Query stag */</div><div class='del'>-#define I40IW_CQPSQ_QUERYSTAG_IDX_SHIFT I40IW_CQPSQ_STAG_IDX_SHIFT</div><div class='del'>-#define I40IW_CQPSQ_QUERYSTAG_IDX_MASK I40IW_CQPSQ_STAG_IDX_MASK</div><div class='del'>-</div><div class='del'>-/* Allocate Local IP Address Entry */</div><div class='del'>-</div><div class='del'>-/* Manage Local IP Address Table - MLIPA */</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_IPV6LO_SHIFT  I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_IPV6LO_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_IPV6HI_SHIFT  I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_IPV6HI_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_IPV4_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_IPV4_MASK \</div><div class='del'>-	(0xffffffffUL &lt;&lt; I40IW_CQPSQ_MLIPA_IPV4_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_IPTABLEIDX_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_IPTABLEIDX_MASK       \</div><div class='del'>-	(0x3fUL &lt;&lt; I40IW_CQPSQ_MLIPA_IPTABLEIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_IPV4VALID_SHIFT 42</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_IPV4VALID_MASK        \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_MLIPA_IPV4VALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_IPV6VALID_SHIFT 43</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_IPV6VALID_MASK        \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_MLIPA_IPV6VALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_FREEENTRY_SHIFT 62</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_FREEENTRY_MASK        \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_MLIPA_FREEENTRY_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT_SHIFT 61</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT_MASK   \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_MAC0_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_MAC0_MASK (0xffUL &lt;&lt; I40IW_CQPSQ_MLIPA_MAC0_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_MAC1_SHIFT 8</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_MAC1_MASK (0xffUL &lt;&lt; I40IW_CQPSQ_MLIPA_MAC1_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_MAC2_SHIFT 16</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_MAC2_MASK (0xffUL &lt;&lt; I40IW_CQPSQ_MLIPA_MAC2_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_MAC3_SHIFT 24</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_MAC3_MASK (0xffUL &lt;&lt; I40IW_CQPSQ_MLIPA_MAC3_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_MAC4_SHIFT 32</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_MAC4_MASK (0xffULL &lt;&lt; I40IW_CQPSQ_MLIPA_MAC4_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_MAC5_SHIFT 40</div><div class='del'>-#define I40IW_CQPSQ_MLIPA_MAC5_MASK (0xffULL &lt;&lt; I40IW_CQPSQ_MLIPA_MAC5_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Manage ARP Table  - MAT */</div><div class='del'>-#define I40IW_CQPSQ_MAT_REACHMAX_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_MAT_REACHMAX_MASK   \</div><div class='del'>-	(0xffffffffUL &lt;&lt; I40IW_CQPSQ_MAT_REACHMAX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MAT_MACADDR_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_MAT_MACADDR_MASK    \</div><div class='del'>-	(0xffffffffffffULL &lt;&lt; I40IW_CQPSQ_MAT_MACADDR_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MAT_ARPENTRYIDX_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_MAT_ARPENTRYIDX_MASK        \</div><div class='del'>-	(0xfffUL &lt;&lt; I40IW_CQPSQ_MAT_ARPENTRYIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MAT_ENTRYVALID_SHIFT 42</div><div class='del'>-#define I40IW_CQPSQ_MAT_ENTRYVALID_MASK \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_MAT_ENTRYVALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MAT_PERMANENT_SHIFT 43</div><div class='del'>-#define I40IW_CQPSQ_MAT_PERMANENT_MASK  \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_MAT_PERMANENT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MAT_QUERY_SHIFT 44</div><div class='del'>-#define I40IW_CQPSQ_MAT_QUERY_MASK (1ULL &lt;&lt; I40IW_CQPSQ_MAT_QUERY_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Manage VF PBLE Backing Pages - MVPBP*/</div><div class='del'>-#define I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT_MASK \</div><div class='del'>-	(0x3ffULL &lt;&lt; I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MVPBP_FIRST_PD_INX_SHIFT 16</div><div class='del'>-#define I40IW_CQPSQ_MVPBP_FIRST_PD_INX_MASK \</div><div class='del'>-	(0x1ffULL &lt;&lt; I40IW_CQPSQ_MVPBP_FIRST_PD_INX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MVPBP_SD_INX_SHIFT 32</div><div class='del'>-#define I40IW_CQPSQ_MVPBP_SD_INX_MASK \</div><div class='del'>-	(0xfffULL &lt;&lt; I40IW_CQPSQ_MVPBP_SD_INX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MVPBP_INV_PD_ENT_SHIFT 62</div><div class='del'>-#define I40IW_CQPSQ_MVPBP_INV_PD_ENT_MASK \</div><div class='del'>-	(0x1ULL &lt;&lt; I40IW_CQPSQ_MVPBP_INV_PD_ENT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MVPBP_PD_PLPBA_SHIFT 3</div><div class='del'>-#define I40IW_CQPSQ_MVPBP_PD_PLPBA_MASK \</div><div class='del'>-	(0x1fffffffffffffffULL &lt;&lt; I40IW_CQPSQ_MVPBP_PD_PLPBA_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Manage Push Page - MPP */</div><div class='del'>-#define I40IW_INVALID_PUSH_PAGE_INDEX 0xffff</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MPP_QS_HANDLE_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_MPP_QS_HANDLE_MASK (0xffffUL &lt;&lt; \</div><div class='del'>-					I40IW_CQPSQ_MPP_QS_HANDLE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MPP_PPIDX_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_MPP_PPIDX_MASK (0x3ffUL &lt;&lt; I40IW_CQPSQ_MPP_PPIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MPP_FREE_PAGE_SHIFT 62</div><div class='del'>-#define I40IW_CQPSQ_MPP_FREE_PAGE_MASK (1ULL &lt;&lt; I40IW_CQPSQ_MPP_FREE_PAGE_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Upload Context - UCTX */</div><div class='del'>-#define I40IW_CQPSQ_UCTX_QPCTXADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IW_CQPSQ_UCTX_QPCTXADDR_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_UCTX_QPID_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_UCTX_QPID_MASK (0x3ffffUL &lt;&lt; I40IW_CQPSQ_UCTX_QPID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_UCTX_QPTYPE_SHIFT 48</div><div class='del'>-#define I40IW_CQPSQ_UCTX_QPTYPE_MASK (0xfULL &lt;&lt; I40IW_CQPSQ_UCTX_QPTYPE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_UCTX_RAWFORMAT_SHIFT 61</div><div class='del'>-#define I40IW_CQPSQ_UCTX_RAWFORMAT_MASK \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_UCTX_RAWFORMAT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_UCTX_FREEZEQP_SHIFT 62</div><div class='del'>-#define I40IW_CQPSQ_UCTX_FREEZEQP_MASK  \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_UCTX_FREEZEQP_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Manage HMC PM Function Table - MHMC */</div><div class='del'>-#define I40IW_CQPSQ_MHMC_VFIDX_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_MHMC_VFIDX_MASK (0x7fUL &lt;&lt; I40IW_CQPSQ_MHMC_VFIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MHMC_FREEPMFN_SHIFT 62</div><div class='del'>-#define I40IW_CQPSQ_MHMC_FREEPMFN_MASK  \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_MHMC_FREEPMFN_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Set HMC Resource Profile - SHMCRP */</div><div class='del'>-#define I40IW_CQPSQ_SHMCRP_HMC_PROFILE_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_SHMCRP_HMC_PROFILE_MASK \</div><div class='del'>-	(0x7ULL &lt;&lt; I40IW_CQPSQ_SHMCRP_HMC_PROFILE_SHIFT)</div><div class='del'>-#define I40IW_CQPSQ_SHMCRP_VFNUM_SHIFT 32</div><div class='del'>-#define I40IW_CQPSQ_SHMCRP_VFNUM_MASK (0x3fULL &lt;&lt; I40IW_CQPSQ_SHMCRP_VFNUM_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Create/Destroy CEQ */</div><div class='del'>-#define I40IW_CQPSQ_CEQ_CEQSIZE_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_CEQ_CEQSIZE_MASK \</div><div class='del'>-	(0x1ffffUL &lt;&lt; I40IW_CQPSQ_CEQ_CEQSIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_CEQ_CEQID_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_CEQ_CEQID_MASK (0x7fUL &lt;&lt; I40IW_CQPSQ_CEQ_CEQID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_CEQ_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT</div><div class='del'>-#define I40IW_CQPSQ_CEQ_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_CEQ_VMAP_SHIFT 47</div><div class='del'>-#define I40IW_CQPSQ_CEQ_VMAP_MASK (1ULL &lt;&lt; I40IW_CQPSQ_CEQ_VMAP_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_CEQ_FIRSTPMPBLIDX_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_CEQ_FIRSTPMPBLIDX_MASK      \</div><div class='del'>-	(0xfffffffUL &lt;&lt; I40IW_CQPSQ_CEQ_FIRSTPMPBLIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Create/Destroy AEQ */</div><div class='del'>-#define I40IW_CQPSQ_AEQ_AEQECNT_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_AEQ_AEQECNT_MASK \</div><div class='del'>-	(0x7ffffUL &lt;&lt; I40IW_CQPSQ_AEQ_AEQECNT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_AEQ_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT</div><div class='del'>-#define I40IW_CQPSQ_AEQ_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_AEQ_VMAP_SHIFT 47</div><div class='del'>-#define I40IW_CQPSQ_AEQ_VMAP_MASK (1ULL &lt;&lt; I40IW_CQPSQ_AEQ_VMAP_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_AEQ_FIRSTPMPBLIDX_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_AEQ_FIRSTPMPBLIDX_MASK      \</div><div class='del'>-	(0xfffffffUL &lt;&lt; I40IW_CQPSQ_AEQ_FIRSTPMPBLIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Commit FPM Values - CFPM */</div><div class='del'>-#define I40IW_CQPSQ_CFPM_HMCFNID_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_CFPM_HMCFNID_MASK (0x3fUL &lt;&lt; I40IW_CQPSQ_CFPM_HMCFNID_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Flush WQEs - FWQE */</div><div class='del'>-#define I40IW_CQPSQ_FWQE_AECODE_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_FWQE_AECODE_MASK (0xffffUL &lt;&lt; I40IW_CQPSQ_FWQE_AECODE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_FWQE_AESOURCE_SHIFT 16</div><div class='del'>-#define I40IW_CQPSQ_FWQE_AESOURCE_MASK \</div><div class='del'>-	(0xfUL &lt;&lt; I40IW_CQPSQ_FWQE_AESOURCE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_FWQE_RQMNERR_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_FWQE_RQMNERR_MASK \</div><div class='del'>-	(0xffffUL &lt;&lt; I40IW_CQPSQ_FWQE_RQMNERR_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_FWQE_RQMJERR_SHIFT 16</div><div class='del'>-#define I40IW_CQPSQ_FWQE_RQMJERR_MASK \</div><div class='del'>-	(0xffffUL &lt;&lt; I40IW_CQPSQ_FWQE_RQMJERR_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_FWQE_SQMNERR_SHIFT 32</div><div class='del'>-#define I40IW_CQPSQ_FWQE_SQMNERR_MASK   \</div><div class='del'>-	(0xffffULL &lt;&lt; I40IW_CQPSQ_FWQE_SQMNERR_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_FWQE_SQMJERR_SHIFT 48</div><div class='del'>-#define I40IW_CQPSQ_FWQE_SQMJERR_MASK   \</div><div class='del'>-	(0xffffULL &lt;&lt; I40IW_CQPSQ_FWQE_SQMJERR_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_FWQE_QPID_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_FWQE_QPID_MASK (0x3ffffULL &lt;&lt; I40IW_CQPSQ_FWQE_QPID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_FWQE_GENERATE_AE_SHIFT 59</div><div class='del'>-#define I40IW_CQPSQ_FWQE_GENERATE_AE_MASK (1ULL &lt;&lt;      \</div><div class='del'>-					   I40IW_CQPSQ_FWQE_GENERATE_AE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_FWQE_USERFLCODE_SHIFT 60</div><div class='del'>-#define I40IW_CQPSQ_FWQE_USERFLCODE_MASK        \</div><div class='del'>-	(1ULL &lt;&lt; I40IW_CQPSQ_FWQE_USERFLCODE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_FWQE_FLUSHSQ_SHIFT 61</div><div class='del'>-#define I40IW_CQPSQ_FWQE_FLUSHSQ_MASK (1ULL &lt;&lt; I40IW_CQPSQ_FWQE_FLUSHSQ_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_FWQE_FLUSHRQ_SHIFT 62</div><div class='del'>-#define I40IW_CQPSQ_FWQE_FLUSHRQ_MASK (1ULL &lt;&lt; I40IW_CQPSQ_FWQE_FLUSHRQ_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Manage Accelerated Port Table - MAPT */</div><div class='del'>-#define I40IW_CQPSQ_MAPT_PORT_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_MAPT_PORT_MASK (0xffffUL &lt;&lt; I40IW_CQPSQ_MAPT_PORT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_MAPT_ADDPORT_SHIFT 62</div><div class='del'>-#define I40IW_CQPSQ_MAPT_ADDPORT_MASK (1ULL &lt;&lt; I40IW_CQPSQ_MAPT_ADDPORT_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Update Protocol Engine SDs */</div><div class='del'>-#define I40IW_CQPSQ_UPESD_SDCMD_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_UPESD_SDCMD_MASK (0xffffffffUL &lt;&lt; I40IW_CQPSQ_UPESD_SDCMD_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_UPESD_SDDATALOW_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_UPESD_SDDATALOW_MASK        \</div><div class='del'>-	(0xffffffffUL &lt;&lt; I40IW_CQPSQ_UPESD_SDDATALOW_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_UPESD_SDDATAHI_SHIFT 32</div><div class='del'>-#define I40IW_CQPSQ_UPESD_SDDATAHI_MASK \</div><div class='del'>-	(0xffffffffULL &lt;&lt; I40IW_CQPSQ_UPESD_SDDATAHI_SHIFT)</div><div class='del'>-#define I40IW_CQPSQ_UPESD_HMCFNID_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_UPESD_HMCFNID_MASK  \</div><div class='del'>-	(0x3fUL &lt;&lt; I40IW_CQPSQ_UPESD_HMCFNID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_UPESD_ENTRY_VALID_SHIFT 63</div><div class='del'>-#define I40IW_CQPSQ_UPESD_ENTRY_VALID_MASK      \</div><div class='del'>-	((u64)1 &lt;&lt; I40IW_CQPSQ_UPESD_ENTRY_VALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_UPESD_ENTRY_COUNT_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_UPESD_ENTRY_COUNT_MASK      \</div><div class='del'>-	(0xfUL &lt;&lt; I40IW_CQPSQ_UPESD_ENTRY_COUNT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_UPESD_SKIP_ENTRY_SHIFT 7</div><div class='del'>-#define I40IW_CQPSQ_UPESD_SKIP_ENTRY_MASK       \</div><div class='del'>-	(0x1UL &lt;&lt; I40IW_CQPSQ_UPESD_SKIP_ENTRY_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Suspend QP */</div><div class='del'>-#define I40IW_CQPSQ_SUSPENDQP_QPID_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_SUSPENDQP_QPID_MASK (0x3FFFFUL)</div><div class='del'>-/* I40IWCQ_QPID_MASK */</div><div class='del'>-</div><div class='del'>-/* Resume QP */</div><div class='del'>-#define I40IW_CQPSQ_RESUMEQP_QSHANDLE_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_RESUMEQP_QSHANDLE_MASK      \</div><div class='del'>-	(0xffffffffUL &lt;&lt; I40IW_CQPSQ_RESUMEQP_QSHANDLE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_CQPSQ_RESUMEQP_QPID_SHIFT 0</div><div class='del'>-#define I40IW_CQPSQ_RESUMEQP_QPID_MASK (0x3FFFFUL)</div><div class='del'>-/* I40IWCQ_QPID_MASK */</div><div class='del'>-</div><div class='del'>-/* IW QP Context */</div><div class='del'>-#define I40IWQPC_DDP_VER_SHIFT 0</div><div class='del'>-#define I40IWQPC_DDP_VER_MASK (3UL &lt;&lt; I40IWQPC_DDP_VER_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SNAP_SHIFT 2</div><div class='del'>-#define I40IWQPC_SNAP_MASK (1UL &lt;&lt; I40IWQPC_SNAP_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_IPV4_SHIFT 3</div><div class='del'>-#define I40IWQPC_IPV4_MASK (1UL &lt;&lt; I40IWQPC_IPV4_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_NONAGLE_SHIFT 4</div><div class='del'>-#define I40IWQPC_NONAGLE_MASK (1UL &lt;&lt; I40IWQPC_NONAGLE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_INSERTVLANTAG_SHIFT 5</div><div class='del'>-#define I40IWQPC_INSERTVLANTAG_MASK (1 &lt;&lt; I40IWQPC_INSERTVLANTAG_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_USESRQ_SHIFT 6</div><div class='del'>-#define I40IWQPC_USESRQ_MASK (1UL &lt;&lt; I40IWQPC_USESRQ_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_TIMESTAMP_SHIFT 7</div><div class='del'>-#define I40IWQPC_TIMESTAMP_MASK (1UL &lt;&lt; I40IWQPC_TIMESTAMP_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_RQWQESIZE_SHIFT 8</div><div class='del'>-#define I40IWQPC_RQWQESIZE_MASK (3UL &lt;&lt; I40IWQPC_RQWQESIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_INSERTL2TAG2_SHIFT 11</div><div class='del'>-#define I40IWQPC_INSERTL2TAG2_MASK (1UL &lt;&lt; I40IWQPC_INSERTL2TAG2_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_LIMIT_SHIFT 12</div><div class='del'>-#define I40IWQPC_LIMIT_MASK (3UL &lt;&lt; I40IWQPC_LIMIT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_DROPOOOSEG_SHIFT 15</div><div class='del'>-#define I40IWQPC_DROPOOOSEG_MASK (1UL &lt;&lt; I40IWQPC_DROPOOOSEG_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_DUPACK_THRESH_SHIFT 16</div><div class='del'>-#define I40IWQPC_DUPACK_THRESH_MASK (7UL &lt;&lt; I40IWQPC_DUPACK_THRESH_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_ERR_RQ_IDX_VALID_SHIFT 19</div><div class='del'>-#define I40IWQPC_ERR_RQ_IDX_VALID_MASK  (1UL &lt;&lt; I40IWQPC_ERR_RQ_IDX_VALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_DIS_VLAN_CHECKS_SHIFT 19</div><div class='del'>-#define I40IWQPC_DIS_VLAN_CHECKS_MASK (7UL &lt;&lt; I40IWQPC_DIS_VLAN_CHECKS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_RCVTPHEN_SHIFT 28</div><div class='del'>-#define I40IWQPC_RCVTPHEN_MASK (1UL &lt;&lt; I40IWQPC_RCVTPHEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_XMITTPHEN_SHIFT 29</div><div class='del'>-#define I40IWQPC_XMITTPHEN_MASK (1ULL &lt;&lt; I40IWQPC_XMITTPHEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_RQTPHEN_SHIFT 30</div><div class='del'>-#define I40IWQPC_RQTPHEN_MASK (1UL &lt;&lt; I40IWQPC_RQTPHEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SQTPHEN_SHIFT 31</div><div class='del'>-#define I40IWQPC_SQTPHEN_MASK (1ULL &lt;&lt; I40IWQPC_SQTPHEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_PPIDX_SHIFT 32</div><div class='del'>-#define I40IWQPC_PPIDX_MASK (0x3ffULL &lt;&lt; I40IWQPC_PPIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_PMENA_SHIFT 47</div><div class='del'>-#define I40IWQPC_PMENA_MASK (1ULL &lt;&lt; I40IWQPC_PMENA_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_RDMAP_VER_SHIFT 62</div><div class='del'>-#define I40IWQPC_RDMAP_VER_MASK (3ULL &lt;&lt; I40IWQPC_RDMAP_VER_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SQADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IWQPC_SQADDR_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_RQADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IWQPC_RQADDR_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_TTL_SHIFT 0</div><div class='del'>-#define I40IWQPC_TTL_MASK (0xffUL &lt;&lt; I40IWQPC_TTL_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_RQSIZE_SHIFT 8</div><div class='del'>-#define I40IWQPC_RQSIZE_MASK (0xfUL &lt;&lt; I40IWQPC_RQSIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SQSIZE_SHIFT 12</div><div class='del'>-#define I40IWQPC_SQSIZE_MASK (0xfUL &lt;&lt; I40IWQPC_SQSIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SRCMACADDRIDX_SHIFT 16</div><div class='del'>-#define I40IWQPC_SRCMACADDRIDX_MASK (0x3fUL &lt;&lt; I40IWQPC_SRCMACADDRIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_AVOIDSTRETCHACK_SHIFT 23</div><div class='del'>-#define I40IWQPC_AVOIDSTRETCHACK_MASK (1UL &lt;&lt; I40IWQPC_AVOIDSTRETCHACK_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_TOS_SHIFT 24</div><div class='del'>-#define I40IWQPC_TOS_MASK (0xffUL &lt;&lt; I40IWQPC_TOS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SRCPORTNUM_SHIFT 32</div><div class='del'>-#define I40IWQPC_SRCPORTNUM_MASK (0xffffULL &lt;&lt; I40IWQPC_SRCPORTNUM_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_DESTPORTNUM_SHIFT 48</div><div class='del'>-#define I40IWQPC_DESTPORTNUM_MASK (0xffffULL &lt;&lt; I40IWQPC_DESTPORTNUM_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_DESTIPADDR0_SHIFT 32</div><div class='del'>-#define I40IWQPC_DESTIPADDR0_MASK       \</div><div class='del'>-	(0xffffffffULL &lt;&lt; I40IWQPC_DESTIPADDR0_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_DESTIPADDR1_SHIFT 0</div><div class='del'>-#define I40IWQPC_DESTIPADDR1_MASK       \</div><div class='del'>-	(0xffffffffULL &lt;&lt; I40IWQPC_DESTIPADDR1_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_DESTIPADDR2_SHIFT 32</div><div class='del'>-#define I40IWQPC_DESTIPADDR2_MASK       \</div><div class='del'>-	(0xffffffffULL &lt;&lt; I40IWQPC_DESTIPADDR2_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_DESTIPADDR3_SHIFT 0</div><div class='del'>-#define I40IWQPC_DESTIPADDR3_MASK       \</div><div class='del'>-	(0xffffffffULL &lt;&lt; I40IWQPC_DESTIPADDR3_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SNDMSS_SHIFT 16</div><div class='del'>-#define I40IWQPC_SNDMSS_MASK (0x3fffUL &lt;&lt; I40IWQPC_SNDMSS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_UDA_QPC_MAXFRAMESIZE_SHIFT 16</div><div class='del'>-#define I40IW_UDA_QPC_MAXFRAMESIZE_MASK (0x3fffUL &lt;&lt; I40IW_UDA_QPC_MAXFRAMESIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_VLANTAG_SHIFT 32</div><div class='del'>-#define I40IWQPC_VLANTAG_MASK (0xffffULL &lt;&lt; I40IWQPC_VLANTAG_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_ARPIDX_SHIFT 48</div><div class='del'>-#define I40IWQPC_ARPIDX_MASK (0xffffULL &lt;&lt; I40IWQPC_ARPIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_FLOWLABEL_SHIFT 0</div><div class='del'>-#define I40IWQPC_FLOWLABEL_MASK (0xfffffUL &lt;&lt; I40IWQPC_FLOWLABEL_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_WSCALE_SHIFT 20</div><div class='del'>-#define I40IWQPC_WSCALE_MASK (1UL &lt;&lt; I40IWQPC_WSCALE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_KEEPALIVE_SHIFT 21</div><div class='del'>-#define I40IWQPC_KEEPALIVE_MASK (1UL &lt;&lt; I40IWQPC_KEEPALIVE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_IGNORE_TCP_OPT_SHIFT 22</div><div class='del'>-#define I40IWQPC_IGNORE_TCP_OPT_MASK (1UL &lt;&lt; I40IWQPC_IGNORE_TCP_OPT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_IGNORE_TCP_UNS_OPT_SHIFT 23</div><div class='del'>-#define I40IWQPC_IGNORE_TCP_UNS_OPT_MASK        \</div><div class='del'>-	(1UL &lt;&lt; I40IWQPC_IGNORE_TCP_UNS_OPT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_TCPSTATE_SHIFT 28</div><div class='del'>-#define I40IWQPC_TCPSTATE_MASK (0xfUL &lt;&lt; I40IWQPC_TCPSTATE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_RCVSCALE_SHIFT 32</div><div class='del'>-#define I40IWQPC_RCVSCALE_MASK (0xfULL &lt;&lt; I40IWQPC_RCVSCALE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SNDSCALE_SHIFT 40</div><div class='del'>-#define I40IWQPC_SNDSCALE_MASK (0xfULL &lt;&lt; I40IWQPC_SNDSCALE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_PDIDX_SHIFT 48</div><div class='del'>-#define I40IWQPC_PDIDX_MASK (0x7fffULL &lt;&lt; I40IWQPC_PDIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_KALIVE_TIMER_MAX_PROBES_SHIFT 16</div><div class='del'>-#define I40IWQPC_KALIVE_TIMER_MAX_PROBES_MASK   \</div><div class='del'>-	(0xffUL &lt;&lt; I40IWQPC_KALIVE_TIMER_MAX_PROBES_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_KEEPALIVE_INTERVAL_SHIFT 24</div><div class='del'>-#define I40IWQPC_KEEPALIVE_INTERVAL_MASK        \</div><div class='del'>-	(0xffUL &lt;&lt; I40IWQPC_KEEPALIVE_INTERVAL_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_TIMESTAMP_RECENT_SHIFT 0</div><div class='del'>-#define I40IWQPC_TIMESTAMP_RECENT_MASK  \</div><div class='del'>-	(0xffffffffUL &lt;&lt; I40IWQPC_TIMESTAMP_RECENT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_TIMESTAMP_AGE_SHIFT 32</div><div class='del'>-#define I40IWQPC_TIMESTAMP_AGE_MASK     \</div><div class='del'>-	(0xffffffffULL &lt;&lt; I40IWQPC_TIMESTAMP_AGE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SNDNXT_SHIFT 0</div><div class='del'>-#define I40IWQPC_SNDNXT_MASK (0xffffffffUL &lt;&lt; I40IWQPC_SNDNXT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SNDWND_SHIFT 32</div><div class='del'>-#define I40IWQPC_SNDWND_MASK (0xffffffffULL &lt;&lt; I40IWQPC_SNDWND_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_RCVNXT_SHIFT 0</div><div class='del'>-#define I40IWQPC_RCVNXT_MASK (0xffffffffUL &lt;&lt; I40IWQPC_RCVNXT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_RCVWND_SHIFT 32</div><div class='del'>-#define I40IWQPC_RCVWND_MASK (0xffffffffULL &lt;&lt; I40IWQPC_RCVWND_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SNDMAX_SHIFT 0</div><div class='del'>-#define I40IWQPC_SNDMAX_MASK (0xffffffffUL &lt;&lt; I40IWQPC_SNDMAX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SNDUNA_SHIFT 32</div><div class='del'>-#define I40IWQPC_SNDUNA_MASK (0xffffffffULL &lt;&lt; I40IWQPC_SNDUNA_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SRTT_SHIFT 0</div><div class='del'>-#define I40IWQPC_SRTT_MASK (0xffffffffUL &lt;&lt; I40IWQPC_SRTT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_RTTVAR_SHIFT 32</div><div class='del'>-#define I40IWQPC_RTTVAR_MASK (0xffffffffULL &lt;&lt; I40IWQPC_RTTVAR_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SSTHRESH_SHIFT 0</div><div class='del'>-#define I40IWQPC_SSTHRESH_MASK (0xffffffffUL &lt;&lt; I40IWQPC_SSTHRESH_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_CWND_SHIFT 32</div><div class='del'>-#define I40IWQPC_CWND_MASK (0xffffffffULL &lt;&lt; I40IWQPC_CWND_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SNDWL1_SHIFT 0</div><div class='del'>-#define I40IWQPC_SNDWL1_MASK (0xffffffffUL &lt;&lt; I40IWQPC_SNDWL1_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SNDWL2_SHIFT 32</div><div class='del'>-#define I40IWQPC_SNDWL2_MASK (0xffffffffULL &lt;&lt; I40IWQPC_SNDWL2_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_ERR_RQ_IDX_SHIFT 32</div><div class='del'>-#define I40IWQPC_ERR_RQ_IDX_MASK  (0x3fffULL &lt;&lt; I40IWQPC_ERR_RQ_IDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_MAXSNDWND_SHIFT 0</div><div class='del'>-#define I40IWQPC_MAXSNDWND_MASK (0xffffffffUL &lt;&lt; I40IWQPC_MAXSNDWND_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_REXMIT_THRESH_SHIFT 48</div><div class='del'>-#define I40IWQPC_REXMIT_THRESH_MASK (0x3fULL &lt;&lt; I40IWQPC_REXMIT_THRESH_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_TXCQNUM_SHIFT 0</div><div class='del'>-#define I40IWQPC_TXCQNUM_MASK (0x1ffffUL &lt;&lt; I40IWQPC_TXCQNUM_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_RXCQNUM_SHIFT 32</div><div class='del'>-#define I40IWQPC_RXCQNUM_MASK (0x1ffffULL &lt;&lt; I40IWQPC_RXCQNUM_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_STAT_INDEX_SHIFT 0</div><div class='del'>-#define I40IWQPC_STAT_INDEX_MASK (0x1fULL &lt;&lt; I40IWQPC_STAT_INDEX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_Q2ADDR_SHIFT 0</div><div class='del'>-#define I40IWQPC_Q2ADDR_MASK (0xffffffffffffff00ULL &lt;&lt; I40IWQPC_Q2ADDR_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_LASTBYTESENT_SHIFT 0</div><div class='del'>-#define I40IWQPC_LASTBYTESENT_MASK (0xffUL &lt;&lt; I40IWQPC_LASTBYTESENT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SRQID_SHIFT 32</div><div class='del'>-#define I40IWQPC_SRQID_MASK (0xffULL &lt;&lt; I40IWQPC_SRQID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_ORDSIZE_SHIFT 0</div><div class='del'>-#define I40IWQPC_ORDSIZE_MASK (0x7fUL &lt;&lt; I40IWQPC_ORDSIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_IRDSIZE_SHIFT 16</div><div class='del'>-#define I40IWQPC_IRDSIZE_MASK (0x3UL &lt;&lt; I40IWQPC_IRDSIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_WRRDRSPOK_SHIFT 20</div><div class='del'>-#define I40IWQPC_WRRDRSPOK_MASK (1UL &lt;&lt; I40IWQPC_WRRDRSPOK_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_RDOK_SHIFT 21</div><div class='del'>-#define I40IWQPC_RDOK_MASK (1UL &lt;&lt; I40IWQPC_RDOK_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SNDMARKERS_SHIFT 22</div><div class='del'>-#define I40IWQPC_SNDMARKERS_MASK (1UL &lt;&lt; I40IWQPC_SNDMARKERS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_BINDEN_SHIFT 23</div><div class='del'>-#define I40IWQPC_BINDEN_MASK (1UL &lt;&lt; I40IWQPC_BINDEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_FASTREGEN_SHIFT 24</div><div class='del'>-#define I40IWQPC_FASTREGEN_MASK (1UL &lt;&lt; I40IWQPC_FASTREGEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_PRIVEN_SHIFT 25</div><div class='del'>-#define I40IWQPC_PRIVEN_MASK (1UL &lt;&lt; I40IWQPC_PRIVEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_USESTATSINSTANCE_SHIFT 26</div><div class='del'>-#define I40IWQPC_USESTATSINSTANCE_MASK (1UL &lt;&lt; I40IWQPC_USESTATSINSTANCE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_IWARPMODE_SHIFT 28</div><div class='del'>-#define I40IWQPC_IWARPMODE_MASK (1UL &lt;&lt; I40IWQPC_IWARPMODE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_RCVMARKERS_SHIFT 29</div><div class='del'>-#define I40IWQPC_RCVMARKERS_MASK (1UL &lt;&lt; I40IWQPC_RCVMARKERS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_ALIGNHDRS_SHIFT 30</div><div class='del'>-#define I40IWQPC_ALIGNHDRS_MASK (1UL &lt;&lt; I40IWQPC_ALIGNHDRS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_RCVNOMPACRC_SHIFT 31</div><div class='del'>-#define I40IWQPC_RCVNOMPACRC_MASK (1UL &lt;&lt; I40IWQPC_RCVNOMPACRC_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_RCVMARKOFFSET_SHIFT 33</div><div class='del'>-#define I40IWQPC_RCVMARKOFFSET_MASK (0x1ffULL &lt;&lt; I40IWQPC_RCVMARKOFFSET_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SNDMARKOFFSET_SHIFT 48</div><div class='del'>-#define I40IWQPC_SNDMARKOFFSET_MASK (0x1ffULL &lt;&lt; I40IWQPC_SNDMARKOFFSET_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_QPCOMPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IWQPC_QPCOMPCTX_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_SQTPHVAL_SHIFT 0</div><div class='del'>-#define I40IWQPC_SQTPHVAL_MASK (0xffUL &lt;&lt; I40IWQPC_SQTPHVAL_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_RQTPHVAL_SHIFT 8</div><div class='del'>-#define I40IWQPC_RQTPHVAL_MASK (0xffUL &lt;&lt; I40IWQPC_RQTPHVAL_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_QSHANDLE_SHIFT 16</div><div class='del'>-#define I40IWQPC_QSHANDLE_MASK (0x3ffUL &lt;&lt; I40IWQPC_QSHANDLE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_EXCEPTION_LAN_QUEUE_SHIFT 32</div><div class='del'>-#define I40IWQPC_EXCEPTION_LAN_QUEUE_MASK (0xfffULL &lt;&lt;  \</div><div class='del'>-					   I40IWQPC_EXCEPTION_LAN_QUEUE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_LOCAL_IPADDR3_SHIFT 0</div><div class='del'>-#define I40IWQPC_LOCAL_IPADDR3_MASK \</div><div class='del'>-	(0xffffffffUL &lt;&lt; I40IWQPC_LOCAL_IPADDR3_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_LOCAL_IPADDR2_SHIFT 32</div><div class='del'>-#define I40IWQPC_LOCAL_IPADDR2_MASK     \</div><div class='del'>-	(0xffffffffULL &lt;&lt; I40IWQPC_LOCAL_IPADDR2_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_LOCAL_IPADDR1_SHIFT 0</div><div class='del'>-#define I40IWQPC_LOCAL_IPADDR1_MASK     \</div><div class='del'>-	(0xffffffffUL &lt;&lt; I40IWQPC_LOCAL_IPADDR1_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPC_LOCAL_IPADDR0_SHIFT 32</div><div class='del'>-#define I40IWQPC_LOCAL_IPADDR0_MASK     \</div><div class='del'>-	(0xffffffffULL &lt;&lt; I40IWQPC_LOCAL_IPADDR0_SHIFT)</div><div class='del'>-</div><div class='del'>-/* wqe size considering 32 bytes per wqe*/</div><div class='del'>-#define I40IW_QP_SW_MIN_WQSIZE 4		/*in WRs*/</div><div class='del'>-#define I40IW_SQ_RSVD 2</div><div class='del'>-#define I40IW_RQ_RSVD 1</div><div class='del'>-#define I40IW_MAX_QUANTAS_PER_WR 2</div><div class='del'>-#define I40IW_QP_SW_MAX_SQ_QUANTAS 2048</div><div class='del'>-#define I40IW_QP_SW_MAX_RQ_QUANTAS 16384</div><div class='del'>-#define I40IW_MAX_QP_WRS ((I40IW_QP_SW_MAX_SQ_QUANTAS / I40IW_MAX_QUANTAS_PER_WR) - 1)</div><div class='del'>-</div><div class='del'>-#define I40IWQP_OP_RDMA_WRITE 0</div><div class='del'>-#define I40IWQP_OP_RDMA_READ 1</div><div class='del'>-#define I40IWQP_OP_RDMA_SEND 3</div><div class='del'>-#define I40IWQP_OP_RDMA_SEND_INV 4</div><div class='del'>-#define I40IWQP_OP_RDMA_SEND_SOL_EVENT 5</div><div class='del'>-#define I40IWQP_OP_RDMA_SEND_SOL_EVENT_INV 6</div><div class='del'>-#define I40IWQP_OP_BIND_MW 8</div><div class='del'>-#define I40IWQP_OP_FAST_REGISTER 9</div><div class='del'>-#define I40IWQP_OP_LOCAL_INVALIDATE 10</div><div class='del'>-#define I40IWQP_OP_RDMA_READ_LOC_INV 11</div><div class='del'>-#define I40IWQP_OP_NOP 12</div><div class='del'>-</div><div class='del'>-#define I40IW_RSVD_SHIFT        41</div><div class='del'>-#define I40IW_RSVD_MASK (0x7fffULL &lt;&lt; I40IW_RSVD_SHIFT)</div><div class='del'>-</div><div class='del'>-/* iwarp QP SQ WQE common fields */</div><div class='del'>-#define I40IWQPSQ_OPCODE_SHIFT 32</div><div class='del'>-#define I40IWQPSQ_OPCODE_MASK (0x3fULL &lt;&lt; I40IWQPSQ_OPCODE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_ADDFRAGCNT_SHIFT 38</div><div class='del'>-#define I40IWQPSQ_ADDFRAGCNT_MASK (0x7ULL &lt;&lt; I40IWQPSQ_ADDFRAGCNT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_PUSHWQE_SHIFT 56</div><div class='del'>-#define I40IWQPSQ_PUSHWQE_MASK (1ULL &lt;&lt; I40IWQPSQ_PUSHWQE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_STREAMMODE_SHIFT 58</div><div class='del'>-#define I40IWQPSQ_STREAMMODE_MASK (1ULL &lt;&lt; I40IWQPSQ_STREAMMODE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_WAITFORRCVPDU_SHIFT 59</div><div class='del'>-#define I40IWQPSQ_WAITFORRCVPDU_MASK (1ULL &lt;&lt; I40IWQPSQ_WAITFORRCVPDU_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_READFENCE_SHIFT 60</div><div class='del'>-#define I40IWQPSQ_READFENCE_MASK (1ULL &lt;&lt; I40IWQPSQ_READFENCE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_LOCALFENCE_SHIFT 61</div><div class='del'>-#define I40IWQPSQ_LOCALFENCE_MASK (1ULL &lt;&lt; I40IWQPSQ_LOCALFENCE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_SIGCOMPL_SHIFT 62</div><div class='del'>-#define I40IWQPSQ_SIGCOMPL_MASK (1ULL &lt;&lt; I40IWQPSQ_SIGCOMPL_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_VALID_SHIFT 63</div><div class='del'>-#define I40IWQPSQ_VALID_MASK (1ULL &lt;&lt; I40IWQPSQ_VALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_FRAG_TO_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IWQPSQ_FRAG_TO_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_FRAG_LEN_SHIFT 0</div><div class='del'>-#define I40IWQPSQ_FRAG_LEN_MASK (0xffffffffUL &lt;&lt; I40IWQPSQ_FRAG_LEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_FRAG_STAG_SHIFT 32</div><div class='del'>-#define I40IWQPSQ_FRAG_STAG_MASK (0xffffffffULL &lt;&lt; I40IWQPSQ_FRAG_STAG_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_REMSTAGINV_SHIFT 0</div><div class='del'>-#define I40IWQPSQ_REMSTAGINV_MASK (0xffffffffUL &lt;&lt; I40IWQPSQ_REMSTAGINV_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_INLINEDATAFLAG_SHIFT 57</div><div class='del'>-#define I40IWQPSQ_INLINEDATAFLAG_MASK (1ULL &lt;&lt; I40IWQPSQ_INLINEDATAFLAG_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_INLINEDATALEN_SHIFT 48</div><div class='del'>-#define I40IWQPSQ_INLINEDATALEN_MASK    \</div><div class='del'>-	(0x7fULL &lt;&lt; I40IWQPSQ_INLINEDATALEN_SHIFT)</div><div class='del'>-</div><div class='del'>-/* iwarp send with push mode */</div><div class='del'>-#define I40IWQPSQ_WQDESCIDX_SHIFT 0</div><div class='del'>-#define I40IWQPSQ_WQDESCIDX_MASK (0x3fffUL &lt;&lt; I40IWQPSQ_WQDESCIDX_SHIFT)</div><div class='del'>-</div><div class='del'>-/* rdma write */</div><div class='del'>-#define I40IWQPSQ_REMSTAG_SHIFT 0</div><div class='del'>-#define I40IWQPSQ_REMSTAG_MASK (0xffffffffUL &lt;&lt; I40IWQPSQ_REMSTAG_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_REMTO_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IWQPSQ_REMTO_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-/* memory window */</div><div class='del'>-#define I40IWQPSQ_STAGRIGHTS_SHIFT 48</div><div class='del'>-#define I40IWQPSQ_STAGRIGHTS_MASK (0x1fULL &lt;&lt; I40IWQPSQ_STAGRIGHTS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_VABASEDTO_SHIFT 53</div><div class='del'>-#define I40IWQPSQ_VABASEDTO_MASK (1ULL &lt;&lt; I40IWQPSQ_VABASEDTO_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_MWLEN_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IWQPSQ_MWLEN_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_PARENTMRSTAG_SHIFT 0</div><div class='del'>-#define I40IWQPSQ_PARENTMRSTAG_MASK \</div><div class='del'>-	(0xffffffffUL &lt;&lt; I40IWQPSQ_PARENTMRSTAG_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_MWSTAG_SHIFT 32</div><div class='del'>-#define I40IWQPSQ_MWSTAG_MASK (0xffffffffULL &lt;&lt; I40IWQPSQ_MWSTAG_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_BASEVA_TO_FBO_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IWQPSQ_BASEVA_TO_FBO_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-/* Local Invalidate */</div><div class='del'>-#define I40IWQPSQ_LOCSTAG_SHIFT 32</div><div class='del'>-#define I40IWQPSQ_LOCSTAG_MASK (0xffffffffULL &lt;&lt; I40IWQPSQ_LOCSTAG_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Fast Register */</div><div class='del'>-#define I40IWQPSQ_STAGKEY_SHIFT 0</div><div class='del'>-#define I40IWQPSQ_STAGKEY_MASK (0xffUL &lt;&lt; I40IWQPSQ_STAGKEY_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_STAGINDEX_SHIFT 8</div><div class='del'>-#define I40IWQPSQ_STAGINDEX_MASK (0xffffffUL &lt;&lt; I40IWQPSQ_STAGINDEX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_COPYHOSTPBLS_SHIFT 43</div><div class='del'>-#define I40IWQPSQ_COPYHOSTPBLS_MASK (1ULL &lt;&lt; I40IWQPSQ_COPYHOSTPBLS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_LPBLSIZE_SHIFT 44</div><div class='del'>-#define I40IWQPSQ_LPBLSIZE_MASK (3ULL &lt;&lt; I40IWQPSQ_LPBLSIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_HPAGESIZE_SHIFT 46</div><div class='del'>-#define I40IWQPSQ_HPAGESIZE_MASK (3ULL &lt;&lt; I40IWQPSQ_HPAGESIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_STAGLEN_SHIFT 0</div><div class='del'>-#define I40IWQPSQ_STAGLEN_MASK (0x1ffffffffffULL &lt;&lt; I40IWQPSQ_STAGLEN_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_FIRSTPMPBLIDXLO_SHIFT 48</div><div class='del'>-#define I40IWQPSQ_FIRSTPMPBLIDXLO_MASK  \</div><div class='del'>-	(0xffffULL &lt;&lt; I40IWQPSQ_FIRSTPMPBLIDXLO_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_FIRSTPMPBLIDXHI_SHIFT 0</div><div class='del'>-#define I40IWQPSQ_FIRSTPMPBLIDXHI_MASK  \</div><div class='del'>-	(0xfffUL &lt;&lt; I40IWQPSQ_FIRSTPMPBLIDXHI_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IWQPSQ_PBLADDR_SHIFT 12</div><div class='del'>-#define I40IWQPSQ_PBLADDR_MASK (0xfffffffffffffULL &lt;&lt; I40IWQPSQ_PBLADDR_SHIFT)</div><div class='del'>-</div><div class='del'>-/*  iwarp QP RQ WQE common fields */</div><div class='del'>-#define I40IWQPRQ_ADDFRAGCNT_SHIFT I40IWQPSQ_ADDFRAGCNT_SHIFT</div><div class='del'>-#define I40IWQPRQ_ADDFRAGCNT_MASK I40IWQPSQ_ADDFRAGCNT_MASK</div><div class='del'>-</div><div class='del'>-#define I40IWQPRQ_VALID_SHIFT I40IWQPSQ_VALID_SHIFT</div><div class='del'>-#define I40IWQPRQ_VALID_MASK I40IWQPSQ_VALID_MASK</div><div class='del'>-</div><div class='del'>-#define I40IWQPRQ_COMPLCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT</div><div class='del'>-#define I40IWQPRQ_COMPLCTX_MASK I40IW_CQPHC_QPCTX_MASK</div><div class='del'>-</div><div class='del'>-#define I40IWQPRQ_FRAG_LEN_SHIFT I40IWQPSQ_FRAG_LEN_SHIFT</div><div class='del'>-#define I40IWQPRQ_FRAG_LEN_MASK I40IWQPSQ_FRAG_LEN_MASK</div><div class='del'>-</div><div class='del'>-#define I40IWQPRQ_STAG_SHIFT I40IWQPSQ_FRAG_STAG_SHIFT</div><div class='del'>-#define I40IWQPRQ_STAG_MASK I40IWQPSQ_FRAG_STAG_MASK</div><div class='del'>-</div><div class='del'>-#define I40IWQPRQ_TO_SHIFT I40IWQPSQ_FRAG_TO_SHIFT</div><div class='del'>-#define I40IWQPRQ_TO_MASK I40IWQPSQ_FRAG_TO_MASK</div><div class='del'>-</div><div class='del'>-/* Query FPM CQP buf */</div><div class='del'>-#define I40IW_QUERY_FPM_MAX_QPS_SHIFT 0</div><div class='del'>-#define I40IW_QUERY_FPM_MAX_QPS_MASK               \</div><div class='del'>-	(0x7ffffUL &lt;&lt; I40IW_QUERY_FPM_MAX_QPS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_QUERY_FPM_MAX_CQS_SHIFT 0</div><div class='del'>-#define I40IW_QUERY_FPM_MAX_CQS_MASK               \</div><div class='del'>-	(0x3ffffUL &lt;&lt; I40IW_QUERY_FPM_MAX_CQS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_QUERY_FPM_FIRST_PE_SD_INDEX_SHIFT 0</div><div class='del'>-#define I40IW_QUERY_FPM_FIRST_PE_SD_INDEX_MASK  \</div><div class='del'>-	(0x3fffUL &lt;&lt; I40IW_QUERY_FPM_FIRST_PE_SD_INDEX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_QUERY_FPM_MAX_PE_SDS_SHIFT 32</div><div class='del'>-#define I40IW_QUERY_FPM_MAX_PE_SDS_MASK \</div><div class='del'>-	(0x3fffULL &lt;&lt; I40IW_QUERY_FPM_MAX_PE_SDS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_QUERY_FPM_MAX_QPS_SHIFT 0</div><div class='del'>-#define I40IW_QUERY_FPM_MAX_QPS_MASK    \</div><div class='del'>-	(0x7ffffUL &lt;&lt; I40IW_QUERY_FPM_MAX_QPS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_QUERY_FPM_MAX_CQS_SHIFT 0</div><div class='del'>-#define I40IW_QUERY_FPM_MAX_CQS_MASK    \</div><div class='del'>-	(0x3ffffUL &lt;&lt; I40IW_QUERY_FPM_MAX_CQS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_QUERY_FPM_MAX_CEQS_SHIFT 0</div><div class='del'>-#define I40IW_QUERY_FPM_MAX_CEQS_MASK   \</div><div class='del'>-	(0xffUL &lt;&lt; I40IW_QUERY_FPM_MAX_CEQS_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_QUERY_FPM_XFBLOCKSIZE_SHIFT 32</div><div class='del'>-#define I40IW_QUERY_FPM_XFBLOCKSIZE_MASK        \</div><div class='del'>-	(0xffffffffULL &lt;&lt; I40IW_QUERY_FPM_XFBLOCKSIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_QUERY_FPM_Q1BLOCKSIZE_SHIFT 32</div><div class='del'>-#define I40IW_QUERY_FPM_Q1BLOCKSIZE_MASK        \</div><div class='del'>-	(0xffffffffULL &lt;&lt; I40IW_QUERY_FPM_Q1BLOCKSIZE_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_QUERY_FPM_HTMULTIPLIER_SHIFT 16</div><div class='del'>-#define I40IW_QUERY_FPM_HTMULTIPLIER_MASK       \</div><div class='del'>-	(0xfUL &lt;&lt; I40IW_QUERY_FPM_HTMULTIPLIER_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_QUERY_FPM_TIMERBUCKET_SHIFT 32</div><div class='del'>-#define I40IW_QUERY_FPM_TIMERBUCKET_MASK        \</div><div class='del'>-	(0xffFFULL &lt;&lt; I40IW_QUERY_FPM_TIMERBUCKET_SHIFT)</div><div class='del'>-</div><div class='del'>-/* Static HMC pages allocated buf */</div><div class='del'>-#define I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID_SHIFT 0</div><div class='del'>-#define I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID_MASK        \</div><div class='del'>-	(0x3fUL &lt;&lt; I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40IW_HW_PAGE_SIZE	4096</div><div class='del'>-#define I40IW_DONE_COUNT	1000</div><div class='del'>-#define I40IW_SLEEP_COUNT	10</div><div class='del'>-</div><div class='del'>-enum {</div><div class='del'>-	I40IW_QUEUES_ALIGNMENT_MASK =		(128 - 1),</div><div class='del'>-	I40IW_AEQ_ALIGNMENT_MASK =		(256 - 1),</div><div class='del'>-	I40IW_Q2_ALIGNMENT_MASK =		(256 - 1),</div><div class='del'>-	I40IW_CEQ_ALIGNMENT_MASK =		(256 - 1),</div><div class='del'>-	I40IW_CQ0_ALIGNMENT_MASK =		(256 - 1),</div><div class='del'>-	I40IW_HOST_CTX_ALIGNMENT_MASK =		(4 - 1),</div><div class='del'>-	I40IW_SHADOWAREA_MASK =			(128 - 1),</div><div class='del'>-	I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK =	(4 - 1),</div><div class='del'>-	I40IW_FPM_COMMIT_BUF_ALIGNMENT_MASK =	(4 - 1)</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_alignment {</div><div class='del'>-	I40IW_CQP_ALIGNMENT =		0x200,</div><div class='del'>-	I40IW_AEQ_ALIGNMENT =		0x100,</div><div class='del'>-	I40IW_CEQ_ALIGNMENT =		0x100,</div><div class='del'>-	I40IW_CQ0_ALIGNMENT =		0x100,</div><div class='del'>-	I40IW_SD_BUF_ALIGNMENT =	0x80</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define I40IW_WQE_SIZE_64	64</div><div class='del'>-</div><div class='del'>-#define I40IW_QP_WQE_MIN_SIZE	32</div><div class='del'>-#define I40IW_QP_WQE_MAX_SIZE	128</div><div class='del'>-</div><div class='del'>-#define I40IW_UPDATE_SD_BUF_SIZE 128</div><div class='del'>-</div><div class='del'>-#define I40IW_CQE_QTYPE_RQ 0</div><div class='del'>-#define I40IW_CQE_QTYPE_SQ 1</div><div class='del'>-</div><div class='del'>-#define I40IW_RING_INIT(_ring, _size) \</div><div class='del'>-	{ \</div><div class='del'>-		(_ring).head = 0; \</div><div class='del'>-		(_ring).tail = 0; \</div><div class='del'>-		(_ring).size = (_size); \</div><div class='del'>-	}</div><div class='del'>-#define I40IW_RING_GETSIZE(_ring) ((_ring).size)</div><div class='del'>-#define I40IW_RING_GETCURRENT_HEAD(_ring) ((_ring).head)</div><div class='del'>-#define I40IW_RING_GETCURRENT_TAIL(_ring) ((_ring).tail)</div><div class='del'>-</div><div class='del'>-#define I40IW_RING_MOVE_HEAD(_ring, _retcode) \</div><div class='del'>-	{ \</div><div class='del'>-		register u32 size; \</div><div class='del'>-		size = (_ring).size;  \</div><div class='del'>-		if (!I40IW_RING_FULL_ERR(_ring)) { \</div><div class='del'>-			(_ring).head = ((_ring).head + 1) % size; \</div><div class='del'>-			(_retcode) = 0; \</div><div class='del'>-		} else { \</div><div class='del'>-			(_retcode) = I40IW_ERR_RING_FULL; \</div><div class='del'>-		} \</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-#define I40IW_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \</div><div class='del'>-	{ \</div><div class='del'>-		register u32 size; \</div><div class='del'>-		size = (_ring).size; \</div><div class='del'>-		if ((I40IW_RING_WORK_AVAILABLE(_ring) + (_count)) &lt; size) { \</div><div class='del'>-			(_ring).head = ((_ring).head + (_count)) % size; \</div><div class='del'>-			(_retcode) = 0; \</div><div class='del'>-		} else { \</div><div class='del'>-			(_retcode) = I40IW_ERR_RING_FULL; \</div><div class='del'>-		} \</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-#define I40IW_RING_MOVE_TAIL(_ring) \</div><div class='del'>-	(_ring).tail = ((_ring).tail + 1) % (_ring).size</div><div class='del'>-</div><div class='del'>-#define I40IW_RING_MOVE_HEAD_NOCHECK(_ring) \</div><div class='del'>-	(_ring).head = ((_ring).head + 1) % (_ring).size</div><div class='del'>-</div><div class='del'>-#define I40IW_RING_MOVE_TAIL_BY_COUNT(_ring, _count) \</div><div class='del'>-	(_ring).tail = ((_ring).tail + (_count)) % (_ring).size</div><div class='del'>-</div><div class='del'>-#define I40IW_RING_SET_TAIL(_ring, _pos) \</div><div class='del'>-	(_ring).tail = (_pos) % (_ring).size</div><div class='del'>-</div><div class='del'>-#define I40IW_RING_FULL_ERR(_ring) \</div><div class='del'>-	( \</div><div class='del'>-		(I40IW_RING_WORK_AVAILABLE(_ring) == ((_ring).size - 1))  \</div><div class='del'>-	)</div><div class='del'>-</div><div class='del'>-#define I40IW_ERR_RING_FULL2(_ring) \</div><div class='del'>-	( \</div><div class='del'>-		(I40IW_RING_WORK_AVAILABLE(_ring) == ((_ring).size - 2))  \</div><div class='del'>-	)</div><div class='del'>-</div><div class='del'>-#define I40IW_ERR_RING_FULL3(_ring) \</div><div class='del'>-	( \</div><div class='del'>-		(I40IW_RING_WORK_AVAILABLE(_ring) == ((_ring).size - 3))  \</div><div class='del'>-	)</div><div class='del'>-</div><div class='del'>-#define I40IW_RING_MORE_WORK(_ring) \</div><div class='del'>-	( \</div><div class='del'>-		(I40IW_RING_WORK_AVAILABLE(_ring) != 0) \</div><div class='del'>-	)</div><div class='del'>-</div><div class='del'>-#define I40IW_RING_WORK_AVAILABLE(_ring) \</div><div class='del'>-	( \</div><div class='del'>-		(((_ring).head + (_ring).size - (_ring).tail) % (_ring).size) \</div><div class='del'>-	)</div><div class='del'>-</div><div class='del'>-#define I40IW_RING_GET_WQES_AVAILABLE(_ring) \</div><div class='del'>-	( \</div><div class='del'>-		((_ring).size - I40IW_RING_WORK_AVAILABLE(_ring) - 1) \</div><div class='del'>-	)</div><div class='del'>-</div><div class='del'>-#define I40IW_ATOMIC_RING_MOVE_HEAD(_ring, index, _retcode) \</div><div class='del'>-	{ \</div><div class='del'>-		index = I40IW_RING_GETCURRENT_HEAD(_ring); \</div><div class='del'>-		I40IW_RING_MOVE_HEAD(_ring, _retcode); \</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-/* Async Events codes */</div><div class='del'>-#define I40IW_AE_AMP_UNALLOCATED_STAG                                   0x0102</div><div class='del'>-#define I40IW_AE_AMP_INVALID_STAG                                       0x0103</div><div class='del'>-#define I40IW_AE_AMP_BAD_QP                                             0x0104</div><div class='del'>-#define I40IW_AE_AMP_BAD_PD                                             0x0105</div><div class='del'>-#define I40IW_AE_AMP_BAD_STAG_KEY                                       0x0106</div><div class='del'>-#define I40IW_AE_AMP_BAD_STAG_INDEX                                     0x0107</div><div class='del'>-#define I40IW_AE_AMP_BOUNDS_VIOLATION                                   0x0108</div><div class='del'>-#define I40IW_AE_AMP_RIGHTS_VIOLATION                                   0x0109</div><div class='del'>-#define I40IW_AE_AMP_TO_WRAP                                            0x010a</div><div class='del'>-#define I40IW_AE_AMP_FASTREG_SHARED                                     0x010b</div><div class='del'>-#define I40IW_AE_AMP_FASTREG_VALID_STAG                                 0x010c</div><div class='del'>-#define I40IW_AE_AMP_FASTREG_MW_STAG                                    0x010d</div><div class='del'>-#define I40IW_AE_AMP_FASTREG_INVALID_RIGHTS                             0x010e</div><div class='del'>-#define I40IW_AE_AMP_FASTREG_PBL_TABLE_OVERFLOW                         0x010f</div><div class='del'>-#define I40IW_AE_AMP_FASTREG_INVALID_LENGTH                             0x0110</div><div class='del'>-#define I40IW_AE_AMP_INVALIDATE_SHARED                                  0x0111</div><div class='del'>-#define I40IW_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS                 0x0112</div><div class='del'>-#define I40IW_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS                   0x0113</div><div class='del'>-#define I40IW_AE_AMP_MWBIND_VALID_STAG                                  0x0114</div><div class='del'>-#define I40IW_AE_AMP_MWBIND_OF_MR_STAG                                  0x0115</div><div class='del'>-#define I40IW_AE_AMP_MWBIND_TO_ZERO_BASED_STAG                          0x0116</div><div class='del'>-#define I40IW_AE_AMP_MWBIND_TO_MW_STAG                                  0x0117</div><div class='del'>-#define I40IW_AE_AMP_MWBIND_INVALID_RIGHTS                              0x0118</div><div class='del'>-#define I40IW_AE_AMP_MWBIND_INVALID_BOUNDS                              0x0119</div><div class='del'>-#define I40IW_AE_AMP_MWBIND_TO_INVALID_PARENT                           0x011a</div><div class='del'>-#define I40IW_AE_AMP_MWBIND_BIND_DISABLED                               0x011b</div><div class='del'>-#define I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG                                0x0132</div><div class='del'>-#define I40IW_AE_UDA_XMIT_DGRAM_TOO_SHORT                               0x0134</div><div class='del'>-#define I40IW_AE_BAD_CLOSE                                              0x0201</div><div class='del'>-#define I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE                                0x0202</div><div class='del'>-#define I40IW_AE_CQ_OPERATION_ERROR                                     0x0203</div><div class='del'>-#define I40IW_AE_PRIV_OPERATION_DENIED                                  0x011c</div><div class='del'>-#define I40IW_AE_RDMA_READ_WHILE_ORD_ZERO                               0x0205</div><div class='del'>-#define I40IW_AE_STAG_ZERO_INVALID                                      0x0206</div><div class='del'>-#define I40IW_AE_IB_RREQ_AND_Q1_FULL                                    0x0207</div><div class='del'>-#define I40IW_AE_WQE_UNEXPECTED_OPCODE                                  0x020a</div><div class='del'>-#define I40IW_AE_WQE_INVALID_PARAMETER                                  0x020b</div><div class='del'>-#define I40IW_AE_WQE_LSMM_TOO_LONG                                      0x0220</div><div class='del'>-#define I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN                             0x0301</div><div class='del'>-#define I40IW_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER      0x0303</div><div class='del'>-#define I40IW_AE_DDP_UBE_INVALID_DDP_VERSION                            0x0304</div><div class='del'>-#define I40IW_AE_DDP_UBE_INVALID_MO                                     0x0305</div><div class='del'>-#define I40IW_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE                0x0306</div><div class='del'>-#define I40IW_AE_DDP_UBE_INVALID_QN                                     0x0307</div><div class='del'>-#define I40IW_AE_DDP_NO_L_BIT                                           0x0308</div><div class='del'>-#define I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION                        0x0311</div><div class='del'>-#define I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE                            0x0312</div><div class='del'>-#define I40IW_AE_ROE_INVALID_RDMA_READ_REQUEST                          0x0313</div><div class='del'>-#define I40IW_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP                    0x0314</div><div class='del'>-#define I40IW_AE_INVALID_ARP_ENTRY                                      0x0401</div><div class='del'>-#define I40IW_AE_INVALID_TCP_OPTION_RCVD                                0x0402</div><div class='del'>-#define I40IW_AE_STALE_ARP_ENTRY                                        0x0403</div><div class='del'>-#define I40IW_AE_INVALID_MAC_ENTRY                                      0x0405</div><div class='del'>-#define I40IW_AE_LLP_CLOSE_COMPLETE                                     0x0501</div><div class='del'>-#define I40IW_AE_LLP_CONNECTION_RESET                                   0x0502</div><div class='del'>-#define I40IW_AE_LLP_FIN_RECEIVED                                       0x0503</div><div class='del'>-#define I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR                             0x0505</div><div class='del'>-#define I40IW_AE_LLP_SEGMENT_TOO_LARGE                                  0x0506</div><div class='del'>-#define I40IW_AE_LLP_SEGMENT_TOO_SMALL                                  0x0507</div><div class='del'>-#define I40IW_AE_LLP_SYN_RECEIVED                                       0x0508</div><div class='del'>-#define I40IW_AE_LLP_TERMINATE_RECEIVED                                 0x0509</div><div class='del'>-#define I40IW_AE_LLP_TOO_MANY_RETRIES                                   0x050a</div><div class='del'>-#define I40IW_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES                         0x050b</div><div class='del'>-#define I40IW_AE_LLP_DOUBT_REACHABILITY                                 0x050c</div><div class='del'>-#define I40IW_AE_LLP_RX_VLAN_MISMATCH                                   0x050d</div><div class='del'>-#define I40IW_AE_RESOURCE_EXHAUSTION                                    0x0520</div><div class='del'>-#define I40IW_AE_RESET_SENT                                             0x0601</div><div class='del'>-#define I40IW_AE_TERMINATE_SENT                                         0x0602</div><div class='del'>-#define I40IW_AE_RESET_NOT_SENT                                         0x0603</div><div class='del'>-#define I40IW_AE_LCE_QP_CATASTROPHIC                                    0x0700</div><div class='del'>-#define I40IW_AE_LCE_FUNCTION_CATASTROPHIC                              0x0701</div><div class='del'>-#define I40IW_AE_LCE_CQ_CATASTROPHIC                                    0x0702</div><div class='del'>-#define I40IW_AE_QP_SUSPEND_COMPLETE                                    0x0900</div><div class='del'>-</div><div class='del'>-#define OP_DELETE_LOCAL_MAC_IPADDR_ENTRY        1</div><div class='del'>-#define OP_CEQ_DESTROY                          2</div><div class='del'>-#define OP_AEQ_DESTROY                          3</div><div class='del'>-#define OP_DELETE_ARP_CACHE_ENTRY               4</div><div class='del'>-#define OP_MANAGE_APBVT_ENTRY                   5</div><div class='del'>-#define OP_CEQ_CREATE                           6</div><div class='del'>-#define OP_AEQ_CREATE                           7</div><div class='del'>-#define OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY         8</div><div class='del'>-#define OP_ADD_LOCAL_MAC_IPADDR_ENTRY           9</div><div class='del'>-#define OP_MANAGE_QHASH_TABLE_ENTRY             10</div><div class='del'>-#define OP_QP_MODIFY                            11</div><div class='del'>-#define OP_QP_UPLOAD_CONTEXT                    12</div><div class='del'>-#define OP_CQ_CREATE                            13</div><div class='del'>-#define OP_CQ_DESTROY                           14</div><div class='del'>-#define OP_QP_CREATE                            15</div><div class='del'>-#define OP_QP_DESTROY                           16</div><div class='del'>-#define OP_ALLOC_STAG                           17</div><div class='del'>-#define OP_MR_REG_NON_SHARED                    18</div><div class='del'>-#define OP_DEALLOC_STAG                         19</div><div class='del'>-#define OP_MW_ALLOC                             20</div><div class='del'>-#define OP_QP_FLUSH_WQES                        21</div><div class='del'>-#define OP_ADD_ARP_CACHE_ENTRY                  22</div><div class='del'>-#define OP_MANAGE_PUSH_PAGE                     23</div><div class='del'>-#define OP_UPDATE_PE_SDS                        24</div><div class='del'>-#define OP_MANAGE_HMC_PM_FUNC_TABLE             25</div><div class='del'>-#define OP_SUSPEND                              26</div><div class='del'>-#define OP_RESUME                               27</div><div class='del'>-#define OP_MANAGE_VF_PBLE_BP                    28</div><div class='del'>-#define OP_QUERY_FPM_VALUES                     29</div><div class='del'>-#define OP_COMMIT_FPM_VALUES                    30</div><div class='del'>-#define OP_REQUESTED_COMMANDS                   31</div><div class='del'>-#define OP_COMPLETED_COMMANDS                   32</div><div class='del'>-#define OP_GEN_AE                               33</div><div class='del'>-#define OP_SIZE_CQP_STAT_ARRAY                  34</div><div class='del'>-</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_hmc.c b/drivers/infiniband/hw/i40iw/i40iw_hmc.c<br/>deleted file mode 100644<br/>index 5484cbf55f0f..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_hmc.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_hmc.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,821 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#include "i40iw_osdep.h"</div><div class='del'>-#include "i40iw_register.h"</div><div class='del'>-#include "i40iw_status.h"</div><div class='del'>-#include "i40iw_hmc.h"</div><div class='del'>-#include "i40iw_d.h"</div><div class='del'>-#include "i40iw_type.h"</div><div class='del'>-#include "i40iw_p.h"</div><div class='del'>-#include "i40iw_vf.h"</div><div class='del'>-#include "i40iw_virtchnl.h"</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_find_sd_index_limit - finds segment descriptor index limit</div><div class='del'>- * @hmc_info: pointer to the HMC configuration information structure</div><div class='del'>- * @type: type of HMC resources we're searching</div><div class='del'>- * @index: starting index for the object</div><div class='del'>- * @cnt: number of objects we're trying to create</div><div class='del'>- * @sd_idx: pointer to return index of the segment descriptor in question</div><div class='del'>- * @sd_limit: pointer to return the maximum number of segment descriptors</div><div class='del'>- *</div><div class='del'>- * This function calculates the segment descriptor index and index limit</div><div class='del'>- * for the resource defined by i40iw_hmc_rsrc_type.</div><div class='del'>- */</div><div class='del'>-</div><div class='del'>-static inline void i40iw_find_sd_index_limit(struct i40iw_hmc_info *hmc_info,</div><div class='del'>-					     u32 type,</div><div class='del'>-					     u32 idx,</div><div class='del'>-					     u32 cnt,</div><div class='del'>-					     u32 *sd_idx,</div><div class='del'>-					     u32 *sd_limit)</div><div class='del'>-{</div><div class='del'>-	u64 fpm_addr, fpm_limit;</div><div class='del'>-</div><div class='del'>-	fpm_addr = hmc_info-&gt;hmc_obj[(type)].base +</div><div class='del'>-			hmc_info-&gt;hmc_obj[type].size * idx;</div><div class='del'>-	fpm_limit = fpm_addr + hmc_info-&gt;hmc_obj[type].size * cnt;</div><div class='del'>-	*sd_idx = (u32)(fpm_addr / I40IW_HMC_DIRECT_BP_SIZE);</div><div class='del'>-	*sd_limit = (u32)((fpm_limit - 1) / I40IW_HMC_DIRECT_BP_SIZE);</div><div class='del'>-	*sd_limit += 1;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_find_pd_index_limit - finds page descriptor index limit</div><div class='del'>- * @hmc_info: pointer to the HMC configuration information struct</div><div class='del'>- * @type: HMC resource type we're examining</div><div class='del'>- * @idx: starting index for the object</div><div class='del'>- * @cnt: number of objects we're trying to create</div><div class='del'>- * @pd_index: pointer to return page descriptor index</div><div class='del'>- * @pd_limit: pointer to return page descriptor index limit</div><div class='del'>- *</div><div class='del'>- * Calculates the page descriptor index and index limit for the resource</div><div class='del'>- * defined by i40iw_hmc_rsrc_type.</div><div class='del'>- */</div><div class='del'>-</div><div class='del'>-static inline void i40iw_find_pd_index_limit(struct i40iw_hmc_info *hmc_info,</div><div class='del'>-					     u32 type,</div><div class='del'>-					     u32 idx,</div><div class='del'>-					     u32 cnt,</div><div class='del'>-					     u32 *pd_idx,</div><div class='del'>-					     u32 *pd_limit)</div><div class='del'>-{</div><div class='del'>-	u64 fpm_adr, fpm_limit;</div><div class='del'>-</div><div class='del'>-	fpm_adr = hmc_info-&gt;hmc_obj[type].base +</div><div class='del'>-			hmc_info-&gt;hmc_obj[type].size * idx;</div><div class='del'>-	fpm_limit = fpm_adr + (hmc_info)-&gt;hmc_obj[(type)].size * (cnt);</div><div class='del'>-	*(pd_idx) = (u32)(fpm_adr / I40IW_HMC_PAGED_BP_SIZE);</div><div class='del'>-	*(pd_limit) = (u32)((fpm_limit - 1) / I40IW_HMC_PAGED_BP_SIZE);</div><div class='del'>-	*(pd_limit) += 1;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_set_sd_entry - setup entry for sd programming</div><div class='del'>- * @pa: physical addr</div><div class='del'>- * @idx: sd index</div><div class='del'>- * @type: paged or direct sd</div><div class='del'>- * @entry: sd entry ptr</div><div class='del'>- */</div><div class='del'>-static inline void i40iw_set_sd_entry(u64 pa,</div><div class='del'>-				      u32 idx,</div><div class='del'>-				      enum i40iw_sd_entry_type type,</div><div class='del'>-				      struct update_sd_entry *entry)</div><div class='del'>-{</div><div class='del'>-	entry-&gt;data = pa | (I40IW_HMC_MAX_BP_COUNT &lt;&lt; I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) |</div><div class='del'>-			(((type == I40IW_SD_TYPE_PAGED) ? 0 : 1) &lt;&lt;</div><div class='del'>-				I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT) |</div><div class='del'>-			(1 &lt;&lt; I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT);</div><div class='del'>-	entry-&gt;cmd = (idx | (1 &lt;&lt; I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | (1 &lt;&lt; 15));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_clr_sd_entry - setup entry for sd clear</div><div class='del'>- * @idx: sd index</div><div class='del'>- * @type: paged or direct sd</div><div class='del'>- * @entry: sd entry ptr</div><div class='del'>- */</div><div class='del'>-static inline void i40iw_clr_sd_entry(u32 idx, enum i40iw_sd_entry_type type,</div><div class='del'>-				      struct update_sd_entry *entry)</div><div class='del'>-{</div><div class='del'>-	entry-&gt;data = (I40IW_HMC_MAX_BP_COUNT &lt;&lt;</div><div class='del'>-			I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) |</div><div class='del'>-			(((type == I40IW_SD_TYPE_PAGED) ? 0 : 1) &lt;&lt;</div><div class='del'>-				I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT);</div><div class='del'>-	entry-&gt;cmd = (idx | (1 &lt;&lt; I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | (1 &lt;&lt; 15));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hmc_sd_one - setup 1 sd entry for cqp</div><div class='del'>- * @dev: pointer to the device structure</div><div class='del'>- * @hmc_fn_id: hmc's function id</div><div class='del'>- * @pa: physical addr</div><div class='del'>- * @sd_idx: sd index</div><div class='del'>- * @type: paged or direct sd</div><div class='del'>- * @setsd: flag to set or clear sd</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_hmc_sd_one(struct i40iw_sc_dev *dev,</div><div class='del'>-					u8 hmc_fn_id,</div><div class='del'>-					u64 pa, u32 sd_idx,</div><div class='del'>-					enum i40iw_sd_entry_type type,</div><div class='del'>-					bool setsd)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_update_sds_info sdinfo;</div><div class='del'>-</div><div class='del'>-	sdinfo.cnt = 1;</div><div class='del'>-	sdinfo.hmc_fn_id = hmc_fn_id;</div><div class='del'>-	if (setsd)</div><div class='del'>-		i40iw_set_sd_entry(pa, sd_idx, type, sdinfo.entry);</div><div class='del'>-	else</div><div class='del'>-		i40iw_clr_sd_entry(sd_idx, type, sdinfo.entry);</div><div class='del'>-</div><div class='del'>-	return dev-&gt;cqp-&gt;process_cqp_sds(dev, &amp;sdinfo);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hmc_sd_grp - setup group od sd entries for cqp</div><div class='del'>- * @dev: pointer to the device structure</div><div class='del'>- * @hmc_info: pointer to the HMC configuration information struct</div><div class='del'>- * @sd_index: sd index</div><div class='del'>- * @sd_cnt: number of sd entries</div><div class='del'>- * @setsd: flag to set or clear sd</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_hmc_sd_grp(struct i40iw_sc_dev *dev,</div><div class='del'>-					       struct i40iw_hmc_info *hmc_info,</div><div class='del'>-					       u32 sd_index,</div><div class='del'>-					       u32 sd_cnt,</div><div class='del'>-					       bool setsd)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_hmc_sd_entry *sd_entry;</div><div class='del'>-	struct i40iw_update_sds_info sdinfo;</div><div class='del'>-	u64 pa;</div><div class='del'>-	u32 i;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-</div><div class='del'>-	memset(&amp;sdinfo, 0, sizeof(sdinfo));</div><div class='del'>-	sdinfo.hmc_fn_id = hmc_info-&gt;hmc_fn_id;</div><div class='del'>-	for (i = sd_index; i &lt; sd_index + sd_cnt; i++) {</div><div class='del'>-		sd_entry = &amp;hmc_info-&gt;sd_table.sd_entry[i];</div><div class='del'>-		if (!sd_entry ||</div><div class='del'>-		    (!sd_entry-&gt;valid &amp;&amp; setsd) ||</div><div class='del'>-		    (sd_entry-&gt;valid &amp;&amp; !setsd))</div><div class='del'>-			continue;</div><div class='del'>-		if (setsd) {</div><div class='del'>-			pa = (sd_entry-&gt;entry_type == I40IW_SD_TYPE_PAGED) ?</div><div class='del'>-			    sd_entry-&gt;u.pd_table.pd_page_addr.pa :</div><div class='del'>-			    sd_entry-&gt;u.bp.addr.pa;</div><div class='del'>-			i40iw_set_sd_entry(pa, i, sd_entry-&gt;entry_type,</div><div class='del'>-					   &amp;sdinfo.entry[sdinfo.cnt]);</div><div class='del'>-		} else {</div><div class='del'>-			i40iw_clr_sd_entry(i, sd_entry-&gt;entry_type,</div><div class='del'>-					   &amp;sdinfo.entry[sdinfo.cnt]);</div><div class='del'>-		}</div><div class='del'>-		sdinfo.cnt++;</div><div class='del'>-		if (sdinfo.cnt == I40IW_MAX_SD_ENTRIES) {</div><div class='del'>-			ret_code = dev-&gt;cqp-&gt;process_cqp_sds(dev, &amp;sdinfo);</div><div class='del'>-			if (ret_code) {</div><div class='del'>-				i40iw_debug(dev, I40IW_DEBUG_HMC,</div><div class='del'>-					    "i40iw_hmc_sd_grp: sd_programming failed err=%d\n",</div><div class='del'>-					    ret_code);</div><div class='del'>-				return ret_code;</div><div class='del'>-			}</div><div class='del'>-			sdinfo.cnt = 0;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	if (sdinfo.cnt)</div><div class='del'>-		ret_code = dev-&gt;cqp-&gt;process_cqp_sds(dev, &amp;sdinfo);</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vfdev_from_fpm - return vf dev ptr for hmc function id</div><div class='del'>- * @dev: pointer to the device structure</div><div class='del'>- * @hmc_fn_id: hmc's function id</div><div class='del'>- */</div><div class='del'>-struct i40iw_vfdev *i40iw_vfdev_from_fpm(struct i40iw_sc_dev *dev, u8 hmc_fn_id)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_vfdev *vf_dev = NULL;</div><div class='del'>-	u16 idx;</div><div class='del'>-</div><div class='del'>-	for (idx = 0; idx &lt; I40IW_MAX_PE_ENABLED_VF_COUNT; idx++) {</div><div class='del'>-		if (dev-&gt;vf_dev[idx] &amp;&amp;</div><div class='del'>-		    ((u8)dev-&gt;vf_dev[idx]-&gt;pmf_index == hmc_fn_id)) {</div><div class='del'>-			vf_dev = dev-&gt;vf_dev[idx];</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	return vf_dev;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vf_hmcinfo_from_fpm - get ptr to hmc for func_id</div><div class='del'>- * @dev: pointer to the device structure</div><div class='del'>- * @hmc_fn_id: hmc's function id</div><div class='del'>- */</div><div class='del'>-struct i40iw_hmc_info *i40iw_vf_hmcinfo_from_fpm(struct i40iw_sc_dev *dev,</div><div class='del'>-						 u8 hmc_fn_id)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_hmc_info *hmc_info = NULL;</div><div class='del'>-	u16 idx;</div><div class='del'>-</div><div class='del'>-	for (idx = 0; idx &lt; I40IW_MAX_PE_ENABLED_VF_COUNT; idx++) {</div><div class='del'>-		if (dev-&gt;vf_dev[idx] &amp;&amp;</div><div class='del'>-		    ((u8)dev-&gt;vf_dev[idx]-&gt;pmf_index == hmc_fn_id)) {</div><div class='del'>-			hmc_info = &amp;dev-&gt;vf_dev[idx]-&gt;hmc_info;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	return hmc_info;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hmc_finish_add_sd_reg - program sd entries for objects</div><div class='del'>- * @dev: pointer to the device structure</div><div class='del'>- * @info: create obj info</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_hmc_finish_add_sd_reg(struct i40iw_sc_dev *dev,</div><div class='del'>-							  struct i40iw_hmc_create_obj_info *info)</div><div class='del'>-{</div><div class='del'>-	if (info-&gt;start_idx &gt;= info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt)</div><div class='del'>-		return I40IW_ERR_INVALID_HMC_OBJ_INDEX;</div><div class='del'>-</div><div class='del'>-	if ((info-&gt;start_idx + info-&gt;count) &gt;</div><div class='del'>-			info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt)</div><div class='del'>-		return I40IW_ERR_INVALID_HMC_OBJ_COUNT;</div><div class='del'>-</div><div class='del'>-	if (!info-&gt;add_sd_cnt)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	return i40iw_hmc_sd_grp(dev, info-&gt;hmc_info,</div><div class='del'>-				info-&gt;hmc_info-&gt;sd_indexes[0],</div><div class='del'>-				info-&gt;add_sd_cnt, true);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_create_iw_hmc_obj - allocate backing store for hmc objects</div><div class='del'>- * @dev: pointer to the device structure</div><div class='del'>- * @info: pointer to i40iw_hmc_iw_create_obj_info struct</div><div class='del'>- *</div><div class='del'>- * This will allocate memory for PDs and backing pages and populate</div><div class='del'>- * the sd and pd entries.</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_sc_create_hmc_obj(struct i40iw_sc_dev *dev,</div><div class='del'>-					       struct i40iw_hmc_create_obj_info *info)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_hmc_sd_entry *sd_entry;</div><div class='del'>-	u32 sd_idx, sd_lmt;</div><div class='del'>-	u32 pd_idx = 0, pd_lmt = 0;</div><div class='del'>-	u32 pd_idx1 = 0, pd_lmt1 = 0;</div><div class='del'>-	u32 i, j;</div><div class='del'>-	bool pd_error = false;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;start_idx &gt;= info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt)</div><div class='del'>-		return I40IW_ERR_INVALID_HMC_OBJ_INDEX;</div><div class='del'>-</div><div class='del'>-	if ((info-&gt;start_idx + info-&gt;count) &gt;</div><div class='del'>-	    info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_HMC,</div><div class='del'>-			    "%s: error type %u, start = %u, req cnt %u, cnt = %u\n",</div><div class='del'>-			    __func__, info-&gt;rsrc_type, info-&gt;start_idx, info-&gt;count,</div><div class='del'>-			    info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt);</div><div class='del'>-		return I40IW_ERR_INVALID_HMC_OBJ_COUNT;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!dev-&gt;is_pf)</div><div class='del'>-		return i40iw_vchnl_vf_add_hmc_objs(dev, info-&gt;rsrc_type, 0, info-&gt;count);</div><div class='del'>-</div><div class='del'>-	i40iw_find_sd_index_limit(info-&gt;hmc_info, info-&gt;rsrc_type,</div><div class='del'>-				  info-&gt;start_idx, info-&gt;count,</div><div class='del'>-				  &amp;sd_idx, &amp;sd_lmt);</div><div class='del'>-	if (sd_idx &gt;= info-&gt;hmc_info-&gt;sd_table.sd_cnt ||</div><div class='del'>-	    sd_lmt &gt; info-&gt;hmc_info-&gt;sd_table.sd_cnt) {</div><div class='del'>-		return I40IW_ERR_INVALID_SD_INDEX;</div><div class='del'>-	}</div><div class='del'>-	i40iw_find_pd_index_limit(info-&gt;hmc_info, info-&gt;rsrc_type,</div><div class='del'>-				  info-&gt;start_idx, info-&gt;count, &amp;pd_idx, &amp;pd_lmt);</div><div class='del'>-</div><div class='del'>-	for (j = sd_idx; j &lt; sd_lmt; j++) {</div><div class='del'>-		ret_code = i40iw_add_sd_table_entry(dev-&gt;hw, info-&gt;hmc_info,</div><div class='del'>-						    j,</div><div class='del'>-						    info-&gt;entry_type,</div><div class='del'>-						    I40IW_HMC_DIRECT_BP_SIZE);</div><div class='del'>-		if (ret_code)</div><div class='del'>-			goto exit_sd_error;</div><div class='del'>-		sd_entry = &amp;info-&gt;hmc_info-&gt;sd_table.sd_entry[j];</div><div class='del'>-</div><div class='del'>-		if ((sd_entry-&gt;entry_type == I40IW_SD_TYPE_PAGED) &amp;&amp;</div><div class='del'>-		    ((dev-&gt;hmc_info == info-&gt;hmc_info) &amp;&amp;</div><div class='del'>-		     (info-&gt;rsrc_type != I40IW_HMC_IW_PBLE))) {</div><div class='del'>-			pd_idx1 = max(pd_idx, (j * I40IW_HMC_MAX_BP_COUNT));</div><div class='del'>-			pd_lmt1 = min(pd_lmt,</div><div class='del'>-				      (j + 1) * I40IW_HMC_MAX_BP_COUNT);</div><div class='del'>-			for (i = pd_idx1; i &lt; pd_lmt1; i++) {</div><div class='del'>-				/* update the pd table entry */</div><div class='del'>-				ret_code = i40iw_add_pd_table_entry(dev-&gt;hw, info-&gt;hmc_info,</div><div class='del'>-								    i, NULL);</div><div class='del'>-				if (ret_code) {</div><div class='del'>-					pd_error = true;</div><div class='del'>-					break;</div><div class='del'>-				}</div><div class='del'>-			}</div><div class='del'>-			if (pd_error) {</div><div class='del'>-				while (i &amp;&amp; (i &gt; pd_idx1)) {</div><div class='del'>-					i40iw_remove_pd_bp(dev-&gt;hw, info-&gt;hmc_info, (i - 1),</div><div class='del'>-							   info-&gt;is_pf);</div><div class='del'>-					i--;</div><div class='del'>-				}</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-		if (sd_entry-&gt;valid)</div><div class='del'>-			continue;</div><div class='del'>-</div><div class='del'>-		info-&gt;hmc_info-&gt;sd_indexes[info-&gt;add_sd_cnt] = (u16)j;</div><div class='del'>-		info-&gt;add_sd_cnt++;</div><div class='del'>-		sd_entry-&gt;valid = true;</div><div class='del'>-	}</div><div class='del'>-	return i40iw_hmc_finish_add_sd_reg(dev, info);</div><div class='del'>-</div><div class='del'>-exit_sd_error:</div><div class='del'>-	while (j &amp;&amp; (j &gt; sd_idx)) {</div><div class='del'>-		sd_entry = &amp;info-&gt;hmc_info-&gt;sd_table.sd_entry[j - 1];</div><div class='del'>-		switch (sd_entry-&gt;entry_type) {</div><div class='del'>-		case I40IW_SD_TYPE_PAGED:</div><div class='del'>-			pd_idx1 = max(pd_idx,</div><div class='del'>-				      (j - 1) * I40IW_HMC_MAX_BP_COUNT);</div><div class='del'>-			pd_lmt1 = min(pd_lmt, (j * I40IW_HMC_MAX_BP_COUNT));</div><div class='del'>-			for (i = pd_idx1; i &lt; pd_lmt1; i++)</div><div class='del'>-				i40iw_prep_remove_pd_page(info-&gt;hmc_info, i);</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_SD_TYPE_DIRECT:</div><div class='del'>-			i40iw_prep_remove_pd_page(info-&gt;hmc_info, (j - 1));</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			ret_code = I40IW_ERR_INVALID_SD_TYPE;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		j--;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_finish_del_sd_reg - delete sd entries for objects</div><div class='del'>- * @dev: pointer to the device structure</div><div class='del'>- * @info: dele obj info</div><div class='del'>- * @reset: true if called before reset</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_finish_del_sd_reg(struct i40iw_sc_dev *dev,</div><div class='del'>-						      struct i40iw_hmc_del_obj_info *info,</div><div class='del'>-						      bool reset)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_hmc_sd_entry *sd_entry;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-	u32 i, sd_idx;</div><div class='del'>-	struct i40iw_dma_mem *mem;</div><div class='del'>-</div><div class='del'>-	if (dev-&gt;is_pf &amp;&amp; !reset)</div><div class='del'>-		ret_code = i40iw_hmc_sd_grp(dev, info-&gt;hmc_info,</div><div class='del'>-					    info-&gt;hmc_info-&gt;sd_indexes[0],</div><div class='del'>-					    info-&gt;del_sd_cnt, false);</div><div class='del'>-</div><div class='del'>-	if (ret_code)</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: error cqp sd sd_grp\n", __func__);</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; info-&gt;del_sd_cnt; i++) {</div><div class='del'>-		sd_idx = info-&gt;hmc_info-&gt;sd_indexes[i];</div><div class='del'>-		sd_entry = &amp;info-&gt;hmc_info-&gt;sd_table.sd_entry[sd_idx];</div><div class='del'>-		if (!sd_entry)</div><div class='del'>-			continue;</div><div class='del'>-		mem = (sd_entry-&gt;entry_type == I40IW_SD_TYPE_PAGED) ?</div><div class='del'>-			&amp;sd_entry-&gt;u.pd_table.pd_page_addr :</div><div class='del'>-			&amp;sd_entry-&gt;u.bp.addr;</div><div class='del'>-</div><div class='del'>-		if (!mem || !mem-&gt;va)</div><div class='del'>-			i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: error cqp sd mem\n", __func__);</div><div class='del'>-		else</div><div class='del'>-			i40iw_free_dma_mem(dev-&gt;hw, mem);</div><div class='del'>-	}</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_del_iw_hmc_obj - remove pe hmc objects</div><div class='del'>- * @dev: pointer to the device structure</div><div class='del'>- * @info: pointer to i40iw_hmc_del_obj_info struct</div><div class='del'>- * @reset: true if called before reset</div><div class='del'>- *</div><div class='del'>- * This will de-populate the SDs and PDs.  It frees</div><div class='del'>- * the memory for PDS and backing storage.  After this function is returned,</div><div class='del'>- * caller should deallocate memory allocated previously for</div><div class='del'>- * book-keeping information about PDs and backing storage.</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_sc_del_hmc_obj(struct i40iw_sc_dev *dev,</div><div class='del'>-					    struct i40iw_hmc_del_obj_info *info,</div><div class='del'>-					    bool reset)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_hmc_pd_table *pd_table;</div><div class='del'>-	u32 sd_idx, sd_lmt;</div><div class='del'>-	u32 pd_idx, pd_lmt, rel_pd_idx;</div><div class='del'>-	u32 i, j;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;start_idx &gt;= info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_HMC,</div><div class='del'>-			    "%s: error start_idx[%04d]  &gt;= [type %04d].cnt[%04d]\n",</div><div class='del'>-			    __func__, info-&gt;start_idx, info-&gt;rsrc_type,</div><div class='del'>-			    info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt);</div><div class='del'>-		return I40IW_ERR_INVALID_HMC_OBJ_INDEX;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if ((info-&gt;start_idx + info-&gt;count) &gt;</div><div class='del'>-	    info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_HMC,</div><div class='del'>-			    "%s: error start_idx[%04d] + count %04d  &gt;= [type %04d].cnt[%04d]\n",</div><div class='del'>-			    __func__, info-&gt;start_idx, info-&gt;count,</div><div class='del'>-			    info-&gt;rsrc_type,</div><div class='del'>-			    info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt);</div><div class='del'>-		return I40IW_ERR_INVALID_HMC_OBJ_COUNT;</div><div class='del'>-	}</div><div class='del'>-	if (!dev-&gt;is_pf) {</div><div class='del'>-		ret_code = i40iw_vchnl_vf_del_hmc_obj(dev, info-&gt;rsrc_type, 0,</div><div class='del'>-						      info-&gt;count);</div><div class='del'>-		if (info-&gt;rsrc_type != I40IW_HMC_IW_PBLE)</div><div class='del'>-			return ret_code;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i40iw_find_pd_index_limit(info-&gt;hmc_info, info-&gt;rsrc_type,</div><div class='del'>-				  info-&gt;start_idx, info-&gt;count, &amp;pd_idx, &amp;pd_lmt);</div><div class='del'>-</div><div class='del'>-	for (j = pd_idx; j &lt; pd_lmt; j++) {</div><div class='del'>-		sd_idx = j / I40IW_HMC_PD_CNT_IN_SD;</div><div class='del'>-</div><div class='del'>-		if (info-&gt;hmc_info-&gt;sd_table.sd_entry[sd_idx].entry_type !=</div><div class='del'>-		    I40IW_SD_TYPE_PAGED)</div><div class='del'>-			continue;</div><div class='del'>-</div><div class='del'>-		rel_pd_idx = j % I40IW_HMC_PD_CNT_IN_SD;</div><div class='del'>-		pd_table = &amp;info-&gt;hmc_info-&gt;sd_table.sd_entry[sd_idx].u.pd_table;</div><div class='del'>-		if (pd_table-&gt;pd_entry[rel_pd_idx].valid) {</div><div class='del'>-			ret_code = i40iw_remove_pd_bp(dev-&gt;hw, info-&gt;hmc_info, j,</div><div class='del'>-						      info-&gt;is_pf);</div><div class='del'>-			if (ret_code) {</div><div class='del'>-				i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: error\n", __func__);</div><div class='del'>-				return ret_code;</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i40iw_find_sd_index_limit(info-&gt;hmc_info, info-&gt;rsrc_type,</div><div class='del'>-				  info-&gt;start_idx, info-&gt;count, &amp;sd_idx, &amp;sd_lmt);</div><div class='del'>-	if (sd_idx &gt;= info-&gt;hmc_info-&gt;sd_table.sd_cnt ||</div><div class='del'>-	    sd_lmt &gt; info-&gt;hmc_info-&gt;sd_table.sd_cnt) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: error invalid sd_idx\n", __func__);</div><div class='del'>-		return I40IW_ERR_INVALID_SD_INDEX;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	for (i = sd_idx; i &lt; sd_lmt; i++) {</div><div class='del'>-		if (!info-&gt;hmc_info-&gt;sd_table.sd_entry[i].valid)</div><div class='del'>-			continue;</div><div class='del'>-		switch (info-&gt;hmc_info-&gt;sd_table.sd_entry[i].entry_type) {</div><div class='del'>-		case I40IW_SD_TYPE_DIRECT:</div><div class='del'>-			ret_code = i40iw_prep_remove_sd_bp(info-&gt;hmc_info, i);</div><div class='del'>-			if (!ret_code) {</div><div class='del'>-				info-&gt;hmc_info-&gt;sd_indexes[info-&gt;del_sd_cnt] = (u16)i;</div><div class='del'>-				info-&gt;del_sd_cnt++;</div><div class='del'>-			}</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_SD_TYPE_PAGED:</div><div class='del'>-			ret_code = i40iw_prep_remove_pd_page(info-&gt;hmc_info, i);</div><div class='del'>-			if (!ret_code) {</div><div class='del'>-				info-&gt;hmc_info-&gt;sd_indexes[info-&gt;del_sd_cnt] = (u16)i;</div><div class='del'>-				info-&gt;del_sd_cnt++;</div><div class='del'>-			}</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	return i40iw_finish_del_sd_reg(dev, info, reset);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_add_sd_table_entry - Adds a segment descriptor to the table</div><div class='del'>- * @hw: pointer to our hw struct</div><div class='del'>- * @hmc_info: pointer to the HMC configuration information struct</div><div class='del'>- * @sd_index: segment descriptor index to manipulate</div><div class='del'>- * @type: what type of segment descriptor we're manipulating</div><div class='del'>- * @direct_mode_sz: size to alloc in direct mode</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_add_sd_table_entry(struct i40iw_hw *hw,</div><div class='del'>-						struct i40iw_hmc_info *hmc_info,</div><div class='del'>-						u32 sd_index,</div><div class='del'>-						enum i40iw_sd_entry_type type,</div><div class='del'>-						u64 direct_mode_sz)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-	struct i40iw_hmc_sd_entry *sd_entry;</div><div class='del'>-	bool dma_mem_alloc_done = false;</div><div class='del'>-	struct i40iw_dma_mem mem;</div><div class='del'>-	u64 alloc_len;</div><div class='del'>-</div><div class='del'>-	sd_entry = &amp;hmc_info-&gt;sd_table.sd_entry[sd_index];</div><div class='del'>-	if (!sd_entry-&gt;valid) {</div><div class='del'>-		if (type == I40IW_SD_TYPE_PAGED)</div><div class='del'>-			alloc_len = I40IW_HMC_PAGED_BP_SIZE;</div><div class='del'>-		else</div><div class='del'>-			alloc_len = direct_mode_sz;</div><div class='del'>-</div><div class='del'>-		/* allocate a 4K pd page or 2M backing page */</div><div class='del'>-		ret_code = i40iw_allocate_dma_mem(hw, &amp;mem, alloc_len,</div><div class='del'>-						  I40IW_HMC_PD_BP_BUF_ALIGNMENT);</div><div class='del'>-		if (ret_code)</div><div class='del'>-			goto exit;</div><div class='del'>-		dma_mem_alloc_done = true;</div><div class='del'>-		if (type == I40IW_SD_TYPE_PAGED) {</div><div class='del'>-			ret_code = i40iw_allocate_virt_mem(hw,</div><div class='del'>-							   &amp;sd_entry-&gt;u.pd_table.pd_entry_virt_mem,</div><div class='del'>-							   sizeof(struct i40iw_hmc_pd_entry) * 512);</div><div class='del'>-			if (ret_code)</div><div class='del'>-				goto exit;</div><div class='del'>-			sd_entry-&gt;u.pd_table.pd_entry = (struct i40iw_hmc_pd_entry *)</div><div class='del'>-							 sd_entry-&gt;u.pd_table.pd_entry_virt_mem.va;</div><div class='del'>-</div><div class='del'>-			memcpy(&amp;sd_entry-&gt;u.pd_table.pd_page_addr, &amp;mem, sizeof(struct i40iw_dma_mem));</div><div class='del'>-		} else {</div><div class='del'>-			memcpy(&amp;sd_entry-&gt;u.bp.addr, &amp;mem, sizeof(struct i40iw_dma_mem));</div><div class='del'>-			sd_entry-&gt;u.bp.sd_pd_index = sd_index;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		hmc_info-&gt;sd_table.sd_entry[sd_index].entry_type = type;</div><div class='del'>-</div><div class='del'>-		I40IW_INC_SD_REFCNT(&amp;hmc_info-&gt;sd_table);</div><div class='del'>-	}</div><div class='del'>-	if (sd_entry-&gt;entry_type == I40IW_SD_TYPE_DIRECT)</div><div class='del'>-		I40IW_INC_BP_REFCNT(&amp;sd_entry-&gt;u.bp);</div><div class='del'>-exit:</div><div class='del'>-	if (ret_code)</div><div class='del'>-		if (dma_mem_alloc_done)</div><div class='del'>-			i40iw_free_dma_mem(hw, &amp;mem);</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_add_pd_table_entry - Adds page descriptor to the specified table</div><div class='del'>- * @hw: pointer to our HW structure</div><div class='del'>- * @hmc_info: pointer to the HMC configuration information structure</div><div class='del'>- * @pd_index: which page descriptor index to manipulate</div><div class='del'>- * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one.</div><div class='del'>- *</div><div class='del'>- * This function:</div><div class='del'>- *	1. Initializes the pd entry</div><div class='del'>- *	2. Adds pd_entry in the pd_table</div><div class='del'>- *	3. Mark the entry valid in i40iw_hmc_pd_entry structure</div><div class='del'>- *	4. Initializes the pd_entry's ref count to 1</div><div class='del'>- * assumptions:</div><div class='del'>- *	1. The memory for pd should be pinned down, physically contiguous and</div><div class='del'>- *	   aligned on 4K boundary and zeroed memory.</div><div class='del'>- *	2. It should be 4K in size.</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_add_pd_table_entry(struct i40iw_hw *hw,</div><div class='del'>-						struct i40iw_hmc_info *hmc_info,</div><div class='del'>-						u32 pd_index,</div><div class='del'>-						struct i40iw_dma_mem *rsrc_pg)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-	struct i40iw_hmc_pd_table *pd_table;</div><div class='del'>-	struct i40iw_hmc_pd_entry *pd_entry;</div><div class='del'>-	struct i40iw_dma_mem mem;</div><div class='del'>-	struct i40iw_dma_mem *page = &amp;mem;</div><div class='del'>-	u32 sd_idx, rel_pd_idx;</div><div class='del'>-	u64 *pd_addr;</div><div class='del'>-	u64 page_desc;</div><div class='del'>-</div><div class='del'>-	if (pd_index / I40IW_HMC_PD_CNT_IN_SD &gt;= hmc_info-&gt;sd_table.sd_cnt)</div><div class='del'>-		return I40IW_ERR_INVALID_PAGE_DESC_INDEX;</div><div class='del'>-</div><div class='del'>-	sd_idx = (pd_index / I40IW_HMC_PD_CNT_IN_SD);</div><div class='del'>-	if (hmc_info-&gt;sd_table.sd_entry[sd_idx].entry_type != I40IW_SD_TYPE_PAGED)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	rel_pd_idx = (pd_index % I40IW_HMC_PD_CNT_IN_SD);</div><div class='del'>-	pd_table = &amp;hmc_info-&gt;sd_table.sd_entry[sd_idx].u.pd_table;</div><div class='del'>-	pd_entry = &amp;pd_table-&gt;pd_entry[rel_pd_idx];</div><div class='del'>-	if (!pd_entry-&gt;valid) {</div><div class='del'>-		if (rsrc_pg) {</div><div class='del'>-			pd_entry-&gt;rsrc_pg = true;</div><div class='del'>-			page = rsrc_pg;</div><div class='del'>-		} else {</div><div class='del'>-			ret_code = i40iw_allocate_dma_mem(hw, page,</div><div class='del'>-							  I40IW_HMC_PAGED_BP_SIZE,</div><div class='del'>-							  I40IW_HMC_PD_BP_BUF_ALIGNMENT);</div><div class='del'>-			if (ret_code)</div><div class='del'>-				return ret_code;</div><div class='del'>-			pd_entry-&gt;rsrc_pg = false;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		memcpy(&amp;pd_entry-&gt;bp.addr, page, sizeof(struct i40iw_dma_mem));</div><div class='del'>-		pd_entry-&gt;bp.sd_pd_index = pd_index;</div><div class='del'>-		pd_entry-&gt;bp.entry_type = I40IW_SD_TYPE_PAGED;</div><div class='del'>-		page_desc = page-&gt;pa | 0x1;</div><div class='del'>-</div><div class='del'>-		pd_addr = (u64 *)pd_table-&gt;pd_page_addr.va;</div><div class='del'>-		pd_addr += rel_pd_idx;</div><div class='del'>-</div><div class='del'>-		memcpy(pd_addr, &amp;page_desc, sizeof(*pd_addr));</div><div class='del'>-</div><div class='del'>-		pd_entry-&gt;sd_index = sd_idx;</div><div class='del'>-		pd_entry-&gt;valid = true;</div><div class='del'>-		I40IW_INC_PD_REFCNT(pd_table);</div><div class='del'>-		if (hmc_info-&gt;hmc_fn_id &lt; I40IW_FIRST_VF_FPM_ID)</div><div class='del'>-			I40IW_INVALIDATE_PF_HMC_PD(hw, sd_idx, rel_pd_idx);</div><div class='del'>-		else if (hw-&gt;hmc.hmc_fn_id != hmc_info-&gt;hmc_fn_id)</div><div class='del'>-			I40IW_INVALIDATE_VF_HMC_PD(hw, sd_idx, rel_pd_idx,</div><div class='del'>-						   hmc_info-&gt;hmc_fn_id);</div><div class='del'>-	}</div><div class='del'>-	I40IW_INC_BP_REFCNT(&amp;pd_entry-&gt;bp);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_remove_pd_bp - remove a backing page from a page descriptor</div><div class='del'>- * @hw: pointer to our HW structure</div><div class='del'>- * @hmc_info: pointer to the HMC configuration information structure</div><div class='del'>- * @idx: the page index</div><div class='del'>- * @is_pf: distinguishes a VF from a PF</div><div class='del'>- *</div><div class='del'>- * This function:</div><div class='del'>- *	1. Marks the entry in pd table (for paged address mode) or in sd table</div><div class='del'>- *	   (for direct address mode) invalid.</div><div class='del'>- *	2. Write to register PMPDINV to invalidate the backing page in FV cache</div><div class='del'>- *	3. Decrement the ref count for the pd _entry</div><div class='del'>- * assumptions:</div><div class='del'>- *	1. Caller can deallocate the memory used by backing storage after this</div><div class='del'>- *	   function returns.</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_remove_pd_bp(struct i40iw_hw *hw,</div><div class='del'>-					  struct i40iw_hmc_info *hmc_info,</div><div class='del'>-					  u32 idx,</div><div class='del'>-					  bool is_pf)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_hmc_pd_entry *pd_entry;</div><div class='del'>-	struct i40iw_hmc_pd_table *pd_table;</div><div class='del'>-	struct i40iw_hmc_sd_entry *sd_entry;</div><div class='del'>-	u32 sd_idx, rel_pd_idx;</div><div class='del'>-	struct i40iw_dma_mem *mem;</div><div class='del'>-	u64 *pd_addr;</div><div class='del'>-</div><div class='del'>-	sd_idx = idx / I40IW_HMC_PD_CNT_IN_SD;</div><div class='del'>-	rel_pd_idx = idx % I40IW_HMC_PD_CNT_IN_SD;</div><div class='del'>-	if (sd_idx &gt;= hmc_info-&gt;sd_table.sd_cnt)</div><div class='del'>-		return I40IW_ERR_INVALID_PAGE_DESC_INDEX;</div><div class='del'>-</div><div class='del'>-	sd_entry = &amp;hmc_info-&gt;sd_table.sd_entry[sd_idx];</div><div class='del'>-	if (sd_entry-&gt;entry_type != I40IW_SD_TYPE_PAGED)</div><div class='del'>-		return I40IW_ERR_INVALID_SD_TYPE;</div><div class='del'>-</div><div class='del'>-	pd_table = &amp;hmc_info-&gt;sd_table.sd_entry[sd_idx].u.pd_table;</div><div class='del'>-	pd_entry = &amp;pd_table-&gt;pd_entry[rel_pd_idx];</div><div class='del'>-	I40IW_DEC_BP_REFCNT(&amp;pd_entry-&gt;bp);</div><div class='del'>-	if (pd_entry-&gt;bp.ref_cnt)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	pd_entry-&gt;valid = false;</div><div class='del'>-	I40IW_DEC_PD_REFCNT(pd_table);</div><div class='del'>-	pd_addr = (u64 *)pd_table-&gt;pd_page_addr.va;</div><div class='del'>-	pd_addr += rel_pd_idx;</div><div class='del'>-	memset(pd_addr, 0, sizeof(u64));</div><div class='del'>-	if (is_pf)</div><div class='del'>-		I40IW_INVALIDATE_PF_HMC_PD(hw, sd_idx, idx);</div><div class='del'>-	else</div><div class='del'>-		I40IW_INVALIDATE_VF_HMC_PD(hw, sd_idx, idx,</div><div class='del'>-					   hmc_info-&gt;hmc_fn_id);</div><div class='del'>-</div><div class='del'>-	if (!pd_entry-&gt;rsrc_pg) {</div><div class='del'>-		mem = &amp;pd_entry-&gt;bp.addr;</div><div class='del'>-		if (!mem || !mem-&gt;va)</div><div class='del'>-			return I40IW_ERR_PARAM;</div><div class='del'>-		i40iw_free_dma_mem(hw, mem);</div><div class='del'>-	}</div><div class='del'>-	if (!pd_table-&gt;ref_cnt)</div><div class='del'>-		i40iw_free_virt_mem(hw, &amp;pd_table-&gt;pd_entry_virt_mem);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry</div><div class='del'>- * @hmc_info: pointer to the HMC configuration information structure</div><div class='del'>- * @idx: the page index</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_prep_remove_sd_bp(struct i40iw_hmc_info *hmc_info, u32 idx)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_hmc_sd_entry *sd_entry;</div><div class='del'>-</div><div class='del'>-	sd_entry = &amp;hmc_info-&gt;sd_table.sd_entry[idx];</div><div class='del'>-	I40IW_DEC_BP_REFCNT(&amp;sd_entry-&gt;u.bp);</div><div class='del'>-	if (sd_entry-&gt;u.bp.ref_cnt)</div><div class='del'>-		return I40IW_ERR_NOT_READY;</div><div class='del'>-</div><div class='del'>-	I40IW_DEC_SD_REFCNT(&amp;hmc_info-&gt;sd_table);</div><div class='del'>-	sd_entry-&gt;valid = false;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_prep_remove_pd_page - Prepares to remove a PD page from sd entry.</div><div class='del'>- * @hmc_info: pointer to the HMC configuration information structure</div><div class='del'>- * @idx: segment descriptor index to find the relevant page descriptor</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_prep_remove_pd_page(struct i40iw_hmc_info *hmc_info,</div><div class='del'>-						 u32 idx)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_hmc_sd_entry *sd_entry;</div><div class='del'>-</div><div class='del'>-	sd_entry = &amp;hmc_info-&gt;sd_table.sd_entry[idx];</div><div class='del'>-</div><div class='del'>-	if (sd_entry-&gt;u.pd_table.ref_cnt)</div><div class='del'>-		return I40IW_ERR_NOT_READY;</div><div class='del'>-</div><div class='del'>-	sd_entry-&gt;valid = false;</div><div class='del'>-	I40IW_DEC_SD_REFCNT(&amp;hmc_info-&gt;sd_table);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_pf_init_vfhmc -</div><div class='del'>- * @vf_cnt_array: array of cnt values of iwarp hmc objects</div><div class='del'>- * @vf_hmc_fn_id: hmc function id ofr vf driver</div><div class='del'>- * @dev: pointer to i40iw_dev struct</div><div class='del'>- *</div><div class='del'>- * Called by pf driver to initialize hmc_info for vf driver instance.</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_pf_init_vfhmc(struct i40iw_sc_dev *dev,</div><div class='del'>-					   u8 vf_hmc_fn_id,</div><div class='del'>-					   u32 *vf_cnt_array)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_hmc_info *hmc_info;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-	u32 i;</div><div class='del'>-</div><div class='del'>-	if ((vf_hmc_fn_id &lt; I40IW_FIRST_VF_FPM_ID) ||</div><div class='del'>-	    (vf_hmc_fn_id &gt;= I40IW_FIRST_VF_FPM_ID +</div><div class='del'>-	     I40IW_MAX_PE_ENABLED_VF_COUNT)) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: invalid vf_hmc_fn_id  0x%x\n",</div><div class='del'>-			    __func__, vf_hmc_fn_id);</div><div class='del'>-		return I40IW_ERR_INVALID_HMCFN_ID;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret_code = i40iw_sc_init_iw_hmc(dev, vf_hmc_fn_id);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return ret_code;</div><div class='del'>-</div><div class='del'>-	hmc_info = i40iw_vf_hmcinfo_from_fpm(dev, vf_hmc_fn_id);</div><div class='del'>-</div><div class='del'>-	for (i = I40IW_HMC_IW_QP; i &lt; I40IW_HMC_IW_MAX; i++)</div><div class='del'>-		if (vf_cnt_array)</div><div class='del'>-			hmc_info-&gt;hmc_obj[i].cnt =</div><div class='del'>-			    vf_cnt_array[i - I40IW_HMC_IW_QP];</div><div class='del'>-		else</div><div class='del'>-			hmc_info-&gt;hmc_obj[i].cnt = hmc_info-&gt;hmc_obj[i].max_cnt;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_hmc.h b/drivers/infiniband/hw/i40iw/i40iw_hmc.h<br/>deleted file mode 100644<br/>index 4c3fdd875621..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_hmc.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_hmc.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,241 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#ifndef I40IW_HMC_H</div><div class='del'>-#define I40IW_HMC_H</div><div class='del'>-</div><div class='del'>-#include "i40iw_d.h"</div><div class='del'>-</div><div class='del'>-struct i40iw_hw;</div><div class='del'>-enum i40iw_status_code;</div><div class='del'>-</div><div class='del'>-#define I40IW_HMC_MAX_BP_COUNT 512</div><div class='del'>-#define I40IW_MAX_SD_ENTRIES 11</div><div class='del'>-#define I40IW_HW_DBG_HMC_INVALID_BP_MARK     0xCA</div><div class='del'>-</div><div class='del'>-#define I40IW_HMC_INFO_SIGNATURE	0x484D5347</div><div class='del'>-#define I40IW_HMC_PD_CNT_IN_SD		512</div><div class='del'>-#define I40IW_HMC_DIRECT_BP_SIZE	0x200000</div><div class='del'>-#define I40IW_HMC_MAX_SD_COUNT		4096</div><div class='del'>-#define I40IW_HMC_PAGED_BP_SIZE		4096</div><div class='del'>-#define I40IW_HMC_PD_BP_BUF_ALIGNMENT	4096</div><div class='del'>-#define I40IW_FIRST_VF_FPM_ID		16</div><div class='del'>-#define FPM_MULTIPLIER			1024</div><div class='del'>-</div><div class='del'>-#define I40IW_INC_SD_REFCNT(sd_table)   ((sd_table)-&gt;ref_cnt++)</div><div class='del'>-#define I40IW_INC_PD_REFCNT(pd_table)   ((pd_table)-&gt;ref_cnt++)</div><div class='del'>-#define I40IW_INC_BP_REFCNT(bp)         ((bp)-&gt;ref_cnt++)</div><div class='del'>-</div><div class='del'>-#define I40IW_DEC_SD_REFCNT(sd_table)   ((sd_table)-&gt;ref_cnt--)</div><div class='del'>-#define I40IW_DEC_PD_REFCNT(pd_table)   ((pd_table)-&gt;ref_cnt--)</div><div class='del'>-#define I40IW_DEC_BP_REFCNT(bp)         ((bp)-&gt;ref_cnt--)</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * I40IW_INVALIDATE_PF_HMC_PD - Invalidates the pd cache in the hardware</div><div class='del'>- * @hw: pointer to our hw struct</div><div class='del'>- * @sd_idx: segment descriptor index</div><div class='del'>- * @pd_idx: page descriptor index</div><div class='del'>- */</div><div class='del'>-#define I40IW_INVALIDATE_PF_HMC_PD(hw, sd_idx, pd_idx)                  \</div><div class='del'>-	i40iw_wr32((hw), I40E_PFHMC_PDINV,                                    \</div><div class='del'>-		(((sd_idx) &lt;&lt; I40E_PFHMC_PDINV_PMSDIDX_SHIFT) |             \</div><div class='del'>-		(0x1 &lt;&lt; I40E_PFHMC_PDINV_PMSDPARTSEL_SHIFT) | \</div><div class='del'>-		((pd_idx) &lt;&lt; I40E_PFHMC_PDINV_PMPDIDX_SHIFT)))</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * I40IW_INVALIDATE_VF_HMC_PD - Invalidates the pd cache in the hardware</div><div class='del'>- * @hw: pointer to our hw struct</div><div class='del'>- * @sd_idx: segment descriptor index</div><div class='del'>- * @pd_idx: page descriptor index</div><div class='del'>- * @hmc_fn_id: VF's function id</div><div class='del'>- */</div><div class='del'>-#define I40IW_INVALIDATE_VF_HMC_PD(hw, sd_idx, pd_idx, hmc_fn_id)        \</div><div class='del'>-	i40iw_wr32(hw, I40E_GLHMC_VFPDINV(hmc_fn_id - I40IW_FIRST_VF_FPM_ID),  \</div><div class='del'>-	     ((sd_idx &lt;&lt; I40E_PFHMC_PDINV_PMSDIDX_SHIFT) |              \</div><div class='del'>-	      (pd_idx &lt;&lt; I40E_PFHMC_PDINV_PMPDIDX_SHIFT)))</div><div class='del'>-</div><div class='del'>-struct i40iw_hmc_obj_info {</div><div class='del'>-	u64 base;</div><div class='del'>-	u32 max_cnt;</div><div class='del'>-	u32 cnt;</div><div class='del'>-	u64 size;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_sd_entry_type {</div><div class='del'>-	I40IW_SD_TYPE_INVALID = 0,</div><div class='del'>-	I40IW_SD_TYPE_PAGED = 1,</div><div class='del'>-	I40IW_SD_TYPE_DIRECT = 2</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_hmc_bp {</div><div class='del'>-	enum i40iw_sd_entry_type entry_type;</div><div class='del'>-	struct i40iw_dma_mem addr;</div><div class='del'>-	u32 sd_pd_index;</div><div class='del'>-	u32 ref_cnt;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_hmc_pd_entry {</div><div class='del'>-	struct i40iw_hmc_bp bp;</div><div class='del'>-	u32 sd_index;</div><div class='del'>-	bool rsrc_pg;</div><div class='del'>-	bool valid;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_hmc_pd_table {</div><div class='del'>-	struct i40iw_dma_mem pd_page_addr;</div><div class='del'>-	struct i40iw_hmc_pd_entry *pd_entry;</div><div class='del'>-	struct i40iw_virt_mem pd_entry_virt_mem;</div><div class='del'>-	u32 ref_cnt;</div><div class='del'>-	u32 sd_index;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_hmc_sd_entry {</div><div class='del'>-	enum i40iw_sd_entry_type entry_type;</div><div class='del'>-	bool valid;</div><div class='del'>-</div><div class='del'>-	union {</div><div class='del'>-		struct i40iw_hmc_pd_table pd_table;</div><div class='del'>-		struct i40iw_hmc_bp bp;</div><div class='del'>-	} u;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_hmc_sd_table {</div><div class='del'>-	struct i40iw_virt_mem addr;</div><div class='del'>-	u32 sd_cnt;</div><div class='del'>-	u32 ref_cnt;</div><div class='del'>-	struct i40iw_hmc_sd_entry *sd_entry;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_hmc_info {</div><div class='del'>-	u32 signature;</div><div class='del'>-	u8 hmc_fn_id;</div><div class='del'>-	u16 first_sd_index;</div><div class='del'>-</div><div class='del'>-	struct i40iw_hmc_obj_info *hmc_obj;</div><div class='del'>-	struct i40iw_virt_mem hmc_obj_virt_mem;</div><div class='del'>-	struct i40iw_hmc_sd_table sd_table;</div><div class='del'>-	u16 sd_indexes[I40IW_HMC_MAX_SD_COUNT];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct update_sd_entry {</div><div class='del'>-	u64 cmd;</div><div class='del'>-	u64 data;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_update_sds_info {</div><div class='del'>-	u32 cnt;</div><div class='del'>-	u8 hmc_fn_id;</div><div class='del'>-	struct update_sd_entry entry[I40IW_MAX_SD_ENTRIES];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_ccq_cqe_info;</div><div class='del'>-struct i40iw_hmc_fcn_info {</div><div class='del'>-	void (*callback_fcn)(struct i40iw_sc_dev *, void *,</div><div class='del'>-			     struct i40iw_ccq_cqe_info *);</div><div class='del'>-	void *cqp_callback_param;</div><div class='del'>-	u32 vf_id;</div><div class='del'>-	u16 iw_vf_idx;</div><div class='del'>-	bool free_fcn;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_hmc_rsrc_type {</div><div class='del'>-	I40IW_HMC_IW_QP = 0,</div><div class='del'>-	I40IW_HMC_IW_CQ = 1,</div><div class='del'>-	I40IW_HMC_IW_SRQ = 2,</div><div class='del'>-	I40IW_HMC_IW_HTE = 3,</div><div class='del'>-	I40IW_HMC_IW_ARP = 4,</div><div class='del'>-	I40IW_HMC_IW_APBVT_ENTRY = 5,</div><div class='del'>-	I40IW_HMC_IW_MR = 6,</div><div class='del'>-	I40IW_HMC_IW_XF = 7,</div><div class='del'>-	I40IW_HMC_IW_XFFL = 8,</div><div class='del'>-	I40IW_HMC_IW_Q1 = 9,</div><div class='del'>-	I40IW_HMC_IW_Q1FL = 10,</div><div class='del'>-	I40IW_HMC_IW_TIMER = 11,</div><div class='del'>-	I40IW_HMC_IW_FSIMC = 12,</div><div class='del'>-	I40IW_HMC_IW_FSIAV = 13,</div><div class='del'>-	I40IW_HMC_IW_PBLE = 14,</div><div class='del'>-	I40IW_HMC_IW_MAX = 15,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_hmc_create_obj_info {</div><div class='del'>-	struct i40iw_hmc_info *hmc_info;</div><div class='del'>-	struct i40iw_virt_mem add_sd_virt_mem;</div><div class='del'>-	u32 rsrc_type;</div><div class='del'>-	u32 start_idx;</div><div class='del'>-	u32 count;</div><div class='del'>-	u32 add_sd_cnt;</div><div class='del'>-	enum i40iw_sd_entry_type entry_type;</div><div class='del'>-	bool is_pf;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_hmc_del_obj_info {</div><div class='del'>-	struct i40iw_hmc_info *hmc_info;</div><div class='del'>-	struct i40iw_virt_mem del_sd_virt_mem;</div><div class='del'>-	u32 rsrc_type;</div><div class='del'>-	u32 start_idx;</div><div class='del'>-	u32 count;</div><div class='del'>-	u32 del_sd_cnt;</div><div class='del'>-	bool is_pf;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_copy_dma_mem(struct i40iw_hw *hw, void *dest_buf,</div><div class='del'>-					  struct i40iw_dma_mem *src_mem, u64 src_offset, u64 size);</div><div class='del'>-enum i40iw_status_code i40iw_sc_create_hmc_obj(struct i40iw_sc_dev *dev,</div><div class='del'>-					       struct i40iw_hmc_create_obj_info *info);</div><div class='del'>-enum i40iw_status_code i40iw_sc_del_hmc_obj(struct i40iw_sc_dev *dev,</div><div class='del'>-					    struct i40iw_hmc_del_obj_info *info,</div><div class='del'>-					    bool reset);</div><div class='del'>-enum i40iw_status_code i40iw_hmc_sd_one(struct i40iw_sc_dev *dev, u8 hmc_fn_id,</div><div class='del'>-					u64 pa, u32 sd_idx, enum i40iw_sd_entry_type type,</div><div class='del'>-					bool setsd);</div><div class='del'>-enum i40iw_status_code i40iw_update_sds_noccq(struct i40iw_sc_dev *dev,</div><div class='del'>-					      struct i40iw_update_sds_info *info);</div><div class='del'>-struct i40iw_vfdev *i40iw_vfdev_from_fpm(struct i40iw_sc_dev *dev, u8 hmc_fn_id);</div><div class='del'>-struct i40iw_hmc_info *i40iw_vf_hmcinfo_from_fpm(struct i40iw_sc_dev *dev,</div><div class='del'>-						 u8 hmc_fn_id);</div><div class='del'>-enum i40iw_status_code i40iw_add_sd_table_entry(struct i40iw_hw *hw,</div><div class='del'>-						struct i40iw_hmc_info *hmc_info, u32 sd_index,</div><div class='del'>-						enum i40iw_sd_entry_type type, u64 direct_mode_sz);</div><div class='del'>-enum i40iw_status_code i40iw_add_pd_table_entry(struct i40iw_hw *hw,</div><div class='del'>-						struct i40iw_hmc_info *hmc_info, u32 pd_index,</div><div class='del'>-						struct i40iw_dma_mem *rsrc_pg);</div><div class='del'>-enum i40iw_status_code i40iw_remove_pd_bp(struct i40iw_hw *hw,</div><div class='del'>-					  struct i40iw_hmc_info *hmc_info, u32 idx, bool is_pf);</div><div class='del'>-enum i40iw_status_code i40iw_prep_remove_sd_bp(struct i40iw_hmc_info *hmc_info, u32 idx);</div><div class='del'>-enum i40iw_status_code i40iw_prep_remove_pd_page(struct i40iw_hmc_info *hmc_info, u32 idx);</div><div class='del'>-</div><div class='del'>-#define     ENTER_SHARED_FUNCTION()</div><div class='del'>-#define     EXIT_SHARED_FUNCTION()</div><div class='del'>-</div><div class='del'>-#endif				/* I40IW_HMC_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_hw.c b/drivers/infiniband/hw/i40iw/i40iw_hw.c<br/>deleted file mode 100644<br/>index 55a1fbf0e670..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_hw.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_hw.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,852 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='del'>-#include &lt;linux/moduleparam.h&gt;</div><div class='del'>-#include &lt;linux/netdevice.h&gt;</div><div class='del'>-#include &lt;linux/etherdevice.h&gt;</div><div class='del'>-#include &lt;linux/ip.h&gt;</div><div class='del'>-#include &lt;linux/tcp.h&gt;</div><div class='del'>-#include &lt;linux/if_vlan.h&gt;</div><div class='del'>-</div><div class='del'>-#include "i40iw.h"</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_initialize_hw_resources - initialize hw resource during open</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- */</div><div class='del'>-u32 i40iw_initialize_hw_resources(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	unsigned long num_pds;</div><div class='del'>-	u32 resources_size;</div><div class='del'>-	u32 max_mr;</div><div class='del'>-	u32 max_qp;</div><div class='del'>-	u32 max_cq;</div><div class='del'>-	u32 arp_table_size;</div><div class='del'>-	u32 mrdrvbits;</div><div class='del'>-	void *resource_ptr;</div><div class='del'>-</div><div class='del'>-	max_qp = iwdev-&gt;sc_dev.hmc_info-&gt;hmc_obj[I40IW_HMC_IW_QP].cnt;</div><div class='del'>-	max_cq = iwdev-&gt;sc_dev.hmc_info-&gt;hmc_obj[I40IW_HMC_IW_CQ].cnt;</div><div class='del'>-	max_mr = iwdev-&gt;sc_dev.hmc_info-&gt;hmc_obj[I40IW_HMC_IW_MR].cnt;</div><div class='del'>-	arp_table_size = iwdev-&gt;sc_dev.hmc_info-&gt;hmc_obj[I40IW_HMC_IW_ARP].cnt;</div><div class='del'>-	iwdev-&gt;max_cqe = 0xFFFFF;</div><div class='del'>-	num_pds = I40IW_MAX_PDS;</div><div class='del'>-	resources_size = sizeof(struct i40iw_arp_entry) * arp_table_size;</div><div class='del'>-	resources_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);</div><div class='del'>-	resources_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);</div><div class='del'>-	resources_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);</div><div class='del'>-	resources_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);</div><div class='del'>-	resources_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);</div><div class='del'>-	resources_size += sizeof(struct i40iw_qp **) * max_qp;</div><div class='del'>-	iwdev-&gt;mem_resources = kzalloc(resources_size, GFP_KERNEL);</div><div class='del'>-</div><div class='del'>-	if (!iwdev-&gt;mem_resources)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	iwdev-&gt;max_qp = max_qp;</div><div class='del'>-	iwdev-&gt;max_mr = max_mr;</div><div class='del'>-	iwdev-&gt;max_cq = max_cq;</div><div class='del'>-	iwdev-&gt;max_pd = num_pds;</div><div class='del'>-	iwdev-&gt;arp_table_size = arp_table_size;</div><div class='del'>-	iwdev-&gt;arp_table = (struct i40iw_arp_entry *)iwdev-&gt;mem_resources;</div><div class='del'>-	resource_ptr = iwdev-&gt;mem_resources + (sizeof(struct i40iw_arp_entry) * arp_table_size);</div><div class='del'>-</div><div class='del'>-	iwdev-&gt;device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |</div><div class='del'>-	    IB_DEVICE_MEM_WINDOW | IB_DEVICE_MEM_MGT_EXTENSIONS;</div><div class='del'>-</div><div class='del'>-	iwdev-&gt;allocated_qps = resource_ptr;</div><div class='del'>-	iwdev-&gt;allocated_cqs = &amp;iwdev-&gt;allocated_qps[BITS_TO_LONGS(max_qp)];</div><div class='del'>-	iwdev-&gt;allocated_mrs = &amp;iwdev-&gt;allocated_cqs[BITS_TO_LONGS(max_cq)];</div><div class='del'>-	iwdev-&gt;allocated_pds = &amp;iwdev-&gt;allocated_mrs[BITS_TO_LONGS(max_mr)];</div><div class='del'>-	iwdev-&gt;allocated_arps = &amp;iwdev-&gt;allocated_pds[BITS_TO_LONGS(num_pds)];</div><div class='del'>-	iwdev-&gt;qp_table = (struct i40iw_qp **)(&amp;iwdev-&gt;allocated_arps[BITS_TO_LONGS(arp_table_size)]);</div><div class='del'>-	set_bit(0, iwdev-&gt;allocated_mrs);</div><div class='del'>-	set_bit(0, iwdev-&gt;allocated_qps);</div><div class='del'>-	set_bit(0, iwdev-&gt;allocated_cqs);</div><div class='del'>-	set_bit(0, iwdev-&gt;allocated_pds);</div><div class='del'>-	set_bit(0, iwdev-&gt;allocated_arps);</div><div class='del'>-</div><div class='del'>-	/* Following for ILQ/IEQ */</div><div class='del'>-	set_bit(1, iwdev-&gt;allocated_qps);</div><div class='del'>-	set_bit(1, iwdev-&gt;allocated_cqs);</div><div class='del'>-	set_bit(1, iwdev-&gt;allocated_pds);</div><div class='del'>-	set_bit(2, iwdev-&gt;allocated_cqs);</div><div class='del'>-	set_bit(2, iwdev-&gt;allocated_pds);</div><div class='del'>-</div><div class='del'>-	spin_lock_init(&amp;iwdev-&gt;resource_lock);</div><div class='del'>-	spin_lock_init(&amp;iwdev-&gt;qptable_lock);</div><div class='del'>-	/* stag index mask has a minimum of 14 bits */</div><div class='del'>-	mrdrvbits = 24 - max(get_count_order(iwdev-&gt;max_mr), 14);</div><div class='del'>-	iwdev-&gt;mr_stagmask = ~(((1 &lt;&lt; mrdrvbits) - 1) &lt;&lt; (32 - mrdrvbits));</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cqp_ce_handler - handle cqp completions</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @arm: flag to arm after completions</div><div class='del'>- * @cq: cq for cqp completions</div><div class='del'>- */</div><div class='del'>-static void i40iw_cqp_ce_handler(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq, bool arm)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	u32 cqe_count = 0;</div><div class='del'>-	struct i40iw_ccq_cqe_info info;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	do {</div><div class='del'>-		memset(&amp;info, 0, sizeof(info));</div><div class='del'>-		ret = dev-&gt;ccq_ops-&gt;ccq_get_cqe_info(cq, &amp;info);</div><div class='del'>-		if (ret)</div><div class='del'>-			break;</div><div class='del'>-		cqp_request = (struct i40iw_cqp_request *)(unsigned long)info.scratch;</div><div class='del'>-		if (info.error)</div><div class='del'>-			i40iw_pr_err("opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n",</div><div class='del'>-				     info.op_code, info.maj_err_code, info.min_err_code);</div><div class='del'>-		if (cqp_request) {</div><div class='del'>-			cqp_request-&gt;compl_info.maj_err_code = info.maj_err_code;</div><div class='del'>-			cqp_request-&gt;compl_info.min_err_code = info.min_err_code;</div><div class='del'>-			cqp_request-&gt;compl_info.op_ret_val = info.op_ret_val;</div><div class='del'>-			cqp_request-&gt;compl_info.error = info.error;</div><div class='del'>-</div><div class='del'>-			if (cqp_request-&gt;waiting) {</div><div class='del'>-				cqp_request-&gt;request_done = true;</div><div class='del'>-				wake_up(&amp;cqp_request-&gt;waitq);</div><div class='del'>-				i40iw_put_cqp_request(&amp;iwdev-&gt;cqp, cqp_request);</div><div class='del'>-			} else {</div><div class='del'>-				if (cqp_request-&gt;callback_fcn)</div><div class='del'>-					cqp_request-&gt;callback_fcn(cqp_request, 1);</div><div class='del'>-				i40iw_put_cqp_request(&amp;iwdev-&gt;cqp, cqp_request);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		cqe_count++;</div><div class='del'>-	} while (1);</div><div class='del'>-</div><div class='del'>-	if (arm &amp;&amp; cqe_count) {</div><div class='del'>-		i40iw_process_bh(dev);</div><div class='del'>-		dev-&gt;ccq_ops-&gt;ccq_arm(cq);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_iwarp_ce_handler - handle iwarp completions</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @iwcp: iwarp cq receiving event</div><div class='del'>- */</div><div class='del'>-static void i40iw_iwarp_ce_handler(struct i40iw_device *iwdev,</div><div class='del'>-				   struct i40iw_sc_cq *iwcq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cq *i40iwcq = iwcq-&gt;back_cq;</div><div class='del'>-</div><div class='del'>-	if (i40iwcq-&gt;ibcq.comp_handler)</div><div class='del'>-		i40iwcq-&gt;ibcq.comp_handler(&amp;i40iwcq-&gt;ibcq,</div><div class='del'>-					   i40iwcq-&gt;ibcq.cq_context);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_ce_handler - handle puda completion events</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @cq: puda completion q for event</div><div class='del'>- */</div><div class='del'>-static void i40iw_puda_ce_handler(struct i40iw_device *iwdev,</div><div class='del'>-				  struct i40iw_sc_cq *cq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_dev *dev = (struct i40iw_sc_dev *)&amp;iwdev-&gt;sc_dev;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	u32 compl_error;</div><div class='del'>-</div><div class='del'>-	do {</div><div class='del'>-		status = i40iw_puda_poll_completion(dev, cq, &amp;compl_error);</div><div class='del'>-		if (status == I40IW_ERR_QUEUE_EMPTY)</div><div class='del'>-			break;</div><div class='del'>-		if (status) {</div><div class='del'>-			i40iw_pr_err("puda  status = %d\n", status);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		if (compl_error) {</div><div class='del'>-			i40iw_pr_err("puda compl_err  =0x%x\n", compl_error);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-	} while (1);</div><div class='del'>-</div><div class='del'>-	dev-&gt;ccq_ops-&gt;ccq_arm(cq);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_process_ceq - handle ceq for completions</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @ceq: ceq having cq for completion</div><div class='del'>- */</div><div class='del'>-void i40iw_process_ceq(struct i40iw_device *iwdev, struct i40iw_ceq *ceq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_sc_ceq *sc_ceq;</div><div class='del'>-	struct i40iw_sc_cq *cq;</div><div class='del'>-	bool arm = true;</div><div class='del'>-</div><div class='del'>-	sc_ceq = &amp;ceq-&gt;sc_ceq;</div><div class='del'>-	do {</div><div class='del'>-		cq = dev-&gt;ceq_ops-&gt;process_ceq(dev, sc_ceq);</div><div class='del'>-		if (!cq)</div><div class='del'>-			break;</div><div class='del'>-</div><div class='del'>-		if (cq-&gt;cq_type == I40IW_CQ_TYPE_CQP)</div><div class='del'>-			i40iw_cqp_ce_handler(iwdev, cq, arm);</div><div class='del'>-		else if (cq-&gt;cq_type == I40IW_CQ_TYPE_IWARP)</div><div class='del'>-			i40iw_iwarp_ce_handler(iwdev, cq);</div><div class='del'>-		else if ((cq-&gt;cq_type == I40IW_CQ_TYPE_ILQ) ||</div><div class='del'>-			 (cq-&gt;cq_type == I40IW_CQ_TYPE_IEQ))</div><div class='del'>-			i40iw_puda_ce_handler(iwdev, cq);</div><div class='del'>-	} while (1);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_next_iw_state - modify qp state</div><div class='del'>- * @iwqp: iwarp qp to modify</div><div class='del'>- * @state: next state for qp</div><div class='del'>- * @del_hash: del hash</div><div class='del'>- * @term: term message</div><div class='del'>- * @termlen: length of term message</div><div class='del'>- */</div><div class='del'>-void i40iw_next_iw_state(struct i40iw_qp *iwqp,</div><div class='del'>-			 u8 state,</div><div class='del'>-			 u8 del_hash,</div><div class='del'>-			 u8 term,</div><div class='del'>-			 u8 termlen)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_modify_qp_info info;</div><div class='del'>-</div><div class='del'>-	memset(&amp;info, 0, sizeof(info));</div><div class='del'>-	info.next_iwarp_state = state;</div><div class='del'>-	info.remove_hash_idx = del_hash;</div><div class='del'>-	info.cq_num_valid = true;</div><div class='del'>-	info.arp_cache_idx_valid = true;</div><div class='del'>-	info.dont_send_term = true;</div><div class='del'>-	info.dont_send_fin = true;</div><div class='del'>-	info.termlen = termlen;</div><div class='del'>-</div><div class='del'>-	if (term &amp; I40IWQP_TERM_SEND_TERM_ONLY)</div><div class='del'>-		info.dont_send_term = false;</div><div class='del'>-	if (term &amp; I40IWQP_TERM_SEND_FIN_ONLY)</div><div class='del'>-		info.dont_send_fin = false;</div><div class='del'>-	if (iwqp-&gt;sc_qp.term_flags &amp;&amp; (state == I40IW_QP_STATE_ERROR))</div><div class='del'>-		info.reset_tcp_conn = true;</div><div class='del'>-	iwqp-&gt;hw_iwarp_state = state;</div><div class='del'>-	i40iw_hw_modify_qp(iwqp-&gt;iwdev, iwqp, &amp;info, 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_process_aeq - handle aeq events</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- */</div><div class='del'>-void i40iw_process_aeq(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_aeq *aeq = &amp;iwdev-&gt;aeq;</div><div class='del'>-	struct i40iw_sc_aeq *sc_aeq = &amp;aeq-&gt;sc_aeq;</div><div class='del'>-	struct i40iw_aeqe_info aeinfo;</div><div class='del'>-	struct i40iw_aeqe_info *info = &amp;aeinfo;</div><div class='del'>-	int ret;</div><div class='del'>-	struct i40iw_qp *iwqp = NULL;</div><div class='del'>-	struct i40iw_sc_cq *cq = NULL;</div><div class='del'>-	struct i40iw_cq *iwcq = NULL;</div><div class='del'>-	struct i40iw_sc_qp *qp = NULL;</div><div class='del'>-	struct i40iw_qp_host_ctx_info *ctx_info = NULL;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	u32 aeqcnt = 0;</div><div class='del'>-</div><div class='del'>-	if (!sc_aeq-&gt;size)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	do {</div><div class='del'>-		memset(info, 0, sizeof(*info));</div><div class='del'>-		ret = dev-&gt;aeq_ops-&gt;get_next_aeqe(sc_aeq, info);</div><div class='del'>-		if (ret)</div><div class='del'>-			break;</div><div class='del'>-</div><div class='del'>-		aeqcnt++;</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_AEQ,</div><div class='del'>-			    "%s ae_id = 0x%x bool qp=%d qp_id = %d\n",</div><div class='del'>-			    __func__, info-&gt;ae_id, info-&gt;qp, info-&gt;qp_cq_id);</div><div class='del'>-		if (info-&gt;qp) {</div><div class='del'>-			spin_lock_irqsave(&amp;iwdev-&gt;qptable_lock, flags);</div><div class='del'>-			iwqp = iwdev-&gt;qp_table[info-&gt;qp_cq_id];</div><div class='del'>-			if (!iwqp) {</div><div class='del'>-				spin_unlock_irqrestore(&amp;iwdev-&gt;qptable_lock, flags);</div><div class='del'>-				i40iw_debug(dev, I40IW_DEBUG_AEQ,</div><div class='del'>-					    "%s qp_id %d is already freed\n",</div><div class='del'>-					    __func__, info-&gt;qp_cq_id);</div><div class='del'>-				continue;</div><div class='del'>-			}</div><div class='del'>-			i40iw_add_ref(&amp;iwqp-&gt;ibqp);</div><div class='del'>-			spin_unlock_irqrestore(&amp;iwdev-&gt;qptable_lock, flags);</div><div class='del'>-			qp = &amp;iwqp-&gt;sc_qp;</div><div class='del'>-			spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-			iwqp-&gt;hw_tcp_state = info-&gt;tcp_state;</div><div class='del'>-			iwqp-&gt;hw_iwarp_state = info-&gt;iwarp_state;</div><div class='del'>-			iwqp-&gt;last_aeq = info-&gt;ae_id;</div><div class='del'>-			spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-			ctx_info = &amp;iwqp-&gt;ctx_info;</div><div class='del'>-			ctx_info-&gt;err_rq_idx_valid = true;</div><div class='del'>-		} else {</div><div class='del'>-			if (info-&gt;ae_id != I40IW_AE_CQ_OPERATION_ERROR)</div><div class='del'>-				continue;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		switch (info-&gt;ae_id) {</div><div class='del'>-		case I40IW_AE_LLP_FIN_RECEIVED:</div><div class='del'>-			if (qp-&gt;term_flags)</div><div class='del'>-				break;</div><div class='del'>-			if (atomic_inc_return(&amp;iwqp-&gt;close_timer_started) == 1) {</div><div class='del'>-				iwqp-&gt;hw_tcp_state = I40IW_TCP_STATE_CLOSE_WAIT;</div><div class='del'>-				if ((iwqp-&gt;hw_tcp_state == I40IW_TCP_STATE_CLOSE_WAIT) &amp;&amp;</div><div class='del'>-				    (iwqp-&gt;ibqp_state == IB_QPS_RTS)) {</div><div class='del'>-					i40iw_next_iw_state(iwqp,</div><div class='del'>-							    I40IW_QP_STATE_CLOSING, 0, 0, 0);</div><div class='del'>-					i40iw_cm_disconn(iwqp);</div><div class='del'>-				}</div><div class='del'>-				iwqp-&gt;cm_id-&gt;add_ref(iwqp-&gt;cm_id);</div><div class='del'>-				i40iw_schedule_cm_timer(iwqp-&gt;cm_node,</div><div class='del'>-							(struct i40iw_puda_buf *)iwqp,</div><div class='del'>-							I40IW_TIMER_TYPE_CLOSE, 1, 0);</div><div class='del'>-			}</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_AE_LLP_CLOSE_COMPLETE:</div><div class='del'>-			if (qp-&gt;term_flags)</div><div class='del'>-				i40iw_terminate_done(qp, 0);</div><div class='del'>-			else</div><div class='del'>-				i40iw_cm_disconn(iwqp);</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_AE_BAD_CLOSE:</div><div class='del'>-			/* fall through */</div><div class='del'>-		case I40IW_AE_RESET_SENT:</div><div class='del'>-			i40iw_next_iw_state(iwqp, I40IW_QP_STATE_ERROR, 1, 0, 0);</div><div class='del'>-			i40iw_cm_disconn(iwqp);</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_AE_LLP_CONNECTION_RESET:</div><div class='del'>-			if (atomic_read(&amp;iwqp-&gt;close_timer_started))</div><div class='del'>-				break;</div><div class='del'>-			i40iw_cm_disconn(iwqp);</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_AE_QP_SUSPEND_COMPLETE:</div><div class='del'>-			i40iw_qp_suspend_resume(dev, &amp;iwqp-&gt;sc_qp, false);</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_AE_TERMINATE_SENT:</div><div class='del'>-			i40iw_terminate_send_fin(qp);</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_AE_LLP_TERMINATE_RECEIVED:</div><div class='del'>-			i40iw_terminate_received(qp, info);</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_AE_CQ_OPERATION_ERROR:</div><div class='del'>-			i40iw_pr_err("Processing an iWARP related AE for CQ misc = 0x%04X\n",</div><div class='del'>-				     info-&gt;ae_id);</div><div class='del'>-			cq = (struct i40iw_sc_cq *)(unsigned long)info-&gt;compl_ctx;</div><div class='del'>-			iwcq = (struct i40iw_cq *)cq-&gt;back_cq;</div><div class='del'>-</div><div class='del'>-			if (iwcq-&gt;ibcq.event_handler) {</div><div class='del'>-				struct ib_event ibevent;</div><div class='del'>-</div><div class='del'>-				ibevent.device = iwcq-&gt;ibcq.device;</div><div class='del'>-				ibevent.event = IB_EVENT_CQ_ERR;</div><div class='del'>-				ibevent.element.cq = &amp;iwcq-&gt;ibcq;</div><div class='del'>-				iwcq-&gt;ibcq.event_handler(&amp;ibevent, iwcq-&gt;ibcq.cq_context);</div><div class='del'>-			}</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_AE_LLP_DOUBT_REACHABILITY:</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_AE_PRIV_OPERATION_DENIED:</div><div class='del'>-		case I40IW_AE_STAG_ZERO_INVALID:</div><div class='del'>-		case I40IW_AE_IB_RREQ_AND_Q1_FULL:</div><div class='del'>-		case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION:</div><div class='del'>-		case I40IW_AE_DDP_UBE_INVALID_MO:</div><div class='del'>-		case I40IW_AE_DDP_UBE_INVALID_QN:</div><div class='del'>-		case I40IW_AE_DDP_NO_L_BIT:</div><div class='del'>-		case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:</div><div class='del'>-		case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE:</div><div class='del'>-		case I40IW_AE_ROE_INVALID_RDMA_READ_REQUEST:</div><div class='del'>-		case I40IW_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:</div><div class='del'>-		case I40IW_AE_INVALID_ARP_ENTRY:</div><div class='del'>-		case I40IW_AE_INVALID_TCP_OPTION_RCVD:</div><div class='del'>-		case I40IW_AE_STALE_ARP_ENTRY:</div><div class='del'>-		case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR:</div><div class='del'>-		case I40IW_AE_LLP_SEGMENT_TOO_SMALL:</div><div class='del'>-		case I40IW_AE_LLP_SYN_RECEIVED:</div><div class='del'>-		case I40IW_AE_LLP_TOO_MANY_RETRIES:</div><div class='del'>-		case I40IW_AE_LCE_QP_CATASTROPHIC:</div><div class='del'>-		case I40IW_AE_LCE_FUNCTION_CATASTROPHIC:</div><div class='del'>-		case I40IW_AE_LCE_CQ_CATASTROPHIC:</div><div class='del'>-		case I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG:</div><div class='del'>-		case I40IW_AE_UDA_XMIT_DGRAM_TOO_SHORT:</div><div class='del'>-			ctx_info-&gt;err_rq_idx_valid = false;</div><div class='del'>-			/* fall through */</div><div class='del'>-		default:</div><div class='del'>-			if (!info-&gt;sq &amp;&amp; ctx_info-&gt;err_rq_idx_valid) {</div><div class='del'>-				ctx_info-&gt;err_rq_idx = info-&gt;wqe_idx;</div><div class='del'>-				ctx_info-&gt;tcp_info_valid = false;</div><div class='del'>-				ctx_info-&gt;iwarp_info_valid = false;</div><div class='del'>-				ret = dev-&gt;iw_priv_qp_ops-&gt;qp_setctx(&amp;iwqp-&gt;sc_qp,</div><div class='del'>-								     iwqp-&gt;host_ctx.va,</div><div class='del'>-								     ctx_info);</div><div class='del'>-			}</div><div class='del'>-			i40iw_terminate_connection(qp, info);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		if (info-&gt;qp)</div><div class='del'>-			i40iw_rem_ref(&amp;iwqp-&gt;ibqp);</div><div class='del'>-	} while (1);</div><div class='del'>-</div><div class='del'>-	if (aeqcnt)</div><div class='del'>-		dev-&gt;aeq_ops-&gt;repost_aeq_entries(dev, aeqcnt);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cqp_manage_abvpt_cmd - send cqp command manage abpvt</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @accel_local_port: port for apbvt</div><div class='del'>- * @add_port: add or delete port</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code</div><div class='del'>-i40iw_cqp_manage_abvpt_cmd(struct i40iw_device *iwdev,</div><div class='del'>-			   u16 accel_local_port,</div><div class='del'>-			   bool add_port)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_apbvt_info *info;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, add_port);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	info = &amp;cqp_info-&gt;in.u.manage_apbvt_entry.info;</div><div class='del'>-</div><div class='del'>-	memset(info, 0, sizeof(*info));</div><div class='del'>-	info-&gt;add = add_port;</div><div class='del'>-	info-&gt;port = cpu_to_le16(accel_local_port);</div><div class='del'>-</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_MANAGE_APBVT_ENTRY;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.manage_apbvt_entry.cqp = &amp;iwdev-&gt;cqp.sc_cqp;</div><div class='del'>-	cqp_info-&gt;in.u.manage_apbvt_entry.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("CQP-OP Manage APBVT entry fail");</div><div class='del'>-</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_manage_apbvt - add or delete tcp port</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @accel_local_port: port for apbvt</div><div class='del'>- * @add_port: add or delete port</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_manage_apbvt(struct i40iw_device *iwdev,</div><div class='del'>-					  u16 accel_local_port,</div><div class='del'>-					  bool add_port)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cm_core *cm_core = &amp;iwdev-&gt;cm_core;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	bool in_use;</div><div class='del'>-</div><div class='del'>-	/* apbvt_lock is held across CQP delete APBVT OP (non-waiting) to</div><div class='del'>-	 * protect against race where add APBVT CQP can race ahead of the delete</div><div class='del'>-	 * APBVT for same port.</div><div class='del'>-	 */</div><div class='del'>-	if (add_port) {</div><div class='del'>-		spin_lock_irqsave(&amp;cm_core-&gt;apbvt_lock, flags);</div><div class='del'>-		in_use = __test_and_set_bit(accel_local_port,</div><div class='del'>-					    cm_core-&gt;ports_in_use);</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_core-&gt;apbvt_lock, flags);</div><div class='del'>-		if (in_use)</div><div class='del'>-			return 0;</div><div class='del'>-		return i40iw_cqp_manage_abvpt_cmd(iwdev, accel_local_port,</div><div class='del'>-						  true);</div><div class='del'>-	} else {</div><div class='del'>-		spin_lock_irqsave(&amp;cm_core-&gt;apbvt_lock, flags);</div><div class='del'>-		in_use = i40iw_port_in_use(cm_core, accel_local_port);</div><div class='del'>-		if (in_use) {</div><div class='del'>-			spin_unlock_irqrestore(&amp;cm_core-&gt;apbvt_lock, flags);</div><div class='del'>-			return 0;</div><div class='del'>-		}</div><div class='del'>-		__clear_bit(accel_local_port, cm_core-&gt;ports_in_use);</div><div class='del'>-		status = i40iw_cqp_manage_abvpt_cmd(iwdev, accel_local_port,</div><div class='del'>-						    false);</div><div class='del'>-		spin_unlock_irqrestore(&amp;cm_core-&gt;apbvt_lock, flags);</div><div class='del'>-		return status;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_manage_arp_cache - manage hw arp cache</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @mac_addr: mac address ptr</div><div class='del'>- * @ip_addr: ip addr for arp cache</div><div class='del'>- * @action: add, delete or modify</div><div class='del'>- */</div><div class='del'>-void i40iw_manage_arp_cache(struct i40iw_device *iwdev,</div><div class='del'>-			    unsigned char *mac_addr,</div><div class='del'>-			    u32 *ip_addr,</div><div class='del'>-			    bool ipv4,</div><div class='del'>-			    u32 action)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_add_arp_cache_entry_info *info;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	int arp_index;</div><div class='del'>-</div><div class='del'>-	arp_index = i40iw_arp_table(iwdev, ip_addr, ipv4, mac_addr, action);</div><div class='del'>-	if (arp_index == -1)</div><div class='del'>-		return;</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, false);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	if (action == I40IW_ARP_ADD) {</div><div class='del'>-		cqp_info-&gt;cqp_cmd = OP_ADD_ARP_CACHE_ENTRY;</div><div class='del'>-		info = &amp;cqp_info-&gt;in.u.add_arp_cache_entry.info;</div><div class='del'>-		memset(info, 0, sizeof(*info));</div><div class='del'>-		info-&gt;arp_index = cpu_to_le16((u16)arp_index);</div><div class='del'>-		info-&gt;permanent = true;</div><div class='del'>-		ether_addr_copy(info-&gt;mac_addr, mac_addr);</div><div class='del'>-		cqp_info-&gt;in.u.add_arp_cache_entry.scratch = (uintptr_t)cqp_request;</div><div class='del'>-		cqp_info-&gt;in.u.add_arp_cache_entry.cqp = &amp;iwdev-&gt;cqp.sc_cqp;</div><div class='del'>-	} else {</div><div class='del'>-		cqp_info-&gt;cqp_cmd = OP_DELETE_ARP_CACHE_ENTRY;</div><div class='del'>-		cqp_info-&gt;in.u.del_arp_cache_entry.scratch = (uintptr_t)cqp_request;</div><div class='del'>-		cqp_info-&gt;in.u.del_arp_cache_entry.cqp = &amp;iwdev-&gt;cqp.sc_cqp;</div><div class='del'>-		cqp_info-&gt;in.u.del_arp_cache_entry.arp_index = arp_index;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cqp_info-&gt;in.u.add_arp_cache_entry.cqp = &amp;iwdev-&gt;cqp.sc_cqp;</div><div class='del'>-	cqp_info-&gt;in.u.add_arp_cache_entry.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	if (i40iw_handle_cqp_op(iwdev, cqp_request))</div><div class='del'>-		i40iw_pr_err("CQP-OP Add/Del Arp Cache entry fail");</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_send_syn_cqp_callback - do syn/ack after qhash</div><div class='del'>- * @cqp_request: qhash cqp completion</div><div class='del'>- * @send_ack: flag send ack</div><div class='del'>- */</div><div class='del'>-static void i40iw_send_syn_cqp_callback(struct i40iw_cqp_request *cqp_request, u32 send_ack)</div><div class='del'>-{</div><div class='del'>-	i40iw_send_syn(cqp_request-&gt;param, send_ack);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_manage_qhash - add or modify qhash</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @cminfo: cm info for qhash</div><div class='del'>- * @etype: type (syn or quad)</div><div class='del'>- * @mtype: type of qhash</div><div class='del'>- * @cmnode: cmnode associated with connection</div><div class='del'>- * @wait: wait for completion</div><div class='del'>- * @user_pri:user pri of the connection</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_manage_qhash(struct i40iw_device *iwdev,</div><div class='del'>-					  struct i40iw_cm_info *cminfo,</div><div class='del'>-					  enum i40iw_quad_entry_type etype,</div><div class='del'>-					  enum i40iw_quad_hash_manage_type mtype,</div><div class='del'>-					  void *cmnode,</div><div class='del'>-					  bool wait)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qhash_table_info *info;</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_sc_vsi *vsi = &amp;iwdev-&gt;vsi;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_cqp *iwcqp = &amp;iwdev-&gt;cqp;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(iwcqp, wait);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	info = &amp;cqp_info-&gt;in.u.manage_qhash_table_entry.info;</div><div class='del'>-	memset(info, 0, sizeof(*info));</div><div class='del'>-</div><div class='del'>-	info-&gt;vsi = &amp;iwdev-&gt;vsi;</div><div class='del'>-	info-&gt;manage = mtype;</div><div class='del'>-	info-&gt;entry_type = etype;</div><div class='del'>-	if (cminfo-&gt;vlan_id != 0xFFFF) {</div><div class='del'>-		info-&gt;vlan_valid = true;</div><div class='del'>-		info-&gt;vlan_id = cpu_to_le16(cminfo-&gt;vlan_id);</div><div class='del'>-	} else {</div><div class='del'>-		info-&gt;vlan_valid = false;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	info-&gt;ipv4_valid = cminfo-&gt;ipv4;</div><div class='del'>-	info-&gt;user_pri = cminfo-&gt;user_pri;</div><div class='del'>-	ether_addr_copy(info-&gt;mac_addr, iwdev-&gt;netdev-&gt;dev_addr);</div><div class='del'>-	info-&gt;qp_num = cpu_to_le32(vsi-&gt;ilq-&gt;qp_id);</div><div class='del'>-	info-&gt;dest_port = cpu_to_le16(cminfo-&gt;loc_port);</div><div class='del'>-	info-&gt;dest_ip[0] = cpu_to_le32(cminfo-&gt;loc_addr[0]);</div><div class='del'>-	info-&gt;dest_ip[1] = cpu_to_le32(cminfo-&gt;loc_addr[1]);</div><div class='del'>-	info-&gt;dest_ip[2] = cpu_to_le32(cminfo-&gt;loc_addr[2]);</div><div class='del'>-	info-&gt;dest_ip[3] = cpu_to_le32(cminfo-&gt;loc_addr[3]);</div><div class='del'>-	if (etype == I40IW_QHASH_TYPE_TCP_ESTABLISHED) {</div><div class='del'>-		info-&gt;src_port = cpu_to_le16(cminfo-&gt;rem_port);</div><div class='del'>-		info-&gt;src_ip[0] = cpu_to_le32(cminfo-&gt;rem_addr[0]);</div><div class='del'>-		info-&gt;src_ip[1] = cpu_to_le32(cminfo-&gt;rem_addr[1]);</div><div class='del'>-		info-&gt;src_ip[2] = cpu_to_le32(cminfo-&gt;rem_addr[2]);</div><div class='del'>-		info-&gt;src_ip[3] = cpu_to_le32(cminfo-&gt;rem_addr[3]);</div><div class='del'>-	}</div><div class='del'>-	if (cmnode) {</div><div class='del'>-		cqp_request-&gt;callback_fcn = i40iw_send_syn_cqp_callback;</div><div class='del'>-		cqp_request-&gt;param = (void *)cmnode;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (info-&gt;ipv4_valid)</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_CM,</div><div class='del'>-			    "%s:%s IP=%pI4, port=%d, mac=%pM, vlan_id=%d\n",</div><div class='del'>-			    __func__, (!mtype) ? "DELETE" : "ADD",</div><div class='del'>-			    info-&gt;dest_ip,</div><div class='del'>-			    info-&gt;dest_port, info-&gt;mac_addr, cminfo-&gt;vlan_id);</div><div class='del'>-	else</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_CM,</div><div class='del'>-			    "%s:%s IP=%pI6, port=%d, mac=%pM, vlan_id=%d\n",</div><div class='del'>-			    __func__, (!mtype) ? "DELETE" : "ADD",</div><div class='del'>-			    info-&gt;dest_ip,</div><div class='del'>-			    info-&gt;dest_port, info-&gt;mac_addr, cminfo-&gt;vlan_id);</div><div class='del'>-	cqp_info-&gt;in.u.manage_qhash_table_entry.cqp = &amp;iwdev-&gt;cqp.sc_cqp;</div><div class='del'>-	cqp_info-&gt;in.u.manage_qhash_table_entry.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_MANAGE_QHASH_TABLE_ENTRY;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("CQP-OP Manage Qhash Entry fail");</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hw_flush_wqes - flush qp's wqe</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @qp: hardware control qp</div><div class='del'>- * @info: info for flush</div><div class='del'>- * @wait: flag wait for completion</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_hw_flush_wqes(struct i40iw_device *iwdev,</div><div class='del'>-					   struct i40iw_sc_qp *qp,</div><div class='del'>-					   struct i40iw_qp_flush_info *info,</div><div class='del'>-					   bool wait)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_qp_flush_info *hw_info;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	struct i40iw_qp *iwqp = (struct i40iw_qp *)qp-&gt;back_qp;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, wait);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	hw_info = &amp;cqp_request-&gt;info.in.u.qp_flush_wqes.info;</div><div class='del'>-	memcpy(hw_info, info, sizeof(*hw_info));</div><div class='del'>-</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_QP_FLUSH_WQES;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.qp_flush_wqes.qp = qp;</div><div class='del'>-	cqp_info-&gt;in.u.qp_flush_wqes.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status) {</div><div class='del'>-		i40iw_pr_err("CQP-OP Flush WQE's fail");</div><div class='del'>-		complete(&amp;iwqp-&gt;sq_drained);</div><div class='del'>-		complete(&amp;iwqp-&gt;rq_drained);</div><div class='del'>-		return status;</div><div class='del'>-	}</div><div class='del'>-	if (!cqp_request-&gt;compl_info.maj_err_code) {</div><div class='del'>-		switch (cqp_request-&gt;compl_info.min_err_code) {</div><div class='del'>-		case I40IW_CQP_COMPL_RQ_WQE_FLUSHED:</div><div class='del'>-			complete(&amp;iwqp-&gt;sq_drained);</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_CQP_COMPL_SQ_WQE_FLUSHED:</div><div class='del'>-			complete(&amp;iwqp-&gt;rq_drained);</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_CQP_COMPL_RQ_SQ_WQE_FLUSHED:</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			complete(&amp;iwqp-&gt;sq_drained);</div><div class='del'>-			complete(&amp;iwqp-&gt;rq_drained);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_gen_ae - generate AE</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @qp: qp associated with AE</div><div class='del'>- * @info: info for ae</div><div class='del'>- * @wait: wait for completion</div><div class='del'>- */</div><div class='del'>-void i40iw_gen_ae(struct i40iw_device *iwdev,</div><div class='del'>-		  struct i40iw_sc_qp *qp,</div><div class='del'>-		  struct i40iw_gen_ae_info *info,</div><div class='del'>-		  bool wait)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_gen_ae_info *ae_info;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, wait);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	ae_info = &amp;cqp_request-&gt;info.in.u.gen_ae.info;</div><div class='del'>-	memcpy(ae_info, info, sizeof(*ae_info));</div><div class='del'>-</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_GEN_AE;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.gen_ae.qp = qp;</div><div class='del'>-	cqp_info-&gt;in.u.gen_ae.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	if (i40iw_handle_cqp_op(iwdev, cqp_request))</div><div class='del'>-		i40iw_pr_err("CQP OP failed attempting to generate ae_code=0x%x\n",</div><div class='del'>-			     info-&gt;ae_code);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hw_manage_vf_pble_bp - manage vf pbles</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @info: info for managing pble</div><div class='del'>- * @wait: flag wait for completion</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_hw_manage_vf_pble_bp(struct i40iw_device *iwdev,</div><div class='del'>-						  struct i40iw_manage_vf_pble_info *info,</div><div class='del'>-						  bool wait)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_manage_vf_pble_info *hw_info;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-</div><div class='del'>-	if ((iwdev-&gt;init_state &lt; CCQ_CREATED) &amp;&amp; wait)</div><div class='del'>-		wait = false;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, wait);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	hw_info = &amp;cqp_request-&gt;info.in.u.manage_vf_pble_bp.info;</div><div class='del'>-	memcpy(hw_info, info, sizeof(*hw_info));</div><div class='del'>-</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_MANAGE_VF_PBLE_BP;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.manage_vf_pble_bp.cqp = &amp;iwdev-&gt;cqp.sc_cqp;</div><div class='del'>-	cqp_info-&gt;in.u.manage_vf_pble_bp.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("CQP-OP Manage VF pble_bp fail");</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_ib_wc - return change flush code to IB's</div><div class='del'>- * @opcode: iwarp flush code</div><div class='del'>- */</div><div class='del'>-static enum ib_wc_status i40iw_get_ib_wc(enum i40iw_flush_opcode opcode)</div><div class='del'>-{</div><div class='del'>-	switch (opcode) {</div><div class='del'>-	case FLUSH_PROT_ERR:</div><div class='del'>-		return IB_WC_LOC_PROT_ERR;</div><div class='del'>-	case FLUSH_REM_ACCESS_ERR:</div><div class='del'>-		return IB_WC_REM_ACCESS_ERR;</div><div class='del'>-	case FLUSH_LOC_QP_OP_ERR:</div><div class='del'>-		return IB_WC_LOC_QP_OP_ERR;</div><div class='del'>-	case FLUSH_REM_OP_ERR:</div><div class='del'>-		return IB_WC_REM_OP_ERR;</div><div class='del'>-	case FLUSH_LOC_LEN_ERR:</div><div class='del'>-		return IB_WC_LOC_LEN_ERR;</div><div class='del'>-	case FLUSH_GENERAL_ERR:</div><div class='del'>-		return IB_WC_GENERAL_ERR;</div><div class='del'>-	case FLUSH_FATAL_ERR:</div><div class='del'>-	default:</div><div class='del'>-		return IB_WC_FATAL_ERR;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_set_flush_info - set flush info</div><div class='del'>- * @pinfo: set flush info</div><div class='del'>- * @min: minor err</div><div class='del'>- * @maj: major err</div><div class='del'>- * @opcode: flush error code</div><div class='del'>- */</div><div class='del'>-static void i40iw_set_flush_info(struct i40iw_qp_flush_info *pinfo,</div><div class='del'>-				 u16 *min,</div><div class='del'>-				 u16 *maj,</div><div class='del'>-				 enum i40iw_flush_opcode opcode)</div><div class='del'>-{</div><div class='del'>-	*min = (u16)i40iw_get_ib_wc(opcode);</div><div class='del'>-	*maj = CQE_MAJOR_DRV;</div><div class='del'>-	pinfo-&gt;userflushcode = true;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_flush_wqes - flush wqe for qp</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @iwqp: qp to flush wqes</div><div class='del'>- */</div><div class='del'>-void i40iw_flush_wqes(struct i40iw_device *iwdev, struct i40iw_qp *iwqp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp_flush_info info;</div><div class='del'>-	struct i40iw_qp_flush_info *pinfo = &amp;info;</div><div class='del'>-</div><div class='del'>-	struct i40iw_sc_qp *qp = &amp;iwqp-&gt;sc_qp;</div><div class='del'>-</div><div class='del'>-	memset(pinfo, 0, sizeof(*pinfo));</div><div class='del'>-	info.sq = true;</div><div class='del'>-	info.rq = true;</div><div class='del'>-	if (qp-&gt;term_flags) {</div><div class='del'>-		i40iw_set_flush_info(pinfo, &amp;pinfo-&gt;sq_minor_code,</div><div class='del'>-				     &amp;pinfo-&gt;sq_major_code, qp-&gt;flush_code);</div><div class='del'>-		i40iw_set_flush_info(pinfo, &amp;pinfo-&gt;rq_minor_code,</div><div class='del'>-				     &amp;pinfo-&gt;rq_major_code, qp-&gt;flush_code);</div><div class='del'>-	}</div><div class='del'>-	(void)i40iw_hw_flush_wqes(iwdev, &amp;iwqp-&gt;sc_qp, &amp;info, true);</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_main.c b/drivers/infiniband/hw/i40iw/i40iw_main.c<br/>deleted file mode 100644<br/>index 10932baee279..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_main.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_main.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,2066 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='del'>-#include &lt;linux/moduleparam.h&gt;</div><div class='del'>-#include &lt;linux/netdevice.h&gt;</div><div class='del'>-#include &lt;linux/etherdevice.h&gt;</div><div class='del'>-#include &lt;linux/ip.h&gt;</div><div class='del'>-#include &lt;linux/tcp.h&gt;</div><div class='del'>-#include &lt;linux/if_vlan.h&gt;</div><div class='del'>-#include &lt;net/addrconf.h&gt;</div><div class='del'>-</div><div class='del'>-#include "i40iw.h"</div><div class='del'>-#include "i40iw_register.h"</div><div class='del'>-#include &lt;net/netevent.h&gt;</div><div class='del'>-#define CLIENT_IW_INTERFACE_VERSION_MAJOR 0</div><div class='del'>-#define CLIENT_IW_INTERFACE_VERSION_MINOR 01</div><div class='del'>-#define CLIENT_IW_INTERFACE_VERSION_BUILD 00</div><div class='del'>-</div><div class='del'>-#define DRV_VERSION_MAJOR 0</div><div class='del'>-#define DRV_VERSION_MINOR 5</div><div class='del'>-#define DRV_VERSION_BUILD 123</div><div class='del'>-#define DRV_VERSION	__stringify(DRV_VERSION_MAJOR) "."		\</div><div class='del'>-	__stringify(DRV_VERSION_MINOR) "." __stringify(DRV_VERSION_BUILD)</div><div class='del'>-</div><div class='del'>-static int push_mode;</div><div class='del'>-module_param(push_mode, int, 0644);</div><div class='del'>-MODULE_PARM_DESC(push_mode, "Low latency mode: 0=disabled (default), 1=enabled)");</div><div class='del'>-</div><div class='del'>-static int debug;</div><div class='del'>-module_param(debug, int, 0644);</div><div class='del'>-MODULE_PARM_DESC(debug, "debug flags: 0=disabled (default), 0x7fffffff=all");</div><div class='del'>-</div><div class='del'>-static int resource_profile;</div><div class='del'>-module_param(resource_profile, int, 0644);</div><div class='del'>-MODULE_PARM_DESC(resource_profile,</div><div class='del'>-		 "Resource Profile: 0=no VF RDMA support (default), 1=Weighted VF, 2=Even Distribution");</div><div class='del'>-</div><div class='del'>-static int max_rdma_vfs = 32;</div><div class='del'>-module_param(max_rdma_vfs, int, 0644);</div><div class='del'>-MODULE_PARM_DESC(max_rdma_vfs, "Maximum VF count: 0-32 32=default");</div><div class='del'>-static int mpa_version = 2;</div><div class='del'>-module_param(mpa_version, int, 0644);</div><div class='del'>-MODULE_PARM_DESC(mpa_version, "MPA version to be used in MPA Req/Resp 1 or 2");</div><div class='del'>-</div><div class='del'>-MODULE_AUTHOR("Intel Corporation, &lt;e1000-rdma@lists.sourceforge.net&gt;");</div><div class='del'>-MODULE_DESCRIPTION("Intel(R) Ethernet Connection X722 iWARP RDMA Driver");</div><div class='del'>-MODULE_LICENSE("Dual BSD/GPL");</div><div class='del'>-</div><div class='del'>-static struct i40e_client i40iw_client;</div><div class='del'>-static char i40iw_client_name[I40E_CLIENT_STR_LENGTH] = "i40iw";</div><div class='del'>-</div><div class='del'>-static LIST_HEAD(i40iw_handlers);</div><div class='del'>-static spinlock_t i40iw_handler_lock;</div><div class='del'>-</div><div class='del'>-static enum i40iw_status_code i40iw_virtchnl_send(struct i40iw_sc_dev *dev,</div><div class='del'>-						  u32 vf_id, u8 *msg, u16 len);</div><div class='del'>-</div><div class='del'>-static struct notifier_block i40iw_inetaddr_notifier = {</div><div class='del'>-	.notifier_call = i40iw_inetaddr_event</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct notifier_block i40iw_inetaddr6_notifier = {</div><div class='del'>-	.notifier_call = i40iw_inet6addr_event</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct notifier_block i40iw_net_notifier = {</div><div class='del'>-	.notifier_call = i40iw_net_event</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct notifier_block i40iw_netdevice_notifier = {</div><div class='del'>-	.notifier_call = i40iw_netdevice_event</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_find_i40e_handler - find a handler given a client info</div><div class='del'>- * @ldev: pointer to a client info</div><div class='del'>- */</div><div class='del'>-static struct i40iw_handler *i40iw_find_i40e_handler(struct i40e_info *ldev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_handler *hdl;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;i40iw_handler_lock, flags);</div><div class='del'>-	list_for_each_entry(hdl, &amp;i40iw_handlers, list) {</div><div class='del'>-		if (hdl-&gt;ldev.netdev == ldev-&gt;netdev) {</div><div class='del'>-			spin_unlock_irqrestore(&amp;i40iw_handler_lock, flags);</div><div class='del'>-			return hdl;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;i40iw_handler_lock, flags);</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_find_netdev - find a handler given a netdev</div><div class='del'>- * @netdev: pointer to net_device</div><div class='del'>- */</div><div class='del'>-struct i40iw_handler *i40iw_find_netdev(struct net_device *netdev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_handler *hdl;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;i40iw_handler_lock, flags);</div><div class='del'>-	list_for_each_entry(hdl, &amp;i40iw_handlers, list) {</div><div class='del'>-		if (hdl-&gt;ldev.netdev == netdev) {</div><div class='del'>-			spin_unlock_irqrestore(&amp;i40iw_handler_lock, flags);</div><div class='del'>-			return hdl;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;i40iw_handler_lock, flags);</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_add_handler - add a handler to the list</div><div class='del'>- * @hdl: handler to be added to the handler list</div><div class='del'>- */</div><div class='del'>-static void i40iw_add_handler(struct i40iw_handler *hdl)</div><div class='del'>-{</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;i40iw_handler_lock, flags);</div><div class='del'>-	list_add(&amp;hdl-&gt;list, &amp;i40iw_handlers);</div><div class='del'>-	spin_unlock_irqrestore(&amp;i40iw_handler_lock, flags);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_del_handler - delete a handler from the list</div><div class='del'>- * @hdl: handler to be deleted from the handler list</div><div class='del'>- */</div><div class='del'>-static int i40iw_del_handler(struct i40iw_handler *hdl)</div><div class='del'>-{</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;i40iw_handler_lock, flags);</div><div class='del'>-	list_del(&amp;hdl-&gt;list);</div><div class='del'>-	spin_unlock_irqrestore(&amp;i40iw_handler_lock, flags);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_enable_intr - set up device interrupts</div><div class='del'>- * @dev: hardware control device structure</div><div class='del'>- * @msix_id: id of the interrupt to be enabled</div><div class='del'>- */</div><div class='del'>-static void i40iw_enable_intr(struct i40iw_sc_dev *dev, u32 msix_id)</div><div class='del'>-{</div><div class='del'>-	u32 val;</div><div class='del'>-</div><div class='del'>-	val = I40E_PFINT_DYN_CTLN_INTENA_MASK |</div><div class='del'>-		I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |</div><div class='del'>-		(3 &lt;&lt; I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);</div><div class='del'>-	if (dev-&gt;is_pf)</div><div class='del'>-		i40iw_wr32(dev-&gt;hw, I40E_PFINT_DYN_CTLN(msix_id - 1), val);</div><div class='del'>-	else</div><div class='del'>-		i40iw_wr32(dev-&gt;hw, I40E_VFINT_DYN_CTLN1(msix_id - 1), val);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_dpc - tasklet for aeq and ceq 0</div><div class='del'>- * @data: iwarp device</div><div class='del'>- */</div><div class='del'>-static void i40iw_dpc(unsigned long data)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)data;</div><div class='del'>-</div><div class='del'>-	if (iwdev-&gt;msix_shared)</div><div class='del'>-		i40iw_process_ceq(iwdev, iwdev-&gt;ceqlist);</div><div class='del'>-	i40iw_process_aeq(iwdev);</div><div class='del'>-	i40iw_enable_intr(&amp;iwdev-&gt;sc_dev, iwdev-&gt;iw_msixtbl[0].idx);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ceq_dpc - dpc handler for CEQ</div><div class='del'>- * @data: data points to CEQ</div><div class='del'>- */</div><div class='del'>-static void i40iw_ceq_dpc(unsigned long data)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_ceq *iwceq = (struct i40iw_ceq *)data;</div><div class='del'>-	struct i40iw_device *iwdev = iwceq-&gt;iwdev;</div><div class='del'>-</div><div class='del'>-	i40iw_process_ceq(iwdev, iwceq);</div><div class='del'>-	i40iw_enable_intr(&amp;iwdev-&gt;sc_dev, iwceq-&gt;msix_idx);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_irq_handler - interrupt handler for aeq and ceq0</div><div class='del'>- * @irq: Interrupt request number</div><div class='del'>- * @data: iwarp device</div><div class='del'>- */</div><div class='del'>-static irqreturn_t i40iw_irq_handler(int irq, void *data)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)data;</div><div class='del'>-</div><div class='del'>-	tasklet_schedule(&amp;iwdev-&gt;dpc_tasklet);</div><div class='del'>-	return IRQ_HANDLED;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_destroy_cqp  - destroy control qp</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @create_done: 1 if cqp create poll was success</div><div class='del'>- *</div><div class='del'>- * Issue destroy cqp request and</div><div class='del'>- * free the resources associated with the cqp</div><div class='del'>- */</div><div class='del'>-static void i40iw_destroy_cqp(struct i40iw_device *iwdev, bool free_hwcqp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_cqp *cqp = &amp;iwdev-&gt;cqp;</div><div class='del'>-</div><div class='del'>-	if (free_hwcqp)</div><div class='del'>-		dev-&gt;cqp_ops-&gt;cqp_destroy(dev-&gt;cqp);</div><div class='del'>-</div><div class='del'>-	i40iw_cleanup_pending_cqp_op(iwdev);</div><div class='del'>-</div><div class='del'>-	i40iw_free_dma_mem(dev-&gt;hw, &amp;cqp-&gt;sq);</div><div class='del'>-	kfree(cqp-&gt;scratch_array);</div><div class='del'>-	iwdev-&gt;cqp.scratch_array = NULL;</div><div class='del'>-</div><div class='del'>-	kfree(cqp-&gt;cqp_requests);</div><div class='del'>-	cqp-&gt;cqp_requests = NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_disable_irqs - disable device interrupts</div><div class='del'>- * @dev: hardware control device structure</div><div class='del'>- * @msic_vec: msix vector to disable irq</div><div class='del'>- * @dev_id: parameter to pass to free_irq (used during irq setup)</div><div class='del'>- *</div><div class='del'>- * The function is called when destroying aeq/ceq</div><div class='del'>- */</div><div class='del'>-static void i40iw_disable_irq(struct i40iw_sc_dev *dev,</div><div class='del'>-			      struct i40iw_msix_vector *msix_vec,</div><div class='del'>-			      void *dev_id)</div><div class='del'>-{</div><div class='del'>-	if (dev-&gt;is_pf)</div><div class='del'>-		i40iw_wr32(dev-&gt;hw, I40E_PFINT_DYN_CTLN(msix_vec-&gt;idx - 1), 0);</div><div class='del'>-	else</div><div class='del'>-		i40iw_wr32(dev-&gt;hw, I40E_VFINT_DYN_CTLN1(msix_vec-&gt;idx - 1), 0);</div><div class='del'>-	irq_set_affinity_hint(msix_vec-&gt;irq, NULL);</div><div class='del'>-	free_irq(msix_vec-&gt;irq, dev_id);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_destroy_aeq - destroy aeq</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- *</div><div class='del'>- * Issue a destroy aeq request and</div><div class='del'>- * free the resources associated with the aeq</div><div class='del'>- * The function is called during driver unload</div><div class='del'>- */</div><div class='del'>-static void i40iw_destroy_aeq(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status = I40IW_ERR_NOT_READY;</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_aeq *aeq = &amp;iwdev-&gt;aeq;</div><div class='del'>-</div><div class='del'>-	if (!iwdev-&gt;msix_shared)</div><div class='del'>-		i40iw_disable_irq(dev, iwdev-&gt;iw_msixtbl, (void *)iwdev);</div><div class='del'>-	if (iwdev-&gt;reset)</div><div class='del'>-		goto exit;</div><div class='del'>-</div><div class='del'>-	if (!dev-&gt;aeq_ops-&gt;aeq_destroy(&amp;aeq-&gt;sc_aeq, 0, 1))</div><div class='del'>-		status = dev-&gt;aeq_ops-&gt;aeq_destroy_done(&amp;aeq-&gt;sc_aeq);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("destroy aeq failed %d\n", status);</div><div class='del'>-</div><div class='del'>-exit:</div><div class='del'>-	i40iw_free_dma_mem(dev-&gt;hw, &amp;aeq-&gt;mem);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_destroy_ceq - destroy ceq</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @iwceq: ceq to be destroyed</div><div class='del'>- *</div><div class='del'>- * Issue a destroy ceq request and</div><div class='del'>- * free the resources associated with the ceq</div><div class='del'>- */</div><div class='del'>-static void i40iw_destroy_ceq(struct i40iw_device *iwdev,</div><div class='del'>-			      struct i40iw_ceq *iwceq)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-</div><div class='del'>-	if (iwdev-&gt;reset)</div><div class='del'>-		goto exit;</div><div class='del'>-</div><div class='del'>-	status = dev-&gt;ceq_ops-&gt;ceq_destroy(&amp;iwceq-&gt;sc_ceq, 0, 1);</div><div class='del'>-	if (status) {</div><div class='del'>-		i40iw_pr_err("ceq destroy command failed %d\n", status);</div><div class='del'>-		goto exit;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	status = dev-&gt;ceq_ops-&gt;cceq_destroy_done(&amp;iwceq-&gt;sc_ceq);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("ceq destroy completion failed %d\n", status);</div><div class='del'>-exit:</div><div class='del'>-	i40iw_free_dma_mem(dev-&gt;hw, &amp;iwceq-&gt;mem);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_dele_ceqs - destroy all ceq's</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- *</div><div class='del'>- * Go through all of the device ceq's and for each ceq</div><div class='del'>- * disable the ceq interrupt and destroy the ceq</div><div class='del'>- */</div><div class='del'>-static void i40iw_dele_ceqs(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	u32 i = 0;</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_ceq *iwceq = iwdev-&gt;ceqlist;</div><div class='del'>-	struct i40iw_msix_vector *msix_vec = iwdev-&gt;iw_msixtbl;</div><div class='del'>-</div><div class='del'>-	if (iwdev-&gt;msix_shared) {</div><div class='del'>-		i40iw_disable_irq(dev, msix_vec, (void *)iwdev);</div><div class='del'>-		i40iw_destroy_ceq(iwdev, iwceq);</div><div class='del'>-		iwceq++;</div><div class='del'>-		i++;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	for (msix_vec++; i &lt; iwdev-&gt;ceqs_count; i++, msix_vec++, iwceq++) {</div><div class='del'>-		i40iw_disable_irq(dev, msix_vec, (void *)iwceq);</div><div class='del'>-		i40iw_destroy_ceq(iwdev, iwceq);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	iwdev-&gt;sc_dev.ceq_valid = false;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_destroy_ccq - destroy control cq</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- *</div><div class='del'>- * Issue destroy ccq request and</div><div class='del'>- * free the resources associated with the ccq</div><div class='del'>- */</div><div class='del'>-static void i40iw_destroy_ccq(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_ccq *ccq = &amp;iwdev-&gt;ccq;</div><div class='del'>-	enum i40iw_status_code status = 0;</div><div class='del'>-</div><div class='del'>-	if (!iwdev-&gt;reset)</div><div class='del'>-		status = dev-&gt;ccq_ops-&gt;ccq_destroy(dev-&gt;ccq, 0, true);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("ccq destroy failed %d\n", status);</div><div class='del'>-	i40iw_free_dma_mem(dev-&gt;hw, &amp;ccq-&gt;mem_cq);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* types of hmc objects */</div><div class='del'>-static enum i40iw_hmc_rsrc_type iw_hmc_obj_types[] = {</div><div class='del'>-	I40IW_HMC_IW_QP,</div><div class='del'>-	I40IW_HMC_IW_CQ,</div><div class='del'>-	I40IW_HMC_IW_HTE,</div><div class='del'>-	I40IW_HMC_IW_ARP,</div><div class='del'>-	I40IW_HMC_IW_APBVT_ENTRY,</div><div class='del'>-	I40IW_HMC_IW_MR,</div><div class='del'>-	I40IW_HMC_IW_XF,</div><div class='del'>-	I40IW_HMC_IW_XFFL,</div><div class='del'>-	I40IW_HMC_IW_Q1,</div><div class='del'>-	I40IW_HMC_IW_Q1FL,</div><div class='del'>-	I40IW_HMC_IW_TIMER,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_close_hmc_objects_type - delete hmc objects of a given type</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @obj_type: the hmc object type to be deleted</div><div class='del'>- * @is_pf: true if the function is PF otherwise false</div><div class='del'>- * @reset: true if called before reset</div><div class='del'>- */</div><div class='del'>-static void i40iw_close_hmc_objects_type(struct i40iw_sc_dev *dev,</div><div class='del'>-					 enum i40iw_hmc_rsrc_type obj_type,</div><div class='del'>-					 struct i40iw_hmc_info *hmc_info,</div><div class='del'>-					 bool is_pf,</div><div class='del'>-					 bool reset)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_hmc_del_obj_info info;</div><div class='del'>-</div><div class='del'>-	memset(&amp;info, 0, sizeof(info));</div><div class='del'>-	info.hmc_info = hmc_info;</div><div class='del'>-	info.rsrc_type = obj_type;</div><div class='del'>-	info.count = hmc_info-&gt;hmc_obj[obj_type].cnt;</div><div class='del'>-	info.is_pf = is_pf;</div><div class='del'>-	if (dev-&gt;hmc_ops-&gt;del_hmc_object(dev, &amp;info, reset))</div><div class='del'>-		i40iw_pr_err("del obj of type %d failed\n", obj_type);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_del_hmc_objects - remove all device hmc objects</div><div class='del'>- * @dev: iwarp device</div><div class='del'>- * @hmc_info: hmc_info to free</div><div class='del'>- * @is_pf: true if hmc_info belongs to PF, not vf nor allocated</div><div class='del'>- *	   by PF on behalf of VF</div><div class='del'>- * @reset: true if called before reset</div><div class='del'>- */</div><div class='del'>-static void i40iw_del_hmc_objects(struct i40iw_sc_dev *dev,</div><div class='del'>-				  struct i40iw_hmc_info *hmc_info,</div><div class='del'>-				  bool is_pf,</div><div class='del'>-				  bool reset)</div><div class='del'>-{</div><div class='del'>-	unsigned int i;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; IW_HMC_OBJ_TYPE_NUM; i++)</div><div class='del'>-		i40iw_close_hmc_objects_type(dev, iw_hmc_obj_types[i], hmc_info, is_pf, reset);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ceq_handler - interrupt handler for ceq</div><div class='del'>- * @data: ceq pointer</div><div class='del'>- */</div><div class='del'>-static irqreturn_t i40iw_ceq_handler(int irq, void *data)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_ceq *iwceq = (struct i40iw_ceq *)data;</div><div class='del'>-</div><div class='del'>-	if (iwceq-&gt;irq != irq)</div><div class='del'>-		i40iw_pr_err("expected irq = %d received irq = %d\n", iwceq-&gt;irq, irq);</div><div class='del'>-	tasklet_schedule(&amp;iwceq-&gt;dpc_tasklet);</div><div class='del'>-	return IRQ_HANDLED;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_create_hmc_obj_type - create hmc object of a given type</div><div class='del'>- * @dev: hardware control device structure</div><div class='del'>- * @info: information for the hmc object to create</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_create_hmc_obj_type(struct i40iw_sc_dev *dev,</div><div class='del'>-							struct i40iw_hmc_create_obj_info *info)</div><div class='del'>-{</div><div class='del'>-	return dev-&gt;hmc_ops-&gt;create_hmc_object(dev, info);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_create_hmc_objs - create all hmc objects for the device</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @is_pf: true if the function is PF otherwise false</div><div class='del'>- *</div><div class='del'>- * Create the device hmc objects and allocate hmc pages</div><div class='del'>- * Return 0 if successful, otherwise clean up and return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_create_hmc_objs(struct i40iw_device *iwdev,</div><div class='del'>-						    bool is_pf)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_hmc_create_obj_info info;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	memset(&amp;info, 0, sizeof(info));</div><div class='del'>-	info.hmc_info = dev-&gt;hmc_info;</div><div class='del'>-	info.is_pf = is_pf;</div><div class='del'>-	info.entry_type = iwdev-&gt;sd_type;</div><div class='del'>-	for (i = 0; i &lt; IW_HMC_OBJ_TYPE_NUM; i++) {</div><div class='del'>-		info.rsrc_type = iw_hmc_obj_types[i];</div><div class='del'>-		info.count = dev-&gt;hmc_info-&gt;hmc_obj[info.rsrc_type].cnt;</div><div class='del'>-		info.add_sd_cnt = 0;</div><div class='del'>-		status = i40iw_create_hmc_obj_type(dev, &amp;info);</div><div class='del'>-		if (status) {</div><div class='del'>-			i40iw_pr_err("create obj type %d status = %d\n",</div><div class='del'>-				     iw_hmc_obj_types[i], status);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	if (!status)</div><div class='del'>-		return (dev-&gt;cqp_misc_ops-&gt;static_hmc_pages_allocated(dev-&gt;cqp, 0,</div><div class='del'>-								      dev-&gt;hmc_fn_id,</div><div class='del'>-								      true, true));</div><div class='del'>-</div><div class='del'>-	while (i) {</div><div class='del'>-		i--;</div><div class='del'>-		/* destroy the hmc objects of a given type */</div><div class='del'>-		i40iw_close_hmc_objects_type(dev,</div><div class='del'>-					     iw_hmc_obj_types[i],</div><div class='del'>-					     dev-&gt;hmc_info,</div><div class='del'>-					     is_pf,</div><div class='del'>-					     false);</div><div class='del'>-	}</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_obj_aligned_mem - get aligned memory from device allocated memory</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @memptr: points to the memory addresses</div><div class='del'>- * @size: size of memory needed</div><div class='del'>- * @mask: mask for the aligned memory</div><div class='del'>- *</div><div class='del'>- * Get aligned memory of the requested size and</div><div class='del'>- * update the memptr to point to the new aligned memory</div><div class='del'>- * Return 0 if successful, otherwise return no memory error</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_obj_aligned_mem(struct i40iw_device *iwdev,</div><div class='del'>-					     struct i40iw_dma_mem *memptr,</div><div class='del'>-					     u32 size,</div><div class='del'>-					     u32 mask)</div><div class='del'>-{</div><div class='del'>-	unsigned long va, newva;</div><div class='del'>-	unsigned long extra;</div><div class='del'>-</div><div class='del'>-	va = (unsigned long)iwdev-&gt;obj_next.va;</div><div class='del'>-	newva = va;</div><div class='del'>-	if (mask)</div><div class='del'>-		newva = ALIGN(va, (mask + 1));</div><div class='del'>-	extra = newva - va;</div><div class='del'>-	memptr-&gt;va = (u8 *)va + extra;</div><div class='del'>-	memptr-&gt;pa = iwdev-&gt;obj_next.pa + extra;</div><div class='del'>-	memptr-&gt;size = size;</div><div class='del'>-	if ((memptr-&gt;va + size) &gt; (iwdev-&gt;obj_mem.va + iwdev-&gt;obj_mem.size))</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-</div><div class='del'>-	iwdev-&gt;obj_next.va = memptr-&gt;va + size;</div><div class='del'>-	iwdev-&gt;obj_next.pa = memptr-&gt;pa + size;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_create_cqp - create control qp</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- *</div><div class='del'>- * Return 0, if the cqp and all the resources associated with it</div><div class='del'>- * are successfully created, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_create_cqp(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	u32 sqsize = I40IW_CQP_SW_SQSIZE_2048;</div><div class='del'>-	struct i40iw_dma_mem mem;</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_cqp_init_info cqp_init_info;</div><div class='del'>-	struct i40iw_cqp *cqp = &amp;iwdev-&gt;cqp;</div><div class='del'>-	u16 maj_err, min_err;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	cqp-&gt;cqp_requests = kcalloc(sqsize, sizeof(*cqp-&gt;cqp_requests), GFP_KERNEL);</div><div class='del'>-	if (!cqp-&gt;cqp_requests)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	cqp-&gt;scratch_array = kcalloc(sqsize, sizeof(*cqp-&gt;scratch_array), GFP_KERNEL);</div><div class='del'>-	if (!cqp-&gt;scratch_array) {</div><div class='del'>-		kfree(cqp-&gt;cqp_requests);</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	}</div><div class='del'>-	dev-&gt;cqp = &amp;cqp-&gt;sc_cqp;</div><div class='del'>-	dev-&gt;cqp-&gt;dev = dev;</div><div class='del'>-	memset(&amp;cqp_init_info, 0, sizeof(cqp_init_info));</div><div class='del'>-	status = i40iw_allocate_dma_mem(dev-&gt;hw, &amp;cqp-&gt;sq,</div><div class='del'>-					(sizeof(struct i40iw_cqp_sq_wqe) * sqsize),</div><div class='del'>-					I40IW_CQP_ALIGNMENT);</div><div class='del'>-	if (status)</div><div class='del'>-		goto exit;</div><div class='del'>-	status = i40iw_obj_aligned_mem(iwdev, &amp;mem, sizeof(struct i40iw_cqp_ctx),</div><div class='del'>-				       I40IW_HOST_CTX_ALIGNMENT_MASK);</div><div class='del'>-	if (status)</div><div class='del'>-		goto exit;</div><div class='del'>-	dev-&gt;cqp-&gt;host_ctx_pa = mem.pa;</div><div class='del'>-	dev-&gt;cqp-&gt;host_ctx = mem.va;</div><div class='del'>-	/* populate the cqp init info */</div><div class='del'>-	cqp_init_info.dev = dev;</div><div class='del'>-	cqp_init_info.sq_size = sqsize;</div><div class='del'>-	cqp_init_info.sq = cqp-&gt;sq.va;</div><div class='del'>-	cqp_init_info.sq_pa = cqp-&gt;sq.pa;</div><div class='del'>-	cqp_init_info.host_ctx_pa = mem.pa;</div><div class='del'>-	cqp_init_info.host_ctx = mem.va;</div><div class='del'>-	cqp_init_info.hmc_profile = iwdev-&gt;resource_profile;</div><div class='del'>-	cqp_init_info.enabled_vf_count = iwdev-&gt;max_rdma_vfs;</div><div class='del'>-	cqp_init_info.scratch_array = cqp-&gt;scratch_array;</div><div class='del'>-	status = dev-&gt;cqp_ops-&gt;cqp_init(dev-&gt;cqp, &amp;cqp_init_info);</div><div class='del'>-	if (status) {</div><div class='del'>-		i40iw_pr_err("cqp init status %d\n", status);</div><div class='del'>-		goto exit;</div><div class='del'>-	}</div><div class='del'>-	status = dev-&gt;cqp_ops-&gt;cqp_create(dev-&gt;cqp, &amp;maj_err, &amp;min_err);</div><div class='del'>-	if (status) {</div><div class='del'>-		i40iw_pr_err("cqp create status %d maj_err %d min_err %d\n",</div><div class='del'>-			     status, maj_err, min_err);</div><div class='del'>-		goto exit;</div><div class='del'>-	}</div><div class='del'>-	spin_lock_init(&amp;cqp-&gt;req_lock);</div><div class='del'>-	INIT_LIST_HEAD(&amp;cqp-&gt;cqp_avail_reqs);</div><div class='del'>-	INIT_LIST_HEAD(&amp;cqp-&gt;cqp_pending_reqs);</div><div class='del'>-	/* init the waitq of the cqp_requests and add them to the list */</div><div class='del'>-	for (i = 0; i &lt; sqsize; i++) {</div><div class='del'>-		init_waitqueue_head(&amp;cqp-&gt;cqp_requests[i].waitq);</div><div class='del'>-		list_add_tail(&amp;cqp-&gt;cqp_requests[i].list, &amp;cqp-&gt;cqp_avail_reqs);</div><div class='del'>-	}</div><div class='del'>-	return 0;</div><div class='del'>-exit:</div><div class='del'>-	/* clean up the created resources */</div><div class='del'>-	i40iw_destroy_cqp(iwdev, false);</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_create_ccq - create control cq</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- *</div><div class='del'>- * Return 0, if the ccq and the resources associated with it</div><div class='del'>- * are successfully created, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_create_ccq(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_dma_mem mem;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_ccq_init_info info;</div><div class='del'>-	struct i40iw_ccq *ccq = &amp;iwdev-&gt;ccq;</div><div class='del'>-</div><div class='del'>-	memset(&amp;info, 0, sizeof(info));</div><div class='del'>-	dev-&gt;ccq = &amp;ccq-&gt;sc_cq;</div><div class='del'>-	dev-&gt;ccq-&gt;dev = dev;</div><div class='del'>-	info.dev = dev;</div><div class='del'>-	ccq-&gt;shadow_area.size = sizeof(struct i40iw_cq_shadow_area);</div><div class='del'>-	ccq-&gt;mem_cq.size = sizeof(struct i40iw_cqe) * IW_CCQ_SIZE;</div><div class='del'>-	status = i40iw_allocate_dma_mem(dev-&gt;hw, &amp;ccq-&gt;mem_cq,</div><div class='del'>-					ccq-&gt;mem_cq.size, I40IW_CQ0_ALIGNMENT);</div><div class='del'>-	if (status)</div><div class='del'>-		goto exit;</div><div class='del'>-	status = i40iw_obj_aligned_mem(iwdev, &amp;mem, ccq-&gt;shadow_area.size,</div><div class='del'>-				       I40IW_SHADOWAREA_MASK);</div><div class='del'>-	if (status)</div><div class='del'>-		goto exit;</div><div class='del'>-	ccq-&gt;sc_cq.back_cq = (void *)ccq;</div><div class='del'>-	/* populate the ccq init info */</div><div class='del'>-	info.cq_base = ccq-&gt;mem_cq.va;</div><div class='del'>-	info.cq_pa = ccq-&gt;mem_cq.pa;</div><div class='del'>-	info.num_elem = IW_CCQ_SIZE;</div><div class='del'>-	info.shadow_area = mem.va;</div><div class='del'>-	info.shadow_area_pa = mem.pa;</div><div class='del'>-	info.ceqe_mask = false;</div><div class='del'>-	info.ceq_id_valid = true;</div><div class='del'>-	info.shadow_read_threshold = 16;</div><div class='del'>-	status = dev-&gt;ccq_ops-&gt;ccq_init(dev-&gt;ccq, &amp;info);</div><div class='del'>-	if (!status)</div><div class='del'>-		status = dev-&gt;ccq_ops-&gt;ccq_create(dev-&gt;ccq, 0, true, true);</div><div class='del'>-exit:</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_free_dma_mem(dev-&gt;hw, &amp;ccq-&gt;mem_cq);</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_configure_ceq_vector - set up the msix interrupt vector for ceq</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @msix_vec: interrupt vector information</div><div class='del'>- * @iwceq: ceq associated with the vector</div><div class='del'>- * @ceq_id: the id number of the iwceq</div><div class='del'>- *</div><div class='del'>- * Allocate interrupt resources and enable irq handling</div><div class='del'>- * Return 0 if successful, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_configure_ceq_vector(struct i40iw_device *iwdev,</div><div class='del'>-							 struct i40iw_ceq *iwceq,</div><div class='del'>-							 u32 ceq_id,</div><div class='del'>-							 struct i40iw_msix_vector *msix_vec)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-</div><div class='del'>-	if (iwdev-&gt;msix_shared &amp;&amp; !ceq_id) {</div><div class='del'>-		tasklet_init(&amp;iwdev-&gt;dpc_tasklet, i40iw_dpc, (unsigned long)iwdev);</div><div class='del'>-		status = request_irq(msix_vec-&gt;irq, i40iw_irq_handler, 0, "AEQCEQ", iwdev);</div><div class='del'>-	} else {</div><div class='del'>-		tasklet_init(&amp;iwceq-&gt;dpc_tasklet, i40iw_ceq_dpc, (unsigned long)iwceq);</div><div class='del'>-		status = request_irq(msix_vec-&gt;irq, i40iw_ceq_handler, 0, "CEQ", iwceq);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cpumask_clear(&amp;msix_vec-&gt;mask);</div><div class='del'>-	cpumask_set_cpu(msix_vec-&gt;cpu_affinity, &amp;msix_vec-&gt;mask);</div><div class='del'>-	irq_set_affinity_hint(msix_vec-&gt;irq, &amp;msix_vec-&gt;mask);</div><div class='del'>-</div><div class='del'>-	if (status) {</div><div class='del'>-		i40iw_pr_err("ceq irq config fail\n");</div><div class='del'>-		return I40IW_ERR_CONFIG;</div><div class='del'>-	}</div><div class='del'>-	msix_vec-&gt;ceq_id = ceq_id;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_create_ceq - create completion event queue</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @iwceq: pointer to the ceq resources to be created</div><div class='del'>- * @ceq_id: the id number of the iwceq</div><div class='del'>- *</div><div class='del'>- * Return 0, if the ceq and the resources associated with it</div><div class='del'>- * are successfully created, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_create_ceq(struct i40iw_device *iwdev,</div><div class='del'>-					       struct i40iw_ceq *iwceq,</div><div class='del'>-					       u32 ceq_id)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_ceq_init_info info;</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	u64 scratch;</div><div class='del'>-</div><div class='del'>-	memset(&amp;info, 0, sizeof(info));</div><div class='del'>-	info.ceq_id = ceq_id;</div><div class='del'>-	iwceq-&gt;iwdev = iwdev;</div><div class='del'>-	iwceq-&gt;mem.size = sizeof(struct i40iw_ceqe) *</div><div class='del'>-		iwdev-&gt;sc_dev.hmc_info-&gt;hmc_obj[I40IW_HMC_IW_CQ].cnt;</div><div class='del'>-	status = i40iw_allocate_dma_mem(dev-&gt;hw, &amp;iwceq-&gt;mem, iwceq-&gt;mem.size,</div><div class='del'>-					I40IW_CEQ_ALIGNMENT);</div><div class='del'>-	if (status)</div><div class='del'>-		goto exit;</div><div class='del'>-	info.ceq_id = ceq_id;</div><div class='del'>-	info.ceqe_base = iwceq-&gt;mem.va;</div><div class='del'>-	info.ceqe_pa = iwceq-&gt;mem.pa;</div><div class='del'>-</div><div class='del'>-	info.elem_cnt = iwdev-&gt;sc_dev.hmc_info-&gt;hmc_obj[I40IW_HMC_IW_CQ].cnt;</div><div class='del'>-	iwceq-&gt;sc_ceq.ceq_id = ceq_id;</div><div class='del'>-	info.dev = dev;</div><div class='del'>-	scratch = (uintptr_t)&amp;iwdev-&gt;cqp.sc_cqp;</div><div class='del'>-	status = dev-&gt;ceq_ops-&gt;ceq_init(&amp;iwceq-&gt;sc_ceq, &amp;info);</div><div class='del'>-	if (!status)</div><div class='del'>-		status = dev-&gt;ceq_ops-&gt;cceq_create(&amp;iwceq-&gt;sc_ceq, scratch);</div><div class='del'>-</div><div class='del'>-exit:</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_free_dma_mem(dev-&gt;hw, &amp;iwceq-&gt;mem);</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void i40iw_request_reset(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct i40e_info *ldev = iwdev-&gt;ldev;</div><div class='del'>-</div><div class='del'>-	ldev-&gt;ops-&gt;request_reset(ldev, iwdev-&gt;client, 1);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_setup_ceqs - manage the device ceq's and their interrupt resources</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @ldev: i40e lan device</div><div class='del'>- *</div><div class='del'>- * Allocate a list for all device completion event queues</div><div class='del'>- * Create the ceq's and configure their msix interrupt vectors</div><div class='del'>- * Return 0, if at least one ceq is successfully set up, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_setup_ceqs(struct i40iw_device *iwdev,</div><div class='del'>-					       struct i40e_info *ldev)</div><div class='del'>-{</div><div class='del'>-	u32 i;</div><div class='del'>-	u32 ceq_id;</div><div class='del'>-	struct i40iw_ceq *iwceq;</div><div class='del'>-	struct i40iw_msix_vector *msix_vec;</div><div class='del'>-	enum i40iw_status_code status = 0;</div><div class='del'>-	u32 num_ceqs;</div><div class='del'>-</div><div class='del'>-	if (ldev &amp;&amp; ldev-&gt;ops &amp;&amp; ldev-&gt;ops-&gt;setup_qvlist) {</div><div class='del'>-		status = ldev-&gt;ops-&gt;setup_qvlist(ldev, &amp;i40iw_client,</div><div class='del'>-						 iwdev-&gt;iw_qvlist);</div><div class='del'>-		if (status)</div><div class='del'>-			goto exit;</div><div class='del'>-	} else {</div><div class='del'>-		status = I40IW_ERR_BAD_PTR;</div><div class='del'>-		goto exit;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	num_ceqs = min(iwdev-&gt;msix_count, iwdev-&gt;sc_dev.hmc_fpm_misc.max_ceqs);</div><div class='del'>-	iwdev-&gt;ceqlist = kcalloc(num_ceqs, sizeof(*iwdev-&gt;ceqlist), GFP_KERNEL);</div><div class='del'>-	if (!iwdev-&gt;ceqlist) {</div><div class='del'>-		status = I40IW_ERR_NO_MEMORY;</div><div class='del'>-		goto exit;</div><div class='del'>-	}</div><div class='del'>-	i = (iwdev-&gt;msix_shared) ? 0 : 1;</div><div class='del'>-	for (ceq_id = 0; i &lt; num_ceqs; i++, ceq_id++) {</div><div class='del'>-		iwceq = &amp;iwdev-&gt;ceqlist[ceq_id];</div><div class='del'>-		status = i40iw_create_ceq(iwdev, iwceq, ceq_id);</div><div class='del'>-		if (status) {</div><div class='del'>-			i40iw_pr_err("create ceq status = %d\n", status);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		msix_vec = &amp;iwdev-&gt;iw_msixtbl[i];</div><div class='del'>-		iwceq-&gt;irq = msix_vec-&gt;irq;</div><div class='del'>-		iwceq-&gt;msix_idx = msix_vec-&gt;idx;</div><div class='del'>-		status = i40iw_configure_ceq_vector(iwdev, iwceq, ceq_id, msix_vec);</div><div class='del'>-		if (status) {</div><div class='del'>-			i40iw_destroy_ceq(iwdev, iwceq);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		i40iw_enable_intr(&amp;iwdev-&gt;sc_dev, msix_vec-&gt;idx);</div><div class='del'>-		iwdev-&gt;ceqs_count++;</div><div class='del'>-	}</div><div class='del'>-exit:</div><div class='del'>-	if (status &amp;&amp; !iwdev-&gt;ceqs_count) {</div><div class='del'>-		kfree(iwdev-&gt;ceqlist);</div><div class='del'>-		iwdev-&gt;ceqlist = NULL;</div><div class='del'>-		return status;</div><div class='del'>-	} else {</div><div class='del'>-		iwdev-&gt;sc_dev.ceq_valid = true;</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_configure_aeq_vector - set up the msix vector for aeq</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- *</div><div class='del'>- * Allocate interrupt resources and enable irq handling</div><div class='del'>- * Return 0 if successful, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_configure_aeq_vector(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_msix_vector *msix_vec = iwdev-&gt;iw_msixtbl;</div><div class='del'>-	u32 ret = 0;</div><div class='del'>-</div><div class='del'>-	if (!iwdev-&gt;msix_shared) {</div><div class='del'>-		tasklet_init(&amp;iwdev-&gt;dpc_tasklet, i40iw_dpc, (unsigned long)iwdev);</div><div class='del'>-		ret = request_irq(msix_vec-&gt;irq, i40iw_irq_handler, 0, "i40iw", iwdev);</div><div class='del'>-	}</div><div class='del'>-	if (ret) {</div><div class='del'>-		i40iw_pr_err("aeq irq config fail\n");</div><div class='del'>-		return I40IW_ERR_CONFIG;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_create_aeq - create async event queue</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- *</div><div class='del'>- * Return 0, if the aeq and the resources associated with it</div><div class='del'>- * are successfully created, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_create_aeq(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_aeq_init_info info;</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_aeq *aeq = &amp;iwdev-&gt;aeq;</div><div class='del'>-	u64 scratch = 0;</div><div class='del'>-	u32 aeq_size;</div><div class='del'>-</div><div class='del'>-	aeq_size = 2 * iwdev-&gt;sc_dev.hmc_info-&gt;hmc_obj[I40IW_HMC_IW_QP].cnt +</div><div class='del'>-		iwdev-&gt;sc_dev.hmc_info-&gt;hmc_obj[I40IW_HMC_IW_CQ].cnt;</div><div class='del'>-	memset(&amp;info, 0, sizeof(info));</div><div class='del'>-	aeq-&gt;mem.size = sizeof(struct i40iw_sc_aeqe) * aeq_size;</div><div class='del'>-	status = i40iw_allocate_dma_mem(dev-&gt;hw, &amp;aeq-&gt;mem, aeq-&gt;mem.size,</div><div class='del'>-					I40IW_AEQ_ALIGNMENT);</div><div class='del'>-	if (status)</div><div class='del'>-		goto exit;</div><div class='del'>-</div><div class='del'>-	info.aeqe_base = aeq-&gt;mem.va;</div><div class='del'>-	info.aeq_elem_pa = aeq-&gt;mem.pa;</div><div class='del'>-	info.elem_cnt = aeq_size;</div><div class='del'>-	info.dev = dev;</div><div class='del'>-	status = dev-&gt;aeq_ops-&gt;aeq_init(&amp;aeq-&gt;sc_aeq, &amp;info);</div><div class='del'>-	if (status)</div><div class='del'>-		goto exit;</div><div class='del'>-	status = dev-&gt;aeq_ops-&gt;aeq_create(&amp;aeq-&gt;sc_aeq, scratch, 1);</div><div class='del'>-	if (!status)</div><div class='del'>-		status = dev-&gt;aeq_ops-&gt;aeq_create_done(&amp;aeq-&gt;sc_aeq);</div><div class='del'>-exit:</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_free_dma_mem(dev-&gt;hw, &amp;aeq-&gt;mem);</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_setup_aeq - set up the device aeq</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- *</div><div class='del'>- * Create the aeq and configure its msix interrupt vector</div><div class='del'>- * Return 0 if successful, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_setup_aeq(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-</div><div class='del'>-	status = i40iw_create_aeq(iwdev);</div><div class='del'>-	if (status)</div><div class='del'>-		return status;</div><div class='del'>-</div><div class='del'>-	status = i40iw_configure_aeq_vector(iwdev);</div><div class='del'>-	if (status) {</div><div class='del'>-		i40iw_destroy_aeq(iwdev);</div><div class='del'>-		return status;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!iwdev-&gt;msix_shared)</div><div class='del'>-		i40iw_enable_intr(dev, iwdev-&gt;iw_msixtbl[0].idx);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_initialize_ilq - create iwarp local queue for cm</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- *</div><div class='del'>- * Return 0 if successful, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_initialize_ilq(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_rsrc_info info;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-</div><div class='del'>-	memset(&amp;info, 0, sizeof(info));</div><div class='del'>-	info.type = I40IW_PUDA_RSRC_TYPE_ILQ;</div><div class='del'>-	info.cq_id = 1;</div><div class='del'>-	info.qp_id = 0;</div><div class='del'>-	info.count = 1;</div><div class='del'>-	info.pd_id = 1;</div><div class='del'>-	info.sq_size = 8192;</div><div class='del'>-	info.rq_size = 8192;</div><div class='del'>-	info.buf_size = 1024;</div><div class='del'>-	info.tx_buf_cnt = 16384;</div><div class='del'>-	info.receive = i40iw_receive_ilq;</div><div class='del'>-	info.xmit_complete = i40iw_free_sqbuf;</div><div class='del'>-	status = i40iw_puda_create_rsrc(&amp;iwdev-&gt;vsi, &amp;info);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("ilq create fail\n");</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_initialize_ieq - create iwarp exception queue</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- *</div><div class='del'>- * Return 0 if successful, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_initialize_ieq(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_rsrc_info info;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-</div><div class='del'>-	memset(&amp;info, 0, sizeof(info));</div><div class='del'>-	info.type = I40IW_PUDA_RSRC_TYPE_IEQ;</div><div class='del'>-	info.cq_id = 2;</div><div class='del'>-	info.qp_id = iwdev-&gt;vsi.exception_lan_queue;</div><div class='del'>-	info.count = 1;</div><div class='del'>-	info.pd_id = 2;</div><div class='del'>-	info.sq_size = 8192;</div><div class='del'>-	info.rq_size = 8192;</div><div class='del'>-	info.buf_size = iwdev-&gt;vsi.mtu + VLAN_ETH_HLEN;</div><div class='del'>-	info.tx_buf_cnt = 4096;</div><div class='del'>-	status = i40iw_puda_create_rsrc(&amp;iwdev-&gt;vsi, &amp;info);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("ieq create fail\n");</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_reinitialize_ieq - destroy and re-create ieq</div><div class='del'>- * @dev: iwarp device</div><div class='del'>- */</div><div class='del'>-void i40iw_reinitialize_ieq(struct i40iw_sc_dev *dev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)dev-&gt;back_dev;</div><div class='del'>-</div><div class='del'>-	i40iw_puda_dele_resources(&amp;iwdev-&gt;vsi, I40IW_PUDA_RSRC_TYPE_IEQ, false);</div><div class='del'>-	if (i40iw_initialize_ieq(iwdev)) {</div><div class='del'>-		iwdev-&gt;reset = true;</div><div class='del'>-		i40iw_request_reset(iwdev);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hmc_setup - create hmc objects for the device</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- *</div><div class='del'>- * Set up the device private memory space for the number and size of</div><div class='del'>- * the hmc objects and create the objects</div><div class='del'>- * Return 0 if successful, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_hmc_setup(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-</div><div class='del'>-	iwdev-&gt;sd_type = I40IW_SD_TYPE_DIRECT;</div><div class='del'>-	status = i40iw_config_fpm_values(&amp;iwdev-&gt;sc_dev, IW_CFG_FPM_QP_COUNT);</div><div class='del'>-	if (status)</div><div class='del'>-		goto exit;</div><div class='del'>-	status = i40iw_create_hmc_objs(iwdev, true);</div><div class='del'>-	if (status)</div><div class='del'>-		goto exit;</div><div class='del'>-	iwdev-&gt;init_state = HMC_OBJS_CREATED;</div><div class='del'>-exit:</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_del_init_mem - deallocate memory resources</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- */</div><div class='del'>-static void i40iw_del_init_mem(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-</div><div class='del'>-	i40iw_free_dma_mem(&amp;iwdev-&gt;hw, &amp;iwdev-&gt;obj_mem);</div><div class='del'>-	kfree(dev-&gt;hmc_info-&gt;sd_table.sd_entry);</div><div class='del'>-	dev-&gt;hmc_info-&gt;sd_table.sd_entry = NULL;</div><div class='del'>-	kfree(iwdev-&gt;mem_resources);</div><div class='del'>-	iwdev-&gt;mem_resources = NULL;</div><div class='del'>-	kfree(iwdev-&gt;ceqlist);</div><div class='del'>-	iwdev-&gt;ceqlist = NULL;</div><div class='del'>-	kfree(iwdev-&gt;iw_msixtbl);</div><div class='del'>-	iwdev-&gt;iw_msixtbl = NULL;</div><div class='del'>-	kfree(iwdev-&gt;hmc_info_mem);</div><div class='del'>-	iwdev-&gt;hmc_info_mem = NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_del_macip_entry - remove a mac ip address entry from the hw table</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @idx: the index of the mac ip address to delete</div><div class='del'>- */</div><div class='del'>-static void i40iw_del_macip_entry(struct i40iw_device *iwdev, u8 idx)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cqp *iwcqp = &amp;iwdev-&gt;cqp;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	enum i40iw_status_code status = 0;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(iwcqp, true);</div><div class='del'>-	if (!cqp_request) {</div><div class='del'>-		i40iw_pr_err("cqp_request memory failed\n");</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_DELETE_LOCAL_MAC_IPADDR_ENTRY;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.del_local_mac_ipaddr_entry.cqp = &amp;iwcqp-&gt;sc_cqp;</div><div class='del'>-	cqp_info-&gt;in.u.del_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	cqp_info-&gt;in.u.del_local_mac_ipaddr_entry.entry_idx = idx;</div><div class='del'>-	cqp_info-&gt;in.u.del_local_mac_ipaddr_entry.ignore_ref_count = 0;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("CQP-OP Del MAC Ip entry fail");</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_add_mac_ipaddr_entry - add a mac ip address entry to the hw table</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @mac_addr: pointer to mac address</div><div class='del'>- * @idx: the index of the mac ip address to add</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_add_mac_ipaddr_entry(struct i40iw_device *iwdev,</div><div class='del'>-							 u8 *mac_addr,</div><div class='del'>-							 u8 idx)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_local_mac_ipaddr_entry_info *info;</div><div class='del'>-	struct i40iw_cqp *iwcqp = &amp;iwdev-&gt;cqp;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	enum i40iw_status_code status = 0;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(iwcqp, true);</div><div class='del'>-	if (!cqp_request) {</div><div class='del'>-		i40iw_pr_err("cqp_request memory failed\n");</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	info = &amp;cqp_info-&gt;in.u.add_local_mac_ipaddr_entry.info;</div><div class='del'>-	ether_addr_copy(info-&gt;mac_addr, mac_addr);</div><div class='del'>-	info-&gt;entry_idx = idx;</div><div class='del'>-	cqp_info-&gt;in.u.add_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_ADD_LOCAL_MAC_IPADDR_ENTRY;</div><div class='del'>-	cqp_info-&gt;in.u.add_local_mac_ipaddr_entry.cqp = &amp;iwcqp-&gt;sc_cqp;</div><div class='del'>-	cqp_info-&gt;in.u.add_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("CQP-OP Add MAC Ip entry fail");</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_alloc_local_mac_ipaddr_entry - allocate a mac ip address entry</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @mac_ip_tbl_idx: the index of the new mac ip address</div><div class='del'>- *</div><div class='del'>- * Allocate a mac ip address entry and update the mac_ip_tbl_idx</div><div class='del'>- * to hold the index of the newly created mac ip address</div><div class='del'>- * Return 0 if successful, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_alloc_local_mac_ipaddr_entry(struct i40iw_device *iwdev,</div><div class='del'>-								 u16 *mac_ip_tbl_idx)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cqp *iwcqp = &amp;iwdev-&gt;cqp;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	enum i40iw_status_code status = 0;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(iwcqp, true);</div><div class='del'>-	if (!cqp_request) {</div><div class='del'>-		i40iw_pr_err("cqp_request memory failed\n");</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* increment refcount, because we need the cqp request ret value */</div><div class='del'>-	atomic_inc(&amp;cqp_request-&gt;refcount);</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.alloc_local_mac_ipaddr_entry.cqp = &amp;iwcqp-&gt;sc_cqp;</div><div class='del'>-	cqp_info-&gt;in.u.alloc_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (!status)</div><div class='del'>-		*mac_ip_tbl_idx = cqp_request-&gt;compl_info.op_ret_val;</div><div class='del'>-	else</div><div class='del'>-		i40iw_pr_err("CQP-OP Alloc MAC Ip entry fail");</div><div class='del'>-	/* decrement refcount and free the cqp request, if no longer used */</div><div class='del'>-	i40iw_put_cqp_request(iwcqp, cqp_request);</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_alloc_set_mac_ipaddr - set up a mac ip address table entry</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @macaddr: pointer to mac address</div><div class='del'>- *</div><div class='del'>- * Allocate a mac ip address entry and add it to the hw table</div><div class='del'>- * Return 0 if successful, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_alloc_set_mac_ipaddr(struct i40iw_device *iwdev,</div><div class='del'>-							 u8 *macaddr)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-</div><div class='del'>-	status = i40iw_alloc_local_mac_ipaddr_entry(iwdev, &amp;iwdev-&gt;mac_ip_table_idx);</div><div class='del'>-	if (!status) {</div><div class='del'>-		status = i40iw_add_mac_ipaddr_entry(iwdev, macaddr,</div><div class='del'>-						    (u8)iwdev-&gt;mac_ip_table_idx);</div><div class='del'>-		if (status)</div><div class='del'>-			i40iw_del_macip_entry(iwdev, (u8)iwdev-&gt;mac_ip_table_idx);</div><div class='del'>-	}</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_add_ipv6_addr - add ipv6 address to the hw arp table</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- */</div><div class='del'>-static void i40iw_add_ipv6_addr(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct net_device *ip_dev;</div><div class='del'>-	struct inet6_dev *idev;</div><div class='del'>-	struct inet6_ifaddr *ifp, *tmp;</div><div class='del'>-	u32 local_ipaddr6[4];</div><div class='del'>-</div><div class='del'>-	rcu_read_lock();</div><div class='del'>-	for_each_netdev_rcu(&amp;init_net, ip_dev) {</div><div class='del'>-		if ((((rdma_vlan_dev_vlan_id(ip_dev) &lt; 0xFFFF) &amp;&amp;</div><div class='del'>-		      (rdma_vlan_dev_real_dev(ip_dev) == iwdev-&gt;netdev)) ||</div><div class='del'>-		     (ip_dev == iwdev-&gt;netdev)) &amp;&amp; (ip_dev-&gt;flags &amp; IFF_UP)) {</div><div class='del'>-			idev = __in6_dev_get(ip_dev);</div><div class='del'>-			if (!idev) {</div><div class='del'>-				i40iw_pr_err("ipv6 inet device not found\n");</div><div class='del'>-				break;</div><div class='del'>-			}</div><div class='del'>-			list_for_each_entry_safe(ifp, tmp, &amp;idev-&gt;addr_list, if_list) {</div><div class='del'>-				i40iw_pr_info("IP=%pI6, vlan_id=%d, MAC=%pM\n", &amp;ifp-&gt;addr,</div><div class='del'>-					      rdma_vlan_dev_vlan_id(ip_dev), ip_dev-&gt;dev_addr);</div><div class='del'>-				i40iw_copy_ip_ntohl(local_ipaddr6,</div><div class='del'>-						    ifp-&gt;addr.in6_u.u6_addr32);</div><div class='del'>-				i40iw_manage_arp_cache(iwdev,</div><div class='del'>-						       ip_dev-&gt;dev_addr,</div><div class='del'>-						       local_ipaddr6,</div><div class='del'>-						       false,</div><div class='del'>-						       I40IW_ARP_ADD);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	rcu_read_unlock();</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_add_ipv4_addr - add ipv4 address to the hw arp table</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- */</div><div class='del'>-static void i40iw_add_ipv4_addr(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct net_device *dev;</div><div class='del'>-	struct in_device *idev;</div><div class='del'>-	bool got_lock = true;</div><div class='del'>-	u32 ip_addr;</div><div class='del'>-</div><div class='del'>-	if (!rtnl_trylock())</div><div class='del'>-		got_lock = false;</div><div class='del'>-</div><div class='del'>-	for_each_netdev(&amp;init_net, dev) {</div><div class='del'>-		if ((((rdma_vlan_dev_vlan_id(dev) &lt; 0xFFFF) &amp;&amp;</div><div class='del'>-		      (rdma_vlan_dev_real_dev(dev) == iwdev-&gt;netdev)) ||</div><div class='del'>-		    (dev == iwdev-&gt;netdev)) &amp;&amp; (dev-&gt;flags &amp; IFF_UP)) {</div><div class='del'>-			idev = in_dev_get(dev);</div><div class='del'>-			for_ifa(idev) {</div><div class='del'>-				i40iw_debug(&amp;iwdev-&gt;sc_dev, I40IW_DEBUG_CM,</div><div class='del'>-					    "IP=%pI4, vlan_id=%d, MAC=%pM\n", &amp;ifa-&gt;ifa_address,</div><div class='del'>-					     rdma_vlan_dev_vlan_id(dev), dev-&gt;dev_addr);</div><div class='del'>-</div><div class='del'>-				ip_addr = ntohl(ifa-&gt;ifa_address);</div><div class='del'>-				i40iw_manage_arp_cache(iwdev,</div><div class='del'>-						       dev-&gt;dev_addr,</div><div class='del'>-						       &amp;ip_addr,</div><div class='del'>-						       true,</div><div class='del'>-						       I40IW_ARP_ADD);</div><div class='del'>-			}</div><div class='del'>-			endfor_ifa(idev);</div><div class='del'>-			in_dev_put(idev);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	if (got_lock)</div><div class='del'>-		rtnl_unlock();</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_add_mac_ip - add mac and ip addresses</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- *</div><div class='del'>- * Create and add a mac ip address entry to the hw table and</div><div class='del'>- * ipv4/ipv6 addresses to the arp cache</div><div class='del'>- * Return 0 if successful, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_add_mac_ip(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct net_device *netdev = iwdev-&gt;netdev;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-</div><div class='del'>-	status = i40iw_alloc_set_mac_ipaddr(iwdev, (u8 *)netdev-&gt;dev_addr);</div><div class='del'>-	if (status)</div><div class='del'>-		return status;</div><div class='del'>-	i40iw_add_ipv4_addr(iwdev);</div><div class='del'>-	i40iw_add_ipv6_addr(iwdev);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_wait_pe_ready - Check if firmware is ready</div><div class='del'>- * @hw: provides access to registers</div><div class='del'>- */</div><div class='del'>-static void i40iw_wait_pe_ready(struct i40iw_hw *hw)</div><div class='del'>-{</div><div class='del'>-	u32 statusfw;</div><div class='del'>-	u32 statuscpu0;</div><div class='del'>-	u32 statuscpu1;</div><div class='del'>-	u32 statuscpu2;</div><div class='del'>-	u32 retrycount = 0;</div><div class='del'>-</div><div class='del'>-	do {</div><div class='del'>-		statusfw = i40iw_rd32(hw, I40E_GLPE_FWLDSTATUS);</div><div class='del'>-		i40iw_pr_info("[%04d] fm load status[x%04X]\n", __LINE__, statusfw);</div><div class='del'>-		statuscpu0 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS0);</div><div class='del'>-		i40iw_pr_info("[%04d] CSR_CQP status[x%04X]\n", __LINE__, statuscpu0);</div><div class='del'>-		statuscpu1 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS1);</div><div class='del'>-		i40iw_pr_info("[%04d] I40E_GLPE_CPUSTATUS1 status[x%04X]\n",</div><div class='del'>-			      __LINE__, statuscpu1);</div><div class='del'>-		statuscpu2 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS2);</div><div class='del'>-		i40iw_pr_info("[%04d] I40E_GLPE_CPUSTATUS2 status[x%04X]\n",</div><div class='del'>-			      __LINE__, statuscpu2);</div><div class='del'>-		if ((statuscpu0 == 0x80) &amp;&amp; (statuscpu1 == 0x80) &amp;&amp; (statuscpu2 == 0x80))</div><div class='del'>-			break;	/* SUCCESS */</div><div class='del'>-		msleep(1000);</div><div class='del'>-		retrycount++;</div><div class='del'>-	} while (retrycount &lt; 14);</div><div class='del'>-	i40iw_wr32(hw, 0xb4040, 0x4C104C5);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_initialize_dev - initialize device</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @ldev: lan device information</div><div class='del'>- *</div><div class='del'>- * Allocate memory for the hmc objects and initialize iwdev</div><div class='del'>- * Return 0 if successful, otherwise clean up the resources</div><div class='del'>- * and return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_initialize_dev(struct i40iw_device *iwdev,</div><div class='del'>-						   struct i40e_info *ldev)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_device_init_info info;</div><div class='del'>-	struct i40iw_vsi_init_info vsi_info;</div><div class='del'>-	struct i40iw_dma_mem mem;</div><div class='del'>-	struct i40iw_l2params l2params;</div><div class='del'>-	u32 size;</div><div class='del'>-	struct i40iw_vsi_stats_info stats_info;</div><div class='del'>-	u16 last_qset = I40IW_NO_QSET;</div><div class='del'>-	u16 qset;</div><div class='del'>-	u32 i;</div><div class='del'>-</div><div class='del'>-	memset(&amp;l2params, 0, sizeof(l2params));</div><div class='del'>-	memset(&amp;info, 0, sizeof(info));</div><div class='del'>-	size = sizeof(struct i40iw_hmc_pble_rsrc) + sizeof(struct i40iw_hmc_info) +</div><div class='del'>-				(sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX);</div><div class='del'>-	iwdev-&gt;hmc_info_mem = kzalloc(size, GFP_KERNEL);</div><div class='del'>-	if (!iwdev-&gt;hmc_info_mem)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-</div><div class='del'>-	iwdev-&gt;pble_rsrc = (struct i40iw_hmc_pble_rsrc *)iwdev-&gt;hmc_info_mem;</div><div class='del'>-	dev-&gt;hmc_info = &amp;iwdev-&gt;hw.hmc;</div><div class='del'>-	dev-&gt;hmc_info-&gt;hmc_obj = (struct i40iw_hmc_obj_info *)(iwdev-&gt;pble_rsrc + 1);</div><div class='del'>-	status = i40iw_obj_aligned_mem(iwdev, &amp;mem, I40IW_QUERY_FPM_BUF_SIZE,</div><div class='del'>-				       I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK);</div><div class='del'>-	if (status)</div><div class='del'>-		goto error;</div><div class='del'>-	info.fpm_query_buf_pa = mem.pa;</div><div class='del'>-	info.fpm_query_buf = mem.va;</div><div class='del'>-	status = i40iw_obj_aligned_mem(iwdev, &amp;mem, I40IW_COMMIT_FPM_BUF_SIZE,</div><div class='del'>-				       I40IW_FPM_COMMIT_BUF_ALIGNMENT_MASK);</div><div class='del'>-	if (status)</div><div class='del'>-		goto error;</div><div class='del'>-	info.fpm_commit_buf_pa = mem.pa;</div><div class='del'>-	info.fpm_commit_buf = mem.va;</div><div class='del'>-	info.hmc_fn_id = ldev-&gt;fid;</div><div class='del'>-	info.is_pf = (ldev-&gt;ftype) ? false : true;</div><div class='del'>-	info.bar0 = ldev-&gt;hw_addr;</div><div class='del'>-	info.hw = &amp;iwdev-&gt;hw;</div><div class='del'>-	info.debug_mask = debug;</div><div class='del'>-	l2params.mtu =</div><div class='del'>-		(ldev-&gt;params.mtu) ? ldev-&gt;params.mtu : I40IW_DEFAULT_MTU;</div><div class='del'>-	for (i = 0; i &lt; I40E_CLIENT_MAX_USER_PRIORITY; i++) {</div><div class='del'>-		qset = ldev-&gt;params.qos.prio_qos[i].qs_handle;</div><div class='del'>-		l2params.qs_handle_list[i] = qset;</div><div class='del'>-		if (last_qset == I40IW_NO_QSET)</div><div class='del'>-			last_qset = qset;</div><div class='del'>-		else if ((qset != last_qset) &amp;&amp; (qset != I40IW_NO_QSET))</div><div class='del'>-			iwdev-&gt;dcb = true;</div><div class='del'>-	}</div><div class='del'>-	i40iw_pr_info("DCB is set/clear = %d\n", iwdev-&gt;dcb);</div><div class='del'>-	info.vchnl_send = i40iw_virtchnl_send;</div><div class='del'>-	status = i40iw_device_init(&amp;iwdev-&gt;sc_dev, &amp;info);</div><div class='del'>-</div><div class='del'>-	if (status)</div><div class='del'>-		goto error;</div><div class='del'>-	memset(&amp;vsi_info, 0, sizeof(vsi_info));</div><div class='del'>-	vsi_info.dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	vsi_info.back_vsi = (void *)iwdev;</div><div class='del'>-	vsi_info.params = &amp;l2params;</div><div class='del'>-	vsi_info.exception_lan_queue = 1;</div><div class='del'>-	i40iw_sc_vsi_init(&amp;iwdev-&gt;vsi, &amp;vsi_info);</div><div class='del'>-</div><div class='del'>-	if (dev-&gt;is_pf) {</div><div class='del'>-		memset(&amp;stats_info, 0, sizeof(stats_info));</div><div class='del'>-		stats_info.fcn_id = ldev-&gt;fid;</div><div class='del'>-		stats_info.pestat = kzalloc(sizeof(*stats_info.pestat), GFP_KERNEL);</div><div class='del'>-		if (!stats_info.pestat) {</div><div class='del'>-			status = I40IW_ERR_NO_MEMORY;</div><div class='del'>-			goto error;</div><div class='del'>-		}</div><div class='del'>-		stats_info.stats_initialize = true;</div><div class='del'>-		if (stats_info.pestat)</div><div class='del'>-			i40iw_vsi_stats_init(&amp;iwdev-&gt;vsi, &amp;stats_info);</div><div class='del'>-	}</div><div class='del'>-	return status;</div><div class='del'>-error:</div><div class='del'>-	kfree(iwdev-&gt;hmc_info_mem);</div><div class='del'>-	iwdev-&gt;hmc_info_mem = NULL;</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_register_notifiers - register tcp ip notifiers</div><div class='del'>- */</div><div class='del'>-static void i40iw_register_notifiers(void)</div><div class='del'>-{</div><div class='del'>-	register_inetaddr_notifier(&amp;i40iw_inetaddr_notifier);</div><div class='del'>-	register_inet6addr_notifier(&amp;i40iw_inetaddr6_notifier);</div><div class='del'>-	register_netevent_notifier(&amp;i40iw_net_notifier);</div><div class='del'>-	register_netdevice_notifier(&amp;i40iw_netdevice_notifier);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_unregister_notifiers - unregister tcp ip notifiers</div><div class='del'>- */</div><div class='del'>-</div><div class='del'>-static void i40iw_unregister_notifiers(void)</div><div class='del'>-{</div><div class='del'>-	unregister_netevent_notifier(&amp;i40iw_net_notifier);</div><div class='del'>-	unregister_inetaddr_notifier(&amp;i40iw_inetaddr_notifier);</div><div class='del'>-	unregister_inet6addr_notifier(&amp;i40iw_inetaddr6_notifier);</div><div class='del'>-	unregister_netdevice_notifier(&amp;i40iw_netdevice_notifier);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_save_msix_info - copy msix vector information to iwarp device</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @ldev: lan device information</div><div class='del'>- *</div><div class='del'>- * Allocate iwdev msix table and copy the ldev msix info to the table</div><div class='del'>- * Return 0 if successful, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_save_msix_info(struct i40iw_device *iwdev,</div><div class='del'>-						   struct i40e_info *ldev)</div><div class='del'>-{</div><div class='del'>-	struct i40e_qvlist_info *iw_qvlist;</div><div class='del'>-	struct i40e_qv_info *iw_qvinfo;</div><div class='del'>-	u32 ceq_idx;</div><div class='del'>-	u32 i;</div><div class='del'>-	u32 size;</div><div class='del'>-</div><div class='del'>-	if (!ldev-&gt;msix_count) {</div><div class='del'>-		i40iw_pr_err("No MSI-X vectors\n");</div><div class='del'>-		return I40IW_ERR_CONFIG;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	iwdev-&gt;msix_count = ldev-&gt;msix_count;</div><div class='del'>-</div><div class='del'>-	size = sizeof(struct i40iw_msix_vector) * iwdev-&gt;msix_count;</div><div class='del'>-	size += sizeof(struct i40e_qvlist_info);</div><div class='del'>-	size +=  sizeof(struct i40e_qv_info) * iwdev-&gt;msix_count - 1;</div><div class='del'>-	iwdev-&gt;iw_msixtbl = kzalloc(size, GFP_KERNEL);</div><div class='del'>-</div><div class='del'>-	if (!iwdev-&gt;iw_msixtbl)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	iwdev-&gt;iw_qvlist = (struct i40e_qvlist_info *)(&amp;iwdev-&gt;iw_msixtbl[iwdev-&gt;msix_count]);</div><div class='del'>-	iw_qvlist = iwdev-&gt;iw_qvlist;</div><div class='del'>-	iw_qvinfo = iw_qvlist-&gt;qv_info;</div><div class='del'>-	iw_qvlist-&gt;num_vectors = iwdev-&gt;msix_count;</div><div class='del'>-	if (iwdev-&gt;msix_count &lt;= num_online_cpus())</div><div class='del'>-		iwdev-&gt;msix_shared = true;</div><div class='del'>-	for (i = 0, ceq_idx = 0; i &lt; iwdev-&gt;msix_count; i++, iw_qvinfo++) {</div><div class='del'>-		iwdev-&gt;iw_msixtbl[i].idx = ldev-&gt;msix_entries[i].entry;</div><div class='del'>-		iwdev-&gt;iw_msixtbl[i].irq = ldev-&gt;msix_entries[i].vector;</div><div class='del'>-		iwdev-&gt;iw_msixtbl[i].cpu_affinity = ceq_idx;</div><div class='del'>-		if (i == 0) {</div><div class='del'>-			iw_qvinfo-&gt;aeq_idx = 0;</div><div class='del'>-			if (iwdev-&gt;msix_shared)</div><div class='del'>-				iw_qvinfo-&gt;ceq_idx = ceq_idx++;</div><div class='del'>-			else</div><div class='del'>-				iw_qvinfo-&gt;ceq_idx = I40E_QUEUE_INVALID_IDX;</div><div class='del'>-		} else {</div><div class='del'>-			iw_qvinfo-&gt;aeq_idx = I40E_QUEUE_INVALID_IDX;</div><div class='del'>-			iw_qvinfo-&gt;ceq_idx = ceq_idx++;</div><div class='del'>-		}</div><div class='del'>-		iw_qvinfo-&gt;itr_idx = 3;</div><div class='del'>-		iw_qvinfo-&gt;v_idx = iwdev-&gt;iw_msixtbl[i].idx;</div><div class='del'>-	}</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_deinit_device - clean up the device resources</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- *</div><div class='del'>- * Destroy the ib device interface, remove the mac ip entry and ipv4/ipv6 addresses,</div><div class='del'>- * destroy the device queues and free the pble and the hmc objects</div><div class='del'>- */</div><div class='del'>-static void i40iw_deinit_device(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct i40e_info *ldev = iwdev-&gt;ldev;</div><div class='del'>-</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-</div><div class='del'>-	i40iw_pr_info("state = %d\n", iwdev-&gt;init_state);</div><div class='del'>-	if (iwdev-&gt;param_wq)</div><div class='del'>-		destroy_workqueue(iwdev-&gt;param_wq);</div><div class='del'>-</div><div class='del'>-	switch (iwdev-&gt;init_state) {</div><div class='del'>-	case RDMA_DEV_REGISTERED:</div><div class='del'>-		iwdev-&gt;iw_status = 0;</div><div class='del'>-		i40iw_port_ibevent(iwdev);</div><div class='del'>-		i40iw_destroy_rdma_device(iwdev-&gt;iwibdev);</div><div class='del'>-		/* fallthrough */</div><div class='del'>-	case IP_ADDR_REGISTERED:</div><div class='del'>-		if (!iwdev-&gt;reset)</div><div class='del'>-			i40iw_del_macip_entry(iwdev, (u8)iwdev-&gt;mac_ip_table_idx);</div><div class='del'>-		/* fallthrough */</div><div class='del'>-		/* fallthrough */</div><div class='del'>-	case PBLE_CHUNK_MEM:</div><div class='del'>-		i40iw_destroy_pble_pool(dev, iwdev-&gt;pble_rsrc);</div><div class='del'>-		/* fallthrough */</div><div class='del'>-	case CEQ_CREATED:</div><div class='del'>-		i40iw_dele_ceqs(iwdev);</div><div class='del'>-		/* fallthrough */</div><div class='del'>-	case AEQ_CREATED:</div><div class='del'>-		i40iw_destroy_aeq(iwdev);</div><div class='del'>-		/* fallthrough */</div><div class='del'>-	case IEQ_CREATED:</div><div class='del'>-		i40iw_puda_dele_resources(&amp;iwdev-&gt;vsi, I40IW_PUDA_RSRC_TYPE_IEQ, iwdev-&gt;reset);</div><div class='del'>-		/* fallthrough */</div><div class='del'>-	case ILQ_CREATED:</div><div class='del'>-		i40iw_puda_dele_resources(&amp;iwdev-&gt;vsi, I40IW_PUDA_RSRC_TYPE_ILQ, iwdev-&gt;reset);</div><div class='del'>-		/* fallthrough */</div><div class='del'>-	case CCQ_CREATED:</div><div class='del'>-		i40iw_destroy_ccq(iwdev);</div><div class='del'>-		/* fallthrough */</div><div class='del'>-	case HMC_OBJS_CREATED:</div><div class='del'>-		i40iw_del_hmc_objects(dev, dev-&gt;hmc_info, true, iwdev-&gt;reset);</div><div class='del'>-		/* fallthrough */</div><div class='del'>-	case CQP_CREATED:</div><div class='del'>-		i40iw_destroy_cqp(iwdev, true);</div><div class='del'>-		/* fallthrough */</div><div class='del'>-	case INITIAL_STATE:</div><div class='del'>-		i40iw_cleanup_cm_core(&amp;iwdev-&gt;cm_core);</div><div class='del'>-		if (iwdev-&gt;vsi.pestat) {</div><div class='del'>-			i40iw_vsi_stats_free(&amp;iwdev-&gt;vsi);</div><div class='del'>-			kfree(iwdev-&gt;vsi.pestat);</div><div class='del'>-		}</div><div class='del'>-		i40iw_del_init_mem(iwdev);</div><div class='del'>-		break;</div><div class='del'>-	case INVALID_STATE:</div><div class='del'>-		/* fallthrough */</div><div class='del'>-	default:</div><div class='del'>-		i40iw_pr_err("bad init_state = %d\n", iwdev-&gt;init_state);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i40iw_del_handler(i40iw_find_i40e_handler(ldev));</div><div class='del'>-	kfree(iwdev-&gt;hdl);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_setup_init_state - set up the initial device struct</div><div class='del'>- * @hdl: handler for iwarp device - one per instance</div><div class='del'>- * @ldev: lan device information</div><div class='del'>- * @client: iwarp client information, provided during registration</div><div class='del'>- *</div><div class='del'>- * Initialize the iwarp device and its hdl information</div><div class='del'>- * using the ldev and client information</div><div class='del'>- * Return 0 if successful, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_setup_init_state(struct i40iw_handler *hdl,</div><div class='del'>-						     struct i40e_info *ldev,</div><div class='del'>-						     struct i40e_client *client)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = &amp;hdl-&gt;device;</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-</div><div class='del'>-	memcpy(&amp;hdl-&gt;ldev, ldev, sizeof(*ldev));</div><div class='del'>-</div><div class='del'>-	iwdev-&gt;mpa_version = mpa_version;</div><div class='del'>-	iwdev-&gt;resource_profile = (resource_profile &lt; I40IW_HMC_PROFILE_EQUAL) ?</div><div class='del'>-	    (u8)resource_profile + I40IW_HMC_PROFILE_DEFAULT :</div><div class='del'>-	    I40IW_HMC_PROFILE_DEFAULT;</div><div class='del'>-	iwdev-&gt;max_rdma_vfs =</div><div class='del'>-		(iwdev-&gt;resource_profile != I40IW_HMC_PROFILE_DEFAULT) ?  max_rdma_vfs : 0;</div><div class='del'>-	iwdev-&gt;max_enabled_vfs = iwdev-&gt;max_rdma_vfs;</div><div class='del'>-	iwdev-&gt;netdev = ldev-&gt;netdev;</div><div class='del'>-	hdl-&gt;client = client;</div><div class='del'>-	if (!ldev-&gt;ftype)</div><div class='del'>-		iwdev-&gt;db_start = pci_resource_start(ldev-&gt;pcidev, 0) + I40IW_DB_ADDR_OFFSET;</div><div class='del'>-	else</div><div class='del'>-		iwdev-&gt;db_start = pci_resource_start(ldev-&gt;pcidev, 0) + I40IW_VF_DB_ADDR_OFFSET;</div><div class='del'>-</div><div class='del'>-	status = i40iw_save_msix_info(iwdev, ldev);</div><div class='del'>-	if (status)</div><div class='del'>-		return status;</div><div class='del'>-	iwdev-&gt;hw.dev_context = (void *)ldev-&gt;pcidev;</div><div class='del'>-	iwdev-&gt;hw.hw_addr = ldev-&gt;hw_addr;</div><div class='del'>-	status = i40iw_allocate_dma_mem(&amp;iwdev-&gt;hw,</div><div class='del'>-					&amp;iwdev-&gt;obj_mem, 8192, 4096);</div><div class='del'>-	if (status)</div><div class='del'>-		goto exit;</div><div class='del'>-	iwdev-&gt;obj_next = iwdev-&gt;obj_mem;</div><div class='del'>-	iwdev-&gt;push_mode = push_mode;</div><div class='del'>-</div><div class='del'>-	init_waitqueue_head(&amp;iwdev-&gt;vchnl_waitq);</div><div class='del'>-	init_waitqueue_head(&amp;dev-&gt;vf_reqs);</div><div class='del'>-	init_waitqueue_head(&amp;iwdev-&gt;close_wq);</div><div class='del'>-</div><div class='del'>-	status = i40iw_initialize_dev(iwdev, ldev);</div><div class='del'>-exit:</div><div class='del'>-	if (status) {</div><div class='del'>-		kfree(iwdev-&gt;iw_msixtbl);</div><div class='del'>-		i40iw_free_dma_mem(dev-&gt;hw, &amp;iwdev-&gt;obj_mem);</div><div class='del'>-		iwdev-&gt;iw_msixtbl = NULL;</div><div class='del'>-	}</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_used_rsrc - determine resources used internally</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- *</div><div class='del'>- * Called after internal allocations</div><div class='del'>- */</div><div class='del'>-static void i40iw_get_used_rsrc(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	iwdev-&gt;used_pds = find_next_zero_bit(iwdev-&gt;allocated_pds, iwdev-&gt;max_pd, 0);</div><div class='del'>-	iwdev-&gt;used_qps = find_next_zero_bit(iwdev-&gt;allocated_qps, iwdev-&gt;max_qp, 0);</div><div class='del'>-	iwdev-&gt;used_cqs = find_next_zero_bit(iwdev-&gt;allocated_cqs, iwdev-&gt;max_cq, 0);</div><div class='del'>-	iwdev-&gt;used_mrs = find_next_zero_bit(iwdev-&gt;allocated_mrs, iwdev-&gt;max_mr, 0);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_open - client interface operation open for iwarp/uda device</div><div class='del'>- * @ldev: lan device information</div><div class='del'>- * @client: iwarp client information, provided during registration</div><div class='del'>- *</div><div class='del'>- * Called by the lan driver during the processing of client register</div><div class='del'>- * Create device resources, set up queues, pble and hmc objects and</div><div class='del'>- * register the device with the ib verbs interface</div><div class='del'>- * Return 0 if successful, otherwise return error</div><div class='del'>- */</div><div class='del'>-static int i40iw_open(struct i40e_info *ldev, struct i40e_client *client)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_handler *hdl;</div><div class='del'>-</div><div class='del'>-	hdl = i40iw_find_netdev(ldev-&gt;netdev);</div><div class='del'>-	if (hdl)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	hdl = kzalloc(sizeof(*hdl), GFP_KERNEL);</div><div class='del'>-	if (!hdl)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	iwdev = &amp;hdl-&gt;device;</div><div class='del'>-	iwdev-&gt;hdl = hdl;</div><div class='del'>-	dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	if (i40iw_setup_cm_core(iwdev)) {</div><div class='del'>-		kfree(iwdev-&gt;hdl);</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	dev-&gt;back_dev = (void *)iwdev;</div><div class='del'>-	iwdev-&gt;ldev = &amp;hdl-&gt;ldev;</div><div class='del'>-	iwdev-&gt;client = client;</div><div class='del'>-	mutex_init(&amp;iwdev-&gt;pbl_mutex);</div><div class='del'>-	i40iw_add_handler(hdl);</div><div class='del'>-</div><div class='del'>-	do {</div><div class='del'>-		status = i40iw_setup_init_state(hdl, ldev, client);</div><div class='del'>-		if (status)</div><div class='del'>-			break;</div><div class='del'>-		iwdev-&gt;init_state = INITIAL_STATE;</div><div class='del'>-		if (dev-&gt;is_pf)</div><div class='del'>-			i40iw_wait_pe_ready(dev-&gt;hw);</div><div class='del'>-		status = i40iw_create_cqp(iwdev);</div><div class='del'>-		if (status)</div><div class='del'>-			break;</div><div class='del'>-		iwdev-&gt;init_state = CQP_CREATED;</div><div class='del'>-		status = i40iw_hmc_setup(iwdev);</div><div class='del'>-		if (status)</div><div class='del'>-			break;</div><div class='del'>-		status = i40iw_create_ccq(iwdev);</div><div class='del'>-		if (status)</div><div class='del'>-			break;</div><div class='del'>-		iwdev-&gt;init_state = CCQ_CREATED;</div><div class='del'>-		status = i40iw_initialize_ilq(iwdev);</div><div class='del'>-		if (status)</div><div class='del'>-			break;</div><div class='del'>-		iwdev-&gt;init_state = ILQ_CREATED;</div><div class='del'>-		status = i40iw_initialize_ieq(iwdev);</div><div class='del'>-		if (status)</div><div class='del'>-			break;</div><div class='del'>-		iwdev-&gt;init_state = IEQ_CREATED;</div><div class='del'>-		status = i40iw_setup_aeq(iwdev);</div><div class='del'>-		if (status)</div><div class='del'>-			break;</div><div class='del'>-		iwdev-&gt;init_state = AEQ_CREATED;</div><div class='del'>-		status = i40iw_setup_ceqs(iwdev, ldev);</div><div class='del'>-		if (status)</div><div class='del'>-			break;</div><div class='del'>-		iwdev-&gt;init_state = CEQ_CREATED;</div><div class='del'>-		status = i40iw_initialize_hw_resources(iwdev);</div><div class='del'>-		if (status)</div><div class='del'>-			break;</div><div class='del'>-		i40iw_get_used_rsrc(iwdev);</div><div class='del'>-		dev-&gt;ccq_ops-&gt;ccq_arm(dev-&gt;ccq);</div><div class='del'>-		status = i40iw_hmc_init_pble(&amp;iwdev-&gt;sc_dev, iwdev-&gt;pble_rsrc);</div><div class='del'>-		if (status)</div><div class='del'>-			break;</div><div class='del'>-		iwdev-&gt;init_state = PBLE_CHUNK_MEM;</div><div class='del'>-		iwdev-&gt;virtchnl_wq = alloc_ordered_workqueue("iwvch", WQ_MEM_RECLAIM);</div><div class='del'>-		status = i40iw_add_mac_ip(iwdev);</div><div class='del'>-		if (status)</div><div class='del'>-			break;</div><div class='del'>-		iwdev-&gt;init_state = IP_ADDR_REGISTERED;</div><div class='del'>-		if (i40iw_register_rdma_device(iwdev)) {</div><div class='del'>-			i40iw_pr_err("register rdma device fail\n");</div><div class='del'>-			break;</div><div class='del'>-		};</div><div class='del'>-</div><div class='del'>-		iwdev-&gt;init_state = RDMA_DEV_REGISTERED;</div><div class='del'>-		iwdev-&gt;iw_status = 1;</div><div class='del'>-		i40iw_port_ibevent(iwdev);</div><div class='del'>-		iwdev-&gt;param_wq = alloc_ordered_workqueue("l2params", WQ_MEM_RECLAIM);</div><div class='del'>-		if(iwdev-&gt;param_wq == NULL)</div><div class='del'>-			break;</div><div class='del'>-		i40iw_pr_info("i40iw_open completed\n");</div><div class='del'>-		return 0;</div><div class='del'>-	} while (0);</div><div class='del'>-</div><div class='del'>-	i40iw_pr_err("status = %d last completion = %d\n", status, iwdev-&gt;init_state);</div><div class='del'>-	i40iw_deinit_device(iwdev);</div><div class='del'>-	return -ERESTART;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_l2params_worker - worker for l2 params change</div><div class='del'>- * @work: work pointer for l2 params</div><div class='del'>- */</div><div class='del'>-static void i40iw_l2params_worker(struct work_struct *work)</div><div class='del'>-{</div><div class='del'>-	struct l2params_work *dwork =</div><div class='del'>-	    container_of(work, struct l2params_work, work);</div><div class='del'>-	struct i40iw_device *iwdev = dwork-&gt;iwdev;</div><div class='del'>-</div><div class='del'>-	i40iw_change_l2params(&amp;iwdev-&gt;vsi, &amp;dwork-&gt;l2params);</div><div class='del'>-	atomic_dec(&amp;iwdev-&gt;params_busy);</div><div class='del'>-	kfree(work);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_l2param_change - handle qs handles for qos and mss change</div><div class='del'>- * @ldev: lan device information</div><div class='del'>- * @client: client for paramater change</div><div class='del'>- * @params: new parameters from L2</div><div class='del'>- */</div><div class='del'>-static void i40iw_l2param_change(struct i40e_info *ldev, struct i40e_client *client,</div><div class='del'>-				 struct i40e_params *params)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_handler *hdl;</div><div class='del'>-	struct i40iw_l2params *l2params;</div><div class='del'>-	struct l2params_work *work;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	hdl = i40iw_find_i40e_handler(ldev);</div><div class='del'>-	if (!hdl)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	iwdev = &amp;hdl-&gt;device;</div><div class='del'>-</div><div class='del'>-	if (atomic_read(&amp;iwdev-&gt;params_busy))</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-</div><div class='del'>-	work = kzalloc(sizeof(*work), GFP_KERNEL);</div><div class='del'>-	if (!work)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	atomic_inc(&amp;iwdev-&gt;params_busy);</div><div class='del'>-</div><div class='del'>-	work-&gt;iwdev = iwdev;</div><div class='del'>-	l2params = &amp;work-&gt;l2params;</div><div class='del'>-	for (i = 0; i &lt; I40E_CLIENT_MAX_USER_PRIORITY; i++)</div><div class='del'>-		l2params-&gt;qs_handle_list[i] = params-&gt;qos.prio_qos[i].qs_handle;</div><div class='del'>-</div><div class='del'>-	l2params-&gt;mtu = (params-&gt;mtu) ? params-&gt;mtu : iwdev-&gt;vsi.mtu;</div><div class='del'>-</div><div class='del'>-	INIT_WORK(&amp;work-&gt;work, i40iw_l2params_worker);</div><div class='del'>-	queue_work(iwdev-&gt;param_wq, &amp;work-&gt;work);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_close - client interface operation close for iwarp/uda device</div><div class='del'>- * @ldev: lan device information</div><div class='del'>- * @client: client to close</div><div class='del'>- *</div><div class='del'>- * Called by the lan driver during the processing of client unregister</div><div class='del'>- * Destroy and clean up the driver resources</div><div class='del'>- */</div><div class='del'>-static void i40iw_close(struct i40e_info *ldev, struct i40e_client *client, bool reset)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_handler *hdl;</div><div class='del'>-</div><div class='del'>-	hdl = i40iw_find_i40e_handler(ldev);</div><div class='del'>-	if (!hdl)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	iwdev = &amp;hdl-&gt;device;</div><div class='del'>-	iwdev-&gt;closing = true;</div><div class='del'>-</div><div class='del'>-	if (reset)</div><div class='del'>-		iwdev-&gt;reset = true;</div><div class='del'>-</div><div class='del'>-	i40iw_cm_teardown_connections(iwdev, NULL, NULL, true);</div><div class='del'>-	destroy_workqueue(iwdev-&gt;virtchnl_wq);</div><div class='del'>-	i40iw_deinit_device(iwdev);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vf_reset - process VF reset</div><div class='del'>- * @ldev: lan device information</div><div class='del'>- * @client: client interface instance</div><div class='del'>- * @vf_id: virtual function id</div><div class='del'>- *</div><div class='del'>- * Called when a VF is reset by the PF</div><div class='del'>- * Destroy and clean up the VF resources</div><div class='del'>- */</div><div class='del'>-static void i40iw_vf_reset(struct i40e_info *ldev, struct i40e_client *client, u32 vf_id)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_handler *hdl;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	struct i40iw_hmc_fcn_info hmc_fcn_info;</div><div class='del'>-	struct i40iw_virt_mem vf_dev_mem;</div><div class='del'>-	struct i40iw_vfdev *tmp_vfdev;</div><div class='del'>-	unsigned int i;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-</div><div class='del'>-	hdl = i40iw_find_i40e_handler(ldev);</div><div class='del'>-	if (!hdl)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	dev = &amp;hdl-&gt;device.sc_dev;</div><div class='del'>-	iwdev = (struct i40iw_device *)dev-&gt;back_dev;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; I40IW_MAX_PE_ENABLED_VF_COUNT; i++) {</div><div class='del'>-		if (!dev-&gt;vf_dev[i] || (dev-&gt;vf_dev[i]-&gt;vf_id != vf_id))</div><div class='del'>-			continue;</div><div class='del'>-		/* free all resources allocated on behalf of vf */</div><div class='del'>-		tmp_vfdev = dev-&gt;vf_dev[i];</div><div class='del'>-		spin_lock_irqsave(&amp;iwdev-&gt;vsi.pestat-&gt;lock, flags);</div><div class='del'>-		dev-&gt;vf_dev[i] = NULL;</div><div class='del'>-		spin_unlock_irqrestore(&amp;iwdev-&gt;vsi.pestat-&gt;lock, flags);</div><div class='del'>-		i40iw_del_hmc_objects(dev, &amp;tmp_vfdev-&gt;hmc_info, false, false);</div><div class='del'>-		/* remove vf hmc function */</div><div class='del'>-		memset(&amp;hmc_fcn_info, 0, sizeof(hmc_fcn_info));</div><div class='del'>-		hmc_fcn_info.vf_id = vf_id;</div><div class='del'>-		hmc_fcn_info.iw_vf_idx = tmp_vfdev-&gt;iw_vf_idx;</div><div class='del'>-		hmc_fcn_info.free_fcn = true;</div><div class='del'>-		i40iw_cqp_manage_hmc_fcn_cmd(dev, &amp;hmc_fcn_info);</div><div class='del'>-		/* free vf_dev */</div><div class='del'>-		vf_dev_mem.va = tmp_vfdev;</div><div class='del'>-		vf_dev_mem.size = sizeof(struct i40iw_vfdev) +</div><div class='del'>-					sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX;</div><div class='del'>-		i40iw_free_virt_mem(dev-&gt;hw, &amp;vf_dev_mem);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vf_enable - enable a number of VFs</div><div class='del'>- * @ldev: lan device information</div><div class='del'>- * @client: client interface instance</div><div class='del'>- * @num_vfs: number of VFs for the PF</div><div class='del'>- *</div><div class='del'>- * Called when the number of VFs changes</div><div class='del'>- */</div><div class='del'>-static void i40iw_vf_enable(struct i40e_info *ldev,</div><div class='del'>-			    struct i40e_client *client,</div><div class='del'>-			    u32 num_vfs)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_handler *hdl;</div><div class='del'>-</div><div class='del'>-	hdl = i40iw_find_i40e_handler(ldev);</div><div class='del'>-	if (!hdl)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	if (num_vfs &gt; I40IW_MAX_PE_ENABLED_VF_COUNT)</div><div class='del'>-		hdl-&gt;device.max_enabled_vfs = I40IW_MAX_PE_ENABLED_VF_COUNT;</div><div class='del'>-	else</div><div class='del'>-		hdl-&gt;device.max_enabled_vfs = num_vfs;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vf_capable - check if VF capable</div><div class='del'>- * @ldev: lan device information</div><div class='del'>- * @client: client interface instance</div><div class='del'>- * @vf_id: virtual function id</div><div class='del'>- *</div><div class='del'>- * Return 1 if a VF slot is available or if VF is already RDMA enabled</div><div class='del'>- * Return 0 otherwise</div><div class='del'>- */</div><div class='del'>-static int i40iw_vf_capable(struct i40e_info *ldev,</div><div class='del'>-			    struct i40e_client *client,</div><div class='del'>-			    u32 vf_id)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_handler *hdl;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	unsigned int i;</div><div class='del'>-</div><div class='del'>-	hdl = i40iw_find_i40e_handler(ldev);</div><div class='del'>-	if (!hdl)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	dev = &amp;hdl-&gt;device.sc_dev;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; hdl-&gt;device.max_enabled_vfs; i++) {</div><div class='del'>-		if (!dev-&gt;vf_dev[i] || (dev-&gt;vf_dev[i]-&gt;vf_id == vf_id))</div><div class='del'>-			return 1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_virtchnl_receive - receive a message through the virtual channel</div><div class='del'>- * @ldev: lan device information</div><div class='del'>- * @client: client interface instance</div><div class='del'>- * @vf_id: virtual function id associated with the message</div><div class='del'>- * @msg: message buffer pointer</div><div class='del'>- * @len: length of the message</div><div class='del'>- *</div><div class='del'>- * Invoke virtual channel receive operation for the given msg</div><div class='del'>- * Return 0 if successful, otherwise return error</div><div class='del'>- */</div><div class='del'>-static int i40iw_virtchnl_receive(struct i40e_info *ldev,</div><div class='del'>-				  struct i40e_client *client,</div><div class='del'>-				  u32 vf_id,</div><div class='del'>-				  u8 *msg,</div><div class='del'>-				  u16 len)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_handler *hdl;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	int ret_code = I40IW_NOT_SUPPORTED;</div><div class='del'>-</div><div class='del'>-	if (!len || !msg)</div><div class='del'>-		return I40IW_ERR_PARAM;</div><div class='del'>-</div><div class='del'>-	hdl = i40iw_find_i40e_handler(ldev);</div><div class='del'>-	if (!hdl)</div><div class='del'>-		return I40IW_ERR_PARAM;</div><div class='del'>-</div><div class='del'>-	dev = &amp;hdl-&gt;device.sc_dev;</div><div class='del'>-	iwdev = dev-&gt;back_dev;</div><div class='del'>-</div><div class='del'>-	if (dev-&gt;vchnl_if.vchnl_recv) {</div><div class='del'>-		ret_code = dev-&gt;vchnl_if.vchnl_recv(dev, vf_id, msg, len);</div><div class='del'>-		if (!dev-&gt;is_pf) {</div><div class='del'>-			atomic_dec(&amp;iwdev-&gt;vchnl_msgs);</div><div class='del'>-			wake_up(&amp;iwdev-&gt;vchnl_waitq);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vf_clear_to_send - wait to send virtual channel message</div><div class='del'>- * @dev: iwarp device *</div><div class='del'>- * Wait for until virtual channel is clear</div><div class='del'>- * before sending the next message</div><div class='del'>- *</div><div class='del'>- * Returns false if error</div><div class='del'>- * Returns true if clear to send</div><div class='del'>- */</div><div class='del'>-bool i40iw_vf_clear_to_send(struct i40iw_sc_dev *dev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	wait_queue_entry_t wait;</div><div class='del'>-</div><div class='del'>-	iwdev = dev-&gt;back_dev;</div><div class='del'>-</div><div class='del'>-	if (!wq_has_sleeper(&amp;dev-&gt;vf_reqs) &amp;&amp;</div><div class='del'>-	    (atomic_read(&amp;iwdev-&gt;vchnl_msgs) == 0))</div><div class='del'>-		return true; /* virtual channel is clear */</div><div class='del'>-</div><div class='del'>-	init_wait(&amp;wait);</div><div class='del'>-	add_wait_queue_exclusive(&amp;dev-&gt;vf_reqs, &amp;wait);</div><div class='del'>-</div><div class='del'>-	if (!wait_event_timeout(dev-&gt;vf_reqs,</div><div class='del'>-				(atomic_read(&amp;iwdev-&gt;vchnl_msgs) == 0),</div><div class='del'>-				I40IW_VCHNL_EVENT_TIMEOUT))</div><div class='del'>-		dev-&gt;vchnl_up = false;</div><div class='del'>-</div><div class='del'>-	remove_wait_queue(&amp;dev-&gt;vf_reqs, &amp;wait);</div><div class='del'>-</div><div class='del'>-	return dev-&gt;vchnl_up;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_virtchnl_send - send a message through the virtual channel</div><div class='del'>- * @dev: iwarp device</div><div class='del'>- * @vf_id: virtual function id associated with the message</div><div class='del'>- * @msg: virtual channel message buffer pointer</div><div class='del'>- * @len: length of the message</div><div class='del'>- *</div><div class='del'>- * Invoke virtual channel send operation for the given msg</div><div class='del'>- * Return 0 if successful, otherwise return error</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_virtchnl_send(struct i40iw_sc_dev *dev,</div><div class='del'>-						  u32 vf_id,</div><div class='del'>-						  u8 *msg,</div><div class='del'>-						  u16 len)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40e_info *ldev;</div><div class='del'>-</div><div class='del'>-	if (!dev || !dev-&gt;back_dev)</div><div class='del'>-		return I40IW_ERR_BAD_PTR;</div><div class='del'>-</div><div class='del'>-	iwdev = dev-&gt;back_dev;</div><div class='del'>-	ldev = iwdev-&gt;ldev;</div><div class='del'>-</div><div class='del'>-	if (ldev &amp;&amp; ldev-&gt;ops &amp;&amp; ldev-&gt;ops-&gt;virtchnl_send)</div><div class='del'>-		return ldev-&gt;ops-&gt;virtchnl_send(ldev, &amp;i40iw_client, vf_id, msg, len);</div><div class='del'>-	return I40IW_ERR_BAD_PTR;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* client interface functions */</div><div class='del'>-static const struct i40e_client_ops i40e_ops = {</div><div class='del'>-	.open = i40iw_open,</div><div class='del'>-	.close = i40iw_close,</div><div class='del'>-	.l2_param_change = i40iw_l2param_change,</div><div class='del'>-	.virtchnl_receive = i40iw_virtchnl_receive,</div><div class='del'>-	.vf_reset = i40iw_vf_reset,</div><div class='del'>-	.vf_enable = i40iw_vf_enable,</div><div class='del'>-	.vf_capable = i40iw_vf_capable</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_init_module - driver initialization function</div><div class='del'>- *</div><div class='del'>- * First function to call when the driver is loaded</div><div class='del'>- * Register the driver as i40e client and port mapper client</div><div class='del'>- */</div><div class='del'>-static int __init i40iw_init_module(void)</div><div class='del'>-{</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	memset(&amp;i40iw_client, 0, sizeof(i40iw_client));</div><div class='del'>-	i40iw_client.version.major = CLIENT_IW_INTERFACE_VERSION_MAJOR;</div><div class='del'>-	i40iw_client.version.minor = CLIENT_IW_INTERFACE_VERSION_MINOR;</div><div class='del'>-	i40iw_client.version.build = CLIENT_IW_INTERFACE_VERSION_BUILD;</div><div class='del'>-	i40iw_client.ops = &amp;i40e_ops;</div><div class='del'>-	memcpy(i40iw_client.name, i40iw_client_name, I40E_CLIENT_STR_LENGTH);</div><div class='del'>-	i40iw_client.type = I40E_CLIENT_IWARP;</div><div class='del'>-	spin_lock_init(&amp;i40iw_handler_lock);</div><div class='del'>-	ret = i40e_register_client(&amp;i40iw_client);</div><div class='del'>-	i40iw_register_notifiers();</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_exit_module - driver exit clean up function</div><div class='del'>- *</div><div class='del'>- * The function is called just before the driver is unloaded</div><div class='del'>- * Unregister the driver as i40e client and port mapper client</div><div class='del'>- */</div><div class='del'>-static void __exit i40iw_exit_module(void)</div><div class='del'>-{</div><div class='del'>-	i40iw_unregister_notifiers();</div><div class='del'>-	i40e_unregister_client(&amp;i40iw_client);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-module_init(i40iw_init_module);</div><div class='del'>-module_exit(i40iw_exit_module);</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_osdep.h b/drivers/infiniband/hw/i40iw/i40iw_osdep.h<br/>deleted file mode 100644<br/>index d474aad62a81..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_osdep.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_osdep.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,217 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#ifndef I40IW_OSDEP_H</div><div class='del'>-#define I40IW_OSDEP_H</div><div class='del'>-</div><div class='del'>-#include &lt;linux/kernel.h&gt;</div><div class='del'>-#include &lt;linux/string.h&gt;</div><div class='del'>-#include &lt;linux/bitops.h&gt;</div><div class='del'>-#include &lt;net/tcp.h&gt;</div><div class='del'>-#include &lt;crypto/hash.h&gt;</div><div class='del'>-/* get readq/writeq support for 32 bit kernels, use the low-first version */</div><div class='del'>-#include &lt;linux/io-64-nonatomic-lo-hi.h&gt;</div><div class='del'>-</div><div class='del'>-#define STATS_TIMER_DELAY 1000</div><div class='del'>-</div><div class='del'>-static inline void set_64bit_val(u64 *wqe_words, u32 byte_index, u64 value)</div><div class='del'>-{</div><div class='del'>-	wqe_words[byte_index &gt;&gt; 3] = value;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * set_32bit_val - set 32 value to hw wqe</div><div class='del'>- * @wqe_words: wqe addr to write</div><div class='del'>- * @byte_index: index in wqe</div><div class='del'>- * @value: value to write</div><div class='del'>- **/</div><div class='del'>-static inline void set_32bit_val(u32 *wqe_words, u32 byte_index, u32 value)</div><div class='del'>-{</div><div class='del'>-	wqe_words[byte_index &gt;&gt; 2] = value;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * get_64bit_val - read 64 bit value from wqe</div><div class='del'>- * @wqe_words: wqe addr</div><div class='del'>- * @byte_index: index to read from</div><div class='del'>- * @value: read value</div><div class='del'>- **/</div><div class='del'>-static inline void get_64bit_val(u64 *wqe_words, u32 byte_index, u64 *value)</div><div class='del'>-{</div><div class='del'>-	*value = wqe_words[byte_index &gt;&gt; 3];</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * get_32bit_val - read 32 bit value from wqe</div><div class='del'>- * @wqe_words: wqe addr</div><div class='del'>- * @byte_index: index to reaad from</div><div class='del'>- * @value: return 32 bit value</div><div class='del'>- **/</div><div class='del'>-static inline void get_32bit_val(u32 *wqe_words, u32 byte_index, u32 *value)</div><div class='del'>-{</div><div class='del'>-	*value = wqe_words[byte_index &gt;&gt; 2];</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct i40iw_dma_mem {</div><div class='del'>-	void *va;</div><div class='del'>-	dma_addr_t pa;</div><div class='del'>-	u32 size;</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-struct i40iw_virt_mem {</div><div class='del'>-	void *va;</div><div class='del'>-	u32 size;</div><div class='del'>-} __packed;</div><div class='del'>-</div><div class='del'>-#define i40iw_debug(h, m, s, ...)                               \</div><div class='del'>-do {                                                            \</div><div class='del'>-	if (((m) &amp; (h)-&gt;debug_mask))                            \</div><div class='del'>-		pr_info("i40iw " s, ##__VA_ARGS__);             \</div><div class='del'>-} while (0)</div><div class='del'>-</div><div class='del'>-#define i40iw_flush(a)          readl((a)-&gt;hw_addr + I40E_GLGEN_STAT)</div><div class='del'>-</div><div class='del'>-#define I40E_GLHMC_VFSDCMD(_i)  (0x000C8000 + ((_i) * 4)) \</div><div class='del'>-				/* _i=0...31 */</div><div class='del'>-#define I40E_GLHMC_VFSDCMD_MAX_INDEX    31</div><div class='del'>-#define I40E_GLHMC_VFSDCMD_PMSDIDX_SHIFT  0</div><div class='del'>-#define I40E_GLHMC_VFSDCMD_PMSDIDX_MASK  (0xFFF \</div><div class='del'>-					  &lt;&lt; I40E_GLHMC_VFSDCMD_PMSDIDX_SHIFT)</div><div class='del'>-#define I40E_GLHMC_VFSDCMD_PF_SHIFT       16</div><div class='del'>-#define I40E_GLHMC_VFSDCMD_PF_MASK        (0xF &lt;&lt; I40E_GLHMC_VFSDCMD_PF_SHIFT)</div><div class='del'>-#define I40E_GLHMC_VFSDCMD_VF_SHIFT       20</div><div class='del'>-#define I40E_GLHMC_VFSDCMD_VF_MASK        (0x1FF &lt;&lt; I40E_GLHMC_VFSDCMD_VF_SHIFT)</div><div class='del'>-#define I40E_GLHMC_VFSDCMD_PMF_TYPE_SHIFT 29</div><div class='del'>-#define I40E_GLHMC_VFSDCMD_PMF_TYPE_MASK  (0x3 \</div><div class='del'>-					   &lt;&lt; I40E_GLHMC_VFSDCMD_PMF_TYPE_SHIFT)</div><div class='del'>-#define I40E_GLHMC_VFSDCMD_PMSDWR_SHIFT   31</div><div class='del'>-#define I40E_GLHMC_VFSDCMD_PMSDWR_MASK  (0x1 &lt;&lt; I40E_GLHMC_VFSDCMD_PMSDWR_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40E_GLHMC_VFSDDATAHIGH(_i)     (0x000C8200 + ((_i) * 4)) \</div><div class='del'>-				/* _i=0...31 */</div><div class='del'>-#define I40E_GLHMC_VFSDDATAHIGH_MAX_INDEX       31</div><div class='del'>-#define I40E_GLHMC_VFSDDATAHIGH_PMSDDATAHIGH_SHIFT 0</div><div class='del'>-#define I40E_GLHMC_VFSDDATAHIGH_PMSDDATAHIGH_MASK  (0xFFFFFFFF \</div><div class='del'>-			&lt;&lt; I40E_GLHMC_VFSDDATAHIGH_PMSDDATAHIGH_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40E_GLHMC_VFSDDATALOW(_i)      (0x000C8100 + ((_i) * 4)) \</div><div class='del'>-				/* _i=0...31 */</div><div class='del'>-#define I40E_GLHMC_VFSDDATALOW_MAX_INDEX        31</div><div class='del'>-#define I40E_GLHMC_VFSDDATALOW_PMSDVALID_SHIFT   0</div><div class='del'>-#define I40E_GLHMC_VFSDDATALOW_PMSDVALID_MASK  (0x1 \</div><div class='del'>-			&lt;&lt; I40E_GLHMC_VFSDDATALOW_PMSDVALID_SHIFT)</div><div class='del'>-#define I40E_GLHMC_VFSDDATALOW_PMSDTYPE_SHIFT    1</div><div class='del'>-#define I40E_GLHMC_VFSDDATALOW_PMSDTYPE_MASK  (0x1 \</div><div class='del'>-			&lt;&lt; I40E_GLHMC_VFSDDATALOW_PMSDTYPE_SHIFT)</div><div class='del'>-#define I40E_GLHMC_VFSDDATALOW_PMSDBPCOUNT_SHIFT 2</div><div class='del'>-#define I40E_GLHMC_VFSDDATALOW_PMSDBPCOUNT_MASK  (0x3FF \</div><div class='del'>-			&lt;&lt; I40E_GLHMC_VFSDDATALOW_PMSDBPCOUNT_SHIFT)</div><div class='del'>-#define I40E_GLHMC_VFSDDATALOW_PMSDDATALOW_SHIFT 12</div><div class='del'>-#define I40E_GLHMC_VFSDDATALOW_PMSDDATALOW_MASK  (0xFFFFF \</div><div class='del'>-			&lt;&lt; I40E_GLHMC_VFSDDATALOW_PMSDDATALOW_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40E_GLPE_FWLDSTATUS                     0x0000D200</div><div class='del'>-#define I40E_GLPE_FWLDSTATUS_LOAD_REQUESTED_SHIFT 0</div><div class='del'>-#define I40E_GLPE_FWLDSTATUS_LOAD_REQUESTED_MASK  (0x1 \</div><div class='del'>-			&lt;&lt; I40E_GLPE_FWLDSTATUS_LOAD_REQUESTED_SHIFT)</div><div class='del'>-#define I40E_GLPE_FWLDSTATUS_DONE_SHIFT           1</div><div class='del'>-#define I40E_GLPE_FWLDSTATUS_DONE_MASK  (0x1 &lt;&lt; I40E_GLPE_FWLDSTATUS_DONE_SHIFT)</div><div class='del'>-#define I40E_GLPE_FWLDSTATUS_CQP_FAIL_SHIFT       2</div><div class='del'>-#define I40E_GLPE_FWLDSTATUS_CQP_FAIL_MASK  (0x1 \</div><div class='del'>-			 &lt;&lt; I40E_GLPE_FWLDSTATUS_CQP_FAIL_SHIFT)</div><div class='del'>-#define I40E_GLPE_FWLDSTATUS_TEP_FAIL_SHIFT       3</div><div class='del'>-#define I40E_GLPE_FWLDSTATUS_TEP_FAIL_MASK  (0x1 \</div><div class='del'>-			 &lt;&lt; I40E_GLPE_FWLDSTATUS_TEP_FAIL_SHIFT)</div><div class='del'>-#define I40E_GLPE_FWLDSTATUS_OOP_FAIL_SHIFT       4</div><div class='del'>-#define I40E_GLPE_FWLDSTATUS_OOP_FAIL_MASK  (0x1 \</div><div class='del'>-			 &lt;&lt; I40E_GLPE_FWLDSTATUS_OOP_FAIL_SHIFT)</div><div class='del'>-</div><div class='del'>-struct i40iw_sc_dev;</div><div class='del'>-struct i40iw_sc_qp;</div><div class='del'>-struct i40iw_puda_buf;</div><div class='del'>-struct i40iw_puda_completion_info;</div><div class='del'>-struct i40iw_update_sds_info;</div><div class='del'>-struct i40iw_hmc_fcn_info;</div><div class='del'>-struct i40iw_virtchnl_work_info;</div><div class='del'>-struct i40iw_manage_vf_pble_info;</div><div class='del'>-struct i40iw_device;</div><div class='del'>-struct i40iw_hmc_info;</div><div class='del'>-struct i40iw_hw;</div><div class='del'>-</div><div class='del'>-u8 __iomem *i40iw_get_hw_addr(void *dev);</div><div class='del'>-void i40iw_ieq_mpa_crc_ae(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp);</div><div class='del'>-enum i40iw_status_code i40iw_vf_wait_vchnl_resp(struct i40iw_sc_dev *dev);</div><div class='del'>-bool i40iw_vf_clear_to_send(struct i40iw_sc_dev *dev);</div><div class='del'>-enum i40iw_status_code i40iw_ieq_check_mpacrc(struct shash_desc *desc, void *addr,</div><div class='del'>-					      u32 length, u32 value);</div><div class='del'>-struct i40iw_sc_qp *i40iw_ieq_get_qp(struct i40iw_sc_dev *dev, struct i40iw_puda_buf *buf);</div><div class='del'>-void i40iw_ieq_update_tcpip_info(struct i40iw_puda_buf *buf, u16 length, u32 seqnum);</div><div class='del'>-void i40iw_free_hash_desc(struct shash_desc *);</div><div class='del'>-enum i40iw_status_code i40iw_init_hash_desc(struct shash_desc **);</div><div class='del'>-enum i40iw_status_code i40iw_puda_get_tcpip_info(struct i40iw_puda_completion_info *info,</div><div class='del'>-						 struct i40iw_puda_buf *buf);</div><div class='del'>-enum i40iw_status_code i40iw_cqp_sds_cmd(struct i40iw_sc_dev *dev,</div><div class='del'>-					 struct i40iw_update_sds_info *info);</div><div class='del'>-enum i40iw_status_code i40iw_cqp_manage_hmc_fcn_cmd(struct i40iw_sc_dev *dev,</div><div class='del'>-						    struct i40iw_hmc_fcn_info *hmcfcninfo);</div><div class='del'>-enum i40iw_status_code i40iw_cqp_query_fpm_values_cmd(struct i40iw_sc_dev *dev,</div><div class='del'>-						      struct i40iw_dma_mem *values_mem,</div><div class='del'>-						      u8 hmc_fn_id);</div><div class='del'>-enum i40iw_status_code i40iw_cqp_commit_fpm_values_cmd(struct i40iw_sc_dev *dev,</div><div class='del'>-						       struct i40iw_dma_mem *values_mem,</div><div class='del'>-						       u8 hmc_fn_id);</div><div class='del'>-enum i40iw_status_code i40iw_alloc_query_fpm_buf(struct i40iw_sc_dev *dev,</div><div class='del'>-						 struct i40iw_dma_mem *mem);</div><div class='del'>-enum i40iw_status_code i40iw_cqp_manage_vf_pble_bp(struct i40iw_sc_dev *dev,</div><div class='del'>-						   struct i40iw_manage_vf_pble_info *info);</div><div class='del'>-void i40iw_cqp_spawn_worker(struct i40iw_sc_dev *dev,</div><div class='del'>-			    struct i40iw_virtchnl_work_info *work_info, u32 iw_vf_idx);</div><div class='del'>-void *i40iw_remove_head(struct list_head *list);</div><div class='del'>-void i40iw_qp_suspend_resume(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp, bool suspend);</div><div class='del'>-</div><div class='del'>-void i40iw_term_modify_qp(struct i40iw_sc_qp *qp, u8 next_state, u8 term, u8 term_len);</div><div class='del'>-void i40iw_terminate_done(struct i40iw_sc_qp *qp, int timeout_occurred);</div><div class='del'>-void i40iw_terminate_start_timer(struct i40iw_sc_qp *qp);</div><div class='del'>-void i40iw_terminate_del_timer(struct i40iw_sc_qp *qp);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_hw_manage_vf_pble_bp(struct i40iw_device *iwdev,</div><div class='del'>-						  struct i40iw_manage_vf_pble_info *info,</div><div class='del'>-						  bool wait);</div><div class='del'>-struct i40iw_sc_vsi;</div><div class='del'>-void i40iw_hw_stats_start_timer(struct i40iw_sc_vsi *vsi);</div><div class='del'>-void i40iw_hw_stats_stop_timer(struct i40iw_sc_vsi *vsi);</div><div class='del'>-#define i40iw_mmiowb() do { } while (0)</div><div class='del'>-void i40iw_wr32(struct i40iw_hw *hw, u32 reg, u32 value);</div><div class='del'>-u32  i40iw_rd32(struct i40iw_hw *hw, u32 reg);</div><div class='del'>-#endif				/* _I40IW_OSDEP_H_ */</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_p.h b/drivers/infiniband/hw/i40iw/i40iw_p.h<br/>deleted file mode 100644<br/>index 11d3a2a72100..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_p.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_p.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,128 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#ifndef I40IW_P_H</div><div class='del'>-#define I40IW_P_H</div><div class='del'>-</div><div class='del'>-#define PAUSE_TIMER_VALUE       0xFFFF</div><div class='del'>-#define REFRESH_THRESHOLD       0x7FFF</div><div class='del'>-#define HIGH_THRESHOLD          0x800</div><div class='del'>-#define LOW_THRESHOLD           0x200</div><div class='del'>-#define ALL_TC2PFC              0xFF</div><div class='del'>-#define CQP_COMPL_WAIT_TIME     0x3E8</div><div class='del'>-#define CQP_TIMEOUT_THRESHOLD   5</div><div class='del'>-</div><div class='del'>-void i40iw_debug_buf(struct i40iw_sc_dev *dev, enum i40iw_debug_flag mask,</div><div class='del'>-		     char *desc, u64 *buf, u32 size);</div><div class='del'>-/* init operations */</div><div class='del'>-enum i40iw_status_code i40iw_device_init(struct i40iw_sc_dev *dev,</div><div class='del'>-					 struct i40iw_device_init_info *info);</div><div class='del'>-</div><div class='del'>-void i40iw_sc_cqp_post_sq(struct i40iw_sc_cqp *cqp);</div><div class='del'>-</div><div class='del'>-u64 *i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp *cqp, u64 scratch);</div><div class='del'>-</div><div class='del'>-void i40iw_check_cqp_progress(struct i40iw_cqp_timeout *cqp_timeout, struct i40iw_sc_dev *dev);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_sc_mr_fast_register(struct i40iw_sc_qp *qp,</div><div class='del'>-						 struct i40iw_fast_reg_stag_info *info,</div><div class='del'>-						 bool post_sq);</div><div class='del'>-</div><div class='del'>-void i40iw_insert_wqe_hdr(u64 *wqe, u64 header);</div><div class='del'>-</div><div class='del'>-/* HMC/FPM functions */</div><div class='del'>-enum i40iw_status_code i40iw_sc_init_iw_hmc(struct i40iw_sc_dev *dev,</div><div class='del'>-					    u8 hmc_fn_id);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_pf_init_vfhmc(struct i40iw_sc_dev *dev, u8 vf_hmc_fn_id,</div><div class='del'>-					   u32 *vf_cnt_array);</div><div class='del'>-</div><div class='del'>-/* stats functions */</div><div class='del'>-void i40iw_hw_stats_refresh_all(struct i40iw_vsi_pestat *stats);</div><div class='del'>-void i40iw_hw_stats_read_all(struct i40iw_vsi_pestat *stats, struct i40iw_dev_hw_stats *stats_values);</div><div class='del'>-void i40iw_hw_stats_read_32(struct i40iw_vsi_pestat *stats,</div><div class='del'>-			    enum i40iw_hw_stats_index_32b index,</div><div class='del'>-			    u64 *value);</div><div class='del'>-void i40iw_hw_stats_read_64(struct i40iw_vsi_pestat *stats,</div><div class='del'>-			    enum i40iw_hw_stats_index_64b index,</div><div class='del'>-			    u64 *value);</div><div class='del'>-void i40iw_hw_stats_init(struct i40iw_vsi_pestat *stats, u8 index, bool is_pf);</div><div class='del'>-</div><div class='del'>-/* vsi misc functions */</div><div class='del'>-enum i40iw_status_code i40iw_vsi_stats_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_stats_info *info);</div><div class='del'>-void i40iw_vsi_stats_free(struct i40iw_sc_vsi *vsi);</div><div class='del'>-void i40iw_sc_vsi_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_init_info *info);</div><div class='del'>-</div><div class='del'>-void i40iw_change_l2params(struct i40iw_sc_vsi *vsi, struct i40iw_l2params *l2params);</div><div class='del'>-void i40iw_qp_add_qos(struct i40iw_sc_qp *qp);</div><div class='del'>-void i40iw_qp_rem_qos(struct i40iw_sc_qp *qp);</div><div class='del'>-void i40iw_terminate_send_fin(struct i40iw_sc_qp *qp);</div><div class='del'>-</div><div class='del'>-void i40iw_terminate_connection(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info);</div><div class='del'>-</div><div class='del'>-void i40iw_terminate_received(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_sc_suspend_qp(struct i40iw_sc_cqp *cqp,</div><div class='del'>-					   struct i40iw_sc_qp *qp, u64 scratch);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_sc_resume_qp(struct i40iw_sc_cqp *cqp,</div><div class='del'>-					  struct i40iw_sc_qp *qp, u64 scratch);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_sc_static_hmc_pages_allocated(struct i40iw_sc_cqp *cqp,</div><div class='del'>-							   u64 scratch, u8 hmc_fn_id,</div><div class='del'>-							   bool post_sq,</div><div class='del'>-							   bool poll_registers);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_count);</div><div class='del'>-</div><div class='del'>-void free_sd_mem(struct i40iw_sc_dev *dev);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_process_cqp_cmd(struct i40iw_sc_dev *dev,</div><div class='del'>-					     struct cqp_commands_info *pcmdinfo);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_process_bh(struct i40iw_sc_dev *dev);</div><div class='del'>-</div><div class='del'>-/* prototype for functions used for dynamic memory allocation */</div><div class='del'>-enum i40iw_status_code i40iw_allocate_dma_mem(struct i40iw_hw *hw,</div><div class='del'>-					      struct i40iw_dma_mem *mem, u64 size,</div><div class='del'>-					      u32 alignment);</div><div class='del'>-void i40iw_free_dma_mem(struct i40iw_hw *hw, struct i40iw_dma_mem *mem);</div><div class='del'>-enum i40iw_status_code i40iw_allocate_virt_mem(struct i40iw_hw *hw,</div><div class='del'>-					       struct i40iw_virt_mem *mem, u32 size);</div><div class='del'>-enum i40iw_status_code i40iw_free_virt_mem(struct i40iw_hw *hw,</div><div class='del'>-					   struct i40iw_virt_mem *mem);</div><div class='del'>-u8 i40iw_get_encoded_wqe_size(u32 wqsize, bool cqpsq);</div><div class='del'>-void i40iw_reinitialize_ieq(struct i40iw_sc_dev *dev);</div><div class='del'>-</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_pble.c b/drivers/infiniband/hw/i40iw/i40iw_pble.c<br/>deleted file mode 100644<br/>index 540aab5e502d..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_pble.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_pble.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,612 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#include "i40iw_status.h"</div><div class='del'>-#include "i40iw_osdep.h"</div><div class='del'>-#include "i40iw_register.h"</div><div class='del'>-#include "i40iw_hmc.h"</div><div class='del'>-</div><div class='del'>-#include "i40iw_d.h"</div><div class='del'>-#include "i40iw_type.h"</div><div class='del'>-#include "i40iw_p.h"</div><div class='del'>-</div><div class='del'>-#include &lt;linux/pci.h&gt;</div><div class='del'>-#include &lt;linux/genalloc.h&gt;</div><div class='del'>-#include &lt;linux/vmalloc.h&gt;</div><div class='del'>-#include "i40iw_pble.h"</div><div class='del'>-#include "i40iw.h"</div><div class='del'>-</div><div class='del'>-struct i40iw_device;</div><div class='del'>-static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev,</div><div class='del'>-					    struct i40iw_hmc_pble_rsrc *pble_rsrc);</div><div class='del'>-static void i40iw_free_vmalloc_mem(struct i40iw_hw *hw, struct i40iw_chunk *chunk);</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_destroy_pble_pool - destroy pool during module unload</div><div class='del'>- * @pble_rsrc:	pble resources</div><div class='del'>- */</div><div class='del'>-void i40iw_destroy_pble_pool(struct i40iw_sc_dev *dev, struct i40iw_hmc_pble_rsrc *pble_rsrc)</div><div class='del'>-{</div><div class='del'>-	struct list_head *clist;</div><div class='del'>-	struct list_head *tlist;</div><div class='del'>-	struct i40iw_chunk *chunk;</div><div class='del'>-	struct i40iw_pble_pool *pinfo = &amp;pble_rsrc-&gt;pinfo;</div><div class='del'>-</div><div class='del'>-	if (pinfo-&gt;pool) {</div><div class='del'>-		list_for_each_safe(clist, tlist, &amp;pinfo-&gt;clist) {</div><div class='del'>-			chunk = list_entry(clist, struct i40iw_chunk, list);</div><div class='del'>-			if (chunk-&gt;type == I40IW_VMALLOC)</div><div class='del'>-				i40iw_free_vmalloc_mem(dev-&gt;hw, chunk);</div><div class='del'>-			kfree(chunk);</div><div class='del'>-		}</div><div class='del'>-		gen_pool_destroy(pinfo-&gt;pool);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hmc_init_pble - Initialize pble resources during module load</div><div class='del'>- * @dev: i40iw_sc_dev struct</div><div class='del'>- * @pble_rsrc:	pble resources</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_hmc_init_pble(struct i40iw_sc_dev *dev,</div><div class='del'>-					   struct i40iw_hmc_pble_rsrc *pble_rsrc)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_hmc_info *hmc_info;</div><div class='del'>-	u32 fpm_idx = 0;</div><div class='del'>-</div><div class='del'>-	hmc_info = dev-&gt;hmc_info;</div><div class='del'>-	pble_rsrc-&gt;fpm_base_addr = hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].base;</div><div class='del'>-	/* Now start the pble' on 4k boundary */</div><div class='del'>-	if (pble_rsrc-&gt;fpm_base_addr &amp; 0xfff)</div><div class='del'>-		fpm_idx = (PAGE_SIZE - (pble_rsrc-&gt;fpm_base_addr &amp; 0xfff)) &gt;&gt; 3;</div><div class='del'>-</div><div class='del'>-	pble_rsrc-&gt;unallocated_pble =</div><div class='del'>-	    hmc_info-&gt;hmc_obj[I40IW_HMC_IW_PBLE].cnt - fpm_idx;</div><div class='del'>-	pble_rsrc-&gt;next_fpm_addr = pble_rsrc-&gt;fpm_base_addr + (fpm_idx &lt;&lt; 3);</div><div class='del'>-</div><div class='del'>-	pble_rsrc-&gt;pinfo.pool_shift = POOL_SHIFT;</div><div class='del'>-	pble_rsrc-&gt;pinfo.pool = gen_pool_create(pble_rsrc-&gt;pinfo.pool_shift, -1);</div><div class='del'>-	INIT_LIST_HEAD(&amp;pble_rsrc-&gt;pinfo.clist);</div><div class='del'>-	if (!pble_rsrc-&gt;pinfo.pool)</div><div class='del'>-		goto error;</div><div class='del'>-</div><div class='del'>-	if (add_pble_pool(dev, pble_rsrc))</div><div class='del'>-		goto error;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>- error:i40iw_destroy_pble_pool(dev, pble_rsrc);</div><div class='del'>-	return I40IW_ERR_NO_MEMORY;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * get_sd_pd_idx -  Returns sd index, pd index and rel_pd_idx from fpm address</div><div class='del'>- * @ pble_rsrc:	structure containing fpm address</div><div class='del'>- * @ idx: where to return indexes</div><div class='del'>- */</div><div class='del'>-static inline void get_sd_pd_idx(struct i40iw_hmc_pble_rsrc *pble_rsrc,</div><div class='del'>-				 struct sd_pd_idx *idx)</div><div class='del'>-{</div><div class='del'>-	idx-&gt;sd_idx = (u32)(pble_rsrc-&gt;next_fpm_addr) / I40IW_HMC_DIRECT_BP_SIZE;</div><div class='del'>-	idx-&gt;pd_idx = (u32)(pble_rsrc-&gt;next_fpm_addr) / I40IW_HMC_PAGED_BP_SIZE;</div><div class='del'>-	idx-&gt;rel_pd_idx = (idx-&gt;pd_idx % I40IW_HMC_PD_CNT_IN_SD);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * add_sd_direct - add sd direct for pble</div><div class='del'>- * @dev: hardware control device structure</div><div class='del'>- * @pble_rsrc: pble resource ptr</div><div class='del'>- * @info: page info for sd</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code add_sd_direct(struct i40iw_sc_dev *dev,</div><div class='del'>-					    struct i40iw_hmc_pble_rsrc *pble_rsrc,</div><div class='del'>-					    struct i40iw_add_page_info *info)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-	struct sd_pd_idx *idx = &amp;info-&gt;idx;</div><div class='del'>-	struct i40iw_chunk *chunk = info-&gt;chunk;</div><div class='del'>-	struct i40iw_hmc_info *hmc_info = info-&gt;hmc_info;</div><div class='del'>-	struct i40iw_hmc_sd_entry *sd_entry = info-&gt;sd_entry;</div><div class='del'>-	u32 offset = 0;</div><div class='del'>-</div><div class='del'>-	if (!sd_entry-&gt;valid) {</div><div class='del'>-		if (dev-&gt;is_pf) {</div><div class='del'>-			ret_code = i40iw_add_sd_table_entry(dev-&gt;hw, hmc_info,</div><div class='del'>-							    info-&gt;idx.sd_idx,</div><div class='del'>-							    I40IW_SD_TYPE_DIRECT,</div><div class='del'>-							    I40IW_HMC_DIRECT_BP_SIZE);</div><div class='del'>-			if (ret_code)</div><div class='del'>-				return ret_code;</div><div class='del'>-			chunk-&gt;type = I40IW_DMA_COHERENT;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	offset = idx-&gt;rel_pd_idx &lt;&lt; I40IW_HMC_PAGED_BP_SHIFT;</div><div class='del'>-	chunk-&gt;size = info-&gt;pages &lt;&lt; I40IW_HMC_PAGED_BP_SHIFT;</div><div class='del'>-	chunk-&gt;vaddr = ((u8 *)sd_entry-&gt;u.bp.addr.va + offset);</div><div class='del'>-	chunk-&gt;fpm_addr = pble_rsrc-&gt;next_fpm_addr;</div><div class='del'>-	i40iw_debug(dev, I40IW_DEBUG_PBLE, "chunk_size[%d] = 0x%x vaddr=%p fpm_addr = %llx\n",</div><div class='del'>-		    chunk-&gt;size, chunk-&gt;size, chunk-&gt;vaddr, chunk-&gt;fpm_addr);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_free_vmalloc_mem - free vmalloc during close</div><div class='del'>- * @hw: hw struct</div><div class='del'>- * @chunk: chunk information for vmalloc</div><div class='del'>- */</div><div class='del'>-static void i40iw_free_vmalloc_mem(struct i40iw_hw *hw, struct i40iw_chunk *chunk)</div><div class='del'>-{</div><div class='del'>-	struct pci_dev *pcidev = (struct pci_dev *)hw-&gt;dev_context;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	if (!chunk-&gt;pg_cnt)</div><div class='del'>-		goto done;</div><div class='del'>-	for (i = 0; i &lt; chunk-&gt;pg_cnt; i++)</div><div class='del'>-		dma_unmap_page(&amp;pcidev-&gt;dev, chunk-&gt;dmaaddrs[i], PAGE_SIZE, DMA_BIDIRECTIONAL);</div><div class='del'>-</div><div class='del'>- done:</div><div class='del'>-	kfree(chunk-&gt;dmaaddrs);</div><div class='del'>-	chunk-&gt;dmaaddrs = NULL;</div><div class='del'>-	vfree(chunk-&gt;vaddr);</div><div class='del'>-	chunk-&gt;vaddr = NULL;</div><div class='del'>-	chunk-&gt;type = 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_vmalloc_mem - get 2M page for sd</div><div class='del'>- * @hw: hardware address</div><div class='del'>- * @chunk: chunk to adf</div><div class='del'>- * @pg_cnt: #of 4 K pages</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_get_vmalloc_mem(struct i40iw_hw *hw,</div><div class='del'>-						    struct i40iw_chunk *chunk,</div><div class='del'>-						    int pg_cnt)</div><div class='del'>-{</div><div class='del'>-	struct pci_dev *pcidev = (struct pci_dev *)hw-&gt;dev_context;</div><div class='del'>-	struct page *page;</div><div class='del'>-	u8 *addr;</div><div class='del'>-	u32 size;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	chunk-&gt;dmaaddrs = kzalloc(pg_cnt &lt;&lt; 3, GFP_KERNEL);</div><div class='del'>-	if (!chunk-&gt;dmaaddrs)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	size = PAGE_SIZE * pg_cnt;</div><div class='del'>-	chunk-&gt;vaddr = vmalloc(size);</div><div class='del'>-	if (!chunk-&gt;vaddr) {</div><div class='del'>-		kfree(chunk-&gt;dmaaddrs);</div><div class='del'>-		chunk-&gt;dmaaddrs = NULL;</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	}</div><div class='del'>-	chunk-&gt;size = size;</div><div class='del'>-	addr = (u8 *)chunk-&gt;vaddr;</div><div class='del'>-	for (i = 0; i &lt; pg_cnt; i++) {</div><div class='del'>-		page = vmalloc_to_page((void *)addr);</div><div class='del'>-		if (!page)</div><div class='del'>-			break;</div><div class='del'>-		chunk-&gt;dmaaddrs[i] = dma_map_page(&amp;pcidev-&gt;dev, page, 0,</div><div class='del'>-						  PAGE_SIZE, DMA_BIDIRECTIONAL);</div><div class='del'>-		if (dma_mapping_error(&amp;pcidev-&gt;dev, chunk-&gt;dmaaddrs[i]))</div><div class='del'>-			break;</div><div class='del'>-		addr += PAGE_SIZE;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	chunk-&gt;pg_cnt = i;</div><div class='del'>-	chunk-&gt;type = I40IW_VMALLOC;</div><div class='del'>-	if (i == pg_cnt)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	i40iw_free_vmalloc_mem(hw, chunk);</div><div class='del'>-	return I40IW_ERR_NO_MEMORY;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * fpm_to_idx - given fpm address, get pble index</div><div class='del'>- * @pble_rsrc: pble resource management</div><div class='del'>- * @addr: fpm address for index</div><div class='del'>- */</div><div class='del'>-static inline u32 fpm_to_idx(struct i40iw_hmc_pble_rsrc *pble_rsrc, u64 addr)</div><div class='del'>-{</div><div class='del'>-	return (addr - (pble_rsrc-&gt;fpm_base_addr)) &gt;&gt; 3;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * add_bp_pages - add backing pages for sd</div><div class='del'>- * @dev: hardware control device structure</div><div class='del'>- * @pble_rsrc: pble resource management</div><div class='del'>- * @info: page info for sd</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code add_bp_pages(struct i40iw_sc_dev *dev,</div><div class='del'>-					   struct i40iw_hmc_pble_rsrc *pble_rsrc,</div><div class='del'>-					   struct i40iw_add_page_info *info)</div><div class='del'>-{</div><div class='del'>-	u8 *addr;</div><div class='del'>-	struct i40iw_dma_mem mem;</div><div class='del'>-	struct i40iw_hmc_pd_entry *pd_entry;</div><div class='del'>-	struct i40iw_hmc_sd_entry *sd_entry = info-&gt;sd_entry;</div><div class='del'>-	struct i40iw_hmc_info *hmc_info = info-&gt;hmc_info;</div><div class='del'>-	struct i40iw_chunk *chunk = info-&gt;chunk;</div><div class='del'>-	struct i40iw_manage_vf_pble_info vf_pble_info;</div><div class='del'>-	enum i40iw_status_code status = 0;</div><div class='del'>-	u32 rel_pd_idx = info-&gt;idx.rel_pd_idx;</div><div class='del'>-	u32 pd_idx = info-&gt;idx.pd_idx;</div><div class='del'>-	u32 i;</div><div class='del'>-</div><div class='del'>-	status = i40iw_get_vmalloc_mem(dev-&gt;hw, chunk, info-&gt;pages);</div><div class='del'>-	if (status)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	status = i40iw_add_sd_table_entry(dev-&gt;hw, hmc_info,</div><div class='del'>-					  info-&gt;idx.sd_idx, I40IW_SD_TYPE_PAGED,</div><div class='del'>-					  I40IW_HMC_DIRECT_BP_SIZE);</div><div class='del'>-	if (status)</div><div class='del'>-		goto error;</div><div class='del'>-	if (!dev-&gt;is_pf) {</div><div class='del'>-		status = i40iw_vchnl_vf_add_hmc_objs(dev, I40IW_HMC_IW_PBLE,</div><div class='del'>-						     fpm_to_idx(pble_rsrc,</div><div class='del'>-								pble_rsrc-&gt;next_fpm_addr),</div><div class='del'>-						     (info-&gt;pages &lt;&lt; PBLE_512_SHIFT));</div><div class='del'>-		if (status) {</div><div class='del'>-			i40iw_pr_err("allocate PBLEs in the PF.  Error %i\n", status);</div><div class='del'>-			goto error;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	addr = chunk-&gt;vaddr;</div><div class='del'>-	for (i = 0; i &lt; info-&gt;pages; i++) {</div><div class='del'>-		mem.pa = chunk-&gt;dmaaddrs[i];</div><div class='del'>-		mem.size = PAGE_SIZE;</div><div class='del'>-		mem.va = (void *)(addr);</div><div class='del'>-		pd_entry = &amp;sd_entry-&gt;u.pd_table.pd_entry[rel_pd_idx++];</div><div class='del'>-		if (!pd_entry-&gt;valid) {</div><div class='del'>-			status = i40iw_add_pd_table_entry(dev-&gt;hw, hmc_info, pd_idx++, &amp;mem);</div><div class='del'>-			if (status)</div><div class='del'>-				goto error;</div><div class='del'>-			addr += PAGE_SIZE;</div><div class='del'>-		} else {</div><div class='del'>-			i40iw_pr_err("pd entry is valid expecting to be invalid\n");</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	if (!dev-&gt;is_pf) {</div><div class='del'>-		vf_pble_info.first_pd_index = info-&gt;idx.rel_pd_idx;</div><div class='del'>-		vf_pble_info.inv_pd_ent = false;</div><div class='del'>-		vf_pble_info.pd_entry_cnt = PBLE_PER_PAGE;</div><div class='del'>-		vf_pble_info.pd_pl_pba = sd_entry-&gt;u.pd_table.pd_page_addr.pa;</div><div class='del'>-		vf_pble_info.sd_index = info-&gt;idx.sd_idx;</div><div class='del'>-		status = i40iw_hw_manage_vf_pble_bp(dev-&gt;back_dev,</div><div class='del'>-						    &amp;vf_pble_info, true);</div><div class='del'>-		if (status) {</div><div class='del'>-			i40iw_pr_err("CQP manage VF PBLE BP failed.  %i\n", status);</div><div class='del'>-			goto error;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	chunk-&gt;fpm_addr = pble_rsrc-&gt;next_fpm_addr;</div><div class='del'>-	return 0;</div><div class='del'>-error:</div><div class='del'>-	i40iw_free_vmalloc_mem(dev-&gt;hw, chunk);</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * add_pble_pool - add a sd entry for pble resoure</div><div class='del'>- * @dev: hardware control device structure</div><div class='del'>- * @pble_rsrc: pble resource management</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev,</div><div class='del'>-					    struct i40iw_hmc_pble_rsrc *pble_rsrc)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_hmc_sd_entry *sd_entry;</div><div class='del'>-	struct i40iw_hmc_info *hmc_info;</div><div class='del'>-	struct i40iw_chunk *chunk;</div><div class='del'>-	struct i40iw_add_page_info info;</div><div class='del'>-	struct sd_pd_idx *idx = &amp;info.idx;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-	enum i40iw_sd_entry_type sd_entry_type;</div><div class='del'>-	u64 sd_reg_val = 0;</div><div class='del'>-	u32 pages;</div><div class='del'>-</div><div class='del'>-	if (pble_rsrc-&gt;unallocated_pble &lt; PBLE_PER_PAGE)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	if (pble_rsrc-&gt;next_fpm_addr &amp; 0xfff) {</div><div class='del'>-		i40iw_pr_err("next fpm_addr %llx\n", pble_rsrc-&gt;next_fpm_addr);</div><div class='del'>-		return I40IW_ERR_INVALID_PAGE_DESC_INDEX;</div><div class='del'>-	}</div><div class='del'>-	chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);</div><div class='del'>-	if (!chunk)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	hmc_info = dev-&gt;hmc_info;</div><div class='del'>-	chunk-&gt;fpm_addr = pble_rsrc-&gt;next_fpm_addr;</div><div class='del'>-	get_sd_pd_idx(pble_rsrc, idx);</div><div class='del'>-	sd_entry = &amp;hmc_info-&gt;sd_table.sd_entry[idx-&gt;sd_idx];</div><div class='del'>-	pages = (idx-&gt;rel_pd_idx) ? (I40IW_HMC_PD_CNT_IN_SD -</div><div class='del'>-			idx-&gt;rel_pd_idx) : I40IW_HMC_PD_CNT_IN_SD;</div><div class='del'>-	pages = min(pages, pble_rsrc-&gt;unallocated_pble &gt;&gt; PBLE_512_SHIFT);</div><div class='del'>-	info.chunk = chunk;</div><div class='del'>-	info.hmc_info = hmc_info;</div><div class='del'>-	info.pages = pages;</div><div class='del'>-	info.sd_entry = sd_entry;</div><div class='del'>-	if (!sd_entry-&gt;valid) {</div><div class='del'>-		sd_entry_type = (!idx-&gt;rel_pd_idx &amp;&amp;</div><div class='del'>-				 (pages == I40IW_HMC_PD_CNT_IN_SD) &amp;&amp;</div><div class='del'>-				 dev-&gt;is_pf) ? I40IW_SD_TYPE_DIRECT : I40IW_SD_TYPE_PAGED;</div><div class='del'>-	} else {</div><div class='del'>-		sd_entry_type = sd_entry-&gt;entry_type;</div><div class='del'>-	}</div><div class='del'>-	i40iw_debug(dev, I40IW_DEBUG_PBLE,</div><div class='del'>-		    "pages = %d, unallocated_pble[%u] current_fpm_addr = %llx\n",</div><div class='del'>-		    pages, pble_rsrc-&gt;unallocated_pble, pble_rsrc-&gt;next_fpm_addr);</div><div class='del'>-	i40iw_debug(dev, I40IW_DEBUG_PBLE, "sd_entry_type = %d sd_entry valid = %d\n",</div><div class='del'>-		    sd_entry_type, sd_entry-&gt;valid);</div><div class='del'>-</div><div class='del'>-	if (sd_entry_type == I40IW_SD_TYPE_DIRECT)</div><div class='del'>-		ret_code = add_sd_direct(dev, pble_rsrc, &amp;info);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		sd_entry_type = I40IW_SD_TYPE_PAGED;</div><div class='del'>-	else</div><div class='del'>-		pble_rsrc-&gt;stats_direct_sds++;</div><div class='del'>-</div><div class='del'>-	if (sd_entry_type == I40IW_SD_TYPE_PAGED) {</div><div class='del'>-		ret_code = add_bp_pages(dev, pble_rsrc, &amp;info);</div><div class='del'>-		if (ret_code)</div><div class='del'>-			goto error;</div><div class='del'>-		else</div><div class='del'>-			pble_rsrc-&gt;stats_paged_sds++;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (gen_pool_add_virt(pble_rsrc-&gt;pinfo.pool, (unsigned long)chunk-&gt;vaddr,</div><div class='del'>-			      (phys_addr_t)chunk-&gt;fpm_addr, chunk-&gt;size, -1)) {</div><div class='del'>-		i40iw_pr_err("could not allocate memory by gen_pool_addr_virt()\n");</div><div class='del'>-		ret_code = I40IW_ERR_NO_MEMORY;</div><div class='del'>-		goto error;</div><div class='del'>-	}</div><div class='del'>-	pble_rsrc-&gt;next_fpm_addr += chunk-&gt;size;</div><div class='del'>-	i40iw_debug(dev, I40IW_DEBUG_PBLE, "next_fpm_addr = %llx chunk_size[%u] = 0x%x\n",</div><div class='del'>-		    pble_rsrc-&gt;next_fpm_addr, chunk-&gt;size, chunk-&gt;size);</div><div class='del'>-	pble_rsrc-&gt;unallocated_pble -= (chunk-&gt;size &gt;&gt; 3);</div><div class='del'>-	list_add(&amp;chunk-&gt;list, &amp;pble_rsrc-&gt;pinfo.clist);</div><div class='del'>-	sd_reg_val = (sd_entry_type == I40IW_SD_TYPE_PAGED) ?</div><div class='del'>-			sd_entry-&gt;u.pd_table.pd_page_addr.pa : sd_entry-&gt;u.bp.addr.pa;</div><div class='del'>-	if (sd_entry-&gt;valid)</div><div class='del'>-		return 0;</div><div class='del'>-	if (dev-&gt;is_pf) {</div><div class='del'>-		ret_code = i40iw_hmc_sd_one(dev, hmc_info-&gt;hmc_fn_id,</div><div class='del'>-					    sd_reg_val, idx-&gt;sd_idx,</div><div class='del'>-					    sd_entry-&gt;entry_type, true);</div><div class='del'>-		if (ret_code) {</div><div class='del'>-			i40iw_pr_err("cqp cmd failed for sd (pbles)\n");</div><div class='del'>-			goto error;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	sd_entry-&gt;valid = true;</div><div class='del'>-	return 0;</div><div class='del'>- error:</div><div class='del'>-	kfree(chunk);</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * free_lvl2 - fee level 2 pble</div><div class='del'>- * @pble_rsrc: pble resource management</div><div class='del'>- * @palloc: level 2 pble allocation</div><div class='del'>- */</div><div class='del'>-static void free_lvl2(struct i40iw_hmc_pble_rsrc *pble_rsrc,</div><div class='del'>-		      struct i40iw_pble_alloc *palloc)</div><div class='del'>-{</div><div class='del'>-	u32 i;</div><div class='del'>-	struct gen_pool *pool;</div><div class='del'>-	struct i40iw_pble_level2 *lvl2 = &amp;palloc-&gt;level2;</div><div class='del'>-	struct i40iw_pble_info *root = &amp;lvl2-&gt;root;</div><div class='del'>-	struct i40iw_pble_info *leaf = lvl2-&gt;leaf;</div><div class='del'>-</div><div class='del'>-	pool = pble_rsrc-&gt;pinfo.pool;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; lvl2-&gt;leaf_cnt; i++, leaf++) {</div><div class='del'>-		if (leaf-&gt;addr)</div><div class='del'>-			gen_pool_free(pool, leaf-&gt;addr, (leaf-&gt;cnt &lt;&lt; 3));</div><div class='del'>-		else</div><div class='del'>-			break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (root-&gt;addr)</div><div class='del'>-		gen_pool_free(pool, root-&gt;addr, (root-&gt;cnt &lt;&lt; 3));</div><div class='del'>-</div><div class='del'>-	kfree(lvl2-&gt;leaf);</div><div class='del'>-	lvl2-&gt;leaf = NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * get_lvl2_pble - get level 2 pble resource</div><div class='del'>- * @pble_rsrc: pble resource management</div><div class='del'>- * @palloc: level 2 pble allocation</div><div class='del'>- * @pool: pool pointer</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code get_lvl2_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc,</div><div class='del'>-					    struct i40iw_pble_alloc *palloc,</div><div class='del'>-					    struct gen_pool *pool)</div><div class='del'>-{</div><div class='del'>-	u32 lf4k, lflast, total, i;</div><div class='del'>-	u32 pblcnt = PBLE_PER_PAGE;</div><div class='del'>-	u64 *addr;</div><div class='del'>-	struct i40iw_pble_level2 *lvl2 = &amp;palloc-&gt;level2;</div><div class='del'>-	struct i40iw_pble_info *root = &amp;lvl2-&gt;root;</div><div class='del'>-	struct i40iw_pble_info *leaf;</div><div class='del'>-</div><div class='del'>-	/* number of full 512 (4K) leafs) */</div><div class='del'>-	lf4k = palloc-&gt;total_cnt &gt;&gt; 9;</div><div class='del'>-	lflast = palloc-&gt;total_cnt % PBLE_PER_PAGE;</div><div class='del'>-	total = (lflast == 0) ? lf4k : lf4k + 1;</div><div class='del'>-	lvl2-&gt;leaf_cnt = total;</div><div class='del'>-</div><div class='del'>-	leaf = kzalloc((sizeof(*leaf) * total), GFP_ATOMIC);</div><div class='del'>-	if (!leaf)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	lvl2-&gt;leaf = leaf;</div><div class='del'>-	/* allocate pbles for the root */</div><div class='del'>-	root-&gt;addr = gen_pool_alloc(pool, (total &lt;&lt; 3));</div><div class='del'>-	if (!root-&gt;addr) {</div><div class='del'>-		kfree(lvl2-&gt;leaf);</div><div class='del'>-		lvl2-&gt;leaf = NULL;</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	}</div><div class='del'>-	root-&gt;idx = fpm_to_idx(pble_rsrc,</div><div class='del'>-			       (u64)gen_pool_virt_to_phys(pool, root-&gt;addr));</div><div class='del'>-	root-&gt;cnt = total;</div><div class='del'>-	addr = (u64 *)root-&gt;addr;</div><div class='del'>-	for (i = 0; i &lt; total; i++, leaf++) {</div><div class='del'>-		pblcnt = (lflast &amp;&amp; ((i + 1) == total)) ? lflast : PBLE_PER_PAGE;</div><div class='del'>-		leaf-&gt;addr = gen_pool_alloc(pool, (pblcnt &lt;&lt; 3));</div><div class='del'>-		if (!leaf-&gt;addr)</div><div class='del'>-			goto error;</div><div class='del'>-		leaf-&gt;idx = fpm_to_idx(pble_rsrc, (u64)gen_pool_virt_to_phys(pool, leaf-&gt;addr));</div><div class='del'>-</div><div class='del'>-		leaf-&gt;cnt = pblcnt;</div><div class='del'>-		*addr = (u64)leaf-&gt;idx;</div><div class='del'>-		addr++;</div><div class='del'>-	}</div><div class='del'>-	palloc-&gt;level = I40IW_LEVEL_2;</div><div class='del'>-	pble_rsrc-&gt;stats_lvl2++;</div><div class='del'>-	return 0;</div><div class='del'>- error:</div><div class='del'>-	free_lvl2(pble_rsrc, palloc);</div><div class='del'>-	return I40IW_ERR_NO_MEMORY;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * get_lvl1_pble - get level 1 pble resource</div><div class='del'>- * @dev: hardware control device structure</div><div class='del'>- * @pble_rsrc: pble resource management</div><div class='del'>- * @palloc: level 1 pble allocation</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code get_lvl1_pble(struct i40iw_sc_dev *dev,</div><div class='del'>-					    struct i40iw_hmc_pble_rsrc *pble_rsrc,</div><div class='del'>-					    struct i40iw_pble_alloc *palloc)</div><div class='del'>-{</div><div class='del'>-	u64 *addr;</div><div class='del'>-	struct gen_pool *pool;</div><div class='del'>-	struct i40iw_pble_info *lvl1 = &amp;palloc-&gt;level1;</div><div class='del'>-</div><div class='del'>-	pool = pble_rsrc-&gt;pinfo.pool;</div><div class='del'>-	addr = (u64 *)gen_pool_alloc(pool, (palloc-&gt;total_cnt &lt;&lt; 3));</div><div class='del'>-</div><div class='del'>-	if (!addr)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-</div><div class='del'>-	palloc-&gt;level = I40IW_LEVEL_1;</div><div class='del'>-	lvl1-&gt;addr = (unsigned long)addr;</div><div class='del'>-	lvl1-&gt;idx = fpm_to_idx(pble_rsrc, (u64)gen_pool_virt_to_phys(pool,</div><div class='del'>-			       (unsigned long)addr));</div><div class='del'>-	lvl1-&gt;cnt = palloc-&gt;total_cnt;</div><div class='del'>-	pble_rsrc-&gt;stats_lvl1++;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * get_lvl1_lvl2_pble - calls get_lvl1 and get_lvl2 pble routine</div><div class='del'>- * @dev: i40iw_sc_dev struct</div><div class='del'>- * @pble_rsrc:	pble resources</div><div class='del'>- * @palloc: contains all inforamtion regarding pble (idx + pble addr)</div><div class='del'>- * @pool: pointer to general purpose special memory pool descriptor</div><div class='del'>- */</div><div class='del'>-static inline enum i40iw_status_code get_lvl1_lvl2_pble(struct i40iw_sc_dev *dev,</div><div class='del'>-							struct i40iw_hmc_pble_rsrc *pble_rsrc,</div><div class='del'>-							struct i40iw_pble_alloc *palloc,</div><div class='del'>-							struct gen_pool *pool)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status = 0;</div><div class='del'>-</div><div class='del'>-	status = get_lvl1_pble(dev, pble_rsrc, palloc);</div><div class='del'>-	if (status &amp;&amp; (palloc-&gt;total_cnt &gt; PBLE_PER_PAGE))</div><div class='del'>-		status = get_lvl2_pble(pble_rsrc, palloc, pool);</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_pble - allocate pbles from the pool</div><div class='del'>- * @dev: i40iw_sc_dev struct</div><div class='del'>- * @pble_rsrc:	pble resources</div><div class='del'>- * @palloc: contains all inforamtion regarding pble (idx + pble addr)</div><div class='del'>- * @pble_cnt: #of pbles requested</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_get_pble(struct i40iw_sc_dev *dev,</div><div class='del'>-				      struct i40iw_hmc_pble_rsrc *pble_rsrc,</div><div class='del'>-				      struct i40iw_pble_alloc *palloc,</div><div class='del'>-				      u32 pble_cnt)</div><div class='del'>-{</div><div class='del'>-	struct gen_pool *pool;</div><div class='del'>-	enum i40iw_status_code status = 0;</div><div class='del'>-	u32 max_sds = 0;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	pool = pble_rsrc-&gt;pinfo.pool;</div><div class='del'>-	palloc-&gt;total_cnt = pble_cnt;</div><div class='del'>-	palloc-&gt;level = I40IW_LEVEL_0;</div><div class='del'>-	/*check first to see if we can get pble's without acquiring additional sd's */</div><div class='del'>-	status = get_lvl1_lvl2_pble(dev, pble_rsrc, palloc, pool);</div><div class='del'>-	if (!status)</div><div class='del'>-		goto exit;</div><div class='del'>-	max_sds = (palloc-&gt;total_cnt &gt;&gt; 18) + 1;</div><div class='del'>-	for (i = 0; i &lt; max_sds; i++) {</div><div class='del'>-		status = add_pble_pool(dev, pble_rsrc);</div><div class='del'>-		if (status)</div><div class='del'>-			break;</div><div class='del'>-		status = get_lvl1_lvl2_pble(dev, pble_rsrc, palloc, pool);</div><div class='del'>-		if (!status)</div><div class='del'>-			break;</div><div class='del'>-	}</div><div class='del'>-exit:</div><div class='del'>-	if (!status)</div><div class='del'>-		pble_rsrc-&gt;stats_alloc_ok++;</div><div class='del'>-	else</div><div class='del'>-		pble_rsrc-&gt;stats_alloc_fail++;</div><div class='del'>-</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_free_pble - put pbles back into pool</div><div class='del'>- * @pble_rsrc:	pble resources</div><div class='del'>- * @palloc: contains all inforamtion regarding pble resource being freed</div><div class='del'>- */</div><div class='del'>-void i40iw_free_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc,</div><div class='del'>-		     struct i40iw_pble_alloc *palloc)</div><div class='del'>-{</div><div class='del'>-	struct gen_pool *pool;</div><div class='del'>-</div><div class='del'>-	pool = pble_rsrc-&gt;pinfo.pool;</div><div class='del'>-	if (palloc-&gt;level == I40IW_LEVEL_2)</div><div class='del'>-		free_lvl2(pble_rsrc, palloc);</div><div class='del'>-	else</div><div class='del'>-		gen_pool_free(pool, palloc-&gt;level1.addr,</div><div class='del'>-			      (palloc-&gt;level1.cnt &lt;&lt; 3));</div><div class='del'>-	pble_rsrc-&gt;stats_alloc_freed++;</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_pble.h b/drivers/infiniband/hw/i40iw/i40iw_pble.h<br/>deleted file mode 100644<br/>index 7b1851d21cc0..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_pble.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_pble.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,131 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#ifndef I40IW_PBLE_H</div><div class='del'>-#define I40IW_PBLE_H</div><div class='del'>-</div><div class='del'>-#define POOL_SHIFT      6</div><div class='del'>-#define PBLE_PER_PAGE   512</div><div class='del'>-#define I40IW_HMC_PAGED_BP_SHIFT 12</div><div class='del'>-#define PBLE_512_SHIFT  9</div><div class='del'>-</div><div class='del'>-enum i40iw_pble_level {</div><div class='del'>-	I40IW_LEVEL_0 = 0,</div><div class='del'>-	I40IW_LEVEL_1 = 1,</div><div class='del'>-	I40IW_LEVEL_2 = 2</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_alloc_type {</div><div class='del'>-	I40IW_NO_ALLOC = 0,</div><div class='del'>-	I40IW_DMA_COHERENT = 1,</div><div class='del'>-	I40IW_VMALLOC = 2</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_pble_info {</div><div class='del'>-	unsigned long addr;</div><div class='del'>-	u32 idx;</div><div class='del'>-	u32 cnt;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_pble_level2 {</div><div class='del'>-	struct i40iw_pble_info root;</div><div class='del'>-	struct i40iw_pble_info *leaf;</div><div class='del'>-	u32 leaf_cnt;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_pble_alloc {</div><div class='del'>-	u32 total_cnt;</div><div class='del'>-	enum i40iw_pble_level level;</div><div class='del'>-	union {</div><div class='del'>-		struct i40iw_pble_info level1;</div><div class='del'>-		struct i40iw_pble_level2 level2;</div><div class='del'>-	};</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct sd_pd_idx {</div><div class='del'>-	u32 sd_idx;</div><div class='del'>-	u32 pd_idx;</div><div class='del'>-	u32 rel_pd_idx;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_add_page_info {</div><div class='del'>-	struct i40iw_chunk *chunk;</div><div class='del'>-	struct i40iw_hmc_sd_entry *sd_entry;</div><div class='del'>-	struct i40iw_hmc_info *hmc_info;</div><div class='del'>-	struct sd_pd_idx idx;</div><div class='del'>-	u32 pages;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_chunk {</div><div class='del'>-	struct list_head list;</div><div class='del'>-	u32 size;</div><div class='del'>-	void *vaddr;</div><div class='del'>-	u64 fpm_addr;</div><div class='del'>-	u32 pg_cnt;</div><div class='del'>-	dma_addr_t *dmaaddrs;</div><div class='del'>-	enum i40iw_alloc_type type;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_pble_pool {</div><div class='del'>-	struct gen_pool *pool;</div><div class='del'>-	struct list_head clist;</div><div class='del'>-	u32 total_pble_alloc;</div><div class='del'>-	u32 free_pble_cnt;</div><div class='del'>-	u32 pool_shift;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_hmc_pble_rsrc {</div><div class='del'>-	u32 unallocated_pble;</div><div class='del'>-	u64 fpm_base_addr;</div><div class='del'>-	u64 next_fpm_addr;</div><div class='del'>-	struct i40iw_pble_pool pinfo;</div><div class='del'>-</div><div class='del'>-	u32 stats_direct_sds;</div><div class='del'>-	u32 stats_paged_sds;</div><div class='del'>-	u64 stats_alloc_ok;</div><div class='del'>-	u64 stats_alloc_fail;</div><div class='del'>-	u64 stats_alloc_freed;</div><div class='del'>-	u64 stats_lvl1;</div><div class='del'>-	u64 stats_lvl2;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-void i40iw_destroy_pble_pool(struct i40iw_sc_dev *dev, struct i40iw_hmc_pble_rsrc *pble_rsrc);</div><div class='del'>-enum i40iw_status_code i40iw_hmc_init_pble(struct i40iw_sc_dev *dev,</div><div class='del'>-					   struct i40iw_hmc_pble_rsrc *pble_rsrc);</div><div class='del'>-void i40iw_free_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc, struct i40iw_pble_alloc *palloc);</div><div class='del'>-enum i40iw_status_code i40iw_get_pble(struct i40iw_sc_dev *dev,</div><div class='del'>-				      struct i40iw_hmc_pble_rsrc *pble_rsrc,</div><div class='del'>-				      struct i40iw_pble_alloc *palloc,</div><div class='del'>-				      u32 pble_cnt);</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_puda.c b/drivers/infiniband/hw/i40iw/i40iw_puda.c<br/>deleted file mode 100644<br/>index d9c7ae6a7030..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_puda.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_puda.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,1493 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#include "i40iw_osdep.h"</div><div class='del'>-#include "i40iw_register.h"</div><div class='del'>-#include "i40iw_status.h"</div><div class='del'>-#include "i40iw_hmc.h"</div><div class='del'>-</div><div class='del'>-#include "i40iw_d.h"</div><div class='del'>-#include "i40iw_type.h"</div><div class='del'>-#include "i40iw_p.h"</div><div class='del'>-#include "i40iw_puda.h"</div><div class='del'>-</div><div class='del'>-static void i40iw_ieq_receive(struct i40iw_sc_vsi *vsi,</div><div class='del'>-			      struct i40iw_puda_buf *buf);</div><div class='del'>-static void i40iw_ieq_tx_compl(struct i40iw_sc_vsi *vsi, void *sqwrid);</div><div class='del'>-static void i40iw_ilq_putback_rcvbuf(struct i40iw_sc_qp *qp, u32 wqe_idx);</div><div class='del'>-static enum i40iw_status_code i40iw_puda_replenish_rq(struct i40iw_puda_rsrc</div><div class='del'>-						      *rsrc, bool initial);</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_get_listbuf - get buffer from puda list</div><div class='del'>- * @list: list to use for buffers (ILQ or IEQ)</div><div class='del'>- */</div><div class='del'>-static struct i40iw_puda_buf *i40iw_puda_get_listbuf(struct list_head *list)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_buf *buf = NULL;</div><div class='del'>-</div><div class='del'>-	if (!list_empty(list)) {</div><div class='del'>-		buf = (struct i40iw_puda_buf *)list-&gt;next;</div><div class='del'>-		list_del((struct list_head *)&amp;buf-&gt;list);</div><div class='del'>-	}</div><div class='del'>-	return buf;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_get_bufpool - return buffer from resource</div><div class='del'>- * @rsrc: resource to use for buffer</div><div class='del'>- */</div><div class='del'>-struct i40iw_puda_buf *i40iw_puda_get_bufpool(struct i40iw_puda_rsrc *rsrc)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_buf *buf = NULL;</div><div class='del'>-	struct list_head *list = &amp;rsrc-&gt;bufpool;</div><div class='del'>-	unsigned long	flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='del'>-	buf = i40iw_puda_get_listbuf(list);</div><div class='del'>-	if (buf)</div><div class='del'>-		rsrc-&gt;avail_buf_count--;</div><div class='del'>-	else</div><div class='del'>-		rsrc-&gt;stats_buf_alloc_fail++;</div><div class='del'>-	spin_unlock_irqrestore(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='del'>-	return buf;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_ret_bufpool - return buffer to rsrc list</div><div class='del'>- * @rsrc: resource to use for buffer</div><div class='del'>- * @buf: buffe to return to resouce</div><div class='del'>- */</div><div class='del'>-void i40iw_puda_ret_bufpool(struct i40iw_puda_rsrc *rsrc,</div><div class='del'>-			    struct i40iw_puda_buf *buf)</div><div class='del'>-{</div><div class='del'>-	unsigned long	flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='del'>-	list_add(&amp;buf-&gt;list, &amp;rsrc-&gt;bufpool);</div><div class='del'>-	spin_unlock_irqrestore(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='del'>-	rsrc-&gt;avail_buf_count++;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_post_recvbuf - set wqe for rcv buffer</div><div class='del'>- * @rsrc: resource ptr</div><div class='del'>- * @wqe_idx: wqe index to use</div><div class='del'>- * @buf: puda buffer for rcv q</div><div class='del'>- * @initial: flag if during init time</div><div class='del'>- */</div><div class='del'>-static void i40iw_puda_post_recvbuf(struct i40iw_puda_rsrc *rsrc, u32 wqe_idx,</div><div class='del'>-				    struct i40iw_puda_buf *buf, bool initial)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_sc_qp *qp = &amp;rsrc-&gt;qp;</div><div class='del'>-	u64 offset24 = 0;</div><div class='del'>-</div><div class='del'>-	qp-&gt;qp_uk.rq_wrid_array[wqe_idx] = (uintptr_t)buf;</div><div class='del'>-	wqe = qp-&gt;qp_uk.rq_base[wqe_idx].elem;</div><div class='del'>-	i40iw_debug(rsrc-&gt;dev, I40IW_DEBUG_PUDA,</div><div class='del'>-		    "%s: wqe_idx= %d buf = %p wqe = %p\n", __func__,</div><div class='del'>-		    wqe_idx, buf, wqe);</div><div class='del'>-	if (!initial)</div><div class='del'>-		get_64bit_val(wqe, 24, &amp;offset24);</div><div class='del'>-</div><div class='del'>-	offset24 = (offset24) ? 0 : LS_64(1, I40IWQPSQ_VALID);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 0, buf-&gt;mem.pa);</div><div class='del'>-	set_64bit_val(wqe, 8,</div><div class='del'>-		      LS_64(buf-&gt;mem.size, I40IWQPSQ_FRAG_LEN));</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, offset24);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_replenish_rq - post rcv buffers</div><div class='del'>- * @rsrc: resource to use for buffer</div><div class='del'>- * @initial: flag if during init time</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_puda_replenish_rq(struct i40iw_puda_rsrc *rsrc,</div><div class='del'>-						      bool initial)</div><div class='del'>-{</div><div class='del'>-	u32 i;</div><div class='del'>-	u32 invalid_cnt = rsrc-&gt;rxq_invalid_cnt;</div><div class='del'>-	struct i40iw_puda_buf *buf = NULL;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; invalid_cnt; i++) {</div><div class='del'>-		buf = i40iw_puda_get_bufpool(rsrc);</div><div class='del'>-		if (!buf)</div><div class='del'>-			return I40IW_ERR_list_empty;</div><div class='del'>-		i40iw_puda_post_recvbuf(rsrc, rsrc-&gt;rx_wqe_idx, buf,</div><div class='del'>-					initial);</div><div class='del'>-		rsrc-&gt;rx_wqe_idx =</div><div class='del'>-		    ((rsrc-&gt;rx_wqe_idx + 1) % rsrc-&gt;rq_size);</div><div class='del'>-		rsrc-&gt;rxq_invalid_cnt--;</div><div class='del'>-	}</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_alloc_buf - allocate mem for buffer</div><div class='del'>- * @dev: iwarp device</div><div class='del'>- * @length: length of buffer</div><div class='del'>- */</div><div class='del'>-static struct i40iw_puda_buf *i40iw_puda_alloc_buf(struct i40iw_sc_dev *dev,</div><div class='del'>-						   u32 length)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_buf *buf = NULL;</div><div class='del'>-	struct i40iw_virt_mem buf_mem;</div><div class='del'>-	enum i40iw_status_code ret;</div><div class='del'>-</div><div class='del'>-	ret = i40iw_allocate_virt_mem(dev-&gt;hw, &amp;buf_mem,</div><div class='del'>-				      sizeof(struct i40iw_puda_buf));</div><div class='del'>-	if (ret) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_PUDA,</div><div class='del'>-			    "%s: error mem for buf\n", __func__);</div><div class='del'>-		return NULL;</div><div class='del'>-	}</div><div class='del'>-	buf = (struct i40iw_puda_buf *)buf_mem.va;</div><div class='del'>-	ret = i40iw_allocate_dma_mem(dev-&gt;hw, &amp;buf-&gt;mem, length, 1);</div><div class='del'>-	if (ret) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_PUDA,</div><div class='del'>-			    "%s: error dma mem for buf\n", __func__);</div><div class='del'>-		i40iw_free_virt_mem(dev-&gt;hw, &amp;buf_mem);</div><div class='del'>-		return NULL;</div><div class='del'>-	}</div><div class='del'>-	buf-&gt;buf_mem.va = buf_mem.va;</div><div class='del'>-	buf-&gt;buf_mem.size = buf_mem.size;</div><div class='del'>-	return buf;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_dele_buf - delete buffer back to system</div><div class='del'>- * @dev: iwarp device</div><div class='del'>- * @buf: buffer to free</div><div class='del'>- */</div><div class='del'>-static void i40iw_puda_dele_buf(struct i40iw_sc_dev *dev,</div><div class='del'>-				struct i40iw_puda_buf *buf)</div><div class='del'>-{</div><div class='del'>-	i40iw_free_dma_mem(dev-&gt;hw, &amp;buf-&gt;mem);</div><div class='del'>-	i40iw_free_virt_mem(dev-&gt;hw, &amp;buf-&gt;buf_mem);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_get_next_send_wqe - return next wqe for processing</div><div class='del'>- * @qp: puda qp for wqe</div><div class='del'>- * @wqe_idx: wqe index for caller</div><div class='del'>- */</div><div class='del'>-static u64 *i40iw_puda_get_next_send_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe = NULL;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-</div><div class='del'>-	*wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp-&gt;sq_ring);</div><div class='del'>-	if (!*wqe_idx)</div><div class='del'>-		qp-&gt;swqe_polarity = !qp-&gt;swqe_polarity;</div><div class='del'>-	I40IW_RING_MOVE_HEAD(qp-&gt;sq_ring, ret_code);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return wqe;</div><div class='del'>-	wqe = qp-&gt;sq_base[*wqe_idx].elem;</div><div class='del'>-</div><div class='del'>-	return wqe;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_poll_info - poll cq for completion</div><div class='del'>- * @cq: cq for poll</div><div class='del'>- * @info: info return for successful completion</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_puda_poll_info(struct i40iw_sc_cq *cq,</div><div class='del'>-						   struct i40iw_puda_completion_info *info)</div><div class='del'>-{</div><div class='del'>-	u64 qword0, qword2, qword3;</div><div class='del'>-	u64 *cqe;</div><div class='del'>-	u64 comp_ctx;</div><div class='del'>-	bool valid_bit;</div><div class='del'>-	u32 major_err, minor_err;</div><div class='del'>-	bool error;</div><div class='del'>-</div><div class='del'>-	cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(&amp;cq-&gt;cq_uk);</div><div class='del'>-	get_64bit_val(cqe, 24, &amp;qword3);</div><div class='del'>-	valid_bit = (bool)RS_64(qword3, I40IW_CQ_VALID);</div><div class='del'>-</div><div class='del'>-	if (valid_bit != cq-&gt;cq_uk.polarity)</div><div class='del'>-		return I40IW_ERR_QUEUE_EMPTY;</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cq-&gt;dev, I40IW_DEBUG_PUDA, "PUDA CQE", cqe, 32);</div><div class='del'>-	error = (bool)RS_64(qword3, I40IW_CQ_ERROR);</div><div class='del'>-	if (error) {</div><div class='del'>-		i40iw_debug(cq-&gt;dev, I40IW_DEBUG_PUDA, "%s receive error\n", __func__);</div><div class='del'>-		major_err = (u32)(RS_64(qword3, I40IW_CQ_MAJERR));</div><div class='del'>-		minor_err = (u32)(RS_64(qword3, I40IW_CQ_MINERR));</div><div class='del'>-		info-&gt;compl_error = major_err &lt;&lt; 16 | minor_err;</div><div class='del'>-		return I40IW_ERR_CQ_COMPL_ERROR;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	get_64bit_val(cqe, 0, &amp;qword0);</div><div class='del'>-	get_64bit_val(cqe, 16, &amp;qword2);</div><div class='del'>-</div><div class='del'>-	info-&gt;q_type = (u8)RS_64(qword3, I40IW_CQ_SQ);</div><div class='del'>-	info-&gt;qp_id = (u32)RS_64(qword2, I40IWCQ_QPID);</div><div class='del'>-</div><div class='del'>-	get_64bit_val(cqe, 8, &amp;comp_ctx);</div><div class='del'>-	info-&gt;qp = (struct i40iw_qp_uk *)(unsigned long)comp_ctx;</div><div class='del'>-	info-&gt;wqe_idx = (u32)RS_64(qword3, I40IW_CQ_WQEIDX);</div><div class='del'>-</div><div class='del'>-	if (info-&gt;q_type == I40IW_CQE_QTYPE_RQ) {</div><div class='del'>-		info-&gt;vlan_valid = (bool)RS_64(qword3, I40IW_VLAN_TAG_VALID);</div><div class='del'>-		info-&gt;l4proto = (u8)RS_64(qword2, I40IW_UDA_L4PROTO);</div><div class='del'>-		info-&gt;l3proto = (u8)RS_64(qword2, I40IW_UDA_L3PROTO);</div><div class='del'>-		info-&gt;payload_len = (u16)RS_64(qword0, I40IW_UDA_PAYLOADLEN);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_poll_completion - processes completion for cq</div><div class='del'>- * @dev: iwarp device</div><div class='del'>- * @cq: cq getting interrupt</div><div class='del'>- * @compl_err: return any completion err</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_puda_poll_completion(struct i40iw_sc_dev *dev,</div><div class='del'>-						  struct i40iw_sc_cq *cq, u32 *compl_err)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp_uk *qp;</div><div class='del'>-	struct i40iw_cq_uk *cq_uk = &amp;cq-&gt;cq_uk;</div><div class='del'>-	struct i40iw_puda_completion_info info;</div><div class='del'>-	enum i40iw_status_code ret = 0;</div><div class='del'>-	struct i40iw_puda_buf *buf;</div><div class='del'>-	struct i40iw_puda_rsrc *rsrc;</div><div class='del'>-	void *sqwrid;</div><div class='del'>-	u8 cq_type = cq-&gt;cq_type;</div><div class='del'>-	unsigned long	flags;</div><div class='del'>-</div><div class='del'>-	if ((cq_type == I40IW_CQ_TYPE_ILQ) || (cq_type == I40IW_CQ_TYPE_IEQ)) {</div><div class='del'>-		rsrc = (cq_type == I40IW_CQ_TYPE_ILQ) ? cq-&gt;vsi-&gt;ilq : cq-&gt;vsi-&gt;ieq;</div><div class='del'>-	} else {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s qp_type error\n", __func__);</div><div class='del'>-		return I40IW_ERR_BAD_PTR;</div><div class='del'>-	}</div><div class='del'>-	memset(&amp;info, 0, sizeof(info));</div><div class='del'>-	ret = i40iw_puda_poll_info(cq, &amp;info);</div><div class='del'>-	*compl_err = info.compl_error;</div><div class='del'>-	if (ret == I40IW_ERR_QUEUE_EMPTY)</div><div class='del'>-		return ret;</div><div class='del'>-	if (ret)</div><div class='del'>-		goto done;</div><div class='del'>-</div><div class='del'>-	qp = info.qp;</div><div class='del'>-	if (!qp || !rsrc) {</div><div class='del'>-		ret = I40IW_ERR_BAD_PTR;</div><div class='del'>-		goto done;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (qp-&gt;qp_id != rsrc-&gt;qp_id) {</div><div class='del'>-		ret = I40IW_ERR_BAD_PTR;</div><div class='del'>-		goto done;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (info.q_type == I40IW_CQE_QTYPE_RQ) {</div><div class='del'>-		buf = (struct i40iw_puda_buf *)(uintptr_t)qp-&gt;rq_wrid_array[info.wqe_idx];</div><div class='del'>-		/* Get all the tcpip information in the buf header */</div><div class='del'>-		ret = i40iw_puda_get_tcpip_info(&amp;info, buf);</div><div class='del'>-		if (ret) {</div><div class='del'>-			rsrc-&gt;stats_rcvd_pkt_err++;</div><div class='del'>-			if (cq_type == I40IW_CQ_TYPE_ILQ) {</div><div class='del'>-				i40iw_ilq_putback_rcvbuf(&amp;rsrc-&gt;qp,</div><div class='del'>-							 info.wqe_idx);</div><div class='del'>-			} else {</div><div class='del'>-				i40iw_puda_ret_bufpool(rsrc, buf);</div><div class='del'>-				i40iw_puda_replenish_rq(rsrc, false);</div><div class='del'>-			}</div><div class='del'>-			goto done;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		rsrc-&gt;stats_pkt_rcvd++;</div><div class='del'>-		rsrc-&gt;compl_rxwqe_idx = info.wqe_idx;</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s RQ completion\n", __func__);</div><div class='del'>-		rsrc-&gt;receive(rsrc-&gt;vsi, buf);</div><div class='del'>-		if (cq_type == I40IW_CQ_TYPE_ILQ)</div><div class='del'>-			i40iw_ilq_putback_rcvbuf(&amp;rsrc-&gt;qp, info.wqe_idx);</div><div class='del'>-		else</div><div class='del'>-			i40iw_puda_replenish_rq(rsrc, false);</div><div class='del'>-</div><div class='del'>-	} else {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s SQ completion\n", __func__);</div><div class='del'>-		sqwrid = (void *)(uintptr_t)qp-&gt;sq_wrtrk_array[info.wqe_idx].wrid;</div><div class='del'>-		I40IW_RING_SET_TAIL(qp-&gt;sq_ring, info.wqe_idx);</div><div class='del'>-		rsrc-&gt;xmit_complete(rsrc-&gt;vsi, sqwrid);</div><div class='del'>-		spin_lock_irqsave(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='del'>-		rsrc-&gt;tx_wqe_avail_cnt++;</div><div class='del'>-		spin_unlock_irqrestore(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='del'>-		if (!list_empty(&amp;rsrc-&gt;txpend))</div><div class='del'>-			i40iw_puda_send_buf(rsrc, NULL);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-done:</div><div class='del'>-	I40IW_RING_MOVE_HEAD(cq_uk-&gt;cq_ring, ret);</div><div class='del'>-	if (I40IW_RING_GETCURRENT_HEAD(cq_uk-&gt;cq_ring) == 0)</div><div class='del'>-		cq_uk-&gt;polarity = !cq_uk-&gt;polarity;</div><div class='del'>-	/* update cq tail in cq shadow memory also */</div><div class='del'>-	I40IW_RING_MOVE_TAIL(cq_uk-&gt;cq_ring);</div><div class='del'>-	set_64bit_val(cq_uk-&gt;shadow_area, 0,</div><div class='del'>-		      I40IW_RING_GETCURRENT_HEAD(cq_uk-&gt;cq_ring));</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_send - complete send wqe for transmit</div><div class='del'>- * @qp: puda qp for send</div><div class='del'>- * @info: buffer information for transmit</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_puda_send(struct i40iw_sc_qp *qp,</div><div class='del'>-				       struct i40iw_puda_send_info *info)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u32 iplen, l4len;</div><div class='del'>-	u64 header[2];</div><div class='del'>-	u32 wqe_idx;</div><div class='del'>-	u8 iipt;</div><div class='del'>-</div><div class='del'>-	/* number of 32 bits DWORDS in header */</div><div class='del'>-	l4len = info-&gt;tcplen &gt;&gt; 2;</div><div class='del'>-	if (info-&gt;ipv4) {</div><div class='del'>-		iipt = 3;</div><div class='del'>-		iplen = 5;</div><div class='del'>-	} else {</div><div class='del'>-		iipt = 1;</div><div class='del'>-		iplen = 10;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_puda_get_next_send_wqe(&amp;qp-&gt;qp_uk, &amp;wqe_idx);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_QP_TOOMANY_WRS_POSTED;</div><div class='del'>-	qp-&gt;qp_uk.sq_wrtrk_array[wqe_idx].wrid = (uintptr_t)info-&gt;scratch;</div><div class='del'>-	/* Third line of WQE descriptor */</div><div class='del'>-	/* maclen is in words */</div><div class='del'>-	header[0] = LS_64((info-&gt;maclen &gt;&gt; 1), I40IW_UDA_QPSQ_MACLEN) |</div><div class='del'>-		    LS_64(iplen, I40IW_UDA_QPSQ_IPLEN) | LS_64(1, I40IW_UDA_QPSQ_L4T) |</div><div class='del'>-		    LS_64(iipt, I40IW_UDA_QPSQ_IIPT) |</div><div class='del'>-		    LS_64(l4len, I40IW_UDA_QPSQ_L4LEN);</div><div class='del'>-	/* Forth line of WQE descriptor */</div><div class='del'>-	header[1] = LS_64(I40IW_OP_TYPE_SEND, I40IW_UDA_QPSQ_OPCODE) |</div><div class='del'>-		    LS_64(1, I40IW_UDA_QPSQ_SIGCOMPL) |</div><div class='del'>-		    LS_64(info-&gt;doloopback, I40IW_UDA_QPSQ_DOLOOPBACK) |</div><div class='del'>-		    LS_64(qp-&gt;qp_uk.swqe_polarity, I40IW_UDA_QPSQ_VALID);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 0, info-&gt;paddr);</div><div class='del'>-	set_64bit_val(wqe, 8, LS_64(info-&gt;len, I40IWQPSQ_FRAG_LEN));</div><div class='del'>-	set_64bit_val(wqe, 16, header[0]);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header[1]);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(qp-&gt;dev, I40IW_DEBUG_PUDA, "PUDA SEND WQE", wqe, 32);</div><div class='del'>-	i40iw_qp_post_wr(&amp;qp-&gt;qp_uk);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_send_buf - transmit puda buffer</div><div class='del'>- * @rsrc: resource to use for buffer</div><div class='del'>- * @buf: puda buffer to transmit</div><div class='del'>- */</div><div class='del'>-void i40iw_puda_send_buf(struct i40iw_puda_rsrc *rsrc, struct i40iw_puda_buf *buf)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_send_info info;</div><div class='del'>-	enum i40iw_status_code ret = 0;</div><div class='del'>-	unsigned long	flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='del'>-	/* if no wqe available or not from a completion and we have</div><div class='del'>-	 * pending buffers, we must queue new buffer</div><div class='del'>-	 */</div><div class='del'>-	if (!rsrc-&gt;tx_wqe_avail_cnt || (buf &amp;&amp; !list_empty(&amp;rsrc-&gt;txpend))) {</div><div class='del'>-		list_add_tail(&amp;buf-&gt;list, &amp;rsrc-&gt;txpend);</div><div class='del'>-		spin_unlock_irqrestore(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='del'>-		rsrc-&gt;stats_sent_pkt_q++;</div><div class='del'>-		if (rsrc-&gt;type == I40IW_PUDA_RSRC_TYPE_ILQ)</div><div class='del'>-			i40iw_debug(rsrc-&gt;dev, I40IW_DEBUG_PUDA,</div><div class='del'>-				    "%s: adding to txpend\n", __func__);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-	rsrc-&gt;tx_wqe_avail_cnt--;</div><div class='del'>-	/* if we are coming from a completion and have pending buffers</div><div class='del'>-	 * then Get one from pending list</div><div class='del'>-	 */</div><div class='del'>-	if (!buf) {</div><div class='del'>-		buf = i40iw_puda_get_listbuf(&amp;rsrc-&gt;txpend);</div><div class='del'>-		if (!buf)</div><div class='del'>-			goto done;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	info.scratch = (void *)buf;</div><div class='del'>-	info.paddr = buf-&gt;mem.pa;</div><div class='del'>-	info.len = buf-&gt;totallen;</div><div class='del'>-	info.tcplen = buf-&gt;tcphlen;</div><div class='del'>-	info.maclen = buf-&gt;maclen;</div><div class='del'>-	info.ipv4 = buf-&gt;ipv4;</div><div class='del'>-	info.doloopback = (rsrc-&gt;type == I40IW_PUDA_RSRC_TYPE_IEQ);</div><div class='del'>-</div><div class='del'>-	ret = i40iw_puda_send(&amp;rsrc-&gt;qp, &amp;info);</div><div class='del'>-	if (ret) {</div><div class='del'>-		rsrc-&gt;tx_wqe_avail_cnt++;</div><div class='del'>-		rsrc-&gt;stats_sent_pkt_q++;</div><div class='del'>-		list_add(&amp;buf-&gt;list, &amp;rsrc-&gt;txpend);</div><div class='del'>-		if (rsrc-&gt;type == I40IW_PUDA_RSRC_TYPE_ILQ)</div><div class='del'>-			i40iw_debug(rsrc-&gt;dev, I40IW_DEBUG_PUDA,</div><div class='del'>-				    "%s: adding to puda_send\n", __func__);</div><div class='del'>-	} else {</div><div class='del'>-		rsrc-&gt;stats_pkt_sent++;</div><div class='del'>-	}</div><div class='del'>-done:</div><div class='del'>-	spin_unlock_irqrestore(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_qp_setctx - during init, set qp's context</div><div class='del'>- * @rsrc: qp's resource</div><div class='del'>- */</div><div class='del'>-static void i40iw_puda_qp_setctx(struct i40iw_puda_rsrc *rsrc)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_qp *qp = &amp;rsrc-&gt;qp;</div><div class='del'>-	u64 *qp_ctx = qp-&gt;hw_host_ctx;</div><div class='del'>-</div><div class='del'>-	set_64bit_val(qp_ctx, 8, qp-&gt;sq_pa);</div><div class='del'>-	set_64bit_val(qp_ctx, 16, qp-&gt;rq_pa);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(qp_ctx, 24,</div><div class='del'>-		      LS_64(qp-&gt;hw_rq_size, I40IWQPC_RQSIZE) |</div><div class='del'>-		      LS_64(qp-&gt;hw_sq_size, I40IWQPC_SQSIZE));</div><div class='del'>-</div><div class='del'>-	set_64bit_val(qp_ctx, 48, LS_64(rsrc-&gt;buf_size, I40IW_UDA_QPC_MAXFRAMESIZE));</div><div class='del'>-	set_64bit_val(qp_ctx, 56, 0);</div><div class='del'>-	set_64bit_val(qp_ctx, 64, 1);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(qp_ctx, 136,</div><div class='del'>-		      LS_64(rsrc-&gt;cq_id, I40IWQPC_TXCQNUM) |</div><div class='del'>-		      LS_64(rsrc-&gt;cq_id, I40IWQPC_RXCQNUM));</div><div class='del'>-</div><div class='del'>-	set_64bit_val(qp_ctx, 160, LS_64(1, I40IWQPC_PRIVEN));</div><div class='del'>-</div><div class='del'>-	set_64bit_val(qp_ctx, 168,</div><div class='del'>-		      LS_64((uintptr_t)qp, I40IWQPC_QPCOMPCTX));</div><div class='del'>-</div><div class='del'>-	set_64bit_val(qp_ctx, 176,</div><div class='del'>-		      LS_64(qp-&gt;sq_tph_val, I40IWQPC_SQTPHVAL) |</div><div class='del'>-		      LS_64(qp-&gt;rq_tph_val, I40IWQPC_RQTPHVAL) |</div><div class='del'>-		      LS_64(qp-&gt;qs_handle, I40IWQPC_QSHANDLE));</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(rsrc-&gt;dev, I40IW_DEBUG_PUDA, "PUDA QP CONTEXT",</div><div class='del'>-			qp_ctx, I40IW_QP_CTX_SIZE);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_qp_wqe - setup wqe for qp create</div><div class='del'>- * @rsrc: resource for qp</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_puda_qp_wqe(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-	struct i40iw_ccq_cqe_info compl_info;</div><div class='del'>-	enum i40iw_status_code status = 0;</div><div class='del'>-</div><div class='del'>-	cqp = dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, 0);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 16, qp-&gt;hw_host_ctx_pa);</div><div class='del'>-	set_64bit_val(wqe, 40, qp-&gt;shadow_area_pa);</div><div class='del'>-	header = qp-&gt;qp_uk.qp_id |</div><div class='del'>-		 LS_64(I40IW_CQP_OP_CREATE_QP, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(I40IW_QP_TYPE_UDA, I40IW_CQPSQ_QP_QPTYPE) |</div><div class='del'>-		 LS_64(1, I40IW_CQPSQ_QP_CQNUMVALID) |</div><div class='del'>-		 LS_64(2, I40IW_CQPSQ_QP_NEXTIWSTATE) |</div><div class='del'>-		 LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_PUDA, "PUDA CQE", wqe, 32);</div><div class='del'>-	i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	status = dev-&gt;cqp_ops-&gt;poll_for_cqp_op_done(dev-&gt;cqp,</div><div class='del'>-						    I40IW_CQP_OP_CREATE_QP,</div><div class='del'>-						    &amp;compl_info);</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_qp_create - create qp for resource</div><div class='del'>- * @rsrc: resource to use for buffer</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_puda_qp_create(struct i40iw_puda_rsrc *rsrc)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_qp *qp = &amp;rsrc-&gt;qp;</div><div class='del'>-	struct i40iw_qp_uk *ukqp = &amp;qp-&gt;qp_uk;</div><div class='del'>-	enum i40iw_status_code ret = 0;</div><div class='del'>-	u32 sq_size, rq_size, t_size;</div><div class='del'>-	struct i40iw_dma_mem *mem;</div><div class='del'>-</div><div class='del'>-	sq_size = rsrc-&gt;sq_size * I40IW_QP_WQE_MIN_SIZE;</div><div class='del'>-	rq_size = rsrc-&gt;rq_size * I40IW_QP_WQE_MIN_SIZE;</div><div class='del'>-	t_size = (sq_size + rq_size + (I40IW_SHADOW_AREA_SIZE &lt;&lt; 3) +</div><div class='del'>-		  I40IW_QP_CTX_SIZE);</div><div class='del'>-	/* Get page aligned memory */</div><div class='del'>-	ret =</div><div class='del'>-	    i40iw_allocate_dma_mem(rsrc-&gt;dev-&gt;hw, &amp;rsrc-&gt;qpmem, t_size,</div><div class='del'>-				   I40IW_HW_PAGE_SIZE);</div><div class='del'>-	if (ret) {</div><div class='del'>-		i40iw_debug(rsrc-&gt;dev, I40IW_DEBUG_PUDA, "%s: error dma mem\n", __func__);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mem = &amp;rsrc-&gt;qpmem;</div><div class='del'>-	memset(mem-&gt;va, 0, t_size);</div><div class='del'>-	qp-&gt;hw_sq_size = i40iw_get_encoded_wqe_size(rsrc-&gt;sq_size, false);</div><div class='del'>-	qp-&gt;hw_rq_size = i40iw_get_encoded_wqe_size(rsrc-&gt;rq_size, false);</div><div class='del'>-	qp-&gt;pd = &amp;rsrc-&gt;sc_pd;</div><div class='del'>-	qp-&gt;qp_type = I40IW_QP_TYPE_UDA;</div><div class='del'>-	qp-&gt;dev = rsrc-&gt;dev;</div><div class='del'>-	qp-&gt;back_qp = (void *)rsrc;</div><div class='del'>-	qp-&gt;sq_pa = mem-&gt;pa;</div><div class='del'>-	qp-&gt;rq_pa = qp-&gt;sq_pa + sq_size;</div><div class='del'>-	qp-&gt;vsi = rsrc-&gt;vsi;</div><div class='del'>-	ukqp-&gt;sq_base = mem-&gt;va;</div><div class='del'>-	ukqp-&gt;rq_base = &amp;ukqp-&gt;sq_base[rsrc-&gt;sq_size];</div><div class='del'>-	ukqp-&gt;shadow_area = ukqp-&gt;rq_base[rsrc-&gt;rq_size].elem;</div><div class='del'>-	qp-&gt;shadow_area_pa = qp-&gt;rq_pa + rq_size;</div><div class='del'>-	qp-&gt;hw_host_ctx = ukqp-&gt;shadow_area + I40IW_SHADOW_AREA_SIZE;</div><div class='del'>-	qp-&gt;hw_host_ctx_pa =</div><div class='del'>-		qp-&gt;shadow_area_pa + (I40IW_SHADOW_AREA_SIZE &lt;&lt; 3);</div><div class='del'>-	ukqp-&gt;qp_id = rsrc-&gt;qp_id;</div><div class='del'>-	ukqp-&gt;sq_wrtrk_array = rsrc-&gt;sq_wrtrk_array;</div><div class='del'>-	ukqp-&gt;rq_wrid_array = rsrc-&gt;rq_wrid_array;</div><div class='del'>-</div><div class='del'>-	ukqp-&gt;qp_id = rsrc-&gt;qp_id;</div><div class='del'>-	ukqp-&gt;sq_size = rsrc-&gt;sq_size;</div><div class='del'>-	ukqp-&gt;rq_size = rsrc-&gt;rq_size;</div><div class='del'>-</div><div class='del'>-	I40IW_RING_INIT(ukqp-&gt;sq_ring, ukqp-&gt;sq_size);</div><div class='del'>-	I40IW_RING_INIT(ukqp-&gt;initial_ring, ukqp-&gt;sq_size);</div><div class='del'>-	I40IW_RING_INIT(ukqp-&gt;rq_ring, ukqp-&gt;rq_size);</div><div class='del'>-</div><div class='del'>-	if (qp-&gt;pd-&gt;dev-&gt;is_pf)</div><div class='del'>-		ukqp-&gt;wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp-&gt;pd-&gt;dev) +</div><div class='del'>-						    I40E_PFPE_WQEALLOC);</div><div class='del'>-	else</div><div class='del'>-		ukqp-&gt;wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp-&gt;pd-&gt;dev) +</div><div class='del'>-						    I40E_VFPE_WQEALLOC1);</div><div class='del'>-</div><div class='del'>-	qp-&gt;user_pri = 0;</div><div class='del'>-	i40iw_qp_add_qos(qp);</div><div class='del'>-	i40iw_puda_qp_setctx(rsrc);</div><div class='del'>-	if (rsrc-&gt;dev-&gt;ceq_valid)</div><div class='del'>-		ret = i40iw_cqp_qp_create_cmd(rsrc-&gt;dev, qp);</div><div class='del'>-	else</div><div class='del'>-		ret = i40iw_puda_qp_wqe(rsrc-&gt;dev, qp);</div><div class='del'>-	if (ret) {</div><div class='del'>-		i40iw_qp_rem_qos(qp);</div><div class='del'>-		i40iw_free_dma_mem(rsrc-&gt;dev-&gt;hw, &amp;rsrc-&gt;qpmem);</div><div class='del'>-	}</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_cq_wqe - setup wqe for cq create</div><div class='del'>- * @rsrc: resource for cq</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_puda_cq_wqe(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 header;</div><div class='del'>-	struct i40iw_ccq_cqe_info compl_info;</div><div class='del'>-	enum i40iw_status_code status = 0;</div><div class='del'>-</div><div class='del'>-	cqp = dev-&gt;cqp;</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, 0);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 0, cq-&gt;cq_uk.cq_size);</div><div class='del'>-	set_64bit_val(wqe, 8, RS_64_1(cq, 1));</div><div class='del'>-	set_64bit_val(wqe, 16,</div><div class='del'>-		      LS_64(cq-&gt;shadow_read_threshold,</div><div class='del'>-			    I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));</div><div class='del'>-	set_64bit_val(wqe, 32, cq-&gt;cq_pa);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 40, cq-&gt;shadow_area_pa);</div><div class='del'>-</div><div class='del'>-	header = cq-&gt;cq_uk.cq_id |</div><div class='del'>-	    LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-	    LS_64(1, I40IW_CQPSQ_CQ_CHKOVERFLOW) |</div><div class='del'>-	    LS_64(1, I40IW_CQPSQ_CQ_ENCEQEMASK) |</div><div class='del'>-	    LS_64(1, I40IW_CQPSQ_CQ_CEQIDVALID) |</div><div class='del'>-	    LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-	i40iw_insert_wqe_hdr(wqe, header);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(dev, I40IW_DEBUG_PUDA, "PUDA CQE",</div><div class='del'>-			wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	i40iw_sc_cqp_post_sq(dev-&gt;cqp);</div><div class='del'>-	status = dev-&gt;cqp_ops-&gt;poll_for_cqp_op_done(dev-&gt;cqp,</div><div class='del'>-						 I40IW_CQP_OP_CREATE_CQ,</div><div class='del'>-						 &amp;compl_info);</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_cq_create - create cq for resource</div><div class='del'>- * @rsrc: resource for which cq to create</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_puda_cq_create(struct i40iw_puda_rsrc *rsrc)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_dev *dev = rsrc-&gt;dev;</div><div class='del'>-	struct i40iw_sc_cq *cq = &amp;rsrc-&gt;cq;</div><div class='del'>-	enum i40iw_status_code ret = 0;</div><div class='del'>-	u32 tsize, cqsize;</div><div class='del'>-	struct i40iw_dma_mem *mem;</div><div class='del'>-	struct i40iw_cq_init_info info;</div><div class='del'>-	struct i40iw_cq_uk_init_info *init_info = &amp;info.cq_uk_init_info;</div><div class='del'>-</div><div class='del'>-	cq-&gt;vsi = rsrc-&gt;vsi;</div><div class='del'>-	cqsize = rsrc-&gt;cq_size * (sizeof(struct i40iw_cqe));</div><div class='del'>-	tsize = cqsize + sizeof(struct i40iw_cq_shadow_area);</div><div class='del'>-	ret = i40iw_allocate_dma_mem(dev-&gt;hw, &amp;rsrc-&gt;cqmem, tsize,</div><div class='del'>-				     I40IW_CQ0_ALIGNMENT);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	mem = &amp;rsrc-&gt;cqmem;</div><div class='del'>-	memset(&amp;info, 0, sizeof(info));</div><div class='del'>-	info.dev = dev;</div><div class='del'>-	info.type = (rsrc-&gt;type == I40IW_PUDA_RSRC_TYPE_ILQ) ?</div><div class='del'>-			 I40IW_CQ_TYPE_ILQ : I40IW_CQ_TYPE_IEQ;</div><div class='del'>-	info.shadow_read_threshold = rsrc-&gt;cq_size &gt;&gt; 2;</div><div class='del'>-	info.ceq_id_valid = true;</div><div class='del'>-	info.cq_base_pa = mem-&gt;pa;</div><div class='del'>-	info.shadow_area_pa = mem-&gt;pa + cqsize;</div><div class='del'>-	init_info-&gt;cq_base = mem-&gt;va;</div><div class='del'>-	init_info-&gt;shadow_area = (u64 *)((u8 *)mem-&gt;va + cqsize);</div><div class='del'>-	init_info-&gt;cq_size = rsrc-&gt;cq_size;</div><div class='del'>-	init_info-&gt;cq_id = rsrc-&gt;cq_id;</div><div class='del'>-	info.ceqe_mask = true;</div><div class='del'>-	info.ceq_id_valid = true;</div><div class='del'>-	ret = dev-&gt;iw_priv_cq_ops-&gt;cq_init(cq, &amp;info);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto error;</div><div class='del'>-	if (rsrc-&gt;dev-&gt;ceq_valid)</div><div class='del'>-		ret = i40iw_cqp_cq_create_cmd(dev, cq);</div><div class='del'>-	else</div><div class='del'>-		ret = i40iw_puda_cq_wqe(dev, cq);</div><div class='del'>-error:</div><div class='del'>-	if (ret)</div><div class='del'>-		i40iw_free_dma_mem(dev-&gt;hw, &amp;rsrc-&gt;cqmem);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_free_qp - free qp for resource</div><div class='del'>- * @rsrc: resource for which qp to free</div><div class='del'>- */</div><div class='del'>-static void i40iw_puda_free_qp(struct i40iw_puda_rsrc *rsrc)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code ret;</div><div class='del'>-	struct i40iw_ccq_cqe_info compl_info;</div><div class='del'>-	struct i40iw_sc_dev *dev = rsrc-&gt;dev;</div><div class='del'>-</div><div class='del'>-	if (rsrc-&gt;dev-&gt;ceq_valid) {</div><div class='del'>-		i40iw_cqp_qp_destroy_cmd(dev, &amp;rsrc-&gt;qp);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = dev-&gt;iw_priv_qp_ops-&gt;qp_destroy(&amp;rsrc-&gt;qp,</div><div class='del'>-			0, false, true, true);</div><div class='del'>-	if (ret)</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_PUDA,</div><div class='del'>-			    "%s error puda qp destroy wqe\n",</div><div class='del'>-			    __func__);</div><div class='del'>-</div><div class='del'>-	if (!ret) {</div><div class='del'>-		ret = dev-&gt;cqp_ops-&gt;poll_for_cqp_op_done(dev-&gt;cqp,</div><div class='del'>-				I40IW_CQP_OP_DESTROY_QP,</div><div class='del'>-				&amp;compl_info);</div><div class='del'>-		if (ret)</div><div class='del'>-			i40iw_debug(dev, I40IW_DEBUG_PUDA,</div><div class='del'>-				    "%s error puda qp destroy failed\n",</div><div class='del'>-				    __func__);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_free_cq - free cq for resource</div><div class='del'>- * @rsrc: resource for which cq to free</div><div class='del'>- */</div><div class='del'>-static void i40iw_puda_free_cq(struct i40iw_puda_rsrc *rsrc)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code ret;</div><div class='del'>-	struct i40iw_ccq_cqe_info compl_info;</div><div class='del'>-	struct i40iw_sc_dev *dev = rsrc-&gt;dev;</div><div class='del'>-</div><div class='del'>-	if (rsrc-&gt;dev-&gt;ceq_valid) {</div><div class='del'>-		i40iw_cqp_cq_destroy_cmd(dev, &amp;rsrc-&gt;cq);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-	ret = dev-&gt;iw_priv_cq_ops-&gt;cq_destroy(&amp;rsrc-&gt;cq, 0, true);</div><div class='del'>-</div><div class='del'>-	if (ret)</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_PUDA,</div><div class='del'>-			    "%s error ieq cq destroy\n",</div><div class='del'>-			    __func__);</div><div class='del'>-</div><div class='del'>-	if (!ret) {</div><div class='del'>-		ret = dev-&gt;cqp_ops-&gt;poll_for_cqp_op_done(dev-&gt;cqp,</div><div class='del'>-				I40IW_CQP_OP_DESTROY_CQ,</div><div class='del'>-				&amp;compl_info);</div><div class='del'>-		if (ret)</div><div class='del'>-			i40iw_debug(dev, I40IW_DEBUG_PUDA,</div><div class='del'>-				    "%s error ieq qp destroy done\n",</div><div class='del'>-				    __func__);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_dele_resources - delete all resources during close</div><div class='del'>- * @dev: iwarp device</div><div class='del'>- * @type: type of resource to dele</div><div class='del'>- * @reset: true if reset chip</div><div class='del'>- */</div><div class='del'>-void i40iw_puda_dele_resources(struct i40iw_sc_vsi *vsi,</div><div class='del'>-			       enum puda_resource_type type,</div><div class='del'>-			       bool reset)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_dev *dev = vsi-&gt;dev;</div><div class='del'>-	struct i40iw_puda_rsrc *rsrc;</div><div class='del'>-	struct i40iw_puda_buf *buf = NULL;</div><div class='del'>-	struct i40iw_puda_buf *nextbuf = NULL;</div><div class='del'>-	struct i40iw_virt_mem *vmem;</div><div class='del'>-</div><div class='del'>-	switch (type) {</div><div class='del'>-	case I40IW_PUDA_RSRC_TYPE_ILQ:</div><div class='del'>-		rsrc = vsi-&gt;ilq;</div><div class='del'>-		vmem = &amp;vsi-&gt;ilq_mem;</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_PUDA_RSRC_TYPE_IEQ:</div><div class='del'>-		rsrc = vsi-&gt;ieq;</div><div class='del'>-		vmem = &amp;vsi-&gt;ieq_mem;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s: error resource type = 0x%x\n",</div><div class='del'>-			    __func__, type);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	switch (rsrc-&gt;completion) {</div><div class='del'>-	case PUDA_HASH_CRC_COMPLETE:</div><div class='del'>-		i40iw_free_hash_desc(rsrc-&gt;hash_desc);</div><div class='del'>-		/* fall through */</div><div class='del'>-	case PUDA_QP_CREATED:</div><div class='del'>-		if (!reset)</div><div class='del'>-			i40iw_puda_free_qp(rsrc);</div><div class='del'>-</div><div class='del'>-		i40iw_free_dma_mem(dev-&gt;hw, &amp;rsrc-&gt;qpmem);</div><div class='del'>-		/* fallthrough */</div><div class='del'>-	case PUDA_CQ_CREATED:</div><div class='del'>-		if (!reset)</div><div class='del'>-			i40iw_puda_free_cq(rsrc);</div><div class='del'>-</div><div class='del'>-		i40iw_free_dma_mem(dev-&gt;hw, &amp;rsrc-&gt;cqmem);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		i40iw_debug(rsrc-&gt;dev, I40IW_DEBUG_PUDA, "%s error no resources\n", __func__);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	/* Free all allocated puda buffers for both tx and rx */</div><div class='del'>-	buf = rsrc-&gt;alloclist;</div><div class='del'>-	while (buf) {</div><div class='del'>-		nextbuf = buf-&gt;next;</div><div class='del'>-		i40iw_puda_dele_buf(dev, buf);</div><div class='del'>-		buf = nextbuf;</div><div class='del'>-		rsrc-&gt;alloc_buf_count--;</div><div class='del'>-	}</div><div class='del'>-	i40iw_free_virt_mem(dev-&gt;hw, vmem);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_allocbufs - allocate buffers for resource</div><div class='del'>- * @rsrc: resource for buffer allocation</div><div class='del'>- * @count: number of buffers to create</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_puda_allocbufs(struct i40iw_puda_rsrc *rsrc,</div><div class='del'>-						   u32 count)</div><div class='del'>-{</div><div class='del'>-	u32 i;</div><div class='del'>-	struct i40iw_puda_buf *buf;</div><div class='del'>-	struct i40iw_puda_buf *nextbuf;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; count; i++) {</div><div class='del'>-		buf = i40iw_puda_alloc_buf(rsrc-&gt;dev, rsrc-&gt;buf_size);</div><div class='del'>-		if (!buf) {</div><div class='del'>-			rsrc-&gt;stats_buf_alloc_fail++;</div><div class='del'>-			return I40IW_ERR_NO_MEMORY;</div><div class='del'>-		}</div><div class='del'>-		i40iw_puda_ret_bufpool(rsrc, buf);</div><div class='del'>-		rsrc-&gt;alloc_buf_count++;</div><div class='del'>-		if (!rsrc-&gt;alloclist) {</div><div class='del'>-			rsrc-&gt;alloclist = buf;</div><div class='del'>-		} else {</div><div class='del'>-			nextbuf = rsrc-&gt;alloclist;</div><div class='del'>-			rsrc-&gt;alloclist = buf;</div><div class='del'>-			buf-&gt;next = nextbuf;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	rsrc-&gt;avail_buf_count = rsrc-&gt;alloc_buf_count;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_create_rsrc - create resouce (ilq or ieq)</div><div class='del'>- * @dev: iwarp device</div><div class='del'>- * @info: resource information</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_puda_create_rsrc(struct i40iw_sc_vsi *vsi,</div><div class='del'>-					      struct i40iw_puda_rsrc_info *info)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_dev *dev = vsi-&gt;dev;</div><div class='del'>-	enum i40iw_status_code ret = 0;</div><div class='del'>-	struct i40iw_puda_rsrc *rsrc;</div><div class='del'>-	u32 pudasize;</div><div class='del'>-	u32 sqwridsize, rqwridsize;</div><div class='del'>-	struct i40iw_virt_mem *vmem;</div><div class='del'>-</div><div class='del'>-	info-&gt;count = 1;</div><div class='del'>-	pudasize = sizeof(struct i40iw_puda_rsrc);</div><div class='del'>-	sqwridsize = info-&gt;sq_size * sizeof(struct i40iw_sq_uk_wr_trk_info);</div><div class='del'>-	rqwridsize = info-&gt;rq_size * 8;</div><div class='del'>-	switch (info-&gt;type) {</div><div class='del'>-	case I40IW_PUDA_RSRC_TYPE_ILQ:</div><div class='del'>-		vmem = &amp;vsi-&gt;ilq_mem;</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_PUDA_RSRC_TYPE_IEQ:</div><div class='del'>-		vmem = &amp;vsi-&gt;ieq_mem;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return I40IW_NOT_SUPPORTED;</div><div class='del'>-	}</div><div class='del'>-	ret =</div><div class='del'>-	    i40iw_allocate_virt_mem(dev-&gt;hw, vmem,</div><div class='del'>-				    pudasize + sqwridsize + rqwridsize);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-	rsrc = (struct i40iw_puda_rsrc *)vmem-&gt;va;</div><div class='del'>-	spin_lock_init(&amp;rsrc-&gt;bufpool_lock);</div><div class='del'>-	if (info-&gt;type == I40IW_PUDA_RSRC_TYPE_ILQ) {</div><div class='del'>-		vsi-&gt;ilq = (struct i40iw_puda_rsrc *)vmem-&gt;va;</div><div class='del'>-		vsi-&gt;ilq_count = info-&gt;count;</div><div class='del'>-		rsrc-&gt;receive = info-&gt;receive;</div><div class='del'>-		rsrc-&gt;xmit_complete = info-&gt;xmit_complete;</div><div class='del'>-	} else {</div><div class='del'>-		vmem = &amp;vsi-&gt;ieq_mem;</div><div class='del'>-		vsi-&gt;ieq_count = info-&gt;count;</div><div class='del'>-		vsi-&gt;ieq = (struct i40iw_puda_rsrc *)vmem-&gt;va;</div><div class='del'>-		rsrc-&gt;receive = i40iw_ieq_receive;</div><div class='del'>-		rsrc-&gt;xmit_complete = i40iw_ieq_tx_compl;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	rsrc-&gt;type = info-&gt;type;</div><div class='del'>-	rsrc-&gt;sq_wrtrk_array = (struct i40iw_sq_uk_wr_trk_info *)((u8 *)vmem-&gt;va + pudasize);</div><div class='del'>-	rsrc-&gt;rq_wrid_array = (u64 *)((u8 *)vmem-&gt;va + pudasize + sqwridsize);</div><div class='del'>-	/* Initialize all ieq lists */</div><div class='del'>-	INIT_LIST_HEAD(&amp;rsrc-&gt;bufpool);</div><div class='del'>-	INIT_LIST_HEAD(&amp;rsrc-&gt;txpend);</div><div class='del'>-</div><div class='del'>-	rsrc-&gt;tx_wqe_avail_cnt = info-&gt;sq_size - 1;</div><div class='del'>-	dev-&gt;iw_pd_ops-&gt;pd_init(dev, &amp;rsrc-&gt;sc_pd, info-&gt;pd_id, -1);</div><div class='del'>-	rsrc-&gt;qp_id = info-&gt;qp_id;</div><div class='del'>-	rsrc-&gt;cq_id = info-&gt;cq_id;</div><div class='del'>-	rsrc-&gt;sq_size = info-&gt;sq_size;</div><div class='del'>-	rsrc-&gt;rq_size = info-&gt;rq_size;</div><div class='del'>-	rsrc-&gt;cq_size = info-&gt;rq_size + info-&gt;sq_size;</div><div class='del'>-	rsrc-&gt;buf_size = info-&gt;buf_size;</div><div class='del'>-	rsrc-&gt;dev = dev;</div><div class='del'>-	rsrc-&gt;vsi = vsi;</div><div class='del'>-</div><div class='del'>-	ret = i40iw_puda_cq_create(rsrc);</div><div class='del'>-	if (!ret) {</div><div class='del'>-		rsrc-&gt;completion = PUDA_CQ_CREATED;</div><div class='del'>-		ret = i40iw_puda_qp_create(rsrc);</div><div class='del'>-	}</div><div class='del'>-	if (ret) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_PUDA, "[%s] error qp_create\n",</div><div class='del'>-			    __func__);</div><div class='del'>-		goto error;</div><div class='del'>-	}</div><div class='del'>-	rsrc-&gt;completion = PUDA_QP_CREATED;</div><div class='del'>-</div><div class='del'>-	ret = i40iw_puda_allocbufs(rsrc, info-&gt;tx_buf_cnt + info-&gt;rq_size);</div><div class='del'>-	if (ret) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_PUDA, "[%s] error alloc_buf\n",</div><div class='del'>-			    __func__);</div><div class='del'>-		goto error;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	rsrc-&gt;rxq_invalid_cnt = info-&gt;rq_size;</div><div class='del'>-	ret = i40iw_puda_replenish_rq(rsrc, true);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto error;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;type == I40IW_PUDA_RSRC_TYPE_IEQ) {</div><div class='del'>-		if (!i40iw_init_hash_desc(&amp;rsrc-&gt;hash_desc)) {</div><div class='del'>-			rsrc-&gt;check_crc = true;</div><div class='del'>-			rsrc-&gt;completion = PUDA_HASH_CRC_COMPLETE;</div><div class='del'>-			ret = 0;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	dev-&gt;ccq_ops-&gt;ccq_arm(&amp;rsrc-&gt;cq);</div><div class='del'>-	return ret;</div><div class='del'>- error:</div><div class='del'>-	i40iw_puda_dele_resources(vsi, info-&gt;type, false);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ilq_putback_rcvbuf - ilq buffer to put back on rq</div><div class='del'>- * @qp: ilq's qp resource</div><div class='del'>- * @wqe_idx:  wqe index of completed rcvbuf</div><div class='del'>- */</div><div class='del'>-static void i40iw_ilq_putback_rcvbuf(struct i40iw_sc_qp *qp, u32 wqe_idx)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 offset24;</div><div class='del'>-</div><div class='del'>-	wqe = qp-&gt;qp_uk.rq_base[wqe_idx].elem;</div><div class='del'>-	get_64bit_val(wqe, 24, &amp;offset24);</div><div class='del'>-	offset24 = (offset24) ? 0 : LS_64(1, I40IWQPSQ_VALID);</div><div class='del'>-	set_64bit_val(wqe, 24, offset24);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_get_fpdu - given length return fpdu length</div><div class='del'>- * @length: length if fpdu</div><div class='del'>- */</div><div class='del'>-static u16 i40iw_ieq_get_fpdu_length(u16 length)</div><div class='del'>-{</div><div class='del'>-	u16 fpdu_len;</div><div class='del'>-</div><div class='del'>-	fpdu_len = length + I40IW_IEQ_MPA_FRAMING;</div><div class='del'>-	fpdu_len = (fpdu_len + 3) &amp; 0xfffffffc;</div><div class='del'>-	return fpdu_len;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_copy_to_txbuf - copydata from rcv buf to tx buf</div><div class='del'>- * @buf: rcv buffer with partial</div><div class='del'>- * @txbuf: tx buffer for sendign back</div><div class='del'>- * @buf_offset: rcv buffer offset to copy from</div><div class='del'>- * @txbuf_offset: at offset in tx buf to copy</div><div class='del'>- * @length: length of data to copy</div><div class='del'>- */</div><div class='del'>-static void i40iw_ieq_copy_to_txbuf(struct i40iw_puda_buf *buf,</div><div class='del'>-				    struct i40iw_puda_buf *txbuf,</div><div class='del'>-				    u16 buf_offset, u32 txbuf_offset,</div><div class='del'>-				    u32 length)</div><div class='del'>-{</div><div class='del'>-	void *mem1 = (u8 *)buf-&gt;mem.va + buf_offset;</div><div class='del'>-	void *mem2 = (u8 *)txbuf-&gt;mem.va + txbuf_offset;</div><div class='del'>-</div><div class='del'>-	memcpy(mem2, mem1, length);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_setup_tx_buf - setup tx buffer for partial handling</div><div class='del'>- * @buf: reeive buffer with partial</div><div class='del'>- * @txbuf: buffer to prepare</div><div class='del'>- */</div><div class='del'>-static void i40iw_ieq_setup_tx_buf(struct i40iw_puda_buf *buf,</div><div class='del'>-				   struct i40iw_puda_buf *txbuf)</div><div class='del'>-{</div><div class='del'>-	txbuf-&gt;maclen = buf-&gt;maclen;</div><div class='del'>-	txbuf-&gt;tcphlen = buf-&gt;tcphlen;</div><div class='del'>-	txbuf-&gt;ipv4 = buf-&gt;ipv4;</div><div class='del'>-	txbuf-&gt;hdrlen = buf-&gt;hdrlen;</div><div class='del'>-	i40iw_ieq_copy_to_txbuf(buf, txbuf, 0, 0, buf-&gt;hdrlen);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_check_first_buf - check if rcv buffer's seq is in range</div><div class='del'>- * @buf: receive exception buffer</div><div class='del'>- * @fps: first partial sequence number</div><div class='del'>- */</div><div class='del'>-static void i40iw_ieq_check_first_buf(struct i40iw_puda_buf *buf, u32 fps)</div><div class='del'>-{</div><div class='del'>-	u32 offset;</div><div class='del'>-</div><div class='del'>-	if (buf-&gt;seqnum &lt; fps) {</div><div class='del'>-		offset = fps - buf-&gt;seqnum;</div><div class='del'>-		if (offset &gt; buf-&gt;datalen)</div><div class='del'>-			return;</div><div class='del'>-		buf-&gt;data += offset;</div><div class='del'>-		buf-&gt;datalen -= (u16)offset;</div><div class='del'>-		buf-&gt;seqnum = fps;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_compl_pfpdu - write txbuf with full fpdu</div><div class='del'>- * @ieq: ieq resource</div><div class='del'>- * @rxlist: ieq's received buffer list</div><div class='del'>- * @pbufl: temporary list for buffers for fpddu</div><div class='del'>- * @txbuf: tx buffer for fpdu</div><div class='del'>- * @fpdu_len: total length of fpdu</div><div class='del'>- */</div><div class='del'>-static void  i40iw_ieq_compl_pfpdu(struct i40iw_puda_rsrc *ieq,</div><div class='del'>-				   struct list_head *rxlist,</div><div class='del'>-				   struct list_head *pbufl,</div><div class='del'>-				   struct i40iw_puda_buf *txbuf,</div><div class='del'>-				   u16 fpdu_len)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_buf *buf;</div><div class='del'>-	u32 nextseqnum;</div><div class='del'>-	u16 txoffset, bufoffset;</div><div class='del'>-</div><div class='del'>-	buf = i40iw_puda_get_listbuf(pbufl);</div><div class='del'>-	if (!buf)</div><div class='del'>-		return;</div><div class='del'>-	nextseqnum = buf-&gt;seqnum + fpdu_len;</div><div class='del'>-	txbuf-&gt;totallen = buf-&gt;hdrlen + fpdu_len;</div><div class='del'>-	txbuf-&gt;data = (u8 *)txbuf-&gt;mem.va + buf-&gt;hdrlen;</div><div class='del'>-	i40iw_ieq_setup_tx_buf(buf, txbuf);</div><div class='del'>-</div><div class='del'>-	txoffset = buf-&gt;hdrlen;</div><div class='del'>-	bufoffset = (u16)(buf-&gt;data - (u8 *)buf-&gt;mem.va);</div><div class='del'>-</div><div class='del'>-	do {</div><div class='del'>-		if (buf-&gt;datalen &gt;= fpdu_len) {</div><div class='del'>-			/* copied full fpdu */</div><div class='del'>-			i40iw_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset, fpdu_len);</div><div class='del'>-			buf-&gt;datalen -= fpdu_len;</div><div class='del'>-			buf-&gt;data += fpdu_len;</div><div class='del'>-			buf-&gt;seqnum = nextseqnum;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		/* copy partial fpdu */</div><div class='del'>-		i40iw_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset, buf-&gt;datalen);</div><div class='del'>-		txoffset += buf-&gt;datalen;</div><div class='del'>-		fpdu_len -= buf-&gt;datalen;</div><div class='del'>-		i40iw_puda_ret_bufpool(ieq, buf);</div><div class='del'>-		buf = i40iw_puda_get_listbuf(pbufl);</div><div class='del'>-		if (!buf)</div><div class='del'>-			return;</div><div class='del'>-		bufoffset = (u16)(buf-&gt;data - (u8 *)buf-&gt;mem.va);</div><div class='del'>-	} while (1);</div><div class='del'>-</div><div class='del'>-	/* last buffer on the list*/</div><div class='del'>-	if (buf-&gt;datalen)</div><div class='del'>-		list_add(&amp;buf-&gt;list, rxlist);</div><div class='del'>-	else</div><div class='del'>-		i40iw_puda_ret_bufpool(ieq, buf);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_create_pbufl - create buffer list for single fpdu</div><div class='del'>- * @rxlist: resource list for receive ieq buffes</div><div class='del'>- * @pbufl: temp. list for buffers for fpddu</div><div class='del'>- * @buf: first receive buffer</div><div class='del'>- * @fpdu_len: total length of fpdu</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_ieq_create_pbufl(</div><div class='del'>-						     struct i40iw_pfpdu *pfpdu,</div><div class='del'>-						     struct list_head *rxlist,</div><div class='del'>-						     struct list_head *pbufl,</div><div class='del'>-						     struct i40iw_puda_buf *buf,</div><div class='del'>-						     u16 fpdu_len)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status = 0;</div><div class='del'>-	struct i40iw_puda_buf *nextbuf;</div><div class='del'>-	u32	nextseqnum;</div><div class='del'>-	u16 plen = fpdu_len - buf-&gt;datalen;</div><div class='del'>-	bool done = false;</div><div class='del'>-</div><div class='del'>-	nextseqnum = buf-&gt;seqnum + buf-&gt;datalen;</div><div class='del'>-	do {</div><div class='del'>-		nextbuf = i40iw_puda_get_listbuf(rxlist);</div><div class='del'>-		if (!nextbuf) {</div><div class='del'>-			status = I40IW_ERR_list_empty;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		list_add_tail(&amp;nextbuf-&gt;list, pbufl);</div><div class='del'>-		if (nextbuf-&gt;seqnum != nextseqnum) {</div><div class='del'>-			pfpdu-&gt;bad_seq_num++;</div><div class='del'>-			status = I40IW_ERR_SEQ_NUM;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		if (nextbuf-&gt;datalen &gt;= plen) {</div><div class='del'>-			done = true;</div><div class='del'>-		} else {</div><div class='del'>-			plen -= nextbuf-&gt;datalen;</div><div class='del'>-			nextseqnum = nextbuf-&gt;seqnum + nextbuf-&gt;datalen;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-	} while (!done);</div><div class='del'>-</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_handle_partial - process partial fpdu buffer</div><div class='del'>- * @ieq: ieq resource</div><div class='del'>- * @pfpdu: partial management per user qp</div><div class='del'>- * @buf: receive buffer</div><div class='del'>- * @fpdu_len: fpdu len in the buffer</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_ieq_handle_partial(struct i40iw_puda_rsrc *ieq,</div><div class='del'>-						       struct i40iw_pfpdu *pfpdu,</div><div class='del'>-						       struct i40iw_puda_buf *buf,</div><div class='del'>-						       u16 fpdu_len)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status = 0;</div><div class='del'>-	u8 *crcptr;</div><div class='del'>-	u32 mpacrc;</div><div class='del'>-	u32 seqnum = buf-&gt;seqnum;</div><div class='del'>-	struct list_head pbufl;	/* partial buffer list */</div><div class='del'>-	struct i40iw_puda_buf *txbuf = NULL;</div><div class='del'>-	struct list_head *rxlist = &amp;pfpdu-&gt;rxlist;</div><div class='del'>-</div><div class='del'>-	INIT_LIST_HEAD(&amp;pbufl);</div><div class='del'>-	list_add(&amp;buf-&gt;list, &amp;pbufl);</div><div class='del'>-</div><div class='del'>-	status = i40iw_ieq_create_pbufl(pfpdu, rxlist, &amp;pbufl, buf, fpdu_len);</div><div class='del'>-	if (status)</div><div class='del'>-		goto error;</div><div class='del'>-</div><div class='del'>-	txbuf = i40iw_puda_get_bufpool(ieq);</div><div class='del'>-	if (!txbuf) {</div><div class='del'>-		pfpdu-&gt;no_tx_bufs++;</div><div class='del'>-		status = I40IW_ERR_NO_TXBUFS;</div><div class='del'>-		goto error;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i40iw_ieq_compl_pfpdu(ieq, rxlist, &amp;pbufl, txbuf, fpdu_len);</div><div class='del'>-	i40iw_ieq_update_tcpip_info(txbuf, fpdu_len, seqnum);</div><div class='del'>-	crcptr = txbuf-&gt;data + fpdu_len - 4;</div><div class='del'>-	mpacrc = *(u32 *)crcptr;</div><div class='del'>-	if (ieq-&gt;check_crc) {</div><div class='del'>-		status = i40iw_ieq_check_mpacrc(ieq-&gt;hash_desc, txbuf-&gt;data,</div><div class='del'>-						(fpdu_len - 4), mpacrc);</div><div class='del'>-		if (status) {</div><div class='del'>-			i40iw_debug(ieq-&gt;dev, I40IW_DEBUG_IEQ,</div><div class='del'>-				    "%s: error bad crc\n", __func__);</div><div class='del'>-			goto error;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(ieq-&gt;dev, I40IW_DEBUG_IEQ, "IEQ TX BUFFER",</div><div class='del'>-			txbuf-&gt;mem.va, txbuf-&gt;totallen);</div><div class='del'>-	i40iw_puda_send_buf(ieq, txbuf);</div><div class='del'>-	pfpdu-&gt;rcv_nxt = seqnum + fpdu_len;</div><div class='del'>-	return status;</div><div class='del'>- error:</div><div class='del'>-	while (!list_empty(&amp;pbufl)) {</div><div class='del'>-		buf = (struct i40iw_puda_buf *)(pbufl.prev);</div><div class='del'>-		list_del(&amp;buf-&gt;list);</div><div class='del'>-		list_add(&amp;buf-&gt;list, rxlist);</div><div class='del'>-	}</div><div class='del'>-	if (txbuf)</div><div class='del'>-		i40iw_puda_ret_bufpool(ieq, txbuf);</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_process_buf - process buffer rcvd for ieq</div><div class='del'>- * @ieq: ieq resource</div><div class='del'>- * @pfpdu: partial management per user qp</div><div class='del'>- * @buf: receive buffer</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_ieq_process_buf(struct i40iw_puda_rsrc *ieq,</div><div class='del'>-						    struct i40iw_pfpdu *pfpdu,</div><div class='del'>-						    struct i40iw_puda_buf *buf)</div><div class='del'>-{</div><div class='del'>-	u16 fpdu_len = 0;</div><div class='del'>-	u16 datalen = buf-&gt;datalen;</div><div class='del'>-	u8 *datap = buf-&gt;data;</div><div class='del'>-	u8 *crcptr;</div><div class='del'>-	u16 ioffset = 0;</div><div class='del'>-	u32 mpacrc;</div><div class='del'>-	u32 seqnum = buf-&gt;seqnum;</div><div class='del'>-	u16 length = 0;</div><div class='del'>-	u16 full = 0;</div><div class='del'>-	bool partial = false;</div><div class='del'>-	struct i40iw_puda_buf *txbuf;</div><div class='del'>-	struct list_head *rxlist = &amp;pfpdu-&gt;rxlist;</div><div class='del'>-	enum i40iw_status_code ret = 0;</div><div class='del'>-	enum i40iw_status_code status = 0;</div><div class='del'>-</div><div class='del'>-	ioffset = (u16)(buf-&gt;data - (u8 *)buf-&gt;mem.va);</div><div class='del'>-	while (datalen) {</div><div class='del'>-		fpdu_len = i40iw_ieq_get_fpdu_length(ntohs(*(__be16 *)datap));</div><div class='del'>-		if (fpdu_len &gt; pfpdu-&gt;max_fpdu_data) {</div><div class='del'>-			i40iw_debug(ieq-&gt;dev, I40IW_DEBUG_IEQ,</div><div class='del'>-				    "%s: error bad fpdu_len\n", __func__);</div><div class='del'>-			status = I40IW_ERR_MPA_CRC;</div><div class='del'>-			list_add(&amp;buf-&gt;list, rxlist);</div><div class='del'>-			return status;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (datalen &lt; fpdu_len) {</div><div class='del'>-			partial = true;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		crcptr = datap + fpdu_len - 4;</div><div class='del'>-		mpacrc = *(u32 *)crcptr;</div><div class='del'>-		if (ieq-&gt;check_crc)</div><div class='del'>-			ret = i40iw_ieq_check_mpacrc(ieq-&gt;hash_desc,</div><div class='del'>-						     datap, fpdu_len - 4, mpacrc);</div><div class='del'>-		if (ret) {</div><div class='del'>-			status = I40IW_ERR_MPA_CRC;</div><div class='del'>-			list_add(&amp;buf-&gt;list, rxlist);</div><div class='del'>-			return status;</div><div class='del'>-		}</div><div class='del'>-		full++;</div><div class='del'>-		pfpdu-&gt;fpdu_processed++;</div><div class='del'>-		datap += fpdu_len;</div><div class='del'>-		length += fpdu_len;</div><div class='del'>-		datalen -= fpdu_len;</div><div class='del'>-	}</div><div class='del'>-	if (full) {</div><div class='del'>-		/* copy full pdu's in the txbuf and send them out */</div><div class='del'>-		txbuf = i40iw_puda_get_bufpool(ieq);</div><div class='del'>-		if (!txbuf) {</div><div class='del'>-			pfpdu-&gt;no_tx_bufs++;</div><div class='del'>-			status = I40IW_ERR_NO_TXBUFS;</div><div class='del'>-			list_add(&amp;buf-&gt;list, rxlist);</div><div class='del'>-			return status;</div><div class='del'>-		}</div><div class='del'>-		/* modify txbuf's buffer header */</div><div class='del'>-		i40iw_ieq_setup_tx_buf(buf, txbuf);</div><div class='del'>-		/* copy full fpdu's to new buffer */</div><div class='del'>-		i40iw_ieq_copy_to_txbuf(buf, txbuf, ioffset, buf-&gt;hdrlen,</div><div class='del'>-					length);</div><div class='del'>-		txbuf-&gt;totallen = buf-&gt;hdrlen + length;</div><div class='del'>-</div><div class='del'>-		i40iw_ieq_update_tcpip_info(txbuf, length, buf-&gt;seqnum);</div><div class='del'>-		i40iw_puda_send_buf(ieq, txbuf);</div><div class='del'>-</div><div class='del'>-		if (!datalen) {</div><div class='del'>-			pfpdu-&gt;rcv_nxt = buf-&gt;seqnum + length;</div><div class='del'>-			i40iw_puda_ret_bufpool(ieq, buf);</div><div class='del'>-			return status;</div><div class='del'>-		}</div><div class='del'>-		buf-&gt;data = datap;</div><div class='del'>-		buf-&gt;seqnum = seqnum + length;</div><div class='del'>-		buf-&gt;datalen = datalen;</div><div class='del'>-		pfpdu-&gt;rcv_nxt = buf-&gt;seqnum;</div><div class='del'>-	}</div><div class='del'>-	if (partial)</div><div class='del'>-		status = i40iw_ieq_handle_partial(ieq, pfpdu, buf, fpdu_len);</div><div class='del'>-</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_process_fpdus - process fpdu's buffers on its list</div><div class='del'>- * @qp: qp for which partial fpdus</div><div class='del'>- * @ieq: ieq resource</div><div class='del'>- */</div><div class='del'>-static void i40iw_ieq_process_fpdus(struct i40iw_sc_qp *qp,</div><div class='del'>-				    struct i40iw_puda_rsrc *ieq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_pfpdu *pfpdu = &amp;qp-&gt;pfpdu;</div><div class='del'>-	struct list_head *rxlist = &amp;pfpdu-&gt;rxlist;</div><div class='del'>-	struct i40iw_puda_buf *buf;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-</div><div class='del'>-	do {</div><div class='del'>-		if (list_empty(rxlist))</div><div class='del'>-			break;</div><div class='del'>-		buf = i40iw_puda_get_listbuf(rxlist);</div><div class='del'>-		if (!buf) {</div><div class='del'>-			i40iw_debug(ieq-&gt;dev, I40IW_DEBUG_IEQ,</div><div class='del'>-				    "%s: error no buf\n", __func__);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		if (buf-&gt;seqnum != pfpdu-&gt;rcv_nxt) {</div><div class='del'>-			/* This could be out of order or missing packet */</div><div class='del'>-			pfpdu-&gt;out_of_order++;</div><div class='del'>-			list_add(&amp;buf-&gt;list, rxlist);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		/* keep processing buffers from the head of the list */</div><div class='del'>-		status = i40iw_ieq_process_buf(ieq, pfpdu, buf);</div><div class='del'>-		if (status == I40IW_ERR_MPA_CRC) {</div><div class='del'>-			pfpdu-&gt;mpa_crc_err = true;</div><div class='del'>-			while (!list_empty(rxlist)) {</div><div class='del'>-				buf = i40iw_puda_get_listbuf(rxlist);</div><div class='del'>-				i40iw_puda_ret_bufpool(ieq, buf);</div><div class='del'>-				pfpdu-&gt;crc_err++;</div><div class='del'>-			}</div><div class='del'>-			/* create CQP for AE */</div><div class='del'>-			i40iw_ieq_mpa_crc_ae(ieq-&gt;dev, qp);</div><div class='del'>-		}</div><div class='del'>-	} while (!status);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_handle_exception - handle qp's exception</div><div class='del'>- * @ieq: ieq resource</div><div class='del'>- * @qp: qp receiving excpetion</div><div class='del'>- * @buf: receive buffer</div><div class='del'>- */</div><div class='del'>-static void i40iw_ieq_handle_exception(struct i40iw_puda_rsrc *ieq,</div><div class='del'>-				       struct i40iw_sc_qp *qp,</div><div class='del'>-				       struct i40iw_puda_buf *buf)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_buf *tmpbuf = NULL;</div><div class='del'>-	struct i40iw_pfpdu *pfpdu = &amp;qp-&gt;pfpdu;</div><div class='del'>-	u32 *hw_host_ctx = (u32 *)qp-&gt;hw_host_ctx;</div><div class='del'>-	u32 rcv_wnd = hw_host_ctx[23];</div><div class='del'>-	/* first partial seq # in q2 */</div><div class='del'>-	u32 fps = *(u32 *)(qp-&gt;q2_buf + Q2_FPSN_OFFSET);</div><div class='del'>-	struct list_head *rxlist = &amp;pfpdu-&gt;rxlist;</div><div class='del'>-	struct list_head *plist;</div><div class='del'>-</div><div class='del'>-	pfpdu-&gt;total_ieq_bufs++;</div><div class='del'>-</div><div class='del'>-	if (pfpdu-&gt;mpa_crc_err) {</div><div class='del'>-		pfpdu-&gt;crc_err++;</div><div class='del'>-		goto error;</div><div class='del'>-	}</div><div class='del'>-	if (pfpdu-&gt;mode &amp;&amp; (fps != pfpdu-&gt;fps)) {</div><div class='del'>-		/* clean up qp as it is new partial sequence */</div><div class='del'>-		i40iw_ieq_cleanup_qp(ieq, qp);</div><div class='del'>-		i40iw_debug(ieq-&gt;dev, I40IW_DEBUG_IEQ,</div><div class='del'>-			    "%s: restarting new partial\n", __func__);</div><div class='del'>-		pfpdu-&gt;mode = false;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!pfpdu-&gt;mode) {</div><div class='del'>-		i40iw_debug_buf(ieq-&gt;dev, I40IW_DEBUG_IEQ, "Q2 BUFFER", (u64 *)qp-&gt;q2_buf, 128);</div><div class='del'>-		/* First_Partial_Sequence_Number check */</div><div class='del'>-		pfpdu-&gt;rcv_nxt = fps;</div><div class='del'>-		pfpdu-&gt;fps = fps;</div><div class='del'>-		pfpdu-&gt;mode = true;</div><div class='del'>-		pfpdu-&gt;max_fpdu_data = (buf-&gt;ipv4) ? (ieq-&gt;vsi-&gt;mtu - I40IW_MTU_TO_MSS_IPV4) :</div><div class='del'>-				       (ieq-&gt;vsi-&gt;mtu - I40IW_MTU_TO_MSS_IPV6);</div><div class='del'>-		pfpdu-&gt;pmode_count++;</div><div class='del'>-		INIT_LIST_HEAD(rxlist);</div><div class='del'>-		i40iw_ieq_check_first_buf(buf, fps);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!(rcv_wnd &gt;= (buf-&gt;seqnum - pfpdu-&gt;rcv_nxt))) {</div><div class='del'>-		pfpdu-&gt;bad_seq_num++;</div><div class='del'>-		goto error;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!list_empty(rxlist)) {</div><div class='del'>-		tmpbuf = (struct i40iw_puda_buf *)rxlist-&gt;next;</div><div class='del'>-		while ((struct list_head *)tmpbuf != rxlist) {</div><div class='del'>-			if ((int)(buf-&gt;seqnum - tmpbuf-&gt;seqnum) &lt; 0)</div><div class='del'>-				break;</div><div class='del'>-			plist = &amp;tmpbuf-&gt;list;</div><div class='del'>-			tmpbuf = (struct i40iw_puda_buf *)plist-&gt;next;</div><div class='del'>-		}</div><div class='del'>-		/* Insert buf before tmpbuf */</div><div class='del'>-		list_add_tail(&amp;buf-&gt;list, &amp;tmpbuf-&gt;list);</div><div class='del'>-	} else {</div><div class='del'>-		list_add_tail(&amp;buf-&gt;list, rxlist);</div><div class='del'>-	}</div><div class='del'>-	i40iw_ieq_process_fpdus(qp, ieq);</div><div class='del'>-	return;</div><div class='del'>- error:</div><div class='del'>-	i40iw_puda_ret_bufpool(ieq, buf);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_receive - received exception buffer</div><div class='del'>- * @dev: iwarp device</div><div class='del'>- * @buf: exception buffer received</div><div class='del'>- */</div><div class='del'>-static void i40iw_ieq_receive(struct i40iw_sc_vsi *vsi,</div><div class='del'>-			      struct i40iw_puda_buf *buf)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_rsrc *ieq = vsi-&gt;ieq;</div><div class='del'>-	struct i40iw_sc_qp *qp = NULL;</div><div class='del'>-	u32 wqe_idx = ieq-&gt;compl_rxwqe_idx;</div><div class='del'>-</div><div class='del'>-	qp = i40iw_ieq_get_qp(vsi-&gt;dev, buf);</div><div class='del'>-	if (!qp) {</div><div class='del'>-		ieq-&gt;stats_bad_qp_id++;</div><div class='del'>-		i40iw_puda_ret_bufpool(ieq, buf);</div><div class='del'>-	} else {</div><div class='del'>-		i40iw_ieq_handle_exception(ieq, qp, buf);</div><div class='del'>-	}</div><div class='del'>-	/*</div><div class='del'>-	 * ieq-&gt;rx_wqe_idx is used by i40iw_puda_replenish_rq()</div><div class='del'>-	 * on which wqe_idx to start replenish rq</div><div class='del'>-	 */</div><div class='del'>-	if (!ieq-&gt;rxq_invalid_cnt)</div><div class='del'>-		ieq-&gt;rx_wqe_idx = wqe_idx;</div><div class='del'>-	ieq-&gt;rxq_invalid_cnt++;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_tx_compl - put back after sending completed exception buffer</div><div class='del'>- * @vsi: pointer to the vsi structure</div><div class='del'>- * @sqwrid: pointer to puda buffer</div><div class='del'>- */</div><div class='del'>-static void i40iw_ieq_tx_compl(struct i40iw_sc_vsi *vsi, void *sqwrid)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_rsrc *ieq = vsi-&gt;ieq;</div><div class='del'>-	struct i40iw_puda_buf *buf = (struct i40iw_puda_buf *)sqwrid;</div><div class='del'>-</div><div class='del'>-	i40iw_puda_ret_bufpool(ieq, buf);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_cleanup_qp - qp is being destroyed</div><div class='del'>- * @ieq: ieq resource</div><div class='del'>- * @qp: all pending fpdu buffers</div><div class='del'>- */</div><div class='del'>-void i40iw_ieq_cleanup_qp(struct i40iw_puda_rsrc *ieq, struct i40iw_sc_qp *qp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_puda_buf *buf;</div><div class='del'>-	struct i40iw_pfpdu *pfpdu = &amp;qp-&gt;pfpdu;</div><div class='del'>-	struct list_head *rxlist = &amp;pfpdu-&gt;rxlist;</div><div class='del'>-</div><div class='del'>-	if (!pfpdu-&gt;mode)</div><div class='del'>-		return;</div><div class='del'>-	while (!list_empty(rxlist)) {</div><div class='del'>-		buf = i40iw_puda_get_listbuf(rxlist);</div><div class='del'>-		i40iw_puda_ret_bufpool(ieq, buf);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_puda.h b/drivers/infiniband/hw/i40iw/i40iw_puda.h<br/>deleted file mode 100644<br/>index 53a7d58c84b5..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_puda.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_puda.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,188 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#ifndef I40IW_PUDA_H</div><div class='del'>-#define I40IW_PUDA_H</div><div class='del'>-</div><div class='del'>-#define I40IW_IEQ_MPA_FRAMING 6</div><div class='del'>-</div><div class='del'>-struct i40iw_sc_dev;</div><div class='del'>-struct i40iw_sc_qp;</div><div class='del'>-struct i40iw_sc_cq;</div><div class='del'>-</div><div class='del'>-enum puda_resource_type {</div><div class='del'>-	I40IW_PUDA_RSRC_TYPE_ILQ = 1,</div><div class='del'>-	I40IW_PUDA_RSRC_TYPE_IEQ</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum puda_rsrc_complete {</div><div class='del'>-	PUDA_CQ_CREATED = 1,</div><div class='del'>-	PUDA_QP_CREATED,</div><div class='del'>-	PUDA_TX_COMPLETE,</div><div class='del'>-	PUDA_RX_COMPLETE,</div><div class='del'>-	PUDA_HASH_CRC_COMPLETE</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_puda_completion_info {</div><div class='del'>-	struct i40iw_qp_uk *qp;</div><div class='del'>-	u8 q_type;</div><div class='del'>-	u8 vlan_valid;</div><div class='del'>-	u8 l3proto;</div><div class='del'>-	u8 l4proto;</div><div class='del'>-	u16 payload_len;</div><div class='del'>-	u32 compl_error;	/* No_err=0, else major and minor err code */</div><div class='del'>-	u32 qp_id;</div><div class='del'>-	u32 wqe_idx;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_puda_send_info {</div><div class='del'>-	u64 paddr;		/* Physical address */</div><div class='del'>-	u32 len;</div><div class='del'>-	u8 tcplen;</div><div class='del'>-	u8 maclen;</div><div class='del'>-	bool ipv4;</div><div class='del'>-	bool doloopback;</div><div class='del'>-	void *scratch;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_puda_buf {</div><div class='del'>-	struct list_head list;	/* MUST be first entry */</div><div class='del'>-	struct i40iw_dma_mem mem;	/* DMA memory for the buffer */</div><div class='del'>-	struct i40iw_puda_buf *next;	/* for alloclist in rsrc struct */</div><div class='del'>-	struct i40iw_virt_mem buf_mem;	/* Buffer memory for this buffer */</div><div class='del'>-	void *scratch;</div><div class='del'>-	u8 *iph;</div><div class='del'>-	u8 *tcph;</div><div class='del'>-	u8 *data;</div><div class='del'>-	u16 datalen;</div><div class='del'>-	u16 vlan_id;</div><div class='del'>-	u8 tcphlen;		/* tcp length in bytes */</div><div class='del'>-	u8 maclen;		/* mac length in bytes */</div><div class='del'>-	u32 totallen;		/* machlen+iphlen+tcphlen+datalen */</div><div class='del'>-	atomic_t refcount;</div><div class='del'>-	u8 hdrlen;</div><div class='del'>-	bool ipv4;</div><div class='del'>-	u32 seqnum;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_puda_rsrc_info {</div><div class='del'>-	enum puda_resource_type type;	/* ILQ or IEQ */</div><div class='del'>-	u32 count;</div><div class='del'>-	u16 pd_id;</div><div class='del'>-	u32 cq_id;</div><div class='del'>-	u32 qp_id;</div><div class='del'>-	u32 sq_size;</div><div class='del'>-	u32 rq_size;</div><div class='del'>-	u16 buf_size;</div><div class='del'>-	u16 mss;</div><div class='del'>-	u32 tx_buf_cnt;		/* total bufs allocated will be rq_size + tx_buf_cnt */</div><div class='del'>-	void (*receive)(struct i40iw_sc_vsi *, struct i40iw_puda_buf *);</div><div class='del'>-	void (*xmit_complete)(struct i40iw_sc_vsi *, void *);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_puda_rsrc {</div><div class='del'>-	struct i40iw_sc_cq cq;</div><div class='del'>-	struct i40iw_sc_qp qp;</div><div class='del'>-	struct i40iw_sc_pd sc_pd;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	struct i40iw_sc_vsi *vsi;</div><div class='del'>-	struct i40iw_dma_mem cqmem;</div><div class='del'>-	struct i40iw_dma_mem qpmem;</div><div class='del'>-	struct i40iw_virt_mem ilq_mem;</div><div class='del'>-	enum puda_rsrc_complete completion;</div><div class='del'>-	enum puda_resource_type type;</div><div class='del'>-	u16 buf_size;		/*buffer must be max datalen + tcpip hdr + mac */</div><div class='del'>-	u16 mss;</div><div class='del'>-	u32 cq_id;</div><div class='del'>-	u32 qp_id;</div><div class='del'>-	u32 sq_size;</div><div class='del'>-	u32 rq_size;</div><div class='del'>-	u32 cq_size;</div><div class='del'>-	struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array;</div><div class='del'>-	u64 *rq_wrid_array;</div><div class='del'>-	u32 compl_rxwqe_idx;</div><div class='del'>-	u32 rx_wqe_idx;</div><div class='del'>-	u32 rxq_invalid_cnt;</div><div class='del'>-	u32 tx_wqe_avail_cnt;</div><div class='del'>-	bool check_crc;</div><div class='del'>-	struct shash_desc *hash_desc;</div><div class='del'>-	struct list_head txpend;</div><div class='del'>-	struct list_head bufpool;	/* free buffers pool list for recv and xmit */</div><div class='del'>-	u32 alloc_buf_count;</div><div class='del'>-	u32 avail_buf_count;		/* snapshot of currently available buffers */</div><div class='del'>-	spinlock_t bufpool_lock;</div><div class='del'>-	struct i40iw_puda_buf *alloclist;</div><div class='del'>-	void (*receive)(struct i40iw_sc_vsi *, struct i40iw_puda_buf *);</div><div class='del'>-	void (*xmit_complete)(struct i40iw_sc_vsi *, void *);</div><div class='del'>-	/* puda stats */</div><div class='del'>-	u64 stats_buf_alloc_fail;</div><div class='del'>-	u64 stats_pkt_rcvd;</div><div class='del'>-	u64 stats_pkt_sent;</div><div class='del'>-	u64 stats_rcvd_pkt_err;</div><div class='del'>-	u64 stats_sent_pkt_q;</div><div class='del'>-	u64 stats_bad_qp_id;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_puda_buf *i40iw_puda_get_bufpool(struct i40iw_puda_rsrc *rsrc);</div><div class='del'>-void i40iw_puda_ret_bufpool(struct i40iw_puda_rsrc *rsrc,</div><div class='del'>-			    struct i40iw_puda_buf *buf);</div><div class='del'>-void i40iw_puda_send_buf(struct i40iw_puda_rsrc *rsrc,</div><div class='del'>-			 struct i40iw_puda_buf *buf);</div><div class='del'>-enum i40iw_status_code i40iw_puda_send(struct i40iw_sc_qp *qp,</div><div class='del'>-				       struct i40iw_puda_send_info *info);</div><div class='del'>-enum i40iw_status_code i40iw_puda_create_rsrc(struct i40iw_sc_vsi *vsi,</div><div class='del'>-					      struct i40iw_puda_rsrc_info *info);</div><div class='del'>-void i40iw_puda_dele_resources(struct i40iw_sc_vsi *vsi,</div><div class='del'>-			       enum puda_resource_type type,</div><div class='del'>-			       bool reset);</div><div class='del'>-enum i40iw_status_code i40iw_puda_poll_completion(struct i40iw_sc_dev *dev,</div><div class='del'>-						  struct i40iw_sc_cq *cq, u32 *compl_err);</div><div class='del'>-</div><div class='del'>-struct i40iw_sc_qp *i40iw_ieq_get_qp(struct i40iw_sc_dev *dev,</div><div class='del'>-				     struct i40iw_puda_buf *buf);</div><div class='del'>-enum i40iw_status_code i40iw_puda_get_tcpip_info(struct i40iw_puda_completion_info *info,</div><div class='del'>-						 struct i40iw_puda_buf *buf);</div><div class='del'>-enum i40iw_status_code i40iw_ieq_check_mpacrc(struct shash_desc *desc,</div><div class='del'>-					      void *addr, u32 length, u32 value);</div><div class='del'>-enum i40iw_status_code i40iw_init_hash_desc(struct shash_desc **desc);</div><div class='del'>-void i40iw_ieq_mpa_crc_ae(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp);</div><div class='del'>-void i40iw_free_hash_desc(struct shash_desc *desc);</div><div class='del'>-void i40iw_ieq_update_tcpip_info(struct i40iw_puda_buf *buf, u16 length,</div><div class='del'>-				 u32 seqnum);</div><div class='del'>-enum i40iw_status_code i40iw_cqp_qp_create_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp);</div><div class='del'>-enum i40iw_status_code i40iw_cqp_cq_create_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq);</div><div class='del'>-void i40iw_cqp_qp_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp);</div><div class='del'>-void i40iw_cqp_cq_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq);</div><div class='del'>-void i40iw_ieq_cleanup_qp(struct i40iw_puda_rsrc *ieq, struct i40iw_sc_qp *qp);</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_register.h b/drivers/infiniband/hw/i40iw/i40iw_register.h<br/>deleted file mode 100644<br/>index 57768184e251..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_register.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_register.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,1030 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#ifndef I40IW_REGISTER_H</div><div class='del'>-#define I40IW_REGISTER_H</div><div class='del'>-</div><div class='del'>-#define I40E_GLGEN_STAT               0x000B612C /* Reset: POR */</div><div class='del'>-</div><div class='del'>-#define I40E_PFHMC_PDINV               0x000C0300 /* Reset: PFR */</div><div class='del'>-#define I40E_PFHMC_PDINV_PMSDIDX_SHIFT 0</div><div class='del'>-#define I40E_PFHMC_PDINV_PMSDIDX_MASK  (0xFFF &lt;&lt;  I40E_PFHMC_PDINV_PMSDIDX_SHIFT)</div><div class='del'>-#define I40E_PFHMC_PDINV_PMPDIDX_SHIFT 16</div><div class='del'>-#define I40E_PFHMC_PDINV_PMPDIDX_MASK  (0x1FF &lt;&lt;  I40E_PFHMC_PDINV_PMPDIDX_SHIFT)</div><div class='del'>-#define I40E_PFHMC_SDCMD_PMSDWR_SHIFT  31</div><div class='del'>-#define I40E_PFHMC_SDCMD_PMSDWR_MASK   (0x1 &lt;&lt;  I40E_PFHMC_SDCMD_PMSDWR_SHIFT)</div><div class='del'>-#define I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT   0</div><div class='del'>-#define I40E_PFHMC_SDDATALOW_PMSDVALID_MASK    (0x1 &lt;&lt;  I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)</div><div class='del'>-#define I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT    1</div><div class='del'>-#define I40E_PFHMC_SDDATALOW_PMSDTYPE_MASK     (0x1 &lt;&lt;  I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT)</div><div class='del'>-#define I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT 2</div><div class='del'>-#define I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_MASK  (0x3FF &lt;&lt;  I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40E_PFINT_DYN_CTLN(_INTPF) (0x00034800 + ((_INTPF) * 4)) /* _i=0...511 */	/* Reset: PFR */</div><div class='del'>-#define I40E_PFINT_DYN_CTLN_INTENA_SHIFT          0</div><div class='del'>-#define I40E_PFINT_DYN_CTLN_INTENA_MASK           (0x1 &lt;&lt;  I40E_PFINT_DYN_CTLN_INTENA_SHIFT)</div><div class='del'>-#define I40E_PFINT_DYN_CTLN_CLEARPBA_SHIFT        1</div><div class='del'>-#define I40E_PFINT_DYN_CTLN_CLEARPBA_MASK         (0x1 &lt;&lt;  I40E_PFINT_DYN_CTLN_CLEARPBA_SHIFT)</div><div class='del'>-#define I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT        3</div><div class='del'>-#define I40E_PFINT_DYN_CTLN_ITR_INDX_MASK         (0x3 &lt;&lt;  I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40E_VFINT_DYN_CTLN1(_INTVF)               (0x00003800 + ((_INTVF) * 4)) /* _i=0...15 */ /* Reset: VFR */</div><div class='del'>-#define I40E_GLHMC_VFPDINV(_i)               (0x000C8300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */</div><div class='del'>-</div><div class='del'>-#define I40E_PFHMC_PDINV_PMSDPARTSEL_SHIFT 15</div><div class='del'>-#define I40E_PFHMC_PDINV_PMSDPARTSEL_MASK  (0x1 &lt;&lt;  I40E_PFHMC_PDINV_PMSDPARTSEL_SHIFT)</div><div class='del'>-#define I40E_GLPCI_LBARCTRL                    0x000BE484 /* Reset: POR */</div><div class='del'>-#define I40E_GLPCI_LBARCTRL_PE_DB_SIZE_SHIFT    4</div><div class='del'>-#define I40E_GLPCI_LBARCTRL_PE_DB_SIZE_MASK     (0x3 &lt;&lt;  I40E_GLPCI_LBARCTRL_PE_DB_SIZE_SHIFT)</div><div class='del'>-#define I40E_GLPCI_DREVID			0x0009C480 /* Reset: PCIR */</div><div class='del'>-#define I40E_GLPCI_DREVID_DEFAULT_REVID_SHIFT 0</div><div class='del'>-#define I40E_GLPCI_DREVID_DEFAULT_REVID_MASK 0xFF</div><div class='del'>-</div><div class='del'>-#define I40E_PFPE_AEQALLOC               0x00131180 /* Reset: PFR */</div><div class='del'>-#define I40E_PFPE_AEQALLOC_AECOUNT_SHIFT 0</div><div class='del'>-#define I40E_PFPE_AEQALLOC_AECOUNT_MASK  (0xFFFFFFFF &lt;&lt;  I40E_PFPE_AEQALLOC_AECOUNT_SHIFT)</div><div class='del'>-#define I40E_PFPE_CCQPHIGH                  0x00008200 /* Reset: PFR */</div><div class='del'>-#define I40E_PFPE_CCQPHIGH_PECCQPHIGH_SHIFT 0</div><div class='del'>-#define I40E_PFPE_CCQPHIGH_PECCQPHIGH_MASK  (0xFFFFFFFF &lt;&lt;  I40E_PFPE_CCQPHIGH_PECCQPHIGH_SHIFT)</div><div class='del'>-#define I40E_PFPE_CCQPLOW                 0x00008180 /* Reset: PFR */</div><div class='del'>-#define I40E_PFPE_CCQPLOW_PECCQPLOW_SHIFT 0</div><div class='del'>-#define I40E_PFPE_CCQPLOW_PECCQPLOW_MASK  (0xFFFFFFFF &lt;&lt;  I40E_PFPE_CCQPLOW_PECCQPLOW_SHIFT)</div><div class='del'>-#define I40E_PFPE_CCQPSTATUS                   0x00008100 /* Reset: PFR */</div><div class='del'>-#define I40E_PFPE_CCQPSTATUS_CCQP_DONE_SHIFT   0</div><div class='del'>-#define I40E_PFPE_CCQPSTATUS_CCQP_DONE_MASK    (0x1 &lt;&lt;  I40E_PFPE_CCQPSTATUS_CCQP_DONE_SHIFT)</div><div class='del'>-#define I40E_PFPE_CCQPSTATUS_HMC_PROFILE_SHIFT 4</div><div class='del'>-#define I40E_PFPE_CCQPSTATUS_HMC_PROFILE_MASK  (0x7 &lt;&lt;  I40E_PFPE_CCQPSTATUS_HMC_PROFILE_SHIFT)</div><div class='del'>-#define I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT 16</div><div class='del'>-#define I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_MASK  (0x3F &lt;&lt;  I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT)</div><div class='del'>-#define I40E_PFPE_CCQPSTATUS_CCQP_ERR_SHIFT    31</div><div class='del'>-#define I40E_PFPE_CCQPSTATUS_CCQP_ERR_MASK     (0x1 &lt;&lt;  I40E_PFPE_CCQPSTATUS_CCQP_ERR_SHIFT)</div><div class='del'>-#define I40E_PFPE_CQACK              0x00131100 /* Reset: PFR */</div><div class='del'>-#define I40E_PFPE_CQACK_PECQID_SHIFT 0</div><div class='del'>-#define I40E_PFPE_CQACK_PECQID_MASK  (0x1FFFF &lt;&lt;  I40E_PFPE_CQACK_PECQID_SHIFT)</div><div class='del'>-#define I40E_PFPE_CQARM              0x00131080 /* Reset: PFR */</div><div class='del'>-#define I40E_PFPE_CQARM_PECQID_SHIFT 0</div><div class='del'>-#define I40E_PFPE_CQARM_PECQID_MASK  (0x1FFFF &lt;&lt;  I40E_PFPE_CQARM_PECQID_SHIFT)</div><div class='del'>-#define I40E_PFPE_CQPDB              0x00008000 /* Reset: PFR */</div><div class='del'>-#define I40E_PFPE_CQPDB_WQHEAD_SHIFT 0</div><div class='del'>-#define I40E_PFPE_CQPDB_WQHEAD_MASK  (0x7FF &lt;&lt;  I40E_PFPE_CQPDB_WQHEAD_SHIFT)</div><div class='del'>-#define I40E_PFPE_CQPERRCODES                      0x00008880 /* Reset: PFR */</div><div class='del'>-#define I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT 0</div><div class='del'>-#define I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_MASK  (0xFFFF &lt;&lt;  I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT)</div><div class='del'>-#define I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT 16</div><div class='del'>-#define I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_MASK  (0xFFFF &lt;&lt;  I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT)</div><div class='del'>-#define I40E_PFPE_CQPTAIL                  0x00008080 /* Reset: PFR */</div><div class='del'>-#define I40E_PFPE_CQPTAIL_WQTAIL_SHIFT     0</div><div class='del'>-#define I40E_PFPE_CQPTAIL_WQTAIL_MASK      (0x7FF &lt;&lt;  I40E_PFPE_CQPTAIL_WQTAIL_SHIFT)</div><div class='del'>-#define I40E_PFPE_CQPTAIL_CQP_OP_ERR_SHIFT 31</div><div class='del'>-#define I40E_PFPE_CQPTAIL_CQP_OP_ERR_MASK  (0x1 &lt;&lt;  I40E_PFPE_CQPTAIL_CQP_OP_ERR_SHIFT)</div><div class='del'>-#define I40E_PFPE_FLMQ1ALLOCERR                   0x00008980 /* Reset: PFR */</div><div class='del'>-#define I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_SHIFT 0</div><div class='del'>-#define I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_MASK  (0xFFFF &lt;&lt;  I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_SHIFT)</div><div class='del'>-#define I40E_PFPE_FLMXMITALLOCERR                   0x00008900 /* Reset: PFR */</div><div class='del'>-#define I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_SHIFT 0</div><div class='del'>-#define I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_MASK  (0xFFFF &lt;&lt;  I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_SHIFT)</div><div class='del'>-#define I40E_PFPE_IPCONFIG0                        0x00008280 /* Reset: PFR */</div><div class='del'>-#define I40E_PFPE_IPCONFIG0_PEIPID_SHIFT           0</div><div class='del'>-#define I40E_PFPE_IPCONFIG0_PEIPID_MASK            (0xFFFF &lt;&lt;  I40E_PFPE_IPCONFIG0_PEIPID_SHIFT)</div><div class='del'>-#define I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT 16</div><div class='del'>-#define I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_MASK  (0x1 &lt;&lt;  I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT)</div><div class='del'>-#define I40E_PFPE_MRTEIDXMASK                       0x00008600 /* Reset: PFR */</div><div class='del'>-#define I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT 0</div><div class='del'>-#define I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_MASK  (0x1F &lt;&lt;  I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT)</div><div class='del'>-#define I40E_PFPE_RCVUNEXPECTEDERROR                        0x00008680 /* Reset: PFR */</div><div class='del'>-#define I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT 0</div><div class='del'>-#define I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_MASK  (0xFFFFFF &lt;&lt;  I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT)</div><div class='del'>-#define I40E_PFPE_TCPNOWTIMER               0x00008580 /* Reset: PFR */</div><div class='del'>-#define I40E_PFPE_TCPNOWTIMER_TCP_NOW_SHIFT 0</div><div class='del'>-#define I40E_PFPE_TCPNOWTIMER_TCP_NOW_MASK  (0xFFFFFFFF &lt;&lt;  I40E_PFPE_TCPNOWTIMER_TCP_NOW_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40E_PFPE_WQEALLOC                      0x00138C00 /* Reset: PFR */</div><div class='del'>-#define I40E_PFPE_WQEALLOC_PEQPID_SHIFT         0</div><div class='del'>-#define I40E_PFPE_WQEALLOC_PEQPID_MASK          (0x3FFFF &lt;&lt;  I40E_PFPE_WQEALLOC_PEQPID_SHIFT)</div><div class='del'>-#define I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT 20</div><div class='del'>-#define I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_MASK  (0xFFF &lt;&lt;  I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40E_VFPE_AEQALLOC(_VF)          (0x00130C00 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_AEQALLOC_MAX_INDEX     127</div><div class='del'>-#define I40E_VFPE_AEQALLOC_AECOUNT_SHIFT 0</div><div class='del'>-#define I40E_VFPE_AEQALLOC_AECOUNT_MASK  (0xFFFFFFFF &lt;&lt;  I40E_VFPE_AEQALLOC_AECOUNT_SHIFT)</div><div class='del'>-#define I40E_VFPE_CCQPHIGH(_VF)             (0x00001000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_CCQPHIGH_MAX_INDEX        127</div><div class='del'>-#define I40E_VFPE_CCQPHIGH_PECCQPHIGH_SHIFT 0</div><div class='del'>-#define I40E_VFPE_CCQPHIGH_PECCQPHIGH_MASK  (0xFFFFFFFF &lt;&lt;  I40E_VFPE_CCQPHIGH_PECCQPHIGH_SHIFT)</div><div class='del'>-#define I40E_VFPE_CCQPLOW(_VF)            (0x00000C00 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_CCQPLOW_MAX_INDEX       127</div><div class='del'>-#define I40E_VFPE_CCQPLOW_PECCQPLOW_SHIFT 0</div><div class='del'>-#define I40E_VFPE_CCQPLOW_PECCQPLOW_MASK  (0xFFFFFFFF &lt;&lt;  I40E_VFPE_CCQPLOW_PECCQPLOW_SHIFT)</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS(_VF)              (0x00000800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS_MAX_INDEX         127</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS_CCQP_DONE_SHIFT   0</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS_CCQP_DONE_MASK    (0x1 &lt;&lt;  I40E_VFPE_CCQPSTATUS_CCQP_DONE_SHIFT)</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS_HMC_PROFILE_SHIFT 4</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS_HMC_PROFILE_MASK  (0x7 &lt;&lt;  I40E_VFPE_CCQPSTATUS_HMC_PROFILE_SHIFT)</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT 16</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_MASK  (0x3F &lt;&lt;  I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT)</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS_CCQP_ERR_SHIFT    31</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS_CCQP_ERR_MASK     (0x1 &lt;&lt;  I40E_VFPE_CCQPSTATUS_CCQP_ERR_SHIFT)</div><div class='del'>-#define I40E_VFPE_CQACK(_VF)         (0x00130800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_CQACK_MAX_INDEX    127</div><div class='del'>-#define I40E_VFPE_CQACK_PECQID_SHIFT 0</div><div class='del'>-#define I40E_VFPE_CQACK_PECQID_MASK  (0x1FFFF &lt;&lt;  I40E_VFPE_CQACK_PECQID_SHIFT)</div><div class='del'>-#define I40E_VFPE_CQARM(_VF)         (0x00130400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_CQARM_MAX_INDEX    127</div><div class='del'>-#define I40E_VFPE_CQARM_PECQID_SHIFT 0</div><div class='del'>-#define I40E_VFPE_CQARM_PECQID_MASK  (0x1FFFF &lt;&lt;  I40E_VFPE_CQARM_PECQID_SHIFT)</div><div class='del'>-#define I40E_VFPE_CQPDB(_VF)         (0x00000000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_CQPDB_MAX_INDEX    127</div><div class='del'>-#define I40E_VFPE_CQPDB_WQHEAD_SHIFT 0</div><div class='del'>-#define I40E_VFPE_CQPDB_WQHEAD_MASK  (0x7FF &lt;&lt;  I40E_VFPE_CQPDB_WQHEAD_SHIFT)</div><div class='del'>-#define I40E_VFPE_CQPERRCODES(_VF)                 (0x00001800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_CQPERRCODES_MAX_INDEX            127</div><div class='del'>-#define I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT 0</div><div class='del'>-#define I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_MASK  (0xFFFF &lt;&lt;  I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT)</div><div class='del'>-#define I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT 16</div><div class='del'>-#define I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_MASK  (0xFFFF &lt;&lt;  I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT)</div><div class='del'>-#define I40E_VFPE_CQPTAIL(_VF)             (0x00000400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_CQPTAIL_MAX_INDEX        127</div><div class='del'>-#define I40E_VFPE_CQPTAIL_WQTAIL_SHIFT     0</div><div class='del'>-#define I40E_VFPE_CQPTAIL_WQTAIL_MASK      (0x7FF &lt;&lt;  I40E_VFPE_CQPTAIL_WQTAIL_SHIFT)</div><div class='del'>-#define I40E_VFPE_CQPTAIL_CQP_OP_ERR_SHIFT 31</div><div class='del'>-#define I40E_VFPE_CQPTAIL_CQP_OP_ERR_MASK  (0x1 &lt;&lt;  I40E_VFPE_CQPTAIL_CQP_OP_ERR_SHIFT)</div><div class='del'>-#define I40E_VFPE_IPCONFIG0(_VF)                   (0x00001400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_IPCONFIG0_MAX_INDEX              127</div><div class='del'>-#define I40E_VFPE_IPCONFIG0_PEIPID_SHIFT           0</div><div class='del'>-#define I40E_VFPE_IPCONFIG0_PEIPID_MASK            (0xFFFF &lt;&lt;  I40E_VFPE_IPCONFIG0_PEIPID_SHIFT)</div><div class='del'>-#define I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT 16</div><div class='del'>-#define I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_MASK  (0x1 &lt;&lt;  I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT)</div><div class='del'>-#define I40E_VFPE_MRTEIDXMASK(_VF)                  (0x00003000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_MRTEIDXMASK_MAX_INDEX             127</div><div class='del'>-#define I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT 0</div><div class='del'>-#define I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_MASK  (0x1F &lt;&lt;  I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT)</div><div class='del'>-#define I40E_VFPE_RCVUNEXPECTEDERROR(_VF)                   (0x00003400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_RCVUNEXPECTEDERROR_MAX_INDEX              127</div><div class='del'>-#define I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT 0</div><div class='del'>-#define I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_MASK  (0xFFFFFF &lt;&lt;  I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT)</div><div class='del'>-#define I40E_VFPE_TCPNOWTIMER(_VF)          (0x00002C00 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_TCPNOWTIMER_MAX_INDEX     127</div><div class='del'>-#define I40E_VFPE_TCPNOWTIMER_TCP_NOW_SHIFT 0</div><div class='del'>-#define I40E_VFPE_TCPNOWTIMER_TCP_NOW_MASK  (0xFFFFFFFF &lt;&lt;  I40E_VFPE_TCPNOWTIMER_TCP_NOW_SHIFT)</div><div class='del'>-#define I40E_VFPE_WQEALLOC(_VF)                 (0x00138000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_WQEALLOC_MAX_INDEX            127</div><div class='del'>-#define I40E_VFPE_WQEALLOC_PEQPID_SHIFT         0</div><div class='del'>-#define I40E_VFPE_WQEALLOC_PEQPID_MASK          (0x3FFFF &lt;&lt;  I40E_VFPE_WQEALLOC_PEQPID_SHIFT)</div><div class='del'>-#define I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT 20</div><div class='del'>-#define I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_MASK  (0xFFF &lt;&lt;  I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40E_GLPE_CPUSTATUS0                    0x0000D040 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_SHIFT 0</div><div class='del'>-#define I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_SHIFT)</div><div class='del'>-#define I40E_GLPE_CPUSTATUS1                    0x0000D044 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_SHIFT 0</div><div class='del'>-#define I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_SHIFT)</div><div class='del'>-#define I40E_GLPE_CPUSTATUS2                    0x0000D048 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_SHIFT 0</div><div class='del'>-#define I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_SHIFT)</div><div class='del'>-#define I40E_GLPE_CPUTRIG0                   0x0000D060 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPE_CPUTRIG0_PECPUTRIG0_SHIFT  0</div><div class='del'>-#define I40E_GLPE_CPUTRIG0_PECPUTRIG0_MASK   (0xFFFF &lt;&lt;  I40E_GLPE_CPUTRIG0_PECPUTRIG0_SHIFT)</div><div class='del'>-#define I40E_GLPE_CPUTRIG0_TEPREQUEST0_SHIFT 17</div><div class='del'>-#define I40E_GLPE_CPUTRIG0_TEPREQUEST0_MASK  (0x1 &lt;&lt;  I40E_GLPE_CPUTRIG0_TEPREQUEST0_SHIFT)</div><div class='del'>-#define I40E_GLPE_CPUTRIG0_OOPREQUEST0_SHIFT 18</div><div class='del'>-#define I40E_GLPE_CPUTRIG0_OOPREQUEST0_MASK  (0x1 &lt;&lt;  I40E_GLPE_CPUTRIG0_OOPREQUEST0_SHIFT)</div><div class='del'>-#define I40E_GLPE_DUAL40_RUPM                     0x0000DA04 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_SHIFT 0</div><div class='del'>-#define I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_MASK  (0x1 &lt;&lt;  I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_SHIFT)</div><div class='del'>-#define I40E_GLPE_PFAEQEDROPCNT(_i)               (0x00131440 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */</div><div class='del'>-#define I40E_GLPE_PFAEQEDROPCNT_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_SHIFT 0</div><div class='del'>-#define I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_MASK  (0xFFFF &lt;&lt;  I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_SHIFT)</div><div class='del'>-#define I40E_GLPE_PFCEQEDROPCNT(_i)               (0x001313C0 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */</div><div class='del'>-#define I40E_GLPE_PFCEQEDROPCNT_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_SHIFT 0</div><div class='del'>-#define I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_MASK  (0xFFFF &lt;&lt;  I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_SHIFT)</div><div class='del'>-#define I40E_GLPE_PFCQEDROPCNT(_i)              (0x00131340 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */</div><div class='del'>-#define I40E_GLPE_PFCQEDROPCNT_MAX_INDEX        15</div><div class='del'>-#define I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_SHIFT 0</div><div class='del'>-#define I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_MASK  (0xFFFF &lt;&lt;  I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_SHIFT)</div><div class='del'>-#define I40E_GLPE_RUPM_CQPPOOL                0x0000DACC /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_SHIFT 0</div><div class='del'>-#define I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_MASK  (0xFF &lt;&lt;  I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_SHIFT)</div><div class='del'>-#define I40E_GLPE_RUPM_FLRPOOL                0x0000DAC4 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_SHIFT 0</div><div class='del'>-#define I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_MASK  (0xFF &lt;&lt;  I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_SHIFT)</div><div class='del'>-#define I40E_GLPE_RUPM_GCTL                   0x0000DA00 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPE_RUPM_GCTL_ALLOFFTH_SHIFT    0</div><div class='del'>-#define I40E_GLPE_RUPM_GCTL_ALLOFFTH_MASK     (0xFF &lt;&lt;  I40E_GLPE_RUPM_GCTL_ALLOFFTH_SHIFT)</div><div class='del'>-#define I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_SHIFT 26</div><div class='del'>-#define I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_MASK  (0x1 &lt;&lt;  I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_SHIFT)</div><div class='del'>-#define I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_SHIFT 27</div><div class='del'>-#define I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_MASK  (0x1 &lt;&lt;  I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_SHIFT)</div><div class='del'>-#define I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_SHIFT 28</div><div class='del'>-#define I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_MASK  (0x1 &lt;&lt;  I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_SHIFT)</div><div class='del'>-#define I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_SHIFT 29</div><div class='del'>-#define I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_MASK  (0x1 &lt;&lt;  I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_SHIFT)</div><div class='del'>-#define I40E_GLPE_RUPM_GCTL_RUPM_DIS_SHIFT    30</div><div class='del'>-#define I40E_GLPE_RUPM_GCTL_RUPM_DIS_MASK     (0x1 &lt;&lt;  I40E_GLPE_RUPM_GCTL_RUPM_DIS_SHIFT)</div><div class='del'>-#define I40E_GLPE_RUPM_GCTL_SWLB_MODE_SHIFT   31</div><div class='del'>-#define I40E_GLPE_RUPM_GCTL_SWLB_MODE_MASK    (0x1 &lt;&lt;  I40E_GLPE_RUPM_GCTL_SWLB_MODE_SHIFT)</div><div class='del'>-#define I40E_GLPE_RUPM_PTXPOOL                0x0000DAC8 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_SHIFT 0</div><div class='del'>-#define I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_MASK  (0xFF &lt;&lt;  I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_SHIFT)</div><div class='del'>-#define I40E_GLPE_RUPM_PUSHPOOL                 0x0000DAC0 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_SHIFT 0</div><div class='del'>-#define I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_MASK  (0xFF &lt;&lt;  I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_SHIFT)</div><div class='del'>-#define I40E_GLPE_RUPM_TXHOST_EN                 0x0000DA08 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_SHIFT 0</div><div class='del'>-#define I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_MASK  (0x1 &lt;&lt;  I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_SHIFT)</div><div class='del'>-#define I40E_GLPE_VFAEQEDROPCNT(_i)               (0x00132540 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */</div><div class='del'>-#define I40E_GLPE_VFAEQEDROPCNT_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_SHIFT 0</div><div class='del'>-#define I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_MASK  (0xFFFF &lt;&lt;  I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_SHIFT)</div><div class='del'>-#define I40E_GLPE_VFCEQEDROPCNT(_i)               (0x00132440 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */</div><div class='del'>-#define I40E_GLPE_VFCEQEDROPCNT_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_SHIFT 0</div><div class='del'>-#define I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_MASK  (0xFFFF &lt;&lt;  I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_SHIFT)</div><div class='del'>-#define I40E_GLPE_VFCQEDROPCNT(_i)              (0x00132340 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */</div><div class='del'>-#define I40E_GLPE_VFCQEDROPCNT_MAX_INDEX        31</div><div class='del'>-#define I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_SHIFT 0</div><div class='del'>-#define I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_MASK  (0xFFFF &lt;&lt;  I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_SHIFT)</div><div class='del'>-#define I40E_GLPE_VFFLMOBJCTRL(_i)                  (0x0000D400 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPE_VFFLMOBJCTRL_MAX_INDEX            31</div><div class='del'>-#define I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_SHIFT 0</div><div class='del'>-#define I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_MASK  (0x7 &lt;&lt;  I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_SHIFT)</div><div class='del'>-#define I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_SHIFT   8</div><div class='del'>-#define I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_MASK    (0x7 &lt;&lt;  I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_SHIFT)</div><div class='del'>-#define I40E_GLPE_VFFLMQ1ALLOCERR(_i)               (0x0000C700 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPE_VFFLMQ1ALLOCERR_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_SHIFT 0</div><div class='del'>-#define I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_MASK  (0xFFFF &lt;&lt;  I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_SHIFT)</div><div class='del'>-#define I40E_GLPE_VFFLMXMITALLOCERR(_i)               (0x0000C600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPE_VFFLMXMITALLOCERR_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_SHIFT 0</div><div class='del'>-#define I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_MASK  (0xFFFF &lt;&lt;  I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_SHIFT)</div><div class='del'>-#define I40E_GLPE_VFUDACTRL(_i)                    (0x0000C000 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPE_VFUDACTRL_MAX_INDEX              31</div><div class='del'>-#define I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_SHIFT  0</div><div class='del'>-#define I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_MASK   (0x1 &lt;&lt;  I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_SHIFT)</div><div class='del'>-#define I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_SHIFT  1</div><div class='del'>-#define I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_MASK   (0x1 &lt;&lt;  I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_SHIFT)</div><div class='del'>-#define I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_SHIFT  2</div><div class='del'>-#define I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_MASK   (0x1 &lt;&lt;  I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_SHIFT)</div><div class='del'>-#define I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_SHIFT  3</div><div class='del'>-#define I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_MASK   (0x1 &lt;&lt;  I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_SHIFT)</div><div class='del'>-#define I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_SHIFT 4</div><div class='del'>-#define I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_MASK  (0x1 &lt;&lt;  I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_SHIFT)</div><div class='del'>-#define I40E_GLPE_VFUDAUCFBQPN(_i)         (0x0000C100 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPE_VFUDAUCFBQPN_MAX_INDEX   31</div><div class='del'>-#define I40E_GLPE_VFUDAUCFBQPN_QPN_SHIFT   0</div><div class='del'>-#define I40E_GLPE_VFUDAUCFBQPN_QPN_MASK    (0x3FFFF &lt;&lt;  I40E_GLPE_VFUDAUCFBQPN_QPN_SHIFT)</div><div class='del'>-#define I40E_GLPE_VFUDAUCFBQPN_VALID_SHIFT 31</div><div class='del'>-#define I40E_GLPE_VFUDAUCFBQPN_VALID_MASK  (0x1 &lt;&lt;  I40E_GLPE_VFUDAUCFBQPN_VALID_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40E_GLPES_PFIP4RXDISCARD(_i)                (0x00010600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4RXDISCARD_MAX_INDEX          15</div><div class='del'>-#define I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4RXFRAGSHI(_i)                (0x00010804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4RXFRAGSHI_MAX_INDEX          15</div><div class='del'>-#define I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4RXFRAGSLO(_i)                (0x00010800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4RXFRAGSLO_MAX_INDEX          15</div><div class='del'>-#define I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4RXMCOCTSHI(_i)                 (0x00010A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4RXMCOCTSHI_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4RXMCOCTSLO(_i)                 (0x00010A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4RXMCOCTSLO_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4RXMCPKTSHI(_i)                 (0x00010C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4RXMCPKTSHI_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4RXMCPKTSLO(_i)                 (0x00010C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4RXMCPKTSLO_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4RXOCTSHI(_i)               (0x00010204 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4RXOCTSHI_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4RXOCTSLO(_i)               (0x00010200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4RXOCTSLO_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4RXPKTSHI(_i)               (0x00010404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4RXPKTSHI_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4RXPKTSLO(_i)               (0x00010400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4RXPKTSLO_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4RXTRUNC(_i)              (0x00010700 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4RXTRUNC_MAX_INDEX        15</div><div class='del'>-#define I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4TXFRAGSHI(_i)                (0x00011E04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4TXFRAGSHI_MAX_INDEX          15</div><div class='del'>-#define I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4TXFRAGSLO(_i)                (0x00011E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4TXFRAGSLO_MAX_INDEX          15</div><div class='del'>-#define I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4TXMCOCTSHI(_i)                 (0x00012004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4TXMCOCTSHI_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4TXMCOCTSLO(_i)                 (0x00012000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4TXMCOCTSLO_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4TXMCPKTSHI(_i)                 (0x00012204 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4TXMCPKTSHI_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4TXMCPKTSLO(_i)                 (0x00012200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4TXMCPKTSLO_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4TXNOROUTE(_i)                (0x00012E00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4TXNOROUTE_MAX_INDEX          15</div><div class='del'>-#define I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4TXOCTSHI(_i)               (0x00011A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4TXOCTSHI_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4TXOCTSLO(_i)               (0x00011A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4TXOCTSLO_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4TXPKTSHI(_i)               (0x00011C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4TXPKTSHI_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP4TXPKTSLO(_i)               (0x00011C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP4TXPKTSLO_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6RXDISCARD(_i)                (0x00011200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6RXDISCARD_MAX_INDEX          15</div><div class='del'>-#define I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6RXFRAGSHI(_i)                (0x00011404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6RXFRAGSHI_MAX_INDEX          15</div><div class='del'>-#define I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6RXFRAGSLO(_i)                (0x00011400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6RXFRAGSLO_MAX_INDEX          15</div><div class='del'>-#define I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6RXMCOCTSHI(_i)                 (0x00011604 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6RXMCOCTSHI_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6RXMCOCTSLO(_i)                 (0x00011600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6RXMCOCTSLO_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6RXMCPKTSHI(_i)                 (0x00011804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6RXMCPKTSHI_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6RXMCPKTSLO(_i)                 (0x00011800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6RXMCPKTSLO_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6RXOCTSHI(_i)               (0x00010E04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6RXOCTSHI_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6RXOCTSLO(_i)               (0x00010E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6RXOCTSLO_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6RXPKTSHI(_i)               (0x00011004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6RXPKTSHI_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6RXPKTSLO(_i)               (0x00011000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6RXPKTSLO_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6RXTRUNC(_i)              (0x00011300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6RXTRUNC_MAX_INDEX        15</div><div class='del'>-#define I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6TXFRAGSHI(_i)                (0x00012804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6TXFRAGSHI_MAX_INDEX          15</div><div class='del'>-#define I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6TXFRAGSLO(_i)                (0x00012800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6TXFRAGSLO_MAX_INDEX          15</div><div class='del'>-#define I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6TXMCOCTSHI(_i)                 (0x00012A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6TXMCOCTSHI_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6TXMCOCTSLO(_i)                 (0x00012A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6TXMCOCTSLO_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6TXMCPKTSHI(_i)                 (0x00012C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6TXMCPKTSHI_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6TXMCPKTSLO(_i)                 (0x00012C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6TXMCPKTSLO_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6TXNOROUTE(_i)                (0x00012F00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6TXNOROUTE_MAX_INDEX          15</div><div class='del'>-#define I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6TXOCTSHI(_i)               (0x00012404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6TXOCTSHI_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6TXOCTSLO(_i)               (0x00012400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6TXOCTSLO_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6TXPKTSHI(_i)               (0x00012604 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6TXPKTSHI_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFIP6TXPKTSLO(_i)               (0x00012600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFIP6TXPKTSLO_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRDMARXRDSHI(_i)               (0x00013E04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRDMARXRDSHI_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRDMARXRDSLO(_i)               (0x00013E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRDMARXRDSLO_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRDMARXSNDSHI(_i)                (0x00014004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRDMARXSNDSHI_MAX_INDEX          15</div><div class='del'>-#define I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRDMARXSNDSLO(_i)                (0x00014000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRDMARXSNDSLO_MAX_INDEX          15</div><div class='del'>-#define I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRDMARXWRSHI(_i)               (0x00013C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRDMARXWRSHI_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRDMARXWRSLO(_i)               (0x00013C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRDMARXWRSLO_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRDMATXRDSHI(_i)               (0x00014404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRDMATXRDSHI_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRDMATXRDSLO(_i)               (0x00014400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRDMATXRDSLO_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRDMATXSNDSHI(_i)                (0x00014604 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRDMATXSNDSHI_MAX_INDEX          15</div><div class='del'>-#define I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRDMATXSNDSLO(_i)                (0x00014600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRDMATXSNDSLO_MAX_INDEX          15</div><div class='del'>-#define I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRDMATXWRSHI(_i)               (0x00014204 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRDMATXWRSHI_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRDMATXWRSLO(_i)               (0x00014200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRDMATXWRSLO_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRDMAVBNDHI(_i)              (0x00014804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRDMAVBNDHI_MAX_INDEX        15</div><div class='del'>-#define I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRDMAVBNDLO(_i)              (0x00014800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRDMAVBNDLO_MAX_INDEX        15</div><div class='del'>-#define I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRDMAVINVHI(_i)              (0x00014A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRDMAVINVHI_MAX_INDEX        15</div><div class='del'>-#define I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRDMAVINVLO(_i)              (0x00014A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRDMAVINVLO_MAX_INDEX        15</div><div class='del'>-#define I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFRXVLANERR(_i)             (0x00010000 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFRXVLANERR_MAX_INDEX       15</div><div class='del'>-#define I40E_GLPES_PFRXVLANERR_RXVLANERR_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFRXVLANERR_RXVLANERR_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_PFRXVLANERR_RXVLANERR_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFTCPRTXSEG(_i)             (0x00013600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFTCPRTXSEG_MAX_INDEX       15</div><div class='del'>-#define I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFTCPRXOPTERR(_i)               (0x00013200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFTCPRXOPTERR_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFTCPRXPROTOERR(_i)                 (0x00013300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFTCPRXPROTOERR_MAX_INDEX           15</div><div class='del'>-#define I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFTCPRXSEGSHI(_i)               (0x00013004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFTCPRXSEGSHI_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFTCPRXSEGSLO(_i)               (0x00013000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFTCPRXSEGSLO_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFTCPTXSEGHI(_i)              (0x00013404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFTCPTXSEGHI_MAX_INDEX        15</div><div class='del'>-#define I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFTCPTXSEGLO(_i)              (0x00013400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFTCPTXSEGLO_MAX_INDEX        15</div><div class='del'>-#define I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFUDPRXPKTSHI(_i)               (0x00013804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFUDPRXPKTSHI_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFUDPRXPKTSLO(_i)               (0x00013800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFUDPRXPKTSLO_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFUDPTXPKTSHI(_i)               (0x00013A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFUDPTXPKTSHI_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_PFUDPTXPKTSLO(_i)               (0x00013A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_PFUDPTXPKTSLO_MAX_INDEX         15</div><div class='del'>-#define I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_RDMARXMULTFPDUSHI                         0x0001E014 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_RDMARXMULTFPDUSLO                         0x0001E010 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_RDMARXOOODDPHI                      0x0001E01C /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_RDMARXOOODDPLO                      0x0001E018 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_RDMARXOOONOMARK                     0x0001E004 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_SHIFT 0</div><div class='del'>-#define I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_SHIFT)</div><div class='del'>-#define I40E_GLPES_RDMARXUNALIGN                     0x0001E000 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_SHIFT 0</div><div class='del'>-#define I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_SHIFT)</div><div class='del'>-#define I40E_GLPES_TCPRXFOURHOLEHI                       0x0001E044 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_TCPRXFOURHOLELO                       0x0001E040 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_SHIFT)</div><div class='del'>-#define I40E_GLPES_TCPRXONEHOLEHI                      0x0001E02C /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_TCPRXONEHOLELO                      0x0001E028 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_SHIFT)</div><div class='del'>-#define I40E_GLPES_TCPRXPUREACKHI                       0x0001E024 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_TCPRXPUREACKSLO                      0x0001E020 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_TCPRXTHREEHOLEHI                        0x0001E03C /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_TCPRXTHREEHOLELO                        0x0001E038 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_SHIFT)</div><div class='del'>-#define I40E_GLPES_TCPRXTWOHOLEHI                      0x0001E034 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_TCPRXTWOHOLELO                      0x0001E030 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_SHIFT)</div><div class='del'>-#define I40E_GLPES_TCPTXRETRANSFASTHI                          0x0001E04C /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_TCPTXRETRANSFASTLO                          0x0001E048 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_TCPTXTOUTSFASTHI                        0x0001E054 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_TCPTXTOUTSFASTLO                        0x0001E050 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_TCPTXTOUTSHI                    0x0001E05C /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_TCPTXTOUTSLO                    0x0001E058 /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4RXDISCARD(_i)                (0x00018600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4RXDISCARD_MAX_INDEX          31</div><div class='del'>-#define I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4RXFRAGSHI(_i)                (0x00018804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4RXFRAGSHI_MAX_INDEX          31</div><div class='del'>-#define I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4RXFRAGSLO(_i)                (0x00018800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4RXFRAGSLO_MAX_INDEX          31</div><div class='del'>-#define I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4RXMCOCTSHI(_i)                 (0x00018A04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4RXMCOCTSHI_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4RXMCOCTSLO(_i)                 (0x00018A00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4RXMCOCTSLO_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4RXMCPKTSHI(_i)                 (0x00018C04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4RXMCPKTSHI_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4RXMCPKTSLO(_i)                 (0x00018C00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4RXMCPKTSLO_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4RXOCTSHI(_i)               (0x00018204 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4RXOCTSHI_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4RXOCTSLO(_i)               (0x00018200 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4RXOCTSLO_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4RXPKTSHI(_i)               (0x00018404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4RXPKTSHI_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4RXPKTSLO(_i)               (0x00018400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4RXPKTSLO_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4RXTRUNC(_i)              (0x00018700 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4RXTRUNC_MAX_INDEX        31</div><div class='del'>-#define I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4TXFRAGSHI(_i)                (0x00019E04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4TXFRAGSHI_MAX_INDEX          31</div><div class='del'>-#define I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4TXFRAGSLO(_i)                (0x00019E00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4TXFRAGSLO_MAX_INDEX          31</div><div class='del'>-#define I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4TXMCOCTSHI(_i)                 (0x0001A004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4TXMCOCTSHI_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4TXMCOCTSLO(_i)                 (0x0001A000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4TXMCOCTSLO_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4TXMCPKTSHI(_i)                 (0x0001A204 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4TXMCPKTSHI_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4TXMCPKTSLO(_i)                 (0x0001A200 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4TXMCPKTSLO_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4TXNOROUTE(_i)                (0x0001AE00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4TXNOROUTE_MAX_INDEX          31</div><div class='del'>-#define I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4TXOCTSHI(_i)               (0x00019A04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4TXOCTSHI_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4TXOCTSLO(_i)               (0x00019A00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4TXOCTSLO_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4TXPKTSHI(_i)               (0x00019C04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4TXPKTSHI_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP4TXPKTSLO(_i)               (0x00019C00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP4TXPKTSLO_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6RXDISCARD(_i)                (0x00019200 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6RXDISCARD_MAX_INDEX          31</div><div class='del'>-#define I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6RXFRAGSHI(_i)                (0x00019404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6RXFRAGSHI_MAX_INDEX          31</div><div class='del'>-#define I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6RXFRAGSLO(_i)                (0x00019400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6RXFRAGSLO_MAX_INDEX          31</div><div class='del'>-#define I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6RXMCOCTSHI(_i)                 (0x00019604 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6RXMCOCTSHI_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6RXMCOCTSLO(_i)                 (0x00019600 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6RXMCOCTSLO_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6RXMCPKTSHI(_i)                 (0x00019804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6RXMCPKTSHI_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6RXMCPKTSLO(_i)                 (0x00019800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6RXMCPKTSLO_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6RXOCTSHI(_i)               (0x00018E04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6RXOCTSHI_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6RXOCTSLO(_i)               (0x00018E00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6RXOCTSLO_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6RXPKTSHI(_i)               (0x00019004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6RXPKTSHI_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6RXPKTSLO(_i)               (0x00019000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6RXPKTSLO_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6RXTRUNC(_i)              (0x00019300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6RXTRUNC_MAX_INDEX        31</div><div class='del'>-#define I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6TXFRAGSHI(_i)                (0x0001A804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6TXFRAGSHI_MAX_INDEX          31</div><div class='del'>-#define I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6TXFRAGSLO(_i)                (0x0001A800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6TXFRAGSLO_MAX_INDEX          31</div><div class='del'>-#define I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6TXMCOCTSHI(_i)                 (0x0001AA04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6TXMCOCTSHI_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6TXMCOCTSLO(_i)                 (0x0001AA00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6TXMCOCTSLO_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6TXMCPKTSHI(_i)                 (0x0001AC04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6TXMCPKTSHI_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6TXMCPKTSLO(_i)                 (0x0001AC00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6TXMCPKTSLO_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6TXNOROUTE(_i)                (0x0001AF00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6TXNOROUTE_MAX_INDEX          31</div><div class='del'>-#define I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6TXOCTSHI(_i)               (0x0001A404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6TXOCTSHI_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6TXOCTSLO(_i)               (0x0001A400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6TXOCTSLO_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6TXPKTSHI(_i)               (0x0001A604 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6TXPKTSHI_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFIP6TXPKTSLO(_i)               (0x0001A600 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFIP6TXPKTSLO_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRDMARXRDSHI(_i)               (0x0001BE04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRDMARXRDSHI_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRDMARXRDSLO(_i)               (0x0001BE00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRDMARXRDSLO_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRDMARXSNDSHI(_i)                (0x0001C004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRDMARXSNDSHI_MAX_INDEX          31</div><div class='del'>-#define I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRDMARXSNDSLO(_i)                (0x0001C000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRDMARXSNDSLO_MAX_INDEX          31</div><div class='del'>-#define I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRDMARXWRSHI(_i)               (0x0001BC04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRDMARXWRSHI_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRDMARXWRSLO(_i)               (0x0001BC00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRDMARXWRSLO_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRDMATXRDSHI(_i)               (0x0001C404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRDMATXRDSHI_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRDMATXRDSLO(_i)               (0x0001C400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRDMATXRDSLO_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRDMATXSNDSHI(_i)                (0x0001C604 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRDMATXSNDSHI_MAX_INDEX          31</div><div class='del'>-#define I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRDMATXSNDSLO(_i)                (0x0001C600 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRDMATXSNDSLO_MAX_INDEX          31</div><div class='del'>-#define I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRDMATXWRSHI(_i)               (0x0001C204 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRDMATXWRSHI_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRDMATXWRSLO(_i)               (0x0001C200 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRDMATXWRSLO_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRDMAVBNDHI(_i)              (0x0001C804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRDMAVBNDHI_MAX_INDEX        31</div><div class='del'>-#define I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRDMAVBNDLO(_i)              (0x0001C800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRDMAVBNDLO_MAX_INDEX        31</div><div class='del'>-#define I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRDMAVINVHI(_i)              (0x0001CA04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRDMAVINVHI_MAX_INDEX        31</div><div class='del'>-#define I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRDMAVINVLO(_i)              (0x0001CA00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRDMAVINVLO_MAX_INDEX        31</div><div class='del'>-#define I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFRXVLANERR(_i)             (0x00018000 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFRXVLANERR_MAX_INDEX       31</div><div class='del'>-#define I40E_GLPES_VFRXVLANERR_RXVLANERR_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFRXVLANERR_RXVLANERR_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_VFRXVLANERR_RXVLANERR_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFTCPRTXSEG(_i)             (0x0001B600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFTCPRTXSEG_MAX_INDEX       31</div><div class='del'>-#define I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFTCPRXOPTERR(_i)               (0x0001B200 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFTCPRXOPTERR_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFTCPRXPROTOERR(_i)                 (0x0001B300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFTCPRXPROTOERR_MAX_INDEX           31</div><div class='del'>-#define I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_MASK  (0xFFFFFF &lt;&lt;  I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFTCPRXSEGSHI(_i)               (0x0001B004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFTCPRXSEGSHI_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFTCPRXSEGSLO(_i)               (0x0001B000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFTCPRXSEGSLO_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFTCPTXSEGHI(_i)              (0x0001B404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFTCPTXSEGHI_MAX_INDEX        31</div><div class='del'>-#define I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFTCPTXSEGLO(_i)              (0x0001B400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFTCPTXSEGLO_MAX_INDEX        31</div><div class='del'>-#define I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFUDPRXPKTSHI(_i)               (0x0001B804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFUDPRXPKTSHI_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFUDPRXPKTSLO(_i)               (0x0001B800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFUDPRXPKTSLO_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFUDPTXPKTSHI(_i)               (0x0001BA04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFUDPTXPKTSHI_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_MASK  (0xFFFF &lt;&lt;  I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT)</div><div class='del'>-#define I40E_GLPES_VFUDPTXPKTSLO(_i)               (0x0001BA00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */</div><div class='del'>-#define I40E_GLPES_VFUDPTXPKTSLO_MAX_INDEX         31</div><div class='del'>-#define I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT 0</div><div class='del'>-#define I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_MASK  (0xFFFFFFFF &lt;&lt;  I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT)</div><div class='del'>-</div><div class='del'>-#define I40E_VFPE_AEQALLOC1               0x0000A400 /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_AEQALLOC1_AECOUNT_SHIFT 0</div><div class='del'>-#define I40E_VFPE_AEQALLOC1_AECOUNT_MASK  (0xFFFFFFFF &lt;&lt;  I40E_VFPE_AEQALLOC1_AECOUNT_SHIFT)</div><div class='del'>-#define I40E_VFPE_CCQPHIGH1                  0x00009800 /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_CCQPHIGH1_PECCQPHIGH_SHIFT 0</div><div class='del'>-#define I40E_VFPE_CCQPHIGH1_PECCQPHIGH_MASK  (0xFFFFFFFF &lt;&lt;  I40E_VFPE_CCQPHIGH1_PECCQPHIGH_SHIFT)</div><div class='del'>-#define I40E_VFPE_CCQPLOW1                 0x0000AC00 /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_CCQPLOW1_PECCQPLOW_SHIFT 0</div><div class='del'>-#define I40E_VFPE_CCQPLOW1_PECCQPLOW_MASK  (0xFFFFFFFF &lt;&lt;  I40E_VFPE_CCQPLOW1_PECCQPLOW_SHIFT)</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS1                   0x0000B800 /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS1_CCQP_DONE_SHIFT   0</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS1_CCQP_DONE_MASK    (0x1 &lt;&lt;  I40E_VFPE_CCQPSTATUS1_CCQP_DONE_SHIFT)</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_SHIFT 4</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_MASK  (0x7 &lt;&lt;  I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_SHIFT)</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_SHIFT 16</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_MASK  (0x3F &lt;&lt;  I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_SHIFT)</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS1_CCQP_ERR_SHIFT    31</div><div class='del'>-#define I40E_VFPE_CCQPSTATUS1_CCQP_ERR_MASK     (0x1 &lt;&lt;  I40E_VFPE_CCQPSTATUS1_CCQP_ERR_SHIFT)</div><div class='del'>-#define I40E_VFPE_CQACK1              0x0000B000 /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_CQACK1_PECQID_SHIFT 0</div><div class='del'>-#define I40E_VFPE_CQACK1_PECQID_MASK  (0x1FFFF &lt;&lt;  I40E_VFPE_CQACK1_PECQID_SHIFT)</div><div class='del'>-#define I40E_VFPE_CQARM1              0x0000B400 /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_CQARM1_PECQID_SHIFT 0</div><div class='del'>-#define I40E_VFPE_CQARM1_PECQID_MASK  (0x1FFFF &lt;&lt;  I40E_VFPE_CQARM1_PECQID_SHIFT)</div><div class='del'>-#define I40E_VFPE_CQPDB1              0x0000BC00 /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_CQPDB1_WQHEAD_SHIFT 0</div><div class='del'>-#define I40E_VFPE_CQPDB1_WQHEAD_MASK  (0x7FF &lt;&lt;  I40E_VFPE_CQPDB1_WQHEAD_SHIFT)</div><div class='del'>-#define I40E_VFPE_CQPERRCODES1                      0x00009C00 /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_SHIFT 0</div><div class='del'>-#define I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_MASK  (0xFFFF &lt;&lt;  I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_SHIFT)</div><div class='del'>-#define I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_SHIFT 16</div><div class='del'>-#define I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_MASK  (0xFFFF &lt;&lt;  I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_SHIFT)</div><div class='del'>-#define I40E_VFPE_CQPTAIL1                  0x0000A000 /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_CQPTAIL1_WQTAIL_SHIFT     0</div><div class='del'>-#define I40E_VFPE_CQPTAIL1_WQTAIL_MASK      (0x7FF &lt;&lt;  I40E_VFPE_CQPTAIL1_WQTAIL_SHIFT)</div><div class='del'>-#define I40E_VFPE_CQPTAIL1_CQP_OP_ERR_SHIFT 31</div><div class='del'>-#define I40E_VFPE_CQPTAIL1_CQP_OP_ERR_MASK  (0x1 &lt;&lt;  I40E_VFPE_CQPTAIL1_CQP_OP_ERR_SHIFT)</div><div class='del'>-#define I40E_VFPE_IPCONFIG01                        0x00008C00 /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_IPCONFIG01_PEIPID_SHIFT           0</div><div class='del'>-#define I40E_VFPE_IPCONFIG01_PEIPID_MASK            (0xFFFF &lt;&lt;  I40E_VFPE_IPCONFIG01_PEIPID_SHIFT)</div><div class='del'>-#define I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_SHIFT 16</div><div class='del'>-#define I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_MASK  (0x1 &lt;&lt;  I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_SHIFT)</div><div class='del'>-#define I40E_VFPE_MRTEIDXMASK1                       0x00009000 /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_SHIFT 0</div><div class='del'>-#define I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_MASK  (0x1F &lt;&lt;  I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_SHIFT)</div><div class='del'>-#define I40E_VFPE_RCVUNEXPECTEDERROR1                        0x00009400 /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_SHIFT 0</div><div class='del'>-#define I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_MASK  (0xFFFFFF &lt;&lt;  I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_SHIFT)</div><div class='del'>-#define I40E_VFPE_TCPNOWTIMER1               0x0000A800 /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_TCPNOWTIMER1_TCP_NOW_SHIFT 0</div><div class='del'>-#define I40E_VFPE_TCPNOWTIMER1_TCP_NOW_MASK  (0xFFFFFFFF &lt;&lt;  I40E_VFPE_TCPNOWTIMER1_TCP_NOW_SHIFT)</div><div class='del'>-#define I40E_VFPE_WQEALLOC1                      0x0000C000 /* Reset: VFR */</div><div class='del'>-#define I40E_VFPE_WQEALLOC1_PEQPID_SHIFT         0</div><div class='del'>-#define I40E_VFPE_WQEALLOC1_PEQPID_MASK          (0x3FFFF &lt;&lt;  I40E_VFPE_WQEALLOC1_PEQPID_SHIFT)</div><div class='del'>-#define I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_SHIFT 20</div><div class='del'>-#define I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_MASK  (0xFFF &lt;&lt;  I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_SHIFT)</div><div class='del'>-#endif /* I40IW_REGISTER_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_status.h b/drivers/infiniband/hw/i40iw/i40iw_status.h<br/>deleted file mode 100644<br/>index f7013f11d808..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_status.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_status.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,101 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#ifndef I40IW_STATUS_H</div><div class='del'>-#define I40IW_STATUS_H</div><div class='del'>-</div><div class='del'>-/* Error Codes */</div><div class='del'>-enum i40iw_status_code {</div><div class='del'>-	I40IW_SUCCESS = 0,</div><div class='del'>-	I40IW_ERR_NVM = -1,</div><div class='del'>-	I40IW_ERR_NVM_CHECKSUM = -2,</div><div class='del'>-	I40IW_ERR_CONFIG = -4,</div><div class='del'>-	I40IW_ERR_PARAM = -5,</div><div class='del'>-	I40IW_ERR_DEVICE_NOT_SUPPORTED = -6,</div><div class='del'>-	I40IW_ERR_RESET_FAILED = -7,</div><div class='del'>-	I40IW_ERR_SWFW_SYNC = -8,</div><div class='del'>-	I40IW_ERR_NO_MEMORY = -9,</div><div class='del'>-	I40IW_ERR_BAD_PTR = -10,</div><div class='del'>-	I40IW_ERR_INVALID_PD_ID = -11,</div><div class='del'>-	I40IW_ERR_INVALID_QP_ID = -12,</div><div class='del'>-	I40IW_ERR_INVALID_CQ_ID = -13,</div><div class='del'>-	I40IW_ERR_INVALID_CEQ_ID = -14,</div><div class='del'>-	I40IW_ERR_INVALID_AEQ_ID = -15,</div><div class='del'>-	I40IW_ERR_INVALID_SIZE = -16,</div><div class='del'>-	I40IW_ERR_INVALID_ARP_INDEX = -17,</div><div class='del'>-	I40IW_ERR_INVALID_FPM_FUNC_ID = -18,</div><div class='del'>-	I40IW_ERR_QP_INVALID_MSG_SIZE = -19,</div><div class='del'>-	I40IW_ERR_QP_TOOMANY_WRS_POSTED = -20,</div><div class='del'>-	I40IW_ERR_INVALID_FRAG_COUNT = -21,</div><div class='del'>-	I40IW_ERR_QUEUE_EMPTY = -22,</div><div class='del'>-	I40IW_ERR_INVALID_ALIGNMENT = -23,</div><div class='del'>-	I40IW_ERR_FLUSHED_QUEUE = -24,</div><div class='del'>-	I40IW_ERR_INVALID_PUSH_PAGE_INDEX = -25,</div><div class='del'>-	I40IW_ERR_INVALID_INLINE_DATA_SIZE = -26,</div><div class='del'>-	I40IW_ERR_TIMEOUT = -27,</div><div class='del'>-	I40IW_ERR_OPCODE_MISMATCH = -28,</div><div class='del'>-	I40IW_ERR_CQP_COMPL_ERROR = -29,</div><div class='del'>-	I40IW_ERR_INVALID_VF_ID = -30,</div><div class='del'>-	I40IW_ERR_INVALID_HMCFN_ID = -31,</div><div class='del'>-	I40IW_ERR_BACKING_PAGE_ERROR = -32,</div><div class='del'>-	I40IW_ERR_NO_PBLCHUNKS_AVAILABLE = -33,</div><div class='del'>-	I40IW_ERR_INVALID_PBLE_INDEX = -34,</div><div class='del'>-	I40IW_ERR_INVALID_SD_INDEX = -35,</div><div class='del'>-	I40IW_ERR_INVALID_PAGE_DESC_INDEX = -36,</div><div class='del'>-	I40IW_ERR_INVALID_SD_TYPE = -37,</div><div class='del'>-	I40IW_ERR_MEMCPY_FAILED = -38,</div><div class='del'>-	I40IW_ERR_INVALID_HMC_OBJ_INDEX = -39,</div><div class='del'>-	I40IW_ERR_INVALID_HMC_OBJ_COUNT = -40,</div><div class='del'>-	I40IW_ERR_INVALID_SRQ_ARM_LIMIT = -41,</div><div class='del'>-	I40IW_ERR_SRQ_ENABLED = -42,</div><div class='del'>-	I40IW_ERR_BUF_TOO_SHORT = -43,</div><div class='del'>-	I40IW_ERR_BAD_IWARP_CQE = -44,</div><div class='del'>-	I40IW_ERR_NVM_BLANK_MODE = -45,</div><div class='del'>-	I40IW_ERR_NOT_IMPLEMENTED = -46,</div><div class='del'>-	I40IW_ERR_PE_DOORBELL_NOT_ENABLED = -47,</div><div class='del'>-	I40IW_ERR_NOT_READY = -48,</div><div class='del'>-	I40IW_NOT_SUPPORTED = -49,</div><div class='del'>-	I40IW_ERR_FIRMWARE_API_VERSION = -50,</div><div class='del'>-	I40IW_ERR_RING_FULL = -51,</div><div class='del'>-	I40IW_ERR_MPA_CRC = -61,</div><div class='del'>-	I40IW_ERR_NO_TXBUFS = -62,</div><div class='del'>-	I40IW_ERR_SEQ_NUM = -63,</div><div class='del'>-	I40IW_ERR_list_empty = -64,</div><div class='del'>-	I40IW_ERR_INVALID_MAC_ADDR = -65,</div><div class='del'>-	I40IW_ERR_BAD_STAG      = -66,</div><div class='del'>-	I40IW_ERR_CQ_COMPL_ERROR = -67,</div><div class='del'>-	I40IW_ERR_QUEUE_DESTROYED = -68</div><div class='del'>-</div><div class='del'>-};</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_type.h b/drivers/infiniband/hw/i40iw/i40iw_type.h<br/>deleted file mode 100644<br/>index adc8d2ec523d..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_type.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_type.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,1363 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#ifndef I40IW_TYPE_H</div><div class='del'>-#define I40IW_TYPE_H</div><div class='del'>-#include "i40iw_user.h"</div><div class='del'>-#include "i40iw_hmc.h"</div><div class='del'>-#include "i40iw_vf.h"</div><div class='del'>-#include "i40iw_virtchnl.h"</div><div class='del'>-</div><div class='del'>-struct i40iw_cqp_sq_wqe {</div><div class='del'>-	u64 buf[I40IW_CQP_WQE_SIZE];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_sc_aeqe {</div><div class='del'>-	u64 buf[I40IW_AEQE_SIZE];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_ceqe {</div><div class='del'>-	u64 buf[I40IW_CEQE_SIZE];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cqp_ctx {</div><div class='del'>-	u64 buf[I40IW_CQP_CTX_SIZE];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cq_shadow_area {</div><div class='del'>-	u64 buf[I40IW_SHADOW_AREA_SIZE];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_sc_dev;</div><div class='del'>-struct i40iw_hmc_info;</div><div class='del'>-struct i40iw_vsi_pestat;</div><div class='del'>-</div><div class='del'>-struct i40iw_cqp_ops;</div><div class='del'>-struct i40iw_ccq_ops;</div><div class='del'>-struct i40iw_ceq_ops;</div><div class='del'>-struct i40iw_aeq_ops;</div><div class='del'>-struct i40iw_mr_ops;</div><div class='del'>-struct i40iw_cqp_misc_ops;</div><div class='del'>-struct i40iw_pd_ops;</div><div class='del'>-struct i40iw_priv_qp_ops;</div><div class='del'>-struct i40iw_priv_cq_ops;</div><div class='del'>-struct i40iw_hmc_ops;</div><div class='del'>-</div><div class='del'>-enum i40iw_page_size {</div><div class='del'>-	I40IW_PAGE_SIZE_4K,</div><div class='del'>-	I40IW_PAGE_SIZE_2M</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_resource_indicator_type {</div><div class='del'>-	I40IW_RSRC_INDICATOR_TYPE_ADAPTER = 0,</div><div class='del'>-	I40IW_RSRC_INDICATOR_TYPE_CQ,</div><div class='del'>-	I40IW_RSRC_INDICATOR_TYPE_QP,</div><div class='del'>-	I40IW_RSRC_INDICATOR_TYPE_SRQ</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_hdrct_flags {</div><div class='del'>-	DDP_LEN_FLAG = 0x80,</div><div class='del'>-	DDP_HDR_FLAG = 0x40,</div><div class='del'>-	RDMA_HDR_FLAG = 0x20</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_term_layers {</div><div class='del'>-	LAYER_RDMA = 0,</div><div class='del'>-	LAYER_DDP = 1,</div><div class='del'>-	LAYER_MPA = 2</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_term_error_types {</div><div class='del'>-	RDMAP_REMOTE_PROT = 1,</div><div class='del'>-	RDMAP_REMOTE_OP = 2,</div><div class='del'>-	DDP_CATASTROPHIC = 0,</div><div class='del'>-	DDP_TAGGED_BUFFER = 1,</div><div class='del'>-	DDP_UNTAGGED_BUFFER = 2,</div><div class='del'>-	DDP_LLP = 3</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_term_rdma_errors {</div><div class='del'>-	RDMAP_INV_STAG = 0x00,</div><div class='del'>-	RDMAP_INV_BOUNDS = 0x01,</div><div class='del'>-	RDMAP_ACCESS = 0x02,</div><div class='del'>-	RDMAP_UNASSOC_STAG = 0x03,</div><div class='del'>-	RDMAP_TO_WRAP = 0x04,</div><div class='del'>-	RDMAP_INV_RDMAP_VER = 0x05,</div><div class='del'>-	RDMAP_UNEXPECTED_OP = 0x06,</div><div class='del'>-	RDMAP_CATASTROPHIC_LOCAL = 0x07,</div><div class='del'>-	RDMAP_CATASTROPHIC_GLOBAL = 0x08,</div><div class='del'>-	RDMAP_CANT_INV_STAG = 0x09,</div><div class='del'>-	RDMAP_UNSPECIFIED = 0xff</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_term_ddp_errors {</div><div class='del'>-	DDP_CATASTROPHIC_LOCAL = 0x00,</div><div class='del'>-	DDP_TAGGED_INV_STAG = 0x00,</div><div class='del'>-	DDP_TAGGED_BOUNDS = 0x01,</div><div class='del'>-	DDP_TAGGED_UNASSOC_STAG = 0x02,</div><div class='del'>-	DDP_TAGGED_TO_WRAP = 0x03,</div><div class='del'>-	DDP_TAGGED_INV_DDP_VER = 0x04,</div><div class='del'>-	DDP_UNTAGGED_INV_QN = 0x01,</div><div class='del'>-	DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02,</div><div class='del'>-	DDP_UNTAGGED_INV_MSN_RANGE = 0x03,</div><div class='del'>-	DDP_UNTAGGED_INV_MO = 0x04,</div><div class='del'>-	DDP_UNTAGGED_INV_TOO_LONG = 0x05,</div><div class='del'>-	DDP_UNTAGGED_INV_DDP_VER = 0x06</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_term_mpa_errors {</div><div class='del'>-	MPA_CLOSED = 0x01,</div><div class='del'>-	MPA_CRC = 0x02,</div><div class='del'>-	MPA_MARKER = 0x03,</div><div class='del'>-	MPA_REQ_RSP = 0x04,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_flush_opcode {</div><div class='del'>-	FLUSH_INVALID = 0,</div><div class='del'>-	FLUSH_PROT_ERR,</div><div class='del'>-	FLUSH_REM_ACCESS_ERR,</div><div class='del'>-	FLUSH_LOC_QP_OP_ERR,</div><div class='del'>-	FLUSH_REM_OP_ERR,</div><div class='del'>-	FLUSH_LOC_LEN_ERR,</div><div class='del'>-	FLUSH_GENERAL_ERR,</div><div class='del'>-	FLUSH_FATAL_ERR</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_term_eventtypes {</div><div class='del'>-	TERM_EVENT_QP_FATAL,</div><div class='del'>-	TERM_EVENT_QP_ACCESS_ERR</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_terminate_hdr {</div><div class='del'>-	u8 layer_etype;</div><div class='del'>-	u8 error_code;</div><div class='del'>-	u8 hdrct;</div><div class='del'>-	u8 rsvd;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_debug_flag {</div><div class='del'>-	I40IW_DEBUG_NONE	= 0x00000000,</div><div class='del'>-	I40IW_DEBUG_ERR		= 0x00000001,</div><div class='del'>-	I40IW_DEBUG_INIT	= 0x00000002,</div><div class='del'>-	I40IW_DEBUG_DEV		= 0x00000004,</div><div class='del'>-	I40IW_DEBUG_CM		= 0x00000008,</div><div class='del'>-	I40IW_DEBUG_VERBS	= 0x00000010,</div><div class='del'>-	I40IW_DEBUG_PUDA	= 0x00000020,</div><div class='del'>-	I40IW_DEBUG_ILQ		= 0x00000040,</div><div class='del'>-	I40IW_DEBUG_IEQ		= 0x00000080,</div><div class='del'>-	I40IW_DEBUG_QP		= 0x00000100,</div><div class='del'>-	I40IW_DEBUG_CQ		= 0x00000200,</div><div class='del'>-	I40IW_DEBUG_MR		= 0x00000400,</div><div class='del'>-	I40IW_DEBUG_PBLE	= 0x00000800,</div><div class='del'>-	I40IW_DEBUG_WQE		= 0x00001000,</div><div class='del'>-	I40IW_DEBUG_AEQ		= 0x00002000,</div><div class='del'>-	I40IW_DEBUG_CQP		= 0x00004000,</div><div class='del'>-	I40IW_DEBUG_HMC		= 0x00008000,</div><div class='del'>-	I40IW_DEBUG_USER	= 0x00010000,</div><div class='del'>-	I40IW_DEBUG_VIRT	= 0x00020000,</div><div class='del'>-	I40IW_DEBUG_DCB		= 0x00040000,</div><div class='del'>-	I40IW_DEBUG_CQE		= 0x00800000,</div><div class='del'>-	I40IW_DEBUG_ALL		= 0xFFFFFFFF</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_hw_stats_index_32b {</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP4RXDISCARD = 0,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP4RXTRUNC,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP4TXNOROUTE,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP6RXDISCARD,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP6RXTRUNC,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP6TXNOROUTE,</div><div class='del'>-	I40IW_HW_STAT_INDEX_TCPRTXSEG,</div><div class='del'>-	I40IW_HW_STAT_INDEX_TCPRXOPTERR,</div><div class='del'>-	I40IW_HW_STAT_INDEX_TCPRXPROTOERR,</div><div class='del'>-	I40IW_HW_STAT_INDEX_MAX_32</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_hw_stats_index_64b {</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP4RXOCTS = 0,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP4RXPKTS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP4RXFRAGS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP4RXMCPKTS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP4TXOCTS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP4TXPKTS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP4TXFRAGS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP4TXMCPKTS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP6RXOCTS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP6RXPKTS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP6RXFRAGS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP6RXMCPKTS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP6TXOCTS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP6TXPKTS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP6TXFRAGS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_IP6TXMCPKTS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_TCPRXSEGS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_TCPTXSEG,</div><div class='del'>-	I40IW_HW_STAT_INDEX_RDMARXRDS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_RDMARXSNDS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_RDMARXWRS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_RDMATXRDS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_RDMATXSNDS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_RDMATXWRS,</div><div class='del'>-	I40IW_HW_STAT_INDEX_RDMAVBND,</div><div class='del'>-	I40IW_HW_STAT_INDEX_RDMAVINV,</div><div class='del'>-	I40IW_HW_STAT_INDEX_MAX_64</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_dev_hw_stats_offsets {</div><div class='del'>-	u32 stats_offset_32[I40IW_HW_STAT_INDEX_MAX_32];</div><div class='del'>-	u32 stats_offset_64[I40IW_HW_STAT_INDEX_MAX_64];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_dev_hw_stats {</div><div class='del'>-	u64 stats_value_32[I40IW_HW_STAT_INDEX_MAX_32];</div><div class='del'>-	u64 stats_value_64[I40IW_HW_STAT_INDEX_MAX_64];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_vsi_pestat {</div><div class='del'>-	struct i40iw_hw *hw;</div><div class='del'>-	struct i40iw_dev_hw_stats hw_stats;</div><div class='del'>-	struct i40iw_dev_hw_stats last_read_hw_stats;</div><div class='del'>-	struct i40iw_dev_hw_stats_offsets hw_stats_offsets;</div><div class='del'>-	struct timer_list stats_timer;</div><div class='del'>-	struct i40iw_sc_vsi *vsi;</div><div class='del'>-	spinlock_t lock; /* rdma stats lock */</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_hw {</div><div class='del'>-	u8 __iomem *hw_addr;</div><div class='del'>-	void *dev_context;</div><div class='del'>-	struct i40iw_hmc_info hmc;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_pfpdu {</div><div class='del'>-	struct list_head rxlist;</div><div class='del'>-	u32 rcv_nxt;</div><div class='del'>-	u32 fps;</div><div class='del'>-	u32 max_fpdu_data;</div><div class='del'>-	bool mode;</div><div class='del'>-	bool mpa_crc_err;</div><div class='del'>-	u64 total_ieq_bufs;</div><div class='del'>-	u64 fpdu_processed;</div><div class='del'>-	u64 bad_seq_num;</div><div class='del'>-	u64 crc_err;</div><div class='del'>-	u64 no_tx_bufs;</div><div class='del'>-	u64 tx_err;</div><div class='del'>-	u64 out_of_order;</div><div class='del'>-	u64 pmode_count;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_sc_pd {</div><div class='del'>-	u32 size;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	u16 pd_id;</div><div class='del'>-	int abi_ver;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cqp_quanta {</div><div class='del'>-	u64 elem[I40IW_CQP_WQE_SIZE];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_sc_cqp {</div><div class='del'>-	u32 size;</div><div class='del'>-	u64 sq_pa;</div><div class='del'>-	u64 host_ctx_pa;</div><div class='del'>-	void *back_cqp;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	enum i40iw_status_code (*process_cqp_sds)(struct i40iw_sc_dev *,</div><div class='del'>-						  struct i40iw_update_sds_info *);</div><div class='del'>-	struct i40iw_dma_mem sdbuf;</div><div class='del'>-	struct i40iw_ring sq_ring;</div><div class='del'>-	struct i40iw_cqp_quanta *sq_base;</div><div class='del'>-	u64 *host_ctx;</div><div class='del'>-	u64 *scratch_array;</div><div class='del'>-	u32 cqp_id;</div><div class='del'>-	u32 sq_size;</div><div class='del'>-	u32 hw_sq_size;</div><div class='del'>-	u8 struct_ver;</div><div class='del'>-	u8 polarity;</div><div class='del'>-	bool en_datacenter_tcp;</div><div class='del'>-	u8 hmc_profile;</div><div class='del'>-	u8 enabled_vf_count;</div><div class='del'>-	u8 timeout_count;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_sc_aeq {</div><div class='del'>-	u32 size;</div><div class='del'>-	u64 aeq_elem_pa;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	struct i40iw_sc_aeqe *aeqe_base;</div><div class='del'>-	void *pbl_list;</div><div class='del'>-	u32 elem_cnt;</div><div class='del'>-	struct i40iw_ring aeq_ring;</div><div class='del'>-	bool virtual_map;</div><div class='del'>-	u8 pbl_chunk_size;</div><div class='del'>-	u32 first_pm_pbl_idx;</div><div class='del'>-	u8 polarity;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_sc_ceq {</div><div class='del'>-	u32 size;</div><div class='del'>-	u64 ceq_elem_pa;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	struct i40iw_ceqe *ceqe_base;</div><div class='del'>-	void *pbl_list;</div><div class='del'>-	u32 ceq_id;</div><div class='del'>-	u32 elem_cnt;</div><div class='del'>-	struct i40iw_ring ceq_ring;</div><div class='del'>-	bool virtual_map;</div><div class='del'>-	u8 pbl_chunk_size;</div><div class='del'>-	bool tph_en;</div><div class='del'>-	u8 tph_val;</div><div class='del'>-	u32 first_pm_pbl_idx;</div><div class='del'>-	u8 polarity;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_sc_cq {</div><div class='del'>-	struct i40iw_cq_uk cq_uk;</div><div class='del'>-	u64 cq_pa;</div><div class='del'>-	u64 shadow_area_pa;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	struct i40iw_sc_vsi *vsi;</div><div class='del'>-	void *pbl_list;</div><div class='del'>-	void *back_cq;</div><div class='del'>-	u32 ceq_id;</div><div class='del'>-	u32 shadow_read_threshold;</div><div class='del'>-	bool ceqe_mask;</div><div class='del'>-	bool virtual_map;</div><div class='del'>-	u8 pbl_chunk_size;</div><div class='del'>-	u8 cq_type;</div><div class='del'>-	bool ceq_id_valid;</div><div class='del'>-	bool tph_en;</div><div class='del'>-	u8 tph_val;</div><div class='del'>-	u32 first_pm_pbl_idx;</div><div class='del'>-	bool check_overflow;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_sc_qp {</div><div class='del'>-	struct i40iw_qp_uk qp_uk;</div><div class='del'>-	u64 sq_pa;</div><div class='del'>-	u64 rq_pa;</div><div class='del'>-	u64 hw_host_ctx_pa;</div><div class='del'>-	u64 shadow_area_pa;</div><div class='del'>-	u64 q2_pa;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	struct i40iw_sc_vsi *vsi;</div><div class='del'>-	struct i40iw_sc_pd *pd;</div><div class='del'>-	u64 *hw_host_ctx;</div><div class='del'>-	void *llp_stream_handle;</div><div class='del'>-	void *back_qp;</div><div class='del'>-	struct i40iw_pfpdu pfpdu;</div><div class='del'>-	u8 *q2_buf;</div><div class='del'>-	u64 qp_compl_ctx;</div><div class='del'>-	u16 qs_handle;</div><div class='del'>-	u16 push_idx;</div><div class='del'>-	u8 sq_tph_val;</div><div class='del'>-	u8 rq_tph_val;</div><div class='del'>-	u8 qp_state;</div><div class='del'>-	u8 qp_type;</div><div class='del'>-	u8 hw_sq_size;</div><div class='del'>-	u8 hw_rq_size;</div><div class='del'>-	u8 src_mac_addr_idx;</div><div class='del'>-	bool sq_tph_en;</div><div class='del'>-	bool rq_tph_en;</div><div class='del'>-	bool rcv_tph_en;</div><div class='del'>-	bool xmit_tph_en;</div><div class='del'>-	bool virtual_map;</div><div class='del'>-	bool flush_sq;</div><div class='del'>-	bool flush_rq;</div><div class='del'>-	u8 user_pri;</div><div class='del'>-	struct list_head list;</div><div class='del'>-	bool on_qoslist;</div><div class='del'>-	bool sq_flush;</div><div class='del'>-	enum i40iw_flush_opcode flush_code;</div><div class='del'>-	enum i40iw_term_eventtypes eventtype;</div><div class='del'>-	u8 term_flags;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_hmc_fpm_misc {</div><div class='del'>-	u32 max_ceqs;</div><div class='del'>-	u32 max_sds;</div><div class='del'>-	u32 xf_block_size;</div><div class='del'>-	u32 q1_block_size;</div><div class='del'>-	u32 ht_multiplier;</div><div class='del'>-	u32 timer_bucket;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_vchnl_if {</div><div class='del'>-	enum i40iw_status_code (*vchnl_recv)(struct i40iw_sc_dev *, u32, u8 *, u16);</div><div class='del'>-	enum i40iw_status_code (*vchnl_send)(struct i40iw_sc_dev *dev, u32, u8 *, u16);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define I40IW_VCHNL_MAX_VF_MSG_SIZE 512</div><div class='del'>-</div><div class='del'>-struct i40iw_vchnl_vf_msg_buffer {</div><div class='del'>-	struct i40iw_virtchnl_op_buf vchnl_msg;</div><div class='del'>-	char parm_buffer[I40IW_VCHNL_MAX_VF_MSG_SIZE - 1];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_qos {</div><div class='del'>-	struct list_head qplist;</div><div class='del'>-	spinlock_t lock;	/* qos list */</div><div class='del'>-	u16 qs_handle;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_vfdev {</div><div class='del'>-	struct i40iw_sc_dev *pf_dev;</div><div class='del'>-	u8 *hmc_info_mem;</div><div class='del'>-	struct i40iw_vsi_pestat pestat;</div><div class='del'>-	struct i40iw_hmc_pble_info *pble_info;</div><div class='del'>-	struct i40iw_hmc_info hmc_info;</div><div class='del'>-	struct i40iw_vchnl_vf_msg_buffer vf_msg_buffer;</div><div class='del'>-	u64 fpm_query_buf_pa;</div><div class='del'>-	u64 *fpm_query_buf;</div><div class='del'>-	u32 vf_id;</div><div class='del'>-	u32 msg_count;</div><div class='del'>-	bool pf_hmc_initialized;</div><div class='del'>-	u16 pmf_index;</div><div class='del'>-	u16 iw_vf_idx;		/* VF Device table index */</div><div class='del'>-	bool stats_initialized;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define I40IW_INVALID_FCN_ID 0xff</div><div class='del'>-struct i40iw_sc_vsi {</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	void *back_vsi; /* Owned by OS */</div><div class='del'>-	u32 ilq_count;</div><div class='del'>-	struct i40iw_virt_mem ilq_mem;</div><div class='del'>-	struct i40iw_puda_rsrc *ilq;</div><div class='del'>-	u32 ieq_count;</div><div class='del'>-	struct i40iw_virt_mem ieq_mem;</div><div class='del'>-	struct i40iw_puda_rsrc *ieq;</div><div class='del'>-	u16 exception_lan_queue;</div><div class='del'>-	u16 mtu;</div><div class='del'>-	u8 fcn_id;</div><div class='del'>-	bool stats_fcn_id_alloc;</div><div class='del'>-	struct i40iw_qos qos[I40IW_MAX_USER_PRIORITY];</div><div class='del'>-	struct i40iw_vsi_pestat *pestat;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_sc_dev {</div><div class='del'>-	struct list_head cqp_cmd_head;	/* head of the CQP command list */</div><div class='del'>-	spinlock_t cqp_lock; /* cqp list sync */</div><div class='del'>-	struct i40iw_dev_uk dev_uk;</div><div class='del'>-	bool fcn_id_array[I40IW_MAX_STATS_COUNT];</div><div class='del'>-	struct i40iw_dma_mem vf_fpm_query_buf[I40IW_MAX_PE_ENABLED_VF_COUNT];</div><div class='del'>-	u64 fpm_query_buf_pa;</div><div class='del'>-	u64 fpm_commit_buf_pa;</div><div class='del'>-	u64 *fpm_query_buf;</div><div class='del'>-	u64 *fpm_commit_buf;</div><div class='del'>-	void *back_dev;</div><div class='del'>-	struct i40iw_hw *hw;</div><div class='del'>-	u8 __iomem *db_addr;</div><div class='del'>-	struct i40iw_hmc_info *hmc_info;</div><div class='del'>-	struct i40iw_hmc_pble_info *pble_info;</div><div class='del'>-	struct i40iw_vfdev *vf_dev[I40IW_MAX_PE_ENABLED_VF_COUNT];</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	struct i40iw_sc_aeq *aeq;</div><div class='del'>-	struct i40iw_sc_ceq *ceq[I40IW_CEQ_MAX_COUNT];</div><div class='del'>-	struct i40iw_sc_cq *ccq;</div><div class='del'>-	struct i40iw_cqp_ops *cqp_ops;</div><div class='del'>-	struct i40iw_ccq_ops *ccq_ops;</div><div class='del'>-	struct i40iw_ceq_ops *ceq_ops;</div><div class='del'>-	struct i40iw_aeq_ops *aeq_ops;</div><div class='del'>-	struct i40iw_pd_ops *iw_pd_ops;</div><div class='del'>-	struct i40iw_priv_qp_ops *iw_priv_qp_ops;</div><div class='del'>-	struct i40iw_priv_cq_ops *iw_priv_cq_ops;</div><div class='del'>-	struct i40iw_mr_ops *mr_ops;</div><div class='del'>-	struct i40iw_cqp_misc_ops *cqp_misc_ops;</div><div class='del'>-	struct i40iw_hmc_ops *hmc_ops;</div><div class='del'>-	struct i40iw_vchnl_if vchnl_if;</div><div class='del'>-	const struct i40iw_vf_cqp_ops *iw_vf_cqp_ops;</div><div class='del'>-</div><div class='del'>-	struct i40iw_hmc_fpm_misc hmc_fpm_misc;</div><div class='del'>-	u32 debug_mask;</div><div class='del'>-	u8 hmc_fn_id;</div><div class='del'>-	bool is_pf;</div><div class='del'>-	bool vchnl_up;</div><div class='del'>-	bool ceq_valid;</div><div class='del'>-	u8 vf_id;</div><div class='del'>-	wait_queue_head_t vf_reqs;</div><div class='del'>-	u64 cqp_cmd_stats[OP_SIZE_CQP_STAT_ARRAY];</div><div class='del'>-	struct i40iw_vchnl_vf_msg_buffer vchnl_vf_msg_buf;</div><div class='del'>-	u8 hw_rev;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_modify_cq_info {</div><div class='del'>-	u64 cq_pa;</div><div class='del'>-	struct i40iw_cqe *cq_base;</div><div class='del'>-	void *pbl_list;</div><div class='del'>-	u32 ceq_id;</div><div class='del'>-	u32 cq_size;</div><div class='del'>-	u32 shadow_read_threshold;</div><div class='del'>-	bool virtual_map;</div><div class='del'>-	u8 pbl_chunk_size;</div><div class='del'>-	bool check_overflow;</div><div class='del'>-	bool cq_resize;</div><div class='del'>-	bool ceq_change;</div><div class='del'>-	bool check_overflow_change;</div><div class='del'>-	u32 first_pm_pbl_idx;</div><div class='del'>-	bool ceq_valid;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_create_qp_info {</div><div class='del'>-	u8 next_iwarp_state;</div><div class='del'>-	bool ord_valid;</div><div class='del'>-	bool tcp_ctx_valid;</div><div class='del'>-	bool cq_num_valid;</div><div class='del'>-	bool arp_cache_idx_valid;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_modify_qp_info {</div><div class='del'>-	u64 rx_win0;</div><div class='del'>-	u64 rx_win1;</div><div class='del'>-	u8 next_iwarp_state;</div><div class='del'>-	u8 termlen;</div><div class='del'>-	bool ord_valid;</div><div class='del'>-	bool tcp_ctx_valid;</div><div class='del'>-	bool cq_num_valid;</div><div class='del'>-	bool arp_cache_idx_valid;</div><div class='del'>-	bool reset_tcp_conn;</div><div class='del'>-	bool remove_hash_idx;</div><div class='del'>-	bool dont_send_term;</div><div class='del'>-	bool dont_send_fin;</div><div class='del'>-	bool cached_var_valid;</div><div class='del'>-	bool force_loopback;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_ccq_cqe_info {</div><div class='del'>-	struct i40iw_sc_cqp *cqp;</div><div class='del'>-	u64 scratch;</div><div class='del'>-	u32 op_ret_val;</div><div class='del'>-	u16 maj_err_code;</div><div class='del'>-	u16 min_err_code;</div><div class='del'>-	u8 op_code;</div><div class='del'>-	bool error;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_l2params {</div><div class='del'>-	u16 qs_handle_list[I40IW_MAX_USER_PRIORITY];</div><div class='del'>-	u16 mtu;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_vsi_init_info {</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	void  *back_vsi;</div><div class='del'>-	struct i40iw_l2params *params;</div><div class='del'>-	u16 exception_lan_queue;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_vsi_stats_info {</div><div class='del'>-	struct i40iw_vsi_pestat *pestat;</div><div class='del'>-	u8 fcn_id;</div><div class='del'>-	bool alloc_fcn_id;</div><div class='del'>-	bool stats_initialize;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_device_init_info {</div><div class='del'>-	u64 fpm_query_buf_pa;</div><div class='del'>-	u64 fpm_commit_buf_pa;</div><div class='del'>-	u64 *fpm_query_buf;</div><div class='del'>-	u64 *fpm_commit_buf;</div><div class='del'>-	struct i40iw_hw *hw;</div><div class='del'>-	void __iomem *bar0;</div><div class='del'>-	enum i40iw_status_code (*vchnl_send)(struct i40iw_sc_dev *, u32, u8 *, u16);</div><div class='del'>-	u8 hmc_fn_id;</div><div class='del'>-	bool is_pf;</div><div class='del'>-	u32 debug_mask;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_cqp_hmc_profile {</div><div class='del'>-	I40IW_HMC_PROFILE_DEFAULT = 1,</div><div class='del'>-	I40IW_HMC_PROFILE_FAVOR_VF = 2,</div><div class='del'>-	I40IW_HMC_PROFILE_EQUAL = 3,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cqp_init_info {</div><div class='del'>-	u64 cqp_compl_ctx;</div><div class='del'>-	u64 host_ctx_pa;</div><div class='del'>-	u64 sq_pa;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	struct i40iw_cqp_quanta *sq;</div><div class='del'>-	u64 *host_ctx;</div><div class='del'>-	u64 *scratch_array;</div><div class='del'>-	u32 sq_size;</div><div class='del'>-	u8 struct_ver;</div><div class='del'>-	bool en_datacenter_tcp;</div><div class='del'>-	u8 hmc_profile;</div><div class='del'>-	u8 enabled_vf_count;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_ceq_init_info {</div><div class='del'>-	u64 ceqe_pa;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	u64 *ceqe_base;</div><div class='del'>-	void *pbl_list;</div><div class='del'>-	u32 elem_cnt;</div><div class='del'>-	u32 ceq_id;</div><div class='del'>-	bool virtual_map;</div><div class='del'>-	u8 pbl_chunk_size;</div><div class='del'>-	bool tph_en;</div><div class='del'>-	u8 tph_val;</div><div class='del'>-	u32 first_pm_pbl_idx;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_aeq_init_info {</div><div class='del'>-	u64 aeq_elem_pa;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	u32 *aeqe_base;</div><div class='del'>-	void *pbl_list;</div><div class='del'>-	u32 elem_cnt;</div><div class='del'>-	bool virtual_map;</div><div class='del'>-	u8 pbl_chunk_size;</div><div class='del'>-	u32 first_pm_pbl_idx;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_ccq_init_info {</div><div class='del'>-	u64 cq_pa;</div><div class='del'>-	u64 shadow_area_pa;</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	struct i40iw_cqe *cq_base;</div><div class='del'>-	u64 *shadow_area;</div><div class='del'>-	void *pbl_list;</div><div class='del'>-	u32 num_elem;</div><div class='del'>-	u32 ceq_id;</div><div class='del'>-	u32 shadow_read_threshold;</div><div class='del'>-	bool ceqe_mask;</div><div class='del'>-	bool ceq_id_valid;</div><div class='del'>-	bool tph_en;</div><div class='del'>-	u8 tph_val;</div><div class='del'>-	bool avoid_mem_cflct;</div><div class='del'>-	bool virtual_map;</div><div class='del'>-	u8 pbl_chunk_size;</div><div class='del'>-	u32 first_pm_pbl_idx;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iwarp_offload_info {</div><div class='del'>-	u16 rcv_mark_offset;</div><div class='del'>-	u16 snd_mark_offset;</div><div class='del'>-	u16 pd_id;</div><div class='del'>-	u8 ddp_ver;</div><div class='del'>-	u8 rdmap_ver;</div><div class='del'>-	u8 ord_size;</div><div class='del'>-	u8 ird_size;</div><div class='del'>-	bool wr_rdresp_en;</div><div class='del'>-	bool rd_enable;</div><div class='del'>-	bool snd_mark_en;</div><div class='del'>-	bool rcv_mark_en;</div><div class='del'>-	bool bind_en;</div><div class='del'>-	bool fast_reg_en;</div><div class='del'>-	bool priv_mode_en;</div><div class='del'>-	bool lsmm_present;</div><div class='del'>-	u8 iwarp_mode;</div><div class='del'>-	bool align_hdrs;</div><div class='del'>-	bool rcv_no_mpa_crc;</div><div class='del'>-</div><div class='del'>-	u8 last_byte_sent;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_tcp_offload_info {</div><div class='del'>-	bool ipv4;</div><div class='del'>-	bool no_nagle;</div><div class='del'>-	bool insert_vlan_tag;</div><div class='del'>-	bool time_stamp;</div><div class='del'>-	u8 cwnd_inc_limit;</div><div class='del'>-	bool drop_ooo_seg;</div><div class='del'>-	u8 dup_ack_thresh;</div><div class='del'>-	u8 ttl;</div><div class='del'>-	u8 src_mac_addr_idx;</div><div class='del'>-	bool avoid_stretch_ack;</div><div class='del'>-	u8 tos;</div><div class='del'>-	u16 src_port;</div><div class='del'>-	u16 dst_port;</div><div class='del'>-	u32 dest_ip_addr0;</div><div class='del'>-	u32 dest_ip_addr1;</div><div class='del'>-	u32 dest_ip_addr2;</div><div class='del'>-	u32 dest_ip_addr3;</div><div class='del'>-	u32 snd_mss;</div><div class='del'>-	u16 vlan_tag;</div><div class='del'>-	u16 arp_idx;</div><div class='del'>-	u32 flow_label;</div><div class='del'>-	bool wscale;</div><div class='del'>-	u8 tcp_state;</div><div class='del'>-	u8 snd_wscale;</div><div class='del'>-	u8 rcv_wscale;</div><div class='del'>-	u32 time_stamp_recent;</div><div class='del'>-	u32 time_stamp_age;</div><div class='del'>-	u32 snd_nxt;</div><div class='del'>-	u32 snd_wnd;</div><div class='del'>-	u32 rcv_nxt;</div><div class='del'>-	u32 rcv_wnd;</div><div class='del'>-	u32 snd_max;</div><div class='del'>-	u32 snd_una;</div><div class='del'>-	u32 srtt;</div><div class='del'>-	u32 rtt_var;</div><div class='del'>-	u32 ss_thresh;</div><div class='del'>-	u32 cwnd;</div><div class='del'>-	u32 snd_wl1;</div><div class='del'>-	u32 snd_wl2;</div><div class='del'>-	u32 max_snd_window;</div><div class='del'>-	u8 rexmit_thresh;</div><div class='del'>-	u32 local_ipaddr0;</div><div class='del'>-	u32 local_ipaddr1;</div><div class='del'>-	u32 local_ipaddr2;</div><div class='del'>-	u32 local_ipaddr3;</div><div class='del'>-	bool ignore_tcp_opt;</div><div class='del'>-	bool ignore_tcp_uns_opt;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_qp_host_ctx_info {</div><div class='del'>-	u64 qp_compl_ctx;</div><div class='del'>-	struct i40iw_tcp_offload_info *tcp_info;</div><div class='del'>-	struct i40iwarp_offload_info *iwarp_info;</div><div class='del'>-	u32 send_cq_num;</div><div class='del'>-	u32 rcv_cq_num;</div><div class='del'>-	u16 push_idx;</div><div class='del'>-	bool push_mode_en;</div><div class='del'>-	bool tcp_info_valid;</div><div class='del'>-	bool iwarp_info_valid;</div><div class='del'>-	bool err_rq_idx_valid;</div><div class='del'>-	u16 err_rq_idx;</div><div class='del'>-	bool add_to_qoslist;</div><div class='del'>-	u8 user_pri;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_aeqe_info {</div><div class='del'>-	u64 compl_ctx;</div><div class='del'>-	u32 qp_cq_id;</div><div class='del'>-	u16 ae_id;</div><div class='del'>-	u16 wqe_idx;</div><div class='del'>-	u8 tcp_state;</div><div class='del'>-	u8 iwarp_state;</div><div class='del'>-	bool qp;</div><div class='del'>-	bool cq;</div><div class='del'>-	bool sq;</div><div class='del'>-	bool in_rdrsp_wr;</div><div class='del'>-	bool out_rdrsp;</div><div class='del'>-	u8 q2_data_written;</div><div class='del'>-	bool aeqe_overflow;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_allocate_stag_info {</div><div class='del'>-	u64 total_len;</div><div class='del'>-	u32 chunk_size;</div><div class='del'>-	u32 stag_idx;</div><div class='del'>-	u32 page_size;</div><div class='del'>-	u16 pd_id;</div><div class='del'>-	u16 access_rights;</div><div class='del'>-	bool remote_access;</div><div class='del'>-	bool use_hmc_fcn_index;</div><div class='del'>-	u8 hmc_fcn_index;</div><div class='del'>-	bool use_pf_rid;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_reg_ns_stag_info {</div><div class='del'>-	u64 reg_addr_pa;</div><div class='del'>-	u64 fbo;</div><div class='del'>-	void *va;</div><div class='del'>-	u64 total_len;</div><div class='del'>-	u32 page_size;</div><div class='del'>-	u32 chunk_size;</div><div class='del'>-	u32 first_pm_pbl_index;</div><div class='del'>-	enum i40iw_addressing_type addr_type;</div><div class='del'>-	i40iw_stag_index stag_idx;</div><div class='del'>-	u16 access_rights;</div><div class='del'>-	u16 pd_id;</div><div class='del'>-	i40iw_stag_key stag_key;</div><div class='del'>-	bool use_hmc_fcn_index;</div><div class='del'>-	u8 hmc_fcn_index;</div><div class='del'>-	bool use_pf_rid;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_fast_reg_stag_info {</div><div class='del'>-	u64 wr_id;</div><div class='del'>-	u64 reg_addr_pa;</div><div class='del'>-	u64 fbo;</div><div class='del'>-	void *va;</div><div class='del'>-	u64 total_len;</div><div class='del'>-	u32 page_size;</div><div class='del'>-	u32 chunk_size;</div><div class='del'>-	u32 first_pm_pbl_index;</div><div class='del'>-	enum i40iw_addressing_type addr_type;</div><div class='del'>-	i40iw_stag_index stag_idx;</div><div class='del'>-	u16 access_rights;</div><div class='del'>-	u16 pd_id;</div><div class='del'>-	i40iw_stag_key stag_key;</div><div class='del'>-	bool local_fence;</div><div class='del'>-	bool read_fence;</div><div class='del'>-	bool signaled;</div><div class='del'>-	bool use_hmc_fcn_index;</div><div class='del'>-	u8 hmc_fcn_index;</div><div class='del'>-	bool use_pf_rid;</div><div class='del'>-	bool defer_flag;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_dealloc_stag_info {</div><div class='del'>-	u32 stag_idx;</div><div class='del'>-	u16 pd_id;</div><div class='del'>-	bool mr;</div><div class='del'>-	bool dealloc_pbl;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_register_shared_stag {</div><div class='del'>-	void *va;</div><div class='del'>-	enum i40iw_addressing_type addr_type;</div><div class='del'>-	i40iw_stag_index new_stag_idx;</div><div class='del'>-	i40iw_stag_index parent_stag_idx;</div><div class='del'>-	u32 access_rights;</div><div class='del'>-	u16 pd_id;</div><div class='del'>-	i40iw_stag_key new_stag_key;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_qp_init_info {</div><div class='del'>-	struct i40iw_qp_uk_init_info qp_uk_init_info;</div><div class='del'>-	struct i40iw_sc_pd *pd;</div><div class='del'>-	struct i40iw_sc_vsi *vsi;</div><div class='del'>-	u64 *host_ctx;</div><div class='del'>-	u8 *q2;</div><div class='del'>-	u64 sq_pa;</div><div class='del'>-	u64 rq_pa;</div><div class='del'>-	u64 host_ctx_pa;</div><div class='del'>-	u64 q2_pa;</div><div class='del'>-	u64 shadow_area_pa;</div><div class='del'>-	int abi_ver;</div><div class='del'>-	u8 sq_tph_val;</div><div class='del'>-	u8 rq_tph_val;</div><div class='del'>-	u8 type;</div><div class='del'>-	bool sq_tph_en;</div><div class='del'>-	bool rq_tph_en;</div><div class='del'>-	bool rcv_tph_en;</div><div class='del'>-	bool xmit_tph_en;</div><div class='del'>-	bool virtual_map;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cq_init_info {</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	u64 cq_base_pa;</div><div class='del'>-	u64 shadow_area_pa;</div><div class='del'>-	u32 ceq_id;</div><div class='del'>-	u32 shadow_read_threshold;</div><div class='del'>-	bool virtual_map;</div><div class='del'>-	bool ceqe_mask;</div><div class='del'>-	u8 pbl_chunk_size;</div><div class='del'>-	u32 first_pm_pbl_idx;</div><div class='del'>-	bool ceq_id_valid;</div><div class='del'>-	bool tph_en;</div><div class='del'>-	u8 tph_val;</div><div class='del'>-	u8 type;</div><div class='del'>-	struct i40iw_cq_uk_init_info cq_uk_init_info;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_upload_context_info {</div><div class='del'>-	u64 buf_pa;</div><div class='del'>-	bool freeze_qp;</div><div class='del'>-	bool raw_format;</div><div class='del'>-	u32 qp_id;</div><div class='del'>-	u8 qp_type;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_add_arp_cache_entry_info {</div><div class='del'>-	u8 mac_addr[6];</div><div class='del'>-	u32 reach_max;</div><div class='del'>-	u16 arp_index;</div><div class='del'>-	bool permanent;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_apbvt_info {</div><div class='del'>-	u16 port;</div><div class='del'>-	bool add;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_quad_entry_type {</div><div class='del'>-	I40IW_QHASH_TYPE_TCP_ESTABLISHED = 1,</div><div class='del'>-	I40IW_QHASH_TYPE_TCP_SYN,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_quad_hash_manage_type {</div><div class='del'>-	I40IW_QHASH_MANAGE_TYPE_DELETE = 0,</div><div class='del'>-	I40IW_QHASH_MANAGE_TYPE_ADD,</div><div class='del'>-	I40IW_QHASH_MANAGE_TYPE_MODIFY</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_qhash_table_info {</div><div class='del'>-	struct i40iw_sc_vsi *vsi;</div><div class='del'>-	enum i40iw_quad_hash_manage_type manage;</div><div class='del'>-	enum i40iw_quad_entry_type entry_type;</div><div class='del'>-	bool vlan_valid;</div><div class='del'>-	bool ipv4_valid;</div><div class='del'>-	u8 mac_addr[6];</div><div class='del'>-	u16 vlan_id;</div><div class='del'>-	u8 user_pri;</div><div class='del'>-	u32 qp_num;</div><div class='del'>-	u32 dest_ip[4];</div><div class='del'>-	u32 src_ip[4];</div><div class='del'>-	u16 dest_port;</div><div class='del'>-	u16 src_port;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_local_mac_ipaddr_entry_info {</div><div class='del'>-	u8 mac_addr[6];</div><div class='del'>-	u8 entry_idx;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cqp_manage_push_page_info {</div><div class='del'>-	u32 push_idx;</div><div class='del'>-	u16 qs_handle;</div><div class='del'>-	u8 free_page;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_qp_flush_info {</div><div class='del'>-	u16 sq_minor_code;</div><div class='del'>-	u16 sq_major_code;</div><div class='del'>-	u16 rq_minor_code;</div><div class='del'>-	u16 rq_major_code;</div><div class='del'>-	u16 ae_code;</div><div class='del'>-	u8 ae_source;</div><div class='del'>-	bool sq;</div><div class='del'>-	bool rq;</div><div class='del'>-	bool userflushcode;</div><div class='del'>-	bool generate_ae;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cqp_commit_fpm_values {</div><div class='del'>-	u64 qp_base;</div><div class='del'>-	u64 cq_base;</div><div class='del'>-	u32 hte_base;</div><div class='del'>-	u32 arp_base;</div><div class='del'>-	u32 apbvt_inuse_base;</div><div class='del'>-	u32 mr_base;</div><div class='del'>-	u32 xf_base;</div><div class='del'>-	u32 xffl_base;</div><div class='del'>-	u32 q1_base;</div><div class='del'>-	u32 q1fl_base;</div><div class='del'>-	u32 fsimc_base;</div><div class='del'>-	u32 fsiav_base;</div><div class='del'>-	u32 pbl_base;</div><div class='del'>-</div><div class='del'>-	u32 qp_cnt;</div><div class='del'>-	u32 cq_cnt;</div><div class='del'>-	u32 hte_cnt;</div><div class='del'>-	u32 arp_cnt;</div><div class='del'>-	u32 mr_cnt;</div><div class='del'>-	u32 xf_cnt;</div><div class='del'>-	u32 xffl_cnt;</div><div class='del'>-	u32 q1_cnt;</div><div class='del'>-	u32 q1fl_cnt;</div><div class='del'>-	u32 fsimc_cnt;</div><div class='del'>-	u32 fsiav_cnt;</div><div class='del'>-	u32 pbl_cnt;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cqp_query_fpm_values {</div><div class='del'>-	u16 first_pe_sd_index;</div><div class='del'>-	u32 qp_objsize;</div><div class='del'>-	u32 cq_objsize;</div><div class='del'>-	u32 hte_objsize;</div><div class='del'>-	u32 arp_objsize;</div><div class='del'>-	u32 mr_objsize;</div><div class='del'>-	u32 xf_objsize;</div><div class='del'>-	u32 q1_objsize;</div><div class='del'>-	u32 fsimc_objsize;</div><div class='del'>-	u32 fsiav_objsize;</div><div class='del'>-</div><div class='del'>-	u32 qp_max;</div><div class='del'>-	u32 cq_max;</div><div class='del'>-	u32 hte_max;</div><div class='del'>-	u32 arp_max;</div><div class='del'>-	u32 mr_max;</div><div class='del'>-	u32 xf_max;</div><div class='del'>-	u32 xffl_max;</div><div class='del'>-	u32 q1_max;</div><div class='del'>-	u32 q1fl_max;</div><div class='del'>-	u32 fsimc_max;</div><div class='del'>-	u32 fsiav_max;</div><div class='del'>-	u32 pbl_max;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_gen_ae_info {</div><div class='del'>-	u16 ae_code;</div><div class='del'>-	u8 ae_source;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cqp_ops {</div><div class='del'>-	enum i40iw_status_code (*cqp_init)(struct i40iw_sc_cqp *,</div><div class='del'>-					   struct i40iw_cqp_init_info *);</div><div class='del'>-	enum i40iw_status_code (*cqp_create)(struct i40iw_sc_cqp *, u16 *, u16 *);</div><div class='del'>-	void (*cqp_post_sq)(struct i40iw_sc_cqp *);</div><div class='del'>-	u64 *(*cqp_get_next_send_wqe)(struct i40iw_sc_cqp *, u64 scratch);</div><div class='del'>-	enum i40iw_status_code (*cqp_destroy)(struct i40iw_sc_cqp *);</div><div class='del'>-	enum i40iw_status_code (*poll_for_cqp_op_done)(struct i40iw_sc_cqp *, u8,</div><div class='del'>-						       struct i40iw_ccq_cqe_info *);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_ccq_ops {</div><div class='del'>-	enum i40iw_status_code (*ccq_init)(struct i40iw_sc_cq *,</div><div class='del'>-					   struct i40iw_ccq_init_info *);</div><div class='del'>-	enum i40iw_status_code (*ccq_create)(struct i40iw_sc_cq *, u64, bool, bool);</div><div class='del'>-	enum i40iw_status_code (*ccq_destroy)(struct i40iw_sc_cq *, u64, bool);</div><div class='del'>-	enum i40iw_status_code (*ccq_create_done)(struct i40iw_sc_cq *);</div><div class='del'>-	enum i40iw_status_code (*ccq_get_cqe_info)(struct i40iw_sc_cq *,</div><div class='del'>-						   struct i40iw_ccq_cqe_info *);</div><div class='del'>-	void (*ccq_arm)(struct i40iw_sc_cq *);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_ceq_ops {</div><div class='del'>-	enum i40iw_status_code (*ceq_init)(struct i40iw_sc_ceq *,</div><div class='del'>-					   struct i40iw_ceq_init_info *);</div><div class='del'>-	enum i40iw_status_code (*ceq_create)(struct i40iw_sc_ceq *, u64, bool);</div><div class='del'>-	enum i40iw_status_code (*cceq_create_done)(struct i40iw_sc_ceq *);</div><div class='del'>-	enum i40iw_status_code (*cceq_destroy_done)(struct i40iw_sc_ceq *);</div><div class='del'>-	enum i40iw_status_code (*cceq_create)(struct i40iw_sc_ceq *, u64);</div><div class='del'>-	enum i40iw_status_code (*ceq_destroy)(struct i40iw_sc_ceq *, u64, bool);</div><div class='del'>-	void *(*process_ceq)(struct i40iw_sc_dev *, struct i40iw_sc_ceq *);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_aeq_ops {</div><div class='del'>-	enum i40iw_status_code (*aeq_init)(struct i40iw_sc_aeq *,</div><div class='del'>-					   struct i40iw_aeq_init_info *);</div><div class='del'>-	enum i40iw_status_code (*aeq_create)(struct i40iw_sc_aeq *, u64, bool);</div><div class='del'>-	enum i40iw_status_code (*aeq_destroy)(struct i40iw_sc_aeq *, u64, bool);</div><div class='del'>-	enum i40iw_status_code (*get_next_aeqe)(struct i40iw_sc_aeq *,</div><div class='del'>-						struct i40iw_aeqe_info *);</div><div class='del'>-	enum i40iw_status_code (*repost_aeq_entries)(struct i40iw_sc_dev *, u32);</div><div class='del'>-	enum i40iw_status_code (*aeq_create_done)(struct i40iw_sc_aeq *);</div><div class='del'>-	enum i40iw_status_code (*aeq_destroy_done)(struct i40iw_sc_aeq *);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_pd_ops {</div><div class='del'>-	void (*pd_init)(struct i40iw_sc_dev *, struct i40iw_sc_pd *, u16, int);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_priv_qp_ops {</div><div class='del'>-	enum i40iw_status_code (*qp_init)(struct i40iw_sc_qp *, struct i40iw_qp_init_info *);</div><div class='del'>-	enum i40iw_status_code (*qp_create)(struct i40iw_sc_qp *,</div><div class='del'>-					    struct i40iw_create_qp_info *, u64, bool);</div><div class='del'>-	enum i40iw_status_code (*qp_modify)(struct i40iw_sc_qp *,</div><div class='del'>-					    struct i40iw_modify_qp_info *, u64, bool);</div><div class='del'>-	enum i40iw_status_code (*qp_destroy)(struct i40iw_sc_qp *, u64, bool, bool, bool);</div><div class='del'>-	enum i40iw_status_code (*qp_flush_wqes)(struct i40iw_sc_qp *,</div><div class='del'>-						struct i40iw_qp_flush_info *, u64, bool);</div><div class='del'>-	enum i40iw_status_code (*qp_upload_context)(struct i40iw_sc_dev *,</div><div class='del'>-						    struct i40iw_upload_context_info *,</div><div class='del'>-						    u64, bool);</div><div class='del'>-	enum i40iw_status_code (*qp_setctx)(struct i40iw_sc_qp *, u64 *,</div><div class='del'>-					    struct i40iw_qp_host_ctx_info *);</div><div class='del'>-</div><div class='del'>-	void (*qp_send_lsmm)(struct i40iw_sc_qp *, void *, u32, i40iw_stag);</div><div class='del'>-	void (*qp_send_lsmm_nostag)(struct i40iw_sc_qp *, void *, u32);</div><div class='del'>-	void (*qp_send_rtt)(struct i40iw_sc_qp *, bool);</div><div class='del'>-	enum i40iw_status_code (*qp_post_wqe0)(struct i40iw_sc_qp *, u8);</div><div class='del'>-	enum i40iw_status_code (*iw_mr_fast_register)(struct i40iw_sc_qp *,</div><div class='del'>-						      struct i40iw_fast_reg_stag_info *,</div><div class='del'>-						      bool);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_priv_cq_ops {</div><div class='del'>-	enum i40iw_status_code (*cq_init)(struct i40iw_sc_cq *, struct i40iw_cq_init_info *);</div><div class='del'>-	enum i40iw_status_code (*cq_create)(struct i40iw_sc_cq *, u64, bool, bool);</div><div class='del'>-	enum i40iw_status_code (*cq_destroy)(struct i40iw_sc_cq *, u64, bool);</div><div class='del'>-	enum i40iw_status_code (*cq_modify)(struct i40iw_sc_cq *,</div><div class='del'>-					    struct i40iw_modify_cq_info *, u64, bool);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_mr_ops {</div><div class='del'>-	enum i40iw_status_code (*alloc_stag)(struct i40iw_sc_dev *,</div><div class='del'>-					     struct i40iw_allocate_stag_info *, u64, bool);</div><div class='del'>-	enum i40iw_status_code (*mr_reg_non_shared)(struct i40iw_sc_dev *,</div><div class='del'>-						    struct i40iw_reg_ns_stag_info *,</div><div class='del'>-						    u64, bool);</div><div class='del'>-	enum i40iw_status_code (*mr_reg_shared)(struct i40iw_sc_dev *,</div><div class='del'>-						struct i40iw_register_shared_stag *,</div><div class='del'>-						u64, bool);</div><div class='del'>-	enum i40iw_status_code (*dealloc_stag)(struct i40iw_sc_dev *,</div><div class='del'>-					       struct i40iw_dealloc_stag_info *,</div><div class='del'>-					       u64, bool);</div><div class='del'>-	enum i40iw_status_code (*query_stag)(struct i40iw_sc_dev *, u64, u32, bool);</div><div class='del'>-	enum i40iw_status_code (*mw_alloc)(struct i40iw_sc_dev *, u64, u32, u16, bool);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cqp_misc_ops {</div><div class='del'>-	enum i40iw_status_code (*manage_push_page)(struct i40iw_sc_cqp *,</div><div class='del'>-						   struct i40iw_cqp_manage_push_page_info *,</div><div class='del'>-						   u64, bool);</div><div class='del'>-	enum i40iw_status_code (*manage_hmc_pm_func_table)(struct i40iw_sc_cqp *,</div><div class='del'>-							   u64, u8, bool, bool);</div><div class='del'>-	enum i40iw_status_code (*set_hmc_resource_profile)(struct i40iw_sc_cqp *,</div><div class='del'>-							   u64, u8, u8, bool, bool);</div><div class='del'>-	enum i40iw_status_code (*commit_fpm_values)(struct i40iw_sc_cqp *, u64, u8,</div><div class='del'>-						    struct i40iw_dma_mem *, bool, u8);</div><div class='del'>-	enum i40iw_status_code (*query_fpm_values)(struct i40iw_sc_cqp *, u64, u8,</div><div class='del'>-						   struct i40iw_dma_mem *, bool, u8);</div><div class='del'>-	enum i40iw_status_code (*static_hmc_pages_allocated)(struct i40iw_sc_cqp *,</div><div class='del'>-							     u64, u8, bool, bool);</div><div class='del'>-	enum i40iw_status_code (*add_arp_cache_entry)(struct i40iw_sc_cqp *,</div><div class='del'>-						      struct i40iw_add_arp_cache_entry_info *,</div><div class='del'>-						      u64, bool);</div><div class='del'>-	enum i40iw_status_code (*del_arp_cache_entry)(struct i40iw_sc_cqp *, u64, u16, bool);</div><div class='del'>-	enum i40iw_status_code (*query_arp_cache_entry)(struct i40iw_sc_cqp *, u64, u16, bool);</div><div class='del'>-	enum i40iw_status_code (*manage_apbvt_entry)(struct i40iw_sc_cqp *,</div><div class='del'>-						     struct i40iw_apbvt_info *, u64, bool);</div><div class='del'>-	enum i40iw_status_code (*manage_qhash_table_entry)(struct i40iw_sc_cqp *,</div><div class='del'>-							   struct i40iw_qhash_table_info *, u64, bool);</div><div class='del'>-	enum i40iw_status_code (*alloc_local_mac_ipaddr_table_entry)(struct i40iw_sc_cqp *, u64, bool);</div><div class='del'>-	enum i40iw_status_code (*add_local_mac_ipaddr_entry)(struct i40iw_sc_cqp *,</div><div class='del'>-							     struct i40iw_local_mac_ipaddr_entry_info *,</div><div class='del'>-							     u64, bool);</div><div class='del'>-	enum i40iw_status_code (*del_local_mac_ipaddr_entry)(struct i40iw_sc_cqp *, u64, u8, u8, bool);</div><div class='del'>-	enum i40iw_status_code (*cqp_nop)(struct i40iw_sc_cqp *, u64, bool);</div><div class='del'>-	enum i40iw_status_code (*commit_fpm_values_done)(struct i40iw_sc_cqp</div><div class='del'>-							  *);</div><div class='del'>-	enum i40iw_status_code (*query_fpm_values_done)(struct i40iw_sc_cqp *);</div><div class='del'>-	enum i40iw_status_code (*manage_hmc_pm_func_table_done)(struct i40iw_sc_cqp *);</div><div class='del'>-	enum i40iw_status_code (*update_suspend_qp)(struct i40iw_sc_cqp *, struct i40iw_sc_qp *, u64);</div><div class='del'>-	enum i40iw_status_code (*update_resume_qp)(struct i40iw_sc_cqp *, struct i40iw_sc_qp *, u64);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_hmc_ops {</div><div class='del'>-	enum i40iw_status_code (*init_iw_hmc)(struct i40iw_sc_dev *, u8);</div><div class='del'>-	enum i40iw_status_code (*parse_fpm_query_buf)(u64 *, struct i40iw_hmc_info *,</div><div class='del'>-						      struct i40iw_hmc_fpm_misc *);</div><div class='del'>-	enum i40iw_status_code (*configure_iw_fpm)(struct i40iw_sc_dev *, u8);</div><div class='del'>-	enum i40iw_status_code (*parse_fpm_commit_buf)(u64 *, struct i40iw_hmc_obj_info *, u32 *sd);</div><div class='del'>-	enum i40iw_status_code (*create_hmc_object)(struct i40iw_sc_dev *dev,</div><div class='del'>-						    struct i40iw_hmc_create_obj_info *);</div><div class='del'>-	enum i40iw_status_code (*del_hmc_object)(struct i40iw_sc_dev *dev,</div><div class='del'>-						 struct i40iw_hmc_del_obj_info *,</div><div class='del'>-						 bool reset);</div><div class='del'>-	enum i40iw_status_code (*pf_init_vfhmc)(struct i40iw_sc_dev *, u8, u32 *);</div><div class='del'>-	enum i40iw_status_code (*vf_configure_vffpm)(struct i40iw_sc_dev *, u32 *);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct cqp_info {</div><div class='del'>-	union {</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_qp *qp;</div><div class='del'>-			struct i40iw_create_qp_info info;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} qp_create;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_qp *qp;</div><div class='del'>-			struct i40iw_modify_qp_info info;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} qp_modify;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_qp *qp;</div><div class='del'>-			u64 scratch;</div><div class='del'>-			bool remove_hash_idx;</div><div class='del'>-			bool ignore_mw_bnd;</div><div class='del'>-		} qp_destroy;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_cq *cq;</div><div class='del'>-			u64 scratch;</div><div class='del'>-			bool check_overflow;</div><div class='del'>-		} cq_create;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_cq *cq;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} cq_destroy;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_dev *dev;</div><div class='del'>-			struct i40iw_allocate_stag_info info;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} alloc_stag;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_dev *dev;</div><div class='del'>-			u64 scratch;</div><div class='del'>-			u32 mw_stag_index;</div><div class='del'>-			u16 pd_id;</div><div class='del'>-		} mw_alloc;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_dev *dev;</div><div class='del'>-			struct i40iw_reg_ns_stag_info info;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} mr_reg_non_shared;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_dev *dev;</div><div class='del'>-			struct i40iw_dealloc_stag_info info;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} dealloc_stag;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_cqp *cqp;</div><div class='del'>-			struct i40iw_local_mac_ipaddr_entry_info info;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} add_local_mac_ipaddr_entry;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_cqp *cqp;</div><div class='del'>-			struct i40iw_add_arp_cache_entry_info info;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} add_arp_cache_entry;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_cqp *cqp;</div><div class='del'>-			u64 scratch;</div><div class='del'>-			u8 entry_idx;</div><div class='del'>-			u8 ignore_ref_count;</div><div class='del'>-		} del_local_mac_ipaddr_entry;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_cqp *cqp;</div><div class='del'>-			u64 scratch;</div><div class='del'>-			u16 arp_index;</div><div class='del'>-		} del_arp_cache_entry;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_cqp *cqp;</div><div class='del'>-			struct i40iw_manage_vf_pble_info info;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} manage_vf_pble_bp;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_cqp *cqp;</div><div class='del'>-			struct i40iw_cqp_manage_push_page_info info;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} manage_push_page;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_dev *dev;</div><div class='del'>-			struct i40iw_upload_context_info info;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} qp_upload_context;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_cqp *cqp;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} alloc_local_mac_ipaddr_entry;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_dev *dev;</div><div class='del'>-			struct i40iw_hmc_fcn_info info;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} manage_hmc_pm;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_ceq *ceq;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} ceq_create;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_ceq *ceq;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} ceq_destroy;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_aeq *aeq;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} aeq_create;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_aeq *aeq;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} aeq_destroy;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_qp *qp;</div><div class='del'>-			struct i40iw_qp_flush_info info;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} qp_flush_wqes;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_qp *qp;</div><div class='del'>-			struct i40iw_gen_ae_info info;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} gen_ae;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_cqp *cqp;</div><div class='del'>-			void *fpm_values_va;</div><div class='del'>-			u64 fpm_values_pa;</div><div class='del'>-			u8 hmc_fn_id;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} query_fpm_values;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_cqp *cqp;</div><div class='del'>-			void *fpm_values_va;</div><div class='del'>-			u64 fpm_values_pa;</div><div class='del'>-			u8 hmc_fn_id;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} commit_fpm_values;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_cqp *cqp;</div><div class='del'>-			struct i40iw_apbvt_info info;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} manage_apbvt_entry;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_cqp *cqp;</div><div class='del'>-			struct i40iw_qhash_table_info info;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} manage_qhash_table_entry;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_dev *dev;</div><div class='del'>-			struct i40iw_update_sds_info info;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} update_pe_sds;</div><div class='del'>-</div><div class='del'>-		struct {</div><div class='del'>-			struct i40iw_sc_cqp *cqp;</div><div class='del'>-			struct i40iw_sc_qp *qp;</div><div class='del'>-			u64 scratch;</div><div class='del'>-		} suspend_resume;</div><div class='del'>-	} u;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct cqp_commands_info {</div><div class='del'>-	struct list_head cqp_cmd_entry;</div><div class='del'>-	u8 cqp_cmd;</div><div class='del'>-	u8 post_sq;</div><div class='del'>-	struct cqp_info in;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_virtchnl_work_info {</div><div class='del'>-	void (*callback_fcn)(void *vf_dev);</div><div class='del'>-	void *worker_vf_dev;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cqp_timeout {</div><div class='del'>-	u64 compl_cqp_cmds;</div><div class='del'>-	u8 count;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_uk.c b/drivers/infiniband/hw/i40iw/i40iw_uk.c<br/>deleted file mode 100644<br/>index 8afa5a67a86b..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_uk.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_uk.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,1232 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#include "i40iw_osdep.h"</div><div class='del'>-#include "i40iw_status.h"</div><div class='del'>-#include "i40iw_d.h"</div><div class='del'>-#include "i40iw_user.h"</div><div class='del'>-#include "i40iw_register.h"</div><div class='del'>-</div><div class='del'>-static u32 nop_signature = 0x55550000;</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_nop_1 - insert a nop wqe and move head. no post work</div><div class='del'>- * @qp: hw qp ptr</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_nop_1(struct i40iw_qp_uk *qp)</div><div class='del'>-{</div><div class='del'>-	u64 header, *wqe;</div><div class='del'>-	u64 *wqe_0 = NULL;</div><div class='del'>-	u32 wqe_idx, peek_head;</div><div class='del'>-	bool signaled = false;</div><div class='del'>-</div><div class='del'>-	if (!qp-&gt;sq_ring.head)</div><div class='del'>-		return I40IW_ERR_PARAM;</div><div class='del'>-</div><div class='del'>-	wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp-&gt;sq_ring);</div><div class='del'>-	wqe = qp-&gt;sq_base[wqe_idx].elem;</div><div class='del'>-</div><div class='del'>-	qp-&gt;sq_wrtrk_array[wqe_idx].wqe_size = I40IW_QP_WQE_MIN_SIZE;</div><div class='del'>-</div><div class='del'>-	peek_head = (qp-&gt;sq_ring.head + 1) % qp-&gt;sq_ring.size;</div><div class='del'>-	wqe_0 = qp-&gt;sq_base[peek_head].elem;</div><div class='del'>-	if (peek_head)</div><div class='del'>-		wqe_0[3] = LS_64(!qp-&gt;swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-	else</div><div class='del'>-		wqe_0[3] = LS_64(qp-&gt;swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 0, 0);</div><div class='del'>-	set_64bit_val(wqe, 8, 0);</div><div class='del'>-	set_64bit_val(wqe, 16, 0);</div><div class='del'>-</div><div class='del'>-	header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) |</div><div class='del'>-	    LS_64(signaled, I40IWQPSQ_SIGCOMPL) |</div><div class='del'>-	    LS_64(qp-&gt;swqe_polarity, I40IWQPSQ_VALID) | nop_signature++;</div><div class='del'>-</div><div class='del'>-	wmb();	/* Memory barrier to ensure data is written before valid bit is set */</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 24, header);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_qp_post_wr - post wr to hrdware</div><div class='del'>- * @qp: hw qp ptr</div><div class='del'>- */</div><div class='del'>-void i40iw_qp_post_wr(struct i40iw_qp_uk *qp)</div><div class='del'>-{</div><div class='del'>-	u64 temp;</div><div class='del'>-	u32 hw_sq_tail;</div><div class='del'>-	u32 sw_sq_head;</div><div class='del'>-</div><div class='del'>-	mb(); /* valid bit is written and loads completed before reading shadow */</div><div class='del'>-</div><div class='del'>-	/* read the doorbell shadow area */</div><div class='del'>-	get_64bit_val(qp-&gt;shadow_area, 0, &amp;temp);</div><div class='del'>-</div><div class='del'>-	hw_sq_tail = (u32)RS_64(temp, I40IW_QP_DBSA_HW_SQ_TAIL);</div><div class='del'>-	sw_sq_head = I40IW_RING_GETCURRENT_HEAD(qp-&gt;sq_ring);</div><div class='del'>-	if (sw_sq_head != hw_sq_tail) {</div><div class='del'>-		if (sw_sq_head &gt; qp-&gt;initial_ring.head) {</div><div class='del'>-			if ((hw_sq_tail &gt;= qp-&gt;initial_ring.head) &amp;&amp;</div><div class='del'>-			    (hw_sq_tail &lt; sw_sq_head)) {</div><div class='del'>-				writel(qp-&gt;qp_id, qp-&gt;wqe_alloc_reg);</div><div class='del'>-			}</div><div class='del'>-		} else if (sw_sq_head != qp-&gt;initial_ring.head) {</div><div class='del'>-			if ((hw_sq_tail &gt;= qp-&gt;initial_ring.head) ||</div><div class='del'>-			    (hw_sq_tail &lt; sw_sq_head)) {</div><div class='del'>-				writel(qp-&gt;qp_id, qp-&gt;wqe_alloc_reg);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	qp-&gt;initial_ring.head = qp-&gt;sq_ring.head;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_qp_ring_push_db -  ring qp doorbell</div><div class='del'>- * @qp: hw qp ptr</div><div class='del'>- * @wqe_idx: wqe index</div><div class='del'>- */</div><div class='del'>-static void i40iw_qp_ring_push_db(struct i40iw_qp_uk *qp, u32 wqe_idx)</div><div class='del'>-{</div><div class='del'>-	set_32bit_val(qp-&gt;push_db, 0, LS_32((wqe_idx &gt;&gt; 2), I40E_PFPE_WQEALLOC_WQE_DESC_INDEX) | qp-&gt;qp_id);</div><div class='del'>-	qp-&gt;initial_ring.head = I40IW_RING_GETCURRENT_HEAD(qp-&gt;sq_ring);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_qp_get_next_send_wqe - return next wqe ptr</div><div class='del'>- * @qp: hw qp ptr</div><div class='del'>- * @wqe_idx: return wqe index</div><div class='del'>- * @wqe_size: size of sq wqe</div><div class='del'>- */</div><div class='del'>-u64 *i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp,</div><div class='del'>-				u32 *wqe_idx,</div><div class='del'>-				u8 wqe_size,</div><div class='del'>-				u32 total_size,</div><div class='del'>-				u64 wr_id</div><div class='del'>-				)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe = NULL;</div><div class='del'>-	u64 wqe_ptr;</div><div class='del'>-	u32 peek_head = 0;</div><div class='del'>-	u16 offset;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-	u8 nop_wqe_cnt = 0, i;</div><div class='del'>-	u64 *wqe_0 = NULL;</div><div class='del'>-</div><div class='del'>-	*wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp-&gt;sq_ring);</div><div class='del'>-</div><div class='del'>-	if (!*wqe_idx)</div><div class='del'>-		qp-&gt;swqe_polarity = !qp-&gt;swqe_polarity;</div><div class='del'>-	wqe_ptr = (uintptr_t)qp-&gt;sq_base[*wqe_idx].elem;</div><div class='del'>-	offset = (u16)(wqe_ptr) &amp; 0x7F;</div><div class='del'>-	if ((offset + wqe_size) &gt; I40IW_QP_WQE_MAX_SIZE) {</div><div class='del'>-		nop_wqe_cnt = (u8)(I40IW_QP_WQE_MAX_SIZE - offset) / I40IW_QP_WQE_MIN_SIZE;</div><div class='del'>-		for (i = 0; i &lt; nop_wqe_cnt; i++) {</div><div class='del'>-			i40iw_nop_1(qp);</div><div class='del'>-			I40IW_RING_MOVE_HEAD(qp-&gt;sq_ring, ret_code);</div><div class='del'>-			if (ret_code)</div><div class='del'>-				return NULL;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		*wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp-&gt;sq_ring);</div><div class='del'>-		if (!*wqe_idx)</div><div class='del'>-			qp-&gt;swqe_polarity = !qp-&gt;swqe_polarity;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (((*wqe_idx &amp; 3) == 1) &amp;&amp; (wqe_size == I40IW_WQE_SIZE_64)) {</div><div class='del'>-		i40iw_nop_1(qp);</div><div class='del'>-		I40IW_RING_MOVE_HEAD(qp-&gt;sq_ring, ret_code);</div><div class='del'>-		if (ret_code)</div><div class='del'>-			return NULL;</div><div class='del'>-		*wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp-&gt;sq_ring);</div><div class='del'>-		if (!*wqe_idx)</div><div class='del'>-			qp-&gt;swqe_polarity = !qp-&gt;swqe_polarity;</div><div class='del'>-	}</div><div class='del'>-	I40IW_RING_MOVE_HEAD_BY_COUNT(qp-&gt;sq_ring,</div><div class='del'>-				      wqe_size / I40IW_QP_WQE_MIN_SIZE, ret_code);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	wqe = qp-&gt;sq_base[*wqe_idx].elem;</div><div class='del'>-</div><div class='del'>-	peek_head = I40IW_RING_GETCURRENT_HEAD(qp-&gt;sq_ring);</div><div class='del'>-	wqe_0 = qp-&gt;sq_base[peek_head].elem;</div><div class='del'>-</div><div class='del'>-	if (((peek_head &amp; 3) == 1) || ((peek_head &amp; 3) == 3)) {</div><div class='del'>-		if (RS_64(wqe_0[3], I40IWQPSQ_VALID) != !qp-&gt;swqe_polarity)</div><div class='del'>-			wqe_0[3] = LS_64(!qp-&gt;swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	qp-&gt;sq_wrtrk_array[*wqe_idx].wrid = wr_id;</div><div class='del'>-	qp-&gt;sq_wrtrk_array[*wqe_idx].wr_len = total_size;</div><div class='del'>-	qp-&gt;sq_wrtrk_array[*wqe_idx].wqe_size = wqe_size;</div><div class='del'>-	return wqe;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_set_fragment - set fragment in wqe</div><div class='del'>- * @wqe: wqe for setting fragment</div><div class='del'>- * @offset: offset value</div><div class='del'>- * @sge: sge length and stag</div><div class='del'>- */</div><div class='del'>-static void i40iw_set_fragment(u64 *wqe, u32 offset, struct i40iw_sge *sge)</div><div class='del'>-{</div><div class='del'>-	if (sge) {</div><div class='del'>-		set_64bit_val(wqe, offset, LS_64(sge-&gt;tag_off, I40IWQPSQ_FRAG_TO));</div><div class='del'>-		set_64bit_val(wqe, (offset + 8),</div><div class='del'>-			      (LS_64(sge-&gt;len, I40IWQPSQ_FRAG_LEN) |</div><div class='del'>-			       LS_64(sge-&gt;stag, I40IWQPSQ_FRAG_STAG)));</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_qp_get_next_recv_wqe - get next qp's rcv wqe</div><div class='del'>- * @qp: hw qp ptr</div><div class='del'>- * @wqe_idx: return wqe index</div><div class='del'>- */</div><div class='del'>-u64 *i40iw_qp_get_next_recv_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe = NULL;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-</div><div class='del'>-	if (I40IW_RING_FULL_ERR(qp-&gt;rq_ring))</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	I40IW_ATOMIC_RING_MOVE_HEAD(qp-&gt;rq_ring, *wqe_idx, ret_code);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return NULL;</div><div class='del'>-	if (!*wqe_idx)</div><div class='del'>-		qp-&gt;rwqe_polarity = !qp-&gt;rwqe_polarity;</div><div class='del'>-	/* rq_wqe_size_multiplier is no of qwords in one rq wqe */</div><div class='del'>-	wqe = qp-&gt;rq_base[*wqe_idx * (qp-&gt;rq_wqe_size_multiplier &gt;&gt; 2)].elem;</div><div class='del'>-</div><div class='del'>-	return wqe;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_rdma_write - rdma write operation</div><div class='del'>- * @qp: hw qp ptr</div><div class='del'>- * @info: post sq information</div><div class='del'>- * @post_sq: flag to post sq</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_rdma_write(struct i40iw_qp_uk *qp,</div><div class='del'>-					       struct i40iw_post_sq_info *info,</div><div class='del'>-					       bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 header;</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_rdma_write *op_info;</div><div class='del'>-	u32 i, wqe_idx;</div><div class='del'>-	u32 total_size = 0, byte_off;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-	bool read_fence = false;</div><div class='del'>-	u8 wqe_size;</div><div class='del'>-</div><div class='del'>-	op_info = &amp;info-&gt;op.rdma_write;</div><div class='del'>-	if (op_info-&gt;num_lo_sges &gt; qp-&gt;max_sq_frag_cnt)</div><div class='del'>-		return I40IW_ERR_INVALID_FRAG_COUNT;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; op_info-&gt;num_lo_sges; i++)</div><div class='del'>-		total_size += op_info-&gt;lo_sg_list[i].len;</div><div class='del'>-</div><div class='del'>-	if (total_size &gt; I40IW_MAX_OUTBOUND_MESSAGE_SIZE)</div><div class='del'>-		return I40IW_ERR_QP_INVALID_MSG_SIZE;</div><div class='del'>-</div><div class='del'>-	read_fence |= info-&gt;read_fence;</div><div class='del'>-</div><div class='del'>-	ret_code = i40iw_fragcnt_to_wqesize_sq(op_info-&gt;num_lo_sges, &amp;wqe_size);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return ret_code;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_qp_get_next_send_wqe(qp, &amp;wqe_idx, wqe_size, total_size, info-&gt;wr_id);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_QP_TOOMANY_WRS_POSTED;</div><div class='del'>-	set_64bit_val(wqe, 16,</div><div class='del'>-		      LS_64(op_info-&gt;rem_addr.tag_off, I40IWQPSQ_FRAG_TO));</div><div class='del'>-	if (!op_info-&gt;rem_addr.stag)</div><div class='del'>-		return I40IW_ERR_BAD_STAG;</div><div class='del'>-</div><div class='del'>-	header = LS_64(op_info-&gt;rem_addr.stag, I40IWQPSQ_REMSTAG) |</div><div class='del'>-		 LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) |</div><div class='del'>-		 LS_64((op_info-&gt;num_lo_sges &gt; 1 ?  (op_info-&gt;num_lo_sges - 1) : 0), I40IWQPSQ_ADDFRAGCNT) |</div><div class='del'>-		 LS_64(read_fence, I40IWQPSQ_READFENCE) |</div><div class='del'>-		 LS_64(info-&gt;local_fence, I40IWQPSQ_LOCALFENCE) |</div><div class='del'>-		 LS_64(info-&gt;signaled, I40IWQPSQ_SIGCOMPL) |</div><div class='del'>-		 LS_64(qp-&gt;swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-</div><div class='del'>-	i40iw_set_fragment(wqe, 0, op_info-&gt;lo_sg_list);</div><div class='del'>-</div><div class='del'>-	for (i = 1, byte_off = 32; i &lt; op_info-&gt;num_lo_sges; i++) {</div><div class='del'>-		i40iw_set_fragment(wqe, byte_off, &amp;op_info-&gt;lo_sg_list[i]);</div><div class='del'>-		byte_off += 16;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 24, header);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_qp_post_wr(qp);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_rdma_read - rdma read command</div><div class='del'>- * @qp: hw qp ptr</div><div class='del'>- * @info: post sq information</div><div class='del'>- * @inv_stag: flag for inv_stag</div><div class='del'>- * @post_sq: flag to post sq</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_rdma_read(struct i40iw_qp_uk *qp,</div><div class='del'>-					      struct i40iw_post_sq_info *info,</div><div class='del'>-					      bool inv_stag,</div><div class='del'>-					      bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_rdma_read *op_info;</div><div class='del'>-	u64 header;</div><div class='del'>-	u32 wqe_idx;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-	u8 wqe_size;</div><div class='del'>-	bool local_fence = false;</div><div class='del'>-</div><div class='del'>-	op_info = &amp;info-&gt;op.rdma_read;</div><div class='del'>-	ret_code = i40iw_fragcnt_to_wqesize_sq(1, &amp;wqe_size);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return ret_code;</div><div class='del'>-	wqe = i40iw_qp_get_next_send_wqe(qp, &amp;wqe_idx, wqe_size, op_info-&gt;lo_addr.len, info-&gt;wr_id);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_QP_TOOMANY_WRS_POSTED;</div><div class='del'>-	local_fence |= info-&gt;local_fence;</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 16, LS_64(op_info-&gt;rem_addr.tag_off, I40IWQPSQ_FRAG_TO));</div><div class='del'>-	header = LS_64(op_info-&gt;rem_addr.stag, I40IWQPSQ_REMSTAG) |</div><div class='del'>-		 LS_64((inv_stag ? I40IWQP_OP_RDMA_READ_LOC_INV : I40IWQP_OP_RDMA_READ), I40IWQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(info-&gt;read_fence, I40IWQPSQ_READFENCE) |</div><div class='del'>-		 LS_64(local_fence, I40IWQPSQ_LOCALFENCE) |</div><div class='del'>-		 LS_64(info-&gt;signaled, I40IWQPSQ_SIGCOMPL) |</div><div class='del'>-		 LS_64(qp-&gt;swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-</div><div class='del'>-	i40iw_set_fragment(wqe, 0, &amp;op_info-&gt;lo_addr);</div><div class='del'>-</div><div class='del'>-	wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 24, header);</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_qp_post_wr(qp);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_send - rdma send command</div><div class='del'>- * @qp: hw qp ptr</div><div class='del'>- * @info: post sq information</div><div class='del'>- * @stag_to_inv: stag_to_inv value</div><div class='del'>- * @post_sq: flag to post sq</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_send(struct i40iw_qp_uk *qp,</div><div class='del'>-					 struct i40iw_post_sq_info *info,</div><div class='del'>-					 u32 stag_to_inv,</div><div class='del'>-					 bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_post_send *op_info;</div><div class='del'>-	u64 header;</div><div class='del'>-	u32 i, wqe_idx, total_size = 0, byte_off;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-	bool read_fence = false;</div><div class='del'>-	u8 wqe_size;</div><div class='del'>-</div><div class='del'>-	op_info = &amp;info-&gt;op.send;</div><div class='del'>-	if (qp-&gt;max_sq_frag_cnt &lt; op_info-&gt;num_sges)</div><div class='del'>-		return I40IW_ERR_INVALID_FRAG_COUNT;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; op_info-&gt;num_sges; i++)</div><div class='del'>-		total_size += op_info-&gt;sg_list[i].len;</div><div class='del'>-	ret_code = i40iw_fragcnt_to_wqesize_sq(op_info-&gt;num_sges, &amp;wqe_size);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return ret_code;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_qp_get_next_send_wqe(qp, &amp;wqe_idx, wqe_size, total_size, info-&gt;wr_id);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_QP_TOOMANY_WRS_POSTED;</div><div class='del'>-</div><div class='del'>-	read_fence |= info-&gt;read_fence;</div><div class='del'>-	set_64bit_val(wqe, 16, 0);</div><div class='del'>-	header = LS_64(stag_to_inv, I40IWQPSQ_REMSTAG) |</div><div class='del'>-		 LS_64(info-&gt;op_type, I40IWQPSQ_OPCODE) |</div><div class='del'>-		 LS_64((op_info-&gt;num_sges &gt; 1 ? (op_info-&gt;num_sges - 1) : 0),</div><div class='del'>-		       I40IWQPSQ_ADDFRAGCNT) |</div><div class='del'>-		 LS_64(read_fence, I40IWQPSQ_READFENCE) |</div><div class='del'>-		 LS_64(info-&gt;local_fence, I40IWQPSQ_LOCALFENCE) |</div><div class='del'>-		 LS_64(info-&gt;signaled, I40IWQPSQ_SIGCOMPL) |</div><div class='del'>-		 LS_64(qp-&gt;swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-</div><div class='del'>-	i40iw_set_fragment(wqe, 0, op_info-&gt;sg_list);</div><div class='del'>-</div><div class='del'>-	for (i = 1, byte_off = 32; i &lt; op_info-&gt;num_sges; i++) {</div><div class='del'>-		i40iw_set_fragment(wqe, byte_off, &amp;op_info-&gt;sg_list[i]);</div><div class='del'>-		byte_off += 16;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 24, header);</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_qp_post_wr(qp);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_inline_rdma_write - inline rdma write operation</div><div class='del'>- * @qp: hw qp ptr</div><div class='del'>- * @info: post sq information</div><div class='del'>- * @post_sq: flag to post sq</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_inline_rdma_write(struct i40iw_qp_uk *qp,</div><div class='del'>-						      struct i40iw_post_sq_info *info,</div><div class='del'>-						      bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u8 *dest, *src;</div><div class='del'>-	struct i40iw_inline_rdma_write *op_info;</div><div class='del'>-	u64 *push;</div><div class='del'>-	u64 header = 0;</div><div class='del'>-	u32 wqe_idx;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-	bool read_fence = false;</div><div class='del'>-	u8 wqe_size;</div><div class='del'>-</div><div class='del'>-	op_info = &amp;info-&gt;op.inline_rdma_write;</div><div class='del'>-	if (op_info-&gt;len &gt; I40IW_MAX_INLINE_DATA_SIZE)</div><div class='del'>-		return I40IW_ERR_INVALID_INLINE_DATA_SIZE;</div><div class='del'>-</div><div class='del'>-	ret_code = i40iw_inline_data_size_to_wqesize(op_info-&gt;len, &amp;wqe_size);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return ret_code;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_qp_get_next_send_wqe(qp, &amp;wqe_idx, wqe_size, op_info-&gt;len, info-&gt;wr_id);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_QP_TOOMANY_WRS_POSTED;</div><div class='del'>-</div><div class='del'>-	read_fence |= info-&gt;read_fence;</div><div class='del'>-	set_64bit_val(wqe, 16,</div><div class='del'>-		      LS_64(op_info-&gt;rem_addr.tag_off, I40IWQPSQ_FRAG_TO));</div><div class='del'>-</div><div class='del'>-	header = LS_64(op_info-&gt;rem_addr.stag, I40IWQPSQ_REMSTAG) |</div><div class='del'>-		 LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) |</div><div class='del'>-		 LS_64(op_info-&gt;len, I40IWQPSQ_INLINEDATALEN) |</div><div class='del'>-		 LS_64(1, I40IWQPSQ_INLINEDATAFLAG) |</div><div class='del'>-		 LS_64((qp-&gt;push_db ? 1 : 0), I40IWQPSQ_PUSHWQE) |</div><div class='del'>-		 LS_64(read_fence, I40IWQPSQ_READFENCE) |</div><div class='del'>-		 LS_64(info-&gt;local_fence, I40IWQPSQ_LOCALFENCE) |</div><div class='del'>-		 LS_64(info-&gt;signaled, I40IWQPSQ_SIGCOMPL) |</div><div class='del'>-		 LS_64(qp-&gt;swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-</div><div class='del'>-	dest = (u8 *)wqe;</div><div class='del'>-	src = (u8 *)(op_info-&gt;data);</div><div class='del'>-</div><div class='del'>-	if (op_info-&gt;len &lt;= 16) {</div><div class='del'>-		memcpy(dest, src, op_info-&gt;len);</div><div class='del'>-	} else {</div><div class='del'>-		memcpy(dest, src, 16);</div><div class='del'>-		src += 16;</div><div class='del'>-		dest = (u8 *)wqe + 32;</div><div class='del'>-		memcpy(dest, src, op_info-&gt;len - 16);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 24, header);</div><div class='del'>-</div><div class='del'>-	if (qp-&gt;push_db) {</div><div class='del'>-		push = (u64 *)((uintptr_t)qp-&gt;push_wqe + (wqe_idx &amp; 0x3) * 0x20);</div><div class='del'>-		memcpy(push, wqe, (op_info-&gt;len &gt; 16) ? op_info-&gt;len + 16 : 32);</div><div class='del'>-		i40iw_qp_ring_push_db(qp, wqe_idx);</div><div class='del'>-	} else {</div><div class='del'>-		if (post_sq)</div><div class='del'>-			i40iw_qp_post_wr(qp);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_inline_send - inline send operation</div><div class='del'>- * @qp: hw qp ptr</div><div class='del'>- * @info: post sq information</div><div class='del'>- * @stag_to_inv: remote stag</div><div class='del'>- * @post_sq: flag to post sq</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_inline_send(struct i40iw_qp_uk *qp,</div><div class='del'>-						struct i40iw_post_sq_info *info,</div><div class='del'>-						u32 stag_to_inv,</div><div class='del'>-						bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u8 *dest, *src;</div><div class='del'>-	struct i40iw_post_inline_send *op_info;</div><div class='del'>-	u64 header;</div><div class='del'>-	u32 wqe_idx;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-	bool read_fence = false;</div><div class='del'>-	u8 wqe_size;</div><div class='del'>-	u64 *push;</div><div class='del'>-</div><div class='del'>-	op_info = &amp;info-&gt;op.inline_send;</div><div class='del'>-	if (op_info-&gt;len &gt; I40IW_MAX_INLINE_DATA_SIZE)</div><div class='del'>-		return I40IW_ERR_INVALID_INLINE_DATA_SIZE;</div><div class='del'>-</div><div class='del'>-	ret_code = i40iw_inline_data_size_to_wqesize(op_info-&gt;len, &amp;wqe_size);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return ret_code;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_qp_get_next_send_wqe(qp, &amp;wqe_idx, wqe_size, op_info-&gt;len, info-&gt;wr_id);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_QP_TOOMANY_WRS_POSTED;</div><div class='del'>-</div><div class='del'>-	read_fence |= info-&gt;read_fence;</div><div class='del'>-	header = LS_64(stag_to_inv, I40IWQPSQ_REMSTAG) |</div><div class='del'>-	    LS_64(info-&gt;op_type, I40IWQPSQ_OPCODE) |</div><div class='del'>-	    LS_64(op_info-&gt;len, I40IWQPSQ_INLINEDATALEN) |</div><div class='del'>-	    LS_64(1, I40IWQPSQ_INLINEDATAFLAG) |</div><div class='del'>-	    LS_64((qp-&gt;push_db ? 1 : 0), I40IWQPSQ_PUSHWQE) |</div><div class='del'>-	    LS_64(read_fence, I40IWQPSQ_READFENCE) |</div><div class='del'>-	    LS_64(info-&gt;local_fence, I40IWQPSQ_LOCALFENCE) |</div><div class='del'>-	    LS_64(info-&gt;signaled, I40IWQPSQ_SIGCOMPL) |</div><div class='del'>-	    LS_64(qp-&gt;swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-</div><div class='del'>-	dest = (u8 *)wqe;</div><div class='del'>-	src = (u8 *)(op_info-&gt;data);</div><div class='del'>-</div><div class='del'>-	if (op_info-&gt;len &lt;= 16) {</div><div class='del'>-		memcpy(dest, src, op_info-&gt;len);</div><div class='del'>-	} else {</div><div class='del'>-		memcpy(dest, src, 16);</div><div class='del'>-		src += 16;</div><div class='del'>-		dest = (u8 *)wqe + 32;</div><div class='del'>-		memcpy(dest, src, op_info-&gt;len - 16);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 24, header);</div><div class='del'>-</div><div class='del'>-	if (qp-&gt;push_db) {</div><div class='del'>-		push = (u64 *)((uintptr_t)qp-&gt;push_wqe + (wqe_idx &amp; 0x3) * 0x20);</div><div class='del'>-		memcpy(push, wqe, (op_info-&gt;len &gt; 16) ? op_info-&gt;len + 16 : 32);</div><div class='del'>-		i40iw_qp_ring_push_db(qp, wqe_idx);</div><div class='del'>-	} else {</div><div class='del'>-		if (post_sq)</div><div class='del'>-			i40iw_qp_post_wr(qp);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_stag_local_invalidate - stag invalidate operation</div><div class='del'>- * @qp: hw qp ptr</div><div class='del'>- * @info: post sq information</div><div class='del'>- * @post_sq: flag to post sq</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_stag_local_invalidate(struct i40iw_qp_uk *qp,</div><div class='del'>-							  struct i40iw_post_sq_info *info,</div><div class='del'>-							  bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_inv_local_stag *op_info;</div><div class='del'>-	u64 header;</div><div class='del'>-	u32 wqe_idx;</div><div class='del'>-	bool local_fence = false;</div><div class='del'>-</div><div class='del'>-	op_info = &amp;info-&gt;op.inv_local_stag;</div><div class='del'>-	local_fence = info-&gt;local_fence;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_qp_get_next_send_wqe(qp, &amp;wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, info-&gt;wr_id);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_QP_TOOMANY_WRS_POSTED;</div><div class='del'>-	set_64bit_val(wqe, 0, 0);</div><div class='del'>-	set_64bit_val(wqe, 8,</div><div class='del'>-		      LS_64(op_info-&gt;target_stag, I40IWQPSQ_LOCSTAG));</div><div class='del'>-	set_64bit_val(wqe, 16, 0);</div><div class='del'>-	header = LS_64(I40IW_OP_TYPE_INV_STAG, I40IWQPSQ_OPCODE) |</div><div class='del'>-	    LS_64(info-&gt;read_fence, I40IWQPSQ_READFENCE) |</div><div class='del'>-	    LS_64(local_fence, I40IWQPSQ_LOCALFENCE) |</div><div class='del'>-	    LS_64(info-&gt;signaled, I40IWQPSQ_SIGCOMPL) |</div><div class='del'>-	    LS_64(qp-&gt;swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-</div><div class='del'>-	wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 24, header);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_qp_post_wr(qp);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_mw_bind - Memory Window bind operation</div><div class='del'>- * @qp: hw qp ptr</div><div class='del'>- * @info: post sq information</div><div class='del'>- * @post_sq: flag to post sq</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_mw_bind(struct i40iw_qp_uk *qp,</div><div class='del'>-					    struct i40iw_post_sq_info *info,</div><div class='del'>-					    bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	struct i40iw_bind_window *op_info;</div><div class='del'>-	u64 header;</div><div class='del'>-	u32 wqe_idx;</div><div class='del'>-	bool local_fence = false;</div><div class='del'>-</div><div class='del'>-	op_info = &amp;info-&gt;op.bind_window;</div><div class='del'>-</div><div class='del'>-	local_fence |= info-&gt;local_fence;</div><div class='del'>-	wqe = i40iw_qp_get_next_send_wqe(qp, &amp;wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, info-&gt;wr_id);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_QP_TOOMANY_WRS_POSTED;</div><div class='del'>-	set_64bit_val(wqe, 0, (uintptr_t)op_info-&gt;va);</div><div class='del'>-	set_64bit_val(wqe, 8,</div><div class='del'>-		      LS_64(op_info-&gt;mr_stag, I40IWQPSQ_PARENTMRSTAG) |</div><div class='del'>-		      LS_64(op_info-&gt;mw_stag, I40IWQPSQ_MWSTAG));</div><div class='del'>-	set_64bit_val(wqe, 16, op_info-&gt;bind_length);</div><div class='del'>-	header = LS_64(I40IW_OP_TYPE_BIND_MW, I40IWQPSQ_OPCODE) |</div><div class='del'>-	    LS_64(((op_info-&gt;enable_reads &lt;&lt; 2) |</div><div class='del'>-		   (op_info-&gt;enable_writes &lt;&lt; 3)),</div><div class='del'>-		  I40IWQPSQ_STAGRIGHTS) |</div><div class='del'>-	    LS_64((op_info-&gt;addressing_type == I40IW_ADDR_TYPE_VA_BASED ?  1 : 0),</div><div class='del'>-		  I40IWQPSQ_VABASEDTO) |</div><div class='del'>-	    LS_64(info-&gt;read_fence, I40IWQPSQ_READFENCE) |</div><div class='del'>-	    LS_64(local_fence, I40IWQPSQ_LOCALFENCE) |</div><div class='del'>-	    LS_64(info-&gt;signaled, I40IWQPSQ_SIGCOMPL) |</div><div class='del'>-	    LS_64(qp-&gt;swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-</div><div class='del'>-	wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 24, header);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_qp_post_wr(qp);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_post_receive - post receive wqe</div><div class='del'>- * @qp: hw qp ptr</div><div class='del'>- * @info: post rq information</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_post_receive(struct i40iw_qp_uk *qp,</div><div class='del'>-						 struct i40iw_post_rq_info *info)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 header;</div><div class='del'>-	u32 total_size = 0, wqe_idx, i, byte_off;</div><div class='del'>-</div><div class='del'>-	if (qp-&gt;max_rq_frag_cnt &lt; info-&gt;num_sges)</div><div class='del'>-		return I40IW_ERR_INVALID_FRAG_COUNT;</div><div class='del'>-	for (i = 0; i &lt; info-&gt;num_sges; i++)</div><div class='del'>-		total_size += info-&gt;sg_list[i].len;</div><div class='del'>-	wqe = i40iw_qp_get_next_recv_wqe(qp, &amp;wqe_idx);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_QP_TOOMANY_WRS_POSTED;</div><div class='del'>-</div><div class='del'>-	qp-&gt;rq_wrid_array[wqe_idx] = info-&gt;wr_id;</div><div class='del'>-	set_64bit_val(wqe, 16, 0);</div><div class='del'>-</div><div class='del'>-	header = LS_64((info-&gt;num_sges &gt; 1 ? (info-&gt;num_sges - 1) : 0),</div><div class='del'>-		       I40IWQPSQ_ADDFRAGCNT) |</div><div class='del'>-	    LS_64(qp-&gt;rwqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-</div><div class='del'>-	i40iw_set_fragment(wqe, 0, info-&gt;sg_list);</div><div class='del'>-</div><div class='del'>-	for (i = 1, byte_off = 32; i &lt; info-&gt;num_sges; i++) {</div><div class='del'>-		i40iw_set_fragment(wqe, byte_off, &amp;info-&gt;sg_list[i]);</div><div class='del'>-		byte_off += 16;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 24, header);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cq_request_notification - cq notification request (door bell)</div><div class='del'>- * @cq: hw cq</div><div class='del'>- * @cq_notify: notification type</div><div class='del'>- */</div><div class='del'>-static void i40iw_cq_request_notification(struct i40iw_cq_uk *cq,</div><div class='del'>-					  enum i40iw_completion_notify cq_notify)</div><div class='del'>-{</div><div class='del'>-	u64 temp_val;</div><div class='del'>-	u16 sw_cq_sel;</div><div class='del'>-	u8 arm_next_se = 0;</div><div class='del'>-	u8 arm_next = 0;</div><div class='del'>-	u8 arm_seq_num;</div><div class='del'>-</div><div class='del'>-	get_64bit_val(cq-&gt;shadow_area, 32, &amp;temp_val);</div><div class='del'>-	arm_seq_num = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_SEQ_NUM);</div><div class='del'>-	arm_seq_num++;</div><div class='del'>-</div><div class='del'>-	sw_cq_sel = (u16)RS_64(temp_val, I40IW_CQ_DBSA_SW_CQ_SELECT);</div><div class='del'>-	arm_next_se = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_NEXT_SE);</div><div class='del'>-	arm_next_se |= 1;</div><div class='del'>-	if (cq_notify == IW_CQ_COMPL_EVENT)</div><div class='del'>-		arm_next = 1;</div><div class='del'>-	temp_val = LS_64(arm_seq_num, I40IW_CQ_DBSA_ARM_SEQ_NUM) |</div><div class='del'>-	    LS_64(sw_cq_sel, I40IW_CQ_DBSA_SW_CQ_SELECT) |</div><div class='del'>-	    LS_64(arm_next_se, I40IW_CQ_DBSA_ARM_NEXT_SE) |</div><div class='del'>-	    LS_64(arm_next, I40IW_CQ_DBSA_ARM_NEXT);</div><div class='del'>-</div><div class='del'>-	set_64bit_val(cq-&gt;shadow_area, 32, temp_val);</div><div class='del'>-</div><div class='del'>-	wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='del'>-</div><div class='del'>-	writel(cq-&gt;cq_id, cq-&gt;cqe_alloc_reg);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cq_post_entries - update tail in shadow memory</div><div class='del'>- * @cq: hw cq</div><div class='del'>- * @count: # of entries processed</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_cq_post_entries(struct i40iw_cq_uk *cq,</div><div class='del'>-						    u8 count)</div><div class='del'>-{</div><div class='del'>-	I40IW_RING_MOVE_TAIL_BY_COUNT(cq-&gt;cq_ring, count);</div><div class='del'>-	set_64bit_val(cq-&gt;shadow_area, 0,</div><div class='del'>-		      I40IW_RING_GETCURRENT_HEAD(cq-&gt;cq_ring));</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cq_poll_completion - get cq completion info</div><div class='del'>- * @cq: hw cq</div><div class='del'>- * @info: cq poll information returned</div><div class='del'>- * @post_cq: update cq tail</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code i40iw_cq_poll_completion(struct i40iw_cq_uk *cq,</div><div class='del'>-						       struct i40iw_cq_poll_info *info)</div><div class='del'>-{</div><div class='del'>-	u64 comp_ctx, qword0, qword2, qword3, wqe_qword;</div><div class='del'>-	u64 *cqe, *sw_wqe;</div><div class='del'>-	struct i40iw_qp_uk *qp;</div><div class='del'>-	struct i40iw_ring *pring = NULL;</div><div class='del'>-	u32 wqe_idx, q_type, array_idx = 0;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-	bool move_cq_head = true;</div><div class='del'>-	u8 polarity;</div><div class='del'>-	u8 addl_wqes = 0;</div><div class='del'>-</div><div class='del'>-	if (cq-&gt;avoid_mem_cflct)</div><div class='del'>-		cqe = (u64 *)I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(cq);</div><div class='del'>-	else</div><div class='del'>-		cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(cq);</div><div class='del'>-</div><div class='del'>-	get_64bit_val(cqe, 24, &amp;qword3);</div><div class='del'>-	polarity = (u8)RS_64(qword3, I40IW_CQ_VALID);</div><div class='del'>-</div><div class='del'>-	if (polarity != cq-&gt;polarity)</div><div class='del'>-		return I40IW_ERR_QUEUE_EMPTY;</div><div class='del'>-</div><div class='del'>-	q_type = (u8)RS_64(qword3, I40IW_CQ_SQ);</div><div class='del'>-	info-&gt;error = (bool)RS_64(qword3, I40IW_CQ_ERROR);</div><div class='del'>-	info-&gt;push_dropped = (bool)RS_64(qword3, I40IWCQ_PSHDROP);</div><div class='del'>-	if (info-&gt;error) {</div><div class='del'>-		info-&gt;comp_status = I40IW_COMPL_STATUS_FLUSHED;</div><div class='del'>-		info-&gt;major_err = (bool)RS_64(qword3, I40IW_CQ_MAJERR);</div><div class='del'>-		info-&gt;minor_err = (bool)RS_64(qword3, I40IW_CQ_MINERR);</div><div class='del'>-	} else {</div><div class='del'>-		info-&gt;comp_status = I40IW_COMPL_STATUS_SUCCESS;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	get_64bit_val(cqe, 0, &amp;qword0);</div><div class='del'>-	get_64bit_val(cqe, 16, &amp;qword2);</div><div class='del'>-</div><div class='del'>-	info-&gt;tcp_seq_num = (u32)RS_64(qword0, I40IWCQ_TCPSEQNUM);</div><div class='del'>-</div><div class='del'>-	info-&gt;qp_id = (u32)RS_64(qword2, I40IWCQ_QPID);</div><div class='del'>-</div><div class='del'>-	get_64bit_val(cqe, 8, &amp;comp_ctx);</div><div class='del'>-</div><div class='del'>-	info-&gt;solicited_event = (bool)RS_64(qword3, I40IWCQ_SOEVENT);</div><div class='del'>-	info-&gt;is_srq = (bool)RS_64(qword3, I40IWCQ_SRQ);</div><div class='del'>-</div><div class='del'>-	qp = (struct i40iw_qp_uk *)(unsigned long)comp_ctx;</div><div class='del'>-	if (!qp) {</div><div class='del'>-		ret_code = I40IW_ERR_QUEUE_DESTROYED;</div><div class='del'>-		goto exit;</div><div class='del'>-	}</div><div class='del'>-	wqe_idx = (u32)RS_64(qword3, I40IW_CQ_WQEIDX);</div><div class='del'>-	info-&gt;qp_handle = (i40iw_qp_handle)(unsigned long)qp;</div><div class='del'>-</div><div class='del'>-	if (q_type == I40IW_CQE_QTYPE_RQ) {</div><div class='del'>-		array_idx = (wqe_idx * 4) / qp-&gt;rq_wqe_size_multiplier;</div><div class='del'>-		if (info-&gt;comp_status == I40IW_COMPL_STATUS_FLUSHED) {</div><div class='del'>-			info-&gt;wr_id = qp-&gt;rq_wrid_array[qp-&gt;rq_ring.tail];</div><div class='del'>-			array_idx = qp-&gt;rq_ring.tail;</div><div class='del'>-		} else {</div><div class='del'>-			info-&gt;wr_id = qp-&gt;rq_wrid_array[array_idx];</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		info-&gt;op_type = I40IW_OP_TYPE_REC;</div><div class='del'>-		if (qword3 &amp; I40IWCQ_STAG_MASK) {</div><div class='del'>-			info-&gt;stag_invalid_set = true;</div><div class='del'>-			info-&gt;inv_stag = (u32)RS_64(qword2, I40IWCQ_INVSTAG);</div><div class='del'>-		} else {</div><div class='del'>-			info-&gt;stag_invalid_set = false;</div><div class='del'>-		}</div><div class='del'>-		info-&gt;bytes_xfered = (u32)RS_64(qword0, I40IWCQ_PAYLDLEN);</div><div class='del'>-		I40IW_RING_SET_TAIL(qp-&gt;rq_ring, array_idx + 1);</div><div class='del'>-		pring = &amp;qp-&gt;rq_ring;</div><div class='del'>-	} else {</div><div class='del'>-		if (qp-&gt;first_sq_wq) {</div><div class='del'>-			qp-&gt;first_sq_wq = false;</div><div class='del'>-			if (!wqe_idx &amp;&amp; (qp-&gt;sq_ring.head == qp-&gt;sq_ring.tail)) {</div><div class='del'>-				I40IW_RING_MOVE_HEAD_NOCHECK(cq-&gt;cq_ring);</div><div class='del'>-				I40IW_RING_MOVE_TAIL(cq-&gt;cq_ring);</div><div class='del'>-				set_64bit_val(cq-&gt;shadow_area, 0,</div><div class='del'>-					      I40IW_RING_GETCURRENT_HEAD(cq-&gt;cq_ring));</div><div class='del'>-				memset(info, 0, sizeof(struct i40iw_cq_poll_info));</div><div class='del'>-				return i40iw_cq_poll_completion(cq, info);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (info-&gt;comp_status != I40IW_COMPL_STATUS_FLUSHED) {</div><div class='del'>-			info-&gt;wr_id = qp-&gt;sq_wrtrk_array[wqe_idx].wrid;</div><div class='del'>-			info-&gt;bytes_xfered = qp-&gt;sq_wrtrk_array[wqe_idx].wr_len;</div><div class='del'>-</div><div class='del'>-			info-&gt;op_type = (u8)RS_64(qword3, I40IWCQ_OP);</div><div class='del'>-			sw_wqe = qp-&gt;sq_base[wqe_idx].elem;</div><div class='del'>-			get_64bit_val(sw_wqe, 24, &amp;wqe_qword);</div><div class='del'>-</div><div class='del'>-			addl_wqes = qp-&gt;sq_wrtrk_array[wqe_idx].wqe_size / I40IW_QP_WQE_MIN_SIZE;</div><div class='del'>-			I40IW_RING_SET_TAIL(qp-&gt;sq_ring, (wqe_idx + addl_wqes));</div><div class='del'>-		} else {</div><div class='del'>-			do {</div><div class='del'>-				u8 op_type;</div><div class='del'>-				u32 tail;</div><div class='del'>-</div><div class='del'>-				tail = qp-&gt;sq_ring.tail;</div><div class='del'>-				sw_wqe = qp-&gt;sq_base[tail].elem;</div><div class='del'>-				get_64bit_val(sw_wqe, 24, &amp;wqe_qword);</div><div class='del'>-				op_type = (u8)RS_64(wqe_qword, I40IWQPSQ_OPCODE);</div><div class='del'>-				info-&gt;op_type = op_type;</div><div class='del'>-				addl_wqes = qp-&gt;sq_wrtrk_array[tail].wqe_size / I40IW_QP_WQE_MIN_SIZE;</div><div class='del'>-				I40IW_RING_SET_TAIL(qp-&gt;sq_ring, (tail + addl_wqes));</div><div class='del'>-				if (op_type != I40IWQP_OP_NOP) {</div><div class='del'>-					info-&gt;wr_id = qp-&gt;sq_wrtrk_array[tail].wrid;</div><div class='del'>-					info-&gt;bytes_xfered = qp-&gt;sq_wrtrk_array[tail].wr_len;</div><div class='del'>-					break;</div><div class='del'>-				}</div><div class='del'>-			} while (1);</div><div class='del'>-		}</div><div class='del'>-		pring = &amp;qp-&gt;sq_ring;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret_code = 0;</div><div class='del'>-</div><div class='del'>-exit:</div><div class='del'>-	if (!ret_code &amp;&amp;</div><div class='del'>-	    (info-&gt;comp_status == I40IW_COMPL_STATUS_FLUSHED))</div><div class='del'>-		if (pring &amp;&amp; (I40IW_RING_MORE_WORK(*pring)))</div><div class='del'>-			move_cq_head = false;</div><div class='del'>-</div><div class='del'>-	if (move_cq_head) {</div><div class='del'>-		I40IW_RING_MOVE_HEAD_NOCHECK(cq-&gt;cq_ring);</div><div class='del'>-</div><div class='del'>-		if (I40IW_RING_GETCURRENT_HEAD(cq-&gt;cq_ring) == 0)</div><div class='del'>-			cq-&gt;polarity ^= 1;</div><div class='del'>-</div><div class='del'>-		I40IW_RING_MOVE_TAIL(cq-&gt;cq_ring);</div><div class='del'>-		set_64bit_val(cq-&gt;shadow_area, 0,</div><div class='del'>-			      I40IW_RING_GETCURRENT_HEAD(cq-&gt;cq_ring));</div><div class='del'>-	} else {</div><div class='del'>-		if (info-&gt;is_srq)</div><div class='del'>-			return ret_code;</div><div class='del'>-		qword3 &amp;= ~I40IW_CQ_WQEIDX_MASK;</div><div class='del'>-		qword3 |= LS_64(pring-&gt;tail, I40IW_CQ_WQEIDX);</div><div class='del'>-		set_64bit_val(cqe, 24, qword3);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_wqe_shift - get shift count for maximum wqe size</div><div class='del'>- * @sge: Maximum Scatter Gather Elements wqe</div><div class='del'>- * @inline_data: Maximum inline data size</div><div class='del'>- * @shift: Returns the shift needed based on sge</div><div class='del'>- *</div><div class='del'>- * Shift can be used to left shift the wqe size based on number of SGEs and inlind data size.</div><div class='del'>- * For 1 SGE or inline data &lt;= 16, shift = 0 (wqe size of 32 bytes).</div><div class='del'>- * For 2 or 3 SGEs or inline data &lt;= 48, shift = 1 (wqe size of 64 bytes).</div><div class='del'>- * Shift of 2 otherwise (wqe size of 128 bytes).</div><div class='del'>- */</div><div class='del'>-void i40iw_get_wqe_shift(u32 sge, u32 inline_data, u8 *shift)</div><div class='del'>-{</div><div class='del'>-	*shift = 0;</div><div class='del'>-	if (sge &gt; 1 || inline_data &gt; 16)</div><div class='del'>-		*shift = (sge &lt; 4 &amp;&amp; inline_data &lt;= 48) ? 1 : 2;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * i40iw_get_sqdepth - get SQ depth (quantas)</div><div class='del'>- * @sq_size: SQ size</div><div class='del'>- * @shift: shift which determines size of WQE</div><div class='del'>- * @sqdepth: depth of SQ</div><div class='del'>- *</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_get_sqdepth(u32 sq_size, u8 shift, u32 *sqdepth)</div><div class='del'>-{</div><div class='del'>-	*sqdepth = roundup_pow_of_two((sq_size &lt;&lt; shift) + I40IW_SQ_RSVD);</div><div class='del'>-</div><div class='del'>-	if (*sqdepth &lt; (I40IW_QP_SW_MIN_WQSIZE &lt;&lt; shift))</div><div class='del'>-		*sqdepth = I40IW_QP_SW_MIN_WQSIZE &lt;&lt; shift;</div><div class='del'>-	else if (*sqdepth &gt; I40IW_QP_SW_MAX_SQ_QUANTAS)</div><div class='del'>-		return I40IW_ERR_INVALID_SIZE;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * i40iw_get_rq_depth - get RQ depth (quantas)</div><div class='del'>- * @rq_size: RQ size</div><div class='del'>- * @shift: shift which determines size of WQE</div><div class='del'>- * @rqdepth: depth of RQ</div><div class='del'>- *</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_get_rqdepth(u32 rq_size, u8 shift, u32 *rqdepth)</div><div class='del'>-{</div><div class='del'>-	*rqdepth = roundup_pow_of_two((rq_size &lt;&lt; shift) + I40IW_RQ_RSVD);</div><div class='del'>-</div><div class='del'>-	if (*rqdepth &lt; (I40IW_QP_SW_MIN_WQSIZE &lt;&lt; shift))</div><div class='del'>-		*rqdepth = I40IW_QP_SW_MIN_WQSIZE &lt;&lt; shift;</div><div class='del'>-	else if (*rqdepth &gt; I40IW_QP_SW_MAX_RQ_QUANTAS)</div><div class='del'>-		return I40IW_ERR_INVALID_SIZE;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static const struct i40iw_qp_uk_ops iw_qp_uk_ops = {</div><div class='del'>-	.iw_qp_post_wr = i40iw_qp_post_wr,</div><div class='del'>-	.iw_qp_ring_push_db = i40iw_qp_ring_push_db,</div><div class='del'>-	.iw_rdma_write = i40iw_rdma_write,</div><div class='del'>-	.iw_rdma_read = i40iw_rdma_read,</div><div class='del'>-	.iw_send = i40iw_send,</div><div class='del'>-	.iw_inline_rdma_write = i40iw_inline_rdma_write,</div><div class='del'>-	.iw_inline_send = i40iw_inline_send,</div><div class='del'>-	.iw_stag_local_invalidate = i40iw_stag_local_invalidate,</div><div class='del'>-	.iw_mw_bind = i40iw_mw_bind,</div><div class='del'>-	.iw_post_receive = i40iw_post_receive,</div><div class='del'>-	.iw_post_nop = i40iw_nop</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static const struct i40iw_cq_ops iw_cq_ops = {</div><div class='del'>-	.iw_cq_request_notification = i40iw_cq_request_notification,</div><div class='del'>-	.iw_cq_poll_completion = i40iw_cq_poll_completion,</div><div class='del'>-	.iw_cq_post_entries = i40iw_cq_post_entries,</div><div class='del'>-	.iw_cq_clean = i40iw_clean_cq</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static const struct i40iw_device_uk_ops iw_device_uk_ops = {</div><div class='del'>-	.iwarp_cq_uk_init = i40iw_cq_uk_init,</div><div class='del'>-	.iwarp_qp_uk_init = i40iw_qp_uk_init,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_qp_uk_init - initialize shared qp</div><div class='del'>- * @qp: hw qp (user and kernel)</div><div class='del'>- * @info: qp initialization info</div><div class='del'>- *</div><div class='del'>- * initializes the vars used in both user and kernel mode.</div><div class='del'>- * size of the wqe depends on numbers of max. fragements</div><div class='del'>- * allowed. Then size of wqe * the number of wqes should be the</div><div class='del'>- * amount of memory allocated for sq and rq. If srq is used,</div><div class='del'>- * then rq_base will point to one rq wqe only (not the whole</div><div class='del'>- * array of wqes)</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_qp_uk_init(struct i40iw_qp_uk *qp,</div><div class='del'>-					struct i40iw_qp_uk_init_info *info)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-	u32 sq_ring_size;</div><div class='del'>-	u8 sqshift, rqshift;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;max_sq_frag_cnt &gt; I40IW_MAX_WQ_FRAGMENT_COUNT)</div><div class='del'>-		return I40IW_ERR_INVALID_FRAG_COUNT;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;max_rq_frag_cnt &gt; I40IW_MAX_WQ_FRAGMENT_COUNT)</div><div class='del'>-		return I40IW_ERR_INVALID_FRAG_COUNT;</div><div class='del'>-	i40iw_get_wqe_shift(info-&gt;max_sq_frag_cnt, info-&gt;max_inline_data, &amp;sqshift);</div><div class='del'>-</div><div class='del'>-	qp-&gt;sq_base = info-&gt;sq;</div><div class='del'>-	qp-&gt;rq_base = info-&gt;rq;</div><div class='del'>-	qp-&gt;shadow_area = info-&gt;shadow_area;</div><div class='del'>-	qp-&gt;sq_wrtrk_array = info-&gt;sq_wrtrk_array;</div><div class='del'>-	qp-&gt;rq_wrid_array = info-&gt;rq_wrid_array;</div><div class='del'>-</div><div class='del'>-	qp-&gt;wqe_alloc_reg = info-&gt;wqe_alloc_reg;</div><div class='del'>-	qp-&gt;qp_id = info-&gt;qp_id;</div><div class='del'>-</div><div class='del'>-	qp-&gt;sq_size = info-&gt;sq_size;</div><div class='del'>-	qp-&gt;push_db = info-&gt;push_db;</div><div class='del'>-	qp-&gt;push_wqe = info-&gt;push_wqe;</div><div class='del'>-</div><div class='del'>-	qp-&gt;max_sq_frag_cnt = info-&gt;max_sq_frag_cnt;</div><div class='del'>-	sq_ring_size = qp-&gt;sq_size &lt;&lt; sqshift;</div><div class='del'>-</div><div class='del'>-	I40IW_RING_INIT(qp-&gt;sq_ring, sq_ring_size);</div><div class='del'>-	I40IW_RING_INIT(qp-&gt;initial_ring, sq_ring_size);</div><div class='del'>-	I40IW_RING_MOVE_HEAD(qp-&gt;sq_ring, ret_code);</div><div class='del'>-	I40IW_RING_MOVE_TAIL(qp-&gt;sq_ring);</div><div class='del'>-	I40IW_RING_MOVE_HEAD(qp-&gt;initial_ring, ret_code);</div><div class='del'>-	qp-&gt;swqe_polarity = 1;</div><div class='del'>-	qp-&gt;first_sq_wq = true;</div><div class='del'>-	qp-&gt;swqe_polarity_deferred = 1;</div><div class='del'>-	qp-&gt;rwqe_polarity = 0;</div><div class='del'>-</div><div class='del'>-	if (!qp-&gt;use_srq) {</div><div class='del'>-		qp-&gt;rq_size = info-&gt;rq_size;</div><div class='del'>-		qp-&gt;max_rq_frag_cnt = info-&gt;max_rq_frag_cnt;</div><div class='del'>-		I40IW_RING_INIT(qp-&gt;rq_ring, qp-&gt;rq_size);</div><div class='del'>-		switch (info-&gt;abi_ver) {</div><div class='del'>-		case 4:</div><div class='del'>-			i40iw_get_wqe_shift(info-&gt;max_rq_frag_cnt, 0, &amp;rqshift);</div><div class='del'>-			break;</div><div class='del'>-		case 5: /* fallthrough until next ABI version */</div><div class='del'>-		default:</div><div class='del'>-			rqshift = I40IW_MAX_RQ_WQE_SHIFT;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		qp-&gt;rq_wqe_size = rqshift;</div><div class='del'>-		qp-&gt;rq_wqe_size_multiplier = 4 &lt;&lt; rqshift;</div><div class='del'>-	}</div><div class='del'>-	qp-&gt;ops = iw_qp_uk_ops;</div><div class='del'>-</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cq_uk_init - initialize shared cq (user and kernel)</div><div class='del'>- * @cq: hw cq</div><div class='del'>- * @info: hw cq initialization info</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_cq_uk_init(struct i40iw_cq_uk *cq,</div><div class='del'>-					struct i40iw_cq_uk_init_info *info)</div><div class='del'>-{</div><div class='del'>-	if ((info-&gt;cq_size &lt; I40IW_MIN_CQ_SIZE) ||</div><div class='del'>-	    (info-&gt;cq_size &gt; I40IW_MAX_CQ_SIZE))</div><div class='del'>-		return I40IW_ERR_INVALID_SIZE;</div><div class='del'>-	cq-&gt;cq_base = (struct i40iw_cqe *)info-&gt;cq_base;</div><div class='del'>-	cq-&gt;cq_id = info-&gt;cq_id;</div><div class='del'>-	cq-&gt;cq_size = info-&gt;cq_size;</div><div class='del'>-	cq-&gt;cqe_alloc_reg = info-&gt;cqe_alloc_reg;</div><div class='del'>-	cq-&gt;shadow_area = info-&gt;shadow_area;</div><div class='del'>-	cq-&gt;avoid_mem_cflct = info-&gt;avoid_mem_cflct;</div><div class='del'>-</div><div class='del'>-	I40IW_RING_INIT(cq-&gt;cq_ring, cq-&gt;cq_size);</div><div class='del'>-	cq-&gt;polarity = 1;</div><div class='del'>-	cq-&gt;ops = iw_cq_ops;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_device_init_uk - setup routines for iwarp shared device</div><div class='del'>- * @dev: iwarp shared (user and kernel)</div><div class='del'>- */</div><div class='del'>-void i40iw_device_init_uk(struct i40iw_dev_uk *dev)</div><div class='del'>-{</div><div class='del'>-	dev-&gt;ops_uk = iw_device_uk_ops;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_clean_cq - clean cq entries</div><div class='del'>- * @ queue completion context</div><div class='del'>- * @cq: cq to clean</div><div class='del'>- */</div><div class='del'>-void i40iw_clean_cq(void *queue, struct i40iw_cq_uk *cq)</div><div class='del'>-{</div><div class='del'>-	u64 *cqe;</div><div class='del'>-	u64 qword3, comp_ctx;</div><div class='del'>-	u32 cq_head;</div><div class='del'>-	u8 polarity, temp;</div><div class='del'>-</div><div class='del'>-	cq_head = cq-&gt;cq_ring.head;</div><div class='del'>-	temp = cq-&gt;polarity;</div><div class='del'>-	do {</div><div class='del'>-		if (cq-&gt;avoid_mem_cflct)</div><div class='del'>-			cqe = (u64 *)&amp;(((struct i40iw_extended_cqe *)cq-&gt;cq_base)[cq_head]);</div><div class='del'>-		else</div><div class='del'>-			cqe = (u64 *)&amp;cq-&gt;cq_base[cq_head];</div><div class='del'>-		get_64bit_val(cqe, 24, &amp;qword3);</div><div class='del'>-		polarity = (u8)RS_64(qword3, I40IW_CQ_VALID);</div><div class='del'>-</div><div class='del'>-		if (polarity != temp)</div><div class='del'>-			break;</div><div class='del'>-</div><div class='del'>-		get_64bit_val(cqe, 8, &amp;comp_ctx);</div><div class='del'>-		if ((void *)(unsigned long)comp_ctx == queue)</div><div class='del'>-			set_64bit_val(cqe, 8, 0);</div><div class='del'>-</div><div class='del'>-		cq_head = (cq_head + 1) % cq-&gt;cq_ring.size;</div><div class='del'>-		if (!cq_head)</div><div class='del'>-			temp ^= 1;</div><div class='del'>-	} while (true);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_nop - send a nop</div><div class='del'>- * @qp: hw qp ptr</div><div class='del'>- * @wr_id: work request id</div><div class='del'>- * @signaled: flag if signaled for completion</div><div class='del'>- * @post_sq: flag to post sq</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_nop(struct i40iw_qp_uk *qp,</div><div class='del'>-				 u64 wr_id,</div><div class='del'>-				 bool signaled,</div><div class='del'>-				 bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 header, *wqe;</div><div class='del'>-	u32 wqe_idx;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_qp_get_next_send_wqe(qp, &amp;wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, wr_id);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_QP_TOOMANY_WRS_POSTED;</div><div class='del'>-	set_64bit_val(wqe, 0, 0);</div><div class='del'>-	set_64bit_val(wqe, 8, 0);</div><div class='del'>-	set_64bit_val(wqe, 16, 0);</div><div class='del'>-</div><div class='del'>-	header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) |</div><div class='del'>-	    LS_64(signaled, I40IWQPSQ_SIGCOMPL) |</div><div class='del'>-	    LS_64(qp-&gt;swqe_polarity, I40IWQPSQ_VALID);</div><div class='del'>-</div><div class='del'>-	wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='del'>-</div><div class='del'>-	set_64bit_val(wqe, 24, header);</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_qp_post_wr(qp);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_fragcnt_to_wqesize_sq - calculate wqe size based on fragment count for SQ</div><div class='del'>- * @frag_cnt: number of fragments</div><div class='del'>- * @wqe_size: size of sq wqe returned</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_fragcnt_to_wqesize_sq(u32 frag_cnt, u8 *wqe_size)</div><div class='del'>-{</div><div class='del'>-	switch (frag_cnt) {</div><div class='del'>-	case 0:</div><div class='del'>-	case 1:</div><div class='del'>-		*wqe_size = I40IW_QP_WQE_MIN_SIZE;</div><div class='del'>-		break;</div><div class='del'>-	case 2:</div><div class='del'>-	case 3:</div><div class='del'>-		*wqe_size = 64;</div><div class='del'>-		break;</div><div class='del'>-	case 4:</div><div class='del'>-	case 5:</div><div class='del'>-		*wqe_size = 96;</div><div class='del'>-		break;</div><div class='del'>-	case 6:</div><div class='del'>-	case 7:</div><div class='del'>-		*wqe_size = 128;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return I40IW_ERR_INVALID_FRAG_COUNT;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_fragcnt_to_wqesize_rq - calculate wqe size based on fragment count for RQ</div><div class='del'>- * @frag_cnt: number of fragments</div><div class='del'>- * @wqe_size: size of rq wqe returned</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_fragcnt_to_wqesize_rq(u32 frag_cnt, u8 *wqe_size)</div><div class='del'>-{</div><div class='del'>-	switch (frag_cnt) {</div><div class='del'>-	case 0:</div><div class='del'>-	case 1:</div><div class='del'>-		*wqe_size = 32;</div><div class='del'>-		break;</div><div class='del'>-	case 2:</div><div class='del'>-	case 3:</div><div class='del'>-		*wqe_size = 64;</div><div class='del'>-		break;</div><div class='del'>-	case 4:</div><div class='del'>-	case 5:</div><div class='del'>-	case 6:</div><div class='del'>-	case 7:</div><div class='del'>-		*wqe_size = 128;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return I40IW_ERR_INVALID_FRAG_COUNT;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_inline_data_size_to_wqesize - based on inline data, wqe size</div><div class='del'>- * @data_size: data size for inline</div><div class='del'>- * @wqe_size: size of sq wqe returned</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_inline_data_size_to_wqesize(u32 data_size,</div><div class='del'>-							 u8 *wqe_size)</div><div class='del'>-{</div><div class='del'>-	if (data_size &gt; I40IW_MAX_INLINE_DATA_SIZE)</div><div class='del'>-		return I40IW_ERR_INVALID_INLINE_DATA_SIZE;</div><div class='del'>-</div><div class='del'>-	if (data_size &lt;= 16)</div><div class='del'>-		*wqe_size = I40IW_QP_WQE_MIN_SIZE;</div><div class='del'>-	else</div><div class='del'>-		*wqe_size = 64;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_user.h b/drivers/infiniband/hw/i40iw/i40iw_user.h<br/>deleted file mode 100644<br/>index b125925641e0..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_user.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_user.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,430 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#ifndef I40IW_USER_H</div><div class='del'>-#define I40IW_USER_H</div><div class='del'>-</div><div class='del'>-enum i40iw_device_capabilities_const {</div><div class='del'>-	I40IW_WQE_SIZE =			4,</div><div class='del'>-	I40IW_CQP_WQE_SIZE =			8,</div><div class='del'>-	I40IW_CQE_SIZE =			4,</div><div class='del'>-	I40IW_EXTENDED_CQE_SIZE =		8,</div><div class='del'>-	I40IW_AEQE_SIZE =			2,</div><div class='del'>-	I40IW_CEQE_SIZE =			1,</div><div class='del'>-	I40IW_CQP_CTX_SIZE =			8,</div><div class='del'>-	I40IW_SHADOW_AREA_SIZE =		8,</div><div class='del'>-	I40IW_CEQ_MAX_COUNT =			256,</div><div class='del'>-	I40IW_QUERY_FPM_BUF_SIZE =		128,</div><div class='del'>-	I40IW_COMMIT_FPM_BUF_SIZE =		128,</div><div class='del'>-	I40IW_MIN_IW_QP_ID =			1,</div><div class='del'>-	I40IW_MAX_IW_QP_ID =			262143,</div><div class='del'>-	I40IW_MIN_CEQID =			0,</div><div class='del'>-	I40IW_MAX_CEQID =			256,</div><div class='del'>-	I40IW_MIN_CQID =			0,</div><div class='del'>-	I40IW_MAX_CQID =			131071,</div><div class='del'>-	I40IW_MIN_AEQ_ENTRIES =			1,</div><div class='del'>-	I40IW_MAX_AEQ_ENTRIES =			524287,</div><div class='del'>-	I40IW_MIN_CEQ_ENTRIES =			1,</div><div class='del'>-	I40IW_MAX_CEQ_ENTRIES =			131071,</div><div class='del'>-	I40IW_MIN_CQ_SIZE =			1,</div><div class='del'>-	I40IW_MAX_CQ_SIZE =			1048575,</div><div class='del'>-	I40IW_DB_ID_ZERO =			0,</div><div class='del'>-	I40IW_MAX_WQ_FRAGMENT_COUNT =		3,</div><div class='del'>-	I40IW_MAX_SGE_RD =			1,</div><div class='del'>-	I40IW_MAX_OUTBOUND_MESSAGE_SIZE =	2147483647,</div><div class='del'>-	I40IW_MAX_INBOUND_MESSAGE_SIZE =	2147483647,</div><div class='del'>-	I40IW_MAX_PUSH_PAGE_COUNT =		4096,</div><div class='del'>-	I40IW_MAX_PE_ENABLED_VF_COUNT =		32,</div><div class='del'>-	I40IW_MAX_VF_FPM_ID =			47,</div><div class='del'>-	I40IW_MAX_VF_PER_PF =			127,</div><div class='del'>-	I40IW_MAX_SQ_PAYLOAD_SIZE =		2145386496,</div><div class='del'>-	I40IW_MAX_INLINE_DATA_SIZE =		48,</div><div class='del'>-	I40IW_MAX_PUSHMODE_INLINE_DATA_SIZE =	48,</div><div class='del'>-	I40IW_MAX_IRD_SIZE =			64,</div><div class='del'>-	I40IW_MAX_ORD_SIZE =			127,</div><div class='del'>-	I40IW_MAX_WQ_ENTRIES =			2048,</div><div class='del'>-	I40IW_Q2_BUFFER_SIZE =			(248 + 100),</div><div class='del'>-	I40IW_MAX_WQE_SIZE_RQ =			128,</div><div class='del'>-	I40IW_QP_CTX_SIZE =			248,</div><div class='del'>-	I40IW_MAX_PDS = 			32768</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define i40iw_handle void *</div><div class='del'>-#define i40iw_adapter_handle i40iw_handle</div><div class='del'>-#define i40iw_qp_handle i40iw_handle</div><div class='del'>-#define i40iw_cq_handle i40iw_handle</div><div class='del'>-#define i40iw_srq_handle i40iw_handle</div><div class='del'>-#define i40iw_pd_id i40iw_handle</div><div class='del'>-#define i40iw_stag_handle i40iw_handle</div><div class='del'>-#define i40iw_stag_index u32</div><div class='del'>-#define i40iw_stag u32</div><div class='del'>-#define i40iw_stag_key u8</div><div class='del'>-</div><div class='del'>-#define i40iw_tagged_offset u64</div><div class='del'>-#define i40iw_access_privileges u32</div><div class='del'>-#define i40iw_physical_fragment u64</div><div class='del'>-#define i40iw_address_list u64 *</div><div class='del'>-</div><div class='del'>-#define	I40IW_MAX_MR_SIZE	0x10000000000L</div><div class='del'>-#define	I40IW_MAX_RQ_WQE_SHIFT	2</div><div class='del'>-</div><div class='del'>-struct i40iw_qp_uk;</div><div class='del'>-struct i40iw_cq_uk;</div><div class='del'>-struct i40iw_srq_uk;</div><div class='del'>-struct i40iw_qp_uk_init_info;</div><div class='del'>-struct i40iw_cq_uk_init_info;</div><div class='del'>-struct i40iw_srq_uk_init_info;</div><div class='del'>-</div><div class='del'>-struct i40iw_sge {</div><div class='del'>-	i40iw_tagged_offset tag_off;</div><div class='del'>-	u32 len;</div><div class='del'>-	i40iw_stag stag;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define i40iw_sgl struct i40iw_sge *</div><div class='del'>-</div><div class='del'>-struct i40iw_ring {</div><div class='del'>-	u32 head;</div><div class='del'>-	u32 tail;</div><div class='del'>-	u32 size;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cqe {</div><div class='del'>-	u64 buf[I40IW_CQE_SIZE];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_extended_cqe {</div><div class='del'>-	u64 buf[I40IW_EXTENDED_CQE_SIZE];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_wqe {</div><div class='del'>-	u64 buf[I40IW_WQE_SIZE];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_qp_uk_ops;</div><div class='del'>-</div><div class='del'>-enum i40iw_addressing_type {</div><div class='del'>-	I40IW_ADDR_TYPE_ZERO_BASED = 0,</div><div class='del'>-	I40IW_ADDR_TYPE_VA_BASED = 1,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define I40IW_ACCESS_FLAGS_LOCALREAD		0x01</div><div class='del'>-#define I40IW_ACCESS_FLAGS_LOCALWRITE		0x02</div><div class='del'>-#define I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY	0x04</div><div class='del'>-#define I40IW_ACCESS_FLAGS_REMOTEREAD		0x05</div><div class='del'>-#define I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY	0x08</div><div class='del'>-#define I40IW_ACCESS_FLAGS_REMOTEWRITE		0x0a</div><div class='del'>-#define I40IW_ACCESS_FLAGS_BIND_WINDOW		0x10</div><div class='del'>-#define I40IW_ACCESS_FLAGS_ALL			0x1F</div><div class='del'>-</div><div class='del'>-#define I40IW_OP_TYPE_RDMA_WRITE	0</div><div class='del'>-#define I40IW_OP_TYPE_RDMA_READ		1</div><div class='del'>-#define I40IW_OP_TYPE_SEND		3</div><div class='del'>-#define I40IW_OP_TYPE_SEND_INV		4</div><div class='del'>-#define I40IW_OP_TYPE_SEND_SOL		5</div><div class='del'>-#define I40IW_OP_TYPE_SEND_SOL_INV	6</div><div class='del'>-#define I40IW_OP_TYPE_REC		7</div><div class='del'>-#define I40IW_OP_TYPE_BIND_MW		8</div><div class='del'>-#define I40IW_OP_TYPE_FAST_REG_NSMR	9</div><div class='del'>-#define I40IW_OP_TYPE_INV_STAG		10</div><div class='del'>-#define I40IW_OP_TYPE_RDMA_READ_INV_STAG 11</div><div class='del'>-#define I40IW_OP_TYPE_NOP		12</div><div class='del'>-</div><div class='del'>-enum i40iw_completion_status {</div><div class='del'>-	I40IW_COMPL_STATUS_SUCCESS = 0,</div><div class='del'>-	I40IW_COMPL_STATUS_FLUSHED,</div><div class='del'>-	I40IW_COMPL_STATUS_INVALID_WQE,</div><div class='del'>-	I40IW_COMPL_STATUS_QP_CATASTROPHIC,</div><div class='del'>-	I40IW_COMPL_STATUS_REMOTE_TERMINATION,</div><div class='del'>-	I40IW_COMPL_STATUS_INVALID_STAG,</div><div class='del'>-	I40IW_COMPL_STATUS_BASE_BOUND_VIOLATION,</div><div class='del'>-	I40IW_COMPL_STATUS_ACCESS_VIOLATION,</div><div class='del'>-	I40IW_COMPL_STATUS_INVALID_PD_ID,</div><div class='del'>-	I40IW_COMPL_STATUS_WRAP_ERROR,</div><div class='del'>-	I40IW_COMPL_STATUS_STAG_INVALID_PDID,</div><div class='del'>-	I40IW_COMPL_STATUS_RDMA_READ_ZERO_ORD,</div><div class='del'>-	I40IW_COMPL_STATUS_QP_NOT_PRIVLEDGED,</div><div class='del'>-	I40IW_COMPL_STATUS_STAG_NOT_INVALID,</div><div class='del'>-	I40IW_COMPL_STATUS_INVALID_PHYS_BUFFER_SIZE,</div><div class='del'>-	I40IW_COMPL_STATUS_INVALID_PHYS_BUFFER_ENTRY,</div><div class='del'>-	I40IW_COMPL_STATUS_INVALID_FBO,</div><div class='del'>-	I40IW_COMPL_STATUS_INVALID_LENGTH,</div><div class='del'>-	I40IW_COMPL_STATUS_INVALID_ACCESS,</div><div class='del'>-	I40IW_COMPL_STATUS_PHYS_BUFFER_LIST_TOO_LONG,</div><div class='del'>-	I40IW_COMPL_STATUS_INVALID_VIRT_ADDRESS,</div><div class='del'>-	I40IW_COMPL_STATUS_INVALID_REGION,</div><div class='del'>-	I40IW_COMPL_STATUS_INVALID_WINDOW,</div><div class='del'>-	I40IW_COMPL_STATUS_INVALID_TOTAL_LENGTH</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_completion_notify {</div><div class='del'>-	IW_CQ_COMPL_EVENT = 0,</div><div class='del'>-	IW_CQ_COMPL_SOLICITED = 1</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_post_send {</div><div class='del'>-	i40iw_sgl sg_list;</div><div class='del'>-	u32 num_sges;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_post_inline_send {</div><div class='del'>-	void *data;</div><div class='del'>-	u32 len;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_rdma_write {</div><div class='del'>-	i40iw_sgl lo_sg_list;</div><div class='del'>-	u32 num_lo_sges;</div><div class='del'>-	struct i40iw_sge rem_addr;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_inline_rdma_write {</div><div class='del'>-	void *data;</div><div class='del'>-	u32 len;</div><div class='del'>-	struct i40iw_sge rem_addr;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_rdma_read {</div><div class='del'>-	struct i40iw_sge lo_addr;</div><div class='del'>-	struct i40iw_sge rem_addr;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_bind_window {</div><div class='del'>-	i40iw_stag mr_stag;</div><div class='del'>-	u64 bind_length;</div><div class='del'>-	void *va;</div><div class='del'>-	enum i40iw_addressing_type addressing_type;</div><div class='del'>-	bool enable_reads;</div><div class='del'>-	bool enable_writes;</div><div class='del'>-	i40iw_stag mw_stag;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_inv_local_stag {</div><div class='del'>-	i40iw_stag target_stag;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_post_sq_info {</div><div class='del'>-	u64 wr_id;</div><div class='del'>-	u8 op_type;</div><div class='del'>-	bool signaled;</div><div class='del'>-	bool read_fence;</div><div class='del'>-	bool local_fence;</div><div class='del'>-	bool inline_data;</div><div class='del'>-	bool defer_flag;</div><div class='del'>-	union {</div><div class='del'>-		struct i40iw_post_send send;</div><div class='del'>-		struct i40iw_rdma_write rdma_write;</div><div class='del'>-		struct i40iw_rdma_read rdma_read;</div><div class='del'>-		struct i40iw_rdma_read rdma_read_inv;</div><div class='del'>-		struct i40iw_bind_window bind_window;</div><div class='del'>-		struct i40iw_inv_local_stag inv_local_stag;</div><div class='del'>-		struct i40iw_inline_rdma_write inline_rdma_write;</div><div class='del'>-		struct i40iw_post_inline_send inline_send;</div><div class='del'>-	} op;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_post_rq_info {</div><div class='del'>-	u64 wr_id;</div><div class='del'>-	i40iw_sgl sg_list;</div><div class='del'>-	u32 num_sges;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cq_poll_info {</div><div class='del'>-	u64 wr_id;</div><div class='del'>-	i40iw_qp_handle qp_handle;</div><div class='del'>-	u32 bytes_xfered;</div><div class='del'>-	u32 tcp_seq_num;</div><div class='del'>-	u32 qp_id;</div><div class='del'>-	i40iw_stag inv_stag;</div><div class='del'>-	enum i40iw_completion_status comp_status;</div><div class='del'>-	u16 major_err;</div><div class='del'>-	u16 minor_err;</div><div class='del'>-	u8 op_type;</div><div class='del'>-	bool stag_invalid_set;</div><div class='del'>-	bool push_dropped;</div><div class='del'>-	bool error;</div><div class='del'>-	bool is_srq;</div><div class='del'>-	bool solicited_event;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_qp_uk_ops {</div><div class='del'>-	void (*iw_qp_post_wr)(struct i40iw_qp_uk *);</div><div class='del'>-	void (*iw_qp_ring_push_db)(struct i40iw_qp_uk *, u32);</div><div class='del'>-	enum i40iw_status_code (*iw_rdma_write)(struct i40iw_qp_uk *,</div><div class='del'>-						struct i40iw_post_sq_info *, bool);</div><div class='del'>-	enum i40iw_status_code (*iw_rdma_read)(struct i40iw_qp_uk *,</div><div class='del'>-					       struct i40iw_post_sq_info *, bool, bool);</div><div class='del'>-	enum i40iw_status_code (*iw_send)(struct i40iw_qp_uk *,</div><div class='del'>-					  struct i40iw_post_sq_info *, u32, bool);</div><div class='del'>-	enum i40iw_status_code (*iw_inline_rdma_write)(struct i40iw_qp_uk *,</div><div class='del'>-						       struct i40iw_post_sq_info *, bool);</div><div class='del'>-	enum i40iw_status_code (*iw_inline_send)(struct i40iw_qp_uk *,</div><div class='del'>-						 struct i40iw_post_sq_info *, u32, bool);</div><div class='del'>-	enum i40iw_status_code (*iw_stag_local_invalidate)(struct i40iw_qp_uk *,</div><div class='del'>-							   struct i40iw_post_sq_info *, bool);</div><div class='del'>-	enum i40iw_status_code (*iw_mw_bind)(struct i40iw_qp_uk *,</div><div class='del'>-					     struct i40iw_post_sq_info *, bool);</div><div class='del'>-	enum i40iw_status_code (*iw_post_receive)(struct i40iw_qp_uk *,</div><div class='del'>-						  struct i40iw_post_rq_info *);</div><div class='del'>-	enum i40iw_status_code (*iw_post_nop)(struct i40iw_qp_uk *, u64, bool, bool);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cq_ops {</div><div class='del'>-	void (*iw_cq_request_notification)(struct i40iw_cq_uk *,</div><div class='del'>-					   enum i40iw_completion_notify);</div><div class='del'>-	enum i40iw_status_code (*iw_cq_poll_completion)(struct i40iw_cq_uk *,</div><div class='del'>-							struct i40iw_cq_poll_info *);</div><div class='del'>-	enum i40iw_status_code (*iw_cq_post_entries)(struct i40iw_cq_uk *, u8 count);</div><div class='del'>-	void (*iw_cq_clean)(void *, struct i40iw_cq_uk *);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_dev_uk;</div><div class='del'>-</div><div class='del'>-struct i40iw_device_uk_ops {</div><div class='del'>-	enum i40iw_status_code (*iwarp_cq_uk_init)(struct i40iw_cq_uk *,</div><div class='del'>-						   struct i40iw_cq_uk_init_info *);</div><div class='del'>-	enum i40iw_status_code (*iwarp_qp_uk_init)(struct i40iw_qp_uk *,</div><div class='del'>-						   struct i40iw_qp_uk_init_info *);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_dev_uk {</div><div class='del'>-	struct i40iw_device_uk_ops ops_uk;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_sq_uk_wr_trk_info {</div><div class='del'>-	u64 wrid;</div><div class='del'>-	u32 wr_len;</div><div class='del'>-	u8 wqe_size;</div><div class='del'>-	u8 reserved[3];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_qp_quanta {</div><div class='del'>-	u64 elem[I40IW_WQE_SIZE];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_qp_uk {</div><div class='del'>-	struct i40iw_qp_quanta *sq_base;</div><div class='del'>-	struct i40iw_qp_quanta *rq_base;</div><div class='del'>-	u32 __iomem *wqe_alloc_reg;</div><div class='del'>-	struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array;</div><div class='del'>-	u64 *rq_wrid_array;</div><div class='del'>-	u64 *shadow_area;</div><div class='del'>-	u32 *push_db;</div><div class='del'>-	u64 *push_wqe;</div><div class='del'>-	struct i40iw_ring sq_ring;</div><div class='del'>-	struct i40iw_ring rq_ring;</div><div class='del'>-	struct i40iw_ring initial_ring;</div><div class='del'>-	u32 qp_id;</div><div class='del'>-	u32 sq_size;</div><div class='del'>-	u32 rq_size;</div><div class='del'>-	u32 max_sq_frag_cnt;</div><div class='del'>-	u32 max_rq_frag_cnt;</div><div class='del'>-	struct i40iw_qp_uk_ops ops;</div><div class='del'>-	bool use_srq;</div><div class='del'>-	u8 swqe_polarity;</div><div class='del'>-	u8 swqe_polarity_deferred;</div><div class='del'>-	u8 rwqe_polarity;</div><div class='del'>-	u8 rq_wqe_size;</div><div class='del'>-	u8 rq_wqe_size_multiplier;</div><div class='del'>-	bool first_sq_wq;</div><div class='del'>-	bool deferred_flag;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cq_uk {</div><div class='del'>-	struct i40iw_cqe *cq_base;</div><div class='del'>-	u32 __iomem *cqe_alloc_reg;</div><div class='del'>-	u64 *shadow_area;</div><div class='del'>-	u32 cq_id;</div><div class='del'>-	u32 cq_size;</div><div class='del'>-	struct i40iw_ring cq_ring;</div><div class='del'>-	u8 polarity;</div><div class='del'>-	bool avoid_mem_cflct;</div><div class='del'>-</div><div class='del'>-	struct i40iw_cq_ops ops;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_qp_uk_init_info {</div><div class='del'>-	struct i40iw_qp_quanta *sq;</div><div class='del'>-	struct i40iw_qp_quanta *rq;</div><div class='del'>-	u32 __iomem *wqe_alloc_reg;</div><div class='del'>-	u64 *shadow_area;</div><div class='del'>-	struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array;</div><div class='del'>-	u64 *rq_wrid_array;</div><div class='del'>-	u32 *push_db;</div><div class='del'>-	u64 *push_wqe;</div><div class='del'>-	u32 qp_id;</div><div class='del'>-	u32 sq_size;</div><div class='del'>-	u32 rq_size;</div><div class='del'>-	u32 max_sq_frag_cnt;</div><div class='del'>-	u32 max_rq_frag_cnt;</div><div class='del'>-	u32 max_inline_data;</div><div class='del'>-	int abi_ver;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cq_uk_init_info {</div><div class='del'>-	u32 __iomem *cqe_alloc_reg;</div><div class='del'>-	struct i40iw_cqe *cq_base;</div><div class='del'>-	u64 *shadow_area;</div><div class='del'>-	u32 cq_size;</div><div class='del'>-	u32 cq_id;</div><div class='del'>-	bool avoid_mem_cflct;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-void i40iw_device_init_uk(struct i40iw_dev_uk *dev);</div><div class='del'>-</div><div class='del'>-void i40iw_qp_post_wr(struct i40iw_qp_uk *qp);</div><div class='del'>-u64 *i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx,</div><div class='del'>-				u8 wqe_size,</div><div class='del'>-				u32 total_size,</div><div class='del'>-				u64 wr_id</div><div class='del'>-				);</div><div class='del'>-u64 *i40iw_qp_get_next_recv_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx);</div><div class='del'>-u64 *i40iw_qp_get_next_srq_wqe(struct i40iw_srq_uk *srq, u32 *wqe_idx);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_cq_uk_init(struct i40iw_cq_uk *cq,</div><div class='del'>-					struct i40iw_cq_uk_init_info *info);</div><div class='del'>-enum i40iw_status_code i40iw_qp_uk_init(struct i40iw_qp_uk *qp,</div><div class='del'>-					struct i40iw_qp_uk_init_info *info);</div><div class='del'>-</div><div class='del'>-void i40iw_clean_cq(void *queue, struct i40iw_cq_uk *cq);</div><div class='del'>-enum i40iw_status_code i40iw_nop(struct i40iw_qp_uk *qp, u64 wr_id,</div><div class='del'>-				 bool signaled, bool post_sq);</div><div class='del'>-enum i40iw_status_code i40iw_fragcnt_to_wqesize_sq(u32 frag_cnt, u8 *wqe_size);</div><div class='del'>-enum i40iw_status_code i40iw_fragcnt_to_wqesize_rq(u32 frag_cnt, u8 *wqe_size);</div><div class='del'>-enum i40iw_status_code i40iw_inline_data_size_to_wqesize(u32 data_size,</div><div class='del'>-							 u8 *wqe_size);</div><div class='del'>-void i40iw_get_wqe_shift(u32 sge, u32 inline_data, u8 *shift);</div><div class='del'>-enum i40iw_status_code i40iw_get_sqdepth(u32 sq_size, u8 shift, u32 *sqdepth);</div><div class='del'>-enum i40iw_status_code i40iw_get_rqdepth(u32 rq_size, u8 shift, u32 *rqdepth);</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_utils.c b/drivers/infiniband/hw/i40iw/i40iw_utils.c<br/>deleted file mode 100644<br/>index 337410f40860..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_utils.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_utils.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,1553 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='del'>-#include &lt;linux/moduleparam.h&gt;</div><div class='del'>-#include &lt;linux/netdevice.h&gt;</div><div class='del'>-#include &lt;linux/etherdevice.h&gt;</div><div class='del'>-#include &lt;linux/ethtool.h&gt;</div><div class='del'>-#include &lt;linux/mii.h&gt;</div><div class='del'>-#include &lt;linux/if_vlan.h&gt;</div><div class='del'>-#include &lt;linux/crc32.h&gt;</div><div class='del'>-#include &lt;linux/in.h&gt;</div><div class='del'>-#include &lt;linux/ip.h&gt;</div><div class='del'>-#include &lt;linux/tcp.h&gt;</div><div class='del'>-#include &lt;linux/init.h&gt;</div><div class='del'>-#include &lt;linux/io.h&gt;</div><div class='del'>-#include &lt;asm/irq.h&gt;</div><div class='del'>-#include &lt;asm/byteorder.h&gt;</div><div class='del'>-#include &lt;net/netevent.h&gt;</div><div class='del'>-#include &lt;net/neighbour.h&gt;</div><div class='del'>-#include "i40iw.h"</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_arp_table - manage arp table</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @ip_addr: ip address for device</div><div class='del'>- * @mac_addr: mac address ptr</div><div class='del'>- * @action: modify, delete or add</div><div class='del'>- */</div><div class='del'>-int i40iw_arp_table(struct i40iw_device *iwdev,</div><div class='del'>-		    u32 *ip_addr,</div><div class='del'>-		    bool ipv4,</div><div class='del'>-		    u8 *mac_addr,</div><div class='del'>-		    u32 action)</div><div class='del'>-{</div><div class='del'>-	int arp_index;</div><div class='del'>-	int err;</div><div class='del'>-	u32 ip[4];</div><div class='del'>-</div><div class='del'>-	if (ipv4) {</div><div class='del'>-		memset(ip, 0, sizeof(ip));</div><div class='del'>-		ip[0] = *ip_addr;</div><div class='del'>-	} else {</div><div class='del'>-		memcpy(ip, ip_addr, sizeof(ip));</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	for (arp_index = 0; (u32)arp_index &lt; iwdev-&gt;arp_table_size; arp_index++)</div><div class='del'>-		if (memcmp(iwdev-&gt;arp_table[arp_index].ip_addr, ip, sizeof(ip)) == 0)</div><div class='del'>-			break;</div><div class='del'>-	switch (action) {</div><div class='del'>-	case I40IW_ARP_ADD:</div><div class='del'>-		if (arp_index != iwdev-&gt;arp_table_size)</div><div class='del'>-			return -1;</div><div class='del'>-</div><div class='del'>-		arp_index = 0;</div><div class='del'>-		err = i40iw_alloc_resource(iwdev, iwdev-&gt;allocated_arps,</div><div class='del'>-					   iwdev-&gt;arp_table_size,</div><div class='del'>-					   (u32 *)&amp;arp_index,</div><div class='del'>-					   &amp;iwdev-&gt;next_arp_index);</div><div class='del'>-</div><div class='del'>-		if (err)</div><div class='del'>-			return err;</div><div class='del'>-</div><div class='del'>-		memcpy(iwdev-&gt;arp_table[arp_index].ip_addr, ip, sizeof(ip));</div><div class='del'>-		ether_addr_copy(iwdev-&gt;arp_table[arp_index].mac_addr, mac_addr);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_ARP_RESOLVE:</div><div class='del'>-		if (arp_index == iwdev-&gt;arp_table_size)</div><div class='del'>-			return -1;</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_ARP_DELETE:</div><div class='del'>-		if (arp_index == iwdev-&gt;arp_table_size)</div><div class='del'>-			return -1;</div><div class='del'>-		memset(iwdev-&gt;arp_table[arp_index].ip_addr, 0,</div><div class='del'>-		       sizeof(iwdev-&gt;arp_table[arp_index].ip_addr));</div><div class='del'>-		eth_zero_addr(iwdev-&gt;arp_table[arp_index].mac_addr);</div><div class='del'>-		i40iw_free_resource(iwdev, iwdev-&gt;allocated_arps, arp_index);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return -1;</div><div class='del'>-	}</div><div class='del'>-	return arp_index;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_wr32 - write 32 bits to hw register</div><div class='del'>- * @hw: hardware information including registers</div><div class='del'>- * @reg: register offset</div><div class='del'>- * @value: vvalue to write to register</div><div class='del'>- */</div><div class='del'>-inline void i40iw_wr32(struct i40iw_hw *hw, u32 reg, u32 value)</div><div class='del'>-{</div><div class='del'>-	writel(value, hw-&gt;hw_addr + reg);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_rd32 - read a 32 bit hw register</div><div class='del'>- * @hw: hardware information including registers</div><div class='del'>- * @reg: register offset</div><div class='del'>- *</div><div class='del'>- * Return value of register content</div><div class='del'>- */</div><div class='del'>-inline u32 i40iw_rd32(struct i40iw_hw *hw, u32 reg)</div><div class='del'>-{</div><div class='del'>-	return readl(hw-&gt;hw_addr + reg);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_inetaddr_event - system notifier for ipv4 addr events</div><div class='del'>- * @notfier: not used</div><div class='del'>- * @event: event for notifier</div><div class='del'>- * @ptr: if address</div><div class='del'>- */</div><div class='del'>-int i40iw_inetaddr_event(struct notifier_block *notifier,</div><div class='del'>-			 unsigned long event,</div><div class='del'>-			 void *ptr)</div><div class='del'>-{</div><div class='del'>-	struct in_ifaddr *ifa = ptr;</div><div class='del'>-	struct net_device *event_netdev = ifa-&gt;ifa_dev-&gt;dev;</div><div class='del'>-	struct net_device *netdev;</div><div class='del'>-	struct net_device *upper_dev;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_handler *hdl;</div><div class='del'>-	u32 local_ipaddr;</div><div class='del'>-	u32 action = I40IW_ARP_ADD;</div><div class='del'>-</div><div class='del'>-	hdl = i40iw_find_netdev(event_netdev);</div><div class='del'>-	if (!hdl)</div><div class='del'>-		return NOTIFY_DONE;</div><div class='del'>-</div><div class='del'>-	iwdev = &amp;hdl-&gt;device;</div><div class='del'>-	if (iwdev-&gt;init_state &lt; IP_ADDR_REGISTERED || iwdev-&gt;closing)</div><div class='del'>-		return NOTIFY_DONE;</div><div class='del'>-</div><div class='del'>-	netdev = iwdev-&gt;ldev-&gt;netdev;</div><div class='del'>-	upper_dev = netdev_master_upper_dev_get(netdev);</div><div class='del'>-	if (netdev != event_netdev)</div><div class='del'>-		return NOTIFY_DONE;</div><div class='del'>-</div><div class='del'>-	if (upper_dev) {</div><div class='del'>-		struct in_device *in;</div><div class='del'>-</div><div class='del'>-		rcu_read_lock();</div><div class='del'>-		in = __in_dev_get_rcu(upper_dev);</div><div class='del'>-</div><div class='del'>-		if (!in-&gt;ifa_list)</div><div class='del'>-			local_ipaddr = 0;</div><div class='del'>-		else</div><div class='del'>-			local_ipaddr = ntohl(in-&gt;ifa_list-&gt;ifa_address);</div><div class='del'>-</div><div class='del'>-		rcu_read_unlock();</div><div class='del'>-	} else {</div><div class='del'>-		local_ipaddr = ntohl(ifa-&gt;ifa_address);</div><div class='del'>-	}</div><div class='del'>-	switch (event) {</div><div class='del'>-	case NETDEV_DOWN:</div><div class='del'>-		action = I40IW_ARP_DELETE;</div><div class='del'>-		/* Fall through */</div><div class='del'>-	case NETDEV_UP:</div><div class='del'>-		/* Fall through */</div><div class='del'>-	case NETDEV_CHANGEADDR:</div><div class='del'>-</div><div class='del'>-		/* Just skip if no need to handle ARP cache */</div><div class='del'>-		if (!local_ipaddr)</div><div class='del'>-			break;</div><div class='del'>-</div><div class='del'>-		i40iw_manage_arp_cache(iwdev,</div><div class='del'>-				       netdev-&gt;dev_addr,</div><div class='del'>-				       &amp;local_ipaddr,</div><div class='del'>-				       true,</div><div class='del'>-				       action);</div><div class='del'>-		i40iw_if_notify(iwdev, netdev, &amp;local_ipaddr, true,</div><div class='del'>-				(action == I40IW_ARP_ADD) ? true : false);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	return NOTIFY_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_inet6addr_event - system notifier for ipv6 addr events</div><div class='del'>- * @notfier: not used</div><div class='del'>- * @event: event for notifier</div><div class='del'>- * @ptr: if address</div><div class='del'>- */</div><div class='del'>-int i40iw_inet6addr_event(struct notifier_block *notifier,</div><div class='del'>-			  unsigned long event,</div><div class='del'>-			  void *ptr)</div><div class='del'>-{</div><div class='del'>-	struct inet6_ifaddr *ifa = (struct inet6_ifaddr *)ptr;</div><div class='del'>-	struct net_device *event_netdev = ifa-&gt;idev-&gt;dev;</div><div class='del'>-	struct net_device *netdev;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_handler *hdl;</div><div class='del'>-	u32 local_ipaddr6[4];</div><div class='del'>-	u32 action = I40IW_ARP_ADD;</div><div class='del'>-</div><div class='del'>-	hdl = i40iw_find_netdev(event_netdev);</div><div class='del'>-	if (!hdl)</div><div class='del'>-		return NOTIFY_DONE;</div><div class='del'>-</div><div class='del'>-	iwdev = &amp;hdl-&gt;device;</div><div class='del'>-	if (iwdev-&gt;init_state &lt; IP_ADDR_REGISTERED || iwdev-&gt;closing)</div><div class='del'>-		return NOTIFY_DONE;</div><div class='del'>-</div><div class='del'>-	netdev = iwdev-&gt;ldev-&gt;netdev;</div><div class='del'>-	if (netdev != event_netdev)</div><div class='del'>-		return NOTIFY_DONE;</div><div class='del'>-</div><div class='del'>-	i40iw_copy_ip_ntohl(local_ipaddr6, ifa-&gt;addr.in6_u.u6_addr32);</div><div class='del'>-	switch (event) {</div><div class='del'>-	case NETDEV_DOWN:</div><div class='del'>-		action = I40IW_ARP_DELETE;</div><div class='del'>-		/* Fall through */</div><div class='del'>-	case NETDEV_UP:</div><div class='del'>-		/* Fall through */</div><div class='del'>-	case NETDEV_CHANGEADDR:</div><div class='del'>-		i40iw_manage_arp_cache(iwdev,</div><div class='del'>-				       netdev-&gt;dev_addr,</div><div class='del'>-				       local_ipaddr6,</div><div class='del'>-				       false,</div><div class='del'>-				       action);</div><div class='del'>-		i40iw_if_notify(iwdev, netdev, local_ipaddr6, false,</div><div class='del'>-				(action == I40IW_ARP_ADD) ? true : false);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	return NOTIFY_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_net_event - system notifier for netevents</div><div class='del'>- * @notfier: not used</div><div class='del'>- * @event: event for notifier</div><div class='del'>- * @ptr: neighbor</div><div class='del'>- */</div><div class='del'>-int i40iw_net_event(struct notifier_block *notifier, unsigned long event, void *ptr)</div><div class='del'>-{</div><div class='del'>-	struct neighbour *neigh = ptr;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_handler *iwhdl;</div><div class='del'>-	__be32 *p;</div><div class='del'>-	u32 local_ipaddr[4];</div><div class='del'>-</div><div class='del'>-	switch (event) {</div><div class='del'>-	case NETEVENT_NEIGH_UPDATE:</div><div class='del'>-		iwhdl = i40iw_find_netdev((struct net_device *)neigh-&gt;dev);</div><div class='del'>-		if (!iwhdl)</div><div class='del'>-			return NOTIFY_DONE;</div><div class='del'>-		iwdev = &amp;iwhdl-&gt;device;</div><div class='del'>-		if (iwdev-&gt;init_state &lt; IP_ADDR_REGISTERED || iwdev-&gt;closing)</div><div class='del'>-			return NOTIFY_DONE;</div><div class='del'>-		p = (__be32 *)neigh-&gt;primary_key;</div><div class='del'>-		i40iw_copy_ip_ntohl(local_ipaddr, p);</div><div class='del'>-		if (neigh-&gt;nud_state &amp; NUD_VALID) {</div><div class='del'>-			i40iw_manage_arp_cache(iwdev,</div><div class='del'>-					       neigh-&gt;ha,</div><div class='del'>-					       local_ipaddr,</div><div class='del'>-					       false,</div><div class='del'>-					       I40IW_ARP_ADD);</div><div class='del'>-</div><div class='del'>-		} else {</div><div class='del'>-			i40iw_manage_arp_cache(iwdev,</div><div class='del'>-					       neigh-&gt;ha,</div><div class='del'>-					       local_ipaddr,</div><div class='del'>-					       false,</div><div class='del'>-					       I40IW_ARP_DELETE);</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	return NOTIFY_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_netdevice_event - system notifier for netdev events</div><div class='del'>- * @notfier: not used</div><div class='del'>- * @event: event for notifier</div><div class='del'>- * @ptr: netdev</div><div class='del'>- */</div><div class='del'>-int i40iw_netdevice_event(struct notifier_block *notifier,</div><div class='del'>-			  unsigned long event,</div><div class='del'>-			  void *ptr)</div><div class='del'>-{</div><div class='del'>-	struct net_device *event_netdev;</div><div class='del'>-	struct net_device *netdev;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_handler *hdl;</div><div class='del'>-</div><div class='del'>-	event_netdev = netdev_notifier_info_to_dev(ptr);</div><div class='del'>-</div><div class='del'>-	hdl = i40iw_find_netdev(event_netdev);</div><div class='del'>-	if (!hdl)</div><div class='del'>-		return NOTIFY_DONE;</div><div class='del'>-</div><div class='del'>-	iwdev = &amp;hdl-&gt;device;</div><div class='del'>-	if (iwdev-&gt;init_state &lt; RDMA_DEV_REGISTERED || iwdev-&gt;closing)</div><div class='del'>-		return NOTIFY_DONE;</div><div class='del'>-</div><div class='del'>-	netdev = iwdev-&gt;ldev-&gt;netdev;</div><div class='del'>-	if (netdev != event_netdev)</div><div class='del'>-		return NOTIFY_DONE;</div><div class='del'>-</div><div class='del'>-	iwdev-&gt;iw_status = 1;</div><div class='del'>-</div><div class='del'>-	switch (event) {</div><div class='del'>-	case NETDEV_DOWN:</div><div class='del'>-		iwdev-&gt;iw_status = 0;</div><div class='del'>-		/* Fall through */</div><div class='del'>-	case NETDEV_UP:</div><div class='del'>-		i40iw_port_ibevent(iwdev);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-	return NOTIFY_DONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_cqp_request - get cqp struct</div><div class='del'>- * @cqp: device cqp ptr</div><div class='del'>- * @wait: cqp to be used in wait mode</div><div class='del'>- */</div><div class='del'>-struct i40iw_cqp_request *i40iw_get_cqp_request(struct i40iw_cqp *cqp, bool wait)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cqp_request *cqp_request = NULL;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;cqp-&gt;req_lock, flags);</div><div class='del'>-	if (!list_empty(&amp;cqp-&gt;cqp_avail_reqs)) {</div><div class='del'>-		cqp_request = list_entry(cqp-&gt;cqp_avail_reqs.next,</div><div class='del'>-					 struct i40iw_cqp_request, list);</div><div class='del'>-		list_del_init(&amp;cqp_request-&gt;list);</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;cqp-&gt;req_lock, flags);</div><div class='del'>-	if (!cqp_request) {</div><div class='del'>-		cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);</div><div class='del'>-		if (cqp_request) {</div><div class='del'>-			cqp_request-&gt;dynamic = true;</div><div class='del'>-			INIT_LIST_HEAD(&amp;cqp_request-&gt;list);</div><div class='del'>-			init_waitqueue_head(&amp;cqp_request-&gt;waitq);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	if (!cqp_request) {</div><div class='del'>-		i40iw_pr_err("CQP Request Fail: No Memory");</div><div class='del'>-		return NULL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (wait) {</div><div class='del'>-		atomic_set(&amp;cqp_request-&gt;refcount, 2);</div><div class='del'>-		cqp_request-&gt;waiting = true;</div><div class='del'>-	} else {</div><div class='del'>-		atomic_set(&amp;cqp_request-&gt;refcount, 1);</div><div class='del'>-	}</div><div class='del'>-	return cqp_request;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_free_cqp_request - free cqp request</div><div class='del'>- * @cqp: cqp ptr</div><div class='del'>- * @cqp_request: to be put back in cqp list</div><div class='del'>- */</div><div class='del'>-void i40iw_free_cqp_request(struct i40iw_cqp *cqp, struct i40iw_cqp_request *cqp_request)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = container_of(cqp, struct i40iw_device, cqp);</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	if (cqp_request-&gt;dynamic) {</div><div class='del'>-		kfree(cqp_request);</div><div class='del'>-	} else {</div><div class='del'>-		cqp_request-&gt;request_done = false;</div><div class='del'>-		cqp_request-&gt;callback_fcn = NULL;</div><div class='del'>-		cqp_request-&gt;waiting = false;</div><div class='del'>-</div><div class='del'>-		spin_lock_irqsave(&amp;cqp-&gt;req_lock, flags);</div><div class='del'>-		list_add_tail(&amp;cqp_request-&gt;list, &amp;cqp-&gt;cqp_avail_reqs);</div><div class='del'>-		spin_unlock_irqrestore(&amp;cqp-&gt;req_lock, flags);</div><div class='del'>-	}</div><div class='del'>-	wake_up(&amp;iwdev-&gt;close_wq);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_put_cqp_request - dec ref count and free if 0</div><div class='del'>- * @cqp: cqp ptr</div><div class='del'>- * @cqp_request: to be put back in cqp list</div><div class='del'>- */</div><div class='del'>-void i40iw_put_cqp_request(struct i40iw_cqp *cqp,</div><div class='del'>-			   struct i40iw_cqp_request *cqp_request)</div><div class='del'>-{</div><div class='del'>-	if (atomic_dec_and_test(&amp;cqp_request-&gt;refcount))</div><div class='del'>-		i40iw_free_cqp_request(cqp, cqp_request);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_free_pending_cqp_request -free pending cqp request objs</div><div class='del'>- * @cqp: cqp ptr</div><div class='del'>- * @cqp_request: to be put back in cqp list</div><div class='del'>- */</div><div class='del'>-static void i40iw_free_pending_cqp_request(struct i40iw_cqp *cqp,</div><div class='del'>-					   struct i40iw_cqp_request *cqp_request)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = container_of(cqp, struct i40iw_device, cqp);</div><div class='del'>-</div><div class='del'>-	if (cqp_request-&gt;waiting) {</div><div class='del'>-		cqp_request-&gt;compl_info.error = true;</div><div class='del'>-		cqp_request-&gt;request_done = true;</div><div class='del'>-		wake_up(&amp;cqp_request-&gt;waitq);</div><div class='del'>-	}</div><div class='del'>-	i40iw_put_cqp_request(cqp, cqp_request);</div><div class='del'>-	wait_event_timeout(iwdev-&gt;close_wq,</div><div class='del'>-			   !atomic_read(&amp;cqp_request-&gt;refcount),</div><div class='del'>-			   1000);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cleanup_pending_cqp_op - clean-up cqp with no completions</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- */</div><div class='del'>-void i40iw_cleanup_pending_cqp_op(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_cqp *cqp = &amp;iwdev-&gt;cqp;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request = NULL;</div><div class='del'>-	struct cqp_commands_info *pcmdinfo = NULL;</div><div class='del'>-	u32 i, pending_work, wqe_idx;</div><div class='del'>-</div><div class='del'>-	pending_work = I40IW_RING_WORK_AVAILABLE(cqp-&gt;sc_cqp.sq_ring);</div><div class='del'>-	wqe_idx = I40IW_RING_GETCURRENT_TAIL(cqp-&gt;sc_cqp.sq_ring);</div><div class='del'>-	for (i = 0; i &lt; pending_work; i++) {</div><div class='del'>-		cqp_request = (struct i40iw_cqp_request *)(unsigned long)cqp-&gt;scratch_array[wqe_idx];</div><div class='del'>-		if (cqp_request)</div><div class='del'>-			i40iw_free_pending_cqp_request(cqp, cqp_request);</div><div class='del'>-		wqe_idx = (wqe_idx + 1) % I40IW_RING_GETSIZE(cqp-&gt;sc_cqp.sq_ring);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	while (!list_empty(&amp;dev-&gt;cqp_cmd_head)) {</div><div class='del'>-		pcmdinfo = (struct cqp_commands_info *)i40iw_remove_head(&amp;dev-&gt;cqp_cmd_head);</div><div class='del'>-		cqp_request = container_of(pcmdinfo, struct i40iw_cqp_request, info);</div><div class='del'>-		if (cqp_request)</div><div class='del'>-			i40iw_free_pending_cqp_request(cqp, cqp_request);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_free_qp - callback after destroy cqp completes</div><div class='del'>- * @cqp_request: cqp request for destroy qp</div><div class='del'>- * @num: not used</div><div class='del'>- */</div><div class='del'>-static void i40iw_free_qp(struct i40iw_cqp_request *cqp_request, u32 num)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_qp *qp = (struct i40iw_sc_qp *)cqp_request-&gt;param;</div><div class='del'>-	struct i40iw_qp *iwqp = (struct i40iw_qp *)qp-&gt;back_qp;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	u32 qp_num = iwqp-&gt;ibqp.qp_num;</div><div class='del'>-</div><div class='del'>-	iwdev = iwqp-&gt;iwdev;</div><div class='del'>-</div><div class='del'>-	i40iw_rem_pdusecount(iwqp-&gt;iwpd, iwdev);</div><div class='del'>-	i40iw_free_qp_resources(iwdev, iwqp, qp_num);</div><div class='del'>-	i40iw_rem_devusecount(iwdev);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_wait_event - wait for completion</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @cqp_request: cqp request to wait</div><div class='del'>- */</div><div class='del'>-static int i40iw_wait_event(struct i40iw_device *iwdev,</div><div class='del'>-			    struct i40iw_cqp_request *cqp_request)</div><div class='del'>-{</div><div class='del'>-	struct cqp_commands_info *info = &amp;cqp_request-&gt;info;</div><div class='del'>-	struct i40iw_cqp *iwcqp = &amp;iwdev-&gt;cqp;</div><div class='del'>-	struct i40iw_cqp_timeout cqp_timeout;</div><div class='del'>-	bool cqp_error = false;</div><div class='del'>-	int err_code = 0;</div><div class='del'>-	memset(&amp;cqp_timeout, 0, sizeof(cqp_timeout));</div><div class='del'>-	cqp_timeout.compl_cqp_cmds = iwdev-&gt;sc_dev.cqp_cmd_stats[OP_COMPLETED_COMMANDS];</div><div class='del'>-	do {</div><div class='del'>-		if (wait_event_timeout(cqp_request-&gt;waitq,</div><div class='del'>-				       cqp_request-&gt;request_done, CQP_COMPL_WAIT_TIME))</div><div class='del'>-			break;</div><div class='del'>-</div><div class='del'>-		i40iw_check_cqp_progress(&amp;cqp_timeout, &amp;iwdev-&gt;sc_dev);</div><div class='del'>-</div><div class='del'>-		if (cqp_timeout.count &lt; CQP_TIMEOUT_THRESHOLD)</div><div class='del'>-			continue;</div><div class='del'>-</div><div class='del'>-		i40iw_pr_err("error cqp command 0x%x timed out", info-&gt;cqp_cmd);</div><div class='del'>-		err_code = -ETIME;</div><div class='del'>-		if (!iwdev-&gt;reset) {</div><div class='del'>-			iwdev-&gt;reset = true;</div><div class='del'>-			i40iw_request_reset(iwdev);</div><div class='del'>-		}</div><div class='del'>-		goto done;</div><div class='del'>-	} while (1);</div><div class='del'>-	cqp_error = cqp_request-&gt;compl_info.error;</div><div class='del'>-	if (cqp_error) {</div><div class='del'>-		i40iw_pr_err("error cqp command 0x%x completion maj = 0x%x min=0x%x\n",</div><div class='del'>-			     info-&gt;cqp_cmd, cqp_request-&gt;compl_info.maj_err_code,</div><div class='del'>-			     cqp_request-&gt;compl_info.min_err_code);</div><div class='del'>-		err_code = -EPROTO;</div><div class='del'>-		goto done;</div><div class='del'>-	}</div><div class='del'>-done:</div><div class='del'>-	i40iw_put_cqp_request(iwcqp, cqp_request);</div><div class='del'>-	return err_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_handle_cqp_op - process cqp command</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @cqp_request: cqp request to process</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_handle_cqp_op(struct i40iw_device *iwdev,</div><div class='del'>-					   struct i40iw_cqp_request</div><div class='del'>-					   *cqp_request)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct cqp_commands_info *info = &amp;cqp_request-&gt;info;</div><div class='del'>-	int err_code = 0;</div><div class='del'>-</div><div class='del'>-	if (iwdev-&gt;reset) {</div><div class='del'>-		i40iw_free_cqp_request(&amp;iwdev-&gt;cqp, cqp_request);</div><div class='del'>-		return I40IW_ERR_CQP_COMPL_ERROR;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	status = i40iw_process_cqp_cmd(dev, info);</div><div class='del'>-	if (status) {</div><div class='del'>-		i40iw_pr_err("error cqp command 0x%x failed\n", info-&gt;cqp_cmd);</div><div class='del'>-		i40iw_free_cqp_request(&amp;iwdev-&gt;cqp, cqp_request);</div><div class='del'>-		return status;</div><div class='del'>-	}</div><div class='del'>-	if (cqp_request-&gt;waiting)</div><div class='del'>-		err_code = i40iw_wait_event(iwdev, cqp_request);</div><div class='del'>-	if (err_code)</div><div class='del'>-		status = I40IW_ERR_CQP_COMPL_ERROR;</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_add_devusecount - add dev refcount</div><div class='del'>- * @iwdev: dev for refcount</div><div class='del'>- */</div><div class='del'>-void i40iw_add_devusecount(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	atomic64_inc(&amp;iwdev-&gt;use_count);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_rem_devusecount - decrement refcount for dev</div><div class='del'>- * @iwdev: device</div><div class='del'>- */</div><div class='del'>-void i40iw_rem_devusecount(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	if (!atomic64_dec_and_test(&amp;iwdev-&gt;use_count))</div><div class='del'>-		return;</div><div class='del'>-	wake_up(&amp;iwdev-&gt;close_wq);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_add_pdusecount - add pd refcount</div><div class='del'>- * @iwpd: pd for refcount</div><div class='del'>- */</div><div class='del'>-void i40iw_add_pdusecount(struct i40iw_pd *iwpd)</div><div class='del'>-{</div><div class='del'>-	atomic_inc(&amp;iwpd-&gt;usecount);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_rem_pdusecount - decrement refcount for pd and free if 0</div><div class='del'>- * @iwpd: pd for refcount</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- */</div><div class='del'>-void i40iw_rem_pdusecount(struct i40iw_pd *iwpd, struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	if (!atomic_dec_and_test(&amp;iwpd-&gt;usecount))</div><div class='del'>-		return;</div><div class='del'>-	i40iw_free_resource(iwdev, iwdev-&gt;allocated_pds, iwpd-&gt;sc_pd.pd_id);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_add_ref - add refcount for qp</div><div class='del'>- * @ibqp: iqarp qp</div><div class='del'>- */</div><div class='del'>-void i40iw_add_ref(struct ib_qp *ibqp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp *iwqp = (struct i40iw_qp *)ibqp;</div><div class='del'>-</div><div class='del'>-	atomic_inc(&amp;iwqp-&gt;refcount);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_rem_ref - rem refcount for qp and free if 0</div><div class='del'>- * @ibqp: iqarp qp</div><div class='del'>- */</div><div class='del'>-void i40iw_rem_ref(struct ib_qp *ibqp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	u32 qp_num;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	iwqp = to_iwqp(ibqp);</div><div class='del'>-	iwdev = iwqp-&gt;iwdev;</div><div class='del'>-	spin_lock_irqsave(&amp;iwdev-&gt;qptable_lock, flags);</div><div class='del'>-	if (!atomic_dec_and_test(&amp;iwqp-&gt;refcount)) {</div><div class='del'>-		spin_unlock_irqrestore(&amp;iwdev-&gt;qptable_lock, flags);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	qp_num = iwqp-&gt;ibqp.qp_num;</div><div class='del'>-	iwdev-&gt;qp_table[qp_num] = NULL;</div><div class='del'>-	spin_unlock_irqrestore(&amp;iwdev-&gt;qptable_lock, flags);</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, false);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	cqp_request-&gt;callback_fcn = i40iw_free_qp;</div><div class='del'>-	cqp_request-&gt;param = (void *)&amp;iwqp-&gt;sc_qp;</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_QP_DESTROY;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.qp_destroy.qp = &amp;iwqp-&gt;sc_qp;</div><div class='del'>-	cqp_info-&gt;in.u.qp_destroy.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	cqp_info-&gt;in.u.qp_destroy.remove_hash_idx = true;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (!status)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	i40iw_rem_pdusecount(iwqp-&gt;iwpd, iwdev);</div><div class='del'>-	i40iw_free_qp_resources(iwdev, iwqp, qp_num);</div><div class='del'>-	i40iw_rem_devusecount(iwdev);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_qp - get qp address</div><div class='del'>- * @device: iwarp device</div><div class='del'>- * @qpn: qp number</div><div class='del'>- */</div><div class='del'>-struct ib_qp *i40iw_get_qp(struct ib_device *device, int qpn)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = to_iwdev(device);</div><div class='del'>-</div><div class='del'>-	if ((qpn &lt; IW_FIRST_QPN) || (qpn &gt;= iwdev-&gt;max_qp))</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	return &amp;iwdev-&gt;qp_table[qpn]-&gt;ibqp;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_debug_buf - print debug msg and buffer is mask set</div><div class='del'>- * @dev: hardware control device structure</div><div class='del'>- * @mask: mask to compare if to print debug buffer</div><div class='del'>- * @buf: points buffer addr</div><div class='del'>- * @size: saize of buffer to print</div><div class='del'>- */</div><div class='del'>-void i40iw_debug_buf(struct i40iw_sc_dev *dev,</div><div class='del'>-		     enum i40iw_debug_flag mask,</div><div class='del'>-		     char *desc,</div><div class='del'>-		     u64 *buf,</div><div class='del'>-		     u32 size)</div><div class='del'>-{</div><div class='del'>-	u32 i;</div><div class='del'>-</div><div class='del'>-	if (!(dev-&gt;debug_mask &amp; mask))</div><div class='del'>-		return;</div><div class='del'>-	i40iw_debug(dev, mask, "%s\n", desc);</div><div class='del'>-	i40iw_debug(dev, mask, "starting address virt=%p phy=%llxh\n", buf,</div><div class='del'>-		    (unsigned long long)virt_to_phys(buf));</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; size; i += 8)</div><div class='del'>-		i40iw_debug(dev, mask, "index %03d val: %016llx\n", i, buf[i / 8]);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_hw_addr - return hw addr</div><div class='del'>- * @par: points to shared dev</div><div class='del'>- */</div><div class='del'>-u8 __iomem *i40iw_get_hw_addr(void *par)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_dev *dev = (struct i40iw_sc_dev *)par;</div><div class='del'>-</div><div class='del'>-	return dev-&gt;hw-&gt;hw_addr;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_remove_head - return head entry and remove from list</div><div class='del'>- * @list: list for entry</div><div class='del'>- */</div><div class='del'>-void *i40iw_remove_head(struct list_head *list)</div><div class='del'>-{</div><div class='del'>-	struct list_head *entry;</div><div class='del'>-</div><div class='del'>-	if (list_empty(list))</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	entry = (void *)list-&gt;next;</div><div class='del'>-	list_del(entry);</div><div class='del'>-	return (void *)entry;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_allocate_dma_mem - Memory alloc helper fn</div><div class='del'>- * @hw:   pointer to the HW structure</div><div class='del'>- * @mem:  ptr to mem struct to fill out</div><div class='del'>- * @size: size of memory requested</div><div class='del'>- * @alignment: what to align the allocation to</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_allocate_dma_mem(struct i40iw_hw *hw,</div><div class='del'>-					      struct i40iw_dma_mem *mem,</div><div class='del'>-					      u64 size,</div><div class='del'>-					      u32 alignment)</div><div class='del'>-{</div><div class='del'>-	struct pci_dev *pcidev = (struct pci_dev *)hw-&gt;dev_context;</div><div class='del'>-</div><div class='del'>-	if (!mem)</div><div class='del'>-		return I40IW_ERR_PARAM;</div><div class='del'>-	mem-&gt;size = ALIGN(size, alignment);</div><div class='del'>-	mem-&gt;va = dma_alloc_coherent(&amp;pcidev-&gt;dev, mem-&gt;size,</div><div class='del'>-				     (dma_addr_t *)&amp;mem-&gt;pa, GFP_KERNEL);</div><div class='del'>-	if (!mem-&gt;va)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_free_dma_mem - Memory free helper fn</div><div class='del'>- * @hw:   pointer to the HW structure</div><div class='del'>- * @mem:  ptr to mem struct to free</div><div class='del'>- */</div><div class='del'>-void i40iw_free_dma_mem(struct i40iw_hw *hw, struct i40iw_dma_mem *mem)</div><div class='del'>-{</div><div class='del'>-	struct pci_dev *pcidev = (struct pci_dev *)hw-&gt;dev_context;</div><div class='del'>-</div><div class='del'>-	if (!mem || !mem-&gt;va)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	dma_free_coherent(&amp;pcidev-&gt;dev, mem-&gt;size,</div><div class='del'>-			  mem-&gt;va, (dma_addr_t)mem-&gt;pa);</div><div class='del'>-	mem-&gt;va = NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_allocate_virt_mem - virtual memory alloc helper fn</div><div class='del'>- * @hw:   pointer to the HW structure</div><div class='del'>- * @mem:  ptr to mem struct to fill out</div><div class='del'>- * @size: size of memory requested</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_allocate_virt_mem(struct i40iw_hw *hw,</div><div class='del'>-					       struct i40iw_virt_mem *mem,</div><div class='del'>-					       u32 size)</div><div class='del'>-{</div><div class='del'>-	if (!mem)</div><div class='del'>-		return I40IW_ERR_PARAM;</div><div class='del'>-</div><div class='del'>-	mem-&gt;size = size;</div><div class='del'>-	mem-&gt;va = kzalloc(size, GFP_KERNEL);</div><div class='del'>-</div><div class='del'>-	if (mem-&gt;va)</div><div class='del'>-		return 0;</div><div class='del'>-	else</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_free_virt_mem - virtual memory free helper fn</div><div class='del'>- * @hw:   pointer to the HW structure</div><div class='del'>- * @mem:  ptr to mem struct to free</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_free_virt_mem(struct i40iw_hw *hw,</div><div class='del'>-					   struct i40iw_virt_mem *mem)</div><div class='del'>-{</div><div class='del'>-	if (!mem)</div><div class='del'>-		return I40IW_ERR_PARAM;</div><div class='del'>-	/*</div><div class='del'>-	 * mem-&gt;va points to the parent of mem, so both mem and mem-&gt;va</div><div class='del'>-	 * can not be touched once mem-&gt;va is freed</div><div class='del'>-	 */</div><div class='del'>-	kfree(mem-&gt;va);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cqp_sds_cmd - create cqp command for sd</div><div class='del'>- * @dev: hardware control device structure</div><div class='del'>- * @sd_info: information  for sd cqp</div><div class='del'>- *</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_cqp_sds_cmd(struct i40iw_sc_dev *dev,</div><div class='del'>-					 struct i40iw_update_sds_info *sdinfo)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)dev-&gt;back_dev;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, true);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	memcpy(&amp;cqp_info-&gt;in.u.update_pe_sds.info, sdinfo,</div><div class='del'>-	       sizeof(cqp_info-&gt;in.u.update_pe_sds.info));</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_UPDATE_PE_SDS;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.update_pe_sds.dev = dev;</div><div class='del'>-	cqp_info-&gt;in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("CQP-OP Update SD's fail");</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_qp_suspend_resume - cqp command for suspend/resume</div><div class='del'>- * @dev: hardware control device structure</div><div class='del'>- * @qp: hardware control qp</div><div class='del'>- * @suspend: flag if suspend or resume</div><div class='del'>- */</div><div class='del'>-void i40iw_qp_suspend_resume(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp, bool suspend)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)dev-&gt;back_dev;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct i40iw_sc_cqp *cqp = dev-&gt;cqp;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, false);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	cqp_info-&gt;cqp_cmd = (suspend) ? OP_SUSPEND : OP_RESUME;</div><div class='del'>-	cqp_info-&gt;in.u.suspend_resume.cqp = cqp;</div><div class='del'>-	cqp_info-&gt;in.u.suspend_resume.qp = qp;</div><div class='del'>-	cqp_info-&gt;in.u.suspend_resume.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("CQP-OP QP Suspend/Resume fail");</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_term_modify_qp - modify qp for term message</div><div class='del'>- * @qp: hardware control qp</div><div class='del'>- * @next_state: qp's next state</div><div class='del'>- * @term: terminate code</div><div class='del'>- * @term_len: length</div><div class='del'>- */</div><div class='del'>-void i40iw_term_modify_qp(struct i40iw_sc_qp *qp, u8 next_state, u8 term, u8 term_len)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-</div><div class='del'>-	iwqp = (struct i40iw_qp *)qp-&gt;back_qp;</div><div class='del'>-	i40iw_next_iw_state(iwqp, next_state, 0, term, term_len);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_terminate_done - after terminate is completed</div><div class='del'>- * @qp: hardware control qp</div><div class='del'>- * @timeout_occurred: indicates if terminate timer expired</div><div class='del'>- */</div><div class='del'>-void i40iw_terminate_done(struct i40iw_sc_qp *qp, int timeout_occurred)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-	u32 next_iwarp_state = I40IW_QP_STATE_ERROR;</div><div class='del'>-	u8 hte = 0;</div><div class='del'>-	bool first_time;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	iwqp = (struct i40iw_qp *)qp-&gt;back_qp;</div><div class='del'>-	spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-	if (iwqp-&gt;hte_added) {</div><div class='del'>-		iwqp-&gt;hte_added = 0;</div><div class='del'>-		hte = 1;</div><div class='del'>-	}</div><div class='del'>-	first_time = !(qp-&gt;term_flags &amp; I40IW_TERM_DONE);</div><div class='del'>-	qp-&gt;term_flags |= I40IW_TERM_DONE;</div><div class='del'>-	spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-	if (first_time) {</div><div class='del'>-		if (!timeout_occurred)</div><div class='del'>-			i40iw_terminate_del_timer(qp);</div><div class='del'>-		else</div><div class='del'>-			next_iwarp_state = I40IW_QP_STATE_CLOSING;</div><div class='del'>-</div><div class='del'>-		i40iw_next_iw_state(iwqp, next_iwarp_state, hte, 0, 0);</div><div class='del'>-		i40iw_cm_disconn(iwqp);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_terminate_imeout - timeout happened</div><div class='del'>- * @context: points to iwarp qp</div><div class='del'>- */</div><div class='del'>-static void i40iw_terminate_timeout(struct timer_list *t)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp *iwqp = from_timer(iwqp, t, terminate_timer);</div><div class='del'>-	struct i40iw_sc_qp *qp = (struct i40iw_sc_qp *)&amp;iwqp-&gt;sc_qp;</div><div class='del'>-</div><div class='del'>-	i40iw_terminate_done(qp, 1);</div><div class='del'>-	i40iw_rem_ref(&amp;iwqp-&gt;ibqp);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_terminate_start_timer - start terminate timeout</div><div class='del'>- * @qp: hardware control qp</div><div class='del'>- */</div><div class='del'>-void i40iw_terminate_start_timer(struct i40iw_sc_qp *qp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-</div><div class='del'>-	iwqp = (struct i40iw_qp *)qp-&gt;back_qp;</div><div class='del'>-	i40iw_add_ref(&amp;iwqp-&gt;ibqp);</div><div class='del'>-	timer_setup(&amp;iwqp-&gt;terminate_timer, i40iw_terminate_timeout, 0);</div><div class='del'>-	iwqp-&gt;terminate_timer.expires = jiffies + HZ;</div><div class='del'>-	add_timer(&amp;iwqp-&gt;terminate_timer);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_terminate_del_timer - delete terminate timeout</div><div class='del'>- * @qp: hardware control qp</div><div class='del'>- */</div><div class='del'>-void i40iw_terminate_del_timer(struct i40iw_sc_qp *qp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-</div><div class='del'>-	iwqp = (struct i40iw_qp *)qp-&gt;back_qp;</div><div class='del'>-	if (del_timer(&amp;iwqp-&gt;terminate_timer))</div><div class='del'>-		i40iw_rem_ref(&amp;iwqp-&gt;ibqp);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cqp_generic_worker - generic worker for cqp</div><div class='del'>- * @work: work pointer</div><div class='del'>- */</div><div class='del'>-static void i40iw_cqp_generic_worker(struct work_struct *work)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_virtchnl_work_info *work_info =</div><div class='del'>-	    &amp;((struct virtchnl_work *)work)-&gt;work_info;</div><div class='del'>-</div><div class='del'>-	if (work_info-&gt;worker_vf_dev)</div><div class='del'>-		work_info-&gt;callback_fcn(work_info-&gt;worker_vf_dev);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cqp_spawn_worker - spawn worket thread</div><div class='del'>- * @iwdev: device struct pointer</div><div class='del'>- * @work_info: work request info</div><div class='del'>- * @iw_vf_idx: virtual function index</div><div class='del'>- */</div><div class='del'>-void i40iw_cqp_spawn_worker(struct i40iw_sc_dev *dev,</div><div class='del'>-			    struct i40iw_virtchnl_work_info *work_info,</div><div class='del'>-			    u32 iw_vf_idx)</div><div class='del'>-{</div><div class='del'>-	struct virtchnl_work *work;</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)dev-&gt;back_dev;</div><div class='del'>-</div><div class='del'>-	work = &amp;iwdev-&gt;virtchnl_w[iw_vf_idx];</div><div class='del'>-	memcpy(&amp;work-&gt;work_info, work_info, sizeof(*work_info));</div><div class='del'>-	INIT_WORK(&amp;work-&gt;work, i40iw_cqp_generic_worker);</div><div class='del'>-	queue_work(iwdev-&gt;virtchnl_wq, &amp;work-&gt;work);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cqp_manage_hmc_fcn_worker -</div><div class='del'>- * @work: work pointer for hmc info</div><div class='del'>- */</div><div class='del'>-static void i40iw_cqp_manage_hmc_fcn_worker(struct work_struct *work)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cqp_request *cqp_request =</div><div class='del'>-	    ((struct virtchnl_work *)work)-&gt;cqp_request;</div><div class='del'>-	struct i40iw_ccq_cqe_info ccq_cqe_info;</div><div class='del'>-	struct i40iw_hmc_fcn_info *hmcfcninfo =</div><div class='del'>-			&amp;cqp_request-&gt;info.in.u.manage_hmc_pm.info;</div><div class='del'>-	struct i40iw_device *iwdev =</div><div class='del'>-	    (struct i40iw_device *)cqp_request-&gt;info.in.u.manage_hmc_pm.dev-&gt;back_dev;</div><div class='del'>-</div><div class='del'>-	ccq_cqe_info.cqp = NULL;</div><div class='del'>-	ccq_cqe_info.maj_err_code = cqp_request-&gt;compl_info.maj_err_code;</div><div class='del'>-	ccq_cqe_info.min_err_code = cqp_request-&gt;compl_info.min_err_code;</div><div class='del'>-	ccq_cqe_info.op_code = cqp_request-&gt;compl_info.op_code;</div><div class='del'>-	ccq_cqe_info.op_ret_val = cqp_request-&gt;compl_info.op_ret_val;</div><div class='del'>-	ccq_cqe_info.scratch = 0;</div><div class='del'>-	ccq_cqe_info.error = cqp_request-&gt;compl_info.error;</div><div class='del'>-	hmcfcninfo-&gt;callback_fcn(cqp_request-&gt;info.in.u.manage_hmc_pm.dev,</div><div class='del'>-				 hmcfcninfo-&gt;cqp_callback_param, &amp;ccq_cqe_info);</div><div class='del'>-	i40iw_put_cqp_request(&amp;iwdev-&gt;cqp, cqp_request);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cqp_manage_hmc_fcn_callback - called function after cqp completion</div><div class='del'>- * @cqp_request: cqp request info struct for hmc fun</div><div class='del'>- * @unused: unused param of callback</div><div class='del'>- */</div><div class='del'>-static void i40iw_cqp_manage_hmc_fcn_callback(struct i40iw_cqp_request *cqp_request,</div><div class='del'>-					      u32 unused)</div><div class='del'>-{</div><div class='del'>-	struct virtchnl_work *work;</div><div class='del'>-	struct i40iw_hmc_fcn_info *hmcfcninfo =</div><div class='del'>-	    &amp;cqp_request-&gt;info.in.u.manage_hmc_pm.info;</div><div class='del'>-	struct i40iw_device *iwdev =</div><div class='del'>-	    (struct i40iw_device *)cqp_request-&gt;info.in.u.manage_hmc_pm.dev-&gt;</div><div class='del'>-	    back_dev;</div><div class='del'>-</div><div class='del'>-	if (hmcfcninfo &amp;&amp; hmcfcninfo-&gt;callback_fcn) {</div><div class='del'>-		i40iw_debug(&amp;iwdev-&gt;sc_dev, I40IW_DEBUG_HMC, "%s1\n", __func__);</div><div class='del'>-		atomic_inc(&amp;cqp_request-&gt;refcount);</div><div class='del'>-		work = &amp;iwdev-&gt;virtchnl_w[hmcfcninfo-&gt;iw_vf_idx];</div><div class='del'>-		work-&gt;cqp_request = cqp_request;</div><div class='del'>-		INIT_WORK(&amp;work-&gt;work, i40iw_cqp_manage_hmc_fcn_worker);</div><div class='del'>-		queue_work(iwdev-&gt;virtchnl_wq, &amp;work-&gt;work);</div><div class='del'>-		i40iw_debug(&amp;iwdev-&gt;sc_dev, I40IW_DEBUG_HMC, "%s2\n", __func__);</div><div class='del'>-	} else {</div><div class='del'>-		i40iw_debug(&amp;iwdev-&gt;sc_dev, I40IW_DEBUG_HMC, "%s: Something wrong\n", __func__);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cqp_manage_hmc_fcn_cmd - issue cqp command to manage hmc</div><div class='del'>- * @dev: hardware control device structure</div><div class='del'>- * @hmcfcninfo: info for hmc</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_cqp_manage_hmc_fcn_cmd(struct i40iw_sc_dev *dev,</div><div class='del'>-						    struct i40iw_hmc_fcn_info *hmcfcninfo)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)dev-&gt;back_dev;</div><div class='del'>-</div><div class='del'>-	i40iw_debug(&amp;iwdev-&gt;sc_dev, I40IW_DEBUG_HMC, "%s\n", __func__);</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, false);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	cqp_request-&gt;callback_fcn = i40iw_cqp_manage_hmc_fcn_callback;</div><div class='del'>-	cqp_request-&gt;param = hmcfcninfo;</div><div class='del'>-	memcpy(&amp;cqp_info-&gt;in.u.manage_hmc_pm.info, hmcfcninfo,</div><div class='del'>-	       sizeof(*hmcfcninfo));</div><div class='del'>-	cqp_info-&gt;in.u.manage_hmc_pm.dev = dev;</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_MANAGE_HMC_PM_FUNC_TABLE;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.manage_hmc_pm.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("CQP-OP Manage HMC fail");</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cqp_query_fpm_values_cmd - send cqp command for fpm</div><div class='del'>- * @iwdev: function device struct</div><div class='del'>- * @values_mem: buffer for fpm</div><div class='del'>- * @hmc_fn_id: function id for fpm</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_cqp_query_fpm_values_cmd(struct i40iw_sc_dev *dev,</div><div class='del'>-						      struct i40iw_dma_mem *values_mem,</div><div class='del'>-						      u8 hmc_fn_id)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)dev-&gt;back_dev;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, true);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	cqp_request-&gt;param = NULL;</div><div class='del'>-	cqp_info-&gt;in.u.query_fpm_values.cqp = dev-&gt;cqp;</div><div class='del'>-	cqp_info-&gt;in.u.query_fpm_values.fpm_values_pa = values_mem-&gt;pa;</div><div class='del'>-	cqp_info-&gt;in.u.query_fpm_values.fpm_values_va = values_mem-&gt;va;</div><div class='del'>-	cqp_info-&gt;in.u.query_fpm_values.hmc_fn_id = hmc_fn_id;</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_QUERY_FPM_VALUES;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.query_fpm_values.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("CQP-OP Query FPM fail");</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cqp_commit_fpm_values_cmd - commit fpm values in hw</div><div class='del'>- * @dev: hardware control device structure</div><div class='del'>- * @values_mem: buffer with fpm values</div><div class='del'>- * @hmc_fn_id: function id for fpm</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_cqp_commit_fpm_values_cmd(struct i40iw_sc_dev *dev,</div><div class='del'>-						       struct i40iw_dma_mem *values_mem,</div><div class='del'>-						       u8 hmc_fn_id)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)dev-&gt;back_dev;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, true);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	cqp_request-&gt;param = NULL;</div><div class='del'>-	cqp_info-&gt;in.u.commit_fpm_values.cqp = dev-&gt;cqp;</div><div class='del'>-	cqp_info-&gt;in.u.commit_fpm_values.fpm_values_pa = values_mem-&gt;pa;</div><div class='del'>-	cqp_info-&gt;in.u.commit_fpm_values.fpm_values_va = values_mem-&gt;va;</div><div class='del'>-	cqp_info-&gt;in.u.commit_fpm_values.hmc_fn_id = hmc_fn_id;</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_COMMIT_FPM_VALUES;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.commit_fpm_values.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("CQP-OP Commit FPM fail");</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vf_wait_vchnl_resp - wait for channel msg</div><div class='del'>- * @iwdev: function's device struct</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_vf_wait_vchnl_resp(struct i40iw_sc_dev *dev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = dev-&gt;back_dev;</div><div class='del'>-	int timeout_ret;</div><div class='del'>-</div><div class='del'>-	i40iw_debug(dev, I40IW_DEBUG_VIRT, "%s[%u] dev %p, iwdev %p\n",</div><div class='del'>-		    __func__, __LINE__, dev, iwdev);</div><div class='del'>-</div><div class='del'>-	atomic_set(&amp;iwdev-&gt;vchnl_msgs, 2);</div><div class='del'>-	timeout_ret = wait_event_timeout(iwdev-&gt;vchnl_waitq,</div><div class='del'>-					 (atomic_read(&amp;iwdev-&gt;vchnl_msgs) == 1),</div><div class='del'>-					 I40IW_VCHNL_EVENT_TIMEOUT);</div><div class='del'>-	atomic_dec(&amp;iwdev-&gt;vchnl_msgs);</div><div class='del'>-	if (!timeout_ret) {</div><div class='del'>-		i40iw_pr_err("virt channel completion timeout = 0x%x\n", timeout_ret);</div><div class='del'>-		atomic_set(&amp;iwdev-&gt;vchnl_msgs, 0);</div><div class='del'>-		dev-&gt;vchnl_up = false;</div><div class='del'>-		return I40IW_ERR_TIMEOUT;</div><div class='del'>-	}</div><div class='del'>-	wake_up(&amp;dev-&gt;vf_reqs);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cqp_cq_create_cmd - create a cq for the cqp</div><div class='del'>- * @dev: device pointer</div><div class='del'>- * @cq: pointer to created cq</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_cqp_cq_create_cmd(struct i40iw_sc_dev *dev,</div><div class='del'>-					       struct i40iw_sc_cq *cq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)dev-&gt;back_dev;</div><div class='del'>-	struct i40iw_cqp *iwcqp = &amp;iwdev-&gt;cqp;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(iwcqp, true);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_CQ_CREATE;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.cq_create.cq = cq;</div><div class='del'>-	cqp_info-&gt;in.u.cq_create.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("CQP-OP Create QP fail");</div><div class='del'>-</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cqp_qp_create_cmd - create a qp for the cqp</div><div class='del'>- * @dev: device pointer</div><div class='del'>- * @qp: pointer to created qp</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_cqp_qp_create_cmd(struct i40iw_sc_dev *dev,</div><div class='del'>-					       struct i40iw_sc_qp *qp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)dev-&gt;back_dev;</div><div class='del'>-	struct i40iw_cqp *iwcqp = &amp;iwdev-&gt;cqp;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	struct i40iw_create_qp_info *qp_info;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(iwcqp, true);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return I40IW_ERR_NO_MEMORY;</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	qp_info = &amp;cqp_request-&gt;info.in.u.qp_create.info;</div><div class='del'>-</div><div class='del'>-	memset(qp_info, 0, sizeof(*qp_info));</div><div class='del'>-</div><div class='del'>-	qp_info-&gt;cq_num_valid = true;</div><div class='del'>-	qp_info-&gt;next_iwarp_state = I40IW_QP_STATE_RTS;</div><div class='del'>-</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_QP_CREATE;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.qp_create.qp = qp;</div><div class='del'>-	cqp_info-&gt;in.u.qp_create.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("CQP-OP QP create fail");</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cqp_cq_destroy_cmd - destroy the cqp cq</div><div class='del'>- * @dev: device pointer</div><div class='del'>- * @cq: pointer to cq</div><div class='del'>- */</div><div class='del'>-void i40iw_cqp_cq_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)dev-&gt;back_dev;</div><div class='del'>-</div><div class='del'>-	i40iw_cq_wq_destroy(iwdev, cq);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cqp_qp_destroy_cmd - destroy the cqp</div><div class='del'>- * @dev: device pointer</div><div class='del'>- * @qp: pointer to qp</div><div class='del'>- */</div><div class='del'>-void i40iw_cqp_qp_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)dev-&gt;back_dev;</div><div class='del'>-	struct i40iw_cqp *iwcqp = &amp;iwdev-&gt;cqp;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(iwcqp, true);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	memset(cqp_info, 0, sizeof(*cqp_info));</div><div class='del'>-</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_QP_DESTROY;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.qp_destroy.qp = qp;</div><div class='del'>-	cqp_info-&gt;in.u.qp_destroy.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	cqp_info-&gt;in.u.qp_destroy.remove_hash_idx = true;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("CQP QP_DESTROY fail");</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_mpa_crc_ae - generate AE for crc error</div><div class='del'>- * @dev: hardware control device structure</div><div class='del'>- * @qp: hardware control qp</div><div class='del'>- */</div><div class='del'>-void i40iw_ieq_mpa_crc_ae(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_gen_ae_info info;</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)dev-&gt;back_dev;</div><div class='del'>-</div><div class='del'>-	i40iw_debug(dev, I40IW_DEBUG_AEQ, "%s entered\n", __func__);</div><div class='del'>-	info.ae_code = I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR;</div><div class='del'>-	info.ae_source = I40IW_AE_SOURCE_RQ;</div><div class='del'>-	i40iw_gen_ae(iwdev, qp, &amp;info, false);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_init_hash_desc - initialize hash for crc calculation</div><div class='del'>- * @desc: cryption type</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_init_hash_desc(struct shash_desc **desc)</div><div class='del'>-{</div><div class='del'>-	struct crypto_shash *tfm;</div><div class='del'>-	struct shash_desc *tdesc;</div><div class='del'>-</div><div class='del'>-	tfm = crypto_alloc_shash("crc32c", 0, 0);</div><div class='del'>-	if (IS_ERR(tfm))</div><div class='del'>-		return I40IW_ERR_MPA_CRC;</div><div class='del'>-</div><div class='del'>-	tdesc = kzalloc(sizeof(*tdesc) + crypto_shash_descsize(tfm),</div><div class='del'>-			GFP_KERNEL);</div><div class='del'>-	if (!tdesc) {</div><div class='del'>-		crypto_free_shash(tfm);</div><div class='del'>-		return I40IW_ERR_MPA_CRC;</div><div class='del'>-	}</div><div class='del'>-	tdesc-&gt;tfm = tfm;</div><div class='del'>-	*desc = tdesc;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_free_hash_desc - free hash desc</div><div class='del'>- * @desc: to be freed</div><div class='del'>- */</div><div class='del'>-void i40iw_free_hash_desc(struct shash_desc *desc)</div><div class='del'>-{</div><div class='del'>-	if (desc) {</div><div class='del'>-		crypto_free_shash(desc-&gt;tfm);</div><div class='del'>-		kfree(desc);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_alloc_query_fpm_buf - allocate buffer for fpm</div><div class='del'>- * @dev: hardware control device structure</div><div class='del'>- * @mem: buffer ptr for fpm to be allocated</div><div class='del'>- * @return: memory allocation status</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_alloc_query_fpm_buf(struct i40iw_sc_dev *dev,</div><div class='del'>-						 struct i40iw_dma_mem *mem)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)dev-&gt;back_dev;</div><div class='del'>-</div><div class='del'>-	status = i40iw_obj_aligned_mem(iwdev, mem, I40IW_QUERY_FPM_BUF_SIZE,</div><div class='del'>-				       I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK);</div><div class='del'>-	return status;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_check_mpacrc - check if mpa crc is OK</div><div class='del'>- * @desc: desc for hash</div><div class='del'>- * @addr: address of buffer for crc</div><div class='del'>- * @length: length of buffer</div><div class='del'>- * @value: value to be compared</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_ieq_check_mpacrc(struct shash_desc *desc,</div><div class='del'>-					      void *addr,</div><div class='del'>-					      u32 length,</div><div class='del'>-					      u32 value)</div><div class='del'>-{</div><div class='del'>-	u32 crc = 0;</div><div class='del'>-	int ret;</div><div class='del'>-	enum i40iw_status_code ret_code = 0;</div><div class='del'>-</div><div class='del'>-	crypto_shash_init(desc);</div><div class='del'>-	ret = crypto_shash_update(desc, addr, length);</div><div class='del'>-	if (!ret)</div><div class='del'>-		crypto_shash_final(desc, (u8 *)&amp;crc);</div><div class='del'>-	if (crc != value) {</div><div class='del'>-		i40iw_pr_err("mpa crc check fail\n");</div><div class='del'>-		ret_code = I40IW_ERR_MPA_CRC;</div><div class='del'>-	}</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_get_qp - get qp based on quad in puda buffer</div><div class='del'>- * @dev: hardware control device structure</div><div class='del'>- * @buf: receive puda buffer on exception q</div><div class='del'>- */</div><div class='del'>-struct i40iw_sc_qp *i40iw_ieq_get_qp(struct i40iw_sc_dev *dev,</div><div class='del'>-				     struct i40iw_puda_buf *buf)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = (struct i40iw_device *)dev-&gt;back_dev;</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-	struct i40iw_cm_node *cm_node;</div><div class='del'>-	u32 loc_addr[4], rem_addr[4];</div><div class='del'>-	u16 loc_port, rem_port;</div><div class='del'>-	struct ipv6hdr *ip6h;</div><div class='del'>-	struct iphdr *iph = (struct iphdr *)buf-&gt;iph;</div><div class='del'>-	struct tcphdr *tcph = (struct tcphdr *)buf-&gt;tcph;</div><div class='del'>-</div><div class='del'>-	if (iph-&gt;version == 4) {</div><div class='del'>-		memset(loc_addr, 0, sizeof(loc_addr));</div><div class='del'>-		loc_addr[0] = ntohl(iph-&gt;daddr);</div><div class='del'>-		memset(rem_addr, 0, sizeof(rem_addr));</div><div class='del'>-		rem_addr[0] = ntohl(iph-&gt;saddr);</div><div class='del'>-	} else {</div><div class='del'>-		ip6h = (struct ipv6hdr *)buf-&gt;iph;</div><div class='del'>-		i40iw_copy_ip_ntohl(loc_addr, ip6h-&gt;daddr.in6_u.u6_addr32);</div><div class='del'>-		i40iw_copy_ip_ntohl(rem_addr, ip6h-&gt;saddr.in6_u.u6_addr32);</div><div class='del'>-	}</div><div class='del'>-	loc_port = ntohs(tcph-&gt;dest);</div><div class='del'>-	rem_port = ntohs(tcph-&gt;source);</div><div class='del'>-</div><div class='del'>-	cm_node = i40iw_find_node(&amp;iwdev-&gt;cm_core, rem_port, rem_addr, loc_port,</div><div class='del'>-				  loc_addr, false, true);</div><div class='del'>-	if (!cm_node)</div><div class='del'>-		return NULL;</div><div class='del'>-	iwqp = cm_node-&gt;iwqp;</div><div class='del'>-	return &amp;iwqp-&gt;sc_qp;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_ieq_update_tcpip_info - update tcpip in the buffer</div><div class='del'>- * @buf: puda to update</div><div class='del'>- * @length: length of buffer</div><div class='del'>- * @seqnum: seq number for tcp</div><div class='del'>- */</div><div class='del'>-void i40iw_ieq_update_tcpip_info(struct i40iw_puda_buf *buf, u16 length, u32 seqnum)</div><div class='del'>-{</div><div class='del'>-	struct tcphdr *tcph;</div><div class='del'>-	struct iphdr *iph;</div><div class='del'>-	u16 iphlen;</div><div class='del'>-	u16 packetsize;</div><div class='del'>-	u8 *addr = (u8 *)buf-&gt;mem.va;</div><div class='del'>-</div><div class='del'>-	iphlen = (buf-&gt;ipv4) ? 20 : 40;</div><div class='del'>-	iph = (struct iphdr *)(addr + buf-&gt;maclen);</div><div class='del'>-	tcph = (struct tcphdr *)(addr + buf-&gt;maclen + iphlen);</div><div class='del'>-	packetsize = length + buf-&gt;tcphlen + iphlen;</div><div class='del'>-</div><div class='del'>-	iph-&gt;tot_len = htons(packetsize);</div><div class='del'>-	tcph-&gt;seq = htonl(seqnum);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_puda_get_tcpip_info - get tcpip info from puda buffer</div><div class='del'>- * @info: to get information</div><div class='del'>- * @buf: puda buffer</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_puda_get_tcpip_info(struct i40iw_puda_completion_info *info,</div><div class='del'>-						 struct i40iw_puda_buf *buf)</div><div class='del'>-{</div><div class='del'>-	struct iphdr *iph;</div><div class='del'>-	struct ipv6hdr *ip6h;</div><div class='del'>-	struct tcphdr *tcph;</div><div class='del'>-	u16 iphlen;</div><div class='del'>-	u16 pkt_len;</div><div class='del'>-	u8 *mem = (u8 *)buf-&gt;mem.va;</div><div class='del'>-	struct ethhdr *ethh = (struct ethhdr *)buf-&gt;mem.va;</div><div class='del'>-</div><div class='del'>-	if (ethh-&gt;h_proto == htons(0x8100)) {</div><div class='del'>-		info-&gt;vlan_valid = true;</div><div class='del'>-		buf-&gt;vlan_id = ntohs(((struct vlan_ethhdr *)ethh)-&gt;h_vlan_TCI) &amp; VLAN_VID_MASK;</div><div class='del'>-	}</div><div class='del'>-	buf-&gt;maclen = (info-&gt;vlan_valid) ? 18 : 14;</div><div class='del'>-	iphlen = (info-&gt;l3proto) ? 40 : 20;</div><div class='del'>-	buf-&gt;ipv4 = (info-&gt;l3proto) ? false : true;</div><div class='del'>-	buf-&gt;iph = mem + buf-&gt;maclen;</div><div class='del'>-	iph = (struct iphdr *)buf-&gt;iph;</div><div class='del'>-</div><div class='del'>-	buf-&gt;tcph = buf-&gt;iph + iphlen;</div><div class='del'>-	tcph = (struct tcphdr *)buf-&gt;tcph;</div><div class='del'>-</div><div class='del'>-	if (buf-&gt;ipv4) {</div><div class='del'>-		pkt_len = ntohs(iph-&gt;tot_len);</div><div class='del'>-	} else {</div><div class='del'>-		ip6h = (struct ipv6hdr *)buf-&gt;iph;</div><div class='del'>-		pkt_len = ntohs(ip6h-&gt;payload_len) + iphlen;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	buf-&gt;totallen = pkt_len + buf-&gt;maclen;</div><div class='del'>-</div><div class='del'>-	if (info-&gt;payload_len &lt; buf-&gt;totallen) {</div><div class='del'>-		i40iw_pr_err("payload_len = 0x%x totallen expected0x%x\n",</div><div class='del'>-			     info-&gt;payload_len, buf-&gt;totallen);</div><div class='del'>-		return I40IW_ERR_INVALID_SIZE;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	buf-&gt;tcphlen = (tcph-&gt;doff) &lt;&lt; 2;</div><div class='del'>-	buf-&gt;datalen = pkt_len - iphlen - buf-&gt;tcphlen;</div><div class='del'>-	buf-&gt;data = (buf-&gt;datalen) ? buf-&gt;tcph + buf-&gt;tcphlen : NULL;</div><div class='del'>-	buf-&gt;hdrlen = buf-&gt;maclen + iphlen + buf-&gt;tcphlen;</div><div class='del'>-	buf-&gt;seqnum = ntohl(tcph-&gt;seq);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hw_stats_timeout - Stats timer-handler which updates all HW stats</div><div class='del'>- * @vsi: pointer to the vsi structure</div><div class='del'>- */</div><div class='del'>-static void i40iw_hw_stats_timeout(struct timer_list *t)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_vsi_pestat *pf_devstat = from_timer(pf_devstat, t,</div><div class='del'>-						       stats_timer);</div><div class='del'>-	struct i40iw_sc_vsi *sc_vsi = pf_devstat-&gt;vsi;</div><div class='del'>-	struct i40iw_sc_dev *pf_dev = sc_vsi-&gt;dev;</div><div class='del'>-	struct i40iw_vsi_pestat *vf_devstat = NULL;</div><div class='del'>-	u16 iw_vf_idx;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	/*PF*/</div><div class='del'>-	i40iw_hw_stats_read_all(pf_devstat, &amp;pf_devstat-&gt;hw_stats);</div><div class='del'>-</div><div class='del'>-	for (iw_vf_idx = 0; iw_vf_idx &lt; I40IW_MAX_PE_ENABLED_VF_COUNT; iw_vf_idx++) {</div><div class='del'>-		spin_lock_irqsave(&amp;pf_devstat-&gt;lock, flags);</div><div class='del'>-		if (pf_dev-&gt;vf_dev[iw_vf_idx]) {</div><div class='del'>-			if (pf_dev-&gt;vf_dev[iw_vf_idx]-&gt;stats_initialized) {</div><div class='del'>-				vf_devstat = &amp;pf_dev-&gt;vf_dev[iw_vf_idx]-&gt;pestat;</div><div class='del'>-				i40iw_hw_stats_read_all(vf_devstat, &amp;vf_devstat-&gt;hw_stats);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-		spin_unlock_irqrestore(&amp;pf_devstat-&gt;lock, flags);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mod_timer(&amp;pf_devstat-&gt;stats_timer,</div><div class='del'>-		  jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hw_stats_start_timer - Start periodic stats timer</div><div class='del'>- * @vsi: pointer to the vsi structure</div><div class='del'>- */</div><div class='del'>-void i40iw_hw_stats_start_timer(struct i40iw_sc_vsi *vsi)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_vsi_pestat *devstat = vsi-&gt;pestat;</div><div class='del'>-</div><div class='del'>-	timer_setup(&amp;devstat-&gt;stats_timer, i40iw_hw_stats_timeout, 0);</div><div class='del'>-	mod_timer(&amp;devstat-&gt;stats_timer,</div><div class='del'>-		  jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hw_stats_stop_timer - Delete periodic stats timer</div><div class='del'>- * @vsi: pointer to the vsi structure</div><div class='del'>- */</div><div class='del'>-void i40iw_hw_stats_stop_timer(struct i40iw_sc_vsi *vsi)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_vsi_pestat *devstat = vsi-&gt;pestat;</div><div class='del'>-</div><div class='del'>-	del_timer_sync(&amp;devstat-&gt;stats_timer);</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c<br/>deleted file mode 100644<br/>index 5689d742bafb..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_verbs.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_verbs.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,2807 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='del'>-#include &lt;linux/moduleparam.h&gt;</div><div class='del'>-#include &lt;linux/random.h&gt;</div><div class='del'>-#include &lt;linux/highmem.h&gt;</div><div class='del'>-#include &lt;linux/time.h&gt;</div><div class='del'>-#include &lt;linux/hugetlb.h&gt;</div><div class='del'>-#include &lt;linux/irq.h&gt;</div><div class='del'>-#include &lt;asm/byteorder.h&gt;</div><div class='del'>-#include &lt;net/ip.h&gt;</div><div class='del'>-#include &lt;rdma/ib_verbs.h&gt;</div><div class='del'>-#include &lt;rdma/iw_cm.h&gt;</div><div class='del'>-#include &lt;rdma/ib_user_verbs.h&gt;</div><div class='del'>-#include &lt;rdma/ib_umem.h&gt;</div><div class='del'>-#include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='del'>-#include "i40iw.h"</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_query_device - get device attributes</div><div class='del'>- * @ibdev: device pointer from stack</div><div class='del'>- * @props: returning device attributes</div><div class='del'>- * @udata: user data</div><div class='del'>- */</div><div class='del'>-static int i40iw_query_device(struct ib_device *ibdev,</div><div class='del'>-			      struct ib_device_attr *props,</div><div class='del'>-			      struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = to_iwdev(ibdev);</div><div class='del'>-</div><div class='del'>-	if (udata-&gt;inlen || udata-&gt;outlen)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	memset(props, 0, sizeof(*props));</div><div class='del'>-	ether_addr_copy((u8 *)&amp;props-&gt;sys_image_guid, iwdev-&gt;netdev-&gt;dev_addr);</div><div class='del'>-	props-&gt;fw_ver = I40IW_FW_VERSION;</div><div class='del'>-	props-&gt;device_cap_flags = iwdev-&gt;device_cap_flags;</div><div class='del'>-	props-&gt;vendor_id = iwdev-&gt;ldev-&gt;pcidev-&gt;vendor;</div><div class='del'>-	props-&gt;vendor_part_id = iwdev-&gt;ldev-&gt;pcidev-&gt;device;</div><div class='del'>-	props-&gt;hw_ver = (u32)iwdev-&gt;sc_dev.hw_rev;</div><div class='del'>-	props-&gt;max_mr_size = I40IW_MAX_OUTBOUND_MESSAGE_SIZE;</div><div class='del'>-	props-&gt;max_qp = iwdev-&gt;max_qp - iwdev-&gt;used_qps;</div><div class='del'>-	props-&gt;max_qp_wr = I40IW_MAX_QP_WRS;</div><div class='del'>-	props-&gt;max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;</div><div class='del'>-	props-&gt;max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;</div><div class='del'>-	props-&gt;max_cq = iwdev-&gt;max_cq - iwdev-&gt;used_cqs;</div><div class='del'>-	props-&gt;max_cqe = iwdev-&gt;max_cqe;</div><div class='del'>-	props-&gt;max_mr = iwdev-&gt;max_mr - iwdev-&gt;used_mrs;</div><div class='del'>-	props-&gt;max_pd = iwdev-&gt;max_pd - iwdev-&gt;used_pds;</div><div class='del'>-	props-&gt;max_sge_rd = I40IW_MAX_SGE_RD;</div><div class='del'>-	props-&gt;max_qp_rd_atom = I40IW_MAX_IRD_SIZE;</div><div class='del'>-	props-&gt;max_qp_init_rd_atom = props-&gt;max_qp_rd_atom;</div><div class='del'>-	props-&gt;atomic_cap = IB_ATOMIC_NONE;</div><div class='del'>-	props-&gt;max_map_per_fmr = 1;</div><div class='del'>-	props-&gt;max_fast_reg_page_list_len = I40IW_MAX_PAGES_PER_FMR;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_query_port - get port attrubutes</div><div class='del'>- * @ibdev: device pointer from stack</div><div class='del'>- * @port: port number for query</div><div class='del'>- * @props: returning device attributes</div><div class='del'>- */</div><div class='del'>-static int i40iw_query_port(struct ib_device *ibdev,</div><div class='del'>-			    u8 port,</div><div class='del'>-			    struct ib_port_attr *props)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = to_iwdev(ibdev);</div><div class='del'>-	struct net_device *netdev = iwdev-&gt;netdev;</div><div class='del'>-</div><div class='del'>-	/* props being zeroed by the caller, avoid zeroing it here */</div><div class='del'>-	props-&gt;max_mtu = IB_MTU_4096;</div><div class='del'>-	props-&gt;active_mtu = ib_mtu_int_to_enum(netdev-&gt;mtu);</div><div class='del'>-</div><div class='del'>-	props-&gt;lid = 1;</div><div class='del'>-	if (netif_carrier_ok(iwdev-&gt;netdev))</div><div class='del'>-		props-&gt;state = IB_PORT_ACTIVE;</div><div class='del'>-	else</div><div class='del'>-		props-&gt;state = IB_PORT_DOWN;</div><div class='del'>-	props-&gt;port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |</div><div class='del'>-		IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;</div><div class='del'>-	props-&gt;gid_tbl_len = 1;</div><div class='del'>-	props-&gt;pkey_tbl_len = 1;</div><div class='del'>-	props-&gt;active_width = IB_WIDTH_4X;</div><div class='del'>-	props-&gt;active_speed = 1;</div><div class='del'>-	props-&gt;max_msg_sz = I40IW_MAX_OUTBOUND_MESSAGE_SIZE;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_alloc_ucontext - Allocate the user context data structure</div><div class='del'>- * @uctx: Uverbs context pointer from stack</div><div class='del'>- * @udata: user data</div><div class='del'>- *</div><div class='del'>- * This keeps track of all objects associated with a particular</div><div class='del'>- * user-mode client.</div><div class='del'>- */</div><div class='del'>-static int i40iw_alloc_ucontext(struct ib_ucontext *uctx,</div><div class='del'>-				struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct ib_device *ibdev = uctx-&gt;device;</div><div class='del'>-	struct i40iw_device *iwdev = to_iwdev(ibdev);</div><div class='del'>-	struct i40iw_alloc_ucontext_req req;</div><div class='del'>-	struct i40iw_alloc_ucontext_resp uresp = {};</div><div class='del'>-	struct i40iw_ucontext *ucontext = to_ucontext(uctx);</div><div class='del'>-</div><div class='del'>-	if (ib_copy_from_udata(&amp;req, udata, sizeof(req)))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	if (req.userspace_ver &lt; 4 || req.userspace_ver &gt; I40IW_ABI_VER) {</div><div class='del'>-		i40iw_pr_err("Unsupported provider library version %u.\n", req.userspace_ver);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	uresp.max_qps = iwdev-&gt;max_qp;</div><div class='del'>-	uresp.max_pds = iwdev-&gt;max_pd;</div><div class='del'>-	uresp.wq_size = iwdev-&gt;max_qp_wr * 2;</div><div class='del'>-	uresp.kernel_ver = req.userspace_ver;</div><div class='del'>-</div><div class='del'>-	ucontext-&gt;iwdev = iwdev;</div><div class='del'>-	ucontext-&gt;abi_ver = req.userspace_ver;</div><div class='del'>-</div><div class='del'>-	if (ib_copy_to_udata(udata, &amp;uresp, sizeof(uresp)))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	INIT_LIST_HEAD(&amp;ucontext-&gt;cq_reg_mem_list);</div><div class='del'>-	spin_lock_init(&amp;ucontext-&gt;cq_reg_mem_list_lock);</div><div class='del'>-	INIT_LIST_HEAD(&amp;ucontext-&gt;qp_reg_mem_list);</div><div class='del'>-	spin_lock_init(&amp;ucontext-&gt;qp_reg_mem_list_lock);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_dealloc_ucontext - deallocate the user context data structure</div><div class='del'>- * @context: user context created during alloc</div><div class='del'>- */</div><div class='del'>-static void i40iw_dealloc_ucontext(struct ib_ucontext *context)</div><div class='del'>-{</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_mmap - user memory map</div><div class='del'>- * @context: context created during alloc</div><div class='del'>- * @vma: kernel info for user memory map</div><div class='del'>- */</div><div class='del'>-static int i40iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_ucontext *ucontext;</div><div class='del'>-	u64 db_addr_offset;</div><div class='del'>-	u64 push_offset;</div><div class='del'>-</div><div class='del'>-	ucontext = to_ucontext(context);</div><div class='del'>-	if (ucontext-&gt;iwdev-&gt;sc_dev.is_pf) {</div><div class='del'>-		db_addr_offset = I40IW_DB_ADDR_OFFSET;</div><div class='del'>-		push_offset = I40IW_PUSH_OFFSET;</div><div class='del'>-		if (vma-&gt;vm_pgoff)</div><div class='del'>-			vma-&gt;vm_pgoff += I40IW_PF_FIRST_PUSH_PAGE_INDEX - 1;</div><div class='del'>-	} else {</div><div class='del'>-		db_addr_offset = I40IW_VF_DB_ADDR_OFFSET;</div><div class='del'>-		push_offset = I40IW_VF_PUSH_OFFSET;</div><div class='del'>-		if (vma-&gt;vm_pgoff)</div><div class='del'>-			vma-&gt;vm_pgoff += I40IW_VF_FIRST_PUSH_PAGE_INDEX - 1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	vma-&gt;vm_pgoff += db_addr_offset &gt;&gt; PAGE_SHIFT;</div><div class='del'>-</div><div class='del'>-	if (vma-&gt;vm_pgoff == (db_addr_offset &gt;&gt; PAGE_SHIFT)) {</div><div class='del'>-		vma-&gt;vm_page_prot = pgprot_noncached(vma-&gt;vm_page_prot);</div><div class='del'>-		vma-&gt;vm_private_data = ucontext;</div><div class='del'>-	} else {</div><div class='del'>-		if ((vma-&gt;vm_pgoff - (push_offset &gt;&gt; PAGE_SHIFT)) % 2)</div><div class='del'>-			vma-&gt;vm_page_prot = pgprot_noncached(vma-&gt;vm_page_prot);</div><div class='del'>-		else</div><div class='del'>-			vma-&gt;vm_page_prot = pgprot_writecombine(vma-&gt;vm_page_prot);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (io_remap_pfn_range(vma, vma-&gt;vm_start,</div><div class='del'>-			       vma-&gt;vm_pgoff + (pci_resource_start(ucontext-&gt;iwdev-&gt;ldev-&gt;pcidev, 0) &gt;&gt; PAGE_SHIFT),</div><div class='del'>-			       PAGE_SIZE, vma-&gt;vm_page_prot))</div><div class='del'>-		return -EAGAIN;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_alloc_push_page - allocate a push page for qp</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @qp: hardware control qp</div><div class='del'>- */</div><div class='del'>-static void i40iw_alloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_qp *qp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-</div><div class='del'>-	if (qp-&gt;push_idx != I40IW_INVALID_PUSH_PAGE_INDEX)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, true);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	atomic_inc(&amp;cqp_request-&gt;refcount);</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_MANAGE_PUSH_PAGE;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-</div><div class='del'>-	cqp_info-&gt;in.u.manage_push_page.info.qs_handle = qp-&gt;qs_handle;</div><div class='del'>-	cqp_info-&gt;in.u.manage_push_page.info.free_page = 0;</div><div class='del'>-	cqp_info-&gt;in.u.manage_push_page.cqp = &amp;iwdev-&gt;cqp.sc_cqp;</div><div class='del'>-	cqp_info-&gt;in.u.manage_push_page.scratch = (uintptr_t)cqp_request;</div><div class='del'>-</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (!status)</div><div class='del'>-		qp-&gt;push_idx = cqp_request-&gt;compl_info.op_ret_val;</div><div class='del'>-	else</div><div class='del'>-		i40iw_pr_err("CQP-OP Push page fail");</div><div class='del'>-	i40iw_put_cqp_request(&amp;iwdev-&gt;cqp, cqp_request);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_dealloc_push_page - free a push page for qp</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @qp: hardware control qp</div><div class='del'>- */</div><div class='del'>-static void i40iw_dealloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_qp *qp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-</div><div class='del'>-	if (qp-&gt;push_idx == I40IW_INVALID_PUSH_PAGE_INDEX)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, false);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_MANAGE_PUSH_PAGE;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-</div><div class='del'>-	cqp_info-&gt;in.u.manage_push_page.info.push_idx = qp-&gt;push_idx;</div><div class='del'>-	cqp_info-&gt;in.u.manage_push_page.info.qs_handle = qp-&gt;qs_handle;</div><div class='del'>-	cqp_info-&gt;in.u.manage_push_page.info.free_page = 1;</div><div class='del'>-	cqp_info-&gt;in.u.manage_push_page.cqp = &amp;iwdev-&gt;cqp.sc_cqp;</div><div class='del'>-	cqp_info-&gt;in.u.manage_push_page.scratch = (uintptr_t)cqp_request;</div><div class='del'>-</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (!status)</div><div class='del'>-		qp-&gt;push_idx = I40IW_INVALID_PUSH_PAGE_INDEX;</div><div class='del'>-	else</div><div class='del'>-		i40iw_pr_err("CQP-OP Push page fail");</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_alloc_pd - allocate protection domain</div><div class='del'>- * @pd: PD pointer</div><div class='del'>- * @udata: user data</div><div class='del'>- */</div><div class='del'>-static int i40iw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_pd *iwpd = to_iwpd(pd);</div><div class='del'>-	struct i40iw_device *iwdev = to_iwdev(pd-&gt;device);</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_alloc_pd_resp uresp;</div><div class='del'>-	struct i40iw_sc_pd *sc_pd;</div><div class='del'>-	u32 pd_id = 0;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	if (iwdev-&gt;closing)</div><div class='del'>-		return -ENODEV;</div><div class='del'>-</div><div class='del'>-	err = i40iw_alloc_resource(iwdev, iwdev-&gt;allocated_pds,</div><div class='del'>-				   iwdev-&gt;max_pd, &amp;pd_id, &amp;iwdev-&gt;next_pd);</div><div class='del'>-	if (err) {</div><div class='del'>-		i40iw_pr_err("alloc resource failed\n");</div><div class='del'>-		return err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	sc_pd = &amp;iwpd-&gt;sc_pd;</div><div class='del'>-</div><div class='del'>-	if (udata) {</div><div class='del'>-		struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context(</div><div class='del'>-			udata, struct i40iw_ucontext, ibucontext);</div><div class='del'>-		dev-&gt;iw_pd_ops-&gt;pd_init(dev, sc_pd, pd_id, ucontext-&gt;abi_ver);</div><div class='del'>-		memset(&amp;uresp, 0, sizeof(uresp));</div><div class='del'>-		uresp.pd_id = pd_id;</div><div class='del'>-		if (ib_copy_to_udata(udata, &amp;uresp, sizeof(uresp))) {</div><div class='del'>-			err = -EFAULT;</div><div class='del'>-			goto error;</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		dev-&gt;iw_pd_ops-&gt;pd_init(dev, sc_pd, pd_id, -1);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i40iw_add_pdusecount(iwpd);</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-error:</div><div class='del'>-	i40iw_free_resource(iwdev, iwdev-&gt;allocated_pds, pd_id);</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_dealloc_pd - deallocate pd</div><div class='del'>- * @ibpd: ptr of pd to be deallocated</div><div class='del'>- * @udata: user data or null for kernel object</div><div class='del'>- */</div><div class='del'>-static void i40iw_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_pd *iwpd = to_iwpd(ibpd);</div><div class='del'>-	struct i40iw_device *iwdev = to_iwdev(ibpd-&gt;device);</div><div class='del'>-</div><div class='del'>-	i40iw_rem_pdusecount(iwpd, iwdev);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_pbl - Retrieve pbl from a list given a virtual</div><div class='del'>- * address</div><div class='del'>- * @va: user virtual address</div><div class='del'>- * @pbl_list: pbl list to search in (QP's or CQ's)</div><div class='del'>- */</div><div class='del'>-static struct i40iw_pbl *i40iw_get_pbl(unsigned long va,</div><div class='del'>-				       struct list_head *pbl_list)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_pbl *iwpbl;</div><div class='del'>-</div><div class='del'>-	list_for_each_entry(iwpbl, pbl_list, list) {</div><div class='del'>-		if (iwpbl-&gt;user_base == va) {</div><div class='del'>-			iwpbl-&gt;on_list = false;</div><div class='del'>-			list_del(&amp;iwpbl-&gt;list);</div><div class='del'>-			return iwpbl;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_free_qp_resources - free up memory resources for qp</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @iwqp: qp ptr (user or kernel)</div><div class='del'>- * @qp_num: qp number assigned</div><div class='del'>- */</div><div class='del'>-void i40iw_free_qp_resources(struct i40iw_device *iwdev,</div><div class='del'>-			     struct i40iw_qp *iwqp,</div><div class='del'>-			     u32 qp_num)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_pbl *iwpbl = &amp;iwqp-&gt;iwpbl;</div><div class='del'>-</div><div class='del'>-	i40iw_ieq_cleanup_qp(iwdev-&gt;vsi.ieq, &amp;iwqp-&gt;sc_qp);</div><div class='del'>-	i40iw_dealloc_push_page(iwdev, &amp;iwqp-&gt;sc_qp);</div><div class='del'>-	if (qp_num)</div><div class='del'>-		i40iw_free_resource(iwdev, iwdev-&gt;allocated_qps, qp_num);</div><div class='del'>-	if (iwpbl-&gt;pbl_allocated)</div><div class='del'>-		i40iw_free_pble(iwdev-&gt;pble_rsrc, &amp;iwpbl-&gt;pble_alloc);</div><div class='del'>-	i40iw_free_dma_mem(iwdev-&gt;sc_dev.hw, &amp;iwqp-&gt;q2_ctx_mem);</div><div class='del'>-	i40iw_free_dma_mem(iwdev-&gt;sc_dev.hw, &amp;iwqp-&gt;kqp.dma_mem);</div><div class='del'>-	kfree(iwqp-&gt;kqp.wrid_mem);</div><div class='del'>-	iwqp-&gt;kqp.wrid_mem = NULL;</div><div class='del'>-	kfree(iwqp-&gt;allocated_buffer);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_clean_cqes - clean cq entries for qp</div><div class='del'>- * @iwqp: qp ptr (user or kernel)</div><div class='del'>- * @iwcq: cq ptr</div><div class='del'>- */</div><div class='del'>-static void i40iw_clean_cqes(struct i40iw_qp *iwqp, struct i40iw_cq *iwcq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cq_uk *ukcq = &amp;iwcq-&gt;sc_cq.cq_uk;</div><div class='del'>-</div><div class='del'>-	ukcq-&gt;ops.iw_cq_clean(&amp;iwqp-&gt;sc_qp.qp_uk, ukcq);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_destroy_qp - destroy qp</div><div class='del'>- * @ibqp: qp's ib pointer also to get to device's qp address</div><div class='del'>- */</div><div class='del'>-static int i40iw_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp *iwqp = to_iwqp(ibqp);</div><div class='del'>-</div><div class='del'>-	iwqp-&gt;destroyed = 1;</div><div class='del'>-</div><div class='del'>-	if (iwqp-&gt;ibqp_state &gt;= IB_QPS_INIT &amp;&amp; iwqp-&gt;ibqp_state &lt; IB_QPS_RTS)</div><div class='del'>-		i40iw_next_iw_state(iwqp, I40IW_QP_STATE_ERROR, 0, 0, 0);</div><div class='del'>-</div><div class='del'>-	if (!iwqp-&gt;user_mode) {</div><div class='del'>-		if (iwqp-&gt;iwscq) {</div><div class='del'>-			i40iw_clean_cqes(iwqp, iwqp-&gt;iwscq);</div><div class='del'>-			if (iwqp-&gt;iwrcq != iwqp-&gt;iwscq)</div><div class='del'>-				i40iw_clean_cqes(iwqp, iwqp-&gt;iwrcq);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i40iw_rem_ref(&amp;iwqp-&gt;ibqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_setup_virt_qp - setup for allocation of virtual qp</div><div class='del'>- * @dev: iwarp device</div><div class='del'>- * @qp: qp ptr</div><div class='del'>- * @init_info: initialize info to return</div><div class='del'>- */</div><div class='del'>-static int i40iw_setup_virt_qp(struct i40iw_device *iwdev,</div><div class='del'>-			       struct i40iw_qp *iwqp,</div><div class='del'>-			       struct i40iw_qp_init_info *init_info)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_pbl *iwpbl = &amp;iwqp-&gt;iwpbl;</div><div class='del'>-	struct i40iw_qp_mr *qpmr = &amp;iwpbl-&gt;qp_mr;</div><div class='del'>-</div><div class='del'>-	iwqp-&gt;page = qpmr-&gt;sq_page;</div><div class='del'>-	init_info-&gt;shadow_area_pa = cpu_to_le64(qpmr-&gt;shadow);</div><div class='del'>-	if (iwpbl-&gt;pbl_allocated) {</div><div class='del'>-		init_info-&gt;virtual_map = true;</div><div class='del'>-		init_info-&gt;sq_pa = qpmr-&gt;sq_pbl.idx;</div><div class='del'>-		init_info-&gt;rq_pa = qpmr-&gt;rq_pbl.idx;</div><div class='del'>-	} else {</div><div class='del'>-		init_info-&gt;sq_pa = qpmr-&gt;sq_pbl.addr;</div><div class='del'>-		init_info-&gt;rq_pa = qpmr-&gt;rq_pbl.addr;</div><div class='del'>-	}</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_setup_kmode_qp - setup initialization for kernel mode qp</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @iwqp: qp ptr (user or kernel)</div><div class='del'>- * @info: initialize info to return</div><div class='del'>- */</div><div class='del'>-static int i40iw_setup_kmode_qp(struct i40iw_device *iwdev,</div><div class='del'>-				struct i40iw_qp *iwqp,</div><div class='del'>-				struct i40iw_qp_init_info *info)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_dma_mem *mem = &amp;iwqp-&gt;kqp.dma_mem;</div><div class='del'>-	u32 sqdepth, rqdepth;</div><div class='del'>-	u8 sqshift;</div><div class='del'>-	u32 size;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_qp_uk_init_info *ukinfo = &amp;info-&gt;qp_uk_init_info;</div><div class='del'>-</div><div class='del'>-	i40iw_get_wqe_shift(ukinfo-&gt;max_sq_frag_cnt, ukinfo-&gt;max_inline_data, &amp;sqshift);</div><div class='del'>-	status = i40iw_get_sqdepth(ukinfo-&gt;sq_size, sqshift, &amp;sqdepth);</div><div class='del'>-	if (status)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	status = i40iw_get_rqdepth(ukinfo-&gt;rq_size, I40IW_MAX_RQ_WQE_SHIFT, &amp;rqdepth);</div><div class='del'>-	if (status)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	size = sqdepth * sizeof(struct i40iw_sq_uk_wr_trk_info) + (rqdepth &lt;&lt; 3);</div><div class='del'>-	iwqp-&gt;kqp.wrid_mem = kzalloc(size, GFP_KERNEL);</div><div class='del'>-</div><div class='del'>-	ukinfo-&gt;sq_wrtrk_array = (struct i40iw_sq_uk_wr_trk_info *)iwqp-&gt;kqp.wrid_mem;</div><div class='del'>-	if (!ukinfo-&gt;sq_wrtrk_array)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	ukinfo-&gt;rq_wrid_array = (u64 *)&amp;ukinfo-&gt;sq_wrtrk_array[sqdepth];</div><div class='del'>-</div><div class='del'>-	size = (sqdepth + rqdepth) * I40IW_QP_WQE_MIN_SIZE;</div><div class='del'>-	size += (I40IW_SHADOW_AREA_SIZE &lt;&lt; 3);</div><div class='del'>-</div><div class='del'>-	status = i40iw_allocate_dma_mem(iwdev-&gt;sc_dev.hw, mem, size, 256);</div><div class='del'>-	if (status) {</div><div class='del'>-		kfree(ukinfo-&gt;sq_wrtrk_array);</div><div class='del'>-		ukinfo-&gt;sq_wrtrk_array = NULL;</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ukinfo-&gt;sq = mem-&gt;va;</div><div class='del'>-	info-&gt;sq_pa = mem-&gt;pa;</div><div class='del'>-</div><div class='del'>-	ukinfo-&gt;rq = &amp;ukinfo-&gt;sq[sqdepth];</div><div class='del'>-	info-&gt;rq_pa = info-&gt;sq_pa + (sqdepth * I40IW_QP_WQE_MIN_SIZE);</div><div class='del'>-</div><div class='del'>-	ukinfo-&gt;shadow_area = ukinfo-&gt;rq[rqdepth].elem;</div><div class='del'>-	info-&gt;shadow_area_pa = info-&gt;rq_pa + (rqdepth * I40IW_QP_WQE_MIN_SIZE);</div><div class='del'>-</div><div class='del'>-	ukinfo-&gt;sq_size = sqdepth &gt;&gt; sqshift;</div><div class='del'>-	ukinfo-&gt;rq_size = rqdepth &gt;&gt; I40IW_MAX_RQ_WQE_SHIFT;</div><div class='del'>-	ukinfo-&gt;qp_id = iwqp-&gt;ibqp.qp_num;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_create_qp - create qp</div><div class='del'>- * @ibpd: ptr of pd</div><div class='del'>- * @init_attr: attributes for qp</div><div class='del'>- * @udata: user data for create qp</div><div class='del'>- */</div><div class='del'>-static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd,</div><div class='del'>-				     struct ib_qp_init_attr *init_attr,</div><div class='del'>-				     struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_pd *iwpd = to_iwpd(ibpd);</div><div class='del'>-	struct i40iw_device *iwdev = to_iwdev(ibpd-&gt;device);</div><div class='del'>-	struct i40iw_cqp *iwcqp = &amp;iwdev-&gt;cqp;</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-	struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context(</div><div class='del'>-		udata, struct i40iw_ucontext, ibucontext);</div><div class='del'>-	struct i40iw_create_qp_req req;</div><div class='del'>-	struct i40iw_create_qp_resp uresp;</div><div class='del'>-	u32 qp_num = 0;</div><div class='del'>-	void *mem;</div><div class='del'>-	enum i40iw_status_code ret;</div><div class='del'>-	int err_code;</div><div class='del'>-	int sq_size;</div><div class='del'>-	int rq_size;</div><div class='del'>-	struct i40iw_sc_qp *qp;</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_qp_init_info init_info;</div><div class='del'>-	struct i40iw_create_qp_info *qp_info;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-</div><div class='del'>-	struct i40iw_qp_host_ctx_info *ctx_info;</div><div class='del'>-	struct i40iwarp_offload_info *iwarp_info;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	if (iwdev-&gt;closing)</div><div class='del'>-		return ERR_PTR(-ENODEV);</div><div class='del'>-</div><div class='del'>-	if (init_attr-&gt;create_flags)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-	if (init_attr-&gt;cap.max_inline_data &gt; I40IW_MAX_INLINE_DATA_SIZE)</div><div class='del'>-		init_attr-&gt;cap.max_inline_data = I40IW_MAX_INLINE_DATA_SIZE;</div><div class='del'>-</div><div class='del'>-	if (init_attr-&gt;cap.max_send_sge &gt; I40IW_MAX_WQ_FRAGMENT_COUNT)</div><div class='del'>-		init_attr-&gt;cap.max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;</div><div class='del'>-</div><div class='del'>-	if (init_attr-&gt;cap.max_recv_sge &gt; I40IW_MAX_WQ_FRAGMENT_COUNT)</div><div class='del'>-		init_attr-&gt;cap.max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;</div><div class='del'>-</div><div class='del'>-	memset(&amp;init_info, 0, sizeof(init_info));</div><div class='del'>-</div><div class='del'>-	sq_size = init_attr-&gt;cap.max_send_wr;</div><div class='del'>-	rq_size = init_attr-&gt;cap.max_recv_wr;</div><div class='del'>-</div><div class='del'>-	init_info.vsi = &amp;iwdev-&gt;vsi;</div><div class='del'>-	init_info.qp_uk_init_info.sq_size = sq_size;</div><div class='del'>-	init_info.qp_uk_init_info.rq_size = rq_size;</div><div class='del'>-	init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr-&gt;cap.max_send_sge;</div><div class='del'>-	init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr-&gt;cap.max_recv_sge;</div><div class='del'>-	init_info.qp_uk_init_info.max_inline_data = init_attr-&gt;cap.max_inline_data;</div><div class='del'>-</div><div class='del'>-	mem = kzalloc(sizeof(*iwqp), GFP_KERNEL);</div><div class='del'>-	if (!mem)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	iwqp = (struct i40iw_qp *)mem;</div><div class='del'>-	iwqp-&gt;allocated_buffer = mem;</div><div class='del'>-	qp = &amp;iwqp-&gt;sc_qp;</div><div class='del'>-	qp-&gt;back_qp = (void *)iwqp;</div><div class='del'>-	qp-&gt;push_idx = I40IW_INVALID_PUSH_PAGE_INDEX;</div><div class='del'>-</div><div class='del'>-	iwqp-&gt;ctx_info.iwarp_info = &amp;iwqp-&gt;iwarp_info;</div><div class='del'>-</div><div class='del'>-	if (i40iw_allocate_dma_mem(dev-&gt;hw,</div><div class='del'>-				   &amp;iwqp-&gt;q2_ctx_mem,</div><div class='del'>-				   I40IW_Q2_BUFFER_SIZE + I40IW_QP_CTX_SIZE,</div><div class='del'>-				   256)) {</div><div class='del'>-		i40iw_pr_err("dma_mem failed\n");</div><div class='del'>-		err_code = -ENOMEM;</div><div class='del'>-		goto error;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	init_info.q2 = iwqp-&gt;q2_ctx_mem.va;</div><div class='del'>-	init_info.q2_pa = iwqp-&gt;q2_ctx_mem.pa;</div><div class='del'>-</div><div class='del'>-	init_info.host_ctx = (void *)init_info.q2 + I40IW_Q2_BUFFER_SIZE;</div><div class='del'>-	init_info.host_ctx_pa = init_info.q2_pa + I40IW_Q2_BUFFER_SIZE;</div><div class='del'>-</div><div class='del'>-	err_code = i40iw_alloc_resource(iwdev, iwdev-&gt;allocated_qps, iwdev-&gt;max_qp,</div><div class='del'>-					&amp;qp_num, &amp;iwdev-&gt;next_qp);</div><div class='del'>-	if (err_code) {</div><div class='del'>-		i40iw_pr_err("qp resource\n");</div><div class='del'>-		goto error;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	iwqp-&gt;iwdev = iwdev;</div><div class='del'>-	iwqp-&gt;iwpd = iwpd;</div><div class='del'>-	iwqp-&gt;ibqp.qp_num = qp_num;</div><div class='del'>-	qp = &amp;iwqp-&gt;sc_qp;</div><div class='del'>-	iwqp-&gt;iwscq = to_iwcq(init_attr-&gt;send_cq);</div><div class='del'>-	iwqp-&gt;iwrcq = to_iwcq(init_attr-&gt;recv_cq);</div><div class='del'>-</div><div class='del'>-	iwqp-&gt;host_ctx.va = init_info.host_ctx;</div><div class='del'>-	iwqp-&gt;host_ctx.pa = init_info.host_ctx_pa;</div><div class='del'>-	iwqp-&gt;host_ctx.size = I40IW_QP_CTX_SIZE;</div><div class='del'>-</div><div class='del'>-	init_info.pd = &amp;iwpd-&gt;sc_pd;</div><div class='del'>-	init_info.qp_uk_init_info.qp_id = iwqp-&gt;ibqp.qp_num;</div><div class='del'>-	iwqp-&gt;ctx_info.qp_compl_ctx = (uintptr_t)qp;</div><div class='del'>-</div><div class='del'>-	if (init_attr-&gt;qp_type != IB_QPT_RC) {</div><div class='del'>-		err_code = -EINVAL;</div><div class='del'>-		goto error;</div><div class='del'>-	}</div><div class='del'>-	if (iwdev-&gt;push_mode)</div><div class='del'>-		i40iw_alloc_push_page(iwdev, qp);</div><div class='del'>-	if (udata) {</div><div class='del'>-		err_code = ib_copy_from_udata(&amp;req, udata, sizeof(req));</div><div class='del'>-		if (err_code) {</div><div class='del'>-			i40iw_pr_err("ib_copy_from_data\n");</div><div class='del'>-			goto error;</div><div class='del'>-		}</div><div class='del'>-		iwqp-&gt;ctx_info.qp_compl_ctx = req.user_compl_ctx;</div><div class='del'>-		iwqp-&gt;user_mode = 1;</div><div class='del'>-</div><div class='del'>-		if (req.user_wqe_buffers) {</div><div class='del'>-			struct i40iw_pbl *iwpbl;</div><div class='del'>-</div><div class='del'>-			spin_lock_irqsave(</div><div class='del'>-			    &amp;ucontext-&gt;qp_reg_mem_list_lock, flags);</div><div class='del'>-			iwpbl = i40iw_get_pbl(</div><div class='del'>-			    (unsigned long)req.user_wqe_buffers,</div><div class='del'>-			    &amp;ucontext-&gt;qp_reg_mem_list);</div><div class='del'>-			spin_unlock_irqrestore(</div><div class='del'>-			    &amp;ucontext-&gt;qp_reg_mem_list_lock, flags);</div><div class='del'>-</div><div class='del'>-			if (!iwpbl) {</div><div class='del'>-				err_code = -ENODATA;</div><div class='del'>-				i40iw_pr_err("no pbl info\n");</div><div class='del'>-				goto error;</div><div class='del'>-			}</div><div class='del'>-			memcpy(&amp;iwqp-&gt;iwpbl, iwpbl, sizeof(iwqp-&gt;iwpbl));</div><div class='del'>-		}</div><div class='del'>-		err_code = i40iw_setup_virt_qp(iwdev, iwqp, &amp;init_info);</div><div class='del'>-	} else {</div><div class='del'>-		err_code = i40iw_setup_kmode_qp(iwdev, iwqp, &amp;init_info);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (err_code) {</div><div class='del'>-		i40iw_pr_err("setup qp failed\n");</div><div class='del'>-		goto error;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	init_info.type = I40IW_QP_TYPE_IWARP;</div><div class='del'>-	ret = dev-&gt;iw_priv_qp_ops-&gt;qp_init(qp, &amp;init_info);</div><div class='del'>-	if (ret) {</div><div class='del'>-		err_code = -EPROTO;</div><div class='del'>-		i40iw_pr_err("qp_init fail\n");</div><div class='del'>-		goto error;</div><div class='del'>-	}</div><div class='del'>-	ctx_info = &amp;iwqp-&gt;ctx_info;</div><div class='del'>-	iwarp_info = &amp;iwqp-&gt;iwarp_info;</div><div class='del'>-	iwarp_info-&gt;rd_enable = true;</div><div class='del'>-	iwarp_info-&gt;wr_rdresp_en = true;</div><div class='del'>-	if (!iwqp-&gt;user_mode) {</div><div class='del'>-		iwarp_info-&gt;fast_reg_en = true;</div><div class='del'>-		iwarp_info-&gt;priv_mode_en = true;</div><div class='del'>-	}</div><div class='del'>-	iwarp_info-&gt;ddp_ver = 1;</div><div class='del'>-	iwarp_info-&gt;rdmap_ver = 1;</div><div class='del'>-</div><div class='del'>-	ctx_info-&gt;iwarp_info_valid = true;</div><div class='del'>-	ctx_info-&gt;send_cq_num = iwqp-&gt;iwscq-&gt;sc_cq.cq_uk.cq_id;</div><div class='del'>-	ctx_info-&gt;rcv_cq_num = iwqp-&gt;iwrcq-&gt;sc_cq.cq_uk.cq_id;</div><div class='del'>-	if (qp-&gt;push_idx == I40IW_INVALID_PUSH_PAGE_INDEX) {</div><div class='del'>-		ctx_info-&gt;push_mode_en = false;</div><div class='del'>-	} else {</div><div class='del'>-		ctx_info-&gt;push_mode_en = true;</div><div class='del'>-		ctx_info-&gt;push_idx = qp-&gt;push_idx;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = dev-&gt;iw_priv_qp_ops-&gt;qp_setctx(&amp;iwqp-&gt;sc_qp,</div><div class='del'>-					     (u64 *)iwqp-&gt;host_ctx.va,</div><div class='del'>-					     ctx_info);</div><div class='del'>-	ctx_info-&gt;iwarp_info_valid = false;</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(iwcqp, true);</div><div class='del'>-	if (!cqp_request) {</div><div class='del'>-		err_code = -ENOMEM;</div><div class='del'>-		goto error;</div><div class='del'>-	}</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	qp_info = &amp;cqp_request-&gt;info.in.u.qp_create.info;</div><div class='del'>-</div><div class='del'>-	memset(qp_info, 0, sizeof(*qp_info));</div><div class='del'>-</div><div class='del'>-	qp_info-&gt;cq_num_valid = true;</div><div class='del'>-	qp_info-&gt;next_iwarp_state = I40IW_QP_STATE_IDLE;</div><div class='del'>-</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_QP_CREATE;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.qp_create.qp = qp;</div><div class='del'>-	cqp_info-&gt;in.u.qp_create.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	ret = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (ret) {</div><div class='del'>-		i40iw_pr_err("CQP-OP QP create fail");</div><div class='del'>-		err_code = -EACCES;</div><div class='del'>-		goto error;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i40iw_add_ref(&amp;iwqp-&gt;ibqp);</div><div class='del'>-	spin_lock_init(&amp;iwqp-&gt;lock);</div><div class='del'>-	iwqp-&gt;sig_all = (init_attr-&gt;sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0;</div><div class='del'>-	iwdev-&gt;qp_table[qp_num] = iwqp;</div><div class='del'>-	i40iw_add_pdusecount(iwqp-&gt;iwpd);</div><div class='del'>-	i40iw_add_devusecount(iwdev);</div><div class='del'>-	if (udata) {</div><div class='del'>-		memset(&amp;uresp, 0, sizeof(uresp));</div><div class='del'>-		uresp.actual_sq_size = sq_size;</div><div class='del'>-		uresp.actual_rq_size = rq_size;</div><div class='del'>-		uresp.qp_id = qp_num;</div><div class='del'>-		uresp.push_idx = qp-&gt;push_idx;</div><div class='del'>-		err_code = ib_copy_to_udata(udata, &amp;uresp, sizeof(uresp));</div><div class='del'>-		if (err_code) {</div><div class='del'>-			i40iw_pr_err("copy_to_udata failed\n");</div><div class='del'>-			i40iw_destroy_qp(&amp;iwqp-&gt;ibqp, udata);</div><div class='del'>-			/* let the completion of the qp destroy free the qp */</div><div class='del'>-			return ERR_PTR(err_code);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	init_completion(&amp;iwqp-&gt;sq_drained);</div><div class='del'>-	init_completion(&amp;iwqp-&gt;rq_drained);</div><div class='del'>-</div><div class='del'>-	return &amp;iwqp-&gt;ibqp;</div><div class='del'>-error:</div><div class='del'>-	i40iw_free_qp_resources(iwdev, iwqp, qp_num);</div><div class='del'>-	return ERR_PTR(err_code);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_query - query qp attributes</div><div class='del'>- * @ibqp: qp pointer</div><div class='del'>- * @attr: attributes pointer</div><div class='del'>- * @attr_mask: Not used</div><div class='del'>- * @init_attr: qp attributes to return</div><div class='del'>- */</div><div class='del'>-static int i40iw_query_qp(struct ib_qp *ibqp,</div><div class='del'>-			  struct ib_qp_attr *attr,</div><div class='del'>-			  int attr_mask,</div><div class='del'>-			  struct ib_qp_init_attr *init_attr)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp *iwqp = to_iwqp(ibqp);</div><div class='del'>-	struct i40iw_sc_qp *qp = &amp;iwqp-&gt;sc_qp;</div><div class='del'>-</div><div class='del'>-	attr-&gt;qp_access_flags = 0;</div><div class='del'>-	attr-&gt;cap.max_send_wr = qp-&gt;qp_uk.sq_size;</div><div class='del'>-	attr-&gt;cap.max_recv_wr = qp-&gt;qp_uk.rq_size;</div><div class='del'>-	attr-&gt;cap.max_inline_data = I40IW_MAX_INLINE_DATA_SIZE;</div><div class='del'>-	attr-&gt;cap.max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;</div><div class='del'>-	attr-&gt;cap.max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;</div><div class='del'>-	attr-&gt;port_num = 1;</div><div class='del'>-	init_attr-&gt;event_handler = iwqp-&gt;ibqp.event_handler;</div><div class='del'>-	init_attr-&gt;qp_context = iwqp-&gt;ibqp.qp_context;</div><div class='del'>-	init_attr-&gt;send_cq = iwqp-&gt;ibqp.send_cq;</div><div class='del'>-	init_attr-&gt;recv_cq = iwqp-&gt;ibqp.recv_cq;</div><div class='del'>-	init_attr-&gt;srq = iwqp-&gt;ibqp.srq;</div><div class='del'>-	init_attr-&gt;cap = attr-&gt;cap;</div><div class='del'>-	init_attr-&gt;port_num = 1;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hw_modify_qp - setup cqp for modify qp</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @iwqp: qp ptr (user or kernel)</div><div class='del'>- * @info: info for modify qp</div><div class='del'>- * @wait: flag to wait or not for modify qp completion</div><div class='del'>- */</div><div class='del'>-void i40iw_hw_modify_qp(struct i40iw_device *iwdev, struct i40iw_qp *iwqp,</div><div class='del'>-			struct i40iw_modify_qp_info *info, bool wait)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	struct i40iw_modify_qp_info *m_info;</div><div class='del'>-	struct i40iw_gen_ae_info ae_info;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, wait);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	m_info = &amp;cqp_info-&gt;in.u.qp_modify.info;</div><div class='del'>-	memcpy(m_info, info, sizeof(*m_info));</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_QP_MODIFY;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.qp_modify.qp = &amp;iwqp-&gt;sc_qp;</div><div class='del'>-	cqp_info-&gt;in.u.qp_modify.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	if (!i40iw_handle_cqp_op(iwdev, cqp_request))</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	switch (m_info-&gt;next_iwarp_state) {</div><div class='del'>-	case I40IW_QP_STATE_RTS:</div><div class='del'>-		if (iwqp-&gt;iwarp_state == I40IW_QP_STATE_IDLE)</div><div class='del'>-			i40iw_send_reset(iwqp-&gt;cm_node);</div><div class='del'>-		/* fall through */</div><div class='del'>-	case I40IW_QP_STATE_IDLE:</div><div class='del'>-	case I40IW_QP_STATE_TERMINATE:</div><div class='del'>-	case I40IW_QP_STATE_CLOSING:</div><div class='del'>-		ae_info.ae_code = I40IW_AE_BAD_CLOSE;</div><div class='del'>-		ae_info.ae_source = 0;</div><div class='del'>-		i40iw_gen_ae(iwdev, &amp;iwqp-&gt;sc_qp, &amp;ae_info, false);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_QP_STATE_ERROR:</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_modify_qp - modify qp request</div><div class='del'>- * @ibqp: qp's pointer for modify</div><div class='del'>- * @attr: access attributes</div><div class='del'>- * @attr_mask: state mask</div><div class='del'>- * @udata: user data</div><div class='del'>- */</div><div class='del'>-int i40iw_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='del'>-		    int attr_mask, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp *iwqp = to_iwqp(ibqp);</div><div class='del'>-	struct i40iw_device *iwdev = iwqp-&gt;iwdev;</div><div class='del'>-	struct i40iw_qp_host_ctx_info *ctx_info;</div><div class='del'>-	struct i40iwarp_offload_info *iwarp_info;</div><div class='del'>-	struct i40iw_modify_qp_info info;</div><div class='del'>-	u8 issue_modify_qp = 0;</div><div class='del'>-	u8 dont_wait = 0;</div><div class='del'>-	u32 err;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	memset(&amp;info, 0, sizeof(info));</div><div class='del'>-	ctx_info = &amp;iwqp-&gt;ctx_info;</div><div class='del'>-	iwarp_info = &amp;iwqp-&gt;iwarp_info;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-	if (attr_mask &amp; IB_QP_STATE) {</div><div class='del'>-		if (iwdev-&gt;closing &amp;&amp; attr-&gt;qp_state != IB_QPS_ERR) {</div><div class='del'>-			err = -EINVAL;</div><div class='del'>-			goto exit;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		switch (attr-&gt;qp_state) {</div><div class='del'>-		case IB_QPS_INIT:</div><div class='del'>-		case IB_QPS_RTR:</div><div class='del'>-			if (iwqp-&gt;iwarp_state &gt; (u32)I40IW_QP_STATE_IDLE) {</div><div class='del'>-				err = -EINVAL;</div><div class='del'>-				goto exit;</div><div class='del'>-			}</div><div class='del'>-			if (iwqp-&gt;iwarp_state == I40IW_QP_STATE_INVALID) {</div><div class='del'>-				info.next_iwarp_state = I40IW_QP_STATE_IDLE;</div><div class='del'>-				issue_modify_qp = 1;</div><div class='del'>-			}</div><div class='del'>-			break;</div><div class='del'>-		case IB_QPS_RTS:</div><div class='del'>-			if ((iwqp-&gt;iwarp_state &gt; (u32)I40IW_QP_STATE_RTS) ||</div><div class='del'>-			    (!iwqp-&gt;cm_id)) {</div><div class='del'>-				err = -EINVAL;</div><div class='del'>-				goto exit;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			issue_modify_qp = 1;</div><div class='del'>-			iwqp-&gt;hw_tcp_state = I40IW_TCP_STATE_ESTABLISHED;</div><div class='del'>-			iwqp-&gt;hte_added = 1;</div><div class='del'>-			info.next_iwarp_state = I40IW_QP_STATE_RTS;</div><div class='del'>-			info.tcp_ctx_valid = true;</div><div class='del'>-			info.ord_valid = true;</div><div class='del'>-			info.arp_cache_idx_valid = true;</div><div class='del'>-			info.cq_num_valid = true;</div><div class='del'>-			break;</div><div class='del'>-		case IB_QPS_SQD:</div><div class='del'>-			if (iwqp-&gt;hw_iwarp_state &gt; (u32)I40IW_QP_STATE_RTS) {</div><div class='del'>-				err = 0;</div><div class='del'>-				goto exit;</div><div class='del'>-			}</div><div class='del'>-			if ((iwqp-&gt;iwarp_state == (u32)I40IW_QP_STATE_CLOSING) ||</div><div class='del'>-			    (iwqp-&gt;iwarp_state &lt; (u32)I40IW_QP_STATE_RTS)) {</div><div class='del'>-				err = 0;</div><div class='del'>-				goto exit;</div><div class='del'>-			}</div><div class='del'>-			if (iwqp-&gt;iwarp_state &gt; (u32)I40IW_QP_STATE_CLOSING) {</div><div class='del'>-				err = -EINVAL;</div><div class='del'>-				goto exit;</div><div class='del'>-			}</div><div class='del'>-			info.next_iwarp_state = I40IW_QP_STATE_CLOSING;</div><div class='del'>-			issue_modify_qp = 1;</div><div class='del'>-			break;</div><div class='del'>-		case IB_QPS_SQE:</div><div class='del'>-			if (iwqp-&gt;iwarp_state &gt;= (u32)I40IW_QP_STATE_TERMINATE) {</div><div class='del'>-				err = -EINVAL;</div><div class='del'>-				goto exit;</div><div class='del'>-			}</div><div class='del'>-			info.next_iwarp_state = I40IW_QP_STATE_TERMINATE;</div><div class='del'>-			issue_modify_qp = 1;</div><div class='del'>-			break;</div><div class='del'>-		case IB_QPS_ERR:</div><div class='del'>-		case IB_QPS_RESET:</div><div class='del'>-			if (iwqp-&gt;iwarp_state == (u32)I40IW_QP_STATE_ERROR) {</div><div class='del'>-				err = -EINVAL;</div><div class='del'>-				goto exit;</div><div class='del'>-			}</div><div class='del'>-			if (iwqp-&gt;sc_qp.term_flags)</div><div class='del'>-				i40iw_terminate_del_timer(&amp;iwqp-&gt;sc_qp);</div><div class='del'>-			info.next_iwarp_state = I40IW_QP_STATE_ERROR;</div><div class='del'>-			if ((iwqp-&gt;hw_tcp_state &gt; I40IW_TCP_STATE_CLOSED) &amp;&amp;</div><div class='del'>-			    iwdev-&gt;iw_status &amp;&amp;</div><div class='del'>-			    (iwqp-&gt;hw_tcp_state != I40IW_TCP_STATE_TIME_WAIT))</div><div class='del'>-				info.reset_tcp_conn = true;</div><div class='del'>-			else</div><div class='del'>-				dont_wait = 1;</div><div class='del'>-			issue_modify_qp = 1;</div><div class='del'>-			info.next_iwarp_state = I40IW_QP_STATE_ERROR;</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			err = -EINVAL;</div><div class='del'>-			goto exit;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		iwqp-&gt;ibqp_state = attr-&gt;qp_state;</div><div class='del'>-</div><div class='del'>-	}</div><div class='del'>-	if (attr_mask &amp; IB_QP_ACCESS_FLAGS) {</div><div class='del'>-		ctx_info-&gt;iwarp_info_valid = true;</div><div class='del'>-		if (attr-&gt;qp_access_flags &amp; IB_ACCESS_LOCAL_WRITE)</div><div class='del'>-			iwarp_info-&gt;wr_rdresp_en = true;</div><div class='del'>-		if (attr-&gt;qp_access_flags &amp; IB_ACCESS_REMOTE_WRITE)</div><div class='del'>-			iwarp_info-&gt;wr_rdresp_en = true;</div><div class='del'>-		if (attr-&gt;qp_access_flags &amp; IB_ACCESS_REMOTE_READ)</div><div class='del'>-			iwarp_info-&gt;rd_enable = true;</div><div class='del'>-		if (attr-&gt;qp_access_flags &amp; IB_ACCESS_MW_BIND)</div><div class='del'>-			iwarp_info-&gt;bind_en = true;</div><div class='del'>-</div><div class='del'>-		if (iwqp-&gt;user_mode) {</div><div class='del'>-			iwarp_info-&gt;rd_enable = true;</div><div class='del'>-			iwarp_info-&gt;wr_rdresp_en = true;</div><div class='del'>-			iwarp_info-&gt;priv_mode_en = false;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (ctx_info-&gt;iwarp_info_valid) {</div><div class='del'>-		struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-		int ret;</div><div class='del'>-</div><div class='del'>-		ctx_info-&gt;send_cq_num = iwqp-&gt;iwscq-&gt;sc_cq.cq_uk.cq_id;</div><div class='del'>-		ctx_info-&gt;rcv_cq_num = iwqp-&gt;iwrcq-&gt;sc_cq.cq_uk.cq_id;</div><div class='del'>-		ret = dev-&gt;iw_priv_qp_ops-&gt;qp_setctx(&amp;iwqp-&gt;sc_qp,</div><div class='del'>-						     (u64 *)iwqp-&gt;host_ctx.va,</div><div class='del'>-						     ctx_info);</div><div class='del'>-		if (ret) {</div><div class='del'>-			i40iw_pr_err("setting QP context\n");</div><div class='del'>-			err = -EINVAL;</div><div class='del'>-			goto exit;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-	if (issue_modify_qp) {</div><div class='del'>-		i40iw_hw_modify_qp(iwdev, iwqp, &amp;info, true);</div><div class='del'>-</div><div class='del'>-		spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-		iwqp-&gt;iwarp_state = info.next_iwarp_state;</div><div class='del'>-		spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (issue_modify_qp &amp;&amp; (iwqp-&gt;ibqp_state &gt; IB_QPS_RTS)) {</div><div class='del'>-		if (dont_wait) {</div><div class='del'>-			if (iwqp-&gt;cm_id &amp;&amp; iwqp-&gt;hw_tcp_state) {</div><div class='del'>-				spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-				iwqp-&gt;hw_tcp_state = I40IW_TCP_STATE_CLOSED;</div><div class='del'>-				iwqp-&gt;last_aeq = I40IW_AE_RESET_SENT;</div><div class='del'>-				spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-				i40iw_cm_disconn(iwqp);</div><div class='del'>-			}</div><div class='del'>-		} else {</div><div class='del'>-			spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-			if (iwqp-&gt;cm_id) {</div><div class='del'>-				if (atomic_inc_return(&amp;iwqp-&gt;close_timer_started) == 1) {</div><div class='del'>-					iwqp-&gt;cm_id-&gt;add_ref(iwqp-&gt;cm_id);</div><div class='del'>-					i40iw_schedule_cm_timer(iwqp-&gt;cm_node,</div><div class='del'>-								(struct i40iw_puda_buf *)iwqp,</div><div class='del'>-								 I40IW_TIMER_TYPE_CLOSE, 1, 0);</div><div class='del'>-				}</div><div class='del'>-			}</div><div class='del'>-			spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	return 0;</div><div class='del'>-exit:</div><div class='del'>-	spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * cq_free_resources - free up recources for cq</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @iwcq: cq ptr</div><div class='del'>- */</div><div class='del'>-static void cq_free_resources(struct i40iw_device *iwdev, struct i40iw_cq *iwcq)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_sc_cq *cq = &amp;iwcq-&gt;sc_cq;</div><div class='del'>-</div><div class='del'>-	if (!iwcq-&gt;user_mode)</div><div class='del'>-		i40iw_free_dma_mem(iwdev-&gt;sc_dev.hw, &amp;iwcq-&gt;kmem);</div><div class='del'>-	i40iw_free_resource(iwdev, iwdev-&gt;allocated_cqs, cq-&gt;cq_uk.cq_id);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_cq_wq_destroy - send cq destroy cqp</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @cq: hardware control cq</div><div class='del'>- */</div><div class='del'>-void i40iw_cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, true);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_CQ_DESTROY;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.cq_destroy.cq = cq;</div><div class='del'>-	cqp_info-&gt;in.u.cq_destroy.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("CQP-OP Destroy QP fail");</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_destroy_cq - destroy cq</div><div class='del'>- * @ib_cq: cq pointer</div><div class='del'>- * @udata: user data or NULL for kernel object</div><div class='del'>- */</div><div class='del'>-static int i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cq *iwcq;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_sc_cq *cq;</div><div class='del'>-</div><div class='del'>-	if (!ib_cq) {</div><div class='del'>-		i40iw_pr_err("ib_cq == NULL\n");</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	iwcq = to_iwcq(ib_cq);</div><div class='del'>-	iwdev = to_iwdev(ib_cq-&gt;device);</div><div class='del'>-	cq = &amp;iwcq-&gt;sc_cq;</div><div class='del'>-	i40iw_cq_wq_destroy(iwdev, cq);</div><div class='del'>-	cq_free_resources(iwdev, iwcq);</div><div class='del'>-	kfree(iwcq);</div><div class='del'>-	i40iw_rem_devusecount(iwdev);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_create_cq - create cq</div><div class='del'>- * @ibdev: device pointer from stack</div><div class='del'>- * @attr: attributes for cq</div><div class='del'>- * @udata: user data</div><div class='del'>- */</div><div class='del'>-static struct ib_cq *i40iw_create_cq(struct ib_device *ibdev,</div><div class='del'>-				     const struct ib_cq_init_attr *attr,</div><div class='del'>-				     struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = to_iwdev(ibdev);</div><div class='del'>-	struct i40iw_cq *iwcq;</div><div class='del'>-	struct i40iw_pbl *iwpbl;</div><div class='del'>-	u32 cq_num = 0;</div><div class='del'>-	struct i40iw_sc_cq *cq;</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_cq_init_info info;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	struct i40iw_cq_uk_init_info *ukinfo = &amp;info.cq_uk_init_info;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int err_code;</div><div class='del'>-	int entries = attr-&gt;cqe;</div><div class='del'>-</div><div class='del'>-	if (iwdev-&gt;closing)</div><div class='del'>-		return ERR_PTR(-ENODEV);</div><div class='del'>-</div><div class='del'>-	if (entries &gt; iwdev-&gt;max_cqe)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	iwcq = kzalloc(sizeof(*iwcq), GFP_KERNEL);</div><div class='del'>-	if (!iwcq)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	memset(&amp;info, 0, sizeof(info));</div><div class='del'>-</div><div class='del'>-	err_code = i40iw_alloc_resource(iwdev, iwdev-&gt;allocated_cqs,</div><div class='del'>-					iwdev-&gt;max_cq, &amp;cq_num,</div><div class='del'>-					&amp;iwdev-&gt;next_cq);</div><div class='del'>-	if (err_code)</div><div class='del'>-		goto error;</div><div class='del'>-</div><div class='del'>-	cq = &amp;iwcq-&gt;sc_cq;</div><div class='del'>-	cq-&gt;back_cq = (void *)iwcq;</div><div class='del'>-	spin_lock_init(&amp;iwcq-&gt;lock);</div><div class='del'>-</div><div class='del'>-	info.dev = dev;</div><div class='del'>-	ukinfo-&gt;cq_size = max(entries, 4);</div><div class='del'>-	ukinfo-&gt;cq_id = cq_num;</div><div class='del'>-	iwcq-&gt;ibcq.cqe = info.cq_uk_init_info.cq_size;</div><div class='del'>-	info.ceqe_mask = 0;</div><div class='del'>-	if (attr-&gt;comp_vector &lt; iwdev-&gt;ceqs_count)</div><div class='del'>-		info.ceq_id = attr-&gt;comp_vector;</div><div class='del'>-	info.ceq_id_valid = true;</div><div class='del'>-	info.ceqe_mask = 1;</div><div class='del'>-	info.type = I40IW_CQ_TYPE_IWARP;</div><div class='del'>-	if (udata) {</div><div class='del'>-		struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context(</div><div class='del'>-			udata, struct i40iw_ucontext, ibucontext);</div><div class='del'>-		struct i40iw_create_cq_req req;</div><div class='del'>-		struct i40iw_cq_mr *cqmr;</div><div class='del'>-</div><div class='del'>-		memset(&amp;req, 0, sizeof(req));</div><div class='del'>-		iwcq-&gt;user_mode = true;</div><div class='del'>-		if (ib_copy_from_udata(&amp;req, udata, sizeof(struct i40iw_create_cq_req))) {</div><div class='del'>-			err_code = -EFAULT;</div><div class='del'>-			goto cq_free_resources;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		spin_lock_irqsave(&amp;ucontext-&gt;cq_reg_mem_list_lock, flags);</div><div class='del'>-		iwpbl = i40iw_get_pbl((unsigned long)req.user_cq_buffer,</div><div class='del'>-				      &amp;ucontext-&gt;cq_reg_mem_list);</div><div class='del'>-		spin_unlock_irqrestore(&amp;ucontext-&gt;cq_reg_mem_list_lock, flags);</div><div class='del'>-		if (!iwpbl) {</div><div class='del'>-			err_code = -EPROTO;</div><div class='del'>-			goto cq_free_resources;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		iwcq-&gt;iwpbl = iwpbl;</div><div class='del'>-		iwcq-&gt;cq_mem_size = 0;</div><div class='del'>-		cqmr = &amp;iwpbl-&gt;cq_mr;</div><div class='del'>-		info.shadow_area_pa = cpu_to_le64(cqmr-&gt;shadow);</div><div class='del'>-		if (iwpbl-&gt;pbl_allocated) {</div><div class='del'>-			info.virtual_map = true;</div><div class='del'>-			info.pbl_chunk_size = 1;</div><div class='del'>-			info.first_pm_pbl_idx = cqmr-&gt;cq_pbl.idx;</div><div class='del'>-		} else {</div><div class='del'>-			info.cq_base_pa = cqmr-&gt;cq_pbl.addr;</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		/* Kmode allocations */</div><div class='del'>-		int rsize;</div><div class='del'>-		int shadow;</div><div class='del'>-</div><div class='del'>-		rsize = info.cq_uk_init_info.cq_size * sizeof(struct i40iw_cqe);</div><div class='del'>-		rsize = round_up(rsize, 256);</div><div class='del'>-		shadow = I40IW_SHADOW_AREA_SIZE &lt;&lt; 3;</div><div class='del'>-		status = i40iw_allocate_dma_mem(dev-&gt;hw, &amp;iwcq-&gt;kmem,</div><div class='del'>-						rsize + shadow, 256);</div><div class='del'>-		if (status) {</div><div class='del'>-			err_code = -ENOMEM;</div><div class='del'>-			goto cq_free_resources;</div><div class='del'>-		}</div><div class='del'>-		ukinfo-&gt;cq_base = iwcq-&gt;kmem.va;</div><div class='del'>-		info.cq_base_pa = iwcq-&gt;kmem.pa;</div><div class='del'>-		info.shadow_area_pa = info.cq_base_pa + rsize;</div><div class='del'>-		ukinfo-&gt;shadow_area = iwcq-&gt;kmem.va + rsize;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (dev-&gt;iw_priv_cq_ops-&gt;cq_init(cq, &amp;info)) {</div><div class='del'>-		i40iw_pr_err("init cq fail\n");</div><div class='del'>-		err_code = -EPROTO;</div><div class='del'>-		goto cq_free_resources;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, true);</div><div class='del'>-	if (!cqp_request) {</div><div class='del'>-		err_code = -ENOMEM;</div><div class='del'>-		goto cq_free_resources;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_CQ_CREATE;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.cq_create.cq = cq;</div><div class='del'>-	cqp_info-&gt;in.u.cq_create.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status) {</div><div class='del'>-		i40iw_pr_err("CQP-OP Create QP fail");</div><div class='del'>-		err_code = -EPROTO;</div><div class='del'>-		goto cq_free_resources;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (udata) {</div><div class='del'>-		struct i40iw_create_cq_resp resp;</div><div class='del'>-</div><div class='del'>-		memset(&amp;resp, 0, sizeof(resp));</div><div class='del'>-		resp.cq_id = info.cq_uk_init_info.cq_id;</div><div class='del'>-		resp.cq_size = info.cq_uk_init_info.cq_size;</div><div class='del'>-		if (ib_copy_to_udata(udata, &amp;resp, sizeof(resp))) {</div><div class='del'>-			i40iw_pr_err("copy to user data\n");</div><div class='del'>-			err_code = -EPROTO;</div><div class='del'>-			goto cq_destroy;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i40iw_add_devusecount(iwdev);</div><div class='del'>-	return (struct ib_cq *)iwcq;</div><div class='del'>-</div><div class='del'>-cq_destroy:</div><div class='del'>-	i40iw_cq_wq_destroy(iwdev, cq);</div><div class='del'>-cq_free_resources:</div><div class='del'>-	cq_free_resources(iwdev, iwcq);</div><div class='del'>-error:</div><div class='del'>-	kfree(iwcq);</div><div class='del'>-	return ERR_PTR(err_code);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_user_access - get hw access from IB access</div><div class='del'>- * @acc: IB access to return hw access</div><div class='del'>- */</div><div class='del'>-static inline u16 i40iw_get_user_access(int acc)</div><div class='del'>-{</div><div class='del'>-	u16 access = 0;</div><div class='del'>-</div><div class='del'>-	access |= (acc &amp; IB_ACCESS_LOCAL_WRITE) ? I40IW_ACCESS_FLAGS_LOCALWRITE : 0;</div><div class='del'>-	access |= (acc &amp; IB_ACCESS_REMOTE_WRITE) ? I40IW_ACCESS_FLAGS_REMOTEWRITE : 0;</div><div class='del'>-	access |= (acc &amp; IB_ACCESS_REMOTE_READ) ? I40IW_ACCESS_FLAGS_REMOTEREAD : 0;</div><div class='del'>-	access |= (acc &amp; IB_ACCESS_MW_BIND) ? I40IW_ACCESS_FLAGS_BIND_WINDOW : 0;</div><div class='del'>-	return access;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_free_stag - free stag resource</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @stag: stag to free</div><div class='del'>- */</div><div class='del'>-static void i40iw_free_stag(struct i40iw_device *iwdev, u32 stag)</div><div class='del'>-{</div><div class='del'>-	u32 stag_idx;</div><div class='del'>-</div><div class='del'>-	stag_idx = (stag &amp; iwdev-&gt;mr_stagmask) &gt;&gt; I40IW_CQPSQ_STAG_IDX_SHIFT;</div><div class='del'>-	i40iw_free_resource(iwdev, iwdev-&gt;allocated_mrs, stag_idx);</div><div class='del'>-	i40iw_rem_devusecount(iwdev);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_create_stag - create random stag</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- */</div><div class='del'>-static u32 i40iw_create_stag(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	u32 stag = 0;</div><div class='del'>-	u32 stag_index = 0;</div><div class='del'>-	u32 next_stag_index;</div><div class='del'>-	u32 driver_key;</div><div class='del'>-	u32 random;</div><div class='del'>-	u8 consumer_key;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	get_random_bytes(&amp;random, sizeof(random));</div><div class='del'>-	consumer_key = (u8)random;</div><div class='del'>-</div><div class='del'>-	driver_key = random &amp; ~iwdev-&gt;mr_stagmask;</div><div class='del'>-	next_stag_index = (random &amp; iwdev-&gt;mr_stagmask) &gt;&gt; 8;</div><div class='del'>-	next_stag_index %= iwdev-&gt;max_mr;</div><div class='del'>-</div><div class='del'>-	ret = i40iw_alloc_resource(iwdev,</div><div class='del'>-				   iwdev-&gt;allocated_mrs, iwdev-&gt;max_mr,</div><div class='del'>-				   &amp;stag_index, &amp;next_stag_index);</div><div class='del'>-	if (!ret) {</div><div class='del'>-		stag = stag_index &lt;&lt; I40IW_CQPSQ_STAG_IDX_SHIFT;</div><div class='del'>-		stag |= driver_key;</div><div class='del'>-		stag += (u32)consumer_key;</div><div class='del'>-		i40iw_add_devusecount(iwdev);</div><div class='del'>-	}</div><div class='del'>-	return stag;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_next_pbl_addr - Get next pbl address</div><div class='del'>- * @pbl: pointer to a pble</div><div class='del'>- * @pinfo: info pointer</div><div class='del'>- * @idx: index</div><div class='del'>- */</div><div class='del'>-static inline u64 *i40iw_next_pbl_addr(u64 *pbl,</div><div class='del'>-				       struct i40iw_pble_info **pinfo,</div><div class='del'>-				       u32 *idx)</div><div class='del'>-{</div><div class='del'>-	*idx += 1;</div><div class='del'>-	if ((!(*pinfo)) || (*idx != (*pinfo)-&gt;cnt))</div><div class='del'>-		return ++pbl;</div><div class='del'>-	*idx = 0;</div><div class='del'>-	(*pinfo)++;</div><div class='del'>-	return (u64 *)(*pinfo)-&gt;addr;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_copy_user_pgaddrs - copy user page address to pble's os locally</div><div class='del'>- * @iwmr: iwmr for IB's user page addresses</div><div class='del'>- * @pbl: ple pointer to save 1 level or 0 level pble</div><div class='del'>- * @level: indicated level 0, 1 or 2</div><div class='del'>- */</div><div class='del'>-static void i40iw_copy_user_pgaddrs(struct i40iw_mr *iwmr,</div><div class='del'>-				    u64 *pbl,</div><div class='del'>-				    enum i40iw_pble_level level)</div><div class='del'>-{</div><div class='del'>-	struct ib_umem *region = iwmr-&gt;region;</div><div class='del'>-	struct i40iw_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='del'>-	struct i40iw_pble_alloc *palloc = &amp;iwpbl-&gt;pble_alloc;</div><div class='del'>-	struct i40iw_pble_info *pinfo;</div><div class='del'>-	struct ib_block_iter biter;</div><div class='del'>-	u32 idx = 0;</div><div class='del'>-</div><div class='del'>-	pinfo = (level == I40IW_LEVEL_1) ? NULL : palloc-&gt;level2.leaf;</div><div class='del'>-</div><div class='del'>-	if (iwmr-&gt;type == IW_MEMREG_TYPE_QP)</div><div class='del'>-		iwpbl-&gt;qp_mr.sq_page = sg_page(region-&gt;sg_head.sgl);</div><div class='del'>-</div><div class='del'>-	rdma_for_each_block(region-&gt;sg_head.sgl, &amp;biter, region-&gt;nmap,</div><div class='del'>-			    iwmr-&gt;page_size) {</div><div class='del'>-		*pbl = rdma_block_iter_dma_address(&amp;biter);</div><div class='del'>-		pbl = i40iw_next_pbl_addr(pbl, &amp;pinfo, &amp;idx);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_check_mem_contiguous - check if pbls stored in arr are contiguous</div><div class='del'>- * @arr: lvl1 pbl array</div><div class='del'>- * @npages: page count</div><div class='del'>- * pg_size: page size</div><div class='del'>- *</div><div class='del'>- */</div><div class='del'>-static bool i40iw_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size)</div><div class='del'>-{</div><div class='del'>-	u32 pg_idx;</div><div class='del'>-</div><div class='del'>-	for (pg_idx = 0; pg_idx &lt; npages; pg_idx++) {</div><div class='del'>-		if ((*arr + (pg_size * pg_idx)) != arr[pg_idx])</div><div class='del'>-			return false;</div><div class='del'>-	}</div><div class='del'>-	return true;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_check_mr_contiguous - check if MR is physically contiguous</div><div class='del'>- * @palloc: pbl allocation struct</div><div class='del'>- * pg_size: page size</div><div class='del'>- */</div><div class='del'>-static bool i40iw_check_mr_contiguous(struct i40iw_pble_alloc *palloc, u32 pg_size)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_pble_level2 *lvl2 = &amp;palloc-&gt;level2;</div><div class='del'>-	struct i40iw_pble_info *leaf = lvl2-&gt;leaf;</div><div class='del'>-	u64 *arr = NULL;</div><div class='del'>-	u64 *start_addr = NULL;</div><div class='del'>-	int i;</div><div class='del'>-	bool ret;</div><div class='del'>-</div><div class='del'>-	if (palloc-&gt;level == I40IW_LEVEL_1) {</div><div class='del'>-		arr = (u64 *)palloc-&gt;level1.addr;</div><div class='del'>-		ret = i40iw_check_mem_contiguous(arr, palloc-&gt;total_cnt, pg_size);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	start_addr = (u64 *)leaf-&gt;addr;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; lvl2-&gt;leaf_cnt; i++, leaf++) {</div><div class='del'>-		arr = (u64 *)leaf-&gt;addr;</div><div class='del'>-		if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr)</div><div class='del'>-			return false;</div><div class='del'>-		ret = i40iw_check_mem_contiguous(arr, leaf-&gt;cnt, pg_size);</div><div class='del'>-		if (!ret)</div><div class='del'>-			return false;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return true;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_setup_pbles - copy user pg address to pble's</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @iwmr: mr pointer for this memory registration</div><div class='del'>- * @use_pbles: flag if to use pble's</div><div class='del'>- */</div><div class='del'>-static int i40iw_setup_pbles(struct i40iw_device *iwdev,</div><div class='del'>-			     struct i40iw_mr *iwmr,</div><div class='del'>-			     bool use_pbles)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='del'>-	struct i40iw_pble_alloc *palloc = &amp;iwpbl-&gt;pble_alloc;</div><div class='del'>-	struct i40iw_pble_info *pinfo;</div><div class='del'>-	u64 *pbl;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	enum i40iw_pble_level level = I40IW_LEVEL_1;</div><div class='del'>-</div><div class='del'>-	if (use_pbles) {</div><div class='del'>-		mutex_lock(&amp;iwdev-&gt;pbl_mutex);</div><div class='del'>-		status = i40iw_get_pble(&amp;iwdev-&gt;sc_dev, iwdev-&gt;pble_rsrc, palloc, iwmr-&gt;page_cnt);</div><div class='del'>-		mutex_unlock(&amp;iwdev-&gt;pbl_mutex);</div><div class='del'>-		if (status)</div><div class='del'>-			return -ENOMEM;</div><div class='del'>-</div><div class='del'>-		iwpbl-&gt;pbl_allocated = true;</div><div class='del'>-		level = palloc-&gt;level;</div><div class='del'>-		pinfo = (level == I40IW_LEVEL_1) ? &amp;palloc-&gt;level1 : palloc-&gt;level2.leaf;</div><div class='del'>-		pbl = (u64 *)pinfo-&gt;addr;</div><div class='del'>-	} else {</div><div class='del'>-		pbl = iwmr-&gt;pgaddrmem;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i40iw_copy_user_pgaddrs(iwmr, pbl, level);</div><div class='del'>-</div><div class='del'>-	if (use_pbles)</div><div class='del'>-		iwmr-&gt;pgaddrmem[0] = *pbl;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_handle_q_mem - handle memory for qp and cq</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @req: information for q memory management</div><div class='del'>- * @iwpbl: pble struct</div><div class='del'>- * @use_pbles: flag to use pble</div><div class='del'>- */</div><div class='del'>-static int i40iw_handle_q_mem(struct i40iw_device *iwdev,</div><div class='del'>-			      struct i40iw_mem_reg_req *req,</div><div class='del'>-			      struct i40iw_pbl *iwpbl,</div><div class='del'>-			      bool use_pbles)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_pble_alloc *palloc = &amp;iwpbl-&gt;pble_alloc;</div><div class='del'>-	struct i40iw_mr *iwmr = iwpbl-&gt;iwmr;</div><div class='del'>-	struct i40iw_qp_mr *qpmr = &amp;iwpbl-&gt;qp_mr;</div><div class='del'>-	struct i40iw_cq_mr *cqmr = &amp;iwpbl-&gt;cq_mr;</div><div class='del'>-	struct i40iw_hmc_pble *hmc_p;</div><div class='del'>-	u64 *arr = iwmr-&gt;pgaddrmem;</div><div class='del'>-	u32 pg_size;</div><div class='del'>-	int err;</div><div class='del'>-	int total;</div><div class='del'>-	bool ret = true;</div><div class='del'>-</div><div class='del'>-	total = req-&gt;sq_pages + req-&gt;rq_pages + req-&gt;cq_pages;</div><div class='del'>-	pg_size = iwmr-&gt;page_size;</div><div class='del'>-</div><div class='del'>-	err = i40iw_setup_pbles(iwdev, iwmr, use_pbles);</div><div class='del'>-	if (err)</div><div class='del'>-		return err;</div><div class='del'>-</div><div class='del'>-	if (use_pbles &amp;&amp; (palloc-&gt;level != I40IW_LEVEL_1)) {</div><div class='del'>-		i40iw_free_pble(iwdev-&gt;pble_rsrc, palloc);</div><div class='del'>-		iwpbl-&gt;pbl_allocated = false;</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (use_pbles)</div><div class='del'>-		arr = (u64 *)palloc-&gt;level1.addr;</div><div class='del'>-</div><div class='del'>-	if (iwmr-&gt;type == IW_MEMREG_TYPE_QP) {</div><div class='del'>-		hmc_p = &amp;qpmr-&gt;sq_pbl;</div><div class='del'>-		qpmr-&gt;shadow = (dma_addr_t)arr[total];</div><div class='del'>-</div><div class='del'>-		if (use_pbles) {</div><div class='del'>-			ret = i40iw_check_mem_contiguous(arr, req-&gt;sq_pages, pg_size);</div><div class='del'>-			if (ret)</div><div class='del'>-				ret = i40iw_check_mem_contiguous(&amp;arr[req-&gt;sq_pages], req-&gt;rq_pages, pg_size);</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (!ret) {</div><div class='del'>-			hmc_p-&gt;idx = palloc-&gt;level1.idx;</div><div class='del'>-			hmc_p = &amp;qpmr-&gt;rq_pbl;</div><div class='del'>-			hmc_p-&gt;idx = palloc-&gt;level1.idx + req-&gt;sq_pages;</div><div class='del'>-		} else {</div><div class='del'>-			hmc_p-&gt;addr = arr[0];</div><div class='del'>-			hmc_p = &amp;qpmr-&gt;rq_pbl;</div><div class='del'>-			hmc_p-&gt;addr = arr[req-&gt;sq_pages];</div><div class='del'>-		}</div><div class='del'>-	} else {		/* CQ */</div><div class='del'>-		hmc_p = &amp;cqmr-&gt;cq_pbl;</div><div class='del'>-		cqmr-&gt;shadow = (dma_addr_t)arr[total];</div><div class='del'>-</div><div class='del'>-		if (use_pbles)</div><div class='del'>-			ret = i40iw_check_mem_contiguous(arr, req-&gt;cq_pages, pg_size);</div><div class='del'>-</div><div class='del'>-		if (!ret)</div><div class='del'>-			hmc_p-&gt;idx = palloc-&gt;level1.idx;</div><div class='del'>-		else</div><div class='del'>-			hmc_p-&gt;addr = arr[0];</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (use_pbles &amp;&amp; ret) {</div><div class='del'>-		i40iw_free_pble(iwdev-&gt;pble_rsrc, palloc);</div><div class='del'>-		iwpbl-&gt;pbl_allocated = false;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hw_alloc_stag - cqp command to allocate stag</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @iwmr: iwarp mr pointer</div><div class='del'>- */</div><div class='del'>-static int i40iw_hw_alloc_stag(struct i40iw_device *iwdev, struct i40iw_mr *iwmr)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_allocate_stag_info *info;</div><div class='del'>-	struct i40iw_pd *iwpd = to_iwpd(iwmr-&gt;ibmr.pd);</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	int err = 0;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, true);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	info = &amp;cqp_info-&gt;in.u.alloc_stag.info;</div><div class='del'>-	memset(info, 0, sizeof(*info));</div><div class='del'>-	info-&gt;page_size = PAGE_SIZE;</div><div class='del'>-	info-&gt;stag_idx = iwmr-&gt;stag &gt;&gt; I40IW_CQPSQ_STAG_IDX_SHIFT;</div><div class='del'>-	info-&gt;pd_id = iwpd-&gt;sc_pd.pd_id;</div><div class='del'>-	info-&gt;total_len = iwmr-&gt;length;</div><div class='del'>-	info-&gt;remote_access = true;</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_ALLOC_STAG;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.alloc_stag.dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	cqp_info-&gt;in.u.alloc_stag.scratch = (uintptr_t)cqp_request;</div><div class='del'>-</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status) {</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		i40iw_pr_err("CQP-OP MR Reg fail");</div><div class='del'>-	}</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_alloc_mr - register stag for fast memory registration</div><div class='del'>- * @pd: ibpd pointer</div><div class='del'>- * @mr_type: memory for stag registrion</div><div class='del'>- * @max_num_sg: man number of pages</div><div class='del'>- * @udata: user data or NULL for kernel objects</div><div class='del'>- */</div><div class='del'>-static struct ib_mr *i40iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,</div><div class='del'>-				    u32 max_num_sg, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_pd *iwpd = to_iwpd(pd);</div><div class='del'>-	struct i40iw_device *iwdev = to_iwdev(pd-&gt;device);</div><div class='del'>-	struct i40iw_pble_alloc *palloc;</div><div class='del'>-	struct i40iw_pbl *iwpbl;</div><div class='del'>-	struct i40iw_mr *iwmr;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	u32 stag;</div><div class='del'>-	int err_code = -ENOMEM;</div><div class='del'>-</div><div class='del'>-	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);</div><div class='del'>-	if (!iwmr)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	stag = i40iw_create_stag(iwdev);</div><div class='del'>-	if (!stag) {</div><div class='del'>-		err_code = -EOVERFLOW;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-	stag &amp;= ~I40IW_CQPSQ_STAG_KEY_MASK;</div><div class='del'>-	iwmr-&gt;stag = stag;</div><div class='del'>-	iwmr-&gt;ibmr.rkey = stag;</div><div class='del'>-	iwmr-&gt;ibmr.lkey = stag;</div><div class='del'>-	iwmr-&gt;ibmr.pd = pd;</div><div class='del'>-	iwmr-&gt;ibmr.device = pd-&gt;device;</div><div class='del'>-	iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='del'>-	iwpbl-&gt;iwmr = iwmr;</div><div class='del'>-	iwmr-&gt;type = IW_MEMREG_TYPE_MEM;</div><div class='del'>-	palloc = &amp;iwpbl-&gt;pble_alloc;</div><div class='del'>-	iwmr-&gt;page_cnt = max_num_sg;</div><div class='del'>-	mutex_lock(&amp;iwdev-&gt;pbl_mutex);</div><div class='del'>-	status = i40iw_get_pble(&amp;iwdev-&gt;sc_dev, iwdev-&gt;pble_rsrc, palloc, iwmr-&gt;page_cnt);</div><div class='del'>-	mutex_unlock(&amp;iwdev-&gt;pbl_mutex);</div><div class='del'>-	if (status)</div><div class='del'>-		goto err1;</div><div class='del'>-</div><div class='del'>-	if (palloc-&gt;level != I40IW_LEVEL_1)</div><div class='del'>-		goto err2;</div><div class='del'>-	err_code = i40iw_hw_alloc_stag(iwdev, iwmr);</div><div class='del'>-	if (err_code)</div><div class='del'>-		goto err2;</div><div class='del'>-	iwpbl-&gt;pbl_allocated = true;</div><div class='del'>-	i40iw_add_pdusecount(iwpd);</div><div class='del'>-	return &amp;iwmr-&gt;ibmr;</div><div class='del'>-err2:</div><div class='del'>-	i40iw_free_pble(iwdev-&gt;pble_rsrc, palloc);</div><div class='del'>-err1:</div><div class='del'>-	i40iw_free_stag(iwdev, stag);</div><div class='del'>-err:</div><div class='del'>-	kfree(iwmr);</div><div class='del'>-	return ERR_PTR(err_code);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_set_page - populate pbl list for fmr</div><div class='del'>- * @ibmr: ib mem to access iwarp mr pointer</div><div class='del'>- * @addr: page dma address fro pbl list</div><div class='del'>- */</div><div class='del'>-static int i40iw_set_page(struct ib_mr *ibmr, u64 addr)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_mr *iwmr = to_iwmr(ibmr);</div><div class='del'>-	struct i40iw_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='del'>-	struct i40iw_pble_alloc *palloc = &amp;iwpbl-&gt;pble_alloc;</div><div class='del'>-	u64 *pbl;</div><div class='del'>-</div><div class='del'>-	if (unlikely(iwmr-&gt;npages == iwmr-&gt;page_cnt))</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	pbl = (u64 *)palloc-&gt;level1.addr;</div><div class='del'>-	pbl[iwmr-&gt;npages++] = cpu_to_le64(addr);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_map_mr_sg - map of sg list for fmr</div><div class='del'>- * @ibmr: ib mem to access iwarp mr pointer</div><div class='del'>- * @sg: scatter gather list for fmr</div><div class='del'>- * @sg_nents: number of sg pages</div><div class='del'>- */</div><div class='del'>-static int i40iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,</div><div class='del'>-			   int sg_nents, unsigned int *sg_offset)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_mr *iwmr = to_iwmr(ibmr);</div><div class='del'>-</div><div class='del'>-	iwmr-&gt;npages = 0;</div><div class='del'>-	return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, i40iw_set_page);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_drain_sq - drain the send queue</div><div class='del'>- * @ibqp: ib qp pointer</div><div class='del'>- */</div><div class='del'>-static void i40iw_drain_sq(struct ib_qp *ibqp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp *iwqp = to_iwqp(ibqp);</div><div class='del'>-	struct i40iw_sc_qp *qp = &amp;iwqp-&gt;sc_qp;</div><div class='del'>-</div><div class='del'>-	if (I40IW_RING_MORE_WORK(qp-&gt;qp_uk.sq_ring))</div><div class='del'>-		wait_for_completion(&amp;iwqp-&gt;sq_drained);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_drain_rq - drain the receive queue</div><div class='del'>- * @ibqp: ib qp pointer</div><div class='del'>- */</div><div class='del'>-static void i40iw_drain_rq(struct ib_qp *ibqp)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp *iwqp = to_iwqp(ibqp);</div><div class='del'>-	struct i40iw_sc_qp *qp = &amp;iwqp-&gt;sc_qp;</div><div class='del'>-</div><div class='del'>-	if (I40IW_RING_MORE_WORK(qp-&gt;qp_uk.rq_ring))</div><div class='del'>-		wait_for_completion(&amp;iwqp-&gt;rq_drained);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_hwreg_mr - send cqp command for memory registration</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- * @iwmr: iwarp mr pointer</div><div class='del'>- * @access: access for MR</div><div class='del'>- */</div><div class='del'>-static int i40iw_hwreg_mr(struct i40iw_device *iwdev,</div><div class='del'>-			  struct i40iw_mr *iwmr,</div><div class='del'>-			  u16 access)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='del'>-	struct i40iw_reg_ns_stag_info *stag_info;</div><div class='del'>-	struct i40iw_pd *iwpd = to_iwpd(iwmr-&gt;ibmr.pd);</div><div class='del'>-	struct i40iw_pble_alloc *palloc = &amp;iwpbl-&gt;pble_alloc;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	int err = 0;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, true);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	stag_info = &amp;cqp_info-&gt;in.u.mr_reg_non_shared.info;</div><div class='del'>-	memset(stag_info, 0, sizeof(*stag_info));</div><div class='del'>-	stag_info-&gt;va = (void *)(unsigned long)iwpbl-&gt;user_base;</div><div class='del'>-	stag_info-&gt;stag_idx = iwmr-&gt;stag &gt;&gt; I40IW_CQPSQ_STAG_IDX_SHIFT;</div><div class='del'>-	stag_info-&gt;stag_key = (u8)iwmr-&gt;stag;</div><div class='del'>-	stag_info-&gt;total_len = iwmr-&gt;length;</div><div class='del'>-	stag_info-&gt;access_rights = access;</div><div class='del'>-	stag_info-&gt;pd_id = iwpd-&gt;sc_pd.pd_id;</div><div class='del'>-	stag_info-&gt;addr_type = I40IW_ADDR_TYPE_VA_BASED;</div><div class='del'>-	stag_info-&gt;page_size = iwmr-&gt;page_size;</div><div class='del'>-</div><div class='del'>-	if (iwpbl-&gt;pbl_allocated) {</div><div class='del'>-		if (palloc-&gt;level == I40IW_LEVEL_1) {</div><div class='del'>-			stag_info-&gt;first_pm_pbl_index = palloc-&gt;level1.idx;</div><div class='del'>-			stag_info-&gt;chunk_size = 1;</div><div class='del'>-		} else {</div><div class='del'>-			stag_info-&gt;first_pm_pbl_index = palloc-&gt;level2.root.idx;</div><div class='del'>-			stag_info-&gt;chunk_size = 3;</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		stag_info-&gt;reg_addr_pa = iwmr-&gt;pgaddrmem[0];</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_MR_REG_NON_SHARED;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.mr_reg_non_shared.dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	cqp_info-&gt;in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request;</div><div class='del'>-</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status) {</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		i40iw_pr_err("CQP-OP MR Reg fail");</div><div class='del'>-	}</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_reg_user_mr - Register a user memory region</div><div class='del'>- * @pd: ptr of pd</div><div class='del'>- * @start: virtual start address</div><div class='del'>- * @length: length of mr</div><div class='del'>- * @virt: virtual address</div><div class='del'>- * @acc: access of mr</div><div class='del'>- * @udata: user data</div><div class='del'>- */</div><div class='del'>-static struct ib_mr *i40iw_reg_user_mr(struct ib_pd *pd,</div><div class='del'>-				       u64 start,</div><div class='del'>-				       u64 length,</div><div class='del'>-				       u64 virt,</div><div class='del'>-				       int acc,</div><div class='del'>-				       struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_pd *iwpd = to_iwpd(pd);</div><div class='del'>-	struct i40iw_device *iwdev = to_iwdev(pd-&gt;device);</div><div class='del'>-	struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context(</div><div class='del'>-		udata, struct i40iw_ucontext, ibucontext);</div><div class='del'>-	struct i40iw_pble_alloc *palloc;</div><div class='del'>-	struct i40iw_pbl *iwpbl;</div><div class='del'>-	struct i40iw_mr *iwmr;</div><div class='del'>-	struct ib_umem *region;</div><div class='del'>-	struct i40iw_mem_reg_req req;</div><div class='del'>-	u64 pbl_depth = 0;</div><div class='del'>-	u32 stag = 0;</div><div class='del'>-	u16 access;</div><div class='del'>-	u64 region_length;</div><div class='del'>-	bool use_pbles = false;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int err = -ENOSYS;</div><div class='del'>-	int ret;</div><div class='del'>-	int pg_shift;</div><div class='del'>-</div><div class='del'>-	if (iwdev-&gt;closing)</div><div class='del'>-		return ERR_PTR(-ENODEV);</div><div class='del'>-</div><div class='del'>-	if (length &gt; I40IW_MAX_MR_SIZE)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-	region = ib_umem_get(udata, start, length, acc, 0);</div><div class='del'>-	if (IS_ERR(region))</div><div class='del'>-		return (struct ib_mr *)region;</div><div class='del'>-</div><div class='del'>-	if (ib_copy_from_udata(&amp;req, udata, sizeof(req))) {</div><div class='del'>-		ib_umem_release(region);</div><div class='del'>-		return ERR_PTR(-EFAULT);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);</div><div class='del'>-	if (!iwmr) {</div><div class='del'>-		ib_umem_release(region);</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='del'>-	iwpbl-&gt;iwmr = iwmr;</div><div class='del'>-	iwmr-&gt;region = region;</div><div class='del'>-	iwmr-&gt;ibmr.pd = pd;</div><div class='del'>-	iwmr-&gt;ibmr.device = pd-&gt;device;</div><div class='del'>-</div><div class='del'>-	iwmr-&gt;page_size = PAGE_SIZE;</div><div class='del'>-	if (req.reg_type == IW_MEMREG_TYPE_MEM)</div><div class='del'>-		iwmr-&gt;page_size = ib_umem_find_best_pgsz(region, SZ_4K | SZ_2M,</div><div class='del'>-							 virt);</div><div class='del'>-</div><div class='del'>-	region_length = region-&gt;length + (start &amp; (iwmr-&gt;page_size - 1));</div><div class='del'>-	pg_shift = ffs(iwmr-&gt;page_size) - 1;</div><div class='del'>-	pbl_depth = region_length &gt;&gt; pg_shift;</div><div class='del'>-	pbl_depth += (region_length &amp; (iwmr-&gt;page_size - 1)) ? 1 : 0;</div><div class='del'>-	iwmr-&gt;length = region-&gt;length;</div><div class='del'>-</div><div class='del'>-	iwpbl-&gt;user_base = virt;</div><div class='del'>-	palloc = &amp;iwpbl-&gt;pble_alloc;</div><div class='del'>-</div><div class='del'>-	iwmr-&gt;type = req.reg_type;</div><div class='del'>-	iwmr-&gt;page_cnt = (u32)pbl_depth;</div><div class='del'>-</div><div class='del'>-	switch (req.reg_type) {</div><div class='del'>-	case IW_MEMREG_TYPE_QP:</div><div class='del'>-		use_pbles = ((req.sq_pages + req.rq_pages) &gt; 2);</div><div class='del'>-		err = i40iw_handle_q_mem(iwdev, &amp;req, iwpbl, use_pbles);</div><div class='del'>-		if (err)</div><div class='del'>-			goto error;</div><div class='del'>-		spin_lock_irqsave(&amp;ucontext-&gt;qp_reg_mem_list_lock, flags);</div><div class='del'>-		list_add_tail(&amp;iwpbl-&gt;list, &amp;ucontext-&gt;qp_reg_mem_list);</div><div class='del'>-		iwpbl-&gt;on_list = true;</div><div class='del'>-		spin_unlock_irqrestore(&amp;ucontext-&gt;qp_reg_mem_list_lock, flags);</div><div class='del'>-		break;</div><div class='del'>-	case IW_MEMREG_TYPE_CQ:</div><div class='del'>-		use_pbles = (req.cq_pages &gt; 1);</div><div class='del'>-		err = i40iw_handle_q_mem(iwdev, &amp;req, iwpbl, use_pbles);</div><div class='del'>-		if (err)</div><div class='del'>-			goto error;</div><div class='del'>-</div><div class='del'>-		spin_lock_irqsave(&amp;ucontext-&gt;cq_reg_mem_list_lock, flags);</div><div class='del'>-		list_add_tail(&amp;iwpbl-&gt;list, &amp;ucontext-&gt;cq_reg_mem_list);</div><div class='del'>-		iwpbl-&gt;on_list = true;</div><div class='del'>-		spin_unlock_irqrestore(&amp;ucontext-&gt;cq_reg_mem_list_lock, flags);</div><div class='del'>-		break;</div><div class='del'>-	case IW_MEMREG_TYPE_MEM:</div><div class='del'>-		use_pbles = (iwmr-&gt;page_cnt != 1);</div><div class='del'>-		access = I40IW_ACCESS_FLAGS_LOCALREAD;</div><div class='del'>-</div><div class='del'>-		err = i40iw_setup_pbles(iwdev, iwmr, use_pbles);</div><div class='del'>-		if (err)</div><div class='del'>-			goto error;</div><div class='del'>-</div><div class='del'>-		if (use_pbles) {</div><div class='del'>-			ret = i40iw_check_mr_contiguous(palloc, iwmr-&gt;page_size);</div><div class='del'>-			if (ret) {</div><div class='del'>-				i40iw_free_pble(iwdev-&gt;pble_rsrc, palloc);</div><div class='del'>-				iwpbl-&gt;pbl_allocated = false;</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		access |= i40iw_get_user_access(acc);</div><div class='del'>-		stag = i40iw_create_stag(iwdev);</div><div class='del'>-		if (!stag) {</div><div class='del'>-			err = -ENOMEM;</div><div class='del'>-			goto error;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		iwmr-&gt;stag = stag;</div><div class='del'>-		iwmr-&gt;ibmr.rkey = stag;</div><div class='del'>-		iwmr-&gt;ibmr.lkey = stag;</div><div class='del'>-</div><div class='del'>-		err = i40iw_hwreg_mr(iwdev, iwmr, access);</div><div class='del'>-		if (err) {</div><div class='del'>-			i40iw_free_stag(iwdev, stag);</div><div class='del'>-			goto error;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		goto error;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	iwmr-&gt;type = req.reg_type;</div><div class='del'>-	if (req.reg_type == IW_MEMREG_TYPE_MEM)</div><div class='del'>-		i40iw_add_pdusecount(iwpd);</div><div class='del'>-	return &amp;iwmr-&gt;ibmr;</div><div class='del'>-</div><div class='del'>-error:</div><div class='del'>-	if (palloc-&gt;level != I40IW_LEVEL_0 &amp;&amp; iwpbl-&gt;pbl_allocated)</div><div class='del'>-		i40iw_free_pble(iwdev-&gt;pble_rsrc, palloc);</div><div class='del'>-	ib_umem_release(region);</div><div class='del'>-	kfree(iwmr);</div><div class='del'>-	return ERR_PTR(err);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_reg_phys_mr - register kernel physical memory</div><div class='del'>- * @pd: ibpd pointer</div><div class='del'>- * @addr: physical address of memory to register</div><div class='del'>- * @size: size of memory to register</div><div class='del'>- * @acc: Access rights</div><div class='del'>- * @iova_start: start of virtual address for physical buffers</div><div class='del'>- */</div><div class='del'>-struct ib_mr *i40iw_reg_phys_mr(struct ib_pd *pd,</div><div class='del'>-				u64 addr,</div><div class='del'>-				u64 size,</div><div class='del'>-				int acc,</div><div class='del'>-				u64 *iova_start)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_pd *iwpd = to_iwpd(pd);</div><div class='del'>-	struct i40iw_device *iwdev = to_iwdev(pd-&gt;device);</div><div class='del'>-	struct i40iw_pbl *iwpbl;</div><div class='del'>-	struct i40iw_mr *iwmr;</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	u32 stag;</div><div class='del'>-	u16 access = I40IW_ACCESS_FLAGS_LOCALREAD;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);</div><div class='del'>-	if (!iwmr)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-	iwmr-&gt;ibmr.pd = pd;</div><div class='del'>-	iwmr-&gt;ibmr.device = pd-&gt;device;</div><div class='del'>-	iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='del'>-	iwpbl-&gt;iwmr = iwmr;</div><div class='del'>-	iwmr-&gt;type = IW_MEMREG_TYPE_MEM;</div><div class='del'>-	iwpbl-&gt;user_base = *iova_start;</div><div class='del'>-	stag = i40iw_create_stag(iwdev);</div><div class='del'>-	if (!stag) {</div><div class='del'>-		ret = -EOVERFLOW;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-	access |= i40iw_get_user_access(acc);</div><div class='del'>-	iwmr-&gt;stag = stag;</div><div class='del'>-	iwmr-&gt;ibmr.rkey = stag;</div><div class='del'>-	iwmr-&gt;ibmr.lkey = stag;</div><div class='del'>-	iwmr-&gt;page_cnt = 1;</div><div class='del'>-	iwmr-&gt;pgaddrmem[0]  = addr;</div><div class='del'>-	iwmr-&gt;length = size;</div><div class='del'>-	status = i40iw_hwreg_mr(iwdev, iwmr, access);</div><div class='del'>-	if (status) {</div><div class='del'>-		i40iw_free_stag(iwdev, stag);</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i40iw_add_pdusecount(iwpd);</div><div class='del'>-	return &amp;iwmr-&gt;ibmr;</div><div class='del'>- err:</div><div class='del'>-	kfree(iwmr);</div><div class='del'>-	return ERR_PTR(ret);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_dma_mr - register physical mem</div><div class='del'>- * @pd: ptr of pd</div><div class='del'>- * @acc: access for memory</div><div class='del'>- */</div><div class='del'>-static struct ib_mr *i40iw_get_dma_mr(struct ib_pd *pd, int acc)</div><div class='del'>-{</div><div class='del'>-	u64 kva = 0;</div><div class='del'>-</div><div class='del'>-	return i40iw_reg_phys_mr(pd, 0, 0, acc, &amp;kva);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_del_mem_list - Deleting pbl list entries for CQ/QP</div><div class='del'>- * @iwmr: iwmr for IB's user page addresses</div><div class='del'>- * @ucontext: ptr to user context</div><div class='del'>- */</div><div class='del'>-static void i40iw_del_memlist(struct i40iw_mr *iwmr,</div><div class='del'>-			      struct i40iw_ucontext *ucontext)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	switch (iwmr-&gt;type) {</div><div class='del'>-	case IW_MEMREG_TYPE_CQ:</div><div class='del'>-		spin_lock_irqsave(&amp;ucontext-&gt;cq_reg_mem_list_lock, flags);</div><div class='del'>-		if (iwpbl-&gt;on_list) {</div><div class='del'>-			iwpbl-&gt;on_list = false;</div><div class='del'>-			list_del(&amp;iwpbl-&gt;list);</div><div class='del'>-		}</div><div class='del'>-		spin_unlock_irqrestore(&amp;ucontext-&gt;cq_reg_mem_list_lock, flags);</div><div class='del'>-		break;</div><div class='del'>-	case IW_MEMREG_TYPE_QP:</div><div class='del'>-		spin_lock_irqsave(&amp;ucontext-&gt;qp_reg_mem_list_lock, flags);</div><div class='del'>-		if (iwpbl-&gt;on_list) {</div><div class='del'>-			iwpbl-&gt;on_list = false;</div><div class='del'>-			list_del(&amp;iwpbl-&gt;list);</div><div class='del'>-		}</div><div class='del'>-		spin_unlock_irqrestore(&amp;ucontext-&gt;qp_reg_mem_list_lock, flags);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_dereg_mr - deregister mr</div><div class='del'>- * @ib_mr: mr ptr for dereg</div><div class='del'>- */</div><div class='del'>-static int i40iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct ib_pd *ibpd = ib_mr-&gt;pd;</div><div class='del'>-	struct i40iw_pd *iwpd = to_iwpd(ibpd);</div><div class='del'>-	struct i40iw_mr *iwmr = to_iwmr(ib_mr);</div><div class='del'>-	struct i40iw_device *iwdev = to_iwdev(ib_mr-&gt;device);</div><div class='del'>-	enum i40iw_status_code status;</div><div class='del'>-	struct i40iw_dealloc_stag_info *info;</div><div class='del'>-	struct i40iw_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='del'>-	struct i40iw_pble_alloc *palloc = &amp;iwpbl-&gt;pble_alloc;</div><div class='del'>-	struct i40iw_cqp_request *cqp_request;</div><div class='del'>-	struct cqp_commands_info *cqp_info;</div><div class='del'>-	u32 stag_idx;</div><div class='del'>-</div><div class='del'>-	if (iwmr-&gt;region)</div><div class='del'>-		ib_umem_release(iwmr-&gt;region);</div><div class='del'>-</div><div class='del'>-	if (iwmr-&gt;type != IW_MEMREG_TYPE_MEM) {</div><div class='del'>-		/* region is released. only test for userness. */</div><div class='del'>-		if (iwmr-&gt;region) {</div><div class='del'>-			struct i40iw_ucontext *ucontext =</div><div class='del'>-				rdma_udata_to_drv_context(</div><div class='del'>-					udata,</div><div class='del'>-					struct i40iw_ucontext,</div><div class='del'>-					ibucontext);</div><div class='del'>-</div><div class='del'>-			i40iw_del_memlist(iwmr, ucontext);</div><div class='del'>-		}</div><div class='del'>-		if (iwpbl-&gt;pbl_allocated &amp;&amp; iwmr-&gt;type != IW_MEMREG_TYPE_QP)</div><div class='del'>-			i40iw_free_pble(iwdev-&gt;pble_rsrc, palloc);</div><div class='del'>-		kfree(iwmr);</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	cqp_request = i40iw_get_cqp_request(&amp;iwdev-&gt;cqp, true);</div><div class='del'>-	if (!cqp_request)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	cqp_info = &amp;cqp_request-&gt;info;</div><div class='del'>-	info = &amp;cqp_info-&gt;in.u.dealloc_stag.info;</div><div class='del'>-	memset(info, 0, sizeof(*info));</div><div class='del'>-</div><div class='del'>-	info-&gt;pd_id = cpu_to_le32(iwpd-&gt;sc_pd.pd_id &amp; 0x00007fff);</div><div class='del'>-	info-&gt;stag_idx = RS_64_1(ib_mr-&gt;rkey, I40IW_CQPSQ_STAG_IDX_SHIFT);</div><div class='del'>-	stag_idx = info-&gt;stag_idx;</div><div class='del'>-	info-&gt;mr = true;</div><div class='del'>-	if (iwpbl-&gt;pbl_allocated)</div><div class='del'>-		info-&gt;dealloc_pbl = true;</div><div class='del'>-</div><div class='del'>-	cqp_info-&gt;cqp_cmd = OP_DEALLOC_STAG;</div><div class='del'>-	cqp_info-&gt;post_sq = 1;</div><div class='del'>-	cqp_info-&gt;in.u.dealloc_stag.dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	cqp_info-&gt;in.u.dealloc_stag.scratch = (uintptr_t)cqp_request;</div><div class='del'>-	status = i40iw_handle_cqp_op(iwdev, cqp_request);</div><div class='del'>-	if (status)</div><div class='del'>-		i40iw_pr_err("CQP-OP dealloc failed for stag_idx = 0x%x\n", stag_idx);</div><div class='del'>-	i40iw_rem_pdusecount(iwpd, iwdev);</div><div class='del'>-	i40iw_free_stag(iwdev, iwmr-&gt;stag);</div><div class='del'>-	if (iwpbl-&gt;pbl_allocated)</div><div class='del'>-		i40iw_free_pble(iwdev-&gt;pble_rsrc, palloc);</div><div class='del'>-	kfree(iwmr);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * hw_rev_show</div><div class='del'>- */</div><div class='del'>-static ssize_t hw_rev_show(struct device *dev,</div><div class='del'>-			   struct device_attribute *attr, char *buf)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_ib_device *iwibdev =</div><div class='del'>-		rdma_device_to_drv_device(dev, struct i40iw_ib_device, ibdev);</div><div class='del'>-	u32 hw_rev = iwibdev-&gt;iwdev-&gt;sc_dev.hw_rev;</div><div class='del'>-</div><div class='del'>-	return sprintf(buf, "%x\n", hw_rev);</div><div class='del'>-}</div><div class='del'>-static DEVICE_ATTR_RO(hw_rev);</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * hca_type_show</div><div class='del'>- */</div><div class='del'>-static ssize_t hca_type_show(struct device *dev,</div><div class='del'>-			     struct device_attribute *attr, char *buf)</div><div class='del'>-{</div><div class='del'>-	return sprintf(buf, "I40IW\n");</div><div class='del'>-}</div><div class='del'>-static DEVICE_ATTR_RO(hca_type);</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * board_id_show</div><div class='del'>- */</div><div class='del'>-static ssize_t board_id_show(struct device *dev,</div><div class='del'>-			     struct device_attribute *attr, char *buf)</div><div class='del'>-{</div><div class='del'>-	return sprintf(buf, "%.*s\n", 32, "I40IW Board ID");</div><div class='del'>-}</div><div class='del'>-static DEVICE_ATTR_RO(board_id);</div><div class='del'>-</div><div class='del'>-static struct attribute *i40iw_dev_attributes[] = {</div><div class='del'>-	&amp;dev_attr_hw_rev.attr,</div><div class='del'>-	&amp;dev_attr_hca_type.attr,</div><div class='del'>-	&amp;dev_attr_board_id.attr,</div><div class='del'>-	NULL</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static const struct attribute_group i40iw_attr_group = {</div><div class='del'>-	.attrs = i40iw_dev_attributes,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_copy_sg_list - copy sg list for qp</div><div class='del'>- * @sg_list: copied into sg_list</div><div class='del'>- * @sgl: copy from sgl</div><div class='del'>- * @num_sges: count of sg entries</div><div class='del'>- */</div><div class='del'>-static void i40iw_copy_sg_list(struct i40iw_sge *sg_list, struct ib_sge *sgl, int num_sges)</div><div class='del'>-{</div><div class='del'>-	unsigned int i;</div><div class='del'>-</div><div class='del'>-	for (i = 0; (i &lt; num_sges) &amp;&amp; (i &lt; I40IW_MAX_WQ_FRAGMENT_COUNT); i++) {</div><div class='del'>-		sg_list[i].tag_off = sgl[i].addr;</div><div class='del'>-		sg_list[i].len = sgl[i].length;</div><div class='del'>-		sg_list[i].stag = sgl[i].lkey;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_post_send -  kernel application wr</div><div class='del'>- * @ibqp: qp ptr for wr</div><div class='del'>- * @ib_wr: work request ptr</div><div class='del'>- * @bad_wr: return of bad wr if err</div><div class='del'>- */</div><div class='del'>-static int i40iw_post_send(struct ib_qp *ibqp,</div><div class='del'>-			   const struct ib_send_wr *ib_wr,</div><div class='del'>-			   const struct ib_send_wr **bad_wr)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-	struct i40iw_qp_uk *ukqp;</div><div class='del'>-	struct i40iw_post_sq_info info;</div><div class='del'>-	enum i40iw_status_code ret;</div><div class='del'>-	int err = 0;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	bool inv_stag;</div><div class='del'>-</div><div class='del'>-	iwqp = (struct i40iw_qp *)ibqp;</div><div class='del'>-	ukqp = &amp;iwqp-&gt;sc_qp.qp_uk;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-	if (iwqp-&gt;flush_issued) {</div><div class='del'>-		err = -EINVAL;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	while (ib_wr) {</div><div class='del'>-		inv_stag = false;</div><div class='del'>-		memset(&amp;info, 0, sizeof(info));</div><div class='del'>-		info.wr_id = (u64)(ib_wr-&gt;wr_id);</div><div class='del'>-		if ((ib_wr-&gt;send_flags &amp; IB_SEND_SIGNALED) || iwqp-&gt;sig_all)</div><div class='del'>-			info.signaled = true;</div><div class='del'>-		if (ib_wr-&gt;send_flags &amp; IB_SEND_FENCE)</div><div class='del'>-			info.read_fence = true;</div><div class='del'>-</div><div class='del'>-		switch (ib_wr-&gt;opcode) {</div><div class='del'>-		case IB_WR_SEND:</div><div class='del'>-			/* fall-through */</div><div class='del'>-		case IB_WR_SEND_WITH_INV:</div><div class='del'>-			if (ib_wr-&gt;opcode == IB_WR_SEND) {</div><div class='del'>-				if (ib_wr-&gt;send_flags &amp; IB_SEND_SOLICITED)</div><div class='del'>-					info.op_type = I40IW_OP_TYPE_SEND_SOL;</div><div class='del'>-				else</div><div class='del'>-					info.op_type = I40IW_OP_TYPE_SEND;</div><div class='del'>-			} else {</div><div class='del'>-				if (ib_wr-&gt;send_flags &amp; IB_SEND_SOLICITED)</div><div class='del'>-					info.op_type = I40IW_OP_TYPE_SEND_SOL_INV;</div><div class='del'>-				else</div><div class='del'>-					info.op_type = I40IW_OP_TYPE_SEND_INV;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			if (ib_wr-&gt;send_flags &amp; IB_SEND_INLINE) {</div><div class='del'>-				info.op.inline_send.data = (void *)(unsigned long)ib_wr-&gt;sg_list[0].addr;</div><div class='del'>-				info.op.inline_send.len = ib_wr-&gt;sg_list[0].length;</div><div class='del'>-				ret = ukqp-&gt;ops.iw_inline_send(ukqp, &amp;info, ib_wr-&gt;ex.invalidate_rkey, false);</div><div class='del'>-			} else {</div><div class='del'>-				info.op.send.num_sges = ib_wr-&gt;num_sge;</div><div class='del'>-				info.op.send.sg_list = (struct i40iw_sge *)ib_wr-&gt;sg_list;</div><div class='del'>-				ret = ukqp-&gt;ops.iw_send(ukqp, &amp;info, ib_wr-&gt;ex.invalidate_rkey, false);</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			if (ret) {</div><div class='del'>-				if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)</div><div class='del'>-					err = -ENOMEM;</div><div class='del'>-				else</div><div class='del'>-					err = -EINVAL;</div><div class='del'>-			}</div><div class='del'>-			break;</div><div class='del'>-		case IB_WR_RDMA_WRITE:</div><div class='del'>-			info.op_type = I40IW_OP_TYPE_RDMA_WRITE;</div><div class='del'>-</div><div class='del'>-			if (ib_wr-&gt;send_flags &amp; IB_SEND_INLINE) {</div><div class='del'>-				info.op.inline_rdma_write.data = (void *)(unsigned long)ib_wr-&gt;sg_list[0].addr;</div><div class='del'>-				info.op.inline_rdma_write.len = ib_wr-&gt;sg_list[0].length;</div><div class='del'>-				info.op.inline_rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)-&gt;remote_addr;</div><div class='del'>-				info.op.inline_rdma_write.rem_addr.stag = rdma_wr(ib_wr)-&gt;rkey;</div><div class='del'>-				ret = ukqp-&gt;ops.iw_inline_rdma_write(ukqp, &amp;info, false);</div><div class='del'>-			} else {</div><div class='del'>-				info.op.rdma_write.lo_sg_list = (void *)ib_wr-&gt;sg_list;</div><div class='del'>-				info.op.rdma_write.num_lo_sges = ib_wr-&gt;num_sge;</div><div class='del'>-				info.op.rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)-&gt;remote_addr;</div><div class='del'>-				info.op.rdma_write.rem_addr.stag = rdma_wr(ib_wr)-&gt;rkey;</div><div class='del'>-				ret = ukqp-&gt;ops.iw_rdma_write(ukqp, &amp;info, false);</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			if (ret) {</div><div class='del'>-				if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)</div><div class='del'>-					err = -ENOMEM;</div><div class='del'>-				else</div><div class='del'>-					err = -EINVAL;</div><div class='del'>-			}</div><div class='del'>-			break;</div><div class='del'>-		case IB_WR_RDMA_READ_WITH_INV:</div><div class='del'>-			inv_stag = true;</div><div class='del'>-			/* fall-through*/</div><div class='del'>-		case IB_WR_RDMA_READ:</div><div class='del'>-			if (ib_wr-&gt;num_sge &gt; I40IW_MAX_SGE_RD) {</div><div class='del'>-				err = -EINVAL;</div><div class='del'>-				break;</div><div class='del'>-			}</div><div class='del'>-			info.op_type = I40IW_OP_TYPE_RDMA_READ;</div><div class='del'>-			info.op.rdma_read.rem_addr.tag_off = rdma_wr(ib_wr)-&gt;remote_addr;</div><div class='del'>-			info.op.rdma_read.rem_addr.stag = rdma_wr(ib_wr)-&gt;rkey;</div><div class='del'>-			info.op.rdma_read.lo_addr.tag_off = ib_wr-&gt;sg_list-&gt;addr;</div><div class='del'>-			info.op.rdma_read.lo_addr.stag = ib_wr-&gt;sg_list-&gt;lkey;</div><div class='del'>-			info.op.rdma_read.lo_addr.len = ib_wr-&gt;sg_list-&gt;length;</div><div class='del'>-			ret = ukqp-&gt;ops.iw_rdma_read(ukqp, &amp;info, inv_stag, false);</div><div class='del'>-			if (ret) {</div><div class='del'>-				if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)</div><div class='del'>-					err = -ENOMEM;</div><div class='del'>-				else</div><div class='del'>-					err = -EINVAL;</div><div class='del'>-			}</div><div class='del'>-			break;</div><div class='del'>-		case IB_WR_LOCAL_INV:</div><div class='del'>-			info.op_type = I40IW_OP_TYPE_INV_STAG;</div><div class='del'>-			info.op.inv_local_stag.target_stag = ib_wr-&gt;ex.invalidate_rkey;</div><div class='del'>-			ret = ukqp-&gt;ops.iw_stag_local_invalidate(ukqp, &amp;info, true);</div><div class='del'>-			if (ret)</div><div class='del'>-				err = -ENOMEM;</div><div class='del'>-			break;</div><div class='del'>-		case IB_WR_REG_MR:</div><div class='del'>-		{</div><div class='del'>-			struct i40iw_mr *iwmr = to_iwmr(reg_wr(ib_wr)-&gt;mr);</div><div class='del'>-			int flags = reg_wr(ib_wr)-&gt;access;</div><div class='del'>-			struct i40iw_pble_alloc *palloc = &amp;iwmr-&gt;iwpbl.pble_alloc;</div><div class='del'>-			struct i40iw_sc_dev *dev = &amp;iwqp-&gt;iwdev-&gt;sc_dev;</div><div class='del'>-			struct i40iw_fast_reg_stag_info info;</div><div class='del'>-</div><div class='del'>-			memset(&amp;info, 0, sizeof(info));</div><div class='del'>-			info.access_rights = I40IW_ACCESS_FLAGS_LOCALREAD;</div><div class='del'>-			info.access_rights |= i40iw_get_user_access(flags);</div><div class='del'>-			info.stag_key = reg_wr(ib_wr)-&gt;key &amp; 0xff;</div><div class='del'>-			info.stag_idx = reg_wr(ib_wr)-&gt;key &gt;&gt; 8;</div><div class='del'>-			info.page_size = reg_wr(ib_wr)-&gt;mr-&gt;page_size;</div><div class='del'>-			info.wr_id = ib_wr-&gt;wr_id;</div><div class='del'>-</div><div class='del'>-			info.addr_type = I40IW_ADDR_TYPE_VA_BASED;</div><div class='del'>-			info.va = (void *)(uintptr_t)iwmr-&gt;ibmr.iova;</div><div class='del'>-			info.total_len = iwmr-&gt;ibmr.length;</div><div class='del'>-			info.reg_addr_pa = *(u64 *)palloc-&gt;level1.addr;</div><div class='del'>-			info.first_pm_pbl_index = palloc-&gt;level1.idx;</div><div class='del'>-			info.local_fence = ib_wr-&gt;send_flags &amp; IB_SEND_FENCE;</div><div class='del'>-			info.signaled = ib_wr-&gt;send_flags &amp; IB_SEND_SIGNALED;</div><div class='del'>-</div><div class='del'>-			if (iwmr-&gt;npages &gt; I40IW_MIN_PAGES_PER_FMR)</div><div class='del'>-				info.chunk_size = 1;</div><div class='del'>-</div><div class='del'>-			ret = dev-&gt;iw_priv_qp_ops-&gt;iw_mr_fast_register(&amp;iwqp-&gt;sc_qp, &amp;info, true);</div><div class='del'>-			if (ret)</div><div class='del'>-				err = -ENOMEM;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		default:</div><div class='del'>-			err = -EINVAL;</div><div class='del'>-			i40iw_pr_err(" upost_send bad opcode = 0x%x\n",</div><div class='del'>-				     ib_wr-&gt;opcode);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (err)</div><div class='del'>-			break;</div><div class='del'>-		ib_wr = ib_wr-&gt;next;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-out:</div><div class='del'>-	if (err)</div><div class='del'>-		*bad_wr = ib_wr;</div><div class='del'>-	else</div><div class='del'>-		ukqp-&gt;ops.iw_qp_post_wr(ukqp);</div><div class='del'>-	spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_post_recv - post receive wr for kernel application</div><div class='del'>- * @ibqp: ib qp pointer</div><div class='del'>- * @ib_wr: work request for receive</div><div class='del'>- * @bad_wr: bad wr caused an error</div><div class='del'>- */</div><div class='del'>-static int i40iw_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *ib_wr,</div><div class='del'>-			   const struct ib_recv_wr **bad_wr)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-	struct i40iw_qp_uk *ukqp;</div><div class='del'>-	struct i40iw_post_rq_info post_recv;</div><div class='del'>-	struct i40iw_sge sg_list[I40IW_MAX_WQ_FRAGMENT_COUNT];</div><div class='del'>-	enum i40iw_status_code ret = 0;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int err = 0;</div><div class='del'>-</div><div class='del'>-	iwqp = (struct i40iw_qp *)ibqp;</div><div class='del'>-	ukqp = &amp;iwqp-&gt;sc_qp.qp_uk;</div><div class='del'>-</div><div class='del'>-	memset(&amp;post_recv, 0, sizeof(post_recv));</div><div class='del'>-	spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-	if (iwqp-&gt;flush_issued) {</div><div class='del'>-		err = -EINVAL;</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	while (ib_wr) {</div><div class='del'>-		post_recv.num_sges = ib_wr-&gt;num_sge;</div><div class='del'>-		post_recv.wr_id = ib_wr-&gt;wr_id;</div><div class='del'>-		i40iw_copy_sg_list(sg_list, ib_wr-&gt;sg_list, ib_wr-&gt;num_sge);</div><div class='del'>-		post_recv.sg_list = sg_list;</div><div class='del'>-		ret = ukqp-&gt;ops.iw_post_receive(ukqp, &amp;post_recv);</div><div class='del'>-		if (ret) {</div><div class='del'>-			i40iw_pr_err(" post_recv err %d\n", ret);</div><div class='del'>-			if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)</div><div class='del'>-				err = -ENOMEM;</div><div class='del'>-			else</div><div class='del'>-				err = -EINVAL;</div><div class='del'>-			*bad_wr = ib_wr;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-		ib_wr = ib_wr-&gt;next;</div><div class='del'>-	}</div><div class='del'>- out:</div><div class='del'>-	spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_poll_cq - poll cq for completion (kernel apps)</div><div class='del'>- * @ibcq: cq to poll</div><div class='del'>- * @num_entries: number of entries to poll</div><div class='del'>- * @entry: wr of entry completed</div><div class='del'>- */</div><div class='del'>-static int i40iw_poll_cq(struct ib_cq *ibcq,</div><div class='del'>-			 int num_entries,</div><div class='del'>-			 struct ib_wc *entry)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cq *iwcq;</div><div class='del'>-	int cqe_count = 0;</div><div class='del'>-	struct i40iw_cq_poll_info cq_poll_info;</div><div class='del'>-	enum i40iw_status_code ret;</div><div class='del'>-	struct i40iw_cq_uk *ukcq;</div><div class='del'>-	struct i40iw_sc_qp *qp;</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-</div><div class='del'>-	iwcq = (struct i40iw_cq *)ibcq;</div><div class='del'>-	ukcq = &amp;iwcq-&gt;sc_cq.cq_uk;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;iwcq-&gt;lock, flags);</div><div class='del'>-	while (cqe_count &lt; num_entries) {</div><div class='del'>-		ret = ukcq-&gt;ops.iw_cq_poll_completion(ukcq, &amp;cq_poll_info);</div><div class='del'>-		if (ret == I40IW_ERR_QUEUE_EMPTY) {</div><div class='del'>-			break;</div><div class='del'>-		} else if (ret == I40IW_ERR_QUEUE_DESTROYED) {</div><div class='del'>-			continue;</div><div class='del'>-		} else if (ret) {</div><div class='del'>-			if (!cqe_count)</div><div class='del'>-				cqe_count = -1;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		entry-&gt;wc_flags = 0;</div><div class='del'>-		entry-&gt;wr_id = cq_poll_info.wr_id;</div><div class='del'>-		if (cq_poll_info.error) {</div><div class='del'>-			entry-&gt;status = IB_WC_WR_FLUSH_ERR;</div><div class='del'>-			entry-&gt;vendor_err = cq_poll_info.major_err &lt;&lt; 16 | cq_poll_info.minor_err;</div><div class='del'>-		} else {</div><div class='del'>-			entry-&gt;status = IB_WC_SUCCESS;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		switch (cq_poll_info.op_type) {</div><div class='del'>-		case I40IW_OP_TYPE_RDMA_WRITE:</div><div class='del'>-			entry-&gt;opcode = IB_WC_RDMA_WRITE;</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_OP_TYPE_RDMA_READ_INV_STAG:</div><div class='del'>-		case I40IW_OP_TYPE_RDMA_READ:</div><div class='del'>-			entry-&gt;opcode = IB_WC_RDMA_READ;</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_OP_TYPE_SEND_SOL:</div><div class='del'>-		case I40IW_OP_TYPE_SEND_SOL_INV:</div><div class='del'>-		case I40IW_OP_TYPE_SEND_INV:</div><div class='del'>-		case I40IW_OP_TYPE_SEND:</div><div class='del'>-			entry-&gt;opcode = IB_WC_SEND;</div><div class='del'>-			break;</div><div class='del'>-		case I40IW_OP_TYPE_REC:</div><div class='del'>-			entry-&gt;opcode = IB_WC_RECV;</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			entry-&gt;opcode = IB_WC_RECV;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		entry-&gt;ex.imm_data = 0;</div><div class='del'>-		qp = (struct i40iw_sc_qp *)cq_poll_info.qp_handle;</div><div class='del'>-		entry-&gt;qp = (struct ib_qp *)qp-&gt;back_qp;</div><div class='del'>-		entry-&gt;src_qp = cq_poll_info.qp_id;</div><div class='del'>-		iwqp = (struct i40iw_qp *)qp-&gt;back_qp;</div><div class='del'>-		if (iwqp-&gt;iwarp_state &gt; I40IW_QP_STATE_RTS) {</div><div class='del'>-			if (!I40IW_RING_MORE_WORK(qp-&gt;qp_uk.sq_ring))</div><div class='del'>-				complete(&amp;iwqp-&gt;sq_drained);</div><div class='del'>-			if (!I40IW_RING_MORE_WORK(qp-&gt;qp_uk.rq_ring))</div><div class='del'>-				complete(&amp;iwqp-&gt;rq_drained);</div><div class='del'>-		}</div><div class='del'>-		entry-&gt;byte_len = cq_poll_info.bytes_xfered;</div><div class='del'>-		entry++;</div><div class='del'>-		cqe_count++;</div><div class='del'>-	}</div><div class='del'>-	spin_unlock_irqrestore(&amp;iwcq-&gt;lock, flags);</div><div class='del'>-	return cqe_count;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_req_notify_cq - arm cq kernel application</div><div class='del'>- * @ibcq: cq to arm</div><div class='del'>- * @notify_flags: notofication flags</div><div class='del'>- */</div><div class='del'>-static int i40iw_req_notify_cq(struct ib_cq *ibcq,</div><div class='del'>-			       enum ib_cq_notify_flags notify_flags)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_cq *iwcq;</div><div class='del'>-	struct i40iw_cq_uk *ukcq;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	enum i40iw_completion_notify cq_notify = IW_CQ_COMPL_EVENT;</div><div class='del'>-</div><div class='del'>-	iwcq = (struct i40iw_cq *)ibcq;</div><div class='del'>-	ukcq = &amp;iwcq-&gt;sc_cq.cq_uk;</div><div class='del'>-	if (notify_flags == IB_CQ_SOLICITED)</div><div class='del'>-		cq_notify = IW_CQ_COMPL_SOLICITED;</div><div class='del'>-	spin_lock_irqsave(&amp;iwcq-&gt;lock, flags);</div><div class='del'>-	ukcq-&gt;ops.iw_cq_request_notification(ukcq, cq_notify);</div><div class='del'>-	spin_unlock_irqrestore(&amp;iwcq-&gt;lock, flags);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_port_immutable - return port's immutable data</div><div class='del'>- * @ibdev: ib dev struct</div><div class='del'>- * @port_num: port number</div><div class='del'>- * @immutable: immutable data for the port return</div><div class='del'>- */</div><div class='del'>-static int i40iw_port_immutable(struct ib_device *ibdev, u8 port_num,</div><div class='del'>-				struct ib_port_immutable *immutable)</div><div class='del'>-{</div><div class='del'>-	struct ib_port_attr attr;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	immutable-&gt;core_cap_flags = RDMA_CORE_PORT_IWARP;</div><div class='del'>-</div><div class='del'>-	err = ib_query_port(ibdev, port_num, &amp;attr);</div><div class='del'>-</div><div class='del'>-	if (err)</div><div class='del'>-		return err;</div><div class='del'>-</div><div class='del'>-	immutable-&gt;pkey_tbl_len = attr.pkey_tbl_len;</div><div class='del'>-	immutable-&gt;gid_tbl_len = attr.gid_tbl_len;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static const char * const i40iw_hw_stat_names[] = {</div><div class='del'>-	// 32bit names</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP4RXDISCARD] = "ip4InDiscards",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP4RXTRUNC] = "ip4InTruncatedPkts",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] = "ip4OutNoRoutes",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP6RXDISCARD] = "ip6InDiscards",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP6RXTRUNC] = "ip6InTruncatedPkts",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] = "ip6OutNoRoutes",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_TCPRTXSEG] = "tcpRetransSegs",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_TCPRXOPTERR] = "tcpInOptErrors",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] = "tcpInProtoErrors",</div><div class='del'>-	// 64bit names</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP4RXOCTS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"ip4InOctets",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP4RXPKTS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"ip4InPkts",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP4RXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"ip4InReasmRqd",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP4RXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"ip4InMcastPkts",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP4TXOCTS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"ip4OutOctets",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP4TXPKTS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"ip4OutPkts",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP4TXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"ip4OutSegRqd",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP4TXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"ip4OutMcastPkts",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP6RXOCTS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"ip6InOctets",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP6RXPKTS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"ip6InPkts",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP6RXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"ip6InReasmRqd",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP6RXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"ip6InMcastPkts",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP6TXOCTS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"ip6OutOctets",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP6TXPKTS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"ip6OutPkts",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP6TXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"ip6OutSegRqd",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_IP6TXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"ip6OutMcastPkts",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_TCPRXSEGS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"tcpInSegs",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_TCPTXSEG + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"tcpOutSegs",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_RDMARXRDS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"iwInRdmaReads",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_RDMARXSNDS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"iwInRdmaSends",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_RDMARXWRS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"iwInRdmaWrites",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_RDMATXRDS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"iwOutRdmaReads",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_RDMATXSNDS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"iwOutRdmaSends",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_RDMATXWRS + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"iwOutRdmaWrites",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_RDMAVBND + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"iwRdmaBnd",</div><div class='del'>-	[I40IW_HW_STAT_INDEX_RDMAVINV + I40IW_HW_STAT_INDEX_MAX_32] =</div><div class='del'>-		"iwRdmaInv"</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static void i40iw_get_dev_fw_str(struct ib_device *dev, char *str)</div><div class='del'>-{</div><div class='del'>-	u32 firmware_version = I40IW_FW_VERSION;</div><div class='del'>-</div><div class='del'>-	snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u", firmware_version,</div><div class='del'>-		 (firmware_version &amp; 0x000000ff));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_alloc_hw_stats - Allocate a hw stats structure</div><div class='del'>- * @ibdev: device pointer from stack</div><div class='del'>- * @port_num: port number</div><div class='del'>- */</div><div class='del'>-static struct rdma_hw_stats *i40iw_alloc_hw_stats(struct ib_device *ibdev,</div><div class='del'>-						  u8 port_num)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = to_iwdev(ibdev);</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	int num_counters = I40IW_HW_STAT_INDEX_MAX_32 +</div><div class='del'>-		I40IW_HW_STAT_INDEX_MAX_64;</div><div class='del'>-	unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN;</div><div class='del'>-</div><div class='del'>-	BUILD_BUG_ON(ARRAY_SIZE(i40iw_hw_stat_names) !=</div><div class='del'>-		     (I40IW_HW_STAT_INDEX_MAX_32 +</div><div class='del'>-		      I40IW_HW_STAT_INDEX_MAX_64));</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * PFs get the default update lifespan, but VFs only update once</div><div class='del'>-	 * per second</div><div class='del'>-	 */</div><div class='del'>-	if (!dev-&gt;is_pf)</div><div class='del'>-		lifespan = 1000;</div><div class='del'>-	return rdma_alloc_hw_stats_struct(i40iw_hw_stat_names, num_counters,</div><div class='del'>-					  lifespan);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_get_hw_stats - Populates the rdma_hw_stats structure</div><div class='del'>- * @ibdev: device pointer from stack</div><div class='del'>- * @stats: stats pointer from stack</div><div class='del'>- * @port_num: port number</div><div class='del'>- * @index: which hw counter the stack is requesting we update</div><div class='del'>- */</div><div class='del'>-static int i40iw_get_hw_stats(struct ib_device *ibdev,</div><div class='del'>-			      struct rdma_hw_stats *stats,</div><div class='del'>-			      u8 port_num, int index)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = to_iwdev(ibdev);</div><div class='del'>-	struct i40iw_sc_dev *dev = &amp;iwdev-&gt;sc_dev;</div><div class='del'>-	struct i40iw_vsi_pestat *devstat = iwdev-&gt;vsi.pestat;</div><div class='del'>-	struct i40iw_dev_hw_stats *hw_stats = &amp;devstat-&gt;hw_stats;</div><div class='del'>-</div><div class='del'>-	if (dev-&gt;is_pf) {</div><div class='del'>-		i40iw_hw_stats_read_all(devstat, &amp;devstat-&gt;hw_stats);</div><div class='del'>-	} else {</div><div class='del'>-		if (i40iw_vchnl_vf_get_pe_stats(dev, &amp;devstat-&gt;hw_stats))</div><div class='del'>-			return -ENOSYS;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	memcpy(&amp;stats-&gt;value[0], hw_stats, sizeof(*hw_stats));</div><div class='del'>-</div><div class='del'>-	return stats-&gt;num_counters;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_query_gid - Query port GID</div><div class='del'>- * @ibdev: device pointer from stack</div><div class='del'>- * @port: port number</div><div class='del'>- * @index: Entry index</div><div class='del'>- * @gid: Global ID</div><div class='del'>- */</div><div class='del'>-static int i40iw_query_gid(struct ib_device *ibdev,</div><div class='del'>-			   u8 port,</div><div class='del'>-			   int index,</div><div class='del'>-			   union ib_gid *gid)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_device *iwdev = to_iwdev(ibdev);</div><div class='del'>-</div><div class='del'>-	memset(gid-&gt;raw, 0, sizeof(gid-&gt;raw));</div><div class='del'>-	ether_addr_copy(gid-&gt;raw, iwdev-&gt;netdev-&gt;dev_addr);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_query_pkey - Query partition key</div><div class='del'>- * @ibdev: device pointer from stack</div><div class='del'>- * @port: port number</div><div class='del'>- * @index: index of pkey</div><div class='del'>- * @pkey: pointer to store the pkey</div><div class='del'>- */</div><div class='del'>-static int i40iw_query_pkey(struct ib_device *ibdev,</div><div class='del'>-			    u8 port,</div><div class='del'>-			    u16 index,</div><div class='del'>-			    u16 *pkey)</div><div class='del'>-{</div><div class='del'>-	*pkey = 0;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static const struct ib_device_ops i40iw_dev_ops = {</div><div class='del'>-	.alloc_hw_stats = i40iw_alloc_hw_stats,</div><div class='del'>-	.alloc_mr = i40iw_alloc_mr,</div><div class='del'>-	.alloc_pd = i40iw_alloc_pd,</div><div class='del'>-	.alloc_ucontext = i40iw_alloc_ucontext,</div><div class='del'>-	.create_cq = i40iw_create_cq,</div><div class='del'>-	.create_qp = i40iw_create_qp,</div><div class='del'>-	.dealloc_pd = i40iw_dealloc_pd,</div><div class='del'>-	.dealloc_ucontext = i40iw_dealloc_ucontext,</div><div class='del'>-	.dereg_mr = i40iw_dereg_mr,</div><div class='del'>-	.destroy_cq = i40iw_destroy_cq,</div><div class='del'>-	.destroy_qp = i40iw_destroy_qp,</div><div class='del'>-	.drain_rq = i40iw_drain_rq,</div><div class='del'>-	.drain_sq = i40iw_drain_sq,</div><div class='del'>-	.get_dev_fw_str = i40iw_get_dev_fw_str,</div><div class='del'>-	.get_dma_mr = i40iw_get_dma_mr,</div><div class='del'>-	.get_hw_stats = i40iw_get_hw_stats,</div><div class='del'>-	.get_port_immutable = i40iw_port_immutable,</div><div class='del'>-	.iw_accept = i40iw_accept,</div><div class='del'>-	.iw_add_ref = i40iw_add_ref,</div><div class='del'>-	.iw_connect = i40iw_connect,</div><div class='del'>-	.iw_create_listen = i40iw_create_listen,</div><div class='del'>-	.iw_destroy_listen = i40iw_destroy_listen,</div><div class='del'>-	.iw_get_qp = i40iw_get_qp,</div><div class='del'>-	.iw_reject = i40iw_reject,</div><div class='del'>-	.iw_rem_ref = i40iw_rem_ref,</div><div class='del'>-	.map_mr_sg = i40iw_map_mr_sg,</div><div class='del'>-	.mmap = i40iw_mmap,</div><div class='del'>-	.modify_qp = i40iw_modify_qp,</div><div class='del'>-	.poll_cq = i40iw_poll_cq,</div><div class='del'>-	.post_recv = i40iw_post_recv,</div><div class='del'>-	.post_send = i40iw_post_send,</div><div class='del'>-	.query_device = i40iw_query_device,</div><div class='del'>-	.query_gid = i40iw_query_gid,</div><div class='del'>-	.query_pkey = i40iw_query_pkey,</div><div class='del'>-	.query_port = i40iw_query_port,</div><div class='del'>-	.query_qp = i40iw_query_qp,</div><div class='del'>-	.reg_user_mr = i40iw_reg_user_mr,</div><div class='del'>-	.req_notify_cq = i40iw_req_notify_cq,</div><div class='del'>-	INIT_RDMA_OBJ_SIZE(ib_pd, i40iw_pd, ibpd),</div><div class='del'>-	INIT_RDMA_OBJ_SIZE(ib_ucontext, i40iw_ucontext, ibucontext),</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_init_rdma_device - initialization of iwarp device</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- */</div><div class='del'>-static struct i40iw_ib_device *i40iw_init_rdma_device(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_ib_device *iwibdev;</div><div class='del'>-	struct net_device *netdev = iwdev-&gt;netdev;</div><div class='del'>-	struct pci_dev *pcidev = (struct pci_dev *)iwdev-&gt;hw.dev_context;</div><div class='del'>-</div><div class='del'>-	iwibdev = ib_alloc_device(i40iw_ib_device, ibdev);</div><div class='del'>-	if (!iwibdev) {</div><div class='del'>-		i40iw_pr_err("iwdev == NULL\n");</div><div class='del'>-		return NULL;</div><div class='del'>-	}</div><div class='del'>-	iwibdev-&gt;ibdev.owner = THIS_MODULE;</div><div class='del'>-	iwdev-&gt;iwibdev = iwibdev;</div><div class='del'>-	iwibdev-&gt;iwdev = iwdev;</div><div class='del'>-</div><div class='del'>-	iwibdev-&gt;ibdev.node_type = RDMA_NODE_RNIC;</div><div class='del'>-	ether_addr_copy((u8 *)&amp;iwibdev-&gt;ibdev.node_guid, netdev-&gt;dev_addr);</div><div class='del'>-</div><div class='del'>-	iwibdev-&gt;ibdev.uverbs_cmd_mask =</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_GET_CONTEXT) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_DEVICE) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_PORT) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_ALLOC_PD) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_DEALLOC_PD) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_REG_MR) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_DEREG_MR) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_CQ) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_CQ) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_QP) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_MODIFY_QP) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_QP) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_POLL_CQ) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_AH) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_AH) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_QP) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_POST_RECV) |</div><div class='del'>-	    (1ull &lt;&lt; IB_USER_VERBS_CMD_POST_SEND);</div><div class='del'>-	iwibdev-&gt;ibdev.phys_port_cnt = 1;</div><div class='del'>-	iwibdev-&gt;ibdev.num_comp_vectors = iwdev-&gt;ceqs_count;</div><div class='del'>-	iwibdev-&gt;ibdev.dev.parent = &amp;pcidev-&gt;dev;</div><div class='del'>-	memcpy(iwibdev-&gt;ibdev.iw_ifname, netdev-&gt;name,</div><div class='del'>-	       sizeof(iwibdev-&gt;ibdev.iw_ifname));</div><div class='del'>-	ib_set_device_ops(&amp;iwibdev-&gt;ibdev, &amp;i40iw_dev_ops);</div><div class='del'>-</div><div class='del'>-	return iwibdev;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_port_ibevent - indicate port event</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- */</div><div class='del'>-void i40iw_port_ibevent(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_ib_device *iwibdev = iwdev-&gt;iwibdev;</div><div class='del'>-	struct ib_event event;</div><div class='del'>-</div><div class='del'>-	event.device = &amp;iwibdev-&gt;ibdev;</div><div class='del'>-	event.element.port_num = 1;</div><div class='del'>-	event.event = iwdev-&gt;iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;</div><div class='del'>-	ib_dispatch_event(&amp;event);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_destroy_rdma_device - destroy rdma device and free resources</div><div class='del'>- * @iwibdev: IB device ptr</div><div class='del'>- */</div><div class='del'>-void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev)</div><div class='del'>-{</div><div class='del'>-	if (!iwibdev)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	ib_unregister_device(&amp;iwibdev-&gt;ibdev);</div><div class='del'>-	wait_event_timeout(iwibdev-&gt;iwdev-&gt;close_wq,</div><div class='del'>-			   !atomic64_read(&amp;iwibdev-&gt;iwdev-&gt;use_count),</div><div class='del'>-			   I40IW_EVENT_TIMEOUT);</div><div class='del'>-	ib_dealloc_device(&amp;iwibdev-&gt;ibdev);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_register_rdma_device - register iwarp device to IB</div><div class='del'>- * @iwdev: iwarp device</div><div class='del'>- */</div><div class='del'>-int i40iw_register_rdma_device(struct i40iw_device *iwdev)</div><div class='del'>-{</div><div class='del'>-	int ret;</div><div class='del'>-	struct i40iw_ib_device *iwibdev;</div><div class='del'>-</div><div class='del'>-	iwdev-&gt;iwibdev = i40iw_init_rdma_device(iwdev);</div><div class='del'>-	if (!iwdev-&gt;iwibdev)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-	iwibdev = iwdev-&gt;iwibdev;</div><div class='del'>-	rdma_set_device_sysfs_group(&amp;iwibdev-&gt;ibdev, &amp;i40iw_attr_group);</div><div class='del'>-	iwibdev-&gt;ibdev.driver_id = RDMA_DRIVER_I40IW;</div><div class='del'>-	ret = ib_register_device(&amp;iwibdev-&gt;ibdev, "i40iw%d");</div><div class='del'>-	if (ret)</div><div class='del'>-		goto error;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-error:</div><div class='del'>-	ib_dealloc_device(&amp;iwdev-&gt;iwibdev-&gt;ibdev);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.h b/drivers/infiniband/hw/i40iw/i40iw_verbs.h<br/>deleted file mode 100644<br/>index 3a413752ccc3..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_verbs.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_verbs.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,179 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#ifndef I40IW_VERBS_H</div><div class='del'>-#define I40IW_VERBS_H</div><div class='del'>-</div><div class='del'>-struct i40iw_ucontext {</div><div class='del'>-	struct ib_ucontext ibucontext;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct list_head cq_reg_mem_list;</div><div class='del'>-	spinlock_t cq_reg_mem_list_lock; /* memory list for cq's */</div><div class='del'>-	struct list_head qp_reg_mem_list;</div><div class='del'>-	spinlock_t qp_reg_mem_list_lock; /* memory list for qp's */</div><div class='del'>-	int abi_ver;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_pd {</div><div class='del'>-	struct ib_pd ibpd;</div><div class='del'>-	struct i40iw_sc_pd sc_pd;</div><div class='del'>-	atomic_t usecount;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_hmc_pble {</div><div class='del'>-	union {</div><div class='del'>-		u32 idx;</div><div class='del'>-		dma_addr_t addr;</div><div class='del'>-	};</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cq_mr {</div><div class='del'>-	struct i40iw_hmc_pble cq_pbl;</div><div class='del'>-	dma_addr_t shadow;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_qp_mr {</div><div class='del'>-	struct i40iw_hmc_pble sq_pbl;</div><div class='del'>-	struct i40iw_hmc_pble rq_pbl;</div><div class='del'>-	dma_addr_t shadow;</div><div class='del'>-	struct page *sq_page;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_pbl {</div><div class='del'>-	struct list_head list;</div><div class='del'>-	union {</div><div class='del'>-		struct i40iw_qp_mr qp_mr;</div><div class='del'>-		struct i40iw_cq_mr cq_mr;</div><div class='del'>-	};</div><div class='del'>-</div><div class='del'>-	bool pbl_allocated;</div><div class='del'>-	bool on_list;</div><div class='del'>-	u64 user_base;</div><div class='del'>-	struct i40iw_pble_alloc pble_alloc;</div><div class='del'>-	struct i40iw_mr *iwmr;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define MAX_SAVE_PAGE_ADDRS     4</div><div class='del'>-struct i40iw_mr {</div><div class='del'>-	union {</div><div class='del'>-		struct ib_mr ibmr;</div><div class='del'>-		struct ib_mw ibmw;</div><div class='del'>-		struct ib_fmr ibfmr;</div><div class='del'>-	};</div><div class='del'>-	struct ib_umem *region;</div><div class='del'>-	u16 type;</div><div class='del'>-	u32 page_cnt;</div><div class='del'>-	u64 page_size;</div><div class='del'>-	u32 npages;</div><div class='del'>-	u32 stag;</div><div class='del'>-	u64 length;</div><div class='del'>-	u64 pgaddrmem[MAX_SAVE_PAGE_ADDRS];</div><div class='del'>-	struct i40iw_pbl iwpbl;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_cq {</div><div class='del'>-	struct ib_cq ibcq;</div><div class='del'>-	struct i40iw_sc_cq sc_cq;</div><div class='del'>-	u16 cq_head;</div><div class='del'>-	u16 cq_size;</div><div class='del'>-	u16 cq_number;</div><div class='del'>-	bool user_mode;</div><div class='del'>-	u32 polled_completions;</div><div class='del'>-	u32 cq_mem_size;</div><div class='del'>-	struct i40iw_dma_mem kmem;</div><div class='del'>-	spinlock_t lock; /* for poll cq */</div><div class='del'>-	struct i40iw_pbl *iwpbl;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct disconn_work {</div><div class='del'>-	struct work_struct work;</div><div class='del'>-	struct i40iw_qp *iwqp;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct iw_cm_id;</div><div class='del'>-struct ietf_mpa_frame;</div><div class='del'>-struct i40iw_ud_file;</div><div class='del'>-</div><div class='del'>-struct i40iw_qp_kmode {</div><div class='del'>-	struct i40iw_dma_mem dma_mem;</div><div class='del'>-	u64 *wrid_mem;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_qp {</div><div class='del'>-	struct ib_qp ibqp;</div><div class='del'>-	struct i40iw_sc_qp sc_qp;</div><div class='del'>-	struct i40iw_device *iwdev;</div><div class='del'>-	struct i40iw_cq *iwscq;</div><div class='del'>-	struct i40iw_cq *iwrcq;</div><div class='del'>-	struct i40iw_pd *iwpd;</div><div class='del'>-	struct i40iw_qp_host_ctx_info ctx_info;</div><div class='del'>-	struct i40iwarp_offload_info iwarp_info;</div><div class='del'>-	void *allocated_buffer;</div><div class='del'>-	atomic_t refcount;</div><div class='del'>-	struct iw_cm_id *cm_id;</div><div class='del'>-	void *cm_node;</div><div class='del'>-	struct ib_mr *lsmm_mr;</div><div class='del'>-	struct work_struct work;</div><div class='del'>-	enum ib_qp_state ibqp_state;</div><div class='del'>-	u32 iwarp_state;</div><div class='del'>-	u32 qp_mem_size;</div><div class='del'>-	u32 last_aeq;</div><div class='del'>-	atomic_t close_timer_started;</div><div class='del'>-	spinlock_t lock; /* for post work requests */</div><div class='del'>-	struct i40iw_qp_context *iwqp_context;</div><div class='del'>-	void *pbl_vbase;</div><div class='del'>-	dma_addr_t pbl_pbase;</div><div class='del'>-	struct page *page;</div><div class='del'>-	u8 active_conn:1;</div><div class='del'>-	u8 user_mode:1;</div><div class='del'>-	u8 hte_added:1;</div><div class='del'>-	u8 flush_issued:1;</div><div class='del'>-	u8 destroyed:1;</div><div class='del'>-	u8 sig_all:1;</div><div class='del'>-	u8 pau_mode:1;</div><div class='del'>-	u8 rsvd:1;</div><div class='del'>-	u16 term_sq_flush_code;</div><div class='del'>-	u16 term_rq_flush_code;</div><div class='del'>-	u8 hw_iwarp_state;</div><div class='del'>-	u8 hw_tcp_state;</div><div class='del'>-	struct i40iw_qp_kmode kqp;</div><div class='del'>-	struct i40iw_dma_mem host_ctx;</div><div class='del'>-	struct timer_list terminate_timer;</div><div class='del'>-	struct i40iw_pbl iwpbl;</div><div class='del'>-	struct i40iw_dma_mem q2_ctx_mem;</div><div class='del'>-	struct i40iw_dma_mem ietf_mem;</div><div class='del'>-	struct completion sq_drained;</div><div class='del'>-	struct completion rq_drained;</div><div class='del'>-};</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_vf.c b/drivers/infiniband/hw/i40iw/i40iw_vf.c<br/>deleted file mode 100644<br/>index e33d4810965c..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_vf.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_vf.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,85 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#include "i40iw_osdep.h"</div><div class='del'>-#include "i40iw_register.h"</div><div class='del'>-#include "i40iw_status.h"</div><div class='del'>-#include "i40iw_hmc.h"</div><div class='del'>-#include "i40iw_d.h"</div><div class='del'>-#include "i40iw_type.h"</div><div class='del'>-#include "i40iw_p.h"</div><div class='del'>-#include "i40iw_vf.h"</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_manage_vf_pble_bp - manage vf pble</div><div class='del'>- * @cqp: cqp for cqp' sq wqe</div><div class='del'>- * @info: pble info</div><div class='del'>- * @scratch: pointer for completion</div><div class='del'>- * @post_sq: to post and ring</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_manage_vf_pble_bp(struct i40iw_sc_cqp *cqp,</div><div class='del'>-					       struct i40iw_manage_vf_pble_info *info,</div><div class='del'>-					       u64 scratch,</div><div class='del'>-					       bool post_sq)</div><div class='del'>-{</div><div class='del'>-	u64 *wqe;</div><div class='del'>-	u64 temp, header, pd_pl_pba = 0;</div><div class='del'>-</div><div class='del'>-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='del'>-	if (!wqe)</div><div class='del'>-		return I40IW_ERR_RING_FULL;</div><div class='del'>-</div><div class='del'>-	temp = LS_64(info-&gt;pd_entry_cnt, I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT) |</div><div class='del'>-	    LS_64(info-&gt;first_pd_index, I40IW_CQPSQ_MVPBP_FIRST_PD_INX) |</div><div class='del'>-	    LS_64(info-&gt;sd_index, I40IW_CQPSQ_MVPBP_SD_INX);</div><div class='del'>-	set_64bit_val(wqe, 16, temp);</div><div class='del'>-</div><div class='del'>-	header = LS_64((info-&gt;inv_pd_ent ? 1 : 0), I40IW_CQPSQ_MVPBP_INV_PD_ENT) |</div><div class='del'>-	    LS_64(I40IW_CQP_OP_MANAGE_VF_PBLE_BP, I40IW_CQPSQ_OPCODE) |</div><div class='del'>-	    LS_64(cqp-&gt;polarity, I40IW_CQPSQ_WQEVALID);</div><div class='del'>-	set_64bit_val(wqe, 24, header);</div><div class='del'>-</div><div class='del'>-	pd_pl_pba = LS_64(info-&gt;pd_pl_pba &gt;&gt; 3, I40IW_CQPSQ_MVPBP_PD_PLPBA);</div><div class='del'>-	set_64bit_val(wqe, 32, pd_pl_pba);</div><div class='del'>-</div><div class='del'>-	i40iw_debug_buf(cqp-&gt;dev, I40IW_DEBUG_WQE, "MANAGE VF_PBLE_BP WQE", wqe, I40IW_CQP_WQE_SIZE * 8);</div><div class='del'>-</div><div class='del'>-	if (post_sq)</div><div class='del'>-		i40iw_sc_cqp_post_sq(cqp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-const struct i40iw_vf_cqp_ops iw_vf_cqp_ops = {</div><div class='del'>-	i40iw_manage_vf_pble_bp</div><div class='del'>-};</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_vf.h b/drivers/infiniband/hw/i40iw/i40iw_vf.h<br/>deleted file mode 100644<br/>index 4359559ece9c..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_vf.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_vf.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,62 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#ifndef I40IW_VF_H</div><div class='del'>-#define I40IW_VF_H</div><div class='del'>-</div><div class='del'>-struct i40iw_sc_cqp;</div><div class='del'>-</div><div class='del'>-struct i40iw_manage_vf_pble_info {</div><div class='del'>-	u32 sd_index;</div><div class='del'>-	u16 first_pd_index;</div><div class='del'>-	u16 pd_entry_cnt;</div><div class='del'>-	u8 inv_pd_ent;</div><div class='del'>-	u64 pd_pl_pba;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_vf_cqp_ops {</div><div class='del'>-	enum i40iw_status_code (*manage_vf_pble_bp)(struct i40iw_sc_cqp *,</div><div class='del'>-						    struct i40iw_manage_vf_pble_info *,</div><div class='del'>-						    u64,</div><div class='del'>-						    bool);</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_manage_vf_pble_bp(struct i40iw_sc_cqp *cqp,</div><div class='del'>-					       struct i40iw_manage_vf_pble_info *info,</div><div class='del'>-					       u64 scratch,</div><div class='del'>-					       bool post_sq);</div><div class='del'>-</div><div class='del'>-extern const struct i40iw_vf_cqp_ops iw_vf_cqp_ops;</div><div class='del'>-</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c b/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c<br/>deleted file mode 100644<br/>index 48fd327f876b..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_virtchnl.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,756 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#include "i40iw_osdep.h"</div><div class='del'>-#include "i40iw_register.h"</div><div class='del'>-#include "i40iw_status.h"</div><div class='del'>-#include "i40iw_hmc.h"</div><div class='del'>-#include "i40iw_d.h"</div><div class='del'>-#include "i40iw_type.h"</div><div class='del'>-#include "i40iw_p.h"</div><div class='del'>-#include "i40iw_virtchnl.h"</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * vchnl_vf_send_get_ver_req - Request Channel version</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @vchnl_req: Virtual channel message request pointer</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code vchnl_vf_send_get_ver_req(struct i40iw_sc_dev *dev,</div><div class='del'>-							struct i40iw_virtchnl_req *vchnl_req)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code ret_code = I40IW_ERR_NOT_READY;</div><div class='del'>-	struct i40iw_virtchnl_op_buf *vchnl_msg = vchnl_req-&gt;vchnl_msg;</div><div class='del'>-</div><div class='del'>-	if (!dev-&gt;vchnl_up)</div><div class='del'>-		return ret_code;</div><div class='del'>-</div><div class='del'>-	memset(vchnl_msg, 0, sizeof(*vchnl_msg));</div><div class='del'>-	vchnl_msg-&gt;iw_chnl_op_ctx = (uintptr_t)vchnl_req;</div><div class='del'>-	vchnl_msg-&gt;iw_chnl_buf_len = sizeof(*vchnl_msg);</div><div class='del'>-	vchnl_msg-&gt;iw_op_code = I40IW_VCHNL_OP_GET_VER;</div><div class='del'>-	vchnl_msg-&gt;iw_op_ver = I40IW_VCHNL_OP_GET_VER_V0;</div><div class='del'>-	ret_code = dev-&gt;vchnl_if.vchnl_send(dev, 0, (u8 *)vchnl_msg, vchnl_msg-&gt;iw_chnl_buf_len);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "%s: virt channel send failed 0x%x\n", __func__, ret_code);</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * vchnl_vf_send_get_hmc_fcn_req - Request HMC Function from VF</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @vchnl_req: Virtual channel message request pointer</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code vchnl_vf_send_get_hmc_fcn_req(struct i40iw_sc_dev *dev,</div><div class='del'>-							    struct i40iw_virtchnl_req *vchnl_req)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code ret_code = I40IW_ERR_NOT_READY;</div><div class='del'>-	struct i40iw_virtchnl_op_buf *vchnl_msg = vchnl_req-&gt;vchnl_msg;</div><div class='del'>-</div><div class='del'>-	if (!dev-&gt;vchnl_up)</div><div class='del'>-		return ret_code;</div><div class='del'>-</div><div class='del'>-	memset(vchnl_msg, 0, sizeof(*vchnl_msg));</div><div class='del'>-	vchnl_msg-&gt;iw_chnl_op_ctx = (uintptr_t)vchnl_req;</div><div class='del'>-	vchnl_msg-&gt;iw_chnl_buf_len = sizeof(*vchnl_msg);</div><div class='del'>-	vchnl_msg-&gt;iw_op_code = I40IW_VCHNL_OP_GET_HMC_FCN;</div><div class='del'>-	vchnl_msg-&gt;iw_op_ver = I40IW_VCHNL_OP_GET_HMC_FCN_V0;</div><div class='del'>-	ret_code = dev-&gt;vchnl_if.vchnl_send(dev, 0, (u8 *)vchnl_msg, vchnl_msg-&gt;iw_chnl_buf_len);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "%s: virt channel send failed 0x%x\n", __func__, ret_code);</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * vchnl_vf_send_get_pe_stats_req - Request PE stats from VF</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @vchnl_req: Virtual channel message request pointer</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code vchnl_vf_send_get_pe_stats_req(struct i40iw_sc_dev *dev,</div><div class='del'>-							     struct i40iw_virtchnl_req  *vchnl_req)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code ret_code = I40IW_ERR_NOT_READY;</div><div class='del'>-	struct i40iw_virtchnl_op_buf *vchnl_msg = vchnl_req-&gt;vchnl_msg;</div><div class='del'>-</div><div class='del'>-	if (!dev-&gt;vchnl_up)</div><div class='del'>-		return ret_code;</div><div class='del'>-</div><div class='del'>-	memset(vchnl_msg, 0, sizeof(*vchnl_msg));</div><div class='del'>-	vchnl_msg-&gt;iw_chnl_op_ctx = (uintptr_t)vchnl_req;</div><div class='del'>-	vchnl_msg-&gt;iw_chnl_buf_len = sizeof(*vchnl_msg) + sizeof(struct i40iw_dev_hw_stats) - 1;</div><div class='del'>-	vchnl_msg-&gt;iw_op_code = I40IW_VCHNL_OP_GET_STATS;</div><div class='del'>-	vchnl_msg-&gt;iw_op_ver = I40IW_VCHNL_OP_GET_STATS_V0;</div><div class='del'>-	ret_code = dev-&gt;vchnl_if.vchnl_send(dev, 0, (u8 *)vchnl_msg, vchnl_msg-&gt;iw_chnl_buf_len);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "%s: virt channel send failed 0x%x\n", __func__, ret_code);</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * vchnl_vf_send_add_hmc_objs_req - Add HMC objects</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @vchnl_req: Virtual channel message request pointer</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code vchnl_vf_send_add_hmc_objs_req(struct i40iw_sc_dev *dev,</div><div class='del'>-							     struct i40iw_virtchnl_req *vchnl_req,</div><div class='del'>-							     enum i40iw_hmc_rsrc_type rsrc_type,</div><div class='del'>-							     u32 start_index,</div><div class='del'>-							     u32 rsrc_count)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code ret_code = I40IW_ERR_NOT_READY;</div><div class='del'>-	struct i40iw_virtchnl_op_buf *vchnl_msg = vchnl_req-&gt;vchnl_msg;</div><div class='del'>-	struct i40iw_virtchnl_hmc_obj_range *add_hmc_obj;</div><div class='del'>-</div><div class='del'>-	if (!dev-&gt;vchnl_up)</div><div class='del'>-		return ret_code;</div><div class='del'>-</div><div class='del'>-	add_hmc_obj = (struct i40iw_virtchnl_hmc_obj_range *)vchnl_msg-&gt;iw_chnl_buf;</div><div class='del'>-	memset(vchnl_msg, 0, sizeof(*vchnl_msg));</div><div class='del'>-	memset(add_hmc_obj, 0, sizeof(*add_hmc_obj));</div><div class='del'>-	vchnl_msg-&gt;iw_chnl_op_ctx = (uintptr_t)vchnl_req;</div><div class='del'>-	vchnl_msg-&gt;iw_chnl_buf_len = sizeof(*vchnl_msg) + sizeof(struct i40iw_virtchnl_hmc_obj_range) - 1;</div><div class='del'>-	vchnl_msg-&gt;iw_op_code = I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE;</div><div class='del'>-	vchnl_msg-&gt;iw_op_ver = I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE_V0;</div><div class='del'>-	add_hmc_obj-&gt;obj_type = (u16)rsrc_type;</div><div class='del'>-	add_hmc_obj-&gt;start_index = start_index;</div><div class='del'>-	add_hmc_obj-&gt;obj_count = rsrc_count;</div><div class='del'>-	ret_code = dev-&gt;vchnl_if.vchnl_send(dev, 0, (u8 *)vchnl_msg, vchnl_msg-&gt;iw_chnl_buf_len);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "%s: virt channel send failed 0x%x\n", __func__, ret_code);</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * vchnl_vf_send_del_hmc_objs_req - del HMC objects</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @vchnl_req: Virtual channel message request pointer</div><div class='del'>- * @ rsrc_type - resource type to delete</div><div class='del'>- * @ start_index - starting index for resource</div><div class='del'>- * @ rsrc_count - number of resource type to delete</div><div class='del'>- */</div><div class='del'>-static enum i40iw_status_code vchnl_vf_send_del_hmc_objs_req(struct i40iw_sc_dev *dev,</div><div class='del'>-							     struct i40iw_virtchnl_req *vchnl_req,</div><div class='del'>-							     enum i40iw_hmc_rsrc_type rsrc_type,</div><div class='del'>-							     u32 start_index,</div><div class='del'>-							     u32 rsrc_count)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code ret_code = I40IW_ERR_NOT_READY;</div><div class='del'>-	struct i40iw_virtchnl_op_buf *vchnl_msg = vchnl_req-&gt;vchnl_msg;</div><div class='del'>-	struct i40iw_virtchnl_hmc_obj_range *add_hmc_obj;</div><div class='del'>-</div><div class='del'>-	if (!dev-&gt;vchnl_up)</div><div class='del'>-		return ret_code;</div><div class='del'>-</div><div class='del'>-	add_hmc_obj = (struct i40iw_virtchnl_hmc_obj_range *)vchnl_msg-&gt;iw_chnl_buf;</div><div class='del'>-	memset(vchnl_msg, 0, sizeof(*vchnl_msg));</div><div class='del'>-	memset(add_hmc_obj, 0, sizeof(*add_hmc_obj));</div><div class='del'>-	vchnl_msg-&gt;iw_chnl_op_ctx = (uintptr_t)vchnl_req;</div><div class='del'>-	vchnl_msg-&gt;iw_chnl_buf_len = sizeof(*vchnl_msg) + sizeof(struct i40iw_virtchnl_hmc_obj_range) - 1;</div><div class='del'>-	vchnl_msg-&gt;iw_op_code = I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE;</div><div class='del'>-	vchnl_msg-&gt;iw_op_ver = I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE_V0;</div><div class='del'>-	add_hmc_obj-&gt;obj_type = (u16)rsrc_type;</div><div class='del'>-	add_hmc_obj-&gt;start_index = start_index;</div><div class='del'>-	add_hmc_obj-&gt;obj_count = rsrc_count;</div><div class='del'>-	ret_code = dev-&gt;vchnl_if.vchnl_send(dev, 0, (u8 *)vchnl_msg, vchnl_msg-&gt;iw_chnl_buf_len);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "%s: virt channel send failed 0x%x\n", __func__, ret_code);</div><div class='del'>-	return ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * vchnl_pf_send_get_ver_resp - Send channel version to VF</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @vf_id: Virtual function ID associated with the message</div><div class='del'>- * @vchnl_msg: Virtual channel message buffer pointer</div><div class='del'>- */</div><div class='del'>-static void vchnl_pf_send_get_ver_resp(struct i40iw_sc_dev *dev,</div><div class='del'>-				       u32 vf_id,</div><div class='del'>-				       struct i40iw_virtchnl_op_buf *vchnl_msg)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-	u8 resp_buffer[sizeof(struct i40iw_virtchnl_resp_buf) + sizeof(u32) - 1];</div><div class='del'>-	struct i40iw_virtchnl_resp_buf *vchnl_msg_resp = (struct i40iw_virtchnl_resp_buf *)resp_buffer;</div><div class='del'>-</div><div class='del'>-	memset(resp_buffer, 0, sizeof(*resp_buffer));</div><div class='del'>-	vchnl_msg_resp-&gt;iw_chnl_op_ctx = vchnl_msg-&gt;iw_chnl_op_ctx;</div><div class='del'>-	vchnl_msg_resp-&gt;iw_chnl_buf_len = sizeof(resp_buffer);</div><div class='del'>-	vchnl_msg_resp-&gt;iw_op_ret_code = I40IW_SUCCESS;</div><div class='del'>-	*((u32 *)vchnl_msg_resp-&gt;iw_chnl_buf) = I40IW_VCHNL_CHNL_VER_V0;</div><div class='del'>-	ret_code = dev-&gt;vchnl_if.vchnl_send(dev, vf_id, resp_buffer, sizeof(resp_buffer));</div><div class='del'>-	if (ret_code)</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "%s: virt channel send failed 0x%x\n", __func__, ret_code);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * vchnl_pf_send_get_hmc_fcn_resp - Send HMC Function to VF</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @vf_id: Virtual function ID associated with the message</div><div class='del'>- * @vchnl_msg: Virtual channel message buffer pointer</div><div class='del'>- */</div><div class='del'>-static void vchnl_pf_send_get_hmc_fcn_resp(struct i40iw_sc_dev *dev,</div><div class='del'>-					   u32 vf_id,</div><div class='del'>-					   struct i40iw_virtchnl_op_buf *vchnl_msg,</div><div class='del'>-					   u16 hmc_fcn)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-	u8 resp_buffer[sizeof(struct i40iw_virtchnl_resp_buf) + sizeof(u16) - 1];</div><div class='del'>-	struct i40iw_virtchnl_resp_buf *vchnl_msg_resp = (struct i40iw_virtchnl_resp_buf *)resp_buffer;</div><div class='del'>-</div><div class='del'>-	memset(resp_buffer, 0, sizeof(*resp_buffer));</div><div class='del'>-	vchnl_msg_resp-&gt;iw_chnl_op_ctx = vchnl_msg-&gt;iw_chnl_op_ctx;</div><div class='del'>-	vchnl_msg_resp-&gt;iw_chnl_buf_len = sizeof(resp_buffer);</div><div class='del'>-	vchnl_msg_resp-&gt;iw_op_ret_code = I40IW_SUCCESS;</div><div class='del'>-	*((u16 *)vchnl_msg_resp-&gt;iw_chnl_buf) = hmc_fcn;</div><div class='del'>-	ret_code = dev-&gt;vchnl_if.vchnl_send(dev, vf_id, resp_buffer, sizeof(resp_buffer));</div><div class='del'>-	if (ret_code)</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "%s: virt channel send failed 0x%x\n", __func__, ret_code);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * vchnl_pf_send_get_pe_stats_resp - Send PE Stats to VF</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @vf_id: Virtual function ID associated with the message</div><div class='del'>- * @vchnl_msg: Virtual channel message buffer pointer</div><div class='del'>- * @hw_stats: HW Stats struct</div><div class='del'>- */</div><div class='del'>-</div><div class='del'>-static void vchnl_pf_send_get_pe_stats_resp(struct i40iw_sc_dev *dev,</div><div class='del'>-					    u32 vf_id,</div><div class='del'>-					    struct i40iw_virtchnl_op_buf *vchnl_msg,</div><div class='del'>-					    struct i40iw_dev_hw_stats *hw_stats)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-	u8 resp_buffer[sizeof(struct i40iw_virtchnl_resp_buf) + sizeof(struct i40iw_dev_hw_stats) - 1];</div><div class='del'>-	struct i40iw_virtchnl_resp_buf *vchnl_msg_resp = (struct i40iw_virtchnl_resp_buf *)resp_buffer;</div><div class='del'>-</div><div class='del'>-	memset(resp_buffer, 0, sizeof(*resp_buffer));</div><div class='del'>-	vchnl_msg_resp-&gt;iw_chnl_op_ctx = vchnl_msg-&gt;iw_chnl_op_ctx;</div><div class='del'>-	vchnl_msg_resp-&gt;iw_chnl_buf_len = sizeof(resp_buffer);</div><div class='del'>-	vchnl_msg_resp-&gt;iw_op_ret_code = I40IW_SUCCESS;</div><div class='del'>-	*((struct i40iw_dev_hw_stats *)vchnl_msg_resp-&gt;iw_chnl_buf) = *hw_stats;</div><div class='del'>-	ret_code = dev-&gt;vchnl_if.vchnl_send(dev, vf_id, resp_buffer, sizeof(resp_buffer));</div><div class='del'>-	if (ret_code)</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "%s: virt channel send failed 0x%x\n", __func__, ret_code);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * vchnl_pf_send_error_resp - Send an error response to VF</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @vf_id: Virtual function ID associated with the message</div><div class='del'>- * @vchnl_msg: Virtual channel message buffer pointer</div><div class='del'>- */</div><div class='del'>-static void vchnl_pf_send_error_resp(struct i40iw_sc_dev *dev, u32 vf_id,</div><div class='del'>-				     struct i40iw_virtchnl_op_buf *vchnl_msg,</div><div class='del'>-				     u16 op_ret_code)</div><div class='del'>-{</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-	u8 resp_buffer[sizeof(struct i40iw_virtchnl_resp_buf)];</div><div class='del'>-	struct i40iw_virtchnl_resp_buf *vchnl_msg_resp = (struct i40iw_virtchnl_resp_buf *)resp_buffer;</div><div class='del'>-</div><div class='del'>-	memset(resp_buffer, 0, sizeof(resp_buffer));</div><div class='del'>-	vchnl_msg_resp-&gt;iw_chnl_op_ctx = vchnl_msg-&gt;iw_chnl_op_ctx;</div><div class='del'>-	vchnl_msg_resp-&gt;iw_chnl_buf_len = sizeof(resp_buffer);</div><div class='del'>-	vchnl_msg_resp-&gt;iw_op_ret_code = (u16)op_ret_code;</div><div class='del'>-	ret_code = dev-&gt;vchnl_if.vchnl_send(dev, vf_id, resp_buffer, sizeof(resp_buffer));</div><div class='del'>-	if (ret_code)</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "%s: virt channel send failed 0x%x\n", __func__, ret_code);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * pf_cqp_get_hmc_fcn_callback - Callback for Get HMC Fcn</div><div class='del'>- * @cqp_req_param: CQP Request param value</div><div class='del'>- * @not_used: unused CQP callback parameter</div><div class='del'>- */</div><div class='del'>-static void pf_cqp_get_hmc_fcn_callback(struct i40iw_sc_dev *dev, void *callback_param,</div><div class='del'>-					struct i40iw_ccq_cqe_info *cqe_info)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_vfdev *vf_dev = callback_param;</div><div class='del'>-	struct i40iw_virt_mem vf_dev_mem;</div><div class='del'>-</div><div class='del'>-	if (cqe_info-&gt;error) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "CQP Completion Error on Get HMC Function.  Maj = 0x%04x, Minor = 0x%04x\n",</div><div class='del'>-			    cqe_info-&gt;maj_err_code, cqe_info-&gt;min_err_code);</div><div class='del'>-		dev-&gt;vf_dev[vf_dev-&gt;iw_vf_idx] = NULL;</div><div class='del'>-		vchnl_pf_send_error_resp(dev, vf_dev-&gt;vf_id, &amp;vf_dev-&gt;vf_msg_buffer.vchnl_msg,</div><div class='del'>-					 (u16)I40IW_ERR_CQP_COMPL_ERROR);</div><div class='del'>-		vf_dev_mem.va = vf_dev;</div><div class='del'>-		vf_dev_mem.size = sizeof(*vf_dev);</div><div class='del'>-		i40iw_free_virt_mem(dev-&gt;hw, &amp;vf_dev_mem);</div><div class='del'>-	} else {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "CQP Completion Operation Return information = 0x%08x\n",</div><div class='del'>-			    cqe_info-&gt;op_ret_val);</div><div class='del'>-		vf_dev-&gt;pmf_index = (u16)cqe_info-&gt;op_ret_val;</div><div class='del'>-		vf_dev-&gt;msg_count--;</div><div class='del'>-		vchnl_pf_send_get_hmc_fcn_resp(dev,</div><div class='del'>-					       vf_dev-&gt;vf_id,</div><div class='del'>-					       &amp;vf_dev-&gt;vf_msg_buffer.vchnl_msg,</div><div class='del'>-					       vf_dev-&gt;pmf_index);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * pf_add_hmc_obj - Callback for Add HMC Object</div><div class='del'>- * @vf_dev: pointer to the VF Device</div><div class='del'>- */</div><div class='del'>-static void pf_add_hmc_obj_callback(void *work_vf_dev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_vfdev *vf_dev = (struct i40iw_vfdev *)work_vf_dev;</div><div class='del'>-	struct i40iw_hmc_info *hmc_info = &amp;vf_dev-&gt;hmc_info;</div><div class='del'>-	struct i40iw_virtchnl_op_buf *vchnl_msg = &amp;vf_dev-&gt;vf_msg_buffer.vchnl_msg;</div><div class='del'>-	struct i40iw_hmc_create_obj_info info;</div><div class='del'>-	struct i40iw_virtchnl_hmc_obj_range *add_hmc_obj;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-</div><div class='del'>-	if (!vf_dev-&gt;pf_hmc_initialized) {</div><div class='del'>-		ret_code = i40iw_pf_init_vfhmc(vf_dev-&gt;pf_dev, (u8)vf_dev-&gt;pmf_index, NULL);</div><div class='del'>-		if (ret_code)</div><div class='del'>-			goto add_out;</div><div class='del'>-		vf_dev-&gt;pf_hmc_initialized = true;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	add_hmc_obj = (struct i40iw_virtchnl_hmc_obj_range *)vchnl_msg-&gt;iw_chnl_buf;</div><div class='del'>-</div><div class='del'>-	memset(&amp;info, 0, sizeof(info));</div><div class='del'>-	info.hmc_info = hmc_info;</div><div class='del'>-	info.is_pf = false;</div><div class='del'>-	info.rsrc_type = (u32)add_hmc_obj-&gt;obj_type;</div><div class='del'>-	info.entry_type = (info.rsrc_type == I40IW_HMC_IW_PBLE) ? I40IW_SD_TYPE_PAGED : I40IW_SD_TYPE_DIRECT;</div><div class='del'>-	info.start_idx = add_hmc_obj-&gt;start_index;</div><div class='del'>-	info.count = add_hmc_obj-&gt;obj_count;</div><div class='del'>-	i40iw_debug(vf_dev-&gt;pf_dev, I40IW_DEBUG_VIRT,</div><div class='del'>-		    "I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE.  Add %u type %u objects\n",</div><div class='del'>-		    info.count, info.rsrc_type);</div><div class='del'>-	ret_code = i40iw_sc_create_hmc_obj(vf_dev-&gt;pf_dev, &amp;info);</div><div class='del'>-	if (!ret_code)</div><div class='del'>-		vf_dev-&gt;hmc_info.hmc_obj[add_hmc_obj-&gt;obj_type].cnt = add_hmc_obj-&gt;obj_count;</div><div class='del'>-add_out:</div><div class='del'>-	vf_dev-&gt;msg_count--;</div><div class='del'>-	vchnl_pf_send_error_resp(vf_dev-&gt;pf_dev, vf_dev-&gt;vf_id, vchnl_msg, (u16)ret_code);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * pf_del_hmc_obj_callback - Callback for delete HMC Object</div><div class='del'>- * @work_vf_dev: pointer to the VF Device</div><div class='del'>- */</div><div class='del'>-static void pf_del_hmc_obj_callback(void *work_vf_dev)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_vfdev *vf_dev = (struct i40iw_vfdev *)work_vf_dev;</div><div class='del'>-	struct i40iw_hmc_info *hmc_info = &amp;vf_dev-&gt;hmc_info;</div><div class='del'>-	struct i40iw_virtchnl_op_buf *vchnl_msg = &amp;vf_dev-&gt;vf_msg_buffer.vchnl_msg;</div><div class='del'>-	struct i40iw_hmc_del_obj_info info;</div><div class='del'>-	struct i40iw_virtchnl_hmc_obj_range *del_hmc_obj;</div><div class='del'>-	enum i40iw_status_code ret_code = I40IW_SUCCESS;</div><div class='del'>-</div><div class='del'>-	if (!vf_dev-&gt;pf_hmc_initialized)</div><div class='del'>-		goto del_out;</div><div class='del'>-</div><div class='del'>-	del_hmc_obj = (struct i40iw_virtchnl_hmc_obj_range *)vchnl_msg-&gt;iw_chnl_buf;</div><div class='del'>-</div><div class='del'>-	memset(&amp;info, 0, sizeof(info));</div><div class='del'>-	info.hmc_info = hmc_info;</div><div class='del'>-	info.is_pf = false;</div><div class='del'>-	info.rsrc_type = (u32)del_hmc_obj-&gt;obj_type;</div><div class='del'>-	info.start_idx = del_hmc_obj-&gt;start_index;</div><div class='del'>-	info.count = del_hmc_obj-&gt;obj_count;</div><div class='del'>-	i40iw_debug(vf_dev-&gt;pf_dev, I40IW_DEBUG_VIRT,</div><div class='del'>-		    "I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE.  Delete %u type %u objects\n",</div><div class='del'>-		    info.count, info.rsrc_type);</div><div class='del'>-	ret_code = i40iw_sc_del_hmc_obj(vf_dev-&gt;pf_dev, &amp;info, false);</div><div class='del'>-del_out:</div><div class='del'>-	vf_dev-&gt;msg_count--;</div><div class='del'>-	vchnl_pf_send_error_resp(vf_dev-&gt;pf_dev, vf_dev-&gt;vf_id, vchnl_msg, (u16)ret_code);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vf_init_pestat - Initialize stats for VF</div><div class='del'>- * @devL pointer to the VF Device</div><div class='del'>- * @stats: Statistics structure pointer</div><div class='del'>- * @index: Stats index</div><div class='del'>- */</div><div class='del'>-static void i40iw_vf_init_pestat(struct i40iw_sc_dev *dev, struct i40iw_vsi_pestat *stats, u16 index)</div><div class='del'>-{</div><div class='del'>-	stats-&gt;hw = dev-&gt;hw;</div><div class='del'>-	i40iw_hw_stats_init(stats, (u8)index, false);</div><div class='del'>-	spin_lock_init(&amp;stats-&gt;lock);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vchnl_recv_pf - Receive PF virtual channel messages</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @vf_id: Virtual function ID associated with the message</div><div class='del'>- * @msg: Virtual channel message buffer pointer</div><div class='del'>- * @len: Length of the virtual channels message</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_vchnl_recv_pf(struct i40iw_sc_dev *dev,</div><div class='del'>-					   u32 vf_id,</div><div class='del'>-					   u8 *msg,</div><div class='del'>-					   u16 len)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_virtchnl_op_buf *vchnl_msg = (struct i40iw_virtchnl_op_buf *)msg;</div><div class='del'>-	struct i40iw_vfdev *vf_dev = NULL;</div><div class='del'>-	struct i40iw_hmc_fcn_info hmc_fcn_info;</div><div class='del'>-	u16 iw_vf_idx;</div><div class='del'>-	u16 first_avail_iw_vf = I40IW_MAX_PE_ENABLED_VF_COUNT;</div><div class='del'>-	struct i40iw_virt_mem vf_dev_mem;</div><div class='del'>-	struct i40iw_virtchnl_work_info work_info;</div><div class='del'>-	struct i40iw_vsi_pestat *stats;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-</div><div class='del'>-	if (!dev || !msg || !len)</div><div class='del'>-		return I40IW_ERR_PARAM;</div><div class='del'>-</div><div class='del'>-	if (!dev-&gt;vchnl_up)</div><div class='del'>-		return I40IW_ERR_NOT_READY;</div><div class='del'>-	if (vchnl_msg-&gt;iw_op_code == I40IW_VCHNL_OP_GET_VER) {</div><div class='del'>-		vchnl_pf_send_get_ver_resp(dev, vf_id, vchnl_msg);</div><div class='del'>-		return I40IW_SUCCESS;</div><div class='del'>-	}</div><div class='del'>-	for (iw_vf_idx = 0; iw_vf_idx &lt; I40IW_MAX_PE_ENABLED_VF_COUNT; iw_vf_idx++) {</div><div class='del'>-		if (!dev-&gt;vf_dev[iw_vf_idx]) {</div><div class='del'>-			if (first_avail_iw_vf == I40IW_MAX_PE_ENABLED_VF_COUNT)</div><div class='del'>-				first_avail_iw_vf = iw_vf_idx;</div><div class='del'>-			continue;</div><div class='del'>-		}</div><div class='del'>-		if (dev-&gt;vf_dev[iw_vf_idx]-&gt;vf_id == vf_id) {</div><div class='del'>-			vf_dev = dev-&gt;vf_dev[iw_vf_idx];</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	if (vf_dev) {</div><div class='del'>-		if (!vf_dev-&gt;msg_count) {</div><div class='del'>-			vf_dev-&gt;msg_count++;</div><div class='del'>-		} else {</div><div class='del'>-			i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-				    "VF%u already has a channel message in progress.\n",</div><div class='del'>-				    vf_id);</div><div class='del'>-			return I40IW_SUCCESS;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	switch (vchnl_msg-&gt;iw_op_code) {</div><div class='del'>-	case I40IW_VCHNL_OP_GET_HMC_FCN:</div><div class='del'>-		if (!vf_dev &amp;&amp;</div><div class='del'>-		    (first_avail_iw_vf != I40IW_MAX_PE_ENABLED_VF_COUNT)) {</div><div class='del'>-			ret_code = i40iw_allocate_virt_mem(dev-&gt;hw, &amp;vf_dev_mem, sizeof(struct i40iw_vfdev) +</div><div class='del'>-							   (sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX));</div><div class='del'>-			if (!ret_code) {</div><div class='del'>-				vf_dev = vf_dev_mem.va;</div><div class='del'>-				vf_dev-&gt;stats_initialized = false;</div><div class='del'>-				vf_dev-&gt;pf_dev = dev;</div><div class='del'>-				vf_dev-&gt;msg_count = 1;</div><div class='del'>-				vf_dev-&gt;vf_id = vf_id;</div><div class='del'>-				vf_dev-&gt;iw_vf_idx = first_avail_iw_vf;</div><div class='del'>-				vf_dev-&gt;pf_hmc_initialized = false;</div><div class='del'>-				vf_dev-&gt;hmc_info.hmc_obj = (struct i40iw_hmc_obj_info *)(&amp;vf_dev[1]);</div><div class='del'>-				i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-					    "vf_dev %p, hmc_info %p, hmc_obj %p\n",</div><div class='del'>-					    vf_dev, &amp;vf_dev-&gt;hmc_info, vf_dev-&gt;hmc_info.hmc_obj);</div><div class='del'>-				dev-&gt;vf_dev[first_avail_iw_vf] = vf_dev;</div><div class='del'>-				iw_vf_idx = first_avail_iw_vf;</div><div class='del'>-			} else {</div><div class='del'>-				i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-					    "VF%u Unable to allocate a VF device structure.\n",</div><div class='del'>-					    vf_id);</div><div class='del'>-				vchnl_pf_send_error_resp(dev, vf_id, vchnl_msg, (u16)I40IW_ERR_NO_MEMORY);</div><div class='del'>-				return I40IW_SUCCESS;</div><div class='del'>-			}</div><div class='del'>-			memcpy(&amp;vf_dev-&gt;vf_msg_buffer.vchnl_msg, vchnl_msg, len);</div><div class='del'>-			hmc_fcn_info.callback_fcn = pf_cqp_get_hmc_fcn_callback;</div><div class='del'>-			hmc_fcn_info.vf_id = vf_id;</div><div class='del'>-			hmc_fcn_info.iw_vf_idx = vf_dev-&gt;iw_vf_idx;</div><div class='del'>-			hmc_fcn_info.cqp_callback_param = vf_dev;</div><div class='del'>-			hmc_fcn_info.free_fcn = false;</div><div class='del'>-			ret_code = i40iw_cqp_manage_hmc_fcn_cmd(dev, &amp;hmc_fcn_info);</div><div class='del'>-			if (ret_code)</div><div class='del'>-				i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-					    "VF%u error CQP HMC Function operation.\n",</div><div class='del'>-					    vf_id);</div><div class='del'>-			i40iw_vf_init_pestat(dev, &amp;vf_dev-&gt;pestat, vf_dev-&gt;pmf_index);</div><div class='del'>-			vf_dev-&gt;stats_initialized = true;</div><div class='del'>-		} else {</div><div class='del'>-			if (vf_dev) {</div><div class='del'>-				vf_dev-&gt;msg_count--;</div><div class='del'>-				vchnl_pf_send_get_hmc_fcn_resp(dev, vf_id, vchnl_msg, vf_dev-&gt;pmf_index);</div><div class='del'>-			} else {</div><div class='del'>-				vchnl_pf_send_error_resp(dev, vf_id, vchnl_msg,</div><div class='del'>-							 (u16)I40IW_ERR_NO_MEMORY);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE:</div><div class='del'>-		if (!vf_dev)</div><div class='del'>-			return I40IW_ERR_BAD_PTR;</div><div class='del'>-		work_info.worker_vf_dev = vf_dev;</div><div class='del'>-		work_info.callback_fcn = pf_add_hmc_obj_callback;</div><div class='del'>-		memcpy(&amp;vf_dev-&gt;vf_msg_buffer.vchnl_msg, vchnl_msg, len);</div><div class='del'>-		i40iw_cqp_spawn_worker(dev, &amp;work_info, vf_dev-&gt;iw_vf_idx);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE:</div><div class='del'>-		if (!vf_dev)</div><div class='del'>-			return I40IW_ERR_BAD_PTR;</div><div class='del'>-		work_info.worker_vf_dev = vf_dev;</div><div class='del'>-		work_info.callback_fcn = pf_del_hmc_obj_callback;</div><div class='del'>-		memcpy(&amp;vf_dev-&gt;vf_msg_buffer.vchnl_msg, vchnl_msg, len);</div><div class='del'>-		i40iw_cqp_spawn_worker(dev, &amp;work_info, vf_dev-&gt;iw_vf_idx);</div><div class='del'>-		break;</div><div class='del'>-	case I40IW_VCHNL_OP_GET_STATS:</div><div class='del'>-		if (!vf_dev)</div><div class='del'>-			return I40IW_ERR_BAD_PTR;</div><div class='del'>-		stats = &amp;vf_dev-&gt;pestat;</div><div class='del'>-		i40iw_hw_stats_read_all(stats, &amp;stats-&gt;hw_stats);</div><div class='del'>-		vf_dev-&gt;msg_count--;</div><div class='del'>-		vchnl_pf_send_get_pe_stats_resp(dev, vf_id, vchnl_msg, &amp;stats-&gt;hw_stats);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "40iw_vchnl_recv_pf: Invalid OpCode 0x%x\n",</div><div class='del'>-			    vchnl_msg-&gt;iw_op_code);</div><div class='del'>-		vchnl_pf_send_error_resp(dev, vf_id,</div><div class='del'>-					 vchnl_msg, (u16)I40IW_ERR_NOT_IMPLEMENTED);</div><div class='del'>-	}</div><div class='del'>-	return I40IW_SUCCESS;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vchnl_recv_vf - Receive VF virtual channel messages</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @vf_id: Virtual function ID associated with the message</div><div class='del'>- * @msg: Virtual channel message buffer pointer</div><div class='del'>- * @len: Length of the virtual channels message</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_vchnl_recv_vf(struct i40iw_sc_dev *dev,</div><div class='del'>-					   u32 vf_id,</div><div class='del'>-					   u8 *msg,</div><div class='del'>-					   u16 len)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_virtchnl_resp_buf *vchnl_msg_resp = (struct i40iw_virtchnl_resp_buf *)msg;</div><div class='del'>-	struct i40iw_virtchnl_req *vchnl_req;</div><div class='del'>-</div><div class='del'>-	vchnl_req = (struct i40iw_virtchnl_req *)(uintptr_t)vchnl_msg_resp-&gt;iw_chnl_op_ctx;</div><div class='del'>-	vchnl_req-&gt;ret_code = (enum i40iw_status_code)vchnl_msg_resp-&gt;iw_op_ret_code;</div><div class='del'>-	if (len == (sizeof(*vchnl_msg_resp) + vchnl_req-&gt;parm_len - 1)) {</div><div class='del'>-		if (vchnl_req-&gt;parm_len &amp;&amp; vchnl_req-&gt;parm)</div><div class='del'>-			memcpy(vchnl_req-&gt;parm, vchnl_msg_resp-&gt;iw_chnl_buf, vchnl_req-&gt;parm_len);</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "%s: Got response, data size %u\n", __func__,</div><div class='del'>-			    vchnl_req-&gt;parm_len);</div><div class='del'>-	} else {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "%s: error length on response, Got %u, expected %u\n", __func__,</div><div class='del'>-			    len, (u32)(sizeof(*vchnl_msg_resp) + vchnl_req-&gt;parm_len - 1));</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return I40IW_SUCCESS;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vchnl_vf_get_ver - Request Channel version</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @vchnl_ver: Virtual channel message version pointer</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_vchnl_vf_get_ver(struct i40iw_sc_dev *dev,</div><div class='del'>-					      u32 *vchnl_ver)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_virtchnl_req vchnl_req;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-</div><div class='del'>-	if (!i40iw_vf_clear_to_send(dev))</div><div class='del'>-		return I40IW_ERR_TIMEOUT;</div><div class='del'>-	memset(&amp;vchnl_req, 0, sizeof(vchnl_req));</div><div class='del'>-	vchnl_req.dev = dev;</div><div class='del'>-	vchnl_req.parm = vchnl_ver;</div><div class='del'>-	vchnl_req.parm_len = sizeof(*vchnl_ver);</div><div class='del'>-	vchnl_req.vchnl_msg = &amp;dev-&gt;vchnl_vf_msg_buf.vchnl_msg;</div><div class='del'>-</div><div class='del'>-	ret_code = vchnl_vf_send_get_ver_req(dev, &amp;vchnl_req);</div><div class='del'>-	if (ret_code) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "%s Send message failed 0x%0x\n", __func__, ret_code);</div><div class='del'>-		return ret_code;</div><div class='del'>-	}</div><div class='del'>-	ret_code = i40iw_vf_wait_vchnl_resp(dev);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return ret_code;</div><div class='del'>-	else</div><div class='del'>-		return vchnl_req.ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vchnl_vf_get_hmc_fcn - Request HMC Function</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @hmc_fcn: HMC function index pointer</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_vchnl_vf_get_hmc_fcn(struct i40iw_sc_dev *dev,</div><div class='del'>-						  u16 *hmc_fcn)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_virtchnl_req vchnl_req;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-</div><div class='del'>-	if (!i40iw_vf_clear_to_send(dev))</div><div class='del'>-		return I40IW_ERR_TIMEOUT;</div><div class='del'>-	memset(&amp;vchnl_req, 0, sizeof(vchnl_req));</div><div class='del'>-	vchnl_req.dev = dev;</div><div class='del'>-	vchnl_req.parm = hmc_fcn;</div><div class='del'>-	vchnl_req.parm_len = sizeof(*hmc_fcn);</div><div class='del'>-	vchnl_req.vchnl_msg = &amp;dev-&gt;vchnl_vf_msg_buf.vchnl_msg;</div><div class='del'>-</div><div class='del'>-	ret_code = vchnl_vf_send_get_hmc_fcn_req(dev, &amp;vchnl_req);</div><div class='del'>-	if (ret_code) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "%s Send message failed 0x%0x\n", __func__, ret_code);</div><div class='del'>-		return ret_code;</div><div class='del'>-	}</div><div class='del'>-	ret_code = i40iw_vf_wait_vchnl_resp(dev);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return ret_code;</div><div class='del'>-	else</div><div class='del'>-		return vchnl_req.ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vchnl_vf_add_hmc_objs - Add HMC Object</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @rsrc_type: HMC Resource type</div><div class='del'>- * @start_index: Starting index of the objects to be added</div><div class='del'>- * @rsrc_count: Number of resources to be added</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_vchnl_vf_add_hmc_objs(struct i40iw_sc_dev *dev,</div><div class='del'>-						   enum i40iw_hmc_rsrc_type rsrc_type,</div><div class='del'>-						   u32 start_index,</div><div class='del'>-						   u32 rsrc_count)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_virtchnl_req vchnl_req;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-</div><div class='del'>-	if (!i40iw_vf_clear_to_send(dev))</div><div class='del'>-		return I40IW_ERR_TIMEOUT;</div><div class='del'>-	memset(&amp;vchnl_req, 0, sizeof(vchnl_req));</div><div class='del'>-	vchnl_req.dev = dev;</div><div class='del'>-	vchnl_req.vchnl_msg = &amp;dev-&gt;vchnl_vf_msg_buf.vchnl_msg;</div><div class='del'>-</div><div class='del'>-	ret_code = vchnl_vf_send_add_hmc_objs_req(dev,</div><div class='del'>-						  &amp;vchnl_req,</div><div class='del'>-						  rsrc_type,</div><div class='del'>-						  start_index,</div><div class='del'>-						  rsrc_count);</div><div class='del'>-	if (ret_code) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "%s Send message failed 0x%0x\n", __func__, ret_code);</div><div class='del'>-		return ret_code;</div><div class='del'>-	}</div><div class='del'>-	ret_code = i40iw_vf_wait_vchnl_resp(dev);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return ret_code;</div><div class='del'>-	else</div><div class='del'>-		return vchnl_req.ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vchnl_vf_del_hmc_obj - del HMC obj</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @rsrc_type: HMC Resource type</div><div class='del'>- * @start_index: Starting index of the object to delete</div><div class='del'>- * @rsrc_count: Number of resources to be delete</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_vchnl_vf_del_hmc_obj(struct i40iw_sc_dev *dev,</div><div class='del'>-						  enum i40iw_hmc_rsrc_type rsrc_type,</div><div class='del'>-						  u32 start_index,</div><div class='del'>-						  u32 rsrc_count)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_virtchnl_req vchnl_req;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-</div><div class='del'>-	if (!i40iw_vf_clear_to_send(dev))</div><div class='del'>-		return I40IW_ERR_TIMEOUT;</div><div class='del'>-	memset(&amp;vchnl_req, 0, sizeof(vchnl_req));</div><div class='del'>-	vchnl_req.dev = dev;</div><div class='del'>-	vchnl_req.vchnl_msg = &amp;dev-&gt;vchnl_vf_msg_buf.vchnl_msg;</div><div class='del'>-</div><div class='del'>-	ret_code = vchnl_vf_send_del_hmc_objs_req(dev,</div><div class='del'>-						  &amp;vchnl_req,</div><div class='del'>-						  rsrc_type,</div><div class='del'>-						  start_index,</div><div class='del'>-						  rsrc_count);</div><div class='del'>-	if (ret_code) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "%s Send message failed 0x%0x\n", __func__, ret_code);</div><div class='del'>-		return ret_code;</div><div class='del'>-	}</div><div class='del'>-	ret_code = i40iw_vf_wait_vchnl_resp(dev);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return ret_code;</div><div class='del'>-	else</div><div class='del'>-		return vchnl_req.ret_code;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/**</div><div class='del'>- * i40iw_vchnl_vf_get_pe_stats - Get PE stats</div><div class='del'>- * @dev: IWARP device pointer</div><div class='del'>- * @hw_stats: HW stats struct</div><div class='del'>- */</div><div class='del'>-enum i40iw_status_code i40iw_vchnl_vf_get_pe_stats(struct i40iw_sc_dev *dev,</div><div class='del'>-						   struct i40iw_dev_hw_stats *hw_stats)</div><div class='del'>-{</div><div class='del'>-	struct i40iw_virtchnl_req  vchnl_req;</div><div class='del'>-	enum i40iw_status_code ret_code;</div><div class='del'>-</div><div class='del'>-	if (!i40iw_vf_clear_to_send(dev))</div><div class='del'>-		return I40IW_ERR_TIMEOUT;</div><div class='del'>-	memset(&amp;vchnl_req, 0, sizeof(vchnl_req));</div><div class='del'>-	vchnl_req.dev = dev;</div><div class='del'>-	vchnl_req.parm = hw_stats;</div><div class='del'>-	vchnl_req.parm_len = sizeof(*hw_stats);</div><div class='del'>-	vchnl_req.vchnl_msg = &amp;dev-&gt;vchnl_vf_msg_buf.vchnl_msg;</div><div class='del'>-</div><div class='del'>-	ret_code = vchnl_vf_send_get_pe_stats_req(dev, &amp;vchnl_req);</div><div class='del'>-	if (ret_code) {</div><div class='del'>-		i40iw_debug(dev, I40IW_DEBUG_VIRT,</div><div class='del'>-			    "%s Send message failed 0x%0x\n", __func__, ret_code);</div><div class='del'>-		return ret_code;</div><div class='del'>-	}</div><div class='del'>-	ret_code = i40iw_vf_wait_vchnl_resp(dev);</div><div class='del'>-	if (ret_code)</div><div class='del'>-		return ret_code;</div><div class='del'>-	else</div><div class='del'>-		return vchnl_req.ret_code;</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/i40iw/i40iw_virtchnl.h b/drivers/infiniband/hw/i40iw/i40iw_virtchnl.h<br/>deleted file mode 100644<br/>index 24886ef08293..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/i40iw/i40iw_virtchnl.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/i40iw/i40iw_virtchnl.h</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,124 +0,0 @@</div><div class='del'>-/*******************************************************************************</div><div class='del'>-*</div><div class='del'>-* Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.</div><div class='del'>-*</div><div class='del'>-* This software is available to you under a choice of one of two</div><div class='del'>-* licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>-* General Public License (GPL) Version 2, available from the file</div><div class='del'>-* COPYING in the main directory of this source tree, or the</div><div class='del'>-* OpenFabrics.org BSD license below:</div><div class='del'>-*</div><div class='del'>-*   Redistribution and use in source and binary forms, with or</div><div class='del'>-*   without modification, are permitted provided that the following</div><div class='del'>-*   conditions are met:</div><div class='del'>-*</div><div class='del'>-*    - Redistributions of source code must retain the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer.</div><div class='del'>-*</div><div class='del'>-*    - Redistributions in binary form must reproduce the above</div><div class='del'>-*	copyright notice, this list of conditions and the following</div><div class='del'>-*	disclaimer in the documentation and/or other materials</div><div class='del'>-*	provided with the distribution.</div><div class='del'>-*</div><div class='del'>-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>-* SOFTWARE.</div><div class='del'>-*</div><div class='del'>-*******************************************************************************/</div><div class='del'>-</div><div class='del'>-#ifndef I40IW_VIRTCHNL_H</div><div class='del'>-#define I40IW_VIRTCHNL_H</div><div class='del'>-</div><div class='del'>-#include "i40iw_hmc.h"</div><div class='del'>-</div><div class='del'>-#pragma pack(push, 1)</div><div class='del'>-</div><div class='del'>-struct i40iw_virtchnl_op_buf {</div><div class='del'>-	u16 iw_op_code;</div><div class='del'>-	u16 iw_op_ver;</div><div class='del'>-	u16 iw_chnl_buf_len;</div><div class='del'>-	u16 rsvd;</div><div class='del'>-	u64 iw_chnl_op_ctx;</div><div class='del'>-	/* Member alignment MUST be maintained above this location */</div><div class='del'>-	u8 iw_chnl_buf[1];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct i40iw_virtchnl_resp_buf {</div><div class='del'>-	u64 iw_chnl_op_ctx;</div><div class='del'>-	u16 iw_chnl_buf_len;</div><div class='del'>-	s16 iw_op_ret_code;</div><div class='del'>-	/* Member alignment MUST be maintained above this location */</div><div class='del'>-	u16 rsvd[2];</div><div class='del'>-	u8 iw_chnl_buf[1];</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_virtchnl_ops {</div><div class='del'>-	I40IW_VCHNL_OP_GET_VER = 0,</div><div class='del'>-	I40IW_VCHNL_OP_GET_HMC_FCN,</div><div class='del'>-	I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE,</div><div class='del'>-	I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE,</div><div class='del'>-	I40IW_VCHNL_OP_GET_STATS</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define I40IW_VCHNL_OP_GET_VER_V0 0</div><div class='del'>-#define I40IW_VCHNL_OP_GET_HMC_FCN_V0 0</div><div class='del'>-#define I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE_V0 0</div><div class='del'>-#define I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE_V0 0</div><div class='del'>-#define I40IW_VCHNL_OP_GET_STATS_V0 0</div><div class='del'>-#define I40IW_VCHNL_CHNL_VER_V0 0</div><div class='del'>-</div><div class='del'>-struct i40iw_dev_hw_stats;</div><div class='del'>-</div><div class='del'>-struct i40iw_virtchnl_hmc_obj_range {</div><div class='del'>-	u16 obj_type;</div><div class='del'>-	u16 rsvd;</div><div class='del'>-	u32 start_index;</div><div class='del'>-	u32 obj_count;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_vchnl_recv_pf(struct i40iw_sc_dev *dev,</div><div class='del'>-					   u32 vf_id,</div><div class='del'>-					   u8 *msg,</div><div class='del'>-					   u16 len);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_vchnl_recv_vf(struct i40iw_sc_dev *dev,</div><div class='del'>-					   u32 vf_id,</div><div class='del'>-					   u8 *msg,</div><div class='del'>-					   u16 len);</div><div class='del'>-</div><div class='del'>-struct i40iw_virtchnl_req {</div><div class='del'>-	struct i40iw_sc_dev *dev;</div><div class='del'>-	struct i40iw_virtchnl_op_buf *vchnl_msg;</div><div class='del'>-	void *parm;</div><div class='del'>-	u32 vf_id;</div><div class='del'>-	u16 parm_len;</div><div class='del'>-	s16 ret_code;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#pragma pack(pop)</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_vchnl_vf_get_ver(struct i40iw_sc_dev *dev,</div><div class='del'>-					      u32 *vchnl_ver);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_vchnl_vf_get_hmc_fcn(struct i40iw_sc_dev *dev,</div><div class='del'>-						  u16 *hmc_fcn);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_vchnl_vf_add_hmc_objs(struct i40iw_sc_dev *dev,</div><div class='del'>-						   enum i40iw_hmc_rsrc_type rsrc_type,</div><div class='del'>-						   u32 start_index,</div><div class='del'>-						   u32 rsrc_count);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_vchnl_vf_del_hmc_obj(struct i40iw_sc_dev *dev,</div><div class='del'>-						  enum i40iw_hmc_rsrc_type rsrc_type,</div><div class='del'>-						  u32 start_index,</div><div class='del'>-						  u32 rsrc_count);</div><div class='del'>-</div><div class='del'>-enum i40iw_status_code i40iw_vchnl_vf_get_pe_stats(struct i40iw_sc_dev *dev,</div><div class='del'>-						   struct i40iw_dev_hw_stats *hw_stats);</div><div class='del'>-#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/ionic/Kconfig b/drivers/infiniband/hw/ionic/Kconfig<br/>new file mode 100644<br/>index 000000000000..de6f10e9b6e9<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/ionic/Kconfig?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/ionic/Kconfig</a></div><div class='hunk'>@@ -0,0 +1,15 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0</div><div class='add'>+# Copyright (C) 2018-2025, Advanced Micro Devices, Inc.</div><div class='add'>+</div><div class='add'>+config INFINIBAND_IONIC</div><div class='add'>+	tristate "AMD Pensando DSC RDMA/RoCE Support"</div><div class='add'>+	depends on NETDEVICES &amp;&amp; ETHERNET &amp;&amp; PCI &amp;&amp; INET &amp;&amp; IONIC</div><div class='add'>+	help</div><div class='add'>+	  This enables RDMA/RoCE support for the AMD Pensando family of</div><div class='add'>+	  Distributed Services Cards (DSCs).</div><div class='add'>+</div><div class='add'>+	  To learn more, visit our website at</div><div class='add'>+	  &lt;https://www.amd.com/en/products/accelerators/pensando.html&gt;.</div><div class='add'>+</div><div class='add'>+	  To compile this driver as a module, choose M here. The module</div><div class='add'>+	  will be called ionic_rdma.</div><div class='head'>diff --git a/drivers/infiniband/hw/ionic/Makefile b/drivers/infiniband/hw/ionic/Makefile<br/>new file mode 100644<br/>index 000000000000..957973742820<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/ionic/Makefile?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/ionic/Makefile</a></div><div class='hunk'>@@ -0,0 +1,9 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0</div><div class='add'>+</div><div class='add'>+ccflags-y :=  -I $(srctree)/drivers/net/ethernet/pensando/ionic</div><div class='add'>+</div><div class='add'>+obj-$(CONFIG_INFINIBAND_IONIC)	+= ionic_rdma.o</div><div class='add'>+</div><div class='add'>+ionic_rdma-y :=	\</div><div class='add'>+	ionic_ibdev.o ionic_lif_cfg.o ionic_queue.o ionic_pgtbl.o ionic_admin.o \</div><div class='add'>+	ionic_controlpath.o ionic_datapath.o ionic_hw_stats.o</div><div class='head'>diff --git a/drivers/infiniband/hw/ionic/ionic_admin.c b/drivers/infiniband/hw/ionic/ionic_admin.c<br/>new file mode 100644<br/>index 000000000000..2537aa55d12d<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/ionic/ionic_admin.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/ionic/ionic_admin.c</a></div><div class='hunk'>@@ -0,0 +1,1229 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0</div><div class='add'>+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/interrupt.h&gt;</div><div class='add'>+#include &lt;linux/module.h&gt;</div><div class='add'>+#include &lt;linux/printk.h&gt;</div><div class='add'>+</div><div class='add'>+#include "ionic_fw.h"</div><div class='add'>+#include "ionic_ibdev.h"</div><div class='add'>+</div><div class='add'>+#define IONIC_EQ_COUNT_MIN	4</div><div class='add'>+#define IONIC_AQ_COUNT_MIN	1</div><div class='add'>+</div><div class='add'>+/* not a valid queue position or negative error status */</div><div class='add'>+#define IONIC_ADMIN_POSTED	0x10000</div><div class='add'>+</div><div class='add'>+/* cpu can be held with irq disabled for COUNT * MS  (for create/destroy_ah) */</div><div class='add'>+#define IONIC_ADMIN_BUSY_RETRY_COUNT	2000</div><div class='add'>+#define IONIC_ADMIN_BUSY_RETRY_MS	1</div><div class='add'>+</div><div class='add'>+/* admin queue will be considered failed if a command takes longer */</div><div class='add'>+#define IONIC_ADMIN_TIMEOUT	(HZ * 2)</div><div class='add'>+#define IONIC_ADMIN_WARN	(HZ / 8)</div><div class='add'>+</div><div class='add'>+/* will poll for admin cq to tolerate and report from missed event */</div><div class='add'>+#define IONIC_ADMIN_DELAY	(HZ / 8)</div><div class='add'>+</div><div class='add'>+/* work queue for polling the event queue and admin cq */</div><div class='add'>+struct workqueue_struct *ionic_evt_workq;</div><div class='add'>+</div><div class='add'>+static void ionic_admin_timedout(struct ionic_aq *aq)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = aq-&gt;dev;</div><div class='add'>+	unsigned long irqflags;</div><div class='add'>+	u16 pos;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;aq-&gt;lock, irqflags);</div><div class='add'>+	if (ionic_queue_empty(&amp;aq-&gt;q))</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	/* Reset ALL adminq if any one times out */</div><div class='add'>+	if (atomic_read(&amp;aq-&gt;admin_state) &lt; IONIC_ADMIN_KILLED)</div><div class='add'>+		queue_work(ionic_evt_workq, &amp;dev-&gt;reset_work);</div><div class='add'>+</div><div class='add'>+	ibdev_err(&amp;dev-&gt;ibdev, "admin command timed out, aq %d after: %ums\n",</div><div class='add'>+		  aq-&gt;aqid, (u32)jiffies_to_msecs(jiffies - aq-&gt;stamp));</div><div class='add'>+</div><div class='add'>+	pos = (aq-&gt;q.prod - 1) &amp; aq-&gt;q.mask;</div><div class='add'>+	if (pos == aq-&gt;q.cons)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	ibdev_warn(&amp;dev-&gt;ibdev, "admin pos %u (last posted)\n", pos);</div><div class='add'>+	print_hex_dump(KERN_WARNING, "cmd ", DUMP_PREFIX_OFFSET, 16, 1,</div><div class='add'>+		       ionic_queue_at(&amp;aq-&gt;q, pos),</div><div class='add'>+		       BIT(aq-&gt;q.stride_log2), true);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	spin_unlock_irqrestore(&amp;aq-&gt;lock, irqflags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_admin_reset_dwork(struct ionic_ibdev *dev)</div><div class='add'>+{</div><div class='add'>+	if (atomic_read(&amp;dev-&gt;admin_state) == IONIC_ADMIN_KILLED)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	queue_delayed_work(ionic_evt_workq, &amp;dev-&gt;admin_dwork,</div><div class='add'>+			   IONIC_ADMIN_DELAY);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_admin_reset_wdog(struct ionic_aq *aq)</div><div class='add'>+{</div><div class='add'>+	if (atomic_read(&amp;aq-&gt;admin_state) == IONIC_ADMIN_KILLED)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	aq-&gt;stamp = jiffies;</div><div class='add'>+	ionic_admin_reset_dwork(aq-&gt;dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool ionic_admin_next_cqe(struct ionic_ibdev *dev, struct ionic_cq *cq,</div><div class='add'>+				 struct ionic_v1_cqe **cqe)</div><div class='add'>+{</div><div class='add'>+	struct ionic_v1_cqe *qcqe = ionic_queue_at_prod(&amp;cq-&gt;q);</div><div class='add'>+</div><div class='add'>+	if (unlikely(cq-&gt;color != ionic_v1_cqe_color(qcqe)))</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	/* Prevent out-of-order reads of the CQE */</div><div class='add'>+	dma_rmb();</div><div class='add'>+	*cqe = qcqe;</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_admin_poll_locked(struct ionic_aq *aq)</div><div class='add'>+{</div><div class='add'>+	struct ionic_cq *cq = &amp;aq-&gt;vcq-&gt;cq[0];</div><div class='add'>+	struct ionic_admin_wr *wr, *wr_next;</div><div class='add'>+	struct ionic_ibdev *dev = aq-&gt;dev;</div><div class='add'>+	u32 wr_strides, avlbl_strides;</div><div class='add'>+	struct ionic_v1_cqe *cqe;</div><div class='add'>+	u32 qtf, qid;</div><div class='add'>+	u16 old_prod;</div><div class='add'>+	u8 type;</div><div class='add'>+</div><div class='add'>+	lockdep_assert_held(&amp;aq-&gt;lock);</div><div class='add'>+</div><div class='add'>+	if (atomic_read(&amp;aq-&gt;admin_state) == IONIC_ADMIN_KILLED) {</div><div class='add'>+		list_for_each_entry_safe(wr, wr_next, &amp;aq-&gt;wr_prod, aq_ent) {</div><div class='add'>+			INIT_LIST_HEAD(&amp;wr-&gt;aq_ent);</div><div class='add'>+			aq-&gt;q_wr[wr-&gt;status].wr = NULL;</div><div class='add'>+			wr-&gt;status = atomic_read(&amp;aq-&gt;admin_state);</div><div class='add'>+			complete_all(&amp;wr-&gt;work);</div><div class='add'>+		}</div><div class='add'>+		INIT_LIST_HEAD(&amp;aq-&gt;wr_prod);</div><div class='add'>+</div><div class='add'>+		list_for_each_entry_safe(wr, wr_next, &amp;aq-&gt;wr_post, aq_ent) {</div><div class='add'>+			INIT_LIST_HEAD(&amp;wr-&gt;aq_ent);</div><div class='add'>+			wr-&gt;status = atomic_read(&amp;aq-&gt;admin_state);</div><div class='add'>+			complete_all(&amp;wr-&gt;work);</div><div class='add'>+		}</div><div class='add'>+		INIT_LIST_HEAD(&amp;aq-&gt;wr_post);</div><div class='add'>+</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	old_prod = cq-&gt;q.prod;</div><div class='add'>+</div><div class='add'>+	while (ionic_admin_next_cqe(dev, cq, &amp;cqe)) {</div><div class='add'>+		qtf = ionic_v1_cqe_qtf(cqe);</div><div class='add'>+		qid = ionic_v1_cqe_qtf_qid(qtf);</div><div class='add'>+		type = ionic_v1_cqe_qtf_type(qtf);</div><div class='add'>+</div><div class='add'>+		if (unlikely(type != IONIC_V1_CQE_TYPE_ADMIN)) {</div><div class='add'>+			ibdev_warn_ratelimited(&amp;dev-&gt;ibdev,</div><div class='add'>+					       "bad cqe type %u\n", type);</div><div class='add'>+			goto cq_next;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (unlikely(qid != aq-&gt;aqid)) {</div><div class='add'>+			ibdev_warn_ratelimited(&amp;dev-&gt;ibdev,</div><div class='add'>+					       "bad cqe qid %u\n", qid);</div><div class='add'>+			goto cq_next;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (unlikely(be16_to_cpu(cqe-&gt;admin.cmd_idx) != aq-&gt;q.cons)) {</div><div class='add'>+			ibdev_warn_ratelimited(&amp;dev-&gt;ibdev,</div><div class='add'>+					       "bad idx %u cons %u qid %u\n",</div><div class='add'>+					       be16_to_cpu(cqe-&gt;admin.cmd_idx),</div><div class='add'>+					       aq-&gt;q.cons, qid);</div><div class='add'>+			goto cq_next;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (unlikely(ionic_queue_empty(&amp;aq-&gt;q))) {</div><div class='add'>+			ibdev_warn_ratelimited(&amp;dev-&gt;ibdev,</div><div class='add'>+					       "bad cqe for empty adminq\n");</div><div class='add'>+			goto cq_next;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		wr = aq-&gt;q_wr[aq-&gt;q.cons].wr;</div><div class='add'>+		if (wr) {</div><div class='add'>+			aq-&gt;q_wr[aq-&gt;q.cons].wr = NULL;</div><div class='add'>+			list_del_init(&amp;wr-&gt;aq_ent);</div><div class='add'>+</div><div class='add'>+			wr-&gt;cqe = *cqe;</div><div class='add'>+			wr-&gt;status = atomic_read(&amp;aq-&gt;admin_state);</div><div class='add'>+			complete_all(&amp;wr-&gt;work);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ionic_queue_consume_entries(&amp;aq-&gt;q,</div><div class='add'>+					    aq-&gt;q_wr[aq-&gt;q.cons].wqe_strides);</div><div class='add'>+</div><div class='add'>+cq_next:</div><div class='add'>+		ionic_queue_produce(&amp;cq-&gt;q);</div><div class='add'>+		cq-&gt;color = ionic_color_wrap(cq-&gt;q.prod, cq-&gt;color);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (old_prod != cq-&gt;q.prod) {</div><div class='add'>+		ionic_admin_reset_wdog(aq);</div><div class='add'>+		cq-&gt;q.cons = cq-&gt;q.prod;</div><div class='add'>+		ionic_dbell_ring(dev-&gt;lif_cfg.dbpage, dev-&gt;lif_cfg.cq_qtype,</div><div class='add'>+				 ionic_queue_dbell_val(&amp;cq-&gt;q));</div><div class='add'>+		queue_work(ionic_evt_workq, &amp;aq-&gt;work);</div><div class='add'>+	} else if (!aq-&gt;armed) {</div><div class='add'>+		aq-&gt;armed = true;</div><div class='add'>+		cq-&gt;arm_any_prod = ionic_queue_next(&amp;cq-&gt;q, cq-&gt;arm_any_prod);</div><div class='add'>+		ionic_dbell_ring(dev-&gt;lif_cfg.dbpage, dev-&gt;lif_cfg.cq_qtype,</div><div class='add'>+				 cq-&gt;q.dbell | IONIC_CQ_RING_ARM |</div><div class='add'>+				 cq-&gt;arm_any_prod);</div><div class='add'>+		queue_work(ionic_evt_workq, &amp;aq-&gt;work);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (atomic_read(&amp;aq-&gt;admin_state) != IONIC_ADMIN_ACTIVE)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	old_prod = aq-&gt;q.prod;</div><div class='add'>+</div><div class='add'>+	if (ionic_queue_empty(&amp;aq-&gt;q) &amp;&amp; !list_empty(&amp;aq-&gt;wr_post))</div><div class='add'>+		ionic_admin_reset_wdog(aq);</div><div class='add'>+</div><div class='add'>+	if (list_empty(&amp;aq-&gt;wr_post))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		u8 *src;</div><div class='add'>+		int i, src_len;</div><div class='add'>+		size_t stride_len;</div><div class='add'>+</div><div class='add'>+		wr = list_first_entry(&amp;aq-&gt;wr_post, struct ionic_admin_wr,</div><div class='add'>+				      aq_ent);</div><div class='add'>+		wr_strides = (le16_to_cpu(wr-&gt;wqe.len) + ADMIN_WQE_HDR_LEN +</div><div class='add'>+			     (ADMIN_WQE_STRIDE - 1)) &gt;&gt; aq-&gt;q.stride_log2;</div><div class='add'>+		avlbl_strides = ionic_queue_length_remaining(&amp;aq-&gt;q);</div><div class='add'>+</div><div class='add'>+		if (wr_strides &gt; avlbl_strides)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		list_move(&amp;wr-&gt;aq_ent, &amp;aq-&gt;wr_prod);</div><div class='add'>+		wr-&gt;status = aq-&gt;q.prod;</div><div class='add'>+		aq-&gt;q_wr[aq-&gt;q.prod].wr = wr;</div><div class='add'>+		aq-&gt;q_wr[aq-&gt;q.prod].wqe_strides = wr_strides;</div><div class='add'>+</div><div class='add'>+		src_len = le16_to_cpu(wr-&gt;wqe.len);</div><div class='add'>+		src = (uint8_t *)&amp;wr-&gt;wqe.cmd;</div><div class='add'>+</div><div class='add'>+		/* First stride */</div><div class='add'>+		memcpy(ionic_queue_at_prod(&amp;aq-&gt;q), &amp;wr-&gt;wqe,</div><div class='add'>+		       ADMIN_WQE_HDR_LEN);</div><div class='add'>+		stride_len = ADMIN_WQE_STRIDE - ADMIN_WQE_HDR_LEN;</div><div class='add'>+		if (stride_len &gt; src_len)</div><div class='add'>+			stride_len = src_len;</div><div class='add'>+		memcpy(ionic_queue_at_prod(&amp;aq-&gt;q) + ADMIN_WQE_HDR_LEN,</div><div class='add'>+		       src, stride_len);</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev, "post admin prod %u (%u strides)\n",</div><div class='add'>+			  aq-&gt;q.prod, wr_strides);</div><div class='add'>+		print_hex_dump_debug("wqe ", DUMP_PREFIX_OFFSET, 16, 1,</div><div class='add'>+				     ionic_queue_at_prod(&amp;aq-&gt;q),</div><div class='add'>+				     BIT(aq-&gt;q.stride_log2), true);</div><div class='add'>+		ionic_queue_produce(&amp;aq-&gt;q);</div><div class='add'>+</div><div class='add'>+		/* Remaining strides */</div><div class='add'>+		for (i = stride_len; i &lt; src_len; i += stride_len) {</div><div class='add'>+			stride_len = ADMIN_WQE_STRIDE;</div><div class='add'>+</div><div class='add'>+			if (i + stride_len &gt; src_len)</div><div class='add'>+				stride_len = src_len - i;</div><div class='add'>+</div><div class='add'>+			memcpy(ionic_queue_at_prod(&amp;aq-&gt;q), src + i,</div><div class='add'>+			       stride_len);</div><div class='add'>+			print_hex_dump_debug("wqe ", DUMP_PREFIX_OFFSET, 16, 1,</div><div class='add'>+					     ionic_queue_at_prod(&amp;aq-&gt;q),</div><div class='add'>+					     BIT(aq-&gt;q.stride_log2), true);</div><div class='add'>+			ionic_queue_produce(&amp;aq-&gt;q);</div><div class='add'>+		}</div><div class='add'>+	} while (!list_empty(&amp;aq-&gt;wr_post));</div><div class='add'>+</div><div class='add'>+	if (old_prod != aq-&gt;q.prod)</div><div class='add'>+		ionic_dbell_ring(dev-&gt;lif_cfg.dbpage, dev-&gt;lif_cfg.aq_qtype,</div><div class='add'>+				 ionic_queue_dbell_val(&amp;aq-&gt;q));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_admin_dwork(struct work_struct *ws)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev =</div><div class='add'>+		container_of(ws, struct ionic_ibdev, admin_dwork.work);</div><div class='add'>+	struct ionic_aq *aq, *bad_aq = NULL;</div><div class='add'>+	bool do_reschedule = false;</div><div class='add'>+	unsigned long irqflags;</div><div class='add'>+	bool do_reset = false;</div><div class='add'>+	u16 pos;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; dev-&gt;lif_cfg.aq_count; i++) {</div><div class='add'>+		aq = dev-&gt;aq_vec[i];</div><div class='add'>+</div><div class='add'>+		spin_lock_irqsave(&amp;aq-&gt;lock, irqflags);</div><div class='add'>+</div><div class='add'>+		if (ionic_queue_empty(&amp;aq-&gt;q))</div><div class='add'>+			goto next_aq;</div><div class='add'>+</div><div class='add'>+		/* Reschedule if any queue has outstanding work */</div><div class='add'>+		do_reschedule = true;</div><div class='add'>+</div><div class='add'>+		if (time_is_after_eq_jiffies(aq-&gt;stamp + IONIC_ADMIN_WARN))</div><div class='add'>+			/* Warning threshold not met, nothing to do */</div><div class='add'>+			goto next_aq;</div><div class='add'>+</div><div class='add'>+		/* See if polling now makes some progress */</div><div class='add'>+		pos = aq-&gt;q.cons;</div><div class='add'>+		ionic_admin_poll_locked(aq);</div><div class='add'>+		if (pos != aq-&gt;q.cons) {</div><div class='add'>+			ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='add'>+				  "missed event for acq %d\n", aq-&gt;cqid);</div><div class='add'>+			goto next_aq;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (time_is_after_eq_jiffies(aq-&gt;stamp +</div><div class='add'>+					     IONIC_ADMIN_TIMEOUT)) {</div><div class='add'>+			/* Timeout threshold not met */</div><div class='add'>+			ibdev_dbg(&amp;dev-&gt;ibdev, "no progress after %ums\n",</div><div class='add'>+				  (u32)jiffies_to_msecs(jiffies - aq-&gt;stamp));</div><div class='add'>+			goto next_aq;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		/* Queue timed out */</div><div class='add'>+		bad_aq = aq;</div><div class='add'>+		do_reset = true;</div><div class='add'>+next_aq:</div><div class='add'>+		spin_unlock_irqrestore(&amp;aq-&gt;lock, irqflags);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (do_reset)</div><div class='add'>+		/* Reset RDMA lif on a timeout */</div><div class='add'>+		ionic_admin_timedout(bad_aq);</div><div class='add'>+	else if (do_reschedule)</div><div class='add'>+		/* Try to poll again later */</div><div class='add'>+		ionic_admin_reset_dwork(dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_admin_work(struct work_struct *ws)</div><div class='add'>+{</div><div class='add'>+	struct ionic_aq *aq = container_of(ws, struct ionic_aq, work);</div><div class='add'>+	unsigned long irqflags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;aq-&gt;lock, irqflags);</div><div class='add'>+	ionic_admin_poll_locked(aq);</div><div class='add'>+	spin_unlock_irqrestore(&amp;aq-&gt;lock, irqflags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_admin_post_aq(struct ionic_aq *aq, struct ionic_admin_wr *wr)</div><div class='add'>+{</div><div class='add'>+	unsigned long irqflags;</div><div class='add'>+	bool poll;</div><div class='add'>+</div><div class='add'>+	wr-&gt;status = IONIC_ADMIN_POSTED;</div><div class='add'>+	wr-&gt;aq = aq;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;aq-&gt;lock, irqflags);</div><div class='add'>+	poll = list_empty(&amp;aq-&gt;wr_post);</div><div class='add'>+	list_add(&amp;wr-&gt;aq_ent, &amp;aq-&gt;wr_post);</div><div class='add'>+	if (poll)</div><div class='add'>+		ionic_admin_poll_locked(aq);</div><div class='add'>+	spin_unlock_irqrestore(&amp;aq-&gt;lock, irqflags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr)</div><div class='add'>+{</div><div class='add'>+	int aq_idx;</div><div class='add'>+</div><div class='add'>+	/* Use cpu id for the adminq selection */</div><div class='add'>+	aq_idx = raw_smp_processor_id() % dev-&gt;lif_cfg.aq_count;</div><div class='add'>+	ionic_admin_post_aq(dev-&gt;aq_vec[aq_idx], wr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_admin_cancel(struct ionic_admin_wr *wr)</div><div class='add'>+{</div><div class='add'>+	struct ionic_aq *aq = wr-&gt;aq;</div><div class='add'>+	unsigned long irqflags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;aq-&gt;lock, irqflags);</div><div class='add'>+</div><div class='add'>+	if (!list_empty(&amp;wr-&gt;aq_ent)) {</div><div class='add'>+		list_del(&amp;wr-&gt;aq_ent);</div><div class='add'>+		if (wr-&gt;status != IONIC_ADMIN_POSTED)</div><div class='add'>+			aq-&gt;q_wr[wr-&gt;status].wr = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_unlock_irqrestore(&amp;aq-&gt;lock, irqflags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_admin_busy_wait(struct ionic_admin_wr *wr)</div><div class='add'>+{</div><div class='add'>+	struct ionic_aq *aq = wr-&gt;aq;</div><div class='add'>+	unsigned long irqflags;</div><div class='add'>+	int try_i;</div><div class='add'>+</div><div class='add'>+	for (try_i = 0; try_i &lt; IONIC_ADMIN_BUSY_RETRY_COUNT; ++try_i) {</div><div class='add'>+		if (completion_done(&amp;wr-&gt;work))</div><div class='add'>+			return 0;</div><div class='add'>+</div><div class='add'>+		mdelay(IONIC_ADMIN_BUSY_RETRY_MS);</div><div class='add'>+</div><div class='add'>+		spin_lock_irqsave(&amp;aq-&gt;lock, irqflags);</div><div class='add'>+		ionic_admin_poll_locked(aq);</div><div class='add'>+		spin_unlock_irqrestore(&amp;aq-&gt;lock, irqflags);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * we timed out. Initiate RDMA LIF reset and indicate</div><div class='add'>+	 * error to caller.</div><div class='add'>+	 */</div><div class='add'>+	ionic_admin_timedout(aq);</div><div class='add'>+	return -ETIMEDOUT;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr,</div><div class='add'>+		     enum ionic_admin_flags flags)</div><div class='add'>+{</div><div class='add'>+	int rc, timo;</div><div class='add'>+</div><div class='add'>+	if (flags &amp; IONIC_ADMIN_F_BUSYWAIT) {</div><div class='add'>+		/* Spin */</div><div class='add'>+		rc = ionic_admin_busy_wait(wr);</div><div class='add'>+	} else if (flags &amp; IONIC_ADMIN_F_INTERRUPT) {</div><div class='add'>+		/*</div><div class='add'>+		 * Interruptible sleep, 1s timeout</div><div class='add'>+		 * This is used for commands which are safe for the caller</div><div class='add'>+		 * to clean up without killing and resetting the adminq.</div><div class='add'>+		 */</div><div class='add'>+		timo = wait_for_completion_interruptible_timeout(&amp;wr-&gt;work,</div><div class='add'>+								 HZ);</div><div class='add'>+		if (timo &gt; 0)</div><div class='add'>+			rc = 0;</div><div class='add'>+		else if (timo == 0)</div><div class='add'>+			rc = -ETIMEDOUT;</div><div class='add'>+		else</div><div class='add'>+			rc = timo;</div><div class='add'>+	} else {</div><div class='add'>+		/*</div><div class='add'>+		 * Uninterruptible sleep</div><div class='add'>+		 * This is used for commands which are NOT safe for the</div><div class='add'>+		 * caller to clean up. Cleanup must be handled by the</div><div class='add'>+		 * adminq kill and reset process so that host memory is</div><div class='add'>+		 * not corrupted by the device.</div><div class='add'>+		 */</div><div class='add'>+		wait_for_completion(&amp;wr-&gt;work);</div><div class='add'>+		rc = 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (rc) {</div><div class='add'>+		ibdev_warn(&amp;dev-&gt;ibdev, "wait status %d\n", rc);</div><div class='add'>+		ionic_admin_cancel(wr);</div><div class='add'>+	} else if (wr-&gt;status == IONIC_ADMIN_KILLED) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev, "admin killed\n");</div><div class='add'>+</div><div class='add'>+		/* No error if admin already killed during teardown */</div><div class='add'>+		rc = (flags &amp; IONIC_ADMIN_F_TEARDOWN) ? 0 : -ENODEV;</div><div class='add'>+	} else if (ionic_v1_cqe_error(&amp;wr-&gt;cqe)) {</div><div class='add'>+		ibdev_warn(&amp;dev-&gt;ibdev, "opcode %u error %u\n",</div><div class='add'>+			   wr-&gt;wqe.op,</div><div class='add'>+			   be32_to_cpu(wr-&gt;cqe.status_length));</div><div class='add'>+		rc = -EINVAL;</div><div class='add'>+	}</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_rdma_devcmd(struct ionic_ibdev *dev,</div><div class='add'>+			     struct ionic_admin_ctx *admin)</div><div class='add'>+{</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rc = ionic_adminq_post_wait(dev-&gt;lif_cfg.lif, admin);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	return ionic_error_to_errno(admin-&gt;comp.comp.status);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev)</div><div class='add'>+{</div><div class='add'>+	struct ionic_admin_ctx admin = {</div><div class='add'>+		.work = COMPLETION_INITIALIZER_ONSTACK(admin.work),</div><div class='add'>+		.cmd.rdma_reset = {</div><div class='add'>+			.opcode = IONIC_CMD_RDMA_RESET_LIF,</div><div class='add'>+			.lif_index = cpu_to_le16(dev-&gt;lif_cfg.lif_index),</div><div class='add'>+		},</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	return ionic_rdma_devcmd(dev, &amp;admin);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_rdma_queue_devcmd(struct ionic_ibdev *dev,</div><div class='add'>+				   struct ionic_queue *q,</div><div class='add'>+				   u32 qid, u32 cid, u16 opcode)</div><div class='add'>+{</div><div class='add'>+	struct ionic_admin_ctx admin = {</div><div class='add'>+		.work = COMPLETION_INITIALIZER_ONSTACK(admin.work),</div><div class='add'>+		.cmd.rdma_queue = {</div><div class='add'>+			.opcode = opcode,</div><div class='add'>+			.lif_index = cpu_to_le16(dev-&gt;lif_cfg.lif_index),</div><div class='add'>+			.qid_ver = cpu_to_le32(qid),</div><div class='add'>+			.cid = cpu_to_le32(cid),</div><div class='add'>+			.dbid = cpu_to_le16(dev-&gt;lif_cfg.dbid),</div><div class='add'>+			.depth_log2 = q-&gt;depth_log2,</div><div class='add'>+			.stride_log2 = q-&gt;stride_log2,</div><div class='add'>+			.dma_addr = cpu_to_le64(q-&gt;dma),</div><div class='add'>+		},</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	return ionic_rdma_devcmd(dev, &amp;admin);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_rdma_admincq_comp(struct ib_cq *ibcq, void *cq_context)</div><div class='add'>+{</div><div class='add'>+	struct ionic_aq *aq = cq_context;</div><div class='add'>+	unsigned long irqflags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;aq-&gt;lock, irqflags);</div><div class='add'>+	aq-&gt;armed = false;</div><div class='add'>+	if (atomic_read(&amp;aq-&gt;admin_state) &lt; IONIC_ADMIN_KILLED)</div><div class='add'>+		queue_work(ionic_evt_workq, &amp;aq-&gt;work);</div><div class='add'>+	spin_unlock_irqrestore(&amp;aq-&gt;lock, irqflags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_rdma_admincq_event(struct ib_event *event, void *cq_context)</div><div class='add'>+{</div><div class='add'>+	struct ionic_aq *aq = cq_context;</div><div class='add'>+</div><div class='add'>+	ibdev_err(&amp;aq-&gt;dev-&gt;ibdev, "admincq event %d\n", event-&gt;event);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct ionic_vcq *ionic_create_rdma_admincq(struct ionic_ibdev *dev,</div><div class='add'>+						   int comp_vector)</div><div class='add'>+{</div><div class='add'>+	struct ib_cq_init_attr attr = {</div><div class='add'>+		.cqe = IONIC_AQ_DEPTH,</div><div class='add'>+		.comp_vector = comp_vector,</div><div class='add'>+	};</div><div class='add'>+	struct ionic_tbl_buf buf = {};</div><div class='add'>+	struct ionic_vcq *vcq;</div><div class='add'>+	struct ionic_cq *cq;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	vcq = kzalloc(sizeof(*vcq), GFP_KERNEL);</div><div class='add'>+	if (!vcq)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	vcq-&gt;ibcq.device = &amp;dev-&gt;ibdev;</div><div class='add'>+	vcq-&gt;ibcq.comp_handler = ionic_rdma_admincq_comp;</div><div class='add'>+	vcq-&gt;ibcq.event_handler = ionic_rdma_admincq_event;</div><div class='add'>+	atomic_set(&amp;vcq-&gt;ibcq.usecnt, 0);</div><div class='add'>+</div><div class='add'>+	vcq-&gt;udma_mask = 1;</div><div class='add'>+	cq = &amp;vcq-&gt;cq[0];</div><div class='add'>+</div><div class='add'>+	rc = ionic_create_cq_common(vcq, &amp;buf, &amp;attr, NULL, NULL,</div><div class='add'>+				    NULL, NULL, 0);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_init;</div><div class='add'>+</div><div class='add'>+	rc = ionic_rdma_queue_devcmd(dev, &amp;cq-&gt;q, cq-&gt;cqid, cq-&gt;eqid,</div><div class='add'>+				     IONIC_CMD_RDMA_CREATE_CQ);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_cmd;</div><div class='add'>+</div><div class='add'>+	return vcq;</div><div class='add'>+</div><div class='add'>+err_cmd:</div><div class='add'>+	ionic_destroy_cq_common(dev, cq);</div><div class='add'>+err_init:</div><div class='add'>+	kfree(vcq);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(rc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct ionic_aq *__ionic_create_rdma_adminq(struct ionic_ibdev *dev,</div><div class='add'>+						   u32 aqid, u32 cqid)</div><div class='add'>+{</div><div class='add'>+	struct ionic_aq *aq;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	aq = kzalloc(sizeof(*aq), GFP_KERNEL);</div><div class='add'>+	if (!aq)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	atomic_set(&amp;aq-&gt;admin_state, IONIC_ADMIN_KILLED);</div><div class='add'>+	aq-&gt;dev = dev;</div><div class='add'>+	aq-&gt;aqid = aqid;</div><div class='add'>+	aq-&gt;cqid = cqid;</div><div class='add'>+	spin_lock_init(&amp;aq-&gt;lock);</div><div class='add'>+</div><div class='add'>+	rc = ionic_queue_init(&amp;aq-&gt;q, dev-&gt;lif_cfg.hwdev, IONIC_EQ_DEPTH,</div><div class='add'>+			      ADMIN_WQE_STRIDE);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_q;</div><div class='add'>+</div><div class='add'>+	ionic_queue_dbell_init(&amp;aq-&gt;q, aq-&gt;aqid);</div><div class='add'>+</div><div class='add'>+	aq-&gt;q_wr = kcalloc((u32)aq-&gt;q.mask + 1, sizeof(*aq-&gt;q_wr), GFP_KERNEL);</div><div class='add'>+	if (!aq-&gt;q_wr) {</div><div class='add'>+		rc = -ENOMEM;</div><div class='add'>+		goto err_wr;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;aq-&gt;wr_prod);</div><div class='add'>+	INIT_LIST_HEAD(&amp;aq-&gt;wr_post);</div><div class='add'>+</div><div class='add'>+	INIT_WORK(&amp;aq-&gt;work, ionic_admin_work);</div><div class='add'>+	aq-&gt;armed = false;</div><div class='add'>+</div><div class='add'>+	return aq;</div><div class='add'>+</div><div class='add'>+err_wr:</div><div class='add'>+	ionic_queue_destroy(&amp;aq-&gt;q, dev-&gt;lif_cfg.hwdev);</div><div class='add'>+err_q:</div><div class='add'>+	kfree(aq);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(rc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void __ionic_destroy_rdma_adminq(struct ionic_ibdev *dev,</div><div class='add'>+					struct ionic_aq *aq)</div><div class='add'>+{</div><div class='add'>+	kfree(aq-&gt;q_wr);</div><div class='add'>+	ionic_queue_destroy(&amp;aq-&gt;q, dev-&gt;lif_cfg.hwdev);</div><div class='add'>+	kfree(aq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct ionic_aq *ionic_create_rdma_adminq(struct ionic_ibdev *dev,</div><div class='add'>+						 u32 aqid, u32 cqid)</div><div class='add'>+{</div><div class='add'>+	struct ionic_aq *aq;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	aq = __ionic_create_rdma_adminq(dev, aqid, cqid);</div><div class='add'>+	if (IS_ERR(aq))</div><div class='add'>+		return aq;</div><div class='add'>+</div><div class='add'>+	rc = ionic_rdma_queue_devcmd(dev, &amp;aq-&gt;q, aq-&gt;aqid, aq-&gt;cqid,</div><div class='add'>+				     IONIC_CMD_RDMA_CREATE_ADMINQ);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_cmd;</div><div class='add'>+</div><div class='add'>+	return aq;</div><div class='add'>+</div><div class='add'>+err_cmd:</div><div class='add'>+	__ionic_destroy_rdma_adminq(dev, aq);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(rc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_flush_qs(struct ionic_ibdev *dev)</div><div class='add'>+{</div><div class='add'>+	struct ionic_qp *qp, *qp_tmp;</div><div class='add'>+	struct ionic_cq *cq, *cq_tmp;</div><div class='add'>+	LIST_HEAD(flush_list);</div><div class='add'>+	unsigned long index;</div><div class='add'>+</div><div class='add'>+	WARN_ON(!irqs_disabled());</div><div class='add'>+</div><div class='add'>+	/* Flush qp send and recv */</div><div class='add'>+	xa_lock(&amp;dev-&gt;qp_tbl);</div><div class='add'>+	xa_for_each(&amp;dev-&gt;qp_tbl, index, qp) {</div><div class='add'>+		kref_get(&amp;qp-&gt;qp_kref);</div><div class='add'>+		list_add_tail(&amp;qp-&gt;ibkill_flush_ent, &amp;flush_list);</div><div class='add'>+	}</div><div class='add'>+	xa_unlock(&amp;dev-&gt;qp_tbl);</div><div class='add'>+</div><div class='add'>+	list_for_each_entry_safe(qp, qp_tmp, &amp;flush_list, ibkill_flush_ent) {</div><div class='add'>+		ionic_flush_qp(dev, qp);</div><div class='add'>+		kref_put(&amp;qp-&gt;qp_kref, ionic_qp_complete);</div><div class='add'>+		list_del(&amp;qp-&gt;ibkill_flush_ent);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Notify completions */</div><div class='add'>+	xa_lock(&amp;dev-&gt;cq_tbl);</div><div class='add'>+	xa_for_each(&amp;dev-&gt;cq_tbl, index, cq) {</div><div class='add'>+		kref_get(&amp;cq-&gt;cq_kref);</div><div class='add'>+		list_add_tail(&amp;cq-&gt;ibkill_flush_ent, &amp;flush_list);</div><div class='add'>+	}</div><div class='add'>+	xa_unlock(&amp;dev-&gt;cq_tbl);</div><div class='add'>+</div><div class='add'>+	list_for_each_entry_safe(cq, cq_tmp, &amp;flush_list, ibkill_flush_ent) {</div><div class='add'>+		ionic_notify_flush_cq(cq);</div><div class='add'>+		kref_put(&amp;cq-&gt;cq_kref, ionic_cq_complete);</div><div class='add'>+		list_del(&amp;cq-&gt;ibkill_flush_ent);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_kill_ibdev(struct ionic_ibdev *dev, bool fatal_path)</div><div class='add'>+{</div><div class='add'>+	unsigned long irqflags;</div><div class='add'>+	bool do_flush = false;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	/* Mark AQs for drain and flush the QPs while irq is disabled */</div><div class='add'>+	local_irq_save(irqflags);</div><div class='add'>+</div><div class='add'>+	/* Mark the admin queue, flushing at most once */</div><div class='add'>+	for (i = 0; i &lt; dev-&gt;lif_cfg.aq_count; i++) {</div><div class='add'>+		struct ionic_aq *aq = dev-&gt;aq_vec[i];</div><div class='add'>+</div><div class='add'>+		spin_lock(&amp;aq-&gt;lock);</div><div class='add'>+		if (atomic_read(&amp;aq-&gt;admin_state) != IONIC_ADMIN_KILLED) {</div><div class='add'>+			atomic_set(&amp;aq-&gt;admin_state, IONIC_ADMIN_KILLED);</div><div class='add'>+			/* Flush incomplete admin commands */</div><div class='add'>+			ionic_admin_poll_locked(aq);</div><div class='add'>+			do_flush = true;</div><div class='add'>+		}</div><div class='add'>+		spin_unlock(&amp;aq-&gt;lock);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (do_flush)</div><div class='add'>+		ionic_flush_qs(dev);</div><div class='add'>+</div><div class='add'>+	local_irq_restore(irqflags);</div><div class='add'>+</div><div class='add'>+	/* Post a fatal event if requested */</div><div class='add'>+	if (fatal_path) {</div><div class='add'>+		struct ib_event ev;</div><div class='add'>+</div><div class='add'>+		ev.device = &amp;dev-&gt;ibdev;</div><div class='add'>+		ev.element.port_num = 1;</div><div class='add'>+		ev.event = IB_EVENT_DEVICE_FATAL;</div><div class='add'>+</div><div class='add'>+		ib_dispatch_event(&amp;ev);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	atomic_set(&amp;dev-&gt;admin_state, IONIC_ADMIN_KILLED);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path)</div><div class='add'>+{</div><div class='add'>+	enum ionic_admin_state old_state;</div><div class='add'>+	unsigned long irqflags = 0;</div><div class='add'>+	int i, rc;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;aq_vec)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Admin queues are transitioned from active to paused to killed state.</div><div class='add'>+	 * When in paused state, no new commands are issued to the device,</div><div class='add'>+	 * nor are any completed locally. After resetting the lif, it will be</div><div class='add'>+	 * safe to resume the rdma admin queues in the killed state. Commands</div><div class='add'>+	 * will not be issued to the device, but will complete locally with status</div><div class='add'>+	 * IONIC_ADMIN_KILLED. Handling completion will ensure that creating or</div><div class='add'>+	 * modifying resources fails, but destroying resources succeeds.</div><div class='add'>+	 * If there was a failure resetting the lif using this strategy,</div><div class='add'>+	 * then the state of the device is unknown.</div><div class='add'>+	 */</div><div class='add'>+	old_state = atomic_cmpxchg(&amp;dev-&gt;admin_state, IONIC_ADMIN_ACTIVE,</div><div class='add'>+				   IONIC_ADMIN_PAUSED);</div><div class='add'>+	if (old_state != IONIC_ADMIN_ACTIVE)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	/* Pause all the AQs */</div><div class='add'>+	local_irq_save(irqflags);</div><div class='add'>+	for (i = 0; i &lt; dev-&gt;lif_cfg.aq_count; i++) {</div><div class='add'>+		struct ionic_aq *aq = dev-&gt;aq_vec[i];</div><div class='add'>+</div><div class='add'>+		spin_lock(&amp;aq-&gt;lock);</div><div class='add'>+		/* pause rdma admin queues to reset lif */</div><div class='add'>+		if (atomic_read(&amp;aq-&gt;admin_state) == IONIC_ADMIN_ACTIVE)</div><div class='add'>+			atomic_set(&amp;aq-&gt;admin_state, IONIC_ADMIN_PAUSED);</div><div class='add'>+		spin_unlock(&amp;aq-&gt;lock);</div><div class='add'>+	}</div><div class='add'>+	local_irq_restore(irqflags);</div><div class='add'>+</div><div class='add'>+	rc = ionic_rdma_reset_devcmd(dev);</div><div class='add'>+	if (unlikely(rc)) {</div><div class='add'>+		ibdev_err(&amp;dev-&gt;ibdev, "failed to reset rdma %d\n", rc);</div><div class='add'>+		ionic_request_rdma_reset(dev-&gt;lif_cfg.lif);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ionic_kill_ibdev(dev, fatal_path);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_reset_work(struct work_struct *ws)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev =</div><div class='add'>+		container_of(ws, struct ionic_ibdev, reset_work);</div><div class='add'>+</div><div class='add'>+	ionic_kill_rdma_admin(dev, true);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool ionic_next_eqe(struct ionic_eq *eq, struct ionic_v1_eqe *eqe)</div><div class='add'>+{</div><div class='add'>+	struct ionic_v1_eqe *qeqe;</div><div class='add'>+	bool color;</div><div class='add'>+</div><div class='add'>+	qeqe = ionic_queue_at_prod(&amp;eq-&gt;q);</div><div class='add'>+	color = ionic_v1_eqe_color(qeqe);</div><div class='add'>+</div><div class='add'>+	/* cons is color for eq */</div><div class='add'>+	if (eq-&gt;q.cons != color)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	/* Prevent out-of-order reads of the EQE */</div><div class='add'>+	dma_rmb();</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;eq-&gt;dev-&gt;ibdev, "poll eq prod %u\n", eq-&gt;q.prod);</div><div class='add'>+	print_hex_dump_debug("eqe ", DUMP_PREFIX_OFFSET, 16, 1,</div><div class='add'>+			     qeqe, BIT(eq-&gt;q.stride_log2), true);</div><div class='add'>+	*eqe = *qeqe;</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_cq_event(struct ionic_ibdev *dev, u32 cqid, u8 code)</div><div class='add'>+{</div><div class='add'>+	unsigned long irqflags;</div><div class='add'>+	struct ib_event ibev;</div><div class='add'>+	struct ionic_cq *cq;</div><div class='add'>+</div><div class='add'>+	xa_lock_irqsave(&amp;dev-&gt;cq_tbl, irqflags);</div><div class='add'>+	cq = xa_load(&amp;dev-&gt;cq_tbl, cqid);</div><div class='add'>+	if (cq)</div><div class='add'>+		kref_get(&amp;cq-&gt;cq_kref);</div><div class='add'>+	xa_unlock_irqrestore(&amp;dev-&gt;cq_tbl, irqflags);</div><div class='add'>+</div><div class='add'>+	if (!cq) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='add'>+			  "missing cqid %#x code %u\n", cqid, code);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (code) {</div><div class='add'>+	case IONIC_V1_EQE_CQ_NOTIFY:</div><div class='add'>+		if (cq-&gt;vcq-&gt;ibcq.comp_handler)</div><div class='add'>+			cq-&gt;vcq-&gt;ibcq.comp_handler(&amp;cq-&gt;vcq-&gt;ibcq,</div><div class='add'>+						   cq-&gt;vcq-&gt;ibcq.cq_context);</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	case IONIC_V1_EQE_CQ_ERR:</div><div class='add'>+		if (cq-&gt;vcq-&gt;ibcq.event_handler) {</div><div class='add'>+			ibev.event = IB_EVENT_CQ_ERR;</div><div class='add'>+			ibev.device = &amp;dev-&gt;ibdev;</div><div class='add'>+			ibev.element.cq = &amp;cq-&gt;vcq-&gt;ibcq;</div><div class='add'>+</div><div class='add'>+			cq-&gt;vcq-&gt;ibcq.event_handler(&amp;ibev,</div><div class='add'>+						    cq-&gt;vcq-&gt;ibcq.cq_context);</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	default:</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='add'>+			  "unrecognized cqid %#x code %u\n", cqid, code);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	kref_put(&amp;cq-&gt;cq_kref, ionic_cq_complete);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_qp_event(struct ionic_ibdev *dev, u32 qpid, u8 code)</div><div class='add'>+{</div><div class='add'>+	unsigned long irqflags;</div><div class='add'>+	struct ib_event ibev;</div><div class='add'>+	struct ionic_qp *qp;</div><div class='add'>+</div><div class='add'>+	xa_lock_irqsave(&amp;dev-&gt;qp_tbl, irqflags);</div><div class='add'>+	qp = xa_load(&amp;dev-&gt;qp_tbl, qpid);</div><div class='add'>+	if (qp)</div><div class='add'>+		kref_get(&amp;qp-&gt;qp_kref);</div><div class='add'>+	xa_unlock_irqrestore(&amp;dev-&gt;qp_tbl, irqflags);</div><div class='add'>+</div><div class='add'>+	if (!qp) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='add'>+			  "missing qpid %#x code %u\n", qpid, code);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ibev.device = &amp;dev-&gt;ibdev;</div><div class='add'>+	ibev.element.qp = &amp;qp-&gt;ibqp;</div><div class='add'>+</div><div class='add'>+	switch (code) {</div><div class='add'>+	case IONIC_V1_EQE_SQ_DRAIN:</div><div class='add'>+		ibev.event = IB_EVENT_SQ_DRAINED;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	case IONIC_V1_EQE_QP_COMM_EST:</div><div class='add'>+		ibev.event = IB_EVENT_COMM_EST;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	case IONIC_V1_EQE_QP_LAST_WQE:</div><div class='add'>+		ibev.event = IB_EVENT_QP_LAST_WQE_REACHED;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	case IONIC_V1_EQE_QP_ERR:</div><div class='add'>+		ibev.event = IB_EVENT_QP_FATAL;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	case IONIC_V1_EQE_QP_ERR_REQUEST:</div><div class='add'>+		ibev.event = IB_EVENT_QP_REQ_ERR;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	case IONIC_V1_EQE_QP_ERR_ACCESS:</div><div class='add'>+		ibev.event = IB_EVENT_QP_ACCESS_ERR;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	default:</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='add'>+			  "unrecognized qpid %#x code %u\n", qpid, code);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;ibqp.event_handler)</div><div class='add'>+		qp-&gt;ibqp.event_handler(&amp;ibev, qp-&gt;ibqp.qp_context);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	kref_put(&amp;qp-&gt;qp_kref, ionic_qp_complete);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u16 ionic_poll_eq(struct ionic_eq *eq, u16 budget)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = eq-&gt;dev;</div><div class='add'>+	struct ionic_v1_eqe eqe;</div><div class='add'>+	u16 npolled = 0;</div><div class='add'>+	u8 type, code;</div><div class='add'>+	u32 evt, qid;</div><div class='add'>+</div><div class='add'>+	while (npolled &lt; budget) {</div><div class='add'>+		if (!ionic_next_eqe(eq, &amp;eqe))</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		ionic_queue_produce(&amp;eq-&gt;q);</div><div class='add'>+</div><div class='add'>+		/* cons is color for eq */</div><div class='add'>+		eq-&gt;q.cons = ionic_color_wrap(eq-&gt;q.prod, eq-&gt;q.cons);</div><div class='add'>+</div><div class='add'>+		++npolled;</div><div class='add'>+</div><div class='add'>+		evt = ionic_v1_eqe_evt(&amp;eqe);</div><div class='add'>+		type = ionic_v1_eqe_evt_type(evt);</div><div class='add'>+		code = ionic_v1_eqe_evt_code(evt);</div><div class='add'>+		qid = ionic_v1_eqe_evt_qid(evt);</div><div class='add'>+</div><div class='add'>+		switch (type) {</div><div class='add'>+		case IONIC_V1_EQE_TYPE_CQ:</div><div class='add'>+			ionic_cq_event(dev, qid, code);</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		case IONIC_V1_EQE_TYPE_QP:</div><div class='add'>+			ionic_qp_event(dev, qid, code);</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		default:</div><div class='add'>+			ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='add'>+				  "unknown event %#x type %u\n", evt, type);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return npolled;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_poll_eq_work(struct work_struct *work)</div><div class='add'>+{</div><div class='add'>+	struct ionic_eq *eq = container_of(work, struct ionic_eq, work);</div><div class='add'>+	u32 npolled;</div><div class='add'>+</div><div class='add'>+	if (unlikely(!eq-&gt;enable) || WARN_ON(eq-&gt;armed))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	npolled = ionic_poll_eq(eq, IONIC_EQ_WORK_BUDGET);</div><div class='add'>+	if (npolled == IONIC_EQ_WORK_BUDGET) {</div><div class='add'>+		ionic_intr_credits(eq-&gt;dev-&gt;lif_cfg.intr_ctrl, eq-&gt;intr,</div><div class='add'>+				   npolled, 0);</div><div class='add'>+		queue_work(ionic_evt_workq, &amp;eq-&gt;work);</div><div class='add'>+	} else {</div><div class='add'>+		xchg(&amp;eq-&gt;armed, 1);</div><div class='add'>+		ionic_intr_credits(eq-&gt;dev-&gt;lif_cfg.intr_ctrl, eq-&gt;intr,</div><div class='add'>+				   0, IONIC_INTR_CRED_UNMASK);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static irqreturn_t ionic_poll_eq_isr(int irq, void *eqptr)</div><div class='add'>+{</div><div class='add'>+	struct ionic_eq *eq = eqptr;</div><div class='add'>+	int was_armed;</div><div class='add'>+	u32 npolled;</div><div class='add'>+</div><div class='add'>+	was_armed = xchg(&amp;eq-&gt;armed, 0);</div><div class='add'>+</div><div class='add'>+	if (unlikely(!eq-&gt;enable) || !was_armed)</div><div class='add'>+		return IRQ_HANDLED;</div><div class='add'>+</div><div class='add'>+	npolled = ionic_poll_eq(eq, IONIC_EQ_ISR_BUDGET);</div><div class='add'>+	if (npolled == IONIC_EQ_ISR_BUDGET) {</div><div class='add'>+		ionic_intr_credits(eq-&gt;dev-&gt;lif_cfg.intr_ctrl, eq-&gt;intr,</div><div class='add'>+				   npolled, 0);</div><div class='add'>+		queue_work(ionic_evt_workq, &amp;eq-&gt;work);</div><div class='add'>+	} else {</div><div class='add'>+		xchg(&amp;eq-&gt;armed, 1);</div><div class='add'>+		ionic_intr_credits(eq-&gt;dev-&gt;lif_cfg.intr_ctrl, eq-&gt;intr,</div><div class='add'>+				   0, IONIC_INTR_CRED_UNMASK);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return IRQ_HANDLED;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct ionic_eq *ionic_create_eq(struct ionic_ibdev *dev, int eqid)</div><div class='add'>+{</div><div class='add'>+	struct ionic_intr_info intr_obj = { };</div><div class='add'>+	struct ionic_eq *eq;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	eq = kzalloc(sizeof(*eq), GFP_KERNEL);</div><div class='add'>+	if (!eq)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	eq-&gt;dev = dev;</div><div class='add'>+</div><div class='add'>+	rc = ionic_queue_init(&amp;eq-&gt;q, dev-&gt;lif_cfg.hwdev, IONIC_EQ_DEPTH,</div><div class='add'>+			      sizeof(struct ionic_v1_eqe));</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_q;</div><div class='add'>+</div><div class='add'>+	eq-&gt;eqid = eqid;</div><div class='add'>+</div><div class='add'>+	eq-&gt;armed = true;</div><div class='add'>+	eq-&gt;enable = false;</div><div class='add'>+	INIT_WORK(&amp;eq-&gt;work, ionic_poll_eq_work);</div><div class='add'>+</div><div class='add'>+	rc = ionic_intr_alloc(dev-&gt;lif_cfg.lif, &amp;intr_obj);</div><div class='add'>+	if (rc &lt; 0)</div><div class='add'>+		goto err_intr;</div><div class='add'>+</div><div class='add'>+	eq-&gt;irq = intr_obj.vector;</div><div class='add'>+	eq-&gt;intr = intr_obj.index;</div><div class='add'>+</div><div class='add'>+	ionic_queue_dbell_init(&amp;eq-&gt;q, eq-&gt;eqid);</div><div class='add'>+</div><div class='add'>+	/* cons is color for eq */</div><div class='add'>+	eq-&gt;q.cons = true;</div><div class='add'>+</div><div class='add'>+	snprintf(eq-&gt;name, sizeof(eq-&gt;name), "%s-%d-%d-eq",</div><div class='add'>+		 "ionr", dev-&gt;lif_cfg.lif_index, eq-&gt;eqid);</div><div class='add'>+</div><div class='add'>+	ionic_intr_mask(dev-&gt;lif_cfg.intr_ctrl, eq-&gt;intr, IONIC_INTR_MASK_SET);</div><div class='add'>+	ionic_intr_mask_assert(dev-&gt;lif_cfg.intr_ctrl, eq-&gt;intr, IONIC_INTR_MASK_SET);</div><div class='add'>+	ionic_intr_coal_init(dev-&gt;lif_cfg.intr_ctrl, eq-&gt;intr, 0);</div><div class='add'>+	ionic_intr_clean(dev-&gt;lif_cfg.intr_ctrl, eq-&gt;intr);</div><div class='add'>+</div><div class='add'>+	eq-&gt;enable = true;</div><div class='add'>+</div><div class='add'>+	rc = request_irq(eq-&gt;irq, ionic_poll_eq_isr, 0, eq-&gt;name, eq);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_irq;</div><div class='add'>+</div><div class='add'>+	rc = ionic_rdma_queue_devcmd(dev, &amp;eq-&gt;q, eq-&gt;eqid, eq-&gt;intr,</div><div class='add'>+				     IONIC_CMD_RDMA_CREATE_EQ);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_cmd;</div><div class='add'>+</div><div class='add'>+	ionic_intr_mask(dev-&gt;lif_cfg.intr_ctrl, eq-&gt;intr, IONIC_INTR_MASK_CLEAR);</div><div class='add'>+</div><div class='add'>+	return eq;</div><div class='add'>+</div><div class='add'>+err_cmd:</div><div class='add'>+	eq-&gt;enable = false;</div><div class='add'>+	free_irq(eq-&gt;irq, eq);</div><div class='add'>+	flush_work(&amp;eq-&gt;work);</div><div class='add'>+err_irq:</div><div class='add'>+	ionic_intr_free(dev-&gt;lif_cfg.lif, eq-&gt;intr);</div><div class='add'>+err_intr:</div><div class='add'>+	ionic_queue_destroy(&amp;eq-&gt;q, dev-&gt;lif_cfg.hwdev);</div><div class='add'>+err_q:</div><div class='add'>+	kfree(eq);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(rc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_destroy_eq(struct ionic_eq *eq)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = eq-&gt;dev;</div><div class='add'>+</div><div class='add'>+	eq-&gt;enable = false;</div><div class='add'>+	free_irq(eq-&gt;irq, eq);</div><div class='add'>+	flush_work(&amp;eq-&gt;work);</div><div class='add'>+</div><div class='add'>+	ionic_intr_free(dev-&gt;lif_cfg.lif, eq-&gt;intr);</div><div class='add'>+	ionic_queue_destroy(&amp;eq-&gt;q, dev-&gt;lif_cfg.hwdev);</div><div class='add'>+	kfree(eq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_create_rdma_admin(struct ionic_ibdev *dev)</div><div class='add'>+{</div><div class='add'>+	int eq_i = 0, aq_i = 0, rc = 0;</div><div class='add'>+	struct ionic_vcq *vcq;</div><div class='add'>+	struct ionic_aq *aq;</div><div class='add'>+	struct ionic_eq *eq;</div><div class='add'>+</div><div class='add'>+	dev-&gt;eq_vec = NULL;</div><div class='add'>+	dev-&gt;aq_vec = NULL;</div><div class='add'>+</div><div class='add'>+	INIT_WORK(&amp;dev-&gt;reset_work, ionic_reset_work);</div><div class='add'>+	INIT_DELAYED_WORK(&amp;dev-&gt;admin_dwork, ionic_admin_dwork);</div><div class='add'>+	atomic_set(&amp;dev-&gt;admin_state, IONIC_ADMIN_KILLED);</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;lif_cfg.aq_count &gt; IONIC_AQ_COUNT) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev, "limiting adminq count to %d\n",</div><div class='add'>+			  IONIC_AQ_COUNT);</div><div class='add'>+		dev-&gt;lif_cfg.aq_count = IONIC_AQ_COUNT;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;lif_cfg.eq_count &gt; IONIC_EQ_COUNT) {</div><div class='add'>+		dev_dbg(&amp;dev-&gt;ibdev.dev, "limiting eventq count to %d\n",</div><div class='add'>+			IONIC_EQ_COUNT);</div><div class='add'>+		dev-&gt;lif_cfg.eq_count = IONIC_EQ_COUNT;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* need at least two eq and one aq */</div><div class='add'>+	if (dev-&gt;lif_cfg.eq_count &lt; IONIC_EQ_COUNT_MIN ||</div><div class='add'>+	    dev-&gt;lif_cfg.aq_count &lt; IONIC_AQ_COUNT_MIN) {</div><div class='add'>+		rc = -EINVAL;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dev-&gt;eq_vec = kmalloc_array(dev-&gt;lif_cfg.eq_count, sizeof(*dev-&gt;eq_vec),</div><div class='add'>+				    GFP_KERNEL);</div><div class='add'>+	if (!dev-&gt;eq_vec) {</div><div class='add'>+		rc = -ENOMEM;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (eq_i = 0; eq_i &lt; dev-&gt;lif_cfg.eq_count; ++eq_i) {</div><div class='add'>+		eq = ionic_create_eq(dev, eq_i + dev-&gt;lif_cfg.eq_base);</div><div class='add'>+		if (IS_ERR(eq)) {</div><div class='add'>+			rc = PTR_ERR(eq);</div><div class='add'>+</div><div class='add'>+			if (eq_i &lt; IONIC_EQ_COUNT_MIN) {</div><div class='add'>+				ibdev_err(&amp;dev-&gt;ibdev,</div><div class='add'>+					  "fail create eq %pe\n", eq);</div><div class='add'>+				goto out;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			/* ok, just fewer eq than device supports */</div><div class='add'>+			ibdev_dbg(&amp;dev-&gt;ibdev, "eq count %d want %d rc %pe\n",</div><div class='add'>+				  eq_i, dev-&gt;lif_cfg.eq_count, eq);</div><div class='add'>+</div><div class='add'>+			rc = 0;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		dev-&gt;eq_vec[eq_i] = eq;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dev-&gt;lif_cfg.eq_count = eq_i;</div><div class='add'>+</div><div class='add'>+	dev-&gt;aq_vec = kmalloc_array(dev-&gt;lif_cfg.aq_count, sizeof(*dev-&gt;aq_vec),</div><div class='add'>+				    GFP_KERNEL);</div><div class='add'>+	if (!dev-&gt;aq_vec) {</div><div class='add'>+		rc = -ENOMEM;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Create one CQ per AQ */</div><div class='add'>+	for (aq_i = 0; aq_i &lt; dev-&gt;lif_cfg.aq_count; ++aq_i) {</div><div class='add'>+		vcq = ionic_create_rdma_admincq(dev, aq_i % eq_i);</div><div class='add'>+		if (IS_ERR(vcq)) {</div><div class='add'>+			rc = PTR_ERR(vcq);</div><div class='add'>+</div><div class='add'>+			if (!aq_i) {</div><div class='add'>+				ibdev_err(&amp;dev-&gt;ibdev,</div><div class='add'>+					  "failed to create acq %pe\n", vcq);</div><div class='add'>+				goto out;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			/* ok, just fewer adminq than device supports */</div><div class='add'>+			ibdev_dbg(&amp;dev-&gt;ibdev, "acq count %d want %d rc %pe\n",</div><div class='add'>+				  aq_i, dev-&gt;lif_cfg.aq_count, vcq);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		aq = ionic_create_rdma_adminq(dev, aq_i + dev-&gt;lif_cfg.aq_base,</div><div class='add'>+					      vcq-&gt;cq[0].cqid);</div><div class='add'>+		if (IS_ERR(aq)) {</div><div class='add'>+			/* Clean up the dangling CQ */</div><div class='add'>+			ionic_destroy_cq_common(dev, &amp;vcq-&gt;cq[0]);</div><div class='add'>+			kfree(vcq);</div><div class='add'>+</div><div class='add'>+			rc = PTR_ERR(aq);</div><div class='add'>+</div><div class='add'>+			if (!aq_i) {</div><div class='add'>+				ibdev_err(&amp;dev-&gt;ibdev,</div><div class='add'>+					  "failed to create aq %pe\n", aq);</div><div class='add'>+				goto out;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			/* ok, just fewer adminq than device supports */</div><div class='add'>+			ibdev_dbg(&amp;dev-&gt;ibdev, "aq count %d want %d rc %pe\n",</div><div class='add'>+				  aq_i, dev-&gt;lif_cfg.aq_count, aq);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		vcq-&gt;ibcq.cq_context = aq;</div><div class='add'>+		aq-&gt;vcq = vcq;</div><div class='add'>+</div><div class='add'>+		atomic_set(&amp;aq-&gt;admin_state, IONIC_ADMIN_ACTIVE);</div><div class='add'>+		dev-&gt;aq_vec[aq_i] = aq;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	atomic_set(&amp;dev-&gt;admin_state, IONIC_ADMIN_ACTIVE);</div><div class='add'>+out:</div><div class='add'>+	dev-&gt;lif_cfg.eq_count = eq_i;</div><div class='add'>+	dev-&gt;lif_cfg.aq_count = aq_i;</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void ionic_destroy_rdma_admin(struct ionic_ibdev *dev)</div><div class='add'>+{</div><div class='add'>+	struct ionic_vcq *vcq;</div><div class='add'>+	struct ionic_aq *aq;</div><div class='add'>+	struct ionic_eq *eq;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Killing the admin before destroy makes sure all admin and</div><div class='add'>+	 * completions are flushed. admin_state = IONIC_ADMIN_KILLED</div><div class='add'>+	 * stops queueing up further works.</div><div class='add'>+	 */</div><div class='add'>+	cancel_delayed_work_sync(&amp;dev-&gt;admin_dwork);</div><div class='add'>+	cancel_work_sync(&amp;dev-&gt;reset_work);</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;aq_vec) {</div><div class='add'>+		while (dev-&gt;lif_cfg.aq_count &gt; 0) {</div><div class='add'>+			aq = dev-&gt;aq_vec[--dev-&gt;lif_cfg.aq_count];</div><div class='add'>+			vcq = aq-&gt;vcq;</div><div class='add'>+</div><div class='add'>+			cancel_work_sync(&amp;aq-&gt;work);</div><div class='add'>+</div><div class='add'>+			__ionic_destroy_rdma_adminq(dev, aq);</div><div class='add'>+			if (vcq) {</div><div class='add'>+				ionic_destroy_cq_common(dev, &amp;vcq-&gt;cq[0]);</div><div class='add'>+				kfree(vcq);</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		kfree(dev-&gt;aq_vec);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;eq_vec) {</div><div class='add'>+		while (dev-&gt;lif_cfg.eq_count &gt; 0) {</div><div class='add'>+			eq = dev-&gt;eq_vec[--dev-&gt;lif_cfg.eq_count];</div><div class='add'>+			ionic_destroy_eq(eq);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		kfree(dev-&gt;eq_vec);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c<br/>new file mode 100644<br/>index 000000000000..ea12d9b8e125<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/ionic/ionic_controlpath.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/ionic/ionic_controlpath.c</a></div><div class='hunk'>@@ -0,0 +1,2679 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0</div><div class='add'>+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/module.h&gt;</div><div class='add'>+#include &lt;linux/printk.h&gt;</div><div class='add'>+#include &lt;rdma/ib_addr.h&gt;</div><div class='add'>+#include &lt;rdma/ib_cache.h&gt;</div><div class='add'>+#include &lt;rdma/ib_user_verbs.h&gt;</div><div class='add'>+#include &lt;ionic_api.h&gt;</div><div class='add'>+</div><div class='add'>+#include "ionic_fw.h"</div><div class='add'>+#include "ionic_ibdev.h"</div><div class='add'>+</div><div class='add'>+#define ionic_set_ecn(tos)   (((tos) | 2u) &amp; ~1u)</div><div class='add'>+#define ionic_clear_ecn(tos)  ((tos) &amp; ~3u)</div><div class='add'>+</div><div class='add'>+static int ionic_validate_qdesc(struct ionic_qdesc *q)</div><div class='add'>+{</div><div class='add'>+	if (!q-&gt;addr || !q-&gt;size || !q-&gt;mask ||</div><div class='add'>+	    !q-&gt;depth_log2 || !q-&gt;stride_log2)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (q-&gt;addr &amp; (PAGE_SIZE - 1))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (q-&gt;mask != BIT(q-&gt;depth_log2) - 1)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (q-&gt;size &lt; BIT_ULL(q-&gt;depth_log2 + q-&gt;stride_log2))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u32 ionic_get_eqid(struct ionic_ibdev *dev, u32 comp_vector, u8 udma_idx)</div><div class='add'>+{</div><div class='add'>+	/* EQ per vector per udma, and the first eqs reserved for async events.</div><div class='add'>+	 * The rest of the vectors can be requested for completions.</div><div class='add'>+	 */</div><div class='add'>+	u32 comp_vec_count = dev-&gt;lif_cfg.eq_count / dev-&gt;lif_cfg.udma_count - 1;</div><div class='add'>+</div><div class='add'>+	return (comp_vector % comp_vec_count + 1) * dev-&gt;lif_cfg.udma_count + udma_idx;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_get_cqid(struct ionic_ibdev *dev, u32 *cqid, u8 udma_idx)</div><div class='add'>+{</div><div class='add'>+	unsigned int size, base, bound;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	size = dev-&gt;lif_cfg.cq_count / dev-&gt;lif_cfg.udma_count;</div><div class='add'>+	base = size * udma_idx;</div><div class='add'>+	bound = base + size;</div><div class='add'>+</div><div class='add'>+	rc = ionic_resid_get_shared(&amp;dev-&gt;inuse_cqid, base, bound);</div><div class='add'>+	if (rc &gt;= 0) {</div><div class='add'>+		/* cq_base is zero or a multiple of two queue groups */</div><div class='add'>+		*cqid = dev-&gt;lif_cfg.cq_base +</div><div class='add'>+			ionic_bitid_to_qid(rc, dev-&gt;lif_cfg.udma_qgrp_shift,</div><div class='add'>+					   dev-&gt;half_cqid_udma_shift);</div><div class='add'>+</div><div class='add'>+		rc = 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_put_cqid(struct ionic_ibdev *dev, u32 cqid)</div><div class='add'>+{</div><div class='add'>+	u32 bitid = ionic_qid_to_bitid(cqid - dev-&gt;lif_cfg.cq_base,</div><div class='add'>+				       dev-&gt;lif_cfg.udma_qgrp_shift,</div><div class='add'>+				       dev-&gt;half_cqid_udma_shift);</div><div class='add'>+</div><div class='add'>+	ionic_resid_put(&amp;dev-&gt;inuse_cqid, bitid);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_create_cq_common(struct ionic_vcq *vcq,</div><div class='add'>+			   struct ionic_tbl_buf *buf,</div><div class='add'>+			   const struct ib_cq_init_attr *attr,</div><div class='add'>+			   struct ionic_ctx *ctx,</div><div class='add'>+			   struct ib_udata *udata,</div><div class='add'>+			   struct ionic_qdesc *req_cq,</div><div class='add'>+			   __u32 *resp_cqid,</div><div class='add'>+			   int udma_idx)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(vcq-&gt;ibcq.device);</div><div class='add'>+	struct ionic_cq *cq = &amp;vcq-&gt;cq[udma_idx];</div><div class='add'>+	void *entry;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	cq-&gt;vcq = vcq;</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;cqe &lt; 1 || attr-&gt;cqe + IONIC_CQ_GRACE &gt; 0xffff) {</div><div class='add'>+		rc = -EINVAL;</div><div class='add'>+		goto err_args;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rc = ionic_get_cqid(dev, &amp;cq-&gt;cqid, udma_idx);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_args;</div><div class='add'>+</div><div class='add'>+	cq-&gt;eqid = ionic_get_eqid(dev, attr-&gt;comp_vector, udma_idx);</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;cq-&gt;lock);</div><div class='add'>+	INIT_LIST_HEAD(&amp;cq-&gt;poll_sq);</div><div class='add'>+	INIT_LIST_HEAD(&amp;cq-&gt;flush_sq);</div><div class='add'>+	INIT_LIST_HEAD(&amp;cq-&gt;flush_rq);</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		rc = ionic_validate_qdesc(req_cq);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto err_qdesc;</div><div class='add'>+</div><div class='add'>+		cq-&gt;umem = ib_umem_get(&amp;dev-&gt;ibdev, req_cq-&gt;addr, req_cq-&gt;size,</div><div class='add'>+				       IB_ACCESS_LOCAL_WRITE);</div><div class='add'>+		if (IS_ERR(cq-&gt;umem)) {</div><div class='add'>+			rc = PTR_ERR(cq-&gt;umem);</div><div class='add'>+			goto err_qdesc;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		cq-&gt;q.ptr = NULL;</div><div class='add'>+		cq-&gt;q.size = req_cq-&gt;size;</div><div class='add'>+		cq-&gt;q.mask = req_cq-&gt;mask;</div><div class='add'>+		cq-&gt;q.depth_log2 = req_cq-&gt;depth_log2;</div><div class='add'>+		cq-&gt;q.stride_log2 = req_cq-&gt;stride_log2;</div><div class='add'>+</div><div class='add'>+		*resp_cqid = cq-&gt;cqid;</div><div class='add'>+	} else {</div><div class='add'>+		rc = ionic_queue_init(&amp;cq-&gt;q, dev-&gt;lif_cfg.hwdev,</div><div class='add'>+				      attr-&gt;cqe + IONIC_CQ_GRACE,</div><div class='add'>+				      sizeof(struct ionic_v1_cqe));</div><div class='add'>+		if (rc)</div><div class='add'>+			goto err_q_init;</div><div class='add'>+</div><div class='add'>+		ionic_queue_dbell_init(&amp;cq-&gt;q, cq-&gt;cqid);</div><div class='add'>+		cq-&gt;color = true;</div><div class='add'>+		cq-&gt;credit = cq-&gt;q.mask;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rc = ionic_pgtbl_init(dev, buf, cq-&gt;umem, cq-&gt;q.dma, 1, PAGE_SIZE);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_pgtbl_init;</div><div class='add'>+</div><div class='add'>+	init_completion(&amp;cq-&gt;cq_rel_comp);</div><div class='add'>+	kref_init(&amp;cq-&gt;cq_kref);</div><div class='add'>+</div><div class='add'>+	entry = xa_store_irq(&amp;dev-&gt;cq_tbl, cq-&gt;cqid, cq, GFP_KERNEL);</div><div class='add'>+	if (entry) {</div><div class='add'>+		if (!xa_is_err(entry))</div><div class='add'>+			rc = -EINVAL;</div><div class='add'>+		else</div><div class='add'>+			rc = xa_err(entry);</div><div class='add'>+</div><div class='add'>+		goto err_xa;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_xa:</div><div class='add'>+	ionic_pgtbl_unbuf(dev, buf);</div><div class='add'>+err_pgtbl_init:</div><div class='add'>+	if (!udata)</div><div class='add'>+		ionic_queue_destroy(&amp;cq-&gt;q, dev-&gt;lif_cfg.hwdev);</div><div class='add'>+err_q_init:</div><div class='add'>+	if (cq-&gt;umem)</div><div class='add'>+		ib_umem_release(cq-&gt;umem);</div><div class='add'>+err_qdesc:</div><div class='add'>+	ionic_put_cqid(dev, cq-&gt;cqid);</div><div class='add'>+err_args:</div><div class='add'>+	cq-&gt;vcq = NULL;</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq)</div><div class='add'>+{</div><div class='add'>+	if (!cq-&gt;vcq)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	xa_erase_irq(&amp;dev-&gt;cq_tbl, cq-&gt;cqid);</div><div class='add'>+</div><div class='add'>+	kref_put(&amp;cq-&gt;cq_kref, ionic_cq_complete);</div><div class='add'>+	wait_for_completion(&amp;cq-&gt;cq_rel_comp);</div><div class='add'>+</div><div class='add'>+	if (cq-&gt;umem)</div><div class='add'>+		ib_umem_release(cq-&gt;umem);</div><div class='add'>+	else</div><div class='add'>+		ionic_queue_destroy(&amp;cq-&gt;q, dev-&gt;lif_cfg.hwdev);</div><div class='add'>+</div><div class='add'>+	ionic_put_cqid(dev, cq-&gt;cqid);</div><div class='add'>+</div><div class='add'>+	cq-&gt;vcq = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_validate_qdesc_zero(struct ionic_qdesc *q)</div><div class='add'>+{</div><div class='add'>+	if (q-&gt;addr || q-&gt;size || q-&gt;mask || q-&gt;depth_log2 || q-&gt;stride_log2)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_get_pdid(struct ionic_ibdev *dev, u32 *pdid)</div><div class='add'>+{</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rc = ionic_resid_get(&amp;dev-&gt;inuse_pdid);</div><div class='add'>+	if (rc &lt; 0)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	*pdid = rc;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_get_ahid(struct ionic_ibdev *dev, u32 *ahid)</div><div class='add'>+{</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rc = ionic_resid_get(&amp;dev-&gt;inuse_ahid);</div><div class='add'>+	if (rc &lt; 0)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	*ahid = rc;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_get_mrid(struct ionic_ibdev *dev, u32 *mrid)</div><div class='add'>+{</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	/* wrap to 1, skip reserved lkey */</div><div class='add'>+	rc = ionic_resid_get_shared(&amp;dev-&gt;inuse_mrid, 1,</div><div class='add'>+				    dev-&gt;inuse_mrid.inuse_size);</div><div class='add'>+	if (rc &lt; 0)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	*mrid = ionic_mrid(rc, dev-&gt;next_mrkey++);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_get_gsi_qpid(struct ionic_ibdev *dev, u32 *qpid)</div><div class='add'>+{</div><div class='add'>+	int rc = 0;</div><div class='add'>+</div><div class='add'>+	rc = ionic_resid_get_shared(&amp;dev-&gt;inuse_qpid, IB_QPT_GSI, IB_QPT_GSI + 1);</div><div class='add'>+	if (rc &lt; 0)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	*qpid = IB_QPT_GSI;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_get_qpid(struct ionic_ibdev *dev, u32 *qpid,</div><div class='add'>+			  u8 *udma_idx, u8 udma_mask)</div><div class='add'>+{</div><div class='add'>+	unsigned int size, base, bound;</div><div class='add'>+	int udma_i, udma_x, udma_ix;</div><div class='add'>+	int rc = -EINVAL;</div><div class='add'>+</div><div class='add'>+	udma_x = dev-&gt;next_qpid_udma_idx;</div><div class='add'>+</div><div class='add'>+	dev-&gt;next_qpid_udma_idx ^= dev-&gt;lif_cfg.udma_count - 1;</div><div class='add'>+</div><div class='add'>+	for (udma_i = 0; udma_i &lt; dev-&gt;lif_cfg.udma_count; ++udma_i) {</div><div class='add'>+		udma_ix = udma_i ^ udma_x;</div><div class='add'>+</div><div class='add'>+		if (!(udma_mask &amp; BIT(udma_ix)))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		size = dev-&gt;lif_cfg.qp_count / dev-&gt;lif_cfg.udma_count;</div><div class='add'>+		base = size * udma_ix;</div><div class='add'>+		bound = base + size;</div><div class='add'>+</div><div class='add'>+		/* skip reserved SMI and GSI qpids in group zero */</div><div class='add'>+		if (!base)</div><div class='add'>+			base = 2;</div><div class='add'>+</div><div class='add'>+		rc = ionic_resid_get_shared(&amp;dev-&gt;inuse_qpid, base, bound);</div><div class='add'>+		if (rc &gt;= 0) {</div><div class='add'>+			*qpid = ionic_bitid_to_qid(rc,</div><div class='add'>+						   dev-&gt;lif_cfg.udma_qgrp_shift,</div><div class='add'>+						   dev-&gt;half_qpid_udma_shift);</div><div class='add'>+			*udma_idx = udma_ix;</div><div class='add'>+</div><div class='add'>+			rc = 0;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_get_dbid(struct ionic_ibdev *dev, u32 *dbid, phys_addr_t *addr)</div><div class='add'>+{</div><div class='add'>+	int rc, dbpage_num;</div><div class='add'>+</div><div class='add'>+	/* wrap to 1, skip kernel reserved */</div><div class='add'>+	rc = ionic_resid_get_shared(&amp;dev-&gt;inuse_dbid, 1,</div><div class='add'>+				    dev-&gt;inuse_dbid.inuse_size);</div><div class='add'>+	if (rc &lt; 0)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	dbpage_num = (dev-&gt;lif_cfg.lif_hw_index * dev-&gt;lif_cfg.dbid_count) + rc;</div><div class='add'>+	*addr = dev-&gt;lif_cfg.db_phys + ((phys_addr_t)dbpage_num &lt;&lt; PAGE_SHIFT);</div><div class='add'>+</div><div class='add'>+	*dbid = rc;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_put_pdid(struct ionic_ibdev *dev, u32 pdid)</div><div class='add'>+{</div><div class='add'>+	ionic_resid_put(&amp;dev-&gt;inuse_pdid, pdid);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_put_ahid(struct ionic_ibdev *dev, u32 ahid)</div><div class='add'>+{</div><div class='add'>+	ionic_resid_put(&amp;dev-&gt;inuse_ahid, ahid);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_put_mrid(struct ionic_ibdev *dev, u32 mrid)</div><div class='add'>+{</div><div class='add'>+	ionic_resid_put(&amp;dev-&gt;inuse_mrid, ionic_mrid_index(mrid));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_put_qpid(struct ionic_ibdev *dev, u32 qpid)</div><div class='add'>+{</div><div class='add'>+	u32 bitid = ionic_qid_to_bitid(qpid,</div><div class='add'>+				       dev-&gt;lif_cfg.udma_qgrp_shift,</div><div class='add'>+				       dev-&gt;half_qpid_udma_shift);</div><div class='add'>+</div><div class='add'>+	ionic_resid_put(&amp;dev-&gt;inuse_qpid, bitid);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_put_dbid(struct ionic_ibdev *dev, u32 dbid)</div><div class='add'>+{</div><div class='add'>+	ionic_resid_put(&amp;dev-&gt;inuse_dbid, dbid);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct rdma_user_mmap_entry*</div><div class='add'>+ionic_mmap_entry_insert(struct ionic_ctx *ctx, unsigned long size,</div><div class='add'>+			unsigned long pfn, u8 mmap_flags, u64 *offset)</div><div class='add'>+{</div><div class='add'>+	struct ionic_mmap_entry *entry;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	entry = kzalloc(sizeof(*entry), GFP_KERNEL);</div><div class='add'>+	if (!entry)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	entry-&gt;size = size;</div><div class='add'>+	entry-&gt;pfn = pfn;</div><div class='add'>+	entry-&gt;mmap_flags = mmap_flags;</div><div class='add'>+</div><div class='add'>+	rc = rdma_user_mmap_entry_insert(&amp;ctx-&gt;ibctx, &amp;entry-&gt;rdma_entry,</div><div class='add'>+					 entry-&gt;size);</div><div class='add'>+	if (rc) {</div><div class='add'>+		kfree(entry);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (offset)</div><div class='add'>+		*offset = rdma_user_mmap_get_offset(&amp;entry-&gt;rdma_entry);</div><div class='add'>+</div><div class='add'>+	return &amp;entry-&gt;rdma_entry;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibctx-&gt;device);</div><div class='add'>+	struct ionic_ctx *ctx = to_ionic_ctx(ibctx);</div><div class='add'>+	struct ionic_ctx_resp resp = {};</div><div class='add'>+	struct ionic_ctx_req req;</div><div class='add'>+	phys_addr_t db_phys = 0;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rc = ib_copy_from_udata(&amp;req, udata, sizeof(req));</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	/* try to allocate dbid for user ctx */</div><div class='add'>+	rc = ionic_get_dbid(dev, &amp;ctx-&gt;dbid, &amp;db_phys);</div><div class='add'>+	if (rc &lt; 0)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ibdev, "user space dbid %u\n", ctx-&gt;dbid);</div><div class='add'>+</div><div class='add'>+	ctx-&gt;mmap_dbell = ionic_mmap_entry_insert(ctx, PAGE_SIZE,</div><div class='add'>+						  PHYS_PFN(db_phys), 0, NULL);</div><div class='add'>+	if (!ctx-&gt;mmap_dbell) {</div><div class='add'>+		rc = -ENOMEM;</div><div class='add'>+		goto err_mmap_dbell;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	resp.page_shift = PAGE_SHIFT;</div><div class='add'>+</div><div class='add'>+	resp.dbell_offset = db_phys &amp; ~PAGE_MASK;</div><div class='add'>+</div><div class='add'>+	resp.version = dev-&gt;lif_cfg.rdma_version;</div><div class='add'>+	resp.qp_opcodes = dev-&gt;lif_cfg.qp_opcodes;</div><div class='add'>+	resp.admin_opcodes = dev-&gt;lif_cfg.admin_opcodes;</div><div class='add'>+</div><div class='add'>+	resp.sq_qtype = dev-&gt;lif_cfg.sq_qtype;</div><div class='add'>+	resp.rq_qtype = dev-&gt;lif_cfg.rq_qtype;</div><div class='add'>+	resp.cq_qtype = dev-&gt;lif_cfg.cq_qtype;</div><div class='add'>+	resp.admin_qtype = dev-&gt;lif_cfg.aq_qtype;</div><div class='add'>+	resp.max_stride = dev-&gt;lif_cfg.max_stride;</div><div class='add'>+	resp.max_spec = IONIC_SPEC_HIGH;</div><div class='add'>+</div><div class='add'>+	resp.udma_count = dev-&gt;lif_cfg.udma_count;</div><div class='add'>+	resp.expdb_mask = dev-&gt;lif_cfg.expdb_mask;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;lif_cfg.sq_expdb)</div><div class='add'>+		resp.expdb_qtypes |= IONIC_EXPDB_SQ;</div><div class='add'>+	if (dev-&gt;lif_cfg.rq_expdb)</div><div class='add'>+		resp.expdb_qtypes |= IONIC_EXPDB_RQ;</div><div class='add'>+</div><div class='add'>+	rc = ib_copy_to_udata(udata, &amp;resp, sizeof(resp));</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_resp;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_resp:</div><div class='add'>+	rdma_user_mmap_entry_remove(ctx-&gt;mmap_dbell);</div><div class='add'>+err_mmap_dbell:</div><div class='add'>+	ionic_put_dbid(dev, ctx-&gt;dbid);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void ionic_dealloc_ucontext(struct ib_ucontext *ibctx)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibctx-&gt;device);</div><div class='add'>+	struct ionic_ctx *ctx = to_ionic_ctx(ibctx);</div><div class='add'>+</div><div class='add'>+	rdma_user_mmap_entry_remove(ctx-&gt;mmap_dbell);</div><div class='add'>+	ionic_put_dbid(dev, ctx-&gt;dbid);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibctx-&gt;device);</div><div class='add'>+	struct ionic_ctx *ctx = to_ionic_ctx(ibctx);</div><div class='add'>+	struct rdma_user_mmap_entry *rdma_entry;</div><div class='add'>+	struct ionic_mmap_entry *ionic_entry;</div><div class='add'>+	int rc = 0;</div><div class='add'>+</div><div class='add'>+	rdma_entry = rdma_user_mmap_entry_get(&amp;ctx-&gt;ibctx, vma);</div><div class='add'>+	if (!rdma_entry) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev, "not found %#lx\n",</div><div class='add'>+			  vma-&gt;vm_pgoff &lt;&lt; PAGE_SHIFT);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ionic_entry = container_of(rdma_entry, struct ionic_mmap_entry,</div><div class='add'>+				   rdma_entry);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ibdev, "writecombine? %d\n",</div><div class='add'>+		  ionic_entry-&gt;mmap_flags &amp; IONIC_MMAP_WC);</div><div class='add'>+	if (ionic_entry-&gt;mmap_flags &amp; IONIC_MMAP_WC)</div><div class='add'>+		vma-&gt;vm_page_prot = pgprot_writecombine(vma-&gt;vm_page_prot);</div><div class='add'>+	else</div><div class='add'>+		vma-&gt;vm_page_prot = pgprot_noncached(vma-&gt;vm_page_prot);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ibdev, "remap st %#lx pf %#lx sz %#lx\n",</div><div class='add'>+		  vma-&gt;vm_start, ionic_entry-&gt;pfn, ionic_entry-&gt;size);</div><div class='add'>+	rc = rdma_user_mmap_io(&amp;ctx-&gt;ibctx, vma, ionic_entry-&gt;pfn,</div><div class='add'>+			       ionic_entry-&gt;size, vma-&gt;vm_page_prot,</div><div class='add'>+			       rdma_entry);</div><div class='add'>+	if (rc)</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev, "remap failed %d\n", rc);</div><div class='add'>+</div><div class='add'>+	rdma_user_mmap_entry_put(rdma_entry);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry)</div><div class='add'>+{</div><div class='add'>+	struct ionic_mmap_entry *ionic_entry;</div><div class='add'>+</div><div class='add'>+	ionic_entry = container_of(rdma_entry, struct ionic_mmap_entry,</div><div class='add'>+				   rdma_entry);</div><div class='add'>+	kfree(ionic_entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibpd-&gt;device);</div><div class='add'>+	struct ionic_pd *pd = to_ionic_pd(ibpd);</div><div class='add'>+</div><div class='add'>+	return ionic_get_pdid(dev, &amp;pd-&gt;pdid);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibpd-&gt;device);</div><div class='add'>+	struct ionic_pd *pd = to_ionic_pd(ibpd);</div><div class='add'>+</div><div class='add'>+	ionic_put_pdid(dev, pd-&gt;pdid);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_build_hdr(struct ionic_ibdev *dev,</div><div class='add'>+			   struct ib_ud_header *hdr,</div><div class='add'>+			   const struct rdma_ah_attr *attr,</div><div class='add'>+			   u16 sport, bool want_ecn)</div><div class='add'>+{</div><div class='add'>+	const struct ib_global_route *grh;</div><div class='add'>+	enum rdma_network_type net;</div><div class='add'>+	u16 vlan;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;ah_flags != IB_AH_GRH)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	if (attr-&gt;type != RDMA_AH_ATTR_TYPE_ROCE)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	grh = rdma_ah_read_grh(attr);</div><div class='add'>+</div><div class='add'>+	rc = rdma_read_gid_l2_fields(grh-&gt;sgid_attr, &amp;vlan, &amp;hdr-&gt;eth.smac_h[0]);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	net = rdma_gid_attr_network_type(grh-&gt;sgid_attr);</div><div class='add'>+</div><div class='add'>+	rc = ib_ud_header_init(0,	/* no payload */</div><div class='add'>+			       0,	/* no lrh */</div><div class='add'>+			       1,	/* yes eth */</div><div class='add'>+			       vlan != 0xffff,</div><div class='add'>+			       0,	/* no grh */</div><div class='add'>+			       net == RDMA_NETWORK_IPV4 ? 4 : 6,</div><div class='add'>+			       1,	/* yes udp */</div><div class='add'>+			       0,	/* no imm */</div><div class='add'>+			       hdr);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	ether_addr_copy(hdr-&gt;eth.dmac_h, attr-&gt;roce.dmac);</div><div class='add'>+</div><div class='add'>+	if (net == RDMA_NETWORK_IPV4) {</div><div class='add'>+		hdr-&gt;eth.type = cpu_to_be16(ETH_P_IP);</div><div class='add'>+		hdr-&gt;ip4.frag_off = cpu_to_be16(0x4000); /* don't fragment */</div><div class='add'>+		hdr-&gt;ip4.ttl = grh-&gt;hop_limit;</div><div class='add'>+		hdr-&gt;ip4.tot_len = cpu_to_be16(0xffff);</div><div class='add'>+		hdr-&gt;ip4.saddr =</div><div class='add'>+			*(const __be32 *)(grh-&gt;sgid_attr-&gt;gid.raw + 12);</div><div class='add'>+		hdr-&gt;ip4.daddr = *(const __be32 *)(grh-&gt;dgid.raw + 12);</div><div class='add'>+</div><div class='add'>+		if (want_ecn)</div><div class='add'>+			hdr-&gt;ip4.tos = ionic_set_ecn(grh-&gt;traffic_class);</div><div class='add'>+		else</div><div class='add'>+			hdr-&gt;ip4.tos = ionic_clear_ecn(grh-&gt;traffic_class);</div><div class='add'>+	} else {</div><div class='add'>+		hdr-&gt;eth.type = cpu_to_be16(ETH_P_IPV6);</div><div class='add'>+		hdr-&gt;grh.flow_label = cpu_to_be32(grh-&gt;flow_label);</div><div class='add'>+		hdr-&gt;grh.hop_limit = grh-&gt;hop_limit;</div><div class='add'>+		hdr-&gt;grh.source_gid = grh-&gt;sgid_attr-&gt;gid;</div><div class='add'>+		hdr-&gt;grh.destination_gid = grh-&gt;dgid;</div><div class='add'>+</div><div class='add'>+		if (want_ecn)</div><div class='add'>+			hdr-&gt;grh.traffic_class =</div><div class='add'>+				ionic_set_ecn(grh-&gt;traffic_class);</div><div class='add'>+		else</div><div class='add'>+			hdr-&gt;grh.traffic_class =</div><div class='add'>+				ionic_clear_ecn(grh-&gt;traffic_class);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (vlan != 0xffff) {</div><div class='add'>+		vlan |= rdma_ah_get_sl(attr) &lt;&lt; VLAN_PRIO_SHIFT;</div><div class='add'>+		hdr-&gt;vlan.tag = cpu_to_be16(vlan);</div><div class='add'>+		hdr-&gt;vlan.type = hdr-&gt;eth.type;</div><div class='add'>+		hdr-&gt;eth.type = cpu_to_be16(ETH_P_8021Q);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hdr-&gt;udp.sport = cpu_to_be16(sport);</div><div class='add'>+	hdr-&gt;udp.dport = cpu_to_be16(ROCE_V2_UDP_DPORT);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_set_ah_attr(struct ionic_ibdev *dev,</div><div class='add'>+			      struct rdma_ah_attr *ah_attr,</div><div class='add'>+			      struct ib_ud_header *hdr,</div><div class='add'>+			      int sgid_index)</div><div class='add'>+{</div><div class='add'>+	u32 flow_label;</div><div class='add'>+	u16 vlan = 0;</div><div class='add'>+	u8  tos, ttl;</div><div class='add'>+</div><div class='add'>+	if (hdr-&gt;vlan_present)</div><div class='add'>+		vlan = be16_to_cpu(hdr-&gt;vlan.tag);</div><div class='add'>+</div><div class='add'>+	if (hdr-&gt;ipv4_present) {</div><div class='add'>+		flow_label = 0;</div><div class='add'>+		ttl = hdr-&gt;ip4.ttl;</div><div class='add'>+		tos = hdr-&gt;ip4.tos;</div><div class='add'>+		*(__be16 *)(hdr-&gt;grh.destination_gid.raw + 10) = cpu_to_be16(0xffff);</div><div class='add'>+		*(__be32 *)(hdr-&gt;grh.destination_gid.raw + 12) = hdr-&gt;ip4.daddr;</div><div class='add'>+	} else {</div><div class='add'>+		flow_label = be32_to_cpu(hdr-&gt;grh.flow_label);</div><div class='add'>+		ttl = hdr-&gt;grh.hop_limit;</div><div class='add'>+		tos = hdr-&gt;grh.traffic_class;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	memset(ah_attr, 0, sizeof(*ah_attr));</div><div class='add'>+	ah_attr-&gt;type = RDMA_AH_ATTR_TYPE_ROCE;</div><div class='add'>+	if (hdr-&gt;eth_present)</div><div class='add'>+		ether_addr_copy(ah_attr-&gt;roce.dmac, hdr-&gt;eth.dmac_h);</div><div class='add'>+	rdma_ah_set_sl(ah_attr, vlan &gt;&gt; VLAN_PRIO_SHIFT);</div><div class='add'>+	rdma_ah_set_port_num(ah_attr, 1);</div><div class='add'>+	rdma_ah_set_grh(ah_attr, NULL, flow_label, sgid_index, ttl, tos);</div><div class='add'>+	rdma_ah_set_dgid_raw(ah_attr, &amp;hdr-&gt;grh.destination_gid);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_create_ah_cmd(struct ionic_ibdev *dev,</div><div class='add'>+			       struct ionic_ah *ah,</div><div class='add'>+			       struct ionic_pd *pd,</div><div class='add'>+			       struct rdma_ah_attr *attr,</div><div class='add'>+			       u32 flags)</div><div class='add'>+{</div><div class='add'>+	struct ionic_admin_wr wr = {</div><div class='add'>+		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),</div><div class='add'>+		.wqe = {</div><div class='add'>+			.op = IONIC_V1_ADMIN_CREATE_AH,</div><div class='add'>+			.len = cpu_to_le16(IONIC_ADMIN_CREATE_AH_IN_V1_LEN),</div><div class='add'>+			.cmd.create_ah = {</div><div class='add'>+				.pd_id = cpu_to_le32(pd-&gt;pdid),</div><div class='add'>+				.dbid_flags = cpu_to_le16(dev-&gt;lif_cfg.dbid),</div><div class='add'>+				.id_ver = cpu_to_le32(ah-&gt;ahid),</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	};</div><div class='add'>+	enum ionic_admin_flags admin_flags = 0;</div><div class='add'>+	dma_addr_t hdr_dma = 0;</div><div class='add'>+	void *hdr_buf;</div><div class='add'>+	gfp_t gfp = GFP_ATOMIC;</div><div class='add'>+	int rc, hdr_len = 0;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;lif_cfg.admin_opcodes &lt;= IONIC_V1_ADMIN_CREATE_AH)</div><div class='add'>+		return -EBADRQC;</div><div class='add'>+</div><div class='add'>+	if (flags &amp; RDMA_CREATE_AH_SLEEPABLE)</div><div class='add'>+		gfp = GFP_KERNEL;</div><div class='add'>+	else</div><div class='add'>+		admin_flags |= IONIC_ADMIN_F_BUSYWAIT;</div><div class='add'>+</div><div class='add'>+	rc = ionic_build_hdr(dev, &amp;ah-&gt;hdr, attr, IONIC_ROCE_UDP_SPORT, false);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	if (ah-&gt;hdr.eth.type == cpu_to_be16(ETH_P_8021Q)) {</div><div class='add'>+		if (ah-&gt;hdr.vlan.type == cpu_to_be16(ETH_P_IP))</div><div class='add'>+			wr.wqe.cmd.create_ah.csum_profile =</div><div class='add'>+				IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP;</div><div class='add'>+		else</div><div class='add'>+			wr.wqe.cmd.create_ah.csum_profile =</div><div class='add'>+				IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV6_UDP;</div><div class='add'>+	} else {</div><div class='add'>+		if (ah-&gt;hdr.eth.type == cpu_to_be16(ETH_P_IP))</div><div class='add'>+			wr.wqe.cmd.create_ah.csum_profile =</div><div class='add'>+				IONIC_TFP_CSUM_PROF_ETH_IPV4_UDP;</div><div class='add'>+		else</div><div class='add'>+			wr.wqe.cmd.create_ah.csum_profile =</div><div class='add'>+				IONIC_TFP_CSUM_PROF_ETH_IPV6_UDP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ah-&gt;sgid_index = rdma_ah_read_grh(attr)-&gt;sgid_index;</div><div class='add'>+</div><div class='add'>+	hdr_buf = kmalloc(PAGE_SIZE, gfp);</div><div class='add'>+	if (!hdr_buf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	hdr_len = ib_ud_header_pack(&amp;ah-&gt;hdr, hdr_buf);</div><div class='add'>+	hdr_len -= IB_BTH_BYTES;</div><div class='add'>+	hdr_len -= IB_DETH_BYTES;</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ibdev, "roce packet header template\n");</div><div class='add'>+	print_hex_dump_debug("hdr ", DUMP_PREFIX_OFFSET, 16, 1,</div><div class='add'>+			     hdr_buf, hdr_len, true);</div><div class='add'>+</div><div class='add'>+	hdr_dma = dma_map_single(dev-&gt;lif_cfg.hwdev, hdr_buf, hdr_len,</div><div class='add'>+				 DMA_TO_DEVICE);</div><div class='add'>+</div><div class='add'>+	rc = dma_mapping_error(dev-&gt;lif_cfg.hwdev, hdr_dma);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_dma;</div><div class='add'>+</div><div class='add'>+	wr.wqe.cmd.create_ah.dma_addr = cpu_to_le64(hdr_dma);</div><div class='add'>+	wr.wqe.cmd.create_ah.length = cpu_to_le32(hdr_len);</div><div class='add'>+</div><div class='add'>+	ionic_admin_post(dev, &amp;wr);</div><div class='add'>+	rc = ionic_admin_wait(dev, &amp;wr, admin_flags);</div><div class='add'>+</div><div class='add'>+	dma_unmap_single(dev-&gt;lif_cfg.hwdev, hdr_dma, hdr_len,</div><div class='add'>+			 DMA_TO_DEVICE);</div><div class='add'>+err_dma:</div><div class='add'>+	kfree(hdr_buf);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_destroy_ah_cmd(struct ionic_ibdev *dev, u32 ahid, u32 flags)</div><div class='add'>+{</div><div class='add'>+	struct ionic_admin_wr wr = {</div><div class='add'>+		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),</div><div class='add'>+		.wqe = {</div><div class='add'>+			.op = IONIC_V1_ADMIN_DESTROY_AH,</div><div class='add'>+			.len = cpu_to_le16(IONIC_ADMIN_DESTROY_AH_IN_V1_LEN),</div><div class='add'>+			.cmd.destroy_ah = {</div><div class='add'>+				.ah_id = cpu_to_le32(ahid),</div><div class='add'>+			},</div><div class='add'>+		}</div><div class='add'>+	};</div><div class='add'>+	enum ionic_admin_flags admin_flags = IONIC_ADMIN_F_TEARDOWN;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;lif_cfg.admin_opcodes &lt;= IONIC_V1_ADMIN_DESTROY_AH)</div><div class='add'>+		return -EBADRQC;</div><div class='add'>+</div><div class='add'>+	if (!(flags &amp; RDMA_CREATE_AH_SLEEPABLE))</div><div class='add'>+		admin_flags |= IONIC_ADMIN_F_BUSYWAIT;</div><div class='add'>+</div><div class='add'>+	ionic_admin_post(dev, &amp;wr);</div><div class='add'>+	ionic_admin_wait(dev, &amp;wr, admin_flags);</div><div class='add'>+</div><div class='add'>+	/* No host-memory resource is associated with ah, so it is ok</div><div class='add'>+	 * to "succeed" and complete this destroy ah on the host.</div><div class='add'>+	 */</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,</div><div class='add'>+		    struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibah-&gt;device);</div><div class='add'>+	struct rdma_ah_attr *attr = init_attr-&gt;ah_attr;</div><div class='add'>+	struct ionic_pd *pd = to_ionic_pd(ibah-&gt;pd);</div><div class='add'>+	struct ionic_ah *ah = to_ionic_ah(ibah);</div><div class='add'>+	struct ionic_ah_resp resp = {};</div><div class='add'>+	u32 flags = init_attr-&gt;flags;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rc = ionic_get_ahid(dev, &amp;ah-&gt;ahid);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	rc = ionic_create_ah_cmd(dev, ah, pd, attr, flags);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_cmd;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		resp.ahid = ah-&gt;ahid;</div><div class='add'>+</div><div class='add'>+		rc = ib_copy_to_udata(udata, &amp;resp, sizeof(resp));</div><div class='add'>+		if (rc)</div><div class='add'>+			goto err_resp;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_resp:</div><div class='add'>+	ionic_destroy_ah_cmd(dev, ah-&gt;ahid, flags);</div><div class='add'>+err_cmd:</div><div class='add'>+	ionic_put_ahid(dev, ah-&gt;ahid);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibah-&gt;device);</div><div class='add'>+	struct ionic_ah *ah = to_ionic_ah(ibah);</div><div class='add'>+</div><div class='add'>+	ionic_set_ah_attr(dev, ah_attr, &amp;ah-&gt;hdr, ah-&gt;sgid_index);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_destroy_ah(struct ib_ah *ibah, u32 flags)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibah-&gt;device);</div><div class='add'>+	struct ionic_ah *ah = to_ionic_ah(ibah);</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rc = ionic_destroy_ah_cmd(dev, ah-&gt;ahid, flags);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	ionic_put_ahid(dev, ah-&gt;ahid);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_create_mr_cmd(struct ionic_ibdev *dev,</div><div class='add'>+			       struct ionic_pd *pd,</div><div class='add'>+			       struct ionic_mr *mr,</div><div class='add'>+			       u64 addr,</div><div class='add'>+			       u64 length)</div><div class='add'>+{</div><div class='add'>+	struct ionic_admin_wr wr = {</div><div class='add'>+		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),</div><div class='add'>+		.wqe = {</div><div class='add'>+			.op = IONIC_V1_ADMIN_CREATE_MR,</div><div class='add'>+			.len = cpu_to_le16(IONIC_ADMIN_CREATE_MR_IN_V1_LEN),</div><div class='add'>+			.cmd.create_mr = {</div><div class='add'>+				.va = cpu_to_le64(addr),</div><div class='add'>+				.length = cpu_to_le64(length),</div><div class='add'>+				.pd_id = cpu_to_le32(pd-&gt;pdid),</div><div class='add'>+				.page_size_log2 = mr-&gt;buf.page_size_log2,</div><div class='add'>+				.tbl_index = cpu_to_le32(~0),</div><div class='add'>+				.map_count = cpu_to_le32(mr-&gt;buf.tbl_pages),</div><div class='add'>+				.dma_addr = ionic_pgtbl_dma(&amp;mr-&gt;buf, addr),</div><div class='add'>+				.dbid_flags = cpu_to_le16(mr-&gt;flags),</div><div class='add'>+				.id_ver = cpu_to_le32(mr-&gt;mrid),</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	};</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;lif_cfg.admin_opcodes &lt;= IONIC_V1_ADMIN_CREATE_MR)</div><div class='add'>+		return -EBADRQC;</div><div class='add'>+</div><div class='add'>+	ionic_admin_post(dev, &amp;wr);</div><div class='add'>+	rc = ionic_admin_wait(dev, &amp;wr, 0);</div><div class='add'>+	if (!rc)</div><div class='add'>+		mr-&gt;created = true;</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_destroy_mr_cmd(struct ionic_ibdev *dev, u32 mrid)</div><div class='add'>+{</div><div class='add'>+	struct ionic_admin_wr wr = {</div><div class='add'>+		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),</div><div class='add'>+		.wqe = {</div><div class='add'>+			.op = IONIC_V1_ADMIN_DESTROY_MR,</div><div class='add'>+			.len = cpu_to_le16(IONIC_ADMIN_DESTROY_MR_IN_V1_LEN),</div><div class='add'>+			.cmd.destroy_mr = {</div><div class='add'>+				.mr_id = cpu_to_le32(mrid),</div><div class='add'>+			},</div><div class='add'>+		}</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;lif_cfg.admin_opcodes &lt;= IONIC_V1_ADMIN_DESTROY_MR)</div><div class='add'>+		return -EBADRQC;</div><div class='add'>+</div><div class='add'>+	ionic_admin_post(dev, &amp;wr);</div><div class='add'>+</div><div class='add'>+	return ionic_admin_wait(dev, &amp;wr, IONIC_ADMIN_F_TEARDOWN);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access)</div><div class='add'>+{</div><div class='add'>+	struct ionic_pd *pd = to_ionic_pd(ibpd);</div><div class='add'>+	struct ionic_mr *mr;</div><div class='add'>+</div><div class='add'>+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='add'>+	if (!mr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	mr-&gt;ibmr.lkey = IONIC_DMA_LKEY;</div><div class='add'>+	mr-&gt;ibmr.rkey = IONIC_DMA_RKEY;</div><div class='add'>+</div><div class='add'>+	if (pd)</div><div class='add'>+		pd-&gt;flags |= IONIC_QPF_PRIVILEGED;</div><div class='add'>+</div><div class='add'>+	return &amp;mr-&gt;ibmr;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,</div><div class='add'>+				u64 addr, int access, struct ib_dmah *dmah,</div><div class='add'>+				struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibpd-&gt;device);</div><div class='add'>+	struct ionic_pd *pd = to_ionic_pd(ibpd);</div><div class='add'>+	struct ionic_mr *mr;</div><div class='add'>+	unsigned long pg_sz;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (dmah)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='add'>+	if (!mr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	rc = ionic_get_mrid(dev, &amp;mr-&gt;mrid);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_mrid;</div><div class='add'>+</div><div class='add'>+	mr-&gt;ibmr.lkey = mr-&gt;mrid;</div><div class='add'>+	mr-&gt;ibmr.rkey = mr-&gt;mrid;</div><div class='add'>+	mr-&gt;ibmr.iova = addr;</div><div class='add'>+	mr-&gt;ibmr.length = length;</div><div class='add'>+</div><div class='add'>+	mr-&gt;flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access);</div><div class='add'>+</div><div class='add'>+	mr-&gt;umem = ib_umem_get(&amp;dev-&gt;ibdev, start, length, access);</div><div class='add'>+	if (IS_ERR(mr-&gt;umem)) {</div><div class='add'>+		rc = PTR_ERR(mr-&gt;umem);</div><div class='add'>+		goto err_umem;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	pg_sz = ib_umem_find_best_pgsz(mr-&gt;umem,</div><div class='add'>+				       dev-&gt;lif_cfg.page_size_supported,</div><div class='add'>+				       addr);</div><div class='add'>+	if (!pg_sz) {</div><div class='add'>+		rc = -EINVAL;</div><div class='add'>+		goto err_pgtbl;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rc = ionic_pgtbl_init(dev, &amp;mr-&gt;buf, mr-&gt;umem, 0, 1, pg_sz);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_pgtbl;</div><div class='add'>+</div><div class='add'>+	rc = ionic_create_mr_cmd(dev, pd, mr, addr, length);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_cmd;</div><div class='add'>+</div><div class='add'>+	ionic_pgtbl_unbuf(dev, &amp;mr-&gt;buf);</div><div class='add'>+</div><div class='add'>+	return &amp;mr-&gt;ibmr;</div><div class='add'>+</div><div class='add'>+err_cmd:</div><div class='add'>+	ionic_pgtbl_unbuf(dev, &amp;mr-&gt;buf);</div><div class='add'>+err_pgtbl:</div><div class='add'>+	ib_umem_release(mr-&gt;umem);</div><div class='add'>+err_umem:</div><div class='add'>+	ionic_put_mrid(dev, mr-&gt;mrid);</div><div class='add'>+err_mrid:</div><div class='add'>+	kfree(mr);</div><div class='add'>+	return ERR_PTR(rc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset,</div><div class='add'>+				       u64 length, u64 addr, int fd, int access,</div><div class='add'>+				       struct ib_dmah *dmah,</div><div class='add'>+				       struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibpd-&gt;device);</div><div class='add'>+	struct ionic_pd *pd = to_ionic_pd(ibpd);</div><div class='add'>+	struct ib_umem_dmabuf *umem_dmabuf;</div><div class='add'>+	struct ionic_mr *mr;</div><div class='add'>+	u64 pg_sz;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (dmah)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='add'>+	if (!mr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	rc = ionic_get_mrid(dev, &amp;mr-&gt;mrid);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_mrid;</div><div class='add'>+</div><div class='add'>+	mr-&gt;ibmr.lkey = mr-&gt;mrid;</div><div class='add'>+	mr-&gt;ibmr.rkey = mr-&gt;mrid;</div><div class='add'>+	mr-&gt;ibmr.iova = addr;</div><div class='add'>+	mr-&gt;ibmr.length = length;</div><div class='add'>+</div><div class='add'>+	mr-&gt;flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access);</div><div class='add'>+</div><div class='add'>+	umem_dmabuf = ib_umem_dmabuf_get_pinned(&amp;dev-&gt;ibdev, offset, length,</div><div class='add'>+						fd, access);</div><div class='add'>+	if (IS_ERR(umem_dmabuf)) {</div><div class='add'>+		rc = PTR_ERR(umem_dmabuf);</div><div class='add'>+		goto err_umem;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mr-&gt;umem = &amp;umem_dmabuf-&gt;umem;</div><div class='add'>+</div><div class='add'>+	pg_sz = ib_umem_find_best_pgsz(mr-&gt;umem,</div><div class='add'>+				       dev-&gt;lif_cfg.page_size_supported,</div><div class='add'>+				       addr);</div><div class='add'>+	if (!pg_sz) {</div><div class='add'>+		rc = -EINVAL;</div><div class='add'>+		goto err_pgtbl;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rc = ionic_pgtbl_init(dev, &amp;mr-&gt;buf, mr-&gt;umem, 0, 1, pg_sz);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_pgtbl;</div><div class='add'>+</div><div class='add'>+	rc = ionic_create_mr_cmd(dev, pd, mr, addr, length);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_cmd;</div><div class='add'>+</div><div class='add'>+	ionic_pgtbl_unbuf(dev, &amp;mr-&gt;buf);</div><div class='add'>+</div><div class='add'>+	return &amp;mr-&gt;ibmr;</div><div class='add'>+</div><div class='add'>+err_cmd:</div><div class='add'>+	ionic_pgtbl_unbuf(dev, &amp;mr-&gt;buf);</div><div class='add'>+err_pgtbl:</div><div class='add'>+	ib_umem_release(mr-&gt;umem);</div><div class='add'>+err_umem:</div><div class='add'>+	ionic_put_mrid(dev, mr-&gt;mrid);</div><div class='add'>+err_mrid:</div><div class='add'>+	kfree(mr);</div><div class='add'>+	return ERR_PTR(rc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibmr-&gt;device);</div><div class='add'>+	struct ionic_mr *mr = to_ionic_mr(ibmr);</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (!mr-&gt;ibmr.lkey)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	if (mr-&gt;created) {</div><div class='add'>+		rc = ionic_destroy_mr_cmd(dev, mr-&gt;mrid);</div><div class='add'>+		if (rc)</div><div class='add'>+			return rc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ionic_pgtbl_unbuf(dev, &amp;mr-&gt;buf);</div><div class='add'>+</div><div class='add'>+	if (mr-&gt;umem)</div><div class='add'>+		ib_umem_release(mr-&gt;umem);</div><div class='add'>+</div><div class='add'>+	ionic_put_mrid(dev, mr-&gt;mrid);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	kfree(mr);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type,</div><div class='add'>+			     u32 max_sg)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibpd-&gt;device);</div><div class='add'>+	struct ionic_pd *pd = to_ionic_pd(ibpd);</div><div class='add'>+	struct ionic_mr *mr;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (type != IB_MR_TYPE_MEM_REG)</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='add'>+	if (!mr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	rc = ionic_get_mrid(dev, &amp;mr-&gt;mrid);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_mrid;</div><div class='add'>+</div><div class='add'>+	mr-&gt;ibmr.lkey = mr-&gt;mrid;</div><div class='add'>+	mr-&gt;ibmr.rkey = mr-&gt;mrid;</div><div class='add'>+</div><div class='add'>+	mr-&gt;flags = IONIC_MRF_PHYS_MR;</div><div class='add'>+</div><div class='add'>+	rc = ionic_pgtbl_init(dev, &amp;mr-&gt;buf, mr-&gt;umem, 0, max_sg, PAGE_SIZE);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_pgtbl;</div><div class='add'>+</div><div class='add'>+	mr-&gt;buf.tbl_pages = 0;</div><div class='add'>+</div><div class='add'>+	rc = ionic_create_mr_cmd(dev, pd, mr, 0, 0);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_cmd;</div><div class='add'>+</div><div class='add'>+	return &amp;mr-&gt;ibmr;</div><div class='add'>+</div><div class='add'>+err_cmd:</div><div class='add'>+	ionic_pgtbl_unbuf(dev, &amp;mr-&gt;buf);</div><div class='add'>+err_pgtbl:</div><div class='add'>+	ionic_put_mrid(dev, mr-&gt;mrid);</div><div class='add'>+err_mrid:</div><div class='add'>+	kfree(mr);</div><div class='add'>+	return ERR_PTR(rc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_map_mr_page(struct ib_mr *ibmr, u64 dma)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibmr-&gt;device);</div><div class='add'>+	struct ionic_mr *mr = to_ionic_mr(ibmr);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ibdev, "dma %p\n", (void *)dma);</div><div class='add'>+	return ionic_pgtbl_page(&amp;mr-&gt;buf, dma);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,</div><div class='add'>+		    unsigned int *sg_offset)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibmr-&gt;device);</div><div class='add'>+	struct ionic_mr *mr = to_ionic_mr(ibmr);</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	/* mr must be allocated using ib_alloc_mr() */</div><div class='add'>+	if (unlikely(!mr-&gt;buf.tbl_limit))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	mr-&gt;buf.tbl_pages = 0;</div><div class='add'>+</div><div class='add'>+	if (mr-&gt;buf.tbl_buf)</div><div class='add'>+		dma_sync_single_for_cpu(dev-&gt;lif_cfg.hwdev, mr-&gt;buf.tbl_dma,</div><div class='add'>+					mr-&gt;buf.tbl_size, DMA_TO_DEVICE);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ibdev, "sg %p nent %d\n", sg, sg_nents);</div><div class='add'>+	rc = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ionic_map_mr_page);</div><div class='add'>+</div><div class='add'>+	mr-&gt;buf.page_size_log2 = order_base_2(ibmr-&gt;page_size);</div><div class='add'>+</div><div class='add'>+	if (mr-&gt;buf.tbl_buf)</div><div class='add'>+		dma_sync_single_for_device(dev-&gt;lif_cfg.hwdev, mr-&gt;buf.tbl_dma,</div><div class='add'>+					   mr-&gt;buf.tbl_size, DMA_TO_DEVICE);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibmw-&gt;device);</div><div class='add'>+	struct ionic_pd *pd = to_ionic_pd(ibmw-&gt;pd);</div><div class='add'>+	struct ionic_mr *mr = to_ionic_mw(ibmw);</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rc = ionic_get_mrid(dev, &amp;mr-&gt;mrid);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	mr-&gt;ibmw.rkey = mr-&gt;mrid;</div><div class='add'>+</div><div class='add'>+	if (mr-&gt;ibmw.type == IB_MW_TYPE_1)</div><div class='add'>+		mr-&gt;flags = IONIC_MRF_MW_1;</div><div class='add'>+	else</div><div class='add'>+		mr-&gt;flags = IONIC_MRF_MW_2;</div><div class='add'>+</div><div class='add'>+	rc = ionic_create_mr_cmd(dev, pd, mr, 0, 0);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_cmd;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_cmd:</div><div class='add'>+	ionic_put_mrid(dev, mr-&gt;mrid);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_dealloc_mw(struct ib_mw *ibmw)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibmw-&gt;device);</div><div class='add'>+	struct ionic_mr *mr = to_ionic_mw(ibmw);</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rc = ionic_destroy_mr_cmd(dev, mr-&gt;mrid);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	ionic_put_mrid(dev, mr-&gt;mrid);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_create_cq_cmd(struct ionic_ibdev *dev,</div><div class='add'>+			       struct ionic_ctx *ctx,</div><div class='add'>+			       struct ionic_cq *cq,</div><div class='add'>+			       struct ionic_tbl_buf *buf)</div><div class='add'>+{</div><div class='add'>+	const u16 dbid = ionic_ctx_dbid(dev, ctx);</div><div class='add'>+	struct ionic_admin_wr wr = {</div><div class='add'>+		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),</div><div class='add'>+		.wqe = {</div><div class='add'>+			.op = IONIC_V1_ADMIN_CREATE_CQ,</div><div class='add'>+			.len = cpu_to_le16(IONIC_ADMIN_CREATE_CQ_IN_V1_LEN),</div><div class='add'>+			.cmd.create_cq = {</div><div class='add'>+				.eq_id = cpu_to_le32(cq-&gt;eqid),</div><div class='add'>+				.depth_log2 = cq-&gt;q.depth_log2,</div><div class='add'>+				.stride_log2 = cq-&gt;q.stride_log2,</div><div class='add'>+				.page_size_log2 = buf-&gt;page_size_log2,</div><div class='add'>+				.tbl_index = cpu_to_le32(~0),</div><div class='add'>+				.map_count = cpu_to_le32(buf-&gt;tbl_pages),</div><div class='add'>+				.dma_addr = ionic_pgtbl_dma(buf, 0),</div><div class='add'>+				.dbid_flags = cpu_to_le16(dbid),</div><div class='add'>+				.id_ver = cpu_to_le32(cq-&gt;cqid),</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;lif_cfg.admin_opcodes &lt;= IONIC_V1_ADMIN_CREATE_CQ)</div><div class='add'>+		return -EBADRQC;</div><div class='add'>+</div><div class='add'>+	ionic_admin_post(dev, &amp;wr);</div><div class='add'>+</div><div class='add'>+	return ionic_admin_wait(dev, &amp;wr, 0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_destroy_cq_cmd(struct ionic_ibdev *dev, u32 cqid)</div><div class='add'>+{</div><div class='add'>+	struct ionic_admin_wr wr = {</div><div class='add'>+		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),</div><div class='add'>+		.wqe = {</div><div class='add'>+			.op = IONIC_V1_ADMIN_DESTROY_CQ,</div><div class='add'>+			.len = cpu_to_le16(IONIC_ADMIN_DESTROY_CQ_IN_V1_LEN),</div><div class='add'>+			.cmd.destroy_cq = {</div><div class='add'>+				.cq_id = cpu_to_le32(cqid),</div><div class='add'>+			},</div><div class='add'>+		}</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;lif_cfg.admin_opcodes &lt;= IONIC_V1_ADMIN_DESTROY_CQ)</div><div class='add'>+		return -EBADRQC;</div><div class='add'>+</div><div class='add'>+	ionic_admin_post(dev, &amp;wr);</div><div class='add'>+</div><div class='add'>+	return ionic_admin_wait(dev, &amp;wr, IONIC_ADMIN_F_TEARDOWN);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		    struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibcq-&gt;device);</div><div class='add'>+	struct ib_udata *udata = &amp;attrs-&gt;driver_udata;</div><div class='add'>+	struct ionic_ctx *ctx =</div><div class='add'>+		rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx);</div><div class='add'>+	struct ionic_vcq *vcq = to_ionic_vcq(ibcq);</div><div class='add'>+	struct ionic_tbl_buf buf = {};</div><div class='add'>+	struct ionic_cq_resp resp;</div><div class='add'>+	struct ionic_cq_req req;</div><div class='add'>+	int udma_idx = 0, rc;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		rc = ib_copy_from_udata(&amp;req, udata, sizeof(req));</div><div class='add'>+		if (rc)</div><div class='add'>+			return rc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	vcq-&gt;udma_mask = BIT(dev-&gt;lif_cfg.udma_count) - 1;</div><div class='add'>+</div><div class='add'>+	if (udata)</div><div class='add'>+		vcq-&gt;udma_mask &amp;= req.udma_mask;</div><div class='add'>+</div><div class='add'>+	if (!vcq-&gt;udma_mask) {</div><div class='add'>+		rc = -EINVAL;</div><div class='add'>+		goto err_init;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (; udma_idx &lt; dev-&gt;lif_cfg.udma_count; ++udma_idx) {</div><div class='add'>+		if (!(vcq-&gt;udma_mask &amp; BIT(udma_idx)))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		rc = ionic_create_cq_common(vcq, &amp;buf, attr, ctx, udata,</div><div class='add'>+					    &amp;req.cq[udma_idx],</div><div class='add'>+					    &amp;resp.cqid[udma_idx],</div><div class='add'>+					    udma_idx);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto err_init;</div><div class='add'>+</div><div class='add'>+		rc = ionic_create_cq_cmd(dev, ctx, &amp;vcq-&gt;cq[udma_idx], &amp;buf);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto err_cmd;</div><div class='add'>+</div><div class='add'>+		ionic_pgtbl_unbuf(dev, &amp;buf);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	vcq-&gt;ibcq.cqe = attr-&gt;cqe;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		resp.udma_mask = vcq-&gt;udma_mask;</div><div class='add'>+</div><div class='add'>+		rc = ib_copy_to_udata(udata, &amp;resp, sizeof(resp));</div><div class='add'>+		if (rc)</div><div class='add'>+			goto err_resp;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_resp:</div><div class='add'>+	while (udma_idx) {</div><div class='add'>+		--udma_idx;</div><div class='add'>+		if (!(vcq-&gt;udma_mask &amp; BIT(udma_idx)))</div><div class='add'>+			continue;</div><div class='add'>+		ionic_destroy_cq_cmd(dev, vcq-&gt;cq[udma_idx].cqid);</div><div class='add'>+err_cmd:</div><div class='add'>+		ionic_pgtbl_unbuf(dev, &amp;buf);</div><div class='add'>+		ionic_destroy_cq_common(dev, &amp;vcq-&gt;cq[udma_idx]);</div><div class='add'>+err_init:</div><div class='add'>+		;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibcq-&gt;device);</div><div class='add'>+	struct ionic_vcq *vcq = to_ionic_vcq(ibcq);</div><div class='add'>+	int udma_idx, rc_tmp, rc = 0;</div><div class='add'>+</div><div class='add'>+	for (udma_idx = dev-&gt;lif_cfg.udma_count; udma_idx; ) {</div><div class='add'>+		--udma_idx;</div><div class='add'>+</div><div class='add'>+		if (!(vcq-&gt;udma_mask &amp; BIT(udma_idx)))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		rc_tmp = ionic_destroy_cq_cmd(dev, vcq-&gt;cq[udma_idx].cqid);</div><div class='add'>+		if (rc_tmp) {</div><div class='add'>+			if (!rc)</div><div class='add'>+				rc = rc_tmp;</div><div class='add'>+</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ionic_destroy_cq_common(dev, &amp;vcq-&gt;cq[udma_idx]);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool pd_remote_privileged(struct ib_pd *pd)</div><div class='add'>+{</div><div class='add'>+	return pd-&gt;flags &amp; IB_PD_UNSAFE_GLOBAL_RKEY;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_create_qp_cmd(struct ionic_ibdev *dev,</div><div class='add'>+			       struct ionic_pd *pd,</div><div class='add'>+			       struct ionic_cq *send_cq,</div><div class='add'>+			       struct ionic_cq *recv_cq,</div><div class='add'>+			       struct ionic_qp *qp,</div><div class='add'>+			       struct ionic_tbl_buf *sq_buf,</div><div class='add'>+			       struct ionic_tbl_buf *rq_buf,</div><div class='add'>+			       struct ib_qp_init_attr *attr)</div><div class='add'>+{</div><div class='add'>+	const u16 dbid = ionic_obj_dbid(dev, pd-&gt;ibpd.uobject);</div><div class='add'>+	const u32 flags = to_ionic_qp_flags(0, 0,</div><div class='add'>+					    qp-&gt;sq_cmb &amp; IONIC_CMB_ENABLE,</div><div class='add'>+					    qp-&gt;rq_cmb &amp; IONIC_CMB_ENABLE,</div><div class='add'>+					    qp-&gt;sq_spec, qp-&gt;rq_spec,</div><div class='add'>+					    pd-&gt;flags &amp; IONIC_QPF_PRIVILEGED,</div><div class='add'>+					    pd_remote_privileged(&amp;pd-&gt;ibpd));</div><div class='add'>+	struct ionic_admin_wr wr = {</div><div class='add'>+		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),</div><div class='add'>+		.wqe = {</div><div class='add'>+			.op = IONIC_V1_ADMIN_CREATE_QP,</div><div class='add'>+			.len = cpu_to_le16(IONIC_ADMIN_CREATE_QP_IN_V1_LEN),</div><div class='add'>+			.cmd.create_qp = {</div><div class='add'>+				.pd_id = cpu_to_le32(pd-&gt;pdid),</div><div class='add'>+				.priv_flags = cpu_to_be32(flags),</div><div class='add'>+				.type_state = to_ionic_qp_type(attr-&gt;qp_type),</div><div class='add'>+				.dbid_flags = cpu_to_le16(dbid),</div><div class='add'>+				.id_ver = cpu_to_le32(qp-&gt;qpid),</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;lif_cfg.admin_opcodes &lt;= IONIC_V1_ADMIN_CREATE_QP)</div><div class='add'>+		return -EBADRQC;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;has_sq) {</div><div class='add'>+		wr.wqe.cmd.create_qp.sq_cq_id = cpu_to_le32(send_cq-&gt;cqid);</div><div class='add'>+		wr.wqe.cmd.create_qp.sq_depth_log2 = qp-&gt;sq.depth_log2;</div><div class='add'>+		wr.wqe.cmd.create_qp.sq_stride_log2 = qp-&gt;sq.stride_log2;</div><div class='add'>+		wr.wqe.cmd.create_qp.sq_page_size_log2 = sq_buf-&gt;page_size_log2;</div><div class='add'>+		wr.wqe.cmd.create_qp.sq_tbl_index_xrcd_id = cpu_to_le32(~0);</div><div class='add'>+		wr.wqe.cmd.create_qp.sq_map_count =</div><div class='add'>+			cpu_to_le32(sq_buf-&gt;tbl_pages);</div><div class='add'>+		wr.wqe.cmd.create_qp.sq_dma_addr = ionic_pgtbl_dma(sq_buf, 0);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;has_rq) {</div><div class='add'>+		wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq-&gt;cqid);</div><div class='add'>+		wr.wqe.cmd.create_qp.rq_depth_log2 = qp-&gt;rq.depth_log2;</div><div class='add'>+		wr.wqe.cmd.create_qp.rq_stride_log2 = qp-&gt;rq.stride_log2;</div><div class='add'>+		wr.wqe.cmd.create_qp.rq_page_size_log2 = rq_buf-&gt;page_size_log2;</div><div class='add'>+		wr.wqe.cmd.create_qp.rq_tbl_index_srq_id = cpu_to_le32(~0);</div><div class='add'>+		wr.wqe.cmd.create_qp.rq_map_count =</div><div class='add'>+			cpu_to_le32(rq_buf-&gt;tbl_pages);</div><div class='add'>+		wr.wqe.cmd.create_qp.rq_dma_addr = ionic_pgtbl_dma(rq_buf, 0);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ionic_admin_post(dev, &amp;wr);</div><div class='add'>+</div><div class='add'>+	return ionic_admin_wait(dev, &amp;wr, 0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_modify_qp_cmd(struct ionic_ibdev *dev,</div><div class='add'>+			       struct ionic_pd *pd,</div><div class='add'>+			       struct ionic_qp *qp,</div><div class='add'>+			       struct ib_qp_attr *attr,</div><div class='add'>+			       int mask)</div><div class='add'>+{</div><div class='add'>+	const u32 flags = to_ionic_qp_flags(attr-&gt;qp_access_flags,</div><div class='add'>+					    attr-&gt;en_sqd_async_notify,</div><div class='add'>+					    qp-&gt;sq_cmb &amp; IONIC_CMB_ENABLE,</div><div class='add'>+					    qp-&gt;rq_cmb &amp; IONIC_CMB_ENABLE,</div><div class='add'>+					    qp-&gt;sq_spec, qp-&gt;rq_spec,</div><div class='add'>+					    pd-&gt;flags &amp; IONIC_QPF_PRIVILEGED,</div><div class='add'>+					    pd_remote_privileged(qp-&gt;ibqp.pd));</div><div class='add'>+	const u8 state = to_ionic_qp_modify_state(attr-&gt;qp_state,</div><div class='add'>+						  attr-&gt;cur_qp_state);</div><div class='add'>+	struct ionic_admin_wr wr = {</div><div class='add'>+		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),</div><div class='add'>+		.wqe = {</div><div class='add'>+			.op = IONIC_V1_ADMIN_MODIFY_QP,</div><div class='add'>+			.len = cpu_to_le16(IONIC_ADMIN_MODIFY_QP_IN_V1_LEN),</div><div class='add'>+			.cmd.mod_qp = {</div><div class='add'>+				.attr_mask = cpu_to_be32(mask),</div><div class='add'>+				.access_flags = cpu_to_be16(flags),</div><div class='add'>+				.rq_psn = cpu_to_le32(attr-&gt;rq_psn),</div><div class='add'>+				.sq_psn = cpu_to_le32(attr-&gt;sq_psn),</div><div class='add'>+				.rate_limit_kbps =</div><div class='add'>+					cpu_to_le32(attr-&gt;rate_limit),</div><div class='add'>+				.pmtu = (attr-&gt;path_mtu + 7),</div><div class='add'>+				.retry = (attr-&gt;retry_cnt |</div><div class='add'>+					  (attr-&gt;rnr_retry &lt;&lt; 4)),</div><div class='add'>+				.rnr_timer = attr-&gt;min_rnr_timer,</div><div class='add'>+				.retry_timeout = attr-&gt;timeout,</div><div class='add'>+				.type_state = state,</div><div class='add'>+				.id_ver = cpu_to_le32(qp-&gt;qpid),</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	};</div><div class='add'>+	const struct ib_global_route *grh = rdma_ah_read_grh(&amp;attr-&gt;ah_attr);</div><div class='add'>+	void *hdr_buf = NULL;</div><div class='add'>+	dma_addr_t hdr_dma = 0;</div><div class='add'>+	int rc, hdr_len = 0;</div><div class='add'>+	u16 sport;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;lif_cfg.admin_opcodes &lt;= IONIC_V1_ADMIN_MODIFY_QP)</div><div class='add'>+		return -EBADRQC;</div><div class='add'>+</div><div class='add'>+	if ((mask &amp; IB_QP_MAX_DEST_RD_ATOMIC) &amp;&amp; attr-&gt;max_dest_rd_atomic) {</div><div class='add'>+		/* Note, round up/down was already done for allocating</div><div class='add'>+		 * resources on the device. The allocation order is in cache</div><div class='add'>+		 * line size.  We can't use the order of the resource</div><div class='add'>+		 * allocation to determine the order wqes here, because for</div><div class='add'>+		 * queue length &lt;= one cache line it is not distinct.</div><div class='add'>+		 *</div><div class='add'>+		 * Therefore, order wqes is computed again here.</div><div class='add'>+		 *</div><div class='add'>+		 * Account for hole and round up to the next order.</div><div class='add'>+		 */</div><div class='add'>+		wr.wqe.cmd.mod_qp.rsq_depth =</div><div class='add'>+			order_base_2(attr-&gt;max_dest_rd_atomic + 1);</div><div class='add'>+		wr.wqe.cmd.mod_qp.rsq_index = cpu_to_le32(~0);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if ((mask &amp; IB_QP_MAX_QP_RD_ATOMIC) &amp;&amp; attr-&gt;max_rd_atomic) {</div><div class='add'>+		/* Account for hole and round down to the next order */</div><div class='add'>+		wr.wqe.cmd.mod_qp.rrq_depth =</div><div class='add'>+			order_base_2(attr-&gt;max_rd_atomic + 2) - 1;</div><div class='add'>+		wr.wqe.cmd.mod_qp.rrq_index = cpu_to_le32(~0);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;ibqp.qp_type == IB_QPT_RC || qp-&gt;ibqp.qp_type == IB_QPT_UC)</div><div class='add'>+		wr.wqe.cmd.mod_qp.qkey_dest_qpn =</div><div class='add'>+			cpu_to_le32(attr-&gt;dest_qp_num);</div><div class='add'>+	else</div><div class='add'>+		wr.wqe.cmd.mod_qp.qkey_dest_qpn = cpu_to_le32(attr-&gt;qkey);</div><div class='add'>+</div><div class='add'>+	if (mask &amp; IB_QP_AV) {</div><div class='add'>+		if (!qp-&gt;hdr)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+</div><div class='add'>+		sport = rdma_get_udp_sport(grh-&gt;flow_label,</div><div class='add'>+					   qp-&gt;qpid,</div><div class='add'>+					   attr-&gt;dest_qp_num);</div><div class='add'>+</div><div class='add'>+		rc = ionic_build_hdr(dev, qp-&gt;hdr, &amp;attr-&gt;ah_attr, sport, true);</div><div class='add'>+		if (rc)</div><div class='add'>+			return rc;</div><div class='add'>+</div><div class='add'>+		qp-&gt;sgid_index = grh-&gt;sgid_index;</div><div class='add'>+</div><div class='add'>+		hdr_buf = kmalloc(PAGE_SIZE, GFP_KERNEL);</div><div class='add'>+		if (!hdr_buf)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+</div><div class='add'>+		hdr_len = ib_ud_header_pack(qp-&gt;hdr, hdr_buf);</div><div class='add'>+		hdr_len -= IB_BTH_BYTES;</div><div class='add'>+		hdr_len -= IB_DETH_BYTES;</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev, "roce packet header template\n");</div><div class='add'>+		print_hex_dump_debug("hdr ", DUMP_PREFIX_OFFSET, 16, 1,</div><div class='add'>+				     hdr_buf, hdr_len, true);</div><div class='add'>+</div><div class='add'>+		hdr_dma = dma_map_single(dev-&gt;lif_cfg.hwdev, hdr_buf, hdr_len,</div><div class='add'>+					 DMA_TO_DEVICE);</div><div class='add'>+</div><div class='add'>+		rc = dma_mapping_error(dev-&gt;lif_cfg.hwdev, hdr_dma);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto err_dma;</div><div class='add'>+</div><div class='add'>+		if (qp-&gt;hdr-&gt;ipv4_present) {</div><div class='add'>+			wr.wqe.cmd.mod_qp.tfp_csum_profile =</div><div class='add'>+				qp-&gt;hdr-&gt;vlan_present ?</div><div class='add'>+					IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP :</div><div class='add'>+					IONIC_TFP_CSUM_PROF_ETH_IPV4_UDP;</div><div class='add'>+		} else {</div><div class='add'>+			wr.wqe.cmd.mod_qp.tfp_csum_profile =</div><div class='add'>+				qp-&gt;hdr-&gt;vlan_present ?</div><div class='add'>+					IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV6_UDP :</div><div class='add'>+					IONIC_TFP_CSUM_PROF_ETH_IPV6_UDP;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		wr.wqe.cmd.mod_qp.ah_id_len =</div><div class='add'>+			cpu_to_le32(qp-&gt;ahid | (hdr_len &lt;&lt; 24));</div><div class='add'>+		wr.wqe.cmd.mod_qp.dma_addr = cpu_to_le64(hdr_dma);</div><div class='add'>+</div><div class='add'>+		wr.wqe.cmd.mod_qp.en_pcp = attr-&gt;ah_attr.sl;</div><div class='add'>+		wr.wqe.cmd.mod_qp.ip_dscp = grh-&gt;traffic_class &gt;&gt; 2;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ionic_admin_post(dev, &amp;wr);</div><div class='add'>+</div><div class='add'>+	rc = ionic_admin_wait(dev, &amp;wr, 0);</div><div class='add'>+</div><div class='add'>+	if (mask &amp; IB_QP_AV)</div><div class='add'>+		dma_unmap_single(dev-&gt;lif_cfg.hwdev, hdr_dma, hdr_len,</div><div class='add'>+				 DMA_TO_DEVICE);</div><div class='add'>+err_dma:</div><div class='add'>+	if (mask &amp; IB_QP_AV)</div><div class='add'>+		kfree(hdr_buf);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_query_qp_cmd(struct ionic_ibdev *dev,</div><div class='add'>+			      struct ionic_qp *qp,</div><div class='add'>+			      struct ib_qp_attr *attr,</div><div class='add'>+			      int mask)</div><div class='add'>+{</div><div class='add'>+	struct ionic_admin_wr wr = {</div><div class='add'>+		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),</div><div class='add'>+		.wqe = {</div><div class='add'>+			.op = IONIC_V1_ADMIN_QUERY_QP,</div><div class='add'>+			.len = cpu_to_le16(IONIC_ADMIN_QUERY_QP_IN_V1_LEN),</div><div class='add'>+			.cmd.query_qp = {</div><div class='add'>+				.id_ver = cpu_to_le32(qp-&gt;qpid),</div><div class='add'>+			},</div><div class='add'>+		}</div><div class='add'>+	};</div><div class='add'>+	struct ionic_v1_admin_query_qp_sq *query_sqbuf;</div><div class='add'>+	struct ionic_v1_admin_query_qp_rq *query_rqbuf;</div><div class='add'>+	dma_addr_t query_sqdma;</div><div class='add'>+	dma_addr_t query_rqdma;</div><div class='add'>+	dma_addr_t hdr_dma = 0;</div><div class='add'>+	void *hdr_buf = NULL;</div><div class='add'>+	int flags, rc;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;lif_cfg.admin_opcodes &lt;= IONIC_V1_ADMIN_QUERY_QP)</div><div class='add'>+		return -EBADRQC;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;has_sq) {</div><div class='add'>+		bool expdb = !!(qp-&gt;sq_cmb &amp; IONIC_CMB_EXPDB);</div><div class='add'>+</div><div class='add'>+		attr-&gt;cap.max_send_sge =</div><div class='add'>+			ionic_v1_send_wqe_max_sge(qp-&gt;sq.stride_log2,</div><div class='add'>+						  qp-&gt;sq_spec,</div><div class='add'>+						  expdb);</div><div class='add'>+		attr-&gt;cap.max_inline_data =</div><div class='add'>+			ionic_v1_send_wqe_max_data(qp-&gt;sq.stride_log2, expdb);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;has_rq) {</div><div class='add'>+		attr-&gt;cap.max_recv_sge =</div><div class='add'>+			ionic_v1_recv_wqe_max_sge(qp-&gt;rq.stride_log2,</div><div class='add'>+						  qp-&gt;rq_spec,</div><div class='add'>+						  qp-&gt;rq_cmb &amp; IONIC_CMB_EXPDB);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	query_sqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL);</div><div class='add'>+	if (!query_sqbuf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	query_rqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL);</div><div class='add'>+	if (!query_rqbuf) {</div><div class='add'>+		rc = -ENOMEM;</div><div class='add'>+		goto err_rqbuf;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	query_sqdma = dma_map_single(dev-&gt;lif_cfg.hwdev, query_sqbuf, PAGE_SIZE,</div><div class='add'>+				     DMA_FROM_DEVICE);</div><div class='add'>+	rc = dma_mapping_error(dev-&gt;lif_cfg.hwdev, query_sqdma);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_sqdma;</div><div class='add'>+</div><div class='add'>+	query_rqdma = dma_map_single(dev-&gt;lif_cfg.hwdev, query_rqbuf, PAGE_SIZE,</div><div class='add'>+				     DMA_FROM_DEVICE);</div><div class='add'>+	rc = dma_mapping_error(dev-&gt;lif_cfg.hwdev, query_rqdma);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_rqdma;</div><div class='add'>+</div><div class='add'>+	if (mask &amp; IB_QP_AV) {</div><div class='add'>+		hdr_buf = kmalloc(PAGE_SIZE, GFP_KERNEL);</div><div class='add'>+		if (!hdr_buf) {</div><div class='add'>+			rc = -ENOMEM;</div><div class='add'>+			goto err_hdrbuf;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		hdr_dma = dma_map_single(dev-&gt;lif_cfg.hwdev, hdr_buf,</div><div class='add'>+					 PAGE_SIZE, DMA_FROM_DEVICE);</div><div class='add'>+		rc = dma_mapping_error(dev-&gt;lif_cfg.hwdev, hdr_dma);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto err_hdrdma;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	wr.wqe.cmd.query_qp.sq_dma_addr = cpu_to_le64(query_sqdma);</div><div class='add'>+	wr.wqe.cmd.query_qp.rq_dma_addr = cpu_to_le64(query_rqdma);</div><div class='add'>+	wr.wqe.cmd.query_qp.hdr_dma_addr = cpu_to_le64(hdr_dma);</div><div class='add'>+	wr.wqe.cmd.query_qp.ah_id = cpu_to_le32(qp-&gt;ahid);</div><div class='add'>+</div><div class='add'>+	ionic_admin_post(dev, &amp;wr);</div><div class='add'>+</div><div class='add'>+	rc = ionic_admin_wait(dev, &amp;wr, 0);</div><div class='add'>+</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_hdrdma;</div><div class='add'>+</div><div class='add'>+	flags = be16_to_cpu(query_sqbuf-&gt;access_perms_flags |</div><div class='add'>+			    query_rqbuf-&gt;access_perms_flags);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("sqbuf ", DUMP_PREFIX_OFFSET, 16, 1,</div><div class='add'>+			     query_sqbuf, sizeof(*query_sqbuf), true);</div><div class='add'>+	print_hex_dump_debug("rqbuf ", DUMP_PREFIX_OFFSET, 16, 1,</div><div class='add'>+			     query_rqbuf, sizeof(*query_rqbuf), true);</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ibdev, "query qp %u state_pmtu %#x flags %#x",</div><div class='add'>+		  qp-&gt;qpid, query_rqbuf-&gt;state_pmtu, flags);</div><div class='add'>+</div><div class='add'>+	attr-&gt;qp_state = from_ionic_qp_state(query_rqbuf-&gt;state_pmtu &gt;&gt; 4);</div><div class='add'>+	attr-&gt;cur_qp_state = attr-&gt;qp_state;</div><div class='add'>+	attr-&gt;path_mtu = (query_rqbuf-&gt;state_pmtu &amp; 0xf) - 7;</div><div class='add'>+	attr-&gt;path_mig_state = IB_MIG_MIGRATED;</div><div class='add'>+	attr-&gt;qkey = be32_to_cpu(query_sqbuf-&gt;qkey_dest_qpn);</div><div class='add'>+	attr-&gt;rq_psn = be32_to_cpu(query_sqbuf-&gt;rq_psn);</div><div class='add'>+	attr-&gt;sq_psn = be32_to_cpu(query_rqbuf-&gt;sq_psn);</div><div class='add'>+	attr-&gt;dest_qp_num = attr-&gt;qkey;</div><div class='add'>+	attr-&gt;qp_access_flags = from_ionic_qp_flags(flags);</div><div class='add'>+	attr-&gt;pkey_index = 0;</div><div class='add'>+	attr-&gt;alt_pkey_index = 0;</div><div class='add'>+	attr-&gt;en_sqd_async_notify = !!(flags &amp; IONIC_QPF_SQD_NOTIFY);</div><div class='add'>+	attr-&gt;sq_draining = !!(flags &amp; IONIC_QPF_SQ_DRAINING);</div><div class='add'>+	attr-&gt;max_rd_atomic = BIT(query_rqbuf-&gt;rrq_depth) - 1;</div><div class='add'>+	attr-&gt;max_dest_rd_atomic = BIT(query_rqbuf-&gt;rsq_depth) - 1;</div><div class='add'>+	attr-&gt;min_rnr_timer = query_sqbuf-&gt;rnr_timer;</div><div class='add'>+	attr-&gt;port_num = 0;</div><div class='add'>+	attr-&gt;timeout = query_sqbuf-&gt;retry_timeout;</div><div class='add'>+	attr-&gt;retry_cnt = query_rqbuf-&gt;retry_rnrtry &amp; 0xf;</div><div class='add'>+	attr-&gt;rnr_retry = query_rqbuf-&gt;retry_rnrtry &gt;&gt; 4;</div><div class='add'>+	attr-&gt;alt_port_num = 0;</div><div class='add'>+	attr-&gt;alt_timeout = 0;</div><div class='add'>+	attr-&gt;rate_limit = be32_to_cpu(query_sqbuf-&gt;rate_limit_kbps);</div><div class='add'>+</div><div class='add'>+	if (mask &amp; IB_QP_AV)</div><div class='add'>+		ionic_set_ah_attr(dev, &amp;attr-&gt;ah_attr,</div><div class='add'>+				  qp-&gt;hdr, qp-&gt;sgid_index);</div><div class='add'>+</div><div class='add'>+err_hdrdma:</div><div class='add'>+	if (mask &amp; IB_QP_AV) {</div><div class='add'>+		dma_unmap_single(dev-&gt;lif_cfg.hwdev, hdr_dma,</div><div class='add'>+				 PAGE_SIZE, DMA_FROM_DEVICE);</div><div class='add'>+		kfree(hdr_buf);</div><div class='add'>+	}</div><div class='add'>+err_hdrbuf:</div><div class='add'>+	dma_unmap_single(dev-&gt;lif_cfg.hwdev, query_rqdma, sizeof(*query_rqbuf),</div><div class='add'>+			 DMA_FROM_DEVICE);</div><div class='add'>+err_rqdma:</div><div class='add'>+	dma_unmap_single(dev-&gt;lif_cfg.hwdev, query_sqdma, sizeof(*query_sqbuf),</div><div class='add'>+			 DMA_FROM_DEVICE);</div><div class='add'>+err_sqdma:</div><div class='add'>+	kfree(query_rqbuf);</div><div class='add'>+err_rqbuf:</div><div class='add'>+	kfree(query_sqbuf);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_destroy_qp_cmd(struct ionic_ibdev *dev, u32 qpid)</div><div class='add'>+{</div><div class='add'>+	struct ionic_admin_wr wr = {</div><div class='add'>+		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),</div><div class='add'>+		.wqe = {</div><div class='add'>+			.op = IONIC_V1_ADMIN_DESTROY_QP,</div><div class='add'>+			.len = cpu_to_le16(IONIC_ADMIN_DESTROY_QP_IN_V1_LEN),</div><div class='add'>+			.cmd.destroy_qp = {</div><div class='add'>+				.qp_id = cpu_to_le32(qpid),</div><div class='add'>+			},</div><div class='add'>+		}</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;lif_cfg.admin_opcodes &lt;= IONIC_V1_ADMIN_DESTROY_QP)</div><div class='add'>+		return -EBADRQC;</div><div class='add'>+</div><div class='add'>+	ionic_admin_post(dev, &amp;wr);</div><div class='add'>+</div><div class='add'>+	return ionic_admin_wait(dev, &amp;wr, IONIC_ADMIN_F_TEARDOWN);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool ionic_expdb_wqe_size_supported(struct ionic_ibdev *dev,</div><div class='add'>+					   uint32_t wqe_size)</div><div class='add'>+{</div><div class='add'>+	switch (wqe_size) {</div><div class='add'>+	case 64: return dev-&gt;lif_cfg.expdb_mask &amp; IONIC_EXPDB_64;</div><div class='add'>+	case 128: return dev-&gt;lif_cfg.expdb_mask &amp; IONIC_EXPDB_128;</div><div class='add'>+	case 256: return dev-&gt;lif_cfg.expdb_mask &amp; IONIC_EXPDB_256;</div><div class='add'>+	case 512: return dev-&gt;lif_cfg.expdb_mask &amp; IONIC_EXPDB_512;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_qp_sq_init_cmb(struct ionic_ibdev *dev,</div><div class='add'>+				 struct ionic_qp *qp,</div><div class='add'>+				 struct ib_udata *udata,</div><div class='add'>+				 int max_data)</div><div class='add'>+{</div><div class='add'>+	u8 expdb_stride_log2 = 0;</div><div class='add'>+	bool expdb;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (!(qp-&gt;sq_cmb &amp; IONIC_CMB_ENABLE))</div><div class='add'>+		goto not_in_cmb;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;sq_cmb &amp; ~IONIC_CMB_SUPPORTED) {</div><div class='add'>+		if (qp-&gt;sq_cmb &amp; IONIC_CMB_REQUIRE)</div><div class='add'>+			goto not_in_cmb;</div><div class='add'>+</div><div class='add'>+		qp-&gt;sq_cmb &amp;= IONIC_CMB_SUPPORTED;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if ((qp-&gt;sq_cmb &amp; IONIC_CMB_EXPDB) &amp;&amp; !dev-&gt;lif_cfg.sq_expdb) {</div><div class='add'>+		if (qp-&gt;sq_cmb &amp; IONIC_CMB_REQUIRE)</div><div class='add'>+			goto not_in_cmb;</div><div class='add'>+</div><div class='add'>+		qp-&gt;sq_cmb &amp;= ~IONIC_CMB_EXPDB;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	qp-&gt;sq_cmb_order = order_base_2(qp-&gt;sq.size / PAGE_SIZE);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;sq_cmb_order &gt;= IONIC_SQCMB_ORDER)</div><div class='add'>+		goto not_in_cmb;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;sq_cmb &amp; IONIC_CMB_EXPDB)</div><div class='add'>+		expdb_stride_log2 = qp-&gt;sq.stride_log2;</div><div class='add'>+</div><div class='add'>+	rc = ionic_get_cmb(dev-&gt;lif_cfg.lif, &amp;qp-&gt;sq_cmb_pgid,</div><div class='add'>+			   &amp;qp-&gt;sq_cmb_addr, qp-&gt;sq_cmb_order,</div><div class='add'>+			   expdb_stride_log2, &amp;expdb);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto not_in_cmb;</div><div class='add'>+</div><div class='add'>+	if ((qp-&gt;sq_cmb &amp; IONIC_CMB_EXPDB) &amp;&amp; !expdb) {</div><div class='add'>+		if (qp-&gt;sq_cmb &amp; IONIC_CMB_REQUIRE)</div><div class='add'>+			goto err_map;</div><div class='add'>+</div><div class='add'>+		qp-&gt;sq_cmb &amp;= ~IONIC_CMB_EXPDB;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return;</div><div class='add'>+</div><div class='add'>+err_map:</div><div class='add'>+	ionic_put_cmb(dev-&gt;lif_cfg.lif, qp-&gt;sq_cmb_pgid, qp-&gt;sq_cmb_order);</div><div class='add'>+not_in_cmb:</div><div class='add'>+	if (qp-&gt;sq_cmb &amp; IONIC_CMB_REQUIRE)</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev, "could not place sq in cmb as required\n");</div><div class='add'>+</div><div class='add'>+	qp-&gt;sq_cmb = 0;</div><div class='add'>+	qp-&gt;sq_cmb_order = IONIC_RES_INVALID;</div><div class='add'>+	qp-&gt;sq_cmb_pgid = 0;</div><div class='add'>+	qp-&gt;sq_cmb_addr = 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_qp_sq_destroy_cmb(struct ionic_ibdev *dev,</div><div class='add'>+				    struct ionic_ctx *ctx,</div><div class='add'>+				    struct ionic_qp *qp)</div><div class='add'>+{</div><div class='add'>+	if (!(qp-&gt;sq_cmb &amp; IONIC_CMB_ENABLE))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	if (ctx)</div><div class='add'>+		rdma_user_mmap_entry_remove(qp-&gt;mmap_sq_cmb);</div><div class='add'>+</div><div class='add'>+	ionic_put_cmb(dev-&gt;lif_cfg.lif, qp-&gt;sq_cmb_pgid, qp-&gt;sq_cmb_order);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_qp_sq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx,</div><div class='add'>+			    struct ionic_qp *qp, struct ionic_qdesc *sq,</div><div class='add'>+			    struct ionic_tbl_buf *buf, int max_wr, int max_sge,</div><div class='add'>+			    int max_data, int sq_spec, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	u32 wqe_size;</div><div class='add'>+	int rc = 0;</div><div class='add'>+</div><div class='add'>+	qp-&gt;sq_msn_prod = 0;</div><div class='add'>+	qp-&gt;sq_msn_cons = 0;</div><div class='add'>+</div><div class='add'>+	if (!qp-&gt;has_sq) {</div><div class='add'>+		if (buf) {</div><div class='add'>+			buf-&gt;tbl_buf = NULL;</div><div class='add'>+			buf-&gt;tbl_limit = 0;</div><div class='add'>+			buf-&gt;tbl_pages = 0;</div><div class='add'>+		}</div><div class='add'>+		if (udata)</div><div class='add'>+			rc = ionic_validate_qdesc_zero(sq);</div><div class='add'>+</div><div class='add'>+		return rc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rc = -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (max_wr &lt; 0 || max_wr &gt; 0xffff)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	if (max_sge &lt; 1)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	if (max_sge &gt; min(ionic_v1_send_wqe_max_sge(dev-&gt;lif_cfg.max_stride, 0,</div><div class='add'>+						    qp-&gt;sq_cmb &amp;</div><div class='add'>+						    IONIC_CMB_EXPDB),</div><div class='add'>+			  IONIC_SPEC_HIGH))</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	if (max_data &lt; 0)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	if (max_data &gt; ionic_v1_send_wqe_max_data(dev-&gt;lif_cfg.max_stride,</div><div class='add'>+						  qp-&gt;sq_cmb &amp; IONIC_CMB_EXPDB))</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		rc = ionic_validate_qdesc(sq);</div><div class='add'>+		if (rc)</div><div class='add'>+			return rc;</div><div class='add'>+</div><div class='add'>+		qp-&gt;sq_spec = sq_spec;</div><div class='add'>+</div><div class='add'>+		qp-&gt;sq.ptr = NULL;</div><div class='add'>+		qp-&gt;sq.size = sq-&gt;size;</div><div class='add'>+		qp-&gt;sq.mask = sq-&gt;mask;</div><div class='add'>+		qp-&gt;sq.depth_log2 = sq-&gt;depth_log2;</div><div class='add'>+		qp-&gt;sq.stride_log2 = sq-&gt;stride_log2;</div><div class='add'>+</div><div class='add'>+		qp-&gt;sq_meta = NULL;</div><div class='add'>+		qp-&gt;sq_msn_idx = NULL;</div><div class='add'>+</div><div class='add'>+		qp-&gt;sq_umem = ib_umem_get(&amp;dev-&gt;ibdev, sq-&gt;addr, sq-&gt;size, 0);</div><div class='add'>+		if (IS_ERR(qp-&gt;sq_umem))</div><div class='add'>+			return PTR_ERR(qp-&gt;sq_umem);</div><div class='add'>+	} else {</div><div class='add'>+		qp-&gt;sq_umem = NULL;</div><div class='add'>+</div><div class='add'>+		qp-&gt;sq_spec = ionic_v1_use_spec_sge(max_sge, sq_spec);</div><div class='add'>+		if (sq_spec &amp;&amp; !qp-&gt;sq_spec)</div><div class='add'>+			ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='add'>+				  "init sq: max_sge %u disables spec\n",</div><div class='add'>+				  max_sge);</div><div class='add'>+</div><div class='add'>+		if (qp-&gt;sq_cmb &amp; IONIC_CMB_EXPDB) {</div><div class='add'>+			wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data,</div><div class='add'>+							      qp-&gt;sq_spec,</div><div class='add'>+							      true);</div><div class='add'>+</div><div class='add'>+			if (!ionic_expdb_wqe_size_supported(dev, wqe_size))</div><div class='add'>+				qp-&gt;sq_cmb &amp;= ~IONIC_CMB_EXPDB;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (!(qp-&gt;sq_cmb &amp; IONIC_CMB_EXPDB))</div><div class='add'>+			wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data,</div><div class='add'>+							      qp-&gt;sq_spec,</div><div class='add'>+							      false);</div><div class='add'>+</div><div class='add'>+		rc = ionic_queue_init(&amp;qp-&gt;sq, dev-&gt;lif_cfg.hwdev,</div><div class='add'>+				      max_wr, wqe_size);</div><div class='add'>+		if (rc)</div><div class='add'>+			return rc;</div><div class='add'>+</div><div class='add'>+		ionic_queue_dbell_init(&amp;qp-&gt;sq, qp-&gt;qpid);</div><div class='add'>+</div><div class='add'>+		qp-&gt;sq_meta = kmalloc_array((u32)qp-&gt;sq.mask + 1,</div><div class='add'>+					    sizeof(*qp-&gt;sq_meta),</div><div class='add'>+					    GFP_KERNEL);</div><div class='add'>+		if (!qp-&gt;sq_meta) {</div><div class='add'>+			rc = -ENOMEM;</div><div class='add'>+			goto err_sq_meta;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		qp-&gt;sq_msn_idx = kmalloc_array((u32)qp-&gt;sq.mask + 1,</div><div class='add'>+					       sizeof(*qp-&gt;sq_msn_idx),</div><div class='add'>+					       GFP_KERNEL);</div><div class='add'>+		if (!qp-&gt;sq_msn_idx) {</div><div class='add'>+			rc = -ENOMEM;</div><div class='add'>+			goto err_sq_msn;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ionic_qp_sq_init_cmb(dev, qp, udata, max_data);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;sq_cmb &amp; IONIC_CMB_ENABLE)</div><div class='add'>+		rc = ionic_pgtbl_init(dev, buf, NULL,</div><div class='add'>+				      (u64)qp-&gt;sq_cmb_pgid &lt;&lt; PAGE_SHIFT,</div><div class='add'>+				      1, PAGE_SIZE);</div><div class='add'>+	else</div><div class='add'>+		rc = ionic_pgtbl_init(dev, buf,</div><div class='add'>+				      qp-&gt;sq_umem, qp-&gt;sq.dma, 1, PAGE_SIZE);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_sq_tbl;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_sq_tbl:</div><div class='add'>+	ionic_qp_sq_destroy_cmb(dev, ctx, qp);</div><div class='add'>+	kfree(qp-&gt;sq_msn_idx);</div><div class='add'>+err_sq_msn:</div><div class='add'>+	kfree(qp-&gt;sq_meta);</div><div class='add'>+err_sq_meta:</div><div class='add'>+	if (qp-&gt;sq_umem)</div><div class='add'>+		ib_umem_release(qp-&gt;sq_umem);</div><div class='add'>+	else</div><div class='add'>+		ionic_queue_destroy(&amp;qp-&gt;sq, dev-&gt;lif_cfg.hwdev);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_qp_sq_destroy(struct ionic_ibdev *dev,</div><div class='add'>+				struct ionic_ctx *ctx,</div><div class='add'>+				struct ionic_qp *qp)</div><div class='add'>+{</div><div class='add'>+	if (!qp-&gt;has_sq)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	ionic_qp_sq_destroy_cmb(dev, ctx, qp);</div><div class='add'>+</div><div class='add'>+	kfree(qp-&gt;sq_msn_idx);</div><div class='add'>+	kfree(qp-&gt;sq_meta);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;sq_umem)</div><div class='add'>+		ib_umem_release(qp-&gt;sq_umem);</div><div class='add'>+	else</div><div class='add'>+		ionic_queue_destroy(&amp;qp-&gt;sq, dev-&gt;lif_cfg.hwdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_qp_rq_init_cmb(struct ionic_ibdev *dev,</div><div class='add'>+				 struct ionic_qp *qp,</div><div class='add'>+				 struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	u8 expdb_stride_log2 = 0;</div><div class='add'>+	bool expdb;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (!(qp-&gt;rq_cmb &amp; IONIC_CMB_ENABLE))</div><div class='add'>+		goto not_in_cmb;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;rq_cmb &amp; ~IONIC_CMB_SUPPORTED) {</div><div class='add'>+		if (qp-&gt;rq_cmb &amp; IONIC_CMB_REQUIRE)</div><div class='add'>+			goto not_in_cmb;</div><div class='add'>+</div><div class='add'>+		qp-&gt;rq_cmb &amp;= IONIC_CMB_SUPPORTED;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if ((qp-&gt;rq_cmb &amp; IONIC_CMB_EXPDB) &amp;&amp; !dev-&gt;lif_cfg.rq_expdb) {</div><div class='add'>+		if (qp-&gt;rq_cmb &amp; IONIC_CMB_REQUIRE)</div><div class='add'>+			goto not_in_cmb;</div><div class='add'>+</div><div class='add'>+		qp-&gt;rq_cmb &amp;= ~IONIC_CMB_EXPDB;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	qp-&gt;rq_cmb_order = order_base_2(qp-&gt;rq.size / PAGE_SIZE);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;rq_cmb_order &gt;= IONIC_RQCMB_ORDER)</div><div class='add'>+		goto not_in_cmb;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;rq_cmb &amp; IONIC_CMB_EXPDB)</div><div class='add'>+		expdb_stride_log2 = qp-&gt;rq.stride_log2;</div><div class='add'>+</div><div class='add'>+	rc = ionic_get_cmb(dev-&gt;lif_cfg.lif, &amp;qp-&gt;rq_cmb_pgid,</div><div class='add'>+			   &amp;qp-&gt;rq_cmb_addr, qp-&gt;rq_cmb_order,</div><div class='add'>+			   expdb_stride_log2, &amp;expdb);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto not_in_cmb;</div><div class='add'>+</div><div class='add'>+	if ((qp-&gt;rq_cmb &amp; IONIC_CMB_EXPDB) &amp;&amp; !expdb) {</div><div class='add'>+		if (qp-&gt;rq_cmb &amp; IONIC_CMB_REQUIRE)</div><div class='add'>+			goto err_map;</div><div class='add'>+</div><div class='add'>+		qp-&gt;rq_cmb &amp;= ~IONIC_CMB_EXPDB;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return;</div><div class='add'>+</div><div class='add'>+err_map:</div><div class='add'>+	ionic_put_cmb(dev-&gt;lif_cfg.lif, qp-&gt;rq_cmb_pgid, qp-&gt;rq_cmb_order);</div><div class='add'>+not_in_cmb:</div><div class='add'>+	if (qp-&gt;rq_cmb &amp; IONIC_CMB_REQUIRE)</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev, "could not place rq in cmb as required\n");</div><div class='add'>+</div><div class='add'>+	qp-&gt;rq_cmb = 0;</div><div class='add'>+	qp-&gt;rq_cmb_order = IONIC_RES_INVALID;</div><div class='add'>+	qp-&gt;rq_cmb_pgid = 0;</div><div class='add'>+	qp-&gt;rq_cmb_addr = 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_qp_rq_destroy_cmb(struct ionic_ibdev *dev,</div><div class='add'>+				    struct ionic_ctx *ctx,</div><div class='add'>+				    struct ionic_qp *qp)</div><div class='add'>+{</div><div class='add'>+	if (!(qp-&gt;rq_cmb &amp; IONIC_CMB_ENABLE))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	if (ctx)</div><div class='add'>+		rdma_user_mmap_entry_remove(qp-&gt;mmap_rq_cmb);</div><div class='add'>+</div><div class='add'>+	ionic_put_cmb(dev-&gt;lif_cfg.lif, qp-&gt;rq_cmb_pgid, qp-&gt;rq_cmb_order);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_qp_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx,</div><div class='add'>+			    struct ionic_qp *qp, struct ionic_qdesc *rq,</div><div class='add'>+			    struct ionic_tbl_buf *buf, int max_wr, int max_sge,</div><div class='add'>+			    int rq_spec, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	int rc = 0, i;</div><div class='add'>+	u32 wqe_size;</div><div class='add'>+</div><div class='add'>+	if (!qp-&gt;has_rq) {</div><div class='add'>+		if (buf) {</div><div class='add'>+			buf-&gt;tbl_buf = NULL;</div><div class='add'>+			buf-&gt;tbl_limit = 0;</div><div class='add'>+			buf-&gt;tbl_pages = 0;</div><div class='add'>+		}</div><div class='add'>+		if (udata)</div><div class='add'>+			rc = ionic_validate_qdesc_zero(rq);</div><div class='add'>+</div><div class='add'>+		return rc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rc = -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (max_wr &lt; 0 || max_wr &gt; 0xffff)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	if (max_sge &lt; 1)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	if (max_sge &gt; min(ionic_v1_recv_wqe_max_sge(dev-&gt;lif_cfg.max_stride, 0, false),</div><div class='add'>+			  IONIC_SPEC_HIGH))</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		rc = ionic_validate_qdesc(rq);</div><div class='add'>+		if (rc)</div><div class='add'>+			return rc;</div><div class='add'>+</div><div class='add'>+		qp-&gt;rq_spec = rq_spec;</div><div class='add'>+</div><div class='add'>+		qp-&gt;rq.ptr = NULL;</div><div class='add'>+		qp-&gt;rq.size = rq-&gt;size;</div><div class='add'>+		qp-&gt;rq.mask = rq-&gt;mask;</div><div class='add'>+		qp-&gt;rq.depth_log2 = rq-&gt;depth_log2;</div><div class='add'>+		qp-&gt;rq.stride_log2 = rq-&gt;stride_log2;</div><div class='add'>+</div><div class='add'>+		qp-&gt;rq_meta = NULL;</div><div class='add'>+</div><div class='add'>+		qp-&gt;rq_umem = ib_umem_get(&amp;dev-&gt;ibdev, rq-&gt;addr, rq-&gt;size, 0);</div><div class='add'>+		if (IS_ERR(qp-&gt;rq_umem))</div><div class='add'>+			return PTR_ERR(qp-&gt;rq_umem);</div><div class='add'>+	} else {</div><div class='add'>+		qp-&gt;rq_umem = NULL;</div><div class='add'>+</div><div class='add'>+		qp-&gt;rq_spec = ionic_v1_use_spec_sge(max_sge, rq_spec);</div><div class='add'>+		if (rq_spec &amp;&amp; !qp-&gt;rq_spec)</div><div class='add'>+			ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='add'>+				  "init rq: max_sge %u disables spec\n",</div><div class='add'>+				  max_sge);</div><div class='add'>+</div><div class='add'>+		if (qp-&gt;rq_cmb &amp; IONIC_CMB_EXPDB) {</div><div class='add'>+			wqe_size = ionic_v1_recv_wqe_min_size(max_sge,</div><div class='add'>+							      qp-&gt;rq_spec,</div><div class='add'>+							      true);</div><div class='add'>+</div><div class='add'>+			if (!ionic_expdb_wqe_size_supported(dev, wqe_size))</div><div class='add'>+				qp-&gt;rq_cmb &amp;= ~IONIC_CMB_EXPDB;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (!(qp-&gt;rq_cmb &amp; IONIC_CMB_EXPDB))</div><div class='add'>+			wqe_size = ionic_v1_recv_wqe_min_size(max_sge,</div><div class='add'>+							      qp-&gt;rq_spec,</div><div class='add'>+							      false);</div><div class='add'>+</div><div class='add'>+		rc = ionic_queue_init(&amp;qp-&gt;rq, dev-&gt;lif_cfg.hwdev,</div><div class='add'>+				      max_wr, wqe_size);</div><div class='add'>+		if (rc)</div><div class='add'>+			return rc;</div><div class='add'>+</div><div class='add'>+		ionic_queue_dbell_init(&amp;qp-&gt;rq, qp-&gt;qpid);</div><div class='add'>+</div><div class='add'>+		qp-&gt;rq_meta = kmalloc_array((u32)qp-&gt;rq.mask + 1,</div><div class='add'>+					    sizeof(*qp-&gt;rq_meta),</div><div class='add'>+					    GFP_KERNEL);</div><div class='add'>+		if (!qp-&gt;rq_meta) {</div><div class='add'>+			rc = -ENOMEM;</div><div class='add'>+			goto err_rq_meta;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		for (i = 0; i &lt; qp-&gt;rq.mask; ++i)</div><div class='add'>+			qp-&gt;rq_meta[i].next = &amp;qp-&gt;rq_meta[i + 1];</div><div class='add'>+		qp-&gt;rq_meta[i].next = IONIC_META_LAST;</div><div class='add'>+		qp-&gt;rq_meta_head = &amp;qp-&gt;rq_meta[0];</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ionic_qp_rq_init_cmb(dev, qp, udata);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;rq_cmb &amp; IONIC_CMB_ENABLE)</div><div class='add'>+		rc = ionic_pgtbl_init(dev, buf, NULL,</div><div class='add'>+				      (u64)qp-&gt;rq_cmb_pgid &lt;&lt; PAGE_SHIFT,</div><div class='add'>+				      1, PAGE_SIZE);</div><div class='add'>+	else</div><div class='add'>+		rc = ionic_pgtbl_init(dev, buf,</div><div class='add'>+				      qp-&gt;rq_umem, qp-&gt;rq.dma, 1, PAGE_SIZE);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_rq_tbl;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_rq_tbl:</div><div class='add'>+	ionic_qp_rq_destroy_cmb(dev, ctx, qp);</div><div class='add'>+	kfree(qp-&gt;rq_meta);</div><div class='add'>+err_rq_meta:</div><div class='add'>+	if (qp-&gt;rq_umem)</div><div class='add'>+		ib_umem_release(qp-&gt;rq_umem);</div><div class='add'>+	else</div><div class='add'>+		ionic_queue_destroy(&amp;qp-&gt;rq, dev-&gt;lif_cfg.hwdev);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_qp_rq_destroy(struct ionic_ibdev *dev,</div><div class='add'>+				struct ionic_ctx *ctx,</div><div class='add'>+				struct ionic_qp *qp)</div><div class='add'>+{</div><div class='add'>+	if (!qp-&gt;has_rq)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	ionic_qp_rq_destroy_cmb(dev, ctx, qp);</div><div class='add'>+</div><div class='add'>+	kfree(qp-&gt;rq_meta);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;rq_umem)</div><div class='add'>+		ib_umem_release(qp-&gt;rq_umem);</div><div class='add'>+	else</div><div class='add'>+		ionic_queue_destroy(&amp;qp-&gt;rq, dev-&gt;lif_cfg.hwdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,</div><div class='add'>+		    struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibqp-&gt;device);</div><div class='add'>+	struct ionic_tbl_buf sq_buf = {}, rq_buf = {};</div><div class='add'>+	struct ionic_pd *pd = to_ionic_pd(ibqp-&gt;pd);</div><div class='add'>+	struct ionic_qp *qp = to_ionic_qp(ibqp);</div><div class='add'>+	struct ionic_ctx *ctx =</div><div class='add'>+		rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx);</div><div class='add'>+	struct ionic_qp_resp resp = {};</div><div class='add'>+	struct ionic_qp_req req = {};</div><div class='add'>+	struct ionic_cq *cq;</div><div class='add'>+	u8 udma_mask;</div><div class='add'>+	void *entry;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		rc = ib_copy_from_udata(&amp;req, udata, sizeof(req));</div><div class='add'>+		if (rc)</div><div class='add'>+			return rc;</div><div class='add'>+	} else {</div><div class='add'>+		req.sq_spec = IONIC_SPEC_HIGH;</div><div class='add'>+		req.rq_spec = IONIC_SPEC_HIGH;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;qp_type == IB_QPT_SMI || attr-&gt;qp_type &gt; IB_QPT_UD)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	qp-&gt;state = IB_QPS_RESET;</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;qp-&gt;cq_poll_sq);</div><div class='add'>+	INIT_LIST_HEAD(&amp;qp-&gt;cq_flush_sq);</div><div class='add'>+	INIT_LIST_HEAD(&amp;qp-&gt;cq_flush_rq);</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;qp-&gt;sq_lock);</div><div class='add'>+	spin_lock_init(&amp;qp-&gt;rq_lock);</div><div class='add'>+</div><div class='add'>+	qp-&gt;has_sq = 1;</div><div class='add'>+	qp-&gt;has_rq = 1;</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;qp_type == IB_QPT_GSI) {</div><div class='add'>+		rc = ionic_get_gsi_qpid(dev, &amp;qp-&gt;qpid);</div><div class='add'>+	} else {</div><div class='add'>+		udma_mask = BIT(dev-&gt;lif_cfg.udma_count) - 1;</div><div class='add'>+</div><div class='add'>+		if (qp-&gt;has_sq)</div><div class='add'>+			udma_mask &amp;= to_ionic_vcq(attr-&gt;send_cq)-&gt;udma_mask;</div><div class='add'>+</div><div class='add'>+		if (qp-&gt;has_rq)</div><div class='add'>+			udma_mask &amp;= to_ionic_vcq(attr-&gt;recv_cq)-&gt;udma_mask;</div><div class='add'>+</div><div class='add'>+		if (udata &amp;&amp; req.udma_mask)</div><div class='add'>+			udma_mask &amp;= req.udma_mask;</div><div class='add'>+</div><div class='add'>+		if (!udma_mask)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		rc = ionic_get_qpid(dev, &amp;qp-&gt;qpid, &amp;qp-&gt;udma_idx, udma_mask);</div><div class='add'>+	}</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	qp-&gt;sig_all = attr-&gt;sq_sig_type == IB_SIGNAL_ALL_WR;</div><div class='add'>+	qp-&gt;has_ah = attr-&gt;qp_type == IB_QPT_RC;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;has_ah) {</div><div class='add'>+		qp-&gt;hdr = kzalloc(sizeof(*qp-&gt;hdr), GFP_KERNEL);</div><div class='add'>+		if (!qp-&gt;hdr) {</div><div class='add'>+			rc = -ENOMEM;</div><div class='add'>+			goto err_ah_alloc;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		rc = ionic_get_ahid(dev, &amp;qp-&gt;ahid);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto err_ahid;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		if (req.rq_cmb &amp; IONIC_CMB_ENABLE)</div><div class='add'>+			qp-&gt;rq_cmb = req.rq_cmb;</div><div class='add'>+</div><div class='add'>+		if (req.sq_cmb &amp; IONIC_CMB_ENABLE)</div><div class='add'>+			qp-&gt;sq_cmb = req.sq_cmb;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rc = ionic_qp_sq_init(dev, ctx, qp, &amp;req.sq, &amp;sq_buf,</div><div class='add'>+			      attr-&gt;cap.max_send_wr, attr-&gt;cap.max_send_sge,</div><div class='add'>+			      attr-&gt;cap.max_inline_data, req.sq_spec, udata);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_sq;</div><div class='add'>+</div><div class='add'>+	rc = ionic_qp_rq_init(dev, ctx, qp, &amp;req.rq, &amp;rq_buf,</div><div class='add'>+			      attr-&gt;cap.max_recv_wr, attr-&gt;cap.max_recv_sge,</div><div class='add'>+			      req.rq_spec, udata);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_rq;</div><div class='add'>+</div><div class='add'>+	rc = ionic_create_qp_cmd(dev, pd,</div><div class='add'>+				 to_ionic_vcq_cq(attr-&gt;send_cq, qp-&gt;udma_idx),</div><div class='add'>+				 to_ionic_vcq_cq(attr-&gt;recv_cq, qp-&gt;udma_idx),</div><div class='add'>+				 qp, &amp;sq_buf, &amp;rq_buf, attr);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_cmd;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		resp.qpid = qp-&gt;qpid;</div><div class='add'>+		resp.udma_idx = qp-&gt;udma_idx;</div><div class='add'>+</div><div class='add'>+		if (qp-&gt;sq_cmb &amp; IONIC_CMB_ENABLE) {</div><div class='add'>+			bool wc;</div><div class='add'>+</div><div class='add'>+			if ((qp-&gt;sq_cmb &amp; (IONIC_CMB_WC | IONIC_CMB_UC)) ==</div><div class='add'>+				(IONIC_CMB_WC | IONIC_CMB_UC)) {</div><div class='add'>+				ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='add'>+					  "Both sq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC are set, using default driver mapping\n");</div><div class='add'>+				qp-&gt;sq_cmb &amp;= ~(IONIC_CMB_WC | IONIC_CMB_UC);</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			wc = (qp-&gt;sq_cmb &amp; (IONIC_CMB_WC | IONIC_CMB_UC))</div><div class='add'>+					!= IONIC_CMB_UC;</div><div class='add'>+</div><div class='add'>+			/* let userspace know the mapping */</div><div class='add'>+			if (wc)</div><div class='add'>+				qp-&gt;sq_cmb |= IONIC_CMB_WC;</div><div class='add'>+			else</div><div class='add'>+				qp-&gt;sq_cmb |= IONIC_CMB_UC;</div><div class='add'>+</div><div class='add'>+			qp-&gt;mmap_sq_cmb =</div><div class='add'>+			    ionic_mmap_entry_insert(ctx,</div><div class='add'>+						    qp-&gt;sq.size,</div><div class='add'>+						    PHYS_PFN(qp-&gt;sq_cmb_addr),</div><div class='add'>+						    wc ? IONIC_MMAP_WC : 0,</div><div class='add'>+						    &amp;resp.sq_cmb_offset);</div><div class='add'>+			if (!qp-&gt;mmap_sq_cmb) {</div><div class='add'>+				rc = -ENOMEM;</div><div class='add'>+				goto err_mmap_sq;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			resp.sq_cmb = qp-&gt;sq_cmb;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (qp-&gt;rq_cmb &amp; IONIC_CMB_ENABLE) {</div><div class='add'>+			bool wc;</div><div class='add'>+</div><div class='add'>+			if ((qp-&gt;rq_cmb &amp; (IONIC_CMB_WC | IONIC_CMB_UC)) ==</div><div class='add'>+				(IONIC_CMB_WC | IONIC_CMB_UC)) {</div><div class='add'>+				ibdev_dbg(&amp;dev-&gt;ibdev,</div><div class='add'>+					  "Both rq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC are set, using default driver mapping\n");</div><div class='add'>+				qp-&gt;rq_cmb &amp;= ~(IONIC_CMB_WC | IONIC_CMB_UC);</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			if (qp-&gt;rq_cmb &amp; IONIC_CMB_EXPDB)</div><div class='add'>+				wc = (qp-&gt;rq_cmb &amp; (IONIC_CMB_WC | IONIC_CMB_UC))</div><div class='add'>+					== IONIC_CMB_WC;</div><div class='add'>+			else</div><div class='add'>+				wc = (qp-&gt;rq_cmb &amp; (IONIC_CMB_WC | IONIC_CMB_UC))</div><div class='add'>+					!= IONIC_CMB_UC;</div><div class='add'>+</div><div class='add'>+			/* let userspace know the mapping */</div><div class='add'>+			if (wc)</div><div class='add'>+				qp-&gt;rq_cmb |= IONIC_CMB_WC;</div><div class='add'>+			else</div><div class='add'>+				qp-&gt;rq_cmb |= IONIC_CMB_UC;</div><div class='add'>+</div><div class='add'>+			qp-&gt;mmap_rq_cmb =</div><div class='add'>+			    ionic_mmap_entry_insert(ctx,</div><div class='add'>+						    qp-&gt;rq.size,</div><div class='add'>+						    PHYS_PFN(qp-&gt;rq_cmb_addr),</div><div class='add'>+						    wc ? IONIC_MMAP_WC : 0,</div><div class='add'>+						    &amp;resp.rq_cmb_offset);</div><div class='add'>+			if (!qp-&gt;mmap_rq_cmb) {</div><div class='add'>+				rc = -ENOMEM;</div><div class='add'>+				goto err_mmap_rq;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			resp.rq_cmb = qp-&gt;rq_cmb;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		rc = ib_copy_to_udata(udata, &amp;resp, sizeof(resp));</div><div class='add'>+		if (rc)</div><div class='add'>+			goto err_resp;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ionic_pgtbl_unbuf(dev, &amp;rq_buf);</div><div class='add'>+	ionic_pgtbl_unbuf(dev, &amp;sq_buf);</div><div class='add'>+</div><div class='add'>+	qp-&gt;ibqp.qp_num = qp-&gt;qpid;</div><div class='add'>+</div><div class='add'>+	init_completion(&amp;qp-&gt;qp_rel_comp);</div><div class='add'>+	kref_init(&amp;qp-&gt;qp_kref);</div><div class='add'>+</div><div class='add'>+	entry = xa_store_irq(&amp;dev-&gt;qp_tbl, qp-&gt;qpid, qp, GFP_KERNEL);</div><div class='add'>+	if (entry) {</div><div class='add'>+		if (!xa_is_err(entry))</div><div class='add'>+			rc = -EINVAL;</div><div class='add'>+		else</div><div class='add'>+			rc = xa_err(entry);</div><div class='add'>+</div><div class='add'>+		goto err_resp;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;has_sq) {</div><div class='add'>+		cq = to_ionic_vcq_cq(attr-&gt;send_cq, qp-&gt;udma_idx);</div><div class='add'>+</div><div class='add'>+		attr-&gt;cap.max_send_wr = qp-&gt;sq.mask;</div><div class='add'>+		attr-&gt;cap.max_send_sge =</div><div class='add'>+			ionic_v1_send_wqe_max_sge(qp-&gt;sq.stride_log2,</div><div class='add'>+						  qp-&gt;sq_spec,</div><div class='add'>+						  qp-&gt;sq_cmb &amp; IONIC_CMB_EXPDB);</div><div class='add'>+		attr-&gt;cap.max_inline_data =</div><div class='add'>+			ionic_v1_send_wqe_max_data(qp-&gt;sq.stride_log2,</div><div class='add'>+						   qp-&gt;sq_cmb &amp;</div><div class='add'>+						   IONIC_CMB_EXPDB);</div><div class='add'>+		qp-&gt;sq_cqid = cq-&gt;cqid;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;has_rq) {</div><div class='add'>+		cq = to_ionic_vcq_cq(attr-&gt;recv_cq, qp-&gt;udma_idx);</div><div class='add'>+</div><div class='add'>+		attr-&gt;cap.max_recv_wr = qp-&gt;rq.mask;</div><div class='add'>+		attr-&gt;cap.max_recv_sge =</div><div class='add'>+			ionic_v1_recv_wqe_max_sge(qp-&gt;rq.stride_log2,</div><div class='add'>+						  qp-&gt;rq_spec,</div><div class='add'>+						  qp-&gt;rq_cmb &amp; IONIC_CMB_EXPDB);</div><div class='add'>+		qp-&gt;rq_cqid = cq-&gt;cqid;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_resp:</div><div class='add'>+	if (udata &amp;&amp; (qp-&gt;rq_cmb &amp; IONIC_CMB_ENABLE))</div><div class='add'>+		rdma_user_mmap_entry_remove(qp-&gt;mmap_rq_cmb);</div><div class='add'>+err_mmap_rq:</div><div class='add'>+	if (udata &amp;&amp; (qp-&gt;sq_cmb &amp; IONIC_CMB_ENABLE))</div><div class='add'>+		rdma_user_mmap_entry_remove(qp-&gt;mmap_sq_cmb);</div><div class='add'>+err_mmap_sq:</div><div class='add'>+	ionic_destroy_qp_cmd(dev, qp-&gt;qpid);</div><div class='add'>+err_cmd:</div><div class='add'>+	ionic_pgtbl_unbuf(dev, &amp;rq_buf);</div><div class='add'>+	ionic_qp_rq_destroy(dev, ctx, qp);</div><div class='add'>+err_rq:</div><div class='add'>+	ionic_pgtbl_unbuf(dev, &amp;sq_buf);</div><div class='add'>+	ionic_qp_sq_destroy(dev, ctx, qp);</div><div class='add'>+err_sq:</div><div class='add'>+	if (qp-&gt;has_ah)</div><div class='add'>+		ionic_put_ahid(dev, qp-&gt;ahid);</div><div class='add'>+err_ahid:</div><div class='add'>+	kfree(qp-&gt;hdr);</div><div class='add'>+err_ah_alloc:</div><div class='add'>+	ionic_put_qpid(dev, qp-&gt;qpid);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void ionic_notify_flush_cq(struct ionic_cq *cq)</div><div class='add'>+{</div><div class='add'>+	if (cq-&gt;flush &amp;&amp; cq-&gt;vcq-&gt;ibcq.comp_handler)</div><div class='add'>+		cq-&gt;vcq-&gt;ibcq.comp_handler(&amp;cq-&gt;vcq-&gt;ibcq,</div><div class='add'>+					   cq-&gt;vcq-&gt;ibcq.cq_context);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_notify_qp_cqs(struct ionic_ibdev *dev, struct ionic_qp *qp)</div><div class='add'>+{</div><div class='add'>+	if (qp-&gt;ibqp.send_cq)</div><div class='add'>+		ionic_notify_flush_cq(to_ionic_vcq_cq(qp-&gt;ibqp.send_cq,</div><div class='add'>+						      qp-&gt;udma_idx));</div><div class='add'>+	if (qp-&gt;ibqp.recv_cq &amp;&amp; qp-&gt;ibqp.recv_cq != qp-&gt;ibqp.send_cq)</div><div class='add'>+		ionic_notify_flush_cq(to_ionic_vcq_cq(qp-&gt;ibqp.recv_cq,</div><div class='add'>+						      qp-&gt;udma_idx));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp)</div><div class='add'>+{</div><div class='add'>+	unsigned long irqflags;</div><div class='add'>+	struct ionic_cq *cq;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;ibqp.send_cq) {</div><div class='add'>+		cq = to_ionic_vcq_cq(qp-&gt;ibqp.send_cq, qp-&gt;udma_idx);</div><div class='add'>+</div><div class='add'>+		/* Hold the CQ lock and QP sq_lock to set up flush */</div><div class='add'>+		spin_lock_irqsave(&amp;cq-&gt;lock, irqflags);</div><div class='add'>+		spin_lock(&amp;qp-&gt;sq_lock);</div><div class='add'>+		qp-&gt;sq_flush = true;</div><div class='add'>+		if (!ionic_queue_empty(&amp;qp-&gt;sq)) {</div><div class='add'>+			cq-&gt;flush = true;</div><div class='add'>+			list_move_tail(&amp;qp-&gt;cq_flush_sq, &amp;cq-&gt;flush_sq);</div><div class='add'>+		}</div><div class='add'>+		spin_unlock(&amp;qp-&gt;sq_lock);</div><div class='add'>+		spin_unlock_irqrestore(&amp;cq-&gt;lock, irqflags);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;ibqp.recv_cq) {</div><div class='add'>+		cq = to_ionic_vcq_cq(qp-&gt;ibqp.recv_cq, qp-&gt;udma_idx);</div><div class='add'>+</div><div class='add'>+		/* Hold the CQ lock and QP rq_lock to set up flush */</div><div class='add'>+		spin_lock_irqsave(&amp;cq-&gt;lock, irqflags);</div><div class='add'>+		spin_lock(&amp;qp-&gt;rq_lock);</div><div class='add'>+		qp-&gt;rq_flush = true;</div><div class='add'>+		if (!ionic_queue_empty(&amp;qp-&gt;rq)) {</div><div class='add'>+			cq-&gt;flush = true;</div><div class='add'>+			list_move_tail(&amp;qp-&gt;cq_flush_rq, &amp;cq-&gt;flush_rq);</div><div class='add'>+		}</div><div class='add'>+		spin_unlock(&amp;qp-&gt;rq_lock);</div><div class='add'>+		spin_unlock_irqrestore(&amp;cq-&gt;lock, irqflags);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_clean_cq(struct ionic_cq *cq, u32 qpid)</div><div class='add'>+{</div><div class='add'>+	struct ionic_v1_cqe *qcqe;</div><div class='add'>+	int prod, qtf, qid, type;</div><div class='add'>+	bool color;</div><div class='add'>+</div><div class='add'>+	if (!cq-&gt;q.ptr)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	color = cq-&gt;color;</div><div class='add'>+	prod = cq-&gt;q.prod;</div><div class='add'>+	qcqe = ionic_queue_at(&amp;cq-&gt;q, prod);</div><div class='add'>+</div><div class='add'>+	while (color == ionic_v1_cqe_color(qcqe)) {</div><div class='add'>+		qtf = ionic_v1_cqe_qtf(qcqe);</div><div class='add'>+		qid = ionic_v1_cqe_qtf_qid(qtf);</div><div class='add'>+		type = ionic_v1_cqe_qtf_type(qtf);</div><div class='add'>+</div><div class='add'>+		if (qid == qpid &amp;&amp; type != IONIC_V1_CQE_TYPE_ADMIN)</div><div class='add'>+			ionic_v1_cqe_clean(qcqe);</div><div class='add'>+</div><div class='add'>+		prod = ionic_queue_next(&amp;cq-&gt;q, prod);</div><div class='add'>+		qcqe = ionic_queue_at(&amp;cq-&gt;q, prod);</div><div class='add'>+		color = ionic_color_wrap(prod, color);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_reset_qp(struct ionic_ibdev *dev, struct ionic_qp *qp)</div><div class='add'>+{</div><div class='add'>+	unsigned long irqflags;</div><div class='add'>+	struct ionic_cq *cq;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	local_irq_save(irqflags);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;ibqp.send_cq) {</div><div class='add'>+		cq = to_ionic_vcq_cq(qp-&gt;ibqp.send_cq, qp-&gt;udma_idx);</div><div class='add'>+		spin_lock(&amp;cq-&gt;lock);</div><div class='add'>+		ionic_clean_cq(cq, qp-&gt;qpid);</div><div class='add'>+		spin_unlock(&amp;cq-&gt;lock);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;ibqp.recv_cq) {</div><div class='add'>+		cq = to_ionic_vcq_cq(qp-&gt;ibqp.recv_cq, qp-&gt;udma_idx);</div><div class='add'>+		spin_lock(&amp;cq-&gt;lock);</div><div class='add'>+		ionic_clean_cq(cq, qp-&gt;qpid);</div><div class='add'>+		spin_unlock(&amp;cq-&gt;lock);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;has_sq) {</div><div class='add'>+		spin_lock(&amp;qp-&gt;sq_lock);</div><div class='add'>+		qp-&gt;sq_flush = false;</div><div class='add'>+		qp-&gt;sq_flush_rcvd = false;</div><div class='add'>+		qp-&gt;sq_msn_prod = 0;</div><div class='add'>+		qp-&gt;sq_msn_cons = 0;</div><div class='add'>+		qp-&gt;sq.prod = 0;</div><div class='add'>+		qp-&gt;sq.cons = 0;</div><div class='add'>+		spin_unlock(&amp;qp-&gt;sq_lock);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;has_rq) {</div><div class='add'>+		spin_lock(&amp;qp-&gt;rq_lock);</div><div class='add'>+		qp-&gt;rq_flush = false;</div><div class='add'>+		qp-&gt;rq.prod = 0;</div><div class='add'>+		qp-&gt;rq.cons = 0;</div><div class='add'>+		if (qp-&gt;rq_meta) {</div><div class='add'>+			for (i = 0; i &lt; qp-&gt;rq.mask; ++i)</div><div class='add'>+				qp-&gt;rq_meta[i].next = &amp;qp-&gt;rq_meta[i + 1];</div><div class='add'>+			qp-&gt;rq_meta[i].next = IONIC_META_LAST;</div><div class='add'>+		}</div><div class='add'>+		qp-&gt;rq_meta_head = &amp;qp-&gt;rq_meta[0];</div><div class='add'>+		spin_unlock(&amp;qp-&gt;rq_lock);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	local_irq_restore(irqflags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool ionic_qp_cur_state_is_ok(enum ib_qp_state q_state,</div><div class='add'>+				     enum ib_qp_state attr_state)</div><div class='add'>+{</div><div class='add'>+	if (q_state == attr_state)</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	if (attr_state == IB_QPS_ERR)</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	if (attr_state == IB_QPS_SQE)</div><div class='add'>+		return q_state == IB_QPS_RTS || q_state == IB_QPS_SQD;</div><div class='add'>+</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_check_modify_qp(struct ionic_qp *qp, struct ib_qp_attr *attr,</div><div class='add'>+				 int mask)</div><div class='add'>+{</div><div class='add'>+	enum ib_qp_state cur_state = (mask &amp; IB_QP_CUR_STATE) ?</div><div class='add'>+		attr-&gt;cur_qp_state : qp-&gt;state;</div><div class='add'>+	enum ib_qp_state next_state = (mask &amp; IB_QP_STATE) ?</div><div class='add'>+		attr-&gt;qp_state : cur_state;</div><div class='add'>+</div><div class='add'>+	if ((mask &amp; IB_QP_CUR_STATE) &amp;&amp;</div><div class='add'>+	    !ionic_qp_cur_state_is_ok(qp-&gt;state, attr-&gt;cur_qp_state))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!ib_modify_qp_is_ok(cur_state, next_state, qp-&gt;ibqp.qp_type, mask))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	/* unprivileged qp not allowed privileged qkey */</div><div class='add'>+	if ((mask &amp; IB_QP_QKEY) &amp;&amp; (attr-&gt;qkey &amp; 0x80000000) &amp;&amp;</div><div class='add'>+	    qp-&gt;ibqp.uobject)</div><div class='add'>+		return -EPERM;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,</div><div class='add'>+		    struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibqp-&gt;device);</div><div class='add'>+	struct ionic_pd *pd = to_ionic_pd(ibqp-&gt;pd);</div><div class='add'>+	struct ionic_qp *qp = to_ionic_qp(ibqp);</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rc = ionic_check_modify_qp(qp, attr, mask);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	if (mask &amp; IB_QP_CAP)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	rc = ionic_modify_qp_cmd(dev, pd, qp, attr, mask);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	if (mask &amp; IB_QP_STATE) {</div><div class='add'>+		qp-&gt;state = attr-&gt;qp_state;</div><div class='add'>+</div><div class='add'>+		if (attr-&gt;qp_state == IB_QPS_ERR) {</div><div class='add'>+			ionic_flush_qp(dev, qp);</div><div class='add'>+			ionic_notify_qp_cqs(dev, qp);</div><div class='add'>+		} else if (attr-&gt;qp_state == IB_QPS_RESET) {</div><div class='add'>+			ionic_reset_qp(dev, qp);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='add'>+		   int mask, struct ib_qp_init_attr *init_attr)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibqp-&gt;device);</div><div class='add'>+	struct ionic_qp *qp = to_ionic_qp(ibqp);</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	memset(attr, 0, sizeof(*attr));</div><div class='add'>+	memset(init_attr, 0, sizeof(*init_attr));</div><div class='add'>+</div><div class='add'>+	rc = ionic_query_qp_cmd(dev, qp, attr, mask);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;has_sq)</div><div class='add'>+		attr-&gt;cap.max_send_wr = qp-&gt;sq.mask;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;has_rq)</div><div class='add'>+		attr-&gt;cap.max_recv_wr = qp-&gt;rq.mask;</div><div class='add'>+</div><div class='add'>+	init_attr-&gt;event_handler = ibqp-&gt;event_handler;</div><div class='add'>+	init_attr-&gt;qp_context = ibqp-&gt;qp_context;</div><div class='add'>+	init_attr-&gt;send_cq = ibqp-&gt;send_cq;</div><div class='add'>+	init_attr-&gt;recv_cq = ibqp-&gt;recv_cq;</div><div class='add'>+	init_attr-&gt;srq = ibqp-&gt;srq;</div><div class='add'>+	init_attr-&gt;xrcd = ibqp-&gt;xrcd;</div><div class='add'>+	init_attr-&gt;cap = attr-&gt;cap;</div><div class='add'>+	init_attr-&gt;sq_sig_type = qp-&gt;sig_all ?</div><div class='add'>+		IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;</div><div class='add'>+	init_attr-&gt;qp_type = ibqp-&gt;qp_type;</div><div class='add'>+	init_attr-&gt;create_flags = 0;</div><div class='add'>+	init_attr-&gt;port_num = 0;</div><div class='add'>+	init_attr-&gt;rwq_ind_tbl = ibqp-&gt;rwq_ind_tbl;</div><div class='add'>+	init_attr-&gt;source_qpn = 0;</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ctx *ctx =</div><div class='add'>+		rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx);</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibqp-&gt;device);</div><div class='add'>+	struct ionic_qp *qp = to_ionic_qp(ibqp);</div><div class='add'>+	unsigned long irqflags;</div><div class='add'>+	struct ionic_cq *cq;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rc = ionic_destroy_qp_cmd(dev, qp-&gt;qpid);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	xa_erase_irq(&amp;dev-&gt;qp_tbl, qp-&gt;qpid);</div><div class='add'>+</div><div class='add'>+	kref_put(&amp;qp-&gt;qp_kref, ionic_qp_complete);</div><div class='add'>+	wait_for_completion(&amp;qp-&gt;qp_rel_comp);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;ibqp.send_cq) {</div><div class='add'>+		cq = to_ionic_vcq_cq(qp-&gt;ibqp.send_cq, qp-&gt;udma_idx);</div><div class='add'>+		spin_lock_irqsave(&amp;cq-&gt;lock, irqflags);</div><div class='add'>+		ionic_clean_cq(cq, qp-&gt;qpid);</div><div class='add'>+		list_del(&amp;qp-&gt;cq_poll_sq);</div><div class='add'>+		list_del(&amp;qp-&gt;cq_flush_sq);</div><div class='add'>+		spin_unlock_irqrestore(&amp;cq-&gt;lock, irqflags);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;ibqp.recv_cq) {</div><div class='add'>+		cq = to_ionic_vcq_cq(qp-&gt;ibqp.recv_cq, qp-&gt;udma_idx);</div><div class='add'>+		spin_lock_irqsave(&amp;cq-&gt;lock, irqflags);</div><div class='add'>+		ionic_clean_cq(cq, qp-&gt;qpid);</div><div class='add'>+		list_del(&amp;qp-&gt;cq_flush_rq);</div><div class='add'>+		spin_unlock_irqrestore(&amp;cq-&gt;lock, irqflags);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ionic_qp_rq_destroy(dev, ctx, qp);</div><div class='add'>+	ionic_qp_sq_destroy(dev, ctx, qp);</div><div class='add'>+	if (qp-&gt;has_ah) {</div><div class='add'>+		ionic_put_ahid(dev, qp-&gt;ahid);</div><div class='add'>+		kfree(qp-&gt;hdr);</div><div class='add'>+	}</div><div class='add'>+	ionic_put_qpid(dev, qp-&gt;qpid);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/ionic/ionic_datapath.c b/drivers/infiniband/hw/ionic/ionic_datapath.c<br/>new file mode 100644<br/>index 000000000000..aa2944887f23<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/ionic/ionic_datapath.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/ionic/ionic_datapath.c</a></div><div class='hunk'>@@ -0,0 +1,1399 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0</div><div class='add'>+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/module.h&gt;</div><div class='add'>+#include &lt;linux/printk.h&gt;</div><div class='add'>+#include &lt;rdma/ib_addr.h&gt;</div><div class='add'>+#include &lt;rdma/ib_user_verbs.h&gt;</div><div class='add'>+</div><div class='add'>+#include "ionic_fw.h"</div><div class='add'>+#include "ionic_ibdev.h"</div><div class='add'>+</div><div class='add'>+#define IONIC_OP(version, opname) \</div><div class='add'>+	((version) &lt; 2 ? IONIC_V1_OP_##opname : IONIC_V2_OP_##opname)</div><div class='add'>+</div><div class='add'>+static bool ionic_next_cqe(struct ionic_ibdev *dev, struct ionic_cq *cq,</div><div class='add'>+			   struct ionic_v1_cqe **cqe)</div><div class='add'>+{</div><div class='add'>+	struct ionic_v1_cqe *qcqe = ionic_queue_at_prod(&amp;cq-&gt;q);</div><div class='add'>+</div><div class='add'>+	if (unlikely(cq-&gt;color != ionic_v1_cqe_color(qcqe)))</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	/* Prevent out-of-order reads of the CQE */</div><div class='add'>+	dma_rmb();</div><div class='add'>+</div><div class='add'>+	*cqe = qcqe;</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc)</div><div class='add'>+{</div><div class='add'>+	struct ionic_rq_meta *meta;</div><div class='add'>+	struct ionic_v1_wqe *wqe;</div><div class='add'>+</div><div class='add'>+	if (!qp-&gt;rq_flush)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (ionic_queue_empty(&amp;qp-&gt;rq))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	wqe = ionic_queue_at_cons(&amp;qp-&gt;rq);</div><div class='add'>+</div><div class='add'>+	/* wqe_id must be a valid queue index */</div><div class='add'>+	if (unlikely(wqe-&gt;base.wqe_id &gt;&gt; qp-&gt;rq.depth_log2)) {</div><div class='add'>+		ibdev_warn(qp-&gt;ibqp.device,</div><div class='add'>+			   "flush qp %u recv index %llu invalid\n",</div><div class='add'>+			   qp-&gt;qpid, (unsigned long long)wqe-&gt;base.wqe_id);</div><div class='add'>+		return -EIO;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* wqe_id must indicate a request that is outstanding */</div><div class='add'>+	meta = &amp;qp-&gt;rq_meta[wqe-&gt;base.wqe_id];</div><div class='add'>+	if (unlikely(meta-&gt;next != IONIC_META_POSTED)) {</div><div class='add'>+		ibdev_warn(qp-&gt;ibqp.device,</div><div class='add'>+			   "flush qp %u recv index %llu not posted\n",</div><div class='add'>+			   qp-&gt;qpid, (unsigned long long)wqe-&gt;base.wqe_id);</div><div class='add'>+		return -EIO;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ionic_queue_consume(&amp;qp-&gt;rq);</div><div class='add'>+</div><div class='add'>+	memset(wc, 0, sizeof(*wc));</div><div class='add'>+</div><div class='add'>+	wc-&gt;status = IB_WC_WR_FLUSH_ERR;</div><div class='add'>+	wc-&gt;wr_id = meta-&gt;wrid;</div><div class='add'>+	wc-&gt;qp = &amp;qp-&gt;ibqp;</div><div class='add'>+</div><div class='add'>+	meta-&gt;next = qp-&gt;rq_meta_head;</div><div class='add'>+	qp-&gt;rq_meta_head = meta;</div><div class='add'>+</div><div class='add'>+	return 1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_flush_recv_many(struct ionic_qp *qp,</div><div class='add'>+				 struct ib_wc *wc, int nwc)</div><div class='add'>+{</div><div class='add'>+	int rc = 0, npolled = 0;</div><div class='add'>+</div><div class='add'>+	while (npolled &lt; nwc) {</div><div class='add'>+		rc = ionic_flush_recv(qp, wc + npolled);</div><div class='add'>+		if (rc &lt;= 0)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		npolled += rc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return npolled ?: rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_flush_send(struct ionic_qp *qp, struct ib_wc *wc)</div><div class='add'>+{</div><div class='add'>+	struct ionic_sq_meta *meta;</div><div class='add'>+</div><div class='add'>+	if (!qp-&gt;sq_flush)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (ionic_queue_empty(&amp;qp-&gt;sq))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	meta = &amp;qp-&gt;sq_meta[qp-&gt;sq.cons];</div><div class='add'>+</div><div class='add'>+	ionic_queue_consume(&amp;qp-&gt;sq);</div><div class='add'>+</div><div class='add'>+	memset(wc, 0, sizeof(*wc));</div><div class='add'>+</div><div class='add'>+	wc-&gt;status = IB_WC_WR_FLUSH_ERR;</div><div class='add'>+	wc-&gt;wr_id = meta-&gt;wrid;</div><div class='add'>+	wc-&gt;qp = &amp;qp-&gt;ibqp;</div><div class='add'>+</div><div class='add'>+	return 1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_flush_send_many(struct ionic_qp *qp,</div><div class='add'>+				 struct ib_wc *wc, int nwc)</div><div class='add'>+{</div><div class='add'>+	int rc = 0, npolled = 0;</div><div class='add'>+</div><div class='add'>+	while (npolled &lt; nwc) {</div><div class='add'>+		rc = ionic_flush_send(qp, wc + npolled);</div><div class='add'>+		if (rc &lt;= 0)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		npolled += rc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return npolled ?: rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq,</div><div class='add'>+			   struct ionic_qp *cqe_qp, struct ionic_v1_cqe *cqe,</div><div class='add'>+			   struct ib_wc *wc)</div><div class='add'>+{</div><div class='add'>+	struct ionic_qp *qp = NULL;</div><div class='add'>+	struct ionic_rq_meta *meta;</div><div class='add'>+	u32 src_qpn, st_len;</div><div class='add'>+	u16 vlan_tag;</div><div class='add'>+	u8 op;</div><div class='add'>+</div><div class='add'>+	if (cqe_qp-&gt;rq_flush)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	qp = cqe_qp;</div><div class='add'>+</div><div class='add'>+	st_len = be32_to_cpu(cqe-&gt;status_length);</div><div class='add'>+</div><div class='add'>+	/* ignore wqe_id in case of flush error */</div><div class='add'>+	if (ionic_v1_cqe_error(cqe) &amp;&amp; st_len == IONIC_STS_WQE_FLUSHED_ERR) {</div><div class='add'>+		cqe_qp-&gt;rq_flush = true;</div><div class='add'>+		cq-&gt;flush = true;</div><div class='add'>+		list_move_tail(&amp;qp-&gt;cq_flush_rq, &amp;cq-&gt;flush_rq);</div><div class='add'>+</div><div class='add'>+		/* posted recvs (if any) flushed by ionic_flush_recv */</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* there had better be something in the recv queue to complete */</div><div class='add'>+	if (ionic_queue_empty(&amp;qp-&gt;rq)) {</div><div class='add'>+		ibdev_warn(&amp;dev-&gt;ibdev, "qp %u is empty\n", qp-&gt;qpid);</div><div class='add'>+		return -EIO;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* wqe_id must be a valid queue index */</div><div class='add'>+	if (unlikely(cqe-&gt;recv.wqe_id &gt;&gt; qp-&gt;rq.depth_log2)) {</div><div class='add'>+		ibdev_warn(&amp;dev-&gt;ibdev,</div><div class='add'>+			   "qp %u recv index %llu invalid\n",</div><div class='add'>+			   qp-&gt;qpid, (unsigned long long)cqe-&gt;recv.wqe_id);</div><div class='add'>+		return -EIO;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* wqe_id must indicate a request that is outstanding */</div><div class='add'>+	meta = &amp;qp-&gt;rq_meta[cqe-&gt;recv.wqe_id];</div><div class='add'>+	if (unlikely(meta-&gt;next != IONIC_META_POSTED)) {</div><div class='add'>+		ibdev_warn(&amp;dev-&gt;ibdev,</div><div class='add'>+			   "qp %u recv index %llu not posted\n",</div><div class='add'>+			   qp-&gt;qpid, (unsigned long long)cqe-&gt;recv.wqe_id);</div><div class='add'>+		return -EIO;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	meta-&gt;next = qp-&gt;rq_meta_head;</div><div class='add'>+	qp-&gt;rq_meta_head = meta;</div><div class='add'>+</div><div class='add'>+	memset(wc, 0, sizeof(*wc));</div><div class='add'>+</div><div class='add'>+	wc-&gt;wr_id = meta-&gt;wrid;</div><div class='add'>+</div><div class='add'>+	wc-&gt;qp = &amp;cqe_qp-&gt;ibqp;</div><div class='add'>+</div><div class='add'>+	if (ionic_v1_cqe_error(cqe)) {</div><div class='add'>+		wc-&gt;vendor_err = st_len;</div><div class='add'>+		wc-&gt;status = ionic_to_ib_status(st_len);</div><div class='add'>+</div><div class='add'>+		cqe_qp-&gt;rq_flush = true;</div><div class='add'>+		cq-&gt;flush = true;</div><div class='add'>+		list_move_tail(&amp;qp-&gt;cq_flush_rq, &amp;cq-&gt;flush_rq);</div><div class='add'>+</div><div class='add'>+		ibdev_warn(&amp;dev-&gt;ibdev,</div><div class='add'>+			   "qp %d recv cqe with error\n", qp-&gt;qpid);</div><div class='add'>+		print_hex_dump(KERN_WARNING, "cqe ", DUMP_PREFIX_OFFSET, 16, 1,</div><div class='add'>+			       cqe, BIT(cq-&gt;q.stride_log2), true);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	wc-&gt;vendor_err = 0;</div><div class='add'>+	wc-&gt;status = IB_WC_SUCCESS;</div><div class='add'>+</div><div class='add'>+	src_qpn = be32_to_cpu(cqe-&gt;recv.src_qpn_op);</div><div class='add'>+	op = src_qpn &gt;&gt; IONIC_V1_CQE_RECV_OP_SHIFT;</div><div class='add'>+</div><div class='add'>+	src_qpn &amp;= IONIC_V1_CQE_RECV_QPN_MASK;</div><div class='add'>+	op &amp;= IONIC_V1_CQE_RECV_OP_MASK;</div><div class='add'>+</div><div class='add'>+	wc-&gt;opcode = IB_WC_RECV;</div><div class='add'>+	switch (op) {</div><div class='add'>+	case IONIC_V1_CQE_RECV_OP_RDMA_IMM:</div><div class='add'>+		wc-&gt;opcode = IB_WC_RECV_RDMA_WITH_IMM;</div><div class='add'>+		wc-&gt;wc_flags |= IB_WC_WITH_IMM;</div><div class='add'>+		wc-&gt;ex.imm_data = cqe-&gt;recv.imm_data_rkey; /* be32 in wc */</div><div class='add'>+		break;</div><div class='add'>+	case IONIC_V1_CQE_RECV_OP_SEND_IMM:</div><div class='add'>+		wc-&gt;wc_flags |= IB_WC_WITH_IMM;</div><div class='add'>+		wc-&gt;ex.imm_data = cqe-&gt;recv.imm_data_rkey; /* be32 in wc */</div><div class='add'>+		break;</div><div class='add'>+	case IONIC_V1_CQE_RECV_OP_SEND_INV:</div><div class='add'>+		wc-&gt;wc_flags |= IB_WC_WITH_INVALIDATE;</div><div class='add'>+		wc-&gt;ex.invalidate_rkey = be32_to_cpu(cqe-&gt;recv.imm_data_rkey);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	wc-&gt;byte_len = st_len;</div><div class='add'>+	wc-&gt;src_qp = src_qpn;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;ibqp.qp_type == IB_QPT_UD ||</div><div class='add'>+	    qp-&gt;ibqp.qp_type == IB_QPT_GSI) {</div><div class='add'>+		wc-&gt;wc_flags |= IB_WC_GRH | IB_WC_WITH_SMAC;</div><div class='add'>+		ether_addr_copy(wc-&gt;smac, cqe-&gt;recv.src_mac);</div><div class='add'>+</div><div class='add'>+		wc-&gt;wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;</div><div class='add'>+		if (ionic_v1_cqe_recv_is_ipv4(cqe))</div><div class='add'>+			wc-&gt;network_hdr_type = RDMA_NETWORK_IPV4;</div><div class='add'>+		else</div><div class='add'>+			wc-&gt;network_hdr_type = RDMA_NETWORK_IPV6;</div><div class='add'>+</div><div class='add'>+		if (ionic_v1_cqe_recv_is_vlan(cqe))</div><div class='add'>+			wc-&gt;wc_flags |= IB_WC_WITH_VLAN;</div><div class='add'>+</div><div class='add'>+		/* vlan_tag in cqe will be valid from dpath even if no vlan */</div><div class='add'>+		vlan_tag = be16_to_cpu(cqe-&gt;recv.vlan_tag);</div><div class='add'>+		wc-&gt;vlan_id = vlan_tag &amp; 0xfff; /* 802.1q VID */</div><div class='add'>+		wc-&gt;sl = vlan_tag &gt;&gt; VLAN_PRIO_SHIFT; /* 802.1q PCP */</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	wc-&gt;pkey_index = 0;</div><div class='add'>+	wc-&gt;port_num = 1;</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	ionic_queue_consume(&amp;qp-&gt;rq);</div><div class='add'>+</div><div class='add'>+	return 1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool ionic_peek_send(struct ionic_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct ionic_sq_meta *meta;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;sq_flush)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	/* completed all send queue requests */</div><div class='add'>+	if (ionic_queue_empty(&amp;qp-&gt;sq))</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	meta = &amp;qp-&gt;sq_meta[qp-&gt;sq.cons];</div><div class='add'>+</div><div class='add'>+	/* waiting for remote completion */</div><div class='add'>+	if (meta-&gt;remote &amp;&amp; meta-&gt;seq == qp-&gt;sq_msn_cons)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	/* waiting for local completion */</div><div class='add'>+	if (!meta-&gt;remote &amp;&amp; !meta-&gt;local_comp)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_poll_send(struct ionic_ibdev *dev, struct ionic_cq *cq,</div><div class='add'>+			   struct ionic_qp *qp, struct ib_wc *wc)</div><div class='add'>+{</div><div class='add'>+	struct ionic_sq_meta *meta;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;sq_flush)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		/* completed all send queue requests */</div><div class='add'>+		if (ionic_queue_empty(&amp;qp-&gt;sq))</div><div class='add'>+			goto out_empty;</div><div class='add'>+</div><div class='add'>+		meta = &amp;qp-&gt;sq_meta[qp-&gt;sq.cons];</div><div class='add'>+</div><div class='add'>+		/* waiting for remote completion */</div><div class='add'>+		if (meta-&gt;remote &amp;&amp; meta-&gt;seq == qp-&gt;sq_msn_cons)</div><div class='add'>+			goto out_empty;</div><div class='add'>+</div><div class='add'>+		/* waiting for local completion */</div><div class='add'>+		if (!meta-&gt;remote &amp;&amp; !meta-&gt;local_comp)</div><div class='add'>+			goto out_empty;</div><div class='add'>+</div><div class='add'>+		ionic_queue_consume(&amp;qp-&gt;sq);</div><div class='add'>+</div><div class='add'>+		/* produce wc only if signaled or error status */</div><div class='add'>+	} while (!meta-&gt;signal &amp;&amp; meta-&gt;ibsts == IB_WC_SUCCESS);</div><div class='add'>+</div><div class='add'>+	memset(wc, 0, sizeof(*wc));</div><div class='add'>+</div><div class='add'>+	wc-&gt;status = meta-&gt;ibsts;</div><div class='add'>+	wc-&gt;wr_id = meta-&gt;wrid;</div><div class='add'>+	wc-&gt;qp = &amp;qp-&gt;ibqp;</div><div class='add'>+</div><div class='add'>+	if (meta-&gt;ibsts == IB_WC_SUCCESS) {</div><div class='add'>+		wc-&gt;byte_len = meta-&gt;len;</div><div class='add'>+		wc-&gt;opcode = meta-&gt;ibop;</div><div class='add'>+	} else {</div><div class='add'>+		wc-&gt;vendor_err = meta-&gt;len;</div><div class='add'>+</div><div class='add'>+		qp-&gt;sq_flush = true;</div><div class='add'>+		cq-&gt;flush = true;</div><div class='add'>+		list_move_tail(&amp;qp-&gt;cq_flush_sq, &amp;cq-&gt;flush_sq);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 1;</div><div class='add'>+</div><div class='add'>+out_empty:</div><div class='add'>+	if (qp-&gt;sq_flush_rcvd) {</div><div class='add'>+		qp-&gt;sq_flush = true;</div><div class='add'>+		cq-&gt;flush = true;</div><div class='add'>+		list_move_tail(&amp;qp-&gt;cq_flush_sq, &amp;cq-&gt;flush_sq);</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_poll_send_many(struct ionic_ibdev *dev, struct ionic_cq *cq,</div><div class='add'>+				struct ionic_qp *qp, struct ib_wc *wc, int nwc)</div><div class='add'>+{</div><div class='add'>+	int rc = 0, npolled = 0;</div><div class='add'>+</div><div class='add'>+	while (npolled &lt; nwc) {</div><div class='add'>+		rc = ionic_poll_send(dev, cq, qp, wc + npolled);</div><div class='add'>+		if (rc &lt;= 0)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		npolled += rc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return npolled ?: rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_validate_cons(u16 prod, u16 cons,</div><div class='add'>+			       u16 comp, u16 mask)</div><div class='add'>+{</div><div class='add'>+	if (((prod - cons) &amp; mask) &lt;= ((comp - cons) &amp; mask))</div><div class='add'>+		return -EIO;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_comp_msn(struct ionic_qp *qp, struct ionic_v1_cqe *cqe)</div><div class='add'>+{</div><div class='add'>+	struct ionic_sq_meta *meta;</div><div class='add'>+	u16 cqe_seq, cqe_idx;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;sq_flush)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	cqe_seq = be32_to_cpu(cqe-&gt;send.msg_msn) &amp; qp-&gt;sq.mask;</div><div class='add'>+</div><div class='add'>+	rc = ionic_validate_cons(qp-&gt;sq_msn_prod,</div><div class='add'>+				 qp-&gt;sq_msn_cons,</div><div class='add'>+				 cqe_seq - 1,</div><div class='add'>+				 qp-&gt;sq.mask);</div><div class='add'>+	if (rc) {</div><div class='add'>+		ibdev_warn(qp-&gt;ibqp.device,</div><div class='add'>+			   "qp %u bad msn %#x seq %u for prod %u cons %u\n",</div><div class='add'>+			   qp-&gt;qpid, be32_to_cpu(cqe-&gt;send.msg_msn),</div><div class='add'>+			   cqe_seq, qp-&gt;sq_msn_prod, qp-&gt;sq_msn_cons);</div><div class='add'>+		return rc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	qp-&gt;sq_msn_cons = cqe_seq;</div><div class='add'>+</div><div class='add'>+	if (ionic_v1_cqe_error(cqe)) {</div><div class='add'>+		cqe_idx = qp-&gt;sq_msn_idx[(cqe_seq - 1) &amp; qp-&gt;sq.mask];</div><div class='add'>+</div><div class='add'>+		meta = &amp;qp-&gt;sq_meta[cqe_idx];</div><div class='add'>+		meta-&gt;len = be32_to_cpu(cqe-&gt;status_length);</div><div class='add'>+		meta-&gt;ibsts = ionic_to_ib_status(meta-&gt;len);</div><div class='add'>+</div><div class='add'>+		ibdev_warn(qp-&gt;ibqp.device,</div><div class='add'>+			   "qp %d msn cqe with error\n", qp-&gt;qpid);</div><div class='add'>+		print_hex_dump(KERN_WARNING, "cqe ", DUMP_PREFIX_OFFSET, 16, 1,</div><div class='add'>+			       cqe, sizeof(*cqe), true);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_comp_npg(struct ionic_qp *qp, struct ionic_v1_cqe *cqe)</div><div class='add'>+{</div><div class='add'>+	struct ionic_sq_meta *meta;</div><div class='add'>+	u16 cqe_idx;</div><div class='add'>+	u32 st_len;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;sq_flush)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	st_len = be32_to_cpu(cqe-&gt;status_length);</div><div class='add'>+</div><div class='add'>+	if (ionic_v1_cqe_error(cqe) &amp;&amp; st_len == IONIC_STS_WQE_FLUSHED_ERR) {</div><div class='add'>+		/*</div><div class='add'>+		 * Flush cqe does not consume a wqe on the device, and maybe</div><div class='add'>+		 * no such work request is posted.</div><div class='add'>+		 *</div><div class='add'>+		 * The driver should begin flushing after the last indicated</div><div class='add'>+		 * normal or error completion.	Here, only set a hint that the</div><div class='add'>+		 * flush request was indicated.	 In poll_send, if nothing more</div><div class='add'>+		 * can be polled normally, then begin flushing.</div><div class='add'>+		 */</div><div class='add'>+		qp-&gt;sq_flush_rcvd = true;</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cqe_idx = cqe-&gt;send.npg_wqe_id &amp; qp-&gt;sq.mask;</div><div class='add'>+	meta = &amp;qp-&gt;sq_meta[cqe_idx];</div><div class='add'>+	meta-&gt;local_comp = true;</div><div class='add'>+</div><div class='add'>+	if (ionic_v1_cqe_error(cqe)) {</div><div class='add'>+		meta-&gt;len = st_len;</div><div class='add'>+		meta-&gt;ibsts = ionic_to_ib_status(st_len);</div><div class='add'>+		meta-&gt;remote = false;</div><div class='add'>+		ibdev_warn(qp-&gt;ibqp.device,</div><div class='add'>+			   "qp %d npg cqe with error\n", qp-&gt;qpid);</div><div class='add'>+		print_hex_dump(KERN_WARNING, "cqe ", DUMP_PREFIX_OFFSET, 16, 1,</div><div class='add'>+			       cqe, sizeof(*cqe), true);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_reserve_sync_cq(struct ionic_ibdev *dev, struct ionic_cq *cq)</div><div class='add'>+{</div><div class='add'>+	if (!ionic_queue_empty(&amp;cq-&gt;q)) {</div><div class='add'>+		cq-&gt;credit += ionic_queue_length(&amp;cq-&gt;q);</div><div class='add'>+		cq-&gt;q.cons = cq-&gt;q.prod;</div><div class='add'>+</div><div class='add'>+		ionic_dbell_ring(dev-&gt;lif_cfg.dbpage, dev-&gt;lif_cfg.cq_qtype,</div><div class='add'>+				 ionic_queue_dbell_val(&amp;cq-&gt;q));</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_reserve_cq(struct ionic_ibdev *dev, struct ionic_cq *cq,</div><div class='add'>+			     int spend)</div><div class='add'>+{</div><div class='add'>+	cq-&gt;credit -= spend;</div><div class='add'>+</div><div class='add'>+	if (cq-&gt;credit &lt;= 0)</div><div class='add'>+		ionic_reserve_sync_cq(dev, cq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_poll_vcq_cq(struct ionic_ibdev *dev,</div><div class='add'>+			     struct ionic_cq *cq,</div><div class='add'>+			     int nwc, struct ib_wc *wc)</div><div class='add'>+{</div><div class='add'>+	struct ionic_qp *qp, *qp_next;</div><div class='add'>+	struct ionic_v1_cqe *cqe;</div><div class='add'>+	int rc = 0, npolled = 0;</div><div class='add'>+	unsigned long irqflags;</div><div class='add'>+	u32 qtf, qid;</div><div class='add'>+	bool peek;</div><div class='add'>+	u8 type;</div><div class='add'>+</div><div class='add'>+	if (nwc &lt; 1)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cq-&gt;lock, irqflags);</div><div class='add'>+</div><div class='add'>+	/* poll already indicated work completions for send queue */</div><div class='add'>+	list_for_each_entry_safe(qp, qp_next, &amp;cq-&gt;poll_sq, cq_poll_sq) {</div><div class='add'>+		if (npolled == nwc)</div><div class='add'>+			goto out;</div><div class='add'>+</div><div class='add'>+		spin_lock(&amp;qp-&gt;sq_lock);</div><div class='add'>+		rc = ionic_poll_send_many(dev, cq, qp, wc + npolled,</div><div class='add'>+					  nwc - npolled);</div><div class='add'>+		spin_unlock(&amp;qp-&gt;sq_lock);</div><div class='add'>+</div><div class='add'>+		if (rc &gt; 0)</div><div class='add'>+			npolled += rc;</div><div class='add'>+</div><div class='add'>+		if (npolled &lt; nwc)</div><div class='add'>+			list_del_init(&amp;qp-&gt;cq_poll_sq);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* poll for more work completions */</div><div class='add'>+	while (likely(ionic_next_cqe(dev, cq, &amp;cqe))) {</div><div class='add'>+		if (npolled == nwc)</div><div class='add'>+			goto out;</div><div class='add'>+</div><div class='add'>+		qtf = ionic_v1_cqe_qtf(cqe);</div><div class='add'>+		qid = ionic_v1_cqe_qtf_qid(qtf);</div><div class='add'>+		type = ionic_v1_cqe_qtf_type(qtf);</div><div class='add'>+</div><div class='add'>+		/*</div><div class='add'>+		 * Safe to access QP without additional reference here as,</div><div class='add'>+		 * 1. We hold cq-&gt;lock throughout</div><div class='add'>+		 * 2. ionic_destroy_qp() acquires the same cq-&gt;lock before cleanup</div><div class='add'>+		 * 3. QP is removed from qp_tbl before any cleanup begins</div><div class='add'>+		 * This ensures no concurrent access between polling and destruction.</div><div class='add'>+		 */</div><div class='add'>+		qp = xa_load(&amp;dev-&gt;qp_tbl, qid);</div><div class='add'>+		if (unlikely(!qp)) {</div><div class='add'>+			ibdev_dbg(&amp;dev-&gt;ibdev, "missing qp for qid %u\n", qid);</div><div class='add'>+			goto cq_next;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		switch (type) {</div><div class='add'>+		case IONIC_V1_CQE_TYPE_RECV:</div><div class='add'>+			spin_lock(&amp;qp-&gt;rq_lock);</div><div class='add'>+			rc = ionic_poll_recv(dev, cq, qp, cqe, wc + npolled);</div><div class='add'>+			spin_unlock(&amp;qp-&gt;rq_lock);</div><div class='add'>+</div><div class='add'>+			if (rc &lt; 0)</div><div class='add'>+				goto out;</div><div class='add'>+</div><div class='add'>+			npolled += rc;</div><div class='add'>+</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		case IONIC_V1_CQE_TYPE_SEND_MSN:</div><div class='add'>+			spin_lock(&amp;qp-&gt;sq_lock);</div><div class='add'>+			rc = ionic_comp_msn(qp, cqe);</div><div class='add'>+			if (!rc) {</div><div class='add'>+				rc = ionic_poll_send_many(dev, cq, qp,</div><div class='add'>+							  wc + npolled,</div><div class='add'>+							  nwc - npolled);</div><div class='add'>+				peek = ionic_peek_send(qp);</div><div class='add'>+			}</div><div class='add'>+			spin_unlock(&amp;qp-&gt;sq_lock);</div><div class='add'>+</div><div class='add'>+			if (rc &lt; 0)</div><div class='add'>+				goto out;</div><div class='add'>+</div><div class='add'>+			npolled += rc;</div><div class='add'>+</div><div class='add'>+			if (peek)</div><div class='add'>+				list_move_tail(&amp;qp-&gt;cq_poll_sq, &amp;cq-&gt;poll_sq);</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		case IONIC_V1_CQE_TYPE_SEND_NPG:</div><div class='add'>+			spin_lock(&amp;qp-&gt;sq_lock);</div><div class='add'>+			rc = ionic_comp_npg(qp, cqe);</div><div class='add'>+			if (!rc) {</div><div class='add'>+				rc = ionic_poll_send_many(dev, cq, qp,</div><div class='add'>+							  wc + npolled,</div><div class='add'>+							  nwc - npolled);</div><div class='add'>+				peek = ionic_peek_send(qp);</div><div class='add'>+			}</div><div class='add'>+			spin_unlock(&amp;qp-&gt;sq_lock);</div><div class='add'>+</div><div class='add'>+			if (rc &lt; 0)</div><div class='add'>+				goto out;</div><div class='add'>+</div><div class='add'>+			npolled += rc;</div><div class='add'>+</div><div class='add'>+			if (peek)</div><div class='add'>+				list_move_tail(&amp;qp-&gt;cq_poll_sq, &amp;cq-&gt;poll_sq);</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		default:</div><div class='add'>+			ibdev_warn(&amp;dev-&gt;ibdev,</div><div class='add'>+				   "unexpected cqe type %u\n", type);</div><div class='add'>+			rc = -EIO;</div><div class='add'>+			goto out;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+cq_next:</div><div class='add'>+		ionic_queue_produce(&amp;cq-&gt;q);</div><div class='add'>+		cq-&gt;color = ionic_color_wrap(cq-&gt;q.prod, cq-&gt;color);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* lastly, flush send and recv queues */</div><div class='add'>+	if (likely(!cq-&gt;flush))</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	cq-&gt;flush = false;</div><div class='add'>+</div><div class='add'>+	list_for_each_entry_safe(qp, qp_next, &amp;cq-&gt;flush_sq, cq_flush_sq) {</div><div class='add'>+		if (npolled == nwc)</div><div class='add'>+			goto out;</div><div class='add'>+</div><div class='add'>+		spin_lock(&amp;qp-&gt;sq_lock);</div><div class='add'>+		rc = ionic_flush_send_many(qp, wc + npolled, nwc - npolled);</div><div class='add'>+		spin_unlock(&amp;qp-&gt;sq_lock);</div><div class='add'>+</div><div class='add'>+		if (rc &gt; 0)</div><div class='add'>+			npolled += rc;</div><div class='add'>+</div><div class='add'>+		if (npolled &lt; nwc)</div><div class='add'>+			list_del_init(&amp;qp-&gt;cq_flush_sq);</div><div class='add'>+		else</div><div class='add'>+			cq-&gt;flush = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	list_for_each_entry_safe(qp, qp_next, &amp;cq-&gt;flush_rq, cq_flush_rq) {</div><div class='add'>+		if (npolled == nwc)</div><div class='add'>+			goto out;</div><div class='add'>+</div><div class='add'>+		spin_lock(&amp;qp-&gt;rq_lock);</div><div class='add'>+		rc = ionic_flush_recv_many(qp, wc + npolled, nwc - npolled);</div><div class='add'>+		spin_unlock(&amp;qp-&gt;rq_lock);</div><div class='add'>+</div><div class='add'>+		if (rc &gt; 0)</div><div class='add'>+			npolled += rc;</div><div class='add'>+</div><div class='add'>+		if (npolled &lt; nwc)</div><div class='add'>+			list_del_init(&amp;qp-&gt;cq_flush_rq);</div><div class='add'>+		else</div><div class='add'>+			cq-&gt;flush = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	/* in case credit was depleted (more work posted than cq depth) */</div><div class='add'>+	if (cq-&gt;credit &lt;= 0)</div><div class='add'>+		ionic_reserve_sync_cq(dev, cq);</div><div class='add'>+</div><div class='add'>+	spin_unlock_irqrestore(&amp;cq-&gt;lock, irqflags);</div><div class='add'>+</div><div class='add'>+	return npolled ?: rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_poll_cq(struct ib_cq *ibcq, int nwc, struct ib_wc *wc)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibcq-&gt;device);</div><div class='add'>+	struct ionic_vcq *vcq = to_ionic_vcq(ibcq);</div><div class='add'>+	int rc_tmp, rc = 0, npolled = 0;</div><div class='add'>+	int cq_i, cq_x, cq_ix;</div><div class='add'>+</div><div class='add'>+	cq_x = vcq-&gt;poll_idx;</div><div class='add'>+	vcq-&gt;poll_idx ^= dev-&gt;lif_cfg.udma_count - 1;</div><div class='add'>+</div><div class='add'>+	for (cq_i = 0; npolled &lt; nwc &amp;&amp; cq_i &lt; dev-&gt;lif_cfg.udma_count; ++cq_i) {</div><div class='add'>+		cq_ix = cq_i ^ cq_x;</div><div class='add'>+</div><div class='add'>+		if (!(vcq-&gt;udma_mask &amp; BIT(cq_ix)))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		rc_tmp = ionic_poll_vcq_cq(dev, &amp;vcq-&gt;cq[cq_ix],</div><div class='add'>+					   nwc - npolled,</div><div class='add'>+					   wc + npolled);</div><div class='add'>+</div><div class='add'>+		if (rc_tmp &gt;= 0)</div><div class='add'>+			npolled += rc_tmp;</div><div class='add'>+		else if (!rc)</div><div class='add'>+			rc = rc_tmp;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return npolled ?: rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_req_notify_vcq_cq(struct ionic_ibdev *dev, struct ionic_cq *cq,</div><div class='add'>+				   enum ib_cq_notify_flags flags)</div><div class='add'>+{</div><div class='add'>+	u64 dbell_val = cq-&gt;q.dbell;</div><div class='add'>+</div><div class='add'>+	if (flags &amp; IB_CQ_SOLICITED) {</div><div class='add'>+		cq-&gt;arm_sol_prod = ionic_queue_next(&amp;cq-&gt;q, cq-&gt;arm_sol_prod);</div><div class='add'>+		dbell_val |= cq-&gt;arm_sol_prod | IONIC_CQ_RING_SOL;</div><div class='add'>+	} else {</div><div class='add'>+		cq-&gt;arm_any_prod = ionic_queue_next(&amp;cq-&gt;q, cq-&gt;arm_any_prod);</div><div class='add'>+		dbell_val |= cq-&gt;arm_any_prod | IONIC_CQ_RING_ARM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ionic_reserve_sync_cq(dev, cq);</div><div class='add'>+</div><div class='add'>+	ionic_dbell_ring(dev-&gt;lif_cfg.dbpage, dev-&gt;lif_cfg.cq_qtype, dbell_val);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * IB_CQ_REPORT_MISSED_EVENTS:</div><div class='add'>+	 *</div><div class='add'>+	 * The queue index in ring zero guarantees no missed events.</div><div class='add'>+	 *</div><div class='add'>+	 * Here, we check if the color bit in the next cqe is flipped.	If it</div><div class='add'>+	 * is flipped, then progress can be made by immediately polling the cq.</div><div class='add'>+	 * Still, the cq will be armed, and an event will be generated.	 The cq</div><div class='add'>+	 * may be empty when polled after the event, because the next poll</div><div class='add'>+	 * after arming the cq can empty it.</div><div class='add'>+	 */</div><div class='add'>+	return (flags &amp; IB_CQ_REPORT_MISSED_EVENTS) &amp;&amp;</div><div class='add'>+		cq-&gt;color == ionic_v1_cqe_color(ionic_queue_at_prod(&amp;cq-&gt;q));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibcq-&gt;device);</div><div class='add'>+	struct ionic_vcq *vcq = to_ionic_vcq(ibcq);</div><div class='add'>+	int rc = 0, cq_i;</div><div class='add'>+</div><div class='add'>+	for (cq_i = 0; cq_i &lt; dev-&gt;lif_cfg.udma_count; ++cq_i) {</div><div class='add'>+		if (!(vcq-&gt;udma_mask &amp; BIT(cq_i)))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		if (ionic_req_notify_vcq_cq(dev, &amp;vcq-&gt;cq[cq_i], flags))</div><div class='add'>+			rc = 1;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static s64 ionic_prep_inline(void *data, u32 max_data,</div><div class='add'>+			     const struct ib_sge *ib_sgl, int num_sge)</div><div class='add'>+{</div><div class='add'>+	static const s64 bit_31 = 1u &lt;&lt; 31;</div><div class='add'>+	s64 len = 0, sg_len;</div><div class='add'>+	int sg_i;</div><div class='add'>+</div><div class='add'>+	for (sg_i = 0; sg_i &lt; num_sge; ++sg_i) {</div><div class='add'>+		sg_len = ib_sgl[sg_i].length;</div><div class='add'>+</div><div class='add'>+		/* sge length zero means 2GB */</div><div class='add'>+		if (unlikely(sg_len == 0))</div><div class='add'>+			sg_len = bit_31;</div><div class='add'>+</div><div class='add'>+		/* greater than max inline data is invalid */</div><div class='add'>+		if (unlikely(len + sg_len &gt; max_data))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		memcpy(data + len, (void *)ib_sgl[sg_i].addr, sg_len);</div><div class='add'>+</div><div class='add'>+		len += sg_len;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return len;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static s64 ionic_prep_pld(struct ionic_v1_wqe *wqe,</div><div class='add'>+			  union ionic_v1_pld *pld,</div><div class='add'>+			  int spec, u32 max_sge,</div><div class='add'>+			  const struct ib_sge *ib_sgl,</div><div class='add'>+			  int num_sge)</div><div class='add'>+{</div><div class='add'>+	static const s64 bit_31 = 1l &lt;&lt; 31;</div><div class='add'>+	struct ionic_sge *sgl;</div><div class='add'>+	__be32 *spec32 = NULL;</div><div class='add'>+	__be16 *spec16 = NULL;</div><div class='add'>+	s64 len = 0, sg_len;</div><div class='add'>+	int sg_i = 0;</div><div class='add'>+</div><div class='add'>+	if (unlikely(num_sge &lt; 0 || (u32)num_sge &gt; max_sge))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (spec &amp;&amp; num_sge &gt; IONIC_V1_SPEC_FIRST_SGE) {</div><div class='add'>+		sg_i = IONIC_V1_SPEC_FIRST_SGE;</div><div class='add'>+</div><div class='add'>+		if (num_sge &gt; 8) {</div><div class='add'>+			wqe-&gt;base.flags |= cpu_to_be16(IONIC_V1_FLAG_SPEC16);</div><div class='add'>+			spec16 = pld-&gt;spec16;</div><div class='add'>+		} else {</div><div class='add'>+			wqe-&gt;base.flags |= cpu_to_be16(IONIC_V1_FLAG_SPEC32);</div><div class='add'>+			spec32 = pld-&gt;spec32;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	sgl = &amp;pld-&gt;sgl[sg_i];</div><div class='add'>+</div><div class='add'>+	for (sg_i = 0; sg_i &lt; num_sge; ++sg_i) {</div><div class='add'>+		sg_len = ib_sgl[sg_i].length;</div><div class='add'>+</div><div class='add'>+		/* sge length zero means 2GB */</div><div class='add'>+		if (unlikely(sg_len == 0))</div><div class='add'>+			sg_len = bit_31;</div><div class='add'>+</div><div class='add'>+		/* greater than 2GB data is invalid */</div><div class='add'>+		if (unlikely(len + sg_len &gt; bit_31))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		sgl[sg_i].va = cpu_to_be64(ib_sgl[sg_i].addr);</div><div class='add'>+		sgl[sg_i].len = cpu_to_be32(sg_len);</div><div class='add'>+		sgl[sg_i].lkey = cpu_to_be32(ib_sgl[sg_i].lkey);</div><div class='add'>+</div><div class='add'>+		if (spec32) {</div><div class='add'>+			spec32[sg_i] = sgl[sg_i].len;</div><div class='add'>+		} else if (spec16) {</div><div class='add'>+			if (unlikely(sg_len &gt; U16_MAX))</div><div class='add'>+				return -EINVAL;</div><div class='add'>+			spec16[sg_i] = cpu_to_be16(sg_len);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		len += sg_len;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return len;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_prep_base(struct ionic_qp *qp,</div><div class='add'>+			    const struct ib_send_wr *wr,</div><div class='add'>+			    struct ionic_sq_meta *meta,</div><div class='add'>+			    struct ionic_v1_wqe *wqe)</div><div class='add'>+{</div><div class='add'>+	meta-&gt;wrid = wr-&gt;wr_id;</div><div class='add'>+	meta-&gt;ibsts = IB_WC_SUCCESS;</div><div class='add'>+	meta-&gt;signal = false;</div><div class='add'>+	meta-&gt;local_comp = false;</div><div class='add'>+</div><div class='add'>+	wqe-&gt;base.wqe_id = qp-&gt;sq.prod;</div><div class='add'>+</div><div class='add'>+	if (wr-&gt;send_flags &amp; IB_SEND_FENCE)</div><div class='add'>+		wqe-&gt;base.flags |= cpu_to_be16(IONIC_V1_FLAG_FENCE);</div><div class='add'>+</div><div class='add'>+	if (wr-&gt;send_flags &amp; IB_SEND_SOLICITED)</div><div class='add'>+		wqe-&gt;base.flags |= cpu_to_be16(IONIC_V1_FLAG_SOL);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;sig_all || wr-&gt;send_flags &amp; IB_SEND_SIGNALED) {</div><div class='add'>+		wqe-&gt;base.flags |= cpu_to_be16(IONIC_V1_FLAG_SIG);</div><div class='add'>+		meta-&gt;signal = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	meta-&gt;seq = qp-&gt;sq_msn_prod;</div><div class='add'>+	meta-&gt;remote =</div><div class='add'>+		qp-&gt;ibqp.qp_type != IB_QPT_UD &amp;&amp;</div><div class='add'>+		qp-&gt;ibqp.qp_type != IB_QPT_GSI &amp;&amp;</div><div class='add'>+		!ionic_ibop_is_local(wr-&gt;opcode);</div><div class='add'>+</div><div class='add'>+	if (meta-&gt;remote) {</div><div class='add'>+		qp-&gt;sq_msn_idx[meta-&gt;seq] = qp-&gt;sq.prod;</div><div class='add'>+		qp-&gt;sq_msn_prod = ionic_queue_next(&amp;qp-&gt;sq, qp-&gt;sq_msn_prod);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ionic_queue_produce(&amp;qp-&gt;sq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_prep_common(struct ionic_qp *qp,</div><div class='add'>+			     const struct ib_send_wr *wr,</div><div class='add'>+			     struct ionic_sq_meta *meta,</div><div class='add'>+			     struct ionic_v1_wqe *wqe)</div><div class='add'>+{</div><div class='add'>+	s64 signed_len;</div><div class='add'>+	u32 mval;</div><div class='add'>+</div><div class='add'>+	if (wr-&gt;send_flags &amp; IB_SEND_INLINE) {</div><div class='add'>+		wqe-&gt;base.num_sge_key = 0;</div><div class='add'>+		wqe-&gt;base.flags |= cpu_to_be16(IONIC_V1_FLAG_INL);</div><div class='add'>+		mval = ionic_v1_send_wqe_max_data(qp-&gt;sq.stride_log2, false);</div><div class='add'>+		signed_len = ionic_prep_inline(wqe-&gt;common.pld.data, mval,</div><div class='add'>+					       wr-&gt;sg_list, wr-&gt;num_sge);</div><div class='add'>+	} else {</div><div class='add'>+		wqe-&gt;base.num_sge_key = wr-&gt;num_sge;</div><div class='add'>+		mval = ionic_v1_send_wqe_max_sge(qp-&gt;sq.stride_log2,</div><div class='add'>+						 qp-&gt;sq_spec,</div><div class='add'>+						 false);</div><div class='add'>+		signed_len = ionic_prep_pld(wqe, &amp;wqe-&gt;common.pld,</div><div class='add'>+					    qp-&gt;sq_spec, mval,</div><div class='add'>+					    wr-&gt;sg_list, wr-&gt;num_sge);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (unlikely(signed_len &lt; 0))</div><div class='add'>+		return signed_len;</div><div class='add'>+</div><div class='add'>+	meta-&gt;len = signed_len;</div><div class='add'>+	wqe-&gt;common.length = cpu_to_be32(signed_len);</div><div class='add'>+</div><div class='add'>+	ionic_prep_base(qp, wr, meta, wqe);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_prep_sq_wqe(struct ionic_qp *qp, void *wqe)</div><div class='add'>+{</div><div class='add'>+	memset(wqe, 0, 1u &lt;&lt; qp-&gt;sq.stride_log2);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_prep_rq_wqe(struct ionic_qp *qp, void *wqe)</div><div class='add'>+{</div><div class='add'>+	memset(wqe, 0, 1u &lt;&lt; qp-&gt;rq.stride_log2);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_prep_send(struct ionic_qp *qp,</div><div class='add'>+			   const struct ib_send_wr *wr)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(qp-&gt;ibqp.device);</div><div class='add'>+	struct ionic_sq_meta *meta;</div><div class='add'>+	struct ionic_v1_wqe *wqe;</div><div class='add'>+</div><div class='add'>+	meta = &amp;qp-&gt;sq_meta[qp-&gt;sq.prod];</div><div class='add'>+	wqe = ionic_queue_at_prod(&amp;qp-&gt;sq);</div><div class='add'>+</div><div class='add'>+	ionic_prep_sq_wqe(qp, wqe);</div><div class='add'>+</div><div class='add'>+	meta-&gt;ibop = IB_WC_SEND;</div><div class='add'>+</div><div class='add'>+	switch (wr-&gt;opcode) {</div><div class='add'>+	case IB_WR_SEND:</div><div class='add'>+		wqe-&gt;base.op = IONIC_OP(dev-&gt;lif_cfg.rdma_version, SEND);</div><div class='add'>+		break;</div><div class='add'>+	case IB_WR_SEND_WITH_IMM:</div><div class='add'>+		wqe-&gt;base.op = IONIC_OP(dev-&gt;lif_cfg.rdma_version, SEND_IMM);</div><div class='add'>+		wqe-&gt;base.imm_data_key = wr-&gt;ex.imm_data;</div><div class='add'>+		break;</div><div class='add'>+	case IB_WR_SEND_WITH_INV:</div><div class='add'>+		wqe-&gt;base.op = IONIC_OP(dev-&gt;lif_cfg.rdma_version, SEND_INV);</div><div class='add'>+		wqe-&gt;base.imm_data_key =</div><div class='add'>+			cpu_to_be32(wr-&gt;ex.invalidate_rkey);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ionic_prep_common(qp, wr, meta, wqe);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_prep_send_ud(struct ionic_qp *qp,</div><div class='add'>+			      const struct ib_ud_wr *wr)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(qp-&gt;ibqp.device);</div><div class='add'>+	struct ionic_sq_meta *meta;</div><div class='add'>+	struct ionic_v1_wqe *wqe;</div><div class='add'>+	struct ionic_ah *ah;</div><div class='add'>+</div><div class='add'>+	if (unlikely(!wr-&gt;ah))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ah = to_ionic_ah(wr-&gt;ah);</div><div class='add'>+</div><div class='add'>+	meta = &amp;qp-&gt;sq_meta[qp-&gt;sq.prod];</div><div class='add'>+	wqe = ionic_queue_at_prod(&amp;qp-&gt;sq);</div><div class='add'>+</div><div class='add'>+	ionic_prep_sq_wqe(qp, wqe);</div><div class='add'>+</div><div class='add'>+	wqe-&gt;common.send.ah_id = cpu_to_be32(ah-&gt;ahid);</div><div class='add'>+	wqe-&gt;common.send.dest_qpn = cpu_to_be32(wr-&gt;remote_qpn);</div><div class='add'>+	wqe-&gt;common.send.dest_qkey = cpu_to_be32(wr-&gt;remote_qkey);</div><div class='add'>+</div><div class='add'>+	meta-&gt;ibop = IB_WC_SEND;</div><div class='add'>+</div><div class='add'>+	switch (wr-&gt;wr.opcode) {</div><div class='add'>+	case IB_WR_SEND:</div><div class='add'>+		wqe-&gt;base.op = IONIC_OP(dev-&gt;lif_cfg.rdma_version, SEND);</div><div class='add'>+		break;</div><div class='add'>+	case IB_WR_SEND_WITH_IMM:</div><div class='add'>+		wqe-&gt;base.op = IONIC_OP(dev-&gt;lif_cfg.rdma_version, SEND_IMM);</div><div class='add'>+		wqe-&gt;base.imm_data_key = wr-&gt;wr.ex.imm_data;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ionic_prep_common(qp, &amp;wr-&gt;wr, meta, wqe);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_prep_rdma(struct ionic_qp *qp,</div><div class='add'>+			   const struct ib_rdma_wr *wr)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(qp-&gt;ibqp.device);</div><div class='add'>+	struct ionic_sq_meta *meta;</div><div class='add'>+	struct ionic_v1_wqe *wqe;</div><div class='add'>+</div><div class='add'>+	meta = &amp;qp-&gt;sq_meta[qp-&gt;sq.prod];</div><div class='add'>+	wqe = ionic_queue_at_prod(&amp;qp-&gt;sq);</div><div class='add'>+</div><div class='add'>+	ionic_prep_sq_wqe(qp, wqe);</div><div class='add'>+</div><div class='add'>+	meta-&gt;ibop = IB_WC_RDMA_WRITE;</div><div class='add'>+</div><div class='add'>+	switch (wr-&gt;wr.opcode) {</div><div class='add'>+	case IB_WR_RDMA_READ:</div><div class='add'>+		if (wr-&gt;wr.send_flags &amp; (IB_SEND_SOLICITED | IB_SEND_INLINE))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		meta-&gt;ibop = IB_WC_RDMA_READ;</div><div class='add'>+		wqe-&gt;base.op = IONIC_OP(dev-&gt;lif_cfg.rdma_version, RDMA_READ);</div><div class='add'>+		break;</div><div class='add'>+	case IB_WR_RDMA_WRITE:</div><div class='add'>+		if (wr-&gt;wr.send_flags &amp; IB_SEND_SOLICITED)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		wqe-&gt;base.op = IONIC_OP(dev-&gt;lif_cfg.rdma_version, RDMA_WRITE);</div><div class='add'>+		break;</div><div class='add'>+	case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='add'>+		wqe-&gt;base.op = IONIC_OP(dev-&gt;lif_cfg.rdma_version, RDMA_WRITE_IMM);</div><div class='add'>+		wqe-&gt;base.imm_data_key = wr-&gt;wr.ex.imm_data;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	wqe-&gt;common.rdma.remote_va_high = cpu_to_be32(wr-&gt;remote_addr &gt;&gt; 32);</div><div class='add'>+	wqe-&gt;common.rdma.remote_va_low = cpu_to_be32(wr-&gt;remote_addr);</div><div class='add'>+	wqe-&gt;common.rdma.remote_rkey = cpu_to_be32(wr-&gt;rkey);</div><div class='add'>+</div><div class='add'>+	return ionic_prep_common(qp, &amp;wr-&gt;wr, meta, wqe);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_prep_atomic(struct ionic_qp *qp,</div><div class='add'>+			     const struct ib_atomic_wr *wr)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(qp-&gt;ibqp.device);</div><div class='add'>+	struct ionic_sq_meta *meta;</div><div class='add'>+	struct ionic_v1_wqe *wqe;</div><div class='add'>+</div><div class='add'>+	if (wr-&gt;wr.num_sge != 1 || wr-&gt;wr.sg_list[0].length != 8)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (wr-&gt;wr.send_flags &amp; (IB_SEND_SOLICITED | IB_SEND_INLINE))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	meta = &amp;qp-&gt;sq_meta[qp-&gt;sq.prod];</div><div class='add'>+	wqe = ionic_queue_at_prod(&amp;qp-&gt;sq);</div><div class='add'>+</div><div class='add'>+	ionic_prep_sq_wqe(qp, wqe);</div><div class='add'>+</div><div class='add'>+	meta-&gt;ibop = IB_WC_RDMA_WRITE;</div><div class='add'>+</div><div class='add'>+	switch (wr-&gt;wr.opcode) {</div><div class='add'>+	case IB_WR_ATOMIC_CMP_AND_SWP:</div><div class='add'>+		meta-&gt;ibop = IB_WC_COMP_SWAP;</div><div class='add'>+		wqe-&gt;base.op = IONIC_OP(dev-&gt;lif_cfg.rdma_version, ATOMIC_CS);</div><div class='add'>+		wqe-&gt;atomic.swap_add_high = cpu_to_be32(wr-&gt;swap &gt;&gt; 32);</div><div class='add'>+		wqe-&gt;atomic.swap_add_low = cpu_to_be32(wr-&gt;swap);</div><div class='add'>+		wqe-&gt;atomic.compare_high = cpu_to_be32(wr-&gt;compare_add &gt;&gt; 32);</div><div class='add'>+		wqe-&gt;atomic.compare_low = cpu_to_be32(wr-&gt;compare_add);</div><div class='add'>+		break;</div><div class='add'>+	case IB_WR_ATOMIC_FETCH_AND_ADD:</div><div class='add'>+		meta-&gt;ibop = IB_WC_FETCH_ADD;</div><div class='add'>+		wqe-&gt;base.op = IONIC_OP(dev-&gt;lif_cfg.rdma_version, ATOMIC_FA);</div><div class='add'>+		wqe-&gt;atomic.swap_add_high = cpu_to_be32(wr-&gt;compare_add &gt;&gt; 32);</div><div class='add'>+		wqe-&gt;atomic.swap_add_low = cpu_to_be32(wr-&gt;compare_add);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	wqe-&gt;atomic.remote_va_high = cpu_to_be32(wr-&gt;remote_addr &gt;&gt; 32);</div><div class='add'>+	wqe-&gt;atomic.remote_va_low = cpu_to_be32(wr-&gt;remote_addr);</div><div class='add'>+	wqe-&gt;atomic.remote_rkey = cpu_to_be32(wr-&gt;rkey);</div><div class='add'>+</div><div class='add'>+	wqe-&gt;base.num_sge_key = 1;</div><div class='add'>+	wqe-&gt;atomic.sge.va = cpu_to_be64(wr-&gt;wr.sg_list[0].addr);</div><div class='add'>+	wqe-&gt;atomic.sge.len = cpu_to_be32(8);</div><div class='add'>+	wqe-&gt;atomic.sge.lkey = cpu_to_be32(wr-&gt;wr.sg_list[0].lkey);</div><div class='add'>+</div><div class='add'>+	return ionic_prep_common(qp, &amp;wr-&gt;wr, meta, wqe);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_prep_inv(struct ionic_qp *qp,</div><div class='add'>+			  const struct ib_send_wr *wr)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(qp-&gt;ibqp.device);</div><div class='add'>+	struct ionic_sq_meta *meta;</div><div class='add'>+	struct ionic_v1_wqe *wqe;</div><div class='add'>+</div><div class='add'>+	if (wr-&gt;send_flags &amp; (IB_SEND_SOLICITED | IB_SEND_INLINE))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	meta = &amp;qp-&gt;sq_meta[qp-&gt;sq.prod];</div><div class='add'>+	wqe = ionic_queue_at_prod(&amp;qp-&gt;sq);</div><div class='add'>+</div><div class='add'>+	ionic_prep_sq_wqe(qp, wqe);</div><div class='add'>+</div><div class='add'>+	wqe-&gt;base.op = IONIC_OP(dev-&gt;lif_cfg.rdma_version, LOCAL_INV);</div><div class='add'>+	wqe-&gt;base.imm_data_key = cpu_to_be32(wr-&gt;ex.invalidate_rkey);</div><div class='add'>+</div><div class='add'>+	meta-&gt;len = 0;</div><div class='add'>+	meta-&gt;ibop = IB_WC_LOCAL_INV;</div><div class='add'>+</div><div class='add'>+	ionic_prep_base(qp, wr, meta, wqe);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_prep_reg(struct ionic_qp *qp,</div><div class='add'>+			  const struct ib_reg_wr *wr)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(qp-&gt;ibqp.device);</div><div class='add'>+	struct ionic_mr *mr = to_ionic_mr(wr-&gt;mr);</div><div class='add'>+	struct ionic_sq_meta *meta;</div><div class='add'>+	struct ionic_v1_wqe *wqe;</div><div class='add'>+	__le64 dma_addr;</div><div class='add'>+	int flags;</div><div class='add'>+</div><div class='add'>+	if (wr-&gt;wr.send_flags &amp; (IB_SEND_SOLICITED | IB_SEND_INLINE))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	/* must call ib_map_mr_sg before posting reg wr */</div><div class='add'>+	if (!mr-&gt;buf.tbl_pages)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	meta = &amp;qp-&gt;sq_meta[qp-&gt;sq.prod];</div><div class='add'>+	wqe = ionic_queue_at_prod(&amp;qp-&gt;sq);</div><div class='add'>+</div><div class='add'>+	ionic_prep_sq_wqe(qp, wqe);</div><div class='add'>+</div><div class='add'>+	flags = to_ionic_mr_flags(wr-&gt;access);</div><div class='add'>+</div><div class='add'>+	wqe-&gt;base.op = IONIC_OP(dev-&gt;lif_cfg.rdma_version, REG_MR);</div><div class='add'>+	wqe-&gt;base.num_sge_key = wr-&gt;key;</div><div class='add'>+	wqe-&gt;base.imm_data_key = cpu_to_be32(mr-&gt;ibmr.lkey);</div><div class='add'>+	wqe-&gt;reg_mr.va = cpu_to_be64(mr-&gt;ibmr.iova);</div><div class='add'>+	wqe-&gt;reg_mr.length = cpu_to_be64(mr-&gt;ibmr.length);</div><div class='add'>+	wqe-&gt;reg_mr.offset = ionic_pgtbl_off(&amp;mr-&gt;buf, mr-&gt;ibmr.iova);</div><div class='add'>+	dma_addr = ionic_pgtbl_dma(&amp;mr-&gt;buf, mr-&gt;ibmr.iova);</div><div class='add'>+	wqe-&gt;reg_mr.dma_addr = cpu_to_be64(le64_to_cpu(dma_addr));</div><div class='add'>+</div><div class='add'>+	wqe-&gt;reg_mr.map_count = cpu_to_be32(mr-&gt;buf.tbl_pages);</div><div class='add'>+	wqe-&gt;reg_mr.flags = cpu_to_be16(flags);</div><div class='add'>+	wqe-&gt;reg_mr.dir_size_log2 = 0;</div><div class='add'>+	wqe-&gt;reg_mr.page_size_log2 = order_base_2(mr-&gt;ibmr.page_size);</div><div class='add'>+</div><div class='add'>+	meta-&gt;len = 0;</div><div class='add'>+	meta-&gt;ibop = IB_WC_REG_MR;</div><div class='add'>+</div><div class='add'>+	ionic_prep_base(qp, &amp;wr-&gt;wr, meta, wqe);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_prep_one_rc(struct ionic_qp *qp,</div><div class='add'>+			     const struct ib_send_wr *wr)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(qp-&gt;ibqp.device);</div><div class='add'>+	int rc = 0;</div><div class='add'>+</div><div class='add'>+	switch (wr-&gt;opcode) {</div><div class='add'>+	case IB_WR_SEND:</div><div class='add'>+	case IB_WR_SEND_WITH_IMM:</div><div class='add'>+	case IB_WR_SEND_WITH_INV:</div><div class='add'>+		rc = ionic_prep_send(qp, wr);</div><div class='add'>+		break;</div><div class='add'>+	case IB_WR_RDMA_READ:</div><div class='add'>+	case IB_WR_RDMA_WRITE:</div><div class='add'>+	case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='add'>+		rc = ionic_prep_rdma(qp, rdma_wr(wr));</div><div class='add'>+		break;</div><div class='add'>+	case IB_WR_ATOMIC_CMP_AND_SWP:</div><div class='add'>+	case IB_WR_ATOMIC_FETCH_AND_ADD:</div><div class='add'>+		rc = ionic_prep_atomic(qp, atomic_wr(wr));</div><div class='add'>+		break;</div><div class='add'>+	case IB_WR_LOCAL_INV:</div><div class='add'>+		rc = ionic_prep_inv(qp, wr);</div><div class='add'>+		break;</div><div class='add'>+	case IB_WR_REG_MR:</div><div class='add'>+		rc = ionic_prep_reg(qp, reg_wr(wr));</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev, "invalid opcode %d\n", wr-&gt;opcode);</div><div class='add'>+		rc = -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_prep_one_ud(struct ionic_qp *qp,</div><div class='add'>+			     const struct ib_send_wr *wr)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(qp-&gt;ibqp.device);</div><div class='add'>+	int rc = 0;</div><div class='add'>+</div><div class='add'>+	switch (wr-&gt;opcode) {</div><div class='add'>+	case IB_WR_SEND:</div><div class='add'>+	case IB_WR_SEND_WITH_IMM:</div><div class='add'>+		rc = ionic_prep_send_ud(qp, ud_wr(wr));</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ibdev, "invalid opcode %d\n", wr-&gt;opcode);</div><div class='add'>+		rc = -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_prep_recv(struct ionic_qp *qp,</div><div class='add'>+			   const struct ib_recv_wr *wr)</div><div class='add'>+{</div><div class='add'>+	struct ionic_rq_meta *meta;</div><div class='add'>+	struct ionic_v1_wqe *wqe;</div><div class='add'>+	s64 signed_len;</div><div class='add'>+	u32 mval;</div><div class='add'>+</div><div class='add'>+	wqe = ionic_queue_at_prod(&amp;qp-&gt;rq);</div><div class='add'>+</div><div class='add'>+	/* if wqe is owned by device, caller can try posting again soon */</div><div class='add'>+	if (wqe-&gt;base.flags &amp; cpu_to_be16(IONIC_V1_FLAG_FENCE))</div><div class='add'>+		return -EAGAIN;</div><div class='add'>+</div><div class='add'>+	meta = qp-&gt;rq_meta_head;</div><div class='add'>+	if (unlikely(meta == IONIC_META_LAST) ||</div><div class='add'>+	    unlikely(meta == IONIC_META_POSTED))</div><div class='add'>+		return -EIO;</div><div class='add'>+</div><div class='add'>+	ionic_prep_rq_wqe(qp, wqe);</div><div class='add'>+</div><div class='add'>+	mval = ionic_v1_recv_wqe_max_sge(qp-&gt;rq.stride_log2, qp-&gt;rq_spec,</div><div class='add'>+					 false);</div><div class='add'>+	signed_len = ionic_prep_pld(wqe, &amp;wqe-&gt;recv.pld,</div><div class='add'>+				    qp-&gt;rq_spec, mval,</div><div class='add'>+				    wr-&gt;sg_list, wr-&gt;num_sge);</div><div class='add'>+	if (signed_len &lt; 0)</div><div class='add'>+		return signed_len;</div><div class='add'>+</div><div class='add'>+	meta-&gt;wrid = wr-&gt;wr_id;</div><div class='add'>+</div><div class='add'>+	wqe-&gt;base.wqe_id = meta - qp-&gt;rq_meta;</div><div class='add'>+	wqe-&gt;base.num_sge_key = wr-&gt;num_sge;</div><div class='add'>+</div><div class='add'>+	/* total length for recv goes in base imm_data_key */</div><div class='add'>+	wqe-&gt;base.imm_data_key = cpu_to_be32(signed_len);</div><div class='add'>+</div><div class='add'>+	ionic_queue_produce(&amp;qp-&gt;rq);</div><div class='add'>+</div><div class='add'>+	qp-&gt;rq_meta_head = meta-&gt;next;</div><div class='add'>+	meta-&gt;next = IONIC_META_POSTED;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_post_send_common(struct ionic_ibdev *dev,</div><div class='add'>+				  struct ionic_vcq *vcq,</div><div class='add'>+				  struct ionic_cq *cq,</div><div class='add'>+				  struct ionic_qp *qp,</div><div class='add'>+				  const struct ib_send_wr *wr,</div><div class='add'>+				  const struct ib_send_wr **bad)</div><div class='add'>+{</div><div class='add'>+	unsigned long irqflags;</div><div class='add'>+	bool notify = false;</div><div class='add'>+	int spend, rc = 0;</div><div class='add'>+</div><div class='add'>+	if (!bad)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!qp-&gt;has_sq) {</div><div class='add'>+		*bad = wr;</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;state &lt; IB_QPS_RTS) {</div><div class='add'>+		*bad = wr;</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;qp-&gt;sq_lock, irqflags);</div><div class='add'>+</div><div class='add'>+	while (wr) {</div><div class='add'>+		if (ionic_queue_full(&amp;qp-&gt;sq)) {</div><div class='add'>+			ibdev_dbg(&amp;dev-&gt;ibdev, "queue full");</div><div class='add'>+			rc = -ENOMEM;</div><div class='add'>+			goto out;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (qp-&gt;ibqp.qp_type == IB_QPT_UD ||</div><div class='add'>+		    qp-&gt;ibqp.qp_type == IB_QPT_GSI)</div><div class='add'>+			rc = ionic_prep_one_ud(qp, wr);</div><div class='add'>+		else</div><div class='add'>+			rc = ionic_prep_one_rc(qp, wr);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto out;</div><div class='add'>+</div><div class='add'>+		wr = wr-&gt;next;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	spin_unlock_irqrestore(&amp;qp-&gt;sq_lock, irqflags);</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cq-&gt;lock, irqflags);</div><div class='add'>+	spin_lock(&amp;qp-&gt;sq_lock);</div><div class='add'>+</div><div class='add'>+	if (likely(qp-&gt;sq.prod != qp-&gt;sq_old_prod)) {</div><div class='add'>+		/* ring cq doorbell just in time */</div><div class='add'>+		spend = (qp-&gt;sq.prod - qp-&gt;sq_old_prod) &amp; qp-&gt;sq.mask;</div><div class='add'>+		ionic_reserve_cq(dev, cq, spend);</div><div class='add'>+</div><div class='add'>+		qp-&gt;sq_old_prod = qp-&gt;sq.prod;</div><div class='add'>+</div><div class='add'>+		ionic_dbell_ring(dev-&gt;lif_cfg.dbpage, dev-&gt;lif_cfg.sq_qtype,</div><div class='add'>+				 ionic_queue_dbell_val(&amp;qp-&gt;sq));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;sq_flush) {</div><div class='add'>+		notify = true;</div><div class='add'>+		cq-&gt;flush = true;</div><div class='add'>+		list_move_tail(&amp;qp-&gt;cq_flush_sq, &amp;cq-&gt;flush_sq);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_unlock(&amp;qp-&gt;sq_lock);</div><div class='add'>+	spin_unlock_irqrestore(&amp;cq-&gt;lock, irqflags);</div><div class='add'>+</div><div class='add'>+	if (notify &amp;&amp; vcq-&gt;ibcq.comp_handler)</div><div class='add'>+		vcq-&gt;ibcq.comp_handler(&amp;vcq-&gt;ibcq, vcq-&gt;ibcq.cq_context);</div><div class='add'>+</div><div class='add'>+	*bad = wr;</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_post_recv_common(struct ionic_ibdev *dev,</div><div class='add'>+				  struct ionic_vcq *vcq,</div><div class='add'>+				  struct ionic_cq *cq,</div><div class='add'>+				  struct ionic_qp *qp,</div><div class='add'>+				  const struct ib_recv_wr *wr,</div><div class='add'>+				  const struct ib_recv_wr **bad)</div><div class='add'>+{</div><div class='add'>+	unsigned long irqflags;</div><div class='add'>+	bool notify = false;</div><div class='add'>+	int spend, rc = 0;</div><div class='add'>+</div><div class='add'>+	if (!bad)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!qp-&gt;has_rq) {</div><div class='add'>+		*bad = wr;</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;state &lt; IB_QPS_INIT) {</div><div class='add'>+		*bad = wr;</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;qp-&gt;rq_lock, irqflags);</div><div class='add'>+</div><div class='add'>+	while (wr) {</div><div class='add'>+		if (ionic_queue_full(&amp;qp-&gt;rq)) {</div><div class='add'>+			ibdev_dbg(&amp;dev-&gt;ibdev, "queue full");</div><div class='add'>+			rc = -ENOMEM;</div><div class='add'>+			goto out;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		rc = ionic_prep_recv(qp, wr);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto out;</div><div class='add'>+</div><div class='add'>+		wr = wr-&gt;next;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	if (!cq) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;qp-&gt;rq_lock, irqflags);</div><div class='add'>+		goto out_unlocked;</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;qp-&gt;rq_lock, irqflags);</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cq-&gt;lock, irqflags);</div><div class='add'>+	spin_lock(&amp;qp-&gt;rq_lock);</div><div class='add'>+</div><div class='add'>+	if (likely(qp-&gt;rq.prod != qp-&gt;rq_old_prod)) {</div><div class='add'>+		/* ring cq doorbell just in time */</div><div class='add'>+		spend = (qp-&gt;rq.prod - qp-&gt;rq_old_prod) &amp; qp-&gt;rq.mask;</div><div class='add'>+		ionic_reserve_cq(dev, cq, spend);</div><div class='add'>+</div><div class='add'>+		qp-&gt;rq_old_prod = qp-&gt;rq.prod;</div><div class='add'>+</div><div class='add'>+		ionic_dbell_ring(dev-&gt;lif_cfg.dbpage, dev-&gt;lif_cfg.rq_qtype,</div><div class='add'>+				 ionic_queue_dbell_val(&amp;qp-&gt;rq));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;rq_flush) {</div><div class='add'>+		notify = true;</div><div class='add'>+		cq-&gt;flush = true;</div><div class='add'>+		list_move_tail(&amp;qp-&gt;cq_flush_rq, &amp;cq-&gt;flush_rq);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_unlock(&amp;qp-&gt;rq_lock);</div><div class='add'>+	spin_unlock_irqrestore(&amp;cq-&gt;lock, irqflags);</div><div class='add'>+</div><div class='add'>+	if (notify &amp;&amp; vcq-&gt;ibcq.comp_handler)</div><div class='add'>+		vcq-&gt;ibcq.comp_handler(&amp;vcq-&gt;ibcq, vcq-&gt;ibcq.cq_context);</div><div class='add'>+</div><div class='add'>+out_unlocked:</div><div class='add'>+	*bad = wr;</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,</div><div class='add'>+		    const struct ib_send_wr **bad)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibqp-&gt;device);</div><div class='add'>+	struct ionic_vcq *vcq = to_ionic_vcq(ibqp-&gt;send_cq);</div><div class='add'>+	struct ionic_qp *qp = to_ionic_qp(ibqp);</div><div class='add'>+	struct ionic_cq *cq =</div><div class='add'>+		to_ionic_vcq_cq(ibqp-&gt;send_cq, qp-&gt;udma_idx);</div><div class='add'>+</div><div class='add'>+	return ionic_post_send_common(dev, vcq, cq, qp, wr, bad);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,</div><div class='add'>+		    const struct ib_recv_wr **bad)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibqp-&gt;device);</div><div class='add'>+	struct ionic_vcq *vcq = to_ionic_vcq(ibqp-&gt;recv_cq);</div><div class='add'>+	struct ionic_qp *qp = to_ionic_qp(ibqp);</div><div class='add'>+	struct ionic_cq *cq =</div><div class='add'>+		to_ionic_vcq_cq(ibqp-&gt;recv_cq, qp-&gt;udma_idx);</div><div class='add'>+</div><div class='add'>+	return ionic_post_recv_common(dev, vcq, cq, qp, wr, bad);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/ionic/ionic_fw.h b/drivers/infiniband/hw/ionic/ionic_fw.h<br/>new file mode 100644<br/>index 000000000000..adfbb89d856c<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/ionic/ionic_fw.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/ionic/ionic_fw.h</a></div><div class='hunk'>@@ -0,0 +1,1029 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 */</div><div class='add'>+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */</div><div class='add'>+</div><div class='add'>+#ifndef _IONIC_FW_H_</div><div class='add'>+#define _IONIC_FW_H_</div><div class='add'>+</div><div class='add'>+#include &lt;linux/kernel.h&gt;</div><div class='add'>+#include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+</div><div class='add'>+/* common for ib spec */</div><div class='add'>+</div><div class='add'>+#define IONIC_EXP_DBELL_SZ		8</div><div class='add'>+</div><div class='add'>+enum ionic_mrid_bits {</div><div class='add'>+	IONIC_MRID_INDEX_SHIFT		= 8,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline u32 ionic_mrid(u32 index, u8 key)</div><div class='add'>+{</div><div class='add'>+	return (index &lt;&lt; IONIC_MRID_INDEX_SHIFT) | key;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 ionic_mrid_index(u32 lrkey)</div><div class='add'>+{</div><div class='add'>+	return lrkey &gt;&gt; IONIC_MRID_INDEX_SHIFT;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* common to all versions */</div><div class='add'>+</div><div class='add'>+/* wqe scatter gather element */</div><div class='add'>+struct ionic_sge {</div><div class='add'>+	__be64				va;</div><div class='add'>+	__be32				len;</div><div class='add'>+	__be32				lkey;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* admin queue mr type */</div><div class='add'>+enum ionic_mr_flags {</div><div class='add'>+	/* bits that determine mr access */</div><div class='add'>+	IONIC_MRF_LOCAL_WRITE		= BIT(0),</div><div class='add'>+	IONIC_MRF_REMOTE_WRITE		= BIT(1),</div><div class='add'>+	IONIC_MRF_REMOTE_READ		= BIT(2),</div><div class='add'>+	IONIC_MRF_REMOTE_ATOMIC		= BIT(3),</div><div class='add'>+	IONIC_MRF_MW_BIND		= BIT(4),</div><div class='add'>+	IONIC_MRF_ZERO_BASED		= BIT(5),</div><div class='add'>+	IONIC_MRF_ON_DEMAND		= BIT(6),</div><div class='add'>+	IONIC_MRF_PB			= BIT(7),</div><div class='add'>+	IONIC_MRF_ACCESS_MASK		= BIT(12) - 1,</div><div class='add'>+</div><div class='add'>+	/* bits that determine mr type */</div><div class='add'>+	IONIC_MRF_UKEY_EN		= BIT(13),</div><div class='add'>+	IONIC_MRF_IS_MW			= BIT(14),</div><div class='add'>+	IONIC_MRF_INV_EN		= BIT(15),</div><div class='add'>+</div><div class='add'>+	/* base flags combinations for mr types */</div><div class='add'>+	IONIC_MRF_USER_MR		= 0,</div><div class='add'>+	IONIC_MRF_PHYS_MR		= (IONIC_MRF_UKEY_EN |</div><div class='add'>+					   IONIC_MRF_INV_EN),</div><div class='add'>+	IONIC_MRF_MW_1			= (IONIC_MRF_UKEY_EN |</div><div class='add'>+					   IONIC_MRF_IS_MW),</div><div class='add'>+	IONIC_MRF_MW_2			= (IONIC_MRF_UKEY_EN |</div><div class='add'>+					   IONIC_MRF_IS_MW |</div><div class='add'>+					   IONIC_MRF_INV_EN),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline int to_ionic_mr_flags(int access)</div><div class='add'>+{</div><div class='add'>+	int flags = 0;</div><div class='add'>+</div><div class='add'>+	if (access &amp; IB_ACCESS_LOCAL_WRITE)</div><div class='add'>+		flags |= IONIC_MRF_LOCAL_WRITE;</div><div class='add'>+</div><div class='add'>+	if (access &amp; IB_ACCESS_REMOTE_READ)</div><div class='add'>+		flags |= IONIC_MRF_REMOTE_READ;</div><div class='add'>+</div><div class='add'>+	if (access &amp; IB_ACCESS_REMOTE_WRITE)</div><div class='add'>+		flags |= IONIC_MRF_REMOTE_WRITE;</div><div class='add'>+</div><div class='add'>+	if (access &amp; IB_ACCESS_REMOTE_ATOMIC)</div><div class='add'>+		flags |= IONIC_MRF_REMOTE_ATOMIC;</div><div class='add'>+</div><div class='add'>+	if (access &amp; IB_ACCESS_MW_BIND)</div><div class='add'>+		flags |= IONIC_MRF_MW_BIND;</div><div class='add'>+</div><div class='add'>+	if (access &amp; IB_ZERO_BASED)</div><div class='add'>+		flags |= IONIC_MRF_ZERO_BASED;</div><div class='add'>+</div><div class='add'>+	return flags;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+enum ionic_qp_flags {</div><div class='add'>+	/* bits that determine qp access */</div><div class='add'>+	IONIC_QPF_REMOTE_WRITE		= BIT(0),</div><div class='add'>+	IONIC_QPF_REMOTE_READ		= BIT(1),</div><div class='add'>+	IONIC_QPF_REMOTE_ATOMIC		= BIT(2),</div><div class='add'>+</div><div class='add'>+	/* bits that determine other qp behavior */</div><div class='add'>+	IONIC_QPF_SQ_PB			= BIT(6),</div><div class='add'>+	IONIC_QPF_RQ_PB			= BIT(7),</div><div class='add'>+	IONIC_QPF_SQ_SPEC		= BIT(8),</div><div class='add'>+	IONIC_QPF_RQ_SPEC		= BIT(9),</div><div class='add'>+	IONIC_QPF_REMOTE_PRIVILEGED	= BIT(10),</div><div class='add'>+	IONIC_QPF_SQ_DRAINING		= BIT(11),</div><div class='add'>+	IONIC_QPF_SQD_NOTIFY		= BIT(12),</div><div class='add'>+	IONIC_QPF_SQ_CMB		= BIT(13),</div><div class='add'>+	IONIC_QPF_RQ_CMB		= BIT(14),</div><div class='add'>+	IONIC_QPF_PRIVILEGED		= BIT(15),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline int from_ionic_qp_flags(int flags)</div><div class='add'>+{</div><div class='add'>+	int access_flags = 0;</div><div class='add'>+</div><div class='add'>+	if (flags &amp; IONIC_QPF_REMOTE_WRITE)</div><div class='add'>+		access_flags |= IB_ACCESS_REMOTE_WRITE;</div><div class='add'>+</div><div class='add'>+	if (flags &amp; IONIC_QPF_REMOTE_READ)</div><div class='add'>+		access_flags |= IB_ACCESS_REMOTE_READ;</div><div class='add'>+</div><div class='add'>+	if (flags &amp; IONIC_QPF_REMOTE_ATOMIC)</div><div class='add'>+		access_flags |= IB_ACCESS_REMOTE_ATOMIC;</div><div class='add'>+</div><div class='add'>+	return access_flags;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int to_ionic_qp_flags(int access, bool sqd_notify,</div><div class='add'>+				    bool sq_is_cmb, bool rq_is_cmb,</div><div class='add'>+				    bool sq_spec, bool rq_spec,</div><div class='add'>+				    bool privileged, bool remote_privileged)</div><div class='add'>+{</div><div class='add'>+	int flags = 0;</div><div class='add'>+</div><div class='add'>+	if (access &amp; IB_ACCESS_REMOTE_WRITE)</div><div class='add'>+		flags |= IONIC_QPF_REMOTE_WRITE;</div><div class='add'>+</div><div class='add'>+	if (access &amp; IB_ACCESS_REMOTE_READ)</div><div class='add'>+		flags |= IONIC_QPF_REMOTE_READ;</div><div class='add'>+</div><div class='add'>+	if (access &amp; IB_ACCESS_REMOTE_ATOMIC)</div><div class='add'>+		flags |= IONIC_QPF_REMOTE_ATOMIC;</div><div class='add'>+</div><div class='add'>+	if (sqd_notify)</div><div class='add'>+		flags |= IONIC_QPF_SQD_NOTIFY;</div><div class='add'>+</div><div class='add'>+	if (sq_is_cmb)</div><div class='add'>+		flags |= IONIC_QPF_SQ_CMB;</div><div class='add'>+</div><div class='add'>+	if (rq_is_cmb)</div><div class='add'>+		flags |= IONIC_QPF_RQ_CMB;</div><div class='add'>+</div><div class='add'>+	if (sq_spec)</div><div class='add'>+		flags |= IONIC_QPF_SQ_SPEC;</div><div class='add'>+</div><div class='add'>+	if (rq_spec)</div><div class='add'>+		flags |= IONIC_QPF_RQ_SPEC;</div><div class='add'>+</div><div class='add'>+	if (privileged)</div><div class='add'>+		flags |= IONIC_QPF_PRIVILEGED;</div><div class='add'>+</div><div class='add'>+	if (remote_privileged)</div><div class='add'>+		flags |= IONIC_QPF_REMOTE_PRIVILEGED;</div><div class='add'>+</div><div class='add'>+	return flags;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* cqe non-admin status indicated in status_length field when err bit is set */</div><div class='add'>+enum ionic_status {</div><div class='add'>+	IONIC_STS_OK,</div><div class='add'>+	IONIC_STS_LOCAL_LEN_ERR,</div><div class='add'>+	IONIC_STS_LOCAL_QP_OPER_ERR,</div><div class='add'>+	IONIC_STS_LOCAL_PROT_ERR,</div><div class='add'>+	IONIC_STS_WQE_FLUSHED_ERR,</div><div class='add'>+	IONIC_STS_MEM_MGMT_OPER_ERR,</div><div class='add'>+	IONIC_STS_BAD_RESP_ERR,</div><div class='add'>+	IONIC_STS_LOCAL_ACC_ERR,</div><div class='add'>+	IONIC_STS_REMOTE_INV_REQ_ERR,</div><div class='add'>+	IONIC_STS_REMOTE_ACC_ERR,</div><div class='add'>+	IONIC_STS_REMOTE_OPER_ERR,</div><div class='add'>+	IONIC_STS_RETRY_EXCEEDED,</div><div class='add'>+	IONIC_STS_RNR_RETRY_EXCEEDED,</div><div class='add'>+	IONIC_STS_XRC_VIO_ERR,</div><div class='add'>+	IONIC_STS_LOCAL_SGL_INV_ERR,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline int ionic_to_ib_status(int sts)</div><div class='add'>+{</div><div class='add'>+	switch (sts) {</div><div class='add'>+	case IONIC_STS_OK:</div><div class='add'>+		return IB_WC_SUCCESS;</div><div class='add'>+	case IONIC_STS_LOCAL_LEN_ERR:</div><div class='add'>+		return IB_WC_LOC_LEN_ERR;</div><div class='add'>+	case IONIC_STS_LOCAL_QP_OPER_ERR:</div><div class='add'>+	case IONIC_STS_LOCAL_SGL_INV_ERR:</div><div class='add'>+		return IB_WC_LOC_QP_OP_ERR;</div><div class='add'>+	case IONIC_STS_LOCAL_PROT_ERR:</div><div class='add'>+		return IB_WC_LOC_PROT_ERR;</div><div class='add'>+	case IONIC_STS_WQE_FLUSHED_ERR:</div><div class='add'>+		return IB_WC_WR_FLUSH_ERR;</div><div class='add'>+	case IONIC_STS_MEM_MGMT_OPER_ERR:</div><div class='add'>+		return IB_WC_MW_BIND_ERR;</div><div class='add'>+	case IONIC_STS_BAD_RESP_ERR:</div><div class='add'>+		return IB_WC_BAD_RESP_ERR;</div><div class='add'>+	case IONIC_STS_LOCAL_ACC_ERR:</div><div class='add'>+		return IB_WC_LOC_ACCESS_ERR;</div><div class='add'>+	case IONIC_STS_REMOTE_INV_REQ_ERR:</div><div class='add'>+		return IB_WC_REM_INV_REQ_ERR;</div><div class='add'>+	case IONIC_STS_REMOTE_ACC_ERR:</div><div class='add'>+		return IB_WC_REM_ACCESS_ERR;</div><div class='add'>+	case IONIC_STS_REMOTE_OPER_ERR:</div><div class='add'>+		return IB_WC_REM_OP_ERR;</div><div class='add'>+	case IONIC_STS_RETRY_EXCEEDED:</div><div class='add'>+		return IB_WC_RETRY_EXC_ERR;</div><div class='add'>+	case IONIC_STS_RNR_RETRY_EXCEEDED:</div><div class='add'>+		return IB_WC_RNR_RETRY_EXC_ERR;</div><div class='add'>+	case IONIC_STS_XRC_VIO_ERR:</div><div class='add'>+	default:</div><div class='add'>+		return IB_WC_GENERAL_ERR;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* admin queue qp type */</div><div class='add'>+enum ionic_qp_type {</div><div class='add'>+	IONIC_QPT_RC,</div><div class='add'>+	IONIC_QPT_UC,</div><div class='add'>+	IONIC_QPT_RD,</div><div class='add'>+	IONIC_QPT_UD,</div><div class='add'>+	IONIC_QPT_SRQ,</div><div class='add'>+	IONIC_QPT_XRC_INI,</div><div class='add'>+	IONIC_QPT_XRC_TGT,</div><div class='add'>+	IONIC_QPT_XRC_SRQ,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline int to_ionic_qp_type(enum ib_qp_type type)</div><div class='add'>+{</div><div class='add'>+	switch (type) {</div><div class='add'>+	case IB_QPT_GSI:</div><div class='add'>+	case IB_QPT_UD:</div><div class='add'>+		return IONIC_QPT_UD;</div><div class='add'>+	case IB_QPT_RC:</div><div class='add'>+		return IONIC_QPT_RC;</div><div class='add'>+	case IB_QPT_UC:</div><div class='add'>+		return IONIC_QPT_UC;</div><div class='add'>+	case IB_QPT_XRC_INI:</div><div class='add'>+		return IONIC_QPT_XRC_INI;</div><div class='add'>+	case IB_QPT_XRC_TGT:</div><div class='add'>+		return IONIC_QPT_XRC_TGT;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* admin queue qp state */</div><div class='add'>+enum ionic_qp_state {</div><div class='add'>+	IONIC_QPS_RESET,</div><div class='add'>+	IONIC_QPS_INIT,</div><div class='add'>+	IONIC_QPS_RTR,</div><div class='add'>+	IONIC_QPS_RTS,</div><div class='add'>+	IONIC_QPS_SQD,</div><div class='add'>+	IONIC_QPS_SQE,</div><div class='add'>+	IONIC_QPS_ERR,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline int from_ionic_qp_state(enum ionic_qp_state state)</div><div class='add'>+{</div><div class='add'>+	switch (state) {</div><div class='add'>+	case IONIC_QPS_RESET:</div><div class='add'>+		return IB_QPS_RESET;</div><div class='add'>+	case IONIC_QPS_INIT:</div><div class='add'>+		return IB_QPS_INIT;</div><div class='add'>+	case IONIC_QPS_RTR:</div><div class='add'>+		return IB_QPS_RTR;</div><div class='add'>+	case IONIC_QPS_RTS:</div><div class='add'>+		return IB_QPS_RTS;</div><div class='add'>+	case IONIC_QPS_SQD:</div><div class='add'>+		return IB_QPS_SQD;</div><div class='add'>+	case IONIC_QPS_SQE:</div><div class='add'>+		return IB_QPS_SQE;</div><div class='add'>+	case IONIC_QPS_ERR:</div><div class='add'>+		return IB_QPS_ERR;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int to_ionic_qp_state(enum ib_qp_state state)</div><div class='add'>+{</div><div class='add'>+	switch (state) {</div><div class='add'>+	case IB_QPS_RESET:</div><div class='add'>+		return IONIC_QPS_RESET;</div><div class='add'>+	case IB_QPS_INIT:</div><div class='add'>+		return IONIC_QPS_INIT;</div><div class='add'>+	case IB_QPS_RTR:</div><div class='add'>+		return IONIC_QPS_RTR;</div><div class='add'>+	case IB_QPS_RTS:</div><div class='add'>+		return IONIC_QPS_RTS;</div><div class='add'>+	case IB_QPS_SQD:</div><div class='add'>+		return IONIC_QPS_SQD;</div><div class='add'>+	case IB_QPS_SQE:</div><div class='add'>+		return IONIC_QPS_SQE;</div><div class='add'>+	case IB_QPS_ERR:</div><div class='add'>+		return IONIC_QPS_ERR;</div><div class='add'>+	default:</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int to_ionic_qp_modify_state(enum ib_qp_state to_state,</div><div class='add'>+					   enum ib_qp_state from_state)</div><div class='add'>+{</div><div class='add'>+	return to_ionic_qp_state(to_state) |</div><div class='add'>+		(to_ionic_qp_state(from_state) &lt;&lt; 4);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* fw abi v1 */</div><div class='add'>+</div><div class='add'>+/* data payload part of v1 wqe */</div><div class='add'>+union ionic_v1_pld {</div><div class='add'>+	struct ionic_sge	sgl[2];</div><div class='add'>+	__be32			spec32[8];</div><div class='add'>+	__be16			spec16[16];</div><div class='add'>+	__u8			data[32];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* completion queue v1 cqe */</div><div class='add'>+struct ionic_v1_cqe {</div><div class='add'>+	union {</div><div class='add'>+		struct {</div><div class='add'>+			__be16		cmd_idx;</div><div class='add'>+			__u8		cmd_op;</div><div class='add'>+			__u8		rsvd[17];</div><div class='add'>+			__le16		old_sq_cindex;</div><div class='add'>+			__le16		old_rq_cq_cindex;</div><div class='add'>+		} admin;</div><div class='add'>+		struct {</div><div class='add'>+			__u64		wqe_id;</div><div class='add'>+			__be32		src_qpn_op;</div><div class='add'>+			__u8		src_mac[6];</div><div class='add'>+			__be16		vlan_tag;</div><div class='add'>+			__be32		imm_data_rkey;</div><div class='add'>+		} recv;</div><div class='add'>+		struct {</div><div class='add'>+			__u8		rsvd[4];</div><div class='add'>+			__be32		msg_msn;</div><div class='add'>+			__u8		rsvd2[8];</div><div class='add'>+			__u64		npg_wqe_id;</div><div class='add'>+		} send;</div><div class='add'>+	};</div><div class='add'>+	__be32				status_length;</div><div class='add'>+	__be32				qid_type_flags;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* bits for cqe recv */</div><div class='add'>+enum ionic_v1_cqe_src_qpn_bits {</div><div class='add'>+	IONIC_V1_CQE_RECV_QPN_MASK	= 0xffffff,</div><div class='add'>+	IONIC_V1_CQE_RECV_OP_SHIFT	= 24,</div><div class='add'>+</div><div class='add'>+	/* MASK could be 0x3, but need 0x1f for makeshift values:</div><div class='add'>+	 * OP_TYPE_RDMA_OPER_WITH_IMM, OP_TYPE_SEND_RCVD</div><div class='add'>+	 */</div><div class='add'>+	IONIC_V1_CQE_RECV_OP_MASK	= 0x1f,</div><div class='add'>+	IONIC_V1_CQE_RECV_OP_SEND	= 0,</div><div class='add'>+	IONIC_V1_CQE_RECV_OP_SEND_INV	= 1,</div><div class='add'>+	IONIC_V1_CQE_RECV_OP_SEND_IMM	= 2,</div><div class='add'>+	IONIC_V1_CQE_RECV_OP_RDMA_IMM	= 3,</div><div class='add'>+</div><div class='add'>+	IONIC_V1_CQE_RECV_IS_IPV4	= BIT(7 + IONIC_V1_CQE_RECV_OP_SHIFT),</div><div class='add'>+	IONIC_V1_CQE_RECV_IS_VLAN	= BIT(6 + IONIC_V1_CQE_RECV_OP_SHIFT),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* bits for cqe qid_type_flags */</div><div class='add'>+enum ionic_v1_cqe_qtf_bits {</div><div class='add'>+	IONIC_V1_CQE_COLOR		= BIT(0),</div><div class='add'>+	IONIC_V1_CQE_ERROR		= BIT(1),</div><div class='add'>+	IONIC_V1_CQE_TYPE_SHIFT		= 5,</div><div class='add'>+	IONIC_V1_CQE_TYPE_MASK		= 0x7,</div><div class='add'>+	IONIC_V1_CQE_QID_SHIFT		= 8,</div><div class='add'>+</div><div class='add'>+	IONIC_V1_CQE_TYPE_ADMIN		= 0,</div><div class='add'>+	IONIC_V1_CQE_TYPE_RECV		= 1,</div><div class='add'>+	IONIC_V1_CQE_TYPE_SEND_MSN	= 2,</div><div class='add'>+	IONIC_V1_CQE_TYPE_SEND_NPG	= 3,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline bool ionic_v1_cqe_color(struct ionic_v1_cqe *cqe)</div><div class='add'>+{</div><div class='add'>+	return cqe-&gt;qid_type_flags &amp; cpu_to_be32(IONIC_V1_CQE_COLOR);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool ionic_v1_cqe_error(struct ionic_v1_cqe *cqe)</div><div class='add'>+{</div><div class='add'>+	return cqe-&gt;qid_type_flags &amp; cpu_to_be32(IONIC_V1_CQE_ERROR);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool ionic_v1_cqe_recv_is_ipv4(struct ionic_v1_cqe *cqe)</div><div class='add'>+{</div><div class='add'>+	return cqe-&gt;recv.src_qpn_op &amp; cpu_to_be32(IONIC_V1_CQE_RECV_IS_IPV4);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool ionic_v1_cqe_recv_is_vlan(struct ionic_v1_cqe *cqe)</div><div class='add'>+{</div><div class='add'>+	return cqe-&gt;recv.src_qpn_op &amp; cpu_to_be32(IONIC_V1_CQE_RECV_IS_VLAN);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void ionic_v1_cqe_clean(struct ionic_v1_cqe *cqe)</div><div class='add'>+{</div><div class='add'>+	cqe-&gt;qid_type_flags |= cpu_to_be32(~0u &lt;&lt; IONIC_V1_CQE_QID_SHIFT);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 ionic_v1_cqe_qtf(struct ionic_v1_cqe *cqe)</div><div class='add'>+{</div><div class='add'>+	return be32_to_cpu(cqe-&gt;qid_type_flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u8 ionic_v1_cqe_qtf_type(u32 qtf)</div><div class='add'>+{</div><div class='add'>+	return (qtf &gt;&gt; IONIC_V1_CQE_TYPE_SHIFT) &amp; IONIC_V1_CQE_TYPE_MASK;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 ionic_v1_cqe_qtf_qid(u32 qtf)</div><div class='add'>+{</div><div class='add'>+	return qtf &gt;&gt; IONIC_V1_CQE_QID_SHIFT;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* v1 base wqe header */</div><div class='add'>+struct ionic_v1_base_hdr {</div><div class='add'>+	__u64				wqe_id;</div><div class='add'>+	__u8				op;</div><div class='add'>+	__u8				num_sge_key;</div><div class='add'>+	__be16				flags;</div><div class='add'>+	__be32				imm_data_key;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* v1 receive wqe body */</div><div class='add'>+struct ionic_v1_recv_bdy {</div><div class='add'>+	__u8				rsvd[16];</div><div class='add'>+	union ionic_v1_pld		pld;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* v1 send/rdma wqe body (common, has sgl) */</div><div class='add'>+struct ionic_v1_common_bdy {</div><div class='add'>+	union {</div><div class='add'>+		struct {</div><div class='add'>+			__be32		ah_id;</div><div class='add'>+			__be32		dest_qpn;</div><div class='add'>+			__be32		dest_qkey;</div><div class='add'>+		} send;</div><div class='add'>+		struct {</div><div class='add'>+			__be32		remote_va_high;</div><div class='add'>+			__be32		remote_va_low;</div><div class='add'>+			__be32		remote_rkey;</div><div class='add'>+		} rdma;</div><div class='add'>+	};</div><div class='add'>+	__be32				length;</div><div class='add'>+	union ionic_v1_pld		pld;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* v1 atomic wqe body */</div><div class='add'>+struct ionic_v1_atomic_bdy {</div><div class='add'>+	__be32				remote_va_high;</div><div class='add'>+	__be32				remote_va_low;</div><div class='add'>+	__be32				remote_rkey;</div><div class='add'>+	__be32				swap_add_high;</div><div class='add'>+	__be32				swap_add_low;</div><div class='add'>+	__be32				compare_high;</div><div class='add'>+	__be32				compare_low;</div><div class='add'>+	__u8				rsvd[4];</div><div class='add'>+	struct ionic_sge		sge;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* v1 reg mr wqe body */</div><div class='add'>+struct ionic_v1_reg_mr_bdy {</div><div class='add'>+	__be64				va;</div><div class='add'>+	__be64				length;</div><div class='add'>+	__be64				offset;</div><div class='add'>+	__be64				dma_addr;</div><div class='add'>+	__be32				map_count;</div><div class='add'>+	__be16				flags;</div><div class='add'>+	__u8				dir_size_log2;</div><div class='add'>+	__u8				page_size_log2;</div><div class='add'>+	__u8				rsvd[8];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* v1 bind mw wqe body */</div><div class='add'>+struct ionic_v1_bind_mw_bdy {</div><div class='add'>+	__be64				va;</div><div class='add'>+	__be64				length;</div><div class='add'>+	__be32				lkey;</div><div class='add'>+	__be16				flags;</div><div class='add'>+	__u8				rsvd[26];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* v1 send/recv wqe */</div><div class='add'>+struct ionic_v1_wqe {</div><div class='add'>+	struct ionic_v1_base_hdr	base;</div><div class='add'>+	union {</div><div class='add'>+		struct ionic_v1_recv_bdy	recv;</div><div class='add'>+		struct ionic_v1_common_bdy	common;</div><div class='add'>+		struct ionic_v1_atomic_bdy	atomic;</div><div class='add'>+		struct ionic_v1_reg_mr_bdy	reg_mr;</div><div class='add'>+		struct ionic_v1_bind_mw_bdy	bind_mw;</div><div class='add'>+	};</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* queue pair v1 send opcodes */</div><div class='add'>+enum ionic_v1_op {</div><div class='add'>+	IONIC_V1_OP_SEND,</div><div class='add'>+	IONIC_V1_OP_SEND_INV,</div><div class='add'>+	IONIC_V1_OP_SEND_IMM,</div><div class='add'>+	IONIC_V1_OP_RDMA_READ,</div><div class='add'>+	IONIC_V1_OP_RDMA_WRITE,</div><div class='add'>+	IONIC_V1_OP_RDMA_WRITE_IMM,</div><div class='add'>+	IONIC_V1_OP_ATOMIC_CS,</div><div class='add'>+	IONIC_V1_OP_ATOMIC_FA,</div><div class='add'>+	IONIC_V1_OP_REG_MR,</div><div class='add'>+	IONIC_V1_OP_LOCAL_INV,</div><div class='add'>+	IONIC_V1_OP_BIND_MW,</div><div class='add'>+</div><div class='add'>+	/* flags */</div><div class='add'>+	IONIC_V1_FLAG_FENCE		= BIT(0),</div><div class='add'>+	IONIC_V1_FLAG_SOL		= BIT(1),</div><div class='add'>+	IONIC_V1_FLAG_INL		= BIT(2),</div><div class='add'>+	IONIC_V1_FLAG_SIG		= BIT(3),</div><div class='add'>+</div><div class='add'>+	/* flags last four bits for sgl spec format */</div><div class='add'>+	IONIC_V1_FLAG_SPEC32		= (1u &lt;&lt; 12),</div><div class='add'>+	IONIC_V1_FLAG_SPEC16		= (2u &lt;&lt; 12),</div><div class='add'>+	IONIC_V1_SPEC_FIRST_SGE		= 2,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* queue pair v2 send opcodes */</div><div class='add'>+enum ionic_v2_op {</div><div class='add'>+	IONIC_V2_OPSL_OUT          = 0x20,</div><div class='add'>+	IONIC_V2_OPSL_IMM          = 0x40,</div><div class='add'>+	IONIC_V2_OPSL_INV          = 0x80,</div><div class='add'>+</div><div class='add'>+	IONIC_V2_OP_SEND           = 0x0 | IONIC_V2_OPSL_OUT,</div><div class='add'>+	IONIC_V2_OP_SEND_IMM       = IONIC_V2_OP_SEND | IONIC_V2_OPSL_IMM,</div><div class='add'>+	IONIC_V2_OP_SEND_INV       = IONIC_V2_OP_SEND | IONIC_V2_OPSL_INV,</div><div class='add'>+</div><div class='add'>+	IONIC_V2_OP_RDMA_WRITE     = 0x1 | IONIC_V2_OPSL_OUT,</div><div class='add'>+	IONIC_V2_OP_RDMA_WRITE_IMM = IONIC_V2_OP_RDMA_WRITE | IONIC_V2_OPSL_IMM,</div><div class='add'>+</div><div class='add'>+	IONIC_V2_OP_RDMA_READ      = 0x2,</div><div class='add'>+</div><div class='add'>+	IONIC_V2_OP_ATOMIC_CS      = 0x4,</div><div class='add'>+	IONIC_V2_OP_ATOMIC_FA      = 0x5,</div><div class='add'>+	IONIC_V2_OP_REG_MR         = 0x6,</div><div class='add'>+	IONIC_V2_OP_LOCAL_INV      = 0x7,</div><div class='add'>+	IONIC_V2_OP_BIND_MW        = 0x8,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline size_t ionic_v1_send_wqe_min_size(int min_sge, int min_data,</div><div class='add'>+						int spec, bool expdb)</div><div class='add'>+{</div><div class='add'>+	size_t sz_wqe, sz_sgl, sz_data;</div><div class='add'>+</div><div class='add'>+	if (spec &gt; IONIC_V1_SPEC_FIRST_SGE)</div><div class='add'>+		min_sge += IONIC_V1_SPEC_FIRST_SGE;</div><div class='add'>+</div><div class='add'>+	if (expdb) {</div><div class='add'>+		min_sge += 1;</div><div class='add'>+		min_data += IONIC_EXP_DBELL_SZ;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	sz_wqe = sizeof(struct ionic_v1_wqe);</div><div class='add'>+	sz_sgl = offsetof(struct ionic_v1_wqe, common.pld.sgl[min_sge]);</div><div class='add'>+	sz_data = offsetof(struct ionic_v1_wqe, common.pld.data[min_data]);</div><div class='add'>+</div><div class='add'>+	if (sz_sgl &gt; sz_wqe)</div><div class='add'>+		sz_wqe = sz_sgl;</div><div class='add'>+</div><div class='add'>+	if (sz_data &gt; sz_wqe)</div><div class='add'>+		sz_wqe = sz_data;</div><div class='add'>+</div><div class='add'>+	return sz_wqe;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int ionic_v1_send_wqe_max_sge(u8 stride_log2, int spec,</div><div class='add'>+					    bool expdb)</div><div class='add'>+{</div><div class='add'>+	struct ionic_sge *sge = (void *)(1ull &lt;&lt; stride_log2);</div><div class='add'>+	struct ionic_v1_wqe *wqe = (void *)0;</div><div class='add'>+	int num_sge = 0;</div><div class='add'>+</div><div class='add'>+	if (expdb)</div><div class='add'>+		sge -= 1;</div><div class='add'>+</div><div class='add'>+	if (spec &gt; IONIC_V1_SPEC_FIRST_SGE)</div><div class='add'>+		num_sge = IONIC_V1_SPEC_FIRST_SGE;</div><div class='add'>+</div><div class='add'>+	num_sge = sge - &amp;wqe-&gt;common.pld.sgl[num_sge];</div><div class='add'>+</div><div class='add'>+	if (spec &amp;&amp; num_sge &gt; spec)</div><div class='add'>+		num_sge = spec;</div><div class='add'>+</div><div class='add'>+	return num_sge;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int ionic_v1_send_wqe_max_data(u8 stride_log2, bool expdb)</div><div class='add'>+{</div><div class='add'>+	struct ionic_v1_wqe *wqe = (void *)0;</div><div class='add'>+	__u8 *data = (void *)(1ull &lt;&lt; stride_log2);</div><div class='add'>+</div><div class='add'>+	if (expdb)</div><div class='add'>+		data -= IONIC_EXP_DBELL_SZ;</div><div class='add'>+</div><div class='add'>+	return data - wqe-&gt;common.pld.data;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline size_t ionic_v1_recv_wqe_min_size(int min_sge, int spec,</div><div class='add'>+						bool expdb)</div><div class='add'>+{</div><div class='add'>+	size_t sz_wqe, sz_sgl;</div><div class='add'>+</div><div class='add'>+	if (spec &gt; IONIC_V1_SPEC_FIRST_SGE)</div><div class='add'>+		min_sge += IONIC_V1_SPEC_FIRST_SGE;</div><div class='add'>+</div><div class='add'>+	if (expdb)</div><div class='add'>+		min_sge += 1;</div><div class='add'>+</div><div class='add'>+	sz_wqe = sizeof(struct ionic_v1_wqe);</div><div class='add'>+	sz_sgl = offsetof(struct ionic_v1_wqe, recv.pld.sgl[min_sge]);</div><div class='add'>+</div><div class='add'>+	if (sz_sgl &gt; sz_wqe)</div><div class='add'>+		sz_wqe = sz_sgl;</div><div class='add'>+</div><div class='add'>+	return sz_wqe;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int ionic_v1_recv_wqe_max_sge(u8 stride_log2, int spec,</div><div class='add'>+					    bool expdb)</div><div class='add'>+{</div><div class='add'>+	struct ionic_sge *sge = (void *)(1ull &lt;&lt; stride_log2);</div><div class='add'>+	struct ionic_v1_wqe *wqe = (void *)0;</div><div class='add'>+	int num_sge = 0;</div><div class='add'>+</div><div class='add'>+	if (expdb)</div><div class='add'>+		sge -= 1;</div><div class='add'>+</div><div class='add'>+	if (spec &gt; IONIC_V1_SPEC_FIRST_SGE)</div><div class='add'>+		num_sge = IONIC_V1_SPEC_FIRST_SGE;</div><div class='add'>+</div><div class='add'>+	num_sge = sge - &amp;wqe-&gt;recv.pld.sgl[num_sge];</div><div class='add'>+</div><div class='add'>+	if (spec &amp;&amp; num_sge &gt; spec)</div><div class='add'>+		num_sge = spec;</div><div class='add'>+</div><div class='add'>+	return num_sge;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int ionic_v1_use_spec_sge(int min_sge, int spec)</div><div class='add'>+{</div><div class='add'>+	if (!spec || min_sge &gt; spec)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (min_sge &lt;= IONIC_V1_SPEC_FIRST_SGE)</div><div class='add'>+		return IONIC_V1_SPEC_FIRST_SGE;</div><div class='add'>+</div><div class='add'>+	return spec;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ionic_admin_stats_hdr {</div><div class='add'>+	__le64		dma_addr;</div><div class='add'>+	__le32		length;</div><div class='add'>+	__le32		id_ver;</div><div class='add'>+	__u8		type_state;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+#define IONIC_ADMIN_STATS_HDRS_IN_V1_LEN 17</div><div class='add'>+static_assert(sizeof(struct ionic_admin_stats_hdr) ==</div><div class='add'>+	       IONIC_ADMIN_STATS_HDRS_IN_V1_LEN);</div><div class='add'>+</div><div class='add'>+struct ionic_admin_create_ah {</div><div class='add'>+	__le64		dma_addr;</div><div class='add'>+	__le32		length;</div><div class='add'>+	__le32		pd_id;</div><div class='add'>+	__le32		id_ver;</div><div class='add'>+	__le16		dbid_flags;</div><div class='add'>+	__u8		csum_profile;</div><div class='add'>+	__u8		crypto;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+#define IONIC_ADMIN_CREATE_AH_IN_V1_LEN 24</div><div class='add'>+static_assert(sizeof(struct ionic_admin_create_ah) ==</div><div class='add'>+	       IONIC_ADMIN_CREATE_AH_IN_V1_LEN);</div><div class='add'>+</div><div class='add'>+struct ionic_admin_destroy_ah {</div><div class='add'>+	__le32		ah_id;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+#define IONIC_ADMIN_DESTROY_AH_IN_V1_LEN 4</div><div class='add'>+static_assert(sizeof(struct ionic_admin_destroy_ah) ==</div><div class='add'>+	       IONIC_ADMIN_DESTROY_AH_IN_V1_LEN);</div><div class='add'>+</div><div class='add'>+struct ionic_admin_query_ah {</div><div class='add'>+	__le64		dma_addr;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+#define IONIC_ADMIN_QUERY_AH_IN_V1_LEN 8</div><div class='add'>+static_assert(sizeof(struct ionic_admin_query_ah) ==</div><div class='add'>+	       IONIC_ADMIN_QUERY_AH_IN_V1_LEN);</div><div class='add'>+</div><div class='add'>+struct ionic_admin_create_mr {</div><div class='add'>+	__le64		va;</div><div class='add'>+	__le64		length;</div><div class='add'>+	__le32		pd_id;</div><div class='add'>+	__le32		id_ver;</div><div class='add'>+	__le32		tbl_index;</div><div class='add'>+	__le32		map_count;</div><div class='add'>+	__le64		dma_addr;</div><div class='add'>+	__le16		dbid_flags;</div><div class='add'>+	__u8		pt_type;</div><div class='add'>+	__u8		dir_size_log2;</div><div class='add'>+	__u8		page_size_log2;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+#define IONIC_ADMIN_CREATE_MR_IN_V1_LEN 45</div><div class='add'>+static_assert(sizeof(struct ionic_admin_create_mr) ==</div><div class='add'>+	       IONIC_ADMIN_CREATE_MR_IN_V1_LEN);</div><div class='add'>+</div><div class='add'>+struct ionic_admin_destroy_mr {</div><div class='add'>+	__le32		mr_id;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+#define IONIC_ADMIN_DESTROY_MR_IN_V1_LEN 4</div><div class='add'>+static_assert(sizeof(struct ionic_admin_destroy_mr) ==</div><div class='add'>+	       IONIC_ADMIN_DESTROY_MR_IN_V1_LEN);</div><div class='add'>+</div><div class='add'>+struct ionic_admin_create_cq {</div><div class='add'>+	__le32		eq_id;</div><div class='add'>+	__u8		depth_log2;</div><div class='add'>+	__u8		stride_log2;</div><div class='add'>+	__u8		dir_size_log2_rsvd;</div><div class='add'>+	__u8		page_size_log2;</div><div class='add'>+	__le32		cq_flags;</div><div class='add'>+	__le32		id_ver;</div><div class='add'>+	__le32		tbl_index;</div><div class='add'>+	__le32		map_count;</div><div class='add'>+	__le64		dma_addr;</div><div class='add'>+	__le16		dbid_flags;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+#define IONIC_ADMIN_CREATE_CQ_IN_V1_LEN 34</div><div class='add'>+static_assert(sizeof(struct ionic_admin_create_cq) ==</div><div class='add'>+	       IONIC_ADMIN_CREATE_CQ_IN_V1_LEN);</div><div class='add'>+</div><div class='add'>+struct ionic_admin_destroy_cq {</div><div class='add'>+	__le32		cq_id;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+#define IONIC_ADMIN_DESTROY_CQ_IN_V1_LEN 4</div><div class='add'>+static_assert(sizeof(struct ionic_admin_destroy_cq) ==</div><div class='add'>+	       IONIC_ADMIN_DESTROY_CQ_IN_V1_LEN);</div><div class='add'>+</div><div class='add'>+struct ionic_admin_create_qp {</div><div class='add'>+	__le32		pd_id;</div><div class='add'>+	__be32		priv_flags;</div><div class='add'>+	__le32		sq_cq_id;</div><div class='add'>+	__u8		sq_depth_log2;</div><div class='add'>+	__u8		sq_stride_log2;</div><div class='add'>+	__u8		sq_dir_size_log2_rsvd;</div><div class='add'>+	__u8		sq_page_size_log2;</div><div class='add'>+	__le32		sq_tbl_index_xrcd_id;</div><div class='add'>+	__le32		sq_map_count;</div><div class='add'>+	__le64		sq_dma_addr;</div><div class='add'>+	__le32		rq_cq_id;</div><div class='add'>+	__u8		rq_depth_log2;</div><div class='add'>+	__u8		rq_stride_log2;</div><div class='add'>+	__u8		rq_dir_size_log2_rsvd;</div><div class='add'>+	__u8		rq_page_size_log2;</div><div class='add'>+	__le32		rq_tbl_index_srq_id;</div><div class='add'>+	__le32		rq_map_count;</div><div class='add'>+	__le64		rq_dma_addr;</div><div class='add'>+	__le32		id_ver;</div><div class='add'>+	__le16		dbid_flags;</div><div class='add'>+	__u8		type_state;</div><div class='add'>+	__u8		rsvd;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+#define IONIC_ADMIN_CREATE_QP_IN_V1_LEN 64</div><div class='add'>+static_assert(sizeof(struct ionic_admin_create_qp) ==</div><div class='add'>+	       IONIC_ADMIN_CREATE_QP_IN_V1_LEN);</div><div class='add'>+</div><div class='add'>+struct ionic_admin_destroy_qp {</div><div class='add'>+	__le32		qp_id;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+#define IONIC_ADMIN_DESTROY_QP_IN_V1_LEN 4</div><div class='add'>+static_assert(sizeof(struct ionic_admin_destroy_qp) ==</div><div class='add'>+	       IONIC_ADMIN_DESTROY_QP_IN_V1_LEN);</div><div class='add'>+</div><div class='add'>+struct ionic_admin_mod_qp {</div><div class='add'>+	__be32		attr_mask;</div><div class='add'>+	__u8		dcqcn_profile;</div><div class='add'>+	__u8		tfp_csum_profile;</div><div class='add'>+	__be16		access_flags;</div><div class='add'>+	__le32		rq_psn;</div><div class='add'>+	__le32		sq_psn;</div><div class='add'>+	__le32		qkey_dest_qpn;</div><div class='add'>+	__le32		rate_limit_kbps;</div><div class='add'>+	__u8		pmtu;</div><div class='add'>+	__u8		retry;</div><div class='add'>+	__u8		rnr_timer;</div><div class='add'>+	__u8		retry_timeout;</div><div class='add'>+	__u8		rsq_depth;</div><div class='add'>+	__u8		rrq_depth;</div><div class='add'>+	__le16		pkey_id;</div><div class='add'>+	__le32		ah_id_len;</div><div class='add'>+	__u8		en_pcp;</div><div class='add'>+	__u8		ip_dscp;</div><div class='add'>+	__u8		rsvd2;</div><div class='add'>+	__u8		type_state;</div><div class='add'>+	union {</div><div class='add'>+		struct {</div><div class='add'>+			__le16		rsvd1;</div><div class='add'>+		};</div><div class='add'>+		__le32		rrq_index;</div><div class='add'>+	};</div><div class='add'>+	__le32		rsq_index;</div><div class='add'>+	__le64		dma_addr;</div><div class='add'>+	__le32		id_ver;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+#define IONIC_ADMIN_MODIFY_QP_IN_V1_LEN 60</div><div class='add'>+static_assert(sizeof(struct ionic_admin_mod_qp) ==</div><div class='add'>+	       IONIC_ADMIN_MODIFY_QP_IN_V1_LEN);</div><div class='add'>+</div><div class='add'>+struct ionic_admin_query_qp {</div><div class='add'>+	__le64		hdr_dma_addr;</div><div class='add'>+	__le64		sq_dma_addr;</div><div class='add'>+	__le64		rq_dma_addr;</div><div class='add'>+	__le32		ah_id;</div><div class='add'>+	__le32		id_ver;</div><div class='add'>+	__le16		dbid_flags;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+#define IONIC_ADMIN_QUERY_QP_IN_V1_LEN 34</div><div class='add'>+static_assert(sizeof(struct ionic_admin_query_qp) ==</div><div class='add'>+	       IONIC_ADMIN_QUERY_QP_IN_V1_LEN);</div><div class='add'>+</div><div class='add'>+#define ADMIN_WQE_STRIDE	64</div><div class='add'>+#define ADMIN_WQE_HDR_LEN	4</div><div class='add'>+</div><div class='add'>+/* admin queue v1 wqe */</div><div class='add'>+struct ionic_v1_admin_wqe {</div><div class='add'>+	__u8				op;</div><div class='add'>+	__u8				rsvd;</div><div class='add'>+	__le16				len;</div><div class='add'>+</div><div class='add'>+	union {</div><div class='add'>+		struct ionic_admin_stats_hdr stats;</div><div class='add'>+		struct ionic_admin_create_ah create_ah;</div><div class='add'>+		struct ionic_admin_destroy_ah destroy_ah;</div><div class='add'>+		struct ionic_admin_query_ah query_ah;</div><div class='add'>+		struct ionic_admin_create_mr create_mr;</div><div class='add'>+		struct ionic_admin_destroy_mr destroy_mr;</div><div class='add'>+		struct ionic_admin_create_cq create_cq;</div><div class='add'>+		struct ionic_admin_destroy_cq destroy_cq;</div><div class='add'>+		struct ionic_admin_create_qp create_qp;</div><div class='add'>+		struct ionic_admin_destroy_qp destroy_qp;</div><div class='add'>+		struct ionic_admin_mod_qp mod_qp;</div><div class='add'>+		struct ionic_admin_query_qp query_qp;</div><div class='add'>+	} cmd;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* side data for query qp */</div><div class='add'>+struct ionic_v1_admin_query_qp_sq {</div><div class='add'>+	__u8				rnr_timer;</div><div class='add'>+	__u8				retry_timeout;</div><div class='add'>+	__be16				access_perms_flags;</div><div class='add'>+	__be16				rsvd;</div><div class='add'>+	__be16				pkey_id;</div><div class='add'>+	__be32				qkey_dest_qpn;</div><div class='add'>+	__be32				rate_limit_kbps;</div><div class='add'>+	__be32				rq_psn;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_v1_admin_query_qp_rq {</div><div class='add'>+	__u8				state_pmtu;</div><div class='add'>+	__u8				retry_rnrtry;</div><div class='add'>+	__u8				rrq_depth;</div><div class='add'>+	__u8				rsq_depth;</div><div class='add'>+	__be32				sq_psn;</div><div class='add'>+	__be16				access_perms_flags;</div><div class='add'>+	__be16				rsvd;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* admin queue v1 opcodes */</div><div class='add'>+enum ionic_v1_admin_op {</div><div class='add'>+	IONIC_V1_ADMIN_NOOP,</div><div class='add'>+	IONIC_V1_ADMIN_CREATE_CQ,</div><div class='add'>+	IONIC_V1_ADMIN_CREATE_QP,</div><div class='add'>+	IONIC_V1_ADMIN_CREATE_MR,</div><div class='add'>+	IONIC_V1_ADMIN_STATS_HDRS,</div><div class='add'>+	IONIC_V1_ADMIN_STATS_VALS,</div><div class='add'>+	IONIC_V1_ADMIN_DESTROY_MR,</div><div class='add'>+	IONIC_V1_ADMIN_RSVD_7,		/* RESIZE_CQ */</div><div class='add'>+	IONIC_V1_ADMIN_DESTROY_CQ,</div><div class='add'>+	IONIC_V1_ADMIN_MODIFY_QP,</div><div class='add'>+	IONIC_V1_ADMIN_QUERY_QP,</div><div class='add'>+	IONIC_V1_ADMIN_DESTROY_QP,</div><div class='add'>+	IONIC_V1_ADMIN_DEBUG,</div><div class='add'>+	IONIC_V1_ADMIN_CREATE_AH,</div><div class='add'>+	IONIC_V1_ADMIN_QUERY_AH,</div><div class='add'>+	IONIC_V1_ADMIN_MODIFY_DCQCN,</div><div class='add'>+	IONIC_V1_ADMIN_DESTROY_AH,</div><div class='add'>+	IONIC_V1_ADMIN_QP_STATS_HDRS,</div><div class='add'>+	IONIC_V1_ADMIN_QP_STATS_VALS,</div><div class='add'>+	IONIC_V1_ADMIN_OPCODES_MAX,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* admin queue v1 cqe status */</div><div class='add'>+enum ionic_v1_admin_status {</div><div class='add'>+	IONIC_V1_ASTS_OK,</div><div class='add'>+	IONIC_V1_ASTS_BAD_CMD,</div><div class='add'>+	IONIC_V1_ASTS_BAD_INDEX,</div><div class='add'>+	IONIC_V1_ASTS_BAD_STATE,</div><div class='add'>+	IONIC_V1_ASTS_BAD_TYPE,</div><div class='add'>+	IONIC_V1_ASTS_BAD_ATTR,</div><div class='add'>+	IONIC_V1_ASTS_MSG_TOO_BIG,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* event queue v1 eqe */</div><div class='add'>+struct ionic_v1_eqe {</div><div class='add'>+	__be32				evt;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* bits for cqe queue_type_flags */</div><div class='add'>+enum ionic_v1_eqe_evt_bits {</div><div class='add'>+	IONIC_V1_EQE_COLOR		= BIT(0),</div><div class='add'>+	IONIC_V1_EQE_TYPE_SHIFT		= 1,</div><div class='add'>+	IONIC_V1_EQE_TYPE_MASK		= 0x7,</div><div class='add'>+	IONIC_V1_EQE_CODE_SHIFT		= 4,</div><div class='add'>+	IONIC_V1_EQE_CODE_MASK		= 0xf,</div><div class='add'>+	IONIC_V1_EQE_QID_SHIFT		= 8,</div><div class='add'>+</div><div class='add'>+	/* cq events */</div><div class='add'>+	IONIC_V1_EQE_TYPE_CQ		= 0,</div><div class='add'>+	/* cq normal events */</div><div class='add'>+	IONIC_V1_EQE_CQ_NOTIFY		= 0,</div><div class='add'>+	/* cq error events */</div><div class='add'>+	IONIC_V1_EQE_CQ_ERR		= 8,</div><div class='add'>+</div><div class='add'>+	/* qp and srq events */</div><div class='add'>+	IONIC_V1_EQE_TYPE_QP		= 1,</div><div class='add'>+	/* qp normal events */</div><div class='add'>+	IONIC_V1_EQE_SRQ_LEVEL		= 0,</div><div class='add'>+	IONIC_V1_EQE_SQ_DRAIN		= 1,</div><div class='add'>+	IONIC_V1_EQE_QP_COMM_EST	= 2,</div><div class='add'>+	IONIC_V1_EQE_QP_LAST_WQE	= 3,</div><div class='add'>+	/* qp error events */</div><div class='add'>+	IONIC_V1_EQE_QP_ERR		= 8,</div><div class='add'>+	IONIC_V1_EQE_QP_ERR_REQUEST	= 9,</div><div class='add'>+	IONIC_V1_EQE_QP_ERR_ACCESS	= 10,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum ionic_tfp_csum_profiles {</div><div class='add'>+	IONIC_TFP_CSUM_PROF_ETH_IPV4_UDP				= 0,</div><div class='add'>+	IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP				= 1,</div><div class='add'>+	IONIC_TFP_CSUM_PROF_ETH_IPV6_UDP				= 2,</div><div class='add'>+	IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV6_UDP				= 3,</div><div class='add'>+	IONIC_TFP_CSUM_PROF_IPV4_UDP_VXLAN_ETH_QTAG_IPV4_UDP		= 4,</div><div class='add'>+	IONIC_TFP_CSUM_PROF_IPV4_UDP_VXLAN_ETH_QTAG_IPV6_UDP		= 5,</div><div class='add'>+	IONIC_TFP_CSUM_PROF_QTAG_IPV4_UDP_VXLAN_ETH_QTAG_IPV4_UDP	= 6,</div><div class='add'>+	IONIC_TFP_CSUM_PROF_QTAG_IPV4_UDP_VXLAN_ETH_QTAG_IPV6_UDP	= 7,</div><div class='add'>+	IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP_ESP_IPV4_UDP		= 8,</div><div class='add'>+	IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_ESP_UDP			= 9,</div><div class='add'>+	IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP_ESP_UDP			= 10,</div><div class='add'>+	IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV6_ESP_UDP			= 11,</div><div class='add'>+	IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP_CSUM			= 12,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline bool ionic_v1_eqe_color(struct ionic_v1_eqe *eqe)</div><div class='add'>+{</div><div class='add'>+	return eqe-&gt;evt &amp; cpu_to_be32(IONIC_V1_EQE_COLOR);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 ionic_v1_eqe_evt(struct ionic_v1_eqe *eqe)</div><div class='add'>+{</div><div class='add'>+	return be32_to_cpu(eqe-&gt;evt);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u8 ionic_v1_eqe_evt_type(u32 evt)</div><div class='add'>+{</div><div class='add'>+	return (evt &gt;&gt; IONIC_V1_EQE_TYPE_SHIFT) &amp; IONIC_V1_EQE_TYPE_MASK;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u8 ionic_v1_eqe_evt_code(u32 evt)</div><div class='add'>+{</div><div class='add'>+	return (evt &gt;&gt; IONIC_V1_EQE_CODE_SHIFT) &amp; IONIC_V1_EQE_CODE_MASK;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 ionic_v1_eqe_evt_qid(u32 evt)</div><div class='add'>+{</div><div class='add'>+	return evt &gt;&gt; IONIC_V1_EQE_QID_SHIFT;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+enum ionic_v1_stat_bits {</div><div class='add'>+	IONIC_V1_STAT_TYPE_SHIFT	= 28,</div><div class='add'>+	IONIC_V1_STAT_TYPE_NONE		= 0,</div><div class='add'>+	IONIC_V1_STAT_TYPE_8		= 1,</div><div class='add'>+	IONIC_V1_STAT_TYPE_LE16		= 2,</div><div class='add'>+	IONIC_V1_STAT_TYPE_LE32		= 3,</div><div class='add'>+	IONIC_V1_STAT_TYPE_LE64		= 4,</div><div class='add'>+	IONIC_V1_STAT_TYPE_BE16		= 5,</div><div class='add'>+	IONIC_V1_STAT_TYPE_BE32		= 6,</div><div class='add'>+	IONIC_V1_STAT_TYPE_BE64		= 7,</div><div class='add'>+	IONIC_V1_STAT_OFF_MASK		= BIT(IONIC_V1_STAT_TYPE_SHIFT) - 1,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_v1_stat {</div><div class='add'>+	union {</div><div class='add'>+		__be32		be_type_off;</div><div class='add'>+		u32		type_off;</div><div class='add'>+	};</div><div class='add'>+	char			name[28];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline int ionic_v1_stat_type(struct ionic_v1_stat *hdr)</div><div class='add'>+{</div><div class='add'>+	return hdr-&gt;type_off &gt;&gt; IONIC_V1_STAT_TYPE_SHIFT;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline unsigned int ionic_v1_stat_off(struct ionic_v1_stat *hdr)</div><div class='add'>+{</div><div class='add'>+	return hdr-&gt;type_off &amp; IONIC_V1_STAT_OFF_MASK;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#endif /* _IONIC_FW_H_ */</div><div class='head'>diff --git a/drivers/infiniband/hw/ionic/ionic_hw_stats.c b/drivers/infiniband/hw/ionic/ionic_hw_stats.c<br/>new file mode 100644<br/>index 000000000000..244a80dde08f<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/ionic/ionic_hw_stats.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/ionic/ionic_hw_stats.c</a></div><div class='hunk'>@@ -0,0 +1,484 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0</div><div class='add'>+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/dma-mapping.h&gt;</div><div class='add'>+</div><div class='add'>+#include "ionic_fw.h"</div><div class='add'>+#include "ionic_ibdev.h"</div><div class='add'>+</div><div class='add'>+static int ionic_v1_stat_normalize(struct ionic_v1_stat *hw_stats,</div><div class='add'>+				   int hw_stats_count)</div><div class='add'>+{</div><div class='add'>+	int hw_stat_i;</div><div class='add'>+</div><div class='add'>+	for (hw_stat_i = 0; hw_stat_i &lt; hw_stats_count; ++hw_stat_i) {</div><div class='add'>+		struct ionic_v1_stat *stat = &amp;hw_stats[hw_stat_i];</div><div class='add'>+</div><div class='add'>+		stat-&gt;type_off = be32_to_cpu(stat-&gt;be_type_off);</div><div class='add'>+		stat-&gt;name[sizeof(stat-&gt;name) - 1] = 0;</div><div class='add'>+		if (ionic_v1_stat_type(stat) == IONIC_V1_STAT_TYPE_NONE)</div><div class='add'>+			break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return hw_stat_i;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_fill_stats_desc(struct rdma_stat_desc *hw_stats_hdrs,</div><div class='add'>+				  struct ionic_v1_stat *hw_stats,</div><div class='add'>+				  int hw_stats_count)</div><div class='add'>+{</div><div class='add'>+	int hw_stat_i;</div><div class='add'>+</div><div class='add'>+	for (hw_stat_i = 0; hw_stat_i &lt; hw_stats_count; ++hw_stat_i) {</div><div class='add'>+		struct ionic_v1_stat *stat = &amp;hw_stats[hw_stat_i];</div><div class='add'>+</div><div class='add'>+		hw_stats_hdrs[hw_stat_i].name = stat-&gt;name;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u64 ionic_v1_stat_val(struct ionic_v1_stat *stat,</div><div class='add'>+			     void *vals_buf, size_t vals_len)</div><div class='add'>+{</div><div class='add'>+	unsigned int off = ionic_v1_stat_off(stat);</div><div class='add'>+	int type = ionic_v1_stat_type(stat);</div><div class='add'>+</div><div class='add'>+#define __ionic_v1_stat_validate(__type)		\</div><div class='add'>+	((off + sizeof(__type) &lt;= vals_len) &amp;&amp;		\</div><div class='add'>+	 (IS_ALIGNED(off, sizeof(__type))))</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case IONIC_V1_STAT_TYPE_8:</div><div class='add'>+		if (__ionic_v1_stat_validate(u8))</div><div class='add'>+			return *(u8 *)(vals_buf + off);</div><div class='add'>+		break;</div><div class='add'>+	case IONIC_V1_STAT_TYPE_LE16:</div><div class='add'>+		if (__ionic_v1_stat_validate(__le16))</div><div class='add'>+			return le16_to_cpu(*(__le16 *)(vals_buf + off));</div><div class='add'>+		break;</div><div class='add'>+	case IONIC_V1_STAT_TYPE_LE32:</div><div class='add'>+		if (__ionic_v1_stat_validate(__le32))</div><div class='add'>+			return le32_to_cpu(*(__le32 *)(vals_buf + off));</div><div class='add'>+		break;</div><div class='add'>+	case IONIC_V1_STAT_TYPE_LE64:</div><div class='add'>+		if (__ionic_v1_stat_validate(__le64))</div><div class='add'>+			return le64_to_cpu(*(__le64 *)(vals_buf + off));</div><div class='add'>+		break;</div><div class='add'>+	case IONIC_V1_STAT_TYPE_BE16:</div><div class='add'>+		if (__ionic_v1_stat_validate(__be16))</div><div class='add'>+			return be16_to_cpu(*(__be16 *)(vals_buf + off));</div><div class='add'>+		break;</div><div class='add'>+	case IONIC_V1_STAT_TYPE_BE32:</div><div class='add'>+		if (__ionic_v1_stat_validate(__be32))</div><div class='add'>+			return be32_to_cpu(*(__be32 *)(vals_buf + off));</div><div class='add'>+		break;</div><div class='add'>+	case IONIC_V1_STAT_TYPE_BE64:</div><div class='add'>+		if (__ionic_v1_stat_validate(__be64))</div><div class='add'>+			return be64_to_cpu(*(__be64 *)(vals_buf + off));</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ~0ull;</div><div class='add'>+#undef __ionic_v1_stat_validate</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_hw_stats_cmd(struct ionic_ibdev *dev,</div><div class='add'>+			      dma_addr_t dma, size_t len, int qid, int op)</div><div class='add'>+{</div><div class='add'>+	struct ionic_admin_wr wr = {</div><div class='add'>+		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),</div><div class='add'>+		.wqe = {</div><div class='add'>+			.op = op,</div><div class='add'>+			.len = cpu_to_le16(IONIC_ADMIN_STATS_HDRS_IN_V1_LEN),</div><div class='add'>+			.cmd.stats = {</div><div class='add'>+				.dma_addr = cpu_to_le64(dma),</div><div class='add'>+				.length = cpu_to_le32(len),</div><div class='add'>+				.id_ver = cpu_to_le32(qid),</div><div class='add'>+			},</div><div class='add'>+		}</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;lif_cfg.admin_opcodes &lt;= op)</div><div class='add'>+		return -EBADRQC;</div><div class='add'>+</div><div class='add'>+	ionic_admin_post(dev, &amp;wr);</div><div class='add'>+</div><div class='add'>+	return ionic_admin_wait(dev, &amp;wr, IONIC_ADMIN_F_INTERRUPT);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_init_hw_stats(struct ionic_ibdev *dev)</div><div class='add'>+{</div><div class='add'>+	dma_addr_t hw_stats_dma;</div><div class='add'>+	int rc, hw_stats_count;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_stats_hdrs)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	dev-&gt;hw_stats_count = 0;</div><div class='add'>+</div><div class='add'>+	/* buffer for current values from the device */</div><div class='add'>+	dev-&gt;hw_stats_buf = kzalloc(PAGE_SIZE, GFP_KERNEL);</div><div class='add'>+	if (!dev-&gt;hw_stats_buf) {</div><div class='add'>+		rc = -ENOMEM;</div><div class='add'>+		goto err_buf;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* buffer for names, sizes, offsets of values */</div><div class='add'>+	dev-&gt;hw_stats = kzalloc(PAGE_SIZE, GFP_KERNEL);</div><div class='add'>+	if (!dev-&gt;hw_stats) {</div><div class='add'>+		rc = -ENOMEM;</div><div class='add'>+		goto err_hw_stats;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* request the names, sizes, offsets */</div><div class='add'>+	hw_stats_dma = dma_map_single(dev-&gt;lif_cfg.hwdev, dev-&gt;hw_stats,</div><div class='add'>+				      PAGE_SIZE, DMA_FROM_DEVICE);</div><div class='add'>+	rc = dma_mapping_error(dev-&gt;lif_cfg.hwdev, hw_stats_dma);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_dma;</div><div class='add'>+</div><div class='add'>+	rc = ionic_hw_stats_cmd(dev, hw_stats_dma, PAGE_SIZE, 0,</div><div class='add'>+				IONIC_V1_ADMIN_STATS_HDRS);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_cmd;</div><div class='add'>+</div><div class='add'>+	dma_unmap_single(dev-&gt;lif_cfg.hwdev, hw_stats_dma, PAGE_SIZE, DMA_FROM_DEVICE);</div><div class='add'>+</div><div class='add'>+	/* normalize and count the number of hw_stats */</div><div class='add'>+	hw_stats_count =</div><div class='add'>+		ionic_v1_stat_normalize(dev-&gt;hw_stats,</div><div class='add'>+					PAGE_SIZE / sizeof(*dev-&gt;hw_stats));</div><div class='add'>+	if (!hw_stats_count) {</div><div class='add'>+		rc = -ENODATA;</div><div class='add'>+		goto err_dma;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dev-&gt;hw_stats_count = hw_stats_count;</div><div class='add'>+</div><div class='add'>+	/* alloc and init array of names, for alloc_hw_stats */</div><div class='add'>+	dev-&gt;hw_stats_hdrs = kcalloc(hw_stats_count,</div><div class='add'>+				     sizeof(*dev-&gt;hw_stats_hdrs),</div><div class='add'>+				     GFP_KERNEL);</div><div class='add'>+	if (!dev-&gt;hw_stats_hdrs) {</div><div class='add'>+		rc = -ENOMEM;</div><div class='add'>+		goto err_dma;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ionic_fill_stats_desc(dev-&gt;hw_stats_hdrs, dev-&gt;hw_stats,</div><div class='add'>+			      hw_stats_count);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_cmd:</div><div class='add'>+	dma_unmap_single(dev-&gt;lif_cfg.hwdev, hw_stats_dma, PAGE_SIZE, DMA_FROM_DEVICE);</div><div class='add'>+err_dma:</div><div class='add'>+	kfree(dev-&gt;hw_stats);</div><div class='add'>+err_hw_stats:</div><div class='add'>+	kfree(dev-&gt;hw_stats_buf);</div><div class='add'>+err_buf:</div><div class='add'>+	dev-&gt;hw_stats_count = 0;</div><div class='add'>+	dev-&gt;hw_stats = NULL;</div><div class='add'>+	dev-&gt;hw_stats_buf = NULL;</div><div class='add'>+	dev-&gt;hw_stats_hdrs = NULL;</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct rdma_hw_stats *ionic_alloc_hw_stats(struct ib_device *ibdev,</div><div class='add'>+						  u32 port)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibdev);</div><div class='add'>+</div><div class='add'>+	if (port != 1)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	return rdma_alloc_hw_stats_struct(dev-&gt;hw_stats_hdrs,</div><div class='add'>+					  dev-&gt;hw_stats_count,</div><div class='add'>+					  RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_get_hw_stats(struct ib_device *ibdev,</div><div class='add'>+			      struct rdma_hw_stats *hw_stats,</div><div class='add'>+			      u32 port, int index)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibdev);</div><div class='add'>+	dma_addr_t hw_stats_dma;</div><div class='add'>+	int rc, hw_stat_i;</div><div class='add'>+</div><div class='add'>+	if (port != 1)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	hw_stats_dma = dma_map_single(dev-&gt;lif_cfg.hwdev, dev-&gt;hw_stats_buf,</div><div class='add'>+				      PAGE_SIZE, DMA_FROM_DEVICE);</div><div class='add'>+	rc = dma_mapping_error(dev-&gt;lif_cfg.hwdev, hw_stats_dma);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_dma;</div><div class='add'>+</div><div class='add'>+	rc = ionic_hw_stats_cmd(dev, hw_stats_dma, PAGE_SIZE,</div><div class='add'>+				0, IONIC_V1_ADMIN_STATS_VALS);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_cmd;</div><div class='add'>+</div><div class='add'>+	dma_unmap_single(dev-&gt;lif_cfg.hwdev, hw_stats_dma,</div><div class='add'>+			 PAGE_SIZE, DMA_FROM_DEVICE);</div><div class='add'>+</div><div class='add'>+	for (hw_stat_i = 0; hw_stat_i &lt; dev-&gt;hw_stats_count; ++hw_stat_i)</div><div class='add'>+		hw_stats-&gt;value[hw_stat_i] =</div><div class='add'>+			ionic_v1_stat_val(&amp;dev-&gt;hw_stats[hw_stat_i],</div><div class='add'>+					  dev-&gt;hw_stats_buf, PAGE_SIZE);</div><div class='add'>+</div><div class='add'>+	return hw_stat_i;</div><div class='add'>+</div><div class='add'>+err_cmd:</div><div class='add'>+	dma_unmap_single(dev-&gt;lif_cfg.hwdev, hw_stats_dma,</div><div class='add'>+			 PAGE_SIZE, DMA_FROM_DEVICE);</div><div class='add'>+err_dma:</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct rdma_hw_stats *</div><div class='add'>+ionic_counter_alloc_stats(struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(counter-&gt;device);</div><div class='add'>+	struct ionic_counter *cntr;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	cntr = kzalloc(sizeof(*cntr), GFP_KERNEL);</div><div class='add'>+	if (!cntr)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	/* buffer for current values from the device */</div><div class='add'>+	cntr-&gt;vals = kzalloc(PAGE_SIZE, GFP_KERNEL);</div><div class='add'>+	if (!cntr-&gt;vals)</div><div class='add'>+		goto err_vals;</div><div class='add'>+</div><div class='add'>+	err = xa_alloc(&amp;dev-&gt;counter_stats-&gt;xa_counters, &amp;counter-&gt;id,</div><div class='add'>+		       cntr,</div><div class='add'>+		       XA_LIMIT(0, IONIC_MAX_QPID),</div><div class='add'>+		       GFP_KERNEL);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_xa;</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;cntr-&gt;qp_list);</div><div class='add'>+</div><div class='add'>+	return rdma_alloc_hw_stats_struct(dev-&gt;counter_stats-&gt;stats_hdrs,</div><div class='add'>+					 dev-&gt;counter_stats-&gt;queue_stats_count,</div><div class='add'>+					 RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='add'>+err_xa:</div><div class='add'>+	kfree(cntr-&gt;vals);</div><div class='add'>+err_vals:</div><div class='add'>+	kfree(cntr);</div><div class='add'>+</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_counter_dealloc(struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(counter-&gt;device);</div><div class='add'>+	struct ionic_counter *cntr;</div><div class='add'>+</div><div class='add'>+	cntr = xa_erase(&amp;dev-&gt;counter_stats-&gt;xa_counters, counter-&gt;id);</div><div class='add'>+	if (!cntr)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	kfree(cntr-&gt;vals);</div><div class='add'>+	kfree(cntr);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_counter_bind_qp(struct rdma_counter *counter,</div><div class='add'>+				 struct ib_qp *ibqp,</div><div class='add'>+				 u32 port)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(counter-&gt;device);</div><div class='add'>+	struct ionic_qp *qp = to_ionic_qp(ibqp);</div><div class='add'>+	struct ionic_counter *cntr;</div><div class='add'>+</div><div class='add'>+	cntr = xa_load(&amp;dev-&gt;counter_stats-&gt;xa_counters, counter-&gt;id);</div><div class='add'>+	if (!cntr)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	list_add_tail(&amp;qp-&gt;qp_list_counter, &amp;cntr-&gt;qp_list);</div><div class='add'>+	ibqp-&gt;counter = counter;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_counter_unbind_qp(struct ib_qp *ibqp, u32 port)</div><div class='add'>+{</div><div class='add'>+	struct ionic_qp *qp = to_ionic_qp(ibqp);</div><div class='add'>+</div><div class='add'>+	if (ibqp-&gt;counter) {</div><div class='add'>+		list_del(&amp;qp-&gt;qp_list_counter);</div><div class='add'>+		ibqp-&gt;counter = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_get_qp_stats(struct ib_device *ibdev,</div><div class='add'>+			      struct rdma_hw_stats *hw_stats,</div><div class='add'>+			      u32 counter_id)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibdev);</div><div class='add'>+	struct ionic_counter_stats *cs;</div><div class='add'>+	struct ionic_counter *cntr;</div><div class='add'>+	dma_addr_t hw_stats_dma;</div><div class='add'>+	struct ionic_qp *qp;</div><div class='add'>+	int rc, stat_i = 0;</div><div class='add'>+</div><div class='add'>+	cs = dev-&gt;counter_stats;</div><div class='add'>+	cntr = xa_load(&amp;cs-&gt;xa_counters, counter_id);</div><div class='add'>+	if (!cntr)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	hw_stats_dma = dma_map_single(dev-&gt;lif_cfg.hwdev, cntr-&gt;vals,</div><div class='add'>+				      PAGE_SIZE, DMA_FROM_DEVICE);</div><div class='add'>+	rc = dma_mapping_error(dev-&gt;lif_cfg.hwdev, hw_stats_dma);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	memset(hw_stats-&gt;value, 0, sizeof(u64) * hw_stats-&gt;num_counters);</div><div class='add'>+</div><div class='add'>+	list_for_each_entry(qp, &amp;cntr-&gt;qp_list, qp_list_counter) {</div><div class='add'>+		rc = ionic_hw_stats_cmd(dev, hw_stats_dma, PAGE_SIZE,</div><div class='add'>+					qp-&gt;qpid,</div><div class='add'>+					IONIC_V1_ADMIN_QP_STATS_VALS);</div><div class='add'>+		if (rc)</div><div class='add'>+			goto err_cmd;</div><div class='add'>+</div><div class='add'>+		for (stat_i = 0; stat_i &lt; cs-&gt;queue_stats_count; ++stat_i)</div><div class='add'>+			hw_stats-&gt;value[stat_i] +=</div><div class='add'>+				ionic_v1_stat_val(&amp;cs-&gt;hdr[stat_i],</div><div class='add'>+						  cntr-&gt;vals,</div><div class='add'>+						  PAGE_SIZE);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dma_unmap_single(dev-&gt;lif_cfg.hwdev, hw_stats_dma, PAGE_SIZE, DMA_FROM_DEVICE);</div><div class='add'>+	return stat_i;</div><div class='add'>+</div><div class='add'>+err_cmd:</div><div class='add'>+	dma_unmap_single(dev-&gt;lif_cfg.hwdev, hw_stats_dma, PAGE_SIZE, DMA_FROM_DEVICE);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_counter_update_stats(struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	return ionic_get_qp_stats(counter-&gt;device, counter-&gt;stats, counter-&gt;id);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_alloc_counters(struct ionic_ibdev *dev)</div><div class='add'>+{</div><div class='add'>+	struct ionic_counter_stats *cs = dev-&gt;counter_stats;</div><div class='add'>+	int rc, hw_stats_count;</div><div class='add'>+	dma_addr_t hdr_dma;</div><div class='add'>+</div><div class='add'>+	/* buffer for names, sizes, offsets of values */</div><div class='add'>+	cs-&gt;hdr = kzalloc(PAGE_SIZE, GFP_KERNEL);</div><div class='add'>+	if (!cs-&gt;hdr)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	hdr_dma = dma_map_single(dev-&gt;lif_cfg.hwdev, cs-&gt;hdr,</div><div class='add'>+				 PAGE_SIZE, DMA_FROM_DEVICE);</div><div class='add'>+	rc = dma_mapping_error(dev-&gt;lif_cfg.hwdev, hdr_dma);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_dma;</div><div class='add'>+</div><div class='add'>+	rc = ionic_hw_stats_cmd(dev, hdr_dma, PAGE_SIZE, 0,</div><div class='add'>+				IONIC_V1_ADMIN_QP_STATS_HDRS);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_cmd;</div><div class='add'>+</div><div class='add'>+	dma_unmap_single(dev-&gt;lif_cfg.hwdev, hdr_dma, PAGE_SIZE, DMA_FROM_DEVICE);</div><div class='add'>+</div><div class='add'>+	/* normalize and count the number of hw_stats */</div><div class='add'>+	hw_stats_count = ionic_v1_stat_normalize(cs-&gt;hdr,</div><div class='add'>+						 PAGE_SIZE / sizeof(*cs-&gt;hdr));</div><div class='add'>+	if (!hw_stats_count) {</div><div class='add'>+		rc = -ENODATA;</div><div class='add'>+		goto err_dma;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cs-&gt;queue_stats_count = hw_stats_count;</div><div class='add'>+</div><div class='add'>+	/* alloc and init array of names */</div><div class='add'>+	cs-&gt;stats_hdrs = kcalloc(hw_stats_count, sizeof(*cs-&gt;stats_hdrs),</div><div class='add'>+				 GFP_KERNEL);</div><div class='add'>+	if (!cs-&gt;stats_hdrs) {</div><div class='add'>+		rc = -ENOMEM;</div><div class='add'>+		goto err_dma;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ionic_fill_stats_desc(cs-&gt;stats_hdrs, cs-&gt;hdr, hw_stats_count);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_cmd:</div><div class='add'>+	dma_unmap_single(dev-&gt;lif_cfg.hwdev, hdr_dma, PAGE_SIZE, DMA_FROM_DEVICE);</div><div class='add'>+err_dma:</div><div class='add'>+	kfree(cs-&gt;hdr);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops ionic_hw_stats_ops = {</div><div class='add'>+	.driver_id = RDMA_DRIVER_IONIC,</div><div class='add'>+	.alloc_hw_port_stats = ionic_alloc_hw_stats,</div><div class='add'>+	.get_hw_stats = ionic_get_hw_stats,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops ionic_counter_stats_ops = {</div><div class='add'>+	.counter_alloc_stats = ionic_counter_alloc_stats,</div><div class='add'>+	.counter_dealloc = ionic_counter_dealloc,</div><div class='add'>+	.counter_bind_qp = ionic_counter_bind_qp,</div><div class='add'>+	.counter_unbind_qp = ionic_counter_unbind_qp,</div><div class='add'>+	.counter_update_stats = ionic_counter_update_stats,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+void ionic_stats_init(struct ionic_ibdev *dev)</div><div class='add'>+{</div><div class='add'>+	u16 stats_type = dev-&gt;lif_cfg.stats_type;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	if (stats_type &amp; IONIC_LIF_RDMA_STAT_GLOBAL) {</div><div class='add'>+		rc = ionic_init_hw_stats(dev);</div><div class='add'>+		if (rc)</div><div class='add'>+			ibdev_dbg(&amp;dev-&gt;ibdev, "Failed to init hw stats\n");</div><div class='add'>+		else</div><div class='add'>+			ib_set_device_ops(&amp;dev-&gt;ibdev, &amp;ionic_hw_stats_ops);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (stats_type &amp; IONIC_LIF_RDMA_STAT_QP) {</div><div class='add'>+		dev-&gt;counter_stats = kzalloc(sizeof(*dev-&gt;counter_stats),</div><div class='add'>+					     GFP_KERNEL);</div><div class='add'>+		if (!dev-&gt;counter_stats)</div><div class='add'>+			return;</div><div class='add'>+</div><div class='add'>+		rc = ionic_alloc_counters(dev);</div><div class='add'>+		if (rc) {</div><div class='add'>+			ibdev_dbg(&amp;dev-&gt;ibdev, "Failed to init counter stats\n");</div><div class='add'>+			kfree(dev-&gt;counter_stats);</div><div class='add'>+			dev-&gt;counter_stats = NULL;</div><div class='add'>+			return;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		xa_init_flags(&amp;dev-&gt;counter_stats-&gt;xa_counters, XA_FLAGS_ALLOC);</div><div class='add'>+</div><div class='add'>+		ib_set_device_ops(&amp;dev-&gt;ibdev, &amp;ionic_counter_stats_ops);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void ionic_stats_cleanup(struct ionic_ibdev *dev)</div><div class='add'>+{</div><div class='add'>+	if (dev-&gt;counter_stats) {</div><div class='add'>+		xa_destroy(&amp;dev-&gt;counter_stats-&gt;xa_counters);</div><div class='add'>+		kfree(dev-&gt;counter_stats-&gt;hdr);</div><div class='add'>+		kfree(dev-&gt;counter_stats-&gt;stats_hdrs);</div><div class='add'>+		kfree(dev-&gt;counter_stats);</div><div class='add'>+		dev-&gt;counter_stats = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	kfree(dev-&gt;hw_stats);</div><div class='add'>+	kfree(dev-&gt;hw_stats_buf);</div><div class='add'>+	kfree(dev-&gt;hw_stats_hdrs);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c<br/>new file mode 100644<br/>index 000000000000..164046d00e5d<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/ionic/ionic_ibdev.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/ionic/ionic_ibdev.c</a></div><div class='hunk'>@@ -0,0 +1,440 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0</div><div class='add'>+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/module.h&gt;</div><div class='add'>+#include &lt;linux/printk.h&gt;</div><div class='add'>+#include &lt;linux/pci.h&gt;</div><div class='add'>+#include &lt;linux/irq.h&gt;</div><div class='add'>+#include &lt;net/addrconf.h&gt;</div><div class='add'>+#include &lt;rdma/ib_addr.h&gt;</div><div class='add'>+#include &lt;rdma/ib_mad.h&gt;</div><div class='add'>+</div><div class='add'>+#include "ionic_ibdev.h"</div><div class='add'>+</div><div class='add'>+#define DRIVER_DESCRIPTION "AMD Pensando RoCE HCA driver"</div><div class='add'>+#define DEVICE_DESCRIPTION "AMD Pensando RoCE HCA"</div><div class='add'>+</div><div class='add'>+MODULE_AUTHOR("Allen Hubbe &lt;allen.hubbe@amd.com&gt;");</div><div class='add'>+MODULE_DESCRIPTION(DRIVER_DESCRIPTION);</div><div class='add'>+MODULE_LICENSE("GPL");</div><div class='add'>+MODULE_IMPORT_NS("NET_IONIC");</div><div class='add'>+</div><div class='add'>+static int ionic_query_device(struct ib_device *ibdev,</div><div class='add'>+			      struct ib_device_attr *attr,</div><div class='add'>+			      struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibdev);</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+</div><div class='add'>+	ndev = ib_device_get_netdev(ibdev, 1);</div><div class='add'>+	addrconf_ifid_eui48((u8 *)&amp;attr-&gt;sys_image_guid, ndev);</div><div class='add'>+	dev_put(ndev);</div><div class='add'>+	attr-&gt;max_mr_size = dev-&gt;lif_cfg.npts_per_lif * PAGE_SIZE / 2;</div><div class='add'>+	attr-&gt;page_size_cap = dev-&gt;lif_cfg.page_size_supported;</div><div class='add'>+</div><div class='add'>+	attr-&gt;vendor_id = to_pci_dev(dev-&gt;lif_cfg.hwdev)-&gt;vendor;</div><div class='add'>+	attr-&gt;vendor_part_id = to_pci_dev(dev-&gt;lif_cfg.hwdev)-&gt;device;</div><div class='add'>+</div><div class='add'>+	attr-&gt;hw_ver = ionic_lif_asic_rev(dev-&gt;lif_cfg.lif);</div><div class='add'>+	attr-&gt;fw_ver = 0;</div><div class='add'>+	attr-&gt;max_qp = dev-&gt;lif_cfg.qp_count;</div><div class='add'>+	attr-&gt;max_qp_wr = IONIC_MAX_DEPTH;</div><div class='add'>+	attr-&gt;device_cap_flags =</div><div class='add'>+		IB_DEVICE_MEM_WINDOW |</div><div class='add'>+		IB_DEVICE_MEM_MGT_EXTENSIONS |</div><div class='add'>+		IB_DEVICE_MEM_WINDOW_TYPE_2B |</div><div class='add'>+		0;</div><div class='add'>+	attr-&gt;max_send_sge =</div><div class='add'>+		min(ionic_v1_send_wqe_max_sge(dev-&gt;lif_cfg.max_stride, 0, false),</div><div class='add'>+		    IONIC_SPEC_HIGH);</div><div class='add'>+	attr-&gt;max_recv_sge =</div><div class='add'>+		min(ionic_v1_recv_wqe_max_sge(dev-&gt;lif_cfg.max_stride, 0, false),</div><div class='add'>+		    IONIC_SPEC_HIGH);</div><div class='add'>+	attr-&gt;max_sge_rd = attr-&gt;max_send_sge;</div><div class='add'>+	attr-&gt;max_cq = dev-&gt;lif_cfg.cq_count / dev-&gt;lif_cfg.udma_count;</div><div class='add'>+	attr-&gt;max_cqe = IONIC_MAX_CQ_DEPTH - IONIC_CQ_GRACE;</div><div class='add'>+	attr-&gt;max_mr = dev-&gt;lif_cfg.nmrs_per_lif;</div><div class='add'>+	attr-&gt;max_pd = IONIC_MAX_PD;</div><div class='add'>+	attr-&gt;max_qp_rd_atom = IONIC_MAX_RD_ATOM;</div><div class='add'>+	attr-&gt;max_ee_rd_atom = 0;</div><div class='add'>+	attr-&gt;max_res_rd_atom = IONIC_MAX_RD_ATOM;</div><div class='add'>+	attr-&gt;max_qp_init_rd_atom = IONIC_MAX_RD_ATOM;</div><div class='add'>+	attr-&gt;max_ee_init_rd_atom = 0;</div><div class='add'>+	attr-&gt;atomic_cap = IB_ATOMIC_GLOB;</div><div class='add'>+	attr-&gt;masked_atomic_cap = IB_ATOMIC_GLOB;</div><div class='add'>+	attr-&gt;max_mw = dev-&gt;lif_cfg.nmrs_per_lif;</div><div class='add'>+	attr-&gt;max_mcast_grp = 0;</div><div class='add'>+	attr-&gt;max_mcast_qp_attach = 0;</div><div class='add'>+	attr-&gt;max_ah = dev-&gt;lif_cfg.nahs_per_lif;</div><div class='add'>+	attr-&gt;max_fast_reg_page_list_len = dev-&gt;lif_cfg.npts_per_lif / 2;</div><div class='add'>+	attr-&gt;max_pkeys = IONIC_PKEY_TBL_LEN;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_query_port(struct ib_device *ibdev, u32 port,</div><div class='add'>+			    struct ib_port_attr *attr)</div><div class='add'>+{</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+</div><div class='add'>+	if (port != 1)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ndev = ib_device_get_netdev(ibdev, port);</div><div class='add'>+</div><div class='add'>+	if (netif_running(ndev) &amp;&amp; netif_carrier_ok(ndev)) {</div><div class='add'>+		attr-&gt;state = IB_PORT_ACTIVE;</div><div class='add'>+		attr-&gt;phys_state = IB_PORT_PHYS_STATE_LINK_UP;</div><div class='add'>+	} else if (netif_running(ndev)) {</div><div class='add'>+		attr-&gt;state = IB_PORT_DOWN;</div><div class='add'>+		attr-&gt;phys_state = IB_PORT_PHYS_STATE_POLLING;</div><div class='add'>+	} else {</div><div class='add'>+		attr-&gt;state = IB_PORT_DOWN;</div><div class='add'>+		attr-&gt;phys_state = IB_PORT_PHYS_STATE_DISABLED;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	attr-&gt;max_mtu = iboe_get_mtu(ndev-&gt;max_mtu);</div><div class='add'>+	attr-&gt;active_mtu = min(attr-&gt;max_mtu, iboe_get_mtu(ndev-&gt;mtu));</div><div class='add'>+	attr-&gt;gid_tbl_len = IONIC_GID_TBL_LEN;</div><div class='add'>+	attr-&gt;ip_gids = true;</div><div class='add'>+	attr-&gt;port_cap_flags = 0;</div><div class='add'>+	attr-&gt;max_msg_sz = 0x80000000;</div><div class='add'>+	attr-&gt;pkey_tbl_len = IONIC_PKEY_TBL_LEN;</div><div class='add'>+	attr-&gt;max_vl_num = 1;</div><div class='add'>+	attr-&gt;subnet_prefix = 0xfe80000000000000ull;</div><div class='add'>+</div><div class='add'>+	dev_put(ndev);</div><div class='add'>+</div><div class='add'>+	return ib_get_eth_speed(ibdev, port,</div><div class='add'>+				&amp;attr-&gt;active_speed,</div><div class='add'>+				&amp;attr-&gt;active_width);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static enum rdma_link_layer ionic_get_link_layer(struct ib_device *ibdev,</div><div class='add'>+						 u32 port)</div><div class='add'>+{</div><div class='add'>+	return IB_LINK_LAYER_ETHERNET;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_query_pkey(struct ib_device *ibdev, u32 port, u16 index,</div><div class='add'>+			    u16 *pkey)</div><div class='add'>+{</div><div class='add'>+	if (port != 1)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (index != 0)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	*pkey = IB_DEFAULT_PKEY_FULL;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_modify_device(struct ib_device *ibdev, int mask,</div><div class='add'>+			       struct ib_device_modify *attr)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibdev);</div><div class='add'>+</div><div class='add'>+	if (mask &amp; ~IB_DEVICE_MODIFY_NODE_DESC)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (mask &amp; IB_DEVICE_MODIFY_NODE_DESC)</div><div class='add'>+		memcpy(dev-&gt;ibdev.node_desc, attr-&gt;node_desc,</div><div class='add'>+		       IB_DEVICE_NODE_DESC_MAX);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_get_port_immutable(struct ib_device *ibdev, u32 port,</div><div class='add'>+				    struct ib_port_immutable *attr)</div><div class='add'>+{</div><div class='add'>+	if (port != 1)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	attr-&gt;core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;</div><div class='add'>+</div><div class='add'>+	attr-&gt;pkey_tbl_len = IONIC_PKEY_TBL_LEN;</div><div class='add'>+	attr-&gt;gid_tbl_len = IONIC_GID_TBL_LEN;</div><div class='add'>+	attr-&gt;max_mad_size = IB_MGMT_MAD_SIZE;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_get_dev_fw_str(struct ib_device *ibdev, char *str)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = to_ionic_ibdev(ibdev);</div><div class='add'>+</div><div class='add'>+	ionic_lif_fw_version(dev-&gt;lif_cfg.lif, str, IB_FW_VERSION_NAME_MAX);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static ssize_t hw_rev_show(struct device *device, struct device_attribute *attr,</div><div class='add'>+			   char *buf)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev =</div><div class='add'>+		rdma_device_to_drv_device(device, struct ionic_ibdev, ibdev);</div><div class='add'>+</div><div class='add'>+	return sysfs_emit(buf, "0x%x\n", ionic_lif_asic_rev(dev-&gt;lif_cfg.lif));</div><div class='add'>+}</div><div class='add'>+static DEVICE_ATTR_RO(hw_rev);</div><div class='add'>+</div><div class='add'>+static ssize_t hca_type_show(struct device *device,</div><div class='add'>+			     struct device_attribute *attr, char *buf)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev =</div><div class='add'>+		rdma_device_to_drv_device(device, struct ionic_ibdev, ibdev);</div><div class='add'>+</div><div class='add'>+	return sysfs_emit(buf, "%s\n", dev-&gt;ibdev.node_desc);</div><div class='add'>+}</div><div class='add'>+static DEVICE_ATTR_RO(hca_type);</div><div class='add'>+</div><div class='add'>+static struct attribute *ionic_rdma_attributes[] = {</div><div class='add'>+	&amp;dev_attr_hw_rev.attr,</div><div class='add'>+	&amp;dev_attr_hca_type.attr,</div><div class='add'>+	NULL</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct attribute_group ionic_rdma_attr_group = {</div><div class='add'>+	.attrs = ionic_rdma_attributes,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static void ionic_disassociate_ucontext(struct ib_ucontext *ibcontext)</div><div class='add'>+{</div><div class='add'>+	/*</div><div class='add'>+	 * Dummy define disassociate_ucontext so that it does not</div><div class='add'>+	 * wait for user context before cleaning up hw resources.</div><div class='add'>+	 */</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops ionic_dev_ops = {</div><div class='add'>+	.owner = THIS_MODULE,</div><div class='add'>+	.driver_id = RDMA_DRIVER_IONIC,</div><div class='add'>+	.uverbs_abi_ver = IONIC_ABI_VERSION,</div><div class='add'>+</div><div class='add'>+	.alloc_ucontext = ionic_alloc_ucontext,</div><div class='add'>+	.dealloc_ucontext = ionic_dealloc_ucontext,</div><div class='add'>+	.mmap = ionic_mmap,</div><div class='add'>+	.mmap_free = ionic_mmap_free,</div><div class='add'>+	.alloc_pd = ionic_alloc_pd,</div><div class='add'>+	.dealloc_pd = ionic_dealloc_pd,</div><div class='add'>+	.create_ah = ionic_create_ah,</div><div class='add'>+	.query_ah = ionic_query_ah,</div><div class='add'>+	.destroy_ah = ionic_destroy_ah,</div><div class='add'>+	.create_user_ah = ionic_create_ah,</div><div class='add'>+	.get_dma_mr = ionic_get_dma_mr,</div><div class='add'>+	.reg_user_mr = ionic_reg_user_mr,</div><div class='add'>+	.reg_user_mr_dmabuf = ionic_reg_user_mr_dmabuf,</div><div class='add'>+	.dereg_mr = ionic_dereg_mr,</div><div class='add'>+	.alloc_mr = ionic_alloc_mr,</div><div class='add'>+	.map_mr_sg = ionic_map_mr_sg,</div><div class='add'>+	.alloc_mw = ionic_alloc_mw,</div><div class='add'>+	.dealloc_mw = ionic_dealloc_mw,</div><div class='add'>+	.create_cq = ionic_create_cq,</div><div class='add'>+	.destroy_cq = ionic_destroy_cq,</div><div class='add'>+	.create_qp = ionic_create_qp,</div><div class='add'>+	.modify_qp = ionic_modify_qp,</div><div class='add'>+	.query_qp = ionic_query_qp,</div><div class='add'>+	.destroy_qp = ionic_destroy_qp,</div><div class='add'>+</div><div class='add'>+	.post_send = ionic_post_send,</div><div class='add'>+	.post_recv = ionic_post_recv,</div><div class='add'>+	.poll_cq = ionic_poll_cq,</div><div class='add'>+	.req_notify_cq = ionic_req_notify_cq,</div><div class='add'>+</div><div class='add'>+	.query_device = ionic_query_device,</div><div class='add'>+	.query_port = ionic_query_port,</div><div class='add'>+	.get_link_layer = ionic_get_link_layer,</div><div class='add'>+	.query_pkey = ionic_query_pkey,</div><div class='add'>+	.modify_device = ionic_modify_device,</div><div class='add'>+	.get_port_immutable = ionic_get_port_immutable,</div><div class='add'>+	.get_dev_fw_str = ionic_get_dev_fw_str,</div><div class='add'>+	.device_group = &amp;ionic_rdma_attr_group,</div><div class='add'>+	.disassociate_ucontext = ionic_disassociate_ucontext,</div><div class='add'>+</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_ucontext, ionic_ctx, ibctx),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_pd, ionic_pd, ibpd),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_ah, ionic_ah, ibah),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_cq, ionic_vcq, ibcq),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_qp, ionic_qp, ibqp),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_mw, ionic_mr, ibmw),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static void ionic_init_resids(struct ionic_ibdev *dev)</div><div class='add'>+{</div><div class='add'>+	ionic_resid_init(&amp;dev-&gt;inuse_cqid, dev-&gt;lif_cfg.cq_count);</div><div class='add'>+	dev-&gt;half_cqid_udma_shift =</div><div class='add'>+		order_base_2(dev-&gt;lif_cfg.cq_count / dev-&gt;lif_cfg.udma_count);</div><div class='add'>+	ionic_resid_init(&amp;dev-&gt;inuse_pdid, IONIC_MAX_PD);</div><div class='add'>+	ionic_resid_init(&amp;dev-&gt;inuse_ahid, dev-&gt;lif_cfg.nahs_per_lif);</div><div class='add'>+	ionic_resid_init(&amp;dev-&gt;inuse_mrid, dev-&gt;lif_cfg.nmrs_per_lif);</div><div class='add'>+	/* skip reserved lkey */</div><div class='add'>+	dev-&gt;next_mrkey = 1;</div><div class='add'>+	ionic_resid_init(&amp;dev-&gt;inuse_qpid, dev-&gt;lif_cfg.qp_count);</div><div class='add'>+	/* skip reserved SMI and GSI qpids */</div><div class='add'>+	dev-&gt;half_qpid_udma_shift =</div><div class='add'>+		order_base_2(dev-&gt;lif_cfg.qp_count / dev-&gt;lif_cfg.udma_count);</div><div class='add'>+	ionic_resid_init(&amp;dev-&gt;inuse_dbid, dev-&gt;lif_cfg.dbid_count);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_destroy_resids(struct ionic_ibdev *dev)</div><div class='add'>+{</div><div class='add'>+	ionic_resid_destroy(&amp;dev-&gt;inuse_cqid);</div><div class='add'>+	ionic_resid_destroy(&amp;dev-&gt;inuse_pdid);</div><div class='add'>+	ionic_resid_destroy(&amp;dev-&gt;inuse_ahid);</div><div class='add'>+	ionic_resid_destroy(&amp;dev-&gt;inuse_mrid);</div><div class='add'>+	ionic_resid_destroy(&amp;dev-&gt;inuse_qpid);</div><div class='add'>+	ionic_resid_destroy(&amp;dev-&gt;inuse_dbid);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_destroy_ibdev(struct ionic_ibdev *dev)</div><div class='add'>+{</div><div class='add'>+	ionic_kill_rdma_admin(dev, false);</div><div class='add'>+	ib_unregister_device(&amp;dev-&gt;ibdev);</div><div class='add'>+	ionic_stats_cleanup(dev);</div><div class='add'>+	ionic_destroy_rdma_admin(dev);</div><div class='add'>+	ionic_destroy_resids(dev);</div><div class='add'>+	WARN_ON(!xa_empty(&amp;dev-&gt;qp_tbl));</div><div class='add'>+	xa_destroy(&amp;dev-&gt;qp_tbl);</div><div class='add'>+	WARN_ON(!xa_empty(&amp;dev-&gt;cq_tbl));</div><div class='add'>+	xa_destroy(&amp;dev-&gt;cq_tbl);</div><div class='add'>+	ib_dealloc_device(&amp;dev-&gt;ibdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev;</div><div class='add'>+	struct ionic_ibdev *dev;</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	dev = ib_alloc_device(ionic_ibdev, ibdev);</div><div class='add'>+	if (!dev)</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	ionic_fill_lif_cfg(ionic_adev-&gt;lif, &amp;dev-&gt;lif_cfg);</div><div class='add'>+</div><div class='add'>+	xa_init_flags(&amp;dev-&gt;qp_tbl, GFP_ATOMIC);</div><div class='add'>+	xa_init_flags(&amp;dev-&gt;cq_tbl, GFP_ATOMIC);</div><div class='add'>+</div><div class='add'>+	ionic_init_resids(dev);</div><div class='add'>+</div><div class='add'>+	rc = ionic_rdma_reset_devcmd(dev);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_reset;</div><div class='add'>+</div><div class='add'>+	rc = ionic_create_rdma_admin(dev);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_admin;</div><div class='add'>+</div><div class='add'>+	ibdev = &amp;dev-&gt;ibdev;</div><div class='add'>+	ibdev-&gt;dev.parent = dev-&gt;lif_cfg.hwdev;</div><div class='add'>+</div><div class='add'>+	strscpy(ibdev-&gt;name, "ionic_%d", IB_DEVICE_NAME_MAX);</div><div class='add'>+	strscpy(ibdev-&gt;node_desc, DEVICE_DESCRIPTION, IB_DEVICE_NODE_DESC_MAX);</div><div class='add'>+</div><div class='add'>+	ibdev-&gt;node_type = RDMA_NODE_IB_CA;</div><div class='add'>+	ibdev-&gt;phys_port_cnt = 1;</div><div class='add'>+</div><div class='add'>+	/* the first two eq are reserved for async events */</div><div class='add'>+	ibdev-&gt;num_comp_vectors = dev-&gt;lif_cfg.eq_count - 2;</div><div class='add'>+</div><div class='add'>+	ndev = ionic_lif_netdev(ionic_adev-&gt;lif);</div><div class='add'>+	addrconf_ifid_eui48((u8 *)&amp;ibdev-&gt;node_guid, ndev);</div><div class='add'>+	rc = ib_device_set_netdev(ibdev, ndev, 1);</div><div class='add'>+	/* ionic_lif_netdev() returns ndev with refcount held */</div><div class='add'>+	dev_put(ndev);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_admin;</div><div class='add'>+</div><div class='add'>+	ib_set_device_ops(&amp;dev-&gt;ibdev, &amp;ionic_dev_ops);</div><div class='add'>+</div><div class='add'>+	ionic_stats_init(dev);</div><div class='add'>+</div><div class='add'>+	rc = ib_register_device(ibdev, "ionic_%d", ibdev-&gt;dev.parent);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_register;</div><div class='add'>+</div><div class='add'>+	return dev;</div><div class='add'>+</div><div class='add'>+err_register:</div><div class='add'>+	ionic_stats_cleanup(dev);</div><div class='add'>+err_admin:</div><div class='add'>+	ionic_kill_rdma_admin(dev, false);</div><div class='add'>+	ionic_destroy_rdma_admin(dev);</div><div class='add'>+err_reset:</div><div class='add'>+	ionic_destroy_resids(dev);</div><div class='add'>+	xa_destroy(&amp;dev-&gt;qp_tbl);</div><div class='add'>+	xa_destroy(&amp;dev-&gt;cq_tbl);</div><div class='add'>+	ib_dealloc_device(&amp;dev-&gt;ibdev);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(rc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_aux_probe(struct auxiliary_device *adev,</div><div class='add'>+			   const struct auxiliary_device_id *id)</div><div class='add'>+{</div><div class='add'>+	struct ionic_aux_dev *ionic_adev;</div><div class='add'>+	struct ionic_ibdev *dev;</div><div class='add'>+</div><div class='add'>+	ionic_adev = container_of(adev, struct ionic_aux_dev, adev);</div><div class='add'>+	dev = ionic_create_ibdev(ionic_adev);</div><div class='add'>+	if (IS_ERR(dev))</div><div class='add'>+		return dev_err_probe(&amp;adev-&gt;dev, PTR_ERR(dev),</div><div class='add'>+				     "Failed to register ibdev\n");</div><div class='add'>+</div><div class='add'>+	auxiliary_set_drvdata(adev, dev);</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ibdev, "registered\n");</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ionic_aux_remove(struct auxiliary_device *adev)</div><div class='add'>+{</div><div class='add'>+	struct ionic_ibdev *dev = auxiliary_get_drvdata(adev);</div><div class='add'>+</div><div class='add'>+	dev_dbg(&amp;adev-&gt;dev, "unregister ibdev\n");</div><div class='add'>+	ionic_destroy_ibdev(dev);</div><div class='add'>+	dev_dbg(&amp;adev-&gt;dev, "unregistered\n");</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct auxiliary_device_id ionic_aux_id_table[] = {</div><div class='add'>+	{ .name = "ionic.rdma", },</div><div class='add'>+	{},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+MODULE_DEVICE_TABLE(auxiliary, ionic_aux_id_table);</div><div class='add'>+</div><div class='add'>+static struct auxiliary_driver ionic_aux_r_driver = {</div><div class='add'>+	.name = "rdma",</div><div class='add'>+	.probe = ionic_aux_probe,</div><div class='add'>+	.remove = ionic_aux_remove,</div><div class='add'>+	.id_table = ionic_aux_id_table,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int __init ionic_mod_init(void)</div><div class='add'>+{</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	ionic_evt_workq = create_workqueue(KBUILD_MODNAME "-evt");</div><div class='add'>+	if (!ionic_evt_workq)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	rc = auxiliary_driver_register(&amp;ionic_aux_r_driver);</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_aux;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_aux:</div><div class='add'>+	destroy_workqueue(ionic_evt_workq);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void __exit ionic_mod_exit(void)</div><div class='add'>+{</div><div class='add'>+	auxiliary_driver_unregister(&amp;ionic_aux_r_driver);</div><div class='add'>+	destroy_workqueue(ionic_evt_workq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+module_init(ionic_mod_init);</div><div class='add'>+module_exit(ionic_mod_exit);</div><div class='head'>diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.h b/drivers/infiniband/hw/ionic/ionic_ibdev.h<br/>new file mode 100644<br/>index 000000000000..82fda1e3cdb6<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/ionic/ionic_ibdev.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/ionic/ionic_ibdev.h</a></div><div class='hunk'>@@ -0,0 +1,517 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 */</div><div class='add'>+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */</div><div class='add'>+</div><div class='add'>+#ifndef _IONIC_IBDEV_H_</div><div class='add'>+#define _IONIC_IBDEV_H_</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/ib_umem.h&gt;</div><div class='add'>+#include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+#include &lt;rdma/ib_pack.h&gt;</div><div class='add'>+#include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/ionic-abi.h&gt;</div><div class='add'>+#include &lt;ionic_api.h&gt;</div><div class='add'>+#include &lt;ionic_regs.h&gt;</div><div class='add'>+</div><div class='add'>+#include "ionic_fw.h"</div><div class='add'>+#include "ionic_queue.h"</div><div class='add'>+#include "ionic_res.h"</div><div class='add'>+</div><div class='add'>+#include "ionic_lif_cfg.h"</div><div class='add'>+</div><div class='add'>+/* Config knobs */</div><div class='add'>+#define IONIC_EQ_DEPTH 511</div><div class='add'>+#define IONIC_EQ_COUNT 32</div><div class='add'>+#define IONIC_AQ_DEPTH 63</div><div class='add'>+#define IONIC_AQ_COUNT 4</div><div class='add'>+#define IONIC_EQ_ISR_BUDGET 10</div><div class='add'>+#define IONIC_EQ_WORK_BUDGET 1000</div><div class='add'>+#define IONIC_MAX_RD_ATOM 16</div><div class='add'>+#define IONIC_PKEY_TBL_LEN 1</div><div class='add'>+#define IONIC_GID_TBL_LEN 256</div><div class='add'>+</div><div class='add'>+#define IONIC_MAX_QPID 0xffffff</div><div class='add'>+#define IONIC_SPEC_HIGH 8</div><div class='add'>+#define IONIC_MAX_PD 1024</div><div class='add'>+#define IONIC_SPEC_HIGH 8</div><div class='add'>+#define IONIC_SQCMB_ORDER 5</div><div class='add'>+#define IONIC_RQCMB_ORDER 0</div><div class='add'>+</div><div class='add'>+#define IONIC_META_LAST		((void *)1ul)</div><div class='add'>+#define IONIC_META_POSTED	((void *)2ul)</div><div class='add'>+</div><div class='add'>+#define IONIC_CQ_GRACE 100</div><div class='add'>+</div><div class='add'>+#define IONIC_ROCE_UDP_SPORT	28272</div><div class='add'>+#define IONIC_DMA_LKEY		0</div><div class='add'>+#define IONIC_DMA_RKEY		IONIC_DMA_LKEY</div><div class='add'>+</div><div class='add'>+#define IONIC_CMB_SUPPORTED \</div><div class='add'>+	(IONIC_CMB_ENABLE | IONIC_CMB_REQUIRE | IONIC_CMB_EXPDB | \</div><div class='add'>+	 IONIC_CMB_WC | IONIC_CMB_UC)</div><div class='add'>+</div><div class='add'>+/* resource is not reserved on the device, indicated in tbl_order */</div><div class='add'>+#define IONIC_RES_INVALID	-1</div><div class='add'>+</div><div class='add'>+struct ionic_aq;</div><div class='add'>+struct ionic_cq;</div><div class='add'>+struct ionic_eq;</div><div class='add'>+struct ionic_vcq;</div><div class='add'>+</div><div class='add'>+enum ionic_admin_state {</div><div class='add'>+	IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */</div><div class='add'>+	IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */</div><div class='add'>+	IONIC_ADMIN_KILLED, /* not submitting, locally completed */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum ionic_admin_flags {</div><div class='add'>+	IONIC_ADMIN_F_BUSYWAIT  = BIT(0),	/* Don't sleep */</div><div class='add'>+	IONIC_ADMIN_F_TEARDOWN  = BIT(1),	/* In destroy path */</div><div class='add'>+	IONIC_ADMIN_F_INTERRUPT = BIT(2),	/* Interruptible w/timeout */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum ionic_mmap_flag {</div><div class='add'>+	IONIC_MMAP_WC = BIT(0),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_mmap_entry {</div><div class='add'>+	struct rdma_user_mmap_entry rdma_entry;</div><div class='add'>+	unsigned long size;</div><div class='add'>+	unsigned long pfn;</div><div class='add'>+	u8 mmap_flags;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_ibdev {</div><div class='add'>+	struct ib_device	ibdev;</div><div class='add'>+</div><div class='add'>+	struct ionic_lif_cfg	lif_cfg;</div><div class='add'>+</div><div class='add'>+	struct xarray		qp_tbl;</div><div class='add'>+	struct xarray		cq_tbl;</div><div class='add'>+</div><div class='add'>+	struct ionic_resid_bits	inuse_dbid;</div><div class='add'>+	struct ionic_resid_bits	inuse_pdid;</div><div class='add'>+	struct ionic_resid_bits	inuse_ahid;</div><div class='add'>+	struct ionic_resid_bits	inuse_mrid;</div><div class='add'>+	struct ionic_resid_bits	inuse_qpid;</div><div class='add'>+	struct ionic_resid_bits	inuse_cqid;</div><div class='add'>+</div><div class='add'>+	u8			half_cqid_udma_shift;</div><div class='add'>+	u8			half_qpid_udma_shift;</div><div class='add'>+	u8			next_qpid_udma_idx;</div><div class='add'>+	u8			next_mrkey;</div><div class='add'>+</div><div class='add'>+	struct work_struct	reset_work;</div><div class='add'>+	bool			reset_posted;</div><div class='add'>+	u32			reset_cnt;</div><div class='add'>+</div><div class='add'>+	struct delayed_work	admin_dwork;</div><div class='add'>+	struct ionic_aq		**aq_vec;</div><div class='add'>+	atomic_t		admin_state;</div><div class='add'>+</div><div class='add'>+	struct ionic_eq		**eq_vec;</div><div class='add'>+</div><div class='add'>+	struct ionic_v1_stat	*hw_stats;</div><div class='add'>+	void			*hw_stats_buf;</div><div class='add'>+	struct rdma_stat_desc	*hw_stats_hdrs;</div><div class='add'>+	struct ionic_counter_stats *counter_stats;</div><div class='add'>+	int			hw_stats_count;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_eq {</div><div class='add'>+	struct ionic_ibdev	*dev;</div><div class='add'>+</div><div class='add'>+	u32			eqid;</div><div class='add'>+	u32			intr;</div><div class='add'>+</div><div class='add'>+	struct ionic_queue	q;</div><div class='add'>+</div><div class='add'>+	int			armed;</div><div class='add'>+	bool			enable;</div><div class='add'>+</div><div class='add'>+	struct work_struct	work;</div><div class='add'>+</div><div class='add'>+	int			irq;</div><div class='add'>+	char			name[32];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_admin_wr {</div><div class='add'>+	struct completion		work;</div><div class='add'>+	struct list_head		aq_ent;</div><div class='add'>+	struct ionic_v1_admin_wqe	wqe;</div><div class='add'>+	struct ionic_v1_cqe		cqe;</div><div class='add'>+	struct ionic_aq			*aq;</div><div class='add'>+	int				status;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_admin_wr_q {</div><div class='add'>+	struct ionic_admin_wr	*wr;</div><div class='add'>+	int			wqe_strides;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_aq {</div><div class='add'>+	struct ionic_ibdev	*dev;</div><div class='add'>+	struct ionic_vcq	*vcq;</div><div class='add'>+</div><div class='add'>+	struct work_struct	work;</div><div class='add'>+</div><div class='add'>+	atomic_t		admin_state;</div><div class='add'>+	unsigned long		stamp;</div><div class='add'>+	bool			armed;</div><div class='add'>+</div><div class='add'>+	u32			aqid;</div><div class='add'>+	u32			cqid;</div><div class='add'>+</div><div class='add'>+	spinlock_t		lock; /* for posting */</div><div class='add'>+	struct ionic_queue	q;</div><div class='add'>+	struct ionic_admin_wr_q	*q_wr;</div><div class='add'>+	struct list_head	wr_prod;</div><div class='add'>+	struct list_head	wr_post;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_ctx {</div><div class='add'>+	struct ib_ucontext	ibctx;</div><div class='add'>+	u32			dbid;</div><div class='add'>+	struct rdma_user_mmap_entry	*mmap_dbell;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_tbl_buf {</div><div class='add'>+	u32		tbl_limit;</div><div class='add'>+	u32		tbl_pages;</div><div class='add'>+	size_t		tbl_size;</div><div class='add'>+	__le64		*tbl_buf;</div><div class='add'>+	dma_addr_t	tbl_dma;</div><div class='add'>+	u8		page_size_log2;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_pd {</div><div class='add'>+	struct ib_pd		ibpd;</div><div class='add'>+</div><div class='add'>+	u32			pdid;</div><div class='add'>+	u32			flags;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_cq {</div><div class='add'>+	struct ionic_vcq	*vcq;</div><div class='add'>+</div><div class='add'>+	u32			cqid;</div><div class='add'>+	u32			eqid;</div><div class='add'>+</div><div class='add'>+	spinlock_t		lock; /* for polling */</div><div class='add'>+	struct list_head	poll_sq;</div><div class='add'>+	bool			flush;</div><div class='add'>+	struct list_head	flush_sq;</div><div class='add'>+	struct list_head	flush_rq;</div><div class='add'>+	struct list_head	ibkill_flush_ent;</div><div class='add'>+</div><div class='add'>+	struct ionic_queue	q;</div><div class='add'>+	bool			color;</div><div class='add'>+	int			credit;</div><div class='add'>+	u16			arm_any_prod;</div><div class='add'>+	u16			arm_sol_prod;</div><div class='add'>+</div><div class='add'>+	struct kref		cq_kref;</div><div class='add'>+	struct completion	cq_rel_comp;</div><div class='add'>+</div><div class='add'>+	/* infrequently accessed, keep at end */</div><div class='add'>+	struct ib_umem		*umem;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_vcq {</div><div class='add'>+	struct ib_cq		ibcq;</div><div class='add'>+	struct ionic_cq		cq[2];</div><div class='add'>+	u8			udma_mask;</div><div class='add'>+	u8			poll_idx;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_sq_meta {</div><div class='add'>+	u64			wrid;</div><div class='add'>+	u32			len;</div><div class='add'>+	u16			seq;</div><div class='add'>+	u8			ibop;</div><div class='add'>+	u8			ibsts;</div><div class='add'>+	u8			remote:1;</div><div class='add'>+	u8			signal:1;</div><div class='add'>+	u8			local_comp:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_rq_meta {</div><div class='add'>+	struct ionic_rq_meta	*next;</div><div class='add'>+	u64			wrid;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_qp {</div><div class='add'>+	struct ib_qp		ibqp;</div><div class='add'>+	enum ib_qp_state	state;</div><div class='add'>+</div><div class='add'>+	u32			qpid;</div><div class='add'>+	u32			ahid;</div><div class='add'>+	u32			sq_cqid;</div><div class='add'>+	u32			rq_cqid;</div><div class='add'>+	u8			udma_idx;</div><div class='add'>+	u8			has_ah:1;</div><div class='add'>+	u8			has_sq:1;</div><div class='add'>+	u8			has_rq:1;</div><div class='add'>+	u8			sig_all:1;</div><div class='add'>+</div><div class='add'>+	struct list_head	qp_list_counter;</div><div class='add'>+</div><div class='add'>+	struct list_head	cq_poll_sq;</div><div class='add'>+	struct list_head	cq_flush_sq;</div><div class='add'>+	struct list_head	cq_flush_rq;</div><div class='add'>+	struct list_head	ibkill_flush_ent;</div><div class='add'>+</div><div class='add'>+	spinlock_t		sq_lock; /* for posting and polling */</div><div class='add'>+	struct ionic_queue	sq;</div><div class='add'>+	struct ionic_sq_meta	*sq_meta;</div><div class='add'>+	u16			*sq_msn_idx;</div><div class='add'>+	int			sq_spec;</div><div class='add'>+	u16			sq_old_prod;</div><div class='add'>+	u16			sq_msn_prod;</div><div class='add'>+	u16			sq_msn_cons;</div><div class='add'>+	u8			sq_cmb;</div><div class='add'>+	bool			sq_flush;</div><div class='add'>+	bool			sq_flush_rcvd;</div><div class='add'>+</div><div class='add'>+	spinlock_t		rq_lock; /* for posting and polling */</div><div class='add'>+	struct ionic_queue	rq;</div><div class='add'>+	struct ionic_rq_meta	*rq_meta;</div><div class='add'>+	struct ionic_rq_meta	*rq_meta_head;</div><div class='add'>+	int			rq_spec;</div><div class='add'>+	u16			rq_old_prod;</div><div class='add'>+	u8			rq_cmb;</div><div class='add'>+	bool			rq_flush;</div><div class='add'>+</div><div class='add'>+	struct kref		qp_kref;</div><div class='add'>+	struct completion	qp_rel_comp;</div><div class='add'>+</div><div class='add'>+	/* infrequently accessed, keep at end */</div><div class='add'>+	int			sgid_index;</div><div class='add'>+	int			sq_cmb_order;</div><div class='add'>+	u32			sq_cmb_pgid;</div><div class='add'>+	phys_addr_t		sq_cmb_addr;</div><div class='add'>+	struct rdma_user_mmap_entry *mmap_sq_cmb;</div><div class='add'>+</div><div class='add'>+	struct ib_umem		*sq_umem;</div><div class='add'>+</div><div class='add'>+	int			rq_cmb_order;</div><div class='add'>+	u32			rq_cmb_pgid;</div><div class='add'>+	phys_addr_t		rq_cmb_addr;</div><div class='add'>+	struct rdma_user_mmap_entry *mmap_rq_cmb;</div><div class='add'>+</div><div class='add'>+	struct ib_umem		*rq_umem;</div><div class='add'>+</div><div class='add'>+	int			dcqcn_profile;</div><div class='add'>+</div><div class='add'>+	struct ib_ud_header	*hdr;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_ah {</div><div class='add'>+	struct ib_ah		ibah;</div><div class='add'>+	u32			ahid;</div><div class='add'>+	int			sgid_index;</div><div class='add'>+	struct ib_ud_header	hdr;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_mr {</div><div class='add'>+	union {</div><div class='add'>+		struct ib_mr	ibmr;</div><div class='add'>+		struct ib_mw	ibmw;</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	u32			mrid;</div><div class='add'>+	int			flags;</div><div class='add'>+</div><div class='add'>+	struct ib_umem		*umem;</div><div class='add'>+	struct ionic_tbl_buf	buf;</div><div class='add'>+	bool			created;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_counter_stats {</div><div class='add'>+	int queue_stats_count;</div><div class='add'>+	struct ionic_v1_stat *hdr;</div><div class='add'>+	struct rdma_stat_desc *stats_hdrs;</div><div class='add'>+	struct xarray xa_counters;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ionic_counter {</div><div class='add'>+	void *vals;</div><div class='add'>+	struct list_head qp_list;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibdev, struct ionic_ibdev, ibdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct ionic_ctx *to_ionic_ctx(struct ib_ucontext *ibctx)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibctx, struct ionic_ctx, ibctx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct ionic_ctx *to_ionic_ctx_uobj(struct ib_uobject *uobj)</div><div class='add'>+{</div><div class='add'>+	if (!uobj)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	if (!uobj-&gt;context)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	return to_ionic_ctx(uobj-&gt;context);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct ionic_pd *to_ionic_pd(struct ib_pd *ibpd)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibpd, struct ionic_pd, ibpd);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct ionic_mr *to_ionic_mr(struct ib_mr *ibmr)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibmr, struct ionic_mr, ibmr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct ionic_mr *to_ionic_mw(struct ib_mw *ibmw)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibmw, struct ionic_mr, ibmw);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct ionic_vcq *to_ionic_vcq(struct ib_cq *ibcq)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibcq, struct ionic_vcq, ibcq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct ionic_cq *to_ionic_vcq_cq(struct ib_cq *ibcq,</div><div class='add'>+					       uint8_t udma_idx)</div><div class='add'>+{</div><div class='add'>+	return &amp;to_ionic_vcq(ibcq)-&gt;cq[udma_idx];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibqp, struct ionic_qp, ibqp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibah, struct ionic_ah, ibah);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 ionic_ctx_dbid(struct ionic_ibdev *dev,</div><div class='add'>+				 struct ionic_ctx *ctx)</div><div class='add'>+{</div><div class='add'>+	if (!ctx)</div><div class='add'>+		return dev-&gt;lif_cfg.dbid;</div><div class='add'>+</div><div class='add'>+	return ctx-&gt;dbid;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u32 ionic_obj_dbid(struct ionic_ibdev *dev,</div><div class='add'>+				 struct ib_uobject *uobj)</div><div class='add'>+{</div><div class='add'>+	return ionic_ctx_dbid(dev, to_ionic_ctx_uobj(uobj));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool ionic_ibop_is_local(enum ib_wr_opcode op)</div><div class='add'>+{</div><div class='add'>+	return op == IB_WR_LOCAL_INV || op == IB_WR_REG_MR;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void ionic_qp_complete(struct kref *kref)</div><div class='add'>+{</div><div class='add'>+	struct ionic_qp *qp = container_of(kref, struct ionic_qp, qp_kref);</div><div class='add'>+</div><div class='add'>+	complete(&amp;qp-&gt;qp_rel_comp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void ionic_cq_complete(struct kref *kref)</div><div class='add'>+{</div><div class='add'>+	struct ionic_cq *cq = container_of(kref, struct ionic_cq, cq_kref);</div><div class='add'>+</div><div class='add'>+	complete(&amp;cq-&gt;cq_rel_comp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* ionic_admin.c */</div><div class='add'>+extern struct workqueue_struct *ionic_evt_workq;</div><div class='add'>+void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr);</div><div class='add'>+int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr,</div><div class='add'>+		     enum ionic_admin_flags);</div><div class='add'>+</div><div class='add'>+int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev);</div><div class='add'>+</div><div class='add'>+int ionic_create_rdma_admin(struct ionic_ibdev *dev);</div><div class='add'>+void ionic_destroy_rdma_admin(struct ionic_ibdev *dev);</div><div class='add'>+void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path);</div><div class='add'>+</div><div class='add'>+/* ionic_controlpath.c */</div><div class='add'>+int ionic_create_cq_common(struct ionic_vcq *vcq,</div><div class='add'>+			   struct ionic_tbl_buf *buf,</div><div class='add'>+			   const struct ib_cq_init_attr *attr,</div><div class='add'>+			   struct ionic_ctx *ctx,</div><div class='add'>+			   struct ib_udata *udata,</div><div class='add'>+			   struct ionic_qdesc *req_cq,</div><div class='add'>+			   __u32 *resp_cqid,</div><div class='add'>+			   int udma_idx);</div><div class='add'>+void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq);</div><div class='add'>+void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp);</div><div class='add'>+void ionic_notify_flush_cq(struct ionic_cq *cq);</div><div class='add'>+</div><div class='add'>+int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata);</div><div class='add'>+void ionic_dealloc_ucontext(struct ib_ucontext *ibctx);</div><div class='add'>+int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma);</div><div class='add'>+void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry);</div><div class='add'>+int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);</div><div class='add'>+int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);</div><div class='add'>+int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,</div><div class='add'>+		    struct ib_udata *udata);</div><div class='add'>+int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);</div><div class='add'>+int ionic_destroy_ah(struct ib_ah *ibah, u32 flags);</div><div class='add'>+struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access);</div><div class='add'>+struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,</div><div class='add'>+				u64 addr, int access, struct ib_dmah *dmah,</div><div class='add'>+				struct ib_udata *udata);</div><div class='add'>+struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset,</div><div class='add'>+				       u64 length, u64 addr, int fd, int access,</div><div class='add'>+					   struct ib_dmah *dmah,</div><div class='add'>+				       struct uverbs_attr_bundle *attrs);</div><div class='add'>+int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);</div><div class='add'>+struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type,</div><div class='add'>+			     u32 max_sg);</div><div class='add'>+int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,</div><div class='add'>+		    unsigned int *sg_offset);</div><div class='add'>+int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);</div><div class='add'>+int ionic_dealloc_mw(struct ib_mw *ibmw);</div><div class='add'>+int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		    struct uverbs_attr_bundle *attrs);</div><div class='add'>+int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);</div><div class='add'>+int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,</div><div class='add'>+		    struct ib_udata *udata);</div><div class='add'>+int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,</div><div class='add'>+		    struct ib_udata *udata);</div><div class='add'>+int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,</div><div class='add'>+		   struct ib_qp_init_attr *init_attr);</div><div class='add'>+int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);</div><div class='add'>+</div><div class='add'>+/* ionic_datapath.c */</div><div class='add'>+int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,</div><div class='add'>+		    const struct ib_send_wr **bad);</div><div class='add'>+int ionic_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,</div><div class='add'>+		    const struct ib_recv_wr **bad);</div><div class='add'>+int ionic_poll_cq(struct ib_cq *ibcq, int nwc, struct ib_wc *wc);</div><div class='add'>+int ionic_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);</div><div class='add'>+</div><div class='add'>+/* ionic_hw_stats.c */</div><div class='add'>+void ionic_stats_init(struct ionic_ibdev *dev);</div><div class='add'>+void ionic_stats_cleanup(struct ionic_ibdev *dev);</div><div class='add'>+</div><div class='add'>+/* ionic_pgtbl.c */</div><div class='add'>+__le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va);</div><div class='add'>+__be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va);</div><div class='add'>+int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma);</div><div class='add'>+int ionic_pgtbl_init(struct ionic_ibdev *dev,</div><div class='add'>+		     struct ionic_tbl_buf *buf,</div><div class='add'>+		     struct ib_umem *umem,</div><div class='add'>+		     dma_addr_t dma,</div><div class='add'>+		     int limit,</div><div class='add'>+		     u64 page_size);</div><div class='add'>+void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf);</div><div class='add'>+#endif /* _IONIC_IBDEV_H_ */</div><div class='head'>diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c<br/>new file mode 100644<br/>index 000000000000..f3cd281c3a2f<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/ionic/ionic_lif_cfg.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/ionic/ionic_lif_cfg.c</a></div><div class='hunk'>@@ -0,0 +1,111 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0</div><div class='add'>+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/kernel.h&gt;</div><div class='add'>+</div><div class='add'>+#include &lt;ionic.h&gt;</div><div class='add'>+#include &lt;ionic_lif.h&gt;</div><div class='add'>+</div><div class='add'>+#include "ionic_lif_cfg.h"</div><div class='add'>+</div><div class='add'>+#define IONIC_MIN_RDMA_VERSION	0</div><div class='add'>+#define IONIC_MAX_RDMA_VERSION	2</div><div class='add'>+</div><div class='add'>+static u8 ionic_get_expdb(struct ionic_lif *lif)</div><div class='add'>+{</div><div class='add'>+	u8 expdb_support = 0;</div><div class='add'>+</div><div class='add'>+	if (lif-&gt;ionic-&gt;idev.phy_cmb_expdb64_pages)</div><div class='add'>+		expdb_support |= IONIC_EXPDB_64B_WQE;</div><div class='add'>+	if (lif-&gt;ionic-&gt;idev.phy_cmb_expdb128_pages)</div><div class='add'>+		expdb_support |= IONIC_EXPDB_128B_WQE;</div><div class='add'>+	if (lif-&gt;ionic-&gt;idev.phy_cmb_expdb256_pages)</div><div class='add'>+		expdb_support |= IONIC_EXPDB_256B_WQE;</div><div class='add'>+	if (lif-&gt;ionic-&gt;idev.phy_cmb_expdb512_pages)</div><div class='add'>+		expdb_support |= IONIC_EXPDB_512B_WQE;</div><div class='add'>+</div><div class='add'>+	return expdb_support;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg)</div><div class='add'>+{</div><div class='add'>+	union ionic_lif_identity *ident = &amp;lif-&gt;ionic-&gt;ident.lif;</div><div class='add'>+</div><div class='add'>+	cfg-&gt;lif = lif;</div><div class='add'>+	cfg-&gt;hwdev = &amp;lif-&gt;ionic-&gt;pdev-&gt;dev;</div><div class='add'>+	cfg-&gt;lif_index = lif-&gt;index;</div><div class='add'>+	cfg-&gt;lif_hw_index = lif-&gt;hw_index;</div><div class='add'>+</div><div class='add'>+	cfg-&gt;dbid = lif-&gt;kern_pid;</div><div class='add'>+	cfg-&gt;dbid_count = le32_to_cpu(lif-&gt;ionic-&gt;ident.dev.ndbpgs_per_lif);</div><div class='add'>+	cfg-&gt;dbpage = lif-&gt;kern_dbpage;</div><div class='add'>+	cfg-&gt;intr_ctrl = lif-&gt;ionic-&gt;idev.intr_ctrl;</div><div class='add'>+</div><div class='add'>+	cfg-&gt;db_phys = lif-&gt;ionic-&gt;bars[IONIC_PCI_BAR_DBELL].bus_addr;</div><div class='add'>+</div><div class='add'>+	if (IONIC_VERSION(ident-&gt;rdma.version, ident-&gt;rdma.minor_version) &gt;=</div><div class='add'>+	    IONIC_VERSION(2, 1))</div><div class='add'>+		cfg-&gt;page_size_supported =</div><div class='add'>+		    le64_to_cpu(ident-&gt;rdma.page_size_cap);</div><div class='add'>+	else</div><div class='add'>+		cfg-&gt;page_size_supported = IONIC_PAGE_SIZE_SUPPORTED;</div><div class='add'>+</div><div class='add'>+	cfg-&gt;rdma_version = ident-&gt;rdma.version;</div><div class='add'>+	cfg-&gt;qp_opcodes = ident-&gt;rdma.qp_opcodes;</div><div class='add'>+	cfg-&gt;admin_opcodes = ident-&gt;rdma.admin_opcodes;</div><div class='add'>+</div><div class='add'>+	cfg-&gt;stats_type = le16_to_cpu(ident-&gt;rdma.stats_type);</div><div class='add'>+	cfg-&gt;npts_per_lif = le32_to_cpu(ident-&gt;rdma.npts_per_lif);</div><div class='add'>+	cfg-&gt;nmrs_per_lif = le32_to_cpu(ident-&gt;rdma.nmrs_per_lif);</div><div class='add'>+	cfg-&gt;nahs_per_lif = le32_to_cpu(ident-&gt;rdma.nahs_per_lif);</div><div class='add'>+</div><div class='add'>+	cfg-&gt;aq_base = le32_to_cpu(ident-&gt;rdma.aq_qtype.qid_base);</div><div class='add'>+	cfg-&gt;cq_base = le32_to_cpu(ident-&gt;rdma.cq_qtype.qid_base);</div><div class='add'>+	cfg-&gt;eq_base = le32_to_cpu(ident-&gt;rdma.eq_qtype.qid_base);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * ionic_create_rdma_admin() may reduce aq_count or eq_count if</div><div class='add'>+	 * it is unable to allocate all that were requested.</div><div class='add'>+	 * aq_count is tunable; see ionic_aq_count</div><div class='add'>+	 * eq_count is tunable; see ionic_eq_count</div><div class='add'>+	 */</div><div class='add'>+	cfg-&gt;aq_count = le32_to_cpu(ident-&gt;rdma.aq_qtype.qid_count);</div><div class='add'>+	cfg-&gt;eq_count = le32_to_cpu(ident-&gt;rdma.eq_qtype.qid_count);</div><div class='add'>+	cfg-&gt;cq_count = le32_to_cpu(ident-&gt;rdma.cq_qtype.qid_count);</div><div class='add'>+	cfg-&gt;qp_count = le32_to_cpu(ident-&gt;rdma.sq_qtype.qid_count);</div><div class='add'>+	cfg-&gt;dbid_count = le32_to_cpu(lif-&gt;ionic-&gt;ident.dev.ndbpgs_per_lif);</div><div class='add'>+</div><div class='add'>+	cfg-&gt;aq_qtype = ident-&gt;rdma.aq_qtype.qtype;</div><div class='add'>+	cfg-&gt;sq_qtype = ident-&gt;rdma.sq_qtype.qtype;</div><div class='add'>+	cfg-&gt;rq_qtype = ident-&gt;rdma.rq_qtype.qtype;</div><div class='add'>+	cfg-&gt;cq_qtype = ident-&gt;rdma.cq_qtype.qtype;</div><div class='add'>+	cfg-&gt;eq_qtype = ident-&gt;rdma.eq_qtype.qtype;</div><div class='add'>+	cfg-&gt;udma_qgrp_shift = ident-&gt;rdma.udma_shift;</div><div class='add'>+	cfg-&gt;udma_count = 2;</div><div class='add'>+</div><div class='add'>+	cfg-&gt;max_stride = ident-&gt;rdma.max_stride;</div><div class='add'>+	cfg-&gt;expdb_mask = ionic_get_expdb(lif);</div><div class='add'>+</div><div class='add'>+	cfg-&gt;sq_expdb =</div><div class='add'>+	    !!(lif-&gt;qtype_info[IONIC_QTYPE_TXQ].features &amp; IONIC_QIDENT_F_EXPDB);</div><div class='add'>+	cfg-&gt;rq_expdb =</div><div class='add'>+	    !!(lif-&gt;qtype_info[IONIC_QTYPE_RXQ].features &amp; IONIC_QIDENT_F_EXPDB);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct net_device *ionic_lif_netdev(struct ionic_lif *lif)</div><div class='add'>+{</div><div class='add'>+	struct net_device *netdev = lif-&gt;netdev;</div><div class='add'>+</div><div class='add'>+	dev_hold(netdev);</div><div class='add'>+	return netdev;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void ionic_lif_fw_version(struct ionic_lif *lif, char *str, size_t len)</div><div class='add'>+{</div><div class='add'>+	strscpy(str, lif-&gt;ionic-&gt;idev.dev_info.fw_version, len);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+u8 ionic_lif_asic_rev(struct ionic_lif *lif)</div><div class='add'>+{</div><div class='add'>+	return lif-&gt;ionic-&gt;idev.dev_info.asic_rev;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h<br/>new file mode 100644<br/>index 000000000000..20853429f623<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/ionic/ionic_lif_cfg.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/ionic/ionic_lif_cfg.h</a></div><div class='hunk'>@@ -0,0 +1,66 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 */</div><div class='add'>+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */</div><div class='add'>+</div><div class='add'>+#ifndef _IONIC_LIF_CFG_H_</div><div class='add'>+</div><div class='add'>+#define IONIC_VERSION(a, b) (((a) &lt;&lt; 16) + ((b) &lt;&lt; 8))</div><div class='add'>+#define IONIC_PAGE_SIZE_SUPPORTED	0x40201000 /* 4kb, 2Mb, 1Gb */</div><div class='add'>+</div><div class='add'>+#define IONIC_EXPDB_64B_WQE	BIT(0)</div><div class='add'>+#define IONIC_EXPDB_128B_WQE	BIT(1)</div><div class='add'>+#define IONIC_EXPDB_256B_WQE	BIT(2)</div><div class='add'>+#define IONIC_EXPDB_512B_WQE	BIT(3)</div><div class='add'>+</div><div class='add'>+struct ionic_lif_cfg {</div><div class='add'>+	struct device *hwdev;</div><div class='add'>+	struct ionic_lif *lif;</div><div class='add'>+</div><div class='add'>+	int lif_index;</div><div class='add'>+	int lif_hw_index;</div><div class='add'>+</div><div class='add'>+	u32 dbid;</div><div class='add'>+	int dbid_count;</div><div class='add'>+	u64 __iomem *dbpage;</div><div class='add'>+	struct ionic_intr __iomem *intr_ctrl;</div><div class='add'>+	phys_addr_t db_phys;</div><div class='add'>+</div><div class='add'>+	u64 page_size_supported;</div><div class='add'>+	u32 npts_per_lif;</div><div class='add'>+	u32 nmrs_per_lif;</div><div class='add'>+	u32 nahs_per_lif;</div><div class='add'>+</div><div class='add'>+	u32 aq_base;</div><div class='add'>+	u32 cq_base;</div><div class='add'>+	u32 eq_base;</div><div class='add'>+</div><div class='add'>+	int aq_count;</div><div class='add'>+	int eq_count;</div><div class='add'>+	int cq_count;</div><div class='add'>+	int qp_count;</div><div class='add'>+</div><div class='add'>+	u16 stats_type;</div><div class='add'>+	u8 aq_qtype;</div><div class='add'>+	u8 sq_qtype;</div><div class='add'>+	u8 rq_qtype;</div><div class='add'>+	u8 cq_qtype;</div><div class='add'>+	u8 eq_qtype;</div><div class='add'>+</div><div class='add'>+	u8 udma_count;</div><div class='add'>+	u8 udma_qgrp_shift;</div><div class='add'>+</div><div class='add'>+	u8 rdma_version;</div><div class='add'>+	u8 qp_opcodes;</div><div class='add'>+	u8 admin_opcodes;</div><div class='add'>+</div><div class='add'>+	u8 max_stride;</div><div class='add'>+	bool sq_expdb;</div><div class='add'>+	bool rq_expdb;</div><div class='add'>+	u8 expdb_mask;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg);</div><div class='add'>+struct net_device *ionic_lif_netdev(struct ionic_lif *lif);</div><div class='add'>+void ionic_lif_fw_version(struct ionic_lif *lif, char *str, size_t len);</div><div class='add'>+u8 ionic_lif_asic_rev(struct ionic_lif *lif);</div><div class='add'>+</div><div class='add'>+#endif /* _IONIC_LIF_CFG_H_ */</div><div class='head'>diff --git a/drivers/infiniband/hw/ionic/ionic_pgtbl.c b/drivers/infiniband/hw/ionic/ionic_pgtbl.c<br/>new file mode 100644<br/>index 000000000000..e74db73c9246<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/ionic/ionic_pgtbl.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/ionic/ionic_pgtbl.c</a></div><div class='hunk'>@@ -0,0 +1,143 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0</div><div class='add'>+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/mman.h&gt;</div><div class='add'>+#include &lt;linux/dma-mapping.h&gt;</div><div class='add'>+</div><div class='add'>+#include "ionic_fw.h"</div><div class='add'>+#include "ionic_ibdev.h"</div><div class='add'>+</div><div class='add'>+__le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va)</div><div class='add'>+{</div><div class='add'>+	u64 pg_mask = BIT_ULL(buf-&gt;page_size_log2) - 1;</div><div class='add'>+	u64 dma;</div><div class='add'>+</div><div class='add'>+	if (!buf-&gt;tbl_pages)</div><div class='add'>+		return cpu_to_le64(0);</div><div class='add'>+</div><div class='add'>+	if (buf-&gt;tbl_pages &gt; 1)</div><div class='add'>+		return cpu_to_le64(buf-&gt;tbl_dma);</div><div class='add'>+</div><div class='add'>+	if (buf-&gt;tbl_buf)</div><div class='add'>+		dma = le64_to_cpu(buf-&gt;tbl_buf[0]);</div><div class='add'>+	else</div><div class='add'>+		dma = buf-&gt;tbl_dma;</div><div class='add'>+</div><div class='add'>+	return cpu_to_le64(dma + (va &amp; pg_mask));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+__be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va)</div><div class='add'>+{</div><div class='add'>+	if (buf-&gt;tbl_pages &gt; 1) {</div><div class='add'>+		u64 pg_mask = BIT_ULL(buf-&gt;page_size_log2) - 1;</div><div class='add'>+</div><div class='add'>+		return cpu_to_be64(va &amp; pg_mask);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma)</div><div class='add'>+{</div><div class='add'>+	if (unlikely(buf-&gt;tbl_pages == buf-&gt;tbl_limit))</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	if (buf-&gt;tbl_buf)</div><div class='add'>+		buf-&gt;tbl_buf[buf-&gt;tbl_pages] = cpu_to_le64(dma);</div><div class='add'>+	else</div><div class='add'>+		buf-&gt;tbl_dma = dma;</div><div class='add'>+</div><div class='add'>+	++buf-&gt;tbl_pages;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_tbl_buf_alloc(struct ionic_ibdev *dev,</div><div class='add'>+			       struct ionic_tbl_buf *buf)</div><div class='add'>+{</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	buf-&gt;tbl_size = buf-&gt;tbl_limit * sizeof(*buf-&gt;tbl_buf);</div><div class='add'>+	buf-&gt;tbl_buf = kmalloc(buf-&gt;tbl_size, GFP_KERNEL);</div><div class='add'>+	if (!buf-&gt;tbl_buf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	buf-&gt;tbl_dma = dma_map_single(dev-&gt;lif_cfg.hwdev, buf-&gt;tbl_buf,</div><div class='add'>+				      buf-&gt;tbl_size, DMA_TO_DEVICE);</div><div class='add'>+	rc = dma_mapping_error(dev-&gt;lif_cfg.hwdev, buf-&gt;tbl_dma);</div><div class='add'>+	if (rc) {</div><div class='add'>+		kfree(buf-&gt;tbl_buf);</div><div class='add'>+		return rc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ionic_pgtbl_umem(struct ionic_tbl_buf *buf, struct ib_umem *umem)</div><div class='add'>+{</div><div class='add'>+	struct ib_block_iter biter;</div><div class='add'>+	u64 page_dma;</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	rdma_umem_for_each_dma_block(umem, &amp;biter, BIT_ULL(buf-&gt;page_size_log2)) {</div><div class='add'>+		page_dma = rdma_block_iter_dma_address(&amp;biter);</div><div class='add'>+		rc = ionic_pgtbl_page(buf, page_dma);</div><div class='add'>+		if (rc)</div><div class='add'>+			return rc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf)</div><div class='add'>+{</div><div class='add'>+	if (buf-&gt;tbl_buf)</div><div class='add'>+		dma_unmap_single(dev-&gt;lif_cfg.hwdev, buf-&gt;tbl_dma,</div><div class='add'>+				 buf-&gt;tbl_size, DMA_TO_DEVICE);</div><div class='add'>+</div><div class='add'>+	kfree(buf-&gt;tbl_buf);</div><div class='add'>+	memset(buf, 0, sizeof(*buf));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ionic_pgtbl_init(struct ionic_ibdev *dev,</div><div class='add'>+		     struct ionic_tbl_buf *buf,</div><div class='add'>+		     struct ib_umem *umem,</div><div class='add'>+		     dma_addr_t dma,</div><div class='add'>+		     int limit,</div><div class='add'>+		     u64 page_size)</div><div class='add'>+{</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	memset(buf, 0, sizeof(*buf));</div><div class='add'>+</div><div class='add'>+	if (umem) {</div><div class='add'>+		limit = ib_umem_num_dma_blocks(umem, page_size);</div><div class='add'>+		buf-&gt;page_size_log2 = order_base_2(page_size);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (limit &lt; 1)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	buf-&gt;tbl_limit = limit;</div><div class='add'>+</div><div class='add'>+	/* skip pgtbl if contiguous / direct translation */</div><div class='add'>+	if (limit &gt; 1) {</div><div class='add'>+		rc = ionic_tbl_buf_alloc(dev, buf);</div><div class='add'>+		if (rc)</div><div class='add'>+			return rc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (umem)</div><div class='add'>+		rc = ionic_pgtbl_umem(buf, umem);</div><div class='add'>+	else</div><div class='add'>+		rc = ionic_pgtbl_page(buf, dma);</div><div class='add'>+</div><div class='add'>+	if (rc)</div><div class='add'>+		goto err_unbuf;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_unbuf:</div><div class='add'>+	ionic_pgtbl_unbuf(dev, buf);</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/ionic/ionic_queue.c b/drivers/infiniband/hw/ionic/ionic_queue.c<br/>new file mode 100644<br/>index 000000000000..aa897ed2a412<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/ionic/ionic_queue.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/ionic/ionic_queue.c</a></div><div class='hunk'>@@ -0,0 +1,52 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0</div><div class='add'>+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/dma-mapping.h&gt;</div><div class='add'>+</div><div class='add'>+#include "ionic_queue.h"</div><div class='add'>+</div><div class='add'>+int ionic_queue_init(struct ionic_queue *q, struct device *dma_dev,</div><div class='add'>+		     int depth, size_t stride)</div><div class='add'>+{</div><div class='add'>+	if (depth &lt; 0 || depth &gt; 0xffff)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (stride == 0 || stride &gt; 0x10000)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (depth == 0)</div><div class='add'>+		depth = 1;</div><div class='add'>+</div><div class='add'>+	q-&gt;depth_log2 = order_base_2(depth + 1);</div><div class='add'>+	q-&gt;stride_log2 = order_base_2(stride);</div><div class='add'>+</div><div class='add'>+	if (q-&gt;depth_log2 + q-&gt;stride_log2 &lt; PAGE_SHIFT)</div><div class='add'>+		q-&gt;depth_log2 = PAGE_SHIFT - q-&gt;stride_log2;</div><div class='add'>+</div><div class='add'>+	if (q-&gt;depth_log2 &gt; 16 || q-&gt;stride_log2 &gt; 16)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	q-&gt;size = BIT_ULL(q-&gt;depth_log2 + q-&gt;stride_log2);</div><div class='add'>+	q-&gt;mask = BIT(q-&gt;depth_log2) - 1;</div><div class='add'>+</div><div class='add'>+	q-&gt;ptr = dma_alloc_coherent(dma_dev, q-&gt;size, &amp;q-&gt;dma, GFP_KERNEL);</div><div class='add'>+	if (!q-&gt;ptr)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	/* it will always be page aligned, but just to be sure... */</div><div class='add'>+	if (!PAGE_ALIGNED(q-&gt;ptr)) {</div><div class='add'>+		dma_free_coherent(dma_dev, q-&gt;size, q-&gt;ptr, q-&gt;dma);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	q-&gt;prod = 0;</div><div class='add'>+	q-&gt;cons = 0;</div><div class='add'>+	q-&gt;dbell = 0;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void ionic_queue_destroy(struct ionic_queue *q, struct device *dma_dev)</div><div class='add'>+{</div><div class='add'>+	dma_free_coherent(dma_dev, q-&gt;size, q-&gt;ptr, q-&gt;dma);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/ionic/ionic_queue.h b/drivers/infiniband/hw/ionic/ionic_queue.h<br/>new file mode 100644<br/>index 000000000000..d18020d4cad5<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/ionic/ionic_queue.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/ionic/ionic_queue.h</a></div><div class='hunk'>@@ -0,0 +1,234 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 */</div><div class='add'>+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */</div><div class='add'>+</div><div class='add'>+#ifndef _IONIC_QUEUE_H_</div><div class='add'>+#define _IONIC_QUEUE_H_</div><div class='add'>+</div><div class='add'>+#include &lt;linux/io.h&gt;</div><div class='add'>+#include &lt;ionic_regs.h&gt;</div><div class='add'>+</div><div class='add'>+#define IONIC_MAX_DEPTH		0xffff</div><div class='add'>+#define IONIC_MAX_CQ_DEPTH	0xffff</div><div class='add'>+#define IONIC_CQ_RING_ARM	IONIC_DBELL_RING_1</div><div class='add'>+#define IONIC_CQ_RING_SOL	IONIC_DBELL_RING_2</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * struct ionic_queue - Ring buffer used between device and driver</div><div class='add'>+ * @size:	Size of the buffer, in bytes</div><div class='add'>+ * @dma:	Dma address of the buffer</div><div class='add'>+ * @ptr:	Buffer virtual address</div><div class='add'>+ * @prod:	Driver position in the queue</div><div class='add'>+ * @cons:	Device position in the queue</div><div class='add'>+ * @mask:	Capacity of the queue, subtracting the hole</div><div class='add'>+ *		This value is equal to ((1 &lt;&lt; depth_log2) - 1)</div><div class='add'>+ * @depth_log2: Log base two size depth of the queue</div><div class='add'>+ * @stride_log2: Log base two size of an element in the queue</div><div class='add'>+ * @dbell:	Doorbell identifying bits</div><div class='add'>+ */</div><div class='add'>+struct ionic_queue {</div><div class='add'>+	size_t size;</div><div class='add'>+	dma_addr_t dma;</div><div class='add'>+	void *ptr;</div><div class='add'>+	u16 prod;</div><div class='add'>+	u16 cons;</div><div class='add'>+	u16 mask;</div><div class='add'>+	u8 depth_log2;</div><div class='add'>+	u8 stride_log2;</div><div class='add'>+	u64 dbell;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_queue_init() - Initialize user space queue</div><div class='add'>+ * @q:		Uninitialized queue structure</div><div class='add'>+ * @dma_dev:	DMA device for mapping</div><div class='add'>+ * @depth:	Depth of the queue</div><div class='add'>+ * @stride:	Size of each element of the queue</div><div class='add'>+ *</div><div class='add'>+ * Return: status code</div><div class='add'>+ */</div><div class='add'>+int ionic_queue_init(struct ionic_queue *q, struct device *dma_dev,</div><div class='add'>+		     int depth, size_t stride);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_queue_destroy() - Destroy user space queue</div><div class='add'>+ * @q:		Queue structure</div><div class='add'>+ * @dma_dev:	DMA device for mapping</div><div class='add'>+ *</div><div class='add'>+ * Return: status code</div><div class='add'>+ */</div><div class='add'>+void ionic_queue_destroy(struct ionic_queue *q, struct device *dma_dev);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_queue_empty() - Test if queue is empty</div><div class='add'>+ * @q:		Queue structure</div><div class='add'>+ *</div><div class='add'>+ * This is only valid for to-device queues.</div><div class='add'>+ *</div><div class='add'>+ * Return: is empty</div><div class='add'>+ */</div><div class='add'>+static inline bool ionic_queue_empty(struct ionic_queue *q)</div><div class='add'>+{</div><div class='add'>+	return q-&gt;prod == q-&gt;cons;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_queue_length() - Get the current length of the queue</div><div class='add'>+ * @q:		Queue structure</div><div class='add'>+ *</div><div class='add'>+ * This is only valid for to-device queues.</div><div class='add'>+ *</div><div class='add'>+ * Return: length</div><div class='add'>+ */</div><div class='add'>+static inline u16 ionic_queue_length(struct ionic_queue *q)</div><div class='add'>+{</div><div class='add'>+	return (q-&gt;prod - q-&gt;cons) &amp; q-&gt;mask;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_queue_length_remaining() - Get the remaining length of the queue</div><div class='add'>+ * @q:		Queue structure</div><div class='add'>+ *</div><div class='add'>+ * This is only valid for to-device queues.</div><div class='add'>+ *</div><div class='add'>+ * Return: length remaining</div><div class='add'>+ */</div><div class='add'>+static inline u16 ionic_queue_length_remaining(struct ionic_queue *q)</div><div class='add'>+{</div><div class='add'>+	return q-&gt;mask - ionic_queue_length(q);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_queue_full() - Test if queue is full</div><div class='add'>+ * @q:		Queue structure</div><div class='add'>+ *</div><div class='add'>+ * This is only valid for to-device queues.</div><div class='add'>+ *</div><div class='add'>+ * Return: is full</div><div class='add'>+ */</div><div class='add'>+static inline bool ionic_queue_full(struct ionic_queue *q)</div><div class='add'>+{</div><div class='add'>+	return q-&gt;mask == ionic_queue_length(q);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_color_wrap() - Flip the color if prod is wrapped</div><div class='add'>+ * @prod:	Queue index just after advancing</div><div class='add'>+ * @color:	Queue color just prior to advancing the index</div><div class='add'>+ *</div><div class='add'>+ * Return: color after advancing the index</div><div class='add'>+ */</div><div class='add'>+static inline bool ionic_color_wrap(u16 prod, bool color)</div><div class='add'>+{</div><div class='add'>+	/* logical xor color with (prod == 0) */</div><div class='add'>+	return color != (prod == 0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_queue_at() - Get the element at the given index</div><div class='add'>+ * @q:		Queue structure</div><div class='add'>+ * @idx:	Index in the queue</div><div class='add'>+ *</div><div class='add'>+ * The index must be within the bounds of the queue.  It is not checked here.</div><div class='add'>+ *</div><div class='add'>+ * Return: pointer to element at index</div><div class='add'>+ */</div><div class='add'>+static inline void *ionic_queue_at(struct ionic_queue *q, u16 idx)</div><div class='add'>+{</div><div class='add'>+	return q-&gt;ptr + ((unsigned long)idx &lt;&lt; q-&gt;stride_log2);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_queue_at_prod() - Get the element at the producer index</div><div class='add'>+ * @q:		Queue structure</div><div class='add'>+ *</div><div class='add'>+ * Return: pointer to element at producer index</div><div class='add'>+ */</div><div class='add'>+static inline void *ionic_queue_at_prod(struct ionic_queue *q)</div><div class='add'>+{</div><div class='add'>+	return ionic_queue_at(q, q-&gt;prod);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_queue_at_cons() - Get the element at the consumer index</div><div class='add'>+ * @q:		Queue structure</div><div class='add'>+ *</div><div class='add'>+ * Return: pointer to element at consumer index</div><div class='add'>+ */</div><div class='add'>+static inline void *ionic_queue_at_cons(struct ionic_queue *q)</div><div class='add'>+{</div><div class='add'>+	return ionic_queue_at(q, q-&gt;cons);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_queue_next() - Compute the next index</div><div class='add'>+ * @q:		Queue structure</div><div class='add'>+ * @idx:	Index</div><div class='add'>+ *</div><div class='add'>+ * Return: next index after idx</div><div class='add'>+ */</div><div class='add'>+static inline u16 ionic_queue_next(struct ionic_queue *q, u16 idx)</div><div class='add'>+{</div><div class='add'>+	return (idx + 1) &amp; q-&gt;mask;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_queue_produce() - Increase the producer index</div><div class='add'>+ * @q:		Queue structure</div><div class='add'>+ *</div><div class='add'>+ * Caller must ensure that the queue is not full.  It is not checked here.</div><div class='add'>+ */</div><div class='add'>+static inline void ionic_queue_produce(struct ionic_queue *q)</div><div class='add'>+{</div><div class='add'>+	q-&gt;prod = ionic_queue_next(q, q-&gt;prod);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_queue_consume() - Increase the consumer index</div><div class='add'>+ * @q:		Queue structure</div><div class='add'>+ *</div><div class='add'>+ * Caller must ensure that the queue is not empty.  It is not checked here.</div><div class='add'>+ *</div><div class='add'>+ * This is only valid for to-device queues.</div><div class='add'>+ */</div><div class='add'>+static inline void ionic_queue_consume(struct ionic_queue *q)</div><div class='add'>+{</div><div class='add'>+	q-&gt;cons = ionic_queue_next(q, q-&gt;cons);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_queue_consume_entries() - Increase the consumer index by entries</div><div class='add'>+ * @q:				Queue structure</div><div class='add'>+ * @entries:		Number of entries to increment</div><div class='add'>+ *</div><div class='add'>+ * Caller must ensure that the queue is not empty.  It is not checked here.</div><div class='add'>+ *</div><div class='add'>+ * This is only valid for to-device queues.</div><div class='add'>+ */</div><div class='add'>+static inline void ionic_queue_consume_entries(struct ionic_queue *q,</div><div class='add'>+					       u16 entries)</div><div class='add'>+{</div><div class='add'>+	q-&gt;cons = (q-&gt;cons + entries) &amp; q-&gt;mask;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_queue_dbell_init() - Initialize doorbell bits for queue id</div><div class='add'>+ * @q:		Queue structure</div><div class='add'>+ * @qid:	Queue identifying number</div><div class='add'>+ */</div><div class='add'>+static inline void ionic_queue_dbell_init(struct ionic_queue *q, u32 qid)</div><div class='add'>+{</div><div class='add'>+	q-&gt;dbell = IONIC_DBELL_QID(qid);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_queue_dbell_val() - Get current doorbell update value</div><div class='add'>+ * @q:		Queue structure</div><div class='add'>+ *</div><div class='add'>+ * Return: current doorbell update value</div><div class='add'>+ */</div><div class='add'>+static inline u64 ionic_queue_dbell_val(struct ionic_queue *q)</div><div class='add'>+{</div><div class='add'>+	return q-&gt;dbell | q-&gt;prod;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#endif /* _IONIC_QUEUE_H_ */</div><div class='head'>diff --git a/drivers/infiniband/hw/ionic/ionic_res.h b/drivers/infiniband/hw/ionic/ionic_res.h<br/>new file mode 100644<br/>index 000000000000..46c8c584bd9a<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/ionic/ionic_res.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/ionic/ionic_res.h</a></div><div class='hunk'>@@ -0,0 +1,154 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 */</div><div class='add'>+/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */</div><div class='add'>+</div><div class='add'>+#ifndef _IONIC_RES_H_</div><div class='add'>+#define _IONIC_RES_H_</div><div class='add'>+</div><div class='add'>+#include &lt;linux/kernel.h&gt;</div><div class='add'>+#include &lt;linux/idr.h&gt;</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * struct ionic_resid_bits - Number allocator based on IDA</div><div class='add'>+ *</div><div class='add'>+ * @inuse:      IDA handle</div><div class='add'>+ * @inuse_size: Highest ID limit for IDA</div><div class='add'>+ */</div><div class='add'>+struct ionic_resid_bits {</div><div class='add'>+	struct ida inuse;</div><div class='add'>+	unsigned int inuse_size;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_resid_init() - Initialize a resid allocator</div><div class='add'>+ * @resid:  Uninitialized resid allocator</div><div class='add'>+ * @size:   Capacity of the allocator</div><div class='add'>+ *</div><div class='add'>+ * Return: Zero on success, or negative error number</div><div class='add'>+ */</div><div class='add'>+static inline void ionic_resid_init(struct ionic_resid_bits *resid,</div><div class='add'>+				    unsigned int size)</div><div class='add'>+{</div><div class='add'>+	resid-&gt;inuse_size = size;</div><div class='add'>+	ida_init(&amp;resid-&gt;inuse);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_resid_destroy() - Destroy a resid allocator</div><div class='add'>+ * @resid:  Resid allocator</div><div class='add'>+ */</div><div class='add'>+static inline void ionic_resid_destroy(struct ionic_resid_bits *resid)</div><div class='add'>+{</div><div class='add'>+	ida_destroy(&amp;resid-&gt;inuse);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_resid_get_shared() - Allocate an available shared resource id</div><div class='add'>+ * @resid:   Resid allocator</div><div class='add'>+ * @min:     Smallest valid resource id</div><div class='add'>+ * @size:    One after largest valid resource id</div><div class='add'>+ *</div><div class='add'>+ * Return: Resource id, or negative error number</div><div class='add'>+ */</div><div class='add'>+static inline int ionic_resid_get_shared(struct ionic_resid_bits *resid,</div><div class='add'>+					 unsigned int min,</div><div class='add'>+					 unsigned int size)</div><div class='add'>+{</div><div class='add'>+	return ida_alloc_range(&amp;resid-&gt;inuse, min, size - 1, GFP_KERNEL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_resid_get() - Allocate an available resource id</div><div class='add'>+ * @resid: Resid allocator</div><div class='add'>+ *</div><div class='add'>+ * Return: Resource id, or negative error number</div><div class='add'>+ */</div><div class='add'>+static inline int ionic_resid_get(struct ionic_resid_bits *resid)</div><div class='add'>+{</div><div class='add'>+	return ionic_resid_get_shared(resid, 0, resid-&gt;inuse_size);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_resid_put() - Free a resource id</div><div class='add'>+ * @resid:  Resid allocator</div><div class='add'>+ * @id:     Resource id</div><div class='add'>+ */</div><div class='add'>+static inline void ionic_resid_put(struct ionic_resid_bits *resid, int id)</div><div class='add'>+{</div><div class='add'>+	ida_free(&amp;resid-&gt;inuse, id);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_bitid_to_qid() - Transform a resource bit index into a queue id</div><div class='add'>+ * @bitid:           Bit index</div><div class='add'>+ * @qgrp_shift:      Log2 number of queues per queue group</div><div class='add'>+ * @half_qid_shift:  Log2 of half the total number of queues</div><div class='add'>+ *</div><div class='add'>+ * Return: Queue id</div><div class='add'>+ *</div><div class='add'>+ * Udma-constrained queues (QPs and CQs) are associated with their udma by</div><div class='add'>+ * queue group. Even queue groups are associated with udma0, and odd queue</div><div class='add'>+ * groups with udma1.</div><div class='add'>+ *</div><div class='add'>+ * For allocating queue ids, we want to arrange the bits into two halves,</div><div class='add'>+ * with the even queue groups of udma0 in the lower half of the bitset,</div><div class='add'>+ * and the odd queue groups of udma1 in the upper half of the bitset.</div><div class='add'>+ * Then, one or two calls of find_next_zero_bit can examine all the bits</div><div class='add'>+ * for queues of an entire udma.</div><div class='add'>+ *</div><div class='add'>+ * For example, assuming eight queue groups with qgrp qids per group:</div><div class='add'>+ *</div><div class='add'>+ * bitid 0*qgrp..1*qgrp-1 : qid 0*qgrp..1*qgrp-1</div><div class='add'>+ * bitid 1*qgrp..2*qgrp-1 : qid 2*qgrp..3*qgrp-1</div><div class='add'>+ * bitid 2*qgrp..3*qgrp-1 : qid 4*qgrp..5*qgrp-1</div><div class='add'>+ * bitid 3*qgrp..4*qgrp-1 : qid 6*qgrp..7*qgrp-1</div><div class='add'>+ * bitid 4*qgrp..5*qgrp-1 : qid 1*qgrp..2*qgrp-1</div><div class='add'>+ * bitid 5*qgrp..6*qgrp-1 : qid 3*qgrp..4*qgrp-1</div><div class='add'>+ * bitid 6*qgrp..7*qgrp-1 : qid 5*qgrp..6*qgrp-1</div><div class='add'>+ * bitid 7*qgrp..8*qgrp-1 : qid 7*qgrp..8*qgrp-1</div><div class='add'>+ *</div><div class='add'>+ * There are three important ranges of bits in the qid.  There is the udma</div><div class='add'>+ * bit "U" at qgrp_shift, which is the least significant bit of the group</div><div class='add'>+ * index, and determines which udma a queue is associated with.</div><div class='add'>+ * The bits of lesser significance we can call the idx bits "I", which are</div><div class='add'>+ * the index of the queue within the group.  The bits of greater significance</div><div class='add'>+ * we can call the grp bits "G", which are other bits of the group index that</div><div class='add'>+ * do not determine the udma.  Those bits are just rearranged in the bit index</div><div class='add'>+ * in the bitset.  A bitid has the udma bit in the most significant place,</div><div class='add'>+ * then the grp bits, then the idx bits.</div><div class='add'>+ *</div><div class='add'>+ * bitid: 00000000000000 U GGG IIIIII</div><div class='add'>+ * qid:   00000000000000 GGG U IIIIII</div><div class='add'>+ *</div><div class='add'>+ * Transforming from bit index to qid, or from qid to bit index, can be</div><div class='add'>+ * accomplished by rearranging the bits by masking and shifting.</div><div class='add'>+ */</div><div class='add'>+static inline u32 ionic_bitid_to_qid(u32 bitid, u8 qgrp_shift,</div><div class='add'>+				     u8 half_qid_shift)</div><div class='add'>+{</div><div class='add'>+	u32 udma_bit =</div><div class='add'>+		(bitid &amp; BIT(half_qid_shift)) &gt;&gt; (half_qid_shift - qgrp_shift);</div><div class='add'>+	u32 grp_bits = (bitid &amp; GENMASK(half_qid_shift - 1, qgrp_shift)) &lt;&lt; 1;</div><div class='add'>+	u32 idx_bits = bitid &amp; (BIT(qgrp_shift) - 1);</div><div class='add'>+</div><div class='add'>+	return grp_bits | udma_bit | idx_bits;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ionic_qid_to_bitid() - Transform a queue id into a resource bit index</div><div class='add'>+ * @qid:            queue index</div><div class='add'>+ * @qgrp_shift:     Log2 number of queues per queue group</div><div class='add'>+ * @half_qid_shift: Log2 of half the total number of queues</div><div class='add'>+ *</div><div class='add'>+ * Return: Resource bit index</div><div class='add'>+ *</div><div class='add'>+ * This is the inverse of ionic_bitid_to_qid().</div><div class='add'>+ */</div><div class='add'>+static inline u32 ionic_qid_to_bitid(u32 qid, u8 qgrp_shift, u8 half_qid_shift)</div><div class='add'>+{</div><div class='add'>+	u32 udma_bit = (qid &amp; BIT(qgrp_shift)) &lt;&lt; (half_qid_shift - qgrp_shift);</div><div class='add'>+	u32 grp_bits = (qid &amp; GENMASK(half_qid_shift, qgrp_shift + 1)) &gt;&gt; 1;</div><div class='add'>+	u32 idx_bits = qid &amp; (BIT(qgrp_shift) - 1);</div><div class='add'>+</div><div class='add'>+	return udma_bit | grp_bits | idx_bits;</div><div class='add'>+}</div><div class='add'>+#endif /* _IONIC_RES_H_ */</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/Kconfig b/drivers/infiniband/hw/irdma/Kconfig<br/>new file mode 100644<br/>index 000000000000..0bd7e3fca1fb<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/Kconfig?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/Kconfig</a></div><div class='hunk'>@@ -0,0 +1,14 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+config INFINIBAND_IRDMA</div><div class='add'>+	tristate "Intel(R) Ethernet Protocol Driver for RDMA"</div><div class='add'>+	depends on INET</div><div class='add'>+	depends on IPV6 || !IPV6</div><div class='add'>+	depends on PCI</div><div class='add'>+	depends on IDPF &amp;&amp; ICE &amp;&amp; I40E</div><div class='add'>+	select GENERIC_ALLOCATOR</div><div class='add'>+	select AUXILIARY_BUS</div><div class='add'>+	select CRC32</div><div class='add'>+	help</div><div class='add'>+	  This is an Intel(R) Ethernet Protocol Driver for RDMA that</div><div class='add'>+	  supports IPU E2000 (RoCEv2), E810 (iWARP/RoCEv2) and X722 (iWARP)</div><div class='add'>+	  network devices.</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/Makefile b/drivers/infiniband/hw/irdma/Makefile<br/>new file mode 100644<br/>index 000000000000..03ceb9e5475f<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/Makefile?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/Makefile</a></div><div class='hunk'>@@ -0,0 +1,31 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+# Copyright (c) 2019, Intel Corporation.</div><div class='add'>+</div><div class='add'>+#</div><div class='add'>+# Makefile for the Intel(R) Ethernet Connection RDMA Linux Driver</div><div class='add'>+#</div><div class='add'>+</div><div class='add'>+obj-$(CONFIG_INFINIBAND_IRDMA) += irdma.o</div><div class='add'>+</div><div class='add'>+irdma-objs := cm.o        \</div><div class='add'>+              ctrl.o      \</div><div class='add'>+              hmc.o       \</div><div class='add'>+              hw.o        \</div><div class='add'>+              i40iw_hw.o  \</div><div class='add'>+              i40iw_if.o  \</div><div class='add'>+	      ig3rdma_if.o\</div><div class='add'>+	      icrdma_if.o \</div><div class='add'>+              icrdma_hw.o \</div><div class='add'>+	      ig3rdma_hw.o\</div><div class='add'>+              main.o      \</div><div class='add'>+              pble.o      \</div><div class='add'>+              puda.o      \</div><div class='add'>+              trace.o     \</div><div class='add'>+              uda.o       \</div><div class='add'>+              uk.o        \</div><div class='add'>+              utils.o     \</div><div class='add'>+              verbs.o     \</div><div class='add'>+	      virtchnl.o  \</div><div class='add'>+              ws.o        \</div><div class='add'>+</div><div class='add'>+CFLAGS_trace.o = -I$(src)</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/cm.c b/drivers/infiniband/hw/irdma/cm.c<br/>new file mode 100644<br/>index 000000000000..f4f4f92ba63a<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/cm.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/cm.c</a></div><div class='hunk'>@@ -0,0 +1,4434 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#include "main.h"</div><div class='add'>+#include "trace.h"</div><div class='add'>+</div><div class='add'>+static void irdma_cm_post_event(struct irdma_cm_event *event);</div><div class='add'>+static void irdma_disconnect_worker(struct work_struct *work);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_free_sqbuf - put back puda buffer if refcount is 0</div><div class='add'>+ * @vsi: The VSI structure of the device</div><div class='add'>+ * @bufp: puda buffer to free</div><div class='add'>+ */</div><div class='add'>+void irdma_free_sqbuf(struct irdma_sc_vsi *vsi, void *bufp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_buf *buf = bufp;</div><div class='add'>+	struct irdma_puda_rsrc *ilq = vsi-&gt;ilq;</div><div class='add'>+</div><div class='add'>+	if (refcount_dec_and_test(&amp;buf-&gt;refcount))</div><div class='add'>+		irdma_puda_ret_bufpool(ilq, buf);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_record_ird_ord - Record IRD/ORD passed in</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @conn_ird: connection IRD</div><div class='add'>+ * @conn_ord: connection ORD</div><div class='add'>+ */</div><div class='add'>+static void irdma_record_ird_ord(struct irdma_cm_node *cm_node, u32 conn_ird,</div><div class='add'>+				 u32 conn_ord)</div><div class='add'>+{</div><div class='add'>+	if (conn_ird &gt; cm_node-&gt;dev-&gt;hw_attrs.max_hw_ird)</div><div class='add'>+		conn_ird = cm_node-&gt;dev-&gt;hw_attrs.max_hw_ird;</div><div class='add'>+</div><div class='add'>+	if (conn_ord &gt; cm_node-&gt;dev-&gt;hw_attrs.max_hw_ord)</div><div class='add'>+		conn_ord = cm_node-&gt;dev-&gt;hw_attrs.max_hw_ord;</div><div class='add'>+	else if (!conn_ord &amp;&amp; cm_node-&gt;send_rdma0_op == SEND_RDMA_READ_ZERO)</div><div class='add'>+		conn_ord = 1;</div><div class='add'>+	cm_node-&gt;ird_size = conn_ird;</div><div class='add'>+	cm_node-&gt;ord_size = conn_ord;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_copy_ip_ntohl - copy IP address from  network to host</div><div class='add'>+ * @dst: IP address in host order</div><div class='add'>+ * @src: IP address in network order (big endian)</div><div class='add'>+ */</div><div class='add'>+void irdma_copy_ip_ntohl(u32 *dst, __be32 *src)</div><div class='add'>+{</div><div class='add'>+	*dst++ = ntohl(*src++);</div><div class='add'>+	*dst++ = ntohl(*src++);</div><div class='add'>+	*dst++ = ntohl(*src++);</div><div class='add'>+	*dst = ntohl(*src);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_copy_ip_htonl - copy IP address from host to network order</div><div class='add'>+ * @dst: IP address in network order (big endian)</div><div class='add'>+ * @src: IP address in host order</div><div class='add'>+ */</div><div class='add'>+void irdma_copy_ip_htonl(__be32 *dst, u32 *src)</div><div class='add'>+{</div><div class='add'>+	*dst++ = htonl(*src++);</div><div class='add'>+	*dst++ = htonl(*src++);</div><div class='add'>+	*dst++ = htonl(*src++);</div><div class='add'>+	*dst = htonl(*src);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_addr_info</div><div class='add'>+ * @cm_node: contains ip/tcp info</div><div class='add'>+ * @cm_info: to get a copy of the cm_node ip/tcp info</div><div class='add'>+ */</div><div class='add'>+static void irdma_get_addr_info(struct irdma_cm_node *cm_node,</div><div class='add'>+				struct irdma_cm_info *cm_info)</div><div class='add'>+{</div><div class='add'>+	memset(cm_info, 0, sizeof(*cm_info));</div><div class='add'>+	cm_info-&gt;ipv4 = cm_node-&gt;ipv4;</div><div class='add'>+	cm_info-&gt;vlan_id = cm_node-&gt;vlan_id;</div><div class='add'>+	memcpy(cm_info-&gt;loc_addr, cm_node-&gt;loc_addr, sizeof(cm_info-&gt;loc_addr));</div><div class='add'>+	memcpy(cm_info-&gt;rem_addr, cm_node-&gt;rem_addr, sizeof(cm_info-&gt;rem_addr));</div><div class='add'>+	cm_info-&gt;loc_port = cm_node-&gt;loc_port;</div><div class='add'>+	cm_info-&gt;rem_port = cm_node-&gt;rem_port;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_fill_sockaddr4 - fill in addr info for IPv4 connection</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @event: upper layer's cm event</div><div class='add'>+ */</div><div class='add'>+static inline void irdma_fill_sockaddr4(struct irdma_cm_node *cm_node,</div><div class='add'>+					struct iw_cm_event *event)</div><div class='add'>+{</div><div class='add'>+	struct sockaddr_in *laddr = (struct sockaddr_in *)&amp;event-&gt;local_addr;</div><div class='add'>+	struct sockaddr_in *raddr = (struct sockaddr_in *)&amp;event-&gt;remote_addr;</div><div class='add'>+</div><div class='add'>+	laddr-&gt;sin_family = AF_INET;</div><div class='add'>+	raddr-&gt;sin_family = AF_INET;</div><div class='add'>+</div><div class='add'>+	laddr-&gt;sin_port = htons(cm_node-&gt;loc_port);</div><div class='add'>+	raddr-&gt;sin_port = htons(cm_node-&gt;rem_port);</div><div class='add'>+</div><div class='add'>+	laddr-&gt;sin_addr.s_addr = htonl(cm_node-&gt;loc_addr[0]);</div><div class='add'>+	raddr-&gt;sin_addr.s_addr = htonl(cm_node-&gt;rem_addr[0]);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_fill_sockaddr6 - fill in addr info for IPv6 connection</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @event: upper layer's cm event</div><div class='add'>+ */</div><div class='add'>+static inline void irdma_fill_sockaddr6(struct irdma_cm_node *cm_node,</div><div class='add'>+					struct iw_cm_event *event)</div><div class='add'>+{</div><div class='add'>+	struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)&amp;event-&gt;local_addr;</div><div class='add'>+	struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *)&amp;event-&gt;remote_addr;</div><div class='add'>+</div><div class='add'>+	laddr6-&gt;sin6_family = AF_INET6;</div><div class='add'>+	raddr6-&gt;sin6_family = AF_INET6;</div><div class='add'>+</div><div class='add'>+	laddr6-&gt;sin6_port = htons(cm_node-&gt;loc_port);</div><div class='add'>+	raddr6-&gt;sin6_port = htons(cm_node-&gt;rem_port);</div><div class='add'>+</div><div class='add'>+	irdma_copy_ip_htonl(laddr6-&gt;sin6_addr.in6_u.u6_addr32,</div><div class='add'>+			    cm_node-&gt;loc_addr);</div><div class='add'>+	irdma_copy_ip_htonl(raddr6-&gt;sin6_addr.in6_u.u6_addr32,</div><div class='add'>+			    cm_node-&gt;rem_addr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_cmevent_info - for cm event upcall</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @cm_id: upper layers cm struct for the event</div><div class='add'>+ * @event: upper layer's cm event</div><div class='add'>+ */</div><div class='add'>+static inline void irdma_get_cmevent_info(struct irdma_cm_node *cm_node,</div><div class='add'>+					  struct iw_cm_id *cm_id,</div><div class='add'>+					  struct iw_cm_event *event)</div><div class='add'>+{</div><div class='add'>+	memcpy(&amp;event-&gt;local_addr, &amp;cm_id-&gt;m_local_addr,</div><div class='add'>+	       sizeof(event-&gt;local_addr));</div><div class='add'>+	memcpy(&amp;event-&gt;remote_addr, &amp;cm_id-&gt;m_remote_addr,</div><div class='add'>+	       sizeof(event-&gt;remote_addr));</div><div class='add'>+	if (cm_node) {</div><div class='add'>+		event-&gt;private_data = cm_node-&gt;pdata_buf;</div><div class='add'>+		event-&gt;private_data_len = (u8)cm_node-&gt;pdata.size;</div><div class='add'>+		event-&gt;ird = cm_node-&gt;ird_size;</div><div class='add'>+		event-&gt;ord = cm_node-&gt;ord_size;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_send_cm_event - upcall cm's event handler</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @cm_id: upper layer's cm info struct</div><div class='add'>+ * @type: Event type to indicate</div><div class='add'>+ * @status: status for the event type</div><div class='add'>+ */</div><div class='add'>+static int irdma_send_cm_event(struct irdma_cm_node *cm_node,</div><div class='add'>+			       struct iw_cm_id *cm_id,</div><div class='add'>+			       enum iw_cm_event_type type, int status)</div><div class='add'>+{</div><div class='add'>+	struct iw_cm_event event = {};</div><div class='add'>+</div><div class='add'>+	event.event = type;</div><div class='add'>+	event.status = status;</div><div class='add'>+	trace_irdma_send_cm_event(cm_node, cm_id, type, status,</div><div class='add'>+				  __builtin_return_address(0));</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+		  "CM: cm_node %p cm_id=%p state=%d accel=%d event_type=%d status=%d\n",</div><div class='add'>+		  cm_node, cm_id, cm_node-&gt;accelerated, cm_node-&gt;state, type,</div><div class='add'>+		  status);</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case IW_CM_EVENT_CONNECT_REQUEST:</div><div class='add'>+		if (cm_node-&gt;ipv4)</div><div class='add'>+			irdma_fill_sockaddr4(cm_node, &amp;event);</div><div class='add'>+		else</div><div class='add'>+			irdma_fill_sockaddr6(cm_node, &amp;event);</div><div class='add'>+		event.provider_data = cm_node;</div><div class='add'>+		event.private_data = cm_node-&gt;pdata_buf;</div><div class='add'>+		event.private_data_len = (u8)cm_node-&gt;pdata.size;</div><div class='add'>+		event.ird = cm_node-&gt;ird_size;</div><div class='add'>+		break;</div><div class='add'>+	case IW_CM_EVENT_CONNECT_REPLY:</div><div class='add'>+		irdma_get_cmevent_info(cm_node, cm_id, &amp;event);</div><div class='add'>+		break;</div><div class='add'>+	case IW_CM_EVENT_ESTABLISHED:</div><div class='add'>+		event.ird = cm_node-&gt;ird_size;</div><div class='add'>+		event.ord = cm_node-&gt;ord_size;</div><div class='add'>+		break;</div><div class='add'>+	case IW_CM_EVENT_DISCONNECT:</div><div class='add'>+	case IW_CM_EVENT_CLOSE:</div><div class='add'>+		/* Wait if we are in RTS but havent issued the iwcm event upcall */</div><div class='add'>+		if (!cm_node-&gt;accelerated)</div><div class='add'>+			wait_for_completion(&amp;cm_node-&gt;establish_comp);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return cm_id-&gt;event_handler(cm_id, &amp;event);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_timer_list_prep - add connection nodes to a list to perform timer tasks</div><div class='add'>+ * @cm_core: cm's core</div><div class='add'>+ * @timer_list: a timer list to which cm_node will be selected</div><div class='add'>+ */</div><div class='add'>+static void irdma_timer_list_prep(struct irdma_cm_core *cm_core,</div><div class='add'>+				  struct list_head *timer_list)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+	int bkt;</div><div class='add'>+</div><div class='add'>+	hash_for_each_rcu(cm_core-&gt;cm_hash_tbl, bkt, cm_node, list) {</div><div class='add'>+		if ((cm_node-&gt;close_entry || cm_node-&gt;send_entry) &amp;&amp;</div><div class='add'>+		    refcount_inc_not_zero(&amp;cm_node-&gt;refcnt))</div><div class='add'>+			list_add(&amp;cm_node-&gt;timer_entry, timer_list);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_create_event - create cm event</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @type: Event type to generate</div><div class='add'>+ */</div><div class='add'>+static struct irdma_cm_event *irdma_create_event(struct irdma_cm_node *cm_node,</div><div class='add'>+						 enum irdma_cm_event_type type)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_event *event;</div><div class='add'>+</div><div class='add'>+	if (!cm_node-&gt;cm_id)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	event = kzalloc(sizeof(*event), GFP_ATOMIC);</div><div class='add'>+</div><div class='add'>+	if (!event)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	event-&gt;type = type;</div><div class='add'>+	event-&gt;cm_node = cm_node;</div><div class='add'>+	memcpy(event-&gt;cm_info.rem_addr, cm_node-&gt;rem_addr,</div><div class='add'>+	       sizeof(event-&gt;cm_info.rem_addr));</div><div class='add'>+	memcpy(event-&gt;cm_info.loc_addr, cm_node-&gt;loc_addr,</div><div class='add'>+	       sizeof(event-&gt;cm_info.loc_addr));</div><div class='add'>+	event-&gt;cm_info.rem_port = cm_node-&gt;rem_port;</div><div class='add'>+	event-&gt;cm_info.loc_port = cm_node-&gt;loc_port;</div><div class='add'>+	event-&gt;cm_info.cm_id = cm_node-&gt;cm_id;</div><div class='add'>+	ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+		  "CM: node=%p event=%p type=%u dst=%pI4 src=%pI4\n", cm_node,</div><div class='add'>+		  event, type, event-&gt;cm_info.loc_addr,</div><div class='add'>+		  event-&gt;cm_info.rem_addr);</div><div class='add'>+	trace_irdma_create_event(cm_node, type, __builtin_return_address(0));</div><div class='add'>+	irdma_cm_post_event(event);</div><div class='add'>+</div><div class='add'>+	return event;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_free_retrans_entry - free send entry</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ */</div><div class='add'>+static void irdma_free_retrans_entry(struct irdma_cm_node *cm_node)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = cm_node-&gt;iwdev;</div><div class='add'>+	struct irdma_timer_entry *send_entry;</div><div class='add'>+</div><div class='add'>+	send_entry = cm_node-&gt;send_entry;</div><div class='add'>+	if (!send_entry)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;send_entry = NULL;</div><div class='add'>+	irdma_free_sqbuf(&amp;iwdev-&gt;vsi, send_entry-&gt;sqbuf);</div><div class='add'>+	kfree(send_entry);</div><div class='add'>+	refcount_dec(&amp;cm_node-&gt;refcnt);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cleanup_retrans_entry - free send entry with lock</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ */</div><div class='add'>+static void irdma_cleanup_retrans_entry(struct irdma_cm_node *cm_node)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='add'>+	irdma_free_retrans_entry(cm_node);</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_form_ah_cm_frame - get a free packet and build frame with address handle</div><div class='add'>+ * @cm_node: connection's node ionfo to use in frame</div><div class='add'>+ * @options: pointer to options info</div><div class='add'>+ * @hdr: pointer mpa header</div><div class='add'>+ * @pdata: pointer to private data</div><div class='add'>+ * @flags:  indicates FIN or ACK</div><div class='add'>+ */</div><div class='add'>+static struct irdma_puda_buf *irdma_form_ah_cm_frame(struct irdma_cm_node *cm_node,</div><div class='add'>+						     struct irdma_kmem_info *options,</div><div class='add'>+						     struct irdma_kmem_info *hdr,</div><div class='add'>+						     struct irdma_mpa_priv_info *pdata,</div><div class='add'>+						     u8 flags)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_buf *sqbuf;</div><div class='add'>+	struct irdma_sc_vsi *vsi = &amp;cm_node-&gt;iwdev-&gt;vsi;</div><div class='add'>+	u8 *buf;</div><div class='add'>+	struct tcphdr *tcph;</div><div class='add'>+	u16 pktsize;</div><div class='add'>+	u32 opts_len = 0;</div><div class='add'>+	u32 pd_len = 0;</div><div class='add'>+	u32 hdr_len = 0;</div><div class='add'>+</div><div class='add'>+	if (!cm_node-&gt;ah || !cm_node-&gt;ah-&gt;ah_info.ah_valid) {</div><div class='add'>+		ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev, "CM: AH invalid\n");</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	sqbuf = irdma_puda_get_bufpool(vsi-&gt;ilq);</div><div class='add'>+	if (!sqbuf) {</div><div class='add'>+		ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev, "CM: SQ buf NULL\n");</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	sqbuf-&gt;ah_id = cm_node-&gt;ah-&gt;ah_info.ah_idx;</div><div class='add'>+	buf = sqbuf-&gt;mem.va;</div><div class='add'>+	if (options)</div><div class='add'>+		opts_len = (u32)options-&gt;size;</div><div class='add'>+</div><div class='add'>+	if (hdr)</div><div class='add'>+		hdr_len = hdr-&gt;size;</div><div class='add'>+</div><div class='add'>+	if (pdata)</div><div class='add'>+		pd_len = pdata-&gt;size;</div><div class='add'>+</div><div class='add'>+	pktsize = sizeof(*tcph) + opts_len + hdr_len + pd_len;</div><div class='add'>+</div><div class='add'>+	memset(buf, 0, sizeof(*tcph));</div><div class='add'>+</div><div class='add'>+	sqbuf-&gt;totallen = pktsize;</div><div class='add'>+	sqbuf-&gt;tcphlen = sizeof(*tcph) + opts_len;</div><div class='add'>+	sqbuf-&gt;scratch = cm_node;</div><div class='add'>+</div><div class='add'>+	tcph = (struct tcphdr *)buf;</div><div class='add'>+	buf += sizeof(*tcph);</div><div class='add'>+</div><div class='add'>+	tcph-&gt;source = htons(cm_node-&gt;loc_port);</div><div class='add'>+	tcph-&gt;dest = htons(cm_node-&gt;rem_port);</div><div class='add'>+	tcph-&gt;seq = htonl(cm_node-&gt;tcp_cntxt.loc_seq_num);</div><div class='add'>+</div><div class='add'>+	if (flags &amp; SET_ACK) {</div><div class='add'>+		cm_node-&gt;tcp_cntxt.loc_ack_num = cm_node-&gt;tcp_cntxt.rcv_nxt;</div><div class='add'>+		tcph-&gt;ack_seq = htonl(cm_node-&gt;tcp_cntxt.loc_ack_num);</div><div class='add'>+		tcph-&gt;ack = 1;</div><div class='add'>+	} else {</div><div class='add'>+		tcph-&gt;ack_seq = 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (flags &amp; SET_SYN) {</div><div class='add'>+		cm_node-&gt;tcp_cntxt.loc_seq_num++;</div><div class='add'>+		tcph-&gt;syn = 1;</div><div class='add'>+	} else {</div><div class='add'>+		cm_node-&gt;tcp_cntxt.loc_seq_num += hdr_len + pd_len;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (flags &amp; SET_FIN) {</div><div class='add'>+		cm_node-&gt;tcp_cntxt.loc_seq_num++;</div><div class='add'>+		tcph-&gt;fin = 1;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (flags &amp; SET_RST)</div><div class='add'>+		tcph-&gt;rst = 1;</div><div class='add'>+</div><div class='add'>+	tcph-&gt;doff = (u16)((sizeof(*tcph) + opts_len + 3) &gt;&gt; 2);</div><div class='add'>+	sqbuf-&gt;tcphlen = tcph-&gt;doff &lt;&lt; 2;</div><div class='add'>+	tcph-&gt;window = htons(cm_node-&gt;tcp_cntxt.rcv_wnd);</div><div class='add'>+	tcph-&gt;urg_ptr = 0;</div><div class='add'>+</div><div class='add'>+	if (opts_len) {</div><div class='add'>+		memcpy(buf, options-&gt;addr, opts_len);</div><div class='add'>+		buf += opts_len;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (hdr_len) {</div><div class='add'>+		memcpy(buf, hdr-&gt;addr, hdr_len);</div><div class='add'>+		buf += hdr_len;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (pdata &amp;&amp; pdata-&gt;addr)</div><div class='add'>+		memcpy(buf, pdata-&gt;addr, pdata-&gt;size);</div><div class='add'>+</div><div class='add'>+	refcount_set(&amp;sqbuf-&gt;refcount, 1);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("ILQ: TRANSMIT ILQ BUFFER", DUMP_PREFIX_OFFSET,</div><div class='add'>+			     16, 8, sqbuf-&gt;mem.va, sqbuf-&gt;totallen, false);</div><div class='add'>+</div><div class='add'>+	return sqbuf;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_form_uda_cm_frame - get a free packet and build frame full tcpip packet</div><div class='add'>+ * @cm_node: connection's node ionfo to use in frame</div><div class='add'>+ * @options: pointer to options info</div><div class='add'>+ * @hdr: pointer mpa header</div><div class='add'>+ * @pdata: pointer to private data</div><div class='add'>+ * @flags:  indicates FIN or ACK</div><div class='add'>+ */</div><div class='add'>+static struct irdma_puda_buf *irdma_form_uda_cm_frame(struct irdma_cm_node *cm_node,</div><div class='add'>+						      struct irdma_kmem_info *options,</div><div class='add'>+						      struct irdma_kmem_info *hdr,</div><div class='add'>+						      struct irdma_mpa_priv_info *pdata,</div><div class='add'>+						      u8 flags)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_buf *sqbuf;</div><div class='add'>+	struct irdma_sc_vsi *vsi = &amp;cm_node-&gt;iwdev-&gt;vsi;</div><div class='add'>+	u8 *buf;</div><div class='add'>+</div><div class='add'>+	struct tcphdr *tcph;</div><div class='add'>+	struct iphdr *iph;</div><div class='add'>+	struct ipv6hdr *ip6h;</div><div class='add'>+	struct ethhdr *ethh;</div><div class='add'>+	u16 pktsize;</div><div class='add'>+	u16 eth_hlen = ETH_HLEN;</div><div class='add'>+	u32 opts_len = 0;</div><div class='add'>+	u32 pd_len = 0;</div><div class='add'>+	u32 hdr_len = 0;</div><div class='add'>+</div><div class='add'>+	u16 vtag;</div><div class='add'>+</div><div class='add'>+	sqbuf = irdma_puda_get_bufpool(vsi-&gt;ilq);</div><div class='add'>+	if (!sqbuf)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	buf = sqbuf-&gt;mem.va;</div><div class='add'>+</div><div class='add'>+	if (options)</div><div class='add'>+		opts_len = (u32)options-&gt;size;</div><div class='add'>+</div><div class='add'>+	if (hdr)</div><div class='add'>+		hdr_len = hdr-&gt;size;</div><div class='add'>+</div><div class='add'>+	if (pdata)</div><div class='add'>+		pd_len = pdata-&gt;size;</div><div class='add'>+</div><div class='add'>+	if (cm_node-&gt;vlan_id &lt; VLAN_N_VID)</div><div class='add'>+		eth_hlen += 4;</div><div class='add'>+</div><div class='add'>+	if (cm_node-&gt;ipv4)</div><div class='add'>+		pktsize = sizeof(*iph) + sizeof(*tcph);</div><div class='add'>+	else</div><div class='add'>+		pktsize = sizeof(*ip6h) + sizeof(*tcph);</div><div class='add'>+	pktsize += opts_len + hdr_len + pd_len;</div><div class='add'>+</div><div class='add'>+	memset(buf, 0, eth_hlen + pktsize);</div><div class='add'>+</div><div class='add'>+	sqbuf-&gt;totallen = pktsize + eth_hlen;</div><div class='add'>+	sqbuf-&gt;maclen = eth_hlen;</div><div class='add'>+	sqbuf-&gt;tcphlen = sizeof(*tcph) + opts_len;</div><div class='add'>+	sqbuf-&gt;scratch = cm_node;</div><div class='add'>+</div><div class='add'>+	ethh = (struct ethhdr *)buf;</div><div class='add'>+	buf += eth_hlen;</div><div class='add'>+</div><div class='add'>+	if (cm_node-&gt;do_lpb)</div><div class='add'>+		sqbuf-&gt;do_lpb = true;</div><div class='add'>+</div><div class='add'>+	if (cm_node-&gt;ipv4) {</div><div class='add'>+		sqbuf-&gt;ipv4 = true;</div><div class='add'>+</div><div class='add'>+		iph = (struct iphdr *)buf;</div><div class='add'>+		buf += sizeof(*iph);</div><div class='add'>+		tcph = (struct tcphdr *)buf;</div><div class='add'>+		buf += sizeof(*tcph);</div><div class='add'>+</div><div class='add'>+		ether_addr_copy(ethh-&gt;h_dest, cm_node-&gt;rem_mac);</div><div class='add'>+		ether_addr_copy(ethh-&gt;h_source, cm_node-&gt;loc_mac);</div><div class='add'>+		if (cm_node-&gt;vlan_id &lt; VLAN_N_VID) {</div><div class='add'>+			((struct vlan_ethhdr *)ethh)-&gt;h_vlan_proto =</div><div class='add'>+				htons(ETH_P_8021Q);</div><div class='add'>+			vtag = (cm_node-&gt;user_pri &lt;&lt; VLAN_PRIO_SHIFT) |</div><div class='add'>+			       cm_node-&gt;vlan_id;</div><div class='add'>+			((struct vlan_ethhdr *)ethh)-&gt;h_vlan_TCI = htons(vtag);</div><div class='add'>+</div><div class='add'>+			((struct vlan_ethhdr *)ethh)-&gt;h_vlan_encapsulated_proto =</div><div class='add'>+				htons(ETH_P_IP);</div><div class='add'>+		} else {</div><div class='add'>+			ethh-&gt;h_proto = htons(ETH_P_IP);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		iph-&gt;version = IPVERSION;</div><div class='add'>+		iph-&gt;ihl = 5; /* 5 * 4Byte words, IP headr len */</div><div class='add'>+		iph-&gt;tos = cm_node-&gt;tos;</div><div class='add'>+		iph-&gt;tot_len = htons(pktsize);</div><div class='add'>+		iph-&gt;id = htons(++cm_node-&gt;tcp_cntxt.loc_id);</div><div class='add'>+</div><div class='add'>+		iph-&gt;frag_off = htons(0x4000);</div><div class='add'>+		iph-&gt;ttl = 0x40;</div><div class='add'>+		iph-&gt;protocol = IPPROTO_TCP;</div><div class='add'>+		iph-&gt;saddr = htonl(cm_node-&gt;loc_addr[0]);</div><div class='add'>+		iph-&gt;daddr = htonl(cm_node-&gt;rem_addr[0]);</div><div class='add'>+	} else {</div><div class='add'>+		sqbuf-&gt;ipv4 = false;</div><div class='add'>+		ip6h = (struct ipv6hdr *)buf;</div><div class='add'>+		buf += sizeof(*ip6h);</div><div class='add'>+		tcph = (struct tcphdr *)buf;</div><div class='add'>+		buf += sizeof(*tcph);</div><div class='add'>+</div><div class='add'>+		ether_addr_copy(ethh-&gt;h_dest, cm_node-&gt;rem_mac);</div><div class='add'>+		ether_addr_copy(ethh-&gt;h_source, cm_node-&gt;loc_mac);</div><div class='add'>+		if (cm_node-&gt;vlan_id &lt; VLAN_N_VID) {</div><div class='add'>+			((struct vlan_ethhdr *)ethh)-&gt;h_vlan_proto =</div><div class='add'>+				htons(ETH_P_8021Q);</div><div class='add'>+			vtag = (cm_node-&gt;user_pri &lt;&lt; VLAN_PRIO_SHIFT) |</div><div class='add'>+			       cm_node-&gt;vlan_id;</div><div class='add'>+			((struct vlan_ethhdr *)ethh)-&gt;h_vlan_TCI = htons(vtag);</div><div class='add'>+			((struct vlan_ethhdr *)ethh)-&gt;h_vlan_encapsulated_proto =</div><div class='add'>+				htons(ETH_P_IPV6);</div><div class='add'>+		} else {</div><div class='add'>+			ethh-&gt;h_proto = htons(ETH_P_IPV6);</div><div class='add'>+		}</div><div class='add'>+		ip6h-&gt;version = 6;</div><div class='add'>+		ip6h-&gt;priority = cm_node-&gt;tos &gt;&gt; 4;</div><div class='add'>+		ip6h-&gt;flow_lbl[0] = cm_node-&gt;tos &lt;&lt; 4;</div><div class='add'>+		ip6h-&gt;flow_lbl[1] = 0;</div><div class='add'>+		ip6h-&gt;flow_lbl[2] = 0;</div><div class='add'>+		ip6h-&gt;payload_len = htons(pktsize - sizeof(*ip6h));</div><div class='add'>+		ip6h-&gt;nexthdr = 6;</div><div class='add'>+		ip6h-&gt;hop_limit = 128;</div><div class='add'>+		irdma_copy_ip_htonl(ip6h-&gt;saddr.in6_u.u6_addr32,</div><div class='add'>+				    cm_node-&gt;loc_addr);</div><div class='add'>+		irdma_copy_ip_htonl(ip6h-&gt;daddr.in6_u.u6_addr32,</div><div class='add'>+				    cm_node-&gt;rem_addr);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	tcph-&gt;source = htons(cm_node-&gt;loc_port);</div><div class='add'>+	tcph-&gt;dest = htons(cm_node-&gt;rem_port);</div><div class='add'>+	tcph-&gt;seq = htonl(cm_node-&gt;tcp_cntxt.loc_seq_num);</div><div class='add'>+</div><div class='add'>+	if (flags &amp; SET_ACK) {</div><div class='add'>+		cm_node-&gt;tcp_cntxt.loc_ack_num = cm_node-&gt;tcp_cntxt.rcv_nxt;</div><div class='add'>+		tcph-&gt;ack_seq = htonl(cm_node-&gt;tcp_cntxt.loc_ack_num);</div><div class='add'>+		tcph-&gt;ack = 1;</div><div class='add'>+	} else {</div><div class='add'>+		tcph-&gt;ack_seq = 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (flags &amp; SET_SYN) {</div><div class='add'>+		cm_node-&gt;tcp_cntxt.loc_seq_num++;</div><div class='add'>+		tcph-&gt;syn = 1;</div><div class='add'>+	} else {</div><div class='add'>+		cm_node-&gt;tcp_cntxt.loc_seq_num += hdr_len + pd_len;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (flags &amp; SET_FIN) {</div><div class='add'>+		cm_node-&gt;tcp_cntxt.loc_seq_num++;</div><div class='add'>+		tcph-&gt;fin = 1;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (flags &amp; SET_RST)</div><div class='add'>+		tcph-&gt;rst = 1;</div><div class='add'>+</div><div class='add'>+	tcph-&gt;doff = (u16)((sizeof(*tcph) + opts_len + 3) &gt;&gt; 2);</div><div class='add'>+	sqbuf-&gt;tcphlen = tcph-&gt;doff &lt;&lt; 2;</div><div class='add'>+	tcph-&gt;window = htons(cm_node-&gt;tcp_cntxt.rcv_wnd);</div><div class='add'>+	tcph-&gt;urg_ptr = 0;</div><div class='add'>+</div><div class='add'>+	if (opts_len) {</div><div class='add'>+		memcpy(buf, options-&gt;addr, opts_len);</div><div class='add'>+		buf += opts_len;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (hdr_len) {</div><div class='add'>+		memcpy(buf, hdr-&gt;addr, hdr_len);</div><div class='add'>+		buf += hdr_len;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (pdata &amp;&amp; pdata-&gt;addr)</div><div class='add'>+		memcpy(buf, pdata-&gt;addr, pdata-&gt;size);</div><div class='add'>+</div><div class='add'>+	refcount_set(&amp;sqbuf-&gt;refcount, 1);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("ILQ: TRANSMIT ILQ BUFFER", DUMP_PREFIX_OFFSET,</div><div class='add'>+			     16, 8, sqbuf-&gt;mem.va, sqbuf-&gt;totallen, false);</div><div class='add'>+	return sqbuf;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_send_reset - Send RST packet</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ */</div><div class='add'>+int irdma_send_reset(struct irdma_cm_node *cm_node)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_buf *sqbuf;</div><div class='add'>+	int flags = SET_RST | SET_ACK;</div><div class='add'>+</div><div class='add'>+	trace_irdma_send_reset(cm_node, 0, __builtin_return_address(0));</div><div class='add'>+	sqbuf = cm_node-&gt;cm_core-&gt;form_cm_frame(cm_node, NULL, NULL, NULL,</div><div class='add'>+						flags);</div><div class='add'>+	if (!sqbuf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+		  "CM: caller: %pS cm_node %p cm_id=%p accel=%d state=%d rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n",</div><div class='add'>+		  __builtin_return_address(0), cm_node, cm_node-&gt;cm_id,</div><div class='add'>+		  cm_node-&gt;accelerated, cm_node-&gt;state, cm_node-&gt;rem_port,</div><div class='add'>+		  cm_node-&gt;loc_port, cm_node-&gt;rem_addr, cm_node-&gt;loc_addr);</div><div class='add'>+</div><div class='add'>+	return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 0,</div><div class='add'>+				       1);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_active_open_err - send event for active side cm error</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @reset: Flag to send reset or not</div><div class='add'>+ */</div><div class='add'>+static void irdma_active_open_err(struct irdma_cm_node *cm_node, bool reset)</div><div class='add'>+{</div><div class='add'>+	trace_irdma_active_open_err(cm_node, reset,</div><div class='add'>+				    __builtin_return_address(0));</div><div class='add'>+	irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+	cm_node-&gt;cm_core-&gt;stats_connect_errs++;</div><div class='add'>+	if (reset) {</div><div class='add'>+		ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: cm_node=%p state=%d\n", cm_node,</div><div class='add'>+			  cm_node-&gt;state);</div><div class='add'>+		refcount_inc(&amp;cm_node-&gt;refcnt);</div><div class='add'>+		irdma_send_reset(cm_node);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;state = IRDMA_CM_STATE_CLOSED;</div><div class='add'>+	irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_passive_open_err - handle passive side cm error</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @reset: send reset or just free cm_node</div><div class='add'>+ */</div><div class='add'>+static void irdma_passive_open_err(struct irdma_cm_node *cm_node, bool reset)</div><div class='add'>+{</div><div class='add'>+	irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+	cm_node-&gt;cm_core-&gt;stats_passive_errs++;</div><div class='add'>+	cm_node-&gt;state = IRDMA_CM_STATE_CLOSED;</div><div class='add'>+	ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev, "CM: cm_node=%p state =%d\n",</div><div class='add'>+		  cm_node, cm_node-&gt;state);</div><div class='add'>+	trace_irdma_passive_open_err(cm_node, reset,</div><div class='add'>+				     __builtin_return_address(0));</div><div class='add'>+	if (reset)</div><div class='add'>+		irdma_send_reset(cm_node);</div><div class='add'>+	else</div><div class='add'>+		irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_event_connect_error - to create connect error event</div><div class='add'>+ * @event: cm information for connect event</div><div class='add'>+ */</div><div class='add'>+static void irdma_event_connect_error(struct irdma_cm_event *event)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+	struct iw_cm_id *cm_id;</div><div class='add'>+</div><div class='add'>+	cm_id = event-&gt;cm_node-&gt;cm_id;</div><div class='add'>+	if (!cm_id)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	iwqp = cm_id-&gt;provider_data;</div><div class='add'>+</div><div class='add'>+	if (!iwqp || !iwqp-&gt;iwdev)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	iwqp-&gt;cm_id = NULL;</div><div class='add'>+	cm_id-&gt;provider_data = NULL;</div><div class='add'>+	irdma_send_cm_event(event-&gt;cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY,</div><div class='add'>+			    -ECONNRESET);</div><div class='add'>+	irdma_rem_ref_cm_node(event-&gt;cm_node);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_process_options - process options from TCP header</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @optionsloc: point to start of options</div><div class='add'>+ * @optionsize: size of all options</div><div class='add'>+ * @syn_pkt: flag if syn packet</div><div class='add'>+ */</div><div class='add'>+static int irdma_process_options(struct irdma_cm_node *cm_node, u8 *optionsloc,</div><div class='add'>+				 u32 optionsize, u32 syn_pkt)</div><div class='add'>+{</div><div class='add'>+	u32 tmp;</div><div class='add'>+	u32 offset = 0;</div><div class='add'>+	union all_known_options *all_options;</div><div class='add'>+	char got_mss_option = 0;</div><div class='add'>+</div><div class='add'>+	while (offset &lt; optionsize) {</div><div class='add'>+		all_options = (union all_known_options *)(optionsloc + offset);</div><div class='add'>+		switch (all_options-&gt;base.optionnum) {</div><div class='add'>+		case OPTION_NUM_EOL:</div><div class='add'>+			offset = optionsize;</div><div class='add'>+			break;</div><div class='add'>+		case OPTION_NUM_NONE:</div><div class='add'>+			offset += 1;</div><div class='add'>+			continue;</div><div class='add'>+		case OPTION_NUM_MSS:</div><div class='add'>+			ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: MSS Length: %d Offset: %d Size: %d\n",</div><div class='add'>+				  all_options-&gt;mss.len, offset, optionsize);</div><div class='add'>+			got_mss_option = 1;</div><div class='add'>+			if (all_options-&gt;mss.len != 4)</div><div class='add'>+				return -EINVAL;</div><div class='add'>+			tmp = ntohs(all_options-&gt;mss.mss);</div><div class='add'>+			if ((cm_node-&gt;ipv4 &amp;&amp;</div><div class='add'>+			     (tmp + IRDMA_MTU_TO_MSS_IPV4) &lt; IRDMA_MIN_MTU_IPV4) ||</div><div class='add'>+			    (!cm_node-&gt;ipv4 &amp;&amp;</div><div class='add'>+			     (tmp + IRDMA_MTU_TO_MSS_IPV6) &lt; IRDMA_MIN_MTU_IPV6))</div><div class='add'>+				return -EINVAL;</div><div class='add'>+			if (tmp &lt; cm_node-&gt;tcp_cntxt.mss)</div><div class='add'>+				cm_node-&gt;tcp_cntxt.mss = tmp;</div><div class='add'>+			break;</div><div class='add'>+		case OPTION_NUM_WINDOW_SCALE:</div><div class='add'>+			cm_node-&gt;tcp_cntxt.snd_wscale =</div><div class='add'>+				all_options-&gt;windowscale.shiftcount;</div><div class='add'>+			break;</div><div class='add'>+		default:</div><div class='add'>+			ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: Unsupported TCP Option: %x\n",</div><div class='add'>+				  all_options-&gt;base.optionnum);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		offset += all_options-&gt;base.len;</div><div class='add'>+	}</div><div class='add'>+	if (!got_mss_option &amp;&amp; syn_pkt)</div><div class='add'>+		cm_node-&gt;tcp_cntxt.mss = IRDMA_CM_DEFAULT_MSS;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_handle_tcp_options - setup TCP context info after parsing TCP options</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @tcph: pointer tcp header</div><div class='add'>+ * @optionsize: size of options rcvd</div><div class='add'>+ * @passive: active or passive flag</div><div class='add'>+ */</div><div class='add'>+static int irdma_handle_tcp_options(struct irdma_cm_node *cm_node,</div><div class='add'>+				    struct tcphdr *tcph, int optionsize,</div><div class='add'>+				    int passive)</div><div class='add'>+{</div><div class='add'>+	u8 *optionsloc = (u8 *)&amp;tcph[1];</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (optionsize) {</div><div class='add'>+		ret = irdma_process_options(cm_node, optionsloc, optionsize,</div><div class='add'>+					    (u32)tcph-&gt;syn);</div><div class='add'>+		if (ret) {</div><div class='add'>+			ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: Node %p, Sending Reset\n", cm_node);</div><div class='add'>+			if (passive)</div><div class='add'>+				irdma_passive_open_err(cm_node, true);</div><div class='add'>+			else</div><div class='add'>+				irdma_active_open_err(cm_node, true);</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;tcp_cntxt.snd_wnd = ntohs(tcph-&gt;window)</div><div class='add'>+				     &lt;&lt; cm_node-&gt;tcp_cntxt.snd_wscale;</div><div class='add'>+</div><div class='add'>+	if (cm_node-&gt;tcp_cntxt.snd_wnd &gt; cm_node-&gt;tcp_cntxt.max_snd_wnd)</div><div class='add'>+		cm_node-&gt;tcp_cntxt.max_snd_wnd = cm_node-&gt;tcp_cntxt.snd_wnd;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_build_mpa_v1 - build a MPA V1 frame</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @start_addr: address where to build frame</div><div class='add'>+ * @mpa_key: to do read0 or write0</div><div class='add'>+ */</div><div class='add'>+static void irdma_build_mpa_v1(struct irdma_cm_node *cm_node, void *start_addr,</div><div class='add'>+			       u8 mpa_key)</div><div class='add'>+{</div><div class='add'>+	struct ietf_mpa_v1 *mpa_frame = start_addr;</div><div class='add'>+</div><div class='add'>+	switch (mpa_key) {</div><div class='add'>+	case MPA_KEY_REQUEST:</div><div class='add'>+		memcpy(mpa_frame-&gt;key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE);</div><div class='add'>+		break;</div><div class='add'>+	case MPA_KEY_REPLY:</div><div class='add'>+		memcpy(mpa_frame-&gt;key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+	mpa_frame-&gt;flags = IETF_MPA_FLAGS_CRC;</div><div class='add'>+	mpa_frame-&gt;rev = cm_node-&gt;mpa_frame_rev;</div><div class='add'>+	mpa_frame-&gt;priv_data_len = htons(cm_node-&gt;pdata.size);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_build_mpa_v2 - build a MPA V2 frame</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @start_addr: buffer start address</div><div class='add'>+ * @mpa_key: to do read0 or write0</div><div class='add'>+ */</div><div class='add'>+static void irdma_build_mpa_v2(struct irdma_cm_node *cm_node, void *start_addr,</div><div class='add'>+			       u8 mpa_key)</div><div class='add'>+{</div><div class='add'>+	struct ietf_mpa_v2 *mpa_frame = start_addr;</div><div class='add'>+	struct ietf_rtr_msg *rtr_msg = &amp;mpa_frame-&gt;rtr_msg;</div><div class='add'>+	u16 ctrl_ird, ctrl_ord;</div><div class='add'>+</div><div class='add'>+	/* initialize the upper 5 bytes of the frame */</div><div class='add'>+	irdma_build_mpa_v1(cm_node, start_addr, mpa_key);</div><div class='add'>+	mpa_frame-&gt;flags |= IETF_MPA_V2_FLAG;</div><div class='add'>+	if (cm_node-&gt;iwdev-&gt;iw_ooo) {</div><div class='add'>+		mpa_frame-&gt;flags |= IETF_MPA_FLAGS_MARKERS;</div><div class='add'>+		cm_node-&gt;rcv_mark_en = true;</div><div class='add'>+	}</div><div class='add'>+	mpa_frame-&gt;priv_data_len = cpu_to_be16(be16_to_cpu(mpa_frame-&gt;priv_data_len) +</div><div class='add'>+					       IETF_RTR_MSG_SIZE);</div><div class='add'>+</div><div class='add'>+	/* initialize RTR msg */</div><div class='add'>+	if (cm_node-&gt;mpav2_ird_ord == IETF_NO_IRD_ORD) {</div><div class='add'>+		ctrl_ird = IETF_NO_IRD_ORD;</div><div class='add'>+		ctrl_ord = IETF_NO_IRD_ORD;</div><div class='add'>+	} else {</div><div class='add'>+		ctrl_ird = (cm_node-&gt;ird_size &gt; IETF_NO_IRD_ORD) ?</div><div class='add'>+				   IETF_NO_IRD_ORD :</div><div class='add'>+				   cm_node-&gt;ird_size;</div><div class='add'>+		ctrl_ord = (cm_node-&gt;ord_size &gt; IETF_NO_IRD_ORD) ?</div><div class='add'>+				   IETF_NO_IRD_ORD :</div><div class='add'>+				   cm_node-&gt;ord_size;</div><div class='add'>+	}</div><div class='add'>+	ctrl_ird |= IETF_PEER_TO_PEER;</div><div class='add'>+</div><div class='add'>+	switch (mpa_key) {</div><div class='add'>+	case MPA_KEY_REQUEST:</div><div class='add'>+		ctrl_ord |= IETF_RDMA0_WRITE;</div><div class='add'>+		ctrl_ord |= IETF_RDMA0_READ;</div><div class='add'>+		break;</div><div class='add'>+	case MPA_KEY_REPLY:</div><div class='add'>+		switch (cm_node-&gt;send_rdma0_op) {</div><div class='add'>+		case SEND_RDMA_WRITE_ZERO:</div><div class='add'>+			ctrl_ord |= IETF_RDMA0_WRITE;</div><div class='add'>+			break;</div><div class='add'>+		case SEND_RDMA_READ_ZERO:</div><div class='add'>+			ctrl_ord |= IETF_RDMA0_READ;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+	rtr_msg-&gt;ctrl_ird = htons(ctrl_ird);</div><div class='add'>+	rtr_msg-&gt;ctrl_ord = htons(ctrl_ord);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cm_build_mpa_frame - build mpa frame for mpa version 1 or version 2</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @mpa: mpa: data buffer</div><div class='add'>+ * @mpa_key: to do read0 or write0</div><div class='add'>+ */</div><div class='add'>+static int irdma_cm_build_mpa_frame(struct irdma_cm_node *cm_node,</div><div class='add'>+				    struct irdma_kmem_info *mpa, u8 mpa_key)</div><div class='add'>+{</div><div class='add'>+	int hdr_len = 0;</div><div class='add'>+</div><div class='add'>+	switch (cm_node-&gt;mpa_frame_rev) {</div><div class='add'>+	case IETF_MPA_V1:</div><div class='add'>+		hdr_len = sizeof(struct ietf_mpa_v1);</div><div class='add'>+		irdma_build_mpa_v1(cm_node, mpa-&gt;addr, mpa_key);</div><div class='add'>+		break;</div><div class='add'>+	case IETF_MPA_V2:</div><div class='add'>+		hdr_len = sizeof(struct ietf_mpa_v2);</div><div class='add'>+		irdma_build_mpa_v2(cm_node, mpa-&gt;addr, mpa_key);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return hdr_len;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_send_mpa_request - active node send mpa request to passive node</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ */</div><div class='add'>+static int irdma_send_mpa_request(struct irdma_cm_node *cm_node)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_buf *sqbuf;</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;mpa_hdr.addr = &amp;cm_node-&gt;mpa_v2_frame;</div><div class='add'>+	cm_node-&gt;mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node,</div><div class='add'>+							 &amp;cm_node-&gt;mpa_hdr,</div><div class='add'>+							 MPA_KEY_REQUEST);</div><div class='add'>+	if (!cm_node-&gt;mpa_hdr.size) {</div><div class='add'>+		ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: mpa size = %d\n", cm_node-&gt;mpa_hdr.size);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	sqbuf = cm_node-&gt;cm_core-&gt;form_cm_frame(cm_node, NULL,</div><div class='add'>+						&amp;cm_node-&gt;mpa_hdr,</div><div class='add'>+						&amp;cm_node-&gt;pdata, SET_ACK);</div><div class='add'>+	if (!sqbuf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1,</div><div class='add'>+				       0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_send_mpa_reject -</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @pdata: reject data for connection</div><div class='add'>+ * @plen: length of reject data</div><div class='add'>+ */</div><div class='add'>+static int irdma_send_mpa_reject(struct irdma_cm_node *cm_node,</div><div class='add'>+				 const void *pdata, u8 plen)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_buf *sqbuf;</div><div class='add'>+	struct irdma_mpa_priv_info priv_info;</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;mpa_hdr.addr = &amp;cm_node-&gt;mpa_v2_frame;</div><div class='add'>+	cm_node-&gt;mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node,</div><div class='add'>+							 &amp;cm_node-&gt;mpa_hdr,</div><div class='add'>+							 MPA_KEY_REPLY);</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;mpa_frame.flags |= IETF_MPA_FLAGS_REJECT;</div><div class='add'>+	priv_info.addr = pdata;</div><div class='add'>+	priv_info.size = plen;</div><div class='add'>+</div><div class='add'>+	sqbuf = cm_node-&gt;cm_core-&gt;form_cm_frame(cm_node, NULL,</div><div class='add'>+						&amp;cm_node-&gt;mpa_hdr, &amp;priv_info,</div><div class='add'>+						SET_ACK | SET_FIN);</div><div class='add'>+	if (!sqbuf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;state = IRDMA_CM_STATE_FIN_WAIT1;</div><div class='add'>+</div><div class='add'>+	return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1,</div><div class='add'>+				       0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_negotiate_mpa_v2_ird_ord - negotiate MPAv2 IRD/ORD</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @buf: Data pointer</div><div class='add'>+ */</div><div class='add'>+static int irdma_negotiate_mpa_v2_ird_ord(struct irdma_cm_node *cm_node,</div><div class='add'>+					  u8 *buf)</div><div class='add'>+{</div><div class='add'>+	struct ietf_mpa_v2 *mpa_v2_frame;</div><div class='add'>+	struct ietf_rtr_msg *rtr_msg;</div><div class='add'>+	u16 ird_size;</div><div class='add'>+	u16 ord_size;</div><div class='add'>+	u16 ctrl_ord;</div><div class='add'>+	u16 ctrl_ird;</div><div class='add'>+</div><div class='add'>+	mpa_v2_frame = (struct ietf_mpa_v2 *)buf;</div><div class='add'>+	rtr_msg = &amp;mpa_v2_frame-&gt;rtr_msg;</div><div class='add'>+</div><div class='add'>+	/* parse rtr message */</div><div class='add'>+	ctrl_ord = ntohs(rtr_msg-&gt;ctrl_ord);</div><div class='add'>+	ctrl_ird = ntohs(rtr_msg-&gt;ctrl_ird);</div><div class='add'>+	ird_size = ctrl_ird &amp; IETF_NO_IRD_ORD;</div><div class='add'>+	ord_size = ctrl_ord &amp; IETF_NO_IRD_ORD;</div><div class='add'>+</div><div class='add'>+	if (!(ctrl_ird &amp; IETF_PEER_TO_PEER))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (ird_size == IETF_NO_IRD_ORD || ord_size == IETF_NO_IRD_ORD) {</div><div class='add'>+		cm_node-&gt;mpav2_ird_ord = IETF_NO_IRD_ORD;</div><div class='add'>+		goto negotiate_done;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (cm_node-&gt;state != IRDMA_CM_STATE_MPAREQ_SENT) {</div><div class='add'>+		/* responder */</div><div class='add'>+		if (!ord_size &amp;&amp; (ctrl_ord &amp; IETF_RDMA0_READ))</div><div class='add'>+			cm_node-&gt;ird_size = 1;</div><div class='add'>+		if (cm_node-&gt;ord_size &gt; ird_size)</div><div class='add'>+			cm_node-&gt;ord_size = ird_size;</div><div class='add'>+	} else {</div><div class='add'>+		/* initiator */</div><div class='add'>+		if (!ird_size &amp;&amp; (ctrl_ord &amp; IETF_RDMA0_READ))</div><div class='add'>+			/* Remote peer doesn't support RDMA0_READ */</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+		if (cm_node-&gt;ord_size &gt; ird_size)</div><div class='add'>+			cm_node-&gt;ord_size = ird_size;</div><div class='add'>+</div><div class='add'>+		if (cm_node-&gt;ird_size &lt; ord_size)</div><div class='add'>+		/* no resources available */</div><div class='add'>+			return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+negotiate_done:</div><div class='add'>+	if (ctrl_ord &amp; IETF_RDMA0_READ)</div><div class='add'>+		cm_node-&gt;send_rdma0_op = SEND_RDMA_READ_ZERO;</div><div class='add'>+	else if (ctrl_ord &amp; IETF_RDMA0_WRITE)</div><div class='add'>+		cm_node-&gt;send_rdma0_op = SEND_RDMA_WRITE_ZERO;</div><div class='add'>+	else</div><div class='add'>+		/* Not supported RDMA0 operation */</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+		  "CM: MPAV2 Negotiated ORD: %d, IRD: %d\n",</div><div class='add'>+		  cm_node-&gt;ord_size, cm_node-&gt;ird_size);</div><div class='add'>+	trace_irdma_negotiate_mpa_v2(cm_node);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_parse_mpa - process an IETF MPA frame</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @buf: Data pointer</div><div class='add'>+ * @type: to return accept or reject</div><div class='add'>+ * @len: Len of mpa buffer</div><div class='add'>+ */</div><div class='add'>+static int irdma_parse_mpa(struct irdma_cm_node *cm_node, u8 *buf, u32 *type,</div><div class='add'>+			   u32 len)</div><div class='add'>+{</div><div class='add'>+	struct ietf_mpa_v1 *mpa_frame;</div><div class='add'>+	int mpa_hdr_len, priv_data_len, ret;</div><div class='add'>+</div><div class='add'>+	*type = IRDMA_MPA_REQUEST_ACCEPT;</div><div class='add'>+</div><div class='add'>+	if (len &lt; sizeof(struct ietf_mpa_v1)) {</div><div class='add'>+		ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: ietf buffer small (%x)\n", len);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mpa_frame = (struct ietf_mpa_v1 *)buf;</div><div class='add'>+	mpa_hdr_len = sizeof(struct ietf_mpa_v1);</div><div class='add'>+	priv_data_len = ntohs(mpa_frame-&gt;priv_data_len);</div><div class='add'>+</div><div class='add'>+	if (priv_data_len &gt; IETF_MAX_PRIV_DATA_LEN) {</div><div class='add'>+		ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: private_data too big %d\n", priv_data_len);</div><div class='add'>+		return -EOVERFLOW;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (mpa_frame-&gt;rev != IETF_MPA_V1 &amp;&amp; mpa_frame-&gt;rev != IETF_MPA_V2) {</div><div class='add'>+		ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: unsupported mpa rev = %d\n", mpa_frame-&gt;rev);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (mpa_frame-&gt;rev &gt; cm_node-&gt;mpa_frame_rev) {</div><div class='add'>+		ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev, "CM: rev %d\n",</div><div class='add'>+			  mpa_frame-&gt;rev);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;mpa_frame_rev = mpa_frame-&gt;rev;</div><div class='add'>+	if (cm_node-&gt;state != IRDMA_CM_STATE_MPAREQ_SENT) {</div><div class='add'>+		if (memcmp(mpa_frame-&gt;key, IEFT_MPA_KEY_REQ,</div><div class='add'>+			   IETF_MPA_KEY_SIZE)) {</div><div class='add'>+			ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: Unexpected MPA Key received\n");</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		if (memcmp(mpa_frame-&gt;key, IEFT_MPA_KEY_REP,</div><div class='add'>+			   IETF_MPA_KEY_SIZE)) {</div><div class='add'>+			ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: Unexpected MPA Key received\n");</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (priv_data_len + mpa_hdr_len &gt; len) {</div><div class='add'>+		ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: ietf buffer len(%x + %x != %x)\n",</div><div class='add'>+			  priv_data_len, mpa_hdr_len, len);</div><div class='add'>+		return -EOVERFLOW;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (len &gt; IRDMA_MAX_CM_BUF) {</div><div class='add'>+		ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: ietf buffer large len = %d\n", len);</div><div class='add'>+		return -EOVERFLOW;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (mpa_frame-&gt;rev) {</div><div class='add'>+	case IETF_MPA_V2:</div><div class='add'>+		mpa_hdr_len += IETF_RTR_MSG_SIZE;</div><div class='add'>+		ret = irdma_negotiate_mpa_v2_ird_ord(cm_node, buf);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+		break;</div><div class='add'>+	case IETF_MPA_V1:</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	memcpy(cm_node-&gt;pdata_buf, buf + mpa_hdr_len, priv_data_len);</div><div class='add'>+	cm_node-&gt;pdata.size = priv_data_len;</div><div class='add'>+</div><div class='add'>+	if (mpa_frame-&gt;flags &amp; IETF_MPA_FLAGS_REJECT)</div><div class='add'>+		*type = IRDMA_MPA_REQUEST_REJECT;</div><div class='add'>+</div><div class='add'>+	if (mpa_frame-&gt;flags &amp; IETF_MPA_FLAGS_MARKERS)</div><div class='add'>+		cm_node-&gt;snd_mark_en = true;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_schedule_cm_timer</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @sqbuf: buffer to send</div><div class='add'>+ * @type: if it is send or close</div><div class='add'>+ * @send_retrans: if rexmits to be done</div><div class='add'>+ * @close_when_complete: is cm_node to be removed</div><div class='add'>+ *</div><div class='add'>+ * note - cm_node needs to be protected before calling this. Encase in:</div><div class='add'>+ *		irdma_rem_ref_cm_node(cm_core, cm_node);</div><div class='add'>+ *		irdma_schedule_cm_timer(...)</div><div class='add'>+ *		refcount_inc(&amp;cm_node-&gt;refcnt);</div><div class='add'>+ */</div><div class='add'>+int irdma_schedule_cm_timer(struct irdma_cm_node *cm_node,</div><div class='add'>+			    struct irdma_puda_buf *sqbuf,</div><div class='add'>+			    enum irdma_timer_type type, int send_retrans,</div><div class='add'>+			    int close_when_complete)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_vsi *vsi = &amp;cm_node-&gt;iwdev-&gt;vsi;</div><div class='add'>+	struct irdma_cm_core *cm_core = cm_node-&gt;cm_core;</div><div class='add'>+	struct irdma_timer_entry *new_send;</div><div class='add'>+	u32 was_timer_set;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC);</div><div class='add'>+	if (!new_send) {</div><div class='add'>+		if (type != IRDMA_TIMER_TYPE_CLOSE)</div><div class='add'>+			irdma_free_sqbuf(vsi, sqbuf);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	new_send-&gt;retrycount = IRDMA_DEFAULT_RETRYS;</div><div class='add'>+	new_send-&gt;retranscount = IRDMA_DEFAULT_RETRANS;</div><div class='add'>+	new_send-&gt;sqbuf = sqbuf;</div><div class='add'>+	new_send-&gt;timetosend = jiffies;</div><div class='add'>+	new_send-&gt;type = type;</div><div class='add'>+	new_send-&gt;send_retrans = send_retrans;</div><div class='add'>+	new_send-&gt;close_when_complete = close_when_complete;</div><div class='add'>+</div><div class='add'>+	if (type == IRDMA_TIMER_TYPE_CLOSE) {</div><div class='add'>+		new_send-&gt;timetosend += (HZ / 10);</div><div class='add'>+		if (cm_node-&gt;close_entry) {</div><div class='add'>+			kfree(new_send);</div><div class='add'>+			ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: already close entry\n");</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		cm_node-&gt;close_entry = new_send;</div><div class='add'>+	} else { /* type == IRDMA_TIMER_TYPE_SEND */</div><div class='add'>+		spin_lock_irqsave(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='add'>+		cm_node-&gt;send_entry = new_send;</div><div class='add'>+		refcount_inc(&amp;cm_node-&gt;refcnt);</div><div class='add'>+		spin_unlock_irqrestore(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='add'>+		new_send-&gt;timetosend = jiffies + IRDMA_RETRY_TIMEOUT;</div><div class='add'>+</div><div class='add'>+		refcount_inc(&amp;sqbuf-&gt;refcount);</div><div class='add'>+		irdma_puda_send_buf(vsi-&gt;ilq, sqbuf);</div><div class='add'>+		if (!send_retrans) {</div><div class='add'>+			irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+			if (close_when_complete)</div><div class='add'>+				irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cm_core-&gt;ht_lock, flags);</div><div class='add'>+	was_timer_set = timer_pending(&amp;cm_core-&gt;tcp_timer);</div><div class='add'>+</div><div class='add'>+	if (!was_timer_set) {</div><div class='add'>+		cm_core-&gt;tcp_timer.expires = new_send-&gt;timetosend;</div><div class='add'>+		add_timer(&amp;cm_core-&gt;tcp_timer);</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_retrans_expired - Could not rexmit the packet</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ */</div><div class='add'>+static void irdma_retrans_expired(struct irdma_cm_node *cm_node)</div><div class='add'>+{</div><div class='add'>+	enum irdma_cm_node_state state = cm_node-&gt;state;</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;state = IRDMA_CM_STATE_CLOSED;</div><div class='add'>+	switch (state) {</div><div class='add'>+	case IRDMA_CM_STATE_SYN_RCVD:</div><div class='add'>+	case IRDMA_CM_STATE_CLOSING:</div><div class='add'>+		irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_FIN_WAIT1:</div><div class='add'>+	case IRDMA_CM_STATE_LAST_ACK:</div><div class='add'>+		irdma_send_reset(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		refcount_inc(&amp;cm_node-&gt;refcnt);</div><div class='add'>+		irdma_send_reset(cm_node);</div><div class='add'>+		irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_handle_close_entry - for handling retry/timeouts</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @rem_node: flag for remove cm_node</div><div class='add'>+ */</div><div class='add'>+static void irdma_handle_close_entry(struct irdma_cm_node *cm_node,</div><div class='add'>+				     u32 rem_node)</div><div class='add'>+{</div><div class='add'>+	struct irdma_timer_entry *close_entry = cm_node-&gt;close_entry;</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	if (!close_entry)</div><div class='add'>+		return;</div><div class='add'>+	iwqp = (struct irdma_qp *)close_entry-&gt;sqbuf;</div><div class='add'>+	if (iwqp) {</div><div class='add'>+		spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+		if (iwqp-&gt;cm_id) {</div><div class='add'>+			iwqp-&gt;hw_tcp_state = IRDMA_TCP_STATE_CLOSED;</div><div class='add'>+			iwqp-&gt;hw_iwarp_state = IRDMA_QP_STATE_ERROR;</div><div class='add'>+			iwqp-&gt;last_aeq = IRDMA_AE_RESET_SENT;</div><div class='add'>+			iwqp-&gt;ibqp_state = IB_QPS_ERR;</div><div class='add'>+			spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+			irdma_cm_disconn(iwqp);</div><div class='add'>+		} else {</div><div class='add'>+			spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+		}</div><div class='add'>+	} else if (rem_node) {</div><div class='add'>+		/* TIME_WAIT state */</div><div class='add'>+		irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	kfree(close_entry);</div><div class='add'>+	cm_node-&gt;close_entry = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cm_timer_tick - system's timer expired callback</div><div class='add'>+ * @t: Pointer to timer_list</div><div class='add'>+ */</div><div class='add'>+static void irdma_cm_timer_tick(struct timer_list *t)</div><div class='add'>+{</div><div class='add'>+	unsigned long nexttimeout = jiffies + IRDMA_LONG_TIME;</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+	struct irdma_timer_entry *send_entry, *close_entry;</div><div class='add'>+	struct list_head *list_core_temp;</div><div class='add'>+	struct list_head *list_node;</div><div class='add'>+	struct irdma_cm_core *cm_core = timer_container_of(cm_core, t,</div><div class='add'>+							   tcp_timer);</div><div class='add'>+	struct irdma_sc_vsi *vsi;</div><div class='add'>+	u32 settimer = 0;</div><div class='add'>+	unsigned long timetosend;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	struct list_head timer_list;</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;timer_list);</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	irdma_timer_list_prep(cm_core, &amp;timer_list);</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+</div><div class='add'>+	list_for_each_safe (list_node, list_core_temp, &amp;timer_list) {</div><div class='add'>+		cm_node = container_of(list_node, struct irdma_cm_node,</div><div class='add'>+				       timer_entry);</div><div class='add'>+		close_entry = cm_node-&gt;close_entry;</div><div class='add'>+</div><div class='add'>+		if (close_entry) {</div><div class='add'>+			if (time_after(close_entry-&gt;timetosend, jiffies)) {</div><div class='add'>+				if (nexttimeout &gt; close_entry-&gt;timetosend ||</div><div class='add'>+				    !settimer) {</div><div class='add'>+					nexttimeout = close_entry-&gt;timetosend;</div><div class='add'>+					settimer = 1;</div><div class='add'>+				}</div><div class='add'>+			} else {</div><div class='add'>+				irdma_handle_close_entry(cm_node, 1);</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		spin_lock_irqsave(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='add'>+</div><div class='add'>+		send_entry = cm_node-&gt;send_entry;</div><div class='add'>+		if (!send_entry)</div><div class='add'>+			goto done;</div><div class='add'>+		if (time_after(send_entry-&gt;timetosend, jiffies)) {</div><div class='add'>+			if (cm_node-&gt;state != IRDMA_CM_STATE_OFFLOADED) {</div><div class='add'>+				if (nexttimeout &gt; send_entry-&gt;timetosend ||</div><div class='add'>+				    !settimer) {</div><div class='add'>+					nexttimeout = send_entry-&gt;timetosend;</div><div class='add'>+					settimer = 1;</div><div class='add'>+				}</div><div class='add'>+			} else {</div><div class='add'>+				irdma_free_retrans_entry(cm_node);</div><div class='add'>+			}</div><div class='add'>+			goto done;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (cm_node-&gt;state == IRDMA_CM_STATE_OFFLOADED ||</div><div class='add'>+		    cm_node-&gt;state == IRDMA_CM_STATE_CLOSED) {</div><div class='add'>+			irdma_free_retrans_entry(cm_node);</div><div class='add'>+			goto done;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (!send_entry-&gt;retranscount || !send_entry-&gt;retrycount) {</div><div class='add'>+			irdma_free_retrans_entry(cm_node);</div><div class='add'>+</div><div class='add'>+			spin_unlock_irqrestore(&amp;cm_node-&gt;retrans_list_lock,</div><div class='add'>+					       flags);</div><div class='add'>+			irdma_retrans_expired(cm_node);</div><div class='add'>+			cm_node-&gt;state = IRDMA_CM_STATE_CLOSED;</div><div class='add'>+			spin_lock_irqsave(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='add'>+			goto done;</div><div class='add'>+		}</div><div class='add'>+		spin_unlock_irqrestore(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='add'>+</div><div class='add'>+		vsi = &amp;cm_node-&gt;iwdev-&gt;vsi;</div><div class='add'>+		if (!cm_node-&gt;ack_rcvd) {</div><div class='add'>+			refcount_inc(&amp;send_entry-&gt;sqbuf-&gt;refcount);</div><div class='add'>+			irdma_puda_send_buf(vsi-&gt;ilq, send_entry-&gt;sqbuf);</div><div class='add'>+			cm_node-&gt;cm_core-&gt;stats_pkt_retrans++;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		spin_lock_irqsave(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='add'>+		if (send_entry-&gt;send_retrans) {</div><div class='add'>+			send_entry-&gt;retranscount--;</div><div class='add'>+			timetosend = (IRDMA_RETRY_TIMEOUT &lt;&lt;</div><div class='add'>+				      (IRDMA_DEFAULT_RETRANS -</div><div class='add'>+				       send_entry-&gt;retranscount));</div><div class='add'>+</div><div class='add'>+			send_entry-&gt;timetosend = jiffies +</div><div class='add'>+			    min(timetosend, IRDMA_MAX_TIMEOUT);</div><div class='add'>+			if (nexttimeout &gt; send_entry-&gt;timetosend || !settimer) {</div><div class='add'>+				nexttimeout = send_entry-&gt;timetosend;</div><div class='add'>+				settimer = 1;</div><div class='add'>+			}</div><div class='add'>+		} else {</div><div class='add'>+			int close_when_complete;</div><div class='add'>+</div><div class='add'>+			close_when_complete = send_entry-&gt;close_when_complete;</div><div class='add'>+			irdma_free_retrans_entry(cm_node);</div><div class='add'>+			if (close_when_complete)</div><div class='add'>+				irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+		}</div><div class='add'>+done:</div><div class='add'>+		spin_unlock_irqrestore(&amp;cm_node-&gt;retrans_list_lock, flags);</div><div class='add'>+		irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (settimer) {</div><div class='add'>+		spin_lock_irqsave(&amp;cm_core-&gt;ht_lock, flags);</div><div class='add'>+		if (!timer_pending(&amp;cm_core-&gt;tcp_timer)) {</div><div class='add'>+			cm_core-&gt;tcp_timer.expires = nexttimeout;</div><div class='add'>+			add_timer(&amp;cm_core-&gt;tcp_timer);</div><div class='add'>+		}</div><div class='add'>+		spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_send_syn - send SYN packet</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @sendack: flag to set ACK bit or not</div><div class='add'>+ */</div><div class='add'>+int irdma_send_syn(struct irdma_cm_node *cm_node, u32 sendack)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_buf *sqbuf;</div><div class='add'>+	int flags = SET_SYN;</div><div class='add'>+	char optionsbuf[sizeof(struct option_mss) +</div><div class='add'>+			sizeof(struct option_windowscale) +</div><div class='add'>+			sizeof(struct option_base) + TCP_OPTIONS_PADDING];</div><div class='add'>+	struct irdma_kmem_info opts;</div><div class='add'>+	int optionssize = 0;</div><div class='add'>+	/* Sending MSS option */</div><div class='add'>+	union all_known_options *options;</div><div class='add'>+</div><div class='add'>+	opts.addr = optionsbuf;</div><div class='add'>+	if (!cm_node)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	options = (union all_known_options *)&amp;optionsbuf[optionssize];</div><div class='add'>+	options-&gt;mss.optionnum = OPTION_NUM_MSS;</div><div class='add'>+	options-&gt;mss.len = sizeof(struct option_mss);</div><div class='add'>+	options-&gt;mss.mss = htons(cm_node-&gt;tcp_cntxt.mss);</div><div class='add'>+	optionssize += sizeof(struct option_mss);</div><div class='add'>+</div><div class='add'>+	options = (union all_known_options *)&amp;optionsbuf[optionssize];</div><div class='add'>+	options-&gt;windowscale.optionnum = OPTION_NUM_WINDOW_SCALE;</div><div class='add'>+	options-&gt;windowscale.len = sizeof(struct option_windowscale);</div><div class='add'>+	options-&gt;windowscale.shiftcount = cm_node-&gt;tcp_cntxt.rcv_wscale;</div><div class='add'>+	optionssize += sizeof(struct option_windowscale);</div><div class='add'>+	options = (union all_known_options *)&amp;optionsbuf[optionssize];</div><div class='add'>+	options-&gt;eol = OPTION_NUM_EOL;</div><div class='add'>+	optionssize += 1;</div><div class='add'>+</div><div class='add'>+	if (sendack)</div><div class='add'>+		flags |= SET_ACK;</div><div class='add'>+</div><div class='add'>+	opts.size = optionssize;</div><div class='add'>+</div><div class='add'>+	sqbuf = cm_node-&gt;cm_core-&gt;form_cm_frame(cm_node, &amp;opts, NULL, NULL,</div><div class='add'>+						flags);</div><div class='add'>+	if (!sqbuf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1,</div><div class='add'>+				       0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_send_ack - Send ACK packet</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ */</div><div class='add'>+void irdma_send_ack(struct irdma_cm_node *cm_node)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_buf *sqbuf;</div><div class='add'>+	struct irdma_sc_vsi *vsi = &amp;cm_node-&gt;iwdev-&gt;vsi;</div><div class='add'>+</div><div class='add'>+	sqbuf = cm_node-&gt;cm_core-&gt;form_cm_frame(cm_node, NULL, NULL, NULL,</div><div class='add'>+						SET_ACK);</div><div class='add'>+	if (sqbuf)</div><div class='add'>+		irdma_puda_send_buf(vsi-&gt;ilq, sqbuf);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_send_fin - Send FIN pkt</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ */</div><div class='add'>+static int irdma_send_fin(struct irdma_cm_node *cm_node)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_buf *sqbuf;</div><div class='add'>+</div><div class='add'>+	sqbuf = cm_node-&gt;cm_core-&gt;form_cm_frame(cm_node, NULL, NULL, NULL,</div><div class='add'>+						SET_ACK | SET_FIN);</div><div class='add'>+	if (!sqbuf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1,</div><div class='add'>+				       0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_find_listener - find a cm node listening on this addr-port pair</div><div class='add'>+ * @cm_core: cm's core</div><div class='add'>+ * @dst_addr: listener ip addr</div><div class='add'>+ * @ipv4: flag indicating IPv4 when true</div><div class='add'>+ * @dst_port: listener tcp port num</div><div class='add'>+ * @vlan_id: virtual LAN ID</div><div class='add'>+ * @listener_state: state to match with listen node's</div><div class='add'>+ */</div><div class='add'>+static struct irdma_cm_listener *</div><div class='add'>+irdma_find_listener(struct irdma_cm_core *cm_core, u32 *dst_addr, bool ipv4,</div><div class='add'>+		    u16 dst_port, u16 vlan_id,</div><div class='add'>+		    enum irdma_cm_listener_state listener_state)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_listener *listen_node;</div><div class='add'>+	static const u32 ip_zero[4] = { 0, 0, 0, 0 };</div><div class='add'>+	u32 listen_addr[4];</div><div class='add'>+	u16 listen_port;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	/* walk list and find cm_node associated with this session ID */</div><div class='add'>+	spin_lock_irqsave(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='add'>+	list_for_each_entry (listen_node, &amp;cm_core-&gt;listen_list, list) {</div><div class='add'>+		memcpy(listen_addr, listen_node-&gt;loc_addr, sizeof(listen_addr));</div><div class='add'>+		listen_port = listen_node-&gt;loc_port;</div><div class='add'>+		if (listen_node-&gt;ipv4 != ipv4 || listen_port != dst_port ||</div><div class='add'>+		    !(listener_state &amp; listen_node-&gt;listener_state))</div><div class='add'>+			continue;</div><div class='add'>+		/* compare node pair, return node handle if a match */</div><div class='add'>+		if (!memcmp(listen_addr, ip_zero, sizeof(listen_addr)) ||</div><div class='add'>+		    (!memcmp(listen_addr, dst_addr, sizeof(listen_addr)) &amp;&amp;</div><div class='add'>+		     vlan_id == listen_node-&gt;vlan_id)) {</div><div class='add'>+			refcount_inc(&amp;listen_node-&gt;refcnt);</div><div class='add'>+			spin_unlock_irqrestore(&amp;cm_core-&gt;listen_list_lock,</div><div class='add'>+					       flags);</div><div class='add'>+			trace_irdma_find_listener(listen_node);</div><div class='add'>+			return listen_node;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='add'>+</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_del_multiple_qhash - Remove qhash and child listens</div><div class='add'>+ * @iwdev: iWarp device</div><div class='add'>+ * @cm_info: CM info for parent listen node</div><div class='add'>+ * @cm_parent_listen_node: The parent listen node</div><div class='add'>+ */</div><div class='add'>+static int irdma_del_multiple_qhash(struct irdma_device *iwdev,</div><div class='add'>+				    struct irdma_cm_info *cm_info,</div><div class='add'>+				    struct irdma_cm_listener *cm_parent_listen_node)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_listener *child_listen_node;</div><div class='add'>+	struct list_head *pos, *tpos;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int ret = -EINVAL;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwdev-&gt;cm_core.listen_list_lock, flags);</div><div class='add'>+	list_for_each_safe (pos, tpos,</div><div class='add'>+			    &amp;cm_parent_listen_node-&gt;child_listen_list) {</div><div class='add'>+		child_listen_node = list_entry(pos, struct irdma_cm_listener,</div><div class='add'>+					       child_listen_list);</div><div class='add'>+		if (child_listen_node-&gt;ipv4)</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: removing child listen for IP=%pI4, port=%d, vlan=%d\n",</div><div class='add'>+				  child_listen_node-&gt;loc_addr,</div><div class='add'>+				  child_listen_node-&gt;loc_port,</div><div class='add'>+				  child_listen_node-&gt;vlan_id);</div><div class='add'>+		else</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: removing child listen for IP=%pI6, port=%d, vlan=%d\n",</div><div class='add'>+				  child_listen_node-&gt;loc_addr,</div><div class='add'>+				  child_listen_node-&gt;loc_port,</div><div class='add'>+				  child_listen_node-&gt;vlan_id);</div><div class='add'>+		trace_irdma_del_multiple_qhash(child_listen_node);</div><div class='add'>+		list_del(pos);</div><div class='add'>+		memcpy(cm_info-&gt;loc_addr, child_listen_node-&gt;loc_addr,</div><div class='add'>+		       sizeof(cm_info-&gt;loc_addr));</div><div class='add'>+		cm_info-&gt;vlan_id = child_listen_node-&gt;vlan_id;</div><div class='add'>+		if (child_listen_node-&gt;qhash_set) {</div><div class='add'>+			ret = irdma_manage_qhash(iwdev, cm_info,</div><div class='add'>+						 IRDMA_QHASH_TYPE_TCP_SYN,</div><div class='add'>+						 IRDMA_QHASH_MANAGE_TYPE_DELETE,</div><div class='add'>+						 NULL, false);</div><div class='add'>+			child_listen_node-&gt;qhash_set = false;</div><div class='add'>+		} else {</div><div class='add'>+			ret = 0;</div><div class='add'>+		}</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: Child listen node freed = %p\n",</div><div class='add'>+			  child_listen_node);</div><div class='add'>+		kfree(child_listen_node);</div><div class='add'>+		cm_parent_listen_node-&gt;cm_core-&gt;stats_listen_nodes_destroyed++;</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwdev-&gt;cm_core.listen_list_lock, flags);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u8 irdma_iw_get_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4)</div><div class='add'>+{</div><div class='add'>+	struct net_device *ndev = NULL;</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	if (ipv4) {</div><div class='add'>+		ndev = ip_dev_find(&amp;init_net, htonl(loc_addr[0]));</div><div class='add'>+	} else if (IS_ENABLED(CONFIG_IPV6)) {</div><div class='add'>+		struct net_device *ip_dev;</div><div class='add'>+		struct in6_addr laddr6;</div><div class='add'>+</div><div class='add'>+		irdma_copy_ip_htonl(laddr6.in6_u.u6_addr32, loc_addr);</div><div class='add'>+</div><div class='add'>+		for_each_netdev_rcu (&amp;init_net, ip_dev) {</div><div class='add'>+			if (ipv6_chk_addr(&amp;init_net, &amp;laddr6, ip_dev, 1)) {</div><div class='add'>+				ndev = ip_dev;</div><div class='add'>+				break;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!ndev)</div><div class='add'>+		goto done;</div><div class='add'>+	if (is_vlan_dev(ndev))</div><div class='add'>+		prio = (vlan_dev_get_egress_qos_mask(ndev, prio) &amp; VLAN_PRIO_MASK)</div><div class='add'>+			&gt;&gt; VLAN_PRIO_SHIFT;</div><div class='add'>+	if (ipv4)</div><div class='add'>+		dev_put(ndev);</div><div class='add'>+</div><div class='add'>+done:</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+</div><div class='add'>+	return prio;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_vlan_mac_ipv6 - Gets the vlan and mac</div><div class='add'>+ * @addr: local IPv6 address</div><div class='add'>+ * @vlan_id: vlan id for the given IPv6 address</div><div class='add'>+ * @mac: mac address for the given IPv6 address</div><div class='add'>+ *</div><div class='add'>+ * Returns the vlan id and mac for an IPv6 address.</div><div class='add'>+ */</div><div class='add'>+void irdma_get_vlan_mac_ipv6(u32 *addr, u16 *vlan_id, u8 *mac)</div><div class='add'>+{</div><div class='add'>+	struct net_device *ip_dev = NULL;</div><div class='add'>+	struct in6_addr laddr6;</div><div class='add'>+</div><div class='add'>+	if (!IS_ENABLED(CONFIG_IPV6))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	irdma_copy_ip_htonl(laddr6.in6_u.u6_addr32, addr);</div><div class='add'>+	if (vlan_id)</div><div class='add'>+		*vlan_id = 0xFFFF;	/* Match rdma_vlan_dev_vlan_id() */</div><div class='add'>+	if (mac)</div><div class='add'>+		eth_zero_addr(mac);</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	for_each_netdev_rcu (&amp;init_net, ip_dev) {</div><div class='add'>+		if (ipv6_chk_addr(&amp;init_net, &amp;laddr6, ip_dev, 1)) {</div><div class='add'>+			if (vlan_id)</div><div class='add'>+				*vlan_id = rdma_vlan_dev_vlan_id(ip_dev);</div><div class='add'>+			if (ip_dev-&gt;dev_addr &amp;&amp; mac)</div><div class='add'>+				ether_addr_copy(mac, ip_dev-&gt;dev_addr);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_vlan_ipv4 - Returns the vlan_id for IPv4 address</div><div class='add'>+ * @addr: local IPv4 address</div><div class='add'>+ */</div><div class='add'>+u16 irdma_get_vlan_ipv4(u32 *addr)</div><div class='add'>+{</div><div class='add'>+	struct net_device *netdev;</div><div class='add'>+	u16 vlan_id = 0xFFFF;</div><div class='add'>+</div><div class='add'>+	netdev = ip_dev_find(&amp;init_net, htonl(addr[0]));</div><div class='add'>+	if (netdev) {</div><div class='add'>+		vlan_id = rdma_vlan_dev_vlan_id(netdev);</div><div class='add'>+		dev_put(netdev);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return vlan_id;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_add_mqh_6 - Adds multiple qhashes for IPv6</div><div class='add'>+ * @iwdev: iWarp device</div><div class='add'>+ * @cm_info: CM info for parent listen node</div><div class='add'>+ * @cm_parent_listen_node: The parent listen node</div><div class='add'>+ *</div><div class='add'>+ * Adds a qhash and a child listen node for every IPv6 address</div><div class='add'>+ * on the adapter and adds the associated qhash filter</div><div class='add'>+ */</div><div class='add'>+static int irdma_add_mqh_6(struct irdma_device *iwdev,</div><div class='add'>+			   struct irdma_cm_info *cm_info,</div><div class='add'>+			   struct irdma_cm_listener *cm_parent_listen_node)</div><div class='add'>+{</div><div class='add'>+	struct net_device *ip_dev;</div><div class='add'>+	struct inet6_dev *idev;</div><div class='add'>+	struct inet6_ifaddr *ifp, *tmp;</div><div class='add'>+	struct irdma_cm_listener *child_listen_node;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	rtnl_lock();</div><div class='add'>+	for_each_netdev(&amp;init_net, ip_dev) {</div><div class='add'>+		if (!(ip_dev-&gt;flags &amp; IFF_UP))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		if (((rdma_vlan_dev_vlan_id(ip_dev) &gt;= VLAN_N_VID) ||</div><div class='add'>+		     (rdma_vlan_dev_real_dev(ip_dev) != iwdev-&gt;netdev)) &amp;&amp;</div><div class='add'>+		    ip_dev != iwdev-&gt;netdev)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		idev = __in6_dev_get(ip_dev);</div><div class='add'>+		if (!idev) {</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev, "CM: idev == NULL\n");</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		list_for_each_entry_safe (ifp, tmp, &amp;idev-&gt;addr_list, if_list) {</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev, "CM: IP=%pI6, vlan_id=%d, MAC=%pM\n",</div><div class='add'>+				  &amp;ifp-&gt;addr, rdma_vlan_dev_vlan_id(ip_dev),</div><div class='add'>+				  ip_dev-&gt;dev_addr);</div><div class='add'>+			child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL);</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev, "CM: Allocating child listener %p\n",</div><div class='add'>+				  child_listen_node);</div><div class='add'>+			if (!child_listen_node) {</div><div class='add'>+				ibdev_dbg(&amp;iwdev-&gt;ibdev, "CM: listener memory allocation\n");</div><div class='add'>+				ret = -ENOMEM;</div><div class='add'>+				goto exit;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			cm_info-&gt;vlan_id = rdma_vlan_dev_vlan_id(ip_dev);</div><div class='add'>+			cm_parent_listen_node-&gt;vlan_id = cm_info-&gt;vlan_id;</div><div class='add'>+			memcpy(child_listen_node, cm_parent_listen_node,</div><div class='add'>+			       sizeof(*child_listen_node));</div><div class='add'>+			irdma_copy_ip_ntohl(child_listen_node-&gt;loc_addr,</div><div class='add'>+					    ifp-&gt;addr.in6_u.u6_addr32);</div><div class='add'>+			memcpy(cm_info-&gt;loc_addr, child_listen_node-&gt;loc_addr,</div><div class='add'>+			       sizeof(cm_info-&gt;loc_addr));</div><div class='add'>+			if (!iwdev-&gt;vsi.dscp_mode)</div><div class='add'>+				cm_info-&gt;user_pri =</div><div class='add'>+				irdma_iw_get_vlan_prio(child_listen_node-&gt;loc_addr,</div><div class='add'>+						       cm_info-&gt;user_pri,</div><div class='add'>+						       false);</div><div class='add'>+</div><div class='add'>+			ret = irdma_manage_qhash(iwdev, cm_info,</div><div class='add'>+						 IRDMA_QHASH_TYPE_TCP_SYN,</div><div class='add'>+						 IRDMA_QHASH_MANAGE_TYPE_ADD,</div><div class='add'>+						 NULL, true);</div><div class='add'>+			if (ret) {</div><div class='add'>+				kfree(child_listen_node);</div><div class='add'>+				continue;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			trace_irdma_add_mqh_6(iwdev, child_listen_node,</div><div class='add'>+					      ip_dev-&gt;dev_addr);</div><div class='add'>+</div><div class='add'>+			child_listen_node-&gt;qhash_set = true;</div><div class='add'>+			spin_lock_irqsave(&amp;iwdev-&gt;cm_core.listen_list_lock, flags);</div><div class='add'>+			list_add(&amp;child_listen_node-&gt;child_listen_list,</div><div class='add'>+				 &amp;cm_parent_listen_node-&gt;child_listen_list);</div><div class='add'>+			spin_unlock_irqrestore(&amp;iwdev-&gt;cm_core.listen_list_lock, flags);</div><div class='add'>+			cm_parent_listen_node-&gt;cm_core-&gt;stats_listen_nodes_created++;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+exit:</div><div class='add'>+	rtnl_unlock();</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_add_mqh_4 - Adds multiple qhashes for IPv4</div><div class='add'>+ * @iwdev: iWarp device</div><div class='add'>+ * @cm_info: CM info for parent listen node</div><div class='add'>+ * @cm_parent_listen_node: The parent listen node</div><div class='add'>+ *</div><div class='add'>+ * Adds a qhash and a child listen node for every IPv4 address</div><div class='add'>+ * on the adapter and adds the associated qhash filter</div><div class='add'>+ */</div><div class='add'>+static int irdma_add_mqh_4(struct irdma_device *iwdev,</div><div class='add'>+			   struct irdma_cm_info *cm_info,</div><div class='add'>+			   struct irdma_cm_listener *cm_parent_listen_node)</div><div class='add'>+{</div><div class='add'>+	struct net_device *ip_dev;</div><div class='add'>+	struct in_device *idev;</div><div class='add'>+	struct irdma_cm_listener *child_listen_node;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	const struct in_ifaddr *ifa;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	rtnl_lock();</div><div class='add'>+	for_each_netdev(&amp;init_net, ip_dev) {</div><div class='add'>+		if (!(ip_dev-&gt;flags &amp; IFF_UP))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		if (((rdma_vlan_dev_vlan_id(ip_dev) &gt;= VLAN_N_VID) ||</div><div class='add'>+		     (rdma_vlan_dev_real_dev(ip_dev) != iwdev-&gt;netdev)) &amp;&amp;</div><div class='add'>+		    ip_dev != iwdev-&gt;netdev)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		idev = in_dev_get(ip_dev);</div><div class='add'>+		if (!idev)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		in_dev_for_each_ifa_rtnl(ifa, idev) {</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: Allocating child CM Listener forIP=%pI4, vlan_id=%d, MAC=%pM\n",</div><div class='add'>+				  &amp;ifa-&gt;ifa_address, rdma_vlan_dev_vlan_id(ip_dev),</div><div class='add'>+				  ip_dev-&gt;dev_addr);</div><div class='add'>+			child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL);</div><div class='add'>+			cm_parent_listen_node-&gt;cm_core-&gt;stats_listen_nodes_created++;</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev, "CM: Allocating child listener %p\n",</div><div class='add'>+				  child_listen_node);</div><div class='add'>+			if (!child_listen_node) {</div><div class='add'>+				ibdev_dbg(&amp;iwdev-&gt;ibdev, "CM: listener memory allocation\n");</div><div class='add'>+				in_dev_put(idev);</div><div class='add'>+				ret = -ENOMEM;</div><div class='add'>+				goto exit;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			cm_info-&gt;vlan_id = rdma_vlan_dev_vlan_id(ip_dev);</div><div class='add'>+			cm_parent_listen_node-&gt;vlan_id = cm_info-&gt;vlan_id;</div><div class='add'>+			memcpy(child_listen_node, cm_parent_listen_node,</div><div class='add'>+			       sizeof(*child_listen_node));</div><div class='add'>+			child_listen_node-&gt;loc_addr[0] =</div><div class='add'>+				ntohl(ifa-&gt;ifa_address);</div><div class='add'>+			memcpy(cm_info-&gt;loc_addr, child_listen_node-&gt;loc_addr,</div><div class='add'>+			       sizeof(cm_info-&gt;loc_addr));</div><div class='add'>+			if (!iwdev-&gt;vsi.dscp_mode)</div><div class='add'>+				cm_info-&gt;user_pri =</div><div class='add'>+				irdma_iw_get_vlan_prio(child_listen_node-&gt;loc_addr,</div><div class='add'>+						       cm_info-&gt;user_pri,</div><div class='add'>+						       true);</div><div class='add'>+			ret = irdma_manage_qhash(iwdev, cm_info,</div><div class='add'>+						 IRDMA_QHASH_TYPE_TCP_SYN,</div><div class='add'>+						 IRDMA_QHASH_MANAGE_TYPE_ADD,</div><div class='add'>+						 NULL, true);</div><div class='add'>+			if (ret) {</div><div class='add'>+				kfree(child_listen_node);</div><div class='add'>+				cm_parent_listen_node-&gt;cm_core</div><div class='add'>+					-&gt;stats_listen_nodes_created--;</div><div class='add'>+				continue;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			trace_irdma_add_mqh_4(iwdev, child_listen_node,</div><div class='add'>+					      ip_dev-&gt;dev_addr);</div><div class='add'>+</div><div class='add'>+			child_listen_node-&gt;qhash_set = true;</div><div class='add'>+			spin_lock_irqsave(&amp;iwdev-&gt;cm_core.listen_list_lock,</div><div class='add'>+					  flags);</div><div class='add'>+			list_add(&amp;child_listen_node-&gt;child_listen_list,</div><div class='add'>+				 &amp;cm_parent_listen_node-&gt;child_listen_list);</div><div class='add'>+			spin_unlock_irqrestore(&amp;iwdev-&gt;cm_core.listen_list_lock, flags);</div><div class='add'>+		}</div><div class='add'>+		in_dev_put(idev);</div><div class='add'>+	}</div><div class='add'>+exit:</div><div class='add'>+	rtnl_unlock();</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_add_mqh - Adds multiple qhashes</div><div class='add'>+ * @iwdev: iWarp device</div><div class='add'>+ * @cm_info: CM info for parent listen node</div><div class='add'>+ * @cm_listen_node: The parent listen node</div><div class='add'>+ */</div><div class='add'>+static int irdma_add_mqh(struct irdma_device *iwdev,</div><div class='add'>+			 struct irdma_cm_info *cm_info,</div><div class='add'>+			 struct irdma_cm_listener *cm_listen_node)</div><div class='add'>+{</div><div class='add'>+	if (cm_info-&gt;ipv4)</div><div class='add'>+		return irdma_add_mqh_4(iwdev, cm_info, cm_listen_node);</div><div class='add'>+	else</div><div class='add'>+		return irdma_add_mqh_6(iwdev, cm_info, cm_listen_node);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_reset_list_prep - add connection nodes slated for reset to list</div><div class='add'>+ * @cm_core: cm's core</div><div class='add'>+ * @listener: pointer to listener node</div><div class='add'>+ * @reset_list: a list to which cm_node will be selected</div><div class='add'>+ */</div><div class='add'>+static void irdma_reset_list_prep(struct irdma_cm_core *cm_core,</div><div class='add'>+				  struct irdma_cm_listener *listener,</div><div class='add'>+				  struct list_head *reset_list)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+	int bkt;</div><div class='add'>+</div><div class='add'>+	hash_for_each_rcu(cm_core-&gt;cm_hash_tbl, bkt, cm_node, list) {</div><div class='add'>+		if (cm_node-&gt;listener == listener &amp;&amp;</div><div class='add'>+		    !cm_node-&gt;accelerated &amp;&amp;</div><div class='add'>+		    refcount_inc_not_zero(&amp;cm_node-&gt;refcnt))</div><div class='add'>+			list_add(&amp;cm_node-&gt;reset_entry, reset_list);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_dec_refcnt_listen - delete listener and associated cm nodes</div><div class='add'>+ * @cm_core: cm's core</div><div class='add'>+ * @listener: pointer to listener node</div><div class='add'>+ * @free_hanging_nodes: to free associated cm_nodes</div><div class='add'>+ * @apbvt_del: flag to delete the apbvt</div><div class='add'>+ */</div><div class='add'>+static int irdma_dec_refcnt_listen(struct irdma_cm_core *cm_core,</div><div class='add'>+				   struct irdma_cm_listener *listener,</div><div class='add'>+				   int free_hanging_nodes, bool apbvt_del)</div><div class='add'>+{</div><div class='add'>+	int err;</div><div class='add'>+	struct list_head *list_pos;</div><div class='add'>+	struct list_head *list_temp;</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+	struct list_head reset_list;</div><div class='add'>+	struct irdma_cm_info nfo;</div><div class='add'>+	enum irdma_cm_node_state old_state;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	trace_irdma_dec_refcnt_listen(listener, __builtin_return_address(0));</div><div class='add'>+	/* free non-accelerated child nodes for this listener */</div><div class='add'>+	INIT_LIST_HEAD(&amp;reset_list);</div><div class='add'>+	if (free_hanging_nodes) {</div><div class='add'>+		rcu_read_lock();</div><div class='add'>+		irdma_reset_list_prep(cm_core, listener, &amp;reset_list);</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	list_for_each_safe (list_pos, list_temp, &amp;reset_list) {</div><div class='add'>+		cm_node = container_of(list_pos, struct irdma_cm_node,</div><div class='add'>+				       reset_entry);</div><div class='add'>+		if (cm_node-&gt;state &gt;= IRDMA_CM_STATE_FIN_WAIT1) {</div><div class='add'>+			irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		err = irdma_send_reset(cm_node);</div><div class='add'>+		if (err) {</div><div class='add'>+			cm_node-&gt;state = IRDMA_CM_STATE_CLOSED;</div><div class='add'>+			ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: send reset failed\n");</div><div class='add'>+		} else {</div><div class='add'>+			old_state = cm_node-&gt;state;</div><div class='add'>+			cm_node-&gt;state = IRDMA_CM_STATE_LISTENER_DESTROYED;</div><div class='add'>+			if (old_state != IRDMA_CM_STATE_MPAREQ_RCVD)</div><div class='add'>+				irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (refcount_dec_and_test(&amp;listener-&gt;refcnt)) {</div><div class='add'>+		spin_lock_irqsave(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='add'>+		list_del(&amp;listener-&gt;list);</div><div class='add'>+		spin_unlock_irqrestore(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='add'>+</div><div class='add'>+		if (apbvt_del)</div><div class='add'>+			irdma_del_apbvt(listener-&gt;iwdev,</div><div class='add'>+					listener-&gt;apbvt_entry);</div><div class='add'>+		memcpy(nfo.loc_addr, listener-&gt;loc_addr, sizeof(nfo.loc_addr));</div><div class='add'>+		nfo.loc_port = listener-&gt;loc_port;</div><div class='add'>+		nfo.ipv4 = listener-&gt;ipv4;</div><div class='add'>+		nfo.vlan_id = listener-&gt;vlan_id;</div><div class='add'>+		nfo.user_pri = listener-&gt;user_pri;</div><div class='add'>+		nfo.qh_qpid = listener-&gt;iwdev-&gt;vsi.ilq-&gt;qp_id;</div><div class='add'>+</div><div class='add'>+		if (!list_empty(&amp;listener-&gt;child_listen_list)) {</div><div class='add'>+			irdma_del_multiple_qhash(listener-&gt;iwdev, &amp;nfo,</div><div class='add'>+						 listener);</div><div class='add'>+		} else {</div><div class='add'>+			if (listener-&gt;qhash_set)</div><div class='add'>+				irdma_manage_qhash(listener-&gt;iwdev,</div><div class='add'>+						   &amp;nfo,</div><div class='add'>+						   IRDMA_QHASH_TYPE_TCP_SYN,</div><div class='add'>+						   IRDMA_QHASH_MANAGE_TYPE_DELETE,</div><div class='add'>+						   NULL, false);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		cm_core-&gt;stats_listen_destroyed++;</div><div class='add'>+		cm_core-&gt;stats_listen_nodes_destroyed++;</div><div class='add'>+		ibdev_dbg(&amp;listener-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d apbvt_del=%d\n",</div><div class='add'>+			  listener-&gt;loc_port, listener-&gt;loc_addr, listener,</div><div class='add'>+			  listener-&gt;cm_id, listener-&gt;qhash_set,</div><div class='add'>+			  listener-&gt;vlan_id, apbvt_del);</div><div class='add'>+		kfree(listener);</div><div class='add'>+		listener = NULL;</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return -EINVAL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cm_del_listen - delete a listener</div><div class='add'>+ * @cm_core: cm's core</div><div class='add'>+ * @listener: passive connection's listener</div><div class='add'>+ * @apbvt_del: flag to delete apbvt</div><div class='add'>+ */</div><div class='add'>+static int irdma_cm_del_listen(struct irdma_cm_core *cm_core,</div><div class='add'>+			       struct irdma_cm_listener *listener,</div><div class='add'>+			       bool apbvt_del)</div><div class='add'>+{</div><div class='add'>+	listener-&gt;listener_state = IRDMA_CM_LISTENER_PASSIVE_STATE;</div><div class='add'>+	listener-&gt;cm_id = NULL;</div><div class='add'>+</div><div class='add'>+	return irdma_dec_refcnt_listen(cm_core, listener, 1, apbvt_del);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_addr_resolve_neigh - resolve neighbor address</div><div class='add'>+ * @iwdev: iwarp device structure</div><div class='add'>+ * @src_ip: local ip address</div><div class='add'>+ * @dst_ip: remote ip address</div><div class='add'>+ * @arpindex: if there is an arp entry</div><div class='add'>+ */</div><div class='add'>+static int irdma_addr_resolve_neigh(struct irdma_device *iwdev, u32 src_ip,</div><div class='add'>+				    u32 dst_ip, int arpindex)</div><div class='add'>+{</div><div class='add'>+	struct rtable *rt;</div><div class='add'>+	struct neighbour *neigh;</div><div class='add'>+	int rc = arpindex;</div><div class='add'>+	__be32 dst_ipaddr = htonl(dst_ip);</div><div class='add'>+	__be32 src_ipaddr = htonl(src_ip);</div><div class='add'>+</div><div class='add'>+	rt = ip_route_output(&amp;init_net, dst_ipaddr, src_ipaddr, 0, 0,</div><div class='add'>+			     RT_SCOPE_UNIVERSE);</div><div class='add'>+	if (IS_ERR(rt)) {</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev, "CM: ip_route_output fail\n");</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	neigh = dst_neigh_lookup(&amp;rt-&gt;dst, &amp;dst_ipaddr);</div><div class='add'>+	if (!neigh)</div><div class='add'>+		goto exit;</div><div class='add'>+</div><div class='add'>+	if (neigh-&gt;nud_state &amp; NUD_VALID)</div><div class='add'>+		rc = irdma_add_arp(iwdev-&gt;rf, &amp;dst_ip, true, neigh-&gt;ha);</div><div class='add'>+	else</div><div class='add'>+		neigh_event_send(neigh, NULL);</div><div class='add'>+	if (neigh)</div><div class='add'>+		neigh_release(neigh);</div><div class='add'>+exit:</div><div class='add'>+	ip_rt_put(rt);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_dst_ipv6 - get destination cache entry via ipv6 lookup</div><div class='add'>+ * @src_addr: local ipv6 sock address</div><div class='add'>+ * @dst_addr: destination ipv6 sock address</div><div class='add'>+ */</div><div class='add'>+static struct dst_entry *irdma_get_dst_ipv6(struct sockaddr_in6 *src_addr,</div><div class='add'>+					    struct sockaddr_in6 *dst_addr)</div><div class='add'>+{</div><div class='add'>+	struct dst_entry *dst = NULL;</div><div class='add'>+</div><div class='add'>+	if ((IS_ENABLED(CONFIG_IPV6))) {</div><div class='add'>+		struct flowi6 fl6 = {};</div><div class='add'>+</div><div class='add'>+		fl6.daddr = dst_addr-&gt;sin6_addr;</div><div class='add'>+		fl6.saddr = src_addr-&gt;sin6_addr;</div><div class='add'>+		if (ipv6_addr_type(&amp;fl6.daddr) &amp; IPV6_ADDR_LINKLOCAL)</div><div class='add'>+			fl6.flowi6_oif = dst_addr-&gt;sin6_scope_id;</div><div class='add'>+</div><div class='add'>+		dst = ip6_route_output(&amp;init_net, NULL, &amp;fl6);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return dst;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_addr_resolve_neigh_ipv6 - resolve neighbor ipv6 address</div><div class='add'>+ * @iwdev: iwarp device structure</div><div class='add'>+ * @src: local ip address</div><div class='add'>+ * @dest: remote ip address</div><div class='add'>+ * @arpindex: if there is an arp entry</div><div class='add'>+ */</div><div class='add'>+static int irdma_addr_resolve_neigh_ipv6(struct irdma_device *iwdev, u32 *src,</div><div class='add'>+					 u32 *dest, int arpindex)</div><div class='add'>+{</div><div class='add'>+	struct neighbour *neigh;</div><div class='add'>+	int rc = arpindex;</div><div class='add'>+	struct dst_entry *dst;</div><div class='add'>+	struct sockaddr_in6 dst_addr = {};</div><div class='add'>+	struct sockaddr_in6 src_addr = {};</div><div class='add'>+</div><div class='add'>+	dst_addr.sin6_family = AF_INET6;</div><div class='add'>+	irdma_copy_ip_htonl(dst_addr.sin6_addr.in6_u.u6_addr32, dest);</div><div class='add'>+	src_addr.sin6_family = AF_INET6;</div><div class='add'>+	irdma_copy_ip_htonl(src_addr.sin6_addr.in6_u.u6_addr32, src);</div><div class='add'>+	dst = irdma_get_dst_ipv6(&amp;src_addr, &amp;dst_addr);</div><div class='add'>+	if (!dst || dst-&gt;error) {</div><div class='add'>+		if (dst) {</div><div class='add'>+			dst_release(dst);</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: ip6_route_output returned dst-&gt;error = %d\n",</div><div class='add'>+				  dst-&gt;error);</div><div class='add'>+		}</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	neigh = dst_neigh_lookup(dst, dst_addr.sin6_addr.in6_u.u6_addr32);</div><div class='add'>+	if (!neigh)</div><div class='add'>+		goto exit;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev, "CM: dst_neigh_lookup MAC=%pM\n",</div><div class='add'>+		  neigh-&gt;ha);</div><div class='add'>+</div><div class='add'>+	trace_irdma_addr_resolve(iwdev, neigh-&gt;ha);</div><div class='add'>+</div><div class='add'>+	if (neigh-&gt;nud_state &amp; NUD_VALID)</div><div class='add'>+		rc = irdma_add_arp(iwdev-&gt;rf, dest, false, neigh-&gt;ha);</div><div class='add'>+	else</div><div class='add'>+		neigh_event_send(neigh, NULL);</div><div class='add'>+	if (neigh)</div><div class='add'>+		neigh_release(neigh);</div><div class='add'>+exit:</div><div class='add'>+	dst_release(dst);</div><div class='add'>+</div><div class='add'>+	return rc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_find_node - find a cm node that matches the reference cm node</div><div class='add'>+ * @cm_core: cm's core</div><div class='add'>+ * @rem_port: remote tcp port num</div><div class='add'>+ * @rem_addr: remote ip addr</div><div class='add'>+ * @loc_port: local tcp port num</div><div class='add'>+ * @loc_addr: local ip addr</div><div class='add'>+ * @vlan_id: local VLAN ID</div><div class='add'>+ */</div><div class='add'>+struct irdma_cm_node *irdma_find_node(struct irdma_cm_core *cm_core,</div><div class='add'>+				      u16 rem_port, u32 *rem_addr, u16 loc_port,</div><div class='add'>+				      u32 *loc_addr, u16 vlan_id)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+	u32 key = (rem_port &lt;&lt; 16) | loc_port;</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	hash_for_each_possible_rcu(cm_core-&gt;cm_hash_tbl, cm_node, list, key) {</div><div class='add'>+		if (cm_node-&gt;vlan_id == vlan_id &amp;&amp;</div><div class='add'>+		    cm_node-&gt;loc_port == loc_port &amp;&amp; cm_node-&gt;rem_port == rem_port &amp;&amp;</div><div class='add'>+		    !memcmp(cm_node-&gt;loc_addr, loc_addr, sizeof(cm_node-&gt;loc_addr)) &amp;&amp;</div><div class='add'>+		    !memcmp(cm_node-&gt;rem_addr, rem_addr, sizeof(cm_node-&gt;rem_addr))) {</div><div class='add'>+			if (!refcount_inc_not_zero(&amp;cm_node-&gt;refcnt))</div><div class='add'>+				goto exit;</div><div class='add'>+			rcu_read_unlock();</div><div class='add'>+			trace_irdma_find_node(cm_node, 0, NULL);</div><div class='add'>+			return cm_node;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+exit:</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+</div><div class='add'>+	/* no owner node */</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_add_hte_node - add a cm node to the hash table</div><div class='add'>+ * @cm_core: cm's core</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ */</div><div class='add'>+static void irdma_add_hte_node(struct irdma_cm_core *cm_core,</div><div class='add'>+			       struct irdma_cm_node *cm_node)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	u32 key = (cm_node-&gt;rem_port &lt;&lt; 16) | cm_node-&gt;loc_port;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cm_core-&gt;ht_lock, flags);</div><div class='add'>+	hash_add_rcu(cm_core-&gt;cm_hash_tbl, &amp;cm_node-&gt;list, key);</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ipv4_is_lpb - check if loopback</div><div class='add'>+ * @loc_addr: local addr to compare</div><div class='add'>+ * @rem_addr: remote address</div><div class='add'>+ */</div><div class='add'>+bool irdma_ipv4_is_lpb(u32 loc_addr, u32 rem_addr)</div><div class='add'>+{</div><div class='add'>+	return ipv4_is_loopback(htonl(rem_addr)) || (loc_addr == rem_addr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ipv6_is_lpb - check if loopback</div><div class='add'>+ * @loc_addr: local addr to compare</div><div class='add'>+ * @rem_addr: remote address</div><div class='add'>+ */</div><div class='add'>+bool irdma_ipv6_is_lpb(u32 *loc_addr, u32 *rem_addr)</div><div class='add'>+{</div><div class='add'>+	struct in6_addr raddr6;</div><div class='add'>+</div><div class='add'>+	irdma_copy_ip_htonl(raddr6.in6_u.u6_addr32, rem_addr);</div><div class='add'>+</div><div class='add'>+	return !memcmp(loc_addr, rem_addr, 16) || ipv6_addr_loopback(&amp;raddr6);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cm_create_ah - create a cm address handle</div><div class='add'>+ * @cm_node: The connection manager node to create AH for</div><div class='add'>+ * @wait: Provides option to wait for ah creation or not</div><div class='add'>+ */</div><div class='add'>+static int irdma_cm_create_ah(struct irdma_cm_node *cm_node, bool wait)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ah_info ah_info = {};</div><div class='add'>+	struct irdma_device *iwdev = cm_node-&gt;iwdev;</div><div class='add'>+</div><div class='add'>+	ether_addr_copy(ah_info.mac_addr, iwdev-&gt;netdev-&gt;dev_addr);</div><div class='add'>+</div><div class='add'>+	ah_info.hop_ttl = 0x40;</div><div class='add'>+	ah_info.tc_tos = cm_node-&gt;tos;</div><div class='add'>+	ah_info.vsi = &amp;iwdev-&gt;vsi;</div><div class='add'>+</div><div class='add'>+	if (cm_node-&gt;ipv4) {</div><div class='add'>+		ah_info.ipv4_valid = true;</div><div class='add'>+		ah_info.dest_ip_addr[0] = cm_node-&gt;rem_addr[0];</div><div class='add'>+		ah_info.src_ip_addr[0] = cm_node-&gt;loc_addr[0];</div><div class='add'>+		ah_info.do_lpbk = irdma_ipv4_is_lpb(ah_info.src_ip_addr[0],</div><div class='add'>+						    ah_info.dest_ip_addr[0]);</div><div class='add'>+	} else {</div><div class='add'>+		memcpy(ah_info.dest_ip_addr, cm_node-&gt;rem_addr,</div><div class='add'>+		       sizeof(ah_info.dest_ip_addr));</div><div class='add'>+		memcpy(ah_info.src_ip_addr, cm_node-&gt;loc_addr,</div><div class='add'>+		       sizeof(ah_info.src_ip_addr));</div><div class='add'>+		ah_info.do_lpbk = irdma_ipv6_is_lpb(ah_info.src_ip_addr,</div><div class='add'>+						    ah_info.dest_ip_addr);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ah_info.vlan_tag = cm_node-&gt;vlan_id;</div><div class='add'>+	if (cm_node-&gt;vlan_id &lt; VLAN_N_VID) {</div><div class='add'>+		ah_info.insert_vlan_tag = 1;</div><div class='add'>+		ah_info.vlan_tag |= cm_node-&gt;user_pri &lt;&lt; VLAN_PRIO_SHIFT;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ah_info.dst_arpindex =</div><div class='add'>+		irdma_arp_table(iwdev-&gt;rf, ah_info.dest_ip_addr,</div><div class='add'>+				ah_info.ipv4_valid, NULL, IRDMA_ARP_RESOLVE);</div><div class='add'>+</div><div class='add'>+	if (irdma_puda_create_ah(&amp;iwdev-&gt;rf-&gt;sc_dev, &amp;ah_info, wait,</div><div class='add'>+				 IRDMA_PUDA_RSRC_TYPE_ILQ, cm_node,</div><div class='add'>+				 &amp;cm_node-&gt;ah))</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	trace_irdma_create_ah(cm_node);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cm_free_ah - free a cm address handle</div><div class='add'>+ * @cm_node: The connection manager node to create AH for</div><div class='add'>+ */</div><div class='add'>+static void irdma_cm_free_ah(struct irdma_cm_node *cm_node)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = cm_node-&gt;iwdev;</div><div class='add'>+</div><div class='add'>+	trace_irdma_cm_free_ah(cm_node);</div><div class='add'>+	irdma_puda_free_ah(&amp;iwdev-&gt;rf-&gt;sc_dev, cm_node-&gt;ah);</div><div class='add'>+	cm_node-&gt;ah = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_make_cm_node - create a new instance of a cm node</div><div class='add'>+ * @cm_core: cm's core</div><div class='add'>+ * @iwdev: iwarp device structure</div><div class='add'>+ * @cm_info: quad info for connection</div><div class='add'>+ * @listener: passive connection's listener</div><div class='add'>+ */</div><div class='add'>+static struct irdma_cm_node *</div><div class='add'>+irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev,</div><div class='add'>+		   struct irdma_cm_info *cm_info,</div><div class='add'>+		   struct irdma_cm_listener *listener)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+	int oldarpindex;</div><div class='add'>+	int arpindex;</div><div class='add'>+	struct net_device *netdev = iwdev-&gt;netdev;</div><div class='add'>+</div><div class='add'>+	/* create an hte and cm_node for this instance */</div><div class='add'>+	cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC);</div><div class='add'>+	if (!cm_node)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	/* set our node specific transport info */</div><div class='add'>+	cm_node-&gt;ipv4 = cm_info-&gt;ipv4;</div><div class='add'>+	cm_node-&gt;vlan_id = cm_info-&gt;vlan_id;</div><div class='add'>+	if (cm_node-&gt;vlan_id &gt;= VLAN_N_VID &amp;&amp; iwdev-&gt;dcb_vlan_mode)</div><div class='add'>+		cm_node-&gt;vlan_id = 0;</div><div class='add'>+	cm_node-&gt;tos = cm_info-&gt;tos;</div><div class='add'>+	cm_node-&gt;user_pri = cm_info-&gt;user_pri;</div><div class='add'>+	if (listener) {</div><div class='add'>+		if (listener-&gt;tos != cm_info-&gt;tos)</div><div class='add'>+			ibdev_warn(&amp;iwdev-&gt;ibdev,</div><div class='add'>+				   "application TOS[%d] and remote client TOS[%d] mismatch\n",</div><div class='add'>+				   listener-&gt;tos, cm_info-&gt;tos);</div><div class='add'>+		if (iwdev-&gt;vsi.dscp_mode) {</div><div class='add'>+			cm_node-&gt;user_pri = listener-&gt;user_pri;</div><div class='add'>+		} else {</div><div class='add'>+			cm_node-&gt;tos = max(listener-&gt;tos, cm_info-&gt;tos);</div><div class='add'>+			cm_node-&gt;user_pri = rt_tos2priority(cm_node-&gt;tos);</div><div class='add'>+			cm_node-&gt;user_pri =</div><div class='add'>+				irdma_iw_get_vlan_prio(cm_info-&gt;loc_addr,</div><div class='add'>+						       cm_node-&gt;user_pri,</div><div class='add'>+						       cm_info-&gt;ipv4);</div><div class='add'>+		}</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+			  "DCB: listener: TOS:[%d] UP:[%d]\n", cm_node-&gt;tos,</div><div class='add'>+			  cm_node-&gt;user_pri);</div><div class='add'>+		trace_irdma_listener_tos(iwdev, cm_node-&gt;tos,</div><div class='add'>+					 cm_node-&gt;user_pri);</div><div class='add'>+	}</div><div class='add'>+	memcpy(cm_node-&gt;loc_addr, cm_info-&gt;loc_addr, sizeof(cm_node-&gt;loc_addr));</div><div class='add'>+	memcpy(cm_node-&gt;rem_addr, cm_info-&gt;rem_addr, sizeof(cm_node-&gt;rem_addr));</div><div class='add'>+	cm_node-&gt;loc_port = cm_info-&gt;loc_port;</div><div class='add'>+	cm_node-&gt;rem_port = cm_info-&gt;rem_port;</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;mpa_frame_rev = IRDMA_CM_DEFAULT_MPA_VER;</div><div class='add'>+	cm_node-&gt;send_rdma0_op = SEND_RDMA_READ_ZERO;</div><div class='add'>+	cm_node-&gt;iwdev = iwdev;</div><div class='add'>+	cm_node-&gt;dev = &amp;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;ird_size = cm_node-&gt;dev-&gt;hw_attrs.max_hw_ird;</div><div class='add'>+	cm_node-&gt;ord_size = cm_node-&gt;dev-&gt;hw_attrs.max_hw_ord;</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;listener = listener;</div><div class='add'>+	cm_node-&gt;cm_id = cm_info-&gt;cm_id;</div><div class='add'>+	ether_addr_copy(cm_node-&gt;loc_mac, netdev-&gt;dev_addr);</div><div class='add'>+	spin_lock_init(&amp;cm_node-&gt;retrans_list_lock);</div><div class='add'>+	cm_node-&gt;ack_rcvd = false;</div><div class='add'>+</div><div class='add'>+	init_completion(&amp;cm_node-&gt;establish_comp);</div><div class='add'>+	refcount_set(&amp;cm_node-&gt;refcnt, 1);</div><div class='add'>+	/* associate our parent CM core */</div><div class='add'>+	cm_node-&gt;cm_core = cm_core;</div><div class='add'>+	cm_node-&gt;tcp_cntxt.loc_id = IRDMA_CM_DEFAULT_LOCAL_ID;</div><div class='add'>+	cm_node-&gt;tcp_cntxt.rcv_wscale = iwdev-&gt;rcv_wscale;</div><div class='add'>+	cm_node-&gt;tcp_cntxt.rcv_wnd = iwdev-&gt;rcv_wnd &gt;&gt; cm_node-&gt;tcp_cntxt.rcv_wscale;</div><div class='add'>+	if (cm_node-&gt;ipv4) {</div><div class='add'>+		cm_node-&gt;tcp_cntxt.loc_seq_num = secure_tcp_seq(htonl(cm_node-&gt;loc_addr[0]),</div><div class='add'>+								htonl(cm_node-&gt;rem_addr[0]),</div><div class='add'>+								htons(cm_node-&gt;loc_port),</div><div class='add'>+								htons(cm_node-&gt;rem_port));</div><div class='add'>+		cm_node-&gt;tcp_cntxt.mss = iwdev-&gt;vsi.mtu - IRDMA_MTU_TO_MSS_IPV4;</div><div class='add'>+	} else if (IS_ENABLED(CONFIG_IPV6)) {</div><div class='add'>+		__be32 loc[4] = {</div><div class='add'>+			htonl(cm_node-&gt;loc_addr[0]), htonl(cm_node-&gt;loc_addr[1]),</div><div class='add'>+			htonl(cm_node-&gt;loc_addr[2]), htonl(cm_node-&gt;loc_addr[3])</div><div class='add'>+		};</div><div class='add'>+		__be32 rem[4] = {</div><div class='add'>+			htonl(cm_node-&gt;rem_addr[0]), htonl(cm_node-&gt;rem_addr[1]),</div><div class='add'>+			htonl(cm_node-&gt;rem_addr[2]), htonl(cm_node-&gt;rem_addr[3])</div><div class='add'>+		};</div><div class='add'>+		cm_node-&gt;tcp_cntxt.loc_seq_num = secure_tcpv6_seq(loc, rem,</div><div class='add'>+								  htons(cm_node-&gt;loc_port),</div><div class='add'>+								  htons(cm_node-&gt;rem_port));</div><div class='add'>+		cm_node-&gt;tcp_cntxt.mss = iwdev-&gt;vsi.mtu - IRDMA_MTU_TO_MSS_IPV6;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if ((cm_node-&gt;ipv4 &amp;&amp;</div><div class='add'>+	     irdma_ipv4_is_lpb(cm_node-&gt;loc_addr[0], cm_node-&gt;rem_addr[0])) ||</div><div class='add'>+	    (!cm_node-&gt;ipv4 &amp;&amp;</div><div class='add'>+	     irdma_ipv6_is_lpb(cm_node-&gt;loc_addr, cm_node-&gt;rem_addr))) {</div><div class='add'>+		cm_node-&gt;do_lpb = true;</div><div class='add'>+		arpindex = irdma_arp_table(iwdev-&gt;rf, cm_node-&gt;rem_addr,</div><div class='add'>+					   cm_node-&gt;ipv4, NULL,</div><div class='add'>+					   IRDMA_ARP_RESOLVE);</div><div class='add'>+	} else {</div><div class='add'>+		oldarpindex = irdma_arp_table(iwdev-&gt;rf, cm_node-&gt;rem_addr,</div><div class='add'>+					      cm_node-&gt;ipv4, NULL,</div><div class='add'>+					      IRDMA_ARP_RESOLVE);</div><div class='add'>+		if (cm_node-&gt;ipv4)</div><div class='add'>+			arpindex = irdma_addr_resolve_neigh(iwdev,</div><div class='add'>+							    cm_info-&gt;loc_addr[0],</div><div class='add'>+							    cm_info-&gt;rem_addr[0],</div><div class='add'>+							    oldarpindex);</div><div class='add'>+		else if (IS_ENABLED(CONFIG_IPV6))</div><div class='add'>+			arpindex = irdma_addr_resolve_neigh_ipv6(iwdev,</div><div class='add'>+								 cm_info-&gt;loc_addr,</div><div class='add'>+								 cm_info-&gt;rem_addr,</div><div class='add'>+								 oldarpindex);</div><div class='add'>+		else</div><div class='add'>+			arpindex = -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (arpindex &lt; 0)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	ether_addr_copy(cm_node-&gt;rem_mac,</div><div class='add'>+			iwdev-&gt;rf-&gt;arp_table[arpindex].mac_addr);</div><div class='add'>+	irdma_add_hte_node(cm_core, cm_node);</div><div class='add'>+	cm_core-&gt;stats_nodes_created++;</div><div class='add'>+	return cm_node;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	kfree(cm_node);</div><div class='add'>+</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_destroy_connection(struct irdma_cm_node *cm_node)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_core *cm_core = cm_node-&gt;cm_core;</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+	struct irdma_cm_info nfo;</div><div class='add'>+</div><div class='add'>+	/* if the node is destroyed before connection was accelerated */</div><div class='add'>+	if (!cm_node-&gt;accelerated &amp;&amp; cm_node-&gt;accept_pend) {</div><div class='add'>+		ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: node destroyed before established\n");</div><div class='add'>+		atomic_dec(&amp;cm_node-&gt;listener-&gt;pend_accepts_cnt);</div><div class='add'>+	}</div><div class='add'>+	if (cm_node-&gt;close_entry)</div><div class='add'>+		irdma_handle_close_entry(cm_node, 0);</div><div class='add'>+	if (cm_node-&gt;listener) {</div><div class='add'>+		irdma_dec_refcnt_listen(cm_core, cm_node-&gt;listener, 0, true);</div><div class='add'>+	} else {</div><div class='add'>+		if (cm_node-&gt;apbvt_set) {</div><div class='add'>+			irdma_del_apbvt(cm_node-&gt;iwdev, cm_node-&gt;apbvt_entry);</div><div class='add'>+			cm_node-&gt;apbvt_set = 0;</div><div class='add'>+		}</div><div class='add'>+		irdma_get_addr_info(cm_node, &amp;nfo);</div><div class='add'>+		if (cm_node-&gt;qhash_set) {</div><div class='add'>+			nfo.qh_qpid = cm_node-&gt;iwdev-&gt;vsi.ilq-&gt;qp_id;</div><div class='add'>+			irdma_manage_qhash(cm_node-&gt;iwdev, &amp;nfo,</div><div class='add'>+					   IRDMA_QHASH_TYPE_TCP_ESTABLISHED,</div><div class='add'>+					   IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL,</div><div class='add'>+					   false);</div><div class='add'>+			cm_node-&gt;qhash_set = 0;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	iwqp = cm_node-&gt;iwqp;</div><div class='add'>+	if (iwqp) {</div><div class='add'>+		cm_node-&gt;cm_id-&gt;rem_ref(cm_node-&gt;cm_id);</div><div class='add'>+		cm_node-&gt;cm_id = NULL;</div><div class='add'>+		iwqp-&gt;cm_id = NULL;</div><div class='add'>+		irdma_qp_rem_ref(&amp;iwqp-&gt;ibqp);</div><div class='add'>+		cm_node-&gt;iwqp = NULL;</div><div class='add'>+	} else if (cm_node-&gt;qhash_set) {</div><div class='add'>+		irdma_get_addr_info(cm_node, &amp;nfo);</div><div class='add'>+		nfo.qh_qpid = cm_node-&gt;iwdev-&gt;vsi.ilq-&gt;qp_id;</div><div class='add'>+		irdma_manage_qhash(cm_node-&gt;iwdev, &amp;nfo,</div><div class='add'>+				   IRDMA_QHASH_TYPE_TCP_ESTABLISHED,</div><div class='add'>+				   IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL, false);</div><div class='add'>+		cm_node-&gt;qhash_set = 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cm_core-&gt;cm_free_ah(cm_node);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_rem_ref_cm_node - destroy an instance of a cm node</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ */</div><div class='add'>+void irdma_rem_ref_cm_node(struct irdma_cm_node *cm_node)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_core *cm_core = cm_node-&gt;cm_core;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	trace_irdma_rem_ref_cm_node(cm_node, 0, __builtin_return_address(0));</div><div class='add'>+	spin_lock_irqsave(&amp;cm_core-&gt;ht_lock, flags);</div><div class='add'>+</div><div class='add'>+	if (!refcount_dec_and_test(&amp;cm_node-&gt;refcnt)) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+	if (cm_node-&gt;iwqp) {</div><div class='add'>+		cm_node-&gt;iwqp-&gt;cm_node = NULL;</div><div class='add'>+		cm_node-&gt;iwqp-&gt;cm_id = NULL;</div><div class='add'>+	}</div><div class='add'>+	hash_del_rcu(&amp;cm_node-&gt;list);</div><div class='add'>+	cm_node-&gt;cm_core-&gt;stats_nodes_destroyed++;</div><div class='add'>+</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_core-&gt;ht_lock, flags);</div><div class='add'>+</div><div class='add'>+	irdma_destroy_connection(cm_node);</div><div class='add'>+</div><div class='add'>+	kfree_rcu(cm_node, rcu_head);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_handle_fin_pkt - FIN packet received</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ */</div><div class='add'>+static void irdma_handle_fin_pkt(struct irdma_cm_node *cm_node)</div><div class='add'>+{</div><div class='add'>+	switch (cm_node-&gt;state) {</div><div class='add'>+	case IRDMA_CM_STATE_SYN_RCVD:</div><div class='add'>+	case IRDMA_CM_STATE_SYN_SENT:</div><div class='add'>+	case IRDMA_CM_STATE_ESTABLISHED:</div><div class='add'>+	case IRDMA_CM_STATE_MPAREJ_RCVD:</div><div class='add'>+		cm_node-&gt;tcp_cntxt.rcv_nxt++;</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_LAST_ACK;</div><div class='add'>+		irdma_send_fin(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_MPAREQ_SENT:</div><div class='add'>+		irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED);</div><div class='add'>+		cm_node-&gt;tcp_cntxt.rcv_nxt++;</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_CLOSED;</div><div class='add'>+		refcount_inc(&amp;cm_node-&gt;refcnt);</div><div class='add'>+		irdma_send_reset(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_FIN_WAIT1:</div><div class='add'>+		cm_node-&gt;tcp_cntxt.rcv_nxt++;</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_CLOSING;</div><div class='add'>+		irdma_send_ack(cm_node);</div><div class='add'>+		/*</div><div class='add'>+		 * Wait for ACK as this is simultaneous close.</div><div class='add'>+		 * After we receive ACK, do not send anything.</div><div class='add'>+		 * Just rm the node.</div><div class='add'>+		 */</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_FIN_WAIT2:</div><div class='add'>+		cm_node-&gt;tcp_cntxt.rcv_nxt++;</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_TIME_WAIT;</div><div class='add'>+		irdma_send_ack(cm_node);</div><div class='add'>+		irdma_schedule_cm_timer(cm_node, NULL, IRDMA_TIMER_TYPE_CLOSE,</div><div class='add'>+					1, 0);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_TIME_WAIT:</div><div class='add'>+		cm_node-&gt;tcp_cntxt.rcv_nxt++;</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_CLOSED;</div><div class='add'>+		irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_OFFLOADED:</div><div class='add'>+	default:</div><div class='add'>+		ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: bad state node state = %d\n", cm_node-&gt;state);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_handle_rst_pkt - process received RST packet</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @rbuf: receive buffer</div><div class='add'>+ */</div><div class='add'>+static void irdma_handle_rst_pkt(struct irdma_cm_node *cm_node,</div><div class='add'>+				 struct irdma_puda_buf *rbuf)</div><div class='add'>+{</div><div class='add'>+	ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+		  "CM: caller: %pS cm_node=%p state=%d rem_port=0x%04x loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n",</div><div class='add'>+		  __builtin_return_address(0), cm_node, cm_node-&gt;state,</div><div class='add'>+		  cm_node-&gt;rem_port, cm_node-&gt;loc_port, cm_node-&gt;rem_addr,</div><div class='add'>+		  cm_node-&gt;loc_addr);</div><div class='add'>+</div><div class='add'>+	irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+	switch (cm_node-&gt;state) {</div><div class='add'>+	case IRDMA_CM_STATE_SYN_SENT:</div><div class='add'>+	case IRDMA_CM_STATE_MPAREQ_SENT:</div><div class='add'>+		switch (cm_node-&gt;mpa_frame_rev) {</div><div class='add'>+		case IETF_MPA_V2:</div><div class='add'>+			/* Drop down to MPA_V1*/</div><div class='add'>+			cm_node-&gt;mpa_frame_rev = IETF_MPA_V1;</div><div class='add'>+			/* send a syn and goto syn sent state */</div><div class='add'>+			cm_node-&gt;state = IRDMA_CM_STATE_SYN_SENT;</div><div class='add'>+			if (irdma_send_syn(cm_node, 0))</div><div class='add'>+				irdma_active_open_err(cm_node, false);</div><div class='add'>+			break;</div><div class='add'>+		case IETF_MPA_V1:</div><div class='add'>+		default:</div><div class='add'>+			irdma_active_open_err(cm_node, false);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_MPAREQ_RCVD:</div><div class='add'>+		atomic_inc(&amp;cm_node-&gt;passive_state);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_ESTABLISHED:</div><div class='add'>+	case IRDMA_CM_STATE_SYN_RCVD:</div><div class='add'>+	case IRDMA_CM_STATE_LISTENING:</div><div class='add'>+		irdma_passive_open_err(cm_node, false);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_OFFLOADED:</div><div class='add'>+		irdma_active_open_err(cm_node, false);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_CLOSED:</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_FIN_WAIT2:</div><div class='add'>+	case IRDMA_CM_STATE_FIN_WAIT1:</div><div class='add'>+	case IRDMA_CM_STATE_LAST_ACK:</div><div class='add'>+	case IRDMA_CM_STATE_TIME_WAIT:</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_CLOSED;</div><div class='add'>+		irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_handle_rcv_mpa - Process a recv'd mpa buffer</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @rbuf: receive buffer</div><div class='add'>+ */</div><div class='add'>+static void irdma_handle_rcv_mpa(struct irdma_cm_node *cm_node,</div><div class='add'>+				 struct irdma_puda_buf *rbuf)</div><div class='add'>+{</div><div class='add'>+	int err;</div><div class='add'>+	int datasize = rbuf-&gt;datalen;</div><div class='add'>+	u8 *dataloc = rbuf-&gt;data;</div><div class='add'>+</div><div class='add'>+	enum irdma_cm_event_type type = IRDMA_CM_EVENT_UNKNOWN;</div><div class='add'>+	u32 res_type;</div><div class='add'>+</div><div class='add'>+	err = irdma_parse_mpa(cm_node, dataloc, &amp;res_type, datasize);</div><div class='add'>+	if (err) {</div><div class='add'>+		if (cm_node-&gt;state == IRDMA_CM_STATE_MPAREQ_SENT)</div><div class='add'>+			irdma_active_open_err(cm_node, true);</div><div class='add'>+		else</div><div class='add'>+			irdma_passive_open_err(cm_node, true);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (cm_node-&gt;state) {</div><div class='add'>+	case IRDMA_CM_STATE_ESTABLISHED:</div><div class='add'>+		if (res_type == IRDMA_MPA_REQUEST_REJECT)</div><div class='add'>+			ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: state for reject\n");</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_MPAREQ_RCVD;</div><div class='add'>+		type = IRDMA_CM_EVENT_MPA_REQ;</div><div class='add'>+		irdma_send_ack(cm_node); /* ACK received MPA request */</div><div class='add'>+		atomic_set(&amp;cm_node-&gt;passive_state,</div><div class='add'>+			   IRDMA_PASSIVE_STATE_INDICATED);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_MPAREQ_SENT:</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		if (res_type == IRDMA_MPA_REQUEST_REJECT) {</div><div class='add'>+			type = IRDMA_CM_EVENT_MPA_REJECT;</div><div class='add'>+			cm_node-&gt;state = IRDMA_CM_STATE_MPAREJ_RCVD;</div><div class='add'>+		} else {</div><div class='add'>+			type = IRDMA_CM_EVENT_CONNECTED;</div><div class='add'>+			cm_node-&gt;state = IRDMA_CM_STATE_OFFLOADED;</div><div class='add'>+		}</div><div class='add'>+		irdma_send_ack(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: wrong cm_node state =%d\n", cm_node-&gt;state);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+	irdma_create_event(cm_node, type);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_check_syn - Check for error on received syn ack</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @tcph: pointer tcp header</div><div class='add'>+ */</div><div class='add'>+static int irdma_check_syn(struct irdma_cm_node *cm_node, struct tcphdr *tcph)</div><div class='add'>+{</div><div class='add'>+	if (ntohl(tcph-&gt;ack_seq) != cm_node-&gt;tcp_cntxt.loc_seq_num) {</div><div class='add'>+		irdma_active_open_err(cm_node, true);</div><div class='add'>+		return 1;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_check_seq - check seq numbers if OK</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @tcph: pointer tcp header</div><div class='add'>+ */</div><div class='add'>+static int irdma_check_seq(struct irdma_cm_node *cm_node, struct tcphdr *tcph)</div><div class='add'>+{</div><div class='add'>+	u32 seq;</div><div class='add'>+	u32 ack_seq;</div><div class='add'>+	u32 loc_seq_num = cm_node-&gt;tcp_cntxt.loc_seq_num;</div><div class='add'>+	u32 rcv_nxt = cm_node-&gt;tcp_cntxt.rcv_nxt;</div><div class='add'>+	u32 rcv_wnd;</div><div class='add'>+	int err = 0;</div><div class='add'>+</div><div class='add'>+	seq = ntohl(tcph-&gt;seq);</div><div class='add'>+	ack_seq = ntohl(tcph-&gt;ack_seq);</div><div class='add'>+	rcv_wnd = cm_node-&gt;tcp_cntxt.rcv_wnd;</div><div class='add'>+	if (ack_seq != loc_seq_num ||</div><div class='add'>+	    !between(seq, rcv_nxt, (rcv_nxt + rcv_wnd)))</div><div class='add'>+		err = -1;</div><div class='add'>+	if (err)</div><div class='add'>+		ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: seq number err\n");</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void irdma_add_conn_est_qh(struct irdma_cm_node *cm_node)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_info nfo;</div><div class='add'>+</div><div class='add'>+	irdma_get_addr_info(cm_node, &amp;nfo);</div><div class='add'>+	nfo.qh_qpid = cm_node-&gt;iwdev-&gt;vsi.ilq-&gt;qp_id;</div><div class='add'>+	irdma_manage_qhash(cm_node-&gt;iwdev, &amp;nfo,</div><div class='add'>+			   IRDMA_QHASH_TYPE_TCP_ESTABLISHED,</div><div class='add'>+			   IRDMA_QHASH_MANAGE_TYPE_ADD,</div><div class='add'>+			   cm_node, false);</div><div class='add'>+	cm_node-&gt;qhash_set = true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_handle_syn_pkt - is for Passive node</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @rbuf: receive buffer</div><div class='add'>+ */</div><div class='add'>+static void irdma_handle_syn_pkt(struct irdma_cm_node *cm_node,</div><div class='add'>+				 struct irdma_puda_buf *rbuf)</div><div class='add'>+{</div><div class='add'>+	struct tcphdr *tcph = (struct tcphdr *)rbuf-&gt;tcph;</div><div class='add'>+	int err;</div><div class='add'>+	u32 inc_sequence;</div><div class='add'>+	int optionsize;</div><div class='add'>+</div><div class='add'>+	optionsize = (tcph-&gt;doff &lt;&lt; 2) - sizeof(struct tcphdr);</div><div class='add'>+	inc_sequence = ntohl(tcph-&gt;seq);</div><div class='add'>+</div><div class='add'>+	switch (cm_node-&gt;state) {</div><div class='add'>+	case IRDMA_CM_STATE_SYN_SENT:</div><div class='add'>+	case IRDMA_CM_STATE_MPAREQ_SENT:</div><div class='add'>+		/* Rcvd syn on active open connection */</div><div class='add'>+		irdma_active_open_err(cm_node, 1);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_LISTENING:</div><div class='add'>+		/* Passive OPEN */</div><div class='add'>+		if (atomic_read(&amp;cm_node-&gt;listener-&gt;pend_accepts_cnt) &gt;</div><div class='add'>+		    cm_node-&gt;listener-&gt;backlog) {</div><div class='add'>+			cm_node-&gt;cm_core-&gt;stats_backlog_drops++;</div><div class='add'>+			irdma_passive_open_err(cm_node, false);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 1);</div><div class='add'>+		if (err) {</div><div class='add'>+			irdma_passive_open_err(cm_node, false);</div><div class='add'>+			/* drop pkt */</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		err = cm_node-&gt;cm_core-&gt;cm_create_ah(cm_node, false);</div><div class='add'>+		if (err) {</div><div class='add'>+			irdma_passive_open_err(cm_node, false);</div><div class='add'>+			/* drop pkt */</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		cm_node-&gt;tcp_cntxt.rcv_nxt = inc_sequence + 1;</div><div class='add'>+		cm_node-&gt;accept_pend = 1;</div><div class='add'>+		atomic_inc(&amp;cm_node-&gt;listener-&gt;pend_accepts_cnt);</div><div class='add'>+</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_SYN_RCVD;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_CLOSED:</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		refcount_inc(&amp;cm_node-&gt;refcnt);</div><div class='add'>+		irdma_send_reset(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_OFFLOADED:</div><div class='add'>+	case IRDMA_CM_STATE_ESTABLISHED:</div><div class='add'>+	case IRDMA_CM_STATE_FIN_WAIT1:</div><div class='add'>+	case IRDMA_CM_STATE_FIN_WAIT2:</div><div class='add'>+	case IRDMA_CM_STATE_MPAREQ_RCVD:</div><div class='add'>+	case IRDMA_CM_STATE_LAST_ACK:</div><div class='add'>+	case IRDMA_CM_STATE_CLOSING:</div><div class='add'>+	case IRDMA_CM_STATE_UNKNOWN:</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_handle_synack_pkt - Process SYN+ACK packet (active side)</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @rbuf: receive buffer</div><div class='add'>+ */</div><div class='add'>+static void irdma_handle_synack_pkt(struct irdma_cm_node *cm_node,</div><div class='add'>+				    struct irdma_puda_buf *rbuf)</div><div class='add'>+{</div><div class='add'>+	struct tcphdr *tcph = (struct tcphdr *)rbuf-&gt;tcph;</div><div class='add'>+	int err;</div><div class='add'>+	u32 inc_sequence;</div><div class='add'>+	int optionsize;</div><div class='add'>+</div><div class='add'>+	optionsize = (tcph-&gt;doff &lt;&lt; 2) - sizeof(struct tcphdr);</div><div class='add'>+	inc_sequence = ntohl(tcph-&gt;seq);</div><div class='add'>+	switch (cm_node-&gt;state) {</div><div class='add'>+	case IRDMA_CM_STATE_SYN_SENT:</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		/* active open */</div><div class='add'>+		if (irdma_check_syn(cm_node, tcph)) {</div><div class='add'>+			ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: check syn fail\n");</div><div class='add'>+			return;</div><div class='add'>+		}</div><div class='add'>+		cm_node-&gt;tcp_cntxt.rem_ack_num = ntohl(tcph-&gt;ack_seq);</div><div class='add'>+		/* setup options */</div><div class='add'>+		err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 0);</div><div class='add'>+		if (err) {</div><div class='add'>+			ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: cm_node=%p tcp_options failed\n",</div><div class='add'>+				  cm_node);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		cm_node-&gt;tcp_cntxt.rcv_nxt = inc_sequence + 1;</div><div class='add'>+		irdma_send_ack(cm_node); /* ACK  for the syn_ack */</div><div class='add'>+		err = irdma_send_mpa_request(cm_node);</div><div class='add'>+		if (err) {</div><div class='add'>+			ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: cm_node=%p irdma_send_mpa_request failed\n",</div><div class='add'>+				  cm_node);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_MPAREQ_SENT;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_MPAREQ_RCVD:</div><div class='add'>+		irdma_passive_open_err(cm_node, true);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_LISTENING:</div><div class='add'>+		cm_node-&gt;tcp_cntxt.loc_seq_num = ntohl(tcph-&gt;ack_seq);</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_CLOSED;</div><div class='add'>+		irdma_send_reset(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_CLOSED:</div><div class='add'>+		cm_node-&gt;tcp_cntxt.loc_seq_num = ntohl(tcph-&gt;ack_seq);</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		refcount_inc(&amp;cm_node-&gt;refcnt);</div><div class='add'>+		irdma_send_reset(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_ESTABLISHED:</div><div class='add'>+	case IRDMA_CM_STATE_FIN_WAIT1:</div><div class='add'>+	case IRDMA_CM_STATE_FIN_WAIT2:</div><div class='add'>+	case IRDMA_CM_STATE_LAST_ACK:</div><div class='add'>+	case IRDMA_CM_STATE_OFFLOADED:</div><div class='add'>+	case IRDMA_CM_STATE_CLOSING:</div><div class='add'>+	case IRDMA_CM_STATE_UNKNOWN:</div><div class='add'>+	case IRDMA_CM_STATE_MPAREQ_SENT:</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_handle_ack_pkt - process packet with ACK</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @rbuf: receive buffer</div><div class='add'>+ */</div><div class='add'>+static int irdma_handle_ack_pkt(struct irdma_cm_node *cm_node,</div><div class='add'>+				struct irdma_puda_buf *rbuf)</div><div class='add'>+{</div><div class='add'>+	struct tcphdr *tcph = (struct tcphdr *)rbuf-&gt;tcph;</div><div class='add'>+	u32 inc_sequence;</div><div class='add'>+	int ret;</div><div class='add'>+	int optionsize;</div><div class='add'>+	u32 datasize = rbuf-&gt;datalen;</div><div class='add'>+</div><div class='add'>+	optionsize = (tcph-&gt;doff &lt;&lt; 2) - sizeof(struct tcphdr);</div><div class='add'>+</div><div class='add'>+	if (irdma_check_seq(cm_node, tcph))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	inc_sequence = ntohl(tcph-&gt;seq);</div><div class='add'>+	switch (cm_node-&gt;state) {</div><div class='add'>+	case IRDMA_CM_STATE_SYN_RCVD:</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		ret = irdma_handle_tcp_options(cm_node, tcph, optionsize, 1);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+		cm_node-&gt;tcp_cntxt.rem_ack_num = ntohl(tcph-&gt;ack_seq);</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_ESTABLISHED;</div><div class='add'>+		if (datasize) {</div><div class='add'>+			cm_node-&gt;tcp_cntxt.rcv_nxt = inc_sequence + datasize;</div><div class='add'>+			irdma_handle_rcv_mpa(cm_node, rbuf);</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_ESTABLISHED:</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		if (datasize) {</div><div class='add'>+			cm_node-&gt;tcp_cntxt.rcv_nxt = inc_sequence + datasize;</div><div class='add'>+			irdma_handle_rcv_mpa(cm_node, rbuf);</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_MPAREQ_SENT:</div><div class='add'>+		cm_node-&gt;tcp_cntxt.rem_ack_num = ntohl(tcph-&gt;ack_seq);</div><div class='add'>+		if (datasize) {</div><div class='add'>+			cm_node-&gt;tcp_cntxt.rcv_nxt = inc_sequence + datasize;</div><div class='add'>+			cm_node-&gt;ack_rcvd = false;</div><div class='add'>+			irdma_handle_rcv_mpa(cm_node, rbuf);</div><div class='add'>+		} else {</div><div class='add'>+			cm_node-&gt;ack_rcvd = true;</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_LISTENING:</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_CLOSED;</div><div class='add'>+		irdma_send_reset(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_CLOSED:</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		refcount_inc(&amp;cm_node-&gt;refcnt);</div><div class='add'>+		irdma_send_reset(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_LAST_ACK:</div><div class='add'>+	case IRDMA_CM_STATE_CLOSING:</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_CLOSED;</div><div class='add'>+		irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_FIN_WAIT1:</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_FIN_WAIT2;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_SYN_SENT:</div><div class='add'>+	case IRDMA_CM_STATE_FIN_WAIT2:</div><div class='add'>+	case IRDMA_CM_STATE_OFFLOADED:</div><div class='add'>+	case IRDMA_CM_STATE_MPAREQ_RCVD:</div><div class='add'>+	case IRDMA_CM_STATE_UNKNOWN:</div><div class='add'>+	default:</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_process_pkt - process cm packet</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @rbuf: receive buffer</div><div class='add'>+ */</div><div class='add'>+static void irdma_process_pkt(struct irdma_cm_node *cm_node,</div><div class='add'>+			      struct irdma_puda_buf *rbuf)</div><div class='add'>+{</div><div class='add'>+	enum irdma_tcpip_pkt_type pkt_type = IRDMA_PKT_TYPE_UNKNOWN;</div><div class='add'>+	struct tcphdr *tcph = (struct tcphdr *)rbuf-&gt;tcph;</div><div class='add'>+	u32 fin_set = 0;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (tcph-&gt;rst) {</div><div class='add'>+		pkt_type = IRDMA_PKT_TYPE_RST;</div><div class='add'>+	} else if (tcph-&gt;syn) {</div><div class='add'>+		pkt_type = IRDMA_PKT_TYPE_SYN;</div><div class='add'>+		if (tcph-&gt;ack)</div><div class='add'>+			pkt_type = IRDMA_PKT_TYPE_SYNACK;</div><div class='add'>+	} else if (tcph-&gt;ack) {</div><div class='add'>+		pkt_type = IRDMA_PKT_TYPE_ACK;</div><div class='add'>+	}</div><div class='add'>+	if (tcph-&gt;fin)</div><div class='add'>+		fin_set = 1;</div><div class='add'>+</div><div class='add'>+	switch (pkt_type) {</div><div class='add'>+	case IRDMA_PKT_TYPE_SYN:</div><div class='add'>+		irdma_handle_syn_pkt(cm_node, rbuf);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_PKT_TYPE_SYNACK:</div><div class='add'>+		irdma_handle_synack_pkt(cm_node, rbuf);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_PKT_TYPE_ACK:</div><div class='add'>+		err = irdma_handle_ack_pkt(cm_node, rbuf);</div><div class='add'>+		if (fin_set &amp;&amp; !err)</div><div class='add'>+			irdma_handle_fin_pkt(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_PKT_TYPE_RST:</div><div class='add'>+		irdma_handle_rst_pkt(cm_node, rbuf);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		if (fin_set &amp;&amp;</div><div class='add'>+		    (!irdma_check_seq(cm_node, (struct tcphdr *)rbuf-&gt;tcph)))</div><div class='add'>+			irdma_handle_fin_pkt(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_make_listen_node - create a listen node with params</div><div class='add'>+ * @cm_core: cm's core</div><div class='add'>+ * @iwdev: iwarp device structure</div><div class='add'>+ * @cm_info: quad info for connection</div><div class='add'>+ */</div><div class='add'>+static struct irdma_cm_listener *</div><div class='add'>+irdma_make_listen_node(struct irdma_cm_core *cm_core,</div><div class='add'>+		       struct irdma_device *iwdev,</div><div class='add'>+		       struct irdma_cm_info *cm_info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_listener *listener;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	/* cannot have multiple matching listeners */</div><div class='add'>+	listener =</div><div class='add'>+		irdma_find_listener(cm_core, cm_info-&gt;loc_addr, cm_info-&gt;ipv4,</div><div class='add'>+				    cm_info-&gt;loc_port, cm_info-&gt;vlan_id,</div><div class='add'>+				    IRDMA_CM_LISTENER_EITHER_STATE);</div><div class='add'>+	if (listener &amp;&amp;</div><div class='add'>+	    listener-&gt;listener_state == IRDMA_CM_LISTENER_ACTIVE_STATE) {</div><div class='add'>+		refcount_dec(&amp;listener-&gt;refcnt);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!listener) {</div><div class='add'>+		/* create a CM listen node</div><div class='add'>+		 * 1/2 node to compare incoming traffic to</div><div class='add'>+		 */</div><div class='add'>+		listener = kzalloc(sizeof(*listener), GFP_KERNEL);</div><div class='add'>+		if (!listener)</div><div class='add'>+			return NULL;</div><div class='add'>+		cm_core-&gt;stats_listen_nodes_created++;</div><div class='add'>+		memcpy(listener-&gt;loc_addr, cm_info-&gt;loc_addr,</div><div class='add'>+		       sizeof(listener-&gt;loc_addr));</div><div class='add'>+		listener-&gt;loc_port = cm_info-&gt;loc_port;</div><div class='add'>+</div><div class='add'>+		INIT_LIST_HEAD(&amp;listener-&gt;child_listen_list);</div><div class='add'>+</div><div class='add'>+		refcount_set(&amp;listener-&gt;refcnt, 1);</div><div class='add'>+	} else {</div><div class='add'>+		listener-&gt;reused_node = 1;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	listener-&gt;cm_id = cm_info-&gt;cm_id;</div><div class='add'>+	listener-&gt;ipv4 = cm_info-&gt;ipv4;</div><div class='add'>+	listener-&gt;vlan_id = cm_info-&gt;vlan_id;</div><div class='add'>+	atomic_set(&amp;listener-&gt;pend_accepts_cnt, 0);</div><div class='add'>+	listener-&gt;cm_core = cm_core;</div><div class='add'>+	listener-&gt;iwdev = iwdev;</div><div class='add'>+</div><div class='add'>+	listener-&gt;backlog = cm_info-&gt;backlog;</div><div class='add'>+	listener-&gt;listener_state = IRDMA_CM_LISTENER_ACTIVE_STATE;</div><div class='add'>+</div><div class='add'>+	if (!listener-&gt;reused_node) {</div><div class='add'>+		spin_lock_irqsave(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='add'>+		list_add(&amp;listener-&gt;list, &amp;cm_core-&gt;listen_list);</div><div class='add'>+		spin_unlock_irqrestore(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return listener;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_create_cm_node - make a connection node with params</div><div class='add'>+ * @cm_core: cm's core</div><div class='add'>+ * @iwdev: iwarp device structure</div><div class='add'>+ * @conn_param: connection parameters</div><div class='add'>+ * @cm_info: quad info for connection</div><div class='add'>+ * @caller_cm_node: pointer to cm_node structure to return</div><div class='add'>+ */</div><div class='add'>+static int irdma_create_cm_node(struct irdma_cm_core *cm_core,</div><div class='add'>+				struct irdma_device *iwdev,</div><div class='add'>+				struct iw_cm_conn_param *conn_param,</div><div class='add'>+				struct irdma_cm_info *cm_info,</div><div class='add'>+				struct irdma_cm_node **caller_cm_node)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+	u16 private_data_len = conn_param-&gt;private_data_len;</div><div class='add'>+	const void *private_data = conn_param-&gt;private_data;</div><div class='add'>+</div><div class='add'>+	/* create a CM connection node */</div><div class='add'>+	cm_node = irdma_make_cm_node(cm_core, iwdev, cm_info, NULL);</div><div class='add'>+	if (!cm_node)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	/* set our node side to client (active) side */</div><div class='add'>+	cm_node-&gt;tcp_cntxt.client = 1;</div><div class='add'>+	cm_node-&gt;tcp_cntxt.rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;</div><div class='add'>+</div><div class='add'>+	irdma_record_ird_ord(cm_node, conn_param-&gt;ird, conn_param-&gt;ord);</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;pdata.size = private_data_len;</div><div class='add'>+	cm_node-&gt;pdata.addr = cm_node-&gt;pdata_buf;</div><div class='add'>+</div><div class='add'>+	memcpy(cm_node-&gt;pdata_buf, private_data, private_data_len);</div><div class='add'>+	*caller_cm_node = cm_node;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cm_reject - reject and teardown a connection</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @pdata: ptr to private data for reject</div><div class='add'>+ * @plen: size of private data</div><div class='add'>+ */</div><div class='add'>+static int irdma_cm_reject(struct irdma_cm_node *cm_node, const void *pdata,</div><div class='add'>+			   u8 plen)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+	int passive_state;</div><div class='add'>+</div><div class='add'>+	if (cm_node-&gt;tcp_cntxt.client)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+</div><div class='add'>+	passive_state = atomic_add_return(1, &amp;cm_node-&gt;passive_state);</div><div class='add'>+	if (passive_state == IRDMA_SEND_RESET_EVENT) {</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_CLOSED;</div><div class='add'>+		irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (cm_node-&gt;state == IRDMA_CM_STATE_LISTENER_DESTROYED) {</div><div class='add'>+		irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = irdma_send_mpa_reject(cm_node, pdata, plen);</div><div class='add'>+	if (!ret)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;state = IRDMA_CM_STATE_CLOSED;</div><div class='add'>+	if (irdma_send_reset(cm_node))</div><div class='add'>+		ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: send reset failed\n");</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cm_close - close of cm connection</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ */</div><div class='add'>+static int irdma_cm_close(struct irdma_cm_node *cm_node)</div><div class='add'>+{</div><div class='add'>+	switch (cm_node-&gt;state) {</div><div class='add'>+	case IRDMA_CM_STATE_SYN_RCVD:</div><div class='add'>+	case IRDMA_CM_STATE_SYN_SENT:</div><div class='add'>+	case IRDMA_CM_STATE_ONE_SIDE_ESTABLISHED:</div><div class='add'>+	case IRDMA_CM_STATE_ESTABLISHED:</div><div class='add'>+	case IRDMA_CM_STATE_ACCEPTING:</div><div class='add'>+	case IRDMA_CM_STATE_MPAREQ_SENT:</div><div class='add'>+	case IRDMA_CM_STATE_MPAREQ_RCVD:</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		irdma_send_reset(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_CLOSE_WAIT:</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_LAST_ACK;</div><div class='add'>+		irdma_send_fin(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_FIN_WAIT1:</div><div class='add'>+	case IRDMA_CM_STATE_FIN_WAIT2:</div><div class='add'>+	case IRDMA_CM_STATE_LAST_ACK:</div><div class='add'>+	case IRDMA_CM_STATE_TIME_WAIT:</div><div class='add'>+	case IRDMA_CM_STATE_CLOSING:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	case IRDMA_CM_STATE_LISTENING:</div><div class='add'>+		irdma_cleanup_retrans_entry(cm_node);</div><div class='add'>+		irdma_send_reset(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_MPAREJ_RCVD:</div><div class='add'>+	case IRDMA_CM_STATE_UNKNOWN:</div><div class='add'>+	case IRDMA_CM_STATE_INITED:</div><div class='add'>+	case IRDMA_CM_STATE_CLOSED:</div><div class='add'>+	case IRDMA_CM_STATE_LISTENER_DESTROYED:</div><div class='add'>+		irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_STATE_OFFLOADED:</div><div class='add'>+		if (cm_node-&gt;send_entry)</div><div class='add'>+			ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: CM send_entry in OFFLOADED state\n");</div><div class='add'>+		irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_receive_ilq - recv an ETHERNET packet, and process it</div><div class='add'>+ * through CM</div><div class='add'>+ * @vsi: VSI structure of dev</div><div class='add'>+ * @rbuf: receive buffer</div><div class='add'>+ */</div><div class='add'>+void irdma_receive_ilq(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *rbuf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+	struct irdma_cm_listener *listener;</div><div class='add'>+	struct iphdr *iph;</div><div class='add'>+	struct ipv6hdr *ip6h;</div><div class='add'>+	struct tcphdr *tcph;</div><div class='add'>+	struct irdma_cm_info cm_info = {};</div><div class='add'>+	struct irdma_device *iwdev = vsi-&gt;back_vsi;</div><div class='add'>+	struct irdma_cm_core *cm_core = &amp;iwdev-&gt;cm_core;</div><div class='add'>+	struct vlan_ethhdr *ethh;</div><div class='add'>+	u16 vtag;</div><div class='add'>+</div><div class='add'>+	/* if vlan, then maclen = 18 else 14 */</div><div class='add'>+	iph = (struct iphdr *)rbuf-&gt;iph;</div><div class='add'>+	print_hex_dump_debug("ILQ: RECEIVE ILQ BUFFER", DUMP_PREFIX_OFFSET,</div><div class='add'>+			     16, 8, rbuf-&gt;mem.va, rbuf-&gt;totallen, false);</div><div class='add'>+	if (iwdev-&gt;rf-&gt;sc_dev.hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+		if (rbuf-&gt;vlan_valid) {</div><div class='add'>+			vtag = rbuf-&gt;vlan_id;</div><div class='add'>+			cm_info.user_pri = (vtag &amp; VLAN_PRIO_MASK) &gt;&gt;</div><div class='add'>+					   VLAN_PRIO_SHIFT;</div><div class='add'>+			cm_info.vlan_id = vtag &amp; VLAN_VID_MASK;</div><div class='add'>+		} else {</div><div class='add'>+			cm_info.vlan_id = 0xFFFF;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		ethh = rbuf-&gt;mem.va;</div><div class='add'>+</div><div class='add'>+		if (ethh-&gt;h_vlan_proto == htons(ETH_P_8021Q)) {</div><div class='add'>+			vtag = ntohs(ethh-&gt;h_vlan_TCI);</div><div class='add'>+			cm_info.user_pri = (vtag &amp; VLAN_PRIO_MASK) &gt;&gt;</div><div class='add'>+					   VLAN_PRIO_SHIFT;</div><div class='add'>+			cm_info.vlan_id = vtag &amp; VLAN_VID_MASK;</div><div class='add'>+			ibdev_dbg(&amp;cm_core-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: vlan_id=%d\n", cm_info.vlan_id);</div><div class='add'>+		} else {</div><div class='add'>+			cm_info.vlan_id = 0xFFFF;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	tcph = (struct tcphdr *)rbuf-&gt;tcph;</div><div class='add'>+</div><div class='add'>+	if (rbuf-&gt;ipv4) {</div><div class='add'>+		cm_info.loc_addr[0] = ntohl(iph-&gt;daddr);</div><div class='add'>+		cm_info.rem_addr[0] = ntohl(iph-&gt;saddr);</div><div class='add'>+		cm_info.ipv4 = true;</div><div class='add'>+		cm_info.tos = iph-&gt;tos;</div><div class='add'>+	} else {</div><div class='add'>+		ip6h = (struct ipv6hdr *)rbuf-&gt;iph;</div><div class='add'>+		irdma_copy_ip_ntohl(cm_info.loc_addr,</div><div class='add'>+				    ip6h-&gt;daddr.in6_u.u6_addr32);</div><div class='add'>+		irdma_copy_ip_ntohl(cm_info.rem_addr,</div><div class='add'>+				    ip6h-&gt;saddr.in6_u.u6_addr32);</div><div class='add'>+		cm_info.ipv4 = false;</div><div class='add'>+		cm_info.tos = (ip6h-&gt;priority &lt;&lt; 4) | (ip6h-&gt;flow_lbl[0] &gt;&gt; 4);</div><div class='add'>+	}</div><div class='add'>+	cm_info.loc_port = ntohs(tcph-&gt;dest);</div><div class='add'>+	cm_info.rem_port = ntohs(tcph-&gt;source);</div><div class='add'>+	cm_node = irdma_find_node(cm_core, cm_info.rem_port, cm_info.rem_addr,</div><div class='add'>+				  cm_info.loc_port, cm_info.loc_addr, cm_info.vlan_id);</div><div class='add'>+</div><div class='add'>+	if (!cm_node) {</div><div class='add'>+		/* Only type of packet accepted are for the</div><div class='add'>+		 * PASSIVE open (syn only)</div><div class='add'>+		 */</div><div class='add'>+		if (!tcph-&gt;syn || tcph-&gt;ack)</div><div class='add'>+			return;</div><div class='add'>+</div><div class='add'>+		listener = irdma_find_listener(cm_core,</div><div class='add'>+					       cm_info.loc_addr,</div><div class='add'>+					       cm_info.ipv4,</div><div class='add'>+					       cm_info.loc_port,</div><div class='add'>+					       cm_info.vlan_id,</div><div class='add'>+					       IRDMA_CM_LISTENER_ACTIVE_STATE);</div><div class='add'>+		if (!listener) {</div><div class='add'>+			cm_info.cm_id = NULL;</div><div class='add'>+			ibdev_dbg(&amp;cm_core-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: no listener found\n");</div><div class='add'>+			return;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		cm_info.cm_id = listener-&gt;cm_id;</div><div class='add'>+		cm_node = irdma_make_cm_node(cm_core, iwdev, &amp;cm_info,</div><div class='add'>+					     listener);</div><div class='add'>+		if (!cm_node) {</div><div class='add'>+			ibdev_dbg(&amp;cm_core-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: allocate node failed\n");</div><div class='add'>+			refcount_dec(&amp;listener-&gt;refcnt);</div><div class='add'>+			return;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (!tcph-&gt;rst &amp;&amp; !tcph-&gt;fin) {</div><div class='add'>+			cm_node-&gt;state = IRDMA_CM_STATE_LISTENING;</div><div class='add'>+		} else {</div><div class='add'>+			irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+			return;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		refcount_inc(&amp;cm_node-&gt;refcnt);</div><div class='add'>+	} else if (cm_node-&gt;state == IRDMA_CM_STATE_OFFLOADED) {</div><div class='add'>+		irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_process_pkt(cm_node, rbuf);</div><div class='add'>+	irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_add_qh(struct irdma_cm_node *cm_node, bool active)</div><div class='add'>+{</div><div class='add'>+	if (!active)</div><div class='add'>+		irdma_add_conn_est_qh(cm_node);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_cm_free_ah_nop(struct irdma_cm_node *cm_node)</div><div class='add'>+{</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_setup_cm_core - setup top level instance of a cm core</div><div class='add'>+ * @iwdev: iwarp device structure</div><div class='add'>+ * @rdma_ver: HW version</div><div class='add'>+ */</div><div class='add'>+int irdma_setup_cm_core(struct irdma_device *iwdev, u8 rdma_ver)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_core *cm_core = &amp;iwdev-&gt;cm_core;</div><div class='add'>+</div><div class='add'>+	cm_core-&gt;iwdev = iwdev;</div><div class='add'>+	cm_core-&gt;dev = &amp;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+</div><div class='add'>+	/* Handles CM event work items send to Iwarp core */</div><div class='add'>+	cm_core-&gt;event_wq = alloc_ordered_workqueue("iwarp-event-wq", 0);</div><div class='add'>+	if (!cm_core-&gt;event_wq)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;cm_core-&gt;listen_list);</div><div class='add'>+</div><div class='add'>+	timer_setup(&amp;cm_core-&gt;tcp_timer, irdma_cm_timer_tick, 0);</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;cm_core-&gt;ht_lock);</div><div class='add'>+	spin_lock_init(&amp;cm_core-&gt;listen_list_lock);</div><div class='add'>+	spin_lock_init(&amp;cm_core-&gt;apbvt_lock);</div><div class='add'>+	switch (rdma_ver) {</div><div class='add'>+	case IRDMA_GEN_1:</div><div class='add'>+		cm_core-&gt;form_cm_frame = irdma_form_uda_cm_frame;</div><div class='add'>+		cm_core-&gt;cm_create_ah = irdma_add_qh;</div><div class='add'>+		cm_core-&gt;cm_free_ah = irdma_cm_free_ah_nop;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_GEN_2:</div><div class='add'>+	default:</div><div class='add'>+		cm_core-&gt;form_cm_frame = irdma_form_ah_cm_frame;</div><div class='add'>+		cm_core-&gt;cm_create_ah = irdma_cm_create_ah;</div><div class='add'>+		cm_core-&gt;cm_free_ah = irdma_cm_free_ah;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cleanup_cm_core - deallocate a top level instance of a</div><div class='add'>+ * cm core</div><div class='add'>+ * @cm_core: cm's core</div><div class='add'>+ */</div><div class='add'>+void irdma_cleanup_cm_core(struct irdma_cm_core *cm_core)</div><div class='add'>+{</div><div class='add'>+	if (!cm_core)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	timer_delete_sync(&amp;cm_core-&gt;tcp_timer);</div><div class='add'>+</div><div class='add'>+	destroy_workqueue(cm_core-&gt;event_wq);</div><div class='add'>+	cm_core-&gt;dev-&gt;ws_reset(&amp;cm_core-&gt;iwdev-&gt;vsi);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_init_tcp_ctx - setup qp context</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ * @tcp_info: offload info for tcp</div><div class='add'>+ * @iwqp: associate qp for the connection</div><div class='add'>+ */</div><div class='add'>+static void irdma_init_tcp_ctx(struct irdma_cm_node *cm_node,</div><div class='add'>+			       struct irdma_tcp_offload_info *tcp_info,</div><div class='add'>+			       struct irdma_qp *iwqp)</div><div class='add'>+{</div><div class='add'>+	tcp_info-&gt;ipv4 = cm_node-&gt;ipv4;</div><div class='add'>+	tcp_info-&gt;drop_ooo_seg = !iwqp-&gt;iwdev-&gt;iw_ooo;</div><div class='add'>+	tcp_info-&gt;wscale = true;</div><div class='add'>+	tcp_info-&gt;ignore_tcp_opt = true;</div><div class='add'>+	tcp_info-&gt;ignore_tcp_uns_opt = true;</div><div class='add'>+	tcp_info-&gt;no_nagle = false;</div><div class='add'>+</div><div class='add'>+	tcp_info-&gt;ttl = IRDMA_DEFAULT_TTL;</div><div class='add'>+	tcp_info-&gt;rtt_var = IRDMA_DEFAULT_RTT_VAR;</div><div class='add'>+	tcp_info-&gt;ss_thresh = IRDMA_DEFAULT_SS_THRESH;</div><div class='add'>+	tcp_info-&gt;rexmit_thresh = IRDMA_DEFAULT_REXMIT_THRESH;</div><div class='add'>+</div><div class='add'>+	tcp_info-&gt;tcp_state = IRDMA_TCP_STATE_ESTABLISHED;</div><div class='add'>+	tcp_info-&gt;snd_wscale = cm_node-&gt;tcp_cntxt.snd_wscale;</div><div class='add'>+	tcp_info-&gt;rcv_wscale = cm_node-&gt;tcp_cntxt.rcv_wscale;</div><div class='add'>+</div><div class='add'>+	tcp_info-&gt;snd_nxt = cm_node-&gt;tcp_cntxt.loc_seq_num;</div><div class='add'>+	tcp_info-&gt;snd_wnd = cm_node-&gt;tcp_cntxt.snd_wnd;</div><div class='add'>+	tcp_info-&gt;rcv_nxt = cm_node-&gt;tcp_cntxt.rcv_nxt;</div><div class='add'>+	tcp_info-&gt;snd_max = cm_node-&gt;tcp_cntxt.loc_seq_num;</div><div class='add'>+</div><div class='add'>+	tcp_info-&gt;snd_una = cm_node-&gt;tcp_cntxt.loc_seq_num;</div><div class='add'>+	tcp_info-&gt;cwnd = 2 * cm_node-&gt;tcp_cntxt.mss;</div><div class='add'>+	tcp_info-&gt;snd_wl1 = cm_node-&gt;tcp_cntxt.rcv_nxt;</div><div class='add'>+	tcp_info-&gt;snd_wl2 = cm_node-&gt;tcp_cntxt.loc_seq_num;</div><div class='add'>+	tcp_info-&gt;max_snd_window = cm_node-&gt;tcp_cntxt.max_snd_wnd;</div><div class='add'>+	tcp_info-&gt;rcv_wnd = cm_node-&gt;tcp_cntxt.rcv_wnd</div><div class='add'>+			    &lt;&lt; cm_node-&gt;tcp_cntxt.rcv_wscale;</div><div class='add'>+</div><div class='add'>+	tcp_info-&gt;flow_label = 0;</div><div class='add'>+	tcp_info-&gt;snd_mss = (u32)cm_node-&gt;tcp_cntxt.mss;</div><div class='add'>+	tcp_info-&gt;tos = cm_node-&gt;tos;</div><div class='add'>+	if (cm_node-&gt;vlan_id &lt; VLAN_N_VID) {</div><div class='add'>+		tcp_info-&gt;insert_vlan_tag = true;</div><div class='add'>+		tcp_info-&gt;vlan_tag = cm_node-&gt;vlan_id;</div><div class='add'>+		tcp_info-&gt;vlan_tag |= cm_node-&gt;user_pri &lt;&lt; VLAN_PRIO_SHIFT;</div><div class='add'>+	}</div><div class='add'>+	if (cm_node-&gt;ipv4) {</div><div class='add'>+		tcp_info-&gt;src_port = cm_node-&gt;loc_port;</div><div class='add'>+		tcp_info-&gt;dst_port = cm_node-&gt;rem_port;</div><div class='add'>+</div><div class='add'>+		tcp_info-&gt;dest_ip_addr[3] = cm_node-&gt;rem_addr[0];</div><div class='add'>+		tcp_info-&gt;local_ipaddr[3] = cm_node-&gt;loc_addr[0];</div><div class='add'>+		tcp_info-&gt;arp_idx = (u16)irdma_arp_table(iwqp-&gt;iwdev-&gt;rf,</div><div class='add'>+							 &amp;tcp_info-&gt;dest_ip_addr[3],</div><div class='add'>+							 true, NULL,</div><div class='add'>+							 IRDMA_ARP_RESOLVE);</div><div class='add'>+	} else {</div><div class='add'>+		tcp_info-&gt;src_port = cm_node-&gt;loc_port;</div><div class='add'>+		tcp_info-&gt;dst_port = cm_node-&gt;rem_port;</div><div class='add'>+		memcpy(tcp_info-&gt;dest_ip_addr, cm_node-&gt;rem_addr,</div><div class='add'>+		       sizeof(tcp_info-&gt;dest_ip_addr));</div><div class='add'>+		memcpy(tcp_info-&gt;local_ipaddr, cm_node-&gt;loc_addr,</div><div class='add'>+		       sizeof(tcp_info-&gt;local_ipaddr));</div><div class='add'>+</div><div class='add'>+		tcp_info-&gt;arp_idx = (u16)irdma_arp_table(iwqp-&gt;iwdev-&gt;rf,</div><div class='add'>+							 &amp;tcp_info-&gt;dest_ip_addr[0],</div><div class='add'>+							 false, NULL,</div><div class='add'>+							 IRDMA_ARP_RESOLVE);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cm_init_tsa_conn - setup qp for RTS</div><div class='add'>+ * @iwqp: associate qp for the connection</div><div class='add'>+ * @cm_node: connection's node</div><div class='add'>+ */</div><div class='add'>+static void irdma_cm_init_tsa_conn(struct irdma_qp *iwqp,</div><div class='add'>+				   struct irdma_cm_node *cm_node)</div><div class='add'>+{</div><div class='add'>+	struct irdma_iwarp_offload_info *iwarp_info;</div><div class='add'>+	struct irdma_qp_host_ctx_info *ctx_info;</div><div class='add'>+</div><div class='add'>+	iwarp_info = &amp;iwqp-&gt;iwarp_info;</div><div class='add'>+	ctx_info = &amp;iwqp-&gt;ctx_info;</div><div class='add'>+</div><div class='add'>+	ctx_info-&gt;tcp_info = &amp;iwqp-&gt;tcp_info;</div><div class='add'>+	ctx_info-&gt;send_cq_num = iwqp-&gt;iwscq-&gt;sc_cq.cq_uk.cq_id;</div><div class='add'>+	ctx_info-&gt;rcv_cq_num = iwqp-&gt;iwrcq-&gt;sc_cq.cq_uk.cq_id;</div><div class='add'>+</div><div class='add'>+	iwarp_info-&gt;ord_size = cm_node-&gt;ord_size;</div><div class='add'>+	iwarp_info-&gt;ird_size = cm_node-&gt;ird_size;</div><div class='add'>+	iwarp_info-&gt;rd_en = true;</div><div class='add'>+	iwarp_info-&gt;rdmap_ver = 1;</div><div class='add'>+	iwarp_info-&gt;ddp_ver = 1;</div><div class='add'>+	iwarp_info-&gt;pd_id = iwqp-&gt;iwpd-&gt;sc_pd.pd_id;</div><div class='add'>+</div><div class='add'>+	ctx_info-&gt;tcp_info_valid = true;</div><div class='add'>+	ctx_info-&gt;iwarp_info_valid = true;</div><div class='add'>+	ctx_info-&gt;user_pri = cm_node-&gt;user_pri;</div><div class='add'>+</div><div class='add'>+	irdma_init_tcp_ctx(cm_node, &amp;iwqp-&gt;tcp_info, iwqp);</div><div class='add'>+	if (cm_node-&gt;snd_mark_en) {</div><div class='add'>+		iwarp_info-&gt;snd_mark_en = true;</div><div class='add'>+		iwarp_info-&gt;snd_mark_offset = (iwqp-&gt;tcp_info.snd_nxt &amp; SNDMARKER_SEQNMASK) +</div><div class='add'>+					       cm_node-&gt;lsmm_size;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;state = IRDMA_CM_STATE_OFFLOADED;</div><div class='add'>+	iwqp-&gt;tcp_info.tcp_state = IRDMA_TCP_STATE_ESTABLISHED;</div><div class='add'>+	iwqp-&gt;tcp_info.src_mac_addr_idx = iwqp-&gt;iwdev-&gt;mac_ip_table_idx;</div><div class='add'>+</div><div class='add'>+	if (cm_node-&gt;rcv_mark_en) {</div><div class='add'>+		iwarp_info-&gt;rcv_mark_en = true;</div><div class='add'>+		iwarp_info-&gt;align_hdrs = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_sc_qp_setctx(&amp;iwqp-&gt;sc_qp, iwqp-&gt;host_ctx.va, ctx_info);</div><div class='add'>+</div><div class='add'>+	/* once tcp_info is set, no need to do it again */</div><div class='add'>+	ctx_info-&gt;tcp_info_valid = false;</div><div class='add'>+	ctx_info-&gt;iwarp_info_valid = false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cm_disconn - when a connection is being closed</div><div class='add'>+ * @iwqp: associated qp for the connection</div><div class='add'>+ */</div><div class='add'>+void irdma_cm_disconn(struct irdma_qp *iwqp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = iwqp-&gt;iwdev;</div><div class='add'>+	struct disconn_work *work;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	work = kzalloc(sizeof(*work), GFP_ATOMIC);</div><div class='add'>+	if (!work)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwdev-&gt;rf-&gt;qptable_lock, flags);</div><div class='add'>+	if (!iwdev-&gt;rf-&gt;qp_table[iwqp-&gt;ibqp.qp_num]) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;iwdev-&gt;rf-&gt;qptable_lock, flags);</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: qp_id %d is already freed\n",</div><div class='add'>+			  iwqp-&gt;ibqp.qp_num);</div><div class='add'>+		kfree(work);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+	irdma_qp_add_ref(&amp;iwqp-&gt;ibqp);</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwdev-&gt;rf-&gt;qptable_lock, flags);</div><div class='add'>+</div><div class='add'>+	work-&gt;iwqp = iwqp;</div><div class='add'>+	INIT_WORK(&amp;work-&gt;work, irdma_disconnect_worker);</div><div class='add'>+	queue_work(iwdev-&gt;cleanup_wq, &amp;work-&gt;work);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_qp_disconnect - free qp and close cm</div><div class='add'>+ * @iwqp: associate qp for the connection</div><div class='add'>+ */</div><div class='add'>+static void irdma_qp_disconnect(struct irdma_qp *iwqp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = iwqp-&gt;iwdev;</div><div class='add'>+</div><div class='add'>+	iwqp-&gt;active_conn = 0;</div><div class='add'>+	/* close the CM node down if it is still active */</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev, "CM: Call close API\n");</div><div class='add'>+	irdma_cm_close(iwqp-&gt;cm_node);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cm_disconn_true - called by worker thread to disconnect qp</div><div class='add'>+ * @iwqp: associate qp for the connection</div><div class='add'>+ */</div><div class='add'>+static void irdma_cm_disconn_true(struct irdma_qp *iwqp)</div><div class='add'>+{</div><div class='add'>+	struct iw_cm_id *cm_id;</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct irdma_sc_qp *qp = &amp;iwqp-&gt;sc_qp;</div><div class='add'>+	u16 last_ae;</div><div class='add'>+	u8 original_hw_tcp_state;</div><div class='add'>+	u8 original_ibqp_state;</div><div class='add'>+	int disconn_status = 0;</div><div class='add'>+	int issue_disconn = 0;</div><div class='add'>+	int issue_close = 0;</div><div class='add'>+	int issue_flush = 0;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	iwdev = iwqp-&gt;iwdev;</div><div class='add'>+	spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+	if (rdma_protocol_roce(&amp;iwdev-&gt;ibdev, 1)) {</div><div class='add'>+		struct ib_qp_attr attr;</div><div class='add'>+</div><div class='add'>+		if (iwqp-&gt;flush_issued || iwqp-&gt;sc_qp.qp_uk.destroy_pending) {</div><div class='add'>+			spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+			return;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+		attr.qp_state = IB_QPS_ERR;</div><div class='add'>+		irdma_modify_qp_roce(&amp;iwqp-&gt;ibqp, &amp;attr, IB_QP_STATE, NULL);</div><div class='add'>+		irdma_ib_qp_event(iwqp, qp-&gt;event_type);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cm_id = iwqp-&gt;cm_id;</div><div class='add'>+	original_hw_tcp_state = iwqp-&gt;hw_tcp_state;</div><div class='add'>+	original_ibqp_state = iwqp-&gt;ibqp_state;</div><div class='add'>+	last_ae = iwqp-&gt;last_aeq;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;term_flags) {</div><div class='add'>+		issue_disconn = 1;</div><div class='add'>+		issue_close = 1;</div><div class='add'>+		iwqp-&gt;cm_id = NULL;</div><div class='add'>+		irdma_terminate_del_timer(qp);</div><div class='add'>+		if (!iwqp-&gt;flush_issued) {</div><div class='add'>+			iwqp-&gt;flush_issued = 1;</div><div class='add'>+			issue_flush = 1;</div><div class='add'>+		}</div><div class='add'>+	} else if ((original_hw_tcp_state == IRDMA_TCP_STATE_CLOSE_WAIT) ||</div><div class='add'>+		   ((original_ibqp_state == IB_QPS_RTS) &amp;&amp;</div><div class='add'>+		    (last_ae == IRDMA_AE_LLP_CONNECTION_RESET))) {</div><div class='add'>+		issue_disconn = 1;</div><div class='add'>+		if (last_ae == IRDMA_AE_LLP_CONNECTION_RESET)</div><div class='add'>+			disconn_status = -ECONNRESET;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (original_hw_tcp_state == IRDMA_TCP_STATE_CLOSED ||</div><div class='add'>+	    original_hw_tcp_state == IRDMA_TCP_STATE_TIME_WAIT ||</div><div class='add'>+	    last_ae == IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE ||</div><div class='add'>+	    last_ae == IRDMA_AE_BAD_CLOSE ||</div><div class='add'>+	    last_ae == IRDMA_AE_LLP_CONNECTION_RESET || iwdev-&gt;rf-&gt;reset || !cm_id) {</div><div class='add'>+		issue_close = 1;</div><div class='add'>+		iwqp-&gt;cm_id = NULL;</div><div class='add'>+		qp-&gt;term_flags = 0;</div><div class='add'>+		if (!iwqp-&gt;flush_issued) {</div><div class='add'>+			iwqp-&gt;flush_issued = 1;</div><div class='add'>+			issue_flush = 1;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+	if (issue_flush &amp;&amp; !iwqp-&gt;sc_qp.qp_uk.destroy_pending) {</div><div class='add'>+		irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | IRDMA_FLUSH_RQ |</div><div class='add'>+				 IRDMA_FLUSH_WAIT);</div><div class='add'>+</div><div class='add'>+		if (qp-&gt;term_flags)</div><div class='add'>+			irdma_ib_qp_event(iwqp, qp-&gt;event_type);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!cm_id || !cm_id-&gt;event_handler)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwdev-&gt;cm_core.ht_lock, flags);</div><div class='add'>+	if (!iwqp-&gt;cm_node) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;iwdev-&gt;cm_core.ht_lock, flags);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+	refcount_inc(&amp;iwqp-&gt;cm_node-&gt;refcnt);</div><div class='add'>+</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwdev-&gt;cm_core.ht_lock, flags);</div><div class='add'>+</div><div class='add'>+	if (issue_disconn) {</div><div class='add'>+		err = irdma_send_cm_event(iwqp-&gt;cm_node, cm_id,</div><div class='add'>+					  IW_CM_EVENT_DISCONNECT,</div><div class='add'>+					  disconn_status);</div><div class='add'>+		if (err)</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: disconnect event failed: - cm_id = %p\n",</div><div class='add'>+				  cm_id);</div><div class='add'>+	}</div><div class='add'>+	if (issue_close) {</div><div class='add'>+		cm_id-&gt;provider_data = iwqp;</div><div class='add'>+		err = irdma_send_cm_event(iwqp-&gt;cm_node, cm_id,</div><div class='add'>+					  IW_CM_EVENT_CLOSE, 0);</div><div class='add'>+		if (err)</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: close event failed: - cm_id = %p\n",</div><div class='add'>+				  cm_id);</div><div class='add'>+		irdma_qp_disconnect(iwqp);</div><div class='add'>+	}</div><div class='add'>+	irdma_rem_ref_cm_node(iwqp-&gt;cm_node);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_disconnect_worker - worker for connection close</div><div class='add'>+ * @work: points or disconn structure</div><div class='add'>+ */</div><div class='add'>+static void irdma_disconnect_worker(struct work_struct *work)</div><div class='add'>+{</div><div class='add'>+	struct disconn_work *dwork = container_of(work, struct disconn_work, work);</div><div class='add'>+	struct irdma_qp *iwqp = dwork-&gt;iwqp;</div><div class='add'>+</div><div class='add'>+	kfree(dwork);</div><div class='add'>+	irdma_cm_disconn_true(iwqp);</div><div class='add'>+	irdma_qp_rem_ref(&amp;iwqp-&gt;ibqp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_free_lsmm_rsrc - free lsmm memory and deregister</div><div class='add'>+ * @iwqp: associate qp for the connection</div><div class='add'>+ */</div><div class='add'>+void irdma_free_lsmm_rsrc(struct irdma_qp *iwqp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+</div><div class='add'>+	iwdev = iwqp-&gt;iwdev;</div><div class='add'>+</div><div class='add'>+	if (iwqp-&gt;ietf_mem.va) {</div><div class='add'>+		if (iwqp-&gt;lsmm_mr)</div><div class='add'>+			iwdev-&gt;ibdev.ops.dereg_mr(iwqp-&gt;lsmm_mr, NULL);</div><div class='add'>+		dma_free_coherent(iwdev-&gt;rf-&gt;sc_dev.hw-&gt;device,</div><div class='add'>+				  iwqp-&gt;ietf_mem.size, iwqp-&gt;ietf_mem.va,</div><div class='add'>+				  iwqp-&gt;ietf_mem.pa);</div><div class='add'>+		iwqp-&gt;ietf_mem.va = NULL;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_accept - registered call for connection to be accepted</div><div class='add'>+ * @cm_id: cm information for passive connection</div><div class='add'>+ * @conn_param: accept parameters</div><div class='add'>+ */</div><div class='add'>+int irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)</div><div class='add'>+{</div><div class='add'>+	struct ib_qp *ibqp;</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+	struct ib_qp_attr attr = {};</div><div class='add'>+	int passive_state;</div><div class='add'>+	struct ib_mr *ibmr;</div><div class='add'>+	struct irdma_pd *iwpd;</div><div class='add'>+	u16 buf_len = 0;</div><div class='add'>+	struct irdma_kmem_info accept;</div><div class='add'>+	u64 tagged_offset;</div><div class='add'>+	int wait_ret;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	ibqp = irdma_get_qp(cm_id-&gt;device, conn_param-&gt;qpn);</div><div class='add'>+	if (!ibqp)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	iwqp = to_iwqp(ibqp);</div><div class='add'>+	iwdev = iwqp-&gt;iwdev;</div><div class='add'>+	dev = &amp;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+	cm_node = cm_id-&gt;provider_data;</div><div class='add'>+</div><div class='add'>+	if (((struct sockaddr_in *)&amp;cm_id-&gt;local_addr)-&gt;sin_family == AF_INET) {</div><div class='add'>+		cm_node-&gt;ipv4 = true;</div><div class='add'>+		cm_node-&gt;vlan_id = irdma_get_vlan_ipv4(cm_node-&gt;loc_addr);</div><div class='add'>+	} else {</div><div class='add'>+		cm_node-&gt;ipv4 = false;</div><div class='add'>+		irdma_get_vlan_mac_ipv6(cm_node-&gt;loc_addr, &amp;cm_node-&gt;vlan_id,</div><div class='add'>+					NULL);</div><div class='add'>+	}</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev, "CM: Accept vlan_id=%d\n",</div><div class='add'>+		  cm_node-&gt;vlan_id);</div><div class='add'>+</div><div class='add'>+	trace_irdma_accept(cm_node, 0, NULL);</div><div class='add'>+</div><div class='add'>+	if (cm_node-&gt;state == IRDMA_CM_STATE_LISTENER_DESTROYED) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	passive_state = atomic_add_return(1, &amp;cm_node-&gt;passive_state);</div><div class='add'>+	if (passive_state == IRDMA_SEND_RESET_EVENT) {</div><div class='add'>+		ret = -ECONNRESET;</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	buf_len = conn_param-&gt;private_data_len + IRDMA_MAX_IETF_SIZE;</div><div class='add'>+	iwqp-&gt;ietf_mem.size = ALIGN(buf_len, 1);</div><div class='add'>+	iwqp-&gt;ietf_mem.va = dma_alloc_coherent(dev-&gt;hw-&gt;device,</div><div class='add'>+					       iwqp-&gt;ietf_mem.size,</div><div class='add'>+					       &amp;iwqp-&gt;ietf_mem.pa, GFP_KERNEL);</div><div class='add'>+	if (!iwqp-&gt;ietf_mem.va) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;pdata.size = conn_param-&gt;private_data_len;</div><div class='add'>+	accept.addr = iwqp-&gt;ietf_mem.va;</div><div class='add'>+	accept.size = irdma_cm_build_mpa_frame(cm_node, &amp;accept, MPA_KEY_REPLY);</div><div class='add'>+	memcpy((u8 *)accept.addr + accept.size, conn_param-&gt;private_data,</div><div class='add'>+	       conn_param-&gt;private_data_len);</div><div class='add'>+</div><div class='add'>+	if (cm_node-&gt;dev-&gt;ws_add(iwqp-&gt;sc_qp.vsi, cm_node-&gt;user_pri)) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+	iwqp-&gt;sc_qp.user_pri = cm_node-&gt;user_pri;</div><div class='add'>+	irdma_qp_add_qos(&amp;iwqp-&gt;sc_qp);</div><div class='add'>+	/* setup our first outgoing iWarp send WQE (the IETF frame response) */</div><div class='add'>+	iwpd = iwqp-&gt;iwpd;</div><div class='add'>+	tagged_offset = (uintptr_t)iwqp-&gt;ietf_mem.va;</div><div class='add'>+	ibmr = irdma_reg_phys_mr(&amp;iwpd-&gt;ibpd, iwqp-&gt;ietf_mem.pa, buf_len,</div><div class='add'>+				 IB_ACCESS_LOCAL_WRITE, &amp;tagged_offset, false);</div><div class='add'>+	if (IS_ERR(ibmr)) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ibmr-&gt;pd = &amp;iwpd-&gt;ibpd;</div><div class='add'>+	ibmr-&gt;device = iwpd-&gt;ibpd.device;</div><div class='add'>+	iwqp-&gt;lsmm_mr = ibmr;</div><div class='add'>+	if (iwqp-&gt;page)</div><div class='add'>+		iwqp-&gt;sc_qp.qp_uk.sq_base = kmap_local_page(iwqp-&gt;page);</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;lsmm_size = accept.size + conn_param-&gt;private_data_len;</div><div class='add'>+	irdma_sc_send_lsmm(&amp;iwqp-&gt;sc_qp, iwqp-&gt;ietf_mem.va, cm_node-&gt;lsmm_size,</div><div class='add'>+			   ibmr-&gt;lkey);</div><div class='add'>+</div><div class='add'>+	if (iwqp-&gt;page)</div><div class='add'>+		kunmap_local(iwqp-&gt;sc_qp.qp_uk.sq_base);</div><div class='add'>+</div><div class='add'>+	iwqp-&gt;cm_id = cm_id;</div><div class='add'>+	cm_node-&gt;cm_id = cm_id;</div><div class='add'>+</div><div class='add'>+	cm_id-&gt;provider_data = iwqp;</div><div class='add'>+	iwqp-&gt;active_conn = 0;</div><div class='add'>+	iwqp-&gt;cm_node = cm_node;</div><div class='add'>+	cm_node-&gt;iwqp = iwqp;</div><div class='add'>+	irdma_cm_init_tsa_conn(iwqp, cm_node);</div><div class='add'>+	irdma_qp_add_ref(&amp;iwqp-&gt;ibqp);</div><div class='add'>+	cm_id-&gt;add_ref(cm_id);</div><div class='add'>+</div><div class='add'>+	attr.qp_state = IB_QPS_RTS;</div><div class='add'>+	cm_node-&gt;qhash_set = false;</div><div class='add'>+	cm_node-&gt;cm_core-&gt;cm_free_ah(cm_node);</div><div class='add'>+</div><div class='add'>+	irdma_modify_qp(&amp;iwqp-&gt;ibqp, &amp;attr, IB_QP_STATE, NULL);</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.feature_flags &amp; IRDMA_FEATURE_RTS_AE) {</div><div class='add'>+		wait_ret = wait_event_interruptible_timeout(iwqp-&gt;waitq,</div><div class='add'>+							    iwqp-&gt;rts_ae_rcvd,</div><div class='add'>+							    IRDMA_MAX_TIMEOUT);</div><div class='add'>+		if (!wait_ret) {</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n",</div><div class='add'>+				  cm_node, cm_node-&gt;loc_port,</div><div class='add'>+				  cm_node-&gt;rem_port, cm_node-&gt;cm_id);</div><div class='add'>+			ret = -ECONNRESET;</div><div class='add'>+			goto error;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_send_cm_event(cm_node, cm_id, IW_CM_EVENT_ESTABLISHED, 0);</div><div class='add'>+	cm_node-&gt;accelerated = true;</div><div class='add'>+	complete(&amp;cm_node-&gt;establish_comp);</div><div class='add'>+</div><div class='add'>+	if (cm_node-&gt;accept_pend) {</div><div class='add'>+		atomic_dec(&amp;cm_node-&gt;listener-&gt;pend_accepts_cnt);</div><div class='add'>+		cm_node-&gt;accept_pend = 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+		  "CM: rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n",</div><div class='add'>+		  cm_node-&gt;rem_port, cm_node-&gt;loc_port, cm_node-&gt;rem_addr,</div><div class='add'>+		  cm_node-&gt;loc_addr, cm_node, cm_id, ibqp-&gt;qp_num);</div><div class='add'>+	cm_node-&gt;cm_core-&gt;stats_accepts++;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+error:</div><div class='add'>+	irdma_free_lsmm_rsrc(iwqp);</div><div class='add'>+	irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_reject - registered call for connection to be rejected</div><div class='add'>+ * @cm_id: cm information for passive connection</div><div class='add'>+ * @pdata: private data to be sent</div><div class='add'>+ * @pdata_len: private data length</div><div class='add'>+ */</div><div class='add'>+int irdma_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+</div><div class='add'>+	cm_node = cm_id-&gt;provider_data;</div><div class='add'>+	cm_node-&gt;pdata.size = pdata_len;</div><div class='add'>+</div><div class='add'>+	trace_irdma_reject(cm_node, 0, NULL);</div><div class='add'>+</div><div class='add'>+	iwdev = to_iwdev(cm_id-&gt;device);</div><div class='add'>+	if (!iwdev)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;cm_core-&gt;stats_rejects++;</div><div class='add'>+</div><div class='add'>+	if (pdata_len + sizeof(struct ietf_mpa_v2) &gt; IRDMA_MAX_CM_BUF)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	return irdma_cm_reject(cm_node, pdata, pdata_len);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_connect - registered call for connection to be established</div><div class='add'>+ * @cm_id: cm information for passive connection</div><div class='add'>+ * @conn_param: Information about the connection</div><div class='add'>+ */</div><div class='add'>+int irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)</div><div class='add'>+{</div><div class='add'>+	struct ib_qp *ibqp;</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+	struct irdma_cm_info cm_info;</div><div class='add'>+	struct sockaddr_in *laddr;</div><div class='add'>+	struct sockaddr_in *raddr;</div><div class='add'>+	struct sockaddr_in6 *laddr6;</div><div class='add'>+	struct sockaddr_in6 *raddr6;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	ibqp = irdma_get_qp(cm_id-&gt;device, conn_param-&gt;qpn);</div><div class='add'>+	if (!ibqp)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	iwqp = to_iwqp(ibqp);</div><div class='add'>+	if (!iwqp)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	iwdev = iwqp-&gt;iwdev;</div><div class='add'>+	if (!iwdev)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	laddr = (struct sockaddr_in *)&amp;cm_id-&gt;m_local_addr;</div><div class='add'>+	raddr = (struct sockaddr_in *)&amp;cm_id-&gt;m_remote_addr;</div><div class='add'>+	laddr6 = (struct sockaddr_in6 *)&amp;cm_id-&gt;m_local_addr;</div><div class='add'>+	raddr6 = (struct sockaddr_in6 *)&amp;cm_id-&gt;m_remote_addr;</div><div class='add'>+</div><div class='add'>+	if (!(laddr-&gt;sin_port) || !(raddr-&gt;sin_port))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	iwqp-&gt;active_conn = 1;</div><div class='add'>+	iwqp-&gt;cm_id = NULL;</div><div class='add'>+	cm_id-&gt;provider_data = iwqp;</div><div class='add'>+</div><div class='add'>+	/* set up the connection params for the node */</div><div class='add'>+	if (cm_id-&gt;remote_addr.ss_family == AF_INET) {</div><div class='add'>+		if (iwdev-&gt;vsi.mtu &lt; IRDMA_MIN_MTU_IPV4)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		cm_info.ipv4 = true;</div><div class='add'>+		memset(cm_info.loc_addr, 0, sizeof(cm_info.loc_addr));</div><div class='add'>+		memset(cm_info.rem_addr, 0, sizeof(cm_info.rem_addr));</div><div class='add'>+		cm_info.loc_addr[0] = ntohl(laddr-&gt;sin_addr.s_addr);</div><div class='add'>+		cm_info.rem_addr[0] = ntohl(raddr-&gt;sin_addr.s_addr);</div><div class='add'>+		cm_info.loc_port = ntohs(laddr-&gt;sin_port);</div><div class='add'>+		cm_info.rem_port = ntohs(raddr-&gt;sin_port);</div><div class='add'>+		cm_info.vlan_id = irdma_get_vlan_ipv4(cm_info.loc_addr);</div><div class='add'>+	} else {</div><div class='add'>+		if (iwdev-&gt;vsi.mtu &lt; IRDMA_MIN_MTU_IPV6)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		cm_info.ipv4 = false;</div><div class='add'>+		irdma_copy_ip_ntohl(cm_info.loc_addr,</div><div class='add'>+				    laddr6-&gt;sin6_addr.in6_u.u6_addr32);</div><div class='add'>+		irdma_copy_ip_ntohl(cm_info.rem_addr,</div><div class='add'>+				    raddr6-&gt;sin6_addr.in6_u.u6_addr32);</div><div class='add'>+		cm_info.loc_port = ntohs(laddr6-&gt;sin6_port);</div><div class='add'>+		cm_info.rem_port = ntohs(raddr6-&gt;sin6_port);</div><div class='add'>+		irdma_get_vlan_mac_ipv6(cm_info.loc_addr, &amp;cm_info.vlan_id,</div><div class='add'>+					NULL);</div><div class='add'>+	}</div><div class='add'>+	cm_info.cm_id = cm_id;</div><div class='add'>+	cm_info.qh_qpid = iwdev-&gt;vsi.ilq-&gt;qp_id;</div><div class='add'>+	cm_info.tos = cm_id-&gt;tos;</div><div class='add'>+	if (iwdev-&gt;vsi.dscp_mode) {</div><div class='add'>+		cm_info.user_pri =</div><div class='add'>+			iwqp-&gt;sc_qp.vsi-&gt;dscp_map[irdma_tos2dscp(cm_info.tos)];</div><div class='add'>+	} else {</div><div class='add'>+		cm_info.user_pri = rt_tos2priority(cm_id-&gt;tos);</div><div class='add'>+		cm_info.user_pri = irdma_iw_get_vlan_prio(cm_info.loc_addr,</div><div class='add'>+							  cm_info.user_pri,</div><div class='add'>+							  cm_info.ipv4);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (iwqp-&gt;sc_qp.dev-&gt;ws_add(iwqp-&gt;sc_qp.vsi, cm_info.user_pri))</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	iwqp-&gt;sc_qp.user_pri = cm_info.user_pri;</div><div class='add'>+	irdma_qp_add_qos(&amp;iwqp-&gt;sc_qp);</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev, "DCB: TOS:[%d] UP:[%d]\n", cm_id-&gt;tos,</div><div class='add'>+		  cm_info.user_pri);</div><div class='add'>+</div><div class='add'>+	trace_irdma_dcb_tos(iwdev, cm_id-&gt;tos, cm_info.user_pri);</div><div class='add'>+</div><div class='add'>+	ret = irdma_create_cm_node(&amp;iwdev-&gt;cm_core, iwdev, conn_param, &amp;cm_info,</div><div class='add'>+				   &amp;cm_node);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+	ret = cm_node-&gt;cm_core-&gt;cm_create_ah(cm_node, true);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err;</div><div class='add'>+	if (irdma_manage_qhash(iwdev, &amp;cm_info,</div><div class='add'>+			       IRDMA_QHASH_TYPE_TCP_ESTABLISHED,</div><div class='add'>+			       IRDMA_QHASH_MANAGE_TYPE_ADD, NULL, true)) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+	cm_node-&gt;qhash_set = true;</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;apbvt_entry = irdma_add_apbvt(iwdev, cm_info.loc_port);</div><div class='add'>+	if (!cm_node-&gt;apbvt_entry) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;apbvt_set = true;</div><div class='add'>+	iwqp-&gt;cm_node = cm_node;</div><div class='add'>+	cm_node-&gt;iwqp = iwqp;</div><div class='add'>+	iwqp-&gt;cm_id = cm_id;</div><div class='add'>+	irdma_qp_add_ref(&amp;iwqp-&gt;ibqp);</div><div class='add'>+	cm_id-&gt;add_ref(cm_id);</div><div class='add'>+</div><div class='add'>+	if (cm_node-&gt;state != IRDMA_CM_STATE_OFFLOADED) {</div><div class='add'>+		cm_node-&gt;state = IRDMA_CM_STATE_SYN_SENT;</div><div class='add'>+		ret = irdma_send_syn(cm_node, 0);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+		  "CM: rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n",</div><div class='add'>+		  cm_node-&gt;rem_port, cm_node-&gt;loc_port, cm_node-&gt;rem_addr,</div><div class='add'>+		  cm_node-&gt;loc_addr, cm_node, cm_id, ibqp-&gt;qp_num);</div><div class='add'>+</div><div class='add'>+	trace_irdma_connect(cm_node, 0, NULL);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	if (cm_info.ipv4)</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: connect() FAILED: dest addr=%pI4",</div><div class='add'>+			  cm_info.rem_addr);</div><div class='add'>+	else</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: connect() FAILED: dest addr=%pI6",</div><div class='add'>+			  cm_info.rem_addr);</div><div class='add'>+	irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+	iwdev-&gt;cm_core.stats_connect_errs++;</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_create_listen - registered call creating listener</div><div class='add'>+ * @cm_id: cm information for passive connection</div><div class='add'>+ * @backlog: to max accept pending count</div><div class='add'>+ */</div><div class='add'>+int irdma_create_listen(struct iw_cm_id *cm_id, int backlog)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct irdma_cm_listener *cm_listen_node;</div><div class='add'>+	struct irdma_cm_info cm_info = {};</div><div class='add'>+	struct sockaddr_in *laddr;</div><div class='add'>+	struct sockaddr_in6 *laddr6;</div><div class='add'>+	bool wildcard = false;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	iwdev = to_iwdev(cm_id-&gt;device);</div><div class='add'>+	if (!iwdev)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	laddr = (struct sockaddr_in *)&amp;cm_id-&gt;m_local_addr;</div><div class='add'>+	laddr6 = (struct sockaddr_in6 *)&amp;cm_id-&gt;m_local_addr;</div><div class='add'>+	cm_info.qh_qpid = iwdev-&gt;vsi.ilq-&gt;qp_id;</div><div class='add'>+</div><div class='add'>+	if (laddr-&gt;sin_family == AF_INET) {</div><div class='add'>+		if (iwdev-&gt;vsi.mtu &lt; IRDMA_MIN_MTU_IPV4)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		cm_info.ipv4 = true;</div><div class='add'>+		cm_info.loc_addr[0] = ntohl(laddr-&gt;sin_addr.s_addr);</div><div class='add'>+		cm_info.loc_port = ntohs(laddr-&gt;sin_port);</div><div class='add'>+</div><div class='add'>+		if (laddr-&gt;sin_addr.s_addr != htonl(INADDR_ANY)) {</div><div class='add'>+			cm_info.vlan_id = irdma_get_vlan_ipv4(cm_info.loc_addr);</div><div class='add'>+		} else {</div><div class='add'>+			cm_info.vlan_id = 0xFFFF;</div><div class='add'>+			wildcard = true;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		if (iwdev-&gt;vsi.mtu &lt; IRDMA_MIN_MTU_IPV6)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		cm_info.ipv4 = false;</div><div class='add'>+		irdma_copy_ip_ntohl(cm_info.loc_addr,</div><div class='add'>+				    laddr6-&gt;sin6_addr.in6_u.u6_addr32);</div><div class='add'>+		cm_info.loc_port = ntohs(laddr6-&gt;sin6_port);</div><div class='add'>+		if (ipv6_addr_type(&amp;laddr6-&gt;sin6_addr) != IPV6_ADDR_ANY) {</div><div class='add'>+			irdma_get_vlan_mac_ipv6(cm_info.loc_addr,</div><div class='add'>+						&amp;cm_info.vlan_id, NULL);</div><div class='add'>+		} else {</div><div class='add'>+			cm_info.vlan_id = 0xFFFF;</div><div class='add'>+			wildcard = true;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (cm_info.vlan_id &gt;= VLAN_N_VID &amp;&amp; iwdev-&gt;dcb_vlan_mode)</div><div class='add'>+		cm_info.vlan_id = 0;</div><div class='add'>+	cm_info.backlog = backlog;</div><div class='add'>+	cm_info.cm_id = cm_id;</div><div class='add'>+</div><div class='add'>+	trace_irdma_create_listen(iwdev, &amp;cm_info);</div><div class='add'>+</div><div class='add'>+	cm_listen_node = irdma_make_listen_node(&amp;iwdev-&gt;cm_core, iwdev,</div><div class='add'>+						&amp;cm_info);</div><div class='add'>+	if (!cm_listen_node) {</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: cm_listen_node == NULL\n");</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cm_id-&gt;provider_data = cm_listen_node;</div><div class='add'>+</div><div class='add'>+	cm_listen_node-&gt;tos = cm_id-&gt;tos;</div><div class='add'>+	if (iwdev-&gt;vsi.dscp_mode)</div><div class='add'>+		cm_listen_node-&gt;user_pri =</div><div class='add'>+		iwdev-&gt;vsi.dscp_map[irdma_tos2dscp(cm_id-&gt;tos)];</div><div class='add'>+	else</div><div class='add'>+		cm_listen_node-&gt;user_pri = rt_tos2priority(cm_id-&gt;tos);</div><div class='add'>+	cm_info.user_pri = cm_listen_node-&gt;user_pri;</div><div class='add'>+	if (!cm_listen_node-&gt;reused_node) {</div><div class='add'>+		if (wildcard) {</div><div class='add'>+			err = irdma_add_mqh(iwdev, &amp;cm_info, cm_listen_node);</div><div class='add'>+			if (err)</div><div class='add'>+				goto error;</div><div class='add'>+		} else {</div><div class='add'>+			if (!iwdev-&gt;vsi.dscp_mode)</div><div class='add'>+				cm_listen_node-&gt;user_pri =</div><div class='add'>+				irdma_iw_get_vlan_prio(cm_info.loc_addr,</div><div class='add'>+						       cm_info.user_pri,</div><div class='add'>+						       cm_info.ipv4);</div><div class='add'>+			cm_info.user_pri = cm_listen_node-&gt;user_pri;</div><div class='add'>+			err = irdma_manage_qhash(iwdev, &amp;cm_info,</div><div class='add'>+						 IRDMA_QHASH_TYPE_TCP_SYN,</div><div class='add'>+						 IRDMA_QHASH_MANAGE_TYPE_ADD,</div><div class='add'>+						 NULL, true);</div><div class='add'>+			if (err)</div><div class='add'>+				goto error;</div><div class='add'>+</div><div class='add'>+			cm_listen_node-&gt;qhash_set = true;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		cm_listen_node-&gt;apbvt_entry = irdma_add_apbvt(iwdev,</div><div class='add'>+							      cm_info.loc_port);</div><div class='add'>+		if (!cm_listen_node-&gt;apbvt_entry)</div><div class='add'>+			goto error;</div><div class='add'>+	}</div><div class='add'>+	cm_id-&gt;add_ref(cm_id);</div><div class='add'>+	cm_listen_node-&gt;cm_core-&gt;stats_listen_created++;</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+		  "CM: loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d\n",</div><div class='add'>+		  cm_listen_node-&gt;loc_port, cm_listen_node-&gt;loc_addr,</div><div class='add'>+		  cm_listen_node, cm_listen_node-&gt;cm_id,</div><div class='add'>+		  cm_listen_node-&gt;qhash_set, cm_listen_node-&gt;vlan_id);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+error:</div><div class='add'>+</div><div class='add'>+	irdma_cm_del_listen(&amp;iwdev-&gt;cm_core, cm_listen_node, false);</div><div class='add'>+</div><div class='add'>+	return -EINVAL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_destroy_listen - registered call to destroy listener</div><div class='add'>+ * @cm_id: cm information for passive connection</div><div class='add'>+ */</div><div class='add'>+int irdma_destroy_listen(struct iw_cm_id *cm_id)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+</div><div class='add'>+	iwdev = to_iwdev(cm_id-&gt;device);</div><div class='add'>+	if (cm_id-&gt;provider_data)</div><div class='add'>+		irdma_cm_del_listen(&amp;iwdev-&gt;cm_core, cm_id-&gt;provider_data,</div><div class='add'>+				    true);</div><div class='add'>+	else</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: cm_id-&gt;provider_data was NULL\n");</div><div class='add'>+</div><div class='add'>+	cm_id-&gt;rem_ref(cm_id);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_teardown_list_prep - add conn nodes slated for tear down to list</div><div class='add'>+ * @cm_core: cm's core</div><div class='add'>+ * @teardown_list: a list to which cm_node will be selected</div><div class='add'>+ * @ipaddr: pointer to ip address</div><div class='add'>+ * @nfo: pointer to cm_info structure instance</div><div class='add'>+ * @disconnect_all: flag indicating disconnect all QPs</div><div class='add'>+ */</div><div class='add'>+static void irdma_teardown_list_prep(struct irdma_cm_core *cm_core,</div><div class='add'>+				     struct list_head *teardown_list,</div><div class='add'>+				     u32 *ipaddr,</div><div class='add'>+				     struct irdma_cm_info *nfo,</div><div class='add'>+				     bool disconnect_all)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+	int bkt;</div><div class='add'>+</div><div class='add'>+	hash_for_each_rcu(cm_core-&gt;cm_hash_tbl, bkt, cm_node, list) {</div><div class='add'>+		if ((disconnect_all ||</div><div class='add'>+		     (nfo-&gt;vlan_id == cm_node-&gt;vlan_id &amp;&amp;</div><div class='add'>+		      !memcmp(cm_node-&gt;loc_addr, ipaddr, nfo-&gt;ipv4 ? 4 : 16))) &amp;&amp;</div><div class='add'>+		    refcount_inc_not_zero(&amp;cm_node-&gt;refcnt))</div><div class='add'>+			list_add(&amp;cm_node-&gt;teardown_entry, teardown_list);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cm_event_connected - handle connected active node</div><div class='add'>+ * @event: the info for cm_node of connection</div><div class='add'>+ */</div><div class='add'>+static void irdma_cm_event_connected(struct irdma_cm_event *event)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	struct ib_qp_attr attr = {};</div><div class='add'>+	struct iw_cm_id *cm_id;</div><div class='add'>+	int status;</div><div class='add'>+	bool read0;</div><div class='add'>+	int wait_ret = 0;</div><div class='add'>+</div><div class='add'>+	cm_node = event-&gt;cm_node;</div><div class='add'>+	cm_id = cm_node-&gt;cm_id;</div><div class='add'>+	iwqp = cm_id-&gt;provider_data;</div><div class='add'>+	iwdev = iwqp-&gt;iwdev;</div><div class='add'>+	dev = &amp;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+	if (iwqp-&gt;sc_qp.qp_uk.destroy_pending) {</div><div class='add'>+		status = -ETIMEDOUT;</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_cm_init_tsa_conn(iwqp, cm_node);</div><div class='add'>+	read0 = (cm_node-&gt;send_rdma0_op == SEND_RDMA_READ_ZERO);</div><div class='add'>+	if (iwqp-&gt;page)</div><div class='add'>+		iwqp-&gt;sc_qp.qp_uk.sq_base = kmap_local_page(iwqp-&gt;page);</div><div class='add'>+	irdma_sc_send_rtt(&amp;iwqp-&gt;sc_qp, read0);</div><div class='add'>+	if (iwqp-&gt;page)</div><div class='add'>+		kunmap_local(iwqp-&gt;sc_qp.qp_uk.sq_base);</div><div class='add'>+</div><div class='add'>+	attr.qp_state = IB_QPS_RTS;</div><div class='add'>+	cm_node-&gt;qhash_set = false;</div><div class='add'>+	irdma_modify_qp(&amp;iwqp-&gt;ibqp, &amp;attr, IB_QP_STATE, NULL);</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.feature_flags &amp; IRDMA_FEATURE_RTS_AE) {</div><div class='add'>+		wait_ret = wait_event_interruptible_timeout(iwqp-&gt;waitq,</div><div class='add'>+							    iwqp-&gt;rts_ae_rcvd,</div><div class='add'>+							    IRDMA_MAX_TIMEOUT);</div><div class='add'>+		if (!wait_ret)</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+				  "CM: Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n",</div><div class='add'>+				  cm_node, cm_node-&gt;loc_port,</div><div class='add'>+				  cm_node-&gt;rem_port, cm_node-&gt;cm_id);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_send_cm_event(cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY, 0);</div><div class='add'>+	cm_node-&gt;accelerated = true;</div><div class='add'>+	complete(&amp;cm_node-&gt;establish_comp);</div><div class='add'>+	cm_node-&gt;cm_core-&gt;cm_free_ah(cm_node);</div><div class='add'>+	return;</div><div class='add'>+</div><div class='add'>+error:</div><div class='add'>+	iwqp-&gt;cm_id = NULL;</div><div class='add'>+	cm_id-&gt;provider_data = NULL;</div><div class='add'>+	irdma_send_cm_event(event-&gt;cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY,</div><div class='add'>+			    status);</div><div class='add'>+	irdma_rem_ref_cm_node(event-&gt;cm_node);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cm_event_reset - handle reset</div><div class='add'>+ * @event: the info for cm_node of connection</div><div class='add'>+ */</div><div class='add'>+static void irdma_cm_event_reset(struct irdma_cm_event *event)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_node *cm_node = event-&gt;cm_node;</div><div class='add'>+	struct iw_cm_id *cm_id = cm_node-&gt;cm_id;</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+</div><div class='add'>+	if (!cm_id)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	iwqp = cm_id-&gt;provider_data;</div><div class='add'>+	if (!iwqp)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+		  "CM: reset event %p - cm_id = %p\n", event-&gt;cm_node, cm_id);</div><div class='add'>+	iwqp-&gt;cm_id = NULL;</div><div class='add'>+</div><div class='add'>+	irdma_send_cm_event(cm_node, cm_node-&gt;cm_id, IW_CM_EVENT_DISCONNECT,</div><div class='add'>+			    -ECONNRESET);</div><div class='add'>+	irdma_send_cm_event(cm_node, cm_node-&gt;cm_id, IW_CM_EVENT_CLOSE, 0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cm_event_handler - send event to cm upper layer</div><div class='add'>+ * @work: pointer of cm event info.</div><div class='add'>+ */</div><div class='add'>+static void irdma_cm_event_handler(struct work_struct *work)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_event *event = container_of(work, struct irdma_cm_event, event_work);</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+</div><div class='add'>+	if (!event || !event-&gt;cm_node || !event-&gt;cm_node-&gt;cm_core)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	cm_node = event-&gt;cm_node;</div><div class='add'>+	trace_irdma_cm_event_handler(cm_node, event-&gt;type, NULL);</div><div class='add'>+</div><div class='add'>+	switch (event-&gt;type) {</div><div class='add'>+	case IRDMA_CM_EVENT_MPA_REQ:</div><div class='add'>+		irdma_send_cm_event(cm_node, cm_node-&gt;cm_id,</div><div class='add'>+				    IW_CM_EVENT_CONNECT_REQUEST, 0);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_EVENT_RESET:</div><div class='add'>+		irdma_cm_event_reset(event);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_EVENT_CONNECTED:</div><div class='add'>+		if (!event-&gt;cm_node-&gt;cm_id ||</div><div class='add'>+		    event-&gt;cm_node-&gt;state != IRDMA_CM_STATE_OFFLOADED)</div><div class='add'>+			break;</div><div class='add'>+		irdma_cm_event_connected(event);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_EVENT_MPA_REJECT:</div><div class='add'>+		if (!event-&gt;cm_node-&gt;cm_id ||</div><div class='add'>+		    cm_node-&gt;state == IRDMA_CM_STATE_OFFLOADED)</div><div class='add'>+			break;</div><div class='add'>+		irdma_send_cm_event(cm_node, cm_node-&gt;cm_id,</div><div class='add'>+				    IW_CM_EVENT_CONNECT_REPLY, -ECONNREFUSED);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_CM_EVENT_ABORTED:</div><div class='add'>+		if (!event-&gt;cm_node-&gt;cm_id ||</div><div class='add'>+		    event-&gt;cm_node-&gt;state == IRDMA_CM_STATE_OFFLOADED)</div><div class='add'>+			break;</div><div class='add'>+		irdma_event_connect_error(event);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ibdev_dbg(&amp;cm_node-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: bad event type = %d\n", event-&gt;type);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_rem_ref_cm_node(event-&gt;cm_node);</div><div class='add'>+	kfree(event);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cm_post_event - queue event request for worker thread</div><div class='add'>+ * @event: cm node's info for up event call</div><div class='add'>+ */</div><div class='add'>+static void irdma_cm_post_event(struct irdma_cm_event *event)</div><div class='add'>+{</div><div class='add'>+	refcount_inc(&amp;event-&gt;cm_node-&gt;refcnt);</div><div class='add'>+	INIT_WORK(&amp;event-&gt;event_work, irdma_cm_event_handler);</div><div class='add'>+	queue_work(event-&gt;cm_node-&gt;cm_core-&gt;event_wq, &amp;event-&gt;event_work);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cm_teardown_connections - teardown QPs</div><div class='add'>+ * @iwdev: device pointer</div><div class='add'>+ * @ipaddr: Pointer to IPv4 or IPv6 address</div><div class='add'>+ * @nfo: Connection info</div><div class='add'>+ * @disconnect_all: flag indicating disconnect all QPs</div><div class='add'>+ *</div><div class='add'>+ * teardown QPs where source or destination addr matches ip addr</div><div class='add'>+ */</div><div class='add'>+void irdma_cm_teardown_connections(struct irdma_device *iwdev, u32 *ipaddr,</div><div class='add'>+				   struct irdma_cm_info *nfo,</div><div class='add'>+				   bool disconnect_all)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_core *cm_core = &amp;iwdev-&gt;cm_core;</div><div class='add'>+	struct list_head *list_core_temp;</div><div class='add'>+	struct list_head *list_node;</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+	struct list_head teardown_list;</div><div class='add'>+	struct ib_qp_attr attr;</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;teardown_list);</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	irdma_teardown_list_prep(cm_core, &amp;teardown_list, ipaddr, nfo, disconnect_all);</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+</div><div class='add'>+	list_for_each_safe (list_node, list_core_temp, &amp;teardown_list) {</div><div class='add'>+		cm_node = container_of(list_node, struct irdma_cm_node,</div><div class='add'>+				       teardown_entry);</div><div class='add'>+		attr.qp_state = IB_QPS_ERR;</div><div class='add'>+		irdma_modify_qp(&amp;cm_node-&gt;iwqp-&gt;ibqp, &amp;attr, IB_QP_STATE, NULL);</div><div class='add'>+		if (iwdev-&gt;rf-&gt;reset)</div><div class='add'>+			irdma_cm_disconn(cm_node-&gt;iwqp);</div><div class='add'>+		irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_qhash_ctrl - enable/disable qhash for list</div><div class='add'>+ * @iwdev: device pointer</div><div class='add'>+ * @parent_listen_node: parent listen node</div><div class='add'>+ * @nfo: cm info node</div><div class='add'>+ * @ipaddr: Pointer to IPv4 or IPv6 address</div><div class='add'>+ * @ipv4: flag indicating IPv4 when true</div><div class='add'>+ * @ifup: flag indicating interface up when true</div><div class='add'>+ *</div><div class='add'>+ * Enables or disables the qhash for the node in the child</div><div class='add'>+ * listen list that matches ipaddr. If no matching IP was found</div><div class='add'>+ * it will allocate and add a new child listen node to the</div><div class='add'>+ * parent listen node. The listen_list_lock is assumed to be</div><div class='add'>+ * held when called.</div><div class='add'>+ */</div><div class='add'>+static void irdma_qhash_ctrl(struct irdma_device *iwdev,</div><div class='add'>+			     struct irdma_cm_listener *parent_listen_node,</div><div class='add'>+			     struct irdma_cm_info *nfo, u32 *ipaddr, bool ipv4,</div><div class='add'>+			     bool ifup)</div><div class='add'>+{</div><div class='add'>+	struct list_head *child_listen_list = &amp;parent_listen_node-&gt;child_listen_list;</div><div class='add'>+	struct irdma_cm_listener *child_listen_node;</div><div class='add'>+	struct list_head *pos, *tpos;</div><div class='add'>+	bool node_allocated = false;</div><div class='add'>+	enum irdma_quad_hash_manage_type op = ifup ?</div><div class='add'>+					      IRDMA_QHASH_MANAGE_TYPE_ADD :</div><div class='add'>+					      IRDMA_QHASH_MANAGE_TYPE_DELETE;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	list_for_each_safe (pos, tpos, child_listen_list) {</div><div class='add'>+		child_listen_node = list_entry(pos, struct irdma_cm_listener,</div><div class='add'>+					       child_listen_list);</div><div class='add'>+		if (!memcmp(child_listen_node-&gt;loc_addr, ipaddr, ipv4 ? 4 : 16))</div><div class='add'>+			goto set_qhash;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* if not found then add a child listener if interface is going up */</div><div class='add'>+	if (!ifup)</div><div class='add'>+		return;</div><div class='add'>+	child_listen_node = kmemdup(parent_listen_node,</div><div class='add'>+				    sizeof(*child_listen_node), GFP_ATOMIC);</div><div class='add'>+	if (!child_listen_node)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	node_allocated = true;</div><div class='add'>+	memcpy(child_listen_node-&gt;loc_addr, ipaddr, ipv4 ? 4 : 16);</div><div class='add'>+</div><div class='add'>+set_qhash:</div><div class='add'>+	memcpy(nfo-&gt;loc_addr, child_listen_node-&gt;loc_addr,</div><div class='add'>+	       sizeof(nfo-&gt;loc_addr));</div><div class='add'>+	nfo-&gt;vlan_id = child_listen_node-&gt;vlan_id;</div><div class='add'>+	err = irdma_manage_qhash(iwdev, nfo, IRDMA_QHASH_TYPE_TCP_SYN, op, NULL,</div><div class='add'>+				 false);</div><div class='add'>+	if (!err) {</div><div class='add'>+		child_listen_node-&gt;qhash_set = ifup;</div><div class='add'>+		if (node_allocated)</div><div class='add'>+			list_add(&amp;child_listen_node-&gt;child_listen_list,</div><div class='add'>+				 &amp;parent_listen_node-&gt;child_listen_list);</div><div class='add'>+	} else if (node_allocated) {</div><div class='add'>+		kfree(child_listen_node);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_if_notify - process an ifdown on an interface</div><div class='add'>+ * @iwdev: device pointer</div><div class='add'>+ * @netdev: network device structure</div><div class='add'>+ * @ipaddr: Pointer to IPv4 or IPv6 address</div><div class='add'>+ * @ipv4: flag indicating IPv4 when true</div><div class='add'>+ * @ifup: flag indicating interface up when true</div><div class='add'>+ */</div><div class='add'>+void irdma_if_notify(struct irdma_device *iwdev, struct net_device *netdev,</div><div class='add'>+		     u32 *ipaddr, bool ipv4, bool ifup)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_core *cm_core = &amp;iwdev-&gt;cm_core;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	struct irdma_cm_listener *listen_node;</div><div class='add'>+	static const u32 ip_zero[4] = { 0, 0, 0, 0 };</div><div class='add'>+	struct irdma_cm_info nfo = {};</div><div class='add'>+	u16 vlan_id = rdma_vlan_dev_vlan_id(netdev);</div><div class='add'>+	enum irdma_quad_hash_manage_type op = ifup ?</div><div class='add'>+					      IRDMA_QHASH_MANAGE_TYPE_ADD :</div><div class='add'>+					      IRDMA_QHASH_MANAGE_TYPE_DELETE;</div><div class='add'>+</div><div class='add'>+	nfo.vlan_id = vlan_id;</div><div class='add'>+	nfo.ipv4 = ipv4;</div><div class='add'>+	nfo.qh_qpid = 1;</div><div class='add'>+</div><div class='add'>+	/* Disable or enable qhash for listeners */</div><div class='add'>+	spin_lock_irqsave(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='add'>+	list_for_each_entry (listen_node, &amp;cm_core-&gt;listen_list, list) {</div><div class='add'>+		if (vlan_id != listen_node-&gt;vlan_id ||</div><div class='add'>+		    (memcmp(listen_node-&gt;loc_addr, ipaddr, ipv4 ? 4 : 16) &amp;&amp;</div><div class='add'>+		     memcmp(listen_node-&gt;loc_addr, ip_zero, ipv4 ? 4 : 16)))</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		memcpy(nfo.loc_addr, listen_node-&gt;loc_addr,</div><div class='add'>+		       sizeof(nfo.loc_addr));</div><div class='add'>+		nfo.loc_port = listen_node-&gt;loc_port;</div><div class='add'>+		nfo.user_pri = listen_node-&gt;user_pri;</div><div class='add'>+		if (!list_empty(&amp;listen_node-&gt;child_listen_list)) {</div><div class='add'>+			irdma_qhash_ctrl(iwdev, listen_node, &amp;nfo, ipaddr, ipv4,</div><div class='add'>+					 ifup);</div><div class='add'>+		} else if (memcmp(listen_node-&gt;loc_addr, ip_zero,</div><div class='add'>+				  ipv4 ? 4 : 16)) {</div><div class='add'>+			if (!irdma_manage_qhash(iwdev, &amp;nfo,</div><div class='add'>+						IRDMA_QHASH_TYPE_TCP_SYN, op,</div><div class='add'>+						NULL, false))</div><div class='add'>+				listen_node-&gt;qhash_set = ifup;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_core-&gt;listen_list_lock, flags);</div><div class='add'>+</div><div class='add'>+	/* disconnect any connected qp's on ifdown */</div><div class='add'>+	if (!ifup)</div><div class='add'>+		irdma_cm_teardown_connections(iwdev, ipaddr, &amp;nfo, false);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/cm.h b/drivers/infiniband/hw/irdma/cm.h<br/>new file mode 100644<br/>index 000000000000..48ee285cf745<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/cm.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/cm.h</a></div><div class='hunk'>@@ -0,0 +1,416 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#ifndef IRDMA_CM_H</div><div class='add'>+#define IRDMA_CM_H</div><div class='add'>+</div><div class='add'>+#define IRDMA_MPA_REQUEST_ACCEPT	1</div><div class='add'>+#define IRDMA_MPA_REQUEST_REJECT	2</div><div class='add'>+</div><div class='add'>+/* IETF MPA -- defines */</div><div class='add'>+#define IEFT_MPA_KEY_REQ	"MPA ID Req Frame"</div><div class='add'>+#define IEFT_MPA_KEY_REP	"MPA ID Rep Frame"</div><div class='add'>+#define IETF_MPA_KEY_SIZE	16</div><div class='add'>+#define IETF_MPA_VER		1</div><div class='add'>+#define IETF_MAX_PRIV_DATA_LEN	512</div><div class='add'>+#define IETF_MPA_FRAME_SIZE	20</div><div class='add'>+#define IETF_RTR_MSG_SIZE	4</div><div class='add'>+#define IETF_MPA_V2_FLAG	0x10</div><div class='add'>+#define SNDMARKER_SEQNMASK	0x000001ff</div><div class='add'>+#define IRDMA_MAX_IETF_SIZE	32</div><div class='add'>+</div><div class='add'>+/* IETF RTR MSG Fields */</div><div class='add'>+#define IETF_PEER_TO_PEER	0x8000</div><div class='add'>+#define IETF_FLPDU_ZERO_LEN	0x4000</div><div class='add'>+#define IETF_RDMA0_WRITE	0x8000</div><div class='add'>+#define IETF_RDMA0_READ		0x4000</div><div class='add'>+#define IETF_NO_IRD_ORD		0x3fff</div><div class='add'>+</div><div class='add'>+#define MAX_PORTS	65536</div><div class='add'>+</div><div class='add'>+#define IRDMA_PASSIVE_STATE_INDICATED	0</div><div class='add'>+#define IRDMA_DO_NOT_SEND_RESET_EVENT	1</div><div class='add'>+#define IRDMA_SEND_RESET_EVENT		2</div><div class='add'>+</div><div class='add'>+#define MAX_IRDMA_IFS	4</div><div class='add'>+</div><div class='add'>+#define SET_ACK		1</div><div class='add'>+#define SET_SYN		2</div><div class='add'>+#define SET_FIN		4</div><div class='add'>+#define SET_RST		8</div><div class='add'>+</div><div class='add'>+#define TCP_OPTIONS_PADDING	3</div><div class='add'>+</div><div class='add'>+#define IRDMA_DEFAULT_RETRYS	64</div><div class='add'>+#define IRDMA_DEFAULT_RETRANS	32</div><div class='add'>+#define IRDMA_DEFAULT_TTL		0x40</div><div class='add'>+#define IRDMA_DEFAULT_RTT_VAR		6</div><div class='add'>+#define IRDMA_DEFAULT_SS_THRESH		0x3fffffff</div><div class='add'>+#define IRDMA_DEFAULT_REXMIT_THRESH	8</div><div class='add'>+</div><div class='add'>+#define IRDMA_RETRY_TIMEOUT	HZ</div><div class='add'>+#define IRDMA_SHORT_TIME	10</div><div class='add'>+#define IRDMA_LONG_TIME		(2 * HZ)</div><div class='add'>+#define IRDMA_MAX_TIMEOUT	((unsigned long)(12 * HZ))</div><div class='add'>+</div><div class='add'>+#define IRDMA_CM_HASHTABLE_SIZE		1024</div><div class='add'>+#define IRDMA_CM_TCP_TIMER_INTERVAL	3000</div><div class='add'>+#define IRDMA_CM_DEFAULT_MTU		1540</div><div class='add'>+#define IRDMA_CM_DEFAULT_FRAME_CNT	10</div><div class='add'>+#define IRDMA_CM_THREAD_STACK_SIZE	256</div><div class='add'>+#define IRDMA_CM_DEFAULT_RCV_WND	64240</div><div class='add'>+#define IRDMA_CM_DEFAULT_RCV_WND_SCALED	0x3FFFC</div><div class='add'>+#define IRDMA_CM_DEFAULT_RCV_WND_SCALE	2</div><div class='add'>+#define IRDMA_CM_DEFAULT_FREE_PKTS	10</div><div class='add'>+#define IRDMA_CM_FREE_PKT_LO_WATERMARK	2</div><div class='add'>+#define IRDMA_CM_DEFAULT_MSS		536</div><div class='add'>+#define IRDMA_CM_DEFAULT_MPA_VER	2</div><div class='add'>+#define IRDMA_CM_DEFAULT_SEQ		0x159bf75f</div><div class='add'>+#define IRDMA_CM_DEFAULT_LOCAL_ID	0x3b47</div><div class='add'>+#define IRDMA_CM_DEFAULT_SEQ2		0x18ed5740</div><div class='add'>+#define IRDMA_CM_DEFAULT_LOCAL_ID2	0xb807</div><div class='add'>+#define IRDMA_MAX_CM_BUF		(IRDMA_MAX_IETF_SIZE + IETF_MAX_PRIV_DATA_LEN)</div><div class='add'>+</div><div class='add'>+enum ietf_mpa_flags {</div><div class='add'>+	IETF_MPA_FLAGS_REJECT  = 0x20,</div><div class='add'>+	IETF_MPA_FLAGS_CRC     = 0x40,</div><div class='add'>+	IETF_MPA_FLAGS_MARKERS = 0x80,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_timer_type {</div><div class='add'>+	IRDMA_TIMER_TYPE_SEND,</div><div class='add'>+	IRDMA_TIMER_TYPE_CLOSE,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum option_nums {</div><div class='add'>+	OPTION_NUM_EOL,</div><div class='add'>+	OPTION_NUM_NONE,</div><div class='add'>+	OPTION_NUM_MSS,</div><div class='add'>+	OPTION_NUM_WINDOW_SCALE,</div><div class='add'>+	OPTION_NUM_SACK_PERM,</div><div class='add'>+	OPTION_NUM_SACK,</div><div class='add'>+	OPTION_NUM_WRITE0 = 0xbc,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* cm node transition states */</div><div class='add'>+enum irdma_cm_node_state {</div><div class='add'>+	IRDMA_CM_STATE_UNKNOWN,</div><div class='add'>+	IRDMA_CM_STATE_INITED,</div><div class='add'>+	IRDMA_CM_STATE_LISTENING,</div><div class='add'>+	IRDMA_CM_STATE_SYN_RCVD,</div><div class='add'>+	IRDMA_CM_STATE_SYN_SENT,</div><div class='add'>+	IRDMA_CM_STATE_ONE_SIDE_ESTABLISHED,</div><div class='add'>+	IRDMA_CM_STATE_ESTABLISHED,</div><div class='add'>+	IRDMA_CM_STATE_ACCEPTING,</div><div class='add'>+	IRDMA_CM_STATE_MPAREQ_SENT,</div><div class='add'>+	IRDMA_CM_STATE_MPAREQ_RCVD,</div><div class='add'>+	IRDMA_CM_STATE_MPAREJ_RCVD,</div><div class='add'>+	IRDMA_CM_STATE_OFFLOADED,</div><div class='add'>+	IRDMA_CM_STATE_FIN_WAIT1,</div><div class='add'>+	IRDMA_CM_STATE_FIN_WAIT2,</div><div class='add'>+	IRDMA_CM_STATE_CLOSE_WAIT,</div><div class='add'>+	IRDMA_CM_STATE_TIME_WAIT,</div><div class='add'>+	IRDMA_CM_STATE_LAST_ACK,</div><div class='add'>+	IRDMA_CM_STATE_CLOSING,</div><div class='add'>+	IRDMA_CM_STATE_LISTENER_DESTROYED,</div><div class='add'>+	IRDMA_CM_STATE_CLOSED,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum mpa_frame_ver {</div><div class='add'>+	IETF_MPA_V1 = 1,</div><div class='add'>+	IETF_MPA_V2 = 2,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum mpa_frame_key {</div><div class='add'>+	MPA_KEY_REQUEST,</div><div class='add'>+	MPA_KEY_REPLY,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum send_rdma0 {</div><div class='add'>+	SEND_RDMA_READ_ZERO  = 1,</div><div class='add'>+	SEND_RDMA_WRITE_ZERO = 2,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_tcpip_pkt_type {</div><div class='add'>+	IRDMA_PKT_TYPE_UNKNOWN,</div><div class='add'>+	IRDMA_PKT_TYPE_SYN,</div><div class='add'>+	IRDMA_PKT_TYPE_SYNACK,</div><div class='add'>+	IRDMA_PKT_TYPE_ACK,</div><div class='add'>+	IRDMA_PKT_TYPE_FIN,</div><div class='add'>+	IRDMA_PKT_TYPE_RST,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_cm_listener_state {</div><div class='add'>+	IRDMA_CM_LISTENER_PASSIVE_STATE = 1,</div><div class='add'>+	IRDMA_CM_LISTENER_ACTIVE_STATE  = 2,</div><div class='add'>+	IRDMA_CM_LISTENER_EITHER_STATE  = 3,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* CM event codes */</div><div class='add'>+enum irdma_cm_event_type {</div><div class='add'>+	IRDMA_CM_EVENT_UNKNOWN,</div><div class='add'>+	IRDMA_CM_EVENT_ESTABLISHED,</div><div class='add'>+	IRDMA_CM_EVENT_MPA_REQ,</div><div class='add'>+	IRDMA_CM_EVENT_MPA_CONNECT,</div><div class='add'>+	IRDMA_CM_EVENT_MPA_ACCEPT,</div><div class='add'>+	IRDMA_CM_EVENT_MPA_REJECT,</div><div class='add'>+	IRDMA_CM_EVENT_MPA_ESTABLISHED,</div><div class='add'>+	IRDMA_CM_EVENT_CONNECTED,</div><div class='add'>+	IRDMA_CM_EVENT_RESET,</div><div class='add'>+	IRDMA_CM_EVENT_ABORTED,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ietf_mpa_v1 {</div><div class='add'>+	u8 key[IETF_MPA_KEY_SIZE];</div><div class='add'>+	u8 flags;</div><div class='add'>+	u8 rev;</div><div class='add'>+	__be16 priv_data_len;</div><div class='add'>+	u8 priv_data[];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ietf_rtr_msg {</div><div class='add'>+	__be16 ctrl_ird;</div><div class='add'>+	__be16 ctrl_ord;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ietf_mpa_v2 {</div><div class='add'>+	u8 key[IETF_MPA_KEY_SIZE];</div><div class='add'>+	u8 flags;</div><div class='add'>+	u8 rev;</div><div class='add'>+	__be16 priv_data_len;</div><div class='add'>+	struct ietf_rtr_msg rtr_msg;</div><div class='add'>+	u8 priv_data[];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct option_base {</div><div class='add'>+	u8 optionnum;</div><div class='add'>+	u8 len;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct option_mss {</div><div class='add'>+	u8 optionnum;</div><div class='add'>+	u8 len;</div><div class='add'>+	__be16 mss;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct option_windowscale {</div><div class='add'>+	u8 optionnum;</div><div class='add'>+	u8 len;</div><div class='add'>+	u8 shiftcount;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+union all_known_options {</div><div class='add'>+	char eol;</div><div class='add'>+	struct option_base base;</div><div class='add'>+	struct option_mss mss;</div><div class='add'>+	struct option_windowscale windowscale;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_timer_entry {</div><div class='add'>+	struct list_head list;</div><div class='add'>+	unsigned long timetosend; /* jiffies */</div><div class='add'>+	struct irdma_puda_buf *sqbuf;</div><div class='add'>+	u32 type;</div><div class='add'>+	u32 retrycount;</div><div class='add'>+	u32 retranscount;</div><div class='add'>+	u32 context;</div><div class='add'>+	u32 send_retrans;</div><div class='add'>+	int close_when_complete;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* CM context params */</div><div class='add'>+struct irdma_cm_tcp_context {</div><div class='add'>+	u8 client;</div><div class='add'>+	u32 loc_seq_num;</div><div class='add'>+	u32 loc_ack_num;</div><div class='add'>+	u32 rem_ack_num;</div><div class='add'>+	u32 rcv_nxt;</div><div class='add'>+	u32 loc_id;</div><div class='add'>+	u32 rem_id;</div><div class='add'>+	u32 snd_wnd;</div><div class='add'>+	u32 max_snd_wnd;</div><div class='add'>+	u32 rcv_wnd;</div><div class='add'>+	u32 mss;</div><div class='add'>+	u8 snd_wscale;</div><div class='add'>+	u8 rcv_wscale;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_apbvt_entry {</div><div class='add'>+	struct hlist_node hlist;</div><div class='add'>+	u32 use_cnt;</div><div class='add'>+	u16 port;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cm_listener {</div><div class='add'>+	struct list_head list;</div><div class='add'>+	struct iw_cm_id *cm_id;</div><div class='add'>+	struct irdma_cm_core *cm_core;</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct list_head child_listen_list;</div><div class='add'>+	struct irdma_apbvt_entry *apbvt_entry;</div><div class='add'>+	enum irdma_cm_listener_state listener_state;</div><div class='add'>+	refcount_t refcnt;</div><div class='add'>+	atomic_t pend_accepts_cnt;</div><div class='add'>+	u32 loc_addr[4];</div><div class='add'>+	u32 reused_node;</div><div class='add'>+	int backlog;</div><div class='add'>+	u16 loc_port;</div><div class='add'>+	u16 vlan_id;</div><div class='add'>+	u8 loc_mac[ETH_ALEN];</div><div class='add'>+	u8 user_pri;</div><div class='add'>+	u8 tos;</div><div class='add'>+	bool qhash_set:1;</div><div class='add'>+	bool ipv4:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_kmem_info {</div><div class='add'>+	void *addr;</div><div class='add'>+	u32 size;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_mpa_priv_info {</div><div class='add'>+	const void *addr;</div><div class='add'>+	u32 size;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cm_node {</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	struct irdma_cm_tcp_context tcp_cntxt;</div><div class='add'>+	struct irdma_cm_core *cm_core;</div><div class='add'>+	struct irdma_timer_entry *send_entry;</div><div class='add'>+	struct irdma_timer_entry *close_entry;</div><div class='add'>+	struct irdma_cm_listener *listener;</div><div class='add'>+	struct list_head timer_entry;</div><div class='add'>+	struct list_head reset_entry;</div><div class='add'>+	struct list_head teardown_entry;</div><div class='add'>+	struct irdma_apbvt_entry *apbvt_entry;</div><div class='add'>+	struct rcu_head rcu_head;</div><div class='add'>+	struct irdma_mpa_priv_info pdata;</div><div class='add'>+	struct irdma_sc_ah *ah;</div><div class='add'>+	union {</div><div class='add'>+		struct ietf_mpa_v1 mpa_frame;</div><div class='add'>+		struct ietf_mpa_v2 mpa_v2_frame;</div><div class='add'>+	};</div><div class='add'>+	struct irdma_kmem_info mpa_hdr;</div><div class='add'>+	struct iw_cm_id *cm_id;</div><div class='add'>+	struct hlist_node list;</div><div class='add'>+	struct completion establish_comp;</div><div class='add'>+	spinlock_t retrans_list_lock; /* protect CM node rexmit updates*/</div><div class='add'>+	atomic_t passive_state;</div><div class='add'>+	refcount_t refcnt;</div><div class='add'>+	enum irdma_cm_node_state state;</div><div class='add'>+	enum send_rdma0 send_rdma0_op;</div><div class='add'>+	enum mpa_frame_ver mpa_frame_rev;</div><div class='add'>+	u32 loc_addr[4], rem_addr[4];</div><div class='add'>+	u16 loc_port, rem_port;</div><div class='add'>+	int apbvt_set;</div><div class='add'>+	int accept_pend;</div><div class='add'>+	u16 vlan_id;</div><div class='add'>+	u16 ird_size;</div><div class='add'>+	u16 ord_size;</div><div class='add'>+	u16 mpav2_ird_ord;</div><div class='add'>+	u16 lsmm_size;</div><div class='add'>+	u8 pdata_buf[IETF_MAX_PRIV_DATA_LEN];</div><div class='add'>+	u8 loc_mac[ETH_ALEN];</div><div class='add'>+	u8 rem_mac[ETH_ALEN];</div><div class='add'>+	u8 user_pri;</div><div class='add'>+	u8 tos;</div><div class='add'>+	bool ack_rcvd:1;</div><div class='add'>+	bool qhash_set:1;</div><div class='add'>+	bool ipv4:1;</div><div class='add'>+	bool snd_mark_en:1;</div><div class='add'>+	bool rcv_mark_en:1;</div><div class='add'>+	bool do_lpb:1;</div><div class='add'>+	bool accelerated:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* Used by internal CM APIs to pass CM information*/</div><div class='add'>+struct irdma_cm_info {</div><div class='add'>+	struct iw_cm_id *cm_id;</div><div class='add'>+	u16 loc_port;</div><div class='add'>+	u16 rem_port;</div><div class='add'>+	u32 loc_addr[4];</div><div class='add'>+	u32 rem_addr[4];</div><div class='add'>+	u32 qh_qpid;</div><div class='add'>+	u16 vlan_id;</div><div class='add'>+	int backlog;</div><div class='add'>+	u8 user_pri;</div><div class='add'>+	u8 tos;</div><div class='add'>+	bool ipv4;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cm_event {</div><div class='add'>+	enum irdma_cm_event_type type;</div><div class='add'>+	struct irdma_cm_info cm_info;</div><div class='add'>+	struct work_struct event_work;</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cm_core {</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	struct list_head listen_list;</div><div class='add'>+	DECLARE_HASHTABLE(cm_hash_tbl, 8);</div><div class='add'>+	DECLARE_HASHTABLE(apbvt_hash_tbl, 8);</div><div class='add'>+	struct timer_list tcp_timer;</div><div class='add'>+	struct workqueue_struct *event_wq;</div><div class='add'>+	spinlock_t ht_lock; /* protect CM node (active side) list */</div><div class='add'>+	spinlock_t listen_list_lock; /* protect listener list */</div><div class='add'>+	spinlock_t apbvt_lock; /*serialize apbvt add/del entries*/</div><div class='add'>+	u64 stats_nodes_created;</div><div class='add'>+	u64 stats_nodes_destroyed;</div><div class='add'>+	u64 stats_listen_created;</div><div class='add'>+	u64 stats_listen_destroyed;</div><div class='add'>+	u64 stats_listen_nodes_created;</div><div class='add'>+	u64 stats_listen_nodes_destroyed;</div><div class='add'>+	u64 stats_lpbs;</div><div class='add'>+	u64 stats_accepts;</div><div class='add'>+	u64 stats_rejects;</div><div class='add'>+	u64 stats_connect_errs;</div><div class='add'>+	u64 stats_passive_errs;</div><div class='add'>+	u64 stats_pkt_retrans;</div><div class='add'>+	u64 stats_backlog_drops;</div><div class='add'>+	struct irdma_puda_buf *(*form_cm_frame)(struct irdma_cm_node *cm_node,</div><div class='add'>+						struct irdma_kmem_info *options,</div><div class='add'>+						struct irdma_kmem_info *hdr,</div><div class='add'>+						struct irdma_mpa_priv_info *pdata,</div><div class='add'>+						u8 flags);</div><div class='add'>+	int (*cm_create_ah)(struct irdma_cm_node *cm_node, bool wait);</div><div class='add'>+	void (*cm_free_ah)(struct irdma_cm_node *cm_node);</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+int irdma_schedule_cm_timer(struct irdma_cm_node *cm_node,</div><div class='add'>+			    struct irdma_puda_buf *sqbuf,</div><div class='add'>+			    enum irdma_timer_type type, int send_retrans,</div><div class='add'>+			    int close_when_complete);</div><div class='add'>+</div><div class='add'>+static inline u8 irdma_tos2dscp(u8 tos)</div><div class='add'>+{</div><div class='add'>+#define IRDMA_DSCP_VAL GENMASK(7, 2)</div><div class='add'>+	return (u8)FIELD_GET(IRDMA_DSCP_VAL, tos);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);</div><div class='add'>+int irdma_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);</div><div class='add'>+int irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);</div><div class='add'>+int irdma_create_listen(struct iw_cm_id *cm_id, int backlog);</div><div class='add'>+int irdma_destroy_listen(struct iw_cm_id *cm_id);</div><div class='add'>+int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, const u8 *mac);</div><div class='add'>+void irdma_cm_teardown_connections(struct irdma_device *iwdev, u32 *ipaddr,</div><div class='add'>+				   struct irdma_cm_info *nfo,</div><div class='add'>+				   bool disconnect_all);</div><div class='add'>+int irdma_cm_start(struct irdma_device *dev);</div><div class='add'>+int irdma_cm_stop(struct irdma_device *dev);</div><div class='add'>+bool irdma_ipv4_is_lpb(u32 loc_addr, u32 rem_addr);</div><div class='add'>+bool irdma_ipv6_is_lpb(u32 *loc_addr, u32 *rem_addr);</div><div class='add'>+int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4,</div><div class='add'>+		    const u8 *mac_addr, u32 action);</div><div class='add'>+void irdma_if_notify(struct irdma_device *iwdev, struct net_device *netdev,</div><div class='add'>+		     u32 *ipaddr, bool ipv4, bool ifup);</div><div class='add'>+bool irdma_port_in_use(struct irdma_cm_core *cm_core, u16 port);</div><div class='add'>+void irdma_send_ack(struct irdma_cm_node *cm_node);</div><div class='add'>+void irdma_lpb_nop(struct irdma_sc_qp *qp);</div><div class='add'>+void irdma_rem_ref_cm_node(struct irdma_cm_node *cm_node);</div><div class='add'>+void irdma_add_conn_est_qh(struct irdma_cm_node *cm_node);</div><div class='add'>+#endif /* IRDMA_CM_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c<br/>new file mode 100644<br/>index 000000000000..ce5cf89c463c<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/ctrl.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/ctrl.c</a></div><div class='hunk'>@@ -0,0 +1,6602 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#include &lt;linux/etherdevice.h&gt;</div><div class='add'>+</div><div class='add'>+#include "osdep.h"</div><div class='add'>+#include "hmc.h"</div><div class='add'>+#include "defs.h"</div><div class='add'>+#include "type.h"</div><div class='add'>+#include "ws.h"</div><div class='add'>+#include "protos.h"</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_qp_from_list - get next qp from a list</div><div class='add'>+ * @head: Listhead of qp's</div><div class='add'>+ * @qp: current qp</div><div class='add'>+ */</div><div class='add'>+struct irdma_sc_qp *irdma_get_qp_from_list(struct list_head *head,</div><div class='add'>+					   struct irdma_sc_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct list_head *lastentry;</div><div class='add'>+	struct list_head *entry = NULL;</div><div class='add'>+</div><div class='add'>+	if (list_empty(head))</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	if (!qp) {</div><div class='add'>+		entry = head-&gt;next;</div><div class='add'>+	} else {</div><div class='add'>+		lastentry = &amp;qp-&gt;list;</div><div class='add'>+		entry = lastentry-&gt;next;</div><div class='add'>+		if (entry == head)</div><div class='add'>+			return NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return container_of(entry, struct irdma_sc_qp, list);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_suspend_resume_qps - suspend/resume all qp's on VSI</div><div class='add'>+ * @vsi: the VSI struct pointer</div><div class='add'>+ * @op: Set to IRDMA_OP_RESUME or IRDMA_OP_SUSPEND</div><div class='add'>+ */</div><div class='add'>+void irdma_sc_suspend_resume_qps(struct irdma_sc_vsi *vsi, u8 op)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_qp *qp = NULL;</div><div class='add'>+	u8 i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; IRDMA_MAX_USER_PRIORITY; i++) {</div><div class='add'>+		mutex_lock(&amp;vsi-&gt;qos[i].qos_mutex);</div><div class='add'>+		qp = irdma_get_qp_from_list(&amp;vsi-&gt;qos[i].qplist, qp);</div><div class='add'>+		while (qp) {</div><div class='add'>+			if (op == IRDMA_OP_RESUME) {</div><div class='add'>+				if (!qp-&gt;dev-&gt;ws_add(vsi, i)) {</div><div class='add'>+					qp-&gt;qs_handle =</div><div class='add'>+						vsi-&gt;qos[qp-&gt;user_pri].qs_handle;</div><div class='add'>+					irdma_cqp_qp_suspend_resume(qp, op);</div><div class='add'>+				} else {</div><div class='add'>+					irdma_cqp_qp_suspend_resume(qp, op);</div><div class='add'>+					irdma_modify_qp_to_err(qp);</div><div class='add'>+				}</div><div class='add'>+			} else if (op == IRDMA_OP_SUSPEND) {</div><div class='add'>+				/* issue cqp suspend command */</div><div class='add'>+				if (!irdma_cqp_qp_suspend_resume(qp, op))</div><div class='add'>+					atomic_inc(&amp;vsi-&gt;qp_suspend_reqs);</div><div class='add'>+			}</div><div class='add'>+			qp = irdma_get_qp_from_list(&amp;vsi-&gt;qos[i].qplist, qp);</div><div class='add'>+		}</div><div class='add'>+		mutex_unlock(&amp;vsi-&gt;qos[i].qos_mutex);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_set_qos_info(struct irdma_sc_vsi  *vsi,</div><div class='add'>+			       struct irdma_l2params *l2p)</div><div class='add'>+{</div><div class='add'>+	u8 i;</div><div class='add'>+</div><div class='add'>+	if (vsi-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3) {</div><div class='add'>+		for (i = 0; i &lt; IRDMA_MAX_USER_PRIORITY; i++) {</div><div class='add'>+			vsi-&gt;qos[i].qs_handle = vsi-&gt;dev-&gt;qos[i].qs_handle;</div><div class='add'>+			vsi-&gt;qos[i].valid = true;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+	vsi-&gt;qos_rel_bw = l2p-&gt;vsi_rel_bw;</div><div class='add'>+	vsi-&gt;qos_prio_type = l2p-&gt;vsi_prio_type;</div><div class='add'>+	vsi-&gt;dscp_mode = l2p-&gt;dscp_mode;</div><div class='add'>+	if (l2p-&gt;dscp_mode) {</div><div class='add'>+		memcpy(vsi-&gt;dscp_map, l2p-&gt;dscp_map, sizeof(vsi-&gt;dscp_map));</div><div class='add'>+		for (i = 0; i &lt; IRDMA_MAX_USER_PRIORITY; i++)</div><div class='add'>+			l2p-&gt;up2tc[i] = i;</div><div class='add'>+	}</div><div class='add'>+	for (i = 0; i &lt; IRDMA_MAX_USER_PRIORITY; i++) {</div><div class='add'>+		if (vsi-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)</div><div class='add'>+			vsi-&gt;qos[i].qs_handle = l2p-&gt;qs_handle_list[i];</div><div class='add'>+		vsi-&gt;qos[i].traffic_class = l2p-&gt;up2tc[i];</div><div class='add'>+		vsi-&gt;qos[i].rel_bw =</div><div class='add'>+			l2p-&gt;tc_info[vsi-&gt;qos[i].traffic_class].rel_bw;</div><div class='add'>+		vsi-&gt;qos[i].prio_type =</div><div class='add'>+			l2p-&gt;tc_info[vsi-&gt;qos[i].traffic_class].prio_type;</div><div class='add'>+		vsi-&gt;qos[i].valid = false;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_change_l2params - given the new l2 parameters, change all qp</div><div class='add'>+ * @vsi: RDMA VSI pointer</div><div class='add'>+ * @l2params: New parameters from l2</div><div class='add'>+ */</div><div class='add'>+void irdma_change_l2params(struct irdma_sc_vsi *vsi,</div><div class='add'>+			   struct irdma_l2params *l2params)</div><div class='add'>+{</div><div class='add'>+	if (l2params-&gt;mtu_changed) {</div><div class='add'>+		vsi-&gt;mtu = l2params-&gt;mtu;</div><div class='add'>+		if (vsi-&gt;ieq)</div><div class='add'>+			irdma_reinitialize_ieq(vsi);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!l2params-&gt;tc_changed)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	vsi-&gt;tc_change_pending = false;</div><div class='add'>+	irdma_set_qos_info(vsi, l2params);</div><div class='add'>+	irdma_sc_suspend_resume_qps(vsi, IRDMA_OP_RESUME);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_qp_rem_qos - remove qp from qos lists during destroy qp</div><div class='add'>+ * @qp: qp to be removed from qos</div><div class='add'>+ */</div><div class='add'>+void irdma_qp_rem_qos(struct irdma_sc_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_vsi *vsi = qp-&gt;vsi;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(to_ibdev(qp-&gt;dev),</div><div class='add'>+		  "DCB: DCB: Remove qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n",</div><div class='add'>+		  qp-&gt;qp_uk.qp_id, qp-&gt;user_pri, qp-&gt;qs_handle,</div><div class='add'>+		  qp-&gt;on_qoslist);</div><div class='add'>+	mutex_lock(&amp;vsi-&gt;qos[qp-&gt;user_pri].qos_mutex);</div><div class='add'>+	if (qp-&gt;on_qoslist) {</div><div class='add'>+		qp-&gt;on_qoslist = false;</div><div class='add'>+		list_del(&amp;qp-&gt;list);</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;vsi-&gt;qos[qp-&gt;user_pri].qos_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_qp_add_qos - called during setctx for qp to be added to qos</div><div class='add'>+ * @qp: qp to be added to qos</div><div class='add'>+ */</div><div class='add'>+void irdma_qp_add_qos(struct irdma_sc_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_vsi *vsi = qp-&gt;vsi;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(to_ibdev(qp-&gt;dev),</div><div class='add'>+		  "DCB: DCB: Add qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n",</div><div class='add'>+		  qp-&gt;qp_uk.qp_id, qp-&gt;user_pri, qp-&gt;qs_handle,</div><div class='add'>+		  qp-&gt;on_qoslist);</div><div class='add'>+	mutex_lock(&amp;vsi-&gt;qos[qp-&gt;user_pri].qos_mutex);</div><div class='add'>+	if (!qp-&gt;on_qoslist) {</div><div class='add'>+		list_add(&amp;qp-&gt;list, &amp;vsi-&gt;qos[qp-&gt;user_pri].qplist);</div><div class='add'>+		qp-&gt;on_qoslist = true;</div><div class='add'>+		qp-&gt;qs_handle = vsi-&gt;qos[qp-&gt;user_pri].qs_handle;</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;vsi-&gt;qos[qp-&gt;user_pri].qos_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_pd_init - initialize sc pd struct</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @pd: sc pd ptr</div><div class='add'>+ * @pd_id: pd_id for allocated pd</div><div class='add'>+ * @abi_ver: User/Kernel ABI version</div><div class='add'>+ */</div><div class='add'>+void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id,</div><div class='add'>+		      int abi_ver)</div><div class='add'>+{</div><div class='add'>+	pd-&gt;pd_id = pd_id;</div><div class='add'>+	pd-&gt;abi_ver = abi_ver;</div><div class='add'>+	pd-&gt;dev = dev;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_add_arp_cache_entry - cqp wqe add arp cache entry</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @info: arp entry information</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_add_arp_cache_entry(struct irdma_sc_cqp *cqp,</div><div class='add'>+					struct irdma_add_arp_cache_entry_info *info,</div><div class='add'>+					u64 scratch, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	set_64bit_val(wqe, 8, info-&gt;reach_max);</div><div class='add'>+	set_64bit_val(wqe, 16, ether_addr_to_u64(info-&gt;mac_addr));</div><div class='add'>+</div><div class='add'>+	hdr = info-&gt;arp_index |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_MAT_PERMANENT, (info-&gt;permanent ? 1 : 0)) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_MAT_ENTRYVALID, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: ARP_CACHE_ENTRY WQE", DUMP_PREFIX_OFFSET,</div><div class='add'>+			     16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_del_arp_cache_entry - dele arp cache entry</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @arp_index: arp index to delete arp entry</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_del_arp_cache_entry(struct irdma_sc_cqp *cqp, u64 scratch,</div><div class='add'>+					u16 arp_index, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	hdr = arp_index |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: ARP_CACHE_DEL_ENTRY WQE",</div><div class='add'>+			     DUMP_PREFIX_OFFSET, 16, 8, wqe,</div><div class='add'>+			     IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_manage_apbvt_entry - for adding and deleting apbvt entries</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @info: info for apbvt entry to add or delete</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_manage_apbvt_entry(struct irdma_sc_cqp *cqp,</div><div class='add'>+				       struct irdma_apbvt_info *info,</div><div class='add'>+				       u64 scratch, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16, info-&gt;port);</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_APBVT) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_MAPT_ADDPORT, info-&gt;add) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: MANAGE_APBVT WQE", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_manage_qhash_table_entry - manage quad hash entries</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @info: info for quad hash to manage</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ *</div><div class='add'>+ * This is called before connection establishment is started.</div><div class='add'>+ * For passive connections, when listener is created, it will</div><div class='add'>+ * call with entry type of  IRDMA_QHASH_TYPE_TCP_SYN with local</div><div class='add'>+ * ip address and tcp port. When SYN is received (passive</div><div class='add'>+ * connections) or sent (active connections), this routine is</div><div class='add'>+ * called with entry type of IRDMA_QHASH_TYPE_TCP_ESTABLISHED</div><div class='add'>+ * and quad is passed in info.</div><div class='add'>+ *</div><div class='add'>+ * When iwarp connection is done and its state moves to RTS, the</div><div class='add'>+ * quad hash entry in the hardware will point to iwarp's qp</div><div class='add'>+ * number and requires no calls from the driver.</div><div class='add'>+ */</div><div class='add'>+static int</div><div class='add'>+irdma_sc_manage_qhash_table_entry(struct irdma_sc_cqp *cqp,</div><div class='add'>+				  struct irdma_qhash_table_info *info,</div><div class='add'>+				  u64 scratch, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 qw1 = 0;</div><div class='add'>+	u64 qw2 = 0;</div><div class='add'>+	u64 temp;</div><div class='add'>+	struct irdma_sc_vsi *vsi = info-&gt;vsi;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0, ether_addr_to_u64(info-&gt;mac_addr));</div><div class='add'>+</div><div class='add'>+	qw1 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QPN, info-&gt;qp_num) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QHASH_DEST_PORT, info-&gt;dest_port);</div><div class='add'>+	if (info-&gt;ipv4_valid) {</div><div class='add'>+		set_64bit_val(wqe, 48,</div><div class='add'>+			      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info-&gt;dest_ip[0]));</div><div class='add'>+	} else {</div><div class='add'>+		set_64bit_val(wqe, 56,</div><div class='add'>+			      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info-&gt;dest_ip[0]) |</div><div class='add'>+			      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info-&gt;dest_ip[1]));</div><div class='add'>+</div><div class='add'>+		set_64bit_val(wqe, 48,</div><div class='add'>+			      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info-&gt;dest_ip[2]) |</div><div class='add'>+			      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info-&gt;dest_ip[3]));</div><div class='add'>+	}</div><div class='add'>+	qw2 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QS_HANDLE,</div><div class='add'>+			 vsi-&gt;qos[info-&gt;user_pri].qs_handle);</div><div class='add'>+	if (info-&gt;vlan_valid)</div><div class='add'>+		qw2 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANID, info-&gt;vlan_id);</div><div class='add'>+	set_64bit_val(wqe, 16, qw2);</div><div class='add'>+	if (info-&gt;entry_type == IRDMA_QHASH_TYPE_TCP_ESTABLISHED) {</div><div class='add'>+		qw1 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_SRC_PORT, info-&gt;src_port);</div><div class='add'>+		if (!info-&gt;ipv4_valid) {</div><div class='add'>+			set_64bit_val(wqe, 40,</div><div class='add'>+				      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info-&gt;src_ip[0]) |</div><div class='add'>+				      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info-&gt;src_ip[1]));</div><div class='add'>+			set_64bit_val(wqe, 32,</div><div class='add'>+				      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info-&gt;src_ip[2]) |</div><div class='add'>+				      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info-&gt;src_ip[3]));</div><div class='add'>+		} else {</div><div class='add'>+			set_64bit_val(wqe, 32,</div><div class='add'>+				      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info-&gt;src_ip[0]));</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 8, qw1);</div><div class='add'>+	temp = FIELD_PREP(IRDMA_CQPSQ_QHASH_WQEVALID, cqp-&gt;polarity) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_QHASH_OPCODE,</div><div class='add'>+			  IRDMA_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_QHASH_MANAGE, info-&gt;manage) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_QHASH_IPV4VALID, info-&gt;ipv4_valid) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANVALID, info-&gt;vlan_valid) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_QHASH_ENTRYTYPE, info-&gt;entry_type);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, temp);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: MANAGE_QHASH WQE", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_qp_init - initialize qp</div><div class='add'>+ * @qp: sc qp</div><div class='add'>+ * @info: initialization qp info</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info)</div><div class='add'>+{</div><div class='add'>+	int ret_code;</div><div class='add'>+	u32 pble_obj_cnt;</div><div class='add'>+	u16 wqe_size;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;qp_uk_init_info.max_sq_frag_cnt &gt;</div><div class='add'>+	    info-&gt;pd-&gt;dev-&gt;hw_attrs.uk_attrs.max_hw_wq_frags ||</div><div class='add'>+	    info-&gt;qp_uk_init_info.max_rq_frag_cnt &gt;</div><div class='add'>+	    info-&gt;pd-&gt;dev-&gt;hw_attrs.uk_attrs.max_hw_wq_frags)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	qp-&gt;dev = info-&gt;pd-&gt;dev;</div><div class='add'>+	qp-&gt;vsi = info-&gt;vsi;</div><div class='add'>+	qp-&gt;ieq_qp = info-&gt;vsi-&gt;exception_lan_q;</div><div class='add'>+	qp-&gt;sq_pa = info-&gt;sq_pa;</div><div class='add'>+	qp-&gt;rq_pa = info-&gt;rq_pa;</div><div class='add'>+	qp-&gt;hw_host_ctx_pa = info-&gt;host_ctx_pa;</div><div class='add'>+	qp-&gt;q2_pa = info-&gt;q2_pa;</div><div class='add'>+	qp-&gt;shadow_area_pa = info-&gt;shadow_area_pa;</div><div class='add'>+	qp-&gt;q2_buf = info-&gt;q2;</div><div class='add'>+	qp-&gt;pd = info-&gt;pd;</div><div class='add'>+	qp-&gt;hw_host_ctx = info-&gt;host_ctx;</div><div class='add'>+	info-&gt;qp_uk_init_info.wqe_alloc_db = qp-&gt;pd-&gt;dev-&gt;wqe_alloc_db;</div><div class='add'>+	ret_code = irdma_uk_qp_init(&amp;qp-&gt;qp_uk, &amp;info-&gt;qp_uk_init_info);</div><div class='add'>+	if (ret_code)</div><div class='add'>+		return ret_code;</div><div class='add'>+</div><div class='add'>+	qp-&gt;virtual_map = info-&gt;virtual_map;</div><div class='add'>+	pble_obj_cnt = info-&gt;pd-&gt;dev-&gt;hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].cnt;</div><div class='add'>+</div><div class='add'>+	if ((info-&gt;virtual_map &amp;&amp; info-&gt;sq_pa &gt;= pble_obj_cnt) ||</div><div class='add'>+	    (!info-&gt;qp_uk_init_info.srq_uk &amp;&amp;</div><div class='add'>+	     info-&gt;virtual_map &amp;&amp; info-&gt;rq_pa &gt;= pble_obj_cnt))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	qp-&gt;llp_stream_handle = (void *)(-1);</div><div class='add'>+	qp-&gt;hw_sq_size = irdma_get_encoded_wqe_size(qp-&gt;qp_uk.sq_ring.size,</div><div class='add'>+						    IRDMA_QUEUE_TYPE_SQ_RQ);</div><div class='add'>+	ibdev_dbg(to_ibdev(qp-&gt;dev),</div><div class='add'>+		  "WQE: hw_sq_size[%04d] sq_ring.size[%04d]\n",</div><div class='add'>+		  qp-&gt;hw_sq_size, qp-&gt;qp_uk.sq_ring.size);</div><div class='add'>+	if (qp-&gt;qp_uk.uk_attrs-&gt;hw_rev == IRDMA_GEN_1 &amp;&amp; qp-&gt;pd-&gt;abi_ver &gt; 4)</div><div class='add'>+		wqe_size = IRDMA_WQE_SIZE_128;</div><div class='add'>+	else</div><div class='add'>+		ret_code = irdma_fragcnt_to_wqesize_rq(qp-&gt;qp_uk.max_rq_frag_cnt,</div><div class='add'>+						       &amp;wqe_size);</div><div class='add'>+	if (ret_code)</div><div class='add'>+		return ret_code;</div><div class='add'>+</div><div class='add'>+	qp-&gt;hw_rq_size = irdma_get_encoded_wqe_size(qp-&gt;qp_uk.rq_size *</div><div class='add'>+				(wqe_size / IRDMA_QP_WQE_MIN_SIZE), IRDMA_QUEUE_TYPE_SQ_RQ);</div><div class='add'>+	ibdev_dbg(to_ibdev(qp-&gt;dev),</div><div class='add'>+		  "WQE: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n",</div><div class='add'>+		  qp-&gt;hw_rq_size, qp-&gt;qp_uk.rq_size, wqe_size);</div><div class='add'>+	qp-&gt;sq_tph_val = info-&gt;sq_tph_val;</div><div class='add'>+	qp-&gt;rq_tph_val = info-&gt;rq_tph_val;</div><div class='add'>+	qp-&gt;sq_tph_en = info-&gt;sq_tph_en;</div><div class='add'>+	qp-&gt;rq_tph_en = info-&gt;rq_tph_en;</div><div class='add'>+	qp-&gt;rcv_tph_en = info-&gt;rcv_tph_en;</div><div class='add'>+	qp-&gt;xmit_tph_en = info-&gt;xmit_tph_en;</div><div class='add'>+	qp-&gt;qp_uk.first_sq_wq = info-&gt;qp_uk_init_info.first_sq_wq;</div><div class='add'>+	qp-&gt;qs_handle = qp-&gt;vsi-&gt;qos[qp-&gt;user_pri].qs_handle;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_srq_init - init sc_srq structure</div><div class='add'>+ * @srq: srq sc struct</div><div class='add'>+ * @info: parameters for srq init</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_srq_init(struct irdma_sc_srq *srq,</div><div class='add'>+		      struct irdma_srq_init_info *info)</div><div class='add'>+{</div><div class='add'>+	u32 srq_size_quanta;</div><div class='add'>+	int ret_code;</div><div class='add'>+</div><div class='add'>+	ret_code = irdma_uk_srq_init(&amp;srq-&gt;srq_uk, &amp;info-&gt;srq_uk_init_info);</div><div class='add'>+	if (ret_code)</div><div class='add'>+		return ret_code;</div><div class='add'>+</div><div class='add'>+	srq-&gt;dev = info-&gt;pd-&gt;dev;</div><div class='add'>+	srq-&gt;pd = info-&gt;pd;</div><div class='add'>+	srq-&gt;vsi = info-&gt;vsi;</div><div class='add'>+	srq-&gt;srq_pa = info-&gt;srq_pa;</div><div class='add'>+	srq-&gt;first_pm_pbl_idx = info-&gt;first_pm_pbl_idx;</div><div class='add'>+	srq-&gt;pasid = info-&gt;pasid;</div><div class='add'>+	srq-&gt;pasid_valid = info-&gt;pasid_valid;</div><div class='add'>+	srq-&gt;srq_limit = info-&gt;srq_limit;</div><div class='add'>+	srq-&gt;leaf_pbl_size = info-&gt;leaf_pbl_size;</div><div class='add'>+	srq-&gt;virtual_map = info-&gt;virtual_map;</div><div class='add'>+	srq-&gt;tph_en = info-&gt;tph_en;</div><div class='add'>+	srq-&gt;arm_limit_event = info-&gt;arm_limit_event;</div><div class='add'>+	srq-&gt;tph_val = info-&gt;tph_value;</div><div class='add'>+	srq-&gt;shadow_area_pa = info-&gt;shadow_area_pa;</div><div class='add'>+</div><div class='add'>+	/* Smallest SRQ size is 256B i.e. 8 quanta */</div><div class='add'>+	srq_size_quanta = max((u32)IRDMA_SRQ_MIN_QUANTA,</div><div class='add'>+			      srq-&gt;srq_uk.srq_size *</div><div class='add'>+			      srq-&gt;srq_uk.wqe_size_multiplier);</div><div class='add'>+	srq-&gt;hw_srq_size = irdma_get_encoded_wqe_size(srq_size_quanta,</div><div class='add'>+						      IRDMA_QUEUE_TYPE_SRQ);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_srq_create - send srq create CQP WQE</div><div class='add'>+ * @srq: srq sc struct</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_srq_create(struct irdma_sc_srq *srq, u64 scratch,</div><div class='add'>+			       bool post_sq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	cqp = srq-&gt;pd-&gt;dev-&gt;cqp;</div><div class='add'>+	if (srq-&gt;srq_uk.srq_id &lt; cqp-&gt;dev-&gt;hw_attrs.min_hw_srq_id ||</div><div class='add'>+	    srq-&gt;srq_uk.srq_id &gt;</div><div class='add'>+	    (cqp-&gt;dev-&gt;hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_SRQ].max_cnt - 1))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_SRQ_SRQ_LIMIT, srq-&gt;srq_limit) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_SRQ_RQSIZE, srq-&gt;hw_srq_size) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_SRQ_RQ_WQE_SIZE, srq-&gt;srq_uk.wqe_size));</div><div class='add'>+	set_64bit_val(wqe, 8, (uintptr_t)srq);</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_SRQ_PD_ID, srq-&gt;pd-&gt;pd_id));</div><div class='add'>+	set_64bit_val(wqe, 32,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_SRQ_PHYSICAL_BUFFER_ADDR,</div><div class='add'>+				 srq-&gt;srq_pa &gt;&gt;</div><div class='add'>+				 IRDMA_CQPSQ_SRQ_PHYSICAL_BUFFER_ADDR_S));</div><div class='add'>+	set_64bit_val(wqe, 40,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_SRQ_DB_SHADOW_ADDR,</div><div class='add'>+				 srq-&gt;shadow_area_pa &gt;&gt;</div><div class='add'>+				 IRDMA_CQPSQ_SRQ_DB_SHADOW_ADDR_S));</div><div class='add'>+	set_64bit_val(wqe, 48,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_SRQ_FIRST_PM_PBL_IDX,</div><div class='add'>+				 srq-&gt;first_pm_pbl_idx));</div><div class='add'>+</div><div class='add'>+	hdr = srq-&gt;srq_uk.srq_id |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_SRQ) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_SRQ_LEAF_PBL_SIZE, srq-&gt;leaf_pbl_size) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_SRQ_VIRTMAP, srq-&gt;virtual_map) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_SRQ_ARM_LIMIT_EVENT,</div><div class='add'>+			 srq-&gt;arm_limit_event) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: SRQ_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_srq_modify - send modify_srq CQP WQE</div><div class='add'>+ * @srq: srq sc struct</div><div class='add'>+ * @info: parameters for srq modification</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_srq_modify(struct irdma_sc_srq *srq,</div><div class='add'>+			       struct irdma_modify_srq_info *info, u64 scratch,</div><div class='add'>+			       bool post_sq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	cqp = srq-&gt;dev-&gt;cqp;</div><div class='add'>+	if (srq-&gt;srq_uk.srq_id &lt; cqp-&gt;dev-&gt;hw_attrs.min_hw_srq_id ||</div><div class='add'>+	    srq-&gt;srq_uk.srq_id &gt;</div><div class='add'>+	    (cqp-&gt;dev-&gt;hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_SRQ].max_cnt - 1))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_SRQ_SRQ_LIMIT, info-&gt;srq_limit) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_SRQ_RQSIZE, srq-&gt;hw_srq_size) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_SRQ_RQ_WQE_SIZE, srq-&gt;srq_uk.wqe_size));</div><div class='add'>+	set_64bit_val(wqe, 8,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_SRQ_SRQCTX, srq-&gt;srq_uk.srq_id));</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_SRQ_PD_ID, srq-&gt;pd-&gt;pd_id));</div><div class='add'>+	set_64bit_val(wqe, 32,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_SRQ_PHYSICAL_BUFFER_ADDR,</div><div class='add'>+				 srq-&gt;srq_pa &gt;&gt;</div><div class='add'>+				 IRDMA_CQPSQ_SRQ_PHYSICAL_BUFFER_ADDR_S));</div><div class='add'>+	set_64bit_val(wqe, 40,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_SRQ_DB_SHADOW_ADDR,</div><div class='add'>+				 srq-&gt;shadow_area_pa &gt;&gt;</div><div class='add'>+				 IRDMA_CQPSQ_SRQ_DB_SHADOW_ADDR_S));</div><div class='add'>+	set_64bit_val(wqe, 48,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_SRQ_FIRST_PM_PBL_IDX,</div><div class='add'>+				 srq-&gt;first_pm_pbl_idx));</div><div class='add'>+</div><div class='add'>+	hdr = srq-&gt;srq_uk.srq_id |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_SRQ) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_SRQ_LEAF_PBL_SIZE, srq-&gt;leaf_pbl_size) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_SRQ_VIRTMAP, srq-&gt;virtual_map) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_SRQ_ARM_LIMIT_EVENT,</div><div class='add'>+			 info-&gt;arm_limit_event) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: SRQ_MODIFY WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_srq_destroy - send srq_destroy CQP WQE</div><div class='add'>+ * @srq: srq sc struct</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_srq_destroy(struct irdma_sc_srq *srq, u64 scratch,</div><div class='add'>+				bool post_sq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	cqp = srq-&gt;dev-&gt;cqp;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 8, (uintptr_t)srq);</div><div class='add'>+</div><div class='add'>+	hdr = srq-&gt;srq_uk.srq_id |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_SRQ) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: SRQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_qp_create - create qp</div><div class='add'>+ * @qp: sc qp</div><div class='add'>+ * @info: qp create info</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_qp_create(struct irdma_sc_qp *qp, struct irdma_create_qp_info *info,</div><div class='add'>+		       u64 scratch, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	cqp = qp-&gt;dev-&gt;cqp;</div><div class='add'>+	if (qp-&gt;qp_uk.qp_id &lt; cqp-&gt;dev-&gt;hw_attrs.min_hw_qp_id ||</div><div class='add'>+	    qp-&gt;qp_uk.qp_id &gt;= cqp-&gt;dev-&gt;hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_QP].max_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16, qp-&gt;hw_host_ctx_pa);</div><div class='add'>+	set_64bit_val(wqe, 40, qp-&gt;shadow_area_pa);</div><div class='add'>+</div><div class='add'>+	hdr = qp-&gt;qp_uk.qp_id |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_QP) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, (info-&gt;ord_valid ? 1 : 0)) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info-&gt;tcp_ctx_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info-&gt;mac_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp-&gt;qp_uk.qp_type) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp-&gt;virtual_map) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info-&gt;force_lpb) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info-&gt;cq_num_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_ARPTABIDXVALID,</div><div class='add'>+			 info-&gt;arp_cache_idx_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info-&gt;next_iwarp_state) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: QP_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_qp_modify - modify qp cqp wqe</div><div class='add'>+ * @qp: sc qp</div><div class='add'>+ * @info: modify qp info</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_qp_modify(struct irdma_sc_qp *qp, struct irdma_modify_qp_info *info,</div><div class='add'>+		       u64 scratch, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u8 term_actions = 0;</div><div class='add'>+	u8 term_len = 0;</div><div class='add'>+</div><div class='add'>+	cqp = qp-&gt;dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;next_iwarp_state == IRDMA_QP_STATE_TERMINATE) {</div><div class='add'>+		if (info-&gt;dont_send_fin)</div><div class='add'>+			term_actions += IRDMAQP_TERM_SEND_TERM_ONLY;</div><div class='add'>+		if (info-&gt;dont_send_term)</div><div class='add'>+			term_actions += IRDMAQP_TERM_SEND_FIN_ONLY;</div><div class='add'>+		if (term_actions == IRDMAQP_TERM_SEND_TERM_AND_FIN ||</div><div class='add'>+		    term_actions == IRDMAQP_TERM_SEND_TERM_ONLY)</div><div class='add'>+			term_len = info-&gt;termlen;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 8,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_QP_NEWMSS, info-&gt;new_mss) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_QP_TERMLEN, term_len));</div><div class='add'>+	set_64bit_val(wqe, 16, qp-&gt;hw_host_ctx_pa);</div><div class='add'>+	set_64bit_val(wqe, 40, qp-&gt;shadow_area_pa);</div><div class='add'>+</div><div class='add'>+	hdr = qp-&gt;qp_uk.qp_id |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_QP) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, info-&gt;ord_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info-&gt;tcp_ctx_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_CACHEDVARVALID,</div><div class='add'>+			 info-&gt;cached_var_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp-&gt;virtual_map) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info-&gt;force_lpb) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info-&gt;cq_num_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info-&gt;mac_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp-&gt;qp_uk.qp_type) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_MSSCHANGE, info-&gt;mss_change) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_REMOVEHASHENTRY,</div><div class='add'>+			 info-&gt;remove_hash_idx) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_TERMACT, term_actions) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_RESETCON, info-&gt;reset_tcp_conn) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_ARPTABIDXVALID,</div><div class='add'>+			 info-&gt;arp_cache_idx_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info-&gt;next_iwarp_state) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: QP_MODIFY WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_qp_destroy - cqp destroy qp</div><div class='add'>+ * @qp: sc qp</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @remove_hash_idx: flag if to remove hash idx</div><div class='add'>+ * @ignore_mw_bnd: memory window bind flag</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch,</div><div class='add'>+			bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	cqp = qp-&gt;dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16, qp-&gt;hw_host_ctx_pa);</div><div class='add'>+	set_64bit_val(wqe, 40, qp-&gt;shadow_area_pa);</div><div class='add'>+</div><div class='add'>+	hdr = qp-&gt;qp_uk.qp_id |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_QP) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp-&gt;qp_uk.qp_type) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_IGNOREMWBOUND, ignore_mw_bnd) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_REMOVEHASHENTRY, remove_hash_idx) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: QP_DESTROY WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_get_encoded_ird_size -</div><div class='add'>+ * @ird_size: IRD size</div><div class='add'>+ * The ird from the connection is rounded to a supported HW setting and then encoded</div><div class='add'>+ * for ird_size field of qp_ctx. Consumers are expected to provide valid ird size based</div><div class='add'>+ * on hardware attributes. IRD size defaults to a value of 4 in case of invalid input</div><div class='add'>+ */</div><div class='add'>+static u8 irdma_sc_get_encoded_ird_size(u16 ird_size)</div><div class='add'>+{</div><div class='add'>+	switch (ird_size ?</div><div class='add'>+		roundup_pow_of_two(2 * ird_size) : 4) {</div><div class='add'>+	case 256:</div><div class='add'>+		return IRDMA_IRD_HW_SIZE_256;</div><div class='add'>+	case 128:</div><div class='add'>+		return IRDMA_IRD_HW_SIZE_128;</div><div class='add'>+	case 64:</div><div class='add'>+	case 32:</div><div class='add'>+		return IRDMA_IRD_HW_SIZE_64;</div><div class='add'>+	case 16:</div><div class='add'>+	case 8:</div><div class='add'>+		return IRDMA_IRD_HW_SIZE_16;</div><div class='add'>+	case 4:</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return IRDMA_IRD_HW_SIZE_4;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_qp_setctx_roce_gen_2 - set qp's context</div><div class='add'>+ * @qp: sc qp</div><div class='add'>+ * @qp_ctx: context ptr</div><div class='add'>+ * @info: ctx info</div><div class='add'>+ */</div><div class='add'>+static void irdma_sc_qp_setctx_roce_gen_2(struct irdma_sc_qp *qp,</div><div class='add'>+					  __le64 *qp_ctx,</div><div class='add'>+					  struct irdma_qp_host_ctx_info *info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_roce_offload_info *roce_info;</div><div class='add'>+	struct irdma_udp_offload_info *udp;</div><div class='add'>+	u8 push_mode_en;</div><div class='add'>+	u32 push_idx;</div><div class='add'>+</div><div class='add'>+	roce_info = info-&gt;roce_info;</div><div class='add'>+	udp = info-&gt;udp_info;</div><div class='add'>+	qp-&gt;user_pri = info-&gt;user_pri;</div><div class='add'>+	if (qp-&gt;push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {</div><div class='add'>+		push_mode_en = 0;</div><div class='add'>+		push_idx = 0;</div><div class='add'>+	} else {</div><div class='add'>+		push_mode_en = 1;</div><div class='add'>+		push_idx = qp-&gt;push_idx;</div><div class='add'>+	}</div><div class='add'>+	set_64bit_val(qp_ctx, 0,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RQWQESIZE, qp-&gt;qp_uk.rq_wqe_size) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RCVTPHEN, qp-&gt;rcv_tph_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_XMITTPHEN, qp-&gt;xmit_tph_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RQTPHEN, qp-&gt;rq_tph_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_SQTPHEN, qp-&gt;sq_tph_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_PPIDX, push_idx) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_PMENA, push_mode_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_PDIDXHI, roce_info-&gt;pd_id &gt;&gt; 16) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_DC_TCP_EN, roce_info-&gt;dctcp_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID, roce_info-&gt;err_rq_idx_valid) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_ISQP1, roce_info-&gt;is_qp1) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_ROCE_TVER, roce_info-&gt;roce_tver) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_IPV4, udp-&gt;ipv4) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_INSERTVLANTAG, udp-&gt;insert_vlan_tag));</div><div class='add'>+	set_64bit_val(qp_ctx, 8, qp-&gt;sq_pa);</div><div class='add'>+	set_64bit_val(qp_ctx, 16, qp-&gt;rq_pa);</div><div class='add'>+	if ((roce_info-&gt;dcqcn_en || roce_info-&gt;dctcp_en) &amp;&amp;</div><div class='add'>+	    !(udp-&gt;tos &amp; 0x03))</div><div class='add'>+		udp-&gt;tos |= ECN_CODE_PT_VAL;</div><div class='add'>+	set_64bit_val(qp_ctx, 24,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RQSIZE, qp-&gt;hw_rq_size) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_SQSIZE, qp-&gt;hw_sq_size) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_TTL, udp-&gt;ttl) | FIELD_PREP(IRDMAQPC_TOS, udp-&gt;tos) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_SRCPORTNUM, udp-&gt;src_port) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_DESTPORTNUM, udp-&gt;dst_port));</div><div class='add'>+	set_64bit_val(qp_ctx, 32,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_DESTIPADDR2, udp-&gt;dest_ip_addr[2]) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_DESTIPADDR3, udp-&gt;dest_ip_addr[3]));</div><div class='add'>+	set_64bit_val(qp_ctx, 40,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_DESTIPADDR0, udp-&gt;dest_ip_addr[0]) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_DESTIPADDR1, udp-&gt;dest_ip_addr[1]));</div><div class='add'>+	set_64bit_val(qp_ctx, 48,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_SNDMSS, udp-&gt;snd_mss) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_VLANTAG, udp-&gt;vlan_tag) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_ARPIDX, udp-&gt;arp_idx));</div><div class='add'>+	set_64bit_val(qp_ctx, 56,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_PKEY, roce_info-&gt;p_key) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_PDIDX, roce_info-&gt;pd_id) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_ACKCREDITS, roce_info-&gt;ack_credits) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_FLOWLABEL, udp-&gt;flow_label));</div><div class='add'>+	set_64bit_val(qp_ctx, 64,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_QKEY, roce_info-&gt;qkey) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_DESTQP, roce_info-&gt;dest_qp));</div><div class='add'>+	set_64bit_val(qp_ctx, 80,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_PSNNXT, udp-&gt;psn_nxt) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_LSN, udp-&gt;lsn));</div><div class='add'>+	set_64bit_val(qp_ctx, 88,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_EPSN, udp-&gt;epsn));</div><div class='add'>+	set_64bit_val(qp_ctx, 96,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_PSNMAX, udp-&gt;psn_max) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_PSNUNA, udp-&gt;psn_una));</div><div class='add'>+	set_64bit_val(qp_ctx, 112,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_CWNDROCE, udp-&gt;cwnd));</div><div class='add'>+	set_64bit_val(qp_ctx, 128,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, roce_info-&gt;err_rq_idx) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RNRNAK_THRESH, udp-&gt;rnr_nak_thresh) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_REXMIT_THRESH, udp-&gt;rexmit_thresh) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RTOMIN, roce_info-&gt;rtomin));</div><div class='add'>+	set_64bit_val(qp_ctx, 136,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_TXCQNUM, info-&gt;send_cq_num) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RXCQNUM, info-&gt;rcv_cq_num));</div><div class='add'>+	set_64bit_val(qp_ctx, 144,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_STAT_INDEX, info-&gt;stats_idx));</div><div class='add'>+	set_64bit_val(qp_ctx, 152, ether_addr_to_u64(roce_info-&gt;mac_addr) &lt;&lt; 16);</div><div class='add'>+	set_64bit_val(qp_ctx, 160,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_ORDSIZE, roce_info-&gt;ord_size) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(roce_info-&gt;ird_size)) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_WRRDRSPOK, roce_info-&gt;wr_rdresp_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RDOK, roce_info-&gt;rd_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info-&gt;stats_idx_valid) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_BINDEN, roce_info-&gt;bind_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_FASTREGEN, roce_info-&gt;fast_reg_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_DCQCNENABLE, roce_info-&gt;dcqcn_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RCVNOICRC, roce_info-&gt;rcv_no_icrc) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_FW_CC_ENABLE, roce_info-&gt;fw_cc_enable) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_UDPRIVCQENABLE, roce_info-&gt;udprivcq_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_PRIVEN, roce_info-&gt;priv_mode_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_TIMELYENABLE, roce_info-&gt;timely_en));</div><div class='add'>+	set_64bit_val(qp_ctx, 168,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_QPCOMPCTX, info-&gt;qp_compl_ctx));</div><div class='add'>+	set_64bit_val(qp_ctx, 176,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_SQTPHVAL, qp-&gt;sq_tph_val) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RQTPHVAL, qp-&gt;rq_tph_val) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_QSHANDLE, qp-&gt;qs_handle));</div><div class='add'>+	set_64bit_val(qp_ctx, 184,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, udp-&gt;local_ipaddr[3]) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, udp-&gt;local_ipaddr[2]));</div><div class='add'>+	set_64bit_val(qp_ctx, 192,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, udp-&gt;local_ipaddr[1]) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, udp-&gt;local_ipaddr[0]));</div><div class='add'>+	set_64bit_val(qp_ctx, 200,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_THIGH, roce_info-&gt;t_high) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_TLOW, roce_info-&gt;t_low));</div><div class='add'>+	set_64bit_val(qp_ctx, 208,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info-&gt;rem_endpoint_idx));</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: QP_HOST CTX WQE", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, qp_ctx, IRDMA_QP_CTX_SIZE, false);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_get_encoded_ird_size_gen_3 - get encoded IRD size for GEN 3</div><div class='add'>+ * @ird_size: IRD size</div><div class='add'>+ * The ird from the connection is rounded to a supported HW setting and then encoded</div><div class='add'>+ * for ird_size field of qp_ctx. Consumers are expected to provide valid ird size based</div><div class='add'>+ * on hardware attributes. IRD size defaults to a value of 4 in case of invalid input.</div><div class='add'>+ */</div><div class='add'>+static u8 irdma_sc_get_encoded_ird_size_gen_3(u16 ird_size)</div><div class='add'>+{</div><div class='add'>+	switch (ird_size ?</div><div class='add'>+		roundup_pow_of_two(2 * ird_size) : 4) {</div><div class='add'>+	case 4096:</div><div class='add'>+		return IRDMA_IRD_HW_SIZE_4096_GEN3;</div><div class='add'>+	case 2048:</div><div class='add'>+		return IRDMA_IRD_HW_SIZE_2048_GEN3;</div><div class='add'>+	case 1024:</div><div class='add'>+		return IRDMA_IRD_HW_SIZE_1024_GEN3;</div><div class='add'>+	case 512:</div><div class='add'>+		return IRDMA_IRD_HW_SIZE_512_GEN3;</div><div class='add'>+	case 256:</div><div class='add'>+		return IRDMA_IRD_HW_SIZE_256_GEN3;</div><div class='add'>+	case 128:</div><div class='add'>+		return IRDMA_IRD_HW_SIZE_128_GEN3;</div><div class='add'>+	case 64:</div><div class='add'>+		return IRDMA_IRD_HW_SIZE_64_GEN3;</div><div class='add'>+	case 32:</div><div class='add'>+		return IRDMA_IRD_HW_SIZE_32_GEN3;</div><div class='add'>+	case 16:</div><div class='add'>+		return IRDMA_IRD_HW_SIZE_16_GEN3;</div><div class='add'>+	case 8:</div><div class='add'>+		return IRDMA_IRD_HW_SIZE_8_GEN3;</div><div class='add'>+	case 4:</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return IRDMA_IRD_HW_SIZE_4_GEN3;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_qp_setctx_roce_gen_3 - set qp's context</div><div class='add'>+ * @qp: sc qp</div><div class='add'>+ * @qp_ctx: context ptr</div><div class='add'>+ * @info: ctx info</div><div class='add'>+ */</div><div class='add'>+static void irdma_sc_qp_setctx_roce_gen_3(struct irdma_sc_qp *qp,</div><div class='add'>+					  __le64 *qp_ctx,</div><div class='add'>+					  struct irdma_qp_host_ctx_info *info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_roce_offload_info *roce_info = info-&gt;roce_info;</div><div class='add'>+	struct irdma_udp_offload_info *udp = info-&gt;udp_info;</div><div class='add'>+	u64 qw0, qw3, qw7 = 0, qw8 = 0;</div><div class='add'>+	u8 push_mode_en;</div><div class='add'>+	u32 push_idx;</div><div class='add'>+</div><div class='add'>+	qp-&gt;user_pri = info-&gt;user_pri;</div><div class='add'>+	if (qp-&gt;push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {</div><div class='add'>+		push_mode_en = 0;</div><div class='add'>+		push_idx = 0;</div><div class='add'>+	} else {</div><div class='add'>+		push_mode_en = 1;</div><div class='add'>+		push_idx = qp-&gt;push_idx;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	qw0 = FIELD_PREP(IRDMAQPC_RQWQESIZE, qp-&gt;qp_uk.rq_wqe_size) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_RCVTPHEN, qp-&gt;rcv_tph_en) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_XMITTPHEN, qp-&gt;xmit_tph_en) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_RQTPHEN, qp-&gt;rq_tph_en) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_SQTPHEN, qp-&gt;sq_tph_en) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_PPIDX, push_idx) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_PMENA, push_mode_en) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_DC_TCP_EN, roce_info-&gt;dctcp_en) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_ISQP1, roce_info-&gt;is_qp1) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_ROCE_TVER, roce_info-&gt;roce_tver) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_IPV4, udp-&gt;ipv4) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_USE_SRQ, !qp-&gt;qp_uk.srq_uk ? 0 : 1) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_INSERTVLANTAG, udp-&gt;insert_vlan_tag);</div><div class='add'>+	set_64bit_val(qp_ctx, 0, qw0);</div><div class='add'>+	set_64bit_val(qp_ctx, 8, qp-&gt;sq_pa);</div><div class='add'>+	set_64bit_val(qp_ctx, 16, qp-&gt;rq_pa);</div><div class='add'>+	qw3 = FIELD_PREP(IRDMAQPC_RQSIZE, qp-&gt;hw_rq_size) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_SQSIZE, qp-&gt;hw_sq_size) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_TTL, udp-&gt;ttl) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_TOS, udp-&gt;tos) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_SRCPORTNUM, udp-&gt;src_port) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_DESTPORTNUM, udp-&gt;dst_port);</div><div class='add'>+	set_64bit_val(qp_ctx, 24, qw3);</div><div class='add'>+	set_64bit_val(qp_ctx, 32,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_DESTIPADDR2, udp-&gt;dest_ip_addr[2]) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_DESTIPADDR3, udp-&gt;dest_ip_addr[3]));</div><div class='add'>+	set_64bit_val(qp_ctx, 40,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_DESTIPADDR0, udp-&gt;dest_ip_addr[0]) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_DESTIPADDR1, udp-&gt;dest_ip_addr[1]));</div><div class='add'>+	set_64bit_val(qp_ctx, 48,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_SNDMSS, udp-&gt;snd_mss) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_VLANTAG, udp-&gt;vlan_tag) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_ARPIDX, udp-&gt;arp_idx));</div><div class='add'>+	qw7 =  FIELD_PREP(IRDMAQPC_PKEY, roce_info-&gt;p_key) |</div><div class='add'>+	       FIELD_PREP(IRDMAQPC_ACKCREDITS, roce_info-&gt;ack_credits) |</div><div class='add'>+	       FIELD_PREP(IRDMAQPC_FLOWLABEL, udp-&gt;flow_label);</div><div class='add'>+	set_64bit_val(qp_ctx, 56, qw7);</div><div class='add'>+	qw8 = FIELD_PREP(IRDMAQPC_QKEY, roce_info-&gt;qkey) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_DESTQP, roce_info-&gt;dest_qp);</div><div class='add'>+	set_64bit_val(qp_ctx, 64, qw8);</div><div class='add'>+	set_64bit_val(qp_ctx, 80,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_PSNNXT, udp-&gt;psn_nxt) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_LSN, udp-&gt;lsn));</div><div class='add'>+	set_64bit_val(qp_ctx, 88,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_EPSN, udp-&gt;epsn));</div><div class='add'>+	set_64bit_val(qp_ctx, 96,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_PSNMAX, udp-&gt;psn_max) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_PSNUNA, udp-&gt;psn_una));</div><div class='add'>+	set_64bit_val(qp_ctx, 112,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_CWNDROCE, udp-&gt;cwnd));</div><div class='add'>+	set_64bit_val(qp_ctx, 128,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_MINRNR_TIMER, udp-&gt;min_rnr_timer) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RNRNAK_THRESH, udp-&gt;rnr_nak_thresh) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_REXMIT_THRESH, udp-&gt;rexmit_thresh) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RNRNAK_TMR, udp-&gt;rnr_nak_tmr) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RTOMIN, roce_info-&gt;rtomin));</div><div class='add'>+	set_64bit_val(qp_ctx, 136,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_TXCQNUM, info-&gt;send_cq_num) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RXCQNUM, info-&gt;rcv_cq_num));</div><div class='add'>+	set_64bit_val(qp_ctx, 152,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_MACADDRESS,</div><div class='add'>+				 ether_addr_to_u64(roce_info-&gt;mac_addr)) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_LOCALACKTIMEOUT,</div><div class='add'>+				 roce_info-&gt;local_ack_timeout));</div><div class='add'>+	set_64bit_val(qp_ctx, 160,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_ORDSIZE_GEN3, roce_info-&gt;ord_size) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_IRDSIZE_GEN3,</div><div class='add'>+				 irdma_sc_get_encoded_ird_size_gen_3(roce_info-&gt;ird_size)) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_WRRDRSPOK, roce_info-&gt;wr_rdresp_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RDOK, roce_info-&gt;rd_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_USESTATSINSTANCE,</div><div class='add'>+				 info-&gt;stats_idx_valid) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_BINDEN, roce_info-&gt;bind_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_FASTREGEN, roce_info-&gt;fast_reg_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_DCQCNENABLE, roce_info-&gt;dcqcn_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RCVNOICRC, roce_info-&gt;rcv_no_icrc) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_FW_CC_ENABLE,</div><div class='add'>+				 roce_info-&gt;fw_cc_enable) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_UDPRIVCQENABLE,</div><div class='add'>+				 roce_info-&gt;udprivcq_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_PRIVEN, roce_info-&gt;priv_mode_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_REMOTE_ATOMIC_EN,</div><div class='add'>+				 info-&gt;remote_atomics_en) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_TIMELYENABLE, roce_info-&gt;timely_en));</div><div class='add'>+	set_64bit_val(qp_ctx, 168,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_QPCOMPCTX, info-&gt;qp_compl_ctx));</div><div class='add'>+	set_64bit_val(qp_ctx, 176,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_SQTPHVAL, qp-&gt;sq_tph_val) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RQTPHVAL, qp-&gt;rq_tph_val) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_QSHANDLE, qp-&gt;qs_handle));</div><div class='add'>+	set_64bit_val(qp_ctx, 184,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, udp-&gt;local_ipaddr[3]) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, udp-&gt;local_ipaddr[2]));</div><div class='add'>+	set_64bit_val(qp_ctx, 192,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, udp-&gt;local_ipaddr[1]) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, udp-&gt;local_ipaddr[0]));</div><div class='add'>+	set_64bit_val(qp_ctx, 200,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_THIGH, roce_info-&gt;t_high) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_SRQ_ID,</div><div class='add'>+				 !qp-&gt;qp_uk.srq_uk ?</div><div class='add'>+					0 : qp-&gt;qp_uk.srq_uk-&gt;srq_id) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_TLOW, roce_info-&gt;t_low));</div><div class='add'>+	set_64bit_val(qp_ctx, 208, roce_info-&gt;pd_id |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_STAT_INDEX_GEN3, info-&gt;stats_idx) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_PKT_LIMIT, qp-&gt;pkt_limit));</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: QP_HOST ROCE CTX WQE", DUMP_PREFIX_OFFSET,</div><div class='add'>+			     16, 8, qp_ctx, IRDMA_QP_CTX_SIZE, false);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx,</div><div class='add'>+			     struct irdma_qp_host_ctx_info *info)</div><div class='add'>+{</div><div class='add'>+	if (qp-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2)</div><div class='add'>+		irdma_sc_qp_setctx_roce_gen_2(qp, qp_ctx, info);</div><div class='add'>+	else</div><div class='add'>+		irdma_sc_qp_setctx_roce_gen_3(qp, qp_ctx, info);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* irdma_sc_alloc_local_mac_entry - allocate a mac entry</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_alloc_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch,</div><div class='add'>+					  bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE,</div><div class='add'>+			 IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: ALLOCATE_LOCAL_MAC WQE",</div><div class='add'>+			     DUMP_PREFIX_OFFSET, 16, 8, wqe,</div><div class='add'>+			     IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_add_local_mac_entry - add mac enry</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @info:mac addr info</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_add_local_mac_entry(struct irdma_sc_cqp *cqp,</div><div class='add'>+					struct irdma_local_mac_entry_info *info,</div><div class='add'>+					u64 scratch, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 header;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 32, ether_addr_to_u64(info-&gt;mac_addr));</div><div class='add'>+</div><div class='add'>+	header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, info-&gt;entry_idx) |</div><div class='add'>+		 FIELD_PREP(IRDMA_CQPSQ_OPCODE,</div><div class='add'>+			    IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE) |</div><div class='add'>+		 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, header);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: ADD_LOCAL_MAC WQE", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_del_local_mac_entry - cqp wqe to dele local mac</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @entry_idx: index of mac entry</div><div class='add'>+ * @ignore_ref_count: to force mac adde delete</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_del_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch,</div><div class='add'>+					u16 entry_idx, u8 ignore_ref_count,</div><div class='add'>+					bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 header;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, entry_idx) |</div><div class='add'>+		 FIELD_PREP(IRDMA_CQPSQ_OPCODE,</div><div class='add'>+			    IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE) |</div><div class='add'>+		 FIELD_PREP(IRDMA_CQPSQ_MLM_FREEENTRY, 1) |</div><div class='add'>+		 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity) |</div><div class='add'>+		 FIELD_PREP(IRDMA_CQPSQ_MLM_IGNORE_REF_CNT, ignore_ref_count);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, header);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: DEL_LOCAL_MAC_IPADDR WQE",</div><div class='add'>+			     DUMP_PREFIX_OFFSET, 16, 8, wqe,</div><div class='add'>+			     IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_qp_setctx - set qp's context</div><div class='add'>+ * @qp: sc qp</div><div class='add'>+ * @qp_ctx: context ptr</div><div class='add'>+ * @info: ctx info</div><div class='add'>+ */</div><div class='add'>+void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx,</div><div class='add'>+			struct irdma_qp_host_ctx_info *info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_iwarp_offload_info *iw;</div><div class='add'>+	struct irdma_tcp_offload_info *tcp;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	u8 push_mode_en;</div><div class='add'>+	u32 push_idx;</div><div class='add'>+	u64 qw0, qw3, qw7 = 0, qw16 = 0;</div><div class='add'>+	u64 mac = 0;</div><div class='add'>+</div><div class='add'>+	iw = info-&gt;iwarp_info;</div><div class='add'>+	tcp = info-&gt;tcp_info;</div><div class='add'>+	dev = qp-&gt;dev;</div><div class='add'>+	if (iw-&gt;rcv_mark_en) {</div><div class='add'>+		qp-&gt;pfpdu.marker_len = 4;</div><div class='add'>+		qp-&gt;pfpdu.rcv_start_seq = tcp-&gt;rcv_nxt;</div><div class='add'>+	}</div><div class='add'>+	qp-&gt;user_pri = info-&gt;user_pri;</div><div class='add'>+	if (qp-&gt;push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {</div><div class='add'>+		push_mode_en = 0;</div><div class='add'>+		push_idx = 0;</div><div class='add'>+	} else {</div><div class='add'>+		push_mode_en = 1;</div><div class='add'>+		push_idx = qp-&gt;push_idx;</div><div class='add'>+	}</div><div class='add'>+	qw0 = FIELD_PREP(IRDMAQPC_RQWQESIZE, qp-&gt;qp_uk.rq_wqe_size) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_RCVTPHEN, qp-&gt;rcv_tph_en) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_XMITTPHEN, qp-&gt;xmit_tph_en) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_RQTPHEN, qp-&gt;rq_tph_en) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_SQTPHEN, qp-&gt;sq_tph_en) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_PPIDX, push_idx) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_PMENA, push_mode_en);</div><div class='add'>+</div><div class='add'>+	set_64bit_val(qp_ctx, 8, qp-&gt;sq_pa);</div><div class='add'>+	set_64bit_val(qp_ctx, 16, qp-&gt;rq_pa);</div><div class='add'>+</div><div class='add'>+	qw3 = FIELD_PREP(IRDMAQPC_RQSIZE, qp-&gt;hw_rq_size) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPC_SQSIZE, qp-&gt;hw_sq_size);</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)</div><div class='add'>+		qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX,</div><div class='add'>+				  qp-&gt;src_mac_addr_idx);</div><div class='add'>+	set_64bit_val(qp_ctx, 136,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_TXCQNUM, info-&gt;send_cq_num) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RXCQNUM, info-&gt;rcv_cq_num));</div><div class='add'>+	set_64bit_val(qp_ctx, 168,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_QPCOMPCTX, info-&gt;qp_compl_ctx));</div><div class='add'>+	set_64bit_val(qp_ctx, 176,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_SQTPHVAL, qp-&gt;sq_tph_val) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RQTPHVAL, qp-&gt;rq_tph_val) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_QSHANDLE, qp-&gt;qs_handle) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_EXCEPTION_LAN_QUEUE, qp-&gt;ieq_qp));</div><div class='add'>+	if (info-&gt;iwarp_info_valid) {</div><div class='add'>+		qw0 |= FIELD_PREP(IRDMAQPC_DDP_VER, iw-&gt;ddp_ver) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_RDMAP_VER, iw-&gt;rdmap_ver) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_DC_TCP_EN, iw-&gt;dctcp_en) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_ECN_EN, iw-&gt;ecn_en) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_IBRDENABLE, iw-&gt;ib_rd_en) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_PDIDXHI, iw-&gt;pd_id &gt;&gt; 16) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID,</div><div class='add'>+				  iw-&gt;err_rq_idx_valid);</div><div class='add'>+		qw7 |= FIELD_PREP(IRDMAQPC_PDIDX, iw-&gt;pd_id);</div><div class='add'>+		qw16 |= FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, iw-&gt;err_rq_idx) |</div><div class='add'>+			FIELD_PREP(IRDMAQPC_RTOMIN, iw-&gt;rtomin);</div><div class='add'>+		set_64bit_val(qp_ctx, 144,</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_Q2ADDR, qp-&gt;q2_pa &gt;&gt; 8) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_STAT_INDEX, info-&gt;stats_idx));</div><div class='add'>+</div><div class='add'>+		if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2)</div><div class='add'>+			mac = ether_addr_to_u64(iw-&gt;mac_addr);</div><div class='add'>+</div><div class='add'>+		set_64bit_val(qp_ctx, 152,</div><div class='add'>+			      mac &lt;&lt; 16 | FIELD_PREP(IRDMAQPC_LASTBYTESENT, iw-&gt;last_byte_sent));</div><div class='add'>+		set_64bit_val(qp_ctx, 160,</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_ORDSIZE, iw-&gt;ord_size) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(iw-&gt;ird_size)) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_WRRDRSPOK, iw-&gt;wr_rdresp_en) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_RDOK, iw-&gt;rd_en) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_SNDMARKERS, iw-&gt;snd_mark_en) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_BINDEN, iw-&gt;bind_en) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_FASTREGEN, iw-&gt;fast_reg_en) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_PRIVEN, iw-&gt;priv_mode_en) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info-&gt;stats_idx_valid) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_IWARPMODE, 1) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_RCVMARKERS, iw-&gt;rcv_mark_en) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_ALIGNHDRS, iw-&gt;align_hdrs) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_RCVNOMPACRC, iw-&gt;rcv_no_mpa_crc) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_RCVMARKOFFSET, iw-&gt;rcv_mark_offset || !tcp ? iw-&gt;rcv_mark_offset : tcp-&gt;rcv_nxt) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_SNDMARKOFFSET, iw-&gt;snd_mark_offset || !tcp ? iw-&gt;snd_mark_offset : tcp-&gt;snd_nxt) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_TIMELYENABLE, iw-&gt;timely_en));</div><div class='add'>+	}</div><div class='add'>+	if (info-&gt;tcp_info_valid) {</div><div class='add'>+		qw0 |= FIELD_PREP(IRDMAQPC_IPV4, tcp-&gt;ipv4) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_NONAGLE, tcp-&gt;no_nagle) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_INSERTVLANTAG,</div><div class='add'>+				  tcp-&gt;insert_vlan_tag) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_TIMESTAMP, tcp-&gt;time_stamp) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_LIMIT, tcp-&gt;cwnd_inc_limit) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_DROPOOOSEG, tcp-&gt;drop_ooo_seg) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_DUPACK_THRESH, tcp-&gt;dup_ack_thresh);</div><div class='add'>+</div><div class='add'>+		if ((iw-&gt;ecn_en || iw-&gt;dctcp_en) &amp;&amp; !(tcp-&gt;tos &amp; 0x03))</div><div class='add'>+			tcp-&gt;tos |= ECN_CODE_PT_VAL;</div><div class='add'>+</div><div class='add'>+		qw3 |= FIELD_PREP(IRDMAQPC_TTL, tcp-&gt;ttl) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_AVOIDSTRETCHACK, tcp-&gt;avoid_stretch_ack) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_TOS, tcp-&gt;tos) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_SRCPORTNUM, tcp-&gt;src_port) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_DESTPORTNUM, tcp-&gt;dst_port);</div><div class='add'>+		if (dev-&gt;hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {</div><div class='add'>+			qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX, tcp-&gt;src_mac_addr_idx);</div><div class='add'>+</div><div class='add'>+			qp-&gt;src_mac_addr_idx = tcp-&gt;src_mac_addr_idx;</div><div class='add'>+		}</div><div class='add'>+		set_64bit_val(qp_ctx, 32,</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_DESTIPADDR2, tcp-&gt;dest_ip_addr[2]) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_DESTIPADDR3, tcp-&gt;dest_ip_addr[3]));</div><div class='add'>+		set_64bit_val(qp_ctx, 40,</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_DESTIPADDR0, tcp-&gt;dest_ip_addr[0]) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_DESTIPADDR1, tcp-&gt;dest_ip_addr[1]));</div><div class='add'>+		set_64bit_val(qp_ctx, 48,</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_SNDMSS, tcp-&gt;snd_mss) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_SYN_RST_HANDLING, tcp-&gt;syn_rst_handling) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_VLANTAG, tcp-&gt;vlan_tag) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_ARPIDX, tcp-&gt;arp_idx));</div><div class='add'>+		qw7 |= FIELD_PREP(IRDMAQPC_FLOWLABEL, tcp-&gt;flow_label) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_WSCALE, tcp-&gt;wscale) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_IGNORE_TCP_OPT,</div><div class='add'>+				  tcp-&gt;ignore_tcp_opt) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_IGNORE_TCP_UNS_OPT,</div><div class='add'>+				  tcp-&gt;ignore_tcp_uns_opt) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_TCPSTATE, tcp-&gt;tcp_state) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_RCVSCALE, tcp-&gt;rcv_wscale) |</div><div class='add'>+		       FIELD_PREP(IRDMAQPC_SNDSCALE, tcp-&gt;snd_wscale);</div><div class='add'>+		set_64bit_val(qp_ctx, 72,</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_TIMESTAMP_RECENT, tcp-&gt;time_stamp_recent) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_TIMESTAMP_AGE, tcp-&gt;time_stamp_age));</div><div class='add'>+		set_64bit_val(qp_ctx, 80,</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_SNDNXT, tcp-&gt;snd_nxt) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_SNDWND, tcp-&gt;snd_wnd));</div><div class='add'>+		set_64bit_val(qp_ctx, 88,</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_RCVNXT, tcp-&gt;rcv_nxt) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_RCVWND, tcp-&gt;rcv_wnd));</div><div class='add'>+		set_64bit_val(qp_ctx, 96,</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_SNDMAX, tcp-&gt;snd_max) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_SNDUNA, tcp-&gt;snd_una));</div><div class='add'>+		set_64bit_val(qp_ctx, 104,</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_SRTT, tcp-&gt;srtt) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_RTTVAR, tcp-&gt;rtt_var));</div><div class='add'>+		set_64bit_val(qp_ctx, 112,</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_SSTHRESH, tcp-&gt;ss_thresh) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_CWND, tcp-&gt;cwnd));</div><div class='add'>+		set_64bit_val(qp_ctx, 120,</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_SNDWL1, tcp-&gt;snd_wl1) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_SNDWL2, tcp-&gt;snd_wl2));</div><div class='add'>+		qw16 |= FIELD_PREP(IRDMAQPC_MAXSNDWND, tcp-&gt;max_snd_window) |</div><div class='add'>+			FIELD_PREP(IRDMAQPC_REXMIT_THRESH, tcp-&gt;rexmit_thresh);</div><div class='add'>+		set_64bit_val(qp_ctx, 184,</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, tcp-&gt;local_ipaddr[3]) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, tcp-&gt;local_ipaddr[2]));</div><div class='add'>+		set_64bit_val(qp_ctx, 192,</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, tcp-&gt;local_ipaddr[1]) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, tcp-&gt;local_ipaddr[0]));</div><div class='add'>+		set_64bit_val(qp_ctx, 200,</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_THIGH, iw-&gt;t_high) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_TLOW, iw-&gt;t_low));</div><div class='add'>+		set_64bit_val(qp_ctx, 208,</div><div class='add'>+			      FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info-&gt;rem_endpoint_idx));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	set_64bit_val(qp_ctx, 0, qw0);</div><div class='add'>+	set_64bit_val(qp_ctx, 24, qw3);</div><div class='add'>+	set_64bit_val(qp_ctx, 56, qw7);</div><div class='add'>+	set_64bit_val(qp_ctx, 128, qw16);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: QP_HOST CTX", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     qp_ctx, IRDMA_QP_CTX_SIZE, false);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_alloc_stag - mr stag alloc</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @info: stag info</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_alloc_stag(struct irdma_sc_dev *dev,</div><div class='add'>+			       struct irdma_allocate_stag_info *info,</div><div class='add'>+			       u64 scratch, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	enum irdma_page_size page_size;</div><div class='add'>+</div><div class='add'>+	if (!info-&gt;total_len &amp;&amp; !info-&gt;all_memory)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;page_size == 0x40000000)</div><div class='add'>+		page_size = IRDMA_PAGE_SIZE_1G;</div><div class='add'>+	else if (info-&gt;page_size == 0x200000)</div><div class='add'>+		page_size = IRDMA_PAGE_SIZE_2M;</div><div class='add'>+	else</div><div class='add'>+		page_size = IRDMA_PAGE_SIZE_4K;</div><div class='add'>+</div><div class='add'>+	cqp = dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 8,</div><div class='add'>+		      FLD_LS_64(dev, info-&gt;pd_id, IRDMA_CQPSQ_STAG_PDID) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info-&gt;total_len));</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info-&gt;stag_idx) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_STAG_PDID_HI, info-&gt;pd_id &gt;&gt; 18));</div><div class='add'>+	set_64bit_val(wqe, 40,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_STAG_HMCFNIDX, info-&gt;hmc_fcn_index));</div><div class='add'>+</div><div class='add'>+	if (info-&gt;chunk_size)</div><div class='add'>+		set_64bit_val(wqe, 48,</div><div class='add'>+			      FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info-&gt;first_pm_pbl_idx));</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_ALLOC_STAG) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_MR, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info-&gt;access_rights) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info-&gt;chunk_size) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_HPAGESIZE, page_size) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, info-&gt;remote_access) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info-&gt;use_hmc_fcn_index) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info-&gt;use_pf_rid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_REMOTE_ATOMIC_EN,</div><div class='add'>+			 info-&gt;remote_atomics_en) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: ALLOC_STAG WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_mr_reg_non_shared - non-shared mr registration</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @info: mr info</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_mr_reg_non_shared(struct irdma_sc_dev *dev,</div><div class='add'>+				      struct irdma_reg_ns_stag_info *info,</div><div class='add'>+				      u64 scratch, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 fbo;</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 pble_obj_cnt;</div><div class='add'>+	bool remote_access;</div><div class='add'>+	u8 addr_type;</div><div class='add'>+	enum irdma_page_size page_size;</div><div class='add'>+</div><div class='add'>+	if (!info-&gt;total_len &amp;&amp; !info-&gt;all_memory)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;page_size == 0x40000000)</div><div class='add'>+		page_size = IRDMA_PAGE_SIZE_1G;</div><div class='add'>+	else if (info-&gt;page_size == 0x200000)</div><div class='add'>+		page_size = IRDMA_PAGE_SIZE_2M;</div><div class='add'>+	else if (info-&gt;page_size == 0x1000)</div><div class='add'>+		page_size = IRDMA_PAGE_SIZE_4K;</div><div class='add'>+	else</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;access_rights &amp; (IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY |</div><div class='add'>+				   IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY))</div><div class='add'>+		remote_access = true;</div><div class='add'>+	else</div><div class='add'>+		remote_access = false;</div><div class='add'>+</div><div class='add'>+	pble_obj_cnt = dev-&gt;hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].cnt;</div><div class='add'>+	if (info-&gt;chunk_size &amp;&amp; info-&gt;first_pm_pbl_index &gt;= pble_obj_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	cqp = dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	fbo = info-&gt;va &amp; (info-&gt;page_size - 1);</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0,</div><div class='add'>+		      (info-&gt;addr_type == IRDMA_ADDR_TYPE_VA_BASED ?</div><div class='add'>+		      info-&gt;va : fbo));</div><div class='add'>+	set_64bit_val(wqe, 8,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info-&gt;total_len) |</div><div class='add'>+		      FLD_LS_64(dev, info-&gt;pd_id, IRDMA_CQPSQ_STAG_PDID));</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_STAG_KEY, info-&gt;stag_key) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_STAG_PDID_HI, info-&gt;pd_id &gt;&gt; 18) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info-&gt;stag_idx));</div><div class='add'>+	if (!info-&gt;chunk_size) {</div><div class='add'>+		set_64bit_val(wqe, 32, info-&gt;reg_addr_pa);</div><div class='add'>+		set_64bit_val(wqe, 48, 0);</div><div class='add'>+	} else {</div><div class='add'>+		set_64bit_val(wqe, 32, 0);</div><div class='add'>+		set_64bit_val(wqe, 48,</div><div class='add'>+			      FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info-&gt;first_pm_pbl_index));</div><div class='add'>+	}</div><div class='add'>+	set_64bit_val(wqe, 40, info-&gt;hmc_fcn_index);</div><div class='add'>+	set_64bit_val(wqe, 56, 0);</div><div class='add'>+</div><div class='add'>+	addr_type = (info-&gt;addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1 : 0;</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_REG_MR) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_MR, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info-&gt;chunk_size) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_HPAGESIZE, page_size) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info-&gt;access_rights) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, remote_access) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_VABASEDTO, addr_type) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info-&gt;use_hmc_fcn_index) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info-&gt;use_pf_rid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_REMOTE_ATOMIC_EN,</div><div class='add'>+			 info-&gt;remote_atomics_en) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: MR_REG_NS WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_dealloc_stag - deallocate stag</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @info: dealloc stag info</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_dealloc_stag(struct irdma_sc_dev *dev,</div><div class='add'>+				 struct irdma_dealloc_stag_info *info,</div><div class='add'>+				 u64 scratch, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	u64 hdr;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+</div><div class='add'>+	cqp = dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 8,</div><div class='add'>+		      FLD_LS_64(dev, info-&gt;pd_id, IRDMA_CQPSQ_STAG_PDID));</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info-&gt;stag_idx) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_STAG_PDID_HI, info-&gt;pd_id &gt;&gt; 18));</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DEALLOC_STAG) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_MR, info-&gt;mr) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: DEALLOC_STAG WQE", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_mw_alloc - mw allocate</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @info: memory window allocation information</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_mw_alloc(struct irdma_sc_dev *dev,</div><div class='add'>+			     struct irdma_mw_alloc_info *info, u64 scratch,</div><div class='add'>+			     bool post_sq)</div><div class='add'>+{</div><div class='add'>+	u64 hdr;</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+</div><div class='add'>+	cqp = dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 8,</div><div class='add'>+		      FLD_LS_64(dev, info-&gt;pd_id, IRDMA_CQPSQ_STAG_PDID));</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info-&gt;mw_stag_index) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_STAG_PDID_HI, info-&gt;pd_id &gt;&gt; 18));</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_ALLOC_STAG) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_MWTYPE, info-&gt;mw_wide) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY,</div><div class='add'>+			 info-&gt;mw1_bind_dont_vldt_key) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: MW_ALLOC WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp</div><div class='add'>+ * @qp: sc qp struct</div><div class='add'>+ * @info: fast mr info</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,</div><div class='add'>+			      struct irdma_fast_reg_stag_info *info,</div><div class='add'>+			      bool post_sq)</div><div class='add'>+{</div><div class='add'>+	u64 temp, hdr;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+	enum irdma_page_size page_size;</div><div class='add'>+	struct irdma_post_sq_info sq_info = {};</div><div class='add'>+</div><div class='add'>+	if (info-&gt;page_size == 0x40000000)</div><div class='add'>+		page_size = IRDMA_PAGE_SIZE_1G;</div><div class='add'>+	else if (info-&gt;page_size == 0x200000)</div><div class='add'>+		page_size = IRDMA_PAGE_SIZE_2M;</div><div class='add'>+	else</div><div class='add'>+		page_size = IRDMA_PAGE_SIZE_4K;</div><div class='add'>+</div><div class='add'>+	sq_info.wr_id = info-&gt;wr_id;</div><div class='add'>+	sq_info.signaled = info-&gt;signaled;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_qp_get_next_send_wqe(&amp;qp-&gt;qp_uk, &amp;wqe_idx,</div><div class='add'>+					 IRDMA_QP_WQE_MIN_QUANTA, 0, &amp;sq_info);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	irdma_clr_wqes(&amp;qp-&gt;qp_uk, wqe_idx);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(to_ibdev(qp-&gt;dev),</div><div class='add'>+		  "MR: wr_id[%llxh] wqe_idx[%04d] location[%p]\n",</div><div class='add'>+		  info-&gt;wr_id, wqe_idx,</div><div class='add'>+		  &amp;qp-&gt;qp_uk.sq_wrtrk_array[wqe_idx].wrid);</div><div class='add'>+</div><div class='add'>+	temp = (info-&gt;addr_type == IRDMA_ADDR_TYPE_VA_BASED) ?</div><div class='add'>+		(uintptr_t)info-&gt;va : info-&gt;fbo;</div><div class='add'>+	set_64bit_val(wqe, 0, temp);</div><div class='add'>+</div><div class='add'>+	temp = FIELD_GET(IRDMAQPSQ_FIRSTPMPBLIDXHI,</div><div class='add'>+			 info-&gt;first_pm_pbl_index &gt;&gt; 16);</div><div class='add'>+	set_64bit_val(wqe, 8,</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXHI, temp) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_PBLADDR &gt;&gt; IRDMA_HW_PAGE_SHIFT, info-&gt;reg_addr_pa));</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      info-&gt;total_len |</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXLO, info-&gt;first_pm_pbl_index));</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMAQPSQ_STAGKEY, info-&gt;stag_key) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_STAGINDEX, info-&gt;stag_idx) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_FAST_REGISTER) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_LPBLSIZE, info-&gt;chunk_size) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_HPAGESIZE, page_size) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, info-&gt;access_rights) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VABASEDTO, info-&gt;addr_type) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_READFENCE, info-&gt;read_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info-&gt;local_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info-&gt;signaled) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_REMOTE_ATOMICS_EN, info-&gt;remote_atomics_en) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;qp_uk.swqe_polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: FAST_REG WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_QP_WQE_MIN_SIZE, false);</div><div class='add'>+</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_uk_qp_post_wr(&amp;qp-&gt;qp_uk);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_gen_rts_ae - request AE generated after RTS</div><div class='add'>+ * @qp: sc qp struct</div><div class='add'>+ */</div><div class='add'>+static void irdma_sc_gen_rts_ae(struct irdma_sc_qp *qp)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	struct irdma_qp_uk *qp_uk;</div><div class='add'>+</div><div class='add'>+	qp_uk = &amp;qp-&gt;qp_uk;</div><div class='add'>+</div><div class='add'>+	wqe = qp_uk-&gt;sq_base[1].elem;</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;qp_uk.swqe_polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+	print_hex_dump_debug("QP: NOP W/LOCAL FENCE WQE", DUMP_PREFIX_OFFSET,</div><div class='add'>+			     16, 8, wqe, IRDMA_QP_WQE_MIN_SIZE, false);</div><div class='add'>+</div><div class='add'>+	wqe = qp_uk-&gt;sq_base[2].elem;</div><div class='add'>+	hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_GEN_RTS_AE) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;qp_uk.swqe_polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+	print_hex_dump_debug("QP: CONN EST WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_QP_WQE_MIN_SIZE, false);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_send_lsmm - send last streaming mode message</div><div class='add'>+ * @qp: sc qp struct</div><div class='add'>+ * @lsmm_buf: buffer with lsmm message</div><div class='add'>+ * @size: size of lsmm buffer</div><div class='add'>+ * @stag: stag of lsmm buffer</div><div class='add'>+ */</div><div class='add'>+void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,</div><div class='add'>+			irdma_stag stag)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	struct irdma_qp_uk *qp_uk;</div><div class='add'>+</div><div class='add'>+	qp_uk = &amp;qp-&gt;qp_uk;</div><div class='add'>+	wqe = qp_uk-&gt;sq_base-&gt;elem;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);</div><div class='add'>+	if (qp-&gt;qp_uk.uk_attrs-&gt;hw_rev == IRDMA_GEN_1) {</div><div class='add'>+		set_64bit_val(wqe, 8,</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, size) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, stag));</div><div class='add'>+	} else {</div><div class='add'>+		set_64bit_val(wqe, 8,</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_FRAG_LEN, size) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_FRAG_STAG, stag) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;qp_uk.swqe_polarity));</div><div class='add'>+	}</div><div class='add'>+	set_64bit_val(wqe, 16, 0);</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_SEND) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_STREAMMODE, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_WAITFORRCVPDU, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;qp_uk.swqe_polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: SEND_LSMM WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_QP_WQE_MIN_SIZE, false);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;dev-&gt;hw_attrs.uk_attrs.feature_flags &amp; IRDMA_FEATURE_RTS_AE)</div><div class='add'>+		irdma_sc_gen_rts_ae(qp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_send_rtt - send last read0 or write0</div><div class='add'>+ * @qp: sc qp struct</div><div class='add'>+ * @read: Do read0 or write0</div><div class='add'>+ */</div><div class='add'>+void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	struct irdma_qp_uk *qp_uk;</div><div class='add'>+</div><div class='add'>+	qp_uk = &amp;qp-&gt;qp_uk;</div><div class='add'>+	wqe = qp_uk-&gt;sq_base-&gt;elem;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0, 0);</div><div class='add'>+	set_64bit_val(wqe, 16, 0);</div><div class='add'>+	if (read) {</div><div class='add'>+		if (qp-&gt;qp_uk.uk_attrs-&gt;hw_rev == IRDMA_GEN_1) {</div><div class='add'>+			set_64bit_val(wqe, 8,</div><div class='add'>+				      FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, 0xabcd));</div><div class='add'>+		} else {</div><div class='add'>+			set_64bit_val(wqe, 8,</div><div class='add'>+				      (u64)0xabcd | FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;qp_uk.swqe_polarity));</div><div class='add'>+		}</div><div class='add'>+		hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, 0x1234) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_READ) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;qp_uk.swqe_polarity);</div><div class='add'>+</div><div class='add'>+	} else {</div><div class='add'>+		if (qp-&gt;qp_uk.uk_attrs-&gt;hw_rev == IRDMA_GEN_1) {</div><div class='add'>+			set_64bit_val(wqe, 8, 0);</div><div class='add'>+		} else {</div><div class='add'>+			set_64bit_val(wqe, 8,</div><div class='add'>+				      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;qp_uk.swqe_polarity));</div><div class='add'>+		}</div><div class='add'>+		hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_WRITE) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;qp_uk.swqe_polarity);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: RTR WQE", DUMP_PREFIX_OFFSET, 16, 8, wqe,</div><div class='add'>+			     IRDMA_QP_WQE_MIN_SIZE, false);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;dev-&gt;hw_attrs.uk_attrs.feature_flags &amp; IRDMA_FEATURE_RTS_AE)</div><div class='add'>+		irdma_sc_gen_rts_ae(qp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_iwarp_opcode - determine if incoming is rdma layer</div><div class='add'>+ * @info: aeq info for the packet</div><div class='add'>+ * @pkt: packet for error</div><div class='add'>+ */</div><div class='add'>+static u32 irdma_iwarp_opcode(struct irdma_aeqe_info *info, u8 *pkt)</div><div class='add'>+{</div><div class='add'>+	__be16 *mpa;</div><div class='add'>+	u32 opcode = 0xffffffff;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;q2_data_written) {</div><div class='add'>+		mpa = (__be16 *)pkt;</div><div class='add'>+		opcode = ntohs(mpa[1]) &amp; 0xf;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return opcode;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_locate_mpa - return pointer to mpa in the pkt</div><div class='add'>+ * @pkt: packet with data</div><div class='add'>+ */</div><div class='add'>+static u8 *irdma_locate_mpa(u8 *pkt)</div><div class='add'>+{</div><div class='add'>+	/* skip over ethernet header */</div><div class='add'>+	pkt += IRDMA_MAC_HLEN;</div><div class='add'>+</div><div class='add'>+	/* Skip over IP and TCP headers */</div><div class='add'>+	pkt += 4 * (pkt[0] &amp; 0x0f);</div><div class='add'>+	pkt += 4 * ((pkt[12] &gt;&gt; 4) &amp; 0x0f);</div><div class='add'>+</div><div class='add'>+	return pkt;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_bld_termhdr_ctrl - setup terminate hdr control fields</div><div class='add'>+ * @qp: sc qp ptr for pkt</div><div class='add'>+ * @hdr: term hdr</div><div class='add'>+ * @opcode: flush opcode for termhdr</div><div class='add'>+ * @layer_etype: error layer + error type</div><div class='add'>+ * @err: error cod ein the header</div><div class='add'>+ */</div><div class='add'>+static void irdma_bld_termhdr_ctrl(struct irdma_sc_qp *qp,</div><div class='add'>+				   struct irdma_terminate_hdr *hdr,</div><div class='add'>+				   enum irdma_flush_opcode opcode,</div><div class='add'>+				   u8 layer_etype, u8 err)</div><div class='add'>+{</div><div class='add'>+	qp-&gt;flush_code = opcode;</div><div class='add'>+	hdr-&gt;layer_etype = layer_etype;</div><div class='add'>+	hdr-&gt;error_code = err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_bld_termhdr_ddp_rdma - setup ddp and rdma hdrs in terminate hdr</div><div class='add'>+ * @pkt: ptr to mpa in offending pkt</div><div class='add'>+ * @hdr: term hdr</div><div class='add'>+ * @copy_len: offending pkt length to be copied to term hdr</div><div class='add'>+ * @is_tagged: DDP tagged or untagged</div><div class='add'>+ */</div><div class='add'>+static void irdma_bld_termhdr_ddp_rdma(u8 *pkt, struct irdma_terminate_hdr *hdr,</div><div class='add'>+				       int *copy_len, u8 *is_tagged)</div><div class='add'>+{</div><div class='add'>+	u16 ddp_seg_len;</div><div class='add'>+</div><div class='add'>+	ddp_seg_len = ntohs(*(__be16 *)pkt);</div><div class='add'>+	if (ddp_seg_len) {</div><div class='add'>+		*copy_len = 2;</div><div class='add'>+		hdr-&gt;hdrct = DDP_LEN_FLAG;</div><div class='add'>+		if (pkt[2] &amp; 0x80) {</div><div class='add'>+			*is_tagged = 1;</div><div class='add'>+			if (ddp_seg_len &gt;= TERM_DDP_LEN_TAGGED) {</div><div class='add'>+				*copy_len += TERM_DDP_LEN_TAGGED;</div><div class='add'>+				hdr-&gt;hdrct |= DDP_HDR_FLAG;</div><div class='add'>+			}</div><div class='add'>+		} else {</div><div class='add'>+			if (ddp_seg_len &gt;= TERM_DDP_LEN_UNTAGGED) {</div><div class='add'>+				*copy_len += TERM_DDP_LEN_UNTAGGED;</div><div class='add'>+				hdr-&gt;hdrct |= DDP_HDR_FLAG;</div><div class='add'>+			}</div><div class='add'>+			if (ddp_seg_len &gt;= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN) &amp;&amp;</div><div class='add'>+			    ((pkt[3] &amp; RDMA_OPCODE_M) == RDMA_READ_REQ_OPCODE)) {</div><div class='add'>+				*copy_len += TERM_RDMA_LEN;</div><div class='add'>+				hdr-&gt;hdrct |= RDMA_HDR_FLAG;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_bld_terminate_hdr - build terminate message header</div><div class='add'>+ * @qp: qp associated with received terminate AE</div><div class='add'>+ * @info: the struct contiaing AE information</div><div class='add'>+ */</div><div class='add'>+static int irdma_bld_terminate_hdr(struct irdma_sc_qp *qp,</div><div class='add'>+				   struct irdma_aeqe_info *info)</div><div class='add'>+{</div><div class='add'>+	u8 *pkt = qp-&gt;q2_buf + Q2_BAD_FRAME_OFFSET;</div><div class='add'>+	int copy_len = 0;</div><div class='add'>+	u8 is_tagged = 0;</div><div class='add'>+	u32 opcode;</div><div class='add'>+	struct irdma_terminate_hdr *termhdr;</div><div class='add'>+</div><div class='add'>+	termhdr = (struct irdma_terminate_hdr *)qp-&gt;q2_buf;</div><div class='add'>+	memset(termhdr, 0, Q2_BAD_FRAME_OFFSET);</div><div class='add'>+</div><div class='add'>+	if (info-&gt;q2_data_written) {</div><div class='add'>+		pkt = irdma_locate_mpa(pkt);</div><div class='add'>+		irdma_bld_termhdr_ddp_rdma(pkt, termhdr, &amp;copy_len, &amp;is_tagged);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	opcode = irdma_iwarp_opcode(info, pkt);</div><div class='add'>+	qp-&gt;event_type = IRDMA_QP_EVENT_CATASTROPHIC;</div><div class='add'>+	qp-&gt;sq_flush_code = info-&gt;sq;</div><div class='add'>+	qp-&gt;rq_flush_code = info-&gt;rq;</div><div class='add'>+</div><div class='add'>+	switch (info-&gt;ae_id) {</div><div class='add'>+	case IRDMA_AE_AMP_UNALLOCATED_STAG:</div><div class='add'>+		qp-&gt;event_type = IRDMA_QP_EVENT_ACCESS_ERR;</div><div class='add'>+		if (opcode == IRDMA_OP_TYPE_RDMA_WRITE)</div><div class='add'>+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,</div><div class='add'>+					       (LAYER_DDP &lt;&lt; 4) | DDP_TAGGED_BUF,</div><div class='add'>+					       DDP_TAGGED_INV_STAG);</div><div class='add'>+		else</div><div class='add'>+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,</div><div class='add'>+					       (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_PROT,</div><div class='add'>+					       RDMAP_INV_STAG);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_AMP_BOUNDS_VIOLATION:</div><div class='add'>+		qp-&gt;event_type = IRDMA_QP_EVENT_ACCESS_ERR;</div><div class='add'>+		if (info-&gt;q2_data_written)</div><div class='add'>+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,</div><div class='add'>+					       (LAYER_DDP &lt;&lt; 4) | DDP_TAGGED_BUF,</div><div class='add'>+					       DDP_TAGGED_BOUNDS);</div><div class='add'>+		else</div><div class='add'>+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,</div><div class='add'>+					       (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_PROT,</div><div class='add'>+					       RDMAP_INV_BOUNDS);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_AMP_BAD_PD:</div><div class='add'>+		switch (opcode) {</div><div class='add'>+		case IRDMA_OP_TYPE_RDMA_WRITE:</div><div class='add'>+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,</div><div class='add'>+					       (LAYER_DDP &lt;&lt; 4) | DDP_TAGGED_BUF,</div><div class='add'>+					       DDP_TAGGED_UNASSOC_STAG);</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_OP_TYPE_SEND_INV:</div><div class='add'>+		case IRDMA_OP_TYPE_SEND_SOL_INV:</div><div class='add'>+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,</div><div class='add'>+					       (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_PROT,</div><div class='add'>+					       RDMAP_CANT_INV_STAG);</div><div class='add'>+			break;</div><div class='add'>+		default:</div><div class='add'>+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,</div><div class='add'>+					       (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_PROT,</div><div class='add'>+					       RDMAP_UNASSOC_STAG);</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_AMP_INVALID_STAG:</div><div class='add'>+		qp-&gt;event_type = IRDMA_QP_EVENT_ACCESS_ERR;</div><div class='add'>+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,</div><div class='add'>+				       (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_PROT,</div><div class='add'>+				       RDMAP_INV_STAG);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_AMP_BAD_QP:</div><div class='add'>+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_QP_OP_ERR,</div><div class='add'>+				       (LAYER_DDP &lt;&lt; 4) | DDP_UNTAGGED_BUF,</div><div class='add'>+				       DDP_UNTAGGED_INV_QN);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_AMP_BAD_STAG_KEY:</div><div class='add'>+	case IRDMA_AE_AMP_BAD_STAG_INDEX:</div><div class='add'>+		qp-&gt;event_type = IRDMA_QP_EVENT_ACCESS_ERR;</div><div class='add'>+		switch (opcode) {</div><div class='add'>+		case IRDMA_OP_TYPE_SEND_INV:</div><div class='add'>+		case IRDMA_OP_TYPE_SEND_SOL_INV:</div><div class='add'>+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_OP_ERR,</div><div class='add'>+					       (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_OP,</div><div class='add'>+					       RDMAP_CANT_INV_STAG);</div><div class='add'>+			break;</div><div class='add'>+		default:</div><div class='add'>+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,</div><div class='add'>+					       (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_OP,</div><div class='add'>+					       RDMAP_INV_STAG);</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_AMP_RIGHTS_VIOLATION:</div><div class='add'>+	case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:</div><div class='add'>+	case IRDMA_AE_PRIV_OPERATION_DENIED:</div><div class='add'>+		qp-&gt;event_type = IRDMA_QP_EVENT_ACCESS_ERR;</div><div class='add'>+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,</div><div class='add'>+				       (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_PROT,</div><div class='add'>+				       RDMAP_ACCESS);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_AMP_TO_WRAP:</div><div class='add'>+		qp-&gt;event_type = IRDMA_QP_EVENT_ACCESS_ERR;</div><div class='add'>+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,</div><div class='add'>+				       (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_PROT,</div><div class='add'>+				       RDMAP_TO_WRAP);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:</div><div class='add'>+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,</div><div class='add'>+				       (LAYER_MPA &lt;&lt; 4) | DDP_LLP, MPA_CRC);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:</div><div class='add'>+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_LEN_ERR,</div><div class='add'>+				       (LAYER_DDP &lt;&lt; 4) | DDP_CATASTROPHIC,</div><div class='add'>+				       DDP_CATASTROPHIC_LOCAL);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_LCE_QP_CATASTROPHIC:</div><div class='add'>+	case IRDMA_AE_DDP_NO_L_BIT:</div><div class='add'>+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_FATAL_ERR,</div><div class='add'>+				       (LAYER_DDP &lt;&lt; 4) | DDP_CATASTROPHIC,</div><div class='add'>+				       DDP_CATASTROPHIC_LOCAL);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN:</div><div class='add'>+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,</div><div class='add'>+				       (LAYER_DDP &lt;&lt; 4) | DDP_UNTAGGED_BUF,</div><div class='add'>+				       DDP_UNTAGGED_INV_MSN_RANGE);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:</div><div class='add'>+		qp-&gt;event_type = IRDMA_QP_EVENT_ACCESS_ERR;</div><div class='add'>+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_LEN_ERR,</div><div class='add'>+				       (LAYER_DDP &lt;&lt; 4) | DDP_UNTAGGED_BUF,</div><div class='add'>+				       DDP_UNTAGGED_INV_TOO_LONG);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:</div><div class='add'>+		if (is_tagged)</div><div class='add'>+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,</div><div class='add'>+					       (LAYER_DDP &lt;&lt; 4) | DDP_TAGGED_BUF,</div><div class='add'>+					       DDP_TAGGED_INV_DDP_VER);</div><div class='add'>+		else</div><div class='add'>+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,</div><div class='add'>+					       (LAYER_DDP &lt;&lt; 4) | DDP_UNTAGGED_BUF,</div><div class='add'>+					       DDP_UNTAGGED_INV_DDP_VER);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_DDP_UBE_INVALID_MO:</div><div class='add'>+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,</div><div class='add'>+				       (LAYER_DDP &lt;&lt; 4) | DDP_UNTAGGED_BUF,</div><div class='add'>+				       DDP_UNTAGGED_INV_MO);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:</div><div class='add'>+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_OP_ERR,</div><div class='add'>+				       (LAYER_DDP &lt;&lt; 4) | DDP_UNTAGGED_BUF,</div><div class='add'>+				       DDP_UNTAGGED_INV_MSN_NO_BUF);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_DDP_UBE_INVALID_QN:</div><div class='add'>+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,</div><div class='add'>+				       (LAYER_DDP &lt;&lt; 4) | DDP_UNTAGGED_BUF,</div><div class='add'>+				       DDP_UNTAGGED_INV_QN);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:</div><div class='add'>+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,</div><div class='add'>+				       (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_OP,</div><div class='add'>+				       RDMAP_INV_RDMAP_VER);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_FATAL_ERR,</div><div class='add'>+				       (LAYER_RDMA &lt;&lt; 4) | RDMAP_REMOTE_OP,</div><div class='add'>+				       RDMAP_UNSPECIFIED);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (copy_len)</div><div class='add'>+		memcpy(termhdr + 1, pkt, copy_len);</div><div class='add'>+</div><div class='add'>+	return sizeof(struct irdma_terminate_hdr) + copy_len;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_terminate_send_fin() - Send fin for terminate message</div><div class='add'>+ * @qp: qp associated with received terminate AE</div><div class='add'>+ */</div><div class='add'>+void irdma_terminate_send_fin(struct irdma_sc_qp *qp)</div><div class='add'>+{</div><div class='add'>+	irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE,</div><div class='add'>+			     IRDMAQP_TERM_SEND_FIN_ONLY, 0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_terminate_connection() - Bad AE and send terminate to remote QP</div><div class='add'>+ * @qp: qp associated with received terminate AE</div><div class='add'>+ * @info: the struct contiaing AE information</div><div class='add'>+ */</div><div class='add'>+void irdma_terminate_connection(struct irdma_sc_qp *qp,</div><div class='add'>+				struct irdma_aeqe_info *info)</div><div class='add'>+{</div><div class='add'>+	u8 termlen = 0;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;term_flags &amp; IRDMA_TERM_SENT)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	termlen = irdma_bld_terminate_hdr(qp, info);</div><div class='add'>+	irdma_terminate_start_timer(qp);</div><div class='add'>+	qp-&gt;term_flags |= IRDMA_TERM_SENT;</div><div class='add'>+	irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE,</div><div class='add'>+			     IRDMAQP_TERM_SEND_TERM_ONLY, termlen);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_terminate_received - handle terminate received AE</div><div class='add'>+ * @qp: qp associated with received terminate AE</div><div class='add'>+ * @info: the struct contiaing AE information</div><div class='add'>+ */</div><div class='add'>+void irdma_terminate_received(struct irdma_sc_qp *qp,</div><div class='add'>+			      struct irdma_aeqe_info *info)</div><div class='add'>+{</div><div class='add'>+	u8 *pkt = qp-&gt;q2_buf + Q2_BAD_FRAME_OFFSET;</div><div class='add'>+	__be32 *mpa;</div><div class='add'>+	u8 ddp_ctl;</div><div class='add'>+	u8 rdma_ctl;</div><div class='add'>+	u16 aeq_id = 0;</div><div class='add'>+	struct irdma_terminate_hdr *termhdr;</div><div class='add'>+</div><div class='add'>+	mpa = (__be32 *)irdma_locate_mpa(pkt);</div><div class='add'>+	if (info-&gt;q2_data_written) {</div><div class='add'>+		/* did not validate the frame - do it now */</div><div class='add'>+		ddp_ctl = (ntohl(mpa[0]) &gt;&gt; 8) &amp; 0xff;</div><div class='add'>+		rdma_ctl = ntohl(mpa[0]) &amp; 0xff;</div><div class='add'>+		if ((ddp_ctl &amp; 0xc0) != 0x40)</div><div class='add'>+			aeq_id = IRDMA_AE_LCE_QP_CATASTROPHIC;</div><div class='add'>+		else if ((ddp_ctl &amp; 0x03) != 1)</div><div class='add'>+			aeq_id = IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION;</div><div class='add'>+		else if (ntohl(mpa[2]) != 2)</div><div class='add'>+			aeq_id = IRDMA_AE_DDP_UBE_INVALID_QN;</div><div class='add'>+		else if (ntohl(mpa[3]) != 1)</div><div class='add'>+			aeq_id = IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN;</div><div class='add'>+		else if (ntohl(mpa[4]) != 0)</div><div class='add'>+			aeq_id = IRDMA_AE_DDP_UBE_INVALID_MO;</div><div class='add'>+		else if ((rdma_ctl &amp; 0xc0) != 0x40)</div><div class='add'>+			aeq_id = IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION;</div><div class='add'>+</div><div class='add'>+		info-&gt;ae_id = aeq_id;</div><div class='add'>+		if (info-&gt;ae_id) {</div><div class='add'>+			/* Bad terminate recvd - send back a terminate */</div><div class='add'>+			irdma_terminate_connection(qp, info);</div><div class='add'>+			return;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	qp-&gt;term_flags |= IRDMA_TERM_RCVD;</div><div class='add'>+	qp-&gt;event_type = IRDMA_QP_EVENT_CATASTROPHIC;</div><div class='add'>+	termhdr = (struct irdma_terminate_hdr *)&amp;mpa[5];</div><div class='add'>+	if (termhdr-&gt;layer_etype == RDMAP_REMOTE_PROT ||</div><div class='add'>+	    termhdr-&gt;layer_etype == RDMAP_REMOTE_OP) {</div><div class='add'>+		irdma_terminate_done(qp, 0);</div><div class='add'>+	} else {</div><div class='add'>+		irdma_terminate_start_timer(qp);</div><div class='add'>+		irdma_terminate_send_fin(qp);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_null_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri)</div><div class='add'>+{</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_null_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri)</div><div class='add'>+{</div><div class='add'>+	/* do nothing */</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_null_ws_reset(struct irdma_sc_vsi *vsi)</div><div class='add'>+{</div><div class='add'>+	/* do nothing */</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_vsi_init - Init the vsi structure</div><div class='add'>+ * @vsi: pointer to vsi structure to initialize</div><div class='add'>+ * @info: the info used to initialize the vsi struct</div><div class='add'>+ */</div><div class='add'>+void irdma_sc_vsi_init(struct irdma_sc_vsi  *vsi,</div><div class='add'>+		       struct irdma_vsi_init_info *info)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	vsi-&gt;dev = info-&gt;dev;</div><div class='add'>+	vsi-&gt;back_vsi = info-&gt;back_vsi;</div><div class='add'>+	vsi-&gt;register_qset = info-&gt;register_qset;</div><div class='add'>+	vsi-&gt;unregister_qset = info-&gt;unregister_qset;</div><div class='add'>+	vsi-&gt;mtu = info-&gt;params-&gt;mtu;</div><div class='add'>+	vsi-&gt;exception_lan_q = info-&gt;exception_lan_q;</div><div class='add'>+	vsi-&gt;vsi_idx = info-&gt;pf_data_vsi_num;</div><div class='add'>+</div><div class='add'>+	irdma_set_qos_info(vsi, info-&gt;params);</div><div class='add'>+	for (i = 0; i &lt; IRDMA_MAX_USER_PRIORITY; i++) {</div><div class='add'>+		mutex_init(&amp;vsi-&gt;qos[i].qos_mutex);</div><div class='add'>+		INIT_LIST_HEAD(&amp;vsi-&gt;qos[i].qplist);</div><div class='add'>+	}</div><div class='add'>+	if (vsi-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) {</div><div class='add'>+		vsi-&gt;dev-&gt;ws_add = irdma_ws_add;</div><div class='add'>+		vsi-&gt;dev-&gt;ws_remove = irdma_ws_remove;</div><div class='add'>+		vsi-&gt;dev-&gt;ws_reset = irdma_ws_reset;</div><div class='add'>+	} else {</div><div class='add'>+		vsi-&gt;dev-&gt;ws_add = irdma_null_ws_add;</div><div class='add'>+		vsi-&gt;dev-&gt;ws_remove = irdma_null_ws_remove;</div><div class='add'>+		vsi-&gt;dev-&gt;ws_reset = irdma_null_ws_reset;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_stats_idx - Return stats index</div><div class='add'>+ * @vsi: pointer to the vsi</div><div class='add'>+ */</div><div class='add'>+static u16 irdma_get_stats_idx(struct irdma_sc_vsi *vsi)</div><div class='add'>+{</div><div class='add'>+	struct irdma_stats_inst_info stats_info = {};</div><div class='add'>+	struct irdma_sc_dev *dev = vsi-&gt;dev;</div><div class='add'>+	u8 i;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+		if (!irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_ALLOCATE,</div><div class='add'>+					      &amp;stats_info))</div><div class='add'>+			return stats_info.stats_idx;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; IRDMA_MAX_STATS_COUNT_GEN_1; i++) {</div><div class='add'>+		if (!dev-&gt;stats_idx_array[i]) {</div><div class='add'>+			dev-&gt;stats_idx_array[i] = true;</div><div class='add'>+			return i;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return IRDMA_INVALID_STATS_IDX;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_hw_stats_init_gen1 - Initialize stat reg table used for gen1</div><div class='add'>+ * @vsi: vsi structure where hw_regs are set</div><div class='add'>+ *</div><div class='add'>+ * Populate the HW stats table</div><div class='add'>+ */</div><div class='add'>+static void irdma_hw_stats_init_gen1(struct irdma_sc_vsi *vsi)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = vsi-&gt;dev;</div><div class='add'>+	const struct irdma_hw_stat_map *map;</div><div class='add'>+	u64 *stat_reg = vsi-&gt;hw_stats_regs;</div><div class='add'>+	u64 *regs = dev-&gt;hw_stats_regs;</div><div class='add'>+	u16 i, stats_reg_set = vsi-&gt;stats_idx;</div><div class='add'>+</div><div class='add'>+	map = dev-&gt;hw_stats_map;</div><div class='add'>+</div><div class='add'>+	/* First 4 stat instances are reserved for port level statistics. */</div><div class='add'>+	stats_reg_set += vsi-&gt;stats_inst_alloc ? IRDMA_FIRST_NON_PF_STAT : 0;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; dev-&gt;hw_attrs.max_stat_idx; i++) {</div><div class='add'>+		if (map[i].bitmask &lt;= IRDMA_MAX_STATS_32)</div><div class='add'>+			stat_reg[i] = regs[i] + stats_reg_set * sizeof(u32);</div><div class='add'>+		else</div><div class='add'>+			stat_reg[i] = regs[i] + stats_reg_set * sizeof(u64);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_vsi_stats_init - Initialize the vsi statistics</div><div class='add'>+ * @vsi: pointer to the vsi structure</div><div class='add'>+ * @info: The info structure used for initialization</div><div class='add'>+ */</div><div class='add'>+int irdma_vsi_stats_init(struct irdma_sc_vsi *vsi,</div><div class='add'>+			 struct irdma_vsi_stats_info *info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_dma_mem *stats_buff_mem;</div><div class='add'>+</div><div class='add'>+	vsi-&gt;pestat = info-&gt;pestat;</div><div class='add'>+	vsi-&gt;pestat-&gt;hw = vsi-&gt;dev-&gt;hw;</div><div class='add'>+	vsi-&gt;pestat-&gt;vsi = vsi;</div><div class='add'>+	stats_buff_mem = &amp;vsi-&gt;pestat-&gt;gather_info.stats_buff_mem;</div><div class='add'>+	stats_buff_mem-&gt;size = ALIGN(IRDMA_GATHER_STATS_BUF_SIZE * 2, 1);</div><div class='add'>+	stats_buff_mem-&gt;va = dma_alloc_coherent(vsi-&gt;pestat-&gt;hw-&gt;device,</div><div class='add'>+						stats_buff_mem-&gt;size,</div><div class='add'>+						&amp;stats_buff_mem-&gt;pa,</div><div class='add'>+						GFP_KERNEL);</div><div class='add'>+	if (!stats_buff_mem-&gt;va)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	vsi-&gt;pestat-&gt;gather_info.gather_stats_va = stats_buff_mem-&gt;va;</div><div class='add'>+	vsi-&gt;pestat-&gt;gather_info.last_gather_stats_va =</div><div class='add'>+		(void *)((uintptr_t)stats_buff_mem-&gt;va +</div><div class='add'>+			 IRDMA_GATHER_STATS_BUF_SIZE);</div><div class='add'>+</div><div class='add'>+	if (vsi-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &lt; IRDMA_GEN_3)</div><div class='add'>+		irdma_hw_stats_start_timer(vsi);</div><div class='add'>+</div><div class='add'>+	/* when stat allocation is not required default to fcn_id. */</div><div class='add'>+	vsi-&gt;stats_idx = info-&gt;fcn_id;</div><div class='add'>+	if (info-&gt;alloc_stats_inst) {</div><div class='add'>+		u16 stats_idx = irdma_get_stats_idx(vsi);</div><div class='add'>+</div><div class='add'>+		if (stats_idx != IRDMA_INVALID_STATS_IDX) {</div><div class='add'>+			vsi-&gt;stats_inst_alloc = true;</div><div class='add'>+			vsi-&gt;stats_idx = stats_idx;</div><div class='add'>+			vsi-&gt;pestat-&gt;gather_info.use_stats_inst = true;</div><div class='add'>+			vsi-&gt;pestat-&gt;gather_info.stats_inst_index = stats_idx;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (vsi-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)</div><div class='add'>+		irdma_hw_stats_init_gen1(vsi);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_vsi_stats_free - Free the vsi stats</div><div class='add'>+ * @vsi: pointer to the vsi structure</div><div class='add'>+ */</div><div class='add'>+void irdma_vsi_stats_free(struct irdma_sc_vsi *vsi)</div><div class='add'>+{</div><div class='add'>+	struct irdma_stats_inst_info stats_info = {};</div><div class='add'>+	struct irdma_sc_dev *dev = vsi-&gt;dev;</div><div class='add'>+	u16 stats_idx = vsi-&gt;stats_idx;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+		if (vsi-&gt;stats_inst_alloc) {</div><div class='add'>+			stats_info.stats_idx = vsi-&gt;stats_idx;</div><div class='add'>+			irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_FREE,</div><div class='add'>+						 &amp;stats_info);</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		if (vsi-&gt;stats_inst_alloc &amp;&amp;</div><div class='add'>+		    stats_idx &lt; vsi-&gt;dev-&gt;hw_attrs.max_stat_inst)</div><div class='add'>+			vsi-&gt;dev-&gt;stats_idx_array[stats_idx] = false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!vsi-&gt;pestat)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &lt; IRDMA_GEN_3)</div><div class='add'>+		irdma_hw_stats_stop_timer(vsi);</div><div class='add'>+	dma_free_coherent(vsi-&gt;pestat-&gt;hw-&gt;device,</div><div class='add'>+			  vsi-&gt;pestat-&gt;gather_info.stats_buff_mem.size,</div><div class='add'>+			  vsi-&gt;pestat-&gt;gather_info.stats_buff_mem.va,</div><div class='add'>+			  vsi-&gt;pestat-&gt;gather_info.stats_buff_mem.pa);</div><div class='add'>+	vsi-&gt;pestat-&gt;gather_info.stats_buff_mem.va = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_encoded_wqe_size - given wq size, returns hardware encoded size</div><div class='add'>+ * @wqsize: size of the wq (sq, rq) to encoded_size</div><div class='add'>+ * @queue_type: queue type selected for the calculation algorithm</div><div class='add'>+ */</div><div class='add'>+u8 irdma_get_encoded_wqe_size(u32 wqsize, enum irdma_queue_type queue_type)</div><div class='add'>+{</div><div class='add'>+	u8 encoded_size = 0;</div><div class='add'>+</div><div class='add'>+	if (queue_type == IRDMA_QUEUE_TYPE_SRQ) {</div><div class='add'>+		/* Smallest SRQ size is 256B (8 quanta) that gets</div><div class='add'>+		 * encoded to 0.</div><div class='add'>+		 */</div><div class='add'>+		encoded_size = ilog2(wqsize) - 3;</div><div class='add'>+</div><div class='add'>+		return encoded_size;</div><div class='add'>+	}</div><div class='add'>+	/* cqp sq's hw coded value starts from 1 for size of 4</div><div class='add'>+	 * while it starts from 0 for qp' wq's.</div><div class='add'>+	 */</div><div class='add'>+	if (queue_type == IRDMA_QUEUE_TYPE_CQP)</div><div class='add'>+		encoded_size = 1;</div><div class='add'>+	wqsize &gt;&gt;= 2;</div><div class='add'>+	while (wqsize &gt;&gt;= 1)</div><div class='add'>+		encoded_size++;</div><div class='add'>+</div><div class='add'>+	return encoded_size;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_gather_stats - collect the statistics</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @info: gather stats info structure</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_gather_stats(struct irdma_sc_cqp *cqp,</div><div class='add'>+				 struct irdma_stats_gather_info *info,</div><div class='add'>+				 u64 scratch)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 temp;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;stats_buff_mem.size &lt; IRDMA_GATHER_STATS_BUF_SIZE)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 40,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info-&gt;hmc_fcn_index));</div><div class='add'>+	set_64bit_val(wqe, 32, info-&gt;stats_buff_mem.pa);</div><div class='add'>+</div><div class='add'>+	temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp-&gt;polarity) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_STATS_USE_INST, info-&gt;use_stats_inst) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX,</div><div class='add'>+			  info-&gt;stats_inst_index) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX,</div><div class='add'>+			  info-&gt;use_hmc_fcn_index) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_STATS_OP, IRDMA_CQP_OP_GATHER_STATS);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, temp);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("STATS: GATHER_STATS WQE", DUMP_PREFIX_OFFSET,</div><div class='add'>+			     16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+</div><div class='add'>+	irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+	ibdev_dbg(to_ibdev(cqp-&gt;dev),</div><div class='add'>+		  "STATS: CQP SQ head 0x%x tail 0x%x size 0x%x\n",</div><div class='add'>+		  cqp-&gt;sq_ring.head, cqp-&gt;sq_ring.tail, cqp-&gt;sq_ring.size);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_manage_stats_inst - allocate or free stats instance</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @info: stats info structure</div><div class='add'>+ * @alloc: alloc vs. delete flag</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_manage_stats_inst(struct irdma_sc_cqp *cqp,</div><div class='add'>+				      struct irdma_stats_inst_info *info,</div><div class='add'>+				      bool alloc, u64 scratch)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 temp;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 40,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info-&gt;hmc_fn_id));</div><div class='add'>+	temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp-&gt;polarity) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_STATS_ALLOC_INST, alloc) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX,</div><div class='add'>+			  info-&gt;use_hmc_fcn_index) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX, info-&gt;stats_idx) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_STATS_OP, IRDMA_CQP_OP_MANAGE_STATS);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, temp);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: MANAGE_STATS WQE", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+</div><div class='add'>+	irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_set_up_map - set the up map table</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @info: User priority map info</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_set_up_map(struct irdma_sc_cqp *cqp,</div><div class='add'>+			       struct irdma_up_info *info, u64 scratch)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 temp = 0;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; IRDMA_MAX_USER_PRIORITY; i++)</div><div class='add'>+		temp |= (u64)info-&gt;map[i] &lt;&lt; (i * 8);</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0, temp);</div><div class='add'>+	set_64bit_val(wqe, 40,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_UP_CNPOVERRIDE, info-&gt;cnp_up_override) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_UP_HMCFCNIDX, info-&gt;hmc_fcn_idx));</div><div class='add'>+</div><div class='add'>+	temp = FIELD_PREP(IRDMA_CQPSQ_UP_WQEVALID, cqp-&gt;polarity) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_UP_USEVLAN, info-&gt;use_vlan) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_UP_USEOVERRIDE,</div><div class='add'>+			  info-&gt;use_cnp_up_override) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_UP_OP, IRDMA_CQP_OP_UP_MAP);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, temp);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: UPMAP WQE", DUMP_PREFIX_OFFSET, 16, 8, wqe,</div><div class='add'>+			     IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_manage_ws_node - create/modify/destroy WS node</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @info: node info structure</div><div class='add'>+ * @node_op: 0 for add 1 for modify, 2 for delete</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp,</div><div class='add'>+				   struct irdma_ws_node_info *info,</div><div class='add'>+				   enum irdma_ws_node_op node_op, u64 scratch)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 temp = 0;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 32,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_WS_VSI, info-&gt;vsi) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_WS_WEIGHT, info-&gt;weight));</div><div class='add'>+</div><div class='add'>+	temp = FIELD_PREP(IRDMA_CQPSQ_WS_WQEVALID, cqp-&gt;polarity) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_WS_NODEOP, node_op) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_WS_ENABLENODE, info-&gt;enable) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_WS_NODETYPE, info-&gt;type_leaf) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_WS_PRIOTYPE, info-&gt;prio_type) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_WS_TC, info-&gt;tc) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_WS_OP, IRDMA_CQP_OP_WORK_SCHED_NODE) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_WS_PARENTID, info-&gt;parent_id) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_WS_NODEID, info-&gt;id);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, temp);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: MANAGE_WS WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_qp_flush_wqes - flush qp's wqe</div><div class='add'>+ * @qp: sc qp</div><div class='add'>+ * @info: dlush information</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,</div><div class='add'>+			   struct irdma_qp_flush_info *info, u64 scratch,</div><div class='add'>+			   bool post_sq)</div><div class='add'>+{</div><div class='add'>+	u64 temp = 0;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	bool flush_sq = false, flush_rq = false;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;rq &amp;&amp; !qp-&gt;flush_rq)</div><div class='add'>+		flush_rq = true;</div><div class='add'>+	if (info-&gt;sq &amp;&amp; !qp-&gt;flush_sq)</div><div class='add'>+		flush_sq = true;</div><div class='add'>+	qp-&gt;flush_sq |= flush_sq;</div><div class='add'>+	qp-&gt;flush_rq |= flush_rq;</div><div class='add'>+</div><div class='add'>+	if (!flush_sq &amp;&amp; !flush_rq) {</div><div class='add'>+		ibdev_dbg(to_ibdev(qp-&gt;dev),</div><div class='add'>+			  "CQP: Additional flush request ignored for qp %x\n",</div><div class='add'>+			  qp-&gt;qp_uk.qp_id);</div><div class='add'>+		return -EALREADY;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cqp = qp-&gt;pd-&gt;dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;userflushcode) {</div><div class='add'>+		if (flush_rq)</div><div class='add'>+			temp |= FIELD_PREP(IRDMA_CQPSQ_FWQE_RQMNERR,</div><div class='add'>+					   info-&gt;rq_minor_code) |</div><div class='add'>+				FIELD_PREP(IRDMA_CQPSQ_FWQE_RQMJERR,</div><div class='add'>+					   info-&gt;rq_major_code);</div><div class='add'>+		if (flush_sq)</div><div class='add'>+			temp |= FIELD_PREP(IRDMA_CQPSQ_FWQE_SQMNERR,</div><div class='add'>+					   info-&gt;sq_minor_code) |</div><div class='add'>+				FIELD_PREP(IRDMA_CQPSQ_FWQE_SQMJERR,</div><div class='add'>+					   info-&gt;sq_major_code);</div><div class='add'>+	}</div><div class='add'>+	set_64bit_val(wqe, 16, temp);</div><div class='add'>+</div><div class='add'>+	temp = (info-&gt;generate_ae) ?</div><div class='add'>+		info-&gt;ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE,</div><div class='add'>+					   info-&gt;ae_src) : 0;</div><div class='add'>+	set_64bit_val(wqe, 8, temp);</div><div class='add'>+	if (cqp-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3) {</div><div class='add'>+		set_64bit_val(wqe, 40,</div><div class='add'>+			      FIELD_PREP(IRDMA_CQPSQ_FWQE_ERR_SQ_IDX, info-&gt;err_sq_idx));</div><div class='add'>+		set_64bit_val(wqe, 48,</div><div class='add'>+			      FIELD_PREP(IRDMA_CQPSQ_FWQE_ERR_RQ_IDX, info-&gt;err_rq_idx));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hdr = qp-&gt;qp_uk.qp_id |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_FLUSH_WQES) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, info-&gt;generate_ae) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_FWQE_USERFLCODE, info-&gt;userflushcode) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_FWQE_FLUSHSQ, flush_sq) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_FWQE_FLUSHRQ, flush_rq) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	if (cqp-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3)</div><div class='add'>+		hdr |= FIELD_PREP(IRDMA_CQPSQ_FWQE_ERR_SQ_IDX_VALID, info-&gt;err_sq_idx_valid) |</div><div class='add'>+		       FIELD_PREP(IRDMA_CQPSQ_FWQE_ERR_RQ_IDX_VALID, info-&gt;err_rq_idx_valid);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: QP_FLUSH WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_gen_ae - generate AE, uses flush WQE CQP OP</div><div class='add'>+ * @qp: sc qp</div><div class='add'>+ * @info: gen ae information</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_gen_ae(struct irdma_sc_qp *qp,</div><div class='add'>+			   struct irdma_gen_ae_info *info, u64 scratch,</div><div class='add'>+			   bool post_sq)</div><div class='add'>+{</div><div class='add'>+	u64 temp;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	cqp = qp-&gt;pd-&gt;dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	temp = info-&gt;ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE,</div><div class='add'>+					  info-&gt;ae_src);</div><div class='add'>+	set_64bit_val(wqe, 8, temp);</div><div class='add'>+</div><div class='add'>+	hdr = qp-&gt;qp_uk.qp_id | FIELD_PREP(IRDMA_CQPSQ_OPCODE,</div><div class='add'>+					   IRDMA_CQP_OP_GEN_AE) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: GEN_AE WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*** irdma_sc_qp_upload_context - upload qp's context</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @info: upload context info ptr for return</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_qp_upload_context(struct irdma_sc_dev *dev,</div><div class='add'>+				      struct irdma_upload_context_info *info,</div><div class='add'>+				      u64 scratch, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	cqp = dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16, info-&gt;buf_pa);</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_UCTX_QPID, info-&gt;qp_id) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_UPLOAD_CONTEXT) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_UCTX_QPTYPE, info-&gt;qp_type) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_UCTX_RAWFORMAT, info-&gt;raw_format) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_UCTX_FREEZEQP, info-&gt;freeze_qp) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: QP_UPLOAD_CTX WQE", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_manage_push_page - Handle push page</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @info: push page info</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_manage_push_page(struct irdma_sc_cqp *cqp,</div><div class='add'>+				     struct irdma_cqp_manage_push_page_info *info,</div><div class='add'>+				     u64 scratch, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;free_page &amp;&amp;</div><div class='add'>+	    info-&gt;push_idx &gt;= cqp-&gt;dev-&gt;hw_attrs.max_hw_device_pages)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16, info-&gt;qs_handle);</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_MPP_PPIDX, info-&gt;push_idx) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_MPP_PPTYPE, info-&gt;push_page_type) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_PUSH_PAGES) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_MPP_FREE_PAGE, info-&gt;free_page);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: MANAGE_PUSH_PAGES WQE", DUMP_PREFIX_OFFSET,</div><div class='add'>+			     16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_suspend_qp - suspend qp for param change</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @qp: sc qp struct</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_suspend_qp(struct irdma_sc_cqp *cqp, struct irdma_sc_qp *qp,</div><div class='add'>+			       u64 scratch)</div><div class='add'>+{</div><div class='add'>+	u64 hdr;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_SUSPENDQP_QPID, qp-&gt;qp_uk.qp_id) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_SUSPEND_QP) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: SUSPEND_QP WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_resume_qp - resume qp after suspend</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @qp: sc qp struct</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_resume_qp(struct irdma_sc_cqp *cqp, struct irdma_sc_qp *qp,</div><div class='add'>+			      u64 scratch)</div><div class='add'>+{</div><div class='add'>+	u64 hdr;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QSHANDLE, qp-&gt;qs_handle));</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QPID, qp-&gt;qp_uk.qp_id) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_RESUME_QP) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: RESUME_QP WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cq_ack - acknowledge completion q</div><div class='add'>+ * @cq: cq struct</div><div class='add'>+ */</div><div class='add'>+static inline void irdma_sc_cq_ack(struct irdma_sc_cq *cq)</div><div class='add'>+{</div><div class='add'>+	writel(cq-&gt;cq_uk.cq_id, cq-&gt;cq_uk.cq_ack_db);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cq_init - initialize completion q</div><div class='add'>+ * @cq: cq struct</div><div class='add'>+ * @info: cq initialization info</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info)</div><div class='add'>+{</div><div class='add'>+	u32 pble_obj_cnt;</div><div class='add'>+</div><div class='add'>+	pble_obj_cnt = info-&gt;dev-&gt;hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].cnt;</div><div class='add'>+	if (info-&gt;virtual_map &amp;&amp; info-&gt;first_pm_pbl_idx &gt;= pble_obj_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	cq-&gt;cq_pa = info-&gt;cq_base_pa;</div><div class='add'>+	cq-&gt;dev = info-&gt;dev;</div><div class='add'>+	cq-&gt;ceq_id = info-&gt;ceq_id;</div><div class='add'>+	info-&gt;cq_uk_init_info.cqe_alloc_db = cq-&gt;dev-&gt;cq_arm_db;</div><div class='add'>+	info-&gt;cq_uk_init_info.cq_ack_db = cq-&gt;dev-&gt;cq_ack_db;</div><div class='add'>+	irdma_uk_cq_init(&amp;cq-&gt;cq_uk, &amp;info-&gt;cq_uk_init_info);</div><div class='add'>+</div><div class='add'>+	cq-&gt;virtual_map = info-&gt;virtual_map;</div><div class='add'>+	cq-&gt;pbl_chunk_size = info-&gt;pbl_chunk_size;</div><div class='add'>+	cq-&gt;ceqe_mask = info-&gt;ceqe_mask;</div><div class='add'>+	cq-&gt;cq_type = (info-&gt;type) ? info-&gt;type : IRDMA_CQ_TYPE_IWARP;</div><div class='add'>+	cq-&gt;shadow_area_pa = info-&gt;shadow_area_pa;</div><div class='add'>+	cq-&gt;shadow_read_threshold = info-&gt;shadow_read_threshold;</div><div class='add'>+	cq-&gt;ceq_id_valid = info-&gt;ceq_id_valid;</div><div class='add'>+	cq-&gt;tph_en = info-&gt;tph_en;</div><div class='add'>+	cq-&gt;tph_val = info-&gt;tph_val;</div><div class='add'>+	cq-&gt;first_pm_pbl_idx = info-&gt;first_pm_pbl_idx;</div><div class='add'>+	cq-&gt;vsi = info-&gt;vsi;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cq_create - create completion q</div><div class='add'>+ * @cq: cq struct</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @check_overflow: flag for overflow check</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_cq_create(struct irdma_sc_cq *cq, u64 scratch,</div><div class='add'>+			      bool check_overflow, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	cqp = cq-&gt;dev-&gt;cqp;</div><div class='add'>+	if (cq-&gt;cq_uk.cq_id &gt;= cqp-&gt;dev-&gt;hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_CQ].max_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (cq-&gt;ceq_id &gt;= cq-&gt;dev-&gt;hmc_fpm_misc.max_ceqs)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0, cq-&gt;cq_uk.cq_size);</div><div class='add'>+	set_64bit_val(wqe, 8, (uintptr_t)cq &gt;&gt; 1);</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq-&gt;shadow_read_threshold));</div><div class='add'>+	set_64bit_val(wqe, 32, (cq-&gt;virtual_map ? 0 : cq-&gt;cq_pa));</div><div class='add'>+	set_64bit_val(wqe, 40, cq-&gt;shadow_area_pa);</div><div class='add'>+	set_64bit_val(wqe, 48,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX, (cq-&gt;virtual_map ? cq-&gt;first_pm_pbl_idx : 0)));</div><div class='add'>+	set_64bit_val(wqe, 56,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq-&gt;tph_val) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq-&gt;vsi-&gt;vsi_idx));</div><div class='add'>+</div><div class='add'>+	hdr = FLD_LS_64(cq-&gt;dev, cq-&gt;cq_uk.cq_id, IRDMA_CQPSQ_CQ_CQID) |</div><div class='add'>+	      FLD_LS_64(cq-&gt;dev, (cq-&gt;ceq_id_valid ? cq-&gt;ceq_id : 0),</div><div class='add'>+			IRDMA_CQPSQ_CQ_CEQID) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq-&gt;pbl_chunk_size) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, check_overflow) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq-&gt;virtual_map) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_CQID_HIGH, cq-&gt;cq_uk.cq_id &gt;&gt; 22) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_CEQID_HIGH,</div><div class='add'>+			 (cq-&gt;ceq_id_valid ? cq-&gt;ceq_id : 0) &gt;&gt; 10) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq-&gt;ceqe_mask) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq-&gt;ceq_id_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq-&gt;tph_en) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT,</div><div class='add'>+			 cq-&gt;cq_uk.avoid_mem_cflct) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: CQ_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cq_destroy - destroy completion q</div><div class='add'>+ * @cq: cq struct</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	cqp = cq-&gt;dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0, cq-&gt;cq_uk.cq_size);</div><div class='add'>+	set_64bit_val(wqe, 8, (uintptr_t)cq &gt;&gt; 1);</div><div class='add'>+	set_64bit_val(wqe, 40, cq-&gt;shadow_area_pa);</div><div class='add'>+	set_64bit_val(wqe, 48,</div><div class='add'>+		      (cq-&gt;virtual_map ? cq-&gt;first_pm_pbl_idx : 0));</div><div class='add'>+</div><div class='add'>+	hdr = cq-&gt;cq_uk.cq_id |</div><div class='add'>+	      FLD_LS_64(cq-&gt;dev, (cq-&gt;ceq_id_valid ? cq-&gt;ceq_id : 0),</div><div class='add'>+			IRDMA_CQPSQ_CQ_CEQID) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CQ) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq-&gt;pbl_chunk_size) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq-&gt;virtual_map) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq-&gt;ceqe_mask) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq-&gt;ceq_id_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq-&gt;tph_en) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, cq-&gt;cq_uk.avoid_mem_cflct) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: CQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cq_resize - set resized cq buffer info</div><div class='add'>+ * @cq: resized cq</div><div class='add'>+ * @info: resized cq buffer info</div><div class='add'>+ */</div><div class='add'>+void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info)</div><div class='add'>+{</div><div class='add'>+	cq-&gt;virtual_map = info-&gt;virtual_map;</div><div class='add'>+	cq-&gt;cq_pa = info-&gt;cq_pa;</div><div class='add'>+	cq-&gt;first_pm_pbl_idx = info-&gt;first_pm_pbl_idx;</div><div class='add'>+	cq-&gt;pbl_chunk_size = info-&gt;pbl_chunk_size;</div><div class='add'>+	irdma_uk_cq_resize(&amp;cq-&gt;cq_uk, info-&gt;cq_base, info-&gt;cq_size);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cq_modify - modify a Completion Queue</div><div class='add'>+ * @cq: cq struct</div><div class='add'>+ * @info: modification info struct</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag to post to sq</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_cq_modify(struct irdma_sc_cq *cq,</div><div class='add'>+			      struct irdma_modify_cq_info *info, u64 scratch,</div><div class='add'>+			      bool post_sq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 pble_obj_cnt;</div><div class='add'>+</div><div class='add'>+	pble_obj_cnt = cq-&gt;dev-&gt;hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].cnt;</div><div class='add'>+	if (info-&gt;cq_resize &amp;&amp; info-&gt;virtual_map &amp;&amp;</div><div class='add'>+	    info-&gt;first_pm_pbl_idx &gt;= pble_obj_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	cqp = cq-&gt;dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0, info-&gt;cq_size);</div><div class='add'>+	set_64bit_val(wqe, 8, (uintptr_t)cq &gt;&gt; 1);</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, info-&gt;shadow_read_threshold));</div><div class='add'>+	set_64bit_val(wqe, 32, info-&gt;cq_pa);</div><div class='add'>+	set_64bit_val(wqe, 40, cq-&gt;shadow_area_pa);</div><div class='add'>+	set_64bit_val(wqe, 48, info-&gt;first_pm_pbl_idx);</div><div class='add'>+	set_64bit_val(wqe, 56,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq-&gt;tph_val) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq-&gt;vsi-&gt;vsi_idx));</div><div class='add'>+</div><div class='add'>+	hdr = cq-&gt;cq_uk.cq_id |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_CQ) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_CQRESIZE, info-&gt;cq_resize) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, info-&gt;pbl_chunk_size) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, info-&gt;check_overflow) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, info-&gt;virtual_map) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq-&gt;ceqe_mask) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq-&gt;tph_en) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT,</div><div class='add'>+			 cq-&gt;cq_uk.avoid_mem_cflct) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: CQ_MODIFY WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_get_decoded_ird_size_gen_3 - get decoded IRD size for GEN 3</div><div class='add'>+ * @ird_enc: IRD encoding</div><div class='add'>+ * IRD size defaults to a value of 4 in case of invalid input.</div><div class='add'>+ */</div><div class='add'>+static u16 irdma_sc_get_decoded_ird_size_gen_3(u8 ird_enc)</div><div class='add'>+{</div><div class='add'>+	switch (ird_enc) {</div><div class='add'>+	case IRDMA_IRD_HW_SIZE_4096_GEN3:</div><div class='add'>+		return 4096;</div><div class='add'>+	case IRDMA_IRD_HW_SIZE_2048_GEN3:</div><div class='add'>+		return 2048;</div><div class='add'>+	case IRDMA_IRD_HW_SIZE_1024_GEN3:</div><div class='add'>+		return 1024;</div><div class='add'>+	case IRDMA_IRD_HW_SIZE_512_GEN3:</div><div class='add'>+		return 512;</div><div class='add'>+	case IRDMA_IRD_HW_SIZE_256_GEN3:</div><div class='add'>+		return 256;</div><div class='add'>+	case IRDMA_IRD_HW_SIZE_128_GEN3:</div><div class='add'>+		return 128;</div><div class='add'>+	case IRDMA_IRD_HW_SIZE_64_GEN3:</div><div class='add'>+		return 64;</div><div class='add'>+	case IRDMA_IRD_HW_SIZE_32_GEN3:</div><div class='add'>+		return 32;</div><div class='add'>+	case IRDMA_IRD_HW_SIZE_16_GEN3:</div><div class='add'>+		return 16;</div><div class='add'>+	case IRDMA_IRD_HW_SIZE_8_GEN3:</div><div class='add'>+		return 8;</div><div class='add'>+	case IRDMA_IRD_HW_SIZE_4_GEN3:</div><div class='add'>+		return 4;</div><div class='add'>+	default:</div><div class='add'>+		return 4;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_check_cqp_progress - check cqp processing progress</div><div class='add'>+ * @timeout: timeout info struct</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ */</div><div class='add'>+void irdma_check_cqp_progress(struct irdma_cqp_timeout *timeout, struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	u64 completed_ops = atomic64_read(&amp;dev-&gt;cqp-&gt;completed_ops);</div><div class='add'>+</div><div class='add'>+	if (timeout-&gt;compl_cqp_cmds != completed_ops) {</div><div class='add'>+		timeout-&gt;compl_cqp_cmds = completed_ops;</div><div class='add'>+		timeout-&gt;count = 0;</div><div class='add'>+	} else if (timeout-&gt;compl_cqp_cmds != dev-&gt;cqp-&gt;requested_ops) {</div><div class='add'>+		timeout-&gt;count++;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_cqp_reg_info - get head and tail for cqp using registers</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @val: cqp tail register value</div><div class='add'>+ * @tail: wqtail register value</div><div class='add'>+ * @error: cqp processing err</div><div class='add'>+ */</div><div class='add'>+static inline void irdma_get_cqp_reg_info(struct irdma_sc_cqp *cqp, u32 *val,</div><div class='add'>+					  u32 *tail, u32 *error)</div><div class='add'>+{</div><div class='add'>+	*val = readl(cqp-&gt;dev-&gt;hw_regs[IRDMA_CQPTAIL]);</div><div class='add'>+	*tail = FIELD_GET(IRDMA_CQPTAIL_WQTAIL, *val);</div><div class='add'>+	*error = FIELD_GET(IRDMA_CQPTAIL_CQP_OP_ERR, *val);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cqp_def_cmpl_ae_handler - remove completed requests from pending list</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @info: AE entry info</div><div class='add'>+ * @first: true if this is the first call to this handler for given AEQE</div><div class='add'>+ * @scratch: (out) scratch entry pointer</div><div class='add'>+ * @sw_def_info: (in/out) SW ticket value for this AE</div><div class='add'>+ *</div><div class='add'>+ * In case of AE_DEF_CMPL event, this function should be called in a loop</div><div class='add'>+ * until it returns NULL-ptr via scratch.</div><div class='add'>+ * For each call, it looks for a matching CQP request on pending list,</div><div class='add'>+ * removes it from the list and returns the pointer to the associated scratch</div><div class='add'>+ * entry.</div><div class='add'>+ * If this is the first call to this function for given AEQE, sw_def_info</div><div class='add'>+ * value is not used to find matching requests.  Instead, it is populated</div><div class='add'>+ * with the value from the first matching cqp_request on the list.</div><div class='add'>+ * For subsequent calls, ooo_op-&gt;sw_def_info need to match the value passed</div><div class='add'>+ * by a caller.</div><div class='add'>+ *</div><div class='add'>+ * Return: scratch entry pointer for cqp_request to be released or NULL</div><div class='add'>+ * if no matching request is found.</div><div class='add'>+ */</div><div class='add'>+void irdma_sc_cqp_def_cmpl_ae_handler(struct irdma_sc_dev *dev,</div><div class='add'>+				      struct irdma_aeqe_info *info,</div><div class='add'>+				      bool first, u64 *scratch,</div><div class='add'>+				      u32 *sw_def_info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ooo_cqp_op *ooo_op;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	*scratch = 0;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;dev-&gt;cqp-&gt;ooo_list_lock, flags);</div><div class='add'>+	list_for_each_entry(ooo_op, &amp;dev-&gt;cqp-&gt;ooo_pnd, list_entry) {</div><div class='add'>+		if (ooo_op-&gt;deferred &amp;&amp;</div><div class='add'>+		    ((first &amp;&amp; ooo_op-&gt;def_info == info-&gt;def_info) ||</div><div class='add'>+		     (!first &amp;&amp; ooo_op-&gt;sw_def_info == *sw_def_info))) {</div><div class='add'>+			*sw_def_info = ooo_op-&gt;sw_def_info;</div><div class='add'>+			*scratch = ooo_op-&gt;scratch;</div><div class='add'>+</div><div class='add'>+			list_move(&amp;ooo_op-&gt;list_entry, &amp;dev-&gt;cqp-&gt;ooo_avail);</div><div class='add'>+			atomic64_inc(&amp;dev-&gt;cqp-&gt;completed_ops);</div><div class='add'>+</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;dev-&gt;cqp-&gt;ooo_list_lock, flags);</div><div class='add'>+</div><div class='add'>+	if (first &amp;&amp; !*scratch)</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "AEQ: deferred completion with unknown ticket: def_info 0x%x\n",</div><div class='add'>+			   info-&gt;def_info);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cqp_cleanup_handler - remove requests from pending list</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ *</div><div class='add'>+ * This function should be called in a loop from irdma_cleanup_pending_cqp_op.</div><div class='add'>+ * For each call, it returns first CQP request on pending list, removes it</div><div class='add'>+ * from the list and returns the pointer to the associated scratch entry.</div><div class='add'>+ *</div><div class='add'>+ * Return: scratch entry pointer for cqp_request to be released or NULL</div><div class='add'>+ * if pending list is empty.</div><div class='add'>+ */</div><div class='add'>+u64 irdma_sc_cqp_cleanup_handler(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ooo_cqp_op *ooo_op;</div><div class='add'>+	u64 scratch = 0;</div><div class='add'>+</div><div class='add'>+	list_for_each_entry(ooo_op, &amp;dev-&gt;cqp-&gt;ooo_pnd, list_entry) {</div><div class='add'>+		scratch = ooo_op-&gt;scratch;</div><div class='add'>+</div><div class='add'>+		list_del(&amp;ooo_op-&gt;list_entry);</div><div class='add'>+		list_add(&amp;ooo_op-&gt;list_entry, &amp;dev-&gt;cqp-&gt;ooo_avail);</div><div class='add'>+		atomic64_inc(&amp;dev-&gt;cqp-&gt;completed_ops);</div><div class='add'>+</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return scratch;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_poll_registers - poll cqp registers</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @tail: wqtail register value</div><div class='add'>+ * @count: how many times to try for completion</div><div class='add'>+ */</div><div class='add'>+static int irdma_cqp_poll_registers(struct irdma_sc_cqp *cqp, u32 tail,</div><div class='add'>+				    u32 count)</div><div class='add'>+{</div><div class='add'>+	u32 i = 0;</div><div class='add'>+	u32 newtail, error, val;</div><div class='add'>+</div><div class='add'>+	while (i++ &lt; count) {</div><div class='add'>+		irdma_get_cqp_reg_info(cqp, &amp;val, &amp;newtail, &amp;error);</div><div class='add'>+		if (error) {</div><div class='add'>+			error = readl(cqp-&gt;dev-&gt;hw_regs[IRDMA_CQPERRCODES]);</div><div class='add'>+			ibdev_dbg(to_ibdev(cqp-&gt;dev),</div><div class='add'>+				  "CQP: CQPERRCODES error_code[x%08X]\n",</div><div class='add'>+				  error);</div><div class='add'>+			return -EIO;</div><div class='add'>+		}</div><div class='add'>+		if (newtail != tail) {</div><div class='add'>+			/* SUCCESS */</div><div class='add'>+			IRDMA_RING_MOVE_TAIL(cqp-&gt;sq_ring);</div><div class='add'>+			atomic64_inc(&amp;cqp-&gt;completed_ops);</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+		udelay(cqp-&gt;dev-&gt;hw_attrs.max_sleep_count);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return -ETIMEDOUT;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_decode_fpm_commit - decode a 64 bit value into count and base</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @buf: pointer to commit buffer</div><div class='add'>+ * @buf_idx: buffer index</div><div class='add'>+ * @obj_info: object info pointer</div><div class='add'>+ * @rsrc_idx: indexs of memory resource</div><div class='add'>+ */</div><div class='add'>+static u64 irdma_sc_decode_fpm_commit(struct irdma_sc_dev *dev, __le64 *buf,</div><div class='add'>+				      u32 buf_idx, struct irdma_hmc_obj_info *obj_info,</div><div class='add'>+				      u32 rsrc_idx)</div><div class='add'>+{</div><div class='add'>+	u64 temp;</div><div class='add'>+</div><div class='add'>+	get_64bit_val(buf, buf_idx, &amp;temp);</div><div class='add'>+</div><div class='add'>+	switch (rsrc_idx) {</div><div class='add'>+	case IRDMA_HMC_IW_QP:</div><div class='add'>+		obj_info[rsrc_idx].cnt = (u32)FIELD_GET(IRDMA_COMMIT_FPM_QPCNT, temp);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_HMC_IW_CQ:</div><div class='add'>+		obj_info[rsrc_idx].cnt = (u32)FLD_RS_64(dev, temp, IRDMA_COMMIT_FPM_CQCNT);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_HMC_IW_APBVT_ENTRY:</div><div class='add'>+		if (dev-&gt;hw_attrs.uk_attrs.hw_rev &lt;= IRDMA_GEN_2)</div><div class='add'>+			obj_info[rsrc_idx].cnt = 1;</div><div class='add'>+		else</div><div class='add'>+			obj_info[rsrc_idx].cnt = 0;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		obj_info[rsrc_idx].cnt = (u32)temp;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	obj_info[rsrc_idx].base = (temp &gt;&gt; IRDMA_COMMIT_FPM_BASE_S) * 512;</div><div class='add'>+</div><div class='add'>+	return temp;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_parse_fpm_commit_buf - parse fpm commit buffer</div><div class='add'>+ * @dev: pointer to dev struct</div><div class='add'>+ * @buf: ptr to fpm commit buffer</div><div class='add'>+ * @info: ptr to irdma_hmc_obj_info struct</div><div class='add'>+ * @sd: number of SDs for HMC objects</div><div class='add'>+ *</div><div class='add'>+ * parses fpm commit info and copy base value</div><div class='add'>+ * of hmc objects in hmc_info</div><div class='add'>+ */</div><div class='add'>+static void</div><div class='add'>+irdma_sc_parse_fpm_commit_buf(struct irdma_sc_dev *dev, __le64 *buf,</div><div class='add'>+			      struct irdma_hmc_obj_info *info, u32 *sd)</div><div class='add'>+{</div><div class='add'>+	u64 size;</div><div class='add'>+	u32 i;</div><div class='add'>+	u64 max_base = 0;</div><div class='add'>+	u32 last_hmc_obj = 0;</div><div class='add'>+</div><div class='add'>+	irdma_sc_decode_fpm_commit(dev, buf, 0, info,</div><div class='add'>+				   IRDMA_HMC_IW_QP);</div><div class='add'>+	irdma_sc_decode_fpm_commit(dev, buf, 8, info,</div><div class='add'>+				   IRDMA_HMC_IW_CQ);</div><div class='add'>+	irdma_sc_decode_fpm_commit(dev, buf, 16, info,</div><div class='add'>+				   IRDMA_HMC_IW_SRQ);</div><div class='add'>+	irdma_sc_decode_fpm_commit(dev, buf, 24, info,</div><div class='add'>+				   IRDMA_HMC_IW_HTE);</div><div class='add'>+	irdma_sc_decode_fpm_commit(dev, buf, 32, info,</div><div class='add'>+				   IRDMA_HMC_IW_ARP);</div><div class='add'>+	irdma_sc_decode_fpm_commit(dev, buf, 40, info,</div><div class='add'>+				   IRDMA_HMC_IW_APBVT_ENTRY);</div><div class='add'>+	irdma_sc_decode_fpm_commit(dev, buf, 48, info,</div><div class='add'>+				   IRDMA_HMC_IW_MR);</div><div class='add'>+	irdma_sc_decode_fpm_commit(dev, buf, 56, info,</div><div class='add'>+				   IRDMA_HMC_IW_XF);</div><div class='add'>+	irdma_sc_decode_fpm_commit(dev, buf, 64, info,</div><div class='add'>+				   IRDMA_HMC_IW_XFFL);</div><div class='add'>+	irdma_sc_decode_fpm_commit(dev, buf, 72, info,</div><div class='add'>+				   IRDMA_HMC_IW_Q1);</div><div class='add'>+	irdma_sc_decode_fpm_commit(dev, buf, 80, info,</div><div class='add'>+				   IRDMA_HMC_IW_Q1FL);</div><div class='add'>+	irdma_sc_decode_fpm_commit(dev, buf, 88, info,</div><div class='add'>+				   IRDMA_HMC_IW_TIMER);</div><div class='add'>+	irdma_sc_decode_fpm_commit(dev, buf, 112, info,</div><div class='add'>+				   IRDMA_HMC_IW_PBLE);</div><div class='add'>+	/* skipping RSVD. */</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {</div><div class='add'>+		irdma_sc_decode_fpm_commit(dev, buf, 96, info,</div><div class='add'>+					   IRDMA_HMC_IW_FSIMC);</div><div class='add'>+		irdma_sc_decode_fpm_commit(dev, buf, 104, info,</div><div class='add'>+					   IRDMA_HMC_IW_FSIAV);</div><div class='add'>+		irdma_sc_decode_fpm_commit(dev, buf, 128, info,</div><div class='add'>+					   IRDMA_HMC_IW_RRF);</div><div class='add'>+		irdma_sc_decode_fpm_commit(dev, buf, 136, info,</div><div class='add'>+					   IRDMA_HMC_IW_RRFFL);</div><div class='add'>+		irdma_sc_decode_fpm_commit(dev, buf, 144, info,</div><div class='add'>+					   IRDMA_HMC_IW_HDR);</div><div class='add'>+		irdma_sc_decode_fpm_commit(dev, buf, 152, info,</div><div class='add'>+					   IRDMA_HMC_IW_MD);</div><div class='add'>+		if (dev-&gt;cqp-&gt;protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) {</div><div class='add'>+			irdma_sc_decode_fpm_commit(dev, buf, 160, info,</div><div class='add'>+						   IRDMA_HMC_IW_OOISC);</div><div class='add'>+			irdma_sc_decode_fpm_commit(dev, buf, 168, info,</div><div class='add'>+						   IRDMA_HMC_IW_OOISCFFL);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* searching for the last object in HMC to find the size of the HMC area. */</div><div class='add'>+	for (i = IRDMA_HMC_IW_QP; i &lt; IRDMA_HMC_IW_MAX; i++) {</div><div class='add'>+		if (info[i].base &gt; max_base &amp;&amp; info[i].cnt) {</div><div class='add'>+			max_base = info[i].base;</div><div class='add'>+			last_hmc_obj = i;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	size = info[last_hmc_obj].cnt * info[last_hmc_obj].size +</div><div class='add'>+	       info[last_hmc_obj].base;</div><div class='add'>+</div><div class='add'>+	if (size &amp; 0x1FFFFF)</div><div class='add'>+		*sd = (u32)((size &gt;&gt; 21) + 1); /* add 1 for remainder */</div><div class='add'>+	else</div><div class='add'>+		*sd = (u32)(size &gt;&gt; 21);</div><div class='add'>+</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_decode_fpm_query() - Decode a 64 bit value into max count and size</div><div class='add'>+ * @buf: ptr to fpm query buffer</div><div class='add'>+ * @buf_idx: index into buf</div><div class='add'>+ * @obj_info: ptr to irdma_hmc_obj_info struct</div><div class='add'>+ * @rsrc_idx: resource index into info</div><div class='add'>+ *</div><div class='add'>+ * Decode a 64 bit value from fpm query buffer into max count and size</div><div class='add'>+ */</div><div class='add'>+static u64 irdma_sc_decode_fpm_query(__le64 *buf, u32 buf_idx,</div><div class='add'>+				     struct irdma_hmc_obj_info *obj_info,</div><div class='add'>+				     u32 rsrc_idx)</div><div class='add'>+{</div><div class='add'>+	u64 temp;</div><div class='add'>+	u32 size;</div><div class='add'>+</div><div class='add'>+	get_64bit_val(buf, buf_idx, &amp;temp);</div><div class='add'>+	obj_info[rsrc_idx].max_cnt = (u32)temp;</div><div class='add'>+	size = (u32)(temp &gt;&gt; 32);</div><div class='add'>+	obj_info[rsrc_idx].size = BIT_ULL(size);</div><div class='add'>+</div><div class='add'>+	return temp;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_parse_fpm_query_buf() - parses fpm query buffer</div><div class='add'>+ * @dev: ptr to shared code device</div><div class='add'>+ * @buf: ptr to fpm query buffer</div><div class='add'>+ * @hmc_info: ptr to irdma_hmc_obj_info struct</div><div class='add'>+ * @hmc_fpm_misc: ptr to fpm data</div><div class='add'>+ *</div><div class='add'>+ * parses fpm query buffer and copy max_cnt and</div><div class='add'>+ * size value of hmc objects in hmc_info</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_parse_fpm_query_buf(struct irdma_sc_dev *dev, __le64 *buf,</div><div class='add'>+					struct irdma_hmc_info *hmc_info,</div><div class='add'>+					struct irdma_hmc_fpm_misc *hmc_fpm_misc)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_obj_info *obj_info;</div><div class='add'>+	u8 ird_encoding;</div><div class='add'>+	u64 temp;</div><div class='add'>+	u32 size;</div><div class='add'>+	u16 max_pe_sds;</div><div class='add'>+</div><div class='add'>+	obj_info = hmc_info-&gt;hmc_obj;</div><div class='add'>+</div><div class='add'>+	get_64bit_val(buf, 0, &amp;temp);</div><div class='add'>+	hmc_info-&gt;first_sd_index = (u16)FIELD_GET(IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX, temp);</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3)</div><div class='add'>+		max_pe_sds = (u16)FIELD_GET(IRDMA_QUERY_FPM_MAX_PE_SDS_GEN3, temp);</div><div class='add'>+	else</div><div class='add'>+		max_pe_sds = (u16)FIELD_GET(IRDMA_QUERY_FPM_MAX_PE_SDS, temp);</div><div class='add'>+</div><div class='add'>+	/* Reduce SD count for unprivleged functions by 1 to account for PBLE</div><div class='add'>+	 * backing page rounding</div><div class='add'>+	 */</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &lt;= IRDMA_GEN_2 &amp;&amp;</div><div class='add'>+	    (hmc_info-&gt;hmc_fn_id &gt;= dev-&gt;hw_attrs.first_hw_vf_fpm_id ||</div><div class='add'>+	    !dev-&gt;privileged))</div><div class='add'>+		max_pe_sds--;</div><div class='add'>+</div><div class='add'>+	hmc_fpm_misc-&gt;max_sds = max_pe_sds;</div><div class='add'>+	hmc_info-&gt;sd_table.sd_cnt = max_pe_sds + hmc_info-&gt;first_sd_index;</div><div class='add'>+	get_64bit_val(buf, 8, &amp;temp);</div><div class='add'>+	obj_info[IRDMA_HMC_IW_QP].max_cnt = (u32)FIELD_GET(IRDMA_QUERY_FPM_MAX_QPS, temp);</div><div class='add'>+	size = (u32)(temp &gt;&gt; 32);</div><div class='add'>+	obj_info[IRDMA_HMC_IW_QP].size = BIT_ULL(size);</div><div class='add'>+</div><div class='add'>+	get_64bit_val(buf, 16, &amp;temp);</div><div class='add'>+	obj_info[IRDMA_HMC_IW_CQ].max_cnt = (u32)FIELD_GET(IRDMA_QUERY_FPM_MAX_CQS, temp);</div><div class='add'>+	size = (u32)(temp &gt;&gt; 32);</div><div class='add'>+	obj_info[IRDMA_HMC_IW_CQ].size = BIT_ULL(size);</div><div class='add'>+</div><div class='add'>+	irdma_sc_decode_fpm_query(buf, 24, obj_info, IRDMA_HMC_IW_SRQ);</div><div class='add'>+	irdma_sc_decode_fpm_query(buf, 32, obj_info, IRDMA_HMC_IW_HTE);</div><div class='add'>+	irdma_sc_decode_fpm_query(buf, 40, obj_info, IRDMA_HMC_IW_ARP);</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3) {</div><div class='add'>+		obj_info[IRDMA_HMC_IW_APBVT_ENTRY].size = 0;</div><div class='add'>+		obj_info[IRDMA_HMC_IW_APBVT_ENTRY].max_cnt = 0;</div><div class='add'>+	} else {</div><div class='add'>+		obj_info[IRDMA_HMC_IW_APBVT_ENTRY].size = 8192;</div><div class='add'>+		obj_info[IRDMA_HMC_IW_APBVT_ENTRY].max_cnt = 1;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_sc_decode_fpm_query(buf, 48, obj_info, IRDMA_HMC_IW_MR);</div><div class='add'>+	irdma_sc_decode_fpm_query(buf, 56, obj_info, IRDMA_HMC_IW_XF);</div><div class='add'>+</div><div class='add'>+	get_64bit_val(buf, 64, &amp;temp);</div><div class='add'>+	obj_info[IRDMA_HMC_IW_XFFL].max_cnt = (u32)temp;</div><div class='add'>+	obj_info[IRDMA_HMC_IW_XFFL].size = 4;</div><div class='add'>+	hmc_fpm_misc-&gt;xf_block_size = FIELD_GET(IRDMA_QUERY_FPM_XFBLOCKSIZE, temp);</div><div class='add'>+	if (obj_info[IRDMA_HMC_IW_XF].max_cnt &amp;&amp; !hmc_fpm_misc-&gt;xf_block_size)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	irdma_sc_decode_fpm_query(buf, 72, obj_info, IRDMA_HMC_IW_Q1);</div><div class='add'>+	get_64bit_val(buf, 80, &amp;temp);</div><div class='add'>+	obj_info[IRDMA_HMC_IW_Q1FL].max_cnt = (u32)temp;</div><div class='add'>+	obj_info[IRDMA_HMC_IW_Q1FL].size = 4;</div><div class='add'>+</div><div class='add'>+	hmc_fpm_misc-&gt;q1_block_size = FIELD_GET(IRDMA_QUERY_FPM_Q1BLOCKSIZE, temp);</div><div class='add'>+	if (!hmc_fpm_misc-&gt;q1_block_size)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	irdma_sc_decode_fpm_query(buf, 88, obj_info, IRDMA_HMC_IW_TIMER);</div><div class='add'>+</div><div class='add'>+	get_64bit_val(buf, 112, &amp;temp);</div><div class='add'>+	obj_info[IRDMA_HMC_IW_PBLE].max_cnt = (u32)temp;</div><div class='add'>+	obj_info[IRDMA_HMC_IW_PBLE].size = 8;</div><div class='add'>+</div><div class='add'>+	get_64bit_val(buf, 120, &amp;temp);</div><div class='add'>+	hmc_fpm_misc-&gt;max_ceqs = FIELD_GET(IRDMA_QUERY_FPM_MAX_CEQS, temp);</div><div class='add'>+	hmc_fpm_misc-&gt;ht_multiplier = FIELD_GET(IRDMA_QUERY_FPM_HTMULTIPLIER, temp);</div><div class='add'>+	hmc_fpm_misc-&gt;timer_bucket = FIELD_GET(IRDMA_QUERY_FPM_TIMERBUCKET, temp);</div><div class='add'>+	if (FIELD_GET(IRDMA_MANAGE_RSRC_VER2,</div><div class='add'>+		      dev-&gt;feature_info[IRDMA_FTN_FLAGS])) {</div><div class='add'>+		ird_encoding = (u8)FIELD_GET(IRDMA_QUERY_FPM_MAX_IRD, temp);</div><div class='add'>+		hmc_fpm_misc-&gt;ird =</div><div class='add'>+			irdma_sc_get_decoded_ird_size_gen_3(ird_encoding) / 2;</div><div class='add'>+		dev-&gt;hw_attrs.max_hw_ird = hmc_fpm_misc-&gt;ird;</div><div class='add'>+		dev-&gt;hw_attrs.max_hw_ord = hmc_fpm_misc-&gt;ird;</div><div class='add'>+	}</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)</div><div class='add'>+		return 0;</div><div class='add'>+	irdma_sc_decode_fpm_query(buf, 96, obj_info, IRDMA_HMC_IW_FSIMC);</div><div class='add'>+	irdma_sc_decode_fpm_query(buf, 104, obj_info, IRDMA_HMC_IW_FSIAV);</div><div class='add'>+	irdma_sc_decode_fpm_query(buf, 128, obj_info, IRDMA_HMC_IW_RRF);</div><div class='add'>+</div><div class='add'>+	get_64bit_val(buf, 136, &amp;temp);</div><div class='add'>+	obj_info[IRDMA_HMC_IW_RRFFL].max_cnt = (u32)temp;</div><div class='add'>+	obj_info[IRDMA_HMC_IW_RRFFL].size = 4;</div><div class='add'>+	hmc_fpm_misc-&gt;rrf_block_size = FIELD_GET(IRDMA_QUERY_FPM_RRFBLOCKSIZE, temp);</div><div class='add'>+	if (!hmc_fpm_misc-&gt;rrf_block_size &amp;&amp;</div><div class='add'>+	    obj_info[IRDMA_HMC_IW_RRFFL].max_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	irdma_sc_decode_fpm_query(buf, 144, obj_info, IRDMA_HMC_IW_HDR);</div><div class='add'>+	irdma_sc_decode_fpm_query(buf, 152, obj_info, IRDMA_HMC_IW_MD);</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;cqp-&gt;protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) {</div><div class='add'>+		irdma_sc_decode_fpm_query(buf, 160, obj_info, IRDMA_HMC_IW_OOISC);</div><div class='add'>+</div><div class='add'>+		get_64bit_val(buf, 168, &amp;temp);</div><div class='add'>+		obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp;</div><div class='add'>+		obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4;</div><div class='add'>+		hmc_fpm_misc-&gt;ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp);</div><div class='add'>+		if (!hmc_fpm_misc-&gt;ooiscf_block_size &amp;&amp;</div><div class='add'>+		    obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3) {</div><div class='add'>+		get_64bit_val(buf, 176, &amp;temp);</div><div class='add'>+		hmc_fpm_misc-&gt;loc_mem_pages = (u32)FIELD_GET(IRDMA_QUERY_FPM_LOC_MEM_PAGES, temp);</div><div class='add'>+		if (!hmc_fpm_misc-&gt;loc_mem_pages)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cqp_init - Initialize buffers for a control Queue Pair</div><div class='add'>+ * @cqp: IWARP control queue pair pointer</div><div class='add'>+ * @info: IWARP control queue pair init info pointer</div><div class='add'>+ *</div><div class='add'>+ * Initializes the object and context buffers for a control Queue Pair.</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp,</div><div class='add'>+		      struct irdma_cqp_init_info *info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ooo_cqp_op *ooo_op;</div><div class='add'>+	u32 num_ooo_ops;</div><div class='add'>+	u8 hw_sq_size;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;sq_size &gt; IRDMA_CQP_SW_SQSIZE_2048 ||</div><div class='add'>+	    info-&gt;sq_size &lt; IRDMA_CQP_SW_SQSIZE_4 ||</div><div class='add'>+	    ((info-&gt;sq_size &amp; (info-&gt;sq_size - 1))))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	hw_sq_size = irdma_get_encoded_wqe_size(info-&gt;sq_size,</div><div class='add'>+						IRDMA_QUEUE_TYPE_CQP);</div><div class='add'>+	cqp-&gt;size = sizeof(*cqp);</div><div class='add'>+	cqp-&gt;sq_size = info-&gt;sq_size;</div><div class='add'>+	cqp-&gt;hw_sq_size = hw_sq_size;</div><div class='add'>+	cqp-&gt;sq_base = info-&gt;sq;</div><div class='add'>+	cqp-&gt;host_ctx = info-&gt;host_ctx;</div><div class='add'>+	cqp-&gt;sq_pa = info-&gt;sq_pa;</div><div class='add'>+	cqp-&gt;host_ctx_pa = info-&gt;host_ctx_pa;</div><div class='add'>+	cqp-&gt;dev = info-&gt;dev;</div><div class='add'>+	cqp-&gt;struct_ver = info-&gt;struct_ver;</div><div class='add'>+	cqp-&gt;hw_maj_ver = info-&gt;hw_maj_ver;</div><div class='add'>+	cqp-&gt;hw_min_ver = info-&gt;hw_min_ver;</div><div class='add'>+	cqp-&gt;scratch_array = info-&gt;scratch_array;</div><div class='add'>+	cqp-&gt;polarity = 0;</div><div class='add'>+	cqp-&gt;en_datacenter_tcp = info-&gt;en_datacenter_tcp;</div><div class='add'>+	cqp-&gt;ena_vf_count = info-&gt;ena_vf_count;</div><div class='add'>+	cqp-&gt;hmc_profile = info-&gt;hmc_profile;</div><div class='add'>+	cqp-&gt;ceqs_per_vf = info-&gt;ceqs_per_vf;</div><div class='add'>+	cqp-&gt;disable_packed = info-&gt;disable_packed;</div><div class='add'>+	cqp-&gt;rocev2_rto_policy = info-&gt;rocev2_rto_policy;</div><div class='add'>+	cqp-&gt;protocol_used = info-&gt;protocol_used;</div><div class='add'>+	memcpy(&amp;cqp-&gt;dcqcn_params, &amp;info-&gt;dcqcn_params, sizeof(cqp-&gt;dcqcn_params));</div><div class='add'>+	if (cqp-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3) {</div><div class='add'>+		cqp-&gt;ooisc_blksize = info-&gt;ooisc_blksize;</div><div class='add'>+		cqp-&gt;rrsp_blksize = info-&gt;rrsp_blksize;</div><div class='add'>+		cqp-&gt;q1_blksize = info-&gt;q1_blksize;</div><div class='add'>+		cqp-&gt;xmit_blksize = info-&gt;xmit_blksize;</div><div class='add'>+		cqp-&gt;blksizes_valid = info-&gt;blksizes_valid;</div><div class='add'>+		cqp-&gt;ts_shift = info-&gt;ts_shift;</div><div class='add'>+		cqp-&gt;ts_override = info-&gt;ts_override;</div><div class='add'>+		cqp-&gt;en_fine_grained_timers = info-&gt;en_fine_grained_timers;</div><div class='add'>+		cqp-&gt;pe_en_vf_cnt = info-&gt;pe_en_vf_cnt;</div><div class='add'>+		cqp-&gt;ooo_op_array = info-&gt;ooo_op_array;</div><div class='add'>+		/* initialize the OOO lists */</div><div class='add'>+		INIT_LIST_HEAD(&amp;cqp-&gt;ooo_avail);</div><div class='add'>+		INIT_LIST_HEAD(&amp;cqp-&gt;ooo_pnd);</div><div class='add'>+		if (cqp-&gt;ooo_op_array) {</div><div class='add'>+			/* Populate avail list entries */</div><div class='add'>+			for (num_ooo_ops = 0, ooo_op = info-&gt;ooo_op_array;</div><div class='add'>+			     num_ooo_ops &lt; cqp-&gt;sq_size;</div><div class='add'>+			     num_ooo_ops++, ooo_op++)</div><div class='add'>+				list_add(&amp;ooo_op-&gt;list_entry, &amp;cqp-&gt;ooo_avail);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	info-&gt;dev-&gt;cqp = cqp;</div><div class='add'>+</div><div class='add'>+	IRDMA_RING_INIT(cqp-&gt;sq_ring, cqp-&gt;sq_size);</div><div class='add'>+	cqp-&gt;last_def_cmpl_ticket = 0;</div><div class='add'>+	cqp-&gt;sw_def_cmpl_ticket = 0;</div><div class='add'>+	cqp-&gt;requested_ops = 0;</div><div class='add'>+	atomic64_set(&amp;cqp-&gt;completed_ops, 0);</div><div class='add'>+	/* for the cqp commands backlog. */</div><div class='add'>+	INIT_LIST_HEAD(&amp;cqp-&gt;dev-&gt;cqp_cmd_head);</div><div class='add'>+</div><div class='add'>+	writel(0, cqp-&gt;dev-&gt;hw_regs[IRDMA_CQPTAIL]);</div><div class='add'>+	if (cqp-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &lt;= IRDMA_GEN_2) {</div><div class='add'>+		writel(0, cqp-&gt;dev-&gt;hw_regs[IRDMA_CQPDB]);</div><div class='add'>+		writel(0, cqp-&gt;dev-&gt;hw_regs[IRDMA_CCQPSTATUS]);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(to_ibdev(cqp-&gt;dev),</div><div class='add'>+		  "WQE: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%p] cqp[%p] polarity[x%04x]\n",</div><div class='add'>+		  cqp-&gt;sq_size, cqp-&gt;hw_sq_size, cqp-&gt;sq_base,</div><div class='add'>+		  (u64 *)(uintptr_t)cqp-&gt;sq_pa, cqp, cqp-&gt;polarity);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cqp_create - create cqp during bringup</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @maj_err: If error, major err number</div><div class='add'>+ * @min_err: If error, minor err number</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err)</div><div class='add'>+{</div><div class='add'>+	u64 temp;</div><div class='add'>+	u8 hw_rev;</div><div class='add'>+	u32 cnt = 0, p1, p2, val = 0, err_code;</div><div class='add'>+	int ret_code;</div><div class='add'>+</div><div class='add'>+	hw_rev = cqp-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev;</div><div class='add'>+	cqp-&gt;sdbuf.size = ALIGN(IRDMA_UPDATE_SD_BUFF_SIZE * cqp-&gt;sq_size,</div><div class='add'>+				IRDMA_SD_BUF_ALIGNMENT);</div><div class='add'>+	cqp-&gt;sdbuf.va = dma_alloc_coherent(cqp-&gt;dev-&gt;hw-&gt;device,</div><div class='add'>+					   cqp-&gt;sdbuf.size, &amp;cqp-&gt;sdbuf.pa,</div><div class='add'>+					   GFP_KERNEL);</div><div class='add'>+	if (!cqp-&gt;sdbuf.va)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;cqp-&gt;dev-&gt;cqp_lock);</div><div class='add'>+	spin_lock_init(&amp;cqp-&gt;ooo_list_lock);</div><div class='add'>+</div><div class='add'>+	temp = FIELD_PREP(IRDMA_CQPHC_SQSIZE, cqp-&gt;hw_sq_size) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPHC_SVER, cqp-&gt;struct_ver) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPHC_DISABLE_PFPDUS, cqp-&gt;disable_packed) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPHC_CEQPERVF, cqp-&gt;ceqs_per_vf);</div><div class='add'>+	if (hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+		temp |= FIELD_PREP(IRDMA_CQPHC_ROCEV2_RTO_POLICY,</div><div class='add'>+				   cqp-&gt;rocev2_rto_policy) |</div><div class='add'>+			FIELD_PREP(IRDMA_CQPHC_PROTOCOL_USED,</div><div class='add'>+				   cqp-&gt;protocol_used);</div><div class='add'>+	}</div><div class='add'>+	if (hw_rev &gt;= IRDMA_GEN_3)</div><div class='add'>+		temp |= FIELD_PREP(IRDMA_CQPHC_EN_FINE_GRAINED_TIMERS,</div><div class='add'>+				   cqp-&gt;en_fine_grained_timers);</div><div class='add'>+</div><div class='add'>+	set_64bit_val(cqp-&gt;host_ctx, 0, temp);</div><div class='add'>+	set_64bit_val(cqp-&gt;host_ctx, 8, cqp-&gt;sq_pa);</div><div class='add'>+</div><div class='add'>+	temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp-&gt;ena_vf_count) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp-&gt;hmc_profile);</div><div class='add'>+</div><div class='add'>+	if (hw_rev &gt;= IRDMA_GEN_3)</div><div class='add'>+		temp |= FIELD_PREP(IRDMA_CQPHC_OOISC_BLKSIZE,</div><div class='add'>+				   cqp-&gt;ooisc_blksize) |</div><div class='add'>+			FIELD_PREP(IRDMA_CQPHC_RRSP_BLKSIZE,</div><div class='add'>+				   cqp-&gt;rrsp_blksize) |</div><div class='add'>+			FIELD_PREP(IRDMA_CQPHC_Q1_BLKSIZE, cqp-&gt;q1_blksize) |</div><div class='add'>+			FIELD_PREP(IRDMA_CQPHC_XMIT_BLKSIZE,</div><div class='add'>+				   cqp-&gt;xmit_blksize) |</div><div class='add'>+			FIELD_PREP(IRDMA_CQPHC_BLKSIZES_VALID,</div><div class='add'>+				   cqp-&gt;blksizes_valid) |</div><div class='add'>+			FIELD_PREP(IRDMA_CQPHC_TIMESTAMP_OVERRIDE,</div><div class='add'>+				   cqp-&gt;ts_override) |</div><div class='add'>+			FIELD_PREP(IRDMA_CQPHC_TS_SHIFT, cqp-&gt;ts_shift);</div><div class='add'>+	set_64bit_val(cqp-&gt;host_ctx, 16, temp);</div><div class='add'>+	set_64bit_val(cqp-&gt;host_ctx, 24, (uintptr_t)cqp);</div><div class='add'>+	temp = FIELD_PREP(IRDMA_CQPHC_HW_MAJVER, cqp-&gt;hw_maj_ver) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPHC_HW_MINVER, cqp-&gt;hw_min_ver);</div><div class='add'>+	if (hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+		temp |= FIELD_PREP(IRDMA_CQPHC_MIN_RATE, cqp-&gt;dcqcn_params.min_rate) |</div><div class='add'>+			FIELD_PREP(IRDMA_CQPHC_MIN_DEC_FACTOR, cqp-&gt;dcqcn_params.min_dec_factor);</div><div class='add'>+	}</div><div class='add'>+	set_64bit_val(cqp-&gt;host_ctx, 32, temp);</div><div class='add'>+	set_64bit_val(cqp-&gt;host_ctx, 40, 0);</div><div class='add'>+	temp = 0;</div><div class='add'>+	if (hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+		temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_T, cqp-&gt;dcqcn_params.dcqcn_t) |</div><div class='add'>+			FIELD_PREP(IRDMA_CQPHC_RAI_FACTOR, cqp-&gt;dcqcn_params.rai_factor) |</div><div class='add'>+			FIELD_PREP(IRDMA_CQPHC_HAI_FACTOR, cqp-&gt;dcqcn_params.hai_factor);</div><div class='add'>+	}</div><div class='add'>+	set_64bit_val(cqp-&gt;host_ctx, 48, temp);</div><div class='add'>+	temp = 0;</div><div class='add'>+	if (hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+		temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_B, cqp-&gt;dcqcn_params.dcqcn_b) |</div><div class='add'>+			FIELD_PREP(IRDMA_CQPHC_DCQCN_F, cqp-&gt;dcqcn_params.dcqcn_f) |</div><div class='add'>+			FIELD_PREP(IRDMA_CQPHC_CC_CFG_VALID, cqp-&gt;dcqcn_params.cc_cfg_valid) |</div><div class='add'>+			FIELD_PREP(IRDMA_CQPHC_RREDUCE_MPERIOD, cqp-&gt;dcqcn_params.rreduce_mperiod);</div><div class='add'>+	}</div><div class='add'>+	set_64bit_val(cqp-&gt;host_ctx, 56, temp);</div><div class='add'>+	print_hex_dump_debug("WQE: CQP_HOST_CTX WQE", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, cqp-&gt;host_ctx, IRDMA_CQP_CTX_SIZE * 8, false);</div><div class='add'>+	p1 = cqp-&gt;host_ctx_pa &gt;&gt; 32;</div><div class='add'>+	p2 = (u32)cqp-&gt;host_ctx_pa;</div><div class='add'>+</div><div class='add'>+	writel(p1, cqp-&gt;dev-&gt;hw_regs[IRDMA_CCQPHIGH]);</div><div class='add'>+	writel(p2, cqp-&gt;dev-&gt;hw_regs[IRDMA_CCQPLOW]);</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		if (cnt++ &gt; cqp-&gt;dev-&gt;hw_attrs.max_done_count) {</div><div class='add'>+			ret_code = -ETIMEDOUT;</div><div class='add'>+			goto err;</div><div class='add'>+		}</div><div class='add'>+		udelay(cqp-&gt;dev-&gt;hw_attrs.max_sleep_count);</div><div class='add'>+		val = readl(cqp-&gt;dev-&gt;hw_regs[IRDMA_CCQPSTATUS]);</div><div class='add'>+	} while (!val);</div><div class='add'>+</div><div class='add'>+	if (FLD_RS_32(cqp-&gt;dev, val, IRDMA_CCQPSTATUS_CCQP_ERR)) {</div><div class='add'>+		ret_code = -EOPNOTSUPP;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cqp-&gt;process_cqp_sds = irdma_update_sds_noccq;</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	dma_free_coherent(cqp-&gt;dev-&gt;hw-&gt;device, cqp-&gt;sdbuf.size,</div><div class='add'>+			  cqp-&gt;sdbuf.va, cqp-&gt;sdbuf.pa);</div><div class='add'>+	cqp-&gt;sdbuf.va = NULL;</div><div class='add'>+	err_code = readl(cqp-&gt;dev-&gt;hw_regs[IRDMA_CQPERRCODES]);</div><div class='add'>+	*min_err = FIELD_GET(IRDMA_CQPERRCODES_CQP_MINOR_CODE, err_code);</div><div class='add'>+	*maj_err = FIELD_GET(IRDMA_CQPERRCODES_CQP_MAJOR_CODE, err_code);</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cqp_post_sq - post of cqp's sq</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ */</div><div class='add'>+void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp)</div><div class='add'>+{</div><div class='add'>+	writel(IRDMA_RING_CURRENT_HEAD(cqp-&gt;sq_ring), cqp-&gt;dev-&gt;cqp_db);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(to_ibdev(cqp-&gt;dev),</div><div class='add'>+		  "WQE: CQP SQ head 0x%x tail 0x%x size 0x%x\n",</div><div class='add'>+		  cqp-&gt;sq_ring.head, cqp-&gt;sq_ring.tail, cqp-&gt;sq_ring.size);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cqp_get_next_send_wqe_idx - get next wqe on cqp sq</div><div class='add'>+ * and pass back index</div><div class='add'>+ * @cqp: CQP HW structure</div><div class='add'>+ * @scratch: private data for CQP WQE</div><div class='add'>+ * @wqe_idx: WQE index of CQP SQ</div><div class='add'>+ */</div><div class='add'>+__le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch,</div><div class='add'>+					   u32 *wqe_idx)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe = NULL;</div><div class='add'>+	int ret_code;</div><div class='add'>+</div><div class='add'>+	if (IRDMA_RING_FULL_ERR(cqp-&gt;sq_ring)) {</div><div class='add'>+		ibdev_dbg(to_ibdev(cqp-&gt;dev),</div><div class='add'>+			  "WQE: CQP SQ is full, head 0x%x tail 0x%x size 0x%x\n",</div><div class='add'>+			  cqp-&gt;sq_ring.head, cqp-&gt;sq_ring.tail,</div><div class='add'>+			  cqp-&gt;sq_ring.size);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+	IRDMA_ATOMIC_RING_MOVE_HEAD(cqp-&gt;sq_ring, *wqe_idx, ret_code);</div><div class='add'>+	if (ret_code)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	cqp-&gt;requested_ops++;</div><div class='add'>+	if (!*wqe_idx)</div><div class='add'>+		cqp-&gt;polarity = !cqp-&gt;polarity;</div><div class='add'>+	wqe = cqp-&gt;sq_base[*wqe_idx].elem;</div><div class='add'>+	cqp-&gt;scratch_array[*wqe_idx] = scratch;</div><div class='add'>+	IRDMA_CQP_INIT_WQE(wqe);</div><div class='add'>+</div><div class='add'>+	return wqe;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cqp_destroy - destroy cqp during close</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp)</div><div class='add'>+{</div><div class='add'>+	u32 cnt = 0, val;</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+</div><div class='add'>+	writel(0, cqp-&gt;dev-&gt;hw_regs[IRDMA_CCQPHIGH]);</div><div class='add'>+	writel(0, cqp-&gt;dev-&gt;hw_regs[IRDMA_CCQPLOW]);</div><div class='add'>+	do {</div><div class='add'>+		if (cnt++ &gt; cqp-&gt;dev-&gt;hw_attrs.max_done_count) {</div><div class='add'>+			ret_code = -ETIMEDOUT;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		udelay(cqp-&gt;dev-&gt;hw_attrs.max_sleep_count);</div><div class='add'>+		val = readl(cqp-&gt;dev-&gt;hw_regs[IRDMA_CCQPSTATUS]);</div><div class='add'>+	} while (FLD_RS_32(cqp-&gt;dev, val, IRDMA_CCQPSTATUS_CCQP_DONE));</div><div class='add'>+</div><div class='add'>+	dma_free_coherent(cqp-&gt;dev-&gt;hw-&gt;device, cqp-&gt;sdbuf.size,</div><div class='add'>+			  cqp-&gt;sdbuf.va, cqp-&gt;sdbuf.pa);</div><div class='add'>+	cqp-&gt;sdbuf.va = NULL;</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_ccq_arm - enable intr for control cq</div><div class='add'>+ * @ccq: ccq sc struct</div><div class='add'>+ */</div><div class='add'>+void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	u64 temp_val;</div><div class='add'>+	u16 sw_cq_sel;</div><div class='add'>+	u8 arm_next_se;</div><div class='add'>+	u8 arm_seq_num;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;ccq-&gt;dev-&gt;cqp_lock, flags);</div><div class='add'>+	get_64bit_val(ccq-&gt;cq_uk.shadow_area, 32, &amp;temp_val);</div><div class='add'>+	sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);</div><div class='add'>+	arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);</div><div class='add'>+	arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);</div><div class='add'>+	arm_seq_num++;</div><div class='add'>+	temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |</div><div class='add'>+		   FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |</div><div class='add'>+		   FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |</div><div class='add'>+		   FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, 1);</div><div class='add'>+	set_64bit_val(ccq-&gt;cq_uk.shadow_area, 32, temp_val);</div><div class='add'>+	spin_unlock_irqrestore(&amp;ccq-&gt;dev-&gt;cqp_lock, flags);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure shadow area is updated before arming */</div><div class='add'>+</div><div class='add'>+	writel(ccq-&gt;cq_uk.cq_id, ccq-&gt;dev-&gt;cq_arm_db);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_process_def_cmpl - process deferred or pending completion</div><div class='add'>+ * @cqp: CQP sc struct</div><div class='add'>+ * @info: CQP CQE info</div><div class='add'>+ * @wqe_idx: CQP WQE descriptor index</div><div class='add'>+ * @def_info: deferred op ticket value or out-of-order completion id</div><div class='add'>+ * @def_cmpl: true for deferred completion, false for pending (RCA)</div><div class='add'>+ */</div><div class='add'>+static void irdma_sc_process_def_cmpl(struct irdma_sc_cqp *cqp,</div><div class='add'>+				      struct irdma_ccq_cqe_info *info,</div><div class='add'>+				      u32 wqe_idx, u32 def_info, bool def_cmpl)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ooo_cqp_op *ooo_op;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	/* Deferred and out-of-order completions share the same list of pending</div><div class='add'>+	 * completions.  Since the list can be also accessed from AE handler,</div><div class='add'>+	 * it must be protected by a lock.</div><div class='add'>+	 */</div><div class='add'>+	spin_lock_irqsave(&amp;cqp-&gt;ooo_list_lock, flags);</div><div class='add'>+</div><div class='add'>+	/* For deferred completions bump up SW completion ticket value. */</div><div class='add'>+	if (def_cmpl) {</div><div class='add'>+		cqp-&gt;last_def_cmpl_ticket = def_info;</div><div class='add'>+		cqp-&gt;sw_def_cmpl_ticket++;</div><div class='add'>+	}</div><div class='add'>+	if (!list_empty(&amp;cqp-&gt;ooo_avail)) {</div><div class='add'>+		ooo_op = (struct irdma_ooo_cqp_op *)</div><div class='add'>+			 list_entry(cqp-&gt;ooo_avail.next,</div><div class='add'>+				    struct irdma_ooo_cqp_op, list_entry);</div><div class='add'>+</div><div class='add'>+		list_del(&amp;ooo_op-&gt;list_entry);</div><div class='add'>+		ooo_op-&gt;scratch = info-&gt;scratch;</div><div class='add'>+		ooo_op-&gt;def_info = def_info;</div><div class='add'>+		ooo_op-&gt;sw_def_info = cqp-&gt;sw_def_cmpl_ticket;</div><div class='add'>+		ooo_op-&gt;deferred = def_cmpl;</div><div class='add'>+		ooo_op-&gt;wqe_idx = wqe_idx;</div><div class='add'>+		/* Pending completions must be chronologically ordered,</div><div class='add'>+		 * so adding at the end of list.</div><div class='add'>+		 */</div><div class='add'>+		list_add_tail(&amp;ooo_op-&gt;list_entry, &amp;cqp-&gt;ooo_pnd);</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;cqp-&gt;ooo_list_lock, flags);</div><div class='add'>+</div><div class='add'>+	info-&gt;pending = true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_process_ooo_cmpl - process out-of-order (final) completion</div><div class='add'>+ * @cqp: CQP sc struct</div><div class='add'>+ * @info: CQP CQE info</div><div class='add'>+ * @def_info: out-of-order completion id</div><div class='add'>+ */</div><div class='add'>+static void irdma_sc_process_ooo_cmpl(struct irdma_sc_cqp *cqp,</div><div class='add'>+				      struct irdma_ccq_cqe_info *info,</div><div class='add'>+				      u32 def_info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ooo_cqp_op *ooo_op_tmp;</div><div class='add'>+	struct irdma_ooo_cqp_op *ooo_op;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	info-&gt;scratch = 0;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cqp-&gt;ooo_list_lock, flags);</div><div class='add'>+	list_for_each_entry_safe(ooo_op, ooo_op_tmp, &amp;cqp-&gt;ooo_pnd,</div><div class='add'>+				 list_entry) {</div><div class='add'>+		if (!ooo_op-&gt;deferred &amp;&amp; ooo_op-&gt;def_info == def_info) {</div><div class='add'>+			list_del(&amp;ooo_op-&gt;list_entry);</div><div class='add'>+			info-&gt;scratch = ooo_op-&gt;scratch;</div><div class='add'>+			list_add(&amp;ooo_op-&gt;list_entry, &amp;cqp-&gt;ooo_avail);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;cqp-&gt;ooo_list_lock, flags);</div><div class='add'>+</div><div class='add'>+	if (!info-&gt;scratch)</div><div class='add'>+		ibdev_dbg(to_ibdev(cqp-&gt;dev),</div><div class='add'>+			  "CQP: DEBUG_FW_OOO out-of-order completion with unknown def_info = 0x%x\n",</div><div class='add'>+			  def_info);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_ccq_get_cqe_info - get ccq's cq entry</div><div class='add'>+ * @ccq: ccq sc struct</div><div class='add'>+ * @info: completion q entry to return</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,</div><div class='add'>+			      struct irdma_ccq_cqe_info *info)</div><div class='add'>+{</div><div class='add'>+	u32 def_info;</div><div class='add'>+	bool def_cmpl = false;</div><div class='add'>+	bool pend_cmpl = false;</div><div class='add'>+	bool ooo_final_cmpl = false;</div><div class='add'>+	u64 qp_ctx, temp, temp1;</div><div class='add'>+	__le64 *cqe;</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+	u32 error;</div><div class='add'>+	u8 polarity;</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	if (ccq-&gt;cq_uk.avoid_mem_cflct)</div><div class='add'>+		cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(&amp;ccq-&gt;cq_uk);</div><div class='add'>+	else</div><div class='add'>+		cqe = IRDMA_GET_CURRENT_CQ_ELEM(&amp;ccq-&gt;cq_uk);</div><div class='add'>+</div><div class='add'>+	get_64bit_val(cqe, 24, &amp;temp);</div><div class='add'>+	polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, temp);</div><div class='add'>+	if (polarity != ccq-&gt;cq_uk.polarity)</div><div class='add'>+		return -ENOENT;</div><div class='add'>+</div><div class='add'>+	/* Ensure CEQE contents are read after valid bit is checked */</div><div class='add'>+	dma_rmb();</div><div class='add'>+</div><div class='add'>+	get_64bit_val(cqe, 8, &amp;qp_ctx);</div><div class='add'>+	cqp = (struct irdma_sc_cqp *)(unsigned long)qp_ctx;</div><div class='add'>+	info-&gt;error = (bool)FIELD_GET(IRDMA_CQ_ERROR, temp);</div><div class='add'>+	info-&gt;maj_err_code = IRDMA_CQPSQ_MAJ_NO_ERROR;</div><div class='add'>+	info-&gt;min_err_code = (u16)FIELD_GET(IRDMA_CQ_MINERR, temp);</div><div class='add'>+	if (info-&gt;error) {</div><div class='add'>+		info-&gt;maj_err_code = (u16)FIELD_GET(IRDMA_CQ_MAJERR, temp);</div><div class='add'>+		error = readl(cqp-&gt;dev-&gt;hw_regs[IRDMA_CQPERRCODES]);</div><div class='add'>+		ibdev_dbg(to_ibdev(cqp-&gt;dev),</div><div class='add'>+			  "CQP: CQPERRCODES error_code[x%08X]\n", error);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, temp);</div><div class='add'>+	info-&gt;scratch = cqp-&gt;scratch_array[wqe_idx];</div><div class='add'>+</div><div class='add'>+	get_64bit_val(cqe, 16, &amp;temp1);</div><div class='add'>+	info-&gt;op_ret_val = (u32)FIELD_GET(IRDMA_CCQ_OPRETVAL, temp1);</div><div class='add'>+	if (cqp-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3) {</div><div class='add'>+		def_cmpl = info-&gt;maj_err_code == IRDMA_CQPSQ_MAJ_NO_ERROR &amp;&amp;</div><div class='add'>+			   info-&gt;min_err_code == IRDMA_CQPSQ_MIN_DEF_CMPL;</div><div class='add'>+		def_info = (u32)FIELD_GET(IRDMA_CCQ_DEFINFO, temp1);</div><div class='add'>+</div><div class='add'>+		pend_cmpl = info-&gt;maj_err_code == IRDMA_CQPSQ_MAJ_NO_ERROR &amp;&amp;</div><div class='add'>+			    info-&gt;min_err_code == IRDMA_CQPSQ_MIN_OOO_CMPL;</div><div class='add'>+</div><div class='add'>+		ooo_final_cmpl = (bool)FIELD_GET(IRDMA_OOO_CMPL, temp);</div><div class='add'>+</div><div class='add'>+		if (def_cmpl || pend_cmpl || ooo_final_cmpl) {</div><div class='add'>+			if (ooo_final_cmpl)</div><div class='add'>+				irdma_sc_process_ooo_cmpl(cqp, info, def_info);</div><div class='add'>+			else</div><div class='add'>+				irdma_sc_process_def_cmpl(cqp, info, wqe_idx,</div><div class='add'>+							  def_info, def_cmpl);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	get_64bit_val(cqp-&gt;sq_base[wqe_idx].elem, 24, &amp;temp1);</div><div class='add'>+	info-&gt;op_code = (u8)FIELD_GET(IRDMA_CQPSQ_OPCODE, temp1);</div><div class='add'>+	info-&gt;cqp = cqp;</div><div class='add'>+</div><div class='add'>+	/*  move the head for cq */</div><div class='add'>+	IRDMA_RING_MOVE_HEAD(ccq-&gt;cq_uk.cq_ring, ret_code);</div><div class='add'>+	if (!IRDMA_RING_CURRENT_HEAD(ccq-&gt;cq_uk.cq_ring))</div><div class='add'>+		ccq-&gt;cq_uk.polarity ^= 1;</div><div class='add'>+</div><div class='add'>+	/* update cq tail in cq shadow memory also */</div><div class='add'>+	IRDMA_RING_MOVE_TAIL(ccq-&gt;cq_uk.cq_ring);</div><div class='add'>+	set_64bit_val(ccq-&gt;cq_uk.shadow_area, 0,</div><div class='add'>+		      IRDMA_RING_CURRENT_HEAD(ccq-&gt;cq_uk.cq_ring));</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure shadow area is updated before moving tail */</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cqp-&gt;dev-&gt;cqp_lock, flags);</div><div class='add'>+	if (!ooo_final_cmpl)</div><div class='add'>+		IRDMA_RING_MOVE_TAIL(cqp-&gt;sq_ring);</div><div class='add'>+	spin_unlock_irqrestore(&amp;cqp-&gt;dev-&gt;cqp_lock, flags);</div><div class='add'>+</div><div class='add'>+	/* Do not increment completed_ops counter on pending or deferred</div><div class='add'>+	 * completions.</div><div class='add'>+	 */</div><div class='add'>+	if (pend_cmpl || def_cmpl)</div><div class='add'>+		return ret_code;</div><div class='add'>+	atomic64_inc(&amp;cqp-&gt;completed_ops);</div><div class='add'>+</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @op_code: cqp opcode for completion</div><div class='add'>+ * @compl_info: completion q entry to return</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 op_code,</div><div class='add'>+				  struct irdma_ccq_cqe_info *compl_info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ccq_cqe_info info = {};</div><div class='add'>+	struct irdma_sc_cq *ccq;</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+	u32 cnt = 0;</div><div class='add'>+</div><div class='add'>+	ccq = cqp-&gt;dev-&gt;ccq;</div><div class='add'>+	while (1) {</div><div class='add'>+		if (cnt++ &gt; 100 * cqp-&gt;dev-&gt;hw_attrs.max_done_count)</div><div class='add'>+			return -ETIMEDOUT;</div><div class='add'>+</div><div class='add'>+		if (irdma_sc_ccq_get_cqe_info(ccq, &amp;info)) {</div><div class='add'>+			udelay(cqp-&gt;dev-&gt;hw_attrs.max_sleep_count);</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+		if (info.error &amp;&amp; info.op_code != IRDMA_CQP_OP_QUERY_STAG) {</div><div class='add'>+			ret_code = -EIO;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		/* make sure op code matches*/</div><div class='add'>+		if (op_code == info.op_code)</div><div class='add'>+			break;</div><div class='add'>+		ibdev_dbg(to_ibdev(cqp-&gt;dev),</div><div class='add'>+			  "WQE: opcode mismatch for my op code 0x%x, returned opcode %x\n",</div><div class='add'>+			  op_code, info.op_code);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (compl_info)</div><div class='add'>+		memcpy(compl_info, &amp;info, sizeof(*compl_info));</div><div class='add'>+</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_manage_hmc_pm_func_table - manage of function table</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @info: info for the manage function table operation</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_manage_hmc_pm_func_table(struct irdma_sc_cqp *cqp,</div><div class='add'>+					     struct irdma_hmc_fcn_info *info,</div><div class='add'>+					     u64 scratch, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0, 0);</div><div class='add'>+	set_64bit_val(wqe, 8, 0);</div><div class='add'>+	set_64bit_val(wqe, 16, 0);</div><div class='add'>+	set_64bit_val(wqe, 32, 0);</div><div class='add'>+	set_64bit_val(wqe, 40, 0);</div><div class='add'>+	set_64bit_val(wqe, 48, 0);</div><div class='add'>+	set_64bit_val(wqe, 56, 0);</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_MHMC_VFIDX, info-&gt;vf_id) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE,</div><div class='add'>+			 IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_MHMC_FREEPMFN, info-&gt;free_fcn) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: MANAGE_HMC_PM_FUNC_TABLE WQE",</div><div class='add'>+			     DUMP_PREFIX_OFFSET, 16, 8, wqe,</div><div class='add'>+			     IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_commit_fpm_val_done - wait for cqp eqe completion</div><div class='add'>+ * for fpm commit</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_commit_fpm_val_done(struct irdma_sc_cqp *cqp)</div><div class='add'>+{</div><div class='add'>+	return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_COMMIT_FPM_VAL,</div><div class='add'>+					     NULL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_commit_fpm_val - cqp wqe for commit fpm values</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @hmc_fn_id: hmc function id</div><div class='add'>+ * @commit_fpm_mem: Memory for fpm values</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ * @wait_type: poll ccq or cqp registers for cqp completion</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_commit_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch,</div><div class='add'>+				   u8 hmc_fn_id,</div><div class='add'>+				   struct irdma_dma_mem *commit_fpm_mem,</div><div class='add'>+				   bool post_sq, u8 wait_type)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 tail, val, error;</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16, hmc_fn_id);</div><div class='add'>+	set_64bit_val(wqe, 32, commit_fpm_mem-&gt;pa);</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_BUFSIZE, IRDMA_COMMIT_FPM_BUF_SIZE) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_COMMIT_FPM_VAL) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: COMMIT_FPM_VAL WQE", DUMP_PREFIX_OFFSET,</div><div class='add'>+			     16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	irdma_get_cqp_reg_info(cqp, &amp;val, &amp;tail, &amp;error);</div><div class='add'>+</div><div class='add'>+	if (post_sq) {</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+		if (wait_type == IRDMA_CQP_WAIT_POLL_REGS)</div><div class='add'>+			ret_code = irdma_cqp_poll_registers(cqp, tail,</div><div class='add'>+							    cqp-&gt;dev-&gt;hw_attrs.max_done_count);</div><div class='add'>+		else if (wait_type == IRDMA_CQP_WAIT_POLL_CQ)</div><div class='add'>+			ret_code = irdma_sc_commit_fpm_val_done(cqp);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_query_fpm_val_done - poll for cqp wqe completion for</div><div class='add'>+ * query fpm</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_query_fpm_val_done(struct irdma_sc_cqp *cqp)</div><div class='add'>+{</div><div class='add'>+	return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_QUERY_FPM_VAL,</div><div class='add'>+					     NULL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_query_fpm_val - cqp wqe query fpm values</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @hmc_fn_id: hmc function id</div><div class='add'>+ * @query_fpm_mem: memory for return fpm values</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ * @wait_type: poll ccq or cqp registers for cqp completion</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_query_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch,</div><div class='add'>+				  u8 hmc_fn_id,</div><div class='add'>+				  struct irdma_dma_mem *query_fpm_mem,</div><div class='add'>+				  bool post_sq, u8 wait_type)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 tail, val, error;</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16, hmc_fn_id);</div><div class='add'>+	set_64bit_val(wqe, 32, query_fpm_mem-&gt;pa);</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_QUERY_FPM_VAL) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: QUERY_FPM WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	irdma_get_cqp_reg_info(cqp, &amp;val, &amp;tail, &amp;error);</div><div class='add'>+</div><div class='add'>+	if (post_sq) {</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+		if (wait_type == IRDMA_CQP_WAIT_POLL_REGS)</div><div class='add'>+			ret_code = irdma_cqp_poll_registers(cqp, tail,</div><div class='add'>+							    cqp-&gt;dev-&gt;hw_attrs.max_done_count);</div><div class='add'>+		else if (wait_type == IRDMA_CQP_WAIT_POLL_CQ)</div><div class='add'>+			ret_code = irdma_sc_query_fpm_val_done(cqp);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_ceq_init - initialize ceq</div><div class='add'>+ * @ceq: ceq sc structure</div><div class='add'>+ * @info: ceq initialization info</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_ceq_init(struct irdma_sc_ceq *ceq,</div><div class='add'>+		      struct irdma_ceq_init_info *info)</div><div class='add'>+{</div><div class='add'>+	u32 pble_obj_cnt;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;elem_cnt &lt; info-&gt;dev-&gt;hw_attrs.min_hw_ceq_size ||</div><div class='add'>+	    info-&gt;elem_cnt &gt; info-&gt;dev-&gt;hw_attrs.max_hw_ceq_size)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;ceq_id &gt;= info-&gt;dev-&gt;hmc_fpm_misc.max_ceqs)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	pble_obj_cnt = info-&gt;dev-&gt;hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].cnt;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;virtual_map &amp;&amp; info-&gt;first_pm_pbl_idx &gt;= pble_obj_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ceq-&gt;size = sizeof(*ceq);</div><div class='add'>+	ceq-&gt;ceqe_base = (struct irdma_ceqe *)info-&gt;ceqe_base;</div><div class='add'>+	ceq-&gt;ceq_id = info-&gt;ceq_id;</div><div class='add'>+	ceq-&gt;dev = info-&gt;dev;</div><div class='add'>+	ceq-&gt;elem_cnt = info-&gt;elem_cnt;</div><div class='add'>+	ceq-&gt;ceq_elem_pa = info-&gt;ceqe_pa;</div><div class='add'>+	ceq-&gt;virtual_map = info-&gt;virtual_map;</div><div class='add'>+	ceq-&gt;itr_no_expire = info-&gt;itr_no_expire;</div><div class='add'>+	ceq-&gt;pbl_chunk_size = (ceq-&gt;virtual_map ? info-&gt;pbl_chunk_size : 0);</div><div class='add'>+	ceq-&gt;first_pm_pbl_idx = (ceq-&gt;virtual_map ? info-&gt;first_pm_pbl_idx : 0);</div><div class='add'>+	ceq-&gt;pbl_list = (ceq-&gt;virtual_map ? info-&gt;pbl_list : NULL);</div><div class='add'>+	ceq-&gt;tph_en = info-&gt;tph_en;</div><div class='add'>+	ceq-&gt;tph_val = info-&gt;tph_val;</div><div class='add'>+	ceq-&gt;vsi_idx = info-&gt;vsi_idx;</div><div class='add'>+	ceq-&gt;polarity = 1;</div><div class='add'>+	IRDMA_RING_INIT(ceq-&gt;ceq_ring, ceq-&gt;elem_cnt);</div><div class='add'>+	ceq-&gt;dev-&gt;ceq[info-&gt;ceq_id] = ceq;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_ceq_create - create ceq wqe</div><div class='add'>+ * @ceq: ceq sc structure</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+static int irdma_sc_ceq_create(struct irdma_sc_ceq *ceq, u64 scratch,</div><div class='add'>+			       bool post_sq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	cqp = ceq-&gt;dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	set_64bit_val(wqe, 16, ceq-&gt;elem_cnt);</div><div class='add'>+	set_64bit_val(wqe, 32,</div><div class='add'>+		      (ceq-&gt;virtual_map ? 0 : ceq-&gt;ceq_elem_pa));</div><div class='add'>+	set_64bit_val(wqe, 48,</div><div class='add'>+		      (ceq-&gt;virtual_map ? ceq-&gt;first_pm_pbl_idx : 0));</div><div class='add'>+	set_64bit_val(wqe, 56,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_TPHVAL, ceq-&gt;tph_val) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_PASID, ceq-&gt;pasid) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_VSIIDX, ceq-&gt;vsi_idx));</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_CEQ_CEQID, ceq-&gt;ceq_id) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CEQ_CEQID_HIGH, ceq-&gt;ceq_id &gt;&gt; 10) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CEQ) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq-&gt;pbl_chunk_size) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq-&gt;virtual_map) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CEQ_ITRNOEXPIRE, ceq-&gt;itr_no_expire) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq-&gt;tph_en) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_PASID_VALID, ceq-&gt;pasid_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: CEQ_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cceq_create_done - poll for control ceq wqe to complete</div><div class='add'>+ * @ceq: ceq sc structure</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_cceq_create_done(struct irdma_sc_ceq *ceq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+</div><div class='add'>+	cqp = ceq-&gt;dev-&gt;cqp;</div><div class='add'>+	return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CEQ,</div><div class='add'>+					     NULL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cceq_destroy_done - poll for destroy cceq to complete</div><div class='add'>+ * @ceq: ceq sc structure</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+</div><div class='add'>+	cqp = ceq-&gt;dev-&gt;cqp;</div><div class='add'>+	cqp-&gt;process_cqp_sds = irdma_update_sds_noccq;</div><div class='add'>+</div><div class='add'>+	return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_DESTROY_CEQ,</div><div class='add'>+					     NULL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cceq_create - create cceq</div><div class='add'>+ * @ceq: ceq sc structure</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch)</div><div class='add'>+{</div><div class='add'>+	int ret_code;</div><div class='add'>+	struct irdma_sc_dev *dev = ceq-&gt;dev;</div><div class='add'>+</div><div class='add'>+	dev-&gt;ccq-&gt;vsi_idx = ceq-&gt;vsi_idx;</div><div class='add'>+</div><div class='add'>+	ret_code = irdma_sc_ceq_create(ceq, scratch, true);</div><div class='add'>+	if (!ret_code)</div><div class='add'>+		return irdma_sc_cceq_create_done(ceq);</div><div class='add'>+</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_ceq_destroy - destroy ceq</div><div class='add'>+ * @ceq: ceq sc structure</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	cqp = ceq-&gt;dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16, ceq-&gt;elem_cnt);</div><div class='add'>+	set_64bit_val(wqe, 48, ceq-&gt;first_pm_pbl_idx);</div><div class='add'>+	set_64bit_val(wqe, 56,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_PASID, ceq-&gt;pasid));</div><div class='add'>+	hdr = ceq-&gt;ceq_id |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CEQ) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq-&gt;pbl_chunk_size) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq-&gt;virtual_map) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq-&gt;tph_en) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_PASID_VALID, ceq-&gt;pasid_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: CEQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_process_ceq - process ceq</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @ceq: ceq sc structure</div><div class='add'>+ *</div><div class='add'>+ * It is expected caller serializes this function with cleanup_ceqes()</div><div class='add'>+ * because these functions manipulate the same ceq</div><div class='add'>+ */</div><div class='add'>+void *irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq)</div><div class='add'>+{</div><div class='add'>+	u64 temp;</div><div class='add'>+	__le64 *ceqe;</div><div class='add'>+	struct irdma_sc_cq *cq = NULL;</div><div class='add'>+	struct irdma_sc_cq *temp_cq;</div><div class='add'>+	u8 polarity;</div><div class='add'>+	u32 cq_idx;</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		cq_idx = 0;</div><div class='add'>+		ceqe = IRDMA_GET_CURRENT_CEQ_ELEM(ceq);</div><div class='add'>+		get_64bit_val(ceqe, 0, &amp;temp);</div><div class='add'>+		polarity = (u8)FIELD_GET(IRDMA_CEQE_VALID, temp);</div><div class='add'>+		if (polarity != ceq-&gt;polarity)</div><div class='add'>+			return NULL;</div><div class='add'>+</div><div class='add'>+		temp_cq = (struct irdma_sc_cq *)(unsigned long)(temp &lt;&lt; 1);</div><div class='add'>+		if (!temp_cq) {</div><div class='add'>+			cq_idx = IRDMA_INVALID_CQ_IDX;</div><div class='add'>+			IRDMA_RING_MOVE_TAIL(ceq-&gt;ceq_ring);</div><div class='add'>+</div><div class='add'>+			if (!IRDMA_RING_CURRENT_TAIL(ceq-&gt;ceq_ring))</div><div class='add'>+				ceq-&gt;polarity ^= 1;</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		cq = temp_cq;</div><div class='add'>+</div><div class='add'>+		IRDMA_RING_MOVE_TAIL(ceq-&gt;ceq_ring);</div><div class='add'>+		if (!IRDMA_RING_CURRENT_TAIL(ceq-&gt;ceq_ring))</div><div class='add'>+			ceq-&gt;polarity ^= 1;</div><div class='add'>+	} while (cq_idx == IRDMA_INVALID_CQ_IDX);</div><div class='add'>+</div><div class='add'>+	if (cq)</div><div class='add'>+		irdma_sc_cq_ack(cq);</div><div class='add'>+	return cq;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cleanup_ceqes - clear the valid ceqes ctx matching the cq</div><div class='add'>+ * @cq: cq for which the ceqes need to be cleaned up</div><div class='add'>+ * @ceq: ceq ptr</div><div class='add'>+ *</div><div class='add'>+ * The function is called after the cq is destroyed to cleanup</div><div class='add'>+ * its pending ceqe entries. It is expected caller serializes this</div><div class='add'>+ * function with process_ceq() in interrupt context.</div><div class='add'>+ */</div><div class='add'>+void irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_cq *next_cq;</div><div class='add'>+	u8 ceq_polarity = ceq-&gt;polarity;</div><div class='add'>+	__le64 *ceqe;</div><div class='add'>+	u8 polarity;</div><div class='add'>+	u64 temp;</div><div class='add'>+	int next;</div><div class='add'>+	u32 i;</div><div class='add'>+</div><div class='add'>+	next = IRDMA_RING_GET_NEXT_TAIL(ceq-&gt;ceq_ring, 0);</div><div class='add'>+</div><div class='add'>+	for (i = 1; i &lt;= IRDMA_RING_SIZE(*ceq); i++) {</div><div class='add'>+		ceqe = IRDMA_GET_CEQ_ELEM_AT_POS(ceq, next);</div><div class='add'>+</div><div class='add'>+		get_64bit_val(ceqe, 0, &amp;temp);</div><div class='add'>+		polarity = (u8)FIELD_GET(IRDMA_CEQE_VALID, temp);</div><div class='add'>+		if (polarity != ceq_polarity)</div><div class='add'>+			return;</div><div class='add'>+</div><div class='add'>+		next_cq = (struct irdma_sc_cq *)(unsigned long)(temp &lt;&lt; 1);</div><div class='add'>+		if (cq == next_cq)</div><div class='add'>+			set_64bit_val(ceqe, 0, temp &amp; IRDMA_CEQE_VALID);</div><div class='add'>+</div><div class='add'>+		next = IRDMA_RING_GET_NEXT_TAIL(ceq-&gt;ceq_ring, i);</div><div class='add'>+		if (!next)</div><div class='add'>+			ceq_polarity ^= 1;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_aeq_init - initialize aeq</div><div class='add'>+ * @aeq: aeq structure ptr</div><div class='add'>+ * @info: aeq initialization info</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_aeq_init(struct irdma_sc_aeq *aeq,</div><div class='add'>+		      struct irdma_aeq_init_info *info)</div><div class='add'>+{</div><div class='add'>+	u32 pble_obj_cnt;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;elem_cnt &lt; info-&gt;dev-&gt;hw_attrs.min_hw_aeq_size ||</div><div class='add'>+	    info-&gt;elem_cnt &gt; info-&gt;dev-&gt;hw_attrs.max_hw_aeq_size)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	pble_obj_cnt = info-&gt;dev-&gt;hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].cnt;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;virtual_map &amp;&amp; info-&gt;first_pm_pbl_idx &gt;= pble_obj_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	aeq-&gt;size = sizeof(*aeq);</div><div class='add'>+	aeq-&gt;polarity = 1;</div><div class='add'>+	aeq-&gt;aeqe_base = (struct irdma_sc_aeqe *)info-&gt;aeqe_base;</div><div class='add'>+	aeq-&gt;dev = info-&gt;dev;</div><div class='add'>+	aeq-&gt;elem_cnt = info-&gt;elem_cnt;</div><div class='add'>+	aeq-&gt;aeq_elem_pa = info-&gt;aeq_elem_pa;</div><div class='add'>+	IRDMA_RING_INIT(aeq-&gt;aeq_ring, aeq-&gt;elem_cnt);</div><div class='add'>+	aeq-&gt;virtual_map = info-&gt;virtual_map;</div><div class='add'>+	aeq-&gt;pbl_list = (aeq-&gt;virtual_map ? info-&gt;pbl_list : NULL);</div><div class='add'>+	aeq-&gt;pbl_chunk_size = (aeq-&gt;virtual_map ? info-&gt;pbl_chunk_size : 0);</div><div class='add'>+	aeq-&gt;first_pm_pbl_idx = (aeq-&gt;virtual_map ? info-&gt;first_pm_pbl_idx : 0);</div><div class='add'>+	aeq-&gt;msix_idx = info-&gt;msix_idx;</div><div class='add'>+	info-&gt;dev-&gt;aeq = aeq;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_aeq_create - create aeq</div><div class='add'>+ * @aeq: aeq structure ptr</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_aeq_create(struct irdma_sc_aeq *aeq, u64 scratch,</div><div class='add'>+			       bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	cqp = aeq-&gt;dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	set_64bit_val(wqe, 16, aeq-&gt;elem_cnt);</div><div class='add'>+	set_64bit_val(wqe, 32,</div><div class='add'>+		      (aeq-&gt;virtual_map ? 0 : aeq-&gt;aeq_elem_pa));</div><div class='add'>+	set_64bit_val(wqe, 48,</div><div class='add'>+		      (aeq-&gt;virtual_map ? aeq-&gt;first_pm_pbl_idx : 0));</div><div class='add'>+	set_64bit_val(wqe, 56,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_PASID, aeq-&gt;pasid));</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_AEQ) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq-&gt;pbl_chunk_size) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq-&gt;virtual_map) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_PASID_VALID, aeq-&gt;pasid_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: AEQ_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_aeq_destroy - destroy aeq during close</div><div class='add'>+ * @aeq: aeq structure ptr</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_aeq_destroy(struct irdma_sc_aeq *aeq, u64 scratch,</div><div class='add'>+				bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	dev = aeq-&gt;dev;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &lt;= IRDMA_GEN_2)</div><div class='add'>+		writel(0, dev-&gt;hw_regs[IRDMA_PFINT_AEQCTL]);</div><div class='add'>+</div><div class='add'>+	cqp = dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	set_64bit_val(wqe, 16, aeq-&gt;elem_cnt);</div><div class='add'>+	set_64bit_val(wqe, 48, aeq-&gt;first_pm_pbl_idx);</div><div class='add'>+	set_64bit_val(wqe, 56,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_PASID, aeq-&gt;pasid));</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_AEQ) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq-&gt;pbl_chunk_size) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq-&gt;virtual_map) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_PASID_VALID, aeq-&gt;pasid_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: AEQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_get_next_aeqe - get next aeq entry</div><div class='add'>+ * @aeq: aeq structure ptr</div><div class='add'>+ * @info: aeqe info to be returned</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq,</div><div class='add'>+			   struct irdma_aeqe_info *info)</div><div class='add'>+{</div><div class='add'>+	u64 temp, compl_ctx;</div><div class='add'>+	__le64 *aeqe;</div><div class='add'>+	u8 ae_src;</div><div class='add'>+	u8 polarity;</div><div class='add'>+</div><div class='add'>+	aeqe = IRDMA_GET_CURRENT_AEQ_ELEM(aeq);</div><div class='add'>+	get_64bit_val(aeqe, 8, &amp;temp);</div><div class='add'>+	polarity = (u8)FIELD_GET(IRDMA_AEQE_VALID, temp);</div><div class='add'>+</div><div class='add'>+	if (aeq-&gt;polarity != polarity)</div><div class='add'>+		return -ENOENT;</div><div class='add'>+</div><div class='add'>+	/* Ensure AEQE contents are read after valid bit is checked */</div><div class='add'>+	dma_rmb();</div><div class='add'>+</div><div class='add'>+	get_64bit_val(aeqe, 0, &amp;compl_ctx);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: AEQ_ENTRY WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     aeqe, 16, false);</div><div class='add'>+</div><div class='add'>+	if (aeq-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3) {</div><div class='add'>+		ae_src = (u8)FIELD_GET(IRDMA_AEQE_AESRC_GEN_3, temp);</div><div class='add'>+		info-&gt;wqe_idx = (u16)FIELD_GET(IRDMA_AEQE_WQDESCIDX_GEN_3,</div><div class='add'>+					       temp);</div><div class='add'>+		info-&gt;qp_cq_id = (u32)FIELD_GET(IRDMA_AEQE_QPCQID_GEN_3, temp);</div><div class='add'>+		info-&gt;ae_id = (u16)FIELD_GET(IRDMA_AEQE_AECODE_GEN_3, temp);</div><div class='add'>+		info-&gt;tcp_state = (u8)FIELD_GET(IRDMA_AEQE_TCPSTATE_GEN_3, compl_ctx);</div><div class='add'>+		info-&gt;iwarp_state = (u8)FIELD_GET(IRDMA_AEQE_IWSTATE_GEN_3, temp);</div><div class='add'>+		info-&gt;q2_data_written = (u8)FIELD_GET(IRDMA_AEQE_Q2DATA_GEN_3, compl_ctx);</div><div class='add'>+		info-&gt;aeqe_overflow = (bool)FIELD_GET(IRDMA_AEQE_OVERFLOW_GEN_3, temp);</div><div class='add'>+		info-&gt;compl_ctx = FIELD_GET(IRDMA_AEQE_CMPL_CTXT, compl_ctx);</div><div class='add'>+		compl_ctx = FIELD_GET(IRDMA_AEQE_CMPL_CTXT, compl_ctx) &lt;&lt; IRDMA_AEQE_CMPL_CTXT_S;</div><div class='add'>+	} else {</div><div class='add'>+		ae_src = (u8)FIELD_GET(IRDMA_AEQE_AESRC, temp);</div><div class='add'>+		info-&gt;wqe_idx = (u16)FIELD_GET(IRDMA_AEQE_WQDESCIDX, temp);</div><div class='add'>+		info-&gt;qp_cq_id = (u32)FIELD_GET(IRDMA_AEQE_QPCQID_LOW, temp) |</div><div class='add'>+			 ((u32)FIELD_GET(IRDMA_AEQE_QPCQID_HI, temp) &lt;&lt; 18);</div><div class='add'>+		info-&gt;ae_id = (u16)FIELD_GET(IRDMA_AEQE_AECODE, temp);</div><div class='add'>+		info-&gt;tcp_state = (u8)FIELD_GET(IRDMA_AEQE_TCPSTATE, temp);</div><div class='add'>+		info-&gt;iwarp_state = (u8)FIELD_GET(IRDMA_AEQE_IWSTATE, temp);</div><div class='add'>+		info-&gt;q2_data_written = (u8)FIELD_GET(IRDMA_AEQE_Q2DATA, temp);</div><div class='add'>+		info-&gt;aeqe_overflow = (bool)FIELD_GET(IRDMA_AEQE_OVERFLOW,</div><div class='add'>+						      temp);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	info-&gt;ae_src = ae_src;</div><div class='add'>+	switch (info-&gt;ae_id) {</div><div class='add'>+	case IRDMA_AE_SRQ_LIMIT:</div><div class='add'>+		info-&gt;srq = true;</div><div class='add'>+		/* [63:6] from CMPL_CTXT, [5:0] from WQDESCIDX. */</div><div class='add'>+		info-&gt;compl_ctx = compl_ctx;</div><div class='add'>+		ae_src = IRDMA_AE_SOURCE_RSVD;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_PRIV_OPERATION_DENIED:</div><div class='add'>+	case IRDMA_AE_AMP_INVALIDATE_TYPE1_MW:</div><div class='add'>+	case IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW:</div><div class='add'>+	case IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG:</div><div class='add'>+	case IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH:</div><div class='add'>+	case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:</div><div class='add'>+	case IRDMA_AE_UDA_XMIT_BAD_PD:</div><div class='add'>+	case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT:</div><div class='add'>+	case IRDMA_AE_BAD_CLOSE:</div><div class='add'>+	case IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO:</div><div class='add'>+	case IRDMA_AE_STAG_ZERO_INVALID:</div><div class='add'>+	case IRDMA_AE_IB_RREQ_AND_Q1_FULL:</div><div class='add'>+	case IRDMA_AE_IB_INVALID_REQUEST:</div><div class='add'>+	case IRDMA_AE_WQE_UNEXPECTED_OPCODE:</div><div class='add'>+	case IRDMA_AE_IB_REMOTE_ACCESS_ERROR:</div><div class='add'>+	case IRDMA_AE_IB_REMOTE_OP_ERROR:</div><div class='add'>+	case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:</div><div class='add'>+	case IRDMA_AE_DDP_UBE_INVALID_MO:</div><div class='add'>+	case IRDMA_AE_DDP_UBE_INVALID_QN:</div><div class='add'>+	case IRDMA_AE_DDP_NO_L_BIT:</div><div class='add'>+	case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:</div><div class='add'>+	case IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE:</div><div class='add'>+	case IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST:</div><div class='add'>+	case IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:</div><div class='add'>+	case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:</div><div class='add'>+	case IRDMA_AE_INVALID_ARP_ENTRY:</div><div class='add'>+	case IRDMA_AE_INVALID_TCP_OPTION_RCVD:</div><div class='add'>+	case IRDMA_AE_STALE_ARP_ENTRY:</div><div class='add'>+	case IRDMA_AE_INVALID_AH_ENTRY:</div><div class='add'>+	case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:</div><div class='add'>+	case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:</div><div class='add'>+	case IRDMA_AE_LLP_TOO_MANY_RETRIES:</div><div class='add'>+	case IRDMA_AE_LLP_TOO_MANY_RNRS:</div><div class='add'>+	case IRDMA_AE_REMOTE_QP_CATASTROPHIC:</div><div class='add'>+	case IRDMA_AE_LOCAL_QP_CATASTROPHIC:</div><div class='add'>+	case IRDMA_AE_RCE_QP_CATASTROPHIC:</div><div class='add'>+	case IRDMA_AE_LLP_DOUBT_REACHABILITY:</div><div class='add'>+	case IRDMA_AE_LLP_CONNECTION_ESTABLISHED:</div><div class='add'>+	case IRDMA_AE_RESET_SENT:</div><div class='add'>+	case IRDMA_AE_TERMINATE_SENT:</div><div class='add'>+	case IRDMA_AE_RESET_NOT_SENT:</div><div class='add'>+	case IRDMA_AE_LCE_QP_CATASTROPHIC:</div><div class='add'>+	case IRDMA_AE_QP_SUSPEND_COMPLETE:</div><div class='add'>+	case IRDMA_AE_UDA_L4LEN_INVALID:</div><div class='add'>+		info-&gt;qp = true;</div><div class='add'>+		info-&gt;compl_ctx = compl_ctx;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_LCE_CQ_CATASTROPHIC:</div><div class='add'>+		info-&gt;cq = true;</div><div class='add'>+		info-&gt;compl_ctx = compl_ctx &lt;&lt; 1;</div><div class='add'>+		ae_src = IRDMA_AE_SOURCE_RSVD;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_CQP_DEFERRED_COMPLETE:</div><div class='add'>+		info-&gt;def_info = info-&gt;wqe_idx;</div><div class='add'>+		ae_src = IRDMA_AE_SOURCE_RSVD;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_ROCE_EMPTY_MCG:</div><div class='add'>+	case IRDMA_AE_ROCE_BAD_MC_IP_ADDR:</div><div class='add'>+	case IRDMA_AE_ROCE_BAD_MC_QPID:</div><div class='add'>+	case IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH:</div><div class='add'>+		fallthrough;</div><div class='add'>+	case IRDMA_AE_LLP_CONNECTION_RESET:</div><div class='add'>+	case IRDMA_AE_LLP_SYN_RECEIVED:</div><div class='add'>+	case IRDMA_AE_LLP_FIN_RECEIVED:</div><div class='add'>+	case IRDMA_AE_LLP_CLOSE_COMPLETE:</div><div class='add'>+	case IRDMA_AE_LLP_TERMINATE_RECEIVED:</div><div class='add'>+	case IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE:</div><div class='add'>+		ae_src = IRDMA_AE_SOURCE_RSVD;</div><div class='add'>+		info-&gt;qp = true;</div><div class='add'>+		info-&gt;compl_ctx = compl_ctx;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (ae_src) {</div><div class='add'>+	case IRDMA_AE_SOURCE_RQ:</div><div class='add'>+	case IRDMA_AE_SOURCE_RQ_0011:</div><div class='add'>+		info-&gt;qp = true;</div><div class='add'>+		info-&gt;rq = true;</div><div class='add'>+		info-&gt;compl_ctx = compl_ctx;</div><div class='add'>+		info-&gt;err_rq_idx_valid = true;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_SOURCE_CQ:</div><div class='add'>+	case IRDMA_AE_SOURCE_CQ_0110:</div><div class='add'>+	case IRDMA_AE_SOURCE_CQ_1010:</div><div class='add'>+	case IRDMA_AE_SOURCE_CQ_1110:</div><div class='add'>+		info-&gt;cq = true;</div><div class='add'>+		info-&gt;compl_ctx = compl_ctx &lt;&lt; 1;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_SOURCE_SQ:</div><div class='add'>+	case IRDMA_AE_SOURCE_SQ_0111:</div><div class='add'>+		info-&gt;qp = true;</div><div class='add'>+		info-&gt;sq = true;</div><div class='add'>+		info-&gt;compl_ctx = compl_ctx;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_SOURCE_IN_RR_WR:</div><div class='add'>+		info-&gt;qp = true;</div><div class='add'>+		if (aeq-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3)</div><div class='add'>+			info-&gt;err_rq_idx_valid = true;</div><div class='add'>+		info-&gt;compl_ctx = compl_ctx;</div><div class='add'>+		info-&gt;in_rdrsp_wr = true;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_SOURCE_IN_RR_WR_1011:</div><div class='add'>+		info-&gt;qp = true;</div><div class='add'>+		if (aeq-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3) {</div><div class='add'>+			info-&gt;sq = true;</div><div class='add'>+			info-&gt;err_rq_idx_valid = true;</div><div class='add'>+		}</div><div class='add'>+		info-&gt;compl_ctx = compl_ctx;</div><div class='add'>+		info-&gt;in_rdrsp_wr = true;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_SOURCE_OUT_RR:</div><div class='add'>+	case IRDMA_AE_SOURCE_OUT_RR_1111:</div><div class='add'>+		info-&gt;qp = true;</div><div class='add'>+		info-&gt;compl_ctx = compl_ctx;</div><div class='add'>+		info-&gt;out_rdrsp = true;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_SOURCE_RSVD:</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	IRDMA_RING_MOVE_TAIL(aeq-&gt;aeq_ring);</div><div class='add'>+	if (!IRDMA_RING_CURRENT_TAIL(aeq-&gt;aeq_ring))</div><div class='add'>+		aeq-&gt;polarity ^= 1;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_repost_aeq_entries - repost completed aeq entries</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @count: allocate count</div><div class='add'>+ */</div><div class='add'>+void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count)</div><div class='add'>+{</div><div class='add'>+	writel(count, dev-&gt;hw_regs[IRDMA_AEQALLOC]);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_ccq_init - initialize control cq</div><div class='add'>+ * @cq: sc's cq ctruct</div><div class='add'>+ * @info: info for control cq initialization</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_ccq_init(struct irdma_sc_cq *cq, struct irdma_ccq_init_info *info)</div><div class='add'>+{</div><div class='add'>+	u32 pble_obj_cnt;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;num_elem &lt; info-&gt;dev-&gt;hw_attrs.uk_attrs.min_hw_cq_size ||</div><div class='add'>+	    info-&gt;num_elem &gt; info-&gt;dev-&gt;hw_attrs.uk_attrs.max_hw_cq_size)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;ceq_id &gt;= info-&gt;dev-&gt;hmc_fpm_misc.max_ceqs)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	pble_obj_cnt = info-&gt;dev-&gt;hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].cnt;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;virtual_map &amp;&amp; info-&gt;first_pm_pbl_idx &gt;= pble_obj_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	cq-&gt;cq_pa = info-&gt;cq_pa;</div><div class='add'>+	cq-&gt;cq_uk.cq_base = info-&gt;cq_base;</div><div class='add'>+	cq-&gt;shadow_area_pa = info-&gt;shadow_area_pa;</div><div class='add'>+	cq-&gt;cq_uk.shadow_area = info-&gt;shadow_area;</div><div class='add'>+	cq-&gt;shadow_read_threshold = info-&gt;shadow_read_threshold;</div><div class='add'>+	cq-&gt;dev = info-&gt;dev;</div><div class='add'>+	cq-&gt;ceq_id = info-&gt;ceq_id;</div><div class='add'>+	cq-&gt;cq_uk.cq_size = info-&gt;num_elem;</div><div class='add'>+	cq-&gt;cq_type = IRDMA_CQ_TYPE_CQP;</div><div class='add'>+	cq-&gt;ceqe_mask = info-&gt;ceqe_mask;</div><div class='add'>+	IRDMA_RING_INIT(cq-&gt;cq_uk.cq_ring, info-&gt;num_elem);</div><div class='add'>+	cq-&gt;cq_uk.cq_id = 0; /* control cq is id 0 always */</div><div class='add'>+	cq-&gt;ceq_id_valid = info-&gt;ceq_id_valid;</div><div class='add'>+	cq-&gt;tph_en = info-&gt;tph_en;</div><div class='add'>+	cq-&gt;tph_val = info-&gt;tph_val;</div><div class='add'>+	cq-&gt;cq_uk.avoid_mem_cflct = info-&gt;avoid_mem_cflct;</div><div class='add'>+	cq-&gt;pbl_list = info-&gt;pbl_list;</div><div class='add'>+	cq-&gt;virtual_map = info-&gt;virtual_map;</div><div class='add'>+	cq-&gt;pbl_chunk_size = info-&gt;pbl_chunk_size;</div><div class='add'>+	cq-&gt;first_pm_pbl_idx = info-&gt;first_pm_pbl_idx;</div><div class='add'>+	cq-&gt;cq_uk.polarity = true;</div><div class='add'>+	cq-&gt;vsi = info-&gt;vsi;</div><div class='add'>+	cq-&gt;cq_uk.cq_ack_db = cq-&gt;dev-&gt;cq_ack_db;</div><div class='add'>+</div><div class='add'>+	/* Only applicable to CQs other than CCQ so initialize to zero */</div><div class='add'>+	cq-&gt;cq_uk.cqe_alloc_db = NULL;</div><div class='add'>+</div><div class='add'>+	info-&gt;dev-&gt;ccq = cq;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_ccq_create_done - poll cqp for ccq create</div><div class='add'>+ * @ccq: ccq sc struct</div><div class='add'>+ */</div><div class='add'>+static inline int irdma_sc_ccq_create_done(struct irdma_sc_cq *ccq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+</div><div class='add'>+	cqp = ccq-&gt;dev-&gt;cqp;</div><div class='add'>+</div><div class='add'>+	return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CQ, NULL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_ccq_create - create control cq</div><div class='add'>+ * @ccq: ccq sc struct</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @check_overflow: overlow flag for ccq</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch,</div><div class='add'>+			bool check_overflow, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	int ret_code;</div><div class='add'>+</div><div class='add'>+	ret_code = irdma_sc_cq_create(ccq, scratch, check_overflow, post_sq);</div><div class='add'>+	if (ret_code)</div><div class='add'>+		return ret_code;</div><div class='add'>+</div><div class='add'>+	if (post_sq) {</div><div class='add'>+		ret_code = irdma_sc_ccq_create_done(ccq);</div><div class='add'>+		if (ret_code)</div><div class='add'>+			return ret_code;</div><div class='add'>+	}</div><div class='add'>+	ccq-&gt;dev-&gt;cqp-&gt;process_cqp_sds = irdma_cqp_sds_cmd;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_ccq_destroy - destroy ccq during close</div><div class='add'>+ * @ccq: ccq sc struct</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+	u32 tail, val, error;</div><div class='add'>+</div><div class='add'>+	cqp = ccq-&gt;dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0, ccq-&gt;cq_uk.cq_size);</div><div class='add'>+	set_64bit_val(wqe, 8, (uintptr_t)ccq &gt;&gt; 1);</div><div class='add'>+	set_64bit_val(wqe, 40, ccq-&gt;shadow_area_pa);</div><div class='add'>+</div><div class='add'>+	hdr = ccq-&gt;cq_uk.cq_id |</div><div class='add'>+	      FLD_LS_64(ccq-&gt;dev, (ccq-&gt;ceq_id_valid ? ccq-&gt;ceq_id : 0),</div><div class='add'>+			IRDMA_CQPSQ_CQ_CEQID) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CQ) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, ccq-&gt;ceqe_mask) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, ccq-&gt;ceq_id_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, ccq-&gt;tph_en) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, ccq-&gt;cq_uk.avoid_mem_cflct) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: CCQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	irdma_get_cqp_reg_info(cqp, &amp;val, &amp;tail, &amp;error);</div><div class='add'>+</div><div class='add'>+	if (post_sq) {</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+		ret_code = irdma_cqp_poll_registers(cqp, tail,</div><div class='add'>+						    cqp-&gt;dev-&gt;hw_attrs.max_done_count);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cqp-&gt;process_cqp_sds = irdma_update_sds_noccq;</div><div class='add'>+</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info</div><div class='add'>+ * @dev : ptr to irdma_dev struct</div><div class='add'>+ * @hmc_fn_id: hmc function id</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u8 hmc_fn_id)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_info *hmc_info;</div><div class='add'>+	struct irdma_hmc_fpm_misc *hmc_fpm_misc;</div><div class='add'>+	struct irdma_dma_mem query_fpm_mem;</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+	u8 wait_type;</div><div class='add'>+</div><div class='add'>+	hmc_info = dev-&gt;hmc_info;</div><div class='add'>+	hmc_fpm_misc = &amp;dev-&gt;hmc_fpm_misc;</div><div class='add'>+	query_fpm_mem.pa = dev-&gt;fpm_query_buf_pa;</div><div class='add'>+	query_fpm_mem.va = dev-&gt;fpm_query_buf;</div><div class='add'>+	hmc_info-&gt;hmc_fn_id = hmc_fn_id;</div><div class='add'>+	wait_type = (u8)IRDMA_CQP_WAIT_POLL_REGS;</div><div class='add'>+</div><div class='add'>+	ret_code = irdma_sc_query_fpm_val(dev-&gt;cqp, 0, hmc_info-&gt;hmc_fn_id,</div><div class='add'>+					  &amp;query_fpm_mem, true, wait_type);</div><div class='add'>+	if (ret_code)</div><div class='add'>+		return ret_code;</div><div class='add'>+</div><div class='add'>+	/* parse the fpm_query_buf and fill hmc obj info */</div><div class='add'>+	ret_code = irdma_sc_parse_fpm_query_buf(dev, query_fpm_mem.va, hmc_info,</div><div class='add'>+						hmc_fpm_misc);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("HMC: QUERY FPM BUFFER", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, query_fpm_mem.va, IRDMA_QUERY_FPM_BUF_SIZE,</div><div class='add'>+			     false);</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_set_loc_mem() - set a local memory bit field</div><div class='add'>+ * @buf: ptr to a buffer where local memory gets enabled</div><div class='add'>+ */</div><div class='add'>+static void irdma_set_loc_mem(__le64 *buf)</div><div class='add'>+{</div><div class='add'>+	u64 loc_mem_en = BIT_ULL(ENABLE_LOC_MEM);</div><div class='add'>+	u32 offset;</div><div class='add'>+	u64 temp;</div><div class='add'>+</div><div class='add'>+	for (offset = 0; offset &lt; IRDMA_COMMIT_FPM_BUF_SIZE;</div><div class='add'>+	     offset += sizeof(__le64)) {</div><div class='add'>+		if (offset == IRDMA_PBLE_COMMIT_OFFSET)</div><div class='add'>+			continue;</div><div class='add'>+		get_64bit_val(buf, offset, &amp;temp);</div><div class='add'>+		if (temp)</div><div class='add'>+			set_64bit_val(buf, offset, temp | loc_mem_en);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cfg_iw_fpm() - commits hmc obj cnt values using cqp</div><div class='add'>+ * command and populates fpm base address in hmc_info</div><div class='add'>+ * @dev : ptr to irdma_dev struct</div><div class='add'>+ * @hmc_fn_id: hmc function id</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_cfg_iw_fpm(struct irdma_sc_dev *dev, u8 hmc_fn_id)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_info *hmc_info;</div><div class='add'>+	struct irdma_hmc_obj_info *obj_info;</div><div class='add'>+	__le64 *buf;</div><div class='add'>+	struct irdma_dma_mem commit_fpm_mem;</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+	u8 wait_type;</div><div class='add'>+</div><div class='add'>+	hmc_info = dev-&gt;hmc_info;</div><div class='add'>+	obj_info = hmc_info-&gt;hmc_obj;</div><div class='add'>+	buf = dev-&gt;fpm_commit_buf;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(buf, 0, (u64)obj_info[IRDMA_HMC_IW_QP].cnt);</div><div class='add'>+	set_64bit_val(buf, 8, (u64)obj_info[IRDMA_HMC_IW_CQ].cnt);</div><div class='add'>+	set_64bit_val(buf, 16, (u64)obj_info[IRDMA_HMC_IW_SRQ].cnt);</div><div class='add'>+	set_64bit_val(buf, 24, (u64)obj_info[IRDMA_HMC_IW_HTE].cnt);</div><div class='add'>+	set_64bit_val(buf, 32, (u64)obj_info[IRDMA_HMC_IW_ARP].cnt);</div><div class='add'>+	set_64bit_val(buf, 40, (u64)0); /* RSVD */</div><div class='add'>+	set_64bit_val(buf, 48, (u64)obj_info[IRDMA_HMC_IW_MR].cnt);</div><div class='add'>+	set_64bit_val(buf, 56, (u64)obj_info[IRDMA_HMC_IW_XF].cnt);</div><div class='add'>+	set_64bit_val(buf, 64, (u64)obj_info[IRDMA_HMC_IW_XFFL].cnt);</div><div class='add'>+	set_64bit_val(buf, 72, (u64)obj_info[IRDMA_HMC_IW_Q1].cnt);</div><div class='add'>+	set_64bit_val(buf, 80, (u64)obj_info[IRDMA_HMC_IW_Q1FL].cnt);</div><div class='add'>+	set_64bit_val(buf, 88,</div><div class='add'>+		      (u64)obj_info[IRDMA_HMC_IW_TIMER].cnt);</div><div class='add'>+	set_64bit_val(buf, 96,</div><div class='add'>+		      (u64)obj_info[IRDMA_HMC_IW_FSIMC].cnt);</div><div class='add'>+	set_64bit_val(buf, 104,</div><div class='add'>+		      (u64)obj_info[IRDMA_HMC_IW_FSIAV].cnt);</div><div class='add'>+	set_64bit_val(buf, 112,</div><div class='add'>+		      (u64)obj_info[IRDMA_HMC_IW_PBLE].cnt);</div><div class='add'>+	set_64bit_val(buf, 120, (u64)0); /* RSVD */</div><div class='add'>+	set_64bit_val(buf, 128, (u64)obj_info[IRDMA_HMC_IW_RRF].cnt);</div><div class='add'>+	set_64bit_val(buf, 136,</div><div class='add'>+		      (u64)obj_info[IRDMA_HMC_IW_RRFFL].cnt);</div><div class='add'>+	set_64bit_val(buf, 144, (u64)obj_info[IRDMA_HMC_IW_HDR].cnt);</div><div class='add'>+	set_64bit_val(buf, 152, (u64)obj_info[IRDMA_HMC_IW_MD].cnt);</div><div class='add'>+	set_64bit_val(buf, 160,</div><div class='add'>+		      (u64)obj_info[IRDMA_HMC_IW_OOISC].cnt);</div><div class='add'>+	set_64bit_val(buf, 168,</div><div class='add'>+		      (u64)obj_info[IRDMA_HMC_IW_OOISCFFL].cnt);</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3 &amp;&amp;</div><div class='add'>+	    dev-&gt;hmc_fpm_misc.loc_mem_pages)</div><div class='add'>+		irdma_set_loc_mem(buf);</div><div class='add'>+	commit_fpm_mem.pa = dev-&gt;fpm_commit_buf_pa;</div><div class='add'>+	commit_fpm_mem.va = dev-&gt;fpm_commit_buf;</div><div class='add'>+</div><div class='add'>+	wait_type = (u8)IRDMA_CQP_WAIT_POLL_REGS;</div><div class='add'>+	print_hex_dump_debug("HMC: COMMIT FPM BUFFER", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE,</div><div class='add'>+			     false);</div><div class='add'>+	ret_code = irdma_sc_commit_fpm_val(dev-&gt;cqp, 0, hmc_info-&gt;hmc_fn_id,</div><div class='add'>+					   &amp;commit_fpm_mem, true, wait_type);</div><div class='add'>+	if (!ret_code)</div><div class='add'>+		irdma_sc_parse_fpm_commit_buf(dev, dev-&gt;fpm_commit_buf,</div><div class='add'>+					      hmc_info-&gt;hmc_obj,</div><div class='add'>+					      &amp;hmc_info-&gt;sd_table.sd_cnt);</div><div class='add'>+	print_hex_dump_debug("HMC: COMMIT FPM BUFFER", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE,</div><div class='add'>+			     false);</div><div class='add'>+</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * cqp_sds_wqe_fill - fill cqp wqe doe sd</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @info: sd info for wqe</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ */</div><div class='add'>+static int cqp_sds_wqe_fill(struct irdma_sc_cqp *cqp,</div><div class='add'>+			    struct irdma_update_sds_info *info, u64 scratch)</div><div class='add'>+{</div><div class='add'>+	u64 data;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	int mem_entries, wqe_entries;</div><div class='add'>+	struct irdma_dma_mem *sdbuf = &amp;cqp-&gt;sdbuf;</div><div class='add'>+	u64 offset = 0;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &amp;wqe_idx);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	wqe_entries = (info-&gt;cnt &gt; 3) ? 3 : info-&gt;cnt;</div><div class='add'>+	mem_entries = info-&gt;cnt - wqe_entries;</div><div class='add'>+</div><div class='add'>+	if (mem_entries) {</div><div class='add'>+		offset = wqe_idx * IRDMA_UPDATE_SD_BUFF_SIZE;</div><div class='add'>+		memcpy(((char *)sdbuf-&gt;va + offset), &amp;info-&gt;entry[3], mem_entries &lt;&lt; 4);</div><div class='add'>+</div><div class='add'>+		data = (u64)sdbuf-&gt;pa + offset;</div><div class='add'>+	} else {</div><div class='add'>+		data = 0;</div><div class='add'>+	}</div><div class='add'>+	data |= FIELD_PREP(IRDMA_CQPSQ_UPESD_HMCFNID, info-&gt;hmc_fn_id);</div><div class='add'>+	set_64bit_val(wqe, 16, data);</div><div class='add'>+</div><div class='add'>+	switch (wqe_entries) {</div><div class='add'>+	case 3:</div><div class='add'>+		set_64bit_val(wqe, 48,</div><div class='add'>+			      (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info-&gt;entry[2].cmd) |</div><div class='add'>+			       FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_VALID, 1)));</div><div class='add'>+</div><div class='add'>+		set_64bit_val(wqe, 56, info-&gt;entry[2].data);</div><div class='add'>+		fallthrough;</div><div class='add'>+	case 2:</div><div class='add'>+		set_64bit_val(wqe, 32,</div><div class='add'>+			      (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info-&gt;entry[1].cmd) |</div><div class='add'>+			       FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_VALID, 1)));</div><div class='add'>+</div><div class='add'>+		set_64bit_val(wqe, 40, info-&gt;entry[1].data);</div><div class='add'>+		fallthrough;</div><div class='add'>+	case 1:</div><div class='add'>+		set_64bit_val(wqe, 0,</div><div class='add'>+			      FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info-&gt;entry[0].cmd));</div><div class='add'>+</div><div class='add'>+		set_64bit_val(wqe, 8, info-&gt;entry[0].data);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_UPDATE_PE_SDS) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_COUNT, mem_entries);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	if (mem_entries)</div><div class='add'>+		print_hex_dump_debug("WQE: UPDATE_PE_SDS WQE Buffer",</div><div class='add'>+				     DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+				     (char *)sdbuf-&gt;va + offset,</div><div class='add'>+				     mem_entries &lt;&lt; 4, false);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: UPDATE_PE_SDS WQE", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_update_pe_sds - cqp wqe for sd</div><div class='add'>+ * @dev: ptr to irdma_dev struct</div><div class='add'>+ * @info: sd info for sd's</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ */</div><div class='add'>+static int irdma_update_pe_sds(struct irdma_sc_dev *dev,</div><div class='add'>+			       struct irdma_update_sds_info *info, u64 scratch)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_cqp *cqp = dev-&gt;cqp;</div><div class='add'>+	int ret_code;</div><div class='add'>+</div><div class='add'>+	ret_code = cqp_sds_wqe_fill(cqp, info, scratch);</div><div class='add'>+	if (!ret_code)</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_update_sds_noccq - update sd before ccq created</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @info: sd info for sd's</div><div class='add'>+ */</div><div class='add'>+int irdma_update_sds_noccq(struct irdma_sc_dev *dev,</div><div class='add'>+			   struct irdma_update_sds_info *info)</div><div class='add'>+{</div><div class='add'>+	u32 error, val, tail;</div><div class='add'>+	struct irdma_sc_cqp *cqp = dev-&gt;cqp;</div><div class='add'>+	int ret_code;</div><div class='add'>+</div><div class='add'>+	ret_code = cqp_sds_wqe_fill(cqp, info, 0);</div><div class='add'>+	if (ret_code)</div><div class='add'>+		return ret_code;</div><div class='add'>+</div><div class='add'>+	irdma_get_cqp_reg_info(cqp, &amp;val, &amp;tail, &amp;error);</div><div class='add'>+</div><div class='add'>+	irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+	return irdma_cqp_poll_registers(cqp, tail,</div><div class='add'>+					cqp-&gt;dev-&gt;hw_attrs.max_done_count);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ * @hmc_fn_id: hmc function id</div><div class='add'>+ * @post_sq: flag for cqp db to ring</div><div class='add'>+ * @poll_registers: flag to poll register for cqp completion</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,</div><div class='add'>+					u8 hmc_fn_id, bool post_sq,</div><div class='add'>+					bool poll_registers)</div><div class='add'>+{</div><div class='add'>+	u64 hdr;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u32 tail, val, error;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID, hmc_fn_id));</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE,</div><div class='add'>+			 IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: SHMC_PAGES_ALLOCATED WQE",</div><div class='add'>+			     DUMP_PREFIX_OFFSET, 16, 8, wqe,</div><div class='add'>+			     IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	irdma_get_cqp_reg_info(cqp, &amp;val, &amp;tail, &amp;error);</div><div class='add'>+</div><div class='add'>+	if (post_sq) {</div><div class='add'>+		irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+		if (poll_registers)</div><div class='add'>+			/* check for cqp sq tail update */</div><div class='add'>+			return irdma_cqp_poll_registers(cqp, tail,</div><div class='add'>+							cqp-&gt;dev-&gt;hw_attrs.max_done_count);</div><div class='add'>+		else</div><div class='add'>+			return irdma_sc_poll_for_cqp_op_done(cqp,</div><div class='add'>+							     IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED,</div><div class='add'>+							     NULL);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_ring_full - check if cqp ring is full</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ */</div><div class='add'>+static bool irdma_cqp_ring_full(struct irdma_sc_cqp *cqp)</div><div class='add'>+{</div><div class='add'>+	return IRDMA_RING_FULL_ERR(cqp-&gt;sq_ring);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_est_sd - returns approximate number of SDs for HMC</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @hmc_info: hmc structure, size and count for HMC objects</div><div class='add'>+ */</div><div class='add'>+static u32 irdma_est_sd(struct irdma_sc_dev *dev,</div><div class='add'>+			struct irdma_hmc_info *hmc_info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_obj_info *pble_info;</div><div class='add'>+	int i;</div><div class='add'>+	u64 size = 0;</div><div class='add'>+	u64 sd;</div><div class='add'>+</div><div class='add'>+	for (i = IRDMA_HMC_IW_QP; i &lt; IRDMA_HMC_IW_MAX; i++)</div><div class='add'>+		if (i != IRDMA_HMC_IW_PBLE)</div><div class='add'>+			size += round_up(hmc_info-&gt;hmc_obj[i].cnt *</div><div class='add'>+					 hmc_info-&gt;hmc_obj[i].size, 512);</div><div class='add'>+</div><div class='add'>+	pble_info = &amp;hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE];</div><div class='add'>+	if (dev-&gt;privileged)</div><div class='add'>+		size += round_up(pble_info-&gt;cnt * pble_info-&gt;size, 512);</div><div class='add'>+	if (size &amp; 0x1FFFFF)</div><div class='add'>+		sd = (size &gt;&gt; 21) + 1; /* add 1 for remainder */</div><div class='add'>+	else</div><div class='add'>+		sd = size &gt;&gt; 21;</div><div class='add'>+	if (!dev-&gt;privileged &amp;&amp; !dev-&gt;hmc_fpm_misc.loc_mem_pages) {</div><div class='add'>+		/* 2MB alignment for VF PBLE HMC */</div><div class='add'>+		size = pble_info-&gt;cnt * pble_info-&gt;size;</div><div class='add'>+		if (size &amp; 0x1FFFFF)</div><div class='add'>+			sd += (size &gt;&gt; 21) + 1; /* add 1 for remainder */</div><div class='add'>+		else</div><div class='add'>+			sd += size &gt;&gt; 21;</div><div class='add'>+	}</div><div class='add'>+	if (sd &gt; 0xFFFFFFFF) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "HMC: sd overflow[%lld]\n", sd);</div><div class='add'>+		sd = 0xFFFFFFFF - 1;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return (u32)sd;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_query_rdma_features - query RDMA features and FW ver</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @buf: buffer to hold query info</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ */</div><div class='add'>+static int irdma_sc_query_rdma_features(struct irdma_sc_cqp *cqp,</div><div class='add'>+					struct irdma_dma_mem *buf, u64 scratch)</div><div class='add'>+{</div><div class='add'>+	u32 tail, val, error;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	int status;</div><div class='add'>+	u64 temp;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	temp = buf-&gt;pa;</div><div class='add'>+	set_64bit_val(wqe, 32, temp);</div><div class='add'>+</div><div class='add'>+	temp = FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID,</div><div class='add'>+			  cqp-&gt;polarity) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN, buf-&gt;size) |</div><div class='add'>+	       FIELD_PREP(IRDMA_CQPSQ_UP_OP, IRDMA_CQP_OP_QUERY_RDMA_FEATURES);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, temp);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: QUERY RDMA FEATURES", DUMP_PREFIX_OFFSET,</div><div class='add'>+			     16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	irdma_get_cqp_reg_info(cqp, &amp;val, &amp;tail, &amp;error);</div><div class='add'>+</div><div class='add'>+	irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+	status = irdma_cqp_poll_registers(cqp, tail,</div><div class='add'>+					  cqp-&gt;dev-&gt;hw_attrs.max_done_count);</div><div class='add'>+	if (error || status)</div><div class='add'>+		status = -EINVAL;</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_rdma_features - get RDMA features</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ */</div><div class='add'>+int irdma_get_rdma_features(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int ret_code;</div><div class='add'>+	struct irdma_dma_mem feat_buf;</div><div class='add'>+	u64 temp;</div><div class='add'>+	u16 byte_idx, feat_type, feat_cnt, feat_idx;</div><div class='add'>+</div><div class='add'>+	feat_buf.size = ALIGN(IRDMA_FEATURE_BUF_SIZE,</div><div class='add'>+			      IRDMA_FEATURE_BUF_ALIGNMENT);</div><div class='add'>+	feat_buf.va = dma_alloc_coherent(dev-&gt;hw-&gt;device, feat_buf.size,</div><div class='add'>+					 &amp;feat_buf.pa, GFP_KERNEL);</div><div class='add'>+	if (!feat_buf.va)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	ret_code = irdma_sc_query_rdma_features(dev-&gt;cqp, &amp;feat_buf, 0);</div><div class='add'>+	if (ret_code)</div><div class='add'>+		goto exit;</div><div class='add'>+</div><div class='add'>+	get_64bit_val(feat_buf.va, 0, &amp;temp);</div><div class='add'>+	feat_cnt = (u16)FIELD_GET(IRDMA_FEATURE_CNT, temp);</div><div class='add'>+	if (feat_cnt &lt; 2) {</div><div class='add'>+		ret_code = -EINVAL;</div><div class='add'>+		goto exit;</div><div class='add'>+	} else if (feat_cnt &gt; IRDMA_MAX_FEATURES) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "DEV: feature buf size insufficient, retrying with larger buffer\n");</div><div class='add'>+		dma_free_coherent(dev-&gt;hw-&gt;device, feat_buf.size, feat_buf.va,</div><div class='add'>+				  feat_buf.pa);</div><div class='add'>+		feat_buf.va = NULL;</div><div class='add'>+		feat_buf.size = ALIGN(8 * feat_cnt,</div><div class='add'>+				      IRDMA_FEATURE_BUF_ALIGNMENT);</div><div class='add'>+		feat_buf.va = dma_alloc_coherent(dev-&gt;hw-&gt;device,</div><div class='add'>+						 feat_buf.size, &amp;feat_buf.pa,</div><div class='add'>+						 GFP_KERNEL);</div><div class='add'>+		if (!feat_buf.va)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+</div><div class='add'>+		ret_code = irdma_sc_query_rdma_features(dev-&gt;cqp, &amp;feat_buf, 0);</div><div class='add'>+		if (ret_code)</div><div class='add'>+			goto exit;</div><div class='add'>+</div><div class='add'>+		get_64bit_val(feat_buf.va, 0, &amp;temp);</div><div class='add'>+		feat_cnt = (u16)FIELD_GET(IRDMA_FEATURE_CNT, temp);</div><div class='add'>+		if (feat_cnt &lt; 2) {</div><div class='add'>+			ret_code = -EINVAL;</div><div class='add'>+			goto exit;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: QUERY RDMA FEATURES", DUMP_PREFIX_OFFSET,</div><div class='add'>+			     16, 8, feat_buf.va, feat_cnt * 8, false);</div><div class='add'>+</div><div class='add'>+	for (byte_idx = 0, feat_idx = 0; feat_idx &lt; min(feat_cnt, (u16)IRDMA_MAX_FEATURES);</div><div class='add'>+	     feat_idx++, byte_idx += 8) {</div><div class='add'>+		get_64bit_val(feat_buf.va, byte_idx, &amp;temp);</div><div class='add'>+		feat_type = FIELD_GET(IRDMA_FEATURE_TYPE, temp);</div><div class='add'>+		if (feat_type &gt;= IRDMA_MAX_FEATURES) {</div><div class='add'>+			ibdev_dbg(to_ibdev(dev),</div><div class='add'>+				  "DEV: found unrecognized feature type %d\n",</div><div class='add'>+				  feat_type);</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+		dev-&gt;feature_info[feat_type] = temp;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;feature_info[IRDMA_FTN_FLAGS] &amp; IRDMA_ATOMICS_ALLOWED_BIT)</div><div class='add'>+		dev-&gt;hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_ATOMIC_OPS;</div><div class='add'>+</div><div class='add'>+exit:</div><div class='add'>+	dma_free_coherent(dev-&gt;hw-&gt;device, feat_buf.size, feat_buf.va,</div><div class='add'>+			  feat_buf.pa);</div><div class='add'>+	feat_buf.va = NULL;</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u32 irdma_q1_cnt(struct irdma_sc_dev *dev,</div><div class='add'>+			struct irdma_hmc_info *hmc_info, u32 qpwanted)</div><div class='add'>+{</div><div class='add'>+	u32 q1_cnt;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {</div><div class='add'>+		q1_cnt = roundup_pow_of_two(dev-&gt;hw_attrs.max_hw_ird * 2 * qpwanted);</div><div class='add'>+	} else {</div><div class='add'>+		if (dev-&gt;cqp-&gt;protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)</div><div class='add'>+			q1_cnt = roundup_pow_of_two(dev-&gt;hw_attrs.max_hw_ird * 2 * qpwanted + 512);</div><div class='add'>+		else</div><div class='add'>+			q1_cnt = dev-&gt;hw_attrs.max_hw_ird * 2 * qpwanted;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return q1_cnt;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void cfg_fpm_value_gen_1(struct irdma_sc_dev *dev,</div><div class='add'>+				struct irdma_hmc_info *hmc_info, u32 qpwanted)</div><div class='add'>+{</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_XF].cnt = roundup_pow_of_two(qpwanted * dev-&gt;hw_attrs.max_hw_wqes);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void cfg_fpm_value_gen_2(struct irdma_sc_dev *dev,</div><div class='add'>+				struct irdma_hmc_info *hmc_info, u32 qpwanted)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_fpm_misc *hmc_fpm_misc = &amp;dev-&gt;hmc_fpm_misc;</div><div class='add'>+</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_XF].cnt =</div><div class='add'>+		4 * hmc_fpm_misc-&gt;xf_block_size * qpwanted;</div><div class='add'>+</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_HDR].cnt = qpwanted;</div><div class='add'>+</div><div class='add'>+	if (hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_RRF].max_cnt)</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_RRF].cnt = 32 * qpwanted;</div><div class='add'>+	if (hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_RRFFL].max_cnt)</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_RRFFL].cnt =</div><div class='add'>+			hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_RRF].cnt /</div><div class='add'>+			hmc_fpm_misc-&gt;rrf_block_size;</div><div class='add'>+	if (hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_OOISC].max_cnt)</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_OOISC].cnt = 32 * qpwanted;</div><div class='add'>+	if (hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_OOISCFFL].max_cnt)</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_OOISCFFL].cnt =</div><div class='add'>+			hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_OOISC].cnt /</div><div class='add'>+			hmc_fpm_misc-&gt;ooiscf_block_size;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_rsrc_mem_config - configure resources if local memory or host</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @is_mrte_loc_mem: if true, MR's to be in local memory because sd=loc pages</div><div class='add'>+ *</div><div class='add'>+ * Only mr can be configured host or local memory if qp's are in local memory.</div><div class='add'>+ * If qp is in local memory, then all resource object will be in local memory</div><div class='add'>+ * except mr which can be either host or local memory.  The only exception</div><div class='add'>+ * is pble's which are always in host memory.</div><div class='add'>+ */</div><div class='add'>+static void irdma_get_rsrc_mem_config(struct irdma_sc_dev *dev, bool is_mrte_loc_mem)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_info *hmc_info = dev-&gt;hmc_info;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = IRDMA_HMC_IW_QP; i &lt; IRDMA_HMC_IW_MAX; i++)</div><div class='add'>+		hmc_info-&gt;hmc_obj[i].mem_loc = IRDMA_LOC_MEM;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;feature_info[IRDMA_OBJ_1] &amp;&amp; !is_mrte_loc_mem) {</div><div class='add'>+		u8 mem_type;</div><div class='add'>+</div><div class='add'>+		mem_type = (u8)FIELD_GET(IRDMA_MR_MEM_LOC, dev-&gt;feature_info[IRDMA_OBJ_1]);</div><div class='add'>+</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_MR].mem_loc =</div><div class='add'>+			(mem_type &amp; IRDMA_OBJ_LOC_MEM_BIT) ?</div><div class='add'>+			IRDMA_LOC_MEM : IRDMA_HOST_MEM;</div><div class='add'>+	} else {</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_MR].mem_loc = IRDMA_LOC_MEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].mem_loc = IRDMA_HOST_MEM;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(to_ibdev(dev), "HMC: INFO: mrte_mem_loc = %d pble = %d\n",</div><div class='add'>+		  hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_MR].mem_loc,</div><div class='add'>+		  hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].mem_loc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cfg_sd_mem - allocate sd memory</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @hmc_info: ptr to irdma_hmc_obj_info struct</div><div class='add'>+ */</div><div class='add'>+static int irdma_cfg_sd_mem(struct irdma_sc_dev *dev,</div><div class='add'>+			    struct irdma_hmc_info *hmc_info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_virt_mem virt_mem;</div><div class='add'>+	u32 mem_size;</div><div class='add'>+</div><div class='add'>+	mem_size = sizeof(struct irdma_hmc_sd_entry) * hmc_info-&gt;sd_table.sd_cnt;</div><div class='add'>+	virt_mem.size = mem_size;</div><div class='add'>+	virt_mem.va = kzalloc(virt_mem.size, GFP_KERNEL);</div><div class='add'>+	if (!virt_mem.va)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	hmc_info-&gt;sd_table.sd_entry = virt_mem.va;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_objs_pages - get number of 2M pages needed</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @hmc_info: pointer to the HMC configuration information struct</div><div class='add'>+ * @mem_loc: pages for local or host memory</div><div class='add'>+ */</div><div class='add'>+static u32 irdma_get_objs_pages(struct irdma_sc_dev *dev,</div><div class='add'>+				struct irdma_hmc_info *hmc_info,</div><div class='add'>+				enum irdma_hmc_obj_mem mem_loc)</div><div class='add'>+{</div><div class='add'>+	u64 size = 0;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = IRDMA_HMC_IW_QP; i &lt; IRDMA_HMC_IW_MAX; i++) {</div><div class='add'>+		if (hmc_info-&gt;hmc_obj[i].mem_loc == mem_loc) {</div><div class='add'>+			size += round_up(hmc_info-&gt;hmc_obj[i].cnt *</div><div class='add'>+					 hmc_info-&gt;hmc_obj[i].size, 512);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return DIV_ROUND_UP(size, IRDMA_HMC_PAGE_SIZE);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_set_host_hmc_rsrc_gen_3 - calculate host hmc resources for gen 3</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ */</div><div class='add'>+static void irdma_set_host_hmc_rsrc_gen_3(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_fpm_misc *hmc_fpm_misc;</div><div class='add'>+	struct irdma_hmc_info *hmc_info;</div><div class='add'>+	enum irdma_hmc_obj_mem mrte_loc;</div><div class='add'>+	u32 mrwanted, pblewanted;</div><div class='add'>+	u32  avail_sds, mr_sds;</div><div class='add'>+</div><div class='add'>+	hmc_info = dev-&gt;hmc_info;</div><div class='add'>+	hmc_fpm_misc = &amp;dev-&gt;hmc_fpm_misc;</div><div class='add'>+	avail_sds = hmc_fpm_misc-&gt;max_sds;</div><div class='add'>+	mrte_loc = hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_MR].mem_loc;</div><div class='add'>+	mrwanted = hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_MR].cnt;</div><div class='add'>+	pblewanted = hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt;</div><div class='add'>+</div><div class='add'>+	if (mrte_loc == IRDMA_HOST_MEM &amp;&amp; avail_sds &gt; IRDMA_MIN_PBLE_PAGES) {</div><div class='add'>+		mr_sds = avail_sds - IRDMA_MIN_PBLE_PAGES;</div><div class='add'>+		mrwanted = min(mrwanted, mr_sds * MAX_MR_PER_SD);</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_MR].cnt = mrwanted;</div><div class='add'>+		avail_sds -= DIV_ROUND_UP(mrwanted, MAX_MR_PER_SD);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (FIELD_GET(IRDMA_MANAGE_RSRC_VER2, dev-&gt;feature_info[IRDMA_FTN_FLAGS]) &amp;&amp;</div><div class='add'>+	    pblewanted &gt; avail_sds * MAX_PBLE_PER_SD)</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "HMC: Warn: Resource version 2: pble wanted = 0x%x available = 0x%x\n",</div><div class='add'>+			  pblewanted, avail_sds * MAX_PBLE_PER_SD);</div><div class='add'>+</div><div class='add'>+	pblewanted = min(pblewanted, avail_sds * MAX_PBLE_PER_SD);</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_verify_commit_fpm_gen_3 - verify query fpm values</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @max_pages: max local memory available</div><div class='add'>+ * @qpwanted: number of qp's wanted</div><div class='add'>+ */</div><div class='add'>+static int irdma_verify_commit_fpm_gen_3(struct irdma_sc_dev *dev,</div><div class='add'>+					 u32 max_pages,</div><div class='add'>+					 u32 qpwanted)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_fpm_misc *hmc_fpm_misc;</div><div class='add'>+	u32 rrf_cnt, xf_cnt, timer_cnt, pages_needed;</div><div class='add'>+	struct irdma_hmc_info *hmc_info;</div><div class='add'>+	u32 rrffl_cnt = 0;</div><div class='add'>+	u32 xffl_cnt = 0;</div><div class='add'>+	u32 q1fl_cnt;</div><div class='add'>+</div><div class='add'>+	hmc_info = dev-&gt;hmc_info;</div><div class='add'>+	hmc_fpm_misc = &amp;dev-&gt;hmc_fpm_misc;</div><div class='add'>+</div><div class='add'>+	rrf_cnt = roundup_pow_of_two(IRDMA_RRF_MULTIPLIER * qpwanted);</div><div class='add'>+</div><div class='add'>+	if (hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_RRFFL].max_cnt)</div><div class='add'>+		rrffl_cnt =</div><div class='add'>+			hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_RRF].cnt /</div><div class='add'>+			hmc_fpm_misc-&gt;rrf_block_size;</div><div class='add'>+</div><div class='add'>+	xf_cnt = roundup_pow_of_two(IRDMA_XF_MULTIPLIER * qpwanted);</div><div class='add'>+</div><div class='add'>+	if (xf_cnt)</div><div class='add'>+		xffl_cnt = xf_cnt / hmc_fpm_misc-&gt;xf_block_size;</div><div class='add'>+</div><div class='add'>+	timer_cnt = (round_up(qpwanted, 512) / 512 + 1) *</div><div class='add'>+		hmc_fpm_misc-&gt;timer_bucket;</div><div class='add'>+</div><div class='add'>+	q1fl_cnt = hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_Q1].cnt / hmc_fpm_misc-&gt;q1_block_size;</div><div class='add'>+</div><div class='add'>+	pages_needed = irdma_get_objs_pages(dev, hmc_info, IRDMA_LOC_MEM);</div><div class='add'>+	if (pages_needed &gt; max_pages) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "HMC: FAIL: SW counts rrf_cnt = %u rrffl_cnt = %u timer_cnt = %u",</div><div class='add'>+			  rrf_cnt, rrffl_cnt, timer_cnt);</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "HMC: FAIL: SW counts xf_cnt = %u xffl_cnt = %u q1fl_cnt = %u",</div><div class='add'>+			  xf_cnt, xffl_cnt, q1fl_cnt);</div><div class='add'>+</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hmc_fpm_misc-&gt;max_sds -= pages_needed;</div><div class='add'>+	hmc_fpm_misc-&gt;loc_mem_pages -= pages_needed;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_set_loc_hmc_rsrc_gen_3 - calculate hmc resources for gen 3</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @max_pages: max local memory available</div><div class='add'>+ * @qpwanted: number of qp's wanted</div><div class='add'>+ */</div><div class='add'>+static int irdma_set_loc_hmc_rsrc_gen_3(struct irdma_sc_dev *dev,</div><div class='add'>+					u32 max_pages,</div><div class='add'>+					u32 qpwanted)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_fpm_misc *hmc_fpm_misc;</div><div class='add'>+	u32 rrf_cnt, xf_cnt, timer_cnt, pages_needed;</div><div class='add'>+	struct irdma_hmc_info *hmc_info;</div><div class='add'>+	u32 ird, ord;</div><div class='add'>+</div><div class='add'>+	if (FIELD_GET(IRDMA_MANAGE_RSRC_VER2, dev-&gt;feature_info[IRDMA_FTN_FLAGS]))</div><div class='add'>+		return irdma_verify_commit_fpm_gen_3(dev, max_pages, qpwanted);</div><div class='add'>+</div><div class='add'>+	hmc_info = dev-&gt;hmc_info;</div><div class='add'>+	hmc_fpm_misc = &amp;dev-&gt;hmc_fpm_misc;</div><div class='add'>+	ird = dev-&gt;hw_attrs.max_hw_ird;</div><div class='add'>+	ord = dev-&gt;hw_attrs.max_hw_ord;</div><div class='add'>+</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_HDR].cnt = qpwanted;</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_QP].cnt = qpwanted;</div><div class='add'>+</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_CQ].cnt =</div><div class='add'>+		min(hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_CQ].cnt, qpwanted * 2);</div><div class='add'>+</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIAV].cnt =</div><div class='add'>+		min(qpwanted * 8, hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt);</div><div class='add'>+</div><div class='add'>+	rrf_cnt = roundup_pow_of_two(IRDMA_RRF_MULTIPLIER * qpwanted);</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_RRF].cnt =</div><div class='add'>+		min(hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_RRF].max_cnt, rrf_cnt);</div><div class='add'>+</div><div class='add'>+	if (hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_RRFFL].max_cnt)</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_RRFFL].cnt =</div><div class='add'>+			hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_RRF].cnt /</div><div class='add'>+			hmc_fpm_misc-&gt;rrf_block_size;</div><div class='add'>+</div><div class='add'>+	xf_cnt = roundup_pow_of_two(IRDMA_XF_MULTIPLIER * qpwanted);</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_XF].cnt =</div><div class='add'>+		min(hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_XF].max_cnt, xf_cnt);</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_XFFL].cnt =</div><div class='add'>+			xf_cnt / hmc_fpm_misc-&gt;xf_block_size;</div><div class='add'>+</div><div class='add'>+	timer_cnt = (round_up(qpwanted, 512) / 512 + 1) *</div><div class='add'>+		hmc_fpm_misc-&gt;timer_bucket;</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_TIMER].cnt =</div><div class='add'>+		min(timer_cnt, hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_TIMER].cnt);</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_Q1].cnt = roundup_pow_of_two(ird * 2 * qpwanted);</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_Q1FL].cnt =</div><div class='add'>+			hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_Q1].cnt / hmc_fpm_misc-&gt;q1_block_size;</div><div class='add'>+</div><div class='add'>+		pages_needed = irdma_get_objs_pages(dev, hmc_info, IRDMA_LOC_MEM);</div><div class='add'>+		if (pages_needed &lt;= max_pages)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		ird /= 2;</div><div class='add'>+		ord /= 2;</div><div class='add'>+	} while (ird &gt;= IRDMA_MIN_IRD);</div><div class='add'>+</div><div class='add'>+	if (ird &lt; IRDMA_MIN_IRD) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "HMC: FAIL: IRD=%u Q1 CNT = %u\n",</div><div class='add'>+			  ird, hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_Q1].cnt);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_ird = ird;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_ord = ord;</div><div class='add'>+	hmc_fpm_misc-&gt;max_sds -= pages_needed;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * cfg_fpm_value_gen_3 - configure fpm for gen 3</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @hmc_info: ptr to irdma_hmc_obj_info struct</div><div class='add'>+ * @hmc_fpm_misc: ptr to fpm data</div><div class='add'>+ */</div><div class='add'>+static int cfg_fpm_value_gen_3(struct irdma_sc_dev *dev,</div><div class='add'>+			       struct irdma_hmc_info *hmc_info,</div><div class='add'>+			       struct irdma_hmc_fpm_misc *hmc_fpm_misc)</div><div class='add'>+{</div><div class='add'>+	enum irdma_hmc_obj_mem mrte_loc;</div><div class='add'>+	u32 mrwanted,  qpwanted;</div><div class='add'>+	int i, ret_code = 0;</div><div class='add'>+	u32 loc_mem_pages;</div><div class='add'>+	bool is_mrte_loc_mem;</div><div class='add'>+</div><div class='add'>+	loc_mem_pages = hmc_fpm_misc-&gt;loc_mem_pages;</div><div class='add'>+	is_mrte_loc_mem = hmc_fpm_misc-&gt;loc_mem_pages == hmc_fpm_misc-&gt;max_sds ?</div><div class='add'>+			true : false;</div><div class='add'>+</div><div class='add'>+	irdma_get_rsrc_mem_config(dev, is_mrte_loc_mem);</div><div class='add'>+	mrte_loc = hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_MR].mem_loc;</div><div class='add'>+</div><div class='add'>+	if (is_mrte_loc_mem)</div><div class='add'>+		loc_mem_pages -= IRDMA_MIN_PBLE_PAGES;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(to_ibdev(dev),</div><div class='add'>+		  "HMC: mrte_loc %d loc_mem %u fpm max sds %u host_obj %d\n",</div><div class='add'>+		  hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_MR].mem_loc,</div><div class='add'>+		  hmc_fpm_misc-&gt;loc_mem_pages, hmc_fpm_misc-&gt;max_sds,</div><div class='add'>+		  is_mrte_loc_mem);</div><div class='add'>+</div><div class='add'>+	mrwanted = hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_MR].max_cnt;</div><div class='add'>+	qpwanted = hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_QP].max_cnt;</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_HDR].cnt = qpwanted;</div><div class='add'>+</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_OOISC].max_cnt = 0;</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_OOISCFFL].max_cnt = 0;</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_HTE].max_cnt = 0;</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt = 0;</div><div class='add'>+</div><div class='add'>+	if (!FIELD_GET(IRDMA_MANAGE_RSRC_VER2, dev-&gt;feature_info[IRDMA_FTN_FLAGS]))</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt =</div><div class='add'>+			min(hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt,</div><div class='add'>+			(u32)IRDMA_FSIAV_CNT_MAX);</div><div class='add'>+</div><div class='add'>+	for (i = IRDMA_HMC_IW_QP; i &lt; IRDMA_HMC_IW_MAX; i++)</div><div class='add'>+		hmc_info-&gt;hmc_obj[i].cnt = hmc_info-&gt;hmc_obj[i].max_cnt;</div><div class='add'>+</div><div class='add'>+	while (qpwanted &gt;= IRDMA_MIN_QP_CNT) {</div><div class='add'>+		if (!irdma_set_loc_hmc_rsrc_gen_3(dev, loc_mem_pages, qpwanted))</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		if (FIELD_GET(IRDMA_MANAGE_RSRC_VER2, dev-&gt;feature_info[IRDMA_FTN_FLAGS]))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		qpwanted /= 2;</div><div class='add'>+		if (mrte_loc == IRDMA_LOC_MEM) {</div><div class='add'>+			mrwanted = qpwanted * IRDMA_MIN_MR_PER_QP;</div><div class='add'>+			hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_MR].cnt =</div><div class='add'>+				min(hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_MR].max_cnt, mrwanted);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qpwanted &lt; IRDMA_MIN_QP_CNT) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "HMC: ERROR: could not allocate fpm resources\n");</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_set_host_hmc_rsrc_gen_3(dev);</div><div class='add'>+	ret_code = irdma_sc_cfg_iw_fpm(dev, dev-&gt;hmc_fn_id);</div><div class='add'>+	if (ret_code) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "HMC: cfg_iw_fpm returned error_code[x%08X]\n",</div><div class='add'>+			  readl(dev-&gt;hw_regs[IRDMA_CQPERRCODES]));</div><div class='add'>+</div><div class='add'>+		return ret_code;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return irdma_cfg_sd_mem(dev, hmc_info);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cfg_fpm_val - configure HMC objects</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @qp_count: desired qp count</div><div class='add'>+ */</div><div class='add'>+int irdma_cfg_fpm_val(struct irdma_sc_dev *dev, u32 qp_count)</div><div class='add'>+{</div><div class='add'>+	u32 qpwanted, mrwanted, pblewanted;</div><div class='add'>+	u32 powerof2, hte, i;</div><div class='add'>+	u32 sd_needed;</div><div class='add'>+	u32 sd_diff;</div><div class='add'>+	u32 loop_count = 0;</div><div class='add'>+	struct irdma_hmc_info *hmc_info;</div><div class='add'>+	struct irdma_hmc_fpm_misc *hmc_fpm_misc;</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+	u32 max_sds;</div><div class='add'>+</div><div class='add'>+	hmc_info = dev-&gt;hmc_info;</div><div class='add'>+	hmc_fpm_misc = &amp;dev-&gt;hmc_fpm_misc;</div><div class='add'>+</div><div class='add'>+	ret_code = irdma_sc_init_iw_hmc(dev, dev-&gt;hmc_fn_id);</div><div class='add'>+	if (ret_code) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "HMC: irdma_sc_init_iw_hmc returned error_code = %d\n",</div><div class='add'>+			  ret_code);</div><div class='add'>+		return ret_code;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	max_sds = hmc_fpm_misc-&gt;max_sds;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3)</div><div class='add'>+		return cfg_fpm_value_gen_3(dev, hmc_info, hmc_fpm_misc);</div><div class='add'>+</div><div class='add'>+	for (i = IRDMA_HMC_IW_QP; i &lt; IRDMA_HMC_IW_MAX; i++)</div><div class='add'>+		hmc_info-&gt;hmc_obj[i].cnt = hmc_info-&gt;hmc_obj[i].max_cnt;</div><div class='add'>+	sd_needed = irdma_est_sd(dev, hmc_info);</div><div class='add'>+	ibdev_dbg(to_ibdev(dev), "HMC: sd count %u where max sd is %u\n",</div><div class='add'>+		  hmc_info-&gt;sd_table.sd_cnt, max_sds);</div><div class='add'>+</div><div class='add'>+	qpwanted = min(qp_count, hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_QP].max_cnt);</div><div class='add'>+</div><div class='add'>+	powerof2 = 1;</div><div class='add'>+	while (powerof2 &lt;= qpwanted)</div><div class='add'>+		powerof2 *= 2;</div><div class='add'>+	powerof2 /= 2;</div><div class='add'>+	qpwanted = powerof2;</div><div class='add'>+</div><div class='add'>+	mrwanted = hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_MR].max_cnt;</div><div class='add'>+	pblewanted = hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(to_ibdev(dev),</div><div class='add'>+		  "HMC: req_qp=%d max_sd=%u, max_qp = %u, max_cq=%u, max_mr=%u, max_pble=%u, mc=%d, av=%u\n",</div><div class='add'>+		  qp_count, max_sds,</div><div class='add'>+		  hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_QP].max_cnt,</div><div class='add'>+		  hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_CQ].max_cnt,</div><div class='add'>+		  hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_MR].max_cnt,</div><div class='add'>+		  hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt,</div><div class='add'>+		  hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt,</div><div class='add'>+		  hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt);</div><div class='add'>+</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIMC].cnt =</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt;</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIAV].cnt =</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt;</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_ARP].cnt =</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_ARP].max_cnt;</div><div class='add'>+	hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_APBVT_ENTRY].cnt = 1;</div><div class='add'>+</div><div class='add'>+	while (irdma_q1_cnt(dev, hmc_info, qpwanted) &gt; hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_Q1].max_cnt)</div><div class='add'>+		qpwanted /= 2;</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		++loop_count;</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_QP].cnt = qpwanted;</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_CQ].cnt =</div><div class='add'>+			min(2 * qpwanted, hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_CQ].cnt);</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_SRQ].cnt = 0; /* Reserved */</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_MR].cnt = mrwanted;</div><div class='add'>+</div><div class='add'>+		hte = round_up(qpwanted + hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIMC].cnt, 512);</div><div class='add'>+		powerof2 = 1;</div><div class='add'>+		while (powerof2 &lt; hte)</div><div class='add'>+			powerof2 *= 2;</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_HTE].cnt =</div><div class='add'>+			powerof2 * hmc_fpm_misc-&gt;ht_multiplier;</div><div class='add'>+		if (dev-&gt;hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)</div><div class='add'>+			cfg_fpm_value_gen_1(dev, hmc_info, qpwanted);</div><div class='add'>+		else</div><div class='add'>+			cfg_fpm_value_gen_2(dev, hmc_info, qpwanted);</div><div class='add'>+</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_Q1].cnt = irdma_q1_cnt(dev, hmc_info, qpwanted);</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_XFFL].cnt =</div><div class='add'>+			hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_XF].cnt / hmc_fpm_misc-&gt;xf_block_size;</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_Q1FL].cnt =</div><div class='add'>+			hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_Q1].cnt / hmc_fpm_misc-&gt;q1_block_size;</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_TIMER].cnt =</div><div class='add'>+			(round_up(qpwanted, 512) / 512 + 1) * hmc_fpm_misc-&gt;timer_bucket;</div><div class='add'>+</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted;</div><div class='add'>+		sd_needed = irdma_est_sd(dev, hmc_info);</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "HMC: sd_needed = %d, hmc_fpm_misc-&gt;max_sds=%d, mrwanted=%d, pblewanted=%d qpwanted=%d\n",</div><div class='add'>+			  sd_needed, hmc_fpm_misc-&gt;max_sds, mrwanted,</div><div class='add'>+			  pblewanted, qpwanted);</div><div class='add'>+</div><div class='add'>+		/* Do not reduce resources further. All objects fit with max SDs */</div><div class='add'>+		if (sd_needed &lt;= hmc_fpm_misc-&gt;max_sds)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		sd_diff = sd_needed - hmc_fpm_misc-&gt;max_sds;</div><div class='add'>+		if (sd_diff &gt; 128) {</div><div class='add'>+			if (!(loop_count % 2) &amp;&amp; qpwanted &gt; 128) {</div><div class='add'>+				qpwanted /= 2;</div><div class='add'>+			} else {</div><div class='add'>+				pblewanted /= 2;</div><div class='add'>+				mrwanted /= 2;</div><div class='add'>+			}</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (dev-&gt;cqp-&gt;hmc_profile != IRDMA_HMC_PROFILE_FAVOR_VF &amp;&amp;</div><div class='add'>+		    pblewanted &gt; (512 * FPM_MULTIPLIER * sd_diff)) {</div><div class='add'>+			pblewanted -= 256 * FPM_MULTIPLIER * sd_diff;</div><div class='add'>+			continue;</div><div class='add'>+		} else if (pblewanted &gt; (100 * FPM_MULTIPLIER)) {</div><div class='add'>+			pblewanted -= 10 * FPM_MULTIPLIER;</div><div class='add'>+		} else if (pblewanted &gt; FPM_MULTIPLIER) {</div><div class='add'>+			pblewanted -= FPM_MULTIPLIER;</div><div class='add'>+		} else if (qpwanted &lt;= 128) {</div><div class='add'>+			if (hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIMC].cnt &gt; 256)</div><div class='add'>+				hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIMC].cnt /= 2;</div><div class='add'>+			if (hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIAV].cnt &gt; 256)</div><div class='add'>+				hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2;</div><div class='add'>+		}</div><div class='add'>+		if (mrwanted &gt; FPM_MULTIPLIER)</div><div class='add'>+			mrwanted -= FPM_MULTIPLIER;</div><div class='add'>+		if (!(loop_count % 10) &amp;&amp; qpwanted &gt; 128) {</div><div class='add'>+			qpwanted /= 2;</div><div class='add'>+			if (hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIAV].cnt &gt; 256)</div><div class='add'>+				hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2;</div><div class='add'>+		}</div><div class='add'>+	} while (loop_count &lt; 2000);</div><div class='add'>+</div><div class='add'>+	if (sd_needed &gt; hmc_fpm_misc-&gt;max_sds) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "HMC: cfg_fpm failed loop_cnt=%u, sd_needed=%u, max sd count %u\n",</div><div class='add'>+			  loop_count, sd_needed, hmc_info-&gt;sd_table.sd_cnt);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (loop_count &gt; 1 &amp;&amp; sd_needed &lt; max_sds) {</div><div class='add'>+		pblewanted += (max_sds - sd_needed) * 256 * FPM_MULTIPLIER;</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted;</div><div class='add'>+		sd_needed = irdma_est_sd(dev, hmc_info);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(to_ibdev(dev),</div><div class='add'>+		  "HMC: loop_cnt=%d, sd_needed=%d, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d, mc=%d, ah=%d, max sd count %d, first sd index %d\n",</div><div class='add'>+		  loop_count, sd_needed,</div><div class='add'>+		  hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_QP].cnt,</div><div class='add'>+		  hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_CQ].cnt,</div><div class='add'>+		  hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_MR].cnt,</div><div class='add'>+		  hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].cnt,</div><div class='add'>+		  hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIMC].cnt,</div><div class='add'>+		  hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIAV].cnt,</div><div class='add'>+		  hmc_info-&gt;sd_table.sd_cnt, hmc_info-&gt;first_sd_index);</div><div class='add'>+</div><div class='add'>+	ret_code = irdma_sc_cfg_iw_fpm(dev, dev-&gt;hmc_fn_id);</div><div class='add'>+	if (ret_code) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "HMC: cfg_iw_fpm returned error_code[x%08X]\n",</div><div class='add'>+			  readl(dev-&gt;hw_regs[IRDMA_CQPERRCODES]));</div><div class='add'>+		return ret_code;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return irdma_cfg_sd_mem(dev, hmc_info);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_exec_cqp_cmd - execute cqp cmd when wqe are available</div><div class='add'>+ * @dev: rdma device</div><div class='add'>+ * @pcmdinfo: cqp command info</div><div class='add'>+ */</div><div class='add'>+static int irdma_exec_cqp_cmd(struct irdma_sc_dev *dev,</div><div class='add'>+			      struct cqp_cmds_info *pcmdinfo)</div><div class='add'>+{</div><div class='add'>+	int status;</div><div class='add'>+	struct irdma_dma_mem val_mem;</div><div class='add'>+	bool alloc = false;</div><div class='add'>+</div><div class='add'>+	dev-&gt;cqp_cmd_stats[pcmdinfo-&gt;cqp_cmd]++;</div><div class='add'>+	switch (pcmdinfo-&gt;cqp_cmd) {</div><div class='add'>+	case IRDMA_OP_CEQ_DESTROY:</div><div class='add'>+		status = irdma_sc_ceq_destroy(pcmdinfo-&gt;in.u.ceq_destroy.ceq,</div><div class='add'>+					      pcmdinfo-&gt;in.u.ceq_destroy.scratch,</div><div class='add'>+					      pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_AEQ_DESTROY:</div><div class='add'>+		status = irdma_sc_aeq_destroy(pcmdinfo-&gt;in.u.aeq_destroy.aeq,</div><div class='add'>+					      pcmdinfo-&gt;in.u.aeq_destroy.scratch,</div><div class='add'>+					      pcmdinfo-&gt;post_sq);</div><div class='add'>+</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_CEQ_CREATE:</div><div class='add'>+		status = irdma_sc_ceq_create(pcmdinfo-&gt;in.u.ceq_create.ceq,</div><div class='add'>+					     pcmdinfo-&gt;in.u.ceq_create.scratch,</div><div class='add'>+					     pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_AEQ_CREATE:</div><div class='add'>+		status = irdma_sc_aeq_create(pcmdinfo-&gt;in.u.aeq_create.aeq,</div><div class='add'>+					     pcmdinfo-&gt;in.u.aeq_create.scratch,</div><div class='add'>+					     pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_QP_UPLOAD_CONTEXT:</div><div class='add'>+		status = irdma_sc_qp_upload_context(pcmdinfo-&gt;in.u.qp_upload_context.dev,</div><div class='add'>+						    &amp;pcmdinfo-&gt;in.u.qp_upload_context.info,</div><div class='add'>+						    pcmdinfo-&gt;in.u.qp_upload_context.scratch,</div><div class='add'>+						    pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_CQ_CREATE:</div><div class='add'>+		status = irdma_sc_cq_create(pcmdinfo-&gt;in.u.cq_create.cq,</div><div class='add'>+					    pcmdinfo-&gt;in.u.cq_create.scratch,</div><div class='add'>+					    pcmdinfo-&gt;in.u.cq_create.check_overflow,</div><div class='add'>+					    pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_CQ_MODIFY:</div><div class='add'>+		status = irdma_sc_cq_modify(pcmdinfo-&gt;in.u.cq_modify.cq,</div><div class='add'>+					    &amp;pcmdinfo-&gt;in.u.cq_modify.info,</div><div class='add'>+					    pcmdinfo-&gt;in.u.cq_modify.scratch,</div><div class='add'>+					    pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_CQ_DESTROY:</div><div class='add'>+		status = irdma_sc_cq_destroy(pcmdinfo-&gt;in.u.cq_destroy.cq,</div><div class='add'>+					     pcmdinfo-&gt;in.u.cq_destroy.scratch,</div><div class='add'>+					     pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_QP_FLUSH_WQES:</div><div class='add'>+		status = irdma_sc_qp_flush_wqes(pcmdinfo-&gt;in.u.qp_flush_wqes.qp,</div><div class='add'>+						&amp;pcmdinfo-&gt;in.u.qp_flush_wqes.info,</div><div class='add'>+						pcmdinfo-&gt;in.u.qp_flush_wqes.scratch,</div><div class='add'>+						pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_GEN_AE:</div><div class='add'>+		status = irdma_sc_gen_ae(pcmdinfo-&gt;in.u.gen_ae.qp,</div><div class='add'>+					 &amp;pcmdinfo-&gt;in.u.gen_ae.info,</div><div class='add'>+					 pcmdinfo-&gt;in.u.gen_ae.scratch,</div><div class='add'>+					 pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_MANAGE_PUSH_PAGE:</div><div class='add'>+		status = irdma_sc_manage_push_page(pcmdinfo-&gt;in.u.manage_push_page.cqp,</div><div class='add'>+						   &amp;pcmdinfo-&gt;in.u.manage_push_page.info,</div><div class='add'>+						   pcmdinfo-&gt;in.u.manage_push_page.scratch,</div><div class='add'>+						   pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_UPDATE_PE_SDS:</div><div class='add'>+		status = irdma_update_pe_sds(pcmdinfo-&gt;in.u.update_pe_sds.dev,</div><div class='add'>+					     &amp;pcmdinfo-&gt;in.u.update_pe_sds.info,</div><div class='add'>+					     pcmdinfo-&gt;in.u.update_pe_sds.scratch);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE:</div><div class='add'>+		/* switch to calling through the call table */</div><div class='add'>+		status =</div><div class='add'>+			irdma_sc_manage_hmc_pm_func_table(pcmdinfo-&gt;in.u.manage_hmc_pm.dev-&gt;cqp,</div><div class='add'>+							  &amp;pcmdinfo-&gt;in.u.manage_hmc_pm.info,</div><div class='add'>+							  pcmdinfo-&gt;in.u.manage_hmc_pm.scratch,</div><div class='add'>+							  true);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_SUSPEND:</div><div class='add'>+		status = irdma_sc_suspend_qp(pcmdinfo-&gt;in.u.suspend_resume.cqp,</div><div class='add'>+					     pcmdinfo-&gt;in.u.suspend_resume.qp,</div><div class='add'>+					     pcmdinfo-&gt;in.u.suspend_resume.scratch);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_RESUME:</div><div class='add'>+		status = irdma_sc_resume_qp(pcmdinfo-&gt;in.u.suspend_resume.cqp,</div><div class='add'>+					    pcmdinfo-&gt;in.u.suspend_resume.qp,</div><div class='add'>+					    pcmdinfo-&gt;in.u.suspend_resume.scratch);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_QUERY_FPM_VAL:</div><div class='add'>+		val_mem.pa = pcmdinfo-&gt;in.u.query_fpm_val.fpm_val_pa;</div><div class='add'>+		val_mem.va = pcmdinfo-&gt;in.u.query_fpm_val.fpm_val_va;</div><div class='add'>+		status = irdma_sc_query_fpm_val(pcmdinfo-&gt;in.u.query_fpm_val.cqp,</div><div class='add'>+						pcmdinfo-&gt;in.u.query_fpm_val.scratch,</div><div class='add'>+						pcmdinfo-&gt;in.u.query_fpm_val.hmc_fn_id,</div><div class='add'>+						&amp;val_mem, true, IRDMA_CQP_WAIT_EVENT);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_COMMIT_FPM_VAL:</div><div class='add'>+		val_mem.pa = pcmdinfo-&gt;in.u.commit_fpm_val.fpm_val_pa;</div><div class='add'>+		val_mem.va = pcmdinfo-&gt;in.u.commit_fpm_val.fpm_val_va;</div><div class='add'>+		status = irdma_sc_commit_fpm_val(pcmdinfo-&gt;in.u.commit_fpm_val.cqp,</div><div class='add'>+						 pcmdinfo-&gt;in.u.commit_fpm_val.scratch,</div><div class='add'>+						 pcmdinfo-&gt;in.u.commit_fpm_val.hmc_fn_id,</div><div class='add'>+						 &amp;val_mem,</div><div class='add'>+						 true,</div><div class='add'>+						 IRDMA_CQP_WAIT_EVENT);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_STATS_ALLOCATE:</div><div class='add'>+		alloc = true;</div><div class='add'>+		fallthrough;</div><div class='add'>+	case IRDMA_OP_STATS_FREE:</div><div class='add'>+		status = irdma_sc_manage_stats_inst(pcmdinfo-&gt;in.u.stats_manage.cqp,</div><div class='add'>+						    &amp;pcmdinfo-&gt;in.u.stats_manage.info,</div><div class='add'>+						    alloc,</div><div class='add'>+						    pcmdinfo-&gt;in.u.stats_manage.scratch);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_STATS_GATHER:</div><div class='add'>+		status = irdma_sc_gather_stats(pcmdinfo-&gt;in.u.stats_gather.cqp,</div><div class='add'>+					       &amp;pcmdinfo-&gt;in.u.stats_gather.info,</div><div class='add'>+					       pcmdinfo-&gt;in.u.stats_gather.scratch);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_WS_MODIFY_NODE:</div><div class='add'>+		status = irdma_sc_manage_ws_node(pcmdinfo-&gt;in.u.ws_node.cqp,</div><div class='add'>+						 &amp;pcmdinfo-&gt;in.u.ws_node.info,</div><div class='add'>+						 IRDMA_MODIFY_NODE,</div><div class='add'>+						 pcmdinfo-&gt;in.u.ws_node.scratch);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_WS_DELETE_NODE:</div><div class='add'>+		status = irdma_sc_manage_ws_node(pcmdinfo-&gt;in.u.ws_node.cqp,</div><div class='add'>+						 &amp;pcmdinfo-&gt;in.u.ws_node.info,</div><div class='add'>+						 IRDMA_DEL_NODE,</div><div class='add'>+						 pcmdinfo-&gt;in.u.ws_node.scratch);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_WS_ADD_NODE:</div><div class='add'>+		status = irdma_sc_manage_ws_node(pcmdinfo-&gt;in.u.ws_node.cqp,</div><div class='add'>+						 &amp;pcmdinfo-&gt;in.u.ws_node.info,</div><div class='add'>+						 IRDMA_ADD_NODE,</div><div class='add'>+						 pcmdinfo-&gt;in.u.ws_node.scratch);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_SET_UP_MAP:</div><div class='add'>+		status = irdma_sc_set_up_map(pcmdinfo-&gt;in.u.up_map.cqp,</div><div class='add'>+					     &amp;pcmdinfo-&gt;in.u.up_map.info,</div><div class='add'>+					     pcmdinfo-&gt;in.u.up_map.scratch);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_QUERY_RDMA_FEATURES:</div><div class='add'>+		status = irdma_sc_query_rdma_features(pcmdinfo-&gt;in.u.query_rdma.cqp,</div><div class='add'>+						      &amp;pcmdinfo-&gt;in.u.query_rdma.query_buff_mem,</div><div class='add'>+						      pcmdinfo-&gt;in.u.query_rdma.scratch);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_DELETE_ARP_CACHE_ENTRY:</div><div class='add'>+		status = irdma_sc_del_arp_cache_entry(pcmdinfo-&gt;in.u.del_arp_cache_entry.cqp,</div><div class='add'>+						      pcmdinfo-&gt;in.u.del_arp_cache_entry.scratch,</div><div class='add'>+						      pcmdinfo-&gt;in.u.del_arp_cache_entry.arp_index,</div><div class='add'>+						      pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_MANAGE_APBVT_ENTRY:</div><div class='add'>+		status = irdma_sc_manage_apbvt_entry(pcmdinfo-&gt;in.u.manage_apbvt_entry.cqp,</div><div class='add'>+						     &amp;pcmdinfo-&gt;in.u.manage_apbvt_entry.info,</div><div class='add'>+						     pcmdinfo-&gt;in.u.manage_apbvt_entry.scratch,</div><div class='add'>+						     pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY:</div><div class='add'>+		status = irdma_sc_manage_qhash_table_entry(pcmdinfo-&gt;in.u.manage_qhash_table_entry.cqp,</div><div class='add'>+							   &amp;pcmdinfo-&gt;in.u.manage_qhash_table_entry.info,</div><div class='add'>+							   pcmdinfo-&gt;in.u.manage_qhash_table_entry.scratch,</div><div class='add'>+							   pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_QP_MODIFY:</div><div class='add'>+		status = irdma_sc_qp_modify(pcmdinfo-&gt;in.u.qp_modify.qp,</div><div class='add'>+					    &amp;pcmdinfo-&gt;in.u.qp_modify.info,</div><div class='add'>+					    pcmdinfo-&gt;in.u.qp_modify.scratch,</div><div class='add'>+					    pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_QP_CREATE:</div><div class='add'>+		status = irdma_sc_qp_create(pcmdinfo-&gt;in.u.qp_create.qp,</div><div class='add'>+					    &amp;pcmdinfo-&gt;in.u.qp_create.info,</div><div class='add'>+					    pcmdinfo-&gt;in.u.qp_create.scratch,</div><div class='add'>+					    pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_QP_DESTROY:</div><div class='add'>+		status = irdma_sc_qp_destroy(pcmdinfo-&gt;in.u.qp_destroy.qp,</div><div class='add'>+					     pcmdinfo-&gt;in.u.qp_destroy.scratch,</div><div class='add'>+					     pcmdinfo-&gt;in.u.qp_destroy.remove_hash_idx,</div><div class='add'>+					     pcmdinfo-&gt;in.u.qp_destroy.ignore_mw_bnd,</div><div class='add'>+					     pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_ALLOC_STAG:</div><div class='add'>+		status = irdma_sc_alloc_stag(pcmdinfo-&gt;in.u.alloc_stag.dev,</div><div class='add'>+					     &amp;pcmdinfo-&gt;in.u.alloc_stag.info,</div><div class='add'>+					     pcmdinfo-&gt;in.u.alloc_stag.scratch,</div><div class='add'>+					     pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_MR_REG_NON_SHARED:</div><div class='add'>+		status = irdma_sc_mr_reg_non_shared(pcmdinfo-&gt;in.u.mr_reg_non_shared.dev,</div><div class='add'>+						    &amp;pcmdinfo-&gt;in.u.mr_reg_non_shared.info,</div><div class='add'>+						    pcmdinfo-&gt;in.u.mr_reg_non_shared.scratch,</div><div class='add'>+						    pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_DEALLOC_STAG:</div><div class='add'>+		status = irdma_sc_dealloc_stag(pcmdinfo-&gt;in.u.dealloc_stag.dev,</div><div class='add'>+					       &amp;pcmdinfo-&gt;in.u.dealloc_stag.info,</div><div class='add'>+					       pcmdinfo-&gt;in.u.dealloc_stag.scratch,</div><div class='add'>+					       pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_MW_ALLOC:</div><div class='add'>+		status = irdma_sc_mw_alloc(pcmdinfo-&gt;in.u.mw_alloc.dev,</div><div class='add'>+					   &amp;pcmdinfo-&gt;in.u.mw_alloc.info,</div><div class='add'>+					   pcmdinfo-&gt;in.u.mw_alloc.scratch,</div><div class='add'>+					   pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_ADD_ARP_CACHE_ENTRY:</div><div class='add'>+		status = irdma_sc_add_arp_cache_entry(pcmdinfo-&gt;in.u.add_arp_cache_entry.cqp,</div><div class='add'>+						      &amp;pcmdinfo-&gt;in.u.add_arp_cache_entry.info,</div><div class='add'>+						      pcmdinfo-&gt;in.u.add_arp_cache_entry.scratch,</div><div class='add'>+						      pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY:</div><div class='add'>+		status = irdma_sc_alloc_local_mac_entry(pcmdinfo-&gt;in.u.alloc_local_mac_entry.cqp,</div><div class='add'>+							pcmdinfo-&gt;in.u.alloc_local_mac_entry.scratch,</div><div class='add'>+							pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_ADD_LOCAL_MAC_ENTRY:</div><div class='add'>+		status = irdma_sc_add_local_mac_entry(pcmdinfo-&gt;in.u.add_local_mac_entry.cqp,</div><div class='add'>+						      &amp;pcmdinfo-&gt;in.u.add_local_mac_entry.info,</div><div class='add'>+						      pcmdinfo-&gt;in.u.add_local_mac_entry.scratch,</div><div class='add'>+						      pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_DELETE_LOCAL_MAC_ENTRY:</div><div class='add'>+		status = irdma_sc_del_local_mac_entry(pcmdinfo-&gt;in.u.del_local_mac_entry.cqp,</div><div class='add'>+						      pcmdinfo-&gt;in.u.del_local_mac_entry.scratch,</div><div class='add'>+						      pcmdinfo-&gt;in.u.del_local_mac_entry.entry_idx,</div><div class='add'>+						      pcmdinfo-&gt;in.u.del_local_mac_entry.ignore_ref_count,</div><div class='add'>+						      pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_AH_CREATE:</div><div class='add'>+		status = irdma_sc_create_ah(pcmdinfo-&gt;in.u.ah_create.cqp,</div><div class='add'>+					    &amp;pcmdinfo-&gt;in.u.ah_create.info,</div><div class='add'>+					    pcmdinfo-&gt;in.u.ah_create.scratch);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_AH_DESTROY:</div><div class='add'>+		status = irdma_sc_destroy_ah(pcmdinfo-&gt;in.u.ah_destroy.cqp,</div><div class='add'>+					     &amp;pcmdinfo-&gt;in.u.ah_destroy.info,</div><div class='add'>+					     pcmdinfo-&gt;in.u.ah_destroy.scratch);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_MC_CREATE:</div><div class='add'>+		status = irdma_sc_create_mcast_grp(pcmdinfo-&gt;in.u.mc_create.cqp,</div><div class='add'>+						   &amp;pcmdinfo-&gt;in.u.mc_create.info,</div><div class='add'>+						   pcmdinfo-&gt;in.u.mc_create.scratch);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_MC_DESTROY:</div><div class='add'>+		status = irdma_sc_destroy_mcast_grp(pcmdinfo-&gt;in.u.mc_destroy.cqp,</div><div class='add'>+						    &amp;pcmdinfo-&gt;in.u.mc_destroy.info,</div><div class='add'>+						    pcmdinfo-&gt;in.u.mc_destroy.scratch);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_MC_MODIFY:</div><div class='add'>+		status = irdma_sc_modify_mcast_grp(pcmdinfo-&gt;in.u.mc_modify.cqp,</div><div class='add'>+						   &amp;pcmdinfo-&gt;in.u.mc_modify.info,</div><div class='add'>+						   pcmdinfo-&gt;in.u.mc_modify.scratch);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_SRQ_CREATE:</div><div class='add'>+		status = irdma_sc_srq_create(pcmdinfo-&gt;in.u.srq_create.srq,</div><div class='add'>+					     pcmdinfo-&gt;in.u.srq_create.scratch,</div><div class='add'>+					     pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_SRQ_MODIFY:</div><div class='add'>+		status = irdma_sc_srq_modify(pcmdinfo-&gt;in.u.srq_modify.srq,</div><div class='add'>+					     &amp;pcmdinfo-&gt;in.u.srq_modify.info,</div><div class='add'>+					     pcmdinfo-&gt;in.u.srq_modify.scratch,</div><div class='add'>+					     pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_SRQ_DESTROY:</div><div class='add'>+		status = irdma_sc_srq_destroy(pcmdinfo-&gt;in.u.srq_destroy.srq,</div><div class='add'>+					      pcmdinfo-&gt;in.u.srq_destroy.scratch,</div><div class='add'>+					      pcmdinfo-&gt;post_sq);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		status = -EOPNOTSUPP;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_process_cqp_cmd - process all cqp commands</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ * @pcmdinfo: cqp command info</div><div class='add'>+ */</div><div class='add'>+int irdma_process_cqp_cmd(struct irdma_sc_dev *dev,</div><div class='add'>+			  struct cqp_cmds_info *pcmdinfo)</div><div class='add'>+{</div><div class='add'>+	int status = 0;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;dev-&gt;cqp_lock, flags);</div><div class='add'>+	if (list_empty(&amp;dev-&gt;cqp_cmd_head) &amp;&amp; !irdma_cqp_ring_full(dev-&gt;cqp))</div><div class='add'>+		status = irdma_exec_cqp_cmd(dev, pcmdinfo);</div><div class='add'>+	else</div><div class='add'>+		list_add_tail(&amp;pcmdinfo-&gt;cqp_cmd_entry, &amp;dev-&gt;cqp_cmd_head);</div><div class='add'>+	spin_unlock_irqrestore(&amp;dev-&gt;cqp_lock, flags);</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_process_bh - called from tasklet for cqp list</div><div class='add'>+ * @dev: sc device struct</div><div class='add'>+ */</div><div class='add'>+int irdma_process_bh(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int status = 0;</div><div class='add'>+	struct cqp_cmds_info *pcmdinfo;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;dev-&gt;cqp_lock, flags);</div><div class='add'>+	while (!list_empty(&amp;dev-&gt;cqp_cmd_head) &amp;&amp;</div><div class='add'>+	       !irdma_cqp_ring_full(dev-&gt;cqp)) {</div><div class='add'>+		pcmdinfo = (struct cqp_cmds_info *)irdma_remove_cqp_head(dev);</div><div class='add'>+		status = irdma_exec_cqp_cmd(dev, pcmdinfo);</div><div class='add'>+		if (status)</div><div class='add'>+			break;</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;dev-&gt;cqp_lock, flags);</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cfg_aeq- Configure AEQ interrupt</div><div class='add'>+ * @dev: pointer to the device structure</div><div class='add'>+ * @idx: vector index</div><div class='add'>+ * @enable: True to enable, False disables</div><div class='add'>+ */</div><div class='add'>+void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable)</div><div class='add'>+{</div><div class='add'>+	u32 reg_val;</div><div class='add'>+</div><div class='add'>+	reg_val = FIELD_PREP(IRDMA_PFINT_AEQCTL_CAUSE_ENA, enable) |</div><div class='add'>+		  FIELD_PREP(IRDMA_PFINT_AEQCTL_MSIX_INDX, idx) |</div><div class='add'>+		  FIELD_PREP(IRDMA_PFINT_AEQCTL_ITR_INDX, 3);</div><div class='add'>+	writel(reg_val, dev-&gt;hw_regs[IRDMA_PFINT_AEQCTL]);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * sc_vsi_update_stats - Update statistics</div><div class='add'>+ * @vsi: sc_vsi instance to update</div><div class='add'>+ */</div><div class='add'>+void sc_vsi_update_stats(struct irdma_sc_vsi *vsi)</div><div class='add'>+{</div><div class='add'>+	struct irdma_dev_hw_stats *hw_stats = &amp;vsi-&gt;pestat-&gt;hw_stats;</div><div class='add'>+	struct irdma_gather_stats *gather_stats =</div><div class='add'>+		vsi-&gt;pestat-&gt;gather_info.gather_stats_va;</div><div class='add'>+	struct irdma_gather_stats *last_gather_stats =</div><div class='add'>+		vsi-&gt;pestat-&gt;gather_info.last_gather_stats_va;</div><div class='add'>+	const struct irdma_hw_stat_map *map = vsi-&gt;dev-&gt;hw_stats_map;</div><div class='add'>+	u16 max_stat_idx = vsi-&gt;dev-&gt;hw_attrs.max_stat_idx;</div><div class='add'>+	u16 i;</div><div class='add'>+</div><div class='add'>+	if (vsi-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3) {</div><div class='add'>+		for (i = 0; i &lt; max_stat_idx; i++) {</div><div class='add'>+			u16 idx = map[i].byteoff / sizeof(u64);</div><div class='add'>+</div><div class='add'>+			hw_stats-&gt;stats_val[i] = gather_stats-&gt;val[idx];</div><div class='add'>+		}</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_update_stats(hw_stats, gather_stats, last_gather_stats,</div><div class='add'>+			   map, max_stat_idx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_wait_pe_ready - Check if firmware is ready</div><div class='add'>+ * @dev: provides access to registers</div><div class='add'>+ */</div><div class='add'>+static int irdma_wait_pe_ready(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	u32 statuscpu0;</div><div class='add'>+	u32 statuscpu1;</div><div class='add'>+	u32 statuscpu2;</div><div class='add'>+	u32 retrycount = 0;</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		statuscpu0 = readl(dev-&gt;hw_regs[IRDMA_GLPE_CPUSTATUS0]);</div><div class='add'>+		statuscpu1 = readl(dev-&gt;hw_regs[IRDMA_GLPE_CPUSTATUS1]);</div><div class='add'>+		statuscpu2 = readl(dev-&gt;hw_regs[IRDMA_GLPE_CPUSTATUS2]);</div><div class='add'>+		if (statuscpu0 == 0x80 &amp;&amp; statuscpu1 == 0x80 &amp;&amp;</div><div class='add'>+		    statuscpu2 == 0x80)</div><div class='add'>+			return 0;</div><div class='add'>+		mdelay(1000);</div><div class='add'>+	} while (retrycount++ &lt; dev-&gt;hw_attrs.max_pe_ready_count);</div><div class='add'>+	return -1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void irdma_sc_init_hw(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	switch (dev-&gt;hw_attrs.uk_attrs.hw_rev) {</div><div class='add'>+	case IRDMA_GEN_1:</div><div class='add'>+		i40iw_init_hw(dev);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_GEN_2:</div><div class='add'>+		icrdma_init_hw(dev);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_GEN_3:</div><div class='add'>+		ig3rdma_init_hw(dev);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_dev_init - Initialize control part of device</div><div class='add'>+ * @ver: version</div><div class='add'>+ * @dev: Device pointer</div><div class='add'>+ * @info: Device init info</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_dev_init(enum irdma_vers ver, struct irdma_sc_dev *dev,</div><div class='add'>+		      struct irdma_device_init_info *info)</div><div class='add'>+{</div><div class='add'>+	u32 val;</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+	u8 db_size;</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;dev-&gt;cqp_cmd_head); /* for CQP command backlog */</div><div class='add'>+	mutex_init(&amp;dev-&gt;ws_mutex);</div><div class='add'>+	dev-&gt;hmc_fn_id = info-&gt;hmc_fn_id;</div><div class='add'>+	dev-&gt;fpm_query_buf_pa = info-&gt;fpm_query_buf_pa;</div><div class='add'>+	dev-&gt;fpm_query_buf = info-&gt;fpm_query_buf;</div><div class='add'>+	dev-&gt;fpm_commit_buf_pa = info-&gt;fpm_commit_buf_pa;</div><div class='add'>+	dev-&gt;fpm_commit_buf = info-&gt;fpm_commit_buf;</div><div class='add'>+	dev-&gt;hw = info-&gt;hw;</div><div class='add'>+	dev-&gt;hw-&gt;hw_addr = info-&gt;bar0;</div><div class='add'>+	dev-&gt;protocol_used = info-&gt;protocol_used;</div><div class='add'>+	/* Setup the hardware limits, hmc may limit further */</div><div class='add'>+	dev-&gt;hw_attrs.min_hw_qp_id = IRDMA_MIN_IW_QP_ID;</div><div class='add'>+	dev-&gt;hw_attrs.min_hw_srq_id = IRDMA_MIN_IW_SRQ_ID;</div><div class='add'>+	dev-&gt;hw_attrs.min_hw_aeq_size = IRDMA_MIN_AEQ_ENTRIES;</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3)</div><div class='add'>+		dev-&gt;hw_attrs.max_hw_aeq_size = IRDMA_MAX_AEQ_ENTRIES_GEN_3;</div><div class='add'>+	else</div><div class='add'>+		dev-&gt;hw_attrs.max_hw_aeq_size = IRDMA_MAX_AEQ_ENTRIES;</div><div class='add'>+	dev-&gt;hw_attrs.min_hw_ceq_size = IRDMA_MIN_CEQ_ENTRIES;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_ceq_size = IRDMA_MAX_CEQ_ENTRIES;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.min_hw_cq_size = IRDMA_MIN_CQ_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_cq_size = IRDMA_MAX_CQ_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_wq_frags = IRDMA_MAX_WQ_FRAGMENT_COUNT;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_read_sges = IRDMA_MAX_SGE_RD;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_outbound_msg_size = IRDMA_MAX_OUTBOUND_MSG_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.max_mr_size = IRDMA_MAX_MR_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_inbound_msg_size = IRDMA_MAX_INBOUND_MSG_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_device_pages = IRDMA_MAX_PUSH_PAGE_COUNT;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_inline = IRDMA_MAX_INLINE_DATA_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_wqes = IRDMA_MAX_WQ_ENTRIES;</div><div class='add'>+	dev-&gt;hw_attrs.max_qp_wr = IRDMA_MAX_QP_WRS(IRDMA_MAX_QUANTA_PER_WR);</div><div class='add'>+</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_rq_quanta = IRDMA_QP_SW_MAX_RQ_QUANTA;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_wq_quanta = IRDMA_QP_SW_MAX_WQ_QUANTA;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_pds = IRDMA_MAX_PDS;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_ena_vf_count = IRDMA_MAX_PE_ENA_VF_COUNT;</div><div class='add'>+</div><div class='add'>+	dev-&gt;hw_attrs.max_pe_ready_count = 14;</div><div class='add'>+	dev-&gt;hw_attrs.max_done_count = IRDMA_DONE_COUNT;</div><div class='add'>+	dev-&gt;hw_attrs.max_sleep_count = IRDMA_SLEEP_COUNT;</div><div class='add'>+	dev-&gt;hw_attrs.max_cqp_compl_wait_time_ms = CQP_COMPL_WAIT_TIME_MS;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;privileged) {</div><div class='add'>+		ret_code = irdma_vchnl_req_get_hmc_fcn(dev);</div><div class='add'>+		if (ret_code) {</div><div class='add'>+			ibdev_dbg(to_ibdev(dev),</div><div class='add'>+				  "DEV: Get HMC function ret = %d\n",</div><div class='add'>+				  ret_code);</div><div class='add'>+</div><div class='add'>+			return ret_code;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_sc_init_hw(dev);</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;privileged) {</div><div class='add'>+		if (irdma_wait_pe_ready(dev))</div><div class='add'>+			return -ETIMEDOUT;</div><div class='add'>+</div><div class='add'>+		val = readl(dev-&gt;hw_regs[IRDMA_GLPCI_LBARCTRL]);</div><div class='add'>+		db_size = (u8)FIELD_GET(IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE, val);</div><div class='add'>+		if (db_size != IRDMA_PE_DB_SIZE_4M &amp;&amp;</div><div class='add'>+		    db_size != IRDMA_PE_DB_SIZE_8M) {</div><div class='add'>+			ibdev_dbg(to_ibdev(dev),</div><div class='add'>+				  "DEV: RDMA PE doorbell is not enabled in CSR val 0x%x db_size=%d\n",</div><div class='add'>+				  val, db_size);</div><div class='add'>+			return -ENODEV;</div><div class='add'>+			}</div><div class='add'>+	} else {</div><div class='add'>+		ret_code = irdma_vchnl_req_get_reg_layout(dev);</div><div class='add'>+		if (ret_code)</div><div class='add'>+			ibdev_dbg(to_ibdev(dev),</div><div class='add'>+				  "DEV: Get Register layout failed ret = %d\n",</div><div class='add'>+				  ret_code);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_stat_val - Extract HW counter value from statistics buffer</div><div class='add'>+ * @stats_val: pointer to statistics buffer</div><div class='add'>+ * @byteoff: byte offset of counter value in the buffer (8B-aligned)</div><div class='add'>+ * @bitoff: bit offset of counter value within 8B entry</div><div class='add'>+ * @bitmask: maximum counter value (e.g. 0xffffff for 24-bit counter)</div><div class='add'>+ */</div><div class='add'>+static inline u64 irdma_stat_val(const u64 *stats_val, u16 byteoff, u8 bitoff,</div><div class='add'>+				 u64 bitmask)</div><div class='add'>+{</div><div class='add'>+	u16 idx = byteoff / sizeof(*stats_val);</div><div class='add'>+</div><div class='add'>+	return (stats_val[idx] &gt;&gt; bitoff) &amp; bitmask;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_stat_delta - Calculate counter delta</div><div class='add'>+ * @new_val: updated counter value</div><div class='add'>+ * @old_val: last counter value</div><div class='add'>+ * @max_val: maximum counter value (e.g. 0xffffff for 24-bit counter)</div><div class='add'>+ */</div><div class='add'>+static inline u64 irdma_stat_delta(u64 new_val, u64 old_val, u64 max_val)</div><div class='add'>+{</div><div class='add'>+	if (new_val &gt;= old_val)</div><div class='add'>+		return new_val - old_val;</div><div class='add'>+</div><div class='add'>+	/* roll-over case */</div><div class='add'>+	return max_val - old_val + new_val + 1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_update_stats - Update statistics</div><div class='add'>+ * @hw_stats: hw_stats instance to update</div><div class='add'>+ * @gather_stats: updated stat counters</div><div class='add'>+ * @last_gather_stats: last stat counters</div><div class='add'>+ * @map: HW stat map (hw_stats =&gt; gather_stats)</div><div class='add'>+ * @max_stat_idx: number of HW stats</div><div class='add'>+ */</div><div class='add'>+void irdma_update_stats(struct irdma_dev_hw_stats *hw_stats,</div><div class='add'>+			struct irdma_gather_stats *gather_stats,</div><div class='add'>+			struct irdma_gather_stats *last_gather_stats,</div><div class='add'>+			const struct irdma_hw_stat_map *map, u16 max_stat_idx)</div><div class='add'>+{</div><div class='add'>+	u64 *stats_val = hw_stats-&gt;stats_val;</div><div class='add'>+	u16 i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; max_stat_idx; i++) {</div><div class='add'>+		u64 new_val = irdma_stat_val(gather_stats-&gt;val, map[i].byteoff,</div><div class='add'>+					     map[i].bitoff, map[i].bitmask);</div><div class='add'>+		u64 last_val = irdma_stat_val(last_gather_stats-&gt;val,</div><div class='add'>+					      map[i].byteoff, map[i].bitoff,</div><div class='add'>+					      map[i].bitmask);</div><div class='add'>+</div><div class='add'>+		stats_val[i] +=</div><div class='add'>+			irdma_stat_delta(new_val, last_val, map[i].bitmask);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	memcpy(last_gather_stats, gather_stats, sizeof(*last_gather_stats));</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/defs.h b/drivers/infiniband/hw/irdma/defs.h<br/>new file mode 100644<br/>index 000000000000..983b22d7ae23<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/defs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/defs.h</a></div><div class='hunk'>@@ -0,0 +1,1184 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#ifndef IRDMA_DEFS_H</div><div class='add'>+#define IRDMA_DEFS_H</div><div class='add'>+</div><div class='add'>+#define IRDMA_FIRST_USER_QP_ID	3</div><div class='add'>+</div><div class='add'>+#define ECN_CODE_PT_VAL	2</div><div class='add'>+</div><div class='add'>+#define IRDMA_PUSH_OFFSET		(8 * 1024 * 1024)</div><div class='add'>+#define IRDMA_PF_FIRST_PUSH_PAGE_INDEX	16</div><div class='add'>+#define IRDMA_PF_BAR_RSVD		(60 * 1024)</div><div class='add'>+</div><div class='add'>+#define IRDMA_PE_DB_SIZE_4M	1</div><div class='add'>+#define IRDMA_PE_DB_SIZE_8M	2</div><div class='add'>+</div><div class='add'>+#define IRDMA_IRD_HW_SIZE_4_GEN3	0</div><div class='add'>+#define IRDMA_IRD_HW_SIZE_8_GEN3	1</div><div class='add'>+#define IRDMA_IRD_HW_SIZE_16_GEN3	2</div><div class='add'>+#define IRDMA_IRD_HW_SIZE_32_GEN3	3</div><div class='add'>+#define IRDMA_IRD_HW_SIZE_64_GEN3	4</div><div class='add'>+#define IRDMA_IRD_HW_SIZE_128_GEN3	5</div><div class='add'>+#define IRDMA_IRD_HW_SIZE_256_GEN3	6</div><div class='add'>+#define IRDMA_IRD_HW_SIZE_512_GEN3	7</div><div class='add'>+#define IRDMA_IRD_HW_SIZE_1024_GEN3	8</div><div class='add'>+#define IRDMA_IRD_HW_SIZE_2048_GEN3	9</div><div class='add'>+#define IRDMA_IRD_HW_SIZE_4096_GEN3	10</div><div class='add'>+</div><div class='add'>+#define IRDMA_IRD_HW_SIZE_4	0</div><div class='add'>+#define IRDMA_IRD_HW_SIZE_16	1</div><div class='add'>+#define IRDMA_IRD_HW_SIZE_64	2</div><div class='add'>+#define IRDMA_IRD_HW_SIZE_128	3</div><div class='add'>+#define IRDMA_IRD_HW_SIZE_256	4</div><div class='add'>+</div><div class='add'>+enum irdma_protocol_used {</div><div class='add'>+	IRDMA_ANY_PROTOCOL = 0,</div><div class='add'>+	IRDMA_IWARP_PROTOCOL_ONLY = 1,</div><div class='add'>+	IRDMA_ROCE_PROTOCOL_ONLY = 2,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define IRDMA_QP_STATE_INVALID		0</div><div class='add'>+#define IRDMA_QP_STATE_IDLE		1</div><div class='add'>+#define IRDMA_QP_STATE_RTS		2</div><div class='add'>+#define IRDMA_QP_STATE_CLOSING		3</div><div class='add'>+#define IRDMA_QP_STATE_SQD		3</div><div class='add'>+#define IRDMA_QP_STATE_RTR		4</div><div class='add'>+#define IRDMA_QP_STATE_TERMINATE	5</div><div class='add'>+#define IRDMA_QP_STATE_ERROR		6</div><div class='add'>+</div><div class='add'>+#define IRDMA_MAX_TRAFFIC_CLASS		8</div><div class='add'>+#define	IRDMA_MAX_STATS_COUNT_GEN_1	12</div><div class='add'>+#define IRDMA_MAX_USER_PRIORITY		8</div><div class='add'>+#define IRDMA_MAX_APPS			8</div><div class='add'>+#define IRDMA_MAX_STATS_COUNT		128</div><div class='add'>+#define IRDMA_FIRST_NON_PF_STAT		4</div><div class='add'>+</div><div class='add'>+#define IRDMA_MIN_MTU_IPV4	576</div><div class='add'>+#define IRDMA_MIN_MTU_IPV6	1280</div><div class='add'>+#define IRDMA_MTU_TO_MSS_IPV4	40</div><div class='add'>+#define IRDMA_MTU_TO_MSS_IPV6	60</div><div class='add'>+#define IRDMA_DEFAULT_MTU	1500</div><div class='add'>+</div><div class='add'>+#define Q2_FPSN_OFFSET		64</div><div class='add'>+#define TERM_DDP_LEN_TAGGED	14</div><div class='add'>+#define TERM_DDP_LEN_UNTAGGED	18</div><div class='add'>+#define TERM_RDMA_LEN		28</div><div class='add'>+#define RDMA_OPCODE_M		0x0f</div><div class='add'>+#define RDMA_READ_REQ_OPCODE	1</div><div class='add'>+#define Q2_BAD_FRAME_OFFSET	72</div><div class='add'>+#define CQE_MAJOR_DRV		0x8000</div><div class='add'>+</div><div class='add'>+#define IRDMA_TERM_SENT		1</div><div class='add'>+#define IRDMA_TERM_RCVD		2</div><div class='add'>+#define IRDMA_TERM_DONE		4</div><div class='add'>+#define IRDMA_MAC_HLEN		14</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQP_WAIT_POLL_REGS	1</div><div class='add'>+#define IRDMA_CQP_WAIT_POLL_CQ		2</div><div class='add'>+#define IRDMA_CQP_WAIT_EVENT		3</div><div class='add'>+</div><div class='add'>+#define IRDMA_AE_SOURCE_RSVD		0x0</div><div class='add'>+#define IRDMA_AE_SOURCE_RQ		0x1</div><div class='add'>+#define IRDMA_AE_SOURCE_RQ_0011		0x3</div><div class='add'>+</div><div class='add'>+#define IRDMA_AE_SOURCE_CQ		0x2</div><div class='add'>+#define IRDMA_AE_SOURCE_CQ_0110		0x6</div><div class='add'>+#define IRDMA_AE_SOURCE_CQ_1010		0xa</div><div class='add'>+#define IRDMA_AE_SOURCE_CQ_1110		0xe</div><div class='add'>+</div><div class='add'>+#define IRDMA_AE_SOURCE_SQ		0x5</div><div class='add'>+#define IRDMA_AE_SOURCE_SQ_0111		0x7</div><div class='add'>+</div><div class='add'>+#define IRDMA_AE_SOURCE_IN_RR_WR	0x9</div><div class='add'>+#define IRDMA_AE_SOURCE_IN_RR_WR_1011	0xb</div><div class='add'>+#define IRDMA_AE_SOURCE_OUT_RR		0xd</div><div class='add'>+#define IRDMA_AE_SOURCE_OUT_RR_1111	0xf</div><div class='add'>+</div><div class='add'>+#define IRDMA_TCP_STATE_NON_EXISTENT	0</div><div class='add'>+#define IRDMA_TCP_STATE_CLOSED		1</div><div class='add'>+#define IRDMA_TCP_STATE_LISTEN		2</div><div class='add'>+#define IRDMA_STATE_SYN_SEND		3</div><div class='add'>+#define IRDMA_TCP_STATE_SYN_RECEIVED	4</div><div class='add'>+#define IRDMA_TCP_STATE_ESTABLISHED	5</div><div class='add'>+#define IRDMA_TCP_STATE_CLOSE_WAIT	6</div><div class='add'>+#define IRDMA_TCP_STATE_FIN_WAIT_1	7</div><div class='add'>+#define IRDMA_TCP_STATE_CLOSING		8</div><div class='add'>+#define IRDMA_TCP_STATE_LAST_ACK	9</div><div class='add'>+#define IRDMA_TCP_STATE_FIN_WAIT_2	10</div><div class='add'>+#define IRDMA_TCP_STATE_TIME_WAIT	11</div><div class='add'>+#define IRDMA_TCP_STATE_RESERVED_1	12</div><div class='add'>+#define IRDMA_TCP_STATE_RESERVED_2	13</div><div class='add'>+#define IRDMA_TCP_STATE_RESERVED_3	14</div><div class='add'>+#define IRDMA_TCP_STATE_RESERVED_4	15</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQP_SW_SQSIZE_4		4</div><div class='add'>+#define IRDMA_CQP_SW_SQSIZE_2048	2048</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQ_TYPE_IWARP	1</div><div class='add'>+#define IRDMA_CQ_TYPE_ILQ	2</div><div class='add'>+#define IRDMA_CQ_TYPE_IEQ	3</div><div class='add'>+#define IRDMA_CQ_TYPE_CQP	4</div><div class='add'>+</div><div class='add'>+#define IRDMA_DONE_COUNT	1000</div><div class='add'>+#define IRDMA_SLEEP_COUNT	10</div><div class='add'>+</div><div class='add'>+#define IRDMA_UPDATE_SD_BUFF_SIZE	128</div><div class='add'>+#define IRDMA_FEATURE_BUF_SIZE		(8 * IRDMA_MAX_FEATURES)</div><div class='add'>+</div><div class='add'>+#define ENABLE_LOC_MEM			63</div><div class='add'>+#define IRDMA_ATOMICS_ALLOWED_BIT	1</div><div class='add'>+#define MAX_PBLE_PER_SD			0x40000</div><div class='add'>+#define MAX_PBLE_SD_PER_FCN		0x400</div><div class='add'>+#define MAX_MR_PER_SD			0x8000</div><div class='add'>+#define MAX_MR_SD_PER_FCN		0x80</div><div class='add'>+#define IRDMA_PBLE_COMMIT_OFFSET	112</div><div class='add'>+#define IRDMA_MAX_QUANTA_PER_WR	8</div><div class='add'>+</div><div class='add'>+#define IRDMA_QP_SW_MAX_WQ_QUANTA	32768</div><div class='add'>+#define IRDMA_QP_SW_MAX_SQ_QUANTA	32768</div><div class='add'>+#define IRDMA_QP_SW_MAX_RQ_QUANTA	32768</div><div class='add'>+#define IRDMA_MAX_QP_WRS(max_quanta_per_wr) \</div><div class='add'>+	((IRDMA_QP_SW_MAX_WQ_QUANTA - IRDMA_SQ_RSVD) / (max_quanta_per_wr))</div><div class='add'>+#define IRDMA_SRQ_MIN_QUANTA 8</div><div class='add'>+#define IRDMA_SRQ_MAX_QUANTA 262144</div><div class='add'>+#define IRDMA_MAX_SRQ_WRS \</div><div class='add'>+	((IRDMA_SRQ_MAX_QUANTA - IRDMA_RQ_RSVD) / IRDMA_MAX_QUANTA_PER_WR)</div><div class='add'>+</div><div class='add'>+#define IRDMAQP_TERM_SEND_TERM_AND_FIN		0</div><div class='add'>+#define IRDMAQP_TERM_SEND_TERM_ONLY		1</div><div class='add'>+#define IRDMAQP_TERM_SEND_FIN_ONLY		2</div><div class='add'>+#define IRDMAQP_TERM_DONOT_SEND_TERM_OR_FIN	3</div><div class='add'>+</div><div class='add'>+#define IRDMA_QP_TYPE_IWARP	1</div><div class='add'>+#define IRDMA_QP_TYPE_UDA	2</div><div class='add'>+#define IRDMA_QP_TYPE_ROCE_RC	3</div><div class='add'>+#define IRDMA_QP_TYPE_ROCE_UD	4</div><div class='add'>+</div><div class='add'>+#define IRDMA_HW_PAGE_SIZE	4096</div><div class='add'>+#define IRDMA_HW_PAGE_SHIFT	12</div><div class='add'>+#define IRDMA_CQE_QTYPE_RQ	0</div><div class='add'>+#define IRDMA_CQE_QTYPE_SQ	1</div><div class='add'>+</div><div class='add'>+#define IRDMA_QP_SW_MIN_WQSIZE	8u /* in WRs*/</div><div class='add'>+#define IRDMA_QP_WQE_MIN_SIZE	32</div><div class='add'>+#define IRDMA_QP_WQE_MAX_SIZE	256</div><div class='add'>+#define IRDMA_QP_WQE_MIN_QUANTA 1</div><div class='add'>+#define IRDMA_MAX_RQ_WQE_SHIFT_GEN1 2</div><div class='add'>+#define IRDMA_MAX_RQ_WQE_SHIFT_GEN2 3</div><div class='add'>+</div><div class='add'>+#define IRDMA_SQ_RSVD	258</div><div class='add'>+#define IRDMA_RQ_RSVD	1</div><div class='add'>+</div><div class='add'>+#define IRDMA_FEATURE_RTS_AE			BIT_ULL(0)</div><div class='add'>+#define IRDMA_FEATURE_CQ_RESIZE			BIT_ULL(1)</div><div class='add'>+#define IRDMA_FEATURE_64_BYTE_CQE		BIT_ULL(5)</div><div class='add'>+#define IRDMA_FEATURE_ATOMIC_OPS		BIT_ULL(6)</div><div class='add'>+#define IRDMA_FEATURE_SRQ			BIT_ULL(7)</div><div class='add'>+#define IRDMA_FEATURE_CQE_TIMESTAMPING		BIT_ULL(8)</div><div class='add'>+</div><div class='add'>+#define IRDMAQP_OP_RDMA_WRITE			0x00</div><div class='add'>+#define IRDMAQP_OP_RDMA_READ			0x01</div><div class='add'>+#define IRDMAQP_OP_RDMA_SEND			0x03</div><div class='add'>+#define IRDMAQP_OP_RDMA_SEND_INV		0x04</div><div class='add'>+#define IRDMAQP_OP_RDMA_SEND_SOL_EVENT		0x05</div><div class='add'>+#define IRDMAQP_OP_RDMA_SEND_SOL_EVENT_INV	0x06</div><div class='add'>+#define IRDMAQP_OP_BIND_MW			0x08</div><div class='add'>+#define IRDMAQP_OP_FAST_REGISTER		0x09</div><div class='add'>+#define IRDMAQP_OP_LOCAL_INVALIDATE		0x0a</div><div class='add'>+#define IRDMAQP_OP_RDMA_READ_LOC_INV		0x0b</div><div class='add'>+#define IRDMAQP_OP_NOP				0x0c</div><div class='add'>+#define IRDMAQP_OP_RDMA_WRITE_SOL		0x0d</div><div class='add'>+#define IRDMAQP_OP_ATOMIC_FETCH_ADD		0x0f</div><div class='add'>+#define IRDMAQP_OP_ATOMIC_COMPARE_SWAP_ADD	0x11</div><div class='add'>+#define IRDMAQP_OP_GEN_RTS_AE			0x30</div><div class='add'>+</div><div class='add'>+enum irdma_cqp_op_type {</div><div class='add'>+	IRDMA_OP_CEQ_DESTROY			= 1,</div><div class='add'>+	IRDMA_OP_AEQ_DESTROY			= 2,</div><div class='add'>+	IRDMA_OP_DELETE_ARP_CACHE_ENTRY		= 3,</div><div class='add'>+	IRDMA_OP_MANAGE_APBVT_ENTRY		= 4,</div><div class='add'>+	IRDMA_OP_CEQ_CREATE			= 5,</div><div class='add'>+	IRDMA_OP_AEQ_CREATE			= 6,</div><div class='add'>+	IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY	= 7,</div><div class='add'>+	IRDMA_OP_QP_MODIFY			= 8,</div><div class='add'>+	IRDMA_OP_QP_UPLOAD_CONTEXT		= 9,</div><div class='add'>+	IRDMA_OP_CQ_CREATE			= 10,</div><div class='add'>+	IRDMA_OP_CQ_DESTROY			= 11,</div><div class='add'>+	IRDMA_OP_QP_CREATE			= 12,</div><div class='add'>+	IRDMA_OP_QP_DESTROY			= 13,</div><div class='add'>+	IRDMA_OP_ALLOC_STAG			= 14,</div><div class='add'>+	IRDMA_OP_MR_REG_NON_SHARED		= 15,</div><div class='add'>+	IRDMA_OP_DEALLOC_STAG			= 16,</div><div class='add'>+	IRDMA_OP_MW_ALLOC			= 17,</div><div class='add'>+	IRDMA_OP_QP_FLUSH_WQES			= 18,</div><div class='add'>+	IRDMA_OP_ADD_ARP_CACHE_ENTRY		= 19,</div><div class='add'>+	IRDMA_OP_MANAGE_PUSH_PAGE		= 20,</div><div class='add'>+	IRDMA_OP_UPDATE_PE_SDS			= 21,</div><div class='add'>+	IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE	= 22,</div><div class='add'>+	IRDMA_OP_SUSPEND			= 23,</div><div class='add'>+	IRDMA_OP_RESUME				= 24,</div><div class='add'>+	IRDMA_OP_MANAGE_VF_PBLE_BP		= 25,</div><div class='add'>+	IRDMA_OP_QUERY_FPM_VAL			= 26,</div><div class='add'>+	IRDMA_OP_COMMIT_FPM_VAL			= 27,</div><div class='add'>+	IRDMA_OP_AH_CREATE			= 28,</div><div class='add'>+	IRDMA_OP_AH_MODIFY			= 29,</div><div class='add'>+	IRDMA_OP_AH_DESTROY			= 30,</div><div class='add'>+	IRDMA_OP_MC_CREATE			= 31,</div><div class='add'>+	IRDMA_OP_MC_DESTROY			= 32,</div><div class='add'>+	IRDMA_OP_MC_MODIFY			= 33,</div><div class='add'>+	IRDMA_OP_STATS_ALLOCATE			= 34,</div><div class='add'>+	IRDMA_OP_STATS_FREE			= 35,</div><div class='add'>+	IRDMA_OP_STATS_GATHER			= 36,</div><div class='add'>+	IRDMA_OP_WS_ADD_NODE			= 37,</div><div class='add'>+	IRDMA_OP_WS_MODIFY_NODE			= 38,</div><div class='add'>+	IRDMA_OP_WS_DELETE_NODE			= 39,</div><div class='add'>+	IRDMA_OP_WS_FAILOVER_START		= 40,</div><div class='add'>+	IRDMA_OP_WS_FAILOVER_COMPLETE		= 41,</div><div class='add'>+	IRDMA_OP_SET_UP_MAP			= 42,</div><div class='add'>+	IRDMA_OP_GEN_AE				= 43,</div><div class='add'>+	IRDMA_OP_QUERY_RDMA_FEATURES		= 44,</div><div class='add'>+	IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY		= 45,</div><div class='add'>+	IRDMA_OP_ADD_LOCAL_MAC_ENTRY		= 46,</div><div class='add'>+	IRDMA_OP_DELETE_LOCAL_MAC_ENTRY		= 47,</div><div class='add'>+	IRDMA_OP_CQ_MODIFY			= 48,</div><div class='add'>+	IRDMA_OP_SRQ_CREATE			= 49,</div><div class='add'>+	IRDMA_OP_SRQ_MODIFY			= 50,</div><div class='add'>+	IRDMA_OP_SRQ_DESTROY			= 51,</div><div class='add'>+</div><div class='add'>+	/* Must be last entry*/</div><div class='add'>+	IRDMA_MAX_CQP_OPS			= 52,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* CQP SQ WQES */</div><div class='add'>+#define IRDMA_CQP_OP_CREATE_QP				0</div><div class='add'>+#define IRDMA_CQP_OP_MODIFY_QP				0x1</div><div class='add'>+#define IRDMA_CQP_OP_DESTROY_QP				0x02</div><div class='add'>+#define IRDMA_CQP_OP_CREATE_CQ				0x03</div><div class='add'>+#define IRDMA_CQP_OP_MODIFY_CQ				0x04</div><div class='add'>+#define IRDMA_CQP_OP_DESTROY_CQ				0x05</div><div class='add'>+#define IRDMA_CQP_OP_CREATE_SRQ				0x06</div><div class='add'>+#define IRDMA_CQP_OP_MODIFY_SRQ				0x07</div><div class='add'>+#define IRDMA_CQP_OP_DESTROY_SRQ			0x08</div><div class='add'>+#define IRDMA_CQP_OP_ALLOC_STAG				0x09</div><div class='add'>+#define IRDMA_CQP_OP_REG_MR				0x0a</div><div class='add'>+#define IRDMA_CQP_OP_QUERY_STAG				0x0b</div><div class='add'>+#define IRDMA_CQP_OP_REG_SMR				0x0c</div><div class='add'>+#define IRDMA_CQP_OP_DEALLOC_STAG			0x0d</div><div class='add'>+#define IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE		0x0e</div><div class='add'>+#define IRDMA_CQP_OP_MANAGE_ARP				0x0f</div><div class='add'>+#define IRDMA_CQP_OP_MANAGE_VF_PBLE_BP			0x10</div><div class='add'>+#define IRDMA_CQP_OP_MANAGE_PUSH_PAGES			0x11</div><div class='add'>+#define IRDMA_CQP_OP_QUERY_RDMA_FEATURES		0x12</div><div class='add'>+#define IRDMA_CQP_OP_UPLOAD_CONTEXT			0x13</div><div class='add'>+#define IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY	0x14</div><div class='add'>+#define IRDMA_CQP_OP_UPLOAD_CONTEXT			0x13</div><div class='add'>+#define IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE		0x15</div><div class='add'>+#define IRDMA_CQP_OP_CREATE_CEQ				0x16</div><div class='add'>+#define IRDMA_CQP_OP_DESTROY_CEQ			0x18</div><div class='add'>+#define IRDMA_CQP_OP_CREATE_AEQ				0x19</div><div class='add'>+#define IRDMA_CQP_OP_DESTROY_AEQ			0x1b</div><div class='add'>+#define IRDMA_CQP_OP_CREATE_ADDR_HANDLE			0x1c</div><div class='add'>+#define IRDMA_CQP_OP_MODIFY_ADDR_HANDLE			0x1d</div><div class='add'>+#define IRDMA_CQP_OP_DESTROY_ADDR_HANDLE		0x1e</div><div class='add'>+#define IRDMA_CQP_OP_UPDATE_PE_SDS			0x1f</div><div class='add'>+#define IRDMA_CQP_OP_QUERY_FPM_VAL			0x20</div><div class='add'>+#define IRDMA_CQP_OP_COMMIT_FPM_VAL			0x21</div><div class='add'>+#define IRDMA_CQP_OP_FLUSH_WQES				0x22</div><div class='add'>+/* IRDMA_CQP_OP_GEN_AE is the same value as IRDMA_CQP_OP_FLUSH_WQES */</div><div class='add'>+#define IRDMA_CQP_OP_GEN_AE				0x22</div><div class='add'>+#define IRDMA_CQP_OP_MANAGE_APBVT			0x23</div><div class='add'>+#define IRDMA_CQP_OP_NOP				0x24</div><div class='add'>+#define IRDMA_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY	0x25</div><div class='add'>+#define IRDMA_CQP_OP_CREATE_MCAST_GRP			0x26</div><div class='add'>+#define IRDMA_CQP_OP_MODIFY_MCAST_GRP			0x27</div><div class='add'>+#define IRDMA_CQP_OP_DESTROY_MCAST_GRP			0x28</div><div class='add'>+#define IRDMA_CQP_OP_SUSPEND_QP				0x29</div><div class='add'>+#define IRDMA_CQP_OP_RESUME_QP				0x2a</div><div class='add'>+#define IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED		0x2b</div><div class='add'>+#define IRDMA_CQP_OP_WORK_SCHED_NODE			0x2c</div><div class='add'>+#define IRDMA_CQP_OP_MANAGE_STATS			0x2d</div><div class='add'>+#define IRDMA_CQP_OP_GATHER_STATS			0x2e</div><div class='add'>+#define IRDMA_CQP_OP_UP_MAP				0x2f</div><div class='add'>+</div><div class='add'>+#define FLD_LS_64(dev, val, field)	\</div><div class='add'>+	(((u64)(val) &lt;&lt; (dev)-&gt;hw_shifts[field ## _S]) &amp; (dev)-&gt;hw_masks[field ## _M])</div><div class='add'>+#define FLD_RS_64(dev, val, field)	\</div><div class='add'>+	((u64)((val) &amp; (dev)-&gt;hw_masks[field ## _M]) &gt;&gt; (dev)-&gt;hw_shifts[field ## _S])</div><div class='add'>+#define FLD_LS_32(dev, val, field)	\</div><div class='add'>+	(((val) &lt;&lt; (dev)-&gt;hw_shifts[field ## _S]) &amp; (dev)-&gt;hw_masks[field ## _M])</div><div class='add'>+#define FLD_RS_32(dev, val, field)	\</div><div class='add'>+	((u64)((val) &amp; (dev)-&gt;hw_masks[field ## _M]) &gt;&gt; (dev)-&gt;hw_shifts[field ## _S])</div><div class='add'>+</div><div class='add'>+#define IRDMA_MAX_STATS_24	0xffffffULL</div><div class='add'>+#define IRDMA_MAX_STATS_32	0xffffffffULL</div><div class='add'>+#define IRDMA_MAX_STATS_48	0xffffffffffffULL</div><div class='add'>+#define IRDMA_MAX_STATS_56	0xffffffffffffffULL</div><div class='add'>+#define IRDMA_MAX_STATS_64	0xffffffffffffffffULL</div><div class='add'>+</div><div class='add'>+#define IRDMA_MAX_CQ_READ_THRESH 0x3FFFF</div><div class='add'>+#define IRDMA_CQPSQ_QHASH_VLANID GENMASK_ULL(43, 32)</div><div class='add'>+#define IRDMA_CQPSQ_QHASH_QPN GENMASK_ULL(49, 32)</div><div class='add'>+#define IRDMA_CQPSQ_QHASH_QS_HANDLE GENMASK_ULL(9, 0)</div><div class='add'>+#define IRDMA_CQPSQ_QHASH_SRC_PORT GENMASK_ULL(31, 16)</div><div class='add'>+#define IRDMA_CQPSQ_QHASH_DEST_PORT GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMA_CQPSQ_QHASH_ADDR0 GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMA_CQPSQ_QHASH_ADDR1 GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMA_CQPSQ_QHASH_ADDR2 GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMA_CQPSQ_QHASH_ADDR3 GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMA_CQPSQ_QHASH_WQEVALID BIT_ULL(63)</div><div class='add'>+#define IRDMA_CQPSQ_QHASH_OPCODE GENMASK_ULL(37, 32)</div><div class='add'>+#define IRDMA_CQPSQ_QHASH_MANAGE GENMASK_ULL(62, 61)</div><div class='add'>+#define IRDMA_CQPSQ_QHASH_IPV4VALID BIT_ULL(60)</div><div class='add'>+#define IRDMA_CQPSQ_QHASH_VLANVALID BIT_ULL(59)</div><div class='add'>+#define IRDMA_CQPSQ_QHASH_ENTRYTYPE GENMASK_ULL(44, 42)</div><div class='add'>+#define IRDMA_CQPSQ_STATS_WQEVALID BIT_ULL(63)</div><div class='add'>+#define IRDMA_CQPSQ_STATS_ALLOC_INST BIT_ULL(62)</div><div class='add'>+#define IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX BIT_ULL(60)</div><div class='add'>+#define IRDMA_CQPSQ_STATS_USE_INST BIT_ULL(61)</div><div class='add'>+#define IRDMA_CQPSQ_STATS_OP GENMASK_ULL(37, 32)</div><div class='add'>+#define IRDMA_CQPSQ_STATS_INST_INDEX GENMASK_ULL(6, 0)</div><div class='add'>+#define IRDMA_CQPSQ_STATS_HMC_FCN_INDEX GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMA_CQPSQ_WS_WQEVALID BIT_ULL(63)</div><div class='add'>+#define IRDMA_CQPSQ_WS_NODEOP GENMASK_ULL(55, 52)</div><div class='add'>+#define IRDMA_SD_MAX GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMA_MEM_MAX GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMA_QP_MEM_LOC GENMASK_ULL(47, 44)</div><div class='add'>+#define IRDMA_MR_MEM_LOC GENMASK_ULL(27, 24)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_WS_ENABLENODE BIT_ULL(62)</div><div class='add'>+#define IRDMA_CQPSQ_WS_NODETYPE BIT_ULL(61)</div><div class='add'>+#define IRDMA_CQPSQ_WS_PRIOTYPE GENMASK_ULL(60, 59)</div><div class='add'>+#define IRDMA_CQPSQ_WS_TC GENMASK_ULL(58, 56)</div><div class='add'>+#define IRDMA_CQPSQ_WS_VMVFTYPE GENMASK_ULL(55, 54)</div><div class='add'>+#define IRDMA_CQPSQ_WS_VMVFNUM GENMASK_ULL(51, 42)</div><div class='add'>+#define IRDMA_CQPSQ_WS_OP GENMASK_ULL(37, 32)</div><div class='add'>+#define IRDMA_CQPSQ_WS_PARENTID GENMASK_ULL(29, 16)</div><div class='add'>+#define IRDMA_CQPSQ_WS_NODEID GENMASK_ULL(13, 0)</div><div class='add'>+#define IRDMA_CQPSQ_WS_VSI GENMASK_ULL(63, 48)</div><div class='add'>+#define IRDMA_CQPSQ_WS_WEIGHT GENMASK_ULL(38, 32)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_UP_WQEVALID BIT_ULL(63)</div><div class='add'>+#define IRDMA_CQPSQ_UP_USEVLAN BIT_ULL(62)</div><div class='add'>+#define IRDMA_CQPSQ_UP_USEOVERRIDE BIT_ULL(61)</div><div class='add'>+#define IRDMA_CQPSQ_UP_OP GENMASK_ULL(37, 32)</div><div class='add'>+#define IRDMA_CQPSQ_UP_HMCFCNIDX GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMA_CQPSQ_UP_CNPOVERRIDE GENMASK_ULL(37, 32)</div><div class='add'>+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID BIT_ULL(63)</div><div class='add'>+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_OP GENMASK_ULL(37, 32)</div><div class='add'>+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MODEL_USED GENMASK_ULL(47, 32)</div><div class='add'>+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MAJOR_VERSION GENMASK_ULL(23, 16)</div><div class='add'>+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MINOR_VERSION GENMASK_ULL(7, 0)</div><div class='add'>+#define IRDMA_CQPHC_SQSIZE GENMASK_ULL(11, 8)</div><div class='add'>+#define IRDMA_CQPHC_DISABLE_PFPDUS BIT_ULL(1)</div><div class='add'>+#define IRDMA_CQPHC_ROCEV2_RTO_POLICY BIT_ULL(2)</div><div class='add'>+#define IRDMA_CQPHC_PROTOCOL_USED GENMASK_ULL(4, 3)</div><div class='add'>+#define IRDMA_CQPHC_MIN_RATE GENMASK_ULL(51, 48)</div><div class='add'>+#define IRDMA_CQPHC_MIN_DEC_FACTOR GENMASK_ULL(59, 56)</div><div class='add'>+#define IRDMA_CQPHC_DCQCN_T GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMA_CQPHC_HAI_FACTOR GENMASK_ULL(47, 32)</div><div class='add'>+#define IRDMA_CQPHC_RAI_FACTOR GENMASK_ULL(63, 48)</div><div class='add'>+#define IRDMA_CQPHC_DCQCN_B GENMASK_ULL(24, 0)</div><div class='add'>+#define IRDMA_CQPHC_DCQCN_F GENMASK_ULL(27, 25)</div><div class='add'>+#define IRDMA_CQPHC_CC_CFG_VALID BIT_ULL(31)</div><div class='add'>+#define IRDMA_CQPHC_RREDUCE_MPERIOD GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMA_CQPHC_HW_MINVER GENMASK_ULL(15, 0)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPHC_HW_MAJVER_GEN_1 0</div><div class='add'>+#define IRDMA_CQPHC_HW_MAJVER_GEN_2 1</div><div class='add'>+#define IRDMA_CQPHC_HW_MAJVER_GEN_3 2</div><div class='add'>+#define IRDMA_CQPHC_HW_MAJVER GENMASK_ULL(31, 16)</div><div class='add'>+#define IRDMA_CQPHC_CEQPERVF GENMASK_ULL(39, 32)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPHC_ENABLED_VFS GENMASK_ULL(37, 32)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPHC_HMC_PROFILE GENMASK_ULL(2, 0)</div><div class='add'>+#define IRDMA_CQPHC_SVER GENMASK_ULL(31, 24)</div><div class='add'>+#define IRDMA_CQPHC_SQBASE GENMASK_ULL(63, 9)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPHC_TIMESTAMP_OVERRIDE BIT_ULL(5)</div><div class='add'>+#define IRDMA_CQPHC_TS_SHIFT GENMASK_ULL(12, 8)</div><div class='add'>+#define IRDMA_CQPHC_EN_FINE_GRAINED_TIMERS BIT_ULL(0)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPHC_OOISC_BLKSIZE GENMASK_ULL(63, 60)</div><div class='add'>+#define IRDMA_CQPHC_RRSP_BLKSIZE GENMASK_ULL(59, 56)</div><div class='add'>+#define IRDMA_CQPHC_Q1_BLKSIZE GENMASK_ULL(55, 52)</div><div class='add'>+#define IRDMA_CQPHC_XMIT_BLKSIZE GENMASK_ULL(51, 48)</div><div class='add'>+#define IRDMA_CQPHC_BLKSIZES_VALID BIT_ULL(4)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPHC_QPCTX GENMASK_ULL(63, 0)</div><div class='add'>+#define IRDMA_QP_DBSA_HW_SQ_TAIL GENMASK_ULL(14, 0)</div><div class='add'>+#define IRDMA_CQ_DBSA_CQEIDX GENMASK_ULL(19, 0)</div><div class='add'>+#define IRDMA_CQ_DBSA_SW_CQ_SELECT GENMASK_ULL(13, 0)</div><div class='add'>+#define IRDMA_CQ_DBSA_ARM_NEXT BIT_ULL(14)</div><div class='add'>+#define IRDMA_CQ_DBSA_ARM_NEXT_SE BIT_ULL(15)</div><div class='add'>+#define IRDMA_CQ_DBSA_ARM_SEQ_NUM GENMASK_ULL(17, 16)</div><div class='add'>+</div><div class='add'>+/* CQP and iWARP Completion Queue */</div><div class='add'>+#define IRDMA_CQ_QPCTX IRDMA_CQPHC_QPCTX</div><div class='add'>+</div><div class='add'>+#define IRDMA_CCQ_OPRETVAL GENMASK_ULL(31, 0)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CCQ_DEFINFO GENMASK_ULL(63, 32)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQ_MINERR GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMA_CQ_MAJERR GENMASK_ULL(31, 16)</div><div class='add'>+#define IRDMA_CQ_WQEIDX GENMASK_ULL(46, 32)</div><div class='add'>+#define IRDMA_CQ_EXTCQE BIT_ULL(50)</div><div class='add'>+#define IRDMA_OOO_CMPL BIT_ULL(54)</div><div class='add'>+#define IRDMA_CQ_ERROR BIT_ULL(55)</div><div class='add'>+#define IRDMA_CQ_SQ BIT_ULL(62)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQ_SRQ BIT_ULL(52)</div><div class='add'>+#define IRDMA_CQ_VALID BIT_ULL(63)</div><div class='add'>+#define IRDMA_CQ_IMMVALID BIT_ULL(62)</div><div class='add'>+#define IRDMA_CQ_UDSMACVALID BIT_ULL(61)</div><div class='add'>+#define IRDMA_CQ_UDVLANVALID BIT_ULL(60)</div><div class='add'>+#define IRDMA_CQ_UDSMAC GENMASK_ULL(47, 0)</div><div class='add'>+#define IRDMA_CQ_UDVLAN GENMASK_ULL(63, 48)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQ_IMMDATALOW32 GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMA_CQ_IMMDATAUP32 GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMACQ_PAYLDLEN GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMACQ_TCPSEQNUMRTT GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMACQ_INVSTAG GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMACQ_QPID GENMASK_ULL(55, 32)</div><div class='add'>+</div><div class='add'>+#define IRDMACQ_UDSRCQPN GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMACQ_PSHDROP BIT_ULL(51)</div><div class='add'>+#define IRDMACQ_STAG BIT_ULL(53)</div><div class='add'>+#define IRDMACQ_IPV4 BIT_ULL(53)</div><div class='add'>+#define IRDMACQ_SOEVENT BIT_ULL(54)</div><div class='add'>+#define IRDMACQ_OP GENMASK_ULL(61, 56)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CEQE_CQCTX GENMASK_ULL(62, 0)</div><div class='add'>+#define IRDMA_CEQE_VALID BIT_ULL(63)</div><div class='add'>+</div><div class='add'>+/* AEQE format */</div><div class='add'>+#define IRDMA_AEQE_COMPCTX IRDMA_CQPHC_QPCTX</div><div class='add'>+#define IRDMA_AEQE_QPCQID_LOW GENMASK_ULL(17, 0)</div><div class='add'>+#define IRDMA_AEQE_QPCQID_HI BIT_ULL(46)</div><div class='add'>+#define IRDMA_AEQE_WQDESCIDX GENMASK_ULL(32, 18)</div><div class='add'>+#define IRDMA_AEQE_OVERFLOW BIT_ULL(33)</div><div class='add'>+#define IRDMA_AEQE_AECODE GENMASK_ULL(45, 34)</div><div class='add'>+#define IRDMA_AEQE_AESRC GENMASK_ULL(53, 50)</div><div class='add'>+#define IRDMA_AEQE_IWSTATE GENMASK_ULL(56, 54)</div><div class='add'>+#define IRDMA_AEQE_TCPSTATE GENMASK_ULL(60, 57)</div><div class='add'>+#define IRDMA_AEQE_Q2DATA GENMASK_ULL(62, 61)</div><div class='add'>+#define IRDMA_AEQE_VALID BIT_ULL(63)</div><div class='add'>+</div><div class='add'>+#define IRDMA_AEQE_Q2DATA_GEN_3 GENMASK_ULL(5, 4)</div><div class='add'>+#define IRDMA_AEQE_TCPSTATE_GEN_3 GENMASK_ULL(3, 0)</div><div class='add'>+#define IRDMA_AEQE_QPCQID_GEN_3 GENMASK_ULL(24, 0)</div><div class='add'>+#define IRDMA_AEQE_AECODE_GEN_3 GENMASK_ULL(61, 50)</div><div class='add'>+#define IRDMA_AEQE_OVERFLOW_GEN_3 BIT_ULL(62)</div><div class='add'>+#define IRDMA_AEQE_WQDESCIDX_GEN_3 GENMASK_ULL(49, 32)</div><div class='add'>+#define IRDMA_AEQE_IWSTATE_GEN_3 GENMASK_ULL(31, 29)</div><div class='add'>+#define IRDMA_AEQE_AESRC_GEN_3 GENMASK_ULL(28, 25)</div><div class='add'>+#define IRDMA_AEQE_CMPL_CTXT_S 6</div><div class='add'>+#define IRDMA_AEQE_CMPL_CTXT GENMASK_ULL(63, 6)</div><div class='add'>+</div><div class='add'>+#define IRDMA_UDA_QPSQ_NEXT_HDR GENMASK_ULL(23, 16)</div><div class='add'>+#define IRDMA_UDA_QPSQ_OPCODE GENMASK_ULL(37, 32)</div><div class='add'>+#define IRDMA_UDA_QPSQ_L4LEN GENMASK_ULL(45, 42)</div><div class='add'>+#define IRDMA_GEN1_UDA_QPSQ_L4LEN GENMASK_ULL(27, 24)</div><div class='add'>+#define IRDMA_UDA_QPSQ_AHIDX GENMASK_ULL(16, 0)</div><div class='add'>+#define IRDMA_UDA_QPSQ_VALID BIT_ULL(63)</div><div class='add'>+#define IRDMA_UDA_QPSQ_SIGCOMPL BIT_ULL(62)</div><div class='add'>+#define IRDMA_UDA_QPSQ_MACLEN GENMASK_ULL(62, 56)</div><div class='add'>+#define IRDMA_UDA_QPSQ_IPLEN GENMASK_ULL(54, 48)</div><div class='add'>+#define IRDMA_UDA_QPSQ_L4T GENMASK_ULL(31, 30)</div><div class='add'>+#define IRDMA_UDA_QPSQ_IIPT GENMASK_ULL(29, 28)</div><div class='add'>+#define IRDMA_UDA_PAYLOADLEN GENMASK_ULL(13, 0)</div><div class='add'>+#define IRDMA_UDA_HDRLEN GENMASK_ULL(24, 16)</div><div class='add'>+#define IRDMA_VLAN_TAG_VALID BIT_ULL(50)</div><div class='add'>+#define IRDMA_UDA_L3PROTO GENMASK_ULL(1, 0)</div><div class='add'>+#define IRDMA_UDA_L4PROTO GENMASK_ULL(17, 16)</div><div class='add'>+#define IRDMA_UDA_QPSQ_DOLOOPBACK BIT_ULL(44)</div><div class='add'>+#define IRDMA_CQPSQ_BUFSIZE GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMA_CQPSQ_OPCODE GENMASK_ULL(37, 32)</div><div class='add'>+#define IRDMA_CQPSQ_WQEVALID BIT_ULL(63)</div><div class='add'>+#define IRDMA_CQPSQ_TPHVAL GENMASK_ULL(7, 0)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_VSIIDX GENMASK_ULL(23, 8)</div><div class='add'>+#define IRDMA_CQPSQ_TPHEN BIT_ULL(60)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_PBUFADDR IRDMA_CQPHC_QPCTX</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_PASID GENMASK_ULL(51, 32)</div><div class='add'>+#define IRDMA_CQPSQ_PASID_VALID BIT_ULL(62)</div><div class='add'>+</div><div class='add'>+/* Create/Modify/Destroy QP */</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_QP_NEWMSS GENMASK_ULL(45, 32)</div><div class='add'>+#define IRDMA_CQPSQ_QP_TERMLEN GENMASK_ULL(51, 48)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_QP_QPCTX IRDMA_CQPHC_QPCTX</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_QP_QPID_S 0</div><div class='add'>+#define IRDMA_CQPSQ_QP_QPID_M (0xFFFFFFUL)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_QP_OP_S 32</div><div class='add'>+#define IRDMA_CQPSQ_QP_OP_M IRDMACQ_OP_M</div><div class='add'>+#define IRDMA_CQPSQ_QP_ORDVALID BIT_ULL(42)</div><div class='add'>+#define IRDMA_CQPSQ_QP_TOECTXVALID BIT_ULL(43)</div><div class='add'>+#define IRDMA_CQPSQ_QP_CACHEDVARVALID BIT_ULL(44)</div><div class='add'>+#define IRDMA_CQPSQ_QP_VQ BIT_ULL(45)</div><div class='add'>+#define IRDMA_CQPSQ_QP_FORCELOOPBACK BIT_ULL(46)</div><div class='add'>+#define IRDMA_CQPSQ_QP_CQNUMVALID BIT_ULL(47)</div><div class='add'>+#define IRDMA_CQPSQ_QP_QPTYPE GENMASK_ULL(50, 48)</div><div class='add'>+#define IRDMA_CQPSQ_QP_MACVALID BIT_ULL(51)</div><div class='add'>+#define IRDMA_CQPSQ_QP_MSSCHANGE BIT_ULL(52)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_QP_IGNOREMWBOUND BIT_ULL(54)</div><div class='add'>+#define IRDMA_CQPSQ_QP_REMOVEHASHENTRY BIT_ULL(55)</div><div class='add'>+#define IRDMA_CQPSQ_QP_TERMACT GENMASK_ULL(57, 56)</div><div class='add'>+#define IRDMA_CQPSQ_QP_RESETCON BIT_ULL(58)</div><div class='add'>+#define IRDMA_CQPSQ_QP_ARPTABIDXVALID BIT_ULL(59)</div><div class='add'>+#define IRDMA_CQPSQ_QP_NEXTIWSTATE GENMASK_ULL(62, 60)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_QP_DBSHADOWADDR IRDMA_CQPHC_QPCTX</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_RQSIZE GENMASK_ULL(3, 0)</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_RQ_WQE_SIZE GENMASK_ULL(5, 4)</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_SRQ_LIMIT GENMASK_ULL(43, 32)</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_SRQCTX GENMASK_ULL(63, 6)</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_PD_ID GENMASK_ULL(39, 16)</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_SRQ_ID GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_OP GENMASK_ULL(37, 32)</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_LEAF_PBL_SIZE GENMASK_ULL(45, 44)</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_VIRTMAP BIT_ULL(47)</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_TPH_EN BIT_ULL(60)</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_ARM_LIMIT_EVENT BIT_ULL(61)</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_FIRST_PM_PBL_IDX GENMASK_ULL(27, 0)</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_TPH_VALUE GENMASK_ULL(7, 0)</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_PHYSICAL_BUFFER_ADDR_S 8</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_PHYSICAL_BUFFER_ADDR GENMASK_ULL(63, 8)</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_DB_SHADOW_ADDR_S 6</div><div class='add'>+#define IRDMA_CQPSQ_SRQ_DB_SHADOW_ADDR GENMASK_ULL(63, 6)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_CQ_CQSIZE GENMASK_ULL(20, 0)</div><div class='add'>+#define IRDMA_CQPSQ_CQ_CQCTX GENMASK_ULL(62, 0)</div><div class='add'>+#define IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD GENMASK(17, 0)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_CQ_CQID_HIGH GENMASK_ULL(52, 50)</div><div class='add'>+#define IRDMA_CQPSQ_CQ_CEQID_HIGH GENMASK_ULL(59, 54)</div><div class='add'>+#define IRDMA_CQPSQ_CQ_OP GENMASK_ULL(37, 32)</div><div class='add'>+#define IRDMA_CQPSQ_CQ_CQRESIZE BIT_ULL(43)</div><div class='add'>+#define IRDMA_CQPSQ_CQ_LPBLSIZE GENMASK_ULL(45, 44)</div><div class='add'>+#define IRDMA_CQPSQ_CQ_CHKOVERFLOW BIT_ULL(46)</div><div class='add'>+#define IRDMA_CQPSQ_CQ_VIRTMAP BIT_ULL(47)</div><div class='add'>+#define IRDMA_CQPSQ_CQ_ENCEQEMASK BIT_ULL(48)</div><div class='add'>+#define IRDMA_CQPSQ_CQ_CEQIDVALID BIT_ULL(49)</div><div class='add'>+#define IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT BIT_ULL(61)</div><div class='add'>+#define IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX GENMASK_ULL(27, 0)</div><div class='add'>+</div><div class='add'>+/* Allocate/Register/Register Shared/Deallocate Stag */</div><div class='add'>+#define IRDMA_CQPSQ_STAG_VA_FBO IRDMA_CQPHC_QPCTX</div><div class='add'>+#define IRDMA_CQPSQ_STAG_STAGLEN GENMASK_ULL(45, 0)</div><div class='add'>+#define IRDMA_CQPSQ_STAG_KEY GENMASK_ULL(7, 0)</div><div class='add'>+#define IRDMA_CQPSQ_STAG_IDX GENMASK_ULL(31, 8)</div><div class='add'>+#define IRDMA_CQPSQ_STAG_IDX_S 8</div><div class='add'>+#define IRDMA_CQPSQ_STAG_PARENTSTAGIDX GENMASK_ULL(55, 32)</div><div class='add'>+#define IRDMA_CQPSQ_STAG_MR BIT_ULL(43)</div><div class='add'>+#define IRDMA_CQPSQ_STAG_MWTYPE BIT_ULL(42)</div><div class='add'>+#define IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY BIT_ULL(58)</div><div class='add'>+#define IRDMA_CQPSQ_STAG_PDID_HI GENMASK_ULL(59, 54)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_STAG_LPBLSIZE IRDMA_CQPSQ_CQ_LPBLSIZE</div><div class='add'>+#define IRDMA_CQPSQ_STAG_HPAGESIZE GENMASK_ULL(47, 46)</div><div class='add'>+#define IRDMA_CQPSQ_STAG_ARIGHTS GENMASK_ULL(52, 48)</div><div class='add'>+#define IRDMA_CQPSQ_STAG_REMACCENABLED BIT_ULL(53)</div><div class='add'>+#define IRDMA_CQPSQ_STAG_VABASEDTO BIT_ULL(59)</div><div class='add'>+#define IRDMA_CQPSQ_STAG_USEHMCFNIDX BIT_ULL(60)</div><div class='add'>+#define IRDMA_CQPSQ_STAG_USEPFRID BIT_ULL(61)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_STAG_PBA IRDMA_CQPHC_QPCTX</div><div class='add'>+#define IRDMA_CQPSQ_STAG_HMCFNIDX GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMA_CQPSQ_STAG_REMOTE_ATOMIC_EN BIT_ULL(61)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX GENMASK_ULL(27, 0)</div><div class='add'>+#define IRDMA_CQPSQ_QUERYSTAG_IDX IRDMA_CQPSQ_STAG_IDX</div><div class='add'>+#define IRDMA_CQPSQ_MLM_TABLEIDX GENMASK_ULL(5, 0)</div><div class='add'>+#define IRDMA_CQPSQ_MLM_FREEENTRY BIT_ULL(62)</div><div class='add'>+#define IRDMA_CQPSQ_MLM_IGNORE_REF_CNT BIT_ULL(61)</div><div class='add'>+#define IRDMA_CQPSQ_MLM_MAC0 GENMASK_ULL(7, 0)</div><div class='add'>+#define IRDMA_CQPSQ_MLM_MAC1 GENMASK_ULL(15, 8)</div><div class='add'>+#define IRDMA_CQPSQ_MLM_MAC2 GENMASK_ULL(23, 16)</div><div class='add'>+#define IRDMA_CQPSQ_MLM_MAC3 GENMASK_ULL(31, 24)</div><div class='add'>+#define IRDMA_CQPSQ_MLM_MAC4 GENMASK_ULL(39, 32)</div><div class='add'>+#define IRDMA_CQPSQ_MLM_MAC5 GENMASK_ULL(47, 40)</div><div class='add'>+#define IRDMA_CQPSQ_MAT_REACHMAX GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMA_CQPSQ_MAT_MACADDR GENMASK_ULL(47, 0)</div><div class='add'>+#define IRDMA_CQPSQ_MAT_ARPENTRYIDX GENMASK_ULL(11, 0)</div><div class='add'>+#define IRDMA_CQPSQ_MAT_ENTRYVALID BIT_ULL(42)</div><div class='add'>+#define IRDMA_CQPSQ_MAT_PERMANENT BIT_ULL(43)</div><div class='add'>+#define IRDMA_CQPSQ_MAT_QUERY BIT_ULL(44)</div><div class='add'>+#define IRDMA_CQPSQ_MVPBP_PD_ENTRY_CNT GENMASK_ULL(9, 0)</div><div class='add'>+#define IRDMA_CQPSQ_MVPBP_FIRST_PD_INX GENMASK_ULL(24, 16)</div><div class='add'>+#define IRDMA_CQPSQ_MVPBP_SD_INX GENMASK_ULL(43, 32)</div><div class='add'>+#define IRDMA_CQPSQ_MVPBP_INV_PD_ENT BIT_ULL(62)</div><div class='add'>+#define IRDMA_CQPSQ_MVPBP_PD_PLPBA GENMASK_ULL(63, 3)</div><div class='add'>+</div><div class='add'>+/* Manage Push Page - MPP */</div><div class='add'>+#define IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1 0xffff</div><div class='add'>+#define IRDMA_INVALID_PUSH_PAGE_INDEX 0xffffffff</div><div class='add'>+#define IRDMA_CQPSQ_MPP_PPIDX GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMA_CQPSQ_MPP_PPTYPE GENMASK_ULL(61, 60)</div><div class='add'>+#define IRDMA_CQPSQ_MPP_FREE_PAGE BIT_ULL(62)</div><div class='add'>+</div><div class='add'>+/* Upload Context - UCTX */</div><div class='add'>+#define IRDMA_CQPSQ_UCTX_QPCTXADDR IRDMA_CQPHC_QPCTX</div><div class='add'>+#define IRDMA_CQPSQ_UCTX_QPID GENMASK_ULL(23, 0)</div><div class='add'>+#define IRDMA_CQPSQ_UCTX_QPTYPE GENMASK_ULL(51, 48)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_UCTX_RAWFORMAT BIT_ULL(61)</div><div class='add'>+#define IRDMA_CQPSQ_UCTX_FREEZEQP BIT_ULL(62)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_MHMC_VFIDX GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMA_CQPSQ_MHMC_FREEPMFN BIT_ULL(62)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_SHMCRP_HMC_PROFILE GENMASK_ULL(2, 0)</div><div class='add'>+#define IRDMA_CQPSQ_SHMCRP_VFNUM GENMASK_ULL(37, 32)</div><div class='add'>+#define IRDMA_CQPSQ_CEQ_CEQSIZE GENMASK_ULL(21, 0)</div><div class='add'>+#define IRDMA_CQPSQ_CEQ_CEQID GENMASK_ULL(9, 0)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_CEQ_CEQID_HIGH GENMASK_ULL(15, 10)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_CEQ_LPBLSIZE IRDMA_CQPSQ_CQ_LPBLSIZE</div><div class='add'>+#define IRDMA_CQPSQ_CEQ_VMAP BIT_ULL(47)</div><div class='add'>+#define IRDMA_CQPSQ_CEQ_ITRNOEXPIRE BIT_ULL(46)</div><div class='add'>+#define IRDMA_CQPSQ_CEQ_FIRSTPMPBLIDX GENMASK_ULL(27, 0)</div><div class='add'>+#define IRDMA_CQPSQ_AEQ_AEQECNT GENMASK_ULL(18, 0)</div><div class='add'>+#define IRDMA_CQPSQ_AEQ_LPBLSIZE IRDMA_CQPSQ_CQ_LPBLSIZE</div><div class='add'>+#define IRDMA_CQPSQ_AEQ_VMAP BIT_ULL(47)</div><div class='add'>+#define IRDMA_CQPSQ_AEQ_FIRSTPMPBLIDX GENMASK_ULL(27, 0)</div><div class='add'>+</div><div class='add'>+#define IRDMA_COMMIT_FPM_QPCNT GENMASK_ULL(20, 0)</div><div class='add'>+#define IRDMA_COMMIT_FPM_BASE_S 32</div><div class='add'>+#define IRDMA_CQPSQ_CFPM_HMCFNID GENMASK_ULL(15, 0)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_FWQE_AECODE GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMA_CQPSQ_FWQE_AESOURCE GENMASK_ULL(19, 16)</div><div class='add'>+#define IRDMA_CQPSQ_FWQE_RQMNERR GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMA_CQPSQ_FWQE_RQMJERR GENMASK_ULL(31, 16)</div><div class='add'>+#define IRDMA_CQPSQ_FWQE_SQMNERR GENMASK_ULL(47, 32)</div><div class='add'>+#define IRDMA_CQPSQ_FWQE_SQMJERR GENMASK_ULL(63, 48)</div><div class='add'>+#define IRDMA_CQPSQ_FWQE_QPID GENMASK_ULL(23, 0)</div><div class='add'>+#define IRDMA_CQPSQ_FWQE_GENERATE_AE BIT_ULL(59)</div><div class='add'>+#define IRDMA_CQPSQ_FWQE_USERFLCODE BIT_ULL(60)</div><div class='add'>+#define IRDMA_CQPSQ_FWQE_FLUSHSQ BIT_ULL(61)</div><div class='add'>+#define IRDMA_CQPSQ_FWQE_FLUSHRQ BIT_ULL(62)</div><div class='add'>+#define IRDMA_CQPSQ_FWQE_ERR_SQ_IDX_VALID BIT_ULL(42)</div><div class='add'>+#define IRDMA_CQPSQ_FWQE_ERR_SQ_IDX GENMASK_ULL(49, 32)</div><div class='add'>+#define IRDMA_CQPSQ_FWQE_ERR_RQ_IDX_VALID BIT_ULL(43)</div><div class='add'>+#define IRDMA_CQPSQ_FWQE_ERR_RQ_IDX GENMASK_ULL(46, 32)</div><div class='add'>+#define IRDMA_CQPSQ_MAPT_PORT GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMA_CQPSQ_MAPT_ADDPORT BIT_ULL(62)</div><div class='add'>+#define IRDMA_CQPSQ_UPESD_SDCMD GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMA_CQPSQ_UPESD_SDDATALOW GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMA_CQPSQ_UPESD_SDDATAHI GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMA_CQPSQ_UPESD_HMCFNID GENMASK_ULL(5, 0)</div><div class='add'>+#define IRDMA_CQPSQ_UPESD_ENTRY_VALID BIT_ULL(63)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_UPESD_BM_PF 0</div><div class='add'>+#define IRDMA_CQPSQ_UPESD_BM_CP_LM 1</div><div class='add'>+#define IRDMA_CQPSQ_UPESD_BM_AXF 2</div><div class='add'>+#define IRDMA_CQPSQ_UPESD_BM_LM 4</div><div class='add'>+#define IRDMA_CQPSQ_UPESD_BM GENMASK_ULL(34, 32)</div><div class='add'>+#define IRDMA_CQPSQ_UPESD_ENTRY_COUNT GENMASK_ULL(3, 0)</div><div class='add'>+#define IRDMA_CQPSQ_UPESD_SKIP_ENTRY BIT_ULL(7)</div><div class='add'>+#define IRDMA_CQPSQ_SUSPENDQP_QPID GENMASK_ULL(23, 0)</div><div class='add'>+#define IRDMA_CQPSQ_RESUMEQP_QSHANDLE GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMA_CQPSQ_RESUMEQP_QPID GENMASK(23, 0)</div><div class='add'>+#define IRDMA_MANAGE_RSRC_VER2 BIT_ULL(2)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_MIN_STAG_INVALID 0x0001</div><div class='add'>+#define IRDMA_CQPSQ_MIN_SUSPEND_PND 0x0005</div><div class='add'>+#define IRDMA_CQPSQ_MIN_DEF_CMPL 0x0006</div><div class='add'>+#define IRDMA_CQPSQ_MIN_OOO_CMPL 0x0007</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPSQ_MAJ_NO_ERROR 0x0000</div><div class='add'>+#define IRDMA_CQPSQ_MAJ_OBJCACHE_ERROR 0xF000</div><div class='add'>+#define IRDMA_CQPSQ_MAJ_CNTXTCACHE_ERROR 0xF001</div><div class='add'>+#define IRDMA_CQPSQ_MAJ_ERROR 0xFFFF</div><div class='add'>+#define IRDMAQPC_DDP_VER GENMASK_ULL(1, 0)</div><div class='add'>+#define IRDMAQPC_IBRDENABLE BIT_ULL(2)</div><div class='add'>+#define IRDMAQPC_IPV4 BIT_ULL(3)</div><div class='add'>+#define IRDMAQPC_NONAGLE BIT_ULL(4)</div><div class='add'>+#define IRDMAQPC_INSERTVLANTAG BIT_ULL(5)</div><div class='add'>+#define IRDMAQPC_ISQP1 BIT_ULL(6)</div><div class='add'>+#define IRDMAQPC_TIMESTAMP BIT_ULL(7)</div><div class='add'>+#define IRDMAQPC_RQWQESIZE GENMASK_ULL(9, 8)</div><div class='add'>+#define IRDMAQPC_INSERTL2TAG2 BIT_ULL(11)</div><div class='add'>+#define IRDMAQPC_LIMIT GENMASK_ULL(13, 12)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPC_USE_SRQ BIT_ULL(10)</div><div class='add'>+#define IRDMAQPC_SRQ_ID GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMAQPC_PASID GENMASK_ULL(19, 0)</div><div class='add'>+#define IRDMAQPC_PASID_VALID BIT_ULL(11)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPC_ECN_EN BIT_ULL(14)</div><div class='add'>+#define IRDMAQPC_DROPOOOSEG BIT_ULL(15)</div><div class='add'>+#define IRDMAQPC_DUPACK_THRESH GENMASK_ULL(18, 16)</div><div class='add'>+#define IRDMAQPC_ERR_RQ_IDX_VALID BIT_ULL(19)</div><div class='add'>+#define IRDMAQPC_DIS_VLAN_CHECKS GENMASK_ULL(21, 19)</div><div class='add'>+#define IRDMAQPC_DC_TCP_EN BIT_ULL(25)</div><div class='add'>+#define IRDMAQPC_RCVTPHEN BIT_ULL(28)</div><div class='add'>+#define IRDMAQPC_XMITTPHEN BIT_ULL(29)</div><div class='add'>+#define IRDMAQPC_RQTPHEN BIT_ULL(30)</div><div class='add'>+#define IRDMAQPC_SQTPHEN BIT_ULL(31)</div><div class='add'>+#define IRDMAQPC_PPIDX GENMASK_ULL(41, 32)</div><div class='add'>+#define IRDMAQPC_PMENA BIT_ULL(47)</div><div class='add'>+#define IRDMAQPC_RDMAP_VER GENMASK_ULL(63, 62)</div><div class='add'>+#define IRDMAQPC_ROCE_TVER GENMASK_ULL(63, 60)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPC_SQADDR IRDMA_CQPHC_QPCTX</div><div class='add'>+#define IRDMAQPC_RQADDR IRDMA_CQPHC_QPCTX</div><div class='add'>+#define IRDMAQPC_TTL GENMASK_ULL(7, 0)</div><div class='add'>+#define IRDMAQPC_RQSIZE GENMASK_ULL(11, 8)</div><div class='add'>+#define IRDMAQPC_SQSIZE GENMASK_ULL(15, 12)</div><div class='add'>+#define IRDMAQPC_GEN1_SRCMACADDRIDX GENMASK(21, 16)</div><div class='add'>+#define IRDMAQPC_AVOIDSTRETCHACK BIT_ULL(23)</div><div class='add'>+#define IRDMAQPC_TOS GENMASK_ULL(31, 24)</div><div class='add'>+#define IRDMAQPC_SRCPORTNUM GENMASK_ULL(47, 32)</div><div class='add'>+#define IRDMAQPC_DESTPORTNUM GENMASK_ULL(63, 48)</div><div class='add'>+#define IRDMAQPC_DESTIPADDR0 GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMAQPC_DESTIPADDR1 GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPC_DESTIPADDR2 GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMAQPC_DESTIPADDR3 GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPC_SNDMSS GENMASK_ULL(29, 16)</div><div class='add'>+#define IRDMAQPC_SYN_RST_HANDLING GENMASK_ULL(31, 30)</div><div class='add'>+#define IRDMAQPC_VLANTAG GENMASK_ULL(47, 32)</div><div class='add'>+#define IRDMAQPC_ARPIDX GENMASK_ULL(63, 48)</div><div class='add'>+#define IRDMAQPC_FLOWLABEL GENMASK_ULL(19, 0)</div><div class='add'>+#define IRDMAQPC_WSCALE BIT_ULL(20)</div><div class='add'>+#define IRDMAQPC_KEEPALIVE BIT_ULL(21)</div><div class='add'>+#define IRDMAQPC_IGNORE_TCP_OPT BIT_ULL(22)</div><div class='add'>+#define IRDMAQPC_IGNORE_TCP_UNS_OPT BIT_ULL(23)</div><div class='add'>+#define IRDMAQPC_TCPSTATE GENMASK_ULL(31, 28)</div><div class='add'>+#define IRDMAQPC_RCVSCALE GENMASK_ULL(35, 32)</div><div class='add'>+#define IRDMAQPC_SNDSCALE GENMASK_ULL(43, 40)</div><div class='add'>+#define IRDMAQPC_PDIDX GENMASK_ULL(63, 48)</div><div class='add'>+#define IRDMAQPC_PDIDXHI GENMASK_ULL(21, 20)</div><div class='add'>+#define IRDMAQPC_PKEY GENMASK_ULL(47, 32)</div><div class='add'>+#define IRDMAQPC_ACKCREDITS GENMASK_ULL(24, 20)</div><div class='add'>+#define IRDMAQPC_QKEY GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMAQPC_DESTQP GENMASK_ULL(23, 0)</div><div class='add'>+#define IRDMAQPC_KALIVE_TIMER_MAX_PROBES GENMASK_ULL(23, 16)</div><div class='add'>+#define IRDMAQPC_KEEPALIVE_INTERVAL GENMASK_ULL(31, 24)</div><div class='add'>+#define IRDMAQPC_TIMESTAMP_RECENT GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPC_TIMESTAMP_AGE GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMAQPC_SNDNXT GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPC_ISN GENMASK_ULL(55, 32)</div><div class='add'>+#define IRDMAQPC_PSNNXT GENMASK_ULL(23, 0)</div><div class='add'>+#define IRDMAQPC_LSN GENMASK_ULL(55, 32)</div><div class='add'>+#define IRDMAQPC_SNDWND GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMAQPC_RCVNXT GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPC_EPSN GENMASK_ULL(23, 0)</div><div class='add'>+#define IRDMAQPC_RCVWND GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMAQPC_SNDMAX GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPC_SNDUNA GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMAQPC_PSNMAX GENMASK_ULL(23, 0)</div><div class='add'>+#define IRDMAQPC_PSNUNA GENMASK_ULL(55, 32)</div><div class='add'>+#define IRDMAQPC_SRTT GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPC_RTTVAR GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMAQPC_SSTHRESH GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPC_CWND GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMAQPC_CWNDROCE GENMASK_ULL(55, 32)</div><div class='add'>+#define IRDMAQPC_SNDWL1 GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPC_SNDWL2 GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMAQPC_MINRNR_TIMER GENMASK_ULL(4, 0)</div><div class='add'>+#define IRDMAQPC_ERR_RQ_IDX GENMASK_ULL(46, 32)</div><div class='add'>+#define IRDMAQPC_RTOMIN GENMASK_ULL(63, 57)</div><div class='add'>+#define IRDMAQPC_MAXSNDWND GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPC_REXMIT_THRESH GENMASK_ULL(53, 48)</div><div class='add'>+#define IRDMAQPC_RNRNAK_THRESH GENMASK_ULL(56, 54)</div><div class='add'>+#define IRDMAQPC_TXCQNUM GENMASK_ULL(24, 0)</div><div class='add'>+#define IRDMAQPC_RXCQNUM GENMASK_ULL(56, 32)</div><div class='add'>+#define IRDMAQPC_STAT_INDEX GENMASK_ULL(6, 0)</div><div class='add'>+#define IRDMAQPC_Q2ADDR GENMASK_ULL(63, 8)</div><div class='add'>+#define IRDMAQPC_LASTBYTESENT GENMASK_ULL(7, 0)</div><div class='add'>+#define IRDMAQPC_MACADDRESS GENMASK_ULL(63, 16)</div><div class='add'>+#define IRDMAQPC_ORDSIZE GENMASK_ULL(7, 0)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPC_LOCALACKTIMEOUT GENMASK_ULL(12, 8)</div><div class='add'>+#define IRDMAQPC_RNRNAK_TMR GENMASK_ULL(4, 0)</div><div class='add'>+#define IRDMAQPC_ORDSIZE_GEN3 GENMASK_ULL(10, 0)</div><div class='add'>+#define IRDMAQPC_REMOTE_ATOMIC_EN BIT_ULL(18)</div><div class='add'>+#define IRDMAQPC_STAT_INDEX_GEN3 GENMASK_ULL(47, 32)</div><div class='add'>+#define IRDMAQPC_PKT_LIMIT GENMASK_ULL(55, 48)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPC_IRDSIZE GENMASK_ULL(18, 16)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPC_IRDSIZE_GEN3 GENMASK_ULL(17, 14)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPC_UDPRIVCQENABLE BIT_ULL(19)</div><div class='add'>+#define IRDMAQPC_WRRDRSPOK BIT_ULL(20)</div><div class='add'>+#define IRDMAQPC_RDOK BIT_ULL(21)</div><div class='add'>+#define IRDMAQPC_SNDMARKERS BIT_ULL(22)</div><div class='add'>+#define IRDMAQPC_DCQCNENABLE BIT_ULL(22)</div><div class='add'>+#define IRDMAQPC_FW_CC_ENABLE BIT_ULL(28)</div><div class='add'>+#define IRDMAQPC_RCVNOICRC BIT_ULL(31)</div><div class='add'>+#define IRDMAQPC_BINDEN BIT_ULL(23)</div><div class='add'>+#define IRDMAQPC_FASTREGEN BIT_ULL(24)</div><div class='add'>+#define IRDMAQPC_PRIVEN BIT_ULL(25)</div><div class='add'>+#define IRDMAQPC_TIMELYENABLE BIT_ULL(27)</div><div class='add'>+#define IRDMAQPC_THIGH GENMASK_ULL(63, 52)</div><div class='add'>+#define IRDMAQPC_TLOW GENMASK_ULL(39, 32)</div><div class='add'>+#define IRDMAQPC_REMENDPOINTIDX GENMASK_ULL(16, 0)</div><div class='add'>+#define IRDMAQPC_USESTATSINSTANCE BIT_ULL(26)</div><div class='add'>+#define IRDMAQPC_IWARPMODE BIT_ULL(28)</div><div class='add'>+#define IRDMAQPC_RCVMARKERS BIT_ULL(29)</div><div class='add'>+#define IRDMAQPC_ALIGNHDRS BIT_ULL(30)</div><div class='add'>+#define IRDMAQPC_RCVNOMPACRC BIT_ULL(31)</div><div class='add'>+#define IRDMAQPC_RCVMARKOFFSET GENMASK_ULL(40, 32)</div><div class='add'>+#define IRDMAQPC_SNDMARKOFFSET GENMASK_ULL(56, 48)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPC_QPCOMPCTX IRDMA_CQPHC_QPCTX</div><div class='add'>+#define IRDMAQPC_SQTPHVAL GENMASK_ULL(7, 0)</div><div class='add'>+#define IRDMAQPC_RQTPHVAL GENMASK_ULL(15, 8)</div><div class='add'>+#define IRDMAQPC_QSHANDLE GENMASK_ULL(25, 16)</div><div class='add'>+#define IRDMAQPC_EXCEPTION_LAN_QUEUE GENMASK_ULL(43, 32)</div><div class='add'>+#define IRDMAQPC_LOCAL_IPADDR3 GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPC_LOCAL_IPADDR2 GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMAQPC_LOCAL_IPADDR1 GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPC_LOCAL_IPADDR0 GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMA_FW_VER_MINOR GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMA_FW_VER_MAJOR GENMASK_ULL(31, 16)</div><div class='add'>+#define IRDMA_FEATURE_INFO GENMASK_ULL(47, 0)</div><div class='add'>+#define IRDMA_FEATURE_CNT GENMASK_ULL(47, 32)</div><div class='add'>+#define IRDMA_FEATURE_TYPE GENMASK_ULL(63, 48)</div><div class='add'>+#define IRDMA_FEATURE_RSRC_MAX GENMASK_ULL(31, 0)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPSQ_OPCODE GENMASK_ULL(37, 32)</div><div class='add'>+#define IRDMAQPSQ_COPY_HOST_PBL BIT_ULL(43)</div><div class='add'>+#define IRDMAQPSQ_ADDFRAGCNT GENMASK_ULL(41, 38)</div><div class='add'>+#define IRDMAQPSQ_PUSHWQE BIT_ULL(56)</div><div class='add'>+#define IRDMAQPSQ_STREAMMODE BIT_ULL(58)</div><div class='add'>+#define IRDMAQPSQ_WAITFORRCVPDU BIT_ULL(59)</div><div class='add'>+#define IRDMAQPSQ_READFENCE BIT_ULL(60)</div><div class='add'>+#define IRDMAQPSQ_LOCALFENCE BIT_ULL(61)</div><div class='add'>+#define IRDMAQPSQ_UDPHEADER BIT_ULL(61)</div><div class='add'>+#define IRDMAQPSQ_L4LEN GENMASK_ULL(45, 42)</div><div class='add'>+#define IRDMAQPSQ_SIGCOMPL BIT_ULL(62)</div><div class='add'>+#define IRDMAQPSQ_VALID BIT_ULL(63)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPSQ_FRAG_TO IRDMA_CQPHC_QPCTX</div><div class='add'>+#define IRDMAQPSQ_FRAG_VALID BIT_ULL(63)</div><div class='add'>+#define IRDMAQPSQ_FRAG_LEN GENMASK_ULL(62, 32)</div><div class='add'>+#define IRDMAQPSQ_FRAG_STAG GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPSQ_GEN1_FRAG_LEN GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPSQ_GEN1_FRAG_STAG GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMAQPSQ_REMSTAGINV GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPSQ_DESTQKEY GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPSQ_DESTQPN GENMASK_ULL(55, 32)</div><div class='add'>+#define IRDMAQPSQ_AHID GENMASK_ULL(24, 0)</div><div class='add'>+#define IRDMAQPSQ_INLINEDATAFLAG BIT_ULL(57)</div><div class='add'>+</div><div class='add'>+#define IRDMA_INLINE_VALID_S 7</div><div class='add'>+#define IRDMAQPSQ_INLINEDATALEN GENMASK_ULL(55, 48)</div><div class='add'>+#define IRDMAQPSQ_IMMDATAFLAG BIT_ULL(47)</div><div class='add'>+#define IRDMAQPSQ_REPORTRTT BIT_ULL(46)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPSQ_IMMDATA GENMASK_ULL(63, 0)</div><div class='add'>+#define IRDMAQPSQ_REMSTAG GENMASK_ULL(31, 0)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPSQ_REMTO IRDMA_CQPHC_QPCTX</div><div class='add'>+</div><div class='add'>+#define IRDMAQPSQ_STAG GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMAQPSQ_REMOTE_STAG GENMASK_ULL(31, 0)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPSQ_STAGRIGHTS GENMASK_ULL(52, 48)</div><div class='add'>+#define IRDMAQPSQ_VABASEDTO BIT_ULL(53)</div><div class='add'>+#define IRDMAQPSQ_MEMWINDOWTYPE BIT_ULL(54)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPSQ_MWLEN IRDMA_CQPHC_QPCTX</div><div class='add'>+#define IRDMAQPSQ_PARENTMRSTAG GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMAQPSQ_MWSTAG GENMASK_ULL(31, 0)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPSQ_BASEVA_TO_FBO IRDMA_CQPHC_QPCTX</div><div class='add'>+</div><div class='add'>+#define IRDMAQPSQ_REMOTE_ATOMICS_EN BIT_ULL(55)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPSQ_LOCSTAG GENMASK_ULL(31, 0)</div><div class='add'>+</div><div class='add'>+#define IRDMAQPSQ_STAGKEY GENMASK_ULL(7, 0)</div><div class='add'>+#define IRDMAQPSQ_STAGINDEX GENMASK_ULL(31, 8)</div><div class='add'>+#define IRDMAQPSQ_COPYHOSTPBLS BIT_ULL(43)</div><div class='add'>+#define IRDMAQPSQ_LPBLSIZE GENMASK_ULL(45, 44)</div><div class='add'>+#define IRDMAQPSQ_HPAGESIZE GENMASK_ULL(47, 46)</div><div class='add'>+#define IRDMAQPSQ_STAGLEN GENMASK_ULL(40, 0)</div><div class='add'>+#define IRDMAQPSQ_FIRSTPMPBLIDXLO GENMASK_ULL(63, 48)</div><div class='add'>+#define IRDMAQPSQ_FIRSTPMPBLIDXHI GENMASK_ULL(11, 0)</div><div class='add'>+#define IRDMAQPSQ_PBLADDR GENMASK_ULL(63, 12)</div><div class='add'>+</div><div class='add'>+/* iwarp QP RQ WQE common fields */</div><div class='add'>+#define IRDMAQPRQ_ADDFRAGCNT IRDMAQPSQ_ADDFRAGCNT</div><div class='add'>+#define IRDMAQPRQ_VALID IRDMAQPSQ_VALID</div><div class='add'>+#define IRDMAQPRQ_COMPLCTX IRDMA_CQPHC_QPCTX</div><div class='add'>+#define IRDMAQPRQ_FRAG_LEN IRDMAQPSQ_FRAG_LEN</div><div class='add'>+#define IRDMAQPRQ_STAG IRDMAQPSQ_FRAG_STAG</div><div class='add'>+#define IRDMAQPRQ_TO IRDMAQPSQ_FRAG_TO</div><div class='add'>+</div><div class='add'>+#define IRDMAPFINT_OICR_HMC_ERR_M BIT(26)</div><div class='add'>+#define IRDMAPFINT_OICR_PE_PUSH_M BIT(27)</div><div class='add'>+#define IRDMAPFINT_OICR_PE_CRITERR_M BIT(28)</div><div class='add'>+</div><div class='add'>+#define IRDMA_QUERY_FPM_LOC_MEM_PAGES GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMA_QUERY_FPM_MAX_QPS GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMA_QUERY_FPM_MAX_CQS GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX GENMASK_ULL(13, 0)</div><div class='add'>+#define IRDMA_QUERY_FPM_MAX_PE_SDS GENMASK_ULL(44, 32)</div><div class='add'>+#define IRDMA_QUERY_FPM_MAX_PE_SDS_GEN3 GENMASK_ULL(47, 32)</div><div class='add'>+#define IRDMA_QUERY_FPM_MAX_CEQS GENMASK_ULL(9, 0)</div><div class='add'>+#define IRDMA_QUERY_FPM_MAX_IRD GENMASK_ULL(53, 50)</div><div class='add'>+#define IRDMA_QUERY_FPM_XFBLOCKSIZE GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMA_QUERY_FPM_Q1BLOCKSIZE GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMA_QUERY_FPM_HTMULTIPLIER GENMASK_ULL(19, 16)</div><div class='add'>+#define IRDMA_QUERY_FPM_TIMERBUCKET GENMASK_ULL(47, 32)</div><div class='add'>+#define IRDMA_QUERY_FPM_RRFBLOCKSIZE GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMA_QUERY_FPM_RRFFLBLOCKSIZE GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMA_QUERY_FPM_OOISCFBLOCKSIZE GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID GENMASK_ULL(5, 0)</div><div class='add'>+</div><div class='add'>+#define IRDMA_GET_CURRENT_AEQ_ELEM(_aeq) \</div><div class='add'>+	( \</div><div class='add'>+		(_aeq)-&gt;aeqe_base[IRDMA_RING_CURRENT_TAIL((_aeq)-&gt;aeq_ring)].buf \</div><div class='add'>+	)</div><div class='add'>+</div><div class='add'>+#define IRDMA_GET_CURRENT_CEQ_ELEM(_ceq) \</div><div class='add'>+	( \</div><div class='add'>+		(_ceq)-&gt;ceqe_base[IRDMA_RING_CURRENT_TAIL((_ceq)-&gt;ceq_ring)].buf \</div><div class='add'>+	)</div><div class='add'>+</div><div class='add'>+#define IRDMA_GET_CEQ_ELEM_AT_POS(_ceq, _pos) \</div><div class='add'>+	( \</div><div class='add'>+		(_ceq)-&gt;ceqe_base[_pos].buf  \</div><div class='add'>+	)</div><div class='add'>+</div><div class='add'>+#define IRDMA_RING_GET_NEXT_TAIL(_ring, _idx) \</div><div class='add'>+	( \</div><div class='add'>+		((_ring).tail + (_idx)) % (_ring).size \</div><div class='add'>+	)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQP_INIT_WQE(wqe) memset(wqe, 0, 64)</div><div class='add'>+</div><div class='add'>+#define IRDMA_GET_CURRENT_CQ_ELEM(_cq) \</div><div class='add'>+	( \</div><div class='add'>+		(_cq)-&gt;cq_base[IRDMA_RING_CURRENT_HEAD((_cq)-&gt;cq_ring)].buf  \</div><div class='add'>+	)</div><div class='add'>+#define IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(_cq) \</div><div class='add'>+	( \</div><div class='add'>+		((struct irdma_extended_cqe *) \</div><div class='add'>+		((_cq)-&gt;cq_base))[IRDMA_RING_CURRENT_HEAD((_cq)-&gt;cq_ring)].buf \</div><div class='add'>+	)</div><div class='add'>+</div><div class='add'>+#define IRDMA_RING_INIT(_ring, _size) \</div><div class='add'>+	{ \</div><div class='add'>+		(_ring).head = 0; \</div><div class='add'>+		(_ring).tail = 0; \</div><div class='add'>+		(_ring).size = (_size); \</div><div class='add'>+	}</div><div class='add'>+#define IRDMA_RING_SIZE(_ring) ((_ring).size)</div><div class='add'>+#define IRDMA_RING_CURRENT_HEAD(_ring) ((_ring).head)</div><div class='add'>+#define IRDMA_RING_CURRENT_TAIL(_ring) ((_ring).tail)</div><div class='add'>+</div><div class='add'>+#define IRDMA_RING_MOVE_HEAD(_ring, _retcode) \</div><div class='add'>+	{ \</div><div class='add'>+		register u32 size; \</div><div class='add'>+		size = (_ring).size;  \</div><div class='add'>+		if (!IRDMA_RING_FULL_ERR(_ring)) { \</div><div class='add'>+			(_ring).head = ((_ring).head + 1) % size; \</div><div class='add'>+			(_retcode) = 0; \</div><div class='add'>+		} else { \</div><div class='add'>+			(_retcode) = -ENOMEM; \</div><div class='add'>+		} \</div><div class='add'>+	}</div><div class='add'>+#define IRDMA_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \</div><div class='add'>+	{ \</div><div class='add'>+		register u32 size; \</div><div class='add'>+		size = (_ring).size; \</div><div class='add'>+		if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) &lt; size) { \</div><div class='add'>+			(_ring).head = ((_ring).head + (_count)) % size; \</div><div class='add'>+			(_retcode) = 0; \</div><div class='add'>+		} else { \</div><div class='add'>+			(_retcode) = -ENOMEM; \</div><div class='add'>+		} \</div><div class='add'>+	}</div><div class='add'>+#define IRDMA_SQ_RING_MOVE_HEAD(_ring, _retcode) \</div><div class='add'>+	{ \</div><div class='add'>+		register u32 size; \</div><div class='add'>+		size = (_ring).size;  \</div><div class='add'>+		if (!IRDMA_SQ_RING_FULL_ERR(_ring)) { \</div><div class='add'>+			(_ring).head = ((_ring).head + 1) % size; \</div><div class='add'>+			(_retcode) = 0; \</div><div class='add'>+		} else { \</div><div class='add'>+			(_retcode) = -ENOMEM; \</div><div class='add'>+		} \</div><div class='add'>+	}</div><div class='add'>+#define IRDMA_SQ_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \</div><div class='add'>+	{ \</div><div class='add'>+		register u32 size; \</div><div class='add'>+		size = (_ring).size; \</div><div class='add'>+		if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) &lt; (size - 256)) { \</div><div class='add'>+			(_ring).head = ((_ring).head + (_count)) % size; \</div><div class='add'>+			(_retcode) = 0; \</div><div class='add'>+		} else { \</div><div class='add'>+			(_retcode) = -ENOMEM; \</div><div class='add'>+		} \</div><div class='add'>+	}</div><div class='add'>+#define IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(_ring, _count) \</div><div class='add'>+	(_ring).head = ((_ring).head + (_count)) % (_ring).size</div><div class='add'>+</div><div class='add'>+#define IRDMA_RING_MOVE_TAIL(_ring) \</div><div class='add'>+	(_ring).tail = ((_ring).tail + 1) % (_ring).size</div><div class='add'>+</div><div class='add'>+#define IRDMA_RING_MOVE_HEAD_NOCHECK(_ring) \</div><div class='add'>+	(_ring).head = ((_ring).head + 1) % (_ring).size</div><div class='add'>+</div><div class='add'>+#define IRDMA_RING_MOVE_TAIL_BY_COUNT(_ring, _count) \</div><div class='add'>+	(_ring).tail = ((_ring).tail + (_count)) % (_ring).size</div><div class='add'>+</div><div class='add'>+#define IRDMA_RING_SET_TAIL(_ring, _pos) \</div><div class='add'>+	(_ring).tail = (_pos) % (_ring).size</div><div class='add'>+</div><div class='add'>+#define IRDMA_RING_FULL_ERR(_ring) \</div><div class='add'>+	( \</div><div class='add'>+		(IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 1))  \</div><div class='add'>+	)</div><div class='add'>+</div><div class='add'>+#define IRDMA_ERR_RING_FULL2(_ring) \</div><div class='add'>+	( \</div><div class='add'>+		(IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 2))  \</div><div class='add'>+	)</div><div class='add'>+</div><div class='add'>+#define IRDMA_ERR_RING_FULL3(_ring) \</div><div class='add'>+	( \</div><div class='add'>+		(IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 3))  \</div><div class='add'>+	)</div><div class='add'>+</div><div class='add'>+#define IRDMA_SQ_RING_FULL_ERR(_ring) \</div><div class='add'>+	( \</div><div class='add'>+		(IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 257))  \</div><div class='add'>+	)</div><div class='add'>+</div><div class='add'>+#define IRDMA_ERR_SQ_RING_FULL2(_ring) \</div><div class='add'>+	( \</div><div class='add'>+		(IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 258))  \</div><div class='add'>+	)</div><div class='add'>+#define IRDMA_ERR_SQ_RING_FULL3(_ring) \</div><div class='add'>+	( \</div><div class='add'>+		(IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 259))  \</div><div class='add'>+	)</div><div class='add'>+#define IRDMA_RING_MORE_WORK(_ring) \</div><div class='add'>+	( \</div><div class='add'>+		(IRDMA_RING_USED_QUANTA(_ring) != 0) \</div><div class='add'>+	)</div><div class='add'>+</div><div class='add'>+#define IRDMA_RING_USED_QUANTA(_ring) \</div><div class='add'>+	( \</div><div class='add'>+		(((_ring).head + (_ring).size - (_ring).tail) % (_ring).size) \</div><div class='add'>+	)</div><div class='add'>+</div><div class='add'>+#define IRDMA_RING_FREE_QUANTA(_ring) \</div><div class='add'>+	( \</div><div class='add'>+		((_ring).size - IRDMA_RING_USED_QUANTA(_ring) - 1) \</div><div class='add'>+	)</div><div class='add'>+</div><div class='add'>+#define IRDMA_SQ_RING_FREE_QUANTA(_ring) \</div><div class='add'>+	( \</div><div class='add'>+		((_ring).size - IRDMA_RING_USED_QUANTA(_ring) - 257) \</div><div class='add'>+	)</div><div class='add'>+</div><div class='add'>+#define IRDMA_ATOMIC_RING_MOVE_HEAD(_ring, index, _retcode) \</div><div class='add'>+	{ \</div><div class='add'>+		index = IRDMA_RING_CURRENT_HEAD(_ring); \</div><div class='add'>+		IRDMA_RING_MOVE_HEAD(_ring, _retcode); \</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+enum irdma_qp_wqe_size {</div><div class='add'>+	IRDMA_WQE_SIZE_32  = 32,</div><div class='add'>+	IRDMA_WQE_SIZE_64  = 64,</div><div class='add'>+	IRDMA_WQE_SIZE_96  = 96,</div><div class='add'>+	IRDMA_WQE_SIZE_128 = 128,</div><div class='add'>+	IRDMA_WQE_SIZE_256 = 256,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_ws_node_op {</div><div class='add'>+	IRDMA_ADD_NODE = 0,</div><div class='add'>+	IRDMA_MODIFY_NODE,</div><div class='add'>+	IRDMA_DEL_NODE,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum {	IRDMA_Q_ALIGNMENT_M		 = (128 - 1),</div><div class='add'>+	IRDMA_AEQ_ALIGNMENT_M		 = (256 - 1),</div><div class='add'>+	IRDMA_Q2_ALIGNMENT_M		 = (256 - 1),</div><div class='add'>+	IRDMA_CEQ_ALIGNMENT_M		 = (256 - 1),</div><div class='add'>+	IRDMA_CQ0_ALIGNMENT_M		 = (256 - 1),</div><div class='add'>+	IRDMA_HOST_CTX_ALIGNMENT_M	 = (4 - 1),</div><div class='add'>+	IRDMA_SHADOWAREA_M		 = (128 - 1),</div><div class='add'>+	IRDMA_FPM_QUERY_BUF_ALIGNMENT_M	 = (4 - 1),</div><div class='add'>+	IRDMA_FPM_COMMIT_BUF_ALIGNMENT_M = (4 - 1),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_alignment {</div><div class='add'>+	IRDMA_CQP_ALIGNMENT	    = 0x200,</div><div class='add'>+	IRDMA_AEQ_ALIGNMENT	    = 0x100,</div><div class='add'>+	IRDMA_CEQ_ALIGNMENT	    = 0x100,</div><div class='add'>+	IRDMA_CQ0_ALIGNMENT	    = 0x100,</div><div class='add'>+	IRDMA_SD_BUF_ALIGNMENT      = 0x80,</div><div class='add'>+	IRDMA_FEATURE_BUF_ALIGNMENT = 0x10,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum icrdma_protocol_used {</div><div class='add'>+	ICRDMA_ANY_PROTOCOL	   = 0,</div><div class='add'>+	ICRDMA_IWARP_PROTOCOL_ONLY = 1,</div><div class='add'>+	ICRDMA_ROCE_PROTOCOL_ONLY  = 2,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * set_64bit_val - set 64 bit value to hw wqe</div><div class='add'>+ * @wqe_words: wqe addr to write</div><div class='add'>+ * @byte_index: index in wqe</div><div class='add'>+ * @val: value to write</div><div class='add'>+ **/</div><div class='add'>+static inline void set_64bit_val(__le64 *wqe_words, u32 byte_index, u64 val)</div><div class='add'>+{</div><div class='add'>+	wqe_words[byte_index &gt;&gt; 3] = cpu_to_le64(val);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * set_32bit_val - set 32 bit value to hw wqe</div><div class='add'>+ * @wqe_words: wqe addr to write</div><div class='add'>+ * @byte_index: index in wqe</div><div class='add'>+ * @val: value to write</div><div class='add'>+ **/</div><div class='add'>+static inline void set_32bit_val(__le32 *wqe_words, u32 byte_index, u32 val)</div><div class='add'>+{</div><div class='add'>+	wqe_words[byte_index &gt;&gt; 2] = cpu_to_le32(val);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * get_64bit_val - read 64 bit value from wqe</div><div class='add'>+ * @wqe_words: wqe addr</div><div class='add'>+ * @byte_index: index to read from</div><div class='add'>+ * @val: read value</div><div class='add'>+ **/</div><div class='add'>+static inline void get_64bit_val(__le64 *wqe_words, u32 byte_index, u64 *val)</div><div class='add'>+{</div><div class='add'>+	*val = le64_to_cpu(wqe_words[byte_index &gt;&gt; 3]);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * get_32bit_val - read 32 bit value from wqe</div><div class='add'>+ * @wqe_words: wqe addr</div><div class='add'>+ * @byte_index: index to reaad from</div><div class='add'>+ * @val: return 32 bit value</div><div class='add'>+ **/</div><div class='add'>+static inline void get_32bit_val(__le32 *wqe_words, u32 byte_index, u32 *val)</div><div class='add'>+{</div><div class='add'>+	*val = le32_to_cpu(wqe_words[byte_index &gt;&gt; 2]);</div><div class='add'>+}</div><div class='add'>+#endif /* IRDMA_DEFS_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/hmc.c b/drivers/infiniband/hw/irdma/hmc.c<br/>new file mode 100644<br/>index 000000000000..da18add141da<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/hmc.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/hmc.c</a></div><div class='hunk'>@@ -0,0 +1,709 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#include "osdep.h"</div><div class='add'>+#include "hmc.h"</div><div class='add'>+#include "defs.h"</div><div class='add'>+#include "type.h"</div><div class='add'>+#include "protos.h"</div><div class='add'>+#include "virtchnl.h"</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_find_sd_index_limit - finds segment descriptor index limit</div><div class='add'>+ * @hmc_info: pointer to the HMC configuration information structure</div><div class='add'>+ * @type: type of HMC resources we're searching</div><div class='add'>+ * @idx: starting index for the object</div><div class='add'>+ * @cnt: number of objects we're trying to create</div><div class='add'>+ * @sd_idx: pointer to return index of the segment descriptor in question</div><div class='add'>+ * @sd_limit: pointer to return the maximum number of segment descriptors</div><div class='add'>+ *</div><div class='add'>+ * This function calculates the segment descriptor index and index limit</div><div class='add'>+ * for the resource defined by irdma_hmc_rsrc_type.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+static void irdma_find_sd_index_limit(struct irdma_hmc_info *hmc_info, u32 type,</div><div class='add'>+				      u32 idx, u32 cnt, u32 *sd_idx,</div><div class='add'>+				      u32 *sd_limit)</div><div class='add'>+{</div><div class='add'>+	u64 fpm_addr, fpm_limit;</div><div class='add'>+</div><div class='add'>+	fpm_addr = hmc_info-&gt;hmc_obj[(type)].base +</div><div class='add'>+		   hmc_info-&gt;hmc_obj[type].size * idx;</div><div class='add'>+	fpm_limit = fpm_addr + hmc_info-&gt;hmc_obj[type].size * cnt;</div><div class='add'>+	*sd_idx = (u32)(fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE);</div><div class='add'>+	*sd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_DIRECT_BP_SIZE);</div><div class='add'>+	*sd_limit += 1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_find_pd_index_limit - finds page descriptor index limit</div><div class='add'>+ * @hmc_info: pointer to the HMC configuration information struct</div><div class='add'>+ * @type: HMC resource type we're examining</div><div class='add'>+ * @idx: starting index for the object</div><div class='add'>+ * @cnt: number of objects we're trying to create</div><div class='add'>+ * @pd_idx: pointer to return page descriptor index</div><div class='add'>+ * @pd_limit: pointer to return page descriptor index limit</div><div class='add'>+ *</div><div class='add'>+ * Calculates the page descriptor index and index limit for the resource</div><div class='add'>+ * defined by irdma_hmc_rsrc_type.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+static void irdma_find_pd_index_limit(struct irdma_hmc_info *hmc_info, u32 type,</div><div class='add'>+				      u32 idx, u32 cnt, u32 *pd_idx,</div><div class='add'>+				      u32 *pd_limit)</div><div class='add'>+{</div><div class='add'>+	u64 fpm_adr, fpm_limit;</div><div class='add'>+</div><div class='add'>+	fpm_adr = hmc_info-&gt;hmc_obj[type].base +</div><div class='add'>+		  hmc_info-&gt;hmc_obj[type].size * idx;</div><div class='add'>+	fpm_limit = fpm_adr + (hmc_info)-&gt;hmc_obj[(type)].size * (cnt);</div><div class='add'>+	*pd_idx = (u32)(fpm_adr / IRDMA_HMC_PAGED_BP_SIZE);</div><div class='add'>+	*pd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_PAGED_BP_SIZE);</div><div class='add'>+	*pd_limit += 1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_set_sd_entry - setup entry for sd programming</div><div class='add'>+ * @pa: physical addr</div><div class='add'>+ * @idx: sd index</div><div class='add'>+ * @type: paged or direct sd</div><div class='add'>+ * @entry: sd entry ptr</div><div class='add'>+ */</div><div class='add'>+static void irdma_set_sd_entry(u64 pa, u32 idx, enum irdma_sd_entry_type type,</div><div class='add'>+			       struct irdma_update_sd_entry *entry)</div><div class='add'>+{</div><div class='add'>+	entry-&gt;data = pa |</div><div class='add'>+		      FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) |</div><div class='add'>+		      FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE,</div><div class='add'>+				 type == IRDMA_SD_TYPE_PAGED ? 0 : 1) |</div><div class='add'>+		      FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDVALID, 1);</div><div class='add'>+</div><div class='add'>+	entry-&gt;cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) | BIT(15);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_clr_sd_entry - setup entry for sd clear</div><div class='add'>+ * @idx: sd index</div><div class='add'>+ * @type: paged or direct sd</div><div class='add'>+ * @entry: sd entry ptr</div><div class='add'>+ */</div><div class='add'>+static void irdma_clr_sd_entry(u32 idx, enum irdma_sd_entry_type type,</div><div class='add'>+			       struct irdma_update_sd_entry *entry)</div><div class='add'>+{</div><div class='add'>+	entry-&gt;data = FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) |</div><div class='add'>+		      FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE,</div><div class='add'>+				 type == IRDMA_SD_TYPE_PAGED ? 0 : 1);</div><div class='add'>+</div><div class='add'>+	entry-&gt;cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) | BIT(15);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_invalidate_pf_hmc_pd - Invalidates the pd cache in the hardware for PF</div><div class='add'>+ * @dev: pointer to our device struct</div><div class='add'>+ * @sd_idx: segment descriptor index</div><div class='add'>+ * @pd_idx: page descriptor index</div><div class='add'>+ */</div><div class='add'>+static inline void irdma_invalidate_pf_hmc_pd(struct irdma_sc_dev *dev, u32 sd_idx,</div><div class='add'>+					      u32 pd_idx)</div><div class='add'>+{</div><div class='add'>+	u32 val = FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDIDX, sd_idx) |</div><div class='add'>+		  FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDPARTSEL, 1) |</div><div class='add'>+		  FIELD_PREP(IRDMA_PFHMC_PDINV_PMPDIDX, pd_idx);</div><div class='add'>+</div><div class='add'>+	writel(val, dev-&gt;hw_regs[IRDMA_PFHMC_PDINV]);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_hmc_sd_one - setup 1 sd entry for cqp</div><div class='add'>+ * @dev: pointer to the device structure</div><div class='add'>+ * @hmc_fn_id: hmc's function id</div><div class='add'>+ * @pa: physical addr</div><div class='add'>+ * @sd_idx: sd index</div><div class='add'>+ * @type: paged or direct sd</div><div class='add'>+ * @setsd: flag to set or clear sd</div><div class='add'>+ */</div><div class='add'>+int irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id, u64 pa, u32 sd_idx,</div><div class='add'>+		     enum irdma_sd_entry_type type, bool setsd)</div><div class='add'>+{</div><div class='add'>+	struct irdma_update_sds_info sdinfo;</div><div class='add'>+</div><div class='add'>+	sdinfo.cnt = 1;</div><div class='add'>+	sdinfo.hmc_fn_id = hmc_fn_id;</div><div class='add'>+	if (setsd)</div><div class='add'>+		irdma_set_sd_entry(pa, sd_idx, type, sdinfo.entry);</div><div class='add'>+	else</div><div class='add'>+		irdma_clr_sd_entry(sd_idx, type, sdinfo.entry);</div><div class='add'>+	return dev-&gt;cqp-&gt;process_cqp_sds(dev, &amp;sdinfo);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_hmc_sd_grp - setup group of sd entries for cqp</div><div class='add'>+ * @dev: pointer to the device structure</div><div class='add'>+ * @hmc_info: pointer to the HMC configuration information struct</div><div class='add'>+ * @sd_index: sd index</div><div class='add'>+ * @sd_cnt: number of sd entries</div><div class='add'>+ * @setsd: flag to set or clear sd</div><div class='add'>+ */</div><div class='add'>+static int irdma_hmc_sd_grp(struct irdma_sc_dev *dev,</div><div class='add'>+			    struct irdma_hmc_info *hmc_info, u32 sd_index,</div><div class='add'>+			    u32 sd_cnt, bool setsd)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_sd_entry *sd_entry;</div><div class='add'>+	struct irdma_update_sds_info sdinfo = {};</div><div class='add'>+	u64 pa;</div><div class='add'>+	u32 i;</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+</div><div class='add'>+	sdinfo.hmc_fn_id = hmc_info-&gt;hmc_fn_id;</div><div class='add'>+	for (i = sd_index; i &lt; sd_index + sd_cnt; i++) {</div><div class='add'>+		sd_entry = &amp;hmc_info-&gt;sd_table.sd_entry[i];</div><div class='add'>+		if (!sd_entry || (!sd_entry-&gt;valid &amp;&amp; setsd) ||</div><div class='add'>+		    (sd_entry-&gt;valid &amp;&amp; !setsd))</div><div class='add'>+			continue;</div><div class='add'>+		if (setsd) {</div><div class='add'>+			pa = (sd_entry-&gt;entry_type == IRDMA_SD_TYPE_PAGED) ?</div><div class='add'>+				     sd_entry-&gt;u.pd_table.pd_page_addr.pa :</div><div class='add'>+				     sd_entry-&gt;u.bp.addr.pa;</div><div class='add'>+			irdma_set_sd_entry(pa, i, sd_entry-&gt;entry_type,</div><div class='add'>+					   &amp;sdinfo.entry[sdinfo.cnt]);</div><div class='add'>+		} else {</div><div class='add'>+			irdma_clr_sd_entry(i, sd_entry-&gt;entry_type,</div><div class='add'>+					   &amp;sdinfo.entry[sdinfo.cnt]);</div><div class='add'>+		}</div><div class='add'>+		sdinfo.cnt++;</div><div class='add'>+		if (sdinfo.cnt == IRDMA_MAX_SD_ENTRIES) {</div><div class='add'>+			ret_code = dev-&gt;cqp-&gt;process_cqp_sds(dev, &amp;sdinfo);</div><div class='add'>+			if (ret_code) {</div><div class='add'>+				ibdev_dbg(to_ibdev(dev),</div><div class='add'>+					  "HMC: sd_programming failed err=%d\n",</div><div class='add'>+					  ret_code);</div><div class='add'>+				return ret_code;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			sdinfo.cnt = 0;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	if (sdinfo.cnt)</div><div class='add'>+		ret_code = dev-&gt;cqp-&gt;process_cqp_sds(dev, &amp;sdinfo);</div><div class='add'>+</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_hmc_finish_add_sd_reg - program sd entries for objects</div><div class='add'>+ * @dev: pointer to the device structure</div><div class='add'>+ * @info: create obj info</div><div class='add'>+ */</div><div class='add'>+static int irdma_hmc_finish_add_sd_reg(struct irdma_sc_dev *dev,</div><div class='add'>+				       struct irdma_hmc_create_obj_info *info)</div><div class='add'>+{</div><div class='add'>+	if (info-&gt;start_idx &gt;= info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if ((info-&gt;start_idx + info-&gt;count) &gt;</div><div class='add'>+	    info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!info-&gt;add_sd_cnt)</div><div class='add'>+		return 0;</div><div class='add'>+	return irdma_hmc_sd_grp(dev, info-&gt;hmc_info,</div><div class='add'>+				info-&gt;hmc_info-&gt;sd_indexes[0], info-&gt;add_sd_cnt,</div><div class='add'>+				true);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_create_hmc_obj - allocate backing store for hmc objects</div><div class='add'>+ * @dev: pointer to the device structure</div><div class='add'>+ * @info: pointer to irdma_hmc_create_obj_info struct</div><div class='add'>+ *</div><div class='add'>+ * This will allocate memory for PDs and backing pages and populate</div><div class='add'>+ * the sd and pd entries.</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev,</div><div class='add'>+			    struct irdma_hmc_create_obj_info *info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_sd_entry *sd_entry;</div><div class='add'>+	u32 sd_idx, sd_lmt;</div><div class='add'>+	u32 pd_idx = 0, pd_lmt = 0;</div><div class='add'>+	u32 pd_idx1 = 0, pd_lmt1 = 0;</div><div class='add'>+	u32 i, j;</div><div class='add'>+	bool pd_error = false;</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3 &amp;&amp;</div><div class='add'>+	    dev-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].mem_loc == IRDMA_LOC_MEM)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;start_idx &gt;= info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if ((info-&gt;start_idx + info-&gt;count) &gt;</div><div class='add'>+	    info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "HMC: error type %u, start = %u, req cnt %u, cnt = %u\n",</div><div class='add'>+			  info-&gt;rsrc_type, info-&gt;start_idx, info-&gt;count,</div><div class='add'>+			  info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_find_sd_index_limit(info-&gt;hmc_info, info-&gt;rsrc_type,</div><div class='add'>+				  info-&gt;start_idx, info-&gt;count, &amp;sd_idx,</div><div class='add'>+				  &amp;sd_lmt);</div><div class='add'>+	if (sd_idx &gt;= info-&gt;hmc_info-&gt;sd_table.sd_cnt ||</div><div class='add'>+	    sd_lmt &gt; info-&gt;hmc_info-&gt;sd_table.sd_cnt) {</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_find_pd_index_limit(info-&gt;hmc_info, info-&gt;rsrc_type,</div><div class='add'>+				  info-&gt;start_idx, info-&gt;count, &amp;pd_idx,</div><div class='add'>+				  &amp;pd_lmt);</div><div class='add'>+</div><div class='add'>+	for (j = sd_idx; j &lt; sd_lmt; j++) {</div><div class='add'>+		ret_code = irdma_add_sd_table_entry(dev-&gt;hw, info-&gt;hmc_info, j,</div><div class='add'>+						    info-&gt;entry_type,</div><div class='add'>+						    IRDMA_HMC_DIRECT_BP_SIZE);</div><div class='add'>+		if (ret_code)</div><div class='add'>+			goto exit_sd_error;</div><div class='add'>+</div><div class='add'>+		sd_entry = &amp;info-&gt;hmc_info-&gt;sd_table.sd_entry[j];</div><div class='add'>+		if (sd_entry-&gt;entry_type == IRDMA_SD_TYPE_PAGED &amp;&amp;</div><div class='add'>+		    (dev-&gt;hmc_info == info-&gt;hmc_info &amp;&amp;</div><div class='add'>+		     info-&gt;rsrc_type != IRDMA_HMC_IW_PBLE)) {</div><div class='add'>+			pd_idx1 = max(pd_idx, (j * IRDMA_HMC_MAX_BP_COUNT));</div><div class='add'>+			pd_lmt1 = min(pd_lmt, (j + 1) * IRDMA_HMC_MAX_BP_COUNT);</div><div class='add'>+			for (i = pd_idx1; i &lt; pd_lmt1; i++) {</div><div class='add'>+				/* update the pd table entry */</div><div class='add'>+				ret_code = irdma_add_pd_table_entry(dev,</div><div class='add'>+								    info-&gt;hmc_info,</div><div class='add'>+								    i, NULL);</div><div class='add'>+				if (ret_code) {</div><div class='add'>+					pd_error = true;</div><div class='add'>+					break;</div><div class='add'>+				}</div><div class='add'>+			}</div><div class='add'>+			if (pd_error) {</div><div class='add'>+				while (i &amp;&amp; (i &gt; pd_idx1)) {</div><div class='add'>+					irdma_remove_pd_bp(dev, info-&gt;hmc_info,</div><div class='add'>+							   i - 1);</div><div class='add'>+					i--;</div><div class='add'>+				}</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+		if (sd_entry-&gt;valid)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		info-&gt;hmc_info-&gt;sd_indexes[info-&gt;add_sd_cnt] = (u16)j;</div><div class='add'>+		info-&gt;add_sd_cnt++;</div><div class='add'>+		sd_entry-&gt;valid = true;</div><div class='add'>+	}</div><div class='add'>+	return irdma_hmc_finish_add_sd_reg(dev, info);</div><div class='add'>+</div><div class='add'>+exit_sd_error:</div><div class='add'>+	while (j &amp;&amp; (j &gt; sd_idx)) {</div><div class='add'>+		sd_entry = &amp;info-&gt;hmc_info-&gt;sd_table.sd_entry[j - 1];</div><div class='add'>+		switch (sd_entry-&gt;entry_type) {</div><div class='add'>+		case IRDMA_SD_TYPE_PAGED:</div><div class='add'>+			pd_idx1 = max(pd_idx, (j - 1) * IRDMA_HMC_MAX_BP_COUNT);</div><div class='add'>+			pd_lmt1 = min(pd_lmt, (j * IRDMA_HMC_MAX_BP_COUNT));</div><div class='add'>+			for (i = pd_idx1; i &lt; pd_lmt1; i++)</div><div class='add'>+				irdma_prep_remove_pd_page(info-&gt;hmc_info, i);</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_SD_TYPE_DIRECT:</div><div class='add'>+			irdma_prep_remove_pd_page(info-&gt;hmc_info, (j - 1));</div><div class='add'>+			break;</div><div class='add'>+		default:</div><div class='add'>+			ret_code = -EINVAL;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		j--;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_finish_del_sd_reg - delete sd entries for objects</div><div class='add'>+ * @dev: pointer to the device structure</div><div class='add'>+ * @info: dele obj info</div><div class='add'>+ * @reset: true if called before reset</div><div class='add'>+ */</div><div class='add'>+static int irdma_finish_del_sd_reg(struct irdma_sc_dev *dev,</div><div class='add'>+				   struct irdma_hmc_del_obj_info *info,</div><div class='add'>+				   bool reset)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_sd_entry *sd_entry;</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+	u32 i, sd_idx;</div><div class='add'>+	struct irdma_dma_mem *mem;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;privileged &amp;&amp; !reset)</div><div class='add'>+		ret_code = irdma_hmc_sd_grp(dev, info-&gt;hmc_info,</div><div class='add'>+					    info-&gt;hmc_info-&gt;sd_indexes[0],</div><div class='add'>+					    info-&gt;del_sd_cnt, false);</div><div class='add'>+</div><div class='add'>+	if (ret_code)</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "HMC: error cqp sd sd_grp\n");</div><div class='add'>+	for (i = 0; i &lt; info-&gt;del_sd_cnt; i++) {</div><div class='add'>+		sd_idx = info-&gt;hmc_info-&gt;sd_indexes[i];</div><div class='add'>+		sd_entry = &amp;info-&gt;hmc_info-&gt;sd_table.sd_entry[sd_idx];</div><div class='add'>+		mem = (sd_entry-&gt;entry_type == IRDMA_SD_TYPE_PAGED) ?</div><div class='add'>+			      &amp;sd_entry-&gt;u.pd_table.pd_page_addr :</div><div class='add'>+			      &amp;sd_entry-&gt;u.bp.addr;</div><div class='add'>+</div><div class='add'>+		if (!mem || !mem-&gt;va) {</div><div class='add'>+			ibdev_dbg(to_ibdev(dev), "HMC: error cqp sd mem\n");</div><div class='add'>+		} else {</div><div class='add'>+			dma_free_coherent(dev-&gt;hw-&gt;device, mem-&gt;size, mem-&gt;va,</div><div class='add'>+					  mem-&gt;pa);</div><div class='add'>+			mem-&gt;va = NULL;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_del_hmc_obj - remove pe hmc objects</div><div class='add'>+ * @dev: pointer to the device structure</div><div class='add'>+ * @info: pointer to irdma_hmc_del_obj_info struct</div><div class='add'>+ * @reset: true if called before reset</div><div class='add'>+ *</div><div class='add'>+ * This will de-populate the SDs and PDs.  It frees</div><div class='add'>+ * the memory for PDS and backing storage.  After this function is returned,</div><div class='add'>+ * caller should deallocate memory allocated previously for</div><div class='add'>+ * book-keeping information about PDs and backing storage.</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,</div><div class='add'>+			 struct irdma_hmc_del_obj_info *info, bool reset)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_pd_table *pd_table;</div><div class='add'>+	u32 sd_idx, sd_lmt;</div><div class='add'>+	u32 pd_idx, pd_lmt, rel_pd_idx;</div><div class='add'>+	u32 i, j;</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].mem_loc == IRDMA_LOC_MEM)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;start_idx &gt;= info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "HMC: error start_idx[%04d]  &gt;= [type %04d].cnt[%04d]\n",</div><div class='add'>+			  info-&gt;start_idx, info-&gt;rsrc_type,</div><div class='add'>+			  info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if ((info-&gt;start_idx + info-&gt;count) &gt;</div><div class='add'>+	    info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "HMC: error start_idx[%04d] + count %04d  &gt;= [type %04d].cnt[%04d]\n",</div><div class='add'>+			  info-&gt;start_idx, info-&gt;count, info-&gt;rsrc_type,</div><div class='add'>+			  info-&gt;hmc_info-&gt;hmc_obj[info-&gt;rsrc_type].cnt);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_find_pd_index_limit(info-&gt;hmc_info, info-&gt;rsrc_type,</div><div class='add'>+				  info-&gt;start_idx, info-&gt;count, &amp;pd_idx,</div><div class='add'>+				  &amp;pd_lmt);</div><div class='add'>+</div><div class='add'>+	for (j = pd_idx; j &lt; pd_lmt; j++) {</div><div class='add'>+		sd_idx = j / IRDMA_HMC_PD_CNT_IN_SD;</div><div class='add'>+</div><div class='add'>+		if (!info-&gt;hmc_info-&gt;sd_table.sd_entry[sd_idx].valid)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		if (info-&gt;hmc_info-&gt;sd_table.sd_entry[sd_idx].entry_type !=</div><div class='add'>+		    IRDMA_SD_TYPE_PAGED)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		rel_pd_idx = j % IRDMA_HMC_PD_CNT_IN_SD;</div><div class='add'>+		pd_table = &amp;info-&gt;hmc_info-&gt;sd_table.sd_entry[sd_idx].u.pd_table;</div><div class='add'>+		if (pd_table-&gt;pd_entry &amp;&amp;</div><div class='add'>+		    pd_table-&gt;pd_entry[rel_pd_idx].valid) {</div><div class='add'>+			ret_code = irdma_remove_pd_bp(dev, info-&gt;hmc_info, j);</div><div class='add'>+			if (ret_code) {</div><div class='add'>+				ibdev_dbg(to_ibdev(dev),</div><div class='add'>+					  "HMC: remove_pd_bp error\n");</div><div class='add'>+				return ret_code;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_find_sd_index_limit(info-&gt;hmc_info, info-&gt;rsrc_type,</div><div class='add'>+				  info-&gt;start_idx, info-&gt;count, &amp;sd_idx,</div><div class='add'>+				  &amp;sd_lmt);</div><div class='add'>+	if (sd_idx &gt;= info-&gt;hmc_info-&gt;sd_table.sd_cnt ||</div><div class='add'>+	    sd_lmt &gt; info-&gt;hmc_info-&gt;sd_table.sd_cnt) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "HMC: invalid sd_idx\n");</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = sd_idx; i &lt; sd_lmt; i++) {</div><div class='add'>+		pd_table = &amp;info-&gt;hmc_info-&gt;sd_table.sd_entry[i].u.pd_table;</div><div class='add'>+		if (!info-&gt;hmc_info-&gt;sd_table.sd_entry[i].valid)</div><div class='add'>+			continue;</div><div class='add'>+		switch (info-&gt;hmc_info-&gt;sd_table.sd_entry[i].entry_type) {</div><div class='add'>+		case IRDMA_SD_TYPE_DIRECT:</div><div class='add'>+			ret_code = irdma_prep_remove_sd_bp(info-&gt;hmc_info, i);</div><div class='add'>+			if (!ret_code) {</div><div class='add'>+				info-&gt;hmc_info-&gt;sd_indexes[info-&gt;del_sd_cnt] =</div><div class='add'>+					(u16)i;</div><div class='add'>+				info-&gt;del_sd_cnt++;</div><div class='add'>+			}</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_SD_TYPE_PAGED:</div><div class='add'>+			ret_code = irdma_prep_remove_pd_page(info-&gt;hmc_info, i);</div><div class='add'>+			if (ret_code)</div><div class='add'>+				break;</div><div class='add'>+			if (dev-&gt;hmc_info != info-&gt;hmc_info &amp;&amp;</div><div class='add'>+			    info-&gt;rsrc_type == IRDMA_HMC_IW_PBLE &amp;&amp;</div><div class='add'>+			    pd_table-&gt;pd_entry) {</div><div class='add'>+				kfree(pd_table-&gt;pd_entry_virt_mem.va);</div><div class='add'>+				pd_table-&gt;pd_entry = NULL;</div><div class='add'>+			}</div><div class='add'>+			info-&gt;hmc_info-&gt;sd_indexes[info-&gt;del_sd_cnt] = (u16)i;</div><div class='add'>+			info-&gt;del_sd_cnt++;</div><div class='add'>+			break;</div><div class='add'>+		default:</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	return irdma_finish_del_sd_reg(dev, info, reset);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_add_sd_table_entry - Adds a segment descriptor to the table</div><div class='add'>+ * @hw: pointer to our hw struct</div><div class='add'>+ * @hmc_info: pointer to the HMC configuration information struct</div><div class='add'>+ * @sd_index: segment descriptor index to manipulate</div><div class='add'>+ * @type: what type of segment descriptor we're manipulating</div><div class='add'>+ * @direct_mode_sz: size to alloc in direct mode</div><div class='add'>+ */</div><div class='add'>+int irdma_add_sd_table_entry(struct irdma_hw *hw,</div><div class='add'>+			     struct irdma_hmc_info *hmc_info, u32 sd_index,</div><div class='add'>+			     enum irdma_sd_entry_type type, u64 direct_mode_sz)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_sd_entry *sd_entry;</div><div class='add'>+	struct irdma_dma_mem dma_mem;</div><div class='add'>+	u64 alloc_len;</div><div class='add'>+</div><div class='add'>+	sd_entry = &amp;hmc_info-&gt;sd_table.sd_entry[sd_index];</div><div class='add'>+	if (!sd_entry-&gt;valid) {</div><div class='add'>+		if (type == IRDMA_SD_TYPE_PAGED)</div><div class='add'>+			alloc_len = IRDMA_HMC_PAGED_BP_SIZE;</div><div class='add'>+		else</div><div class='add'>+			alloc_len = direct_mode_sz;</div><div class='add'>+</div><div class='add'>+		/* allocate a 4K pd page or 2M backing page */</div><div class='add'>+		dma_mem.size = ALIGN(alloc_len, IRDMA_HMC_PD_BP_BUF_ALIGNMENT);</div><div class='add'>+		dma_mem.va = dma_alloc_coherent(hw-&gt;device, dma_mem.size,</div><div class='add'>+						&amp;dma_mem.pa, GFP_KERNEL);</div><div class='add'>+		if (!dma_mem.va)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		if (type == IRDMA_SD_TYPE_PAGED) {</div><div class='add'>+			struct irdma_virt_mem *vmem =</div><div class='add'>+				&amp;sd_entry-&gt;u.pd_table.pd_entry_virt_mem;</div><div class='add'>+</div><div class='add'>+			vmem-&gt;size = sizeof(struct irdma_hmc_pd_entry) * 512;</div><div class='add'>+			vmem-&gt;va = kzalloc(vmem-&gt;size, GFP_KERNEL);</div><div class='add'>+			if (!vmem-&gt;va) {</div><div class='add'>+				dma_free_coherent(hw-&gt;device, dma_mem.size,</div><div class='add'>+						  dma_mem.va, dma_mem.pa);</div><div class='add'>+				dma_mem.va = NULL;</div><div class='add'>+				return -ENOMEM;</div><div class='add'>+			}</div><div class='add'>+			sd_entry-&gt;u.pd_table.pd_entry = vmem-&gt;va;</div><div class='add'>+</div><div class='add'>+			memcpy(&amp;sd_entry-&gt;u.pd_table.pd_page_addr, &amp;dma_mem,</div><div class='add'>+			       sizeof(sd_entry-&gt;u.pd_table.pd_page_addr));</div><div class='add'>+		} else {</div><div class='add'>+			memcpy(&amp;sd_entry-&gt;u.bp.addr, &amp;dma_mem,</div><div class='add'>+			       sizeof(sd_entry-&gt;u.bp.addr));</div><div class='add'>+</div><div class='add'>+			sd_entry-&gt;u.bp.sd_pd_index = sd_index;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		hmc_info-&gt;sd_table.sd_entry[sd_index].entry_type = type;</div><div class='add'>+		hmc_info-&gt;sd_table.use_cnt++;</div><div class='add'>+	}</div><div class='add'>+	if (sd_entry-&gt;entry_type == IRDMA_SD_TYPE_DIRECT)</div><div class='add'>+		sd_entry-&gt;u.bp.use_cnt++;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_add_pd_table_entry - Adds page descriptor to the specified table</div><div class='add'>+ * @dev: pointer to our device structure</div><div class='add'>+ * @hmc_info: pointer to the HMC configuration information structure</div><div class='add'>+ * @pd_index: which page descriptor index to manipulate</div><div class='add'>+ * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one.</div><div class='add'>+ *</div><div class='add'>+ * This function:</div><div class='add'>+ *	1. Initializes the pd entry</div><div class='add'>+ *	2. Adds pd_entry in the pd_table</div><div class='add'>+ *	3. Mark the entry valid in irdma_hmc_pd_entry structure</div><div class='add'>+ *	4. Initializes the pd_entry's ref count to 1</div><div class='add'>+ * assumptions:</div><div class='add'>+ *	1. The memory for pd should be pinned down, physically contiguous and</div><div class='add'>+ *	   aligned on 4K boundary and zeroed memory.</div><div class='add'>+ *	2. It should be 4K in size.</div><div class='add'>+ */</div><div class='add'>+int irdma_add_pd_table_entry(struct irdma_sc_dev *dev,</div><div class='add'>+			     struct irdma_hmc_info *hmc_info, u32 pd_index,</div><div class='add'>+			     struct irdma_dma_mem *rsrc_pg)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_pd_table *pd_table;</div><div class='add'>+	struct irdma_hmc_pd_entry *pd_entry;</div><div class='add'>+	struct irdma_dma_mem mem;</div><div class='add'>+	struct irdma_dma_mem *page = &amp;mem;</div><div class='add'>+	u32 sd_idx, rel_pd_idx;</div><div class='add'>+	u64 *pd_addr;</div><div class='add'>+	u64 page_desc;</div><div class='add'>+</div><div class='add'>+	if (pd_index / IRDMA_HMC_PD_CNT_IN_SD &gt;= hmc_info-&gt;sd_table.sd_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	sd_idx = (pd_index / IRDMA_HMC_PD_CNT_IN_SD);</div><div class='add'>+	if (hmc_info-&gt;sd_table.sd_entry[sd_idx].entry_type !=</div><div class='add'>+	    IRDMA_SD_TYPE_PAGED)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	rel_pd_idx = (pd_index % IRDMA_HMC_PD_CNT_IN_SD);</div><div class='add'>+	pd_table = &amp;hmc_info-&gt;sd_table.sd_entry[sd_idx].u.pd_table;</div><div class='add'>+	pd_entry = &amp;pd_table-&gt;pd_entry[rel_pd_idx];</div><div class='add'>+	if (!pd_entry-&gt;valid) {</div><div class='add'>+		if (rsrc_pg) {</div><div class='add'>+			pd_entry-&gt;rsrc_pg = true;</div><div class='add'>+			page = rsrc_pg;</div><div class='add'>+		} else {</div><div class='add'>+			page-&gt;size = ALIGN(IRDMA_HMC_PAGED_BP_SIZE,</div><div class='add'>+					   IRDMA_HMC_PD_BP_BUF_ALIGNMENT);</div><div class='add'>+			page-&gt;va = dma_alloc_coherent(dev-&gt;hw-&gt;device,</div><div class='add'>+						      page-&gt;size, &amp;page-&gt;pa,</div><div class='add'>+						      GFP_KERNEL);</div><div class='add'>+			if (!page-&gt;va)</div><div class='add'>+				return -ENOMEM;</div><div class='add'>+</div><div class='add'>+			pd_entry-&gt;rsrc_pg = false;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		memcpy(&amp;pd_entry-&gt;bp.addr, page, sizeof(pd_entry-&gt;bp.addr));</div><div class='add'>+		pd_entry-&gt;bp.sd_pd_index = pd_index;</div><div class='add'>+		pd_entry-&gt;bp.entry_type = IRDMA_SD_TYPE_PAGED;</div><div class='add'>+		page_desc = page-&gt;pa | 0x1;</div><div class='add'>+		pd_addr = pd_table-&gt;pd_page_addr.va;</div><div class='add'>+		pd_addr += rel_pd_idx;</div><div class='add'>+		memcpy(pd_addr, &amp;page_desc, sizeof(*pd_addr));</div><div class='add'>+		pd_entry-&gt;sd_index = sd_idx;</div><div class='add'>+		pd_entry-&gt;valid = true;</div><div class='add'>+		pd_table-&gt;use_cnt++;</div><div class='add'>+</div><div class='add'>+		if (hmc_info-&gt;hmc_fn_id &lt; dev-&gt;hw_attrs.first_hw_vf_fpm_id &amp;&amp;</div><div class='add'>+		    dev-&gt;privileged)</div><div class='add'>+			irdma_invalidate_pf_hmc_pd(dev, sd_idx, rel_pd_idx);</div><div class='add'>+	}</div><div class='add'>+	pd_entry-&gt;bp.use_cnt++;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_remove_pd_bp - remove a backing page from a page descriptor</div><div class='add'>+ * @dev: pointer to our HW structure</div><div class='add'>+ * @hmc_info: pointer to the HMC configuration information structure</div><div class='add'>+ * @idx: the page index</div><div class='add'>+ *</div><div class='add'>+ * This function:</div><div class='add'>+ *	1. Marks the entry in pd table (for paged address mode) or in sd table</div><div class='add'>+ *	   (for direct address mode) invalid.</div><div class='add'>+ *	2. Write to register PMPDINV to invalidate the backing page in FV cache</div><div class='add'>+ *	3. Decrement the ref count for the pd _entry</div><div class='add'>+ * assumptions:</div><div class='add'>+ *	1. Caller can deallocate the memory used by backing storage after this</div><div class='add'>+ *	   function returns.</div><div class='add'>+ */</div><div class='add'>+int irdma_remove_pd_bp(struct irdma_sc_dev *dev,</div><div class='add'>+		       struct irdma_hmc_info *hmc_info, u32 idx)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_pd_entry *pd_entry;</div><div class='add'>+	struct irdma_hmc_pd_table *pd_table;</div><div class='add'>+	struct irdma_hmc_sd_entry *sd_entry;</div><div class='add'>+	u32 sd_idx, rel_pd_idx;</div><div class='add'>+	struct irdma_dma_mem *mem;</div><div class='add'>+	u64 *pd_addr;</div><div class='add'>+</div><div class='add'>+	sd_idx = idx / IRDMA_HMC_PD_CNT_IN_SD;</div><div class='add'>+	rel_pd_idx = idx % IRDMA_HMC_PD_CNT_IN_SD;</div><div class='add'>+	if (sd_idx &gt;= hmc_info-&gt;sd_table.sd_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	sd_entry = &amp;hmc_info-&gt;sd_table.sd_entry[sd_idx];</div><div class='add'>+	if (sd_entry-&gt;entry_type != IRDMA_SD_TYPE_PAGED)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	pd_table = &amp;hmc_info-&gt;sd_table.sd_entry[sd_idx].u.pd_table;</div><div class='add'>+	pd_entry = &amp;pd_table-&gt;pd_entry[rel_pd_idx];</div><div class='add'>+	if (--pd_entry-&gt;bp.use_cnt)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	pd_entry-&gt;valid = false;</div><div class='add'>+	pd_table-&gt;use_cnt--;</div><div class='add'>+	pd_addr = pd_table-&gt;pd_page_addr.va;</div><div class='add'>+	pd_addr += rel_pd_idx;</div><div class='add'>+	memset(pd_addr, 0, sizeof(u64));</div><div class='add'>+	if (dev-&gt;privileged &amp;&amp; dev-&gt;hmc_fn_id == hmc_info-&gt;hmc_fn_id)</div><div class='add'>+		irdma_invalidate_pf_hmc_pd(dev, sd_idx, idx);</div><div class='add'>+</div><div class='add'>+	if (!pd_entry-&gt;rsrc_pg) {</div><div class='add'>+		mem = &amp;pd_entry-&gt;bp.addr;</div><div class='add'>+		if (!mem || !mem-&gt;va)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		dma_free_coherent(dev-&gt;hw-&gt;device, mem-&gt;size, mem-&gt;va,</div><div class='add'>+				  mem-&gt;pa);</div><div class='add'>+		mem-&gt;va = NULL;</div><div class='add'>+	}</div><div class='add'>+	if (!pd_table-&gt;use_cnt)</div><div class='add'>+		kfree(pd_table-&gt;pd_entry_virt_mem.va);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry</div><div class='add'>+ * @hmc_info: pointer to the HMC configuration information structure</div><div class='add'>+ * @idx: the page index</div><div class='add'>+ */</div><div class='add'>+int irdma_prep_remove_sd_bp(struct irdma_hmc_info *hmc_info, u32 idx)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_sd_entry *sd_entry;</div><div class='add'>+</div><div class='add'>+	sd_entry = &amp;hmc_info-&gt;sd_table.sd_entry[idx];</div><div class='add'>+	if (--sd_entry-&gt;u.bp.use_cnt)</div><div class='add'>+		return -EBUSY;</div><div class='add'>+</div><div class='add'>+	hmc_info-&gt;sd_table.use_cnt--;</div><div class='add'>+	sd_entry-&gt;valid = false;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_prep_remove_pd_page - Prepares to remove a PD page from sd entry.</div><div class='add'>+ * @hmc_info: pointer to the HMC configuration information structure</div><div class='add'>+ * @idx: segment descriptor index to find the relevant page descriptor</div><div class='add'>+ */</div><div class='add'>+int irdma_prep_remove_pd_page(struct irdma_hmc_info *hmc_info, u32 idx)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_sd_entry *sd_entry;</div><div class='add'>+</div><div class='add'>+	sd_entry = &amp;hmc_info-&gt;sd_table.sd_entry[idx];</div><div class='add'>+</div><div class='add'>+	if (sd_entry-&gt;u.pd_table.use_cnt)</div><div class='add'>+		return -EBUSY;</div><div class='add'>+</div><div class='add'>+	sd_entry-&gt;valid = false;</div><div class='add'>+	hmc_info-&gt;sd_table.use_cnt--;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/hmc.h b/drivers/infiniband/hw/irdma/hmc.h<br/>new file mode 100644<br/>index 000000000000..257a5d22aa96<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/hmc.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/hmc.h</a></div><div class='hunk'>@@ -0,0 +1,186 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2015 - 2020 Intel Corporation */</div><div class='add'>+#ifndef IRDMA_HMC_H</div><div class='add'>+#define IRDMA_HMC_H</div><div class='add'>+</div><div class='add'>+#include "defs.h"</div><div class='add'>+</div><div class='add'>+#define IRDMA_HMC_MAX_BP_COUNT			512</div><div class='add'>+#define IRDMA_MAX_SD_ENTRIES			11</div><div class='add'>+#define IRDMA_HW_DBG_HMC_INVALID_BP_MARK	0xca</div><div class='add'>+#define IRDMA_HMC_INFO_SIGNATURE		0x484d5347</div><div class='add'>+#define IRDMA_HMC_PD_CNT_IN_SD			512</div><div class='add'>+#define IRDMA_HMC_DIRECT_BP_SIZE		0x200000</div><div class='add'>+#define IRDMA_HMC_MAX_SD_COUNT			8192</div><div class='add'>+#define IRDMA_HMC_PAGED_BP_SIZE			4096</div><div class='add'>+#define IRDMA_HMC_PD_BP_BUF_ALIGNMENT		4096</div><div class='add'>+#define IRDMA_FIRST_VF_FPM_ID			8</div><div class='add'>+#define FPM_MULTIPLIER				1024</div><div class='add'>+#define IRDMA_OBJ_LOC_MEM_BIT			0x4</div><div class='add'>+#define IRDMA_XF_MULTIPLIER			16</div><div class='add'>+#define IRDMA_RRF_MULTIPLIER			8</div><div class='add'>+#define IRDMA_MIN_PBLE_PAGES			3</div><div class='add'>+#define IRDMA_HMC_PAGE_SIZE			2097152</div><div class='add'>+#define IRDMA_MIN_MR_PER_QP			4</div><div class='add'>+#define IRDMA_MIN_QP_CNT			64</div><div class='add'>+#define IRDMA_FSIAV_CNT_MAX			1048576</div><div class='add'>+#define IRDMA_MIN_IRD				8</div><div class='add'>+#define IRDMA_HMC_MIN_RRF			16</div><div class='add'>+</div><div class='add'>+enum irdma_hmc_rsrc_type {</div><div class='add'>+	IRDMA_HMC_IW_QP		 = 0,</div><div class='add'>+	IRDMA_HMC_IW_CQ		 = 1,</div><div class='add'>+	IRDMA_HMC_IW_SRQ	 = 2,</div><div class='add'>+	IRDMA_HMC_IW_HTE	 = 3,</div><div class='add'>+	IRDMA_HMC_IW_ARP	 = 4,</div><div class='add'>+	IRDMA_HMC_IW_APBVT_ENTRY = 5,</div><div class='add'>+	IRDMA_HMC_IW_MR		 = 6,</div><div class='add'>+	IRDMA_HMC_IW_XF		 = 7,</div><div class='add'>+	IRDMA_HMC_IW_XFFL	 = 8,</div><div class='add'>+	IRDMA_HMC_IW_Q1		 = 9,</div><div class='add'>+	IRDMA_HMC_IW_Q1FL	 = 10,</div><div class='add'>+	IRDMA_HMC_IW_TIMER       = 11,</div><div class='add'>+	IRDMA_HMC_IW_FSIMC       = 12,</div><div class='add'>+	IRDMA_HMC_IW_FSIAV       = 13,</div><div class='add'>+	IRDMA_HMC_IW_PBLE	 = 14,</div><div class='add'>+	IRDMA_HMC_IW_RRF	 = 15,</div><div class='add'>+	IRDMA_HMC_IW_RRFFL       = 16,</div><div class='add'>+	IRDMA_HMC_IW_HDR	 = 17,</div><div class='add'>+	IRDMA_HMC_IW_MD		 = 18,</div><div class='add'>+	IRDMA_HMC_IW_OOISC       = 19,</div><div class='add'>+	IRDMA_HMC_IW_OOISCFFL    = 20,</div><div class='add'>+	IRDMA_HMC_IW_MAX, /* Must be last entry */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_sd_entry_type {</div><div class='add'>+	IRDMA_SD_TYPE_INVALID = 0,</div><div class='add'>+	IRDMA_SD_TYPE_PAGED   = 1,</div><div class='add'>+	IRDMA_SD_TYPE_DIRECT  = 2,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_hmc_obj_mem {</div><div class='add'>+	IRDMA_HOST_MEM = 0,</div><div class='add'>+	IRDMA_LOC_MEM  = 1,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_hmc_obj_info {</div><div class='add'>+	u64 base;</div><div class='add'>+	u32 max_cnt;</div><div class='add'>+	u32 cnt;</div><div class='add'>+	u64 size;</div><div class='add'>+	enum irdma_hmc_obj_mem mem_loc;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_hmc_bp {</div><div class='add'>+	enum irdma_sd_entry_type entry_type;</div><div class='add'>+	struct irdma_dma_mem addr;</div><div class='add'>+	u32 sd_pd_index;</div><div class='add'>+	u32 use_cnt;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_hmc_pd_entry {</div><div class='add'>+	struct irdma_hmc_bp bp;</div><div class='add'>+	u32 sd_index;</div><div class='add'>+	bool rsrc_pg:1;</div><div class='add'>+	bool valid:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_hmc_pd_table {</div><div class='add'>+	struct irdma_dma_mem pd_page_addr;</div><div class='add'>+	struct irdma_hmc_pd_entry *pd_entry;</div><div class='add'>+	struct irdma_virt_mem pd_entry_virt_mem;</div><div class='add'>+	u32 use_cnt;</div><div class='add'>+	u32 sd_index;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_hmc_sd_entry {</div><div class='add'>+	enum irdma_sd_entry_type entry_type;</div><div class='add'>+	bool valid;</div><div class='add'>+	union {</div><div class='add'>+		struct irdma_hmc_pd_table pd_table;</div><div class='add'>+		struct irdma_hmc_bp bp;</div><div class='add'>+	} u;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_hmc_sd_table {</div><div class='add'>+	struct irdma_virt_mem addr;</div><div class='add'>+	u32 sd_cnt;</div><div class='add'>+	u32 use_cnt;</div><div class='add'>+	struct irdma_hmc_sd_entry *sd_entry;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_hmc_info {</div><div class='add'>+	u32 signature;</div><div class='add'>+	u8 hmc_fn_id;</div><div class='add'>+	u16 first_sd_index;</div><div class='add'>+	struct irdma_hmc_obj_info *hmc_obj;</div><div class='add'>+	struct irdma_virt_mem hmc_obj_virt_mem;</div><div class='add'>+	struct irdma_hmc_sd_table sd_table;</div><div class='add'>+	u16 sd_indexes[IRDMA_HMC_MAX_SD_COUNT];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_update_sd_entry {</div><div class='add'>+	u64 cmd;</div><div class='add'>+	u64 data;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_update_sds_info {</div><div class='add'>+	u32 cnt;</div><div class='add'>+	u8 hmc_fn_id;</div><div class='add'>+	struct irdma_update_sd_entry entry[IRDMA_MAX_SD_ENTRIES];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_ccq_cqe_info;</div><div class='add'>+struct irdma_hmc_fcn_info {</div><div class='add'>+	u32 vf_id;</div><div class='add'>+	u8 protocol_used;</div><div class='add'>+	u8 free_fcn;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_hmc_create_obj_info {</div><div class='add'>+	struct irdma_hmc_info *hmc_info;</div><div class='add'>+	struct irdma_virt_mem add_sd_virt_mem;</div><div class='add'>+	u32 rsrc_type;</div><div class='add'>+	u32 start_idx;</div><div class='add'>+	u32 count;</div><div class='add'>+	u32 add_sd_cnt;</div><div class='add'>+	enum irdma_sd_entry_type entry_type;</div><div class='add'>+	bool privileged;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_hmc_del_obj_info {</div><div class='add'>+	struct irdma_hmc_info *hmc_info;</div><div class='add'>+	struct irdma_virt_mem del_sd_virt_mem;</div><div class='add'>+	u32 rsrc_type;</div><div class='add'>+	u32 start_idx;</div><div class='add'>+	u32 count;</div><div class='add'>+	u32 del_sd_cnt;</div><div class='add'>+	bool privileged;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+int irdma_copy_dma_mem(struct irdma_hw *hw, void *dest_buf,</div><div class='add'>+		       struct irdma_dma_mem *src_mem, u64 src_offset, u64 size);</div><div class='add'>+int irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev,</div><div class='add'>+			    struct irdma_hmc_create_obj_info *info);</div><div class='add'>+int irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,</div><div class='add'>+			 struct irdma_hmc_del_obj_info *info, bool reset);</div><div class='add'>+int irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id, u64 pa, u32 sd_idx,</div><div class='add'>+		     enum irdma_sd_entry_type type,</div><div class='add'>+		     bool setsd);</div><div class='add'>+int irdma_update_sds_noccq(struct irdma_sc_dev *dev,</div><div class='add'>+			   struct irdma_update_sds_info *info);</div><div class='add'>+struct irdma_vfdev *irdma_vfdev_from_fpm(struct irdma_sc_dev *dev,</div><div class='add'>+					 u8 hmc_fn_id);</div><div class='add'>+struct irdma_hmc_info *irdma_vf_hmcinfo_from_fpm(struct irdma_sc_dev *dev,</div><div class='add'>+						 u8 hmc_fn_id);</div><div class='add'>+int irdma_add_sd_table_entry(struct irdma_hw *hw,</div><div class='add'>+			     struct irdma_hmc_info *hmc_info, u32 sd_index,</div><div class='add'>+			     enum irdma_sd_entry_type type, u64 direct_mode_sz);</div><div class='add'>+int irdma_add_pd_table_entry(struct irdma_sc_dev *dev,</div><div class='add'>+			     struct irdma_hmc_info *hmc_info, u32 pd_index,</div><div class='add'>+			     struct irdma_dma_mem *rsrc_pg);</div><div class='add'>+int irdma_remove_pd_bp(struct irdma_sc_dev *dev,</div><div class='add'>+		       struct irdma_hmc_info *hmc_info, u32 idx);</div><div class='add'>+int irdma_prep_remove_sd_bp(struct irdma_hmc_info *hmc_info, u32 idx);</div><div class='add'>+int irdma_prep_remove_pd_page(struct irdma_hmc_info *hmc_info, u32 idx);</div><div class='add'>+#endif /* IRDMA_HMC_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c<br/>new file mode 100644<br/>index 000000000000..d1fc5726b979<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/hw.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/hw.c</a></div><div class='hunk'>@@ -0,0 +1,2823 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#include "main.h"</div><div class='add'>+</div><div class='add'>+static struct irdma_rsrc_limits rsrc_limits_table[] = {</div><div class='add'>+	[0] = {</div><div class='add'>+		.qplimit = SZ_128,</div><div class='add'>+	},</div><div class='add'>+	[1] = {</div><div class='add'>+		.qplimit = SZ_1K,</div><div class='add'>+	},</div><div class='add'>+	[2] = {</div><div class='add'>+		.qplimit = SZ_2K,</div><div class='add'>+	},</div><div class='add'>+	[3] = {</div><div class='add'>+		.qplimit = SZ_4K,</div><div class='add'>+	},</div><div class='add'>+	[4] = {</div><div class='add'>+		.qplimit = SZ_16K,</div><div class='add'>+	},</div><div class='add'>+	[5] = {</div><div class='add'>+		.qplimit = SZ_64K,</div><div class='add'>+	},</div><div class='add'>+	[6] = {</div><div class='add'>+		.qplimit = SZ_128K,</div><div class='add'>+	},</div><div class='add'>+	[7] = {</div><div class='add'>+		.qplimit = SZ_256K,</div><div class='add'>+	},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* types of hmc objects */</div><div class='add'>+static enum irdma_hmc_rsrc_type iw_hmc_obj_types[] = {</div><div class='add'>+	IRDMA_HMC_IW_QP,</div><div class='add'>+	IRDMA_HMC_IW_CQ,</div><div class='add'>+	IRDMA_HMC_IW_SRQ,</div><div class='add'>+	IRDMA_HMC_IW_HTE,</div><div class='add'>+	IRDMA_HMC_IW_ARP,</div><div class='add'>+	IRDMA_HMC_IW_APBVT_ENTRY,</div><div class='add'>+	IRDMA_HMC_IW_MR,</div><div class='add'>+	IRDMA_HMC_IW_XF,</div><div class='add'>+	IRDMA_HMC_IW_XFFL,</div><div class='add'>+	IRDMA_HMC_IW_Q1,</div><div class='add'>+	IRDMA_HMC_IW_Q1FL,</div><div class='add'>+	IRDMA_HMC_IW_PBLE,</div><div class='add'>+	IRDMA_HMC_IW_TIMER,</div><div class='add'>+	IRDMA_HMC_IW_FSIMC,</div><div class='add'>+	IRDMA_HMC_IW_FSIAV,</div><div class='add'>+	IRDMA_HMC_IW_RRF,</div><div class='add'>+	IRDMA_HMC_IW_RRFFL,</div><div class='add'>+	IRDMA_HMC_IW_HDR,</div><div class='add'>+	IRDMA_HMC_IW_MD,</div><div class='add'>+	IRDMA_HMC_IW_OOISC,</div><div class='add'>+	IRDMA_HMC_IW_OOISCFFL,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_iwarp_ce_handler - handle iwarp completions</div><div class='add'>+ * @iwcq: iwarp cq receiving event</div><div class='add'>+ */</div><div class='add'>+static void irdma_iwarp_ce_handler(struct irdma_sc_cq *iwcq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cq *cq = iwcq-&gt;back_cq;</div><div class='add'>+</div><div class='add'>+	if (!cq-&gt;user_mode)</div><div class='add'>+		atomic_set(&amp;cq-&gt;armed, 0);</div><div class='add'>+	if (cq-&gt;ibcq.comp_handler)</div><div class='add'>+		cq-&gt;ibcq.comp_handler(&amp;cq-&gt;ibcq, cq-&gt;ibcq.cq_context);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_ce_handler - handle puda completion events</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @cq: puda completion q for event</div><div class='add'>+ */</div><div class='add'>+static void irdma_puda_ce_handler(struct irdma_pci_f *rf,</div><div class='add'>+				  struct irdma_sc_cq *cq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	u32 compl_error;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		status = irdma_puda_poll_cmpl(dev, cq, &amp;compl_error);</div><div class='add'>+		if (status == -ENOENT)</div><div class='add'>+			break;</div><div class='add'>+		if (status) {</div><div class='add'>+			ibdev_dbg(to_ibdev(dev), "ERR: puda status = %d\n", status);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		if (compl_error) {</div><div class='add'>+			ibdev_dbg(to_ibdev(dev), "ERR: puda compl_err  =0x%x\n",</div><div class='add'>+				  compl_error);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	} while (1);</div><div class='add'>+</div><div class='add'>+	irdma_sc_ccq_arm(cq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_process_ceq - handle ceq for completions</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @ceq: ceq having cq for completion</div><div class='add'>+ */</div><div class='add'>+static void irdma_process_ceq(struct irdma_pci_f *rf, struct irdma_ceq *ceq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_sc_ceq *sc_ceq;</div><div class='add'>+	struct irdma_sc_cq *cq;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	sc_ceq = &amp;ceq-&gt;sc_ceq;</div><div class='add'>+	do {</div><div class='add'>+		spin_lock_irqsave(&amp;ceq-&gt;ce_lock, flags);</div><div class='add'>+		cq = irdma_sc_process_ceq(dev, sc_ceq);</div><div class='add'>+		if (!cq) {</div><div class='add'>+			spin_unlock_irqrestore(&amp;ceq-&gt;ce_lock, flags);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (cq-&gt;cq_type == IRDMA_CQ_TYPE_IWARP)</div><div class='add'>+			irdma_iwarp_ce_handler(cq);</div><div class='add'>+</div><div class='add'>+		spin_unlock_irqrestore(&amp;ceq-&gt;ce_lock, flags);</div><div class='add'>+</div><div class='add'>+		if (cq-&gt;cq_type == IRDMA_CQ_TYPE_CQP)</div><div class='add'>+			queue_work(rf-&gt;cqp_cmpl_wq, &amp;rf-&gt;cqp_cmpl_work);</div><div class='add'>+		else if (cq-&gt;cq_type == IRDMA_CQ_TYPE_ILQ ||</div><div class='add'>+			 cq-&gt;cq_type == IRDMA_CQ_TYPE_IEQ)</div><div class='add'>+			irdma_puda_ce_handler(rf, cq);</div><div class='add'>+	} while (1);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_set_flush_fields(struct irdma_sc_qp *qp,</div><div class='add'>+				   struct irdma_aeqe_info *info)</div><div class='add'>+{</div><div class='add'>+	struct qp_err_code qp_err;</div><div class='add'>+</div><div class='add'>+	qp-&gt;sq_flush_code = info-&gt;sq;</div><div class='add'>+	qp-&gt;rq_flush_code = info-&gt;rq;</div><div class='add'>+	if (qp-&gt;qp_uk.uk_attrs-&gt;hw_rev &gt;= IRDMA_GEN_3) {</div><div class='add'>+		if (info-&gt;sq) {</div><div class='add'>+			qp-&gt;err_sq_idx_valid = true;</div><div class='add'>+			qp-&gt;err_sq_idx = info-&gt;wqe_idx;</div><div class='add'>+		}</div><div class='add'>+		if (info-&gt;rq) {</div><div class='add'>+			qp-&gt;err_rq_idx_valid = true;</div><div class='add'>+			qp-&gt;err_rq_idx = info-&gt;wqe_idx;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	qp_err = irdma_ae_to_qp_err_code(info-&gt;ae_id);</div><div class='add'>+	qp-&gt;flush_code = qp_err.flush_code;</div><div class='add'>+	qp-&gt;event_type = qp_err.event_type;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_complete_cqp_request - perform post-completion cleanup</div><div class='add'>+ * @cqp: device CQP</div><div class='add'>+ * @cqp_request: CQP request</div><div class='add'>+ *</div><div class='add'>+ * Mark CQP request as done, wake up waiting thread or invoke</div><div class='add'>+ * callback function and release/free CQP request.</div><div class='add'>+ */</div><div class='add'>+static void irdma_complete_cqp_request(struct irdma_cqp *cqp,</div><div class='add'>+				       struct irdma_cqp_request *cqp_request)</div><div class='add'>+{</div><div class='add'>+	if (cqp_request-&gt;waiting) {</div><div class='add'>+		WRITE_ONCE(cqp_request-&gt;request_done, true);</div><div class='add'>+		wake_up(&amp;cqp_request-&gt;waitq);</div><div class='add'>+	} else if (cqp_request-&gt;callback_fcn) {</div><div class='add'>+		cqp_request-&gt;callback_fcn(cqp_request);</div><div class='add'>+	}</div><div class='add'>+	irdma_put_cqp_request(cqp, cqp_request);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_process_ae_def_cmpl - handle IRDMA_AE_CQP_DEFERRED_COMPLETE event</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @info: AEQ entry info</div><div class='add'>+ */</div><div class='add'>+static void irdma_process_ae_def_cmpl(struct irdma_pci_f *rf,</div><div class='add'>+				      struct irdma_aeqe_info *info)</div><div class='add'>+{</div><div class='add'>+	u32 sw_def_info;</div><div class='add'>+	u64 scratch;</div><div class='add'>+</div><div class='add'>+	irdma_cqp_ce_handler(rf, &amp;rf-&gt;ccq.sc_cq);</div><div class='add'>+</div><div class='add'>+	irdma_sc_cqp_def_cmpl_ae_handler(&amp;rf-&gt;sc_dev, info, true,</div><div class='add'>+					 &amp;scratch, &amp;sw_def_info);</div><div class='add'>+	while (scratch) {</div><div class='add'>+		struct irdma_cqp_request *cqp_request =</div><div class='add'>+			(struct irdma_cqp_request *)(uintptr_t)scratch;</div><div class='add'>+</div><div class='add'>+		irdma_complete_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+		irdma_sc_cqp_def_cmpl_ae_handler(&amp;rf-&gt;sc_dev, info, false,</div><div class='add'>+						 &amp;scratch, &amp;sw_def_info);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_process_aeq - handle aeq events</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ */</div><div class='add'>+static void irdma_process_aeq(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_aeq *aeq = &amp;rf-&gt;aeq;</div><div class='add'>+	struct irdma_sc_aeq *sc_aeq = &amp;aeq-&gt;sc_aeq;</div><div class='add'>+	struct irdma_aeqe_info aeinfo;</div><div class='add'>+	struct irdma_aeqe_info *info = &amp;aeinfo;</div><div class='add'>+	int ret;</div><div class='add'>+	struct irdma_qp *iwqp = NULL;</div><div class='add'>+	struct irdma_cq *iwcq = NULL;</div><div class='add'>+	struct irdma_sc_qp *qp = NULL;</div><div class='add'>+	struct irdma_qp_host_ctx_info *ctx_info = NULL;</div><div class='add'>+	struct irdma_device *iwdev = rf-&gt;iwdev;</div><div class='add'>+	struct irdma_sc_srq *srq;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	u32 aeqcnt = 0;</div><div class='add'>+</div><div class='add'>+	if (!sc_aeq-&gt;size)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		memset(info, 0, sizeof(*info));</div><div class='add'>+		ret = irdma_sc_get_next_aeqe(sc_aeq, info);</div><div class='add'>+		if (ret)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		if (info-&gt;aeqe_overflow) {</div><div class='add'>+			ibdev_err(&amp;iwdev-&gt;ibdev, "AEQ has overflowed\n");</div><div class='add'>+			rf-&gt;reset = true;</div><div class='add'>+			rf-&gt;gen_ops.request_reset(rf);</div><div class='add'>+			return;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		aeqcnt++;</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+			  "AEQ: ae_id = 0x%x bool qp=%d qp_id = %d tcp_state=%d iwarp_state=%d ae_src=%d\n",</div><div class='add'>+			  info-&gt;ae_id, info-&gt;qp, info-&gt;qp_cq_id, info-&gt;tcp_state,</div><div class='add'>+			  info-&gt;iwarp_state, info-&gt;ae_src);</div><div class='add'>+</div><div class='add'>+		if (info-&gt;qp) {</div><div class='add'>+			spin_lock_irqsave(&amp;rf-&gt;qptable_lock, flags);</div><div class='add'>+			iwqp = rf-&gt;qp_table[info-&gt;qp_cq_id];</div><div class='add'>+			if (!iwqp) {</div><div class='add'>+				spin_unlock_irqrestore(&amp;rf-&gt;qptable_lock,</div><div class='add'>+						       flags);</div><div class='add'>+				if (info-&gt;ae_id == IRDMA_AE_QP_SUSPEND_COMPLETE) {</div><div class='add'>+					atomic_dec(&amp;iwdev-&gt;vsi.qp_suspend_reqs);</div><div class='add'>+					wake_up(&amp;iwdev-&gt;suspend_wq);</div><div class='add'>+					continue;</div><div class='add'>+				}</div><div class='add'>+				ibdev_dbg(&amp;iwdev-&gt;ibdev, "AEQ: qp_id %d is already freed\n",</div><div class='add'>+					  info-&gt;qp_cq_id);</div><div class='add'>+				continue;</div><div class='add'>+			}</div><div class='add'>+			irdma_qp_add_ref(&amp;iwqp-&gt;ibqp);</div><div class='add'>+			spin_unlock_irqrestore(&amp;rf-&gt;qptable_lock, flags);</div><div class='add'>+			qp = &amp;iwqp-&gt;sc_qp;</div><div class='add'>+			spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+			iwqp-&gt;hw_tcp_state = info-&gt;tcp_state;</div><div class='add'>+			iwqp-&gt;hw_iwarp_state = info-&gt;iwarp_state;</div><div class='add'>+			if (info-&gt;ae_id != IRDMA_AE_QP_SUSPEND_COMPLETE)</div><div class='add'>+				iwqp-&gt;last_aeq = info-&gt;ae_id;</div><div class='add'>+			spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+		} else if (info-&gt;srq) {</div><div class='add'>+			if (info-&gt;ae_id != IRDMA_AE_SRQ_LIMIT)</div><div class='add'>+				continue;</div><div class='add'>+		} else {</div><div class='add'>+			if (info-&gt;ae_id != IRDMA_AE_CQ_OPERATION_ERROR &amp;&amp;</div><div class='add'>+			    info-&gt;ae_id != IRDMA_AE_CQP_DEFERRED_COMPLETE)</div><div class='add'>+				continue;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		switch (info-&gt;ae_id) {</div><div class='add'>+			struct irdma_cm_node *cm_node;</div><div class='add'>+		case IRDMA_AE_LLP_CONNECTION_ESTABLISHED:</div><div class='add'>+			cm_node = iwqp-&gt;cm_node;</div><div class='add'>+			if (cm_node-&gt;accept_pend) {</div><div class='add'>+				atomic_dec(&amp;cm_node-&gt;listener-&gt;pend_accepts_cnt);</div><div class='add'>+				cm_node-&gt;accept_pend = 0;</div><div class='add'>+			}</div><div class='add'>+			iwqp-&gt;rts_ae_rcvd = 1;</div><div class='add'>+			wake_up_interruptible(&amp;iwqp-&gt;waitq);</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_AE_LLP_FIN_RECEIVED:</div><div class='add'>+		case IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE:</div><div class='add'>+			if (qp-&gt;term_flags)</div><div class='add'>+				break;</div><div class='add'>+			if (atomic_inc_return(&amp;iwqp-&gt;close_timer_started) == 1) {</div><div class='add'>+				iwqp-&gt;hw_tcp_state = IRDMA_TCP_STATE_CLOSE_WAIT;</div><div class='add'>+				if (iwqp-&gt;hw_tcp_state == IRDMA_TCP_STATE_CLOSE_WAIT &amp;&amp;</div><div class='add'>+				    iwqp-&gt;ibqp_state == IB_QPS_RTS) {</div><div class='add'>+					irdma_next_iw_state(iwqp,</div><div class='add'>+							    IRDMA_QP_STATE_CLOSING,</div><div class='add'>+							    0, 0, 0);</div><div class='add'>+					irdma_cm_disconn(iwqp);</div><div class='add'>+				}</div><div class='add'>+				irdma_schedule_cm_timer(iwqp-&gt;cm_node,</div><div class='add'>+							(struct irdma_puda_buf *)iwqp,</div><div class='add'>+							IRDMA_TIMER_TYPE_CLOSE,</div><div class='add'>+							1, 0);</div><div class='add'>+			}</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_AE_LLP_CLOSE_COMPLETE:</div><div class='add'>+			if (qp-&gt;term_flags)</div><div class='add'>+				irdma_terminate_done(qp, 0);</div><div class='add'>+			else</div><div class='add'>+				irdma_cm_disconn(iwqp);</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_AE_BAD_CLOSE:</div><div class='add'>+		case IRDMA_AE_RESET_SENT:</div><div class='add'>+			irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, 1, 0,</div><div class='add'>+					    0);</div><div class='add'>+			irdma_cm_disconn(iwqp);</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_AE_LLP_CONNECTION_RESET:</div><div class='add'>+			if (atomic_read(&amp;iwqp-&gt;close_timer_started))</div><div class='add'>+				break;</div><div class='add'>+			irdma_cm_disconn(iwqp);</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_AE_QP_SUSPEND_COMPLETE:</div><div class='add'>+			if (iwqp-&gt;iwdev-&gt;vsi.tc_change_pending) {</div><div class='add'>+				if (!atomic_dec_return(&amp;qp-&gt;vsi-&gt;qp_suspend_reqs))</div><div class='add'>+					wake_up(&amp;iwqp-&gt;iwdev-&gt;suspend_wq);</div><div class='add'>+			}</div><div class='add'>+			if (iwqp-&gt;suspend_pending) {</div><div class='add'>+				iwqp-&gt;suspend_pending = false;</div><div class='add'>+				wake_up(&amp;iwqp-&gt;iwdev-&gt;suspend_wq);</div><div class='add'>+			}</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_AE_TERMINATE_SENT:</div><div class='add'>+			irdma_terminate_send_fin(qp);</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_AE_LLP_TERMINATE_RECEIVED:</div><div class='add'>+			irdma_terminate_received(qp, info);</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_AE_CQ_OPERATION_ERROR:</div><div class='add'>+			ibdev_err(&amp;iwdev-&gt;ibdev,</div><div class='add'>+				  "Processing an iWARP related AE for CQ misc = 0x%04X\n",</div><div class='add'>+				  info-&gt;ae_id);</div><div class='add'>+</div><div class='add'>+			spin_lock_irqsave(&amp;rf-&gt;cqtable_lock, flags);</div><div class='add'>+			iwcq = rf-&gt;cq_table[info-&gt;qp_cq_id];</div><div class='add'>+			if (!iwcq) {</div><div class='add'>+				spin_unlock_irqrestore(&amp;rf-&gt;cqtable_lock,</div><div class='add'>+						       flags);</div><div class='add'>+				ibdev_dbg(to_ibdev(dev),</div><div class='add'>+					  "cq_id %d is already freed\n", info-&gt;qp_cq_id);</div><div class='add'>+				continue;</div><div class='add'>+			}</div><div class='add'>+			irdma_cq_add_ref(&amp;iwcq-&gt;ibcq);</div><div class='add'>+			spin_unlock_irqrestore(&amp;rf-&gt;cqtable_lock, flags);</div><div class='add'>+</div><div class='add'>+			if (iwcq-&gt;ibcq.event_handler) {</div><div class='add'>+				struct ib_event ibevent;</div><div class='add'>+</div><div class='add'>+				ibevent.device = iwcq-&gt;ibcq.device;</div><div class='add'>+				ibevent.event = IB_EVENT_CQ_ERR;</div><div class='add'>+				ibevent.element.cq = &amp;iwcq-&gt;ibcq;</div><div class='add'>+				iwcq-&gt;ibcq.event_handler(&amp;ibevent,</div><div class='add'>+							 iwcq-&gt;ibcq.cq_context);</div><div class='add'>+			}</div><div class='add'>+			irdma_cq_rem_ref(&amp;iwcq-&gt;ibcq);</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_AE_SRQ_LIMIT:</div><div class='add'>+			srq = (struct irdma_sc_srq *)(uintptr_t)info-&gt;compl_ctx;</div><div class='add'>+			irdma_srq_event(srq);</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_AE_SRQ_CATASTROPHIC_ERROR:</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_AE_CQP_DEFERRED_COMPLETE:</div><div class='add'>+			/* Remove completed CQP requests from pending list</div><div class='add'>+			 * and notify about those CQP ops completion.</div><div class='add'>+			 */</div><div class='add'>+			irdma_process_ae_def_cmpl(rf, info);</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_AE_RESET_NOT_SENT:</div><div class='add'>+		case IRDMA_AE_LLP_DOUBT_REACHABILITY:</div><div class='add'>+		case IRDMA_AE_RESOURCE_EXHAUSTION:</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_AE_PRIV_OPERATION_DENIED:</div><div class='add'>+		case IRDMA_AE_STAG_ZERO_INVALID:</div><div class='add'>+		case IRDMA_AE_IB_RREQ_AND_Q1_FULL:</div><div class='add'>+		case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:</div><div class='add'>+		case IRDMA_AE_DDP_UBE_INVALID_MO:</div><div class='add'>+		case IRDMA_AE_DDP_UBE_INVALID_QN:</div><div class='add'>+		case IRDMA_AE_DDP_NO_L_BIT:</div><div class='add'>+		case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:</div><div class='add'>+		case IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE:</div><div class='add'>+		case IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST:</div><div class='add'>+		case IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:</div><div class='add'>+		case IRDMA_AE_INVALID_ARP_ENTRY:</div><div class='add'>+		case IRDMA_AE_INVALID_TCP_OPTION_RCVD:</div><div class='add'>+		case IRDMA_AE_STALE_ARP_ENTRY:</div><div class='add'>+		case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:</div><div class='add'>+		case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:</div><div class='add'>+		case IRDMA_AE_LLP_SYN_RECEIVED:</div><div class='add'>+		case IRDMA_AE_LLP_TOO_MANY_RETRIES:</div><div class='add'>+		case IRDMA_AE_LCE_QP_CATASTROPHIC:</div><div class='add'>+		case IRDMA_AE_LCE_FUNCTION_CATASTROPHIC:</div><div class='add'>+		case IRDMA_AE_LLP_TOO_MANY_RNRS:</div><div class='add'>+		case IRDMA_AE_LCE_CQ_CATASTROPHIC:</div><div class='add'>+		case IRDMA_AE_REMOTE_QP_CATASTROPHIC:</div><div class='add'>+		case IRDMA_AE_LOCAL_QP_CATASTROPHIC:</div><div class='add'>+		case IRDMA_AE_RCE_QP_CATASTROPHIC:</div><div class='add'>+		case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:</div><div class='add'>+		default:</div><div class='add'>+			ibdev_err(&amp;iwdev-&gt;ibdev, "abnormal ae_id = 0x%x bool qp=%d qp_id = %d, ae_src=%d\n",</div><div class='add'>+				  info-&gt;ae_id, info-&gt;qp, info-&gt;qp_cq_id, info-&gt;ae_src);</div><div class='add'>+			ctx_info = &amp;iwqp-&gt;ctx_info;</div><div class='add'>+			if (rdma_protocol_roce(&amp;iwqp-&gt;iwdev-&gt;ibdev, 1)) {</div><div class='add'>+				ctx_info-&gt;roce_info-&gt;err_rq_idx_valid =</div><div class='add'>+					ctx_info-&gt;srq_valid ? false : info-&gt;err_rq_idx_valid;</div><div class='add'>+				if (ctx_info-&gt;roce_info-&gt;err_rq_idx_valid) {</div><div class='add'>+					ctx_info-&gt;roce_info-&gt;err_rq_idx = info-&gt;wqe_idx;</div><div class='add'>+					irdma_sc_qp_setctx_roce(&amp;iwqp-&gt;sc_qp, iwqp-&gt;host_ctx.va,</div><div class='add'>+								ctx_info);</div><div class='add'>+				}</div><div class='add'>+				irdma_set_flush_fields(qp, info);</div><div class='add'>+				irdma_cm_disconn(iwqp);</div><div class='add'>+				break;</div><div class='add'>+			}</div><div class='add'>+			ctx_info-&gt;iwarp_info-&gt;err_rq_idx_valid = info-&gt;rq;</div><div class='add'>+			if (info-&gt;rq) {</div><div class='add'>+				ctx_info-&gt;iwarp_info-&gt;err_rq_idx = info-&gt;wqe_idx;</div><div class='add'>+				ctx_info-&gt;tcp_info_valid = false;</div><div class='add'>+				ctx_info-&gt;iwarp_info_valid = true;</div><div class='add'>+				irdma_sc_qp_setctx(&amp;iwqp-&gt;sc_qp, iwqp-&gt;host_ctx.va,</div><div class='add'>+						   ctx_info);</div><div class='add'>+			}</div><div class='add'>+			if (iwqp-&gt;hw_iwarp_state != IRDMA_QP_STATE_RTS &amp;&amp;</div><div class='add'>+			    iwqp-&gt;hw_iwarp_state != IRDMA_QP_STATE_TERMINATE) {</div><div class='add'>+				irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, 1, 0, 0);</div><div class='add'>+				irdma_cm_disconn(iwqp);</div><div class='add'>+			} else {</div><div class='add'>+				irdma_terminate_connection(qp, info);</div><div class='add'>+			}</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		if (info-&gt;qp)</div><div class='add'>+			irdma_qp_rem_ref(&amp;iwqp-&gt;ibqp);</div><div class='add'>+	} while (1);</div><div class='add'>+</div><div class='add'>+	if (aeqcnt)</div><div class='add'>+		irdma_sc_repost_aeq_entries(dev, aeqcnt);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ena_intr - set up device interrupts</div><div class='add'>+ * @dev: hardware control device structure</div><div class='add'>+ * @msix_id: id of the interrupt to be enabled</div><div class='add'>+ */</div><div class='add'>+static void irdma_ena_intr(struct irdma_sc_dev *dev, u32 msix_id)</div><div class='add'>+{</div><div class='add'>+	dev-&gt;irq_ops-&gt;irdma_en_irq(dev, msix_id);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_dpc - tasklet for aeq and ceq 0</div><div class='add'>+ * @t: tasklet_struct ptr</div><div class='add'>+ */</div><div class='add'>+static void irdma_dpc(struct tasklet_struct *t)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = from_tasklet(rf, t, dpc_tasklet);</div><div class='add'>+</div><div class='add'>+	if (rf-&gt;msix_shared)</div><div class='add'>+		irdma_process_ceq(rf, rf-&gt;ceqlist);</div><div class='add'>+	irdma_process_aeq(rf);</div><div class='add'>+	irdma_ena_intr(&amp;rf-&gt;sc_dev, rf-&gt;iw_msixtbl[0].idx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ceq_dpc - dpc handler for CEQ</div><div class='add'>+ * @t: tasklet_struct ptr</div><div class='add'>+ */</div><div class='add'>+static void irdma_ceq_dpc(struct tasklet_struct *t)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ceq *iwceq = from_tasklet(iwceq, t, dpc_tasklet);</div><div class='add'>+	struct irdma_pci_f *rf = iwceq-&gt;rf;</div><div class='add'>+</div><div class='add'>+	irdma_process_ceq(rf, iwceq);</div><div class='add'>+	irdma_ena_intr(&amp;rf-&gt;sc_dev, iwceq-&gt;msix_idx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_save_msix_info - copy msix vector information to iwarp device</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ *</div><div class='add'>+ * Allocate iwdev msix table and copy the msix info to the table</div><div class='add'>+ * Return 0 if successful, otherwise return error</div><div class='add'>+ */</div><div class='add'>+static int irdma_save_msix_info(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qvlist_info *iw_qvlist;</div><div class='add'>+	struct irdma_qv_info *iw_qvinfo;</div><div class='add'>+	struct msix_entry *pmsix;</div><div class='add'>+	u32 ceq_idx;</div><div class='add'>+	u32 i;</div><div class='add'>+	size_t size;</div><div class='add'>+</div><div class='add'>+	if (!rf-&gt;msix_count)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	size = sizeof(struct irdma_msix_vector) * rf-&gt;msix_count;</div><div class='add'>+	size += struct_size(iw_qvlist, qv_info, rf-&gt;msix_count);</div><div class='add'>+	rf-&gt;iw_msixtbl = kzalloc(size, GFP_KERNEL);</div><div class='add'>+	if (!rf-&gt;iw_msixtbl)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	rf-&gt;iw_qvlist = (struct irdma_qvlist_info *)</div><div class='add'>+			(&amp;rf-&gt;iw_msixtbl[rf-&gt;msix_count]);</div><div class='add'>+	iw_qvlist = rf-&gt;iw_qvlist;</div><div class='add'>+	iw_qvinfo = iw_qvlist-&gt;qv_info;</div><div class='add'>+	iw_qvlist-&gt;num_vectors = rf-&gt;msix_count;</div><div class='add'>+	if (rf-&gt;msix_count &lt;= num_online_cpus())</div><div class='add'>+		rf-&gt;msix_shared = true;</div><div class='add'>+</div><div class='add'>+	pmsix = rf-&gt;msix_entries;</div><div class='add'>+	for (i = 0, ceq_idx = 0; i &lt; rf-&gt;msix_count; i++, iw_qvinfo++) {</div><div class='add'>+		rf-&gt;iw_msixtbl[i].idx = pmsix-&gt;entry;</div><div class='add'>+		rf-&gt;iw_msixtbl[i].irq = pmsix-&gt;vector;</div><div class='add'>+		rf-&gt;iw_msixtbl[i].cpu_affinity = ceq_idx;</div><div class='add'>+		if (!i) {</div><div class='add'>+			iw_qvinfo-&gt;aeq_idx = 0;</div><div class='add'>+			if (rf-&gt;msix_shared)</div><div class='add'>+				iw_qvinfo-&gt;ceq_idx = ceq_idx++;</div><div class='add'>+			else</div><div class='add'>+				iw_qvinfo-&gt;ceq_idx = IRDMA_Q_INVALID_IDX;</div><div class='add'>+		} else {</div><div class='add'>+			iw_qvinfo-&gt;aeq_idx = IRDMA_Q_INVALID_IDX;</div><div class='add'>+			iw_qvinfo-&gt;ceq_idx = ceq_idx++;</div><div class='add'>+		}</div><div class='add'>+		iw_qvinfo-&gt;itr_idx = 3;</div><div class='add'>+		iw_qvinfo-&gt;v_idx = rf-&gt;iw_msixtbl[i].idx;</div><div class='add'>+		pmsix++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_irq_handler - interrupt handler for aeq and ceq0</div><div class='add'>+ * @irq: Interrupt request number</div><div class='add'>+ * @data: RDMA PCI function</div><div class='add'>+ */</div><div class='add'>+static irqreturn_t irdma_irq_handler(int irq, void *data)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = data;</div><div class='add'>+</div><div class='add'>+	tasklet_schedule(&amp;rf-&gt;dpc_tasklet);</div><div class='add'>+</div><div class='add'>+	return IRQ_HANDLED;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ceq_handler - interrupt handler for ceq</div><div class='add'>+ * @irq: interrupt request number</div><div class='add'>+ * @data: ceq pointer</div><div class='add'>+ */</div><div class='add'>+static irqreturn_t irdma_ceq_handler(int irq, void *data)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ceq *iwceq = data;</div><div class='add'>+</div><div class='add'>+	if (iwceq-&gt;irq != irq)</div><div class='add'>+		ibdev_err(to_ibdev(&amp;iwceq-&gt;rf-&gt;sc_dev), "expected irq = %d received irq = %d\n",</div><div class='add'>+			  iwceq-&gt;irq, irq);</div><div class='add'>+	tasklet_schedule(&amp;iwceq-&gt;dpc_tasklet);</div><div class='add'>+</div><div class='add'>+	return IRQ_HANDLED;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_destroy_irq - destroy device interrupts</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @msix_vec: msix vector to disable irq</div><div class='add'>+ * @dev_id: parameter to pass to free_irq (used during irq setup)</div><div class='add'>+ *</div><div class='add'>+ * The function is called when destroying aeq/ceq</div><div class='add'>+ */</div><div class='add'>+static void irdma_destroy_irq(struct irdma_pci_f *rf,</div><div class='add'>+			      struct irdma_msix_vector *msix_vec, void *dev_id)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+</div><div class='add'>+	dev-&gt;irq_ops-&gt;irdma_dis_irq(dev, msix_vec-&gt;idx);</div><div class='add'>+	irq_update_affinity_hint(msix_vec-&gt;irq, NULL);</div><div class='add'>+	free_irq(msix_vec-&gt;irq, dev_id);</div><div class='add'>+	if (rf == dev_id) {</div><div class='add'>+		tasklet_kill(&amp;rf-&gt;dpc_tasklet);</div><div class='add'>+	} else {</div><div class='add'>+		struct irdma_ceq *iwceq = (struct irdma_ceq *)dev_id;</div><div class='add'>+</div><div class='add'>+		tasklet_kill(&amp;iwceq-&gt;dpc_tasklet);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_destroy_cqp  - destroy control qp</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ *</div><div class='add'>+ * Issue destroy cqp request and</div><div class='add'>+ * free the resources associated with the cqp</div><div class='add'>+ */</div><div class='add'>+static void irdma_destroy_cqp(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_cqp *cqp = &amp;rf-&gt;cqp;</div><div class='add'>+	int status = 0;</div><div class='add'>+</div><div class='add'>+	status = irdma_sc_cqp_destroy(dev-&gt;cqp);</div><div class='add'>+	if (status)</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "ERR: Destroy CQP failed %d\n", status);</div><div class='add'>+</div><div class='add'>+	irdma_cleanup_pending_cqp_op(rf);</div><div class='add'>+	dma_free_coherent(dev-&gt;hw-&gt;device, cqp-&gt;sq.size, cqp-&gt;sq.va,</div><div class='add'>+			  cqp-&gt;sq.pa);</div><div class='add'>+	cqp-&gt;sq.va = NULL;</div><div class='add'>+	kfree(cqp-&gt;oop_op_array);</div><div class='add'>+	cqp-&gt;oop_op_array = NULL;</div><div class='add'>+	kfree(cqp-&gt;scratch_array);</div><div class='add'>+	cqp-&gt;scratch_array = NULL;</div><div class='add'>+	kfree(cqp-&gt;cqp_requests);</div><div class='add'>+	cqp-&gt;cqp_requests = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_destroy_virt_aeq(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_aeq *aeq = &amp;rf-&gt;aeq;</div><div class='add'>+	u32 pg_cnt = DIV_ROUND_UP(aeq-&gt;mem.size, PAGE_SIZE);</div><div class='add'>+	dma_addr_t *pg_arr = (dma_addr_t *)aeq-&gt;palloc.level1.addr;</div><div class='add'>+</div><div class='add'>+	irdma_unmap_vm_page_list(&amp;rf-&gt;hw, pg_arr, pg_cnt);</div><div class='add'>+	irdma_free_pble(rf-&gt;pble_rsrc, &amp;aeq-&gt;palloc);</div><div class='add'>+	vfree(aeq-&gt;mem.va);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_destroy_aeq - destroy aeq</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ *</div><div class='add'>+ * Issue a destroy aeq request and</div><div class='add'>+ * free the resources associated with the aeq</div><div class='add'>+ * The function is called during driver unload</div><div class='add'>+ */</div><div class='add'>+static void irdma_destroy_aeq(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_aeq *aeq = &amp;rf-&gt;aeq;</div><div class='add'>+	int status = -EBUSY;</div><div class='add'>+</div><div class='add'>+	if (!rf-&gt;msix_shared) {</div><div class='add'>+		if (rf-&gt;sc_dev.privileged)</div><div class='add'>+			rf-&gt;sc_dev.irq_ops-&gt;irdma_cfg_aeq(&amp;rf-&gt;sc_dev,</div><div class='add'>+							  rf-&gt;iw_msixtbl-&gt;idx, false);</div><div class='add'>+		irdma_destroy_irq(rf, rf-&gt;iw_msixtbl, rf);</div><div class='add'>+	}</div><div class='add'>+	if (rf-&gt;reset)</div><div class='add'>+		goto exit;</div><div class='add'>+</div><div class='add'>+	aeq-&gt;sc_aeq.size = 0;</div><div class='add'>+	status = irdma_cqp_aeq_cmd(dev, &amp;aeq-&gt;sc_aeq, IRDMA_OP_AEQ_DESTROY);</div><div class='add'>+	if (status)</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "ERR: Destroy AEQ failed %d\n", status);</div><div class='add'>+</div><div class='add'>+exit:</div><div class='add'>+	if (aeq-&gt;virtual_map) {</div><div class='add'>+		irdma_destroy_virt_aeq(rf);</div><div class='add'>+	} else {</div><div class='add'>+		dma_free_coherent(dev-&gt;hw-&gt;device, aeq-&gt;mem.size, aeq-&gt;mem.va,</div><div class='add'>+				  aeq-&gt;mem.pa);</div><div class='add'>+		aeq-&gt;mem.va = NULL;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_destroy_ceq - destroy ceq</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @iwceq: ceq to be destroyed</div><div class='add'>+ *</div><div class='add'>+ * Issue a destroy ceq request and</div><div class='add'>+ * free the resources associated with the ceq</div><div class='add'>+ */</div><div class='add'>+static void irdma_destroy_ceq(struct irdma_pci_f *rf, struct irdma_ceq *iwceq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	if (rf-&gt;reset)</div><div class='add'>+		goto exit;</div><div class='add'>+</div><div class='add'>+	status = irdma_sc_ceq_destroy(&amp;iwceq-&gt;sc_ceq, 0, 1);</div><div class='add'>+	if (status) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "ERR: CEQ destroy command failed %d\n", status);</div><div class='add'>+		goto exit;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	status = irdma_sc_cceq_destroy_done(&amp;iwceq-&gt;sc_ceq);</div><div class='add'>+	if (status)</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "ERR: CEQ destroy completion failed %d\n",</div><div class='add'>+			  status);</div><div class='add'>+exit:</div><div class='add'>+	dma_free_coherent(dev-&gt;hw-&gt;device, iwceq-&gt;mem.size, iwceq-&gt;mem.va,</div><div class='add'>+			  iwceq-&gt;mem.pa);</div><div class='add'>+	iwceq-&gt;mem.va = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_del_ceq_0 - destroy ceq 0</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ *</div><div class='add'>+ * Disable the ceq 0 interrupt and destroy the ceq 0</div><div class='add'>+ */</div><div class='add'>+static void irdma_del_ceq_0(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ceq *iwceq = rf-&gt;ceqlist;</div><div class='add'>+	struct irdma_msix_vector *msix_vec;</div><div class='add'>+</div><div class='add'>+	if (rf-&gt;msix_shared) {</div><div class='add'>+		msix_vec = &amp;rf-&gt;iw_msixtbl[0];</div><div class='add'>+		if (rf-&gt;sc_dev.privileged)</div><div class='add'>+			rf-&gt;sc_dev.irq_ops-&gt;irdma_cfg_ceq(&amp;rf-&gt;sc_dev,</div><div class='add'>+							  msix_vec-&gt;ceq_id,</div><div class='add'>+							  msix_vec-&gt;idx, false);</div><div class='add'>+		irdma_destroy_irq(rf, msix_vec, rf);</div><div class='add'>+	} else {</div><div class='add'>+		msix_vec = &amp;rf-&gt;iw_msixtbl[1];</div><div class='add'>+		irdma_destroy_irq(rf, msix_vec, iwceq);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_destroy_ceq(rf, iwceq);</div><div class='add'>+	rf-&gt;sc_dev.ceq_valid = false;</div><div class='add'>+	rf-&gt;ceqs_count = 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_del_ceqs - destroy all ceq's except CEQ 0</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ *</div><div class='add'>+ * Go through all of the device ceq's, except 0, and for each</div><div class='add'>+ * ceq disable the ceq interrupt and destroy the ceq</div><div class='add'>+ */</div><div class='add'>+static void irdma_del_ceqs(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ceq *iwceq = &amp;rf-&gt;ceqlist[1];</div><div class='add'>+	struct irdma_msix_vector *msix_vec;</div><div class='add'>+	u32 i = 0;</div><div class='add'>+</div><div class='add'>+	if (rf-&gt;msix_shared)</div><div class='add'>+		msix_vec = &amp;rf-&gt;iw_msixtbl[1];</div><div class='add'>+	else</div><div class='add'>+		msix_vec = &amp;rf-&gt;iw_msixtbl[2];</div><div class='add'>+</div><div class='add'>+	for (i = 1; i &lt; rf-&gt;ceqs_count; i++, msix_vec++, iwceq++) {</div><div class='add'>+		if (rf-&gt;sc_dev.privileged)</div><div class='add'>+			rf-&gt;sc_dev.irq_ops-&gt;irdma_cfg_ceq(&amp;rf-&gt;sc_dev,</div><div class='add'>+							  msix_vec-&gt;ceq_id,</div><div class='add'>+							  msix_vec-&gt;idx, false);</div><div class='add'>+		irdma_destroy_irq(rf, msix_vec, iwceq);</div><div class='add'>+		irdma_cqp_ceq_cmd(&amp;rf-&gt;sc_dev, &amp;iwceq-&gt;sc_ceq,</div><div class='add'>+				  IRDMA_OP_CEQ_DESTROY);</div><div class='add'>+		dma_free_coherent(rf-&gt;sc_dev.hw-&gt;device, iwceq-&gt;mem.size,</div><div class='add'>+				  iwceq-&gt;mem.va, iwceq-&gt;mem.pa);</div><div class='add'>+		iwceq-&gt;mem.va = NULL;</div><div class='add'>+	}</div><div class='add'>+	rf-&gt;ceqs_count = 1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_destroy_ccq - destroy control cq</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ *</div><div class='add'>+ * Issue destroy ccq request and</div><div class='add'>+ * free the resources associated with the ccq</div><div class='add'>+ */</div><div class='add'>+static void irdma_destroy_ccq(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_ccq *ccq = &amp;rf-&gt;ccq;</div><div class='add'>+	int status = 0;</div><div class='add'>+</div><div class='add'>+	if (rf-&gt;cqp_cmpl_wq)</div><div class='add'>+		destroy_workqueue(rf-&gt;cqp_cmpl_wq);</div><div class='add'>+</div><div class='add'>+	if (!rf-&gt;reset)</div><div class='add'>+		status = irdma_sc_ccq_destroy(dev-&gt;ccq, 0, true);</div><div class='add'>+	if (status)</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "ERR: CCQ destroy failed %d\n", status);</div><div class='add'>+	dma_free_coherent(dev-&gt;hw-&gt;device, ccq-&gt;mem_cq.size, ccq-&gt;mem_cq.va,</div><div class='add'>+			  ccq-&gt;mem_cq.pa);</div><div class='add'>+	ccq-&gt;mem_cq.va = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_close_hmc_objects_type - delete hmc objects of a given type</div><div class='add'>+ * @dev: iwarp device</div><div class='add'>+ * @obj_type: the hmc object type to be deleted</div><div class='add'>+ * @hmc_info: host memory info struct</div><div class='add'>+ * @privileged: permission to close HMC objects</div><div class='add'>+ * @reset: true if called before reset</div><div class='add'>+ */</div><div class='add'>+static void irdma_close_hmc_objects_type(struct irdma_sc_dev *dev,</div><div class='add'>+					 enum irdma_hmc_rsrc_type obj_type,</div><div class='add'>+					 struct irdma_hmc_info *hmc_info,</div><div class='add'>+					 bool privileged, bool reset)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_del_obj_info info = {};</div><div class='add'>+</div><div class='add'>+	info.hmc_info = hmc_info;</div><div class='add'>+	info.rsrc_type = obj_type;</div><div class='add'>+	info.count = hmc_info-&gt;hmc_obj[obj_type].cnt;</div><div class='add'>+	info.privileged = privileged;</div><div class='add'>+	if (irdma_sc_del_hmc_obj(dev, &amp;info, reset))</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "ERR: del HMC obj of type %d failed\n",</div><div class='add'>+			  obj_type);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_del_hmc_objects - remove all device hmc objects</div><div class='add'>+ * @dev: iwarp device</div><div class='add'>+ * @hmc_info: hmc_info to free</div><div class='add'>+ * @privileged: permission to delete HMC objects</div><div class='add'>+ * @reset: true if called before reset</div><div class='add'>+ * @vers: hardware version</div><div class='add'>+ */</div><div class='add'>+static void irdma_del_hmc_objects(struct irdma_sc_dev *dev,</div><div class='add'>+				  struct irdma_hmc_info *hmc_info, bool privileged,</div><div class='add'>+				  bool reset, enum irdma_vers vers)</div><div class='add'>+{</div><div class='add'>+	unsigned int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; IW_HMC_OBJ_TYPE_NUM; i++) {</div><div class='add'>+		if (dev-&gt;hmc_info-&gt;hmc_obj[iw_hmc_obj_types[i]].cnt)</div><div class='add'>+			irdma_close_hmc_objects_type(dev, iw_hmc_obj_types[i],</div><div class='add'>+						     hmc_info, privileged, reset);</div><div class='add'>+		if (vers == IRDMA_GEN_1 &amp;&amp; i == IRDMA_HMC_IW_TIMER)</div><div class='add'>+			break;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_create_hmc_obj_type - create hmc object of a given type</div><div class='add'>+ * @dev: hardware control device structure</div><div class='add'>+ * @info: information for the hmc object to create</div><div class='add'>+ */</div><div class='add'>+static int irdma_create_hmc_obj_type(struct irdma_sc_dev *dev,</div><div class='add'>+				     struct irdma_hmc_create_obj_info *info)</div><div class='add'>+{</div><div class='add'>+	return irdma_sc_create_hmc_obj(dev, info);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_create_hmc_objs - create all hmc objects for the device</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @privileged: permission to create HMC objects</div><div class='add'>+ * @vers: HW version</div><div class='add'>+ *</div><div class='add'>+ * Create the device hmc objects and allocate hmc pages</div><div class='add'>+ * Return 0 if successful, otherwise clean up and return error</div><div class='add'>+ */</div><div class='add'>+static int irdma_create_hmc_objs(struct irdma_pci_f *rf, bool privileged,</div><div class='add'>+				 enum irdma_vers vers)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_hmc_create_obj_info info = {};</div><div class='add'>+	int i, status = 0;</div><div class='add'>+</div><div class='add'>+	info.hmc_info = dev-&gt;hmc_info;</div><div class='add'>+	info.privileged = privileged;</div><div class='add'>+	info.entry_type = rf-&gt;sd_type;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; IW_HMC_OBJ_TYPE_NUM; i++) {</div><div class='add'>+		if (iw_hmc_obj_types[i] == IRDMA_HMC_IW_PBLE)</div><div class='add'>+			continue;</div><div class='add'>+		if (dev-&gt;hmc_info-&gt;hmc_obj[iw_hmc_obj_types[i]].cnt) {</div><div class='add'>+			info.rsrc_type = iw_hmc_obj_types[i];</div><div class='add'>+			info.count = dev-&gt;hmc_info-&gt;hmc_obj[info.rsrc_type].cnt;</div><div class='add'>+			info.add_sd_cnt = 0;</div><div class='add'>+			status = irdma_create_hmc_obj_type(dev, &amp;info);</div><div class='add'>+			if (status) {</div><div class='add'>+				ibdev_dbg(to_ibdev(dev),</div><div class='add'>+					  "ERR: create obj type %d status = %d\n",</div><div class='add'>+					  iw_hmc_obj_types[i], status);</div><div class='add'>+				break;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+		if (vers == IRDMA_GEN_1 &amp;&amp; i == IRDMA_HMC_IW_TIMER)</div><div class='add'>+			break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!status)</div><div class='add'>+		return irdma_sc_static_hmc_pages_allocated(dev-&gt;cqp, 0, dev-&gt;hmc_fn_id,</div><div class='add'>+							   true, true);</div><div class='add'>+</div><div class='add'>+	while (i) {</div><div class='add'>+		i--;</div><div class='add'>+		/* destroy the hmc objects of a given type */</div><div class='add'>+		if (dev-&gt;hmc_info-&gt;hmc_obj[iw_hmc_obj_types[i]].cnt)</div><div class='add'>+			irdma_close_hmc_objects_type(dev, iw_hmc_obj_types[i],</div><div class='add'>+						     dev-&gt;hmc_info, privileged,</div><div class='add'>+						     false);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_obj_aligned_mem - get aligned memory from device allocated memory</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @memptr: points to the memory addresses</div><div class='add'>+ * @size: size of memory needed</div><div class='add'>+ * @mask: mask for the aligned memory</div><div class='add'>+ *</div><div class='add'>+ * Get aligned memory of the requested size and</div><div class='add'>+ * update the memptr to point to the new aligned memory</div><div class='add'>+ * Return 0 if successful, otherwise return no memory error</div><div class='add'>+ */</div><div class='add'>+static int irdma_obj_aligned_mem(struct irdma_pci_f *rf,</div><div class='add'>+				 struct irdma_dma_mem *memptr, u32 size,</div><div class='add'>+				 u32 mask)</div><div class='add'>+{</div><div class='add'>+	unsigned long va, newva;</div><div class='add'>+	unsigned long extra;</div><div class='add'>+</div><div class='add'>+	va = (unsigned long)rf-&gt;obj_next.va;</div><div class='add'>+	newva = va;</div><div class='add'>+	if (mask)</div><div class='add'>+		newva = ALIGN(va, (unsigned long)mask + 1ULL);</div><div class='add'>+	extra = newva - va;</div><div class='add'>+	memptr-&gt;va = (u8 *)va + extra;</div><div class='add'>+	memptr-&gt;pa = rf-&gt;obj_next.pa + extra;</div><div class='add'>+	memptr-&gt;size = size;</div><div class='add'>+	if (((u8 *)memptr-&gt;va + size) &gt; ((u8 *)rf-&gt;obj_mem.va + rf-&gt;obj_mem.size))</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	rf-&gt;obj_next.va = (u8 *)memptr-&gt;va + size;</div><div class='add'>+	rf-&gt;obj_next.pa = memptr-&gt;pa + size;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_create_cqp - create control qp</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ *</div><div class='add'>+ * Return 0, if the cqp and all the resources associated with it</div><div class='add'>+ * are successfully created, otherwise return error</div><div class='add'>+ */</div><div class='add'>+static int irdma_create_cqp(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	u32 sqsize = IRDMA_CQP_SW_SQSIZE_2048;</div><div class='add'>+	struct irdma_dma_mem mem;</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_cqp_init_info cqp_init_info = {};</div><div class='add'>+	struct irdma_cqp *cqp = &amp;rf-&gt;cqp;</div><div class='add'>+	u16 maj_err, min_err;</div><div class='add'>+	int i, status;</div><div class='add'>+</div><div class='add'>+	cqp-&gt;cqp_requests = kcalloc(sqsize, sizeof(*cqp-&gt;cqp_requests), GFP_KERNEL);</div><div class='add'>+	if (!cqp-&gt;cqp_requests)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp-&gt;scratch_array = kcalloc(sqsize, sizeof(*cqp-&gt;scratch_array), GFP_KERNEL);</div><div class='add'>+	if (!cqp-&gt;scratch_array) {</div><div class='add'>+		status = -ENOMEM;</div><div class='add'>+		goto err_scratch;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cqp-&gt;oop_op_array = kcalloc(sqsize, sizeof(*cqp-&gt;oop_op_array),</div><div class='add'>+				    GFP_KERNEL);</div><div class='add'>+	if (!cqp-&gt;oop_op_array) {</div><div class='add'>+		status = -ENOMEM;</div><div class='add'>+		goto err_oop;</div><div class='add'>+	}</div><div class='add'>+	cqp_init_info.ooo_op_array = cqp-&gt;oop_op_array;</div><div class='add'>+	dev-&gt;cqp = &amp;cqp-&gt;sc_cqp;</div><div class='add'>+	dev-&gt;cqp-&gt;dev = dev;</div><div class='add'>+	cqp-&gt;sq.size = ALIGN(sizeof(struct irdma_cqp_sq_wqe) * sqsize,</div><div class='add'>+			     IRDMA_CQP_ALIGNMENT);</div><div class='add'>+	cqp-&gt;sq.va = dma_alloc_coherent(dev-&gt;hw-&gt;device, cqp-&gt;sq.size,</div><div class='add'>+					&amp;cqp-&gt;sq.pa, GFP_KERNEL);</div><div class='add'>+	if (!cqp-&gt;sq.va) {</div><div class='add'>+		status = -ENOMEM;</div><div class='add'>+		goto err_sq;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	status = irdma_obj_aligned_mem(rf, &amp;mem, sizeof(struct irdma_cqp_ctx),</div><div class='add'>+				       IRDMA_HOST_CTX_ALIGNMENT_M);</div><div class='add'>+	if (status)</div><div class='add'>+		goto err_ctx;</div><div class='add'>+</div><div class='add'>+	dev-&gt;cqp-&gt;host_ctx_pa = mem.pa;</div><div class='add'>+	dev-&gt;cqp-&gt;host_ctx = mem.va;</div><div class='add'>+	/* populate the cqp init info */</div><div class='add'>+	cqp_init_info.dev = dev;</div><div class='add'>+	cqp_init_info.sq_size = sqsize;</div><div class='add'>+	cqp_init_info.sq = cqp-&gt;sq.va;</div><div class='add'>+	cqp_init_info.sq_pa = cqp-&gt;sq.pa;</div><div class='add'>+	cqp_init_info.host_ctx_pa = mem.pa;</div><div class='add'>+	cqp_init_info.host_ctx = mem.va;</div><div class='add'>+	cqp_init_info.hmc_profile = rf-&gt;rsrc_profile;</div><div class='add'>+	cqp_init_info.scratch_array = cqp-&gt;scratch_array;</div><div class='add'>+	cqp_init_info.protocol_used = rf-&gt;protocol_used;</div><div class='add'>+</div><div class='add'>+	switch (rf-&gt;rdma_ver) {</div><div class='add'>+	case IRDMA_GEN_1:</div><div class='add'>+		cqp_init_info.hw_maj_ver = IRDMA_CQPHC_HW_MAJVER_GEN_1;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_GEN_2:</div><div class='add'>+		cqp_init_info.hw_maj_ver = IRDMA_CQPHC_HW_MAJVER_GEN_2;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_GEN_3:</div><div class='add'>+		cqp_init_info.hw_maj_ver = IRDMA_CQPHC_HW_MAJVER_GEN_3;</div><div class='add'>+		cqp_init_info.ts_override = 1;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+	status = irdma_sc_cqp_init(dev-&gt;cqp, &amp;cqp_init_info);</div><div class='add'>+	if (status) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "ERR: cqp init status %d\n", status);</div><div class='add'>+		goto err_ctx;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;cqp-&gt;req_lock);</div><div class='add'>+	spin_lock_init(&amp;cqp-&gt;compl_lock);</div><div class='add'>+</div><div class='add'>+	status = irdma_sc_cqp_create(dev-&gt;cqp, &amp;maj_err, &amp;min_err);</div><div class='add'>+	if (status) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "ERR: cqp create failed - status %d maj_err %d min_err %d\n",</div><div class='add'>+			  status, maj_err, min_err);</div><div class='add'>+		goto err_ctx;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;cqp-&gt;cqp_avail_reqs);</div><div class='add'>+	INIT_LIST_HEAD(&amp;cqp-&gt;cqp_pending_reqs);</div><div class='add'>+</div><div class='add'>+	/* init the waitqueue of the cqp_requests and add them to the list */</div><div class='add'>+	for (i = 0; i &lt; sqsize; i++) {</div><div class='add'>+		init_waitqueue_head(&amp;cqp-&gt;cqp_requests[i].waitq);</div><div class='add'>+		list_add_tail(&amp;cqp-&gt;cqp_requests[i].list, &amp;cqp-&gt;cqp_avail_reqs);</div><div class='add'>+	}</div><div class='add'>+	init_waitqueue_head(&amp;cqp-&gt;remove_wq);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_ctx:</div><div class='add'>+	dma_free_coherent(dev-&gt;hw-&gt;device, cqp-&gt;sq.size,</div><div class='add'>+			  cqp-&gt;sq.va, cqp-&gt;sq.pa);</div><div class='add'>+	cqp-&gt;sq.va = NULL;</div><div class='add'>+err_sq:</div><div class='add'>+	kfree(cqp-&gt;oop_op_array);</div><div class='add'>+	cqp-&gt;oop_op_array = NULL;</div><div class='add'>+err_oop:</div><div class='add'>+	kfree(cqp-&gt;scratch_array);</div><div class='add'>+	cqp-&gt;scratch_array = NULL;</div><div class='add'>+err_scratch:</div><div class='add'>+	kfree(cqp-&gt;cqp_requests);</div><div class='add'>+	cqp-&gt;cqp_requests = NULL;</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_create_ccq - create control cq</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ *</div><div class='add'>+ * Return 0, if the ccq and the resources associated with it</div><div class='add'>+ * are successfully created, otherwise return error</div><div class='add'>+ */</div><div class='add'>+static int irdma_create_ccq(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_ccq_init_info info = {};</div><div class='add'>+	struct irdma_ccq *ccq = &amp;rf-&gt;ccq;</div><div class='add'>+	int ccq_size;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	dev-&gt;ccq = &amp;ccq-&gt;sc_cq;</div><div class='add'>+	dev-&gt;ccq-&gt;dev = dev;</div><div class='add'>+	info.dev = dev;</div><div class='add'>+	ccq_size = (rf-&gt;rdma_ver &gt;= IRDMA_GEN_3) ? IW_GEN_3_CCQ_SIZE : IW_CCQ_SIZE;</div><div class='add'>+	ccq-&gt;shadow_area.size = sizeof(struct irdma_cq_shadow_area);</div><div class='add'>+	ccq-&gt;mem_cq.size = ALIGN(sizeof(struct irdma_cqe) * ccq_size,</div><div class='add'>+				 IRDMA_CQ0_ALIGNMENT);</div><div class='add'>+	ccq-&gt;mem_cq.va = dma_alloc_coherent(dev-&gt;hw-&gt;device, ccq-&gt;mem_cq.size,</div><div class='add'>+					    &amp;ccq-&gt;mem_cq.pa, GFP_KERNEL);</div><div class='add'>+	if (!ccq-&gt;mem_cq.va)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	status = irdma_obj_aligned_mem(rf, &amp;ccq-&gt;shadow_area,</div><div class='add'>+				       ccq-&gt;shadow_area.size,</div><div class='add'>+				       IRDMA_SHADOWAREA_M);</div><div class='add'>+	if (status)</div><div class='add'>+		goto exit;</div><div class='add'>+</div><div class='add'>+	ccq-&gt;sc_cq.back_cq = ccq;</div><div class='add'>+	/* populate the ccq init info */</div><div class='add'>+	info.cq_base = ccq-&gt;mem_cq.va;</div><div class='add'>+	info.cq_pa = ccq-&gt;mem_cq.pa;</div><div class='add'>+	info.num_elem = ccq_size;</div><div class='add'>+	info.shadow_area = ccq-&gt;shadow_area.va;</div><div class='add'>+	info.shadow_area_pa = ccq-&gt;shadow_area.pa;</div><div class='add'>+	info.ceqe_mask = false;</div><div class='add'>+	info.ceq_id_valid = true;</div><div class='add'>+	info.shadow_read_threshold = 16;</div><div class='add'>+	info.vsi = &amp;rf-&gt;default_vsi;</div><div class='add'>+	status = irdma_sc_ccq_init(dev-&gt;ccq, &amp;info);</div><div class='add'>+	if (!status)</div><div class='add'>+		status = irdma_sc_ccq_create(dev-&gt;ccq, 0, true, true);</div><div class='add'>+exit:</div><div class='add'>+	if (status) {</div><div class='add'>+		dma_free_coherent(dev-&gt;hw-&gt;device, ccq-&gt;mem_cq.size,</div><div class='add'>+				  ccq-&gt;mem_cq.va, ccq-&gt;mem_cq.pa);</div><div class='add'>+		ccq-&gt;mem_cq.va = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_alloc_set_mac - set up a mac address table entry</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ *</div><div class='add'>+ * Allocate a mac ip entry and add it to the hw table Return 0</div><div class='add'>+ * if successful, otherwise return error</div><div class='add'>+ */</div><div class='add'>+static int irdma_alloc_set_mac(struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	status = irdma_alloc_local_mac_entry(iwdev-&gt;rf,</div><div class='add'>+					     &amp;iwdev-&gt;mac_ip_table_idx);</div><div class='add'>+	if (!status) {</div><div class='add'>+		status = irdma_add_local_mac_entry(iwdev-&gt;rf,</div><div class='add'>+						   (const u8 *)iwdev-&gt;netdev-&gt;dev_addr,</div><div class='add'>+						   (u8)iwdev-&gt;mac_ip_table_idx);</div><div class='add'>+		if (status)</div><div class='add'>+			irdma_del_local_mac_entry(iwdev-&gt;rf,</div><div class='add'>+						  (u8)iwdev-&gt;mac_ip_table_idx);</div><div class='add'>+	}</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cfg_ceq_vector - set up the msix interrupt vector for</div><div class='add'>+ * ceq</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @iwceq: ceq associated with the vector</div><div class='add'>+ * @ceq_id: the id number of the iwceq</div><div class='add'>+ * @msix_vec: interrupt vector information</div><div class='add'>+ *</div><div class='add'>+ * Allocate interrupt resources and enable irq handling</div><div class='add'>+ * Return 0 if successful, otherwise return error</div><div class='add'>+ */</div><div class='add'>+static int irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,</div><div class='add'>+				u32 ceq_id, struct irdma_msix_vector *msix_vec)</div><div class='add'>+{</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	if (rf-&gt;msix_shared &amp;&amp; !ceq_id) {</div><div class='add'>+		snprintf(msix_vec-&gt;name, sizeof(msix_vec-&gt;name) - 1,</div><div class='add'>+			 "irdma-%s-AEQCEQ-0", dev_name(&amp;rf-&gt;pcidev-&gt;dev));</div><div class='add'>+		tasklet_setup(&amp;rf-&gt;dpc_tasklet, irdma_dpc);</div><div class='add'>+		status = request_irq(msix_vec-&gt;irq, irdma_irq_handler, 0,</div><div class='add'>+				     msix_vec-&gt;name, rf);</div><div class='add'>+	} else {</div><div class='add'>+		snprintf(msix_vec-&gt;name, sizeof(msix_vec-&gt;name) - 1,</div><div class='add'>+			 "irdma-%s-CEQ-%d",</div><div class='add'>+			 dev_name(&amp;rf-&gt;pcidev-&gt;dev), ceq_id);</div><div class='add'>+		tasklet_setup(&amp;iwceq-&gt;dpc_tasklet, irdma_ceq_dpc);</div><div class='add'>+</div><div class='add'>+		status = request_irq(msix_vec-&gt;irq, irdma_ceq_handler, 0,</div><div class='add'>+				     msix_vec-&gt;name, iwceq);</div><div class='add'>+	}</div><div class='add'>+	cpumask_clear(&amp;msix_vec-&gt;mask);</div><div class='add'>+	cpumask_set_cpu(msix_vec-&gt;cpu_affinity, &amp;msix_vec-&gt;mask);</div><div class='add'>+	irq_update_affinity_hint(msix_vec-&gt;irq, &amp;msix_vec-&gt;mask);</div><div class='add'>+	if (status) {</div><div class='add'>+		ibdev_dbg(&amp;rf-&gt;iwdev-&gt;ibdev, "ERR: ceq irq config fail\n");</div><div class='add'>+		return status;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	msix_vec-&gt;ceq_id = ceq_id;</div><div class='add'>+	if (rf-&gt;sc_dev.privileged)</div><div class='add'>+		rf-&gt;sc_dev.irq_ops-&gt;irdma_cfg_ceq(&amp;rf-&gt;sc_dev, ceq_id,</div><div class='add'>+						  msix_vec-&gt;idx, true);</div><div class='add'>+	else</div><div class='add'>+		status = irdma_vchnl_req_ceq_vec_map(&amp;rf-&gt;sc_dev, ceq_id,</div><div class='add'>+						     msix_vec-&gt;idx);</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cfg_aeq_vector - set up the msix vector for aeq</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ *</div><div class='add'>+ * Allocate interrupt resources and enable irq handling</div><div class='add'>+ * Return 0 if successful, otherwise return error</div><div class='add'>+ */</div><div class='add'>+static int irdma_cfg_aeq_vector(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_msix_vector *msix_vec = rf-&gt;iw_msixtbl;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	if (!rf-&gt;msix_shared) {</div><div class='add'>+		snprintf(msix_vec-&gt;name, sizeof(msix_vec-&gt;name) - 1,</div><div class='add'>+			 "irdma-%s-AEQ", dev_name(&amp;rf-&gt;pcidev-&gt;dev));</div><div class='add'>+		tasklet_setup(&amp;rf-&gt;dpc_tasklet, irdma_dpc);</div><div class='add'>+		ret = request_irq(msix_vec-&gt;irq, irdma_irq_handler, 0,</div><div class='add'>+				  msix_vec-&gt;name, rf);</div><div class='add'>+	}</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_dbg(&amp;rf-&gt;iwdev-&gt;ibdev, "ERR: aeq irq config fail\n");</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (rf-&gt;sc_dev.privileged)</div><div class='add'>+		rf-&gt;sc_dev.irq_ops-&gt;irdma_cfg_aeq(&amp;rf-&gt;sc_dev, msix_vec-&gt;idx,</div><div class='add'>+						  true);</div><div class='add'>+	else</div><div class='add'>+		ret = irdma_vchnl_req_aeq_vec_map(&amp;rf-&gt;sc_dev, msix_vec-&gt;idx);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_create_ceq - create completion event queue</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @iwceq: pointer to the ceq resources to be created</div><div class='add'>+ * @ceq_id: the id number of the iwceq</div><div class='add'>+ * @vsi_idx: vsi idx</div><div class='add'>+ *</div><div class='add'>+ * Return 0, if the ceq and the resources associated with it</div><div class='add'>+ * are successfully created, otherwise return error</div><div class='add'>+ */</div><div class='add'>+static int irdma_create_ceq(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,</div><div class='add'>+			    u32 ceq_id, u16 vsi_idx)</div><div class='add'>+{</div><div class='add'>+	int status;</div><div class='add'>+	struct irdma_ceq_init_info info = {};</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	u32 ceq_size;</div><div class='add'>+</div><div class='add'>+	info.ceq_id = ceq_id;</div><div class='add'>+	iwceq-&gt;rf = rf;</div><div class='add'>+	ceq_size = min(rf-&gt;sc_dev.hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_CQ].cnt,</div><div class='add'>+		       dev-&gt;hw_attrs.max_hw_ceq_size);</div><div class='add'>+	iwceq-&gt;mem.size = ALIGN(sizeof(struct irdma_ceqe) * ceq_size,</div><div class='add'>+				IRDMA_CEQ_ALIGNMENT);</div><div class='add'>+	iwceq-&gt;mem.va = dma_alloc_coherent(dev-&gt;hw-&gt;device, iwceq-&gt;mem.size,</div><div class='add'>+					   &amp;iwceq-&gt;mem.pa, GFP_KERNEL);</div><div class='add'>+	if (!iwceq-&gt;mem.va)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	info.ceq_id = ceq_id;</div><div class='add'>+	info.ceqe_base = iwceq-&gt;mem.va;</div><div class='add'>+	info.ceqe_pa = iwceq-&gt;mem.pa;</div><div class='add'>+	info.elem_cnt = ceq_size;</div><div class='add'>+	iwceq-&gt;sc_ceq.ceq_id = ceq_id;</div><div class='add'>+	info.dev = dev;</div><div class='add'>+	info.vsi_idx = vsi_idx;</div><div class='add'>+	status = irdma_sc_ceq_init(&amp;iwceq-&gt;sc_ceq, &amp;info);</div><div class='add'>+	if (!status) {</div><div class='add'>+		if (dev-&gt;ceq_valid)</div><div class='add'>+			status = irdma_cqp_ceq_cmd(&amp;rf-&gt;sc_dev, &amp;iwceq-&gt;sc_ceq,</div><div class='add'>+						   IRDMA_OP_CEQ_CREATE);</div><div class='add'>+		else</div><div class='add'>+			status = irdma_sc_cceq_create(&amp;iwceq-&gt;sc_ceq, 0);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (status) {</div><div class='add'>+		dma_free_coherent(dev-&gt;hw-&gt;device, iwceq-&gt;mem.size,</div><div class='add'>+				  iwceq-&gt;mem.va, iwceq-&gt;mem.pa);</div><div class='add'>+		iwceq-&gt;mem.va = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_setup_ceq_0 - create CEQ 0 and it's interrupt resource</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ *</div><div class='add'>+ * Allocate a list for all device completion event queues</div><div class='add'>+ * Create the ceq 0 and configure it's msix interrupt vector</div><div class='add'>+ * Return 0, if successfully set up, otherwise return error</div><div class='add'>+ */</div><div class='add'>+static int irdma_setup_ceq_0(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ceq *iwceq;</div><div class='add'>+	struct irdma_msix_vector *msix_vec;</div><div class='add'>+	u32 i;</div><div class='add'>+	int status = 0;</div><div class='add'>+	u32 num_ceqs;</div><div class='add'>+</div><div class='add'>+	num_ceqs = min(rf-&gt;msix_count, rf-&gt;sc_dev.hmc_fpm_misc.max_ceqs);</div><div class='add'>+	rf-&gt;ceqlist = kcalloc(num_ceqs, sizeof(*rf-&gt;ceqlist), GFP_KERNEL);</div><div class='add'>+	if (!rf-&gt;ceqlist) {</div><div class='add'>+		status = -ENOMEM;</div><div class='add'>+		goto exit;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	iwceq = &amp;rf-&gt;ceqlist[0];</div><div class='add'>+	status = irdma_create_ceq(rf, iwceq, 0, rf-&gt;default_vsi.vsi_idx);</div><div class='add'>+	if (status) {</div><div class='add'>+		ibdev_dbg(&amp;rf-&gt;iwdev-&gt;ibdev, "ERR: create ceq status = %d\n",</div><div class='add'>+			  status);</div><div class='add'>+		goto exit;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;iwceq-&gt;ce_lock);</div><div class='add'>+	i = rf-&gt;msix_shared ? 0 : 1;</div><div class='add'>+	msix_vec = &amp;rf-&gt;iw_msixtbl[i];</div><div class='add'>+	iwceq-&gt;irq = msix_vec-&gt;irq;</div><div class='add'>+	iwceq-&gt;msix_idx = msix_vec-&gt;idx;</div><div class='add'>+	status = irdma_cfg_ceq_vector(rf, iwceq, 0, msix_vec);</div><div class='add'>+	if (status) {</div><div class='add'>+		irdma_destroy_ceq(rf, iwceq);</div><div class='add'>+		goto exit;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_ena_intr(&amp;rf-&gt;sc_dev, msix_vec-&gt;idx);</div><div class='add'>+	rf-&gt;ceqs_count++;</div><div class='add'>+</div><div class='add'>+exit:</div><div class='add'>+	if (status &amp;&amp; !rf-&gt;ceqs_count) {</div><div class='add'>+		kfree(rf-&gt;ceqlist);</div><div class='add'>+		rf-&gt;ceqlist = NULL;</div><div class='add'>+		return status;</div><div class='add'>+	}</div><div class='add'>+	rf-&gt;sc_dev.ceq_valid = true;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_setup_ceqs - manage the device ceq's and their interrupt resources</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @vsi_idx: vsi_idx for this CEQ</div><div class='add'>+ *</div><div class='add'>+ * Allocate a list for all device completion event queues</div><div class='add'>+ * Create the ceq's and configure their msix interrupt vectors</div><div class='add'>+ * Return 0, if ceqs are successfully set up, otherwise return error</div><div class='add'>+ */</div><div class='add'>+static int irdma_setup_ceqs(struct irdma_pci_f *rf, u16 vsi_idx)</div><div class='add'>+{</div><div class='add'>+	u32 i;</div><div class='add'>+	u32 ceq_id;</div><div class='add'>+	struct irdma_ceq *iwceq;</div><div class='add'>+	struct irdma_msix_vector *msix_vec;</div><div class='add'>+	int status;</div><div class='add'>+	u32 num_ceqs;</div><div class='add'>+</div><div class='add'>+	num_ceqs = min(rf-&gt;msix_count, rf-&gt;sc_dev.hmc_fpm_misc.max_ceqs);</div><div class='add'>+	i = (rf-&gt;msix_shared) ? 1 : 2;</div><div class='add'>+	for (ceq_id = 1; i &lt; num_ceqs; i++, ceq_id++) {</div><div class='add'>+		iwceq = &amp;rf-&gt;ceqlist[ceq_id];</div><div class='add'>+		status = irdma_create_ceq(rf, iwceq, ceq_id, vsi_idx);</div><div class='add'>+		if (status) {</div><div class='add'>+			ibdev_dbg(&amp;rf-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "ERR: create ceq status = %d\n", status);</div><div class='add'>+			goto del_ceqs;</div><div class='add'>+		}</div><div class='add'>+		spin_lock_init(&amp;iwceq-&gt;ce_lock);</div><div class='add'>+		msix_vec = &amp;rf-&gt;iw_msixtbl[i];</div><div class='add'>+		iwceq-&gt;irq = msix_vec-&gt;irq;</div><div class='add'>+		iwceq-&gt;msix_idx = msix_vec-&gt;idx;</div><div class='add'>+		status = irdma_cfg_ceq_vector(rf, iwceq, ceq_id, msix_vec);</div><div class='add'>+		if (status) {</div><div class='add'>+			irdma_destroy_ceq(rf, iwceq);</div><div class='add'>+			goto del_ceqs;</div><div class='add'>+		}</div><div class='add'>+		irdma_ena_intr(&amp;rf-&gt;sc_dev, msix_vec-&gt;idx);</div><div class='add'>+		rf-&gt;ceqs_count++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+del_ceqs:</div><div class='add'>+	irdma_del_ceqs(rf);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_create_virt_aeq(struct irdma_pci_f *rf, u32 size)</div><div class='add'>+{</div><div class='add'>+	struct irdma_aeq *aeq = &amp;rf-&gt;aeq;</div><div class='add'>+	dma_addr_t *pg_arr;</div><div class='add'>+	u32 pg_cnt;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	if (rf-&gt;rdma_ver &lt; IRDMA_GEN_2)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	aeq-&gt;mem.size = sizeof(struct irdma_sc_aeqe) * size;</div><div class='add'>+	aeq-&gt;mem.va = vzalloc(aeq-&gt;mem.size);</div><div class='add'>+</div><div class='add'>+	if (!aeq-&gt;mem.va)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	pg_cnt = DIV_ROUND_UP(aeq-&gt;mem.size, PAGE_SIZE);</div><div class='add'>+	status = irdma_get_pble(rf-&gt;pble_rsrc, &amp;aeq-&gt;palloc, pg_cnt, true);</div><div class='add'>+	if (status) {</div><div class='add'>+		vfree(aeq-&gt;mem.va);</div><div class='add'>+		return status;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	pg_arr = (dma_addr_t *)aeq-&gt;palloc.level1.addr;</div><div class='add'>+	status = irdma_map_vm_page_list(&amp;rf-&gt;hw, aeq-&gt;mem.va, pg_arr, pg_cnt);</div><div class='add'>+	if (status) {</div><div class='add'>+		irdma_free_pble(rf-&gt;pble_rsrc, &amp;aeq-&gt;palloc);</div><div class='add'>+		vfree(aeq-&gt;mem.va);</div><div class='add'>+		return status;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_create_aeq - create async event queue</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ *</div><div class='add'>+ * Return 0, if the aeq and the resources associated with it</div><div class='add'>+ * are successfully created, otherwise return error</div><div class='add'>+ */</div><div class='add'>+static int irdma_create_aeq(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_aeq_init_info info = {};</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_aeq *aeq = &amp;rf-&gt;aeq;</div><div class='add'>+	struct irdma_hmc_info *hmc_info = rf-&gt;sc_dev.hmc_info;</div><div class='add'>+	u32 aeq_size;</div><div class='add'>+	u8 multiplier = (rf-&gt;protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) ? 2 : 1;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	aeq_size = multiplier * hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_QP].cnt +</div><div class='add'>+		   hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_CQ].cnt;</div><div class='add'>+	aeq_size = min(aeq_size, dev-&gt;hw_attrs.max_hw_aeq_size);</div><div class='add'>+	/* GEN_3 does not support virtual AEQ. Cap at max Kernel alloc size */</div><div class='add'>+	if (rf-&gt;rdma_ver == IRDMA_GEN_3)</div><div class='add'>+		aeq_size = min(aeq_size, (u32)((PAGE_SIZE &lt;&lt; MAX_PAGE_ORDER) /</div><div class='add'>+			       sizeof(struct irdma_sc_aeqe)));</div><div class='add'>+	aeq-&gt;mem.size = ALIGN(sizeof(struct irdma_sc_aeqe) * aeq_size,</div><div class='add'>+			      IRDMA_AEQ_ALIGNMENT);</div><div class='add'>+	aeq-&gt;mem.va = dma_alloc_coherent(dev-&gt;hw-&gt;device, aeq-&gt;mem.size,</div><div class='add'>+					 &amp;aeq-&gt;mem.pa,</div><div class='add'>+					 GFP_KERNEL | __GFP_NOWARN);</div><div class='add'>+	if (aeq-&gt;mem.va)</div><div class='add'>+		goto skip_virt_aeq;</div><div class='add'>+	else if (rf-&gt;rdma_ver == IRDMA_GEN_3)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	/* physically mapped aeq failed. setup virtual aeq */</div><div class='add'>+	status = irdma_create_virt_aeq(rf, aeq_size);</div><div class='add'>+	if (status)</div><div class='add'>+		return status;</div><div class='add'>+</div><div class='add'>+	info.virtual_map = true;</div><div class='add'>+	aeq-&gt;virtual_map = info.virtual_map;</div><div class='add'>+	info.pbl_chunk_size = 1;</div><div class='add'>+	info.first_pm_pbl_idx = aeq-&gt;palloc.level1.idx;</div><div class='add'>+</div><div class='add'>+skip_virt_aeq:</div><div class='add'>+	info.aeqe_base = aeq-&gt;mem.va;</div><div class='add'>+	info.aeq_elem_pa = aeq-&gt;mem.pa;</div><div class='add'>+	info.elem_cnt = aeq_size;</div><div class='add'>+	info.dev = dev;</div><div class='add'>+	info.msix_idx = rf-&gt;iw_msixtbl-&gt;idx;</div><div class='add'>+	status = irdma_sc_aeq_init(&amp;aeq-&gt;sc_aeq, &amp;info);</div><div class='add'>+	if (status)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	status = irdma_cqp_aeq_cmd(dev, &amp;aeq-&gt;sc_aeq, IRDMA_OP_AEQ_CREATE);</div><div class='add'>+	if (status)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	if (aeq-&gt;virtual_map) {</div><div class='add'>+		irdma_destroy_virt_aeq(rf);</div><div class='add'>+	} else {</div><div class='add'>+		dma_free_coherent(dev-&gt;hw-&gt;device, aeq-&gt;mem.size, aeq-&gt;mem.va,</div><div class='add'>+				  aeq-&gt;mem.pa);</div><div class='add'>+		aeq-&gt;mem.va = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_setup_aeq - set up the device aeq</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ *</div><div class='add'>+ * Create the aeq and configure its msix interrupt vector</div><div class='add'>+ * Return 0 if successful, otherwise return error</div><div class='add'>+ */</div><div class='add'>+static int irdma_setup_aeq(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	status = irdma_create_aeq(rf);</div><div class='add'>+	if (status)</div><div class='add'>+		return status;</div><div class='add'>+</div><div class='add'>+	status = irdma_cfg_aeq_vector(rf);</div><div class='add'>+	if (status) {</div><div class='add'>+		irdma_destroy_aeq(rf);</div><div class='add'>+		return status;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!rf-&gt;msix_shared)</div><div class='add'>+		irdma_ena_intr(dev, rf-&gt;iw_msixtbl[0].idx);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_initialize_ilq - create iwarp local queue for cm</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ *</div><div class='add'>+ * Return 0 if successful, otherwise return error</div><div class='add'>+ */</div><div class='add'>+static int irdma_initialize_ilq(struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_rsrc_info info = {};</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	info.type = IRDMA_PUDA_RSRC_TYPE_ILQ;</div><div class='add'>+	info.cq_id = 1;</div><div class='add'>+	info.qp_id = 1;</div><div class='add'>+	info.count = 1;</div><div class='add'>+	info.pd_id = 1;</div><div class='add'>+	info.abi_ver = IRDMA_ABI_VER;</div><div class='add'>+	info.sq_size = min(iwdev-&gt;rf-&gt;max_qp / 2, (u32)32768);</div><div class='add'>+	info.rq_size = info.sq_size;</div><div class='add'>+	info.buf_size = 1024;</div><div class='add'>+	info.tx_buf_cnt = 2 * info.sq_size;</div><div class='add'>+	info.receive = irdma_receive_ilq;</div><div class='add'>+	info.xmit_complete = irdma_free_sqbuf;</div><div class='add'>+	status = irdma_puda_create_rsrc(&amp;iwdev-&gt;vsi, &amp;info);</div><div class='add'>+	if (status)</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev, "ERR: ilq create fail\n");</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_initialize_ieq - create iwarp exception queue</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ *</div><div class='add'>+ * Return 0 if successful, otherwise return error</div><div class='add'>+ */</div><div class='add'>+static int irdma_initialize_ieq(struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_rsrc_info info = {};</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	info.type = IRDMA_PUDA_RSRC_TYPE_IEQ;</div><div class='add'>+	info.cq_id = 2;</div><div class='add'>+	info.qp_id = iwdev-&gt;vsi.exception_lan_q;</div><div class='add'>+	info.count = 1;</div><div class='add'>+	info.pd_id = 2;</div><div class='add'>+	info.abi_ver = IRDMA_ABI_VER;</div><div class='add'>+	info.sq_size = min(iwdev-&gt;rf-&gt;max_qp / 2, (u32)32768);</div><div class='add'>+	info.rq_size = info.sq_size;</div><div class='add'>+	info.buf_size = iwdev-&gt;vsi.mtu + IRDMA_IPV4_PAD;</div><div class='add'>+	info.tx_buf_cnt = 4096;</div><div class='add'>+	status = irdma_puda_create_rsrc(&amp;iwdev-&gt;vsi, &amp;info);</div><div class='add'>+	if (status)</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev, "ERR: ieq create fail\n");</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_reinitialize_ieq - destroy and re-create ieq</div><div class='add'>+ * @vsi: VSI structure</div><div class='add'>+ */</div><div class='add'>+void irdma_reinitialize_ieq(struct irdma_sc_vsi *vsi)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = vsi-&gt;back_vsi;</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+</div><div class='add'>+	irdma_puda_dele_rsrc(vsi, IRDMA_PUDA_RSRC_TYPE_IEQ, false);</div><div class='add'>+	if (irdma_initialize_ieq(iwdev)) {</div><div class='add'>+		iwdev-&gt;rf-&gt;reset = true;</div><div class='add'>+		rf-&gt;gen_ops.request_reset(rf);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_hmc_setup - create hmc objects for the device</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ *</div><div class='add'>+ * Set up the device private memory space for the number and size of</div><div class='add'>+ * the hmc objects and create the objects</div><div class='add'>+ * Return 0 if successful, otherwise return error</div><div class='add'>+ */</div><div class='add'>+static int irdma_hmc_setup(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	int status;</div><div class='add'>+	u32 qpcnt;</div><div class='add'>+</div><div class='add'>+	qpcnt = rsrc_limits_table[rf-&gt;limits_sel].qplimit;</div><div class='add'>+</div><div class='add'>+	rf-&gt;sd_type = IRDMA_SD_TYPE_DIRECT;</div><div class='add'>+	status = irdma_cfg_fpm_val(&amp;rf-&gt;sc_dev, qpcnt);</div><div class='add'>+	if (status)</div><div class='add'>+		return status;</div><div class='add'>+</div><div class='add'>+	status = irdma_create_hmc_objs(rf, true, rf-&gt;rdma_ver);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_del_init_mem - deallocate memory resources</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ */</div><div class='add'>+static void irdma_del_init_mem(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+</div><div class='add'>+	if (!rf-&gt;sc_dev.privileged)</div><div class='add'>+		irdma_vchnl_req_put_hmc_fcn(&amp;rf-&gt;sc_dev);</div><div class='add'>+	kfree(dev-&gt;hmc_info-&gt;sd_table.sd_entry);</div><div class='add'>+	dev-&gt;hmc_info-&gt;sd_table.sd_entry = NULL;</div><div class='add'>+	vfree(rf-&gt;mem_rsrc);</div><div class='add'>+	rf-&gt;mem_rsrc = NULL;</div><div class='add'>+	dma_free_coherent(rf-&gt;hw.device, rf-&gt;obj_mem.size, rf-&gt;obj_mem.va,</div><div class='add'>+			  rf-&gt;obj_mem.pa);</div><div class='add'>+	rf-&gt;obj_mem.va = NULL;</div><div class='add'>+	if (rf-&gt;rdma_ver != IRDMA_GEN_1) {</div><div class='add'>+		bitmap_free(rf-&gt;allocated_ws_nodes);</div><div class='add'>+		rf-&gt;allocated_ws_nodes = NULL;</div><div class='add'>+	}</div><div class='add'>+	kfree(rf-&gt;ceqlist);</div><div class='add'>+	rf-&gt;ceqlist = NULL;</div><div class='add'>+	kfree(rf-&gt;iw_msixtbl);</div><div class='add'>+	rf-&gt;iw_msixtbl = NULL;</div><div class='add'>+	kfree(rf-&gt;hmc_info_mem);</div><div class='add'>+	rf-&gt;hmc_info_mem = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_initialize_dev - initialize device</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ *</div><div class='add'>+ * Allocate memory for the hmc objects and initialize iwdev</div><div class='add'>+ * Return 0 if successful, otherwise clean up the resources</div><div class='add'>+ * and return error</div><div class='add'>+ */</div><div class='add'>+static int irdma_initialize_dev(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	int status;</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_device_init_info info = {};</div><div class='add'>+	struct irdma_dma_mem mem;</div><div class='add'>+	u32 size;</div><div class='add'>+</div><div class='add'>+	size = sizeof(struct irdma_hmc_pble_rsrc) +</div><div class='add'>+	       sizeof(struct irdma_hmc_info) +</div><div class='add'>+	       (sizeof(struct irdma_hmc_obj_info) * IRDMA_HMC_IW_MAX);</div><div class='add'>+</div><div class='add'>+	rf-&gt;hmc_info_mem = kzalloc(size, GFP_KERNEL);</div><div class='add'>+	if (!rf-&gt;hmc_info_mem)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	rf-&gt;pble_rsrc = (struct irdma_hmc_pble_rsrc *)rf-&gt;hmc_info_mem;</div><div class='add'>+	dev-&gt;hmc_info = &amp;rf-&gt;hw.hmc;</div><div class='add'>+	dev-&gt;hmc_info-&gt;hmc_obj = (struct irdma_hmc_obj_info *)</div><div class='add'>+				 (rf-&gt;pble_rsrc + 1);</div><div class='add'>+</div><div class='add'>+	status = irdma_obj_aligned_mem(rf, &amp;mem, IRDMA_QUERY_FPM_BUF_SIZE,</div><div class='add'>+				       IRDMA_FPM_QUERY_BUF_ALIGNMENT_M);</div><div class='add'>+	if (status)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	info.fpm_query_buf_pa = mem.pa;</div><div class='add'>+	info.fpm_query_buf = mem.va;</div><div class='add'>+</div><div class='add'>+	status = irdma_obj_aligned_mem(rf, &amp;mem, IRDMA_COMMIT_FPM_BUF_SIZE,</div><div class='add'>+				       IRDMA_FPM_COMMIT_BUF_ALIGNMENT_M);</div><div class='add'>+	if (status)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	info.fpm_commit_buf_pa = mem.pa;</div><div class='add'>+	info.fpm_commit_buf = mem.va;</div><div class='add'>+</div><div class='add'>+	info.bar0 = rf-&gt;hw.hw_addr;</div><div class='add'>+	info.hmc_fn_id = rf-&gt;pf_id;</div><div class='add'>+	info.protocol_used = rf-&gt;protocol_used;</div><div class='add'>+	info.hw = &amp;rf-&gt;hw;</div><div class='add'>+	status = irdma_sc_dev_init(rf-&gt;rdma_ver, &amp;rf-&gt;sc_dev, &amp;info);</div><div class='add'>+	if (status)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+error:</div><div class='add'>+	kfree(rf-&gt;hmc_info_mem);</div><div class='add'>+	rf-&gt;hmc_info_mem = NULL;</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_rt_deinit_hw - clean up the irdma device resources</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ *</div><div class='add'>+ * remove the mac ip entry and ipv4/ipv6 addresses, destroy the</div><div class='add'>+ * device queues and free the pble and the hmc objects</div><div class='add'>+ */</div><div class='add'>+void irdma_rt_deinit_hw(struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev, "INIT: state = %d\n", iwdev-&gt;init_state);</div><div class='add'>+</div><div class='add'>+	switch (iwdev-&gt;init_state) {</div><div class='add'>+	case IP_ADDR_REGISTERED:</div><div class='add'>+		if (iwdev-&gt;rf-&gt;sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)</div><div class='add'>+			irdma_del_local_mac_entry(iwdev-&gt;rf,</div><div class='add'>+						  (u8)iwdev-&gt;mac_ip_table_idx);</div><div class='add'>+		fallthrough;</div><div class='add'>+	case IEQ_CREATED:</div><div class='add'>+		if (!iwdev-&gt;roce_mode)</div><div class='add'>+			irdma_puda_dele_rsrc(&amp;iwdev-&gt;vsi, IRDMA_PUDA_RSRC_TYPE_IEQ,</div><div class='add'>+					     iwdev-&gt;rf-&gt;reset);</div><div class='add'>+		fallthrough;</div><div class='add'>+	case ILQ_CREATED:</div><div class='add'>+		if (!iwdev-&gt;roce_mode)</div><div class='add'>+			irdma_puda_dele_rsrc(&amp;iwdev-&gt;vsi,</div><div class='add'>+					     IRDMA_PUDA_RSRC_TYPE_ILQ,</div><div class='add'>+					     iwdev-&gt;rf-&gt;reset);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ibdev_warn(&amp;iwdev-&gt;ibdev, "bad init_state = %d\n", iwdev-&gt;init_state);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_cleanup_cm_core(&amp;iwdev-&gt;cm_core);</div><div class='add'>+	if (iwdev-&gt;vsi.pestat) {</div><div class='add'>+		irdma_vsi_stats_free(&amp;iwdev-&gt;vsi);</div><div class='add'>+		kfree(iwdev-&gt;vsi.pestat);</div><div class='add'>+	}</div><div class='add'>+	if (iwdev-&gt;cleanup_wq)</div><div class='add'>+		destroy_workqueue(iwdev-&gt;cleanup_wq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_setup_init_state(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	status = irdma_save_msix_info(rf);</div><div class='add'>+	if (status)</div><div class='add'>+		return status;</div><div class='add'>+</div><div class='add'>+	rf-&gt;hw.device = &amp;rf-&gt;pcidev-&gt;dev;</div><div class='add'>+	rf-&gt;obj_mem.size = ALIGN(8192, IRDMA_HW_PAGE_SIZE);</div><div class='add'>+	rf-&gt;obj_mem.va = dma_alloc_coherent(rf-&gt;hw.device, rf-&gt;obj_mem.size,</div><div class='add'>+					    &amp;rf-&gt;obj_mem.pa, GFP_KERNEL);</div><div class='add'>+	if (!rf-&gt;obj_mem.va) {</div><div class='add'>+		status = -ENOMEM;</div><div class='add'>+		goto clean_msixtbl;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rf-&gt;obj_next = rf-&gt;obj_mem;</div><div class='add'>+	status = irdma_initialize_dev(rf);</div><div class='add'>+	if (status)</div><div class='add'>+		goto clean_obj_mem;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+clean_obj_mem:</div><div class='add'>+	dma_free_coherent(rf-&gt;hw.device, rf-&gt;obj_mem.size, rf-&gt;obj_mem.va,</div><div class='add'>+			  rf-&gt;obj_mem.pa);</div><div class='add'>+	rf-&gt;obj_mem.va = NULL;</div><div class='add'>+clean_msixtbl:</div><div class='add'>+	kfree(rf-&gt;iw_msixtbl);</div><div class='add'>+	rf-&gt;iw_msixtbl = NULL;</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_used_rsrc - determine resources used internally</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ *</div><div class='add'>+ * Called at the end of open to get all internal allocations</div><div class='add'>+ */</div><div class='add'>+static void irdma_get_used_rsrc(struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	iwdev-&gt;rf-&gt;used_pds = find_first_zero_bit(iwdev-&gt;rf-&gt;allocated_pds,</div><div class='add'>+						 iwdev-&gt;rf-&gt;max_pd);</div><div class='add'>+	iwdev-&gt;rf-&gt;used_qps = find_first_zero_bit(iwdev-&gt;rf-&gt;allocated_qps,</div><div class='add'>+						 iwdev-&gt;rf-&gt;max_qp);</div><div class='add'>+	iwdev-&gt;rf-&gt;used_cqs = find_first_zero_bit(iwdev-&gt;rf-&gt;allocated_cqs,</div><div class='add'>+						  iwdev-&gt;rf-&gt;max_cq);</div><div class='add'>+	iwdev-&gt;rf-&gt;used_srqs = find_first_zero_bit(iwdev-&gt;rf-&gt;allocated_srqs,</div><div class='add'>+						   iwdev-&gt;rf-&gt;max_srq);</div><div class='add'>+	iwdev-&gt;rf-&gt;used_mrs = find_first_zero_bit(iwdev-&gt;rf-&gt;allocated_mrs,</div><div class='add'>+						 iwdev-&gt;rf-&gt;max_mr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void irdma_ctrl_deinit_hw(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	enum init_completion_state state = rf-&gt;init_state;</div><div class='add'>+</div><div class='add'>+	rf-&gt;init_state = INVALID_STATE;</div><div class='add'>+</div><div class='add'>+	switch (state) {</div><div class='add'>+	case AEQ_CREATED:</div><div class='add'>+		irdma_destroy_aeq(rf);</div><div class='add'>+		fallthrough;</div><div class='add'>+	case PBLE_CHUNK_MEM:</div><div class='add'>+		irdma_destroy_pble_prm(rf-&gt;pble_rsrc);</div><div class='add'>+		fallthrough;</div><div class='add'>+	case CEQS_CREATED:</div><div class='add'>+		irdma_del_ceqs(rf);</div><div class='add'>+		fallthrough;</div><div class='add'>+	case CEQ0_CREATED:</div><div class='add'>+		irdma_del_ceq_0(rf);</div><div class='add'>+		fallthrough;</div><div class='add'>+	case CCQ_CREATED:</div><div class='add'>+		irdma_destroy_ccq(rf);</div><div class='add'>+		fallthrough;</div><div class='add'>+	case HW_RSRC_INITIALIZED:</div><div class='add'>+	case HMC_OBJS_CREATED:</div><div class='add'>+		irdma_del_hmc_objects(&amp;rf-&gt;sc_dev, rf-&gt;sc_dev.hmc_info, true,</div><div class='add'>+				      rf-&gt;reset, rf-&gt;rdma_ver);</div><div class='add'>+		fallthrough;</div><div class='add'>+	case CQP_CREATED:</div><div class='add'>+		irdma_destroy_cqp(rf);</div><div class='add'>+		fallthrough;</div><div class='add'>+	case INITIAL_STATE:</div><div class='add'>+		irdma_del_init_mem(rf);</div><div class='add'>+		break;</div><div class='add'>+	case INVALID_STATE:</div><div class='add'>+	default:</div><div class='add'>+		ibdev_warn(&amp;rf-&gt;iwdev-&gt;ibdev, "bad init_state = %d\n", rf-&gt;init_state);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_rt_init_hw - Initializes runtime portion of HW</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ * @l2params: qos, tc, mtu info from netdev driver</div><div class='add'>+ *</div><div class='add'>+ * Create device queues ILQ, IEQ, CEQs and PBLEs. Setup irdma</div><div class='add'>+ * device resource objects.</div><div class='add'>+ */</div><div class='add'>+int irdma_rt_init_hw(struct irdma_device *iwdev,</div><div class='add'>+		     struct irdma_l2params *l2params)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_vsi_init_info vsi_info = {};</div><div class='add'>+	struct irdma_vsi_stats_info stats_info = {};</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	vsi_info.dev = dev;</div><div class='add'>+	vsi_info.back_vsi = iwdev;</div><div class='add'>+	vsi_info.params = l2params;</div><div class='add'>+	vsi_info.pf_data_vsi_num = iwdev-&gt;vsi_num;</div><div class='add'>+	vsi_info.register_qset = rf-&gt;gen_ops.register_qset;</div><div class='add'>+	vsi_info.unregister_qset = rf-&gt;gen_ops.unregister_qset;</div><div class='add'>+	vsi_info.exception_lan_q = 2;</div><div class='add'>+	irdma_sc_vsi_init(&amp;iwdev-&gt;vsi, &amp;vsi_info);</div><div class='add'>+</div><div class='add'>+	status = irdma_setup_cm_core(iwdev, rf-&gt;rdma_ver);</div><div class='add'>+	if (status)</div><div class='add'>+		return status;</div><div class='add'>+</div><div class='add'>+	stats_info.pestat = kzalloc(sizeof(*stats_info.pestat), GFP_KERNEL);</div><div class='add'>+	if (!stats_info.pestat) {</div><div class='add'>+		irdma_cleanup_cm_core(&amp;iwdev-&gt;cm_core);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+	stats_info.fcn_id = dev-&gt;hmc_fn_id;</div><div class='add'>+	status = irdma_vsi_stats_init(&amp;iwdev-&gt;vsi, &amp;stats_info);</div><div class='add'>+	if (status) {</div><div class='add'>+		irdma_cleanup_cm_core(&amp;iwdev-&gt;cm_core);</div><div class='add'>+		kfree(stats_info.pestat);</div><div class='add'>+		return status;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		if (!iwdev-&gt;roce_mode) {</div><div class='add'>+			status = irdma_initialize_ilq(iwdev);</div><div class='add'>+			if (status)</div><div class='add'>+				break;</div><div class='add'>+			iwdev-&gt;init_state = ILQ_CREATED;</div><div class='add'>+			status = irdma_initialize_ieq(iwdev);</div><div class='add'>+			if (status)</div><div class='add'>+				break;</div><div class='add'>+			iwdev-&gt;init_state = IEQ_CREATED;</div><div class='add'>+		}</div><div class='add'>+		if (iwdev-&gt;rf-&gt;sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)</div><div class='add'>+			irdma_alloc_set_mac(iwdev);</div><div class='add'>+		irdma_add_ip(iwdev);</div><div class='add'>+		iwdev-&gt;init_state = IP_ADDR_REGISTERED;</div><div class='add'>+</div><div class='add'>+		/* handles asynch cleanup tasks - disconnect CM , free qp,</div><div class='add'>+		 * free cq bufs</div><div class='add'>+		 */</div><div class='add'>+		iwdev-&gt;cleanup_wq = alloc_workqueue("irdma-cleanup-wq",</div><div class='add'>+					WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE);</div><div class='add'>+		if (!iwdev-&gt;cleanup_wq)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		irdma_get_used_rsrc(iwdev);</div><div class='add'>+		init_waitqueue_head(&amp;iwdev-&gt;suspend_wq);</div><div class='add'>+</div><div class='add'>+		return 0;</div><div class='add'>+	} while (0);</div><div class='add'>+</div><div class='add'>+	dev_err(&amp;rf-&gt;pcidev-&gt;dev, "HW runtime init FAIL status = %d last cmpl = %d\n",</div><div class='add'>+		status, iwdev-&gt;init_state);</div><div class='add'>+	irdma_rt_deinit_hw(iwdev);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ctrl_init_hw - Initializes control portion of HW</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ *</div><div class='add'>+ * Create admin queues, HMC obejcts and RF resource objects</div><div class='add'>+ */</div><div class='add'>+int irdma_ctrl_init_hw(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	int status;</div><div class='add'>+	do {</div><div class='add'>+		status = irdma_setup_init_state(rf);</div><div class='add'>+		if (status)</div><div class='add'>+			break;</div><div class='add'>+		rf-&gt;init_state = INITIAL_STATE;</div><div class='add'>+</div><div class='add'>+		status = irdma_create_cqp(rf);</div><div class='add'>+		if (status)</div><div class='add'>+			break;</div><div class='add'>+		rf-&gt;init_state = CQP_CREATED;</div><div class='add'>+</div><div class='add'>+		dev-&gt;feature_info[IRDMA_FEATURE_FW_INFO] = IRDMA_FW_VER_DEFAULT;</div><div class='add'>+		if (rf-&gt;rdma_ver != IRDMA_GEN_1) {</div><div class='add'>+			status = irdma_get_rdma_features(dev);</div><div class='add'>+			if (status)</div><div class='add'>+				break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		status = irdma_hmc_setup(rf);</div><div class='add'>+		if (status)</div><div class='add'>+			break;</div><div class='add'>+		rf-&gt;init_state = HMC_OBJS_CREATED;</div><div class='add'>+</div><div class='add'>+		status = irdma_initialize_hw_rsrc(rf);</div><div class='add'>+		if (status)</div><div class='add'>+			break;</div><div class='add'>+		rf-&gt;init_state = HW_RSRC_INITIALIZED;</div><div class='add'>+</div><div class='add'>+		status = irdma_create_ccq(rf);</div><div class='add'>+		if (status)</div><div class='add'>+			break;</div><div class='add'>+		rf-&gt;init_state = CCQ_CREATED;</div><div class='add'>+</div><div class='add'>+		status = irdma_setup_ceq_0(rf);</div><div class='add'>+		if (status)</div><div class='add'>+			break;</div><div class='add'>+		rf-&gt;init_state = CEQ0_CREATED;</div><div class='add'>+		/* Handles processing of CQP completions */</div><div class='add'>+		rf-&gt;cqp_cmpl_wq =</div><div class='add'>+			alloc_ordered_workqueue("cqp_cmpl_wq", WQ_HIGHPRI);</div><div class='add'>+		if (!rf-&gt;cqp_cmpl_wq) {</div><div class='add'>+			status = -ENOMEM;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		INIT_WORK(&amp;rf-&gt;cqp_cmpl_work, cqp_compl_worker);</div><div class='add'>+		irdma_sc_ccq_arm(dev-&gt;ccq);</div><div class='add'>+</div><div class='add'>+		status = irdma_setup_ceqs(rf, rf-&gt;iwdev ? rf-&gt;iwdev-&gt;vsi_num : 0);</div><div class='add'>+		if (status)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		rf-&gt;init_state = CEQS_CREATED;</div><div class='add'>+</div><div class='add'>+		status = irdma_hmc_init_pble(&amp;rf-&gt;sc_dev,</div><div class='add'>+					     rf-&gt;pble_rsrc);</div><div class='add'>+		if (status)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		rf-&gt;init_state = PBLE_CHUNK_MEM;</div><div class='add'>+</div><div class='add'>+		status = irdma_setup_aeq(rf);</div><div class='add'>+		if (status)</div><div class='add'>+			break;</div><div class='add'>+		rf-&gt;init_state = AEQ_CREATED;</div><div class='add'>+</div><div class='add'>+		return 0;</div><div class='add'>+	} while (0);</div><div class='add'>+</div><div class='add'>+	dev_err(&amp;rf-&gt;pcidev-&gt;dev, "IRDMA hardware initialization FAILED init_state=%d status=%d\n",</div><div class='add'>+		rf-&gt;init_state, status);</div><div class='add'>+	irdma_ctrl_deinit_hw(rf);</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_set_hw_rsrc - set hw memory resources.</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ */</div><div class='add'>+static void irdma_set_hw_rsrc(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	rf-&gt;allocated_qps = (void *)(rf-&gt;mem_rsrc +</div><div class='add'>+		   (sizeof(struct irdma_arp_entry) * rf-&gt;arp_table_size));</div><div class='add'>+	rf-&gt;allocated_cqs = &amp;rf-&gt;allocated_qps[BITS_TO_LONGS(rf-&gt;max_qp)];</div><div class='add'>+	rf-&gt;allocated_srqs = &amp;rf-&gt;allocated_cqs[BITS_TO_LONGS(rf-&gt;max_cq)];</div><div class='add'>+	rf-&gt;allocated_mrs = &amp;rf-&gt;allocated_srqs[BITS_TO_LONGS(rf-&gt;max_srq)];</div><div class='add'>+	rf-&gt;allocated_pds = &amp;rf-&gt;allocated_mrs[BITS_TO_LONGS(rf-&gt;max_mr)];</div><div class='add'>+	rf-&gt;allocated_ahs = &amp;rf-&gt;allocated_pds[BITS_TO_LONGS(rf-&gt;max_pd)];</div><div class='add'>+	rf-&gt;allocated_mcgs = &amp;rf-&gt;allocated_ahs[BITS_TO_LONGS(rf-&gt;max_ah)];</div><div class='add'>+	rf-&gt;allocated_arps = &amp;rf-&gt;allocated_mcgs[BITS_TO_LONGS(rf-&gt;max_mcg)];</div><div class='add'>+	rf-&gt;qp_table = (struct irdma_qp **)</div><div class='add'>+		(&amp;rf-&gt;allocated_arps[BITS_TO_LONGS(rf-&gt;arp_table_size)]);</div><div class='add'>+	rf-&gt;cq_table = (struct irdma_cq **)(&amp;rf-&gt;qp_table[rf-&gt;max_qp]);</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;rf-&gt;rsrc_lock);</div><div class='add'>+	spin_lock_init(&amp;rf-&gt;arp_lock);</div><div class='add'>+	spin_lock_init(&amp;rf-&gt;qptable_lock);</div><div class='add'>+	spin_lock_init(&amp;rf-&gt;cqtable_lock);</div><div class='add'>+	spin_lock_init(&amp;rf-&gt;qh_list_lock);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_calc_mem_rsrc_size - calculate memory resources size.</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ */</div><div class='add'>+static u32 irdma_calc_mem_rsrc_size(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	u32 rsrc_size;</div><div class='add'>+</div><div class='add'>+	rsrc_size = sizeof(struct irdma_arp_entry) * rf-&gt;arp_table_size;</div><div class='add'>+	rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf-&gt;max_qp);</div><div class='add'>+	rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf-&gt;max_mr);</div><div class='add'>+	rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf-&gt;max_cq);</div><div class='add'>+	rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf-&gt;max_srq);</div><div class='add'>+	rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf-&gt;max_pd);</div><div class='add'>+	rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf-&gt;arp_table_size);</div><div class='add'>+	rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf-&gt;max_ah);</div><div class='add'>+	rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf-&gt;max_mcg);</div><div class='add'>+	rsrc_size += sizeof(struct irdma_qp **) * rf-&gt;max_qp;</div><div class='add'>+	rsrc_size += sizeof(struct irdma_cq **) * rf-&gt;max_cq;</div><div class='add'>+	rsrc_size += sizeof(struct irdma_srq **) * rf-&gt;max_srq;</div><div class='add'>+</div><div class='add'>+	return rsrc_size;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_initialize_hw_rsrc - initialize hw resource tracking array</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ */</div><div class='add'>+u32 irdma_initialize_hw_rsrc(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	u32 rsrc_size;</div><div class='add'>+	u32 mrdrvbits;</div><div class='add'>+	u32 ret;</div><div class='add'>+</div><div class='add'>+	if (rf-&gt;rdma_ver != IRDMA_GEN_1) {</div><div class='add'>+		rf-&gt;allocated_ws_nodes = bitmap_zalloc(IRDMA_MAX_WS_NODES,</div><div class='add'>+						       GFP_KERNEL);</div><div class='add'>+		if (!rf-&gt;allocated_ws_nodes)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+</div><div class='add'>+		set_bit(0, rf-&gt;allocated_ws_nodes);</div><div class='add'>+		rf-&gt;max_ws_node_id = IRDMA_MAX_WS_NODES;</div><div class='add'>+	}</div><div class='add'>+	rf-&gt;max_cqe = rf-&gt;sc_dev.hw_attrs.uk_attrs.max_hw_cq_size;</div><div class='add'>+	rf-&gt;max_qp = rf-&gt;sc_dev.hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_QP].cnt;</div><div class='add'>+	rf-&gt;max_mr = rf-&gt;sc_dev.hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_MR].cnt;</div><div class='add'>+	rf-&gt;max_cq = rf-&gt;sc_dev.hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_CQ].cnt;</div><div class='add'>+	rf-&gt;max_srq = rf-&gt;sc_dev.hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_SRQ].cnt;</div><div class='add'>+	rf-&gt;max_pd = rf-&gt;sc_dev.hw_attrs.max_hw_pds;</div><div class='add'>+	rf-&gt;arp_table_size = rf-&gt;sc_dev.hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_ARP].cnt;</div><div class='add'>+	rf-&gt;max_ah = rf-&gt;sc_dev.hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_FSIAV].cnt;</div><div class='add'>+	rf-&gt;max_mcg = rf-&gt;max_qp;</div><div class='add'>+</div><div class='add'>+	rsrc_size = irdma_calc_mem_rsrc_size(rf);</div><div class='add'>+	rf-&gt;mem_rsrc = vzalloc(rsrc_size);</div><div class='add'>+	if (!rf-&gt;mem_rsrc) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto mem_rsrc_vzalloc_fail;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rf-&gt;arp_table = (struct irdma_arp_entry *)rf-&gt;mem_rsrc;</div><div class='add'>+</div><div class='add'>+	irdma_set_hw_rsrc(rf);</div><div class='add'>+</div><div class='add'>+	set_bit(0, rf-&gt;allocated_mrs);</div><div class='add'>+	set_bit(0, rf-&gt;allocated_qps);</div><div class='add'>+	set_bit(0, rf-&gt;allocated_cqs);</div><div class='add'>+	set_bit(0, rf-&gt;allocated_srqs);</div><div class='add'>+	set_bit(0, rf-&gt;allocated_pds);</div><div class='add'>+	set_bit(0, rf-&gt;allocated_arps);</div><div class='add'>+	set_bit(0, rf-&gt;allocated_ahs);</div><div class='add'>+	set_bit(0, rf-&gt;allocated_mcgs);</div><div class='add'>+	set_bit(2, rf-&gt;allocated_qps); /* qp 2 IEQ */</div><div class='add'>+	set_bit(1, rf-&gt;allocated_qps); /* qp 1 ILQ */</div><div class='add'>+	set_bit(1, rf-&gt;allocated_cqs);</div><div class='add'>+	set_bit(1, rf-&gt;allocated_pds);</div><div class='add'>+	set_bit(2, rf-&gt;allocated_cqs);</div><div class='add'>+	set_bit(2, rf-&gt;allocated_pds);</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;rf-&gt;mc_qht_list.list);</div><div class='add'>+	/* stag index mask has a minimum of 14 bits */</div><div class='add'>+	mrdrvbits = 24 - max(get_count_order(rf-&gt;max_mr), 14);</div><div class='add'>+	rf-&gt;mr_stagmask = ~(((1 &lt;&lt; mrdrvbits) - 1) &lt;&lt; (32 - mrdrvbits));</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+mem_rsrc_vzalloc_fail:</div><div class='add'>+	bitmap_free(rf-&gt;allocated_ws_nodes);</div><div class='add'>+	rf-&gt;allocated_ws_nodes = NULL;</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_ce_handler - handle cqp completions</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @cq: cq for cqp completions</div><div class='add'>+ */</div><div class='add'>+void irdma_cqp_ce_handler(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	u32 cqe_count = 0;</div><div class='add'>+	struct irdma_ccq_cqe_info info;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		memset(&amp;info, 0, sizeof(info));</div><div class='add'>+		spin_lock_irqsave(&amp;rf-&gt;cqp.compl_lock, flags);</div><div class='add'>+		ret = irdma_sc_ccq_get_cqe_info(cq, &amp;info);</div><div class='add'>+		spin_unlock_irqrestore(&amp;rf-&gt;cqp.compl_lock, flags);</div><div class='add'>+		if (ret)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		cqp_request = (struct irdma_cqp_request *)</div><div class='add'>+			      (unsigned long)info.scratch;</div><div class='add'>+		if (info.error &amp;&amp; irdma_cqp_crit_err(dev, cqp_request-&gt;info.cqp_cmd,</div><div class='add'>+						     info.maj_err_code,</div><div class='add'>+						     info.min_err_code))</div><div class='add'>+			ibdev_err(&amp;rf-&gt;iwdev-&gt;ibdev, "cqp opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n",</div><div class='add'>+				  info.op_code, info.maj_err_code, info.min_err_code);</div><div class='add'>+		if (cqp_request) {</div><div class='add'>+			cqp_request-&gt;compl_info.maj_err_code = info.maj_err_code;</div><div class='add'>+			cqp_request-&gt;compl_info.min_err_code = info.min_err_code;</div><div class='add'>+			cqp_request-&gt;compl_info.op_ret_val = info.op_ret_val;</div><div class='add'>+			cqp_request-&gt;compl_info.error = info.error;</div><div class='add'>+</div><div class='add'>+			/*</div><div class='add'>+			 * If this is deferred or pending completion, then mark</div><div class='add'>+			 * CQP request as pending to not block the CQ, but don't</div><div class='add'>+			 * release CQP request, as it is still on the OOO list.</div><div class='add'>+			 */</div><div class='add'>+			if (info.pending)</div><div class='add'>+				cqp_request-&gt;pending = true;</div><div class='add'>+			else</div><div class='add'>+				irdma_complete_cqp_request(&amp;rf-&gt;cqp,</div><div class='add'>+							   cqp_request);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		cqe_count++;</div><div class='add'>+	} while (1);</div><div class='add'>+</div><div class='add'>+	if (cqe_count) {</div><div class='add'>+		irdma_process_bh(dev);</div><div class='add'>+		irdma_sc_ccq_arm(cq);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * cqp_compl_worker - Handle cqp completions</div><div class='add'>+ * @work: Pointer to work structure</div><div class='add'>+ */</div><div class='add'>+void cqp_compl_worker(struct work_struct *work)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = container_of(work, struct irdma_pci_f,</div><div class='add'>+					      cqp_cmpl_work);</div><div class='add'>+	struct irdma_sc_cq *cq = &amp;rf-&gt;ccq.sc_cq;</div><div class='add'>+</div><div class='add'>+	irdma_cqp_ce_handler(rf, cq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_lookup_apbvt_entry - lookup hash table for an existing apbvt entry corresponding to port</div><div class='add'>+ * @cm_core: cm's core</div><div class='add'>+ * @port: port to identify apbvt entry</div><div class='add'>+ */</div><div class='add'>+static struct irdma_apbvt_entry *irdma_lookup_apbvt_entry(struct irdma_cm_core *cm_core,</div><div class='add'>+							  u16 port)</div><div class='add'>+{</div><div class='add'>+	struct irdma_apbvt_entry *entry;</div><div class='add'>+</div><div class='add'>+	hash_for_each_possible(cm_core-&gt;apbvt_hash_tbl, entry, hlist, port) {</div><div class='add'>+		if (entry-&gt;port == port) {</div><div class='add'>+			entry-&gt;use_cnt++;</div><div class='add'>+			return entry;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_next_iw_state - modify qp state</div><div class='add'>+ * @iwqp: iwarp qp to modify</div><div class='add'>+ * @state: next state for qp</div><div class='add'>+ * @del_hash: del hash</div><div class='add'>+ * @term: term message</div><div class='add'>+ * @termlen: length of term message</div><div class='add'>+ */</div><div class='add'>+void irdma_next_iw_state(struct irdma_qp *iwqp, u8 state, u8 del_hash, u8 term,</div><div class='add'>+			 u8 termlen)</div><div class='add'>+{</div><div class='add'>+	struct irdma_modify_qp_info info = {};</div><div class='add'>+</div><div class='add'>+	info.next_iwarp_state = state;</div><div class='add'>+	info.remove_hash_idx = del_hash;</div><div class='add'>+	info.cq_num_valid = true;</div><div class='add'>+	info.arp_cache_idx_valid = true;</div><div class='add'>+	info.dont_send_term = true;</div><div class='add'>+	info.dont_send_fin = true;</div><div class='add'>+	info.termlen = termlen;</div><div class='add'>+</div><div class='add'>+	if (term &amp; IRDMAQP_TERM_SEND_TERM_ONLY)</div><div class='add'>+		info.dont_send_term = false;</div><div class='add'>+	if (term &amp; IRDMAQP_TERM_SEND_FIN_ONLY)</div><div class='add'>+		info.dont_send_fin = false;</div><div class='add'>+	if (iwqp-&gt;sc_qp.term_flags &amp;&amp; state == IRDMA_QP_STATE_ERROR)</div><div class='add'>+		info.reset_tcp_conn = true;</div><div class='add'>+	iwqp-&gt;hw_iwarp_state = state;</div><div class='add'>+	irdma_hw_modify_qp(iwqp-&gt;iwdev, iwqp, &amp;info, 0);</div><div class='add'>+	iwqp-&gt;iwarp_state = info.next_iwarp_state;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_del_local_mac_entry - remove a mac entry from the hw</div><div class='add'>+ * table</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @idx: the index of the mac ip address to delete</div><div class='add'>+ */</div><div class='add'>+void irdma_del_local_mac_entry(struct irdma_pci_f *rf, u16 idx)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cqp *iwcqp = &amp;rf-&gt;cqp;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_DELETE_LOCAL_MAC_ENTRY;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.del_local_mac_entry.cqp = &amp;iwcqp-&gt;sc_cqp;</div><div class='add'>+	cqp_info-&gt;in.u.del_local_mac_entry.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	cqp_info-&gt;in.u.del_local_mac_entry.entry_idx = idx;</div><div class='add'>+	cqp_info-&gt;in.u.del_local_mac_entry.ignore_ref_count = 0;</div><div class='add'>+</div><div class='add'>+	irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(iwcqp, cqp_request);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_add_local_mac_entry - add a mac ip address entry to the</div><div class='add'>+ * hw table</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @mac_addr: pointer to mac address</div><div class='add'>+ * @idx: the index of the mac ip address to add</div><div class='add'>+ */</div><div class='add'>+int irdma_add_local_mac_entry(struct irdma_pci_f *rf, const u8 *mac_addr, u16 idx)</div><div class='add'>+{</div><div class='add'>+	struct irdma_local_mac_entry_info *info;</div><div class='add'>+	struct irdma_cqp *iwcqp = &amp;rf-&gt;cqp;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	info = &amp;cqp_info-&gt;in.u.add_local_mac_entry.info;</div><div class='add'>+	ether_addr_copy(info-&gt;mac_addr, mac_addr);</div><div class='add'>+	info-&gt;entry_idx = idx;</div><div class='add'>+	cqp_info-&gt;in.u.add_local_mac_entry.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_ADD_LOCAL_MAC_ENTRY;</div><div class='add'>+	cqp_info-&gt;in.u.add_local_mac_entry.cqp = &amp;iwcqp-&gt;sc_cqp;</div><div class='add'>+	cqp_info-&gt;in.u.add_local_mac_entry.scratch = (uintptr_t)cqp_request;</div><div class='add'>+</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(iwcqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_alloc_local_mac_entry - allocate a mac entry</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @mac_tbl_idx: the index of the new mac address</div><div class='add'>+ *</div><div class='add'>+ * Allocate a mac address entry and update the mac_tbl_idx</div><div class='add'>+ * to hold the index of the newly created mac address</div><div class='add'>+ * Return 0 if successful, otherwise return error</div><div class='add'>+ */</div><div class='add'>+int irdma_alloc_local_mac_entry(struct irdma_pci_f *rf, u16 *mac_tbl_idx)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cqp *iwcqp = &amp;rf-&gt;cqp;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	int status = 0;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.alloc_local_mac_entry.cqp = &amp;iwcqp-&gt;sc_cqp;</div><div class='add'>+	cqp_info-&gt;in.u.alloc_local_mac_entry.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	if (!status)</div><div class='add'>+		*mac_tbl_idx = (u16)cqp_request-&gt;compl_info.op_ret_val;</div><div class='add'>+</div><div class='add'>+	irdma_put_cqp_request(iwcqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_manage_apbvt_cmd - send cqp command manage apbvt</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ * @accel_local_port: port for apbvt</div><div class='add'>+ * @add_port: add ordelete port</div><div class='add'>+ */</div><div class='add'>+static int irdma_cqp_manage_apbvt_cmd(struct irdma_device *iwdev,</div><div class='add'>+				      u16 accel_local_port, bool add_port)</div><div class='add'>+{</div><div class='add'>+	struct irdma_apbvt_info *info;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;iwdev-&gt;rf-&gt;cqp, add_port);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	info = &amp;cqp_info-&gt;in.u.manage_apbvt_entry.info;</div><div class='add'>+	info-&gt;add = add_port;</div><div class='add'>+	info-&gt;port = accel_local_port;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_MANAGE_APBVT_ENTRY;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.manage_apbvt_entry.cqp = &amp;iwdev-&gt;rf-&gt;cqp.sc_cqp;</div><div class='add'>+	cqp_info-&gt;in.u.manage_apbvt_entry.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev, "DEV: %s: port=0x%04x\n",</div><div class='add'>+		  (!add_port) ? "DELETE" : "ADD", accel_local_port);</div><div class='add'>+</div><div class='add'>+	status = irdma_handle_cqp_op(iwdev-&gt;rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;iwdev-&gt;rf-&gt;cqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_add_apbvt - add tcp port to HW apbvt table</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ * @port: port for apbvt</div><div class='add'>+ */</div><div class='add'>+struct irdma_apbvt_entry *irdma_add_apbvt(struct irdma_device *iwdev, u16 port)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_core *cm_core = &amp;iwdev-&gt;cm_core;</div><div class='add'>+	struct irdma_apbvt_entry *entry;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cm_core-&gt;apbvt_lock, flags);</div><div class='add'>+	entry = irdma_lookup_apbvt_entry(cm_core, port);</div><div class='add'>+	if (entry) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;cm_core-&gt;apbvt_lock, flags);</div><div class='add'>+		return entry;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	entry = kzalloc(sizeof(*entry), GFP_ATOMIC);</div><div class='add'>+	if (!entry) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;cm_core-&gt;apbvt_lock, flags);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	entry-&gt;port = port;</div><div class='add'>+	entry-&gt;use_cnt = 1;</div><div class='add'>+	hash_add(cm_core-&gt;apbvt_hash_tbl, &amp;entry-&gt;hlist, entry-&gt;port);</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_core-&gt;apbvt_lock, flags);</div><div class='add'>+</div><div class='add'>+	if (irdma_cqp_manage_apbvt_cmd(iwdev, port, true)) {</div><div class='add'>+		kfree(entry);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return entry;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_del_apbvt - delete tcp port from HW apbvt table</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ * @entry: apbvt entry object</div><div class='add'>+ */</div><div class='add'>+void irdma_del_apbvt(struct irdma_device *iwdev,</div><div class='add'>+		     struct irdma_apbvt_entry *entry)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_core *cm_core = &amp;iwdev-&gt;cm_core;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cm_core-&gt;apbvt_lock, flags);</div><div class='add'>+	if (--entry-&gt;use_cnt) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;cm_core-&gt;apbvt_lock, flags);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hash_del(&amp;entry-&gt;hlist);</div><div class='add'>+	/* apbvt_lock is held across CQP delete APBVT OP (non-waiting) to</div><div class='add'>+	 * protect against race where add APBVT CQP can race ahead of the delete</div><div class='add'>+	 * APBVT for same port.</div><div class='add'>+	 */</div><div class='add'>+	irdma_cqp_manage_apbvt_cmd(iwdev, entry-&gt;port, false);</div><div class='add'>+	kfree(entry);</div><div class='add'>+	spin_unlock_irqrestore(&amp;cm_core-&gt;apbvt_lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_manage_arp_cache - manage hw arp cache</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @mac_addr: mac address ptr</div><div class='add'>+ * @ip_addr: ip addr for arp cache</div><div class='add'>+ * @ipv4: flag inicating IPv4</div><div class='add'>+ * @action: add, delete or modify</div><div class='add'>+ */</div><div class='add'>+void irdma_manage_arp_cache(struct irdma_pci_f *rf,</div><div class='add'>+			    const unsigned char *mac_addr,</div><div class='add'>+			    u32 *ip_addr, bool ipv4, u32 action)</div><div class='add'>+{</div><div class='add'>+	struct irdma_add_arp_cache_entry_info *info;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	int arp_index;</div><div class='add'>+</div><div class='add'>+	arp_index = irdma_arp_table(rf, ip_addr, ipv4, mac_addr, action);</div><div class='add'>+	if (arp_index == -1)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, false);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	if (action == IRDMA_ARP_ADD) {</div><div class='add'>+		cqp_info-&gt;cqp_cmd = IRDMA_OP_ADD_ARP_CACHE_ENTRY;</div><div class='add'>+		info = &amp;cqp_info-&gt;in.u.add_arp_cache_entry.info;</div><div class='add'>+		info-&gt;arp_index = (u16)arp_index;</div><div class='add'>+		info-&gt;permanent = true;</div><div class='add'>+		ether_addr_copy(info-&gt;mac_addr, mac_addr);</div><div class='add'>+		cqp_info-&gt;in.u.add_arp_cache_entry.scratch =</div><div class='add'>+			(uintptr_t)cqp_request;</div><div class='add'>+		cqp_info-&gt;in.u.add_arp_cache_entry.cqp = &amp;rf-&gt;cqp.sc_cqp;</div><div class='add'>+	} else {</div><div class='add'>+		cqp_info-&gt;cqp_cmd = IRDMA_OP_DELETE_ARP_CACHE_ENTRY;</div><div class='add'>+		cqp_info-&gt;in.u.del_arp_cache_entry.scratch =</div><div class='add'>+			(uintptr_t)cqp_request;</div><div class='add'>+		cqp_info-&gt;in.u.del_arp_cache_entry.cqp = &amp;rf-&gt;cqp.sc_cqp;</div><div class='add'>+		cqp_info-&gt;in.u.del_arp_cache_entry.arp_index = arp_index;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_send_syn_cqp_callback - do syn/ack after qhash</div><div class='add'>+ * @cqp_request: qhash cqp completion</div><div class='add'>+ */</div><div class='add'>+static void irdma_send_syn_cqp_callback(struct irdma_cqp_request *cqp_request)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_node *cm_node = cqp_request-&gt;param;</div><div class='add'>+</div><div class='add'>+	irdma_send_syn(cm_node, 1);</div><div class='add'>+	irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_manage_qhash - add or modify qhash</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ * @cminfo: cm info for qhash</div><div class='add'>+ * @etype: type (syn or quad)</div><div class='add'>+ * @mtype: type of qhash</div><div class='add'>+ * @cmnode: cmnode associated with connection</div><div class='add'>+ * @wait: wait for completion</div><div class='add'>+ */</div><div class='add'>+int irdma_manage_qhash(struct irdma_device *iwdev, struct irdma_cm_info *cminfo,</div><div class='add'>+		       enum irdma_quad_entry_type etype,</div><div class='add'>+		       enum irdma_quad_hash_manage_type mtype, void *cmnode,</div><div class='add'>+		       bool wait)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qhash_table_info *info;</div><div class='add'>+	struct irdma_cqp *iwcqp = &amp;iwdev-&gt;rf-&gt;cqp;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	struct irdma_cm_node *cm_node = cmnode;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	info = &amp;cqp_info-&gt;in.u.manage_qhash_table_entry.info;</div><div class='add'>+	info-&gt;vsi = &amp;iwdev-&gt;vsi;</div><div class='add'>+	info-&gt;manage = mtype;</div><div class='add'>+	info-&gt;entry_type = etype;</div><div class='add'>+	if (cminfo-&gt;vlan_id &lt; VLAN_N_VID) {</div><div class='add'>+		info-&gt;vlan_valid = true;</div><div class='add'>+		info-&gt;vlan_id = cminfo-&gt;vlan_id;</div><div class='add'>+	} else {</div><div class='add'>+		info-&gt;vlan_valid = false;</div><div class='add'>+	}</div><div class='add'>+	info-&gt;ipv4_valid = cminfo-&gt;ipv4;</div><div class='add'>+	info-&gt;user_pri = cminfo-&gt;user_pri;</div><div class='add'>+	ether_addr_copy(info-&gt;mac_addr, iwdev-&gt;netdev-&gt;dev_addr);</div><div class='add'>+	info-&gt;qp_num = cminfo-&gt;qh_qpid;</div><div class='add'>+	info-&gt;dest_port = cminfo-&gt;loc_port;</div><div class='add'>+	info-&gt;dest_ip[0] = cminfo-&gt;loc_addr[0];</div><div class='add'>+	info-&gt;dest_ip[1] = cminfo-&gt;loc_addr[1];</div><div class='add'>+	info-&gt;dest_ip[2] = cminfo-&gt;loc_addr[2];</div><div class='add'>+	info-&gt;dest_ip[3] = cminfo-&gt;loc_addr[3];</div><div class='add'>+	if (etype == IRDMA_QHASH_TYPE_TCP_ESTABLISHED ||</div><div class='add'>+	    etype == IRDMA_QHASH_TYPE_UDP_UNICAST ||</div><div class='add'>+	    etype == IRDMA_QHASH_TYPE_UDP_MCAST ||</div><div class='add'>+	    etype == IRDMA_QHASH_TYPE_ROCE_MCAST ||</div><div class='add'>+	    etype == IRDMA_QHASH_TYPE_ROCEV2_HW) {</div><div class='add'>+		info-&gt;src_port = cminfo-&gt;rem_port;</div><div class='add'>+		info-&gt;src_ip[0] = cminfo-&gt;rem_addr[0];</div><div class='add'>+		info-&gt;src_ip[1] = cminfo-&gt;rem_addr[1];</div><div class='add'>+		info-&gt;src_ip[2] = cminfo-&gt;rem_addr[2];</div><div class='add'>+		info-&gt;src_ip[3] = cminfo-&gt;rem_addr[3];</div><div class='add'>+	}</div><div class='add'>+	if (cmnode) {</div><div class='add'>+		cqp_request-&gt;callback_fcn = irdma_send_syn_cqp_callback;</div><div class='add'>+		cqp_request-&gt;param = cmnode;</div><div class='add'>+		if (!wait)</div><div class='add'>+			refcount_inc(&amp;cm_node-&gt;refcnt);</div><div class='add'>+	}</div><div class='add'>+	if (info-&gt;ipv4_valid)</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: %s caller: %pS loc_port=0x%04x rem_port=0x%04x loc_addr=%pI4 rem_addr=%pI4 mac=%pM, vlan_id=%d cm_node=%p\n",</div><div class='add'>+			  (!mtype) ? "DELETE" : "ADD",</div><div class='add'>+			  __builtin_return_address(0), info-&gt;dest_port,</div><div class='add'>+			  info-&gt;src_port, info-&gt;dest_ip, info-&gt;src_ip,</div><div class='add'>+			  info-&gt;mac_addr, cminfo-&gt;vlan_id,</div><div class='add'>+			  cmnode ? cmnode : NULL);</div><div class='add'>+	else</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+			  "CM: %s caller: %pS loc_port=0x%04x rem_port=0x%04x loc_addr=%pI6 rem_addr=%pI6 mac=%pM, vlan_id=%d cm_node=%p\n",</div><div class='add'>+			  (!mtype) ? "DELETE" : "ADD",</div><div class='add'>+			  __builtin_return_address(0), info-&gt;dest_port,</div><div class='add'>+			  info-&gt;src_port, info-&gt;dest_ip, info-&gt;src_ip,</div><div class='add'>+			  info-&gt;mac_addr, cminfo-&gt;vlan_id,</div><div class='add'>+			  cmnode ? cmnode : NULL);</div><div class='add'>+</div><div class='add'>+	cqp_info-&gt;in.u.manage_qhash_table_entry.cqp = &amp;iwdev-&gt;rf-&gt;cqp.sc_cqp;</div><div class='add'>+	cqp_info-&gt;in.u.manage_qhash_table_entry.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	status = irdma_handle_cqp_op(iwdev-&gt;rf, cqp_request);</div><div class='add'>+	if (status &amp;&amp; cm_node &amp;&amp; !wait)</div><div class='add'>+		irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+</div><div class='add'>+	irdma_put_cqp_request(iwcqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_hw_flush_wqes_callback - Check return code after flush</div><div class='add'>+ * @cqp_request: qhash cqp completion</div><div class='add'>+ */</div><div class='add'>+static void irdma_hw_flush_wqes_callback(struct irdma_cqp_request *cqp_request)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp_flush_info *hw_info;</div><div class='add'>+	struct irdma_sc_qp *qp;</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	hw_info = &amp;cqp_info-&gt;in.u.qp_flush_wqes.info;</div><div class='add'>+	qp = cqp_info-&gt;in.u.qp_flush_wqes.qp;</div><div class='add'>+	iwqp = qp-&gt;qp_uk.back_qp;</div><div class='add'>+</div><div class='add'>+	if (cqp_request-&gt;compl_info.maj_err_code)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	if (hw_info-&gt;rq &amp;&amp;</div><div class='add'>+	    (cqp_request-&gt;compl_info.min_err_code == IRDMA_CQP_COMPL_SQ_WQE_FLUSHED ||</div><div class='add'>+	     cqp_request-&gt;compl_info.min_err_code == 0)) {</div><div class='add'>+		/* RQ WQE flush was requested but did not happen */</div><div class='add'>+		qp-&gt;qp_uk.rq_flush_complete = true;</div><div class='add'>+	}</div><div class='add'>+	if (hw_info-&gt;sq &amp;&amp;</div><div class='add'>+	    (cqp_request-&gt;compl_info.min_err_code == IRDMA_CQP_COMPL_RQ_WQE_FLUSHED ||</div><div class='add'>+	     cqp_request-&gt;compl_info.min_err_code == 0)) {</div><div class='add'>+		if (IRDMA_RING_MORE_WORK(qp-&gt;qp_uk.sq_ring)) {</div><div class='add'>+			ibdev_err(&amp;iwqp-&gt;iwdev-&gt;ibdev, "Flush QP[%d] failed, SQ has more work",</div><div class='add'>+				  qp-&gt;qp_uk.qp_id);</div><div class='add'>+			irdma_ib_qp_event(iwqp, IRDMA_QP_EVENT_CATASTROPHIC);</div><div class='add'>+		}</div><div class='add'>+		qp-&gt;qp_uk.sq_flush_complete = true;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_hw_flush_wqes - flush qp's wqe</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @qp: hardware control qp</div><div class='add'>+ * @info: info for flush</div><div class='add'>+ * @wait: flag wait for completion</div><div class='add'>+ */</div><div class='add'>+int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,</div><div class='add'>+			struct irdma_qp_flush_info *info, bool wait)</div><div class='add'>+{</div><div class='add'>+	int status;</div><div class='add'>+	struct irdma_qp_flush_info *hw_info;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	struct irdma_qp *iwqp = qp-&gt;qp_uk.back_qp;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, wait);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	if (!wait)</div><div class='add'>+		cqp_request-&gt;callback_fcn = irdma_hw_flush_wqes_callback;</div><div class='add'>+	hw_info = &amp;cqp_request-&gt;info.in.u.qp_flush_wqes.info;</div><div class='add'>+	memcpy(hw_info, info, sizeof(*hw_info));</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_QP_FLUSH_WQES;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.qp_flush_wqes.qp = qp;</div><div class='add'>+	cqp_info-&gt;in.u.qp_flush_wqes.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	if (status) {</div><div class='add'>+		qp-&gt;qp_uk.sq_flush_complete = true;</div><div class='add'>+		qp-&gt;qp_uk.rq_flush_complete = true;</div><div class='add'>+		irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+		return status;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!wait || cqp_request-&gt;compl_info.maj_err_code)</div><div class='add'>+		goto put_cqp;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;rq) {</div><div class='add'>+		if (cqp_request-&gt;compl_info.min_err_code == IRDMA_CQP_COMPL_SQ_WQE_FLUSHED ||</div><div class='add'>+		    cqp_request-&gt;compl_info.min_err_code == 0) {</div><div class='add'>+			/* RQ WQE flush was requested but did not happen */</div><div class='add'>+			qp-&gt;qp_uk.rq_flush_complete = true;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	if (info-&gt;sq) {</div><div class='add'>+		if (cqp_request-&gt;compl_info.min_err_code == IRDMA_CQP_COMPL_RQ_WQE_FLUSHED ||</div><div class='add'>+		    cqp_request-&gt;compl_info.min_err_code == 0) {</div><div class='add'>+			/*</div><div class='add'>+			 * Handling case where WQE is posted to empty SQ when</div><div class='add'>+			 * flush has not completed</div><div class='add'>+			 */</div><div class='add'>+			if (IRDMA_RING_MORE_WORK(qp-&gt;qp_uk.sq_ring)) {</div><div class='add'>+				struct irdma_cqp_request *new_req;</div><div class='add'>+</div><div class='add'>+				if (!qp-&gt;qp_uk.sq_flush_complete)</div><div class='add'>+					goto put_cqp;</div><div class='add'>+				qp-&gt;qp_uk.sq_flush_complete = false;</div><div class='add'>+				qp-&gt;flush_sq = false;</div><div class='add'>+</div><div class='add'>+				info-&gt;rq = false;</div><div class='add'>+				info-&gt;sq = true;</div><div class='add'>+				new_req = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, true);</div><div class='add'>+				if (!new_req) {</div><div class='add'>+					status = -ENOMEM;</div><div class='add'>+					goto put_cqp;</div><div class='add'>+				}</div><div class='add'>+				cqp_info = &amp;new_req-&gt;info;</div><div class='add'>+				hw_info = &amp;new_req-&gt;info.in.u.qp_flush_wqes.info;</div><div class='add'>+				memcpy(hw_info, info, sizeof(*hw_info));</div><div class='add'>+				cqp_info-&gt;cqp_cmd = IRDMA_OP_QP_FLUSH_WQES;</div><div class='add'>+				cqp_info-&gt;post_sq = 1;</div><div class='add'>+				cqp_info-&gt;in.u.qp_flush_wqes.qp = qp;</div><div class='add'>+				cqp_info-&gt;in.u.qp_flush_wqes.scratch = (uintptr_t)new_req;</div><div class='add'>+</div><div class='add'>+				status = irdma_handle_cqp_op(rf, new_req);</div><div class='add'>+				if (new_req-&gt;compl_info.maj_err_code ||</div><div class='add'>+				    new_req-&gt;compl_info.min_err_code != IRDMA_CQP_COMPL_SQ_WQE_FLUSHED ||</div><div class='add'>+				    status) {</div><div class='add'>+					ibdev_err(&amp;iwqp-&gt;iwdev-&gt;ibdev, "fatal QP event: SQ in error but not flushed, qp: %d",</div><div class='add'>+						  iwqp-&gt;ibqp.qp_num);</div><div class='add'>+					qp-&gt;qp_uk.sq_flush_complete = false;</div><div class='add'>+					irdma_ib_qp_event(iwqp, IRDMA_QP_EVENT_CATASTROPHIC);</div><div class='add'>+				}</div><div class='add'>+				irdma_put_cqp_request(&amp;rf-&gt;cqp, new_req);</div><div class='add'>+			} else {</div><div class='add'>+				/* SQ WQE flush was requested but did not happen */</div><div class='add'>+				qp-&gt;qp_uk.sq_flush_complete = true;</div><div class='add'>+			}</div><div class='add'>+		} else {</div><div class='add'>+			if (!IRDMA_RING_MORE_WORK(qp-&gt;qp_uk.sq_ring))</div><div class='add'>+				qp-&gt;qp_uk.sq_flush_complete = true;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;rf-&gt;iwdev-&gt;ibdev,</div><div class='add'>+		  "VERBS: qp_id=%d qp_type=%d qpstate=%d ibqpstate=%d last_aeq=%d hw_iw_state=%d maj_err_code=%d min_err_code=%d\n",</div><div class='add'>+		  iwqp-&gt;ibqp.qp_num, rf-&gt;protocol_used, iwqp-&gt;iwarp_state,</div><div class='add'>+		  iwqp-&gt;ibqp_state, iwqp-&gt;last_aeq, iwqp-&gt;hw_iwarp_state,</div><div class='add'>+		  cqp_request-&gt;compl_info.maj_err_code,</div><div class='add'>+		  cqp_request-&gt;compl_info.min_err_code);</div><div class='add'>+put_cqp:</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_gen_ae - generate AE</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @qp: qp associated with AE</div><div class='add'>+ * @info: info for ae</div><div class='add'>+ * @wait: wait for completion</div><div class='add'>+ */</div><div class='add'>+void irdma_gen_ae(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,</div><div class='add'>+		  struct irdma_gen_ae_info *info, bool wait)</div><div class='add'>+{</div><div class='add'>+	struct irdma_gen_ae_info *ae_info;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, wait);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	ae_info = &amp;cqp_request-&gt;info.in.u.gen_ae.info;</div><div class='add'>+	memcpy(ae_info, info, sizeof(*ae_info));</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_GEN_AE;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.gen_ae.qp = qp;</div><div class='add'>+	cqp_info-&gt;in.u.gen_ae.scratch = (uintptr_t)cqp_request;</div><div class='add'>+</div><div class='add'>+	irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void irdma_flush_wqes(struct irdma_qp *iwqp, u32 flush_mask)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp_flush_info info = {};</div><div class='add'>+	struct irdma_pci_f *rf = iwqp-&gt;iwdev-&gt;rf;</div><div class='add'>+	u8 flush_code = iwqp-&gt;sc_qp.flush_code;</div><div class='add'>+</div><div class='add'>+	if ((!(flush_mask &amp; IRDMA_FLUSH_SQ) &amp;&amp;</div><div class='add'>+	     !(flush_mask &amp; IRDMA_FLUSH_RQ)) ||</div><div class='add'>+	    ((flush_mask &amp; IRDMA_REFLUSH) &amp;&amp; rf-&gt;rdma_ver &gt;= IRDMA_GEN_3))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	/* Set flush info fields*/</div><div class='add'>+	info.sq = flush_mask &amp; IRDMA_FLUSH_SQ;</div><div class='add'>+	info.rq = flush_mask &amp; IRDMA_FLUSH_RQ;</div><div class='add'>+</div><div class='add'>+	/* Generate userflush errors in CQE */</div><div class='add'>+	info.sq_major_code = IRDMA_FLUSH_MAJOR_ERR;</div><div class='add'>+	info.sq_minor_code = FLUSH_GENERAL_ERR;</div><div class='add'>+	info.rq_major_code = IRDMA_FLUSH_MAJOR_ERR;</div><div class='add'>+	info.rq_minor_code = FLUSH_GENERAL_ERR;</div><div class='add'>+	info.userflushcode = true;</div><div class='add'>+	info.err_sq_idx_valid = iwqp-&gt;sc_qp.err_sq_idx_valid;</div><div class='add'>+	info.err_sq_idx = iwqp-&gt;sc_qp.err_sq_idx;</div><div class='add'>+	info.err_rq_idx_valid = iwqp-&gt;sc_qp.err_rq_idx_valid;</div><div class='add'>+	info.err_rq_idx = iwqp-&gt;sc_qp.err_rq_idx;</div><div class='add'>+</div><div class='add'>+	if (flush_mask &amp; IRDMA_REFLUSH) {</div><div class='add'>+		if (info.sq)</div><div class='add'>+			iwqp-&gt;sc_qp.flush_sq = false;</div><div class='add'>+		if (info.rq)</div><div class='add'>+			iwqp-&gt;sc_qp.flush_rq = false;</div><div class='add'>+	} else {</div><div class='add'>+		if (flush_code) {</div><div class='add'>+			if (info.sq &amp;&amp; iwqp-&gt;sc_qp.sq_flush_code)</div><div class='add'>+				info.sq_minor_code = flush_code;</div><div class='add'>+			if (info.rq &amp;&amp; iwqp-&gt;sc_qp.rq_flush_code)</div><div class='add'>+				info.rq_minor_code = flush_code;</div><div class='add'>+		}</div><div class='add'>+		if (!iwqp-&gt;user_mode)</div><div class='add'>+			queue_delayed_work(iwqp-&gt;iwdev-&gt;cleanup_wq,</div><div class='add'>+					   &amp;iwqp-&gt;dwork_flush,</div><div class='add'>+					   msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Issue flush */</div><div class='add'>+	(void)irdma_hw_flush_wqes(rf, &amp;iwqp-&gt;sc_qp, &amp;info,</div><div class='add'>+				  flush_mask &amp; IRDMA_FLUSH_WAIT);</div><div class='add'>+	iwqp-&gt;flush_issued = true;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/i40iw_hw.c b/drivers/infiniband/hw/irdma/i40iw_hw.c<br/>new file mode 100644<br/>index 000000000000..60c1f2b1811d<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/i40iw_hw.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/i40iw_hw.c</a></div><div class='hunk'>@@ -0,0 +1,261 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#include "osdep.h"</div><div class='add'>+#include "type.h"</div><div class='add'>+#include "i40iw_hw.h"</div><div class='add'>+#include "protos.h"</div><div class='add'>+</div><div class='add'>+static u32 i40iw_regs[IRDMA_MAX_REGS] = {</div><div class='add'>+	I40E_PFPE_CQPTAIL,</div><div class='add'>+	I40E_PFPE_CQPDB,</div><div class='add'>+	I40E_PFPE_CCQPSTATUS,</div><div class='add'>+	I40E_PFPE_CCQPHIGH,</div><div class='add'>+	I40E_PFPE_CCQPLOW,</div><div class='add'>+	I40E_PFPE_CQARM,</div><div class='add'>+	I40E_PFPE_CQACK,</div><div class='add'>+	I40E_PFPE_AEQALLOC,</div><div class='add'>+	I40E_PFPE_CQPERRCODES,</div><div class='add'>+	I40E_PFPE_WQEALLOC,</div><div class='add'>+	I40E_PFINT_DYN_CTLN(0),</div><div class='add'>+	I40IW_DB_ADDR_OFFSET,</div><div class='add'>+</div><div class='add'>+	I40E_GLPCI_LBARCTRL,</div><div class='add'>+	I40E_GLPE_CPUSTATUS0,</div><div class='add'>+	I40E_GLPE_CPUSTATUS1,</div><div class='add'>+	I40E_GLPE_CPUSTATUS2,</div><div class='add'>+	I40E_PFINT_AEQCTL,</div><div class='add'>+	I40E_PFINT_CEQCTL(0),</div><div class='add'>+	I40E_VSIQF_CTL(0),</div><div class='add'>+	I40E_PFHMC_PDINV,</div><div class='add'>+	I40E_GLHMC_VFPDINV(0),</div><div class='add'>+	I40E_GLPE_CRITERR,</div><div class='add'>+	0xffffffff      /* PFINT_RATEN not used in FPK */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static u32 i40iw_stat_offsets[] = {</div><div class='add'>+	I40E_GLPES_PFIP4RXDISCARD(0),</div><div class='add'>+	I40E_GLPES_PFIP4RXTRUNC(0),</div><div class='add'>+	I40E_GLPES_PFIP4TXNOROUTE(0),</div><div class='add'>+	I40E_GLPES_PFIP6RXDISCARD(0),</div><div class='add'>+	I40E_GLPES_PFIP6RXTRUNC(0),</div><div class='add'>+	I40E_GLPES_PFIP6TXNOROUTE(0),</div><div class='add'>+	I40E_GLPES_PFTCPRTXSEG(0),</div><div class='add'>+	I40E_GLPES_PFTCPRXOPTERR(0),</div><div class='add'>+	I40E_GLPES_PFTCPRXPROTOERR(0),</div><div class='add'>+	I40E_GLPES_PFRXVLANERR(0),</div><div class='add'>+</div><div class='add'>+	I40E_GLPES_PFIP4RXOCTSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP4RXPKTSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP4RXFRAGSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP4RXMCPKTSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP4TXOCTSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP4TXPKTSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP4TXFRAGSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP4TXMCPKTSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP6RXOCTSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP6RXPKTSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP6RXFRAGSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP6RXMCPKTSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP6TXOCTSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP6TXPKTSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP6TXFRAGSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP6TXMCPKTSLO(0),</div><div class='add'>+	I40E_GLPES_PFTCPRXSEGSLO(0),</div><div class='add'>+	I40E_GLPES_PFTCPTXSEGLO(0),</div><div class='add'>+	I40E_GLPES_PFRDMARXRDSLO(0),</div><div class='add'>+	I40E_GLPES_PFRDMARXSNDSLO(0),</div><div class='add'>+	I40E_GLPES_PFRDMARXWRSLO(0),</div><div class='add'>+	I40E_GLPES_PFRDMATXRDSLO(0),</div><div class='add'>+	I40E_GLPES_PFRDMATXSNDSLO(0),</div><div class='add'>+	I40E_GLPES_PFRDMATXWRSLO(0),</div><div class='add'>+	I40E_GLPES_PFRDMAVBNDLO(0),</div><div class='add'>+	I40E_GLPES_PFRDMAVINVLO(0),</div><div class='add'>+	I40E_GLPES_PFIP4RXMCOCTSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP4TXMCOCTSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP6RXMCOCTSLO(0),</div><div class='add'>+	I40E_GLPES_PFIP6TXMCOCTSLO(0),</div><div class='add'>+	I40E_GLPES_PFUDPRXPKTSLO(0),</div><div class='add'>+	I40E_GLPES_PFUDPTXPKTSLO(0)</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static u64 i40iw_masks[IRDMA_MAX_MASKS] = {</div><div class='add'>+	I40E_PFPE_CCQPSTATUS_CCQP_DONE,</div><div class='add'>+	I40E_PFPE_CCQPSTATUS_CCQP_ERR,</div><div class='add'>+	I40E_CQPSQ_STAG_PDID,</div><div class='add'>+	I40E_CQPSQ_CQ_CEQID,</div><div class='add'>+	I40E_CQPSQ_CQ_CQID,</div><div class='add'>+	I40E_COMMIT_FPM_CQCNT,</div><div class='add'>+	I40E_CQPSQ_UPESD_HMCFNID,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static u64 i40iw_shifts[IRDMA_MAX_SHIFTS] = {</div><div class='add'>+	I40E_PFPE_CCQPSTATUS_CCQP_DONE_S,</div><div class='add'>+	I40E_PFPE_CCQPSTATUS_CCQP_ERR_S,</div><div class='add'>+	I40E_CQPSQ_STAG_PDID_S,</div><div class='add'>+	I40E_CQPSQ_CQ_CEQID_S,</div><div class='add'>+	I40E_CQPSQ_CQ_CQID_S,</div><div class='add'>+	I40E_COMMIT_FPM_CQCNT_S,</div><div class='add'>+	I40E_CQPSQ_UPESD_HMCFNID_S,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * i40iw_config_ceq- Configure CEQ interrupt</div><div class='add'>+ * @dev: pointer to the device structure</div><div class='add'>+ * @ceq_id: Completion Event Queue ID</div><div class='add'>+ * @idx: vector index</div><div class='add'>+ * @enable: Enable CEQ interrupt when true</div><div class='add'>+ */</div><div class='add'>+static void i40iw_config_ceq(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx,</div><div class='add'>+			     bool enable)</div><div class='add'>+{</div><div class='add'>+	u32 reg_val;</div><div class='add'>+</div><div class='add'>+	reg_val = FIELD_PREP(I40E_PFINT_LNKLSTN_FIRSTQ_INDX, ceq_id) |</div><div class='add'>+		  FIELD_PREP(I40E_PFINT_LNKLSTN_FIRSTQ_TYPE, QUEUE_TYPE_CEQ);</div><div class='add'>+	wr32(dev-&gt;hw, I40E_PFINT_LNKLSTN(idx - 1), reg_val);</div><div class='add'>+</div><div class='add'>+	reg_val = FIELD_PREP(I40E_PFINT_DYN_CTLN_ITR_INDX, 0x3) |</div><div class='add'>+		  FIELD_PREP(I40E_PFINT_DYN_CTLN_INTENA, 0x1);</div><div class='add'>+	wr32(dev-&gt;hw, I40E_PFINT_DYN_CTLN(idx - 1), reg_val);</div><div class='add'>+</div><div class='add'>+	reg_val = FIELD_PREP(IRDMA_GLINT_CEQCTL_CAUSE_ENA, enable) |</div><div class='add'>+		  FIELD_PREP(IRDMA_GLINT_CEQCTL_MSIX_INDX, idx) |</div><div class='add'>+		  FIELD_PREP(I40E_PFINT_CEQCTL_NEXTQ_INDX, NULL_QUEUE_INDEX) |</div><div class='add'>+		  FIELD_PREP(IRDMA_GLINT_CEQCTL_ITR_INDX, 0x3);</div><div class='add'>+</div><div class='add'>+	wr32(dev-&gt;hw, i40iw_regs[IRDMA_GLINT_CEQCTL] + 4 * ceq_id, reg_val);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * i40iw_ena_irq - Enable interrupt</div><div class='add'>+ * @dev: pointer to the device structure</div><div class='add'>+ * @idx: vector index</div><div class='add'>+ */</div><div class='add'>+static void i40iw_ena_irq(struct irdma_sc_dev *dev, u32 idx)</div><div class='add'>+{</div><div class='add'>+	u32 val;</div><div class='add'>+</div><div class='add'>+	val = FIELD_PREP(IRDMA_GLINT_DYN_CTL_INTENA, 0x1) |</div><div class='add'>+	      FIELD_PREP(IRDMA_GLINT_DYN_CTL_CLEARPBA, 0x1) |</div><div class='add'>+	      FIELD_PREP(IRDMA_GLINT_DYN_CTL_ITR_INDX, 0x3);</div><div class='add'>+	wr32(dev-&gt;hw, i40iw_regs[IRDMA_GLINT_DYN_CTL] + 4 * (idx - 1), val);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * i40iw_disable_irq - Disable interrupt</div><div class='add'>+ * @dev: pointer to the device structure</div><div class='add'>+ * @idx: vector index</div><div class='add'>+ */</div><div class='add'>+static void i40iw_disable_irq(struct irdma_sc_dev *dev, u32 idx)</div><div class='add'>+{</div><div class='add'>+	wr32(dev-&gt;hw, i40iw_regs[IRDMA_GLINT_DYN_CTL] + 4 * (idx - 1), 0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct irdma_irq_ops i40iw_irq_ops = {</div><div class='add'>+	.irdma_cfg_aeq = irdma_cfg_aeq,</div><div class='add'>+	.irdma_cfg_ceq = i40iw_config_ceq,</div><div class='add'>+	.irdma_dis_irq = i40iw_disable_irq,</div><div class='add'>+	.irdma_en_irq = i40iw_ena_irq,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct irdma_hw_stat_map i40iw_hw_stat_map[] = {</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RXVLANERR]	=	{   0,  0, IRDMA_MAX_STATS_24 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXOCTS] =	{   8,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXPKTS] =	{  16,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXDISCARD] =	{  24,  0, IRDMA_MAX_STATS_32 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXTRUNC] =	{  32,  0, IRDMA_MAX_STATS_32 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXFRAGS] =      {  40,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] =     {  48,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXOCTS] =       {  56,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXPKTS] =       {  64,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXDISCARD] =    {  72,  0, IRDMA_MAX_STATS_32 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXTRUNC] =      {  80,  0, IRDMA_MAX_STATS_32 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXFRAGS] =      {  88,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] =     {  96,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXOCTS] =       { 104,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXPKTS] =       { 112,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXFRAGS] =      { 120,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] =     { 128,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXOCTS] =       { 136,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXPKTS] =       { 144,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXFRAGS] =      { 152,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] =     { 160,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] =    { 168,  0, IRDMA_MAX_STATS_24 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] =    { 176,  0, IRDMA_MAX_STATS_24 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPRXSEGS] =       { 184,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPRXOPTERR] =     { 192,  0, IRDMA_MAX_STATS_24 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] =   { 200,  0, IRDMA_MAX_STATS_24 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPTXSEG] =        { 208,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPRTXSEG] =       { 216,  0, IRDMA_MAX_STATS_32 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMARXWRS] =       { 224,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMARXRDS] =       { 232,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMARXSNDS] =      { 240,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMATXWRS] =       { 248,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMATXRDS] =       { 256,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMATXSNDS] =      { 264,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMAVBND] =        { 272,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMAVINV] =        { 280,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXMCOCTS] =     { 288,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXMCOCTS] =     { 296,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXMCOCTS] =     { 304,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXMCOCTS] =     { 312,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_UDPRXPKTS] =       { 320,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_UDPTXPKTS] =       { 328,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+void i40iw_init_hw(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+	u8 __iomem *hw_addr;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; IRDMA_MAX_REGS; ++i) {</div><div class='add'>+		hw_addr = dev-&gt;hw-&gt;hw_addr;</div><div class='add'>+</div><div class='add'>+		if (i == IRDMA_DB_ADDR_OFFSET)</div><div class='add'>+			hw_addr = NULL;</div><div class='add'>+</div><div class='add'>+		dev-&gt;hw_regs[i] = (u32 __iomem *)(i40iw_regs[i] + hw_addr);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; IRDMA_HW_STAT_INDEX_MAX_GEN_1; ++i)</div><div class='add'>+		dev-&gt;hw_stats_regs[i] = i40iw_stat_offsets[i];</div><div class='add'>+</div><div class='add'>+	dev-&gt;hw_attrs.first_hw_vf_fpm_id = I40IW_FIRST_VF_FPM_ID;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_vf_fpm_id = IRDMA_MAX_VF_FPM_ID;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; IRDMA_MAX_SHIFTS; ++i)</div><div class='add'>+		dev-&gt;hw_shifts[i] = i40iw_shifts[i];</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; IRDMA_MAX_MASKS; ++i)</div><div class='add'>+		dev-&gt;hw_masks[i] = i40iw_masks[i];</div><div class='add'>+</div><div class='add'>+	dev-&gt;wqe_alloc_db = dev-&gt;hw_regs[IRDMA_WQEALLOC];</div><div class='add'>+	dev-&gt;cq_arm_db = dev-&gt;hw_regs[IRDMA_CQARM];</div><div class='add'>+	dev-&gt;aeq_alloc_db = dev-&gt;hw_regs[IRDMA_AEQALLOC];</div><div class='add'>+	dev-&gt;cqp_db = dev-&gt;hw_regs[IRDMA_CQPDB];</div><div class='add'>+	dev-&gt;cq_ack_db = dev-&gt;hw_regs[IRDMA_CQACK];</div><div class='add'>+	dev-&gt;ceq_itr_mask_db = NULL;</div><div class='add'>+	dev-&gt;aeq_itr_mask_db = NULL;</div><div class='add'>+	dev-&gt;irq_ops = &amp;i40iw_irq_ops;</div><div class='add'>+	dev-&gt;hw_stats_map = i40iw_hw_stat_map;</div><div class='add'>+</div><div class='add'>+	/* Setup the hardware limits, hmc may limit further */</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_wq_frags = I40IW_MAX_WQ_FRAGMENT_COUNT;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_read_sges = I40IW_MAX_SGE_RD;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_device_pages = I40IW_MAX_PUSH_PAGE_COUNT;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_inline = I40IW_MAX_INLINE_DATA_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.page_size_cap = SZ_4K | SZ_2M;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_ird = I40IW_MAX_IRD_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_ord = I40IW_MAX_ORD_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_wqes = I40IW_MAX_WQ_ENTRIES;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_rq_quanta = I40IW_QP_SW_MAX_RQ_QUANTA;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_wq_quanta = I40IW_QP_SW_MAX_WQ_QUANTA;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_sq_chunk = I40IW_MAX_QUANTA_PER_WR;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_pds = I40IW_MAX_PDS;</div><div class='add'>+	dev-&gt;hw_attrs.max_stat_inst = I40IW_MAX_STATS_COUNT;</div><div class='add'>+	dev-&gt;hw_attrs.max_stat_idx = IRDMA_HW_STAT_INDEX_MAX_GEN_1;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_outbound_msg_size = I40IW_MAX_OUTBOUND_MSG_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_inbound_msg_size = I40IW_MAX_INBOUND_MSG_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.min_hw_wq_size = I40IW_MIN_WQ_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.max_qp_wr = I40IW_MAX_QP_WRS;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/i40iw_hw.h b/drivers/infiniband/hw/irdma/i40iw_hw.h<br/>new file mode 100644<br/>index 000000000000..0095b327afcc<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/i40iw_hw.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/i40iw_hw.h</a></div><div class='hunk'>@@ -0,0 +1,162 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#ifndef I40IW_HW_H</div><div class='add'>+#define I40IW_HW_H</div><div class='add'>+#define I40E_VFPE_CQPTAIL1            0x0000A000 /* Reset: VFR */</div><div class='add'>+#define I40E_VFPE_CQPDB1              0x0000BC00 /* Reset: VFR */</div><div class='add'>+#define I40E_VFPE_CCQPSTATUS1         0x0000B800 /* Reset: VFR */</div><div class='add'>+#define I40E_VFPE_CCQPHIGH1           0x00009800 /* Reset: VFR */</div><div class='add'>+#define I40E_VFPE_CCQPLOW1            0x0000AC00 /* Reset: VFR */</div><div class='add'>+#define I40E_VFPE_CQARM1              0x0000B400 /* Reset: VFR */</div><div class='add'>+#define I40E_VFPE_CQACK1              0x0000B000 /* Reset: VFR */</div><div class='add'>+#define I40E_VFPE_AEQALLOC1           0x0000A400 /* Reset: VFR */</div><div class='add'>+#define I40E_VFPE_CQPERRCODES1        0x00009C00 /* Reset: VFR */</div><div class='add'>+#define I40E_VFPE_WQEALLOC1           0x0000C000 /* Reset: VFR */</div><div class='add'>+#define I40E_VFINT_DYN_CTLN(_INTVF)   (0x00024800 + ((_INTVF) * 4)) /* _i=0...511 */ /* Reset: VFR */</div><div class='add'>+</div><div class='add'>+#define I40E_PFPE_CQPTAIL             0x00008080 /* Reset: PFR */</div><div class='add'>+</div><div class='add'>+#define I40E_PFPE_CQPDB               0x00008000 /* Reset: PFR */</div><div class='add'>+#define I40E_PFPE_CCQPSTATUS          0x00008100 /* Reset: PFR */</div><div class='add'>+#define I40E_PFPE_CCQPHIGH            0x00008200 /* Reset: PFR */</div><div class='add'>+#define I40E_PFPE_CCQPLOW             0x00008180 /* Reset: PFR */</div><div class='add'>+#define I40E_PFPE_CQARM               0x00131080 /* Reset: PFR */</div><div class='add'>+#define I40E_PFPE_CQACK               0x00131100 /* Reset: PFR */</div><div class='add'>+#define I40E_PFPE_AEQALLOC            0x00131180 /* Reset: PFR */</div><div class='add'>+#define I40E_PFPE_CQPERRCODES         0x00008880 /* Reset: PFR */</div><div class='add'>+#define I40E_PFPE_WQEALLOC            0x00138C00 /* Reset: PFR */</div><div class='add'>+#define I40E_GLPCI_LBARCTRL           0x000BE484 /* Reset: POR */</div><div class='add'>+#define I40E_GLPE_CPUSTATUS0          0x0000D040 /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPE_CPUSTATUS1          0x0000D044 /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPE_CPUSTATUS2          0x0000D048 /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPE_CRITERR             0x000B4000 /* Reset: PE_CORER */</div><div class='add'>+#define I40E_PFHMC_PDINV              0x000C0300 /* Reset: PFR */</div><div class='add'>+#define I40E_GLHMC_VFPDINV(_i)        (0x000C8300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */</div><div class='add'>+#define I40E_PFINT_DYN_CTLN(_INTPF)   (0x00034800 + ((_INTPF) * 4)) /* _i=0...511 */	/* Reset: PFR */</div><div class='add'>+#define I40E_PFINT_AEQCTL             0x00038700 /* Reset: CORER */</div><div class='add'>+</div><div class='add'>+#define I40E_GLPES_PFIP4RXDISCARD(_i)            (0x00010600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP4RXTRUNC(_i)              (0x00010700 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP4TXNOROUTE(_i)            (0x00012E00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP6RXDISCARD(_i)            (0x00011200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP6RXTRUNC(_i)              (0x00011300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+</div><div class='add'>+#define I40E_GLPES_PFRDMAVBNDLO(_i)              (0x00014800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP4TXMCOCTSLO(_i)           (0x00012000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP6RXMCOCTSLO(_i)           (0x00011600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP6TXMCOCTSLO(_i)           (0x00012A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFUDPRXPKTSLO(_i)             (0x00013800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFUDPTXPKTSLO(_i)             (0x00013A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+</div><div class='add'>+#define I40E_GLPES_PFIP6TXNOROUTE(_i)            (0x00012F00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFTCPRTXSEG(_i)               (0x00013600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFTCPRXOPTERR(_i)             (0x00013200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFTCPRXPROTOERR(_i)           (0x00013300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFRXVLANERR(_i)               (0x00010000 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP4RXOCTSLO(_i)             (0x00010200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP4RXPKTSLO(_i)             (0x00010400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP4RXFRAGSLO(_i)            (0x00010800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP4RXMCPKTSLO(_i)           (0x00010C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP4TXOCTSLO(_i)             (0x00011A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP4TXPKTSLO(_i)             (0x00011C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP4TXFRAGSLO(_i)            (0x00011E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP4TXMCPKTSLO(_i)           (0x00012200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP6RXOCTSLO(_i)             (0x00010E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP6RXPKTSLO(_i)             (0x00011000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP6RXFRAGSLO(_i)            (0x00011400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP6TXOCTSLO(_i)             (0x00012400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP6TXPKTSLO(_i)             (0x00012600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP6TXFRAGSLO(_i)            (0x00012800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP6TXMCPKTSLO(_i)           (0x00012C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFTCPTXSEGLO(_i)              (0x00013400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFRDMARXRDSLO(_i)             (0x00013E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFRDMARXSNDSLO(_i)            (0x00014000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFRDMARXWRSLO(_i)             (0x00013C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFRDMATXRDSLO(_i)             (0x00014400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFRDMATXSNDSLO(_i)            (0x00014600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFRDMATXWRSLO(_i)             (0x00014200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP4RXMCOCTSLO(_i)           (0x00010A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFIP6RXMCPKTSLO(_i)           (0x00011800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFTCPRXSEGSLO(_i)             (0x00013000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+#define I40E_GLPES_PFRDMAVINVLO(_i)              (0x00014A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */</div><div class='add'>+</div><div class='add'>+#define I40IW_DB_ADDR_OFFSET    (4 * 1024 * 1024 - 64 * 1024)</div><div class='add'>+</div><div class='add'>+#define I40IW_VF_DB_ADDR_OFFSET (64 * 1024)</div><div class='add'>+</div><div class='add'>+#define I40E_PFINT_LNKLSTN(_INTPF)           (0x00035000 + ((_INTPF) * 4)) /* _i=0...511 */ /* Reset: PFR */</div><div class='add'>+#define I40E_PFINT_LNKLSTN_MAX_INDEX         511</div><div class='add'>+#define I40E_PFINT_LNKLSTN_FIRSTQ_INDX GENMASK(10, 0)</div><div class='add'>+#define I40E_PFINT_LNKLSTN_FIRSTQ_TYPE GENMASK(12, 11)</div><div class='add'>+</div><div class='add'>+#define I40E_PFINT_CEQCTL(_INTPF)          (0x00036800 + ((_INTPF) * 4)) /* _i=0...511 */ /* Reset: CORER */</div><div class='add'>+#define I40E_PFINT_CEQCTL_MAX_INDEX        511</div><div class='add'>+</div><div class='add'>+/* shifts/masks for FLD_[LS/RS]_64 macros used in device table */</div><div class='add'>+#define I40E_PFINT_CEQCTL_MSIX_INDX_S 0</div><div class='add'>+#define I40E_PFINT_CEQCTL_MSIX_INDX GENMASK(7, 0)</div><div class='add'>+#define I40E_PFINT_CEQCTL_ITR_INDX_S 11</div><div class='add'>+#define I40E_PFINT_CEQCTL_ITR_INDX GENMASK(12, 11)</div><div class='add'>+#define I40E_PFINT_CEQCTL_MSIX0_INDX_S 13</div><div class='add'>+#define I40E_PFINT_CEQCTL_MSIX0_INDX GENMASK(15, 13)</div><div class='add'>+#define I40E_PFINT_CEQCTL_NEXTQ_INDX_S 16</div><div class='add'>+#define I40E_PFINT_CEQCTL_NEXTQ_INDX GENMASK(26, 16)</div><div class='add'>+#define I40E_PFINT_CEQCTL_NEXTQ_TYPE_S 27</div><div class='add'>+#define I40E_PFINT_CEQCTL_NEXTQ_TYPE GENMASK(28, 27)</div><div class='add'>+#define I40E_PFINT_CEQCTL_CAUSE_ENA_S 30</div><div class='add'>+#define I40E_PFINT_CEQCTL_CAUSE_ENA BIT(30)</div><div class='add'>+#define I40E_PFINT_CEQCTL_INTEVENT_S 31</div><div class='add'>+#define I40E_PFINT_CEQCTL_INTEVENT BIT(31)</div><div class='add'>+#define I40E_CQPSQ_STAG_PDID_S 48</div><div class='add'>+#define I40E_CQPSQ_STAG_PDID GENMASK_ULL(62, 48)</div><div class='add'>+#define I40E_PFPE_CCQPSTATUS_CCQP_DONE_S 0</div><div class='add'>+#define I40E_PFPE_CCQPSTATUS_CCQP_DONE BIT_ULL(0)</div><div class='add'>+#define I40E_PFPE_CCQPSTATUS_CCQP_ERR_S 31</div><div class='add'>+#define I40E_PFPE_CCQPSTATUS_CCQP_ERR BIT_ULL(31)</div><div class='add'>+#define I40E_PFINT_DYN_CTLN_ITR_INDX_S 3</div><div class='add'>+#define I40E_PFINT_DYN_CTLN_ITR_INDX GENMASK(4, 3)</div><div class='add'>+#define I40E_PFINT_DYN_CTLN_INTENA_S 0</div><div class='add'>+#define I40E_PFINT_DYN_CTLN_INTENA BIT(0)</div><div class='add'>+#define I40E_CQPSQ_CQ_CEQID_S 24</div><div class='add'>+#define I40E_CQPSQ_CQ_CEQID GENMASK(30, 24)</div><div class='add'>+#define I40E_CQPSQ_CQ_CQID_S 0</div><div class='add'>+#define I40E_CQPSQ_CQ_CQID GENMASK_ULL(15, 0)</div><div class='add'>+#define I40E_COMMIT_FPM_CQCNT_S 0</div><div class='add'>+#define I40E_COMMIT_FPM_CQCNT GENMASK_ULL(17, 0)</div><div class='add'>+#define I40E_CQPSQ_UPESD_HMCFNID_S 0</div><div class='add'>+#define I40E_CQPSQ_UPESD_HMCFNID GENMASK_ULL(5, 0)</div><div class='add'>+</div><div class='add'>+#define I40E_VSIQF_CTL(_VSI)             (0x0020D800 + ((_VSI) * 4))</div><div class='add'>+</div><div class='add'>+enum i40iw_device_caps_const {</div><div class='add'>+	I40IW_MAX_WQ_FRAGMENT_COUNT		= 3,</div><div class='add'>+	I40IW_MAX_SGE_RD			= 1,</div><div class='add'>+	I40IW_MAX_PUSH_PAGE_COUNT		= 0,</div><div class='add'>+	I40IW_MAX_INLINE_DATA_SIZE		= 48,</div><div class='add'>+	I40IW_MAX_IRD_SIZE			= 63,</div><div class='add'>+	I40IW_MAX_ORD_SIZE			= 127,</div><div class='add'>+	I40IW_MAX_WQ_ENTRIES			= 2048,</div><div class='add'>+	I40IW_MAX_WQE_SIZE_RQ			= 128,</div><div class='add'>+	I40IW_MAX_PDS				= 32768,</div><div class='add'>+	I40IW_MAX_STATS_COUNT			= 16,</div><div class='add'>+	I40IW_MAX_CQ_SIZE			= 1048575,</div><div class='add'>+	I40IW_MAX_OUTBOUND_MSG_SIZE		= 2147483647,</div><div class='add'>+	I40IW_MAX_INBOUND_MSG_SIZE		= 2147483647,</div><div class='add'>+	I40IW_MIN_WQ_SIZE                       = 4 /* WQEs */,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define I40IW_QP_WQE_MIN_SIZE   32</div><div class='add'>+#define I40IW_QP_WQE_MAX_SIZE   128</div><div class='add'>+#define I40IW_MAX_RQ_WQE_SHIFT  2</div><div class='add'>+#define I40IW_MAX_QUANTA_PER_WR 2</div><div class='add'>+</div><div class='add'>+#define I40IW_QP_SW_MAX_SQ_QUANTA 2048</div><div class='add'>+#define I40IW_QP_SW_MAX_RQ_QUANTA 16384</div><div class='add'>+#define I40IW_QP_SW_MAX_WQ_QUANTA 2048</div><div class='add'>+#define I40IW_MAX_QP_WRS ((I40IW_QP_SW_MAX_SQ_QUANTA - IRDMA_SQ_RSVD) / I40IW_MAX_QUANTA_PER_WR)</div><div class='add'>+#define I40IW_FIRST_VF_FPM_ID 16</div><div class='add'>+#define QUEUE_TYPE_CEQ        2</div><div class='add'>+#define NULL_QUEUE_INDEX      0x7FF</div><div class='add'>+</div><div class='add'>+void i40iw_init_hw(struct irdma_sc_dev *dev);</div><div class='add'>+#endif /* I40IW_HW_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/i40iw_if.c b/drivers/infiniband/hw/irdma/i40iw_if.c<br/>new file mode 100644<br/>index 000000000000..15e036ddaffb<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/i40iw_if.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/i40iw_if.c</a></div><div class='hunk'>@@ -0,0 +1,220 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#include "main.h"</div><div class='add'>+#include "i40iw_hw.h"</div><div class='add'>+#include &lt;linux/net/intel/i40e_client.h&gt;</div><div class='add'>+</div><div class='add'>+static struct i40e_client i40iw_client;</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * i40iw_l2param_change - handle mss change</div><div class='add'>+ * @cdev_info: parent lan device information structure with data/ops</div><div class='add'>+ * @client: client for parameter change</div><div class='add'>+ * @params: new parameters from L2</div><div class='add'>+ */</div><div class='add'>+static void i40iw_l2param_change(struct i40e_info *cdev_info,</div><div class='add'>+				 struct i40e_client *client,</div><div class='add'>+				 struct i40e_params *params)</div><div class='add'>+{</div><div class='add'>+	struct irdma_l2params l2params = {};</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct ib_device *ibdev;</div><div class='add'>+</div><div class='add'>+	ibdev = ib_device_get_by_netdev(cdev_info-&gt;netdev, RDMA_DRIVER_IRDMA);</div><div class='add'>+	if (!ibdev)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	iwdev = to_iwdev(ibdev);</div><div class='add'>+</div><div class='add'>+	if (iwdev-&gt;vsi.mtu != params-&gt;mtu) {</div><div class='add'>+		l2params.mtu_changed = true;</div><div class='add'>+		l2params.mtu = params-&gt;mtu;</div><div class='add'>+	}</div><div class='add'>+	irdma_change_l2params(&amp;iwdev-&gt;vsi, &amp;l2params);</div><div class='add'>+	ib_device_put(ibdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * i40iw_close - client interface operation close for iwarp/uda device</div><div class='add'>+ * @cdev_info: parent lan device information structure with data/ops</div><div class='add'>+ * @client: client to close</div><div class='add'>+ * @reset: flag to indicate close on reset</div><div class='add'>+ *</div><div class='add'>+ * Called by the lan driver during the processing of client unregister</div><div class='add'>+ * Destroy and clean up the driver resources</div><div class='add'>+ */</div><div class='add'>+static void i40iw_close(struct i40e_info *cdev_info, struct i40e_client *client,</div><div class='add'>+			bool reset)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct ib_device *ibdev;</div><div class='add'>+</div><div class='add'>+	ibdev = ib_device_get_by_netdev(cdev_info-&gt;netdev, RDMA_DRIVER_IRDMA);</div><div class='add'>+	if (WARN_ON(!ibdev))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	iwdev = to_iwdev(ibdev);</div><div class='add'>+	if (reset)</div><div class='add'>+		iwdev-&gt;rf-&gt;reset = true;</div><div class='add'>+</div><div class='add'>+	iwdev-&gt;iw_status = 0;</div><div class='add'>+	irdma_port_ibevent(iwdev);</div><div class='add'>+	ib_unregister_device_and_put(ibdev);</div><div class='add'>+	pr_debug("INIT: Gen1 PF[%d] close complete\n", PCI_FUNC(cdev_info-&gt;pcidev-&gt;devfn));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void i40iw_request_reset(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct i40e_info *cdev_info = rf-&gt;cdev;</div><div class='add'>+</div><div class='add'>+	cdev_info-&gt;ops-&gt;request_reset(cdev_info, &amp;i40iw_client, 1);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void i40iw_fill_device_info(struct irdma_device *iwdev, struct i40e_info *cdev_info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+</div><div class='add'>+	rf-&gt;rdma_ver = IRDMA_GEN_1;</div><div class='add'>+	rf-&gt;sc_dev.hw = &amp;rf-&gt;hw;</div><div class='add'>+	rf-&gt;sc_dev.hw_attrs.uk_attrs.hw_rev = IRDMA_GEN_1;</div><div class='add'>+	rf-&gt;sc_dev.privileged = true;</div><div class='add'>+	rf-&gt;gen_ops.request_reset = i40iw_request_reset;</div><div class='add'>+	rf-&gt;pcidev = cdev_info-&gt;pcidev;</div><div class='add'>+	rf-&gt;pf_id = cdev_info-&gt;fid;</div><div class='add'>+	rf-&gt;hw.hw_addr = cdev_info-&gt;hw_addr;</div><div class='add'>+	rf-&gt;cdev = cdev_info;</div><div class='add'>+	rf-&gt;msix_count = cdev_info-&gt;msix_count;</div><div class='add'>+	rf-&gt;msix_entries = cdev_info-&gt;msix_entries;</div><div class='add'>+	rf-&gt;limits_sel = 5;</div><div class='add'>+	rf-&gt;protocol_used = IRDMA_IWARP_PROTOCOL_ONLY;</div><div class='add'>+	rf-&gt;iwdev = iwdev;</div><div class='add'>+</div><div class='add'>+	iwdev-&gt;init_state = INITIAL_STATE;</div><div class='add'>+	iwdev-&gt;rcv_wnd = IRDMA_CM_DEFAULT_RCV_WND_SCALED;</div><div class='add'>+	iwdev-&gt;rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;</div><div class='add'>+	iwdev-&gt;netdev = cdev_info-&gt;netdev;</div><div class='add'>+	iwdev-&gt;vsi_num = 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * i40iw_open - client interface operation open for iwarp/uda device</div><div class='add'>+ * @cdev_info: parent lan device information structure with data/ops</div><div class='add'>+ * @client: iwarp client information, provided during registration</div><div class='add'>+ *</div><div class='add'>+ * Called by the lan driver during the processing of client register</div><div class='add'>+ * Create device resources, set up queues, pble and hmc objects and</div><div class='add'>+ * register the device with the ib verbs interface</div><div class='add'>+ * Return 0 if successful, otherwise return error</div><div class='add'>+ */</div><div class='add'>+static int i40iw_open(struct i40e_info *cdev_info, struct i40e_client *client)</div><div class='add'>+{</div><div class='add'>+	struct irdma_l2params l2params = {};</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct irdma_pci_f *rf;</div><div class='add'>+	int err = -EIO;</div><div class='add'>+	int i;</div><div class='add'>+	u16 qset;</div><div class='add'>+	u16 last_qset = IRDMA_NO_QSET;</div><div class='add'>+</div><div class='add'>+	iwdev = ib_alloc_device(irdma_device, ibdev);</div><div class='add'>+	if (!iwdev)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	iwdev-&gt;rf = kzalloc(sizeof(*rf), GFP_KERNEL);</div><div class='add'>+	if (!iwdev-&gt;rf) {</div><div class='add'>+		ib_dealloc_device(&amp;iwdev-&gt;ibdev);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	i40iw_fill_device_info(iwdev, cdev_info);</div><div class='add'>+	rf = iwdev-&gt;rf;</div><div class='add'>+</div><div class='add'>+	if (irdma_ctrl_init_hw(rf)) {</div><div class='add'>+		err = -EIO;</div><div class='add'>+		goto err_ctrl_init;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	l2params.mtu = (cdev_info-&gt;params.mtu) ? cdev_info-&gt;params.mtu : IRDMA_DEFAULT_MTU;</div><div class='add'>+	for (i = 0; i &lt; I40E_CLIENT_MAX_USER_PRIORITY; i++) {</div><div class='add'>+		qset = cdev_info-&gt;params.qos.prio_qos[i].qs_handle;</div><div class='add'>+		l2params.up2tc[i] = cdev_info-&gt;params.qos.prio_qos[i].tc;</div><div class='add'>+		l2params.qs_handle_list[i] = qset;</div><div class='add'>+		if (last_qset == IRDMA_NO_QSET)</div><div class='add'>+			last_qset = qset;</div><div class='add'>+		else if ((qset != last_qset) &amp;&amp; (qset != IRDMA_NO_QSET))</div><div class='add'>+			iwdev-&gt;dcb_vlan_mode = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (irdma_rt_init_hw(iwdev, &amp;l2params)) {</div><div class='add'>+		err = -EIO;</div><div class='add'>+		goto err_rt_init;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = irdma_ib_register_device(iwdev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_ibreg;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev, "INIT: Gen1 PF[%d] open success\n",</div><div class='add'>+		  PCI_FUNC(rf-&gt;pcidev-&gt;devfn));</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_ibreg:</div><div class='add'>+	irdma_rt_deinit_hw(iwdev);</div><div class='add'>+err_rt_init:</div><div class='add'>+	irdma_ctrl_deinit_hw(rf);</div><div class='add'>+err_ctrl_init:</div><div class='add'>+	kfree(iwdev-&gt;rf);</div><div class='add'>+	ib_dealloc_device(&amp;iwdev-&gt;ibdev);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* client interface functions */</div><div class='add'>+static const struct i40e_client_ops i40e_ops = {</div><div class='add'>+	.open = i40iw_open,</div><div class='add'>+	.close = i40iw_close,</div><div class='add'>+	.l2_param_change = i40iw_l2param_change</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static struct i40e_client i40iw_client = {</div><div class='add'>+	.ops = &amp;i40e_ops,</div><div class='add'>+	.type = I40E_CLIENT_IWARP,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int i40iw_probe(struct auxiliary_device *aux_dev, const struct auxiliary_device_id *id)</div><div class='add'>+{</div><div class='add'>+	struct i40e_auxiliary_device *i40e_adev = container_of(aux_dev,</div><div class='add'>+							       struct i40e_auxiliary_device,</div><div class='add'>+							       aux_dev);</div><div class='add'>+	struct i40e_info *cdev_info = i40e_adev-&gt;ldev;</div><div class='add'>+</div><div class='add'>+	strscpy_pad(i40iw_client.name, "irdma", I40E_CLIENT_STR_LENGTH);</div><div class='add'>+	i40e_client_device_register(cdev_info, &amp;i40iw_client);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void i40iw_remove(struct auxiliary_device *aux_dev)</div><div class='add'>+{</div><div class='add'>+	struct i40e_auxiliary_device *i40e_adev = container_of(aux_dev,</div><div class='add'>+							       struct i40e_auxiliary_device,</div><div class='add'>+							       aux_dev);</div><div class='add'>+	struct i40e_info *cdev_info = i40e_adev-&gt;ldev;</div><div class='add'>+</div><div class='add'>+	i40e_client_device_unregister(cdev_info);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct auxiliary_device_id i40iw_auxiliary_id_table[] = {</div><div class='add'>+	{.name = "i40e.iwarp", },</div><div class='add'>+	{},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+MODULE_DEVICE_TABLE(auxiliary, i40iw_auxiliary_id_table);</div><div class='add'>+</div><div class='add'>+struct auxiliary_driver i40iw_auxiliary_drv = {</div><div class='add'>+	.name = "gen_1",</div><div class='add'>+	.id_table = i40iw_auxiliary_id_table,</div><div class='add'>+	.probe = i40iw_probe,</div><div class='add'>+	.remove = i40iw_remove,</div><div class='add'>+};</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/icrdma_hw.c b/drivers/infiniband/hw/irdma/icrdma_hw.c<br/>new file mode 100644<br/>index 000000000000..32f26284a788<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/icrdma_hw.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/icrdma_hw.c</a></div><div class='hunk'>@@ -0,0 +1,205 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2017 - 2021 Intel Corporation */</div><div class='add'>+#include "osdep.h"</div><div class='add'>+#include "type.h"</div><div class='add'>+#include "icrdma_hw.h"</div><div class='add'>+</div><div class='add'>+static u32 icrdma_regs[IRDMA_MAX_REGS] = {</div><div class='add'>+	PFPE_CQPTAIL,</div><div class='add'>+	PFPE_CQPDB,</div><div class='add'>+	PFPE_CCQPSTATUS,</div><div class='add'>+	PFPE_CCQPHIGH,</div><div class='add'>+	PFPE_CCQPLOW,</div><div class='add'>+	PFPE_CQARM,</div><div class='add'>+	PFPE_CQACK,</div><div class='add'>+	PFPE_AEQALLOC,</div><div class='add'>+	PFPE_CQPERRCODES,</div><div class='add'>+	PFPE_WQEALLOC,</div><div class='add'>+	GLINT_DYN_CTL(0),</div><div class='add'>+	ICRDMA_DB_ADDR_OFFSET,</div><div class='add'>+</div><div class='add'>+	GLPCI_LBARCTRL,</div><div class='add'>+	GLPE_CPUSTATUS0,</div><div class='add'>+	GLPE_CPUSTATUS1,</div><div class='add'>+	GLPE_CPUSTATUS2,</div><div class='add'>+	PFINT_AEQCTL,</div><div class='add'>+	GLINT_CEQCTL(0),</div><div class='add'>+	VSIQF_PE_CTL1(0),</div><div class='add'>+	PFHMC_PDINV,</div><div class='add'>+	GLHMC_VFPDINV(0),</div><div class='add'>+	GLPE_CRITERR,</div><div class='add'>+	GLINT_RATE(0),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static u64 icrdma_masks[IRDMA_MAX_MASKS] = {</div><div class='add'>+	ICRDMA_CCQPSTATUS_CCQP_DONE,</div><div class='add'>+	ICRDMA_CCQPSTATUS_CCQP_ERR,</div><div class='add'>+	ICRDMA_CQPSQ_STAG_PDID,</div><div class='add'>+	ICRDMA_CQPSQ_CQ_CEQID,</div><div class='add'>+	ICRDMA_CQPSQ_CQ_CQID,</div><div class='add'>+	ICRDMA_COMMIT_FPM_CQCNT,</div><div class='add'>+	ICRDMA_CQPSQ_UPESD_HMCFNID,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static u64 icrdma_shifts[IRDMA_MAX_SHIFTS] = {</div><div class='add'>+	ICRDMA_CCQPSTATUS_CCQP_DONE_S,</div><div class='add'>+	ICRDMA_CCQPSTATUS_CCQP_ERR_S,</div><div class='add'>+	ICRDMA_CQPSQ_STAG_PDID_S,</div><div class='add'>+	ICRDMA_CQPSQ_CQ_CEQID_S,</div><div class='add'>+	ICRDMA_CQPSQ_CQ_CQID_S,</div><div class='add'>+	ICRDMA_COMMIT_FPM_CQCNT_S,</div><div class='add'>+	ICRDMA_CQPSQ_UPESD_HMCFNID_S,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * icrdma_ena_irq - Enable interrupt</div><div class='add'>+ * @dev: pointer to the device structure</div><div class='add'>+ * @idx: vector index</div><div class='add'>+ */</div><div class='add'>+static void icrdma_ena_irq(struct irdma_sc_dev *dev, u32 idx)</div><div class='add'>+{</div><div class='add'>+	u32 val;</div><div class='add'>+	u32 interval = 0;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;ceq_itr &amp;&amp; dev-&gt;aeq-&gt;msix_idx != idx)</div><div class='add'>+		interval = dev-&gt;ceq_itr &gt;&gt; 1; /* 2 usec units */</div><div class='add'>+	val = FIELD_PREP(IRDMA_GLINT_DYN_CTL_ITR_INDX, 0) |</div><div class='add'>+	      FIELD_PREP(IRDMA_GLINT_DYN_CTL_INTERVAL, interval) |</div><div class='add'>+	      FIELD_PREP(IRDMA_GLINT_DYN_CTL_INTENA, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMA_GLINT_DYN_CTL_CLEARPBA, 1);</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1)</div><div class='add'>+		writel(val, dev-&gt;hw_regs[IRDMA_GLINT_DYN_CTL] + idx);</div><div class='add'>+	else</div><div class='add'>+		writel(val, dev-&gt;hw_regs[IRDMA_GLINT_DYN_CTL] + (idx - 1));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * icrdma_disable_irq - Disable interrupt</div><div class='add'>+ * @dev: pointer to the device structure</div><div class='add'>+ * @idx: vector index</div><div class='add'>+ */</div><div class='add'>+static void icrdma_disable_irq(struct irdma_sc_dev *dev, u32 idx)</div><div class='add'>+{</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1)</div><div class='add'>+		writel(0, dev-&gt;hw_regs[IRDMA_GLINT_DYN_CTL] + idx);</div><div class='add'>+	else</div><div class='add'>+		writel(0, dev-&gt;hw_regs[IRDMA_GLINT_DYN_CTL] + (idx - 1));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * icrdma_cfg_ceq- Configure CEQ interrupt</div><div class='add'>+ * @dev: pointer to the device structure</div><div class='add'>+ * @ceq_id: Completion Event Queue ID</div><div class='add'>+ * @idx: vector index</div><div class='add'>+ * @enable: True to enable, False disables</div><div class='add'>+ */</div><div class='add'>+static void icrdma_cfg_ceq(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx,</div><div class='add'>+			   bool enable)</div><div class='add'>+{</div><div class='add'>+	u32 reg_val;</div><div class='add'>+</div><div class='add'>+	reg_val = FIELD_PREP(IRDMA_GLINT_CEQCTL_CAUSE_ENA, enable) |</div><div class='add'>+		  FIELD_PREP(IRDMA_GLINT_CEQCTL_MSIX_INDX, idx) |</div><div class='add'>+		  FIELD_PREP(IRDMA_GLINT_CEQCTL_ITR_INDX, 3);</div><div class='add'>+</div><div class='add'>+	writel(reg_val, dev-&gt;hw_regs[IRDMA_GLINT_CEQCTL] + ceq_id);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct irdma_irq_ops icrdma_irq_ops = {</div><div class='add'>+	.irdma_cfg_aeq = irdma_cfg_aeq,</div><div class='add'>+	.irdma_cfg_ceq = icrdma_cfg_ceq,</div><div class='add'>+	.irdma_dis_irq = icrdma_disable_irq,</div><div class='add'>+	.irdma_en_irq = icrdma_ena_irq,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct irdma_hw_stat_map icrdma_hw_stat_map[] = {</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RXVLANERR]	=	{   0, 32, IRDMA_MAX_STATS_24 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXOCTS] =	{   8,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXPKTS] =	{  16,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXDISCARD] =	{  24, 32, IRDMA_MAX_STATS_32 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXTRUNC] =	{  24,  0, IRDMA_MAX_STATS_32 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXFRAGS] =	{  32,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXMCOCTS] =	{  40,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] =	{  48,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXOCTS] =	{  56,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXPKTS] =	{  64,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXDISCARD] =	{  72, 32, IRDMA_MAX_STATS_32 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXTRUNC] =	{  72,  0, IRDMA_MAX_STATS_32 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXFRAGS] =	{  80,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXMCOCTS] =	{  88,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] =	{  96,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXOCTS] =	{ 104,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXPKTS] =	{ 112,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXFRAGS] =	{ 120,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXMCOCTS] =	{ 128,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] =	{ 136,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXOCTS] =	{ 144,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXPKTS] =	{ 152,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXFRAGS] =	{ 160,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXMCOCTS] =	{ 168,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] =	{ 176,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] =	{ 184, 32, IRDMA_MAX_STATS_24 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] =	{ 184,  0, IRDMA_MAX_STATS_24 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPRXSEGS] =	{ 192, 32, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPRXOPTERR] =	{ 200, 32, IRDMA_MAX_STATS_24 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] =	{ 200,  0, IRDMA_MAX_STATS_24 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPTXSEG] =	{ 208,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPRTXSEG] =	{ 216, 32, IRDMA_MAX_STATS_32 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_UDPRXPKTS] =	{ 224,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_UDPTXPKTS] =	{ 232,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMARXWRS] =	{ 240,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMARXRDS] =	{ 248,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMARXSNDS] =	{ 256,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMATXWRS] =	{ 264,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMATXRDS] =	{ 272,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMATXSNDS] =	{ 280,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMAVBND] =	{ 288,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMAVINV] =	{ 296,  0, IRDMA_MAX_STATS_48 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] = { 304,  0, IRDMA_MAX_STATS_56 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] =	{ 312, 32, IRDMA_MAX_STATS_24 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] =	{ 312,  0, IRDMA_MAX_STATS_32 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TXNPCNPSENT] =	{ 320,  0, IRDMA_MAX_STATS_32 },</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+void icrdma_init_hw(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+	u8 __iomem *hw_addr;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; IRDMA_MAX_REGS; ++i) {</div><div class='add'>+		hw_addr = dev-&gt;hw-&gt;hw_addr;</div><div class='add'>+</div><div class='add'>+		if (i == IRDMA_DB_ADDR_OFFSET)</div><div class='add'>+			hw_addr = NULL;</div><div class='add'>+</div><div class='add'>+		dev-&gt;hw_regs[i] = (u32 __iomem *)(hw_addr + icrdma_regs[i]);</div><div class='add'>+	}</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_vf_fpm_id = IRDMA_MAX_VF_FPM_ID;</div><div class='add'>+	dev-&gt;hw_attrs.first_hw_vf_fpm_id = IRDMA_FIRST_VF_FPM_ID;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; IRDMA_MAX_SHIFTS; ++i)</div><div class='add'>+		dev-&gt;hw_shifts[i] = icrdma_shifts[i];</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; IRDMA_MAX_MASKS; ++i)</div><div class='add'>+		dev-&gt;hw_masks[i] = icrdma_masks[i];</div><div class='add'>+</div><div class='add'>+	dev-&gt;wqe_alloc_db = dev-&gt;hw_regs[IRDMA_WQEALLOC];</div><div class='add'>+	dev-&gt;cq_arm_db = dev-&gt;hw_regs[IRDMA_CQARM];</div><div class='add'>+	dev-&gt;aeq_alloc_db = dev-&gt;hw_regs[IRDMA_AEQALLOC];</div><div class='add'>+	dev-&gt;cqp_db = dev-&gt;hw_regs[IRDMA_CQPDB];</div><div class='add'>+	dev-&gt;cq_ack_db = dev-&gt;hw_regs[IRDMA_CQACK];</div><div class='add'>+	dev-&gt;irq_ops = &amp;icrdma_irq_ops;</div><div class='add'>+	dev-&gt;hw_attrs.page_size_cap = SZ_4K | SZ_2M | SZ_1G;</div><div class='add'>+	dev-&gt;hw_stats_map = icrdma_hw_stat_map;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_ird = ICRDMA_MAX_IRD_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_ord = ICRDMA_MAX_ORD_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.max_stat_inst = ICRDMA_MAX_STATS_COUNT;</div><div class='add'>+	dev-&gt;hw_attrs.max_stat_idx = IRDMA_HW_STAT_INDEX_MAX_GEN_2;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_device_pages = ICRDMA_MAX_PUSH_PAGE_COUNT;</div><div class='add'>+</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.min_hw_wq_size = ICRDMA_MIN_WQ_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_sq_chunk = IRDMA_MAX_QUANTA_PER_WR;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_RTS_AE |</div><div class='add'>+						IRDMA_FEATURE_CQ_RESIZE;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/icrdma_hw.h b/drivers/infiniband/hw/irdma/icrdma_hw.h<br/>new file mode 100644<br/>index 000000000000..d97944ab45da<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/icrdma_hw.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/icrdma_hw.h</a></div><div class='hunk'>@@ -0,0 +1,73 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2017 - 2021 Intel Corporation */</div><div class='add'>+#ifndef ICRDMA_HW_H</div><div class='add'>+#define ICRDMA_HW_H</div><div class='add'>+</div><div class='add'>+#include "irdma.h"</div><div class='add'>+</div><div class='add'>+#define VFPE_CQPTAIL1		0x0000a000</div><div class='add'>+#define VFPE_CQPDB1		0x0000bc00</div><div class='add'>+#define VFPE_CCQPSTATUS1	0x0000b800</div><div class='add'>+#define VFPE_CCQPHIGH1		0x00009800</div><div class='add'>+#define VFPE_CCQPLOW1		0x0000ac00</div><div class='add'>+#define VFPE_CQARM1		0x0000b400</div><div class='add'>+#define VFPE_CQARM1		0x0000b400</div><div class='add'>+#define VFPE_CQACK1		0x0000b000</div><div class='add'>+#define VFPE_AEQALLOC1		0x0000a400</div><div class='add'>+#define VFPE_CQPERRCODES1	0x00009c00</div><div class='add'>+#define VFPE_WQEALLOC1		0x0000c000</div><div class='add'>+#define VFINT_DYN_CTLN(_i)	(0x00003800 + ((_i) * 4)) /* _i=0...63 */</div><div class='add'>+</div><div class='add'>+#define PFPE_CQPTAIL		0x00500880</div><div class='add'>+#define PFPE_CQPDB		0x00500800</div><div class='add'>+#define PFPE_CCQPSTATUS		0x0050a000</div><div class='add'>+#define PFPE_CCQPHIGH		0x0050a100</div><div class='add'>+#define PFPE_CCQPLOW		0x0050a080</div><div class='add'>+#define PFPE_CQARM		0x00502c00</div><div class='add'>+#define PFPE_CQACK		0x00502c80</div><div class='add'>+#define PFPE_AEQALLOC		0x00502d00</div><div class='add'>+#define GLINT_DYN_CTL(_INT)	(0x00160000 + ((_INT) * 4)) /* _i=0...2047 */</div><div class='add'>+#define GLPCI_LBARCTRL		0x0009de74</div><div class='add'>+#define GLPE_CPUSTATUS0		0x0050ba5c</div><div class='add'>+#define GLPE_CPUSTATUS1		0x0050ba60</div><div class='add'>+#define GLPE_CPUSTATUS2		0x0050ba64</div><div class='add'>+#define PFINT_AEQCTL		0x0016cb00</div><div class='add'>+#define PFPE_CQPERRCODES	0x0050a200</div><div class='add'>+#define PFPE_WQEALLOC		0x00504400</div><div class='add'>+#define GLINT_CEQCTL(_INT)	(0x0015c000 + ((_INT) * 4)) /* _i=0...2047 */</div><div class='add'>+#define VSIQF_PE_CTL1(_VSI)	(0x00414000 + ((_VSI) * 4)) /* _i=0...767 */</div><div class='add'>+#define PFHMC_PDINV		0x00520300</div><div class='add'>+#define GLHMC_VFPDINV(_i)	(0x00528300 + ((_i) * 4)) /* _i=0...31 */</div><div class='add'>+#define GLPE_CRITERR		0x00534000</div><div class='add'>+#define GLINT_RATE(_INT)	(0x0015A000 + ((_INT) * 4)) /* _i=0...2047 */ /* Reset Source: CORER */</div><div class='add'>+</div><div class='add'>+#define ICRDMA_DB_ADDR_OFFSET		(8 * 1024 * 1024 - 64 * 1024)</div><div class='add'>+</div><div class='add'>+#define ICRDMA_VF_DB_ADDR_OFFSET	(64 * 1024)</div><div class='add'>+</div><div class='add'>+/* shifts/masks for FLD_[LS/RS]_64 macros used in device table */</div><div class='add'>+#define ICRDMA_CCQPSTATUS_CCQP_DONE_S 0</div><div class='add'>+#define ICRDMA_CCQPSTATUS_CCQP_DONE BIT_ULL(0)</div><div class='add'>+#define ICRDMA_CCQPSTATUS_CCQP_ERR_S 31</div><div class='add'>+#define ICRDMA_CCQPSTATUS_CCQP_ERR BIT_ULL(31)</div><div class='add'>+#define ICRDMA_CQPSQ_STAG_PDID_S 46</div><div class='add'>+#define ICRDMA_CQPSQ_STAG_PDID GENMASK_ULL(63, 46)</div><div class='add'>+#define ICRDMA_CQPSQ_CQ_CEQID_S 22</div><div class='add'>+#define ICRDMA_CQPSQ_CQ_CEQID GENMASK_ULL(31, 22)</div><div class='add'>+#define ICRDMA_CQPSQ_CQ_CQID_S 0</div><div class='add'>+#define ICRDMA_CQPSQ_CQ_CQID GENMASK_ULL(18, 0)</div><div class='add'>+#define ICRDMA_COMMIT_FPM_CQCNT_S 0</div><div class='add'>+#define ICRDMA_COMMIT_FPM_CQCNT GENMASK_ULL(19, 0)</div><div class='add'>+#define ICRDMA_CQPSQ_UPESD_HMCFNID_S 0</div><div class='add'>+#define ICRDMA_CQPSQ_UPESD_HMCFNID GENMASK_ULL(5, 0)</div><div class='add'>+enum icrdma_device_caps_const {</div><div class='add'>+	ICRDMA_MAX_STATS_COUNT = 128,</div><div class='add'>+</div><div class='add'>+	ICRDMA_MAX_IRD_SIZE			= 127,</div><div class='add'>+	ICRDMA_MAX_ORD_SIZE			= 255,</div><div class='add'>+	ICRDMA_MIN_WQ_SIZE                      = 8 /* WQEs */,</div><div class='add'>+	ICRDMA_MAX_PUSH_PAGE_COUNT		= 256,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+void icrdma_init_hw(struct irdma_sc_dev *dev);</div><div class='add'>+#endif /* ICRDMA_HW_H*/</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/icrdma_if.c b/drivers/infiniband/hw/irdma/icrdma_if.c<br/>new file mode 100644<br/>index 000000000000..b49fd9cf2476<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/icrdma_if.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/icrdma_if.c</a></div><div class='hunk'>@@ -0,0 +1,347 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2015 - 2024 Intel Corporation */</div><div class='add'>+</div><div class='add'>+#include "main.h"</div><div class='add'>+#include &lt;linux/net/intel/iidc_rdma_ice.h&gt;</div><div class='add'>+</div><div class='add'>+static void icrdma_prep_tc_change(struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	iwdev-&gt;vsi.tc_change_pending = true;</div><div class='add'>+	irdma_sc_suspend_resume_qps(&amp;iwdev-&gt;vsi, IRDMA_OP_SUSPEND);</div><div class='add'>+</div><div class='add'>+	/* Wait for all qp's to suspend */</div><div class='add'>+	wait_event_timeout(iwdev-&gt;suspend_wq,</div><div class='add'>+			   !atomic_read(&amp;iwdev-&gt;vsi.qp_suspend_reqs),</div><div class='add'>+			   msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS));</div><div class='add'>+	irdma_ws_reset(&amp;iwdev-&gt;vsi);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void icrdma_fill_qos_info(struct irdma_l2params *l2params,</div><div class='add'>+			 struct iidc_rdma_qos_params *qos_info)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	l2params-&gt;num_tc = qos_info-&gt;num_tc;</div><div class='add'>+	l2params-&gt;vsi_prio_type = qos_info-&gt;vport_priority_type;</div><div class='add'>+	l2params-&gt;vsi_rel_bw = qos_info-&gt;vport_relative_bw;</div><div class='add'>+	for (i = 0; i &lt; l2params-&gt;num_tc; i++) {</div><div class='add'>+		l2params-&gt;tc_info[i].egress_virt_up =</div><div class='add'>+			qos_info-&gt;tc_info[i].egress_virt_up;</div><div class='add'>+		l2params-&gt;tc_info[i].ingress_virt_up =</div><div class='add'>+			qos_info-&gt;tc_info[i].ingress_virt_up;</div><div class='add'>+		l2params-&gt;tc_info[i].prio_type = qos_info-&gt;tc_info[i].prio_type;</div><div class='add'>+		l2params-&gt;tc_info[i].rel_bw = qos_info-&gt;tc_info[i].rel_bw;</div><div class='add'>+		l2params-&gt;tc_info[i].tc_ctx = qos_info-&gt;tc_info[i].tc_ctx;</div><div class='add'>+	}</div><div class='add'>+	for (i = 0; i &lt; IIDC_MAX_USER_PRIORITY; i++)</div><div class='add'>+		l2params-&gt;up2tc[i] = qos_info-&gt;up2tc[i];</div><div class='add'>+	if (qos_info-&gt;pfc_mode == IIDC_DSCP_PFC_MODE) {</div><div class='add'>+		l2params-&gt;dscp_mode = true;</div><div class='add'>+		memcpy(l2params-&gt;dscp_map, qos_info-&gt;dscp_map, sizeof(l2params-&gt;dscp_map));</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void icrdma_iidc_event_handler(struct iidc_rdma_core_dev_info *cdev_info,</div><div class='add'>+				     struct iidc_rdma_event *event)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = dev_get_drvdata(&amp;cdev_info-&gt;adev-&gt;dev);</div><div class='add'>+	struct irdma_l2params l2params = {};</div><div class='add'>+</div><div class='add'>+	if (*event-&gt;type &amp; BIT(IIDC_RDMA_EVENT_AFTER_MTU_CHANGE)) {</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev, "CLNT: new MTU = %d\n", iwdev-&gt;netdev-&gt;mtu);</div><div class='add'>+		if (iwdev-&gt;vsi.mtu != iwdev-&gt;netdev-&gt;mtu) {</div><div class='add'>+			l2params.mtu = iwdev-&gt;netdev-&gt;mtu;</div><div class='add'>+			l2params.mtu_changed = true;</div><div class='add'>+			irdma_log_invalid_mtu(l2params.mtu, &amp;iwdev-&gt;rf-&gt;sc_dev);</div><div class='add'>+			irdma_change_l2params(&amp;iwdev-&gt;vsi, &amp;l2params);</div><div class='add'>+		}</div><div class='add'>+	} else if (*event-&gt;type &amp; BIT(IIDC_RDMA_EVENT_BEFORE_TC_CHANGE)) {</div><div class='add'>+		if (iwdev-&gt;vsi.tc_change_pending)</div><div class='add'>+			return;</div><div class='add'>+</div><div class='add'>+		icrdma_prep_tc_change(iwdev);</div><div class='add'>+	} else if (*event-&gt;type &amp; BIT(IIDC_RDMA_EVENT_AFTER_TC_CHANGE)) {</div><div class='add'>+		struct iidc_rdma_priv_dev_info *idc_priv = cdev_info-&gt;iidc_priv;</div><div class='add'>+</div><div class='add'>+		if (!iwdev-&gt;vsi.tc_change_pending)</div><div class='add'>+			return;</div><div class='add'>+</div><div class='add'>+		l2params.tc_changed = true;</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev, "CLNT: TC Change\n");</div><div class='add'>+</div><div class='add'>+		icrdma_fill_qos_info(&amp;l2params, &amp;idc_priv-&gt;qos_info);</div><div class='add'>+		if (iwdev-&gt;rf-&gt;protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)</div><div class='add'>+			iwdev-&gt;dcb_vlan_mode =</div><div class='add'>+				l2params.num_tc &gt; 1 &amp;&amp; !l2params.dscp_mode;</div><div class='add'>+		irdma_change_l2params(&amp;iwdev-&gt;vsi, &amp;l2params);</div><div class='add'>+	} else if (*event-&gt;type &amp; BIT(IIDC_RDMA_EVENT_CRIT_ERR)) {</div><div class='add'>+		ibdev_warn(&amp;iwdev-&gt;ibdev, "ICE OICR event notification: oicr = 0x%08x\n",</div><div class='add'>+			   event-&gt;reg);</div><div class='add'>+		if (event-&gt;reg &amp; IRDMAPFINT_OICR_PE_CRITERR_M) {</div><div class='add'>+			u32 pe_criterr;</div><div class='add'>+</div><div class='add'>+			pe_criterr = readl(iwdev-&gt;rf-&gt;sc_dev.hw_regs[IRDMA_GLPE_CRITERR]);</div><div class='add'>+#define IRDMA_Q1_RESOURCE_ERR 0x0001024d</div><div class='add'>+			if (pe_criterr != IRDMA_Q1_RESOURCE_ERR) {</div><div class='add'>+				ibdev_err(&amp;iwdev-&gt;ibdev, "critical PE Error, GLPE_CRITERR=0x%08x\n",</div><div class='add'>+					  pe_criterr);</div><div class='add'>+				iwdev-&gt;rf-&gt;reset = true;</div><div class='add'>+			} else {</div><div class='add'>+				ibdev_warn(&amp;iwdev-&gt;ibdev, "Q1 Resource Check\n");</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+		if (event-&gt;reg &amp; IRDMAPFINT_OICR_HMC_ERR_M) {</div><div class='add'>+			ibdev_err(&amp;iwdev-&gt;ibdev, "HMC Error\n");</div><div class='add'>+			iwdev-&gt;rf-&gt;reset = true;</div><div class='add'>+		}</div><div class='add'>+		if (event-&gt;reg &amp; IRDMAPFINT_OICR_PE_PUSH_M) {</div><div class='add'>+			ibdev_err(&amp;iwdev-&gt;ibdev, "PE Push Error\n");</div><div class='add'>+			iwdev-&gt;rf-&gt;reset = true;</div><div class='add'>+		}</div><div class='add'>+		if (iwdev-&gt;rf-&gt;reset)</div><div class='add'>+			iwdev-&gt;rf-&gt;gen_ops.request_reset(iwdev-&gt;rf);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * icrdma_lan_register_qset - Register qset with LAN driver</div><div class='add'>+ * @vsi: vsi structure</div><div class='add'>+ * @tc_node: Traffic class node</div><div class='add'>+ */</div><div class='add'>+static int icrdma_lan_register_qset(struct irdma_sc_vsi *vsi,</div><div class='add'>+				    struct irdma_ws_node *tc_node)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = vsi-&gt;back_vsi;</div><div class='add'>+	struct iidc_rdma_core_dev_info *cdev_info = iwdev-&gt;rf-&gt;cdev;</div><div class='add'>+	struct iidc_rdma_qset_params qset = {};</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	qset.qs_handle = tc_node-&gt;qs_handle;</div><div class='add'>+	qset.tc = tc_node-&gt;traffic_class;</div><div class='add'>+	qset.vport_id = vsi-&gt;vsi_idx;</div><div class='add'>+	ret = ice_add_rdma_qset(cdev_info, &amp;qset);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev, "WS: LAN alloc_res for rdma qset failed.\n");</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	tc_node-&gt;l2_sched_node_id = qset.teid;</div><div class='add'>+	vsi-&gt;qos[tc_node-&gt;user_pri].l2_sched_node_id = qset.teid;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * icrdma_lan_unregister_qset - Unregister qset with LAN driver</div><div class='add'>+ * @vsi: vsi structure</div><div class='add'>+ * @tc_node: Traffic class node</div><div class='add'>+ */</div><div class='add'>+static void icrdma_lan_unregister_qset(struct irdma_sc_vsi *vsi,</div><div class='add'>+				       struct irdma_ws_node *tc_node)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = vsi-&gt;back_vsi;</div><div class='add'>+	struct iidc_rdma_core_dev_info *cdev_info = iwdev-&gt;rf-&gt;cdev;</div><div class='add'>+	struct iidc_rdma_qset_params qset = {};</div><div class='add'>+</div><div class='add'>+	qset.qs_handle = tc_node-&gt;qs_handle;</div><div class='add'>+	qset.tc = tc_node-&gt;traffic_class;</div><div class='add'>+	qset.vport_id = vsi-&gt;vsi_idx;</div><div class='add'>+	qset.teid = tc_node-&gt;l2_sched_node_id;</div><div class='add'>+</div><div class='add'>+	if (ice_del_rdma_qset(cdev_info, &amp;qset))</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev, "WS: LAN free_res for rdma qset failed.\n");</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * icrdma_request_reset - Request a reset</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ */</div><div class='add'>+static void icrdma_request_reset(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	ibdev_warn(&amp;rf-&gt;iwdev-&gt;ibdev, "Requesting a reset\n");</div><div class='add'>+	ice_rdma_request_reset(rf-&gt;cdev, IIDC_FUNC_RESET);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int icrdma_init_interrupts(struct irdma_pci_f *rf, struct iidc_rdma_core_dev_info *cdev)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	rf-&gt;msix_count = num_online_cpus() + IRDMA_NUM_AEQ_MSIX;</div><div class='add'>+	rf-&gt;msix_entries = kcalloc(rf-&gt;msix_count, sizeof(*rf-&gt;msix_entries),</div><div class='add'>+				   GFP_KERNEL);</div><div class='add'>+	if (!rf-&gt;msix_entries)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; rf-&gt;msix_count; i++)</div><div class='add'>+		if (ice_alloc_rdma_qvector(cdev, &amp;rf-&gt;msix_entries[i]))</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+	if (i &lt; IRDMA_MIN_MSIX) {</div><div class='add'>+		while (--i &gt;= 0)</div><div class='add'>+			ice_free_rdma_qvector(cdev, &amp;rf-&gt;msix_entries[i]);</div><div class='add'>+</div><div class='add'>+		kfree(rf-&gt;msix_entries);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rf-&gt;msix_count = i;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void icrdma_deinit_interrupts(struct irdma_pci_f *rf, struct iidc_rdma_core_dev_info *cdev)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; rf-&gt;msix_count; i++)</div><div class='add'>+		ice_free_rdma_qvector(cdev, &amp;rf-&gt;msix_entries[i]);</div><div class='add'>+</div><div class='add'>+	kfree(rf-&gt;msix_entries);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void icrdma_fill_device_info(struct irdma_device *iwdev,</div><div class='add'>+				    struct iidc_rdma_core_dev_info *cdev_info)</div><div class='add'>+{</div><div class='add'>+	struct iidc_rdma_priv_dev_info *idc_priv = cdev_info-&gt;iidc_priv;</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+</div><div class='add'>+	rf-&gt;sc_dev.hw = &amp;rf-&gt;hw;</div><div class='add'>+	rf-&gt;iwdev = iwdev;</div><div class='add'>+	rf-&gt;cdev = cdev_info;</div><div class='add'>+	rf-&gt;hw.hw_addr = idc_priv-&gt;hw_addr;</div><div class='add'>+	rf-&gt;pcidev = cdev_info-&gt;pdev;</div><div class='add'>+	rf-&gt;hw.device = &amp;rf-&gt;pcidev-&gt;dev;</div><div class='add'>+	rf-&gt;pf_id = idc_priv-&gt;pf_id;</div><div class='add'>+	rf-&gt;rdma_ver = IRDMA_GEN_2;</div><div class='add'>+	rf-&gt;sc_dev.hw_attrs.uk_attrs.hw_rev = IRDMA_GEN_2;</div><div class='add'>+	rf-&gt;sc_dev.is_pf = true;</div><div class='add'>+	rf-&gt;sc_dev.privileged = true;</div><div class='add'>+</div><div class='add'>+	rf-&gt;gen_ops.register_qset = icrdma_lan_register_qset;</div><div class='add'>+	rf-&gt;gen_ops.unregister_qset = icrdma_lan_unregister_qset;</div><div class='add'>+</div><div class='add'>+	rf-&gt;default_vsi.vsi_idx = idc_priv-&gt;vport_id;</div><div class='add'>+	rf-&gt;protocol_used =</div><div class='add'>+		cdev_info-&gt;rdma_protocol == IIDC_RDMA_PROTOCOL_ROCEV2 ?</div><div class='add'>+			IRDMA_ROCE_PROTOCOL_ONLY : IRDMA_IWARP_PROTOCOL_ONLY;</div><div class='add'>+	rf-&gt;rsrc_profile = IRDMA_HMC_PROFILE_DEFAULT;</div><div class='add'>+	rf-&gt;rst_to = IRDMA_RST_TIMEOUT_HZ;</div><div class='add'>+	rf-&gt;gen_ops.request_reset = icrdma_request_reset;</div><div class='add'>+	rf-&gt;limits_sel = 7;</div><div class='add'>+	mutex_init(&amp;rf-&gt;ah_tbl_lock);</div><div class='add'>+</div><div class='add'>+	iwdev-&gt;netdev = idc_priv-&gt;netdev;</div><div class='add'>+	iwdev-&gt;vsi_num = idc_priv-&gt;vport_id;</div><div class='add'>+	iwdev-&gt;init_state = INITIAL_STATE;</div><div class='add'>+	iwdev-&gt;roce_cwnd = IRDMA_ROCE_CWND_DEFAULT;</div><div class='add'>+	iwdev-&gt;roce_ackcreds = IRDMA_ROCE_ACKCREDS_DEFAULT;</div><div class='add'>+	iwdev-&gt;rcv_wnd = IRDMA_CM_DEFAULT_RCV_WND_SCALED;</div><div class='add'>+	iwdev-&gt;rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;</div><div class='add'>+	if (iwdev-&gt;rf-&gt;protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)</div><div class='add'>+		iwdev-&gt;roce_mode = true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int icrdma_probe(struct auxiliary_device *aux_dev, const struct auxiliary_device_id *id)</div><div class='add'>+{</div><div class='add'>+	struct iidc_rdma_core_auxiliary_dev *iidc_adev;</div><div class='add'>+	struct iidc_rdma_core_dev_info *cdev_info;</div><div class='add'>+	struct iidc_rdma_priv_dev_info *idc_priv;</div><div class='add'>+	struct irdma_l2params l2params = {};</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct irdma_pci_f *rf;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	iidc_adev = container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev);</div><div class='add'>+	cdev_info = iidc_adev-&gt;cdev_info;</div><div class='add'>+	idc_priv = cdev_info-&gt;iidc_priv;</div><div class='add'>+</div><div class='add'>+	iwdev = ib_alloc_device(irdma_device, ibdev);</div><div class='add'>+	if (!iwdev)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	iwdev-&gt;rf = kzalloc(sizeof(*rf), GFP_KERNEL);</div><div class='add'>+	if (!iwdev-&gt;rf) {</div><div class='add'>+		ib_dealloc_device(&amp;iwdev-&gt;ibdev);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	icrdma_fill_device_info(iwdev, cdev_info);</div><div class='add'>+	rf = iwdev-&gt;rf;</div><div class='add'>+</div><div class='add'>+	err = icrdma_init_interrupts(rf, cdev_info);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_init_interrupts;</div><div class='add'>+</div><div class='add'>+	err = irdma_ctrl_init_hw(rf);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_ctrl_init;</div><div class='add'>+</div><div class='add'>+	l2params.mtu = iwdev-&gt;netdev-&gt;mtu;</div><div class='add'>+	icrdma_fill_qos_info(&amp;l2params, &amp;idc_priv-&gt;qos_info);</div><div class='add'>+	if (iwdev-&gt;rf-&gt;protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)</div><div class='add'>+		iwdev-&gt;dcb_vlan_mode = l2params.num_tc &gt; 1 &amp;&amp; !l2params.dscp_mode;</div><div class='add'>+</div><div class='add'>+	err = irdma_rt_init_hw(iwdev, &amp;l2params);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_rt_init;</div><div class='add'>+</div><div class='add'>+	err = irdma_ib_register_device(iwdev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_ibreg;</div><div class='add'>+</div><div class='add'>+	ice_rdma_update_vsi_filter(cdev_info, iwdev-&gt;vsi_num, true);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev, "INIT: Gen2 PF[%d] device probe success\n", PCI_FUNC(rf-&gt;pcidev-&gt;devfn));</div><div class='add'>+	auxiliary_set_drvdata(aux_dev, iwdev);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_ibreg:</div><div class='add'>+	irdma_rt_deinit_hw(iwdev);</div><div class='add'>+err_rt_init:</div><div class='add'>+	irdma_ctrl_deinit_hw(rf);</div><div class='add'>+err_ctrl_init:</div><div class='add'>+	icrdma_deinit_interrupts(rf, cdev_info);</div><div class='add'>+err_init_interrupts:</div><div class='add'>+	mutex_destroy(&amp;rf-&gt;ah_tbl_lock);</div><div class='add'>+	kfree(rf);</div><div class='add'>+	ib_dealloc_device(&amp;iwdev-&gt;ibdev);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void icrdma_remove(struct auxiliary_device *aux_dev)</div><div class='add'>+{</div><div class='add'>+	struct iidc_rdma_core_auxiliary_dev *idc_adev =</div><div class='add'>+		container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev);</div><div class='add'>+	struct iidc_rdma_core_dev_info *cdev_info = idc_adev-&gt;cdev_info;</div><div class='add'>+	struct irdma_device *iwdev = auxiliary_get_drvdata(aux_dev);</div><div class='add'>+	u8 rdma_ver = iwdev-&gt;rf-&gt;rdma_ver;</div><div class='add'>+</div><div class='add'>+	ice_rdma_update_vsi_filter(cdev_info, iwdev-&gt;vsi_num, false);</div><div class='add'>+	irdma_ib_unregister_device(iwdev);</div><div class='add'>+	icrdma_deinit_interrupts(iwdev-&gt;rf, cdev_info);</div><div class='add'>+	mutex_destroy(&amp;iwdev-&gt;rf-&gt;ah_tbl_lock);</div><div class='add'>+</div><div class='add'>+	kfree(iwdev-&gt;rf);</div><div class='add'>+</div><div class='add'>+	pr_debug("INIT: Gen[%d] func[%d] device remove success\n",</div><div class='add'>+		 rdma_ver, PCI_FUNC(cdev_info-&gt;pdev-&gt;devfn));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct auxiliary_device_id icrdma_auxiliary_id_table[] = {</div><div class='add'>+	{.name = "ice.iwarp", },</div><div class='add'>+	{.name = "ice.roce", },</div><div class='add'>+	{},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+MODULE_DEVICE_TABLE(auxiliary, icrdma_auxiliary_id_table);</div><div class='add'>+</div><div class='add'>+struct iidc_rdma_core_auxiliary_drv icrdma_core_auxiliary_drv = {</div><div class='add'>+	.adrv = {</div><div class='add'>+	    .name = "gen_2",</div><div class='add'>+	    .id_table = icrdma_auxiliary_id_table,</div><div class='add'>+	    .probe = icrdma_probe,</div><div class='add'>+	    .remove = icrdma_remove,</div><div class='add'>+	},</div><div class='add'>+	.event_handler = icrdma_iidc_event_handler,</div><div class='add'>+};</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/ig3rdma_hw.c b/drivers/infiniband/hw/irdma/ig3rdma_hw.c<br/>new file mode 100644<br/>index 000000000000..2e8bb475e22a<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/ig3rdma_hw.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/ig3rdma_hw.c</a></div><div class='hunk'>@@ -0,0 +1,170 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2018 - 2024 Intel Corporation */</div><div class='add'>+#include "osdep.h"</div><div class='add'>+#include "type.h"</div><div class='add'>+#include "protos.h"</div><div class='add'>+#include "ig3rdma_hw.h"</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ig3rdma_ena_irq - Enable interrupt</div><div class='add'>+ * @dev: pointer to the device structure</div><div class='add'>+ * @idx: vector index</div><div class='add'>+ */</div><div class='add'>+static void ig3rdma_ena_irq(struct irdma_sc_dev *dev, u32 idx)</div><div class='add'>+{</div><div class='add'>+	u32 val;</div><div class='add'>+	u32 int_stride = 1; /* one u32 per register */</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;is_pf)</div><div class='add'>+		int_stride = 0x400;</div><div class='add'>+	else</div><div class='add'>+		idx--; /* VFs use DYN_CTL_N */</div><div class='add'>+</div><div class='add'>+	val = FIELD_PREP(IRDMA_GLINT_DYN_CTL_INTENA, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMA_GLINT_DYN_CTL_CLEARPBA, 1);</div><div class='add'>+</div><div class='add'>+	writel(val, dev-&gt;hw_regs[IRDMA_GLINT_DYN_CTL] + (idx * int_stride));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ig3rdma_disable_irq - Disable interrupt</div><div class='add'>+ * @dev: pointer to the device structure</div><div class='add'>+ * @idx: vector index</div><div class='add'>+ */</div><div class='add'>+static void ig3rdma_disable_irq(struct irdma_sc_dev *dev, u32 idx)</div><div class='add'>+{</div><div class='add'>+	u32 int_stride = 1; /* one u32 per register */</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;is_pf)</div><div class='add'>+		int_stride = 0x400;</div><div class='add'>+	else</div><div class='add'>+		idx--; /* VFs use DYN_CTL_N */</div><div class='add'>+</div><div class='add'>+	writel(0, dev-&gt;hw_regs[IRDMA_GLINT_DYN_CTL] + (idx * int_stride));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct irdma_irq_ops ig3rdma_irq_ops = {</div><div class='add'>+	.irdma_dis_irq = ig3rdma_disable_irq,</div><div class='add'>+	.irdma_en_irq = ig3rdma_ena_irq,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct irdma_hw_stat_map ig3rdma_hw_stat_map[] = {</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RXVLANERR] =	{   0, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXOCTS] =	{   8, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXPKTS] =	{  16, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXDISCARD] =	{  24, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXTRUNC] =	{  32, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXFRAGS] =	{  40, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXMCOCTS] =	{  48, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] =	{  56, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXOCTS] =	{  64, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXPKTS] =	{  72, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXDISCARD] =	{  80, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXTRUNC] =	{  88, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXFRAGS] =	{  96, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXMCOCTS] =	{ 104, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] =	{ 112, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXOCTS] =	{ 120, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXPKTS] =	{ 128, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXFRAGS] =	{ 136, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXMCOCTS] =	{ 144, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] =	{ 152, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXOCTS] =	{ 160, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXPKTS] =	{ 168, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXFRAGS] =	{ 176, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXMCOCTS] =	{ 184, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] =	{ 192, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] =	{ 200, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] =	{ 208, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPRTXSEG] =	{ 216, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPRXOPTERR] =	{ 224, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] =	{ 232, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPTXSEG] =	{ 240, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPRXSEGS] =	{ 248, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_UDPRXPKTS] =	{ 256, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_UDPTXPKTS] =	{ 264, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMARXWRS] =	{ 272, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMARXRDS] =	{ 280, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMARXSNDS] =	{ 288, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMATXWRS] =	{ 296, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMATXRDS] =	{ 304, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMATXSNDS] =	{ 312, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMAVBND] =	{ 320, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMAVINV] =	{ 328, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] = { 336, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] =	{ 344, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] =	{ 352, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TXNPCNPSENT] =	{ 360, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RNR_SENT] =	{ 368, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RNR_RCVD] =	{ 376, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMAORDLMTCNT] =	{ 384, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMAIRDLMTCNT] =	{ 392, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMARXATS] =	{ 408, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMATXATS] =	{ 416, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_NAKSEQERR] =	{ 424, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_NAKSEQERR_IMPLIED] = { 432, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RTO] =		{ 440, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RXOOOPKTS] =	{ 448, 0, 0 },</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_ICRCERR] =		{ 456, 0, 0 },</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+void ig3rdma_init_hw(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	dev-&gt;irq_ops = &amp;ig3rdma_irq_ops;</div><div class='add'>+	dev-&gt;hw_stats_map = ig3rdma_hw_stat_map;</div><div class='add'>+</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.hw_rev = IRDMA_GEN_3;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_wq_frags = IG3RDMA_MAX_WQ_FRAGMENT_COUNT;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_read_sges = IG3RDMA_MAX_SGE_RD;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_sq_chunk = IRDMA_MAX_QUANTA_PER_WR;</div><div class='add'>+	dev-&gt;hw_attrs.first_hw_vf_fpm_id = 0;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_vf_fpm_id = IG3_MAX_APFS + IG3_MAX_AVFS;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_64_BYTE_CQE;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_CQE_TIMESTAMPING;</div><div class='add'>+</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_SRQ;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_RTS_AE |</div><div class='add'>+						IRDMA_FEATURE_CQ_RESIZE;</div><div class='add'>+	dev-&gt;hw_attrs.page_size_cap = SZ_4K | SZ_2M | SZ_1G;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_ird = IG3RDMA_MAX_IRD_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_ord = IG3RDMA_MAX_ORD_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.max_stat_inst = IG3RDMA_MAX_STATS_COUNT;</div><div class='add'>+	dev-&gt;hw_attrs.max_stat_idx = IRDMA_HW_STAT_INDEX_MAX_GEN_3;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.min_hw_wq_size = IG3RDMA_MIN_WQ_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_srq_quanta = IRDMA_SRQ_MAX_QUANTA;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.max_hw_inline = IG3RDMA_MAX_INLINE_DATA_SIZE;</div><div class='add'>+	dev-&gt;hw_attrs.max_hw_device_pages =</div><div class='add'>+		dev-&gt;is_pf ? IG3RDMA_MAX_PF_PUSH_PAGE_COUNT : IG3RDMA_MAX_VF_PUSH_PAGE_COUNT;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void __iomem *__ig3rdma_get_reg_addr(struct irdma_mmio_region *region, u64 reg_offset)</div><div class='add'>+{</div><div class='add'>+	if (reg_offset &gt;= region-&gt;offset &amp;&amp;</div><div class='add'>+	    reg_offset &lt; (region-&gt;offset + region-&gt;len)) {</div><div class='add'>+		reg_offset -= region-&gt;offset;</div><div class='add'>+</div><div class='add'>+		return region-&gt;addr + reg_offset;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void __iomem *ig3rdma_get_reg_addr(struct irdma_hw *hw, u64 reg_offset)</div><div class='add'>+{</div><div class='add'>+	u8 __iomem *reg_addr;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	reg_addr = __ig3rdma_get_reg_addr(&amp;hw-&gt;rdma_reg, reg_offset);</div><div class='add'>+	if (reg_addr)</div><div class='add'>+		return reg_addr;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; hw-&gt;num_io_regions; i++) {</div><div class='add'>+		reg_addr = __ig3rdma_get_reg_addr(&amp;hw-&gt;io_regs[i], reg_offset);</div><div class='add'>+		if (reg_addr)</div><div class='add'>+			return reg_addr;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	WARN_ON_ONCE(1);</div><div class='add'>+</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/ig3rdma_hw.h b/drivers/infiniband/hw/irdma/ig3rdma_hw.h<br/>new file mode 100644<br/>index 000000000000..03d5f1188789<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/ig3rdma_hw.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/ig3rdma_hw.h</a></div><div class='hunk'>@@ -0,0 +1,32 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2021 - 2024 Intel Corporation */</div><div class='add'>+#ifndef IG3RDMA_HW_H</div><div class='add'>+#define IG3RDMA_HW_H</div><div class='add'>+</div><div class='add'>+#define IG3_MAX_APFS 1</div><div class='add'>+#define IG3_MAX_AVFS 0</div><div class='add'>+</div><div class='add'>+#define IG3_PF_RDMA_REGION_OFFSET 0xBC00000</div><div class='add'>+#define IG3_PF_RDMA_REGION_LEN 0x401000</div><div class='add'>+#define IG3_VF_RDMA_REGION_OFFSET 0x8C00</div><div class='add'>+#define IG3_VF_RDMA_REGION_LEN 0x8400</div><div class='add'>+</div><div class='add'>+enum ig3rdma_device_caps_const {</div><div class='add'>+	IG3RDMA_MAX_WQ_FRAGMENT_COUNT		= 14,</div><div class='add'>+	IG3RDMA_MAX_SGE_RD			= 14,</div><div class='add'>+</div><div class='add'>+	IG3RDMA_MAX_STATS_COUNT			= 128,</div><div class='add'>+</div><div class='add'>+	IG3RDMA_MAX_IRD_SIZE			= 64,</div><div class='add'>+	IG3RDMA_MAX_ORD_SIZE			= 64,</div><div class='add'>+	IG3RDMA_MIN_WQ_SIZE			= 16 /* WQEs */,</div><div class='add'>+	IG3RDMA_MAX_INLINE_DATA_SIZE		= 216,</div><div class='add'>+	IG3RDMA_MAX_PF_PUSH_PAGE_COUNT		= 8192,</div><div class='add'>+	IG3RDMA_MAX_VF_PUSH_PAGE_COUNT		= 16,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+void __iomem *ig3rdma_get_reg_addr(struct irdma_hw *hw, u64 reg_offset);</div><div class='add'>+int ig3rdma_vchnl_send_sync(struct irdma_sc_dev *dev, u8 *msg, u16 len,</div><div class='add'>+			    u8 *recv_msg, u16 *recv_len);</div><div class='add'>+</div><div class='add'>+#endif /* IG3RDMA_HW_H*/</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/ig3rdma_if.c b/drivers/infiniband/hw/irdma/ig3rdma_if.c<br/>new file mode 100644<br/>index 000000000000..e1d6670d9396<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/ig3rdma_if.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/ig3rdma_if.c</a></div><div class='hunk'>@@ -0,0 +1,236 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2023 - 2024 Intel Corporation */</div><div class='add'>+</div><div class='add'>+#include "main.h"</div><div class='add'>+#include &lt;linux/net/intel/iidc_rdma_idpf.h&gt;</div><div class='add'>+#include "ig3rdma_hw.h"</div><div class='add'>+</div><div class='add'>+static void ig3rdma_idc_core_event_handler(struct iidc_rdma_core_dev_info *cdev_info,</div><div class='add'>+					   struct iidc_rdma_event *event)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = auxiliary_get_drvdata(cdev_info-&gt;adev);</div><div class='add'>+</div><div class='add'>+	if (*event-&gt;type &amp; BIT(IIDC_RDMA_EVENT_WARN_RESET)) {</div><div class='add'>+		rf-&gt;reset = true;</div><div class='add'>+		rf-&gt;sc_dev.vchnl_up = false;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int ig3rdma_vchnl_send_sync(struct irdma_sc_dev *dev, u8 *msg, u16 len,</div><div class='add'>+			    u8 *recv_msg, u16 *recv_len)</div><div class='add'>+{</div><div class='add'>+	struct iidc_rdma_core_dev_info *cdev_info = dev_to_rf(dev)-&gt;cdev;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = idpf_idc_rdma_vc_send_sync(cdev_info, msg, len, recv_msg,</div><div class='add'>+					 recv_len);</div><div class='add'>+	if (ret == -ETIMEDOUT) {</div><div class='add'>+		ibdev_err(&amp;(dev_to_rf(dev)-&gt;iwdev-&gt;ibdev),</div><div class='add'>+			  "Virtual channel Req &lt;-&gt; Resp completion timeout\n");</div><div class='add'>+		dev-&gt;vchnl_up = false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ig3rdma_vchnl_init(struct irdma_pci_f *rf,</div><div class='add'>+			      struct iidc_rdma_core_dev_info *cdev_info,</div><div class='add'>+			      u8 *rdma_ver)</div><div class='add'>+{</div><div class='add'>+	struct iidc_rdma_priv_dev_info *idc_priv = cdev_info-&gt;iidc_priv;</div><div class='add'>+	struct irdma_vchnl_init_info virt_info;</div><div class='add'>+	u8 gen = rf-&gt;rdma_ver;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	rf-&gt;vchnl_wq = alloc_ordered_workqueue("irdma-virtchnl-wq", 0);</div><div class='add'>+	if (!rf-&gt;vchnl_wq)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	mutex_init(&amp;rf-&gt;sc_dev.vchnl_mutex);</div><div class='add'>+</div><div class='add'>+	virt_info.is_pf = !idc_priv-&gt;ftype;</div><div class='add'>+	virt_info.hw_rev = gen;</div><div class='add'>+	virt_info.privileged = gen == IRDMA_GEN_2;</div><div class='add'>+	virt_info.vchnl_wq = rf-&gt;vchnl_wq;</div><div class='add'>+	ret = irdma_sc_vchnl_init(&amp;rf-&gt;sc_dev, &amp;virt_info);</div><div class='add'>+	if (ret) {</div><div class='add'>+		destroy_workqueue(rf-&gt;vchnl_wq);</div><div class='add'>+		mutex_destroy(&amp;rf-&gt;sc_dev.vchnl_mutex);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*rdma_ver = rf-&gt;sc_dev.hw_attrs.uk_attrs.hw_rev;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ig3rdma_request_reset - Request a reset</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ */</div><div class='add'>+static void ig3rdma_request_reset(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	ibdev_warn(&amp;rf-&gt;iwdev-&gt;ibdev, "Requesting a reset\n");</div><div class='add'>+	idpf_idc_request_reset(rf-&gt;cdev, IIDC_FUNC_RESET);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ig3rdma_cfg_regions(struct irdma_hw *hw,</div><div class='add'>+			       struct iidc_rdma_core_dev_info *cdev_info)</div><div class='add'>+{</div><div class='add'>+	struct iidc_rdma_priv_dev_info *idc_priv = cdev_info-&gt;iidc_priv;</div><div class='add'>+	struct pci_dev *pdev = cdev_info-&gt;pdev;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	switch (idc_priv-&gt;ftype) {</div><div class='add'>+	case IIDC_FUNCTION_TYPE_PF:</div><div class='add'>+		hw-&gt;rdma_reg.len = IG3_PF_RDMA_REGION_LEN;</div><div class='add'>+		hw-&gt;rdma_reg.offset = IG3_PF_RDMA_REGION_OFFSET;</div><div class='add'>+		break;</div><div class='add'>+	case IIDC_FUNCTION_TYPE_VF:</div><div class='add'>+		hw-&gt;rdma_reg.len = IG3_VF_RDMA_REGION_LEN;</div><div class='add'>+		hw-&gt;rdma_reg.offset = IG3_VF_RDMA_REGION_OFFSET;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -ENODEV;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hw-&gt;rdma_reg.addr = ioremap(pci_resource_start(pdev, 0) + hw-&gt;rdma_reg.offset,</div><div class='add'>+				    hw-&gt;rdma_reg.len);</div><div class='add'>+</div><div class='add'>+	if (!hw-&gt;rdma_reg.addr)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	hw-&gt;num_io_regions = le16_to_cpu(idc_priv-&gt;num_memory_regions);</div><div class='add'>+	hw-&gt;io_regs = kcalloc(hw-&gt;num_io_regions,</div><div class='add'>+			      sizeof(struct irdma_mmio_region), GFP_KERNEL);</div><div class='add'>+</div><div class='add'>+	if (!hw-&gt;io_regs) {</div><div class='add'>+		iounmap(hw-&gt;rdma_reg.addr);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; hw-&gt;num_io_regions; i++) {</div><div class='add'>+		hw-&gt;io_regs[i].addr =</div><div class='add'>+			idc_priv-&gt;mapped_mem_regions[i].region_addr;</div><div class='add'>+		hw-&gt;io_regs[i].len =</div><div class='add'>+			le64_to_cpu(idc_priv-&gt;mapped_mem_regions[i].size);</div><div class='add'>+		hw-&gt;io_regs[i].offset =</div><div class='add'>+			le64_to_cpu(idc_priv-&gt;mapped_mem_regions[i].start_offset);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ig3rdma_decfg_rf(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hw *hw = &amp;rf-&gt;hw;</div><div class='add'>+</div><div class='add'>+	mutex_destroy(&amp;rf-&gt;ah_tbl_lock);</div><div class='add'>+	destroy_workqueue(rf-&gt;vchnl_wq);</div><div class='add'>+	mutex_destroy(&amp;rf-&gt;sc_dev.vchnl_mutex);</div><div class='add'>+	kfree(hw-&gt;io_regs);</div><div class='add'>+	iounmap(hw-&gt;rdma_reg.addr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ig3rdma_cfg_rf(struct irdma_pci_f *rf,</div><div class='add'>+			  struct iidc_rdma_core_dev_info *cdev_info)</div><div class='add'>+{</div><div class='add'>+	struct iidc_rdma_priv_dev_info *idc_priv = cdev_info-&gt;iidc_priv;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	rf-&gt;sc_dev.hw = &amp;rf-&gt;hw;</div><div class='add'>+	rf-&gt;cdev = cdev_info;</div><div class='add'>+	rf-&gt;pcidev = cdev_info-&gt;pdev;</div><div class='add'>+	rf-&gt;hw.device = &amp;rf-&gt;pcidev-&gt;dev;</div><div class='add'>+	rf-&gt;msix_count = idc_priv-&gt;msix_count;</div><div class='add'>+	rf-&gt;msix_entries = idc_priv-&gt;msix_entries;</div><div class='add'>+</div><div class='add'>+	err = ig3rdma_vchnl_init(rf, cdev_info, &amp;rf-&gt;rdma_ver);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = ig3rdma_cfg_regions(&amp;rf-&gt;hw, cdev_info);</div><div class='add'>+	if (err) {</div><div class='add'>+		destroy_workqueue(rf-&gt;vchnl_wq);</div><div class='add'>+		mutex_destroy(&amp;rf-&gt;sc_dev.vchnl_mutex);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rf-&gt;protocol_used = IRDMA_ROCE_PROTOCOL_ONLY;</div><div class='add'>+	rf-&gt;rsrc_profile = IRDMA_HMC_PROFILE_DEFAULT;</div><div class='add'>+	rf-&gt;rst_to = IRDMA_RST_TIMEOUT_HZ;</div><div class='add'>+	rf-&gt;gen_ops.request_reset = ig3rdma_request_reset;</div><div class='add'>+	rf-&gt;limits_sel = 7;</div><div class='add'>+	mutex_init(&amp;rf-&gt;ah_tbl_lock);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ig3rdma_core_probe(struct auxiliary_device *aux_dev,</div><div class='add'>+			      const struct auxiliary_device_id *id)</div><div class='add'>+{</div><div class='add'>+	struct iidc_rdma_core_auxiliary_dev *idc_adev =</div><div class='add'>+		container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev);</div><div class='add'>+	struct iidc_rdma_core_dev_info *cdev_info = idc_adev-&gt;cdev_info;</div><div class='add'>+	struct irdma_pci_f *rf;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	rf = kzalloc(sizeof(*rf), GFP_KERNEL);</div><div class='add'>+	if (!rf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	err = ig3rdma_cfg_rf(rf, cdev_info);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_cfg_rf;</div><div class='add'>+</div><div class='add'>+	err = irdma_ctrl_init_hw(rf);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_ctrl_init;</div><div class='add'>+</div><div class='add'>+	auxiliary_set_drvdata(aux_dev, rf);</div><div class='add'>+</div><div class='add'>+	err = idpf_idc_vport_dev_ctrl(cdev_info, true);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_vport_ctrl;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_vport_ctrl:</div><div class='add'>+	irdma_ctrl_deinit_hw(rf);</div><div class='add'>+err_ctrl_init:</div><div class='add'>+	ig3rdma_decfg_rf(rf);</div><div class='add'>+err_cfg_rf:</div><div class='add'>+	kfree(rf);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ig3rdma_core_remove(struct auxiliary_device *aux_dev)</div><div class='add'>+{</div><div class='add'>+	struct iidc_rdma_core_auxiliary_dev *idc_adev =</div><div class='add'>+		container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev);</div><div class='add'>+	struct iidc_rdma_core_dev_info *cdev_info = idc_adev-&gt;cdev_info;</div><div class='add'>+	struct irdma_pci_f *rf = auxiliary_get_drvdata(aux_dev);</div><div class='add'>+</div><div class='add'>+	idpf_idc_vport_dev_ctrl(cdev_info, false);</div><div class='add'>+	irdma_ctrl_deinit_hw(rf);</div><div class='add'>+	ig3rdma_decfg_rf(rf);</div><div class='add'>+	kfree(rf);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct auxiliary_device_id ig3rdma_core_auxiliary_id_table[] = {</div><div class='add'>+	{.name = "idpf.8086.rdma.core", },</div><div class='add'>+	{},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+MODULE_DEVICE_TABLE(auxiliary, ig3rdma_core_auxiliary_id_table);</div><div class='add'>+</div><div class='add'>+struct iidc_rdma_core_auxiliary_drv ig3rdma_core_auxiliary_drv = {</div><div class='add'>+	.adrv = {</div><div class='add'>+		.name = "core",</div><div class='add'>+		.id_table = ig3rdma_core_auxiliary_id_table,</div><div class='add'>+		.probe = ig3rdma_core_probe,</div><div class='add'>+		.remove = ig3rdma_core_remove,</div><div class='add'>+	},</div><div class='add'>+	.event_handler = ig3rdma_idc_core_event_handler,</div><div class='add'>+};</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/irdma.h b/drivers/infiniband/hw/irdma/irdma.h<br/>new file mode 100644<br/>index 000000000000..ff938a01d70c<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/irdma.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/irdma.h</a></div><div class='hunk'>@@ -0,0 +1,174 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2017 - 2021 Intel Corporation */</div><div class='add'>+#ifndef IRDMA_H</div><div class='add'>+#define IRDMA_H</div><div class='add'>+</div><div class='add'>+#define IRDMA_WQEALLOC_WQE_DESC_INDEX GENMASK(31, 20)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPTAIL_WQTAIL GENMASK(10, 0)</div><div class='add'>+#define IRDMA_CQPTAIL_CQP_OP_ERR BIT(31)</div><div class='add'>+</div><div class='add'>+#define IRDMA_CQPERRCODES_CQP_MINOR_CODE GENMASK(15, 0)</div><div class='add'>+#define IRDMA_CQPERRCODES_CQP_MAJOR_CODE GENMASK(31, 16)</div><div class='add'>+#define IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE GENMASK(5, 4)</div><div class='add'>+#define IRDMA_GLINT_RATE_INTERVAL GENMASK(5, 0)</div><div class='add'>+#define IRDMA_GLINT_RATE_INTRL_ENA BIT(6)</div><div class='add'>+#define IRDMA_GLINT_DYN_CTL_INTENA BIT(0)</div><div class='add'>+#define IRDMA_GLINT_DYN_CTL_CLEARPBA BIT(1)</div><div class='add'>+#define IRDMA_GLINT_DYN_CTL_ITR_INDX GENMASK(4, 3)</div><div class='add'>+#define IRDMA_GLINT_DYN_CTL_INTERVAL GENMASK(16, 5)</div><div class='add'>+#define IRDMA_GLINT_CEQCTL_ITR_INDX GENMASK(12, 11)</div><div class='add'>+#define IRDMA_GLINT_CEQCTL_CAUSE_ENA BIT(30)</div><div class='add'>+#define IRDMA_GLINT_CEQCTL_MSIX_INDX GENMASK(10, 0)</div><div class='add'>+#define IRDMA_PFINT_AEQCTL_MSIX_INDX GENMASK(10, 0)</div><div class='add'>+#define IRDMA_PFINT_AEQCTL_ITR_INDX GENMASK(12, 11)</div><div class='add'>+#define IRDMA_PFINT_AEQCTL_CAUSE_ENA BIT(30)</div><div class='add'>+#define IRDMA_PFHMC_PDINV_PMSDIDX GENMASK(11, 0)</div><div class='add'>+#define IRDMA_PFHMC_PDINV_PMSDPARTSEL BIT(15)</div><div class='add'>+#define IRDMA_PFHMC_PDINV_PMPDIDX GENMASK(24, 16)</div><div class='add'>+#define IRDMA_PFHMC_SDDATALOW_PMSDVALID BIT(0)</div><div class='add'>+#define IRDMA_PFHMC_SDDATALOW_PMSDTYPE BIT(1)</div><div class='add'>+#define IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT GENMASK(11, 2)</div><div class='add'>+#define IRDMA_PFHMC_SDDATALOW_PMSDDATALOW GENMASK(31, 12)</div><div class='add'>+#define IRDMA_PFHMC_SDCMD_PMSDWR BIT(31)</div><div class='add'>+</div><div class='add'>+#define IRDMA_INVALID_CQ_IDX 0xffffffff</div><div class='add'>+#define IRDMA_Q_INVALID_IDX 0xffff</div><div class='add'>+</div><div class='add'>+enum irdma_dyn_idx_t {</div><div class='add'>+	IRDMA_IDX_ITR0 = 0,</div><div class='add'>+	IRDMA_IDX_ITR1 = 1,</div><div class='add'>+	IRDMA_IDX_ITR2 = 2,</div><div class='add'>+	IRDMA_IDX_NOITR = 3,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_registers {</div><div class='add'>+	IRDMA_CQPTAIL,</div><div class='add'>+	IRDMA_CQPDB,</div><div class='add'>+	IRDMA_CCQPSTATUS,</div><div class='add'>+	IRDMA_CCQPHIGH,</div><div class='add'>+	IRDMA_CCQPLOW,</div><div class='add'>+	IRDMA_CQARM,</div><div class='add'>+	IRDMA_CQACK,</div><div class='add'>+	IRDMA_AEQALLOC,</div><div class='add'>+	IRDMA_CQPERRCODES,</div><div class='add'>+	IRDMA_WQEALLOC,</div><div class='add'>+	IRDMA_GLINT_DYN_CTL,</div><div class='add'>+	IRDMA_DB_ADDR_OFFSET,</div><div class='add'>+	IRDMA_GLPCI_LBARCTRL,</div><div class='add'>+	IRDMA_GLPE_CPUSTATUS0,</div><div class='add'>+	IRDMA_GLPE_CPUSTATUS1,</div><div class='add'>+	IRDMA_GLPE_CPUSTATUS2,</div><div class='add'>+	IRDMA_PFINT_AEQCTL,</div><div class='add'>+	IRDMA_GLINT_CEQCTL,</div><div class='add'>+	IRDMA_VSIQF_PE_CTL1,</div><div class='add'>+	IRDMA_PFHMC_PDINV,</div><div class='add'>+	IRDMA_GLHMC_VFPDINV,</div><div class='add'>+	IRDMA_GLPE_CRITERR,</div><div class='add'>+	IRDMA_GLINT_RATE,</div><div class='add'>+	IRDMA_MAX_REGS, /* Must be last entry */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_shifts {</div><div class='add'>+	IRDMA_CCQPSTATUS_CCQP_DONE_S,</div><div class='add'>+	IRDMA_CCQPSTATUS_CCQP_ERR_S,</div><div class='add'>+	IRDMA_CQPSQ_STAG_PDID_S,</div><div class='add'>+	IRDMA_CQPSQ_CQ_CEQID_S,</div><div class='add'>+	IRDMA_CQPSQ_CQ_CQID_S,</div><div class='add'>+	IRDMA_COMMIT_FPM_CQCNT_S,</div><div class='add'>+	IRDMA_CQPSQ_UPESD_HMCFNID_S,</div><div class='add'>+	IRDMA_MAX_SHIFTS,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_masks {</div><div class='add'>+	IRDMA_CCQPSTATUS_CCQP_DONE_M,</div><div class='add'>+	IRDMA_CCQPSTATUS_CCQP_ERR_M,</div><div class='add'>+	IRDMA_CQPSQ_STAG_PDID_M,</div><div class='add'>+	IRDMA_CQPSQ_CQ_CEQID_M,</div><div class='add'>+	IRDMA_CQPSQ_CQ_CQID_M,</div><div class='add'>+	IRDMA_COMMIT_FPM_CQCNT_M,</div><div class='add'>+	IRDMA_CQPSQ_UPESD_HMCFNID_M,</div><div class='add'>+	IRDMA_MAX_MASKS, /* Must be last entry */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define IRDMA_MAX_MGS_PER_CTX	8</div><div class='add'>+</div><div class='add'>+struct irdma_mcast_grp_ctx_entry_info {</div><div class='add'>+	u32 qp_id;</div><div class='add'>+	bool valid_entry;</div><div class='add'>+	u16 dest_port;</div><div class='add'>+	u32 use_cnt;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_mcast_grp_info {</div><div class='add'>+	u8 dest_mac_addr[ETH_ALEN];</div><div class='add'>+	u16 vlan_id;</div><div class='add'>+	u16 hmc_fcn_id;</div><div class='add'>+	bool ipv4_valid:1;</div><div class='add'>+	bool vlan_valid:1;</div><div class='add'>+	u16 mg_id;</div><div class='add'>+	u32 no_of_mgs;</div><div class='add'>+	u32 dest_ip_addr[4];</div><div class='add'>+	u16 qs_handle;</div><div class='add'>+	struct irdma_dma_mem dma_mem_mc;</div><div class='add'>+	struct irdma_mcast_grp_ctx_entry_info mg_ctx_info[IRDMA_MAX_MGS_PER_CTX];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_vers {</div><div class='add'>+	IRDMA_GEN_RSVD,</div><div class='add'>+	IRDMA_GEN_1,</div><div class='add'>+	IRDMA_GEN_2,</div><div class='add'>+	IRDMA_GEN_3,</div><div class='add'>+	IRDMA_GEN_NEXT,</div><div class='add'>+	IRDMA_GEN_MAX = IRDMA_GEN_NEXT-1</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_uk_attrs {</div><div class='add'>+	u64 feature_flags;</div><div class='add'>+	u32 max_hw_wq_frags;</div><div class='add'>+	u32 max_hw_read_sges;</div><div class='add'>+	u32 max_hw_inline;</div><div class='add'>+	u32 max_hw_rq_quanta;</div><div class='add'>+	u32 max_hw_wq_quanta;</div><div class='add'>+	u32 min_hw_cq_size;</div><div class='add'>+	u32 max_hw_cq_size;</div><div class='add'>+	u32 max_hw_srq_quanta;</div><div class='add'>+	u16 max_hw_sq_chunk;</div><div class='add'>+	u16 min_hw_wq_size;</div><div class='add'>+	u8 hw_rev;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_hw_attrs {</div><div class='add'>+	struct irdma_uk_attrs uk_attrs;</div><div class='add'>+	u64 max_hw_outbound_msg_size;</div><div class='add'>+	u64 max_hw_inbound_msg_size;</div><div class='add'>+	u64 max_mr_size;</div><div class='add'>+	u64 page_size_cap;</div><div class='add'>+	u32 min_hw_qp_id;</div><div class='add'>+	u32 min_hw_aeq_size;</div><div class='add'>+	u32 max_hw_aeq_size;</div><div class='add'>+	u32 min_hw_ceq_size;</div><div class='add'>+	u32 max_hw_ceq_size;</div><div class='add'>+	u32 max_hw_device_pages;</div><div class='add'>+	u32 max_hw_vf_fpm_id;</div><div class='add'>+	u32 first_hw_vf_fpm_id;</div><div class='add'>+	u32 max_hw_ird;</div><div class='add'>+	u32 max_hw_ord;</div><div class='add'>+	u32 max_hw_wqes;</div><div class='add'>+	u32 max_hw_pds;</div><div class='add'>+	u32 max_hw_ena_vf_count;</div><div class='add'>+	u32 max_qp_wr;</div><div class='add'>+	u32 max_pe_ready_count;</div><div class='add'>+	u32 max_done_count;</div><div class='add'>+	u32 max_sleep_count;</div><div class='add'>+	u32 max_cqp_compl_wait_time_ms;</div><div class='add'>+	u32 min_hw_srq_id;</div><div class='add'>+	u16 max_stat_inst;</div><div class='add'>+	u16 max_stat_idx;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+void i40iw_init_hw(struct irdma_sc_dev *dev);</div><div class='add'>+void icrdma_init_hw(struct irdma_sc_dev *dev);</div><div class='add'>+void ig3rdma_init_hw(struct irdma_sc_dev *dev);</div><div class='add'>+void __iomem *ig3rdma_get_reg_addr(struct irdma_hw *hw, u64 reg_offset);</div><div class='add'>+#endif /* IRDMA_H*/</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/main.c b/drivers/infiniband/hw/irdma/main.c<br/>new file mode 100644<br/>index 000000000000..95957d52883d<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/main.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/main.c</a></div><div class='hunk'>@@ -0,0 +1,211 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#include "main.h"</div><div class='add'>+#include &lt;linux/net/intel/iidc_rdma_idpf.h&gt;</div><div class='add'>+</div><div class='add'>+MODULE_ALIAS("i40iw");</div><div class='add'>+MODULE_DESCRIPTION("Intel(R) Ethernet Protocol Driver for RDMA");</div><div class='add'>+MODULE_LICENSE("Dual BSD/GPL");</div><div class='add'>+</div><div class='add'>+static struct notifier_block irdma_inetaddr_notifier = {</div><div class='add'>+	.notifier_call = irdma_inetaddr_event</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static struct notifier_block irdma_inetaddr6_notifier = {</div><div class='add'>+	.notifier_call = irdma_inet6addr_event</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static struct notifier_block irdma_net_notifier = {</div><div class='add'>+	.notifier_call = irdma_net_event</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static struct notifier_block irdma_netdevice_notifier = {</div><div class='add'>+	.notifier_call = irdma_netdevice_event</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static void irdma_register_notifiers(void)</div><div class='add'>+{</div><div class='add'>+	register_inetaddr_notifier(&amp;irdma_inetaddr_notifier);</div><div class='add'>+	register_inet6addr_notifier(&amp;irdma_inetaddr6_notifier);</div><div class='add'>+	register_netevent_notifier(&amp;irdma_net_notifier);</div><div class='add'>+	register_netdevice_notifier(&amp;irdma_netdevice_notifier);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_unregister_notifiers(void)</div><div class='add'>+{</div><div class='add'>+	unregister_netevent_notifier(&amp;irdma_net_notifier);</div><div class='add'>+	unregister_inetaddr_notifier(&amp;irdma_inetaddr_notifier);</div><div class='add'>+	unregister_inet6addr_notifier(&amp;irdma_inetaddr6_notifier);</div><div class='add'>+	unregister_netdevice_notifier(&amp;irdma_netdevice_notifier);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void irdma_log_invalid_mtu(u16 mtu, struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	if (mtu &lt; IRDMA_MIN_MTU_IPV4)</div><div class='add'>+		ibdev_warn(to_ibdev(dev), "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 576 for IPv4\n", mtu);</div><div class='add'>+	else if (mtu &lt; IRDMA_MIN_MTU_IPV6)</div><div class='add'>+		ibdev_warn(to_ibdev(dev), "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 1280 for IPv6\\n", mtu);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ig3rdma_idc_vport_event_handler(struct iidc_rdma_vport_dev_info *cdev_info,</div><div class='add'>+					    struct iidc_rdma_event *event)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = auxiliary_get_drvdata(cdev_info-&gt;adev);</div><div class='add'>+	struct irdma_l2params l2params = {};</div><div class='add'>+</div><div class='add'>+	if (*event-&gt;type &amp; BIT(IIDC_RDMA_EVENT_AFTER_MTU_CHANGE)) {</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev, "CLNT: new MTU = %d\n", iwdev-&gt;netdev-&gt;mtu);</div><div class='add'>+		if (iwdev-&gt;vsi.mtu != iwdev-&gt;netdev-&gt;mtu) {</div><div class='add'>+			l2params.mtu = iwdev-&gt;netdev-&gt;mtu;</div><div class='add'>+			l2params.mtu_changed = true;</div><div class='add'>+			irdma_log_invalid_mtu(l2params.mtu, &amp;iwdev-&gt;rf-&gt;sc_dev);</div><div class='add'>+			irdma_change_l2params(&amp;iwdev-&gt;vsi, &amp;l2params);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ig3rdma_vport_probe(struct auxiliary_device *aux_dev,</div><div class='add'>+			       const struct auxiliary_device_id *id)</div><div class='add'>+{</div><div class='add'>+	struct iidc_rdma_vport_auxiliary_dev *idc_adev =</div><div class='add'>+		container_of(aux_dev, struct iidc_rdma_vport_auxiliary_dev, adev);</div><div class='add'>+	struct auxiliary_device *aux_core_dev = idc_adev-&gt;vdev_info-&gt;core_adev;</div><div class='add'>+	struct irdma_pci_f *rf = auxiliary_get_drvdata(aux_core_dev);</div><div class='add'>+	struct irdma_l2params l2params = {};</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (!rf) {</div><div class='add'>+		WARN_ON_ONCE(1);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+	iwdev = ib_alloc_device(irdma_device, ibdev);</div><div class='add'>+	/* Fill iwdev info */</div><div class='add'>+	iwdev-&gt;is_vport = true;</div><div class='add'>+	iwdev-&gt;rf = rf;</div><div class='add'>+	iwdev-&gt;vport_id = idc_adev-&gt;vdev_info-&gt;vport_id;</div><div class='add'>+	iwdev-&gt;netdev = idc_adev-&gt;vdev_info-&gt;netdev;</div><div class='add'>+	iwdev-&gt;init_state = INITIAL_STATE;</div><div class='add'>+	iwdev-&gt;roce_cwnd = IRDMA_ROCE_CWND_DEFAULT;</div><div class='add'>+	iwdev-&gt;roce_ackcreds = IRDMA_ROCE_ACKCREDS_DEFAULT;</div><div class='add'>+	iwdev-&gt;rcv_wnd = IRDMA_CM_DEFAULT_RCV_WND_SCALED;</div><div class='add'>+	iwdev-&gt;rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;</div><div class='add'>+	iwdev-&gt;roce_mode = true;</div><div class='add'>+	iwdev-&gt;push_mode = false;</div><div class='add'>+</div><div class='add'>+	l2params.mtu = iwdev-&gt;netdev-&gt;mtu;</div><div class='add'>+</div><div class='add'>+	err = irdma_rt_init_hw(iwdev, &amp;l2params);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_rt_init;</div><div class='add'>+</div><div class='add'>+	err = irdma_ib_register_device(iwdev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_ibreg;</div><div class='add'>+</div><div class='add'>+	auxiliary_set_drvdata(aux_dev, iwdev);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+		  "INIT: Gen[%d] vport[%d] probe success. dev_name = %s, core_dev_name = %s, netdev=%s\n",</div><div class='add'>+		  rf-&gt;rdma_ver, idc_adev-&gt;vdev_info-&gt;vport_id,</div><div class='add'>+		  dev_name(&amp;aux_dev-&gt;dev),</div><div class='add'>+		  dev_name(&amp;idc_adev-&gt;vdev_info-&gt;core_adev-&gt;dev),</div><div class='add'>+		  netdev_name(idc_adev-&gt;vdev_info-&gt;netdev));</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+err_ibreg:</div><div class='add'>+	irdma_rt_deinit_hw(iwdev);</div><div class='add'>+err_rt_init:</div><div class='add'>+	ib_dealloc_device(&amp;iwdev-&gt;ibdev);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void ig3rdma_vport_remove(struct auxiliary_device *aux_dev)</div><div class='add'>+{</div><div class='add'>+	struct iidc_rdma_vport_auxiliary_dev *idc_adev =</div><div class='add'>+		container_of(aux_dev, struct iidc_rdma_vport_auxiliary_dev, adev);</div><div class='add'>+	struct irdma_device *iwdev = auxiliary_get_drvdata(aux_dev);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+		  "INIT: Gen[%d] dev_name = %s, core_dev_name = %s, netdev=%s\n",</div><div class='add'>+		  iwdev-&gt;rf-&gt;rdma_ver, dev_name(&amp;aux_dev-&gt;dev),</div><div class='add'>+		  dev_name(&amp;idc_adev-&gt;vdev_info-&gt;core_adev-&gt;dev),</div><div class='add'>+		  netdev_name(idc_adev-&gt;vdev_info-&gt;netdev));</div><div class='add'>+</div><div class='add'>+	irdma_ib_unregister_device(iwdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct auxiliary_device_id ig3rdma_vport_auxiliary_id_table[] = {</div><div class='add'>+	{.name = "idpf.8086.rdma.vdev", },</div><div class='add'>+	{},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+MODULE_DEVICE_TABLE(auxiliary, ig3rdma_vport_auxiliary_id_table);</div><div class='add'>+</div><div class='add'>+static struct iidc_rdma_vport_auxiliary_drv ig3rdma_vport_auxiliary_drv = {</div><div class='add'>+	.adrv = {</div><div class='add'>+		.name = "vdev",</div><div class='add'>+		.id_table = ig3rdma_vport_auxiliary_id_table,</div><div class='add'>+		.probe = ig3rdma_vport_probe,</div><div class='add'>+		.remove = ig3rdma_vport_remove,</div><div class='add'>+	},</div><div class='add'>+	.event_handler = ig3rdma_idc_vport_event_handler,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+</div><div class='add'>+static int __init irdma_init_module(void)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = auxiliary_driver_register(&amp;i40iw_auxiliary_drv);</div><div class='add'>+	if (ret) {</div><div class='add'>+		pr_err("Failed i40iw(gen_1) auxiliary_driver_register() ret=%d\n",</div><div class='add'>+		       ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = auxiliary_driver_register(&amp;icrdma_core_auxiliary_drv.adrv);</div><div class='add'>+	if (ret) {</div><div class='add'>+		auxiliary_driver_unregister(&amp;i40iw_auxiliary_drv);</div><div class='add'>+		pr_err("Failed icrdma(gen_2) auxiliary_driver_register() ret=%d\n",</div><div class='add'>+		       ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = auxiliary_driver_register(&amp;ig3rdma_core_auxiliary_drv.adrv);</div><div class='add'>+	if (ret) {</div><div class='add'>+		auxiliary_driver_unregister(&amp;icrdma_core_auxiliary_drv.adrv);</div><div class='add'>+		auxiliary_driver_unregister(&amp;i40iw_auxiliary_drv);</div><div class='add'>+		pr_err("Failed ig3rdma(gen_3) core auxiliary_driver_register() ret=%d\n",</div><div class='add'>+		       ret);</div><div class='add'>+</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = auxiliary_driver_register(&amp;ig3rdma_vport_auxiliary_drv.adrv);</div><div class='add'>+	if (ret) {</div><div class='add'>+		auxiliary_driver_unregister(&amp;ig3rdma_core_auxiliary_drv.adrv);</div><div class='add'>+		auxiliary_driver_unregister(&amp;icrdma_core_auxiliary_drv.adrv);</div><div class='add'>+		auxiliary_driver_unregister(&amp;i40iw_auxiliary_drv);</div><div class='add'>+		pr_err("Failed ig3rdma vport auxiliary_driver_register() ret=%d\n",</div><div class='add'>+		       ret);</div><div class='add'>+</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+	irdma_register_notifiers();</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void __exit irdma_exit_module(void)</div><div class='add'>+{</div><div class='add'>+	irdma_unregister_notifiers();</div><div class='add'>+	auxiliary_driver_unregister(&amp;icrdma_core_auxiliary_drv.adrv);</div><div class='add'>+	auxiliary_driver_unregister(&amp;i40iw_auxiliary_drv);</div><div class='add'>+	auxiliary_driver_unregister(&amp;ig3rdma_core_auxiliary_drv.adrv);</div><div class='add'>+	auxiliary_driver_unregister(&amp;ig3rdma_vport_auxiliary_drv.adrv);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+module_init(irdma_init_module);</div><div class='add'>+module_exit(irdma_exit_module);</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/main.h b/drivers/infiniband/hw/irdma/main.h<br/>new file mode 100644<br/>index 000000000000..baab61e424a2<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/main.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/main.h</a></div><div class='hunk'>@@ -0,0 +1,578 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#ifndef IRDMA_MAIN_H</div><div class='add'>+#define IRDMA_MAIN_H</div><div class='add'>+</div><div class='add'>+#include &lt;linux/ip.h&gt;</div><div class='add'>+#include &lt;linux/tcp.h&gt;</div><div class='add'>+#include &lt;linux/if_vlan.h&gt;</div><div class='add'>+#include &lt;net/addrconf.h&gt;</div><div class='add'>+#include &lt;net/netevent.h&gt;</div><div class='add'>+#include &lt;net/tcp.h&gt;</div><div class='add'>+#include &lt;net/ip6_route.h&gt;</div><div class='add'>+#include &lt;net/flow.h&gt;</div><div class='add'>+#include &lt;net/secure_seq.h&gt;</div><div class='add'>+#include &lt;linux/netdevice.h&gt;</div><div class='add'>+#include &lt;linux/etherdevice.h&gt;</div><div class='add'>+#include &lt;linux/inetdevice.h&gt;</div><div class='add'>+#include &lt;linux/spinlock.h&gt;</div><div class='add'>+#include &lt;linux/kernel.h&gt;</div><div class='add'>+#include &lt;linux/delay.h&gt;</div><div class='add'>+#include &lt;linux/pci.h&gt;</div><div class='add'>+#include &lt;linux/dma-mapping.h&gt;</div><div class='add'>+#include &lt;linux/workqueue.h&gt;</div><div class='add'>+#include &lt;linux/slab.h&gt;</div><div class='add'>+#include &lt;linux/io.h&gt;</div><div class='add'>+#include &lt;linux/crc32c.h&gt;</div><div class='add'>+#include &lt;linux/kthread.h&gt;</div><div class='add'>+#ifndef CONFIG_64BIT</div><div class='add'>+#include &lt;linux/io-64-nonatomic-lo-hi.h&gt;</div><div class='add'>+#endif</div><div class='add'>+#include &lt;linux/auxiliary_bus.h&gt;</div><div class='add'>+#include &lt;linux/net/intel/iidc_rdma.h&gt;</div><div class='add'>+#include &lt;rdma/ib_smi.h&gt;</div><div class='add'>+#include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+#include &lt;rdma/ib_pack.h&gt;</div><div class='add'>+#include &lt;rdma/rdma_cm.h&gt;</div><div class='add'>+#include &lt;rdma/iw_cm.h&gt;</div><div class='add'>+#include &lt;rdma/ib_user_verbs.h&gt;</div><div class='add'>+#include &lt;rdma/ib_umem.h&gt;</div><div class='add'>+#include &lt;rdma/ib_cache.h&gt;</div><div class='add'>+#include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='add'>+#include "osdep.h"</div><div class='add'>+#include "defs.h"</div><div class='add'>+#include "hmc.h"</div><div class='add'>+#include "type.h"</div><div class='add'>+#include "ws.h"</div><div class='add'>+#include "protos.h"</div><div class='add'>+#include "pble.h"</div><div class='add'>+#include "cm.h"</div><div class='add'>+#include &lt;rdma/irdma-abi.h&gt;</div><div class='add'>+#include "verbs.h"</div><div class='add'>+#include "user.h"</div><div class='add'>+#include "puda.h"</div><div class='add'>+</div><div class='add'>+extern struct auxiliary_driver i40iw_auxiliary_drv;</div><div class='add'>+extern struct iidc_rdma_core_auxiliary_drv icrdma_core_auxiliary_drv;</div><div class='add'>+extern struct iidc_rdma_core_auxiliary_drv ig3rdma_core_auxiliary_drv;</div><div class='add'>+</div><div class='add'>+#define IRDMA_FW_VER_DEFAULT	2</div><div class='add'>+#define IRDMA_HW_VER	        2</div><div class='add'>+</div><div class='add'>+#define IRDMA_ARP_ADD		1</div><div class='add'>+#define IRDMA_ARP_DELETE	2</div><div class='add'>+#define IRDMA_ARP_RESOLVE	3</div><div class='add'>+</div><div class='add'>+#define IRDMA_MACIP_ADD		1</div><div class='add'>+#define IRDMA_MACIP_DELETE	2</div><div class='add'>+</div><div class='add'>+#define IW_GEN_3_CCQ_SIZE  (2 * IRDMA_CQP_SW_SQSIZE_2048 + 2)</div><div class='add'>+#define IW_CCQ_SIZE	(IRDMA_CQP_SW_SQSIZE_2048 + 2)</div><div class='add'>+#define IW_CEQ_SIZE	2048</div><div class='add'>+#define IW_AEQ_SIZE	2048</div><div class='add'>+</div><div class='add'>+#define RX_BUF_SIZE	(1536 + 8)</div><div class='add'>+#define IW_REG0_SIZE	(4 * 1024)</div><div class='add'>+#define IW_TX_TIMEOUT	(6 * HZ)</div><div class='add'>+#define IW_FIRST_QPN	1</div><div class='add'>+</div><div class='add'>+#define IW_SW_CONTEXT_ALIGN	1024</div><div class='add'>+</div><div class='add'>+#define MAX_DPC_ITERATIONS	128</div><div class='add'>+</div><div class='add'>+#define IRDMA_EVENT_TIMEOUT_MS		5000</div><div class='add'>+#define IRDMA_VCHNL_EVENT_TIMEOUT	100000</div><div class='add'>+#define IRDMA_RST_TIMEOUT_HZ		4</div><div class='add'>+</div><div class='add'>+#define	IRDMA_NO_QSET	0xffff</div><div class='add'>+</div><div class='add'>+#define IW_CFG_FPM_QP_COUNT		32768</div><div class='add'>+#define IRDMA_MAX_PAGES_PER_FMR		262144</div><div class='add'>+#define IRDMA_MIN_PAGES_PER_FMR		1</div><div class='add'>+#define IRDMA_CQP_COMPL_RQ_WQE_FLUSHED	2</div><div class='add'>+#define IRDMA_CQP_COMPL_SQ_WQE_FLUSHED	3</div><div class='add'>+</div><div class='add'>+#define IRDMA_Q_TYPE_PE_AEQ	0x80</div><div class='add'>+#define IRDMA_Q_INVALID_IDX	0xffff</div><div class='add'>+#define IRDMA_REM_ENDPOINT_TRK_QPID	3</div><div class='add'>+</div><div class='add'>+#define IRDMA_DRV_OPT_ENA_MPA_VER_0		0x00000001</div><div class='add'>+#define IRDMA_DRV_OPT_DISABLE_MPA_CRC		0x00000002</div><div class='add'>+#define IRDMA_DRV_OPT_DISABLE_FIRST_WRITE	0x00000004</div><div class='add'>+#define IRDMA_DRV_OPT_DISABLE_INTF		0x00000008</div><div class='add'>+#define IRDMA_DRV_OPT_ENA_MSI			0x00000010</div><div class='add'>+#define IRDMA_DRV_OPT_DUAL_LOGICAL_PORT		0x00000020</div><div class='add'>+#define IRDMA_DRV_OPT_NO_INLINE_DATA		0x00000080</div><div class='add'>+#define IRDMA_DRV_OPT_DISABLE_INT_MOD		0x00000100</div><div class='add'>+#define IRDMA_DRV_OPT_DISABLE_VIRT_WQ		0x00000200</div><div class='add'>+#define IRDMA_DRV_OPT_ENA_PAU			0x00000400</div><div class='add'>+#define IRDMA_DRV_OPT_MCAST_LOGPORT_MAP		0x00000800</div><div class='add'>+</div><div class='add'>+#define IW_HMC_OBJ_TYPE_NUM	ARRAY_SIZE(iw_hmc_obj_types)</div><div class='add'>+#define IRDMA_ROCE_CWND_DEFAULT			0x400</div><div class='add'>+#define IRDMA_ROCE_ACKCREDS_DEFAULT		0x1E</div><div class='add'>+</div><div class='add'>+#define IRDMA_FLUSH_SQ		BIT(0)</div><div class='add'>+#define IRDMA_FLUSH_RQ		BIT(1)</div><div class='add'>+#define IRDMA_REFLUSH		BIT(2)</div><div class='add'>+#define IRDMA_FLUSH_WAIT	BIT(3)</div><div class='add'>+</div><div class='add'>+#define IRDMA_IRQ_NAME_STR_LEN (64)</div><div class='add'>+</div><div class='add'>+#define IRDMA_NUM_AEQ_MSIX	1</div><div class='add'>+#define IRDMA_MIN_MSIX		2</div><div class='add'>+</div><div class='add'>+enum init_completion_state {</div><div class='add'>+	INVALID_STATE = 0,</div><div class='add'>+	INITIAL_STATE,</div><div class='add'>+	CQP_CREATED,</div><div class='add'>+	HMC_OBJS_CREATED,</div><div class='add'>+	HW_RSRC_INITIALIZED,</div><div class='add'>+	CCQ_CREATED,</div><div class='add'>+	CEQ0_CREATED,</div><div class='add'>+	CEQS_CREATED,</div><div class='add'>+	PBLE_CHUNK_MEM,</div><div class='add'>+	AEQ_CREATED,</div><div class='add'>+	ILQ_CREATED,</div><div class='add'>+	IEQ_CREATED, /* Last state of probe */</div><div class='add'>+	IP_ADDR_REGISTERED,  /* Last state of open */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_rsrc_limits {</div><div class='add'>+	u32 qplimit;</div><div class='add'>+	u32 mrlimit;</div><div class='add'>+	u32 cqlimit;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cqp_err_info {</div><div class='add'>+	u16 maj;</div><div class='add'>+	u16 min;</div><div class='add'>+	const char *desc;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cqp_compl_info {</div><div class='add'>+	u32 op_ret_val;</div><div class='add'>+	u16 maj_err_code;</div><div class='add'>+	u16 min_err_code;</div><div class='add'>+	bool error;</div><div class='add'>+	u8 op_code;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cqp_request {</div><div class='add'>+	struct cqp_cmds_info info;</div><div class='add'>+	wait_queue_head_t waitq;</div><div class='add'>+	struct list_head list;</div><div class='add'>+	refcount_t refcnt;</div><div class='add'>+	void (*callback_fcn)(struct irdma_cqp_request *cqp_request);</div><div class='add'>+	void *param;</div><div class='add'>+	struct irdma_cqp_compl_info compl_info;</div><div class='add'>+	bool request_done; /* READ/WRITE_ONCE macros operate on it */</div><div class='add'>+	bool waiting:1;</div><div class='add'>+	bool dynamic:1;</div><div class='add'>+	bool pending:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cqp {</div><div class='add'>+	struct irdma_sc_cqp sc_cqp;</div><div class='add'>+	spinlock_t req_lock; /* protect CQP request list */</div><div class='add'>+	spinlock_t compl_lock; /* protect CQP completion processing */</div><div class='add'>+	wait_queue_head_t waitq;</div><div class='add'>+	wait_queue_head_t remove_wq;</div><div class='add'>+	struct irdma_dma_mem sq;</div><div class='add'>+	struct irdma_dma_mem host_ctx;</div><div class='add'>+	u64 *scratch_array;</div><div class='add'>+	struct irdma_cqp_request *cqp_requests;</div><div class='add'>+	struct irdma_ooo_cqp_op *oop_op_array;</div><div class='add'>+	struct list_head cqp_avail_reqs;</div><div class='add'>+	struct list_head cqp_pending_reqs;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_ccq {</div><div class='add'>+	struct irdma_sc_cq sc_cq;</div><div class='add'>+	struct irdma_dma_mem mem_cq;</div><div class='add'>+	struct irdma_dma_mem shadow_area;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_ceq {</div><div class='add'>+	struct irdma_sc_ceq sc_ceq;</div><div class='add'>+	struct irdma_dma_mem mem;</div><div class='add'>+	u32 irq;</div><div class='add'>+	u32 msix_idx;</div><div class='add'>+	struct irdma_pci_f *rf;</div><div class='add'>+	struct tasklet_struct dpc_tasklet;</div><div class='add'>+	spinlock_t ce_lock; /* sync cq destroy with cq completion event notification */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_aeq {</div><div class='add'>+	struct irdma_sc_aeq sc_aeq;</div><div class='add'>+	struct irdma_dma_mem mem;</div><div class='add'>+	struct irdma_pble_alloc palloc;</div><div class='add'>+	bool virtual_map;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_arp_entry {</div><div class='add'>+	u32 ip_addr[4];</div><div class='add'>+	u8 mac_addr[ETH_ALEN];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_msix_vector {</div><div class='add'>+	u32 idx;</div><div class='add'>+	u32 irq;</div><div class='add'>+	u32 cpu_affinity;</div><div class='add'>+	u32 ceq_id;</div><div class='add'>+	cpumask_t mask;</div><div class='add'>+	char name[IRDMA_IRQ_NAME_STR_LEN];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_mc_table_info {</div><div class='add'>+	u32 mgn;</div><div class='add'>+	u32 dest_ip[4];</div><div class='add'>+	bool lan_fwd:1;</div><div class='add'>+	bool ipv4_valid:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mc_table_list {</div><div class='add'>+	struct list_head list;</div><div class='add'>+	struct irdma_mc_table_info mc_info;</div><div class='add'>+	struct irdma_mcast_grp_info mc_grp_ctx;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_qv_info {</div><div class='add'>+	u32 v_idx; /* msix_vector */</div><div class='add'>+	u16 ceq_idx;</div><div class='add'>+	u16 aeq_idx;</div><div class='add'>+	u8 itr_idx;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_qvlist_info {</div><div class='add'>+	u32 num_vectors;</div><div class='add'>+	struct irdma_qv_info qv_info[] __counted_by(num_vectors);</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_gen_ops {</div><div class='add'>+	void (*request_reset)(struct irdma_pci_f *rf);</div><div class='add'>+	int (*register_qset)(struct irdma_sc_vsi *vsi,</div><div class='add'>+			     struct irdma_ws_node *tc_node);</div><div class='add'>+	void (*unregister_qset)(struct irdma_sc_vsi *vsi,</div><div class='add'>+				struct irdma_ws_node *tc_node);</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_pci_f {</div><div class='add'>+	bool reset:1;</div><div class='add'>+	bool rsrc_created:1;</div><div class='add'>+	bool msix_shared:1;</div><div class='add'>+	bool hwqp1_rsvd:1;</div><div class='add'>+	u8 rsrc_profile;</div><div class='add'>+	u8 *hmc_info_mem;</div><div class='add'>+	u8 *mem_rsrc;</div><div class='add'>+	u8 rdma_ver;</div><div class='add'>+	u8 rst_to;</div><div class='add'>+	u8 pf_id;</div><div class='add'>+	enum irdma_protocol_used protocol_used;</div><div class='add'>+	u32 sd_type;</div><div class='add'>+	u32 msix_count;</div><div class='add'>+	u32 max_mr;</div><div class='add'>+	u32 max_qp;</div><div class='add'>+	u32 max_cq;</div><div class='add'>+	u32 max_srq;</div><div class='add'>+	u32 next_srq;</div><div class='add'>+	u32 max_ah;</div><div class='add'>+	u32 next_ah;</div><div class='add'>+	u32 max_mcg;</div><div class='add'>+	u32 next_mcg;</div><div class='add'>+	u32 max_pd;</div><div class='add'>+	u32 next_qp;</div><div class='add'>+	u32 next_cq;</div><div class='add'>+	u32 next_pd;</div><div class='add'>+	u32 max_mr_size;</div><div class='add'>+	u32 max_cqe;</div><div class='add'>+	u32 mr_stagmask;</div><div class='add'>+	u32 used_pds;</div><div class='add'>+	u32 used_cqs;</div><div class='add'>+	u32 used_srqs;</div><div class='add'>+	u32 used_mrs;</div><div class='add'>+	u32 used_qps;</div><div class='add'>+	u32 arp_table_size;</div><div class='add'>+	u32 next_arp_index;</div><div class='add'>+	u32 ceqs_count;</div><div class='add'>+	u32 next_ws_node_id;</div><div class='add'>+	u32 max_ws_node_id;</div><div class='add'>+	u32 limits_sel;</div><div class='add'>+	unsigned long *allocated_ws_nodes;</div><div class='add'>+	unsigned long *allocated_qps;</div><div class='add'>+	unsigned long *allocated_cqs;</div><div class='add'>+	unsigned long *allocated_srqs;</div><div class='add'>+	unsigned long *allocated_mrs;</div><div class='add'>+	unsigned long *allocated_pds;</div><div class='add'>+	unsigned long *allocated_mcgs;</div><div class='add'>+	unsigned long *allocated_ahs;</div><div class='add'>+	unsigned long *allocated_arps;</div><div class='add'>+	enum init_completion_state init_state;</div><div class='add'>+	struct irdma_sc_dev sc_dev;</div><div class='add'>+	struct pci_dev *pcidev;</div><div class='add'>+	void *cdev;</div><div class='add'>+	struct irdma_hw hw;</div><div class='add'>+	struct irdma_cqp cqp;</div><div class='add'>+	struct irdma_ccq ccq;</div><div class='add'>+	struct irdma_aeq aeq;</div><div class='add'>+	struct irdma_ceq *ceqlist;</div><div class='add'>+	struct irdma_hmc_pble_rsrc *pble_rsrc;</div><div class='add'>+	struct irdma_arp_entry *arp_table;</div><div class='add'>+	spinlock_t arp_lock; /*protect ARP table access*/</div><div class='add'>+	spinlock_t rsrc_lock; /* protect HW resource array access */</div><div class='add'>+	spinlock_t qptable_lock; /*protect QP table access*/</div><div class='add'>+	spinlock_t cqtable_lock; /*protect CQ table access*/</div><div class='add'>+	struct irdma_qp **qp_table;</div><div class='add'>+	struct irdma_cq **cq_table;</div><div class='add'>+	spinlock_t qh_list_lock; /* protect mc_qht_list */</div><div class='add'>+	struct mc_table_list mc_qht_list;</div><div class='add'>+	struct irdma_msix_vector *iw_msixtbl;</div><div class='add'>+	struct irdma_qvlist_info *iw_qvlist;</div><div class='add'>+	struct tasklet_struct dpc_tasklet;</div><div class='add'>+	struct msix_entry *msix_entries;</div><div class='add'>+	struct irdma_dma_mem obj_mem;</div><div class='add'>+	struct irdma_dma_mem obj_next;</div><div class='add'>+	atomic_t vchnl_msgs;</div><div class='add'>+	wait_queue_head_t vchnl_waitq;</div><div class='add'>+	struct workqueue_struct *cqp_cmpl_wq;</div><div class='add'>+	struct work_struct cqp_cmpl_work;</div><div class='add'>+	struct workqueue_struct *vchnl_wq;</div><div class='add'>+	struct irdma_sc_vsi default_vsi;</div><div class='add'>+	void *back_fcn;</div><div class='add'>+	struct irdma_gen_ops gen_ops;</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	DECLARE_HASHTABLE(ah_hash_tbl, 8);</div><div class='add'>+	struct mutex ah_tbl_lock; /* protect AH hash table access */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_device {</div><div class='add'>+	struct ib_device ibdev;</div><div class='add'>+	struct irdma_pci_f *rf;</div><div class='add'>+	struct net_device *netdev;</div><div class='add'>+	struct workqueue_struct *cleanup_wq;</div><div class='add'>+	struct irdma_sc_vsi vsi;</div><div class='add'>+	struct irdma_cm_core cm_core;</div><div class='add'>+	u32 roce_cwnd;</div><div class='add'>+	u32 roce_ackcreds;</div><div class='add'>+	u32 vendor_id;</div><div class='add'>+	u32 vendor_part_id;</div><div class='add'>+	u32 push_mode;</div><div class='add'>+	u32 rcv_wnd;</div><div class='add'>+	u16 mac_ip_table_idx;</div><div class='add'>+	u16 vsi_num;</div><div class='add'>+	u16 vport_id;</div><div class='add'>+	u8 rcv_wscale;</div><div class='add'>+	u8 iw_status;</div><div class='add'>+	bool roce_mode:1;</div><div class='add'>+	bool roce_dcqcn_en:1;</div><div class='add'>+	bool dcb_vlan_mode:1;</div><div class='add'>+	bool iw_ooo:1;</div><div class='add'>+	bool is_vport:1;</div><div class='add'>+	enum init_completion_state init_state;</div><div class='add'>+</div><div class='add'>+	wait_queue_head_t suspend_wq;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline struct irdma_device *to_iwdev(struct ib_device *ibdev)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibdev, struct irdma_device, ibdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct irdma_ucontext *to_ucontext(struct ib_ucontext *ibucontext)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibucontext, struct irdma_ucontext, ibucontext);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct irdma_user_mmap_entry *</div><div class='add'>+to_irdma_mmap_entry(struct rdma_user_mmap_entry *rdma_entry)</div><div class='add'>+{</div><div class='add'>+	return container_of(rdma_entry, struct irdma_user_mmap_entry,</div><div class='add'>+			    rdma_entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct irdma_pd *to_iwpd(struct ib_pd *ibpd)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibpd, struct irdma_pd, ibpd);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct irdma_ah *to_iwah(struct ib_ah *ibah)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibah, struct irdma_ah, ibah);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct irdma_mr *to_iwmr(struct ib_mr *ibmr)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibmr, struct irdma_mr, ibmr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct irdma_mr *to_iwmw(struct ib_mw *ibmw)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibmw, struct irdma_mr, ibmw);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct irdma_cq *to_iwcq(struct ib_cq *ibcq)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibcq, struct irdma_cq, ibcq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct irdma_qp *to_iwqp(struct ib_qp *ibqp)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibqp, struct irdma_qp, ibqp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct irdma_pci_f *dev_to_rf(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	return container_of(dev, struct irdma_pci_f, sc_dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct irdma_srq *to_iwsrq(struct ib_srq *ibsrq)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibsrq, struct irdma_srq, ibsrq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_alloc_resource - allocate a resource</div><div class='add'>+ * @iwdev: device pointer</div><div class='add'>+ * @resource_array: resource bit array:</div><div class='add'>+ * @max_resources: maximum resource number</div><div class='add'>+ * @req_resources_num: Allocated resource number</div><div class='add'>+ * @next: next free id</div><div class='add'>+ **/</div><div class='add'>+static inline int irdma_alloc_rsrc(struct irdma_pci_f *rf,</div><div class='add'>+				   unsigned long *rsrc_array, u32 max_rsrc,</div><div class='add'>+				   u32 *req_rsrc_num, u32 *next)</div><div class='add'>+{</div><div class='add'>+	u32 rsrc_num;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;rf-&gt;rsrc_lock, flags);</div><div class='add'>+	rsrc_num = find_next_zero_bit(rsrc_array, max_rsrc, *next);</div><div class='add'>+	if (rsrc_num &gt;= max_rsrc) {</div><div class='add'>+		rsrc_num = find_first_zero_bit(rsrc_array, max_rsrc);</div><div class='add'>+		if (rsrc_num &gt;= max_rsrc) {</div><div class='add'>+			spin_unlock_irqrestore(&amp;rf-&gt;rsrc_lock, flags);</div><div class='add'>+			ibdev_dbg(&amp;rf-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "ERR: resource [%d] allocation failed\n",</div><div class='add'>+				  rsrc_num);</div><div class='add'>+			return -EOVERFLOW;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	__set_bit(rsrc_num, rsrc_array);</div><div class='add'>+	*next = rsrc_num + 1;</div><div class='add'>+	if (*next == max_rsrc)</div><div class='add'>+		*next = 0;</div><div class='add'>+	*req_rsrc_num = rsrc_num;</div><div class='add'>+	spin_unlock_irqrestore(&amp;rf-&gt;rsrc_lock, flags);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_free_resource - free a resource</div><div class='add'>+ * @iwdev: device pointer</div><div class='add'>+ * @resource_array: resource array for the resource_num</div><div class='add'>+ * @resource_num: resource number to free</div><div class='add'>+ **/</div><div class='add'>+static inline void irdma_free_rsrc(struct irdma_pci_f *rf,</div><div class='add'>+				   unsigned long *rsrc_array, u32 rsrc_num)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;rf-&gt;rsrc_lock, flags);</div><div class='add'>+	__clear_bit(rsrc_num, rsrc_array);</div><div class='add'>+	spin_unlock_irqrestore(&amp;rf-&gt;rsrc_lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int irdma_ctrl_init_hw(struct irdma_pci_f *rf);</div><div class='add'>+void irdma_ctrl_deinit_hw(struct irdma_pci_f *rf);</div><div class='add'>+int irdma_rt_init_hw(struct irdma_device *iwdev,</div><div class='add'>+		     struct irdma_l2params *l2params);</div><div class='add'>+void irdma_rt_deinit_hw(struct irdma_device *iwdev);</div><div class='add'>+void irdma_qp_add_ref(struct ib_qp *ibqp);</div><div class='add'>+void irdma_qp_rem_ref(struct ib_qp *ibqp);</div><div class='add'>+void irdma_free_lsmm_rsrc(struct irdma_qp *iwqp);</div><div class='add'>+struct ib_qp *irdma_get_qp(struct ib_device *ibdev, int qpn);</div><div class='add'>+void irdma_flush_wqes(struct irdma_qp *iwqp, u32 flush_mask);</div><div class='add'>+void irdma_manage_arp_cache(struct irdma_pci_f *rf,</div><div class='add'>+			    const unsigned char *mac_addr,</div><div class='add'>+			    u32 *ip_addr, bool ipv4, u32 action);</div><div class='add'>+struct irdma_apbvt_entry *irdma_add_apbvt(struct irdma_device *iwdev, u16 port);</div><div class='add'>+void irdma_del_apbvt(struct irdma_device *iwdev,</div><div class='add'>+		     struct irdma_apbvt_entry *entry);</div><div class='add'>+struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp,</div><div class='add'>+							  bool wait);</div><div class='add'>+void irdma_free_cqp_request(struct irdma_cqp *cqp,</div><div class='add'>+			    struct irdma_cqp_request *cqp_request);</div><div class='add'>+void irdma_put_cqp_request(struct irdma_cqp *cqp,</div><div class='add'>+			   struct irdma_cqp_request *cqp_request);</div><div class='add'>+int irdma_alloc_local_mac_entry(struct irdma_pci_f *rf, u16 *mac_tbl_idx);</div><div class='add'>+int irdma_add_local_mac_entry(struct irdma_pci_f *rf, const u8 *mac_addr, u16 idx);</div><div class='add'>+void irdma_del_local_mac_entry(struct irdma_pci_f *rf, u16 idx);</div><div class='add'>+</div><div class='add'>+u32 irdma_initialize_hw_rsrc(struct irdma_pci_f *rf);</div><div class='add'>+void irdma_port_ibevent(struct irdma_device *iwdev);</div><div class='add'>+void irdma_cm_disconn(struct irdma_qp *qp);</div><div class='add'>+</div><div class='add'>+bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd,</div><div class='add'>+			u16 maj_err_code, u16 min_err_code);</div><div class='add'>+int irdma_handle_cqp_op(struct irdma_pci_f *rf,</div><div class='add'>+			struct irdma_cqp_request *cqp_request);</div><div class='add'>+</div><div class='add'>+int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,</div><div class='add'>+		    struct ib_udata *udata);</div><div class='add'>+int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='add'>+			 int attr_mask, struct ib_udata *udata);</div><div class='add'>+void irdma_cq_add_ref(struct ib_cq *ibcq);</div><div class='add'>+void irdma_cq_rem_ref(struct ib_cq *ibcq);</div><div class='add'>+void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq);</div><div class='add'>+void irdma_srq_event(struct irdma_sc_srq *srq);</div><div class='add'>+void irdma_srq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_srq *srq);</div><div class='add'>+void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf);</div><div class='add'>+int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp,</div><div class='add'>+		       struct irdma_modify_qp_info *info, bool wait);</div><div class='add'>+int irdma_qp_suspend_resume(struct irdma_sc_qp *qp, bool suspend);</div><div class='add'>+int irdma_manage_qhash(struct irdma_device *iwdev, struct irdma_cm_info *cminfo,</div><div class='add'>+		       enum irdma_quad_entry_type etype,</div><div class='add'>+		       enum irdma_quad_hash_manage_type mtype, void *cmnode,</div><div class='add'>+		       bool wait);</div><div class='add'>+void irdma_receive_ilq(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *rbuf);</div><div class='add'>+void irdma_free_sqbuf(struct irdma_sc_vsi *vsi, void *bufp);</div><div class='add'>+void irdma_free_qp_rsrc(struct irdma_qp *iwqp);</div><div class='add'>+int irdma_setup_cm_core(struct irdma_device *iwdev, u8 ver);</div><div class='add'>+void irdma_cleanup_cm_core(struct irdma_cm_core *cm_core);</div><div class='add'>+void irdma_next_iw_state(struct irdma_qp *iwqp, u8 state, u8 del_hash, u8 term,</div><div class='add'>+			 u8 term_len);</div><div class='add'>+int irdma_send_syn(struct irdma_cm_node *cm_node, u32 sendack);</div><div class='add'>+int irdma_send_reset(struct irdma_cm_node *cm_node);</div><div class='add'>+struct irdma_cm_node *irdma_find_node(struct irdma_cm_core *cm_core,</div><div class='add'>+				      u16 rem_port, u32 *rem_addr, u16 loc_port,</div><div class='add'>+				      u32 *loc_addr, u16 vlan_id);</div><div class='add'>+int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,</div><div class='add'>+			struct irdma_qp_flush_info *info, bool wait);</div><div class='add'>+void irdma_gen_ae(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,</div><div class='add'>+		  struct irdma_gen_ae_info *info, bool wait);</div><div class='add'>+void irdma_copy_ip_ntohl(u32 *dst, __be32 *src);</div><div class='add'>+void irdma_copy_ip_htonl(__be32 *dst, u32 *src);</div><div class='add'>+u16 irdma_get_vlan_ipv4(u32 *addr);</div><div class='add'>+void irdma_get_vlan_mac_ipv6(u32 *addr, u16 *vlan_id, u8 *mac);</div><div class='add'>+struct ib_mr *irdma_reg_phys_mr(struct ib_pd *ib_pd, u64 addr, u64 size,</div><div class='add'>+				int acc, u64 *iova_start, bool dma_mr);</div><div class='add'>+int irdma_upload_qp_context(struct irdma_qp *iwqp, bool freeze, bool raw);</div><div class='add'>+void irdma_cqp_ce_handler(struct irdma_pci_f *rf, struct irdma_sc_cq *cq);</div><div class='add'>+int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,</div><div class='add'>+		    bool wait,</div><div class='add'>+		    void (*callback_fcn)(struct irdma_cqp_request *cqp_request),</div><div class='add'>+		    void *cb_param);</div><div class='add'>+void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request);</div><div class='add'>+int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event,</div><div class='add'>+			 void *ptr);</div><div class='add'>+int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event,</div><div class='add'>+			  void *ptr);</div><div class='add'>+int irdma_net_event(struct notifier_block *notifier, unsigned long event,</div><div class='add'>+		    void *ptr);</div><div class='add'>+int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,</div><div class='add'>+			  void *ptr);</div><div class='add'>+void irdma_add_ip(struct irdma_device *iwdev);</div><div class='add'>+void cqp_compl_worker(struct work_struct *work);</div><div class='add'>+void irdma_log_invalid_mtu(u16 mtu, struct irdma_sc_dev *dev);</div><div class='add'>+#endif /* IRDMA_MAIN_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/osdep.h b/drivers/infiniband/hw/irdma/osdep.h<br/>new file mode 100644<br/>index 000000000000..3f73ceacccb6<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/osdep.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/osdep.h</a></div><div class='hunk'>@@ -0,0 +1,74 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#ifndef IRDMA_OSDEP_H</div><div class='add'>+#define IRDMA_OSDEP_H</div><div class='add'>+</div><div class='add'>+#include &lt;linux/pci.h&gt;</div><div class='add'>+#include &lt;linux/bitfield.h&gt;</div><div class='add'>+#include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+#include &lt;net/dscp.h&gt;</div><div class='add'>+</div><div class='add'>+#define STATS_TIMER_DELAY	60000</div><div class='add'>+</div><div class='add'>+struct irdma_dma_info {</div><div class='add'>+	dma_addr_t *dmaaddrs;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_dma_mem {</div><div class='add'>+	void *va;</div><div class='add'>+	dma_addr_t pa;</div><div class='add'>+	u32 size;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+struct irdma_virt_mem {</div><div class='add'>+	void *va;</div><div class='add'>+	u32 size;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+struct irdma_sc_vsi;</div><div class='add'>+struct irdma_sc_dev;</div><div class='add'>+struct irdma_sc_qp;</div><div class='add'>+struct irdma_puda_buf;</div><div class='add'>+struct irdma_puda_cmpl_info;</div><div class='add'>+struct irdma_update_sds_info;</div><div class='add'>+struct irdma_hmc_fcn_info;</div><div class='add'>+struct irdma_manage_vf_pble_info;</div><div class='add'>+struct irdma_hw;</div><div class='add'>+struct irdma_pci_f;</div><div class='add'>+</div><div class='add'>+struct ib_device *to_ibdev(struct irdma_sc_dev *dev);</div><div class='add'>+void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp);</div><div class='add'>+enum irdma_status_code irdma_vf_wait_vchnl_resp(struct irdma_sc_dev *dev);</div><div class='add'>+bool irdma_vf_clear_to_send(struct irdma_sc_dev *dev);</div><div class='add'>+void irdma_add_dev_ref(struct irdma_sc_dev *dev);</div><div class='add'>+void irdma_put_dev_ref(struct irdma_sc_dev *dev);</div><div class='add'>+int irdma_ieq_check_mpacrc(const void *addr, u32 len, u32 val);</div><div class='add'>+struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,</div><div class='add'>+				     struct irdma_puda_buf *buf);</div><div class='add'>+void irdma_send_ieq_ack(struct irdma_sc_qp *qp);</div><div class='add'>+void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,</div><div class='add'>+				 u32 seqnum);</div><div class='add'>+int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,</div><div class='add'>+			      struct irdma_puda_buf *buf);</div><div class='add'>+int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,</div><div class='add'>+		      struct irdma_update_sds_info *info);</div><div class='add'>+int irdma_cqp_manage_hmc_fcn_cmd(struct irdma_sc_dev *dev,</div><div class='add'>+				 struct irdma_hmc_fcn_info *hmcfcninfo,</div><div class='add'>+				 u16 *pmf_idx);</div><div class='add'>+int irdma_alloc_query_fpm_buf(struct irdma_sc_dev *dev,</div><div class='add'>+			      struct irdma_dma_mem *mem);</div><div class='add'>+void *irdma_remove_cqp_head(struct irdma_sc_dev *dev);</div><div class='add'>+void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term,</div><div class='add'>+			  u8 term_len);</div><div class='add'>+void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred);</div><div class='add'>+void irdma_terminate_start_timer(struct irdma_sc_qp *qp);</div><div class='add'>+void irdma_terminate_del_timer(struct irdma_sc_qp *qp);</div><div class='add'>+void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi);</div><div class='add'>+void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi);</div><div class='add'>+void wr32(struct irdma_hw *hw, u32 reg, u32 val);</div><div class='add'>+u32 rd32(struct irdma_hw *hw, u32 reg);</div><div class='add'>+u64 rd64(struct irdma_hw *hw, u32 reg);</div><div class='add'>+int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma,</div><div class='add'>+			   u32 pg_cnt);</div><div class='add'>+void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt);</div><div class='add'>+#endif /* IRDMA_OSDEP_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/pble.c b/drivers/infiniband/hw/irdma/pble.c<br/>new file mode 100644<br/>index 000000000000..28dfad7f940c<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/pble.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/pble.c</a></div><div class='hunk'>@@ -0,0 +1,519 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#include "osdep.h"</div><div class='add'>+#include "hmc.h"</div><div class='add'>+#include "defs.h"</div><div class='add'>+#include "type.h"</div><div class='add'>+#include "protos.h"</div><div class='add'>+#include "pble.h"</div><div class='add'>+</div><div class='add'>+static int add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_destroy_pble_prm - destroy prm during module unload</div><div class='add'>+ * @pble_rsrc: pble resources</div><div class='add'>+ */</div><div class='add'>+void irdma_destroy_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)</div><div class='add'>+{</div><div class='add'>+	struct irdma_chunk *chunk;</div><div class='add'>+	struct irdma_pble_prm *pinfo = &amp;pble_rsrc-&gt;pinfo;</div><div class='add'>+</div><div class='add'>+	while (!list_empty(&amp;pinfo-&gt;clist)) {</div><div class='add'>+		chunk = (struct irdma_chunk *) pinfo-&gt;clist.next;</div><div class='add'>+		list_del(&amp;chunk-&gt;list);</div><div class='add'>+		if (chunk-&gt;type == PBLE_SD_PAGED)</div><div class='add'>+			irdma_pble_free_paged_mem(chunk);</div><div class='add'>+		bitmap_free(chunk-&gt;bitmapbuf);</div><div class='add'>+		kfree(chunk-&gt;chunkmem.va);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_hmc_init_pble - Initialize pble resources during module load</div><div class='add'>+ * @dev: irdma_sc_dev struct</div><div class='add'>+ * @pble_rsrc: pble resources</div><div class='add'>+ */</div><div class='add'>+int irdma_hmc_init_pble(struct irdma_sc_dev *dev,</div><div class='add'>+			struct irdma_hmc_pble_rsrc *pble_rsrc)</div><div class='add'>+{</div><div class='add'>+	struct irdma_hmc_info *hmc_info;</div><div class='add'>+	u32 fpm_idx = 0;</div><div class='add'>+	int status = 0;</div><div class='add'>+</div><div class='add'>+	hmc_info = dev-&gt;hmc_info;</div><div class='add'>+	pble_rsrc-&gt;dev = dev;</div><div class='add'>+	pble_rsrc-&gt;fpm_base_addr = hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].base;</div><div class='add'>+	/* Start pble' on 4k boundary */</div><div class='add'>+	if (pble_rsrc-&gt;fpm_base_addr &amp; 0xfff)</div><div class='add'>+		fpm_idx = (4096 - (pble_rsrc-&gt;fpm_base_addr &amp; 0xfff)) &gt;&gt; 3;</div><div class='add'>+	pble_rsrc-&gt;unallocated_pble =</div><div class='add'>+		hmc_info-&gt;hmc_obj[IRDMA_HMC_IW_PBLE].cnt - fpm_idx;</div><div class='add'>+	pble_rsrc-&gt;next_fpm_addr = pble_rsrc-&gt;fpm_base_addr + (fpm_idx &lt;&lt; 3);</div><div class='add'>+	pble_rsrc-&gt;pinfo.pble_shift = PBLE_SHIFT;</div><div class='add'>+</div><div class='add'>+	mutex_init(&amp;pble_rsrc-&gt;pble_mutex_lock);</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;pble_rsrc-&gt;pinfo.prm_lock);</div><div class='add'>+	INIT_LIST_HEAD(&amp;pble_rsrc-&gt;pinfo.clist);</div><div class='add'>+	if (add_pble_prm(pble_rsrc)) {</div><div class='add'>+		irdma_destroy_pble_prm(pble_rsrc);</div><div class='add'>+		status = -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * get_sd_pd_idx -  Returns sd index, pd index and rel_pd_idx from fpm address</div><div class='add'>+ * @pble_rsrc: structure containing fpm address</div><div class='add'>+ * @idx: where to return indexes</div><div class='add'>+ */</div><div class='add'>+static void get_sd_pd_idx(struct irdma_hmc_pble_rsrc *pble_rsrc,</div><div class='add'>+			  struct sd_pd_idx *idx)</div><div class='add'>+{</div><div class='add'>+	idx-&gt;sd_idx = pble_rsrc-&gt;next_fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE;</div><div class='add'>+	idx-&gt;pd_idx = (u32)(pble_rsrc-&gt;next_fpm_addr / IRDMA_HMC_PAGED_BP_SIZE);</div><div class='add'>+	idx-&gt;rel_pd_idx = (idx-&gt;pd_idx % IRDMA_HMC_PD_CNT_IN_SD);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * add_sd_direct - add sd direct for pble</div><div class='add'>+ * @pble_rsrc: pble resource ptr</div><div class='add'>+ * @info: page info for sd</div><div class='add'>+ */</div><div class='add'>+static int add_sd_direct(struct irdma_hmc_pble_rsrc *pble_rsrc,</div><div class='add'>+			 struct irdma_add_page_info *info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = pble_rsrc-&gt;dev;</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+	struct sd_pd_idx *idx = &amp;info-&gt;idx;</div><div class='add'>+	struct irdma_chunk *chunk = info-&gt;chunk;</div><div class='add'>+	struct irdma_hmc_info *hmc_info = info-&gt;hmc_info;</div><div class='add'>+	struct irdma_hmc_sd_entry *sd_entry = info-&gt;sd_entry;</div><div class='add'>+	u32 offset = 0;</div><div class='add'>+</div><div class='add'>+	if (!sd_entry-&gt;valid) {</div><div class='add'>+		ret_code = irdma_add_sd_table_entry(dev-&gt;hw, hmc_info,</div><div class='add'>+						    info-&gt;idx.sd_idx,</div><div class='add'>+						    IRDMA_SD_TYPE_DIRECT,</div><div class='add'>+						    IRDMA_HMC_DIRECT_BP_SIZE);</div><div class='add'>+		if (ret_code)</div><div class='add'>+			return ret_code;</div><div class='add'>+</div><div class='add'>+		chunk-&gt;type = PBLE_SD_CONTIGOUS;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	offset = idx-&gt;rel_pd_idx &lt;&lt; HMC_PAGED_BP_SHIFT;</div><div class='add'>+	chunk-&gt;size = info-&gt;pages &lt;&lt; HMC_PAGED_BP_SHIFT;</div><div class='add'>+	chunk-&gt;vaddr = sd_entry-&gt;u.bp.addr.va + offset;</div><div class='add'>+	chunk-&gt;fpm_addr = pble_rsrc-&gt;next_fpm_addr;</div><div class='add'>+	ibdev_dbg(to_ibdev(dev),</div><div class='add'>+		  "PBLE: chunk_size[%lld] = 0x%llx vaddr=0x%p fpm_addr = %llx\n",</div><div class='add'>+		  chunk-&gt;size, chunk-&gt;size, chunk-&gt;vaddr, chunk-&gt;fpm_addr);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * fpm_to_idx - given fpm address, get pble index</div><div class='add'>+ * @pble_rsrc: pble resource management</div><div class='add'>+ * @addr: fpm address for index</div><div class='add'>+ */</div><div class='add'>+static u32 fpm_to_idx(struct irdma_hmc_pble_rsrc *pble_rsrc, u64 addr)</div><div class='add'>+{</div><div class='add'>+	u64 idx;</div><div class='add'>+</div><div class='add'>+	idx = (addr - (pble_rsrc-&gt;fpm_base_addr)) &gt;&gt; 3;</div><div class='add'>+</div><div class='add'>+	return (u32)idx;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * add_bp_pages - add backing pages for sd</div><div class='add'>+ * @pble_rsrc: pble resource management</div><div class='add'>+ * @info: page info for sd</div><div class='add'>+ */</div><div class='add'>+static int add_bp_pages(struct irdma_hmc_pble_rsrc *pble_rsrc,</div><div class='add'>+			struct irdma_add_page_info *info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = pble_rsrc-&gt;dev;</div><div class='add'>+	u8 *addr;</div><div class='add'>+	struct irdma_dma_mem mem;</div><div class='add'>+	struct irdma_hmc_pd_entry *pd_entry;</div><div class='add'>+	struct irdma_hmc_sd_entry *sd_entry = info-&gt;sd_entry;</div><div class='add'>+	struct irdma_hmc_info *hmc_info = info-&gt;hmc_info;</div><div class='add'>+	struct irdma_chunk *chunk = info-&gt;chunk;</div><div class='add'>+	int status = 0;</div><div class='add'>+	u32 rel_pd_idx = info-&gt;idx.rel_pd_idx;</div><div class='add'>+	u32 pd_idx = info-&gt;idx.pd_idx;</div><div class='add'>+	u32 i;</div><div class='add'>+</div><div class='add'>+	if (irdma_pble_get_paged_mem(chunk, info-&gt;pages))</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	status = irdma_add_sd_table_entry(dev-&gt;hw, hmc_info, info-&gt;idx.sd_idx,</div><div class='add'>+					  IRDMA_SD_TYPE_PAGED,</div><div class='add'>+					  IRDMA_HMC_DIRECT_BP_SIZE);</div><div class='add'>+	if (status)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	addr = chunk-&gt;vaddr;</div><div class='add'>+	for (i = 0; i &lt; info-&gt;pages; i++) {</div><div class='add'>+		mem.pa = (u64)chunk-&gt;dmainfo.dmaaddrs[i];</div><div class='add'>+		mem.size = 4096;</div><div class='add'>+		mem.va = addr;</div><div class='add'>+		pd_entry = &amp;sd_entry-&gt;u.pd_table.pd_entry[rel_pd_idx++];</div><div class='add'>+		if (!pd_entry-&gt;valid) {</div><div class='add'>+			status = irdma_add_pd_table_entry(dev, hmc_info,</div><div class='add'>+							  pd_idx++, &amp;mem);</div><div class='add'>+			if (status)</div><div class='add'>+				goto error;</div><div class='add'>+</div><div class='add'>+			addr += 4096;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	chunk-&gt;fpm_addr = pble_rsrc-&gt;next_fpm_addr;</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+error:</div><div class='add'>+	irdma_pble_free_paged_mem(chunk);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_type - add a sd entry type for sd</div><div class='add'>+ * @dev: irdma_sc_dev struct</div><div class='add'>+ * @idx: index of sd</div><div class='add'>+ * @pages: pages in the sd</div><div class='add'>+ */</div><div class='add'>+static enum irdma_sd_entry_type irdma_get_type(struct irdma_sc_dev *dev,</div><div class='add'>+					       struct sd_pd_idx *idx, u32 pages)</div><div class='add'>+{</div><div class='add'>+	enum irdma_sd_entry_type sd_entry_type;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3)</div><div class='add'>+		sd_entry_type = (!idx-&gt;rel_pd_idx &amp;&amp;</div><div class='add'>+				 pages == IRDMA_HMC_PD_CNT_IN_SD) ?</div><div class='add'>+				 IRDMA_SD_TYPE_DIRECT : IRDMA_SD_TYPE_PAGED;</div><div class='add'>+	else</div><div class='add'>+		sd_entry_type = (!idx-&gt;rel_pd_idx &amp;&amp;</div><div class='add'>+				 pages == IRDMA_HMC_PD_CNT_IN_SD &amp;&amp;</div><div class='add'>+				 dev-&gt;privileged) ?</div><div class='add'>+				 IRDMA_SD_TYPE_DIRECT : IRDMA_SD_TYPE_PAGED;</div><div class='add'>+	return sd_entry_type;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * add_pble_prm - add a sd entry for pble resoure</div><div class='add'>+ * @pble_rsrc: pble resource management</div><div class='add'>+ */</div><div class='add'>+static int add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = pble_rsrc-&gt;dev;</div><div class='add'>+	struct irdma_hmc_sd_entry *sd_entry;</div><div class='add'>+	struct irdma_hmc_info *hmc_info;</div><div class='add'>+	struct irdma_chunk *chunk;</div><div class='add'>+	struct irdma_add_page_info info;</div><div class='add'>+	struct sd_pd_idx *idx = &amp;info.idx;</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+	enum irdma_sd_entry_type sd_entry_type;</div><div class='add'>+	u64 sd_reg_val = 0;</div><div class='add'>+	struct irdma_virt_mem chunkmem;</div><div class='add'>+	u32 pages;</div><div class='add'>+</div><div class='add'>+	if (pble_rsrc-&gt;unallocated_pble &lt; PBLE_PER_PAGE)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	if (pble_rsrc-&gt;next_fpm_addr &amp; 0xfff)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	chunkmem.size = sizeof(*chunk);</div><div class='add'>+	chunkmem.va = kzalloc(chunkmem.size, GFP_KERNEL);</div><div class='add'>+	if (!chunkmem.va)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	chunk = chunkmem.va;</div><div class='add'>+	chunk-&gt;chunkmem = chunkmem;</div><div class='add'>+	hmc_info = dev-&gt;hmc_info;</div><div class='add'>+	chunk-&gt;dev = dev;</div><div class='add'>+	chunk-&gt;fpm_addr = pble_rsrc-&gt;next_fpm_addr;</div><div class='add'>+	get_sd_pd_idx(pble_rsrc, idx);</div><div class='add'>+	sd_entry = &amp;hmc_info-&gt;sd_table.sd_entry[idx-&gt;sd_idx];</div><div class='add'>+	pages = (idx-&gt;rel_pd_idx) ? (IRDMA_HMC_PD_CNT_IN_SD - idx-&gt;rel_pd_idx) :</div><div class='add'>+				    IRDMA_HMC_PD_CNT_IN_SD;</div><div class='add'>+	pages = min(pages, pble_rsrc-&gt;unallocated_pble &gt;&gt; PBLE_512_SHIFT);</div><div class='add'>+	info.chunk = chunk;</div><div class='add'>+	info.hmc_info = hmc_info;</div><div class='add'>+	info.pages = pages;</div><div class='add'>+	info.sd_entry = sd_entry;</div><div class='add'>+	if (!sd_entry-&gt;valid)</div><div class='add'>+		sd_entry_type = irdma_get_type(dev, idx, pages);</div><div class='add'>+	else</div><div class='add'>+		sd_entry_type = sd_entry-&gt;entry_type;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(to_ibdev(dev),</div><div class='add'>+		  "PBLE: pages = %d, unallocated_pble[%d] current_fpm_addr = %llx\n",</div><div class='add'>+		  pages, pble_rsrc-&gt;unallocated_pble,</div><div class='add'>+		  pble_rsrc-&gt;next_fpm_addr);</div><div class='add'>+	ibdev_dbg(to_ibdev(dev), "PBLE: sd_entry_type = %d\n", sd_entry_type);</div><div class='add'>+	if (sd_entry_type == IRDMA_SD_TYPE_DIRECT)</div><div class='add'>+		ret_code = add_sd_direct(pble_rsrc, &amp;info);</div><div class='add'>+</div><div class='add'>+	if (ret_code)</div><div class='add'>+		sd_entry_type = IRDMA_SD_TYPE_PAGED;</div><div class='add'>+	else</div><div class='add'>+		pble_rsrc-&gt;stats_direct_sds++;</div><div class='add'>+</div><div class='add'>+	if (sd_entry_type == IRDMA_SD_TYPE_PAGED) {</div><div class='add'>+		ret_code = add_bp_pages(pble_rsrc, &amp;info);</div><div class='add'>+		if (ret_code)</div><div class='add'>+			goto error;</div><div class='add'>+		else</div><div class='add'>+			pble_rsrc-&gt;stats_paged_sds++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret_code = irdma_prm_add_pble_mem(&amp;pble_rsrc-&gt;pinfo, chunk);</div><div class='add'>+	if (ret_code)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	pble_rsrc-&gt;next_fpm_addr += chunk-&gt;size;</div><div class='add'>+	ibdev_dbg(to_ibdev(dev),</div><div class='add'>+		  "PBLE: next_fpm_addr = %llx chunk_size[%llu] = 0x%llx\n",</div><div class='add'>+		  pble_rsrc-&gt;next_fpm_addr, chunk-&gt;size, chunk-&gt;size);</div><div class='add'>+	pble_rsrc-&gt;unallocated_pble -= (u32)(chunk-&gt;size &gt;&gt; 3);</div><div class='add'>+	sd_reg_val = (sd_entry_type == IRDMA_SD_TYPE_PAGED) ?</div><div class='add'>+			     sd_entry-&gt;u.pd_table.pd_page_addr.pa :</div><div class='add'>+			     sd_entry-&gt;u.bp.addr.pa;</div><div class='add'>+	if ((dev-&gt;privileged &amp;&amp; !sd_entry-&gt;valid) ||</div><div class='add'>+	    dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3) {</div><div class='add'>+		ret_code = irdma_hmc_sd_one(dev, hmc_info-&gt;hmc_fn_id,</div><div class='add'>+					    sd_reg_val, idx-&gt;sd_idx,</div><div class='add'>+					    sd_entry-&gt;entry_type, true);</div><div class='add'>+		if (ret_code)</div><div class='add'>+			goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	list_add(&amp;chunk-&gt;list, &amp;pble_rsrc-&gt;pinfo.clist);</div><div class='add'>+	sd_entry-&gt;valid = true;</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+error:</div><div class='add'>+	bitmap_free(chunk-&gt;bitmapbuf);</div><div class='add'>+	kfree(chunk-&gt;chunkmem.va);</div><div class='add'>+</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * free_lvl2 - fee level 2 pble</div><div class='add'>+ * @pble_rsrc: pble resource management</div><div class='add'>+ * @palloc: level 2 pble allocation</div><div class='add'>+ */</div><div class='add'>+static void free_lvl2(struct irdma_hmc_pble_rsrc *pble_rsrc,</div><div class='add'>+		      struct irdma_pble_alloc *palloc)</div><div class='add'>+{</div><div class='add'>+	u32 i;</div><div class='add'>+	struct irdma_pble_level2 *lvl2 = &amp;palloc-&gt;level2;</div><div class='add'>+	struct irdma_pble_info *root = &amp;lvl2-&gt;root;</div><div class='add'>+	struct irdma_pble_info *leaf = lvl2-&gt;leaf;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; lvl2-&gt;leaf_cnt; i++, leaf++) {</div><div class='add'>+		if (leaf-&gt;addr)</div><div class='add'>+			irdma_prm_return_pbles(&amp;pble_rsrc-&gt;pinfo,</div><div class='add'>+					       &amp;leaf-&gt;chunkinfo);</div><div class='add'>+		else</div><div class='add'>+			break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (root-&gt;addr)</div><div class='add'>+		irdma_prm_return_pbles(&amp;pble_rsrc-&gt;pinfo, &amp;root-&gt;chunkinfo);</div><div class='add'>+</div><div class='add'>+	kfree(lvl2-&gt;leafmem.va);</div><div class='add'>+	lvl2-&gt;leaf = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * get_lvl2_pble - get level 2 pble resource</div><div class='add'>+ * @pble_rsrc: pble resource management</div><div class='add'>+ * @palloc: level 2 pble allocation</div><div class='add'>+ */</div><div class='add'>+static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,</div><div class='add'>+			 struct irdma_pble_alloc *palloc)</div><div class='add'>+{</div><div class='add'>+	u32 lf4k, lflast, total, i;</div><div class='add'>+	u32 pblcnt = PBLE_PER_PAGE;</div><div class='add'>+	u64 *addr;</div><div class='add'>+	struct irdma_pble_level2 *lvl2 = &amp;palloc-&gt;level2;</div><div class='add'>+	struct irdma_pble_info *root = &amp;lvl2-&gt;root;</div><div class='add'>+	struct irdma_pble_info *leaf;</div><div class='add'>+	int ret_code;</div><div class='add'>+	u64 fpm_addr;</div><div class='add'>+</div><div class='add'>+	/* number of full 512 (4K) leafs) */</div><div class='add'>+	lf4k = palloc-&gt;total_cnt &gt;&gt; 9;</div><div class='add'>+	lflast = palloc-&gt;total_cnt % PBLE_PER_PAGE;</div><div class='add'>+	total = (lflast == 0) ? lf4k : lf4k + 1;</div><div class='add'>+	lvl2-&gt;leaf_cnt = total;</div><div class='add'>+</div><div class='add'>+	lvl2-&gt;leafmem.size = (sizeof(*leaf) * total);</div><div class='add'>+	lvl2-&gt;leafmem.va = kzalloc(lvl2-&gt;leafmem.size, GFP_KERNEL);</div><div class='add'>+	if (!lvl2-&gt;leafmem.va)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	lvl2-&gt;leaf = lvl2-&gt;leafmem.va;</div><div class='add'>+	leaf = lvl2-&gt;leaf;</div><div class='add'>+	ret_code = irdma_prm_get_pbles(&amp;pble_rsrc-&gt;pinfo, &amp;root-&gt;chunkinfo,</div><div class='add'>+				       total &lt;&lt; 3, &amp;root-&gt;addr, &amp;fpm_addr);</div><div class='add'>+	if (ret_code) {</div><div class='add'>+		kfree(lvl2-&gt;leafmem.va);</div><div class='add'>+		lvl2-&gt;leaf = NULL;</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	root-&gt;idx = fpm_to_idx(pble_rsrc, fpm_addr);</div><div class='add'>+	root-&gt;cnt = total;</div><div class='add'>+	addr = root-&gt;addr;</div><div class='add'>+	for (i = 0; i &lt; total; i++, leaf++) {</div><div class='add'>+		pblcnt = (lflast &amp;&amp; ((i + 1) == total)) ?</div><div class='add'>+				lflast : PBLE_PER_PAGE;</div><div class='add'>+		ret_code = irdma_prm_get_pbles(&amp;pble_rsrc-&gt;pinfo,</div><div class='add'>+					       &amp;leaf-&gt;chunkinfo, pblcnt &lt;&lt; 3,</div><div class='add'>+					       &amp;leaf-&gt;addr, &amp;fpm_addr);</div><div class='add'>+		if (ret_code)</div><div class='add'>+			goto error;</div><div class='add'>+</div><div class='add'>+		leaf-&gt;idx = fpm_to_idx(pble_rsrc, fpm_addr);</div><div class='add'>+</div><div class='add'>+		leaf-&gt;cnt = pblcnt;</div><div class='add'>+		*addr = (u64)leaf-&gt;idx;</div><div class='add'>+		addr++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	palloc-&gt;level = PBLE_LEVEL_2;</div><div class='add'>+	pble_rsrc-&gt;stats_lvl2++;</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+error:</div><div class='add'>+	free_lvl2(pble_rsrc, palloc);</div><div class='add'>+</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * get_lvl1_pble - get level 1 pble resource</div><div class='add'>+ * @pble_rsrc: pble resource management</div><div class='add'>+ * @palloc: level 1 pble allocation</div><div class='add'>+ */</div><div class='add'>+static int get_lvl1_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,</div><div class='add'>+			 struct irdma_pble_alloc *palloc)</div><div class='add'>+{</div><div class='add'>+	int ret_code;</div><div class='add'>+	u64 fpm_addr;</div><div class='add'>+	struct irdma_pble_info *lvl1 = &amp;palloc-&gt;level1;</div><div class='add'>+</div><div class='add'>+	ret_code = irdma_prm_get_pbles(&amp;pble_rsrc-&gt;pinfo, &amp;lvl1-&gt;chunkinfo,</div><div class='add'>+				       palloc-&gt;total_cnt &lt;&lt; 3, &amp;lvl1-&gt;addr,</div><div class='add'>+				       &amp;fpm_addr);</div><div class='add'>+	if (ret_code)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	palloc-&gt;level = PBLE_LEVEL_1;</div><div class='add'>+	lvl1-&gt;idx = fpm_to_idx(pble_rsrc, fpm_addr);</div><div class='add'>+	lvl1-&gt;cnt = palloc-&gt;total_cnt;</div><div class='add'>+	pble_rsrc-&gt;stats_lvl1++;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * get_lvl1_lvl2_pble - calls get_lvl1 and get_lvl2 pble routine</div><div class='add'>+ * @pble_rsrc: pble resources</div><div class='add'>+ * @palloc: contains all inforamtion regarding pble (idx + pble addr)</div><div class='add'>+ * @lvl: Bitmask for requested pble level</div><div class='add'>+ */</div><div class='add'>+static int get_lvl1_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,</div><div class='add'>+			      struct irdma_pble_alloc *palloc, u8 lvl)</div><div class='add'>+{</div><div class='add'>+	int status = 0;</div><div class='add'>+</div><div class='add'>+	status = get_lvl1_pble(pble_rsrc, palloc);</div><div class='add'>+	if (!status || lvl == PBLE_LEVEL_1 || palloc-&gt;total_cnt &lt;= PBLE_PER_PAGE)</div><div class='add'>+		return status;</div><div class='add'>+</div><div class='add'>+	status = get_lvl2_pble(pble_rsrc, palloc);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_pble - allocate pbles from the prm</div><div class='add'>+ * @pble_rsrc: pble resources</div><div class='add'>+ * @palloc: contains all inforamtion regarding pble (idx + pble addr)</div><div class='add'>+ * @pble_cnt: #of pbles requested</div><div class='add'>+ * @lvl: requested pble level mask</div><div class='add'>+ */</div><div class='add'>+int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,</div><div class='add'>+		   struct irdma_pble_alloc *palloc, u32 pble_cnt,</div><div class='add'>+		   u8 lvl)</div><div class='add'>+{</div><div class='add'>+	int status = 0;</div><div class='add'>+	int max_sds = 0;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	palloc-&gt;total_cnt = pble_cnt;</div><div class='add'>+	palloc-&gt;level = PBLE_LEVEL_0;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;pble_rsrc-&gt;pble_mutex_lock);</div><div class='add'>+</div><div class='add'>+	/*check first to see if we can get pble's without acquiring</div><div class='add'>+	 * additional sd's</div><div class='add'>+	 */</div><div class='add'>+	status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl);</div><div class='add'>+	if (!status)</div><div class='add'>+		goto exit;</div><div class='add'>+</div><div class='add'>+	max_sds = (palloc-&gt;total_cnt &gt;&gt; 18) + 1;</div><div class='add'>+	for (i = 0; i &lt; max_sds; i++) {</div><div class='add'>+		status = add_pble_prm(pble_rsrc);</div><div class='add'>+		if (status)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl);</div><div class='add'>+		/* if level1_only, only go through it once */</div><div class='add'>+		if (!status || lvl)</div><div class='add'>+			break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+exit:</div><div class='add'>+	if (!status) {</div><div class='add'>+		pble_rsrc-&gt;allocdpbles += pble_cnt;</div><div class='add'>+		pble_rsrc-&gt;stats_alloc_ok++;</div><div class='add'>+	} else {</div><div class='add'>+		pble_rsrc-&gt;stats_alloc_fail++;</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;pble_rsrc-&gt;pble_mutex_lock);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_free_pble - put pbles back into prm</div><div class='add'>+ * @pble_rsrc: pble resources</div><div class='add'>+ * @palloc: contains all information regarding pble resource being freed</div><div class='add'>+ */</div><div class='add'>+void irdma_free_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,</div><div class='add'>+		     struct irdma_pble_alloc *palloc)</div><div class='add'>+{</div><div class='add'>+	if (palloc-&gt;level == PBLE_LEVEL_2)</div><div class='add'>+		free_lvl2(pble_rsrc, palloc);</div><div class='add'>+	else</div><div class='add'>+		irdma_prm_return_pbles(&amp;pble_rsrc-&gt;pinfo,</div><div class='add'>+				       &amp;palloc-&gt;level1.chunkinfo);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;pble_rsrc-&gt;pble_mutex_lock);</div><div class='add'>+	pble_rsrc-&gt;freedpbles += palloc-&gt;total_cnt;</div><div class='add'>+	pble_rsrc-&gt;stats_alloc_freed++;</div><div class='add'>+	mutex_unlock(&amp;pble_rsrc-&gt;pble_mutex_lock);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/pble.h b/drivers/infiniband/hw/irdma/pble.h<br/>new file mode 100644<br/>index 000000000000..160ad728e9fb<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/pble.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/pble.h</a></div><div class='hunk'>@@ -0,0 +1,132 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2015 - 2019 Intel Corporation */</div><div class='add'>+#ifndef IRDMA_PBLE_H</div><div class='add'>+#define IRDMA_PBLE_H</div><div class='add'>+</div><div class='add'>+#define PBLE_SHIFT		6</div><div class='add'>+#define PBLE_PER_PAGE		512</div><div class='add'>+#define HMC_PAGED_BP_SHIFT	12</div><div class='add'>+#define PBLE_512_SHIFT		9</div><div class='add'>+#define PBLE_INVALID_IDX	0xffffffff</div><div class='add'>+</div><div class='add'>+enum irdma_pble_level {</div><div class='add'>+	PBLE_LEVEL_0 = 0,</div><div class='add'>+	PBLE_LEVEL_1 = 1,</div><div class='add'>+	PBLE_LEVEL_2 = 2,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_alloc_type {</div><div class='add'>+	PBLE_NO_ALLOC	  = 0,</div><div class='add'>+	PBLE_SD_CONTIGOUS = 1,</div><div class='add'>+	PBLE_SD_PAGED	  = 2,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_chunk;</div><div class='add'>+</div><div class='add'>+struct irdma_pble_chunkinfo {</div><div class='add'>+	struct irdma_chunk *pchunk;</div><div class='add'>+	u64 bit_idx;</div><div class='add'>+	u64 bits_used;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_pble_info {</div><div class='add'>+	u64 *addr;</div><div class='add'>+	u32 idx;</div><div class='add'>+	u32 cnt;</div><div class='add'>+	struct irdma_pble_chunkinfo chunkinfo;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_pble_level2 {</div><div class='add'>+	struct irdma_pble_info root;</div><div class='add'>+	struct irdma_pble_info *leaf;</div><div class='add'>+	struct irdma_virt_mem leafmem;</div><div class='add'>+	u32 leaf_cnt;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_pble_alloc {</div><div class='add'>+	u32 total_cnt;</div><div class='add'>+	enum irdma_pble_level level;</div><div class='add'>+	union {</div><div class='add'>+		struct irdma_pble_info level1;</div><div class='add'>+		struct irdma_pble_level2 level2;</div><div class='add'>+	};</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct sd_pd_idx {</div><div class='add'>+	u32 sd_idx;</div><div class='add'>+	u32 pd_idx;</div><div class='add'>+	u32 rel_pd_idx;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_add_page_info {</div><div class='add'>+	struct irdma_chunk *chunk;</div><div class='add'>+	struct irdma_hmc_sd_entry *sd_entry;</div><div class='add'>+	struct irdma_hmc_info *hmc_info;</div><div class='add'>+	struct sd_pd_idx idx;</div><div class='add'>+	u32 pages;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_chunk {</div><div class='add'>+	struct list_head list;</div><div class='add'>+	struct irdma_dma_info dmainfo;</div><div class='add'>+	unsigned long *bitmapbuf;</div><div class='add'>+</div><div class='add'>+	u32 sizeofbitmap;</div><div class='add'>+	u64 size;</div><div class='add'>+	void *vaddr;</div><div class='add'>+	u64 fpm_addr;</div><div class='add'>+	u32 pg_cnt;</div><div class='add'>+	enum irdma_alloc_type type;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	struct irdma_virt_mem chunkmem;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_pble_prm {</div><div class='add'>+	struct list_head clist;</div><div class='add'>+	spinlock_t prm_lock; /* protect prm bitmap */</div><div class='add'>+	u64 total_pble_alloc;</div><div class='add'>+	u64 free_pble_cnt;</div><div class='add'>+	u8 pble_shift;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_hmc_pble_rsrc {</div><div class='add'>+	u32 unallocated_pble;</div><div class='add'>+	struct mutex pble_mutex_lock; /* protect PBLE resource */</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	u64 fpm_base_addr;</div><div class='add'>+	u64 next_fpm_addr;</div><div class='add'>+	struct irdma_pble_prm pinfo;</div><div class='add'>+	u64 allocdpbles;</div><div class='add'>+	u64 freedpbles;</div><div class='add'>+	u32 stats_direct_sds;</div><div class='add'>+	u32 stats_paged_sds;</div><div class='add'>+	u64 stats_alloc_ok;</div><div class='add'>+	u64 stats_alloc_fail;</div><div class='add'>+	u64 stats_alloc_freed;</div><div class='add'>+	u64 stats_lvl1;</div><div class='add'>+	u64 stats_lvl2;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+void irdma_destroy_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc);</div><div class='add'>+int irdma_hmc_init_pble(struct irdma_sc_dev *dev,</div><div class='add'>+			struct irdma_hmc_pble_rsrc *pble_rsrc);</div><div class='add'>+void irdma_free_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,</div><div class='add'>+		     struct irdma_pble_alloc *palloc);</div><div class='add'>+int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,</div><div class='add'>+		   struct irdma_pble_alloc *palloc, u32 pble_cnt,</div><div class='add'>+		   u8 lvl);</div><div class='add'>+int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,</div><div class='add'>+			   struct irdma_chunk *pchunk);</div><div class='add'>+int irdma_prm_get_pbles(struct irdma_pble_prm *pprm,</div><div class='add'>+			struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,</div><div class='add'>+			u64 **vaddr, u64 *fpm_addr);</div><div class='add'>+void irdma_prm_return_pbles(struct irdma_pble_prm *pprm,</div><div class='add'>+			    struct irdma_pble_chunkinfo *chunkinfo);</div><div class='add'>+void irdma_pble_acquire_lock(struct irdma_hmc_pble_rsrc *pble_rsrc,</div><div class='add'>+			     unsigned long *flags);</div><div class='add'>+void irdma_pble_release_lock(struct irdma_hmc_pble_rsrc *pble_rsrc,</div><div class='add'>+			     unsigned long *flags);</div><div class='add'>+void irdma_pble_free_paged_mem(struct irdma_chunk *chunk);</div><div class='add'>+int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt);</div><div class='add'>+void irdma_prm_rem_bitmapmem(struct irdma_hw *hw, struct irdma_chunk *chunk);</div><div class='add'>+#endif /* IRDMA_PBLE_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/protos.h b/drivers/infiniband/hw/irdma/protos.h<br/>new file mode 100644<br/>index 000000000000..324cfbf21764<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/protos.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/protos.h</a></div><div class='hunk'>@@ -0,0 +1,97 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2016 - 2021 Intel Corporation */</div><div class='add'>+#ifndef IRDMA_PROTOS_H</div><div class='add'>+#define IRDMA_PROTOS_H</div><div class='add'>+</div><div class='add'>+#define PAUSE_TIMER_VAL		0xffff</div><div class='add'>+#define REFRESH_THRESHOLD	0x7fff</div><div class='add'>+#define HIGH_THRESHOLD		0x800</div><div class='add'>+#define LOW_THRESHOLD		0x200</div><div class='add'>+#define ALL_TC2PFC		0xff</div><div class='add'>+#define CQP_COMPL_WAIT_TIME_MS	10</div><div class='add'>+#define CQP_TIMEOUT_THRESHOLD	500</div><div class='add'>+#define CQP_DEF_CMPL_TIMEOUT_THRESHOLD	2500</div><div class='add'>+</div><div class='add'>+/* init operations */</div><div class='add'>+int irdma_sc_dev_init(enum irdma_vers ver, struct irdma_sc_dev *dev,</div><div class='add'>+		      struct irdma_device_init_info *info);</div><div class='add'>+void irdma_sc_rt_init(struct irdma_sc_dev *dev);</div><div class='add'>+void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);</div><div class='add'>+__le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch);</div><div class='add'>+int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,</div><div class='add'>+			      struct irdma_fast_reg_stag_info *info,</div><div class='add'>+			      bool post_sq);</div><div class='add'>+/* HMC/FPM functions */</div><div class='add'>+int irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u8 hmc_fn_id);</div><div class='add'>+/* stats misc */</div><div class='add'>+int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,</div><div class='add'>+			       struct irdma_vsi_pestat *pestat, bool wait);</div><div class='add'>+void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev,</div><div class='add'>+				 struct irdma_vsi_pestat *pestat);</div><div class='add'>+void irdma_hw_stats_read_all(struct irdma_vsi_pestat *stats,</div><div class='add'>+			     const u64 *hw_stats_regs);</div><div class='add'>+int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,</div><div class='add'>+			  struct irdma_ws_node_info *node_info);</div><div class='add'>+int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq,</div><div class='add'>+		      u8 op);</div><div class='add'>+int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq,</div><div class='add'>+		      u8 op);</div><div class='add'>+int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,</div><div class='add'>+			     struct irdma_stats_inst_info *stats_info);</div><div class='add'>+u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev);</div><div class='add'>+void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id);</div><div class='add'>+void irdma_update_stats(struct irdma_dev_hw_stats *hw_stats,</div><div class='add'>+			struct irdma_gather_stats *gather_stats,</div><div class='add'>+			struct irdma_gather_stats *last_gather_stats,</div><div class='add'>+			const struct irdma_hw_stat_map *map, u16 max_stat_idx);</div><div class='add'>+</div><div class='add'>+/* vsi functions */</div><div class='add'>+int irdma_vsi_stats_init(struct irdma_sc_vsi *vsi,</div><div class='add'>+			 struct irdma_vsi_stats_info *info);</div><div class='add'>+void irdma_vsi_stats_free(struct irdma_sc_vsi *vsi);</div><div class='add'>+void irdma_sc_vsi_init(struct irdma_sc_vsi *vsi,</div><div class='add'>+		       struct irdma_vsi_init_info *info);</div><div class='add'>+int irdma_sc_add_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq);</div><div class='add'>+void irdma_sc_remove_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq);</div><div class='add'>+/* misc L2 param change functions */</div><div class='add'>+void irdma_change_l2params(struct irdma_sc_vsi *vsi,</div><div class='add'>+			   struct irdma_l2params *l2params);</div><div class='add'>+void irdma_sc_suspend_resume_qps(struct irdma_sc_vsi *vsi, u8 suspend);</div><div class='add'>+int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 cmd);</div><div class='add'>+void irdma_qp_add_qos(struct irdma_sc_qp *qp);</div><div class='add'>+void irdma_qp_rem_qos(struct irdma_sc_qp *qp);</div><div class='add'>+struct irdma_sc_qp *irdma_get_qp_from_list(struct list_head *head,</div><div class='add'>+					   struct irdma_sc_qp *qp);</div><div class='add'>+void irdma_reinitialize_ieq(struct irdma_sc_vsi *vsi);</div><div class='add'>+/* terminate functions*/</div><div class='add'>+void irdma_terminate_send_fin(struct irdma_sc_qp *qp);</div><div class='add'>+</div><div class='add'>+void irdma_terminate_connection(struct irdma_sc_qp *qp,</div><div class='add'>+				struct irdma_aeqe_info *info);</div><div class='add'>+</div><div class='add'>+void irdma_terminate_received(struct irdma_sc_qp *qp,</div><div class='add'>+			      struct irdma_aeqe_info *info);</div><div class='add'>+/* dynamic memory allocation */</div><div class='add'>+/* misc */</div><div class='add'>+u8 irdma_get_encoded_wqe_size(u32 wqsize, enum irdma_queue_type queue_type);</div><div class='add'>+void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp);</div><div class='add'>+int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,</div><div class='add'>+					u8 hmc_fn_id, bool post_sq,</div><div class='add'>+					bool poll_registers);</div><div class='add'>+int irdma_cfg_fpm_val(struct irdma_sc_dev *dev, u32 qp_count);</div><div class='add'>+int irdma_get_rdma_features(struct irdma_sc_dev *dev);</div><div class='add'>+void free_sd_mem(struct irdma_sc_dev *dev);</div><div class='add'>+int irdma_process_cqp_cmd(struct irdma_sc_dev *dev,</div><div class='add'>+			  struct cqp_cmds_info *pcmdinfo);</div><div class='add'>+int irdma_process_bh(struct irdma_sc_dev *dev);</div><div class='add'>+int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,</div><div class='add'>+		      struct irdma_update_sds_info *info);</div><div class='add'>+int irdma_alloc_query_fpm_buf(struct irdma_sc_dev *dev,</div><div class='add'>+			      struct irdma_dma_mem *mem);</div><div class='add'>+int irdma_cqp_manage_hmc_fcn_cmd(struct irdma_sc_dev *dev,</div><div class='add'>+				 struct irdma_hmc_fcn_info *hmcfcninfo,</div><div class='add'>+				 u16 *pmf_idx);</div><div class='add'>+void irdma_add_dev_ref(struct irdma_sc_dev *dev);</div><div class='add'>+void irdma_put_dev_ref(struct irdma_sc_dev *dev);</div><div class='add'>+void *irdma_remove_cqp_head(struct irdma_sc_dev *dev);</div><div class='add'>+#endif /* IRDMA_PROTOS_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/puda.c b/drivers/infiniband/hw/irdma/puda.c<br/>new file mode 100644<br/>index 000000000000..cee47ddbd1b5<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/puda.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/puda.c</a></div><div class='hunk'>@@ -0,0 +1,1718 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#include "osdep.h"</div><div class='add'>+#include "hmc.h"</div><div class='add'>+#include "defs.h"</div><div class='add'>+#include "type.h"</div><div class='add'>+#include "protos.h"</div><div class='add'>+#include "puda.h"</div><div class='add'>+#include "ws.h"</div><div class='add'>+</div><div class='add'>+static void irdma_ieq_receive(struct irdma_sc_vsi *vsi,</div><div class='add'>+			      struct irdma_puda_buf *buf);</div><div class='add'>+static void irdma_ieq_tx_compl(struct irdma_sc_vsi *vsi, void *sqwrid);</div><div class='add'>+static void irdma_ilq_putback_rcvbuf(struct irdma_sc_qp *qp,</div><div class='add'>+				     struct irdma_puda_buf *buf, u32 wqe_idx);</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_get_listbuf - get buffer from puda list</div><div class='add'>+ * @list: list to use for buffers (ILQ or IEQ)</div><div class='add'>+ */</div><div class='add'>+static struct irdma_puda_buf *irdma_puda_get_listbuf(struct list_head *list)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_buf *buf = NULL;</div><div class='add'>+</div><div class='add'>+	if (!list_empty(list)) {</div><div class='add'>+		buf = (struct irdma_puda_buf *)list-&gt;next;</div><div class='add'>+		list_del((struct list_head *)&amp;buf-&gt;list);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return buf;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_get_bufpool - return buffer from resource</div><div class='add'>+ * @rsrc: resource to use for buffer</div><div class='add'>+ */</div><div class='add'>+struct irdma_puda_buf *irdma_puda_get_bufpool(struct irdma_puda_rsrc *rsrc)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_buf *buf = NULL;</div><div class='add'>+	struct list_head *list = &amp;rsrc-&gt;bufpool;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='add'>+	buf = irdma_puda_get_listbuf(list);</div><div class='add'>+	if (buf) {</div><div class='add'>+		rsrc-&gt;avail_buf_count--;</div><div class='add'>+		buf-&gt;vsi = rsrc-&gt;vsi;</div><div class='add'>+	} else {</div><div class='add'>+		rsrc-&gt;stats_buf_alloc_fail++;</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='add'>+</div><div class='add'>+	return buf;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_ret_bufpool - return buffer to rsrc list</div><div class='add'>+ * @rsrc: resource to use for buffer</div><div class='add'>+ * @buf: buffer to return to resource</div><div class='add'>+ */</div><div class='add'>+void irdma_puda_ret_bufpool(struct irdma_puda_rsrc *rsrc,</div><div class='add'>+			    struct irdma_puda_buf *buf)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	buf-&gt;do_lpb = false;</div><div class='add'>+	spin_lock_irqsave(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='add'>+	list_add(&amp;buf-&gt;list, &amp;rsrc-&gt;bufpool);</div><div class='add'>+	spin_unlock_irqrestore(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='add'>+	rsrc-&gt;avail_buf_count++;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_post_recvbuf - set wqe for rcv buffer</div><div class='add'>+ * @rsrc: resource ptr</div><div class='add'>+ * @wqe_idx: wqe index to use</div><div class='add'>+ * @buf: puda buffer for rcv q</div><div class='add'>+ * @initial: flag if during init time</div><div class='add'>+ */</div><div class='add'>+static void irdma_puda_post_recvbuf(struct irdma_puda_rsrc *rsrc, u32 wqe_idx,</div><div class='add'>+				    struct irdma_puda_buf *buf, bool initial)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_sc_qp *qp = &amp;rsrc-&gt;qp;</div><div class='add'>+	u64 offset24 = 0;</div><div class='add'>+</div><div class='add'>+	/* Synch buffer for use by device */</div><div class='add'>+	dma_sync_single_for_device(rsrc-&gt;dev-&gt;hw-&gt;device, buf-&gt;mem.pa,</div><div class='add'>+				   buf-&gt;mem.size, DMA_BIDIRECTIONAL);</div><div class='add'>+	qp-&gt;qp_uk.rq_wrid_array[wqe_idx] = (uintptr_t)buf;</div><div class='add'>+	wqe = qp-&gt;qp_uk.rq_base[wqe_idx].elem;</div><div class='add'>+	if (!initial)</div><div class='add'>+		get_64bit_val(wqe, 24, &amp;offset24);</div><div class='add'>+</div><div class='add'>+	offset24 = (offset24) ? 0 : FIELD_PREP(IRDMAQPSQ_VALID, 1);</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16, 0);</div><div class='add'>+	set_64bit_val(wqe, 0, buf-&gt;mem.pa);</div><div class='add'>+	if (qp-&gt;qp_uk.uk_attrs-&gt;hw_rev == IRDMA_GEN_1) {</div><div class='add'>+		set_64bit_val(wqe, 8,</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, buf-&gt;mem.size));</div><div class='add'>+	} else {</div><div class='add'>+		set_64bit_val(wqe, 8,</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_FRAG_LEN, buf-&gt;mem.size) |</div><div class='add'>+			      offset24);</div><div class='add'>+	}</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, offset24);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_replenish_rq - post rcv buffers</div><div class='add'>+ * @rsrc: resource to use for buffer</div><div class='add'>+ * @initial: flag if during init time</div><div class='add'>+ */</div><div class='add'>+static int irdma_puda_replenish_rq(struct irdma_puda_rsrc *rsrc, bool initial)</div><div class='add'>+{</div><div class='add'>+	u32 i;</div><div class='add'>+	u32 invalid_cnt = rsrc-&gt;rxq_invalid_cnt;</div><div class='add'>+	struct irdma_puda_buf *buf = NULL;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; invalid_cnt; i++) {</div><div class='add'>+		buf = irdma_puda_get_bufpool(rsrc);</div><div class='add'>+		if (!buf)</div><div class='add'>+			return -ENOBUFS;</div><div class='add'>+		irdma_puda_post_recvbuf(rsrc, rsrc-&gt;rx_wqe_idx, buf, initial);</div><div class='add'>+		rsrc-&gt;rx_wqe_idx = ((rsrc-&gt;rx_wqe_idx + 1) % rsrc-&gt;rq_size);</div><div class='add'>+		rsrc-&gt;rxq_invalid_cnt--;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_alloc_buf - allocate mem for buffer</div><div class='add'>+ * @dev: iwarp device</div><div class='add'>+ * @len: length of buffer</div><div class='add'>+ */</div><div class='add'>+static struct irdma_puda_buf *irdma_puda_alloc_buf(struct irdma_sc_dev *dev,</div><div class='add'>+						   u32 len)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_buf *buf;</div><div class='add'>+	struct irdma_virt_mem buf_mem;</div><div class='add'>+</div><div class='add'>+	buf_mem.size = sizeof(struct irdma_puda_buf);</div><div class='add'>+	buf_mem.va = kzalloc(buf_mem.size, GFP_KERNEL);</div><div class='add'>+	if (!buf_mem.va)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	buf = buf_mem.va;</div><div class='add'>+	buf-&gt;mem.size = len;</div><div class='add'>+	buf-&gt;mem.va = kzalloc(buf-&gt;mem.size, GFP_KERNEL);</div><div class='add'>+	if (!buf-&gt;mem.va)</div><div class='add'>+		goto free_virt;</div><div class='add'>+	buf-&gt;mem.pa = dma_map_single(dev-&gt;hw-&gt;device, buf-&gt;mem.va,</div><div class='add'>+				     buf-&gt;mem.size, DMA_BIDIRECTIONAL);</div><div class='add'>+	if (dma_mapping_error(dev-&gt;hw-&gt;device, buf-&gt;mem.pa)) {</div><div class='add'>+		kfree(buf-&gt;mem.va);</div><div class='add'>+		goto free_virt;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	buf-&gt;buf_mem.va = buf_mem.va;</div><div class='add'>+	buf-&gt;buf_mem.size = buf_mem.size;</div><div class='add'>+</div><div class='add'>+	return buf;</div><div class='add'>+</div><div class='add'>+free_virt:</div><div class='add'>+	kfree(buf_mem.va);</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_dele_buf - delete buffer back to system</div><div class='add'>+ * @dev: iwarp device</div><div class='add'>+ * @buf: buffer to free</div><div class='add'>+ */</div><div class='add'>+static void irdma_puda_dele_buf(struct irdma_sc_dev *dev,</div><div class='add'>+				struct irdma_puda_buf *buf)</div><div class='add'>+{</div><div class='add'>+	dma_unmap_single(dev-&gt;hw-&gt;device, buf-&gt;mem.pa, buf-&gt;mem.size,</div><div class='add'>+			 DMA_BIDIRECTIONAL);</div><div class='add'>+	kfree(buf-&gt;mem.va);</div><div class='add'>+	kfree(buf-&gt;buf_mem.va);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_get_next_send_wqe - return next wqe for processing</div><div class='add'>+ * @qp: puda qp for wqe</div><div class='add'>+ * @wqe_idx: wqe index for caller</div><div class='add'>+ */</div><div class='add'>+static __le64 *irdma_puda_get_next_send_wqe(struct irdma_qp_uk *qp,</div><div class='add'>+					    u32 *wqe_idx)</div><div class='add'>+{</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+</div><div class='add'>+	*wqe_idx = IRDMA_RING_CURRENT_HEAD(qp-&gt;sq_ring);</div><div class='add'>+	if (!*wqe_idx)</div><div class='add'>+		qp-&gt;swqe_polarity = !qp-&gt;swqe_polarity;</div><div class='add'>+	IRDMA_RING_MOVE_HEAD(qp-&gt;sq_ring, ret_code);</div><div class='add'>+	if (ret_code)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	return qp-&gt;sq_base[*wqe_idx].elem;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_poll_info - poll cq for completion</div><div class='add'>+ * @cq: cq for poll</div><div class='add'>+ * @info: info return for successful completion</div><div class='add'>+ */</div><div class='add'>+static int irdma_puda_poll_info(struct irdma_sc_cq *cq,</div><div class='add'>+				struct irdma_puda_cmpl_info *info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cq_uk *cq_uk = &amp;cq-&gt;cq_uk;</div><div class='add'>+	u64 qword0, qword2, qword3, qword6;</div><div class='add'>+	__le64 *cqe;</div><div class='add'>+	__le64 *ext_cqe = NULL;</div><div class='add'>+	u64 qword7 = 0;</div><div class='add'>+	u64 comp_ctx;</div><div class='add'>+	bool valid_bit;</div><div class='add'>+	bool ext_valid = 0;</div><div class='add'>+	u32 major_err, minor_err;</div><div class='add'>+	u32 peek_head;</div><div class='add'>+	bool error;</div><div class='add'>+	u8 polarity;</div><div class='add'>+</div><div class='add'>+	cqe = IRDMA_GET_CURRENT_CQ_ELEM(&amp;cq-&gt;cq_uk);</div><div class='add'>+	get_64bit_val(cqe, 24, &amp;qword3);</div><div class='add'>+	valid_bit = (bool)FIELD_GET(IRDMA_CQ_VALID, qword3);</div><div class='add'>+	if (valid_bit != cq_uk-&gt;polarity)</div><div class='add'>+		return -ENOENT;</div><div class='add'>+</div><div class='add'>+	/* Ensure CQE contents are read after valid bit is checked */</div><div class='add'>+	dma_rmb();</div><div class='add'>+</div><div class='add'>+	if (cq-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2)</div><div class='add'>+		ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3);</div><div class='add'>+</div><div class='add'>+	if (ext_valid) {</div><div class='add'>+		peek_head = (cq_uk-&gt;cq_ring.head + 1) % cq_uk-&gt;cq_ring.size;</div><div class='add'>+		ext_cqe = cq_uk-&gt;cq_base[peek_head].buf;</div><div class='add'>+		get_64bit_val(ext_cqe, 24, &amp;qword7);</div><div class='add'>+		polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7);</div><div class='add'>+		if (!peek_head)</div><div class='add'>+			polarity ^= 1;</div><div class='add'>+		if (polarity != cq_uk-&gt;polarity)</div><div class='add'>+			return -ENOENT;</div><div class='add'>+</div><div class='add'>+		/* Ensure ext CQE contents are read after ext valid bit is checked */</div><div class='add'>+		dma_rmb();</div><div class='add'>+</div><div class='add'>+		IRDMA_RING_MOVE_HEAD_NOCHECK(cq_uk-&gt;cq_ring);</div><div class='add'>+		if (!IRDMA_RING_CURRENT_HEAD(cq_uk-&gt;cq_ring))</div><div class='add'>+			cq_uk-&gt;polarity = !cq_uk-&gt;polarity;</div><div class='add'>+		/* update cq tail in cq shadow memory also */</div><div class='add'>+		IRDMA_RING_MOVE_TAIL(cq_uk-&gt;cq_ring);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("PUDA: PUDA CQE", DUMP_PREFIX_OFFSET, 16, 8, cqe,</div><div class='add'>+			     32, false);</div><div class='add'>+	if (ext_valid)</div><div class='add'>+		print_hex_dump_debug("PUDA: PUDA EXT-CQE", DUMP_PREFIX_OFFSET,</div><div class='add'>+				     16, 8, ext_cqe, 32, false);</div><div class='add'>+</div><div class='add'>+	error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);</div><div class='add'>+	if (error) {</div><div class='add'>+		ibdev_dbg(to_ibdev(cq-&gt;dev), "PUDA: receive error\n");</div><div class='add'>+		major_err = (u32)(FIELD_GET(IRDMA_CQ_MAJERR, qword3));</div><div class='add'>+		minor_err = (u32)(FIELD_GET(IRDMA_CQ_MINERR, qword3));</div><div class='add'>+		info-&gt;compl_error = major_err &lt;&lt; 16 | minor_err;</div><div class='add'>+		return -EIO;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	get_64bit_val(cqe, 0, &amp;qword0);</div><div class='add'>+	get_64bit_val(cqe, 16, &amp;qword2);</div><div class='add'>+</div><div class='add'>+	info-&gt;q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);</div><div class='add'>+	info-&gt;qp_id = (u32)FIELD_GET(IRDMACQ_QPID, qword2);</div><div class='add'>+	if (cq-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2)</div><div class='add'>+		info-&gt;ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);</div><div class='add'>+</div><div class='add'>+	get_64bit_val(cqe, 8, &amp;comp_ctx);</div><div class='add'>+	info-&gt;qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx;</div><div class='add'>+	info-&gt;wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);</div><div class='add'>+</div><div class='add'>+	if (info-&gt;q_type == IRDMA_CQE_QTYPE_RQ) {</div><div class='add'>+		if (ext_valid) {</div><div class='add'>+			info-&gt;vlan_valid = (bool)FIELD_GET(IRDMA_CQ_UDVLANVALID, qword7);</div><div class='add'>+			if (info-&gt;vlan_valid) {</div><div class='add'>+				get_64bit_val(ext_cqe, 16, &amp;qword6);</div><div class='add'>+				info-&gt;vlan = (u16)FIELD_GET(IRDMA_CQ_UDVLAN, qword6);</div><div class='add'>+			}</div><div class='add'>+			info-&gt;smac_valid = (bool)FIELD_GET(IRDMA_CQ_UDSMACVALID, qword7);</div><div class='add'>+			if (info-&gt;smac_valid) {</div><div class='add'>+				get_64bit_val(ext_cqe, 16, &amp;qword6);</div><div class='add'>+				info-&gt;smac[0] = (u8)((qword6 &gt;&gt; 40) &amp; 0xFF);</div><div class='add'>+				info-&gt;smac[1] = (u8)((qword6 &gt;&gt; 32) &amp; 0xFF);</div><div class='add'>+				info-&gt;smac[2] = (u8)((qword6 &gt;&gt; 24) &amp; 0xFF);</div><div class='add'>+				info-&gt;smac[3] = (u8)((qword6 &gt;&gt; 16) &amp; 0xFF);</div><div class='add'>+				info-&gt;smac[4] = (u8)((qword6 &gt;&gt; 8) &amp; 0xFF);</div><div class='add'>+				info-&gt;smac[5] = (u8)(qword6 &amp; 0xFF);</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (cq-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {</div><div class='add'>+			info-&gt;vlan_valid = (bool)FIELD_GET(IRDMA_VLAN_TAG_VALID, qword3);</div><div class='add'>+			info-&gt;l4proto = (u8)FIELD_GET(IRDMA_UDA_L4PROTO, qword2);</div><div class='add'>+			info-&gt;l3proto = (u8)FIELD_GET(IRDMA_UDA_L3PROTO, qword2);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		info-&gt;payload_len = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_poll_cmpl - processes completion for cq</div><div class='add'>+ * @dev: iwarp device</div><div class='add'>+ * @cq: cq getting interrupt</div><div class='add'>+ * @compl_err: return any completion err</div><div class='add'>+ */</div><div class='add'>+int irdma_puda_poll_cmpl(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq,</div><div class='add'>+			 u32 *compl_err)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp_uk *qp;</div><div class='add'>+	struct irdma_cq_uk *cq_uk = &amp;cq-&gt;cq_uk;</div><div class='add'>+	struct irdma_puda_cmpl_info info = {};</div><div class='add'>+	int ret = 0;</div><div class='add'>+	struct irdma_puda_buf *buf;</div><div class='add'>+	struct irdma_puda_rsrc *rsrc;</div><div class='add'>+	u8 cq_type = cq-&gt;cq_type;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	if (cq_type == IRDMA_CQ_TYPE_ILQ || cq_type == IRDMA_CQ_TYPE_IEQ) {</div><div class='add'>+		rsrc = (cq_type == IRDMA_CQ_TYPE_ILQ) ? cq-&gt;vsi-&gt;ilq :</div><div class='add'>+							cq-&gt;vsi-&gt;ieq;</div><div class='add'>+	} else {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "PUDA: qp_type error\n");</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = irdma_puda_poll_info(cq, &amp;info);</div><div class='add'>+	*compl_err = info.compl_error;</div><div class='add'>+	if (ret == -ENOENT)</div><div class='add'>+		return ret;</div><div class='add'>+	if (ret)</div><div class='add'>+		goto done;</div><div class='add'>+</div><div class='add'>+	qp = info.qp;</div><div class='add'>+	if (!qp || !rsrc) {</div><div class='add'>+		ret = -EFAULT;</div><div class='add'>+		goto done;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;qp_id != rsrc-&gt;qp_id) {</div><div class='add'>+		ret = -EFAULT;</div><div class='add'>+		goto done;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (info.q_type == IRDMA_CQE_QTYPE_RQ) {</div><div class='add'>+		buf = (struct irdma_puda_buf *)(uintptr_t)</div><div class='add'>+			      qp-&gt;rq_wrid_array[info.wqe_idx];</div><div class='add'>+</div><div class='add'>+		/* reusing so synch the buffer for CPU use */</div><div class='add'>+		dma_sync_single_for_cpu(dev-&gt;hw-&gt;device, buf-&gt;mem.pa,</div><div class='add'>+					buf-&gt;mem.size, DMA_BIDIRECTIONAL);</div><div class='add'>+		/* Get all the tcpip information in the buf header */</div><div class='add'>+		ret = irdma_puda_get_tcpip_info(&amp;info, buf);</div><div class='add'>+		if (ret) {</div><div class='add'>+			rsrc-&gt;stats_rcvd_pkt_err++;</div><div class='add'>+			if (cq_type == IRDMA_CQ_TYPE_ILQ) {</div><div class='add'>+				irdma_ilq_putback_rcvbuf(&amp;rsrc-&gt;qp, buf,</div><div class='add'>+							 info.wqe_idx);</div><div class='add'>+			} else {</div><div class='add'>+				irdma_puda_ret_bufpool(rsrc, buf);</div><div class='add'>+				irdma_puda_replenish_rq(rsrc, false);</div><div class='add'>+			}</div><div class='add'>+			goto done;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		rsrc-&gt;stats_pkt_rcvd++;</div><div class='add'>+		rsrc-&gt;compl_rxwqe_idx = info.wqe_idx;</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "PUDA: RQ completion\n");</div><div class='add'>+		rsrc-&gt;receive(rsrc-&gt;vsi, buf);</div><div class='add'>+		if (cq_type == IRDMA_CQ_TYPE_ILQ)</div><div class='add'>+			irdma_ilq_putback_rcvbuf(&amp;rsrc-&gt;qp, buf, info.wqe_idx);</div><div class='add'>+		else</div><div class='add'>+			irdma_puda_replenish_rq(rsrc, false);</div><div class='add'>+</div><div class='add'>+	} else {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "PUDA: SQ completion\n");</div><div class='add'>+		buf = (struct irdma_puda_buf *)(uintptr_t)</div><div class='add'>+					qp-&gt;sq_wrtrk_array[info.wqe_idx].wrid;</div><div class='add'>+</div><div class='add'>+		/* reusing so synch the buffer for CPU use */</div><div class='add'>+		dma_sync_single_for_cpu(dev-&gt;hw-&gt;device, buf-&gt;mem.pa,</div><div class='add'>+					buf-&gt;mem.size, DMA_BIDIRECTIONAL);</div><div class='add'>+		IRDMA_RING_SET_TAIL(qp-&gt;sq_ring, info.wqe_idx);</div><div class='add'>+		rsrc-&gt;xmit_complete(rsrc-&gt;vsi, buf);</div><div class='add'>+		spin_lock_irqsave(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='add'>+		rsrc-&gt;tx_wqe_avail_cnt++;</div><div class='add'>+		spin_unlock_irqrestore(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='add'>+		if (!list_empty(&amp;rsrc-&gt;txpend))</div><div class='add'>+			irdma_puda_send_buf(rsrc, NULL);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+done:</div><div class='add'>+	IRDMA_RING_MOVE_HEAD_NOCHECK(cq_uk-&gt;cq_ring);</div><div class='add'>+	if (!IRDMA_RING_CURRENT_HEAD(cq_uk-&gt;cq_ring))</div><div class='add'>+		cq_uk-&gt;polarity = !cq_uk-&gt;polarity;</div><div class='add'>+	/* update cq tail in cq shadow memory also */</div><div class='add'>+	IRDMA_RING_MOVE_TAIL(cq_uk-&gt;cq_ring);</div><div class='add'>+	set_64bit_val(cq_uk-&gt;shadow_area, 0,</div><div class='add'>+		      IRDMA_RING_CURRENT_HEAD(cq_uk-&gt;cq_ring));</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_send - complete send wqe for transmit</div><div class='add'>+ * @qp: puda qp for send</div><div class='add'>+ * @info: buffer information for transmit</div><div class='add'>+ */</div><div class='add'>+int irdma_puda_send(struct irdma_sc_qp *qp, struct irdma_puda_send_info *info)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u32 iplen, l4len;</div><div class='add'>+	u64 hdr[2];</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+	u8 iipt;</div><div class='add'>+</div><div class='add'>+	/* number of 32 bits DWORDS in header */</div><div class='add'>+	l4len = info-&gt;tcplen &gt;&gt; 2;</div><div class='add'>+	if (info-&gt;ipv4) {</div><div class='add'>+		iipt = 3;</div><div class='add'>+		iplen = 5;</div><div class='add'>+	} else {</div><div class='add'>+		iipt = 1;</div><div class='add'>+		iplen = 10;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	wqe = irdma_puda_get_next_send_wqe(&amp;qp-&gt;qp_uk, &amp;wqe_idx);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	qp-&gt;qp_uk.sq_wrtrk_array[wqe_idx].wrid = (uintptr_t)info-&gt;scratch;</div><div class='add'>+	/* Third line of WQE descriptor */</div><div class='add'>+	/* maclen is in words */</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+		hdr[0] = 0; /* Dest_QPN and Dest_QKey only for UD */</div><div class='add'>+		hdr[1] = FIELD_PREP(IRDMA_UDA_QPSQ_OPCODE, IRDMA_OP_TYPE_SEND) |</div><div class='add'>+			 FIELD_PREP(IRDMA_UDA_QPSQ_L4LEN, l4len) |</div><div class='add'>+			 FIELD_PREP(IRDMAQPSQ_AHID, info-&gt;ah_id) |</div><div class='add'>+			 FIELD_PREP(IRDMA_UDA_QPSQ_SIGCOMPL, 1) |</div><div class='add'>+			 FIELD_PREP(IRDMA_UDA_QPSQ_VALID,</div><div class='add'>+				    qp-&gt;qp_uk.swqe_polarity);</div><div class='add'>+</div><div class='add'>+		/* Forth line of WQE descriptor */</div><div class='add'>+</div><div class='add'>+		set_64bit_val(wqe, 0, info-&gt;paddr);</div><div class='add'>+		set_64bit_val(wqe, 8,</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_FRAG_LEN, info-&gt;len) |</div><div class='add'>+			      FIELD_PREP(IRDMA_UDA_QPSQ_VALID, qp-&gt;qp_uk.swqe_polarity));</div><div class='add'>+	} else {</div><div class='add'>+		hdr[0] = FIELD_PREP(IRDMA_UDA_QPSQ_MACLEN, info-&gt;maclen &gt;&gt; 1) |</div><div class='add'>+			 FIELD_PREP(IRDMA_UDA_QPSQ_IPLEN, iplen) |</div><div class='add'>+			 FIELD_PREP(IRDMA_UDA_QPSQ_L4T, 1) |</div><div class='add'>+			 FIELD_PREP(IRDMA_UDA_QPSQ_IIPT, iipt) |</div><div class='add'>+			 FIELD_PREP(IRDMA_GEN1_UDA_QPSQ_L4LEN, l4len);</div><div class='add'>+</div><div class='add'>+		hdr[1] = FIELD_PREP(IRDMA_UDA_QPSQ_OPCODE, IRDMA_OP_TYPE_SEND) |</div><div class='add'>+			 FIELD_PREP(IRDMA_UDA_QPSQ_SIGCOMPL, 1) |</div><div class='add'>+			 FIELD_PREP(IRDMA_UDA_QPSQ_DOLOOPBACK, info-&gt;do_lpb) |</div><div class='add'>+			 FIELD_PREP(IRDMA_UDA_QPSQ_VALID, qp-&gt;qp_uk.swqe_polarity);</div><div class='add'>+</div><div class='add'>+		/* Forth line of WQE descriptor */</div><div class='add'>+</div><div class='add'>+		set_64bit_val(wqe, 0, info-&gt;paddr);</div><div class='add'>+		set_64bit_val(wqe, 8,</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, info-&gt;len));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16, hdr[0]);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr[1]);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("PUDA: PUDA SEND WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, 32, false);</div><div class='add'>+	irdma_uk_qp_post_wr(&amp;qp-&gt;qp_uk);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_send_buf - transmit puda buffer</div><div class='add'>+ * @rsrc: resource to use for buffer</div><div class='add'>+ * @buf: puda buffer to transmit</div><div class='add'>+ */</div><div class='add'>+void irdma_puda_send_buf(struct irdma_puda_rsrc *rsrc,</div><div class='add'>+			 struct irdma_puda_buf *buf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_send_info info;</div><div class='add'>+	int ret = 0;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='add'>+	/* if no wqe available or not from a completion and we have</div><div class='add'>+	 * pending buffers, we must queue new buffer</div><div class='add'>+	 */</div><div class='add'>+	if (!rsrc-&gt;tx_wqe_avail_cnt || (buf &amp;&amp; !list_empty(&amp;rsrc-&gt;txpend))) {</div><div class='add'>+		list_add_tail(&amp;buf-&gt;list, &amp;rsrc-&gt;txpend);</div><div class='add'>+		spin_unlock_irqrestore(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='add'>+		rsrc-&gt;stats_sent_pkt_q++;</div><div class='add'>+		if (rsrc-&gt;type == IRDMA_PUDA_RSRC_TYPE_ILQ)</div><div class='add'>+			ibdev_dbg(to_ibdev(rsrc-&gt;dev),</div><div class='add'>+				  "PUDA: adding to txpend\n");</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+	rsrc-&gt;tx_wqe_avail_cnt--;</div><div class='add'>+	/* if we are coming from a completion and have pending buffers</div><div class='add'>+	 * then Get one from pending list</div><div class='add'>+	 */</div><div class='add'>+	if (!buf) {</div><div class='add'>+		buf = irdma_puda_get_listbuf(&amp;rsrc-&gt;txpend);</div><div class='add'>+		if (!buf)</div><div class='add'>+			goto done;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	info.scratch = buf;</div><div class='add'>+	info.paddr = buf-&gt;mem.pa;</div><div class='add'>+	info.len = buf-&gt;totallen;</div><div class='add'>+	info.tcplen = buf-&gt;tcphlen;</div><div class='add'>+	info.ipv4 = buf-&gt;ipv4;</div><div class='add'>+</div><div class='add'>+	if (rsrc-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+		info.ah_id = buf-&gt;ah_id;</div><div class='add'>+	} else {</div><div class='add'>+		info.maclen = buf-&gt;maclen;</div><div class='add'>+		info.do_lpb = buf-&gt;do_lpb;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Synch buffer for use by device */</div><div class='add'>+	dma_sync_single_for_cpu(rsrc-&gt;dev-&gt;hw-&gt;device, buf-&gt;mem.pa,</div><div class='add'>+				buf-&gt;mem.size, DMA_BIDIRECTIONAL);</div><div class='add'>+	ret = irdma_puda_send(&amp;rsrc-&gt;qp, &amp;info);</div><div class='add'>+	if (ret) {</div><div class='add'>+		rsrc-&gt;tx_wqe_avail_cnt++;</div><div class='add'>+		rsrc-&gt;stats_sent_pkt_q++;</div><div class='add'>+		list_add(&amp;buf-&gt;list, &amp;rsrc-&gt;txpend);</div><div class='add'>+		if (rsrc-&gt;type == IRDMA_PUDA_RSRC_TYPE_ILQ)</div><div class='add'>+			ibdev_dbg(to_ibdev(rsrc-&gt;dev),</div><div class='add'>+				  "PUDA: adding to puda_send\n");</div><div class='add'>+	} else {</div><div class='add'>+		rsrc-&gt;stats_pkt_sent++;</div><div class='add'>+	}</div><div class='add'>+done:</div><div class='add'>+	spin_unlock_irqrestore(&amp;rsrc-&gt;bufpool_lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_qp_setctx - during init, set qp's context</div><div class='add'>+ * @rsrc: qp's resource</div><div class='add'>+ */</div><div class='add'>+static void irdma_puda_qp_setctx(struct irdma_puda_rsrc *rsrc)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_qp *qp = &amp;rsrc-&gt;qp;</div><div class='add'>+	__le64 *qp_ctx = qp-&gt;hw_host_ctx;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(qp_ctx, 8, qp-&gt;sq_pa);</div><div class='add'>+	set_64bit_val(qp_ctx, 16, qp-&gt;rq_pa);</div><div class='add'>+	set_64bit_val(qp_ctx, 24,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RQSIZE, qp-&gt;hw_rq_size) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_SQSIZE, qp-&gt;hw_sq_size));</div><div class='add'>+	set_64bit_val(qp_ctx, 48,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_SNDMSS, rsrc-&gt;buf_size));</div><div class='add'>+	set_64bit_val(qp_ctx, 56, 0);</div><div class='add'>+	if (qp-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)</div><div class='add'>+		set_64bit_val(qp_ctx, 64, 1);</div><div class='add'>+	set_64bit_val(qp_ctx, 136,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_TXCQNUM, rsrc-&gt;cq_id) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RXCQNUM, rsrc-&gt;cq_id));</div><div class='add'>+	set_64bit_val(qp_ctx, 144,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_STAT_INDEX, rsrc-&gt;stats_idx));</div><div class='add'>+	set_64bit_val(qp_ctx, 160,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_PRIVEN, 1) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, rsrc-&gt;stats_idx_valid));</div><div class='add'>+	set_64bit_val(qp_ctx, 168,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_QPCOMPCTX, (uintptr_t)qp));</div><div class='add'>+	set_64bit_val(qp_ctx, 176,</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_SQTPHVAL, qp-&gt;sq_tph_val) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_RQTPHVAL, qp-&gt;rq_tph_val) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPC_QSHANDLE, qp-&gt;qs_handle));</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("PUDA: PUDA QP CONTEXT", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, qp_ctx, IRDMA_QP_CTX_SIZE, false);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_qp_wqe - setup wqe for qp create</div><div class='add'>+ * @dev: Device</div><div class='add'>+ * @qp: Resource qp</div><div class='add'>+ */</div><div class='add'>+static int irdma_puda_qp_wqe(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	struct irdma_ccq_cqe_info compl_info;</div><div class='add'>+	int status = 0;</div><div class='add'>+</div><div class='add'>+	cqp = dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, 0);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16, qp-&gt;hw_host_ctx_pa);</div><div class='add'>+	set_64bit_val(wqe, 40, qp-&gt;shadow_area_pa);</div><div class='add'>+</div><div class='add'>+	hdr = qp-&gt;qp_uk.qp_id |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_QP) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, IRDMA_QP_TYPE_UDA) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, 2) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("PUDA: PUDA QP CREATE", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, wqe, 40, false);</div><div class='add'>+	irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+	status = irdma_sc_poll_for_cqp_op_done(dev-&gt;cqp, IRDMA_CQP_OP_CREATE_QP,</div><div class='add'>+					       &amp;compl_info);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_qp_create - create qp for resource</div><div class='add'>+ * @rsrc: resource to use for buffer</div><div class='add'>+ */</div><div class='add'>+static int irdma_puda_qp_create(struct irdma_puda_rsrc *rsrc)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_qp *qp = &amp;rsrc-&gt;qp;</div><div class='add'>+	struct irdma_qp_uk *ukqp = &amp;qp-&gt;qp_uk;</div><div class='add'>+	int ret = 0;</div><div class='add'>+	u32 sq_size, rq_size;</div><div class='add'>+	struct irdma_dma_mem *mem;</div><div class='add'>+</div><div class='add'>+	sq_size = rsrc-&gt;sq_size * IRDMA_QP_WQE_MIN_SIZE;</div><div class='add'>+	rq_size = rsrc-&gt;rq_size * IRDMA_QP_WQE_MIN_SIZE;</div><div class='add'>+	rsrc-&gt;qpmem.size = ALIGN((sq_size + rq_size + (IRDMA_SHADOW_AREA_SIZE &lt;&lt; 3) + IRDMA_QP_CTX_SIZE),</div><div class='add'>+				 IRDMA_HW_PAGE_SIZE);</div><div class='add'>+	rsrc-&gt;qpmem.va = dma_alloc_coherent(rsrc-&gt;dev-&gt;hw-&gt;device,</div><div class='add'>+					    rsrc-&gt;qpmem.size, &amp;rsrc-&gt;qpmem.pa,</div><div class='add'>+					    GFP_KERNEL);</div><div class='add'>+	if (!rsrc-&gt;qpmem.va)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	mem = &amp;rsrc-&gt;qpmem;</div><div class='add'>+	memset(mem-&gt;va, 0, rsrc-&gt;qpmem.size);</div><div class='add'>+	qp-&gt;hw_sq_size = irdma_get_encoded_wqe_size(rsrc-&gt;sq_size, IRDMA_QUEUE_TYPE_SQ_RQ);</div><div class='add'>+	qp-&gt;hw_rq_size = irdma_get_encoded_wqe_size(rsrc-&gt;rq_size, IRDMA_QUEUE_TYPE_SQ_RQ);</div><div class='add'>+	qp-&gt;pd = &amp;rsrc-&gt;sc_pd;</div><div class='add'>+	qp-&gt;qp_uk.qp_type = IRDMA_QP_TYPE_UDA;</div><div class='add'>+	qp-&gt;dev = rsrc-&gt;dev;</div><div class='add'>+	qp-&gt;qp_uk.back_qp = rsrc;</div><div class='add'>+	qp-&gt;sq_pa = mem-&gt;pa;</div><div class='add'>+	qp-&gt;rq_pa = qp-&gt;sq_pa + sq_size;</div><div class='add'>+	qp-&gt;vsi = rsrc-&gt;vsi;</div><div class='add'>+	ukqp-&gt;sq_base = mem-&gt;va;</div><div class='add'>+	ukqp-&gt;rq_base = &amp;ukqp-&gt;sq_base[rsrc-&gt;sq_size];</div><div class='add'>+	ukqp-&gt;shadow_area = ukqp-&gt;rq_base[rsrc-&gt;rq_size].elem;</div><div class='add'>+	ukqp-&gt;uk_attrs = &amp;qp-&gt;dev-&gt;hw_attrs.uk_attrs;</div><div class='add'>+	qp-&gt;shadow_area_pa = qp-&gt;rq_pa + rq_size;</div><div class='add'>+	qp-&gt;hw_host_ctx = ukqp-&gt;shadow_area + IRDMA_SHADOW_AREA_SIZE;</div><div class='add'>+	qp-&gt;hw_host_ctx_pa = qp-&gt;shadow_area_pa + (IRDMA_SHADOW_AREA_SIZE &lt;&lt; 3);</div><div class='add'>+	qp-&gt;push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;</div><div class='add'>+	ukqp-&gt;qp_id = rsrc-&gt;qp_id;</div><div class='add'>+	ukqp-&gt;sq_wrtrk_array = rsrc-&gt;sq_wrtrk_array;</div><div class='add'>+	ukqp-&gt;rq_wrid_array = rsrc-&gt;rq_wrid_array;</div><div class='add'>+	ukqp-&gt;sq_size = rsrc-&gt;sq_size;</div><div class='add'>+	ukqp-&gt;rq_size = rsrc-&gt;rq_size;</div><div class='add'>+</div><div class='add'>+	IRDMA_RING_INIT(ukqp-&gt;sq_ring, ukqp-&gt;sq_size);</div><div class='add'>+	IRDMA_RING_INIT(ukqp-&gt;rq_ring, ukqp-&gt;rq_size);</div><div class='add'>+	ukqp-&gt;wqe_alloc_db = qp-&gt;pd-&gt;dev-&gt;wqe_alloc_db;</div><div class='add'>+</div><div class='add'>+	ret = rsrc-&gt;dev-&gt;ws_add(qp-&gt;vsi, qp-&gt;user_pri);</div><div class='add'>+	if (ret) {</div><div class='add'>+		dma_free_coherent(rsrc-&gt;dev-&gt;hw-&gt;device, rsrc-&gt;qpmem.size,</div><div class='add'>+				  rsrc-&gt;qpmem.va, rsrc-&gt;qpmem.pa);</div><div class='add'>+		rsrc-&gt;qpmem.va = NULL;</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_qp_add_qos(qp);</div><div class='add'>+	irdma_puda_qp_setctx(rsrc);</div><div class='add'>+</div><div class='add'>+	if (rsrc-&gt;dev-&gt;ceq_valid)</div><div class='add'>+		ret = irdma_cqp_qp_create_cmd(rsrc-&gt;dev, qp);</div><div class='add'>+	else</div><div class='add'>+		ret = irdma_puda_qp_wqe(rsrc-&gt;dev, qp);</div><div class='add'>+	if (ret) {</div><div class='add'>+		irdma_qp_rem_qos(qp);</div><div class='add'>+		rsrc-&gt;dev-&gt;ws_remove(qp-&gt;vsi, qp-&gt;user_pri);</div><div class='add'>+		dma_free_coherent(rsrc-&gt;dev-&gt;hw-&gt;device, rsrc-&gt;qpmem.size,</div><div class='add'>+				  rsrc-&gt;qpmem.va, rsrc-&gt;qpmem.pa);</div><div class='add'>+		rsrc-&gt;qpmem.va = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_cq_wqe - setup wqe for CQ create</div><div class='add'>+ * @dev: Device</div><div class='add'>+ * @cq: resource for cq</div><div class='add'>+ */</div><div class='add'>+static int irdma_puda_cq_wqe(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	struct irdma_ccq_cqe_info compl_info;</div><div class='add'>+</div><div class='add'>+	cqp = dev-&gt;cqp;</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, 0);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0, cq-&gt;cq_uk.cq_size);</div><div class='add'>+	set_64bit_val(wqe, 8, (uintptr_t)cq &gt;&gt; 1);</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq-&gt;shadow_read_threshold));</div><div class='add'>+	set_64bit_val(wqe, 32, cq-&gt;cq_pa);</div><div class='add'>+	set_64bit_val(wqe, 40, cq-&gt;shadow_area_pa);</div><div class='add'>+	set_64bit_val(wqe, 56,</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq-&gt;tph_val) |</div><div class='add'>+		      FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq-&gt;vsi-&gt;vsi_idx));</div><div class='add'>+</div><div class='add'>+	hdr = cq-&gt;cq_uk.cq_id |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp-&gt;polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("PUDA: PUDA CREATE CQ", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	irdma_sc_cqp_post_sq(dev-&gt;cqp);</div><div class='add'>+	return irdma_sc_poll_for_cqp_op_done(dev-&gt;cqp, IRDMA_CQP_OP_CREATE_CQ,</div><div class='add'>+					     &amp;compl_info);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_cq_create - create cq for resource</div><div class='add'>+ * @rsrc: resource for which cq to create</div><div class='add'>+ */</div><div class='add'>+static int irdma_puda_cq_create(struct irdma_puda_rsrc *rsrc)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = rsrc-&gt;dev;</div><div class='add'>+	struct irdma_sc_cq *cq = &amp;rsrc-&gt;cq;</div><div class='add'>+	int ret = 0;</div><div class='add'>+	u32 cqsize;</div><div class='add'>+	struct irdma_dma_mem *mem;</div><div class='add'>+	struct irdma_cq_init_info info = {};</div><div class='add'>+	struct irdma_cq_uk_init_info *init_info = &amp;info.cq_uk_init_info;</div><div class='add'>+</div><div class='add'>+	cq-&gt;vsi = rsrc-&gt;vsi;</div><div class='add'>+	cqsize = rsrc-&gt;cq_size * (sizeof(struct irdma_cqe));</div><div class='add'>+	rsrc-&gt;cqmem.size = ALIGN(cqsize + sizeof(struct irdma_cq_shadow_area),</div><div class='add'>+				 IRDMA_CQ0_ALIGNMENT);</div><div class='add'>+	rsrc-&gt;cqmem.va = dma_alloc_coherent(dev-&gt;hw-&gt;device, rsrc-&gt;cqmem.size,</div><div class='add'>+					    &amp;rsrc-&gt;cqmem.pa, GFP_KERNEL);</div><div class='add'>+	if (!rsrc-&gt;cqmem.va)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	mem = &amp;rsrc-&gt;cqmem;</div><div class='add'>+	info.dev = dev;</div><div class='add'>+	info.type = (rsrc-&gt;type == IRDMA_PUDA_RSRC_TYPE_ILQ) ?</div><div class='add'>+		    IRDMA_CQ_TYPE_ILQ : IRDMA_CQ_TYPE_IEQ;</div><div class='add'>+	info.shadow_read_threshold = rsrc-&gt;cq_size &gt;&gt; 2;</div><div class='add'>+	info.cq_base_pa = mem-&gt;pa;</div><div class='add'>+	info.shadow_area_pa = mem-&gt;pa + cqsize;</div><div class='add'>+	init_info-&gt;cq_base = mem-&gt;va;</div><div class='add'>+	init_info-&gt;shadow_area = (__le64 *)((u8 *)mem-&gt;va + cqsize);</div><div class='add'>+	init_info-&gt;cq_size = rsrc-&gt;cq_size;</div><div class='add'>+	init_info-&gt;cq_id = rsrc-&gt;cq_id;</div><div class='add'>+	info.ceqe_mask = true;</div><div class='add'>+	info.ceq_id_valid = true;</div><div class='add'>+	info.vsi = rsrc-&gt;vsi;</div><div class='add'>+</div><div class='add'>+	ret = irdma_sc_cq_init(cq, &amp;info);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	if (rsrc-&gt;dev-&gt;ceq_valid)</div><div class='add'>+		ret = irdma_cqp_cq_create_cmd(dev, cq);</div><div class='add'>+	else</div><div class='add'>+		ret = irdma_puda_cq_wqe(dev, cq);</div><div class='add'>+error:</div><div class='add'>+	if (ret) {</div><div class='add'>+		dma_free_coherent(dev-&gt;hw-&gt;device, rsrc-&gt;cqmem.size,</div><div class='add'>+				  rsrc-&gt;cqmem.va, rsrc-&gt;cqmem.pa);</div><div class='add'>+		rsrc-&gt;cqmem.va = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_free_qp - free qp for resource</div><div class='add'>+ * @rsrc: resource for which qp to free</div><div class='add'>+ */</div><div class='add'>+static void irdma_puda_free_qp(struct irdma_puda_rsrc *rsrc)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+	struct irdma_ccq_cqe_info compl_info;</div><div class='add'>+	struct irdma_sc_dev *dev = rsrc-&gt;dev;</div><div class='add'>+</div><div class='add'>+	if (rsrc-&gt;dev-&gt;ceq_valid) {</div><div class='add'>+		irdma_cqp_qp_destroy_cmd(dev, &amp;rsrc-&gt;qp);</div><div class='add'>+		rsrc-&gt;dev-&gt;ws_remove(rsrc-&gt;qp.vsi, rsrc-&gt;qp.user_pri);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = irdma_sc_qp_destroy(&amp;rsrc-&gt;qp, 0, false, true, true);</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "PUDA: error puda qp destroy wqe, status = %d\n",</div><div class='add'>+			  ret);</div><div class='add'>+	if (!ret) {</div><div class='add'>+		ret = irdma_sc_poll_for_cqp_op_done(dev-&gt;cqp, IRDMA_CQP_OP_DESTROY_QP,</div><div class='add'>+						    &amp;compl_info);</div><div class='add'>+		if (ret)</div><div class='add'>+			ibdev_dbg(to_ibdev(dev),</div><div class='add'>+				  "PUDA: error puda qp destroy failed, status = %d\n",</div><div class='add'>+				  ret);</div><div class='add'>+	}</div><div class='add'>+	rsrc-&gt;dev-&gt;ws_remove(rsrc-&gt;qp.vsi, rsrc-&gt;qp.user_pri);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_free_cq - free cq for resource</div><div class='add'>+ * @rsrc: resource for which cq to free</div><div class='add'>+ */</div><div class='add'>+static void irdma_puda_free_cq(struct irdma_puda_rsrc *rsrc)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+	struct irdma_ccq_cqe_info compl_info;</div><div class='add'>+	struct irdma_sc_dev *dev = rsrc-&gt;dev;</div><div class='add'>+</div><div class='add'>+	if (rsrc-&gt;dev-&gt;ceq_valid) {</div><div class='add'>+		irdma_cqp_cq_destroy_cmd(dev, &amp;rsrc-&gt;cq);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = irdma_sc_cq_destroy(&amp;rsrc-&gt;cq, 0, true);</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "PUDA: error ieq cq destroy\n");</div><div class='add'>+	if (!ret) {</div><div class='add'>+		ret = irdma_sc_poll_for_cqp_op_done(dev-&gt;cqp, IRDMA_CQP_OP_DESTROY_CQ,</div><div class='add'>+						    &amp;compl_info);</div><div class='add'>+		if (ret)</div><div class='add'>+			ibdev_dbg(to_ibdev(dev),</div><div class='add'>+				  "PUDA: error ieq qp destroy done\n");</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_dele_rsrc - delete all resources during close</div><div class='add'>+ * @vsi: VSI structure of device</div><div class='add'>+ * @type: type of resource to dele</div><div class='add'>+ * @reset: true if reset chip</div><div class='add'>+ */</div><div class='add'>+void irdma_puda_dele_rsrc(struct irdma_sc_vsi *vsi, enum puda_rsrc_type type,</div><div class='add'>+			  bool reset)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = vsi-&gt;dev;</div><div class='add'>+	struct irdma_puda_rsrc *rsrc;</div><div class='add'>+	struct irdma_puda_buf *buf = NULL;</div><div class='add'>+	struct irdma_puda_buf *nextbuf = NULL;</div><div class='add'>+	struct irdma_virt_mem *vmem;</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case IRDMA_PUDA_RSRC_TYPE_ILQ:</div><div class='add'>+		rsrc = vsi-&gt;ilq;</div><div class='add'>+		vmem = &amp;vsi-&gt;ilq_mem;</div><div class='add'>+		vsi-&gt;ilq = NULL;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_PUDA_RSRC_TYPE_IEQ:</div><div class='add'>+		rsrc = vsi-&gt;ieq;</div><div class='add'>+		vmem = &amp;vsi-&gt;ieq_mem;</div><div class='add'>+		vsi-&gt;ieq = NULL;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "PUDA: error resource type = 0x%x\n",</div><div class='add'>+			  type);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (rsrc-&gt;cmpl) {</div><div class='add'>+	case PUDA_HASH_CRC_COMPLETE:</div><div class='add'>+	case PUDA_QP_CREATED:</div><div class='add'>+		irdma_qp_rem_qos(&amp;rsrc-&gt;qp);</div><div class='add'>+</div><div class='add'>+		if (!reset)</div><div class='add'>+			irdma_puda_free_qp(rsrc);</div><div class='add'>+</div><div class='add'>+		dma_free_coherent(dev-&gt;hw-&gt;device, rsrc-&gt;qpmem.size,</div><div class='add'>+				  rsrc-&gt;qpmem.va, rsrc-&gt;qpmem.pa);</div><div class='add'>+		rsrc-&gt;qpmem.va = NULL;</div><div class='add'>+		fallthrough;</div><div class='add'>+	case PUDA_CQ_CREATED:</div><div class='add'>+		if (!reset)</div><div class='add'>+			irdma_puda_free_cq(rsrc);</div><div class='add'>+</div><div class='add'>+		dma_free_coherent(dev-&gt;hw-&gt;device, rsrc-&gt;cqmem.size,</div><div class='add'>+				  rsrc-&gt;cqmem.va, rsrc-&gt;cqmem.pa);</div><div class='add'>+		rsrc-&gt;cqmem.va = NULL;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ibdev_dbg(to_ibdev(rsrc-&gt;dev), "PUDA: error no resources\n");</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+	/* Free all allocated puda buffers for both tx and rx */</div><div class='add'>+	buf = rsrc-&gt;alloclist;</div><div class='add'>+	while (buf) {</div><div class='add'>+		nextbuf = buf-&gt;next;</div><div class='add'>+		irdma_puda_dele_buf(dev, buf);</div><div class='add'>+		buf = nextbuf;</div><div class='add'>+		rsrc-&gt;alloc_buf_count--;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	kfree(vmem-&gt;va);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_allocbufs - allocate buffers for resource</div><div class='add'>+ * @rsrc: resource for buffer allocation</div><div class='add'>+ * @count: number of buffers to create</div><div class='add'>+ */</div><div class='add'>+static int irdma_puda_allocbufs(struct irdma_puda_rsrc *rsrc, u32 count)</div><div class='add'>+{</div><div class='add'>+	u32 i;</div><div class='add'>+	struct irdma_puda_buf *buf;</div><div class='add'>+	struct irdma_puda_buf *nextbuf;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; count; i++) {</div><div class='add'>+		buf = irdma_puda_alloc_buf(rsrc-&gt;dev, rsrc-&gt;buf_size);</div><div class='add'>+		if (!buf) {</div><div class='add'>+			rsrc-&gt;stats_buf_alloc_fail++;</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		}</div><div class='add'>+		irdma_puda_ret_bufpool(rsrc, buf);</div><div class='add'>+		rsrc-&gt;alloc_buf_count++;</div><div class='add'>+		if (!rsrc-&gt;alloclist) {</div><div class='add'>+			rsrc-&gt;alloclist = buf;</div><div class='add'>+		} else {</div><div class='add'>+			nextbuf = rsrc-&gt;alloclist;</div><div class='add'>+			rsrc-&gt;alloclist = buf;</div><div class='add'>+			buf-&gt;next = nextbuf;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rsrc-&gt;avail_buf_count = rsrc-&gt;alloc_buf_count;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_create_rsrc - create resource (ilq or ieq)</div><div class='add'>+ * @vsi: sc VSI struct</div><div class='add'>+ * @info: resource information</div><div class='add'>+ */</div><div class='add'>+int irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi,</div><div class='add'>+			   struct irdma_puda_rsrc_info *info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = vsi-&gt;dev;</div><div class='add'>+	int ret = 0;</div><div class='add'>+	struct irdma_puda_rsrc *rsrc;</div><div class='add'>+	u32 pudasize;</div><div class='add'>+	u32 sqwridsize, rqwridsize;</div><div class='add'>+	struct irdma_virt_mem *vmem;</div><div class='add'>+</div><div class='add'>+	info-&gt;count = 1;</div><div class='add'>+	pudasize = sizeof(struct irdma_puda_rsrc);</div><div class='add'>+	sqwridsize = info-&gt;sq_size * sizeof(struct irdma_sq_uk_wr_trk_info);</div><div class='add'>+	rqwridsize = info-&gt;rq_size * 8;</div><div class='add'>+	switch (info-&gt;type) {</div><div class='add'>+	case IRDMA_PUDA_RSRC_TYPE_ILQ:</div><div class='add'>+		vmem = &amp;vsi-&gt;ilq_mem;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_PUDA_RSRC_TYPE_IEQ:</div><div class='add'>+		vmem = &amp;vsi-&gt;ieq_mem;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+	vmem-&gt;size = pudasize + sqwridsize + rqwridsize;</div><div class='add'>+	vmem-&gt;va = kzalloc(vmem-&gt;size, GFP_KERNEL);</div><div class='add'>+	if (!vmem-&gt;va)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	rsrc = vmem-&gt;va;</div><div class='add'>+	spin_lock_init(&amp;rsrc-&gt;bufpool_lock);</div><div class='add'>+	switch (info-&gt;type) {</div><div class='add'>+	case IRDMA_PUDA_RSRC_TYPE_ILQ:</div><div class='add'>+		vsi-&gt;ilq = vmem-&gt;va;</div><div class='add'>+		vsi-&gt;ilq_count = info-&gt;count;</div><div class='add'>+		rsrc-&gt;receive = info-&gt;receive;</div><div class='add'>+		rsrc-&gt;xmit_complete = info-&gt;xmit_complete;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_PUDA_RSRC_TYPE_IEQ:</div><div class='add'>+		vsi-&gt;ieq_count = info-&gt;count;</div><div class='add'>+		vsi-&gt;ieq = vmem-&gt;va;</div><div class='add'>+		rsrc-&gt;receive = irdma_ieq_receive;</div><div class='add'>+		rsrc-&gt;xmit_complete = irdma_ieq_tx_compl;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rsrc-&gt;type = info-&gt;type;</div><div class='add'>+	rsrc-&gt;sq_wrtrk_array = (struct irdma_sq_uk_wr_trk_info *)</div><div class='add'>+			       ((u8 *)vmem-&gt;va + pudasize);</div><div class='add'>+	rsrc-&gt;rq_wrid_array = (u64 *)((u8 *)vmem-&gt;va + pudasize + sqwridsize);</div><div class='add'>+	/* Initialize all ieq lists */</div><div class='add'>+	INIT_LIST_HEAD(&amp;rsrc-&gt;bufpool);</div><div class='add'>+	INIT_LIST_HEAD(&amp;rsrc-&gt;txpend);</div><div class='add'>+</div><div class='add'>+	rsrc-&gt;tx_wqe_avail_cnt = info-&gt;sq_size - 1;</div><div class='add'>+	irdma_sc_pd_init(dev, &amp;rsrc-&gt;sc_pd, info-&gt;pd_id, info-&gt;abi_ver);</div><div class='add'>+	rsrc-&gt;qp_id = info-&gt;qp_id;</div><div class='add'>+	rsrc-&gt;cq_id = info-&gt;cq_id;</div><div class='add'>+	rsrc-&gt;sq_size = info-&gt;sq_size;</div><div class='add'>+	rsrc-&gt;rq_size = info-&gt;rq_size;</div><div class='add'>+	rsrc-&gt;cq_size = info-&gt;rq_size + info-&gt;sq_size;</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+		if (rsrc-&gt;type == IRDMA_PUDA_RSRC_TYPE_ILQ)</div><div class='add'>+			rsrc-&gt;cq_size += info-&gt;rq_size;</div><div class='add'>+	}</div><div class='add'>+	rsrc-&gt;buf_size = info-&gt;buf_size;</div><div class='add'>+	rsrc-&gt;dev = dev;</div><div class='add'>+	rsrc-&gt;vsi = vsi;</div><div class='add'>+	rsrc-&gt;stats_idx = info-&gt;stats_idx;</div><div class='add'>+	rsrc-&gt;stats_idx_valid = info-&gt;stats_idx_valid;</div><div class='add'>+</div><div class='add'>+	ret = irdma_puda_cq_create(rsrc);</div><div class='add'>+	if (!ret) {</div><div class='add'>+		rsrc-&gt;cmpl = PUDA_CQ_CREATED;</div><div class='add'>+		ret = irdma_puda_qp_create(rsrc);</div><div class='add'>+	}</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "PUDA: error qp_create type=%d, status=%d\n",</div><div class='add'>+			  rsrc-&gt;type, ret);</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+	rsrc-&gt;cmpl = PUDA_QP_CREATED;</div><div class='add'>+</div><div class='add'>+	ret = irdma_puda_allocbufs(rsrc, info-&gt;tx_buf_cnt + info-&gt;rq_size);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "PUDA: error alloc_buf\n");</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rsrc-&gt;rxq_invalid_cnt = info-&gt;rq_size;</div><div class='add'>+	ret = irdma_puda_replenish_rq(rsrc, true);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;type == IRDMA_PUDA_RSRC_TYPE_IEQ) {</div><div class='add'>+		rsrc-&gt;check_crc = true;</div><div class='add'>+		rsrc-&gt;cmpl = PUDA_HASH_CRC_COMPLETE;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_sc_ccq_arm(&amp;rsrc-&gt;cq);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+error:</div><div class='add'>+	irdma_puda_dele_rsrc(vsi, info-&gt;type, false);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ilq_putback_rcvbuf - ilq buffer to put back on rq</div><div class='add'>+ * @qp: ilq's qp resource</div><div class='add'>+ * @buf: puda buffer for rcv q</div><div class='add'>+ * @wqe_idx:  wqe index of completed rcvbuf</div><div class='add'>+ */</div><div class='add'>+static void irdma_ilq_putback_rcvbuf(struct irdma_sc_qp *qp,</div><div class='add'>+				     struct irdma_puda_buf *buf, u32 wqe_idx)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 offset8, offset24;</div><div class='add'>+</div><div class='add'>+	/* Synch buffer for use by device */</div><div class='add'>+	dma_sync_single_for_device(qp-&gt;dev-&gt;hw-&gt;device, buf-&gt;mem.pa,</div><div class='add'>+				   buf-&gt;mem.size, DMA_BIDIRECTIONAL);</div><div class='add'>+	wqe = qp-&gt;qp_uk.rq_base[wqe_idx].elem;</div><div class='add'>+	get_64bit_val(wqe, 24, &amp;offset24);</div><div class='add'>+	if (qp-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+		get_64bit_val(wqe, 8, &amp;offset8);</div><div class='add'>+		if (offset24)</div><div class='add'>+			offset8 &amp;= ~FIELD_PREP(IRDMAQPSQ_VALID, 1);</div><div class='add'>+		else</div><div class='add'>+			offset8 |= FIELD_PREP(IRDMAQPSQ_VALID, 1);</div><div class='add'>+		set_64bit_val(wqe, 8, offset8);</div><div class='add'>+		dma_wmb(); /* make sure WQE is written before valid bit is set */</div><div class='add'>+	}</div><div class='add'>+	if (offset24)</div><div class='add'>+		offset24 = 0;</div><div class='add'>+	else</div><div class='add'>+		offset24 = FIELD_PREP(IRDMAQPSQ_VALID, 1);</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, offset24);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_get_fpdu_len - get length of fpdu with or without marker</div><div class='add'>+ * @pfpdu: pointer to fpdu</div><div class='add'>+ * @datap: pointer to data in the buffer</div><div class='add'>+ * @rcv_seq: seqnum of the data buffer</div><div class='add'>+ */</div><div class='add'>+static u16 irdma_ieq_get_fpdu_len(struct irdma_pfpdu *pfpdu, u8 *datap,</div><div class='add'>+				  u32 rcv_seq)</div><div class='add'>+{</div><div class='add'>+	u32 marker_seq, end_seq, blk_start;</div><div class='add'>+	u8 marker_len = pfpdu-&gt;marker_len;</div><div class='add'>+	u16 total_len = 0;</div><div class='add'>+	u16 fpdu_len;</div><div class='add'>+</div><div class='add'>+	blk_start = (pfpdu-&gt;rcv_start_seq - rcv_seq) &amp; (IRDMA_MRK_BLK_SZ - 1);</div><div class='add'>+	if (!blk_start) {</div><div class='add'>+		total_len = marker_len;</div><div class='add'>+		marker_seq = rcv_seq + IRDMA_MRK_BLK_SZ;</div><div class='add'>+		if (marker_len &amp;&amp; *(u32 *)datap)</div><div class='add'>+			return 0;</div><div class='add'>+	} else {</div><div class='add'>+		marker_seq = rcv_seq + blk_start;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	datap += total_len;</div><div class='add'>+	fpdu_len = ntohs(*(__be16 *)datap);</div><div class='add'>+	fpdu_len += IRDMA_IEQ_MPA_FRAMING;</div><div class='add'>+	fpdu_len = (fpdu_len + 3) &amp; 0xfffc;</div><div class='add'>+</div><div class='add'>+	if (fpdu_len &gt; pfpdu-&gt;max_fpdu_data)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	total_len += fpdu_len;</div><div class='add'>+	end_seq = rcv_seq + total_len;</div><div class='add'>+	while ((int)(marker_seq - end_seq) &lt; 0) {</div><div class='add'>+		total_len += marker_len;</div><div class='add'>+		end_seq += marker_len;</div><div class='add'>+		marker_seq += IRDMA_MRK_BLK_SZ;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return total_len;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_copy_to_txbuf - copydata from rcv buf to tx buf</div><div class='add'>+ * @buf: rcv buffer with partial</div><div class='add'>+ * @txbuf: tx buffer for sending back</div><div class='add'>+ * @buf_offset: rcv buffer offset to copy from</div><div class='add'>+ * @txbuf_offset: at offset in tx buf to copy</div><div class='add'>+ * @len: length of data to copy</div><div class='add'>+ */</div><div class='add'>+static void irdma_ieq_copy_to_txbuf(struct irdma_puda_buf *buf,</div><div class='add'>+				    struct irdma_puda_buf *txbuf,</div><div class='add'>+				    u16 buf_offset, u32 txbuf_offset, u32 len)</div><div class='add'>+{</div><div class='add'>+	void *mem1 = (u8 *)buf-&gt;mem.va + buf_offset;</div><div class='add'>+	void *mem2 = (u8 *)txbuf-&gt;mem.va + txbuf_offset;</div><div class='add'>+</div><div class='add'>+	memcpy(mem2, mem1, len);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_setup_tx_buf - setup tx buffer for partial handling</div><div class='add'>+ * @buf: reeive buffer with partial</div><div class='add'>+ * @txbuf: buffer to prepare</div><div class='add'>+ */</div><div class='add'>+static void irdma_ieq_setup_tx_buf(struct irdma_puda_buf *buf,</div><div class='add'>+				   struct irdma_puda_buf *txbuf)</div><div class='add'>+{</div><div class='add'>+	txbuf-&gt;tcphlen = buf-&gt;tcphlen;</div><div class='add'>+	txbuf-&gt;ipv4 = buf-&gt;ipv4;</div><div class='add'>+</div><div class='add'>+	if (buf-&gt;vsi-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+		txbuf-&gt;hdrlen = txbuf-&gt;tcphlen;</div><div class='add'>+		irdma_ieq_copy_to_txbuf(buf, txbuf, IRDMA_TCP_OFFSET, 0,</div><div class='add'>+					txbuf-&gt;hdrlen);</div><div class='add'>+	} else {</div><div class='add'>+		txbuf-&gt;maclen = buf-&gt;maclen;</div><div class='add'>+		txbuf-&gt;hdrlen = buf-&gt;hdrlen;</div><div class='add'>+		irdma_ieq_copy_to_txbuf(buf, txbuf, 0, 0, buf-&gt;hdrlen);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_check_first_buf - check if rcv buffer's seq is in range</div><div class='add'>+ * @buf: receive exception buffer</div><div class='add'>+ * @fps: first partial sequence number</div><div class='add'>+ */</div><div class='add'>+static void irdma_ieq_check_first_buf(struct irdma_puda_buf *buf, u32 fps)</div><div class='add'>+{</div><div class='add'>+	u32 offset;</div><div class='add'>+</div><div class='add'>+	if (buf-&gt;seqnum &lt; fps) {</div><div class='add'>+		offset = fps - buf-&gt;seqnum;</div><div class='add'>+		if (offset &gt; buf-&gt;datalen)</div><div class='add'>+			return;</div><div class='add'>+		buf-&gt;data += offset;</div><div class='add'>+		buf-&gt;datalen -= (u16)offset;</div><div class='add'>+		buf-&gt;seqnum = fps;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_compl_pfpdu - write txbuf with full fpdu</div><div class='add'>+ * @ieq: ieq resource</div><div class='add'>+ * @rxlist: ieq's received buffer list</div><div class='add'>+ * @pbufl: temporary list for buffers for fpddu</div><div class='add'>+ * @txbuf: tx buffer for fpdu</div><div class='add'>+ * @fpdu_len: total length of fpdu</div><div class='add'>+ */</div><div class='add'>+static void irdma_ieq_compl_pfpdu(struct irdma_puda_rsrc *ieq,</div><div class='add'>+				  struct list_head *rxlist,</div><div class='add'>+				  struct list_head *pbufl,</div><div class='add'>+				  struct irdma_puda_buf *txbuf, u16 fpdu_len)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_buf *buf;</div><div class='add'>+	u32 nextseqnum;</div><div class='add'>+	u16 txoffset, bufoffset;</div><div class='add'>+</div><div class='add'>+	buf = irdma_puda_get_listbuf(pbufl);</div><div class='add'>+	if (!buf)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	nextseqnum = buf-&gt;seqnum + fpdu_len;</div><div class='add'>+	irdma_ieq_setup_tx_buf(buf, txbuf);</div><div class='add'>+	if (buf-&gt;vsi-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+		txoffset = txbuf-&gt;hdrlen;</div><div class='add'>+		txbuf-&gt;totallen = txbuf-&gt;hdrlen + fpdu_len;</div><div class='add'>+		txbuf-&gt;data = (u8 *)txbuf-&gt;mem.va + txoffset;</div><div class='add'>+	} else {</div><div class='add'>+		txoffset = buf-&gt;hdrlen;</div><div class='add'>+		txbuf-&gt;totallen = buf-&gt;hdrlen + fpdu_len;</div><div class='add'>+		txbuf-&gt;data = (u8 *)txbuf-&gt;mem.va + buf-&gt;hdrlen;</div><div class='add'>+	}</div><div class='add'>+	bufoffset = (u16)(buf-&gt;data - (u8 *)buf-&gt;mem.va);</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		if (buf-&gt;datalen &gt;= fpdu_len) {</div><div class='add'>+			/* copied full fpdu */</div><div class='add'>+			irdma_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset,</div><div class='add'>+						fpdu_len);</div><div class='add'>+			buf-&gt;datalen -= fpdu_len;</div><div class='add'>+			buf-&gt;data += fpdu_len;</div><div class='add'>+			buf-&gt;seqnum = nextseqnum;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		/* copy partial fpdu */</div><div class='add'>+		irdma_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset,</div><div class='add'>+					buf-&gt;datalen);</div><div class='add'>+		txoffset += buf-&gt;datalen;</div><div class='add'>+		fpdu_len -= buf-&gt;datalen;</div><div class='add'>+		irdma_puda_ret_bufpool(ieq, buf);</div><div class='add'>+		buf = irdma_puda_get_listbuf(pbufl);</div><div class='add'>+		if (!buf)</div><div class='add'>+			return;</div><div class='add'>+</div><div class='add'>+		bufoffset = (u16)(buf-&gt;data - (u8 *)buf-&gt;mem.va);</div><div class='add'>+	} while (1);</div><div class='add'>+</div><div class='add'>+	/* last buffer on the list*/</div><div class='add'>+	if (buf-&gt;datalen)</div><div class='add'>+		list_add(&amp;buf-&gt;list, rxlist);</div><div class='add'>+	else</div><div class='add'>+		irdma_puda_ret_bufpool(ieq, buf);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_create_pbufl - create buffer list for single fpdu</div><div class='add'>+ * @pfpdu: pointer to fpdu</div><div class='add'>+ * @rxlist: resource list for receive ieq buffes</div><div class='add'>+ * @pbufl: temp. list for buffers for fpddu</div><div class='add'>+ * @buf: first receive buffer</div><div class='add'>+ * @fpdu_len: total length of fpdu</div><div class='add'>+ */</div><div class='add'>+static int irdma_ieq_create_pbufl(struct irdma_pfpdu *pfpdu,</div><div class='add'>+				  struct list_head *rxlist,</div><div class='add'>+				  struct list_head *pbufl,</div><div class='add'>+				  struct irdma_puda_buf *buf, u16 fpdu_len)</div><div class='add'>+{</div><div class='add'>+	int status = 0;</div><div class='add'>+	struct irdma_puda_buf *nextbuf;</div><div class='add'>+	u32 nextseqnum;</div><div class='add'>+	u16 plen = fpdu_len - buf-&gt;datalen;</div><div class='add'>+	bool done = false;</div><div class='add'>+</div><div class='add'>+	nextseqnum = buf-&gt;seqnum + buf-&gt;datalen;</div><div class='add'>+	do {</div><div class='add'>+		nextbuf = irdma_puda_get_listbuf(rxlist);</div><div class='add'>+		if (!nextbuf) {</div><div class='add'>+			status = -ENOBUFS;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		list_add_tail(&amp;nextbuf-&gt;list, pbufl);</div><div class='add'>+		if (nextbuf-&gt;seqnum != nextseqnum) {</div><div class='add'>+			pfpdu-&gt;bad_seq_num++;</div><div class='add'>+			status = -ERANGE;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		if (nextbuf-&gt;datalen &gt;= plen) {</div><div class='add'>+			done = true;</div><div class='add'>+		} else {</div><div class='add'>+			plen -= nextbuf-&gt;datalen;</div><div class='add'>+			nextseqnum = nextbuf-&gt;seqnum + nextbuf-&gt;datalen;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+	} while (!done);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_handle_partial - process partial fpdu buffer</div><div class='add'>+ * @ieq: ieq resource</div><div class='add'>+ * @pfpdu: partial management per user qp</div><div class='add'>+ * @buf: receive buffer</div><div class='add'>+ * @fpdu_len: fpdu len in the buffer</div><div class='add'>+ */</div><div class='add'>+static int irdma_ieq_handle_partial(struct irdma_puda_rsrc *ieq,</div><div class='add'>+				    struct irdma_pfpdu *pfpdu,</div><div class='add'>+				    struct irdma_puda_buf *buf, u16 fpdu_len)</div><div class='add'>+{</div><div class='add'>+	int status = 0;</div><div class='add'>+	u8 *crcptr;</div><div class='add'>+	u32 mpacrc;</div><div class='add'>+	u32 seqnum = buf-&gt;seqnum;</div><div class='add'>+	struct list_head pbufl; /* partial buffer list */</div><div class='add'>+	struct irdma_puda_buf *txbuf = NULL;</div><div class='add'>+	struct list_head *rxlist = &amp;pfpdu-&gt;rxlist;</div><div class='add'>+</div><div class='add'>+	ieq-&gt;partials_handled++;</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;pbufl);</div><div class='add'>+	list_add(&amp;buf-&gt;list, &amp;pbufl);</div><div class='add'>+</div><div class='add'>+	status = irdma_ieq_create_pbufl(pfpdu, rxlist, &amp;pbufl, buf, fpdu_len);</div><div class='add'>+	if (status)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	txbuf = irdma_puda_get_bufpool(ieq);</div><div class='add'>+	if (!txbuf) {</div><div class='add'>+		pfpdu-&gt;no_tx_bufs++;</div><div class='add'>+		status = -ENOBUFS;</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_ieq_compl_pfpdu(ieq, rxlist, &amp;pbufl, txbuf, fpdu_len);</div><div class='add'>+	irdma_ieq_update_tcpip_info(txbuf, fpdu_len, seqnum);</div><div class='add'>+</div><div class='add'>+	crcptr = txbuf-&gt;data + fpdu_len - 4;</div><div class='add'>+	mpacrc = *(u32 *)crcptr;</div><div class='add'>+	if (ieq-&gt;check_crc) {</div><div class='add'>+		status = irdma_ieq_check_mpacrc(txbuf-&gt;data, fpdu_len - 4,</div><div class='add'>+						mpacrc);</div><div class='add'>+		if (status) {</div><div class='add'>+			ibdev_dbg(to_ibdev(ieq-&gt;dev), "IEQ: error bad crc\n");</div><div class='add'>+			goto error;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("IEQ: IEQ TX BUFFER", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     txbuf-&gt;mem.va, txbuf-&gt;totallen, false);</div><div class='add'>+	if (ieq-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2)</div><div class='add'>+		txbuf-&gt;ah_id = pfpdu-&gt;ah-&gt;ah_info.ah_idx;</div><div class='add'>+	txbuf-&gt;do_lpb = true;</div><div class='add'>+	irdma_puda_send_buf(ieq, txbuf);</div><div class='add'>+	pfpdu-&gt;rcv_nxt = seqnum + fpdu_len;</div><div class='add'>+	return status;</div><div class='add'>+</div><div class='add'>+error:</div><div class='add'>+	while (!list_empty(&amp;pbufl)) {</div><div class='add'>+		buf = list_last_entry(&amp;pbufl, struct irdma_puda_buf, list);</div><div class='add'>+		list_move(&amp;buf-&gt;list, rxlist);</div><div class='add'>+	}</div><div class='add'>+	if (txbuf)</div><div class='add'>+		irdma_puda_ret_bufpool(ieq, txbuf);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_process_buf - process buffer rcvd for ieq</div><div class='add'>+ * @ieq: ieq resource</div><div class='add'>+ * @pfpdu: partial management per user qp</div><div class='add'>+ * @buf: receive buffer</div><div class='add'>+ */</div><div class='add'>+static int irdma_ieq_process_buf(struct irdma_puda_rsrc *ieq,</div><div class='add'>+				 struct irdma_pfpdu *pfpdu,</div><div class='add'>+				 struct irdma_puda_buf *buf)</div><div class='add'>+{</div><div class='add'>+	u16 fpdu_len = 0;</div><div class='add'>+	u16 datalen = buf-&gt;datalen;</div><div class='add'>+	u8 *datap = buf-&gt;data;</div><div class='add'>+	u8 *crcptr;</div><div class='add'>+	u16 ioffset = 0;</div><div class='add'>+	u32 mpacrc;</div><div class='add'>+	u32 seqnum = buf-&gt;seqnum;</div><div class='add'>+	u16 len = 0;</div><div class='add'>+	u16 full = 0;</div><div class='add'>+	bool partial = false;</div><div class='add'>+	struct irdma_puda_buf *txbuf;</div><div class='add'>+	struct list_head *rxlist = &amp;pfpdu-&gt;rxlist;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	ioffset = (u16)(buf-&gt;data - (u8 *)buf-&gt;mem.va);</div><div class='add'>+	while (datalen) {</div><div class='add'>+		fpdu_len = irdma_ieq_get_fpdu_len(pfpdu, datap, buf-&gt;seqnum);</div><div class='add'>+		if (!fpdu_len) {</div><div class='add'>+			ibdev_dbg(to_ibdev(ieq-&gt;dev),</div><div class='add'>+				  "IEQ: error bad fpdu len\n");</div><div class='add'>+			list_add(&amp;buf-&gt;list, rxlist);</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (datalen &lt; fpdu_len) {</div><div class='add'>+			partial = true;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		crcptr = datap + fpdu_len - 4;</div><div class='add'>+		mpacrc = *(u32 *)crcptr;</div><div class='add'>+		if (ieq-&gt;check_crc)</div><div class='add'>+			ret = irdma_ieq_check_mpacrc(datap, fpdu_len - 4,</div><div class='add'>+						     mpacrc);</div><div class='add'>+		if (ret) {</div><div class='add'>+			list_add(&amp;buf-&gt;list, rxlist);</div><div class='add'>+			ibdev_dbg(to_ibdev(ieq-&gt;dev),</div><div class='add'>+				  "ERR: IRDMA_ERR_MPA_CRC\n");</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		}</div><div class='add'>+		full++;</div><div class='add'>+		pfpdu-&gt;fpdu_processed++;</div><div class='add'>+		ieq-&gt;fpdu_processed++;</div><div class='add'>+		datap += fpdu_len;</div><div class='add'>+		len += fpdu_len;</div><div class='add'>+		datalen -= fpdu_len;</div><div class='add'>+	}</div><div class='add'>+	if (full) {</div><div class='add'>+		/* copy full pdu's in the txbuf and send them out */</div><div class='add'>+		txbuf = irdma_puda_get_bufpool(ieq);</div><div class='add'>+		if (!txbuf) {</div><div class='add'>+			pfpdu-&gt;no_tx_bufs++;</div><div class='add'>+			list_add(&amp;buf-&gt;list, rxlist);</div><div class='add'>+			return -ENOBUFS;</div><div class='add'>+		}</div><div class='add'>+		/* modify txbuf's buffer header */</div><div class='add'>+		irdma_ieq_setup_tx_buf(buf, txbuf);</div><div class='add'>+		/* copy full fpdu's to new buffer */</div><div class='add'>+		if (ieq-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+			irdma_ieq_copy_to_txbuf(buf, txbuf, ioffset,</div><div class='add'>+						txbuf-&gt;hdrlen, len);</div><div class='add'>+			txbuf-&gt;totallen = txbuf-&gt;hdrlen + len;</div><div class='add'>+			txbuf-&gt;ah_id = pfpdu-&gt;ah-&gt;ah_info.ah_idx;</div><div class='add'>+		} else {</div><div class='add'>+			irdma_ieq_copy_to_txbuf(buf, txbuf, ioffset,</div><div class='add'>+						buf-&gt;hdrlen, len);</div><div class='add'>+			txbuf-&gt;totallen = buf-&gt;hdrlen + len;</div><div class='add'>+		}</div><div class='add'>+		irdma_ieq_update_tcpip_info(txbuf, len, buf-&gt;seqnum);</div><div class='add'>+		print_hex_dump_debug("IEQ: IEQ TX BUFFER", DUMP_PREFIX_OFFSET,</div><div class='add'>+				     16, 8, txbuf-&gt;mem.va, txbuf-&gt;totallen,</div><div class='add'>+				     false);</div><div class='add'>+		txbuf-&gt;do_lpb = true;</div><div class='add'>+		irdma_puda_send_buf(ieq, txbuf);</div><div class='add'>+</div><div class='add'>+		if (!datalen) {</div><div class='add'>+			pfpdu-&gt;rcv_nxt = buf-&gt;seqnum + len;</div><div class='add'>+			irdma_puda_ret_bufpool(ieq, buf);</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+		buf-&gt;data = datap;</div><div class='add'>+		buf-&gt;seqnum = seqnum + len;</div><div class='add'>+		buf-&gt;datalen = datalen;</div><div class='add'>+		pfpdu-&gt;rcv_nxt = buf-&gt;seqnum;</div><div class='add'>+	}</div><div class='add'>+	if (partial)</div><div class='add'>+		return irdma_ieq_handle_partial(ieq, pfpdu, buf, fpdu_len);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_process_fpdus - process fpdu's buffers on its list</div><div class='add'>+ * @qp: qp for which partial fpdus</div><div class='add'>+ * @ieq: ieq resource</div><div class='add'>+ */</div><div class='add'>+void irdma_ieq_process_fpdus(struct irdma_sc_qp *qp,</div><div class='add'>+			     struct irdma_puda_rsrc *ieq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pfpdu *pfpdu = &amp;qp-&gt;pfpdu;</div><div class='add'>+	struct list_head *rxlist = &amp;pfpdu-&gt;rxlist;</div><div class='add'>+	struct irdma_puda_buf *buf;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		if (list_empty(rxlist))</div><div class='add'>+			break;</div><div class='add'>+		buf = irdma_puda_get_listbuf(rxlist);</div><div class='add'>+		if (!buf) {</div><div class='add'>+			ibdev_dbg(to_ibdev(ieq-&gt;dev), "IEQ: error no buf\n");</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		if (buf-&gt;seqnum != pfpdu-&gt;rcv_nxt) {</div><div class='add'>+			/* This could be out of order or missing packet */</div><div class='add'>+			pfpdu-&gt;out_of_order++;</div><div class='add'>+			list_add(&amp;buf-&gt;list, rxlist);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		/* keep processing buffers from the head of the list */</div><div class='add'>+		status = irdma_ieq_process_buf(ieq, pfpdu, buf);</div><div class='add'>+		if (status == -EINVAL) {</div><div class='add'>+			pfpdu-&gt;mpa_crc_err = true;</div><div class='add'>+			while (!list_empty(rxlist)) {</div><div class='add'>+				buf = irdma_puda_get_listbuf(rxlist);</div><div class='add'>+				irdma_puda_ret_bufpool(ieq, buf);</div><div class='add'>+				pfpdu-&gt;crc_err++;</div><div class='add'>+				ieq-&gt;crc_err++;</div><div class='add'>+			}</div><div class='add'>+			/* create CQP for AE */</div><div class='add'>+			irdma_ieq_mpa_crc_ae(ieq-&gt;dev, qp);</div><div class='add'>+		}</div><div class='add'>+	} while (!status);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_create_ah - create an address handle for IEQ</div><div class='add'>+ * @qp: qp pointer</div><div class='add'>+ * @buf: buf received on IEQ used to create AH</div><div class='add'>+ */</div><div class='add'>+static int irdma_ieq_create_ah(struct irdma_sc_qp *qp, struct irdma_puda_buf *buf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ah_info ah_info = {};</div><div class='add'>+</div><div class='add'>+	qp-&gt;pfpdu.ah_buf = buf;</div><div class='add'>+	irdma_puda_ieq_get_ah_info(qp, &amp;ah_info);</div><div class='add'>+	return irdma_puda_create_ah(qp-&gt;vsi-&gt;dev, &amp;ah_info, false,</div><div class='add'>+				    IRDMA_PUDA_RSRC_TYPE_IEQ, qp,</div><div class='add'>+				    &amp;qp-&gt;pfpdu.ah);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_handle_exception - handle qp's exception</div><div class='add'>+ * @ieq: ieq resource</div><div class='add'>+ * @qp: qp receiving excpetion</div><div class='add'>+ * @buf: receive buffer</div><div class='add'>+ */</div><div class='add'>+static void irdma_ieq_handle_exception(struct irdma_puda_rsrc *ieq,</div><div class='add'>+				       struct irdma_sc_qp *qp,</div><div class='add'>+				       struct irdma_puda_buf *buf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pfpdu *pfpdu = &amp;qp-&gt;pfpdu;</div><div class='add'>+	u32 *hw_host_ctx = (u32 *)qp-&gt;hw_host_ctx;</div><div class='add'>+	u32 rcv_wnd = hw_host_ctx[23];</div><div class='add'>+	/* first partial seq # in q2 */</div><div class='add'>+	u32 fps = *(u32 *)(qp-&gt;q2_buf + Q2_FPSN_OFFSET);</div><div class='add'>+	struct list_head *rxlist = &amp;pfpdu-&gt;rxlist;</div><div class='add'>+	unsigned long flags = 0;</div><div class='add'>+	u8 hw_rev = qp-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev;</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("IEQ: IEQ RX BUFFER", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     buf-&gt;mem.va, buf-&gt;totallen, false);</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;pfpdu-&gt;lock, flags);</div><div class='add'>+	pfpdu-&gt;total_ieq_bufs++;</div><div class='add'>+	if (pfpdu-&gt;mpa_crc_err) {</div><div class='add'>+		pfpdu-&gt;crc_err++;</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+	if (pfpdu-&gt;mode &amp;&amp; fps != pfpdu-&gt;fps) {</div><div class='add'>+		/* clean up qp as it is new partial sequence */</div><div class='add'>+		irdma_ieq_cleanup_qp(ieq, qp);</div><div class='add'>+		ibdev_dbg(to_ibdev(ieq-&gt;dev), "IEQ: restarting new partial\n");</div><div class='add'>+		pfpdu-&gt;mode = false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!pfpdu-&gt;mode) {</div><div class='add'>+		print_hex_dump_debug("IEQ: Q2 BUFFER", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+				     8, (u64 *)qp-&gt;q2_buf, 128, false);</div><div class='add'>+		/* First_Partial_Sequence_Number check */</div><div class='add'>+		pfpdu-&gt;rcv_nxt = fps;</div><div class='add'>+		pfpdu-&gt;fps = fps;</div><div class='add'>+		pfpdu-&gt;mode = true;</div><div class='add'>+		pfpdu-&gt;max_fpdu_data = (buf-&gt;ipv4) ?</div><div class='add'>+				       (ieq-&gt;vsi-&gt;mtu - IRDMA_MTU_TO_MSS_IPV4) :</div><div class='add'>+				       (ieq-&gt;vsi-&gt;mtu - IRDMA_MTU_TO_MSS_IPV6);</div><div class='add'>+		pfpdu-&gt;pmode_count++;</div><div class='add'>+		ieq-&gt;pmode_count++;</div><div class='add'>+		INIT_LIST_HEAD(rxlist);</div><div class='add'>+		irdma_ieq_check_first_buf(buf, fps);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!(rcv_wnd &gt;= (buf-&gt;seqnum - pfpdu-&gt;rcv_nxt))) {</div><div class='add'>+		pfpdu-&gt;bad_seq_num++;</div><div class='add'>+		ieq-&gt;bad_seq_num++;</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!list_empty(rxlist)) {</div><div class='add'>+		if (buf-&gt;seqnum != pfpdu-&gt;nextseqnum) {</div><div class='add'>+			irdma_send_ieq_ack(qp);</div><div class='add'>+			/* throw away out-of-order, duplicates*/</div><div class='add'>+			goto error;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	/* Insert buf before head */</div><div class='add'>+	list_add_tail(&amp;buf-&gt;list, rxlist);</div><div class='add'>+	pfpdu-&gt;nextseqnum = buf-&gt;seqnum + buf-&gt;datalen;</div><div class='add'>+	pfpdu-&gt;lastrcv_buf = buf;</div><div class='add'>+	if (hw_rev &gt;= IRDMA_GEN_2 &amp;&amp; !pfpdu-&gt;ah) {</div><div class='add'>+		irdma_ieq_create_ah(qp, buf);</div><div class='add'>+		if (!pfpdu-&gt;ah)</div><div class='add'>+			goto error;</div><div class='add'>+		goto exit;</div><div class='add'>+	}</div><div class='add'>+	if (hw_rev == IRDMA_GEN_1)</div><div class='add'>+		irdma_ieq_process_fpdus(qp, ieq);</div><div class='add'>+	else if (pfpdu-&gt;ah &amp;&amp; pfpdu-&gt;ah-&gt;ah_info.ah_valid)</div><div class='add'>+		irdma_ieq_process_fpdus(qp, ieq);</div><div class='add'>+exit:</div><div class='add'>+	spin_unlock_irqrestore(&amp;pfpdu-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	return;</div><div class='add'>+</div><div class='add'>+error:</div><div class='add'>+	irdma_puda_ret_bufpool(ieq, buf);</div><div class='add'>+	spin_unlock_irqrestore(&amp;pfpdu-&gt;lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_receive - received exception buffer</div><div class='add'>+ * @vsi: VSI of device</div><div class='add'>+ * @buf: exception buffer received</div><div class='add'>+ */</div><div class='add'>+static void irdma_ieq_receive(struct irdma_sc_vsi *vsi,</div><div class='add'>+			      struct irdma_puda_buf *buf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_rsrc *ieq = vsi-&gt;ieq;</div><div class='add'>+	struct irdma_sc_qp *qp = NULL;</div><div class='add'>+	u32 wqe_idx = ieq-&gt;compl_rxwqe_idx;</div><div class='add'>+</div><div class='add'>+	qp = irdma_ieq_get_qp(vsi-&gt;dev, buf);</div><div class='add'>+	if (!qp) {</div><div class='add'>+		ieq-&gt;stats_bad_qp_id++;</div><div class='add'>+		irdma_puda_ret_bufpool(ieq, buf);</div><div class='add'>+	} else {</div><div class='add'>+		irdma_ieq_handle_exception(ieq, qp, buf);</div><div class='add'>+	}</div><div class='add'>+	/*</div><div class='add'>+	 * ieq-&gt;rx_wqe_idx is used by irdma_puda_replenish_rq()</div><div class='add'>+	 * on which wqe_idx to start replenish rq</div><div class='add'>+	 */</div><div class='add'>+	if (!ieq-&gt;rxq_invalid_cnt)</div><div class='add'>+		ieq-&gt;rx_wqe_idx = wqe_idx;</div><div class='add'>+	ieq-&gt;rxq_invalid_cnt++;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_tx_compl - put back after sending completed exception buffer</div><div class='add'>+ * @vsi: sc VSI struct</div><div class='add'>+ * @sqwrid: pointer to puda buffer</div><div class='add'>+ */</div><div class='add'>+static void irdma_ieq_tx_compl(struct irdma_sc_vsi *vsi, void *sqwrid)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_rsrc *ieq = vsi-&gt;ieq;</div><div class='add'>+	struct irdma_puda_buf *buf = sqwrid;</div><div class='add'>+</div><div class='add'>+	irdma_puda_ret_bufpool(ieq, buf);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_cleanup_qp - qp is being destroyed</div><div class='add'>+ * @ieq: ieq resource</div><div class='add'>+ * @qp: all pending fpdu buffers</div><div class='add'>+ */</div><div class='add'>+void irdma_ieq_cleanup_qp(struct irdma_puda_rsrc *ieq, struct irdma_sc_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_buf *buf;</div><div class='add'>+	struct irdma_pfpdu *pfpdu = &amp;qp-&gt;pfpdu;</div><div class='add'>+	struct list_head *rxlist = &amp;pfpdu-&gt;rxlist;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;pfpdu.ah) {</div><div class='add'>+		irdma_puda_free_ah(ieq-&gt;dev, qp-&gt;pfpdu.ah);</div><div class='add'>+		qp-&gt;pfpdu.ah = NULL;</div><div class='add'>+		qp-&gt;pfpdu.ah_buf = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!pfpdu-&gt;mode)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	while (!list_empty(rxlist)) {</div><div class='add'>+		buf = irdma_puda_get_listbuf(rxlist);</div><div class='add'>+		irdma_puda_ret_bufpool(ieq, buf);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/puda.h b/drivers/infiniband/hw/irdma/puda.h<br/>new file mode 100644<br/>index 000000000000..d65041bee667<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/puda.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/puda.h</a></div><div class='hunk'>@@ -0,0 +1,182 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2015 - 2020 Intel Corporation */</div><div class='add'>+#ifndef IRDMA_PUDA_H</div><div class='add'>+#define IRDMA_PUDA_H</div><div class='add'>+</div><div class='add'>+#define IRDMA_IEQ_MPA_FRAMING	6</div><div class='add'>+#define IRDMA_TCP_OFFSET	40</div><div class='add'>+#define IRDMA_IPV4_PAD		20</div><div class='add'>+#define IRDMA_MRK_BLK_SZ	512</div><div class='add'>+</div><div class='add'>+enum puda_rsrc_type {</div><div class='add'>+	IRDMA_PUDA_RSRC_TYPE_ILQ = 1,</div><div class='add'>+	IRDMA_PUDA_RSRC_TYPE_IEQ,</div><div class='add'>+	IRDMA_PUDA_RSRC_TYPE_MAX, /* Must be last entry */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum puda_rsrc_complete {</div><div class='add'>+	PUDA_CQ_CREATED = 1,</div><div class='add'>+	PUDA_QP_CREATED,</div><div class='add'>+	PUDA_TX_COMPLETE,</div><div class='add'>+	PUDA_RX_COMPLETE,</div><div class='add'>+	PUDA_HASH_CRC_COMPLETE,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_sc_dev;</div><div class='add'>+struct irdma_sc_qp;</div><div class='add'>+struct irdma_sc_cq;</div><div class='add'>+</div><div class='add'>+struct irdma_puda_cmpl_info {</div><div class='add'>+	struct irdma_qp_uk *qp;</div><div class='add'>+	u8 q_type;</div><div class='add'>+	u8 l3proto;</div><div class='add'>+	u8 l4proto;</div><div class='add'>+	u16 vlan;</div><div class='add'>+	u32 payload_len;</div><div class='add'>+	u32 compl_error; /* No_err=0, else major and minor err code */</div><div class='add'>+	u32 qp_id;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+	bool ipv4:1;</div><div class='add'>+	bool smac_valid:1;</div><div class='add'>+	bool vlan_valid:1;</div><div class='add'>+	u8 smac[ETH_ALEN];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_puda_send_info {</div><div class='add'>+	u64 paddr; /* Physical address */</div><div class='add'>+	u32 len;</div><div class='add'>+	u32 ah_id;</div><div class='add'>+	u8 tcplen;</div><div class='add'>+	u8 maclen;</div><div class='add'>+	bool ipv4:1;</div><div class='add'>+	bool do_lpb:1;</div><div class='add'>+	void *scratch;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_puda_buf {</div><div class='add'>+	struct list_head list; /* MUST be first entry */</div><div class='add'>+	struct irdma_dma_mem mem; /* DMA memory for the buffer */</div><div class='add'>+	struct irdma_puda_buf *next; /* for alloclist in rsrc struct */</div><div class='add'>+	struct irdma_virt_mem buf_mem; /* Buffer memory for this buffer */</div><div class='add'>+	void *scratch;</div><div class='add'>+	u8 *iph;</div><div class='add'>+	u8 *tcph;</div><div class='add'>+	u8 *data;</div><div class='add'>+	u16 datalen;</div><div class='add'>+	u16 vlan_id;</div><div class='add'>+	u8 tcphlen; /* tcp length in bytes */</div><div class='add'>+	u8 maclen; /* mac length in bytes */</div><div class='add'>+	u32 totallen; /* machlen+iphlen+tcphlen+datalen */</div><div class='add'>+	refcount_t refcount;</div><div class='add'>+	u8 hdrlen;</div><div class='add'>+	bool ipv4:1;</div><div class='add'>+	bool vlan_valid:1;</div><div class='add'>+	bool do_lpb:1; /* Loopback buffer */</div><div class='add'>+	bool smac_valid:1;</div><div class='add'>+	u32 seqnum;</div><div class='add'>+	u32 ah_id;</div><div class='add'>+	u8 smac[ETH_ALEN];</div><div class='add'>+	struct irdma_sc_vsi *vsi;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_puda_rsrc_info {</div><div class='add'>+	void (*receive)(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *buf);</div><div class='add'>+	void (*xmit_complete)(struct irdma_sc_vsi *vsi, void *sqwrid);</div><div class='add'>+	enum puda_rsrc_type type; /* ILQ or IEQ */</div><div class='add'>+	u32 count;</div><div class='add'>+	u32 pd_id;</div><div class='add'>+	u32 cq_id;</div><div class='add'>+	u32 qp_id;</div><div class='add'>+	u32 sq_size;</div><div class='add'>+	u32 rq_size;</div><div class='add'>+	u32 tx_buf_cnt; /* total bufs allocated will be rq_size + tx_buf_cnt */</div><div class='add'>+	u16 buf_size;</div><div class='add'>+	u16 stats_idx;</div><div class='add'>+	bool stats_idx_valid:1;</div><div class='add'>+	int abi_ver;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_puda_rsrc {</div><div class='add'>+	struct irdma_sc_cq cq;</div><div class='add'>+	struct irdma_sc_qp qp;</div><div class='add'>+	struct irdma_sc_pd sc_pd;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	struct irdma_sc_vsi *vsi;</div><div class='add'>+	struct irdma_dma_mem cqmem;</div><div class='add'>+	struct irdma_dma_mem qpmem;</div><div class='add'>+	struct irdma_virt_mem ilq_mem;</div><div class='add'>+	enum puda_rsrc_complete cmpl;</div><div class='add'>+	enum puda_rsrc_type type;</div><div class='add'>+	u16 buf_size; /*buf must be max datalen + tcpip hdr + mac */</div><div class='add'>+	u32 cq_id;</div><div class='add'>+	u32 qp_id;</div><div class='add'>+	u32 sq_size;</div><div class='add'>+	u32 rq_size;</div><div class='add'>+	u32 cq_size;</div><div class='add'>+	struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;</div><div class='add'>+	u64 *rq_wrid_array;</div><div class='add'>+	u32 compl_rxwqe_idx;</div><div class='add'>+	u32 rx_wqe_idx;</div><div class='add'>+	u32 rxq_invalid_cnt;</div><div class='add'>+	u32 tx_wqe_avail_cnt;</div><div class='add'>+	struct list_head txpend;</div><div class='add'>+	struct list_head bufpool; /* free buffers pool list for recv and xmit */</div><div class='add'>+	u32 alloc_buf_count;</div><div class='add'>+	u32 avail_buf_count; /* snapshot of currently available buffers */</div><div class='add'>+	spinlock_t bufpool_lock;</div><div class='add'>+	struct irdma_puda_buf *alloclist;</div><div class='add'>+	void (*receive)(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *buf);</div><div class='add'>+	void (*xmit_complete)(struct irdma_sc_vsi *vsi, void *sqwrid);</div><div class='add'>+	/* puda stats */</div><div class='add'>+	u64 stats_buf_alloc_fail;</div><div class='add'>+	u64 stats_pkt_rcvd;</div><div class='add'>+	u64 stats_pkt_sent;</div><div class='add'>+	u64 stats_rcvd_pkt_err;</div><div class='add'>+	u64 stats_sent_pkt_q;</div><div class='add'>+	u64 stats_bad_qp_id;</div><div class='add'>+	/* IEQ stats */</div><div class='add'>+	u64 fpdu_processed;</div><div class='add'>+	u64 bad_seq_num;</div><div class='add'>+	u64 crc_err;</div><div class='add'>+	u64 pmode_count;</div><div class='add'>+	u64 partials_handled;</div><div class='add'>+	u16 stats_idx;</div><div class='add'>+	bool check_crc:1;</div><div class='add'>+	bool stats_idx_valid:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_puda_buf *irdma_puda_get_bufpool(struct irdma_puda_rsrc *rsrc);</div><div class='add'>+void irdma_puda_ret_bufpool(struct irdma_puda_rsrc *rsrc,</div><div class='add'>+			    struct irdma_puda_buf *buf);</div><div class='add'>+void irdma_puda_send_buf(struct irdma_puda_rsrc *rsrc,</div><div class='add'>+			 struct irdma_puda_buf *buf);</div><div class='add'>+int irdma_puda_send(struct irdma_sc_qp *qp, struct irdma_puda_send_info *info);</div><div class='add'>+int irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi,</div><div class='add'>+			   struct irdma_puda_rsrc_info *info);</div><div class='add'>+void irdma_puda_dele_rsrc(struct irdma_sc_vsi *vsi, enum puda_rsrc_type type,</div><div class='add'>+			  bool reset);</div><div class='add'>+int irdma_puda_poll_cmpl(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq,</div><div class='add'>+			 u32 *compl_err);</div><div class='add'>+</div><div class='add'>+struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,</div><div class='add'>+				     struct irdma_puda_buf *buf);</div><div class='add'>+int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,</div><div class='add'>+			      struct irdma_puda_buf *buf);</div><div class='add'>+int irdma_ieq_check_mpacrc(const void *addr, u32 len, u32 val);</div><div class='add'>+void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp);</div><div class='add'>+void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len, u32 seqnum);</div><div class='add'>+int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp);</div><div class='add'>+int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq);</div><div class='add'>+int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp);</div><div class='add'>+void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq);</div><div class='add'>+void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp,</div><div class='add'>+				struct irdma_ah_info *ah_info);</div><div class='add'>+int irdma_puda_create_ah(struct irdma_sc_dev *dev,</div><div class='add'>+			 struct irdma_ah_info *ah_info, bool wait,</div><div class='add'>+			 enum puda_rsrc_type type, void *cb_param,</div><div class='add'>+			 struct irdma_sc_ah **ah);</div><div class='add'>+void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah);</div><div class='add'>+void irdma_ieq_process_fpdus(struct irdma_sc_qp *qp,</div><div class='add'>+			     struct irdma_puda_rsrc *ieq);</div><div class='add'>+void irdma_ieq_cleanup_qp(struct irdma_puda_rsrc *ieq, struct irdma_sc_qp *qp);</div><div class='add'>+#endif /*IRDMA_PROTOS_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/trace.c b/drivers/infiniband/hw/irdma/trace.c<br/>new file mode 100644<br/>index 000000000000..fc2f56697741<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/trace.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/trace.c</a></div><div class='hunk'>@@ -0,0 +1,112 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2019 Intel Corporation */</div><div class='add'>+#define CREATE_TRACE_POINTS</div><div class='add'>+#include "trace.h"</div><div class='add'>+</div><div class='add'>+const char *print_ip_addr(struct trace_seq *p, u32 *addr, u16 port, bool ipv4)</div><div class='add'>+{</div><div class='add'>+	const char *ret = trace_seq_buffer_ptr(p);</div><div class='add'>+</div><div class='add'>+	if (ipv4) {</div><div class='add'>+		__be32 myaddr = htonl(*addr);</div><div class='add'>+</div><div class='add'>+		trace_seq_printf(p, "%pI4:%d", &amp;myaddr, htons(port));</div><div class='add'>+	} else {</div><div class='add'>+		trace_seq_printf(p, "%pI6:%d", addr, htons(port));</div><div class='add'>+	}</div><div class='add'>+	trace_seq_putc(p, 0);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+const char *parse_iw_event_type(enum iw_cm_event_type iw_type)</div><div class='add'>+{</div><div class='add'>+	switch (iw_type) {</div><div class='add'>+	case IW_CM_EVENT_CONNECT_REQUEST:</div><div class='add'>+		return "IwRequest";</div><div class='add'>+	case IW_CM_EVENT_CONNECT_REPLY:</div><div class='add'>+		return "IwReply";</div><div class='add'>+	case IW_CM_EVENT_ESTABLISHED:</div><div class='add'>+		return "IwEstablished";</div><div class='add'>+	case IW_CM_EVENT_DISCONNECT:</div><div class='add'>+		return "IwDisconnect";</div><div class='add'>+	case IW_CM_EVENT_CLOSE:</div><div class='add'>+		return "IwClose";</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return "Unknown";</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+const char *parse_cm_event_type(enum irdma_cm_event_type cm_type)</div><div class='add'>+{</div><div class='add'>+	switch (cm_type) {</div><div class='add'>+	case IRDMA_CM_EVENT_ESTABLISHED:</div><div class='add'>+		return "CmEstablished";</div><div class='add'>+	case IRDMA_CM_EVENT_MPA_REQ:</div><div class='add'>+		return "CmMPA_REQ";</div><div class='add'>+	case IRDMA_CM_EVENT_MPA_CONNECT:</div><div class='add'>+		return "CmMPA_CONNECT";</div><div class='add'>+	case IRDMA_CM_EVENT_MPA_ACCEPT:</div><div class='add'>+		return "CmMPA_ACCEPT";</div><div class='add'>+	case IRDMA_CM_EVENT_MPA_REJECT:</div><div class='add'>+		return "CmMPA_REJECT";</div><div class='add'>+	case IRDMA_CM_EVENT_MPA_ESTABLISHED:</div><div class='add'>+		return "CmMPA_ESTABLISHED";</div><div class='add'>+	case IRDMA_CM_EVENT_CONNECTED:</div><div class='add'>+		return "CmConnected";</div><div class='add'>+	case IRDMA_CM_EVENT_RESET:</div><div class='add'>+		return "CmReset";</div><div class='add'>+	case IRDMA_CM_EVENT_ABORTED:</div><div class='add'>+		return "CmAborted";</div><div class='add'>+	case IRDMA_CM_EVENT_UNKNOWN:</div><div class='add'>+		return "none";</div><div class='add'>+	}</div><div class='add'>+	return "Unknown";</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+const char *parse_cm_state(enum irdma_cm_node_state state)</div><div class='add'>+{</div><div class='add'>+	switch (state) {</div><div class='add'>+	case IRDMA_CM_STATE_UNKNOWN:</div><div class='add'>+		return "UNKNOWN";</div><div class='add'>+	case IRDMA_CM_STATE_INITED:</div><div class='add'>+		return "INITED";</div><div class='add'>+	case IRDMA_CM_STATE_LISTENING:</div><div class='add'>+		return "LISTENING";</div><div class='add'>+	case IRDMA_CM_STATE_SYN_RCVD:</div><div class='add'>+		return "SYN_RCVD";</div><div class='add'>+	case IRDMA_CM_STATE_SYN_SENT:</div><div class='add'>+		return "SYN_SENT";</div><div class='add'>+	case IRDMA_CM_STATE_ONE_SIDE_ESTABLISHED:</div><div class='add'>+		return "ONE_SIDE_ESTABLISHED";</div><div class='add'>+	case IRDMA_CM_STATE_ESTABLISHED:</div><div class='add'>+		return "ESTABLISHED";</div><div class='add'>+	case IRDMA_CM_STATE_ACCEPTING:</div><div class='add'>+		return "ACCEPTING";</div><div class='add'>+	case IRDMA_CM_STATE_MPAREQ_SENT:</div><div class='add'>+		return "MPAREQ_SENT";</div><div class='add'>+	case IRDMA_CM_STATE_MPAREQ_RCVD:</div><div class='add'>+		return "MPAREQ_RCVD";</div><div class='add'>+	case IRDMA_CM_STATE_MPAREJ_RCVD:</div><div class='add'>+		return "MPAREJ_RECVD";</div><div class='add'>+	case IRDMA_CM_STATE_OFFLOADED:</div><div class='add'>+		return "OFFLOADED";</div><div class='add'>+	case IRDMA_CM_STATE_FIN_WAIT1:</div><div class='add'>+		return "FIN_WAIT1";</div><div class='add'>+	case IRDMA_CM_STATE_FIN_WAIT2:</div><div class='add'>+		return "FIN_WAIT2";</div><div class='add'>+	case IRDMA_CM_STATE_CLOSE_WAIT:</div><div class='add'>+		return "CLOSE_WAIT";</div><div class='add'>+	case IRDMA_CM_STATE_TIME_WAIT:</div><div class='add'>+		return "TIME_WAIT";</div><div class='add'>+	case IRDMA_CM_STATE_LAST_ACK:</div><div class='add'>+		return "LAST_ACK";</div><div class='add'>+	case IRDMA_CM_STATE_CLOSING:</div><div class='add'>+		return "CLOSING";</div><div class='add'>+	case IRDMA_CM_STATE_LISTENER_DESTROYED:</div><div class='add'>+		return "LISTENER_DESTROYED";</div><div class='add'>+	case IRDMA_CM_STATE_CLOSED:</div><div class='add'>+		return "CLOSED";</div><div class='add'>+	}</div><div class='add'>+	return ("Bad state");</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/trace.h b/drivers/infiniband/hw/irdma/trace.h<br/>new file mode 100644<br/>index 000000000000..b8085a66b9f8<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/trace.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/trace.h</a></div><div class='hunk'>@@ -0,0 +1,3 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2019 Intel Corporation */</div><div class='add'>+#include "trace_cm.h"</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/trace_cm.h b/drivers/infiniband/hw/irdma/trace_cm.h<br/>new file mode 100644<br/>index 000000000000..0d1699b55241<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/trace_cm.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/trace_cm.h</a></div><div class='hunk'>@@ -0,0 +1,460 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2019 - 2021 Intel Corporation */</div><div class='add'>+#if !defined(__TRACE_CM_H) || defined(TRACE_HEADER_MULTI_READ)</div><div class='add'>+#define __TRACE_CM_H</div><div class='add'>+</div><div class='add'>+#include &lt;linux/tracepoint.h&gt;</div><div class='add'>+#include &lt;linux/trace_seq.h&gt;</div><div class='add'>+</div><div class='add'>+#include "main.h"</div><div class='add'>+</div><div class='add'>+const char *print_ip_addr(struct trace_seq *p, u32 *addr, u16 port, bool ivp4);</div><div class='add'>+const char *parse_iw_event_type(enum iw_cm_event_type iw_type);</div><div class='add'>+const char *parse_cm_event_type(enum irdma_cm_event_type cm_type);</div><div class='add'>+const char *parse_cm_state(enum irdma_cm_node_state);</div><div class='add'>+#define __print_ip_addr(addr, port, ipv4) print_ip_addr(p, addr, port, ipv4)</div><div class='add'>+</div><div class='add'>+#undef TRACE_SYSTEM</div><div class='add'>+#define TRACE_SYSTEM irdma_cm</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(irdma_create_listen,</div><div class='add'>+	    TP_PROTO(struct irdma_device *iwdev, struct irdma_cm_info *cm_info),</div><div class='add'>+	    TP_ARGS(iwdev, cm_info),</div><div class='add'>+	    TP_STRUCT__entry(__field(struct irdma_device *, iwdev)</div><div class='add'>+			     __dynamic_array(u32, laddr, 4)</div><div class='add'>+			     __field(u16, lport)</div><div class='add'>+			     __field(bool, ipv4)</div><div class='add'>+		    ),</div><div class='add'>+	    TP_fast_assign(__entry-&gt;iwdev = iwdev;</div><div class='add'>+			   __entry-&gt;lport = cm_info-&gt;loc_port;</div><div class='add'>+			   __entry-&gt;ipv4 = cm_info-&gt;ipv4;</div><div class='add'>+			   memcpy(__get_dynamic_array(laddr),</div><div class='add'>+				  cm_info-&gt;loc_addr, 4);</div><div class='add'>+		    ),</div><div class='add'>+	    TP_printk("iwdev=%p  loc: %s",</div><div class='add'>+		      __entry-&gt;iwdev,</div><div class='add'>+		      __print_ip_addr(__get_dynamic_array(laddr),</div><div class='add'>+				      __entry-&gt;lport, __entry-&gt;ipv4)</div><div class='add'>+		    )</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(irdma_dec_refcnt_listen,</div><div class='add'>+	    TP_PROTO(struct irdma_cm_listener *listener, void *caller),</div><div class='add'>+	    TP_ARGS(listener, caller),</div><div class='add'>+	    TP_STRUCT__entry(__field(struct irdma_device *, iwdev)</div><div class='add'>+		    __field(u32, refcnt)</div><div class='add'>+		    __dynamic_array(u32, laddr, 4)</div><div class='add'>+		    __field(u16, lport)</div><div class='add'>+		    __field(bool, ipv4)</div><div class='add'>+		    __field(void *, caller)</div><div class='add'>+		    ),</div><div class='add'>+	    TP_fast_assign(__entry-&gt;iwdev = listener-&gt;iwdev;</div><div class='add'>+			   __entry-&gt;lport = listener-&gt;loc_port;</div><div class='add'>+			   __entry-&gt;ipv4 = listener-&gt;ipv4;</div><div class='add'>+			   memcpy(__get_dynamic_array(laddr),</div><div class='add'>+				  listener-&gt;loc_addr, 4);</div><div class='add'>+		    ),</div><div class='add'>+	    TP_printk("iwdev=%p  caller=%pS  loc: %s",</div><div class='add'>+		      __entry-&gt;iwdev,</div><div class='add'>+		      __entry-&gt;caller,</div><div class='add'>+		      __print_ip_addr(__get_dynamic_array(laddr),</div><div class='add'>+				      __entry-&gt;lport, __entry-&gt;ipv4)</div><div class='add'>+		    )</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DECLARE_EVENT_CLASS(listener_template,</div><div class='add'>+		    TP_PROTO(struct irdma_cm_listener *listener),</div><div class='add'>+		    TP_ARGS(listener),</div><div class='add'>+		    TP_STRUCT__entry(__field(struct irdma_device *, iwdev)</div><div class='add'>+				     __field(u16, lport)</div><div class='add'>+				     __field(u16, vlan_id)</div><div class='add'>+				     __field(bool, ipv4)</div><div class='add'>+				     __field(enum irdma_cm_listener_state,</div><div class='add'>+					     state)</div><div class='add'>+				     __dynamic_array(u32, laddr, 4)</div><div class='add'>+			    ),</div><div class='add'>+		    TP_fast_assign(__entry-&gt;iwdev = listener-&gt;iwdev;</div><div class='add'>+				   __entry-&gt;lport = listener-&gt;loc_port;</div><div class='add'>+				   __entry-&gt;vlan_id = listener-&gt;vlan_id;</div><div class='add'>+				   __entry-&gt;ipv4 = listener-&gt;ipv4;</div><div class='add'>+				   __entry-&gt;state = listener-&gt;listener_state;</div><div class='add'>+				   memcpy(__get_dynamic_array(laddr),</div><div class='add'>+					  listener-&gt;loc_addr, 4);</div><div class='add'>+			    ),</div><div class='add'>+		    TP_printk("iwdev=%p  vlan=%d  loc: %s",</div><div class='add'>+			      __entry-&gt;iwdev,</div><div class='add'>+			      __entry-&gt;vlan_id,</div><div class='add'>+			      __print_ip_addr(__get_dynamic_array(laddr),</div><div class='add'>+					      __entry-&gt;lport, __entry-&gt;ipv4)</div><div class='add'>+			    )</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(listener_template, irdma_find_listener,</div><div class='add'>+	     TP_PROTO(struct irdma_cm_listener *listener),</div><div class='add'>+	     TP_ARGS(listener));</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(listener_template, irdma_del_multiple_qhash,</div><div class='add'>+	     TP_PROTO(struct irdma_cm_listener *listener),</div><div class='add'>+	     TP_ARGS(listener));</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(irdma_negotiate_mpa_v2,</div><div class='add'>+	    TP_PROTO(struct irdma_cm_node *cm_node),</div><div class='add'>+	    TP_ARGS(cm_node),</div><div class='add'>+	    TP_STRUCT__entry(__field(struct irdma_cm_node *, cm_node)</div><div class='add'>+			     __field(u16, ord_size)</div><div class='add'>+			     __field(u16, ird_size)</div><div class='add'>+		    ),</div><div class='add'>+	    TP_fast_assign(__entry-&gt;cm_node = cm_node;</div><div class='add'>+			   __entry-&gt;ord_size = cm_node-&gt;ord_size;</div><div class='add'>+			   __entry-&gt;ird_size = cm_node-&gt;ird_size;</div><div class='add'>+		    ),</div><div class='add'>+	    TP_printk("MPVA2 Negotiated cm_node=%p ORD:[%d], IRD:[%d]",</div><div class='add'>+		      __entry-&gt;cm_node,</div><div class='add'>+		      __entry-&gt;ord_size,</div><div class='add'>+		      __entry-&gt;ird_size</div><div class='add'>+		    )</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DECLARE_EVENT_CLASS(tos_template,</div><div class='add'>+		    TP_PROTO(struct irdma_device *iwdev, u8 tos, u8 user_pri),</div><div class='add'>+		    TP_ARGS(iwdev, tos, user_pri),</div><div class='add'>+		    TP_STRUCT__entry(__field(struct irdma_device *, iwdev)</div><div class='add'>+				     __field(u8, tos)</div><div class='add'>+				     __field(u8, user_pri)</div><div class='add'>+			    ),</div><div class='add'>+		    TP_fast_assign(__entry-&gt;iwdev = iwdev;</div><div class='add'>+				   __entry-&gt;tos = tos;</div><div class='add'>+				   __entry-&gt;user_pri = user_pri;</div><div class='add'>+			    ),</div><div class='add'>+		    TP_printk("iwdev=%p  TOS:[%d]  UP:[%d]",</div><div class='add'>+			      __entry-&gt;iwdev,</div><div class='add'>+			      __entry-&gt;tos,</div><div class='add'>+			      __entry-&gt;user_pri</div><div class='add'>+			    )</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(tos_template, irdma_listener_tos,</div><div class='add'>+	     TP_PROTO(struct irdma_device *iwdev, u8 tos, u8 user_pri),</div><div class='add'>+	     TP_ARGS(iwdev, tos, user_pri));</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(tos_template, irdma_dcb_tos,</div><div class='add'>+	     TP_PROTO(struct irdma_device *iwdev, u8 tos, u8 user_pri),</div><div class='add'>+	     TP_ARGS(iwdev, tos, user_pri));</div><div class='add'>+</div><div class='add'>+DECLARE_EVENT_CLASS(qhash_template,</div><div class='add'>+		    TP_PROTO(struct irdma_device *iwdev,</div><div class='add'>+			     struct irdma_cm_listener *listener,</div><div class='add'>+			     const char *dev_addr),</div><div class='add'>+		    TP_ARGS(iwdev, listener, dev_addr),</div><div class='add'>+		    TP_STRUCT__entry(__field(struct irdma_device *, iwdev)</div><div class='add'>+				     __field(u16, lport)</div><div class='add'>+				     __field(u16, vlan_id)</div><div class='add'>+				     __field(bool, ipv4)</div><div class='add'>+				     __dynamic_array(u32, laddr, 4)</div><div class='add'>+				     __dynamic_array(u32, mac, ETH_ALEN)</div><div class='add'>+			    ),</div><div class='add'>+		    TP_fast_assign(__entry-&gt;iwdev = iwdev;</div><div class='add'>+				   __entry-&gt;lport = listener-&gt;loc_port;</div><div class='add'>+				   __entry-&gt;vlan_id = listener-&gt;vlan_id;</div><div class='add'>+				   __entry-&gt;ipv4 = listener-&gt;ipv4;</div><div class='add'>+				   memcpy(__get_dynamic_array(laddr),</div><div class='add'>+					  listener-&gt;loc_addr, 4);</div><div class='add'>+				   ether_addr_copy(__get_dynamic_array(mac),</div><div class='add'>+						   dev_addr);</div><div class='add'>+			    ),</div><div class='add'>+		    TP_printk("iwdev=%p  vlan=%d  MAC=%6phC  loc: %s",</div><div class='add'>+			      __entry-&gt;iwdev,</div><div class='add'>+			      __entry-&gt;vlan_id,</div><div class='add'>+			      __get_dynamic_array(mac),</div><div class='add'>+			      __print_ip_addr(__get_dynamic_array(laddr),</div><div class='add'>+					      __entry-&gt;lport, __entry-&gt;ipv4)</div><div class='add'>+		    )</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(qhash_template, irdma_add_mqh_6,</div><div class='add'>+	     TP_PROTO(struct irdma_device *iwdev,</div><div class='add'>+		      struct irdma_cm_listener *listener,</div><div class='add'>+		      const char *dev_addr),</div><div class='add'>+	     TP_ARGS(iwdev, listener, dev_addr));</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(qhash_template, irdma_add_mqh_4,</div><div class='add'>+	     TP_PROTO(struct irdma_device *iwdev,</div><div class='add'>+		      struct irdma_cm_listener *listener,</div><div class='add'>+		      const char *dev_addr),</div><div class='add'>+	     TP_ARGS(iwdev, listener, dev_addr));</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(irdma_addr_resolve,</div><div class='add'>+	    TP_PROTO(struct irdma_device *iwdev, char *dev_addr),</div><div class='add'>+	    TP_ARGS(iwdev, dev_addr),</div><div class='add'>+	    TP_STRUCT__entry(__field(struct irdma_device *, iwdev)</div><div class='add'>+		    __dynamic_array(u8, mac, ETH_ALEN)</div><div class='add'>+		    ),</div><div class='add'>+	    TP_fast_assign(__entry-&gt;iwdev = iwdev;</div><div class='add'>+		    ether_addr_copy(__get_dynamic_array(mac), dev_addr);</div><div class='add'>+		    ),</div><div class='add'>+	    TP_printk("iwdev=%p   MAC=%6phC", __entry-&gt;iwdev,</div><div class='add'>+		      __get_dynamic_array(mac)</div><div class='add'>+		    )</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(irdma_send_cm_event,</div><div class='add'>+	    TP_PROTO(struct irdma_cm_node *cm_node, struct iw_cm_id *cm_id,</div><div class='add'>+		     enum iw_cm_event_type type, int status, void *caller),</div><div class='add'>+	    TP_ARGS(cm_node, cm_id, type, status, caller),</div><div class='add'>+	    TP_STRUCT__entry(__field(struct irdma_device *, iwdev)</div><div class='add'>+			     __field(struct irdma_cm_node *, cm_node)</div><div class='add'>+			     __field(struct iw_cm_id *, cm_id)</div><div class='add'>+			     __field(u32, refcount)</div><div class='add'>+			     __field(u16, lport)</div><div class='add'>+			     __field(u16, rport)</div><div class='add'>+			     __field(enum irdma_cm_node_state, state)</div><div class='add'>+			     __field(bool, ipv4)</div><div class='add'>+			     __field(u16, vlan_id)</div><div class='add'>+			     __field(int, accel)</div><div class='add'>+			     __field(enum iw_cm_event_type, type)</div><div class='add'>+			     __field(int, status)</div><div class='add'>+			     __field(void *, caller)</div><div class='add'>+			     __dynamic_array(u32, laddr, 4)</div><div class='add'>+			     __dynamic_array(u32, raddr, 4)</div><div class='add'>+		    ),</div><div class='add'>+	    TP_fast_assign(__entry-&gt;iwdev = cm_node-&gt;iwdev;</div><div class='add'>+			   __entry-&gt;cm_node = cm_node;</div><div class='add'>+			   __entry-&gt;cm_id = cm_id;</div><div class='add'>+			   __entry-&gt;refcount = refcount_read(&amp;cm_node-&gt;refcnt);</div><div class='add'>+			   __entry-&gt;state = cm_node-&gt;state;</div><div class='add'>+			   __entry-&gt;lport = cm_node-&gt;loc_port;</div><div class='add'>+			   __entry-&gt;rport = cm_node-&gt;rem_port;</div><div class='add'>+			   __entry-&gt;ipv4 = cm_node-&gt;ipv4;</div><div class='add'>+			   __entry-&gt;vlan_id = cm_node-&gt;vlan_id;</div><div class='add'>+			   __entry-&gt;accel = cm_node-&gt;accelerated;</div><div class='add'>+			   __entry-&gt;type = type;</div><div class='add'>+			   __entry-&gt;status = status;</div><div class='add'>+			   __entry-&gt;caller = caller;</div><div class='add'>+			   memcpy(__get_dynamic_array(laddr),</div><div class='add'>+				  cm_node-&gt;loc_addr, 4);</div><div class='add'>+			   memcpy(__get_dynamic_array(raddr),</div><div class='add'>+				  cm_node-&gt;rem_addr, 4);</div><div class='add'>+		    ),</div><div class='add'>+	    TP_printk("iwdev=%p  caller=%pS  cm_id=%p  node=%p  refcnt=%d  vlan_id=%d  accel=%d  state=%s  event_type=%s  status=%d  loc: %s  rem: %s",</div><div class='add'>+		      __entry-&gt;iwdev,</div><div class='add'>+		      __entry-&gt;caller,</div><div class='add'>+		      __entry-&gt;cm_id,</div><div class='add'>+		      __entry-&gt;cm_node,</div><div class='add'>+		      __entry-&gt;refcount,</div><div class='add'>+		      __entry-&gt;vlan_id,</div><div class='add'>+		      __entry-&gt;accel,</div><div class='add'>+		      parse_cm_state(__entry-&gt;state),</div><div class='add'>+		      parse_iw_event_type(__entry-&gt;type),</div><div class='add'>+		      __entry-&gt;status,</div><div class='add'>+		      __print_ip_addr(__get_dynamic_array(laddr),</div><div class='add'>+				      __entry-&gt;lport, __entry-&gt;ipv4),</div><div class='add'>+		      __print_ip_addr(__get_dynamic_array(raddr),</div><div class='add'>+				      __entry-&gt;rport, __entry-&gt;ipv4)</div><div class='add'>+		    )</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(irdma_send_cm_event_no_node,</div><div class='add'>+	    TP_PROTO(struct iw_cm_id *cm_id, enum iw_cm_event_type type,</div><div class='add'>+		     int status, void *caller),</div><div class='add'>+	    TP_ARGS(cm_id, type, status, caller),</div><div class='add'>+	    TP_STRUCT__entry(__field(struct iw_cm_id *, cm_id)</div><div class='add'>+			     __field(enum iw_cm_event_type, type)</div><div class='add'>+			     __field(int, status)</div><div class='add'>+			     __field(void *, caller)</div><div class='add'>+		    ),</div><div class='add'>+	    TP_fast_assign(__entry-&gt;cm_id = cm_id;</div><div class='add'>+			   __entry-&gt;type = type;</div><div class='add'>+			   __entry-&gt;status = status;</div><div class='add'>+			   __entry-&gt;caller = caller;</div><div class='add'>+		    ),</div><div class='add'>+	    TP_printk("cm_id=%p  caller=%pS  event_type=%s  status=%d",</div><div class='add'>+		      __entry-&gt;cm_id,</div><div class='add'>+		      __entry-&gt;caller,</div><div class='add'>+		      parse_iw_event_type(__entry-&gt;type),</div><div class='add'>+		      __entry-&gt;status</div><div class='add'>+		    )</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DECLARE_EVENT_CLASS(cm_node_template,</div><div class='add'>+		    TP_PROTO(struct irdma_cm_node *cm_node,</div><div class='add'>+			     enum irdma_cm_event_type type, void *caller),</div><div class='add'>+		    TP_ARGS(cm_node, type, caller),</div><div class='add'>+		    TP_STRUCT__entry(__field(struct irdma_device *, iwdev)</div><div class='add'>+				     __field(struct irdma_cm_node *, cm_node)</div><div class='add'>+				     __field(u32, refcount)</div><div class='add'>+				     __field(u16, lport)</div><div class='add'>+				     __field(u16, rport)</div><div class='add'>+				     __field(enum irdma_cm_node_state, state)</div><div class='add'>+				     __field(bool, ipv4)</div><div class='add'>+				     __field(u16, vlan_id)</div><div class='add'>+				     __field(int, accel)</div><div class='add'>+				     __field(enum irdma_cm_event_type, type)</div><div class='add'>+				     __field(void *, caller)</div><div class='add'>+				     __dynamic_array(u32, laddr, 4)</div><div class='add'>+				     __dynamic_array(u32, raddr, 4)</div><div class='add'>+			    ),</div><div class='add'>+		    TP_fast_assign(__entry-&gt;iwdev = cm_node-&gt;iwdev;</div><div class='add'>+				   __entry-&gt;cm_node = cm_node;</div><div class='add'>+				   __entry-&gt;refcount = refcount_read(&amp;cm_node-&gt;refcnt);</div><div class='add'>+				   __entry-&gt;state = cm_node-&gt;state;</div><div class='add'>+				   __entry-&gt;lport = cm_node-&gt;loc_port;</div><div class='add'>+				   __entry-&gt;rport = cm_node-&gt;rem_port;</div><div class='add'>+				   __entry-&gt;ipv4 = cm_node-&gt;ipv4;</div><div class='add'>+				   __entry-&gt;vlan_id = cm_node-&gt;vlan_id;</div><div class='add'>+				   __entry-&gt;accel = cm_node-&gt;accelerated;</div><div class='add'>+				   __entry-&gt;type = type;</div><div class='add'>+				   __entry-&gt;caller = caller;</div><div class='add'>+				   memcpy(__get_dynamic_array(laddr),</div><div class='add'>+					  cm_node-&gt;loc_addr, 4);</div><div class='add'>+				   memcpy(__get_dynamic_array(raddr),</div><div class='add'>+					  cm_node-&gt;rem_addr, 4);</div><div class='add'>+			    ),</div><div class='add'>+		    TP_printk("iwdev=%p  caller=%pS  node=%p  refcnt=%d  vlan_id=%d  accel=%d  state=%s  event_type=%s  loc: %s  rem: %s",</div><div class='add'>+			      __entry-&gt;iwdev,</div><div class='add'>+			      __entry-&gt;caller,</div><div class='add'>+			      __entry-&gt;cm_node,</div><div class='add'>+			      __entry-&gt;refcount,</div><div class='add'>+			      __entry-&gt;vlan_id,</div><div class='add'>+			      __entry-&gt;accel,</div><div class='add'>+			      parse_cm_state(__entry-&gt;state),</div><div class='add'>+			      parse_cm_event_type(__entry-&gt;type),</div><div class='add'>+			      __print_ip_addr(__get_dynamic_array(laddr),</div><div class='add'>+					      __entry-&gt;lport, __entry-&gt;ipv4),</div><div class='add'>+			      __print_ip_addr(__get_dynamic_array(raddr),</div><div class='add'>+					      __entry-&gt;rport, __entry-&gt;ipv4)</div><div class='add'>+		    )</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(cm_node_template, irdma_create_event,</div><div class='add'>+	     TP_PROTO(struct irdma_cm_node *cm_node,</div><div class='add'>+		      enum irdma_cm_event_type type, void *caller),</div><div class='add'>+	     TP_ARGS(cm_node, type, caller));</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(cm_node_template, irdma_accept,</div><div class='add'>+	     TP_PROTO(struct irdma_cm_node *cm_node,</div><div class='add'>+		      enum irdma_cm_event_type type, void *caller),</div><div class='add'>+	     TP_ARGS(cm_node, type, caller));</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(cm_node_template, irdma_connect,</div><div class='add'>+	     TP_PROTO(struct irdma_cm_node *cm_node,</div><div class='add'>+		      enum irdma_cm_event_type type, void *caller),</div><div class='add'>+	     TP_ARGS(cm_node, type, caller));</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(cm_node_template, irdma_reject,</div><div class='add'>+	     TP_PROTO(struct irdma_cm_node *cm_node,</div><div class='add'>+		      enum irdma_cm_event_type type, void *caller),</div><div class='add'>+	     TP_ARGS(cm_node, type, caller));</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(cm_node_template, irdma_find_node,</div><div class='add'>+	     TP_PROTO(struct irdma_cm_node *cm_node,</div><div class='add'>+		      enum irdma_cm_event_type type, void *caller),</div><div class='add'>+	     TP_ARGS(cm_node, type, caller));</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(cm_node_template, irdma_send_reset,</div><div class='add'>+	     TP_PROTO(struct irdma_cm_node *cm_node,</div><div class='add'>+		      enum irdma_cm_event_type type, void *caller),</div><div class='add'>+	     TP_ARGS(cm_node, type, caller));</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(cm_node_template, irdma_rem_ref_cm_node,</div><div class='add'>+	     TP_PROTO(struct irdma_cm_node *cm_node,</div><div class='add'>+		      enum irdma_cm_event_type type, void *caller),</div><div class='add'>+	     TP_ARGS(cm_node, type, caller));</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(cm_node_template, irdma_cm_event_handler,</div><div class='add'>+	     TP_PROTO(struct irdma_cm_node *cm_node,</div><div class='add'>+		      enum irdma_cm_event_type type, void *caller),</div><div class='add'>+	     TP_ARGS(cm_node, type, caller));</div><div class='add'>+</div><div class='add'>+TRACE_EVENT(open_err_template,</div><div class='add'>+	    TP_PROTO(struct irdma_cm_node *cm_node, bool reset, void *caller),</div><div class='add'>+	    TP_ARGS(cm_node, reset, caller),</div><div class='add'>+	    TP_STRUCT__entry(__field(struct irdma_device *, iwdev)</div><div class='add'>+			     __field(struct irdma_cm_node *, cm_node)</div><div class='add'>+			     __field(enum irdma_cm_node_state, state)</div><div class='add'>+			     __field(bool, reset)</div><div class='add'>+			     __field(void *, caller)</div><div class='add'>+		    ),</div><div class='add'>+	    TP_fast_assign(__entry-&gt;iwdev = cm_node-&gt;iwdev;</div><div class='add'>+			   __entry-&gt;cm_node = cm_node;</div><div class='add'>+			   __entry-&gt;state = cm_node-&gt;state;</div><div class='add'>+			   __entry-&gt;reset = reset;</div><div class='add'>+			   __entry-&gt;caller = caller;</div><div class='add'>+		    ),</div><div class='add'>+	    TP_printk("iwdev=%p  caller=%pS  node%p reset=%d  state=%s",</div><div class='add'>+		      __entry-&gt;iwdev,</div><div class='add'>+		      __entry-&gt;caller,</div><div class='add'>+		      __entry-&gt;cm_node,</div><div class='add'>+		      __entry-&gt;reset,</div><div class='add'>+		      parse_cm_state(__entry-&gt;state)</div><div class='add'>+		    )</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(open_err_template, irdma_active_open_err,</div><div class='add'>+	     TP_PROTO(struct irdma_cm_node *cm_node, bool reset, void *caller),</div><div class='add'>+	     TP_ARGS(cm_node, reset, caller));</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(open_err_template, irdma_passive_open_err,</div><div class='add'>+	     TP_PROTO(struct irdma_cm_node *cm_node, bool reset, void *caller),</div><div class='add'>+	     TP_ARGS(cm_node, reset, caller));</div><div class='add'>+</div><div class='add'>+DECLARE_EVENT_CLASS(cm_node_ah_template,</div><div class='add'>+		    TP_PROTO(struct irdma_cm_node *cm_node),</div><div class='add'>+		    TP_ARGS(cm_node),</div><div class='add'>+		    TP_STRUCT__entry(__field(struct irdma_device *, iwdev)</div><div class='add'>+				     __field(struct irdma_cm_node *, cm_node)</div><div class='add'>+				     __field(struct irdma_sc_ah *, ah)</div><div class='add'>+				     __field(u32, refcount)</div><div class='add'>+				     __field(u16, lport)</div><div class='add'>+				     __field(u16, rport)</div><div class='add'>+				     __field(enum irdma_cm_node_state, state)</div><div class='add'>+				     __field(bool, ipv4)</div><div class='add'>+				     __field(u16, vlan_id)</div><div class='add'>+				     __field(int, accel)</div><div class='add'>+				     __dynamic_array(u32, laddr, 4)</div><div class='add'>+				     __dynamic_array(u32, raddr, 4)</div><div class='add'>+			    ),</div><div class='add'>+		    TP_fast_assign(__entry-&gt;iwdev = cm_node-&gt;iwdev;</div><div class='add'>+				   __entry-&gt;cm_node = cm_node;</div><div class='add'>+				   __entry-&gt;ah = cm_node-&gt;ah;</div><div class='add'>+				   __entry-&gt;refcount = refcount_read(&amp;cm_node-&gt;refcnt);</div><div class='add'>+				   __entry-&gt;lport = cm_node-&gt;loc_port;</div><div class='add'>+				   __entry-&gt;rport = cm_node-&gt;rem_port;</div><div class='add'>+				   __entry-&gt;state = cm_node-&gt;state;</div><div class='add'>+				   __entry-&gt;ipv4 = cm_node-&gt;ipv4;</div><div class='add'>+				   __entry-&gt;vlan_id = cm_node-&gt;vlan_id;</div><div class='add'>+				   __entry-&gt;accel = cm_node-&gt;accelerated;</div><div class='add'>+				   memcpy(__get_dynamic_array(laddr),</div><div class='add'>+					  cm_node-&gt;loc_addr, 4);</div><div class='add'>+				   memcpy(__get_dynamic_array(raddr),</div><div class='add'>+					  cm_node-&gt;rem_addr, 4);</div><div class='add'>+			    ),</div><div class='add'>+		    TP_printk("iwdev=%p  node=%p  ah=%p  refcnt=%d  vlan_id=%d  accel=%d  state=%s loc: %s  rem: %s",</div><div class='add'>+			      __entry-&gt;iwdev,</div><div class='add'>+			      __entry-&gt;cm_node,</div><div class='add'>+			      __entry-&gt;ah,</div><div class='add'>+			      __entry-&gt;refcount,</div><div class='add'>+			      __entry-&gt;vlan_id,</div><div class='add'>+			      __entry-&gt;accel,</div><div class='add'>+			      parse_cm_state(__entry-&gt;state),</div><div class='add'>+			      __print_ip_addr(__get_dynamic_array(laddr),</div><div class='add'>+					      __entry-&gt;lport, __entry-&gt;ipv4),</div><div class='add'>+			      __print_ip_addr(__get_dynamic_array(raddr),</div><div class='add'>+					      __entry-&gt;rport, __entry-&gt;ipv4)</div><div class='add'>+		    )</div><div class='add'>+);</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(cm_node_ah_template, irdma_cm_free_ah,</div><div class='add'>+	     TP_PROTO(struct irdma_cm_node *cm_node),</div><div class='add'>+	     TP_ARGS(cm_node));</div><div class='add'>+</div><div class='add'>+DEFINE_EVENT(cm_node_ah_template, irdma_create_ah,</div><div class='add'>+	     TP_PROTO(struct irdma_cm_node *cm_node),</div><div class='add'>+	     TP_ARGS(cm_node));</div><div class='add'>+</div><div class='add'>+#endif  /* __TRACE_CM_H */</div><div class='add'>+</div><div class='add'>+#undef TRACE_INCLUDE_PATH</div><div class='add'>+#undef TRACE_INCLUDE_FILE</div><div class='add'>+#define TRACE_INCLUDE_PATH .</div><div class='add'>+#define TRACE_INCLUDE_FILE trace_cm</div><div class='add'>+#include &lt;trace/define_trace.h&gt;</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/type.h b/drivers/infiniband/hw/irdma/type.h<br/>new file mode 100644<br/>index 000000000000..cab4896640a1<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/type.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/type.h</a></div><div class='hunk'>@@ -0,0 +1,1674 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#ifndef IRDMA_TYPE_H</div><div class='add'>+#define IRDMA_TYPE_H</div><div class='add'>+#include "osdep.h"</div><div class='add'>+#include "irdma.h"</div><div class='add'>+#include "user.h"</div><div class='add'>+#include "hmc.h"</div><div class='add'>+#include "uda.h"</div><div class='add'>+#include "ws.h"</div><div class='add'>+#include "virtchnl.h"</div><div class='add'>+</div><div class='add'>+#define IRDMA_DEBUG_ERR		"ERR"</div><div class='add'>+#define IRDMA_DEBUG_INIT	"INIT"</div><div class='add'>+#define IRDMA_DEBUG_DEV		"DEV"</div><div class='add'>+#define IRDMA_DEBUG_CM		"CM"</div><div class='add'>+#define IRDMA_DEBUG_VERBS	"VERBS"</div><div class='add'>+#define IRDMA_DEBUG_PUDA	"PUDA"</div><div class='add'>+#define IRDMA_DEBUG_ILQ		"ILQ"</div><div class='add'>+#define IRDMA_DEBUG_IEQ		"IEQ"</div><div class='add'>+#define IRDMA_DEBUG_QP		"QP"</div><div class='add'>+#define IRDMA_DEBUG_CQ		"CQ"</div><div class='add'>+#define IRDMA_DEBUG_MR		"MR"</div><div class='add'>+#define IRDMA_DEBUG_PBLE	"PBLE"</div><div class='add'>+#define IRDMA_DEBUG_WQE		"WQE"</div><div class='add'>+#define IRDMA_DEBUG_AEQ		"AEQ"</div><div class='add'>+#define IRDMA_DEBUG_CQP		"CQP"</div><div class='add'>+#define IRDMA_DEBUG_HMC		"HMC"</div><div class='add'>+#define IRDMA_DEBUG_USER	"USER"</div><div class='add'>+#define IRDMA_DEBUG_VIRT	"VIRT"</div><div class='add'>+#define IRDMA_DEBUG_DCB		"DCB"</div><div class='add'>+#define	IRDMA_DEBUG_CQE		"CQE"</div><div class='add'>+#define IRDMA_DEBUG_CLNT	"CLNT"</div><div class='add'>+#define IRDMA_DEBUG_WS		"WS"</div><div class='add'>+#define IRDMA_DEBUG_STATS	"STATS"</div><div class='add'>+</div><div class='add'>+enum irdma_page_size {</div><div class='add'>+	IRDMA_PAGE_SIZE_4K = 0,</div><div class='add'>+	IRDMA_PAGE_SIZE_2M,</div><div class='add'>+	IRDMA_PAGE_SIZE_1G,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_hdrct_flags {</div><div class='add'>+	DDP_LEN_FLAG  = 0x80,</div><div class='add'>+	DDP_HDR_FLAG  = 0x40,</div><div class='add'>+	RDMA_HDR_FLAG = 0x20,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_term_layers {</div><div class='add'>+	LAYER_RDMA = 0,</div><div class='add'>+	LAYER_DDP  = 1,</div><div class='add'>+	LAYER_MPA  = 2,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_term_error_types {</div><div class='add'>+	RDMAP_REMOTE_PROT = 1,</div><div class='add'>+	RDMAP_REMOTE_OP   = 2,</div><div class='add'>+	DDP_CATASTROPHIC  = 0,</div><div class='add'>+	DDP_TAGGED_BUF    = 1,</div><div class='add'>+	DDP_UNTAGGED_BUF  = 2,</div><div class='add'>+	DDP_LLP		  = 3,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_term_rdma_errors {</div><div class='add'>+	RDMAP_INV_STAG		  = 0x00,</div><div class='add'>+	RDMAP_INV_BOUNDS	  = 0x01,</div><div class='add'>+	RDMAP_ACCESS		  = 0x02,</div><div class='add'>+	RDMAP_UNASSOC_STAG	  = 0x03,</div><div class='add'>+	RDMAP_TO_WRAP		  = 0x04,</div><div class='add'>+	RDMAP_INV_RDMAP_VER       = 0x05,</div><div class='add'>+	RDMAP_UNEXPECTED_OP       = 0x06,</div><div class='add'>+	RDMAP_CATASTROPHIC_LOCAL  = 0x07,</div><div class='add'>+	RDMAP_CATASTROPHIC_GLOBAL = 0x08,</div><div class='add'>+	RDMAP_CANT_INV_STAG       = 0x09,</div><div class='add'>+	RDMAP_UNSPECIFIED	  = 0xff,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_term_ddp_errors {</div><div class='add'>+	DDP_CATASTROPHIC_LOCAL      = 0x00,</div><div class='add'>+	DDP_TAGGED_INV_STAG	    = 0x00,</div><div class='add'>+	DDP_TAGGED_BOUNDS	    = 0x01,</div><div class='add'>+	DDP_TAGGED_UNASSOC_STAG     = 0x02,</div><div class='add'>+	DDP_TAGGED_TO_WRAP	    = 0x03,</div><div class='add'>+	DDP_TAGGED_INV_DDP_VER      = 0x04,</div><div class='add'>+	DDP_UNTAGGED_INV_QN	    = 0x01,</div><div class='add'>+	DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02,</div><div class='add'>+	DDP_UNTAGGED_INV_MSN_RANGE  = 0x03,</div><div class='add'>+	DDP_UNTAGGED_INV_MO	    = 0x04,</div><div class='add'>+	DDP_UNTAGGED_INV_TOO_LONG   = 0x05,</div><div class='add'>+	DDP_UNTAGGED_INV_DDP_VER    = 0x06,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_term_mpa_errors {</div><div class='add'>+	MPA_CLOSED  = 0x01,</div><div class='add'>+	MPA_CRC     = 0x02,</div><div class='add'>+	MPA_MARKER  = 0x03,</div><div class='add'>+	MPA_REQ_RSP = 0x04,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_hw_stats_index {</div><div class='add'>+	/* gen1 - 32-bit */</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP4RXDISCARD	= 0,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP4RXTRUNC		= 1,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP4TXNOROUTE	= 2,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP6RXDISCARD	= 3,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP6RXTRUNC		= 4,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP6TXNOROUTE	= 5,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_TCPRTXSEG		= 6,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_TCPRXOPTERR		= 7,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_TCPRXPROTOERR	= 8,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RXVLANERR		= 9,</div><div class='add'>+		/* gen1 - 64-bit */</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP4RXOCTS		= 10,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP4RXPKTS		= 11,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP4RXFRAGS		= 12,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP4RXMCPKTS		= 13,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP4TXOCTS		= 14,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP4TXPKTS		= 15,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP4TXFRAGS		= 16,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP4TXMCPKTS		= 17,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP6RXOCTS		= 18,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP6RXPKTS		= 19,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP6RXFRAGS		= 20,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP6RXMCPKTS		= 21,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP6TXOCTS		= 22,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP6TXPKTS		= 23,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP6TXFRAGS		= 24,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP6TXMCPKTS		= 25,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_TCPRXSEGS		= 26,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_TCPTXSEG		= 27,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RDMARXRDS		= 28,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RDMARXSNDS		= 29,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RDMARXWRS		= 30,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RDMATXRDS		= 31,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RDMATXSNDS		= 32,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RDMATXWRS		= 33,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RDMAVBND		= 34,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RDMAVINV		= 35,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP4RXMCOCTS         = 36,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP4TXMCOCTS         = 37,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP6RXMCOCTS         = 38,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_IP6TXMCOCTS         = 39,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_UDPRXPKTS           = 40,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_UDPTXPKTS           = 41,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_MAX_GEN_1           = 42, /* Must be same value as next entry */</div><div class='add'>+	/* gen2 - 64-bit */</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS   = 42,</div><div class='add'>+	/* gen2 - 32-bit */</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED      = 43,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED      = 44,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_TXNPCNPSENT         = 45,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_MAX_GEN_2		= 46,</div><div class='add'>+</div><div class='add'>+	/* gen3 */</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RNR_SENT		= 46,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RNR_RCVD		= 47,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RDMAORDLMTCNT	= 48,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RDMAIRDLMTCNT	= 49,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RDMARXATS		= 50,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RDMATXATS		= 51,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_NAKSEQERR		= 52,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_NAKSEQERR_IMPLIED	= 53,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RTO			= 54,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_RXOOOPKTS		= 55,</div><div class='add'>+	IRDMA_HW_STAT_INDEX_ICRCERR		= 56,</div><div class='add'>+</div><div class='add'>+	IRDMA_HW_STAT_INDEX_MAX_GEN_3		= 57,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_feature_type {</div><div class='add'>+	IRDMA_FEATURE_FW_INFO = 0,</div><div class='add'>+	IRDMA_HW_VERSION_INFO = 1,</div><div class='add'>+	IRDMA_QP_MAX_INCR     = 2,</div><div class='add'>+	IRDMA_CQ_MAX_INCR     = 3,</div><div class='add'>+	IRDMA_CEQ_MAX_INCR    = 4,</div><div class='add'>+	IRDMA_SD_MAX_INCR     = 5,</div><div class='add'>+	IRDMA_MR_MAX_INCR     = 6,</div><div class='add'>+	IRDMA_Q1_MAX_INCR     = 7,</div><div class='add'>+	IRDMA_AH_MAX_INCR     = 8,</div><div class='add'>+	IRDMA_SRQ_MAX_INCR    = 9,</div><div class='add'>+	IRDMA_TIMER_MAX_INCR  = 10,</div><div class='add'>+	IRDMA_XF_MAX_INCR     = 11,</div><div class='add'>+	IRDMA_RRF_MAX_INCR    = 12,</div><div class='add'>+	IRDMA_PBLE_MAX_INCR   = 13,</div><div class='add'>+	IRDMA_OBJ_1           = 22,</div><div class='add'>+	IRDMA_OBJ_2           = 23,</div><div class='add'>+	IRDMA_ENDPT_TRK       = 24,</div><div class='add'>+	IRDMA_FTN_INLINE_MAX  = 25,</div><div class='add'>+	IRDMA_QSETS_MAX       = 26,</div><div class='add'>+	IRDMA_ASO	      = 27,</div><div class='add'>+	IRDMA_FTN_FLAGS	      = 32,</div><div class='add'>+	IRDMA_FTN_NOP         = 33,</div><div class='add'>+	IRDMA_MAX_FEATURES, /* Must be last entry */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_sched_prio_type {</div><div class='add'>+	IRDMA_PRIO_WEIGHTED_RR     = 1,</div><div class='add'>+	IRDMA_PRIO_STRICT	   = 2,</div><div class='add'>+	IRDMA_PRIO_WEIGHTED_STRICT = 3,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_vm_vf_type {</div><div class='add'>+	IRDMA_VF_TYPE = 0,</div><div class='add'>+	IRDMA_VM_TYPE,</div><div class='add'>+	IRDMA_PF_TYPE,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_cqp_hmc_profile {</div><div class='add'>+	IRDMA_HMC_PROFILE_DEFAULT  = 1,</div><div class='add'>+	IRDMA_HMC_PROFILE_FAVOR_VF = 2,</div><div class='add'>+	IRDMA_HMC_PROFILE_EQUAL    = 3,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_quad_entry_type {</div><div class='add'>+	IRDMA_QHASH_TYPE_TCP_ESTABLISHED = 1,</div><div class='add'>+	IRDMA_QHASH_TYPE_TCP_SYN,</div><div class='add'>+	IRDMA_QHASH_TYPE_UDP_UNICAST,</div><div class='add'>+	IRDMA_QHASH_TYPE_UDP_MCAST,</div><div class='add'>+	IRDMA_QHASH_TYPE_ROCE_MCAST,</div><div class='add'>+	IRDMA_QHASH_TYPE_ROCEV2_HW,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_quad_hash_manage_type {</div><div class='add'>+	IRDMA_QHASH_MANAGE_TYPE_DELETE = 0,</div><div class='add'>+	IRDMA_QHASH_MANAGE_TYPE_ADD,</div><div class='add'>+	IRDMA_QHASH_MANAGE_TYPE_MODIFY,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_syn_rst_handling {</div><div class='add'>+	IRDMA_SYN_RST_HANDLING_HW_TCP_SECURE = 0,</div><div class='add'>+	IRDMA_SYN_RST_HANDLING_HW_TCP,</div><div class='add'>+	IRDMA_SYN_RST_HANDLING_FW_TCP_SECURE,</div><div class='add'>+	IRDMA_SYN_RST_HANDLING_FW_TCP,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_queue_type {</div><div class='add'>+	IRDMA_QUEUE_TYPE_SQ_RQ = 0,</div><div class='add'>+	IRDMA_QUEUE_TYPE_CQP,</div><div class='add'>+	IRDMA_QUEUE_TYPE_SRQ,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_sc_dev;</div><div class='add'>+struct irdma_vsi_pestat;</div><div class='add'>+</div><div class='add'>+struct irdma_dcqcn_cc_params {</div><div class='add'>+	u8 cc_cfg_valid;</div><div class='add'>+	u8 min_dec_factor;</div><div class='add'>+	u8 min_rate;</div><div class='add'>+	u8 dcqcn_f;</div><div class='add'>+	u16 rai_factor;</div><div class='add'>+	u16 hai_factor;</div><div class='add'>+	u16 dcqcn_t;</div><div class='add'>+	u32 dcqcn_b;</div><div class='add'>+	u32 rreduce_mperiod;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cqp_init_info {</div><div class='add'>+	u64 cqp_compl_ctx;</div><div class='add'>+	u64 host_ctx_pa;</div><div class='add'>+	u64 sq_pa;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	struct irdma_cqp_quanta *sq;</div><div class='add'>+	struct irdma_dcqcn_cc_params dcqcn_params;</div><div class='add'>+	__le64 *host_ctx;</div><div class='add'>+	u64 *scratch_array;</div><div class='add'>+	u32 sq_size;</div><div class='add'>+	struct irdma_ooo_cqp_op *ooo_op_array;</div><div class='add'>+	u32 pe_en_vf_cnt;</div><div class='add'>+	u16 hw_maj_ver;</div><div class='add'>+	u16 hw_min_ver;</div><div class='add'>+	u8 struct_ver;</div><div class='add'>+	u8 hmc_profile;</div><div class='add'>+	u8 ena_vf_count;</div><div class='add'>+	u8 ceqs_per_vf;</div><div class='add'>+	u8 ooisc_blksize;</div><div class='add'>+	u8 rrsp_blksize;</div><div class='add'>+	u8 q1_blksize;</div><div class='add'>+	u8 xmit_blksize;</div><div class='add'>+	u8 ts_override;</div><div class='add'>+	u8 ts_shift;</div><div class='add'>+	u8 en_fine_grained_timers;</div><div class='add'>+	u8 blksizes_valid;</div><div class='add'>+	bool en_datacenter_tcp:1;</div><div class='add'>+	bool disable_packed:1;</div><div class='add'>+	bool rocev2_rto_policy:1;</div><div class='add'>+	enum irdma_protocol_used protocol_used;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_terminate_hdr {</div><div class='add'>+	u8 layer_etype;</div><div class='add'>+	u8 error_code;</div><div class='add'>+	u8 hdrct;</div><div class='add'>+	u8 rsvd;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cqp_sq_wqe {</div><div class='add'>+	__le64 buf[IRDMA_CQP_WQE_SIZE];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_sc_aeqe {</div><div class='add'>+	__le64 buf[IRDMA_AEQE_SIZE];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_ceqe {</div><div class='add'>+	__le64 buf[IRDMA_CEQE_SIZE];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cqp_ctx {</div><div class='add'>+	__le64 buf[IRDMA_CQP_CTX_SIZE];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cq_shadow_area {</div><div class='add'>+	__le64 buf[IRDMA_SHADOW_AREA_SIZE];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_dev_hw_stats_offsets {</div><div class='add'>+	u32 stats_offset[IRDMA_HW_STAT_INDEX_MAX_GEN_1];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_dev_hw_stats {</div><div class='add'>+	u64 stats_val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_gather_stats {</div><div class='add'>+	u64 val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_hw_stat_map {</div><div class='add'>+	u16 byteoff;</div><div class='add'>+	u8 bitoff;</div><div class='add'>+	u64 bitmask;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_stats_gather_info {</div><div class='add'>+	bool use_hmc_fcn_index:1;</div><div class='add'>+	bool use_stats_inst:1;</div><div class='add'>+	u8 hmc_fcn_index;</div><div class='add'>+	u8 stats_inst_index;</div><div class='add'>+	struct irdma_dma_mem stats_buff_mem;</div><div class='add'>+	void *gather_stats_va;</div><div class='add'>+	void *last_gather_stats_va;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_vsi_pestat {</div><div class='add'>+	struct irdma_hw *hw;</div><div class='add'>+	struct irdma_dev_hw_stats hw_stats;</div><div class='add'>+	struct irdma_stats_gather_info gather_info;</div><div class='add'>+	struct timer_list stats_timer;</div><div class='add'>+	struct irdma_sc_vsi *vsi;</div><div class='add'>+	struct irdma_dev_hw_stats last_hw_stats;</div><div class='add'>+	spinlock_t lock; /* rdma stats lock */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_mmio_region {</div><div class='add'>+	u8 __iomem *addr;</div><div class='add'>+	resource_size_t len;</div><div class='add'>+	resource_size_t offset;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_hw {</div><div class='add'>+	union {</div><div class='add'>+		u8 __iomem *hw_addr;</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_mmio_region rdma_reg; /* RDMA region */</div><div class='add'>+			struct irdma_mmio_region *io_regs; /* Non-RDMA MMIO regions */</div><div class='add'>+			u16 num_io_regions; /* Number of Non-RDMA MMIO regions */</div><div class='add'>+		};</div><div class='add'>+	};</div><div class='add'>+	struct device *device;</div><div class='add'>+	struct irdma_hmc_info hmc;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_pfpdu {</div><div class='add'>+	struct list_head rxlist;</div><div class='add'>+	u32 rcv_nxt;</div><div class='add'>+	u32 fps;</div><div class='add'>+	u32 max_fpdu_data;</div><div class='add'>+	u32 nextseqnum;</div><div class='add'>+	u32 rcv_start_seq;</div><div class='add'>+	bool mode:1;</div><div class='add'>+	bool mpa_crc_err:1;</div><div class='add'>+	u8  marker_len;</div><div class='add'>+	u64 total_ieq_bufs;</div><div class='add'>+	u64 fpdu_processed;</div><div class='add'>+	u64 bad_seq_num;</div><div class='add'>+	u64 crc_err;</div><div class='add'>+	u64 no_tx_bufs;</div><div class='add'>+	u64 tx_err;</div><div class='add'>+	u64 out_of_order;</div><div class='add'>+	u64 pmode_count;</div><div class='add'>+	struct irdma_sc_ah *ah;</div><div class='add'>+	struct irdma_puda_buf *ah_buf;</div><div class='add'>+	spinlock_t lock; /* fpdu processing lock */</div><div class='add'>+	struct irdma_puda_buf *lastrcv_buf;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_sc_pd {</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	u32 pd_id;</div><div class='add'>+	int abi_ver;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cqp_quanta {</div><div class='add'>+	__le64 elem[IRDMA_CQP_WQE_SIZE];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_ooo_cqp_op {</div><div class='add'>+	struct list_head list_entry;</div><div class='add'>+	u64 scratch;</div><div class='add'>+	u32 def_info;</div><div class='add'>+	u32 sw_def_info;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+	bool deferred:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_sc_cqp {</div><div class='add'>+	spinlock_t ooo_list_lock; /* protects list of pending completions */</div><div class='add'>+	struct list_head ooo_avail;</div><div class='add'>+	struct list_head ooo_pnd;</div><div class='add'>+	u32 last_def_cmpl_ticket;</div><div class='add'>+	u32 sw_def_cmpl_ticket;</div><div class='add'>+	u32 size;</div><div class='add'>+	u64 sq_pa;</div><div class='add'>+	u64 host_ctx_pa;</div><div class='add'>+	void *back_cqp;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	int (*process_cqp_sds)(struct irdma_sc_dev *dev,</div><div class='add'>+			       struct irdma_update_sds_info *info);</div><div class='add'>+	struct irdma_dma_mem sdbuf;</div><div class='add'>+	struct irdma_ring sq_ring;</div><div class='add'>+	struct irdma_cqp_quanta *sq_base;</div><div class='add'>+	struct irdma_dcqcn_cc_params dcqcn_params;</div><div class='add'>+	__le64 *host_ctx;</div><div class='add'>+	u64 *scratch_array;</div><div class='add'>+	u64 requested_ops;</div><div class='add'>+	atomic64_t completed_ops;</div><div class='add'>+	struct irdma_ooo_cqp_op *ooo_op_array;</div><div class='add'>+	u32 cqp_id;</div><div class='add'>+	u32 sq_size;</div><div class='add'>+	u32 pe_en_vf_cnt;</div><div class='add'>+	u32 hw_sq_size;</div><div class='add'>+	u16 hw_maj_ver;</div><div class='add'>+	u16 hw_min_ver;</div><div class='add'>+	u8 struct_ver;</div><div class='add'>+	u8 polarity;</div><div class='add'>+	u8 hmc_profile;</div><div class='add'>+	u8 ena_vf_count;</div><div class='add'>+	u8 timeout_count;</div><div class='add'>+	u8 ceqs_per_vf;</div><div class='add'>+	u8 ooisc_blksize;</div><div class='add'>+	u8 rrsp_blksize;</div><div class='add'>+	u8 q1_blksize;</div><div class='add'>+	u8 xmit_blksize;</div><div class='add'>+	u8 ts_override;</div><div class='add'>+	u8 ts_shift;</div><div class='add'>+	u8 en_fine_grained_timers;</div><div class='add'>+	u8 blksizes_valid;</div><div class='add'>+	bool en_datacenter_tcp:1;</div><div class='add'>+	bool disable_packed:1;</div><div class='add'>+	bool rocev2_rto_policy:1;</div><div class='add'>+	enum irdma_protocol_used protocol_used;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_sc_aeq {</div><div class='add'>+	u32 size;</div><div class='add'>+	u64 aeq_elem_pa;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	struct irdma_sc_aeqe *aeqe_base;</div><div class='add'>+	void *pbl_list;</div><div class='add'>+	u32 elem_cnt;</div><div class='add'>+	struct irdma_ring aeq_ring;</div><div class='add'>+	u8 pbl_chunk_size;</div><div class='add'>+	u32 first_pm_pbl_idx;</div><div class='add'>+	u32 msix_idx;</div><div class='add'>+	u8 polarity;</div><div class='add'>+	bool virtual_map:1;</div><div class='add'>+	bool pasid_valid:1;</div><div class='add'>+	u32 pasid;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_sc_ceq {</div><div class='add'>+	u32 size;</div><div class='add'>+	u64 ceq_elem_pa;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	struct irdma_ceqe *ceqe_base;</div><div class='add'>+	void *pbl_list;</div><div class='add'>+	u32 ceq_id;</div><div class='add'>+	u32 elem_cnt;</div><div class='add'>+	struct irdma_ring ceq_ring;</div><div class='add'>+	u8 pbl_chunk_size;</div><div class='add'>+	u8 tph_val;</div><div class='add'>+	u32 first_pm_pbl_idx;</div><div class='add'>+	u8 polarity;</div><div class='add'>+	u16 vsi_idx;</div><div class='add'>+	bool virtual_map:1;</div><div class='add'>+	bool tph_en:1;</div><div class='add'>+	bool itr_no_expire:1;</div><div class='add'>+	bool pasid_valid:1;</div><div class='add'>+	u32 pasid;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_sc_cq {</div><div class='add'>+	struct irdma_cq_uk cq_uk;</div><div class='add'>+	u64 cq_pa;</div><div class='add'>+	u64 shadow_area_pa;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	u16 vsi_idx;</div><div class='add'>+	struct irdma_sc_vsi *vsi;</div><div class='add'>+	void *pbl_list;</div><div class='add'>+	void *back_cq;</div><div class='add'>+	u32 ceq_id;</div><div class='add'>+	u32 shadow_read_threshold;</div><div class='add'>+	u8 pbl_chunk_size;</div><div class='add'>+	u8 cq_type;</div><div class='add'>+	u8 tph_val;</div><div class='add'>+	u32 first_pm_pbl_idx;</div><div class='add'>+	bool ceqe_mask:1;</div><div class='add'>+	bool virtual_map:1;</div><div class='add'>+	bool check_overflow:1;</div><div class='add'>+	bool ceq_id_valid:1;</div><div class='add'>+	bool tph_en;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_sc_qp {</div><div class='add'>+	struct irdma_qp_uk qp_uk;</div><div class='add'>+	u64 sq_pa;</div><div class='add'>+	u64 rq_pa;</div><div class='add'>+	u64 hw_host_ctx_pa;</div><div class='add'>+	u64 shadow_area_pa;</div><div class='add'>+	u64 q2_pa;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	struct irdma_sc_vsi *vsi;</div><div class='add'>+	struct irdma_sc_pd *pd;</div><div class='add'>+	__le64 *hw_host_ctx;</div><div class='add'>+	void *llp_stream_handle;</div><div class='add'>+	struct irdma_pfpdu pfpdu;</div><div class='add'>+	u32 ieq_qp;</div><div class='add'>+	u8 *q2_buf;</div><div class='add'>+	u64 qp_compl_ctx;</div><div class='add'>+	u32 push_idx;</div><div class='add'>+	u16 qs_handle;</div><div class='add'>+	u16 push_offset;</div><div class='add'>+	u8 flush_wqes_count;</div><div class='add'>+	u8 sq_tph_val;</div><div class='add'>+	u8 rq_tph_val;</div><div class='add'>+	u8 qp_state;</div><div class='add'>+	u8 hw_sq_size;</div><div class='add'>+	u8 hw_rq_size;</div><div class='add'>+	u8 src_mac_addr_idx;</div><div class='add'>+	bool on_qoslist:1;</div><div class='add'>+	bool ieq_pass_thru:1;</div><div class='add'>+	bool sq_tph_en:1;</div><div class='add'>+	bool rq_tph_en:1;</div><div class='add'>+	bool rcv_tph_en:1;</div><div class='add'>+	bool xmit_tph_en:1;</div><div class='add'>+	bool virtual_map:1;</div><div class='add'>+	bool flush_sq:1;</div><div class='add'>+	bool flush_rq:1;</div><div class='add'>+	bool err_sq_idx_valid:1;</div><div class='add'>+	bool err_rq_idx_valid:1;</div><div class='add'>+	u32 err_sq_idx;</div><div class='add'>+	u32 err_rq_idx;</div><div class='add'>+	bool sq_flush_code:1;</div><div class='add'>+	bool rq_flush_code:1;</div><div class='add'>+	u32 pkt_limit;</div><div class='add'>+	enum irdma_flush_opcode flush_code;</div><div class='add'>+	enum irdma_qp_event_type event_type;</div><div class='add'>+	u8 term_flags;</div><div class='add'>+	u8 user_pri;</div><div class='add'>+	struct list_head list;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_stats_inst_info {</div><div class='add'>+	bool use_hmc_fcn_index;</div><div class='add'>+	u8 hmc_fn_id;</div><div class='add'>+	u16 stats_idx;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_up_info {</div><div class='add'>+	u8 map[8];</div><div class='add'>+	u8 cnp_up_override;</div><div class='add'>+	u16 hmc_fcn_idx;</div><div class='add'>+	bool use_vlan:1;</div><div class='add'>+	bool use_cnp_up_override:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define IRDMA_MAX_WS_NODES	0x3FF</div><div class='add'>+#define IRDMA_WS_NODE_INVALID	0xFFFF</div><div class='add'>+</div><div class='add'>+struct irdma_ws_node_info {</div><div class='add'>+	u16 id;</div><div class='add'>+	u16 vsi;</div><div class='add'>+	u16 parent_id;</div><div class='add'>+	u16 qs_handle;</div><div class='add'>+	bool type_leaf:1;</div><div class='add'>+	bool enable:1;</div><div class='add'>+	u8 prio_type;</div><div class='add'>+	u8 tc;</div><div class='add'>+	u8 weight;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_hmc_fpm_misc {</div><div class='add'>+	u32 max_ceqs;</div><div class='add'>+	u32 max_sds;</div><div class='add'>+	u32 loc_mem_pages;</div><div class='add'>+	u8 ird;</div><div class='add'>+	u32 xf_block_size;</div><div class='add'>+	u32 q1_block_size;</div><div class='add'>+	u32 ht_multiplier;</div><div class='add'>+	u32 timer_bucket;</div><div class='add'>+	u32 rrf_block_size;</div><div class='add'>+	u32 ooiscf_block_size;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define IRDMA_VCHNL_MAX_MSG_SIZE 512</div><div class='add'>+#define IRDMA_LEAF_DEFAULT_REL_BW		64</div><div class='add'>+#define IRDMA_PARENT_DEFAULT_REL_BW		1</div><div class='add'>+</div><div class='add'>+struct irdma_qos {</div><div class='add'>+	struct list_head qplist;</div><div class='add'>+	struct mutex qos_mutex; /* protect QoS attributes per QoS level */</div><div class='add'>+	u64 lan_qos_handle;</div><div class='add'>+	u32 l2_sched_node_id;</div><div class='add'>+	u16 qs_handle;</div><div class='add'>+	u8 traffic_class;</div><div class='add'>+	u8 rel_bw;</div><div class='add'>+	u8 prio_type;</div><div class='add'>+	bool valid;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define IRDMA_INVALID_STATS_IDX 0xff</div><div class='add'>+struct irdma_sc_vsi {</div><div class='add'>+	u16 vsi_idx;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	void *back_vsi;</div><div class='add'>+	u32 ilq_count;</div><div class='add'>+	struct irdma_virt_mem ilq_mem;</div><div class='add'>+	struct irdma_puda_rsrc *ilq;</div><div class='add'>+	u32 ieq_count;</div><div class='add'>+	struct irdma_virt_mem ieq_mem;</div><div class='add'>+	struct irdma_puda_rsrc *ieq;</div><div class='add'>+	u32 exception_lan_q;</div><div class='add'>+	u16 mtu;</div><div class='add'>+	u16 vm_id;</div><div class='add'>+	enum irdma_vm_vf_type vm_vf_type;</div><div class='add'>+	bool stats_inst_alloc:1;</div><div class='add'>+	bool tc_change_pending:1;</div><div class='add'>+	struct irdma_vsi_pestat *pestat;</div><div class='add'>+	atomic_t qp_suspend_reqs;</div><div class='add'>+	int (*register_qset)(struct irdma_sc_vsi *vsi,</div><div class='add'>+			     struct irdma_ws_node *tc_node);</div><div class='add'>+	void (*unregister_qset)(struct irdma_sc_vsi *vsi,</div><div class='add'>+				struct irdma_ws_node *tc_node);</div><div class='add'>+	u8 qos_rel_bw;</div><div class='add'>+	u8 qos_prio_type;</div><div class='add'>+	u8 stats_idx;</div><div class='add'>+	u8 dscp_map[DSCP_MAX];</div><div class='add'>+	struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];</div><div class='add'>+	u64 hw_stats_regs[IRDMA_HW_STAT_INDEX_MAX_GEN_1];</div><div class='add'>+	bool dscp_mode:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_sc_dev {</div><div class='add'>+	struct list_head cqp_cmd_head; /* head of the CQP command list */</div><div class='add'>+	spinlock_t cqp_lock; /* protect CQP list access */</div><div class='add'>+	bool stats_idx_array[IRDMA_MAX_STATS_COUNT_GEN_1];</div><div class='add'>+	struct irdma_dma_mem vf_fpm_query_buf[IRDMA_MAX_PE_ENA_VF_COUNT];</div><div class='add'>+	u64 fpm_query_buf_pa;</div><div class='add'>+	u64 fpm_commit_buf_pa;</div><div class='add'>+	__le64 *fpm_query_buf;</div><div class='add'>+	__le64 *fpm_commit_buf;</div><div class='add'>+	struct irdma_hw *hw;</div><div class='add'>+	u8 __iomem *db_addr;</div><div class='add'>+	u32 __iomem *wqe_alloc_db;</div><div class='add'>+	u32 __iomem *cq_arm_db;</div><div class='add'>+	u32 __iomem *aeq_alloc_db;</div><div class='add'>+	u32 __iomem *cqp_db;</div><div class='add'>+	u32 __iomem *cq_ack_db;</div><div class='add'>+	u32 __iomem *ceq_itr_mask_db;</div><div class='add'>+	u32 __iomem *aeq_itr_mask_db;</div><div class='add'>+	u32 __iomem *hw_regs[IRDMA_MAX_REGS];</div><div class='add'>+	u32 ceq_itr;   /* Interrupt throttle, usecs between interrupts: 0 disabled. 2 - 8160 */</div><div class='add'>+	u64 hw_masks[IRDMA_MAX_MASKS];</div><div class='add'>+	u64 hw_shifts[IRDMA_MAX_SHIFTS];</div><div class='add'>+	const struct irdma_hw_stat_map *hw_stats_map;</div><div class='add'>+	u64 hw_stats_regs[IRDMA_HW_STAT_INDEX_MAX_GEN_1];</div><div class='add'>+	u64 feature_info[IRDMA_MAX_FEATURES];</div><div class='add'>+	u64 cqp_cmd_stats[IRDMA_MAX_CQP_OPS];</div><div class='add'>+	struct irdma_hw_attrs hw_attrs;</div><div class='add'>+	struct irdma_hmc_info *hmc_info;</div><div class='add'>+	struct irdma_vchnl_rdma_caps vc_caps;</div><div class='add'>+	u8 vc_recv_buf[IRDMA_VCHNL_MAX_MSG_SIZE];</div><div class='add'>+	u16 vc_recv_len;</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	struct irdma_sc_aeq *aeq;</div><div class='add'>+	struct irdma_sc_ceq *ceq[IRDMA_CEQ_MAX_COUNT];</div><div class='add'>+	struct irdma_sc_cq *ccq;</div><div class='add'>+	const struct irdma_irq_ops *irq_ops;</div><div class='add'>+	struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];</div><div class='add'>+	struct irdma_hmc_fpm_misc hmc_fpm_misc;</div><div class='add'>+	struct irdma_ws_node *ws_tree_root;</div><div class='add'>+	struct mutex ws_mutex; /* ws tree mutex */</div><div class='add'>+	u32 vchnl_ver;</div><div class='add'>+	u16 num_vfs;</div><div class='add'>+	u16 hmc_fn_id;</div><div class='add'>+	u16 vf_id;</div><div class='add'>+	bool privileged:1;</div><div class='add'>+	bool vchnl_up:1;</div><div class='add'>+	bool ceq_valid:1;</div><div class='add'>+	bool is_pf:1;</div><div class='add'>+	u8 protocol_used;</div><div class='add'>+	struct mutex vchnl_mutex; /* mutex to synchronize RDMA virtual channel messages */</div><div class='add'>+	u8 pci_rev;</div><div class='add'>+	int (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri);</div><div class='add'>+	void (*ws_remove)(struct irdma_sc_vsi *vsi, u8 user_pri);</div><div class='add'>+	void (*ws_reset)(struct irdma_sc_vsi *vsi);</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_modify_cq_info {</div><div class='add'>+	u64 cq_pa;</div><div class='add'>+	struct irdma_cqe *cq_base;</div><div class='add'>+	u32 cq_size;</div><div class='add'>+	u32 shadow_read_threshold;</div><div class='add'>+	u8 pbl_chunk_size;</div><div class='add'>+	u32 first_pm_pbl_idx;</div><div class='add'>+	bool virtual_map:1;</div><div class='add'>+	bool check_overflow;</div><div class='add'>+	bool cq_resize:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_srq_init_info {</div><div class='add'>+	struct irdma_sc_pd *pd;</div><div class='add'>+	struct irdma_sc_vsi *vsi;</div><div class='add'>+	u64 srq_pa;</div><div class='add'>+	u64 shadow_area_pa;</div><div class='add'>+	u32 first_pm_pbl_idx;</div><div class='add'>+	u32 pasid;</div><div class='add'>+	u32 srq_size;</div><div class='add'>+	u16 srq_limit;</div><div class='add'>+	u8 pasid_valid;</div><div class='add'>+	u8 wqe_size;</div><div class='add'>+	u8 leaf_pbl_size;</div><div class='add'>+	u8 virtual_map;</div><div class='add'>+	u8 tph_en;</div><div class='add'>+	u8 arm_limit_event;</div><div class='add'>+	u8 tph_value;</div><div class='add'>+	u8 pbl_chunk_size;</div><div class='add'>+	struct irdma_srq_uk_init_info srq_uk_init_info;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_sc_srq {</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	struct irdma_sc_vsi *vsi;</div><div class='add'>+	struct irdma_sc_pd *pd;</div><div class='add'>+	struct irdma_srq_uk srq_uk;</div><div class='add'>+	void *back_srq;</div><div class='add'>+	u64 srq_pa;</div><div class='add'>+	u64 shadow_area_pa;</div><div class='add'>+	u32 first_pm_pbl_idx;</div><div class='add'>+	u32 pasid;</div><div class='add'>+	u32 hw_srq_size;</div><div class='add'>+	u16 srq_limit;</div><div class='add'>+	u8 pasid_valid;</div><div class='add'>+	u8 leaf_pbl_size;</div><div class='add'>+	u8 virtual_map;</div><div class='add'>+	u8 tph_en;</div><div class='add'>+	u8 arm_limit_event;</div><div class='add'>+	u8 tph_val;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_modify_srq_info {</div><div class='add'>+	u16 srq_limit;</div><div class='add'>+	u8 arm_limit_event;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_create_qp_info {</div><div class='add'>+	bool ord_valid:1;</div><div class='add'>+	bool tcp_ctx_valid:1;</div><div class='add'>+	bool cq_num_valid:1;</div><div class='add'>+	bool arp_cache_idx_valid:1;</div><div class='add'>+	bool mac_valid:1;</div><div class='add'>+	bool force_lpb;</div><div class='add'>+	u8 next_iwarp_state;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_modify_qp_info {</div><div class='add'>+	u64 rx_win0;</div><div class='add'>+	u64 rx_win1;</div><div class='add'>+	u16 new_mss;</div><div class='add'>+	u8 next_iwarp_state;</div><div class='add'>+	u8 curr_iwarp_state;</div><div class='add'>+	u8 termlen;</div><div class='add'>+	bool ord_valid:1;</div><div class='add'>+	bool tcp_ctx_valid:1;</div><div class='add'>+	bool udp_ctx_valid:1;</div><div class='add'>+	bool cq_num_valid:1;</div><div class='add'>+	bool arp_cache_idx_valid:1;</div><div class='add'>+	bool reset_tcp_conn:1;</div><div class='add'>+	bool remove_hash_idx:1;</div><div class='add'>+	bool dont_send_term:1;</div><div class='add'>+	bool dont_send_fin:1;</div><div class='add'>+	bool cached_var_valid:1;</div><div class='add'>+	bool mss_change:1;</div><div class='add'>+	bool force_lpb:1;</div><div class='add'>+	bool mac_valid:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_ccq_cqe_info {</div><div class='add'>+	struct irdma_sc_cqp *cqp;</div><div class='add'>+	u64 scratch;</div><div class='add'>+	u32 op_ret_val;</div><div class='add'>+	u16 maj_err_code;</div><div class='add'>+	u16 min_err_code;</div><div class='add'>+	u8 op_code;</div><div class='add'>+	bool error:1;</div><div class='add'>+	bool pending:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_dcb_app_info {</div><div class='add'>+	u8 priority;</div><div class='add'>+	u8 selector;</div><div class='add'>+	u16 prot_id;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_qos_tc_info {</div><div class='add'>+	u64 tc_ctx;</div><div class='add'>+	u8 rel_bw;</div><div class='add'>+	u8 prio_type;</div><div class='add'>+	u8 egress_virt_up;</div><div class='add'>+	u8 ingress_virt_up;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_l2params {</div><div class='add'>+	struct irdma_qos_tc_info tc_info[IRDMA_MAX_USER_PRIORITY];</div><div class='add'>+	struct irdma_dcb_app_info apps[IRDMA_MAX_APPS];</div><div class='add'>+	u32 num_apps;</div><div class='add'>+	u16 qs_handle_list[IRDMA_MAX_USER_PRIORITY];</div><div class='add'>+	u16 mtu;</div><div class='add'>+	u8 up2tc[IRDMA_MAX_USER_PRIORITY];</div><div class='add'>+	u8 dscp_map[DSCP_MAX];</div><div class='add'>+	u8 num_tc;</div><div class='add'>+	u8 vsi_rel_bw;</div><div class='add'>+	u8 vsi_prio_type;</div><div class='add'>+	bool mtu_changed:1;</div><div class='add'>+	bool tc_changed:1;</div><div class='add'>+	bool dscp_mode:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_vsi_init_info {</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	void *back_vsi;</div><div class='add'>+	struct irdma_l2params *params;</div><div class='add'>+	u16 exception_lan_q;</div><div class='add'>+	u16 pf_data_vsi_num;</div><div class='add'>+	enum irdma_vm_vf_type vm_vf_type;</div><div class='add'>+	u16 vm_id;</div><div class='add'>+	int (*register_qset)(struct irdma_sc_vsi *vsi,</div><div class='add'>+			     struct irdma_ws_node *tc_node);</div><div class='add'>+	void (*unregister_qset)(struct irdma_sc_vsi *vsi,</div><div class='add'>+				struct irdma_ws_node *tc_node);</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_vsi_stats_info {</div><div class='add'>+	struct irdma_vsi_pestat *pestat;</div><div class='add'>+	u16 fcn_id;</div><div class='add'>+	bool alloc_stats_inst;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_device_init_info {</div><div class='add'>+	u64 fpm_query_buf_pa;</div><div class='add'>+	u64 fpm_commit_buf_pa;</div><div class='add'>+	__le64 *fpm_query_buf;</div><div class='add'>+	__le64 *fpm_commit_buf;</div><div class='add'>+	struct irdma_hw *hw;</div><div class='add'>+	void __iomem *bar0;</div><div class='add'>+	enum irdma_protocol_used protocol_used;</div><div class='add'>+	u16 hmc_fn_id;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_ceq_init_info {</div><div class='add'>+	u64 ceqe_pa;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	u64 *ceqe_base;</div><div class='add'>+	void *pbl_list;</div><div class='add'>+	u32 elem_cnt;</div><div class='add'>+	u32 ceq_id;</div><div class='add'>+	bool virtual_map:1;</div><div class='add'>+	bool tph_en:1;</div><div class='add'>+	bool itr_no_expire:1;</div><div class='add'>+	u8 pbl_chunk_size;</div><div class='add'>+	u8 tph_val;</div><div class='add'>+	u16 vsi_idx;</div><div class='add'>+	u32 first_pm_pbl_idx;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_aeq_init_info {</div><div class='add'>+	u64 aeq_elem_pa;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	u32 *aeqe_base;</div><div class='add'>+	void *pbl_list;</div><div class='add'>+	u32 elem_cnt;</div><div class='add'>+	bool virtual_map;</div><div class='add'>+	u8 pbl_chunk_size;</div><div class='add'>+	u32 first_pm_pbl_idx;</div><div class='add'>+	u32 msix_idx;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_ccq_init_info {</div><div class='add'>+	u64 cq_pa;</div><div class='add'>+	u64 shadow_area_pa;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	struct irdma_cqe *cq_base;</div><div class='add'>+	__le64 *shadow_area;</div><div class='add'>+	void *pbl_list;</div><div class='add'>+	u32 num_elem;</div><div class='add'>+	u32 ceq_id;</div><div class='add'>+	u32 shadow_read_threshold;</div><div class='add'>+	bool ceqe_mask:1;</div><div class='add'>+	bool ceq_id_valid:1;</div><div class='add'>+	bool avoid_mem_cflct:1;</div><div class='add'>+	bool virtual_map:1;</div><div class='add'>+	bool tph_en:1;</div><div class='add'>+	u8 tph_val;</div><div class='add'>+	u8 pbl_chunk_size;</div><div class='add'>+	u32 first_pm_pbl_idx;</div><div class='add'>+	struct irdma_sc_vsi *vsi;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_udp_offload_info {</div><div class='add'>+	bool ipv4:1;</div><div class='add'>+	bool insert_vlan_tag:1;</div><div class='add'>+	u8 ttl;</div><div class='add'>+	u8 tos;</div><div class='add'>+	u16 src_port;</div><div class='add'>+	u16 dst_port;</div><div class='add'>+	u32 dest_ip_addr[4];</div><div class='add'>+	u32 snd_mss;</div><div class='add'>+	u16 vlan_tag;</div><div class='add'>+	u16 arp_idx;</div><div class='add'>+	u32 flow_label;</div><div class='add'>+	u8 udp_state;</div><div class='add'>+	u32 psn_nxt;</div><div class='add'>+	u32 lsn;</div><div class='add'>+	u32 epsn;</div><div class='add'>+	u32 psn_max;</div><div class='add'>+	u32 psn_una;</div><div class='add'>+	u32 local_ipaddr[4];</div><div class='add'>+	u32 cwnd;</div><div class='add'>+	u8 rexmit_thresh;</div><div class='add'>+	u8 rnr_nak_thresh;</div><div class='add'>+	u8 rnr_nak_tmr;</div><div class='add'>+	u8 min_rnr_timer;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_roce_offload_info {</div><div class='add'>+	u16 p_key;</div><div class='add'>+	u16 err_rq_idx;</div><div class='add'>+	u32 qkey;</div><div class='add'>+	u32 dest_qp;</div><div class='add'>+	u8 roce_tver;</div><div class='add'>+	u8 ack_credits;</div><div class='add'>+	u8 err_rq_idx_valid;</div><div class='add'>+	u32 pd_id;</div><div class='add'>+	u16 ord_size;</div><div class='add'>+	u16 ird_size;</div><div class='add'>+	bool is_qp1:1;</div><div class='add'>+	bool udprivcq_en:1;</div><div class='add'>+	bool dcqcn_en:1;</div><div class='add'>+	bool rcv_no_icrc:1;</div><div class='add'>+	bool wr_rdresp_en:1;</div><div class='add'>+	bool bind_en:1;</div><div class='add'>+	bool fast_reg_en:1;</div><div class='add'>+	bool priv_mode_en:1;</div><div class='add'>+	bool rd_en:1;</div><div class='add'>+	bool timely_en:1;</div><div class='add'>+	bool dctcp_en:1;</div><div class='add'>+	bool fw_cc_enable:1;</div><div class='add'>+	bool use_stats_inst:1;</div><div class='add'>+	u8 local_ack_timeout;</div><div class='add'>+	u16 t_high;</div><div class='add'>+	u16 t_low;</div><div class='add'>+	u8 last_byte_sent;</div><div class='add'>+	u8 mac_addr[ETH_ALEN];</div><div class='add'>+	u8 rtomin;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_iwarp_offload_info {</div><div class='add'>+	u16 rcv_mark_offset;</div><div class='add'>+	u16 snd_mark_offset;</div><div class='add'>+	u8 ddp_ver;</div><div class='add'>+	u8 rdmap_ver;</div><div class='add'>+	u8 iwarp_mode;</div><div class='add'>+	u16 err_rq_idx;</div><div class='add'>+	u32 pd_id;</div><div class='add'>+	u16 ord_size;</div><div class='add'>+	u16 ird_size;</div><div class='add'>+	bool ib_rd_en:1;</div><div class='add'>+	bool align_hdrs:1;</div><div class='add'>+	bool rcv_no_mpa_crc:1;</div><div class='add'>+	bool err_rq_idx_valid:1;</div><div class='add'>+	bool snd_mark_en:1;</div><div class='add'>+	bool rcv_mark_en:1;</div><div class='add'>+	bool wr_rdresp_en:1;</div><div class='add'>+	bool bind_en:1;</div><div class='add'>+	bool fast_reg_en:1;</div><div class='add'>+	bool priv_mode_en:1;</div><div class='add'>+	bool rd_en:1;</div><div class='add'>+	bool timely_en:1;</div><div class='add'>+	bool use_stats_inst:1;</div><div class='add'>+	bool ecn_en:1;</div><div class='add'>+	bool dctcp_en:1;</div><div class='add'>+	u16 t_high;</div><div class='add'>+	u16 t_low;</div><div class='add'>+	u8 last_byte_sent;</div><div class='add'>+	u8 mac_addr[ETH_ALEN];</div><div class='add'>+	u8 rtomin;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_tcp_offload_info {</div><div class='add'>+	bool ipv4:1;</div><div class='add'>+	bool no_nagle:1;</div><div class='add'>+	bool insert_vlan_tag:1;</div><div class='add'>+	bool time_stamp:1;</div><div class='add'>+	bool drop_ooo_seg:1;</div><div class='add'>+	bool avoid_stretch_ack:1;</div><div class='add'>+	bool wscale:1;</div><div class='add'>+	bool ignore_tcp_opt:1;</div><div class='add'>+	bool ignore_tcp_uns_opt:1;</div><div class='add'>+	u8 cwnd_inc_limit;</div><div class='add'>+	u8 dup_ack_thresh;</div><div class='add'>+	u8 ttl;</div><div class='add'>+	u8 src_mac_addr_idx;</div><div class='add'>+	u8 tos;</div><div class='add'>+	u16 src_port;</div><div class='add'>+	u16 dst_port;</div><div class='add'>+	u32 dest_ip_addr[4];</div><div class='add'>+	//u32 dest_ip_addr0;</div><div class='add'>+	//u32 dest_ip_addr1;</div><div class='add'>+	//u32 dest_ip_addr2;</div><div class='add'>+	//u32 dest_ip_addr3;</div><div class='add'>+	u32 snd_mss;</div><div class='add'>+	u16 syn_rst_handling;</div><div class='add'>+	u16 vlan_tag;</div><div class='add'>+	u16 arp_idx;</div><div class='add'>+	u32 flow_label;</div><div class='add'>+	u8 tcp_state;</div><div class='add'>+	u8 snd_wscale;</div><div class='add'>+	u8 rcv_wscale;</div><div class='add'>+	u32 time_stamp_recent;</div><div class='add'>+	u32 time_stamp_age;</div><div class='add'>+	u32 snd_nxt;</div><div class='add'>+	u32 snd_wnd;</div><div class='add'>+	u32 rcv_nxt;</div><div class='add'>+	u32 rcv_wnd;</div><div class='add'>+	u32 snd_max;</div><div class='add'>+	u32 snd_una;</div><div class='add'>+	u32 srtt;</div><div class='add'>+	u32 rtt_var;</div><div class='add'>+	u32 ss_thresh;</div><div class='add'>+	u32 cwnd;</div><div class='add'>+	u32 snd_wl1;</div><div class='add'>+	u32 snd_wl2;</div><div class='add'>+	u32 max_snd_window;</div><div class='add'>+	u8 rexmit_thresh;</div><div class='add'>+	u32 local_ipaddr[4];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_qp_host_ctx_info {</div><div class='add'>+	u64 qp_compl_ctx;</div><div class='add'>+	union {</div><div class='add'>+		struct irdma_tcp_offload_info *tcp_info;</div><div class='add'>+		struct irdma_udp_offload_info *udp_info;</div><div class='add'>+	};</div><div class='add'>+	union {</div><div class='add'>+		struct irdma_iwarp_offload_info *iwarp_info;</div><div class='add'>+		struct irdma_roce_offload_info *roce_info;</div><div class='add'>+	};</div><div class='add'>+	u32 send_cq_num;</div><div class='add'>+	u32 rcv_cq_num;</div><div class='add'>+	u32 srq_id;</div><div class='add'>+	u32 rem_endpoint_idx;</div><div class='add'>+	u16 stats_idx;</div><div class='add'>+	bool remote_atomics_en:1;</div><div class='add'>+	bool srq_valid:1;</div><div class='add'>+	bool tcp_info_valid:1;</div><div class='add'>+	bool iwarp_info_valid:1;</div><div class='add'>+	bool stats_idx_valid:1;</div><div class='add'>+	u8 user_pri;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_aeqe_info {</div><div class='add'>+	u64 compl_ctx;</div><div class='add'>+	u32 qp_cq_id;</div><div class='add'>+	u32 def_info;	/* only valid for DEF_CMPL */</div><div class='add'>+	u16 ae_id;</div><div class='add'>+	u16 wqe_idx;</div><div class='add'>+	u8 tcp_state;</div><div class='add'>+	u8 iwarp_state;</div><div class='add'>+	bool qp:1;</div><div class='add'>+	bool cq:1;</div><div class='add'>+	bool sq:1;</div><div class='add'>+	bool rq:1;</div><div class='add'>+	bool srq:1;</div><div class='add'>+	bool in_rdrsp_wr:1;</div><div class='add'>+	bool out_rdrsp:1;</div><div class='add'>+	bool aeqe_overflow:1;</div><div class='add'>+	bool err_rq_idx_valid:1;</div><div class='add'>+	u8 q2_data_written;</div><div class='add'>+	u8 ae_src;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_allocate_stag_info {</div><div class='add'>+	u64 total_len;</div><div class='add'>+	u64 first_pm_pbl_idx;</div><div class='add'>+	u32 chunk_size;</div><div class='add'>+	u32 stag_idx;</div><div class='add'>+	u32 page_size;</div><div class='add'>+	u32 pd_id;</div><div class='add'>+	u16 access_rights;</div><div class='add'>+	bool remote_access:1;</div><div class='add'>+	bool use_hmc_fcn_index:1;</div><div class='add'>+	bool use_pf_rid:1;</div><div class='add'>+	bool all_memory:1;</div><div class='add'>+	bool remote_atomics_en:1;</div><div class='add'>+	u16 hmc_fcn_index;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_mw_alloc_info {</div><div class='add'>+	u32 mw_stag_index;</div><div class='add'>+	u32 page_size;</div><div class='add'>+	u32 pd_id;</div><div class='add'>+	bool remote_access:1;</div><div class='add'>+	bool mw_wide:1;</div><div class='add'>+	bool mw1_bind_dont_vldt_key:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_reg_ns_stag_info {</div><div class='add'>+	u64 reg_addr_pa;</div><div class='add'>+	u64 va;</div><div class='add'>+	u64 total_len;</div><div class='add'>+	u32 page_size;</div><div class='add'>+	u32 chunk_size;</div><div class='add'>+	u32 first_pm_pbl_index;</div><div class='add'>+	enum irdma_addressing_type addr_type;</div><div class='add'>+	irdma_stag_index stag_idx;</div><div class='add'>+	u16 access_rights;</div><div class='add'>+	u32 pd_id;</div><div class='add'>+	irdma_stag_key stag_key;</div><div class='add'>+	bool use_hmc_fcn_index:1;</div><div class='add'>+	u8 hmc_fcn_index;</div><div class='add'>+	bool use_pf_rid:1;</div><div class='add'>+	bool all_memory:1;</div><div class='add'>+	bool remote_atomics_en:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_fast_reg_stag_info {</div><div class='add'>+	u64 wr_id;</div><div class='add'>+	u64 reg_addr_pa;</div><div class='add'>+	u64 fbo;</div><div class='add'>+	void *va;</div><div class='add'>+	u64 total_len;</div><div class='add'>+	u32 page_size;</div><div class='add'>+	u32 chunk_size;</div><div class='add'>+	u32 first_pm_pbl_index;</div><div class='add'>+	enum irdma_addressing_type addr_type;</div><div class='add'>+	irdma_stag_index stag_idx;</div><div class='add'>+	u16 access_rights;</div><div class='add'>+	u32 pd_id;</div><div class='add'>+	irdma_stag_key stag_key;</div><div class='add'>+	bool local_fence:1;</div><div class='add'>+	bool read_fence:1;</div><div class='add'>+	bool signaled:1;</div><div class='add'>+	bool use_hmc_fcn_index:1;</div><div class='add'>+	u8 hmc_fcn_index;</div><div class='add'>+	bool use_pf_rid:1;</div><div class='add'>+	bool defer_flag:1;</div><div class='add'>+	bool remote_atomics_en:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_dealloc_stag_info {</div><div class='add'>+	u32 stag_idx;</div><div class='add'>+	u32 pd_id;</div><div class='add'>+	bool mr:1;</div><div class='add'>+	bool dealloc_pbl:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_register_shared_stag {</div><div class='add'>+	u64 va;</div><div class='add'>+	enum irdma_addressing_type addr_type;</div><div class='add'>+	irdma_stag_index new_stag_idx;</div><div class='add'>+	irdma_stag_index parent_stag_idx;</div><div class='add'>+	u32 access_rights;</div><div class='add'>+	u32 pd_id;</div><div class='add'>+	u32 page_size;</div><div class='add'>+	irdma_stag_key new_stag_key;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_qp_init_info {</div><div class='add'>+	struct irdma_qp_uk_init_info qp_uk_init_info;</div><div class='add'>+	struct irdma_sc_pd *pd;</div><div class='add'>+	struct irdma_sc_vsi *vsi;</div><div class='add'>+	__le64 *host_ctx;</div><div class='add'>+	u8 *q2;</div><div class='add'>+	u64 sq_pa;</div><div class='add'>+	u64 rq_pa;</div><div class='add'>+	u64 host_ctx_pa;</div><div class='add'>+	u64 q2_pa;</div><div class='add'>+	u64 shadow_area_pa;</div><div class='add'>+	u8 sq_tph_val;</div><div class='add'>+	u8 rq_tph_val;</div><div class='add'>+	bool sq_tph_en:1;</div><div class='add'>+	bool rq_tph_en:1;</div><div class='add'>+	bool rcv_tph_en:1;</div><div class='add'>+	bool xmit_tph_en:1;</div><div class='add'>+	bool virtual_map:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cq_init_info {</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	u64 cq_base_pa;</div><div class='add'>+	u64 shadow_area_pa;</div><div class='add'>+	u32 ceq_id;</div><div class='add'>+	u32 shadow_read_threshold;</div><div class='add'>+	u8 pbl_chunk_size;</div><div class='add'>+	u32 first_pm_pbl_idx;</div><div class='add'>+	bool virtual_map:1;</div><div class='add'>+	bool ceqe_mask:1;</div><div class='add'>+	bool ceq_id_valid:1;</div><div class='add'>+	bool tph_en:1;</div><div class='add'>+	u8 tph_val;</div><div class='add'>+	u8 type;</div><div class='add'>+	struct irdma_cq_uk_init_info cq_uk_init_info;</div><div class='add'>+	struct irdma_sc_vsi *vsi;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_upload_context_info {</div><div class='add'>+	u64 buf_pa;</div><div class='add'>+	u32 qp_id;</div><div class='add'>+	u8 qp_type;</div><div class='add'>+	bool freeze_qp:1;</div><div class='add'>+	bool raw_format:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_local_mac_entry_info {</div><div class='add'>+	u8 mac_addr[6];</div><div class='add'>+	u16 entry_idx;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_add_arp_cache_entry_info {</div><div class='add'>+	u8 mac_addr[ETH_ALEN];</div><div class='add'>+	u32 reach_max;</div><div class='add'>+	u16 arp_index;</div><div class='add'>+	bool permanent;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_apbvt_info {</div><div class='add'>+	u16 port;</div><div class='add'>+	bool add;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_qhash_table_info {</div><div class='add'>+	struct irdma_sc_vsi *vsi;</div><div class='add'>+	enum irdma_quad_hash_manage_type manage;</div><div class='add'>+	enum irdma_quad_entry_type entry_type;</div><div class='add'>+	bool vlan_valid:1;</div><div class='add'>+	bool ipv4_valid:1;</div><div class='add'>+	u8 mac_addr[ETH_ALEN];</div><div class='add'>+	u16 vlan_id;</div><div class='add'>+	u8 user_pri;</div><div class='add'>+	u32 qp_num;</div><div class='add'>+	u32 dest_ip[4];</div><div class='add'>+	u32 src_ip[4];</div><div class='add'>+	u16 dest_port;</div><div class='add'>+	u16 src_port;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cqp_manage_push_page_info {</div><div class='add'>+	u32 push_idx;</div><div class='add'>+	u16 qs_handle;</div><div class='add'>+	u8 free_page;</div><div class='add'>+	u8 push_page_type;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_qp_flush_info {</div><div class='add'>+	u32 err_sq_idx;</div><div class='add'>+	u32 err_rq_idx;</div><div class='add'>+	u16 sq_minor_code;</div><div class='add'>+	u16 sq_major_code;</div><div class='add'>+	u16 rq_minor_code;</div><div class='add'>+	u16 rq_major_code;</div><div class='add'>+	u16 ae_code;</div><div class='add'>+	u8 ae_src;</div><div class='add'>+	bool sq:1;</div><div class='add'>+	bool rq:1;</div><div class='add'>+	bool userflushcode:1;</div><div class='add'>+	bool generate_ae:1;</div><div class='add'>+	bool err_sq_idx_valid:1;</div><div class='add'>+	bool err_rq_idx_valid:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_gen_ae_info {</div><div class='add'>+	u16 ae_code;</div><div class='add'>+	u8 ae_src;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cqp_timeout {</div><div class='add'>+	u64 compl_cqp_cmds;</div><div class='add'>+	u32 count;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_irq_ops {</div><div class='add'>+	void (*irdma_cfg_aeq)(struct irdma_sc_dev *dev, u32 idx, bool enable);</div><div class='add'>+	void (*irdma_cfg_ceq)(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx,</div><div class='add'>+			      bool enable);</div><div class='add'>+	void (*irdma_dis_irq)(struct irdma_sc_dev *dev, u32 idx);</div><div class='add'>+	void (*irdma_en_irq)(struct irdma_sc_dev *dev, u32 idx);</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq);</div><div class='add'>+int irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch,</div><div class='add'>+			bool check_overflow, bool post_sq);</div><div class='add'>+int irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq);</div><div class='add'>+int irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,</div><div class='add'>+			      struct irdma_ccq_cqe_info *info);</div><div class='add'>+int irdma_sc_ccq_init(struct irdma_sc_cq *ccq,</div><div class='add'>+		      struct irdma_ccq_init_info *info);</div><div class='add'>+</div><div class='add'>+int irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch);</div><div class='add'>+int irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq);</div><div class='add'>+</div><div class='add'>+int irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq);</div><div class='add'>+int irdma_sc_ceq_init(struct irdma_sc_ceq *ceq,</div><div class='add'>+		      struct irdma_ceq_init_info *info);</div><div class='add'>+void irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq);</div><div class='add'>+void *irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq);</div><div class='add'>+</div><div class='add'>+int irdma_sc_aeq_init(struct irdma_sc_aeq *aeq,</div><div class='add'>+		      struct irdma_aeq_init_info *info);</div><div class='add'>+int irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq,</div><div class='add'>+			   struct irdma_aeqe_info *info);</div><div class='add'>+void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count);</div><div class='add'>+</div><div class='add'>+void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id,</div><div class='add'>+		      int abi_ver);</div><div class='add'>+void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable);</div><div class='add'>+void irdma_check_cqp_progress(struct irdma_cqp_timeout *cqp_timeout,</div><div class='add'>+			      struct irdma_sc_dev *dev);</div><div class='add'>+void irdma_sc_cqp_def_cmpl_ae_handler(struct irdma_sc_dev *dev,</div><div class='add'>+				      struct irdma_aeqe_info *info,</div><div class='add'>+				      bool first, u64 *scratch,</div><div class='add'>+				      u32 *sw_def_info);</div><div class='add'>+u64 irdma_sc_cqp_cleanup_handler(struct irdma_sc_dev *dev);</div><div class='add'>+int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err);</div><div class='add'>+int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp);</div><div class='add'>+int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp,</div><div class='add'>+		      struct irdma_cqp_init_info *info);</div><div class='add'>+void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);</div><div class='add'>+int irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 opcode,</div><div class='add'>+				  struct irdma_ccq_cqe_info *cmpl_info);</div><div class='add'>+int irdma_sc_fast_register(struct irdma_sc_qp *qp,</div><div class='add'>+			   struct irdma_fast_reg_stag_info *info, bool post_sq);</div><div class='add'>+int irdma_sc_qp_create(struct irdma_sc_qp *qp,</div><div class='add'>+		       struct irdma_create_qp_info *info, u64 scratch,</div><div class='add'>+		       bool post_sq);</div><div class='add'>+int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch,</div><div class='add'>+			bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq);</div><div class='add'>+int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,</div><div class='add'>+			   struct irdma_qp_flush_info *info, u64 scratch,</div><div class='add'>+			   bool post_sq);</div><div class='add'>+int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info);</div><div class='add'>+int irdma_sc_qp_modify(struct irdma_sc_qp *qp,</div><div class='add'>+		       struct irdma_modify_qp_info *info, u64 scratch,</div><div class='add'>+		       bool post_sq);</div><div class='add'>+void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,</div><div class='add'>+			irdma_stag stag);</div><div class='add'>+</div><div class='add'>+void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read);</div><div class='add'>+void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx,</div><div class='add'>+			struct irdma_qp_host_ctx_info *info);</div><div class='add'>+void irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx,</div><div class='add'>+			     struct irdma_qp_host_ctx_info *info);</div><div class='add'>+int irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq);</div><div class='add'>+int irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info);</div><div class='add'>+void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info);</div><div class='add'>+int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,</div><div class='add'>+					u8 hmc_fn_id, bool post_sq,</div><div class='add'>+					bool poll_registers);</div><div class='add'>+int irdma_sc_srq_init(struct irdma_sc_srq *srq,</div><div class='add'>+		      struct irdma_srq_init_info *info);</div><div class='add'>+</div><div class='add'>+void sc_vsi_update_stats(struct irdma_sc_vsi *vsi);</div><div class='add'>+struct cqp_info {</div><div class='add'>+	union {</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_qp *qp;</div><div class='add'>+			struct irdma_create_qp_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} qp_create;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_qp *qp;</div><div class='add'>+			struct irdma_modify_qp_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} qp_modify;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_qp *qp;</div><div class='add'>+			u64 scratch;</div><div class='add'>+			bool remove_hash_idx;</div><div class='add'>+			bool ignore_mw_bnd;</div><div class='add'>+		} qp_destroy;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cq *cq;</div><div class='add'>+			u64 scratch;</div><div class='add'>+			bool check_overflow;</div><div class='add'>+		} cq_create;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cq *cq;</div><div class='add'>+			struct irdma_modify_cq_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} cq_modify;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cq *cq;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} cq_destroy;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_dev *dev;</div><div class='add'>+			struct irdma_allocate_stag_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} alloc_stag;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_dev *dev;</div><div class='add'>+			struct irdma_mw_alloc_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} mw_alloc;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_dev *dev;</div><div class='add'>+			struct irdma_reg_ns_stag_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} mr_reg_non_shared;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_dev *dev;</div><div class='add'>+			struct irdma_dealloc_stag_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} dealloc_stag;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			struct irdma_add_arp_cache_entry_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} add_arp_cache_entry;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			u64 scratch;</div><div class='add'>+			u16 arp_index;</div><div class='add'>+		} del_arp_cache_entry;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			struct irdma_local_mac_entry_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} add_local_mac_entry;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			u64 scratch;</div><div class='add'>+			u8 entry_idx;</div><div class='add'>+			u8 ignore_ref_count;</div><div class='add'>+		} del_local_mac_entry;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} alloc_local_mac_entry;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			struct irdma_cqp_manage_push_page_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} manage_push_page;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_dev *dev;</div><div class='add'>+			struct irdma_upload_context_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} qp_upload_context;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_dev *dev;</div><div class='add'>+			struct irdma_hmc_fcn_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} manage_hmc_pm;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_ceq *ceq;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} ceq_create;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_ceq *ceq;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} ceq_destroy;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_aeq *aeq;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} aeq_create;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_aeq *aeq;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} aeq_destroy;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_qp *qp;</div><div class='add'>+			struct irdma_qp_flush_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} qp_flush_wqes;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_qp *qp;</div><div class='add'>+			struct irdma_gen_ae_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} gen_ae;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			void *fpm_val_va;</div><div class='add'>+			u64 fpm_val_pa;</div><div class='add'>+			u8 hmc_fn_id;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} query_fpm_val;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			void *fpm_val_va;</div><div class='add'>+			u64 fpm_val_pa;</div><div class='add'>+			u8 hmc_fn_id;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} commit_fpm_val;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			struct irdma_apbvt_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} manage_apbvt_entry;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			struct irdma_qhash_table_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} manage_qhash_table_entry;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_dev *dev;</div><div class='add'>+			struct irdma_update_sds_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} update_pe_sds;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			struct irdma_sc_qp *qp;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} suspend_resume;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			struct irdma_ah_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} ah_create;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			struct irdma_ah_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} ah_destroy;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			struct irdma_mcast_grp_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} mc_create;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			struct irdma_mcast_grp_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} mc_destroy;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			struct irdma_mcast_grp_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} mc_modify;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			struct irdma_stats_inst_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} stats_manage;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			struct irdma_stats_gather_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} stats_gather;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			struct irdma_ws_node_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} ws_node;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			struct irdma_up_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} up_map;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_cqp *cqp;</div><div class='add'>+			struct irdma_dma_mem query_buff_mem;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} query_rdma;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_srq *srq;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} srq_create;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_srq *srq;</div><div class='add'>+			struct irdma_modify_srq_info info;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} srq_modify;</div><div class='add'>+</div><div class='add'>+		struct {</div><div class='add'>+			struct irdma_sc_srq *srq;</div><div class='add'>+			u64 scratch;</div><div class='add'>+		} srq_destroy;</div><div class='add'>+</div><div class='add'>+	} u;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct cqp_cmds_info {</div><div class='add'>+	struct list_head cqp_cmd_entry;</div><div class='add'>+	u8 cqp_cmd;</div><div class='add'>+	u8 post_sq;</div><div class='add'>+	struct cqp_info in;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+__le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch,</div><div class='add'>+					   u32 *wqe_idx);</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_cqp_get_next_send_wqe - get next wqe on cqp sq</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @scratch: private data for CQP WQE</div><div class='add'>+ */</div><div class='add'>+static inline __le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch)</div><div class='add'>+{</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+</div><div class='add'>+	return irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &amp;wqe_idx);</div><div class='add'>+}</div><div class='add'>+#endif /* IRDMA_TYPE_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/uda.c b/drivers/infiniband/hw/irdma/uda.c<br/>new file mode 100644<br/>index 000000000000..84051266d948<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/uda.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/uda.c</a></div><div class='hunk'>@@ -0,0 +1,265 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2016 - 2021 Intel Corporation */</div><div class='add'>+#include &lt;linux/etherdevice.h&gt;</div><div class='add'>+</div><div class='add'>+#include "osdep.h"</div><div class='add'>+#include "hmc.h"</div><div class='add'>+#include "defs.h"</div><div class='add'>+#include "type.h"</div><div class='add'>+#include "protos.h"</div><div class='add'>+#include "uda.h"</div><div class='add'>+#include "uda_d.h"</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_access_ah() - Create, modify or delete AH</div><div class='add'>+ * @cqp: struct for cqp hw</div><div class='add'>+ * @info: ah information</div><div class='add'>+ * @op: Operation</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_access_ah(struct irdma_sc_cqp *cqp, struct irdma_ah_info *info,</div><div class='add'>+		       u32 op, u64 scratch)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 qw1, qw2;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0, ether_addr_to_u64(info-&gt;mac_addr) &lt;&lt; 16);</div><div class='add'>+	qw1 = FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_PDINDEXLO, info-&gt;pd_idx) |</div><div class='add'>+	      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_TC, info-&gt;tc_tos) |</div><div class='add'>+	      FIELD_PREP(IRDMA_UDAQPC_VLANTAG, info-&gt;vlan_tag);</div><div class='add'>+</div><div class='add'>+	qw2 = FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ARPINDEX, info-&gt;dst_arpindex) |</div><div class='add'>+	      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_FLOWLABEL, info-&gt;flow_label) |</div><div class='add'>+	      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_HOPLIMIT, info-&gt;hop_ttl) |</div><div class='add'>+	      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_PDINDEXHI, info-&gt;pd_idx &gt;&gt; 16);</div><div class='add'>+</div><div class='add'>+	if (!info-&gt;ipv4_valid) {</div><div class='add'>+		set_64bit_val(wqe, 40,</div><div class='add'>+			      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR0, info-&gt;dest_ip_addr[0]) |</div><div class='add'>+			      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR1, info-&gt;dest_ip_addr[1]));</div><div class='add'>+		set_64bit_val(wqe, 32,</div><div class='add'>+			      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR2, info-&gt;dest_ip_addr[2]) |</div><div class='add'>+			      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info-&gt;dest_ip_addr[3]));</div><div class='add'>+</div><div class='add'>+		set_64bit_val(wqe, 56,</div><div class='add'>+			      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR0, info-&gt;src_ip_addr[0]) |</div><div class='add'>+			      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR1, info-&gt;src_ip_addr[1]));</div><div class='add'>+		set_64bit_val(wqe, 48,</div><div class='add'>+			      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR2, info-&gt;src_ip_addr[2]) |</div><div class='add'>+			      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info-&gt;src_ip_addr[3]));</div><div class='add'>+	} else {</div><div class='add'>+		set_64bit_val(wqe, 32,</div><div class='add'>+			      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info-&gt;dest_ip_addr[0]));</div><div class='add'>+</div><div class='add'>+		set_64bit_val(wqe, 48,</div><div class='add'>+			      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info-&gt;src_ip_addr[0]));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 8, qw1);</div><div class='add'>+	set_64bit_val(wqe, 16, qw2);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* need write block before writing WQE header */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(</div><div class='add'>+		wqe, 24,</div><div class='add'>+		FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_WQEVALID, cqp-&gt;polarity) |</div><div class='add'>+		FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_OPCODE, op) |</div><div class='add'>+		FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_DOLOOPBACKK, info-&gt;do_lpbk) |</div><div class='add'>+		FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_IPV4VALID, info-&gt;ipv4_valid) |</div><div class='add'>+		FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_AVIDX, info-&gt;ah_idx) |</div><div class='add'>+		FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_INSERTVLANTAG, info-&gt;insert_vlan_tag));</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: MANAGE_AH WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_create_mg_ctx() - create a mcg context</div><div class='add'>+ * @info: multicast group context info</div><div class='add'>+ */</div><div class='add'>+static void irdma_create_mg_ctx(struct irdma_mcast_grp_info *info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_mcast_grp_ctx_entry_info *entry_info = NULL;</div><div class='add'>+	u8 idx = 0; /* index in the array */</div><div class='add'>+	u8 ctx_idx = 0; /* index in the MG context */</div><div class='add'>+</div><div class='add'>+	memset(info-&gt;dma_mem_mc.va, 0, IRDMA_MAX_MGS_PER_CTX * sizeof(u64));</div><div class='add'>+</div><div class='add'>+	for (idx = 0; idx &lt; IRDMA_MAX_MGS_PER_CTX; idx++) {</div><div class='add'>+		entry_info = &amp;info-&gt;mg_ctx_info[idx];</div><div class='add'>+		if (entry_info-&gt;valid_entry) {</div><div class='add'>+			set_64bit_val((__le64 *)info-&gt;dma_mem_mc.va,</div><div class='add'>+				      ctx_idx * sizeof(u64),</div><div class='add'>+				      FIELD_PREP(IRDMA_UDA_MGCTX_DESTPORT, entry_info-&gt;dest_port) |</div><div class='add'>+				      FIELD_PREP(IRDMA_UDA_MGCTX_VALIDENT, entry_info-&gt;valid_entry) |</div><div class='add'>+				      FIELD_PREP(IRDMA_UDA_MGCTX_QPID, entry_info-&gt;qp_id));</div><div class='add'>+			ctx_idx++;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_access_mcast_grp() - Access mcast group based on op</div><div class='add'>+ * @cqp: Control QP</div><div class='add'>+ * @info: multicast group context info</div><div class='add'>+ * @op: operation to perform</div><div class='add'>+ * @scratch: u64 saved to be used during cqp completion</div><div class='add'>+ */</div><div class='add'>+int irdma_access_mcast_grp(struct irdma_sc_cqp *cqp,</div><div class='add'>+			   struct irdma_mcast_grp_info *info, u32 op,</div><div class='add'>+			   u64 scratch)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;mg_id &gt;= IRDMA_UDA_MAX_FSI_MGS) {</div><div class='add'>+		ibdev_dbg(to_ibdev(cqp-&gt;dev), "WQE: mg_id out of range\n");</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);</div><div class='add'>+	if (!wqe) {</div><div class='add'>+		ibdev_dbg(to_ibdev(cqp-&gt;dev), "WQE: ring full\n");</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_create_mg_ctx(info);</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 32, info-&gt;dma_mem_mc.pa);</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMA_UDA_CQPSQ_MG_VLANID, info-&gt;vlan_id) |</div><div class='add'>+		      FIELD_PREP(IRDMA_UDA_CQPSQ_QS_HANDLE, info-&gt;qs_handle));</div><div class='add'>+	set_64bit_val(wqe, 0, ether_addr_to_u64(info-&gt;dest_mac_addr));</div><div class='add'>+	set_64bit_val(wqe, 8,</div><div class='add'>+		      FIELD_PREP(IRDMA_UDA_CQPSQ_MG_HMC_FCN_ID, info-&gt;hmc_fcn_id));</div><div class='add'>+</div><div class='add'>+	if (!info-&gt;ipv4_valid) {</div><div class='add'>+		set_64bit_val(wqe, 56,</div><div class='add'>+			      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR0, info-&gt;dest_ip_addr[0]) |</div><div class='add'>+			      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR1, info-&gt;dest_ip_addr[1]));</div><div class='add'>+		set_64bit_val(wqe, 48,</div><div class='add'>+			      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR2, info-&gt;dest_ip_addr[2]) |</div><div class='add'>+			      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info-&gt;dest_ip_addr[3]));</div><div class='add'>+	} else {</div><div class='add'>+		set_64bit_val(wqe, 48,</div><div class='add'>+			      FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info-&gt;dest_ip_addr[0]));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* need write memory block before writing the WQE header. */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24,</div><div class='add'>+		      FIELD_PREP(IRDMA_UDA_CQPSQ_MG_WQEVALID, cqp-&gt;polarity) |</div><div class='add'>+		      FIELD_PREP(IRDMA_UDA_CQPSQ_MG_OPCODE, op) |</div><div class='add'>+		      FIELD_PREP(IRDMA_UDA_CQPSQ_MG_MGIDX, info-&gt;mg_id) |</div><div class='add'>+		      FIELD_PREP(IRDMA_UDA_CQPSQ_MG_VLANVALID, info-&gt;vlan_valid) |</div><div class='add'>+		      FIELD_PREP(IRDMA_UDA_CQPSQ_MG_IPV4VALID, info-&gt;ipv4_valid));</div><div class='add'>+</div><div class='add'>+	print_hex_dump_debug("WQE: MANAGE_MCG WQE", DUMP_PREFIX_OFFSET, 16, 8,</div><div class='add'>+			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);</div><div class='add'>+	print_hex_dump_debug("WQE: MCG_HOST CTX WQE", DUMP_PREFIX_OFFSET, 16,</div><div class='add'>+			     8, info-&gt;dma_mem_mc.va,</div><div class='add'>+			     IRDMA_MAX_MGS_PER_CTX * 8, false);</div><div class='add'>+	irdma_sc_cqp_post_sq(cqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_compare_mgs - Compares two multicast group structures</div><div class='add'>+ * @entry1: Multcast group info</div><div class='add'>+ * @entry2: Multcast group info in context</div><div class='add'>+ */</div><div class='add'>+static bool irdma_compare_mgs(struct irdma_mcast_grp_ctx_entry_info *entry1,</div><div class='add'>+			      struct irdma_mcast_grp_ctx_entry_info *entry2)</div><div class='add'>+{</div><div class='add'>+	if (entry1-&gt;dest_port == entry2-&gt;dest_port &amp;&amp;</div><div class='add'>+	    entry1-&gt;qp_id == entry2-&gt;qp_id)</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_add_mcast_grp - Allocates mcast group entry in ctx</div><div class='add'>+ * @ctx: Multcast group context</div><div class='add'>+ * @mg: Multcast group info</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_add_mcast_grp(struct irdma_mcast_grp_info *ctx,</div><div class='add'>+			   struct irdma_mcast_grp_ctx_entry_info *mg)</div><div class='add'>+{</div><div class='add'>+	u32 idx;</div><div class='add'>+	bool free_entry_found = false;</div><div class='add'>+	u32 free_entry_idx = 0;</div><div class='add'>+</div><div class='add'>+	/* find either an identical or a free entry for a multicast group */</div><div class='add'>+	for (idx = 0; idx &lt; IRDMA_MAX_MGS_PER_CTX; idx++) {</div><div class='add'>+		if (ctx-&gt;mg_ctx_info[idx].valid_entry) {</div><div class='add'>+			if (irdma_compare_mgs(&amp;ctx-&gt;mg_ctx_info[idx], mg)) {</div><div class='add'>+				ctx-&gt;mg_ctx_info[idx].use_cnt++;</div><div class='add'>+				return 0;</div><div class='add'>+			}</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+		if (!free_entry_found) {</div><div class='add'>+			free_entry_found = true;</div><div class='add'>+			free_entry_idx = idx;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (free_entry_found) {</div><div class='add'>+		ctx-&gt;mg_ctx_info[free_entry_idx] = *mg;</div><div class='add'>+		ctx-&gt;mg_ctx_info[free_entry_idx].valid_entry = true;</div><div class='add'>+		ctx-&gt;mg_ctx_info[free_entry_idx].use_cnt = 1;</div><div class='add'>+		ctx-&gt;no_of_mgs++;</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_del_mcast_grp - Delete mcast group</div><div class='add'>+ * @ctx: Multcast group context</div><div class='add'>+ * @mg: Multcast group info</div><div class='add'>+ *</div><div class='add'>+ * Finds and removes a specific mulicast group from context, all</div><div class='add'>+ * parameters must match to remove a multicast group.</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_del_mcast_grp(struct irdma_mcast_grp_info *ctx,</div><div class='add'>+			   struct irdma_mcast_grp_ctx_entry_info *mg)</div><div class='add'>+{</div><div class='add'>+	u32 idx;</div><div class='add'>+</div><div class='add'>+	/* find an entry in multicast group context */</div><div class='add'>+	for (idx = 0; idx &lt; IRDMA_MAX_MGS_PER_CTX; idx++) {</div><div class='add'>+		if (!ctx-&gt;mg_ctx_info[idx].valid_entry)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		if (irdma_compare_mgs(mg, &amp;ctx-&gt;mg_ctx_info[idx])) {</div><div class='add'>+			ctx-&gt;mg_ctx_info[idx].use_cnt--;</div><div class='add'>+</div><div class='add'>+			if (!ctx-&gt;mg_ctx_info[idx].use_cnt) {</div><div class='add'>+				ctx-&gt;mg_ctx_info[idx].valid_entry = false;</div><div class='add'>+				ctx-&gt;no_of_mgs--;</div><div class='add'>+				/* Remove gap if element was not the last */</div><div class='add'>+				if (idx != ctx-&gt;no_of_mgs &amp;&amp;</div><div class='add'>+				    ctx-&gt;no_of_mgs &gt; 0) {</div><div class='add'>+					memcpy(&amp;ctx-&gt;mg_ctx_info[idx],</div><div class='add'>+					       &amp;ctx-&gt;mg_ctx_info[ctx-&gt;no_of_mgs - 1],</div><div class='add'>+					       sizeof(ctx-&gt;mg_ctx_info[idx]));</div><div class='add'>+					ctx-&gt;mg_ctx_info[ctx-&gt;no_of_mgs - 1].valid_entry = false;</div><div class='add'>+				}</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return -EINVAL;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/uda.h b/drivers/infiniband/hw/irdma/uda.h<br/>new file mode 100644<br/>index 000000000000..27b8701cf21b<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/uda.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/uda.h</a></div><div class='hunk'>@@ -0,0 +1,87 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2016 - 2021 Intel Corporation */</div><div class='add'>+#ifndef IRDMA_UDA_H</div><div class='add'>+#define IRDMA_UDA_H</div><div class='add'>+</div><div class='add'>+#define IRDMA_UDA_MAX_FSI_MGS	4096</div><div class='add'>+#define IRDMA_UDA_MAX_PFS	16</div><div class='add'>+#define IRDMA_UDA_MAX_VFS	128</div><div class='add'>+</div><div class='add'>+struct irdma_sc_cqp;</div><div class='add'>+</div><div class='add'>+struct irdma_ah_info {</div><div class='add'>+	struct irdma_sc_vsi *vsi;</div><div class='add'>+	u32 pd_idx;</div><div class='add'>+	u32 dst_arpindex;</div><div class='add'>+	u32 dest_ip_addr[4];</div><div class='add'>+	u32 src_ip_addr[4];</div><div class='add'>+	u32 flow_label;</div><div class='add'>+	u32 ah_idx;</div><div class='add'>+	u16 vlan_tag;</div><div class='add'>+	u8 insert_vlan_tag;</div><div class='add'>+	u8 tc_tos;</div><div class='add'>+	u8 hop_ttl;</div><div class='add'>+	u8 mac_addr[ETH_ALEN];</div><div class='add'>+	bool ah_valid:1;</div><div class='add'>+	bool ipv4_valid:1;</div><div class='add'>+	bool do_lpbk:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_sc_ah {</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	struct irdma_ah_info ah_info;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+int irdma_sc_add_mcast_grp(struct irdma_mcast_grp_info *ctx,</div><div class='add'>+			   struct irdma_mcast_grp_ctx_entry_info *mg);</div><div class='add'>+int irdma_sc_del_mcast_grp(struct irdma_mcast_grp_info *ctx,</div><div class='add'>+			   struct irdma_mcast_grp_ctx_entry_info *mg);</div><div class='add'>+int irdma_sc_access_ah(struct irdma_sc_cqp *cqp, struct irdma_ah_info *info,</div><div class='add'>+		       u32 op, u64 scratch);</div><div class='add'>+int irdma_access_mcast_grp(struct irdma_sc_cqp *cqp,</div><div class='add'>+			   struct irdma_mcast_grp_info *info, u32 op,</div><div class='add'>+			   u64 scratch);</div><div class='add'>+</div><div class='add'>+static inline void irdma_sc_init_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)</div><div class='add'>+{</div><div class='add'>+	ah-&gt;dev = dev;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int irdma_sc_create_ah(struct irdma_sc_cqp *cqp,</div><div class='add'>+				     struct irdma_ah_info *info, u64 scratch)</div><div class='add'>+{</div><div class='add'>+	return irdma_sc_access_ah(cqp, info, IRDMA_CQP_OP_CREATE_ADDR_HANDLE,</div><div class='add'>+				  scratch);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int irdma_sc_destroy_ah(struct irdma_sc_cqp *cqp,</div><div class='add'>+				      struct irdma_ah_info *info, u64 scratch)</div><div class='add'>+{</div><div class='add'>+	return irdma_sc_access_ah(cqp, info, IRDMA_CQP_OP_DESTROY_ADDR_HANDLE,</div><div class='add'>+				  scratch);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int irdma_sc_create_mcast_grp(struct irdma_sc_cqp *cqp,</div><div class='add'>+					    struct irdma_mcast_grp_info *info,</div><div class='add'>+					    u64 scratch)</div><div class='add'>+{</div><div class='add'>+	return irdma_access_mcast_grp(cqp, info, IRDMA_CQP_OP_CREATE_MCAST_GRP,</div><div class='add'>+				      scratch);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int irdma_sc_modify_mcast_grp(struct irdma_sc_cqp *cqp,</div><div class='add'>+					    struct irdma_mcast_grp_info *info,</div><div class='add'>+					    u64 scratch)</div><div class='add'>+{</div><div class='add'>+	return irdma_access_mcast_grp(cqp, info, IRDMA_CQP_OP_MODIFY_MCAST_GRP,</div><div class='add'>+				      scratch);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int irdma_sc_destroy_mcast_grp(struct irdma_sc_cqp *cqp,</div><div class='add'>+					     struct irdma_mcast_grp_info *info,</div><div class='add'>+					     u64 scratch)</div><div class='add'>+{</div><div class='add'>+	return irdma_access_mcast_grp(cqp, info, IRDMA_CQP_OP_DESTROY_MCAST_GRP,</div><div class='add'>+				      scratch);</div><div class='add'>+}</div><div class='add'>+#endif /* IRDMA_UDA_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/uda_d.h b/drivers/infiniband/hw/irdma/uda_d.h<br/>new file mode 100644<br/>index 000000000000..4fb4daa20722<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/uda_d.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/uda_d.h</a></div><div class='hunk'>@@ -0,0 +1,127 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2016 - 2021 Intel Corporation */</div><div class='add'>+#ifndef IRDMA_UDA_D_H</div><div class='add'>+#define IRDMA_UDA_D_H</div><div class='add'>+</div><div class='add'>+/* L4 packet type */</div><div class='add'>+#define IRDMA_E_UDA_SQ_L4T_UNKNOWN	0</div><div class='add'>+#define IRDMA_E_UDA_SQ_L4T_TCP		1</div><div class='add'>+#define IRDMA_E_UDA_SQ_L4T_SCTP		2</div><div class='add'>+#define IRDMA_E_UDA_SQ_L4T_UDP		3</div><div class='add'>+</div><div class='add'>+/* Inner IP header type */</div><div class='add'>+#define IRDMA_E_UDA_SQ_IIPT_UNKNOWN		0</div><div class='add'>+#define IRDMA_E_UDA_SQ_IIPT_IPV6		1</div><div class='add'>+#define IRDMA_E_UDA_SQ_IIPT_IPV4_NO_CSUM	2</div><div class='add'>+#define IRDMA_E_UDA_SQ_IIPT_IPV4_CSUM		3</div><div class='add'>+#define IRDMA_UDA_QPSQ_PUSHWQE BIT_ULL(56)</div><div class='add'>+#define IRDMA_UDA_QPSQ_INLINEDATAFLAG BIT_ULL(57)</div><div class='add'>+#define IRDMA_UDA_QPSQ_INLINEDATALEN GENMASK_ULL(55, 48)</div><div class='add'>+#define IRDMA_UDA_QPSQ_ADDFRAGCNT GENMASK_ULL(41, 38)</div><div class='add'>+#define IRDMA_UDA_QPSQ_IPFRAGFLAGS GENMASK_ULL(43, 42)</div><div class='add'>+#define IRDMA_UDA_QPSQ_NOCHECKSUM BIT_ULL(45)</div><div class='add'>+#define IRDMA_UDA_QPSQ_AHIDXVALID BIT_ULL(46)</div><div class='add'>+#define IRDMA_UDA_QPSQ_LOCAL_FENCE BIT_ULL(61)</div><div class='add'>+#define IRDMA_UDA_QPSQ_AHIDX GENMASK_ULL(16, 0)</div><div class='add'>+#define IRDMA_UDA_QPSQ_PROTOCOL GENMASK_ULL(23, 16)</div><div class='add'>+#define IRDMA_UDA_QPSQ_EXTHDRLEN GENMASK_ULL(40, 32)</div><div class='add'>+#define IRDMA_UDA_QPSQ_MULTICAST BIT_ULL(63)</div><div class='add'>+#define IRDMA_UDA_QPSQ_MACLEN GENMASK_ULL(62, 56)</div><div class='add'>+#define IRDMA_UDA_QPSQ_MACLEN_LINE 2</div><div class='add'>+#define IRDMA_UDA_QPSQ_IPLEN GENMASK_ULL(54, 48)</div><div class='add'>+#define IRDMA_UDA_QPSQ_IPLEN_LINE 2</div><div class='add'>+#define IRDMA_UDA_QPSQ_L4T GENMASK_ULL(31, 30)</div><div class='add'>+#define IRDMA_UDA_QPSQ_L4T_LINE 2</div><div class='add'>+#define IRDMA_UDA_QPSQ_IIPT GENMASK_ULL(29, 28)</div><div class='add'>+#define IRDMA_UDA_QPSQ_IIPT_LINE 2</div><div class='add'>+</div><div class='add'>+#define IRDMA_UDA_QPSQ_DO_LPB_LINE 3</div><div class='add'>+#define IRDMA_UDA_QPSQ_FWD_PROG_CONFIRM BIT_ULL(45)</div><div class='add'>+#define IRDMA_UDA_QPSQ_FWD_PROG_CONFIRM_LINE 3</div><div class='add'>+#define IRDMA_UDA_QPSQ_IMMDATA GENMASK_ULL(63, 0)</div><div class='add'>+</div><div class='add'>+/* Byte Offset 0 */</div><div class='add'>+#define IRDMA_UDAQPC_IPV4_M BIT_ULL(3)</div><div class='add'>+#define IRDMA_UDAQPC_INSERTVLANTAG BIT_ULL(5)</div><div class='add'>+#define IRDMA_UDAQPC_ISQP1 BIT_ULL(6)</div><div class='add'>+</div><div class='add'>+#define IRDMA_UDAQPC_ECNENABLE BIT_ULL(14)</div><div class='add'>+#define IRDMA_UDAQPC_PDINDEXHI GENMASK_ULL(21, 20)</div><div class='add'>+#define IRDMA_UDAQPC_DCTCPENABLE BIT_ULL(25)</div><div class='add'>+</div><div class='add'>+#define IRDMA_UDAQPC_RCVTPHEN IRDMAQPC_RCVTPHEN</div><div class='add'>+#define IRDMA_UDAQPC_XMITTPHEN IRDMAQPC_XMITTPHEN</div><div class='add'>+#define IRDMA_UDAQPC_RQTPHEN IRDMAQPC_RQTPHEN</div><div class='add'>+#define IRDMA_UDAQPC_SQTPHEN IRDMAQPC_SQTPHEN</div><div class='add'>+#define IRDMA_UDAQPC_PPIDX IRDMAQPC_PPIDX</div><div class='add'>+#define IRDMA_UDAQPC_PMENA IRDMAQPC_PMENA</div><div class='add'>+#define IRDMA_UDAQPC_INSERTTAG2 BIT_ULL(11)</div><div class='add'>+#define IRDMA_UDAQPC_INSERTTAG3 BIT_ULL(14)</div><div class='add'>+</div><div class='add'>+#define IRDMA_UDAQPC_RQSIZE IRDMAQPC_RQSIZE</div><div class='add'>+#define IRDMA_UDAQPC_SQSIZE IRDMAQPC_SQSIZE</div><div class='add'>+#define IRDMA_UDAQPC_TXCQNUM IRDMAQPC_TXCQNUM</div><div class='add'>+#define IRDMA_UDAQPC_RXCQNUM IRDMAQPC_RXCQNUM</div><div class='add'>+#define IRDMA_UDAQPC_QPCOMPCTX IRDMAQPC_QPCOMPCTX</div><div class='add'>+#define IRDMA_UDAQPC_SQTPHVAL IRDMAQPC_SQTPHVAL</div><div class='add'>+#define IRDMA_UDAQPC_RQTPHVAL IRDMAQPC_RQTPHVAL</div><div class='add'>+#define IRDMA_UDAQPC_QSHANDLE IRDMAQPC_QSHANDLE</div><div class='add'>+#define IRDMA_UDAQPC_RQHDRRINGBUFSIZE GENMASK_ULL(49, 48)</div><div class='add'>+#define IRDMA_UDAQPC_SQHDRRINGBUFSIZE GENMASK_ULL(33, 32)</div><div class='add'>+#define IRDMA_UDAQPC_PRIVILEGEENABLE BIT_ULL(25)</div><div class='add'>+#define IRDMA_UDAQPC_USE_STATISTICS_INSTANCE BIT_ULL(26)</div><div class='add'>+#define IRDMA_UDAQPC_STATISTICS_INSTANCE_INDEX GENMASK_ULL(6, 0)</div><div class='add'>+#define IRDMA_UDAQPC_PRIVHDRGENENABLE BIT_ULL(0)</div><div class='add'>+#define IRDMA_UDAQPC_RQHDRSPLITENABLE BIT_ULL(3)</div><div class='add'>+#define IRDMA_UDAQPC_RQHDRRINGBUFENABLE BIT_ULL(2)</div><div class='add'>+#define IRDMA_UDAQPC_SQHDRRINGBUFENABLE BIT_ULL(1)</div><div class='add'>+#define IRDMA_UDAQPC_IPID GENMASK_ULL(47, 32)</div><div class='add'>+#define IRDMA_UDAQPC_SNDMSS GENMASK_ULL(29, 16)</div><div class='add'>+#define IRDMA_UDAQPC_VLANTAG GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_PDINDEXHI GENMASK_ULL(27, 20)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_PDINDEXLO GENMASK_ULL(63, 48)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_SRCMACADDRINDEX GENMASK_ULL(29, 24)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_ARPINDEX GENMASK_ULL(63, 48)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_TC GENMASK_ULL(39, 32)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_HOPLIMIT GENMASK_ULL(39, 32)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_FLOWLABEL GENMASK_ULL(19, 0)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_ADDR0 GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_ADDR1 GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_ADDR2 GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_ADDR3 GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_WQEVALID BIT_ULL(63)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_OPCODE GENMASK_ULL(37, 32)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_DOLOOPBACKK BIT_ULL(62)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_IPV4VALID BIT_ULL(59)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_AVIDX GENMASK_ULL(23, 0)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MAV_INSERTVLANTAG BIT_ULL(60)</div><div class='add'>+#define IRDMA_UDA_MGCTX_VFFLAG BIT_ULL(29)</div><div class='add'>+#define IRDMA_UDA_MGCTX_DESTPORT GENMASK_ULL(47, 32)</div><div class='add'>+#define IRDMA_UDA_MGCTX_VFID GENMASK_ULL(28, 22)</div><div class='add'>+#define IRDMA_UDA_MGCTX_VALIDENT BIT_ULL(31)</div><div class='add'>+#define IRDMA_UDA_MGCTX_PFID GENMASK_ULL(21, 18)</div><div class='add'>+#define IRDMA_UDA_MGCTX_FLAGIGNOREDPORT BIT_ULL(30)</div><div class='add'>+#define IRDMA_UDA_MGCTX_QPID GENMASK_ULL(17, 0)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MG_WQEVALID BIT_ULL(63)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MG_OPCODE GENMASK_ULL(37, 32)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MG_MGIDX GENMASK_ULL(12, 0)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MG_IPV4VALID BIT_ULL(60)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MG_VLANVALID BIT_ULL(59)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MG_HMC_FCN_ID GENMASK_ULL(5, 0)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_MG_VLANID GENMASK_ULL(43, 32)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_QS_HANDLE GENMASK_ULL(9, 0)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_QHASH_QPN GENMASK_ULL(49, 32)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_QHASH_ BIT_ULL(0)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_QHASH_SRC_PORT GENMASK_ULL(31, 16)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_QHASH_DEST_PORT GENMASK_ULL(15, 0)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_QHASH_ADDR0 GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_QHASH_ADDR1 GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_QHASH_ADDR2 GENMASK_ULL(63, 32)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_QHASH_ADDR3 GENMASK_ULL(31, 0)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_QHASH_WQEVALID BIT_ULL(63)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_QHASH_OPCODE GENMASK_ULL(37, 32)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_QHASH_MANAGE GENMASK_ULL(62, 61)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_QHASH_IPV4VALID GENMASK_ULL(60, 60)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_QHASH_LANFWD GENMASK_ULL(59, 59)</div><div class='add'>+#define IRDMA_UDA_CQPSQ_QHASH_ENTRYTYPE GENMASK_ULL(44, 42)</div><div class='add'>+#endif /* IRDMA_UDA_D_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/uk.c b/drivers/infiniband/hw/irdma/uk.c<br/>new file mode 100644<br/>index 000000000000..f0846b800913<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/uk.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/uk.c</a></div><div class='hunk'>@@ -0,0 +1,1930 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#include "osdep.h"</div><div class='add'>+#include "defs.h"</div><div class='add'>+#include "user.h"</div><div class='add'>+#include "irdma.h"</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_set_fragment - set fragment in wqe</div><div class='add'>+ * @wqe: wqe for setting fragment</div><div class='add'>+ * @offset: offset value</div><div class='add'>+ * @sge: sge length and stag</div><div class='add'>+ * @valid: The wqe valid</div><div class='add'>+ */</div><div class='add'>+static void irdma_set_fragment(__le64 *wqe, u32 offset, struct ib_sge *sge,</div><div class='add'>+			       u8 valid)</div><div class='add'>+{</div><div class='add'>+	if (sge) {</div><div class='add'>+		set_64bit_val(wqe, offset,</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge-&gt;addr));</div><div class='add'>+		set_64bit_val(wqe, offset + 8,</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_VALID, valid) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge-&gt;length) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge-&gt;lkey));</div><div class='add'>+	} else {</div><div class='add'>+		set_64bit_val(wqe, offset, 0);</div><div class='add'>+		set_64bit_val(wqe, offset + 8,</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_VALID, valid));</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_set_fragment_gen_1 - set fragment in wqe</div><div class='add'>+ * @wqe: wqe for setting fragment</div><div class='add'>+ * @offset: offset value</div><div class='add'>+ * @sge: sge length and stag</div><div class='add'>+ * @valid: wqe valid flag</div><div class='add'>+ */</div><div class='add'>+static void irdma_set_fragment_gen_1(__le64 *wqe, u32 offset,</div><div class='add'>+				     struct ib_sge *sge, u8 valid)</div><div class='add'>+{</div><div class='add'>+	if (sge) {</div><div class='add'>+		set_64bit_val(wqe, offset,</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge-&gt;addr));</div><div class='add'>+		set_64bit_val(wqe, offset + 8,</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge-&gt;length) |</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge-&gt;lkey));</div><div class='add'>+	} else {</div><div class='add'>+		set_64bit_val(wqe, offset, 0);</div><div class='add'>+		set_64bit_val(wqe, offset + 8, 0);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_nop_1 - insert a NOP wqe</div><div class='add'>+ * @qp: hw qp ptr</div><div class='add'>+ */</div><div class='add'>+static int irdma_nop_1(struct irdma_qp_uk *qp)</div><div class='add'>+{</div><div class='add'>+	u64 hdr;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+	bool signaled = false;</div><div class='add'>+</div><div class='add'>+	if (!qp-&gt;sq_ring.head)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	wqe_idx = IRDMA_RING_CURRENT_HEAD(qp-&gt;sq_ring);</div><div class='add'>+	wqe = qp-&gt;sq_base[wqe_idx].elem;</div><div class='add'>+</div><div class='add'>+	qp-&gt;sq_wrtrk_array[wqe_idx].quanta = IRDMA_QP_WQE_MIN_QUANTA;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0, 0);</div><div class='add'>+	set_64bit_val(wqe, 8, 0);</div><div class='add'>+	set_64bit_val(wqe, 16, 0);</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;swqe_polarity);</div><div class='add'>+</div><div class='add'>+	/* make sure WQE is written before valid bit is set */</div><div class='add'>+	dma_wmb();</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_clr_wqes - clear next 128 sq entries</div><div class='add'>+ * @qp: hw qp ptr</div><div class='add'>+ * @qp_wqe_idx: wqe_idx</div><div class='add'>+ */</div><div class='add'>+void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp_quanta *sq;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+</div><div class='add'>+	if (!(qp_wqe_idx &amp; 0x7F)) {</div><div class='add'>+		wqe_idx = (qp_wqe_idx + 128) % qp-&gt;sq_ring.size;</div><div class='add'>+		sq = qp-&gt;sq_base + wqe_idx;</div><div class='add'>+		if (wqe_idx)</div><div class='add'>+			memset(sq, qp-&gt;swqe_polarity ? 0 : 0xFF,</div><div class='add'>+			       128 * sizeof(*sq));</div><div class='add'>+		else</div><div class='add'>+			memset(sq, qp-&gt;swqe_polarity ? 0xFF : 0,</div><div class='add'>+			       128 * sizeof(*sq));</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_qp_post_wr - ring doorbell</div><div class='add'>+ * @qp: hw qp ptr</div><div class='add'>+ */</div><div class='add'>+void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp)</div><div class='add'>+{</div><div class='add'>+	dma_wmb();</div><div class='add'>+	writel(qp-&gt;qp_id, qp-&gt;wqe_alloc_db);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_qp_get_next_send_wqe - pad with NOP if needed, return where next WR should go</div><div class='add'>+ * @qp: hw qp ptr</div><div class='add'>+ * @wqe_idx: return wqe index</div><div class='add'>+ * @quanta: size of WR in quanta</div><div class='add'>+ * @total_size: size of WR in bytes</div><div class='add'>+ * @info: info on WR</div><div class='add'>+ */</div><div class='add'>+__le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,</div><div class='add'>+				   u16 quanta, u32 total_size,</div><div class='add'>+				   struct irdma_post_sq_info *info)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	__le64 *wqe_0 = NULL;</div><div class='add'>+	u16 avail_quanta;</div><div class='add'>+	u16 i;</div><div class='add'>+</div><div class='add'>+	avail_quanta = qp-&gt;uk_attrs-&gt;max_hw_sq_chunk -</div><div class='add'>+		       (IRDMA_RING_CURRENT_HEAD(qp-&gt;sq_ring) %</div><div class='add'>+		       qp-&gt;uk_attrs-&gt;max_hw_sq_chunk);</div><div class='add'>+	if (quanta &lt;= avail_quanta) {</div><div class='add'>+		/* WR fits in current chunk */</div><div class='add'>+		if (quanta &gt; IRDMA_SQ_RING_FREE_QUANTA(qp-&gt;sq_ring))</div><div class='add'>+			return NULL;</div><div class='add'>+	} else {</div><div class='add'>+		/* Need to pad with NOP */</div><div class='add'>+		if (quanta + avail_quanta &gt;</div><div class='add'>+			IRDMA_SQ_RING_FREE_QUANTA(qp-&gt;sq_ring))</div><div class='add'>+			return NULL;</div><div class='add'>+</div><div class='add'>+		for (i = 0; i &lt; avail_quanta; i++) {</div><div class='add'>+			irdma_nop_1(qp);</div><div class='add'>+			IRDMA_RING_MOVE_HEAD_NOCHECK(qp-&gt;sq_ring);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*wqe_idx = IRDMA_RING_CURRENT_HEAD(qp-&gt;sq_ring);</div><div class='add'>+	if (!*wqe_idx)</div><div class='add'>+		qp-&gt;swqe_polarity = !qp-&gt;swqe_polarity;</div><div class='add'>+</div><div class='add'>+	IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp-&gt;sq_ring, quanta);</div><div class='add'>+</div><div class='add'>+	wqe = qp-&gt;sq_base[*wqe_idx].elem;</div><div class='add'>+	if (qp-&gt;uk_attrs-&gt;hw_rev == IRDMA_GEN_1 &amp;&amp; quanta == 1 &amp;&amp;</div><div class='add'>+	    (IRDMA_RING_CURRENT_HEAD(qp-&gt;sq_ring) &amp; 1)) {</div><div class='add'>+		wqe_0 = qp-&gt;sq_base[IRDMA_RING_CURRENT_HEAD(qp-&gt;sq_ring)].elem;</div><div class='add'>+		wqe_0[3] = cpu_to_le64(FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;swqe_polarity ? 0 : 1));</div><div class='add'>+	}</div><div class='add'>+	qp-&gt;sq_wrtrk_array[*wqe_idx].wrid = info-&gt;wr_id;</div><div class='add'>+	qp-&gt;sq_wrtrk_array[*wqe_idx].wr_len = total_size;</div><div class='add'>+	qp-&gt;sq_wrtrk_array[*wqe_idx].quanta = quanta;</div><div class='add'>+	qp-&gt;sq_wrtrk_array[*wqe_idx].signaled = info-&gt;signaled;</div><div class='add'>+</div><div class='add'>+	return wqe;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+__le64 *irdma_srq_get_next_recv_wqe(struct irdma_srq_uk *srq, u32 *wqe_idx)</div><div class='add'>+{</div><div class='add'>+	int ret_code;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+</div><div class='add'>+	if (IRDMA_RING_FULL_ERR(srq-&gt;srq_ring))</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	IRDMA_ATOMIC_RING_MOVE_HEAD(srq-&gt;srq_ring, *wqe_idx, ret_code);</div><div class='add'>+	if (ret_code)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	if (!*wqe_idx)</div><div class='add'>+		srq-&gt;srwqe_polarity = !srq-&gt;srwqe_polarity;</div><div class='add'>+	/* rq_wqe_size_multiplier is no of 32 byte quanta in one rq wqe */</div><div class='add'>+	wqe = srq-&gt;srq_base[*wqe_idx * (srq-&gt;wqe_size_multiplier)].elem;</div><div class='add'>+</div><div class='add'>+	return wqe;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_qp_get_next_recv_wqe - get next qp's rcv wqe</div><div class='add'>+ * @qp: hw qp ptr</div><div class='add'>+ * @wqe_idx: return wqe index</div><div class='add'>+ */</div><div class='add'>+__le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	int ret_code;</div><div class='add'>+</div><div class='add'>+	if (IRDMA_RING_FULL_ERR(qp-&gt;rq_ring))</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	IRDMA_ATOMIC_RING_MOVE_HEAD(qp-&gt;rq_ring, *wqe_idx, ret_code);</div><div class='add'>+	if (ret_code)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	if (!*wqe_idx)</div><div class='add'>+		qp-&gt;rwqe_polarity = !qp-&gt;rwqe_polarity;</div><div class='add'>+	/* rq_wqe_size_multiplier is no of 32 byte quanta in one rq wqe */</div><div class='add'>+	wqe = qp-&gt;rq_base[*wqe_idx * qp-&gt;rq_wqe_size_multiplier].elem;</div><div class='add'>+</div><div class='add'>+	return wqe;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_rdma_write - rdma write operation</div><div class='add'>+ * @qp: hw qp ptr</div><div class='add'>+ * @info: post sq information</div><div class='add'>+ * @post_sq: flag to post sq</div><div class='add'>+ */</div><div class='add'>+int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,</div><div class='add'>+			bool post_sq)</div><div class='add'>+{</div><div class='add'>+	u64 hdr;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_rdma_write *op_info;</div><div class='add'>+	u32 i, wqe_idx;</div><div class='add'>+	u32 total_size = 0, byte_off;</div><div class='add'>+	int ret_code;</div><div class='add'>+	u32 frag_cnt, addl_frag_cnt;</div><div class='add'>+	bool read_fence = false;</div><div class='add'>+	u16 quanta;</div><div class='add'>+</div><div class='add'>+	op_info = &amp;info-&gt;op.rdma_write;</div><div class='add'>+	if (op_info-&gt;num_lo_sges &gt; qp-&gt;max_sq_frag_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; op_info-&gt;num_lo_sges; i++)</div><div class='add'>+		total_size += op_info-&gt;lo_sg_list[i].length;</div><div class='add'>+</div><div class='add'>+	read_fence |= info-&gt;read_fence;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;imm_data_valid)</div><div class='add'>+		frag_cnt = op_info-&gt;num_lo_sges + 1;</div><div class='add'>+	else</div><div class='add'>+		frag_cnt = op_info-&gt;num_lo_sges;</div><div class='add'>+	addl_frag_cnt = frag_cnt &gt; 1 ? (frag_cnt - 1) : 0;</div><div class='add'>+	ret_code = irdma_fragcnt_to_quanta_sq(frag_cnt, &amp;quanta);</div><div class='add'>+	if (ret_code)</div><div class='add'>+		return ret_code;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_qp_get_next_send_wqe(qp, &amp;wqe_idx, quanta, total_size,</div><div class='add'>+					 info);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	irdma_clr_wqes(qp, wqe_idx);</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info-&gt;rem_addr.addr));</div><div class='add'>+</div><div class='add'>+	if (info-&gt;imm_data_valid) {</div><div class='add'>+		set_64bit_val(wqe, 0,</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_IMMDATA, info-&gt;imm_data));</div><div class='add'>+		i = 0;</div><div class='add'>+	} else {</div><div class='add'>+		qp-&gt;wqe_ops.iw_set_fragment(wqe, 0,</div><div class='add'>+					    op_info-&gt;lo_sg_list,</div><div class='add'>+					    qp-&gt;swqe_polarity);</div><div class='add'>+		i = 1;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (byte_off = 32; i &lt; op_info-&gt;num_lo_sges; i++) {</div><div class='add'>+		qp-&gt;wqe_ops.iw_set_fragment(wqe, byte_off,</div><div class='add'>+					    &amp;op_info-&gt;lo_sg_list[i],</div><div class='add'>+					    qp-&gt;swqe_polarity);</div><div class='add'>+		byte_off += 16;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* if not an odd number set valid bit in next fragment */</div><div class='add'>+	if (qp-&gt;uk_attrs-&gt;hw_rev &gt;= IRDMA_GEN_2 &amp;&amp; !(frag_cnt &amp; 0x01) &amp;&amp;</div><div class='add'>+	    frag_cnt) {</div><div class='add'>+		qp-&gt;wqe_ops.iw_set_fragment(wqe, byte_off, NULL,</div><div class='add'>+					    qp-&gt;swqe_polarity);</div><div class='add'>+		if (qp-&gt;uk_attrs-&gt;hw_rev == IRDMA_GEN_2)</div><div class='add'>+			++addl_frag_cnt;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info-&gt;rem_addr.lkey) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_OPCODE, info-&gt;op_type) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info-&gt;imm_data_valid) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_REPORTRTT, info-&gt;report_rtt) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info-&gt;local_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info-&gt;signaled) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;swqe_polarity);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_uk_qp_post_wr(qp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_atomic_fetch_add - atomic fetch and add operation</div><div class='add'>+ * @qp: hw qp ptr</div><div class='add'>+ * @info: post sq information</div><div class='add'>+ * @post_sq: flag to post sq</div><div class='add'>+ */</div><div class='add'>+int irdma_uk_atomic_fetch_add(struct irdma_qp_uk *qp,</div><div class='add'>+			      struct irdma_post_sq_info *info, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_atomic_fetch_add *op_info;</div><div class='add'>+	u32 total_size = 0;</div><div class='add'>+	u16 quanta = 2;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	op_info = &amp;info-&gt;op.atomic_fetch_add;</div><div class='add'>+	wqe = irdma_qp_get_next_send_wqe(qp, &amp;wqe_idx, quanta, total_size,</div><div class='add'>+					 info);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0, op_info-&gt;tagged_offset);</div><div class='add'>+	set_64bit_val(wqe, 8,</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_STAG, op_info-&gt;stag));</div><div class='add'>+	set_64bit_val(wqe, 16, op_info-&gt;remote_tagged_offset);</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_REMOTE_STAG, op_info-&gt;remote_stag) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_ATOMIC_FETCH_ADD) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_READFENCE, info-&gt;read_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info-&gt;local_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info-&gt;signaled) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;swqe_polarity);</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 32, op_info-&gt;fetch_add_data_bytes);</div><div class='add'>+	set_64bit_val(wqe, 40, 0);</div><div class='add'>+	set_64bit_val(wqe, 48, 0);</div><div class='add'>+	set_64bit_val(wqe, 56,</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;swqe_polarity));</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_uk_qp_post_wr(qp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_atomic_compare_swap - atomic compare and swap operation</div><div class='add'>+ * @qp: hw qp ptr</div><div class='add'>+ * @info: post sq information</div><div class='add'>+ * @post_sq: flag to post sq</div><div class='add'>+ */</div><div class='add'>+int irdma_uk_atomic_compare_swap(struct irdma_qp_uk *qp,</div><div class='add'>+				 struct irdma_post_sq_info *info, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_atomic_compare_swap *op_info;</div><div class='add'>+	u32 total_size = 0;</div><div class='add'>+	u16 quanta = 2;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	op_info = &amp;info-&gt;op.atomic_compare_swap;</div><div class='add'>+	wqe = irdma_qp_get_next_send_wqe(qp, &amp;wqe_idx, quanta, total_size,</div><div class='add'>+					 info);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0, op_info-&gt;tagged_offset);</div><div class='add'>+	set_64bit_val(wqe, 8,</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_STAG, op_info-&gt;stag));</div><div class='add'>+	set_64bit_val(wqe, 16, op_info-&gt;remote_tagged_offset);</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_REMOTE_STAG, op_info-&gt;remote_stag) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_ATOMIC_COMPARE_SWAP_ADD) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_READFENCE, info-&gt;read_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info-&gt;local_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info-&gt;signaled) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;swqe_polarity);</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 32, op_info-&gt;swap_data_bytes);</div><div class='add'>+	set_64bit_val(wqe, 40, op_info-&gt;compare_data_bytes);</div><div class='add'>+	set_64bit_val(wqe, 48, 0);</div><div class='add'>+	set_64bit_val(wqe, 56,</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;swqe_polarity));</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_uk_qp_post_wr(qp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_srq_post_receive - post a receive wqe to a shared rq</div><div class='add'>+ * @srq: shared rq ptr</div><div class='add'>+ * @info: post rq information</div><div class='add'>+ */</div><div class='add'>+int irdma_uk_srq_post_receive(struct irdma_srq_uk *srq,</div><div class='add'>+			      struct irdma_post_rq_info *info)</div><div class='add'>+{</div><div class='add'>+	u32 wqe_idx, i, byte_off;</div><div class='add'>+	u32 addl_frag_cnt;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	if (srq-&gt;max_srq_frag_cnt &lt; info-&gt;num_sges)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_srq_get_next_recv_wqe(srq, &amp;wqe_idx);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	addl_frag_cnt = info-&gt;num_sges &gt; 1 ? info-&gt;num_sges - 1 : 0;</div><div class='add'>+	srq-&gt;wqe_ops.iw_set_fragment(wqe, 0, info-&gt;sg_list,</div><div class='add'>+				     srq-&gt;srwqe_polarity);</div><div class='add'>+</div><div class='add'>+	for (i = 1, byte_off = 32; i &lt; info-&gt;num_sges; i++) {</div><div class='add'>+		srq-&gt;wqe_ops.iw_set_fragment(wqe, byte_off, &amp;info-&gt;sg_list[i],</div><div class='add'>+					     srq-&gt;srwqe_polarity);</div><div class='add'>+		byte_off += 16;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* if not an odd number set valid bit in next fragment */</div><div class='add'>+	if (srq-&gt;uk_attrs-&gt;hw_rev &gt;= IRDMA_GEN_2 &amp;&amp; !(info-&gt;num_sges &amp; 0x01) &amp;&amp;</div><div class='add'>+	    info-&gt;num_sges) {</div><div class='add'>+		srq-&gt;wqe_ops.iw_set_fragment(wqe, byte_off, NULL,</div><div class='add'>+					     srq-&gt;srwqe_polarity);</div><div class='add'>+		if (srq-&gt;uk_attrs-&gt;hw_rev == IRDMA_GEN_2)</div><div class='add'>+			++addl_frag_cnt;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16, (u64)info-&gt;wr_id);</div><div class='add'>+	hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VALID, srq-&gt;srwqe_polarity);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	set_64bit_val(srq-&gt;shadow_area, 0, (wqe_idx + 1) % srq-&gt;srq_ring.size);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_rdma_read - rdma read command</div><div class='add'>+ * @qp: hw qp ptr</div><div class='add'>+ * @info: post sq information</div><div class='add'>+ * @inv_stag: flag for inv_stag</div><div class='add'>+ * @post_sq: flag to post sq</div><div class='add'>+ */</div><div class='add'>+int irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,</div><div class='add'>+		       bool inv_stag, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_rdma_read *op_info;</div><div class='add'>+	int ret_code;</div><div class='add'>+	u32 i, byte_off, total_size = 0;</div><div class='add'>+	bool local_fence = false;</div><div class='add'>+	u32 addl_frag_cnt;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+	u16 quanta;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	op_info = &amp;info-&gt;op.rdma_read;</div><div class='add'>+	if (qp-&gt;max_sq_frag_cnt &lt; op_info-&gt;num_lo_sges)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; op_info-&gt;num_lo_sges; i++)</div><div class='add'>+		total_size += op_info-&gt;lo_sg_list[i].length;</div><div class='add'>+</div><div class='add'>+	ret_code = irdma_fragcnt_to_quanta_sq(op_info-&gt;num_lo_sges, &amp;quanta);</div><div class='add'>+	if (ret_code)</div><div class='add'>+		return ret_code;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_qp_get_next_send_wqe(qp, &amp;wqe_idx, quanta, total_size,</div><div class='add'>+					 info);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	irdma_clr_wqes(qp, wqe_idx);</div><div class='add'>+</div><div class='add'>+	addl_frag_cnt = op_info-&gt;num_lo_sges &gt; 1 ?</div><div class='add'>+			(op_info-&gt;num_lo_sges - 1) : 0;</div><div class='add'>+	local_fence |= info-&gt;local_fence;</div><div class='add'>+</div><div class='add'>+	qp-&gt;wqe_ops.iw_set_fragment(wqe, 0, op_info-&gt;lo_sg_list,</div><div class='add'>+				    qp-&gt;swqe_polarity);</div><div class='add'>+	for (i = 1, byte_off = 32; i &lt; op_info-&gt;num_lo_sges; ++i) {</div><div class='add'>+		qp-&gt;wqe_ops.iw_set_fragment(wqe, byte_off,</div><div class='add'>+					    &amp;op_info-&gt;lo_sg_list[i],</div><div class='add'>+					    qp-&gt;swqe_polarity);</div><div class='add'>+		byte_off += 16;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* if not an odd number set valid bit in next fragment */</div><div class='add'>+	if (qp-&gt;uk_attrs-&gt;hw_rev &gt;= IRDMA_GEN_2 &amp;&amp;</div><div class='add'>+	    !(op_info-&gt;num_lo_sges &amp; 0x01) &amp;&amp; op_info-&gt;num_lo_sges) {</div><div class='add'>+		qp-&gt;wqe_ops.iw_set_fragment(wqe, byte_off, NULL,</div><div class='add'>+					    qp-&gt;swqe_polarity);</div><div class='add'>+		if (qp-&gt;uk_attrs-&gt;hw_rev == IRDMA_GEN_2)</div><div class='add'>+			++addl_frag_cnt;</div><div class='add'>+	}</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info-&gt;rem_addr.addr));</div><div class='add'>+	hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info-&gt;rem_addr.lkey) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info-&gt;report_rtt ? 1 : 0)) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_OPCODE,</div><div class='add'>+			 (inv_stag ? IRDMAQP_OP_RDMA_READ_LOC_INV : IRDMAQP_OP_RDMA_READ)) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_READFENCE, info-&gt;read_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info-&gt;signaled) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;swqe_polarity);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_uk_qp_post_wr(qp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_send - rdma send command</div><div class='add'>+ * @qp: hw qp ptr</div><div class='add'>+ * @info: post sq information</div><div class='add'>+ * @post_sq: flag to post sq</div><div class='add'>+ */</div><div class='add'>+int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,</div><div class='add'>+		  bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_post_send *op_info;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 i, wqe_idx, total_size = 0, byte_off;</div><div class='add'>+	int ret_code;</div><div class='add'>+	u32 frag_cnt, addl_frag_cnt;</div><div class='add'>+	bool read_fence = false;</div><div class='add'>+	u16 quanta;</div><div class='add'>+</div><div class='add'>+	op_info = &amp;info-&gt;op.send;</div><div class='add'>+	if (qp-&gt;max_sq_frag_cnt &lt; op_info-&gt;num_sges)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; op_info-&gt;num_sges; i++)</div><div class='add'>+		total_size += op_info-&gt;sg_list[i].length;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;imm_data_valid)</div><div class='add'>+		frag_cnt = op_info-&gt;num_sges + 1;</div><div class='add'>+	else</div><div class='add'>+		frag_cnt = op_info-&gt;num_sges;</div><div class='add'>+	ret_code = irdma_fragcnt_to_quanta_sq(frag_cnt, &amp;quanta);</div><div class='add'>+	if (ret_code)</div><div class='add'>+		return ret_code;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_qp_get_next_send_wqe(qp, &amp;wqe_idx, quanta, total_size,</div><div class='add'>+					 info);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	irdma_clr_wqes(qp, wqe_idx);</div><div class='add'>+</div><div class='add'>+	read_fence |= info-&gt;read_fence;</div><div class='add'>+	addl_frag_cnt = frag_cnt &gt; 1 ? (frag_cnt - 1) : 0;</div><div class='add'>+	if (info-&gt;imm_data_valid) {</div><div class='add'>+		set_64bit_val(wqe, 0,</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_IMMDATA, info-&gt;imm_data));</div><div class='add'>+		i = 0;</div><div class='add'>+	} else {</div><div class='add'>+		qp-&gt;wqe_ops.iw_set_fragment(wqe, 0,</div><div class='add'>+					    frag_cnt ? op_info-&gt;sg_list : NULL,</div><div class='add'>+					    qp-&gt;swqe_polarity);</div><div class='add'>+		i = 1;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (byte_off = 32; i &lt; op_info-&gt;num_sges; i++) {</div><div class='add'>+		qp-&gt;wqe_ops.iw_set_fragment(wqe, byte_off, &amp;op_info-&gt;sg_list[i],</div><div class='add'>+					    qp-&gt;swqe_polarity);</div><div class='add'>+		byte_off += 16;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* if not an odd number set valid bit in next fragment */</div><div class='add'>+	if (qp-&gt;uk_attrs-&gt;hw_rev &gt;= IRDMA_GEN_2 &amp;&amp; !(frag_cnt &amp; 0x01) &amp;&amp;</div><div class='add'>+	    frag_cnt) {</div><div class='add'>+		qp-&gt;wqe_ops.iw_set_fragment(wqe, byte_off, NULL,</div><div class='add'>+					    qp-&gt;swqe_polarity);</div><div class='add'>+		if (qp-&gt;uk_attrs-&gt;hw_rev == IRDMA_GEN_2)</div><div class='add'>+			++addl_frag_cnt;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info-&gt;qkey) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info-&gt;dest_qp));</div><div class='add'>+	hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info-&gt;stag_to_inv) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_AHID, op_info-&gt;ah_id) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,</div><div class='add'>+			 (info-&gt;imm_data_valid ? 1 : 0)) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info-&gt;report_rtt ? 1 : 0)) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_OPCODE, info-&gt;op_type) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info-&gt;local_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info-&gt;signaled) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_UDPHEADER, info-&gt;udp_hdr) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_L4LEN, info-&gt;l4len) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;swqe_polarity);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_uk_qp_post_wr(qp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_set_mw_bind_wqe_gen_1 - set mw bind wqe</div><div class='add'>+ * @wqe: wqe for setting fragment</div><div class='add'>+ * @op_info: info for setting bind wqe values</div><div class='add'>+ */</div><div class='add'>+static void irdma_set_mw_bind_wqe_gen_1(__le64 *wqe,</div><div class='add'>+					struct irdma_bind_window *op_info)</div><div class='add'>+{</div><div class='add'>+	set_64bit_val(wqe, 0, (uintptr_t)op_info-&gt;va);</div><div class='add'>+	set_64bit_val(wqe, 8,</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info-&gt;mw_stag) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info-&gt;mr_stag));</div><div class='add'>+	set_64bit_val(wqe, 16, op_info-&gt;bind_len);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_copy_inline_data_gen_1 - Copy inline data to wqe</div><div class='add'>+ * @wqe: pointer to wqe</div><div class='add'>+ * @sge_list: table of pointers to inline data</div><div class='add'>+ * @num_sges: Total inline data length</div><div class='add'>+ * @polarity: compatibility parameter</div><div class='add'>+ */</div><div class='add'>+static void irdma_copy_inline_data_gen_1(u8 *wqe, struct ib_sge *sge_list,</div><div class='add'>+					 u32 num_sges, u8 polarity)</div><div class='add'>+{</div><div class='add'>+	u32 quanta_bytes_remaining = 16;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; num_sges; i++) {</div><div class='add'>+		u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;</div><div class='add'>+		u32 sge_len = sge_list[i].length;</div><div class='add'>+</div><div class='add'>+		while (sge_len) {</div><div class='add'>+			u32 bytes_copied;</div><div class='add'>+</div><div class='add'>+			bytes_copied = min(sge_len, quanta_bytes_remaining);</div><div class='add'>+			memcpy(wqe, cur_sge, bytes_copied);</div><div class='add'>+			wqe += bytes_copied;</div><div class='add'>+			cur_sge += bytes_copied;</div><div class='add'>+			quanta_bytes_remaining -= bytes_copied;</div><div class='add'>+			sge_len -= bytes_copied;</div><div class='add'>+</div><div class='add'>+			if (!quanta_bytes_remaining) {</div><div class='add'>+				/* Remaining inline bytes reside after hdr */</div><div class='add'>+				wqe += 16;</div><div class='add'>+				quanta_bytes_remaining = 32;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_inline_data_size_to_quanta_gen_1 - based on inline data, quanta</div><div class='add'>+ * @data_size: data size for inline</div><div class='add'>+ *</div><div class='add'>+ * Gets the quanta based on inline and immediate data.</div><div class='add'>+ */</div><div class='add'>+static inline u16 irdma_inline_data_size_to_quanta_gen_1(u32 data_size)</div><div class='add'>+{</div><div class='add'>+	return data_size &lt;= 16 ? IRDMA_QP_WQE_MIN_QUANTA : 2;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_set_mw_bind_wqe - set mw bind in wqe</div><div class='add'>+ * @wqe: wqe for setting mw bind</div><div class='add'>+ * @op_info: info for setting wqe values</div><div class='add'>+ */</div><div class='add'>+static void irdma_set_mw_bind_wqe(__le64 *wqe,</div><div class='add'>+				  struct irdma_bind_window *op_info)</div><div class='add'>+{</div><div class='add'>+	set_64bit_val(wqe, 0, (uintptr_t)op_info-&gt;va);</div><div class='add'>+	set_64bit_val(wqe, 8,</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info-&gt;mr_stag) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info-&gt;mw_stag));</div><div class='add'>+	set_64bit_val(wqe, 16, op_info-&gt;bind_len);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_copy_inline_data - Copy inline data to wqe</div><div class='add'>+ * @wqe: pointer to wqe</div><div class='add'>+ * @sge_list: table of pointers to inline data</div><div class='add'>+ * @num_sges: number of SGE's</div><div class='add'>+ * @polarity: polarity of wqe valid bit</div><div class='add'>+ */</div><div class='add'>+static void irdma_copy_inline_data(u8 *wqe, struct ib_sge *sge_list,</div><div class='add'>+				   u32 num_sges, u8 polarity)</div><div class='add'>+{</div><div class='add'>+	u8 inline_valid = polarity &lt;&lt; IRDMA_INLINE_VALID_S;</div><div class='add'>+	u32 quanta_bytes_remaining = 8;</div><div class='add'>+	bool first_quanta = true;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	wqe += 8;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; num_sges; i++) {</div><div class='add'>+		u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;</div><div class='add'>+		u32 sge_len = sge_list[i].length;</div><div class='add'>+</div><div class='add'>+		while (sge_len) {</div><div class='add'>+			u32 bytes_copied;</div><div class='add'>+</div><div class='add'>+			bytes_copied = min(sge_len, quanta_bytes_remaining);</div><div class='add'>+			memcpy(wqe, cur_sge, bytes_copied);</div><div class='add'>+			wqe += bytes_copied;</div><div class='add'>+			cur_sge += bytes_copied;</div><div class='add'>+			quanta_bytes_remaining -= bytes_copied;</div><div class='add'>+			sge_len -= bytes_copied;</div><div class='add'>+</div><div class='add'>+			if (!quanta_bytes_remaining) {</div><div class='add'>+				quanta_bytes_remaining = 31;</div><div class='add'>+</div><div class='add'>+				/* Remaining inline bytes reside after hdr */</div><div class='add'>+				if (first_quanta) {</div><div class='add'>+					first_quanta = false;</div><div class='add'>+					wqe += 16;</div><div class='add'>+				} else {</div><div class='add'>+					*wqe = inline_valid;</div><div class='add'>+					wqe++;</div><div class='add'>+				}</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	if (!first_quanta &amp;&amp; quanta_bytes_remaining &lt; 31)</div><div class='add'>+		*(wqe + quanta_bytes_remaining) = inline_valid;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_inline_data_size_to_quanta - based on inline data, quanta</div><div class='add'>+ * @data_size: data size for inline</div><div class='add'>+ *</div><div class='add'>+ * Gets the quanta based on inline and immediate data.</div><div class='add'>+ */</div><div class='add'>+static u16 irdma_inline_data_size_to_quanta(u32 data_size)</div><div class='add'>+{</div><div class='add'>+	if (data_size &lt;= 8)</div><div class='add'>+		return IRDMA_QP_WQE_MIN_QUANTA;</div><div class='add'>+	else if (data_size &lt;= 39)</div><div class='add'>+		return 2;</div><div class='add'>+	else if (data_size &lt;= 70)</div><div class='add'>+		return 3;</div><div class='add'>+	else if (data_size &lt;= 101)</div><div class='add'>+		return 4;</div><div class='add'>+	else if (data_size &lt;= 132)</div><div class='add'>+		return 5;</div><div class='add'>+	else if (data_size &lt;= 163)</div><div class='add'>+		return 6;</div><div class='add'>+	else if (data_size &lt;= 194)</div><div class='add'>+		return 7;</div><div class='add'>+	else</div><div class='add'>+		return 8;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_inline_rdma_write - inline rdma write operation</div><div class='add'>+ * @qp: hw qp ptr</div><div class='add'>+ * @info: post sq information</div><div class='add'>+ * @post_sq: flag to post sq</div><div class='add'>+ */</div><div class='add'>+int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,</div><div class='add'>+			       struct irdma_post_sq_info *info, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_rdma_write *op_info;</div><div class='add'>+	u64 hdr = 0;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+	bool read_fence = false;</div><div class='add'>+	u32 i, total_size = 0;</div><div class='add'>+	u16 quanta;</div><div class='add'>+</div><div class='add'>+	op_info = &amp;info-&gt;op.rdma_write;</div><div class='add'>+</div><div class='add'>+	if (unlikely(qp-&gt;max_sq_frag_cnt &lt; op_info-&gt;num_lo_sges))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; op_info-&gt;num_lo_sges; i++)</div><div class='add'>+		total_size += op_info-&gt;lo_sg_list[i].length;</div><div class='add'>+</div><div class='add'>+	if (unlikely(total_size &gt; qp-&gt;max_inline_data))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	quanta = qp-&gt;wqe_ops.iw_inline_data_size_to_quanta(total_size);</div><div class='add'>+	wqe = irdma_qp_get_next_send_wqe(qp, &amp;wqe_idx, quanta, total_size,</div><div class='add'>+					 info);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	irdma_clr_wqes(qp, wqe_idx);</div><div class='add'>+</div><div class='add'>+	read_fence |= info-&gt;read_fence;</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info-&gt;rem_addr.addr));</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info-&gt;rem_addr.lkey) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_OPCODE, info-&gt;op_type) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_REPORTRTT, info-&gt;report_rtt ? 1 : 0) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info-&gt;imm_data_valid ? 1 : 0) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info-&gt;local_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info-&gt;signaled) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;swqe_polarity);</div><div class='add'>+</div><div class='add'>+	if (info-&gt;imm_data_valid)</div><div class='add'>+		set_64bit_val(wqe, 0,</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_IMMDATA, info-&gt;imm_data));</div><div class='add'>+</div><div class='add'>+	qp-&gt;wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info-&gt;lo_sg_list,</div><div class='add'>+					op_info-&gt;num_lo_sges,</div><div class='add'>+					qp-&gt;swqe_polarity);</div><div class='add'>+	dma_wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_uk_qp_post_wr(qp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_inline_send - inline send operation</div><div class='add'>+ * @qp: hw qp ptr</div><div class='add'>+ * @info: post sq information</div><div class='add'>+ * @post_sq: flag to post sq</div><div class='add'>+ */</div><div class='add'>+int irdma_uk_inline_send(struct irdma_qp_uk *qp,</div><div class='add'>+			 struct irdma_post_sq_info *info, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_post_send *op_info;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+	bool read_fence = false;</div><div class='add'>+	u32 i, total_size = 0;</div><div class='add'>+	u16 quanta;</div><div class='add'>+</div><div class='add'>+	op_info = &amp;info-&gt;op.send;</div><div class='add'>+</div><div class='add'>+	if (unlikely(qp-&gt;max_sq_frag_cnt &lt; op_info-&gt;num_sges))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; op_info-&gt;num_sges; i++)</div><div class='add'>+		total_size += op_info-&gt;sg_list[i].length;</div><div class='add'>+</div><div class='add'>+	if (unlikely(total_size &gt; qp-&gt;max_inline_data))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	quanta = qp-&gt;wqe_ops.iw_inline_data_size_to_quanta(total_size);</div><div class='add'>+	wqe = irdma_qp_get_next_send_wqe(qp, &amp;wqe_idx, quanta, total_size,</div><div class='add'>+					 info);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	irdma_clr_wqes(qp, wqe_idx);</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16,</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info-&gt;qkey) |</div><div class='add'>+		      FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info-&gt;dest_qp));</div><div class='add'>+</div><div class='add'>+	read_fence |= info-&gt;read_fence;</div><div class='add'>+	hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info-&gt;stag_to_inv) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_AHID, op_info-&gt;ah_id) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_OPCODE, info-&gt;op_type) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,</div><div class='add'>+			 (info-&gt;imm_data_valid ? 1 : 0)) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info-&gt;report_rtt ? 1 : 0)) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info-&gt;local_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info-&gt;signaled) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_UDPHEADER, info-&gt;udp_hdr) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_L4LEN, info-&gt;l4len) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;swqe_polarity);</div><div class='add'>+</div><div class='add'>+	if (info-&gt;imm_data_valid)</div><div class='add'>+		set_64bit_val(wqe, 0,</div><div class='add'>+			      FIELD_PREP(IRDMAQPSQ_IMMDATA, info-&gt;imm_data));</div><div class='add'>+	qp-&gt;wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info-&gt;sg_list,</div><div class='add'>+					op_info-&gt;num_sges, qp-&gt;swqe_polarity);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_uk_qp_post_wr(qp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_stag_local_invalidate - stag invalidate operation</div><div class='add'>+ * @qp: hw qp ptr</div><div class='add'>+ * @info: post sq information</div><div class='add'>+ * @post_sq: flag to post sq</div><div class='add'>+ */</div><div class='add'>+int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,</div><div class='add'>+				   struct irdma_post_sq_info *info,</div><div class='add'>+				   bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	struct irdma_inv_local_stag *op_info;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+	bool local_fence = false;</div><div class='add'>+	struct ib_sge sge = {};</div><div class='add'>+</div><div class='add'>+	op_info = &amp;info-&gt;op.inv_local_stag;</div><div class='add'>+	local_fence = info-&gt;local_fence;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_qp_get_next_send_wqe(qp, &amp;wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,</div><div class='add'>+					 0, info);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	irdma_clr_wqes(qp, wqe_idx);</div><div class='add'>+</div><div class='add'>+	sge.lkey = op_info-&gt;target_stag;</div><div class='add'>+	qp-&gt;wqe_ops.iw_set_fragment(wqe, 0, &amp;sge, 0);</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16, 0);</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMA_OP_TYPE_INV_STAG) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_READFENCE, info-&gt;read_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info-&gt;signaled) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;swqe_polarity);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_uk_qp_post_wr(qp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_post_receive - post receive wqe</div><div class='add'>+ * @qp: hw qp ptr</div><div class='add'>+ * @info: post rq information</div><div class='add'>+ */</div><div class='add'>+int irdma_uk_post_receive(struct irdma_qp_uk *qp,</div><div class='add'>+			  struct irdma_post_rq_info *info)</div><div class='add'>+{</div><div class='add'>+	u32 wqe_idx, i, byte_off;</div><div class='add'>+	u32 addl_frag_cnt;</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;max_rq_frag_cnt &lt; info-&gt;num_sges)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	wqe = irdma_qp_get_next_recv_wqe(qp, &amp;wqe_idx);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	qp-&gt;rq_wrid_array[wqe_idx] = info-&gt;wr_id;</div><div class='add'>+	addl_frag_cnt = info-&gt;num_sges &gt; 1 ? (info-&gt;num_sges - 1) : 0;</div><div class='add'>+	qp-&gt;wqe_ops.iw_set_fragment(wqe, 0, info-&gt;sg_list,</div><div class='add'>+				    qp-&gt;rwqe_polarity);</div><div class='add'>+</div><div class='add'>+	for (i = 1, byte_off = 32; i &lt; info-&gt;num_sges; i++) {</div><div class='add'>+		qp-&gt;wqe_ops.iw_set_fragment(wqe, byte_off, &amp;info-&gt;sg_list[i],</div><div class='add'>+					    qp-&gt;rwqe_polarity);</div><div class='add'>+		byte_off += 16;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* if not an odd number set valid bit in next fragment */</div><div class='add'>+	if (qp-&gt;uk_attrs-&gt;hw_rev &gt;= IRDMA_GEN_2 &amp;&amp; !(info-&gt;num_sges &amp; 0x01) &amp;&amp;</div><div class='add'>+	    info-&gt;num_sges) {</div><div class='add'>+		qp-&gt;wqe_ops.iw_set_fragment(wqe, byte_off, NULL,</div><div class='add'>+					    qp-&gt;rwqe_polarity);</div><div class='add'>+		if (qp-&gt;uk_attrs-&gt;hw_rev == IRDMA_GEN_2)</div><div class='add'>+			++addl_frag_cnt;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 16, 0);</div><div class='add'>+	hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;rwqe_polarity);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_cq_resize - reset the cq buffer info</div><div class='add'>+ * @cq: cq to resize</div><div class='add'>+ * @cq_base: new cq buffer addr</div><div class='add'>+ * @cq_size: number of cqes</div><div class='add'>+ */</div><div class='add'>+void irdma_uk_cq_resize(struct irdma_cq_uk *cq, void *cq_base, int cq_size)</div><div class='add'>+{</div><div class='add'>+	cq-&gt;cq_base = cq_base;</div><div class='add'>+	cq-&gt;cq_size = cq_size;</div><div class='add'>+	IRDMA_RING_INIT(cq-&gt;cq_ring, cq-&gt;cq_size);</div><div class='add'>+	cq-&gt;polarity = 1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_cq_set_resized_cnt - record the count of the resized buffers</div><div class='add'>+ * @cq: cq to resize</div><div class='add'>+ * @cq_cnt: the count of the resized cq buffers</div><div class='add'>+ */</div><div class='add'>+void irdma_uk_cq_set_resized_cnt(struct irdma_cq_uk *cq, u16 cq_cnt)</div><div class='add'>+{</div><div class='add'>+	u64 temp_val;</div><div class='add'>+	u16 sw_cq_sel;</div><div class='add'>+	u8 arm_next_se;</div><div class='add'>+	u8 arm_next;</div><div class='add'>+	u8 arm_seq_num;</div><div class='add'>+</div><div class='add'>+	get_64bit_val(cq-&gt;shadow_area, 32, &amp;temp_val);</div><div class='add'>+</div><div class='add'>+	sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);</div><div class='add'>+	sw_cq_sel += cq_cnt;</div><div class='add'>+</div><div class='add'>+	arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);</div><div class='add'>+	arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);</div><div class='add'>+	arm_next = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT, temp_val);</div><div class='add'>+</div><div class='add'>+	temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |</div><div class='add'>+		   FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |</div><div class='add'>+		   FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |</div><div class='add'>+		   FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next);</div><div class='add'>+</div><div class='add'>+	set_64bit_val(cq-&gt;shadow_area, 32, temp_val);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_cq_request_notification - cq notification request (door bell)</div><div class='add'>+ * @cq: hw cq</div><div class='add'>+ * @cq_notify: notification type</div><div class='add'>+ */</div><div class='add'>+void irdma_uk_cq_request_notification(struct irdma_cq_uk *cq,</div><div class='add'>+				      enum irdma_cmpl_notify cq_notify)</div><div class='add'>+{</div><div class='add'>+	u64 temp_val;</div><div class='add'>+	u16 sw_cq_sel;</div><div class='add'>+	u8 arm_next_se = 0;</div><div class='add'>+	u8 arm_next = 0;</div><div class='add'>+	u8 arm_seq_num;</div><div class='add'>+</div><div class='add'>+	get_64bit_val(cq-&gt;shadow_area, 32, &amp;temp_val);</div><div class='add'>+	arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);</div><div class='add'>+	arm_seq_num++;</div><div class='add'>+	sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);</div><div class='add'>+	arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);</div><div class='add'>+	arm_next_se |= 1;</div><div class='add'>+	if (cq_notify == IRDMA_CQ_COMPL_EVENT)</div><div class='add'>+		arm_next = 1;</div><div class='add'>+	temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |</div><div class='add'>+		   FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |</div><div class='add'>+		   FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |</div><div class='add'>+		   FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next);</div><div class='add'>+</div><div class='add'>+	set_64bit_val(cq-&gt;shadow_area, 32, temp_val);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='add'>+</div><div class='add'>+	writel(cq-&gt;cq_id, cq-&gt;cqe_alloc_db);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_cq_empty - Check if CQ is empty</div><div class='add'>+ * @cq: hw cq</div><div class='add'>+ */</div><div class='add'>+bool irdma_uk_cq_empty(struct irdma_cq_uk *cq)</div><div class='add'>+{</div><div class='add'>+	__le64 *cqe;</div><div class='add'>+	u8 polarity;</div><div class='add'>+	u64 qword3;</div><div class='add'>+</div><div class='add'>+	if (cq-&gt;avoid_mem_cflct)</div><div class='add'>+		cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(cq);</div><div class='add'>+	else</div><div class='add'>+		cqe = IRDMA_GET_CURRENT_CQ_ELEM(cq);</div><div class='add'>+</div><div class='add'>+	get_64bit_val(cqe, 24, &amp;qword3);</div><div class='add'>+	polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);</div><div class='add'>+</div><div class='add'>+	return polarity != cq-&gt;polarity;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_cq_poll_cmpl - get cq completion info</div><div class='add'>+ * @cq: hw cq</div><div class='add'>+ * @info: cq poll information returned</div><div class='add'>+ */</div><div class='add'>+int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,</div><div class='add'>+			  struct irdma_cq_poll_info *info)</div><div class='add'>+{</div><div class='add'>+	u64 comp_ctx, qword0, qword2, qword3;</div><div class='add'>+	__le64 *cqe;</div><div class='add'>+	struct irdma_qp_uk *qp;</div><div class='add'>+	struct irdma_srq_uk *srq;</div><div class='add'>+	struct qp_err_code qp_err;</div><div class='add'>+	u8 is_srq;</div><div class='add'>+	struct irdma_ring *pring = NULL;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+	int ret_code;</div><div class='add'>+	bool move_cq_head = true;</div><div class='add'>+	u8 polarity;</div><div class='add'>+	bool ext_valid;</div><div class='add'>+	__le64 *ext_cqe;</div><div class='add'>+</div><div class='add'>+	if (cq-&gt;avoid_mem_cflct)</div><div class='add'>+		cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(cq);</div><div class='add'>+	else</div><div class='add'>+		cqe = IRDMA_GET_CURRENT_CQ_ELEM(cq);</div><div class='add'>+</div><div class='add'>+	get_64bit_val(cqe, 24, &amp;qword3);</div><div class='add'>+	polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);</div><div class='add'>+	if (polarity != cq-&gt;polarity)</div><div class='add'>+		return -ENOENT;</div><div class='add'>+</div><div class='add'>+	/* Ensure CQE contents are read after valid bit is checked */</div><div class='add'>+	dma_rmb();</div><div class='add'>+</div><div class='add'>+	ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3);</div><div class='add'>+	if (ext_valid) {</div><div class='add'>+		u64 qword6, qword7;</div><div class='add'>+		u32 peek_head;</div><div class='add'>+</div><div class='add'>+		if (cq-&gt;avoid_mem_cflct) {</div><div class='add'>+			ext_cqe = (__le64 *)((u8 *)cqe + 32);</div><div class='add'>+			get_64bit_val(ext_cqe, 24, &amp;qword7);</div><div class='add'>+			polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7);</div><div class='add'>+		} else {</div><div class='add'>+			peek_head = (cq-&gt;cq_ring.head + 1) % cq-&gt;cq_ring.size;</div><div class='add'>+			ext_cqe = cq-&gt;cq_base[peek_head].buf;</div><div class='add'>+			get_64bit_val(ext_cqe, 24, &amp;qword7);</div><div class='add'>+			polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7);</div><div class='add'>+			if (!peek_head)</div><div class='add'>+				polarity ^= 1;</div><div class='add'>+		}</div><div class='add'>+		if (polarity != cq-&gt;polarity)</div><div class='add'>+			return -ENOENT;</div><div class='add'>+</div><div class='add'>+		/* Ensure ext CQE contents are read after ext valid bit is checked */</div><div class='add'>+		dma_rmb();</div><div class='add'>+</div><div class='add'>+		info-&gt;imm_valid = (bool)FIELD_GET(IRDMA_CQ_IMMVALID, qword7);</div><div class='add'>+		if (info-&gt;imm_valid) {</div><div class='add'>+			u64 qword4;</div><div class='add'>+</div><div class='add'>+			get_64bit_val(ext_cqe, 0, &amp;qword4);</div><div class='add'>+			info-&gt;imm_data = (u32)FIELD_GET(IRDMA_CQ_IMMDATALOW32, qword4);</div><div class='add'>+		}</div><div class='add'>+		info-&gt;ud_smac_valid = (bool)FIELD_GET(IRDMA_CQ_UDSMACVALID, qword7);</div><div class='add'>+		info-&gt;ud_vlan_valid = (bool)FIELD_GET(IRDMA_CQ_UDVLANVALID, qword7);</div><div class='add'>+		if (info-&gt;ud_smac_valid || info-&gt;ud_vlan_valid) {</div><div class='add'>+			get_64bit_val(ext_cqe, 16, &amp;qword6);</div><div class='add'>+			if (info-&gt;ud_vlan_valid)</div><div class='add'>+				info-&gt;ud_vlan = (u16)FIELD_GET(IRDMA_CQ_UDVLAN, qword6);</div><div class='add'>+			if (info-&gt;ud_smac_valid) {</div><div class='add'>+				info-&gt;ud_smac[5] = qword6 &amp; 0xFF;</div><div class='add'>+				info-&gt;ud_smac[4] = (qword6 &gt;&gt; 8) &amp; 0xFF;</div><div class='add'>+				info-&gt;ud_smac[3] = (qword6 &gt;&gt; 16) &amp; 0xFF;</div><div class='add'>+				info-&gt;ud_smac[2] = (qword6 &gt;&gt; 24) &amp; 0xFF;</div><div class='add'>+				info-&gt;ud_smac[1] = (qword6 &gt;&gt; 32) &amp; 0xFF;</div><div class='add'>+				info-&gt;ud_smac[0] = (qword6 &gt;&gt; 40) &amp; 0xFF;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		info-&gt;imm_valid = false;</div><div class='add'>+		info-&gt;ud_smac_valid = false;</div><div class='add'>+		info-&gt;ud_vlan_valid = false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	info-&gt;q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);</div><div class='add'>+	is_srq = (u8)FIELD_GET(IRDMA_CQ_SRQ, qword3);</div><div class='add'>+	info-&gt;error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);</div><div class='add'>+	info-&gt;ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);</div><div class='add'>+	get_64bit_val(cqe, 8, &amp;comp_ctx);</div><div class='add'>+	if (is_srq)</div><div class='add'>+		get_64bit_val(cqe, 40, (u64 *)&amp;qp);</div><div class='add'>+	else</div><div class='add'>+		qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx;</div><div class='add'>+	if (info-&gt;error) {</div><div class='add'>+		info-&gt;major_err = FIELD_GET(IRDMA_CQ_MAJERR, qword3);</div><div class='add'>+		info-&gt;minor_err = FIELD_GET(IRDMA_CQ_MINERR, qword3);</div><div class='add'>+		switch (info-&gt;major_err) {</div><div class='add'>+		case IRDMA_SRQFLUSH_RSVD_MAJOR_ERR:</div><div class='add'>+			qp_err = irdma_ae_to_qp_err_code(info-&gt;minor_err);</div><div class='add'>+			info-&gt;minor_err = qp_err.flush_code;</div><div class='add'>+			fallthrough;</div><div class='add'>+		case IRDMA_FLUSH_MAJOR_ERR:</div><div class='add'>+			/* Set the min error to standard flush error code for remaining cqes */</div><div class='add'>+			if (info-&gt;minor_err != FLUSH_GENERAL_ERR) {</div><div class='add'>+				qword3 &amp;= ~IRDMA_CQ_MINERR;</div><div class='add'>+				qword3 |= FIELD_PREP(IRDMA_CQ_MINERR, FLUSH_GENERAL_ERR);</div><div class='add'>+				set_64bit_val(cqe, 24, qword3);</div><div class='add'>+			}</div><div class='add'>+			info-&gt;comp_status = IRDMA_COMPL_STATUS_FLUSHED;</div><div class='add'>+			break;</div><div class='add'>+		default:</div><div class='add'>+#define IRDMA_CIE_SIGNATURE 0xE</div><div class='add'>+#define IRDMA_CQMAJERR_HIGH_NIBBLE GENMASK(15, 12)</div><div class='add'>+			if (info-&gt;q_type == IRDMA_CQE_QTYPE_SQ &amp;&amp;</div><div class='add'>+			    qp-&gt;qp_type == IRDMA_QP_TYPE_ROCE_UD &amp;&amp;</div><div class='add'>+			    FIELD_GET(IRDMA_CQMAJERR_HIGH_NIBBLE, info-&gt;major_err)</div><div class='add'>+			    == IRDMA_CIE_SIGNATURE) {</div><div class='add'>+				info-&gt;error = 0;</div><div class='add'>+				info-&gt;major_err = 0;</div><div class='add'>+				info-&gt;minor_err = 0;</div><div class='add'>+				info-&gt;comp_status = IRDMA_COMPL_STATUS_SUCCESS;</div><div class='add'>+			} else {</div><div class='add'>+				info-&gt;comp_status = IRDMA_COMPL_STATUS_UNKNOWN;</div><div class='add'>+			}</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		info-&gt;comp_status = IRDMA_COMPL_STATUS_SUCCESS;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	get_64bit_val(cqe, 0, &amp;qword0);</div><div class='add'>+	get_64bit_val(cqe, 16, &amp;qword2);</div><div class='add'>+</div><div class='add'>+	info-&gt;qp_id = (u32)FIELD_GET(IRDMACQ_QPID, qword2);</div><div class='add'>+	info-&gt;ud_src_qpn = (u32)FIELD_GET(IRDMACQ_UDSRCQPN, qword2);</div><div class='add'>+</div><div class='add'>+	get_64bit_val(cqe, 8, &amp;comp_ctx);</div><div class='add'>+</div><div class='add'>+	info-&gt;solicited_event = (bool)FIELD_GET(IRDMACQ_SOEVENT, qword3);</div><div class='add'>+	qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx;</div><div class='add'>+	if (!qp || qp-&gt;destroy_pending) {</div><div class='add'>+		ret_code = -EFAULT;</div><div class='add'>+		goto exit;</div><div class='add'>+	}</div><div class='add'>+	wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);</div><div class='add'>+	info-&gt;qp_handle = (irdma_qp_handle)(unsigned long)qp;</div><div class='add'>+	info-&gt;op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3);</div><div class='add'>+</div><div class='add'>+	if (info-&gt;q_type == IRDMA_CQE_QTYPE_RQ &amp;&amp; is_srq) {</div><div class='add'>+		unsigned long flags;</div><div class='add'>+</div><div class='add'>+		srq = qp-&gt;srq_uk;</div><div class='add'>+</div><div class='add'>+		get_64bit_val(cqe, 8, &amp;info-&gt;wr_id);</div><div class='add'>+		info-&gt;bytes_xfered = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);</div><div class='add'>+</div><div class='add'>+		if (qword3 &amp; IRDMACQ_STAG) {</div><div class='add'>+			info-&gt;stag_invalid_set = true;</div><div class='add'>+			info-&gt;inv_stag = (u32)FIELD_GET(IRDMACQ_INVSTAG,</div><div class='add'>+							qword2);</div><div class='add'>+		} else {</div><div class='add'>+			info-&gt;stag_invalid_set = false;</div><div class='add'>+		}</div><div class='add'>+		spin_lock_irqsave(srq-&gt;lock, flags);</div><div class='add'>+		IRDMA_RING_MOVE_TAIL(srq-&gt;srq_ring);</div><div class='add'>+		spin_unlock_irqrestore(srq-&gt;lock, flags);</div><div class='add'>+		pring = &amp;srq-&gt;srq_ring;</div><div class='add'>+</div><div class='add'>+	} else if (info-&gt;q_type == IRDMA_CQE_QTYPE_RQ &amp;&amp; !is_srq) {</div><div class='add'>+		u32 array_idx;</div><div class='add'>+</div><div class='add'>+		array_idx = wqe_idx / qp-&gt;rq_wqe_size_multiplier;</div><div class='add'>+</div><div class='add'>+		if (info-&gt;comp_status == IRDMA_COMPL_STATUS_FLUSHED ||</div><div class='add'>+		    info-&gt;comp_status == IRDMA_COMPL_STATUS_UNKNOWN) {</div><div class='add'>+			if (!IRDMA_RING_MORE_WORK(qp-&gt;rq_ring)) {</div><div class='add'>+				ret_code = -ENOENT;</div><div class='add'>+				goto exit;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			info-&gt;wr_id = qp-&gt;rq_wrid_array[qp-&gt;rq_ring.tail];</div><div class='add'>+			array_idx = qp-&gt;rq_ring.tail;</div><div class='add'>+		} else {</div><div class='add'>+			info-&gt;wr_id = qp-&gt;rq_wrid_array[array_idx];</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		info-&gt;bytes_xfered = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);</div><div class='add'>+</div><div class='add'>+		if (qword3 &amp; IRDMACQ_STAG) {</div><div class='add'>+			info-&gt;stag_invalid_set = true;</div><div class='add'>+			info-&gt;inv_stag = (u32)FIELD_GET(IRDMACQ_INVSTAG, qword2);</div><div class='add'>+		} else {</div><div class='add'>+			info-&gt;stag_invalid_set = false;</div><div class='add'>+		}</div><div class='add'>+		IRDMA_RING_SET_TAIL(qp-&gt;rq_ring, array_idx + 1);</div><div class='add'>+		if (info-&gt;comp_status == IRDMA_COMPL_STATUS_FLUSHED) {</div><div class='add'>+			qp-&gt;rq_flush_seen = true;</div><div class='add'>+			if (!IRDMA_RING_MORE_WORK(qp-&gt;rq_ring))</div><div class='add'>+				qp-&gt;rq_flush_complete = true;</div><div class='add'>+			else</div><div class='add'>+				move_cq_head = false;</div><div class='add'>+		}</div><div class='add'>+		pring = &amp;qp-&gt;rq_ring;</div><div class='add'>+	} else { /* q_type is IRDMA_CQE_QTYPE_SQ */</div><div class='add'>+		if (qp-&gt;first_sq_wq) {</div><div class='add'>+			if (wqe_idx + 1 &gt;= qp-&gt;conn_wqes)</div><div class='add'>+				qp-&gt;first_sq_wq = false;</div><div class='add'>+</div><div class='add'>+			if (wqe_idx &lt; qp-&gt;conn_wqes &amp;&amp; qp-&gt;sq_ring.head == qp-&gt;sq_ring.tail) {</div><div class='add'>+				IRDMA_RING_MOVE_HEAD_NOCHECK(cq-&gt;cq_ring);</div><div class='add'>+				IRDMA_RING_MOVE_TAIL(cq-&gt;cq_ring);</div><div class='add'>+				set_64bit_val(cq-&gt;shadow_area, 0,</div><div class='add'>+					      IRDMA_RING_CURRENT_HEAD(cq-&gt;cq_ring));</div><div class='add'>+				memset(info, 0,</div><div class='add'>+				       sizeof(struct irdma_cq_poll_info));</div><div class='add'>+				return irdma_uk_cq_poll_cmpl(cq, info);</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+		if (info-&gt;comp_status != IRDMA_COMPL_STATUS_FLUSHED) {</div><div class='add'>+			info-&gt;wr_id = qp-&gt;sq_wrtrk_array[wqe_idx].wrid;</div><div class='add'>+			if (!info-&gt;comp_status)</div><div class='add'>+				info-&gt;bytes_xfered = qp-&gt;sq_wrtrk_array[wqe_idx].wr_len;</div><div class='add'>+			if (!qp-&gt;sq_wrtrk_array[wqe_idx].signaled) {</div><div class='add'>+				ret_code = -EFAULT;</div><div class='add'>+				goto exit;</div><div class='add'>+			}</div><div class='add'>+			info-&gt;op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3);</div><div class='add'>+			IRDMA_RING_SET_TAIL(qp-&gt;sq_ring,</div><div class='add'>+					    wqe_idx + qp-&gt;sq_wrtrk_array[wqe_idx].quanta);</div><div class='add'>+		} else {</div><div class='add'>+			if (!IRDMA_RING_MORE_WORK(qp-&gt;sq_ring)) {</div><div class='add'>+				ret_code = -ENOENT;</div><div class='add'>+				goto exit;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			do {</div><div class='add'>+				__le64 *sw_wqe;</div><div class='add'>+				u64 wqe_qword;</div><div class='add'>+				u32 tail;</div><div class='add'>+</div><div class='add'>+				tail = qp-&gt;sq_ring.tail;</div><div class='add'>+				sw_wqe = qp-&gt;sq_base[tail].elem;</div><div class='add'>+				get_64bit_val(sw_wqe, 24,</div><div class='add'>+					      &amp;wqe_qword);</div><div class='add'>+				info-&gt;op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE,</div><div class='add'>+							      wqe_qword);</div><div class='add'>+				IRDMA_RING_SET_TAIL(qp-&gt;sq_ring,</div><div class='add'>+						    tail + qp-&gt;sq_wrtrk_array[tail].quanta);</div><div class='add'>+				if (info-&gt;op_type != IRDMAQP_OP_NOP) {</div><div class='add'>+					info-&gt;wr_id = qp-&gt;sq_wrtrk_array[tail].wrid;</div><div class='add'>+					info-&gt;bytes_xfered = qp-&gt;sq_wrtrk_array[tail].wr_len;</div><div class='add'>+					break;</div><div class='add'>+				}</div><div class='add'>+			} while (1);</div><div class='add'>+			if (info-&gt;op_type == IRDMA_OP_TYPE_BIND_MW &amp;&amp;</div><div class='add'>+			    info-&gt;minor_err == FLUSH_PROT_ERR)</div><div class='add'>+				info-&gt;minor_err = FLUSH_MW_BIND_ERR;</div><div class='add'>+			qp-&gt;sq_flush_seen = true;</div><div class='add'>+			if (!IRDMA_RING_MORE_WORK(qp-&gt;sq_ring))</div><div class='add'>+				qp-&gt;sq_flush_complete = true;</div><div class='add'>+		}</div><div class='add'>+		pring = &amp;qp-&gt;sq_ring;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret_code = 0;</div><div class='add'>+</div><div class='add'>+exit:</div><div class='add'>+	if (!ret_code &amp;&amp; info-&gt;comp_status == IRDMA_COMPL_STATUS_FLUSHED) {</div><div class='add'>+		if (pring &amp;&amp; IRDMA_RING_MORE_WORK(*pring))</div><div class='add'>+		/* Park CQ head during a flush to generate additional CQEs</div><div class='add'>+		 * from SW for all unprocessed WQEs. For GEN3 and beyond</div><div class='add'>+		 * FW will generate/flush these CQEs so move to the next CQE</div><div class='add'>+		 */</div><div class='add'>+			move_cq_head = qp-&gt;uk_attrs-&gt;hw_rev &lt;= IRDMA_GEN_2 ?</div><div class='add'>+						false : true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (move_cq_head) {</div><div class='add'>+		IRDMA_RING_MOVE_HEAD_NOCHECK(cq-&gt;cq_ring);</div><div class='add'>+		if (!IRDMA_RING_CURRENT_HEAD(cq-&gt;cq_ring))</div><div class='add'>+			cq-&gt;polarity ^= 1;</div><div class='add'>+</div><div class='add'>+		if (ext_valid &amp;&amp; !cq-&gt;avoid_mem_cflct) {</div><div class='add'>+			IRDMA_RING_MOVE_HEAD_NOCHECK(cq-&gt;cq_ring);</div><div class='add'>+			if (!IRDMA_RING_CURRENT_HEAD(cq-&gt;cq_ring))</div><div class='add'>+				cq-&gt;polarity ^= 1;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		IRDMA_RING_MOVE_TAIL(cq-&gt;cq_ring);</div><div class='add'>+		if (!cq-&gt;avoid_mem_cflct &amp;&amp; ext_valid)</div><div class='add'>+			IRDMA_RING_MOVE_TAIL(cq-&gt;cq_ring);</div><div class='add'>+		if (IRDMA_RING_CURRENT_HEAD(cq-&gt;cq_ring) &amp; 0x3F || irdma_uk_cq_empty(cq))</div><div class='add'>+			set_64bit_val(cq-&gt;shadow_area, 0,</div><div class='add'>+				      IRDMA_RING_CURRENT_HEAD(cq-&gt;cq_ring));</div><div class='add'>+	} else {</div><div class='add'>+		qword3 &amp;= ~IRDMA_CQ_WQEIDX;</div><div class='add'>+		qword3 |= FIELD_PREP(IRDMA_CQ_WQEIDX, pring-&gt;tail);</div><div class='add'>+		set_64bit_val(cqe, 24, qword3);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_round_up_wq - return round up qp wq depth</div><div class='add'>+ * @wqdepth: wq depth in quanta to round up</div><div class='add'>+ */</div><div class='add'>+static int irdma_round_up_wq(u32 wqdepth)</div><div class='add'>+{</div><div class='add'>+	int scount = 1;</div><div class='add'>+</div><div class='add'>+	for (wqdepth--; scount &lt;= 16; scount *= 2)</div><div class='add'>+		wqdepth |= wqdepth &gt;&gt; scount;</div><div class='add'>+</div><div class='add'>+	return ++wqdepth;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_wqe_shift - get shift count for maximum wqe size</div><div class='add'>+ * @uk_attrs: qp HW attributes</div><div class='add'>+ * @sge: Maximum Scatter Gather Elements wqe</div><div class='add'>+ * @inline_data: Maximum inline data size</div><div class='add'>+ * @shift: Returns the shift needed based on sge</div><div class='add'>+ *</div><div class='add'>+ * Shift can be used to left shift the wqe size based on number of SGEs and inlind data size.</div><div class='add'>+ * For 1 SGE or inline data &lt;= 8, shift = 0 (wqe size of 32</div><div class='add'>+ * bytes). For 2 or 3 SGEs or inline data &lt;= 39, shift = 1 (wqe</div><div class='add'>+ * size of 64 bytes).</div><div class='add'>+ * For 4-7 SGE's and inline &lt;= 101 Shift of 2 otherwise (wqe</div><div class='add'>+ * size of 256 bytes).</div><div class='add'>+ */</div><div class='add'>+void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,</div><div class='add'>+			 u32 inline_data, u8 *shift)</div><div class='add'>+{</div><div class='add'>+	*shift = 0;</div><div class='add'>+	if (uk_attrs-&gt;hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+		if (sge &gt; 1 || inline_data &gt; 8) {</div><div class='add'>+			if (sge &lt; 4 &amp;&amp; inline_data &lt;= 39)</div><div class='add'>+				*shift = 1;</div><div class='add'>+			else if (sge &lt; 8 &amp;&amp; inline_data &lt;= 101)</div><div class='add'>+				*shift = 2;</div><div class='add'>+			else</div><div class='add'>+				*shift = 3;</div><div class='add'>+		}</div><div class='add'>+	} else if (sge &gt; 1 || inline_data &gt; 16) {</div><div class='add'>+		*shift = (sge &lt; 4 &amp;&amp; inline_data &lt;= 48) ? 1 : 2;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * irdma_get_sqdepth - get SQ depth (quanta)</div><div class='add'>+ * @uk_attrs: qp HW attributes</div><div class='add'>+ * @sq_size: SQ size</div><div class='add'>+ * @shift: shift which determines size of WQE</div><div class='add'>+ * @sqdepth: depth of SQ</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift,</div><div class='add'>+		      u32 *sqdepth)</div><div class='add'>+{</div><div class='add'>+	u32 min_size = (u32)uk_attrs-&gt;min_hw_wq_size &lt;&lt; shift;</div><div class='add'>+</div><div class='add'>+	*sqdepth = irdma_round_up_wq((sq_size &lt;&lt; shift) + IRDMA_SQ_RSVD);</div><div class='add'>+</div><div class='add'>+	if (*sqdepth &lt; min_size)</div><div class='add'>+		*sqdepth = min_size;</div><div class='add'>+	else if (*sqdepth &gt; uk_attrs-&gt;max_hw_wq_quanta)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * irdma_get_rqdepth - get RQ depth (quanta)</div><div class='add'>+ * @uk_attrs: qp HW attributes</div><div class='add'>+ * @rq_size: RQ size</div><div class='add'>+ * @shift: shift which determines size of WQE</div><div class='add'>+ * @rqdepth: depth of RQ</div><div class='add'>+ */</div><div class='add'>+int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift,</div><div class='add'>+		      u32 *rqdepth)</div><div class='add'>+{</div><div class='add'>+	u32 min_size = (u32)uk_attrs-&gt;min_hw_wq_size &lt;&lt; shift;</div><div class='add'>+</div><div class='add'>+	*rqdepth = irdma_round_up_wq((rq_size &lt;&lt; shift) + IRDMA_RQ_RSVD);</div><div class='add'>+</div><div class='add'>+	if (*rqdepth &lt; min_size)</div><div class='add'>+		*rqdepth = min_size;</div><div class='add'>+	else if (*rqdepth &gt; uk_attrs-&gt;max_hw_rq_quanta)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * irdma_get_srqdepth - get SRQ depth (quanta)</div><div class='add'>+ * @uk_attrs: qp HW attributes</div><div class='add'>+ * @srq_size: SRQ size</div><div class='add'>+ * @shift: shift which determines size of WQE</div><div class='add'>+ * @srqdepth: depth of SRQ</div><div class='add'>+ */</div><div class='add'>+int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift,</div><div class='add'>+		       u32 *srqdepth)</div><div class='add'>+{</div><div class='add'>+	*srqdepth = irdma_round_up_wq((srq_size &lt;&lt; shift) + IRDMA_RQ_RSVD);</div><div class='add'>+</div><div class='add'>+	if (*srqdepth &lt; ((u32)uk_attrs-&gt;min_hw_wq_size &lt;&lt; shift))</div><div class='add'>+		*srqdepth = uk_attrs-&gt;min_hw_wq_size &lt;&lt; shift;</div><div class='add'>+	else if (*srqdepth &gt; uk_attrs-&gt;max_hw_srq_quanta)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct irdma_wqe_uk_ops iw_wqe_uk_ops = {</div><div class='add'>+	.iw_copy_inline_data = irdma_copy_inline_data,</div><div class='add'>+	.iw_inline_data_size_to_quanta = irdma_inline_data_size_to_quanta,</div><div class='add'>+	.iw_set_fragment = irdma_set_fragment,</div><div class='add'>+	.iw_set_mw_bind_wqe = irdma_set_mw_bind_wqe,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct irdma_wqe_uk_ops iw_wqe_uk_ops_gen_1 = {</div><div class='add'>+	.iw_copy_inline_data = irdma_copy_inline_data_gen_1,</div><div class='add'>+	.iw_inline_data_size_to_quanta = irdma_inline_data_size_to_quanta_gen_1,</div><div class='add'>+	.iw_set_fragment = irdma_set_fragment_gen_1,</div><div class='add'>+	.iw_set_mw_bind_wqe = irdma_set_mw_bind_wqe_gen_1,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_setup_connection_wqes - setup WQEs necessary to complete</div><div class='add'>+ * connection.</div><div class='add'>+ * @qp: hw qp (user and kernel)</div><div class='add'>+ * @info: qp initialization info</div><div class='add'>+ */</div><div class='add'>+static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp,</div><div class='add'>+					struct irdma_qp_uk_init_info *info)</div><div class='add'>+{</div><div class='add'>+	u16 move_cnt = 1;</div><div class='add'>+</div><div class='add'>+	if (!info-&gt;legacy_mode &amp;&amp;</div><div class='add'>+	    (qp-&gt;uk_attrs-&gt;feature_flags &amp; IRDMA_FEATURE_RTS_AE))</div><div class='add'>+		move_cnt = 3;</div><div class='add'>+</div><div class='add'>+	qp-&gt;conn_wqes = move_cnt;</div><div class='add'>+	IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp-&gt;sq_ring, move_cnt);</div><div class='add'>+	IRDMA_RING_MOVE_TAIL_BY_COUNT(qp-&gt;sq_ring, move_cnt);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_srq_init - initialize shared qp</div><div class='add'>+ * @srq: hw srq (user and kernel)</div><div class='add'>+ * @info: srq initialization info</div><div class='add'>+ *</div><div class='add'>+ * Initializes the vars used in both user and kernel mode.</div><div class='add'>+ * The size of the wqe depends on number of max fragments</div><div class='add'>+ * allowed. Then size of wqe * the number of wqes should be the</div><div class='add'>+ * amount of memory allocated for srq.</div><div class='add'>+ */</div><div class='add'>+int irdma_uk_srq_init(struct irdma_srq_uk *srq,</div><div class='add'>+		      struct irdma_srq_uk_init_info *info)</div><div class='add'>+{</div><div class='add'>+	u8 rqshift;</div><div class='add'>+</div><div class='add'>+	srq-&gt;uk_attrs = info-&gt;uk_attrs;</div><div class='add'>+	if (info-&gt;max_srq_frag_cnt &gt; srq-&gt;uk_attrs-&gt;max_hw_wq_frags)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	irdma_get_wqe_shift(srq-&gt;uk_attrs, info-&gt;max_srq_frag_cnt, 0, &amp;rqshift);</div><div class='add'>+	srq-&gt;srq_caps = info-&gt;srq_caps;</div><div class='add'>+	srq-&gt;srq_base = info-&gt;srq;</div><div class='add'>+	srq-&gt;shadow_area = info-&gt;shadow_area;</div><div class='add'>+	srq-&gt;srq_id = info-&gt;srq_id;</div><div class='add'>+	srq-&gt;srwqe_polarity = 0;</div><div class='add'>+	srq-&gt;srq_size = info-&gt;srq_size;</div><div class='add'>+	srq-&gt;wqe_size = rqshift;</div><div class='add'>+	srq-&gt;max_srq_frag_cnt = min(srq-&gt;uk_attrs-&gt;max_hw_wq_frags,</div><div class='add'>+				    ((u32)2 &lt;&lt; rqshift) - 1);</div><div class='add'>+	IRDMA_RING_INIT(srq-&gt;srq_ring, srq-&gt;srq_size);</div><div class='add'>+	srq-&gt;wqe_size_multiplier = 1 &lt;&lt; rqshift;</div><div class='add'>+	srq-&gt;wqe_ops = iw_wqe_uk_ops;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_calc_shift_wq - calculate WQE shift for both SQ and RQ</div><div class='add'>+ * @ukinfo: qp initialization info</div><div class='add'>+ * @sq_shift: Returns shift of SQ</div><div class='add'>+ * @rq_shift: Returns shift of RQ</div><div class='add'>+ */</div><div class='add'>+void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,</div><div class='add'>+			    u8 *rq_shift)</div><div class='add'>+{</div><div class='add'>+	bool imm_support = ukinfo-&gt;uk_attrs-&gt;hw_rev &gt;= IRDMA_GEN_2;</div><div class='add'>+</div><div class='add'>+	irdma_get_wqe_shift(ukinfo-&gt;uk_attrs,</div><div class='add'>+			    imm_support ? ukinfo-&gt;max_sq_frag_cnt + 1 :</div><div class='add'>+					  ukinfo-&gt;max_sq_frag_cnt,</div><div class='add'>+			    ukinfo-&gt;max_inline_data, sq_shift);</div><div class='add'>+</div><div class='add'>+	irdma_get_wqe_shift(ukinfo-&gt;uk_attrs, ukinfo-&gt;max_rq_frag_cnt, 0,</div><div class='add'>+			    rq_shift);</div><div class='add'>+</div><div class='add'>+	if (ukinfo-&gt;uk_attrs-&gt;hw_rev == IRDMA_GEN_1) {</div><div class='add'>+		if (ukinfo-&gt;abi_ver &gt; 4)</div><div class='add'>+			*rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_calc_depth_shift_sq - calculate depth and shift for SQ size.</div><div class='add'>+ * @ukinfo: qp initialization info</div><div class='add'>+ * @sq_depth: Returns depth of SQ</div><div class='add'>+ * @sq_shift: Returns shift of SQ</div><div class='add'>+ */</div><div class='add'>+int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,</div><div class='add'>+				 u32 *sq_depth, u8 *sq_shift)</div><div class='add'>+{</div><div class='add'>+	bool imm_support = ukinfo-&gt;uk_attrs-&gt;hw_rev &gt;= IRDMA_GEN_2;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	irdma_get_wqe_shift(ukinfo-&gt;uk_attrs,</div><div class='add'>+			    imm_support ? ukinfo-&gt;max_sq_frag_cnt + 1 :</div><div class='add'>+			    ukinfo-&gt;max_sq_frag_cnt,</div><div class='add'>+			    ukinfo-&gt;max_inline_data, sq_shift);</div><div class='add'>+	status = irdma_get_sqdepth(ukinfo-&gt;uk_attrs, ukinfo-&gt;sq_size,</div><div class='add'>+				   *sq_shift, sq_depth);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_calc_depth_shift_rq - calculate depth and shift for RQ size.</div><div class='add'>+ * @ukinfo: qp initialization info</div><div class='add'>+ * @rq_depth: Returns depth of RQ</div><div class='add'>+ * @rq_shift: Returns shift of RQ</div><div class='add'>+ */</div><div class='add'>+int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo,</div><div class='add'>+				 u32 *rq_depth, u8 *rq_shift)</div><div class='add'>+{</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	irdma_get_wqe_shift(ukinfo-&gt;uk_attrs, ukinfo-&gt;max_rq_frag_cnt, 0,</div><div class='add'>+			    rq_shift);</div><div class='add'>+</div><div class='add'>+	if (ukinfo-&gt;uk_attrs-&gt;hw_rev == IRDMA_GEN_1) {</div><div class='add'>+		if (ukinfo-&gt;abi_ver &gt; 4)</div><div class='add'>+			*rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	status = irdma_get_rqdepth(ukinfo-&gt;uk_attrs, ukinfo-&gt;rq_size,</div><div class='add'>+				   *rq_shift, rq_depth);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_qp_init - initialize shared qp</div><div class='add'>+ * @qp: hw qp (user and kernel)</div><div class='add'>+ * @info: qp initialization info</div><div class='add'>+ *</div><div class='add'>+ * initializes the vars used in both user and kernel mode.</div><div class='add'>+ * size of the wqe depends on numbers of max. fragements</div><div class='add'>+ * allowed. Then size of wqe * the number of wqes should be the</div><div class='add'>+ * amount of memory allocated for sq and rq.</div><div class='add'>+ */</div><div class='add'>+int irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info)</div><div class='add'>+{</div><div class='add'>+	int ret_code = 0;</div><div class='add'>+	u32 sq_ring_size;</div><div class='add'>+</div><div class='add'>+	qp-&gt;uk_attrs = info-&gt;uk_attrs;</div><div class='add'>+	if (info-&gt;max_sq_frag_cnt &gt; qp-&gt;uk_attrs-&gt;max_hw_wq_frags ||</div><div class='add'>+	    info-&gt;max_rq_frag_cnt &gt; qp-&gt;uk_attrs-&gt;max_hw_wq_frags)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	qp-&gt;qp_caps = info-&gt;qp_caps;</div><div class='add'>+	qp-&gt;sq_base = info-&gt;sq;</div><div class='add'>+	qp-&gt;rq_base = info-&gt;rq;</div><div class='add'>+	qp-&gt;qp_type = info-&gt;type ? info-&gt;type : IRDMA_QP_TYPE_IWARP;</div><div class='add'>+	qp-&gt;shadow_area = info-&gt;shadow_area;</div><div class='add'>+	qp-&gt;sq_wrtrk_array = info-&gt;sq_wrtrk_array;</div><div class='add'>+</div><div class='add'>+	qp-&gt;rq_wrid_array = info-&gt;rq_wrid_array;</div><div class='add'>+	qp-&gt;wqe_alloc_db = info-&gt;wqe_alloc_db;</div><div class='add'>+	qp-&gt;qp_id = info-&gt;qp_id;</div><div class='add'>+	qp-&gt;sq_size = info-&gt;sq_size;</div><div class='add'>+	qp-&gt;max_sq_frag_cnt = info-&gt;max_sq_frag_cnt;</div><div class='add'>+	sq_ring_size = qp-&gt;sq_size &lt;&lt; info-&gt;sq_shift;</div><div class='add'>+	IRDMA_RING_INIT(qp-&gt;sq_ring, sq_ring_size);</div><div class='add'>+	if (info-&gt;first_sq_wq) {</div><div class='add'>+		irdma_setup_connection_wqes(qp, info);</div><div class='add'>+		qp-&gt;swqe_polarity = 1;</div><div class='add'>+		qp-&gt;first_sq_wq = true;</div><div class='add'>+	} else {</div><div class='add'>+		qp-&gt;swqe_polarity = 0;</div><div class='add'>+	}</div><div class='add'>+	qp-&gt;swqe_polarity_deferred = 1;</div><div class='add'>+	qp-&gt;rwqe_polarity = 0;</div><div class='add'>+	qp-&gt;rq_size = info-&gt;rq_size;</div><div class='add'>+	qp-&gt;max_rq_frag_cnt = info-&gt;max_rq_frag_cnt;</div><div class='add'>+	qp-&gt;max_inline_data = info-&gt;max_inline_data;</div><div class='add'>+	qp-&gt;rq_wqe_size = info-&gt;rq_shift;</div><div class='add'>+	IRDMA_RING_INIT(qp-&gt;rq_ring, qp-&gt;rq_size);</div><div class='add'>+	qp-&gt;rq_wqe_size_multiplier = 1 &lt;&lt; info-&gt;rq_shift;</div><div class='add'>+	if (qp-&gt;uk_attrs-&gt;hw_rev == IRDMA_GEN_1)</div><div class='add'>+		qp-&gt;wqe_ops = iw_wqe_uk_ops_gen_1;</div><div class='add'>+	else</div><div class='add'>+		qp-&gt;wqe_ops = iw_wqe_uk_ops;</div><div class='add'>+	qp-&gt;srq_uk = info-&gt;srq_uk;</div><div class='add'>+	return ret_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_cq_init - initialize shared cq (user and kernel)</div><div class='add'>+ * @cq: hw cq</div><div class='add'>+ * @info: hw cq initialization info</div><div class='add'>+ */</div><div class='add'>+void irdma_uk_cq_init(struct irdma_cq_uk *cq,</div><div class='add'>+		      struct irdma_cq_uk_init_info *info)</div><div class='add'>+{</div><div class='add'>+	cq-&gt;cq_base = info-&gt;cq_base;</div><div class='add'>+	cq-&gt;cq_id = info-&gt;cq_id;</div><div class='add'>+	cq-&gt;cq_size = info-&gt;cq_size;</div><div class='add'>+	cq-&gt;cqe_alloc_db = info-&gt;cqe_alloc_db;</div><div class='add'>+	cq-&gt;cq_ack_db = info-&gt;cq_ack_db;</div><div class='add'>+	cq-&gt;shadow_area = info-&gt;shadow_area;</div><div class='add'>+	cq-&gt;avoid_mem_cflct = info-&gt;avoid_mem_cflct;</div><div class='add'>+	IRDMA_RING_INIT(cq-&gt;cq_ring, cq-&gt;cq_size);</div><div class='add'>+	cq-&gt;polarity = 1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_uk_clean_cq - clean cq entries</div><div class='add'>+ * @q: completion context</div><div class='add'>+ * @cq: cq to clean</div><div class='add'>+ */</div><div class='add'>+void irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq)</div><div class='add'>+{</div><div class='add'>+	__le64 *cqe;</div><div class='add'>+	u64 qword3, comp_ctx;</div><div class='add'>+	u32 cq_head;</div><div class='add'>+	u8 polarity, temp;</div><div class='add'>+</div><div class='add'>+	cq_head = cq-&gt;cq_ring.head;</div><div class='add'>+	temp = cq-&gt;polarity;</div><div class='add'>+	do {</div><div class='add'>+		if (cq-&gt;avoid_mem_cflct)</div><div class='add'>+			cqe = ((struct irdma_extended_cqe *)(cq-&gt;cq_base))[cq_head].buf;</div><div class='add'>+		else</div><div class='add'>+			cqe = cq-&gt;cq_base[cq_head].buf;</div><div class='add'>+		get_64bit_val(cqe, 24, &amp;qword3);</div><div class='add'>+		polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);</div><div class='add'>+</div><div class='add'>+		if (polarity != temp)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		/* Ensure CQE contents are read after valid bit is checked */</div><div class='add'>+		dma_rmb();</div><div class='add'>+</div><div class='add'>+		get_64bit_val(cqe, 8, &amp;comp_ctx);</div><div class='add'>+		if ((void *)(unsigned long)comp_ctx == q)</div><div class='add'>+			set_64bit_val(cqe, 8, 0);</div><div class='add'>+</div><div class='add'>+		cq_head = (cq_head + 1) % cq-&gt;cq_ring.size;</div><div class='add'>+		if (!cq_head)</div><div class='add'>+			temp ^= 1;</div><div class='add'>+	} while (true);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_nop - post a nop</div><div class='add'>+ * @qp: hw qp ptr</div><div class='add'>+ * @wr_id: work request id</div><div class='add'>+ * @signaled: signaled for completion</div><div class='add'>+ * @post_sq: ring doorbell</div><div class='add'>+ */</div><div class='add'>+int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq)</div><div class='add'>+{</div><div class='add'>+	__le64 *wqe;</div><div class='add'>+	u64 hdr;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+	struct irdma_post_sq_info info = {};</div><div class='add'>+</div><div class='add'>+	info.wr_id = wr_id;</div><div class='add'>+	wqe = irdma_qp_get_next_send_wqe(qp, &amp;wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,</div><div class='add'>+					 0, &amp;info);</div><div class='add'>+	if (!wqe)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	irdma_clr_wqes(qp, wqe_idx);</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 0, 0);</div><div class='add'>+	set_64bit_val(wqe, 8, 0);</div><div class='add'>+	set_64bit_val(wqe, 16, 0);</div><div class='add'>+</div><div class='add'>+	hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) |</div><div class='add'>+	      FIELD_PREP(IRDMAQPSQ_VALID, qp-&gt;swqe_polarity);</div><div class='add'>+</div><div class='add'>+	dma_wmb(); /* make sure WQE is populated before valid bit is set */</div><div class='add'>+</div><div class='add'>+	set_64bit_val(wqe, 24, hdr);</div><div class='add'>+	if (post_sq)</div><div class='add'>+		irdma_uk_qp_post_wr(qp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_fragcnt_to_quanta_sq - calculate quanta based on fragment count for SQ</div><div class='add'>+ * @frag_cnt: number of fragments</div><div class='add'>+ * @quanta: quanta for frag_cnt</div><div class='add'>+ */</div><div class='add'>+int irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta)</div><div class='add'>+{</div><div class='add'>+	switch (frag_cnt) {</div><div class='add'>+	case 0:</div><div class='add'>+	case 1:</div><div class='add'>+		*quanta = IRDMA_QP_WQE_MIN_QUANTA;</div><div class='add'>+		break;</div><div class='add'>+	case 2:</div><div class='add'>+	case 3:</div><div class='add'>+		*quanta = 2;</div><div class='add'>+		break;</div><div class='add'>+	case 4:</div><div class='add'>+	case 5:</div><div class='add'>+		*quanta = 3;</div><div class='add'>+		break;</div><div class='add'>+	case 6:</div><div class='add'>+	case 7:</div><div class='add'>+		*quanta = 4;</div><div class='add'>+		break;</div><div class='add'>+	case 8:</div><div class='add'>+	case 9:</div><div class='add'>+		*quanta = 5;</div><div class='add'>+		break;</div><div class='add'>+	case 10:</div><div class='add'>+	case 11:</div><div class='add'>+		*quanta = 6;</div><div class='add'>+		break;</div><div class='add'>+	case 12:</div><div class='add'>+	case 13:</div><div class='add'>+		*quanta = 7;</div><div class='add'>+		break;</div><div class='add'>+	case 14:</div><div class='add'>+	case 15: /* when immediate data is present */</div><div class='add'>+		*quanta = 8;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_fragcnt_to_wqesize_rq - calculate wqe size based on fragment count for RQ</div><div class='add'>+ * @frag_cnt: number of fragments</div><div class='add'>+ * @wqe_size: size in bytes given frag_cnt</div><div class='add'>+ */</div><div class='add'>+int irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size)</div><div class='add'>+{</div><div class='add'>+	switch (frag_cnt) {</div><div class='add'>+	case 0:</div><div class='add'>+	case 1:</div><div class='add'>+		*wqe_size = 32;</div><div class='add'>+		break;</div><div class='add'>+	case 2:</div><div class='add'>+	case 3:</div><div class='add'>+		*wqe_size = 64;</div><div class='add'>+		break;</div><div class='add'>+	case 4:</div><div class='add'>+	case 5:</div><div class='add'>+	case 6:</div><div class='add'>+	case 7:</div><div class='add'>+		*wqe_size = 128;</div><div class='add'>+		break;</div><div class='add'>+	case 8:</div><div class='add'>+	case 9:</div><div class='add'>+	case 10:</div><div class='add'>+	case 11:</div><div class='add'>+	case 12:</div><div class='add'>+	case 13:</div><div class='add'>+	case 14:</div><div class='add'>+		*wqe_size = 256;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/user.h b/drivers/infiniband/hw/irdma/user.h<br/>new file mode 100644<br/>index 000000000000..9eb7fd0b1cbf<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/user.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/user.h</a></div><div class='hunk'>@@ -0,0 +1,676 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2015 - 2020 Intel Corporation */</div><div class='add'>+#ifndef IRDMA_USER_H</div><div class='add'>+#define IRDMA_USER_H</div><div class='add'>+</div><div class='add'>+#define irdma_handle void *</div><div class='add'>+#define irdma_adapter_handle irdma_handle</div><div class='add'>+#define irdma_qp_handle irdma_handle</div><div class='add'>+#define irdma_cq_handle irdma_handle</div><div class='add'>+#define irdma_pd_id irdma_handle</div><div class='add'>+#define irdma_stag_handle irdma_handle</div><div class='add'>+#define irdma_stag_index u32</div><div class='add'>+#define irdma_stag u32</div><div class='add'>+#define irdma_stag_key u8</div><div class='add'>+#define irdma_tagged_offset u64</div><div class='add'>+#define irdma_access_privileges u32</div><div class='add'>+#define irdma_physical_fragment u64</div><div class='add'>+#define irdma_address_list u64 *</div><div class='add'>+</div><div class='add'>+#define	IRDMA_MAX_MR_SIZE       0x200000000000ULL</div><div class='add'>+</div><div class='add'>+#define IRDMA_ACCESS_FLAGS_LOCALREAD		0x01</div><div class='add'>+#define IRDMA_ACCESS_FLAGS_LOCALWRITE		0x02</div><div class='add'>+#define IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY	0x04</div><div class='add'>+#define IRDMA_ACCESS_FLAGS_REMOTEREAD		0x05</div><div class='add'>+#define IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY	0x08</div><div class='add'>+#define IRDMA_ACCESS_FLAGS_REMOTEWRITE		0x0a</div><div class='add'>+#define IRDMA_ACCESS_FLAGS_BIND_WINDOW		0x10</div><div class='add'>+#define IRDMA_ACCESS_FLAGS_ZERO_BASED		0x20</div><div class='add'>+#define IRDMA_ACCESS_FLAGS_ALL			0x3f</div><div class='add'>+</div><div class='add'>+#define IRDMA_OP_TYPE_RDMA_WRITE		0x00</div><div class='add'>+#define IRDMA_OP_TYPE_RDMA_READ			0x01</div><div class='add'>+#define IRDMA_OP_TYPE_SEND			0x03</div><div class='add'>+#define IRDMA_OP_TYPE_SEND_INV			0x04</div><div class='add'>+#define IRDMA_OP_TYPE_SEND_SOL			0x05</div><div class='add'>+#define IRDMA_OP_TYPE_SEND_SOL_INV		0x06</div><div class='add'>+#define IRDMA_OP_TYPE_RDMA_WRITE_SOL		0x0d</div><div class='add'>+#define IRDMA_OP_TYPE_BIND_MW			0x08</div><div class='add'>+#define IRDMA_OP_TYPE_FAST_REG_NSMR		0x09</div><div class='add'>+#define IRDMA_OP_TYPE_INV_STAG			0x0a</div><div class='add'>+#define IRDMA_OP_TYPE_RDMA_READ_INV_STAG	0x0b</div><div class='add'>+#define IRDMA_OP_TYPE_NOP			0x0c</div><div class='add'>+#define IRDMA_OP_TYPE_ATOMIC_FETCH_AND_ADD	0x0f</div><div class='add'>+#define IRDMA_OP_TYPE_ATOMIC_COMPARE_AND_SWAP	0x11</div><div class='add'>+#define IRDMA_OP_TYPE_REC	0x3e</div><div class='add'>+#define IRDMA_OP_TYPE_REC_IMM	0x3f</div><div class='add'>+</div><div class='add'>+#define IRDMA_FLUSH_MAJOR_ERR 1</div><div class='add'>+#define IRDMA_SRQFLUSH_RSVD_MAJOR_ERR 0xfffe</div><div class='add'>+</div><div class='add'>+/* Async Events codes */</div><div class='add'>+#define IRDMA_AE_AMP_UNALLOCATED_STAG					0x0102</div><div class='add'>+#define IRDMA_AE_AMP_INVALID_STAG					0x0103</div><div class='add'>+#define IRDMA_AE_AMP_BAD_QP						0x0104</div><div class='add'>+#define IRDMA_AE_AMP_BAD_PD						0x0105</div><div class='add'>+#define IRDMA_AE_AMP_BAD_STAG_KEY					0x0106</div><div class='add'>+#define IRDMA_AE_AMP_BAD_STAG_INDEX					0x0107</div><div class='add'>+#define IRDMA_AE_AMP_BOUNDS_VIOLATION					0x0108</div><div class='add'>+#define IRDMA_AE_AMP_RIGHTS_VIOLATION					0x0109</div><div class='add'>+#define IRDMA_AE_AMP_TO_WRAP						0x010a</div><div class='add'>+#define IRDMA_AE_AMP_FASTREG_VALID_STAG					0x010c</div><div class='add'>+#define IRDMA_AE_AMP_FASTREG_MW_STAG					0x010d</div><div class='add'>+#define IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS				0x010e</div><div class='add'>+#define IRDMA_AE_AMP_FASTREG_INVALID_LENGTH				0x0110</div><div class='add'>+#define IRDMA_AE_AMP_INVALIDATE_SHARED					0x0111</div><div class='add'>+#define IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS			0x0112</div><div class='add'>+#define IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS			0x0113</div><div class='add'>+#define IRDMA_AE_AMP_MWBIND_VALID_STAG					0x0114</div><div class='add'>+#define IRDMA_AE_AMP_MWBIND_OF_MR_STAG					0x0115</div><div class='add'>+#define IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG				0x0116</div><div class='add'>+#define IRDMA_AE_AMP_MWBIND_TO_MW_STAG					0x0117</div><div class='add'>+#define IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS				0x0118</div><div class='add'>+#define IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS				0x0119</div><div class='add'>+#define IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT				0x011a</div><div class='add'>+#define IRDMA_AE_AMP_MWBIND_BIND_DISABLED				0x011b</div><div class='add'>+#define IRDMA_AE_PRIV_OPERATION_DENIED					0x011c</div><div class='add'>+#define IRDMA_AE_AMP_INVALIDATE_TYPE1_MW				0x011d</div><div class='add'>+#define IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW				0x011e</div><div class='add'>+#define IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG			0x011f</div><div class='add'>+#define IRDMA_AE_AMP_MWBIND_WRONG_TYPE					0x0120</div><div class='add'>+#define IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH				0x0121</div><div class='add'>+#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG				0x0132</div><div class='add'>+#define IRDMA_AE_UDA_XMIT_BAD_PD					0x0133</div><div class='add'>+#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT				0x0134</div><div class='add'>+#define IRDMA_AE_UDA_L4LEN_INVALID					0x0135</div><div class='add'>+#define IRDMA_AE_BAD_CLOSE						0x0201</div><div class='add'>+#define IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE				0x0202</div><div class='add'>+#define IRDMA_AE_CQ_OPERATION_ERROR					0x0203</div><div class='add'>+#define IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO				0x0205</div><div class='add'>+#define IRDMA_AE_STAG_ZERO_INVALID					0x0206</div><div class='add'>+#define IRDMA_AE_IB_RREQ_AND_Q1_FULL					0x0207</div><div class='add'>+#define IRDMA_AE_IB_INVALID_REQUEST					0x0208</div><div class='add'>+#define IRDMA_AE_SRQ_LIMIT						0x0209</div><div class='add'>+#define IRDMA_AE_WQE_UNEXPECTED_OPCODE					0x020a</div><div class='add'>+#define IRDMA_AE_WQE_INVALID_PARAMETER					0x020b</div><div class='add'>+#define IRDMA_AE_WQE_INVALID_FRAG_DATA					0x020c</div><div class='add'>+#define IRDMA_AE_IB_REMOTE_ACCESS_ERROR					0x020d</div><div class='add'>+#define IRDMA_AE_IB_REMOTE_OP_ERROR					0x020e</div><div class='add'>+#define IRDMA_AE_SRQ_CATASTROPHIC_ERROR					0x020f</div><div class='add'>+#define IRDMA_AE_WQE_LSMM_TOO_LONG					0x0220</div><div class='add'>+#define IRDMA_AE_ATOMIC_ALIGNMENT					0x0221</div><div class='add'>+#define IRDMA_AE_ATOMIC_MASK						0x0222</div><div class='add'>+#define IRDMA_AE_INVALID_REQUEST					0x0223</div><div class='add'>+#define IRDMA_AE_PCIE_ATOMIC_DISABLE					0x0224</div><div class='add'>+#define IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN				0x0301</div><div class='add'>+#define IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER	0x0303</div><div class='add'>+#define IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION				0x0304</div><div class='add'>+#define IRDMA_AE_DDP_UBE_INVALID_MO					0x0305</div><div class='add'>+#define IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE		0x0306</div><div class='add'>+#define IRDMA_AE_DDP_UBE_INVALID_QN					0x0307</div><div class='add'>+#define IRDMA_AE_DDP_NO_L_BIT						0x0308</div><div class='add'>+#define IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION			0x0311</div><div class='add'>+#define IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE				0x0312</div><div class='add'>+#define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST				0x0313</div><div class='add'>+#define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP			0x0314</div><div class='add'>+#define IRDMA_AE_ROCE_RSP_LENGTH_ERROR					0x0316</div><div class='add'>+#define IRDMA_AE_ROCE_EMPTY_MCG						0x0380</div><div class='add'>+#define IRDMA_AE_ROCE_BAD_MC_IP_ADDR					0x0381</div><div class='add'>+#define IRDMA_AE_ROCE_BAD_MC_QPID					0x0382</div><div class='add'>+#define IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH				0x0383</div><div class='add'>+#define IRDMA_AE_INVALID_ARP_ENTRY					0x0401</div><div class='add'>+#define IRDMA_AE_INVALID_TCP_OPTION_RCVD				0x0402</div><div class='add'>+#define IRDMA_AE_STALE_ARP_ENTRY					0x0403</div><div class='add'>+#define IRDMA_AE_INVALID_AH_ENTRY					0x0406</div><div class='add'>+#define IRDMA_AE_LLP_CLOSE_COMPLETE					0x0501</div><div class='add'>+#define IRDMA_AE_LLP_CONNECTION_RESET					0x0502</div><div class='add'>+#define IRDMA_AE_LLP_FIN_RECEIVED					0x0503</div><div class='add'>+#define IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH	0x0504</div><div class='add'>+#define IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR				0x0505</div><div class='add'>+#define IRDMA_AE_LLP_SEGMENT_TOO_SMALL					0x0507</div><div class='add'>+#define IRDMA_AE_LLP_SYN_RECEIVED					0x0508</div><div class='add'>+#define IRDMA_AE_LLP_TERMINATE_RECEIVED					0x0509</div><div class='add'>+#define IRDMA_AE_LLP_TOO_MANY_RETRIES					0x050a</div><div class='add'>+#define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES				0x050b</div><div class='add'>+#define IRDMA_AE_LLP_DOUBT_REACHABILITY					0x050c</div><div class='add'>+#define IRDMA_AE_LLP_CONNECTION_ESTABLISHED				0x050e</div><div class='add'>+#define IRDMA_AE_LLP_TOO_MANY_RNRS					0x050f</div><div class='add'>+#define IRDMA_AE_RESOURCE_EXHAUSTION					0x0520</div><div class='add'>+#define IRDMA_AE_RESET_SENT						0x0601</div><div class='add'>+#define IRDMA_AE_TERMINATE_SENT						0x0602</div><div class='add'>+#define IRDMA_AE_RESET_NOT_SENT						0x0603</div><div class='add'>+#define IRDMA_AE_LCE_QP_CATASTROPHIC					0x0700</div><div class='add'>+#define IRDMA_AE_LCE_FUNCTION_CATASTROPHIC				0x0701</div><div class='add'>+#define IRDMA_AE_LCE_CQ_CATASTROPHIC					0x0702</div><div class='add'>+#define IRDMA_AE_REMOTE_QP_CATASTROPHIC					0x0703</div><div class='add'>+#define IRDMA_AE_LOCAL_QP_CATASTROPHIC					0x0704</div><div class='add'>+#define IRDMA_AE_RCE_QP_CATASTROPHIC					0x0705</div><div class='add'>+#define IRDMA_AE_QP_SUSPEND_COMPLETE					0x0900</div><div class='add'>+#define IRDMA_AE_CQP_DEFERRED_COMPLETE					0x0901</div><div class='add'>+#define IRDMA_AE_ADAPTER_CATASTROPHIC					0x0B0B</div><div class='add'>+</div><div class='add'>+enum irdma_device_caps_const {</div><div class='add'>+	IRDMA_WQE_SIZE =			4,</div><div class='add'>+	IRDMA_CQP_WQE_SIZE =			8,</div><div class='add'>+	IRDMA_CQE_SIZE =			4,</div><div class='add'>+	IRDMA_EXTENDED_CQE_SIZE =		8,</div><div class='add'>+	IRDMA_AEQE_SIZE =			2,</div><div class='add'>+	IRDMA_CEQE_SIZE =			1,</div><div class='add'>+	IRDMA_CQP_CTX_SIZE =			8,</div><div class='add'>+	IRDMA_SHADOW_AREA_SIZE =		8,</div><div class='add'>+	IRDMA_QUERY_FPM_BUF_SIZE =		192,</div><div class='add'>+	IRDMA_COMMIT_FPM_BUF_SIZE =		192,</div><div class='add'>+	IRDMA_GATHER_STATS_BUF_SIZE =		1024,</div><div class='add'>+	IRDMA_MIN_IW_QP_ID =			0,</div><div class='add'>+	IRDMA_MAX_IW_QP_ID =			262143,</div><div class='add'>+	IRDMA_MIN_IW_SRQ_ID =			0,</div><div class='add'>+	IRDMA_MIN_CEQID =			0,</div><div class='add'>+	IRDMA_MAX_CEQID =			1023,</div><div class='add'>+	IRDMA_CEQ_MAX_COUNT =			IRDMA_MAX_CEQID + 1,</div><div class='add'>+	IRDMA_MIN_CQID =			0,</div><div class='add'>+	IRDMA_MAX_CQID =			524287,</div><div class='add'>+	IRDMA_MIN_AEQ_ENTRIES =			1,</div><div class='add'>+	IRDMA_MAX_AEQ_ENTRIES =			524287,</div><div class='add'>+	IRDMA_MAX_AEQ_ENTRIES_GEN_3 =           262144,</div><div class='add'>+	IRDMA_MIN_CEQ_ENTRIES =			1,</div><div class='add'>+	IRDMA_MAX_CEQ_ENTRIES =			262143,</div><div class='add'>+	IRDMA_MIN_CQ_SIZE =			1,</div><div class='add'>+	IRDMA_MAX_CQ_SIZE =			1048575,</div><div class='add'>+	IRDMA_DB_ID_ZERO =			0,</div><div class='add'>+	IRDMA_MAX_WQ_FRAGMENT_COUNT =		13,</div><div class='add'>+	IRDMA_MAX_SGE_RD =			13,</div><div class='add'>+	IRDMA_MAX_OUTBOUND_MSG_SIZE =		2147483647,</div><div class='add'>+	IRDMA_MAX_INBOUND_MSG_SIZE =		2147483647,</div><div class='add'>+	IRDMA_MAX_PUSH_PAGE_COUNT =		1024,</div><div class='add'>+	IRDMA_MAX_PE_ENA_VF_COUNT =		32,</div><div class='add'>+	IRDMA_MAX_VF_FPM_ID =			47,</div><div class='add'>+	IRDMA_MAX_SQ_PAYLOAD_SIZE =		2145386496,</div><div class='add'>+	IRDMA_MAX_INLINE_DATA_SIZE =		101,</div><div class='add'>+	IRDMA_MAX_WQ_ENTRIES =			32768,</div><div class='add'>+	IRDMA_Q2_BUF_SIZE =			256,</div><div class='add'>+	IRDMA_QP_CTX_SIZE =			256,</div><div class='add'>+	IRDMA_MAX_PDS =				262144,</div><div class='add'>+	IRDMA_MIN_WQ_SIZE_GEN2 =                8,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_addressing_type {</div><div class='add'>+	IRDMA_ADDR_TYPE_ZERO_BASED = 0,</div><div class='add'>+	IRDMA_ADDR_TYPE_VA_BASED   = 1,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_flush_opcode {</div><div class='add'>+	FLUSH_INVALID = 0,</div><div class='add'>+	FLUSH_GENERAL_ERR,</div><div class='add'>+	FLUSH_PROT_ERR,</div><div class='add'>+	FLUSH_REM_ACCESS_ERR,</div><div class='add'>+	FLUSH_LOC_QP_OP_ERR,</div><div class='add'>+	FLUSH_REM_OP_ERR,</div><div class='add'>+	FLUSH_LOC_LEN_ERR,</div><div class='add'>+	FLUSH_FATAL_ERR,</div><div class='add'>+	FLUSH_RETRY_EXC_ERR,</div><div class='add'>+	FLUSH_MW_BIND_ERR,</div><div class='add'>+	FLUSH_REM_INV_REQ_ERR,</div><div class='add'>+	FLUSH_RNR_RETRY_EXC_ERR,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_qp_event_type {</div><div class='add'>+	IRDMA_QP_EVENT_CATASTROPHIC,</div><div class='add'>+	IRDMA_QP_EVENT_ACCESS_ERR,</div><div class='add'>+	IRDMA_QP_EVENT_REQ_ERR,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_cmpl_status {</div><div class='add'>+	IRDMA_COMPL_STATUS_SUCCESS = 0,</div><div class='add'>+	IRDMA_COMPL_STATUS_FLUSHED,</div><div class='add'>+	IRDMA_COMPL_STATUS_INVALID_WQE,</div><div class='add'>+	IRDMA_COMPL_STATUS_QP_CATASTROPHIC,</div><div class='add'>+	IRDMA_COMPL_STATUS_REMOTE_TERMINATION,</div><div class='add'>+	IRDMA_COMPL_STATUS_INVALID_STAG,</div><div class='add'>+	IRDMA_COMPL_STATUS_BASE_BOUND_VIOLATION,</div><div class='add'>+	IRDMA_COMPL_STATUS_ACCESS_VIOLATION,</div><div class='add'>+	IRDMA_COMPL_STATUS_INVALID_PD_ID,</div><div class='add'>+	IRDMA_COMPL_STATUS_WRAP_ERROR,</div><div class='add'>+	IRDMA_COMPL_STATUS_STAG_INVALID_PDID,</div><div class='add'>+	IRDMA_COMPL_STATUS_RDMA_READ_ZERO_ORD,</div><div class='add'>+	IRDMA_COMPL_STATUS_QP_NOT_PRIVLEDGED,</div><div class='add'>+	IRDMA_COMPL_STATUS_STAG_NOT_INVALID,</div><div class='add'>+	IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_SIZE,</div><div class='add'>+	IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_ENTRY,</div><div class='add'>+	IRDMA_COMPL_STATUS_INVALID_FBO,</div><div class='add'>+	IRDMA_COMPL_STATUS_INVALID_LEN,</div><div class='add'>+	IRDMA_COMPL_STATUS_INVALID_ACCESS,</div><div class='add'>+	IRDMA_COMPL_STATUS_PHYS_BUF_LIST_TOO_LONG,</div><div class='add'>+	IRDMA_COMPL_STATUS_INVALID_VIRT_ADDRESS,</div><div class='add'>+	IRDMA_COMPL_STATUS_INVALID_REGION,</div><div class='add'>+	IRDMA_COMPL_STATUS_INVALID_WINDOW,</div><div class='add'>+	IRDMA_COMPL_STATUS_INVALID_TOTAL_LEN,</div><div class='add'>+	IRDMA_COMPL_STATUS_UNKNOWN,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_cmpl_notify {</div><div class='add'>+	IRDMA_CQ_COMPL_EVENT     = 0,</div><div class='add'>+	IRDMA_CQ_COMPL_SOLICITED = 1,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_qp_caps {</div><div class='add'>+	IRDMA_WRITE_WITH_IMM = 1,</div><div class='add'>+	IRDMA_SEND_WITH_IMM  = 2,</div><div class='add'>+	IRDMA_ROCE	     = 4,</div><div class='add'>+	IRDMA_PUSH_MODE      = 8,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_srq_uk;</div><div class='add'>+struct irdma_srq_uk_init_info;</div><div class='add'>+struct irdma_qp_uk;</div><div class='add'>+struct irdma_cq_uk;</div><div class='add'>+struct irdma_qp_uk_init_info;</div><div class='add'>+struct irdma_cq_uk_init_info;</div><div class='add'>+</div><div class='add'>+struct irdma_ring {</div><div class='add'>+	u32 head;</div><div class='add'>+	u32 tail;</div><div class='add'>+	u32 size;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cqe {</div><div class='add'>+	__le64 buf[IRDMA_CQE_SIZE];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_extended_cqe {</div><div class='add'>+	__le64 buf[IRDMA_EXTENDED_CQE_SIZE];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_post_send {</div><div class='add'>+	struct ib_sge *sg_list;</div><div class='add'>+	u32 num_sges;</div><div class='add'>+	u32 qkey;</div><div class='add'>+	u32 dest_qp;</div><div class='add'>+	u32 ah_id;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_post_rq_info {</div><div class='add'>+	u64 wr_id;</div><div class='add'>+	struct ib_sge *sg_list;</div><div class='add'>+	u32 num_sges;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_rdma_write {</div><div class='add'>+	struct ib_sge *lo_sg_list;</div><div class='add'>+	u32 num_lo_sges;</div><div class='add'>+	struct ib_sge rem_addr;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_rdma_read {</div><div class='add'>+	struct ib_sge *lo_sg_list;</div><div class='add'>+	u32 num_lo_sges;</div><div class='add'>+	struct ib_sge rem_addr;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_bind_window {</div><div class='add'>+	irdma_stag mr_stag;</div><div class='add'>+	u64 bind_len;</div><div class='add'>+	void *va;</div><div class='add'>+	enum irdma_addressing_type addressing_type;</div><div class='add'>+	bool ena_reads:1;</div><div class='add'>+	bool ena_writes:1;</div><div class='add'>+	irdma_stag mw_stag;</div><div class='add'>+	bool mem_window_type_1:1;</div><div class='add'>+	bool remote_atomics_en:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_atomic_fetch_add {</div><div class='add'>+	u64 tagged_offset;</div><div class='add'>+	u64 remote_tagged_offset;</div><div class='add'>+	u64 fetch_add_data_bytes;</div><div class='add'>+	u32 stag;</div><div class='add'>+	u32 remote_stag;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_atomic_compare_swap {</div><div class='add'>+	u64 tagged_offset;</div><div class='add'>+	u64 remote_tagged_offset;</div><div class='add'>+	u64 swap_data_bytes;</div><div class='add'>+	u64 compare_data_bytes;</div><div class='add'>+	u32 stag;</div><div class='add'>+	u32 remote_stag;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_inv_local_stag {</div><div class='add'>+	irdma_stag target_stag;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_post_sq_info {</div><div class='add'>+	u64 wr_id;</div><div class='add'>+	u8 op_type;</div><div class='add'>+	u8 l4len;</div><div class='add'>+	bool signaled:1;</div><div class='add'>+	bool read_fence:1;</div><div class='add'>+	bool local_fence:1;</div><div class='add'>+	bool inline_data:1;</div><div class='add'>+	bool imm_data_valid:1;</div><div class='add'>+	bool report_rtt:1;</div><div class='add'>+	bool udp_hdr:1;</div><div class='add'>+	bool defer_flag:1;</div><div class='add'>+	bool remote_atomic_en:1;</div><div class='add'>+	u32 imm_data;</div><div class='add'>+	u32 stag_to_inv;</div><div class='add'>+	union {</div><div class='add'>+		struct irdma_post_send send;</div><div class='add'>+		struct irdma_rdma_write rdma_write;</div><div class='add'>+		struct irdma_rdma_read rdma_read;</div><div class='add'>+		struct irdma_bind_window bind_window;</div><div class='add'>+		struct irdma_inv_local_stag inv_local_stag;</div><div class='add'>+		struct irdma_atomic_fetch_add atomic_fetch_add;</div><div class='add'>+		struct irdma_atomic_compare_swap atomic_compare_swap;</div><div class='add'>+	} op;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cq_poll_info {</div><div class='add'>+	u64 wr_id;</div><div class='add'>+	irdma_qp_handle qp_handle;</div><div class='add'>+	u32 bytes_xfered;</div><div class='add'>+	u32 tcp_seq_num_rtt;</div><div class='add'>+	u32 qp_id;</div><div class='add'>+	u32 ud_src_qpn;</div><div class='add'>+	u32 imm_data;</div><div class='add'>+	irdma_stag inv_stag; /* or L_R_Key */</div><div class='add'>+	enum irdma_cmpl_status comp_status;</div><div class='add'>+	u16 major_err;</div><div class='add'>+	u16 minor_err;</div><div class='add'>+	u16 ud_vlan;</div><div class='add'>+	u8 ud_smac[6];</div><div class='add'>+	u8 op_type;</div><div class='add'>+	u8 q_type;</div><div class='add'>+	bool stag_invalid_set:1; /* or L_R_Key set */</div><div class='add'>+	bool error:1;</div><div class='add'>+	bool solicited_event:1;</div><div class='add'>+	bool ipv4:1;</div><div class='add'>+	bool ud_vlan_valid:1;</div><div class='add'>+	bool ud_smac_valid:1;</div><div class='add'>+	bool imm_valid:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct qp_err_code {</div><div class='add'>+	enum irdma_flush_opcode flush_code;</div><div class='add'>+	enum irdma_qp_event_type event_type;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+int irdma_uk_atomic_compare_swap(struct irdma_qp_uk *qp,</div><div class='add'>+				 struct irdma_post_sq_info *info, bool post_sq);</div><div class='add'>+int irdma_uk_atomic_fetch_add(struct irdma_qp_uk *qp,</div><div class='add'>+			      struct irdma_post_sq_info *info, bool post_sq);</div><div class='add'>+int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,</div><div class='add'>+			       struct irdma_post_sq_info *info, bool post_sq);</div><div class='add'>+int irdma_uk_inline_send(struct irdma_qp_uk *qp,</div><div class='add'>+			 struct irdma_post_sq_info *info, bool post_sq);</div><div class='add'>+int irdma_uk_post_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled,</div><div class='add'>+		      bool post_sq);</div><div class='add'>+int irdma_uk_post_receive(struct irdma_qp_uk *qp,</div><div class='add'>+			  struct irdma_post_rq_info *info);</div><div class='add'>+void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp);</div><div class='add'>+int irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,</div><div class='add'>+		       bool inv_stag, bool post_sq);</div><div class='add'>+int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,</div><div class='add'>+			bool post_sq);</div><div class='add'>+int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,</div><div class='add'>+		  bool post_sq);</div><div class='add'>+int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,</div><div class='add'>+				   struct irdma_post_sq_info *info,</div><div class='add'>+				   bool post_sq);</div><div class='add'>+</div><div class='add'>+struct irdma_wqe_uk_ops {</div><div class='add'>+	void (*iw_copy_inline_data)(u8 *dest, struct ib_sge *sge_list,</div><div class='add'>+				    u32 num_sges, u8 polarity);</div><div class='add'>+	u16 (*iw_inline_data_size_to_quanta)(u32 data_size);</div><div class='add'>+	void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct ib_sge *sge,</div><div class='add'>+				u8 valid);</div><div class='add'>+	void (*iw_set_mw_bind_wqe)(__le64 *wqe,</div><div class='add'>+				   struct irdma_bind_window *op_info);</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+bool irdma_uk_cq_empty(struct irdma_cq_uk *cq);</div><div class='add'>+int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,</div><div class='add'>+			  struct irdma_cq_poll_info *info);</div><div class='add'>+void irdma_uk_cq_request_notification(struct irdma_cq_uk *cq,</div><div class='add'>+				      enum irdma_cmpl_notify cq_notify);</div><div class='add'>+void irdma_uk_cq_resize(struct irdma_cq_uk *cq, void *cq_base, int size);</div><div class='add'>+void irdma_uk_cq_set_resized_cnt(struct irdma_cq_uk *qp, u16 cnt);</div><div class='add'>+void irdma_uk_cq_init(struct irdma_cq_uk *cq,</div><div class='add'>+		      struct irdma_cq_uk_init_info *info);</div><div class='add'>+int irdma_uk_qp_init(struct irdma_qp_uk *qp,</div><div class='add'>+		     struct irdma_qp_uk_init_info *info);</div><div class='add'>+void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,</div><div class='add'>+			    u8 *rq_shift);</div><div class='add'>+int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,</div><div class='add'>+				 u32 *sq_depth, u8 *sq_shift);</div><div class='add'>+int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo,</div><div class='add'>+				 u32 *rq_depth, u8 *rq_shift);</div><div class='add'>+int irdma_uk_srq_init(struct irdma_srq_uk *srq,</div><div class='add'>+		      struct irdma_srq_uk_init_info *info);</div><div class='add'>+int irdma_uk_srq_post_receive(struct irdma_srq_uk *srq,</div><div class='add'>+			      struct irdma_post_rq_info *info);</div><div class='add'>+</div><div class='add'>+struct irdma_srq_uk {</div><div class='add'>+	u32 srq_caps;</div><div class='add'>+	struct irdma_qp_quanta *srq_base;</div><div class='add'>+	struct irdma_uk_attrs *uk_attrs;</div><div class='add'>+	__le64 *shadow_area;</div><div class='add'>+	struct irdma_ring srq_ring;</div><div class='add'>+	u32 srq_id;</div><div class='add'>+	u32 srq_size;</div><div class='add'>+	u32 max_srq_frag_cnt;</div><div class='add'>+	struct irdma_wqe_uk_ops wqe_ops;</div><div class='add'>+	u8 srwqe_polarity;</div><div class='add'>+	u8 wqe_size;</div><div class='add'>+	u8 wqe_size_multiplier;</div><div class='add'>+	u8 deferred_flag;</div><div class='add'>+	spinlock_t *lock;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_srq_uk_init_info {</div><div class='add'>+	struct irdma_qp_quanta *srq;</div><div class='add'>+	struct irdma_uk_attrs *uk_attrs;</div><div class='add'>+	__le64 *shadow_area;</div><div class='add'>+	u64 *srq_wrid_array;</div><div class='add'>+	u32 srq_id;</div><div class='add'>+	u32 srq_caps;</div><div class='add'>+	u32 srq_size;</div><div class='add'>+	u32 max_srq_frag_cnt;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_sq_uk_wr_trk_info {</div><div class='add'>+	u64 wrid;</div><div class='add'>+	u32 wr_len;</div><div class='add'>+	u16 quanta;</div><div class='add'>+	u8 signaled;</div><div class='add'>+	u8 reserved[1];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_qp_quanta {</div><div class='add'>+	__le64 elem[IRDMA_WQE_SIZE];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_qp_uk {</div><div class='add'>+	struct irdma_qp_quanta *sq_base;</div><div class='add'>+	struct irdma_qp_quanta *rq_base;</div><div class='add'>+	struct irdma_uk_attrs *uk_attrs;</div><div class='add'>+	u32 __iomem *wqe_alloc_db;</div><div class='add'>+	struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;</div><div class='add'>+	u64 *rq_wrid_array;</div><div class='add'>+	__le64 *shadow_area;</div><div class='add'>+	struct irdma_ring sq_ring;</div><div class='add'>+	struct irdma_ring rq_ring;</div><div class='add'>+	struct irdma_ring initial_ring;</div><div class='add'>+	u32 qp_id;</div><div class='add'>+	u32 qp_caps;</div><div class='add'>+	u32 sq_size;</div><div class='add'>+	u32 rq_size;</div><div class='add'>+	u32 max_sq_frag_cnt;</div><div class='add'>+	u32 max_rq_frag_cnt;</div><div class='add'>+	u32 max_inline_data;</div><div class='add'>+	struct irdma_wqe_uk_ops wqe_ops;</div><div class='add'>+	u16 conn_wqes;</div><div class='add'>+	u8 qp_type;</div><div class='add'>+	u8 swqe_polarity;</div><div class='add'>+	u8 swqe_polarity_deferred;</div><div class='add'>+	u8 rwqe_polarity;</div><div class='add'>+	u8 rq_wqe_size;</div><div class='add'>+	u8 rq_wqe_size_multiplier;</div><div class='add'>+	bool deferred_flag:1;</div><div class='add'>+	bool first_sq_wq:1;</div><div class='add'>+	bool sq_flush_complete:1; /* Indicates flush was seen and SQ was empty after the flush */</div><div class='add'>+	bool rq_flush_complete:1; /* Indicates flush was seen and RQ was empty after the flush */</div><div class='add'>+	bool destroy_pending:1; /* Indicates the QP is being destroyed */</div><div class='add'>+	void *back_qp;</div><div class='add'>+	u8 dbg_rq_flushed;</div><div class='add'>+	struct irdma_srq_uk *srq_uk;</div><div class='add'>+	u8 sq_flush_seen;</div><div class='add'>+	u8 rq_flush_seen;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cq_uk {</div><div class='add'>+	struct irdma_cqe *cq_base;</div><div class='add'>+	u32 __iomem *cqe_alloc_db;</div><div class='add'>+	u32 __iomem *cq_ack_db;</div><div class='add'>+	__le64 *shadow_area;</div><div class='add'>+	u32 cq_id;</div><div class='add'>+	u32 cq_size;</div><div class='add'>+	struct irdma_ring cq_ring;</div><div class='add'>+	u8 polarity;</div><div class='add'>+	bool avoid_mem_cflct:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_qp_uk_init_info {</div><div class='add'>+	struct irdma_qp_quanta *sq;</div><div class='add'>+	struct irdma_qp_quanta *rq;</div><div class='add'>+	struct irdma_uk_attrs *uk_attrs;</div><div class='add'>+	u32 __iomem *wqe_alloc_db;</div><div class='add'>+	__le64 *shadow_area;</div><div class='add'>+	struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;</div><div class='add'>+	u64 *rq_wrid_array;</div><div class='add'>+	u32 qp_id;</div><div class='add'>+	u32 qp_caps;</div><div class='add'>+	u32 sq_size;</div><div class='add'>+	u32 rq_size;</div><div class='add'>+	u32 max_sq_frag_cnt;</div><div class='add'>+	u32 max_rq_frag_cnt;</div><div class='add'>+	u32 max_inline_data;</div><div class='add'>+	u32 sq_depth;</div><div class='add'>+	u32 rq_depth;</div><div class='add'>+	u8 first_sq_wq;</div><div class='add'>+	u8 type;</div><div class='add'>+	u8 sq_shift;</div><div class='add'>+	u8 rq_shift;</div><div class='add'>+	int abi_ver;</div><div class='add'>+	bool legacy_mode;</div><div class='add'>+	struct irdma_srq_uk *srq_uk;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cq_uk_init_info {</div><div class='add'>+	u32 __iomem *cqe_alloc_db;</div><div class='add'>+	u32 __iomem *cq_ack_db;</div><div class='add'>+	struct irdma_cqe *cq_base;</div><div class='add'>+	__le64 *shadow_area;</div><div class='add'>+	u32 cq_size;</div><div class='add'>+	u32 cq_id;</div><div class='add'>+	bool avoid_mem_cflct;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+__le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,</div><div class='add'>+				   u16 quanta, u32 total_size,</div><div class='add'>+				   struct irdma_post_sq_info *info);</div><div class='add'>+__le64 *irdma_srq_get_next_recv_wqe(struct irdma_srq_uk *srq, u32 *wqe_idx);</div><div class='add'>+__le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx);</div><div class='add'>+void irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq);</div><div class='add'>+int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq);</div><div class='add'>+int irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta);</div><div class='add'>+int irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size);</div><div class='add'>+void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,</div><div class='add'>+			 u32 inline_data, u8 *shift);</div><div class='add'>+int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift,</div><div class='add'>+		      u32 *wqdepth);</div><div class='add'>+int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift,</div><div class='add'>+		      u32 *wqdepth);</div><div class='add'>+int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift,</div><div class='add'>+		       u32 *srqdepth);</div><div class='add'>+void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);</div><div class='add'>+</div><div class='add'>+static inline struct qp_err_code irdma_ae_to_qp_err_code(u16 ae_id)</div><div class='add'>+{</div><div class='add'>+	struct qp_err_code qp_err = {};</div><div class='add'>+</div><div class='add'>+	switch (ae_id) {</div><div class='add'>+	case IRDMA_AE_AMP_BOUNDS_VIOLATION:</div><div class='add'>+	case IRDMA_AE_AMP_INVALID_STAG:</div><div class='add'>+	case IRDMA_AE_AMP_RIGHTS_VIOLATION:</div><div class='add'>+	case IRDMA_AE_AMP_UNALLOCATED_STAG:</div><div class='add'>+	case IRDMA_AE_AMP_BAD_PD:</div><div class='add'>+	case IRDMA_AE_AMP_BAD_QP:</div><div class='add'>+	case IRDMA_AE_AMP_BAD_STAG_KEY:</div><div class='add'>+	case IRDMA_AE_AMP_BAD_STAG_INDEX:</div><div class='add'>+	case IRDMA_AE_AMP_TO_WRAP:</div><div class='add'>+	case IRDMA_AE_PRIV_OPERATION_DENIED:</div><div class='add'>+		qp_err.flush_code = FLUSH_PROT_ERR;</div><div class='add'>+		qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_UDA_XMIT_BAD_PD:</div><div class='add'>+	case IRDMA_AE_WQE_UNEXPECTED_OPCODE:</div><div class='add'>+		qp_err.flush_code = FLUSH_LOC_QP_OP_ERR;</div><div class='add'>+		qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT:</div><div class='add'>+	case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:</div><div class='add'>+	case IRDMA_AE_UDA_L4LEN_INVALID:</div><div class='add'>+	case IRDMA_AE_DDP_UBE_INVALID_MO:</div><div class='add'>+	case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:</div><div class='add'>+		qp_err.flush_code = FLUSH_LOC_LEN_ERR;</div><div class='add'>+		qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:</div><div class='add'>+	case IRDMA_AE_IB_REMOTE_ACCESS_ERROR:</div><div class='add'>+		qp_err.flush_code = FLUSH_REM_ACCESS_ERR;</div><div class='add'>+		qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS:</div><div class='add'>+	case IRDMA_AE_AMP_MWBIND_BIND_DISABLED:</div><div class='add'>+	case IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS:</div><div class='add'>+	case IRDMA_AE_AMP_MWBIND_VALID_STAG:</div><div class='add'>+		qp_err.flush_code = FLUSH_MW_BIND_ERR;</div><div class='add'>+		qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_LLP_TOO_MANY_RETRIES:</div><div class='add'>+		qp_err.flush_code = FLUSH_RETRY_EXC_ERR;</div><div class='add'>+		qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_IB_INVALID_REQUEST:</div><div class='add'>+		qp_err.flush_code = FLUSH_REM_INV_REQ_ERR;</div><div class='add'>+		qp_err.event_type = IRDMA_QP_EVENT_REQ_ERR;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:</div><div class='add'>+	case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:</div><div class='add'>+	case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:</div><div class='add'>+	case IRDMA_AE_IB_REMOTE_OP_ERROR:</div><div class='add'>+		qp_err.flush_code = FLUSH_REM_OP_ERR;</div><div class='add'>+		qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_LLP_TOO_MANY_RNRS:</div><div class='add'>+		qp_err.flush_code = FLUSH_RNR_RETRY_EXC_ERR;</div><div class='add'>+		qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_AE_LCE_QP_CATASTROPHIC:</div><div class='add'>+	case IRDMA_AE_REMOTE_QP_CATASTROPHIC:</div><div class='add'>+	case IRDMA_AE_LOCAL_QP_CATASTROPHIC:</div><div class='add'>+	case IRDMA_AE_RCE_QP_CATASTROPHIC:</div><div class='add'>+		qp_err.flush_code = FLUSH_FATAL_ERR;</div><div class='add'>+		qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		qp_err.flush_code = FLUSH_GENERAL_ERR;</div><div class='add'>+		qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return qp_err;</div><div class='add'>+}</div><div class='add'>+#endif /* IRDMA_USER_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c<br/>new file mode 100644<br/>index 000000000000..cc2a12f735d3<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/utils.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/utils.c</a></div><div class='hunk'>@@ -0,0 +1,2508 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#include "main.h"</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_arp_table -manage arp table</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @ip_addr: ip address for device</div><div class='add'>+ * @ipv4: IPv4 flag</div><div class='add'>+ * @mac_addr: mac address ptr</div><div class='add'>+ * @action: modify, delete or add</div><div class='add'>+ */</div><div class='add'>+int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4,</div><div class='add'>+		    const u8 *mac_addr, u32 action)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int arp_index;</div><div class='add'>+	u32 ip[4] = {};</div><div class='add'>+</div><div class='add'>+	if (ipv4)</div><div class='add'>+		ip[0] = *ip_addr;</div><div class='add'>+	else</div><div class='add'>+		memcpy(ip, ip_addr, sizeof(ip));</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;rf-&gt;arp_lock, flags);</div><div class='add'>+	for (arp_index = 0; (u32)arp_index &lt; rf-&gt;arp_table_size; arp_index++) {</div><div class='add'>+		if (!memcmp(rf-&gt;arp_table[arp_index].ip_addr, ip, sizeof(ip)))</div><div class='add'>+			break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (action) {</div><div class='add'>+	case IRDMA_ARP_ADD:</div><div class='add'>+		if (arp_index != rf-&gt;arp_table_size) {</div><div class='add'>+			arp_index = -1;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		arp_index = 0;</div><div class='add'>+		if (irdma_alloc_rsrc(rf, rf-&gt;allocated_arps, rf-&gt;arp_table_size,</div><div class='add'>+				     (u32 *)&amp;arp_index, &amp;rf-&gt;next_arp_index)) {</div><div class='add'>+			arp_index = -1;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		memcpy(rf-&gt;arp_table[arp_index].ip_addr, ip,</div><div class='add'>+		       sizeof(rf-&gt;arp_table[arp_index].ip_addr));</div><div class='add'>+		ether_addr_copy(rf-&gt;arp_table[arp_index].mac_addr, mac_addr);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_ARP_RESOLVE:</div><div class='add'>+		if (arp_index == rf-&gt;arp_table_size)</div><div class='add'>+			arp_index = -1;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_ARP_DELETE:</div><div class='add'>+		if (arp_index == rf-&gt;arp_table_size) {</div><div class='add'>+			arp_index = -1;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		memset(rf-&gt;arp_table[arp_index].ip_addr, 0,</div><div class='add'>+		       sizeof(rf-&gt;arp_table[arp_index].ip_addr));</div><div class='add'>+		eth_zero_addr(rf-&gt;arp_table[arp_index].mac_addr);</div><div class='add'>+		irdma_free_rsrc(rf, rf-&gt;allocated_arps, arp_index);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		arp_index = -1;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_unlock_irqrestore(&amp;rf-&gt;arp_lock, flags);</div><div class='add'>+	return arp_index;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_add_arp - add a new arp entry if needed</div><div class='add'>+ * @rf: RDMA function</div><div class='add'>+ * @ip: IP address</div><div class='add'>+ * @ipv4: IPv4 flag</div><div class='add'>+ * @mac: MAC address</div><div class='add'>+ */</div><div class='add'>+int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, const u8 *mac)</div><div class='add'>+{</div><div class='add'>+	int arpidx;</div><div class='add'>+</div><div class='add'>+	arpidx = irdma_arp_table(rf, &amp;ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE);</div><div class='add'>+	if (arpidx &gt;= 0) {</div><div class='add'>+		if (ether_addr_equal(rf-&gt;arp_table[arpidx].mac_addr, mac))</div><div class='add'>+			return arpidx;</div><div class='add'>+</div><div class='add'>+		irdma_manage_arp_cache(rf, rf-&gt;arp_table[arpidx].mac_addr, ip,</div><div class='add'>+				       ipv4, IRDMA_ARP_DELETE);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_manage_arp_cache(rf, mac, ip, ipv4, IRDMA_ARP_ADD);</div><div class='add'>+</div><div class='add'>+	return irdma_arp_table(rf, ip, ipv4, NULL, IRDMA_ARP_RESOLVE);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * wr32 - write 32 bits to hw register</div><div class='add'>+ * @hw: hardware information including registers</div><div class='add'>+ * @reg: register offset</div><div class='add'>+ * @val: value to write to register</div><div class='add'>+ */</div><div class='add'>+inline void wr32(struct irdma_hw *hw, u32 reg, u32 val)</div><div class='add'>+{</div><div class='add'>+	writel(val, hw-&gt;hw_addr + reg);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * rd32 - read a 32 bit hw register</div><div class='add'>+ * @hw: hardware information including registers</div><div class='add'>+ * @reg: register offset</div><div class='add'>+ *</div><div class='add'>+ * Return value of register content</div><div class='add'>+ */</div><div class='add'>+inline u32 rd32(struct irdma_hw *hw, u32 reg)</div><div class='add'>+{</div><div class='add'>+	return readl(hw-&gt;hw_addr + reg);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * rd64 - read a 64 bit hw register</div><div class='add'>+ * @hw: hardware information including registers</div><div class='add'>+ * @reg: register offset</div><div class='add'>+ *</div><div class='add'>+ * Return value of register content</div><div class='add'>+ */</div><div class='add'>+inline u64 rd64(struct irdma_hw *hw, u32 reg)</div><div class='add'>+{</div><div class='add'>+	return readq(hw-&gt;hw_addr + reg);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_gid_change_event(struct ib_device *ibdev)</div><div class='add'>+{</div><div class='add'>+	struct ib_event ib_event;</div><div class='add'>+</div><div class='add'>+	ib_event.event = IB_EVENT_GID_CHANGE;</div><div class='add'>+	ib_event.device = ibdev;</div><div class='add'>+	ib_event.element.port_num = 1;</div><div class='add'>+	ib_dispatch_event(&amp;ib_event);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_inetaddr_event - system notifier for ipv4 addr events</div><div class='add'>+ * @notifier: not used</div><div class='add'>+ * @event: event for notifier</div><div class='add'>+ * @ptr: if address</div><div class='add'>+ */</div><div class='add'>+int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event,</div><div class='add'>+			 void *ptr)</div><div class='add'>+{</div><div class='add'>+	struct in_ifaddr *ifa = ptr;</div><div class='add'>+	struct net_device *real_dev, *netdev = ifa-&gt;ifa_dev-&gt;dev;</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct ib_device *ibdev;</div><div class='add'>+	u32 local_ipaddr;</div><div class='add'>+</div><div class='add'>+	real_dev = rdma_vlan_dev_real_dev(netdev);</div><div class='add'>+	if (!real_dev)</div><div class='add'>+		real_dev = netdev;</div><div class='add'>+</div><div class='add'>+	ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);</div><div class='add'>+	if (!ibdev)</div><div class='add'>+		return NOTIFY_DONE;</div><div class='add'>+</div><div class='add'>+	iwdev = to_iwdev(ibdev);</div><div class='add'>+	local_ipaddr = ntohl(ifa-&gt;ifa_address);</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+		  "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", real_dev,</div><div class='add'>+		  event, &amp;local_ipaddr, real_dev-&gt;dev_addr);</div><div class='add'>+	switch (event) {</div><div class='add'>+	case NETDEV_DOWN:</div><div class='add'>+		irdma_manage_arp_cache(iwdev-&gt;rf, real_dev-&gt;dev_addr,</div><div class='add'>+				       &amp;local_ipaddr, true, IRDMA_ARP_DELETE);</div><div class='add'>+		irdma_if_notify(iwdev, real_dev, &amp;local_ipaddr, true, false);</div><div class='add'>+		irdma_gid_change_event(&amp;iwdev-&gt;ibdev);</div><div class='add'>+		break;</div><div class='add'>+	case NETDEV_UP:</div><div class='add'>+	case NETDEV_CHANGEADDR:</div><div class='add'>+		irdma_add_arp(iwdev-&gt;rf, &amp;local_ipaddr, true, real_dev-&gt;dev_addr);</div><div class='add'>+		irdma_if_notify(iwdev, real_dev, &amp;local_ipaddr, true, true);</div><div class='add'>+		irdma_gid_change_event(&amp;iwdev-&gt;ibdev);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ib_device_put(ibdev);</div><div class='add'>+</div><div class='add'>+	return NOTIFY_DONE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_inet6addr_event - system notifier for ipv6 addr events</div><div class='add'>+ * @notifier: not used</div><div class='add'>+ * @event: event for notifier</div><div class='add'>+ * @ptr: if address</div><div class='add'>+ */</div><div class='add'>+int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event,</div><div class='add'>+			  void *ptr)</div><div class='add'>+{</div><div class='add'>+	struct inet6_ifaddr *ifa = ptr;</div><div class='add'>+	struct net_device *real_dev, *netdev = ifa-&gt;idev-&gt;dev;</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct ib_device *ibdev;</div><div class='add'>+	u32 local_ipaddr6[4];</div><div class='add'>+</div><div class='add'>+	real_dev = rdma_vlan_dev_real_dev(netdev);</div><div class='add'>+	if (!real_dev)</div><div class='add'>+		real_dev = netdev;</div><div class='add'>+</div><div class='add'>+	ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);</div><div class='add'>+	if (!ibdev)</div><div class='add'>+		return NOTIFY_DONE;</div><div class='add'>+</div><div class='add'>+	iwdev = to_iwdev(ibdev);</div><div class='add'>+	irdma_copy_ip_ntohl(local_ipaddr6, ifa-&gt;addr.in6_u.u6_addr32);</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+		  "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", real_dev,</div><div class='add'>+		  event, local_ipaddr6, real_dev-&gt;dev_addr);</div><div class='add'>+	switch (event) {</div><div class='add'>+	case NETDEV_DOWN:</div><div class='add'>+		irdma_manage_arp_cache(iwdev-&gt;rf, real_dev-&gt;dev_addr,</div><div class='add'>+				       local_ipaddr6, false, IRDMA_ARP_DELETE);</div><div class='add'>+		irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, false);</div><div class='add'>+		irdma_gid_change_event(&amp;iwdev-&gt;ibdev);</div><div class='add'>+		break;</div><div class='add'>+	case NETDEV_UP:</div><div class='add'>+	case NETDEV_CHANGEADDR:</div><div class='add'>+		irdma_add_arp(iwdev-&gt;rf, local_ipaddr6, false,</div><div class='add'>+			      real_dev-&gt;dev_addr);</div><div class='add'>+		irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, true);</div><div class='add'>+		irdma_gid_change_event(&amp;iwdev-&gt;ibdev);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ib_device_put(ibdev);</div><div class='add'>+</div><div class='add'>+	return NOTIFY_DONE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_net_event - system notifier for net events</div><div class='add'>+ * @notifier: not used</div><div class='add'>+ * @event: event for notifier</div><div class='add'>+ * @ptr: neighbor</div><div class='add'>+ */</div><div class='add'>+int irdma_net_event(struct notifier_block *notifier, unsigned long event,</div><div class='add'>+		    void *ptr)</div><div class='add'>+{</div><div class='add'>+	struct neighbour *neigh = ptr;</div><div class='add'>+	struct net_device *real_dev, *netdev = (struct net_device *)neigh-&gt;dev;</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct ib_device *ibdev;</div><div class='add'>+	__be32 *p;</div><div class='add'>+	u32 local_ipaddr[4] = {};</div><div class='add'>+	bool ipv4 = true;</div><div class='add'>+</div><div class='add'>+	switch (event) {</div><div class='add'>+	case NETEVENT_NEIGH_UPDATE:</div><div class='add'>+		real_dev = rdma_vlan_dev_real_dev(netdev);</div><div class='add'>+		if (!real_dev)</div><div class='add'>+			real_dev = netdev;</div><div class='add'>+		ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);</div><div class='add'>+		if (!ibdev)</div><div class='add'>+			return NOTIFY_DONE;</div><div class='add'>+</div><div class='add'>+		iwdev = to_iwdev(ibdev);</div><div class='add'>+		p = (__be32 *)neigh-&gt;primary_key;</div><div class='add'>+		if (neigh-&gt;tbl-&gt;family == AF_INET6) {</div><div class='add'>+			ipv4 = false;</div><div class='add'>+			irdma_copy_ip_ntohl(local_ipaddr, p);</div><div class='add'>+		} else {</div><div class='add'>+			local_ipaddr[0] = ntohl(*p);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+			  "DEV: netdev %p state %d local_ip=%pI4 MAC=%pM\n",</div><div class='add'>+			  iwdev-&gt;netdev, neigh-&gt;nud_state, local_ipaddr,</div><div class='add'>+			  neigh-&gt;ha);</div><div class='add'>+</div><div class='add'>+		if (neigh-&gt;nud_state &amp; NUD_VALID)</div><div class='add'>+			irdma_add_arp(iwdev-&gt;rf, local_ipaddr, ipv4, neigh-&gt;ha);</div><div class='add'>+</div><div class='add'>+		else</div><div class='add'>+			irdma_manage_arp_cache(iwdev-&gt;rf, neigh-&gt;ha,</div><div class='add'>+					       local_ipaddr, ipv4,</div><div class='add'>+					       IRDMA_ARP_DELETE);</div><div class='add'>+		ib_device_put(ibdev);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return NOTIFY_DONE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_netdevice_event - system notifier for netdev events</div><div class='add'>+ * @notifier: not used</div><div class='add'>+ * @event: event for notifier</div><div class='add'>+ * @ptr: netdev</div><div class='add'>+ */</div><div class='add'>+int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,</div><div class='add'>+			  void *ptr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct ib_device *ibdev;</div><div class='add'>+	struct net_device *netdev = netdev_notifier_info_to_dev(ptr);</div><div class='add'>+</div><div class='add'>+	ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);</div><div class='add'>+	if (!ibdev)</div><div class='add'>+		return NOTIFY_DONE;</div><div class='add'>+</div><div class='add'>+	iwdev = to_iwdev(ibdev);</div><div class='add'>+	iwdev-&gt;iw_status = 1;</div><div class='add'>+	switch (event) {</div><div class='add'>+	case NETDEV_DOWN:</div><div class='add'>+		iwdev-&gt;iw_status = 0;</div><div class='add'>+		fallthrough;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+	ib_device_put(ibdev);</div><div class='add'>+</div><div class='add'>+	return NOTIFY_DONE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_add_ipv6_addr - add ipv6 address to the hw arp table</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ */</div><div class='add'>+static void irdma_add_ipv6_addr(struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	struct net_device *ip_dev;</div><div class='add'>+	struct inet6_dev *idev;</div><div class='add'>+	struct inet6_ifaddr *ifp, *tmp;</div><div class='add'>+	u32 local_ipaddr6[4];</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	for_each_netdev_rcu (&amp;init_net, ip_dev) {</div><div class='add'>+		if (((rdma_vlan_dev_vlan_id(ip_dev) &lt; 0xFFFF &amp;&amp;</div><div class='add'>+		      rdma_vlan_dev_real_dev(ip_dev) == iwdev-&gt;netdev) ||</div><div class='add'>+		      ip_dev == iwdev-&gt;netdev) &amp;&amp;</div><div class='add'>+		      (READ_ONCE(ip_dev-&gt;flags) &amp; IFF_UP)) {</div><div class='add'>+			idev = __in6_dev_get(ip_dev);</div><div class='add'>+			if (!idev) {</div><div class='add'>+				ibdev_err(&amp;iwdev-&gt;ibdev, "ipv6 inet device not found\n");</div><div class='add'>+				break;</div><div class='add'>+			}</div><div class='add'>+			list_for_each_entry_safe (ifp, tmp, &amp;idev-&gt;addr_list,</div><div class='add'>+						  if_list) {</div><div class='add'>+				ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+					  "INIT: IP=%pI6, vlan_id=%d, MAC=%pM\n",</div><div class='add'>+					  &amp;ifp-&gt;addr,</div><div class='add'>+					  rdma_vlan_dev_vlan_id(ip_dev),</div><div class='add'>+					  ip_dev-&gt;dev_addr);</div><div class='add'>+</div><div class='add'>+				irdma_copy_ip_ntohl(local_ipaddr6,</div><div class='add'>+						    ifp-&gt;addr.in6_u.u6_addr32);</div><div class='add'>+				irdma_manage_arp_cache(iwdev-&gt;rf,</div><div class='add'>+						       ip_dev-&gt;dev_addr,</div><div class='add'>+						       local_ipaddr6, false,</div><div class='add'>+						       IRDMA_ARP_ADD);</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_add_ipv4_addr - add ipv4 address to the hw arp table</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ */</div><div class='add'>+static void irdma_add_ipv4_addr(struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	struct net_device *dev;</div><div class='add'>+	struct in_device *idev;</div><div class='add'>+	u32 ip_addr;</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	for_each_netdev_rcu (&amp;init_net, dev) {</div><div class='add'>+		if (((rdma_vlan_dev_vlan_id(dev) &lt; 0xFFFF &amp;&amp;</div><div class='add'>+		      rdma_vlan_dev_real_dev(dev) == iwdev-&gt;netdev) ||</div><div class='add'>+		      dev == iwdev-&gt;netdev) &amp;&amp; (READ_ONCE(dev-&gt;flags) &amp; IFF_UP)) {</div><div class='add'>+			const struct in_ifaddr *ifa;</div><div class='add'>+</div><div class='add'>+			idev = __in_dev_get_rcu(dev);</div><div class='add'>+			if (!idev)</div><div class='add'>+				continue;</div><div class='add'>+</div><div class='add'>+			in_dev_for_each_ifa_rcu(ifa, idev) {</div><div class='add'>+				ibdev_dbg(&amp;iwdev-&gt;ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n",</div><div class='add'>+					  &amp;ifa-&gt;ifa_address, rdma_vlan_dev_vlan_id(dev),</div><div class='add'>+					  dev-&gt;dev_addr);</div><div class='add'>+</div><div class='add'>+				ip_addr = ntohl(ifa-&gt;ifa_address);</div><div class='add'>+				irdma_manage_arp_cache(iwdev-&gt;rf, dev-&gt;dev_addr,</div><div class='add'>+						       &amp;ip_addr, true,</div><div class='add'>+						       IRDMA_ARP_ADD);</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_add_ip - add ip addresses</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ *</div><div class='add'>+ * Add ipv4/ipv6 addresses to the arp cache</div><div class='add'>+ */</div><div class='add'>+void irdma_add_ip(struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	irdma_add_ipv4_addr(iwdev);</div><div class='add'>+	irdma_add_ipv6_addr(iwdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_alloc_and_get_cqp_request - get cqp struct</div><div class='add'>+ * @cqp: device cqp ptr</div><div class='add'>+ * @wait: cqp to be used in wait mode</div><div class='add'>+ */</div><div class='add'>+struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp,</div><div class='add'>+							  bool wait)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cqp_request *cqp_request = NULL;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cqp-&gt;req_lock, flags);</div><div class='add'>+	if (!list_empty(&amp;cqp-&gt;cqp_avail_reqs)) {</div><div class='add'>+		cqp_request = list_first_entry(&amp;cqp-&gt;cqp_avail_reqs,</div><div class='add'>+					       struct irdma_cqp_request, list);</div><div class='add'>+		list_del_init(&amp;cqp_request-&gt;list);</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;cqp-&gt;req_lock, flags);</div><div class='add'>+	if (!cqp_request) {</div><div class='add'>+		cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);</div><div class='add'>+		if (cqp_request) {</div><div class='add'>+			cqp_request-&gt;dynamic = true;</div><div class='add'>+			if (wait)</div><div class='add'>+				init_waitqueue_head(&amp;cqp_request-&gt;waitq);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	if (!cqp_request) {</div><div class='add'>+		ibdev_dbg(to_ibdev(cqp-&gt;sc_cqp.dev), "ERR: CQP Request Fail: No Memory");</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cqp_request-&gt;waiting = wait;</div><div class='add'>+	refcount_set(&amp;cqp_request-&gt;refcnt, 1);</div><div class='add'>+	memset(&amp;cqp_request-&gt;compl_info, 0, sizeof(cqp_request-&gt;compl_info));</div><div class='add'>+	memset(&amp;cqp_request-&gt;info, 0, sizeof(cqp_request-&gt;info));</div><div class='add'>+</div><div class='add'>+	return cqp_request;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_cqp_request - increase refcount for cqp_request</div><div class='add'>+ * @cqp_request: pointer to cqp_request instance</div><div class='add'>+ */</div><div class='add'>+static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request)</div><div class='add'>+{</div><div class='add'>+	refcount_inc(&amp;cqp_request-&gt;refcnt);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_free_cqp_request - free cqp request</div><div class='add'>+ * @cqp: cqp ptr</div><div class='add'>+ * @cqp_request: to be put back in cqp list</div><div class='add'>+ */</div><div class='add'>+void irdma_free_cqp_request(struct irdma_cqp *cqp,</div><div class='add'>+			    struct irdma_cqp_request *cqp_request)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	if (cqp_request-&gt;dynamic) {</div><div class='add'>+		kfree(cqp_request);</div><div class='add'>+	} else {</div><div class='add'>+		WRITE_ONCE(cqp_request-&gt;request_done, false);</div><div class='add'>+		cqp_request-&gt;callback_fcn = NULL;</div><div class='add'>+		cqp_request-&gt;waiting = false;</div><div class='add'>+		cqp_request-&gt;pending = false;</div><div class='add'>+</div><div class='add'>+		spin_lock_irqsave(&amp;cqp-&gt;req_lock, flags);</div><div class='add'>+		list_add_tail(&amp;cqp_request-&gt;list, &amp;cqp-&gt;cqp_avail_reqs);</div><div class='add'>+		spin_unlock_irqrestore(&amp;cqp-&gt;req_lock, flags);</div><div class='add'>+	}</div><div class='add'>+	wake_up(&amp;cqp-&gt;remove_wq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_put_cqp_request - dec ref count and free if 0</div><div class='add'>+ * @cqp: cqp ptr</div><div class='add'>+ * @cqp_request: to be put back in cqp list</div><div class='add'>+ */</div><div class='add'>+void irdma_put_cqp_request(struct irdma_cqp *cqp,</div><div class='add'>+			   struct irdma_cqp_request *cqp_request)</div><div class='add'>+{</div><div class='add'>+	if (refcount_dec_and_test(&amp;cqp_request-&gt;refcnt))</div><div class='add'>+		irdma_free_cqp_request(cqp, cqp_request);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_free_pending_cqp_request -free pending cqp request objs</div><div class='add'>+ * @cqp: cqp ptr</div><div class='add'>+ * @cqp_request: to be put back in cqp list</div><div class='add'>+ */</div><div class='add'>+static void</div><div class='add'>+irdma_free_pending_cqp_request(struct irdma_cqp *cqp,</div><div class='add'>+			       struct irdma_cqp_request *cqp_request)</div><div class='add'>+{</div><div class='add'>+	if (cqp_request-&gt;waiting) {</div><div class='add'>+		cqp_request-&gt;compl_info.error = true;</div><div class='add'>+		WRITE_ONCE(cqp_request-&gt;request_done, true);</div><div class='add'>+		wake_up(&amp;cqp_request-&gt;waitq);</div><div class='add'>+	}</div><div class='add'>+	wait_event_timeout(cqp-&gt;remove_wq,</div><div class='add'>+			   refcount_read(&amp;cqp_request-&gt;refcnt) == 1, 1000);</div><div class='add'>+	irdma_put_cqp_request(cqp, cqp_request);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cleanup_deferred_cqp_ops - clean-up cqp with no completions</div><div class='add'>+ * @dev: sc_dev</div><div class='add'>+ * @cqp: cqp</div><div class='add'>+ */</div><div class='add'>+static void irdma_cleanup_deferred_cqp_ops(struct irdma_sc_dev *dev,</div><div class='add'>+					   struct irdma_cqp *cqp)</div><div class='add'>+{</div><div class='add'>+	u64 scratch;</div><div class='add'>+</div><div class='add'>+	/* process all CQP requests with deferred/pending completions */</div><div class='add'>+	while ((scratch = irdma_sc_cqp_cleanup_handler(dev)))</div><div class='add'>+		irdma_free_pending_cqp_request(cqp, (struct irdma_cqp_request *)</div><div class='add'>+						    (uintptr_t)scratch);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cleanup_pending_cqp_op - clean-up cqp with no</div><div class='add'>+ * completions</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ */</div><div class='add'>+void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_cqp *cqp = &amp;rf-&gt;cqp;</div><div class='add'>+	struct irdma_cqp_request *cqp_request = NULL;</div><div class='add'>+	struct cqp_cmds_info *pcmdinfo = NULL;</div><div class='add'>+	u32 i, pending_work, wqe_idx;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3)</div><div class='add'>+		irdma_cleanup_deferred_cqp_ops(dev, cqp);</div><div class='add'>+	pending_work = IRDMA_RING_USED_QUANTA(cqp-&gt;sc_cqp.sq_ring);</div><div class='add'>+	wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp-&gt;sc_cqp.sq_ring);</div><div class='add'>+	for (i = 0; i &lt; pending_work; i++) {</div><div class='add'>+		cqp_request = (struct irdma_cqp_request *)(unsigned long)</div><div class='add'>+				      cqp-&gt;scratch_array[wqe_idx];</div><div class='add'>+		if (cqp_request)</div><div class='add'>+			irdma_free_pending_cqp_request(cqp, cqp_request);</div><div class='add'>+		wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp-&gt;sc_cqp.sq_ring);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	while (!list_empty(&amp;dev-&gt;cqp_cmd_head)) {</div><div class='add'>+		pcmdinfo = irdma_remove_cqp_head(dev);</div><div class='add'>+		cqp_request =</div><div class='add'>+			container_of(pcmdinfo, struct irdma_cqp_request, info);</div><div class='add'>+		if (cqp_request)</div><div class='add'>+			irdma_free_pending_cqp_request(cqp, cqp_request);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_get_timeout_threshold(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	u16 time_s = dev-&gt;vc_caps.cqp_timeout_s;</div><div class='add'>+</div><div class='add'>+	if (!time_s)</div><div class='add'>+		return CQP_TIMEOUT_THRESHOLD;</div><div class='add'>+</div><div class='add'>+	return time_s * 1000 / dev-&gt;hw_attrs.max_cqp_compl_wait_time_ms;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_get_def_timeout_threshold(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	u16 time_s = dev-&gt;vc_caps.cqp_def_timeout_s;</div><div class='add'>+</div><div class='add'>+	if (!time_s)</div><div class='add'>+		return CQP_DEF_CMPL_TIMEOUT_THRESHOLD;</div><div class='add'>+</div><div class='add'>+	return time_s * 1000 / dev-&gt;hw_attrs.max_cqp_compl_wait_time_ms;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_wait_event - wait for completion</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @cqp_request: cqp request to wait</div><div class='add'>+ */</div><div class='add'>+static int irdma_wait_event(struct irdma_pci_f *rf,</div><div class='add'>+			    struct irdma_cqp_request *cqp_request)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cqp_timeout cqp_timeout = {};</div><div class='add'>+	int timeout_threshold = irdma_get_timeout_threshold(&amp;rf-&gt;sc_dev);</div><div class='add'>+	bool cqp_error = false;</div><div class='add'>+	int err_code = 0;</div><div class='add'>+</div><div class='add'>+	cqp_timeout.compl_cqp_cmds = atomic64_read(&amp;rf-&gt;sc_dev.cqp-&gt;completed_ops);</div><div class='add'>+	do {</div><div class='add'>+		irdma_cqp_ce_handler(rf, &amp;rf-&gt;ccq.sc_cq);</div><div class='add'>+		if (wait_event_timeout(cqp_request-&gt;waitq,</div><div class='add'>+				       READ_ONCE(cqp_request-&gt;request_done),</div><div class='add'>+				       msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS)))</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		if (cqp_request-&gt;pending)</div><div class='add'>+			/* There was a deferred or pending completion</div><div class='add'>+			 * received for this CQP request, so we need</div><div class='add'>+			 * to wait longer than usual.</div><div class='add'>+			 */</div><div class='add'>+			timeout_threshold =</div><div class='add'>+				irdma_get_def_timeout_threshold(&amp;rf-&gt;sc_dev);</div><div class='add'>+</div><div class='add'>+		irdma_check_cqp_progress(&amp;cqp_timeout, &amp;rf-&gt;sc_dev);</div><div class='add'>+</div><div class='add'>+		if (cqp_timeout.count &lt; timeout_threshold)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		if (!rf-&gt;reset) {</div><div class='add'>+			rf-&gt;reset = true;</div><div class='add'>+			rf-&gt;gen_ops.request_reset(rf);</div><div class='add'>+		}</div><div class='add'>+		return -ETIMEDOUT;</div><div class='add'>+	} while (1);</div><div class='add'>+</div><div class='add'>+	cqp_error = cqp_request-&gt;compl_info.error;</div><div class='add'>+	if (cqp_error) {</div><div class='add'>+		err_code = -EIO;</div><div class='add'>+		if (cqp_request-&gt;compl_info.maj_err_code == 0xFFFF) {</div><div class='add'>+			if (cqp_request-&gt;compl_info.min_err_code == 0x8002)</div><div class='add'>+				err_code = -EBUSY;</div><div class='add'>+			else if (cqp_request-&gt;compl_info.min_err_code == 0x8029) {</div><div class='add'>+				if (!rf-&gt;reset) {</div><div class='add'>+					rf-&gt;reset = true;</div><div class='add'>+					rf-&gt;gen_ops.request_reset(rf);</div><div class='add'>+				}</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return err_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = {</div><div class='add'>+	[IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd",</div><div class='add'>+	[IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd",</div><div class='add'>+	[IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd",</div><div class='add'>+	[IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd",</div><div class='add'>+	[IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd",</div><div class='add'>+	[IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd",</div><div class='add'>+	[IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd",</div><div class='add'>+	[IRDMA_OP_QP_MODIFY] = "Modify QP Cmd",</div><div class='add'>+	[IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd",</div><div class='add'>+	[IRDMA_OP_CQ_CREATE] = "Create CQ Cmd",</div><div class='add'>+	[IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd",</div><div class='add'>+	[IRDMA_OP_QP_CREATE] = "Create QP Cmd",</div><div class='add'>+	[IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd",</div><div class='add'>+	[IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd",</div><div class='add'>+	[IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd",</div><div class='add'>+	[IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd",</div><div class='add'>+	[IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd",</div><div class='add'>+	[IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd",</div><div class='add'>+	[IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd",</div><div class='add'>+	[IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd",</div><div class='add'>+	[IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd",</div><div class='add'>+	[IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd",</div><div class='add'>+	[IRDMA_OP_SUSPEND] = "Suspend QP Cmd",</div><div class='add'>+	[IRDMA_OP_RESUME] = "Resume QP Cmd",</div><div class='add'>+	[IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd",</div><div class='add'>+	[IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd",</div><div class='add'>+	[IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd",</div><div class='add'>+	[IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd",</div><div class='add'>+	[IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd",</div><div class='add'>+	[IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd",</div><div class='add'>+	[IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd",</div><div class='add'>+	[IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd",</div><div class='add'>+	[IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd",</div><div class='add'>+	[IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd",</div><div class='add'>+	[IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd",</div><div class='add'>+	[IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd",</div><div class='add'>+	[IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd",</div><div class='add'>+	[IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd",</div><div class='add'>+	[IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd",</div><div class='add'>+	[IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd",</div><div class='add'>+	[IRDMA_OP_GEN_AE] = "Generate AE Cmd",</div><div class='add'>+	[IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd",</div><div class='add'>+	[IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd",</div><div class='add'>+	[IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd",</div><div class='add'>+	[IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd",</div><div class='add'>+	[IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd",</div><div class='add'>+	[IRDMA_OP_SRQ_CREATE] = "Create SRQ Cmd",</div><div class='add'>+	[IRDMA_OP_SRQ_MODIFY] = "Modify SRQ Cmd",</div><div class='add'>+	[IRDMA_OP_SRQ_DESTROY] = "Destroy SRQ Cmd",</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = {</div><div class='add'>+	{0xffff, 0x8002, "Invalid State"},</div><div class='add'>+	{0xffff, 0x8006, "Flush No Wqe Pending"},</div><div class='add'>+	{0xffff, 0x8007, "Modify QP Bad Close"},</div><div class='add'>+	{0xffff, 0x8009, "LLP Closed"},</div><div class='add'>+	{0xffff, 0x800a, "Reset Not Sent"}</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_crit_err - check if CQP error is critical</div><div class='add'>+ * @dev: pointer to dev structure</div><div class='add'>+ * @cqp_cmd: code for last CQP operation</div><div class='add'>+ * @maj_err_code: major error code</div><div class='add'>+ * @min_err_code: minot error code</div><div class='add'>+ */</div><div class='add'>+bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd,</div><div class='add'>+			u16 maj_err_code, u16 min_err_code)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ARRAY_SIZE(irdma_noncrit_err_list); ++i) {</div><div class='add'>+		if (maj_err_code == irdma_noncrit_err_list[i].maj &amp;&amp;</div><div class='add'>+		    min_err_code == irdma_noncrit_err_list[i].min) {</div><div class='add'>+			ibdev_dbg(to_ibdev(dev),</div><div class='add'>+				  "CQP: [%s Error][%s] maj=0x%x min=0x%x\n",</div><div class='add'>+				  irdma_noncrit_err_list[i].desc,</div><div class='add'>+				  irdma_cqp_cmd_names[cqp_cmd], maj_err_code,</div><div class='add'>+				  min_err_code);</div><div class='add'>+			return false;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_handle_cqp_op - process cqp command</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @cqp_request: cqp request to process</div><div class='add'>+ */</div><div class='add'>+int irdma_handle_cqp_op(struct irdma_pci_f *rf,</div><div class='add'>+			struct irdma_cqp_request *cqp_request)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	struct cqp_cmds_info *info = &amp;cqp_request-&gt;info;</div><div class='add'>+	int status;</div><div class='add'>+	bool put_cqp_request = true;</div><div class='add'>+</div><div class='add'>+	if (rf-&gt;reset)</div><div class='add'>+		return -EBUSY;</div><div class='add'>+</div><div class='add'>+	irdma_get_cqp_request(cqp_request);</div><div class='add'>+	status = irdma_process_cqp_cmd(dev, info);</div><div class='add'>+	if (status)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	if (cqp_request-&gt;waiting) {</div><div class='add'>+		put_cqp_request = false;</div><div class='add'>+		status = irdma_wait_event(rf, cqp_request);</div><div class='add'>+		if (status)</div><div class='add'>+			goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	if (irdma_cqp_crit_err(dev, info-&gt;cqp_cmd,</div><div class='add'>+			       cqp_request-&gt;compl_info.maj_err_code,</div><div class='add'>+			       cqp_request-&gt;compl_info.min_err_code))</div><div class='add'>+		ibdev_err(&amp;rf-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n",</div><div class='add'>+			  irdma_cqp_cmd_names[info-&gt;cqp_cmd], info-&gt;cqp_cmd, status, cqp_request-&gt;waiting,</div><div class='add'>+			  cqp_request-&gt;compl_info.error, cqp_request-&gt;compl_info.maj_err_code,</div><div class='add'>+			  cqp_request-&gt;compl_info.min_err_code);</div><div class='add'>+</div><div class='add'>+	if (put_cqp_request)</div><div class='add'>+		irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void irdma_qp_add_ref(struct ib_qp *ibqp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *iwqp = (struct irdma_qp *)ibqp;</div><div class='add'>+</div><div class='add'>+	refcount_inc(&amp;iwqp-&gt;refcnt);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void irdma_qp_rem_ref(struct ib_qp *ibqp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *iwqp = to_iwqp(ibqp);</div><div class='add'>+	struct irdma_device *iwdev = iwqp-&gt;iwdev;</div><div class='add'>+	u32 qp_num;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwdev-&gt;rf-&gt;qptable_lock, flags);</div><div class='add'>+	if (!refcount_dec_and_test(&amp;iwqp-&gt;refcnt)) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;iwdev-&gt;rf-&gt;qptable_lock, flags);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	qp_num = iwqp-&gt;ibqp.qp_num;</div><div class='add'>+	iwdev-&gt;rf-&gt;qp_table[qp_num] = NULL;</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwdev-&gt;rf-&gt;qptable_lock, flags);</div><div class='add'>+	complete(&amp;iwqp-&gt;free_qp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void irdma_cq_add_ref(struct ib_cq *ibcq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cq *iwcq = to_iwcq(ibcq);</div><div class='add'>+</div><div class='add'>+	refcount_inc(&amp;iwcq-&gt;refcnt);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void irdma_cq_rem_ref(struct ib_cq *ibcq)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = ibcq-&gt;device;</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibdev);</div><div class='add'>+	struct irdma_cq *iwcq = to_iwcq(ibcq);</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwdev-&gt;rf-&gt;cqtable_lock, flags);</div><div class='add'>+	if (!refcount_dec_and_test(&amp;iwcq-&gt;refcnt)) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;iwdev-&gt;rf-&gt;cqtable_lock, flags);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	iwdev-&gt;rf-&gt;cq_table[iwcq-&gt;cq_num] = NULL;</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwdev-&gt;rf-&gt;cqtable_lock, flags);</div><div class='add'>+	complete(&amp;iwcq-&gt;free_cq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_device *to_ibdev(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	return &amp;(container_of(dev, struct irdma_pci_f, sc_dev))-&gt;iwdev-&gt;ibdev;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_qp - get qp address</div><div class='add'>+ * @device: iwarp device</div><div class='add'>+ * @qpn: qp number</div><div class='add'>+ */</div><div class='add'>+struct ib_qp *irdma_get_qp(struct ib_device *device, int qpn)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(device);</div><div class='add'>+</div><div class='add'>+	if (qpn &lt; IW_FIRST_QPN || qpn &gt;= iwdev-&gt;rf-&gt;max_qp)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	return &amp;iwdev-&gt;rf-&gt;qp_table[qpn]-&gt;ibqp;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_remove_cqp_head - return head entry and remove</div><div class='add'>+ * @dev: device</div><div class='add'>+ */</div><div class='add'>+void *irdma_remove_cqp_head(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct list_head *entry;</div><div class='add'>+	struct list_head *list = &amp;dev-&gt;cqp_cmd_head;</div><div class='add'>+</div><div class='add'>+	if (list_empty(list))</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	entry = list-&gt;next;</div><div class='add'>+	list_del(entry);</div><div class='add'>+</div><div class='add'>+	return entry;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_sds_cmd - create cqp command for sd</div><div class='add'>+ * @dev: hardware control device structure</div><div class='add'>+ * @sdinfo: information for sd cqp</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,</div><div class='add'>+		      struct irdma_update_sds_info *sdinfo)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	struct irdma_pci_f *rf = dev_to_rf(dev);</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	memcpy(&amp;cqp_info-&gt;in.u.update_pe_sds.info, sdinfo,</div><div class='add'>+	       sizeof(cqp_info-&gt;in.u.update_pe_sds.info));</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_UPDATE_PE_SDS;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.update_pe_sds.dev = dev;</div><div class='add'>+	cqp_info-&gt;in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;</div><div class='add'>+</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume</div><div class='add'>+ * @qp: hardware control qp</div><div class='add'>+ * @op: suspend or resume</div><div class='add'>+ */</div><div class='add'>+int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = qp-&gt;dev;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct irdma_sc_cqp *cqp = dev-&gt;cqp;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	struct irdma_pci_f *rf = dev_to_rf(dev);</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, false);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = op;</div><div class='add'>+	cqp_info-&gt;in.u.suspend_resume.cqp = cqp;</div><div class='add'>+	cqp_info-&gt;in.u.suspend_resume.qp = qp;</div><div class='add'>+	cqp_info-&gt;in.u.suspend_resume.scratch = (uintptr_t)cqp_request;</div><div class='add'>+</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_term_modify_qp - modify qp for term message</div><div class='add'>+ * @qp: hardware control qp</div><div class='add'>+ * @next_state: qp's next state</div><div class='add'>+ * @term: terminate code</div><div class='add'>+ * @term_len: length</div><div class='add'>+ */</div><div class='add'>+void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term,</div><div class='add'>+			  u8 term_len)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+</div><div class='add'>+	iwqp = qp-&gt;qp_uk.back_qp;</div><div class='add'>+	irdma_next_iw_state(iwqp, next_state, 0, term, term_len);</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_terminate_done - after terminate is completed</div><div class='add'>+ * @qp: hardware control qp</div><div class='add'>+ * @timeout_occurred: indicates if terminate timer expired</div><div class='add'>+ */</div><div class='add'>+void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+	u8 hte = 0;</div><div class='add'>+	bool first_time;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	iwqp = qp-&gt;qp_uk.back_qp;</div><div class='add'>+	spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+	if (iwqp-&gt;hte_added) {</div><div class='add'>+		iwqp-&gt;hte_added = 0;</div><div class='add'>+		hte = 1;</div><div class='add'>+	}</div><div class='add'>+	first_time = !(qp-&gt;term_flags &amp; IRDMA_TERM_DONE);</div><div class='add'>+	qp-&gt;term_flags |= IRDMA_TERM_DONE;</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+	if (first_time) {</div><div class='add'>+		if (!timeout_occurred)</div><div class='add'>+			irdma_terminate_del_timer(qp);</div><div class='add'>+</div><div class='add'>+		irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0);</div><div class='add'>+		irdma_cm_disconn(iwqp);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_terminate_timeout(struct timer_list *t)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *iwqp = timer_container_of(iwqp, t, terminate_timer);</div><div class='add'>+	struct irdma_sc_qp *qp = &amp;iwqp-&gt;sc_qp;</div><div class='add'>+</div><div class='add'>+	irdma_terminate_done(qp, 1);</div><div class='add'>+	irdma_qp_rem_ref(&amp;iwqp-&gt;ibqp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_terminate_start_timer - start terminate timeout</div><div class='add'>+ * @qp: hardware control qp</div><div class='add'>+ */</div><div class='add'>+void irdma_terminate_start_timer(struct irdma_sc_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+</div><div class='add'>+	iwqp = qp-&gt;qp_uk.back_qp;</div><div class='add'>+	irdma_qp_add_ref(&amp;iwqp-&gt;ibqp);</div><div class='add'>+	timer_setup(&amp;iwqp-&gt;terminate_timer, irdma_terminate_timeout, 0);</div><div class='add'>+	iwqp-&gt;terminate_timer.expires = jiffies + HZ;</div><div class='add'>+</div><div class='add'>+	add_timer(&amp;iwqp-&gt;terminate_timer);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_terminate_del_timer - delete terminate timeout</div><div class='add'>+ * @qp: hardware control qp</div><div class='add'>+ */</div><div class='add'>+void irdma_terminate_del_timer(struct irdma_sc_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	iwqp = qp-&gt;qp_uk.back_qp;</div><div class='add'>+	ret = timer_delete(&amp;iwqp-&gt;terminate_timer);</div><div class='add'>+	if (ret)</div><div class='add'>+		irdma_qp_rem_ref(&amp;iwqp-&gt;ibqp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_cq_create_cmd - create a cq for the cqp</div><div class='add'>+ * @dev: device pointer</div><div class='add'>+ * @cq: pointer to created cq</div><div class='add'>+ */</div><div class='add'>+int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = dev_to_rf(dev);</div><div class='add'>+	struct irdma_cqp *iwcqp = &amp;rf-&gt;cqp;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_CQ_CREATE;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.cq_create.cq = cq;</div><div class='add'>+	cqp_info-&gt;in.u.cq_create.scratch = (uintptr_t)cqp_request;</div><div class='add'>+</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(iwcqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_qp_create_cmd - create a qp for the cqp</div><div class='add'>+ * @dev: device pointer</div><div class='add'>+ * @qp: pointer to created qp</div><div class='add'>+ */</div><div class='add'>+int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = dev_to_rf(dev);</div><div class='add'>+	struct irdma_cqp *iwcqp = &amp;rf-&gt;cqp;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	struct irdma_create_qp_info *qp_info;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	qp_info = &amp;cqp_request-&gt;info.in.u.qp_create.info;</div><div class='add'>+	qp_info-&gt;cq_num_valid = true;</div><div class='add'>+	qp_info-&gt;next_iwarp_state = IRDMA_QP_STATE_RTS;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_QP_CREATE;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.qp_create.qp = qp;</div><div class='add'>+	cqp_info-&gt;in.u.qp_create.scratch = (uintptr_t)cqp_request;</div><div class='add'>+</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(iwcqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_dealloc_push_page - free a push page for qp</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @qp: hardware control qp</div><div class='add'>+ */</div><div class='add'>+static void irdma_dealloc_push_page(struct irdma_pci_f *rf,</div><div class='add'>+				    struct irdma_sc_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, false);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.manage_push_page.info.push_idx = qp-&gt;push_idx;</div><div class='add'>+	cqp_info-&gt;in.u.manage_push_page.info.qs_handle = qp-&gt;qs_handle;</div><div class='add'>+	cqp_info-&gt;in.u.manage_push_page.info.free_page = 1;</div><div class='add'>+	cqp_info-&gt;in.u.manage_push_page.info.push_page_type = 0;</div><div class='add'>+	cqp_info-&gt;in.u.manage_push_page.cqp = &amp;rf-&gt;cqp.sc_cqp;</div><div class='add'>+	cqp_info-&gt;in.u.manage_push_page.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	if (!status)</div><div class='add'>+		qp-&gt;push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_free_gsi_qp_rsrc(struct irdma_qp *iwqp, u32 qp_num)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = iwqp-&gt;iwdev;</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	if (rf-&gt;sc_dev.hw_attrs.uk_attrs.hw_rev &lt; IRDMA_GEN_3)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	irdma_vchnl_req_del_vport(&amp;rf-&gt;sc_dev, iwdev-&gt;vport_id, qp_num);</div><div class='add'>+</div><div class='add'>+	if (qp_num == 1) {</div><div class='add'>+		spin_lock_irqsave(&amp;rf-&gt;rsrc_lock, flags);</div><div class='add'>+		rf-&gt;hwqp1_rsvd = false;</div><div class='add'>+		spin_unlock_irqrestore(&amp;rf-&gt;rsrc_lock, flags);</div><div class='add'>+	} else if (qp_num &gt; 2) {</div><div class='add'>+		irdma_free_rsrc(rf, rf-&gt;allocated_qps, qp_num);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_free_qp_rsrc - free up memory resources for qp</div><div class='add'>+ * @iwqp: qp ptr (user or kernel)</div><div class='add'>+ */</div><div class='add'>+void irdma_free_qp_rsrc(struct irdma_qp *iwqp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = iwqp-&gt;iwdev;</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+	u32 qp_num = iwqp-&gt;sc_qp.qp_uk.qp_id;</div><div class='add'>+</div><div class='add'>+	irdma_ieq_cleanup_qp(iwdev-&gt;vsi.ieq, &amp;iwqp-&gt;sc_qp);</div><div class='add'>+	irdma_dealloc_push_page(rf, &amp;iwqp-&gt;sc_qp);</div><div class='add'>+	if (iwqp-&gt;sc_qp.vsi) {</div><div class='add'>+		irdma_qp_rem_qos(&amp;iwqp-&gt;sc_qp);</div><div class='add'>+		iwqp-&gt;sc_qp.dev-&gt;ws_remove(iwqp-&gt;sc_qp.vsi,</div><div class='add'>+					   iwqp-&gt;sc_qp.user_pri);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (iwqp-&gt;ibqp.qp_type == IB_QPT_GSI) {</div><div class='add'>+		irdma_free_gsi_qp_rsrc(iwqp, qp_num);</div><div class='add'>+	} else {</div><div class='add'>+		if (qp_num &gt; 2)</div><div class='add'>+			irdma_free_rsrc(rf, rf-&gt;allocated_qps, qp_num);</div><div class='add'>+	}</div><div class='add'>+	dma_free_coherent(rf-&gt;sc_dev.hw-&gt;device, iwqp-&gt;q2_ctx_mem.size,</div><div class='add'>+			  iwqp-&gt;q2_ctx_mem.va, iwqp-&gt;q2_ctx_mem.pa);</div><div class='add'>+	iwqp-&gt;q2_ctx_mem.va = NULL;</div><div class='add'>+	dma_free_coherent(rf-&gt;sc_dev.hw-&gt;device, iwqp-&gt;kqp.dma_mem.size,</div><div class='add'>+			  iwqp-&gt;kqp.dma_mem.va, iwqp-&gt;kqp.dma_mem.pa);</div><div class='add'>+	iwqp-&gt;kqp.dma_mem.va = NULL;</div><div class='add'>+	kfree(iwqp-&gt;kqp.sq_wrid_mem);</div><div class='add'>+	kfree(iwqp-&gt;kqp.rq_wrid_mem);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_srq_wq_destroy - send srq destroy cqp</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @srq: hardware control srq</div><div class='add'>+ */</div><div class='add'>+void irdma_srq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_srq *srq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_SRQ_DESTROY;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.srq_destroy.srq = srq;</div><div class='add'>+	cqp_info-&gt;in.u.srq_destroy.scratch = (uintptr_t)cqp_request;</div><div class='add'>+</div><div class='add'>+	irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cq_wq_destroy - send cq destroy cqp</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @cq: hardware control cq</div><div class='add'>+ */</div><div class='add'>+void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_CQ_DESTROY;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.cq_destroy.cq = cq;</div><div class='add'>+	cqp_info-&gt;in.u.cq_destroy.scratch = (uintptr_t)cqp_request;</div><div class='add'>+</div><div class='add'>+	irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait</div><div class='add'>+ * @cqp_request: modify QP completion</div><div class='add'>+ */</div><div class='add'>+static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request)</div><div class='add'>+{</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	iwqp = cqp_info-&gt;in.u.qp_modify.qp-&gt;qp_uk.back_qp;</div><div class='add'>+	atomic_dec(&amp;iwqp-&gt;hw_mod_qp_pend);</div><div class='add'>+	wake_up(&amp;iwqp-&gt;mod_qp_waitq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_hw_modify_qp - setup cqp for modify qp</div><div class='add'>+ * @iwdev: RDMA device</div><div class='add'>+ * @iwqp: qp ptr (user or kernel)</div><div class='add'>+ * @info: info for modify qp</div><div class='add'>+ * @wait: flag to wait or not for modify qp completion</div><div class='add'>+ */</div><div class='add'>+int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp,</div><div class='add'>+		       struct irdma_modify_qp_info *info, bool wait)</div><div class='add'>+{</div><div class='add'>+	int status;</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	struct irdma_modify_qp_info *m_info;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, wait);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	if (!wait) {</div><div class='add'>+		cqp_request-&gt;callback_fcn = irdma_hw_modify_qp_callback;</div><div class='add'>+		atomic_inc(&amp;iwqp-&gt;hw_mod_qp_pend);</div><div class='add'>+	}</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	m_info = &amp;cqp_info-&gt;in.u.qp_modify.info;</div><div class='add'>+	memcpy(m_info, info, sizeof(*m_info));</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_QP_MODIFY;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.qp_modify.qp = &amp;iwqp-&gt;sc_qp;</div><div class='add'>+	cqp_info-&gt;in.u.qp_modify.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+	if (status) {</div><div class='add'>+		if (rdma_protocol_roce(&amp;iwdev-&gt;ibdev, 1))</div><div class='add'>+			return status;</div><div class='add'>+</div><div class='add'>+		switch (m_info-&gt;next_iwarp_state) {</div><div class='add'>+			struct irdma_gen_ae_info ae_info;</div><div class='add'>+</div><div class='add'>+		case IRDMA_QP_STATE_RTS:</div><div class='add'>+		case IRDMA_QP_STATE_IDLE:</div><div class='add'>+		case IRDMA_QP_STATE_TERMINATE:</div><div class='add'>+		case IRDMA_QP_STATE_CLOSING:</div><div class='add'>+			if (info-&gt;curr_iwarp_state == IRDMA_QP_STATE_IDLE)</div><div class='add'>+				irdma_send_reset(iwqp-&gt;cm_node);</div><div class='add'>+			else</div><div class='add'>+				iwqp-&gt;sc_qp.term_flags = IRDMA_TERM_DONE;</div><div class='add'>+			if (!wait) {</div><div class='add'>+				ae_info.ae_code = IRDMA_AE_BAD_CLOSE;</div><div class='add'>+				ae_info.ae_src = 0;</div><div class='add'>+				irdma_gen_ae(rf, &amp;iwqp-&gt;sc_qp, &amp;ae_info, false);</div><div class='add'>+			} else {</div><div class='add'>+				cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp,</div><div class='add'>+									      wait);</div><div class='add'>+				if (!cqp_request)</div><div class='add'>+					return -ENOMEM;</div><div class='add'>+</div><div class='add'>+				cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+				m_info = &amp;cqp_info-&gt;in.u.qp_modify.info;</div><div class='add'>+				memcpy(m_info, info, sizeof(*m_info));</div><div class='add'>+				cqp_info-&gt;cqp_cmd = IRDMA_OP_QP_MODIFY;</div><div class='add'>+				cqp_info-&gt;post_sq = 1;</div><div class='add'>+				cqp_info-&gt;in.u.qp_modify.qp = &amp;iwqp-&gt;sc_qp;</div><div class='add'>+				cqp_info-&gt;in.u.qp_modify.scratch = (uintptr_t)cqp_request;</div><div class='add'>+				m_info-&gt;next_iwarp_state = IRDMA_QP_STATE_ERROR;</div><div class='add'>+				m_info-&gt;reset_tcp_conn = true;</div><div class='add'>+				irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+				irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+			}</div><div class='add'>+			break;</div><div class='add'>+		case IRDMA_QP_STATE_ERROR:</div><div class='add'>+		default:</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_cq_destroy_cmd - destroy the cqp cq</div><div class='add'>+ * @dev: device pointer</div><div class='add'>+ * @cq: pointer to cq</div><div class='add'>+ */</div><div class='add'>+void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = dev_to_rf(dev);</div><div class='add'>+</div><div class='add'>+	irdma_cq_wq_destroy(rf, cq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_qp_destroy_cmd - destroy the cqp</div><div class='add'>+ * @dev: device pointer</div><div class='add'>+ * @qp: pointer to qp</div><div class='add'>+ */</div><div class='add'>+int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = dev_to_rf(dev);</div><div class='add'>+	struct irdma_cqp *iwcqp = &amp;rf-&gt;cqp;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_QP_DESTROY;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.qp_destroy.qp = qp;</div><div class='add'>+	cqp_info-&gt;in.u.qp_destroy.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	cqp_info-&gt;in.u.qp_destroy.remove_hash_idx = true;</div><div class='add'>+</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_mpa_crc_ae - generate AE for crc error</div><div class='add'>+ * @dev: hardware control device structure</div><div class='add'>+ * @qp: hardware control qp</div><div class='add'>+ */</div><div class='add'>+void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_gen_ae_info info = {};</div><div class='add'>+	struct irdma_pci_f *rf = dev_to_rf(dev);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;rf-&gt;iwdev-&gt;ibdev, "AEQ: Generate MPA CRC AE\n");</div><div class='add'>+	info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR;</div><div class='add'>+	info.ae_src = IRDMA_AE_SOURCE_RQ;</div><div class='add'>+	irdma_gen_ae(rf, qp, &amp;info, false);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_check_mpacrc - check if mpa crc is OK</div><div class='add'>+ * @addr: address of buffer for crc</div><div class='add'>+ * @len: length of buffer</div><div class='add'>+ * @val: value to be compared</div><div class='add'>+ */</div><div class='add'>+int irdma_ieq_check_mpacrc(const void *addr, u32 len, u32 val)</div><div class='add'>+{</div><div class='add'>+	if ((__force u32)cpu_to_le32(~crc32c(~0, addr, len)) != val)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_get_qp - get qp based on quad in puda buffer</div><div class='add'>+ * @dev: hardware control device structure</div><div class='add'>+ * @buf: receive puda buffer on exception q</div><div class='add'>+ */</div><div class='add'>+struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,</div><div class='add'>+				     struct irdma_puda_buf *buf)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+	struct irdma_device *iwdev = buf-&gt;vsi-&gt;back_vsi;</div><div class='add'>+	u32 loc_addr[4] = {};</div><div class='add'>+	u32 rem_addr[4] = {};</div><div class='add'>+	u16 loc_port, rem_port;</div><div class='add'>+	struct ipv6hdr *ip6h;</div><div class='add'>+	struct iphdr *iph = (struct iphdr *)buf-&gt;iph;</div><div class='add'>+	struct tcphdr *tcph = (struct tcphdr *)buf-&gt;tcph;</div><div class='add'>+</div><div class='add'>+	if (iph-&gt;version == 4) {</div><div class='add'>+		loc_addr[0] = ntohl(iph-&gt;daddr);</div><div class='add'>+		rem_addr[0] = ntohl(iph-&gt;saddr);</div><div class='add'>+	} else {</div><div class='add'>+		ip6h = (struct ipv6hdr *)buf-&gt;iph;</div><div class='add'>+		irdma_copy_ip_ntohl(loc_addr, ip6h-&gt;daddr.in6_u.u6_addr32);</div><div class='add'>+		irdma_copy_ip_ntohl(rem_addr, ip6h-&gt;saddr.in6_u.u6_addr32);</div><div class='add'>+	}</div><div class='add'>+	loc_port = ntohs(tcph-&gt;dest);</div><div class='add'>+	rem_port = ntohs(tcph-&gt;source);</div><div class='add'>+	cm_node = irdma_find_node(&amp;iwdev-&gt;cm_core, rem_port, rem_addr, loc_port,</div><div class='add'>+				  loc_addr, buf-&gt;vlan_valid ? buf-&gt;vlan_id : 0xFFFF);</div><div class='add'>+	if (!cm_node)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	iwqp = cm_node-&gt;iwqp;</div><div class='add'>+	irdma_rem_ref_cm_node(cm_node);</div><div class='add'>+</div><div class='add'>+	return &amp;iwqp-&gt;sc_qp;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs</div><div class='add'>+ * @qp: qp ptr</div><div class='add'>+ */</div><div class='add'>+void irdma_send_ieq_ack(struct irdma_sc_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp-&gt;qp_uk.back_qp)-&gt;cm_node;</div><div class='add'>+	struct irdma_puda_buf *buf = qp-&gt;pfpdu.lastrcv_buf;</div><div class='add'>+	struct tcphdr *tcph = (struct tcphdr *)buf-&gt;tcph;</div><div class='add'>+</div><div class='add'>+	cm_node-&gt;tcp_cntxt.rcv_nxt = qp-&gt;pfpdu.nextseqnum;</div><div class='add'>+	cm_node-&gt;tcp_cntxt.loc_seq_num = ntohl(tcph-&gt;ack_seq);</div><div class='add'>+</div><div class='add'>+	irdma_send_ack(cm_node);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer</div><div class='add'>+ * @qp: qp pointer</div><div class='add'>+ * @ah_info: AH info pointer</div><div class='add'>+ */</div><div class='add'>+void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp,</div><div class='add'>+				struct irdma_ah_info *ah_info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_puda_buf *buf = qp-&gt;pfpdu.ah_buf;</div><div class='add'>+	struct iphdr *iph;</div><div class='add'>+	struct ipv6hdr *ip6h;</div><div class='add'>+</div><div class='add'>+	memset(ah_info, 0, sizeof(*ah_info));</div><div class='add'>+	ah_info-&gt;do_lpbk = true;</div><div class='add'>+	ah_info-&gt;vlan_tag = buf-&gt;vlan_id;</div><div class='add'>+	ah_info-&gt;insert_vlan_tag = buf-&gt;vlan_valid;</div><div class='add'>+	ah_info-&gt;ipv4_valid = buf-&gt;ipv4;</div><div class='add'>+	ah_info-&gt;vsi = qp-&gt;vsi;</div><div class='add'>+</div><div class='add'>+	if (buf-&gt;smac_valid)</div><div class='add'>+		ether_addr_copy(ah_info-&gt;mac_addr, buf-&gt;smac);</div><div class='add'>+</div><div class='add'>+	if (buf-&gt;ipv4) {</div><div class='add'>+		ah_info-&gt;ipv4_valid = true;</div><div class='add'>+		iph = (struct iphdr *)buf-&gt;iph;</div><div class='add'>+		ah_info-&gt;hop_ttl = iph-&gt;ttl;</div><div class='add'>+		ah_info-&gt;tc_tos = iph-&gt;tos;</div><div class='add'>+		ah_info-&gt;dest_ip_addr[0] = ntohl(iph-&gt;daddr);</div><div class='add'>+		ah_info-&gt;src_ip_addr[0] = ntohl(iph-&gt;saddr);</div><div class='add'>+	} else {</div><div class='add'>+		ip6h = (struct ipv6hdr *)buf-&gt;iph;</div><div class='add'>+		ah_info-&gt;hop_ttl = ip6h-&gt;hop_limit;</div><div class='add'>+		ah_info-&gt;tc_tos = ip6h-&gt;priority;</div><div class='add'>+		irdma_copy_ip_ntohl(ah_info-&gt;dest_ip_addr,</div><div class='add'>+				    ip6h-&gt;daddr.in6_u.u6_addr32);</div><div class='add'>+		irdma_copy_ip_ntohl(ah_info-&gt;src_ip_addr,</div><div class='add'>+				    ip6h-&gt;saddr.in6_u.u6_addr32);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ah_info-&gt;dst_arpindex = irdma_arp_table(dev_to_rf(qp-&gt;dev),</div><div class='add'>+						ah_info-&gt;dest_ip_addr,</div><div class='add'>+						ah_info-&gt;ipv4_valid,</div><div class='add'>+						NULL, IRDMA_ARP_RESOLVE);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer</div><div class='add'>+ * @buf: puda to update</div><div class='add'>+ * @len: length of buffer</div><div class='add'>+ * @seqnum: seq number for tcp</div><div class='add'>+ */</div><div class='add'>+static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf,</div><div class='add'>+					     u16 len, u32 seqnum)</div><div class='add'>+{</div><div class='add'>+	struct tcphdr *tcph;</div><div class='add'>+	struct iphdr *iph;</div><div class='add'>+	u16 iphlen;</div><div class='add'>+	u16 pktsize;</div><div class='add'>+	u8 *addr = buf-&gt;mem.va;</div><div class='add'>+</div><div class='add'>+	iphlen = (buf-&gt;ipv4) ? 20 : 40;</div><div class='add'>+	iph = (struct iphdr *)(addr + buf-&gt;maclen);</div><div class='add'>+	tcph = (struct tcphdr *)(addr + buf-&gt;maclen + iphlen);</div><div class='add'>+	pktsize = len + buf-&gt;tcphlen + iphlen;</div><div class='add'>+	iph-&gt;tot_len = htons(pktsize);</div><div class='add'>+	tcph-&gt;seq = htonl(seqnum);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_update_tcpip_info - update tcpip in the buffer</div><div class='add'>+ * @buf: puda to update</div><div class='add'>+ * @len: length of buffer</div><div class='add'>+ * @seqnum: seq number for tcp</div><div class='add'>+ */</div><div class='add'>+void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,</div><div class='add'>+				 u32 seqnum)</div><div class='add'>+{</div><div class='add'>+	struct tcphdr *tcph;</div><div class='add'>+	u8 *addr;</div><div class='add'>+</div><div class='add'>+	if (buf-&gt;vsi-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)</div><div class='add'>+		return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum);</div><div class='add'>+</div><div class='add'>+	addr = buf-&gt;mem.va;</div><div class='add'>+	tcph = (struct tcphdr *)addr;</div><div class='add'>+	tcph-&gt;seq = htonl(seqnum);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda</div><div class='add'>+ * buffer</div><div class='add'>+ * @info: to get information</div><div class='add'>+ * @buf: puda buffer</div><div class='add'>+ */</div><div class='add'>+static int irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,</div><div class='add'>+					  struct irdma_puda_buf *buf)</div><div class='add'>+{</div><div class='add'>+	struct iphdr *iph;</div><div class='add'>+	struct ipv6hdr *ip6h;</div><div class='add'>+	struct tcphdr *tcph;</div><div class='add'>+	u16 iphlen;</div><div class='add'>+	u16 pkt_len;</div><div class='add'>+	u8 *mem = buf-&gt;mem.va;</div><div class='add'>+	struct ethhdr *ethh = buf-&gt;mem.va;</div><div class='add'>+</div><div class='add'>+	if (ethh-&gt;h_proto == htons(0x8100)) {</div><div class='add'>+		info-&gt;vlan_valid = true;</div><div class='add'>+		buf-&gt;vlan_id = ntohs(((struct vlan_ethhdr *)ethh)-&gt;h_vlan_TCI) &amp;</div><div class='add'>+			       VLAN_VID_MASK;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	buf-&gt;maclen = (info-&gt;vlan_valid) ? 18 : 14;</div><div class='add'>+	iphlen = (info-&gt;l3proto) ? 40 : 20;</div><div class='add'>+	buf-&gt;ipv4 = (info-&gt;l3proto) ? false : true;</div><div class='add'>+	buf-&gt;iph = mem + buf-&gt;maclen;</div><div class='add'>+	iph = (struct iphdr *)buf-&gt;iph;</div><div class='add'>+	buf-&gt;tcph = buf-&gt;iph + iphlen;</div><div class='add'>+	tcph = (struct tcphdr *)buf-&gt;tcph;</div><div class='add'>+</div><div class='add'>+	if (buf-&gt;ipv4) {</div><div class='add'>+		pkt_len = ntohs(iph-&gt;tot_len);</div><div class='add'>+	} else {</div><div class='add'>+		ip6h = (struct ipv6hdr *)buf-&gt;iph;</div><div class='add'>+		pkt_len = ntohs(ip6h-&gt;payload_len) + iphlen;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	buf-&gt;totallen = pkt_len + buf-&gt;maclen;</div><div class='add'>+</div><div class='add'>+	if (info-&gt;payload_len &lt; buf-&gt;totallen) {</div><div class='add'>+		ibdev_dbg(to_ibdev(buf-&gt;vsi-&gt;dev),</div><div class='add'>+			  "ERR: payload_len = 0x%x totallen expected0x%x\n",</div><div class='add'>+			  info-&gt;payload_len, buf-&gt;totallen);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	buf-&gt;tcphlen = tcph-&gt;doff &lt;&lt; 2;</div><div class='add'>+	buf-&gt;datalen = pkt_len - iphlen - buf-&gt;tcphlen;</div><div class='add'>+	buf-&gt;data = buf-&gt;datalen ? buf-&gt;tcph + buf-&gt;tcphlen : NULL;</div><div class='add'>+	buf-&gt;hdrlen = buf-&gt;maclen + iphlen + buf-&gt;tcphlen;</div><div class='add'>+	buf-&gt;seqnum = ntohl(tcph-&gt;seq);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_get_tcpip_info - get tcpip info from puda buffer</div><div class='add'>+ * @info: to get information</div><div class='add'>+ * @buf: puda buffer</div><div class='add'>+ */</div><div class='add'>+int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,</div><div class='add'>+			      struct irdma_puda_buf *buf)</div><div class='add'>+{</div><div class='add'>+	struct tcphdr *tcph;</div><div class='add'>+	u32 pkt_len;</div><div class='add'>+	u8 *mem;</div><div class='add'>+</div><div class='add'>+	if (buf-&gt;vsi-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)</div><div class='add'>+		return irdma_gen1_puda_get_tcpip_info(info, buf);</div><div class='add'>+</div><div class='add'>+	mem = buf-&gt;mem.va;</div><div class='add'>+	buf-&gt;vlan_valid = info-&gt;vlan_valid;</div><div class='add'>+	if (info-&gt;vlan_valid)</div><div class='add'>+		buf-&gt;vlan_id = info-&gt;vlan;</div><div class='add'>+</div><div class='add'>+	buf-&gt;ipv4 = info-&gt;ipv4;</div><div class='add'>+	if (buf-&gt;ipv4)</div><div class='add'>+		buf-&gt;iph = mem + IRDMA_IPV4_PAD;</div><div class='add'>+	else</div><div class='add'>+		buf-&gt;iph = mem;</div><div class='add'>+</div><div class='add'>+	buf-&gt;tcph = mem + IRDMA_TCP_OFFSET;</div><div class='add'>+	tcph = (struct tcphdr *)buf-&gt;tcph;</div><div class='add'>+	pkt_len = info-&gt;payload_len;</div><div class='add'>+	buf-&gt;totallen = pkt_len;</div><div class='add'>+	buf-&gt;tcphlen = tcph-&gt;doff &lt;&lt; 2;</div><div class='add'>+	buf-&gt;datalen = pkt_len - IRDMA_TCP_OFFSET - buf-&gt;tcphlen;</div><div class='add'>+	buf-&gt;data = buf-&gt;datalen ? buf-&gt;tcph + buf-&gt;tcphlen : NULL;</div><div class='add'>+	buf-&gt;hdrlen = IRDMA_TCP_OFFSET + buf-&gt;tcphlen;</div><div class='add'>+	buf-&gt;seqnum = ntohl(tcph-&gt;seq);</div><div class='add'>+</div><div class='add'>+	if (info-&gt;smac_valid) {</div><div class='add'>+		ether_addr_copy(buf-&gt;smac, info-&gt;smac);</div><div class='add'>+		buf-&gt;smac_valid = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats</div><div class='add'>+ * @t: timer_list pointer</div><div class='add'>+ */</div><div class='add'>+static void irdma_hw_stats_timeout(struct timer_list *t)</div><div class='add'>+{</div><div class='add'>+	struct irdma_vsi_pestat *pf_devstat =</div><div class='add'>+		timer_container_of(pf_devstat, t, stats_timer);</div><div class='add'>+	struct irdma_sc_vsi *sc_vsi = pf_devstat-&gt;vsi;</div><div class='add'>+</div><div class='add'>+	if (sc_vsi-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2)</div><div class='add'>+		irdma_cqp_gather_stats_cmd(sc_vsi-&gt;dev, sc_vsi-&gt;pestat, false);</div><div class='add'>+	else</div><div class='add'>+		irdma_cqp_gather_stats_gen1(sc_vsi-&gt;dev, sc_vsi-&gt;pestat);</div><div class='add'>+</div><div class='add'>+	mod_timer(&amp;pf_devstat-&gt;stats_timer,</div><div class='add'>+		  jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_hw_stats_start_timer - Start periodic stats timer</div><div class='add'>+ * @vsi: vsi structure pointer</div><div class='add'>+ */</div><div class='add'>+void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi)</div><div class='add'>+{</div><div class='add'>+	struct irdma_vsi_pestat *devstat = vsi-&gt;pestat;</div><div class='add'>+</div><div class='add'>+	timer_setup(&amp;devstat-&gt;stats_timer, irdma_hw_stats_timeout, 0);</div><div class='add'>+	mod_timer(&amp;devstat-&gt;stats_timer,</div><div class='add'>+		  jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_hw_stats_stop_timer - Delete periodic stats timer</div><div class='add'>+ * @vsi: pointer to vsi structure</div><div class='add'>+ */</div><div class='add'>+void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi)</div><div class='add'>+{</div><div class='add'>+	struct irdma_vsi_pestat *devstat = vsi-&gt;pestat;</div><div class='add'>+</div><div class='add'>+	timer_delete_sync(&amp;devstat-&gt;stats_timer);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_process_stats - Checking for wrap and update stats</div><div class='add'>+ * @pestat: stats structure pointer</div><div class='add'>+ */</div><div class='add'>+static inline void irdma_process_stats(struct irdma_vsi_pestat *pestat)</div><div class='add'>+{</div><div class='add'>+	sc_vsi_update_stats(pestat-&gt;vsi);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_gather_stats_gen1 - Gather stats</div><div class='add'>+ * @dev: pointer to device structure</div><div class='add'>+ * @pestat: statistics structure</div><div class='add'>+ */</div><div class='add'>+void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev,</div><div class='add'>+				 struct irdma_vsi_pestat *pestat)</div><div class='add'>+{</div><div class='add'>+	struct irdma_gather_stats *gather_stats =</div><div class='add'>+		pestat-&gt;gather_info.gather_stats_va;</div><div class='add'>+	const struct irdma_hw_stat_map *map = dev-&gt;hw_stats_map;</div><div class='add'>+	u16 max_stats_idx = dev-&gt;hw_attrs.max_stat_idx;</div><div class='add'>+	u32 stats_inst_offset_32;</div><div class='add'>+	u32 stats_inst_offset_64;</div><div class='add'>+	u64 new_val;</div><div class='add'>+	u16 i;</div><div class='add'>+</div><div class='add'>+	stats_inst_offset_32 = (pestat-&gt;gather_info.use_stats_inst) ?</div><div class='add'>+				pestat-&gt;gather_info.stats_inst_index :</div><div class='add'>+				pestat-&gt;hw-&gt;hmc.hmc_fn_id;</div><div class='add'>+	stats_inst_offset_32 *= 4;</div><div class='add'>+	stats_inst_offset_64 = stats_inst_offset_32 * 2;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; max_stats_idx; i++) {</div><div class='add'>+		if (map[i].bitmask &lt;= IRDMA_MAX_STATS_32)</div><div class='add'>+			new_val = rd32(dev-&gt;hw,</div><div class='add'>+				       dev-&gt;hw_stats_regs[i] + stats_inst_offset_32);</div><div class='add'>+		else</div><div class='add'>+			new_val = rd64(dev-&gt;hw,</div><div class='add'>+				       dev-&gt;hw_stats_regs[i] + stats_inst_offset_64);</div><div class='add'>+		gather_stats-&gt;val[map[i].byteoff / sizeof(u64)] = new_val;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_process_stats(pestat);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_process_cqp_stats - Checking for wrap and update stats</div><div class='add'>+ * @cqp_request: cqp_request structure pointer</div><div class='add'>+ */</div><div class='add'>+static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request)</div><div class='add'>+{</div><div class='add'>+	struct irdma_vsi_pestat *pestat = cqp_request-&gt;param;</div><div class='add'>+</div><div class='add'>+	irdma_process_stats(pestat);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_gather_stats_cmd - Gather stats</div><div class='add'>+ * @dev: pointer to device structure</div><div class='add'>+ * @pestat: pointer to stats info</div><div class='add'>+ * @wait: flag to wait or not wait for stats</div><div class='add'>+ */</div><div class='add'>+int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,</div><div class='add'>+			       struct irdma_vsi_pestat *pestat, bool wait)</div><div class='add'>+</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = dev_to_rf(dev);</div><div class='add'>+	struct irdma_cqp *iwcqp = &amp;rf-&gt;cqp;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_STATS_GATHER;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.stats_gather.info = pestat-&gt;gather_info;</div><div class='add'>+	cqp_info-&gt;in.u.stats_gather.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	cqp_info-&gt;in.u.stats_gather.cqp = &amp;rf-&gt;cqp.sc_cqp;</div><div class='add'>+	cqp_request-&gt;param = pestat;</div><div class='add'>+	if (!wait)</div><div class='add'>+		cqp_request-&gt;callback_fcn = irdma_process_cqp_stats;</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	if (wait)</div><div class='add'>+		irdma_process_stats(pestat);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_stats_inst_cmd - Allocate/free stats instance</div><div class='add'>+ * @vsi: pointer to vsi structure</div><div class='add'>+ * @cmd: command to allocate or free</div><div class='add'>+ * @stats_info: pointer to allocate stats info</div><div class='add'>+ */</div><div class='add'>+int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,</div><div class='add'>+			     struct irdma_stats_inst_info *stats_info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = dev_to_rf(vsi-&gt;dev);</div><div class='add'>+	struct irdma_cqp *iwcqp = &amp;rf-&gt;cqp;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	int status;</div><div class='add'>+	bool wait = false;</div><div class='add'>+</div><div class='add'>+	if (cmd == IRDMA_OP_STATS_ALLOCATE)</div><div class='add'>+		wait = true;</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = cmd;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.stats_manage.info = *stats_info;</div><div class='add'>+	cqp_info-&gt;in.u.stats_manage.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	cqp_info-&gt;in.u.stats_manage.cqp = &amp;rf-&gt;cqp.sc_cqp;</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	if (wait)</div><div class='add'>+		stats_info-&gt;stats_idx = cqp_request-&gt;compl_info.op_ret_val;</div><div class='add'>+	irdma_put_cqp_request(iwcqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0</div><div class='add'>+ * @dev: pointer to device info</div><div class='add'>+ * @sc_ceq: pointer to ceq structure</div><div class='add'>+ * @op: Create or Destroy</div><div class='add'>+ */</div><div class='add'>+int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq,</div><div class='add'>+		      u8 op)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	struct irdma_pci_f *rf = dev_to_rf(dev);</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = op;</div><div class='add'>+	cqp_info-&gt;in.u.ceq_create.ceq = sc_ceq;</div><div class='add'>+	cqp_info-&gt;in.u.ceq_create.scratch = (uintptr_t)cqp_request;</div><div class='add'>+</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_aeq_cmd - Create/Destroy AEQ</div><div class='add'>+ * @dev: pointer to device info</div><div class='add'>+ * @sc_aeq: pointer to aeq structure</div><div class='add'>+ * @op: Create or Destroy</div><div class='add'>+ */</div><div class='add'>+int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq,</div><div class='add'>+		      u8 op)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	struct irdma_pci_f *rf = dev_to_rf(dev);</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = op;</div><div class='add'>+	cqp_info-&gt;in.u.aeq_create.aeq = sc_aeq;</div><div class='add'>+	cqp_info-&gt;in.u.aeq_create.scratch = (uintptr_t)cqp_request;</div><div class='add'>+</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cqp_ws_node_cmd - Add/modify/delete ws node</div><div class='add'>+ * @dev: pointer to device structure</div><div class='add'>+ * @cmd: Add, modify or delete</div><div class='add'>+ * @node_info: pointer to ws node info</div><div class='add'>+ */</div><div class='add'>+int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,</div><div class='add'>+			  struct irdma_ws_node_info *node_info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = dev_to_rf(dev);</div><div class='add'>+	struct irdma_cqp *iwcqp = &amp;rf-&gt;cqp;</div><div class='add'>+	struct irdma_sc_cqp *cqp = &amp;iwcqp-&gt;sc_cqp;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	int status;</div><div class='add'>+	bool poll;</div><div class='add'>+</div><div class='add'>+	if (!rf-&gt;sc_dev.ceq_valid)</div><div class='add'>+		poll = true;</div><div class='add'>+	else</div><div class='add'>+		poll = false;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = cmd;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.ws_node.info = *node_info;</div><div class='add'>+	cqp_info-&gt;in.u.ws_node.cqp = cqp;</div><div class='add'>+	cqp_info-&gt;in.u.ws_node.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	if (status)</div><div class='add'>+		goto exit;</div><div class='add'>+</div><div class='add'>+	if (poll) {</div><div class='add'>+		struct irdma_ccq_cqe_info compl_info;</div><div class='add'>+</div><div class='add'>+		status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE,</div><div class='add'>+						       &amp;compl_info);</div><div class='add'>+		node_info-&gt;qs_handle = compl_info.op_ret_val;</div><div class='add'>+		ibdev_dbg(&amp;rf-&gt;iwdev-&gt;ibdev, "DCB: opcode=%d, compl_info.retval=%d\n",</div><div class='add'>+			  compl_info.op_code, compl_info.op_ret_val);</div><div class='add'>+	} else {</div><div class='add'>+		node_info-&gt;qs_handle = cqp_request-&gt;compl_info.op_ret_val;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+exit:</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ah_cqp_op - perform an AH cqp operation</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @sc_ah: address handle</div><div class='add'>+ * @cmd: AH operation</div><div class='add'>+ * @wait: wait if true</div><div class='add'>+ * @callback_fcn: Callback function on CQP op completion</div><div class='add'>+ * @cb_param: parameter for callback function</div><div class='add'>+ *</div><div class='add'>+ * returns errno</div><div class='add'>+ */</div><div class='add'>+int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,</div><div class='add'>+		    bool wait,</div><div class='add'>+		    void (*callback_fcn)(struct irdma_cqp_request *),</div><div class='add'>+		    void *cb_param)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	if (cmd != IRDMA_OP_AH_CREATE &amp;&amp; cmd != IRDMA_OP_AH_DESTROY)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, wait);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = cmd;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	if (cmd == IRDMA_OP_AH_CREATE) {</div><div class='add'>+		cqp_info-&gt;in.u.ah_create.info = sc_ah-&gt;ah_info;</div><div class='add'>+		cqp_info-&gt;in.u.ah_create.scratch = (uintptr_t)cqp_request;</div><div class='add'>+		cqp_info-&gt;in.u.ah_create.cqp = &amp;rf-&gt;cqp.sc_cqp;</div><div class='add'>+	} else if (cmd == IRDMA_OP_AH_DESTROY) {</div><div class='add'>+		cqp_info-&gt;in.u.ah_destroy.info = sc_ah-&gt;ah_info;</div><div class='add'>+		cqp_info-&gt;in.u.ah_destroy.scratch = (uintptr_t)cqp_request;</div><div class='add'>+		cqp_info-&gt;in.u.ah_destroy.cqp = &amp;rf-&gt;cqp.sc_cqp;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!wait) {</div><div class='add'>+		cqp_request-&gt;callback_fcn = callback_fcn;</div><div class='add'>+		cqp_request-&gt;param = cb_param;</div><div class='add'>+	}</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	if (status)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	if (wait)</div><div class='add'>+		sc_ah-&gt;ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ieq_ah_cb - callback after creation of AH for IEQ</div><div class='add'>+ * @cqp_request: pointer to cqp_request of create AH</div><div class='add'>+ */</div><div class='add'>+static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_qp *qp = cqp_request-&gt;param;</div><div class='add'>+	struct irdma_sc_ah *sc_ah = qp-&gt;pfpdu.ah;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;qp-&gt;pfpdu.lock, flags);</div><div class='add'>+	if (!cqp_request-&gt;compl_info.op_ret_val) {</div><div class='add'>+		sc_ah-&gt;ah_info.ah_valid = true;</div><div class='add'>+		irdma_ieq_process_fpdus(qp, qp-&gt;vsi-&gt;ieq);</div><div class='add'>+	} else {</div><div class='add'>+		sc_ah-&gt;ah_info.ah_valid = false;</div><div class='add'>+		irdma_ieq_cleanup_qp(qp-&gt;vsi-&gt;ieq, qp);</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;qp-&gt;pfpdu.lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ilq_ah_cb - callback after creation of AH for ILQ</div><div class='add'>+ * @cqp_request: pointer to cqp_request of create AH</div><div class='add'>+ */</div><div class='add'>+static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cm_node *cm_node = cqp_request-&gt;param;</div><div class='add'>+	struct irdma_sc_ah *sc_ah = cm_node-&gt;ah;</div><div class='add'>+</div><div class='add'>+	sc_ah-&gt;ah_info.ah_valid = !cqp_request-&gt;compl_info.op_ret_val;</div><div class='add'>+	irdma_add_conn_est_qh(cm_node);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_create_ah - create AH for ILQ/IEQ qp's</div><div class='add'>+ * @dev: device pointer</div><div class='add'>+ * @ah_info: Address handle info</div><div class='add'>+ * @wait: When true will wait for operation to complete</div><div class='add'>+ * @type: ILQ/IEQ</div><div class='add'>+ * @cb_param: Callback param when not waiting</div><div class='add'>+ * @ah_ret: Returned pointer to address handle if created</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+int irdma_puda_create_ah(struct irdma_sc_dev *dev,</div><div class='add'>+			 struct irdma_ah_info *ah_info, bool wait,</div><div class='add'>+			 enum puda_rsrc_type type, void *cb_param,</div><div class='add'>+			 struct irdma_sc_ah **ah_ret)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_ah *ah;</div><div class='add'>+	struct irdma_pci_f *rf = dev_to_rf(dev);</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);</div><div class='add'>+	*ah_ret = ah;</div><div class='add'>+	if (!ah)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	err = irdma_alloc_rsrc(rf, rf-&gt;allocated_ahs, rf-&gt;max_ah,</div><div class='add'>+			       &amp;ah_info-&gt;ah_idx, &amp;rf-&gt;next_ah);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_free;</div><div class='add'>+</div><div class='add'>+	ah-&gt;dev = dev;</div><div class='add'>+	ah-&gt;ah_info = *ah_info;</div><div class='add'>+</div><div class='add'>+	if (type == IRDMA_PUDA_RSRC_TYPE_ILQ)</div><div class='add'>+		err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,</div><div class='add'>+				      irdma_ilq_ah_cb, cb_param);</div><div class='add'>+	else</div><div class='add'>+		err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,</div><div class='add'>+				      irdma_ieq_ah_cb, cb_param);</div><div class='add'>+</div><div class='add'>+	if (err)</div><div class='add'>+		goto error;</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+error:</div><div class='add'>+	irdma_free_rsrc(rf, rf-&gt;allocated_ahs, ah-&gt;ah_info.ah_idx);</div><div class='add'>+err_free:</div><div class='add'>+	kfree(ah);</div><div class='add'>+	*ah_ret = NULL;</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_puda_free_ah - free a puda address handle</div><div class='add'>+ * @dev: device pointer</div><div class='add'>+ * @ah: The address handle to free</div><div class='add'>+ */</div><div class='add'>+void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = dev_to_rf(dev);</div><div class='add'>+</div><div class='add'>+	if (!ah)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	if (ah-&gt;ah_info.ah_valid) {</div><div class='add'>+		irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL);</div><div class='add'>+		irdma_free_rsrc(rf, rf-&gt;allocated_ahs, ah-&gt;ah_info.ah_idx);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	kfree(ah);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP</div><div class='add'>+ * @cqp_request: pointer to cqp_request of create AH</div><div class='add'>+ */</div><div class='add'>+void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_ah *sc_ah = cqp_request-&gt;param;</div><div class='add'>+</div><div class='add'>+	if (!cqp_request-&gt;compl_info.op_ret_val)</div><div class='add'>+		sc_ah-&gt;ah_info.ah_valid = true;</div><div class='add'>+	else</div><div class='add'>+		sc_ah-&gt;ah_info.ah_valid = false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_prm_add_pble_mem - add moemory to pble resources</div><div class='add'>+ * @pprm: pble resource manager</div><div class='add'>+ * @pchunk: chunk of memory to add</div><div class='add'>+ */</div><div class='add'>+int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,</div><div class='add'>+			   struct irdma_chunk *pchunk)</div><div class='add'>+{</div><div class='add'>+	u64 sizeofbitmap;</div><div class='add'>+</div><div class='add'>+	if (pchunk-&gt;size &amp; 0xfff)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	sizeofbitmap = (u64)pchunk-&gt;size &gt;&gt; pprm-&gt;pble_shift;</div><div class='add'>+</div><div class='add'>+	pchunk-&gt;bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL);</div><div class='add'>+	if (!pchunk-&gt;bitmapbuf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	pchunk-&gt;sizeofbitmap = sizeofbitmap;</div><div class='add'>+	/* each pble is 8 bytes hence shift by 3 */</div><div class='add'>+	pprm-&gt;total_pble_alloc += pchunk-&gt;size &gt;&gt; 3;</div><div class='add'>+	pprm-&gt;free_pble_cnt += pchunk-&gt;size &gt;&gt; 3;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_prm_get_pbles - get pble's from prm</div><div class='add'>+ * @pprm: pble resource manager</div><div class='add'>+ * @chunkinfo: nformation about chunk where pble's were acquired</div><div class='add'>+ * @mem_size: size of pble memory needed</div><div class='add'>+ * @vaddr: returns virtual address of pble memory</div><div class='add'>+ * @fpm_addr: returns fpm address of pble memory</div><div class='add'>+ */</div><div class='add'>+int irdma_prm_get_pbles(struct irdma_pble_prm *pprm,</div><div class='add'>+			struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,</div><div class='add'>+			u64 **vaddr, u64 *fpm_addr)</div><div class='add'>+{</div><div class='add'>+	u64 bits_needed;</div><div class='add'>+	u64 bit_idx = PBLE_INVALID_IDX;</div><div class='add'>+	struct irdma_chunk *pchunk = NULL;</div><div class='add'>+	struct list_head *chunk_entry = pprm-&gt;clist.next;</div><div class='add'>+	u32 offset;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	*vaddr = NULL;</div><div class='add'>+	*fpm_addr = 0;</div><div class='add'>+</div><div class='add'>+	bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm-&gt;pble_shift));</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;pprm-&gt;prm_lock, flags);</div><div class='add'>+	while (chunk_entry != &amp;pprm-&gt;clist) {</div><div class='add'>+		pchunk = (struct irdma_chunk *)chunk_entry;</div><div class='add'>+		bit_idx = bitmap_find_next_zero_area(pchunk-&gt;bitmapbuf,</div><div class='add'>+						     pchunk-&gt;sizeofbitmap, 0,</div><div class='add'>+						     bits_needed, 0);</div><div class='add'>+		if (bit_idx &lt; pchunk-&gt;sizeofbitmap)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		/* list.next used macro */</div><div class='add'>+		chunk_entry = pchunk-&gt;list.next;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!pchunk || bit_idx &gt;= pchunk-&gt;sizeofbitmap) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;pprm-&gt;prm_lock, flags);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	bitmap_set(pchunk-&gt;bitmapbuf, bit_idx, bits_needed);</div><div class='add'>+	offset = bit_idx &lt;&lt; pprm-&gt;pble_shift;</div><div class='add'>+	*vaddr = pchunk-&gt;vaddr + offset;</div><div class='add'>+	*fpm_addr = pchunk-&gt;fpm_addr + offset;</div><div class='add'>+</div><div class='add'>+	chunkinfo-&gt;pchunk = pchunk;</div><div class='add'>+	chunkinfo-&gt;bit_idx = bit_idx;</div><div class='add'>+	chunkinfo-&gt;bits_used = bits_needed;</div><div class='add'>+	/* 3 is sizeof pble divide */</div><div class='add'>+	pprm-&gt;free_pble_cnt -= chunkinfo-&gt;bits_used &lt;&lt; (pprm-&gt;pble_shift - 3);</div><div class='add'>+	spin_unlock_irqrestore(&amp;pprm-&gt;prm_lock, flags);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_prm_return_pbles - return pbles back to prm</div><div class='add'>+ * @pprm: pble resource manager</div><div class='add'>+ * @chunkinfo: chunk where pble's were acquired and to be freed</div><div class='add'>+ */</div><div class='add'>+void irdma_prm_return_pbles(struct irdma_pble_prm *pprm,</div><div class='add'>+			    struct irdma_pble_chunkinfo *chunkinfo)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;pprm-&gt;prm_lock, flags);</div><div class='add'>+	pprm-&gt;free_pble_cnt += chunkinfo-&gt;bits_used &lt;&lt; (pprm-&gt;pble_shift - 3);</div><div class='add'>+	bitmap_clear(chunkinfo-&gt;pchunk-&gt;bitmapbuf, chunkinfo-&gt;bit_idx,</div><div class='add'>+		     chunkinfo-&gt;bits_used);</div><div class='add'>+	spin_unlock_irqrestore(&amp;pprm-&gt;prm_lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma,</div><div class='add'>+			   u32 pg_cnt)</div><div class='add'>+{</div><div class='add'>+	struct page *vm_page;</div><div class='add'>+	int i;</div><div class='add'>+	u8 *addr;</div><div class='add'>+</div><div class='add'>+	addr = (u8 *)(uintptr_t)va;</div><div class='add'>+	for (i = 0; i &lt; pg_cnt; i++) {</div><div class='add'>+		vm_page = vmalloc_to_page(addr);</div><div class='add'>+		if (!vm_page)</div><div class='add'>+			goto err;</div><div class='add'>+</div><div class='add'>+		pg_dma[i] = dma_map_page(hw-&gt;device, vm_page, 0, PAGE_SIZE,</div><div class='add'>+					 DMA_BIDIRECTIONAL);</div><div class='add'>+		if (dma_mapping_error(hw-&gt;device, pg_dma[i]))</div><div class='add'>+			goto err;</div><div class='add'>+</div><div class='add'>+		addr += PAGE_SIZE;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	irdma_unmap_vm_page_list(hw, pg_dma, i);</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; pg_cnt; i++)</div><div class='add'>+		dma_unmap_page(hw-&gt;device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_pble_free_paged_mem - free virtual paged memory</div><div class='add'>+ * @chunk: chunk to free with paged memory</div><div class='add'>+ */</div><div class='add'>+void irdma_pble_free_paged_mem(struct irdma_chunk *chunk)</div><div class='add'>+{</div><div class='add'>+	if (!chunk-&gt;pg_cnt)</div><div class='add'>+		goto done;</div><div class='add'>+</div><div class='add'>+	irdma_unmap_vm_page_list(chunk-&gt;dev-&gt;hw, chunk-&gt;dmainfo.dmaaddrs,</div><div class='add'>+				 chunk-&gt;pg_cnt);</div><div class='add'>+</div><div class='add'>+done:</div><div class='add'>+	kfree(chunk-&gt;dmainfo.dmaaddrs);</div><div class='add'>+	chunk-&gt;dmainfo.dmaaddrs = NULL;</div><div class='add'>+	vfree(chunk-&gt;vaddr);</div><div class='add'>+	chunk-&gt;vaddr = NULL;</div><div class='add'>+	chunk-&gt;type = 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_pble_get_paged_mem -allocate paged memory for pbles</div><div class='add'>+ * @chunk: chunk to add for paged memory</div><div class='add'>+ * @pg_cnt: number of pages needed</div><div class='add'>+ */</div><div class='add'>+int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt)</div><div class='add'>+{</div><div class='add'>+	u32 size;</div><div class='add'>+	void *va;</div><div class='add'>+</div><div class='add'>+	chunk-&gt;dmainfo.dmaaddrs = kzalloc(pg_cnt &lt;&lt; 3, GFP_KERNEL);</div><div class='add'>+	if (!chunk-&gt;dmainfo.dmaaddrs)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	size = PAGE_SIZE * pg_cnt;</div><div class='add'>+	va = vmalloc(size);</div><div class='add'>+	if (!va)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	if (irdma_map_vm_page_list(chunk-&gt;dev-&gt;hw, va, chunk-&gt;dmainfo.dmaaddrs,</div><div class='add'>+				   pg_cnt)) {</div><div class='add'>+		vfree(va);</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+	chunk-&gt;vaddr = va;</div><div class='add'>+	chunk-&gt;size = size;</div><div class='add'>+	chunk-&gt;pg_cnt = pg_cnt;</div><div class='add'>+	chunk-&gt;type = PBLE_SD_PAGED;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+err:</div><div class='add'>+	kfree(chunk-&gt;dmainfo.dmaaddrs);</div><div class='add'>+	chunk-&gt;dmainfo.dmaaddrs = NULL;</div><div class='add'>+</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID</div><div class='add'>+ * @dev: device pointer</div><div class='add'>+ */</div><div class='add'>+u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = dev_to_rf(dev);</div><div class='add'>+	u32 next = 1;</div><div class='add'>+	u32 node_id;</div><div class='add'>+</div><div class='add'>+	if (irdma_alloc_rsrc(rf, rf-&gt;allocated_ws_nodes, rf-&gt;max_ws_node_id,</div><div class='add'>+			     &amp;node_id, &amp;next))</div><div class='add'>+		return IRDMA_WS_NODE_INVALID;</div><div class='add'>+</div><div class='add'>+	return (u16)node_id;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_free_ws_node_id - Free a tx scheduler node ID</div><div class='add'>+ * @dev: device pointer</div><div class='add'>+ * @node_id: Work scheduler node ID</div><div class='add'>+ */</div><div class='add'>+void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = dev_to_rf(dev);</div><div class='add'>+</div><div class='add'>+	irdma_free_rsrc(rf, rf-&gt;allocated_ws_nodes, (u32)node_id);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_modify_qp_to_err - Modify a QP to error</div><div class='add'>+ * @sc_qp: qp structure</div><div class='add'>+ */</div><div class='add'>+void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *qp = sc_qp-&gt;qp_uk.back_qp;</div><div class='add'>+	struct ib_qp_attr attr;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;iwdev-&gt;rf-&gt;reset)</div><div class='add'>+		return;</div><div class='add'>+	attr.qp_state = IB_QPS_ERR;</div><div class='add'>+</div><div class='add'>+	if (rdma_protocol_roce(qp-&gt;ibqp.device, 1))</div><div class='add'>+		irdma_modify_qp_roce(&amp;qp-&gt;ibqp, &amp;attr, IB_QP_STATE, NULL);</div><div class='add'>+	else</div><div class='add'>+		irdma_modify_qp(&amp;qp-&gt;ibqp, &amp;attr, IB_QP_STATE, NULL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event)</div><div class='add'>+{</div><div class='add'>+	struct ib_event ibevent;</div><div class='add'>+</div><div class='add'>+	if (!iwqp-&gt;ibqp.event_handler)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	switch (event) {</div><div class='add'>+	case IRDMA_QP_EVENT_CATASTROPHIC:</div><div class='add'>+		ibevent.event = IB_EVENT_QP_FATAL;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_QP_EVENT_ACCESS_ERR:</div><div class='add'>+		ibevent.event = IB_EVENT_QP_ACCESS_ERR;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_QP_EVENT_REQ_ERR:</div><div class='add'>+		ibevent.event = IB_EVENT_QP_REQ_ERR;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+	ibevent.device = iwqp-&gt;ibqp.device;</div><div class='add'>+	ibevent.element.qp = &amp;iwqp-&gt;ibqp;</div><div class='add'>+	iwqp-&gt;ibqp.event_handler(&amp;ibevent, iwqp-&gt;ibqp.qp_context);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void irdma_remove_cmpls_list(struct irdma_cq *iwcq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cmpl_gen *cmpl_node;</div><div class='add'>+	struct list_head *tmp_node, *list_node;</div><div class='add'>+</div><div class='add'>+	list_for_each_safe (list_node, tmp_node, &amp;iwcq-&gt;cmpl_generated) {</div><div class='add'>+		cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list);</div><div class='add'>+		list_del(&amp;cmpl_node-&gt;list);</div><div class='add'>+		kfree(cmpl_node);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cmpl_gen *cmpl;</div><div class='add'>+</div><div class='add'>+	if (list_empty(&amp;iwcq-&gt;cmpl_generated))</div><div class='add'>+		return -ENOENT;</div><div class='add'>+	cmpl = list_first_entry_or_null(&amp;iwcq-&gt;cmpl_generated, struct irdma_cmpl_gen, list);</div><div class='add'>+	list_del(&amp;cmpl-&gt;list);</div><div class='add'>+	memcpy(cq_poll_info, &amp;cmpl-&gt;cpi, sizeof(*cq_poll_info));</div><div class='add'>+	kfree(cmpl);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(iwcq-&gt;ibcq.device,</div><div class='add'>+		  "VERBS: %s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%llx\n",</div><div class='add'>+		  __func__, cq_poll_info-&gt;qp_id, cq_poll_info-&gt;op_type,</div><div class='add'>+		  cq_poll_info-&gt;wr_id);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_set_cpi_common_values - fill in values for polling info struct</div><div class='add'>+ * @cpi: resulting structure of cq_poll_info type</div><div class='add'>+ * @qp: QPair</div><div class='add'>+ * @qp_num: id of the QP</div><div class='add'>+ */</div><div class='add'>+static void irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi,</div><div class='add'>+					struct irdma_qp_uk *qp, u32 qp_num)</div><div class='add'>+{</div><div class='add'>+	cpi-&gt;comp_status = IRDMA_COMPL_STATUS_FLUSHED;</div><div class='add'>+	cpi-&gt;error = true;</div><div class='add'>+	cpi-&gt;major_err = IRDMA_FLUSH_MAJOR_ERR;</div><div class='add'>+	cpi-&gt;minor_err = FLUSH_GENERAL_ERR;</div><div class='add'>+	cpi-&gt;qp_handle = (irdma_qp_handle)(uintptr_t)qp;</div><div class='add'>+	cpi-&gt;qp_id = qp_num;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void irdma_comp_handler(struct irdma_cq *cq)</div><div class='add'>+{</div><div class='add'>+	if (!cq-&gt;ibcq.comp_handler)</div><div class='add'>+		return;</div><div class='add'>+	if (atomic_cmpxchg(&amp;cq-&gt;armed, 1, 0))</div><div class='add'>+		cq-&gt;ibcq.comp_handler(&amp;cq-&gt;ibcq, cq-&gt;ibcq.cq_context);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void irdma_generate_flush_completions(struct irdma_qp *iwqp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp_uk *qp = &amp;iwqp-&gt;sc_qp.qp_uk;</div><div class='add'>+	struct irdma_ring *sq_ring = &amp;qp-&gt;sq_ring;</div><div class='add'>+	struct irdma_ring *rq_ring = &amp;qp-&gt;rq_ring;</div><div class='add'>+	struct irdma_cq *iwscq = iwqp-&gt;iwscq;</div><div class='add'>+	struct irdma_cq *iwrcq = iwqp-&gt;iwrcq;</div><div class='add'>+	struct irdma_cmpl_gen *cmpl;</div><div class='add'>+	__le64 *sw_wqe;</div><div class='add'>+	u64 wqe_qword;</div><div class='add'>+	u32 wqe_idx;</div><div class='add'>+	bool compl_generated = false;</div><div class='add'>+	unsigned long flags1;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwscq-&gt;lock, flags1);</div><div class='add'>+	if (irdma_uk_cq_empty(&amp;iwscq-&gt;sc_cq.cq_uk)) {</div><div class='add'>+		unsigned long flags2;</div><div class='add'>+</div><div class='add'>+		spin_lock_irqsave(&amp;iwqp-&gt;lock, flags2);</div><div class='add'>+		while (IRDMA_RING_MORE_WORK(*sq_ring)) {</div><div class='add'>+			cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);</div><div class='add'>+			if (!cmpl) {</div><div class='add'>+				spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags2);</div><div class='add'>+				spin_unlock_irqrestore(&amp;iwscq-&gt;lock, flags1);</div><div class='add'>+				return;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			wqe_idx = sq_ring-&gt;tail;</div><div class='add'>+			irdma_set_cpi_common_values(&amp;cmpl-&gt;cpi, qp, qp-&gt;qp_id);</div><div class='add'>+</div><div class='add'>+			cmpl-&gt;cpi.wr_id = qp-&gt;sq_wrtrk_array[wqe_idx].wrid;</div><div class='add'>+			sw_wqe = qp-&gt;sq_base[wqe_idx].elem;</div><div class='add'>+			get_64bit_val(sw_wqe, 24, &amp;wqe_qword);</div><div class='add'>+			cmpl-&gt;cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE);</div><div class='add'>+			cmpl-&gt;cpi.q_type = IRDMA_CQE_QTYPE_SQ;</div><div class='add'>+			/* remove the SQ WR by moving SQ tail*/</div><div class='add'>+			IRDMA_RING_SET_TAIL(*sq_ring,</div><div class='add'>+				sq_ring-&gt;tail + qp-&gt;sq_wrtrk_array[sq_ring-&gt;tail].quanta);</div><div class='add'>+			if (cmpl-&gt;cpi.op_type == IRDMAQP_OP_NOP) {</div><div class='add'>+				kfree(cmpl);</div><div class='add'>+				continue;</div><div class='add'>+			}</div><div class='add'>+			ibdev_dbg(iwscq-&gt;ibcq.device,</div><div class='add'>+				  "DEV: %s: adding wr_id = 0x%llx SQ Completion to list qp_id=%d\n",</div><div class='add'>+				  __func__, cmpl-&gt;cpi.wr_id, qp-&gt;qp_id);</div><div class='add'>+			list_add_tail(&amp;cmpl-&gt;list, &amp;iwscq-&gt;cmpl_generated);</div><div class='add'>+			compl_generated = true;</div><div class='add'>+		}</div><div class='add'>+		spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags2);</div><div class='add'>+		spin_unlock_irqrestore(&amp;iwscq-&gt;lock, flags1);</div><div class='add'>+		if (compl_generated)</div><div class='add'>+			irdma_comp_handler(iwscq);</div><div class='add'>+	} else {</div><div class='add'>+		spin_unlock_irqrestore(&amp;iwscq-&gt;lock, flags1);</div><div class='add'>+		mod_delayed_work(iwqp-&gt;iwdev-&gt;cleanup_wq, &amp;iwqp-&gt;dwork_flush,</div><div class='add'>+				 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwrcq-&gt;lock, flags1);</div><div class='add'>+	if (irdma_uk_cq_empty(&amp;iwrcq-&gt;sc_cq.cq_uk)) {</div><div class='add'>+		unsigned long flags2;</div><div class='add'>+</div><div class='add'>+		spin_lock_irqsave(&amp;iwqp-&gt;lock, flags2);</div><div class='add'>+		while (IRDMA_RING_MORE_WORK(*rq_ring)) {</div><div class='add'>+			cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);</div><div class='add'>+			if (!cmpl) {</div><div class='add'>+				spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags2);</div><div class='add'>+				spin_unlock_irqrestore(&amp;iwrcq-&gt;lock, flags1);</div><div class='add'>+				return;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			wqe_idx = rq_ring-&gt;tail;</div><div class='add'>+			irdma_set_cpi_common_values(&amp;cmpl-&gt;cpi, qp, qp-&gt;qp_id);</div><div class='add'>+</div><div class='add'>+			cmpl-&gt;cpi.wr_id = qp-&gt;rq_wrid_array[wqe_idx];</div><div class='add'>+			cmpl-&gt;cpi.op_type = IRDMA_OP_TYPE_REC;</div><div class='add'>+			cmpl-&gt;cpi.q_type = IRDMA_CQE_QTYPE_RQ;</div><div class='add'>+			/* remove the RQ WR by moving RQ tail */</div><div class='add'>+			IRDMA_RING_SET_TAIL(*rq_ring, rq_ring-&gt;tail + 1);</div><div class='add'>+			ibdev_dbg(iwrcq-&gt;ibcq.device,</div><div class='add'>+				  "DEV: %s: adding wr_id = 0x%llx RQ Completion to list qp_id=%d, wqe_idx=%d\n",</div><div class='add'>+				  __func__, cmpl-&gt;cpi.wr_id, qp-&gt;qp_id,</div><div class='add'>+				  wqe_idx);</div><div class='add'>+			list_add_tail(&amp;cmpl-&gt;list, &amp;iwrcq-&gt;cmpl_generated);</div><div class='add'>+</div><div class='add'>+			compl_generated = true;</div><div class='add'>+		}</div><div class='add'>+		spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags2);</div><div class='add'>+		spin_unlock_irqrestore(&amp;iwrcq-&gt;lock, flags1);</div><div class='add'>+		if (compl_generated)</div><div class='add'>+			irdma_comp_handler(iwrcq);</div><div class='add'>+	} else {</div><div class='add'>+		spin_unlock_irqrestore(&amp;iwrcq-&gt;lock, flags1);</div><div class='add'>+		mod_delayed_work(iwqp-&gt;iwdev-&gt;cleanup_wq, &amp;iwqp-&gt;dwork_flush,</div><div class='add'>+				 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));</div><div class='add'>+	}</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c<br/>new file mode 100644<br/>index 000000000000..6d9af41a2884<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/verbs.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/verbs.c</a></div><div class='hunk'>@@ -0,0 +1,5517 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#include "main.h"</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_query_device - get device attributes</div><div class='add'>+ * @ibdev: device pointer from stack</div><div class='add'>+ * @props: returning device attributes</div><div class='add'>+ * @udata: user data</div><div class='add'>+ */</div><div class='add'>+static int irdma_query_device(struct ib_device *ibdev,</div><div class='add'>+			      struct ib_device_attr *props,</div><div class='add'>+			      struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibdev);</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+	struct pci_dev *pcidev = iwdev-&gt;rf-&gt;pcidev;</div><div class='add'>+	struct irdma_hw_attrs *hw_attrs = &amp;rf-&gt;sc_dev.hw_attrs;</div><div class='add'>+</div><div class='add'>+	if (udata-&gt;inlen || udata-&gt;outlen)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	memset(props, 0, sizeof(*props));</div><div class='add'>+	addrconf_addr_eui48((u8 *)&amp;props-&gt;sys_image_guid,</div><div class='add'>+			    iwdev-&gt;netdev-&gt;dev_addr);</div><div class='add'>+	props-&gt;fw_ver = (u64)irdma_fw_major_ver(&amp;rf-&gt;sc_dev) &lt;&lt; 32 |</div><div class='add'>+			irdma_fw_minor_ver(&amp;rf-&gt;sc_dev);</div><div class='add'>+	props-&gt;device_cap_flags = IB_DEVICE_MEM_WINDOW |</div><div class='add'>+				  IB_DEVICE_MEM_MGT_EXTENSIONS;</div><div class='add'>+	if (hw_attrs-&gt;uk_attrs.hw_rev &lt; IRDMA_GEN_3)</div><div class='add'>+		props-&gt;kernel_cap_flags = IBK_LOCAL_DMA_LKEY;</div><div class='add'>+	props-&gt;vendor_id = pcidev-&gt;vendor;</div><div class='add'>+	props-&gt;vendor_part_id = pcidev-&gt;device;</div><div class='add'>+</div><div class='add'>+	props-&gt;hw_ver = rf-&gt;pcidev-&gt;revision;</div><div class='add'>+	props-&gt;page_size_cap = hw_attrs-&gt;page_size_cap;</div><div class='add'>+	props-&gt;max_mr_size = hw_attrs-&gt;max_mr_size;</div><div class='add'>+	props-&gt;max_qp = rf-&gt;max_qp - rf-&gt;used_qps;</div><div class='add'>+	props-&gt;max_qp_wr = hw_attrs-&gt;max_qp_wr;</div><div class='add'>+	props-&gt;max_send_sge = hw_attrs-&gt;uk_attrs.max_hw_wq_frags;</div><div class='add'>+	props-&gt;max_recv_sge = hw_attrs-&gt;uk_attrs.max_hw_wq_frags;</div><div class='add'>+	props-&gt;max_cq = rf-&gt;max_cq - rf-&gt;used_cqs;</div><div class='add'>+	props-&gt;max_cqe = rf-&gt;max_cqe - 1;</div><div class='add'>+	props-&gt;max_mr = rf-&gt;max_mr - rf-&gt;used_mrs;</div><div class='add'>+	if (hw_attrs-&gt;uk_attrs.hw_rev &gt;= IRDMA_GEN_3)</div><div class='add'>+		props-&gt;max_mw = props-&gt;max_mr;</div><div class='add'>+	props-&gt;max_pd = rf-&gt;max_pd - rf-&gt;used_pds;</div><div class='add'>+	props-&gt;max_sge_rd = hw_attrs-&gt;uk_attrs.max_hw_read_sges;</div><div class='add'>+	props-&gt;max_qp_rd_atom = hw_attrs-&gt;max_hw_ird;</div><div class='add'>+	props-&gt;max_qp_init_rd_atom = hw_attrs-&gt;max_hw_ord;</div><div class='add'>+	if (rdma_protocol_roce(ibdev, 1)) {</div><div class='add'>+		props-&gt;device_cap_flags |= IB_DEVICE_RC_RNR_NAK_GEN;</div><div class='add'>+		props-&gt;max_pkeys = IRDMA_PKEY_TBL_SZ;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	props-&gt;max_ah = rf-&gt;max_ah;</div><div class='add'>+	props-&gt;max_mcast_grp = rf-&gt;max_mcg;</div><div class='add'>+	props-&gt;max_mcast_qp_attach = IRDMA_MAX_MGS_PER_CTX;</div><div class='add'>+	props-&gt;max_total_mcast_qp_attach = rf-&gt;max_qp * IRDMA_MAX_MGS_PER_CTX;</div><div class='add'>+	props-&gt;max_fast_reg_page_list_len = IRDMA_MAX_PAGES_PER_FMR;</div><div class='add'>+	props-&gt;max_srq = rf-&gt;max_srq - rf-&gt;used_srqs;</div><div class='add'>+	props-&gt;max_srq_wr = IRDMA_MAX_SRQ_WRS;</div><div class='add'>+	props-&gt;max_srq_sge = hw_attrs-&gt;uk_attrs.max_hw_wq_frags;</div><div class='add'>+	if (hw_attrs-&gt;uk_attrs.feature_flags &amp; IRDMA_FEATURE_ATOMIC_OPS)</div><div class='add'>+		props-&gt;atomic_cap = IB_ATOMIC_HCA;</div><div class='add'>+	else</div><div class='add'>+		props-&gt;atomic_cap = IB_ATOMIC_NONE;</div><div class='add'>+	props-&gt;masked_atomic_cap = props-&gt;atomic_cap;</div><div class='add'>+	if (hw_attrs-&gt;uk_attrs.hw_rev &gt;= IRDMA_GEN_3) {</div><div class='add'>+#define HCA_CORE_CLOCK_KHZ 1000000UL</div><div class='add'>+		props-&gt;timestamp_mask = GENMASK(31, 0);</div><div class='add'>+		props-&gt;hca_core_clock = HCA_CORE_CLOCK_KHZ;</div><div class='add'>+	}</div><div class='add'>+	if (hw_attrs-&gt;uk_attrs.hw_rev &gt;= IRDMA_GEN_3)</div><div class='add'>+		props-&gt;device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2B;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_query_port - get port attributes</div><div class='add'>+ * @ibdev: device pointer from stack</div><div class='add'>+ * @port: port number for query</div><div class='add'>+ * @props: returning device attributes</div><div class='add'>+ */</div><div class='add'>+static int irdma_query_port(struct ib_device *ibdev, u32 port,</div><div class='add'>+			    struct ib_port_attr *props)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibdev);</div><div class='add'>+	struct net_device *netdev = iwdev-&gt;netdev;</div><div class='add'>+</div><div class='add'>+	/* no need to zero out pros here. done by caller */</div><div class='add'>+</div><div class='add'>+	props-&gt;max_mtu = IB_MTU_4096;</div><div class='add'>+	props-&gt;active_mtu = ib_mtu_int_to_enum(netdev-&gt;mtu);</div><div class='add'>+	props-&gt;lid = 1;</div><div class='add'>+	props-&gt;lmc = 0;</div><div class='add'>+	props-&gt;sm_lid = 0;</div><div class='add'>+	props-&gt;sm_sl = 0;</div><div class='add'>+	if (netif_carrier_ok(netdev) &amp;&amp; netif_running(netdev)) {</div><div class='add'>+		props-&gt;state = IB_PORT_ACTIVE;</div><div class='add'>+		props-&gt;phys_state = IB_PORT_PHYS_STATE_LINK_UP;</div><div class='add'>+	} else {</div><div class='add'>+		props-&gt;state = IB_PORT_DOWN;</div><div class='add'>+		props-&gt;phys_state = IB_PORT_PHYS_STATE_DISABLED;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ib_get_eth_speed(ibdev, port, &amp;props-&gt;active_speed,</div><div class='add'>+			 &amp;props-&gt;active_width);</div><div class='add'>+</div><div class='add'>+	if (rdma_protocol_roce(ibdev, 1)) {</div><div class='add'>+		props-&gt;gid_tbl_len = 32;</div><div class='add'>+		props-&gt;ip_gids = true;</div><div class='add'>+		props-&gt;pkey_tbl_len = IRDMA_PKEY_TBL_SZ;</div><div class='add'>+	} else {</div><div class='add'>+		props-&gt;gid_tbl_len = 1;</div><div class='add'>+	}</div><div class='add'>+	props-&gt;qkey_viol_cntr = 0;</div><div class='add'>+	props-&gt;port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP;</div><div class='add'>+	props-&gt;max_msg_sz = iwdev-&gt;rf-&gt;sc_dev.hw_attrs.max_hw_outbound_msg_size;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_disassociate_ucontext - Disassociate user context</div><div class='add'>+ * @context: ib user context</div><div class='add'>+ */</div><div class='add'>+static void irdma_disassociate_ucontext(struct ib_ucontext *context)</div><div class='add'>+{</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_mmap_legacy(struct irdma_ucontext *ucontext,</div><div class='add'>+			     struct vm_area_struct *vma)</div><div class='add'>+{</div><div class='add'>+	u64 pfn;</div><div class='add'>+</div><div class='add'>+	if (vma-&gt;vm_pgoff || vma-&gt;vm_end - vma-&gt;vm_start != PAGE_SIZE)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	vma-&gt;vm_private_data = ucontext;</div><div class='add'>+	pfn = ((uintptr_t)ucontext-&gt;iwdev-&gt;rf-&gt;sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET] +</div><div class='add'>+	       pci_resource_start(ucontext-&gt;iwdev-&gt;rf-&gt;pcidev, 0)) &gt;&gt; PAGE_SHIFT;</div><div class='add'>+</div><div class='add'>+	return rdma_user_mmap_io(&amp;ucontext-&gt;ibucontext, vma, pfn, PAGE_SIZE,</div><div class='add'>+				 pgprot_noncached(vma-&gt;vm_page_prot), NULL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry)</div><div class='add'>+{</div><div class='add'>+	struct irdma_user_mmap_entry *entry = to_irdma_mmap_entry(rdma_entry);</div><div class='add'>+</div><div class='add'>+	kfree(entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct rdma_user_mmap_entry*</div><div class='add'>+irdma_user_mmap_entry_insert(struct irdma_ucontext *ucontext, u64 bar_offset,</div><div class='add'>+			     enum irdma_mmap_flag mmap_flag, u64 *mmap_offset)</div><div class='add'>+{</div><div class='add'>+	struct irdma_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!entry)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	entry-&gt;bar_offset = bar_offset;</div><div class='add'>+	entry-&gt;mmap_flag = mmap_flag;</div><div class='add'>+</div><div class='add'>+	ret = rdma_user_mmap_entry_insert(&amp;ucontext-&gt;ibucontext,</div><div class='add'>+					  &amp;entry-&gt;rdma_entry, PAGE_SIZE);</div><div class='add'>+	if (ret) {</div><div class='add'>+		kfree(entry);</div><div class='add'>+		return NULL;</div><div class='add'>+	}</div><div class='add'>+	*mmap_offset = rdma_user_mmap_get_offset(&amp;entry-&gt;rdma_entry);</div><div class='add'>+</div><div class='add'>+	return &amp;entry-&gt;rdma_entry;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_mmap - user memory map</div><div class='add'>+ * @context: context created during alloc</div><div class='add'>+ * @vma: kernel info for user memory map</div><div class='add'>+ */</div><div class='add'>+static int irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)</div><div class='add'>+{</div><div class='add'>+	struct rdma_user_mmap_entry *rdma_entry;</div><div class='add'>+	struct irdma_user_mmap_entry *entry;</div><div class='add'>+	struct irdma_ucontext *ucontext;</div><div class='add'>+	u64 pfn;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ucontext = to_ucontext(context);</div><div class='add'>+</div><div class='add'>+	/* Legacy support for libi40iw with hard-coded mmap key */</div><div class='add'>+	if (ucontext-&gt;legacy_mode)</div><div class='add'>+		return irdma_mmap_legacy(ucontext, vma);</div><div class='add'>+</div><div class='add'>+	rdma_entry = rdma_user_mmap_entry_get(&amp;ucontext-&gt;ibucontext, vma);</div><div class='add'>+	if (!rdma_entry) {</div><div class='add'>+		ibdev_dbg(&amp;ucontext-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "VERBS: pgoff[0x%lx] does not have valid entry\n",</div><div class='add'>+			  vma-&gt;vm_pgoff);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	entry = to_irdma_mmap_entry(rdma_entry);</div><div class='add'>+	ibdev_dbg(&amp;ucontext-&gt;iwdev-&gt;ibdev,</div><div class='add'>+		  "VERBS: bar_offset [0x%llx] mmap_flag [%d]\n",</div><div class='add'>+		  entry-&gt;bar_offset, entry-&gt;mmap_flag);</div><div class='add'>+</div><div class='add'>+	pfn = (entry-&gt;bar_offset +</div><div class='add'>+	      pci_resource_start(ucontext-&gt;iwdev-&gt;rf-&gt;pcidev, 0)) &gt;&gt; PAGE_SHIFT;</div><div class='add'>+</div><div class='add'>+	switch (entry-&gt;mmap_flag) {</div><div class='add'>+	case IRDMA_MMAP_IO_NC:</div><div class='add'>+		ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE,</div><div class='add'>+					pgprot_noncached(vma-&gt;vm_page_prot),</div><div class='add'>+					rdma_entry);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_MMAP_IO_WC:</div><div class='add'>+		ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE,</div><div class='add'>+					pgprot_writecombine(vma-&gt;vm_page_prot),</div><div class='add'>+					rdma_entry);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_dbg(&amp;ucontext-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			  "VERBS: bar_offset [0x%llx] mmap_flag[%d] err[%d]\n",</div><div class='add'>+			  entry-&gt;bar_offset, entry-&gt;mmap_flag, ret);</div><div class='add'>+	rdma_user_mmap_entry_put(rdma_entry);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_alloc_push_page - allocate a push page for qp</div><div class='add'>+ * @iwqp: qp pointer</div><div class='add'>+ */</div><div class='add'>+static void irdma_alloc_push_page(struct irdma_qp *iwqp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	struct irdma_device *iwdev = iwqp-&gt;iwdev;</div><div class='add'>+	struct irdma_sc_qp *qp = &amp;iwqp-&gt;sc_qp;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;iwdev-&gt;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.manage_push_page.info.push_idx = 0;</div><div class='add'>+	cqp_info-&gt;in.u.manage_push_page.info.qs_handle =</div><div class='add'>+		qp-&gt;vsi-&gt;qos[qp-&gt;user_pri].qs_handle;</div><div class='add'>+	cqp_info-&gt;in.u.manage_push_page.info.free_page = 0;</div><div class='add'>+	cqp_info-&gt;in.u.manage_push_page.info.push_page_type = 0;</div><div class='add'>+	cqp_info-&gt;in.u.manage_push_page.cqp = &amp;iwdev-&gt;rf-&gt;cqp.sc_cqp;</div><div class='add'>+	cqp_info-&gt;in.u.manage_push_page.scratch = (uintptr_t)cqp_request;</div><div class='add'>+</div><div class='add'>+	status = irdma_handle_cqp_op(iwdev-&gt;rf, cqp_request);</div><div class='add'>+	if (!status &amp;&amp; cqp_request-&gt;compl_info.op_ret_val &lt;</div><div class='add'>+	    iwdev-&gt;rf-&gt;sc_dev.hw_attrs.max_hw_device_pages) {</div><div class='add'>+		qp-&gt;push_idx = cqp_request-&gt;compl_info.op_ret_val;</div><div class='add'>+		qp-&gt;push_offset = 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_put_cqp_request(&amp;iwdev-&gt;rf-&gt;cqp, cqp_request);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_alloc_ucontext - Allocate the user context data structure</div><div class='add'>+ * @uctx: uverbs context pointer</div><div class='add'>+ * @udata: user data</div><div class='add'>+ *</div><div class='add'>+ * This keeps track of all objects associated with a particular</div><div class='add'>+ * user-mode client.</div><div class='add'>+ */</div><div class='add'>+static int irdma_alloc_ucontext(struct ib_ucontext *uctx,</div><div class='add'>+				struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+#define IRDMA_ALLOC_UCTX_MIN_REQ_LEN offsetofend(struct irdma_alloc_ucontext_req, rsvd8)</div><div class='add'>+#define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd)</div><div class='add'>+	struct ib_device *ibdev = uctx-&gt;device;</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibdev);</div><div class='add'>+	struct irdma_alloc_ucontext_req req = {};</div><div class='add'>+	struct irdma_alloc_ucontext_resp uresp = {};</div><div class='add'>+	struct irdma_ucontext *ucontext = to_ucontext(uctx);</div><div class='add'>+	struct irdma_uk_attrs *uk_attrs = &amp;iwdev-&gt;rf-&gt;sc_dev.hw_attrs.uk_attrs;</div><div class='add'>+</div><div class='add'>+	if (udata-&gt;inlen &lt; IRDMA_ALLOC_UCTX_MIN_REQ_LEN ||</div><div class='add'>+	    udata-&gt;outlen &lt; IRDMA_ALLOC_UCTX_MIN_RESP_LEN)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (ib_copy_from_udata(&amp;req, udata, min(sizeof(req), udata-&gt;inlen)))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (req.userspace_ver &lt; 4 || req.userspace_ver &gt; IRDMA_ABI_VER)</div><div class='add'>+		goto ver_error;</div><div class='add'>+</div><div class='add'>+	ucontext-&gt;iwdev = iwdev;</div><div class='add'>+	ucontext-&gt;abi_ver = req.userspace_ver;</div><div class='add'>+</div><div class='add'>+	if (!(req.comp_mask &amp; IRDMA_SUPPORT_WQE_FORMAT_V2) &amp;&amp;</div><div class='add'>+	    uk_attrs-&gt;hw_rev &gt;= IRDMA_GEN_3)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (req.comp_mask &amp; IRDMA_ALLOC_UCTX_USE_RAW_ATTR)</div><div class='add'>+		ucontext-&gt;use_raw_attrs = true;</div><div class='add'>+</div><div class='add'>+	/* GEN_1 legacy support with libi40iw */</div><div class='add'>+	if (udata-&gt;outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) {</div><div class='add'>+		if (uk_attrs-&gt;hw_rev != IRDMA_GEN_1)</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+		ucontext-&gt;legacy_mode = true;</div><div class='add'>+		uresp.max_qps = iwdev-&gt;rf-&gt;max_qp;</div><div class='add'>+		uresp.max_pds = iwdev-&gt;rf-&gt;sc_dev.hw_attrs.max_hw_pds;</div><div class='add'>+		uresp.wq_size = iwdev-&gt;rf-&gt;sc_dev.hw_attrs.max_qp_wr * 2;</div><div class='add'>+		uresp.kernel_ver = req.userspace_ver;</div><div class='add'>+		if (ib_copy_to_udata(udata, &amp;uresp,</div><div class='add'>+				     min(sizeof(uresp), udata-&gt;outlen)))</div><div class='add'>+			return -EFAULT;</div><div class='add'>+	} else {</div><div class='add'>+		u64 bar_off = (uintptr_t)iwdev-&gt;rf-&gt;sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];</div><div class='add'>+</div><div class='add'>+		ucontext-&gt;db_mmap_entry =</div><div class='add'>+			irdma_user_mmap_entry_insert(ucontext, bar_off,</div><div class='add'>+						     IRDMA_MMAP_IO_NC,</div><div class='add'>+						     &amp;uresp.db_mmap_key);</div><div class='add'>+		if (!ucontext-&gt;db_mmap_entry)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+</div><div class='add'>+		uresp.kernel_ver = IRDMA_ABI_VER;</div><div class='add'>+		uresp.feature_flags = uk_attrs-&gt;feature_flags;</div><div class='add'>+		uresp.max_hw_wq_frags = uk_attrs-&gt;max_hw_wq_frags;</div><div class='add'>+		uresp.max_hw_read_sges = uk_attrs-&gt;max_hw_read_sges;</div><div class='add'>+		uresp.max_hw_inline = uk_attrs-&gt;max_hw_inline;</div><div class='add'>+		uresp.max_hw_rq_quanta = uk_attrs-&gt;max_hw_rq_quanta;</div><div class='add'>+		uresp.max_hw_wq_quanta = uk_attrs-&gt;max_hw_wq_quanta;</div><div class='add'>+		uresp.max_hw_sq_chunk = uk_attrs-&gt;max_hw_sq_chunk;</div><div class='add'>+		uresp.max_hw_cq_size = uk_attrs-&gt;max_hw_cq_size;</div><div class='add'>+		uresp.min_hw_cq_size = uk_attrs-&gt;min_hw_cq_size;</div><div class='add'>+		uresp.hw_rev = uk_attrs-&gt;hw_rev;</div><div class='add'>+		uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR;</div><div class='add'>+		uresp.min_hw_wq_size = uk_attrs-&gt;min_hw_wq_size;</div><div class='add'>+		uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE;</div><div class='add'>+		uresp.max_hw_srq_quanta = uk_attrs-&gt;max_hw_srq_quanta;</div><div class='add'>+		uresp.comp_mask |= IRDMA_ALLOC_UCTX_MAX_HW_SRQ_QUANTA;</div><div class='add'>+		if (ib_copy_to_udata(udata, &amp;uresp,</div><div class='add'>+				     min(sizeof(uresp), udata-&gt;outlen))) {</div><div class='add'>+			rdma_user_mmap_entry_remove(ucontext-&gt;db_mmap_entry);</div><div class='add'>+			return -EFAULT;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;ucontext-&gt;cq_reg_mem_list);</div><div class='add'>+	spin_lock_init(&amp;ucontext-&gt;cq_reg_mem_list_lock);</div><div class='add'>+	INIT_LIST_HEAD(&amp;ucontext-&gt;qp_reg_mem_list);</div><div class='add'>+	spin_lock_init(&amp;ucontext-&gt;qp_reg_mem_list_lock);</div><div class='add'>+	INIT_LIST_HEAD(&amp;ucontext-&gt;srq_reg_mem_list);</div><div class='add'>+	spin_lock_init(&amp;ucontext-&gt;srq_reg_mem_list_lock);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+ver_error:</div><div class='add'>+	ibdev_err(&amp;iwdev-&gt;ibdev,</div><div class='add'>+		  "Invalid userspace driver version detected. Detected version %d, should be %d\n",</div><div class='add'>+		  req.userspace_ver, IRDMA_ABI_VER);</div><div class='add'>+	return -EINVAL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_dealloc_ucontext - deallocate the user context data structure</div><div class='add'>+ * @context: user context created during alloc</div><div class='add'>+ */</div><div class='add'>+static void irdma_dealloc_ucontext(struct ib_ucontext *context)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ucontext *ucontext = to_ucontext(context);</div><div class='add'>+</div><div class='add'>+	rdma_user_mmap_entry_remove(ucontext-&gt;db_mmap_entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_alloc_pd - allocate protection domain</div><div class='add'>+ * @pd: PD pointer</div><div class='add'>+ * @udata: user data</div><div class='add'>+ */</div><div class='add'>+static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+#define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd)</div><div class='add'>+	struct irdma_pd *iwpd = to_iwpd(pd);</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(pd-&gt;device);</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+	struct irdma_alloc_pd_resp uresp = {};</div><div class='add'>+	struct irdma_sc_pd *sc_pd;</div><div class='add'>+	u32 pd_id = 0;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (udata &amp;&amp; udata-&gt;outlen &lt; IRDMA_ALLOC_PD_MIN_RESP_LEN)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	err = irdma_alloc_rsrc(rf, rf-&gt;allocated_pds, rf-&gt;max_pd, &amp;pd_id,</div><div class='add'>+			       &amp;rf-&gt;next_pd);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	sc_pd = &amp;iwpd-&gt;sc_pd;</div><div class='add'>+	if (udata) {</div><div class='add'>+		struct irdma_ucontext *ucontext =</div><div class='add'>+			rdma_udata_to_drv_context(udata, struct irdma_ucontext,</div><div class='add'>+						  ibucontext);</div><div class='add'>+		irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext-&gt;abi_ver);</div><div class='add'>+		uresp.pd_id = pd_id;</div><div class='add'>+		if (ib_copy_to_udata(udata, &amp;uresp,</div><div class='add'>+				     min(sizeof(uresp), udata-&gt;outlen))) {</div><div class='add'>+			err = -EFAULT;</div><div class='add'>+			goto error;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+error:</div><div class='add'>+	irdma_free_rsrc(rf, rf-&gt;allocated_pds, pd_id);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_dealloc_pd - deallocate pd</div><div class='add'>+ * @ibpd: ptr of pd to be deallocated</div><div class='add'>+ * @udata: user data</div><div class='add'>+ */</div><div class='add'>+static int irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pd *iwpd = to_iwpd(ibpd);</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibpd-&gt;device);</div><div class='add'>+</div><div class='add'>+	irdma_free_rsrc(iwdev-&gt;rf, iwdev-&gt;rf-&gt;allocated_pds, iwpd-&gt;sc_pd.pd_id);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_pbl - Retrieve pbl from a list given a virtual</div><div class='add'>+ * address</div><div class='add'>+ * @va: user virtual address</div><div class='add'>+ * @pbl_list: pbl list to search in (QP's or CQ's)</div><div class='add'>+ */</div><div class='add'>+static struct irdma_pbl *irdma_get_pbl(unsigned long va,</div><div class='add'>+				       struct list_head *pbl_list)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pbl *iwpbl;</div><div class='add'>+</div><div class='add'>+	list_for_each_entry (iwpbl, pbl_list, list) {</div><div class='add'>+		if (iwpbl-&gt;user_base == va) {</div><div class='add'>+			list_del(&amp;iwpbl-&gt;list);</div><div class='add'>+			iwpbl-&gt;on_list = false;</div><div class='add'>+			return iwpbl;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_clean_cqes - clean cq entries for qp</div><div class='add'>+ * @iwqp: qp ptr (user or kernel)</div><div class='add'>+ * @iwcq: cq ptr</div><div class='add'>+ */</div><div class='add'>+static void irdma_clean_cqes(struct irdma_qp *iwqp, struct irdma_cq *iwcq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cq_uk *ukcq = &amp;iwcq-&gt;sc_cq.cq_uk;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwcq-&gt;lock, flags);</div><div class='add'>+	irdma_uk_clean_cq(&amp;iwqp-&gt;sc_qp.qp_uk, ukcq);</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwcq-&gt;lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_remove_push_mmap_entries(struct irdma_qp *iwqp)</div><div class='add'>+{</div><div class='add'>+	if (iwqp-&gt;push_db_mmap_entry) {</div><div class='add'>+		rdma_user_mmap_entry_remove(iwqp-&gt;push_db_mmap_entry);</div><div class='add'>+		iwqp-&gt;push_db_mmap_entry = NULL;</div><div class='add'>+	}</div><div class='add'>+	if (iwqp-&gt;push_wqe_mmap_entry) {</div><div class='add'>+		rdma_user_mmap_entry_remove(iwqp-&gt;push_wqe_mmap_entry);</div><div class='add'>+		iwqp-&gt;push_wqe_mmap_entry = NULL;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_setup_push_mmap_entries(struct irdma_ucontext *ucontext,</div><div class='add'>+					 struct irdma_qp *iwqp,</div><div class='add'>+					 u64 *push_wqe_mmap_key,</div><div class='add'>+					 u64 *push_db_mmap_key)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = ucontext-&gt;iwdev;</div><div class='add'>+	u64 rsvd, bar_off;</div><div class='add'>+</div><div class='add'>+	rsvd = IRDMA_PF_BAR_RSVD;</div><div class='add'>+	bar_off = (uintptr_t)iwdev-&gt;rf-&gt;sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];</div><div class='add'>+	/* skip over db page */</div><div class='add'>+	bar_off += IRDMA_HW_PAGE_SIZE;</div><div class='add'>+	/* push wqe page */</div><div class='add'>+	bar_off += rsvd + iwqp-&gt;sc_qp.push_idx * IRDMA_HW_PAGE_SIZE;</div><div class='add'>+	iwqp-&gt;push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext,</div><div class='add'>+					bar_off, IRDMA_MMAP_IO_WC,</div><div class='add'>+					push_wqe_mmap_key);</div><div class='add'>+	if (!iwqp-&gt;push_wqe_mmap_entry)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	/* push doorbell page */</div><div class='add'>+	bar_off += IRDMA_HW_PAGE_SIZE;</div><div class='add'>+	iwqp-&gt;push_db_mmap_entry = irdma_user_mmap_entry_insert(ucontext,</div><div class='add'>+					bar_off, IRDMA_MMAP_IO_NC,</div><div class='add'>+					push_db_mmap_key);</div><div class='add'>+	if (!iwqp-&gt;push_db_mmap_entry) {</div><div class='add'>+		rdma_user_mmap_entry_remove(iwqp-&gt;push_wqe_mmap_entry);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_destroy_qp - destroy qp</div><div class='add'>+ * @ibqp: qp's ib pointer also to get to device's qp address</div><div class='add'>+ * @udata: user data</div><div class='add'>+ */</div><div class='add'>+static int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *iwqp = to_iwqp(ibqp);</div><div class='add'>+	struct irdma_device *iwdev = iwqp-&gt;iwdev;</div><div class='add'>+</div><div class='add'>+	iwqp-&gt;sc_qp.qp_uk.destroy_pending = true;</div><div class='add'>+</div><div class='add'>+	if (iwqp-&gt;iwarp_state &gt;= IRDMA_QP_STATE_IDLE)</div><div class='add'>+		irdma_modify_qp_to_err(&amp;iwqp-&gt;sc_qp);</div><div class='add'>+</div><div class='add'>+	if (!iwqp-&gt;user_mode)</div><div class='add'>+		cancel_delayed_work_sync(&amp;iwqp-&gt;dwork_flush);</div><div class='add'>+</div><div class='add'>+	if (!iwqp-&gt;user_mode) {</div><div class='add'>+		if (iwqp-&gt;iwscq) {</div><div class='add'>+			irdma_clean_cqes(iwqp, iwqp-&gt;iwscq);</div><div class='add'>+			if (iwqp-&gt;iwrcq != iwqp-&gt;iwscq)</div><div class='add'>+				irdma_clean_cqes(iwqp, iwqp-&gt;iwrcq);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_qp_rem_ref(&amp;iwqp-&gt;ibqp);</div><div class='add'>+	wait_for_completion(&amp;iwqp-&gt;free_qp);</div><div class='add'>+	irdma_free_lsmm_rsrc(iwqp);</div><div class='add'>+	irdma_cqp_qp_destroy_cmd(&amp;iwdev-&gt;rf-&gt;sc_dev, &amp;iwqp-&gt;sc_qp);</div><div class='add'>+</div><div class='add'>+	irdma_remove_push_mmap_entries(iwqp);</div><div class='add'>+</div><div class='add'>+	if (iwqp-&gt;sc_qp.qp_uk.qp_id == 1)</div><div class='add'>+		iwdev-&gt;rf-&gt;hwqp1_rsvd = false;</div><div class='add'>+	irdma_free_qp_rsrc(iwqp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_setup_virt_qp - setup for allocation of virtual qp</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ * @iwqp: qp ptr</div><div class='add'>+ * @init_info: initialize info to return</div><div class='add'>+ */</div><div class='add'>+static void irdma_setup_virt_qp(struct irdma_device *iwdev,</div><div class='add'>+			       struct irdma_qp *iwqp,</div><div class='add'>+			       struct irdma_qp_init_info *init_info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pbl *iwpbl = iwqp-&gt;iwpbl;</div><div class='add'>+	struct irdma_qp_mr *qpmr = &amp;iwpbl-&gt;qp_mr;</div><div class='add'>+</div><div class='add'>+	iwqp-&gt;page = qpmr-&gt;sq_page;</div><div class='add'>+	init_info-&gt;shadow_area_pa = qpmr-&gt;shadow;</div><div class='add'>+	if (iwpbl-&gt;pbl_allocated) {</div><div class='add'>+		init_info-&gt;virtual_map = true;</div><div class='add'>+		init_info-&gt;sq_pa = qpmr-&gt;sq_pbl.idx;</div><div class='add'>+		/* Need to use contiguous buffer for RQ of QP</div><div class='add'>+		 * in case it is associated with SRQ.</div><div class='add'>+		 */</div><div class='add'>+		init_info-&gt;rq_pa = init_info-&gt;qp_uk_init_info.srq_uk ?</div><div class='add'>+			qpmr-&gt;rq_pa : qpmr-&gt;rq_pbl.idx;</div><div class='add'>+	} else {</div><div class='add'>+		init_info-&gt;sq_pa = qpmr-&gt;sq_pbl.addr;</div><div class='add'>+		init_info-&gt;rq_pa = qpmr-&gt;rq_pbl.addr;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_setup_umode_qp - setup sq and rq size in user mode qp</div><div class='add'>+ * @udata: udata</div><div class='add'>+ * @iwdev: iwarp device</div><div class='add'>+ * @iwqp: qp ptr (user or kernel)</div><div class='add'>+ * @info: initialize info to return</div><div class='add'>+ * @init_attr: Initial QP create attributes</div><div class='add'>+ */</div><div class='add'>+static int irdma_setup_umode_qp(struct ib_udata *udata,</div><div class='add'>+				struct irdma_device *iwdev,</div><div class='add'>+				struct irdma_qp *iwqp,</div><div class='add'>+				struct irdma_qp_init_info *info,</div><div class='add'>+				struct ib_qp_init_attr *init_attr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ucontext *ucontext = rdma_udata_to_drv_context(udata,</div><div class='add'>+				struct irdma_ucontext, ibucontext);</div><div class='add'>+	struct irdma_qp_uk_init_info *ukinfo = &amp;info-&gt;qp_uk_init_info;</div><div class='add'>+	struct irdma_create_qp_req req;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = ib_copy_from_udata(&amp;req, udata,</div><div class='add'>+				 min(sizeof(req), udata-&gt;inlen));</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev, "VERBS: ib_copy_from_data fail\n");</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	iwqp-&gt;ctx_info.qp_compl_ctx = req.user_compl_ctx;</div><div class='add'>+	iwqp-&gt;user_mode = 1;</div><div class='add'>+	if (req.user_wqe_bufs) {</div><div class='add'>+		info-&gt;qp_uk_init_info.legacy_mode = ucontext-&gt;legacy_mode;</div><div class='add'>+		spin_lock_irqsave(&amp;ucontext-&gt;qp_reg_mem_list_lock, flags);</div><div class='add'>+		iwqp-&gt;iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs,</div><div class='add'>+					    &amp;ucontext-&gt;qp_reg_mem_list);</div><div class='add'>+		spin_unlock_irqrestore(&amp;ucontext-&gt;qp_reg_mem_list_lock, flags);</div><div class='add'>+</div><div class='add'>+		if (!iwqp-&gt;iwpbl) {</div><div class='add'>+			ret = -ENODATA;</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev, "VERBS: no pbl info\n");</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!ucontext-&gt;use_raw_attrs) {</div><div class='add'>+		/**</div><div class='add'>+		 * Maintain backward compat with older ABI which passes sq and</div><div class='add'>+		 * rq depth in quanta in cap.max_send_wr and cap.max_recv_wr.</div><div class='add'>+		 * There is no way to compute the correct value of</div><div class='add'>+		 * iwqp-&gt;max_send_wr/max_recv_wr in the kernel.</div><div class='add'>+		 */</div><div class='add'>+		iwqp-&gt;max_send_wr = init_attr-&gt;cap.max_send_wr;</div><div class='add'>+		iwqp-&gt;max_recv_wr = init_attr-&gt;cap.max_recv_wr;</div><div class='add'>+		ukinfo-&gt;sq_size = init_attr-&gt;cap.max_send_wr;</div><div class='add'>+		ukinfo-&gt;rq_size = init_attr-&gt;cap.max_recv_wr;</div><div class='add'>+		irdma_uk_calc_shift_wq(ukinfo, &amp;ukinfo-&gt;sq_shift,</div><div class='add'>+				       &amp;ukinfo-&gt;rq_shift);</div><div class='add'>+	} else {</div><div class='add'>+		ret = irdma_uk_calc_depth_shift_sq(ukinfo, &amp;ukinfo-&gt;sq_depth,</div><div class='add'>+						   &amp;ukinfo-&gt;sq_shift);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+</div><div class='add'>+		ret = irdma_uk_calc_depth_shift_rq(ukinfo, &amp;ukinfo-&gt;rq_depth,</div><div class='add'>+						   &amp;ukinfo-&gt;rq_shift);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+</div><div class='add'>+		iwqp-&gt;max_send_wr =</div><div class='add'>+			(ukinfo-&gt;sq_depth - IRDMA_SQ_RSVD) &gt;&gt; ukinfo-&gt;sq_shift;</div><div class='add'>+		iwqp-&gt;max_recv_wr =</div><div class='add'>+			(ukinfo-&gt;rq_depth - IRDMA_RQ_RSVD) &gt;&gt; ukinfo-&gt;rq_shift;</div><div class='add'>+		ukinfo-&gt;sq_size = ukinfo-&gt;sq_depth &gt;&gt; ukinfo-&gt;sq_shift;</div><div class='add'>+		ukinfo-&gt;rq_size = ukinfo-&gt;rq_depth &gt;&gt; ukinfo-&gt;rq_shift;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_setup_virt_qp(iwdev, iwqp, info);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_setup_kmode_qp - setup initialization for kernel mode qp</div><div class='add'>+ * @iwdev: iwarp device</div><div class='add'>+ * @iwqp: qp ptr (user or kernel)</div><div class='add'>+ * @info: initialize info to return</div><div class='add'>+ * @init_attr: Initial QP create attributes</div><div class='add'>+ */</div><div class='add'>+static int irdma_setup_kmode_qp(struct irdma_device *iwdev,</div><div class='add'>+				struct irdma_qp *iwqp,</div><div class='add'>+				struct irdma_qp_init_info *info,</div><div class='add'>+				struct ib_qp_init_attr *init_attr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_dma_mem *mem = &amp;iwqp-&gt;kqp.dma_mem;</div><div class='add'>+	u32 size;</div><div class='add'>+	int status;</div><div class='add'>+	struct irdma_qp_uk_init_info *ukinfo = &amp;info-&gt;qp_uk_init_info;</div><div class='add'>+</div><div class='add'>+	status = irdma_uk_calc_depth_shift_sq(ukinfo, &amp;ukinfo-&gt;sq_depth,</div><div class='add'>+					      &amp;ukinfo-&gt;sq_shift);</div><div class='add'>+	if (status)</div><div class='add'>+		return status;</div><div class='add'>+</div><div class='add'>+	status = irdma_uk_calc_depth_shift_rq(ukinfo, &amp;ukinfo-&gt;rq_depth,</div><div class='add'>+					      &amp;ukinfo-&gt;rq_shift);</div><div class='add'>+	if (status)</div><div class='add'>+		return status;</div><div class='add'>+</div><div class='add'>+	iwqp-&gt;kqp.sq_wrid_mem =</div><div class='add'>+		kcalloc(ukinfo-&gt;sq_depth, sizeof(*iwqp-&gt;kqp.sq_wrid_mem), GFP_KERNEL);</div><div class='add'>+	if (!iwqp-&gt;kqp.sq_wrid_mem)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	iwqp-&gt;kqp.rq_wrid_mem =</div><div class='add'>+		kcalloc(ukinfo-&gt;rq_depth, sizeof(*iwqp-&gt;kqp.rq_wrid_mem), GFP_KERNEL);</div><div class='add'>+</div><div class='add'>+	if (!iwqp-&gt;kqp.rq_wrid_mem) {</div><div class='add'>+		kfree(iwqp-&gt;kqp.sq_wrid_mem);</div><div class='add'>+		iwqp-&gt;kqp.sq_wrid_mem = NULL;</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ukinfo-&gt;sq_wrtrk_array = iwqp-&gt;kqp.sq_wrid_mem;</div><div class='add'>+	ukinfo-&gt;rq_wrid_array = iwqp-&gt;kqp.rq_wrid_mem;</div><div class='add'>+</div><div class='add'>+	size = (ukinfo-&gt;sq_depth + ukinfo-&gt;rq_depth) * IRDMA_QP_WQE_MIN_SIZE;</div><div class='add'>+	size += (IRDMA_SHADOW_AREA_SIZE &lt;&lt; 3);</div><div class='add'>+</div><div class='add'>+	mem-&gt;size = ALIGN(size, 256);</div><div class='add'>+	mem-&gt;va = dma_alloc_coherent(iwdev-&gt;rf-&gt;hw.device, mem-&gt;size,</div><div class='add'>+				     &amp;mem-&gt;pa, GFP_KERNEL);</div><div class='add'>+	if (!mem-&gt;va) {</div><div class='add'>+		kfree(iwqp-&gt;kqp.sq_wrid_mem);</div><div class='add'>+		iwqp-&gt;kqp.sq_wrid_mem = NULL;</div><div class='add'>+		kfree(iwqp-&gt;kqp.rq_wrid_mem);</div><div class='add'>+		iwqp-&gt;kqp.rq_wrid_mem = NULL;</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ukinfo-&gt;sq = mem-&gt;va;</div><div class='add'>+	info-&gt;sq_pa = mem-&gt;pa;</div><div class='add'>+	ukinfo-&gt;rq = &amp;ukinfo-&gt;sq[ukinfo-&gt;sq_depth];</div><div class='add'>+	info-&gt;rq_pa = info-&gt;sq_pa + (ukinfo-&gt;sq_depth * IRDMA_QP_WQE_MIN_SIZE);</div><div class='add'>+	ukinfo-&gt;shadow_area = ukinfo-&gt;rq[ukinfo-&gt;rq_depth].elem;</div><div class='add'>+	info-&gt;shadow_area_pa =</div><div class='add'>+		info-&gt;rq_pa + (ukinfo-&gt;rq_depth * IRDMA_QP_WQE_MIN_SIZE);</div><div class='add'>+	ukinfo-&gt;sq_size = ukinfo-&gt;sq_depth &gt;&gt; ukinfo-&gt;sq_shift;</div><div class='add'>+	ukinfo-&gt;rq_size = ukinfo-&gt;rq_depth &gt;&gt; ukinfo-&gt;rq_shift;</div><div class='add'>+	ukinfo-&gt;qp_id = info-&gt;qp_uk_init_info.qp_id;</div><div class='add'>+</div><div class='add'>+	iwqp-&gt;max_send_wr = (ukinfo-&gt;sq_depth - IRDMA_SQ_RSVD) &gt;&gt; ukinfo-&gt;sq_shift;</div><div class='add'>+	iwqp-&gt;max_recv_wr = (ukinfo-&gt;rq_depth - IRDMA_RQ_RSVD) &gt;&gt; ukinfo-&gt;rq_shift;</div><div class='add'>+	init_attr-&gt;cap.max_send_wr = iwqp-&gt;max_send_wr;</div><div class='add'>+	init_attr-&gt;cap.max_recv_wr = iwqp-&gt;max_recv_wr;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = iwqp-&gt;iwdev-&gt;rf;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	struct irdma_create_qp_info *qp_info;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	qp_info = &amp;cqp_request-&gt;info.in.u.qp_create.info;</div><div class='add'>+	qp_info-&gt;mac_valid = true;</div><div class='add'>+	qp_info-&gt;cq_num_valid = true;</div><div class='add'>+	qp_info-&gt;next_iwarp_state = IRDMA_QP_STATE_IDLE;</div><div class='add'>+</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_QP_CREATE;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.qp_create.qp = &amp;iwqp-&gt;sc_qp;</div><div class='add'>+	cqp_info-&gt;in.u.qp_create.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp,</div><div class='add'>+					       struct irdma_qp_host_ctx_info *ctx_info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = iwqp-&gt;iwdev;</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_roce_offload_info *roce_info;</div><div class='add'>+	struct irdma_udp_offload_info *udp_info;</div><div class='add'>+</div><div class='add'>+	udp_info = &amp;iwqp-&gt;udp_info;</div><div class='add'>+	udp_info-&gt;snd_mss = ib_mtu_enum_to_int(ib_mtu_int_to_enum(iwdev-&gt;vsi.mtu));</div><div class='add'>+	udp_info-&gt;cwnd = iwdev-&gt;roce_cwnd;</div><div class='add'>+	udp_info-&gt;rexmit_thresh = 2;</div><div class='add'>+	udp_info-&gt;rnr_nak_thresh = 2;</div><div class='add'>+	udp_info-&gt;src_port = 0xc000;</div><div class='add'>+	udp_info-&gt;dst_port = ROCE_V2_UDP_DPORT;</div><div class='add'>+	roce_info = &amp;iwqp-&gt;roce_info;</div><div class='add'>+	ether_addr_copy(roce_info-&gt;mac_addr, iwdev-&gt;netdev-&gt;dev_addr);</div><div class='add'>+</div><div class='add'>+	if (iwqp-&gt;ibqp.qp_type == IB_QPT_GSI &amp;&amp; iwqp-&gt;ibqp.qp_num != 1)</div><div class='add'>+		roce_info-&gt;is_qp1 = true;</div><div class='add'>+	roce_info-&gt;rd_en = true;</div><div class='add'>+	roce_info-&gt;wr_rdresp_en = true;</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3)</div><div class='add'>+		roce_info-&gt;bind_en = true;</div><div class='add'>+	roce_info-&gt;dcqcn_en = false;</div><div class='add'>+	roce_info-&gt;rtomin = 5;</div><div class='add'>+</div><div class='add'>+	roce_info-&gt;ack_credits = iwdev-&gt;roce_ackcreds;</div><div class='add'>+	roce_info-&gt;ird_size = dev-&gt;hw_attrs.max_hw_ird;</div><div class='add'>+	roce_info-&gt;ord_size = dev-&gt;hw_attrs.max_hw_ord;</div><div class='add'>+</div><div class='add'>+	if (!iwqp-&gt;user_mode) {</div><div class='add'>+		roce_info-&gt;priv_mode_en = true;</div><div class='add'>+		roce_info-&gt;fast_reg_en = true;</div><div class='add'>+		roce_info-&gt;udprivcq_en = true;</div><div class='add'>+	}</div><div class='add'>+	roce_info-&gt;roce_tver = 0;</div><div class='add'>+</div><div class='add'>+	ctx_info-&gt;roce_info = &amp;iwqp-&gt;roce_info;</div><div class='add'>+	ctx_info-&gt;udp_info = &amp;iwqp-&gt;udp_info;</div><div class='add'>+	irdma_sc_qp_setctx_roce(&amp;iwqp-&gt;sc_qp, iwqp-&gt;host_ctx.va, ctx_info);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp,</div><div class='add'>+					     struct irdma_qp_host_ctx_info *ctx_info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = iwqp-&gt;iwdev;</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_iwarp_offload_info *iwarp_info;</div><div class='add'>+</div><div class='add'>+	iwarp_info = &amp;iwqp-&gt;iwarp_info;</div><div class='add'>+	ether_addr_copy(iwarp_info-&gt;mac_addr, iwdev-&gt;netdev-&gt;dev_addr);</div><div class='add'>+	iwarp_info-&gt;rd_en = true;</div><div class='add'>+	iwarp_info-&gt;wr_rdresp_en = true;</div><div class='add'>+	iwarp_info-&gt;ecn_en = true;</div><div class='add'>+	iwarp_info-&gt;rtomin = 5;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2)</div><div class='add'>+		iwarp_info-&gt;ib_rd_en = true;</div><div class='add'>+	if (!iwqp-&gt;user_mode) {</div><div class='add'>+		iwarp_info-&gt;priv_mode_en = true;</div><div class='add'>+		iwarp_info-&gt;fast_reg_en = true;</div><div class='add'>+	}</div><div class='add'>+	iwarp_info-&gt;ddp_ver = 1;</div><div class='add'>+	iwarp_info-&gt;rdmap_ver = 1;</div><div class='add'>+</div><div class='add'>+	ctx_info-&gt;iwarp_info = &amp;iwqp-&gt;iwarp_info;</div><div class='add'>+	ctx_info-&gt;iwarp_info_valid = true;</div><div class='add'>+	irdma_sc_qp_setctx(&amp;iwqp-&gt;sc_qp, iwqp-&gt;host_ctx.va, ctx_info);</div><div class='add'>+	ctx_info-&gt;iwarp_info_valid = false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr,</div><div class='add'>+				   struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_uk_attrs *uk_attrs = &amp;dev-&gt;hw_attrs.uk_attrs;</div><div class='add'>+</div><div class='add'>+	if (init_attr-&gt;create_flags)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (init_attr-&gt;cap.max_inline_data &gt; uk_attrs-&gt;max_hw_inline ||</div><div class='add'>+	    init_attr-&gt;cap.max_send_sge &gt; uk_attrs-&gt;max_hw_wq_frags ||</div><div class='add'>+	    init_attr-&gt;cap.max_recv_sge &gt; uk_attrs-&gt;max_hw_wq_frags ||</div><div class='add'>+	    init_attr-&gt;cap.max_send_wr &gt; uk_attrs-&gt;max_hw_wq_quanta ||</div><div class='add'>+	    init_attr-&gt;cap.max_recv_wr &gt; uk_attrs-&gt;max_hw_rq_quanta)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (rdma_protocol_roce(&amp;iwdev-&gt;ibdev, 1)) {</div><div class='add'>+		if (init_attr-&gt;qp_type != IB_QPT_RC &amp;&amp;</div><div class='add'>+		    init_attr-&gt;qp_type != IB_QPT_UD &amp;&amp;</div><div class='add'>+		    init_attr-&gt;qp_type != IB_QPT_GSI)</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+	} else {</div><div class='add'>+		if (init_attr-&gt;qp_type != IB_QPT_RC)</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_flush_worker(struct work_struct *work)</div><div class='add'>+{</div><div class='add'>+	struct delayed_work *dwork = to_delayed_work(work);</div><div class='add'>+	struct irdma_qp *iwqp = container_of(dwork, struct irdma_qp, dwork_flush);</div><div class='add'>+</div><div class='add'>+	irdma_generate_flush_completions(iwqp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_setup_gsi_qp_rsrc(struct irdma_qp *iwqp, u32 *qp_num)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = iwqp-&gt;iwdev;</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (rf-&gt;rdma_ver &lt;= IRDMA_GEN_2) {</div><div class='add'>+		*qp_num = 1;</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;rf-&gt;rsrc_lock, flags);</div><div class='add'>+	if (!rf-&gt;hwqp1_rsvd) {</div><div class='add'>+		*qp_num = 1;</div><div class='add'>+		rf-&gt;hwqp1_rsvd = true;</div><div class='add'>+		spin_unlock_irqrestore(&amp;rf-&gt;rsrc_lock, flags);</div><div class='add'>+	} else {</div><div class='add'>+		spin_unlock_irqrestore(&amp;rf-&gt;rsrc_lock, flags);</div><div class='add'>+		ret = irdma_alloc_rsrc(rf, rf-&gt;allocated_qps, rf-&gt;max_qp,</div><div class='add'>+				       qp_num, &amp;rf-&gt;next_qp);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = irdma_vchnl_req_add_vport(&amp;rf-&gt;sc_dev, iwdev-&gt;vport_id, *qp_num,</div><div class='add'>+					(&amp;iwdev-&gt;vsi)-&gt;qos);</div><div class='add'>+	if (ret) {</div><div class='add'>+		if (*qp_num != 1) {</div><div class='add'>+			irdma_free_rsrc(rf, rf-&gt;allocated_qps, *qp_num);</div><div class='add'>+		} else {</div><div class='add'>+			spin_lock_irqsave(&amp;rf-&gt;rsrc_lock, flags);</div><div class='add'>+			rf-&gt;hwqp1_rsvd = false;</div><div class='add'>+			spin_unlock_irqrestore(&amp;rf-&gt;rsrc_lock, flags);</div><div class='add'>+		}</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_create_qp - create qp</div><div class='add'>+ * @ibqp: ptr of qp</div><div class='add'>+ * @init_attr: attributes for qp</div><div class='add'>+ * @udata: user data for create qp</div><div class='add'>+ */</div><div class='add'>+static int irdma_create_qp(struct ib_qp *ibqp,</div><div class='add'>+			   struct ib_qp_init_attr *init_attr,</div><div class='add'>+			   struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+#define IRDMA_CREATE_QP_MIN_REQ_LEN offsetofend(struct irdma_create_qp_req, user_compl_ctx)</div><div class='add'>+#define IRDMA_CREATE_QP_MIN_RESP_LEN offsetofend(struct irdma_create_qp_resp, rsvd)</div><div class='add'>+	struct ib_pd *ibpd = ibqp-&gt;pd;</div><div class='add'>+	struct irdma_pd *iwpd = to_iwpd(ibpd);</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibpd-&gt;device);</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+	struct irdma_qp *iwqp = to_iwqp(ibqp);</div><div class='add'>+	struct irdma_create_qp_resp uresp = {};</div><div class='add'>+	u32 qp_num = 0;</div><div class='add'>+	int err_code;</div><div class='add'>+	struct irdma_sc_qp *qp;</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_uk_attrs *uk_attrs = &amp;dev-&gt;hw_attrs.uk_attrs;</div><div class='add'>+	struct irdma_qp_init_info init_info = {};</div><div class='add'>+	struct irdma_qp_host_ctx_info *ctx_info;</div><div class='add'>+	struct irdma_srq *iwsrq;</div><div class='add'>+	bool srq_valid = false;</div><div class='add'>+	u32 srq_id = 0;</div><div class='add'>+</div><div class='add'>+	if (init_attr-&gt;srq) {</div><div class='add'>+		iwsrq = to_iwsrq(init_attr-&gt;srq);</div><div class='add'>+		srq_valid = true;</div><div class='add'>+		srq_id = iwsrq-&gt;srq_num;</div><div class='add'>+		init_attr-&gt;cap.max_recv_sge = uk_attrs-&gt;max_hw_wq_frags;</div><div class='add'>+		init_attr-&gt;cap.max_recv_wr = 4;</div><div class='add'>+		init_info.qp_uk_init_info.srq_uk = &amp;iwsrq-&gt;sc_srq.srq_uk;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err_code = irdma_validate_qp_attrs(init_attr, iwdev);</div><div class='add'>+	if (err_code)</div><div class='add'>+		return err_code;</div><div class='add'>+</div><div class='add'>+	if (udata &amp;&amp; (udata-&gt;inlen &lt; IRDMA_CREATE_QP_MIN_REQ_LEN ||</div><div class='add'>+		      udata-&gt;outlen &lt; IRDMA_CREATE_QP_MIN_RESP_LEN))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	init_info.vsi = &amp;iwdev-&gt;vsi;</div><div class='add'>+	init_info.qp_uk_init_info.uk_attrs = uk_attrs;</div><div class='add'>+	init_info.qp_uk_init_info.sq_size = init_attr-&gt;cap.max_send_wr;</div><div class='add'>+	init_info.qp_uk_init_info.rq_size = init_attr-&gt;cap.max_recv_wr;</div><div class='add'>+	init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr-&gt;cap.max_send_sge;</div><div class='add'>+	init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr-&gt;cap.max_recv_sge;</div><div class='add'>+	init_info.qp_uk_init_info.max_inline_data = init_attr-&gt;cap.max_inline_data;</div><div class='add'>+</div><div class='add'>+	qp = &amp;iwqp-&gt;sc_qp;</div><div class='add'>+	qp-&gt;qp_uk.back_qp = iwqp;</div><div class='add'>+	qp-&gt;push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;</div><div class='add'>+</div><div class='add'>+	iwqp-&gt;iwdev = iwdev;</div><div class='add'>+	iwqp-&gt;q2_ctx_mem.size = ALIGN(IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE,</div><div class='add'>+				      256);</div><div class='add'>+	iwqp-&gt;q2_ctx_mem.va = dma_alloc_coherent(dev-&gt;hw-&gt;device,</div><div class='add'>+						 iwqp-&gt;q2_ctx_mem.size,</div><div class='add'>+						 &amp;iwqp-&gt;q2_ctx_mem.pa,</div><div class='add'>+						 GFP_KERNEL);</div><div class='add'>+	if (!iwqp-&gt;q2_ctx_mem.va)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	init_info.q2 = iwqp-&gt;q2_ctx_mem.va;</div><div class='add'>+	init_info.q2_pa = iwqp-&gt;q2_ctx_mem.pa;</div><div class='add'>+	init_info.host_ctx = (__le64 *)(init_info.q2 + IRDMA_Q2_BUF_SIZE);</div><div class='add'>+	init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE;</div><div class='add'>+</div><div class='add'>+	if (init_attr-&gt;qp_type == IB_QPT_GSI) {</div><div class='add'>+		err_code = irdma_setup_gsi_qp_rsrc(iwqp, &amp;qp_num);</div><div class='add'>+		if (err_code)</div><div class='add'>+			goto error;</div><div class='add'>+		iwqp-&gt;ibqp.qp_num = 1;</div><div class='add'>+	} else {</div><div class='add'>+		err_code = irdma_alloc_rsrc(rf, rf-&gt;allocated_qps, rf-&gt;max_qp,</div><div class='add'>+					    &amp;qp_num, &amp;rf-&gt;next_qp);</div><div class='add'>+		if (err_code)</div><div class='add'>+			goto error;</div><div class='add'>+		iwqp-&gt;ibqp.qp_num = qp_num;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	iwqp-&gt;iwpd = iwpd;</div><div class='add'>+	qp = &amp;iwqp-&gt;sc_qp;</div><div class='add'>+	iwqp-&gt;iwscq = to_iwcq(init_attr-&gt;send_cq);</div><div class='add'>+	iwqp-&gt;iwrcq = to_iwcq(init_attr-&gt;recv_cq);</div><div class='add'>+	iwqp-&gt;host_ctx.va = init_info.host_ctx;</div><div class='add'>+	iwqp-&gt;host_ctx.pa = init_info.host_ctx_pa;</div><div class='add'>+	iwqp-&gt;host_ctx.size = IRDMA_QP_CTX_SIZE;</div><div class='add'>+</div><div class='add'>+	init_info.pd = &amp;iwpd-&gt;sc_pd;</div><div class='add'>+	init_info.qp_uk_init_info.qp_id = qp_num;</div><div class='add'>+	if (!rdma_protocol_roce(&amp;iwdev-&gt;ibdev, 1))</div><div class='add'>+		init_info.qp_uk_init_info.first_sq_wq = 1;</div><div class='add'>+	iwqp-&gt;ctx_info.qp_compl_ctx = (uintptr_t)qp;</div><div class='add'>+	init_waitqueue_head(&amp;iwqp-&gt;waitq);</div><div class='add'>+	init_waitqueue_head(&amp;iwqp-&gt;mod_qp_waitq);</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		init_info.qp_uk_init_info.abi_ver = iwpd-&gt;sc_pd.abi_ver;</div><div class='add'>+		err_code = irdma_setup_umode_qp(udata, iwdev, iwqp, &amp;init_info,</div><div class='add'>+						init_attr);</div><div class='add'>+	} else {</div><div class='add'>+		INIT_DELAYED_WORK(&amp;iwqp-&gt;dwork_flush, irdma_flush_worker);</div><div class='add'>+		init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER;</div><div class='add'>+		err_code = irdma_setup_kmode_qp(iwdev, iwqp, &amp;init_info, init_attr);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (err_code) {</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev, "VERBS: setup qp failed\n");</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (rdma_protocol_roce(&amp;iwdev-&gt;ibdev, 1)) {</div><div class='add'>+		if (init_attr-&gt;qp_type == IB_QPT_RC) {</div><div class='add'>+			init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC;</div><div class='add'>+			init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM |</div><div class='add'>+							    IRDMA_WRITE_WITH_IMM |</div><div class='add'>+							    IRDMA_ROCE;</div><div class='add'>+		} else {</div><div class='add'>+			init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD;</div><div class='add'>+			init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM |</div><div class='add'>+							    IRDMA_ROCE;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP;</div><div class='add'>+		init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt; IRDMA_GEN_1)</div><div class='add'>+		init_info.qp_uk_init_info.qp_caps |= IRDMA_PUSH_MODE;</div><div class='add'>+</div><div class='add'>+	err_code = irdma_sc_qp_init(qp, &amp;init_info);</div><div class='add'>+	if (err_code) {</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev, "VERBS: qp_init fail\n");</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ctx_info = &amp;iwqp-&gt;ctx_info;</div><div class='add'>+	ctx_info-&gt;srq_valid = srq_valid;</div><div class='add'>+	ctx_info-&gt;srq_id = srq_id;</div><div class='add'>+	ctx_info-&gt;send_cq_num = iwqp-&gt;iwscq-&gt;sc_cq.cq_uk.cq_id;</div><div class='add'>+	ctx_info-&gt;rcv_cq_num = iwqp-&gt;iwrcq-&gt;sc_cq.cq_uk.cq_id;</div><div class='add'>+</div><div class='add'>+	if (rdma_protocol_roce(&amp;iwdev-&gt;ibdev, 1)) {</div><div class='add'>+		if (dev-&gt;ws_add(&amp;iwdev-&gt;vsi, 0)) {</div><div class='add'>+			irdma_cqp_qp_destroy_cmd(&amp;rf-&gt;sc_dev, &amp;iwqp-&gt;sc_qp);</div><div class='add'>+			err_code = -EINVAL;</div><div class='add'>+			goto error;</div><div class='add'>+		}</div><div class='add'>+		irdma_qp_add_qos(&amp;iwqp-&gt;sc_qp);</div><div class='add'>+		irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info);</div><div class='add'>+	} else {</div><div class='add'>+		irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err_code = irdma_cqp_create_qp_cmd(iwqp);</div><div class='add'>+	if (err_code)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	refcount_set(&amp;iwqp-&gt;refcnt, 1);</div><div class='add'>+	spin_lock_init(&amp;iwqp-&gt;lock);</div><div class='add'>+	spin_lock_init(&amp;iwqp-&gt;sc_qp.pfpdu.lock);</div><div class='add'>+	iwqp-&gt;sig_all = init_attr-&gt;sq_sig_type == IB_SIGNAL_ALL_WR;</div><div class='add'>+	rf-&gt;qp_table[qp_num] = iwqp;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		/* GEN_1 legacy support with libi40iw does not have expanded uresp struct */</div><div class='add'>+		if (udata-&gt;outlen &lt; sizeof(uresp)) {</div><div class='add'>+			uresp.lsmm = 1;</div><div class='add'>+			uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1;</div><div class='add'>+		} else {</div><div class='add'>+			if (rdma_protocol_iwarp(&amp;iwdev-&gt;ibdev, 1))</div><div class='add'>+				uresp.lsmm = 1;</div><div class='add'>+		}</div><div class='add'>+		uresp.actual_sq_size = init_info.qp_uk_init_info.sq_size;</div><div class='add'>+		uresp.actual_rq_size = init_info.qp_uk_init_info.rq_size;</div><div class='add'>+		uresp.qp_id = qp_num;</div><div class='add'>+		uresp.qp_caps = qp-&gt;qp_uk.qp_caps;</div><div class='add'>+</div><div class='add'>+		err_code = ib_copy_to_udata(udata, &amp;uresp,</div><div class='add'>+					    min(sizeof(uresp), udata-&gt;outlen));</div><div class='add'>+		if (err_code) {</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev, "VERBS: copy_to_udata failed\n");</div><div class='add'>+			irdma_destroy_qp(&amp;iwqp-&gt;ibqp, udata);</div><div class='add'>+			return err_code;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	init_completion(&amp;iwqp-&gt;free_qp);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+error:</div><div class='add'>+	irdma_free_qp_rsrc(iwqp);</div><div class='add'>+	return err_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_get_ib_acc_flags(struct irdma_qp *iwqp)</div><div class='add'>+{</div><div class='add'>+	int acc_flags = 0;</div><div class='add'>+</div><div class='add'>+	if (rdma_protocol_roce(iwqp-&gt;ibqp.device, 1)) {</div><div class='add'>+		if (iwqp-&gt;roce_info.wr_rdresp_en) {</div><div class='add'>+			acc_flags |= IB_ACCESS_LOCAL_WRITE;</div><div class='add'>+			acc_flags |= IB_ACCESS_REMOTE_WRITE;</div><div class='add'>+		}</div><div class='add'>+		if (iwqp-&gt;roce_info.rd_en)</div><div class='add'>+			acc_flags |= IB_ACCESS_REMOTE_READ;</div><div class='add'>+		if (iwqp-&gt;roce_info.bind_en)</div><div class='add'>+			acc_flags |= IB_ACCESS_MW_BIND;</div><div class='add'>+		if (iwqp-&gt;ctx_info.remote_atomics_en)</div><div class='add'>+			acc_flags |= IB_ACCESS_REMOTE_ATOMIC;</div><div class='add'>+	} else {</div><div class='add'>+		if (iwqp-&gt;iwarp_info.wr_rdresp_en) {</div><div class='add'>+			acc_flags |= IB_ACCESS_LOCAL_WRITE;</div><div class='add'>+			acc_flags |= IB_ACCESS_REMOTE_WRITE;</div><div class='add'>+		}</div><div class='add'>+		if (iwqp-&gt;iwarp_info.rd_en)</div><div class='add'>+			acc_flags |= IB_ACCESS_REMOTE_READ;</div><div class='add'>+		if (iwqp-&gt;ctx_info.remote_atomics_en)</div><div class='add'>+			acc_flags |= IB_ACCESS_REMOTE_ATOMIC;</div><div class='add'>+	}</div><div class='add'>+	return acc_flags;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_query_qp - query qp attributes</div><div class='add'>+ * @ibqp: qp pointer</div><div class='add'>+ * @attr: attributes pointer</div><div class='add'>+ * @attr_mask: Not used</div><div class='add'>+ * @init_attr: qp attributes to return</div><div class='add'>+ */</div><div class='add'>+static int irdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='add'>+			  int attr_mask, struct ib_qp_init_attr *init_attr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *iwqp = to_iwqp(ibqp);</div><div class='add'>+	struct irdma_sc_qp *qp = &amp;iwqp-&gt;sc_qp;</div><div class='add'>+</div><div class='add'>+	memset(attr, 0, sizeof(*attr));</div><div class='add'>+	memset(init_attr, 0, sizeof(*init_attr));</div><div class='add'>+</div><div class='add'>+	attr-&gt;qp_state = iwqp-&gt;ibqp_state;</div><div class='add'>+	attr-&gt;cur_qp_state = iwqp-&gt;ibqp_state;</div><div class='add'>+	attr-&gt;cap.max_send_wr = iwqp-&gt;max_send_wr;</div><div class='add'>+	attr-&gt;cap.max_recv_wr = iwqp-&gt;max_recv_wr;</div><div class='add'>+	attr-&gt;cap.max_inline_data = qp-&gt;qp_uk.max_inline_data;</div><div class='add'>+	attr-&gt;cap.max_send_sge = qp-&gt;qp_uk.max_sq_frag_cnt;</div><div class='add'>+	attr-&gt;cap.max_recv_sge = qp-&gt;qp_uk.max_rq_frag_cnt;</div><div class='add'>+	attr-&gt;qp_access_flags = irdma_get_ib_acc_flags(iwqp);</div><div class='add'>+	attr-&gt;port_num = 1;</div><div class='add'>+	if (rdma_protocol_roce(ibqp-&gt;device, 1)) {</div><div class='add'>+		attr-&gt;path_mtu = ib_mtu_int_to_enum(iwqp-&gt;udp_info.snd_mss);</div><div class='add'>+		attr-&gt;qkey = iwqp-&gt;roce_info.qkey;</div><div class='add'>+		attr-&gt;rq_psn = iwqp-&gt;udp_info.epsn;</div><div class='add'>+		attr-&gt;sq_psn = iwqp-&gt;udp_info.psn_nxt;</div><div class='add'>+		attr-&gt;dest_qp_num = iwqp-&gt;roce_info.dest_qp;</div><div class='add'>+		attr-&gt;pkey_index = iwqp-&gt;roce_info.p_key;</div><div class='add'>+		attr-&gt;retry_cnt = iwqp-&gt;udp_info.rexmit_thresh;</div><div class='add'>+		attr-&gt;rnr_retry = iwqp-&gt;udp_info.rnr_nak_thresh;</div><div class='add'>+		attr-&gt;min_rnr_timer = iwqp-&gt;udp_info.min_rnr_timer;</div><div class='add'>+		attr-&gt;max_rd_atomic = iwqp-&gt;roce_info.ord_size;</div><div class='add'>+		attr-&gt;max_dest_rd_atomic = iwqp-&gt;roce_info.ird_size;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	init_attr-&gt;event_handler = iwqp-&gt;ibqp.event_handler;</div><div class='add'>+	init_attr-&gt;qp_context = iwqp-&gt;ibqp.qp_context;</div><div class='add'>+	init_attr-&gt;send_cq = iwqp-&gt;ibqp.send_cq;</div><div class='add'>+	init_attr-&gt;recv_cq = iwqp-&gt;ibqp.recv_cq;</div><div class='add'>+	init_attr-&gt;srq = iwqp-&gt;ibqp.srq;</div><div class='add'>+	init_attr-&gt;cap = attr-&gt;cap;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_query_pkey - Query partition key</div><div class='add'>+ * @ibdev: device pointer from stack</div><div class='add'>+ * @port: port number</div><div class='add'>+ * @index: index of pkey</div><div class='add'>+ * @pkey: pointer to store the pkey</div><div class='add'>+ */</div><div class='add'>+static int irdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index,</div><div class='add'>+			    u16 *pkey)</div><div class='add'>+{</div><div class='add'>+	if (index &gt;= IRDMA_PKEY_TBL_SZ)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	*pkey = IRDMA_DEFAULT_PKEY;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u8 irdma_roce_get_vlan_prio(const struct ib_gid_attr *attr, u8 prio)</div><div class='add'>+{</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	ndev = rcu_dereference(attr-&gt;ndev);</div><div class='add'>+	if (!ndev)</div><div class='add'>+		goto exit;</div><div class='add'>+	if (is_vlan_dev(ndev)) {</div><div class='add'>+		u16 vlan_qos = vlan_dev_get_egress_qos_mask(ndev, prio);</div><div class='add'>+</div><div class='add'>+		prio = (vlan_qos &amp; VLAN_PRIO_MASK) &gt;&gt; VLAN_PRIO_SHIFT;</div><div class='add'>+	}</div><div class='add'>+exit:</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+	return prio;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_wait_for_suspend(struct irdma_qp *iwqp)</div><div class='add'>+{</div><div class='add'>+	if (!wait_event_timeout(iwqp-&gt;iwdev-&gt;suspend_wq,</div><div class='add'>+				!iwqp-&gt;suspend_pending,</div><div class='add'>+				msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS))) {</div><div class='add'>+		iwqp-&gt;suspend_pending = false;</div><div class='add'>+		ibdev_warn(&amp;iwqp-&gt;iwdev-&gt;ibdev,</div><div class='add'>+			   "modify_qp timed out waiting for suspend. qp_id = %d, last_ae = 0x%x\n",</div><div class='add'>+			   iwqp-&gt;ibqp.qp_num, iwqp-&gt;last_aeq);</div><div class='add'>+		return -EBUSY;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_modify_qp_roce - modify qp request</div><div class='add'>+ * @ibqp: qp's pointer for modify</div><div class='add'>+ * @attr: access attributes</div><div class='add'>+ * @attr_mask: state mask</div><div class='add'>+ * @udata: user data</div><div class='add'>+ */</div><div class='add'>+int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='add'>+			 int attr_mask, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+#define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush)</div><div class='add'>+#define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid)</div><div class='add'>+	struct irdma_pd *iwpd = to_iwpd(ibqp-&gt;pd);</div><div class='add'>+	struct irdma_qp *iwqp = to_iwqp(ibqp);</div><div class='add'>+	struct irdma_device *iwdev = iwqp-&gt;iwdev;</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_qp_host_ctx_info *ctx_info;</div><div class='add'>+	struct irdma_roce_offload_info *roce_info;</div><div class='add'>+	struct irdma_udp_offload_info *udp_info;</div><div class='add'>+	struct irdma_modify_qp_info info = {};</div><div class='add'>+	struct irdma_modify_qp_resp uresp = {};</div><div class='add'>+	struct irdma_modify_qp_req ureq = {};</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	u8 issue_modify_qp = 0;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	ctx_info = &amp;iwqp-&gt;ctx_info;</div><div class='add'>+	roce_info = &amp;iwqp-&gt;roce_info;</div><div class='add'>+	udp_info = &amp;iwqp-&gt;udp_info;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		/* udata inlen/outlen can be 0 when supporting legacy libi40iw */</div><div class='add'>+		if ((udata-&gt;inlen &amp;&amp; udata-&gt;inlen &lt; IRDMA_MODIFY_QP_MIN_REQ_LEN) ||</div><div class='add'>+		    (udata-&gt;outlen &amp;&amp; udata-&gt;outlen &lt; IRDMA_MODIFY_QP_MIN_RESP_LEN))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; ~IB_QP_ATTR_STANDARD_BITS)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_DEST_QPN)</div><div class='add'>+		roce_info-&gt;dest_qp = attr-&gt;dest_qp_num;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_PKEY_INDEX) {</div><div class='add'>+		ret = irdma_query_pkey(ibqp-&gt;device, 0, attr-&gt;pkey_index,</div><div class='add'>+				       &amp;roce_info-&gt;p_key);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_QKEY)</div><div class='add'>+		roce_info-&gt;qkey = attr-&gt;qkey;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_PATH_MTU)</div><div class='add'>+		udp_info-&gt;snd_mss = ib_mtu_enum_to_int(attr-&gt;path_mtu);</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_SQ_PSN) {</div><div class='add'>+		udp_info-&gt;psn_nxt = attr-&gt;sq_psn;</div><div class='add'>+		udp_info-&gt;lsn =  0xffff;</div><div class='add'>+		udp_info-&gt;psn_una = attr-&gt;sq_psn;</div><div class='add'>+		udp_info-&gt;psn_max = attr-&gt;sq_psn;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_RQ_PSN)</div><div class='add'>+		udp_info-&gt;epsn = attr-&gt;rq_psn;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_RNR_RETRY)</div><div class='add'>+		udp_info-&gt;rnr_nak_thresh = attr-&gt;rnr_retry;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_MIN_RNR_TIMER &amp;&amp;</div><div class='add'>+	    dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3)</div><div class='add'>+		udp_info-&gt;min_rnr_timer = attr-&gt;min_rnr_timer;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_RETRY_CNT)</div><div class='add'>+		udp_info-&gt;rexmit_thresh = attr-&gt;retry_cnt;</div><div class='add'>+</div><div class='add'>+	ctx_info-&gt;roce_info-&gt;pd_id = iwpd-&gt;sc_pd.pd_id;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_AV) {</div><div class='add'>+		struct irdma_av *av = &amp;iwqp-&gt;roce_ah.av;</div><div class='add'>+		const struct ib_gid_attr *sgid_attr =</div><div class='add'>+				attr-&gt;ah_attr.grh.sgid_attr;</div><div class='add'>+		u16 vlan_id = VLAN_N_VID;</div><div class='add'>+		u32 local_ip[4];</div><div class='add'>+</div><div class='add'>+		memset(&amp;iwqp-&gt;roce_ah, 0, sizeof(iwqp-&gt;roce_ah));</div><div class='add'>+		if (attr-&gt;ah_attr.ah_flags &amp; IB_AH_GRH) {</div><div class='add'>+			udp_info-&gt;ttl = attr-&gt;ah_attr.grh.hop_limit;</div><div class='add'>+			udp_info-&gt;flow_label = attr-&gt;ah_attr.grh.flow_label;</div><div class='add'>+			udp_info-&gt;tos = attr-&gt;ah_attr.grh.traffic_class;</div><div class='add'>+			udp_info-&gt;src_port =</div><div class='add'>+				rdma_get_udp_sport(udp_info-&gt;flow_label,</div><div class='add'>+						   ibqp-&gt;qp_num,</div><div class='add'>+						   roce_info-&gt;dest_qp);</div><div class='add'>+			irdma_qp_rem_qos(&amp;iwqp-&gt;sc_qp);</div><div class='add'>+			dev-&gt;ws_remove(iwqp-&gt;sc_qp.vsi, ctx_info-&gt;user_pri);</div><div class='add'>+			if (iwqp-&gt;sc_qp.vsi-&gt;dscp_mode)</div><div class='add'>+				ctx_info-&gt;user_pri =</div><div class='add'>+				iwqp-&gt;sc_qp.vsi-&gt;dscp_map[irdma_tos2dscp(udp_info-&gt;tos)];</div><div class='add'>+			else</div><div class='add'>+				ctx_info-&gt;user_pri = rt_tos2priority(udp_info-&gt;tos);</div><div class='add'>+		}</div><div class='add'>+		ret = rdma_read_gid_l2_fields(sgid_attr, &amp;vlan_id,</div><div class='add'>+					      ctx_info-&gt;roce_info-&gt;mac_addr);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+		ctx_info-&gt;user_pri = irdma_roce_get_vlan_prio(sgid_attr,</div><div class='add'>+							      ctx_info-&gt;user_pri);</div><div class='add'>+		if (dev-&gt;ws_add(iwqp-&gt;sc_qp.vsi, ctx_info-&gt;user_pri))</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		iwqp-&gt;sc_qp.user_pri = ctx_info-&gt;user_pri;</div><div class='add'>+		irdma_qp_add_qos(&amp;iwqp-&gt;sc_qp);</div><div class='add'>+</div><div class='add'>+		if (vlan_id &gt;= VLAN_N_VID &amp;&amp; iwdev-&gt;dcb_vlan_mode)</div><div class='add'>+			vlan_id = 0;</div><div class='add'>+		if (vlan_id &lt; VLAN_N_VID) {</div><div class='add'>+			udp_info-&gt;insert_vlan_tag = true;</div><div class='add'>+			udp_info-&gt;vlan_tag = vlan_id |</div><div class='add'>+				ctx_info-&gt;user_pri &lt;&lt; VLAN_PRIO_SHIFT;</div><div class='add'>+		} else {</div><div class='add'>+			udp_info-&gt;insert_vlan_tag = false;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		av-&gt;attrs = attr-&gt;ah_attr;</div><div class='add'>+		rdma_gid2ip((struct sockaddr *)&amp;av-&gt;sgid_addr, &amp;sgid_attr-&gt;gid);</div><div class='add'>+		rdma_gid2ip((struct sockaddr *)&amp;av-&gt;dgid_addr, &amp;attr-&gt;ah_attr.grh.dgid);</div><div class='add'>+		av-&gt;net_type = rdma_gid_attr_network_type(sgid_attr);</div><div class='add'>+		if (av-&gt;net_type == RDMA_NETWORK_IPV6) {</div><div class='add'>+			__be32 *daddr =</div><div class='add'>+				av-&gt;dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32;</div><div class='add'>+			__be32 *saddr =</div><div class='add'>+				av-&gt;sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32;</div><div class='add'>+</div><div class='add'>+			irdma_copy_ip_ntohl(&amp;udp_info-&gt;dest_ip_addr[0], daddr);</div><div class='add'>+			irdma_copy_ip_ntohl(&amp;udp_info-&gt;local_ipaddr[0], saddr);</div><div class='add'>+</div><div class='add'>+			udp_info-&gt;ipv4 = false;</div><div class='add'>+			irdma_copy_ip_ntohl(local_ip, daddr);</div><div class='add'>+</div><div class='add'>+		} else if (av-&gt;net_type == RDMA_NETWORK_IPV4) {</div><div class='add'>+			__be32 saddr = av-&gt;sgid_addr.saddr_in.sin_addr.s_addr;</div><div class='add'>+			__be32 daddr = av-&gt;dgid_addr.saddr_in.sin_addr.s_addr;</div><div class='add'>+</div><div class='add'>+			local_ip[0] = ntohl(daddr);</div><div class='add'>+</div><div class='add'>+			udp_info-&gt;ipv4 = true;</div><div class='add'>+			udp_info-&gt;dest_ip_addr[0] = 0;</div><div class='add'>+			udp_info-&gt;dest_ip_addr[1] = 0;</div><div class='add'>+			udp_info-&gt;dest_ip_addr[2] = 0;</div><div class='add'>+			udp_info-&gt;dest_ip_addr[3] = local_ip[0];</div><div class='add'>+</div><div class='add'>+			udp_info-&gt;local_ipaddr[0] = 0;</div><div class='add'>+			udp_info-&gt;local_ipaddr[1] = 0;</div><div class='add'>+			udp_info-&gt;local_ipaddr[2] = 0;</div><div class='add'>+			udp_info-&gt;local_ipaddr[3] = ntohl(saddr);</div><div class='add'>+		}</div><div class='add'>+		udp_info-&gt;arp_idx =</div><div class='add'>+			irdma_add_arp(iwdev-&gt;rf, local_ip, udp_info-&gt;ipv4,</div><div class='add'>+				      attr-&gt;ah_attr.roce.dmac);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_MAX_QP_RD_ATOMIC) {</div><div class='add'>+		if (attr-&gt;max_rd_atomic &gt; dev-&gt;hw_attrs.max_hw_ord) {</div><div class='add'>+			ibdev_err(&amp;iwdev-&gt;ibdev,</div><div class='add'>+				  "rd_atomic = %d, above max_hw_ord=%d\n",</div><div class='add'>+				  attr-&gt;max_rd_atomic,</div><div class='add'>+				  dev-&gt;hw_attrs.max_hw_ord);</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		}</div><div class='add'>+		if (attr-&gt;max_rd_atomic)</div><div class='add'>+			roce_info-&gt;ord_size = attr-&gt;max_rd_atomic;</div><div class='add'>+		info.ord_valid = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_MAX_DEST_RD_ATOMIC) {</div><div class='add'>+		if (attr-&gt;max_dest_rd_atomic &gt; dev-&gt;hw_attrs.max_hw_ird) {</div><div class='add'>+			ibdev_err(&amp;iwdev-&gt;ibdev,</div><div class='add'>+				  "rd_atomic = %d, above max_hw_ird=%d\n",</div><div class='add'>+				   attr-&gt;max_dest_rd_atomic,</div><div class='add'>+				   dev-&gt;hw_attrs.max_hw_ird);</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		}</div><div class='add'>+		if (attr-&gt;max_dest_rd_atomic)</div><div class='add'>+			roce_info-&gt;ird_size = attr-&gt;max_dest_rd_atomic;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_ACCESS_FLAGS) {</div><div class='add'>+		if (attr-&gt;qp_access_flags &amp; IB_ACCESS_LOCAL_WRITE)</div><div class='add'>+			roce_info-&gt;wr_rdresp_en = true;</div><div class='add'>+		if (attr-&gt;qp_access_flags &amp; IB_ACCESS_REMOTE_WRITE)</div><div class='add'>+			roce_info-&gt;wr_rdresp_en = true;</div><div class='add'>+		if (attr-&gt;qp_access_flags &amp; IB_ACCESS_REMOTE_READ)</div><div class='add'>+			roce_info-&gt;rd_en = true;</div><div class='add'>+		if (dev-&gt;hw_attrs.uk_attrs.feature_flags &amp; IRDMA_FEATURE_ATOMIC_OPS)</div><div class='add'>+			if (attr-&gt;qp_access_flags &amp; IB_ACCESS_REMOTE_ATOMIC)</div><div class='add'>+				ctx_info-&gt;remote_atomics_en = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	wait_event(iwqp-&gt;mod_qp_waitq, !atomic_read(&amp;iwqp-&gt;hw_mod_qp_pend));</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+		  "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d attr_mask=0x%x\n",</div><div class='add'>+		  __builtin_return_address(0), ibqp-&gt;qp_num, attr-&gt;qp_state,</div><div class='add'>+		  iwqp-&gt;ibqp_state, iwqp-&gt;iwarp_state, attr_mask);</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+	if (attr_mask &amp; IB_QP_STATE) {</div><div class='add'>+		if (!ib_modify_qp_is_ok(iwqp-&gt;ibqp_state, attr-&gt;qp_state,</div><div class='add'>+					iwqp-&gt;ibqp.qp_type, attr_mask)) {</div><div class='add'>+			ibdev_warn(&amp;iwdev-&gt;ibdev, "modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n",</div><div class='add'>+				   iwqp-&gt;ibqp.qp_num, iwqp-&gt;ibqp_state,</div><div class='add'>+				   attr-&gt;qp_state);</div><div class='add'>+			ret = -EINVAL;</div><div class='add'>+			goto exit;</div><div class='add'>+		}</div><div class='add'>+		info.curr_iwarp_state = iwqp-&gt;iwarp_state;</div><div class='add'>+</div><div class='add'>+		switch (attr-&gt;qp_state) {</div><div class='add'>+		case IB_QPS_INIT:</div><div class='add'>+			if (iwqp-&gt;iwarp_state &gt; IRDMA_QP_STATE_IDLE) {</div><div class='add'>+				ret = -EINVAL;</div><div class='add'>+				goto exit;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			if (iwqp-&gt;iwarp_state == IRDMA_QP_STATE_INVALID) {</div><div class='add'>+				info.next_iwarp_state = IRDMA_QP_STATE_IDLE;</div><div class='add'>+				issue_modify_qp = 1;</div><div class='add'>+			}</div><div class='add'>+			break;</div><div class='add'>+		case IB_QPS_RTR:</div><div class='add'>+			if (iwqp-&gt;iwarp_state &gt; IRDMA_QP_STATE_IDLE) {</div><div class='add'>+				ret = -EINVAL;</div><div class='add'>+				goto exit;</div><div class='add'>+			}</div><div class='add'>+			info.arp_cache_idx_valid = true;</div><div class='add'>+			info.cq_num_valid = true;</div><div class='add'>+			info.next_iwarp_state = IRDMA_QP_STATE_RTR;</div><div class='add'>+			issue_modify_qp = 1;</div><div class='add'>+			break;</div><div class='add'>+		case IB_QPS_RTS:</div><div class='add'>+			if (iwqp-&gt;ibqp_state &lt; IB_QPS_RTR ||</div><div class='add'>+			    iwqp-&gt;ibqp_state == IB_QPS_ERR) {</div><div class='add'>+				ret = -EINVAL;</div><div class='add'>+				goto exit;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			info.arp_cache_idx_valid = true;</div><div class='add'>+			info.cq_num_valid = true;</div><div class='add'>+			info.ord_valid = true;</div><div class='add'>+			info.next_iwarp_state = IRDMA_QP_STATE_RTS;</div><div class='add'>+			issue_modify_qp = 1;</div><div class='add'>+			if (iwdev-&gt;push_mode &amp;&amp; udata &amp;&amp;</div><div class='add'>+			    iwqp-&gt;sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX &amp;&amp;</div><div class='add'>+			    dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+				spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+				irdma_alloc_push_page(iwqp);</div><div class='add'>+				spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+			}</div><div class='add'>+			break;</div><div class='add'>+		case IB_QPS_SQD:</div><div class='add'>+			if (iwqp-&gt;iwarp_state == IRDMA_QP_STATE_SQD)</div><div class='add'>+				goto exit;</div><div class='add'>+</div><div class='add'>+			if (iwqp-&gt;iwarp_state != IRDMA_QP_STATE_RTS) {</div><div class='add'>+				ret = -EINVAL;</div><div class='add'>+				goto exit;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			info.next_iwarp_state = IRDMA_QP_STATE_SQD;</div><div class='add'>+			issue_modify_qp = 1;</div><div class='add'>+			iwqp-&gt;suspend_pending = true;</div><div class='add'>+			break;</div><div class='add'>+		case IB_QPS_SQE:</div><div class='add'>+		case IB_QPS_ERR:</div><div class='add'>+		case IB_QPS_RESET:</div><div class='add'>+			if (iwqp-&gt;iwarp_state == IRDMA_QP_STATE_ERROR) {</div><div class='add'>+				spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+				if (udata &amp;&amp; udata-&gt;inlen) {</div><div class='add'>+					if (ib_copy_from_udata(&amp;ureq, udata,</div><div class='add'>+					    min(sizeof(ureq), udata-&gt;inlen)))</div><div class='add'>+						return -EINVAL;</div><div class='add'>+</div><div class='add'>+					irdma_flush_wqes(iwqp,</div><div class='add'>+					    (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) |</div><div class='add'>+					    (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) |</div><div class='add'>+					    IRDMA_REFLUSH);</div><div class='add'>+				}</div><div class='add'>+				return 0;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			info.next_iwarp_state = IRDMA_QP_STATE_ERROR;</div><div class='add'>+			issue_modify_qp = 1;</div><div class='add'>+			break;</div><div class='add'>+		default:</div><div class='add'>+			ret = -EINVAL;</div><div class='add'>+			goto exit;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		iwqp-&gt;ibqp_state = attr-&gt;qp_state;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ctx_info-&gt;send_cq_num = iwqp-&gt;iwscq-&gt;sc_cq.cq_uk.cq_id;</div><div class='add'>+	ctx_info-&gt;rcv_cq_num = iwqp-&gt;iwrcq-&gt;sc_cq.cq_uk.cq_id;</div><div class='add'>+	irdma_sc_qp_setctx_roce(&amp;iwqp-&gt;sc_qp, iwqp-&gt;host_ctx.va, ctx_info);</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_STATE) {</div><div class='add'>+		if (issue_modify_qp) {</div><div class='add'>+			ctx_info-&gt;rem_endpoint_idx = udp_info-&gt;arp_idx;</div><div class='add'>+			if (irdma_hw_modify_qp(iwdev, iwqp, &amp;info, true))</div><div class='add'>+				return -EINVAL;</div><div class='add'>+			if (info.next_iwarp_state == IRDMA_QP_STATE_SQD) {</div><div class='add'>+				ret = irdma_wait_for_suspend(iwqp);</div><div class='add'>+				if (ret)</div><div class='add'>+					return ret;</div><div class='add'>+			}</div><div class='add'>+			spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+			if (iwqp-&gt;iwarp_state == info.curr_iwarp_state) {</div><div class='add'>+				iwqp-&gt;iwarp_state = info.next_iwarp_state;</div><div class='add'>+				iwqp-&gt;ibqp_state = attr-&gt;qp_state;</div><div class='add'>+			}</div><div class='add'>+			if (iwqp-&gt;ibqp_state &gt; IB_QPS_RTS &amp;&amp;</div><div class='add'>+			    !iwqp-&gt;flush_issued) {</div><div class='add'>+				spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+				irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ |</div><div class='add'>+						       IRDMA_FLUSH_RQ |</div><div class='add'>+						       IRDMA_FLUSH_WAIT);</div><div class='add'>+				iwqp-&gt;flush_issued = 1;</div><div class='add'>+			} else {</div><div class='add'>+				spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+			}</div><div class='add'>+		} else {</div><div class='add'>+			iwqp-&gt;ibqp_state = attr-&gt;qp_state;</div><div class='add'>+		}</div><div class='add'>+		if (udata &amp;&amp; udata-&gt;outlen &amp;&amp; dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+			struct irdma_ucontext *ucontext;</div><div class='add'>+</div><div class='add'>+			ucontext = rdma_udata_to_drv_context(udata,</div><div class='add'>+					struct irdma_ucontext, ibucontext);</div><div class='add'>+			if (iwqp-&gt;sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX &amp;&amp;</div><div class='add'>+			    !iwqp-&gt;push_wqe_mmap_entry &amp;&amp;</div><div class='add'>+			    !irdma_setup_push_mmap_entries(ucontext, iwqp,</div><div class='add'>+				&amp;uresp.push_wqe_mmap_key, &amp;uresp.push_db_mmap_key)) {</div><div class='add'>+				uresp.push_valid = 1;</div><div class='add'>+				uresp.push_offset = iwqp-&gt;sc_qp.push_offset;</div><div class='add'>+			}</div><div class='add'>+			ret = ib_copy_to_udata(udata, &amp;uresp, min(sizeof(uresp),</div><div class='add'>+					       udata-&gt;outlen));</div><div class='add'>+			if (ret) {</div><div class='add'>+				irdma_remove_push_mmap_entries(iwqp);</div><div class='add'>+				ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+					  "VERBS: copy_to_udata failed\n");</div><div class='add'>+				return ret;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+exit:</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_modify_qp - modify qp request</div><div class='add'>+ * @ibqp: qp's pointer for modify</div><div class='add'>+ * @attr: access attributes</div><div class='add'>+ * @attr_mask: state mask</div><div class='add'>+ * @udata: user data</div><div class='add'>+ */</div><div class='add'>+int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,</div><div class='add'>+		    struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+#define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush)</div><div class='add'>+#define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid)</div><div class='add'>+	struct irdma_qp *iwqp = to_iwqp(ibqp);</div><div class='add'>+	struct irdma_device *iwdev = iwqp-&gt;iwdev;</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_qp_host_ctx_info *ctx_info;</div><div class='add'>+	struct irdma_tcp_offload_info *tcp_info;</div><div class='add'>+	struct irdma_iwarp_offload_info *offload_info;</div><div class='add'>+	struct irdma_modify_qp_info info = {};</div><div class='add'>+	struct irdma_modify_qp_resp uresp = {};</div><div class='add'>+	struct irdma_modify_qp_req ureq = {};</div><div class='add'>+	u8 issue_modify_qp = 0;</div><div class='add'>+	u8 dont_wait = 0;</div><div class='add'>+	int err;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		/* udata inlen/outlen can be 0 when supporting legacy libi40iw */</div><div class='add'>+		if ((udata-&gt;inlen &amp;&amp; udata-&gt;inlen &lt; IRDMA_MODIFY_QP_MIN_REQ_LEN) ||</div><div class='add'>+		    (udata-&gt;outlen &amp;&amp; udata-&gt;outlen &lt; IRDMA_MODIFY_QP_MIN_RESP_LEN))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; ~IB_QP_ATTR_STANDARD_BITS)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	ctx_info = &amp;iwqp-&gt;ctx_info;</div><div class='add'>+	offload_info = &amp;iwqp-&gt;iwarp_info;</div><div class='add'>+	tcp_info = &amp;iwqp-&gt;tcp_info;</div><div class='add'>+	wait_event(iwqp-&gt;mod_qp_waitq, !atomic_read(&amp;iwqp-&gt;hw_mod_qp_pend));</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+		  "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d last_aeq=%d hw_tcp_state=%d hw_iwarp_state=%d attr_mask=0x%x\n",</div><div class='add'>+		  __builtin_return_address(0), ibqp-&gt;qp_num, attr-&gt;qp_state,</div><div class='add'>+		  iwqp-&gt;ibqp_state, iwqp-&gt;iwarp_state, iwqp-&gt;last_aeq,</div><div class='add'>+		  iwqp-&gt;hw_tcp_state, iwqp-&gt;hw_iwarp_state, attr_mask);</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+	if (attr_mask &amp; IB_QP_STATE) {</div><div class='add'>+		info.curr_iwarp_state = iwqp-&gt;iwarp_state;</div><div class='add'>+		switch (attr-&gt;qp_state) {</div><div class='add'>+		case IB_QPS_INIT:</div><div class='add'>+		case IB_QPS_RTR:</div><div class='add'>+			if (iwqp-&gt;iwarp_state &gt; IRDMA_QP_STATE_IDLE) {</div><div class='add'>+				err = -EINVAL;</div><div class='add'>+				goto exit;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			if (iwqp-&gt;iwarp_state == IRDMA_QP_STATE_INVALID) {</div><div class='add'>+				info.next_iwarp_state = IRDMA_QP_STATE_IDLE;</div><div class='add'>+				issue_modify_qp = 1;</div><div class='add'>+			}</div><div class='add'>+			if (iwdev-&gt;push_mode &amp;&amp; udata &amp;&amp;</div><div class='add'>+			    iwqp-&gt;sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX &amp;&amp;</div><div class='add'>+			    dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+				spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+				irdma_alloc_push_page(iwqp);</div><div class='add'>+				spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+			}</div><div class='add'>+			break;</div><div class='add'>+		case IB_QPS_RTS:</div><div class='add'>+			if (iwqp-&gt;iwarp_state &gt; IRDMA_QP_STATE_RTS ||</div><div class='add'>+			    !iwqp-&gt;cm_id) {</div><div class='add'>+				err = -EINVAL;</div><div class='add'>+				goto exit;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			issue_modify_qp = 1;</div><div class='add'>+			iwqp-&gt;hw_tcp_state = IRDMA_TCP_STATE_ESTABLISHED;</div><div class='add'>+			iwqp-&gt;hte_added = 1;</div><div class='add'>+			info.next_iwarp_state = IRDMA_QP_STATE_RTS;</div><div class='add'>+			info.tcp_ctx_valid = true;</div><div class='add'>+			info.ord_valid = true;</div><div class='add'>+			info.arp_cache_idx_valid = true;</div><div class='add'>+			info.cq_num_valid = true;</div><div class='add'>+			break;</div><div class='add'>+		case IB_QPS_SQD:</div><div class='add'>+			if (iwqp-&gt;hw_iwarp_state &gt; IRDMA_QP_STATE_RTS) {</div><div class='add'>+				err = 0;</div><div class='add'>+				goto exit;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			if (iwqp-&gt;iwarp_state == IRDMA_QP_STATE_CLOSING ||</div><div class='add'>+			    iwqp-&gt;iwarp_state &lt; IRDMA_QP_STATE_RTS) {</div><div class='add'>+				err = 0;</div><div class='add'>+				goto exit;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			if (iwqp-&gt;iwarp_state &gt; IRDMA_QP_STATE_CLOSING) {</div><div class='add'>+				err = -EINVAL;</div><div class='add'>+				goto exit;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			info.next_iwarp_state = IRDMA_QP_STATE_CLOSING;</div><div class='add'>+			issue_modify_qp = 1;</div><div class='add'>+			break;</div><div class='add'>+		case IB_QPS_SQE:</div><div class='add'>+			if (iwqp-&gt;iwarp_state &gt;= IRDMA_QP_STATE_TERMINATE) {</div><div class='add'>+				err = -EINVAL;</div><div class='add'>+				goto exit;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			info.next_iwarp_state = IRDMA_QP_STATE_TERMINATE;</div><div class='add'>+			issue_modify_qp = 1;</div><div class='add'>+			break;</div><div class='add'>+		case IB_QPS_ERR:</div><div class='add'>+		case IB_QPS_RESET:</div><div class='add'>+			if (iwqp-&gt;iwarp_state == IRDMA_QP_STATE_ERROR) {</div><div class='add'>+				spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+				if (udata &amp;&amp; udata-&gt;inlen) {</div><div class='add'>+					if (ib_copy_from_udata(&amp;ureq, udata,</div><div class='add'>+					    min(sizeof(ureq), udata-&gt;inlen)))</div><div class='add'>+						return -EINVAL;</div><div class='add'>+</div><div class='add'>+					irdma_flush_wqes(iwqp,</div><div class='add'>+					    (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) |</div><div class='add'>+					    (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) |</div><div class='add'>+					    IRDMA_REFLUSH);</div><div class='add'>+				}</div><div class='add'>+				return 0;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			if (iwqp-&gt;sc_qp.term_flags) {</div><div class='add'>+				spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+				irdma_terminate_del_timer(&amp;iwqp-&gt;sc_qp);</div><div class='add'>+				spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+			}</div><div class='add'>+			info.next_iwarp_state = IRDMA_QP_STATE_ERROR;</div><div class='add'>+			if (iwqp-&gt;hw_tcp_state &gt; IRDMA_TCP_STATE_CLOSED &amp;&amp;</div><div class='add'>+			    iwdev-&gt;iw_status &amp;&amp;</div><div class='add'>+			    iwqp-&gt;hw_tcp_state != IRDMA_TCP_STATE_TIME_WAIT)</div><div class='add'>+				info.reset_tcp_conn = true;</div><div class='add'>+			else</div><div class='add'>+				dont_wait = 1;</div><div class='add'>+</div><div class='add'>+			issue_modify_qp = 1;</div><div class='add'>+			info.next_iwarp_state = IRDMA_QP_STATE_ERROR;</div><div class='add'>+			break;</div><div class='add'>+		default:</div><div class='add'>+			err = -EINVAL;</div><div class='add'>+			goto exit;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		iwqp-&gt;ibqp_state = attr-&gt;qp_state;</div><div class='add'>+	}</div><div class='add'>+	if (attr_mask &amp; IB_QP_ACCESS_FLAGS) {</div><div class='add'>+		ctx_info-&gt;iwarp_info_valid = true;</div><div class='add'>+		if (attr-&gt;qp_access_flags &amp; IB_ACCESS_LOCAL_WRITE)</div><div class='add'>+			offload_info-&gt;wr_rdresp_en = true;</div><div class='add'>+		if (attr-&gt;qp_access_flags &amp; IB_ACCESS_REMOTE_WRITE)</div><div class='add'>+			offload_info-&gt;wr_rdresp_en = true;</div><div class='add'>+		if (attr-&gt;qp_access_flags &amp; IB_ACCESS_REMOTE_READ)</div><div class='add'>+			offload_info-&gt;rd_en = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ctx_info-&gt;iwarp_info_valid) {</div><div class='add'>+		ctx_info-&gt;send_cq_num = iwqp-&gt;iwscq-&gt;sc_cq.cq_uk.cq_id;</div><div class='add'>+		ctx_info-&gt;rcv_cq_num = iwqp-&gt;iwrcq-&gt;sc_cq.cq_uk.cq_id;</div><div class='add'>+		irdma_sc_qp_setctx(&amp;iwqp-&gt;sc_qp, iwqp-&gt;host_ctx.va, ctx_info);</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_STATE) {</div><div class='add'>+		if (issue_modify_qp) {</div><div class='add'>+			ctx_info-&gt;rem_endpoint_idx = tcp_info-&gt;arp_idx;</div><div class='add'>+			if (irdma_hw_modify_qp(iwdev, iwqp, &amp;info, true))</div><div class='add'>+				return -EINVAL;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+		if (iwqp-&gt;iwarp_state == info.curr_iwarp_state) {</div><div class='add'>+			iwqp-&gt;iwarp_state = info.next_iwarp_state;</div><div class='add'>+			iwqp-&gt;ibqp_state = attr-&gt;qp_state;</div><div class='add'>+		}</div><div class='add'>+		spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (issue_modify_qp &amp;&amp; iwqp-&gt;ibqp_state &gt; IB_QPS_RTS) {</div><div class='add'>+		if (dont_wait) {</div><div class='add'>+			if (iwqp-&gt;hw_tcp_state) {</div><div class='add'>+				spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+				iwqp-&gt;hw_tcp_state = IRDMA_TCP_STATE_CLOSED;</div><div class='add'>+				iwqp-&gt;last_aeq = IRDMA_AE_RESET_SENT;</div><div class='add'>+				spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+			}</div><div class='add'>+			irdma_cm_disconn(iwqp);</div><div class='add'>+		} else {</div><div class='add'>+			int close_timer_started;</div><div class='add'>+</div><div class='add'>+			spin_lock_irqsave(&amp;iwdev-&gt;cm_core.ht_lock, flags);</div><div class='add'>+</div><div class='add'>+			if (iwqp-&gt;cm_node) {</div><div class='add'>+				refcount_inc(&amp;iwqp-&gt;cm_node-&gt;refcnt);</div><div class='add'>+				spin_unlock_irqrestore(&amp;iwdev-&gt;cm_core.ht_lock, flags);</div><div class='add'>+				close_timer_started = atomic_inc_return(&amp;iwqp-&gt;close_timer_started);</div><div class='add'>+				if (iwqp-&gt;cm_id &amp;&amp; close_timer_started == 1)</div><div class='add'>+					irdma_schedule_cm_timer(iwqp-&gt;cm_node,</div><div class='add'>+						(struct irdma_puda_buf *)iwqp,</div><div class='add'>+						IRDMA_TIMER_TYPE_CLOSE, 1, 0);</div><div class='add'>+</div><div class='add'>+				irdma_rem_ref_cm_node(iwqp-&gt;cm_node);</div><div class='add'>+			} else {</div><div class='add'>+				spin_unlock_irqrestore(&amp;iwdev-&gt;cm_core.ht_lock, flags);</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	if (attr_mask &amp; IB_QP_STATE &amp;&amp; udata &amp;&amp; udata-&gt;outlen &amp;&amp;</div><div class='add'>+	    dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2) {</div><div class='add'>+		struct irdma_ucontext *ucontext;</div><div class='add'>+</div><div class='add'>+		ucontext = rdma_udata_to_drv_context(udata,</div><div class='add'>+					struct irdma_ucontext, ibucontext);</div><div class='add'>+		if (iwqp-&gt;sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX &amp;&amp;</div><div class='add'>+		    !iwqp-&gt;push_wqe_mmap_entry &amp;&amp;</div><div class='add'>+		    !irdma_setup_push_mmap_entries(ucontext, iwqp,</div><div class='add'>+			&amp;uresp.push_wqe_mmap_key, &amp;uresp.push_db_mmap_key)) {</div><div class='add'>+			uresp.push_valid = 1;</div><div class='add'>+			uresp.push_offset = iwqp-&gt;sc_qp.push_offset;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		err = ib_copy_to_udata(udata, &amp;uresp, min(sizeof(uresp),</div><div class='add'>+				       udata-&gt;outlen));</div><div class='add'>+		if (err) {</div><div class='add'>+			irdma_remove_push_mmap_entries(iwqp);</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+				  "VERBS: copy_to_udata failed\n");</div><div class='add'>+			return err;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+exit:</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_srq_free_rsrc - free up resources for srq</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @iwsrq: srq ptr</div><div class='add'>+ */</div><div class='add'>+static void irdma_srq_free_rsrc(struct irdma_pci_f *rf, struct irdma_srq *iwsrq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_srq *srq = &amp;iwsrq-&gt;sc_srq;</div><div class='add'>+</div><div class='add'>+	if (!iwsrq-&gt;user_mode) {</div><div class='add'>+		dma_free_coherent(rf-&gt;sc_dev.hw-&gt;device, iwsrq-&gt;kmem.size,</div><div class='add'>+				  iwsrq-&gt;kmem.va, iwsrq-&gt;kmem.pa);</div><div class='add'>+		iwsrq-&gt;kmem.va = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_free_rsrc(rf, rf-&gt;allocated_srqs, srq-&gt;srq_uk.srq_id);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_cq_free_rsrc - free up resources for cq</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @iwcq: cq ptr</div><div class='add'>+ */</div><div class='add'>+static void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_cq *cq = &amp;iwcq-&gt;sc_cq;</div><div class='add'>+</div><div class='add'>+	if (!iwcq-&gt;user_mode) {</div><div class='add'>+		dma_free_coherent(rf-&gt;sc_dev.hw-&gt;device, iwcq-&gt;kmem.size,</div><div class='add'>+				  iwcq-&gt;kmem.va, iwcq-&gt;kmem.pa);</div><div class='add'>+		iwcq-&gt;kmem.va = NULL;</div><div class='add'>+		dma_free_coherent(rf-&gt;sc_dev.hw-&gt;device,</div><div class='add'>+				  iwcq-&gt;kmem_shadow.size,</div><div class='add'>+				  iwcq-&gt;kmem_shadow.va, iwcq-&gt;kmem_shadow.pa);</div><div class='add'>+		iwcq-&gt;kmem_shadow.va = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_free_rsrc(rf, rf-&gt;allocated_cqs, cq-&gt;cq_uk.cq_id);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_free_cqbuf - worker to free a cq buffer</div><div class='add'>+ * @work: provides access to the cq buffer to free</div><div class='add'>+ */</div><div class='add'>+static void irdma_free_cqbuf(struct work_struct *work)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cq_buf *cq_buf = container_of(work, struct irdma_cq_buf, work);</div><div class='add'>+</div><div class='add'>+	dma_free_coherent(cq_buf-&gt;hw-&gt;device, cq_buf-&gt;kmem_buf.size,</div><div class='add'>+			  cq_buf-&gt;kmem_buf.va, cq_buf-&gt;kmem_buf.pa);</div><div class='add'>+	cq_buf-&gt;kmem_buf.va = NULL;</div><div class='add'>+	kfree(cq_buf);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_process_resize_list - remove resized cq buffers from the resize_list</div><div class='add'>+ * @iwcq: cq which owns the resize_list</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ * @lcqe_buf: the buffer where the last cqe is received</div><div class='add'>+ */</div><div class='add'>+static int irdma_process_resize_list(struct irdma_cq *iwcq,</div><div class='add'>+				     struct irdma_device *iwdev,</div><div class='add'>+				     struct irdma_cq_buf *lcqe_buf)</div><div class='add'>+{</div><div class='add'>+	struct list_head *tmp_node, *list_node;</div><div class='add'>+	struct irdma_cq_buf *cq_buf;</div><div class='add'>+	int cnt = 0;</div><div class='add'>+</div><div class='add'>+	list_for_each_safe(list_node, tmp_node, &amp;iwcq-&gt;resize_list) {</div><div class='add'>+		cq_buf = list_entry(list_node, struct irdma_cq_buf, list);</div><div class='add'>+		if (cq_buf == lcqe_buf)</div><div class='add'>+			return cnt;</div><div class='add'>+</div><div class='add'>+		list_del(&amp;cq_buf-&gt;list);</div><div class='add'>+		queue_work(iwdev-&gt;cleanup_wq, &amp;cq_buf-&gt;work);</div><div class='add'>+		cnt++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return cnt;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_destroy_srq - destroy srq</div><div class='add'>+ * @ibsrq: srq pointer</div><div class='add'>+ * @udata: user data</div><div class='add'>+ */</div><div class='add'>+static int irdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibsrq-&gt;device);</div><div class='add'>+	struct irdma_srq *iwsrq = to_iwsrq(ibsrq);</div><div class='add'>+	struct irdma_sc_srq *srq = &amp;iwsrq-&gt;sc_srq;</div><div class='add'>+</div><div class='add'>+	irdma_srq_wq_destroy(iwdev-&gt;rf, srq);</div><div class='add'>+	irdma_srq_free_rsrc(iwdev-&gt;rf, iwsrq);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_destroy_cq - destroy cq</div><div class='add'>+ * @ib_cq: cq pointer</div><div class='add'>+ * @udata: user data</div><div class='add'>+ */</div><div class='add'>+static int irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ib_cq-&gt;device);</div><div class='add'>+	struct irdma_cq *iwcq = to_iwcq(ib_cq);</div><div class='add'>+	struct irdma_sc_cq *cq = &amp;iwcq-&gt;sc_cq;</div><div class='add'>+	struct irdma_sc_dev *dev = cq-&gt;dev;</div><div class='add'>+	struct irdma_sc_ceq *ceq = dev-&gt;ceq[cq-&gt;ceq_id];</div><div class='add'>+	struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq);</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwcq-&gt;lock, flags);</div><div class='add'>+	if (!list_empty(&amp;iwcq-&gt;cmpl_generated))</div><div class='add'>+		irdma_remove_cmpls_list(iwcq);</div><div class='add'>+	if (!list_empty(&amp;iwcq-&gt;resize_list))</div><div class='add'>+		irdma_process_resize_list(iwcq, iwdev, NULL);</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwcq-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	irdma_cq_rem_ref(ib_cq);</div><div class='add'>+	wait_for_completion(&amp;iwcq-&gt;free_cq);</div><div class='add'>+</div><div class='add'>+	irdma_cq_wq_destroy(iwdev-&gt;rf, cq);</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwceq-&gt;ce_lock, flags);</div><div class='add'>+	irdma_sc_cleanup_ceqes(cq, ceq);</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwceq-&gt;ce_lock, flags);</div><div class='add'>+	irdma_cq_free_rsrc(iwdev-&gt;rf, iwcq);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_resize_cq - resize cq</div><div class='add'>+ * @ibcq: cq to be resized</div><div class='add'>+ * @entries: desired cq size</div><div class='add'>+ * @udata: user data</div><div class='add'>+ */</div><div class='add'>+static int irdma_resize_cq(struct ib_cq *ibcq, int entries,</div><div class='add'>+			   struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+#define IRDMA_RESIZE_CQ_MIN_REQ_LEN offsetofend(struct irdma_resize_cq_req, user_cq_buffer)</div><div class='add'>+	struct irdma_cq *iwcq = to_iwcq(ibcq);</div><div class='add'>+	struct irdma_sc_dev *dev = iwcq-&gt;sc_cq.dev;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	struct irdma_modify_cq_info *m_info;</div><div class='add'>+	struct irdma_modify_cq_info info = {};</div><div class='add'>+	struct irdma_dma_mem kmem_buf;</div><div class='add'>+	struct irdma_cq_mr *cqmr_buf;</div><div class='add'>+	struct irdma_pbl *iwpbl_buf;</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct irdma_pci_f *rf;</div><div class='add'>+	struct irdma_cq_buf *cq_buf = NULL;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	u8 cqe_size;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	iwdev = to_iwdev(ibcq-&gt;device);</div><div class='add'>+	rf = iwdev-&gt;rf;</div><div class='add'>+</div><div class='add'>+	if (!(rf-&gt;sc_dev.hw_attrs.uk_attrs.feature_flags &amp;</div><div class='add'>+	    IRDMA_FEATURE_CQ_RESIZE))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (udata &amp;&amp; udata-&gt;inlen &lt; IRDMA_RESIZE_CQ_MIN_REQ_LEN)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (entries &gt; rf-&gt;max_cqe)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!iwcq-&gt;user_mode) {</div><div class='add'>+		entries += 2;</div><div class='add'>+</div><div class='add'>+		if (!iwcq-&gt;sc_cq.cq_uk.avoid_mem_cflct &amp;&amp;</div><div class='add'>+		    dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2)</div><div class='add'>+			entries *= 2;</div><div class='add'>+</div><div class='add'>+		if (entries &amp; 1)</div><div class='add'>+			entries += 1; /* cq size must be an even number */</div><div class='add'>+</div><div class='add'>+		cqe_size = iwcq-&gt;sc_cq.cq_uk.avoid_mem_cflct ? 64 : 32;</div><div class='add'>+		if (entries * cqe_size == IRDMA_HW_PAGE_SIZE)</div><div class='add'>+			entries += 2;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	info.cq_size = max(entries, 4);</div><div class='add'>+</div><div class='add'>+	if (info.cq_size == iwcq-&gt;sc_cq.cq_uk.cq_size - 1)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		struct irdma_resize_cq_req req = {};</div><div class='add'>+		struct irdma_ucontext *ucontext =</div><div class='add'>+			rdma_udata_to_drv_context(udata, struct irdma_ucontext,</div><div class='add'>+						  ibucontext);</div><div class='add'>+</div><div class='add'>+		/* CQ resize not supported with legacy GEN_1 libi40iw */</div><div class='add'>+		if (ucontext-&gt;legacy_mode)</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+		if (ib_copy_from_udata(&amp;req, udata,</div><div class='add'>+				       min(sizeof(req), udata-&gt;inlen)))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		spin_lock_irqsave(&amp;ucontext-&gt;cq_reg_mem_list_lock, flags);</div><div class='add'>+		iwpbl_buf = irdma_get_pbl((unsigned long)req.user_cq_buffer,</div><div class='add'>+					  &amp;ucontext-&gt;cq_reg_mem_list);</div><div class='add'>+		spin_unlock_irqrestore(&amp;ucontext-&gt;cq_reg_mem_list_lock, flags);</div><div class='add'>+</div><div class='add'>+		if (!iwpbl_buf)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+</div><div class='add'>+		cqmr_buf = &amp;iwpbl_buf-&gt;cq_mr;</div><div class='add'>+		if (iwpbl_buf-&gt;pbl_allocated) {</div><div class='add'>+			info.virtual_map = true;</div><div class='add'>+			info.pbl_chunk_size = 1;</div><div class='add'>+			info.first_pm_pbl_idx = cqmr_buf-&gt;cq_pbl.idx;</div><div class='add'>+		} else {</div><div class='add'>+			info.cq_pa = cqmr_buf-&gt;cq_pbl.addr;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		/* Kmode CQ resize */</div><div class='add'>+		int rsize;</div><div class='add'>+</div><div class='add'>+		rsize = info.cq_size * sizeof(struct irdma_cqe);</div><div class='add'>+		kmem_buf.size = ALIGN(round_up(rsize, 256), 256);</div><div class='add'>+		kmem_buf.va = dma_alloc_coherent(dev-&gt;hw-&gt;device,</div><div class='add'>+						 kmem_buf.size, &amp;kmem_buf.pa,</div><div class='add'>+						 GFP_KERNEL);</div><div class='add'>+		if (!kmem_buf.va)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+</div><div class='add'>+		info.cq_base = kmem_buf.va;</div><div class='add'>+		info.cq_pa = kmem_buf.pa;</div><div class='add'>+		cq_buf = kzalloc(sizeof(*cq_buf), GFP_KERNEL);</div><div class='add'>+		if (!cq_buf) {</div><div class='add'>+			ret = -ENOMEM;</div><div class='add'>+			goto error;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	info.shadow_read_threshold = iwcq-&gt;sc_cq.shadow_read_threshold;</div><div class='add'>+	info.cq_resize = true;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	m_info = &amp;cqp_info-&gt;in.u.cq_modify.info;</div><div class='add'>+	memcpy(m_info, &amp;info, sizeof(*m_info));</div><div class='add'>+</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_CQ_MODIFY;</div><div class='add'>+	cqp_info-&gt;in.u.cq_modify.cq = &amp;iwcq-&gt;sc_cq;</div><div class='add'>+	cqp_info-&gt;in.u.cq_modify.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	ret = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwcq-&gt;lock, flags);</div><div class='add'>+	if (cq_buf) {</div><div class='add'>+		cq_buf-&gt;kmem_buf = iwcq-&gt;kmem;</div><div class='add'>+		cq_buf-&gt;hw = dev-&gt;hw;</div><div class='add'>+		memcpy(&amp;cq_buf-&gt;cq_uk, &amp;iwcq-&gt;sc_cq.cq_uk, sizeof(cq_buf-&gt;cq_uk));</div><div class='add'>+		INIT_WORK(&amp;cq_buf-&gt;work, irdma_free_cqbuf);</div><div class='add'>+		list_add_tail(&amp;cq_buf-&gt;list, &amp;iwcq-&gt;resize_list);</div><div class='add'>+		iwcq-&gt;kmem = kmem_buf;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_sc_cq_resize(&amp;iwcq-&gt;sc_cq, &amp;info);</div><div class='add'>+	ibcq-&gt;cqe = info.cq_size - 1;</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwcq-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+error:</div><div class='add'>+	if (!udata) {</div><div class='add'>+		dma_free_coherent(dev-&gt;hw-&gt;device, kmem_buf.size, kmem_buf.va,</div><div class='add'>+				  kmem_buf.pa);</div><div class='add'>+		kmem_buf.va = NULL;</div><div class='add'>+	}</div><div class='add'>+	kfree(cq_buf);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_srq_event - event notification for srq limit</div><div class='add'>+ * @srq: shared srq struct</div><div class='add'>+ */</div><div class='add'>+void irdma_srq_event(struct irdma_sc_srq *srq)</div><div class='add'>+{</div><div class='add'>+	struct irdma_srq *iwsrq = container_of(srq, struct irdma_srq, sc_srq);</div><div class='add'>+	struct ib_srq *ibsrq = &amp;iwsrq-&gt;ibsrq;</div><div class='add'>+	struct ib_event event;</div><div class='add'>+</div><div class='add'>+	srq-&gt;srq_limit = 0;</div><div class='add'>+</div><div class='add'>+	if (!ibsrq-&gt;event_handler)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	event.device = ibsrq-&gt;device;</div><div class='add'>+	event.element.port_num = 1;</div><div class='add'>+	event.element.srq = ibsrq;</div><div class='add'>+	event.event = IB_EVENT_SRQ_LIMIT_REACHED;</div><div class='add'>+	ibsrq-&gt;event_handler(&amp;event, ibsrq-&gt;srq_context);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_modify_srq - modify srq request</div><div class='add'>+ * @ibsrq: srq's pointer for modify</div><div class='add'>+ * @attr: access attributes</div><div class='add'>+ * @attr_mask: state mask</div><div class='add'>+ * @udata: user data</div><div class='add'>+ */</div><div class='add'>+static int irdma_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,</div><div class='add'>+			    enum ib_srq_attr_mask attr_mask,</div><div class='add'>+			    struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibsrq-&gt;device);</div><div class='add'>+	struct irdma_srq *iwsrq = to_iwsrq(ibsrq);</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+	struct irdma_modify_srq_info *info;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_SRQ_MAX_WR)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!(attr_mask &amp; IB_SRQ_LIMIT))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;srq_limit &gt; iwsrq-&gt;sc_srq.srq_uk.srq_size)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	/* Execute this cqp op synchronously, so we can update srq_limit</div><div class='add'>+	 * upon successful completion.</div><div class='add'>+	 */</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	info = &amp;cqp_info-&gt;in.u.srq_modify.info;</div><div class='add'>+	info-&gt;srq_limit = attr-&gt;srq_limit;</div><div class='add'>+	if (info-&gt;srq_limit &gt; 0xFFF)</div><div class='add'>+		info-&gt;srq_limit = 0xFFF;</div><div class='add'>+	info-&gt;arm_limit_event = 1;</div><div class='add'>+</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_SRQ_MODIFY;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.srq_modify.srq = &amp;iwsrq-&gt;sc_srq;</div><div class='add'>+	cqp_info-&gt;in.u.srq_modify.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	status = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+	if (status)</div><div class='add'>+		return status;</div><div class='add'>+</div><div class='add'>+	iwsrq-&gt;sc_srq.srq_limit = info-&gt;srq_limit;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_setup_umode_srq(struct irdma_device *iwdev,</div><div class='add'>+				 struct irdma_srq *iwsrq,</div><div class='add'>+				 struct irdma_srq_init_info *info,</div><div class='add'>+				 struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+#define IRDMA_CREATE_SRQ_MIN_REQ_LEN \</div><div class='add'>+	offsetofend(struct irdma_create_srq_req, user_shadow_area)</div><div class='add'>+	struct irdma_create_srq_req req = {};</div><div class='add'>+	struct irdma_ucontext *ucontext;</div><div class='add'>+	struct irdma_srq_mr *srqmr;</div><div class='add'>+	struct irdma_pbl *iwpbl;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	iwsrq-&gt;user_mode = true;</div><div class='add'>+	ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,</div><div class='add'>+					     ibucontext);</div><div class='add'>+</div><div class='add'>+	if (udata-&gt;inlen &lt; IRDMA_CREATE_SRQ_MIN_REQ_LEN)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (ib_copy_from_udata(&amp;req, udata,</div><div class='add'>+			       min(sizeof(req), udata-&gt;inlen)))</div><div class='add'>+		return -EFAULT;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;ucontext-&gt;srq_reg_mem_list_lock, flags);</div><div class='add'>+	iwpbl = irdma_get_pbl((unsigned long)req.user_srq_buf,</div><div class='add'>+			      &amp;ucontext-&gt;srq_reg_mem_list);</div><div class='add'>+	spin_unlock_irqrestore(&amp;ucontext-&gt;srq_reg_mem_list_lock, flags);</div><div class='add'>+	if (!iwpbl)</div><div class='add'>+		return -EPROTO;</div><div class='add'>+</div><div class='add'>+	iwsrq-&gt;iwpbl = iwpbl;</div><div class='add'>+	srqmr = &amp;iwpbl-&gt;srq_mr;</div><div class='add'>+</div><div class='add'>+	if (iwpbl-&gt;pbl_allocated) {</div><div class='add'>+		info-&gt;virtual_map = true;</div><div class='add'>+		info-&gt;pbl_chunk_size = 1;</div><div class='add'>+		info-&gt;first_pm_pbl_idx = srqmr-&gt;srq_pbl.idx;</div><div class='add'>+		info-&gt;leaf_pbl_size = 1;</div><div class='add'>+	} else {</div><div class='add'>+		info-&gt;srq_pa = srqmr-&gt;srq_pbl.addr;</div><div class='add'>+	}</div><div class='add'>+	info-&gt;shadow_area_pa = srqmr-&gt;shadow;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_setup_kmode_srq(struct irdma_device *iwdev,</div><div class='add'>+				 struct irdma_srq *iwsrq,</div><div class='add'>+				 struct irdma_srq_init_info *info, u32 depth,</div><div class='add'>+				 u8 shift)</div><div class='add'>+{</div><div class='add'>+	struct irdma_srq_uk_init_info *ukinfo = &amp;info-&gt;srq_uk_init_info;</div><div class='add'>+	struct irdma_dma_mem *mem = &amp;iwsrq-&gt;kmem;</div><div class='add'>+	u32 size, ring_size;</div><div class='add'>+</div><div class='add'>+	ring_size = depth * IRDMA_QP_WQE_MIN_SIZE;</div><div class='add'>+	size = ring_size + (IRDMA_SHADOW_AREA_SIZE &lt;&lt; 3);</div><div class='add'>+</div><div class='add'>+	mem-&gt;size = ALIGN(size, 256);</div><div class='add'>+	mem-&gt;va = dma_alloc_coherent(iwdev-&gt;rf-&gt;hw.device, mem-&gt;size,</div><div class='add'>+				     &amp;mem-&gt;pa, GFP_KERNEL);</div><div class='add'>+	if (!mem-&gt;va)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	ukinfo-&gt;srq = mem-&gt;va;</div><div class='add'>+	ukinfo-&gt;srq_size = depth &gt;&gt; shift;</div><div class='add'>+	ukinfo-&gt;shadow_area = mem-&gt;va + ring_size;</div><div class='add'>+</div><div class='add'>+	info-&gt;srq_pa = mem-&gt;pa;</div><div class='add'>+	info-&gt;shadow_area_pa = info-&gt;srq_pa + ring_size;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_create_srq - create srq</div><div class='add'>+ * @ibsrq: ib's srq pointer</div><div class='add'>+ * @initattrs: attributes for srq</div><div class='add'>+ * @udata: user data for create srq</div><div class='add'>+ */</div><div class='add'>+static int irdma_create_srq(struct ib_srq *ibsrq,</div><div class='add'>+			    struct ib_srq_init_attr *initattrs,</div><div class='add'>+			    struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibsrq-&gt;device);</div><div class='add'>+	struct ib_srq_attr *attr = &amp;initattrs-&gt;attr;</div><div class='add'>+	struct irdma_pd *iwpd = to_iwpd(ibsrq-&gt;pd);</div><div class='add'>+	struct irdma_srq *iwsrq = to_iwsrq(ibsrq);</div><div class='add'>+	struct irdma_srq_uk_init_info *ukinfo;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct irdma_srq_init_info info = {};</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+	struct irdma_uk_attrs *uk_attrs;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	int err_code = 0;</div><div class='add'>+	u32 depth;</div><div class='add'>+	u8 shift;</div><div class='add'>+</div><div class='add'>+	uk_attrs = &amp;rf-&gt;sc_dev.hw_attrs.uk_attrs;</div><div class='add'>+	ukinfo = &amp;info.srq_uk_init_info;</div><div class='add'>+</div><div class='add'>+	if (initattrs-&gt;srq_type != IB_SRQT_BASIC)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (!(uk_attrs-&gt;feature_flags &amp; IRDMA_FEATURE_SRQ) ||</div><div class='add'>+	    attr-&gt;max_sge &gt; uk_attrs-&gt;max_hw_wq_frags)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	refcount_set(&amp;iwsrq-&gt;refcnt, 1);</div><div class='add'>+	spin_lock_init(&amp;iwsrq-&gt;lock);</div><div class='add'>+	err_code = irdma_alloc_rsrc(rf, rf-&gt;allocated_srqs, rf-&gt;max_srq,</div><div class='add'>+				    &amp;iwsrq-&gt;srq_num, &amp;rf-&gt;next_srq);</div><div class='add'>+	if (err_code)</div><div class='add'>+		return err_code;</div><div class='add'>+</div><div class='add'>+	ukinfo-&gt;max_srq_frag_cnt = attr-&gt;max_sge;</div><div class='add'>+	ukinfo-&gt;uk_attrs = uk_attrs;</div><div class='add'>+	ukinfo-&gt;srq_id = iwsrq-&gt;srq_num;</div><div class='add'>+</div><div class='add'>+	irdma_get_wqe_shift(ukinfo-&gt;uk_attrs, ukinfo-&gt;max_srq_frag_cnt, 0,</div><div class='add'>+			    &amp;shift);</div><div class='add'>+</div><div class='add'>+	err_code = irdma_get_srqdepth(ukinfo-&gt;uk_attrs, attr-&gt;max_wr,</div><div class='add'>+				      shift, &amp;depth);</div><div class='add'>+	if (err_code)</div><div class='add'>+		return err_code;</div><div class='add'>+</div><div class='add'>+	/* Actual SRQ size in WRs for ring and HW */</div><div class='add'>+	ukinfo-&gt;srq_size = depth &gt;&gt; shift;</div><div class='add'>+</div><div class='add'>+	/* Max postable WRs to SRQ */</div><div class='add'>+	iwsrq-&gt;max_wr = (depth - IRDMA_RQ_RSVD) &gt;&gt; shift;</div><div class='add'>+	attr-&gt;max_wr = iwsrq-&gt;max_wr;</div><div class='add'>+</div><div class='add'>+	if (udata)</div><div class='add'>+		err_code = irdma_setup_umode_srq(iwdev, iwsrq, &amp;info, udata);</div><div class='add'>+	else</div><div class='add'>+		err_code = irdma_setup_kmode_srq(iwdev, iwsrq, &amp;info, depth,</div><div class='add'>+						 shift);</div><div class='add'>+</div><div class='add'>+	if (err_code)</div><div class='add'>+		goto free_rsrc;</div><div class='add'>+</div><div class='add'>+	info.vsi = &amp;iwdev-&gt;vsi;</div><div class='add'>+	info.pd = &amp;iwpd-&gt;sc_pd;</div><div class='add'>+</div><div class='add'>+	iwsrq-&gt;sc_srq.srq_uk.lock = &amp;iwsrq-&gt;lock;</div><div class='add'>+	err_code = irdma_sc_srq_init(&amp;iwsrq-&gt;sc_srq, &amp;info);</div><div class='add'>+	if (err_code)</div><div class='add'>+		goto free_dmem;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request) {</div><div class='add'>+		err_code = -ENOMEM;</div><div class='add'>+		goto free_dmem;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_SRQ_CREATE;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.srq_create.srq = &amp;iwsrq-&gt;sc_srq;</div><div class='add'>+	cqp_info-&gt;in.u.srq_create.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	err_code = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+	if (err_code)</div><div class='add'>+		goto free_dmem;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		struct irdma_create_srq_resp resp = {};</div><div class='add'>+</div><div class='add'>+		resp.srq_id = iwsrq-&gt;srq_num;</div><div class='add'>+		resp.srq_size = ukinfo-&gt;srq_size;</div><div class='add'>+		if (ib_copy_to_udata(udata, &amp;resp,</div><div class='add'>+				     min(sizeof(resp), udata-&gt;outlen))) {</div><div class='add'>+			err_code = -EPROTO;</div><div class='add'>+			goto srq_destroy;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+srq_destroy:</div><div class='add'>+	irdma_srq_wq_destroy(rf, &amp;iwsrq-&gt;sc_srq);</div><div class='add'>+</div><div class='add'>+free_dmem:</div><div class='add'>+	if (!iwsrq-&gt;user_mode)</div><div class='add'>+		dma_free_coherent(rf-&gt;hw.device, iwsrq-&gt;kmem.size,</div><div class='add'>+				  iwsrq-&gt;kmem.va, iwsrq-&gt;kmem.pa);</div><div class='add'>+free_rsrc:</div><div class='add'>+	irdma_free_rsrc(rf, rf-&gt;allocated_srqs, iwsrq-&gt;srq_num);</div><div class='add'>+	return err_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_query_srq - get SRQ attributes</div><div class='add'>+ * @ibsrq: the SRQ to query</div><div class='add'>+ * @attr: the attributes of the SRQ</div><div class='add'>+ */</div><div class='add'>+static int irdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_srq *iwsrq = to_iwsrq(ibsrq);</div><div class='add'>+</div><div class='add'>+	attr-&gt;max_wr = iwsrq-&gt;max_wr;</div><div class='add'>+	attr-&gt;max_sge = iwsrq-&gt;sc_srq.srq_uk.max_srq_frag_cnt;</div><div class='add'>+	attr-&gt;srq_limit = iwsrq-&gt;sc_srq.srq_limit;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline int cq_validate_flags(u32 flags, u8 hw_rev)</div><div class='add'>+{</div><div class='add'>+	/* GEN1/2 does not support CQ create flags */</div><div class='add'>+	if (hw_rev &lt;= IRDMA_GEN_2)</div><div class='add'>+		return flags ? -EOPNOTSUPP : 0;</div><div class='add'>+</div><div class='add'>+	return flags &amp; ~IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION ? -EOPNOTSUPP : 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_create_cq - create cq</div><div class='add'>+ * @ibcq: CQ allocated</div><div class='add'>+ * @attr: attributes for cq</div><div class='add'>+ * @attrs: uverbs attribute bundle</div><div class='add'>+ */</div><div class='add'>+static int irdma_create_cq(struct ib_cq *ibcq,</div><div class='add'>+			   const struct ib_cq_init_attr *attr,</div><div class='add'>+			   struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+#define IRDMA_CREATE_CQ_MIN_REQ_LEN offsetofend(struct irdma_create_cq_req, user_cq_buf)</div><div class='add'>+#define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size)</div><div class='add'>+	struct ib_udata *udata = &amp;attrs-&gt;driver_udata;</div><div class='add'>+	struct ib_device *ibdev = ibcq-&gt;device;</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibdev);</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+	struct irdma_cq *iwcq = to_iwcq(ibcq);</div><div class='add'>+	u32 cq_num = 0;</div><div class='add'>+	struct irdma_sc_cq *cq;</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;rf-&gt;sc_dev;</div><div class='add'>+	struct irdma_cq_init_info info = {};</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	struct irdma_cq_uk_init_info *ukinfo = &amp;info.cq_uk_init_info;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int err_code;</div><div class='add'>+	int entries = attr-&gt;cqe;</div><div class='add'>+	bool cqe_64byte_ena;</div><div class='add'>+	u8 cqe_size;</div><div class='add'>+</div><div class='add'>+	err_code = cq_validate_flags(attr-&gt;flags, dev-&gt;hw_attrs.uk_attrs.hw_rev);</div><div class='add'>+	if (err_code)</div><div class='add'>+		return err_code;</div><div class='add'>+</div><div class='add'>+	if (udata &amp;&amp; (udata-&gt;inlen &lt; IRDMA_CREATE_CQ_MIN_REQ_LEN ||</div><div class='add'>+		      udata-&gt;outlen &lt; IRDMA_CREATE_CQ_MIN_RESP_LEN))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	err_code = irdma_alloc_rsrc(rf, rf-&gt;allocated_cqs, rf-&gt;max_cq, &amp;cq_num,</div><div class='add'>+				    &amp;rf-&gt;next_cq);</div><div class='add'>+	if (err_code)</div><div class='add'>+		return err_code;</div><div class='add'>+</div><div class='add'>+	cq = &amp;iwcq-&gt;sc_cq;</div><div class='add'>+	cq-&gt;back_cq = iwcq;</div><div class='add'>+	refcount_set(&amp;iwcq-&gt;refcnt, 1);</div><div class='add'>+	spin_lock_init(&amp;iwcq-&gt;lock);</div><div class='add'>+	INIT_LIST_HEAD(&amp;iwcq-&gt;resize_list);</div><div class='add'>+	INIT_LIST_HEAD(&amp;iwcq-&gt;cmpl_generated);</div><div class='add'>+	iwcq-&gt;cq_num = cq_num;</div><div class='add'>+	info.dev = dev;</div><div class='add'>+	ukinfo-&gt;cq_size = max(entries, 4);</div><div class='add'>+	ukinfo-&gt;cq_id = cq_num;</div><div class='add'>+	cqe_64byte_ena = dev-&gt;hw_attrs.uk_attrs.feature_flags &amp; IRDMA_FEATURE_64_BYTE_CQE ?</div><div class='add'>+			 true : false;</div><div class='add'>+	cqe_size = cqe_64byte_ena ? 64 : 32;</div><div class='add'>+	ukinfo-&gt;avoid_mem_cflct = cqe_64byte_ena;</div><div class='add'>+	iwcq-&gt;ibcq.cqe = info.cq_uk_init_info.cq_size;</div><div class='add'>+	if (attr-&gt;comp_vector &lt; rf-&gt;ceqs_count)</div><div class='add'>+		info.ceq_id = attr-&gt;comp_vector;</div><div class='add'>+	info.ceq_id_valid = true;</div><div class='add'>+	info.ceqe_mask = 1;</div><div class='add'>+	info.type = IRDMA_CQ_TYPE_IWARP;</div><div class='add'>+	info.vsi = &amp;iwdev-&gt;vsi;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		struct irdma_ucontext *ucontext;</div><div class='add'>+		struct irdma_create_cq_req req = {};</div><div class='add'>+		struct irdma_cq_mr *cqmr;</div><div class='add'>+		struct irdma_pbl *iwpbl;</div><div class='add'>+		struct irdma_pbl *iwpbl_shadow;</div><div class='add'>+		struct irdma_cq_mr *cqmr_shadow;</div><div class='add'>+</div><div class='add'>+		iwcq-&gt;user_mode = true;</div><div class='add'>+		ucontext =</div><div class='add'>+			rdma_udata_to_drv_context(udata, struct irdma_ucontext,</div><div class='add'>+						  ibucontext);</div><div class='add'>+		if (ib_copy_from_udata(&amp;req, udata,</div><div class='add'>+				       min(sizeof(req), udata-&gt;inlen))) {</div><div class='add'>+			err_code = -EFAULT;</div><div class='add'>+			goto cq_free_rsrc;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		spin_lock_irqsave(&amp;ucontext-&gt;cq_reg_mem_list_lock, flags);</div><div class='add'>+		iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf,</div><div class='add'>+				      &amp;ucontext-&gt;cq_reg_mem_list);</div><div class='add'>+		spin_unlock_irqrestore(&amp;ucontext-&gt;cq_reg_mem_list_lock, flags);</div><div class='add'>+		if (!iwpbl) {</div><div class='add'>+			err_code = -EPROTO;</div><div class='add'>+			goto cq_free_rsrc;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		cqmr = &amp;iwpbl-&gt;cq_mr;</div><div class='add'>+</div><div class='add'>+		if (rf-&gt;sc_dev.hw_attrs.uk_attrs.feature_flags &amp;</div><div class='add'>+		    IRDMA_FEATURE_CQ_RESIZE &amp;&amp; !ucontext-&gt;legacy_mode) {</div><div class='add'>+			spin_lock_irqsave(&amp;ucontext-&gt;cq_reg_mem_list_lock, flags);</div><div class='add'>+			iwpbl_shadow = irdma_get_pbl(</div><div class='add'>+					(unsigned long)req.user_shadow_area,</div><div class='add'>+					&amp;ucontext-&gt;cq_reg_mem_list);</div><div class='add'>+			spin_unlock_irqrestore(&amp;ucontext-&gt;cq_reg_mem_list_lock, flags);</div><div class='add'>+</div><div class='add'>+			if (!iwpbl_shadow) {</div><div class='add'>+				err_code = -EPROTO;</div><div class='add'>+				goto cq_free_rsrc;</div><div class='add'>+			}</div><div class='add'>+			cqmr_shadow = &amp;iwpbl_shadow-&gt;cq_mr;</div><div class='add'>+			info.shadow_area_pa = cqmr_shadow-&gt;cq_pbl.addr;</div><div class='add'>+			cqmr-&gt;split = true;</div><div class='add'>+		} else {</div><div class='add'>+			info.shadow_area_pa = cqmr-&gt;shadow;</div><div class='add'>+		}</div><div class='add'>+		if (iwpbl-&gt;pbl_allocated) {</div><div class='add'>+			info.virtual_map = true;</div><div class='add'>+			info.pbl_chunk_size = 1;</div><div class='add'>+			info.first_pm_pbl_idx = cqmr-&gt;cq_pbl.idx;</div><div class='add'>+		} else {</div><div class='add'>+			info.cq_base_pa = cqmr-&gt;cq_pbl.addr;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		/* Kmode allocations */</div><div class='add'>+		int rsize;</div><div class='add'>+</div><div class='add'>+		if (entries &lt; 1 || entries &gt; rf-&gt;max_cqe) {</div><div class='add'>+			err_code = -EINVAL;</div><div class='add'>+			goto cq_free_rsrc;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		entries += 2;</div><div class='add'>+		if (!cqe_64byte_ena &amp;&amp; dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_2)</div><div class='add'>+			entries *= 2;</div><div class='add'>+</div><div class='add'>+		if (entries &amp; 1)</div><div class='add'>+			entries += 1; /* cq size must be an even number */</div><div class='add'>+</div><div class='add'>+		if (entries * cqe_size == IRDMA_HW_PAGE_SIZE)</div><div class='add'>+			entries += 2;</div><div class='add'>+</div><div class='add'>+		ukinfo-&gt;cq_size = entries;</div><div class='add'>+</div><div class='add'>+		if (cqe_64byte_ena)</div><div class='add'>+			rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_extended_cqe);</div><div class='add'>+		else</div><div class='add'>+			rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe);</div><div class='add'>+		iwcq-&gt;kmem.size = ALIGN(round_up(rsize, 256), 256);</div><div class='add'>+		iwcq-&gt;kmem.va = dma_alloc_coherent(dev-&gt;hw-&gt;device,</div><div class='add'>+						   iwcq-&gt;kmem.size,</div><div class='add'>+						   &amp;iwcq-&gt;kmem.pa, GFP_KERNEL);</div><div class='add'>+		if (!iwcq-&gt;kmem.va) {</div><div class='add'>+			err_code = -ENOMEM;</div><div class='add'>+			goto cq_free_rsrc;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		iwcq-&gt;kmem_shadow.size = ALIGN(IRDMA_SHADOW_AREA_SIZE &lt;&lt; 3,</div><div class='add'>+					       64);</div><div class='add'>+		iwcq-&gt;kmem_shadow.va = dma_alloc_coherent(dev-&gt;hw-&gt;device,</div><div class='add'>+							  iwcq-&gt;kmem_shadow.size,</div><div class='add'>+							  &amp;iwcq-&gt;kmem_shadow.pa,</div><div class='add'>+							  GFP_KERNEL);</div><div class='add'>+		if (!iwcq-&gt;kmem_shadow.va) {</div><div class='add'>+			err_code = -ENOMEM;</div><div class='add'>+			goto cq_free_rsrc;</div><div class='add'>+		}</div><div class='add'>+		info.shadow_area_pa = iwcq-&gt;kmem_shadow.pa;</div><div class='add'>+		ukinfo-&gt;shadow_area = iwcq-&gt;kmem_shadow.va;</div><div class='add'>+		ukinfo-&gt;cq_base = iwcq-&gt;kmem.va;</div><div class='add'>+		info.cq_base_pa = iwcq-&gt;kmem.pa;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2,</div><div class='add'>+					 (u32)IRDMA_MAX_CQ_READ_THRESH);</div><div class='add'>+</div><div class='add'>+	if (irdma_sc_cq_init(cq, &amp;info)) {</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev, "VERBS: init cq fail\n");</div><div class='add'>+		err_code = -EPROTO;</div><div class='add'>+		goto cq_free_rsrc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request) {</div><div class='add'>+		err_code = -ENOMEM;</div><div class='add'>+		goto cq_free_rsrc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_CQ_CREATE;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.cq_create.cq = cq;</div><div class='add'>+	cqp_info-&gt;in.u.cq_create.check_overflow = true;</div><div class='add'>+	cqp_info-&gt;in.u.cq_create.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	err_code = irdma_handle_cqp_op(rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;rf-&gt;cqp, cqp_request);</div><div class='add'>+	if (err_code)</div><div class='add'>+		goto cq_free_rsrc;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		struct irdma_create_cq_resp resp = {};</div><div class='add'>+</div><div class='add'>+		resp.cq_id = info.cq_uk_init_info.cq_id;</div><div class='add'>+		resp.cq_size = info.cq_uk_init_info.cq_size;</div><div class='add'>+		if (ib_copy_to_udata(udata, &amp;resp,</div><div class='add'>+				     min(sizeof(resp), udata-&gt;outlen))) {</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+				  "VERBS: copy to user data\n");</div><div class='add'>+			err_code = -EPROTO;</div><div class='add'>+			goto cq_destroy;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	rf-&gt;cq_table[cq_num] = iwcq;</div><div class='add'>+	init_completion(&amp;iwcq-&gt;free_cq);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+cq_destroy:</div><div class='add'>+	irdma_cq_wq_destroy(rf, cq);</div><div class='add'>+cq_free_rsrc:</div><div class='add'>+	irdma_cq_free_rsrc(rf, iwcq);</div><div class='add'>+</div><div class='add'>+	return err_code;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_mr_access - get hw MR access permissions from IB access flags</div><div class='add'>+ * @access: IB access flags</div><div class='add'>+ * @hw_rev: Hardware version</div><div class='add'>+ */</div><div class='add'>+static inline u16 irdma_get_mr_access(int access, u8 hw_rev)</div><div class='add'>+{</div><div class='add'>+	u16 hw_access = 0;</div><div class='add'>+</div><div class='add'>+	hw_access |= (access &amp; IB_ACCESS_LOCAL_WRITE) ?</div><div class='add'>+		     IRDMA_ACCESS_FLAGS_LOCALWRITE : 0;</div><div class='add'>+	hw_access |= (access &amp; IB_ACCESS_REMOTE_WRITE) ?</div><div class='add'>+		     IRDMA_ACCESS_FLAGS_REMOTEWRITE : 0;</div><div class='add'>+	hw_access |= (access &amp; IB_ACCESS_REMOTE_READ) ?</div><div class='add'>+		     IRDMA_ACCESS_FLAGS_REMOTEREAD : 0;</div><div class='add'>+	if (hw_rev &gt;= IRDMA_GEN_3) {</div><div class='add'>+		hw_access |= (access &amp; IB_ACCESS_MW_BIND) ?</div><div class='add'>+			     IRDMA_ACCESS_FLAGS_BIND_WINDOW : 0;</div><div class='add'>+	}</div><div class='add'>+	hw_access |= (access &amp; IB_ZERO_BASED) ?</div><div class='add'>+		     IRDMA_ACCESS_FLAGS_ZERO_BASED : 0;</div><div class='add'>+	hw_access |= IRDMA_ACCESS_FLAGS_LOCALREAD;</div><div class='add'>+</div><div class='add'>+	return hw_access;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_free_stag - free stag resource</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ * @stag: stag to free</div><div class='add'>+ */</div><div class='add'>+static void irdma_free_stag(struct irdma_device *iwdev, u32 stag)</div><div class='add'>+{</div><div class='add'>+	u32 stag_idx;</div><div class='add'>+</div><div class='add'>+	stag_idx = (stag &amp; iwdev-&gt;rf-&gt;mr_stagmask) &gt;&gt; IRDMA_CQPSQ_STAG_IDX_S;</div><div class='add'>+	irdma_free_rsrc(iwdev-&gt;rf, iwdev-&gt;rf-&gt;allocated_mrs, stag_idx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_create_stag - create random stag</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ */</div><div class='add'>+static u32 irdma_create_stag(struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	u32 stag = 0;</div><div class='add'>+	u32 stag_index = 0;</div><div class='add'>+	u32 next_stag_index;</div><div class='add'>+	u32 driver_key;</div><div class='add'>+	u32 random;</div><div class='add'>+	u8 consumer_key;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	get_random_bytes(&amp;random, sizeof(random));</div><div class='add'>+	consumer_key = (u8)random;</div><div class='add'>+</div><div class='add'>+	driver_key = random &amp; ~iwdev-&gt;rf-&gt;mr_stagmask;</div><div class='add'>+	next_stag_index = (random &amp; iwdev-&gt;rf-&gt;mr_stagmask) &gt;&gt; 8;</div><div class='add'>+	next_stag_index %= iwdev-&gt;rf-&gt;max_mr;</div><div class='add'>+</div><div class='add'>+	ret = irdma_alloc_rsrc(iwdev-&gt;rf, iwdev-&gt;rf-&gt;allocated_mrs,</div><div class='add'>+			       iwdev-&gt;rf-&gt;max_mr, &amp;stag_index,</div><div class='add'>+			       &amp;next_stag_index);</div><div class='add'>+	if (ret)</div><div class='add'>+		return stag;</div><div class='add'>+	stag = stag_index &lt;&lt; IRDMA_CQPSQ_STAG_IDX_S;</div><div class='add'>+	stag |= driver_key;</div><div class='add'>+	stag += (u32)consumer_key;</div><div class='add'>+</div><div class='add'>+	return stag;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_next_pbl_addr - Get next pbl address</div><div class='add'>+ * @pbl: pointer to a pble</div><div class='add'>+ * @pinfo: info pointer</div><div class='add'>+ * @idx: index</div><div class='add'>+ */</div><div class='add'>+static inline u64 *irdma_next_pbl_addr(u64 *pbl, struct irdma_pble_info **pinfo,</div><div class='add'>+				       u32 *idx)</div><div class='add'>+{</div><div class='add'>+	*idx += 1;</div><div class='add'>+	if (!(*pinfo) || *idx != (*pinfo)-&gt;cnt)</div><div class='add'>+		return ++pbl;</div><div class='add'>+	*idx = 0;</div><div class='add'>+	(*pinfo)++;</div><div class='add'>+</div><div class='add'>+	return (*pinfo)-&gt;addr;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_copy_user_pgaddrs - copy user page address to pble's os locally</div><div class='add'>+ * @iwmr: iwmr for IB's user page addresses</div><div class='add'>+ * @pbl: ple pointer to save 1 level or 0 level pble</div><div class='add'>+ * @level: indicated level 0, 1 or 2</div><div class='add'>+ */</div><div class='add'>+static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl,</div><div class='add'>+				    enum irdma_pble_level level)</div><div class='add'>+{</div><div class='add'>+	struct ib_umem *region = iwmr-&gt;region;</div><div class='add'>+	struct irdma_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='add'>+	struct irdma_pble_alloc *palloc = &amp;iwpbl-&gt;pble_alloc;</div><div class='add'>+	struct irdma_pble_info *pinfo;</div><div class='add'>+	struct ib_block_iter biter;</div><div class='add'>+	u32 idx = 0;</div><div class='add'>+	u32 pbl_cnt = 0;</div><div class='add'>+</div><div class='add'>+	pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc-&gt;level2.leaf;</div><div class='add'>+</div><div class='add'>+	if (iwmr-&gt;type == IRDMA_MEMREG_TYPE_QP)</div><div class='add'>+		iwpbl-&gt;qp_mr.sq_page = sg_page(region-&gt;sgt_append.sgt.sgl);</div><div class='add'>+</div><div class='add'>+	rdma_umem_for_each_dma_block(region, &amp;biter, iwmr-&gt;page_size) {</div><div class='add'>+		*pbl = rdma_block_iter_dma_address(&amp;biter);</div><div class='add'>+		if (++pbl_cnt == palloc-&gt;total_cnt)</div><div class='add'>+			break;</div><div class='add'>+		pbl = irdma_next_pbl_addr(pbl, &amp;pinfo, &amp;idx);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_check_mem_contiguous - check if pbls stored in arr are contiguous</div><div class='add'>+ * @arr: lvl1 pbl array</div><div class='add'>+ * @npages: page count</div><div class='add'>+ * @pg_size: page size</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size)</div><div class='add'>+{</div><div class='add'>+	u32 pg_idx;</div><div class='add'>+</div><div class='add'>+	for (pg_idx = 0; pg_idx &lt; npages; pg_idx++) {</div><div class='add'>+		if ((*arr + (pg_size * pg_idx)) != arr[pg_idx])</div><div class='add'>+			return false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_check_mr_contiguous - check if MR is physically contiguous</div><div class='add'>+ * @palloc: pbl allocation struct</div><div class='add'>+ * @pg_size: page size</div><div class='add'>+ */</div><div class='add'>+static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc,</div><div class='add'>+				      u32 pg_size)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pble_level2 *lvl2 = &amp;palloc-&gt;level2;</div><div class='add'>+	struct irdma_pble_info *leaf = lvl2-&gt;leaf;</div><div class='add'>+	u64 *arr = NULL;</div><div class='add'>+	u64 *start_addr = NULL;</div><div class='add'>+	int i;</div><div class='add'>+	bool ret;</div><div class='add'>+</div><div class='add'>+	if (palloc-&gt;level == PBLE_LEVEL_1) {</div><div class='add'>+		arr = palloc-&gt;level1.addr;</div><div class='add'>+		ret = irdma_check_mem_contiguous(arr, palloc-&gt;total_cnt,</div><div class='add'>+						 pg_size);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	start_addr = leaf-&gt;addr;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; lvl2-&gt;leaf_cnt; i++, leaf++) {</div><div class='add'>+		arr = leaf-&gt;addr;</div><div class='add'>+		if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr)</div><div class='add'>+			return false;</div><div class='add'>+		ret = irdma_check_mem_contiguous(arr, leaf-&gt;cnt, pg_size);</div><div class='add'>+		if (!ret)</div><div class='add'>+			return false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_setup_pbles - copy user pg address to pble's</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @iwmr: mr pointer for this memory registration</div><div class='add'>+ * @lvl: requested pble levels</div><div class='add'>+ */</div><div class='add'>+static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr,</div><div class='add'>+			     u8 lvl)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='add'>+	struct irdma_pble_alloc *palloc = &amp;iwpbl-&gt;pble_alloc;</div><div class='add'>+	struct irdma_pble_info *pinfo;</div><div class='add'>+	u64 *pbl;</div><div class='add'>+	int status;</div><div class='add'>+	enum irdma_pble_level level = PBLE_LEVEL_1;</div><div class='add'>+</div><div class='add'>+	if (lvl) {</div><div class='add'>+		status = irdma_get_pble(rf-&gt;pble_rsrc, palloc, iwmr-&gt;page_cnt,</div><div class='add'>+					lvl);</div><div class='add'>+		if (status)</div><div class='add'>+			return status;</div><div class='add'>+</div><div class='add'>+		iwpbl-&gt;pbl_allocated = true;</div><div class='add'>+		level = palloc-&gt;level;</div><div class='add'>+		pinfo = (level == PBLE_LEVEL_1) ? &amp;palloc-&gt;level1 :</div><div class='add'>+						  palloc-&gt;level2.leaf;</div><div class='add'>+		pbl = pinfo-&gt;addr;</div><div class='add'>+	} else {</div><div class='add'>+		pbl = iwmr-&gt;pgaddrmem;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_copy_user_pgaddrs(iwmr, pbl, level);</div><div class='add'>+</div><div class='add'>+	if (lvl)</div><div class='add'>+		iwmr-&gt;pgaddrmem[0] = *pbl;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_handle_q_mem - handle memory for qp and cq</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ * @req: information for q memory management</div><div class='add'>+ * @iwpbl: pble struct</div><div class='add'>+ * @lvl: pble level mask</div><div class='add'>+ */</div><div class='add'>+static int irdma_handle_q_mem(struct irdma_device *iwdev,</div><div class='add'>+			      struct irdma_mem_reg_req *req,</div><div class='add'>+			      struct irdma_pbl *iwpbl, u8 lvl)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pble_alloc *palloc = &amp;iwpbl-&gt;pble_alloc;</div><div class='add'>+	struct irdma_mr *iwmr = iwpbl-&gt;iwmr;</div><div class='add'>+	struct irdma_qp_mr *qpmr = &amp;iwpbl-&gt;qp_mr;</div><div class='add'>+	struct irdma_cq_mr *cqmr = &amp;iwpbl-&gt;cq_mr;</div><div class='add'>+	struct irdma_srq_mr *srqmr = &amp;iwpbl-&gt;srq_mr;</div><div class='add'>+	struct irdma_hmc_pble *hmc_p;</div><div class='add'>+	u64 *arr = iwmr-&gt;pgaddrmem;</div><div class='add'>+	u32 pg_size, total;</div><div class='add'>+	int err = 0;</div><div class='add'>+	bool ret = true;</div><div class='add'>+</div><div class='add'>+	pg_size = iwmr-&gt;page_size;</div><div class='add'>+	err = irdma_setup_pbles(iwdev-&gt;rf, iwmr, lvl);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	if (lvl)</div><div class='add'>+		arr = palloc-&gt;level1.addr;</div><div class='add'>+</div><div class='add'>+	switch (iwmr-&gt;type) {</div><div class='add'>+	case IRDMA_MEMREG_TYPE_QP:</div><div class='add'>+		total = req-&gt;sq_pages + req-&gt;rq_pages;</div><div class='add'>+		hmc_p = &amp;qpmr-&gt;sq_pbl;</div><div class='add'>+		qpmr-&gt;shadow = (dma_addr_t)arr[total];</div><div class='add'>+		/* Need to use physical address for RQ of QP</div><div class='add'>+		 * in case it is associated with SRQ.</div><div class='add'>+		 */</div><div class='add'>+		qpmr-&gt;rq_pa = (dma_addr_t)arr[req-&gt;sq_pages];</div><div class='add'>+		if (lvl) {</div><div class='add'>+			ret = irdma_check_mem_contiguous(arr, req-&gt;sq_pages,</div><div class='add'>+							 pg_size);</div><div class='add'>+			if (ret)</div><div class='add'>+				ret = irdma_check_mem_contiguous(&amp;arr[req-&gt;sq_pages],</div><div class='add'>+								 req-&gt;rq_pages,</div><div class='add'>+								 pg_size);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (!ret) {</div><div class='add'>+			hmc_p-&gt;idx = palloc-&gt;level1.idx;</div><div class='add'>+			hmc_p = &amp;qpmr-&gt;rq_pbl;</div><div class='add'>+			hmc_p-&gt;idx = palloc-&gt;level1.idx + req-&gt;sq_pages;</div><div class='add'>+		} else {</div><div class='add'>+			hmc_p-&gt;addr = arr[0];</div><div class='add'>+			hmc_p = &amp;qpmr-&gt;rq_pbl;</div><div class='add'>+			hmc_p-&gt;addr = arr[req-&gt;sq_pages];</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_MEMREG_TYPE_SRQ:</div><div class='add'>+		hmc_p = &amp;srqmr-&gt;srq_pbl;</div><div class='add'>+		srqmr-&gt;shadow = (dma_addr_t)arr[req-&gt;rq_pages];</div><div class='add'>+		if (lvl)</div><div class='add'>+			ret = irdma_check_mem_contiguous(arr, req-&gt;rq_pages,</div><div class='add'>+							 pg_size);</div><div class='add'>+</div><div class='add'>+		if (!ret)</div><div class='add'>+			hmc_p-&gt;idx = palloc-&gt;level1.idx;</div><div class='add'>+		else</div><div class='add'>+			hmc_p-&gt;addr = arr[0];</div><div class='add'>+	break;</div><div class='add'>+	case IRDMA_MEMREG_TYPE_CQ:</div><div class='add'>+		hmc_p = &amp;cqmr-&gt;cq_pbl;</div><div class='add'>+</div><div class='add'>+		if (!cqmr-&gt;split)</div><div class='add'>+			cqmr-&gt;shadow = (dma_addr_t)arr[req-&gt;cq_pages];</div><div class='add'>+</div><div class='add'>+		if (lvl)</div><div class='add'>+			ret = irdma_check_mem_contiguous(arr, req-&gt;cq_pages,</div><div class='add'>+							 pg_size);</div><div class='add'>+</div><div class='add'>+		if (!ret)</div><div class='add'>+			hmc_p-&gt;idx = palloc-&gt;level1.idx;</div><div class='add'>+		else</div><div class='add'>+			hmc_p-&gt;addr = arr[0];</div><div class='add'>+	break;</div><div class='add'>+	default:</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev, "VERBS: MR type error\n");</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (lvl &amp;&amp; ret) {</div><div class='add'>+		irdma_free_pble(iwdev-&gt;rf-&gt;pble_rsrc, palloc);</div><div class='add'>+		iwpbl-&gt;pbl_allocated = false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_hw_alloc_mw - create the hw memory window</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ * @iwmr: pointer to memory window info</div><div class='add'>+ */</div><div class='add'>+static int irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_mw_alloc_info *info;</div><div class='add'>+	struct irdma_pd *iwpd = to_iwpd(iwmr-&gt;ibmr.pd);</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;iwdev-&gt;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	info = &amp;cqp_info-&gt;in.u.mw_alloc.info;</div><div class='add'>+	memset(info, 0, sizeof(*info));</div><div class='add'>+	if (iwmr-&gt;ibmw.type == IB_MW_TYPE_1)</div><div class='add'>+		info-&gt;mw_wide = true;</div><div class='add'>+</div><div class='add'>+	info-&gt;page_size = PAGE_SIZE;</div><div class='add'>+	info-&gt;mw_stag_index = iwmr-&gt;stag &gt;&gt; IRDMA_CQPSQ_STAG_IDX_S;</div><div class='add'>+	info-&gt;pd_id = iwpd-&gt;sc_pd.pd_id;</div><div class='add'>+	info-&gt;remote_access = true;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_MW_ALLOC;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.mw_alloc.dev = &amp;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+	cqp_info-&gt;in.u.mw_alloc.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	status = irdma_handle_cqp_op(iwdev-&gt;rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;iwdev-&gt;rf-&gt;cqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_alloc_mw - Allocate memory window</div><div class='add'>+ * @ibmw: Memory Window</div><div class='add'>+ * @udata: user data pointer</div><div class='add'>+ */</div><div class='add'>+static int irdma_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibmw-&gt;device);</div><div class='add'>+	struct irdma_mr *iwmr = to_iwmw(ibmw);</div><div class='add'>+	int err_code;</div><div class='add'>+	u32 stag;</div><div class='add'>+</div><div class='add'>+	stag = irdma_create_stag(iwdev);</div><div class='add'>+	if (!stag)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	iwmr-&gt;stag = stag;</div><div class='add'>+	ibmw-&gt;rkey = stag;</div><div class='add'>+</div><div class='add'>+	err_code = irdma_hw_alloc_mw(iwdev, iwmr);</div><div class='add'>+	if (err_code) {</div><div class='add'>+		irdma_free_stag(iwdev, stag);</div><div class='add'>+		return err_code;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_dealloc_mw - Dealloc memory window</div><div class='add'>+ * @ibmw: memory window structure.</div><div class='add'>+ */</div><div class='add'>+static int irdma_dealloc_mw(struct ib_mw *ibmw)</div><div class='add'>+{</div><div class='add'>+	struct ib_pd *ibpd = ibmw-&gt;pd;</div><div class='add'>+	struct irdma_pd *iwpd = to_iwpd(ibpd);</div><div class='add'>+	struct irdma_mr *iwmr = to_iwmr((struct ib_mr *)ibmw);</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibmw-&gt;device);</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	struct irdma_dealloc_stag_info *info;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;iwdev-&gt;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	info = &amp;cqp_info-&gt;in.u.dealloc_stag.info;</div><div class='add'>+	memset(info, 0, sizeof(*info));</div><div class='add'>+	info-&gt;pd_id = iwpd-&gt;sc_pd.pd_id;</div><div class='add'>+	info-&gt;stag_idx = ibmw-&gt;rkey &gt;&gt; IRDMA_CQPSQ_STAG_IDX_S;</div><div class='add'>+	info-&gt;mr = false;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_DEALLOC_STAG;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.dealloc_stag.dev = &amp;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+	cqp_info-&gt;in.u.dealloc_stag.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	irdma_handle_cqp_op(iwdev-&gt;rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;iwdev-&gt;rf-&gt;cqp, cqp_request);</div><div class='add'>+	irdma_free_stag(iwdev, iwmr-&gt;stag);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_hw_alloc_stag - cqp command to allocate stag</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ * @iwmr: irdma mr pointer</div><div class='add'>+ */</div><div class='add'>+static int irdma_hw_alloc_stag(struct irdma_device *iwdev,</div><div class='add'>+			       struct irdma_mr *iwmr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_allocate_stag_info *info;</div><div class='add'>+	struct ib_pd *pd = iwmr-&gt;ibmr.pd;</div><div class='add'>+	struct irdma_pd *iwpd = to_iwpd(pd);</div><div class='add'>+	int status;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;iwdev-&gt;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	info = &amp;cqp_info-&gt;in.u.alloc_stag.info;</div><div class='add'>+	info-&gt;page_size = PAGE_SIZE;</div><div class='add'>+	info-&gt;stag_idx = iwmr-&gt;stag &gt;&gt; IRDMA_CQPSQ_STAG_IDX_S;</div><div class='add'>+	info-&gt;pd_id = iwpd-&gt;sc_pd.pd_id;</div><div class='add'>+	info-&gt;total_len = iwmr-&gt;len;</div><div class='add'>+	info-&gt;remote_access = true;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_ALLOC_STAG;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.alloc_stag.dev = &amp;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+	cqp_info-&gt;in.u.alloc_stag.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	status = irdma_handle_cqp_op(iwdev-&gt;rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;iwdev-&gt;rf-&gt;cqp, cqp_request);</div><div class='add'>+	if (status)</div><div class='add'>+		return status;</div><div class='add'>+</div><div class='add'>+	iwmr-&gt;is_hwreg = true;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_alloc_mr - register stag for fast memory registration</div><div class='add'>+ * @pd: ibpd pointer</div><div class='add'>+ * @mr_type: memory for stag registrion</div><div class='add'>+ * @max_num_sg: man number of pages</div><div class='add'>+ */</div><div class='add'>+static struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,</div><div class='add'>+				    u32 max_num_sg)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(pd-&gt;device);</div><div class='add'>+	struct irdma_pble_alloc *palloc;</div><div class='add'>+	struct irdma_pbl *iwpbl;</div><div class='add'>+	struct irdma_mr *iwmr;</div><div class='add'>+	u32 stag;</div><div class='add'>+	int err_code;</div><div class='add'>+</div><div class='add'>+	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);</div><div class='add'>+	if (!iwmr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	stag = irdma_create_stag(iwdev);</div><div class='add'>+	if (!stag) {</div><div class='add'>+		err_code = -ENOMEM;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	iwmr-&gt;stag = stag;</div><div class='add'>+	iwmr-&gt;ibmr.rkey = stag;</div><div class='add'>+	iwmr-&gt;ibmr.lkey = stag;</div><div class='add'>+	iwmr-&gt;ibmr.pd = pd;</div><div class='add'>+	iwmr-&gt;ibmr.device = pd-&gt;device;</div><div class='add'>+	iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='add'>+	iwpbl-&gt;iwmr = iwmr;</div><div class='add'>+	iwmr-&gt;type = IRDMA_MEMREG_TYPE_MEM;</div><div class='add'>+	palloc = &amp;iwpbl-&gt;pble_alloc;</div><div class='add'>+	iwmr-&gt;page_cnt = max_num_sg;</div><div class='add'>+	/* Use system PAGE_SIZE as the sg page sizes are unknown at this point */</div><div class='add'>+	iwmr-&gt;len = max_num_sg * PAGE_SIZE;</div><div class='add'>+	err_code = irdma_get_pble(iwdev-&gt;rf-&gt;pble_rsrc, palloc, iwmr-&gt;page_cnt,</div><div class='add'>+				  false);</div><div class='add'>+	if (err_code)</div><div class='add'>+		goto err_get_pble;</div><div class='add'>+</div><div class='add'>+	err_code = irdma_hw_alloc_stag(iwdev, iwmr);</div><div class='add'>+	if (err_code)</div><div class='add'>+		goto err_alloc_stag;</div><div class='add'>+</div><div class='add'>+	iwpbl-&gt;pbl_allocated = true;</div><div class='add'>+</div><div class='add'>+	return &amp;iwmr-&gt;ibmr;</div><div class='add'>+err_alloc_stag:</div><div class='add'>+	irdma_free_pble(iwdev-&gt;rf-&gt;pble_rsrc, palloc);</div><div class='add'>+err_get_pble:</div><div class='add'>+	irdma_free_stag(iwdev, stag);</div><div class='add'>+err:</div><div class='add'>+	kfree(iwmr);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(err_code);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_set_page - populate pbl list for fmr</div><div class='add'>+ * @ibmr: ib mem to access iwarp mr pointer</div><div class='add'>+ * @addr: page dma address fro pbl list</div><div class='add'>+ */</div><div class='add'>+static int irdma_set_page(struct ib_mr *ibmr, u64 addr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_mr *iwmr = to_iwmr(ibmr);</div><div class='add'>+	struct irdma_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='add'>+	struct irdma_pble_alloc *palloc = &amp;iwpbl-&gt;pble_alloc;</div><div class='add'>+	u64 *pbl;</div><div class='add'>+</div><div class='add'>+	if (unlikely(iwmr-&gt;npages == iwmr-&gt;page_cnt))</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	if (palloc-&gt;level == PBLE_LEVEL_2) {</div><div class='add'>+		struct irdma_pble_info *palloc_info =</div><div class='add'>+			palloc-&gt;level2.leaf + (iwmr-&gt;npages &gt;&gt; PBLE_512_SHIFT);</div><div class='add'>+</div><div class='add'>+		palloc_info-&gt;addr[iwmr-&gt;npages &amp; (PBLE_PER_PAGE - 1)] = addr;</div><div class='add'>+	} else {</div><div class='add'>+		pbl = palloc-&gt;level1.addr;</div><div class='add'>+		pbl[iwmr-&gt;npages] = addr;</div><div class='add'>+	}</div><div class='add'>+	iwmr-&gt;npages++;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_map_mr_sg - map of sg list for fmr</div><div class='add'>+ * @ibmr: ib mem to access iwarp mr pointer</div><div class='add'>+ * @sg: scatter gather list</div><div class='add'>+ * @sg_nents: number of sg pages</div><div class='add'>+ * @sg_offset: scatter gather list for fmr</div><div class='add'>+ */</div><div class='add'>+static int irdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,</div><div class='add'>+			   int sg_nents, unsigned int *sg_offset)</div><div class='add'>+{</div><div class='add'>+	struct irdma_mr *iwmr = to_iwmr(ibmr);</div><div class='add'>+</div><div class='add'>+	iwmr-&gt;npages = 0;</div><div class='add'>+</div><div class='add'>+	return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, irdma_set_page);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_hwreg_mr - send cqp command for memory registration</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ * @iwmr: irdma mr pointer</div><div class='add'>+ * @access: access for MR</div><div class='add'>+ */</div><div class='add'>+static int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr,</div><div class='add'>+			  u16 access)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='add'>+	struct irdma_reg_ns_stag_info *stag_info;</div><div class='add'>+	struct ib_pd *pd = iwmr-&gt;ibmr.pd;</div><div class='add'>+	struct irdma_pd *iwpd = to_iwpd(pd);</div><div class='add'>+	struct irdma_pble_alloc *palloc = &amp;iwpbl-&gt;pble_alloc;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;iwdev-&gt;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	stag_info = &amp;cqp_info-&gt;in.u.mr_reg_non_shared.info;</div><div class='add'>+	stag_info-&gt;va = iwpbl-&gt;user_base;</div><div class='add'>+	stag_info-&gt;stag_idx = iwmr-&gt;stag &gt;&gt; IRDMA_CQPSQ_STAG_IDX_S;</div><div class='add'>+	stag_info-&gt;stag_key = (u8)iwmr-&gt;stag;</div><div class='add'>+	stag_info-&gt;total_len = iwmr-&gt;len;</div><div class='add'>+	stag_info-&gt;access_rights = irdma_get_mr_access(access,</div><div class='add'>+						       iwdev-&gt;rf-&gt;sc_dev.hw_attrs.uk_attrs.hw_rev);</div><div class='add'>+	if (iwdev-&gt;rf-&gt;sc_dev.hw_attrs.uk_attrs.feature_flags &amp; IRDMA_FEATURE_ATOMIC_OPS)</div><div class='add'>+		stag_info-&gt;remote_atomics_en = (access &amp; IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;</div><div class='add'>+	stag_info-&gt;pd_id = iwpd-&gt;sc_pd.pd_id;</div><div class='add'>+	stag_info-&gt;all_memory = iwmr-&gt;dma_mr;</div><div class='add'>+	if (stag_info-&gt;access_rights &amp; IRDMA_ACCESS_FLAGS_ZERO_BASED)</div><div class='add'>+		stag_info-&gt;addr_type = IRDMA_ADDR_TYPE_ZERO_BASED;</div><div class='add'>+	else</div><div class='add'>+		stag_info-&gt;addr_type = IRDMA_ADDR_TYPE_VA_BASED;</div><div class='add'>+	stag_info-&gt;page_size = iwmr-&gt;page_size;</div><div class='add'>+</div><div class='add'>+	if (iwpbl-&gt;pbl_allocated) {</div><div class='add'>+		if (palloc-&gt;level == PBLE_LEVEL_1) {</div><div class='add'>+			stag_info-&gt;first_pm_pbl_index = palloc-&gt;level1.idx;</div><div class='add'>+			stag_info-&gt;chunk_size = 1;</div><div class='add'>+		} else {</div><div class='add'>+			stag_info-&gt;first_pm_pbl_index = palloc-&gt;level2.root.idx;</div><div class='add'>+			stag_info-&gt;chunk_size = 3;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		stag_info-&gt;reg_addr_pa = iwmr-&gt;pgaddrmem[0];</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_MR_REG_NON_SHARED;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.mr_reg_non_shared.dev = &amp;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+	cqp_info-&gt;in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	ret = irdma_handle_cqp_op(iwdev-&gt;rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;iwdev-&gt;rf-&gt;cqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	if (!ret)</div><div class='add'>+		iwmr-&gt;is_hwreg = true;</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access,</div><div class='add'>+				      bool create_stag)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(iwmr-&gt;ibmr.device);</div><div class='add'>+	struct irdma_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='add'>+	u32 stag = 0;</div><div class='add'>+	u8 lvl;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	lvl = iwmr-&gt;page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0;</div><div class='add'>+</div><div class='add'>+	err = irdma_setup_pbles(iwdev-&gt;rf, iwmr, lvl);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	if (lvl) {</div><div class='add'>+		err = irdma_check_mr_contiguous(&amp;iwpbl-&gt;pble_alloc,</div><div class='add'>+						iwmr-&gt;page_size);</div><div class='add'>+		if (err) {</div><div class='add'>+			irdma_free_pble(iwdev-&gt;rf-&gt;pble_rsrc, &amp;iwpbl-&gt;pble_alloc);</div><div class='add'>+			iwpbl-&gt;pbl_allocated = false;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (create_stag) {</div><div class='add'>+		stag = irdma_create_stag(iwdev);</div><div class='add'>+		if (!stag) {</div><div class='add'>+			err = -ENOMEM;</div><div class='add'>+			goto free_pble;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		iwmr-&gt;stag = stag;</div><div class='add'>+		iwmr-&gt;ibmr.rkey = stag;</div><div class='add'>+		iwmr-&gt;ibmr.lkey = stag;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = irdma_hwreg_mr(iwdev, iwmr, access);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_hwreg;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_hwreg:</div><div class='add'>+	if (stag)</div><div class='add'>+		irdma_free_stag(iwdev, stag);</div><div class='add'>+</div><div class='add'>+free_pble:</div><div class='add'>+	if (iwpbl-&gt;pble_alloc.level != PBLE_LEVEL_0 &amp;&amp; iwpbl-&gt;pbl_allocated)</div><div class='add'>+		irdma_free_pble(iwdev-&gt;rf-&gt;pble_rsrc, &amp;iwpbl-&gt;pble_alloc);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region,</div><div class='add'>+					 struct ib_pd *pd, u64 virt,</div><div class='add'>+					 enum irdma_memreg_type reg_type)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(pd-&gt;device);</div><div class='add'>+	struct irdma_pbl *iwpbl;</div><div class='add'>+	struct irdma_mr *iwmr;</div><div class='add'>+	unsigned long pgsz_bitmap;</div><div class='add'>+</div><div class='add'>+	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);</div><div class='add'>+	if (!iwmr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='add'>+	iwpbl-&gt;iwmr = iwmr;</div><div class='add'>+	iwmr-&gt;region = region;</div><div class='add'>+	iwmr-&gt;ibmr.pd = pd;</div><div class='add'>+	iwmr-&gt;ibmr.device = pd-&gt;device;</div><div class='add'>+	iwmr-&gt;ibmr.iova = virt;</div><div class='add'>+	iwmr-&gt;type = reg_type;</div><div class='add'>+</div><div class='add'>+	pgsz_bitmap = (reg_type == IRDMA_MEMREG_TYPE_MEM) ?</div><div class='add'>+		iwdev-&gt;rf-&gt;sc_dev.hw_attrs.page_size_cap : SZ_4K;</div><div class='add'>+</div><div class='add'>+	iwmr-&gt;page_size = ib_umem_find_best_pgsz(region, pgsz_bitmap, virt);</div><div class='add'>+	if (unlikely(!iwmr-&gt;page_size)) {</div><div class='add'>+		kfree(iwmr);</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	iwmr-&gt;len = region-&gt;length;</div><div class='add'>+	iwpbl-&gt;user_base = virt;</div><div class='add'>+	iwmr-&gt;page_cnt = ib_umem_num_dma_blocks(region, iwmr-&gt;page_size);</div><div class='add'>+</div><div class='add'>+	return iwmr;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_free_iwmr(struct irdma_mr *iwmr)</div><div class='add'>+{</div><div class='add'>+	kfree(iwmr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_reg_user_mr_type_qp(struct irdma_mem_reg_req req,</div><div class='add'>+				     struct ib_udata *udata,</div><div class='add'>+				     struct irdma_mr *iwmr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(iwmr-&gt;ibmr.device);</div><div class='add'>+	struct irdma_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='add'>+	struct irdma_ucontext *ucontext = NULL;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	u32 total;</div><div class='add'>+	int err;</div><div class='add'>+	u8 lvl;</div><div class='add'>+</div><div class='add'>+	/* iWarp: Catch page not starting on OS page boundary */</div><div class='add'>+	if (!rdma_protocol_roce(&amp;iwdev-&gt;ibdev, 1) &amp;&amp;</div><div class='add'>+	    ib_umem_offset(iwmr-&gt;region))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	total = req.sq_pages + req.rq_pages + 1;</div><div class='add'>+	if (total &gt; iwmr-&gt;page_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	total = req.sq_pages + req.rq_pages;</div><div class='add'>+	lvl = total &gt; 2 ? PBLE_LEVEL_1 : PBLE_LEVEL_0;</div><div class='add'>+	err = irdma_handle_q_mem(iwdev, &amp;req, iwpbl, lvl);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,</div><div class='add'>+					     ibucontext);</div><div class='add'>+	spin_lock_irqsave(&amp;ucontext-&gt;qp_reg_mem_list_lock, flags);</div><div class='add'>+	list_add_tail(&amp;iwpbl-&gt;list, &amp;ucontext-&gt;qp_reg_mem_list);</div><div class='add'>+	iwpbl-&gt;on_list = true;</div><div class='add'>+	spin_unlock_irqrestore(&amp;ucontext-&gt;qp_reg_mem_list_lock, flags);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_reg_user_mr_type_srq(struct irdma_mem_reg_req req,</div><div class='add'>+				      struct ib_udata *udata,</div><div class='add'>+				      struct irdma_mr *iwmr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(iwmr-&gt;ibmr.device);</div><div class='add'>+	struct irdma_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='add'>+	struct irdma_ucontext *ucontext;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	u32 total;</div><div class='add'>+	int err;</div><div class='add'>+	u8 lvl;</div><div class='add'>+</div><div class='add'>+	total = req.rq_pages + IRDMA_SHADOW_PGCNT;</div><div class='add'>+	if (total &gt; iwmr-&gt;page_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	lvl = req.rq_pages &gt; 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0;</div><div class='add'>+	err = irdma_handle_q_mem(iwdev, &amp;req, iwpbl, lvl);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,</div><div class='add'>+					     ibucontext);</div><div class='add'>+	spin_lock_irqsave(&amp;ucontext-&gt;srq_reg_mem_list_lock, flags);</div><div class='add'>+	list_add_tail(&amp;iwpbl-&gt;list, &amp;ucontext-&gt;srq_reg_mem_list);</div><div class='add'>+	iwpbl-&gt;on_list = true;</div><div class='add'>+	spin_unlock_irqrestore(&amp;ucontext-&gt;srq_reg_mem_list_lock, flags);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_reg_user_mr_type_cq(struct irdma_mem_reg_req req,</div><div class='add'>+				     struct ib_udata *udata,</div><div class='add'>+				     struct irdma_mr *iwmr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(iwmr-&gt;ibmr.device);</div><div class='add'>+	struct irdma_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='add'>+	struct irdma_ucontext *ucontext = NULL;</div><div class='add'>+	u8 shadow_pgcnt = 1;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	u32 total;</div><div class='add'>+	int err;</div><div class='add'>+	u8 lvl;</div><div class='add'>+</div><div class='add'>+	if (iwdev-&gt;rf-&gt;sc_dev.hw_attrs.uk_attrs.feature_flags &amp; IRDMA_FEATURE_CQ_RESIZE)</div><div class='add'>+		shadow_pgcnt = 0;</div><div class='add'>+	total = req.cq_pages + shadow_pgcnt;</div><div class='add'>+	if (total &gt; iwmr-&gt;page_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	lvl = req.cq_pages &gt; 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0;</div><div class='add'>+	err = irdma_handle_q_mem(iwdev, &amp;req, iwpbl, lvl);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,</div><div class='add'>+					     ibucontext);</div><div class='add'>+	spin_lock_irqsave(&amp;ucontext-&gt;cq_reg_mem_list_lock, flags);</div><div class='add'>+	list_add_tail(&amp;iwpbl-&gt;list, &amp;ucontext-&gt;cq_reg_mem_list);</div><div class='add'>+	iwpbl-&gt;on_list = true;</div><div class='add'>+	spin_unlock_irqrestore(&amp;ucontext-&gt;cq_reg_mem_list_lock, flags);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_reg_user_mr - Register a user memory region</div><div class='add'>+ * @pd: ptr of pd</div><div class='add'>+ * @start: virtual start address</div><div class='add'>+ * @len: length of mr</div><div class='add'>+ * @virt: virtual address</div><div class='add'>+ * @access: access of mr</div><div class='add'>+ * @dmah: dma handle</div><div class='add'>+ * @udata: user data</div><div class='add'>+ */</div><div class='add'>+static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,</div><div class='add'>+				       u64 virt, int access,</div><div class='add'>+				       struct ib_dmah *dmah,</div><div class='add'>+				       struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+#define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages)</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(pd-&gt;device);</div><div class='add'>+	struct irdma_mem_reg_req req = {};</div><div class='add'>+	struct ib_umem *region = NULL;</div><div class='add'>+	struct irdma_mr *iwmr = NULL;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (dmah)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	if (len &gt; iwdev-&gt;rf-&gt;sc_dev.hw_attrs.max_mr_size)</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	if (udata-&gt;inlen &lt; IRDMA_MEM_REG_MIN_REQ_LEN)</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	region = ib_umem_get(pd-&gt;device, start, len, access);</div><div class='add'>+</div><div class='add'>+	if (IS_ERR(region)) {</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+			  "VERBS: Failed to create ib_umem region\n");</div><div class='add'>+		return (struct ib_mr *)region;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ib_copy_from_udata(&amp;req, udata, min(sizeof(req), udata-&gt;inlen))) {</div><div class='add'>+		ib_umem_release(region);</div><div class='add'>+		return ERR_PTR(-EFAULT);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	iwmr = irdma_alloc_iwmr(region, pd, virt, req.reg_type);</div><div class='add'>+	if (IS_ERR(iwmr)) {</div><div class='add'>+		ib_umem_release(region);</div><div class='add'>+		return (struct ib_mr *)iwmr;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (req.reg_type) {</div><div class='add'>+	case IRDMA_MEMREG_TYPE_QP:</div><div class='add'>+		err = irdma_reg_user_mr_type_qp(req, udata, iwmr);</div><div class='add'>+		if (err)</div><div class='add'>+			goto error;</div><div class='add'>+</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_MEMREG_TYPE_SRQ:</div><div class='add'>+		err = irdma_reg_user_mr_type_srq(req, udata, iwmr);</div><div class='add'>+		if (err)</div><div class='add'>+			goto error;</div><div class='add'>+</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_MEMREG_TYPE_CQ:</div><div class='add'>+		err = irdma_reg_user_mr_type_cq(req, udata, iwmr);</div><div class='add'>+		if (err)</div><div class='add'>+			goto error;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_MEMREG_TYPE_MEM:</div><div class='add'>+		err = irdma_reg_user_mr_type_mem(iwmr, access, true);</div><div class='add'>+		if (err)</div><div class='add'>+			goto error;</div><div class='add'>+</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return &amp;iwmr-&gt;ibmr;</div><div class='add'>+error:</div><div class='add'>+	ib_umem_release(region);</div><div class='add'>+	irdma_free_iwmr(iwmr);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(err);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start,</div><div class='add'>+					      u64 len, u64 virt,</div><div class='add'>+					      int fd, int access,</div><div class='add'>+					      struct ib_dmah *dmah,</div><div class='add'>+					      struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(pd-&gt;device);</div><div class='add'>+	struct ib_umem_dmabuf *umem_dmabuf;</div><div class='add'>+	struct irdma_mr *iwmr;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (dmah)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	if (len &gt; iwdev-&gt;rf-&gt;sc_dev.hw_attrs.max_mr_size)</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	umem_dmabuf = ib_umem_dmabuf_get_pinned(pd-&gt;device, start, len, fd, access);</div><div class='add'>+	if (IS_ERR(umem_dmabuf)) {</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev, "Failed to get dmabuf umem[%pe]\n",</div><div class='add'>+			  umem_dmabuf);</div><div class='add'>+		return ERR_CAST(umem_dmabuf);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	iwmr = irdma_alloc_iwmr(&amp;umem_dmabuf-&gt;umem, pd, virt, IRDMA_MEMREG_TYPE_MEM);</div><div class='add'>+	if (IS_ERR(iwmr)) {</div><div class='add'>+		err = PTR_ERR(iwmr);</div><div class='add'>+		goto err_release;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = irdma_reg_user_mr_type_mem(iwmr, access, true);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_iwmr;</div><div class='add'>+</div><div class='add'>+	return &amp;iwmr-&gt;ibmr;</div><div class='add'>+</div><div class='add'>+err_iwmr:</div><div class='add'>+	irdma_free_iwmr(iwmr);</div><div class='add'>+</div><div class='add'>+err_release:</div><div class='add'>+	ib_umem_release(&amp;umem_dmabuf-&gt;umem);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(err);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_hwdereg_mr(struct ib_mr *ib_mr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ib_mr-&gt;device);</div><div class='add'>+	struct irdma_mr *iwmr = to_iwmr(ib_mr);</div><div class='add'>+	struct irdma_pd *iwpd = to_iwpd(ib_mr-&gt;pd);</div><div class='add'>+	struct irdma_dealloc_stag_info *info;</div><div class='add'>+	struct irdma_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	/* Skip HW MR de-register when it is already de-registered</div><div class='add'>+	 * during an MR re-reregister and the re-registration fails</div><div class='add'>+	 */</div><div class='add'>+	if (!iwmr-&gt;is_hwreg)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;iwdev-&gt;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	info = &amp;cqp_info-&gt;in.u.dealloc_stag.info;</div><div class='add'>+	info-&gt;pd_id = iwpd-&gt;sc_pd.pd_id;</div><div class='add'>+	info-&gt;stag_idx = ib_mr-&gt;rkey &gt;&gt; IRDMA_CQPSQ_STAG_IDX_S;</div><div class='add'>+	info-&gt;mr = true;</div><div class='add'>+	if (iwpbl-&gt;pbl_allocated)</div><div class='add'>+		info-&gt;dealloc_pbl = true;</div><div class='add'>+</div><div class='add'>+	cqp_info-&gt;cqp_cmd = IRDMA_OP_DEALLOC_STAG;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.dealloc_stag.dev = &amp;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+	cqp_info-&gt;in.u.dealloc_stag.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	status = irdma_handle_cqp_op(iwdev-&gt;rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;iwdev-&gt;rf-&gt;cqp, cqp_request);</div><div class='add'>+	if (status)</div><div class='add'>+		return status;</div><div class='add'>+</div><div class='add'>+	iwmr-&gt;is_hwreg = false;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * irdma_rereg_mr_trans - Re-register a user MR for a change translation.</div><div class='add'>+ * @iwmr: ptr of iwmr</div><div class='add'>+ * @start: virtual start address</div><div class='add'>+ * @len: length of mr</div><div class='add'>+ * @virt: virtual address</div><div class='add'>+ *</div><div class='add'>+ * Re-register a user memory region when a change translation is requested.</div><div class='add'>+ * Re-register a new region while reusing the stag from the original registration.</div><div class='add'>+ */</div><div class='add'>+static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len,</div><div class='add'>+				u64 virt)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(iwmr-&gt;ibmr.device);</div><div class='add'>+	struct irdma_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='add'>+	struct ib_pd *pd = iwmr-&gt;ibmr.pd;</div><div class='add'>+	struct ib_umem *region;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	region = ib_umem_get(pd-&gt;device, start, len, iwmr-&gt;access);</div><div class='add'>+	if (IS_ERR(region))</div><div class='add'>+		return PTR_ERR(region);</div><div class='add'>+</div><div class='add'>+	iwmr-&gt;region = region;</div><div class='add'>+	iwmr-&gt;ibmr.iova = virt;</div><div class='add'>+	iwmr-&gt;ibmr.pd = pd;</div><div class='add'>+	iwmr-&gt;page_size = ib_umem_find_best_pgsz(region,</div><div class='add'>+				iwdev-&gt;rf-&gt;sc_dev.hw_attrs.page_size_cap,</div><div class='add'>+				virt);</div><div class='add'>+	if (unlikely(!iwmr-&gt;page_size)) {</div><div class='add'>+		err = -EOPNOTSUPP;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	iwmr-&gt;len = region-&gt;length;</div><div class='add'>+	iwpbl-&gt;user_base = virt;</div><div class='add'>+	iwmr-&gt;page_cnt = ib_umem_num_dma_blocks(region, iwmr-&gt;page_size);</div><div class='add'>+</div><div class='add'>+	err = irdma_reg_user_mr_type_mem(iwmr, iwmr-&gt;access, false);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	ib_umem_release(region);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ *  irdma_rereg_user_mr - Re-Register a user memory region(MR)</div><div class='add'>+ *  @ibmr: ib mem to access iwarp mr pointer</div><div class='add'>+ *  @flags: bit mask to indicate which of the attr's of MR modified</div><div class='add'>+ *  @start: virtual start address</div><div class='add'>+ *  @len: length of mr</div><div class='add'>+ *  @virt: virtual address</div><div class='add'>+ *  @new_access: bit mask of access flags</div><div class='add'>+ *  @new_pd: ptr of pd</div><div class='add'>+ *  @udata: user data</div><div class='add'>+ *</div><div class='add'>+ *  Return:</div><div class='add'>+ *  NULL - Success, existing MR updated</div><div class='add'>+ *  ERR_PTR - error occurred</div><div class='add'>+ */</div><div class='add'>+static struct ib_mr *irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags,</div><div class='add'>+					 u64 start, u64 len, u64 virt,</div><div class='add'>+					 int new_access, struct ib_pd *new_pd,</div><div class='add'>+					 struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ib_mr-&gt;device);</div><div class='add'>+	struct irdma_mr *iwmr = to_iwmr(ib_mr);</div><div class='add'>+	struct irdma_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (len &gt; iwdev-&gt;rf-&gt;sc_dev.hw_attrs.max_mr_size)</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	if (flags &amp; ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS))</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	ret = irdma_hwdereg_mr(ib_mr);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ERR_PTR(ret);</div><div class='add'>+</div><div class='add'>+	if (flags &amp; IB_MR_REREG_ACCESS)</div><div class='add'>+		iwmr-&gt;access = new_access;</div><div class='add'>+</div><div class='add'>+	if (flags &amp; IB_MR_REREG_PD) {</div><div class='add'>+		iwmr-&gt;ibmr.pd = new_pd;</div><div class='add'>+		iwmr-&gt;ibmr.device = new_pd-&gt;device;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (flags &amp; IB_MR_REREG_TRANS) {</div><div class='add'>+		if (iwpbl-&gt;pbl_allocated) {</div><div class='add'>+			irdma_free_pble(iwdev-&gt;rf-&gt;pble_rsrc,</div><div class='add'>+					&amp;iwpbl-&gt;pble_alloc);</div><div class='add'>+			iwpbl-&gt;pbl_allocated = false;</div><div class='add'>+		}</div><div class='add'>+		if (iwmr-&gt;region) {</div><div class='add'>+			ib_umem_release(iwmr-&gt;region);</div><div class='add'>+			iwmr-&gt;region = NULL;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ret = irdma_rereg_mr_trans(iwmr, start, len, virt);</div><div class='add'>+	} else</div><div class='add'>+		ret = irdma_hwreg_mr(iwdev, iwmr, iwmr-&gt;access);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ERR_PTR(ret);</div><div class='add'>+</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_reg_phys_mr - register kernel physical memory</div><div class='add'>+ * @pd: ibpd pointer</div><div class='add'>+ * @addr: physical address of memory to register</div><div class='add'>+ * @size: size of memory to register</div><div class='add'>+ * @access: Access rights</div><div class='add'>+ * @iova_start: start of virtual address for physical buffers</div><div class='add'>+ * @dma_mr: Flag indicating whether this region is a PD DMA MR</div><div class='add'>+ */</div><div class='add'>+struct ib_mr *irdma_reg_phys_mr(struct ib_pd *pd, u64 addr, u64 size, int access,</div><div class='add'>+				u64 *iova_start, bool dma_mr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(pd-&gt;device);</div><div class='add'>+	struct irdma_pbl *iwpbl;</div><div class='add'>+	struct irdma_mr *iwmr;</div><div class='add'>+	u32 stag;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);</div><div class='add'>+	if (!iwmr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	iwmr-&gt;ibmr.pd = pd;</div><div class='add'>+	iwmr-&gt;ibmr.device = pd-&gt;device;</div><div class='add'>+	iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='add'>+	iwpbl-&gt;iwmr = iwmr;</div><div class='add'>+	iwmr-&gt;type = IRDMA_MEMREG_TYPE_MEM;</div><div class='add'>+	iwmr-&gt;dma_mr = dma_mr;</div><div class='add'>+	iwpbl-&gt;user_base = *iova_start;</div><div class='add'>+	stag = irdma_create_stag(iwdev);</div><div class='add'>+	if (!stag) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	iwmr-&gt;stag = stag;</div><div class='add'>+	iwmr-&gt;ibmr.iova = *iova_start;</div><div class='add'>+	iwmr-&gt;ibmr.rkey = stag;</div><div class='add'>+	iwmr-&gt;ibmr.lkey = stag;</div><div class='add'>+	iwmr-&gt;page_cnt = 1;</div><div class='add'>+	iwmr-&gt;pgaddrmem[0] = addr;</div><div class='add'>+	iwmr-&gt;len = size;</div><div class='add'>+	iwmr-&gt;page_size = SZ_4K;</div><div class='add'>+	ret = irdma_hwreg_mr(iwdev, iwmr, access);</div><div class='add'>+	if (ret) {</div><div class='add'>+		irdma_free_stag(iwdev, stag);</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return &amp;iwmr-&gt;ibmr;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	kfree(iwmr);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(ret);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_dma_mr - register physical mem</div><div class='add'>+ * @pd: ptr of pd</div><div class='add'>+ * @acc: access for memory</div><div class='add'>+ */</div><div class='add'>+static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc)</div><div class='add'>+{</div><div class='add'>+	u64 kva = 0;</div><div class='add'>+</div><div class='add'>+	return irdma_reg_phys_mr(pd, 0, 0, acc, &amp;kva, true);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_del_memlist - Deleting pbl list entries for CQ/QP</div><div class='add'>+ * @iwmr: iwmr for IB's user page addresses</div><div class='add'>+ * @ucontext: ptr to user context</div><div class='add'>+ */</div><div class='add'>+static void irdma_del_memlist(struct irdma_mr *iwmr,</div><div class='add'>+			      struct irdma_ucontext *ucontext)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	switch (iwmr-&gt;type) {</div><div class='add'>+	case IRDMA_MEMREG_TYPE_CQ:</div><div class='add'>+		spin_lock_irqsave(&amp;ucontext-&gt;cq_reg_mem_list_lock, flags);</div><div class='add'>+		if (iwpbl-&gt;on_list) {</div><div class='add'>+			iwpbl-&gt;on_list = false;</div><div class='add'>+			list_del(&amp;iwpbl-&gt;list);</div><div class='add'>+		}</div><div class='add'>+		spin_unlock_irqrestore(&amp;ucontext-&gt;cq_reg_mem_list_lock, flags);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_MEMREG_TYPE_QP:</div><div class='add'>+		spin_lock_irqsave(&amp;ucontext-&gt;qp_reg_mem_list_lock, flags);</div><div class='add'>+		if (iwpbl-&gt;on_list) {</div><div class='add'>+			iwpbl-&gt;on_list = false;</div><div class='add'>+			list_del(&amp;iwpbl-&gt;list);</div><div class='add'>+		}</div><div class='add'>+		spin_unlock_irqrestore(&amp;ucontext-&gt;qp_reg_mem_list_lock, flags);</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_MEMREG_TYPE_SRQ:</div><div class='add'>+		spin_lock_irqsave(&amp;ucontext-&gt;srq_reg_mem_list_lock, flags);</div><div class='add'>+		if (iwpbl-&gt;on_list) {</div><div class='add'>+			iwpbl-&gt;on_list = false;</div><div class='add'>+			list_del(&amp;iwpbl-&gt;list);</div><div class='add'>+		}</div><div class='add'>+		spin_unlock_irqrestore(&amp;ucontext-&gt;srq_reg_mem_list_lock, flags);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_dereg_mr - deregister mr</div><div class='add'>+ * @ib_mr: mr ptr for dereg</div><div class='add'>+ * @udata: user data</div><div class='add'>+ */</div><div class='add'>+static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct irdma_mr *iwmr = to_iwmr(ib_mr);</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ib_mr-&gt;device);</div><div class='add'>+	struct irdma_pbl *iwpbl = &amp;iwmr-&gt;iwpbl;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (iwmr-&gt;type != IRDMA_MEMREG_TYPE_MEM) {</div><div class='add'>+		if (iwmr-&gt;region) {</div><div class='add'>+			struct irdma_ucontext *ucontext;</div><div class='add'>+</div><div class='add'>+			ucontext = rdma_udata_to_drv_context(udata,</div><div class='add'>+						struct irdma_ucontext,</div><div class='add'>+						ibucontext);</div><div class='add'>+			irdma_del_memlist(iwmr, ucontext);</div><div class='add'>+		}</div><div class='add'>+		goto done;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = irdma_hwdereg_mr(ib_mr);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	irdma_free_stag(iwdev, iwmr-&gt;stag);</div><div class='add'>+done:</div><div class='add'>+	if (iwpbl-&gt;pbl_allocated)</div><div class='add'>+		irdma_free_pble(iwdev-&gt;rf-&gt;pble_rsrc, &amp;iwpbl-&gt;pble_alloc);</div><div class='add'>+</div><div class='add'>+	if (iwmr-&gt;region)</div><div class='add'>+		ib_umem_release(iwmr-&gt;region);</div><div class='add'>+</div><div class='add'>+	kfree(iwmr);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_post_send -  kernel application wr</div><div class='add'>+ * @ibqp: qp ptr for wr</div><div class='add'>+ * @ib_wr: work request ptr</div><div class='add'>+ * @bad_wr: return of bad wr if err</div><div class='add'>+ */</div><div class='add'>+static int irdma_post_send(struct ib_qp *ibqp,</div><div class='add'>+			   const struct ib_send_wr *ib_wr,</div><div class='add'>+			   const struct ib_send_wr **bad_wr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+	struct irdma_qp_uk *ukqp;</div><div class='add'>+	struct irdma_sc_dev *dev;</div><div class='add'>+	struct irdma_post_sq_info info;</div><div class='add'>+	int err = 0;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	bool inv_stag;</div><div class='add'>+	struct irdma_ah *ah;</div><div class='add'>+</div><div class='add'>+	iwqp = to_iwqp(ibqp);</div><div class='add'>+	ukqp = &amp;iwqp-&gt;sc_qp.qp_uk;</div><div class='add'>+	dev = &amp;iwqp-&gt;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+	while (ib_wr) {</div><div class='add'>+		memset(&amp;info, 0, sizeof(info));</div><div class='add'>+		inv_stag = false;</div><div class='add'>+		info.wr_id = (ib_wr-&gt;wr_id);</div><div class='add'>+		if ((ib_wr-&gt;send_flags &amp; IB_SEND_SIGNALED) || iwqp-&gt;sig_all)</div><div class='add'>+			info.signaled = true;</div><div class='add'>+		if (ib_wr-&gt;send_flags &amp; IB_SEND_FENCE)</div><div class='add'>+			info.read_fence = true;</div><div class='add'>+		switch (ib_wr-&gt;opcode) {</div><div class='add'>+		case IB_WR_ATOMIC_CMP_AND_SWP:</div><div class='add'>+			if (unlikely(!(dev-&gt;hw_attrs.uk_attrs.feature_flags &amp;</div><div class='add'>+				       IRDMA_FEATURE_ATOMIC_OPS))) {</div><div class='add'>+				err = -EINVAL;</div><div class='add'>+				break;</div><div class='add'>+			}</div><div class='add'>+			info.op_type = IRDMA_OP_TYPE_ATOMIC_COMPARE_AND_SWAP;</div><div class='add'>+			info.op.atomic_compare_swap.tagged_offset = ib_wr-&gt;sg_list[0].addr;</div><div class='add'>+			info.op.atomic_compare_swap.remote_tagged_offset =</div><div class='add'>+				atomic_wr(ib_wr)-&gt;remote_addr;</div><div class='add'>+			info.op.atomic_compare_swap.swap_data_bytes = atomic_wr(ib_wr)-&gt;swap;</div><div class='add'>+			info.op.atomic_compare_swap.compare_data_bytes =</div><div class='add'>+				atomic_wr(ib_wr)-&gt;compare_add;</div><div class='add'>+			info.op.atomic_compare_swap.stag = ib_wr-&gt;sg_list[0].lkey;</div><div class='add'>+			info.op.atomic_compare_swap.remote_stag = atomic_wr(ib_wr)-&gt;rkey;</div><div class='add'>+			err = irdma_uk_atomic_compare_swap(ukqp, &amp;info, false);</div><div class='add'>+			break;</div><div class='add'>+		case IB_WR_ATOMIC_FETCH_AND_ADD:</div><div class='add'>+			if (unlikely(!(dev-&gt;hw_attrs.uk_attrs.feature_flags &amp;</div><div class='add'>+				       IRDMA_FEATURE_ATOMIC_OPS))) {</div><div class='add'>+				err = -EINVAL;</div><div class='add'>+				break;</div><div class='add'>+			}</div><div class='add'>+			info.op_type = IRDMA_OP_TYPE_ATOMIC_FETCH_AND_ADD;</div><div class='add'>+			info.op.atomic_fetch_add.tagged_offset = ib_wr-&gt;sg_list[0].addr;</div><div class='add'>+			info.op.atomic_fetch_add.remote_tagged_offset =</div><div class='add'>+				atomic_wr(ib_wr)-&gt;remote_addr;</div><div class='add'>+			info.op.atomic_fetch_add.fetch_add_data_bytes =</div><div class='add'>+				atomic_wr(ib_wr)-&gt;compare_add;</div><div class='add'>+			info.op.atomic_fetch_add.stag = ib_wr-&gt;sg_list[0].lkey;</div><div class='add'>+			info.op.atomic_fetch_add.remote_stag =</div><div class='add'>+				atomic_wr(ib_wr)-&gt;rkey;</div><div class='add'>+			err = irdma_uk_atomic_fetch_add(ukqp, &amp;info, false);</div><div class='add'>+			break;</div><div class='add'>+		case IB_WR_SEND_WITH_IMM:</div><div class='add'>+			if (ukqp-&gt;qp_caps &amp; IRDMA_SEND_WITH_IMM) {</div><div class='add'>+				info.imm_data_valid = true;</div><div class='add'>+				info.imm_data = ntohl(ib_wr-&gt;ex.imm_data);</div><div class='add'>+			} else {</div><div class='add'>+				err = -EINVAL;</div><div class='add'>+				break;</div><div class='add'>+			}</div><div class='add'>+			fallthrough;</div><div class='add'>+		case IB_WR_SEND:</div><div class='add'>+		case IB_WR_SEND_WITH_INV:</div><div class='add'>+			if (ib_wr-&gt;opcode == IB_WR_SEND ||</div><div class='add'>+			    ib_wr-&gt;opcode == IB_WR_SEND_WITH_IMM) {</div><div class='add'>+				if (ib_wr-&gt;send_flags &amp; IB_SEND_SOLICITED)</div><div class='add'>+					info.op_type = IRDMA_OP_TYPE_SEND_SOL;</div><div class='add'>+				else</div><div class='add'>+					info.op_type = IRDMA_OP_TYPE_SEND;</div><div class='add'>+			} else {</div><div class='add'>+				if (ib_wr-&gt;send_flags &amp; IB_SEND_SOLICITED)</div><div class='add'>+					info.op_type = IRDMA_OP_TYPE_SEND_SOL_INV;</div><div class='add'>+				else</div><div class='add'>+					info.op_type = IRDMA_OP_TYPE_SEND_INV;</div><div class='add'>+				info.stag_to_inv = ib_wr-&gt;ex.invalidate_rkey;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			info.op.send.num_sges = ib_wr-&gt;num_sge;</div><div class='add'>+			info.op.send.sg_list = ib_wr-&gt;sg_list;</div><div class='add'>+			if (iwqp-&gt;ibqp.qp_type == IB_QPT_UD ||</div><div class='add'>+			    iwqp-&gt;ibqp.qp_type == IB_QPT_GSI) {</div><div class='add'>+				ah = to_iwah(ud_wr(ib_wr)-&gt;ah);</div><div class='add'>+				info.op.send.ah_id = ah-&gt;sc_ah.ah_info.ah_idx;</div><div class='add'>+				info.op.send.qkey = ud_wr(ib_wr)-&gt;remote_qkey;</div><div class='add'>+				info.op.send.dest_qp = ud_wr(ib_wr)-&gt;remote_qpn;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			if (ib_wr-&gt;send_flags &amp; IB_SEND_INLINE)</div><div class='add'>+				err = irdma_uk_inline_send(ukqp, &amp;info, false);</div><div class='add'>+			else</div><div class='add'>+				err = irdma_uk_send(ukqp, &amp;info, false);</div><div class='add'>+			break;</div><div class='add'>+		case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='add'>+			if (ukqp-&gt;qp_caps &amp; IRDMA_WRITE_WITH_IMM) {</div><div class='add'>+				info.imm_data_valid = true;</div><div class='add'>+				info.imm_data = ntohl(ib_wr-&gt;ex.imm_data);</div><div class='add'>+			} else {</div><div class='add'>+				err = -EINVAL;</div><div class='add'>+				break;</div><div class='add'>+			}</div><div class='add'>+			fallthrough;</div><div class='add'>+		case IB_WR_RDMA_WRITE:</div><div class='add'>+			if (ib_wr-&gt;send_flags &amp; IB_SEND_SOLICITED)</div><div class='add'>+				info.op_type = IRDMA_OP_TYPE_RDMA_WRITE_SOL;</div><div class='add'>+			else</div><div class='add'>+				info.op_type = IRDMA_OP_TYPE_RDMA_WRITE;</div><div class='add'>+</div><div class='add'>+			info.op.rdma_write.num_lo_sges = ib_wr-&gt;num_sge;</div><div class='add'>+			info.op.rdma_write.lo_sg_list = ib_wr-&gt;sg_list;</div><div class='add'>+			info.op.rdma_write.rem_addr.addr =</div><div class='add'>+				rdma_wr(ib_wr)-&gt;remote_addr;</div><div class='add'>+			info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)-&gt;rkey;</div><div class='add'>+			if (ib_wr-&gt;send_flags &amp; IB_SEND_INLINE)</div><div class='add'>+				err = irdma_uk_inline_rdma_write(ukqp, &amp;info, false);</div><div class='add'>+			else</div><div class='add'>+				err = irdma_uk_rdma_write(ukqp, &amp;info, false);</div><div class='add'>+			break;</div><div class='add'>+		case IB_WR_RDMA_READ_WITH_INV:</div><div class='add'>+			inv_stag = true;</div><div class='add'>+			fallthrough;</div><div class='add'>+		case IB_WR_RDMA_READ:</div><div class='add'>+			if (ib_wr-&gt;num_sge &gt;</div><div class='add'>+			    dev-&gt;hw_attrs.uk_attrs.max_hw_read_sges) {</div><div class='add'>+				err = -EINVAL;</div><div class='add'>+				break;</div><div class='add'>+			}</div><div class='add'>+			info.op_type = IRDMA_OP_TYPE_RDMA_READ;</div><div class='add'>+			info.op.rdma_read.rem_addr.addr = rdma_wr(ib_wr)-&gt;remote_addr;</div><div class='add'>+			info.op.rdma_read.rem_addr.lkey = rdma_wr(ib_wr)-&gt;rkey;</div><div class='add'>+			info.op.rdma_read.lo_sg_list = (void *)ib_wr-&gt;sg_list;</div><div class='add'>+			info.op.rdma_read.num_lo_sges = ib_wr-&gt;num_sge;</div><div class='add'>+			err = irdma_uk_rdma_read(ukqp, &amp;info, inv_stag, false);</div><div class='add'>+			break;</div><div class='add'>+		case IB_WR_LOCAL_INV:</div><div class='add'>+			info.op_type = IRDMA_OP_TYPE_INV_STAG;</div><div class='add'>+			info.local_fence = true;</div><div class='add'>+			info.op.inv_local_stag.target_stag = ib_wr-&gt;ex.invalidate_rkey;</div><div class='add'>+			err = irdma_uk_stag_local_invalidate(ukqp, &amp;info, true);</div><div class='add'>+			break;</div><div class='add'>+		case IB_WR_REG_MR: {</div><div class='add'>+			struct irdma_mr *iwmr = to_iwmr(reg_wr(ib_wr)-&gt;mr);</div><div class='add'>+			struct irdma_pble_alloc *palloc = &amp;iwmr-&gt;iwpbl.pble_alloc;</div><div class='add'>+			struct irdma_fast_reg_stag_info stag_info = {};</div><div class='add'>+</div><div class='add'>+			stag_info.signaled = info.signaled;</div><div class='add'>+			stag_info.read_fence = info.read_fence;</div><div class='add'>+			stag_info.access_rights =</div><div class='add'>+				irdma_get_mr_access(reg_wr(ib_wr)-&gt;access,</div><div class='add'>+						    dev-&gt;hw_attrs.uk_attrs.hw_rev);</div><div class='add'>+			stag_info.stag_key = reg_wr(ib_wr)-&gt;key &amp; 0xff;</div><div class='add'>+			stag_info.stag_idx = reg_wr(ib_wr)-&gt;key &gt;&gt; 8;</div><div class='add'>+			stag_info.page_size = reg_wr(ib_wr)-&gt;mr-&gt;page_size;</div><div class='add'>+			stag_info.wr_id = ib_wr-&gt;wr_id;</div><div class='add'>+			stag_info.addr_type = IRDMA_ADDR_TYPE_VA_BASED;</div><div class='add'>+			stag_info.va = (void *)(uintptr_t)iwmr-&gt;ibmr.iova;</div><div class='add'>+			stag_info.total_len = iwmr-&gt;ibmr.length;</div><div class='add'>+			stag_info.reg_addr_pa = *palloc-&gt;level1.addr;</div><div class='add'>+			stag_info.first_pm_pbl_index = palloc-&gt;level1.idx;</div><div class='add'>+			stag_info.local_fence = ib_wr-&gt;send_flags &amp; IB_SEND_FENCE;</div><div class='add'>+			if (iwmr-&gt;npages &gt; IRDMA_MIN_PAGES_PER_FMR)</div><div class='add'>+				stag_info.chunk_size = 1;</div><div class='add'>+			err = irdma_sc_mr_fast_register(&amp;iwqp-&gt;sc_qp, &amp;stag_info,</div><div class='add'>+							true);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+		default:</div><div class='add'>+			err = -EINVAL;</div><div class='add'>+			ibdev_dbg(&amp;iwqp-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "VERBS: upost_send bad opcode = 0x%x\n",</div><div class='add'>+				  ib_wr-&gt;opcode);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (err)</div><div class='add'>+			break;</div><div class='add'>+		ib_wr = ib_wr-&gt;next;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!iwqp-&gt;flush_issued) {</div><div class='add'>+		if (iwqp-&gt;hw_iwarp_state &lt;= IRDMA_QP_STATE_RTS)</div><div class='add'>+			irdma_uk_qp_post_wr(ukqp);</div><div class='add'>+		spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+	} else {</div><div class='add'>+		spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+		mod_delayed_work(iwqp-&gt;iwdev-&gt;cleanup_wq, &amp;iwqp-&gt;dwork_flush,</div><div class='add'>+				 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (err)</div><div class='add'>+		*bad_wr = ib_wr;</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_post_srq_recv - post receive wr for kernel application</div><div class='add'>+ * @ibsrq: ib srq pointer</div><div class='add'>+ * @ib_wr: work request for receive</div><div class='add'>+ * @bad_wr: bad wr caused an error</div><div class='add'>+ */</div><div class='add'>+static int irdma_post_srq_recv(struct ib_srq *ibsrq,</div><div class='add'>+			       const struct ib_recv_wr *ib_wr,</div><div class='add'>+			       const struct ib_recv_wr **bad_wr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_srq *iwsrq = to_iwsrq(ibsrq);</div><div class='add'>+	struct irdma_srq_uk *uksrq = &amp;iwsrq-&gt;sc_srq.srq_uk;</div><div class='add'>+	struct irdma_post_rq_info post_recv = {};</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int err = 0;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwsrq-&gt;lock, flags);</div><div class='add'>+	while (ib_wr) {</div><div class='add'>+		if (ib_wr-&gt;num_sge &gt; uksrq-&gt;max_srq_frag_cnt) {</div><div class='add'>+			err = -EINVAL;</div><div class='add'>+			goto out;</div><div class='add'>+		}</div><div class='add'>+		post_recv.num_sges = ib_wr-&gt;num_sge;</div><div class='add'>+		post_recv.wr_id = ib_wr-&gt;wr_id;</div><div class='add'>+		post_recv.sg_list = ib_wr-&gt;sg_list;</div><div class='add'>+		err = irdma_uk_srq_post_receive(uksrq, &amp;post_recv);</div><div class='add'>+		if (err)</div><div class='add'>+			goto out;</div><div class='add'>+</div><div class='add'>+		ib_wr = ib_wr-&gt;next;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwsrq-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	if (err)</div><div class='add'>+		*bad_wr = ib_wr;</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_post_recv - post receive wr for kernel application</div><div class='add'>+ * @ibqp: ib qp pointer</div><div class='add'>+ * @ib_wr: work request for receive</div><div class='add'>+ * @bad_wr: bad wr caused an error</div><div class='add'>+ */</div><div class='add'>+static int irdma_post_recv(struct ib_qp *ibqp,</div><div class='add'>+			   const struct ib_recv_wr *ib_wr,</div><div class='add'>+			   const struct ib_recv_wr **bad_wr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+	struct irdma_qp_uk *ukqp;</div><div class='add'>+	struct irdma_post_rq_info post_recv = {};</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int err = 0;</div><div class='add'>+</div><div class='add'>+	iwqp = to_iwqp(ibqp);</div><div class='add'>+	ukqp = &amp;iwqp-&gt;sc_qp.qp_uk;</div><div class='add'>+</div><div class='add'>+	if (ukqp-&gt;srq_uk) {</div><div class='add'>+		*bad_wr = ib_wr;</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+	while (ib_wr) {</div><div class='add'>+		post_recv.num_sges = ib_wr-&gt;num_sge;</div><div class='add'>+		post_recv.wr_id = ib_wr-&gt;wr_id;</div><div class='add'>+		post_recv.sg_list = ib_wr-&gt;sg_list;</div><div class='add'>+		err = irdma_uk_post_receive(ukqp, &amp;post_recv);</div><div class='add'>+		if (err) {</div><div class='add'>+			ibdev_dbg(&amp;iwqp-&gt;iwdev-&gt;ibdev,</div><div class='add'>+				  "VERBS: post_recv err %d\n", err);</div><div class='add'>+			goto out;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ib_wr = ib_wr-&gt;next;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwqp-&gt;lock, flags);</div><div class='add'>+	if (iwqp-&gt;flush_issued)</div><div class='add'>+		mod_delayed_work(iwqp-&gt;iwdev-&gt;cleanup_wq, &amp;iwqp-&gt;dwork_flush,</div><div class='add'>+				 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));</div><div class='add'>+</div><div class='add'>+	if (err)</div><div class='add'>+		*bad_wr = ib_wr;</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_flush_err_to_ib_wc_status - return change flush error code to IB status</div><div class='add'>+ * @opcode: iwarp flush code</div><div class='add'>+ */</div><div class='add'>+static enum ib_wc_status irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode opcode)</div><div class='add'>+{</div><div class='add'>+	switch (opcode) {</div><div class='add'>+	case FLUSH_PROT_ERR:</div><div class='add'>+		return IB_WC_LOC_PROT_ERR;</div><div class='add'>+	case FLUSH_REM_ACCESS_ERR:</div><div class='add'>+		return IB_WC_REM_ACCESS_ERR;</div><div class='add'>+	case FLUSH_LOC_QP_OP_ERR:</div><div class='add'>+		return IB_WC_LOC_QP_OP_ERR;</div><div class='add'>+	case FLUSH_REM_OP_ERR:</div><div class='add'>+		return IB_WC_REM_OP_ERR;</div><div class='add'>+	case FLUSH_LOC_LEN_ERR:</div><div class='add'>+		return IB_WC_LOC_LEN_ERR;</div><div class='add'>+	case FLUSH_GENERAL_ERR:</div><div class='add'>+		return IB_WC_WR_FLUSH_ERR;</div><div class='add'>+	case FLUSH_RETRY_EXC_ERR:</div><div class='add'>+		return IB_WC_RETRY_EXC_ERR;</div><div class='add'>+	case FLUSH_MW_BIND_ERR:</div><div class='add'>+		return IB_WC_MW_BIND_ERR;</div><div class='add'>+	case FLUSH_REM_INV_REQ_ERR:</div><div class='add'>+		return IB_WC_REM_INV_REQ_ERR;</div><div class='add'>+	case FLUSH_RNR_RETRY_EXC_ERR:</div><div class='add'>+		return IB_WC_RNR_RETRY_EXC_ERR;</div><div class='add'>+	case FLUSH_FATAL_ERR:</div><div class='add'>+	default:</div><div class='add'>+		return IB_WC_FATAL_ERR;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_process_cqe - process cqe info</div><div class='add'>+ * @entry: processed cqe</div><div class='add'>+ * @cq_poll_info: cqe info</div><div class='add'>+ */</div><div class='add'>+static void irdma_process_cqe(struct ib_wc *entry,</div><div class='add'>+			      struct irdma_cq_poll_info *cq_poll_info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_sc_qp *qp;</div><div class='add'>+</div><div class='add'>+	entry-&gt;wc_flags = 0;</div><div class='add'>+	entry-&gt;pkey_index = 0;</div><div class='add'>+	entry-&gt;wr_id = cq_poll_info-&gt;wr_id;</div><div class='add'>+</div><div class='add'>+	qp = cq_poll_info-&gt;qp_handle;</div><div class='add'>+	entry-&gt;qp = qp-&gt;qp_uk.back_qp;</div><div class='add'>+</div><div class='add'>+	if (cq_poll_info-&gt;error) {</div><div class='add'>+		entry-&gt;status = (cq_poll_info-&gt;comp_status == IRDMA_COMPL_STATUS_FLUSHED) ?</div><div class='add'>+				irdma_flush_err_to_ib_wc_status(cq_poll_info-&gt;minor_err) : IB_WC_GENERAL_ERR;</div><div class='add'>+</div><div class='add'>+		entry-&gt;vendor_err = cq_poll_info-&gt;major_err &lt;&lt; 16 |</div><div class='add'>+				    cq_poll_info-&gt;minor_err;</div><div class='add'>+	} else {</div><div class='add'>+		entry-&gt;status = IB_WC_SUCCESS;</div><div class='add'>+		if (cq_poll_info-&gt;imm_valid) {</div><div class='add'>+			entry-&gt;ex.imm_data = htonl(cq_poll_info-&gt;imm_data);</div><div class='add'>+			entry-&gt;wc_flags |= IB_WC_WITH_IMM;</div><div class='add'>+		}</div><div class='add'>+		if (cq_poll_info-&gt;ud_smac_valid) {</div><div class='add'>+			ether_addr_copy(entry-&gt;smac, cq_poll_info-&gt;ud_smac);</div><div class='add'>+			entry-&gt;wc_flags |= IB_WC_WITH_SMAC;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (cq_poll_info-&gt;ud_vlan_valid) {</div><div class='add'>+			u16 vlan = cq_poll_info-&gt;ud_vlan &amp; VLAN_VID_MASK;</div><div class='add'>+</div><div class='add'>+			entry-&gt;sl = cq_poll_info-&gt;ud_vlan &gt;&gt; VLAN_PRIO_SHIFT;</div><div class='add'>+			if (vlan) {</div><div class='add'>+				entry-&gt;vlan_id = vlan;</div><div class='add'>+				entry-&gt;wc_flags |= IB_WC_WITH_VLAN;</div><div class='add'>+			}</div><div class='add'>+		} else {</div><div class='add'>+			entry-&gt;sl = 0;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (cq_poll_info-&gt;q_type == IRDMA_CQE_QTYPE_SQ) {</div><div class='add'>+		set_ib_wc_op_sq(cq_poll_info, entry);</div><div class='add'>+	} else {</div><div class='add'>+		if (qp-&gt;dev-&gt;hw_attrs.uk_attrs.hw_rev &lt;= IRDMA_GEN_2)</div><div class='add'>+			set_ib_wc_op_rq(cq_poll_info, entry,</div><div class='add'>+					qp-&gt;qp_uk.qp_caps &amp; IRDMA_SEND_WITH_IMM ?</div><div class='add'>+					true : false);</div><div class='add'>+		else</div><div class='add'>+			set_ib_wc_op_rq_gen_3(cq_poll_info, entry);</div><div class='add'>+		if (qp-&gt;qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_UD &amp;&amp;</div><div class='add'>+		    cq_poll_info-&gt;stag_invalid_set) {</div><div class='add'>+			entry-&gt;ex.invalidate_rkey = cq_poll_info-&gt;inv_stag;</div><div class='add'>+			entry-&gt;wc_flags |= IB_WC_WITH_INVALIDATE;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) {</div><div class='add'>+		entry-&gt;src_qp = cq_poll_info-&gt;ud_src_qpn;</div><div class='add'>+		entry-&gt;slid = 0;</div><div class='add'>+		entry-&gt;wc_flags |=</div><div class='add'>+			(IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE);</div><div class='add'>+		entry-&gt;network_hdr_type = cq_poll_info-&gt;ipv4 ?</div><div class='add'>+						  RDMA_NETWORK_IPV4 :</div><div class='add'>+						  RDMA_NETWORK_IPV6;</div><div class='add'>+	} else {</div><div class='add'>+		entry-&gt;src_qp = cq_poll_info-&gt;qp_id;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	entry-&gt;byte_len = cq_poll_info-&gt;bytes_xfered;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_poll_one - poll one entry of the CQ</div><div class='add'>+ * @ukcq: ukcq to poll</div><div class='add'>+ * @cur_cqe: current CQE info to be filled in</div><div class='add'>+ * @entry: ibv_wc object to be filled for non-extended CQ or NULL for extended CQ</div><div class='add'>+ *</div><div class='add'>+ * Returns the internal irdma device error code or 0 on success</div><div class='add'>+ */</div><div class='add'>+static inline int irdma_poll_one(struct irdma_cq_uk *ukcq,</div><div class='add'>+				 struct irdma_cq_poll_info *cur_cqe,</div><div class='add'>+				 struct ib_wc *entry)</div><div class='add'>+{</div><div class='add'>+	int ret = irdma_uk_cq_poll_cmpl(ukcq, cur_cqe);</div><div class='add'>+</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	irdma_process_cqe(entry, cur_cqe);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * __irdma_poll_cq - poll cq for completion (kernel apps)</div><div class='add'>+ * @iwcq: cq to poll</div><div class='add'>+ * @num_entries: number of entries to poll</div><div class='add'>+ * @entry: wr of a completed entry</div><div class='add'>+ */</div><div class='add'>+static int __irdma_poll_cq(struct irdma_cq *iwcq, int num_entries, struct ib_wc *entry)</div><div class='add'>+{</div><div class='add'>+	struct list_head *tmp_node, *list_node;</div><div class='add'>+	struct irdma_cq_buf *last_buf = NULL;</div><div class='add'>+	struct irdma_cq_poll_info *cur_cqe = &amp;iwcq-&gt;cur_cqe;</div><div class='add'>+	struct irdma_cq_buf *cq_buf;</div><div class='add'>+	int ret;</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct irdma_cq_uk *ukcq;</div><div class='add'>+	bool cq_new_cqe = false;</div><div class='add'>+	int resized_bufs = 0;</div><div class='add'>+	int npolled = 0;</div><div class='add'>+</div><div class='add'>+	iwdev = to_iwdev(iwcq-&gt;ibcq.device);</div><div class='add'>+	ukcq = &amp;iwcq-&gt;sc_cq.cq_uk;</div><div class='add'>+</div><div class='add'>+	/* go through the list of previously resized CQ buffers */</div><div class='add'>+	list_for_each_safe(list_node, tmp_node, &amp;iwcq-&gt;resize_list) {</div><div class='add'>+		cq_buf = container_of(list_node, struct irdma_cq_buf, list);</div><div class='add'>+		while (npolled &lt; num_entries) {</div><div class='add'>+			ret = irdma_poll_one(&amp;cq_buf-&gt;cq_uk, cur_cqe, entry + npolled);</div><div class='add'>+			if (!ret) {</div><div class='add'>+				++npolled;</div><div class='add'>+				cq_new_cqe = true;</div><div class='add'>+				continue;</div><div class='add'>+			}</div><div class='add'>+			if (ret == -ENOENT)</div><div class='add'>+				break;</div><div class='add'>+			 /* QP using the CQ is destroyed. Skip reporting this CQE */</div><div class='add'>+			if (ret == -EFAULT) {</div><div class='add'>+				cq_new_cqe = true;</div><div class='add'>+				continue;</div><div class='add'>+			}</div><div class='add'>+			goto error;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		/* save the resized CQ buffer which received the last cqe */</div><div class='add'>+		if (cq_new_cqe)</div><div class='add'>+			last_buf = cq_buf;</div><div class='add'>+		cq_new_cqe = false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* check the current CQ for new cqes */</div><div class='add'>+	while (npolled &lt; num_entries) {</div><div class='add'>+		ret = irdma_poll_one(ukcq, cur_cqe, entry + npolled);</div><div class='add'>+		if (ret == -ENOENT) {</div><div class='add'>+			ret = irdma_generated_cmpls(iwcq, cur_cqe);</div><div class='add'>+			if (!ret)</div><div class='add'>+				irdma_process_cqe(entry + npolled, cur_cqe);</div><div class='add'>+		}</div><div class='add'>+		if (!ret) {</div><div class='add'>+			++npolled;</div><div class='add'>+			cq_new_cqe = true;</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (ret == -ENOENT)</div><div class='add'>+			break;</div><div class='add'>+		/* QP using the CQ is destroyed. Skip reporting this CQE */</div><div class='add'>+		if (ret == -EFAULT) {</div><div class='add'>+			cq_new_cqe = true;</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+		goto error;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (cq_new_cqe)</div><div class='add'>+		/* all previous CQ resizes are complete */</div><div class='add'>+		resized_bufs = irdma_process_resize_list(iwcq, iwdev, NULL);</div><div class='add'>+	else if (last_buf)</div><div class='add'>+		/* only CQ resizes up to the last_buf are complete */</div><div class='add'>+		resized_bufs = irdma_process_resize_list(iwcq, iwdev, last_buf);</div><div class='add'>+	if (resized_bufs)</div><div class='add'>+		/* report to the HW the number of complete CQ resizes */</div><div class='add'>+		irdma_uk_cq_set_resized_cnt(ukcq, resized_bufs);</div><div class='add'>+</div><div class='add'>+	return npolled;</div><div class='add'>+error:</div><div class='add'>+	ibdev_dbg(&amp;iwdev-&gt;ibdev, "%s: Error polling CQ, irdma_err: %d\n",</div><div class='add'>+		  __func__, ret);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_poll_cq - poll cq for completion (kernel apps)</div><div class='add'>+ * @ibcq: cq to poll</div><div class='add'>+ * @num_entries: number of entries to poll</div><div class='add'>+ * @entry: wr of a completed entry</div><div class='add'>+ */</div><div class='add'>+static int irdma_poll_cq(struct ib_cq *ibcq, int num_entries,</div><div class='add'>+			 struct ib_wc *entry)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cq *iwcq;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	iwcq = to_iwcq(ibcq);</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwcq-&gt;lock, flags);</div><div class='add'>+	ret = __irdma_poll_cq(iwcq, num_entries, entry);</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwcq-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_req_notify_cq - arm cq kernel application</div><div class='add'>+ * @ibcq: cq to arm</div><div class='add'>+ * @notify_flags: notofication flags</div><div class='add'>+ */</div><div class='add'>+static int irdma_req_notify_cq(struct ib_cq *ibcq,</div><div class='add'>+			       enum ib_cq_notify_flags notify_flags)</div><div class='add'>+{</div><div class='add'>+	struct irdma_cq *iwcq;</div><div class='add'>+	struct irdma_cq_uk *ukcq;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	enum irdma_cmpl_notify cq_notify;</div><div class='add'>+	bool promo_event = false;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	cq_notify = notify_flags == IB_CQ_SOLICITED ?</div><div class='add'>+		    IRDMA_CQ_COMPL_SOLICITED : IRDMA_CQ_COMPL_EVENT;</div><div class='add'>+	iwcq = to_iwcq(ibcq);</div><div class='add'>+	ukcq = &amp;iwcq-&gt;sc_cq.cq_uk;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;iwcq-&gt;lock, flags);</div><div class='add'>+	/* Only promote to arm the CQ for any event if the last arm event was solicited. */</div><div class='add'>+	if (iwcq-&gt;last_notify == IRDMA_CQ_COMPL_SOLICITED &amp;&amp; notify_flags != IB_CQ_SOLICITED)</div><div class='add'>+		promo_event = true;</div><div class='add'>+</div><div class='add'>+	if (!atomic_cmpxchg(&amp;iwcq-&gt;armed, 0, 1) || promo_event) {</div><div class='add'>+		iwcq-&gt;last_notify = cq_notify;</div><div class='add'>+		irdma_uk_cq_request_notification(ukcq, cq_notify);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if ((notify_flags &amp; IB_CQ_REPORT_MISSED_EVENTS) &amp;&amp;</div><div class='add'>+	    (!irdma_uk_cq_empty(ukcq) || !list_empty(&amp;iwcq-&gt;cmpl_generated)))</div><div class='add'>+		ret = 1;</div><div class='add'>+	spin_unlock_irqrestore(&amp;iwcq-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct rdma_stat_desc irdma_hw_stat_descs[] = {</div><div class='add'>+	/* gen1 - 32-bit */</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXDISCARD].name		= "ip4InDiscards",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXTRUNC].name		= "ip4InTruncatedPkts",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE].name		= "ip4OutNoRoutes",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXDISCARD].name		= "ip6InDiscards",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXTRUNC].name		= "ip6InTruncatedPkts",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE].name		= "ip6OutNoRoutes",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RXVLANERR].name		= "rxVlanErrors",</div><div class='add'>+	/* gen1 - 64-bit */</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXOCTS].name		= "ip4InOctets",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXPKTS].name		= "ip4InPkts",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXFRAGS].name		= "ip4InReasmRqd",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS].name		= "ip4InMcastPkts",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXOCTS].name		= "ip4OutOctets",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXPKTS].name		= "ip4OutPkts",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXFRAGS].name		= "ip4OutSegRqd",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS].name		= "ip4OutMcastPkts",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXOCTS].name		= "ip6InOctets",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXPKTS].name		= "ip6InPkts",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXFRAGS].name		= "ip6InReasmRqd",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS].name		= "ip6InMcastPkts",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXOCTS].name		= "ip6OutOctets",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXPKTS].name		= "ip6OutPkts",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXFRAGS].name		= "ip6OutSegRqd",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS].name		= "ip6OutMcastPkts",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMARXRDS].name		= "InRdmaReads",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMARXSNDS].name		= "InRdmaSends",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMARXWRS].name		= "InRdmaWrites",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMATXRDS].name		= "OutRdmaReads",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMATXSNDS].name		= "OutRdmaSends",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMATXWRS].name		= "OutRdmaWrites",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMAVBND].name		= "RdmaBnd",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMAVINV].name		= "RdmaInv",</div><div class='add'>+</div><div class='add'>+	/* gen2 - 32-bit */</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED].name	= "cnpHandled",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED].name	= "cnpIgnored",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TXNPCNPSENT].name		= "cnpSent",</div><div class='add'>+	/* gen2 - 64-bit */</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4RXMCOCTS].name		= "ip4InMcastOctets",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP4TXMCOCTS].name		= "ip4OutMcastOctets",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6RXMCOCTS].name		= "ip6InMcastOctets",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_IP6TXMCOCTS].name		= "ip6OutMcastOctets",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_UDPRXPKTS].name		= "RxUDP",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_UDPTXPKTS].name		= "TxUDP",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS].name	= "RxECNMrkd",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPRTXSEG].name		= "RetransSegs",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPRXOPTERR].name		= "InOptErrors",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPRXPROTOERR].name	= "InProtoErrors",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPRXSEGS].name		= "InSegs",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_TCPTXSEG].name		= "OutSegs",</div><div class='add'>+</div><div class='add'>+	/* gen3 */</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RNR_SENT].name		= "RNR sent",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RNR_RCVD].name		= "RNR received",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMAORDLMTCNT].name	= "ord limit count",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMAIRDLMTCNT].name	= "ird limit count",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMARXATS].name		= "Rx atomics",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RDMATXATS].name		= "Tx atomics",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_NAKSEQERR].name		= "Nak Sequence Error",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_NAKSEQERR_IMPLIED].name	= "Nak Sequence Error Implied",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RTO].name			= "RTO",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_RXOOOPKTS].name		= "Rcvd Out of order packets",</div><div class='add'>+	[IRDMA_HW_STAT_INDEX_ICRCERR].name		= "CRC errors",</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int irdma_roce_port_immutable(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+				     struct ib_port_immutable *immutable)</div><div class='add'>+{</div><div class='add'>+	struct ib_port_attr attr;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	immutable-&gt;core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;</div><div class='add'>+	err = ib_query_port(ibdev, port_num, &amp;attr);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	immutable-&gt;max_mad_size = IB_MGMT_MAD_SIZE;</div><div class='add'>+	immutable-&gt;pkey_tbl_len = attr.pkey_tbl_len;</div><div class='add'>+	immutable-&gt;gid_tbl_len = attr.gid_tbl_len;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_iw_port_immutable(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+				   struct ib_port_immutable *immutable)</div><div class='add'>+{</div><div class='add'>+	struct ib_port_attr attr;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	immutable-&gt;core_cap_flags = RDMA_CORE_PORT_IWARP;</div><div class='add'>+	err = ib_query_port(ibdev, port_num, &amp;attr);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+	immutable-&gt;gid_tbl_len = attr.gid_tbl_len;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_get_dev_fw_str(struct ib_device *dev, char *str)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(dev);</div><div class='add'>+</div><div class='add'>+	snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u",</div><div class='add'>+		 irdma_fw_major_ver(&amp;iwdev-&gt;rf-&gt;sc_dev),</div><div class='add'>+		 irdma_fw_minor_ver(&amp;iwdev-&gt;rf-&gt;sc_dev));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_alloc_hw_port_stats - Allocate a hw stats structure</div><div class='add'>+ * @ibdev: device pointer from stack</div><div class='add'>+ * @port_num: port number</div><div class='add'>+ */</div><div class='add'>+static struct rdma_hw_stats *irdma_alloc_hw_port_stats(struct ib_device *ibdev,</div><div class='add'>+						       u32 port_num)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibdev);</div><div class='add'>+	struct irdma_sc_dev *dev = &amp;iwdev-&gt;rf-&gt;sc_dev;</div><div class='add'>+</div><div class='add'>+	int num_counters = dev-&gt;hw_attrs.max_stat_idx;</div><div class='add'>+	unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN;</div><div class='add'>+</div><div class='add'>+	return rdma_alloc_hw_stats_struct(irdma_hw_stat_descs, num_counters,</div><div class='add'>+					  lifespan);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_get_hw_stats - Populates the rdma_hw_stats structure</div><div class='add'>+ * @ibdev: device pointer from stack</div><div class='add'>+ * @stats: stats pointer from stack</div><div class='add'>+ * @port_num: port number</div><div class='add'>+ * @index: which hw counter the stack is requesting we update</div><div class='add'>+ */</div><div class='add'>+static int irdma_get_hw_stats(struct ib_device *ibdev,</div><div class='add'>+			      struct rdma_hw_stats *stats, u32 port_num,</div><div class='add'>+			      int index)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibdev);</div><div class='add'>+	struct irdma_dev_hw_stats *hw_stats = &amp;iwdev-&gt;vsi.pestat-&gt;hw_stats;</div><div class='add'>+</div><div class='add'>+	if (iwdev-&gt;rf-&gt;rdma_ver &gt;= IRDMA_GEN_2)</div><div class='add'>+		irdma_cqp_gather_stats_cmd(&amp;iwdev-&gt;rf-&gt;sc_dev, iwdev-&gt;vsi.pestat, true);</div><div class='add'>+	else</div><div class='add'>+		irdma_cqp_gather_stats_gen1(&amp;iwdev-&gt;rf-&gt;sc_dev, iwdev-&gt;vsi.pestat);</div><div class='add'>+</div><div class='add'>+	memcpy(&amp;stats-&gt;value[0], hw_stats, sizeof(u64) * stats-&gt;num_counters);</div><div class='add'>+</div><div class='add'>+	return stats-&gt;num_counters;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_query_gid - Query port GID</div><div class='add'>+ * @ibdev: device pointer from stack</div><div class='add'>+ * @port: port number</div><div class='add'>+ * @index: Entry index</div><div class='add'>+ * @gid: Global ID</div><div class='add'>+ */</div><div class='add'>+static int irdma_query_gid(struct ib_device *ibdev, u32 port, int index,</div><div class='add'>+			   union ib_gid *gid)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibdev);</div><div class='add'>+</div><div class='add'>+	memset(gid-&gt;raw, 0, sizeof(gid-&gt;raw));</div><div class='add'>+	ether_addr_copy(gid-&gt;raw, iwdev-&gt;netdev-&gt;dev_addr);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * mcast_list_add -  Add a new mcast item to list</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @new_elem: pointer to element to add</div><div class='add'>+ */</div><div class='add'>+static void mcast_list_add(struct irdma_pci_f *rf,</div><div class='add'>+			   struct mc_table_list *new_elem)</div><div class='add'>+{</div><div class='add'>+	list_add(&amp;new_elem-&gt;list, &amp;rf-&gt;mc_qht_list.list);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * mcast_list_del - Remove an mcast item from list</div><div class='add'>+ * @mc_qht_elem: pointer to mcast table list element</div><div class='add'>+ */</div><div class='add'>+static void mcast_list_del(struct mc_table_list *mc_qht_elem)</div><div class='add'>+{</div><div class='add'>+	if (mc_qht_elem)</div><div class='add'>+		list_del(&amp;mc_qht_elem-&gt;list);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * mcast_list_lookup_ip - Search mcast list for address</div><div class='add'>+ * @rf: RDMA PCI function</div><div class='add'>+ * @ip_mcast: pointer to mcast IP address</div><div class='add'>+ */</div><div class='add'>+static struct mc_table_list *mcast_list_lookup_ip(struct irdma_pci_f *rf,</div><div class='add'>+						  u32 *ip_mcast)</div><div class='add'>+{</div><div class='add'>+	struct mc_table_list *mc_qht_el;</div><div class='add'>+	struct list_head *pos, *q;</div><div class='add'>+</div><div class='add'>+	list_for_each_safe (pos, q, &amp;rf-&gt;mc_qht_list.list) {</div><div class='add'>+		mc_qht_el = list_entry(pos, struct mc_table_list, list);</div><div class='add'>+		if (!memcmp(mc_qht_el-&gt;mc_info.dest_ip, ip_mcast,</div><div class='add'>+			    sizeof(mc_qht_el-&gt;mc_info.dest_ip)))</div><div class='add'>+			return mc_qht_el;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_mcast_cqp_op - perform a mcast cqp operation</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ * @mc_grp_ctx: mcast group info</div><div class='add'>+ * @op: operation</div><div class='add'>+ *</div><div class='add'>+ * returns error status</div><div class='add'>+ */</div><div class='add'>+static int irdma_mcast_cqp_op(struct irdma_device *iwdev,</div><div class='add'>+			      struct irdma_mcast_grp_info *mc_grp_ctx, u8 op)</div><div class='add'>+{</div><div class='add'>+	struct cqp_cmds_info *cqp_info;</div><div class='add'>+	struct irdma_cqp_request *cqp_request;</div><div class='add'>+	int status;</div><div class='add'>+</div><div class='add'>+	cqp_request = irdma_alloc_and_get_cqp_request(&amp;iwdev-&gt;rf-&gt;cqp, true);</div><div class='add'>+	if (!cqp_request)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cqp_request-&gt;info.in.u.mc_create.info = *mc_grp_ctx;</div><div class='add'>+	cqp_info = &amp;cqp_request-&gt;info;</div><div class='add'>+	cqp_info-&gt;cqp_cmd = op;</div><div class='add'>+	cqp_info-&gt;post_sq = 1;</div><div class='add'>+	cqp_info-&gt;in.u.mc_create.scratch = (uintptr_t)cqp_request;</div><div class='add'>+	cqp_info-&gt;in.u.mc_create.cqp = &amp;iwdev-&gt;rf-&gt;cqp.sc_cqp;</div><div class='add'>+	status = irdma_handle_cqp_op(iwdev-&gt;rf, cqp_request);</div><div class='add'>+	irdma_put_cqp_request(&amp;iwdev-&gt;rf-&gt;cqp, cqp_request);</div><div class='add'>+</div><div class='add'>+	return status;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_mcast_mac - Get the multicast MAC for an IP address</div><div class='add'>+ * @ip_addr: IPv4 or IPv6 address</div><div class='add'>+ * @mac: pointer to result MAC address</div><div class='add'>+ * @ipv4: flag indicating IPv4 or IPv6</div><div class='add'>+ *</div><div class='add'>+ */</div><div class='add'>+void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4)</div><div class='add'>+{</div><div class='add'>+	u8 *ip = (u8 *)ip_addr;</div><div class='add'>+</div><div class='add'>+	if (ipv4) {</div><div class='add'>+		unsigned char mac4[ETH_ALEN] = {0x01, 0x00, 0x5E, 0x00,</div><div class='add'>+						0x00, 0x00};</div><div class='add'>+</div><div class='add'>+		mac4[3] = ip[2] &amp; 0x7F;</div><div class='add'>+		mac4[4] = ip[1];</div><div class='add'>+		mac4[5] = ip[0];</div><div class='add'>+		ether_addr_copy(mac, mac4);</div><div class='add'>+	} else {</div><div class='add'>+		unsigned char mac6[ETH_ALEN] = {0x33, 0x33, 0x00, 0x00,</div><div class='add'>+						0x00, 0x00};</div><div class='add'>+</div><div class='add'>+		mac6[2] = ip[3];</div><div class='add'>+		mac6[3] = ip[2];</div><div class='add'>+		mac6[4] = ip[1];</div><div class='add'>+		mac6[5] = ip[0];</div><div class='add'>+		ether_addr_copy(mac, mac6);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_attach_mcast - attach a qp to a multicast group</div><div class='add'>+ * @ibqp: ptr to qp</div><div class='add'>+ * @ibgid: pointer to global ID</div><div class='add'>+ * @lid: local ID</div><div class='add'>+ *</div><div class='add'>+ * returns error status</div><div class='add'>+ */</div><div class='add'>+static int irdma_attach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *iwqp = to_iwqp(ibqp);</div><div class='add'>+	struct irdma_device *iwdev = iwqp-&gt;iwdev;</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+	struct mc_table_list *mc_qht_elem;</div><div class='add'>+	struct irdma_mcast_grp_ctx_entry_info mcg_info = {};</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	u32 ip_addr[4] = {};</div><div class='add'>+	u32 mgn;</div><div class='add'>+	u32 no_mgs;</div><div class='add'>+	int ret = 0;</div><div class='add'>+	bool ipv4;</div><div class='add'>+	u16 vlan_id;</div><div class='add'>+	union irdma_sockaddr sgid_addr;</div><div class='add'>+	unsigned char dmac[ETH_ALEN];</div><div class='add'>+</div><div class='add'>+	rdma_gid2ip((struct sockaddr *)&amp;sgid_addr, ibgid);</div><div class='add'>+</div><div class='add'>+	if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) {</div><div class='add'>+		irdma_copy_ip_ntohl(ip_addr,</div><div class='add'>+				    sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32);</div><div class='add'>+		irdma_get_vlan_mac_ipv6(ip_addr, &amp;vlan_id, NULL);</div><div class='add'>+		ipv4 = false;</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+			  "VERBS: qp_id=%d, IP6address=%pI6\n", ibqp-&gt;qp_num,</div><div class='add'>+			  ip_addr);</div><div class='add'>+		irdma_mcast_mac(ip_addr, dmac, false);</div><div class='add'>+	} else {</div><div class='add'>+		ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr);</div><div class='add'>+		ipv4 = true;</div><div class='add'>+		vlan_id = irdma_get_vlan_ipv4(ip_addr);</div><div class='add'>+		irdma_mcast_mac(ip_addr, dmac, true);</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+			  "VERBS: qp_id=%d, IP4address=%pI4, MAC=%pM\n",</div><div class='add'>+			  ibqp-&gt;qp_num, ip_addr, dmac);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;rf-&gt;qh_list_lock, flags);</div><div class='add'>+	mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr);</div><div class='add'>+	if (!mc_qht_elem) {</div><div class='add'>+		struct irdma_dma_mem *dma_mem_mc;</div><div class='add'>+</div><div class='add'>+		spin_unlock_irqrestore(&amp;rf-&gt;qh_list_lock, flags);</div><div class='add'>+		mc_qht_elem = kzalloc(sizeof(*mc_qht_elem), GFP_KERNEL);</div><div class='add'>+		if (!mc_qht_elem)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+</div><div class='add'>+		mc_qht_elem-&gt;mc_info.ipv4_valid = ipv4;</div><div class='add'>+		memcpy(mc_qht_elem-&gt;mc_info.dest_ip, ip_addr,</div><div class='add'>+		       sizeof(mc_qht_elem-&gt;mc_info.dest_ip));</div><div class='add'>+		ret = irdma_alloc_rsrc(rf, rf-&gt;allocated_mcgs, rf-&gt;max_mcg,</div><div class='add'>+				       &amp;mgn, &amp;rf-&gt;next_mcg);</div><div class='add'>+		if (ret) {</div><div class='add'>+			kfree(mc_qht_elem);</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		mc_qht_elem-&gt;mc_info.mgn = mgn;</div><div class='add'>+		dma_mem_mc = &amp;mc_qht_elem-&gt;mc_grp_ctx.dma_mem_mc;</div><div class='add'>+		dma_mem_mc-&gt;size = ALIGN(sizeof(u64) * IRDMA_MAX_MGS_PER_CTX,</div><div class='add'>+					 IRDMA_HW_PAGE_SIZE);</div><div class='add'>+		dma_mem_mc-&gt;va = dma_alloc_coherent(rf-&gt;hw.device,</div><div class='add'>+						    dma_mem_mc-&gt;size,</div><div class='add'>+						    &amp;dma_mem_mc-&gt;pa,</div><div class='add'>+						    GFP_KERNEL);</div><div class='add'>+		if (!dma_mem_mc-&gt;va) {</div><div class='add'>+			irdma_free_rsrc(rf, rf-&gt;allocated_mcgs, mgn);</div><div class='add'>+			kfree(mc_qht_elem);</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		mc_qht_elem-&gt;mc_grp_ctx.mg_id = (u16)mgn;</div><div class='add'>+		memcpy(mc_qht_elem-&gt;mc_grp_ctx.dest_ip_addr, ip_addr,</div><div class='add'>+		       sizeof(mc_qht_elem-&gt;mc_grp_ctx.dest_ip_addr));</div><div class='add'>+		mc_qht_elem-&gt;mc_grp_ctx.ipv4_valid = ipv4;</div><div class='add'>+		mc_qht_elem-&gt;mc_grp_ctx.vlan_id = vlan_id;</div><div class='add'>+		if (vlan_id &lt; VLAN_N_VID)</div><div class='add'>+			mc_qht_elem-&gt;mc_grp_ctx.vlan_valid = true;</div><div class='add'>+		mc_qht_elem-&gt;mc_grp_ctx.hmc_fcn_id = iwdev-&gt;rf-&gt;sc_dev.hmc_fn_id;</div><div class='add'>+		mc_qht_elem-&gt;mc_grp_ctx.qs_handle =</div><div class='add'>+			iwqp-&gt;sc_qp.vsi-&gt;qos[iwqp-&gt;sc_qp.user_pri].qs_handle;</div><div class='add'>+		ether_addr_copy(mc_qht_elem-&gt;mc_grp_ctx.dest_mac_addr, dmac);</div><div class='add'>+</div><div class='add'>+		spin_lock_irqsave(&amp;rf-&gt;qh_list_lock, flags);</div><div class='add'>+		mcast_list_add(rf, mc_qht_elem);</div><div class='add'>+	} else {</div><div class='add'>+		if (mc_qht_elem-&gt;mc_grp_ctx.no_of_mgs ==</div><div class='add'>+		    IRDMA_MAX_MGS_PER_CTX) {</div><div class='add'>+			spin_unlock_irqrestore(&amp;rf-&gt;qh_list_lock, flags);</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mcg_info.qp_id = iwqp-&gt;ibqp.qp_num;</div><div class='add'>+	no_mgs = mc_qht_elem-&gt;mc_grp_ctx.no_of_mgs;</div><div class='add'>+	irdma_sc_add_mcast_grp(&amp;mc_qht_elem-&gt;mc_grp_ctx, &amp;mcg_info);</div><div class='add'>+	spin_unlock_irqrestore(&amp;rf-&gt;qh_list_lock, flags);</div><div class='add'>+</div><div class='add'>+	/* Only if there is a change do we need to modify or create */</div><div class='add'>+	if (!no_mgs) {</div><div class='add'>+		ret = irdma_mcast_cqp_op(iwdev, &amp;mc_qht_elem-&gt;mc_grp_ctx,</div><div class='add'>+					 IRDMA_OP_MC_CREATE);</div><div class='add'>+	} else if (no_mgs != mc_qht_elem-&gt;mc_grp_ctx.no_of_mgs) {</div><div class='add'>+		ret = irdma_mcast_cqp_op(iwdev, &amp;mc_qht_elem-&gt;mc_grp_ctx,</div><div class='add'>+					 IRDMA_OP_MC_MODIFY);</div><div class='add'>+	} else {</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ret)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+error:</div><div class='add'>+	irdma_sc_del_mcast_grp(&amp;mc_qht_elem-&gt;mc_grp_ctx, &amp;mcg_info);</div><div class='add'>+	if (!mc_qht_elem-&gt;mc_grp_ctx.no_of_mgs) {</div><div class='add'>+		mcast_list_del(mc_qht_elem);</div><div class='add'>+		dma_free_coherent(rf-&gt;hw.device,</div><div class='add'>+				  mc_qht_elem-&gt;mc_grp_ctx.dma_mem_mc.size,</div><div class='add'>+				  mc_qht_elem-&gt;mc_grp_ctx.dma_mem_mc.va,</div><div class='add'>+				  mc_qht_elem-&gt;mc_grp_ctx.dma_mem_mc.pa);</div><div class='add'>+		mc_qht_elem-&gt;mc_grp_ctx.dma_mem_mc.va = NULL;</div><div class='add'>+		irdma_free_rsrc(rf, rf-&gt;allocated_mcgs,</div><div class='add'>+				mc_qht_elem-&gt;mc_grp_ctx.mg_id);</div><div class='add'>+		kfree(mc_qht_elem);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_detach_mcast - detach a qp from a multicast group</div><div class='add'>+ * @ibqp: ptr to qp</div><div class='add'>+ * @ibgid: pointer to global ID</div><div class='add'>+ * @lid: local ID</div><div class='add'>+ *</div><div class='add'>+ * returns error status</div><div class='add'>+ */</div><div class='add'>+static int irdma_detach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid)</div><div class='add'>+{</div><div class='add'>+	struct irdma_qp *iwqp = to_iwqp(ibqp);</div><div class='add'>+	struct irdma_device *iwdev = iwqp-&gt;iwdev;</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+	u32 ip_addr[4] = {};</div><div class='add'>+	struct mc_table_list *mc_qht_elem;</div><div class='add'>+	struct irdma_mcast_grp_ctx_entry_info mcg_info = {};</div><div class='add'>+	int ret;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	union irdma_sockaddr sgid_addr;</div><div class='add'>+</div><div class='add'>+	rdma_gid2ip((struct sockaddr *)&amp;sgid_addr, ibgid);</div><div class='add'>+	if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid))</div><div class='add'>+		irdma_copy_ip_ntohl(ip_addr,</div><div class='add'>+				    sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32);</div><div class='add'>+	else</div><div class='add'>+		ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr);</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;rf-&gt;qh_list_lock, flags);</div><div class='add'>+	mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr);</div><div class='add'>+	if (!mc_qht_elem) {</div><div class='add'>+		spin_unlock_irqrestore(&amp;rf-&gt;qh_list_lock, flags);</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+			  "VERBS: address not found MCG\n");</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mcg_info.qp_id = iwqp-&gt;ibqp.qp_num;</div><div class='add'>+	irdma_sc_del_mcast_grp(&amp;mc_qht_elem-&gt;mc_grp_ctx, &amp;mcg_info);</div><div class='add'>+	if (!mc_qht_elem-&gt;mc_grp_ctx.no_of_mgs) {</div><div class='add'>+		mcast_list_del(mc_qht_elem);</div><div class='add'>+		spin_unlock_irqrestore(&amp;rf-&gt;qh_list_lock, flags);</div><div class='add'>+		ret = irdma_mcast_cqp_op(iwdev, &amp;mc_qht_elem-&gt;mc_grp_ctx,</div><div class='add'>+					 IRDMA_OP_MC_DESTROY);</div><div class='add'>+		if (ret) {</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+				  "VERBS: failed MC_DESTROY MCG\n");</div><div class='add'>+			spin_lock_irqsave(&amp;rf-&gt;qh_list_lock, flags);</div><div class='add'>+			mcast_list_add(rf, mc_qht_elem);</div><div class='add'>+			spin_unlock_irqrestore(&amp;rf-&gt;qh_list_lock, flags);</div><div class='add'>+			return -EAGAIN;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		dma_free_coherent(rf-&gt;hw.device,</div><div class='add'>+				  mc_qht_elem-&gt;mc_grp_ctx.dma_mem_mc.size,</div><div class='add'>+				  mc_qht_elem-&gt;mc_grp_ctx.dma_mem_mc.va,</div><div class='add'>+				  mc_qht_elem-&gt;mc_grp_ctx.dma_mem_mc.pa);</div><div class='add'>+		mc_qht_elem-&gt;mc_grp_ctx.dma_mem_mc.va = NULL;</div><div class='add'>+		irdma_free_rsrc(rf, rf-&gt;allocated_mcgs,</div><div class='add'>+				mc_qht_elem-&gt;mc_grp_ctx.mg_id);</div><div class='add'>+		kfree(mc_qht_elem);</div><div class='add'>+	} else {</div><div class='add'>+		spin_unlock_irqrestore(&amp;rf-&gt;qh_list_lock, flags);</div><div class='add'>+		ret = irdma_mcast_cqp_op(iwdev, &amp;mc_qht_elem-&gt;mc_grp_ctx,</div><div class='add'>+					 IRDMA_OP_MC_MODIFY);</div><div class='add'>+		if (ret) {</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev,</div><div class='add'>+				  "VERBS: failed Modify MCG\n");</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_create_hw_ah(struct irdma_device *iwdev, struct irdma_ah *ah, bool sleep)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	err = irdma_alloc_rsrc(rf, rf-&gt;allocated_ahs, rf-&gt;max_ah, &amp;ah-&gt;sc_ah.ah_info.ah_idx,</div><div class='add'>+			       &amp;rf-&gt;next_ah);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = irdma_ah_cqp_op(rf, &amp;ah-&gt;sc_ah, IRDMA_OP_AH_CREATE, sleep,</div><div class='add'>+			      irdma_gsi_ud_qp_ah_cb, &amp;ah-&gt;sc_ah);</div><div class='add'>+</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev, "VERBS: CQP-OP Create AH fail");</div><div class='add'>+		goto err_ah_create;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!sleep) {</div><div class='add'>+		int cnt = CQP_COMPL_WAIT_TIME_MS * CQP_TIMEOUT_THRESHOLD;</div><div class='add'>+</div><div class='add'>+		do {</div><div class='add'>+			irdma_cqp_ce_handler(rf, &amp;rf-&gt;ccq.sc_cq);</div><div class='add'>+			mdelay(1);</div><div class='add'>+		} while (!ah-&gt;sc_ah.ah_info.ah_valid &amp;&amp; --cnt);</div><div class='add'>+</div><div class='add'>+		if (!cnt) {</div><div class='add'>+			ibdev_dbg(&amp;iwdev-&gt;ibdev, "VERBS: CQP create AH timed out");</div><div class='add'>+			err = -ETIMEDOUT;</div><div class='add'>+			goto err_ah_create;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_ah_create:</div><div class='add'>+	irdma_free_rsrc(iwdev-&gt;rf, iwdev-&gt;rf-&gt;allocated_ahs, ah-&gt;sc_ah.ah_info.ah_idx);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_setup_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_pd *pd = to_iwpd(ibah-&gt;pd);</div><div class='add'>+	struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah);</div><div class='add'>+	struct rdma_ah_attr *ah_attr = attr-&gt;ah_attr;</div><div class='add'>+	const struct ib_gid_attr *sgid_attr;</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibah-&gt;pd-&gt;device);</div><div class='add'>+	struct irdma_pci_f *rf = iwdev-&gt;rf;</div><div class='add'>+	struct irdma_sc_ah *sc_ah;</div><div class='add'>+	struct irdma_ah_info *ah_info;</div><div class='add'>+	union irdma_sockaddr sgid_addr, dgid_addr;</div><div class='add'>+	int err;</div><div class='add'>+	u8 dmac[ETH_ALEN];</div><div class='add'>+</div><div class='add'>+	ah-&gt;pd = pd;</div><div class='add'>+	sc_ah = &amp;ah-&gt;sc_ah;</div><div class='add'>+	sc_ah-&gt;ah_info.vsi = &amp;iwdev-&gt;vsi;</div><div class='add'>+	irdma_sc_init_ah(&amp;rf-&gt;sc_dev, sc_ah);</div><div class='add'>+	ah-&gt;sgid_index = ah_attr-&gt;grh.sgid_index;</div><div class='add'>+	sgid_attr = ah_attr-&gt;grh.sgid_attr;</div><div class='add'>+	memcpy(&amp;ah-&gt;dgid, &amp;ah_attr-&gt;grh.dgid, sizeof(ah-&gt;dgid));</div><div class='add'>+	rdma_gid2ip((struct sockaddr *)&amp;sgid_addr, &amp;sgid_attr-&gt;gid);</div><div class='add'>+	rdma_gid2ip((struct sockaddr *)&amp;dgid_addr, &amp;ah_attr-&gt;grh.dgid);</div><div class='add'>+	ah-&gt;av.attrs = *ah_attr;</div><div class='add'>+	ah-&gt;av.net_type = rdma_gid_attr_network_type(sgid_attr);</div><div class='add'>+	ah_info = &amp;sc_ah-&gt;ah_info;</div><div class='add'>+	ah_info-&gt;pd_idx = pd-&gt;sc_pd.pd_id;</div><div class='add'>+	if (ah_attr-&gt;ah_flags &amp; IB_AH_GRH) {</div><div class='add'>+		ah_info-&gt;flow_label = ah_attr-&gt;grh.flow_label;</div><div class='add'>+		ah_info-&gt;hop_ttl = ah_attr-&gt;grh.hop_limit;</div><div class='add'>+		ah_info-&gt;tc_tos = ah_attr-&gt;grh.traffic_class;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ether_addr_copy(dmac, ah_attr-&gt;roce.dmac);</div><div class='add'>+	if (ah-&gt;av.net_type == RDMA_NETWORK_IPV4) {</div><div class='add'>+		ah_info-&gt;ipv4_valid = true;</div><div class='add'>+		ah_info-&gt;dest_ip_addr[0] =</div><div class='add'>+			ntohl(dgid_addr.saddr_in.sin_addr.s_addr);</div><div class='add'>+		ah_info-&gt;src_ip_addr[0] =</div><div class='add'>+			ntohl(sgid_addr.saddr_in.sin_addr.s_addr);</div><div class='add'>+		ah_info-&gt;do_lpbk = irdma_ipv4_is_lpb(ah_info-&gt;src_ip_addr[0],</div><div class='add'>+						     ah_info-&gt;dest_ip_addr[0]);</div><div class='add'>+		if (ipv4_is_multicast(dgid_addr.saddr_in.sin_addr.s_addr)) {</div><div class='add'>+			ah_info-&gt;do_lpbk = true;</div><div class='add'>+			irdma_mcast_mac(ah_info-&gt;dest_ip_addr, dmac, true);</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		irdma_copy_ip_ntohl(ah_info-&gt;dest_ip_addr,</div><div class='add'>+				    dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32);</div><div class='add'>+		irdma_copy_ip_ntohl(ah_info-&gt;src_ip_addr,</div><div class='add'>+				    sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32);</div><div class='add'>+		ah_info-&gt;do_lpbk = irdma_ipv6_is_lpb(ah_info-&gt;src_ip_addr,</div><div class='add'>+						     ah_info-&gt;dest_ip_addr);</div><div class='add'>+		if (rdma_is_multicast_addr(&amp;dgid_addr.saddr_in6.sin6_addr)) {</div><div class='add'>+			ah_info-&gt;do_lpbk = true;</div><div class='add'>+			irdma_mcast_mac(ah_info-&gt;dest_ip_addr, dmac, false);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = rdma_read_gid_l2_fields(sgid_attr, &amp;ah_info-&gt;vlan_tag,</div><div class='add'>+				      ah_info-&gt;mac_addr);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	ah_info-&gt;dst_arpindex = irdma_add_arp(iwdev-&gt;rf, ah_info-&gt;dest_ip_addr,</div><div class='add'>+					      ah_info-&gt;ipv4_valid, dmac);</div><div class='add'>+</div><div class='add'>+	if (ah_info-&gt;dst_arpindex == -1)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (ah_info-&gt;vlan_tag &gt;= VLAN_N_VID &amp;&amp; iwdev-&gt;dcb_vlan_mode)</div><div class='add'>+		ah_info-&gt;vlan_tag = 0;</div><div class='add'>+</div><div class='add'>+	if (ah_info-&gt;vlan_tag &lt; VLAN_N_VID) {</div><div class='add'>+		u8 prio = rt_tos2priority(ah_info-&gt;tc_tos);</div><div class='add'>+</div><div class='add'>+		prio = irdma_roce_get_vlan_prio(sgid_attr, prio);</div><div class='add'>+</div><div class='add'>+		ah_info-&gt;vlan_tag |= (u16)prio &lt;&lt; VLAN_PRIO_SHIFT;</div><div class='add'>+		ah_info-&gt;insert_vlan_tag = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ah_exists - Check for existing identical AH</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ * @new_ah: AH to check for</div><div class='add'>+ *</div><div class='add'>+ * returns true if AH is found, false if not found.</div><div class='add'>+ */</div><div class='add'>+static bool irdma_ah_exists(struct irdma_device *iwdev,</div><div class='add'>+			    struct irdma_ah *new_ah)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ah *ah;</div><div class='add'>+	u32 key = new_ah-&gt;sc_ah.ah_info.dest_ip_addr[0] ^</div><div class='add'>+		  new_ah-&gt;sc_ah.ah_info.dest_ip_addr[1] ^</div><div class='add'>+		  new_ah-&gt;sc_ah.ah_info.dest_ip_addr[2] ^</div><div class='add'>+		  new_ah-&gt;sc_ah.ah_info.dest_ip_addr[3];</div><div class='add'>+</div><div class='add'>+	hash_for_each_possible(iwdev-&gt;rf-&gt;ah_hash_tbl, ah, list, key) {</div><div class='add'>+		/* Set ah_valid and ah_id the same so memcmp can work */</div><div class='add'>+		new_ah-&gt;sc_ah.ah_info.ah_idx = ah-&gt;sc_ah.ah_info.ah_idx;</div><div class='add'>+		new_ah-&gt;sc_ah.ah_info.ah_valid = ah-&gt;sc_ah.ah_info.ah_valid;</div><div class='add'>+		if (!memcmp(&amp;ah-&gt;sc_ah.ah_info, &amp;new_ah-&gt;sc_ah.ah_info,</div><div class='add'>+			    sizeof(ah-&gt;sc_ah.ah_info))) {</div><div class='add'>+			refcount_inc(&amp;ah-&gt;refcnt);</div><div class='add'>+			new_ah-&gt;parent_ah = ah;</div><div class='add'>+			return true;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_destroy_ah - Destroy address handle</div><div class='add'>+ * @ibah: pointer to address handle</div><div class='add'>+ * @ah_flags: flags for sleepable</div><div class='add'>+ */</div><div class='add'>+static int irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibah-&gt;device);</div><div class='add'>+	struct irdma_ah *ah = to_iwah(ibah);</div><div class='add'>+</div><div class='add'>+	if ((ah_flags &amp; RDMA_DESTROY_AH_SLEEPABLE) &amp;&amp; ah-&gt;parent_ah) {</div><div class='add'>+		mutex_lock(&amp;iwdev-&gt;rf-&gt;ah_tbl_lock);</div><div class='add'>+		if (!refcount_dec_and_test(&amp;ah-&gt;parent_ah-&gt;refcnt)) {</div><div class='add'>+			mutex_unlock(&amp;iwdev-&gt;rf-&gt;ah_tbl_lock);</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+		hash_del(&amp;ah-&gt;parent_ah-&gt;list);</div><div class='add'>+		kfree(ah-&gt;parent_ah);</div><div class='add'>+		mutex_unlock(&amp;iwdev-&gt;rf-&gt;ah_tbl_lock);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	irdma_ah_cqp_op(iwdev-&gt;rf, &amp;ah-&gt;sc_ah, IRDMA_OP_AH_DESTROY,</div><div class='add'>+			false, NULL, ah);</div><div class='add'>+</div><div class='add'>+	irdma_free_rsrc(iwdev-&gt;rf, iwdev-&gt;rf-&gt;allocated_ahs,</div><div class='add'>+			ah-&gt;sc_ah.ah_info.ah_idx);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_create_user_ah - create user address handle</div><div class='add'>+ * @ibah: address handle</div><div class='add'>+ * @attr: address handle attributes</div><div class='add'>+ * @udata: User data</div><div class='add'>+ *</div><div class='add'>+ * returns 0 on success, error otherwise</div><div class='add'>+ */</div><div class='add'>+static int irdma_create_user_ah(struct ib_ah *ibah,</div><div class='add'>+				struct rdma_ah_init_attr *attr,</div><div class='add'>+				struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+#define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd)</div><div class='add'>+	struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah);</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibah-&gt;pd-&gt;device);</div><div class='add'>+	struct irdma_create_ah_resp uresp;</div><div class='add'>+	struct irdma_ah *parent_ah;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (udata-&gt;outlen &lt; IRDMA_CREATE_AH_MIN_RESP_LEN)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	err = irdma_setup_ah(ibah, attr);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+	mutex_lock(&amp;iwdev-&gt;rf-&gt;ah_tbl_lock);</div><div class='add'>+	if (!irdma_ah_exists(iwdev, ah)) {</div><div class='add'>+		err = irdma_create_hw_ah(iwdev, ah, true);</div><div class='add'>+		if (err) {</div><div class='add'>+			mutex_unlock(&amp;iwdev-&gt;rf-&gt;ah_tbl_lock);</div><div class='add'>+			return err;</div><div class='add'>+		}</div><div class='add'>+		/* Add new AH to list */</div><div class='add'>+		parent_ah = kmemdup(ah, sizeof(*ah), GFP_KERNEL);</div><div class='add'>+		if (parent_ah) {</div><div class='add'>+			u32 key = parent_ah-&gt;sc_ah.ah_info.dest_ip_addr[0] ^</div><div class='add'>+				  parent_ah-&gt;sc_ah.ah_info.dest_ip_addr[1] ^</div><div class='add'>+				  parent_ah-&gt;sc_ah.ah_info.dest_ip_addr[2] ^</div><div class='add'>+				  parent_ah-&gt;sc_ah.ah_info.dest_ip_addr[3];</div><div class='add'>+</div><div class='add'>+			ah-&gt;parent_ah = parent_ah;</div><div class='add'>+			hash_add(iwdev-&gt;rf-&gt;ah_hash_tbl, &amp;parent_ah-&gt;list, key);</div><div class='add'>+			refcount_set(&amp;parent_ah-&gt;refcnt, 1);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;iwdev-&gt;rf-&gt;ah_tbl_lock);</div><div class='add'>+</div><div class='add'>+	uresp.ah_id = ah-&gt;sc_ah.ah_info.ah_idx;</div><div class='add'>+	err = ib_copy_to_udata(udata, &amp;uresp, min(sizeof(uresp), udata-&gt;outlen));</div><div class='add'>+	if (err)</div><div class='add'>+		irdma_destroy_ah(ibah, attr-&gt;flags);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_create_ah - create address handle</div><div class='add'>+ * @ibah: address handle</div><div class='add'>+ * @attr: address handle attributes</div><div class='add'>+ * @udata: NULL</div><div class='add'>+ *</div><div class='add'>+ * returns 0 on success, error otherwise</div><div class='add'>+ */</div><div class='add'>+static int irdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr,</div><div class='add'>+			   struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah);</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibah-&gt;pd-&gt;device);</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	err = irdma_setup_ah(ibah, attr);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+	err = irdma_create_hw_ah(iwdev, ah, attr-&gt;flags &amp; RDMA_CREATE_AH_SLEEPABLE);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_query_ah - Query address handle</div><div class='add'>+ * @ibah: pointer to address handle</div><div class='add'>+ * @ah_attr: address handle attributes</div><div class='add'>+ */</div><div class='add'>+static int irdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ah *ah = to_iwah(ibah);</div><div class='add'>+</div><div class='add'>+	memset(ah_attr, 0, sizeof(*ah_attr));</div><div class='add'>+	if (ah-&gt;av.attrs.ah_flags &amp; IB_AH_GRH) {</div><div class='add'>+		ah_attr-&gt;ah_flags = IB_AH_GRH;</div><div class='add'>+		ah_attr-&gt;grh.flow_label = ah-&gt;sc_ah.ah_info.flow_label;</div><div class='add'>+		ah_attr-&gt;grh.traffic_class = ah-&gt;sc_ah.ah_info.tc_tos;</div><div class='add'>+		ah_attr-&gt;grh.hop_limit = ah-&gt;sc_ah.ah_info.hop_ttl;</div><div class='add'>+		ah_attr-&gt;grh.sgid_index = ah-&gt;sgid_index;</div><div class='add'>+		memcpy(&amp;ah_attr-&gt;grh.dgid, &amp;ah-&gt;dgid,</div><div class='add'>+		       sizeof(ah_attr-&gt;grh.dgid));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev,</div><div class='add'>+						 u32 port_num)</div><div class='add'>+{</div><div class='add'>+	return IB_LINK_LAYER_ETHERNET;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops irdma_gen1_dev_ops = {</div><div class='add'>+	.dealloc_driver = irdma_ib_dealloc_device,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops irdma_gen3_dev_ops = {</div><div class='add'>+	.alloc_mw = irdma_alloc_mw,</div><div class='add'>+	.create_srq = irdma_create_srq,</div><div class='add'>+	.dealloc_mw = irdma_dealloc_mw,</div><div class='add'>+	.destroy_srq = irdma_destroy_srq,</div><div class='add'>+	.modify_srq = irdma_modify_srq,</div><div class='add'>+	.post_srq_recv = irdma_post_srq_recv,</div><div class='add'>+	.query_srq = irdma_query_srq,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops irdma_roce_dev_ops = {</div><div class='add'>+	.attach_mcast = irdma_attach_mcast,</div><div class='add'>+	.create_ah = irdma_create_ah,</div><div class='add'>+	.create_user_ah = irdma_create_user_ah,</div><div class='add'>+	.destroy_ah = irdma_destroy_ah,</div><div class='add'>+	.detach_mcast = irdma_detach_mcast,</div><div class='add'>+	.get_link_layer = irdma_get_link_layer,</div><div class='add'>+	.get_port_immutable = irdma_roce_port_immutable,</div><div class='add'>+	.modify_qp = irdma_modify_qp_roce,</div><div class='add'>+	.query_ah = irdma_query_ah,</div><div class='add'>+	.query_pkey = irdma_query_pkey,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops irdma_iw_dev_ops = {</div><div class='add'>+	.get_port_immutable = irdma_iw_port_immutable,</div><div class='add'>+	.iw_accept = irdma_accept,</div><div class='add'>+	.iw_add_ref = irdma_qp_add_ref,</div><div class='add'>+	.iw_connect = irdma_connect,</div><div class='add'>+	.iw_create_listen = irdma_create_listen,</div><div class='add'>+	.iw_destroy_listen = irdma_destroy_listen,</div><div class='add'>+	.iw_get_qp = irdma_get_qp,</div><div class='add'>+	.iw_reject = irdma_reject,</div><div class='add'>+	.iw_rem_ref = irdma_qp_rem_ref,</div><div class='add'>+	.modify_qp = irdma_modify_qp,</div><div class='add'>+	.query_gid = irdma_query_gid,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops irdma_dev_ops = {</div><div class='add'>+	.owner = THIS_MODULE,</div><div class='add'>+	.driver_id = RDMA_DRIVER_IRDMA,</div><div class='add'>+	.uverbs_abi_ver = IRDMA_ABI_VER,</div><div class='add'>+</div><div class='add'>+	.alloc_hw_port_stats = irdma_alloc_hw_port_stats,</div><div class='add'>+	.alloc_mr = irdma_alloc_mr,</div><div class='add'>+	.alloc_pd = irdma_alloc_pd,</div><div class='add'>+	.alloc_ucontext = irdma_alloc_ucontext,</div><div class='add'>+	.create_cq = irdma_create_cq,</div><div class='add'>+	.create_qp = irdma_create_qp,</div><div class='add'>+	.dealloc_driver = irdma_ib_dealloc_device,</div><div class='add'>+	.dealloc_mw = irdma_dealloc_mw,</div><div class='add'>+	.dealloc_pd = irdma_dealloc_pd,</div><div class='add'>+	.dealloc_ucontext = irdma_dealloc_ucontext,</div><div class='add'>+	.dereg_mr = irdma_dereg_mr,</div><div class='add'>+	.destroy_cq = irdma_destroy_cq,</div><div class='add'>+	.destroy_qp = irdma_destroy_qp,</div><div class='add'>+	.disassociate_ucontext = irdma_disassociate_ucontext,</div><div class='add'>+	.get_dev_fw_str = irdma_get_dev_fw_str,</div><div class='add'>+	.get_dma_mr = irdma_get_dma_mr,</div><div class='add'>+	.get_hw_stats = irdma_get_hw_stats,</div><div class='add'>+	.map_mr_sg = irdma_map_mr_sg,</div><div class='add'>+	.mmap = irdma_mmap,</div><div class='add'>+	.mmap_free = irdma_mmap_free,</div><div class='add'>+	.poll_cq = irdma_poll_cq,</div><div class='add'>+	.post_recv = irdma_post_recv,</div><div class='add'>+	.post_send = irdma_post_send,</div><div class='add'>+	.query_device = irdma_query_device,</div><div class='add'>+	.query_port = irdma_query_port,</div><div class='add'>+	.query_qp = irdma_query_qp,</div><div class='add'>+	.reg_user_mr = irdma_reg_user_mr,</div><div class='add'>+	.reg_user_mr_dmabuf = irdma_reg_user_mr_dmabuf,</div><div class='add'>+	.rereg_user_mr = irdma_rereg_user_mr,</div><div class='add'>+	.req_notify_cq = irdma_req_notify_cq,</div><div class='add'>+	.resize_cq = irdma_resize_cq,</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_ucontext, irdma_ucontext, ibucontext),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_ah, irdma_ah, ibah),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_cq, irdma_cq, ibcq),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_mw, irdma_mr, ibmw),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_qp, irdma_qp, ibqp),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_srq, irdma_srq, ibsrq),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_init_roce_device - initialization of roce rdma device</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ */</div><div class='add'>+static void irdma_init_roce_device(struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	iwdev-&gt;ibdev.node_type = RDMA_NODE_IB_CA;</div><div class='add'>+	addrconf_addr_eui48((u8 *)&amp;iwdev-&gt;ibdev.node_guid,</div><div class='add'>+			    iwdev-&gt;netdev-&gt;dev_addr);</div><div class='add'>+	ib_set_device_ops(&amp;iwdev-&gt;ibdev, &amp;irdma_roce_dev_ops);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_init_iw_device - initialization of iwarp rdma device</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ */</div><div class='add'>+static void irdma_init_iw_device(struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	struct net_device *netdev = iwdev-&gt;netdev;</div><div class='add'>+</div><div class='add'>+	iwdev-&gt;ibdev.node_type = RDMA_NODE_RNIC;</div><div class='add'>+	addrconf_addr_eui48((u8 *)&amp;iwdev-&gt;ibdev.node_guid,</div><div class='add'>+			    netdev-&gt;dev_addr);</div><div class='add'>+	memcpy(iwdev-&gt;ibdev.iw_ifname, netdev-&gt;name,</div><div class='add'>+	       sizeof(iwdev-&gt;ibdev.iw_ifname));</div><div class='add'>+	ib_set_device_ops(&amp;iwdev-&gt;ibdev, &amp;irdma_iw_dev_ops);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_init_rdma_device - initialization of rdma device</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ */</div><div class='add'>+static void irdma_init_rdma_device(struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	struct pci_dev *pcidev = iwdev-&gt;rf-&gt;pcidev;</div><div class='add'>+</div><div class='add'>+	if (iwdev-&gt;roce_mode)</div><div class='add'>+		irdma_init_roce_device(iwdev);</div><div class='add'>+	else</div><div class='add'>+		irdma_init_iw_device(iwdev);</div><div class='add'>+</div><div class='add'>+	iwdev-&gt;ibdev.phys_port_cnt = 1;</div><div class='add'>+	iwdev-&gt;ibdev.num_comp_vectors = iwdev-&gt;rf-&gt;ceqs_count;</div><div class='add'>+	iwdev-&gt;ibdev.dev.parent = &amp;pcidev-&gt;dev;</div><div class='add'>+	ib_set_device_ops(&amp;iwdev-&gt;ibdev, &amp;irdma_dev_ops);</div><div class='add'>+	if (iwdev-&gt;rf-&gt;rdma_ver == IRDMA_GEN_1)</div><div class='add'>+		ib_set_device_ops(&amp;iwdev-&gt;ibdev, &amp;irdma_gen1_dev_ops);</div><div class='add'>+	if (iwdev-&gt;rf-&gt;rdma_ver &gt;= IRDMA_GEN_3)</div><div class='add'>+		ib_set_device_ops(&amp;iwdev-&gt;ibdev, &amp;irdma_gen3_dev_ops);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_port_ibevent - indicate port event</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ */</div><div class='add'>+void irdma_port_ibevent(struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	struct ib_event event;</div><div class='add'>+</div><div class='add'>+	event.device = &amp;iwdev-&gt;ibdev;</div><div class='add'>+	event.element.port_num = 1;</div><div class='add'>+	event.event =</div><div class='add'>+		iwdev-&gt;iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;</div><div class='add'>+	ib_dispatch_event(&amp;event);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ib_unregister_device - unregister rdma device from IB</div><div class='add'>+ * core</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ */</div><div class='add'>+void irdma_ib_unregister_device(struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	iwdev-&gt;iw_status = 0;</div><div class='add'>+	irdma_port_ibevent(iwdev);</div><div class='add'>+	ib_unregister_device(&amp;iwdev-&gt;ibdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ib_register_device - register irdma device to IB core</div><div class='add'>+ * @iwdev: irdma device</div><div class='add'>+ */</div><div class='add'>+int irdma_ib_register_device(struct irdma_device *iwdev)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	irdma_init_rdma_device(iwdev);</div><div class='add'>+</div><div class='add'>+	ret = ib_device_set_netdev(&amp;iwdev-&gt;ibdev, iwdev-&gt;netdev, 1);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto error;</div><div class='add'>+	dma_set_max_seg_size(iwdev-&gt;rf-&gt;hw.device, UINT_MAX);</div><div class='add'>+	ret = ib_register_device(&amp;iwdev-&gt;ibdev, "irdma%d", iwdev-&gt;rf-&gt;hw.device);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto error;</div><div class='add'>+</div><div class='add'>+	iwdev-&gt;iw_status = 1;</div><div class='add'>+	irdma_port_ibevent(iwdev);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+error:</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_dbg(&amp;iwdev-&gt;ibdev, "VERBS: Register RDMA device fail\n");</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ib_dealloc_device</div><div class='add'>+ * @ibdev: ib device</div><div class='add'>+ *</div><div class='add'>+ * callback from ibdev dealloc_driver to deallocate resources</div><div class='add'>+ * unber irdma device</div><div class='add'>+ */</div><div class='add'>+void irdma_ib_dealloc_device(struct ib_device *ibdev)</div><div class='add'>+{</div><div class='add'>+	struct irdma_device *iwdev = to_iwdev(ibdev);</div><div class='add'>+</div><div class='add'>+	irdma_rt_deinit_hw(iwdev);</div><div class='add'>+	if (!iwdev-&gt;is_vport) {</div><div class='add'>+		irdma_ctrl_deinit_hw(iwdev-&gt;rf);</div><div class='add'>+		if (iwdev-&gt;rf-&gt;vchnl_wq) {</div><div class='add'>+			destroy_workqueue(iwdev-&gt;rf-&gt;vchnl_wq);</div><div class='add'>+			mutex_destroy(&amp;iwdev-&gt;rf-&gt;sc_dev.vchnl_mutex);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h<br/>new file mode 100644<br/>index 000000000000..aabbb3442098<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/verbs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/verbs.h</a></div><div class='hunk'>@@ -0,0 +1,341 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2015 - 2021 Intel Corporation */</div><div class='add'>+#ifndef IRDMA_VERBS_H</div><div class='add'>+#define IRDMA_VERBS_H</div><div class='add'>+</div><div class='add'>+#define IRDMA_MAX_SAVED_PHY_PGADDR	4</div><div class='add'>+#define IRDMA_FLUSH_DELAY_MS		20</div><div class='add'>+</div><div class='add'>+#define IRDMA_PKEY_TBL_SZ		1</div><div class='add'>+#define IRDMA_DEFAULT_PKEY		0xFFFF</div><div class='add'>+#define IRDMA_SHADOW_PGCNT		1</div><div class='add'>+</div><div class='add'>+struct irdma_ucontext {</div><div class='add'>+	struct ib_ucontext ibucontext;</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct rdma_user_mmap_entry *db_mmap_entry;</div><div class='add'>+	struct list_head cq_reg_mem_list;</div><div class='add'>+	spinlock_t cq_reg_mem_list_lock; /* protect CQ memory list */</div><div class='add'>+	struct list_head qp_reg_mem_list;</div><div class='add'>+	spinlock_t qp_reg_mem_list_lock; /* protect QP memory list */</div><div class='add'>+	struct list_head srq_reg_mem_list;</div><div class='add'>+	spinlock_t srq_reg_mem_list_lock; /* protect SRQ memory list */</div><div class='add'>+	int abi_ver;</div><div class='add'>+	u8 legacy_mode : 1;</div><div class='add'>+	u8 use_raw_attrs : 1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_pd {</div><div class='add'>+	struct ib_pd ibpd;</div><div class='add'>+	struct irdma_sc_pd sc_pd;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+union irdma_sockaddr {</div><div class='add'>+	struct sockaddr_in saddr_in;</div><div class='add'>+	struct sockaddr_in6 saddr_in6;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_av {</div><div class='add'>+	u8 macaddr[16];</div><div class='add'>+	struct rdma_ah_attr attrs;</div><div class='add'>+	union irdma_sockaddr sgid_addr;</div><div class='add'>+	union irdma_sockaddr dgid_addr;</div><div class='add'>+	u8 net_type;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_ah {</div><div class='add'>+	struct ib_ah ibah;</div><div class='add'>+	struct irdma_sc_ah sc_ah;</div><div class='add'>+	struct irdma_pd *pd;</div><div class='add'>+	struct irdma_av av;</div><div class='add'>+	u8 sgid_index;</div><div class='add'>+	union ib_gid dgid;</div><div class='add'>+	struct hlist_node list;</div><div class='add'>+	refcount_t refcnt;</div><div class='add'>+	struct irdma_ah *parent_ah; /* AH from cached list */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_hmc_pble {</div><div class='add'>+	union {</div><div class='add'>+		u32 idx;</div><div class='add'>+		dma_addr_t addr;</div><div class='add'>+	};</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cq_mr {</div><div class='add'>+	struct irdma_hmc_pble cq_pbl;</div><div class='add'>+	dma_addr_t shadow;</div><div class='add'>+	bool split;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_srq_mr {</div><div class='add'>+	struct irdma_hmc_pble srq_pbl;</div><div class='add'>+	dma_addr_t shadow;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_qp_mr {</div><div class='add'>+	struct irdma_hmc_pble sq_pbl;</div><div class='add'>+	struct irdma_hmc_pble rq_pbl;</div><div class='add'>+	dma_addr_t shadow;</div><div class='add'>+	dma_addr_t rq_pa;</div><div class='add'>+	struct page *sq_page;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cq_buf {</div><div class='add'>+	struct irdma_dma_mem kmem_buf;</div><div class='add'>+	struct irdma_cq_uk cq_uk;</div><div class='add'>+	struct irdma_hw *hw;</div><div class='add'>+	struct list_head list;</div><div class='add'>+	struct work_struct work;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_pbl {</div><div class='add'>+	struct list_head list;</div><div class='add'>+	union {</div><div class='add'>+		struct irdma_qp_mr qp_mr;</div><div class='add'>+		struct irdma_cq_mr cq_mr;</div><div class='add'>+		struct irdma_srq_mr srq_mr;</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	bool pbl_allocated:1;</div><div class='add'>+	bool on_list:1;</div><div class='add'>+	u64 user_base;</div><div class='add'>+	struct irdma_pble_alloc pble_alloc;</div><div class='add'>+	struct irdma_mr *iwmr;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_mr {</div><div class='add'>+	union {</div><div class='add'>+		struct ib_mr ibmr;</div><div class='add'>+		struct ib_mw ibmw;</div><div class='add'>+	};</div><div class='add'>+	struct ib_umem *region;</div><div class='add'>+	int access;</div><div class='add'>+	bool is_hwreg:1;</div><div class='add'>+	bool dma_mr:1;</div><div class='add'>+	u16 type;</div><div class='add'>+	u32 page_cnt;</div><div class='add'>+	u64 page_size;</div><div class='add'>+	u32 npages;</div><div class='add'>+	u32 stag;</div><div class='add'>+	u64 len;</div><div class='add'>+	u64 pgaddrmem[IRDMA_MAX_SAVED_PHY_PGADDR];</div><div class='add'>+	struct irdma_pbl iwpbl;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_srq {</div><div class='add'>+	struct ib_srq ibsrq;</div><div class='add'>+	struct irdma_sc_srq sc_srq __aligned(64);</div><div class='add'>+	struct irdma_dma_mem kmem;</div><div class='add'>+	u64 *srq_wrid_mem;</div><div class='add'>+	refcount_t refcnt;</div><div class='add'>+	spinlock_t lock; /* for poll srq */</div><div class='add'>+	struct irdma_pbl *iwpbl;</div><div class='add'>+	struct irdma_sge *sg_list;</div><div class='add'>+	u16 srq_head;</div><div class='add'>+	u32 srq_num;</div><div class='add'>+	u32 max_wr;</div><div class='add'>+	bool user_mode:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cq {</div><div class='add'>+	struct ib_cq ibcq;</div><div class='add'>+	struct irdma_sc_cq sc_cq;</div><div class='add'>+	u32 cq_num;</div><div class='add'>+	bool user_mode;</div><div class='add'>+	atomic_t armed;</div><div class='add'>+	enum irdma_cmpl_notify last_notify;</div><div class='add'>+	struct irdma_dma_mem kmem;</div><div class='add'>+	struct irdma_dma_mem kmem_shadow;</div><div class='add'>+	struct completion free_cq;</div><div class='add'>+	refcount_t refcnt;</div><div class='add'>+	spinlock_t lock; /* for poll cq */</div><div class='add'>+	struct list_head resize_list;</div><div class='add'>+	struct irdma_cq_poll_info cur_cqe;</div><div class='add'>+	struct list_head cmpl_generated;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_cmpl_gen {</div><div class='add'>+	struct list_head list;</div><div class='add'>+	struct irdma_cq_poll_info cpi;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct disconn_work {</div><div class='add'>+	struct work_struct work;</div><div class='add'>+	struct irdma_qp *iwqp;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct iw_cm_id;</div><div class='add'>+</div><div class='add'>+struct irdma_qp_kmode {</div><div class='add'>+	struct irdma_dma_mem dma_mem;</div><div class='add'>+	struct irdma_sq_uk_wr_trk_info *sq_wrid_mem;</div><div class='add'>+	u64 *rq_wrid_mem;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_qp {</div><div class='add'>+	struct ib_qp ibqp;</div><div class='add'>+	struct irdma_sc_qp sc_qp;</div><div class='add'>+	struct irdma_device *iwdev;</div><div class='add'>+	struct irdma_cq *iwscq;</div><div class='add'>+	struct irdma_cq *iwrcq;</div><div class='add'>+	struct irdma_pd *iwpd;</div><div class='add'>+	struct rdma_user_mmap_entry *push_wqe_mmap_entry;</div><div class='add'>+	struct rdma_user_mmap_entry *push_db_mmap_entry;</div><div class='add'>+	struct irdma_qp_host_ctx_info ctx_info;</div><div class='add'>+	union {</div><div class='add'>+		struct irdma_iwarp_offload_info iwarp_info;</div><div class='add'>+		struct irdma_roce_offload_info roce_info;</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	union {</div><div class='add'>+		struct irdma_tcp_offload_info tcp_info;</div><div class='add'>+		struct irdma_udp_offload_info udp_info;</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	struct irdma_ah roce_ah;</div><div class='add'>+	struct list_head teardown_entry;</div><div class='add'>+	refcount_t refcnt;</div><div class='add'>+	struct iw_cm_id *cm_id;</div><div class='add'>+	struct irdma_cm_node *cm_node;</div><div class='add'>+	struct delayed_work dwork_flush;</div><div class='add'>+	struct ib_mr *lsmm_mr;</div><div class='add'>+	atomic_t hw_mod_qp_pend;</div><div class='add'>+	enum ib_qp_state ibqp_state;</div><div class='add'>+	u32 qp_mem_size;</div><div class='add'>+	u32 last_aeq;</div><div class='add'>+	int max_send_wr;</div><div class='add'>+	int max_recv_wr;</div><div class='add'>+	atomic_t close_timer_started;</div><div class='add'>+	spinlock_t lock; /* serialize posting WRs to SQ/RQ */</div><div class='add'>+	struct irdma_qp_context *iwqp_context;</div><div class='add'>+	void *pbl_vbase;</div><div class='add'>+	dma_addr_t pbl_pbase;</div><div class='add'>+	struct page *page;</div><div class='add'>+	u8 active_conn : 1;</div><div class='add'>+	u8 user_mode : 1;</div><div class='add'>+	u8 hte_added : 1;</div><div class='add'>+	u8 flush_issued : 1;</div><div class='add'>+	u8 sig_all : 1;</div><div class='add'>+	u8 pau_mode : 1;</div><div class='add'>+	u8 suspend_pending : 1;</div><div class='add'>+	u8 rsvd : 1;</div><div class='add'>+	u8 iwarp_state;</div><div class='add'>+	u16 term_sq_flush_code;</div><div class='add'>+	u16 term_rq_flush_code;</div><div class='add'>+	u8 hw_iwarp_state;</div><div class='add'>+	u8 hw_tcp_state;</div><div class='add'>+	struct irdma_qp_kmode kqp;</div><div class='add'>+	struct irdma_dma_mem host_ctx;</div><div class='add'>+	struct timer_list terminate_timer;</div><div class='add'>+	struct irdma_pbl *iwpbl;</div><div class='add'>+	struct irdma_dma_mem q2_ctx_mem;</div><div class='add'>+	struct irdma_dma_mem ietf_mem;</div><div class='add'>+	struct completion free_qp;</div><div class='add'>+	wait_queue_head_t waitq;</div><div class='add'>+	wait_queue_head_t mod_qp_waitq;</div><div class='add'>+	u8 rts_ae_rcvd;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_mmap_flag {</div><div class='add'>+	IRDMA_MMAP_IO_NC,</div><div class='add'>+	IRDMA_MMAP_IO_WC,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_user_mmap_entry {</div><div class='add'>+	struct rdma_user_mmap_entry rdma_entry;</div><div class='add'>+	u64 bar_offset;</div><div class='add'>+	u8 mmap_flag;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline u16 irdma_fw_major_ver(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	return (u16)FIELD_GET(IRDMA_FW_VER_MAJOR, dev-&gt;feature_info[IRDMA_FEATURE_FW_INFO]);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline u16 irdma_fw_minor_ver(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	return (u16)FIELD_GET(IRDMA_FW_VER_MINOR, dev-&gt;feature_info[IRDMA_FEATURE_FW_INFO]);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void set_ib_wc_op_sq(struct irdma_cq_poll_info *cq_poll_info,</div><div class='add'>+				   struct ib_wc *entry)</div><div class='add'>+{</div><div class='add'>+	switch (cq_poll_info-&gt;op_type) {</div><div class='add'>+	case IRDMA_OP_TYPE_RDMA_WRITE:</div><div class='add'>+	case IRDMA_OP_TYPE_RDMA_WRITE_SOL:</div><div class='add'>+		entry-&gt;opcode = IB_WC_RDMA_WRITE;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_TYPE_RDMA_READ_INV_STAG:</div><div class='add'>+	case IRDMA_OP_TYPE_RDMA_READ:</div><div class='add'>+		entry-&gt;opcode = IB_WC_RDMA_READ;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_TYPE_SEND_SOL:</div><div class='add'>+	case IRDMA_OP_TYPE_SEND_SOL_INV:</div><div class='add'>+	case IRDMA_OP_TYPE_SEND_INV:</div><div class='add'>+	case IRDMA_OP_TYPE_SEND:</div><div class='add'>+		entry-&gt;opcode = IB_WC_SEND;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_TYPE_FAST_REG_NSMR:</div><div class='add'>+		entry-&gt;opcode = IB_WC_REG_MR;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_TYPE_ATOMIC_COMPARE_AND_SWAP:</div><div class='add'>+		entry-&gt;opcode = IB_WC_COMP_SWAP;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_TYPE_ATOMIC_FETCH_AND_ADD:</div><div class='add'>+		entry-&gt;opcode = IB_WC_FETCH_ADD;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_OP_TYPE_INV_STAG:</div><div class='add'>+		entry-&gt;opcode = IB_WC_LOCAL_INV;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		entry-&gt;status = IB_WC_GENERAL_ERR;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void set_ib_wc_op_rq_gen_3(struct irdma_cq_poll_info *info,</div><div class='add'>+					 struct ib_wc *entry)</div><div class='add'>+{</div><div class='add'>+	switch (info-&gt;op_type) {</div><div class='add'>+	case IRDMA_OP_TYPE_RDMA_WRITE:</div><div class='add'>+	case IRDMA_OP_TYPE_RDMA_WRITE_SOL:</div><div class='add'>+		entry-&gt;opcode = IB_WC_RECV_RDMA_WITH_IMM;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		entry-&gt;opcode = IB_WC_RECV;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void set_ib_wc_op_rq(struct irdma_cq_poll_info *cq_poll_info,</div><div class='add'>+				   struct ib_wc *entry, bool send_imm_support)</div><div class='add'>+{</div><div class='add'>+	/**</div><div class='add'>+	 * iWARP does not support sendImm, so the presence of Imm data</div><div class='add'>+	 * must be WriteImm.</div><div class='add'>+	 */</div><div class='add'>+	if (!send_imm_support) {</div><div class='add'>+		entry-&gt;opcode = cq_poll_info-&gt;imm_valid ?</div><div class='add'>+					IB_WC_RECV_RDMA_WITH_IMM :</div><div class='add'>+					IB_WC_RECV;</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (cq_poll_info-&gt;op_type) {</div><div class='add'>+	case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE:</div><div class='add'>+	case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE:</div><div class='add'>+		entry-&gt;opcode = IB_WC_RECV_RDMA_WITH_IMM;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		entry-&gt;opcode = IB_WC_RECV;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4);</div><div class='add'>+int irdma_ib_register_device(struct irdma_device *iwdev);</div><div class='add'>+void irdma_ib_unregister_device(struct irdma_device *iwdev);</div><div class='add'>+void irdma_ib_dealloc_device(struct ib_device *ibdev);</div><div class='add'>+void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event);</div><div class='add'>+void irdma_generate_flush_completions(struct irdma_qp *iwqp);</div><div class='add'>+void irdma_remove_cmpls_list(struct irdma_cq *iwcq);</div><div class='add'>+int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info);</div><div class='add'>+#endif /* IRDMA_VERBS_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/virtchnl.c b/drivers/infiniband/hw/irdma/virtchnl.c<br/>new file mode 100644<br/>index 000000000000..16ad27247527<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/virtchnl.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/virtchnl.c</a></div><div class='hunk'>@@ -0,0 +1,618 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2015 - 2024 Intel Corporation */</div><div class='add'>+</div><div class='add'>+#include "osdep.h"</div><div class='add'>+#include "hmc.h"</div><div class='add'>+#include "defs.h"</div><div class='add'>+#include "type.h"</div><div class='add'>+#include "protos.h"</div><div class='add'>+#include "virtchnl.h"</div><div class='add'>+#include "ws.h"</div><div class='add'>+#include "i40iw_hw.h"</div><div class='add'>+#include "ig3rdma_hw.h"</div><div class='add'>+</div><div class='add'>+struct vchnl_reg_map_elem {</div><div class='add'>+	u16 reg_id;</div><div class='add'>+	u16 reg_idx;</div><div class='add'>+	bool pg_rel;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct vchnl_regfld_map_elem {</div><div class='add'>+	u16 regfld_id;</div><div class='add'>+	u16 regfld_idx;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static struct vchnl_reg_map_elem vchnl_reg_map[] = {</div><div class='add'>+	{IRDMA_VCHNL_REG_ID_CQPTAIL, IRDMA_CQPTAIL, false},</div><div class='add'>+	{IRDMA_VCHNL_REG_ID_CQPDB, IRDMA_CQPDB, false},</div><div class='add'>+	{IRDMA_VCHNL_REG_ID_CCQPSTATUS, IRDMA_CCQPSTATUS, false},</div><div class='add'>+	{IRDMA_VCHNL_REG_ID_CCQPHIGH, IRDMA_CCQPHIGH, false},</div><div class='add'>+	{IRDMA_VCHNL_REG_ID_CCQPLOW, IRDMA_CCQPLOW, false},</div><div class='add'>+	{IRDMA_VCHNL_REG_ID_CQARM, IRDMA_CQARM, false},</div><div class='add'>+	{IRDMA_VCHNL_REG_ID_CQACK, IRDMA_CQACK, false},</div><div class='add'>+	{IRDMA_VCHNL_REG_ID_AEQALLOC, IRDMA_AEQALLOC, false},</div><div class='add'>+	{IRDMA_VCHNL_REG_ID_CQPERRCODES, IRDMA_CQPERRCODES, false},</div><div class='add'>+	{IRDMA_VCHNL_REG_ID_WQEALLOC, IRDMA_WQEALLOC, false},</div><div class='add'>+	{IRDMA_VCHNL_REG_ID_DB_ADDR_OFFSET, IRDMA_DB_ADDR_OFFSET, false },</div><div class='add'>+	{IRDMA_VCHNL_REG_ID_DYN_CTL, IRDMA_GLINT_DYN_CTL, false },</div><div class='add'>+	{IRDMA_VCHNL_REG_INV_ID, IRDMA_VCHNL_REG_INV_ID, false }</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static struct vchnl_regfld_map_elem vchnl_regfld_map[] = {</div><div class='add'>+	{IRDMA_VCHNL_REGFLD_ID_CCQPSTATUS_CQP_OP_ERR, IRDMA_CCQPSTATUS_CCQP_ERR_M},</div><div class='add'>+	{IRDMA_VCHNL_REGFLD_ID_CCQPSTATUS_CCQP_DONE, IRDMA_CCQPSTATUS_CCQP_DONE_M},</div><div class='add'>+	{IRDMA_VCHNL_REGFLD_ID_CQPSQ_STAG_PDID, IRDMA_CQPSQ_STAG_PDID_M},</div><div class='add'>+	{IRDMA_VCHNL_REGFLD_ID_CQPSQ_CQ_CEQID, IRDMA_CQPSQ_CQ_CEQID_M},</div><div class='add'>+	{IRDMA_VCHNL_REGFLD_ID_CQPSQ_CQ_CQID, IRDMA_CQPSQ_CQ_CQID_M},</div><div class='add'>+	{IRDMA_VCHNL_REGFLD_ID_COMMIT_FPM_CQCNT, IRDMA_COMMIT_FPM_CQCNT_M},</div><div class='add'>+	{IRDMA_VCHNL_REGFLD_ID_UPESD_HMCN_ID, IRDMA_CQPSQ_UPESD_HMCFNID_M},</div><div class='add'>+	{IRDMA_VCHNL_REGFLD_INV_ID, IRDMA_VCHNL_REGFLD_INV_ID}</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define IRDMA_VCHNL_REG_COUNT ARRAY_SIZE(vchnl_reg_map)</div><div class='add'>+#define IRDMA_VCHNL_REGFLD_COUNT ARRAY_SIZE(vchnl_regfld_map)</div><div class='add'>+#define IRDMA_VCHNL_REGFLD_BUF_SIZE \</div><div class='add'>+	(IRDMA_VCHNL_REG_COUNT * sizeof(struct irdma_vchnl_reg_info) + \</div><div class='add'>+	 IRDMA_VCHNL_REGFLD_COUNT * sizeof(struct irdma_vchnl_reg_field_info))</div><div class='add'>+#define IRDMA_REGMAP_RESP_BUF_SIZE (IRDMA_VCHNL_RESP_MIN_SIZE + IRDMA_VCHNL_REGFLD_BUF_SIZE)</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_sc_vchnl_init - Initialize dev virtchannel and get hw_rev</div><div class='add'>+ * @dev: dev structure to update</div><div class='add'>+ * @info: virtchannel info parameters to fill into the dev structure</div><div class='add'>+ */</div><div class='add'>+int irdma_sc_vchnl_init(struct irdma_sc_dev *dev,</div><div class='add'>+			struct irdma_vchnl_init_info *info)</div><div class='add'>+{</div><div class='add'>+	dev-&gt;vchnl_up = true;</div><div class='add'>+	dev-&gt;privileged = info-&gt;privileged;</div><div class='add'>+	dev-&gt;is_pf = info-&gt;is_pf;</div><div class='add'>+	dev-&gt;hw_attrs.uk_attrs.hw_rev = info-&gt;hw_rev;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;privileged) {</div><div class='add'>+		int ret = irdma_vchnl_req_get_ver(dev, IRDMA_VCHNL_CHNL_VER_MAX,</div><div class='add'>+						  &amp;dev-&gt;vchnl_ver);</div><div class='add'>+</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "DEV: Get Channel version ret = %d, version is %u\n",</div><div class='add'>+			  ret, dev-&gt;vchnl_ver);</div><div class='add'>+</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+</div><div class='add'>+		ret = irdma_vchnl_req_get_caps(dev);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+</div><div class='add'>+		dev-&gt;hw_attrs.uk_attrs.hw_rev = dev-&gt;vc_caps.hw_rev;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_vchnl_req_verify_resp - Verify requested response size</div><div class='add'>+ * @vchnl_req: vchnl message requested</div><div class='add'>+ * @resp_len: response length sent from vchnl peer</div><div class='add'>+ */</div><div class='add'>+static int irdma_vchnl_req_verify_resp(struct irdma_vchnl_req *vchnl_req,</div><div class='add'>+				       u16 resp_len)</div><div class='add'>+{</div><div class='add'>+	switch (vchnl_req-&gt;vchnl_msg-&gt;op_code) {</div><div class='add'>+	case IRDMA_VCHNL_OP_GET_VER:</div><div class='add'>+	case IRDMA_VCHNL_OP_GET_HMC_FCN:</div><div class='add'>+	case IRDMA_VCHNL_OP_PUT_HMC_FCN:</div><div class='add'>+		if (resp_len != vchnl_req-&gt;parm_len)</div><div class='add'>+			return -EBADMSG;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_VCHNL_OP_GET_RDMA_CAPS:</div><div class='add'>+		if (resp_len &lt; IRDMA_VCHNL_OP_GET_RDMA_CAPS_MIN_SIZE)</div><div class='add'>+			return -EBADMSG;</div><div class='add'>+		break;</div><div class='add'>+	case IRDMA_VCHNL_OP_GET_REG_LAYOUT:</div><div class='add'>+	case IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP:</div><div class='add'>+	case IRDMA_VCHNL_OP_QUEUE_VECTOR_UNMAP:</div><div class='add'>+	case IRDMA_VCHNL_OP_ADD_VPORT:</div><div class='add'>+	case IRDMA_VCHNL_OP_DEL_VPORT:</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void irdma_free_vchnl_req_msg(struct irdma_vchnl_req *vchnl_req)</div><div class='add'>+{</div><div class='add'>+	kfree(vchnl_req-&gt;vchnl_msg);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_alloc_vchnl_req_msg(struct irdma_vchnl_req *vchnl_req,</div><div class='add'>+				     struct irdma_vchnl_req_init_info *info)</div><div class='add'>+{</div><div class='add'>+	struct irdma_vchnl_op_buf *vchnl_msg;</div><div class='add'>+</div><div class='add'>+	vchnl_msg = kzalloc(IRDMA_VCHNL_MAX_MSG_SIZE, GFP_KERNEL);</div><div class='add'>+</div><div class='add'>+	if (!vchnl_msg)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	vchnl_msg-&gt;op_ctx = (uintptr_t)vchnl_req;</div><div class='add'>+	vchnl_msg-&gt;buf_len = sizeof(*vchnl_msg) + info-&gt;req_parm_len;</div><div class='add'>+	if (info-&gt;req_parm_len)</div><div class='add'>+		memcpy(vchnl_msg-&gt;buf, info-&gt;req_parm, info-&gt;req_parm_len);</div><div class='add'>+	vchnl_msg-&gt;op_code = info-&gt;op_code;</div><div class='add'>+	vchnl_msg-&gt;op_ver = info-&gt;op_ver;</div><div class='add'>+</div><div class='add'>+	vchnl_req-&gt;vchnl_msg = vchnl_msg;</div><div class='add'>+	vchnl_req-&gt;parm = info-&gt;resp_parm;</div><div class='add'>+	vchnl_req-&gt;parm_len = info-&gt;resp_parm_len;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int irdma_vchnl_req_send_sync(struct irdma_sc_dev *dev,</div><div class='add'>+				     struct irdma_vchnl_req_init_info *info)</div><div class='add'>+{</div><div class='add'>+	u16 resp_len = sizeof(dev-&gt;vc_recv_buf);</div><div class='add'>+	struct irdma_vchnl_req vchnl_req = {};</div><div class='add'>+	u16 msg_len;</div><div class='add'>+	u8 *msg;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = irdma_alloc_vchnl_req_msg(&amp;vchnl_req, info);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	msg_len = vchnl_req.vchnl_msg-&gt;buf_len;</div><div class='add'>+	msg = (u8 *)vchnl_req.vchnl_msg;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;dev-&gt;vchnl_mutex);</div><div class='add'>+	ret = ig3rdma_vchnl_send_sync(dev, msg, msg_len, dev-&gt;vc_recv_buf,</div><div class='add'>+				      &amp;resp_len);</div><div class='add'>+	dev-&gt;vc_recv_len = resp_len;</div><div class='add'>+	if (ret)</div><div class='add'>+		goto exit;</div><div class='add'>+</div><div class='add'>+	ret = irdma_vchnl_req_get_resp(dev, &amp;vchnl_req);</div><div class='add'>+exit:</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;vchnl_mutex);</div><div class='add'>+	ibdev_dbg(to_ibdev(dev),</div><div class='add'>+		  "VIRT: virtual channel send %s caller: %pS ret=%d op=%u op_ver=%u req_len=%u parm_len=%u resp_len=%u\n",</div><div class='add'>+		  !ret ? "SUCCEEDS" : "FAILS", __builtin_return_address(0),</div><div class='add'>+		  ret, vchnl_req.vchnl_msg-&gt;op_code,</div><div class='add'>+		  vchnl_req.vchnl_msg-&gt;op_ver, vchnl_req.vchnl_msg-&gt;buf_len,</div><div class='add'>+		  vchnl_req.parm_len, vchnl_req.resp_len);</div><div class='add'>+	irdma_free_vchnl_req_msg(&amp;vchnl_req);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_vchnl_req_get_reg_layout - Get Register Layout</div><div class='add'>+ * @dev: RDMA device pointer</div><div class='add'>+ */</div><div class='add'>+int irdma_vchnl_req_get_reg_layout(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	u16 reg_idx, reg_id, tmp_reg_id, regfld_idx, regfld_id, tmp_regfld_id;</div><div class='add'>+	struct irdma_vchnl_reg_field_info *regfld_array = NULL;</div><div class='add'>+	u8 resp_buffer[IRDMA_REGMAP_RESP_BUF_SIZE] = {};</div><div class='add'>+	struct vchnl_regfld_map_elem *regfld_map_array;</div><div class='add'>+	struct irdma_vchnl_req_init_info info = {};</div><div class='add'>+	struct vchnl_reg_map_elem *reg_map_array;</div><div class='add'>+	struct irdma_vchnl_reg_info *reg_array;</div><div class='add'>+	u8 num_bits, shift_cnt;</div><div class='add'>+	u16 buf_len = 0;</div><div class='add'>+	u64 bitmask;</div><div class='add'>+	u32 rindex;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;vchnl_up)</div><div class='add'>+		return -EBUSY;</div><div class='add'>+</div><div class='add'>+	info.op_code = IRDMA_VCHNL_OP_GET_REG_LAYOUT;</div><div class='add'>+	info.op_ver = IRDMA_VCHNL_OP_GET_REG_LAYOUT_V0;</div><div class='add'>+	info.resp_parm = resp_buffer;</div><div class='add'>+	info.resp_parm_len = sizeof(resp_buffer);</div><div class='add'>+</div><div class='add'>+	ret = irdma_vchnl_req_send_sync(dev, &amp;info);</div><div class='add'>+</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	/* parse the response buffer and update reg info*/</div><div class='add'>+	/* Parse registers till invalid */</div><div class='add'>+	/* Parse register fields till invalid */</div><div class='add'>+	reg_array = (struct irdma_vchnl_reg_info *)resp_buffer;</div><div class='add'>+	for (rindex = 0; rindex &lt; IRDMA_VCHNL_REG_COUNT; rindex++) {</div><div class='add'>+		buf_len += sizeof(struct irdma_vchnl_reg_info);</div><div class='add'>+		if (buf_len &gt;= sizeof(resp_buffer))</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+</div><div class='add'>+		regfld_array =</div><div class='add'>+			(struct irdma_vchnl_reg_field_info *)&amp;reg_array[rindex + 1];</div><div class='add'>+		reg_id = reg_array[rindex].reg_id;</div><div class='add'>+		if (reg_id == IRDMA_VCHNL_REG_INV_ID)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		reg_id &amp;= ~IRDMA_VCHNL_REG_PAGE_REL;</div><div class='add'>+		if (reg_id &gt;= IRDMA_VCHNL_REG_COUNT)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		/* search regmap for register index in hw_regs.*/</div><div class='add'>+		reg_map_array = vchnl_reg_map;</div><div class='add'>+		do {</div><div class='add'>+			tmp_reg_id = reg_map_array-&gt;reg_id;</div><div class='add'>+			if (tmp_reg_id == reg_id)</div><div class='add'>+				break;</div><div class='add'>+</div><div class='add'>+			reg_map_array++;</div><div class='add'>+		} while (tmp_reg_id != IRDMA_VCHNL_REG_INV_ID);</div><div class='add'>+		if (tmp_reg_id != reg_id)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		reg_idx = reg_map_array-&gt;reg_idx;</div><div class='add'>+</div><div class='add'>+		/* Page relative, DB Offset do not need bar offset */</div><div class='add'>+		if (reg_idx == IRDMA_DB_ADDR_OFFSET ||</div><div class='add'>+		    (reg_array[rindex].reg_id &amp; IRDMA_VCHNL_REG_PAGE_REL)) {</div><div class='add'>+			dev-&gt;hw_regs[reg_idx] =</div><div class='add'>+				(u32 __iomem *)(uintptr_t)reg_array[rindex].reg_offset;</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		/* Update the local HW struct */</div><div class='add'>+		dev-&gt;hw_regs[reg_idx] = ig3rdma_get_reg_addr(dev-&gt;hw,</div><div class='add'>+						reg_array[rindex].reg_offset);</div><div class='add'>+		if (!dev-&gt;hw_regs[reg_idx])</div><div class='add'>+			return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!regfld_array)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	/* set up doorbell variables using mapped DB page */</div><div class='add'>+	dev-&gt;wqe_alloc_db = dev-&gt;hw_regs[IRDMA_WQEALLOC];</div><div class='add'>+	dev-&gt;cq_arm_db = dev-&gt;hw_regs[IRDMA_CQARM];</div><div class='add'>+	dev-&gt;aeq_alloc_db = dev-&gt;hw_regs[IRDMA_AEQALLOC];</div><div class='add'>+	dev-&gt;cqp_db = dev-&gt;hw_regs[IRDMA_CQPDB];</div><div class='add'>+	dev-&gt;cq_ack_db = dev-&gt;hw_regs[IRDMA_CQACK];</div><div class='add'>+</div><div class='add'>+	for (rindex = 0; rindex &lt; IRDMA_VCHNL_REGFLD_COUNT; rindex++) {</div><div class='add'>+		buf_len += sizeof(struct irdma_vchnl_reg_field_info);</div><div class='add'>+		if ((buf_len - 1) &gt; sizeof(resp_buffer))</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		if (regfld_array[rindex].fld_id == IRDMA_VCHNL_REGFLD_INV_ID)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		regfld_id = regfld_array[rindex].fld_id;</div><div class='add'>+		regfld_map_array = vchnl_regfld_map;</div><div class='add'>+		do {</div><div class='add'>+			tmp_regfld_id = regfld_map_array-&gt;regfld_id;</div><div class='add'>+			if (tmp_regfld_id == regfld_id)</div><div class='add'>+				break;</div><div class='add'>+</div><div class='add'>+			regfld_map_array++;</div><div class='add'>+		} while (tmp_regfld_id != IRDMA_VCHNL_REGFLD_INV_ID);</div><div class='add'>+</div><div class='add'>+		if (tmp_regfld_id != regfld_id)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		regfld_idx = regfld_map_array-&gt;regfld_idx;</div><div class='add'>+</div><div class='add'>+		num_bits = regfld_array[rindex].fld_bits;</div><div class='add'>+		shift_cnt = regfld_array[rindex].fld_shift;</div><div class='add'>+		if ((num_bits + shift_cnt &gt; 64) || !num_bits) {</div><div class='add'>+			ibdev_dbg(to_ibdev(dev),</div><div class='add'>+				  "ERR: Invalid field mask id %d bits %d shift %d",</div><div class='add'>+				  regfld_id, num_bits, shift_cnt);</div><div class='add'>+</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		bitmask = (1ULL &lt;&lt; num_bits) - 1;</div><div class='add'>+		dev-&gt;hw_masks[regfld_idx] = bitmask &lt;&lt; shift_cnt;</div><div class='add'>+		dev-&gt;hw_shifts[regfld_idx] = shift_cnt;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int irdma_vchnl_req_add_vport(struct irdma_sc_dev *dev, u16 vport_id,</div><div class='add'>+			      u32 qp1_id, struct irdma_qos *qos)</div><div class='add'>+{</div><div class='add'>+	struct irdma_vchnl_resp_vport_info resp_vport = { 0 };</div><div class='add'>+	struct irdma_vchnl_req_vport_info req_vport = { 0 };</div><div class='add'>+	struct irdma_vchnl_req_init_info info = { 0 };</div><div class='add'>+	int ret, i;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;vchnl_up)</div><div class='add'>+		return -EBUSY;</div><div class='add'>+</div><div class='add'>+	info.op_code = IRDMA_VCHNL_OP_ADD_VPORT;</div><div class='add'>+	info.op_ver = IRDMA_VCHNL_OP_ADD_VPORT_V0;</div><div class='add'>+	req_vport.vport_id = vport_id;</div><div class='add'>+	req_vport.qp1_id = qp1_id;</div><div class='add'>+	info.req_parm_len = sizeof(req_vport);</div><div class='add'>+	info.req_parm = &amp;req_vport;</div><div class='add'>+	info.resp_parm = &amp;resp_vport;</div><div class='add'>+	info.resp_parm_len = sizeof(resp_vport);</div><div class='add'>+</div><div class='add'>+	ret = irdma_vchnl_req_send_sync(dev, &amp;info);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	for (i = 0;  i &lt; IRDMA_MAX_USER_PRIORITY; i++) {</div><div class='add'>+		qos[i].qs_handle = resp_vport.qs_handle[i];</div><div class='add'>+		qos[i].valid = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int irdma_vchnl_req_del_vport(struct irdma_sc_dev *dev, u16 vport_id, u32 qp1_id)</div><div class='add'>+{</div><div class='add'>+	struct irdma_vchnl_req_init_info info = { 0 };</div><div class='add'>+	struct irdma_vchnl_req_vport_info req_vport = { 0 };</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;vchnl_up)</div><div class='add'>+		return -EBUSY;</div><div class='add'>+</div><div class='add'>+	info.op_code = IRDMA_VCHNL_OP_DEL_VPORT;</div><div class='add'>+	info.op_ver = IRDMA_VCHNL_OP_DEL_VPORT_V0;</div><div class='add'>+	req_vport.vport_id = vport_id;</div><div class='add'>+	req_vport.qp1_id = qp1_id;</div><div class='add'>+	info.req_parm_len = sizeof(req_vport);</div><div class='add'>+	info.req_parm = &amp;req_vport;</div><div class='add'>+</div><div class='add'>+	return irdma_vchnl_req_send_sync(dev, &amp;info);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_vchnl_req_aeq_vec_map - Map AEQ to vector on this function</div><div class='add'>+ * @dev: RDMA device pointer</div><div class='add'>+ * @v_idx: vector index</div><div class='add'>+ */</div><div class='add'>+int irdma_vchnl_req_aeq_vec_map(struct irdma_sc_dev *dev, u32 v_idx)</div><div class='add'>+{</div><div class='add'>+	struct irdma_vchnl_req_init_info info = {};</div><div class='add'>+	struct irdma_vchnl_qvlist_info *qvl;</div><div class='add'>+	struct irdma_vchnl_qv_info *qv;</div><div class='add'>+	u16 qvl_size, num_vectors = 1;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;vchnl_up)</div><div class='add'>+		return -EBUSY;</div><div class='add'>+</div><div class='add'>+	qvl_size = struct_size(qvl, qv_info, num_vectors);</div><div class='add'>+</div><div class='add'>+	qvl = kzalloc(qvl_size, GFP_KERNEL);</div><div class='add'>+	if (!qvl)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	qvl-&gt;num_vectors = 1;</div><div class='add'>+	qv = qvl-&gt;qv_info;</div><div class='add'>+</div><div class='add'>+	qv-&gt;ceq_idx = IRDMA_Q_INVALID_IDX;</div><div class='add'>+	qv-&gt;v_idx = v_idx;</div><div class='add'>+	qv-&gt;itr_idx = IRDMA_IDX_ITR0;</div><div class='add'>+</div><div class='add'>+	info.op_code = IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP;</div><div class='add'>+	info.op_ver = IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP_V0;</div><div class='add'>+	info.req_parm = qvl;</div><div class='add'>+	info.req_parm_len = qvl_size;</div><div class='add'>+</div><div class='add'>+	ret = irdma_vchnl_req_send_sync(dev, &amp;info);</div><div class='add'>+	kfree(qvl);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_vchnl_req_ceq_vec_map - Map CEQ to vector on this function</div><div class='add'>+ * @dev: RDMA device pointer</div><div class='add'>+ * @ceq_id: CEQ index</div><div class='add'>+ * @v_idx: vector index</div><div class='add'>+ */</div><div class='add'>+int irdma_vchnl_req_ceq_vec_map(struct irdma_sc_dev *dev, u16 ceq_id, u32 v_idx)</div><div class='add'>+{</div><div class='add'>+	struct irdma_vchnl_req_init_info info = {};</div><div class='add'>+	struct irdma_vchnl_qvlist_info *qvl;</div><div class='add'>+	struct irdma_vchnl_qv_info *qv;</div><div class='add'>+	u16 qvl_size, num_vectors = 1;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;vchnl_up)</div><div class='add'>+		return -EBUSY;</div><div class='add'>+</div><div class='add'>+	qvl_size = struct_size(qvl, qv_info, num_vectors);</div><div class='add'>+</div><div class='add'>+	qvl = kzalloc(qvl_size, GFP_KERNEL);</div><div class='add'>+	if (!qvl)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	qvl-&gt;num_vectors = num_vectors;</div><div class='add'>+	qv = qvl-&gt;qv_info;</div><div class='add'>+</div><div class='add'>+	qv-&gt;aeq_idx = IRDMA_Q_INVALID_IDX;</div><div class='add'>+	qv-&gt;ceq_idx = ceq_id;</div><div class='add'>+	qv-&gt;v_idx = v_idx;</div><div class='add'>+	qv-&gt;itr_idx = IRDMA_IDX_ITR0;</div><div class='add'>+</div><div class='add'>+	info.op_code = IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP;</div><div class='add'>+	info.op_ver = IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP_V0;</div><div class='add'>+	info.req_parm = qvl;</div><div class='add'>+	info.req_parm_len = qvl_size;</div><div class='add'>+</div><div class='add'>+	ret = irdma_vchnl_req_send_sync(dev, &amp;info);</div><div class='add'>+	kfree(qvl);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_vchnl_req_get_ver - Request Channel version</div><div class='add'>+ * @dev: RDMA device pointer</div><div class='add'>+ * @ver_req: Virtual channel version requested</div><div class='add'>+ * @ver_res: Virtual channel version response</div><div class='add'>+ */</div><div class='add'>+int irdma_vchnl_req_get_ver(struct irdma_sc_dev *dev, u16 ver_req, u32 *ver_res)</div><div class='add'>+{</div><div class='add'>+	struct irdma_vchnl_req_init_info info = {};</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;vchnl_up)</div><div class='add'>+		return -EBUSY;</div><div class='add'>+</div><div class='add'>+	info.op_code = IRDMA_VCHNL_OP_GET_VER;</div><div class='add'>+	info.op_ver = ver_req;</div><div class='add'>+	info.resp_parm = ver_res;</div><div class='add'>+	info.resp_parm_len = sizeof(*ver_res);</div><div class='add'>+</div><div class='add'>+	ret = irdma_vchnl_req_send_sync(dev, &amp;info);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (*ver_res &lt; IRDMA_VCHNL_CHNL_VER_MIN) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "VIRT: %s unsupported vchnl version 0x%0x\n",</div><div class='add'>+			  __func__, *ver_res);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_vchnl_req_get_hmc_fcn - Request VF HMC Function</div><div class='add'>+ * @dev: RDMA device pointer</div><div class='add'>+ */</div><div class='add'>+int irdma_vchnl_req_get_hmc_fcn(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct irdma_vchnl_req_hmc_info req_hmc = {};</div><div class='add'>+	struct irdma_vchnl_resp_hmc_info resp_hmc = {};</div><div class='add'>+	struct irdma_vchnl_req_init_info info = {};</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;vchnl_up)</div><div class='add'>+		return -EBUSY;</div><div class='add'>+</div><div class='add'>+	info.op_code = IRDMA_VCHNL_OP_GET_HMC_FCN;</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3) {</div><div class='add'>+		info.op_ver = IRDMA_VCHNL_OP_GET_HMC_FCN_V2;</div><div class='add'>+		req_hmc.protocol_used = dev-&gt;protocol_used;</div><div class='add'>+		info.req_parm_len = sizeof(req_hmc);</div><div class='add'>+		info.req_parm = &amp;req_hmc;</div><div class='add'>+		info.resp_parm = &amp;resp_hmc;</div><div class='add'>+		info.resp_parm_len = sizeof(resp_hmc);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = irdma_vchnl_req_send_sync(dev, &amp;info);</div><div class='add'>+</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;hw_attrs.uk_attrs.hw_rev &gt;= IRDMA_GEN_3) {</div><div class='add'>+		int i;</div><div class='add'>+</div><div class='add'>+		dev-&gt;hmc_fn_id = resp_hmc.hmc_func;</div><div class='add'>+</div><div class='add'>+		for (i = 0;  i &lt; IRDMA_MAX_USER_PRIORITY; i++) {</div><div class='add'>+			dev-&gt;qos[i].qs_handle = resp_hmc.qs_handle[i];</div><div class='add'>+			dev-&gt;qos[i].valid = true;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_vchnl_req_put_hmc_fcn - Free VF HMC Function</div><div class='add'>+ * @dev: RDMA device pointer</div><div class='add'>+ */</div><div class='add'>+int irdma_vchnl_req_put_hmc_fcn(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct irdma_vchnl_req_init_info info = {};</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;vchnl_up)</div><div class='add'>+		return -EBUSY;</div><div class='add'>+</div><div class='add'>+	info.op_code = IRDMA_VCHNL_OP_PUT_HMC_FCN;</div><div class='add'>+	info.op_ver = IRDMA_VCHNL_OP_PUT_HMC_FCN_V0;</div><div class='add'>+</div><div class='add'>+	return irdma_vchnl_req_send_sync(dev, &amp;info);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_vchnl_req_get_caps - Request RDMA capabilities</div><div class='add'>+ * @dev: RDMA device pointer</div><div class='add'>+ */</div><div class='add'>+int irdma_vchnl_req_get_caps(struct irdma_sc_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct irdma_vchnl_req_init_info info = {};</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;vchnl_up)</div><div class='add'>+		return -EBUSY;</div><div class='add'>+</div><div class='add'>+	info.op_code = IRDMA_VCHNL_OP_GET_RDMA_CAPS;</div><div class='add'>+	info.op_ver = IRDMA_VCHNL_OP_GET_RDMA_CAPS_V0;</div><div class='add'>+	info.resp_parm = &amp;dev-&gt;vc_caps;</div><div class='add'>+	info.resp_parm_len = sizeof(dev-&gt;vc_caps);</div><div class='add'>+</div><div class='add'>+	ret = irdma_vchnl_req_send_sync(dev, &amp;info);</div><div class='add'>+</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;vc_caps.hw_rev &gt; IRDMA_GEN_MAX ||</div><div class='add'>+	    dev-&gt;vc_caps.hw_rev &lt; IRDMA_GEN_2) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "ERR: %s unsupported hw_rev version 0x%0x\n",</div><div class='add'>+			  __func__, dev-&gt;vc_caps.hw_rev);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_vchnl_req_get_resp - Receive the inbound vchnl response.</div><div class='add'>+ * @dev: Dev pointer</div><div class='add'>+ * @vchnl_req: Vchannel request</div><div class='add'>+ */</div><div class='add'>+int irdma_vchnl_req_get_resp(struct irdma_sc_dev *dev,</div><div class='add'>+			     struct irdma_vchnl_req *vchnl_req)</div><div class='add'>+{</div><div class='add'>+	struct irdma_vchnl_resp_buf *vchnl_msg_resp =</div><div class='add'>+		(struct irdma_vchnl_resp_buf *)dev-&gt;vc_recv_buf;</div><div class='add'>+	u16 resp_len;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if ((uintptr_t)vchnl_req != (uintptr_t)vchnl_msg_resp-&gt;op_ctx) {</div><div class='add'>+		ibdev_dbg(to_ibdev(dev),</div><div class='add'>+			  "VIRT: error vchnl context value does not match\n");</div><div class='add'>+		return -EBADMSG;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	resp_len = dev-&gt;vc_recv_len - sizeof(*vchnl_msg_resp);</div><div class='add'>+	resp_len = min(resp_len, vchnl_req-&gt;parm_len);</div><div class='add'>+</div><div class='add'>+	ret = irdma_vchnl_req_verify_resp(vchnl_req, resp_len);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = (int)vchnl_msg_resp-&gt;op_ret;</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	vchnl_req-&gt;resp_len = 0;</div><div class='add'>+	if (vchnl_req-&gt;parm_len &amp;&amp; vchnl_req-&gt;parm &amp;&amp; resp_len) {</div><div class='add'>+		memcpy(vchnl_req-&gt;parm, vchnl_msg_resp-&gt;buf, resp_len);</div><div class='add'>+		vchnl_req-&gt;resp_len = resp_len;</div><div class='add'>+		ibdev_dbg(to_ibdev(dev), "VIRT: Got response, data size %u\n",</div><div class='add'>+			  resp_len);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/virtchnl.h b/drivers/infiniband/hw/irdma/virtchnl.h<br/>new file mode 100644<br/>index 000000000000..aa955a9125bd<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/virtchnl.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/virtchnl.h</a></div><div class='hunk'>@@ -0,0 +1,176 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2015 - 2024 Intel Corporation */</div><div class='add'>+#ifndef IRDMA_VIRTCHNL_H</div><div class='add'>+#define IRDMA_VIRTCHNL_H</div><div class='add'>+</div><div class='add'>+#include "hmc.h"</div><div class='add'>+#include "irdma.h"</div><div class='add'>+</div><div class='add'>+/* IRDMA_VCHNL_CHNL_VER_V0 is for legacy hw, no longer supported. */</div><div class='add'>+#define IRDMA_VCHNL_CHNL_VER_V2 2</div><div class='add'>+#define IRDMA_VCHNL_CHNL_VER_MIN IRDMA_VCHNL_CHNL_VER_V2</div><div class='add'>+#define IRDMA_VCHNL_CHNL_VER_MAX IRDMA_VCHNL_CHNL_VER_V2</div><div class='add'>+#define IRDMA_VCHNL_OP_GET_HMC_FCN_V0 0</div><div class='add'>+#define IRDMA_VCHNL_OP_GET_HMC_FCN_V1 1</div><div class='add'>+#define IRDMA_VCHNL_OP_GET_HMC_FCN_V2 2</div><div class='add'>+#define IRDMA_VCHNL_OP_PUT_HMC_FCN_V0 0</div><div class='add'>+#define IRDMA_VCHNL_OP_GET_REG_LAYOUT_V0 0</div><div class='add'>+#define IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP_V0 0</div><div class='add'>+#define IRDMA_VCHNL_OP_QUEUE_VECTOR_UNMAP_V0 0</div><div class='add'>+#define IRDMA_VCHNL_OP_ADD_VPORT_V0 0</div><div class='add'>+#define IRDMA_VCHNL_OP_DEL_VPORT_V0 0</div><div class='add'>+#define IRDMA_VCHNL_OP_GET_RDMA_CAPS_V0 0</div><div class='add'>+#define IRDMA_VCHNL_OP_GET_RDMA_CAPS_MIN_SIZE 1</div><div class='add'>+</div><div class='add'>+#define IRDMA_VCHNL_REG_ID_CQPTAIL 0</div><div class='add'>+#define IRDMA_VCHNL_REG_ID_CQPDB 1</div><div class='add'>+#define IRDMA_VCHNL_REG_ID_CCQPSTATUS 2</div><div class='add'>+#define IRDMA_VCHNL_REG_ID_CCQPHIGH 3</div><div class='add'>+#define IRDMA_VCHNL_REG_ID_CCQPLOW 4</div><div class='add'>+#define IRDMA_VCHNL_REG_ID_CQARM 5</div><div class='add'>+#define IRDMA_VCHNL_REG_ID_CQACK 6</div><div class='add'>+#define IRDMA_VCHNL_REG_ID_AEQALLOC 7</div><div class='add'>+#define IRDMA_VCHNL_REG_ID_CQPERRCODES 8</div><div class='add'>+#define IRDMA_VCHNL_REG_ID_WQEALLOC 9</div><div class='add'>+#define IRDMA_VCHNL_REG_ID_IPCONFIG0 10</div><div class='add'>+#define IRDMA_VCHNL_REG_ID_DB_ADDR_OFFSET 11</div><div class='add'>+#define IRDMA_VCHNL_REG_ID_DYN_CTL 12</div><div class='add'>+#define IRDMA_VCHNL_REG_ID_AEQITRMASK 13</div><div class='add'>+#define IRDMA_VCHNL_REG_ID_CEQITRMASK 14</div><div class='add'>+#define IRDMA_VCHNL_REG_INV_ID 0xFFFF</div><div class='add'>+#define IRDMA_VCHNL_REG_PAGE_REL 0x8000</div><div class='add'>+</div><div class='add'>+#define IRDMA_VCHNL_REGFLD_ID_CCQPSTATUS_CQP_OP_ERR 2</div><div class='add'>+#define IRDMA_VCHNL_REGFLD_ID_CCQPSTATUS_CCQP_DONE 5</div><div class='add'>+#define IRDMA_VCHNL_REGFLD_ID_CQPSQ_STAG_PDID 6</div><div class='add'>+#define IRDMA_VCHNL_REGFLD_ID_CQPSQ_CQ_CEQID 7</div><div class='add'>+#define IRDMA_VCHNL_REGFLD_ID_CQPSQ_CQ_CQID 8</div><div class='add'>+#define IRDMA_VCHNL_REGFLD_ID_COMMIT_FPM_CQCNT 9</div><div class='add'>+#define IRDMA_VCHNL_REGFLD_ID_UPESD_HMCN_ID 10</div><div class='add'>+#define IRDMA_VCHNL_REGFLD_INV_ID 0xFFFF</div><div class='add'>+</div><div class='add'>+#define IRDMA_VCHNL_RESP_MIN_SIZE (sizeof(struct irdma_vchnl_resp_buf))</div><div class='add'>+</div><div class='add'>+enum irdma_vchnl_ops {</div><div class='add'>+	IRDMA_VCHNL_OP_GET_VER = 0,</div><div class='add'>+	IRDMA_VCHNL_OP_GET_HMC_FCN = 1,</div><div class='add'>+	IRDMA_VCHNL_OP_PUT_HMC_FCN = 2,</div><div class='add'>+	IRDMA_VCHNL_OP_GET_REG_LAYOUT = 11,</div><div class='add'>+	IRDMA_VCHNL_OP_GET_RDMA_CAPS = 13,</div><div class='add'>+	IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP = 14,</div><div class='add'>+	IRDMA_VCHNL_OP_QUEUE_VECTOR_UNMAP = 15,</div><div class='add'>+	IRDMA_VCHNL_OP_ADD_VPORT = 16,</div><div class='add'>+	IRDMA_VCHNL_OP_DEL_VPORT = 17,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_vchnl_req_hmc_info {</div><div class='add'>+	u8 protocol_used;</div><div class='add'>+	u8 disable_qos;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+struct irdma_vchnl_resp_hmc_info {</div><div class='add'>+	u16 hmc_func;</div><div class='add'>+	u16 qs_handle[IRDMA_MAX_USER_PRIORITY];</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+struct irdma_vchnl_qv_info {</div><div class='add'>+	u32 v_idx;</div><div class='add'>+	u16 ceq_idx;</div><div class='add'>+	u16 aeq_idx;</div><div class='add'>+	u8 itr_idx;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_vchnl_qvlist_info {</div><div class='add'>+	u32 num_vectors;</div><div class='add'>+	struct irdma_vchnl_qv_info qv_info[];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_vchnl_req_vport_info {</div><div class='add'>+	u16 vport_id;</div><div class='add'>+	u32 qp1_id;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_vchnl_resp_vport_info {</div><div class='add'>+	u16 qs_handle[IRDMA_MAX_USER_PRIORITY];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_vchnl_op_buf {</div><div class='add'>+	u16 op_code;</div><div class='add'>+	u16 op_ver;</div><div class='add'>+	u16 buf_len;</div><div class='add'>+	u16 rsvd;</div><div class='add'>+	u64 op_ctx;</div><div class='add'>+	u8 buf[];</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+struct irdma_vchnl_resp_buf {</div><div class='add'>+	u64 op_ctx;</div><div class='add'>+	u16 buf_len;</div><div class='add'>+	s16 op_ret;</div><div class='add'>+	u16 rsvd[2];</div><div class='add'>+	u8 buf[];</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+struct irdma_vchnl_rdma_caps {</div><div class='add'>+	u8 hw_rev;</div><div class='add'>+	u16 cqp_timeout_s;</div><div class='add'>+	u16 cqp_def_timeout_s;</div><div class='add'>+	u16 max_hw_push_len;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+struct irdma_vchnl_init_info {</div><div class='add'>+	struct workqueue_struct *vchnl_wq;</div><div class='add'>+	enum irdma_vers hw_rev;</div><div class='add'>+	bool privileged;</div><div class='add'>+	bool is_pf;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_vchnl_reg_info {</div><div class='add'>+	u32 reg_offset;</div><div class='add'>+	u16 field_cnt;</div><div class='add'>+	u16 reg_id; /* High bit of reg_id: bar or page relative */</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_vchnl_reg_field_info {</div><div class='add'>+	u8 fld_shift;</div><div class='add'>+	u8 fld_bits;</div><div class='add'>+	u16 fld_id;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_vchnl_req {</div><div class='add'>+	struct irdma_vchnl_op_buf *vchnl_msg;</div><div class='add'>+	void *parm;</div><div class='add'>+	u32 vf_id;</div><div class='add'>+	u16 parm_len;</div><div class='add'>+	u16 resp_len;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_vchnl_req_init_info {</div><div class='add'>+	void *req_parm;</div><div class='add'>+	void *resp_parm;</div><div class='add'>+	u16 req_parm_len;</div><div class='add'>+	u16 resp_parm_len;</div><div class='add'>+	u16 op_code;</div><div class='add'>+	u16 op_ver;</div><div class='add'>+} __packed;</div><div class='add'>+</div><div class='add'>+struct irdma_qos;</div><div class='add'>+</div><div class='add'>+int irdma_sc_vchnl_init(struct irdma_sc_dev *dev,</div><div class='add'>+			struct irdma_vchnl_init_info *info);</div><div class='add'>+int irdma_vchnl_req_get_ver(struct irdma_sc_dev *dev, u16 ver_req,</div><div class='add'>+			    u32 *ver_res);</div><div class='add'>+int irdma_vchnl_req_get_hmc_fcn(struct irdma_sc_dev *dev);</div><div class='add'>+int irdma_vchnl_req_put_hmc_fcn(struct irdma_sc_dev *dev);</div><div class='add'>+int irdma_vchnl_req_get_caps(struct irdma_sc_dev *dev);</div><div class='add'>+int irdma_vchnl_req_get_resp(struct irdma_sc_dev *dev,</div><div class='add'>+			     struct irdma_vchnl_req *vc_req);</div><div class='add'>+int irdma_vchnl_req_get_reg_layout(struct irdma_sc_dev *dev);</div><div class='add'>+int irdma_vchnl_req_aeq_vec_map(struct irdma_sc_dev *dev, u32 v_idx);</div><div class='add'>+int irdma_vchnl_req_ceq_vec_map(struct irdma_sc_dev *dev, u16 ceq_id,</div><div class='add'>+				u32 v_idx);</div><div class='add'>+int irdma_vchnl_req_add_vport(struct irdma_sc_dev *dev, u16 vport_id,</div><div class='add'>+			      u32 qp1_id, struct irdma_qos *qos);</div><div class='add'>+int irdma_vchnl_req_del_vport(struct irdma_sc_dev *dev, u16 vport_id,</div><div class='add'>+			      u32 qp1_id);</div><div class='add'>+#endif /* IRDMA_VIRTCHNL_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/ws.c b/drivers/infiniband/hw/irdma/ws.c<br/>new file mode 100644<br/>index 000000000000..542bc0b1bb03<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/ws.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/ws.c</a></div><div class='hunk'>@@ -0,0 +1,406 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2017 - 2021 Intel Corporation */</div><div class='add'>+#include "osdep.h"</div><div class='add'>+#include "hmc.h"</div><div class='add'>+#include "defs.h"</div><div class='add'>+#include "type.h"</div><div class='add'>+#include "protos.h"</div><div class='add'>+</div><div class='add'>+#include "ws.h"</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_alloc_node - Allocate a WS node and init</div><div class='add'>+ * @vsi: vsi pointer</div><div class='add'>+ * @user_pri: user priority</div><div class='add'>+ * @node_type: Type of node, leaf or parent</div><div class='add'>+ * @parent: parent node pointer</div><div class='add'>+ */</div><div class='add'>+static struct irdma_ws_node *irdma_alloc_node(struct irdma_sc_vsi *vsi,</div><div class='add'>+					      u8 user_pri,</div><div class='add'>+					      enum irdma_ws_node_type node_type,</div><div class='add'>+					      struct irdma_ws_node *parent)</div><div class='add'>+{</div><div class='add'>+	struct irdma_virt_mem ws_mem;</div><div class='add'>+	struct irdma_ws_node *node;</div><div class='add'>+	u16 node_index = 0;</div><div class='add'>+</div><div class='add'>+	ws_mem.size = sizeof(struct irdma_ws_node);</div><div class='add'>+	ws_mem.va = kzalloc(ws_mem.size, GFP_KERNEL);</div><div class='add'>+	if (!ws_mem.va)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	if (parent) {</div><div class='add'>+		node_index = irdma_alloc_ws_node_id(vsi-&gt;dev);</div><div class='add'>+		if (node_index == IRDMA_WS_NODE_INVALID) {</div><div class='add'>+			kfree(ws_mem.va);</div><div class='add'>+			return NULL;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	node = ws_mem.va;</div><div class='add'>+	node-&gt;index = node_index;</div><div class='add'>+	node-&gt;vsi_index = vsi-&gt;vsi_idx;</div><div class='add'>+	INIT_LIST_HEAD(&amp;node-&gt;child_list_head);</div><div class='add'>+	if (node_type == WS_NODE_TYPE_LEAF) {</div><div class='add'>+		node-&gt;type_leaf = true;</div><div class='add'>+		node-&gt;traffic_class = vsi-&gt;qos[user_pri].traffic_class;</div><div class='add'>+		node-&gt;user_pri = user_pri;</div><div class='add'>+		node-&gt;rel_bw = vsi-&gt;qos[user_pri].rel_bw;</div><div class='add'>+		if (!node-&gt;rel_bw)</div><div class='add'>+			node-&gt;rel_bw = 1;</div><div class='add'>+</div><div class='add'>+		node-&gt;lan_qs_handle = vsi-&gt;qos[user_pri].lan_qos_handle;</div><div class='add'>+		node-&gt;prio_type = IRDMA_PRIO_WEIGHTED_RR;</div><div class='add'>+	} else {</div><div class='add'>+		node-&gt;rel_bw = 1;</div><div class='add'>+		node-&gt;prio_type = IRDMA_PRIO_WEIGHTED_RR;</div><div class='add'>+		node-&gt;enable = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	node-&gt;parent = parent;</div><div class='add'>+</div><div class='add'>+	return node;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_free_node - Free a WS node</div><div class='add'>+ * @vsi: VSI stricture of device</div><div class='add'>+ * @node: Pointer to node to free</div><div class='add'>+ */</div><div class='add'>+static void irdma_free_node(struct irdma_sc_vsi *vsi,</div><div class='add'>+			    struct irdma_ws_node *node)</div><div class='add'>+{</div><div class='add'>+	struct irdma_virt_mem ws_mem;</div><div class='add'>+</div><div class='add'>+	if (node-&gt;index)</div><div class='add'>+		irdma_free_ws_node_id(vsi-&gt;dev, node-&gt;index);</div><div class='add'>+</div><div class='add'>+	ws_mem.va = node;</div><div class='add'>+	ws_mem.size = sizeof(struct irdma_ws_node);</div><div class='add'>+	kfree(ws_mem.va);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ws_cqp_cmd - Post CQP work scheduler node cmd</div><div class='add'>+ * @vsi: vsi pointer</div><div class='add'>+ * @node: pointer to node</div><div class='add'>+ * @cmd: add, remove or modify</div><div class='add'>+ */</div><div class='add'>+static int irdma_ws_cqp_cmd(struct irdma_sc_vsi *vsi,</div><div class='add'>+			    struct irdma_ws_node *node, u8 cmd)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ws_node_info node_info = {};</div><div class='add'>+</div><div class='add'>+	node_info.id = node-&gt;index;</div><div class='add'>+	node_info.vsi = node-&gt;vsi_index;</div><div class='add'>+	if (node-&gt;parent)</div><div class='add'>+		node_info.parent_id = node-&gt;parent-&gt;index;</div><div class='add'>+	else</div><div class='add'>+		node_info.parent_id = node_info.id;</div><div class='add'>+</div><div class='add'>+	node_info.weight = node-&gt;rel_bw;</div><div class='add'>+	node_info.tc = node-&gt;traffic_class;</div><div class='add'>+	node_info.prio_type = node-&gt;prio_type;</div><div class='add'>+	node_info.type_leaf = node-&gt;type_leaf;</div><div class='add'>+	node_info.enable = node-&gt;enable;</div><div class='add'>+	if (irdma_cqp_ws_node_cmd(vsi-&gt;dev, cmd, &amp;node_info)) {</div><div class='add'>+		ibdev_dbg(to_ibdev(vsi-&gt;dev), "WS: CQP WS CMD failed\n");</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (node-&gt;type_leaf &amp;&amp; cmd == IRDMA_OP_WS_ADD_NODE) {</div><div class='add'>+		node-&gt;qs_handle = node_info.qs_handle;</div><div class='add'>+		vsi-&gt;qos[node-&gt;user_pri].qs_handle = node_info.qs_handle;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * ws_find_node - Find SC WS node based on VSI id or TC</div><div class='add'>+ * @parent: parent node of First VSI or TC node</div><div class='add'>+ * @match_val: value to match</div><div class='add'>+ * @type: match type VSI/TC</div><div class='add'>+ */</div><div class='add'>+static struct irdma_ws_node *ws_find_node(struct irdma_ws_node *parent,</div><div class='add'>+					  u16 match_val,</div><div class='add'>+					  enum irdma_ws_match_type type)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ws_node *node;</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case WS_MATCH_TYPE_VSI:</div><div class='add'>+		list_for_each_entry(node, &amp;parent-&gt;child_list_head, siblings) {</div><div class='add'>+			if (node-&gt;vsi_index == match_val)</div><div class='add'>+				return node;</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	case WS_MATCH_TYPE_TC:</div><div class='add'>+		list_for_each_entry(node, &amp;parent-&gt;child_list_head, siblings) {</div><div class='add'>+			if (node-&gt;traffic_class == match_val)</div><div class='add'>+				return node;</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_tc_in_use - Checks to see if a leaf node is in use</div><div class='add'>+ * @vsi: vsi pointer</div><div class='add'>+ * @user_pri: user priority</div><div class='add'>+ */</div><div class='add'>+static bool irdma_tc_in_use(struct irdma_sc_vsi *vsi, u8 user_pri)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;vsi-&gt;qos[user_pri].qos_mutex);</div><div class='add'>+	if (!list_empty(&amp;vsi-&gt;qos[user_pri].qplist)) {</div><div class='add'>+		mutex_unlock(&amp;vsi-&gt;qos[user_pri].qos_mutex);</div><div class='add'>+		return true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Check if the traffic class associated with the given user priority</div><div class='add'>+	 * is in use by any other user priority. If so, nothing left to do</div><div class='add'>+	 */</div><div class='add'>+	for (i = 0; i &lt; IRDMA_MAX_USER_PRIORITY; i++) {</div><div class='add'>+		if (vsi-&gt;qos[i].traffic_class == vsi-&gt;qos[user_pri].traffic_class &amp;&amp;</div><div class='add'>+		    !list_empty(&amp;vsi-&gt;qos[i].qplist)) {</div><div class='add'>+			mutex_unlock(&amp;vsi-&gt;qos[user_pri].qos_mutex);</div><div class='add'>+			return true;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;vsi-&gt;qos[user_pri].qos_mutex);</div><div class='add'>+</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_remove_leaf - Remove leaf node unconditionally</div><div class='add'>+ * @vsi: vsi pointer</div><div class='add'>+ * @user_pri: user priority</div><div class='add'>+ */</div><div class='add'>+static void irdma_remove_leaf(struct irdma_sc_vsi *vsi, u8 user_pri)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ws_node *ws_tree_root, *vsi_node, *tc_node;</div><div class='add'>+	int i;</div><div class='add'>+	u16 traffic_class;</div><div class='add'>+</div><div class='add'>+	traffic_class = vsi-&gt;qos[user_pri].traffic_class;</div><div class='add'>+	for (i = 0; i &lt; IRDMA_MAX_USER_PRIORITY; i++)</div><div class='add'>+		if (vsi-&gt;qos[i].traffic_class == traffic_class)</div><div class='add'>+			vsi-&gt;qos[i].valid = false;</div><div class='add'>+</div><div class='add'>+	ws_tree_root = vsi-&gt;dev-&gt;ws_tree_root;</div><div class='add'>+	if (!ws_tree_root)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	vsi_node = ws_find_node(ws_tree_root, vsi-&gt;vsi_idx,</div><div class='add'>+				WS_MATCH_TYPE_VSI);</div><div class='add'>+	if (!vsi_node)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	tc_node = ws_find_node(vsi_node,</div><div class='add'>+			       vsi-&gt;qos[user_pri].traffic_class,</div><div class='add'>+			       WS_MATCH_TYPE_TC);</div><div class='add'>+	if (!tc_node)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_DELETE_NODE);</div><div class='add'>+	vsi-&gt;unregister_qset(vsi, tc_node);</div><div class='add'>+	list_del(&amp;tc_node-&gt;siblings);</div><div class='add'>+	irdma_free_node(vsi, tc_node);</div><div class='add'>+	/* Check if VSI node can be freed */</div><div class='add'>+	if (list_empty(&amp;vsi_node-&gt;child_list_head)) {</div><div class='add'>+		irdma_ws_cqp_cmd(vsi, vsi_node, IRDMA_OP_WS_DELETE_NODE);</div><div class='add'>+		list_del(&amp;vsi_node-&gt;siblings);</div><div class='add'>+		irdma_free_node(vsi, vsi_node);</div><div class='add'>+		/* Free head node there are no remaining VSI nodes */</div><div class='add'>+		if (list_empty(&amp;ws_tree_root-&gt;child_list_head)) {</div><div class='add'>+			irdma_ws_cqp_cmd(vsi, ws_tree_root,</div><div class='add'>+					 IRDMA_OP_WS_DELETE_NODE);</div><div class='add'>+			irdma_free_node(vsi, ws_tree_root);</div><div class='add'>+			vsi-&gt;dev-&gt;ws_tree_root = NULL;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ws_add - Build work scheduler tree, set RDMA qs_handle</div><div class='add'>+ * @vsi: vsi pointer</div><div class='add'>+ * @user_pri: user priority</div><div class='add'>+ */</div><div class='add'>+int irdma_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri)</div><div class='add'>+{</div><div class='add'>+	struct irdma_ws_node *ws_tree_root;</div><div class='add'>+	struct irdma_ws_node *vsi_node;</div><div class='add'>+	struct irdma_ws_node *tc_node;</div><div class='add'>+	u16 traffic_class;</div><div class='add'>+	int ret = 0;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;vsi-&gt;dev-&gt;ws_mutex);</div><div class='add'>+	if (vsi-&gt;tc_change_pending) {</div><div class='add'>+		ret = -EBUSY;</div><div class='add'>+		goto exit;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (vsi-&gt;qos[user_pri].valid)</div><div class='add'>+		goto exit;</div><div class='add'>+</div><div class='add'>+	ws_tree_root = vsi-&gt;dev-&gt;ws_tree_root;</div><div class='add'>+	if (!ws_tree_root) {</div><div class='add'>+		ibdev_dbg(to_ibdev(vsi-&gt;dev), "WS: Creating root node\n");</div><div class='add'>+		ws_tree_root = irdma_alloc_node(vsi, user_pri,</div><div class='add'>+						WS_NODE_TYPE_PARENT, NULL);</div><div class='add'>+		if (!ws_tree_root) {</div><div class='add'>+			ret = -ENOMEM;</div><div class='add'>+			goto exit;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ret = irdma_ws_cqp_cmd(vsi, ws_tree_root, IRDMA_OP_WS_ADD_NODE);</div><div class='add'>+		if (ret) {</div><div class='add'>+			irdma_free_node(vsi, ws_tree_root);</div><div class='add'>+			goto exit;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		vsi-&gt;dev-&gt;ws_tree_root = ws_tree_root;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Find a second tier node that matches the VSI */</div><div class='add'>+	vsi_node = ws_find_node(ws_tree_root, vsi-&gt;vsi_idx,</div><div class='add'>+				WS_MATCH_TYPE_VSI);</div><div class='add'>+</div><div class='add'>+	/* If VSI node doesn't exist, add one */</div><div class='add'>+	if (!vsi_node) {</div><div class='add'>+		ibdev_dbg(to_ibdev(vsi-&gt;dev),</div><div class='add'>+			  "WS: Node not found matching VSI %d\n",</div><div class='add'>+			  vsi-&gt;vsi_idx);</div><div class='add'>+		vsi_node = irdma_alloc_node(vsi, user_pri, WS_NODE_TYPE_PARENT,</div><div class='add'>+					    ws_tree_root);</div><div class='add'>+		if (!vsi_node) {</div><div class='add'>+			ret = -ENOMEM;</div><div class='add'>+			goto vsi_add_err;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ret = irdma_ws_cqp_cmd(vsi, vsi_node, IRDMA_OP_WS_ADD_NODE);</div><div class='add'>+		if (ret) {</div><div class='add'>+			irdma_free_node(vsi, vsi_node);</div><div class='add'>+			goto vsi_add_err;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		list_add(&amp;vsi_node-&gt;siblings, &amp;ws_tree_root-&gt;child_list_head);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(to_ibdev(vsi-&gt;dev),</div><div class='add'>+		  "WS: Using node %d which represents VSI %d\n",</div><div class='add'>+		  vsi_node-&gt;index, vsi-&gt;vsi_idx);</div><div class='add'>+	traffic_class = vsi-&gt;qos[user_pri].traffic_class;</div><div class='add'>+	tc_node = ws_find_node(vsi_node, traffic_class,</div><div class='add'>+			       WS_MATCH_TYPE_TC);</div><div class='add'>+	if (!tc_node) {</div><div class='add'>+		/* Add leaf node */</div><div class='add'>+		ibdev_dbg(to_ibdev(vsi-&gt;dev),</div><div class='add'>+			  "WS: Node not found matching VSI %d and TC %d\n",</div><div class='add'>+			  vsi-&gt;vsi_idx, traffic_class);</div><div class='add'>+		tc_node = irdma_alloc_node(vsi, user_pri, WS_NODE_TYPE_LEAF,</div><div class='add'>+					   vsi_node);</div><div class='add'>+		if (!tc_node) {</div><div class='add'>+			ret = -ENOMEM;</div><div class='add'>+			goto leaf_add_err;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ret = irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_ADD_NODE);</div><div class='add'>+		if (ret) {</div><div class='add'>+			irdma_free_node(vsi, tc_node);</div><div class='add'>+			goto leaf_add_err;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		list_add(&amp;tc_node-&gt;siblings, &amp;vsi_node-&gt;child_list_head);</div><div class='add'>+		/*</div><div class='add'>+		 * callback to LAN to update the LAN tree with our node</div><div class='add'>+		 */</div><div class='add'>+		ret = vsi-&gt;register_qset(vsi, tc_node);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto reg_err;</div><div class='add'>+</div><div class='add'>+		tc_node-&gt;enable = true;</div><div class='add'>+		ret = irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_MODIFY_NODE);</div><div class='add'>+		if (ret) {</div><div class='add'>+			vsi-&gt;unregister_qset(vsi, tc_node);</div><div class='add'>+			goto reg_err;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	ibdev_dbg(to_ibdev(vsi-&gt;dev),</div><div class='add'>+		  "WS: Using node %d which represents VSI %d TC %d\n",</div><div class='add'>+		  tc_node-&gt;index, vsi-&gt;vsi_idx, traffic_class);</div><div class='add'>+	/*</div><div class='add'>+	 * Iterate through other UPs and update the QS handle if they have</div><div class='add'>+	 * a matching traffic class.</div><div class='add'>+	 */</div><div class='add'>+	for (i = 0; i &lt; IRDMA_MAX_USER_PRIORITY; i++) {</div><div class='add'>+		if (vsi-&gt;qos[i].traffic_class == traffic_class) {</div><div class='add'>+			vsi-&gt;qos[i].qs_handle = tc_node-&gt;qs_handle;</div><div class='add'>+			vsi-&gt;qos[i].lan_qos_handle = tc_node-&gt;lan_qs_handle;</div><div class='add'>+			vsi-&gt;qos[i].l2_sched_node_id = tc_node-&gt;l2_sched_node_id;</div><div class='add'>+			vsi-&gt;qos[i].valid = true;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	goto exit;</div><div class='add'>+</div><div class='add'>+reg_err:</div><div class='add'>+	irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_DELETE_NODE);</div><div class='add'>+	list_del(&amp;tc_node-&gt;siblings);</div><div class='add'>+	irdma_free_node(vsi, tc_node);</div><div class='add'>+leaf_add_err:</div><div class='add'>+	if (list_empty(&amp;vsi_node-&gt;child_list_head)) {</div><div class='add'>+		if (irdma_ws_cqp_cmd(vsi, vsi_node, IRDMA_OP_WS_DELETE_NODE))</div><div class='add'>+			goto exit;</div><div class='add'>+		list_del(&amp;vsi_node-&gt;siblings);</div><div class='add'>+		irdma_free_node(vsi, vsi_node);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+vsi_add_err:</div><div class='add'>+	/* Free head node there are no remaining VSI nodes */</div><div class='add'>+	if (list_empty(&amp;ws_tree_root-&gt;child_list_head)) {</div><div class='add'>+		irdma_ws_cqp_cmd(vsi, ws_tree_root, IRDMA_OP_WS_DELETE_NODE);</div><div class='add'>+		vsi-&gt;dev-&gt;ws_tree_root = NULL;</div><div class='add'>+		irdma_free_node(vsi, ws_tree_root);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+exit:</div><div class='add'>+	mutex_unlock(&amp;vsi-&gt;dev-&gt;ws_mutex);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ws_remove - Free WS scheduler node, update WS tree</div><div class='add'>+ * @vsi: vsi pointer</div><div class='add'>+ * @user_pri: user priority</div><div class='add'>+ */</div><div class='add'>+void irdma_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri)</div><div class='add'>+{</div><div class='add'>+	mutex_lock(&amp;vsi-&gt;dev-&gt;ws_mutex);</div><div class='add'>+	if (irdma_tc_in_use(vsi, user_pri))</div><div class='add'>+		goto exit;</div><div class='add'>+	irdma_remove_leaf(vsi, user_pri);</div><div class='add'>+exit:</div><div class='add'>+	mutex_unlock(&amp;vsi-&gt;dev-&gt;ws_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * irdma_ws_reset - Reset entire WS tree</div><div class='add'>+ * @vsi: vsi pointer</div><div class='add'>+ */</div><div class='add'>+void irdma_ws_reset(struct irdma_sc_vsi *vsi)</div><div class='add'>+{</div><div class='add'>+	u8 i;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;vsi-&gt;dev-&gt;ws_mutex);</div><div class='add'>+	for (i = 0; i &lt; IRDMA_MAX_USER_PRIORITY; ++i)</div><div class='add'>+		irdma_remove_leaf(vsi, i);</div><div class='add'>+	mutex_unlock(&amp;vsi-&gt;dev-&gt;ws_mutex);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/irdma/ws.h b/drivers/infiniband/hw/irdma/ws.h<br/>new file mode 100644<br/>index 000000000000..45490031a389<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/irdma/ws.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/irdma/ws.h</a></div><div class='hunk'>@@ -0,0 +1,41 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2015 - 2020 Intel Corporation */</div><div class='add'>+#ifndef IRDMA_WS_H</div><div class='add'>+#define IRDMA_WS_H</div><div class='add'>+</div><div class='add'>+#include "osdep.h"</div><div class='add'>+</div><div class='add'>+enum irdma_ws_node_type {</div><div class='add'>+	WS_NODE_TYPE_PARENT,</div><div class='add'>+	WS_NODE_TYPE_LEAF,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum irdma_ws_match_type {</div><div class='add'>+	WS_MATCH_TYPE_VSI,</div><div class='add'>+	WS_MATCH_TYPE_TC,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_ws_node {</div><div class='add'>+	struct list_head siblings;</div><div class='add'>+	struct list_head child_list_head;</div><div class='add'>+	struct irdma_ws_node *parent;</div><div class='add'>+	u64 lan_qs_handle; /* opaque handle used by LAN */</div><div class='add'>+	u32 l2_sched_node_id;</div><div class='add'>+	u16 index;</div><div class='add'>+	u16 qs_handle;</div><div class='add'>+	u16 vsi_index;</div><div class='add'>+	u8 traffic_class;</div><div class='add'>+	u8 user_pri;</div><div class='add'>+	u8 rel_bw;</div><div class='add'>+	u8 abstraction_layer; /* used for splitting a TC */</div><div class='add'>+	u8 prio_type;</div><div class='add'>+	bool type_leaf:1;</div><div class='add'>+	bool enable:1;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct irdma_sc_vsi;</div><div class='add'>+int irdma_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri);</div><div class='add'>+void irdma_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri);</div><div class='add'>+void irdma_ws_reset(struct irdma_sc_vsi *vsi);</div><div class='add'>+</div><div class='add'>+#endif /* IRDMA_WS_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/mana/Kconfig b/drivers/infiniband/hw/mana/Kconfig<br/>new file mode 100644<br/>index 000000000000..546640657bac<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mana/Kconfig?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mana/Kconfig</a></div><div class='hunk'>@@ -0,0 +1,10 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+config MANA_INFINIBAND</div><div class='add'>+	tristate "Microsoft Azure Network Adapter support"</div><div class='add'>+	depends on NETDEVICES &amp;&amp; ETHERNET &amp;&amp; PCI &amp;&amp; MICROSOFT_MANA</div><div class='add'>+	help</div><div class='add'>+	  This driver provides low-level RDMA support for Microsoft Azure</div><div class='add'>+	  Network Adapter (MANA). MANA supports RDMA features that can be used</div><div class='add'>+	  for workloads (e.g. DPDK, MPI etc) that uses RDMA verbs to directly</div><div class='add'>+	  access hardware from user-mode processes in Microsoft Azure cloud</div><div class='add'>+	  environment.</div><div class='head'>diff --git a/drivers/infiniband/hw/mana/Makefile b/drivers/infiniband/hw/mana/Makefile<br/>new file mode 100644<br/>index 000000000000..921c05e08b11<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mana/Makefile?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mana/Makefile</a></div><div class='hunk'>@@ -0,0 +1,4 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+obj-$(CONFIG_MANA_INFINIBAND) += mana_ib.o</div><div class='add'>+</div><div class='add'>+mana_ib-y := device.o main.o wq.o qp.o cq.o mr.o ah.o wr.o counters.o</div><div class='head'>diff --git a/drivers/infiniband/hw/mana/ah.c b/drivers/infiniband/hw/mana/ah.c<br/>new file mode 100644<br/>index 000000000000..f56952eebbaa<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mana/ah.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mana/ah.c</a></div><div class='hunk'>@@ -0,0 +1,58 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2024, Microsoft Corporation. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include "mana_ib.h"</div><div class='add'>+</div><div class='add'>+int mana_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr,</div><div class='add'>+		      struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev = container_of(ibah-&gt;device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct mana_ib_ah *ah = container_of(ibah, struct mana_ib_ah, ibah);</div><div class='add'>+	struct rdma_ah_attr *ah_attr = attr-&gt;ah_attr;</div><div class='add'>+	const struct ib_global_route *grh;</div><div class='add'>+	enum rdma_network_type ntype;</div><div class='add'>+</div><div class='add'>+	if (ah_attr-&gt;type != RDMA_AH_ATTR_TYPE_ROCE ||</div><div class='add'>+	    !(rdma_ah_get_ah_flags(ah_attr) &amp; IB_AH_GRH))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (udata)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ah-&gt;av = dma_pool_zalloc(mdev-&gt;av_pool, GFP_ATOMIC, &amp;ah-&gt;dma_handle);</div><div class='add'>+	if (!ah-&gt;av)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	grh = rdma_ah_read_grh(ah_attr);</div><div class='add'>+	ntype = rdma_gid_attr_network_type(grh-&gt;sgid_attr);</div><div class='add'>+</div><div class='add'>+	copy_in_reverse(ah-&gt;av-&gt;dest_mac, ah_attr-&gt;roce.dmac, ETH_ALEN);</div><div class='add'>+	ah-&gt;av-&gt;udp_src_port = rdma_flow_label_to_udp_sport(grh-&gt;flow_label);</div><div class='add'>+	ah-&gt;av-&gt;hop_limit = grh-&gt;hop_limit;</div><div class='add'>+	ah-&gt;av-&gt;dscp = (grh-&gt;traffic_class &gt;&gt; 2) &amp; 0x3f;</div><div class='add'>+	ah-&gt;av-&gt;is_ipv6 = (ntype == RDMA_NETWORK_IPV6);</div><div class='add'>+</div><div class='add'>+	if (ah-&gt;av-&gt;is_ipv6) {</div><div class='add'>+		copy_in_reverse(ah-&gt;av-&gt;dest_ip, grh-&gt;dgid.raw, 16);</div><div class='add'>+		copy_in_reverse(ah-&gt;av-&gt;src_ip, grh-&gt;sgid_attr-&gt;gid.raw, 16);</div><div class='add'>+	} else {</div><div class='add'>+		ah-&gt;av-&gt;dest_ip[10] = 0xFF;</div><div class='add'>+		ah-&gt;av-&gt;dest_ip[11] = 0xFF;</div><div class='add'>+		copy_in_reverse(&amp;ah-&gt;av-&gt;dest_ip[12], &amp;grh-&gt;dgid.raw[12], 4);</div><div class='add'>+		copy_in_reverse(&amp;ah-&gt;av-&gt;src_ip[12], &amp;grh-&gt;sgid_attr-&gt;gid.raw[12], 4);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_destroy_ah(struct ib_ah *ibah, u32 flags)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev = container_of(ibah-&gt;device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct mana_ib_ah *ah = container_of(ibah, struct mana_ib_ah, ibah);</div><div class='add'>+</div><div class='add'>+	dma_pool_free(mdev-&gt;av_pool, ah-&gt;av, ah-&gt;dma_handle);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mana/counters.c b/drivers/infiniband/hw/mana/counters.c<br/>new file mode 100644<br/>index 000000000000..e964e74be48d<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mana/counters.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mana/counters.c</a></div><div class='hunk'>@@ -0,0 +1,179 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2024, Microsoft Corporation. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include "counters.h"</div><div class='add'>+</div><div class='add'>+static const struct rdma_stat_desc mana_ib_port_stats_desc[] = {</div><div class='add'>+	[MANA_IB_REQUESTER_TIMEOUT].name = "requester_timeout",</div><div class='add'>+	[MANA_IB_REQUESTER_OOS_NAK].name = "requester_oos_nak",</div><div class='add'>+	[MANA_IB_REQUESTER_RNR_NAK].name = "requester_rnr_nak",</div><div class='add'>+	[MANA_IB_RESPONDER_RNR_NAK].name = "responder_rnr_nak",</div><div class='add'>+	[MANA_IB_RESPONDER_OOS].name = "responder_oos",</div><div class='add'>+	[MANA_IB_RESPONDER_DUP_REQUEST].name = "responder_dup_request",</div><div class='add'>+	[MANA_IB_REQUESTER_IMPLICIT_NAK].name = "requester_implicit_nak",</div><div class='add'>+	[MANA_IB_REQUESTER_READRESP_PSN_MISMATCH].name = "requester_readresp_psn_mismatch",</div><div class='add'>+	[MANA_IB_NAK_INV_REQ].name = "nak_inv_req",</div><div class='add'>+	[MANA_IB_NAK_ACCESS_ERR].name = "nak_access_error",</div><div class='add'>+	[MANA_IB_NAK_OPP_ERR].name = "nak_opp_error",</div><div class='add'>+	[MANA_IB_NAK_INV_READ].name = "nak_inv_read",</div><div class='add'>+	[MANA_IB_RESPONDER_LOCAL_LEN_ERR].name = "responder_local_len_error",</div><div class='add'>+	[MANA_IB_REQUESTOR_LOCAL_PROT_ERR].name = "requestor_local_prot_error",</div><div class='add'>+	[MANA_IB_RESPONDER_REM_ACCESS_ERR].name = "responder_rem_access_error",</div><div class='add'>+	[MANA_IB_RESPONDER_LOCAL_QP_ERR].name = "responder_local_qp_error",</div><div class='add'>+	[MANA_IB_RESPONDER_MALFORMED_WQE].name = "responder_malformed_wqe",</div><div class='add'>+	[MANA_IB_GENERAL_HW_ERR].name = "general_hw_error",</div><div class='add'>+	[MANA_IB_REQUESTER_RNR_NAK_RETRIES_EXCEEDED].name = "requester_rnr_nak_retries_exceeded",</div><div class='add'>+	[MANA_IB_REQUESTER_RETRIES_EXCEEDED].name = "requester_retries_exceeded",</div><div class='add'>+	[MANA_IB_TOTAL_FATAL_ERR].name = "total_fatal_error",</div><div class='add'>+	[MANA_IB_RECEIVED_CNPS].name = "received_cnps",</div><div class='add'>+	[MANA_IB_NUM_QPS_CONGESTED].name = "num_qps_congested",</div><div class='add'>+	[MANA_IB_RATE_INC_EVENTS].name = "rate_inc_events",</div><div class='add'>+	[MANA_IB_NUM_QPS_RECOVERED].name = "num_qps_recovered",</div><div class='add'>+	[MANA_IB_CURRENT_RATE].name = "current_rate",</div><div class='add'>+	[MANA_IB_DUP_RX_REQ].name = "dup_rx_requests",</div><div class='add'>+	[MANA_IB_TX_BYTES].name = "tx_bytes",</div><div class='add'>+	[MANA_IB_RX_BYTES].name = "rx_bytes",</div><div class='add'>+	[MANA_IB_RX_SEND_REQ].name = "rx_send_requests",</div><div class='add'>+	[MANA_IB_RX_WRITE_REQ].name = "rx_write_requests",</div><div class='add'>+	[MANA_IB_RX_READ_REQ].name = "rx_read_requests",</div><div class='add'>+	[MANA_IB_TX_PKT].name = "tx_packets",</div><div class='add'>+	[MANA_IB_RX_PKT].name = "rx_packets",</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct rdma_stat_desc mana_ib_device_stats_desc[] = {</div><div class='add'>+	[MANA_IB_SENT_CNPS].name = "sent_cnps",</div><div class='add'>+	[MANA_IB_RECEIVED_ECNS].name = "received_ecns",</div><div class='add'>+	[MANA_IB_RECEIVED_CNP_COUNT].name = "received_cnp_count",</div><div class='add'>+	[MANA_IB_QP_CONGESTED_EVENTS].name = "qp_congested_events",</div><div class='add'>+	[MANA_IB_QP_RECOVERED_EVENTS].name = "qp_recovered_events",</div><div class='add'>+	[MANA_IB_DEV_RATE_INC_EVENTS].name = "rate_inc_events",</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct rdma_hw_stats *mana_ib_alloc_hw_device_stats(struct ib_device *ibdev)</div><div class='add'>+{</div><div class='add'>+	return rdma_alloc_hw_stats_struct(mana_ib_device_stats_desc,</div><div class='add'>+					  ARRAY_SIZE(mana_ib_device_stats_desc),</div><div class='add'>+					  RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct rdma_hw_stats *mana_ib_alloc_hw_port_stats(struct ib_device *ibdev,</div><div class='add'>+						  u32 port_num)</div><div class='add'>+{</div><div class='add'>+	return rdma_alloc_hw_stats_struct(mana_ib_port_stats_desc,</div><div class='add'>+					  ARRAY_SIZE(mana_ib_port_stats_desc),</div><div class='add'>+					  RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_ib_get_hw_device_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev,</div><div class='add'>+						ib_dev);</div><div class='add'>+	struct mana_rnic_query_device_cntrs_resp resp = {};</div><div class='add'>+	struct mana_rnic_query_device_cntrs_req req = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, MANA_IB_QUERY_DEVICE_COUNTERS,</div><div class='add'>+			     sizeof(req), sizeof(resp));</div><div class='add'>+	req.hdr.dev_id = mdev-&gt;gdma_dev-&gt;dev_id;</div><div class='add'>+	req.adapter = mdev-&gt;adapter_handle;</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(mdev_to_gc(mdev), sizeof(req), &amp;req,</div><div class='add'>+				   sizeof(resp), &amp;resp);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to query device counters err %d",</div><div class='add'>+			  err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	stats-&gt;value[MANA_IB_SENT_CNPS] = resp.sent_cnps;</div><div class='add'>+	stats-&gt;value[MANA_IB_RECEIVED_ECNS] = resp.received_ecns;</div><div class='add'>+	stats-&gt;value[MANA_IB_RECEIVED_CNP_COUNT] = resp.received_cnp_count;</div><div class='add'>+	stats-&gt;value[MANA_IB_QP_CONGESTED_EVENTS] = resp.qp_congested_events;</div><div class='add'>+	stats-&gt;value[MANA_IB_QP_RECOVERED_EVENTS] = resp.qp_recovered_events;</div><div class='add'>+	stats-&gt;value[MANA_IB_DEV_RATE_INC_EVENTS] = resp.rate_inc_events;</div><div class='add'>+</div><div class='add'>+	return ARRAY_SIZE(mana_ib_device_stats_desc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_ib_get_hw_port_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,</div><div class='add'>+				     u32 port_num)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev,</div><div class='add'>+						ib_dev);</div><div class='add'>+	struct mana_rnic_query_vf_cntrs_resp resp = {};</div><div class='add'>+	struct mana_rnic_query_vf_cntrs_req req = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, MANA_IB_QUERY_VF_COUNTERS,</div><div class='add'>+			     sizeof(req), sizeof(resp));</div><div class='add'>+	req.hdr.resp.msg_version = GDMA_MESSAGE_V2;</div><div class='add'>+	req.hdr.dev_id = mdev-&gt;gdma_dev-&gt;dev_id;</div><div class='add'>+	req.adapter = mdev-&gt;adapter_handle;</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(mdev_to_gc(mdev), sizeof(req), &amp;req,</div><div class='add'>+				   sizeof(resp), &amp;resp);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to query vf counters err %d",</div><div class='add'>+			  err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	stats-&gt;value[MANA_IB_REQUESTER_TIMEOUT] = resp.requester_timeout;</div><div class='add'>+	stats-&gt;value[MANA_IB_REQUESTER_OOS_NAK] = resp.requester_oos_nak;</div><div class='add'>+	stats-&gt;value[MANA_IB_REQUESTER_RNR_NAK] = resp.requester_rnr_nak;</div><div class='add'>+	stats-&gt;value[MANA_IB_RESPONDER_RNR_NAK] = resp.responder_rnr_nak;</div><div class='add'>+	stats-&gt;value[MANA_IB_RESPONDER_OOS] = resp.responder_oos;</div><div class='add'>+	stats-&gt;value[MANA_IB_RESPONDER_DUP_REQUEST] = resp.responder_dup_request;</div><div class='add'>+	stats-&gt;value[MANA_IB_REQUESTER_IMPLICIT_NAK] =</div><div class='add'>+					resp.requester_implicit_nak;</div><div class='add'>+	stats-&gt;value[MANA_IB_REQUESTER_READRESP_PSN_MISMATCH] =</div><div class='add'>+					resp.requester_readresp_psn_mismatch;</div><div class='add'>+	stats-&gt;value[MANA_IB_NAK_INV_REQ] = resp.nak_inv_req;</div><div class='add'>+	stats-&gt;value[MANA_IB_NAK_ACCESS_ERR] = resp.nak_access_err;</div><div class='add'>+	stats-&gt;value[MANA_IB_NAK_OPP_ERR] = resp.nak_opp_err;</div><div class='add'>+	stats-&gt;value[MANA_IB_NAK_INV_READ] = resp.nak_inv_read;</div><div class='add'>+	stats-&gt;value[MANA_IB_RESPONDER_LOCAL_LEN_ERR] =</div><div class='add'>+					resp.responder_local_len_err;</div><div class='add'>+	stats-&gt;value[MANA_IB_REQUESTOR_LOCAL_PROT_ERR] =</div><div class='add'>+					resp.requestor_local_prot_err;</div><div class='add'>+	stats-&gt;value[MANA_IB_RESPONDER_REM_ACCESS_ERR] =</div><div class='add'>+					resp.responder_rem_access_err;</div><div class='add'>+	stats-&gt;value[MANA_IB_RESPONDER_LOCAL_QP_ERR] =</div><div class='add'>+					resp.responder_local_qp_err;</div><div class='add'>+	stats-&gt;value[MANA_IB_RESPONDER_MALFORMED_WQE] =</div><div class='add'>+					resp.responder_malformed_wqe;</div><div class='add'>+	stats-&gt;value[MANA_IB_GENERAL_HW_ERR] = resp.general_hw_err;</div><div class='add'>+	stats-&gt;value[MANA_IB_REQUESTER_RNR_NAK_RETRIES_EXCEEDED] =</div><div class='add'>+					resp.requester_rnr_nak_retries_exceeded;</div><div class='add'>+	stats-&gt;value[MANA_IB_REQUESTER_RETRIES_EXCEEDED] =</div><div class='add'>+					resp.requester_retries_exceeded;</div><div class='add'>+	stats-&gt;value[MANA_IB_TOTAL_FATAL_ERR] = resp.total_fatal_err;</div><div class='add'>+</div><div class='add'>+	stats-&gt;value[MANA_IB_RECEIVED_CNPS] = resp.received_cnps;</div><div class='add'>+	stats-&gt;value[MANA_IB_NUM_QPS_CONGESTED] = resp.num_qps_congested;</div><div class='add'>+	stats-&gt;value[MANA_IB_RATE_INC_EVENTS] = resp.rate_inc_events;</div><div class='add'>+	stats-&gt;value[MANA_IB_NUM_QPS_RECOVERED] = resp.num_qps_recovered;</div><div class='add'>+	stats-&gt;value[MANA_IB_CURRENT_RATE] = resp.current_rate;</div><div class='add'>+</div><div class='add'>+	stats-&gt;value[MANA_IB_DUP_RX_REQ] = resp.dup_rx_req;</div><div class='add'>+	stats-&gt;value[MANA_IB_TX_BYTES] = resp.tx_bytes;</div><div class='add'>+	stats-&gt;value[MANA_IB_RX_BYTES] = resp.rx_bytes;</div><div class='add'>+	stats-&gt;value[MANA_IB_RX_SEND_REQ] = resp.rx_send_req;</div><div class='add'>+	stats-&gt;value[MANA_IB_RX_WRITE_REQ] = resp.rx_write_req;</div><div class='add'>+	stats-&gt;value[MANA_IB_RX_READ_REQ] = resp.rx_read_req;</div><div class='add'>+	stats-&gt;value[MANA_IB_TX_PKT] = resp.tx_pkt;</div><div class='add'>+	stats-&gt;value[MANA_IB_RX_PKT] = resp.rx_pkt;</div><div class='add'>+</div><div class='add'>+	return ARRAY_SIZE(mana_ib_port_stats_desc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,</div><div class='add'>+			 u32 port_num, int index)</div><div class='add'>+{</div><div class='add'>+	if (!port_num)</div><div class='add'>+		return mana_ib_get_hw_device_stats(ibdev, stats);</div><div class='add'>+	else</div><div class='add'>+		return mana_ib_get_hw_port_stats(ibdev, stats, port_num);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mana/counters.h b/drivers/infiniband/hw/mana/counters.h<br/>new file mode 100644<br/>index 000000000000..f68e776bb41d<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mana/counters.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mana/counters.h</a></div><div class='hunk'>@@ -0,0 +1,62 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0-only */</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2024 Microsoft Corporation. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef _COUNTERS_H_</div><div class='add'>+#define _COUNTERS_H_</div><div class='add'>+</div><div class='add'>+#include "mana_ib.h"</div><div class='add'>+</div><div class='add'>+enum mana_ib_port_counters {</div><div class='add'>+	MANA_IB_REQUESTER_TIMEOUT,</div><div class='add'>+	MANA_IB_REQUESTER_OOS_NAK,</div><div class='add'>+	MANA_IB_REQUESTER_RNR_NAK,</div><div class='add'>+	MANA_IB_RESPONDER_RNR_NAK,</div><div class='add'>+	MANA_IB_RESPONDER_OOS,</div><div class='add'>+	MANA_IB_RESPONDER_DUP_REQUEST,</div><div class='add'>+	MANA_IB_REQUESTER_IMPLICIT_NAK,</div><div class='add'>+	MANA_IB_REQUESTER_READRESP_PSN_MISMATCH,</div><div class='add'>+	MANA_IB_NAK_INV_REQ,</div><div class='add'>+	MANA_IB_NAK_ACCESS_ERR,</div><div class='add'>+	MANA_IB_NAK_OPP_ERR,</div><div class='add'>+	MANA_IB_NAK_INV_READ,</div><div class='add'>+	MANA_IB_RESPONDER_LOCAL_LEN_ERR,</div><div class='add'>+	MANA_IB_REQUESTOR_LOCAL_PROT_ERR,</div><div class='add'>+	MANA_IB_RESPONDER_REM_ACCESS_ERR,</div><div class='add'>+	MANA_IB_RESPONDER_LOCAL_QP_ERR,</div><div class='add'>+	MANA_IB_RESPONDER_MALFORMED_WQE,</div><div class='add'>+	MANA_IB_GENERAL_HW_ERR,</div><div class='add'>+	MANA_IB_REQUESTER_RNR_NAK_RETRIES_EXCEEDED,</div><div class='add'>+	MANA_IB_REQUESTER_RETRIES_EXCEEDED,</div><div class='add'>+	MANA_IB_TOTAL_FATAL_ERR,</div><div class='add'>+	MANA_IB_RECEIVED_CNPS,</div><div class='add'>+	MANA_IB_NUM_QPS_CONGESTED,</div><div class='add'>+	MANA_IB_RATE_INC_EVENTS,</div><div class='add'>+	MANA_IB_NUM_QPS_RECOVERED,</div><div class='add'>+	MANA_IB_CURRENT_RATE,</div><div class='add'>+	MANA_IB_DUP_RX_REQ,</div><div class='add'>+	MANA_IB_TX_BYTES,</div><div class='add'>+	MANA_IB_RX_BYTES,</div><div class='add'>+	MANA_IB_RX_SEND_REQ,</div><div class='add'>+	MANA_IB_RX_WRITE_REQ,</div><div class='add'>+	MANA_IB_RX_READ_REQ,</div><div class='add'>+	MANA_IB_TX_PKT,</div><div class='add'>+	MANA_IB_RX_PKT,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum mana_ib_device_counters {</div><div class='add'>+	MANA_IB_SENT_CNPS,</div><div class='add'>+	MANA_IB_RECEIVED_ECNS,</div><div class='add'>+	MANA_IB_RECEIVED_CNP_COUNT,</div><div class='add'>+	MANA_IB_QP_CONGESTED_EVENTS,</div><div class='add'>+	MANA_IB_QP_RECOVERED_EVENTS,</div><div class='add'>+	MANA_IB_DEV_RATE_INC_EVENTS,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct rdma_hw_stats *mana_ib_alloc_hw_port_stats(struct ib_device *ibdev,</div><div class='add'>+						  u32 port_num);</div><div class='add'>+struct rdma_hw_stats *mana_ib_alloc_hw_device_stats(struct ib_device *ibdev);</div><div class='add'>+int mana_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,</div><div class='add'>+			 u32 port_num, int index);</div><div class='add'>+#endif /* _COUNTERS_H_ */</div><div class='head'>diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c<br/>new file mode 100644<br/>index 000000000000..1becc8779123<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mana/cq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mana/cq.c</a></div><div class='hunk'>@@ -0,0 +1,342 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2022, Microsoft Corporation. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include "mana_ib.h"</div><div class='add'>+</div><div class='add'>+int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		      struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_udata *udata = &amp;attrs-&gt;driver_udata;</div><div class='add'>+	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);</div><div class='add'>+	struct mana_ib_create_cq_resp resp = {};</div><div class='add'>+	struct mana_ib_ucontext *mana_ucontext;</div><div class='add'>+	struct ib_device *ibdev = ibcq-&gt;device;</div><div class='add'>+	struct mana_ib_create_cq ucmd = {};</div><div class='add'>+	struct mana_ib_dev *mdev;</div><div class='add'>+	bool is_rnic_cq;</div><div class='add'>+	u32 doorbell;</div><div class='add'>+	u32 buf_size;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);</div><div class='add'>+</div><div class='add'>+	cq-&gt;comp_vector = attr-&gt;comp_vector % ibdev-&gt;num_comp_vectors;</div><div class='add'>+	cq-&gt;cq_handle = INVALID_MANA_HANDLE;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		if (udata-&gt;inlen &lt; offsetof(struct mana_ib_create_cq, flags))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		err = ib_copy_from_udata(&amp;ucmd, udata, min(sizeof(ucmd), udata-&gt;inlen));</div><div class='add'>+		if (err) {</div><div class='add'>+			ibdev_dbg(ibdev, "Failed to copy from udata for create cq, %d\n", err);</div><div class='add'>+			return err;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		is_rnic_cq = !!(ucmd.flags &amp; MANA_IB_CREATE_RNIC_CQ);</div><div class='add'>+</div><div class='add'>+		if ((!is_rnic_cq &amp;&amp; attr-&gt;cqe &gt; mdev-&gt;adapter_caps.max_qp_wr) ||</div><div class='add'>+		    attr-&gt;cqe &gt; U32_MAX / COMP_ENTRY_SIZE) {</div><div class='add'>+			ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr-&gt;cqe);</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		cq-&gt;cqe = attr-&gt;cqe;</div><div class='add'>+		err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq-&gt;cqe * COMP_ENTRY_SIZE,</div><div class='add'>+					   &amp;cq-&gt;queue);</div><div class='add'>+		if (err) {</div><div class='add'>+			ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err);</div><div class='add'>+			return err;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext,</div><div class='add'>+							  ibucontext);</div><div class='add'>+		doorbell = mana_ucontext-&gt;doorbell;</div><div class='add'>+	} else {</div><div class='add'>+		is_rnic_cq = true;</div><div class='add'>+		buf_size = MANA_PAGE_ALIGN(roundup_pow_of_two(attr-&gt;cqe * COMP_ENTRY_SIZE));</div><div class='add'>+		cq-&gt;cqe = buf_size / COMP_ENTRY_SIZE;</div><div class='add'>+		err = mana_ib_create_kernel_queue(mdev, buf_size, GDMA_CQ, &amp;cq-&gt;queue);</div><div class='add'>+		if (err) {</div><div class='add'>+			ibdev_dbg(ibdev, "Failed to create kernel queue for create cq, %d\n", err);</div><div class='add'>+			return err;</div><div class='add'>+		}</div><div class='add'>+		doorbell = mdev-&gt;gdma_dev-&gt;doorbell;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (is_rnic_cq) {</div><div class='add'>+		err = mana_ib_gd_create_cq(mdev, cq, doorbell);</div><div class='add'>+		if (err) {</div><div class='add'>+			ibdev_dbg(ibdev, "Failed to create RNIC cq, %d\n", err);</div><div class='add'>+			goto err_destroy_queue;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		err = mana_ib_install_cq_cb(mdev, cq);</div><div class='add'>+		if (err) {</div><div class='add'>+			ibdev_dbg(ibdev, "Failed to install cq callback, %d\n", err);</div><div class='add'>+			goto err_destroy_rnic_cq;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		resp.cqid = cq-&gt;queue.id;</div><div class='add'>+		err = ib_copy_to_udata(udata, &amp;resp, min(sizeof(resp), udata-&gt;outlen));</div><div class='add'>+		if (err) {</div><div class='add'>+			ibdev_dbg(&amp;mdev-&gt;ib_dev, "Failed to copy to udata, %d\n", err);</div><div class='add'>+			goto err_remove_cq_cb;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;cq-&gt;cq_lock);</div><div class='add'>+	INIT_LIST_HEAD(&amp;cq-&gt;list_send_qp);</div><div class='add'>+	INIT_LIST_HEAD(&amp;cq-&gt;list_recv_qp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_remove_cq_cb:</div><div class='add'>+	mana_ib_remove_cq_cb(mdev, cq);</div><div class='add'>+err_destroy_rnic_cq:</div><div class='add'>+	mana_ib_gd_destroy_cq(mdev, cq);</div><div class='add'>+err_destroy_queue:</div><div class='add'>+	mana_ib_destroy_queue(mdev, &amp;cq-&gt;queue);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);</div><div class='add'>+	struct ib_device *ibdev = ibcq-&gt;device;</div><div class='add'>+	struct mana_ib_dev *mdev;</div><div class='add'>+</div><div class='add'>+	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);</div><div class='add'>+</div><div class='add'>+	mana_ib_remove_cq_cb(mdev, cq);</div><div class='add'>+</div><div class='add'>+	/* Ignore return code as there is not much we can do about it.</div><div class='add'>+	 * The error message is printed inside.</div><div class='add'>+	 */</div><div class='add'>+	mana_ib_gd_destroy_cq(mdev, cq);</div><div class='add'>+</div><div class='add'>+	mana_ib_destroy_queue(mdev, &amp;cq-&gt;queue);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_cq *cq = ctx;</div><div class='add'>+</div><div class='add'>+	if (cq-&gt;ibcq.comp_handler)</div><div class='add'>+		cq-&gt;ibcq.comp_handler(&amp;cq-&gt;ibcq, cq-&gt;ibcq.cq_context);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)</div><div class='add'>+{</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(mdev);</div><div class='add'>+	struct gdma_queue *gdma_cq;</div><div class='add'>+</div><div class='add'>+	if (cq-&gt;queue.id &gt;= gc-&gt;max_num_cqs)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	/* Create CQ table entry */</div><div class='add'>+	WARN_ON(gc-&gt;cq_table[cq-&gt;queue.id]);</div><div class='add'>+	if (cq-&gt;queue.kmem)</div><div class='add'>+		gdma_cq = cq-&gt;queue.kmem;</div><div class='add'>+	else</div><div class='add'>+		gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);</div><div class='add'>+	if (!gdma_cq)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	gdma_cq-&gt;cq.context = cq;</div><div class='add'>+	gdma_cq-&gt;type = GDMA_CQ;</div><div class='add'>+	gdma_cq-&gt;cq.callback = mana_ib_cq_handler;</div><div class='add'>+	gdma_cq-&gt;id = cq-&gt;queue.id;</div><div class='add'>+	gc-&gt;cq_table[cq-&gt;queue.id] = gdma_cq;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)</div><div class='add'>+{</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(mdev);</div><div class='add'>+</div><div class='add'>+	if (cq-&gt;queue.id &gt;= gc-&gt;max_num_cqs || cq-&gt;queue.id == INVALID_QUEUE_ID)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	if (cq-&gt;queue.kmem)</div><div class='add'>+	/* Then it will be cleaned and removed by the mana */</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	kfree(gc-&gt;cq_table[cq-&gt;queue.id]);</div><div class='add'>+	gc-&gt;cq_table[cq-&gt;queue.id] = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);</div><div class='add'>+	struct gdma_queue *gdma_cq = cq-&gt;queue.kmem;</div><div class='add'>+</div><div class='add'>+	if (!gdma_cq)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	mana_gd_ring_cq(gdma_cq, SET_ARM_BIT);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe)</div><div class='add'>+{</div><div class='add'>+	struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe-&gt;cqe_data;</div><div class='add'>+	struct gdma_queue *wq = qp-&gt;ud_qp.queues[MANA_UD_SEND_QUEUE].kmem;</div><div class='add'>+	struct ud_sq_shadow_wqe *shadow_wqe;</div><div class='add'>+</div><div class='add'>+	shadow_wqe = shadow_queue_get_next_to_complete(&amp;qp-&gt;shadow_sq);</div><div class='add'>+	if (!shadow_wqe)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	shadow_wqe-&gt;header.error_code = rdma_cqe-&gt;ud_send.vendor_error;</div><div class='add'>+</div><div class='add'>+	wq-&gt;tail += shadow_wqe-&gt;header.posted_wqe_size;</div><div class='add'>+	shadow_queue_advance_next_to_complete(&amp;qp-&gt;shadow_sq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe)</div><div class='add'>+{</div><div class='add'>+	struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe-&gt;cqe_data;</div><div class='add'>+	struct gdma_queue *wq = qp-&gt;ud_qp.queues[MANA_UD_RECV_QUEUE].kmem;</div><div class='add'>+	struct ud_rq_shadow_wqe *shadow_wqe;</div><div class='add'>+</div><div class='add'>+	shadow_wqe = shadow_queue_get_next_to_complete(&amp;qp-&gt;shadow_rq);</div><div class='add'>+	if (!shadow_wqe)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	shadow_wqe-&gt;byte_len = rdma_cqe-&gt;ud_recv.msg_len;</div><div class='add'>+	shadow_wqe-&gt;src_qpn = rdma_cqe-&gt;ud_recv.src_qpn;</div><div class='add'>+	shadow_wqe-&gt;header.error_code = IB_WC_SUCCESS;</div><div class='add'>+</div><div class='add'>+	wq-&gt;tail += shadow_wqe-&gt;header.posted_wqe_size;</div><div class='add'>+	shadow_queue_advance_next_to_complete(&amp;qp-&gt;shadow_rq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe-&gt;wq_num, cqe-&gt;is_sq);</div><div class='add'>+</div><div class='add'>+	if (!qp)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;ibqp.qp_type == IB_QPT_GSI || qp-&gt;ibqp.qp_type == IB_QPT_UD) {</div><div class='add'>+		if (cqe-&gt;is_sq)</div><div class='add'>+			handle_ud_sq_cqe(qp, cqe);</div><div class='add'>+		else</div><div class='add'>+			handle_ud_rq_cqe(qp, cqe);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mana_put_qp_ref(qp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc,</div><div class='add'>+				       const struct shadow_wqe_header *shadow_wqe)</div><div class='add'>+{</div><div class='add'>+	const struct ud_rq_shadow_wqe *ud_wqe = (const struct ud_rq_shadow_wqe *)shadow_wqe;</div><div class='add'>+</div><div class='add'>+	wc-&gt;wr_id = shadow_wqe-&gt;wr_id;</div><div class='add'>+	wc-&gt;status = shadow_wqe-&gt;error_code;</div><div class='add'>+	wc-&gt;opcode = shadow_wqe-&gt;opcode;</div><div class='add'>+	wc-&gt;vendor_err = shadow_wqe-&gt;error_code;</div><div class='add'>+	wc-&gt;wc_flags = 0;</div><div class='add'>+	wc-&gt;qp = &amp;qp-&gt;ibqp;</div><div class='add'>+	wc-&gt;pkey_index = 0;</div><div class='add'>+</div><div class='add'>+	if (shadow_wqe-&gt;opcode == IB_WC_RECV) {</div><div class='add'>+		wc-&gt;byte_len = ud_wqe-&gt;byte_len;</div><div class='add'>+		wc-&gt;src_qp = ud_wqe-&gt;src_qpn;</div><div class='add'>+		wc-&gt;wc_flags |= IB_WC_GRH;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc *wc)</div><div class='add'>+{</div><div class='add'>+	struct shadow_wqe_header *shadow_wqe;</div><div class='add'>+	struct mana_ib_qp *qp;</div><div class='add'>+	int wc_index = 0;</div><div class='add'>+</div><div class='add'>+	/* process send shadow queue completions  */</div><div class='add'>+	list_for_each_entry(qp, &amp;cq-&gt;list_send_qp, cq_send_list) {</div><div class='add'>+		while ((shadow_wqe = shadow_queue_get_next_to_consume(&amp;qp-&gt;shadow_sq))</div><div class='add'>+				!= NULL) {</div><div class='add'>+			if (wc_index &gt;= nwc)</div><div class='add'>+				goto out;</div><div class='add'>+</div><div class='add'>+			fill_verbs_from_shadow_wqe(qp, &amp;wc[wc_index], shadow_wqe);</div><div class='add'>+			shadow_queue_advance_consumer(&amp;qp-&gt;shadow_sq);</div><div class='add'>+			wc_index++;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* process recv shadow queue completions */</div><div class='add'>+	list_for_each_entry(qp, &amp;cq-&gt;list_recv_qp, cq_recv_list) {</div><div class='add'>+		while ((shadow_wqe = shadow_queue_get_next_to_consume(&amp;qp-&gt;shadow_rq))</div><div class='add'>+				!= NULL) {</div><div class='add'>+			if (wc_index &gt;= nwc)</div><div class='add'>+				goto out;</div><div class='add'>+</div><div class='add'>+			fill_verbs_from_shadow_wqe(qp, &amp;wc[wc_index], shadow_wqe);</div><div class='add'>+			shadow_queue_advance_consumer(&amp;qp-&gt;shadow_rq);</div><div class='add'>+			wc_index++;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	return wc_index;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mana_drain_gsi_sqs(struct mana_ib_dev *mdev)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_qp *qp = mana_get_qp_ref(mdev, MANA_GSI_QPN, false);</div><div class='add'>+	struct ud_sq_shadow_wqe *shadow_wqe;</div><div class='add'>+	struct mana_ib_cq *cq;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	if (!qp)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	cq = container_of(qp-&gt;ibqp.send_cq, struct mana_ib_cq, ibcq);</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cq-&gt;cq_lock, flags);</div><div class='add'>+	while ((shadow_wqe = shadow_queue_get_next_to_complete(&amp;qp-&gt;shadow_sq))</div><div class='add'>+			!= NULL) {</div><div class='add'>+		shadow_wqe-&gt;header.error_code = IB_WC_GENERAL_ERR;</div><div class='add'>+		shadow_queue_advance_next_to_complete(&amp;qp-&gt;shadow_sq);</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irqrestore(&amp;cq-&gt;cq_lock, flags);</div><div class='add'>+</div><div class='add'>+	if (cq-&gt;ibcq.comp_handler)</div><div class='add'>+		cq-&gt;ibcq.comp_handler(&amp;cq-&gt;ibcq, cq-&gt;ibcq.cq_context);</div><div class='add'>+</div><div class='add'>+	mana_put_qp_ref(qp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);</div><div class='add'>+	struct mana_ib_dev *mdev = container_of(ibcq-&gt;device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct gdma_queue *queue = cq-&gt;queue.kmem;</div><div class='add'>+	struct gdma_comp gdma_cqe;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int num_polled = 0;</div><div class='add'>+	int comp_read, i;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;cq-&gt;cq_lock, flags);</div><div class='add'>+	for (i = 0; i &lt; num_entries; i++) {</div><div class='add'>+		comp_read = mana_gd_poll_cq(queue, &amp;gdma_cqe, 1);</div><div class='add'>+		if (comp_read &lt; 1)</div><div class='add'>+			break;</div><div class='add'>+		mana_handle_cqe(mdev, &amp;gdma_cqe);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	num_polled = mana_process_completions(cq, num_entries, wc);</div><div class='add'>+	spin_unlock_irqrestore(&amp;cq-&gt;cq_lock, flags);</div><div class='add'>+</div><div class='add'>+	return num_polled;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c<br/>new file mode 100644<br/>index 000000000000..bdeddb642b87<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mana/device.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mana/device.c</a></div><div class='hunk'>@@ -0,0 +1,261 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2022, Microsoft Corporation. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include "mana_ib.h"</div><div class='add'>+#include &lt;net/mana/mana_auxiliary.h&gt;</div><div class='add'>+#include &lt;net/addrconf.h&gt;</div><div class='add'>+</div><div class='add'>+MODULE_DESCRIPTION("Microsoft Azure Network Adapter IB driver");</div><div class='add'>+MODULE_LICENSE("GPL");</div><div class='add'>+MODULE_IMPORT_NS("NET_MANA");</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops mana_ib_dev_ops = {</div><div class='add'>+	.owner = THIS_MODULE,</div><div class='add'>+	.driver_id = RDMA_DRIVER_MANA,</div><div class='add'>+	.uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION,</div><div class='add'>+</div><div class='add'>+	.add_gid = mana_ib_gd_add_gid,</div><div class='add'>+	.alloc_pd = mana_ib_alloc_pd,</div><div class='add'>+	.alloc_ucontext = mana_ib_alloc_ucontext,</div><div class='add'>+	.create_ah = mana_ib_create_ah,</div><div class='add'>+	.create_cq = mana_ib_create_cq,</div><div class='add'>+	.create_qp = mana_ib_create_qp,</div><div class='add'>+	.create_rwq_ind_table = mana_ib_create_rwq_ind_table,</div><div class='add'>+	.create_wq = mana_ib_create_wq,</div><div class='add'>+	.dealloc_pd = mana_ib_dealloc_pd,</div><div class='add'>+	.dealloc_ucontext = mana_ib_dealloc_ucontext,</div><div class='add'>+	.del_gid = mana_ib_gd_del_gid,</div><div class='add'>+	.dereg_mr = mana_ib_dereg_mr,</div><div class='add'>+	.destroy_ah = mana_ib_destroy_ah,</div><div class='add'>+	.destroy_cq = mana_ib_destroy_cq,</div><div class='add'>+	.destroy_qp = mana_ib_destroy_qp,</div><div class='add'>+	.destroy_rwq_ind_table = mana_ib_destroy_rwq_ind_table,</div><div class='add'>+	.destroy_wq = mana_ib_destroy_wq,</div><div class='add'>+	.disassociate_ucontext = mana_ib_disassociate_ucontext,</div><div class='add'>+	.get_dma_mr = mana_ib_get_dma_mr,</div><div class='add'>+	.get_link_layer = mana_ib_get_link_layer,</div><div class='add'>+	.get_port_immutable = mana_ib_get_port_immutable,</div><div class='add'>+	.mmap = mana_ib_mmap,</div><div class='add'>+	.modify_qp = mana_ib_modify_qp,</div><div class='add'>+	.modify_wq = mana_ib_modify_wq,</div><div class='add'>+	.poll_cq = mana_ib_poll_cq,</div><div class='add'>+	.post_recv = mana_ib_post_recv,</div><div class='add'>+	.post_send = mana_ib_post_send,</div><div class='add'>+	.query_device = mana_ib_query_device,</div><div class='add'>+	.query_gid = mana_ib_query_gid,</div><div class='add'>+	.query_pkey = mana_ib_query_pkey,</div><div class='add'>+	.query_port = mana_ib_query_port,</div><div class='add'>+	.reg_user_mr = mana_ib_reg_user_mr,</div><div class='add'>+	.reg_user_mr_dmabuf = mana_ib_reg_user_mr_dmabuf,</div><div class='add'>+	.req_notify_cq = mana_ib_arm_cq,</div><div class='add'>+</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_ah, mana_ib_ah, ibah),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_cq, mana_ib_cq, ibcq),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_pd, mana_ib_pd, ibpd),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_qp, mana_ib_qp, ibqp),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_ucontext, mana_ib_ucontext, ibucontext),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mana_ib_rwq_ind_table,</div><div class='add'>+			   ib_ind_table),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops mana_ib_stats_ops = {</div><div class='add'>+	.alloc_hw_port_stats = mana_ib_alloc_hw_port_stats,</div><div class='add'>+	.get_hw_stats = mana_ib_get_hw_stats,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops mana_ib_device_stats_ops = {</div><div class='add'>+	.alloc_hw_device_stats = mana_ib_alloc_hw_device_stats,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int mana_ib_netdev_event(struct notifier_block *this,</div><div class='add'>+				unsigned long event, void *ptr)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *dev = container_of(this, struct mana_ib_dev, nb);</div><div class='add'>+	struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);</div><div class='add'>+	struct gdma_context *gc = dev-&gt;gdma_dev-&gt;gdma_context;</div><div class='add'>+	struct mana_context *mc = gc-&gt;mana.driver_data;</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	/* Only process events from our parent device */</div><div class='add'>+	for (i = 0; i &lt; dev-&gt;ib_dev.phys_port_cnt; i++)</div><div class='add'>+		if (event_dev == mc-&gt;ports[i]) {</div><div class='add'>+			switch (event) {</div><div class='add'>+			case NETDEV_CHANGEUPPER:</div><div class='add'>+				ndev = mana_get_primary_netdev(mc, i, &amp;dev-&gt;dev_tracker);</div><div class='add'>+				/*</div><div class='add'>+				 * RDMA core will setup GID based on updated netdev.</div><div class='add'>+				 * It's not possible to race with the core as rtnl lock is being</div><div class='add'>+				 * held.</div><div class='add'>+				 */</div><div class='add'>+				ib_device_set_netdev(&amp;dev-&gt;ib_dev, ndev, i + 1);</div><div class='add'>+</div><div class='add'>+				/* mana_get_primary_netdev() returns ndev with refcount held */</div><div class='add'>+				if (ndev)</div><div class='add'>+					netdev_put(ndev, &amp;dev-&gt;dev_tracker);</div><div class='add'>+</div><div class='add'>+				return NOTIFY_OK;</div><div class='add'>+			default:</div><div class='add'>+				return NOTIFY_DONE;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	return NOTIFY_DONE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_ib_probe(struct auxiliary_device *adev,</div><div class='add'>+			 const struct auxiliary_device_id *id)</div><div class='add'>+{</div><div class='add'>+	struct mana_adev *madev = container_of(adev, struct mana_adev, adev);</div><div class='add'>+	struct gdma_context *gc = madev-&gt;mdev-&gt;gdma_context;</div><div class='add'>+	struct mana_context *mc = gc-&gt;mana.driver_data;</div><div class='add'>+	struct gdma_dev *mdev = madev-&gt;mdev;</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+	struct mana_ib_dev *dev;</div><div class='add'>+	u8 mac_addr[ETH_ALEN];</div><div class='add'>+	int ret, i;</div><div class='add'>+</div><div class='add'>+	dev = ib_alloc_device(mana_ib_dev, ib_dev);</div><div class='add'>+	if (!dev)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;mana_ib_dev_ops);</div><div class='add'>+	dev-&gt;ib_dev.node_type = RDMA_NODE_IB_CA;</div><div class='add'>+	dev-&gt;ib_dev.num_comp_vectors = gc-&gt;max_num_queues;</div><div class='add'>+	dev-&gt;ib_dev.dev.parent = gc-&gt;dev;</div><div class='add'>+	dev-&gt;gdma_dev = mdev;</div><div class='add'>+	xa_init_flags(&amp;dev-&gt;qp_table_wq, XA_FLAGS_LOCK_IRQ);</div><div class='add'>+</div><div class='add'>+	if (mana_ib_is_rnic(dev)) {</div><div class='add'>+		dev-&gt;ib_dev.phys_port_cnt = 1;</div><div class='add'>+		addrconf_addr_eui48((u8 *)&amp;dev-&gt;ib_dev.node_guid, mc-&gt;ports[0]-&gt;dev_addr);</div><div class='add'>+		ret = mana_ib_gd_query_adapter_caps(dev);</div><div class='add'>+		if (ret) {</div><div class='add'>+			ibdev_err(&amp;dev-&gt;ib_dev, "Failed to query device caps, ret %d", ret);</div><div class='add'>+			goto free_ib_device;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;mana_ib_stats_ops);</div><div class='add'>+		if (dev-&gt;adapter_caps.feature_flags &amp; MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT)</div><div class='add'>+			ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;mana_ib_device_stats_ops);</div><div class='add'>+</div><div class='add'>+		ret = mana_ib_create_eqs(dev);</div><div class='add'>+		if (ret) {</div><div class='add'>+			ibdev_err(&amp;dev-&gt;ib_dev, "Failed to create EQs, ret %d", ret);</div><div class='add'>+			goto free_ib_device;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ret = mana_ib_gd_create_rnic_adapter(dev);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto destroy_eqs;</div><div class='add'>+</div><div class='add'>+		if (dev-&gt;adapter_caps.feature_flags &amp; MANA_IB_FEATURE_MULTI_PORTS_SUPPORT)</div><div class='add'>+			dev-&gt;ib_dev.phys_port_cnt = mc-&gt;num_ports;</div><div class='add'>+</div><div class='add'>+		for (i = 0; i &lt; dev-&gt;ib_dev.phys_port_cnt; i++) {</div><div class='add'>+			ndev = mana_get_primary_netdev(mc, i, &amp;dev-&gt;dev_tracker);</div><div class='add'>+			if (!ndev) {</div><div class='add'>+				ret = -ENODEV;</div><div class='add'>+				ibdev_err(&amp;dev-&gt;ib_dev,</div><div class='add'>+					  "Failed to get netdev for IB port %d", i + 1);</div><div class='add'>+				goto destroy_rnic;</div><div class='add'>+			}</div><div class='add'>+			ether_addr_copy(mac_addr, ndev-&gt;dev_addr);</div><div class='add'>+			ret = ib_device_set_netdev(&amp;dev-&gt;ib_dev, ndev, i + 1);</div><div class='add'>+			/* mana_get_primary_netdev() returns ndev with refcount held */</div><div class='add'>+			netdev_put(ndev, &amp;dev-&gt;dev_tracker);</div><div class='add'>+			if (ret) {</div><div class='add'>+				ibdev_err(&amp;dev-&gt;ib_dev, "Failed to set ib netdev, ret %d", ret);</div><div class='add'>+				goto destroy_rnic;</div><div class='add'>+			}</div><div class='add'>+			ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);</div><div class='add'>+			if (ret) {</div><div class='add'>+				ibdev_err(&amp;dev-&gt;ib_dev, "Failed to add Mac address, ret %d", ret);</div><div class='add'>+				goto destroy_rnic;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+		dev-&gt;nb.notifier_call = mana_ib_netdev_event;</div><div class='add'>+		ret = register_netdevice_notifier(&amp;dev-&gt;nb);</div><div class='add'>+		if (ret) {</div><div class='add'>+			ibdev_err(&amp;dev-&gt;ib_dev, "Failed to register net notifier, %d", ret);</div><div class='add'>+			goto destroy_rnic;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		dev-&gt;ib_dev.phys_port_cnt = mc-&gt;num_ports;</div><div class='add'>+		ret = mana_eth_query_adapter_caps(dev);</div><div class='add'>+		if (ret) {</div><div class='add'>+			ibdev_err(&amp;dev-&gt;ib_dev, "Failed to query ETH device caps, ret %d", ret);</div><div class='add'>+			goto free_ib_device;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dev-&gt;av_pool = dma_pool_create("mana_ib_av", gc-&gt;dev, MANA_AV_BUFFER_SIZE,</div><div class='add'>+				       MANA_AV_BUFFER_SIZE, 0);</div><div class='add'>+	if (!dev-&gt;av_pool) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto deregister_net_notifier;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev,</div><div class='add'>+		  mdev-&gt;dev_id.as_uint32, dev-&gt;ib_dev.phys_port_cnt);</div><div class='add'>+</div><div class='add'>+	ret = ib_register_device(&amp;dev-&gt;ib_dev, mana_ib_is_rnic(dev) ? "mana_%d" : "manae_%d",</div><div class='add'>+				 gc-&gt;dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto deallocate_pool;</div><div class='add'>+</div><div class='add'>+	dev_set_drvdata(&amp;adev-&gt;dev, dev);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+deallocate_pool:</div><div class='add'>+	dma_pool_destroy(dev-&gt;av_pool);</div><div class='add'>+deregister_net_notifier:</div><div class='add'>+	if (mana_ib_is_rnic(dev))</div><div class='add'>+		unregister_netdevice_notifier(&amp;dev-&gt;nb);</div><div class='add'>+destroy_rnic:</div><div class='add'>+	if (mana_ib_is_rnic(dev))</div><div class='add'>+		mana_ib_gd_destroy_rnic_adapter(dev);</div><div class='add'>+destroy_eqs:</div><div class='add'>+	if (mana_ib_is_rnic(dev))</div><div class='add'>+		mana_ib_destroy_eqs(dev);</div><div class='add'>+free_ib_device:</div><div class='add'>+	xa_destroy(&amp;dev-&gt;qp_table_wq);</div><div class='add'>+	ib_dealloc_device(&amp;dev-&gt;ib_dev);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mana_ib_remove(struct auxiliary_device *adev)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *dev = dev_get_drvdata(&amp;adev-&gt;dev);</div><div class='add'>+</div><div class='add'>+	if (mana_ib_is_rnic(dev))</div><div class='add'>+		mana_drain_gsi_sqs(dev);</div><div class='add'>+</div><div class='add'>+	ib_unregister_device(&amp;dev-&gt;ib_dev);</div><div class='add'>+	dma_pool_destroy(dev-&gt;av_pool);</div><div class='add'>+	if (mana_ib_is_rnic(dev)) {</div><div class='add'>+		unregister_netdevice_notifier(&amp;dev-&gt;nb);</div><div class='add'>+		mana_ib_gd_destroy_rnic_adapter(dev);</div><div class='add'>+		mana_ib_destroy_eqs(dev);</div><div class='add'>+	}</div><div class='add'>+	xa_destroy(&amp;dev-&gt;qp_table_wq);</div><div class='add'>+	ib_dealloc_device(&amp;dev-&gt;ib_dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct auxiliary_device_id mana_id_table[] = {</div><div class='add'>+	{ .name = "mana.rdma", },</div><div class='add'>+	{ .name = "mana.eth", },</div><div class='add'>+	{},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+MODULE_DEVICE_TABLE(auxiliary, mana_id_table);</div><div class='add'>+</div><div class='add'>+static struct auxiliary_driver mana_driver = {</div><div class='add'>+	.probe = mana_ib_probe,</div><div class='add'>+	.remove = mana_ib_remove,</div><div class='add'>+	.id_table = mana_id_table,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+module_auxiliary_driver(mana_driver);</div><div class='head'>diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c<br/>new file mode 100644<br/>index 000000000000..fac159f7128d<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mana/main.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mana/main.c</a></div><div class='hunk'>@@ -0,0 +1,1134 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2022, Microsoft Corporation. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include "mana_ib.h"</div><div class='add'>+#include "linux/pci.h"</div><div class='add'>+</div><div class='add'>+void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,</div><div class='add'>+			 u32 port)</div><div class='add'>+{</div><div class='add'>+	struct mana_port_context *mpc;</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+</div><div class='add'>+	ndev = mana_ib_get_netdev(&amp;dev-&gt;ib_dev, port);</div><div class='add'>+	mpc = netdev_priv(ndev);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;pd-&gt;vport_mutex);</div><div class='add'>+</div><div class='add'>+	pd-&gt;vport_use_count--;</div><div class='add'>+	WARN_ON(pd-&gt;vport_use_count &lt; 0);</div><div class='add'>+</div><div class='add'>+	if (!pd-&gt;vport_use_count)</div><div class='add'>+		mana_uncfg_vport(mpc);</div><div class='add'>+</div><div class='add'>+	mutex_unlock(&amp;pd-&gt;vport_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port, struct mana_ib_pd *pd,</div><div class='add'>+		      u32 doorbell_id)</div><div class='add'>+{</div><div class='add'>+	struct mana_port_context *mpc;</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	ndev = mana_ib_get_netdev(&amp;dev-&gt;ib_dev, port);</div><div class='add'>+	mpc = netdev_priv(ndev);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;pd-&gt;vport_mutex);</div><div class='add'>+</div><div class='add'>+	pd-&gt;vport_use_count++;</div><div class='add'>+	if (pd-&gt;vport_use_count &gt; 1) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ib_dev,</div><div class='add'>+			  "Skip as this PD is already configured vport\n");</div><div class='add'>+		mutex_unlock(&amp;pd-&gt;vport_mutex);</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = mana_cfg_vport(mpc, pd-&gt;pdn, doorbell_id);</div><div class='add'>+	if (err) {</div><div class='add'>+		pd-&gt;vport_use_count--;</div><div class='add'>+		mutex_unlock(&amp;pd-&gt;vport_mutex);</div><div class='add'>+</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ib_dev, "Failed to configure vPort %d\n", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_unlock(&amp;pd-&gt;vport_mutex);</div><div class='add'>+</div><div class='add'>+	pd-&gt;tx_shortform_allowed = mpc-&gt;tx_shortform_allowed;</div><div class='add'>+	pd-&gt;tx_vp_offset = mpc-&gt;tx_vp_offset;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ib_dev, "vport handle %llx pdid %x doorbell_id %x\n",</div><div class='add'>+		  mpc-&gt;port_handle, pd-&gt;pdn, doorbell_id);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);</div><div class='add'>+	struct ib_device *ibdev = ibpd-&gt;device;</div><div class='add'>+	struct gdma_create_pd_resp resp = {};</div><div class='add'>+	struct gdma_create_pd_req req = {};</div><div class='add'>+	enum gdma_pd_flags flags = 0;</div><div class='add'>+	struct mana_ib_dev *dev;</div><div class='add'>+	struct gdma_context *gc;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);</div><div class='add'>+	gc = mdev_to_gc(dev);</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, GDMA_CREATE_PD, sizeof(req),</div><div class='add'>+			     sizeof(resp));</div><div class='add'>+</div><div class='add'>+	if (!udata)</div><div class='add'>+		flags |= GDMA_PD_FLAG_ALLOW_GPA_MR;</div><div class='add'>+</div><div class='add'>+	req.flags = flags;</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req,</div><div class='add'>+				   sizeof(resp), &amp;resp);</div><div class='add'>+</div><div class='add'>+	if (err || resp.hdr.status) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ib_dev,</div><div class='add'>+			  "Failed to get pd_id err %d status %u\n", err,</div><div class='add'>+			  resp.hdr.status);</div><div class='add'>+		if (!err)</div><div class='add'>+			err = -EPROTO;</div><div class='add'>+</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	pd-&gt;pd_handle = resp.pd_handle;</div><div class='add'>+	pd-&gt;pdn = resp.pd_id;</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ib_dev, "pd_handle 0x%llx pd_id %d\n",</div><div class='add'>+		  pd-&gt;pd_handle, pd-&gt;pdn);</div><div class='add'>+</div><div class='add'>+	mutex_init(&amp;pd-&gt;vport_mutex);</div><div class='add'>+	pd-&gt;vport_use_count = 0;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);</div><div class='add'>+	struct ib_device *ibdev = ibpd-&gt;device;</div><div class='add'>+	struct gdma_destory_pd_resp resp = {};</div><div class='add'>+	struct gdma_destroy_pd_req req = {};</div><div class='add'>+	struct mana_ib_dev *dev;</div><div class='add'>+	struct gdma_context *gc;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);</div><div class='add'>+	gc = mdev_to_gc(dev);</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, GDMA_DESTROY_PD, sizeof(req),</div><div class='add'>+			     sizeof(resp));</div><div class='add'>+</div><div class='add'>+	req.pd_handle = pd-&gt;pd_handle;</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req,</div><div class='add'>+				   sizeof(resp), &amp;resp);</div><div class='add'>+</div><div class='add'>+	if (err || resp.hdr.status) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ib_dev,</div><div class='add'>+			  "Failed to destroy pd_handle 0x%llx err %d status %u",</div><div class='add'>+			  pd-&gt;pd_handle, err, resp.hdr.status);</div><div class='add'>+		if (!err)</div><div class='add'>+			err = -EPROTO;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_gd_destroy_doorbell_page(struct gdma_context *gc,</div><div class='add'>+					 int doorbell_page)</div><div class='add'>+{</div><div class='add'>+	struct gdma_destroy_resource_range_req req = {};</div><div class='add'>+	struct gdma_resp_hdr resp = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, GDMA_DESTROY_RESOURCE_RANGE,</div><div class='add'>+			     sizeof(req), sizeof(resp));</div><div class='add'>+</div><div class='add'>+	req.resource_type = GDMA_RESOURCE_DOORBELL_PAGE;</div><div class='add'>+	req.num_resources = 1;</div><div class='add'>+	req.allocated_resources = doorbell_page;</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+	if (err || resp.status) {</div><div class='add'>+		dev_err(gc-&gt;dev,</div><div class='add'>+			"Failed to destroy doorbell page: ret %d, 0x%x\n",</div><div class='add'>+			err, resp.status);</div><div class='add'>+		return err ?: -EPROTO;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_gd_allocate_doorbell_page(struct gdma_context *gc,</div><div class='add'>+					  int *doorbell_page)</div><div class='add'>+{</div><div class='add'>+	struct gdma_allocate_resource_range_req req = {};</div><div class='add'>+	struct gdma_allocate_resource_range_resp resp = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, GDMA_ALLOCATE_RESOURCE_RANGE,</div><div class='add'>+			     sizeof(req), sizeof(resp));</div><div class='add'>+</div><div class='add'>+	req.resource_type = GDMA_RESOURCE_DOORBELL_PAGE;</div><div class='add'>+	req.num_resources = 1;</div><div class='add'>+	req.alignment = PAGE_SIZE / MANA_PAGE_SIZE;</div><div class='add'>+</div><div class='add'>+	/* Have GDMA start searching from 0 */</div><div class='add'>+	req.allocated_resources = 0;</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+	if (err || resp.hdr.status) {</div><div class='add'>+		dev_err(gc-&gt;dev,</div><div class='add'>+			"Failed to allocate doorbell page: ret %d, 0x%x\n",</div><div class='add'>+			err, resp.hdr.status);</div><div class='add'>+		return err ?: -EPROTO;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*doorbell_page = resp.allocated_resources;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,</div><div class='add'>+			   struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_ucontext *ucontext =</div><div class='add'>+		container_of(ibcontext, struct mana_ib_ucontext, ibucontext);</div><div class='add'>+	struct ib_device *ibdev = ibcontext-&gt;device;</div><div class='add'>+	struct mana_ib_dev *mdev;</div><div class='add'>+	struct gdma_context *gc;</div><div class='add'>+	int doorbell_page;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);</div><div class='add'>+	gc = mdev_to_gc(mdev);</div><div class='add'>+</div><div class='add'>+	/* Allocate a doorbell page index */</div><div class='add'>+	ret = mana_gd_allocate_doorbell_page(gc, &amp;doorbell_page);</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_dbg(ibdev, "Failed to allocate doorbell page %d\n", ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(ibdev, "Doorbell page allocated %d\n", doorbell_page);</div><div class='add'>+</div><div class='add'>+	ucontext-&gt;doorbell = doorbell_page;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_ucontext *mana_ucontext =</div><div class='add'>+		container_of(ibcontext, struct mana_ib_ucontext, ibucontext);</div><div class='add'>+	struct ib_device *ibdev = ibcontext-&gt;device;</div><div class='add'>+	struct mana_ib_dev *mdev;</div><div class='add'>+	struct gdma_context *gc;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);</div><div class='add'>+	gc = mdev_to_gc(mdev);</div><div class='add'>+</div><div class='add'>+	ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext-&gt;doorbell);</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_dbg(ibdev, "Failed to destroy doorbell page %d\n", ret);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_queue_type type,</div><div class='add'>+				struct mana_ib_queue *queue)</div><div class='add'>+{</div><div class='add'>+	struct gdma_queue_spec spec = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	queue-&gt;id = INVALID_QUEUE_ID;</div><div class='add'>+	queue-&gt;gdma_region = GDMA_INVALID_DMA_REGION;</div><div class='add'>+	spec.type = type;</div><div class='add'>+	spec.monitor_avl_buf = false;</div><div class='add'>+	spec.queue_size = size;</div><div class='add'>+	err = mana_gd_create_mana_wq_cq(mdev-&gt;gdma_dev, &amp;spec, &amp;queue-&gt;kmem);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+	/* take ownership into mana_ib from mana */</div><div class='add'>+	queue-&gt;gdma_region = queue-&gt;kmem-&gt;mem_info.dma_region_handle;</div><div class='add'>+	queue-&gt;kmem-&gt;mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,</div><div class='add'>+			 struct mana_ib_queue *queue)</div><div class='add'>+{</div><div class='add'>+	struct ib_umem *umem;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	queue-&gt;umem = NULL;</div><div class='add'>+	queue-&gt;id = INVALID_QUEUE_ID;</div><div class='add'>+	queue-&gt;gdma_region = GDMA_INVALID_DMA_REGION;</div><div class='add'>+</div><div class='add'>+	umem = ib_umem_get(&amp;mdev-&gt;ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE);</div><div class='add'>+	if (IS_ERR(umem)) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev, "Failed to get umem, %pe\n", umem);</div><div class='add'>+		return PTR_ERR(umem);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = mana_ib_create_zero_offset_dma_region(mdev, umem, &amp;queue-&gt;gdma_region);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev, "Failed to create dma region, %d\n", err);</div><div class='add'>+		goto free_umem;</div><div class='add'>+	}</div><div class='add'>+	queue-&gt;umem = umem;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;mdev-&gt;ib_dev, "created dma region 0x%llx\n", queue-&gt;gdma_region);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+free_umem:</div><div class='add'>+	ib_umem_release(umem);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue)</div><div class='add'>+{</div><div class='add'>+	/* Ignore return code as there is not much we can do about it.</div><div class='add'>+	 * The error message is printed inside.</div><div class='add'>+	 */</div><div class='add'>+	mana_ib_gd_destroy_dma_region(mdev, queue-&gt;gdma_region);</div><div class='add'>+	ib_umem_release(queue-&gt;umem);</div><div class='add'>+	if (queue-&gt;kmem)</div><div class='add'>+		mana_gd_destroy_queue(mdev_to_gc(mdev), queue-&gt;kmem);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int</div><div class='add'>+mana_ib_gd_first_dma_region(struct mana_ib_dev *dev,</div><div class='add'>+			    struct gdma_context *gc,</div><div class='add'>+			    struct gdma_create_dma_region_req *create_req,</div><div class='add'>+			    size_t num_pages, mana_handle_t *gdma_region,</div><div class='add'>+			    u32 expected_status)</div><div class='add'>+{</div><div class='add'>+	struct gdma_create_dma_region_resp create_resp = {};</div><div class='add'>+	unsigned int create_req_msg_size;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	create_req_msg_size =</div><div class='add'>+		struct_size(create_req, page_addr_list, num_pages);</div><div class='add'>+	create_req-&gt;page_addr_list_len = num_pages;</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(gc, create_req_msg_size, create_req,</div><div class='add'>+				   sizeof(create_resp), &amp;create_resp);</div><div class='add'>+	if (err || create_resp.hdr.status != expected_status) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ib_dev,</div><div class='add'>+			  "Failed to create DMA region: %d, 0x%x\n",</div><div class='add'>+			  err, create_resp.hdr.status);</div><div class='add'>+		if (!err)</div><div class='add'>+			err = -EPROTO;</div><div class='add'>+</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*gdma_region = create_resp.dma_region_handle;</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ib_dev, "Created DMA region handle 0x%llx\n",</div><div class='add'>+		  *gdma_region);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int</div><div class='add'>+mana_ib_gd_add_dma_region(struct mana_ib_dev *dev, struct gdma_context *gc,</div><div class='add'>+			  struct gdma_dma_region_add_pages_req *add_req,</div><div class='add'>+			  unsigned int num_pages, u32 expected_status)</div><div class='add'>+{</div><div class='add'>+	unsigned int add_req_msg_size =</div><div class='add'>+		struct_size(add_req, page_addr_list, num_pages);</div><div class='add'>+	struct gdma_general_resp add_resp = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;add_req-&gt;hdr, GDMA_DMA_REGION_ADD_PAGES,</div><div class='add'>+			     add_req_msg_size, sizeof(add_resp));</div><div class='add'>+	add_req-&gt;page_addr_list_len = num_pages;</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(gc, add_req_msg_size, add_req,</div><div class='add'>+				   sizeof(add_resp), &amp;add_resp);</div><div class='add'>+	if (err || add_resp.hdr.status != expected_status) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ib_dev,</div><div class='add'>+			  "Failed to create DMA region: %d, 0x%x\n",</div><div class='add'>+			  err, add_resp.hdr.status);</div><div class='add'>+</div><div class='add'>+		if (!err)</div><div class='add'>+			err = -EPROTO;</div><div class='add'>+</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,</div><div class='add'>+					mana_handle_t *gdma_region, unsigned long page_sz)</div><div class='add'>+{</div><div class='add'>+	struct gdma_dma_region_add_pages_req *add_req = NULL;</div><div class='add'>+	size_t num_pages_processed = 0, num_pages_to_handle;</div><div class='add'>+	struct gdma_create_dma_region_req *create_req;</div><div class='add'>+	unsigned int create_req_msg_size;</div><div class='add'>+	struct hw_channel_context *hwc;</div><div class='add'>+	struct ib_block_iter biter;</div><div class='add'>+	size_t max_pgs_add_cmd = 0;</div><div class='add'>+	size_t max_pgs_create_cmd;</div><div class='add'>+	struct gdma_context *gc;</div><div class='add'>+	size_t num_pages_total;</div><div class='add'>+	unsigned int tail = 0;</div><div class='add'>+	u64 *page_addr_list;</div><div class='add'>+	void *request_buf;</div><div class='add'>+	int err = 0;</div><div class='add'>+</div><div class='add'>+	gc = mdev_to_gc(dev);</div><div class='add'>+	hwc = gc-&gt;hwc.driver_data;</div><div class='add'>+</div><div class='add'>+	num_pages_total = ib_umem_num_dma_blocks(umem, page_sz);</div><div class='add'>+</div><div class='add'>+	max_pgs_create_cmd =</div><div class='add'>+		(hwc-&gt;max_req_msg_size - sizeof(*create_req)) / sizeof(u64);</div><div class='add'>+	num_pages_to_handle =</div><div class='add'>+		min_t(size_t, num_pages_total, max_pgs_create_cmd);</div><div class='add'>+	create_req_msg_size =</div><div class='add'>+		struct_size(create_req, page_addr_list, num_pages_to_handle);</div><div class='add'>+</div><div class='add'>+	request_buf = kzalloc(hwc-&gt;max_req_msg_size, GFP_KERNEL);</div><div class='add'>+	if (!request_buf)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	create_req = request_buf;</div><div class='add'>+	mana_gd_init_req_hdr(&amp;create_req-&gt;hdr, GDMA_CREATE_DMA_REGION,</div><div class='add'>+			     create_req_msg_size,</div><div class='add'>+			     sizeof(struct gdma_create_dma_region_resp));</div><div class='add'>+</div><div class='add'>+	create_req-&gt;length = umem-&gt;length;</div><div class='add'>+	create_req-&gt;offset_in_page = ib_umem_dma_offset(umem, page_sz);</div><div class='add'>+	create_req-&gt;gdma_page_type = order_base_2(page_sz) - MANA_PAGE_SHIFT;</div><div class='add'>+	create_req-&gt;page_count = num_pages_total;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ib_dev, "size_dma_region %lu num_pages_total %lu\n",</div><div class='add'>+		  umem-&gt;length, num_pages_total);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ib_dev, "page_sz %lu offset_in_page %u\n",</div><div class='add'>+		  page_sz, create_req-&gt;offset_in_page);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ib_dev, "num_pages_to_handle %lu, gdma_page_type %u",</div><div class='add'>+		  num_pages_to_handle, create_req-&gt;gdma_page_type);</div><div class='add'>+</div><div class='add'>+	page_addr_list = create_req-&gt;page_addr_list;</div><div class='add'>+	rdma_umem_for_each_dma_block(umem, &amp;biter, page_sz) {</div><div class='add'>+		u32 expected_status = 0;</div><div class='add'>+</div><div class='add'>+		page_addr_list[tail++] = rdma_block_iter_dma_address(&amp;biter);</div><div class='add'>+		if (tail &lt; num_pages_to_handle)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		if (num_pages_processed + num_pages_to_handle &lt;</div><div class='add'>+		    num_pages_total)</div><div class='add'>+			expected_status = GDMA_STATUS_MORE_ENTRIES;</div><div class='add'>+</div><div class='add'>+		if (!num_pages_processed) {</div><div class='add'>+			/* First create message */</div><div class='add'>+			err = mana_ib_gd_first_dma_region(dev, gc, create_req,</div><div class='add'>+							  tail, gdma_region,</div><div class='add'>+							  expected_status);</div><div class='add'>+			if (err)</div><div class='add'>+				goto out;</div><div class='add'>+</div><div class='add'>+			max_pgs_add_cmd = (hwc-&gt;max_req_msg_size -</div><div class='add'>+				sizeof(*add_req)) / sizeof(u64);</div><div class='add'>+</div><div class='add'>+			add_req = request_buf;</div><div class='add'>+			add_req-&gt;dma_region_handle = *gdma_region;</div><div class='add'>+			add_req-&gt;reserved3 = 0;</div><div class='add'>+			page_addr_list = add_req-&gt;page_addr_list;</div><div class='add'>+		} else {</div><div class='add'>+			/* Subsequent create messages */</div><div class='add'>+			err = mana_ib_gd_add_dma_region(dev, gc, add_req, tail,</div><div class='add'>+							expected_status);</div><div class='add'>+			if (err)</div><div class='add'>+				break;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		num_pages_processed += tail;</div><div class='add'>+		tail = 0;</div><div class='add'>+</div><div class='add'>+		/* The remaining pages to create */</div><div class='add'>+		num_pages_to_handle =</div><div class='add'>+			min_t(size_t,</div><div class='add'>+			      num_pages_total - num_pages_processed,</div><div class='add'>+			      max_pgs_add_cmd);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (err)</div><div class='add'>+		mana_ib_gd_destroy_dma_region(dev, *gdma_region);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	kfree(request_buf);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,</div><div class='add'>+			      mana_handle_t *gdma_region, u64 virt)</div><div class='add'>+{</div><div class='add'>+	unsigned long page_sz;</div><div class='add'>+</div><div class='add'>+	page_sz = ib_umem_find_best_pgsz(umem, dev-&gt;adapter_caps.page_size_cap, virt);</div><div class='add'>+	if (!page_sz) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ib_dev, "Failed to find page size.\n");</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return mana_ib_gd_create_dma_region(dev, umem, gdma_region, page_sz);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_create_zero_offset_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,</div><div class='add'>+					  mana_handle_t *gdma_region)</div><div class='add'>+{</div><div class='add'>+	unsigned long page_sz;</div><div class='add'>+</div><div class='add'>+	/* Hardware requires dma region to align to chosen page size */</div><div class='add'>+	page_sz = ib_umem_find_best_pgoff(umem, dev-&gt;adapter_caps.page_size_cap, 0);</div><div class='add'>+	if (!page_sz) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ib_dev, "Failed to find page size.\n");</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return mana_ib_gd_create_dma_region(dev, umem, gdma_region, page_sz);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64 gdma_region)</div><div class='add'>+{</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(dev);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ib_dev, "destroy dma region 0x%llx\n", gdma_region);</div><div class='add'>+</div><div class='add'>+	return mana_gd_destroy_dma_region(gc, gdma_region);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_ucontext *mana_ucontext =</div><div class='add'>+		container_of(ibcontext, struct mana_ib_ucontext, ibucontext);</div><div class='add'>+	struct ib_device *ibdev = ibcontext-&gt;device;</div><div class='add'>+	struct mana_ib_dev *mdev;</div><div class='add'>+	struct gdma_context *gc;</div><div class='add'>+	phys_addr_t pfn;</div><div class='add'>+	pgprot_t prot;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);</div><div class='add'>+	gc = mdev_to_gc(mdev);</div><div class='add'>+</div><div class='add'>+	if (vma-&gt;vm_pgoff != 0) {</div><div class='add'>+		ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n", vma-&gt;vm_pgoff);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Map to the page indexed by ucontext-&gt;doorbell */</div><div class='add'>+	pfn = (gc-&gt;phys_db_page_base +</div><div class='add'>+	       gc-&gt;db_page_size * mana_ucontext-&gt;doorbell) &gt;&gt;</div><div class='add'>+	      PAGE_SHIFT;</div><div class='add'>+	prot = pgprot_writecombine(vma-&gt;vm_page_prot);</div><div class='add'>+</div><div class='add'>+	ret = rdma_user_mmap_io(ibcontext, vma, pfn, PAGE_SIZE, prot,</div><div class='add'>+				NULL);</div><div class='add'>+	if (ret)</div><div class='add'>+		ibdev_dbg(ibdev, "can't rdma_user_mmap_io ret %d\n", ret);</div><div class='add'>+	else</div><div class='add'>+		ibdev_dbg(ibdev, "mapped I/O pfn 0x%llx page_size %lu, ret %d\n",</div><div class='add'>+			  pfn, PAGE_SIZE, ret);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+			       struct ib_port_immutable *immutable)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *dev = container_of(ibdev, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct ib_port_attr attr;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	err = ib_query_port(ibdev, port_num, &amp;attr);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	immutable-&gt;pkey_tbl_len = attr.pkey_tbl_len;</div><div class='add'>+	immutable-&gt;gid_tbl_len = attr.gid_tbl_len;</div><div class='add'>+</div><div class='add'>+	if (mana_ib_is_rnic(dev)) {</div><div class='add'>+		if (port_num == 1) {</div><div class='add'>+			immutable-&gt;core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;</div><div class='add'>+			immutable-&gt;max_mad_size = IB_MGMT_MAD_SIZE;</div><div class='add'>+		} else {</div><div class='add'>+			immutable-&gt;core_cap_flags = RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP</div><div class='add'>+						    | RDMA_CORE_CAP_ETH_AH;</div><div class='add'>+			immutable-&gt;max_mad_size = 0;</div><div class='add'>+		}</div><div class='add'>+	} else {</div><div class='add'>+		immutable-&gt;core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props,</div><div class='add'>+			 struct ib_udata *uhw)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *dev = container_of(ibdev, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct pci_dev *pdev = to_pci_dev(mdev_to_gc(dev)-&gt;dev);</div><div class='add'>+</div><div class='add'>+	memset(props, 0, sizeof(*props));</div><div class='add'>+	props-&gt;vendor_id = pdev-&gt;vendor;</div><div class='add'>+	props-&gt;vendor_part_id = dev-&gt;gdma_dev-&gt;dev_id.type;</div><div class='add'>+	props-&gt;max_mr_size = MANA_IB_MAX_MR_SIZE;</div><div class='add'>+	props-&gt;page_size_cap = dev-&gt;adapter_caps.page_size_cap;</div><div class='add'>+	props-&gt;max_qp = dev-&gt;adapter_caps.max_qp_count;</div><div class='add'>+	props-&gt;max_qp_wr = dev-&gt;adapter_caps.max_qp_wr;</div><div class='add'>+	props-&gt;device_cap_flags = IB_DEVICE_RC_RNR_NAK_GEN;</div><div class='add'>+	props-&gt;max_send_sge = dev-&gt;adapter_caps.max_send_sge_count;</div><div class='add'>+	props-&gt;max_recv_sge = dev-&gt;adapter_caps.max_recv_sge_count;</div><div class='add'>+	props-&gt;max_sge_rd = dev-&gt;adapter_caps.max_recv_sge_count;</div><div class='add'>+	props-&gt;max_cq = dev-&gt;adapter_caps.max_cq_count;</div><div class='add'>+	props-&gt;max_cqe = dev-&gt;adapter_caps.max_qp_wr;</div><div class='add'>+	props-&gt;max_mr = dev-&gt;adapter_caps.max_mr_count;</div><div class='add'>+	props-&gt;max_pd = dev-&gt;adapter_caps.max_pd_count;</div><div class='add'>+	props-&gt;max_qp_rd_atom = dev-&gt;adapter_caps.max_inbound_read_limit;</div><div class='add'>+	props-&gt;max_res_rd_atom = props-&gt;max_qp_rd_atom * props-&gt;max_qp;</div><div class='add'>+	props-&gt;max_qp_init_rd_atom = dev-&gt;adapter_caps.max_outbound_read_limit;</div><div class='add'>+	props-&gt;atomic_cap = IB_ATOMIC_NONE;</div><div class='add'>+	props-&gt;masked_atomic_cap = IB_ATOMIC_NONE;</div><div class='add'>+	props-&gt;max_ah = INT_MAX;</div><div class='add'>+	props-&gt;max_pkeys = 1;</div><div class='add'>+	props-&gt;local_ca_ack_delay = MANA_CA_ACK_DELAY;</div><div class='add'>+	if (!mana_ib_is_rnic(dev))</div><div class='add'>+		props-&gt;raw_packet_caps = IB_RAW_PACKET_CAP_IP_CSUM;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_query_port(struct ib_device *ibdev, u32 port,</div><div class='add'>+		       struct ib_port_attr *props)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *dev = container_of(ibdev, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct net_device *ndev = mana_ib_get_netdev(ibdev, port);</div><div class='add'>+</div><div class='add'>+	if (!ndev)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	memset(props, 0, sizeof(*props));</div><div class='add'>+	props-&gt;max_mtu = IB_MTU_4096;</div><div class='add'>+	props-&gt;active_mtu = ib_mtu_int_to_enum(ndev-&gt;mtu);</div><div class='add'>+</div><div class='add'>+	if (netif_carrier_ok(ndev) &amp;&amp; netif_running(ndev)) {</div><div class='add'>+		props-&gt;state = IB_PORT_ACTIVE;</div><div class='add'>+		props-&gt;phys_state = IB_PORT_PHYS_STATE_LINK_UP;</div><div class='add'>+	} else {</div><div class='add'>+		props-&gt;state = IB_PORT_DOWN;</div><div class='add'>+		props-&gt;phys_state = IB_PORT_PHYS_STATE_DISABLED;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	props-&gt;active_width = IB_WIDTH_4X;</div><div class='add'>+	props-&gt;active_speed = IB_SPEED_EDR;</div><div class='add'>+	props-&gt;pkey_tbl_len = 1;</div><div class='add'>+	if (mana_ib_is_rnic(dev)) {</div><div class='add'>+		props-&gt;gid_tbl_len = 16;</div><div class='add'>+		props-&gt;ip_gids = true;</div><div class='add'>+		if (port == 1)</div><div class='add'>+			props-&gt;port_cap_flags = IB_PORT_CM_SUP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_num)</div><div class='add'>+{</div><div class='add'>+	return IB_LINK_LAYER_ETHERNET;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)</div><div class='add'>+{</div><div class='add'>+	if (index != 0)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	*pkey = IB_DEFAULT_PKEY_FULL;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,</div><div class='add'>+		      union ib_gid *gid)</div><div class='add'>+{</div><div class='add'>+	/* This version doesn't return GID properties */</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)</div><div class='add'>+{</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_adapter_caps *caps = &amp;dev-&gt;adapter_caps;</div><div class='add'>+	struct mana_ib_query_adapter_caps_resp resp = {};</div><div class='add'>+	struct mana_ib_query_adapter_caps_req req = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, MANA_IB_GET_ADAPTER_CAP, sizeof(req),</div><div class='add'>+			     sizeof(resp));</div><div class='add'>+	req.hdr.resp.msg_version = GDMA_MESSAGE_V4;</div><div class='add'>+	req.hdr.dev_id = dev-&gt;gdma_dev-&gt;dev_id;</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req),</div><div class='add'>+				   &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;dev-&gt;ib_dev,</div><div class='add'>+			  "Failed to query adapter caps err %d", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	caps-&gt;max_sq_id = resp.max_sq_id;</div><div class='add'>+	caps-&gt;max_rq_id = resp.max_rq_id;</div><div class='add'>+	caps-&gt;max_cq_id = resp.max_cq_id;</div><div class='add'>+	caps-&gt;max_qp_count = resp.max_qp_count;</div><div class='add'>+	caps-&gt;max_cq_count = resp.max_cq_count;</div><div class='add'>+	caps-&gt;max_mr_count = resp.max_mr_count;</div><div class='add'>+	caps-&gt;max_pd_count = resp.max_pd_count;</div><div class='add'>+	caps-&gt;max_inbound_read_limit = resp.max_inbound_read_limit;</div><div class='add'>+	caps-&gt;max_outbound_read_limit = resp.max_outbound_read_limit;</div><div class='add'>+	caps-&gt;mw_count = resp.mw_count;</div><div class='add'>+	caps-&gt;max_srq_count = resp.max_srq_count;</div><div class='add'>+	caps-&gt;max_qp_wr = min_t(u32,</div><div class='add'>+				resp.max_requester_sq_size / GDMA_MAX_SQE_SIZE,</div><div class='add'>+				resp.max_requester_rq_size / GDMA_MAX_RQE_SIZE);</div><div class='add'>+	caps-&gt;max_inline_data_size = resp.max_inline_data_size;</div><div class='add'>+	caps-&gt;max_send_sge_count = resp.max_send_sge_count;</div><div class='add'>+	caps-&gt;max_recv_sge_count = resp.max_recv_sge_count;</div><div class='add'>+	caps-&gt;feature_flags = resp.feature_flags;</div><div class='add'>+</div><div class='add'>+	caps-&gt;page_size_cap = PAGE_SZ_BM;</div><div class='add'>+	if (mdev_to_gc(dev)-&gt;pf_cap_flags1 &amp; GDMA_DRV_CAP_FLAG_1_GDMA_PAGES_4MB_1GB_2GB)</div><div class='add'>+		caps-&gt;page_size_cap |= (SZ_4M | SZ_1G | SZ_2G);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_eth_query_adapter_caps(struct mana_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_adapter_caps *caps = &amp;dev-&gt;adapter_caps;</div><div class='add'>+	struct gdma_query_max_resources_resp resp = {};</div><div class='add'>+	struct gdma_general_req req = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, GDMA_QUERY_MAX_RESOURCES,</div><div class='add'>+			     sizeof(req), sizeof(resp));</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;dev-&gt;ib_dev,</div><div class='add'>+			  "Failed to query adapter caps err %d", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	caps-&gt;max_qp_count = min_t(u32, resp.max_sq, resp.max_rq);</div><div class='add'>+	caps-&gt;max_cq_count = resp.max_cq;</div><div class='add'>+	caps-&gt;max_mr_count = resp.max_mst;</div><div class='add'>+	caps-&gt;max_pd_count = 0x6000;</div><div class='add'>+	caps-&gt;max_qp_wr = min_t(u32,</div><div class='add'>+				0x100000 / GDMA_MAX_SQE_SIZE,</div><div class='add'>+				0x100000 / GDMA_MAX_RQE_SIZE);</div><div class='add'>+	caps-&gt;max_send_sge_count = 30;</div><div class='add'>+	caps-&gt;max_recv_sge_count = 15;</div><div class='add'>+	caps-&gt;page_size_cap = PAGE_SZ_BM;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void</div><div class='add'>+mana_ib_event_handler(void *ctx, struct gdma_queue *q, struct gdma_event *event)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev = (struct mana_ib_dev *)ctx;</div><div class='add'>+	struct mana_ib_qp *qp;</div><div class='add'>+	struct ib_event ev;</div><div class='add'>+	u32 qpn;</div><div class='add'>+</div><div class='add'>+	switch (event-&gt;type) {</div><div class='add'>+	case GDMA_EQE_RNIC_QP_FATAL:</div><div class='add'>+		qpn = event-&gt;details[0];</div><div class='add'>+		qp = mana_get_qp_ref(mdev, qpn, false);</div><div class='add'>+		if (!qp)</div><div class='add'>+			break;</div><div class='add'>+		if (qp-&gt;ibqp.event_handler) {</div><div class='add'>+			ev.device = qp-&gt;ibqp.device;</div><div class='add'>+			ev.element.qp = &amp;qp-&gt;ibqp;</div><div class='add'>+			ev.event = IB_EVENT_QP_FATAL;</div><div class='add'>+			qp-&gt;ibqp.event_handler(&amp;ev, qp-&gt;ibqp.qp_context);</div><div class='add'>+		}</div><div class='add'>+		mana_put_qp_ref(qp);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_create_eqs(struct mana_ib_dev *mdev)</div><div class='add'>+{</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(mdev);</div><div class='add'>+	struct gdma_queue_spec spec = {};</div><div class='add'>+	int err, i;</div><div class='add'>+</div><div class='add'>+	spec.type = GDMA_EQ;</div><div class='add'>+	spec.monitor_avl_buf = false;</div><div class='add'>+	spec.queue_size = EQ_SIZE;</div><div class='add'>+	spec.eq.callback = mana_ib_event_handler;</div><div class='add'>+	spec.eq.context = mdev;</div><div class='add'>+	spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE;</div><div class='add'>+	spec.eq.msix_index = 0;</div><div class='add'>+</div><div class='add'>+	err = mana_gd_create_mana_eq(mdev-&gt;gdma_dev, &amp;spec, &amp;mdev-&gt;fatal_err_eq);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	mdev-&gt;eqs = kcalloc(mdev-&gt;ib_dev.num_comp_vectors, sizeof(struct gdma_queue *),</div><div class='add'>+			    GFP_KERNEL);</div><div class='add'>+	if (!mdev-&gt;eqs) {</div><div class='add'>+		err = -ENOMEM;</div><div class='add'>+		goto destroy_fatal_eq;</div><div class='add'>+	}</div><div class='add'>+	spec.eq.callback = NULL;</div><div class='add'>+	for (i = 0; i &lt; mdev-&gt;ib_dev.num_comp_vectors; i++) {</div><div class='add'>+		spec.eq.msix_index = (i + 1) % gc-&gt;num_msix_usable;</div><div class='add'>+		err = mana_gd_create_mana_eq(mdev-&gt;gdma_dev, &amp;spec, &amp;mdev-&gt;eqs[i]);</div><div class='add'>+		if (err)</div><div class='add'>+			goto destroy_eqs;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+destroy_eqs:</div><div class='add'>+	while (i-- &gt; 0)</div><div class='add'>+		mana_gd_destroy_queue(gc, mdev-&gt;eqs[i]);</div><div class='add'>+	kfree(mdev-&gt;eqs);</div><div class='add'>+destroy_fatal_eq:</div><div class='add'>+	mana_gd_destroy_queue(gc, mdev-&gt;fatal_err_eq);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mana_ib_destroy_eqs(struct mana_ib_dev *mdev)</div><div class='add'>+{</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(mdev);</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	mana_gd_destroy_queue(gc, mdev-&gt;fatal_err_eq);</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; mdev-&gt;ib_dev.num_comp_vectors; i++)</div><div class='add'>+		mana_gd_destroy_queue(gc, mdev-&gt;eqs[i]);</div><div class='add'>+</div><div class='add'>+	kfree(mdev-&gt;eqs);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev)</div><div class='add'>+{</div><div class='add'>+	struct mana_rnic_create_adapter_resp resp = {};</div><div class='add'>+	struct mana_rnic_create_adapter_req req = {};</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(mdev);</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, MANA_IB_CREATE_ADAPTER, sizeof(req), sizeof(resp));</div><div class='add'>+	req.hdr.req.msg_version = GDMA_MESSAGE_V2;</div><div class='add'>+	req.hdr.dev_id = mdev-&gt;gdma_dev-&gt;dev_id;</div><div class='add'>+	req.notify_eq_id = mdev-&gt;fatal_err_eq-&gt;id;</div><div class='add'>+</div><div class='add'>+	if (mdev-&gt;adapter_caps.feature_flags &amp; MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT)</div><div class='add'>+		req.feature_flags |= MANA_IB_FEATURE_CLIENT_ERROR_CQE_REQUEST;</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to create RNIC adapter err %d", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+	mdev-&gt;adapter_handle = resp.adapter;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev)</div><div class='add'>+{</div><div class='add'>+	struct mana_rnic_destroy_adapter_resp resp = {};</div><div class='add'>+	struct mana_rnic_destroy_adapter_req req = {};</div><div class='add'>+	struct gdma_context *gc;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	gc = mdev_to_gc(mdev);</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, MANA_IB_DESTROY_ADAPTER, sizeof(req), sizeof(resp));</div><div class='add'>+	req.hdr.dev_id = mdev-&gt;gdma_dev-&gt;dev_id;</div><div class='add'>+	req.adapter = mdev-&gt;adapter_handle;</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to destroy RNIC adapter err %d", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, void **context)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev = container_of(attr-&gt;device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	enum rdma_network_type ntype = rdma_gid_attr_network_type(attr);</div><div class='add'>+	struct mana_rnic_config_addr_resp resp = {};</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(mdev);</div><div class='add'>+	struct mana_rnic_config_addr_req req = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (ntype != RDMA_NETWORK_IPV4 &amp;&amp; ntype != RDMA_NETWORK_IPV6) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev, "Unsupported rdma network type %d", ntype);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, MANA_IB_CONFIG_IP_ADDR, sizeof(req), sizeof(resp));</div><div class='add'>+	req.hdr.dev_id = mdev-&gt;gdma_dev-&gt;dev_id;</div><div class='add'>+	req.adapter = mdev-&gt;adapter_handle;</div><div class='add'>+	req.op = ADDR_OP_ADD;</div><div class='add'>+	req.sgid_type = (ntype == RDMA_NETWORK_IPV6) ? SGID_TYPE_IPV6 : SGID_TYPE_IPV4;</div><div class='add'>+	copy_in_reverse(req.ip_addr, attr-&gt;gid.raw, sizeof(union ib_gid));</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to config IP addr err %d\n", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev = container_of(attr-&gt;device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	enum rdma_network_type ntype = rdma_gid_attr_network_type(attr);</div><div class='add'>+	struct mana_rnic_config_addr_resp resp = {};</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(mdev);</div><div class='add'>+	struct mana_rnic_config_addr_req req = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (ntype != RDMA_NETWORK_IPV4 &amp;&amp; ntype != RDMA_NETWORK_IPV6) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev, "Unsupported rdma network type %d", ntype);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, MANA_IB_CONFIG_IP_ADDR, sizeof(req), sizeof(resp));</div><div class='add'>+	req.hdr.dev_id = mdev-&gt;gdma_dev-&gt;dev_id;</div><div class='add'>+	req.adapter = mdev-&gt;adapter_handle;</div><div class='add'>+	req.op = ADDR_OP_REMOVE;</div><div class='add'>+	req.sgid_type = (ntype == RDMA_NETWORK_IPV6) ? SGID_TYPE_IPV6 : SGID_TYPE_IPV4;</div><div class='add'>+	copy_in_reverse(req.ip_addr, attr-&gt;gid.raw, sizeof(union ib_gid));</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to config IP addr err %d\n", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8 *mac)</div><div class='add'>+{</div><div class='add'>+	struct mana_rnic_config_mac_addr_resp resp = {};</div><div class='add'>+	struct mana_rnic_config_mac_addr_req req = {};</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(mdev);</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, MANA_IB_CONFIG_MAC_ADDR, sizeof(req), sizeof(resp));</div><div class='add'>+	req.hdr.dev_id = mdev-&gt;gdma_dev-&gt;dev_id;</div><div class='add'>+	req.adapter = mdev-&gt;adapter_handle;</div><div class='add'>+	req.op = op;</div><div class='add'>+	copy_in_reverse(req.mac_addr, mac, ETH_ALEN);</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to config Mac addr err %d", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq, u32 doorbell)</div><div class='add'>+{</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(mdev);</div><div class='add'>+	struct mana_rnic_create_cq_resp resp = {};</div><div class='add'>+	struct mana_rnic_create_cq_req req = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (!mdev-&gt;eqs)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, MANA_IB_CREATE_CQ, sizeof(req), sizeof(resp));</div><div class='add'>+	req.hdr.dev_id = mdev-&gt;gdma_dev-&gt;dev_id;</div><div class='add'>+	req.adapter = mdev-&gt;adapter_handle;</div><div class='add'>+	req.gdma_region = cq-&gt;queue.gdma_region;</div><div class='add'>+	req.eq_id = mdev-&gt;eqs[cq-&gt;comp_vector]-&gt;id;</div><div class='add'>+	req.doorbell_page = doorbell;</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to create cq err %d", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cq-&gt;queue.id  = resp.cq_id;</div><div class='add'>+	cq-&gt;cq_handle = resp.cq_handle;</div><div class='add'>+	/* The GDMA region is now owned by the CQ handle */</div><div class='add'>+	cq-&gt;queue.gdma_region = GDMA_INVALID_DMA_REGION;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)</div><div class='add'>+{</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(mdev);</div><div class='add'>+	struct mana_rnic_destroy_cq_resp resp = {};</div><div class='add'>+	struct mana_rnic_destroy_cq_req req = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (cq-&gt;cq_handle == INVALID_MANA_HANDLE)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, MANA_IB_DESTROY_CQ, sizeof(req), sizeof(resp));</div><div class='add'>+	req.hdr.dev_id = mdev-&gt;gdma_dev-&gt;dev_id;</div><div class='add'>+	req.adapter = mdev-&gt;adapter_handle;</div><div class='add'>+	req.cq_handle = cq-&gt;cq_handle;</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to destroy cq err %d", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,</div><div class='add'>+			    struct ib_qp_init_attr *attr, u32 doorbell, u64 flags)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_cq *send_cq = container_of(qp-&gt;ibqp.send_cq, struct mana_ib_cq, ibcq);</div><div class='add'>+	struct mana_ib_cq *recv_cq = container_of(qp-&gt;ibqp.recv_cq, struct mana_ib_cq, ibcq);</div><div class='add'>+	struct mana_ib_pd *pd = container_of(qp-&gt;ibqp.pd, struct mana_ib_pd, ibpd);</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(mdev);</div><div class='add'>+	struct mana_rnic_create_qp_resp resp = {};</div><div class='add'>+	struct mana_rnic_create_qp_req req = {};</div><div class='add'>+	int err, i;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, MANA_IB_CREATE_RC_QP, sizeof(req), sizeof(resp));</div><div class='add'>+	req.hdr.dev_id = mdev-&gt;gdma_dev-&gt;dev_id;</div><div class='add'>+	req.adapter = mdev-&gt;adapter_handle;</div><div class='add'>+	req.pd_handle = pd-&gt;pd_handle;</div><div class='add'>+	req.send_cq_handle = send_cq-&gt;cq_handle;</div><div class='add'>+	req.recv_cq_handle = recv_cq-&gt;cq_handle;</div><div class='add'>+	for (i = 0; i &lt; MANA_RC_QUEUE_TYPE_MAX; i++)</div><div class='add'>+		req.dma_region[i] = qp-&gt;rc_qp.queues[i].gdma_region;</div><div class='add'>+	req.doorbell_page = doorbell;</div><div class='add'>+	req.max_send_wr = attr-&gt;cap.max_send_wr;</div><div class='add'>+	req.max_recv_wr = attr-&gt;cap.max_recv_wr;</div><div class='add'>+	req.max_send_sge = attr-&gt;cap.max_send_sge;</div><div class='add'>+	req.max_recv_sge = attr-&gt;cap.max_recv_sge;</div><div class='add'>+	req.flags = flags;</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to create rc qp err %d", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+	qp-&gt;qp_handle = resp.rc_qp_handle;</div><div class='add'>+	for (i = 0; i &lt; MANA_RC_QUEUE_TYPE_MAX; i++) {</div><div class='add'>+		qp-&gt;rc_qp.queues[i].id = resp.queue_ids[i];</div><div class='add'>+		/* The GDMA regions are now owned by the RNIC QP handle */</div><div class='add'>+		qp-&gt;rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_destroy_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct mana_rnic_destroy_rc_qp_resp resp = {0};</div><div class='add'>+	struct mana_rnic_destroy_rc_qp_req req = {0};</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(mdev);</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, MANA_IB_DESTROY_RC_QP, sizeof(req), sizeof(resp));</div><div class='add'>+	req.hdr.dev_id = mdev-&gt;gdma_dev-&gt;dev_id;</div><div class='add'>+	req.adapter = mdev-&gt;adapter_handle;</div><div class='add'>+	req.rc_qp_handle = qp-&gt;qp_handle;</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to destroy rc qp err %d", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_create_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,</div><div class='add'>+			    struct ib_qp_init_attr *attr, u32 doorbell, u32 type)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_cq *send_cq = container_of(qp-&gt;ibqp.send_cq, struct mana_ib_cq, ibcq);</div><div class='add'>+	struct mana_ib_cq *recv_cq = container_of(qp-&gt;ibqp.recv_cq, struct mana_ib_cq, ibcq);</div><div class='add'>+	struct mana_ib_pd *pd = container_of(qp-&gt;ibqp.pd, struct mana_ib_pd, ibpd);</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(mdev);</div><div class='add'>+	struct mana_rnic_create_udqp_resp resp = {};</div><div class='add'>+	struct mana_rnic_create_udqp_req req = {};</div><div class='add'>+	int err, i;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, MANA_IB_CREATE_UD_QP, sizeof(req), sizeof(resp));</div><div class='add'>+	req.hdr.dev_id = mdev-&gt;gdma_dev-&gt;dev_id;</div><div class='add'>+	req.adapter = mdev-&gt;adapter_handle;</div><div class='add'>+	req.pd_handle = pd-&gt;pd_handle;</div><div class='add'>+	req.send_cq_handle = send_cq-&gt;cq_handle;</div><div class='add'>+	req.recv_cq_handle = recv_cq-&gt;cq_handle;</div><div class='add'>+	for (i = 0; i &lt; MANA_UD_QUEUE_TYPE_MAX; i++)</div><div class='add'>+		req.dma_region[i] = qp-&gt;ud_qp.queues[i].gdma_region;</div><div class='add'>+	req.doorbell_page = doorbell;</div><div class='add'>+	req.max_send_wr = attr-&gt;cap.max_send_wr;</div><div class='add'>+	req.max_recv_wr = attr-&gt;cap.max_recv_wr;</div><div class='add'>+	req.max_send_sge = attr-&gt;cap.max_send_sge;</div><div class='add'>+	req.max_recv_sge = attr-&gt;cap.max_recv_sge;</div><div class='add'>+	req.qp_type = type;</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to create ud qp err %d", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+	qp-&gt;qp_handle = resp.qp_handle;</div><div class='add'>+	for (i = 0; i &lt; MANA_UD_QUEUE_TYPE_MAX; i++) {</div><div class='add'>+		qp-&gt;ud_qp.queues[i].id = resp.queue_ids[i];</div><div class='add'>+		/* The GDMA regions are now owned by the RNIC QP handle */</div><div class='add'>+		qp-&gt;ud_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_destroy_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct mana_rnic_destroy_udqp_resp resp = {0};</div><div class='add'>+	struct mana_rnic_destroy_udqp_req req = {0};</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(mdev);</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, MANA_IB_DESTROY_UD_QP, sizeof(req), sizeof(resp));</div><div class='add'>+	req.hdr.dev_id = mdev-&gt;gdma_dev-&gt;dev_id;</div><div class='add'>+	req.adapter = mdev-&gt;adapter_handle;</div><div class='add'>+	req.qp_handle = qp-&gt;qp_handle;</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to destroy ud qp err %d", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h<br/>new file mode 100644<br/>index 000000000000..9d36232ed880<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mana/mana_ib.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mana/mana_ib.h</a></div><div class='hunk'>@@ -0,0 +1,738 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0-only */</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2022 Microsoft Corporation. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef _MANA_IB_H_</div><div class='add'>+#define _MANA_IB_H_</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+#include &lt;rdma/ib_mad.h&gt;</div><div class='add'>+#include &lt;rdma/ib_umem.h&gt;</div><div class='add'>+#include &lt;rdma/mana-abi.h&gt;</div><div class='add'>+#include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='add'>+#include &lt;linux/dmapool.h&gt;</div><div class='add'>+</div><div class='add'>+#include &lt;net/mana/mana.h&gt;</div><div class='add'>+#include "shadow_queue.h"</div><div class='add'>+#include "counters.h"</div><div class='add'>+</div><div class='add'>+#define PAGE_SZ_BM                                                             \</div><div class='add'>+	(SZ_4K | SZ_8K | SZ_16K | SZ_32K | SZ_64K | SZ_128K | SZ_256K |        \</div><div class='add'>+	 SZ_512K | SZ_1M | SZ_2M)</div><div class='add'>+</div><div class='add'>+/* MANA doesn't have any limit for MR size */</div><div class='add'>+#define MANA_IB_MAX_MR_SIZE	U64_MAX</div><div class='add'>+</div><div class='add'>+/* Send queue ID mask */</div><div class='add'>+#define MANA_SENDQ_MASK	BIT(31)</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * The hardware limit of number of MRs is greater than maximum number of MRs</div><div class='add'>+ * that can possibly represent in 24 bits</div><div class='add'>+ */</div><div class='add'>+#define MANA_IB_MAX_MR		0xFFFFFFu</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * The CA timeout is approx. 260ms (4us * 2^(DELAY))</div><div class='add'>+ */</div><div class='add'>+#define MANA_CA_ACK_DELAY	16</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * The buffer used for writing AV</div><div class='add'>+ */</div><div class='add'>+#define MANA_AV_BUFFER_SIZE	64</div><div class='add'>+</div><div class='add'>+#define MANA_GSI_QPN		(1)</div><div class='add'>+</div><div class='add'>+struct mana_ib_adapter_caps {</div><div class='add'>+	u32 max_sq_id;</div><div class='add'>+	u32 max_rq_id;</div><div class='add'>+	u32 max_cq_id;</div><div class='add'>+	u32 max_qp_count;</div><div class='add'>+	u32 max_cq_count;</div><div class='add'>+	u32 max_mr_count;</div><div class='add'>+	u32 max_pd_count;</div><div class='add'>+	u32 max_inbound_read_limit;</div><div class='add'>+	u32 max_outbound_read_limit;</div><div class='add'>+	u32 mw_count;</div><div class='add'>+	u32 max_srq_count;</div><div class='add'>+	u32 max_qp_wr;</div><div class='add'>+	u32 max_send_sge_count;</div><div class='add'>+	u32 max_recv_sge_count;</div><div class='add'>+	u32 max_inline_data_size;</div><div class='add'>+	u64 feature_flags;</div><div class='add'>+	u64 page_size_cap;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_ib_queue {</div><div class='add'>+	struct ib_umem *umem;</div><div class='add'>+	struct gdma_queue *kmem;</div><div class='add'>+	u64 gdma_region;</div><div class='add'>+	u64 id;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_ib_dev {</div><div class='add'>+	struct ib_device ib_dev;</div><div class='add'>+	struct gdma_dev *gdma_dev;</div><div class='add'>+	mana_handle_t adapter_handle;</div><div class='add'>+	struct gdma_queue *fatal_err_eq;</div><div class='add'>+	struct gdma_queue **eqs;</div><div class='add'>+	struct xarray qp_table_wq;</div><div class='add'>+	struct mana_ib_adapter_caps adapter_caps;</div><div class='add'>+	struct dma_pool *av_pool;</div><div class='add'>+	netdevice_tracker dev_tracker;</div><div class='add'>+	struct notifier_block nb;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_ib_wq {</div><div class='add'>+	struct ib_wq ibwq;</div><div class='add'>+	struct mana_ib_queue queue;</div><div class='add'>+	int wqe;</div><div class='add'>+	u32 wq_buf_size;</div><div class='add'>+	mana_handle_t rx_object;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_ib_pd {</div><div class='add'>+	struct ib_pd ibpd;</div><div class='add'>+	u32 pdn;</div><div class='add'>+	mana_handle_t pd_handle;</div><div class='add'>+</div><div class='add'>+	/* Mutex for sharing access to vport_use_count */</div><div class='add'>+	struct mutex vport_mutex;</div><div class='add'>+	int vport_use_count;</div><div class='add'>+</div><div class='add'>+	bool tx_shortform_allowed;</div><div class='add'>+	u32 tx_vp_offset;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_ib_av {</div><div class='add'>+	u8 dest_ip[16];</div><div class='add'>+	u8 dest_mac[ETH_ALEN];</div><div class='add'>+	u16 udp_src_port;</div><div class='add'>+	u8 src_ip[16];</div><div class='add'>+	u32 hop_limit	: 8;</div><div class='add'>+	u32 reserved1	: 12;</div><div class='add'>+	u32 dscp	: 6;</div><div class='add'>+	u32 reserved2	: 5;</div><div class='add'>+	u32 is_ipv6	: 1;</div><div class='add'>+	u32 reserved3	: 32;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_ib_ah {</div><div class='add'>+	struct ib_ah ibah;</div><div class='add'>+	struct mana_ib_av *av;</div><div class='add'>+	dma_addr_t dma_handle;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_ib_mr {</div><div class='add'>+	struct ib_mr ibmr;</div><div class='add'>+	struct ib_umem *umem;</div><div class='add'>+	mana_handle_t mr_handle;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_ib_cq {</div><div class='add'>+	struct ib_cq ibcq;</div><div class='add'>+	struct mana_ib_queue queue;</div><div class='add'>+	/* protects CQ polling */</div><div class='add'>+	spinlock_t cq_lock;</div><div class='add'>+	struct list_head list_send_qp;</div><div class='add'>+	struct list_head list_recv_qp;</div><div class='add'>+	int cqe;</div><div class='add'>+	u32 comp_vector;</div><div class='add'>+	mana_handle_t  cq_handle;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum mana_rc_queue_type {</div><div class='add'>+	MANA_RC_SEND_QUEUE_REQUESTER = 0,</div><div class='add'>+	MANA_RC_SEND_QUEUE_RESPONDER,</div><div class='add'>+	MANA_RC_SEND_QUEUE_FMR,</div><div class='add'>+	MANA_RC_RECV_QUEUE_REQUESTER,</div><div class='add'>+	MANA_RC_RECV_QUEUE_RESPONDER,</div><div class='add'>+	MANA_RC_QUEUE_TYPE_MAX,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_ib_rc_qp {</div><div class='add'>+	struct mana_ib_queue queues[MANA_RC_QUEUE_TYPE_MAX];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum mana_ud_queue_type {</div><div class='add'>+	MANA_UD_SEND_QUEUE = 0,</div><div class='add'>+	MANA_UD_RECV_QUEUE,</div><div class='add'>+	MANA_UD_QUEUE_TYPE_MAX,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_ib_ud_qp {</div><div class='add'>+	struct mana_ib_queue queues[MANA_UD_QUEUE_TYPE_MAX];</div><div class='add'>+	u32 sq_psn;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_ib_qp {</div><div class='add'>+	struct ib_qp ibqp;</div><div class='add'>+</div><div class='add'>+	mana_handle_t qp_handle;</div><div class='add'>+	union {</div><div class='add'>+		struct mana_ib_queue raw_sq;</div><div class='add'>+		struct mana_ib_rc_qp rc_qp;</div><div class='add'>+		struct mana_ib_ud_qp ud_qp;</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	/* The port on the IB device, starting with 1 */</div><div class='add'>+	u32 port;</div><div class='add'>+</div><div class='add'>+	struct list_head cq_send_list;</div><div class='add'>+	struct list_head cq_recv_list;</div><div class='add'>+	struct shadow_queue shadow_rq;</div><div class='add'>+	struct shadow_queue shadow_sq;</div><div class='add'>+</div><div class='add'>+	refcount_t		refcount;</div><div class='add'>+	struct completion	free;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_ib_ucontext {</div><div class='add'>+	struct ib_ucontext ibucontext;</div><div class='add'>+	u32 doorbell;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_ib_rwq_ind_table {</div><div class='add'>+	struct ib_rwq_ind_table ib_ind_table;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum mana_ib_command_code {</div><div class='add'>+	MANA_IB_GET_ADAPTER_CAP = 0x30001,</div><div class='add'>+	MANA_IB_CREATE_ADAPTER  = 0x30002,</div><div class='add'>+	MANA_IB_DESTROY_ADAPTER = 0x30003,</div><div class='add'>+	MANA_IB_CONFIG_IP_ADDR	= 0x30004,</div><div class='add'>+	MANA_IB_CONFIG_MAC_ADDR	= 0x30005,</div><div class='add'>+	MANA_IB_CREATE_UD_QP	= 0x30006,</div><div class='add'>+	MANA_IB_DESTROY_UD_QP	= 0x30007,</div><div class='add'>+	MANA_IB_CREATE_CQ       = 0x30008,</div><div class='add'>+	MANA_IB_DESTROY_CQ      = 0x30009,</div><div class='add'>+	MANA_IB_CREATE_RC_QP    = 0x3000a,</div><div class='add'>+	MANA_IB_DESTROY_RC_QP   = 0x3000b,</div><div class='add'>+	MANA_IB_SET_QP_STATE	= 0x3000d,</div><div class='add'>+	MANA_IB_QUERY_VF_COUNTERS = 0x30022,</div><div class='add'>+	MANA_IB_QUERY_DEVICE_COUNTERS = 0x30023,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_ib_query_adapter_caps_req {</div><div class='add'>+	struct gdma_req_hdr hdr;</div><div class='add'>+}; /*HW Data */</div><div class='add'>+</div><div class='add'>+enum mana_ib_adapter_features {</div><div class='add'>+	MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4),</div><div class='add'>+	MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5),</div><div class='add'>+	MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_ib_query_adapter_caps_resp {</div><div class='add'>+	struct gdma_resp_hdr hdr;</div><div class='add'>+	u32 max_sq_id;</div><div class='add'>+	u32 max_rq_id;</div><div class='add'>+	u32 max_cq_id;</div><div class='add'>+	u32 max_qp_count;</div><div class='add'>+	u32 max_cq_count;</div><div class='add'>+	u32 max_mr_count;</div><div class='add'>+	u32 max_pd_count;</div><div class='add'>+	u32 max_inbound_read_limit;</div><div class='add'>+	u32 max_outbound_read_limit;</div><div class='add'>+	u32 mw_count;</div><div class='add'>+	u32 max_srq_count;</div><div class='add'>+	u32 max_requester_sq_size;</div><div class='add'>+	u32 max_responder_sq_size;</div><div class='add'>+	u32 max_requester_rq_size;</div><div class='add'>+	u32 max_responder_rq_size;</div><div class='add'>+	u32 max_send_sge_count;</div><div class='add'>+	u32 max_recv_sge_count;</div><div class='add'>+	u32 max_inline_data_size;</div><div class='add'>+	u64 feature_flags;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+enum mana_ib_adapter_features_request {</div><div class='add'>+	MANA_IB_FEATURE_CLIENT_ERROR_CQE_REQUEST = BIT(1),</div><div class='add'>+}; /*HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_create_adapter_req {</div><div class='add'>+	struct gdma_req_hdr hdr;</div><div class='add'>+	u32 notify_eq_id;</div><div class='add'>+	u32 reserved;</div><div class='add'>+	u64 feature_flags;</div><div class='add'>+}; /*HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_create_adapter_resp {</div><div class='add'>+	struct gdma_resp_hdr hdr;</div><div class='add'>+	mana_handle_t adapter;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_destroy_adapter_req {</div><div class='add'>+	struct gdma_req_hdr hdr;</div><div class='add'>+	mana_handle_t adapter;</div><div class='add'>+}; /*HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_destroy_adapter_resp {</div><div class='add'>+	struct gdma_resp_hdr hdr;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+enum mana_ib_addr_op {</div><div class='add'>+	ADDR_OP_ADD = 1,</div><div class='add'>+	ADDR_OP_REMOVE = 2,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum sgid_entry_type {</div><div class='add'>+	SGID_TYPE_IPV4 = 1,</div><div class='add'>+	SGID_TYPE_IPV6 = 2,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_rnic_config_addr_req {</div><div class='add'>+	struct gdma_req_hdr hdr;</div><div class='add'>+	mana_handle_t adapter;</div><div class='add'>+	enum mana_ib_addr_op op;</div><div class='add'>+	enum sgid_entry_type sgid_type;</div><div class='add'>+	u8 ip_addr[16];</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_config_addr_resp {</div><div class='add'>+	struct gdma_resp_hdr hdr;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_config_mac_addr_req {</div><div class='add'>+	struct gdma_req_hdr hdr;</div><div class='add'>+	mana_handle_t adapter;</div><div class='add'>+	enum mana_ib_addr_op op;</div><div class='add'>+	u8 mac_addr[ETH_ALEN];</div><div class='add'>+	u8 reserved[6];</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_config_mac_addr_resp {</div><div class='add'>+	struct gdma_resp_hdr hdr;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_create_cq_req {</div><div class='add'>+	struct gdma_req_hdr hdr;</div><div class='add'>+	mana_handle_t adapter;</div><div class='add'>+	u64 gdma_region;</div><div class='add'>+	u32 eq_id;</div><div class='add'>+	u32 doorbell_page;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_create_cq_resp {</div><div class='add'>+	struct gdma_resp_hdr hdr;</div><div class='add'>+	mana_handle_t cq_handle;</div><div class='add'>+	u32 cq_id;</div><div class='add'>+	u32 reserved;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_destroy_cq_req {</div><div class='add'>+	struct gdma_req_hdr hdr;</div><div class='add'>+	mana_handle_t adapter;</div><div class='add'>+	mana_handle_t cq_handle;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_destroy_cq_resp {</div><div class='add'>+	struct gdma_resp_hdr hdr;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+enum mana_rnic_create_rc_flags {</div><div class='add'>+	MANA_RC_FLAG_NO_FMR = 2,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_rnic_create_qp_req {</div><div class='add'>+	struct gdma_req_hdr hdr;</div><div class='add'>+	mana_handle_t adapter;</div><div class='add'>+	mana_handle_t pd_handle;</div><div class='add'>+	mana_handle_t send_cq_handle;</div><div class='add'>+	mana_handle_t recv_cq_handle;</div><div class='add'>+	u64 dma_region[MANA_RC_QUEUE_TYPE_MAX];</div><div class='add'>+	u64 deprecated[2];</div><div class='add'>+	u64 flags;</div><div class='add'>+	u32 doorbell_page;</div><div class='add'>+	u32 max_send_wr;</div><div class='add'>+	u32 max_recv_wr;</div><div class='add'>+	u32 max_send_sge;</div><div class='add'>+	u32 max_recv_sge;</div><div class='add'>+	u32 reserved;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_create_qp_resp {</div><div class='add'>+	struct gdma_resp_hdr hdr;</div><div class='add'>+	mana_handle_t rc_qp_handle;</div><div class='add'>+	u32 queue_ids[MANA_RC_QUEUE_TYPE_MAX];</div><div class='add'>+	u32 reserved;</div><div class='add'>+}; /* HW Data*/</div><div class='add'>+</div><div class='add'>+struct mana_rnic_destroy_rc_qp_req {</div><div class='add'>+	struct gdma_req_hdr hdr;</div><div class='add'>+	mana_handle_t adapter;</div><div class='add'>+	mana_handle_t rc_qp_handle;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_destroy_rc_qp_resp {</div><div class='add'>+	struct gdma_resp_hdr hdr;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_create_udqp_req {</div><div class='add'>+	struct gdma_req_hdr hdr;</div><div class='add'>+	mana_handle_t adapter;</div><div class='add'>+	mana_handle_t pd_handle;</div><div class='add'>+	mana_handle_t send_cq_handle;</div><div class='add'>+	mana_handle_t recv_cq_handle;</div><div class='add'>+	u64 dma_region[MANA_UD_QUEUE_TYPE_MAX];</div><div class='add'>+	u32 qp_type;</div><div class='add'>+	u32 doorbell_page;</div><div class='add'>+	u32 max_send_wr;</div><div class='add'>+	u32 max_recv_wr;</div><div class='add'>+	u32 max_send_sge;</div><div class='add'>+	u32 max_recv_sge;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_create_udqp_resp {</div><div class='add'>+	struct gdma_resp_hdr hdr;</div><div class='add'>+	mana_handle_t qp_handle;</div><div class='add'>+	u32 queue_ids[MANA_UD_QUEUE_TYPE_MAX];</div><div class='add'>+}; /* HW Data*/</div><div class='add'>+</div><div class='add'>+struct mana_rnic_destroy_udqp_req {</div><div class='add'>+	struct gdma_req_hdr hdr;</div><div class='add'>+	mana_handle_t adapter;</div><div class='add'>+	mana_handle_t qp_handle;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_destroy_udqp_resp {</div><div class='add'>+	struct gdma_resp_hdr hdr;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_ib_ah_attr {</div><div class='add'>+	u8 src_addr[16];</div><div class='add'>+	u8 dest_addr[16];</div><div class='add'>+	u8 src_mac[ETH_ALEN];</div><div class='add'>+	u8 dest_mac[ETH_ALEN];</div><div class='add'>+	u8 src_addr_type;</div><div class='add'>+	u8 dest_addr_type;</div><div class='add'>+	u8 hop_limit;</div><div class='add'>+	u8 traffic_class;</div><div class='add'>+	u16 src_port;</div><div class='add'>+	u16 dest_port;</div><div class='add'>+	u32 flow_label;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mana_rnic_set_qp_state_req {</div><div class='add'>+	struct gdma_req_hdr hdr;</div><div class='add'>+	mana_handle_t adapter;</div><div class='add'>+	mana_handle_t qp_handle;</div><div class='add'>+	u64 attr_mask;</div><div class='add'>+	u32 qp_state;</div><div class='add'>+	u32 path_mtu;</div><div class='add'>+	u32 rq_psn;</div><div class='add'>+	u32 sq_psn;</div><div class='add'>+	u32 dest_qpn;</div><div class='add'>+	u32 max_dest_rd_atomic;</div><div class='add'>+	u32 retry_cnt;</div><div class='add'>+	u32 rnr_retry;</div><div class='add'>+	u32 min_rnr_timer;</div><div class='add'>+	u32 rate_limit;</div><div class='add'>+	struct mana_ib_ah_attr ah_attr;</div><div class='add'>+	u64 reserved1;</div><div class='add'>+	u32 qkey;</div><div class='add'>+	u32 qp_access_flags;</div><div class='add'>+	u8 local_ack_timeout;</div><div class='add'>+	u8 max_rd_atomic;</div><div class='add'>+	u16 reserved2;</div><div class='add'>+	u32 reserved3;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_set_qp_state_resp {</div><div class='add'>+	struct gdma_resp_hdr hdr;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+enum WQE_OPCODE_TYPES {</div><div class='add'>+	WQE_TYPE_UD_SEND = 0,</div><div class='add'>+	WQE_TYPE_UD_RECV = 8,</div><div class='add'>+}; /* HW DATA */</div><div class='add'>+</div><div class='add'>+struct rdma_send_oob {</div><div class='add'>+	u32 wqe_type	: 5;</div><div class='add'>+	u32 fence	: 1;</div><div class='add'>+	u32 signaled	: 1;</div><div class='add'>+	u32 solicited	: 1;</div><div class='add'>+	u32 psn		: 24;</div><div class='add'>+</div><div class='add'>+	u32 ssn_or_rqpn	: 24;</div><div class='add'>+	u32 reserved1	: 8;</div><div class='add'>+	union {</div><div class='add'>+		struct {</div><div class='add'>+			u32 remote_qkey;</div><div class='add'>+			u32 immediate;</div><div class='add'>+			u32 reserved1;</div><div class='add'>+			u32 reserved2;</div><div class='add'>+		} ud_send;</div><div class='add'>+	};</div><div class='add'>+}; /* HW DATA */</div><div class='add'>+</div><div class='add'>+struct mana_rdma_cqe {</div><div class='add'>+	union {</div><div class='add'>+		struct {</div><div class='add'>+			u8 cqe_type;</div><div class='add'>+			u8 data[GDMA_COMP_DATA_SIZE - 1];</div><div class='add'>+		};</div><div class='add'>+		struct {</div><div class='add'>+			u32 cqe_type		: 8;</div><div class='add'>+			u32 vendor_error	: 9;</div><div class='add'>+			u32 reserved1		: 15;</div><div class='add'>+			u32 sge_offset		: 5;</div><div class='add'>+			u32 tx_wqe_offset	: 27;</div><div class='add'>+		} ud_send;</div><div class='add'>+		struct {</div><div class='add'>+			u32 cqe_type		: 8;</div><div class='add'>+			u32 reserved1		: 24;</div><div class='add'>+			u32 msg_len;</div><div class='add'>+			u32 src_qpn		: 24;</div><div class='add'>+			u32 reserved2		: 8;</div><div class='add'>+			u32 imm_data;</div><div class='add'>+			u32 rx_wqe_offset;</div><div class='add'>+		} ud_recv;</div><div class='add'>+	};</div><div class='add'>+}; /* HW DATA */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_query_vf_cntrs_req {</div><div class='add'>+	struct gdma_req_hdr hdr;</div><div class='add'>+	mana_handle_t adapter;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_query_vf_cntrs_resp {</div><div class='add'>+	struct gdma_resp_hdr hdr;</div><div class='add'>+	u64 requester_timeout;</div><div class='add'>+	u64 requester_oos_nak;</div><div class='add'>+	u64 requester_rnr_nak;</div><div class='add'>+	u64 responder_rnr_nak;</div><div class='add'>+	u64 responder_oos;</div><div class='add'>+	u64 responder_dup_request;</div><div class='add'>+	u64 requester_implicit_nak;</div><div class='add'>+	u64 requester_readresp_psn_mismatch;</div><div class='add'>+	u64 nak_inv_req;</div><div class='add'>+	u64 nak_access_err;</div><div class='add'>+	u64 nak_opp_err;</div><div class='add'>+	u64 nak_inv_read;</div><div class='add'>+	u64 responder_local_len_err;</div><div class='add'>+	u64 requestor_local_prot_err;</div><div class='add'>+	u64 responder_rem_access_err;</div><div class='add'>+	u64 responder_local_qp_err;</div><div class='add'>+	u64 responder_malformed_wqe;</div><div class='add'>+	u64 general_hw_err;</div><div class='add'>+	u64 requester_rnr_nak_retries_exceeded;</div><div class='add'>+	u64 requester_retries_exceeded;</div><div class='add'>+	u64 total_fatal_err;</div><div class='add'>+	u64 received_cnps;</div><div class='add'>+	u64 num_qps_congested;</div><div class='add'>+	u64 rate_inc_events;</div><div class='add'>+	u64 num_qps_recovered;</div><div class='add'>+	u64 current_rate;</div><div class='add'>+	u64 dup_rx_req;</div><div class='add'>+	u64 tx_bytes;</div><div class='add'>+	u64 rx_bytes;</div><div class='add'>+	u64 rx_send_req;</div><div class='add'>+	u64 rx_write_req;</div><div class='add'>+	u64 rx_read_req;</div><div class='add'>+	u64 tx_pkt;</div><div class='add'>+	u64 rx_pkt;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_query_device_cntrs_req {</div><div class='add'>+	struct gdma_req_hdr hdr;</div><div class='add'>+	mana_handle_t adapter;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+struct mana_rnic_query_device_cntrs_resp {</div><div class='add'>+	struct gdma_resp_hdr hdr;</div><div class='add'>+	u32 sent_cnps;</div><div class='add'>+	u32 received_ecns;</div><div class='add'>+	u32 reserved1;</div><div class='add'>+	u32 received_cnp_count;</div><div class='add'>+	u32 qp_congested_events;</div><div class='add'>+	u32 qp_recovered_events;</div><div class='add'>+	u32 rate_inc_events;</div><div class='add'>+	u32 reserved2;</div><div class='add'>+}; /* HW Data */</div><div class='add'>+</div><div class='add'>+static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)</div><div class='add'>+{</div><div class='add'>+	return mdev-&gt;gdma_dev-&gt;gdma_context;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct mana_ib_qp *mana_get_qp_ref(struct mana_ib_dev *mdev,</div><div class='add'>+						 u32 qid, bool is_sq)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_qp *qp;</div><div class='add'>+	unsigned long flag;</div><div class='add'>+</div><div class='add'>+	if (is_sq)</div><div class='add'>+		qid |= MANA_SENDQ_MASK;</div><div class='add'>+</div><div class='add'>+	xa_lock_irqsave(&amp;mdev-&gt;qp_table_wq, flag);</div><div class='add'>+	qp = xa_load(&amp;mdev-&gt;qp_table_wq, qid);</div><div class='add'>+	if (qp)</div><div class='add'>+		refcount_inc(&amp;qp-&gt;refcount);</div><div class='add'>+	xa_unlock_irqrestore(&amp;mdev-&gt;qp_table_wq, flag);</div><div class='add'>+	return qp;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void mana_put_qp_ref(struct mana_ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	if (refcount_dec_and_test(&amp;qp-&gt;refcount))</div><div class='add'>+		complete(&amp;qp-&gt;free);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool mana_ib_is_rnic(struct mana_ib_dev *mdev)</div><div class='add'>+{</div><div class='add'>+	return mdev-&gt;gdma_dev-&gt;dev_id.type == GDMA_DEVICE_MANA_IB;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32 port)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(mdev);</div><div class='add'>+	struct mana_context *mc = gc-&gt;mana.driver_data;</div><div class='add'>+</div><div class='add'>+	if (port &lt; 1 || port &gt; mc-&gt;num_ports)</div><div class='add'>+		return NULL;</div><div class='add'>+	return mc-&gt;ports[port - 1];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void copy_in_reverse(u8 *dst, const u8 *src, u32 size)</div><div class='add'>+{</div><div class='add'>+	u32 i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; size; i++)</div><div class='add'>+		dst[size - 1 - i] = src[i];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);</div><div class='add'>+void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);</div><div class='add'>+</div><div class='add'>+int mana_ib_create_zero_offset_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,</div><div class='add'>+					  mana_handle_t *gdma_region);</div><div class='add'>+</div><div class='add'>+int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,</div><div class='add'>+			      mana_handle_t *gdma_region, u64 virt);</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev,</div><div class='add'>+				  mana_handle_t gdma_region);</div><div class='add'>+</div><div class='add'>+int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_queue_type type,</div><div class='add'>+				struct mana_ib_queue *queue);</div><div class='add'>+int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,</div><div class='add'>+			 struct mana_ib_queue *queue);</div><div class='add'>+void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue);</div><div class='add'>+</div><div class='add'>+struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,</div><div class='add'>+				struct ib_wq_init_attr *init_attr,</div><div class='add'>+				struct ib_udata *udata);</div><div class='add'>+</div><div class='add'>+int mana_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,</div><div class='add'>+		      u32 wq_attr_mask, struct ib_udata *udata);</div><div class='add'>+</div><div class='add'>+int mana_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata);</div><div class='add'>+</div><div class='add'>+int mana_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,</div><div class='add'>+				 struct ib_rwq_ind_table_init_attr *init_attr,</div><div class='add'>+				 struct ib_udata *udata);</div><div class='add'>+</div><div class='add'>+int mana_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl);</div><div class='add'>+</div><div class='add'>+struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags);</div><div class='add'>+</div><div class='add'>+struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='add'>+				  u64 iova, int access_flags,</div><div class='add'>+				  struct ib_dmah *dmah,</div><div class='add'>+				  struct ib_udata *udata);</div><div class='add'>+</div><div class='add'>+int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);</div><div class='add'>+</div><div class='add'>+int mana_ib_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr,</div><div class='add'>+		      struct ib_udata *udata);</div><div class='add'>+</div><div class='add'>+int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='add'>+		      int attr_mask, struct ib_udata *udata);</div><div class='add'>+</div><div class='add'>+int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);</div><div class='add'>+</div><div class='add'>+int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port_id,</div><div class='add'>+		      struct mana_ib_pd *pd, u32 doorbell_id);</div><div class='add'>+void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,</div><div class='add'>+			 u32 port);</div><div class='add'>+</div><div class='add'>+int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		      struct uverbs_attr_bundle *attrs);</div><div class='add'>+</div><div class='add'>+int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);</div><div class='add'>+</div><div class='add'>+int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);</div><div class='add'>+int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);</div><div class='add'>+</div><div class='add'>+int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,</div><div class='add'>+			   struct ib_udata *udata);</div><div class='add'>+void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext);</div><div class='add'>+</div><div class='add'>+int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma);</div><div class='add'>+</div><div class='add'>+int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+			       struct ib_port_immutable *immutable);</div><div class='add'>+int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props,</div><div class='add'>+			 struct ib_udata *uhw);</div><div class='add'>+int mana_ib_query_port(struct ib_device *ibdev, u32 port,</div><div class='add'>+		       struct ib_port_attr *props);</div><div class='add'>+int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,</div><div class='add'>+		      union ib_gid *gid);</div><div class='add'>+</div><div class='add'>+void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext);</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *mdev);</div><div class='add'>+int mana_eth_query_adapter_caps(struct mana_ib_dev *mdev);</div><div class='add'>+</div><div class='add'>+int mana_ib_create_eqs(struct mana_ib_dev *mdev);</div><div class='add'>+</div><div class='add'>+void mana_ib_destroy_eqs(struct mana_ib_dev *mdev);</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev);</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev);</div><div class='add'>+</div><div class='add'>+int mana_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey);</div><div class='add'>+</div><div class='add'>+enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_num);</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, void **context);</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context);</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8 *mac);</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq, u32 doorbell);</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,</div><div class='add'>+			    struct ib_qp_init_attr *attr, u32 doorbell, u64 flags);</div><div class='add'>+int mana_ib_gd_destroy_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp);</div><div class='add'>+</div><div class='add'>+int mana_ib_gd_create_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,</div><div class='add'>+			    struct ib_qp_init_attr *attr, u32 doorbell, u32 type);</div><div class='add'>+int mana_ib_gd_destroy_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp);</div><div class='add'>+</div><div class='add'>+int mana_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,</div><div class='add'>+		      struct ib_udata *udata);</div><div class='add'>+int mana_ib_destroy_ah(struct ib_ah *ah, u32 flags);</div><div class='add'>+</div><div class='add'>+int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,</div><div class='add'>+		      const struct ib_recv_wr **bad_wr);</div><div class='add'>+int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,</div><div class='add'>+		      const struct ib_send_wr **bad_wr);</div><div class='add'>+</div><div class='add'>+void mana_drain_gsi_sqs(struct mana_ib_dev *mdev);</div><div class='add'>+int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);</div><div class='add'>+int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);</div><div class='add'>+</div><div class='add'>+struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 length,</div><div class='add'>+					 u64 iova, int fd, int mr_access_flags,</div><div class='add'>+					 struct ib_dmah *dmah,</div><div class='add'>+					 struct uverbs_attr_bundle *attrs);</div><div class='add'>+#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/mana/mr.c b/drivers/infiniband/hw/mana/mr.c<br/>new file mode 100644<br/>index 000000000000..3d0245a4c1ed<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mana/mr.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mana/mr.c</a></div><div class='hunk'>@@ -0,0 +1,319 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2022, Microsoft Corporation. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include "mana_ib.h"</div><div class='add'>+</div><div class='add'>+#define VALID_MR_FLAGS (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |\</div><div class='add'>+			IB_ACCESS_REMOTE_ATOMIC | IB_ZERO_BASED)</div><div class='add'>+</div><div class='add'>+#define VALID_DMA_MR_FLAGS (IB_ACCESS_LOCAL_WRITE)</div><div class='add'>+</div><div class='add'>+static enum gdma_mr_access_flags</div><div class='add'>+mana_ib_verbs_to_gdma_access_flags(int access_flags)</div><div class='add'>+{</div><div class='add'>+	enum gdma_mr_access_flags flags = GDMA_ACCESS_FLAG_LOCAL_READ;</div><div class='add'>+</div><div class='add'>+	if (access_flags &amp; IB_ACCESS_LOCAL_WRITE)</div><div class='add'>+		flags |= GDMA_ACCESS_FLAG_LOCAL_WRITE;</div><div class='add'>+</div><div class='add'>+	if (access_flags &amp; IB_ACCESS_REMOTE_WRITE)</div><div class='add'>+		flags |= GDMA_ACCESS_FLAG_REMOTE_WRITE;</div><div class='add'>+</div><div class='add'>+	if (access_flags &amp; IB_ACCESS_REMOTE_READ)</div><div class='add'>+		flags |= GDMA_ACCESS_FLAG_REMOTE_READ;</div><div class='add'>+</div><div class='add'>+	if (access_flags &amp; IB_ACCESS_REMOTE_ATOMIC)</div><div class='add'>+		flags |= GDMA_ACCESS_FLAG_REMOTE_ATOMIC;</div><div class='add'>+</div><div class='add'>+	return flags;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_ib_gd_create_mr(struct mana_ib_dev *dev, struct mana_ib_mr *mr,</div><div class='add'>+				struct gdma_create_mr_params *mr_params)</div><div class='add'>+{</div><div class='add'>+	struct gdma_create_mr_response resp = {};</div><div class='add'>+	struct gdma_create_mr_request req = {};</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(dev);</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, GDMA_CREATE_MR, sizeof(req),</div><div class='add'>+			     sizeof(resp));</div><div class='add'>+	req.pd_handle = mr_params-&gt;pd_handle;</div><div class='add'>+	req.mr_type = mr_params-&gt;mr_type;</div><div class='add'>+</div><div class='add'>+	switch (mr_params-&gt;mr_type) {</div><div class='add'>+	case GDMA_MR_TYPE_GPA:</div><div class='add'>+		break;</div><div class='add'>+	case GDMA_MR_TYPE_GVA:</div><div class='add'>+		req.gva.dma_region_handle = mr_params-&gt;gva.dma_region_handle;</div><div class='add'>+		req.gva.virtual_address = mr_params-&gt;gva.virtual_address;</div><div class='add'>+		req.gva.access_flags = mr_params-&gt;gva.access_flags;</div><div class='add'>+		break;</div><div class='add'>+	case GDMA_MR_TYPE_ZBVA:</div><div class='add'>+		req.zbva.dma_region_handle = mr_params-&gt;zbva.dma_region_handle;</div><div class='add'>+		req.zbva.access_flags = mr_params-&gt;zbva.access_flags;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ib_dev,</div><div class='add'>+			  "invalid param (GDMA_MR_TYPE) passed, type %d\n",</div><div class='add'>+			  req.mr_type);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+</div><div class='add'>+	if (err || resp.hdr.status) {</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ib_dev, "Failed to create mr %d, %u", err,</div><div class='add'>+			  resp.hdr.status);</div><div class='add'>+		if (!err)</div><div class='add'>+			err = -EPROTO;</div><div class='add'>+</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mr-&gt;ibmr.lkey = resp.lkey;</div><div class='add'>+	mr-&gt;ibmr.rkey = resp.rkey;</div><div class='add'>+	mr-&gt;mr_handle = resp.mr_handle;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)</div><div class='add'>+{</div><div class='add'>+	struct gdma_destroy_mr_response resp = {};</div><div class='add'>+	struct gdma_destroy_mr_request req = {};</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(dev);</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, GDMA_DESTROY_MR, sizeof(req),</div><div class='add'>+			     sizeof(resp));</div><div class='add'>+</div><div class='add'>+	req.mr_handle = mr_handle;</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+	if (err || resp.hdr.status) {</div><div class='add'>+		dev_err(gc-&gt;dev, "Failed to destroy MR: %d, 0x%x\n", err,</div><div class='add'>+			resp.hdr.status);</div><div class='add'>+		if (!err)</div><div class='add'>+			err = -EPROTO;</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,</div><div class='add'>+				  u64 iova, int access_flags,</div><div class='add'>+				  struct ib_dmah *dmah,</div><div class='add'>+				  struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);</div><div class='add'>+	struct gdma_create_mr_params mr_params = {};</div><div class='add'>+	struct ib_device *ibdev = ibpd-&gt;device;</div><div class='add'>+	struct mana_ib_dev *dev;</div><div class='add'>+	struct mana_ib_mr *mr;</div><div class='add'>+	u64 dma_region_handle;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (dmah)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(ibdev,</div><div class='add'>+		  "start 0x%llx, iova 0x%llx length 0x%llx access_flags 0x%x",</div><div class='add'>+		  start, iova, length, access_flags);</div><div class='add'>+</div><div class='add'>+	access_flags &amp;= ~IB_ACCESS_OPTIONAL;</div><div class='add'>+	if (access_flags &amp; ~VALID_MR_FLAGS)</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='add'>+	if (!mr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	mr-&gt;umem = ib_umem_get(ibdev, start, length, access_flags);</div><div class='add'>+	if (IS_ERR(mr-&gt;umem)) {</div><div class='add'>+		err = PTR_ERR(mr-&gt;umem);</div><div class='add'>+		ibdev_dbg(ibdev,</div><div class='add'>+			  "Failed to get umem for register user-mr, %pe\n",</div><div class='add'>+			  mr-&gt;umem);</div><div class='add'>+		goto err_free;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = mana_ib_create_dma_region(dev, mr-&gt;umem, &amp;dma_region_handle, iova);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_dbg(ibdev, "Failed create dma region for user-mr, %d\n",</div><div class='add'>+			  err);</div><div class='add'>+		goto err_umem;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(ibdev,</div><div class='add'>+		  "created dma region for user-mr 0x%llx\n",</div><div class='add'>+		  dma_region_handle);</div><div class='add'>+</div><div class='add'>+	mr_params.pd_handle = pd-&gt;pd_handle;</div><div class='add'>+	if (access_flags &amp; IB_ZERO_BASED) {</div><div class='add'>+		mr_params.mr_type = GDMA_MR_TYPE_ZBVA;</div><div class='add'>+		mr_params.zbva.dma_region_handle = dma_region_handle;</div><div class='add'>+		mr_params.zbva.access_flags =</div><div class='add'>+			mana_ib_verbs_to_gdma_access_flags(access_flags);</div><div class='add'>+	} else {</div><div class='add'>+		mr_params.mr_type = GDMA_MR_TYPE_GVA;</div><div class='add'>+		mr_params.gva.dma_region_handle = dma_region_handle;</div><div class='add'>+		mr_params.gva.virtual_address = iova;</div><div class='add'>+		mr_params.gva.access_flags =</div><div class='add'>+			mana_ib_verbs_to_gdma_access_flags(access_flags);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = mana_ib_gd_create_mr(dev, mr, &amp;mr_params);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_dma_region;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * There is no need to keep track of dma_region_handle after MR is</div><div class='add'>+	 * successfully created. The dma_region_handle is tracked in the PF</div><div class='add'>+	 * as part of the lifecycle of this MR.</div><div class='add'>+	 */</div><div class='add'>+</div><div class='add'>+	return &amp;mr-&gt;ibmr;</div><div class='add'>+</div><div class='add'>+err_dma_region:</div><div class='add'>+	mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle);</div><div class='add'>+</div><div class='add'>+err_umem:</div><div class='add'>+	ib_umem_release(mr-&gt;umem);</div><div class='add'>+</div><div class='add'>+err_free:</div><div class='add'>+	kfree(mr);</div><div class='add'>+	return ERR_PTR(err);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 length,</div><div class='add'>+					 u64 iova, int fd, int access_flags,</div><div class='add'>+					 struct ib_dmah *dmah,</div><div class='add'>+					 struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);</div><div class='add'>+	struct gdma_create_mr_params mr_params = {};</div><div class='add'>+	struct ib_device *ibdev = ibpd-&gt;device;</div><div class='add'>+	struct ib_umem_dmabuf *umem_dmabuf;</div><div class='add'>+	struct mana_ib_dev *dev;</div><div class='add'>+	struct mana_ib_mr *mr;</div><div class='add'>+	u64 dma_region_handle;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (dmah)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);</div><div class='add'>+</div><div class='add'>+	access_flags &amp;= ~IB_ACCESS_OPTIONAL;</div><div class='add'>+	if (access_flags &amp; ~VALID_MR_FLAGS)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='add'>+	if (!mr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	umem_dmabuf = ib_umem_dmabuf_get_pinned(ibdev, start, length, fd, access_flags);</div><div class='add'>+	if (IS_ERR(umem_dmabuf)) {</div><div class='add'>+		err = PTR_ERR(umem_dmabuf);</div><div class='add'>+		ibdev_dbg(ibdev, "Failed to get dmabuf umem, %pe\n",</div><div class='add'>+			  umem_dmabuf);</div><div class='add'>+		goto err_free;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mr-&gt;umem = &amp;umem_dmabuf-&gt;umem;</div><div class='add'>+</div><div class='add'>+	err = mana_ib_create_dma_region(dev, mr-&gt;umem, &amp;dma_region_handle, iova);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_dbg(ibdev, "Failed create dma region for user-mr, %d\n",</div><div class='add'>+			  err);</div><div class='add'>+		goto err_umem;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mr_params.pd_handle = pd-&gt;pd_handle;</div><div class='add'>+	mr_params.mr_type = GDMA_MR_TYPE_GVA;</div><div class='add'>+	mr_params.gva.dma_region_handle = dma_region_handle;</div><div class='add'>+	mr_params.gva.virtual_address = iova;</div><div class='add'>+	mr_params.gva.access_flags =</div><div class='add'>+		mana_ib_verbs_to_gdma_access_flags(access_flags);</div><div class='add'>+</div><div class='add'>+	err = mana_ib_gd_create_mr(dev, mr, &amp;mr_params);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_dma_region;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * There is no need to keep track of dma_region_handle after MR is</div><div class='add'>+	 * successfully created. The dma_region_handle is tracked in the PF</div><div class='add'>+	 * as part of the lifecycle of this MR.</div><div class='add'>+	 */</div><div class='add'>+</div><div class='add'>+	return &amp;mr-&gt;ibmr;</div><div class='add'>+</div><div class='add'>+err_dma_region:</div><div class='add'>+	mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle);</div><div class='add'>+</div><div class='add'>+err_umem:</div><div class='add'>+	ib_umem_release(mr-&gt;umem);</div><div class='add'>+</div><div class='add'>+err_free:</div><div class='add'>+	kfree(mr);</div><div class='add'>+	return ERR_PTR(err);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);</div><div class='add'>+	struct gdma_create_mr_params mr_params = {};</div><div class='add'>+	struct ib_device *ibdev = ibpd-&gt;device;</div><div class='add'>+	struct mana_ib_dev *dev;</div><div class='add'>+	struct mana_ib_mr *mr;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);</div><div class='add'>+</div><div class='add'>+	if (access_flags &amp; ~VALID_DMA_MR_FLAGS)</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='add'>+	if (!mr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	mr_params.pd_handle = pd-&gt;pd_handle;</div><div class='add'>+	mr_params.mr_type = GDMA_MR_TYPE_GPA;</div><div class='add'>+</div><div class='add'>+	err = mana_ib_gd_create_mr(dev, mr, &amp;mr_params);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_free;</div><div class='add'>+</div><div class='add'>+	return &amp;mr-&gt;ibmr;</div><div class='add'>+</div><div class='add'>+err_free:</div><div class='add'>+	kfree(mr);</div><div class='add'>+	return ERR_PTR(err);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_mr *mr = container_of(ibmr, struct mana_ib_mr, ibmr);</div><div class='add'>+	struct ib_device *ibdev = ibmr-&gt;device;</div><div class='add'>+	struct mana_ib_dev *dev;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);</div><div class='add'>+</div><div class='add'>+	err = mana_ib_gd_destroy_mr(dev, mr-&gt;mr_handle);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	if (mr-&gt;umem)</div><div class='add'>+		ib_umem_release(mr-&gt;umem);</div><div class='add'>+</div><div class='add'>+	kfree(mr);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c<br/>new file mode 100644<br/>index 000000000000..48c1f4977f21<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mana/qp.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mana/qp.c</a></div><div class='hunk'>@@ -0,0 +1,921 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2022, Microsoft Corporation. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include "mana_ib.h"</div><div class='add'>+</div><div class='add'>+static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,</div><div class='add'>+				      struct net_device *ndev,</div><div class='add'>+				      mana_handle_t default_rxobj,</div><div class='add'>+				      mana_handle_t ind_table[],</div><div class='add'>+				      u32 log_ind_tbl_size, u32 rx_hash_key_len,</div><div class='add'>+				      u8 *rx_hash_key)</div><div class='add'>+{</div><div class='add'>+	struct mana_port_context *mpc = netdev_priv(ndev);</div><div class='add'>+	struct mana_cfg_rx_steer_req_v2 *req;</div><div class='add'>+	struct mana_cfg_rx_steer_resp resp = {};</div><div class='add'>+	struct gdma_context *gc;</div><div class='add'>+	u32 req_buf_size;</div><div class='add'>+	int i, err;</div><div class='add'>+</div><div class='add'>+	gc = mdev_to_gc(dev);</div><div class='add'>+</div><div class='add'>+	req_buf_size = struct_size(req, indir_tab, MANA_INDIRECT_TABLE_DEF_SIZE);</div><div class='add'>+	req = kzalloc(req_buf_size, GFP_KERNEL);</div><div class='add'>+	if (!req)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req-&gt;hdr, MANA_CONFIG_VPORT_RX, req_buf_size,</div><div class='add'>+			     sizeof(resp));</div><div class='add'>+</div><div class='add'>+	req-&gt;hdr.req.msg_version = GDMA_MESSAGE_V2;</div><div class='add'>+</div><div class='add'>+	req-&gt;vport = mpc-&gt;port_handle;</div><div class='add'>+	req-&gt;rx_enable = 1;</div><div class='add'>+	req-&gt;update_default_rxobj = 1;</div><div class='add'>+	req-&gt;default_rxobj = default_rxobj;</div><div class='add'>+	req-&gt;hdr.dev_id = gc-&gt;mana.dev_id;</div><div class='add'>+</div><div class='add'>+	/* If there are more than 1 entries in indirection table, enable RSS */</div><div class='add'>+	if (log_ind_tbl_size)</div><div class='add'>+		req-&gt;rss_enable = true;</div><div class='add'>+</div><div class='add'>+	req-&gt;num_indir_entries = MANA_INDIRECT_TABLE_DEF_SIZE;</div><div class='add'>+	req-&gt;indir_tab_offset = offsetof(struct mana_cfg_rx_steer_req_v2,</div><div class='add'>+					 indir_tab);</div><div class='add'>+	req-&gt;update_indir_tab = true;</div><div class='add'>+	req-&gt;cqe_coalescing_enable = 1;</div><div class='add'>+</div><div class='add'>+	/* The ind table passed to the hardware must have</div><div class='add'>+	 * MANA_INDIRECT_TABLE_DEF_SIZE entries. Adjust the verb</div><div class='add'>+	 * ind_table to MANA_INDIRECT_TABLE_SIZE if required</div><div class='add'>+	 */</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ib_dev, "ind table size %u\n", 1 &lt;&lt; log_ind_tbl_size);</div><div class='add'>+	for (i = 0; i &lt; MANA_INDIRECT_TABLE_DEF_SIZE; i++) {</div><div class='add'>+		req-&gt;indir_tab[i] = ind_table[i % (1 &lt;&lt; log_ind_tbl_size)];</div><div class='add'>+		ibdev_dbg(&amp;dev-&gt;ib_dev, "index %u handle 0x%llx\n", i,</div><div class='add'>+			  req-&gt;indir_tab[i]);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	req-&gt;update_hashkey = true;</div><div class='add'>+	if (rx_hash_key_len)</div><div class='add'>+		memcpy(req-&gt;hashkey, rx_hash_key, rx_hash_key_len);</div><div class='add'>+	else</div><div class='add'>+		netdev_rss_key_fill(req-&gt;hashkey, MANA_HASH_KEY_SIZE);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;dev-&gt;ib_dev, "vport handle %llu default_rxobj 0x%llx\n",</div><div class='add'>+		  req-&gt;vport, default_rxobj);</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(gc, req_buf_size, req, sizeof(resp), &amp;resp);</div><div class='add'>+	if (err) {</div><div class='add'>+		netdev_err(ndev, "Failed to configure vPort RX: %d\n", err);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (resp.hdr.status) {</div><div class='add'>+		netdev_err(ndev, "vPort RX configuration failed: 0x%x\n",</div><div class='add'>+			   resp.hdr.status);</div><div class='add'>+		err = -EPROTO;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	netdev_info(ndev, "Configured steering vPort %llu log_entries %u\n",</div><div class='add'>+		    mpc-&gt;port_handle, log_ind_tbl_size);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	kfree(req);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,</div><div class='add'>+				 struct ib_qp_init_attr *attr,</div><div class='add'>+				 struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);</div><div class='add'>+	struct mana_ib_dev *mdev =</div><div class='add'>+		container_of(pd-&gt;device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct ib_rwq_ind_table *ind_tbl = attr-&gt;rwq_ind_tbl;</div><div class='add'>+	struct mana_ib_create_qp_rss_resp resp = {};</div><div class='add'>+	struct mana_ib_create_qp_rss ucmd = {};</div><div class='add'>+	mana_handle_t *mana_ind_table;</div><div class='add'>+	struct mana_port_context *mpc;</div><div class='add'>+	unsigned int ind_tbl_size;</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+	struct mana_ib_cq *cq;</div><div class='add'>+	struct mana_ib_wq *wq;</div><div class='add'>+	struct mana_eq *eq;</div><div class='add'>+	struct ib_cq *ibcq;</div><div class='add'>+	struct ib_wq *ibwq;</div><div class='add'>+	int i = 0;</div><div class='add'>+	u32 port;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!udata || udata-&gt;inlen &lt; sizeof(ucmd))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ret = ib_copy_from_udata(&amp;ucmd, udata, min(sizeof(ucmd), udata-&gt;inlen));</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev,</div><div class='add'>+			  "Failed copy from udata for create rss-qp, err %d\n",</div><div class='add'>+			  ret);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;cap.max_recv_wr &gt; mdev-&gt;adapter_caps.max_qp_wr) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev,</div><div class='add'>+			  "Requested max_recv_wr %d exceeding limit\n",</div><div class='add'>+			  attr-&gt;cap.max_recv_wr);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;cap.max_recv_sge &gt; MAX_RX_WQE_SGL_ENTRIES) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev,</div><div class='add'>+			  "Requested max_recv_sge %d exceeding limit\n",</div><div class='add'>+			  attr-&gt;cap.max_recv_sge);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ind_tbl_size = 1 &lt;&lt; ind_tbl-&gt;log_ind_tbl_size;</div><div class='add'>+	if (ind_tbl_size &gt; MANA_INDIRECT_TABLE_DEF_SIZE) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev,</div><div class='add'>+			  "Indirect table size %d exceeding limit\n",</div><div class='add'>+			  ind_tbl_size);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ucmd.rx_hash_function != MANA_IB_RX_HASH_FUNC_TOEPLITZ) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev,</div><div class='add'>+			  "RX Hash function is not supported, %d\n",</div><div class='add'>+			  ucmd.rx_hash_function);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* IB ports start with 1, MANA start with 0 */</div><div class='add'>+	port = ucmd.port;</div><div class='add'>+	ndev = mana_ib_get_netdev(pd-&gt;device, port);</div><div class='add'>+	if (!ndev) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev, "Invalid port %u in creating qp\n",</div><div class='add'>+			  port);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+	mpc = netdev_priv(ndev);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;mdev-&gt;ib_dev, "rx_hash_function %d port %d\n",</div><div class='add'>+		  ucmd.rx_hash_function, port);</div><div class='add'>+</div><div class='add'>+	mana_ind_table = kcalloc(ind_tbl_size, sizeof(mana_handle_t),</div><div class='add'>+				 GFP_KERNEL);</div><div class='add'>+	if (!mana_ind_table) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto fail;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	qp-&gt;port = port;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ind_tbl_size; i++) {</div><div class='add'>+		struct mana_obj_spec wq_spec = {};</div><div class='add'>+		struct mana_obj_spec cq_spec = {};</div><div class='add'>+</div><div class='add'>+		ibwq = ind_tbl-&gt;ind_tbl[i];</div><div class='add'>+		wq = container_of(ibwq, struct mana_ib_wq, ibwq);</div><div class='add'>+</div><div class='add'>+		ibcq = ibwq-&gt;cq;</div><div class='add'>+		cq = container_of(ibcq, struct mana_ib_cq, ibcq);</div><div class='add'>+</div><div class='add'>+		wq_spec.gdma_region = wq-&gt;queue.gdma_region;</div><div class='add'>+		wq_spec.queue_size = wq-&gt;wq_buf_size;</div><div class='add'>+</div><div class='add'>+		cq_spec.gdma_region = cq-&gt;queue.gdma_region;</div><div class='add'>+		cq_spec.queue_size = cq-&gt;cqe * COMP_ENTRY_SIZE;</div><div class='add'>+		cq_spec.modr_ctx_id = 0;</div><div class='add'>+		eq = &amp;mpc-&gt;ac-&gt;eqs[cq-&gt;comp_vector];</div><div class='add'>+		cq_spec.attached_eq = eq-&gt;eq-&gt;id;</div><div class='add'>+</div><div class='add'>+		ret = mana_create_wq_obj(mpc, mpc-&gt;port_handle, GDMA_RQ,</div><div class='add'>+					 &amp;wq_spec, &amp;cq_spec, &amp;wq-&gt;rx_object);</div><div class='add'>+		if (ret) {</div><div class='add'>+			/* Do cleanup starting with index i-1 */</div><div class='add'>+			i--;</div><div class='add'>+			goto fail;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		/* The GDMA regions are now owned by the WQ object */</div><div class='add'>+		wq-&gt;queue.gdma_region = GDMA_INVALID_DMA_REGION;</div><div class='add'>+		cq-&gt;queue.gdma_region = GDMA_INVALID_DMA_REGION;</div><div class='add'>+</div><div class='add'>+		wq-&gt;queue.id = wq_spec.queue_index;</div><div class='add'>+		cq-&gt;queue.id = cq_spec.queue_index;</div><div class='add'>+</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev,</div><div class='add'>+			  "rx_object 0x%llx wq id %llu cq id %llu\n",</div><div class='add'>+			  wq-&gt;rx_object, wq-&gt;queue.id, cq-&gt;queue.id);</div><div class='add'>+</div><div class='add'>+		resp.entries[i].cqid = cq-&gt;queue.id;</div><div class='add'>+		resp.entries[i].wqid = wq-&gt;queue.id;</div><div class='add'>+</div><div class='add'>+		mana_ind_table[i] = wq-&gt;rx_object;</div><div class='add'>+</div><div class='add'>+		/* Create CQ table entry */</div><div class='add'>+		ret = mana_ib_install_cq_cb(mdev, cq);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto fail;</div><div class='add'>+	}</div><div class='add'>+	resp.num_entries = i;</div><div class='add'>+</div><div class='add'>+	ret = mana_ib_cfg_vport_steering(mdev, ndev, wq-&gt;rx_object,</div><div class='add'>+					 mana_ind_table,</div><div class='add'>+					 ind_tbl-&gt;log_ind_tbl_size,</div><div class='add'>+					 ucmd.rx_hash_key_len,</div><div class='add'>+					 ucmd.rx_hash_key);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto fail;</div><div class='add'>+</div><div class='add'>+	ret = ib_copy_to_udata(udata, &amp;resp, sizeof(resp));</div><div class='add'>+	if (ret) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev,</div><div class='add'>+			  "Failed to copy to udata create rss-qp, %d\n",</div><div class='add'>+			  ret);</div><div class='add'>+		goto fail;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	kfree(mana_ind_table);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+fail:</div><div class='add'>+	while (i-- &gt; 0) {</div><div class='add'>+		ibwq = ind_tbl-&gt;ind_tbl[i];</div><div class='add'>+		ibcq = ibwq-&gt;cq;</div><div class='add'>+		wq = container_of(ibwq, struct mana_ib_wq, ibwq);</div><div class='add'>+		cq = container_of(ibcq, struct mana_ib_cq, ibcq);</div><div class='add'>+</div><div class='add'>+		mana_ib_remove_cq_cb(mdev, cq);</div><div class='add'>+		mana_destroy_wq_obj(mpc, GDMA_RQ, wq-&gt;rx_object);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	kfree(mana_ind_table);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,</div><div class='add'>+				 struct ib_qp_init_attr *attr,</div><div class='add'>+				 struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);</div><div class='add'>+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);</div><div class='add'>+	struct mana_ib_dev *mdev =</div><div class='add'>+		container_of(ibpd-&gt;device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct mana_ib_cq *send_cq =</div><div class='add'>+		container_of(attr-&gt;send_cq, struct mana_ib_cq, ibcq);</div><div class='add'>+	struct mana_ib_ucontext *mana_ucontext =</div><div class='add'>+		rdma_udata_to_drv_context(udata, struct mana_ib_ucontext,</div><div class='add'>+					  ibucontext);</div><div class='add'>+	struct mana_ib_create_qp_resp resp = {};</div><div class='add'>+	struct mana_ib_create_qp ucmd = {};</div><div class='add'>+	struct mana_obj_spec wq_spec = {};</div><div class='add'>+	struct mana_obj_spec cq_spec = {};</div><div class='add'>+	struct mana_port_context *mpc;</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+	struct mana_eq *eq;</div><div class='add'>+	int eq_vec;</div><div class='add'>+	u32 port;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (!mana_ucontext || udata-&gt;inlen &lt; sizeof(ucmd))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	err = ib_copy_from_udata(&amp;ucmd, udata, min(sizeof(ucmd), udata-&gt;inlen));</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev,</div><div class='add'>+			  "Failed to copy from udata create qp-raw, %d\n", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;cap.max_send_wr &gt; mdev-&gt;adapter_caps.max_qp_wr) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev,</div><div class='add'>+			  "Requested max_send_wr %d exceeding limit\n",</div><div class='add'>+			  attr-&gt;cap.max_send_wr);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;cap.max_send_sge &gt; MAX_TX_WQE_SGL_ENTRIES) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev,</div><div class='add'>+			  "Requested max_send_sge %d exceeding limit\n",</div><div class='add'>+			  attr-&gt;cap.max_send_sge);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	port = ucmd.port;</div><div class='add'>+	ndev = mana_ib_get_netdev(ibpd-&gt;device, port);</div><div class='add'>+	if (!ndev) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev, "Invalid port %u in creating qp\n",</div><div class='add'>+			  port);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+	mpc = netdev_priv(ndev);</div><div class='add'>+	ibdev_dbg(&amp;mdev-&gt;ib_dev, "port %u ndev %p mpc %p\n", port, ndev, mpc);</div><div class='add'>+</div><div class='add'>+	err = mana_ib_cfg_vport(mdev, port, pd, mana_ucontext-&gt;doorbell);</div><div class='add'>+	if (err)</div><div class='add'>+		return -ENODEV;</div><div class='add'>+</div><div class='add'>+	qp-&gt;port = port;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;mdev-&gt;ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n",</div><div class='add'>+		  ucmd.sq_buf_addr, ucmd.port);</div><div class='add'>+</div><div class='add'>+	err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &amp;qp-&gt;raw_sq);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev,</div><div class='add'>+			  "Failed to create queue for create qp-raw, err %d\n", err);</div><div class='add'>+		goto err_free_vport;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Create a WQ on the same port handle used by the Ethernet */</div><div class='add'>+	wq_spec.gdma_region = qp-&gt;raw_sq.gdma_region;</div><div class='add'>+	wq_spec.queue_size = ucmd.sq_buf_size;</div><div class='add'>+</div><div class='add'>+	cq_spec.gdma_region = send_cq-&gt;queue.gdma_region;</div><div class='add'>+	cq_spec.queue_size = send_cq-&gt;cqe * COMP_ENTRY_SIZE;</div><div class='add'>+	cq_spec.modr_ctx_id = 0;</div><div class='add'>+	eq_vec = send_cq-&gt;comp_vector;</div><div class='add'>+	eq = &amp;mpc-&gt;ac-&gt;eqs[eq_vec];</div><div class='add'>+	cq_spec.attached_eq = eq-&gt;eq-&gt;id;</div><div class='add'>+</div><div class='add'>+	err = mana_create_wq_obj(mpc, mpc-&gt;port_handle, GDMA_SQ, &amp;wq_spec,</div><div class='add'>+				 &amp;cq_spec, &amp;qp-&gt;qp_handle);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev,</div><div class='add'>+			  "Failed to create wq for create raw-qp, err %d\n",</div><div class='add'>+			  err);</div><div class='add'>+		goto err_destroy_queue;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* The GDMA regions are now owned by the WQ object */</div><div class='add'>+	qp-&gt;raw_sq.gdma_region = GDMA_INVALID_DMA_REGION;</div><div class='add'>+	send_cq-&gt;queue.gdma_region = GDMA_INVALID_DMA_REGION;</div><div class='add'>+</div><div class='add'>+	qp-&gt;raw_sq.id = wq_spec.queue_index;</div><div class='add'>+	send_cq-&gt;queue.id = cq_spec.queue_index;</div><div class='add'>+</div><div class='add'>+	/* Create CQ table entry */</div><div class='add'>+	err = mana_ib_install_cq_cb(mdev, send_cq);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_destroy_wq_obj;</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;mdev-&gt;ib_dev,</div><div class='add'>+		  "qp-&gt;qp_handle 0x%llx sq id %llu cq id %llu\n",</div><div class='add'>+		  qp-&gt;qp_handle, qp-&gt;raw_sq.id, send_cq-&gt;queue.id);</div><div class='add'>+</div><div class='add'>+	resp.sqid = qp-&gt;raw_sq.id;</div><div class='add'>+	resp.cqid = send_cq-&gt;queue.id;</div><div class='add'>+	resp.tx_vp_offset = pd-&gt;tx_vp_offset;</div><div class='add'>+</div><div class='add'>+	err = ib_copy_to_udata(udata, &amp;resp, sizeof(resp));</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev,</div><div class='add'>+			  "Failed copy udata for create qp-raw, %d\n",</div><div class='add'>+			  err);</div><div class='add'>+		goto err_remove_cq_cb;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_remove_cq_cb:</div><div class='add'>+	mana_ib_remove_cq_cb(mdev, send_cq);</div><div class='add'>+</div><div class='add'>+err_destroy_wq_obj:</div><div class='add'>+	mana_destroy_wq_obj(mpc, GDMA_SQ, qp-&gt;qp_handle);</div><div class='add'>+</div><div class='add'>+err_destroy_queue:</div><div class='add'>+	mana_ib_destroy_queue(mdev, &amp;qp-&gt;raw_sq);</div><div class='add'>+</div><div class='add'>+err_free_vport:</div><div class='add'>+	mana_ib_uncfg_vport(mdev, pd, port);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u32 mana_ib_wqe_size(u32 sge, u32 oob_size)</div><div class='add'>+{</div><div class='add'>+	u32 wqe_size = sge * sizeof(struct gdma_sge) + sizeof(struct gdma_wqe) + oob_size;</div><div class='add'>+</div><div class='add'>+	return ALIGN(wqe_size, GDMA_WQE_BU_SIZE);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u32 mana_ib_queue_size(struct ib_qp_init_attr *attr, u32 queue_type)</div><div class='add'>+{</div><div class='add'>+	u32 queue_size;</div><div class='add'>+</div><div class='add'>+	switch (attr-&gt;qp_type) {</div><div class='add'>+	case IB_QPT_UD:</div><div class='add'>+	case IB_QPT_GSI:</div><div class='add'>+		if (queue_type == MANA_UD_SEND_QUEUE)</div><div class='add'>+			queue_size = attr-&gt;cap.max_send_wr *</div><div class='add'>+				mana_ib_wqe_size(attr-&gt;cap.max_send_sge, INLINE_OOB_LARGE_SIZE);</div><div class='add'>+		else</div><div class='add'>+			queue_size = attr-&gt;cap.max_recv_wr *</div><div class='add'>+				mana_ib_wqe_size(attr-&gt;cap.max_recv_sge, INLINE_OOB_SMALL_SIZE);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return MANA_PAGE_ALIGN(roundup_pow_of_two(queue_size));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static enum gdma_queue_type mana_ib_queue_type(struct ib_qp_init_attr *attr, u32 queue_type)</div><div class='add'>+{</div><div class='add'>+	enum gdma_queue_type type;</div><div class='add'>+</div><div class='add'>+	switch (attr-&gt;qp_type) {</div><div class='add'>+	case IB_QPT_UD:</div><div class='add'>+	case IB_QPT_GSI:</div><div class='add'>+		if (queue_type == MANA_UD_SEND_QUEUE)</div><div class='add'>+			type = GDMA_SQ;</div><div class='add'>+		else</div><div class='add'>+			type = GDMA_RQ;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		type = GDMA_INVALID_QUEUE;</div><div class='add'>+	}</div><div class='add'>+	return type;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_table_store_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	return xa_insert_irq(&amp;mdev-&gt;qp_table_wq, qp-&gt;ibqp.qp_num, qp,</div><div class='add'>+			     GFP_KERNEL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mana_table_remove_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	xa_erase_irq(&amp;mdev-&gt;qp_table_wq, qp-&gt;ibqp.qp_num);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_table_store_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	u32 qids = qp-&gt;ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK;</div><div class='add'>+	u32 qidr = qp-&gt;ud_qp.queues[MANA_UD_RECV_QUEUE].id;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	err = xa_insert_irq(&amp;mdev-&gt;qp_table_wq, qids, qp, GFP_KERNEL);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = xa_insert_irq(&amp;mdev-&gt;qp_table_wq, qidr, qp, GFP_KERNEL);</div><div class='add'>+	if (err)</div><div class='add'>+		goto remove_sq;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+remove_sq:</div><div class='add'>+	xa_erase_irq(&amp;mdev-&gt;qp_table_wq, qids);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mana_table_remove_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	u32 qids = qp-&gt;ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK;</div><div class='add'>+	u32 qidr = qp-&gt;ud_qp.queues[MANA_UD_RECV_QUEUE].id;</div><div class='add'>+</div><div class='add'>+	xa_erase_irq(&amp;mdev-&gt;qp_table_wq, qids);</div><div class='add'>+	xa_erase_irq(&amp;mdev-&gt;qp_table_wq, qidr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_table_store_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	refcount_set(&amp;qp-&gt;refcount, 1);</div><div class='add'>+	init_completion(&amp;qp-&gt;free);</div><div class='add'>+</div><div class='add'>+	switch (qp-&gt;ibqp.qp_type) {</div><div class='add'>+	case IB_QPT_RC:</div><div class='add'>+		return mana_table_store_rc_qp(mdev, qp);</div><div class='add'>+	case IB_QPT_UD:</div><div class='add'>+	case IB_QPT_GSI:</div><div class='add'>+		return mana_table_store_ud_qp(mdev, qp);</div><div class='add'>+	default:</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev, "Unknown QP type for storing in mana table, %d\n",</div><div class='add'>+			  qp-&gt;ibqp.qp_type);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return -EINVAL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mana_table_remove_qp(struct mana_ib_dev *mdev,</div><div class='add'>+				 struct mana_ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	switch (qp-&gt;ibqp.qp_type) {</div><div class='add'>+	case IB_QPT_RC:</div><div class='add'>+		mana_table_remove_rc_qp(mdev, qp);</div><div class='add'>+		break;</div><div class='add'>+	case IB_QPT_UD:</div><div class='add'>+	case IB_QPT_GSI:</div><div class='add'>+		mana_table_remove_ud_qp(mdev, qp);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev, "Unknown QP type for removing from mana table, %d\n",</div><div class='add'>+			  qp-&gt;ibqp.qp_type);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+	mana_put_qp_ref(qp);</div><div class='add'>+	wait_for_completion(&amp;qp-&gt;free);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,</div><div class='add'>+				struct ib_qp_init_attr *attr, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev = container_of(ibpd-&gt;device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);</div><div class='add'>+	struct mana_ib_create_rc_qp_resp resp = {};</div><div class='add'>+	struct mana_ib_ucontext *mana_ucontext;</div><div class='add'>+	struct mana_ib_create_rc_qp ucmd = {};</div><div class='add'>+	int i, err, j;</div><div class='add'>+	u64 flags = 0;</div><div class='add'>+	u32 doorbell;</div><div class='add'>+</div><div class='add'>+	if (!udata || udata-&gt;inlen &lt; sizeof(ucmd))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, ibucontext);</div><div class='add'>+	doorbell = mana_ucontext-&gt;doorbell;</div><div class='add'>+	flags = MANA_RC_FLAG_NO_FMR;</div><div class='add'>+	err = ib_copy_from_udata(&amp;ucmd, udata, min(sizeof(ucmd), udata-&gt;inlen));</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev, "Failed to copy from udata, %d\n", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = 0, j = 0; i &lt; MANA_RC_QUEUE_TYPE_MAX; ++i) {</div><div class='add'>+		/* skip FMR for user-level RC QPs */</div><div class='add'>+		if (i == MANA_RC_SEND_QUEUE_FMR) {</div><div class='add'>+			qp-&gt;rc_qp.queues[i].id = INVALID_QUEUE_ID;</div><div class='add'>+			qp-&gt;rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+		err = mana_ib_create_queue(mdev, ucmd.queue_buf[j], ucmd.queue_size[j],</div><div class='add'>+					   &amp;qp-&gt;rc_qp.queues[i]);</div><div class='add'>+		if (err) {</div><div class='add'>+			ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to create queue %d, err %d\n", i, err);</div><div class='add'>+			goto destroy_queues;</div><div class='add'>+		}</div><div class='add'>+		j++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = mana_ib_gd_create_rc_qp(mdev, qp, attr, doorbell, flags);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to create rc qp  %d\n", err);</div><div class='add'>+		goto destroy_queues;</div><div class='add'>+	}</div><div class='add'>+	qp-&gt;ibqp.qp_num = qp-&gt;rc_qp.queues[MANA_RC_RECV_QUEUE_RESPONDER].id;</div><div class='add'>+	qp-&gt;port = attr-&gt;port_num;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		for (i = 0, j = 0; i &lt; MANA_RC_QUEUE_TYPE_MAX; ++i) {</div><div class='add'>+			if (i == MANA_RC_SEND_QUEUE_FMR)</div><div class='add'>+				continue;</div><div class='add'>+			resp.queue_id[j] = qp-&gt;rc_qp.queues[i].id;</div><div class='add'>+			j++;</div><div class='add'>+		}</div><div class='add'>+		err = ib_copy_to_udata(udata, &amp;resp, min(sizeof(resp), udata-&gt;outlen));</div><div class='add'>+		if (err) {</div><div class='add'>+			ibdev_dbg(&amp;mdev-&gt;ib_dev, "Failed to copy to udata, %d\n", err);</div><div class='add'>+			goto destroy_qp;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = mana_table_store_qp(mdev, qp);</div><div class='add'>+	if (err)</div><div class='add'>+		goto destroy_qp;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+destroy_qp:</div><div class='add'>+	mana_ib_gd_destroy_rc_qp(mdev, qp);</div><div class='add'>+destroy_queues:</div><div class='add'>+	while (i-- &gt; 0)</div><div class='add'>+		mana_ib_destroy_queue(mdev, &amp;qp-&gt;rc_qp.queues[i]);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mana_add_qp_to_cqs(struct mana_ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_cq *send_cq = container_of(qp-&gt;ibqp.send_cq, struct mana_ib_cq, ibcq);</div><div class='add'>+	struct mana_ib_cq *recv_cq = container_of(qp-&gt;ibqp.recv_cq, struct mana_ib_cq, ibcq);</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;send_cq-&gt;cq_lock, flags);</div><div class='add'>+	list_add_tail(&amp;qp-&gt;cq_send_list, &amp;send_cq-&gt;list_send_qp);</div><div class='add'>+	spin_unlock_irqrestore(&amp;send_cq-&gt;cq_lock, flags);</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;recv_cq-&gt;cq_lock, flags);</div><div class='add'>+	list_add_tail(&amp;qp-&gt;cq_recv_list, &amp;recv_cq-&gt;list_recv_qp);</div><div class='add'>+	spin_unlock_irqrestore(&amp;recv_cq-&gt;cq_lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_cq *send_cq = container_of(qp-&gt;ibqp.send_cq, struct mana_ib_cq, ibcq);</div><div class='add'>+	struct mana_ib_cq *recv_cq = container_of(qp-&gt;ibqp.recv_cq, struct mana_ib_cq, ibcq);</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;send_cq-&gt;cq_lock, flags);</div><div class='add'>+	list_del(&amp;qp-&gt;cq_send_list);</div><div class='add'>+	spin_unlock_irqrestore(&amp;send_cq-&gt;cq_lock, flags);</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;recv_cq-&gt;cq_lock, flags);</div><div class='add'>+	list_del(&amp;qp-&gt;cq_recv_list);</div><div class='add'>+	spin_unlock_irqrestore(&amp;recv_cq-&gt;cq_lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,</div><div class='add'>+				struct ib_qp_init_attr *attr, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev = container_of(ibpd-&gt;device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);</div><div class='add'>+	u32 doorbell, queue_size;</div><div class='add'>+	int i, err;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev, "User-level UD QPs are not supported\n");</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; MANA_UD_QUEUE_TYPE_MAX; ++i) {</div><div class='add'>+		queue_size = mana_ib_queue_size(attr, i);</div><div class='add'>+		err = mana_ib_create_kernel_queue(mdev, queue_size, mana_ib_queue_type(attr, i),</div><div class='add'>+						  &amp;qp-&gt;ud_qp.queues[i]);</div><div class='add'>+		if (err) {</div><div class='add'>+			ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to create queue %d, err %d\n",</div><div class='add'>+				  i, err);</div><div class='add'>+			goto destroy_queues;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	doorbell = mdev-&gt;gdma_dev-&gt;doorbell;</div><div class='add'>+</div><div class='add'>+	err = create_shadow_queue(&amp;qp-&gt;shadow_rq, attr-&gt;cap.max_recv_wr,</div><div class='add'>+				  sizeof(struct ud_rq_shadow_wqe));</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to create shadow rq err %d\n", err);</div><div class='add'>+		goto destroy_queues;</div><div class='add'>+	}</div><div class='add'>+	err = create_shadow_queue(&amp;qp-&gt;shadow_sq, attr-&gt;cap.max_send_wr,</div><div class='add'>+				  sizeof(struct ud_sq_shadow_wqe));</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to create shadow sq err %d\n", err);</div><div class='add'>+		goto destroy_shadow_queues;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = mana_ib_gd_create_ud_qp(mdev, qp, attr, doorbell, attr-&gt;qp_type);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed to create ud qp  %d\n", err);</div><div class='add'>+		goto destroy_shadow_queues;</div><div class='add'>+	}</div><div class='add'>+	qp-&gt;ibqp.qp_num = qp-&gt;ud_qp.queues[MANA_UD_RECV_QUEUE].id;</div><div class='add'>+	qp-&gt;port = attr-&gt;port_num;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; MANA_UD_QUEUE_TYPE_MAX; ++i)</div><div class='add'>+		qp-&gt;ud_qp.queues[i].kmem-&gt;id = qp-&gt;ud_qp.queues[i].id;</div><div class='add'>+</div><div class='add'>+	err = mana_table_store_qp(mdev, qp);</div><div class='add'>+	if (err)</div><div class='add'>+		goto destroy_qp;</div><div class='add'>+</div><div class='add'>+	mana_add_qp_to_cqs(qp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+destroy_qp:</div><div class='add'>+	mana_ib_gd_destroy_ud_qp(mdev, qp);</div><div class='add'>+destroy_shadow_queues:</div><div class='add'>+	destroy_shadow_queue(&amp;qp-&gt;shadow_rq);</div><div class='add'>+	destroy_shadow_queue(&amp;qp-&gt;shadow_sq);</div><div class='add'>+destroy_queues:</div><div class='add'>+	while (i-- &gt; 0)</div><div class='add'>+		mana_ib_destroy_queue(mdev, &amp;qp-&gt;ud_qp.queues[i]);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,</div><div class='add'>+		      struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	switch (attr-&gt;qp_type) {</div><div class='add'>+	case IB_QPT_RAW_PACKET:</div><div class='add'>+		/* When rwq_ind_tbl is used, it's for creating WQs for RSS */</div><div class='add'>+		if (attr-&gt;rwq_ind_tbl)</div><div class='add'>+			return mana_ib_create_qp_rss(ibqp, ibqp-&gt;pd, attr,</div><div class='add'>+						     udata);</div><div class='add'>+</div><div class='add'>+		return mana_ib_create_qp_raw(ibqp, ibqp-&gt;pd, attr, udata);</div><div class='add'>+	case IB_QPT_RC:</div><div class='add'>+		return mana_ib_create_rc_qp(ibqp, ibqp-&gt;pd, attr, udata);</div><div class='add'>+	case IB_QPT_UD:</div><div class='add'>+	case IB_QPT_GSI:</div><div class='add'>+		return mana_ib_create_ud_qp(ibqp, ibqp-&gt;pd, attr, udata);</div><div class='add'>+	default:</div><div class='add'>+		ibdev_dbg(ibqp-&gt;device, "Creating QP type %u not supported\n",</div><div class='add'>+			  attr-&gt;qp_type);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return -EINVAL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='add'>+				int attr_mask, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev = container_of(ibqp-&gt;device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);</div><div class='add'>+	struct mana_rnic_set_qp_state_resp resp = {};</div><div class='add'>+	struct mana_rnic_set_qp_state_req req = {};</div><div class='add'>+	struct gdma_context *gc = mdev_to_gc(mdev);</div><div class='add'>+	struct mana_port_context *mpc;</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mana_gd_init_req_hdr(&amp;req.hdr, MANA_IB_SET_QP_STATE, sizeof(req), sizeof(resp));</div><div class='add'>+</div><div class='add'>+	req.hdr.req.msg_version = GDMA_MESSAGE_V3;</div><div class='add'>+	req.hdr.dev_id = mdev-&gt;gdma_dev-&gt;dev_id;</div><div class='add'>+	req.adapter = mdev-&gt;adapter_handle;</div><div class='add'>+	req.qp_handle = qp-&gt;qp_handle;</div><div class='add'>+	req.qp_state = attr-&gt;qp_state;</div><div class='add'>+	req.attr_mask = attr_mask;</div><div class='add'>+	req.path_mtu = attr-&gt;path_mtu;</div><div class='add'>+	req.rq_psn = attr-&gt;rq_psn;</div><div class='add'>+	req.sq_psn = attr-&gt;sq_psn;</div><div class='add'>+	req.dest_qpn = attr-&gt;dest_qp_num;</div><div class='add'>+	req.max_dest_rd_atomic = attr-&gt;max_dest_rd_atomic;</div><div class='add'>+	req.retry_cnt = attr-&gt;retry_cnt;</div><div class='add'>+	req.rnr_retry = attr-&gt;rnr_retry;</div><div class='add'>+	req.min_rnr_timer = attr-&gt;min_rnr_timer;</div><div class='add'>+	req.rate_limit = attr-&gt;rate_limit;</div><div class='add'>+	req.qkey = attr-&gt;qkey;</div><div class='add'>+	req.local_ack_timeout = attr-&gt;timeout;</div><div class='add'>+	req.qp_access_flags = attr-&gt;qp_access_flags;</div><div class='add'>+	req.max_rd_atomic = attr-&gt;max_rd_atomic;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_AV) {</div><div class='add'>+		ndev = mana_ib_get_netdev(&amp;mdev-&gt;ib_dev, ibqp-&gt;port);</div><div class='add'>+		if (!ndev) {</div><div class='add'>+			ibdev_dbg(&amp;mdev-&gt;ib_dev, "Invalid port %u in QP %u\n",</div><div class='add'>+				  ibqp-&gt;port, ibqp-&gt;qp_num);</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		}</div><div class='add'>+		mpc = netdev_priv(ndev);</div><div class='add'>+		copy_in_reverse(req.ah_attr.src_mac, mpc-&gt;mac_addr, ETH_ALEN);</div><div class='add'>+		copy_in_reverse(req.ah_attr.dest_mac, attr-&gt;ah_attr.roce.dmac, ETH_ALEN);</div><div class='add'>+		copy_in_reverse(req.ah_attr.src_addr, attr-&gt;ah_attr.grh.sgid_attr-&gt;gid.raw,</div><div class='add'>+				sizeof(union ib_gid));</div><div class='add'>+		copy_in_reverse(req.ah_attr.dest_addr, attr-&gt;ah_attr.grh.dgid.raw,</div><div class='add'>+				sizeof(union ib_gid));</div><div class='add'>+		if (rdma_gid_attr_network_type(attr-&gt;ah_attr.grh.sgid_attr) == RDMA_NETWORK_IPV4) {</div><div class='add'>+			req.ah_attr.src_addr_type = SGID_TYPE_IPV4;</div><div class='add'>+			req.ah_attr.dest_addr_type = SGID_TYPE_IPV4;</div><div class='add'>+		} else {</div><div class='add'>+			req.ah_attr.src_addr_type = SGID_TYPE_IPV6;</div><div class='add'>+			req.ah_attr.dest_addr_type = SGID_TYPE_IPV6;</div><div class='add'>+		}</div><div class='add'>+		req.ah_attr.dest_port = ROCE_V2_UDP_DPORT;</div><div class='add'>+		req.ah_attr.src_port = rdma_get_udp_sport(attr-&gt;ah_attr.grh.flow_label,</div><div class='add'>+							  ibqp-&gt;qp_num, attr-&gt;dest_qp_num);</div><div class='add'>+		req.ah_attr.traffic_class = attr-&gt;ah_attr.grh.traffic_class &gt;&gt; 2;</div><div class='add'>+		req.ah_attr.hop_limit = attr-&gt;ah_attr.grh.hop_limit;</div><div class='add'>+		req.ah_attr.flow_label = attr-&gt;ah_attr.grh.flow_label;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = mana_gd_send_request(gc, sizeof(req), &amp;req, sizeof(resp), &amp;resp);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_err(&amp;mdev-&gt;ib_dev, "Failed modify qp err %d", err);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='add'>+		      int attr_mask, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	switch (ibqp-&gt;qp_type) {</div><div class='add'>+	case IB_QPT_RC:</div><div class='add'>+	case IB_QPT_UD:</div><div class='add'>+	case IB_QPT_GSI:</div><div class='add'>+		return mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata);</div><div class='add'>+	default:</div><div class='add'>+		ibdev_dbg(ibqp-&gt;device, "Modify QP type %u not supported", ibqp-&gt;qp_type);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp,</div><div class='add'>+				  struct ib_rwq_ind_table *ind_tbl,</div><div class='add'>+				  struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev =</div><div class='add'>+		container_of(qp-&gt;ibqp.device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct mana_port_context *mpc;</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+	struct mana_ib_wq *wq;</div><div class='add'>+	struct ib_wq *ibwq;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	ndev = mana_ib_get_netdev(qp-&gt;ibqp.device, qp-&gt;port);</div><div class='add'>+	mpc = netdev_priv(ndev);</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; (1 &lt;&lt; ind_tbl-&gt;log_ind_tbl_size); i++) {</div><div class='add'>+		ibwq = ind_tbl-&gt;ind_tbl[i];</div><div class='add'>+		wq = container_of(ibwq, struct mana_ib_wq, ibwq);</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev, "destroying wq-&gt;rx_object %llu\n",</div><div class='add'>+			  wq-&gt;rx_object);</div><div class='add'>+		mana_destroy_wq_obj(mpc, GDMA_RQ, wq-&gt;rx_object);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_ib_destroy_qp_raw(struct mana_ib_qp *qp, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev =</div><div class='add'>+		container_of(qp-&gt;ibqp.device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct ib_pd *ibpd = qp-&gt;ibqp.pd;</div><div class='add'>+	struct mana_port_context *mpc;</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+	struct mana_ib_pd *pd;</div><div class='add'>+</div><div class='add'>+	ndev = mana_ib_get_netdev(qp-&gt;ibqp.device, qp-&gt;port);</div><div class='add'>+	mpc = netdev_priv(ndev);</div><div class='add'>+	pd = container_of(ibpd, struct mana_ib_pd, ibpd);</div><div class='add'>+</div><div class='add'>+	mana_destroy_wq_obj(mpc, GDMA_SQ, qp-&gt;qp_handle);</div><div class='add'>+</div><div class='add'>+	mana_ib_destroy_queue(mdev, &amp;qp-&gt;raw_sq);</div><div class='add'>+</div><div class='add'>+	mana_ib_uncfg_vport(mdev, pd, qp-&gt;port);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_ib_destroy_rc_qp(struct mana_ib_qp *qp, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev =</div><div class='add'>+		container_of(qp-&gt;ibqp.device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	mana_table_remove_qp(mdev, qp);</div><div class='add'>+</div><div class='add'>+	/* Ignore return code as there is not much we can do about it.</div><div class='add'>+	 * The error message is printed inside.</div><div class='add'>+	 */</div><div class='add'>+	mana_ib_gd_destroy_rc_qp(mdev, qp);</div><div class='add'>+	for (i = 0; i &lt; MANA_RC_QUEUE_TYPE_MAX; ++i)</div><div class='add'>+		mana_ib_destroy_queue(mdev, &amp;qp-&gt;rc_qp.queues[i]);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_ib_destroy_ud_qp(struct mana_ib_qp *qp, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev =</div><div class='add'>+		container_of(qp-&gt;ibqp.device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	mana_remove_qp_from_cqs(qp);</div><div class='add'>+	mana_table_remove_qp(mdev, qp);</div><div class='add'>+</div><div class='add'>+	destroy_shadow_queue(&amp;qp-&gt;shadow_rq);</div><div class='add'>+	destroy_shadow_queue(&amp;qp-&gt;shadow_sq);</div><div class='add'>+</div><div class='add'>+	/* Ignore return code as there is not much we can do about it.</div><div class='add'>+	 * The error message is printed inside.</div><div class='add'>+	 */</div><div class='add'>+	mana_ib_gd_destroy_ud_qp(mdev, qp);</div><div class='add'>+	for (i = 0; i &lt; MANA_UD_QUEUE_TYPE_MAX; ++i)</div><div class='add'>+		mana_ib_destroy_queue(mdev, &amp;qp-&gt;ud_qp.queues[i]);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);</div><div class='add'>+</div><div class='add'>+	switch (ibqp-&gt;qp_type) {</div><div class='add'>+	case IB_QPT_RAW_PACKET:</div><div class='add'>+		if (ibqp-&gt;rwq_ind_tbl)</div><div class='add'>+			return mana_ib_destroy_qp_rss(qp, ibqp-&gt;rwq_ind_tbl,</div><div class='add'>+						      udata);</div><div class='add'>+</div><div class='add'>+		return mana_ib_destroy_qp_raw(qp, udata);</div><div class='add'>+	case IB_QPT_RC:</div><div class='add'>+		return mana_ib_destroy_rc_qp(qp, udata);</div><div class='add'>+	case IB_QPT_UD:</div><div class='add'>+	case IB_QPT_GSI:</div><div class='add'>+		return mana_ib_destroy_ud_qp(qp, udata);</div><div class='add'>+	default:</div><div class='add'>+		ibdev_dbg(ibqp-&gt;device, "Unexpected QP type %u\n",</div><div class='add'>+			  ibqp-&gt;qp_type);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return -ENOENT;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mana/shadow_queue.h b/drivers/infiniband/hw/mana/shadow_queue.h<br/>new file mode 100644<br/>index 000000000000..a4b3818f9c39<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mana/shadow_queue.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mana/shadow_queue.h</a></div><div class='hunk'>@@ -0,0 +1,115 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2024, Microsoft Corporation. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef _MANA_SHADOW_QUEUE_H_</div><div class='add'>+#define _MANA_SHADOW_QUEUE_H_</div><div class='add'>+</div><div class='add'>+struct shadow_wqe_header {</div><div class='add'>+	u16 opcode;</div><div class='add'>+	u16 error_code;</div><div class='add'>+	u32 posted_wqe_size;</div><div class='add'>+	u64 wr_id;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ud_rq_shadow_wqe {</div><div class='add'>+	struct shadow_wqe_header header;</div><div class='add'>+	u32 byte_len;</div><div class='add'>+	u32 src_qpn;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct ud_sq_shadow_wqe {</div><div class='add'>+	struct shadow_wqe_header header;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct shadow_queue {</div><div class='add'>+	/* Unmasked producer index, Incremented on wqe posting */</div><div class='add'>+	u64 prod_idx;</div><div class='add'>+	/* Unmasked consumer index, Incremented on cq polling */</div><div class='add'>+	u64 cons_idx;</div><div class='add'>+	/* Unmasked index of next-to-complete (from HW) shadow WQE */</div><div class='add'>+	u64 next_to_complete_idx;</div><div class='add'>+	/* queue size in wqes */</div><div class='add'>+	u32 length;</div><div class='add'>+	/* distance between elements in bytes */</div><div class='add'>+	u32 stride;</div><div class='add'>+	/* ring buffer holding wqes */</div><div class='add'>+	void *buffer;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline int create_shadow_queue(struct shadow_queue *queue, uint32_t length, uint32_t stride)</div><div class='add'>+{</div><div class='add'>+	queue-&gt;buffer = kvmalloc_array(length, stride, GFP_KERNEL);</div><div class='add'>+	if (!queue-&gt;buffer)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	queue-&gt;length = length;</div><div class='add'>+	queue-&gt;stride = stride;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void destroy_shadow_queue(struct shadow_queue *queue)</div><div class='add'>+{</div><div class='add'>+	kvfree(queue-&gt;buffer);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool shadow_queue_full(struct shadow_queue *queue)</div><div class='add'>+{</div><div class='add'>+	return (queue-&gt;prod_idx - queue-&gt;cons_idx) &gt;= queue-&gt;length;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool shadow_queue_empty(struct shadow_queue *queue)</div><div class='add'>+{</div><div class='add'>+	return queue-&gt;prod_idx == queue-&gt;cons_idx;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void *</div><div class='add'>+shadow_queue_get_element(const struct shadow_queue *queue, u64 unmasked_index)</div><div class='add'>+{</div><div class='add'>+	u32 index = unmasked_index % queue-&gt;length;</div><div class='add'>+</div><div class='add'>+	return ((u8 *)queue-&gt;buffer + index * queue-&gt;stride);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void *</div><div class='add'>+shadow_queue_producer_entry(struct shadow_queue *queue)</div><div class='add'>+{</div><div class='add'>+	return shadow_queue_get_element(queue, queue-&gt;prod_idx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void *</div><div class='add'>+shadow_queue_get_next_to_consume(const struct shadow_queue *queue)</div><div class='add'>+{</div><div class='add'>+	if (queue-&gt;cons_idx == queue-&gt;next_to_complete_idx)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	return shadow_queue_get_element(queue, queue-&gt;cons_idx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void *</div><div class='add'>+shadow_queue_get_next_to_complete(struct shadow_queue *queue)</div><div class='add'>+{</div><div class='add'>+	if (queue-&gt;next_to_complete_idx == queue-&gt;prod_idx)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	return shadow_queue_get_element(queue, queue-&gt;next_to_complete_idx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void shadow_queue_advance_producer(struct shadow_queue *queue)</div><div class='add'>+{</div><div class='add'>+	queue-&gt;prod_idx++;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void shadow_queue_advance_consumer(struct shadow_queue *queue)</div><div class='add'>+{</div><div class='add'>+	queue-&gt;cons_idx++;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline void shadow_queue_advance_next_to_complete(struct shadow_queue *queue)</div><div class='add'>+{</div><div class='add'>+	queue-&gt;next_to_complete_idx++;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/mana/wq.c b/drivers/infiniband/hw/mana/wq.c<br/>new file mode 100644<br/>index 000000000000..f959f4b9244f<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mana/wq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mana/wq.c</a></div><div class='hunk'>@@ -0,0 +1,92 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2022, Microsoft Corporation. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include "mana_ib.h"</div><div class='add'>+</div><div class='add'>+struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,</div><div class='add'>+				struct ib_wq_init_attr *init_attr,</div><div class='add'>+				struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev =</div><div class='add'>+		container_of(pd-&gt;device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct mana_ib_create_wq ucmd = {};</div><div class='add'>+	struct mana_ib_wq *wq;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (udata-&gt;inlen &lt; sizeof(ucmd))</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	err = ib_copy_from_udata(&amp;ucmd, udata, min(sizeof(ucmd), udata-&gt;inlen));</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev,</div><div class='add'>+			  "Failed to copy from udata for create wq, %d\n", err);</div><div class='add'>+		return ERR_PTR(err);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	wq = kzalloc(sizeof(*wq), GFP_KERNEL);</div><div class='add'>+	if (!wq)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	ibdev_dbg(&amp;mdev-&gt;ib_dev, "ucmd wq_buf_addr 0x%llx\n", ucmd.wq_buf_addr);</div><div class='add'>+</div><div class='add'>+	err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &amp;wq-&gt;queue);</div><div class='add'>+	if (err) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev,</div><div class='add'>+			  "Failed to create queue for create wq, %d\n", err);</div><div class='add'>+		goto err_free_wq;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	wq-&gt;wqe = init_attr-&gt;max_wr;</div><div class='add'>+	wq-&gt;wq_buf_size = ucmd.wq_buf_size;</div><div class='add'>+	wq-&gt;rx_object = INVALID_MANA_HANDLE;</div><div class='add'>+	return &amp;wq-&gt;ibwq;</div><div class='add'>+</div><div class='add'>+err_free_wq:</div><div class='add'>+	kfree(wq);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(err);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,</div><div class='add'>+		      u32 wq_attr_mask, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	/* modify_wq is not supported by this version of the driver */</div><div class='add'>+	return -EOPNOTSUPP;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_wq *wq = container_of(ibwq, struct mana_ib_wq, ibwq);</div><div class='add'>+	struct ib_device *ib_dev = ibwq-&gt;device;</div><div class='add'>+	struct mana_ib_dev *mdev;</div><div class='add'>+</div><div class='add'>+	mdev = container_of(ib_dev, struct mana_ib_dev, ib_dev);</div><div class='add'>+</div><div class='add'>+	mana_ib_destroy_queue(mdev, &amp;wq-&gt;queue);</div><div class='add'>+</div><div class='add'>+	kfree(wq);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,</div><div class='add'>+				 struct ib_rwq_ind_table_init_attr *init_attr,</div><div class='add'>+				 struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	/*</div><div class='add'>+	 * There is no additional data in ind_table to be maintained by this</div><div class='add'>+	 * driver, do nothing</div><div class='add'>+	 */</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)</div><div class='add'>+{</div><div class='add'>+	/*</div><div class='add'>+	 * There is no additional data in ind_table to be maintained by this</div><div class='add'>+	 * driver, do nothing</div><div class='add'>+	 */</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mana/wr.c b/drivers/infiniband/hw/mana/wr.c<br/>new file mode 100644<br/>index 000000000000..1813567d3b16<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mana/wr.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mana/wr.c</a></div><div class='hunk'>@@ -0,0 +1,168 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2024, Microsoft Corporation. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include "mana_ib.h"</div><div class='add'>+</div><div class='add'>+#define MAX_WR_SGL_NUM (2)</div><div class='add'>+</div><div class='add'>+static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *wr)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev = container_of(qp-&gt;ibqp.device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct gdma_queue *queue = qp-&gt;ud_qp.queues[MANA_UD_RECV_QUEUE].kmem;</div><div class='add'>+	struct gdma_posted_wqe_info wqe_info = {0};</div><div class='add'>+	struct gdma_sge gdma_sgl[MAX_WR_SGL_NUM];</div><div class='add'>+	struct gdma_wqe_request wqe_req = {0};</div><div class='add'>+	struct ud_rq_shadow_wqe *shadow_wqe;</div><div class='add'>+	int err, i;</div><div class='add'>+</div><div class='add'>+	if (shadow_queue_full(&amp;qp-&gt;shadow_rq))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (wr-&gt;num_sge &gt; MAX_WR_SGL_NUM)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; wr-&gt;num_sge; ++i) {</div><div class='add'>+		gdma_sgl[i].address = wr-&gt;sg_list[i].addr;</div><div class='add'>+		gdma_sgl[i].mem_key = wr-&gt;sg_list[i].lkey;</div><div class='add'>+		gdma_sgl[i].size = wr-&gt;sg_list[i].length;</div><div class='add'>+	}</div><div class='add'>+	wqe_req.num_sge = wr-&gt;num_sge;</div><div class='add'>+	wqe_req.sgl = gdma_sgl;</div><div class='add'>+</div><div class='add'>+	err = mana_gd_post_work_request(queue, &amp;wqe_req, &amp;wqe_info);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	shadow_wqe = shadow_queue_producer_entry(&amp;qp-&gt;shadow_rq);</div><div class='add'>+	memset(shadow_wqe, 0, sizeof(*shadow_wqe));</div><div class='add'>+	shadow_wqe-&gt;header.opcode = IB_WC_RECV;</div><div class='add'>+	shadow_wqe-&gt;header.wr_id = wr-&gt;wr_id;</div><div class='add'>+	shadow_wqe-&gt;header.posted_wqe_size = wqe_info.wqe_size_in_bu;</div><div class='add'>+	shadow_queue_advance_producer(&amp;qp-&gt;shadow_rq);</div><div class='add'>+</div><div class='add'>+	mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,</div><div class='add'>+		      const struct ib_recv_wr **bad_wr)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);</div><div class='add'>+	int err = 0;</div><div class='add'>+</div><div class='add'>+	for (; wr; wr = wr-&gt;next) {</div><div class='add'>+		switch (ibqp-&gt;qp_type) {</div><div class='add'>+		case IB_QPT_UD:</div><div class='add'>+		case IB_QPT_GSI:</div><div class='add'>+			err = mana_ib_post_recv_ud(qp, wr);</div><div class='add'>+			if (unlikely(err)) {</div><div class='add'>+				*bad_wr = wr;</div><div class='add'>+				return err;</div><div class='add'>+			}</div><div class='add'>+			break;</div><div class='add'>+		default:</div><div class='add'>+			ibdev_dbg(ibqp-&gt;device, "Posting recv wr on qp type %u is not supported\n",</div><div class='add'>+				  ibqp-&gt;qp_type);</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr)</div><div class='add'>+{</div><div class='add'>+	struct mana_ib_dev *mdev = container_of(qp-&gt;ibqp.device, struct mana_ib_dev, ib_dev);</div><div class='add'>+	struct mana_ib_ah *ah = container_of(wr-&gt;ah, struct mana_ib_ah, ibah);</div><div class='add'>+	struct net_device *ndev = mana_ib_get_netdev(&amp;mdev-&gt;ib_dev, qp-&gt;port);</div><div class='add'>+	struct gdma_queue *queue = qp-&gt;ud_qp.queues[MANA_UD_SEND_QUEUE].kmem;</div><div class='add'>+	struct gdma_sge gdma_sgl[MAX_WR_SGL_NUM + 1];</div><div class='add'>+	struct gdma_posted_wqe_info wqe_info = {0};</div><div class='add'>+	struct gdma_wqe_request wqe_req = {0};</div><div class='add'>+	struct rdma_send_oob send_oob = {0};</div><div class='add'>+	struct ud_sq_shadow_wqe *shadow_wqe;</div><div class='add'>+	int err, i;</div><div class='add'>+</div><div class='add'>+	if (!ndev) {</div><div class='add'>+		ibdev_dbg(&amp;mdev-&gt;ib_dev, "Invalid port %u in QP %u\n",</div><div class='add'>+			  qp-&gt;port, qp-&gt;ibqp.qp_num);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (wr-&gt;wr.opcode != IB_WR_SEND)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (shadow_queue_full(&amp;qp-&gt;shadow_sq))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (wr-&gt;wr.num_sge &gt; MAX_WR_SGL_NUM)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	gdma_sgl[0].address = ah-&gt;dma_handle;</div><div class='add'>+	gdma_sgl[0].mem_key = qp-&gt;ibqp.pd-&gt;local_dma_lkey;</div><div class='add'>+	gdma_sgl[0].size = sizeof(struct mana_ib_av);</div><div class='add'>+	for (i = 0; i &lt; wr-&gt;wr.num_sge; ++i) {</div><div class='add'>+		gdma_sgl[i + 1].address = wr-&gt;wr.sg_list[i].addr;</div><div class='add'>+		gdma_sgl[i + 1].mem_key = wr-&gt;wr.sg_list[i].lkey;</div><div class='add'>+		gdma_sgl[i + 1].size = wr-&gt;wr.sg_list[i].length;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	wqe_req.num_sge = wr-&gt;wr.num_sge + 1;</div><div class='add'>+	wqe_req.sgl = gdma_sgl;</div><div class='add'>+	wqe_req.inline_oob_size = sizeof(struct rdma_send_oob);</div><div class='add'>+	wqe_req.inline_oob_data = &amp;send_oob;</div><div class='add'>+	wqe_req.flags = GDMA_WR_OOB_IN_SGL;</div><div class='add'>+	wqe_req.client_data_unit = ib_mtu_enum_to_int(ib_mtu_int_to_enum(ndev-&gt;mtu));</div><div class='add'>+</div><div class='add'>+	send_oob.wqe_type = WQE_TYPE_UD_SEND;</div><div class='add'>+	send_oob.fence = !!(wr-&gt;wr.send_flags &amp; IB_SEND_FENCE);</div><div class='add'>+	send_oob.signaled = !!(wr-&gt;wr.send_flags &amp; IB_SEND_SIGNALED);</div><div class='add'>+	send_oob.solicited = !!(wr-&gt;wr.send_flags &amp; IB_SEND_SOLICITED);</div><div class='add'>+	send_oob.psn = qp-&gt;ud_qp.sq_psn;</div><div class='add'>+	send_oob.ssn_or_rqpn = wr-&gt;remote_qpn;</div><div class='add'>+	send_oob.ud_send.remote_qkey =</div><div class='add'>+		qp-&gt;ibqp.qp_type == IB_QPT_GSI ? IB_QP1_QKEY : wr-&gt;remote_qkey;</div><div class='add'>+</div><div class='add'>+	err = mana_gd_post_work_request(queue, &amp;wqe_req, &amp;wqe_info);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	qp-&gt;ud_qp.sq_psn++;</div><div class='add'>+	shadow_wqe = shadow_queue_producer_entry(&amp;qp-&gt;shadow_sq);</div><div class='add'>+	memset(shadow_wqe, 0, sizeof(*shadow_wqe));</div><div class='add'>+	shadow_wqe-&gt;header.opcode = IB_WC_SEND;</div><div class='add'>+	shadow_wqe-&gt;header.wr_id = wr-&gt;wr.wr_id;</div><div class='add'>+	shadow_wqe-&gt;header.posted_wqe_size = wqe_info.wqe_size_in_bu;</div><div class='add'>+	shadow_queue_advance_producer(&amp;qp-&gt;shadow_sq);</div><div class='add'>+</div><div class='add'>+	mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,</div><div class='add'>+		      const struct ib_send_wr **bad_wr)</div><div class='add'>+{</div><div class='add'>+	int err;</div><div class='add'>+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);</div><div class='add'>+</div><div class='add'>+	for (; wr; wr = wr-&gt;next) {</div><div class='add'>+		switch (ibqp-&gt;qp_type) {</div><div class='add'>+		case IB_QPT_UD:</div><div class='add'>+		case IB_QPT_GSI:</div><div class='add'>+			err = mana_ib_post_send_ud(qp, ud_wr(wr));</div><div class='add'>+			if (unlikely(err)) {</div><div class='add'>+				*bad_wr = wr;</div><div class='add'>+				return err;</div><div class='add'>+			}</div><div class='add'>+			break;</div><div class='add'>+		default:</div><div class='add'>+			ibdev_dbg(ibqp-&gt;device, "Posting send wr on qp type %u is not supported\n",</div><div class='add'>+				  ibqp-&gt;qp_type);</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx4/Kconfig b/drivers/infiniband/hw/mlx4/Kconfig<br/>index fc01deac1d3c..f30ce9dd080a 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/Kconfig?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/Kconfig</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/Kconfig?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx4/Kconfig</a></div><div class='hunk'>@@ -1,9 +1,10 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='ctx'> config MLX4_INFINIBAND</div><div class='ctx'> 	tristate "Mellanox ConnectX HCA support"</div><div class='ctx'> 	depends on NETDEVICES &amp;&amp; ETHERNET &amp;&amp; PCI &amp;&amp; INET</div><div class='ctx'> 	select NET_VENDOR_MELLANOX</div><div class='ctx'> 	select MLX4_CORE</div><div class='del'>-	---help---</div><div class='add'>+	help</div><div class='ctx'> 	  This driver provides low-level InfiniBand support for</div><div class='ctx'> 	  Mellanox ConnectX PCI Express host channel adapters (HCAs).</div><div class='ctx'> 	  This is required to use InfiniBand protocols such as</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx4/Makefile b/drivers/infiniband/hw/mlx4/Makefile<br/>index f4213b3a8fe1..7b6757b02857 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/Makefile?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/Makefile</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/Makefile?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx4/Makefile</a></div><div class='hunk'>@@ -1,3 +1,4 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='ctx'> obj-$(CONFIG_MLX4_INFINIBAND)	+= mlx4_ib.o</div><div class='ctx'> </div><div class='ctx'> mlx4_ib-y :=	ah.o cq.o doorbell.o mad.o main.o mr.o qp.o srq.o mcg.o cm.o alias_GUID.o sysfs.o</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx4/ah.c b/drivers/infiniband/hw/mlx4/ah.c<br/>index 02a169f8027b..7321d6ab5fe1 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/ah.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/ah.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/ah.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx4/ah.c</a></div><div class='hunk'>@@ -141,10 +141,11 @@ static int create_iboe_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx4_ib_create_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr,</div><div class='del'>-		      u32 flags, struct ib_udata *udata)</div><div class='del'>-</div><div class='add'>+int mlx4_ib_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,</div><div class='add'>+		      struct ib_udata *udata)</div><div class='ctx'> {</div><div class='add'>+	struct rdma_ah_attr *ah_attr = init_attr-&gt;ah_attr;</div><div class='add'>+</div><div class='ctx'> 	if (ah_attr-&gt;type == RDMA_AH_ATTR_TYPE_ROCE) {</div><div class='ctx'> 		if (!(rdma_ah_get_ah_flags(ah_attr) &amp; IB_AH_GRH))</div><div class='ctx'> 			return -EINVAL;</div><div class='hunk'>@@ -167,12 +168,14 @@ int mlx4_ib_create_ah_slave(struct ib_ah *ah, struct rdma_ah_attr *ah_attr,</div><div class='ctx'> 			    int slave_sgid_index, u8 *s_mac, u16 vlan_tag)</div><div class='ctx'> {</div><div class='ctx'> 	struct rdma_ah_attr slave_attr = *ah_attr;</div><div class='add'>+	struct rdma_ah_init_attr init_attr = {};</div><div class='ctx'> 	struct mlx4_ib_ah *mah = to_mah(ah);</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	slave_attr.grh.sgid_attr = NULL;</div><div class='ctx'> 	slave_attr.grh.sgid_index = slave_sgid_index;</div><div class='del'>-	ret = mlx4_ib_create_ah(ah, &amp;slave_attr, 0, NULL);</div><div class='add'>+	init_attr.ah_attr = &amp;slave_attr;</div><div class='add'>+	ret = mlx4_ib_create_ah(ah, &amp;init_attr, NULL);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='hunk'>@@ -229,8 +232,3 @@ int mlx4_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='del'>-</div><div class='del'>-void mlx4_ib_destroy_ah(struct ib_ah *ah, u32 flags)</div><div class='del'>-{</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx4/alias_GUID.c b/drivers/infiniband/hw/mlx4/alias_GUID.c<br/>index 2a0b59a4b6eb..d7327735b8d0 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/alias_GUID.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/alias_GUID.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/alias_GUID.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx4/alias_GUID.c</a></div><div class='hunk'>@@ -38,7 +38,6 @@</div><div class='ctx'> #include &lt;rdma/ib_sa.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_pack.h&gt;</div><div class='ctx'> #include &lt;linux/mlx4/cmd.h&gt;</div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='ctx'> #include &lt;linux/init.h&gt;</div><div class='ctx'> #include &lt;linux/errno.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_user_verbs.h&gt;</div><div class='hunk'>@@ -73,12 +72,12 @@ static int get_low_record_time_index(struct mlx4_ib_dev *dev, u8 port,</div><div class='ctx'> 				     int *resched_delay_sec);</div><div class='ctx'> </div><div class='ctx'> void mlx4_ib_update_cache_on_guid_change(struct mlx4_ib_dev *dev, int block_num,</div><div class='del'>-					 u8 port_num, u8 *p_data)</div><div class='add'>+					 u32 port_num, u8 *p_data)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='ctx'> 	u64 guid_indexes;</div><div class='ctx'> 	int slave_id;</div><div class='del'>-	int port_index = port_num - 1;</div><div class='add'>+	u32 port_index = port_num - 1;</div><div class='ctx'> </div><div class='ctx'> 	if (!mlx4_is_master(dev-&gt;dev))</div><div class='ctx'> 		return;</div><div class='hunk'>@@ -86,7 +85,7 @@ void mlx4_ib_update_cache_on_guid_change(struct mlx4_ib_dev *dev, int block_num,</div><div class='ctx'> 	guid_indexes = be64_to_cpu((__force __be64) dev-&gt;sriov.alias_guid.</div><div class='ctx'> 				   ports_guid[port_num - 1].</div><div class='ctx'> 				   all_rec_per_port[block_num].guid_indexes);</div><div class='del'>-	pr_debug("port: %d, guid_indexes: 0x%llx\n", port_num, guid_indexes);</div><div class='add'>+	pr_debug("port: %u, guid_indexes: 0x%llx\n", port_num, guid_indexes);</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; NUM_ALIAS_GUID_IN_REC; i++) {</div><div class='ctx'> 		/* The location of the specific index starts from bit number 4</div><div class='hunk'>@@ -184,7 +183,7 @@ unlock:</div><div class='ctx'>  * port_number - 1 or 2</div><div class='ctx'>  */</div><div class='ctx'> void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev,</div><div class='del'>-					  int block_num, u8 port_num,</div><div class='add'>+					  int block_num, u32 port_num,</div><div class='ctx'> 					  u8 *p_data)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='hunk'>@@ -206,7 +205,7 @@ void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev,</div><div class='ctx'> 	guid_indexes = be64_to_cpu((__force __be64) dev-&gt;sriov.alias_guid.</div><div class='ctx'> 				   ports_guid[port_num - 1].</div><div class='ctx'> 				   all_rec_per_port[block_num].guid_indexes);</div><div class='del'>-	pr_debug("port: %d, guid_indexes: 0x%llx\n", port_num, guid_indexes);</div><div class='add'>+	pr_debug("port: %u, guid_indexes: 0x%llx\n", port_num, guid_indexes);</div><div class='ctx'> </div><div class='ctx'> 	/*calculate the slaves and notify them*/</div><div class='ctx'> 	for (i = 0; i &lt; NUM_ALIAS_GUID_IN_REC; i++) {</div><div class='hunk'>@@ -260,11 +259,11 @@ void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev,</div><div class='ctx'> 			new_state = set_and_calc_slave_port_state(dev-&gt;dev, slave_id, port_num,</div><div class='ctx'> 								  MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID,</div><div class='ctx'> 								  &amp;gen_event);</div><div class='del'>-			pr_debug("slave: %d, port: %d prev_port_state: %d,"</div><div class='add'>+			pr_debug("slave: %d, port: %u prev_port_state: %d,"</div><div class='ctx'> 				 " new_port_state: %d, gen_event: %d\n",</div><div class='ctx'> 				 slave_id, port_num, prev_state, new_state, gen_event);</div><div class='ctx'> 			if (gen_event == SLAVE_PORT_GEN_EVENT_UP) {</div><div class='del'>-				pr_debug("sending PORT_UP event to slave: %d, port: %d\n",</div><div class='add'>+				pr_debug("sending PORT_UP event to slave: %d, port: %u\n",</div><div class='ctx'> 					 slave_id, port_num);</div><div class='ctx'> 				mlx4_gen_port_state_change_eqe(dev-&gt;dev, slave_id,</div><div class='ctx'> 							       port_num, MLX4_PORT_CHANGE_SUBTYPE_ACTIVE);</div><div class='hunk'>@@ -274,7 +273,7 @@ void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev,</div><div class='ctx'> 						      MLX4_PORT_STATE_IB_EVENT_GID_INVALID,</div><div class='ctx'> 						      &amp;gen_event);</div><div class='ctx'> 			if (gen_event == SLAVE_PORT_GEN_EVENT_DOWN) {</div><div class='del'>-				pr_debug("sending PORT DOWN event to slave: %d, port: %d\n",</div><div class='add'>+				pr_debug("sending PORT DOWN event to slave: %d, port: %u\n",</div><div class='ctx'> 					 slave_id, port_num);</div><div class='ctx'> 				mlx4_gen_port_state_change_eqe(dev-&gt;dev,</div><div class='ctx'> 							       slave_id,</div><div class='hunk'>@@ -310,7 +309,7 @@ static void aliasguid_query_handler(int status,</div><div class='ctx'> 	if (status) {</div><div class='ctx'> 		pr_debug("(port: %d) failed: status = %d\n",</div><div class='ctx'> 			 cb_ctx-&gt;port, status);</div><div class='del'>-		rec-&gt;time_to_run = ktime_get_boot_ns() + 1 * NSEC_PER_SEC;</div><div class='add'>+		rec-&gt;time_to_run = ktime_get_boottime_ns() + 1 * NSEC_PER_SEC;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -416,7 +415,7 @@ next_entry:</div><div class='ctx'> 			 be64_to_cpu((__force __be64)rec-&gt;guid_indexes),</div><div class='ctx'> 			 be64_to_cpu((__force __be64)applied_guid_indexes),</div><div class='ctx'> 			 be64_to_cpu((__force __be64)declined_guid_indexes));</div><div class='del'>-		rec-&gt;time_to_run = ktime_get_boot_ns() +</div><div class='add'>+		rec-&gt;time_to_run = ktime_get_boottime_ns() +</div><div class='ctx'> 			resched_delay_sec * NSEC_PER_SEC;</div><div class='ctx'> 	} else {</div><div class='ctx'> 		rec-&gt;status = MLX4_GUID_INFO_STATUS_SET;</div><div class='hunk'>@@ -709,7 +708,7 @@ static int get_low_record_time_index(struct mlx4_ib_dev *dev, u8 port,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> 	if (resched_delay_sec) {</div><div class='del'>-		u64 curr_time = ktime_get_boot_ns();</div><div class='add'>+		u64 curr_time = ktime_get_boottime_ns();</div><div class='ctx'> </div><div class='ctx'> 		*resched_delay_sec = (low_record_time &lt; curr_time) ? 0 :</div><div class='ctx'> 			div_u64((low_record_time - curr_time), NSEC_PER_SEC);</div><div class='hunk'>@@ -822,17 +821,14 @@ void mlx4_ib_destroy_alias_guid_service(struct mlx4_ib_dev *dev)</div><div class='ctx'> 		}</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;sriov-&gt;alias_guid.ag_work_lock, flags);</div><div class='ctx'> 	}</div><div class='del'>-	for (i = 0 ; i &lt; dev-&gt;num_ports; i++) {</div><div class='del'>-		flush_workqueue(dev-&gt;sriov.alias_guid.ports_guid[i].wq);</div><div class='add'>+	for (i = 0 ; i &lt; dev-&gt;num_ports; i++)</div><div class='ctx'> 		destroy_workqueue(dev-&gt;sriov.alias_guid.ports_guid[i].wq);</div><div class='del'>-	}</div><div class='ctx'> 	ib_sa_unregister_client(dev-&gt;sriov.alias_guid.sa_client);</div><div class='ctx'> 	kfree(dev-&gt;sriov.alias_guid.sa_client);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx4_ib_init_alias_guid_service(struct mlx4_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	char alias_wq_name[15];</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> 	int i, j;</div><div class='ctx'> 	union ib_gid gid;</div><div class='hunk'>@@ -878,9 +874,8 @@ int mlx4_ib_init_alias_guid_service(struct mlx4_ib_dev *dev)</div><div class='ctx'> 		dev-&gt;sriov.alias_guid.ports_guid[i].parent = &amp;dev-&gt;sriov.alias_guid;</div><div class='ctx'> 		dev-&gt;sriov.alias_guid.ports_guid[i].port  = i;</div><div class='ctx'> </div><div class='del'>-		snprintf(alias_wq_name, sizeof alias_wq_name, "alias_guid%d", i);</div><div class='ctx'> 		dev-&gt;sriov.alias_guid.ports_guid[i].wq =</div><div class='del'>-			alloc_ordered_workqueue(alias_wq_name, WQ_MEM_RECLAIM);</div><div class='add'>+			alloc_ordered_workqueue("alias_guid%d", WQ_MEM_RECLAIM, i);</div><div class='ctx'> 		if (!dev-&gt;sriov.alias_guid.ports_guid[i].wq) {</div><div class='ctx'> 			ret = -ENOMEM;</div><div class='ctx'> 			goto err_thread;</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx4/cm.c b/drivers/infiniband/hw/mlx4/cm.c<br/>index ecd6cadd529a..03aacd526860 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/cm.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/cm.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/cm.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx4/cm.c</a></div><div class='hunk'>@@ -54,11 +54,20 @@ struct id_map_entry {</div><div class='ctx'> 	struct delayed_work timeout;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct rej_tmout_entry {</div><div class='add'>+	int slave;</div><div class='add'>+	u32 rem_pv_cm_id;</div><div class='add'>+	struct delayed_work timeout;</div><div class='add'>+	struct xarray *xa_rej_tmout;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct cm_generic_msg {</div><div class='ctx'> 	struct ib_mad_hdr hdr;</div><div class='ctx'> </div><div class='ctx'> 	__be32 local_comm_id;</div><div class='ctx'> 	__be32 remote_comm_id;</div><div class='add'>+	unsigned char unused[2];</div><div class='add'>+	__be16 rej_reason;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct cm_sidr_generic_msg {</div><div class='hunk'>@@ -71,6 +80,7 @@ struct cm_req_msg {</div><div class='ctx'> 	union ib_gid primary_path_sgid;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static struct workqueue_struct *cm_wq;</div><div class='ctx'> </div><div class='ctx'> static void set_local_comm_id(struct ib_mad *mad, u32 cm_id)</div><div class='ctx'> {</div><div class='hunk'>@@ -186,23 +196,6 @@ out:</div><div class='ctx'> 	kfree(ent);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void id_map_find_del(struct ib_device *ibdev, int pv_cm_id)</div><div class='del'>-{</div><div class='del'>-	struct mlx4_ib_sriov *sriov = &amp;to_mdev(ibdev)-&gt;sriov;</div><div class='del'>-	struct rb_root *sl_id_map = &amp;sriov-&gt;sl_id_map;</div><div class='del'>-	struct id_map_entry *ent, *found_ent;</div><div class='del'>-</div><div class='del'>-	spin_lock(&amp;sriov-&gt;id_map_lock);</div><div class='del'>-	ent = xa_erase(&amp;sriov-&gt;pv_id_table, pv_cm_id);</div><div class='del'>-	if (!ent)</div><div class='del'>-		goto out;</div><div class='del'>-	found_ent = id_map_find_by_sl_id(ibdev, ent-&gt;slave_id, ent-&gt;sl_cm_id);</div><div class='del'>-	if (found_ent &amp;&amp; found_ent == ent)</div><div class='del'>-		rb_erase(&amp;found_ent-&gt;node, sl_id_map);</div><div class='del'>-out:</div><div class='del'>-	spin_unlock(&amp;sriov-&gt;id_map_lock);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static void sl_id_map_add(struct ib_device *ibdev, struct id_map_entry *new)</div><div class='ctx'> {</div><div class='ctx'> 	struct rb_root *sl_id_map = &amp;to_mdev(ibdev)-&gt;sriov.sl_id_map;</div><div class='hunk'>@@ -294,14 +287,18 @@ static void schedule_delayed(struct ib_device *ibdev, struct id_map_entry *id)</div><div class='ctx'> 	spin_lock(&amp;sriov-&gt;id_map_lock);</div><div class='ctx'> 	spin_lock_irqsave(&amp;sriov-&gt;going_down_lock, flags);</div><div class='ctx'> 	/*make sure that there is no schedule inside the scheduled work.*/</div><div class='del'>-	if (!sriov-&gt;is_going_down) {</div><div class='add'>+	if (!sriov-&gt;is_going_down &amp;&amp; !id-&gt;scheduled_delete) {</div><div class='ctx'> 		id-&gt;scheduled_delete = 1;</div><div class='del'>-		schedule_delayed_work(&amp;id-&gt;timeout, CM_CLEANUP_CACHE_TIMEOUT);</div><div class='add'>+		queue_delayed_work(cm_wq, &amp;id-&gt;timeout, CM_CLEANUP_CACHE_TIMEOUT);</div><div class='add'>+	} else if (id-&gt;scheduled_delete) {</div><div class='add'>+		/* Adjust timeout if already scheduled */</div><div class='add'>+		mod_delayed_work(cm_wq, &amp;id-&gt;timeout, CM_CLEANUP_CACHE_TIMEOUT);</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;sriov-&gt;going_down_lock, flags);</div><div class='ctx'> 	spin_unlock(&amp;sriov-&gt;id_map_lock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+#define REJ_REASON(m) be16_to_cpu(((struct cm_generic_msg *)(m))-&gt;rej_reason)</div><div class='ctx'> int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id,</div><div class='ctx'> 		struct ib_mad *mad)</div><div class='ctx'> {</div><div class='hunk'>@@ -310,8 +307,10 @@ int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id</div><div class='ctx'> 	int pv_cm_id = -1;</div><div class='ctx'> </div><div class='ctx'> 	if (mad-&gt;mad_hdr.attr_id == CM_REQ_ATTR_ID ||</div><div class='del'>-			mad-&gt;mad_hdr.attr_id == CM_REP_ATTR_ID ||</div><div class='del'>-			mad-&gt;mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID) {</div><div class='add'>+	    mad-&gt;mad_hdr.attr_id == CM_REP_ATTR_ID ||</div><div class='add'>+	    mad-&gt;mad_hdr.attr_id == CM_MRA_ATTR_ID ||</div><div class='add'>+	    mad-&gt;mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID ||</div><div class='add'>+	    (mad-&gt;mad_hdr.attr_id == CM_REJ_ATTR_ID &amp;&amp; REJ_REASON(mad) == IB_CM_REJ_TIMEOUT)) {</div><div class='ctx'> 		sl_cm_id = get_local_comm_id(mad);</div><div class='ctx'> 		id = id_map_get(ibdev, &amp;pv_cm_id, slave_id, sl_cm_id);</div><div class='ctx'> 		if (id)</div><div class='hunk'>@@ -331,8 +330,8 @@ int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (!id) {</div><div class='del'>-		pr_debug("id{slave: %d, sl_cm_id: 0x%x} is NULL!\n",</div><div class='del'>-			 slave_id, sl_cm_id);</div><div class='add'>+		pr_debug("id{slave: %d, sl_cm_id: 0x%x} is NULL! attr_id: 0x%x\n",</div><div class='add'>+			 slave_id, sl_cm_id, be16_to_cpu(mad-&gt;mad_hdr.attr_id));</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -341,17 +340,97 @@ cont:</div><div class='ctx'> </div><div class='ctx'> 	if (mad-&gt;mad_hdr.attr_id == CM_DREQ_ATTR_ID)</div><div class='ctx'> 		schedule_delayed(ibdev, id);</div><div class='del'>-	else if (mad-&gt;mad_hdr.attr_id == CM_DREP_ATTR_ID)</div><div class='del'>-		id_map_find_del(ibdev, pv_cm_id);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void rej_tmout_timeout(struct work_struct *work)</div><div class='add'>+{</div><div class='add'>+	struct delayed_work *delay = to_delayed_work(work);</div><div class='add'>+	struct rej_tmout_entry *item = container_of(delay, struct rej_tmout_entry, timeout);</div><div class='add'>+	struct rej_tmout_entry *deleted;</div><div class='add'>+</div><div class='add'>+	deleted = xa_cmpxchg(item-&gt;xa_rej_tmout, item-&gt;rem_pv_cm_id, item, NULL, 0);</div><div class='add'>+</div><div class='add'>+	if (deleted != item)</div><div class='add'>+		pr_debug("deleted(%p) != item(%p)\n", deleted, item);</div><div class='add'>+</div><div class='add'>+	kfree(item);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int alloc_rej_tmout(struct mlx4_ib_sriov *sriov, u32 rem_pv_cm_id, int slave)</div><div class='add'>+{</div><div class='add'>+	struct rej_tmout_entry *item;</div><div class='add'>+	struct rej_tmout_entry *old;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	xa_lock(&amp;sriov-&gt;xa_rej_tmout);</div><div class='add'>+	item = xa_load(&amp;sriov-&gt;xa_rej_tmout, (unsigned long)rem_pv_cm_id);</div><div class='add'>+</div><div class='add'>+	if (item) {</div><div class='add'>+		if (xa_err(item))</div><div class='add'>+			ret =  xa_err(item);</div><div class='add'>+		else</div><div class='add'>+			/* If a retry, adjust delayed work */</div><div class='add'>+			mod_delayed_work(cm_wq, &amp;item-&gt;timeout, CM_CLEANUP_CACHE_TIMEOUT);</div><div class='add'>+		goto err_or_exists;</div><div class='add'>+	}</div><div class='add'>+	xa_unlock(&amp;sriov-&gt;xa_rej_tmout);</div><div class='add'>+</div><div class='add'>+	item = kmalloc(sizeof(*item), GFP_KERNEL);</div><div class='add'>+	if (!item)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	INIT_DELAYED_WORK(&amp;item-&gt;timeout, rej_tmout_timeout);</div><div class='add'>+	item-&gt;slave = slave;</div><div class='add'>+	item-&gt;rem_pv_cm_id = rem_pv_cm_id;</div><div class='add'>+	item-&gt;xa_rej_tmout = &amp;sriov-&gt;xa_rej_tmout;</div><div class='add'>+</div><div class='add'>+	old = xa_cmpxchg(&amp;sriov-&gt;xa_rej_tmout, (unsigned long)rem_pv_cm_id, NULL, item, GFP_KERNEL);</div><div class='add'>+	if (old) {</div><div class='add'>+		pr_debug(</div><div class='add'>+			"Non-null old entry (%p) or error (%d) when inserting\n",</div><div class='add'>+			old, xa_err(old));</div><div class='add'>+		kfree(item);</div><div class='add'>+		return xa_err(old);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	queue_delayed_work(cm_wq, &amp;item-&gt;timeout, CM_CLEANUP_CACHE_TIMEOUT);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='add'>+</div><div class='add'>+err_or_exists:</div><div class='add'>+	xa_unlock(&amp;sriov-&gt;xa_rej_tmout);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int lookup_rej_tmout_slave(struct mlx4_ib_sriov *sriov, u32 rem_pv_cm_id)</div><div class='add'>+{</div><div class='add'>+	struct rej_tmout_entry *item;</div><div class='add'>+	int slave;</div><div class='add'>+</div><div class='add'>+	xa_lock(&amp;sriov-&gt;xa_rej_tmout);</div><div class='add'>+	item = xa_load(&amp;sriov-&gt;xa_rej_tmout, (unsigned long)rem_pv_cm_id);</div><div class='add'>+</div><div class='add'>+	if (!item || xa_err(item)) {</div><div class='add'>+		pr_debug("Could not find slave. rem_pv_cm_id 0x%x error: %d\n",</div><div class='add'>+			 rem_pv_cm_id, xa_err(item));</div><div class='add'>+		slave = !item ? -ENOENT : xa_err(item);</div><div class='add'>+	} else {</div><div class='add'>+		slave = item-&gt;slave;</div><div class='add'>+	}</div><div class='add'>+	xa_unlock(&amp;sriov-&gt;xa_rej_tmout);</div><div class='add'>+</div><div class='add'>+	return slave;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave,</div><div class='ctx'> 			     struct ib_mad *mad)</div><div class='ctx'> {</div><div class='add'>+	struct mlx4_ib_sriov *sriov = &amp;to_mdev(ibdev)-&gt;sriov;</div><div class='add'>+	u32 rem_pv_cm_id = get_local_comm_id(mad);</div><div class='ctx'> 	u32 pv_cm_id;</div><div class='ctx'> 	struct id_map_entry *id;</div><div class='add'>+	int sts;</div><div class='ctx'> </div><div class='ctx'> 	if (mad-&gt;mad_hdr.attr_id == CM_REQ_ATTR_ID ||</div><div class='ctx'> 	    mad-&gt;mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID) {</div><div class='hunk'>@@ -367,6 +446,13 @@ int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave,</div><div class='ctx'> 				     be64_to_cpu(gid.global.interface_id));</div><div class='ctx'> 			return -ENOENT;</div><div class='ctx'> 		}</div><div class='add'>+</div><div class='add'>+		sts = alloc_rej_tmout(sriov, rem_pv_cm_id, *slave);</div><div class='add'>+		if (sts)</div><div class='add'>+			/* Even if this fails, we pass on the REQ to the slave */</div><div class='add'>+			pr_debug("Could not allocate rej_tmout entry. rem_pv_cm_id 0x%x slave %d status %d\n",</div><div class='add'>+				 rem_pv_cm_id, *slave, sts);</div><div class='add'>+</div><div class='ctx'> 		return 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -374,7 +460,14 @@ int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave,</div><div class='ctx'> 	id = id_map_get(ibdev, (int *)&amp;pv_cm_id, -1, -1);</div><div class='ctx'> </div><div class='ctx'> 	if (!id) {</div><div class='del'>-		pr_debug("Couldn't find an entry for pv_cm_id 0x%x\n", pv_cm_id);</div><div class='add'>+		if (mad-&gt;mad_hdr.attr_id == CM_REJ_ATTR_ID &amp;&amp;</div><div class='add'>+		    REJ_REASON(mad) == IB_CM_REJ_TIMEOUT &amp;&amp; slave) {</div><div class='add'>+			*slave = lookup_rej_tmout_slave(sriov, rem_pv_cm_id);</div><div class='add'>+</div><div class='add'>+			return (*slave &lt; 0) ? *slave : 0;</div><div class='add'>+		}</div><div class='add'>+		pr_debug("Couldn't find an entry for pv_cm_id 0x%x, attr_id 0x%x\n",</div><div class='add'>+			 pv_cm_id, be16_to_cpu(mad-&gt;mad_hdr.attr_id));</div><div class='ctx'> 		return -ENOENT;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -382,12 +475,9 @@ int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave,</div><div class='ctx'> 		*slave = id-&gt;slave_id;</div><div class='ctx'> 	set_remote_comm_id(mad, id-&gt;sl_cm_id);</div><div class='ctx'> </div><div class='del'>-	if (mad-&gt;mad_hdr.attr_id == CM_DREQ_ATTR_ID)</div><div class='add'>+	if (mad-&gt;mad_hdr.attr_id == CM_DREQ_ATTR_ID ||</div><div class='add'>+	    mad-&gt;mad_hdr.attr_id == CM_REJ_ATTR_ID)</div><div class='ctx'> 		schedule_delayed(ibdev, id);</div><div class='del'>-	else if (mad-&gt;mad_hdr.attr_id == CM_REJ_ATTR_ID ||</div><div class='del'>-			mad-&gt;mad_hdr.attr_id == CM_DREP_ATTR_ID) {</div><div class='del'>-		id_map_find_del(ibdev, (int) pv_cm_id);</div><div class='del'>-	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -398,6 +488,34 @@ void mlx4_ib_cm_paravirt_init(struct mlx4_ib_dev *dev)</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;dev-&gt;sriov.cm_list);</div><div class='ctx'> 	dev-&gt;sriov.sl_id_map = RB_ROOT;</div><div class='ctx'> 	xa_init_flags(&amp;dev-&gt;sriov.pv_id_table, XA_FLAGS_ALLOC);</div><div class='add'>+	xa_init(&amp;dev-&gt;sriov.xa_rej_tmout);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void rej_tmout_xa_cleanup(struct mlx4_ib_sriov *sriov, int slave)</div><div class='add'>+{</div><div class='add'>+	struct rej_tmout_entry *item;</div><div class='add'>+	bool flush_needed = false;</div><div class='add'>+	unsigned long id;</div><div class='add'>+	int cnt = 0;</div><div class='add'>+</div><div class='add'>+	xa_lock(&amp;sriov-&gt;xa_rej_tmout);</div><div class='add'>+	xa_for_each(&amp;sriov-&gt;xa_rej_tmout, id, item) {</div><div class='add'>+		if (slave &lt; 0 || slave == item-&gt;slave) {</div><div class='add'>+			mod_delayed_work(cm_wq, &amp;item-&gt;timeout, 0);</div><div class='add'>+			flush_needed = true;</div><div class='add'>+			++cnt;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	xa_unlock(&amp;sriov-&gt;xa_rej_tmout);</div><div class='add'>+</div><div class='add'>+	if (flush_needed) {</div><div class='add'>+		flush_workqueue(cm_wq);</div><div class='add'>+		pr_debug("Deleted %d entries in xarray for slave %d during cleanup\n",</div><div class='add'>+			 cnt, slave);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (slave &lt; 0)</div><div class='add'>+		WARN_ON(!xa_empty(&amp;sriov-&gt;xa_rej_tmout));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* slave = -1 ==&gt; all slaves */</div><div class='hunk'>@@ -423,7 +541,7 @@ void mlx4_ib_cm_paravirt_clean(struct mlx4_ib_dev *dev, int slave)</div><div class='ctx'> 	spin_unlock(&amp;sriov-&gt;id_map_lock);</div><div class='ctx'> </div><div class='ctx'> 	if (need_flush)</div><div class='del'>-		flush_scheduled_work(); /* make sure all timers were flushed */</div><div class='add'>+		flush_workqueue(cm_wq); /* make sure all timers were flushed */</div><div class='ctx'> </div><div class='ctx'> 	/* now, remove all leftover entries from databases*/</div><div class='ctx'> 	spin_lock(&amp;sriov-&gt;id_map_lock);</div><div class='hunk'>@@ -467,4 +585,20 @@ void mlx4_ib_cm_paravirt_clean(struct mlx4_ib_dev *dev, int slave)</div><div class='ctx'> 		list_del(&amp;map-&gt;list);</div><div class='ctx'> 		kfree(map);</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	rej_tmout_xa_cleanup(sriov, slave);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx4_ib_cm_init(void)</div><div class='add'>+{</div><div class='add'>+	cm_wq = alloc_workqueue("mlx4_ib_cm", WQ_PERCPU, 0);</div><div class='add'>+	if (!cm_wq)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx4_ib_cm_destroy(void)</div><div class='add'>+{</div><div class='add'>+	destroy_workqueue(cm_wq);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c<br/>index 022a0b4ea452..c592374f4a58 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/cq.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/cq.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/cq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx4/cq.c</a></div><div class='hunk'>@@ -135,7 +135,7 @@ static void mlx4_ib_free_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *</div><div class='ctx'> 	mlx4_buf_free(dev-&gt;dev, (cqe + 1) * buf-&gt;entry_size, &amp;buf-&gt;buf);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev, struct ib_udata *udata,</div><div class='add'>+static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev,</div><div class='ctx'> 			       struct mlx4_ib_cq_buf *buf,</div><div class='ctx'> 			       struct ib_umem **umem, u64 buf_addr, int cqe)</div><div class='ctx'> {</div><div class='hunk'>@@ -144,15 +144,18 @@ static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev, struct ib_udata *udata,</div><div class='ctx'> 	int shift;</div><div class='ctx'> 	int n;</div><div class='ctx'> </div><div class='del'>-	*umem = ib_umem_get(udata, buf_addr, cqe * cqe_size,</div><div class='del'>-			    IB_ACCESS_LOCAL_WRITE, 1);</div><div class='add'>+	*umem = ib_umem_get(&amp;dev-&gt;ib_dev, buf_addr, cqe * cqe_size,</div><div class='add'>+			    IB_ACCESS_LOCAL_WRITE);</div><div class='ctx'> 	if (IS_ERR(*umem))</div><div class='ctx'> 		return PTR_ERR(*umem);</div><div class='ctx'> </div><div class='del'>-	n = ib_umem_page_count(*umem);</div><div class='ctx'> 	shift = mlx4_ib_umem_calc_optimal_mtt_size(*umem, 0, &amp;n);</div><div class='del'>-	err = mlx4_mtt_init(dev-&gt;dev, n, shift, &amp;buf-&gt;mtt);</div><div class='add'>+	if (shift &lt; 0) {</div><div class='add'>+		err = shift;</div><div class='add'>+		goto err_buf;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='add'>+	err = mlx4_mtt_init(dev-&gt;dev, n, shift, &amp;buf-&gt;mtt);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_buf;</div><div class='ctx'> </div><div class='hunk'>@@ -172,14 +175,15 @@ err_buf:</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> #define CQ_CREATE_FLAGS_SUPPORTED IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION</div><div class='del'>-struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,</div><div class='del'>-				const struct ib_cq_init_attr *attr,</div><div class='del'>-				struct ib_udata *udata)</div><div class='add'>+int mlx4_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		      struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='add'>+	struct ib_udata *udata = &amp;attrs-&gt;driver_udata;</div><div class='add'>+	struct ib_device *ibdev = ibcq-&gt;device;</div><div class='ctx'> 	int entries = attr-&gt;cqe;</div><div class='ctx'> 	int vector = attr-&gt;comp_vector;</div><div class='ctx'> 	struct mlx4_ib_dev *dev = to_mdev(ibdev);</div><div class='del'>-	struct mlx4_ib_cq *cq;</div><div class='add'>+	struct mlx4_ib_cq *cq = to_mcq(ibcq);</div><div class='ctx'> 	struct mlx4_uar *uar;</div><div class='ctx'> 	void *buf_addr;</div><div class='ctx'> 	int err;</div><div class='hunk'>@@ -187,14 +191,10 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,</div><div class='ctx'> 		udata, struct mlx4_ib_ucontext, ibucontext);</div><div class='ctx'> </div><div class='ctx'> 	if (entries &lt; 1 || entries &gt; dev-&gt;dev-&gt;caps.max_cqes)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	if (attr-&gt;flags &amp; ~CQ_CREATE_FLAGS_SUPPORTED)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	cq = kzalloc(sizeof(*cq), GFP_KERNEL);</div><div class='del'>-	if (!cq)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	entries      = roundup_pow_of_two(entries + 1);</div><div class='ctx'> 	cq-&gt;ibcq.cqe = entries - 1;</div><div class='hunk'>@@ -215,7 +215,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='ctx'> 		buf_addr = (void *)(unsigned long)ucmd.buf_addr;</div><div class='del'>-		err = mlx4_ib_get_cq_umem(dev, udata, &amp;cq-&gt;buf, &amp;cq-&gt;umem,</div><div class='add'>+		err = mlx4_ib_get_cq_umem(dev, &amp;cq-&gt;buf, &amp;cq-&gt;umem,</div><div class='ctx'> 					  ucmd.buf_addr, entries);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto err_cq;</div><div class='hunk'>@@ -269,7 +269,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,</div><div class='ctx'> 			goto err_cq_free;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-	return &amp;cq-&gt;ibcq;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='ctx'> err_cq_free:</div><div class='ctx'> 	mlx4_cq_free(dev-&gt;dev, &amp;cq-&gt;mcq);</div><div class='hunk'>@@ -281,19 +281,15 @@ err_dbmap:</div><div class='ctx'> err_mtt:</div><div class='ctx'> 	mlx4_mtt_cleanup(dev-&gt;dev, &amp;cq-&gt;buf.mtt);</div><div class='ctx'> </div><div class='del'>-	if (udata)</div><div class='del'>-		ib_umem_release(cq-&gt;umem);</div><div class='del'>-	else</div><div class='add'>+	ib_umem_release(cq-&gt;umem);</div><div class='add'>+	if (!udata)</div><div class='ctx'> 		mlx4_ib_free_cq_buf(dev, &amp;cq-&gt;buf, cq-&gt;ibcq.cqe);</div><div class='ctx'> </div><div class='ctx'> err_db:</div><div class='ctx'> 	if (!udata)</div><div class='ctx'> 		mlx4_db_free(dev-&gt;dev, &amp;cq-&gt;db);</div><div class='del'>-</div><div class='ctx'> err_cq:</div><div class='del'>-	kfree(cq);</div><div class='del'>-</div><div class='del'>-	return ERR_PTR(err);</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq,</div><div class='hunk'>@@ -336,8 +332,8 @@ static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq</div><div class='ctx'> 	if (!cq-&gt;resize_buf)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	err = mlx4_ib_get_cq_umem(dev, udata, &amp;cq-&gt;resize_buf-&gt;buf,</div><div class='del'>-				  &amp;cq-&gt;resize_umem, ucmd.buf_addr, entries);</div><div class='add'>+	err = mlx4_ib_get_cq_umem(dev, &amp;cq-&gt;resize_buf-&gt;buf, &amp;cq-&gt;resize_umem,</div><div class='add'>+				  ucmd.buf_addr, entries);</div><div class='ctx'> 	if (err) {</div><div class='ctx'> 		kfree(cq-&gt;resize_buf);</div><div class='ctx'> 		cq-&gt;resize_buf = NULL;</div><div class='hunk'>@@ -475,11 +471,8 @@ err_buf:</div><div class='ctx'> 	kfree(cq-&gt;resize_buf);</div><div class='ctx'> 	cq-&gt;resize_buf = NULL;</div><div class='ctx'> </div><div class='del'>-	if (cq-&gt;resize_umem) {</div><div class='del'>-		ib_umem_release(cq-&gt;resize_umem);</div><div class='del'>-		cq-&gt;resize_umem = NULL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='add'>+	ib_umem_release(cq-&gt;resize_umem);</div><div class='add'>+	cq-&gt;resize_umem = NULL;</div><div class='ctx'> out:</div><div class='ctx'> 	mutex_unlock(&amp;cq-&gt;resize_mutex);</div><div class='ctx'> </div><div class='hunk'>@@ -501,14 +494,11 @@ int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)</div><div class='ctx'> 				struct mlx4_ib_ucontext,</div><div class='ctx'> 				ibucontext),</div><div class='ctx'> 			&amp;mcq-&gt;db);</div><div class='del'>-		ib_umem_release(mcq-&gt;umem);</div><div class='ctx'> 	} else {</div><div class='ctx'> 		mlx4_ib_free_cq_buf(dev, &amp;mcq-&gt;buf, cq-&gt;cqe);</div><div class='ctx'> 		mlx4_db_free(dev-&gt;dev, &amp;mcq-&gt;db);</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='del'>-	kfree(mcq);</div><div class='del'>-</div><div class='add'>+	ib_umem_release(mcq-&gt;umem);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -583,18 +573,13 @@ static void mlx4_ib_handle_error_cqe(struct mlx4_err_cqe *cqe,</div><div class='ctx'> 	wc-&gt;vendor_err = cqe-&gt;vendor_err_syndrome;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx4_ib_ipoib_csum_ok(__be16 status, __be16 checksum)</div><div class='add'>+static int mlx4_ib_ipoib_csum_ok(__be16 status, u8 badfcs_enc, __be16 checksum)</div><div class='ctx'> {</div><div class='del'>-	return ((status &amp; cpu_to_be16(MLX4_CQE_STATUS_IPV4      |</div><div class='del'>-				      MLX4_CQE_STATUS_IPV4F     |</div><div class='del'>-				      MLX4_CQE_STATUS_IPV4OPT   |</div><div class='del'>-				      MLX4_CQE_STATUS_IPV6      |</div><div class='del'>-				      MLX4_CQE_STATUS_IPOK)) ==</div><div class='del'>-		cpu_to_be16(MLX4_CQE_STATUS_IPV4        |</div><div class='del'>-			    MLX4_CQE_STATUS_IPOK))              &amp;&amp;</div><div class='del'>-		(status &amp; cpu_to_be16(MLX4_CQE_STATUS_UDP       |</div><div class='del'>-				      MLX4_CQE_STATUS_TCP))     &amp;&amp;</div><div class='del'>-		checksum == cpu_to_be16(0xffff);</div><div class='add'>+	return ((badfcs_enc &amp; MLX4_CQE_STATUS_L4_CSUM) ||</div><div class='add'>+		((status &amp; cpu_to_be16(MLX4_CQE_STATUS_IPOK)) &amp;&amp;</div><div class='add'>+		 (status &amp; cpu_to_be16(MLX4_CQE_STATUS_TCP |</div><div class='add'>+				       MLX4_CQE_STATUS_UDP)) &amp;&amp;</div><div class='add'>+		 (checksum == cpu_to_be16(0xffff))));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct ib_wc *wc,</div><div class='hunk'>@@ -785,13 +770,13 @@ repoll:</div><div class='ctx'> 		switch (cqe-&gt;owner_sr_opcode &amp; MLX4_CQE_OPCODE_MASK) {</div><div class='ctx'> 		case MLX4_OPCODE_RDMA_WRITE_IMM:</div><div class='ctx'> 			wc-&gt;wc_flags |= IB_WC_WITH_IMM;</div><div class='del'>-			/* fall through */</div><div class='add'>+			fallthrough;</div><div class='ctx'> 		case MLX4_OPCODE_RDMA_WRITE:</div><div class='ctx'> 			wc-&gt;opcode    = IB_WC_RDMA_WRITE;</div><div class='ctx'> 			break;</div><div class='ctx'> 		case MLX4_OPCODE_SEND_IMM:</div><div class='ctx'> 			wc-&gt;wc_flags |= IB_WC_WITH_IMM;</div><div class='del'>-			/* fall through */</div><div class='add'>+			fallthrough;</div><div class='ctx'> 		case MLX4_OPCODE_SEND:</div><div class='ctx'> 		case MLX4_OPCODE_SEND_INVAL:</div><div class='ctx'> 			wc-&gt;opcode    = IB_WC_SEND;</div><div class='hunk'>@@ -870,6 +855,7 @@ repoll:</div><div class='ctx'> 		wc-&gt;wc_flags	  |= g_mlpath_rqpn &amp; 0x80000000 ? IB_WC_GRH : 0;</div><div class='ctx'> 		wc-&gt;pkey_index     = be32_to_cpu(cqe-&gt;immed_rss_invalid) &amp; 0x7f;</div><div class='ctx'> 		wc-&gt;wc_flags	  |= mlx4_ib_ipoib_csum_ok(cqe-&gt;status,</div><div class='add'>+					cqe-&gt;badfcs_enc,</div><div class='ctx'> 					cqe-&gt;checksum) ? IB_WC_IP_CSUM_OK : 0;</div><div class='ctx'> 		if (is_eth) {</div><div class='ctx'> 			wc-&gt;slid = 0;</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx4/doorbell.c b/drivers/infiniband/hw/mlx4/doorbell.c<br/>index 0f390351cef0..9bbd695a9fd5 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/doorbell.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/doorbell.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/doorbell.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx4/doorbell.c</a></div><div class='hunk'>@@ -64,7 +64,8 @@ int mlx4_ib_db_map_user(struct ib_udata *udata, unsigned long virt,</div><div class='ctx'> </div><div class='ctx'> 	page-&gt;user_virt = (virt &amp; PAGE_MASK);</div><div class='ctx'> 	page-&gt;refcnt    = 0;</div><div class='del'>-	page-&gt;umem = ib_umem_get(udata, virt &amp; PAGE_MASK, PAGE_SIZE, 0, 0);</div><div class='add'>+	page-&gt;umem = ib_umem_get(context-&gt;ibucontext.device, virt &amp; PAGE_MASK,</div><div class='add'>+				 PAGE_SIZE, 0);</div><div class='ctx'> 	if (IS_ERR(page-&gt;umem)) {</div><div class='ctx'> 		err = PTR_ERR(page-&gt;umem);</div><div class='ctx'> 		kfree(page);</div><div class='hunk'>@@ -74,7 +75,8 @@ int mlx4_ib_db_map_user(struct ib_udata *udata, unsigned long virt,</div><div class='ctx'> 	list_add(&amp;page-&gt;list, &amp;context-&gt;db_page_list);</div><div class='ctx'> </div><div class='ctx'> found:</div><div class='del'>-	db-&gt;dma = sg_dma_address(page-&gt;umem-&gt;sg_head.sgl) + (virt &amp; ~PAGE_MASK);</div><div class='add'>+	db-&gt;dma = sg_dma_address(page-&gt;umem-&gt;sgt_append.sgt.sgl) +</div><div class='add'>+		  (virt &amp; ~PAGE_MASK);</div><div class='ctx'> 	db-&gt;u.user_page = page;</div><div class='ctx'> 	++page-&gt;refcnt;</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/hw/mlx4/mad.c b/drivers/infiniband/hw/mlx4/mad.c<br/>index 68c951491a08..91c714f72099 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/mad.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/mad.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/mad.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx4/mad.c</a></div><div class='hunk'>@@ -88,15 +88,15 @@ struct mlx4_rcv_tunnel_mad {</div><div class='ctx'> 	struct ib_mad mad;</div><div class='ctx'> } __packed;</div><div class='ctx'> </div><div class='del'>-static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u8 port_num);</div><div class='del'>-static void handle_lid_change_event(struct mlx4_ib_dev *dev, u8 port_num);</div><div class='add'>+static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u32 port_num);</div><div class='add'>+static void handle_lid_change_event(struct mlx4_ib_dev *dev, u32 port_num);</div><div class='ctx'> static void __propagate_pkey_ev(struct mlx4_ib_dev *dev, int port_num,</div><div class='ctx'> 				int block, u32 change_bitmap);</div><div class='ctx'> </div><div class='ctx'> __be64 mlx4_ib_gen_node_guid(void)</div><div class='ctx'> {</div><div class='ctx'> #define NODE_GUID_HI	((u64) (((u64)IB_OPENIB_OUI) &lt;&lt; 40))</div><div class='del'>-	return cpu_to_be64(NODE_GUID_HI | prandom_u32());</div><div class='add'>+	return cpu_to_be64(NODE_GUID_HI | get_random_u32());</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> __be64 mlx4_ib_get_new_demux_tid(struct mlx4_ib_demux_ctx *ctx)</div><div class='hunk'>@@ -186,7 +186,7 @@ int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int mad_ifc_flags,</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void update_sm_ah(struct mlx4_ib_dev *dev, u8 port_num, u16 lid, u8 sl)</div><div class='add'>+static void update_sm_ah(struct mlx4_ib_dev *dev, u32 port_num, u16 lid, u8 sl)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_ah *new_ah;</div><div class='ctx'> 	struct rdma_ah_attr ah_attr;</div><div class='hunk'>@@ -217,8 +217,8 @@ static void update_sm_ah(struct mlx4_ib_dev *dev, u8 port_num, u16 lid, u8 sl)</div><div class='ctx'>  * Snoop SM MADs for port info, GUID info, and  P_Key table sets, so we can</div><div class='ctx'>  * synthesize LID change, Client-Rereg, GID change, and P_Key change events.</div><div class='ctx'>  */</div><div class='del'>-static void smp_snoop(struct ib_device *ibdev, u8 port_num, const struct ib_mad *mad,</div><div class='del'>-		      u16 prev_lid)</div><div class='add'>+static void smp_snoop(struct ib_device *ibdev, u32 port_num,</div><div class='add'>+		      const struct ib_mad *mad, u16 prev_lid)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_info *pinfo;</div><div class='ctx'> 	u16 lid;</div><div class='hunk'>@@ -274,7 +274,7 @@ static void smp_snoop(struct ib_device *ibdev, u8 port_num, const struct ib_mad</div><div class='ctx'> 						be16_to_cpu(base[i]);</div><div class='ctx'> 				}</div><div class='ctx'> 			}</div><div class='del'>-			pr_debug("PKEY Change event: port=%d, "</div><div class='add'>+			pr_debug("PKEY Change event: port=%u, "</div><div class='ctx'> 				 "block=0x%x, change_bitmap=0x%x\n",</div><div class='ctx'> 				 port_num, bn, pkey_change_bitmap);</div><div class='ctx'> </div><div class='hunk'>@@ -380,7 +380,8 @@ static void node_desc_override(struct ib_device *dev,</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void forward_trap(struct mlx4_ib_dev *dev, u8 port_num, const struct ib_mad *mad)</div><div class='add'>+static void forward_trap(struct mlx4_ib_dev *dev, u32 port_num,</div><div class='add'>+			 const struct ib_mad *mad)</div><div class='ctx'> {</div><div class='ctx'> 	int qpn = mad-&gt;mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED;</div><div class='ctx'> 	struct ib_mad_send_buf *send_buf;</div><div class='hunk'>@@ -429,7 +430,7 @@ static int mlx4_ib_demux_sa_handler(struct ib_device *ibdev, int port, int slave</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx4_ib_find_real_gid(struct ib_device *ibdev, u8 port, __be64 guid)</div><div class='add'>+int mlx4_ib_find_real_gid(struct ib_device *ibdev, u32 port, __be64 guid)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_ib_dev *dev = to_mdev(ibdev);</div><div class='ctx'> 	int i;</div><div class='hunk'>@@ -443,7 +444,7 @@ int mlx4_ib_find_real_gid(struct ib_device *ibdev, u8 port, __be64 guid)</div><div class='ctx'> </div><div class='ctx'> </div><div class='ctx'> static int find_slave_port_pkey_ix(struct mlx4_ib_dev *dev, int slave,</div><div class='del'>-				   u8 port, u16 pkey, u16 *ix)</div><div class='add'>+				   u32 port, u16 pkey, u16 *ix)</div><div class='ctx'> {</div><div class='ctx'> 	int i, ret;</div><div class='ctx'> 	u8 unassigned_pkey_ix, pkey_ix, partial_ix = 0xFF;</div><div class='hunk'>@@ -500,7 +501,14 @@ static int get_gids_from_l3_hdr(struct ib_grh *grh, union ib_gid *sgid,</div><div class='ctx'> 					 sgid, dgid);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port,</div><div class='add'>+static int is_proxy_qp0(struct mlx4_ib_dev *dev, int qpn, int slave)</div><div class='add'>+{</div><div class='add'>+	int proxy_start = dev-&gt;dev-&gt;phys_caps.base_proxy_sqpn + 8 * slave;</div><div class='add'>+</div><div class='add'>+	return (qpn &gt;= proxy_start &amp;&amp; qpn &lt;= proxy_start + 1);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u32 port,</div><div class='ctx'> 			  enum ib_qp_type dest_qpt, struct ib_wc *wc,</div><div class='ctx'> 			  struct ib_grh *grh, struct ib_mad *mad)</div><div class='ctx'> {</div><div class='hunk'>@@ -520,8 +528,10 @@ int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port,</div><div class='ctx'> 	u16 cached_pkey;</div><div class='ctx'> 	u8 is_eth = dev-&gt;dev-&gt;caps.port_type[port] == MLX4_PORT_TYPE_ETH;</div><div class='ctx'> </div><div class='del'>-	if (dest_qpt &gt; IB_QPT_GSI)</div><div class='add'>+	if (dest_qpt &gt; IB_QPT_GSI) {</div><div class='add'>+		pr_debug("dest_qpt (%d) &gt; IB_QPT_GSI\n", dest_qpt);</div><div class='ctx'> 		return -EINVAL;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	tun_ctx = dev-&gt;sriov.demux[port-1].tun[slave];</div><div class='ctx'> </div><div class='hunk'>@@ -538,12 +548,20 @@ int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port,</div><div class='ctx'> 	if (dest_qpt) {</div><div class='ctx'> 		u16 pkey_ix;</div><div class='ctx'> 		ret = ib_get_cached_pkey(&amp;dev-&gt;ib_dev, port, wc-&gt;pkey_index, &amp;cached_pkey);</div><div class='del'>-		if (ret)</div><div class='add'>+		if (ret) {</div><div class='add'>+			pr_debug("unable to get %s cached pkey for index %d, ret %d\n",</div><div class='add'>+				 is_proxy_qp0(dev, wc-&gt;src_qp, slave) ? "SMI" : "GSI",</div><div class='add'>+				 wc-&gt;pkey_index, ret);</div><div class='ctx'> 			return -EINVAL;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='ctx'> 		ret = find_slave_port_pkey_ix(dev, slave, port, cached_pkey, &amp;pkey_ix);</div><div class='del'>-		if (ret)</div><div class='add'>+		if (ret) {</div><div class='add'>+			pr_debug("unable to get %s pkey ix for pkey 0x%x, ret %d\n",</div><div class='add'>+				 is_proxy_qp0(dev, wc-&gt;src_qp, slave) ? "SMI" : "GSI",</div><div class='add'>+				 cached_pkey, ret);</div><div class='ctx'> 			return -EINVAL;</div><div class='add'>+		}</div><div class='ctx'> 		tun_pkey_ix = pkey_ix;</div><div class='ctx'> 	} else</div><div class='ctx'> 		tun_pkey_ix = dev-&gt;pkeys.virt2phys_pkey[slave][port - 1][0];</div><div class='hunk'>@@ -661,7 +679,7 @@ end:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx4_ib_demux_mad(struct ib_device *ibdev, u8 port,</div><div class='add'>+static int mlx4_ib_demux_mad(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 			struct ib_wc *wc, struct ib_grh *grh,</div><div class='ctx'> 			struct ib_mad *mad)</div><div class='ctx'> {</div><div class='hunk'>@@ -715,7 +733,8 @@ static int mlx4_ib_demux_mad(struct ib_device *ibdev, u8 port,</div><div class='ctx'> </div><div class='ctx'> 		err = mlx4_ib_send_to_slave(dev, slave, port, wc-&gt;qp-&gt;qp_type, wc, grh, mad);</div><div class='ctx'> 		if (err)</div><div class='del'>-			pr_debug("failed sending to slave %d via tunnel qp (%d)\n",</div><div class='add'>+			pr_debug("failed sending %s to slave %d via tunnel qp (%d)\n",</div><div class='add'>+				 is_proxy_qp0(dev, wc-&gt;src_qp, slave) ? "SMI" : "GSI",</div><div class='ctx'> 				 slave, err);</div><div class='ctx'> 		return 0;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -794,12 +813,13 @@ static int mlx4_ib_demux_mad(struct ib_device *ibdev, u8 port,</div><div class='ctx'> </div><div class='ctx'> 	err = mlx4_ib_send_to_slave(dev, slave, port, wc-&gt;qp-&gt;qp_type, wc, grh, mad);</div><div class='ctx'> 	if (err)</div><div class='del'>-		pr_debug("failed sending to slave %d via tunnel qp (%d)\n",</div><div class='add'>+		pr_debug("failed sending %s to slave %d via tunnel qp (%d)\n",</div><div class='add'>+			 is_proxy_qp0(dev, wc-&gt;src_qp, slave) ? "SMI" : "GSI",</div><div class='ctx'> 			 slave, err);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,</div><div class='add'>+static int ib_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num,</div><div class='ctx'> 			const struct ib_wc *in_wc, const struct ib_grh *in_grh,</div><div class='ctx'> 			const struct ib_mad *in_mad, struct ib_mad *out_mad)</div><div class='ctx'> {</div><div class='hunk'>@@ -807,27 +827,6 @@ static int ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,</div><div class='ctx'> 	int err;</div><div class='ctx'> 	struct ib_port_attr pattr;</div><div class='ctx'> </div><div class='del'>-	if (in_wc &amp;&amp; in_wc-&gt;qp) {</div><div class='del'>-		pr_debug("received MAD: port:%d slid:%d sqpn:%d "</div><div class='del'>-			 "dlid_bits:%d dqpn:%d wc_flags:0x%x tid:%016llx cls:%x mtd:%x atr:%x\n",</div><div class='del'>-			 port_num,</div><div class='del'>-			 in_wc-&gt;slid, in_wc-&gt;src_qp,</div><div class='del'>-			 in_wc-&gt;dlid_path_bits,</div><div class='del'>-			 in_wc-&gt;qp-&gt;qp_num,</div><div class='del'>-			 in_wc-&gt;wc_flags,</div><div class='del'>-			 be64_to_cpu(in_mad-&gt;mad_hdr.tid),</div><div class='del'>-			 in_mad-&gt;mad_hdr.mgmt_class, in_mad-&gt;mad_hdr.method,</div><div class='del'>-			 be16_to_cpu(in_mad-&gt;mad_hdr.attr_id));</div><div class='del'>-		if (in_wc-&gt;wc_flags &amp; IB_WC_GRH) {</div><div class='del'>-			pr_debug("sgid_hi:0x%016llx sgid_lo:0x%016llx\n",</div><div class='del'>-				 be64_to_cpu(in_grh-&gt;sgid.global.subnet_prefix),</div><div class='del'>-				 be64_to_cpu(in_grh-&gt;sgid.global.interface_id));</div><div class='del'>-			pr_debug("dgid_hi:0x%016llx dgid_lo:0x%016llx\n",</div><div class='del'>-				 be64_to_cpu(in_grh-&gt;dgid.global.subnet_prefix),</div><div class='del'>-				 be64_to_cpu(in_grh-&gt;dgid.global.interface_id));</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	slid = in_wc ? ib_lid_cpu16(in_wc-&gt;slid) : be16_to_cpu(IB_LID_PERMISSIVE);</div><div class='ctx'> </div><div class='ctx'> 	if (in_mad-&gt;mad_hdr.method == IB_MGMT_METHOD_TRAP &amp;&amp; slid == 0) {</div><div class='hunk'>@@ -934,9 +933,10 @@ static int iboe_process_mad_port_info(void *out_mad)</div><div class='ctx'> 	return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int iboe_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,</div><div class='del'>-			const struct ib_wc *in_wc, const struct ib_grh *in_grh,</div><div class='del'>-			const struct ib_mad *in_mad, struct ib_mad *out_mad)</div><div class='add'>+static int iboe_process_mad(struct ib_device *ibdev, int mad_flags,</div><div class='add'>+			    u32 port_num, const struct ib_wc *in_wc,</div><div class='add'>+			    const struct ib_grh *in_grh,</div><div class='add'>+			    const struct ib_mad *in_mad, struct ib_mad *out_mad)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_counter counter_stats;</div><div class='ctx'> 	struct mlx4_ib_dev *dev = to_mdev(ibdev);</div><div class='hunk'>@@ -966,7 +966,6 @@ static int iboe_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,</div><div class='ctx'> 	}</div><div class='ctx'> 	mutex_unlock(&amp;dev-&gt;counters_table[port_num - 1].mutex);</div><div class='ctx'> 	if (stats_avail) {</div><div class='del'>-		memset(out_mad-&gt;data, 0, sizeof out_mad-&gt;data);</div><div class='ctx'> 		switch (counter_stats.counter_mode &amp; 0xf) {</div><div class='ctx'> 		case 0:</div><div class='ctx'> 			edit_counter(&amp;counter_stats,</div><div class='hunk'>@@ -982,40 +981,33 @@ static int iboe_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,</div><div class='add'>+int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num,</div><div class='ctx'> 			const struct ib_wc *in_wc, const struct ib_grh *in_grh,</div><div class='del'>-			const struct ib_mad_hdr *in, size_t in_mad_size,</div><div class='del'>-			struct ib_mad_hdr *out, size_t *out_mad_size,</div><div class='del'>-			u16 *out_mad_pkey_index)</div><div class='add'>+			const struct ib_mad *in, struct ib_mad *out,</div><div class='add'>+			size_t *out_mad_size, u16 *out_mad_pkey_index)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_ib_dev *dev = to_mdev(ibdev);</div><div class='del'>-	const struct ib_mad *in_mad = (const struct ib_mad *)in;</div><div class='del'>-	struct ib_mad *out_mad = (struct ib_mad *)out;</div><div class='ctx'> 	enum rdma_link_layer link = rdma_port_get_link_layer(ibdev, port_num);</div><div class='ctx'> </div><div class='del'>-	if (WARN_ON_ONCE(in_mad_size != sizeof(*in_mad) ||</div><div class='del'>-			 *out_mad_size != sizeof(*out_mad)))</div><div class='del'>-		return IB_MAD_RESULT_FAILURE;</div><div class='del'>-</div><div class='ctx'> 	/* iboe_process_mad() which uses the HCA flow-counters to implement IB PMA</div><div class='ctx'> 	 * queries, should be called only by VFs and for that specific purpose</div><div class='ctx'> 	 */</div><div class='ctx'> 	if (link == IB_LINK_LAYER_INFINIBAND) {</div><div class='ctx'> 		if (mlx4_is_slave(dev-&gt;dev) &amp;&amp;</div><div class='del'>-		    (in_mad-&gt;mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT &amp;&amp;</div><div class='del'>-		     (in_mad-&gt;mad_hdr.attr_id == IB_PMA_PORT_COUNTERS ||</div><div class='del'>-		      in_mad-&gt;mad_hdr.attr_id == IB_PMA_PORT_COUNTERS_EXT ||</div><div class='del'>-		      in_mad-&gt;mad_hdr.attr_id == IB_PMA_CLASS_PORT_INFO)))</div><div class='del'>-			return iboe_process_mad(ibdev, mad_flags, port_num, in_wc,</div><div class='del'>-						in_grh, in_mad, out_mad);</div><div class='add'>+		    (in-&gt;mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT &amp;&amp;</div><div class='add'>+		     (in-&gt;mad_hdr.attr_id == IB_PMA_PORT_COUNTERS ||</div><div class='add'>+		      in-&gt;mad_hdr.attr_id == IB_PMA_PORT_COUNTERS_EXT ||</div><div class='add'>+		      in-&gt;mad_hdr.attr_id == IB_PMA_CLASS_PORT_INFO)))</div><div class='add'>+			return iboe_process_mad(ibdev, mad_flags, port_num,</div><div class='add'>+						in_wc, in_grh, in, out);</div><div class='ctx'> </div><div class='del'>-		return ib_process_mad(ibdev, mad_flags, port_num, in_wc,</div><div class='del'>-				      in_grh, in_mad, out_mad);</div><div class='add'>+		return ib_process_mad(ibdev, mad_flags, port_num, in_wc, in_grh,</div><div class='add'>+				      in, out);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (link == IB_LINK_LAYER_ETHERNET)</div><div class='ctx'> 		return iboe_process_mad(ibdev, mad_flags, port_num, in_wc,</div><div class='del'>-					in_grh, in_mad, out_mad);</div><div class='add'>+					in_grh, in, out);</div><div class='ctx'> </div><div class='ctx'> 	return -EINVAL;</div><div class='ctx'> }</div><div class='hunk'>@@ -1083,7 +1075,7 @@ void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev)</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void handle_lid_change_event(struct mlx4_ib_dev *dev, u8 port_num)</div><div class='add'>+static void handle_lid_change_event(struct mlx4_ib_dev *dev, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	mlx4_ib_dispatch_event(dev, port_num, IB_EVENT_LID_CHANGE);</div><div class='ctx'> </div><div class='hunk'>@@ -1092,7 +1084,7 @@ static void handle_lid_change_event(struct mlx4_ib_dev *dev, u8 port_num)</div><div class='ctx'> 					    MLX4_EQ_PORT_INFO_LID_CHANGE_MASK);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u8 port_num)</div><div class='add'>+static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	/* re-configure the alias-guid and mcg's */</div><div class='ctx'> 	if (mlx4_is_master(dev-&gt;dev)) {</div><div class='hunk'>@@ -1131,7 +1123,7 @@ static void propagate_pkey_ev(struct mlx4_ib_dev *dev, int port_num,</div><div class='ctx'> 			    GET_MASK_FROM_EQE(eqe));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void handle_slaves_guid_change(struct mlx4_ib_dev *dev, u8 port_num,</div><div class='add'>+static void handle_slaves_guid_change(struct mlx4_ib_dev *dev, u32 port_num,</div><div class='ctx'> 				      u32 guid_tbl_blk_num, u32 change_bitmap)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_smp *in_mad  = NULL;</div><div class='hunk'>@@ -1187,7 +1179,7 @@ void handle_port_mgmt_change_event(struct work_struct *work)</div><div class='ctx'> 	struct ib_event_work *ew = container_of(work, struct ib_event_work, work);</div><div class='ctx'> 	struct mlx4_ib_dev *dev = ew-&gt;ib_dev;</div><div class='ctx'> 	struct mlx4_eqe *eqe = &amp;(ew-&gt;ib_eqe);</div><div class='del'>-	u8 port = eqe-&gt;event.port_mgmt_change.port;</div><div class='add'>+	u32 port = eqe-&gt;event.port_mgmt_change.port;</div><div class='ctx'> 	u32 changed_attr;</div><div class='ctx'> 	u32 tbl_block;</div><div class='ctx'> 	u32 change_bitmap;</div><div class='hunk'>@@ -1284,7 +1276,7 @@ void handle_port_mgmt_change_event(struct work_struct *work)</div><div class='ctx'> 	kfree(ew);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void mlx4_ib_dispatch_event(struct mlx4_ib_dev *dev, u8 port_num,</div><div class='add'>+void mlx4_ib_dispatch_event(struct mlx4_ib_dev *dev, u32 port_num,</div><div class='ctx'> 			    enum ib_event_type type)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_event event;</div><div class='hunk'>@@ -1307,6 +1299,18 @@ static void mlx4_ib_tunnel_comp_handler(struct ib_cq *cq, void *arg)</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;dev-&gt;sriov.going_down_lock, flags);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void mlx4_ib_wire_comp_handler(struct ib_cq *cq, void *arg)</div><div class='add'>+{</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	struct mlx4_ib_demux_pv_ctx *ctx = cq-&gt;cq_context;</div><div class='add'>+	struct mlx4_ib_dev *dev = to_mdev(ctx-&gt;ib_dev);</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;dev-&gt;sriov.going_down_lock, flags);</div><div class='add'>+	if (!dev-&gt;sriov.is_going_down &amp;&amp; ctx-&gt;state == DEMUX_PV_STATE_ACTIVE)</div><div class='add'>+		queue_work(ctx-&gt;wi_wq, &amp;ctx-&gt;work);</div><div class='add'>+	spin_unlock_irqrestore(&amp;dev-&gt;sriov.going_down_lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int mlx4_ib_post_pv_qp_buf(struct mlx4_ib_demux_pv_ctx *ctx,</div><div class='ctx'> 				  struct mlx4_ib_demux_pv_qp *tun_qp,</div><div class='ctx'> 				  int index)</div><div class='hunk'>@@ -1349,15 +1353,7 @@ static int mlx4_ib_multiplex_sa_handler(struct ib_device *ibdev, int port,</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int is_proxy_qp0(struct mlx4_ib_dev *dev, int qpn, int slave)</div><div class='del'>-{</div><div class='del'>-	int proxy_start = dev-&gt;dev-&gt;phys_caps.base_proxy_sqpn + 8 * slave;</div><div class='del'>-</div><div class='del'>-	return (qpn &gt;= proxy_start &amp;&amp; qpn &lt;= proxy_start + 1);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-</div><div class='del'>-int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,</div><div class='add'>+int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u32 port,</div><div class='ctx'> 			 enum ib_qp_type dest_qpt, u16 pkey_index,</div><div class='ctx'> 			 u32 remote_qpn, u32 qkey, struct rdma_ah_attr *attr,</div><div class='ctx'> 			 u8 *s_mac, u16 vlan_id, struct ib_mad *mad)</div><div class='hunk'>@@ -1409,10 +1405,10 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,</div><div class='ctx'> </div><div class='ctx'> 	spin_lock(&amp;sqp-&gt;tx_lock);</div><div class='ctx'> 	if (sqp-&gt;tx_ix_head - sqp-&gt;tx_ix_tail &gt;=</div><div class='del'>-	    (MLX4_NUM_TUNNEL_BUFS - 1))</div><div class='add'>+	    (MLX4_NUM_WIRE_BUFS - 1))</div><div class='ctx'> 		ret = -EAGAIN;</div><div class='ctx'> 	else</div><div class='del'>-		wire_tx_ix = (++sqp-&gt;tx_ix_head) &amp; (MLX4_NUM_TUNNEL_BUFS - 1);</div><div class='add'>+		wire_tx_ix = (++sqp-&gt;tx_ix_head) &amp; (MLX4_NUM_WIRE_BUFS - 1);</div><div class='ctx'> 	spin_unlock(&amp;sqp-&gt;tx_lock);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto out;</div><div class='hunk'>@@ -1492,6 +1488,7 @@ static void mlx4_ib_multiplex_mad(struct mlx4_ib_demux_pv_ctx *ctx, struct ib_wc</div><div class='ctx'> 	u16 vlan_id;</div><div class='ctx'> 	u8 qos;</div><div class='ctx'> 	u8 *dmac;</div><div class='add'>+	int sts;</div><div class='ctx'> </div><div class='ctx'> 	/* Get slave that sent this packet */</div><div class='ctx'> 	if (wc-&gt;src_qp &lt; dev-&gt;dev-&gt;phys_caps.base_proxy_sqpn ||</div><div class='hunk'>@@ -1528,6 +1525,7 @@ static void mlx4_ib_multiplex_mad(struct mlx4_ib_demux_pv_ctx *ctx, struct ib_wc</div><div class='ctx'> 			return;</div><div class='ctx'> 		} else</div><div class='ctx'> 			*slave_id = slave;</div><div class='add'>+		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		/* nothing */;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -1588,13 +1586,17 @@ static void mlx4_ib_multiplex_mad(struct mlx4_ib_demux_pv_ctx *ctx, struct ib_wc</div><div class='ctx'> 					&amp;vlan_id, &amp;qos))</div><div class='ctx'> 		rdma_ah_set_sl(&amp;ah_attr, qos);</div><div class='ctx'> </div><div class='del'>-	mlx4_ib_send_to_wire(dev, slave, ctx-&gt;port,</div><div class='del'>-			     is_proxy_qp0(dev, wc-&gt;src_qp, slave) ?</div><div class='del'>-			     IB_QPT_SMI : IB_QPT_GSI,</div><div class='del'>-			     be16_to_cpu(tunnel-&gt;hdr.pkey_index),</div><div class='del'>-			     be32_to_cpu(tunnel-&gt;hdr.remote_qpn),</div><div class='del'>-			     be32_to_cpu(tunnel-&gt;hdr.qkey),</div><div class='del'>-			     &amp;ah_attr, wc-&gt;smac, vlan_id, &amp;tunnel-&gt;mad);</div><div class='add'>+	sts = mlx4_ib_send_to_wire(dev, slave, ctx-&gt;port,</div><div class='add'>+				   is_proxy_qp0(dev, wc-&gt;src_qp, slave) ?</div><div class='add'>+				   IB_QPT_SMI : IB_QPT_GSI,</div><div class='add'>+				   be16_to_cpu(tunnel-&gt;hdr.pkey_index),</div><div class='add'>+				   be32_to_cpu(tunnel-&gt;hdr.remote_qpn),</div><div class='add'>+				   be32_to_cpu(tunnel-&gt;hdr.qkey),</div><div class='add'>+				   &amp;ah_attr, wc-&gt;smac, vlan_id, &amp;tunnel-&gt;mad);</div><div class='add'>+	if (sts)</div><div class='add'>+		pr_debug("failed sending %s to wire on behalf of slave %d (%d)\n",</div><div class='add'>+			 is_proxy_qp0(dev, wc-&gt;src_qp, slave) ? "SMI" : "GSI",</div><div class='add'>+			 slave, sts);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int mlx4_ib_alloc_pv_bufs(struct mlx4_ib_demux_pv_ctx *ctx,</div><div class='hunk'>@@ -1603,19 +1605,20 @@ static int mlx4_ib_alloc_pv_bufs(struct mlx4_ib_demux_pv_ctx *ctx,</div><div class='ctx'> 	int i;</div><div class='ctx'> 	struct mlx4_ib_demux_pv_qp *tun_qp;</div><div class='ctx'> 	int rx_buf_size, tx_buf_size;</div><div class='add'>+	const int nmbr_bufs = is_tun ? MLX4_NUM_TUNNEL_BUFS : MLX4_NUM_WIRE_BUFS;</div><div class='ctx'> </div><div class='ctx'> 	if (qp_type &gt; IB_QPT_GSI)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	tun_qp = &amp;ctx-&gt;qp[qp_type];</div><div class='ctx'> </div><div class='del'>-	tun_qp-&gt;ring = kcalloc(MLX4_NUM_TUNNEL_BUFS,</div><div class='add'>+	tun_qp-&gt;ring = kcalloc(nmbr_bufs,</div><div class='ctx'> 			       sizeof(struct mlx4_ib_buf),</div><div class='ctx'> 			       GFP_KERNEL);</div><div class='ctx'> 	if (!tun_qp-&gt;ring)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	tun_qp-&gt;tx_ring = kcalloc(MLX4_NUM_TUNNEL_BUFS,</div><div class='add'>+	tun_qp-&gt;tx_ring = kcalloc(nmbr_bufs,</div><div class='ctx'> 				  sizeof (struct mlx4_ib_tun_tx_buf),</div><div class='ctx'> 				  GFP_KERNEL);</div><div class='ctx'> 	if (!tun_qp-&gt;tx_ring) {</div><div class='hunk'>@@ -1632,7 +1635,7 @@ static int mlx4_ib_alloc_pv_bufs(struct mlx4_ib_demux_pv_ctx *ctx,</div><div class='ctx'> 		tx_buf_size = sizeof (struct mlx4_mad_snd_buf);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; MLX4_NUM_TUNNEL_BUFS; i++) {</div><div class='add'>+	for (i = 0; i &lt; nmbr_bufs; i++) {</div><div class='ctx'> 		tun_qp-&gt;ring[i].addr = kmalloc(rx_buf_size, GFP_KERNEL);</div><div class='ctx'> 		if (!tun_qp-&gt;ring[i].addr)</div><div class='ctx'> 			goto err;</div><div class='hunk'>@@ -1646,7 +1649,7 @@ static int mlx4_ib_alloc_pv_bufs(struct mlx4_ib_demux_pv_ctx *ctx,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; MLX4_NUM_TUNNEL_BUFS; i++) {</div><div class='add'>+	for (i = 0; i &lt; nmbr_bufs; i++) {</div><div class='ctx'> 		tun_qp-&gt;tx_ring[i].buf.addr =</div><div class='ctx'> 			kmalloc(tx_buf_size, GFP_KERNEL);</div><div class='ctx'> 		if (!tun_qp-&gt;tx_ring[i].buf.addr)</div><div class='hunk'>@@ -1677,9 +1680,7 @@ tx_err:</div><div class='ctx'> 				    tx_buf_size, DMA_TO_DEVICE);</div><div class='ctx'> 		kfree(tun_qp-&gt;tx_ring[i].buf.addr);</div><div class='ctx'> 	}</div><div class='del'>-	kfree(tun_qp-&gt;tx_ring);</div><div class='del'>-	tun_qp-&gt;tx_ring = NULL;</div><div class='del'>-	i = MLX4_NUM_TUNNEL_BUFS;</div><div class='add'>+	i = nmbr_bufs;</div><div class='ctx'> err:</div><div class='ctx'> 	while (i &gt; 0) {</div><div class='ctx'> 		--i;</div><div class='hunk'>@@ -1687,6 +1688,8 @@ err:</div><div class='ctx'> 				    rx_buf_size, DMA_FROM_DEVICE);</div><div class='ctx'> 		kfree(tun_qp-&gt;ring[i].addr);</div><div class='ctx'> 	}</div><div class='add'>+	kfree(tun_qp-&gt;tx_ring);</div><div class='add'>+	tun_qp-&gt;tx_ring = NULL;</div><div class='ctx'> 	kfree(tun_qp-&gt;ring);</div><div class='ctx'> 	tun_qp-&gt;ring = NULL;</div><div class='ctx'> 	return -ENOMEM;</div><div class='hunk'>@@ -1698,6 +1701,7 @@ static void mlx4_ib_free_pv_qp_bufs(struct mlx4_ib_demux_pv_ctx *ctx,</div><div class='ctx'> 	int i;</div><div class='ctx'> 	struct mlx4_ib_demux_pv_qp *tun_qp;</div><div class='ctx'> 	int rx_buf_size, tx_buf_size;</div><div class='add'>+	const int nmbr_bufs = is_tun ? MLX4_NUM_TUNNEL_BUFS : MLX4_NUM_WIRE_BUFS;</div><div class='ctx'> </div><div class='ctx'> 	if (qp_type &gt; IB_QPT_GSI)</div><div class='ctx'> 		return;</div><div class='hunk'>@@ -1712,13 +1716,13 @@ static void mlx4_ib_free_pv_qp_bufs(struct mlx4_ib_demux_pv_ctx *ctx,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; MLX4_NUM_TUNNEL_BUFS; i++) {</div><div class='add'>+	for (i = 0; i &lt; nmbr_bufs; i++) {</div><div class='ctx'> 		ib_dma_unmap_single(ctx-&gt;ib_dev, tun_qp-&gt;ring[i].map,</div><div class='ctx'> 				    rx_buf_size, DMA_FROM_DEVICE);</div><div class='ctx'> 		kfree(tun_qp-&gt;ring[i].addr);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; MLX4_NUM_TUNNEL_BUFS; i++) {</div><div class='add'>+	for (i = 0; i &lt; nmbr_bufs; i++) {</div><div class='ctx'> 		ib_dma_unmap_single(ctx-&gt;ib_dev, tun_qp-&gt;tx_ring[i].buf.map,</div><div class='ctx'> 				    tx_buf_size, DMA_TO_DEVICE);</div><div class='ctx'> 		kfree(tun_qp-&gt;tx_ring[i].buf.addr);</div><div class='hunk'>@@ -1752,9 +1756,6 @@ static void mlx4_ib_tunnel_comp_worker(struct work_struct *work)</div><div class='ctx'> 					       "buf:%lld\n", wc.wr_id);</div><div class='ctx'> 				break;</div><div class='ctx'> 			case IB_WC_SEND:</div><div class='del'>-				pr_debug("received tunnel send completion:"</div><div class='del'>-					 "wrid=0x%llx, status=0x%x\n",</div><div class='del'>-					 wc.wr_id, wc.status);</div><div class='ctx'> 				rdma_destroy_ah(tun_qp-&gt;tx_ring[wc.wr_id &amp;</div><div class='ctx'> 					      (MLX4_NUM_TUNNEL_BUFS - 1)].ah, 0);</div><div class='ctx'> 				tun_qp-&gt;tx_ring[wc.wr_id &amp; (MLX4_NUM_TUNNEL_BUFS - 1)].ah</div><div class='hunk'>@@ -1801,6 +1802,7 @@ static int create_pv_sqp(struct mlx4_ib_demux_pv_ctx *ctx,</div><div class='ctx'> 	struct mlx4_ib_qp_tunnel_init_attr qp_init_attr;</div><div class='ctx'> 	struct ib_qp_attr attr;</div><div class='ctx'> 	int qp_attr_mask_INIT;</div><div class='add'>+	const int nmbr_bufs = create_tun ? MLX4_NUM_TUNNEL_BUFS : MLX4_NUM_WIRE_BUFS;</div><div class='ctx'> </div><div class='ctx'> 	if (qp_type &gt; IB_QPT_GSI)</div><div class='ctx'> 		return -EINVAL;</div><div class='hunk'>@@ -1811,8 +1813,8 @@ static int create_pv_sqp(struct mlx4_ib_demux_pv_ctx *ctx,</div><div class='ctx'> 	qp_init_attr.init_attr.send_cq = ctx-&gt;cq;</div><div class='ctx'> 	qp_init_attr.init_attr.recv_cq = ctx-&gt;cq;</div><div class='ctx'> 	qp_init_attr.init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;</div><div class='del'>-	qp_init_attr.init_attr.cap.max_send_wr = MLX4_NUM_TUNNEL_BUFS;</div><div class='del'>-	qp_init_attr.init_attr.cap.max_recv_wr = MLX4_NUM_TUNNEL_BUFS;</div><div class='add'>+	qp_init_attr.init_attr.cap.max_send_wr = nmbr_bufs;</div><div class='add'>+	qp_init_attr.init_attr.cap.max_recv_wr = nmbr_bufs;</div><div class='ctx'> 	qp_init_attr.init_attr.cap.max_send_sge = 1;</div><div class='ctx'> 	qp_init_attr.init_attr.cap.max_recv_sge = 1;</div><div class='ctx'> 	if (create_tun) {</div><div class='hunk'>@@ -1834,9 +1836,9 @@ static int create_pv_sqp(struct mlx4_ib_demux_pv_ctx *ctx,</div><div class='ctx'> 	tun_qp-&gt;qp = ib_create_qp(ctx-&gt;pd, &amp;qp_init_attr.init_attr);</div><div class='ctx'> 	if (IS_ERR(tun_qp-&gt;qp)) {</div><div class='ctx'> 		ret = PTR_ERR(tun_qp-&gt;qp);</div><div class='add'>+		pr_err("Couldn't create %s QP (%pe)\n",</div><div class='add'>+		       create_tun ? "tunnel" : "special", tun_qp-&gt;qp);</div><div class='ctx'> 		tun_qp-&gt;qp = NULL;</div><div class='del'>-		pr_err("Couldn't create %s QP (%d)\n",</div><div class='del'>-		       create_tun ? "tunnel" : "special", ret);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -1874,7 +1876,7 @@ static int create_pv_sqp(struct mlx4_ib_demux_pv_ctx *ctx,</div><div class='ctx'> 		goto err_qp;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; MLX4_NUM_TUNNEL_BUFS; i++) {</div><div class='add'>+	for (i = 0; i &lt; nmbr_bufs; i++) {</div><div class='ctx'> 		ret = mlx4_ib_post_pv_qp_buf(ctx, tun_qp, i);</div><div class='ctx'> 		if (ret) {</div><div class='ctx'> 			pr_err(" mlx4_ib_post_pv_buf error"</div><div class='hunk'>@@ -1910,8 +1912,8 @@ static void mlx4_ib_sqp_comp_worker(struct work_struct *work)</div><div class='ctx'> 			switch (wc.opcode) {</div><div class='ctx'> 			case IB_WC_SEND:</div><div class='ctx'> 				kfree(sqp-&gt;tx_ring[wc.wr_id &amp;</div><div class='del'>-				      (MLX4_NUM_TUNNEL_BUFS - 1)].ah);</div><div class='del'>-				sqp-&gt;tx_ring[wc.wr_id &amp; (MLX4_NUM_TUNNEL_BUFS - 1)].ah</div><div class='add'>+				      (MLX4_NUM_WIRE_BUFS - 1)].ah);</div><div class='add'>+				sqp-&gt;tx_ring[wc.wr_id &amp; (MLX4_NUM_WIRE_BUFS - 1)].ah</div><div class='ctx'> 					= NULL;</div><div class='ctx'> 				spin_lock(&amp;sqp-&gt;tx_lock);</div><div class='ctx'> 				sqp-&gt;tx_ix_tail++;</div><div class='hunk'>@@ -1920,13 +1922,13 @@ static void mlx4_ib_sqp_comp_worker(struct work_struct *work)</div><div class='ctx'> 			case IB_WC_RECV:</div><div class='ctx'> 				mad = (struct ib_mad *) &amp;(((struct mlx4_mad_rcv_buf *)</div><div class='ctx'> 						(sqp-&gt;ring[wc.wr_id &amp;</div><div class='del'>-						(MLX4_NUM_TUNNEL_BUFS - 1)].addr))-&gt;payload);</div><div class='add'>+						(MLX4_NUM_WIRE_BUFS - 1)].addr))-&gt;payload);</div><div class='ctx'> 				grh = &amp;(((struct mlx4_mad_rcv_buf *)</div><div class='ctx'> 						(sqp-&gt;ring[wc.wr_id &amp;</div><div class='del'>-						(MLX4_NUM_TUNNEL_BUFS - 1)].addr))-&gt;grh);</div><div class='add'>+						(MLX4_NUM_WIRE_BUFS - 1)].addr))-&gt;grh);</div><div class='ctx'> 				mlx4_ib_demux_mad(ctx-&gt;ib_dev, ctx-&gt;port, &amp;wc, grh, mad);</div><div class='ctx'> 				if (mlx4_ib_post_pv_qp_buf(ctx, sqp, wc.wr_id &amp;</div><div class='del'>-							   (MLX4_NUM_TUNNEL_BUFS - 1)))</div><div class='add'>+							   (MLX4_NUM_WIRE_BUFS - 1)))</div><div class='ctx'> 					pr_err("Failed reposting SQP "</div><div class='ctx'> 					       "buf:%lld\n", wc.wr_id);</div><div class='ctx'> 				break;</div><div class='hunk'>@@ -1939,8 +1941,8 @@ static void mlx4_ib_sqp_comp_worker(struct work_struct *work)</div><div class='ctx'> 				 ctx-&gt;slave, wc.status, wc.wr_id);</div><div class='ctx'> 			if (!MLX4_TUN_IS_RECV(wc.wr_id)) {</div><div class='ctx'> 				kfree(sqp-&gt;tx_ring[wc.wr_id &amp;</div><div class='del'>-				      (MLX4_NUM_TUNNEL_BUFS - 1)].ah);</div><div class='del'>-				sqp-&gt;tx_ring[wc.wr_id &amp; (MLX4_NUM_TUNNEL_BUFS - 1)].ah</div><div class='add'>+				      (MLX4_NUM_WIRE_BUFS - 1)].ah);</div><div class='add'>+				sqp-&gt;tx_ring[wc.wr_id &amp; (MLX4_NUM_WIRE_BUFS - 1)].ah</div><div class='ctx'> 					= NULL;</div><div class='ctx'> 				spin_lock(&amp;sqp-&gt;tx_lock);</div><div class='ctx'> 				sqp-&gt;tx_ix_tail++;</div><div class='hunk'>@@ -1980,6 +1982,7 @@ static int create_pv_resources(struct ib_device *ibdev, int slave, int port,</div><div class='ctx'> {</div><div class='ctx'> 	int ret, cq_size;</div><div class='ctx'> 	struct ib_cq_init_attr cq_attr = {};</div><div class='add'>+	const int nmbr_bufs = create_tun ? MLX4_NUM_TUNNEL_BUFS : MLX4_NUM_WIRE_BUFS;</div><div class='ctx'> </div><div class='ctx'> 	if (ctx-&gt;state != DEMUX_PV_STATE_DOWN)</div><div class='ctx'> 		return -EEXIST;</div><div class='hunk'>@@ -2004,23 +2007,24 @@ static int create_pv_resources(struct ib_device *ibdev, int slave, int port,</div><div class='ctx'> 		goto err_out_qp0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	cq_size = 2 * MLX4_NUM_TUNNEL_BUFS;</div><div class='add'>+	cq_size = 2 * nmbr_bufs;</div><div class='ctx'> 	if (ctx-&gt;has_smi)</div><div class='ctx'> 		cq_size *= 2;</div><div class='ctx'> </div><div class='ctx'> 	cq_attr.cqe = cq_size;</div><div class='del'>-	ctx-&gt;cq = ib_create_cq(ctx-&gt;ib_dev, mlx4_ib_tunnel_comp_handler,</div><div class='add'>+	ctx-&gt;cq = ib_create_cq(ctx-&gt;ib_dev,</div><div class='add'>+			       create_tun ? mlx4_ib_tunnel_comp_handler : mlx4_ib_wire_comp_handler,</div><div class='ctx'> 			       NULL, ctx, &amp;cq_attr);</div><div class='ctx'> 	if (IS_ERR(ctx-&gt;cq)) {</div><div class='ctx'> 		ret = PTR_ERR(ctx-&gt;cq);</div><div class='del'>-		pr_err("Couldn't create tunnel CQ (%d)\n", ret);</div><div class='add'>+		pr_err("Couldn't create tunnel CQ (%pe)\n", ctx-&gt;cq);</div><div class='ctx'> 		goto err_buf;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ctx-&gt;pd = ib_alloc_pd(ctx-&gt;ib_dev, 0);</div><div class='ctx'> 	if (IS_ERR(ctx-&gt;pd)) {</div><div class='ctx'> 		ret = PTR_ERR(ctx-&gt;pd);</div><div class='del'>-		pr_err("Couldn't create tunnel PD (%d)\n", ret);</div><div class='add'>+		pr_err("Couldn't create tunnel PD (%pe)\n", ctx-&gt;pd);</div><div class='ctx'> 		goto err_cq;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -2046,6 +2050,7 @@ static int create_pv_resources(struct ib_device *ibdev, int slave, int port,</div><div class='ctx'> 		INIT_WORK(&amp;ctx-&gt;work, mlx4_ib_sqp_comp_worker);</div><div class='ctx'> </div><div class='ctx'> 	ctx-&gt;wq = to_mdev(ibdev)-&gt;sriov.demux[port - 1].wq;</div><div class='add'>+	ctx-&gt;wi_wq = to_mdev(ibdev)-&gt;sriov.demux[port - 1].wi_wq;</div><div class='ctx'> </div><div class='ctx'> 	ret = ib_req_notify_cq(ctx-&gt;cq, IB_CQ_NEXT_COMP);</div><div class='ctx'> 	if (ret) {</div><div class='hunk'>@@ -2153,7 +2158,6 @@ static int mlx4_ib_alloc_demux_ctx(struct mlx4_ib_dev *dev,</div><div class='ctx'> 				       struct mlx4_ib_demux_ctx *ctx,</div><div class='ctx'> 				       int port)</div><div class='ctx'> {</div><div class='del'>-	char name[12];</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='hunk'>@@ -2189,16 +2193,21 @@ static int mlx4_ib_alloc_demux_ctx(struct mlx4_ib_dev *dev,</div><div class='ctx'> 		goto err_mcg;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	snprintf(name, sizeof name, "mlx4_ibt%d", port);</div><div class='del'>-	ctx-&gt;wq = alloc_ordered_workqueue(name, WQ_MEM_RECLAIM);</div><div class='add'>+	ctx-&gt;wq = alloc_ordered_workqueue("mlx4_ibt%d", WQ_MEM_RECLAIM, port);</div><div class='ctx'> 	if (!ctx-&gt;wq) {</div><div class='ctx'> 		pr_err("Failed to create tunnelling WQ for port %d\n", port);</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='ctx'> 		goto err_wq;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	snprintf(name, sizeof name, "mlx4_ibud%d", port);</div><div class='del'>-	ctx-&gt;ud_wq = alloc_ordered_workqueue(name, WQ_MEM_RECLAIM);</div><div class='add'>+	ctx-&gt;wi_wq = alloc_ordered_workqueue("mlx4_ibwi%d", WQ_MEM_RECLAIM, port);</div><div class='add'>+	if (!ctx-&gt;wi_wq) {</div><div class='add'>+		pr_err("Failed to create wire WQ for port %d\n", port);</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err_wiwq;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ctx-&gt;ud_wq = alloc_ordered_workqueue("mlx4_ibud%d", WQ_MEM_RECLAIM, port);</div><div class='ctx'> 	if (!ctx-&gt;ud_wq) {</div><div class='ctx'> 		pr_err("Failed to create up/down WQ for port %d\n", port);</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='hunk'>@@ -2208,6 +2217,10 @@ static int mlx4_ib_alloc_demux_ctx(struct mlx4_ib_dev *dev,</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> err_udwq:</div><div class='add'>+	destroy_workqueue(ctx-&gt;wi_wq);</div><div class='add'>+	ctx-&gt;wi_wq = NULL;</div><div class='add'>+</div><div class='add'>+err_wiwq:</div><div class='ctx'> 	destroy_workqueue(ctx-&gt;wq);</div><div class='ctx'> 	ctx-&gt;wq = NULL;</div><div class='ctx'> </div><div class='hunk'>@@ -2255,12 +2268,14 @@ static void mlx4_ib_free_demux_ctx(struct mlx4_ib_demux_ctx *ctx)</div><div class='ctx'> 				ctx-&gt;tun[i]-&gt;state = DEMUX_PV_STATE_DOWNING;</div><div class='ctx'> 		}</div><div class='ctx'> 		flush_workqueue(ctx-&gt;wq);</div><div class='add'>+		flush_workqueue(ctx-&gt;wi_wq);</div><div class='ctx'> 		for (i = 0; i &lt; dev-&gt;dev-&gt;caps.sqp_demux; i++) {</div><div class='ctx'> 			destroy_pv_resources(dev, i, ctx-&gt;port, ctx-&gt;tun[i], 0);</div><div class='ctx'> 			free_pv_object(dev, i, ctx-&gt;port);</div><div class='ctx'> 		}</div><div class='ctx'> 		kfree(ctx-&gt;tun);</div><div class='ctx'> 		destroy_workqueue(ctx-&gt;ud_wq);</div><div class='add'>+		destroy_workqueue(ctx-&gt;wi_wq);</div><div class='ctx'> 		destroy_workqueue(ctx-&gt;wq);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c<br/>index 25d09d53b51c..dd35e03402ab 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/main.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/main.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/main.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx4/main.c</a></div><div class='hunk'>@@ -81,18 +81,12 @@ static const char mlx4_ib_version[] =</div><div class='ctx'> </div><div class='ctx'> static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init);</div><div class='ctx'> static enum rdma_link_layer mlx4_ib_port_link_layer(struct ib_device *device,</div><div class='del'>-						    u8 port_num);</div><div class='add'>+						    u32 port_num);</div><div class='add'>+static int mlx4_ib_event(struct notifier_block *this, unsigned long event,</div><div class='add'>+			 void *param);</div><div class='ctx'> </div><div class='ctx'> static struct workqueue_struct *wq;</div><div class='ctx'> </div><div class='del'>-static void init_query_mad(struct ib_smp *mad)</div><div class='del'>-{</div><div class='del'>-	mad-&gt;base_version  = 1;</div><div class='del'>-	mad-&gt;mgmt_class    = IB_MGMT_CLASS_SUBN_LID_ROUTED;</div><div class='del'>-	mad-&gt;class_version = 1;</div><div class='del'>-	mad-&gt;method	   = IB_MGMT_METHOD_GET;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int check_flow_steering_support(struct mlx4_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	int eth_num_ports = 0;</div><div class='hunk'>@@ -129,17 +123,20 @@ static int num_ib_ports(struct mlx4_dev *dev)</div><div class='ctx'> 	return ib_ports;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct net_device *mlx4_ib_get_netdev(struct ib_device *device, u8 port_num)</div><div class='add'>+static struct net_device *mlx4_ib_get_netdev(struct ib_device *device,</div><div class='add'>+					     u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_ib_dev *ibdev = to_mdev(device);</div><div class='del'>-	struct net_device *dev;</div><div class='add'>+	struct net_device *dev, *ret = NULL;</div><div class='ctx'> </div><div class='ctx'> 	rcu_read_lock();</div><div class='del'>-	dev = mlx4_get_protocol_dev(ibdev-&gt;dev, MLX4_PROT_ETH, port_num);</div><div class='add'>+	for_each_netdev_rcu(&amp;init_net, dev) {</div><div class='add'>+		if (dev-&gt;dev.parent != ibdev-&gt;ib_dev.dev.parent ||</div><div class='add'>+		    dev-&gt;dev_port + 1 != port_num)</div><div class='add'>+			continue;</div><div class='ctx'> </div><div class='del'>-	if (dev) {</div><div class='ctx'> 		if (mlx4_is_bonded(ibdev-&gt;dev)) {</div><div class='del'>-			struct net_device *upper = NULL;</div><div class='add'>+			struct net_device *upper;</div><div class='ctx'> </div><div class='ctx'> 			upper = netdev_master_upper_dev_get_rcu(dev);</div><div class='ctx'> 			if (upper) {</div><div class='hunk'>@@ -150,17 +147,19 @@ static struct net_device *mlx4_ib_get_netdev(struct ib_device *device, u8 port_n</div><div class='ctx'> 					dev = active;</div><div class='ctx'> 			}</div><div class='ctx'> 		}</div><div class='del'>-	}</div><div class='del'>-	if (dev)</div><div class='add'>+</div><div class='ctx'> 		dev_hold(dev);</div><div class='add'>+		ret = dev;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	rcu_read_unlock();</div><div class='del'>-	return dev;</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int mlx4_ib_update_gids_v1(struct gid_entry *gids,</div><div class='ctx'> 				  struct mlx4_ib_dev *ibdev,</div><div class='del'>-				  u8 port_num)</div><div class='add'>+				  u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_cmd_mailbox *mailbox;</div><div class='ctx'> 	int err;</div><div class='hunk'>@@ -193,7 +192,7 @@ static int mlx4_ib_update_gids_v1(struct gid_entry *gids,</div><div class='ctx'> </div><div class='ctx'> static int mlx4_ib_update_gids_v1_v2(struct gid_entry *gids,</div><div class='ctx'> 				     struct mlx4_ib_dev *ibdev,</div><div class='del'>-				     u8 port_num)</div><div class='add'>+				     u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_cmd_mailbox *mailbox;</div><div class='ctx'> 	int err;</div><div class='hunk'>@@ -238,7 +237,7 @@ static int mlx4_ib_update_gids_v1_v2(struct gid_entry *gids,</div><div class='ctx'> </div><div class='ctx'> static int mlx4_ib_update_gids(struct gid_entry *gids,</div><div class='ctx'> 			       struct mlx4_ib_dev *ibdev,</div><div class='del'>-			       u8 port_num)</div><div class='add'>+			       u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	if (ibdev-&gt;dev-&gt;caps.flags2 &amp; MLX4_DEV_CAP_FLAG2_ROCE_V1_V2)</div><div class='ctx'> 		return mlx4_ib_update_gids_v1_v2(gids, ibdev, port_num);</div><div class='hunk'>@@ -246,6 +245,13 @@ static int mlx4_ib_update_gids(struct gid_entry *gids,</div><div class='ctx'> 	return mlx4_ib_update_gids_v1(gids, ibdev, port_num);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void free_gid_entry(struct gid_entry *entry)</div><div class='add'>+{</div><div class='add'>+	memset(&amp;entry-&gt;gid, 0, sizeof(entry-&gt;gid));</div><div class='add'>+	kfree(entry-&gt;ctx);</div><div class='add'>+	entry-&gt;ctx = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int mlx4_ib_add_gid(const struct ib_gid_attr *attr, void **context)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_ib_dev *ibdev = to_mdev(attr-&gt;device);</div><div class='hunk'>@@ -255,7 +261,9 @@ static int mlx4_ib_add_gid(const struct ib_gid_attr *attr, void **context)</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> 	int hw_update = 0;</div><div class='ctx'> 	int i;</div><div class='del'>-	struct gid_entry *gids = NULL;</div><div class='add'>+	struct gid_entry *gids;</div><div class='add'>+	u16 vlan_id = 0xffff;</div><div class='add'>+	u8 mac[ETH_ALEN];</div><div class='ctx'> </div><div class='ctx'> 	if (!rdma_cap_roce_gid_table(attr-&gt;device, attr-&gt;port_num))</div><div class='ctx'> 		return -EINVAL;</div><div class='hunk'>@@ -266,12 +274,16 @@ static int mlx4_ib_add_gid(const struct ib_gid_attr *attr, void **context)</div><div class='ctx'> 	if (!context)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='add'>+	ret = rdma_read_gid_l2_fields(attr, &amp;vlan_id, &amp;mac[0]);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> 	port_gid_table = &amp;iboe-&gt;gids[attr-&gt;port_num - 1];</div><div class='ctx'> 	spin_lock_bh(&amp;iboe-&gt;lock);</div><div class='ctx'> 	for (i = 0; i &lt; MLX4_MAX_PORT_GIDS; ++i) {</div><div class='ctx'> 		if (!memcmp(&amp;port_gid_table-&gt;gids[i].gid,</div><div class='ctx'> 			    &amp;attr-&gt;gid, sizeof(attr-&gt;gid)) &amp;&amp;</div><div class='del'>-		    port_gid_table-&gt;gids[i].gid_type == attr-&gt;gid_type)  {</div><div class='add'>+		    port_gid_table-&gt;gids[i].gid_type == attr-&gt;gid_type &amp;&amp;</div><div class='add'>+		    port_gid_table-&gt;gids[i].vlan_id == vlan_id)  {</div><div class='ctx'> 			found = i;</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -288,9 +300,9 @@ static int mlx4_ib_add_gid(const struct ib_gid_attr *attr, void **context)</div><div class='ctx'> 				ret = -ENOMEM;</div><div class='ctx'> 			} else {</div><div class='ctx'> 				*context = port_gid_table-&gt;gids[free].ctx;</div><div class='del'>-				memcpy(&amp;port_gid_table-&gt;gids[free].gid,</div><div class='del'>-				       &amp;attr-&gt;gid, sizeof(attr-&gt;gid));</div><div class='add'>+				port_gid_table-&gt;gids[free].gid = attr-&gt;gid;</div><div class='ctx'> 				port_gid_table-&gt;gids[free].gid_type = attr-&gt;gid_type;</div><div class='add'>+				port_gid_table-&gt;gids[free].vlan_id = vlan_id;</div><div class='ctx'> 				port_gid_table-&gt;gids[free].ctx-&gt;real_index = free;</div><div class='ctx'> 				port_gid_table-&gt;gids[free].ctx-&gt;refcount = 1;</div><div class='ctx'> 				hw_update = 1;</div><div class='hunk'>@@ -306,6 +318,8 @@ static int mlx4_ib_add_gid(const struct ib_gid_attr *attr, void **context)</div><div class='ctx'> 				     GFP_ATOMIC);</div><div class='ctx'> 		if (!gids) {</div><div class='ctx'> 			ret = -ENOMEM;</div><div class='add'>+			*context = NULL;</div><div class='add'>+			free_gid_entry(&amp;port_gid_table-&gt;gids[free]);</div><div class='ctx'> 		} else {</div><div class='ctx'> 			for (i = 0; i &lt; MLX4_MAX_PORT_GIDS; i++) {</div><div class='ctx'> 				memcpy(&amp;gids[i].gid, &amp;port_gid_table-&gt;gids[i].gid, sizeof(union ib_gid));</div><div class='hunk'>@@ -317,6 +331,12 @@ static int mlx4_ib_add_gid(const struct ib_gid_attr *attr, void **context)</div><div class='ctx'> </div><div class='ctx'> 	if (!ret &amp;&amp; hw_update) {</div><div class='ctx'> 		ret = mlx4_ib_update_gids(gids, ibdev, attr-&gt;port_num);</div><div class='add'>+		if (ret) {</div><div class='add'>+			spin_lock_bh(&amp;iboe-&gt;lock);</div><div class='add'>+			*context = NULL;</div><div class='add'>+			free_gid_entry(&amp;port_gid_table-&gt;gids[free]);</div><div class='add'>+			spin_unlock_bh(&amp;iboe-&gt;lock);</div><div class='add'>+		}</div><div class='ctx'> 		kfree(gids);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -346,10 +366,7 @@ static int mlx4_ib_del_gid(const struct ib_gid_attr *attr, void **context)</div><div class='ctx'> 		if (!ctx-&gt;refcount) {</div><div class='ctx'> 			unsigned int real_index = ctx-&gt;real_index;</div><div class='ctx'> </div><div class='del'>-			memset(&amp;port_gid_table-&gt;gids[real_index].gid, 0,</div><div class='del'>-			       sizeof(port_gid_table-&gt;gids[real_index].gid));</div><div class='del'>-			kfree(port_gid_table-&gt;gids[real_index].ctx);</div><div class='del'>-			port_gid_table-&gt;gids[real_index].ctx = NULL;</div><div class='add'>+			free_gid_entry(&amp;port_gid_table-&gt;gids[real_index]);</div><div class='ctx'> 			hw_update = 1;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='hunk'>@@ -372,10 +389,10 @@ static int mlx4_ib_del_gid(const struct ib_gid_attr *attr, void **context)</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock_bh(&amp;iboe-&gt;lock);</div><div class='ctx'> </div><div class='del'>-	if (!ret &amp;&amp; hw_update) {</div><div class='add'>+	if (gids)</div><div class='ctx'> 		ret = mlx4_ib_update_gids(gids, ibdev, attr-&gt;port_num);</div><div class='del'>-		kfree(gids);</div><div class='del'>-	}</div><div class='add'>+</div><div class='add'>+	kfree(gids);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -388,7 +405,7 @@ int mlx4_ib_gid_index_to_real_index(struct mlx4_ib_dev *ibdev,</div><div class='ctx'> 	int real_index = -EINVAL;</div><div class='ctx'> 	int i;</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	u8 port_num = attr-&gt;port_num;</div><div class='add'>+	u32 port_num = attr-&gt;port_num;</div><div class='ctx'> </div><div class='ctx'> 	if (port_num &gt; MLX4_MAX_PORTS)</div><div class='ctx'> 		return -EINVAL;</div><div class='hunk'>@@ -415,20 +432,17 @@ int mlx4_ib_gid_index_to_real_index(struct mlx4_ib_dev *ibdev,</div><div class='ctx'> 	return real_index;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-#define field_avail(type, fld, sz) (offsetof(type, fld) + \</div><div class='del'>-				    sizeof(((type *)0)-&gt;fld) &lt;= (sz))</div><div class='del'>-</div><div class='ctx'> static int mlx4_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 				struct ib_device_attr *props,</div><div class='ctx'> 				struct ib_udata *uhw)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_ib_dev *dev = to_mdev(ibdev);</div><div class='del'>-	struct ib_smp *in_mad  = NULL;</div><div class='del'>-	struct ib_smp *out_mad = NULL;</div><div class='add'>+	struct ib_smp *in_mad;</div><div class='add'>+	struct ib_smp *out_mad;</div><div class='ctx'> 	int err;</div><div class='ctx'> 	int have_ib_ports;</div><div class='ctx'> 	struct mlx4_uverbs_ex_query_device cmd;</div><div class='del'>-	struct mlx4_uverbs_ex_query_device_resp resp = {.comp_mask = 0};</div><div class='add'>+	struct mlx4_uverbs_ex_query_device_resp resp = {};</div><div class='ctx'> 	struct mlx4_clock_params clock_params;</div><div class='ctx'> </div><div class='ctx'> 	if (uhw-&gt;inlen) {</div><div class='hunk'>@@ -454,7 +468,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 	if (!in_mad || !out_mad)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	init_query_mad(in_mad);</div><div class='add'>+	ib_init_query_mad(in_mad);</div><div class='ctx'> 	in_mad-&gt;attr_id = IB_SMP_ATTR_NODE_INFO;</div><div class='ctx'> </div><div class='ctx'> 	err = mlx4_MAD_IFC(to_mdev(ibdev), MLX4_MAD_IFC_IGNORE_KEYS,</div><div class='hunk'>@@ -470,8 +484,8 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 	props-&gt;device_cap_flags    = IB_DEVICE_CHANGE_PHY_PORT |</div><div class='ctx'> 		IB_DEVICE_PORT_ACTIVE_EVENT		|</div><div class='ctx'> 		IB_DEVICE_SYS_IMAGE_GUID		|</div><div class='del'>-		IB_DEVICE_RC_RNR_NAK_GEN		|</div><div class='del'>-		IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;</div><div class='add'>+		IB_DEVICE_RC_RNR_NAK_GEN;</div><div class='add'>+	props-&gt;kernel_cap_flags = IBK_BLOCK_MULTICAST_LOOPBACK;</div><div class='ctx'> 	if (dev-&gt;dev-&gt;caps.flags &amp; MLX4_DEV_CAP_FLAG_BAD_PKEY_CNTR)</div><div class='ctx'> 		props-&gt;device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;</div><div class='ctx'> 	if (dev-&gt;dev-&gt;caps.flags &amp; MLX4_DEV_CAP_FLAG_BAD_QKEY_CNTR)</div><div class='hunk'>@@ -485,9 +499,9 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 	if (dev-&gt;dev-&gt;caps.max_gso_sz &amp;&amp;</div><div class='ctx'> 	    (dev-&gt;dev-&gt;rev_id != MLX4_IB_CARD_REV_A0) &amp;&amp;</div><div class='ctx'> 	    (dev-&gt;dev-&gt;caps.flags &amp; MLX4_DEV_CAP_FLAG_BLH))</div><div class='del'>-		props-&gt;device_cap_flags |= IB_DEVICE_UD_TSO;</div><div class='add'>+		props-&gt;kernel_cap_flags |= IBK_UD_TSO;</div><div class='ctx'> 	if (dev-&gt;dev-&gt;caps.bmme_flags &amp; MLX4_BMME_FLAG_RESERVED_LKEY)</div><div class='del'>-		props-&gt;device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY;</div><div class='add'>+		props-&gt;kernel_cap_flags |= IBK_LOCAL_DMA_LKEY;</div><div class='ctx'> 	if ((dev-&gt;dev-&gt;caps.bmme_flags &amp; MLX4_BMME_FLAG_LOCAL_INV) &amp;&amp;</div><div class='ctx'> 	    (dev-&gt;dev-&gt;caps.bmme_flags &amp; MLX4_BMME_FLAG_REMOTE_INV) &amp;&amp;</div><div class='ctx'> 	    (dev-&gt;dev-&gt;caps.bmme_flags &amp; MLX4_BMME_FLAG_FAST_REG_WR))</div><div class='hunk'>@@ -542,7 +556,6 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 	props-&gt;max_mcast_qp_attach = dev-&gt;dev-&gt;caps.num_qp_per_mgm;</div><div class='ctx'> 	props-&gt;max_total_mcast_qp_attach = props-&gt;max_mcast_qp_attach *</div><div class='ctx'> 					   props-&gt;max_mcast_grp;</div><div class='del'>-	props-&gt;max_map_per_fmr = dev-&gt;dev-&gt;caps.max_fmr_maps;</div><div class='ctx'> 	props-&gt;hca_core_clock = dev-&gt;dev-&gt;caps.hca_core_clock * 1000UL;</div><div class='ctx'> 	props-&gt;timestamp_mask = 0xFFFFFFFFFFFFULL;</div><div class='ctx'> 	props-&gt;max_ah = INT_MAX;</div><div class='hunk'>@@ -565,12 +578,9 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 	props-&gt;cq_caps.max_cq_moderation_count = MLX4_MAX_CQ_COUNT;</div><div class='ctx'> 	props-&gt;cq_caps.max_cq_moderation_period = MLX4_MAX_CQ_PERIOD;</div><div class='ctx'> </div><div class='del'>-	if (!mlx4_is_slave(dev-&gt;dev))</div><div class='del'>-		err = mlx4_get_internal_clock_params(dev-&gt;dev, &amp;clock_params);</div><div class='del'>-</div><div class='ctx'> 	if (uhw-&gt;outlen &gt;= resp.response_length + sizeof(resp.hca_core_clock_offset)) {</div><div class='ctx'> 		resp.response_length += sizeof(resp.hca_core_clock_offset);</div><div class='del'>-		if (!err &amp;&amp; !mlx4_is_slave(dev-&gt;dev)) {</div><div class='add'>+		if (!mlx4_get_internal_clock_params(dev-&gt;dev, &amp;clock_params)) {</div><div class='ctx'> 			resp.comp_mask |= MLX4_IB_QUERY_DEV_RESP_MASK_CORE_CLOCK_OFFSET;</div><div class='ctx'> 			resp.hca_core_clock_offset = clock_params.offset % PAGE_SIZE;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -583,7 +593,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 			sizeof(struct mlx4_wqe_data_seg);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (field_avail(typeof(resp), rss_caps, uhw-&gt;outlen)) {</div><div class='add'>+	if (offsetofend(typeof(resp), rss_caps) &lt;= uhw-&gt;outlen) {</div><div class='ctx'> 		if (props-&gt;rss_caps.supported_qpts) {</div><div class='ctx'> 			resp.rss_caps.rx_hash_function =</div><div class='ctx'> 				MLX4_IB_RX_HASH_FUNC_TOEPLITZ;</div><div class='hunk'>@@ -607,7 +617,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 				       sizeof(resp.rss_caps);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (field_avail(typeof(resp), tso_caps, uhw-&gt;outlen)) {</div><div class='add'>+	if (offsetofend(typeof(resp), tso_caps) &lt;= uhw-&gt;outlen) {</div><div class='ctx'> 		if (dev-&gt;dev-&gt;caps.max_gso_sz &amp;&amp;</div><div class='ctx'> 		    ((mlx4_ib_port_link_layer(ibdev, 1) ==</div><div class='ctx'> 		    IB_LINK_LAYER_ETHERNET) ||</div><div class='hunk'>@@ -634,7 +644,7 @@ out:</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static enum rdma_link_layer</div><div class='del'>-mlx4_ib_port_link_layer(struct ib_device *device, u8 port_num)</div><div class='add'>+mlx4_ib_port_link_layer(struct ib_device *device, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_dev *dev = to_mdev(device)-&gt;dev;</div><div class='ctx'> </div><div class='hunk'>@@ -642,11 +652,11 @@ mlx4_ib_port_link_layer(struct ib_device *device, u8 port_num)</div><div class='ctx'> 		IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int ib_link_query_port(struct ib_device *ibdev, u8 port,</div><div class='add'>+static int ib_link_query_port(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 			      struct ib_port_attr *props, int netw_view)</div><div class='ctx'> {</div><div class='del'>-	struct ib_smp *in_mad  = NULL;</div><div class='del'>-	struct ib_smp *out_mad = NULL;</div><div class='add'>+	struct ib_smp *in_mad;</div><div class='add'>+	struct ib_smp *out_mad;</div><div class='ctx'> 	int ext_active_speed;</div><div class='ctx'> 	int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;</div><div class='ctx'> 	int err = -ENOMEM;</div><div class='hunk'>@@ -656,7 +666,7 @@ static int ib_link_query_port(struct ib_device *ibdev, u8 port,</div><div class='ctx'> 	if (!in_mad || !out_mad)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	init_query_mad(in_mad);</div><div class='add'>+	ib_init_query_mad(in_mad);</div><div class='ctx'> 	in_mad-&gt;attr_id  = IB_SMP_ATTR_PORT_INFO;</div><div class='ctx'> 	in_mad-&gt;attr_mod = cpu_to_be32(port);</div><div class='ctx'> </div><div class='hunk'>@@ -708,7 +718,7 @@ static int ib_link_query_port(struct ib_device *ibdev, u8 port,</div><div class='ctx'> </div><div class='ctx'> 	/* If reported active speed is QDR, check if is FDR-10 */</div><div class='ctx'> 	if (props-&gt;active_speed == IB_SPEED_QDR) {</div><div class='del'>-		init_query_mad(in_mad);</div><div class='add'>+		ib_init_query_mad(in_mad);</div><div class='ctx'> 		in_mad-&gt;attr_id = MLX4_ATTR_EXTENDED_PORT_INFO;</div><div class='ctx'> 		in_mad-&gt;attr_mod = cpu_to_be32(port);</div><div class='ctx'> </div><div class='hunk'>@@ -734,10 +744,11 @@ out:</div><div class='ctx'> </div><div class='ctx'> static u8 state_to_phys_state(enum ib_port_state state)</div><div class='ctx'> {</div><div class='del'>-	return state == IB_PORT_ACTIVE ? 5 : 3;</div><div class='add'>+	return state == IB_PORT_ACTIVE ?</div><div class='add'>+		IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int eth_link_query_port(struct ib_device *ibdev, u8 port,</div><div class='add'>+static int eth_link_query_port(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 			       struct ib_port_attr *props)</div><div class='ctx'> {</div><div class='ctx'> </div><div class='hunk'>@@ -768,7 +779,8 @@ static int eth_link_query_port(struct ib_device *ibdev, u8 port,</div><div class='ctx'> 	props-&gt;ip_gids = true;</div><div class='ctx'> 	props-&gt;gid_tbl_len	= mdev-&gt;dev-&gt;caps.gid_table_len[port];</div><div class='ctx'> 	props-&gt;max_msg_sz	= mdev-&gt;dev-&gt;caps.max_msg_sz;</div><div class='del'>-	props-&gt;pkey_tbl_len	= 1;</div><div class='add'>+	if (mdev-&gt;dev-&gt;caps.pkey_table_len[port])</div><div class='add'>+		props-&gt;pkey_tbl_len = 1;</div><div class='ctx'> 	props-&gt;max_mtu		= IB_MTU_4096;</div><div class='ctx'> 	props-&gt;max_vl_num	= 2;</div><div class='ctx'> 	props-&gt;state		= IB_PORT_DOWN;</div><div class='hunk'>@@ -797,7 +809,7 @@ out:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port,</div><div class='add'>+int __mlx4_ib_query_port(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 			 struct ib_port_attr *props, int netw_view)</div><div class='ctx'> {</div><div class='ctx'> 	int err;</div><div class='hunk'>@@ -811,18 +823,18 @@ int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port,</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx4_ib_query_port(struct ib_device *ibdev, u8 port,</div><div class='add'>+static int mlx4_ib_query_port(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 			      struct ib_port_attr *props)</div><div class='ctx'> {</div><div class='ctx'> 	/* returns host view */</div><div class='ctx'> 	return __mlx4_ib_query_port(ibdev, port, props, 0);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index,</div><div class='add'>+int __mlx4_ib_query_gid(struct ib_device *ibdev, u32 port, int index,</div><div class='ctx'> 			union ib_gid *gid, int netw_view)</div><div class='ctx'> {</div><div class='del'>-	struct ib_smp *in_mad  = NULL;</div><div class='del'>-	struct ib_smp *out_mad = NULL;</div><div class='add'>+	struct ib_smp *in_mad;</div><div class='add'>+	struct ib_smp *out_mad;</div><div class='ctx'> 	int err = -ENOMEM;</div><div class='ctx'> 	struct mlx4_ib_dev *dev = to_mdev(ibdev);</div><div class='ctx'> 	int clear = 0;</div><div class='hunk'>@@ -833,7 +845,7 @@ int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index,</div><div class='ctx'> 	if (!in_mad || !out_mad)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	init_query_mad(in_mad);</div><div class='add'>+	ib_init_query_mad(in_mad);</div><div class='ctx'> 	in_mad-&gt;attr_id  = IB_SMP_ATTR_PORT_INFO;</div><div class='ctx'> 	in_mad-&gt;attr_mod = cpu_to_be32(port);</div><div class='ctx'> </div><div class='hunk'>@@ -855,7 +867,7 @@ int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	init_query_mad(in_mad);</div><div class='add'>+	ib_init_query_mad(in_mad);</div><div class='ctx'> 	in_mad-&gt;attr_id  = IB_SMP_ATTR_GUID_INFO;</div><div class='ctx'> 	in_mad-&gt;attr_mod = cpu_to_be32(index / 8);</div><div class='ctx'> </div><div class='hunk'>@@ -874,7 +886,7 @@ out:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index,</div><div class='add'>+static int mlx4_ib_query_gid(struct ib_device *ibdev, u32 port, int index,</div><div class='ctx'> 			     union ib_gid *gid)</div><div class='ctx'> {</div><div class='ctx'> 	if (rdma_protocol_ib(ibdev, port))</div><div class='hunk'>@@ -882,11 +894,12 @@ static int mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx4_ib_query_sl2vl(struct ib_device *ibdev, u8 port, u64 *sl2vl_tbl)</div><div class='add'>+static int mlx4_ib_query_sl2vl(struct ib_device *ibdev, u32 port,</div><div class='add'>+			       u64 *sl2vl_tbl)</div><div class='ctx'> {</div><div class='ctx'> 	union sl2vl_tbl_to_u64 sl2vl64;</div><div class='del'>-	struct ib_smp *in_mad  = NULL;</div><div class='del'>-	struct ib_smp *out_mad = NULL;</div><div class='add'>+	struct ib_smp *in_mad;</div><div class='add'>+	struct ib_smp *out_mad;</div><div class='ctx'> 	int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;</div><div class='ctx'> 	int err = -ENOMEM;</div><div class='ctx'> 	int jj;</div><div class='hunk'>@@ -901,7 +914,7 @@ static int mlx4_ib_query_sl2vl(struct ib_device *ibdev, u8 port, u64 *sl2vl_tbl)</div><div class='ctx'> 	if (!in_mad || !out_mad)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	init_query_mad(in_mad);</div><div class='add'>+	ib_init_query_mad(in_mad);</div><div class='ctx'> 	in_mad-&gt;attr_id  = IB_SMP_ATTR_SL_TO_VL_TABLE;</div><div class='ctx'> 	in_mad-&gt;attr_mod = 0;</div><div class='ctx'> </div><div class='hunk'>@@ -942,11 +955,11 @@ static void mlx4_init_sl2vl_tbl(struct mlx4_ib_dev *mdev)</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,</div><div class='add'>+int __mlx4_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index,</div><div class='ctx'> 			 u16 *pkey, int netw_view)</div><div class='ctx'> {</div><div class='del'>-	struct ib_smp *in_mad  = NULL;</div><div class='del'>-	struct ib_smp *out_mad = NULL;</div><div class='add'>+	struct ib_smp *in_mad;</div><div class='add'>+	struct ib_smp *out_mad;</div><div class='ctx'> 	int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;</div><div class='ctx'> 	int err = -ENOMEM;</div><div class='ctx'> </div><div class='hunk'>@@ -955,7 +968,7 @@ int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,</div><div class='ctx'> 	if (!in_mad || !out_mad)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	init_query_mad(in_mad);</div><div class='add'>+	ib_init_query_mad(in_mad);</div><div class='ctx'> 	in_mad-&gt;attr_id  = IB_SMP_ATTR_PKEY_TABLE;</div><div class='ctx'> 	in_mad-&gt;attr_mod = cpu_to_be32(index / 32);</div><div class='ctx'> </div><div class='hunk'>@@ -975,7 +988,8 @@ out:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)</div><div class='add'>+static int mlx4_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index,</div><div class='add'>+			      u16 *pkey)</div><div class='ctx'> {</div><div class='ctx'> 	return __mlx4_ib_query_pkey(ibdev, port, index, pkey, 0);</div><div class='ctx'> }</div><div class='hunk'>@@ -1016,8 +1030,8 @@ static int mlx4_ib_modify_device(struct ib_device *ibdev, int mask,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx4_ib_SET_PORT(struct mlx4_ib_dev *dev, u8 port, int reset_qkey_viols,</div><div class='del'>-			    u32 cap_mask)</div><div class='add'>+static int mlx4_ib_SET_PORT(struct mlx4_ib_dev *dev, u32 port,</div><div class='add'>+			    int reset_qkey_viols, u32 cap_mask)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_cmd_mailbox *mailbox;</div><div class='ctx'> 	int err;</div><div class='hunk'>@@ -1042,7 +1056,7 @@ static int mlx4_ib_SET_PORT(struct mlx4_ib_dev *dev, u8 port, int reset_qkey_vio</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx4_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,</div><div class='add'>+static int mlx4_ib_modify_port(struct ib_device *ibdev, u32 port, int mask,</div><div class='ctx'> 			       struct ib_port_modify *props)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_ib_dev *mdev = to_mdev(ibdev);</div><div class='hunk'>@@ -1089,7 +1103,8 @@ static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx,</div><div class='ctx'> 	if (!dev-&gt;ib_active)</div><div class='ctx'> 		return -EAGAIN;</div><div class='ctx'> </div><div class='del'>-	if (ibdev-&gt;uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) {</div><div class='add'>+	if (ibdev-&gt;ops.uverbs_abi_ver ==</div><div class='add'>+	    MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) {</div><div class='ctx'> 		resp_v3.qp_tab_size      = dev-&gt;dev-&gt;caps.num_qps;</div><div class='ctx'> 		resp_v3.bf_reg_size      = dev-&gt;dev-&gt;caps.bf_reg_size;</div><div class='ctx'> 		resp_v3.bf_regs_per_page = dev-&gt;dev-&gt;caps.bf_regs_per_page;</div><div class='hunk'>@@ -1111,7 +1126,7 @@ static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx,</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;context-&gt;wqn_ranges_list);</div><div class='ctx'> 	mutex_init(&amp;context-&gt;wqn_ranges_mutex);</div><div class='ctx'> </div><div class='del'>-	if (ibdev-&gt;uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION)</div><div class='add'>+	if (ibdev-&gt;ops.uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION)</div><div class='ctx'> 		err = ib_copy_to_udata(udata, &amp;resp_v3, sizeof(resp_v3));</div><div class='ctx'> 	else</div><div class='ctx'> 		err = ib_copy_to_udata(udata, &amp;resp, sizeof(resp));</div><div class='hunk'>@@ -1144,7 +1159,8 @@ static int mlx4_ib_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)</div><div class='ctx'> 		return rdma_user_mmap_io(context, vma,</div><div class='ctx'> 					 to_mucontext(context)-&gt;uar.pfn,</div><div class='ctx'> 					 PAGE_SIZE,</div><div class='del'>-					 pgprot_noncached(vma-&gt;vm_page_prot));</div><div class='add'>+					 pgprot_noncached(vma-&gt;vm_page_prot),</div><div class='add'>+					 NULL);</div><div class='ctx'> </div><div class='ctx'> 	case 1:</div><div class='ctx'> 		if (dev-&gt;dev-&gt;caps.bf_reg_size == 0)</div><div class='hunk'>@@ -1153,7 +1169,8 @@ static int mlx4_ib_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)</div><div class='ctx'> 			context, vma,</div><div class='ctx'> 			to_mucontext(context)-&gt;uar.pfn +</div><div class='ctx'> 				dev-&gt;dev-&gt;caps.num_uars,</div><div class='del'>-			PAGE_SIZE, pgprot_writecombine(vma-&gt;vm_page_prot));</div><div class='add'>+			PAGE_SIZE, pgprot_writecombine(vma-&gt;vm_page_prot),</div><div class='add'>+			NULL);</div><div class='ctx'> </div><div class='ctx'> 	case 3: {</div><div class='ctx'> 		struct mlx4_clock_params params;</div><div class='hunk'>@@ -1169,7 +1186,8 @@ static int mlx4_ib_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)</div><div class='ctx'> 					    params.bar) +</div><div class='ctx'> 			 params.offset) &gt;&gt;</div><div class='ctx'> 				PAGE_SHIFT,</div><div class='del'>-			PAGE_SIZE, pgprot_noncached(vma-&gt;vm_page_prot));</div><div class='add'>+			PAGE_SIZE, pgprot_noncached(vma-&gt;vm_page_prot),</div><div class='add'>+			NULL);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	default:</div><div class='hunk'>@@ -1194,51 +1212,46 @@ static int mlx4_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void mlx4_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)</div><div class='add'>+static int mlx4_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	mlx4_pd_free(to_mdev(pd-&gt;device)-&gt;dev, to_mpd(pd)-&gt;pdn);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct ib_xrcd *mlx4_ib_alloc_xrcd(struct ib_device *ibdev,</div><div class='del'>-					  struct ib_udata *udata)</div><div class='add'>+static int mlx4_ib_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	struct mlx4_ib_xrcd *xrcd;</div><div class='add'>+	struct mlx4_ib_dev *dev = to_mdev(ibxrcd-&gt;device);</div><div class='add'>+	struct mlx4_ib_xrcd *xrcd = to_mxrcd(ibxrcd);</div><div class='ctx'> 	struct ib_cq_init_attr cq_attr = {};</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	if (!(to_mdev(ibdev)-&gt;dev-&gt;caps.flags &amp; MLX4_DEV_CAP_FLAG_XRC))</div><div class='del'>-		return ERR_PTR(-ENOSYS);</div><div class='del'>-</div><div class='del'>-	xrcd = kmalloc(sizeof *xrcd, GFP_KERNEL);</div><div class='del'>-	if (!xrcd)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='add'>+	if (!(dev-&gt;dev-&gt;caps.flags &amp; MLX4_DEV_CAP_FLAG_XRC))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='del'>-	err = mlx4_xrcd_alloc(to_mdev(ibdev)-&gt;dev, &amp;xrcd-&gt;xrcdn);</div><div class='add'>+	err = mlx4_xrcd_alloc(dev-&gt;dev, &amp;xrcd-&gt;xrcdn);</div><div class='ctx'> 	if (err)</div><div class='del'>-		goto err1;</div><div class='add'>+		return err;</div><div class='ctx'> </div><div class='del'>-	xrcd-&gt;pd = ib_alloc_pd(ibdev, 0);</div><div class='add'>+	xrcd-&gt;pd = ib_alloc_pd(ibxrcd-&gt;device, 0);</div><div class='ctx'> 	if (IS_ERR(xrcd-&gt;pd)) {</div><div class='ctx'> 		err = PTR_ERR(xrcd-&gt;pd);</div><div class='ctx'> 		goto err2;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	cq_attr.cqe = 1;</div><div class='del'>-	xrcd-&gt;cq = ib_create_cq(ibdev, NULL, NULL, xrcd, &amp;cq_attr);</div><div class='add'>+	xrcd-&gt;cq = ib_create_cq(ibxrcd-&gt;device, NULL, NULL, xrcd, &amp;cq_attr);</div><div class='ctx'> 	if (IS_ERR(xrcd-&gt;cq)) {</div><div class='ctx'> 		err = PTR_ERR(xrcd-&gt;cq);</div><div class='ctx'> 		goto err3;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return &amp;xrcd-&gt;ibxrcd;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='ctx'> err3:</div><div class='ctx'> 	ib_dealloc_pd(xrcd-&gt;pd);</div><div class='ctx'> err2:</div><div class='del'>-	mlx4_xrcd_free(to_mdev(ibdev)-&gt;dev, xrcd-&gt;xrcdn);</div><div class='del'>-err1:</div><div class='del'>-	kfree(xrcd);</div><div class='del'>-	return ERR_PTR(err);</div><div class='add'>+	mlx4_xrcd_free(dev-&gt;dev, xrcd-&gt;xrcdn);</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)</div><div class='hunk'>@@ -1246,8 +1259,6 @@ static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)</div><div class='ctx'> 	ib_destroy_cq(to_mxrcd(xrcd)-&gt;cq);</div><div class='ctx'> 	ib_dealloc_pd(to_mxrcd(xrcd)-&gt;pd);</div><div class='ctx'> 	mlx4_xrcd_free(to_mdev(xrcd-&gt;device)-&gt;dev, to_mxrcd(xrcd)-&gt;xrcdn);</div><div class='del'>-	kfree(xrcd);</div><div class='del'>-</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1301,8 +1312,7 @@ int mlx4_ib_add_mc(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp,</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_bh(&amp;mdev-&gt;iboe.lock);</div><div class='ctx'> 	ndev = mdev-&gt;iboe.netdevs[mqp-&gt;port - 1];</div><div class='del'>-	if (ndev)</div><div class='del'>-		dev_hold(ndev);</div><div class='add'>+	dev_hold(ndev);</div><div class='ctx'> 	spin_unlock_bh(&amp;mdev-&gt;iboe.lock);</div><div class='ctx'> </div><div class='ctx'> 	if (ndev) {</div><div class='hunk'>@@ -1478,8 +1488,9 @@ static int __mlx4_ib_create_default_rules(</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; ARRAY_SIZE(pdefault_rules-&gt;rules_create_list); i++) {</div><div class='add'>+		union ib_flow_spec ib_spec = {};</div><div class='ctx'> 		int ret;</div><div class='del'>-		union ib_flow_spec ib_spec;</div><div class='add'>+</div><div class='ctx'> 		switch (pdefault_rules-&gt;rules_create_list[i]) {</div><div class='ctx'> 		case 0:</div><div class='ctx'> 			/* no rule */</div><div class='hunk'>@@ -1520,23 +1531,11 @@ static int __mlx4_ib_create_flow(struct ib_qp *qp, struct ib_flow_attr *flow_att</div><div class='ctx'> 	struct mlx4_net_trans_rule_hw_ctrl *ctrl;</div><div class='ctx'> 	int default_flow;</div><div class='ctx'> </div><div class='del'>-	static const u16 __mlx4_domain[] = {</div><div class='del'>-		[IB_FLOW_DOMAIN_USER] = MLX4_DOMAIN_UVERBS,</div><div class='del'>-		[IB_FLOW_DOMAIN_ETHTOOL] = MLX4_DOMAIN_ETHTOOL,</div><div class='del'>-		[IB_FLOW_DOMAIN_RFS] = MLX4_DOMAIN_RFS,</div><div class='del'>-		[IB_FLOW_DOMAIN_NIC] = MLX4_DOMAIN_NIC,</div><div class='del'>-	};</div><div class='del'>-</div><div class='ctx'> 	if (flow_attr-&gt;priority &gt; MLX4_IB_FLOW_MAX_PRIO) {</div><div class='ctx'> 		pr_err("Invalid priority value %d\n", flow_attr-&gt;priority);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (domain &gt;= IB_FLOW_DOMAIN_NUM) {</div><div class='del'>-		pr_err("Invalid domain value %d\n", domain);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	if (mlx4_map_sw_to_hw_steering_mode(mdev-&gt;dev, flow_type) &lt; 0)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='hunk'>@@ -1545,8 +1544,7 @@ static int __mlx4_ib_create_flow(struct ib_qp *qp, struct ib_flow_attr *flow_att</div><div class='ctx'> 		return PTR_ERR(mailbox);</div><div class='ctx'> 	ctrl = mailbox-&gt;buf;</div><div class='ctx'> </div><div class='del'>-	ctrl-&gt;prio = cpu_to_be16(__mlx4_domain[domain] |</div><div class='del'>-				 flow_attr-&gt;priority);</div><div class='add'>+	ctrl-&gt;prio = cpu_to_be16(domain | flow_attr-&gt;priority);</div><div class='ctx'> 	ctrl-&gt;type = mlx4_map_sw_to_hw_steering_mode(mdev-&gt;dev, flow_type);</div><div class='ctx'> 	ctrl-&gt;port = flow_attr-&gt;port;</div><div class='ctx'> 	ctrl-&gt;qpn = cpu_to_be32(qp-&gt;qp_num);</div><div class='hunk'>@@ -1688,8 +1686,8 @@ static int mlx4_ib_add_dont_trap_rule(struct mlx4_dev *dev,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp,</div><div class='del'>-				    struct ib_flow_attr *flow_attr,</div><div class='del'>-				    int domain, struct ib_udata *udata)</div><div class='add'>+					   struct ib_flow_attr *flow_attr,</div><div class='add'>+					   struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	int err = 0, i = 0, j = 0;</div><div class='ctx'> 	struct mlx4_ib_flow *mflow;</div><div class='hunk'>@@ -1697,9 +1695,6 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp,</div><div class='ctx'> 	struct mlx4_dev *dev = (to_mdev(qp-&gt;device))-&gt;dev;</div><div class='ctx'> 	int is_bonded = mlx4_is_bonded(dev);</div><div class='ctx'> </div><div class='del'>-	if (flow_attr-&gt;port &lt; 1 || flow_attr-&gt;port &gt; qp-&gt;device-&gt;phys_port_cnt)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='ctx'> 	if (flow_attr-&gt;flags &amp; ~IB_FLOW_ATTR_FLAGS_DONT_TRAP)</div><div class='ctx'> 		return ERR_PTR(-EOPNOTSUPP);</div><div class='ctx'> </div><div class='hunk'>@@ -1755,8 +1750,8 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	while (i &lt; ARRAY_SIZE(type) &amp;&amp; type[i]) {</div><div class='del'>-		err = __mlx4_ib_create_flow(qp, flow_attr, domain, type[i],</div><div class='del'>-					    &amp;mflow-&gt;reg_id[i].id);</div><div class='add'>+		err = __mlx4_ib_create_flow(qp, flow_attr, MLX4_DOMAIN_UVERBS,</div><div class='add'>+					    type[i], &amp;mflow-&gt;reg_id[i].id);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto err_create_flow;</div><div class='ctx'> 		if (is_bonded) {</div><div class='hunk'>@@ -1765,7 +1760,7 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp,</div><div class='ctx'> 			 */</div><div class='ctx'> 			flow_attr-&gt;port = 2;</div><div class='ctx'> 			err = __mlx4_ib_create_flow(qp, flow_attr,</div><div class='del'>-						    domain, type[j],</div><div class='add'>+						    MLX4_DOMAIN_UVERBS, type[j],</div><div class='ctx'> 						    &amp;mflow-&gt;reg_id[j].mirror);</div><div class='ctx'> 			flow_attr-&gt;port = 1;</div><div class='ctx'> 			if (err)</div><div class='hunk'>@@ -1964,11 +1959,9 @@ static int mlx4_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)</div><div class='ctx'> 	if (ge) {</div><div class='ctx'> 		spin_lock_bh(&amp;mdev-&gt;iboe.lock);</div><div class='ctx'> 		ndev = ge-&gt;added ? mdev-&gt;iboe.netdevs[ge-&gt;port - 1] : NULL;</div><div class='del'>-		if (ndev)</div><div class='del'>-			dev_hold(ndev);</div><div class='add'>+		dev_hold(ndev);</div><div class='ctx'> 		spin_unlock_bh(&amp;mdev-&gt;iboe.lock);</div><div class='del'>-		if (ndev)</div><div class='del'>-			dev_put(ndev);</div><div class='add'>+		dev_put(ndev);</div><div class='ctx'> 		list_del(&amp;ge-&gt;list);</div><div class='ctx'> 		kfree(ge);</div><div class='ctx'> 	} else</div><div class='hunk'>@@ -1981,8 +1974,8 @@ static int mlx4_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)</div><div class='ctx'> </div><div class='ctx'> static int init_node_data(struct mlx4_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	struct ib_smp *in_mad  = NULL;</div><div class='del'>-	struct ib_smp *out_mad = NULL;</div><div class='add'>+	struct ib_smp *in_mad;</div><div class='add'>+	struct ib_smp *out_mad;</div><div class='ctx'> 	int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;</div><div class='ctx'> 	int err = -ENOMEM;</div><div class='ctx'> </div><div class='hunk'>@@ -1991,7 +1984,7 @@ static int init_node_data(struct mlx4_ib_dev *dev)</div><div class='ctx'> 	if (!in_mad || !out_mad)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	init_query_mad(in_mad);</div><div class='add'>+	ib_init_query_mad(in_mad);</div><div class='ctx'> 	in_mad-&gt;attr_id = IB_SMP_ATTR_NODE_DESC;</div><div class='ctx'> 	if (mlx4_is_master(dev-&gt;dev))</div><div class='ctx'> 		mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW;</div><div class='hunk'>@@ -2022,7 +2015,8 @@ static ssize_t hca_type_show(struct device *device,</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_ib_dev *dev =</div><div class='ctx'> 		rdma_device_to_drv_device(device, struct mlx4_ib_dev, ib_dev);</div><div class='del'>-	return sprintf(buf, "MT%d\n", dev-&gt;dev-&gt;persist-&gt;pdev-&gt;device);</div><div class='add'>+</div><div class='add'>+	return sysfs_emit(buf, "MT%d\n", dev-&gt;dev-&gt;persist-&gt;pdev-&gt;device);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(hca_type);</div><div class='ctx'> </div><div class='hunk'>@@ -2031,7 +2025,8 @@ static ssize_t hw_rev_show(struct device *device,</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_ib_dev *dev =</div><div class='ctx'> 		rdma_device_to_drv_device(device, struct mlx4_ib_dev, ib_dev);</div><div class='del'>-	return sprintf(buf, "%x\n", dev-&gt;dev-&gt;rev_id);</div><div class='add'>+</div><div class='add'>+	return sysfs_emit(buf, "%x\n", dev-&gt;dev-&gt;rev_id);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(hw_rev);</div><div class='ctx'> </div><div class='hunk'>@@ -2041,8 +2036,7 @@ static ssize_t board_id_show(struct device *device,</div><div class='ctx'> 	struct mlx4_ib_dev *dev =</div><div class='ctx'> 		rdma_device_to_drv_device(device, struct mlx4_ib_dev, ib_dev);</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%.*s\n", MLX4_BOARD_ID_LEN,</div><div class='del'>-		       dev-&gt;dev-&gt;board_id);</div><div class='add'>+	return sysfs_emit(buf, "%.*s\n", MLX4_BOARD_ID_LEN, dev-&gt;dev-&gt;board_id);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(board_id);</div><div class='ctx'> </div><div class='hunk'>@@ -2099,23 +2093,35 @@ static const struct diag_counter diag_device_only[] = {</div><div class='ctx'> 	DIAG_COUNTER(rq_num_udsdprd, 0x118),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static struct rdma_hw_stats *mlx4_ib_alloc_hw_stats(struct ib_device *ibdev,</div><div class='del'>-						    u8 port_num)</div><div class='add'>+static struct rdma_hw_stats *</div><div class='add'>+mlx4_ib_alloc_hw_device_stats(struct ib_device *ibdev)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_ib_dev *dev = to_mdev(ibdev);</div><div class='ctx'> 	struct mlx4_ib_diag_counters *diag = dev-&gt;diag_counters;</div><div class='ctx'> </div><div class='del'>-	if (!diag[!!port_num].name)</div><div class='add'>+	if (!diag[0].descs)</div><div class='ctx'> 		return NULL;</div><div class='ctx'> </div><div class='del'>-	return rdma_alloc_hw_stats_struct(diag[!!port_num].name,</div><div class='del'>-					  diag[!!port_num].num_counters,</div><div class='add'>+	return rdma_alloc_hw_stats_struct(diag[0].descs, diag[0].num_counters,</div><div class='add'>+					  RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct rdma_hw_stats *</div><div class='add'>+mlx4_ib_alloc_hw_port_stats(struct ib_device *ibdev, u32 port_num)</div><div class='add'>+{</div><div class='add'>+	struct mlx4_ib_dev *dev = to_mdev(ibdev);</div><div class='add'>+	struct mlx4_ib_diag_counters *diag = dev-&gt;diag_counters;</div><div class='add'>+</div><div class='add'>+	if (!diag[1].descs)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	return rdma_alloc_hw_stats_struct(diag[1].descs, diag[1].num_counters,</div><div class='ctx'> 					  RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int mlx4_ib_get_hw_stats(struct ib_device *ibdev,</div><div class='ctx'> 				struct rdma_hw_stats *stats,</div><div class='del'>-				u8 port, int index)</div><div class='add'>+				u32 port, int index)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_ib_dev *dev = to_mdev(ibdev);</div><div class='ctx'> 	struct mlx4_ib_diag_counters *diag = dev-&gt;diag_counters;</div><div class='hunk'>@@ -2139,10 +2145,8 @@ static int mlx4_ib_get_hw_stats(struct ib_device *ibdev,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __mlx4_ib_alloc_diag_counters(struct mlx4_ib_dev *ibdev,</div><div class='del'>-					 const char ***name,</div><div class='del'>-					 u32 **offset,</div><div class='del'>-					 u32 *num,</div><div class='del'>-					 bool port)</div><div class='add'>+					 struct rdma_stat_desc **pdescs,</div><div class='add'>+					 u32 **offset, u32 *num, bool port)</div><div class='ctx'> {</div><div class='ctx'> 	u32 num_counters;</div><div class='ctx'> </div><div class='hunk'>@@ -2154,53 +2158,59 @@ static int __mlx4_ib_alloc_diag_counters(struct mlx4_ib_dev *ibdev,</div><div class='ctx'> 	if (!port)</div><div class='ctx'> 		num_counters += ARRAY_SIZE(diag_device_only);</div><div class='ctx'> </div><div class='del'>-	*name = kcalloc(num_counters, sizeof(**name), GFP_KERNEL);</div><div class='del'>-	if (!*name)</div><div class='add'>+	*pdescs = kcalloc(num_counters, sizeof(struct rdma_stat_desc),</div><div class='add'>+			  GFP_KERNEL);</div><div class='add'>+	if (!*pdescs)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	*offset = kcalloc(num_counters, sizeof(**offset), GFP_KERNEL);</div><div class='ctx'> 	if (!*offset)</div><div class='del'>-		goto err_name;</div><div class='add'>+		goto err;</div><div class='ctx'> </div><div class='ctx'> 	*num = num_counters;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-err_name:</div><div class='del'>-	kfree(*name);</div><div class='add'>+err:</div><div class='add'>+	kfree(*pdescs);</div><div class='ctx'> 	return -ENOMEM;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void mlx4_ib_fill_diag_counters(struct mlx4_ib_dev *ibdev,</div><div class='del'>-				       const char **name,</div><div class='del'>-				       u32 *offset,</div><div class='del'>-				       bool port)</div><div class='add'>+				       struct rdma_stat_desc *descs,</div><div class='add'>+				       u32 *offset, bool port)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='ctx'> 	int j;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0, j = 0; i &lt; ARRAY_SIZE(diag_basic); i++, j++) {</div><div class='del'>-		name[i] = diag_basic[i].name;</div><div class='add'>+		descs[i].name = diag_basic[i].name;</div><div class='ctx'> 		offset[i] = diag_basic[i].offset;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (ibdev-&gt;dev-&gt;caps.flags2 &amp; MLX4_DEV_CAP_FLAG2_DIAG_PER_PORT) {</div><div class='ctx'> 		for (i = 0; i &lt; ARRAY_SIZE(diag_ext); i++, j++) {</div><div class='del'>-			name[j] = diag_ext[i].name;</div><div class='add'>+			descs[j].name = diag_ext[i].name;</div><div class='ctx'> 			offset[j] = diag_ext[i].offset;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (!port) {</div><div class='ctx'> 		for (i = 0; i &lt; ARRAY_SIZE(diag_device_only); i++, j++) {</div><div class='del'>-			name[j] = diag_device_only[i].name;</div><div class='add'>+			descs[j].name = diag_device_only[i].name;</div><div class='ctx'> 			offset[j] = diag_device_only[i].offset;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static const struct ib_device_ops mlx4_ib_hw_stats_ops = {</div><div class='del'>-	.alloc_hw_stats = mlx4_ib_alloc_hw_stats,</div><div class='add'>+	.alloc_hw_device_stats = mlx4_ib_alloc_hw_device_stats,</div><div class='add'>+	.alloc_hw_port_stats = mlx4_ib_alloc_hw_port_stats,</div><div class='add'>+	.get_hw_stats = mlx4_ib_get_hw_stats,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops mlx4_ib_hw_stats_ops1 = {</div><div class='add'>+	.alloc_hw_device_stats = mlx4_ib_alloc_hw_device_stats,</div><div class='ctx'> 	.get_hw_stats = mlx4_ib_get_hw_stats,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -2216,17 +2226,24 @@ static int mlx4_ib_alloc_diag_counters(struct mlx4_ib_dev *ibdev)</div><div class='ctx'> 		return 0;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; MLX4_DIAG_COUNTERS_TYPES; i++) {</div><div class='del'>-		/* i == 1 means we are building port counters */</div><div class='del'>-		if (i &amp;&amp; !per_port)</div><div class='del'>-			continue;</div><div class='add'>+		/*</div><div class='add'>+		 * i == 1 means we are building port counters, set a different</div><div class='add'>+		 * stats ops without port stats callback.</div><div class='add'>+		 */</div><div class='add'>+		if (i &amp;&amp; !per_port) {</div><div class='add'>+			ib_set_device_ops(&amp;ibdev-&gt;ib_dev,</div><div class='add'>+					  &amp;mlx4_ib_hw_stats_ops1);</div><div class='add'>+</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='del'>-		ret = __mlx4_ib_alloc_diag_counters(ibdev, &amp;diag[i].name,</div><div class='add'>+		ret = __mlx4_ib_alloc_diag_counters(ibdev, &amp;diag[i].descs,</div><div class='ctx'> 						    &amp;diag[i].offset,</div><div class='ctx'> 						    &amp;diag[i].num_counters, i);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto err_alloc;</div><div class='ctx'> </div><div class='del'>-		mlx4_ib_fill_diag_counters(ibdev, diag[i].name,</div><div class='add'>+		mlx4_ib_fill_diag_counters(ibdev, diag[i].descs,</div><div class='ctx'> 					   diag[i].offset, i);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -2236,7 +2253,7 @@ static int mlx4_ib_alloc_diag_counters(struct mlx4_ib_dev *ibdev)</div><div class='ctx'> </div><div class='ctx'> err_alloc:</div><div class='ctx'> 	if (i) {</div><div class='del'>-		kfree(diag[i - 1].name);</div><div class='add'>+		kfree(diag[i - 1].descs);</div><div class='ctx'> 		kfree(diag[i - 1].offset);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -2249,7 +2266,7 @@ static void mlx4_ib_diag_cleanup(struct mlx4_ib_dev *ibdev)</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; MLX4_DIAG_COUNTERS_TYPES; i++) {</div><div class='ctx'> 		kfree(ibdev-&gt;diag_counters[i].offset);</div><div class='del'>-		kfree(ibdev-&gt;diag_counters[i].name);</div><div class='add'>+		kfree(ibdev-&gt;diag_counters[i].descs);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -2262,10 +2279,7 @@ static void mlx4_ib_update_qps(struct mlx4_ib_dev *ibdev,</div><div class='ctx'> 	u64 release_mac = MLX4_IB_INVALID_MAC;</div><div class='ctx'> 	struct mlx4_ib_qp *qp;</div><div class='ctx'> </div><div class='del'>-	read_lock(&amp;dev_base_lock);</div><div class='del'>-	new_smac = mlx4_mac_to_u64(dev-&gt;dev_addr);</div><div class='del'>-	read_unlock(&amp;dev_base_lock);</div><div class='del'>-</div><div class='add'>+	new_smac = ether_addr_to_u64(dev-&gt;dev_addr);</div><div class='ctx'> 	atomic64_set(&amp;ibdev-&gt;iboe.mac[port - 1], new_smac);</div><div class='ctx'> </div><div class='ctx'> 	/* no need for update QP1 and mac registration in non-SRIOV */</div><div class='hunk'>@@ -2311,61 +2325,54 @@ unlock:</div><div class='ctx'> 	mutex_unlock(&amp;ibdev-&gt;qp1_proxy_lock[port - 1]);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void mlx4_ib_scan_netdevs(struct mlx4_ib_dev *ibdev,</div><div class='del'>-				 struct net_device *dev,</div><div class='del'>-				 unsigned long event)</div><div class='add'>+static void mlx4_ib_scan_netdev(struct mlx4_ib_dev *ibdev,</div><div class='add'>+				struct net_device *dev,</div><div class='add'>+				unsigned long event)</div><div class='ctx'> </div><div class='ctx'> {</div><div class='del'>-	struct mlx4_ib_iboe *iboe;</div><div class='del'>-	int update_qps_port = -1;</div><div class='del'>-	int port;</div><div class='add'>+	struct mlx4_ib_iboe *iboe = &amp;ibdev-&gt;iboe;</div><div class='ctx'> </div><div class='ctx'> 	ASSERT_RTNL();</div><div class='ctx'> </div><div class='del'>-	iboe = &amp;ibdev-&gt;iboe;</div><div class='add'>+	if (dev-&gt;dev.parent != ibdev-&gt;ib_dev.dev.parent)</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_bh(&amp;iboe-&gt;lock);</div><div class='del'>-	mlx4_foreach_ib_transport_port(port, ibdev-&gt;dev) {</div><div class='ctx'> </div><div class='del'>-		iboe-&gt;netdevs[port - 1] =</div><div class='del'>-			mlx4_get_protocol_dev(ibdev-&gt;dev, MLX4_PROT_ETH, port);</div><div class='add'>+	iboe-&gt;netdevs[dev-&gt;dev_port] = event != NETDEV_UNREGISTER ? dev : NULL;</div><div class='ctx'> </div><div class='del'>-		if (dev == iboe-&gt;netdevs[port - 1] &amp;&amp;</div><div class='del'>-		    (event == NETDEV_CHANGEADDR || event == NETDEV_REGISTER ||</div><div class='del'>-		     event == NETDEV_UP || event == NETDEV_CHANGE))</div><div class='del'>-			update_qps_port = port;</div><div class='add'>+	spin_unlock_bh(&amp;iboe-&gt;lock);</div><div class='ctx'> </div><div class='del'>-		if (dev == iboe-&gt;netdevs[port - 1] &amp;&amp;</div><div class='del'>-		    (event == NETDEV_UP || event == NETDEV_DOWN)) {</div><div class='del'>-			enum ib_port_state port_state;</div><div class='del'>-			struct ib_event ibev = { };</div><div class='add'>+	if (event == NETDEV_CHANGEADDR || event == NETDEV_REGISTER)</div><div class='add'>+		mlx4_ib_update_qps(ibdev, dev, dev-&gt;dev_port + 1);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-			if (ib_get_cached_port_state(&amp;ibdev-&gt;ib_dev, port,</div><div class='del'>-						     &amp;port_state))</div><div class='del'>-				continue;</div><div class='add'>+static void mlx4_ib_port_event(struct ib_device *ibdev, struct net_device *ndev,</div><div class='add'>+			       unsigned long event)</div><div class='add'>+{</div><div class='add'>+	struct mlx4_ib_dev *mlx4_ibdev =</div><div class='add'>+		container_of(ibdev, struct mlx4_ib_dev, ib_dev);</div><div class='add'>+	struct mlx4_ib_iboe *iboe = &amp;mlx4_ibdev-&gt;iboe;</div><div class='ctx'> </div><div class='del'>-			if (event == NETDEV_UP &amp;&amp;</div><div class='del'>-			    (port_state != IB_PORT_ACTIVE ||</div><div class='del'>-			     iboe-&gt;last_port_state[port - 1] != IB_PORT_DOWN))</div><div class='del'>-				continue;</div><div class='del'>-			if (event == NETDEV_DOWN &amp;&amp;</div><div class='del'>-			    (port_state != IB_PORT_DOWN ||</div><div class='del'>-			     iboe-&gt;last_port_state[port - 1] != IB_PORT_ACTIVE))</div><div class='del'>-				continue;</div><div class='del'>-			iboe-&gt;last_port_state[port - 1] = port_state;</div><div class='add'>+	if (!net_eq(dev_net(ndev), &amp;init_net))</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-			ibev.device = &amp;ibdev-&gt;ib_dev;</div><div class='del'>-			ibev.element.port_num = port;</div><div class='del'>-			ibev.event = event == NETDEV_UP ? IB_EVENT_PORT_ACTIVE :</div><div class='del'>-							  IB_EVENT_PORT_ERR;</div><div class='del'>-			ib_dispatch_event(&amp;ibev);</div><div class='del'>-		}</div><div class='add'>+	ASSERT_RTNL();</div><div class='add'>+</div><div class='add'>+	if (ndev-&gt;dev.parent != mlx4_ibdev-&gt;ib_dev.dev.parent)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	spin_lock_bh(&amp;iboe-&gt;lock);</div><div class='add'>+</div><div class='add'>+	iboe-&gt;netdevs[ndev-&gt;dev_port] = event != NETDEV_UNREGISTER ? ndev : NULL;</div><div class='add'>+</div><div class='add'>+	if (event == NETDEV_UP || event == NETDEV_DOWN)</div><div class='add'>+		ib_dispatch_port_state_event(&amp;mlx4_ibdev-&gt;ib_dev, ndev);</div><div class='ctx'> </div><div class='del'>-	}</div><div class='ctx'> 	spin_unlock_bh(&amp;iboe-&gt;lock);</div><div class='ctx'> </div><div class='del'>-	if (update_qps_port &gt; 0)</div><div class='del'>-		mlx4_ib_update_qps(ibdev, dev, update_qps_port);</div><div class='add'>+	if (event == NETDEV_UP || event == NETDEV_CHANGE)</div><div class='add'>+		mlx4_ib_update_qps(mlx4_ibdev, ndev, ndev-&gt;dev_port + 1);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int mlx4_ib_netdev_event(struct notifier_block *this,</div><div class='hunk'>@@ -2378,7 +2385,7 @@ static int mlx4_ib_netdev_event(struct notifier_block *this,</div><div class='ctx'> 		return NOTIFY_DONE;</div><div class='ctx'> </div><div class='ctx'> 	ibdev = container_of(this, struct mlx4_ib_dev, iboe.nb);</div><div class='del'>-	mlx4_ib_scan_netdevs(ibdev, dev, event);</div><div class='add'>+	mlx4_ib_scan_netdev(ibdev, dev, event);</div><div class='ctx'> </div><div class='ctx'> 	return NOTIFY_DONE;</div><div class='ctx'> }</div><div class='hunk'>@@ -2466,7 +2473,7 @@ static void mlx4_ib_free_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev)</div><div class='ctx'> 	ibdev-&gt;eq_table = NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx4_port_immutable(struct ib_device *ibdev, u8 port_num,</div><div class='add'>+static int mlx4_port_immutable(struct ib_device *ibdev, u32 port_num,</div><div class='ctx'> 			       struct ib_port_immutable *immutable)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr attr;</div><div class='hunk'>@@ -2509,6 +2516,10 @@ static void get_fw_ver_str(struct ib_device *device, char *str)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static const struct ib_device_ops mlx4_ib_dev_ops = {</div><div class='add'>+	.owner = THIS_MODULE,</div><div class='add'>+	.driver_id = RDMA_DRIVER_MLX4,</div><div class='add'>+	.uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION,</div><div class='add'>+</div><div class='ctx'> 	.add_gid = mlx4_ib_add_gid,</div><div class='ctx'> 	.alloc_mr = mlx4_ib_alloc_mr,</div><div class='ctx'> 	.alloc_pd = mlx4_ib_alloc_pd,</div><div class='hunk'>@@ -2527,6 +2538,7 @@ static const struct ib_device_ops mlx4_ib_dev_ops = {</div><div class='ctx'> 	.destroy_qp = mlx4_ib_destroy_qp,</div><div class='ctx'> 	.destroy_srq = mlx4_ib_destroy_srq,</div><div class='ctx'> 	.detach_mcast = mlx4_ib_mcg_detach,</div><div class='add'>+	.device_group = &amp;mlx4_attr_group,</div><div class='ctx'> 	.disassociate_ucontext = mlx4_ib_disassociate_ucontext,</div><div class='ctx'> 	.drain_rq = mlx4_ib_drain_rq,</div><div class='ctx'> 	.drain_sq = mlx4_ib_drain_sq,</div><div class='hunk'>@@ -2558,9 +2570,12 @@ static const struct ib_device_ops mlx4_ib_dev_ops = {</div><div class='ctx'> 	.req_notify_cq = mlx4_ib_arm_cq,</div><div class='ctx'> 	.rereg_user_mr = mlx4_ib_rereg_user_mr,</div><div class='ctx'> 	.resize_cq = mlx4_ib_resize_cq,</div><div class='add'>+	.report_port_event = mlx4_ib_port_event,</div><div class='ctx'> </div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_ah, mlx4_ib_ah, ibah),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_cq, mlx4_ib_cq, ibcq),</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_pd, mlx4_ib_pd, ibpd),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_qp, mlx4_ib_qp, ibqp),</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_srq, mlx4_ib_srq, ibsrq),</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx4_ib_ucontext, ibucontext),</div><div class='ctx'> };</div><div class='hunk'>@@ -2571,23 +2586,23 @@ static const struct ib_device_ops mlx4_ib_dev_wq_ops = {</div><div class='ctx'> 	.destroy_rwq_ind_table = mlx4_ib_destroy_rwq_ind_table,</div><div class='ctx'> 	.destroy_wq = mlx4_ib_destroy_wq,</div><div class='ctx'> 	.modify_wq = mlx4_ib_modify_wq,</div><div class='del'>-};</div><div class='ctx'> </div><div class='del'>-static const struct ib_device_ops mlx4_ib_dev_fmr_ops = {</div><div class='del'>-	.alloc_fmr = mlx4_ib_fmr_alloc,</div><div class='del'>-	.dealloc_fmr = mlx4_ib_fmr_dealloc,</div><div class='del'>-	.map_phys_fmr = mlx4_ib_map_phys_fmr,</div><div class='del'>-	.unmap_fmr = mlx4_ib_unmap_fmr,</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mlx4_ib_rwq_ind_table,</div><div class='add'>+			   ib_rwq_ind_tbl),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static const struct ib_device_ops mlx4_ib_dev_mw_ops = {</div><div class='ctx'> 	.alloc_mw = mlx4_ib_alloc_mw,</div><div class='ctx'> 	.dealloc_mw = mlx4_ib_dealloc_mw,</div><div class='add'>+</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_mw, mlx4_ib_mw, ibmw),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static const struct ib_device_ops mlx4_ib_dev_xrc_ops = {</div><div class='ctx'> 	.alloc_xrcd = mlx4_ib_alloc_xrcd,</div><div class='ctx'> 	.dealloc_xrcd = mlx4_ib_dealloc_xrcd,</div><div class='add'>+</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx4_ib_xrcd, ibxrcd),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static const struct ib_device_ops mlx4_ib_dev_fs_ops = {</div><div class='hunk'>@@ -2595,8 +2610,11 @@ static const struct ib_device_ops mlx4_ib_dev_fs_ops = {</div><div class='ctx'> 	.destroy_flow = mlx4_ib_destroy_flow,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static void *mlx4_ib_add(struct mlx4_dev *dev)</div><div class='add'>+static int mlx4_ib_probe(struct auxiliary_device *adev,</div><div class='add'>+			 const struct auxiliary_device_id *id)</div><div class='ctx'> {</div><div class='add'>+	struct mlx4_adev *madev = container_of(adev, struct mlx4_adev, adev);</div><div class='add'>+	struct mlx4_dev *dev = madev-&gt;mdev;</div><div class='ctx'> 	struct mlx4_ib_dev *ibdev;</div><div class='ctx'> 	int num_ports = 0;</div><div class='ctx'> 	int i, j;</div><div class='hunk'>@@ -2606,7 +2624,7 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)</div><div class='ctx'> 	int num_req_counters;</div><div class='ctx'> 	int allocated;</div><div class='ctx'> 	u32 counter_index;</div><div class='del'>-	struct counter_index *new_counter_index = NULL;</div><div class='add'>+	struct counter_index *new_counter_index;</div><div class='ctx'> </div><div class='ctx'> 	pr_info_once("%s", mlx4_ib_version);</div><div class='ctx'> </div><div class='hunk'>@@ -2616,33 +2634,36 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)</div><div class='ctx'> </div><div class='ctx'> 	/* No point in registering a device with no ports... */</div><div class='ctx'> 	if (num_ports == 0)</div><div class='del'>-		return NULL;</div><div class='add'>+		return -ENODEV;</div><div class='ctx'> </div><div class='ctx'> 	ibdev = ib_alloc_device(mlx4_ib_dev, ib_dev);</div><div class='ctx'> 	if (!ibdev) {</div><div class='ctx'> 		dev_err(&amp;dev-&gt;persist-&gt;pdev-&gt;dev,</div><div class='ctx'> 			"Device struct alloc failed\n");</div><div class='del'>-		return NULL;</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	iboe = &amp;ibdev-&gt;iboe;</div><div class='ctx'> </div><div class='del'>-	if (mlx4_pd_alloc(dev, &amp;ibdev-&gt;priv_pdn))</div><div class='add'>+	err = mlx4_pd_alloc(dev, &amp;ibdev-&gt;priv_pdn);</div><div class='add'>+	if (err)</div><div class='ctx'> 		goto err_dealloc;</div><div class='ctx'> </div><div class='del'>-	if (mlx4_uar_alloc(dev, &amp;ibdev-&gt;priv_uar))</div><div class='add'>+	err = mlx4_uar_alloc(dev, &amp;ibdev-&gt;priv_uar);</div><div class='add'>+	if (err)</div><div class='ctx'> 		goto err_pd;</div><div class='ctx'> </div><div class='ctx'> 	ibdev-&gt;uar_map = ioremap((phys_addr_t) ibdev-&gt;priv_uar.pfn &lt;&lt; PAGE_SHIFT,</div><div class='ctx'> 				 PAGE_SIZE);</div><div class='del'>-	if (!ibdev-&gt;uar_map)</div><div class='add'>+	if (!ibdev-&gt;uar_map) {</div><div class='add'>+		err = -ENOMEM;</div><div class='ctx'> 		goto err_uar;</div><div class='add'>+	}</div><div class='ctx'> 	MLX4_INIT_DOORBELL_LOCK(&amp;ibdev-&gt;uar_lock);</div><div class='ctx'> </div><div class='ctx'> 	ibdev-&gt;dev = dev;</div><div class='ctx'> 	ibdev-&gt;bond_next_port	= 0;</div><div class='ctx'> </div><div class='del'>-	ibdev-&gt;ib_dev.owner		= THIS_MODULE;</div><div class='ctx'> 	ibdev-&gt;ib_dev.node_type		= RDMA_NODE_IB_CA;</div><div class='ctx'> 	ibdev-&gt;ib_dev.local_dma_lkey	= dev-&gt;caps.reserved_lkey;</div><div class='ctx'> 	ibdev-&gt;num_ports		= num_ports;</div><div class='hunk'>@@ -2651,89 +2672,38 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)</div><div class='ctx'> 	ibdev-&gt;ib_dev.num_comp_vectors	= dev-&gt;caps.num_comp_vectors;</div><div class='ctx'> 	ibdev-&gt;ib_dev.dev.parent	= &amp;dev-&gt;persist-&gt;pdev-&gt;dev;</div><div class='ctx'> </div><div class='del'>-	if (dev-&gt;caps.userspace_caps)</div><div class='del'>-		ibdev-&gt;ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION;</div><div class='del'>-	else</div><div class='del'>-		ibdev-&gt;ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION;</div><div class='del'>-</div><div class='del'>-	ibdev-&gt;ib_dev.uverbs_cmd_mask	=</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_GET_CONTEXT)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_DEVICE)	|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_PORT)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_ALLOC_PD)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DEALLOC_PD)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_REG_MR)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_REREG_MR)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DEREG_MR)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL)	|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_CQ)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_RESIZE_CQ)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_CQ)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_QP)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_MODIFY_QP)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_QP)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_QP)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_ATTACH_MCAST)	|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DETACH_MCAST)	|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_SRQ)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_MODIFY_SRQ)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_SRQ)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_SRQ)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_XSRQ)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_OPEN_QP);</div><div class='del'>-</div><div class='ctx'> 	ib_set_device_ops(&amp;ibdev-&gt;ib_dev, &amp;mlx4_ib_dev_ops);</div><div class='del'>-	ibdev-&gt;ib_dev.uverbs_ex_cmd_mask |=</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_MODIFY_CQ) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_QUERY_DEVICE) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_CREATE_CQ) |</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_CREATE_QP);</div><div class='ctx'> </div><div class='ctx'> 	if ((dev-&gt;caps.flags2 &amp; MLX4_DEV_CAP_FLAG2_RSS) &amp;&amp;</div><div class='ctx'> 	    ((mlx4_ib_port_link_layer(&amp;ibdev-&gt;ib_dev, 1) ==</div><div class='ctx'> 	    IB_LINK_LAYER_ETHERNET) ||</div><div class='ctx'> 	    (mlx4_ib_port_link_layer(&amp;ibdev-&gt;ib_dev, 2) ==</div><div class='del'>-	    IB_LINK_LAYER_ETHERNET))) {</div><div class='del'>-		ibdev-&gt;ib_dev.uverbs_ex_cmd_mask |=</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_CREATE_WQ)	  |</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_MODIFY_WQ)	  |</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_DESTROY_WQ)	  |</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) |</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL);</div><div class='add'>+	    IB_LINK_LAYER_ETHERNET)))</div><div class='ctx'> 		ib_set_device_ops(&amp;ibdev-&gt;ib_dev, &amp;mlx4_ib_dev_wq_ops);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!mlx4_is_slave(ibdev-&gt;dev))</div><div class='del'>-		ib_set_device_ops(&amp;ibdev-&gt;ib_dev, &amp;mlx4_ib_dev_fmr_ops);</div><div class='ctx'> </div><div class='ctx'> 	if (dev-&gt;caps.flags &amp; MLX4_DEV_CAP_FLAG_MEM_WINDOW ||</div><div class='del'>-	    dev-&gt;caps.bmme_flags &amp; MLX4_BMME_FLAG_TYPE_2_WIN) {</div><div class='del'>-		ibdev-&gt;ib_dev.uverbs_cmd_mask |=</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_ALLOC_MW) |</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_DEALLOC_MW);</div><div class='add'>+	    dev-&gt;caps.bmme_flags &amp; MLX4_BMME_FLAG_TYPE_2_WIN)</div><div class='ctx'> 		ib_set_device_ops(&amp;ibdev-&gt;ib_dev, &amp;mlx4_ib_dev_mw_ops);</div><div class='del'>-	}</div><div class='ctx'> </div><div class='ctx'> 	if (dev-&gt;caps.flags &amp; MLX4_DEV_CAP_FLAG_XRC) {</div><div class='del'>-		ibdev-&gt;ib_dev.uverbs_cmd_mask |=</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_OPEN_XRCD) |</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_CLOSE_XRCD);</div><div class='ctx'> 		ib_set_device_ops(&amp;ibdev-&gt;ib_dev, &amp;mlx4_ib_dev_xrc_ops);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (check_flow_steering_support(dev)) {</div><div class='ctx'> 		ibdev-&gt;steering_support = MLX4_STEERING_MODE_DEVICE_MANAGED;</div><div class='del'>-		ibdev-&gt;ib_dev.uverbs_ex_cmd_mask	|=</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_CREATE_FLOW) |</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_DESTROY_FLOW);</div><div class='ctx'> 		ib_set_device_ops(&amp;ibdev-&gt;ib_dev, &amp;mlx4_ib_dev_fs_ops);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (!dev-&gt;caps.userspace_caps)</div><div class='add'>+		ibdev-&gt;ib_dev.ops.uverbs_abi_ver =</div><div class='add'>+			MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION;</div><div class='add'>+</div><div class='ctx'> 	mlx4_ib_alloc_eqs(dev, ibdev);</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_init(&amp;iboe-&gt;lock);</div><div class='ctx'> </div><div class='del'>-	if (init_node_data(ibdev))</div><div class='add'>+	err = init_node_data(ibdev);</div><div class='add'>+	if (err)</div><div class='ctx'> 		goto err_map;</div><div class='ctx'> 	mlx4_init_sl2vl_tbl(ibdev);</div><div class='ctx'> </div><div class='hunk'>@@ -2765,6 +2735,7 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)</div><div class='ctx'> 		new_counter_index = kmalloc(sizeof(*new_counter_index),</div><div class='ctx'> 					    GFP_KERNEL);</div><div class='ctx'> 		if (!new_counter_index) {</div><div class='add'>+			err = -ENOMEM;</div><div class='ctx'> 			if (allocated)</div><div class='ctx'> 				mlx4_counter_free(ibdev-&gt;dev, counter_index);</div><div class='ctx'> 			goto err_counter;</div><div class='hunk'>@@ -2782,8 +2753,10 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)</div><div class='ctx'> 			new_counter_index =</div><div class='ctx'> 					kmalloc(sizeof(struct counter_index),</div><div class='ctx'> 						GFP_KERNEL);</div><div class='del'>-			if (!new_counter_index)</div><div class='add'>+			if (!new_counter_index) {</div><div class='add'>+				err = -ENOMEM;</div><div class='ctx'> 				goto err_counter;</div><div class='add'>+			}</div><div class='ctx'> 			new_counter_index-&gt;index = counter_index;</div><div class='ctx'> 			new_counter_index-&gt;allocated = 0;</div><div class='ctx'> 			list_add_tail(&amp;new_counter_index-&gt;list,</div><div class='hunk'>@@ -2810,12 +2783,12 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto err_counter;</div><div class='ctx'> </div><div class='del'>-		ibdev-&gt;ib_uc_qpns_bitmap =</div><div class='del'>-			kmalloc_array(BITS_TO_LONGS(ibdev-&gt;steer_qpn_count),</div><div class='del'>-				      sizeof(long),</div><div class='del'>-				      GFP_KERNEL);</div><div class='del'>-		if (!ibdev-&gt;ib_uc_qpns_bitmap)</div><div class='add'>+		ibdev-&gt;ib_uc_qpns_bitmap = bitmap_alloc(ibdev-&gt;steer_qpn_count,</div><div class='add'>+							GFP_KERNEL);</div><div class='add'>+		if (!ibdev-&gt;ib_uc_qpns_bitmap) {</div><div class='add'>+			err = -ENOMEM;</div><div class='ctx'> 			goto err_steer_qp_release;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='ctx'> 		if (dev-&gt;caps.flags2 &amp; MLX4_DEV_CAP_FLAG2_DMFS_IPOIB) {</div><div class='ctx'> 			bitmap_zero(ibdev-&gt;ib_uc_qpns_bitmap,</div><div class='hunk'>@@ -2835,18 +2808,21 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)</div><div class='ctx'> 	for (j = 1; j &lt;= ibdev-&gt;dev-&gt;caps.num_ports; j++)</div><div class='ctx'> 		atomic64_set(&amp;iboe-&gt;mac[j - 1], ibdev-&gt;dev-&gt;caps.def_mac[j]);</div><div class='ctx'> </div><div class='del'>-	if (mlx4_ib_alloc_diag_counters(ibdev))</div><div class='add'>+	err = mlx4_ib_alloc_diag_counters(ibdev);</div><div class='add'>+	if (err)</div><div class='ctx'> 		goto err_steer_free_bitmap;</div><div class='ctx'> </div><div class='del'>-	rdma_set_device_sysfs_group(&amp;ibdev-&gt;ib_dev, &amp;mlx4_attr_group);</div><div class='del'>-	ibdev-&gt;ib_dev.driver_id = RDMA_DRIVER_MLX4;</div><div class='del'>-	if (ib_register_device(&amp;ibdev-&gt;ib_dev, "mlx4_%d"))</div><div class='add'>+	err = ib_register_device(&amp;ibdev-&gt;ib_dev, "mlx4_%d",</div><div class='add'>+				 &amp;dev-&gt;persist-&gt;pdev-&gt;dev);</div><div class='add'>+	if (err)</div><div class='ctx'> 		goto err_diag_counters;</div><div class='ctx'> </div><div class='del'>-	if (mlx4_ib_mad_init(ibdev))</div><div class='add'>+	err = mlx4_ib_mad_init(ibdev);</div><div class='add'>+	if (err)</div><div class='ctx'> 		goto err_reg;</div><div class='ctx'> </div><div class='del'>-	if (mlx4_ib_init_sriov(ibdev))</div><div class='add'>+	err = mlx4_ib_init_sriov(ibdev);</div><div class='add'>+	if (err)</div><div class='ctx'> 		goto err_mad;</div><div class='ctx'> </div><div class='ctx'> 	if (!iboe-&gt;nb.notifier_call) {</div><div class='hunk'>@@ -2880,7 +2856,14 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)</div><div class='ctx'> 				do_slave_init(ibdev, j, 1);</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='del'>-	return ibdev;</div><div class='add'>+</div><div class='add'>+	/* register mlx4 core notifier */</div><div class='add'>+	ibdev-&gt;mlx_nb.notifier_call = mlx4_ib_event;</div><div class='add'>+	err = mlx4_register_event_notifier(dev, &amp;ibdev-&gt;mlx_nb);</div><div class='add'>+	WARN(err, "failed to register mlx4 event notifier (%d)", err);</div><div class='add'>+</div><div class='add'>+	auxiliary_set_drvdata(adev, ibdev);</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='ctx'> err_notif:</div><div class='ctx'> 	if (ibdev-&gt;iboe.nb.notifier_call) {</div><div class='hunk'>@@ -2902,7 +2885,7 @@ err_diag_counters:</div><div class='ctx'> 	mlx4_ib_diag_cleanup(ibdev);</div><div class='ctx'> </div><div class='ctx'> err_steer_free_bitmap:</div><div class='del'>-	kfree(ibdev-&gt;ib_uc_qpns_bitmap);</div><div class='add'>+	bitmap_free(ibdev-&gt;ib_uc_qpns_bitmap);</div><div class='ctx'> </div><div class='ctx'> err_steer_qp_release:</div><div class='ctx'> 	mlx4_qp_release_range(dev, ibdev-&gt;steer_qpn_base,</div><div class='hunk'>@@ -2924,7 +2907,7 @@ err_pd:</div><div class='ctx'> err_dealloc:</div><div class='ctx'> 	ib_dealloc_device(&amp;ibdev-&gt;ib_dev);</div><div class='ctx'> </div><div class='del'>-	return NULL;</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx4_ib_steer_qp_alloc(struct mlx4_ib_dev *dev, int count, int *qpn)</div><div class='hunk'>@@ -2964,7 +2947,7 @@ int mlx4_ib_steer_qp_reg(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp,</div><div class='ctx'> {</div><div class='ctx'> 	int err;</div><div class='ctx'> 	size_t flow_size;</div><div class='del'>-	struct ib_flow_attr *flow = NULL;</div><div class='add'>+	struct ib_flow_attr *flow;</div><div class='ctx'> 	struct ib_flow_spec_ib *ib_spec;</div><div class='ctx'> </div><div class='ctx'> 	if (is_attach) {</div><div class='hunk'>@@ -2982,41 +2965,44 @@ int mlx4_ib_steer_qp_reg(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp,</div><div class='ctx'> 		/* Add an empty rule for IB L2 */</div><div class='ctx'> 		memset(&amp;ib_spec-&gt;mask, 0, sizeof(ib_spec-&gt;mask));</div><div class='ctx'> </div><div class='del'>-		err = __mlx4_ib_create_flow(&amp;mqp-&gt;ibqp, flow,</div><div class='del'>-					    IB_FLOW_DOMAIN_NIC,</div><div class='del'>-					    MLX4_FS_REGULAR,</div><div class='del'>-					    &amp;mqp-&gt;reg_id);</div><div class='del'>-	} else {</div><div class='del'>-		err = __mlx4_ib_destroy_flow(mdev-&gt;dev, mqp-&gt;reg_id);</div><div class='add'>+		err = __mlx4_ib_create_flow(&amp;mqp-&gt;ibqp, flow, MLX4_DOMAIN_NIC,</div><div class='add'>+					    MLX4_FS_REGULAR, &amp;mqp-&gt;reg_id);</div><div class='add'>+		kfree(flow);</div><div class='add'>+		return err;</div><div class='ctx'> 	}</div><div class='del'>-	kfree(flow);</div><div class='del'>-	return err;</div><div class='add'>+	</div><div class='add'>+	return __mlx4_ib_destroy_flow(mdev-&gt;dev, mqp-&gt;reg_id);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr)</div><div class='add'>+static void mlx4_ib_remove(struct auxiliary_device *adev)</div><div class='ctx'> {</div><div class='del'>-	struct mlx4_ib_dev *ibdev = ibdev_ptr;</div><div class='add'>+	struct mlx4_adev *madev = container_of(adev, struct mlx4_adev, adev);</div><div class='add'>+	struct mlx4_dev *dev = madev-&gt;mdev;</div><div class='add'>+	struct mlx4_ib_dev *ibdev = auxiliary_get_drvdata(adev);</div><div class='ctx'> 	int p;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='add'>+	mlx4_unregister_event_notifier(dev, &amp;ibdev-&gt;mlx_nb);</div><div class='add'>+</div><div class='ctx'> 	mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB)</div><div class='ctx'> 		devlink_port_type_clear(mlx4_get_devlink_port(dev, i));</div><div class='ctx'> 	ibdev-&gt;ib_active = false;</div><div class='ctx'> 	flush_workqueue(wq);</div><div class='ctx'> </div><div class='del'>-	mlx4_ib_close_sriov(ibdev);</div><div class='del'>-	mlx4_ib_mad_cleanup(ibdev);</div><div class='del'>-	ib_unregister_device(&amp;ibdev-&gt;ib_dev);</div><div class='del'>-	mlx4_ib_diag_cleanup(ibdev);</div><div class='ctx'> 	if (ibdev-&gt;iboe.nb.notifier_call) {</div><div class='ctx'> 		if (unregister_netdevice_notifier(&amp;ibdev-&gt;iboe.nb))</div><div class='ctx'> 			pr_warn("failure unregistering notifier\n");</div><div class='ctx'> 		ibdev-&gt;iboe.nb.notifier_call = NULL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	mlx4_ib_close_sriov(ibdev);</div><div class='add'>+	mlx4_ib_mad_cleanup(ibdev);</div><div class='add'>+	ib_unregister_device(&amp;ibdev-&gt;ib_dev);</div><div class='add'>+	mlx4_ib_diag_cleanup(ibdev);</div><div class='add'>+</div><div class='ctx'> 	mlx4_qp_release_range(dev, ibdev-&gt;steer_qpn_base,</div><div class='ctx'> 			      ibdev-&gt;steer_qpn_count);</div><div class='del'>-	kfree(ibdev-&gt;ib_uc_qpns_bitmap);</div><div class='add'>+	bitmap_free(ibdev-&gt;ib_uc_qpns_bitmap);</div><div class='ctx'> </div><div class='ctx'> 	iounmap(ibdev-&gt;uar_map);</div><div class='ctx'> 	for (p = 0; p &lt; ibdev-&gt;num_ports; ++p)</div><div class='hunk'>@@ -3034,7 +3020,7 @@ static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr)</div><div class='ctx'> </div><div class='ctx'> static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init)</div><div class='ctx'> {</div><div class='del'>-	struct mlx4_ib_demux_work **dm = NULL;</div><div class='add'>+	struct mlx4_ib_demux_work **dm;</div><div class='ctx'> 	struct mlx4_dev *dev = ibdev-&gt;dev;</div><div class='ctx'> 	int i;</div><div class='ctx'> 	unsigned long flags;</div><div class='hunk'>@@ -3218,11 +3204,13 @@ void mlx4_sched_ib_sl2vl_update_work(struct mlx4_ib_dev *ibdev,</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,</div><div class='del'>-			  enum mlx4_dev_event event, unsigned long param)</div><div class='add'>+static int mlx4_ib_event(struct notifier_block *this, unsigned long event,</div><div class='add'>+			 void *param)</div><div class='ctx'> {</div><div class='add'>+	struct mlx4_ib_dev *ibdev =</div><div class='add'>+		container_of(this, struct mlx4_ib_dev, mlx_nb);</div><div class='add'>+	struct mlx4_dev *dev = ibdev-&gt;dev;</div><div class='ctx'> 	struct ib_event ibev;</div><div class='del'>-	struct mlx4_ib_dev *ibdev = to_mdev((struct ib_device *) ibdev_ptr);</div><div class='ctx'> 	struct mlx4_eqe *eqe = NULL;</div><div class='ctx'> 	struct ib_event_work *ew;</div><div class='ctx'> 	int p = 0;</div><div class='hunk'>@@ -3232,22 +3220,28 @@ static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,</div><div class='ctx'> 	    (event == MLX4_DEV_EVENT_PORT_DOWN))) {</div><div class='ctx'> 		ew = kmalloc(sizeof(*ew), GFP_ATOMIC);</div><div class='ctx'> 		if (!ew)</div><div class='del'>-			return;</div><div class='add'>+			return NOTIFY_DONE;</div><div class='ctx'> 		INIT_WORK(&amp;ew-&gt;work, handle_bonded_port_state_event);</div><div class='ctx'> 		ew-&gt;ib_dev = ibdev;</div><div class='ctx'> 		queue_work(wq, &amp;ew-&gt;work);</div><div class='del'>-		return;</div><div class='add'>+		return NOTIFY_DONE;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (event == MLX4_DEV_EVENT_PORT_MGMT_CHANGE)</div><div class='add'>+	switch (event) {</div><div class='add'>+	case MLX4_DEV_EVENT_CATASTROPHIC_ERROR:</div><div class='add'>+		break;</div><div class='add'>+	case MLX4_DEV_EVENT_PORT_MGMT_CHANGE:</div><div class='ctx'> 		eqe = (struct mlx4_eqe *)param;</div><div class='del'>-	else</div><div class='del'>-		p = (int) param;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		p = *(int *)param;</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	switch (event) {</div><div class='ctx'> 	case MLX4_DEV_EVENT_PORT_UP:</div><div class='ctx'> 		if (p &gt; ibdev-&gt;num_ports)</div><div class='del'>-			return;</div><div class='add'>+			return NOTIFY_DONE;</div><div class='ctx'> 		if (!mlx4_is_slave(dev) &amp;&amp;</div><div class='ctx'> 		    rdma_port_get_link_layer(&amp;ibdev-&gt;ib_dev, p) ==</div><div class='ctx'> 			IB_LINK_LAYER_INFINIBAND) {</div><div class='hunk'>@@ -3262,7 +3256,7 @@ static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,</div><div class='ctx'> </div><div class='ctx'> 	case MLX4_DEV_EVENT_PORT_DOWN:</div><div class='ctx'> 		if (p &gt; ibdev-&gt;num_ports)</div><div class='del'>-			return;</div><div class='add'>+			return NOTIFY_DONE;</div><div class='ctx'> 		ibev.event = IB_EVENT_PORT_ERR;</div><div class='ctx'> 		break;</div><div class='ctx'> </div><div class='hunk'>@@ -3275,7 +3269,7 @@ static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,</div><div class='ctx'> 	case MLX4_DEV_EVENT_PORT_MGMT_CHANGE:</div><div class='ctx'> 		ew = kmalloc(sizeof *ew, GFP_ATOMIC);</div><div class='ctx'> 		if (!ew)</div><div class='del'>-			break;</div><div class='add'>+			return NOTIFY_DONE;</div><div class='ctx'> </div><div class='ctx'> 		INIT_WORK(&amp;ew-&gt;work, handle_port_mgmt_change_event);</div><div class='ctx'> 		memcpy(&amp;ew-&gt;ib_eqe, eqe, sizeof *eqe);</div><div class='hunk'>@@ -3285,7 +3279,7 @@ static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,</div><div class='ctx'> 			queue_work(wq, &amp;ew-&gt;work);</div><div class='ctx'> 		else</div><div class='ctx'> 			handle_port_mgmt_change_event(&amp;ew-&gt;work);</div><div class='del'>-		return;</div><div class='add'>+		return NOTIFY_DONE;</div><div class='ctx'> </div><div class='ctx'> 	case MLX4_DEV_EVENT_SLAVE_INIT:</div><div class='ctx'> 		/* here, p is the slave id */</div><div class='hunk'>@@ -3301,7 +3295,7 @@ static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,</div><div class='ctx'> 								       1);</div><div class='ctx'> 			}</div><div class='ctx'> 		}</div><div class='del'>-		return;</div><div class='add'>+		return NOTIFY_DONE;</div><div class='ctx'> </div><div class='ctx'> 	case MLX4_DEV_EVENT_SLAVE_SHUTDOWN:</div><div class='ctx'> 		if (mlx4_is_master(dev)) {</div><div class='hunk'>@@ -3317,22 +3311,33 @@ static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,</div><div class='ctx'> 		}</div><div class='ctx'> 		/* here, p is the slave id */</div><div class='ctx'> 		do_slave_init(ibdev, p, 0);</div><div class='del'>-		return;</div><div class='add'>+		return NOTIFY_DONE;</div><div class='ctx'> </div><div class='ctx'> 	default:</div><div class='del'>-		return;</div><div class='add'>+		return NOTIFY_DONE;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ibev.device	      = ibdev_ptr;</div><div class='add'>+	ibev.device	      = &amp;ibdev-&gt;ib_dev;</div><div class='ctx'> 	ibev.element.port_num = mlx4_is_bonded(ibdev-&gt;dev) ? 1 : (u8)p;</div><div class='ctx'> </div><div class='ctx'> 	ib_dispatch_event(&amp;ibev);</div><div class='add'>+	return NOTIFY_DONE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct mlx4_interface mlx4_ib_interface = {</div><div class='del'>-	.add		= mlx4_ib_add,</div><div class='del'>-	.remove		= mlx4_ib_remove,</div><div class='del'>-	.event		= mlx4_ib_event,</div><div class='add'>+static const struct auxiliary_device_id mlx4_ib_id_table[] = {</div><div class='add'>+	{ .name = MLX4_ADEV_NAME ".ib" },</div><div class='add'>+	{},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+MODULE_DEVICE_TABLE(auxiliary, mlx4_ib_id_table);</div><div class='add'>+</div><div class='add'>+static struct mlx4_adrv mlx4_ib_adrv = {</div><div class='add'>+	.adrv = {</div><div class='add'>+		.name	= "ib",</div><div class='add'>+		.probe	= mlx4_ib_probe,</div><div class='add'>+		.remove	= mlx4_ib_remove,</div><div class='add'>+		.id_table = mlx4_ib_id_table,</div><div class='add'>+	},</div><div class='ctx'> 	.protocol	= MLX4_PROT_IB_IPV6,</div><div class='ctx'> 	.flags		= MLX4_INTFF_BONDING</div><div class='ctx'> };</div><div class='hunk'>@@ -3345,11 +3350,19 @@ static int __init mlx4_ib_init(void)</div><div class='ctx'> 	if (!wq)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	err = mlx4_ib_mcg_init();</div><div class='add'>+	err = mlx4_ib_qp_event_init();</div><div class='add'>+	if (err)</div><div class='add'>+		goto clean_qp_event;</div><div class='add'>+</div><div class='add'>+	err = mlx4_ib_cm_init();</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto clean_wq;</div><div class='ctx'> </div><div class='del'>-	err = mlx4_register_interface(&amp;mlx4_ib_interface);</div><div class='add'>+	err = mlx4_ib_mcg_init();</div><div class='add'>+	if (err)</div><div class='add'>+		goto clean_cm;</div><div class='add'>+</div><div class='add'>+	err = mlx4_register_auxiliary_driver(&amp;mlx4_ib_adrv);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto clean_mcg;</div><div class='ctx'> </div><div class='hunk'>@@ -3358,15 +3371,23 @@ static int __init mlx4_ib_init(void)</div><div class='ctx'> clean_mcg:</div><div class='ctx'> 	mlx4_ib_mcg_destroy();</div><div class='ctx'> </div><div class='add'>+clean_cm:</div><div class='add'>+	mlx4_ib_cm_destroy();</div><div class='add'>+</div><div class='ctx'> clean_wq:</div><div class='add'>+	mlx4_ib_qp_event_cleanup();</div><div class='add'>+</div><div class='add'>+clean_qp_event:</div><div class='ctx'> 	destroy_workqueue(wq);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void __exit mlx4_ib_cleanup(void)</div><div class='ctx'> {</div><div class='del'>-	mlx4_unregister_interface(&amp;mlx4_ib_interface);</div><div class='add'>+	mlx4_unregister_auxiliary_driver(&amp;mlx4_ib_adrv);</div><div class='ctx'> 	mlx4_ib_mcg_destroy();</div><div class='add'>+	mlx4_ib_cm_destroy();</div><div class='add'>+	mlx4_ib_qp_event_cleanup();</div><div class='ctx'> 	destroy_workqueue(wq);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/hw/mlx4/mcg.c b/drivers/infiniband/hw/mlx4/mcg.c<br/>index d844831179cf..e279e69b9a51 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/mcg.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/mcg.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/mcg.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx4/mcg.c</a></div><div class='hunk'>@@ -43,7 +43,7 @@</div><div class='ctx'> </div><div class='ctx'> #define MAX_VFS		80</div><div class='ctx'> #define MAX_PEND_REQS_PER_FUNC 4</div><div class='del'>-#define MAD_TIMEOUT_MS	2000</div><div class='add'>+#define MAD_TIMEOUT_SEC	2</div><div class='ctx'> </div><div class='ctx'> #define mcg_warn(fmt, arg...)	pr_warn("MCG WARNING: " fmt, ##arg)</div><div class='ctx'> #define mcg_error(fmt, arg...)	pr_err(fmt, ##arg)</div><div class='hunk'>@@ -270,7 +270,7 @@ static int send_join_to_wire(struct mcast_group *group, struct ib_sa_mad *sa_mad</div><div class='ctx'> 	if (!ret) {</div><div class='ctx'> 		/* calls mlx4_ib_mcg_timeout_handler */</div><div class='ctx'> 		queue_delayed_work(group-&gt;demux-&gt;mcg_wq, &amp;group-&gt;timeout_work,</div><div class='del'>-				msecs_to_jiffies(MAD_TIMEOUT_MS));</div><div class='add'>+				   secs_to_jiffies(MAD_TIMEOUT_SEC));</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -309,7 +309,7 @@ static int send_leave_to_wire(struct mcast_group *group, u8 join_state)</div><div class='ctx'> 	if (!ret) {</div><div class='ctx'> 		/* calls mlx4_ib_mcg_timeout_handler */</div><div class='ctx'> 		queue_delayed_work(group-&gt;demux-&gt;mcg_wq, &amp;group-&gt;timeout_work,</div><div class='del'>-				msecs_to_jiffies(MAD_TIMEOUT_MS));</div><div class='add'>+				   secs_to_jiffies(MAD_TIMEOUT_SEC));</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return ret;</div><div class='hunk'>@@ -944,7 +944,7 @@ int mlx4_ib_mcg_multiplex_handler(struct ib_device *ibdev, int port,</div><div class='ctx'> 	switch (sa_mad-&gt;mad_hdr.method) {</div><div class='ctx'> 	case IB_MGMT_METHOD_SET:</div><div class='ctx'> 		may_create = 1;</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case IB_SA_METHOD_DELETE:</div><div class='ctx'> 		req = kzalloc(sizeof *req, GFP_KERNEL);</div><div class='ctx'> 		if (!req)</div><div class='hunk'>@@ -988,53 +988,63 @@ int mlx4_ib_mcg_multiplex_handler(struct ib_device *ibdev, int port,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t sysfs_show_group(struct device *dev,</div><div class='del'>-		struct device_attribute *attr, char *buf)</div><div class='add'>+				struct device_attribute *attr, char *buf)</div><div class='ctx'> {</div><div class='ctx'> 	struct mcast_group *group =</div><div class='ctx'> 		container_of(attr, struct mcast_group, dentry);</div><div class='ctx'> 	struct mcast_req *req = NULL;</div><div class='del'>-	char pending_str[40];</div><div class='ctx'> 	char state_str[40];</div><div class='del'>-	ssize_t len = 0;</div><div class='del'>-	int f;</div><div class='add'>+	char pending_str[40];</div><div class='add'>+	int len;</div><div class='add'>+	int i;</div><div class='add'>+	u32 hoplimit;</div><div class='ctx'> </div><div class='ctx'> 	if (group-&gt;state == MCAST_IDLE)</div><div class='del'>-		sprintf(state_str, "%s", get_state_string(group-&gt;state));</div><div class='add'>+		scnprintf(state_str, sizeof(state_str), "%s",</div><div class='add'>+			  get_state_string(group-&gt;state));</div><div class='ctx'> 	else</div><div class='del'>-		sprintf(state_str, "%s(TID=0x%llx)",</div><div class='del'>-				get_state_string(group-&gt;state),</div><div class='del'>-				be64_to_cpu(group-&gt;last_req_tid));</div><div class='add'>+		scnprintf(state_str, sizeof(state_str), "%s(TID=0x%llx)",</div><div class='add'>+			  get_state_string(group-&gt;state),</div><div class='add'>+			  be64_to_cpu(group-&gt;last_req_tid));</div><div class='add'>+</div><div class='ctx'> 	if (list_empty(&amp;group-&gt;pending_list)) {</div><div class='del'>-		sprintf(pending_str, "No");</div><div class='add'>+		scnprintf(pending_str, sizeof(pending_str), "No");</div><div class='ctx'> 	} else {</div><div class='del'>-		req = list_first_entry(&amp;group-&gt;pending_list, struct mcast_req, group_list);</div><div class='del'>-		sprintf(pending_str, "Yes(TID=0x%llx)",</div><div class='del'>-				be64_to_cpu(req-&gt;sa_mad.mad_hdr.tid));</div><div class='add'>+		req = list_first_entry(&amp;group-&gt;pending_list, struct mcast_req,</div><div class='add'>+				       group_list);</div><div class='add'>+		scnprintf(pending_str, sizeof(pending_str), "Yes(TID=0x%llx)",</div><div class='add'>+			  be64_to_cpu(req-&gt;sa_mad.mad_hdr.tid));</div><div class='ctx'> 	}</div><div class='del'>-	len += sprintf(buf + len, "%1d [%02d,%02d,%02d] %4d %4s %5s     ",</div><div class='del'>-			group-&gt;rec.scope_join_state &amp; 0xf,</div><div class='del'>-			group-&gt;members[2], group-&gt;members[1], group-&gt;members[0],</div><div class='del'>-			atomic_read(&amp;group-&gt;refcount),</div><div class='del'>-			pending_str,</div><div class='del'>-			state_str);</div><div class='del'>-	for (f = 0; f &lt; MAX_VFS; ++f)</div><div class='del'>-		if (group-&gt;func[f].state == MCAST_MEMBER)</div><div class='del'>-			len += sprintf(buf + len, "%d[%1x] ",</div><div class='del'>-					f, group-&gt;func[f].join_state);</div><div class='del'>-</div><div class='del'>-	len += sprintf(buf + len, "\t\t(%4hx %4x %2x %2x %2x %2x %2x "</div><div class='del'>-		"%4x %4x %2x %2x)\n",</div><div class='del'>-		be16_to_cpu(group-&gt;rec.pkey),</div><div class='del'>-		be32_to_cpu(group-&gt;rec.qkey),</div><div class='del'>-		(group-&gt;rec.mtusel_mtu &amp; 0xc0) &gt;&gt; 6,</div><div class='del'>-		group-&gt;rec.mtusel_mtu &amp; 0x3f,</div><div class='del'>-		group-&gt;rec.tclass,</div><div class='del'>-		(group-&gt;rec.ratesel_rate &amp; 0xc0) &gt;&gt; 6,</div><div class='del'>-		group-&gt;rec.ratesel_rate &amp; 0x3f,</div><div class='del'>-		(be32_to_cpu(group-&gt;rec.sl_flowlabel_hoplimit) &amp; 0xf0000000) &gt;&gt; 28,</div><div class='del'>-		(be32_to_cpu(group-&gt;rec.sl_flowlabel_hoplimit) &amp; 0x0fffff00) &gt;&gt; 8,</div><div class='del'>-		be32_to_cpu(group-&gt;rec.sl_flowlabel_hoplimit) &amp; 0x000000ff,</div><div class='del'>-		group-&gt;rec.proxy_join);</div><div class='add'>+</div><div class='add'>+	len = sysfs_emit(buf, "%1d [%02d,%02d,%02d] %4d %4s %5s     ",</div><div class='add'>+			 group-&gt;rec.scope_join_state &amp; 0xf,</div><div class='add'>+			 group-&gt;members[2],</div><div class='add'>+			 group-&gt;members[1],</div><div class='add'>+			 group-&gt;members[0],</div><div class='add'>+			 atomic_read(&amp;group-&gt;refcount),</div><div class='add'>+			 pending_str,</div><div class='add'>+			 state_str);</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; MAX_VFS; i++) {</div><div class='add'>+		if (group-&gt;func[i].state == MCAST_MEMBER)</div><div class='add'>+			len += sysfs_emit_at(buf, len, "%d[%1x] ", i,</div><div class='add'>+					     group-&gt;func[i].join_state);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	hoplimit = be32_to_cpu(group-&gt;rec.sl_flowlabel_hoplimit);</div><div class='add'>+	len += sysfs_emit_at(buf, len,</div><div class='add'>+			     "\t\t(%4hx %4x %2x %2x %2x %2x %2x %4x %4x %2x %2x)\n",</div><div class='add'>+			     be16_to_cpu(group-&gt;rec.pkey),</div><div class='add'>+			     be32_to_cpu(group-&gt;rec.qkey),</div><div class='add'>+			     (group-&gt;rec.mtusel_mtu &amp; 0xc0) &gt;&gt; 6,</div><div class='add'>+			     (group-&gt;rec.mtusel_mtu &amp; 0x3f),</div><div class='add'>+			     group-&gt;rec.tclass,</div><div class='add'>+			     (group-&gt;rec.ratesel_rate &amp; 0xc0) &gt;&gt; 6,</div><div class='add'>+			     (group-&gt;rec.ratesel_rate &amp; 0x3f),</div><div class='add'>+			     (hoplimit &amp; 0xf0000000) &gt;&gt; 28,</div><div class='add'>+			     (hoplimit &amp; 0x0fffff00) &gt;&gt; 8,</div><div class='add'>+			     (hoplimit &amp; 0x000000ff),</div><div class='add'>+			     group-&gt;rec.proxy_join);</div><div class='ctx'> </div><div class='ctx'> 	return len;</div><div class='ctx'> }</div><div class='hunk'>@@ -1081,7 +1091,7 @@ static void _mlx4_ib_mcg_port_cleanup(struct mlx4_ib_demux_ctx *ctx, int destroy</div><div class='ctx'> 	for (i = 0; i &lt; MAX_VFS; ++i)</div><div class='ctx'> 		clean_vf_mcast(ctx, i);</div><div class='ctx'> </div><div class='del'>-	end = jiffies + msecs_to_jiffies(MAD_TIMEOUT_MS + 3000);</div><div class='add'>+	end = jiffies + secs_to_jiffies(MAD_TIMEOUT_SEC + 3);</div><div class='ctx'> 	do {</div><div class='ctx'> 		count = 0;</div><div class='ctx'> 		mutex_lock(&amp;ctx-&gt;mcg_table_lock);</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h<br/>index 26897102057d..5df5b955114e 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/mlx4_ib.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/mlx4_ib.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/mlx4_ib.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx4/mlx4_ib.h</a></div><div class='hunk'>@@ -38,6 +38,7 @@</div><div class='ctx'> #include &lt;linux/list.h&gt;</div><div class='ctx'> #include &lt;linux/mutex.h&gt;</div><div class='ctx'> #include &lt;linux/idr.h&gt;</div><div class='add'>+#include &lt;linux/notifier.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include &lt;rdma/ib_verbs.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_umem.h&gt;</div><div class='hunk'>@@ -146,11 +147,6 @@ struct mlx4_ib_mw {</div><div class='ctx'> 	struct mlx4_mw		mmw;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct mlx4_ib_fmr {</div><div class='del'>-	struct ib_fmr           ibfmr;</div><div class='del'>-	struct mlx4_fmr         mfmr;</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> #define MAX_REGS_PER_FLOW 2</div><div class='ctx'> </div><div class='ctx'> struct mlx4_flow_reg_id {</div><div class='hunk'>@@ -238,7 +234,8 @@ enum mlx4_ib_mad_ifc_flags {</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum {</div><div class='del'>-	MLX4_NUM_TUNNEL_BUFS		= 256,</div><div class='add'>+	MLX4_NUM_TUNNEL_BUFS		= 512,</div><div class='add'>+	MLX4_NUM_WIRE_BUFS		= 2048,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct mlx4_ib_tunnel_header {</div><div class='hunk'>@@ -303,6 +300,26 @@ struct mlx4_ib_rss {</div><div class='ctx'> 	u8			rss_key[MLX4_EN_RSS_KEY_SIZE];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+enum {</div><div class='add'>+	/*</div><div class='add'>+	 * Largest possible UD header: send with GRH and immediate</div><div class='add'>+	 * data plus 18 bytes for an Ethernet header with VLAN/802.1Q</div><div class='add'>+	 * tag.  (LRH would only use 8 bytes, so Ethernet is the</div><div class='add'>+	 * biggest case)</div><div class='add'>+	 */</div><div class='add'>+	MLX4_IB_UD_HEADER_SIZE		= 82,</div><div class='add'>+	MLX4_IB_LSO_HEADER_SPARE	= 128,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mlx4_ib_sqp {</div><div class='add'>+	int pkey_index;</div><div class='add'>+	u32 qkey;</div><div class='add'>+	u32 send_psn;</div><div class='add'>+	struct ib_ud_header ud_header;</div><div class='add'>+	u8 header_buf[MLX4_IB_UD_HEADER_SIZE];</div><div class='add'>+	struct ib_qp *roce_v2_gsi;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct mlx4_ib_qp {</div><div class='ctx'> 	union {</div><div class='ctx'> 		struct ib_qp	ibqp;</div><div class='hunk'>@@ -348,7 +365,10 @@ struct mlx4_ib_qp {</div><div class='ctx'> 	struct mlx4_wqn_range	*wqn_range;</div><div class='ctx'> 	/* Number of RSS QP parents that uses this WQ */</div><div class='ctx'> 	u32			rss_usecnt;</div><div class='del'>-	struct mlx4_ib_rss	*rss_ctx;</div><div class='add'>+	union {</div><div class='add'>+		struct mlx4_ib_rss *rss_ctx;</div><div class='add'>+		struct mlx4_ib_sqp *sqp;</div><div class='add'>+	};</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct mlx4_ib_srq {</div><div class='hunk'>@@ -371,6 +391,10 @@ struct mlx4_ib_ah {</div><div class='ctx'> 	union mlx4_ext_av       av;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct mlx4_ib_rwq_ind_table {</div><div class='add'>+	struct ib_rwq_ind_table ib_rwq_ind_tbl;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> /****************************************/</div><div class='ctx'> /* alias guid support */</div><div class='ctx'> /****************************************/</div><div class='hunk'>@@ -406,7 +430,7 @@ struct mlx4_sriov_alias_guid_port_rec_det {</div><div class='ctx'> 	struct mlx4_sriov_alias_guid_info_rec_det all_rec_per_port[NUM_ALIAS_GUID_REC_IN_PORT];</div><div class='ctx'> 	struct workqueue_struct *wq;</div><div class='ctx'> 	struct delayed_work alias_guid_work;</div><div class='del'>-	u8 port;</div><div class='add'>+	u32 port;</div><div class='ctx'> 	u32 state_flags;</div><div class='ctx'> 	struct mlx4_sriov_alias_guid *parent;</div><div class='ctx'> 	struct list_head cb_list;</div><div class='hunk'>@@ -459,6 +483,7 @@ struct mlx4_ib_demux_pv_ctx {</div><div class='ctx'> 	struct ib_pd *pd;</div><div class='ctx'> 	struct work_struct work;</div><div class='ctx'> 	struct workqueue_struct *wq;</div><div class='add'>+	struct workqueue_struct *wi_wq;</div><div class='ctx'> 	struct mlx4_ib_demux_pv_qp qp[2];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -466,6 +491,7 @@ struct mlx4_ib_demux_ctx {</div><div class='ctx'> 	struct ib_device *ib_dev;</div><div class='ctx'> 	int port;</div><div class='ctx'> 	struct workqueue_struct *wq;</div><div class='add'>+	struct workqueue_struct *wi_wq;</div><div class='ctx'> 	struct workqueue_struct *ud_wq;</div><div class='ctx'> 	spinlock_t ud_lock;</div><div class='ctx'> 	atomic64_t subnet_prefix;</div><div class='hunk'>@@ -497,6 +523,7 @@ struct mlx4_ib_sriov {</div><div class='ctx'> 	spinlock_t id_map_lock;</div><div class='ctx'> 	struct rb_root sl_id_map;</div><div class='ctx'> 	struct list_head cm_list;</div><div class='add'>+	struct xarray xa_rej_tmout;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct gid_cache_context {</div><div class='hunk'>@@ -508,6 +535,7 @@ struct gid_entry {</div><div class='ctx'> 	union ib_gid	gid;</div><div class='ctx'> 	enum ib_gid_type gid_type;</div><div class='ctx'> 	struct gid_cache_context *ctx;</div><div class='add'>+	u16 vlan_id;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct mlx4_port_gid_table {</div><div class='hunk'>@@ -574,7 +602,7 @@ struct mlx4_ib_counters {</div><div class='ctx'> #define MLX4_DIAG_COUNTERS_TYPES 2</div><div class='ctx'> </div><div class='ctx'> struct mlx4_ib_diag_counters {</div><div class='del'>-	const char **name;</div><div class='add'>+	struct rdma_stat_desc *descs;</div><div class='ctx'> 	u32 *offset;</div><div class='ctx'> 	u32 num_counters;</div><div class='ctx'> };</div><div class='hunk'>@@ -617,6 +645,7 @@ struct mlx4_ib_dev {</div><div class='ctx'> 	spinlock_t		reset_flow_resource_lock;</div><div class='ctx'> 	struct list_head		qp_list;</div><div class='ctx'> 	struct mlx4_ib_diag_counters diag_counters[MLX4_DIAG_COUNTERS_TYPES];</div><div class='add'>+	struct notifier_block	mlx_nb;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct ib_event_work {</div><div class='hunk'>@@ -630,7 +659,7 @@ struct mlx4_ib_qp_tunnel_init_attr {</div><div class='ctx'> 	struct ib_qp_init_attr init_attr;</div><div class='ctx'> 	int slave;</div><div class='ctx'> 	enum ib_qp_type proxy_qp_type;</div><div class='del'>-	u8 port;</div><div class='add'>+	u32 port;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct mlx4_uverbs_ex_query_device {</div><div class='hunk'>@@ -638,6 +667,9 @@ struct mlx4_uverbs_ex_query_device {</div><div class='ctx'> 	__u32 reserved;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+/* 4k - 4G */</div><div class='add'>+#define MLX4_PAGE_SIZE_SUPPORTED	((unsigned long)GENMASK_ULL(31, 12))</div><div class='add'>+</div><div class='ctx'> static inline struct mlx4_ib_dev *to_mdev(struct ib_device *ibdev)</div><div class='ctx'> {</div><div class='ctx'> 	return container_of(ibdev, struct mlx4_ib_dev, ib_dev);</div><div class='hunk'>@@ -678,11 +710,6 @@ static inline struct mlx4_ib_mw *to_mmw(struct ib_mw *ibmw)</div><div class='ctx'> 	return container_of(ibmw, struct mlx4_ib_mw, ibmw);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline struct mlx4_ib_fmr *to_mfmr(struct ib_fmr *ibfmr)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibfmr, struct mlx4_ib_fmr, ibfmr);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static inline struct mlx4_ib_flow *to_mflow(struct ib_flow *ibflow)</div><div class='ctx'> {</div><div class='ctx'> 	return container_of(ibflow, struct mlx4_ib_flow, ibflow);</div><div class='hunk'>@@ -732,46 +759,47 @@ int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt,</div><div class='ctx'> 			   struct ib_umem *umem);</div><div class='ctx'> struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='ctx'> 				  u64 virt_addr, int access_flags,</div><div class='add'>+				  struct ib_dmah *dmah,</div><div class='ctx'> 				  struct ib_udata *udata);</div><div class='ctx'> int mlx4_ib_dereg_mr(struct ib_mr *mr, struct ib_udata *udata);</div><div class='del'>-struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,</div><div class='del'>-			       struct ib_udata *udata);</div><div class='add'>+int mlx4_ib_alloc_mw(struct ib_mw *mw, struct ib_udata *udata);</div><div class='ctx'> int mlx4_ib_dealloc_mw(struct ib_mw *mw);</div><div class='ctx'> struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,</div><div class='del'>-			       u32 max_num_sg, struct ib_udata *udata);</div><div class='add'>+			       u32 max_num_sg);</div><div class='ctx'> int mlx4_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,</div><div class='ctx'> 		      unsigned int *sg_offset);</div><div class='ctx'> int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);</div><div class='ctx'> int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);</div><div class='del'>-struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,</div><div class='del'>-				const struct ib_cq_init_attr *attr,</div><div class='del'>-				struct ib_udata *udata);</div><div class='add'>+int mlx4_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		      struct uverbs_attr_bundle *attrs);</div><div class='ctx'> int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);</div><div class='ctx'> int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);</div><div class='ctx'> int mlx4_ib_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);</div><div class='ctx'> void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq);</div><div class='ctx'> void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq);</div><div class='ctx'> </div><div class='del'>-int mlx4_ib_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,</div><div class='add'>+int mlx4_ib_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr,</div><div class='ctx'> 		      struct ib_udata *udata);</div><div class='ctx'> int mlx4_ib_create_ah_slave(struct ib_ah *ah, struct rdma_ah_attr *ah_attr,</div><div class='ctx'> 			    int slave_sgid_index, u8 *s_mac, u16 vlan_tag);</div><div class='ctx'> int mlx4_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);</div><div class='del'>-void mlx4_ib_destroy_ah(struct ib_ah *ah, u32 flags);</div><div class='add'>+static inline int mlx4_ib_destroy_ah(struct ib_ah *ah, u32 flags)</div><div class='add'>+{</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='ctx'> int mlx4_ib_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *init_attr,</div><div class='ctx'> 		       struct ib_udata *udata);</div><div class='ctx'> int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,</div><div class='ctx'> 		       enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);</div><div class='ctx'> int mlx4_ib_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);</div><div class='del'>-void mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata);</div><div class='add'>+int mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata);</div><div class='ctx'> void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index);</div><div class='ctx'> int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,</div><div class='ctx'> 			  const struct ib_recv_wr **bad_wr);</div><div class='ctx'> </div><div class='del'>-struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,</div><div class='del'>-				struct ib_qp_init_attr *init_attr,</div><div class='del'>-				struct ib_udata *udata);</div><div class='add'>+int mlx4_ib_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr,</div><div class='add'>+		      struct ib_udata *udata);</div><div class='ctx'> int mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata);</div><div class='ctx'> void mlx4_ib_drain_sq(struct ib_qp *qp);</div><div class='ctx'> void mlx4_ib_drain_rq(struct ib_qp *qp);</div><div class='hunk'>@@ -787,31 +815,24 @@ int mlx4_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,</div><div class='ctx'> int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int mad_ifc_flags,</div><div class='ctx'> 		 int port, const struct ib_wc *in_wc, const struct ib_grh *in_grh,</div><div class='ctx'> 		 const void *in_mad, void *response_mad);</div><div class='del'>-int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags,	u8 port_num,</div><div class='add'>+int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num,</div><div class='ctx'> 			const struct ib_wc *in_wc, const struct ib_grh *in_grh,</div><div class='del'>-			const struct ib_mad_hdr *in, size_t in_mad_size,</div><div class='del'>-			struct ib_mad_hdr *out, size_t *out_mad_size,</div><div class='del'>-			u16 *out_mad_pkey_index);</div><div class='add'>+			const struct ib_mad *in, struct ib_mad *out,</div><div class='add'>+			size_t *out_mad_size, u16 *out_mad_pkey_index);</div><div class='ctx'> int mlx4_ib_mad_init(struct mlx4_ib_dev *dev);</div><div class='ctx'> void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev);</div><div class='ctx'> </div><div class='del'>-struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int mr_access_flags,</div><div class='del'>-				  struct ib_fmr_attr *fmr_attr);</div><div class='del'>-int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, int npages,</div><div class='del'>-			 u64 iova);</div><div class='del'>-int mlx4_ib_unmap_fmr(struct list_head *fmr_list);</div><div class='del'>-int mlx4_ib_fmr_dealloc(struct ib_fmr *fmr);</div><div class='del'>-int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port,</div><div class='add'>+int __mlx4_ib_query_port(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 			 struct ib_port_attr *props, int netw_view);</div><div class='del'>-int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,</div><div class='add'>+int __mlx4_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index,</div><div class='ctx'> 			 u16 *pkey, int netw_view);</div><div class='ctx'> </div><div class='del'>-int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index,</div><div class='add'>+int __mlx4_ib_query_gid(struct ib_device *ibdev, u32 port, int index,</div><div class='ctx'> 			union ib_gid *gid, int netw_view);</div><div class='ctx'> </div><div class='ctx'> static inline bool mlx4_ib_ah_grh_present(struct mlx4_ib_ah *ah)</div><div class='ctx'> {</div><div class='del'>-	u8 port = be32_to_cpu(ah-&gt;av.ib.port_pd) &gt;&gt; 24 &amp; 3;</div><div class='add'>+	u32 port = be32_to_cpu(ah-&gt;av.ib.port_pd) &gt;&gt; 24 &amp; 3;</div><div class='ctx'> </div><div class='ctx'> 	if (rdma_port_get_link_layer(ah-&gt;ibah.device, port) == IB_LINK_LAYER_ETHERNET)</div><div class='ctx'> 		return true;</div><div class='hunk'>@@ -825,7 +846,7 @@ void clean_vf_mcast(struct mlx4_ib_demux_ctx *ctx, int slave);</div><div class='ctx'> int mlx4_ib_mcg_init(void);</div><div class='ctx'> void mlx4_ib_mcg_destroy(void);</div><div class='ctx'> </div><div class='del'>-int mlx4_ib_find_real_gid(struct ib_device *ibdev, u8 port, __be64 guid);</div><div class='add'>+int mlx4_ib_find_real_gid(struct ib_device *ibdev, u32 port, __be64 guid);</div><div class='ctx'> </div><div class='ctx'> int mlx4_ib_mcg_multiplex_handler(struct ib_device *ibdev, int port, int slave,</div><div class='ctx'> 				  struct ib_sa_mad *sa_mad);</div><div class='hunk'>@@ -835,16 +856,16 @@ int mlx4_ib_mcg_demux_handler(struct ib_device *ibdev, int port, int slave,</div><div class='ctx'> int mlx4_ib_add_mc(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp,</div><div class='ctx'> 		   union ib_gid *gid);</div><div class='ctx'> </div><div class='del'>-void mlx4_ib_dispatch_event(struct mlx4_ib_dev *dev, u8 port_num,</div><div class='add'>+void mlx4_ib_dispatch_event(struct mlx4_ib_dev *dev, u32 port_num,</div><div class='ctx'> 			    enum ib_event_type type);</div><div class='ctx'> </div><div class='ctx'> void mlx4_ib_tunnels_update_work(struct work_struct *work);</div><div class='ctx'> </div><div class='del'>-int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port,</div><div class='add'>+int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u32 port,</div><div class='ctx'> 			  enum ib_qp_type qpt, struct ib_wc *wc,</div><div class='ctx'> 			  struct ib_grh *grh, struct ib_mad *mad);</div><div class='ctx'> </div><div class='del'>-int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,</div><div class='add'>+int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u32 port,</div><div class='ctx'> 			 enum ib_qp_type dest_qpt, u16 pkey_index, u32 remote_qpn,</div><div class='ctx'> 			 u32 qkey, struct rdma_ah_attr *attr, u8 *s_mac,</div><div class='ctx'> 			 u16 vlan_id, struct ib_mad *mad);</div><div class='hunk'>@@ -868,10 +889,10 @@ void mlx4_ib_invalidate_all_guid_record(struct mlx4_ib_dev *dev, int port);</div><div class='ctx'> </div><div class='ctx'> void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev,</div><div class='ctx'> 					  int block_num,</div><div class='del'>-					  u8 port_num, u8 *p_data);</div><div class='add'>+					  u32 port_num, u8 *p_data);</div><div class='ctx'> </div><div class='ctx'> void mlx4_ib_update_cache_on_guid_change(struct mlx4_ib_dev *dev,</div><div class='del'>-					 int block_num, u8 port_num,</div><div class='add'>+					 int block_num, u32 port_num,</div><div class='ctx'> 					 u8 *p_data);</div><div class='ctx'> </div><div class='ctx'> int add_sysfs_port_mcg_attr(struct mlx4_ib_dev *device, int port_num,</div><div class='hunk'>@@ -892,10 +913,10 @@ int mlx4_ib_steer_qp_alloc(struct mlx4_ib_dev *dev, int count, int *qpn);</div><div class='ctx'> void mlx4_ib_steer_qp_free(struct mlx4_ib_dev *dev, u32 qpn, int count);</div><div class='ctx'> int mlx4_ib_steer_qp_reg(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp,</div><div class='ctx'> 			 int is_attach);</div><div class='del'>-int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags,</div><div class='del'>-			  u64 start, u64 length, u64 virt_addr,</div><div class='del'>-			  int mr_access_flags, struct ib_pd *pd,</div><div class='del'>-			  struct ib_udata *udata);</div><div class='add'>+struct ib_mr *mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, u64 start,</div><div class='add'>+				    u64 length, u64 virt_addr,</div><div class='add'>+				    int mr_access_flags, struct ib_pd *pd,</div><div class='add'>+				    struct ib_udata *udata);</div><div class='ctx'> int mlx4_ib_gid_index_to_real_index(struct mlx4_ib_dev *ibdev,</div><div class='ctx'> 				    const struct ib_gid_attr *attr);</div><div class='ctx'> </div><div class='hunk'>@@ -911,12 +932,32 @@ int mlx4_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata);</div><div class='ctx'> int mlx4_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,</div><div class='ctx'> 		      u32 wq_attr_mask, struct ib_udata *udata);</div><div class='ctx'> </div><div class='del'>-struct ib_rwq_ind_table</div><div class='del'>-*mlx4_ib_create_rwq_ind_table(struct ib_device *device,</div><div class='del'>-			      struct ib_rwq_ind_table_init_attr *init_attr,</div><div class='del'>-			      struct ib_udata *udata);</div><div class='del'>-int mlx4_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);</div><div class='del'>-int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, u64 start_va,</div><div class='del'>-				       int *num_of_mtts);</div><div class='add'>+int mlx4_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl,</div><div class='add'>+				 struct ib_rwq_ind_table_init_attr *init_attr,</div><div class='add'>+				 struct ib_udata *udata);</div><div class='add'>+static inline int</div><div class='add'>+mlx4_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table)</div><div class='add'>+{</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+static inline int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem,</div><div class='add'>+						     u64 start,</div><div class='add'>+						     int *num_of_mtts)</div><div class='add'>+{</div><div class='add'>+	unsigned long pg_sz;</div><div class='add'>+</div><div class='add'>+	pg_sz = ib_umem_find_best_pgsz(umem, MLX4_PAGE_SIZE_SUPPORTED, start);</div><div class='add'>+	if (!pg_sz)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	*num_of_mtts = ib_umem_num_dma_blocks(umem, pg_sz);</div><div class='add'>+	return order_base_2(pg_sz);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx4_ib_cm_init(void);</div><div class='add'>+void mlx4_ib_cm_destroy(void);</div><div class='add'>+</div><div class='add'>+int mlx4_ib_qp_event_init(void);</div><div class='add'>+void mlx4_ib_qp_event_cleanup(void);</div><div class='ctx'> </div><div class='ctx'> #endif /* MLX4_IB_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c<br/>index 355205a28544..94464f1694d9 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/mr.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/mr.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/mr.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx4/mr.c</a></div><div class='hunk'>@@ -87,289 +87,24 @@ err_free:</div><div class='ctx'> 	return ERR_PTR(err);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-enum {</div><div class='del'>-	MLX4_MAX_MTT_SHIFT = 31</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static int mlx4_ib_umem_write_mtt_block(struct mlx4_ib_dev *dev,</div><div class='del'>-					struct mlx4_mtt *mtt,</div><div class='del'>-					u64 mtt_size, u64 mtt_shift, u64 len,</div><div class='del'>-					u64 cur_start_addr, u64 *pages,</div><div class='del'>-					int *start_index, int *npages)</div><div class='del'>-{</div><div class='del'>-	u64 cur_end_addr = cur_start_addr + len;</div><div class='del'>-	u64 cur_end_addr_aligned = 0;</div><div class='del'>-	u64 mtt_entries;</div><div class='del'>-	int err = 0;</div><div class='del'>-	int k;</div><div class='del'>-</div><div class='del'>-	len += (cur_start_addr &amp; (mtt_size - 1ULL));</div><div class='del'>-	cur_end_addr_aligned = round_up(cur_end_addr, mtt_size);</div><div class='del'>-	len += (cur_end_addr_aligned - cur_end_addr);</div><div class='del'>-	if (len &amp; (mtt_size - 1ULL)) {</div><div class='del'>-		pr_warn("write_block: len %llx is not aligned to mtt_size %llx\n",</div><div class='del'>-			len, mtt_size);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mtt_entries = (len &gt;&gt; mtt_shift);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Align the MTT start address to the mtt_size.</div><div class='del'>-	 * Required to handle cases when the MR starts in the middle of an MTT</div><div class='del'>-	 * record. Was not required in old code since the physical addresses</div><div class='del'>-	 * provided by the dma subsystem were page aligned, which was also the</div><div class='del'>-	 * MTT size.</div><div class='del'>-	 */</div><div class='del'>-	cur_start_addr = round_down(cur_start_addr, mtt_size);</div><div class='del'>-	/* A new block is started ... */</div><div class='del'>-	for (k = 0; k &lt; mtt_entries; ++k) {</div><div class='del'>-		pages[*npages] = cur_start_addr + (mtt_size * k);</div><div class='del'>-		(*npages)++;</div><div class='del'>-		/*</div><div class='del'>-		 * Be friendly to mlx4_write_mtt() and pass it chunks of</div><div class='del'>-		 * appropriate size.</div><div class='del'>-		 */</div><div class='del'>-		if (*npages == PAGE_SIZE / sizeof(u64)) {</div><div class='del'>-			err = mlx4_write_mtt(dev-&gt;dev, mtt, *start_index,</div><div class='del'>-					     *npages, pages);</div><div class='del'>-			if (err)</div><div class='del'>-				return err;</div><div class='del'>-</div><div class='del'>-			(*start_index) += *npages;</div><div class='del'>-			*npages = 0;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u64 alignment_of(u64 ptr)</div><div class='del'>-{</div><div class='del'>-	return ilog2(ptr &amp; (~(ptr - 1)));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx4_ib_umem_calc_block_mtt(u64 next_block_start,</div><div class='del'>-				       u64 current_block_end,</div><div class='del'>-				       u64 block_shift)</div><div class='del'>-{</div><div class='del'>-	/* Check whether the alignment of the new block is aligned as well as</div><div class='del'>-	 * the previous block.</div><div class='del'>-	 * Block address must start with zeros till size of entity_size.</div><div class='del'>-	 */</div><div class='del'>-	if ((next_block_start &amp; ((1ULL &lt;&lt; block_shift) - 1ULL)) != 0)</div><div class='del'>-		/*</div><div class='del'>-		 * It is not as well aligned as the previous block-reduce the</div><div class='del'>-		 * mtt size accordingly. Here we take the last right bit which</div><div class='del'>-		 * is 1.</div><div class='del'>-		 */</div><div class='del'>-		block_shift = alignment_of(next_block_start);</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Check whether the alignment of the end of previous block - is it</div><div class='del'>-	 * aligned as well as the start of the block</div><div class='del'>-	 */</div><div class='del'>-	if (((current_block_end) &amp; ((1ULL &lt;&lt; block_shift) - 1ULL)) != 0)</div><div class='del'>-		/*</div><div class='del'>-		 * It is not as well aligned as the start of the block -</div><div class='del'>-		 * reduce the mtt size accordingly.</div><div class='del'>-		 */</div><div class='del'>-		block_shift = alignment_of(current_block_end);</div><div class='del'>-</div><div class='del'>-	return block_shift;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt,</div><div class='ctx'> 			   struct ib_umem *umem)</div><div class='ctx'> {</div><div class='del'>-	u64 *pages;</div><div class='del'>-	u64 len = 0;</div><div class='del'>-	int err = 0;</div><div class='del'>-	u64 mtt_size;</div><div class='del'>-	u64 cur_start_addr = 0;</div><div class='del'>-	u64 mtt_shift;</div><div class='del'>-	int start_index = 0;</div><div class='del'>-	int npages = 0;</div><div class='del'>-	struct scatterlist *sg;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	pages = (u64 *) __get_free_page(GFP_KERNEL);</div><div class='del'>-	if (!pages)</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+	struct ib_block_iter biter;</div><div class='add'>+	int err, i = 0;</div><div class='add'>+	u64 addr;</div><div class='ctx'> </div><div class='del'>-	mtt_shift = mtt-&gt;page_shift;</div><div class='del'>-	mtt_size = 1ULL &lt;&lt; mtt_shift;</div><div class='del'>-</div><div class='del'>-	for_each_sg(umem-&gt;sg_head.sgl, sg, umem-&gt;nmap, i) {</div><div class='del'>-		if (cur_start_addr + len == sg_dma_address(sg)) {</div><div class='del'>-			/* still the same block */</div><div class='del'>-			len += sg_dma_len(sg);</div><div class='del'>-			continue;</div><div class='del'>-		}</div><div class='del'>-		/*</div><div class='del'>-		 * A new block is started ...</div><div class='del'>-		 * If len is malaligned, write an extra mtt entry to cover the</div><div class='del'>-		 * misaligned area (round up the division)</div><div class='del'>-		 */</div><div class='del'>-		err = mlx4_ib_umem_write_mtt_block(dev, mtt, mtt_size,</div><div class='del'>-						   mtt_shift, len,</div><div class='del'>-						   cur_start_addr,</div><div class='del'>-						   pages, &amp;start_index,</div><div class='del'>-						   &amp;npages);</div><div class='add'>+	rdma_umem_for_each_dma_block(umem, &amp;biter, BIT(mtt-&gt;page_shift)) {</div><div class='add'>+		addr = rdma_block_iter_dma_address(&amp;biter);</div><div class='add'>+		err = mlx4_write_mtt(dev-&gt;dev, mtt, i++, 1, &amp;addr);</div><div class='ctx'> 		if (err)</div><div class='del'>-			goto out;</div><div class='del'>-</div><div class='del'>-		cur_start_addr = sg_dma_address(sg);</div><div class='del'>-		len = sg_dma_len(sg);</div><div class='add'>+			return err;</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='del'>-	/* Handle the last block */</div><div class='del'>-	if (len &gt; 0) {</div><div class='del'>-		/*</div><div class='del'>-		 * If len is malaligned, write an extra mtt entry to cover</div><div class='del'>-		 * the misaligned area (round up the division)</div><div class='del'>-		 */</div><div class='del'>-		err = mlx4_ib_umem_write_mtt_block(dev, mtt, mtt_size,</div><div class='del'>-						   mtt_shift, len,</div><div class='del'>-						   cur_start_addr, pages,</div><div class='del'>-						   &amp;start_index, &amp;npages);</div><div class='del'>-		if (err)</div><div class='del'>-			goto out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (npages)</div><div class='del'>-		err = mlx4_write_mtt(dev-&gt;dev, mtt, start_index, npages, pages);</div><div class='del'>-</div><div class='del'>-out:</div><div class='del'>-	free_page((unsigned long) pages);</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/*</div><div class='del'>- * Calculate optimal mtt size based on contiguous pages.</div><div class='del'>- * Function will return also the number of pages that are not aligned to the</div><div class='del'>- * calculated mtt_size to be added to total number of pages. For that we should</div><div class='del'>- * check the first chunk length &amp; last chunk length and if not aligned to</div><div class='del'>- * mtt_size we should increment the non_aligned_pages number. All chunks in the</div><div class='del'>- * middle already handled as part of mtt shift calculation for both their start</div><div class='del'>- * &amp; end addresses.</div><div class='del'>- */</div><div class='del'>-int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, u64 start_va,</div><div class='del'>-				       int *num_of_mtts)</div><div class='del'>-{</div><div class='del'>-	u64 block_shift = MLX4_MAX_MTT_SHIFT;</div><div class='del'>-	u64 min_shift = umem-&gt;page_shift;</div><div class='del'>-	u64 last_block_aligned_end = 0;</div><div class='del'>-	u64 current_block_start = 0;</div><div class='del'>-	u64 first_block_start = 0;</div><div class='del'>-	u64 current_block_len = 0;</div><div class='del'>-	u64 last_block_end = 0;</div><div class='del'>-	struct scatterlist *sg;</div><div class='del'>-	u64 current_block_end;</div><div class='del'>-	u64 misalignment_bits;</div><div class='del'>-	u64 next_block_start;</div><div class='del'>-	u64 total_len = 0;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	for_each_sg(umem-&gt;sg_head.sgl, sg, umem-&gt;nmap, i) {</div><div class='del'>-		/*</div><div class='del'>-		 * Initialization - save the first chunk start as the</div><div class='del'>-		 * current_block_start - block means contiguous pages.</div><div class='del'>-		 */</div><div class='del'>-		if (current_block_len == 0 &amp;&amp; current_block_start == 0) {</div><div class='del'>-			current_block_start = sg_dma_address(sg);</div><div class='del'>-			first_block_start = current_block_start;</div><div class='del'>-			/*</div><div class='del'>-			 * Find the bits that are different between the physical</div><div class='del'>-			 * address and the virtual address for the start of the</div><div class='del'>-			 * MR.</div><div class='del'>-			 * umem_get aligned the start_va to a page boundary.</div><div class='del'>-			 * Therefore, we need to align the start va to the same</div><div class='del'>-			 * boundary.</div><div class='del'>-			 * misalignment_bits is needed to handle the  case of a</div><div class='del'>-			 * single memory region. In this case, the rest of the</div><div class='del'>-			 * logic will not reduce the block size.  If we use a</div><div class='del'>-			 * block size which is bigger than the alignment of the</div><div class='del'>-			 * misalignment bits, we might use the virtual page</div><div class='del'>-			 * number instead of the physical page number, resulting</div><div class='del'>-			 * in access to the wrong data.</div><div class='del'>-			 */</div><div class='del'>-			misalignment_bits =</div><div class='del'>-			(start_va &amp; (~(((u64)(BIT(umem-&gt;page_shift))) - 1ULL)))</div><div class='del'>-			^ current_block_start;</div><div class='del'>-			block_shift = min(alignment_of(misalignment_bits),</div><div class='del'>-					  block_shift);</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		/*</div><div class='del'>-		 * Go over the scatter entries and check if they continue the</div><div class='del'>-		 * previous scatter entry.</div><div class='del'>-		 */</div><div class='del'>-		next_block_start = sg_dma_address(sg);</div><div class='del'>-		current_block_end = current_block_start	+ current_block_len;</div><div class='del'>-		/* If we have a split (non-contig.) between two blocks */</div><div class='del'>-		if (current_block_end != next_block_start) {</div><div class='del'>-			block_shift = mlx4_ib_umem_calc_block_mtt</div><div class='del'>-					(next_block_start,</div><div class='del'>-					 current_block_end,</div><div class='del'>-					 block_shift);</div><div class='del'>-</div><div class='del'>-			/*</div><div class='del'>-			 * If we reached the minimum shift for 4k page we stop</div><div class='del'>-			 * the loop.</div><div class='del'>-			 */</div><div class='del'>-			if (block_shift &lt;= min_shift)</div><div class='del'>-				goto end;</div><div class='del'>-</div><div class='del'>-			/*</div><div class='del'>-			 * If not saved yet we are in first block - we save the</div><div class='del'>-			 * length of first block to calculate the</div><div class='del'>-			 * non_aligned_pages number at the end.</div><div class='del'>-			 */</div><div class='del'>-			total_len += current_block_len;</div><div class='del'>-</div><div class='del'>-			/* Start a new block */</div><div class='del'>-			current_block_start = next_block_start;</div><div class='del'>-			current_block_len = sg_dma_len(sg);</div><div class='del'>-			continue;</div><div class='del'>-		}</div><div class='del'>-		/* The scatter entry is another part of the current block,</div><div class='del'>-		 * increase the block size.</div><div class='del'>-		 * An entry in the scatter can be larger than 4k (page) as of</div><div class='del'>-		 * dma mapping which merge some blocks together.</div><div class='del'>-		 */</div><div class='del'>-		current_block_len += sg_dma_len(sg);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Account for the last block in the total len */</div><div class='del'>-	total_len += current_block_len;</div><div class='del'>-	/* Add to the first block the misalignment that it suffers from. */</div><div class='del'>-	total_len += (first_block_start &amp; ((1ULL &lt;&lt; block_shift) - 1ULL));</div><div class='del'>-	last_block_end = current_block_start + current_block_len;</div><div class='del'>-	last_block_aligned_end = round_up(last_block_end, 1ULL &lt;&lt; block_shift);</div><div class='del'>-	total_len += (last_block_aligned_end - last_block_end);</div><div class='del'>-</div><div class='del'>-	if (total_len &amp; ((1ULL &lt;&lt; block_shift) - 1ULL))</div><div class='del'>-		pr_warn("misaligned total length detected (%llu, %llu)!",</div><div class='del'>-			total_len, block_shift);</div><div class='del'>-</div><div class='del'>-	*num_of_mtts = total_len &gt;&gt; block_shift;</div><div class='del'>-end:</div><div class='del'>-	if (block_shift &lt; min_shift) {</div><div class='del'>-		/*</div><div class='del'>-		 * If shift is less than the min we set a warning and return the</div><div class='del'>-		 * min shift.</div><div class='del'>-		 */</div><div class='del'>-		pr_warn("umem_calc_optimal_mtt_size - unexpected shift %lld\n", block_shift);</div><div class='del'>-</div><div class='del'>-		block_shift = min_shift;</div><div class='del'>-	}</div><div class='del'>-	return block_shift;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct ib_umem *mlx4_get_umem_mr(struct ib_udata *udata, u64 start,</div><div class='del'>-					u64 length, u64 virt_addr,</div><div class='del'>-					int access_flags)</div><div class='add'>+static struct ib_umem *mlx4_get_umem_mr(struct ib_device *device, u64 start,</div><div class='add'>+					u64 length, int access_flags)</div><div class='ctx'> {</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Force registering the memory as writable if the underlying pages</div><div class='hunk'>@@ -378,31 +113,33 @@ static struct ib_umem *mlx4_get_umem_mr(struct ib_udata *udata, u64 start,</div><div class='ctx'> 	 * again</div><div class='ctx'> 	 */</div><div class='ctx'> 	if (!ib_access_writable(access_flags)) {</div><div class='add'>+		unsigned long untagged_start = untagged_addr(start);</div><div class='ctx'> 		struct vm_area_struct *vma;</div><div class='ctx'> </div><div class='del'>-		down_read(&amp;current-&gt;mm-&gt;mmap_sem);</div><div class='add'>+		mmap_read_lock(current-&gt;mm);</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * FIXME: Ideally this would iterate over all the vmas that</div><div class='ctx'> 		 * cover the memory, but for now it requires a single vma to</div><div class='ctx'> 		 * entirely cover the MR to support RO mappings.</div><div class='ctx'> 		 */</div><div class='del'>-		vma = find_vma(current-&gt;mm, start);</div><div class='del'>-		if (vma &amp;&amp; vma-&gt;vm_end &gt;= start + length &amp;&amp;</div><div class='del'>-		    vma-&gt;vm_start &lt;= start) {</div><div class='add'>+		vma = find_vma(current-&gt;mm, untagged_start);</div><div class='add'>+		if (vma &amp;&amp; vma-&gt;vm_end &gt;= untagged_start + length &amp;&amp;</div><div class='add'>+		    vma-&gt;vm_start &lt;= untagged_start) {</div><div class='ctx'> 			if (vma-&gt;vm_flags &amp; VM_WRITE)</div><div class='ctx'> 				access_flags |= IB_ACCESS_LOCAL_WRITE;</div><div class='ctx'> 		} else {</div><div class='ctx'> 			access_flags |= IB_ACCESS_LOCAL_WRITE;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		up_read(&amp;current-&gt;mm-&gt;mmap_sem);</div><div class='add'>+		mmap_read_unlock(current-&gt;mm);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return ib_umem_get(udata, start, length, access_flags, 0);</div><div class='add'>+	return ib_umem_get(device, start, length, access_flags);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='ctx'> 				  u64 virt_addr, int access_flags,</div><div class='add'>+				  struct ib_dmah *dmah,</div><div class='ctx'> 				  struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='hunk'>@@ -411,19 +148,24 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='ctx'> 	int err;</div><div class='ctx'> 	int n;</div><div class='ctx'> </div><div class='add'>+	if (dmah)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='ctx'> 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='ctx'> 	if (!mr)</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='del'>-	mr-&gt;umem =</div><div class='del'>-		mlx4_get_umem_mr(udata, start, length, virt_addr, access_flags);</div><div class='add'>+	mr-&gt;umem = mlx4_get_umem_mr(pd-&gt;device, start, length, access_flags);</div><div class='ctx'> 	if (IS_ERR(mr-&gt;umem)) {</div><div class='ctx'> 		err = PTR_ERR(mr-&gt;umem);</div><div class='ctx'> 		goto err_free;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	n = ib_umem_page_count(mr-&gt;umem);</div><div class='ctx'> 	shift = mlx4_ib_umem_calc_optimal_mtt_size(mr-&gt;umem, start, &amp;n);</div><div class='add'>+	if (shift &lt; 0) {</div><div class='add'>+		err = shift;</div><div class='add'>+		goto err_umem;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	err = mlx4_mr_alloc(dev-&gt;dev, to_mpd(pd)-&gt;pdn, virt_addr, length,</div><div class='ctx'> 			    convert_access(access_flags), n, shift, &amp;mr-&gt;mmr);</div><div class='hunk'>@@ -439,8 +181,6 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='ctx'> 		goto err_mr;</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;ibmr.rkey = mr-&gt;ibmr.lkey = mr-&gt;mmr.key;</div><div class='del'>-	mr-&gt;ibmr.length = length;</div><div class='del'>-	mr-&gt;ibmr.iova = virt_addr;</div><div class='ctx'> 	mr-&gt;ibmr.page_size = 1U &lt;&lt; shift;</div><div class='ctx'> </div><div class='ctx'> 	return &amp;mr-&gt;ibmr;</div><div class='hunk'>@@ -457,10 +197,10 @@ err_free:</div><div class='ctx'> 	return ERR_PTR(err);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags,</div><div class='del'>-			  u64 start, u64 length, u64 virt_addr,</div><div class='del'>-			  int mr_access_flags, struct ib_pd *pd,</div><div class='del'>-			  struct ib_udata *udata)</div><div class='add'>+struct ib_mr *mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, u64 start,</div><div class='add'>+				    u64 length, u64 virt_addr,</div><div class='add'>+				    int mr_access_flags, struct ib_pd *pd,</div><div class='add'>+				    struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_ib_dev *dev = to_mdev(mr-&gt;device);</div><div class='ctx'> 	struct mlx4_ib_mr *mmr = to_mmr(mr);</div><div class='hunk'>@@ -473,9 +213,8 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags,</div><div class='ctx'> 	 * race exists.</div><div class='ctx'> 	 */</div><div class='ctx'> 	err =  mlx4_mr_hw_get_mpt(dev-&gt;dev, &amp;mmr-&gt;mmr, &amp;pmpt_entry);</div><div class='del'>-</div><div class='ctx'> 	if (err)</div><div class='del'>-		return err;</div><div class='add'>+		return ERR_PTR(err);</div><div class='ctx'> </div><div class='ctx'> 	if (flags &amp; IB_MR_REREG_PD) {</div><div class='ctx'> 		err = mlx4_mr_hw_change_pd(dev-&gt;dev, *pmpt_entry,</div><div class='hunk'>@@ -505,7 +244,7 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags,</div><div class='ctx'> </div><div class='ctx'> 		mlx4_mr_rereg_mem_cleanup(dev-&gt;dev, &amp;mmr-&gt;mmr);</div><div class='ctx'> 		ib_umem_release(mmr-&gt;umem);</div><div class='del'>-		mmr-&gt;umem = mlx4_get_umem_mr(udata, start, length, virt_addr,</div><div class='add'>+		mmr-&gt;umem = mlx4_get_umem_mr(mr-&gt;device, start, length,</div><div class='ctx'> 					     mr_access_flags);</div><div class='ctx'> 		if (IS_ERR(mmr-&gt;umem)) {</div><div class='ctx'> 			err = PTR_ERR(mmr-&gt;umem);</div><div class='hunk'>@@ -513,8 +252,8 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags,</div><div class='ctx'> 			mmr-&gt;umem = NULL;</div><div class='ctx'> 			goto release_mpt_entry;</div><div class='ctx'> 		}</div><div class='del'>-		n = ib_umem_page_count(mmr-&gt;umem);</div><div class='del'>-		shift = mmr-&gt;umem-&gt;page_shift;</div><div class='add'>+		n = ib_umem_num_dma_blocks(mmr-&gt;umem, PAGE_SIZE);</div><div class='add'>+		shift = PAGE_SHIFT;</div><div class='ctx'> </div><div class='ctx'> 		err = mlx4_mr_rereg_mem_write(dev-&gt;dev, &amp;mmr-&gt;mmr,</div><div class='ctx'> 					      virt_addr, length, n, shift,</div><div class='hunk'>@@ -543,8 +282,9 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags,</div><div class='ctx'> </div><div class='ctx'> release_mpt_entry:</div><div class='ctx'> 	mlx4_mr_hw_put_mpt(dev-&gt;dev, pmpt_entry);</div><div class='del'>-</div><div class='del'>-	return err;</div><div class='add'>+	if (err)</div><div class='add'>+		return ERR_PTR(err);</div><div class='add'>+	return NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int</div><div class='hunk'>@@ -612,37 +352,27 @@ int mlx4_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,</div><div class='del'>-			       struct ib_udata *udata)</div><div class='add'>+int mlx4_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	struct mlx4_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='del'>-	struct mlx4_ib_mw *mw;</div><div class='add'>+	struct mlx4_ib_dev *dev = to_mdev(ibmw-&gt;device);</div><div class='add'>+	struct mlx4_ib_mw *mw = to_mmw(ibmw);</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	mw = kmalloc(sizeof(*mw), GFP_KERNEL);</div><div class='del'>-	if (!mw)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	err = mlx4_mw_alloc(dev-&gt;dev, to_mpd(pd)-&gt;pdn,</div><div class='del'>-			    to_mlx4_type(type), &amp;mw-&gt;mmw);</div><div class='add'>+	err = mlx4_mw_alloc(dev-&gt;dev, to_mpd(ibmw-&gt;pd)-&gt;pdn,</div><div class='add'>+			    to_mlx4_type(ibmw-&gt;type), &amp;mw-&gt;mmw);</div><div class='ctx'> 	if (err)</div><div class='del'>-		goto err_free;</div><div class='add'>+		return err;</div><div class='ctx'> </div><div class='ctx'> 	err = mlx4_mw_enable(dev-&gt;dev, &amp;mw-&gt;mmw);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_mw;</div><div class='ctx'> </div><div class='del'>-	mw-&gt;ibmw.rkey = mw-&gt;mmw.key;</div><div class='del'>-</div><div class='del'>-	return &amp;mw-&gt;ibmw;</div><div class='add'>+	ibmw-&gt;rkey = mw-&gt;mmw.key;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='ctx'> err_mw:</div><div class='ctx'> 	mlx4_mw_free(dev-&gt;dev, &amp;mw-&gt;mmw);</div><div class='del'>-</div><div class='del'>-err_free:</div><div class='del'>-	kfree(mw);</div><div class='del'>-</div><div class='del'>-	return ERR_PTR(err);</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx4_ib_dealloc_mw(struct ib_mw *ibmw)</div><div class='hunk'>@@ -650,13 +380,11 @@ int mlx4_ib_dealloc_mw(struct ib_mw *ibmw)</div><div class='ctx'> 	struct mlx4_ib_mw *mw = to_mmw(ibmw);</div><div class='ctx'> </div><div class='ctx'> 	mlx4_mw_free(to_mdev(ibmw-&gt;device)-&gt;dev, &amp;mw-&gt;mmw);</div><div class='del'>-	kfree(mw);</div><div class='del'>-</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,</div><div class='del'>-			       u32 max_num_sg, struct ib_udata *udata)</div><div class='add'>+			       u32 max_num_sg)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='ctx'> 	struct mlx4_ib_mr *mr;</div><div class='hunk'>@@ -699,99 +427,6 @@ err_free:</div><div class='ctx'> 	return ERR_PTR(err);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int acc,</div><div class='del'>-				 struct ib_fmr_attr *fmr_attr)</div><div class='del'>-{</div><div class='del'>-	struct mlx4_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='del'>-	struct mlx4_ib_fmr *fmr;</div><div class='del'>-	int err = -ENOMEM;</div><div class='del'>-</div><div class='del'>-	fmr = kmalloc(sizeof *fmr, GFP_KERNEL);</div><div class='del'>-	if (!fmr)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	err = mlx4_fmr_alloc(dev-&gt;dev, to_mpd(pd)-&gt;pdn, convert_access(acc),</div><div class='del'>-			     fmr_attr-&gt;max_pages, fmr_attr-&gt;max_maps,</div><div class='del'>-			     fmr_attr-&gt;page_shift, &amp;fmr-&gt;mfmr);</div><div class='del'>-	if (err)</div><div class='del'>-		goto err_free;</div><div class='del'>-</div><div class='del'>-	err = mlx4_fmr_enable(to_mdev(pd-&gt;device)-&gt;dev, &amp;fmr-&gt;mfmr);</div><div class='del'>-	if (err)</div><div class='del'>-		goto err_mr;</div><div class='del'>-</div><div class='del'>-	fmr-&gt;ibfmr.rkey = fmr-&gt;ibfmr.lkey = fmr-&gt;mfmr.mr.key;</div><div class='del'>-</div><div class='del'>-	return &amp;fmr-&gt;ibfmr;</div><div class='del'>-</div><div class='del'>-err_mr:</div><div class='del'>-	(void) mlx4_mr_free(to_mdev(pd-&gt;device)-&gt;dev, &amp;fmr-&gt;mfmr.mr);</div><div class='del'>-</div><div class='del'>-err_free:</div><div class='del'>-	kfree(fmr);</div><div class='del'>-</div><div class='del'>-	return ERR_PTR(err);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,</div><div class='del'>-		      int npages, u64 iova)</div><div class='del'>-{</div><div class='del'>-	struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);</div><div class='del'>-	struct mlx4_ib_dev *dev = to_mdev(ifmr-&gt;ibfmr.device);</div><div class='del'>-</div><div class='del'>-	return mlx4_map_phys_fmr(dev-&gt;dev, &amp;ifmr-&gt;mfmr, page_list, npages, iova,</div><div class='del'>-				 &amp;ifmr-&gt;ibfmr.lkey, &amp;ifmr-&gt;ibfmr.rkey);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int mlx4_ib_unmap_fmr(struct list_head *fmr_list)</div><div class='del'>-{</div><div class='del'>-	struct ib_fmr *ibfmr;</div><div class='del'>-	int err;</div><div class='del'>-	struct mlx4_dev *mdev = NULL;</div><div class='del'>-</div><div class='del'>-	list_for_each_entry(ibfmr, fmr_list, list) {</div><div class='del'>-		if (mdev &amp;&amp; to_mdev(ibfmr-&gt;device)-&gt;dev != mdev)</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		mdev = to_mdev(ibfmr-&gt;device)-&gt;dev;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!mdev)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	list_for_each_entry(ibfmr, fmr_list, list) {</div><div class='del'>-		struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);</div><div class='del'>-</div><div class='del'>-		mlx4_fmr_unmap(mdev, &amp;ifmr-&gt;mfmr, &amp;ifmr-&gt;ibfmr.lkey, &amp;ifmr-&gt;ibfmr.rkey);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Make sure all MPT status updates are visible before issuing</div><div class='del'>-	 * SYNC_TPT firmware command.</div><div class='del'>-	 */</div><div class='del'>-	wmb();</div><div class='del'>-</div><div class='del'>-	err = mlx4_SYNC_TPT(mdev);</div><div class='del'>-	if (err)</div><div class='del'>-		pr_warn("SYNC_TPT error %d when "</div><div class='del'>-		       "unmapping FMRs\n", err);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int mlx4_ib_fmr_dealloc(struct ib_fmr *ibfmr)</div><div class='del'>-{</div><div class='del'>-	struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);</div><div class='del'>-	struct mlx4_ib_dev *dev = to_mdev(ibfmr-&gt;device);</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	err = mlx4_fmr_free(dev-&gt;dev, &amp;ifmr-&gt;mfmr);</div><div class='del'>-</div><div class='del'>-	if (!err)</div><div class='del'>-		kfree(ifmr);</div><div class='del'>-</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int mlx4_set_page(struct ib_mr *ibmr, u64 addr)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_ib_mr *mr = to_mmr(ibmr);</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c<br/>index 5221c0794d1d..f2887ae6390e 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/qp.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/qp.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/qp.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx4/qp.c</a></div><div class='hunk'>@@ -68,27 +68,6 @@ enum {</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum {</div><div class='del'>-	/*</div><div class='del'>-	 * Largest possible UD header: send with GRH and immediate</div><div class='del'>-	 * data plus 18 bytes for an Ethernet header with VLAN/802.1Q</div><div class='del'>-	 * tag.  (LRH would only use 8 bytes, so Ethernet is the</div><div class='del'>-	 * biggest case)</div><div class='del'>-	 */</div><div class='del'>-	MLX4_IB_UD_HEADER_SIZE		= 82,</div><div class='del'>-	MLX4_IB_LSO_HEADER_SPARE	= 128,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct mlx4_ib_sqp {</div><div class='del'>-	struct mlx4_ib_qp	qp;</div><div class='del'>-	int			pkey_index;</div><div class='del'>-	u32			qkey;</div><div class='del'>-	u32			send_psn;</div><div class='del'>-	struct ib_ud_header	ud_header;</div><div class='del'>-	u8			header_buf[MLX4_IB_UD_HEADER_SIZE];</div><div class='del'>-	struct ib_qp		*roce_v2_gsi;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum {</div><div class='ctx'> 	MLX4_IB_MIN_SQ_STRIDE	= 6,</div><div class='ctx'> 	MLX4_IB_CACHE_LINE_SIZE	= 64,</div><div class='ctx'> };</div><div class='hunk'>@@ -123,10 +102,13 @@ enum mlx4_ib_source_type {</div><div class='ctx'> 	MLX4_IB_RWQ_SRC	= 1,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp)</div><div class='del'>-{</div><div class='del'>-	return container_of(mqp, struct mlx4_ib_sqp, qp);</div><div class='del'>-}</div><div class='add'>+struct mlx4_ib_qp_event_work {</div><div class='add'>+	struct work_struct work;</div><div class='add'>+	struct mlx4_qp *qp;</div><div class='add'>+	enum mlx4_event type;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static struct workqueue_struct *mlx4_ib_qp_event_wq;</div><div class='ctx'> </div><div class='ctx'> static int is_tunnel_qp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)</div><div class='ctx'> {</div><div class='hunk'>@@ -226,50 +208,77 @@ static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n)</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void mlx4_ib_handle_qp_event(struct work_struct *_work)</div><div class='add'>+{</div><div class='add'>+	struct mlx4_ib_qp_event_work *qpe_work =</div><div class='add'>+		container_of(_work, struct mlx4_ib_qp_event_work, work);</div><div class='add'>+	struct ib_qp *ibqp = &amp;to_mibqp(qpe_work-&gt;qp)-&gt;ibqp;</div><div class='add'>+	struct ib_event event = {};</div><div class='add'>+</div><div class='add'>+	event.device = ibqp-&gt;device;</div><div class='add'>+	event.element.qp = ibqp;</div><div class='add'>+</div><div class='add'>+	switch (qpe_work-&gt;type) {</div><div class='add'>+	case MLX4_EVENT_TYPE_PATH_MIG:</div><div class='add'>+		event.event = IB_EVENT_PATH_MIG;</div><div class='add'>+		break;</div><div class='add'>+	case MLX4_EVENT_TYPE_COMM_EST:</div><div class='add'>+		event.event = IB_EVENT_COMM_EST;</div><div class='add'>+		break;</div><div class='add'>+	case MLX4_EVENT_TYPE_SQ_DRAINED:</div><div class='add'>+		event.event = IB_EVENT_SQ_DRAINED;</div><div class='add'>+		break;</div><div class='add'>+	case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:</div><div class='add'>+		event.event = IB_EVENT_QP_LAST_WQE_REACHED;</div><div class='add'>+		break;</div><div class='add'>+	case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:</div><div class='add'>+		event.event = IB_EVENT_QP_FATAL;</div><div class='add'>+		break;</div><div class='add'>+	case MLX4_EVENT_TYPE_PATH_MIG_FAILED:</div><div class='add'>+		event.event = IB_EVENT_PATH_MIG_ERR;</div><div class='add'>+		break;</div><div class='add'>+	case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:</div><div class='add'>+		event.event = IB_EVENT_QP_REQ_ERR;</div><div class='add'>+		break;</div><div class='add'>+	case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:</div><div class='add'>+		event.event = IB_EVENT_QP_ACCESS_ERR;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		pr_warn("Unexpected event type %d on QP %06x\n",</div><div class='add'>+			qpe_work-&gt;type, qpe_work-&gt;qp-&gt;qpn);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ibqp-&gt;event_handler(&amp;event, ibqp-&gt;qp_context);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	mlx4_put_qp(qpe_work-&gt;qp);</div><div class='add'>+	kfree(qpe_work);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)</div><div class='ctx'> {</div><div class='del'>-	struct ib_event event;</div><div class='ctx'> 	struct ib_qp *ibqp = &amp;to_mibqp(qp)-&gt;ibqp;</div><div class='add'>+	struct mlx4_ib_qp_event_work *qpe_work;</div><div class='ctx'> </div><div class='ctx'> 	if (type == MLX4_EVENT_TYPE_PATH_MIG)</div><div class='ctx'> 		to_mibqp(qp)-&gt;port = to_mibqp(qp)-&gt;alt_port;</div><div class='ctx'> </div><div class='del'>-	if (ibqp-&gt;event_handler) {</div><div class='del'>-		event.device     = ibqp-&gt;device;</div><div class='del'>-		event.element.qp = ibqp;</div><div class='del'>-		switch (type) {</div><div class='del'>-		case MLX4_EVENT_TYPE_PATH_MIG:</div><div class='del'>-			event.event = IB_EVENT_PATH_MIG;</div><div class='del'>-			break;</div><div class='del'>-		case MLX4_EVENT_TYPE_COMM_EST:</div><div class='del'>-			event.event = IB_EVENT_COMM_EST;</div><div class='del'>-			break;</div><div class='del'>-		case MLX4_EVENT_TYPE_SQ_DRAINED:</div><div class='del'>-			event.event = IB_EVENT_SQ_DRAINED;</div><div class='del'>-			break;</div><div class='del'>-		case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:</div><div class='del'>-			event.event = IB_EVENT_QP_LAST_WQE_REACHED;</div><div class='del'>-			break;</div><div class='del'>-		case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:</div><div class='del'>-			event.event = IB_EVENT_QP_FATAL;</div><div class='del'>-			break;</div><div class='del'>-		case MLX4_EVENT_TYPE_PATH_MIG_FAILED:</div><div class='del'>-			event.event = IB_EVENT_PATH_MIG_ERR;</div><div class='del'>-			break;</div><div class='del'>-		case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:</div><div class='del'>-			event.event = IB_EVENT_QP_REQ_ERR;</div><div class='del'>-			break;</div><div class='del'>-		case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:</div><div class='del'>-			event.event = IB_EVENT_QP_ACCESS_ERR;</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			pr_warn("Unexpected event type %d "</div><div class='del'>-			       "on QP %06x\n", type, qp-&gt;qpn);</div><div class='del'>-			return;</div><div class='del'>-		}</div><div class='add'>+	if (!ibqp-&gt;event_handler)</div><div class='add'>+		goto out_no_handler;</div><div class='ctx'> </div><div class='del'>-		ibqp-&gt;event_handler(&amp;event, ibqp-&gt;qp_context);</div><div class='del'>-	}</div><div class='add'>+	qpe_work = kzalloc(sizeof(*qpe_work), GFP_ATOMIC);</div><div class='add'>+	if (!qpe_work)</div><div class='add'>+		goto out_no_handler;</div><div class='add'>+</div><div class='add'>+	qpe_work-&gt;qp = qp;</div><div class='add'>+	qpe_work-&gt;type = type;</div><div class='add'>+	INIT_WORK(&amp;qpe_work-&gt;work, mlx4_ib_handle_qp_event);</div><div class='add'>+	queue_work(mlx4_ib_qp_event_wq, &amp;qpe_work-&gt;work);</div><div class='add'>+	return;</div><div class='add'>+</div><div class='add'>+out_no_handler:</div><div class='add'>+	mlx4_put_qp(qp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void mlx4_ib_wq_event(struct mlx4_qp *qp, enum mlx4_event type)</div><div class='hunk'>@@ -325,7 +334,7 @@ static int send_wqe_overhead(enum mlx4_ib_qp_type type, u32 flags)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,</div><div class='del'>-		       bool is_user, int has_rq, struct mlx4_ib_qp *qp,</div><div class='add'>+		       bool is_user, bool has_rq, struct mlx4_ib_qp *qp,</div><div class='ctx'> 		       u32 inl_recv_sz)</div><div class='ctx'> {</div><div class='ctx'> 	/* Sanity check RQ size before proceeding */</div><div class='hunk'>@@ -438,9 +447,13 @@ static int set_user_sq_size(struct mlx4_ib_dev *dev,</div><div class='ctx'> 			    struct mlx4_ib_qp *qp,</div><div class='ctx'> 			    struct mlx4_ib_create_qp *ucmd)</div><div class='ctx'> {</div><div class='add'>+	u32 cnt;</div><div class='add'>+</div><div class='ctx'> 	/* Sanity check SQ size before proceeding */</div><div class='del'>-	if ((1 &lt;&lt; ucmd-&gt;log_sq_bb_count) &gt; dev-&gt;dev-&gt;caps.max_wqes	 ||</div><div class='del'>-	    ucmd-&gt;log_sq_stride &gt;</div><div class='add'>+	if (check_shl_overflow(1, ucmd-&gt;log_sq_bb_count, &amp;cnt) ||</div><div class='add'>+	    cnt &gt; dev-&gt;dev-&gt;caps.max_wqes)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	if (ucmd-&gt;log_sq_stride &gt;</div><div class='ctx'> 		ilog2(roundup_pow_of_two(dev-&gt;dev-&gt;caps.max_sq_desc_sz)) ||</div><div class='ctx'> 	    ucmd-&gt;log_sq_stride &lt; MLX4_IB_MIN_SQ_STRIDE)</div><div class='ctx'> 		return -EINVAL;</div><div class='hunk'>@@ -506,10 +519,10 @@ static void free_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp)</div><div class='ctx'> 	kfree(qp-&gt;sqp_proxy_rcv);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int qp_has_rq(struct ib_qp_init_attr *attr)</div><div class='add'>+static bool qp_has_rq(struct ib_qp_init_attr *attr)</div><div class='ctx'> {</div><div class='ctx'> 	if (attr-&gt;qp_type == IB_QPT_XRC_INI || attr-&gt;qp_type == IB_QPT_XRC_TGT)</div><div class='del'>-		return 0;</div><div class='add'>+		return false;</div><div class='ctx'> </div><div class='ctx'> 	return !attr-&gt;srq;</div><div class='ctx'> }</div><div class='hunk'>@@ -552,15 +565,15 @@ static int set_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_rss *rss_ctx,</div><div class='ctx'> 		return (-EOPNOTSUPP);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (ucmd-&gt;rx_hash_fields_mask &amp; ~(MLX4_IB_RX_HASH_SRC_IPV4	|</div><div class='del'>-					  MLX4_IB_RX_HASH_DST_IPV4	|</div><div class='del'>-					  MLX4_IB_RX_HASH_SRC_IPV6	|</div><div class='del'>-					  MLX4_IB_RX_HASH_DST_IPV6	|</div><div class='del'>-					  MLX4_IB_RX_HASH_SRC_PORT_TCP	|</div><div class='del'>-					  MLX4_IB_RX_HASH_DST_PORT_TCP	|</div><div class='del'>-					  MLX4_IB_RX_HASH_SRC_PORT_UDP	|</div><div class='del'>-					  MLX4_IB_RX_HASH_DST_PORT_UDP  |</div><div class='del'>-					  MLX4_IB_RX_HASH_INNER)) {</div><div class='add'>+	if (ucmd-&gt;rx_hash_fields_mask &amp; ~(u64)(MLX4_IB_RX_HASH_SRC_IPV4	|</div><div class='add'>+					       MLX4_IB_RX_HASH_DST_IPV4	|</div><div class='add'>+					       MLX4_IB_RX_HASH_SRC_IPV6	|</div><div class='add'>+					       MLX4_IB_RX_HASH_DST_IPV6	|</div><div class='add'>+					       MLX4_IB_RX_HASH_SRC_PORT_TCP |</div><div class='add'>+					       MLX4_IB_RX_HASH_DST_PORT_TCP |</div><div class='add'>+					       MLX4_IB_RX_HASH_SRC_PORT_UDP |</div><div class='add'>+					       MLX4_IB_RX_HASH_DST_PORT_UDP |</div><div class='add'>+					       MLX4_IB_RX_HASH_INNER)) {</div><div class='ctx'> 		pr_debug("RX Hash fields_mask has unsupported mask (0x%llx)\n",</div><div class='ctx'> 			 ucmd-&gt;rx_hash_fields_mask);</div><div class='ctx'> 		return (-EOPNOTSUPP);</div><div class='hunk'>@@ -656,8 +669,6 @@ static int create_qp_rss(struct mlx4_ib_dev *dev,</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_qpn;</div><div class='ctx'> </div><div class='del'>-	mutex_init(&amp;qp-&gt;mutex);</div><div class='del'>-</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;qp-&gt;gid_list);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;qp-&gt;steering_rules);</div><div class='ctx'> </div><div class='hunk'>@@ -696,80 +707,72 @@ err_qpn:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct ib_qp *_mlx4_ib_create_qp_rss(struct ib_pd *pd,</div><div class='del'>-					    struct ib_qp_init_attr *init_attr,</div><div class='del'>-					    struct ib_udata *udata)</div><div class='add'>+static int _mlx4_ib_create_qp_rss(struct ib_pd *pd, struct mlx4_ib_qp *qp,</div><div class='add'>+				  struct ib_qp_init_attr *init_attr,</div><div class='add'>+				  struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	struct mlx4_ib_qp *qp;</div><div class='ctx'> 	struct mlx4_ib_create_qp_rss ucmd = {};</div><div class='ctx'> 	size_t required_cmd_sz;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	if (!udata) {</div><div class='ctx'> 		pr_debug("RSS QP with NULL udata\n");</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (udata-&gt;outlen)</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	required_cmd_sz = offsetof(typeof(ucmd), reserved1) +</div><div class='ctx'> 					sizeof(ucmd.reserved1);</div><div class='ctx'> 	if (udata-&gt;inlen &lt; required_cmd_sz) {</div><div class='ctx'> 		pr_debug("invalid inlen\n");</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (ib_copy_from_udata(&amp;ucmd, udata, min(sizeof(ucmd), udata-&gt;inlen))) {</div><div class='ctx'> 		pr_debug("copy failed\n");</div><div class='del'>-		return ERR_PTR(-EFAULT);</div><div class='add'>+		return -EFAULT;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (memchr_inv(ucmd.reserved, 0, sizeof(ucmd.reserved)))</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	if (ucmd.comp_mask || ucmd.reserved1)</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	if (udata-&gt;inlen &gt; sizeof(ucmd) &amp;&amp;</div><div class='ctx'> 	    !ib_is_udata_cleared(udata, sizeof(ucmd),</div><div class='ctx'> 				 udata-&gt;inlen - sizeof(ucmd))) {</div><div class='ctx'> 		pr_debug("inlen is not supported\n");</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (init_attr-&gt;qp_type != IB_QPT_RAW_PACKET) {</div><div class='ctx'> 		pr_debug("RSS QP with unsupported QP type %d\n",</div><div class='ctx'> 			 init_attr-&gt;qp_type);</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (init_attr-&gt;create_flags) {</div><div class='ctx'> 		pr_debug("RSS QP doesn't support create flags\n");</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (init_attr-&gt;send_cq || init_attr-&gt;cap.max_send_wr) {</div><div class='ctx'> 		pr_debug("RSS QP with unsupported send attributes\n");</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	qp = kzalloc(sizeof(*qp), GFP_KERNEL);</div><div class='del'>-	if (!qp)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='ctx'> 	qp-&gt;pri.vid = 0xFFFF;</div><div class='ctx'> 	qp-&gt;alt.vid = 0xFFFF;</div><div class='ctx'> </div><div class='ctx'> 	err = create_qp_rss(to_mdev(pd-&gt;device), init_attr, &amp;ucmd, qp);</div><div class='del'>-	if (err) {</div><div class='del'>-		kfree(qp);</div><div class='del'>-		return ERR_PTR(err);</div><div class='del'>-	}</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='ctx'> </div><div class='ctx'> 	qp-&gt;ibqp.qp_num = qp-&gt;mqp.qpn;</div><div class='del'>-</div><div class='del'>-	return &amp;qp-&gt;ibqp;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -849,28 +852,158 @@ static void mlx4_ib_release_wqn(struct mlx4_ib_ucontext *context,</div><div class='ctx'> 	 * reused for further WQN allocations.</div><div class='ctx'> 	 * The next created WQ will allocate a new range.</div><div class='ctx'> 	 */</div><div class='del'>-		range-&gt;dirty = 1;</div><div class='add'>+		range-&gt;dirty = true;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	mutex_unlock(&amp;context-&gt;wqn_ranges_mutex);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,</div><div class='del'>-			    enum mlx4_ib_source_type src,</div><div class='del'>-			    struct ib_qp_init_attr *init_attr,</div><div class='add'>+static int create_rq(struct ib_pd *pd, struct ib_qp_init_attr *init_attr,</div><div class='add'>+		     struct ib_udata *udata, struct mlx4_ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct mlx4_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='add'>+	int qpn;</div><div class='add'>+	int err;</div><div class='add'>+	struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(</div><div class='add'>+		udata, struct mlx4_ib_ucontext, ibucontext);</div><div class='add'>+	struct mlx4_ib_cq *mcq;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	int range_size;</div><div class='add'>+	struct mlx4_ib_create_wq wq;</div><div class='add'>+	size_t copy_len;</div><div class='add'>+	int shift;</div><div class='add'>+	int n;</div><div class='add'>+</div><div class='add'>+	qp-&gt;mlx4_ib_qp_type = MLX4_IB_QPT_RAW_PACKET;</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;qp-&gt;sq.lock);</div><div class='add'>+	spin_lock_init(&amp;qp-&gt;rq.lock);</div><div class='add'>+	INIT_LIST_HEAD(&amp;qp-&gt;gid_list);</div><div class='add'>+	INIT_LIST_HEAD(&amp;qp-&gt;steering_rules);</div><div class='add'>+</div><div class='add'>+	qp-&gt;state = IB_QPS_RESET;</div><div class='add'>+</div><div class='add'>+	copy_len = min(sizeof(struct mlx4_ib_create_wq), udata-&gt;inlen);</div><div class='add'>+</div><div class='add'>+	if (ib_copy_from_udata(&amp;wq, udata, copy_len)) {</div><div class='add'>+		err = -EFAULT;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (wq.comp_mask || wq.reserved[0] || wq.reserved[1] ||</div><div class='add'>+	    wq.reserved[2]) {</div><div class='add'>+		pr_debug("user command isn't supported\n");</div><div class='add'>+		err = -EOPNOTSUPP;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (wq.log_range_size &gt; ilog2(dev-&gt;dev-&gt;caps.max_rss_tbl_sz)) {</div><div class='add'>+		pr_debug("WQN range size must be equal or smaller than %d\n",</div><div class='add'>+			 dev-&gt;dev-&gt;caps.max_rss_tbl_sz);</div><div class='add'>+		err = -EOPNOTSUPP;</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+	range_size = 1 &lt;&lt; wq.log_range_size;</div><div class='add'>+</div><div class='add'>+	if (init_attr-&gt;create_flags &amp; IB_QP_CREATE_SCATTER_FCS)</div><div class='add'>+		qp-&gt;flags |= MLX4_IB_QP_SCATTER_FCS;</div><div class='add'>+</div><div class='add'>+	err = set_rq_size(dev, &amp;init_attr-&gt;cap, true, true, qp, qp-&gt;inl_recv_sz);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	qp-&gt;sq_no_prefetch = 1;</div><div class='add'>+	qp-&gt;sq.wqe_cnt = 1;</div><div class='add'>+	qp-&gt;sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;</div><div class='add'>+	qp-&gt;buf_size = (qp-&gt;rq.wqe_cnt &lt;&lt; qp-&gt;rq.wqe_shift) +</div><div class='add'>+		       (qp-&gt;sq.wqe_cnt &lt;&lt; qp-&gt;sq.wqe_shift);</div><div class='add'>+</div><div class='add'>+	qp-&gt;umem = ib_umem_get(pd-&gt;device, wq.buf_addr, qp-&gt;buf_size, 0);</div><div class='add'>+	if (IS_ERR(qp-&gt;umem)) {</div><div class='add'>+		err = PTR_ERR(qp-&gt;umem);</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	shift = mlx4_ib_umem_calc_optimal_mtt_size(qp-&gt;umem, 0, &amp;n);</div><div class='add'>+	if (shift &lt; 0) {</div><div class='add'>+		err = shift;</div><div class='add'>+		goto err_buf;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = mlx4_mtt_init(dev-&gt;dev, n, shift, &amp;qp-&gt;mtt);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_buf;</div><div class='add'>+</div><div class='add'>+	err = mlx4_ib_umem_write_mtt(dev, &amp;qp-&gt;mtt, qp-&gt;umem);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_mtt;</div><div class='add'>+</div><div class='add'>+	err = mlx4_ib_db_map_user(udata, wq.db_addr, &amp;qp-&gt;db);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_mtt;</div><div class='add'>+	qp-&gt;mqp.usage = MLX4_RES_USAGE_USER_VERBS;</div><div class='add'>+</div><div class='add'>+	err = mlx4_ib_alloc_wqn(context, qp, range_size, &amp;qpn);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_wrid;</div><div class='add'>+</div><div class='add'>+	err = mlx4_qp_alloc(dev-&gt;dev, qpn, &amp;qp-&gt;mqp);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_qpn;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Hardware wants QPN written in big-endian order (after</div><div class='add'>+	 * shifting) for send doorbell.  Precompute this value to save</div><div class='add'>+	 * a little bit when posting sends.</div><div class='add'>+	 */</div><div class='add'>+	qp-&gt;doorbell_qpn = swab32(qp-&gt;mqp.qpn &lt;&lt; 8);</div><div class='add'>+</div><div class='add'>+	qp-&gt;mqp.event = mlx4_ib_wq_event;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;dev-&gt;reset_flow_resource_lock, flags);</div><div class='add'>+	mlx4_ib_lock_cqs(to_mcq(init_attr-&gt;send_cq),</div><div class='add'>+			 to_mcq(init_attr-&gt;recv_cq));</div><div class='add'>+	/* Maintain device to QPs access, needed for further handling</div><div class='add'>+	 * via reset flow</div><div class='add'>+	 */</div><div class='add'>+	list_add_tail(&amp;qp-&gt;qps_list, &amp;dev-&gt;qp_list);</div><div class='add'>+	/* Maintain CQ to QPs access, needed for further handling</div><div class='add'>+	 * via reset flow</div><div class='add'>+	 */</div><div class='add'>+	mcq = to_mcq(init_attr-&gt;send_cq);</div><div class='add'>+	list_add_tail(&amp;qp-&gt;cq_send_list, &amp;mcq-&gt;send_qp_list);</div><div class='add'>+	mcq = to_mcq(init_attr-&gt;recv_cq);</div><div class='add'>+	list_add_tail(&amp;qp-&gt;cq_recv_list, &amp;mcq-&gt;recv_qp_list);</div><div class='add'>+	mlx4_ib_unlock_cqs(to_mcq(init_attr-&gt;send_cq),</div><div class='add'>+			   to_mcq(init_attr-&gt;recv_cq));</div><div class='add'>+	spin_unlock_irqrestore(&amp;dev-&gt;reset_flow_resource_lock, flags);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_qpn:</div><div class='add'>+	mlx4_ib_release_wqn(context, qp, 0);</div><div class='add'>+err_wrid:</div><div class='add'>+	mlx4_ib_db_unmap_user(context, &amp;qp-&gt;db);</div><div class='add'>+</div><div class='add'>+err_mtt:</div><div class='add'>+	mlx4_mtt_cleanup(dev-&gt;dev, &amp;qp-&gt;mtt);</div><div class='add'>+err_buf:</div><div class='add'>+	ib_umem_release(qp-&gt;umem);</div><div class='add'>+err:</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int create_qp_common(struct ib_pd *pd, struct ib_qp_init_attr *init_attr,</div><div class='ctx'> 			    struct ib_udata *udata, int sqpn,</div><div class='del'>-			    struct mlx4_ib_qp **caller_qp)</div><div class='add'>+			    struct mlx4_ib_qp *qp)</div><div class='ctx'> {</div><div class='add'>+	struct mlx4_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='ctx'> 	int qpn;</div><div class='ctx'> 	int err;</div><div class='del'>-	struct mlx4_ib_sqp *sqp = NULL;</div><div class='del'>-	struct mlx4_ib_qp *qp;</div><div class='ctx'> 	struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(</div><div class='ctx'> 		udata, struct mlx4_ib_ucontext, ibucontext);</div><div class='ctx'> 	enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr-&gt;qp_type;</div><div class='ctx'> 	struct mlx4_ib_cq *mcq;</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	int range_size = 0;</div><div class='ctx'> </div><div class='ctx'> 	/* When tunneling special qps, we use a plain UD qp */</div><div class='ctx'> 	if (sqpn) {</div><div class='hunk'>@@ -913,76 +1046,41 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,</div><div class='ctx'> 		sqpn = qpn;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!*caller_qp) {</div><div class='del'>-		if (qp_type == MLX4_IB_QPT_SMI || qp_type == MLX4_IB_QPT_GSI ||</div><div class='del'>-		    (qp_type &amp; (MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_SMI_OWNER |</div><div class='del'>-				MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) {</div><div class='del'>-			sqp = kzalloc(sizeof(struct mlx4_ib_sqp), GFP_KERNEL);</div><div class='del'>-			if (!sqp)</div><div class='del'>-				return -ENOMEM;</div><div class='del'>-			qp = &amp;sqp-&gt;qp;</div><div class='del'>-			qp-&gt;pri.vid = 0xFFFF;</div><div class='del'>-			qp-&gt;alt.vid = 0xFFFF;</div><div class='del'>-		} else {</div><div class='del'>-			qp = kzalloc(sizeof(struct mlx4_ib_qp), GFP_KERNEL);</div><div class='del'>-			if (!qp)</div><div class='del'>-				return -ENOMEM;</div><div class='del'>-			qp-&gt;pri.vid = 0xFFFF;</div><div class='del'>-			qp-&gt;alt.vid = 0xFFFF;</div><div class='del'>-		}</div><div class='del'>-	} else</div><div class='del'>-		qp = *caller_qp;</div><div class='add'>+	if (init_attr-&gt;qp_type == IB_QPT_SMI ||</div><div class='add'>+	    init_attr-&gt;qp_type == IB_QPT_GSI || qp_type == MLX4_IB_QPT_SMI ||</div><div class='add'>+	    qp_type == MLX4_IB_QPT_GSI ||</div><div class='add'>+	    (qp_type &amp; (MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_SMI_OWNER |</div><div class='add'>+			MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) {</div><div class='add'>+		qp-&gt;sqp = kzalloc(sizeof(struct mlx4_ib_sqp), GFP_KERNEL);</div><div class='add'>+		if (!qp-&gt;sqp)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	qp-&gt;mlx4_ib_qp_type = qp_type;</div><div class='ctx'> </div><div class='del'>-	mutex_init(&amp;qp-&gt;mutex);</div><div class='ctx'> 	spin_lock_init(&amp;qp-&gt;sq.lock);</div><div class='ctx'> 	spin_lock_init(&amp;qp-&gt;rq.lock);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;qp-&gt;gid_list);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;qp-&gt;steering_rules);</div><div class='ctx'> </div><div class='del'>-	qp-&gt;state	 = IB_QPS_RESET;</div><div class='add'>+	qp-&gt;state = IB_QPS_RESET;</div><div class='ctx'> 	if (init_attr-&gt;sq_sig_type == IB_SIGNAL_ALL_WR)</div><div class='ctx'> 		qp-&gt;sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);</div><div class='ctx'> </div><div class='del'>-</div><div class='ctx'> 	if (udata) {</div><div class='del'>-		union {</div><div class='del'>-			struct mlx4_ib_create_qp qp;</div><div class='del'>-			struct mlx4_ib_create_wq wq;</div><div class='del'>-		} ucmd;</div><div class='add'>+		struct mlx4_ib_create_qp ucmd;</div><div class='ctx'> 		size_t copy_len;</div><div class='ctx'> 		int shift;</div><div class='ctx'> 		int n;</div><div class='ctx'> </div><div class='del'>-		copy_len = (src == MLX4_IB_QP_SRC) ?</div><div class='del'>-			   sizeof(struct mlx4_ib_create_qp) :</div><div class='del'>-			   min(sizeof(struct mlx4_ib_create_wq), udata-&gt;inlen);</div><div class='add'>+		copy_len = sizeof(struct mlx4_ib_create_qp);</div><div class='ctx'> </div><div class='ctx'> 		if (ib_copy_from_udata(&amp;ucmd, udata, copy_len)) {</div><div class='ctx'> 			err = -EFAULT;</div><div class='ctx'> 			goto err;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		if (src == MLX4_IB_RWQ_SRC) {</div><div class='del'>-			if (ucmd.wq.comp_mask || ucmd.wq.reserved[0] ||</div><div class='del'>-			    ucmd.wq.reserved[1] || ucmd.wq.reserved[2]) {</div><div class='del'>-				pr_debug("user command isn't supported\n");</div><div class='del'>-				err = -EOPNOTSUPP;</div><div class='del'>-				goto err;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			if (ucmd.wq.log_range_size &gt;</div><div class='del'>-			    ilog2(dev-&gt;dev-&gt;caps.max_rss_tbl_sz)) {</div><div class='del'>-				pr_debug("WQN range size must be equal or smaller than %d\n",</div><div class='del'>-					 dev-&gt;dev-&gt;caps.max_rss_tbl_sz);</div><div class='del'>-				err = -EOPNOTSUPP;</div><div class='del'>-				goto err;</div><div class='del'>-			}</div><div class='del'>-			range_size = 1 &lt;&lt; ucmd.wq.log_range_size;</div><div class='del'>-		} else {</div><div class='del'>-			qp-&gt;inl_recv_sz = ucmd.qp.inl_recv_sz;</div><div class='del'>-		}</div><div class='add'>+		qp-&gt;inl_recv_sz = ucmd.inl_recv_sz;</div><div class='ctx'> </div><div class='ctx'> 		if (init_attr-&gt;create_flags &amp; IB_QP_CREATE_SCATTER_FCS) {</div><div class='ctx'> 			if (!(dev-&gt;dev-&gt;caps.flags &amp;</div><div class='hunk'>@@ -1000,39 +1098,26 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto err;</div><div class='ctx'> </div><div class='del'>-		if (src == MLX4_IB_QP_SRC) {</div><div class='del'>-			qp-&gt;sq_no_prefetch = ucmd.qp.sq_no_prefetch;</div><div class='add'>+		qp-&gt;sq_no_prefetch = ucmd.sq_no_prefetch;</div><div class='ctx'> </div><div class='del'>-			err = set_user_sq_size(dev, qp,</div><div class='del'>-					       (struct mlx4_ib_create_qp *)</div><div class='del'>-					       &amp;ucmd);</div><div class='del'>-			if (err)</div><div class='del'>-				goto err;</div><div class='del'>-		} else {</div><div class='del'>-			qp-&gt;sq_no_prefetch = 1;</div><div class='del'>-			qp-&gt;sq.wqe_cnt = 1;</div><div class='del'>-			qp-&gt;sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;</div><div class='del'>-			/* Allocated buffer expects to have at least that SQ</div><div class='del'>-			 * size.</div><div class='del'>-			 */</div><div class='del'>-			qp-&gt;buf_size = (qp-&gt;rq.wqe_cnt &lt;&lt; qp-&gt;rq.wqe_shift) +</div><div class='del'>-				(qp-&gt;sq.wqe_cnt &lt;&lt; qp-&gt;sq.wqe_shift);</div><div class='del'>-		}</div><div class='add'>+		err = set_user_sq_size(dev, qp, &amp;ucmd);</div><div class='add'>+		if (err)</div><div class='add'>+			goto err;</div><div class='ctx'> </div><div class='ctx'> 		qp-&gt;umem =</div><div class='del'>-			ib_umem_get(udata,</div><div class='del'>-				    (src == MLX4_IB_QP_SRC) ? ucmd.qp.buf_addr :</div><div class='del'>-							      ucmd.wq.buf_addr,</div><div class='del'>-				    qp-&gt;buf_size, 0, 0);</div><div class='add'>+			ib_umem_get(pd-&gt;device, ucmd.buf_addr, qp-&gt;buf_size, 0);</div><div class='ctx'> 		if (IS_ERR(qp-&gt;umem)) {</div><div class='ctx'> 			err = PTR_ERR(qp-&gt;umem);</div><div class='ctx'> 			goto err;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		n = ib_umem_page_count(qp-&gt;umem);</div><div class='ctx'> 		shift = mlx4_ib_umem_calc_optimal_mtt_size(qp-&gt;umem, 0, &amp;n);</div><div class='del'>-		err = mlx4_mtt_init(dev-&gt;dev, n, shift, &amp;qp-&gt;mtt);</div><div class='add'>+		if (shift &lt; 0) {</div><div class='add'>+			err = shift;</div><div class='add'>+			goto err_buf;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='add'>+		err = mlx4_mtt_init(dev-&gt;dev, n, shift, &amp;qp-&gt;mtt);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto err_buf;</div><div class='ctx'> </div><div class='hunk'>@@ -1041,11 +1126,7 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,</div><div class='ctx'> 			goto err_mtt;</div><div class='ctx'> </div><div class='ctx'> 		if (qp_has_rq(init_attr)) {</div><div class='del'>-			err = mlx4_ib_db_map_user(udata,</div><div class='del'>-						  (src == MLX4_IB_QP_SRC) ?</div><div class='del'>-							  ucmd.qp.db_addr :</div><div class='del'>-							  ucmd.wq.db_addr,</div><div class='del'>-						  &amp;qp-&gt;db);</div><div class='add'>+			err = mlx4_ib_db_map_user(udata, ucmd.db_addr, &amp;qp-&gt;db);</div><div class='ctx'> 			if (err)</div><div class='ctx'> 				goto err_mtt;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -1065,8 +1146,10 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,</div><div class='ctx'> 			if (dev-&gt;steering_support ==</div><div class='ctx'> 			    MLX4_STEERING_MODE_DEVICE_MANAGED)</div><div class='ctx'> 				qp-&gt;flags |= MLX4_IB_QP_NETIF;</div><div class='del'>-			else</div><div class='add'>+			else {</div><div class='add'>+				err = -EINVAL;</div><div class='ctx'> 				goto err;</div><div class='add'>+			}</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='ctx'> 		err = set_kernel_sq_size(dev, &amp;init_attr-&gt;cap, qp_type, qp);</div><div class='hunk'>@@ -1115,10 +1198,6 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,</div><div class='ctx'> 				goto err_wrid;</div><div class='ctx'> 			}</div><div class='ctx'> 		}</div><div class='del'>-	} else if (src == MLX4_IB_RWQ_SRC) {</div><div class='del'>-		err = mlx4_ib_alloc_wqn(context, qp, range_size, &amp;qpn);</div><div class='del'>-		if (err)</div><div class='del'>-			goto err_wrid;</div><div class='ctx'> 	} else {</div><div class='ctx'> 		/* Raw packet QPNs may not have bits 6,7 set in their qp_num;</div><div class='ctx'> 		 * otherwise, the WQE BlueFlame setup flow wrongly causes</div><div class='hunk'>@@ -1157,11 +1236,7 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,</div><div class='ctx'> 	 */</div><div class='ctx'> 	qp-&gt;doorbell_qpn = swab32(qp-&gt;mqp.qpn &lt;&lt; 8);</div><div class='ctx'> </div><div class='del'>-	qp-&gt;mqp.event = (src == MLX4_IB_QP_SRC) ? mlx4_ib_qp_event :</div><div class='del'>-						  mlx4_ib_wq_event;</div><div class='del'>-</div><div class='del'>-	if (!*caller_qp)</div><div class='del'>-		*caller_qp = qp;</div><div class='add'>+	qp-&gt;mqp.event = mlx4_ib_qp_event;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;dev-&gt;reset_flow_resource_lock, flags);</div><div class='ctx'> 	mlx4_ib_lock_cqs(to_mcq(init_attr-&gt;send_cq),</div><div class='hunk'>@@ -1186,8 +1261,6 @@ err_qpn:</div><div class='ctx'> 	if (!sqpn) {</div><div class='ctx'> 		if (qp-&gt;flags &amp; MLX4_IB_QP_NETIF)</div><div class='ctx'> 			mlx4_ib_steer_qp_free(dev, qpn, 1);</div><div class='del'>-		else if (src == MLX4_IB_RWQ_SRC)</div><div class='del'>-			mlx4_ib_release_wqn(context, qp, 0);</div><div class='ctx'> 		else</div><div class='ctx'> 			mlx4_qp_release_range(dev-&gt;dev, qpn, 1);</div><div class='ctx'> 	}</div><div class='hunk'>@@ -1207,20 +1280,16 @@ err_mtt:</div><div class='ctx'> 	mlx4_mtt_cleanup(dev-&gt;dev, &amp;qp-&gt;mtt);</div><div class='ctx'> </div><div class='ctx'> err_buf:</div><div class='del'>-	if (qp-&gt;umem)</div><div class='del'>-		ib_umem_release(qp-&gt;umem);</div><div class='del'>-	else</div><div class='add'>+	if (!qp-&gt;umem)</div><div class='ctx'> 		mlx4_buf_free(dev-&gt;dev, qp-&gt;buf_size, &amp;qp-&gt;buf);</div><div class='add'>+	ib_umem_release(qp-&gt;umem);</div><div class='ctx'> </div><div class='ctx'> err_db:</div><div class='ctx'> 	if (!udata &amp;&amp; qp_has_rq(init_attr))</div><div class='ctx'> 		mlx4_db_free(dev-&gt;dev, &amp;qp-&gt;db);</div><div class='ctx'> </div><div class='ctx'> err:</div><div class='del'>-	if (!sqp &amp;&amp; !*caller_qp)</div><div class='del'>-		kfree(qp);</div><div class='del'>-	kfree(sqp);</div><div class='del'>-</div><div class='add'>+	kfree(qp-&gt;sqp);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1334,7 +1403,6 @@ static void destroy_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)</div><div class='ctx'> 	mlx4_qp_free(dev-&gt;dev, &amp;qp-&gt;mqp);</div><div class='ctx'> 	mlx4_qp_release_range(dev-&gt;dev, qp-&gt;mqp.qpn, 1);</div><div class='ctx'> 	del_gid_entries(qp);</div><div class='del'>-	kfree(qp-&gt;rss_ctx);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,</div><div class='hunk'>@@ -1421,7 +1489,6 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,</div><div class='ctx'> </div><div class='ctx'> 			mlx4_ib_db_unmap_user(mcontext, &amp;qp-&gt;db);</div><div class='ctx'> 		}</div><div class='del'>-		ib_umem_release(qp-&gt;umem);</div><div class='ctx'> 	} else {</div><div class='ctx'> 		kvfree(qp-&gt;sq.wrid);</div><div class='ctx'> 		kvfree(qp-&gt;rq.wrid);</div><div class='hunk'>@@ -1432,6 +1499,7 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,</div><div class='ctx'> 		if (qp-&gt;rq.wqe_cnt)</div><div class='ctx'> 			mlx4_db_free(dev-&gt;dev, &amp;qp-&gt;db);</div><div class='ctx'> 	}</div><div class='add'>+	ib_umem_release(qp-&gt;umem);</div><div class='ctx'> </div><div class='ctx'> 	del_gid_entries(qp);</div><div class='ctx'> }</div><div class='hunk'>@@ -1453,17 +1521,16 @@ static u32 get_sqp_num(struct mlx4_ib_dev *dev, struct ib_qp_init_attr *attr)</div><div class='ctx'> 		return dev-&gt;dev-&gt;caps.spec_qps[attr-&gt;port_num - 1].qp1_proxy;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,</div><div class='del'>-					struct ib_qp_init_attr *init_attr,</div><div class='del'>-					struct ib_udata *udata)</div><div class='add'>+static int _mlx4_ib_create_qp(struct ib_pd *pd, struct mlx4_ib_qp *qp,</div><div class='add'>+			      struct ib_qp_init_attr *init_attr,</div><div class='add'>+			      struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	struct mlx4_ib_qp *qp = NULL;</div><div class='ctx'> 	int err;</div><div class='ctx'> 	int sup_u_create_flags = MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;</div><div class='ctx'> 	u16 xrcdn = 0;</div><div class='ctx'> </div><div class='ctx'> 	if (init_attr-&gt;rwq_ind_tbl)</div><div class='del'>-		return _mlx4_ib_create_qp_rss(pd, init_attr, udata);</div><div class='add'>+		return _mlx4_ib_create_qp_rss(pd, qp, init_attr, udata);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * We only support LSO, vendor flag1, and multicast loopback blocking,</div><div class='hunk'>@@ -1475,16 +1542,16 @@ static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,</div><div class='ctx'> 					MLX4_IB_SRIOV_SQP |</div><div class='ctx'> 					MLX4_IB_QP_NETIF |</div><div class='ctx'> 					MLX4_IB_QP_CREATE_ROCE_V2_GSI))</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	if (init_attr-&gt;create_flags &amp; IB_QP_CREATE_NETIF_QP) {</div><div class='ctx'> 		if (init_attr-&gt;qp_type != IB_QPT_UD)</div><div class='del'>-			return ERR_PTR(-EINVAL);</div><div class='add'>+			return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (init_attr-&gt;create_flags) {</div><div class='ctx'> 		if (udata &amp;&amp; init_attr-&gt;create_flags &amp; ~(sup_u_create_flags))</div><div class='del'>-			return ERR_PTR(-EINVAL);</div><div class='add'>+			return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 		if ((init_attr-&gt;create_flags &amp; ~(MLX4_IB_SRIOV_SQP |</div><div class='ctx'> 						 MLX4_IB_QP_CREATE_ROCE_V2_GSI  |</div><div class='hunk'>@@ -1494,7 +1561,7 @@ static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,</div><div class='ctx'> 		     init_attr-&gt;qp_type &gt; IB_QPT_GSI) ||</div><div class='ctx'> 		    (init_attr-&gt;create_flags &amp; MLX4_IB_QP_CREATE_ROCE_V2_GSI &amp;&amp;</div><div class='ctx'> 		     init_attr-&gt;qp_type != IB_QPT_GSI))</div><div class='del'>-			return ERR_PTR(-EINVAL);</div><div class='add'>+			return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	switch (init_attr-&gt;qp_type) {</div><div class='hunk'>@@ -1502,58 +1569,51 @@ static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,</div><div class='ctx'> 		pd = to_mxrcd(init_attr-&gt;xrcd)-&gt;pd;</div><div class='ctx'> 		xrcdn = to_mxrcd(init_attr-&gt;xrcd)-&gt;xrcdn;</div><div class='ctx'> 		init_attr-&gt;send_cq = to_mxrcd(init_attr-&gt;xrcd)-&gt;cq;</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case IB_QPT_XRC_INI:</div><div class='ctx'> 		if (!(to_mdev(pd-&gt;device)-&gt;dev-&gt;caps.flags &amp; MLX4_DEV_CAP_FLAG_XRC))</div><div class='del'>-			return ERR_PTR(-ENOSYS);</div><div class='add'>+			return -ENOSYS;</div><div class='ctx'> 		init_attr-&gt;recv_cq = init_attr-&gt;send_cq;</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case IB_QPT_RC:</div><div class='ctx'> 	case IB_QPT_UC:</div><div class='ctx'> 	case IB_QPT_RAW_PACKET:</div><div class='del'>-		qp = kzalloc(sizeof(*qp), GFP_KERNEL);</div><div class='del'>-		if (!qp)</div><div class='del'>-			return ERR_PTR(-ENOMEM);</div><div class='add'>+	case IB_QPT_UD:</div><div class='ctx'> 		qp-&gt;pri.vid = 0xFFFF;</div><div class='ctx'> 		qp-&gt;alt.vid = 0xFFFF;</div><div class='del'>-		/* fall through */</div><div class='del'>-	case IB_QPT_UD:</div><div class='del'>-	{</div><div class='del'>-		err = create_qp_common(to_mdev(pd-&gt;device), pd,	MLX4_IB_QP_SRC,</div><div class='del'>-				       init_attr, udata, 0, &amp;qp);</div><div class='del'>-		if (err) {</div><div class='del'>-			kfree(qp);</div><div class='del'>-			return ERR_PTR(err);</div><div class='del'>-		}</div><div class='add'>+		err = create_qp_common(pd, init_attr, udata, 0, qp);</div><div class='add'>+		if (err)</div><div class='add'>+			return err;</div><div class='ctx'> </div><div class='ctx'> 		qp-&gt;ibqp.qp_num = qp-&gt;mqp.qpn;</div><div class='ctx'> 		qp-&gt;xrcdn = xrcdn;</div><div class='del'>-</div><div class='ctx'> 		break;</div><div class='del'>-	}</div><div class='ctx'> 	case IB_QPT_SMI:</div><div class='ctx'> 	case IB_QPT_GSI:</div><div class='ctx'> 	{</div><div class='ctx'> 		int sqpn;</div><div class='ctx'> </div><div class='del'>-		/* Userspace is not allowed to create special QPs: */</div><div class='del'>-		if (udata)</div><div class='del'>-			return ERR_PTR(-EINVAL);</div><div class='ctx'> 		if (init_attr-&gt;create_flags &amp; MLX4_IB_QP_CREATE_ROCE_V2_GSI) {</div><div class='ctx'> 			int res = mlx4_qp_reserve_range(to_mdev(pd-&gt;device)-&gt;dev,</div><div class='ctx'> 							1, 1, &amp;sqpn, 0,</div><div class='ctx'> 							MLX4_RES_USAGE_DRIVER);</div><div class='ctx'> </div><div class='ctx'> 			if (res)</div><div class='del'>-				return ERR_PTR(res);</div><div class='add'>+				return res;</div><div class='ctx'> 		} else {</div><div class='ctx'> 			sqpn = get_sqp_num(to_mdev(pd-&gt;device), init_attr);</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		err = create_qp_common(to_mdev(pd-&gt;device), pd, MLX4_IB_QP_SRC,</div><div class='del'>-				       init_attr, udata, sqpn, &amp;qp);</div><div class='add'>+		qp-&gt;pri.vid = 0xFFFF;</div><div class='add'>+		qp-&gt;alt.vid = 0xFFFF;</div><div class='add'>+		err = create_qp_common(pd, init_attr, udata, sqpn, qp);</div><div class='ctx'> 		if (err)</div><div class='del'>-			return ERR_PTR(err);</div><div class='add'>+			return err;</div><div class='add'>+</div><div class='add'>+		if (init_attr-&gt;create_flags &amp;</div><div class='add'>+		    (MLX4_IB_SRIOV_SQP | MLX4_IB_SRIOV_TUNNEL_QP))</div><div class='add'>+			/* Internal QP created with ib_create_qp */</div><div class='add'>+			rdma_restrack_no_track(&amp;qp-&gt;ibqp.res);</div><div class='ctx'> </div><div class='ctx'> 		qp-&gt;port	= init_attr-&gt;port_num;</div><div class='ctx'> 		qp-&gt;ibqp.qp_num = init_attr-&gt;qp_type == IB_QPT_SMI ? 0 :</div><div class='hunk'>@@ -1562,25 +1622,28 @@ static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,</div><div class='ctx'> 	}</div><div class='ctx'> 	default:</div><div class='ctx'> 		/* Don't support raw QPs */</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='del'>-	return &amp;qp-&gt;ibqp;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,</div><div class='del'>-				struct ib_qp_init_attr *init_attr,</div><div class='del'>-				struct ib_udata *udata) {</div><div class='del'>-	struct ib_device *device = pd ? pd-&gt;device : init_attr-&gt;xrcd-&gt;device;</div><div class='del'>-	struct ib_qp *ibqp;</div><div class='add'>+int mlx4_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,</div><div class='add'>+		      struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *device = ibqp-&gt;device;</div><div class='ctx'> 	struct mlx4_ib_dev *dev = to_mdev(device);</div><div class='add'>+	struct mlx4_ib_qp *qp = to_mqp(ibqp);</div><div class='add'>+	struct ib_pd *pd = ibqp-&gt;pd;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	ibqp = _mlx4_ib_create_qp(pd, init_attr, udata);</div><div class='add'>+	mutex_init(&amp;qp-&gt;mutex);</div><div class='add'>+	ret = _mlx4_ib_create_qp(pd, qp, init_attr, udata);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='del'>-	if (!IS_ERR(ibqp) &amp;&amp;</div><div class='del'>-	    (init_attr-&gt;qp_type == IB_QPT_GSI) &amp;&amp;</div><div class='add'>+	if (init_attr-&gt;qp_type == IB_QPT_GSI &amp;&amp;</div><div class='ctx'> 	    !(init_attr-&gt;create_flags &amp; MLX4_IB_QP_CREATE_ROCE_V2_GSI)) {</div><div class='del'>-		struct mlx4_ib_sqp *sqp = to_msqp((to_mqp(ibqp)));</div><div class='add'>+		struct mlx4_ib_sqp *sqp = qp-&gt;sqp;</div><div class='ctx'> 		int is_eth = rdma_cap_eth_ah(&amp;dev-&gt;ib_dev, init_attr-&gt;port_num);</div><div class='ctx'> </div><div class='ctx'> 		if (is_eth &amp;&amp;</div><div class='hunk'>@@ -1589,17 +1652,18 @@ struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,</div><div class='ctx'> 			sqp-&gt;roce_v2_gsi = ib_create_qp(pd, init_attr);</div><div class='ctx'> </div><div class='ctx'> 			if (IS_ERR(sqp-&gt;roce_v2_gsi)) {</div><div class='del'>-				pr_err("Failed to create GSI QP for RoCEv2 (%ld)\n", PTR_ERR(sqp-&gt;roce_v2_gsi));</div><div class='add'>+				pr_err("Failed to create GSI QP for RoCEv2 (%pe)\n",</div><div class='add'>+				       sqp-&gt;roce_v2_gsi);</div><div class='ctx'> 				sqp-&gt;roce_v2_gsi = NULL;</div><div class='ctx'> 			} else {</div><div class='del'>-				sqp = to_msqp(to_mqp(sqp-&gt;roce_v2_gsi));</div><div class='del'>-				sqp-&gt;qp.flags |= MLX4_IB_ROCE_V2_GSI_QP;</div><div class='add'>+				to_mqp(sqp-&gt;roce_v2_gsi)-&gt;flags |=</div><div class='add'>+					MLX4_IB_ROCE_V2_GSI_QP;</div><div class='ctx'> 			}</div><div class='ctx'> </div><div class='ctx'> 			init_attr-&gt;create_flags &amp;= ~MLX4_IB_QP_CREATE_ROCE_V2_GSI;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='del'>-	return ibqp;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int _mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)</div><div class='hunk'>@@ -1626,11 +1690,7 @@ static int _mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)</div><div class='ctx'> 		destroy_qp_common(dev, mqp, MLX4_IB_QP_SRC, udata);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (is_sqp(dev, mqp))</div><div class='del'>-		kfree(to_msqp(mqp));</div><div class='del'>-	else</div><div class='del'>-		kfree(mqp);</div><div class='del'>-</div><div class='add'>+	kfree(mqp-&gt;sqp);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1639,7 +1699,7 @@ int mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)</div><div class='ctx'> 	struct mlx4_ib_qp *mqp = to_mqp(qp);</div><div class='ctx'> </div><div class='ctx'> 	if (mqp-&gt;mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {</div><div class='del'>-		struct mlx4_ib_sqp *sqp = to_msqp(mqp);</div><div class='add'>+		struct mlx4_ib_sqp *sqp = mqp-&gt;sqp;</div><div class='ctx'> </div><div class='ctx'> 		if (sqp-&gt;roce_v2_gsi)</div><div class='ctx'> 			ib_destroy_qp(sqp-&gt;roce_v2_gsi);</div><div class='hunk'>@@ -1843,7 +1903,7 @@ static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_qp_attr *qp,</div><div class='ctx'> 			 u16 vlan_id, u8 *smac)</div><div class='ctx'> {</div><div class='ctx'> 	return _mlx4_set_path(dev, &amp;qp-&gt;ah_attr,</div><div class='del'>-			      mlx4_mac_to_u64(smac),</div><div class='add'>+			      ether_addr_to_u64(smac),</div><div class='ctx'> 			      vlan_id,</div><div class='ctx'> 			      path, &amp;mqp-&gt;pri, port);</div><div class='ctx'> }</div><div class='hunk'>@@ -2501,7 +2561,7 @@ static int __mlx4_ib_modify_qp(void *src, enum mlx4_ib_source_type src_type,</div><div class='ctx'> 		qp-&gt;alt_port = attr-&gt;alt_port_num;</div><div class='ctx'> </div><div class='ctx'> 	if (is_sqp(dev, qp))</div><div class='del'>-		store_sqp_attrs(to_msqp(qp), attr, attr_mask);</div><div class='add'>+		store_sqp_attrs(qp-&gt;sqp, attr, attr_mask);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * If we moved QP0 to RTR, bring the IB link up; if we moved</div><div class='hunk'>@@ -2775,10 +2835,13 @@ int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> 	struct mlx4_ib_qp *mqp = to_mqp(ibqp);</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	if (attr_mask &amp; ~IB_QP_ATTR_STANDARD_BITS)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='ctx'> 	ret = _mlx4_ib_modify_qp(ibqp, attr, attr_mask, udata);</div><div class='ctx'> </div><div class='ctx'> 	if (mqp-&gt;mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {</div><div class='del'>-		struct mlx4_ib_sqp *sqp = to_msqp(mqp);</div><div class='add'>+		struct mlx4_ib_sqp *sqp = mqp-&gt;sqp;</div><div class='ctx'> 		int err = 0;</div><div class='ctx'> </div><div class='ctx'> 		if (sqp-&gt;roce_v2_gsi)</div><div class='hunk'>@@ -2803,12 +2866,13 @@ static int vf_get_qp0_qkey(struct mlx4_dev *dev, int qpn, u32 *qkey)</div><div class='ctx'> 	return -EINVAL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp,</div><div class='add'>+static int build_sriov_qp0_header(struct mlx4_ib_qp *qp,</div><div class='ctx'> 				  const struct ib_ud_wr *wr,</div><div class='ctx'> 				  void *wqe, unsigned *mlx_seg_len)</div><div class='ctx'> {</div><div class='del'>-	struct mlx4_ib_dev *mdev = to_mdev(sqp-&gt;qp.ibqp.device);</div><div class='del'>-	struct ib_device *ib_dev = &amp;mdev-&gt;ib_dev;</div><div class='add'>+	struct mlx4_ib_dev *mdev = to_mdev(qp-&gt;ibqp.device);</div><div class='add'>+	struct mlx4_ib_sqp *sqp = qp-&gt;sqp;</div><div class='add'>+	struct ib_device *ib_dev = qp-&gt;ibqp.device;</div><div class='ctx'> 	struct mlx4_wqe_mlx_seg *mlx = wqe;</div><div class='ctx'> 	struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;</div><div class='ctx'> 	struct mlx4_ib_ah *ah = to_mah(wr-&gt;ah);</div><div class='hunk'>@@ -2817,6 +2881,7 @@ static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp,</div><div class='ctx'> 	int send_size;</div><div class='ctx'> 	int header_size;</div><div class='ctx'> 	int spc;</div><div class='add'>+	int err;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='ctx'> 	if (wr-&gt;wr.opcode != IB_WR_SEND)</div><div class='hunk'>@@ -2829,12 +2894,12 @@ static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp,</div><div class='ctx'> </div><div class='ctx'> 	/* for proxy-qp0 sends, need to add in size of tunnel header */</div><div class='ctx'> 	/* for tunnel-qp0 sends, tunnel header is already in s/g list */</div><div class='del'>-	if (sqp-&gt;qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER)</div><div class='add'>+	if (qp-&gt;mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER)</div><div class='ctx'> 		send_size += sizeof (struct mlx4_ib_tunnel_header);</div><div class='ctx'> </div><div class='ctx'> 	ib_ud_header_init(send_size, 1, 0, 0, 0, 0, 0, 0, &amp;sqp-&gt;ud_header);</div><div class='ctx'> </div><div class='del'>-	if (sqp-&gt;qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) {</div><div class='add'>+	if (qp-&gt;mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) {</div><div class='ctx'> 		sqp-&gt;ud_header.lrh.service_level =</div><div class='ctx'> 			be32_to_cpu(ah-&gt;av.ib.sl_tclass_flowlabel) &gt;&gt; 28;</div><div class='ctx'> 		sqp-&gt;ud_header.lrh.destination_lid =</div><div class='hunk'>@@ -2851,24 +2916,26 @@ static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp,</div><div class='ctx'> </div><div class='ctx'> 	sqp-&gt;ud_header.lrh.virtual_lane    = 0;</div><div class='ctx'> 	sqp-&gt;ud_header.bth.solicited_event = !!(wr-&gt;wr.send_flags &amp; IB_SEND_SOLICITED);</div><div class='del'>-	ib_get_cached_pkey(ib_dev, sqp-&gt;qp.port, 0, &amp;pkey);</div><div class='add'>+	err = ib_get_cached_pkey(ib_dev, qp-&gt;port, 0, &amp;pkey);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='ctx'> 	sqp-&gt;ud_header.bth.pkey = cpu_to_be16(pkey);</div><div class='del'>-	if (sqp-&gt;qp.mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER)</div><div class='add'>+	if (qp-&gt;mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER)</div><div class='ctx'> 		sqp-&gt;ud_header.bth.destination_qpn = cpu_to_be32(wr-&gt;remote_qpn);</div><div class='ctx'> 	else</div><div class='ctx'> 		sqp-&gt;ud_header.bth.destination_qpn =</div><div class='del'>-			cpu_to_be32(mdev-&gt;dev-&gt;caps.spec_qps[sqp-&gt;qp.port - 1].qp0_tunnel);</div><div class='add'>+			cpu_to_be32(mdev-&gt;dev-&gt;caps.spec_qps[qp-&gt;port - 1].qp0_tunnel);</div><div class='ctx'> </div><div class='ctx'> 	sqp-&gt;ud_header.bth.psn = cpu_to_be32((sqp-&gt;send_psn++) &amp; ((1 &lt;&lt; 24) - 1));</div><div class='ctx'> 	if (mlx4_is_master(mdev-&gt;dev)) {</div><div class='del'>-		if (mlx4_get_parav_qkey(mdev-&gt;dev, sqp-&gt;qp.mqp.qpn, &amp;qkey))</div><div class='add'>+		if (mlx4_get_parav_qkey(mdev-&gt;dev, qp-&gt;mqp.qpn, &amp;qkey))</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 	} else {</div><div class='del'>-		if (vf_get_qp0_qkey(mdev-&gt;dev, sqp-&gt;qp.mqp.qpn, &amp;qkey))</div><div class='add'>+		if (vf_get_qp0_qkey(mdev-&gt;dev, qp-&gt;mqp.qpn, &amp;qkey))</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> 	sqp-&gt;ud_header.deth.qkey = cpu_to_be32(qkey);</div><div class='del'>-	sqp-&gt;ud_header.deth.source_qpn = cpu_to_be32(sqp-&gt;qp.mqp.qpn);</div><div class='add'>+	sqp-&gt;ud_header.deth.source_qpn = cpu_to_be32(qp-&gt;mqp.qpn);</div><div class='ctx'> </div><div class='ctx'> 	sqp-&gt;ud_header.bth.opcode        = IB_OPCODE_UD_SEND_ONLY;</div><div class='ctx'> 	sqp-&gt;ud_header.immediate_present = 0;</div><div class='hunk'>@@ -2952,10 +3019,11 @@ static int fill_gid_by_hw_index(struct mlx4_ib_dev *ibdev, u8 port_num,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> #define MLX4_ROCEV2_QP1_SPORT 0xC000</div><div class='del'>-static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr,</div><div class='add'>+static int build_mlx_header(struct mlx4_ib_qp *qp, const struct ib_ud_wr *wr,</div><div class='ctx'> 			    void *wqe, unsigned *mlx_seg_len)</div><div class='ctx'> {</div><div class='del'>-	struct ib_device *ib_dev = sqp-&gt;qp.ibqp.device;</div><div class='add'>+	struct mlx4_ib_sqp *sqp = qp-&gt;sqp;</div><div class='add'>+	struct ib_device *ib_dev = qp-&gt;ibqp.device;</div><div class='ctx'> 	struct mlx4_ib_dev *ibdev = to_mdev(ib_dev);</div><div class='ctx'> 	struct mlx4_wqe_mlx_seg *mlx = wqe;</div><div class='ctx'> 	struct mlx4_wqe_ctrl_seg *ctrl = wqe;</div><div class='hunk'>@@ -2979,7 +3047,7 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr,</div><div class='ctx'> 	for (i = 0; i &lt; wr-&gt;wr.num_sge; ++i)</div><div class='ctx'> 		send_size += wr-&gt;wr.sg_list[i].length;</div><div class='ctx'> </div><div class='del'>-	is_eth = rdma_port_get_link_layer(sqp-&gt;qp.ibqp.device, sqp-&gt;qp.port) == IB_LINK_LAYER_ETHERNET;</div><div class='add'>+	is_eth = rdma_port_get_link_layer(qp-&gt;ibqp.device, qp-&gt;port) == IB_LINK_LAYER_ETHERNET;</div><div class='ctx'> 	is_grh = mlx4_ib_ah_grh_present(ah);</div><div class='ctx'> 	if (is_eth) {</div><div class='ctx'> 		enum ib_gid_type gid_type;</div><div class='hunk'>@@ -2993,9 +3061,9 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr,</div><div class='ctx'> 			if (err)</div><div class='ctx'> 				return err;</div><div class='ctx'> 		} else  {</div><div class='del'>-			err = fill_gid_by_hw_index(ibdev, sqp-&gt;qp.port,</div><div class='del'>-					    ah-&gt;av.ib.gid_index,</div><div class='del'>-					    &amp;sgid, &amp;gid_type);</div><div class='add'>+			err = fill_gid_by_hw_index(ibdev, qp-&gt;port,</div><div class='add'>+						   ah-&gt;av.ib.gid_index, &amp;sgid,</div><div class='add'>+						   &amp;gid_type);</div><div class='ctx'> 			if (!err) {</div><div class='ctx'> 				is_udp = gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;</div><div class='ctx'> 				if (is_udp) {</div><div class='hunk'>@@ -3011,7 +3079,7 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr,</div><div class='ctx'> 		}</div><div class='ctx'> 		if (ah-&gt;av.eth.vlan != cpu_to_be16(0xffff)) {</div><div class='ctx'> 			vlan = be16_to_cpu(ah-&gt;av.eth.vlan) &amp; 0x0fff;</div><div class='del'>-			is_vlan = 1;</div><div class='add'>+			is_vlan = true;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> 	err = ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh,</div><div class='hunk'>@@ -3040,13 +3108,18 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr,</div><div class='ctx'> 				 * indexes don't necessarily match the hw ones, so</div><div class='ctx'> 				 * we must use our own cache</div><div class='ctx'> 				 */</div><div class='del'>-				sqp-&gt;ud_header.grh.source_gid.global.subnet_prefix =</div><div class='del'>-					cpu_to_be64(atomic64_read(&amp;(to_mdev(ib_dev)-&gt;sriov.</div><div class='del'>-								    demux[sqp-&gt;qp.port - 1].</div><div class='del'>-								    subnet_prefix)));</div><div class='del'>-				sqp-&gt;ud_header.grh.source_gid.global.interface_id =</div><div class='del'>-					to_mdev(ib_dev)-&gt;sriov.demux[sqp-&gt;qp.port - 1].</div><div class='del'>-						       guid_cache[ah-&gt;av.ib.gid_index];</div><div class='add'>+				sqp-&gt;ud_header.grh.source_gid.global</div><div class='add'>+					.subnet_prefix =</div><div class='add'>+					cpu_to_be64(atomic64_read(</div><div class='add'>+						&amp;(to_mdev(ib_dev)</div><div class='add'>+							  -&gt;sriov</div><div class='add'>+							  .demux[qp-&gt;port - 1]</div><div class='add'>+							  .subnet_prefix)));</div><div class='add'>+				sqp-&gt;ud_header.grh.source_gid.global</div><div class='add'>+					.interface_id =</div><div class='add'>+					to_mdev(ib_dev)</div><div class='add'>+						-&gt;sriov.demux[qp-&gt;port - 1]</div><div class='add'>+						.guid_cache[ah-&gt;av.ib.gid_index];</div><div class='ctx'> 			} else {</div><div class='ctx'> 				sqp-&gt;ud_header.grh.source_gid =</div><div class='ctx'> 					ah-&gt;ibah.sgid_attr-&gt;gid;</div><div class='hunk'>@@ -3078,10 +3151,13 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr,</div><div class='ctx'> 	mlx-&gt;flags &amp;= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);</div><div class='ctx'> </div><div class='ctx'> 	if (!is_eth) {</div><div class='del'>-		mlx-&gt;flags |= cpu_to_be32((!sqp-&gt;qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) |</div><div class='del'>-					  (sqp-&gt;ud_header.lrh.destination_lid ==</div><div class='del'>-					   IB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) |</div><div class='del'>-					  (sqp-&gt;ud_header.lrh.service_level &lt;&lt; 8));</div><div class='add'>+		mlx-&gt;flags |=</div><div class='add'>+			cpu_to_be32((!qp-&gt;ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) |</div><div class='add'>+				    (sqp-&gt;ud_header.lrh.destination_lid ==</div><div class='add'>+						     IB_LID_PERMISSIVE ?</div><div class='add'>+					     MLX4_WQE_MLX_SLR :</div><div class='add'>+					     0) |</div><div class='add'>+				    (sqp-&gt;ud_header.lrh.service_level &lt;&lt; 8));</div><div class='ctx'> 		if (ah-&gt;av.ib.port_pd &amp; cpu_to_be32(0x80000000))</div><div class='ctx'> 			mlx-&gt;flags |= cpu_to_be32(0x1); /* force loopback */</div><div class='ctx'> 		mlx-&gt;rlid = sqp-&gt;ud_header.lrh.destination_lid;</div><div class='hunk'>@@ -3102,7 +3178,6 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (is_eth) {</div><div class='del'>-		struct in6_addr in6;</div><div class='ctx'> 		u16 ether_type;</div><div class='ctx'> 		u16 pcp = (be32_to_cpu(ah-&gt;av.ib.sl_tclass_flowlabel) &gt;&gt; 29) &lt;&lt; 13;</div><div class='ctx'> </div><div class='hunk'>@@ -3112,11 +3187,9 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr,</div><div class='ctx'> 		mlx-&gt;sched_prio = cpu_to_be16(pcp);</div><div class='ctx'> </div><div class='ctx'> 		ether_addr_copy(sqp-&gt;ud_header.eth.smac_h, ah-&gt;av.eth.s_mac);</div><div class='del'>-		memcpy(sqp-&gt;ud_header.eth.dmac_h, ah-&gt;av.eth.mac, 6);</div><div class='add'>+		ether_addr_copy(sqp-&gt;ud_header.eth.dmac_h, ah-&gt;av.eth.mac);</div><div class='ctx'> 		memcpy(&amp;ctrl-&gt;srcrb_flags16[0], ah-&gt;av.eth.mac, 2);</div><div class='ctx'> 		memcpy(&amp;ctrl-&gt;imm, ah-&gt;av.eth.mac + 2, 4);</div><div class='del'>-		memcpy(&amp;in6, sgid.raw, sizeof(in6));</div><div class='del'>-</div><div class='ctx'> </div><div class='ctx'> 		if (!memcmp(sqp-&gt;ud_header.eth.smac_h, sqp-&gt;ud_header.eth.dmac_h, 6))</div><div class='ctx'> 			mlx-&gt;flags |= cpu_to_be32(MLX4_WQE_CTRL_FORCE_LOOPBACK);</div><div class='hunk'>@@ -3127,26 +3200,33 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr,</div><div class='ctx'> 			sqp-&gt;ud_header.vlan.tag = cpu_to_be16(vlan | pcp);</div><div class='ctx'> 		}</div><div class='ctx'> 	} else {</div><div class='del'>-		sqp-&gt;ud_header.lrh.virtual_lane    = !sqp-&gt;qp.ibqp.qp_num ? 15 :</div><div class='del'>-							sl_to_vl(to_mdev(ib_dev),</div><div class='del'>-								 sqp-&gt;ud_header.lrh.service_level,</div><div class='del'>-								 sqp-&gt;qp.port);</div><div class='del'>-		if (sqp-&gt;qp.ibqp.qp_num &amp;&amp; sqp-&gt;ud_header.lrh.virtual_lane == 15)</div><div class='add'>+		sqp-&gt;ud_header.lrh.virtual_lane =</div><div class='add'>+			!qp-&gt;ibqp.qp_num ?</div><div class='add'>+				15 :</div><div class='add'>+				sl_to_vl(to_mdev(ib_dev),</div><div class='add'>+					 sqp-&gt;ud_header.lrh.service_level,</div><div class='add'>+					 qp-&gt;port);</div><div class='add'>+		if (qp-&gt;ibqp.qp_num &amp;&amp; sqp-&gt;ud_header.lrh.virtual_lane == 15)</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 		if (sqp-&gt;ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)</div><div class='ctx'> 			sqp-&gt;ud_header.lrh.source_lid = IB_LID_PERMISSIVE;</div><div class='ctx'> 	}</div><div class='ctx'> 	sqp-&gt;ud_header.bth.solicited_event = !!(wr-&gt;wr.send_flags &amp; IB_SEND_SOLICITED);</div><div class='del'>-	if (!sqp-&gt;qp.ibqp.qp_num)</div><div class='del'>-		ib_get_cached_pkey(ib_dev, sqp-&gt;qp.port, sqp-&gt;pkey_index, &amp;pkey);</div><div class='add'>+	if (!qp-&gt;ibqp.qp_num)</div><div class='add'>+		err = ib_get_cached_pkey(ib_dev, qp-&gt;port, sqp-&gt;pkey_index,</div><div class='add'>+					 &amp;pkey);</div><div class='ctx'> 	else</div><div class='del'>-		ib_get_cached_pkey(ib_dev, sqp-&gt;qp.port, wr-&gt;pkey_index, &amp;pkey);</div><div class='add'>+		err = ib_get_cached_pkey(ib_dev, qp-&gt;port, wr-&gt;pkey_index,</div><div class='add'>+					 &amp;pkey);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='ctx'> 	sqp-&gt;ud_header.bth.pkey = cpu_to_be16(pkey);</div><div class='ctx'> 	sqp-&gt;ud_header.bth.destination_qpn = cpu_to_be32(wr-&gt;remote_qpn);</div><div class='ctx'> 	sqp-&gt;ud_header.bth.psn = cpu_to_be32((sqp-&gt;send_psn++) &amp; ((1 &lt;&lt; 24) - 1));</div><div class='ctx'> 	sqp-&gt;ud_header.deth.qkey = cpu_to_be32(wr-&gt;remote_qkey &amp; 0x80000000 ?</div><div class='ctx'> 					       sqp-&gt;qkey : wr-&gt;remote_qkey);</div><div class='del'>-	sqp-&gt;ud_header.deth.source_qpn = cpu_to_be32(sqp-&gt;qp.ibqp.qp_num);</div><div class='add'>+	sqp-&gt;ud_header.deth.source_qpn = cpu_to_be32(qp-&gt;ibqp.qp_num);</div><div class='ctx'> </div><div class='ctx'> 	header_size = ib_ud_header_pack(&amp;sqp-&gt;ud_header, sqp-&gt;header_buf);</div><div class='ctx'> </div><div class='hunk'>@@ -3459,24 +3539,24 @@ static int _mlx4_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,</div><div class='ctx'> 	int nreq;</div><div class='ctx'> 	int err = 0;</div><div class='ctx'> 	unsigned ind;</div><div class='del'>-	int uninitialized_var(size);</div><div class='del'>-	unsigned uninitialized_var(seglen);</div><div class='add'>+	int size;</div><div class='add'>+	unsigned seglen;</div><div class='ctx'> 	__be32 dummy;</div><div class='ctx'> 	__be32 *lso_wqe;</div><div class='del'>-	__be32 uninitialized_var(lso_hdr_sz);</div><div class='add'>+	__be32 lso_hdr_sz;</div><div class='ctx'> 	__be32 blh;</div><div class='ctx'> 	int i;</div><div class='ctx'> 	struct mlx4_ib_dev *mdev = to_mdev(ibqp-&gt;device);</div><div class='ctx'> </div><div class='ctx'> 	if (qp-&gt;mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {</div><div class='del'>-		struct mlx4_ib_sqp *sqp = to_msqp(qp);</div><div class='add'>+		struct mlx4_ib_sqp *sqp = qp-&gt;sqp;</div><div class='ctx'> </div><div class='ctx'> 		if (sqp-&gt;roce_v2_gsi) {</div><div class='ctx'> 			struct mlx4_ib_ah *ah = to_mah(ud_wr(wr)-&gt;ah);</div><div class='ctx'> 			enum ib_gid_type gid_type;</div><div class='ctx'> 			union ib_gid gid;</div><div class='ctx'> </div><div class='del'>-			if (!fill_gid_by_hw_index(mdev, sqp-&gt;qp.port,</div><div class='add'>+			if (!fill_gid_by_hw_index(mdev, qp-&gt;port,</div><div class='ctx'> 					   ah-&gt;av.ib.gid_index,</div><div class='ctx'> 					   &amp;gid, &amp;gid_type))</div><div class='ctx'> 				qp = (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ?</div><div class='hunk'>@@ -3596,8 +3676,8 @@ static int _mlx4_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,</div><div class='ctx'> 			break;</div><div class='ctx'> </div><div class='ctx'> 		case MLX4_IB_QPT_TUN_SMI_OWNER:</div><div class='del'>-			err =  build_sriov_qp0_header(to_msqp(qp), ud_wr(wr),</div><div class='del'>-					ctrl, &amp;seglen);</div><div class='add'>+			err = build_sriov_qp0_header(qp, ud_wr(wr), ctrl,</div><div class='add'>+						     &amp;seglen);</div><div class='ctx'> 			if (unlikely(err)) {</div><div class='ctx'> 				*bad_wr = wr;</div><div class='ctx'> 				goto out;</div><div class='hunk'>@@ -3633,8 +3713,8 @@ static int _mlx4_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,</div><div class='ctx'> 			break;</div><div class='ctx'> </div><div class='ctx'> 		case MLX4_IB_QPT_PROXY_SMI_OWNER:</div><div class='del'>-			err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr),</div><div class='del'>-					ctrl, &amp;seglen);</div><div class='add'>+			err = build_sriov_qp0_header(qp, ud_wr(wr), ctrl,</div><div class='add'>+						     &amp;seglen);</div><div class='ctx'> 			if (unlikely(err)) {</div><div class='ctx'> 				*bad_wr = wr;</div><div class='ctx'> 				goto out;</div><div class='hunk'>@@ -3667,8 +3747,7 @@ static int _mlx4_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,</div><div class='ctx'> </div><div class='ctx'> 		case MLX4_IB_QPT_SMI:</div><div class='ctx'> 		case MLX4_IB_QPT_GSI:</div><div class='del'>-			err = build_mlx_header(to_msqp(qp), ud_wr(wr), ctrl,</div><div class='del'>-					&amp;seglen);</div><div class='add'>+			err = build_mlx_header(qp, ud_wr(wr), ctrl, &amp;seglen);</div><div class='ctx'> 			if (unlikely(err)) {</div><div class='ctx'> 				*bad_wr = wr;</div><div class='ctx'> 				goto out;</div><div class='hunk'>@@ -3976,7 +4055,9 @@ int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr</div><div class='ctx'> 	qp_attr-&gt;qp_access_flags     =</div><div class='ctx'> 		to_ib_qp_access_flags(be32_to_cpu(context.params2));</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;ibqp.qp_type == IB_QPT_RC || qp-&gt;ibqp.qp_type == IB_QPT_UC) {</div><div class='add'>+	if (qp-&gt;ibqp.qp_type == IB_QPT_RC || qp-&gt;ibqp.qp_type == IB_QPT_UC ||</div><div class='add'>+	    qp-&gt;ibqp.qp_type == IB_QPT_XRC_INI ||</div><div class='add'>+	    qp-&gt;ibqp.qp_type == IB_QPT_XRC_TGT) {</div><div class='ctx'> 		to_rdma_ah_attr(dev, &amp;qp_attr-&gt;ah_attr, &amp;context.pri_path);</div><div class='ctx'> 		to_rdma_ah_attr(dev, &amp;qp_attr-&gt;alt_ah_attr, &amp;context.alt_path);</div><div class='ctx'> 		qp_attr-&gt;alt_pkey_index = context.alt_path.pkey_index &amp; 0x7f;</div><div class='hunk'>@@ -4048,8 +4129,8 @@ struct ib_wq *mlx4_ib_create_wq(struct ib_pd *pd,</div><div class='ctx'> 				struct ib_wq_init_attr *init_attr,</div><div class='ctx'> 				struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	struct mlx4_ib_dev *dev;</div><div class='del'>-	struct ib_qp_init_attr ib_qp_init_attr;</div><div class='add'>+	struct mlx4_dev *dev = to_mdev(pd-&gt;device)-&gt;dev;</div><div class='add'>+	struct ib_qp_init_attr ib_qp_init_attr = {};</div><div class='ctx'> 	struct mlx4_ib_qp *qp;</div><div class='ctx'> 	struct mlx4_ib_create_wq ucmd;</div><div class='ctx'> 	int err, required_cmd_sz;</div><div class='hunk'>@@ -4074,14 +4155,13 @@ struct ib_wq *mlx4_ib_create_wq(struct ib_pd *pd,</div><div class='ctx'> 	if (udata-&gt;outlen)</div><div class='ctx'> 		return ERR_PTR(-EOPNOTSUPP);</div><div class='ctx'> </div><div class='del'>-	dev = to_mdev(pd-&gt;device);</div><div class='del'>-</div><div class='ctx'> 	if (init_attr-&gt;wq_type != IB_WQT_RQ) {</div><div class='ctx'> 		pr_debug("unsupported wq type %d\n", init_attr-&gt;wq_type);</div><div class='ctx'> 		return ERR_PTR(-EOPNOTSUPP);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;create_flags &amp; ~IB_WQ_FLAGS_SCATTER_FCS) {</div><div class='add'>+	if (init_attr-&gt;create_flags &amp; ~IB_WQ_FLAGS_SCATTER_FCS ||</div><div class='add'>+	    !(dev-&gt;caps.flags &amp; MLX4_DEV_CAP_FLAG_FCS_KEEP)) {</div><div class='ctx'> 		pr_debug("unsupported create_flags %u\n",</div><div class='ctx'> 			 init_attr-&gt;create_flags);</div><div class='ctx'> 		return ERR_PTR(-EOPNOTSUPP);</div><div class='hunk'>@@ -4091,10 +4171,10 @@ struct ib_wq *mlx4_ib_create_wq(struct ib_pd *pd,</div><div class='ctx'> 	if (!qp)</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='add'>+	mutex_init(&amp;qp-&gt;mutex);</div><div class='ctx'> 	qp-&gt;pri.vid = 0xFFFF;</div><div class='ctx'> 	qp-&gt;alt.vid = 0xFFFF;</div><div class='ctx'> </div><div class='del'>-	memset(&amp;ib_qp_init_attr, 0, sizeof(ib_qp_init_attr));</div><div class='ctx'> 	ib_qp_init_attr.qp_context = init_attr-&gt;wq_context;</div><div class='ctx'> 	ib_qp_init_attr.qp_type = IB_QPT_RAW_PACKET;</div><div class='ctx'> 	ib_qp_init_attr.cap.max_recv_wr = init_attr-&gt;max_wr;</div><div class='hunk'>@@ -4105,8 +4185,7 @@ struct ib_wq *mlx4_ib_create_wq(struct ib_pd *pd,</div><div class='ctx'> 	if (init_attr-&gt;create_flags &amp; IB_WQ_FLAGS_SCATTER_FCS)</div><div class='ctx'> 		ib_qp_init_attr.create_flags |= IB_QP_CREATE_SCATTER_FCS;</div><div class='ctx'> </div><div class='del'>-	err = create_qp_common(dev, pd, MLX4_IB_RWQ_SRC, &amp;ib_qp_init_attr,</div><div class='del'>-			       udata, 0, &amp;qp);</div><div class='add'>+	err = create_rq(pd, &amp;ib_qp_init_attr, udata, qp);</div><div class='ctx'> 	if (err) {</div><div class='ctx'> 		kfree(qp);</div><div class='ctx'> 		return ERR_PTR(err);</div><div class='hunk'>@@ -4215,13 +4294,8 @@ int mlx4_ib_modify_wq(struct ib_wq *ibwq, struct ib_wq_attr *wq_attr,</div><div class='ctx'> 	if (wq_attr_mask &amp; IB_WQ_FLAGS)</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='del'>-	cur_state = wq_attr_mask &amp; IB_WQ_CUR_STATE ? wq_attr-&gt;curr_wq_state :</div><div class='del'>-						     ibwq-&gt;state;</div><div class='del'>-	new_state = wq_attr_mask &amp; IB_WQ_STATE ? wq_attr-&gt;wq_state : cur_state;</div><div class='del'>-</div><div class='del'>-	if (cur_state  &lt; IB_WQS_RESET || cur_state  &gt; IB_WQS_ERR ||</div><div class='del'>-	    new_state &lt; IB_WQS_RESET || new_state &gt; IB_WQS_ERR)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	cur_state = wq_attr-&gt;curr_wq_state;</div><div class='add'>+	new_state = wq_attr-&gt;wq_state;</div><div class='ctx'> </div><div class='ctx'> 	if ((new_state == IB_WQS_RDY) &amp;&amp; (cur_state == IB_WQS_ERR))</div><div class='ctx'> 		return -EINVAL;</div><div class='hunk'>@@ -4259,38 +4333,35 @@ int mlx4_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata)</div><div class='ctx'> 	destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, udata);</div><div class='ctx'> </div><div class='ctx'> 	kfree(qp);</div><div class='del'>-</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_rwq_ind_table</div><div class='del'>-*mlx4_ib_create_rwq_ind_table(struct ib_device *device,</div><div class='del'>-			      struct ib_rwq_ind_table_init_attr *init_attr,</div><div class='del'>-			      struct ib_udata *udata)</div><div class='add'>+int mlx4_ib_create_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table,</div><div class='add'>+				 struct ib_rwq_ind_table_init_attr *init_attr,</div><div class='add'>+				 struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	struct ib_rwq_ind_table *rwq_ind_table;</div><div class='ctx'> 	struct mlx4_ib_create_rwq_ind_tbl_resp resp = {};</div><div class='ctx'> 	unsigned int ind_tbl_size = 1 &lt;&lt; init_attr-&gt;log_ind_tbl_size;</div><div class='add'>+	struct ib_device *device = rwq_ind_table-&gt;device;</div><div class='ctx'> 	unsigned int base_wqn;</div><div class='ctx'> 	size_t min_resp_len;</div><div class='del'>-	int i;</div><div class='del'>-	int err;</div><div class='add'>+	int i, err = 0;</div><div class='ctx'> </div><div class='ctx'> 	if (udata-&gt;inlen &gt; 0 &amp;&amp;</div><div class='ctx'> 	    !ib_is_udata_cleared(udata, 0,</div><div class='ctx'> 				 udata-&gt;inlen))</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);</div><div class='ctx'> 	if (udata-&gt;outlen &amp;&amp; udata-&gt;outlen &lt; min_resp_len)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	if (ind_tbl_size &gt;</div><div class='ctx'> 	    device-&gt;attrs.rss_caps.max_rwq_indirection_table_size) {</div><div class='ctx'> 		pr_debug("log_ind_tbl_size = %d is bigger than supported = %d\n",</div><div class='ctx'> 			 ind_tbl_size,</div><div class='ctx'> 			 device-&gt;attrs.rss_caps.max_rwq_indirection_table_size);</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	base_wqn = init_attr-&gt;ind_tbl[0]-&gt;wq_num;</div><div class='hunk'>@@ -4298,39 +4369,23 @@ struct ib_rwq_ind_table</div><div class='ctx'> 	if (base_wqn % ind_tbl_size) {</div><div class='ctx'> 		pr_debug("WQN=0x%x isn't aligned with indirection table size\n",</div><div class='ctx'> 			 base_wqn);</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	for (i = 1; i &lt; ind_tbl_size; i++) {</div><div class='ctx'> 		if (++base_wqn != init_attr-&gt;ind_tbl[i]-&gt;wq_num) {</div><div class='ctx'> 			pr_debug("indirection table's WQNs aren't consecutive\n");</div><div class='del'>-			return ERR_PTR(-EINVAL);</div><div class='add'>+			return -EINVAL;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	rwq_ind_table = kzalloc(sizeof(*rwq_ind_table), GFP_KERNEL);</div><div class='del'>-	if (!rwq_ind_table)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='ctx'> 	if (udata-&gt;outlen) {</div><div class='ctx'> 		resp.response_length = offsetof(typeof(resp), response_length) +</div><div class='ctx'> 					sizeof(resp.response_length);</div><div class='ctx'> 		err = ib_copy_to_udata(udata, &amp;resp, resp.response_length);</div><div class='del'>-		if (err)</div><div class='del'>-			goto err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return rwq_ind_table;</div><div class='del'>-</div><div class='del'>-err:</div><div class='del'>-	kfree(rwq_ind_table);</div><div class='del'>-	return ERR_PTR(err);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int mlx4_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)</div><div class='del'>-{</div><div class='del'>-	kfree(ib_rwq_ind_tbl);</div><div class='del'>-	return 0;</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct mlx4_ib_drain_cqe {</div><div class='hunk'>@@ -4461,3 +4516,17 @@ void mlx4_ib_drain_rq(struct ib_qp *qp)</div><div class='ctx'> </div><div class='ctx'> 	handle_drain_completion(cq, &amp;rdrain, dev);</div><div class='ctx'> }</div><div class='add'>+</div><div class='add'>+int mlx4_ib_qp_event_init(void)</div><div class='add'>+{</div><div class='add'>+	mlx4_ib_qp_event_wq = alloc_ordered_workqueue("mlx4_ib_qp_event_wq", 0);</div><div class='add'>+	if (!mlx4_ib_qp_event_wq)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx4_ib_qp_event_cleanup(void)</div><div class='add'>+{</div><div class='add'>+	destroy_workqueue(mlx4_ib_qp_event_wq);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c<br/>index 4bf2946b9759..c4cf91235eee 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/srq.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/srq.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/srq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx4/srq.c</a></div><div class='hunk'>@@ -86,6 +86,10 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq,</div><div class='ctx'> 	int err;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='add'>+	if (init_attr-&gt;srq_type != IB_SRQT_BASIC &amp;&amp;</div><div class='add'>+	    init_attr-&gt;srq_type != IB_SRQT_XRC)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='ctx'> 	/* Sanity check SRQ size before proceeding */</div><div class='ctx'> 	if (init_attr-&gt;attr.max_wr  &gt;= dev-&gt;dev-&gt;caps.max_srq_wqes ||</div><div class='ctx'> 	    init_attr-&gt;attr.max_sge &gt;  dev-&gt;dev-&gt;caps.max_srq_sge)</div><div class='hunk'>@@ -110,12 +114,14 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq,</div><div class='ctx'> 		if (ib_copy_from_udata(&amp;ucmd, udata, sizeof(ucmd)))</div><div class='ctx'> 			return -EFAULT;</div><div class='ctx'> </div><div class='del'>-		srq-&gt;umem = ib_umem_get(udata, ucmd.buf_addr, buf_size, 0, 0);</div><div class='add'>+		srq-&gt;umem =</div><div class='add'>+			ib_umem_get(ib_srq-&gt;device, ucmd.buf_addr, buf_size, 0);</div><div class='ctx'> 		if (IS_ERR(srq-&gt;umem))</div><div class='ctx'> 			return PTR_ERR(srq-&gt;umem);</div><div class='ctx'> </div><div class='del'>-		err = mlx4_mtt_init(dev-&gt;dev, ib_umem_page_count(srq-&gt;umem),</div><div class='del'>-				    srq-&gt;umem-&gt;page_shift, &amp;srq-&gt;mtt);</div><div class='add'>+		err = mlx4_mtt_init(</div><div class='add'>+			dev-&gt;dev, ib_umem_num_dma_blocks(srq-&gt;umem, PAGE_SIZE),</div><div class='add'>+			PAGE_SHIFT, &amp;srq-&gt;mtt);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto err_buf;</div><div class='ctx'> </div><div class='hunk'>@@ -204,10 +210,9 @@ err_mtt:</div><div class='ctx'> 	mlx4_mtt_cleanup(dev-&gt;dev, &amp;srq-&gt;mtt);</div><div class='ctx'> </div><div class='ctx'> err_buf:</div><div class='del'>-	if (srq-&gt;umem)</div><div class='del'>-		ib_umem_release(srq-&gt;umem);</div><div class='del'>-	else</div><div class='add'>+	if (!srq-&gt;umem)</div><div class='ctx'> 		mlx4_buf_free(dev-&gt;dev, buf_size, &amp;srq-&gt;buf);</div><div class='add'>+	ib_umem_release(srq-&gt;umem);</div><div class='ctx'> </div><div class='ctx'> err_db:</div><div class='ctx'> 	if (!udata)</div><div class='hunk'>@@ -260,7 +265,7 @@ int mlx4_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)</div><div class='add'>+int mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_ib_dev *dev = to_mdev(srq-&gt;device);</div><div class='ctx'> 	struct mlx4_ib_srq *msrq = to_msrq(srq);</div><div class='hunk'>@@ -275,13 +280,14 @@ void mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)</div><div class='ctx'> 				struct mlx4_ib_ucontext,</div><div class='ctx'> 				ibucontext),</div><div class='ctx'> 			&amp;msrq-&gt;db);</div><div class='del'>-		ib_umem_release(msrq-&gt;umem);</div><div class='ctx'> 	} else {</div><div class='ctx'> 		kvfree(msrq-&gt;wrid);</div><div class='ctx'> 		mlx4_buf_free(dev-&gt;dev, msrq-&gt;msrq.max &lt;&lt; msrq-&gt;msrq.wqe_shift,</div><div class='ctx'> 			      &amp;msrq-&gt;buf);</div><div class='ctx'> 		mlx4_db_free(dev-&gt;dev, &amp;msrq-&gt;db);</div><div class='ctx'> 	}</div><div class='add'>+	ib_umem_release(msrq-&gt;umem);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index)</div><div class='hunk'>@@ -314,7 +320,6 @@ int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,</div><div class='ctx'> 	if (mdev-&gt;dev-&gt;persist-&gt;state &amp; MLX4_DEVICE_STATE_INTERNAL_ERROR) {</div><div class='ctx'> 		err = -EIO;</div><div class='ctx'> 		*bad_wr = wr;</div><div class='del'>-		nreq = 0;</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/hw/mlx4/sysfs.c b/drivers/infiniband/hw/mlx4/sysfs.c<br/>index ea1f3a081b05..88f534cf690e 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/sysfs.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx4/sysfs.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx4/sysfs.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx4/sysfs.c</a></div><div class='hunk'>@@ -56,7 +56,7 @@ static ssize_t show_admin_alias_guid(struct device *dev,</div><div class='ctx'> 					      mlx4_ib_iov_dentry-&gt;entry_num,</div><div class='ctx'> 					      port-&gt;num);</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%llx\n", be64_to_cpu(sysadmin_ag_val));</div><div class='add'>+	return sysfs_emit(buf, "%llx\n", be64_to_cpu(sysadmin_ag_val));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* store_admin_alias_guid stores the (new) administratively assigned value of that GUID.</div><div class='hunk'>@@ -117,22 +117,24 @@ static ssize_t show_port_gid(struct device *dev,</div><div class='ctx'> 	struct mlx4_ib_iov_port *port = mlx4_ib_iov_dentry-&gt;ctx;</div><div class='ctx'> 	struct mlx4_ib_dev *mdev = port-&gt;dev;</div><div class='ctx'> 	union ib_gid gid;</div><div class='del'>-	ssize_t ret;</div><div class='add'>+	int ret;</div><div class='add'>+	__be16 *raw;</div><div class='ctx'> </div><div class='ctx'> 	ret = __mlx4_ib_query_gid(&amp;mdev-&gt;ib_dev, port-&gt;num,</div><div class='ctx'> 				  mlx4_ib_iov_dentry-&gt;entry_num, &amp;gid, 1);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='del'>-	ret = sprintf(buf, "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",</div><div class='del'>-		      be16_to_cpu(((__be16 *) gid.raw)[0]),</div><div class='del'>-		      be16_to_cpu(((__be16 *) gid.raw)[1]),</div><div class='del'>-		      be16_to_cpu(((__be16 *) gid.raw)[2]),</div><div class='del'>-		      be16_to_cpu(((__be16 *) gid.raw)[3]),</div><div class='del'>-		      be16_to_cpu(((__be16 *) gid.raw)[4]),</div><div class='del'>-		      be16_to_cpu(((__be16 *) gid.raw)[5]),</div><div class='del'>-		      be16_to_cpu(((__be16 *) gid.raw)[6]),</div><div class='del'>-		      be16_to_cpu(((__be16 *) gid.raw)[7]));</div><div class='del'>-	return ret;</div><div class='add'>+</div><div class='add'>+	raw = (__be16 *)gid.raw;</div><div class='add'>+	return sysfs_emit(buf, "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",</div><div class='add'>+			  be16_to_cpu(raw[0]),</div><div class='add'>+			  be16_to_cpu(raw[1]),</div><div class='add'>+			  be16_to_cpu(raw[2]),</div><div class='add'>+			  be16_to_cpu(raw[3]),</div><div class='add'>+			  be16_to_cpu(raw[4]),</div><div class='add'>+			  be16_to_cpu(raw[5]),</div><div class='add'>+			  be16_to_cpu(raw[6]),</div><div class='add'>+			  be16_to_cpu(raw[7]));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t show_phys_port_pkey(struct device *dev,</div><div class='hunk'>@@ -151,7 +153,7 @@ static ssize_t show_phys_port_pkey(struct device *dev,</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "0x%04x\n", pkey);</div><div class='add'>+	return sysfs_emit(buf, "0x%04x\n", pkey);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> #define DENTRY_REMOVE(_dentry)						\</div><div class='hunk'>@@ -221,7 +223,7 @@ void del_sysfs_port_mcg_attr(struct mlx4_ib_dev *device, int port_num,</div><div class='ctx'> static int add_port_entries(struct mlx4_ib_dev *device, int port_num)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='del'>-	char buff[11];</div><div class='add'>+	char buff[12];</div><div class='ctx'> 	struct mlx4_ib_iov_port *port = NULL;</div><div class='ctx'> 	int ret = 0 ;</div><div class='ctx'> 	struct ib_port_attr attr;</div><div class='hunk'>@@ -441,16 +443,12 @@ static ssize_t show_port_pkey(struct mlx4_port *p, struct port_attribute *attr,</div><div class='ctx'> {</div><div class='ctx'> 	struct port_table_attribute *tab_attr =</div><div class='ctx'> 		container_of(attr, struct port_table_attribute, attr);</div><div class='del'>-	ssize_t ret = -ENODEV;</div><div class='del'>-</div><div class='del'>-	if (p-&gt;dev-&gt;pkeys.virt2phys_pkey[p-&gt;slave][p-&gt;port_num - 1][tab_attr-&gt;index] &gt;=</div><div class='del'>-	    (p-&gt;dev-&gt;dev-&gt;caps.pkey_table_len[p-&gt;port_num]))</div><div class='del'>-		ret = sprintf(buf, "none\n");</div><div class='del'>-	else</div><div class='del'>-		ret = sprintf(buf, "%d\n",</div><div class='del'>-			      p-&gt;dev-&gt;pkeys.virt2phys_pkey[p-&gt;slave]</div><div class='del'>-			      [p-&gt;port_num - 1][tab_attr-&gt;index]);</div><div class='del'>-	return ret;</div><div class='add'>+	struct pkey_mgt *m = &amp;p-&gt;dev-&gt;pkeys;</div><div class='add'>+	u8 key = m-&gt;virt2phys_pkey[p-&gt;slave][p-&gt;port_num - 1][tab_attr-&gt;index];</div><div class='add'>+</div><div class='add'>+	if (key &gt;= p-&gt;dev-&gt;dev-&gt;caps.pkey_table_len[p-&gt;port_num])</div><div class='add'>+		return sysfs_emit(buf, "none\n");</div><div class='add'>+	return sysfs_emit(buf, "%d\n", key);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t store_port_pkey(struct mlx4_port *p, struct port_attribute *attr,</div><div class='hunk'>@@ -488,7 +486,7 @@ static ssize_t store_port_pkey(struct mlx4_port *p, struct port_attribute *attr,</div><div class='ctx'> static ssize_t show_port_gid_idx(struct mlx4_port *p,</div><div class='ctx'> 				 struct port_attribute *attr, char *buf)</div><div class='ctx'> {</div><div class='del'>-	return sprintf(buf, "%d\n", p-&gt;slave);</div><div class='add'>+	return sysfs_emit(buf, "%d\n", p-&gt;slave);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static struct attribute **</div><div class='hunk'>@@ -542,14 +540,10 @@ static ssize_t sysfs_show_smi_enabled(struct device *dev,</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_port *p =</div><div class='ctx'> 		container_of(attr, struct mlx4_port, smi_enabled);</div><div class='del'>-	ssize_t len = 0;</div><div class='del'>-</div><div class='del'>-	if (mlx4_vf_smi_enabled(p-&gt;dev-&gt;dev, p-&gt;slave, p-&gt;port_num))</div><div class='del'>-		len = sprintf(buf, "%d\n", 1);</div><div class='del'>-	else</div><div class='del'>-		len = sprintf(buf, "%d\n", 0);</div><div class='ctx'> </div><div class='del'>-	return len;</div><div class='add'>+	return sysfs_emit(buf, "%d\n",</div><div class='add'>+			  !!mlx4_vf_smi_enabled(p-&gt;dev-&gt;dev, p-&gt;slave,</div><div class='add'>+						p-&gt;port_num));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t sysfs_show_enable_smi_admin(struct device *dev,</div><div class='hunk'>@@ -558,14 +552,10 @@ static ssize_t sysfs_show_enable_smi_admin(struct device *dev,</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx4_port *p =</div><div class='ctx'> 		container_of(attr, struct mlx4_port, enable_smi_admin);</div><div class='del'>-	ssize_t len = 0;</div><div class='ctx'> </div><div class='del'>-	if (mlx4_vf_get_enable_smi_admin(p-&gt;dev-&gt;dev, p-&gt;slave, p-&gt;port_num))</div><div class='del'>-		len = sprintf(buf, "%d\n", 1);</div><div class='del'>-	else</div><div class='del'>-		len = sprintf(buf, "%d\n", 0);</div><div class='del'>-</div><div class='del'>-	return len;</div><div class='add'>+	return sysfs_emit(buf, "%d\n",</div><div class='add'>+			  !!mlx4_vf_get_enable_smi_admin(p-&gt;dev-&gt;dev, p-&gt;slave,</div><div class='add'>+							 p-&gt;port_num));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t sysfs_store_enable_smi_admin(struct device *dev,</div><div class='hunk'>@@ -808,7 +798,7 @@ static void unregister_pkey_tree(struct mlx4_ib_dev *device)</div><div class='ctx'> </div><div class='ctx'> int mlx4_ib_device_register_sysfs(struct mlx4_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	int i;</div><div class='add'>+	unsigned int i;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='ctx'> 	if (!mlx4_is_master(dev-&gt;dev))</div><div class='hunk'>@@ -827,7 +817,7 @@ int mlx4_ib_device_register_sysfs(struct mlx4_ib_dev *dev)</div><div class='ctx'> 		goto err_ports;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	for (i = 1; i &lt;= dev-&gt;ib_dev.phys_port_cnt; ++i) {</div><div class='add'>+	rdma_for_each_port(&amp;dev-&gt;ib_dev, i) {</div><div class='ctx'> 		ret = add_port_entries(dev, i);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto err_add_entries;</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/Kconfig b/drivers/infiniband/hw/mlx5/Kconfig<br/>index 8d651c05de62..ef1ff42eaec5 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/Kconfig?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/Kconfig</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/Kconfig?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/Kconfig</a></div><div class='hunk'>@@ -1,7 +1,8 @@</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='ctx'> config MLX5_INFINIBAND</div><div class='ctx'> 	tristate "Mellanox 5th generation network adapters (ConnectX series) support"</div><div class='ctx'> 	depends on NETDEVICES &amp;&amp; ETHERNET &amp;&amp; PCI &amp;&amp; MLX5_CORE</div><div class='del'>-	---help---</div><div class='add'>+	help</div><div class='ctx'> 	  This driver provides low-level InfiniBand support for</div><div class='ctx'> 	  Mellanox Connect-IB PCI Express host channel adapters (HCAs).</div><div class='ctx'> 	  This is required to use InfiniBand protocols such as</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/Makefile b/drivers/infiniband/hw/mlx5/Makefile<br/>index 33f5adb14e4e..dd7bb377f491 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/Makefile?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/Makefile</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/Makefile?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/Makefile</a></div><div class='hunk'>@@ -1,9 +1,33 @@</div><div class='del'>-obj-$(CONFIG_MLX5_INFINIBAND)	+= mlx5_ib.o</div><div class='add'>+# SPDX-License-Identifier: GPL-2.0-only</div><div class='add'>+obj-$(CONFIG_MLX5_INFINIBAND) += mlx5_ib.o</div><div class='add'>+</div><div class='add'>+mlx5_ib-y := ah.o \</div><div class='add'>+	     cmd.o \</div><div class='add'>+	     cong.o \</div><div class='add'>+	     counters.o \</div><div class='add'>+	     cq.o \</div><div class='add'>+	     data_direct.o \</div><div class='add'>+	     dm.o \</div><div class='add'>+	     dmah.o \</div><div class='add'>+	     doorbell.o \</div><div class='add'>+	     fs.o \</div><div class='add'>+	     gsi.o \</div><div class='add'>+	     ib_virt.o \</div><div class='add'>+	     mad.o \</div><div class='add'>+	     main.o \</div><div class='add'>+	     mem.o \</div><div class='add'>+	     mr.o \</div><div class='add'>+	     qp.o \</div><div class='add'>+	     qpc.o \</div><div class='add'>+	     restrack.o \</div><div class='add'>+	     srq.o \</div><div class='add'>+	     srq_cmd.o \</div><div class='add'>+	     umr.o \</div><div class='add'>+	     wr.o</div><div class='ctx'> </div><div class='del'>-mlx5_ib-y :=	main.o cq.o doorbell.o qp.o mem.o srq_cmd.o \</div><div class='del'>-		srq.o mr.o ah.o mad.o gsi.o ib_virt.o cmd.o \</div><div class='del'>-		cong.o</div><div class='ctx'> mlx5_ib-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += odp.o</div><div class='ctx'> mlx5_ib-$(CONFIG_MLX5_ESWITCH) += ib_rep.o</div><div class='del'>-mlx5_ib-$(CONFIG_INFINIBAND_USER_ACCESS) += devx.o</div><div class='del'>-mlx5_ib-$(CONFIG_INFINIBAND_USER_ACCESS) += flow.o</div><div class='add'>+mlx5_ib-$(CONFIG_INFINIBAND_USER_ACCESS) += devx.o \</div><div class='add'>+					    qos.o \</div><div class='add'>+					    std_types.o</div><div class='add'>+mlx5_ib-$(CONFIG_MLX5_MACSEC) += macsec.o</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/ah.c b/drivers/infiniband/hw/mlx5/ah.c<br/>index 80642dd359bc..531a57f9ee7e 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/ah.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/ah.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/ah.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/ah.c</a></div><div class='hunk'>@@ -32,10 +32,30 @@</div><div class='ctx'> </div><div class='ctx'> #include "mlx5_ib.h"</div><div class='ctx'> </div><div class='del'>-static void create_ib_ah(struct mlx5_ib_dev *dev, struct mlx5_ib_ah *ah,</div><div class='del'>-			 struct rdma_ah_attr *ah_attr)</div><div class='add'>+static __be16 mlx5_ah_get_udp_sport(const struct mlx5_ib_dev *dev,</div><div class='add'>+				  const struct rdma_ah_attr *ah_attr)</div><div class='ctx'> {</div><div class='add'>+	enum ib_gid_type gid_type = ah_attr-&gt;grh.sgid_attr-&gt;gid_type;</div><div class='add'>+	__be16 sport;</div><div class='add'>+</div><div class='add'>+	if ((gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) &amp;&amp;</div><div class='add'>+	    (rdma_ah_get_ah_flags(ah_attr) &amp; IB_AH_GRH) &amp;&amp;</div><div class='add'>+	    (ah_attr-&gt;grh.flow_label &amp; IB_GRH_FLOWLABEL_MASK))</div><div class='add'>+		sport = cpu_to_be16(</div><div class='add'>+			rdma_flow_label_to_udp_sport(ah_attr-&gt;grh.flow_label));</div><div class='add'>+	else</div><div class='add'>+		sport = mlx5_get_roce_udp_sport_min(dev,</div><div class='add'>+						    ah_attr-&gt;grh.sgid_attr);</div><div class='add'>+</div><div class='add'>+	return sport;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int create_ib_ah(struct mlx5_ib_dev *dev, struct mlx5_ib_ah *ah,</div><div class='add'>+			 struct rdma_ah_init_attr *init_attr)</div><div class='add'>+{</div><div class='add'>+	struct rdma_ah_attr *ah_attr = init_attr-&gt;ah_attr;</div><div class='ctx'> 	enum ib_gid_type gid_type;</div><div class='add'>+	int rate_val;</div><div class='ctx'> </div><div class='ctx'> 	if (rdma_ah_get_ah_flags(ah_attr) &amp; IB_AH_GRH) {</div><div class='ctx'> 		const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);</div><div class='hunk'>@@ -48,15 +68,21 @@ static void create_ib_ah(struct mlx5_ib_dev *dev, struct mlx5_ib_ah *ah,</div><div class='ctx'> 		ah-&gt;av.tclass = grh-&gt;traffic_class;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ah-&gt;av.stat_rate_sl = (rdma_ah_get_static_rate(ah_attr) &lt;&lt; 4);</div><div class='add'>+	rate_val = mlx5r_ib_rate(dev, rdma_ah_get_static_rate(ah_attr));</div><div class='add'>+	if (rate_val &lt; 0)</div><div class='add'>+		return rate_val;</div><div class='add'>+	ah-&gt;av.stat_rate_sl = rate_val &lt;&lt; 4;</div><div class='ctx'> </div><div class='ctx'> 	if (ah_attr-&gt;type == RDMA_AH_ATTR_TYPE_ROCE) {</div><div class='add'>+		if (init_attr-&gt;xmit_slave)</div><div class='add'>+			ah-&gt;xmit_port =</div><div class='add'>+				mlx5_lag_get_slave_port(dev-&gt;mdev,</div><div class='add'>+							init_attr-&gt;xmit_slave);</div><div class='ctx'> 		gid_type = ah_attr-&gt;grh.sgid_attr-&gt;gid_type;</div><div class='ctx'> </div><div class='ctx'> 		memcpy(ah-&gt;av.rmac, ah_attr-&gt;roce.dmac,</div><div class='ctx'> 		       sizeof(ah_attr-&gt;roce.dmac));</div><div class='del'>-		ah-&gt;av.udp_sport =</div><div class='del'>-			mlx5_get_roce_udp_sport(dev, ah_attr-&gt;grh.sgid_attr);</div><div class='add'>+		ah-&gt;av.udp_sport = mlx5_ah_get_udp_sport(dev, ah_attr);</div><div class='ctx'> 		ah-&gt;av.stat_rate_sl |= (rdma_ah_get_sl(ah_attr) &amp; 0x7) &lt;&lt; 1;</div><div class='ctx'> 		if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)</div><div class='ctx'> #define MLX5_ECN_ENABLED BIT(1)</div><div class='hunk'>@@ -66,12 +92,15 @@ static void create_ib_ah(struct mlx5_ib_dev *dev, struct mlx5_ib_ah *ah,</div><div class='ctx'> 		ah-&gt;av.fl_mlid = rdma_ah_get_path_bits(ah_attr) &amp; 0x7f;</div><div class='ctx'> 		ah-&gt;av.stat_rate_sl |= (rdma_ah_get_sl(ah_attr) &amp; 0xf);</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr,</div><div class='del'>-		      u32 flags, struct ib_udata *udata)</div><div class='add'>+int mlx5_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,</div><div class='add'>+		      struct ib_udata *udata)</div><div class='ctx'> </div><div class='ctx'> {</div><div class='add'>+	struct rdma_ah_attr *ah_attr = init_attr-&gt;ah_attr;</div><div class='ctx'> 	struct mlx5_ib_ah *ah = to_mah(ibah);</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(ibah-&gt;device);</div><div class='ctx'> 	enum rdma_ah_attr_type ah_type = ah_attr-&gt;type;</div><div class='hunk'>@@ -83,8 +112,8 @@ int mlx5_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr,</div><div class='ctx'> 	if (ah_type == RDMA_AH_ATTR_TYPE_ROCE &amp;&amp; udata) {</div><div class='ctx'> 		int err;</div><div class='ctx'> 		struct mlx5_ib_create_ah_resp resp = {};</div><div class='del'>-		u32 min_resp_len = offsetof(typeof(resp), dmac) +</div><div class='del'>-				   sizeof(resp.dmac);</div><div class='add'>+		u32 min_resp_len =</div><div class='add'>+			offsetofend(struct mlx5_ib_create_ah_resp, dmac);</div><div class='ctx'> </div><div class='ctx'> 		if (udata-&gt;outlen &lt; min_resp_len)</div><div class='ctx'> 			return -EINVAL;</div><div class='hunk'>@@ -97,8 +126,7 @@ int mlx5_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr,</div><div class='ctx'> 			return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	create_ib_ah(dev, ah, ah_attr);</div><div class='del'>-	return 0;</div><div class='add'>+	return create_ib_ah(dev, ah, init_attr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)</div><div class='hunk'>@@ -124,8 +152,3 @@ int mlx5_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='del'>-</div><div class='del'>-void mlx5_ib_destroy_ah(struct ib_ah *ah, u32 flags)</div><div class='del'>-{</div><div class='del'>-	return;</div><div class='del'>-}</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/cmd.c b/drivers/infiniband/hw/mlx5/cmd.c<br/>index e3ec79b8f7f5..7c08e3008927 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/cmd.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/cmd.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/cmd.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/cmd.c</a></div><div class='hunk'>@@ -1,373 +1,101 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='ctx'> /*</div><div class='del'>- * Copyright (c) 2017, Mellanox Technologies. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='add'>+ * Copyright (c) 2017-2020, Mellanox Technologies inc. All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include "cmd.h"</div><div class='ctx'> </div><div class='del'>-int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev *dev, u32 *mkey)</div><div class='del'>-{</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {0};</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)]   = {0};</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	MLX5_SET(query_special_contexts_in, in, opcode,</div><div class='del'>-		 MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);</div><div class='del'>-	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='del'>-	if (!err)</div><div class='del'>-		*mkey = MLX5_GET(query_special_contexts_out, out,</div><div class='del'>-				 dump_fill_mkey);</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey)</div><div class='add'>+int mlx5r_cmd_query_special_mkeys(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {};</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)]   = {};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {};</div><div class='add'>+	bool is_terminate, is_dump, is_null;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	MLX5_SET(query_special_contexts_in, in, opcode,</div><div class='del'>-		 MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);</div><div class='del'>-	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='del'>-	if (!err)</div><div class='del'>-		*null_mkey = MLX5_GET(query_special_contexts_out, out,</div><div class='del'>-				      null_mkey);</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,</div><div class='del'>-			       void *out, int out_size)</div><div class='del'>-{</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(query_cong_params_in)] = { };</div><div class='del'>-</div><div class='del'>-	MLX5_SET(query_cong_params_in, in, opcode,</div><div class='del'>-		 MLX5_CMD_OP_QUERY_CONG_PARAMS);</div><div class='del'>-	MLX5_SET(query_cong_params_in, in, cong_protocol, cong_point);</div><div class='del'>-</div><div class='del'>-	return mlx5_cmd_exec(dev, in, sizeof(in), out, out_size);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int mlx5_cmd_modify_cong_params(struct mlx5_core_dev *dev,</div><div class='del'>-				void *in, int in_size)</div><div class='del'>-{</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(modify_cong_params_out)] = { };</div><div class='del'>-</div><div class='del'>-	return mlx5_cmd_exec(dev, in, in_size, out, sizeof(out));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,</div><div class='del'>-			 u64 length, u32 alignment)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_core_dev *dev = dm-&gt;dev;</div><div class='del'>-	u64 num_memic_hw_pages = MLX5_CAP_DEV_MEM(dev, memic_bar_size)</div><div class='del'>-					&gt;&gt; PAGE_SHIFT;</div><div class='del'>-	u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);</div><div class='del'>-	u32 max_alignment = MLX5_CAP_DEV_MEM(dev, log_max_memic_addr_alignment);</div><div class='del'>-	u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(alloc_memic_out)] = {};</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(alloc_memic_in)] = {};</div><div class='del'>-	u32 mlx5_alignment;</div><div class='del'>-	u64 page_idx = 0;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	if (!length || (length &amp; MLX5_MEMIC_ALLOC_SIZE_MASK))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	/* mlx5 device sets alignment as 64*2^driver_value</div><div class='del'>-	 * so normalizing is needed.</div><div class='del'>-	 */</div><div class='del'>-	mlx5_alignment = (alignment &lt; MLX5_MEMIC_BASE_ALIGN) ? 0 :</div><div class='del'>-			 alignment - MLX5_MEMIC_BASE_ALIGN;</div><div class='del'>-	if (mlx5_alignment &gt; max_alignment)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	MLX5_SET(alloc_memic_in, in, opcode, MLX5_CMD_OP_ALLOC_MEMIC);</div><div class='del'>-	MLX5_SET(alloc_memic_in, in, range_size, num_pages * PAGE_SIZE);</div><div class='del'>-	MLX5_SET(alloc_memic_in, in, memic_size, length);</div><div class='del'>-	MLX5_SET(alloc_memic_in, in, log_memic_addr_alignment,</div><div class='del'>-		 mlx5_alignment);</div><div class='del'>-</div><div class='del'>-	while (page_idx &lt; num_memic_hw_pages) {</div><div class='del'>-		spin_lock(&amp;dm-&gt;lock);</div><div class='del'>-		page_idx = bitmap_find_next_zero_area(dm-&gt;memic_alloc_pages,</div><div class='del'>-						      num_memic_hw_pages,</div><div class='del'>-						      page_idx,</div><div class='del'>-						      num_pages, 0);</div><div class='del'>-</div><div class='del'>-		if (page_idx &lt; num_memic_hw_pages)</div><div class='del'>-			bitmap_set(dm-&gt;memic_alloc_pages,</div><div class='del'>-				   page_idx, num_pages);</div><div class='del'>-</div><div class='del'>-		spin_unlock(&amp;dm-&gt;lock);</div><div class='del'>-</div><div class='del'>-		if (page_idx &gt;= num_memic_hw_pages)</div><div class='del'>-			break;</div><div class='del'>-</div><div class='del'>-		MLX5_SET64(alloc_memic_in, in, range_start_addr,</div><div class='del'>-			   hw_start_addr + (page_idx * PAGE_SIZE));</div><div class='del'>-</div><div class='del'>-		ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='del'>-		if (ret) {</div><div class='del'>-			spin_lock(&amp;dm-&gt;lock);</div><div class='del'>-			bitmap_clear(dm-&gt;memic_alloc_pages,</div><div class='del'>-				     page_idx, num_pages);</div><div class='del'>-			spin_unlock(&amp;dm-&gt;lock);</div><div class='del'>-</div><div class='del'>-			if (ret == -EAGAIN) {</div><div class='del'>-				page_idx++;</div><div class='del'>-				continue;</div><div class='del'>-			}</div><div class='del'>-</div><div class='del'>-			return ret;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		*addr = dev-&gt;bar_addr +</div><div class='del'>-			MLX5_GET64(alloc_memic_out, out, memic_start_addr);</div><div class='add'>+	is_terminate = MLX5_CAP_GEN(dev-&gt;mdev, terminate_scatter_list_mkey);</div><div class='add'>+	is_dump = MLX5_CAP_GEN(dev-&gt;mdev, dump_fill_mkey);</div><div class='add'>+	is_null = MLX5_CAP_GEN(dev-&gt;mdev, null_mkey);</div><div class='ctx'> </div><div class='add'>+	dev-&gt;mkeys.terminate_scatter_list_mkey = MLX5_TERMINATE_SCATTER_LIST_LKEY;</div><div class='add'>+	if (!is_terminate &amp;&amp; !is_dump &amp;&amp; !is_null)</div><div class='ctx'> 		return 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return -ENOMEM;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_core_dev *dev = dm-&gt;dev;</div><div class='del'>-	u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);</div><div class='del'>-	u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(dealloc_memic_out)] = {0};</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(dealloc_memic_in)] = {0};</div><div class='del'>-	u64 start_page_idx;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	addr -= dev-&gt;bar_addr;</div><div class='del'>-	start_page_idx = (addr - hw_start_addr) &gt;&gt; PAGE_SHIFT;</div><div class='del'>-</div><div class='del'>-	MLX5_SET(dealloc_memic_in, in, opcode, MLX5_CMD_OP_DEALLOC_MEMIC);</div><div class='del'>-	MLX5_SET64(dealloc_memic_in, in, memic_start_addr, addr);</div><div class='del'>-	MLX5_SET(dealloc_memic_in, in, memic_size, length);</div><div class='del'>-</div><div class='del'>-	err =  mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='del'>-</div><div class='del'>-	if (!err) {</div><div class='del'>-		spin_lock(&amp;dm-&gt;lock);</div><div class='del'>-		bitmap_clear(dm-&gt;memic_alloc_pages,</div><div class='del'>-			     start_page_idx, num_pages);</div><div class='del'>-		spin_unlock(&amp;dm-&gt;lock);</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int mlx5_cmd_alloc_sw_icm(struct mlx5_dm *dm, int type, u64 length,</div><div class='del'>-			  u16 uid, phys_addr_t *addr, u32 *obj_id)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_core_dev *dev = dm-&gt;dev;</div><div class='del'>-	u32 num_blocks = DIV_ROUND_UP(length, MLX5_SW_ICM_BLOCK_SIZE(dev));</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(create_sw_icm_in)] = {};</div><div class='del'>-	unsigned long *block_map;</div><div class='del'>-	u64 icm_start_addr;</div><div class='del'>-	u32 log_icm_size;</div><div class='del'>-	u32 max_blocks;</div><div class='del'>-	u64 block_idx;</div><div class='del'>-	void *sw_icm;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,</div><div class='del'>-		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);</div><div class='del'>-	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_SW_ICM);</div><div class='del'>-	MLX5_SET(general_obj_in_cmd_hdr, in, uid, uid);</div><div class='del'>-</div><div class='del'>-	switch (type) {</div><div class='del'>-	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:</div><div class='del'>-		icm_start_addr = MLX5_CAP64_DEV_MEM(dev,</div><div class='del'>-						steering_sw_icm_start_address);</div><div class='del'>-		log_icm_size = MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size);</div><div class='del'>-		block_map = dm-&gt;steering_sw_icm_alloc_blocks;</div><div class='del'>-		break;</div><div class='del'>-	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:</div><div class='del'>-		icm_start_addr = MLX5_CAP64_DEV_MEM(dev,</div><div class='del'>-					header_modify_sw_icm_start_address);</div><div class='del'>-		log_icm_size = MLX5_CAP_DEV_MEM(dev,</div><div class='del'>-						log_header_modify_sw_icm_size);</div><div class='del'>-		block_map = dm-&gt;header_modify_sw_icm_alloc_blocks;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	max_blocks = BIT(log_icm_size - MLX5_LOG_SW_ICM_BLOCK_SIZE(dev));</div><div class='del'>-	spin_lock(&amp;dm-&gt;lock);</div><div class='del'>-	block_idx = bitmap_find_next_zero_area(block_map,</div><div class='del'>-					       max_blocks,</div><div class='del'>-					       0,</div><div class='del'>-					       num_blocks, 0);</div><div class='del'>-</div><div class='del'>-	if (block_idx &lt; max_blocks)</div><div class='del'>-		bitmap_set(block_map,</div><div class='del'>-			   block_idx, num_blocks);</div><div class='del'>-</div><div class='del'>-	spin_unlock(&amp;dm-&gt;lock);</div><div class='del'>-</div><div class='del'>-	if (block_idx &gt;= max_blocks)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	sw_icm = MLX5_ADDR_OF(create_sw_icm_in, in, sw_icm);</div><div class='del'>-	icm_start_addr += block_idx &lt;&lt; MLX5_LOG_SW_ICM_BLOCK_SIZE(dev);</div><div class='del'>-	MLX5_SET64(sw_icm, sw_icm, sw_icm_start_addr,</div><div class='del'>-		   icm_start_addr);</div><div class='del'>-	MLX5_SET(sw_icm, sw_icm, log_sw_icm_size, ilog2(length));</div><div class='del'>-</div><div class='del'>-	ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='del'>-	if (ret) {</div><div class='del'>-		spin_lock(&amp;dm-&gt;lock);</div><div class='del'>-		bitmap_clear(block_map,</div><div class='del'>-			     block_idx, num_blocks);</div><div class='del'>-		spin_unlock(&amp;dm-&gt;lock);</div><div class='del'>-</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	*addr = icm_start_addr;</div><div class='del'>-	*obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int mlx5_cmd_dealloc_sw_icm(struct mlx5_dm *dm, int type, u64 length,</div><div class='del'>-			    u16 uid, phys_addr_t addr, u32 obj_id)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_core_dev *dev = dm-&gt;dev;</div><div class='del'>-	u32 num_blocks = DIV_ROUND_UP(length, MLX5_SW_ICM_BLOCK_SIZE(dev));</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};</div><div class='del'>-	unsigned long *block_map;</div><div class='del'>-	u64 start_idx;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	switch (type) {</div><div class='del'>-	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:</div><div class='del'>-		start_idx =</div><div class='del'>-			(addr - MLX5_CAP64_DEV_MEM(</div><div class='del'>-					dev, steering_sw_icm_start_address)) &gt;&gt;</div><div class='del'>-			MLX5_LOG_SW_ICM_BLOCK_SIZE(dev);</div><div class='del'>-		block_map = dm-&gt;steering_sw_icm_alloc_blocks;</div><div class='del'>-		break;</div><div class='del'>-	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:</div><div class='del'>-		start_idx =</div><div class='del'>-			(addr -</div><div class='del'>-			 MLX5_CAP64_DEV_MEM(</div><div class='del'>-				 dev, header_modify_sw_icm_start_address)) &gt;&gt;</div><div class='del'>-			MLX5_LOG_SW_ICM_BLOCK_SIZE(dev);</div><div class='del'>-		block_map = dm-&gt;header_modify_sw_icm_alloc_blocks;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,</div><div class='del'>-		 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);</div><div class='del'>-	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_SW_ICM);</div><div class='del'>-	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id);</div><div class='del'>-	MLX5_SET(general_obj_in_cmd_hdr, in, uid, uid);</div><div class='del'>-</div><div class='del'>-	err =  mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='add'>+	MLX5_SET(query_special_contexts_in, in, opcode,</div><div class='add'>+		 MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);</div><div class='add'>+	err = mlx5_cmd_exec_inout(dev-&gt;mdev, query_special_contexts, in, out);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='del'>-	spin_lock(&amp;dm-&gt;lock);</div><div class='del'>-	bitmap_clear(block_map,</div><div class='del'>-		     start_idx, num_blocks);</div><div class='del'>-	spin_unlock(&amp;dm-&gt;lock);</div><div class='add'>+	if (is_dump)</div><div class='add'>+		dev-&gt;mkeys.dump_fill_mkey = MLX5_GET(query_special_contexts_out,</div><div class='add'>+						     out, dump_fill_mkey);</div><div class='add'>+</div><div class='add'>+	if (is_null)</div><div class='add'>+		dev-&gt;mkeys.null_mkey = cpu_to_be32(</div><div class='add'>+			MLX5_GET(query_special_contexts_out, out, null_mkey));</div><div class='add'>+</div><div class='add'>+	if (is_terminate)</div><div class='add'>+		dev-&gt;mkeys.terminate_scatter_list_mkey =</div><div class='add'>+			cpu_to_be32(MLX5_GET(query_special_contexts_out, out,</div><div class='add'>+					     terminate_scatter_list_mkey));</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_cmd_query_ext_ppcnt_counters(struct mlx5_core_dev *dev, void *out)</div><div class='add'>+int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,</div><div class='add'>+			       void *out)</div><div class='ctx'> {</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};</div><div class='del'>-	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(query_cong_params_in)] = {};</div><div class='ctx'> </div><div class='del'>-	MLX5_SET(ppcnt_reg, in, local_port, 1);</div><div class='add'>+	MLX5_SET(query_cong_params_in, in, opcode,</div><div class='add'>+		 MLX5_CMD_OP_QUERY_CONG_PARAMS);</div><div class='add'>+	MLX5_SET(query_cong_params_in, in, cong_protocol, cong_point);</div><div class='ctx'> </div><div class='del'>-	MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);</div><div class='del'>-	return  mlx5_core_access_reg(dev, in, sz, out, sz, MLX5_REG_PPCNT,</div><div class='del'>-				     0, 0);</div><div class='add'>+	return mlx5_cmd_exec_inout(dev, query_cong_params, in, out);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid)</div><div class='ctx'> {</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(destroy_tir_in)]   = {};</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(destroy_tir_out)] = {};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR);</div><div class='ctx'> 	MLX5_SET(destroy_tir_in, in, tirn, tirn);</div><div class='ctx'> 	MLX5_SET(destroy_tir_in, in, uid, uid);</div><div class='del'>-	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='add'>+	mlx5_cmd_exec_in(dev, destroy_tir, in);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void mlx5_cmd_destroy_tis(struct mlx5_core_dev *dev, u32 tisn, u16 uid)</div><div class='ctx'> {</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(destroy_tis_in)]   = {0};</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(destroy_tis_out)] = {0};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {};</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);</div><div class='ctx'> 	MLX5_SET(destroy_tis_in, in, tisn, tisn);</div><div class='ctx'> 	MLX5_SET(destroy_tis_in, in, uid, uid);</div><div class='del'>-	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='add'>+	mlx5_cmd_exec_in(dev, destroy_tis, in);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void mlx5_cmd_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn, u16 uid)</div><div class='add'>+int mlx5_cmd_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn, u16 uid)</div><div class='ctx'> {</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)]   = {};</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)] = {};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {};</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);</div><div class='ctx'> 	MLX5_SET(destroy_rqt_in, in, rqtn, rqtn);</div><div class='ctx'> 	MLX5_SET(destroy_rqt_in, in, uid, uid);</div><div class='del'>-	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='add'>+	return mlx5_cmd_exec_in(dev, destroy_rqt, in);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_cmd_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn,</div><div class='ctx'> 				    u16 uid)</div><div class='ctx'> {</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)]   = {0};</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {0};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {};</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {};</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(alloc_transport_domain_in, in, opcode,</div><div class='ctx'> 		 MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);</div><div class='ctx'> 	MLX5_SET(alloc_transport_domain_in, in, uid, uid);</div><div class='ctx'> </div><div class='del'>-	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='add'>+	err = mlx5_cmd_exec_inout(dev, alloc_transport_domain, in, out);</div><div class='ctx'> 	if (!err)</div><div class='ctx'> 		*tdn = MLX5_GET(alloc_transport_domain_out, out,</div><div class='ctx'> 				transport_domain);</div><div class='hunk'>@@ -378,32 +106,29 @@ int mlx5_cmd_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn,</div><div class='ctx'> void mlx5_cmd_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn,</div><div class='ctx'> 				       u16 uid)</div><div class='ctx'> {</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)]   = {0};</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(dealloc_transport_domain_out)] = {0};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {};</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(dealloc_transport_domain_in, in, opcode,</div><div class='ctx'> 		 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);</div><div class='ctx'> 	MLX5_SET(dealloc_transport_domain_in, in, uid, uid);</div><div class='ctx'> 	MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn);</div><div class='del'>-	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='add'>+	mlx5_cmd_exec_in(dev, dealloc_transport_domain, in);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid)</div><div class='add'>+int mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid)</div><div class='ctx'> {</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(dealloc_pd_out)] = {};</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)]   = {};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)] = {};</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(dealloc_pd_in, in, opcode, MLX5_CMD_OP_DEALLOC_PD);</div><div class='ctx'> 	MLX5_SET(dealloc_pd_in, in, pd, pdn);</div><div class='ctx'> 	MLX5_SET(dealloc_pd_in, in, uid, uid);</div><div class='del'>-	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='add'>+	return mlx5_cmd_exec_in(dev, dealloc_pd, in);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_cmd_attach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid,</div><div class='ctx'> 			u32 qpn, u16 uid)</div><div class='ctx'> {</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(attach_to_mcg_out)] = {};</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(attach_to_mcg_in)]   = {};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(attach_to_mcg_in)] = {};</div><div class='ctx'> 	void *gid;</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(attach_to_mcg_in, in, opcode, MLX5_CMD_OP_ATTACH_TO_MCG);</div><div class='hunk'>@@ -411,14 +136,13 @@ int mlx5_cmd_attach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid,</div><div class='ctx'> 	MLX5_SET(attach_to_mcg_in, in, uid, uid);</div><div class='ctx'> 	gid = MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid);</div><div class='ctx'> 	memcpy(gid, mgid, sizeof(*mgid));</div><div class='del'>-	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='add'>+	return mlx5_cmd_exec_in(dev, attach_to_mcg, in);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_cmd_detach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid,</div><div class='ctx'> 			u32 qpn, u16 uid)</div><div class='ctx'> {</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(detach_from_mcg_out)] = {};</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(detach_from_mcg_in)]   = {};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(detach_from_mcg_in)] = {};</div><div class='ctx'> 	void *gid;</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(detach_from_mcg_in, in, opcode, MLX5_CMD_OP_DETACH_FROM_MCG);</div><div class='hunk'>@@ -426,18 +150,18 @@ int mlx5_cmd_detach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid,</div><div class='ctx'> 	MLX5_SET(detach_from_mcg_in, in, uid, uid);</div><div class='ctx'> 	gid = MLX5_ADDR_OF(detach_from_mcg_in, in, multicast_gid);</div><div class='ctx'> 	memcpy(gid, mgid, sizeof(*mgid));</div><div class='del'>-	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='add'>+	return mlx5_cmd_exec_in(dev, detach_from_mcg, in);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_cmd_xrcd_alloc(struct mlx5_core_dev *dev, u32 *xrcdn, u16 uid)</div><div class='ctx'> {</div><div class='ctx'> 	u32 out[MLX5_ST_SZ_DW(alloc_xrcd_out)] = {};</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(alloc_xrcd_in)]   = {};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(alloc_xrcd_in)] = {};</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(alloc_xrcd_in, in, opcode, MLX5_CMD_OP_ALLOC_XRCD);</div><div class='ctx'> 	MLX5_SET(alloc_xrcd_in, in, uid, uid);</div><div class='del'>-	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='add'>+	err = mlx5_cmd_exec_inout(dev, alloc_xrcd, in, out);</div><div class='ctx'> 	if (!err)</div><div class='ctx'> 		*xrcdn = MLX5_GET(alloc_xrcd_out, out, xrcd);</div><div class='ctx'> 	return err;</div><div class='hunk'>@@ -445,33 +169,15 @@ int mlx5_cmd_xrcd_alloc(struct mlx5_core_dev *dev, u32 *xrcdn, u16 uid)</div><div class='ctx'> </div><div class='ctx'> int mlx5_cmd_xrcd_dealloc(struct mlx5_core_dev *dev, u32 xrcdn, u16 uid)</div><div class='ctx'> {</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(dealloc_xrcd_out)] = {};</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(dealloc_xrcd_in)]   = {};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(dealloc_xrcd_in)] = {};</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(dealloc_xrcd_in, in, opcode, MLX5_CMD_OP_DEALLOC_XRCD);</div><div class='ctx'> 	MLX5_SET(dealloc_xrcd_in, in, xrcd, xrcdn);</div><div class='ctx'> 	MLX5_SET(dealloc_xrcd_in, in, uid, uid);</div><div class='del'>-	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int mlx5_cmd_alloc_q_counter(struct mlx5_core_dev *dev, u16 *counter_id,</div><div class='del'>-			     u16 uid)</div><div class='del'>-{</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)]   = {0};</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {0};</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);</div><div class='del'>-	MLX5_SET(alloc_q_counter_in, in, uid, uid);</div><div class='del'>-</div><div class='del'>-	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='del'>-	if (!err)</div><div class='del'>-		*counter_id = MLX5_GET(alloc_q_counter_out, out,</div><div class='del'>-				       counter_set_id);</div><div class='del'>-	return err;</div><div class='add'>+	return mlx5_cmd_exec_in(dev, dealloc_xrcd, in);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_cmd_mad_ifc(struct mlx5_core_dev *dev, const void *inb, void *outb,</div><div class='add'>+int mlx5_cmd_mad_ifc(struct mlx5_ib_dev *dev, const void *inb, void *outb,</div><div class='ctx'> 		     u16 opmod, u8 port)</div><div class='ctx'> {</div><div class='ctx'> 	int outlen = MLX5_ST_SZ_BYTES(mad_ifc_out);</div><div class='hunk'>@@ -489,12 +195,18 @@ int mlx5_cmd_mad_ifc(struct mlx5_core_dev *dev, const void *inb, void *outb,</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(mad_ifc_in, in, opcode, MLX5_CMD_OP_MAD_IFC);</div><div class='ctx'> 	MLX5_SET(mad_ifc_in, in, op_mod, opmod);</div><div class='del'>-	MLX5_SET(mad_ifc_in, in, port, port);</div><div class='add'>+	if (dev-&gt;ib_dev.type == RDMA_DEVICE_TYPE_SMI) {</div><div class='add'>+		MLX5_SET(mad_ifc_in, in, plane_index, port);</div><div class='add'>+		MLX5_SET(mad_ifc_in, in, port,</div><div class='add'>+			 smi_to_native_portnum(dev, port));</div><div class='add'>+	} else {</div><div class='add'>+		MLX5_SET(mad_ifc_in, in, port, port);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	data = MLX5_ADDR_OF(mad_ifc_in, in, mad);</div><div class='ctx'> 	memcpy(data, inb, MLX5_FLD_SZ_BYTES(mad_ifc_in, mad));</div><div class='ctx'> </div><div class='del'>-	err = mlx5_cmd_exec(dev, in, inlen, out, outlen);</div><div class='add'>+	err = mlx5_cmd_exec_inout(dev-&gt;mdev, mad_ifc, in, out);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='hunk'>@@ -507,3 +219,50 @@ out:</div><div class='ctx'> 	kfree(in);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='add'>+</div><div class='add'>+int mlx5_cmd_uar_alloc(struct mlx5_core_dev *dev, u32 *uarn, u16 uid)</div><div class='add'>+{</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(alloc_uar_out)] = {};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(alloc_uar_in)] = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(alloc_uar_in, in, opcode, MLX5_CMD_OP_ALLOC_UAR);</div><div class='add'>+	MLX5_SET(alloc_uar_in, in, uid, uid);</div><div class='add'>+	err = mlx5_cmd_exec_inout(dev, alloc_uar, in, out);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	*uarn = MLX5_GET(alloc_uar_out, out, uar);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_cmd_uar_dealloc(struct mlx5_core_dev *dev, u32 uarn, u16 uid)</div><div class='add'>+{</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(dealloc_uar_in)] = {};</div><div class='add'>+</div><div class='add'>+	MLX5_SET(dealloc_uar_in, in, opcode, MLX5_CMD_OP_DEALLOC_UAR);</div><div class='add'>+	MLX5_SET(dealloc_uar_in, in, uar, uarn);</div><div class='add'>+	MLX5_SET(dealloc_uar_in, in, uid, uid);</div><div class='add'>+	return mlx5_cmd_exec_in(dev, dealloc_uar, in);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_cmd_query_vuid(struct mlx5_core_dev *dev, bool data_direct,</div><div class='add'>+			char *out_vuid)</div><div class='add'>+{</div><div class='add'>+	u8 out[MLX5_ST_SZ_BYTES(query_vuid_out) +</div><div class='add'>+		MLX5_ST_SZ_BYTES(array1024_auto)] = {};</div><div class='add'>+	u8 in[MLX5_ST_SZ_BYTES(query_vuid_in)] = {};</div><div class='add'>+	char *vuid;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(query_vuid_in, in, opcode, MLX5_CMD_OPCODE_QUERY_VUID);</div><div class='add'>+	MLX5_SET(query_vuid_in, in, vhca_id, MLX5_CAP_GEN(dev, vhca_id));</div><div class='add'>+	MLX5_SET(query_vuid_in, in, data_direct, data_direct);</div><div class='add'>+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	vuid = MLX5_ADDR_OF(query_vuid_out, out, vuid);</div><div class='add'>+	memcpy(out_vuid, vuid, MLX5_ST_SZ_BYTES(array1024_auto));</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/cmd.h b/drivers/infiniband/hw/mlx5/cmd.h<br/>index 0572dcba6eae..e6c88b6ebd0d 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/cmd.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/cmd.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/cmd.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/cmd.h</a></div><div class='hunk'>@@ -37,20 +37,13 @@</div><div class='ctx'> #include &lt;linux/kernel.h&gt;</div><div class='ctx'> #include &lt;linux/mlx5/driver.h&gt;</div><div class='ctx'> </div><div class='del'>-int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev *dev, u32 *mkey);</div><div class='del'>-int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey);</div><div class='add'>+int mlx5r_cmd_query_special_mkeys(struct mlx5_ib_dev *dev);</div><div class='ctx'> int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,</div><div class='del'>-			       void *out, int out_size);</div><div class='del'>-int mlx5_cmd_query_ext_ppcnt_counters(struct mlx5_core_dev *dev, void *out);</div><div class='del'>-int mlx5_cmd_modify_cong_params(struct mlx5_core_dev *mdev,</div><div class='del'>-				void *in, int in_size);</div><div class='del'>-int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,</div><div class='del'>-			 u64 length, u32 alignment);</div><div class='del'>-int mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length);</div><div class='del'>-void mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid);</div><div class='add'>+			       void *out);</div><div class='add'>+int mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid);</div><div class='ctx'> void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid);</div><div class='ctx'> void mlx5_cmd_destroy_tis(struct mlx5_core_dev *dev, u32 tisn, u16 uid);</div><div class='del'>-void mlx5_cmd_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn, u16 uid);</div><div class='add'>+int mlx5_cmd_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn, u16 uid);</div><div class='ctx'> int mlx5_cmd_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn,</div><div class='ctx'> 				    u16 uid);</div><div class='ctx'> void mlx5_cmd_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn,</div><div class='hunk'>@@ -61,12 +54,10 @@ int mlx5_cmd_detach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid,</div><div class='ctx'> 			u32 qpn, u16 uid);</div><div class='ctx'> int mlx5_cmd_xrcd_alloc(struct mlx5_core_dev *dev, u32 *xrcdn, u16 uid);</div><div class='ctx'> int mlx5_cmd_xrcd_dealloc(struct mlx5_core_dev *dev, u32 xrcdn, u16 uid);</div><div class='del'>-int mlx5_cmd_alloc_q_counter(struct mlx5_core_dev *dev, u16 *counter_id,</div><div class='del'>-			     u16 uid);</div><div class='del'>-int mlx5_cmd_mad_ifc(struct mlx5_core_dev *dev, const void *inb, void *outb,</div><div class='add'>+int mlx5_cmd_mad_ifc(struct mlx5_ib_dev *dev, const void *inb, void *outb,</div><div class='ctx'> 		     u16 opmod, u8 port);</div><div class='del'>-int mlx5_cmd_alloc_sw_icm(struct mlx5_dm *dm, int type, u64 length,</div><div class='del'>-			  u16 uid, phys_addr_t *addr, u32 *obj_id);</div><div class='del'>-int mlx5_cmd_dealloc_sw_icm(struct mlx5_dm *dm, int type, u64 length,</div><div class='del'>-			    u16 uid, phys_addr_t addr, u32 obj_id);</div><div class='add'>+int mlx5_cmd_uar_alloc(struct mlx5_core_dev *dev, u32 *uarn, u16 uid);</div><div class='add'>+int mlx5_cmd_uar_dealloc(struct mlx5_core_dev *dev, u32 uarn, u16 uid);</div><div class='add'>+int mlx5_cmd_query_vuid(struct mlx5_core_dev *dev, bool data_direct,</div><div class='add'>+			char *out_vuid);</div><div class='ctx'> #endif /* MLX5_IB_CMD_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/cong.c b/drivers/infiniband/hw/mlx5/cong.c<br/>index 8ba439fabf7f..a78a067e3ce7 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/cong.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/cong.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/cong.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/cong.c</a></div><div class='hunk'>@@ -38,6 +38,7 @@</div><div class='ctx'> enum mlx5_ib_cong_node_type {</div><div class='ctx'> 	MLX5_IB_RROCE_ECN_RP = 1,</div><div class='ctx'> 	MLX5_IB_RROCE_ECN_NP = 2,</div><div class='add'>+	MLX5_IB_RROCE_GENERAL = 3,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static const char * const mlx5_ib_dbg_cc_name[] = {</div><div class='hunk'>@@ -47,6 +48,7 @@ static const char * const mlx5_ib_dbg_cc_name[] = {</div><div class='ctx'> 	"rp_byte_reset",</div><div class='ctx'> 	"rp_threshold",</div><div class='ctx'> 	"rp_ai_rate",</div><div class='add'>+	"rp_max_rate",</div><div class='ctx'> 	"rp_hai_rate",</div><div class='ctx'> 	"rp_min_dec_fac",</div><div class='ctx'> 	"rp_min_rate",</div><div class='hunk'>@@ -56,9 +58,12 @@ static const char * const mlx5_ib_dbg_cc_name[] = {</div><div class='ctx'> 	"rp_rate_reduce_monitor_period",</div><div class='ctx'> 	"rp_initial_alpha_value",</div><div class='ctx'> 	"rp_gd",</div><div class='add'>+	"np_min_time_between_cnps",</div><div class='ctx'> 	"np_cnp_dscp",</div><div class='ctx'> 	"np_cnp_prio_mode",</div><div class='ctx'> 	"np_cnp_prio",</div><div class='add'>+	"rtt_resp_dscp_valid",</div><div class='add'>+	"rtt_resp_dscp",</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> #define MLX5_IB_RP_CLAMP_TGT_RATE_ATTR			BIT(1)</div><div class='hunk'>@@ -66,6 +71,7 @@ static const char * const mlx5_ib_dbg_cc_name[] = {</div><div class='ctx'> #define MLX5_IB_RP_TIME_RESET_ATTR			BIT(3)</div><div class='ctx'> #define MLX5_IB_RP_BYTE_RESET_ATTR			BIT(4)</div><div class='ctx'> #define MLX5_IB_RP_THRESHOLD_ATTR			BIT(5)</div><div class='add'>+#define MLX5_IB_RP_MAX_RATE_ATTR			BIT(6)</div><div class='ctx'> #define MLX5_IB_RP_AI_RATE_ATTR				BIT(7)</div><div class='ctx'> #define MLX5_IB_RP_HAI_RATE_ATTR			BIT(8)</div><div class='ctx'> #define MLX5_IB_RP_MIN_DEC_FAC_ATTR			BIT(9)</div><div class='hunk'>@@ -77,17 +83,22 @@ static const char * const mlx5_ib_dbg_cc_name[] = {</div><div class='ctx'> #define MLX5_IB_RP_INITIAL_ALPHA_VALUE_ATTR		BIT(15)</div><div class='ctx'> #define MLX5_IB_RP_GD_ATTR				BIT(16)</div><div class='ctx'> </div><div class='add'>+#define MLX5_IB_NP_MIN_TIME_BETWEEN_CNPS_ATTR		BIT(2)</div><div class='ctx'> #define MLX5_IB_NP_CNP_DSCP_ATTR			BIT(3)</div><div class='ctx'> #define MLX5_IB_NP_CNP_PRIO_MODE_ATTR			BIT(4)</div><div class='ctx'> </div><div class='add'>+#define MLX5_IB_GENERAL_RTT_RESP_DSCP_ATTR		BIT(0)</div><div class='add'>+</div><div class='ctx'> static enum mlx5_ib_cong_node_type</div><div class='ctx'> mlx5_ib_param_to_node(enum mlx5_ib_dbg_cc_types param_offset)</div><div class='ctx'> {</div><div class='del'>-	if (param_offset &gt;= MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE &amp;&amp;</div><div class='del'>-	    param_offset &lt;= MLX5_IB_DBG_CC_RP_GD)</div><div class='add'>+	if (param_offset &lt;= MLX5_IB_DBG_CC_RP_GD)</div><div class='ctx'> 		return MLX5_IB_RROCE_ECN_RP;</div><div class='del'>-	else</div><div class='add'>+</div><div class='add'>+	if (param_offset &lt;= MLX5_IB_DBG_CC_NP_CNP_PRIO)</div><div class='ctx'> 		return MLX5_IB_RROCE_ECN_NP;</div><div class='add'>+</div><div class='add'>+	return MLX5_IB_RROCE_GENERAL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static u32 mlx5_get_cc_param_val(void *field, int offset)</div><div class='hunk'>@@ -111,6 +122,9 @@ static u32 mlx5_get_cc_param_val(void *field, int offset)</div><div class='ctx'> 	case MLX5_IB_DBG_CC_RP_AI_RATE:</div><div class='ctx'> 		return MLX5_GET(cong_control_r_roce_ecn_rp, field,</div><div class='ctx'> 				rpg_ai_rate);</div><div class='add'>+	case MLX5_IB_DBG_CC_RP_MAX_RATE:</div><div class='add'>+		return MLX5_GET(cong_control_r_roce_ecn_rp, field,</div><div class='add'>+				rpg_max_rate);</div><div class='ctx'> 	case MLX5_IB_DBG_CC_RP_HAI_RATE:</div><div class='ctx'> 		return MLX5_GET(cong_control_r_roce_ecn_rp, field,</div><div class='ctx'> 				rpg_hai_rate);</div><div class='hunk'>@@ -138,6 +152,9 @@ static u32 mlx5_get_cc_param_val(void *field, int offset)</div><div class='ctx'> 	case MLX5_IB_DBG_CC_RP_GD:</div><div class='ctx'> 		return MLX5_GET(cong_control_r_roce_ecn_rp, field,</div><div class='ctx'> 				rpg_gd);</div><div class='add'>+	case MLX5_IB_DBG_CC_NP_MIN_TIME_BETWEEN_CNPS:</div><div class='add'>+		return MLX5_GET(cong_control_r_roce_ecn_np, field,</div><div class='add'>+				min_time_between_cnps);</div><div class='ctx'> 	case MLX5_IB_DBG_CC_NP_CNP_DSCP:</div><div class='ctx'> 		return MLX5_GET(cong_control_r_roce_ecn_np, field,</div><div class='ctx'> 				cnp_dscp);</div><div class='hunk'>@@ -147,6 +164,12 @@ static u32 mlx5_get_cc_param_val(void *field, int offset)</div><div class='ctx'> 	case MLX5_IB_DBG_CC_NP_CNP_PRIO:</div><div class='ctx'> 		return MLX5_GET(cong_control_r_roce_ecn_np, field,</div><div class='ctx'> 				cnp_802p_prio);</div><div class='add'>+	case MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP_VALID:</div><div class='add'>+		return MLX5_GET(cong_control_r_roce_general, field,</div><div class='add'>+				rtt_resp_dscp_valid);</div><div class='add'>+	case MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP:</div><div class='add'>+		return MLX5_GET(cong_control_r_roce_general, field,</div><div class='add'>+				rtt_resp_dscp);</div><div class='ctx'> 	default:</div><div class='ctx'> 		return 0;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -186,6 +209,11 @@ static void mlx5_ib_set_cc_param_mask_val(void *field, int offset,</div><div class='ctx'> 		MLX5_SET(cong_control_r_roce_ecn_rp, field,</div><div class='ctx'> 			 rpg_ai_rate, var);</div><div class='ctx'> 		break;</div><div class='add'>+	case MLX5_IB_DBG_CC_RP_MAX_RATE:</div><div class='add'>+		*attr_mask |= MLX5_IB_RP_MAX_RATE_ATTR;</div><div class='add'>+		MLX5_SET(cong_control_r_roce_ecn_rp, field,</div><div class='add'>+			 rpg_max_rate, var);</div><div class='add'>+		break;</div><div class='ctx'> 	case MLX5_IB_DBG_CC_RP_HAI_RATE:</div><div class='ctx'> 		*attr_mask |= MLX5_IB_RP_HAI_RATE_ATTR;</div><div class='ctx'> 		MLX5_SET(cong_control_r_roce_ecn_rp, field,</div><div class='hunk'>@@ -231,6 +259,11 @@ static void mlx5_ib_set_cc_param_mask_val(void *field, int offset,</div><div class='ctx'> 		MLX5_SET(cong_control_r_roce_ecn_rp, field,</div><div class='ctx'> 			 rpg_gd, var);</div><div class='ctx'> 		break;</div><div class='add'>+	case MLX5_IB_DBG_CC_NP_MIN_TIME_BETWEEN_CNPS:</div><div class='add'>+		*attr_mask |= MLX5_IB_NP_MIN_TIME_BETWEEN_CNPS_ATTR;</div><div class='add'>+		MLX5_SET(cong_control_r_roce_ecn_np, field,</div><div class='add'>+			 min_time_between_cnps, var);</div><div class='add'>+		break;</div><div class='ctx'> 	case MLX5_IB_DBG_CC_NP_CNP_DSCP:</div><div class='ctx'> 		*attr_mask |= MLX5_IB_NP_CNP_DSCP_ATTR;</div><div class='ctx'> 		MLX5_SET(cong_control_r_roce_ecn_np, field, cnp_dscp, var);</div><div class='hunk'>@@ -244,10 +277,19 @@ static void mlx5_ib_set_cc_param_mask_val(void *field, int offset,</div><div class='ctx'> 		MLX5_SET(cong_control_r_roce_ecn_np, field, cnp_prio_mode, 0);</div><div class='ctx'> 		MLX5_SET(cong_control_r_roce_ecn_np, field, cnp_802p_prio, var);</div><div class='ctx'> 		break;</div><div class='add'>+	case MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP_VALID:</div><div class='add'>+		*attr_mask |= MLX5_IB_GENERAL_RTT_RESP_DSCP_ATTR;</div><div class='add'>+		MLX5_SET(cong_control_r_roce_general, field, rtt_resp_dscp_valid, var);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP:</div><div class='add'>+		*attr_mask |= MLX5_IB_GENERAL_RTT_RESP_DSCP_ATTR;</div><div class='add'>+		MLX5_SET(cong_control_r_roce_general, field, rtt_resp_dscp_valid, 1);</div><div class='add'>+		MLX5_SET(cong_control_r_roce_general, field, rtt_resp_dscp, var);</div><div class='add'>+		break;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_ib_get_cc_params(struct mlx5_ib_dev *dev, u8 port_num,</div><div class='add'>+static int mlx5_ib_get_cc_params(struct mlx5_ib_dev *dev, u32 port_num,</div><div class='ctx'> 				 int offset, u32 *var)</div><div class='ctx'> {</div><div class='ctx'> 	int outlen = MLX5_ST_SZ_BYTES(query_cong_params_out);</div><div class='hunk'>@@ -270,7 +312,7 @@ static int mlx5_ib_get_cc_params(struct mlx5_ib_dev *dev, u8 port_num,</div><div class='ctx'> </div><div class='ctx'> 	node = mlx5_ib_param_to_node(offset);</div><div class='ctx'> </div><div class='del'>-	err = mlx5_cmd_query_cong_params(mdev, node, out, outlen);</div><div class='add'>+	err = mlx5_cmd_query_cong_params(mdev, node, out);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto free;</div><div class='ctx'> </div><div class='hunk'>@@ -284,7 +326,7 @@ alloc_err:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_ib_set_cc_params(struct mlx5_ib_dev *dev, u8 port_num,</div><div class='add'>+static int mlx5_ib_set_cc_params(struct mlx5_ib_dev *dev, u32 port_num,</div><div class='ctx'> 				 int offset, u32 var)</div><div class='ctx'> {</div><div class='ctx'> 	int inlen = MLX5_ST_SZ_BYTES(modify_cong_params_in);</div><div class='hunk'>@@ -319,7 +361,7 @@ static int mlx5_ib_set_cc_params(struct mlx5_ib_dev *dev, u8 port_num,</div><div class='ctx'> 	MLX5_SET(field_select_r_roce_rp, field, field_select_r_roce_rp,</div><div class='ctx'> 		 attr_mask);</div><div class='ctx'> </div><div class='del'>-	err = mlx5_cmd_modify_cong_params(mdev, in, inlen);</div><div class='add'>+	err = mlx5_cmd_exec_in(dev-&gt;mdev, modify_cong_params, in);</div><div class='ctx'> 	kvfree(in);</div><div class='ctx'> alloc_err:</div><div class='ctx'> 	mlx5_ib_put_native_port_mdev(dev, port_num + 1);</div><div class='hunk'>@@ -377,7 +419,7 @@ static const struct file_operations dbg_cc_fops = {</div><div class='ctx'> 	.read	= get_param,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num)</div><div class='add'>+void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	if (!mlx5_debugfs_root ||</div><div class='ctx'> 	    !dev-&gt;port[port_num].dbg_cc_params ||</div><div class='hunk'>@@ -389,7 +431,7 @@ void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num)</div><div class='ctx'> 	dev-&gt;port[port_num].dbg_cc_params = NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num)</div><div class='add'>+void mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dbg_cc_params *dbg_cc_params;</div><div class='ctx'> 	struct mlx5_core_dev *mdev;</div><div class='hunk'>@@ -413,10 +455,15 @@ void mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num)</div><div class='ctx'> </div><div class='ctx'> 	dev-&gt;port[port_num].dbg_cc_params = dbg_cc_params;</div><div class='ctx'> </div><div class='del'>-	dbg_cc_params-&gt;root = debugfs_create_dir("cc_params",</div><div class='del'>-						 mdev-&gt;priv.dbg_root);</div><div class='add'>+	dbg_cc_params-&gt;root = debugfs_create_dir("cc_params", mlx5_debugfs_get_dev_root(mdev));</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; MLX5_IB_DBG_CC_MAX; i++) {</div><div class='add'>+		if ((i == MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP_VALID ||</div><div class='add'>+		     i == MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP))</div><div class='add'>+			if (!MLX5_CAP_GEN(mdev, roce) ||</div><div class='add'>+			    !MLX5_CAP_ROCE(mdev, roce_cc_general))</div><div class='add'>+				continue;</div><div class='add'>+</div><div class='ctx'> 		dbg_cc_params-&gt;params[i].offset = i;</div><div class='ctx'> 		dbg_cc_params-&gt;params[i].dev = dev;</div><div class='ctx'> 		dbg_cc_params-&gt;params[i].port_num = port_num;</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/counters.c b/drivers/infiniband/hw/mlx5/counters.c<br/>new file mode 100644<br/>index 000000000000..e042e0719ead<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/counters.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/counters.c</a></div><div class='hunk'>@@ -0,0 +1,1279 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include "mlx5_ib.h"</div><div class='add'>+#include &lt;linux/mlx5/eswitch.h&gt;</div><div class='add'>+#include &lt;linux/mlx5/vport.h&gt;</div><div class='add'>+#include "counters.h"</div><div class='add'>+#include "ib_rep.h"</div><div class='add'>+#include "qp.h"</div><div class='add'>+</div><div class='add'>+struct mlx5_ib_counter {</div><div class='add'>+	const char *name;</div><div class='add'>+	size_t offset;</div><div class='add'>+	u32 type;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mlx5_rdma_counter {</div><div class='add'>+	struct rdma_counter rdma_counter;</div><div class='add'>+</div><div class='add'>+	struct mlx5_fc *fc[MLX5_IB_OPCOUNTER_MAX];</div><div class='add'>+	struct xarray qpn_opfc_xa;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static struct mlx5_rdma_counter *to_mcounter(struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	return container_of(counter, struct mlx5_rdma_counter, rdma_counter);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define INIT_Q_COUNTER(_name)		\</div><div class='add'>+	{ .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}</div><div class='add'>+</div><div class='add'>+#define INIT_VPORT_Q_COUNTER(_name)		\</div><div class='add'>+	{ .name = "vport_" #_name, .offset =	\</div><div class='add'>+		MLX5_BYTE_OFF(query_q_counter_out, _name)}</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counter basic_q_cnts[] = {</div><div class='add'>+	INIT_Q_COUNTER(rx_write_requests),</div><div class='add'>+	INIT_Q_COUNTER(rx_read_requests),</div><div class='add'>+	INIT_Q_COUNTER(rx_atomic_requests),</div><div class='add'>+	INIT_Q_COUNTER(rx_dct_connect),</div><div class='add'>+	INIT_Q_COUNTER(out_of_buffer),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counter out_of_seq_q_cnts[] = {</div><div class='add'>+	INIT_Q_COUNTER(out_of_sequence),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counter retrans_q_cnts[] = {</div><div class='add'>+	INIT_Q_COUNTER(duplicate_request),</div><div class='add'>+	INIT_Q_COUNTER(rnr_nak_retry_err),</div><div class='add'>+	INIT_Q_COUNTER(packet_seq_err),</div><div class='add'>+	INIT_Q_COUNTER(implied_nak_seq_err),</div><div class='add'>+	INIT_Q_COUNTER(local_ack_timeout_err),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counter vport_basic_q_cnts[] = {</div><div class='add'>+	INIT_VPORT_Q_COUNTER(rx_write_requests),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(rx_read_requests),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(rx_atomic_requests),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(rx_dct_connect),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(out_of_buffer),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counter vport_out_of_seq_q_cnts[] = {</div><div class='add'>+	INIT_VPORT_Q_COUNTER(out_of_sequence),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counter vport_retrans_q_cnts[] = {</div><div class='add'>+	INIT_VPORT_Q_COUNTER(duplicate_request),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(rnr_nak_retry_err),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(packet_seq_err),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(implied_nak_seq_err),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(local_ack_timeout_err),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define INIT_CONG_COUNTER(_name)		\</div><div class='add'>+	{ .name = #_name, .offset =	\</div><div class='add'>+		MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counter cong_cnts[] = {</div><div class='add'>+	INIT_CONG_COUNTER(rp_cnp_ignored),</div><div class='add'>+	INIT_CONG_COUNTER(rp_cnp_handled),</div><div class='add'>+	INIT_CONG_COUNTER(np_ecn_marked_roce_packets),</div><div class='add'>+	INIT_CONG_COUNTER(np_cnp_sent),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counter extended_err_cnts[] = {</div><div class='add'>+	INIT_Q_COUNTER(resp_local_length_error),</div><div class='add'>+	INIT_Q_COUNTER(resp_cqe_error),</div><div class='add'>+	INIT_Q_COUNTER(req_cqe_error),</div><div class='add'>+	INIT_Q_COUNTER(req_remote_invalid_request),</div><div class='add'>+	INIT_Q_COUNTER(req_remote_access_errors),</div><div class='add'>+	INIT_Q_COUNTER(resp_remote_access_errors),</div><div class='add'>+	INIT_Q_COUNTER(resp_cqe_flush_error),</div><div class='add'>+	INIT_Q_COUNTER(req_cqe_flush_error),</div><div class='add'>+	INIT_Q_COUNTER(req_transport_retries_exceeded),</div><div class='add'>+	INIT_Q_COUNTER(req_rnr_retries_exceeded),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counter roce_accl_cnts[] = {</div><div class='add'>+	INIT_Q_COUNTER(roce_adp_retrans),</div><div class='add'>+	INIT_Q_COUNTER(roce_adp_retrans_to),</div><div class='add'>+	INIT_Q_COUNTER(roce_slow_restart),</div><div class='add'>+	INIT_Q_COUNTER(roce_slow_restart_cnps),</div><div class='add'>+	INIT_Q_COUNTER(roce_slow_restart_trans),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counter vport_extended_err_cnts[] = {</div><div class='add'>+	INIT_VPORT_Q_COUNTER(resp_local_length_error),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(resp_cqe_error),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(req_cqe_error),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(req_remote_invalid_request),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(req_remote_access_errors),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(resp_remote_access_errors),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(resp_cqe_flush_error),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(req_cqe_flush_error),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(req_transport_retries_exceeded),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(req_rnr_retries_exceeded),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counter vport_roce_accl_cnts[] = {</div><div class='add'>+	INIT_VPORT_Q_COUNTER(roce_adp_retrans),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(roce_adp_retrans_to),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(roce_slow_restart),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(roce_slow_restart_cnps),</div><div class='add'>+	INIT_VPORT_Q_COUNTER(roce_slow_restart_trans),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define INIT_EXT_PPCNT_COUNTER(_name)		\</div><div class='add'>+	{ .name = #_name, .offset =	\</div><div class='add'>+	MLX5_BYTE_OFF(ppcnt_reg, \</div><div class='add'>+		      counter_set.eth_extended_cntrs_grp_data_layout._name##_high)}</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counter ext_ppcnt_cnts[] = {</div><div class='add'>+	INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define INIT_OP_COUNTER(_name, _type)		\</div><div class='add'>+	{ .name = #_name, .type = MLX5_IB_OPCOUNTER_##_type}</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counter basic_op_cnts[] = {</div><div class='add'>+	INIT_OP_COUNTER(cc_rx_ce_pkts, CC_RX_CE_PKTS),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counter rdmarx_cnp_op_cnts[] = {</div><div class='add'>+	INIT_OP_COUNTER(cc_rx_cnp_pkts, CC_RX_CNP_PKTS),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counter rdmatx_cnp_op_cnts[] = {</div><div class='add'>+	INIT_OP_COUNTER(cc_tx_cnp_pkts, CC_TX_CNP_PKTS),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counter packets_op_cnts[] = {</div><div class='add'>+	INIT_OP_COUNTER(rdma_tx_packets, RDMA_TX_PACKETS),</div><div class='add'>+	INIT_OP_COUNTER(rdma_tx_bytes, RDMA_TX_BYTES),</div><div class='add'>+	INIT_OP_COUNTER(rdma_rx_packets, RDMA_RX_PACKETS),</div><div class='add'>+	INIT_OP_COUNTER(rdma_rx_bytes, RDMA_RX_BYTES),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_read_counters(struct ib_counters *counters,</div><div class='add'>+				 struct ib_counters_read_attr *read_attr,</div><div class='add'>+				 struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);</div><div class='add'>+	struct mlx5_read_counters_attr mread_attr = {};</div><div class='add'>+	struct mlx5_ib_flow_counters_desc *desc;</div><div class='add'>+	int ret, i;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;mcounters-&gt;mcntrs_mutex);</div><div class='add'>+	if (mcounters-&gt;cntrs_max_index &gt; read_attr-&gt;ncounters) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto err_bound;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mread_attr.out = kcalloc(mcounters-&gt;counters_num, sizeof(u64),</div><div class='add'>+				 GFP_KERNEL);</div><div class='add'>+	if (!mread_attr.out) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto err_bound;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mread_attr.hw_cntrs_hndl = mcounters-&gt;hw_cntrs_hndl;</div><div class='add'>+	mread_attr.flags = read_attr-&gt;flags;</div><div class='add'>+	ret = mcounters-&gt;read_counters(counters-&gt;device, &amp;mread_attr);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err_read;</div><div class='add'>+</div><div class='add'>+	/* do the pass over the counters data array to assign according to the</div><div class='add'>+	 * descriptions and indexing pairs</div><div class='add'>+	 */</div><div class='add'>+	desc = mcounters-&gt;counters_data;</div><div class='add'>+	for (i = 0; i &lt; mcounters-&gt;ncounters; i++)</div><div class='add'>+		read_attr-&gt;counters_buff[desc[i].index] += mread_attr.out[desc[i].description];</div><div class='add'>+</div><div class='add'>+err_read:</div><div class='add'>+	kfree(mread_attr.out);</div><div class='add'>+err_bound:</div><div class='add'>+	mutex_unlock(&amp;mcounters-&gt;mcntrs_mutex);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_destroy_counters(struct ib_counters *counters)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);</div><div class='add'>+</div><div class='add'>+	mlx5_ib_counters_clear_description(counters);</div><div class='add'>+	if (mcounters-&gt;hw_cntrs_hndl)</div><div class='add'>+		mlx5_fc_destroy(to_mdev(counters-&gt;device)-&gt;mdev,</div><div class='add'>+				mcounters-&gt;hw_cntrs_hndl);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_create_counters(struct ib_counters *counters,</div><div class='add'>+				   struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);</div><div class='add'>+</div><div class='add'>+	mutex_init(&amp;mcounters-&gt;mcntrs_mutex);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool vport_qcounters_supported(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	return MLX5_CAP_GEN(dev-&gt;mdev, q_counter_other_vport) &amp;&amp;</div><div class='add'>+	       MLX5_CAP_GEN(dev-&gt;mdev, q_counter_aggregation);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev,</div><div class='add'>+						   u32 port_num)</div><div class='add'>+{</div><div class='add'>+	if ((is_mdev_switchdev_mode(dev-&gt;mdev) &amp;&amp;</div><div class='add'>+	     !vport_qcounters_supported(dev)) || !port_num)</div><div class='add'>+		return &amp;dev-&gt;port[0].cnts;</div><div class='add'>+</div><div class='add'>+	return is_mdev_switchdev_mode(dev-&gt;mdev) ?</div><div class='add'>+	       &amp;dev-&gt;port[1].cnts : &amp;dev-&gt;port[port_num - 1].cnts;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/**</div><div class='add'>+ * mlx5_ib_get_counters_id - Returns counters id to use for device+port</div><div class='add'>+ * @dev:	Pointer to mlx5 IB device</div><div class='add'>+ * @port_num:	Zero based port number</div><div class='add'>+ *</div><div class='add'>+ * mlx5_ib_get_counters_id() Returns counters set id to use for given</div><div class='add'>+ * device port combination in switchdev and non switchdev mode of the</div><div class='add'>+ * parent device.</div><div class='add'>+ */</div><div class='add'>+u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u32 port_num)</div><div class='add'>+{</div><div class='add'>+	const struct mlx5_ib_counters *cnts = get_counters(dev, port_num + 1);</div><div class='add'>+</div><div class='add'>+	return cnts-&gt;set_id;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct rdma_hw_stats *do_alloc_stats(const struct mlx5_ib_counters *cnts)</div><div class='add'>+{</div><div class='add'>+	struct rdma_hw_stats *stats;</div><div class='add'>+	u32 num_hw_counters;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	num_hw_counters = cnts-&gt;num_q_counters + cnts-&gt;num_cong_counters +</div><div class='add'>+			  cnts-&gt;num_ext_ppcnt_counters;</div><div class='add'>+	stats = rdma_alloc_hw_stats_struct(cnts-&gt;descs,</div><div class='add'>+					   num_hw_counters +</div><div class='add'>+					   cnts-&gt;num_op_counters,</div><div class='add'>+					   RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='add'>+	if (!stats)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; cnts-&gt;num_op_counters; i++)</div><div class='add'>+		set_bit(num_hw_counters + i, stats-&gt;is_disabled);</div><div class='add'>+</div><div class='add'>+	return stats;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct rdma_hw_stats *</div><div class='add'>+mlx5_ib_alloc_hw_device_stats(struct ib_device *ibdev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='add'>+	const struct mlx5_ib_counters *cnts = &amp;dev-&gt;port[0].cnts;</div><div class='add'>+</div><div class='add'>+	return do_alloc_stats(cnts);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct rdma_hw_stats *</div><div class='add'>+mlx5_ib_alloc_hw_port_stats(struct ib_device *ibdev, u32 port_num)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='add'>+	const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);</div><div class='add'>+</div><div class='add'>+	return do_alloc_stats(cnts);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,</div><div class='add'>+				    const struct mlx5_ib_counters *cnts,</div><div class='add'>+				    struct rdma_hw_stats *stats,</div><div class='add'>+				    u16 set_id)</div><div class='add'>+{</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};</div><div class='add'>+	__be32 val;</div><div class='add'>+	int ret, i;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);</div><div class='add'>+	MLX5_SET(query_q_counter_in, in, counter_set_id, set_id);</div><div class='add'>+	ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; cnts-&gt;num_q_counters; i++) {</div><div class='add'>+		val = *(__be32 *)((void *)out + cnts-&gt;offsets[i]);</div><div class='add'>+		stats-&gt;value[i] = (u64)be32_to_cpu(val);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev,</div><div class='add'>+					    const struct mlx5_ib_counters *cnts,</div><div class='add'>+					    struct rdma_hw_stats *stats)</div><div class='add'>+{</div><div class='add'>+	int offset = cnts-&gt;num_q_counters + cnts-&gt;num_cong_counters;</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};</div><div class='add'>+	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);</div><div class='add'>+	int ret, i;</div><div class='add'>+	void *out;</div><div class='add'>+</div><div class='add'>+	out = kvzalloc(sz, GFP_KERNEL);</div><div class='add'>+	if (!out)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(ppcnt_reg, in, local_port, 1);</div><div class='add'>+	MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);</div><div class='add'>+	ret = mlx5_core_access_reg(dev-&gt;mdev, in, sz, out, sz, MLX5_REG_PPCNT,</div><div class='add'>+				   0, 0);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto free;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; cnts-&gt;num_ext_ppcnt_counters; i++)</div><div class='add'>+		stats-&gt;value[i + offset] =</div><div class='add'>+			be64_to_cpup((__be64 *)(out +</div><div class='add'>+				    cnts-&gt;offsets[i + offset]));</div><div class='add'>+free:</div><div class='add'>+	kvfree(out);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_query_q_counters_vport(struct mlx5_ib_dev *dev,</div><div class='add'>+					  u32 port_num,</div><div class='add'>+					  const struct mlx5_ib_counters *cnts,</div><div class='add'>+					  struct rdma_hw_stats *stats)</div><div class='add'>+</div><div class='add'>+{</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};</div><div class='add'>+	struct mlx5_core_dev *mdev;</div><div class='add'>+	__be32 val;</div><div class='add'>+	int ret, i;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;port[port_num].rep ||</div><div class='add'>+	    dev-&gt;port[port_num].rep-&gt;vport == MLX5_VPORT_UPLINK)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	mdev = mlx5_eswitch_get_core_dev(dev-&gt;port[port_num].rep-&gt;esw);</div><div class='add'>+	if (!mdev)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);</div><div class='add'>+	MLX5_SET(query_q_counter_in, in, other_vport, 1);</div><div class='add'>+	MLX5_SET(query_q_counter_in, in, vport_number,</div><div class='add'>+		 dev-&gt;port[port_num].rep-&gt;vport);</div><div class='add'>+	MLX5_SET(query_q_counter_in, in, aggregate, 1);</div><div class='add'>+	ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; cnts-&gt;num_q_counters; i++) {</div><div class='add'>+		val = *(__be32 *)((void *)out + cnts-&gt;offsets[i]);</div><div class='add'>+		stats-&gt;value[i] = (u64)be32_to_cpu(val);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int do_get_hw_stats(struct ib_device *ibdev,</div><div class='add'>+			   struct rdma_hw_stats *stats,</div><div class='add'>+			   u32 port_num, int index)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='add'>+	const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);</div><div class='add'>+	struct mlx5_core_dev *mdev;</div><div class='add'>+	int ret, num_counters;</div><div class='add'>+</div><div class='add'>+	if (!stats)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	num_counters = cnts-&gt;num_q_counters +</div><div class='add'>+		       cnts-&gt;num_cong_counters +</div><div class='add'>+		       cnts-&gt;num_ext_ppcnt_counters;</div><div class='add'>+</div><div class='add'>+	if (is_mdev_switchdev_mode(dev-&gt;mdev) &amp;&amp; dev-&gt;is_rep &amp;&amp; port_num != 0)</div><div class='add'>+		ret = mlx5_ib_query_q_counters_vport(dev, port_num - 1, cnts,</div><div class='add'>+						     stats);</div><div class='add'>+	else</div><div class='add'>+		ret = mlx5_ib_query_q_counters(dev-&gt;mdev, cnts, stats,</div><div class='add'>+					       cnts-&gt;set_id);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	/* We don't expose device counters over Vports */</div><div class='add'>+	if (is_mdev_switchdev_mode(dev-&gt;mdev) &amp;&amp; dev-&gt;is_rep &amp;&amp; port_num != 0)</div><div class='add'>+		goto done;</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_PCAM_FEATURE(dev-&gt;mdev, rx_icrc_encapsulated_counter)) {</div><div class='add'>+		ret =  mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, cc_query_allowed)) {</div><div class='add'>+		if (!port_num)</div><div class='add'>+			port_num = 1;</div><div class='add'>+		mdev = mlx5_ib_get_native_port_mdev(dev, port_num, NULL);</div><div class='add'>+		if (!mdev) {</div><div class='add'>+			/* If port is not affiliated yet, its in down state</div><div class='add'>+			 * which doesn't have any counters yet, so it would be</div><div class='add'>+			 * zero. So no need to read from the HCA.</div><div class='add'>+			 */</div><div class='add'>+			goto done;</div><div class='add'>+		}</div><div class='add'>+		ret = mlx5_lag_query_cong_counters(mdev,</div><div class='add'>+						   stats-&gt;value +</div><div class='add'>+						   cnts-&gt;num_q_counters,</div><div class='add'>+						   cnts-&gt;num_cong_counters,</div><div class='add'>+						   cnts-&gt;offsets +</div><div class='add'>+						   cnts-&gt;num_q_counters);</div><div class='add'>+</div><div class='add'>+		mlx5_ib_put_native_port_mdev(dev, port_num);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+done:</div><div class='add'>+	return num_counters;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool is_rdma_bytes_counter(u32 type)</div><div class='add'>+{</div><div class='add'>+	if (type == MLX5_IB_OPCOUNTER_RDMA_TX_BYTES ||</div><div class='add'>+	    type == MLX5_IB_OPCOUNTER_RDMA_RX_BYTES ||</div><div class='add'>+	    type == MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP ||</div><div class='add'>+	    type == MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP)</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int do_per_qp_get_op_stat(struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(counter-&gt;device);</div><div class='add'>+	const struct mlx5_ib_counters *cnts = get_counters(dev, counter-&gt;port);</div><div class='add'>+	struct mlx5_rdma_counter *mcounter = to_mcounter(counter);</div><div class='add'>+	int i, ret, index, num_hw_counters;</div><div class='add'>+	u64 packets = 0, bytes = 0;</div><div class='add'>+</div><div class='add'>+	for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;</div><div class='add'>+	     i &lt;= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) {</div><div class='add'>+		if (!mcounter-&gt;fc[i])</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		ret = mlx5_fc_query(dev-&gt;mdev, mcounter-&gt;fc[i],</div><div class='add'>+				    &amp;packets, &amp;bytes);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+</div><div class='add'>+		num_hw_counters = cnts-&gt;num_q_counters +</div><div class='add'>+				  cnts-&gt;num_cong_counters +</div><div class='add'>+				  cnts-&gt;num_ext_ppcnt_counters;</div><div class='add'>+</div><div class='add'>+		index = i - MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP +</div><div class='add'>+			num_hw_counters;</div><div class='add'>+</div><div class='add'>+		if (is_rdma_bytes_counter(i))</div><div class='add'>+			counter-&gt;stats-&gt;value[index] = bytes;</div><div class='add'>+		else</div><div class='add'>+			counter-&gt;stats-&gt;value[index] = packets;</div><div class='add'>+</div><div class='add'>+		clear_bit(index, counter-&gt;stats-&gt;is_disabled);</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int do_get_op_stat(struct ib_device *ibdev,</div><div class='add'>+			  struct rdma_hw_stats *stats,</div><div class='add'>+			  u32 port_num, int index)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='add'>+	const struct mlx5_ib_counters *cnts;</div><div class='add'>+	const struct mlx5_ib_op_fc *opfcs;</div><div class='add'>+	u64 packets, bytes;</div><div class='add'>+	u32 type;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	cnts = get_counters(dev, port_num);</div><div class='add'>+</div><div class='add'>+	opfcs = cnts-&gt;opfcs;</div><div class='add'>+	type = *(u32 *)cnts-&gt;descs[index].priv;</div><div class='add'>+	if (type &gt;= MLX5_IB_OPCOUNTER_MAX)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!opfcs[type].fc)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	ret = mlx5_fc_query(dev-&gt;mdev, opfcs[type].fc,</div><div class='add'>+			    &amp;packets, &amp;bytes);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (is_rdma_bytes_counter(type))</div><div class='add'>+		stats-&gt;value[index] = bytes;</div><div class='add'>+	else</div><div class='add'>+		stats-&gt;value[index] = packets;</div><div class='add'>+out:</div><div class='add'>+	return index;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int do_get_op_stats(struct ib_device *ibdev,</div><div class='add'>+			   struct rdma_hw_stats *stats,</div><div class='add'>+			   u32 port_num)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='add'>+	const struct mlx5_ib_counters *cnts;</div><div class='add'>+	int index, ret, num_hw_counters;</div><div class='add'>+</div><div class='add'>+	cnts = get_counters(dev, port_num);</div><div class='add'>+	num_hw_counters = cnts-&gt;num_q_counters + cnts-&gt;num_cong_counters +</div><div class='add'>+			  cnts-&gt;num_ext_ppcnt_counters;</div><div class='add'>+	for (index = num_hw_counters;</div><div class='add'>+	     index &lt; (num_hw_counters + cnts-&gt;num_op_counters); index++) {</div><div class='add'>+		ret = do_get_op_stat(ibdev, stats, port_num, index);</div><div class='add'>+		if (ret != index)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return cnts-&gt;num_op_counters;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,</div><div class='add'>+				struct rdma_hw_stats *stats,</div><div class='add'>+				u32 port_num, int index)</div><div class='add'>+{</div><div class='add'>+	int num_counters, num_hw_counters, num_op_counters;</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='add'>+	const struct mlx5_ib_counters *cnts;</div><div class='add'>+</div><div class='add'>+	cnts = get_counters(dev, port_num);</div><div class='add'>+	num_hw_counters = cnts-&gt;num_q_counters + cnts-&gt;num_cong_counters +</div><div class='add'>+		cnts-&gt;num_ext_ppcnt_counters;</div><div class='add'>+	num_counters = num_hw_counters + cnts-&gt;num_op_counters;</div><div class='add'>+</div><div class='add'>+	if (index &lt; 0 || index &gt; num_counters)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	else if (index &gt; 0 &amp;&amp; index &lt; num_hw_counters)</div><div class='add'>+		return do_get_hw_stats(ibdev, stats, port_num, index);</div><div class='add'>+	else if (index &gt;= num_hw_counters &amp;&amp; index &lt; num_counters)</div><div class='add'>+		return do_get_op_stat(ibdev, stats, port_num, index);</div><div class='add'>+</div><div class='add'>+	num_hw_counters = do_get_hw_stats(ibdev, stats, port_num, index);</div><div class='add'>+	if (num_hw_counters &lt; 0)</div><div class='add'>+		return num_hw_counters;</div><div class='add'>+</div><div class='add'>+	num_op_counters = do_get_op_stats(ibdev, stats, port_num);</div><div class='add'>+	if (num_op_counters &lt; 0)</div><div class='add'>+		return num_op_counters;</div><div class='add'>+</div><div class='add'>+	return num_hw_counters + num_op_counters;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct rdma_hw_stats *</div><div class='add'>+mlx5_ib_counter_alloc_stats(struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(counter-&gt;device);</div><div class='add'>+	const struct mlx5_ib_counters *cnts = get_counters(dev, counter-&gt;port);</div><div class='add'>+</div><div class='add'>+	return do_alloc_stats(cnts);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_counter_update_stats(struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(counter-&gt;device);</div><div class='add'>+	const struct mlx5_ib_counters *cnts = get_counters(dev, counter-&gt;port);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = mlx5_ib_query_q_counters(dev-&gt;mdev, cnts, counter-&gt;stats,</div><div class='add'>+				       counter-&gt;id);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (!counter-&gt;mode.bind_opcnt)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	return do_per_qp_get_op_stat(counter);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_counter_dealloc(struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_rdma_counter *mcounter = to_mcounter(counter);</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(counter-&gt;device);</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};</div><div class='add'>+</div><div class='add'>+	if (!counter-&gt;id)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	WARN_ON(!xa_empty(&amp;mcounter-&gt;qpn_opfc_xa));</div><div class='add'>+	mlx5r_fs_destroy_fcs(dev, mcounter-&gt;fc);</div><div class='add'>+	MLX5_SET(dealloc_q_counter_in, in, opcode,</div><div class='add'>+		 MLX5_CMD_OP_DEALLOC_Q_COUNTER);</div><div class='add'>+	MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter-&gt;id);</div><div class='add'>+	return mlx5_cmd_exec_in(dev-&gt;mdev, dealloc_q_counter, in);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter,</div><div class='add'>+				   struct ib_qp *qp, u32 port)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_rdma_counter *mcounter = to_mcounter(counter);</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(qp-&gt;device);</div><div class='add'>+	bool new = false;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (!counter-&gt;id) {</div><div class='add'>+		u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};</div><div class='add'>+		u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};</div><div class='add'>+</div><div class='add'>+		MLX5_SET(alloc_q_counter_in, in, opcode,</div><div class='add'>+			 MLX5_CMD_OP_ALLOC_Q_COUNTER);</div><div class='add'>+		MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID);</div><div class='add'>+		err = mlx5_cmd_exec_inout(dev-&gt;mdev, alloc_q_counter, in, out);</div><div class='add'>+		if (err)</div><div class='add'>+			return err;</div><div class='add'>+		counter-&gt;id =</div><div class='add'>+			MLX5_GET(alloc_q_counter_out, out, counter_set_id);</div><div class='add'>+		new = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = mlx5_ib_qp_set_counter(qp, counter);</div><div class='add'>+	if (err)</div><div class='add'>+		goto fail_set_counter;</div><div class='add'>+</div><div class='add'>+	if (!counter-&gt;mode.bind_opcnt)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	err = mlx5r_fs_bind_op_fc(qp, mcounter-&gt;fc, &amp;mcounter-&gt;qpn_opfc_xa,</div><div class='add'>+				  port);</div><div class='add'>+	if (err)</div><div class='add'>+		goto fail_bind_op_fc;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+fail_bind_op_fc:</div><div class='add'>+	mlx5_ib_qp_set_counter(qp, NULL);</div><div class='add'>+fail_set_counter:</div><div class='add'>+	if (new) {</div><div class='add'>+		mlx5_ib_counter_dealloc(counter);</div><div class='add'>+		counter-&gt;id = 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp, u32 port)</div><div class='add'>+{</div><div class='add'>+	struct rdma_counter *counter = qp-&gt;counter;</div><div class='add'>+	struct mlx5_rdma_counter *mcounter;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mcounter = to_mcounter(counter);</div><div class='add'>+</div><div class='add'>+	mlx5r_fs_unbind_op_fc(qp, &amp;mcounter-&gt;qpn_opfc_xa);</div><div class='add'>+</div><div class='add'>+	err = mlx5_ib_qp_set_counter(qp, NULL);</div><div class='add'>+	if (err)</div><div class='add'>+		goto fail_set_counter;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+fail_set_counter:</div><div class='add'>+	if (counter-&gt;mode.bind_opcnt)</div><div class='add'>+		mlx5r_fs_bind_op_fc(qp, mcounter-&gt;fc,</div><div class='add'>+				    &amp;mcounter-&gt;qpn_opfc_xa, port);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,</div><div class='add'>+				  struct rdma_stat_desc *descs, size_t *offsets,</div><div class='add'>+				  u32 port_num)</div><div class='add'>+{</div><div class='add'>+	bool is_vport = is_mdev_switchdev_mode(dev-&gt;mdev) &amp;&amp;</div><div class='add'>+			port_num != MLX5_VPORT_PF;</div><div class='add'>+	const struct mlx5_ib_counter *names;</div><div class='add'>+	int j = 0, i, size;</div><div class='add'>+</div><div class='add'>+	names = is_vport ? vport_basic_q_cnts : basic_q_cnts;</div><div class='add'>+	size = is_vport ? ARRAY_SIZE(vport_basic_q_cnts) :</div><div class='add'>+			  ARRAY_SIZE(basic_q_cnts);</div><div class='add'>+	for (i = 0; i &lt; size; i++, j++) {</div><div class='add'>+		descs[j].name = names[i].name;</div><div class='add'>+		offsets[j] = names[i].offset;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	names = is_vport ? vport_out_of_seq_q_cnts : out_of_seq_q_cnts;</div><div class='add'>+	size = is_vport ? ARRAY_SIZE(vport_out_of_seq_q_cnts) :</div><div class='add'>+			  ARRAY_SIZE(out_of_seq_q_cnts);</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, out_of_seq_cnt)) {</div><div class='add'>+		for (i = 0; i &lt; size; i++, j++) {</div><div class='add'>+			descs[j].name = names[i].name;</div><div class='add'>+			offsets[j] = names[i].offset;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	names = is_vport ? vport_retrans_q_cnts : retrans_q_cnts;</div><div class='add'>+	size = is_vport ? ARRAY_SIZE(vport_retrans_q_cnts) :</div><div class='add'>+			  ARRAY_SIZE(retrans_q_cnts);</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, retransmission_q_counters)) {</div><div class='add'>+		for (i = 0; i &lt; size; i++, j++) {</div><div class='add'>+			descs[j].name = names[i].name;</div><div class='add'>+			offsets[j] = names[i].offset;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	names = is_vport ? vport_extended_err_cnts : extended_err_cnts;</div><div class='add'>+	size = is_vport ? ARRAY_SIZE(vport_extended_err_cnts) :</div><div class='add'>+			  ARRAY_SIZE(extended_err_cnts);</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, enhanced_error_q_counters)) {</div><div class='add'>+		for (i = 0; i &lt; size; i++, j++) {</div><div class='add'>+			descs[j].name = names[i].name;</div><div class='add'>+			offsets[j] = names[i].offset;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	names = is_vport ? vport_roce_accl_cnts : roce_accl_cnts;</div><div class='add'>+	size = is_vport ? ARRAY_SIZE(vport_roce_accl_cnts) :</div><div class='add'>+			  ARRAY_SIZE(roce_accl_cnts);</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, roce_accl)) {</div><div class='add'>+		for (i = 0; i &lt; size; i++, j++) {</div><div class='add'>+			descs[j].name = names[i].name;</div><div class='add'>+			offsets[j] = names[i].offset;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (is_vport)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, cc_query_allowed)) {</div><div class='add'>+		for (i = 0; i &lt; ARRAY_SIZE(cong_cnts); i++, j++) {</div><div class='add'>+			descs[j].name = cong_cnts[i].name;</div><div class='add'>+			offsets[j] = cong_cnts[i].offset;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_PCAM_FEATURE(dev-&gt;mdev, rx_icrc_encapsulated_counter)) {</div><div class='add'>+		for (i = 0; i &lt; ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) {</div><div class='add'>+			descs[j].name = ext_ppcnt_cnts[i].name;</div><div class='add'>+			offsets[j] = ext_ppcnt_cnts[i].offset;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ARRAY_SIZE(basic_op_cnts); i++, j++) {</div><div class='add'>+		descs[j].name = basic_op_cnts[i].name;</div><div class='add'>+		descs[j].flags |= IB_STAT_FLAG_OPTIONAL;</div><div class='add'>+		descs[j].priv = &amp;basic_op_cnts[i].type;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_FLOWTABLE(dev-&gt;mdev,</div><div class='add'>+			       ft_field_support_2_nic_receive_rdma.bth_opcode)) {</div><div class='add'>+		for (i = 0; i &lt; ARRAY_SIZE(rdmarx_cnp_op_cnts); i++, j++) {</div><div class='add'>+			descs[j].name = rdmarx_cnp_op_cnts[i].name;</div><div class='add'>+			descs[j].flags |= IB_STAT_FLAG_OPTIONAL;</div><div class='add'>+			descs[j].priv = &amp;rdmarx_cnp_op_cnts[i].type;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_FLOWTABLE(dev-&gt;mdev,</div><div class='add'>+			       ft_field_support_2_nic_transmit_rdma.bth_opcode)) {</div><div class='add'>+		for (i = 0; i &lt; ARRAY_SIZE(rdmatx_cnp_op_cnts); i++, j++) {</div><div class='add'>+			descs[j].name = rdmatx_cnp_op_cnts[i].name;</div><div class='add'>+			descs[j].flags |= IB_STAT_FLAG_OPTIONAL;</div><div class='add'>+			descs[j].priv = &amp;rdmatx_cnp_op_cnts[i].type;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ARRAY_SIZE(packets_op_cnts); i++, j++) {</div><div class='add'>+		descs[j].name = packets_op_cnts[i].name;</div><div class='add'>+		descs[j].flags |= IB_STAT_FLAG_OPTIONAL;</div><div class='add'>+		descs[j].priv = &amp;packets_op_cnts[i].type;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+</div><div class='add'>+static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,</div><div class='add'>+				    struct mlx5_ib_counters *cnts, u32 port_num)</div><div class='add'>+{</div><div class='add'>+	bool is_vport = is_mdev_switchdev_mode(dev-&gt;mdev) &amp;&amp;</div><div class='add'>+			port_num != MLX5_VPORT_PF;</div><div class='add'>+	u32 num_counters, num_op_counters = 0, size;</div><div class='add'>+</div><div class='add'>+	size = is_vport ? ARRAY_SIZE(vport_basic_q_cnts) :</div><div class='add'>+			  ARRAY_SIZE(basic_q_cnts);</div><div class='add'>+	num_counters = size;</div><div class='add'>+</div><div class='add'>+	size = is_vport ? ARRAY_SIZE(vport_out_of_seq_q_cnts) :</div><div class='add'>+			  ARRAY_SIZE(out_of_seq_q_cnts);</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, out_of_seq_cnt))</div><div class='add'>+		num_counters += size;</div><div class='add'>+</div><div class='add'>+	size = is_vport ? ARRAY_SIZE(vport_retrans_q_cnts) :</div><div class='add'>+			  ARRAY_SIZE(retrans_q_cnts);</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, retransmission_q_counters))</div><div class='add'>+		num_counters += size;</div><div class='add'>+</div><div class='add'>+	size = is_vport ? ARRAY_SIZE(vport_extended_err_cnts) :</div><div class='add'>+			  ARRAY_SIZE(extended_err_cnts);</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, enhanced_error_q_counters))</div><div class='add'>+		num_counters += size;</div><div class='add'>+</div><div class='add'>+	size = is_vport ? ARRAY_SIZE(vport_roce_accl_cnts) :</div><div class='add'>+			  ARRAY_SIZE(roce_accl_cnts);</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, roce_accl))</div><div class='add'>+		num_counters += size;</div><div class='add'>+</div><div class='add'>+	cnts-&gt;num_q_counters = num_counters;</div><div class='add'>+</div><div class='add'>+	if (is_vport)</div><div class='add'>+		goto skip_non_qcounters;</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, cc_query_allowed)) {</div><div class='add'>+		cnts-&gt;num_cong_counters = ARRAY_SIZE(cong_cnts);</div><div class='add'>+		num_counters += ARRAY_SIZE(cong_cnts);</div><div class='add'>+	}</div><div class='add'>+	if (MLX5_CAP_PCAM_FEATURE(dev-&gt;mdev, rx_icrc_encapsulated_counter)) {</div><div class='add'>+		cnts-&gt;num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts);</div><div class='add'>+		num_counters += ARRAY_SIZE(ext_ppcnt_cnts);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	num_op_counters = ARRAY_SIZE(basic_op_cnts);</div><div class='add'>+</div><div class='add'>+	num_op_counters += ARRAY_SIZE(packets_op_cnts);</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_FLOWTABLE(dev-&gt;mdev,</div><div class='add'>+			       ft_field_support_2_nic_receive_rdma.bth_opcode))</div><div class='add'>+		num_op_counters += ARRAY_SIZE(rdmarx_cnp_op_cnts);</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_FLOWTABLE(dev-&gt;mdev,</div><div class='add'>+			       ft_field_support_2_nic_transmit_rdma.bth_opcode))</div><div class='add'>+		num_op_counters += ARRAY_SIZE(rdmatx_cnp_op_cnts);</div><div class='add'>+</div><div class='add'>+skip_non_qcounters:</div><div class='add'>+	cnts-&gt;num_op_counters = num_op_counters;</div><div class='add'>+	num_counters += num_op_counters;</div><div class='add'>+	cnts-&gt;descs = kcalloc(num_counters,</div><div class='add'>+			      sizeof(struct rdma_stat_desc), GFP_KERNEL);</div><div class='add'>+	if (!cnts-&gt;descs)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	cnts-&gt;offsets = kcalloc(num_counters,</div><div class='add'>+				sizeof(*cnts-&gt;offsets), GFP_KERNEL);</div><div class='add'>+	if (!cnts-&gt;offsets)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	kfree(cnts-&gt;descs);</div><div class='add'>+	cnts-&gt;descs = NULL;</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * Checks if the given flow counter type should be sharing the same flow counter</div><div class='add'>+ * with another type and if it should, checks if that other type flow counter</div><div class='add'>+ * was already created, if both conditions are met return true and the counter</div><div class='add'>+ * else return false.</div><div class='add'>+ */</div><div class='add'>+bool mlx5r_is_opfc_shared_and_in_use(struct mlx5_ib_op_fc *opfcs, u32 type,</div><div class='add'>+				     struct mlx5_ib_op_fc **opfc)</div><div class='add'>+{</div><div class='add'>+	u32 shared_fc_type;</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:</div><div class='add'>+		shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:</div><div class='add'>+		shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:</div><div class='add'>+		shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:</div><div class='add'>+		shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP:</div><div class='add'>+		shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:</div><div class='add'>+		shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP:</div><div class='add'>+		shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:</div><div class='add'>+		shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*opfc = &amp;opfcs[shared_fc_type];</div><div class='add'>+	if (!(*opfc)-&gt;fc)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};</div><div class='add'>+	int num_cnt_ports = dev-&gt;num_ports;</div><div class='add'>+	struct mlx5_ib_op_fc *in_use_opfc;</div><div class='add'>+	int i, j;</div><div class='add'>+</div><div class='add'>+	if (is_mdev_switchdev_mode(dev-&gt;mdev))</div><div class='add'>+		num_cnt_ports = min(2, num_cnt_ports);</div><div class='add'>+</div><div class='add'>+	MLX5_SET(dealloc_q_counter_in, in, opcode,</div><div class='add'>+		 MLX5_CMD_OP_DEALLOC_Q_COUNTER);</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; num_cnt_ports; i++) {</div><div class='add'>+		if (dev-&gt;port[i].cnts.set_id) {</div><div class='add'>+			MLX5_SET(dealloc_q_counter_in, in, counter_set_id,</div><div class='add'>+				 dev-&gt;port[i].cnts.set_id);</div><div class='add'>+			mlx5_cmd_exec_in(dev-&gt;mdev, dealloc_q_counter, in);</div><div class='add'>+		}</div><div class='add'>+		kfree(dev-&gt;port[i].cnts.descs);</div><div class='add'>+		kfree(dev-&gt;port[i].cnts.offsets);</div><div class='add'>+</div><div class='add'>+		for (j = 0; j &lt; MLX5_IB_OPCOUNTER_MAX; j++) {</div><div class='add'>+			if (!dev-&gt;port[i].cnts.opfcs[j].fc)</div><div class='add'>+				continue;</div><div class='add'>+</div><div class='add'>+			if (mlx5r_is_opfc_shared_and_in_use(</div><div class='add'>+				    dev-&gt;port[i].cnts.opfcs, j, &amp;in_use_opfc))</div><div class='add'>+				goto skip;</div><div class='add'>+</div><div class='add'>+			mlx5_ib_fs_remove_op_fc(dev,</div><div class='add'>+						&amp;dev-&gt;port[i].cnts.opfcs[j], j);</div><div class='add'>+			mlx5_fc_destroy(dev-&gt;mdev,</div><div class='add'>+					dev-&gt;port[i].cnts.opfcs[j].fc);</div><div class='add'>+skip:</div><div class='add'>+			dev-&gt;port[i].cnts.opfcs[j].fc = NULL;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};</div><div class='add'>+	int num_cnt_ports = dev-&gt;num_ports;</div><div class='add'>+	int err = 0;</div><div class='add'>+	int i;</div><div class='add'>+	bool is_shared;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);</div><div class='add'>+	is_shared = MLX5_CAP_GEN(dev-&gt;mdev, log_max_uctx) != 0;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * In switchdev we need to allocate two ports, one that is used for</div><div class='add'>+	 * the device Q_counters and it is essentially the real Q_counters of</div><div class='add'>+	 * this device, while the other is used as a helper for PF to be able to</div><div class='add'>+	 * query all other vports.</div><div class='add'>+	 */</div><div class='add'>+	if (is_mdev_switchdev_mode(dev-&gt;mdev))</div><div class='add'>+		num_cnt_ports = min(2, num_cnt_ports);</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; num_cnt_ports; i++) {</div><div class='add'>+		err = __mlx5_ib_alloc_counters(dev, &amp;dev-&gt;port[i].cnts, i);</div><div class='add'>+		if (err)</div><div class='add'>+			goto err_alloc;</div><div class='add'>+</div><div class='add'>+		mlx5_ib_fill_counters(dev, dev-&gt;port[i].cnts.descs,</div><div class='add'>+				      dev-&gt;port[i].cnts.offsets, i);</div><div class='add'>+</div><div class='add'>+		MLX5_SET(alloc_q_counter_in, in, uid,</div><div class='add'>+			 is_shared ? MLX5_SHARED_RESOURCE_UID : 0);</div><div class='add'>+</div><div class='add'>+		err = mlx5_cmd_exec_inout(dev-&gt;mdev, alloc_q_counter, in, out);</div><div class='add'>+		if (err) {</div><div class='add'>+			mlx5_ib_warn(dev,</div><div class='add'>+				     "couldn't allocate queue counter for port %d, err %d\n",</div><div class='add'>+				     i + 1, err);</div><div class='add'>+			goto err_alloc;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		dev-&gt;port[i].cnts.set_id =</div><div class='add'>+			MLX5_GET(alloc_q_counter_out, out, counter_set_id);</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_alloc:</div><div class='add'>+	mlx5_ib_dealloc_counters(dev);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int read_flow_counters(struct ib_device *ibdev,</div><div class='add'>+			      struct mlx5_read_counters_attr *read_attr)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_fc *fc = read_attr-&gt;hw_cntrs_hndl;</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='add'>+</div><div class='add'>+	return mlx5_fc_query(dev-&gt;mdev, fc,</div><div class='add'>+			     &amp;read_attr-&gt;out[IB_COUNTER_PACKETS],</div><div class='add'>+			     &amp;read_attr-&gt;out[IB_COUNTER_BYTES]);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* flow counters currently expose two counters packets and bytes */</div><div class='add'>+#define FLOW_COUNTERS_NUM 2</div><div class='add'>+static int counters_set_description(</div><div class='add'>+	struct ib_counters *counters, enum mlx5_ib_counters_type counters_type,</div><div class='add'>+	struct mlx5_ib_flow_counters_desc *desc_data, u32 ncounters)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);</div><div class='add'>+	u32 cntrs_max_index = 0;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (counters_type != MLX5_IB_COUNTERS_FLOW)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	/* init the fields for the object */</div><div class='add'>+	mcounters-&gt;type = counters_type;</div><div class='add'>+	mcounters-&gt;read_counters = read_flow_counters;</div><div class='add'>+	mcounters-&gt;counters_num = FLOW_COUNTERS_NUM;</div><div class='add'>+	mcounters-&gt;ncounters = ncounters;</div><div class='add'>+	/* each counter entry have both description and index pair */</div><div class='add'>+	for (i = 0; i &lt; ncounters; i++) {</div><div class='add'>+		if (desc_data[i].description &gt; IB_COUNTER_BYTES)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		if (cntrs_max_index &lt;= desc_data[i].index)</div><div class='add'>+			cntrs_max_index = desc_data[i].index + 1;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;mcounters-&gt;mcntrs_mutex);</div><div class='add'>+	mcounters-&gt;counters_data = desc_data;</div><div class='add'>+	mcounters-&gt;cntrs_max_index = cntrs_max_index;</div><div class='add'>+	mutex_unlock(&amp;mcounters-&gt;mcntrs_mutex);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2))</div><div class='add'>+int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters,</div><div class='add'>+				   struct mlx5_ib_create_flow *ucmd)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters);</div><div class='add'>+	struct mlx5_ib_flow_counters_data *cntrs_data = NULL;</div><div class='add'>+	struct mlx5_ib_flow_counters_desc *desc_data = NULL;</div><div class='add'>+	bool hw_hndl = false;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	if (ucmd &amp;&amp; ucmd-&gt;ncounters_data != 0) {</div><div class='add'>+		cntrs_data = ucmd-&gt;data;</div><div class='add'>+		if (cntrs_data-&gt;ncounters &gt; MAX_COUNTERS_NUM)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		desc_data = kcalloc(cntrs_data-&gt;ncounters,</div><div class='add'>+				    sizeof(*desc_data),</div><div class='add'>+				    GFP_KERNEL);</div><div class='add'>+		if (!desc_data)</div><div class='add'>+			return  -ENOMEM;</div><div class='add'>+</div><div class='add'>+		if (copy_from_user(desc_data,</div><div class='add'>+				   u64_to_user_ptr(cntrs_data-&gt;counters_data),</div><div class='add'>+				   sizeof(*desc_data) * cntrs_data-&gt;ncounters)) {</div><div class='add'>+			ret = -EFAULT;</div><div class='add'>+			goto free;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!mcounters-&gt;hw_cntrs_hndl) {</div><div class='add'>+		mcounters-&gt;hw_cntrs_hndl = mlx5_fc_create(</div><div class='add'>+			to_mdev(ibcounters-&gt;device)-&gt;mdev, false);</div><div class='add'>+		if (IS_ERR(mcounters-&gt;hw_cntrs_hndl)) {</div><div class='add'>+			ret = PTR_ERR(mcounters-&gt;hw_cntrs_hndl);</div><div class='add'>+			goto free;</div><div class='add'>+		}</div><div class='add'>+		hw_hndl = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (desc_data) {</div><div class='add'>+		/* counters already bound to at least one flow */</div><div class='add'>+		if (mcounters-&gt;cntrs_max_index) {</div><div class='add'>+			ret = -EINVAL;</div><div class='add'>+			goto free_hndl;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		ret = counters_set_description(ibcounters,</div><div class='add'>+					       MLX5_IB_COUNTERS_FLOW,</div><div class='add'>+					       desc_data,</div><div class='add'>+					       cntrs_data-&gt;ncounters);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto free_hndl;</div><div class='add'>+</div><div class='add'>+	} else if (!mcounters-&gt;cntrs_max_index) {</div><div class='add'>+		/* counters not bound yet, must have udata passed */</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto free_hndl;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+free_hndl:</div><div class='add'>+	if (hw_hndl) {</div><div class='add'>+		mlx5_fc_destroy(to_mdev(ibcounters-&gt;device)-&gt;mdev,</div><div class='add'>+				mcounters-&gt;hw_cntrs_hndl);</div><div class='add'>+		mcounters-&gt;hw_cntrs_hndl = NULL;</div><div class='add'>+	}</div><div class='add'>+free:</div><div class='add'>+	kfree(desc_data);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_ib_counters_clear_description(struct ib_counters *counters)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mcounters *mcounters;</div><div class='add'>+</div><div class='add'>+	if (!counters || atomic_read(&amp;counters-&gt;usecnt) != 1)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	mcounters = to_mcounters(counters);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;mcounters-&gt;mcntrs_mutex);</div><div class='add'>+	kfree(mcounters-&gt;counters_data);</div><div class='add'>+	mcounters-&gt;counters_data = NULL;</div><div class='add'>+	mcounters-&gt;cntrs_max_index = 0;</div><div class='add'>+	mutex_unlock(&amp;mcounters-&gt;mcntrs_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_modify_stat(struct ib_device *device, u32 port,</div><div class='add'>+			       unsigned int index, bool enable)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='add'>+	struct mlx5_ib_op_fc *opfc, *in_use_opfc;</div><div class='add'>+	struct mlx5_ib_counters *cnts;</div><div class='add'>+	u32 num_hw_counters, type;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	cnts = &amp;dev-&gt;port[port - 1].cnts;</div><div class='add'>+	num_hw_counters = cnts-&gt;num_q_counters + cnts-&gt;num_cong_counters +</div><div class='add'>+		cnts-&gt;num_ext_ppcnt_counters;</div><div class='add'>+	if (index &lt; num_hw_counters ||</div><div class='add'>+	    index &gt;= (num_hw_counters + cnts-&gt;num_op_counters))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!(cnts-&gt;descs[index].flags &amp; IB_STAT_FLAG_OPTIONAL))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	type = *(u32 *)cnts-&gt;descs[index].priv;</div><div class='add'>+	if (type &gt;= MLX5_IB_OPCOUNTER_MAX)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	opfc = &amp;cnts-&gt;opfcs[type];</div><div class='add'>+</div><div class='add'>+	if (enable) {</div><div class='add'>+		if (opfc-&gt;fc)</div><div class='add'>+			return -EEXIST;</div><div class='add'>+</div><div class='add'>+		if (mlx5r_is_opfc_shared_and_in_use(cnts-&gt;opfcs, type,</div><div class='add'>+						    &amp;in_use_opfc)) {</div><div class='add'>+			opfc-&gt;fc = in_use_opfc-&gt;fc;</div><div class='add'>+			opfc-&gt;rule[0] = in_use_opfc-&gt;rule[0];</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		opfc-&gt;fc = mlx5_fc_create(dev-&gt;mdev, false);</div><div class='add'>+		if (IS_ERR(opfc-&gt;fc))</div><div class='add'>+			return PTR_ERR(opfc-&gt;fc);</div><div class='add'>+</div><div class='add'>+		ret = mlx5_ib_fs_add_op_fc(dev, port, opfc, type);</div><div class='add'>+		if (ret) {</div><div class='add'>+			mlx5_fc_destroy(dev-&gt;mdev, opfc-&gt;fc);</div><div class='add'>+			opfc-&gt;fc = NULL;</div><div class='add'>+		}</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!opfc-&gt;fc)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (mlx5r_is_opfc_shared_and_in_use(cnts-&gt;opfcs, type, &amp;in_use_opfc))</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	mlx5_ib_fs_remove_op_fc(dev, opfc, type);</div><div class='add'>+	mlx5_fc_destroy(dev-&gt;mdev, opfc-&gt;fc);</div><div class='add'>+out:</div><div class='add'>+	opfc-&gt;fc = NULL;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_counter_init(struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_rdma_counter *mcounter = to_mcounter(counter);</div><div class='add'>+</div><div class='add'>+	xa_init(&amp;mcounter-&gt;qpn_opfc_xa);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops hw_stats_ops = {</div><div class='add'>+	.alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats,</div><div class='add'>+	.get_hw_stats = mlx5_ib_get_hw_stats,</div><div class='add'>+	.counter_bind_qp = mlx5_ib_counter_bind_qp,</div><div class='add'>+	.counter_unbind_qp = mlx5_ib_counter_unbind_qp,</div><div class='add'>+	.counter_dealloc = mlx5_ib_counter_dealloc,</div><div class='add'>+	.counter_alloc_stats = mlx5_ib_counter_alloc_stats,</div><div class='add'>+	.counter_update_stats = mlx5_ib_counter_update_stats,</div><div class='add'>+	.modify_hw_stat = mlx5_ib_modify_stat,</div><div class='add'>+	.counter_init = mlx5_ib_counter_init,</div><div class='add'>+</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(rdma_counter, mlx5_rdma_counter, rdma_counter),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops hw_switchdev_vport_op = {</div><div class='add'>+	.alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops hw_switchdev_stats_ops = {</div><div class='add'>+	.alloc_hw_device_stats = mlx5_ib_alloc_hw_device_stats,</div><div class='add'>+	.get_hw_stats = mlx5_ib_get_hw_stats,</div><div class='add'>+	.counter_bind_qp = mlx5_ib_counter_bind_qp,</div><div class='add'>+	.counter_unbind_qp = mlx5_ib_counter_unbind_qp,</div><div class='add'>+	.counter_dealloc = mlx5_ib_counter_dealloc,</div><div class='add'>+	.counter_alloc_stats = mlx5_ib_counter_alloc_stats,</div><div class='add'>+	.counter_update_stats = mlx5_ib_counter_update_stats,</div><div class='add'>+	.counter_init = mlx5_ib_counter_init,</div><div class='add'>+</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(rdma_counter, mlx5_rdma_counter, rdma_counter),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops counters_ops = {</div><div class='add'>+	.create_counters = mlx5_ib_create_counters,</div><div class='add'>+	.destroy_counters = mlx5_ib_destroy_counters,</div><div class='add'>+	.read_counters = mlx5_ib_read_counters,</div><div class='add'>+</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+int mlx5_ib_counters_init(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;counters_ops);</div><div class='add'>+</div><div class='add'>+	if (!MLX5_CAP_GEN(dev-&gt;mdev, max_qp_cnt))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (is_mdev_switchdev_mode(dev-&gt;mdev)) {</div><div class='add'>+		ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;hw_switchdev_stats_ops);</div><div class='add'>+		if (vport_qcounters_supported(dev))</div><div class='add'>+			ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;hw_switchdev_vport_op);</div><div class='add'>+	} else</div><div class='add'>+		ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;hw_stats_ops);</div><div class='add'>+	return mlx5_ib_alloc_counters(dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	if (!MLX5_CAP_GEN(dev-&gt;mdev, max_qp_cnt))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	mlx5_ib_dealloc_counters(dev);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/counters.h b/drivers/infiniband/hw/mlx5/counters.h<br/>new file mode 100644<br/>index 000000000000..a04e7dd59455<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/counters.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/counters.h</a></div><div class='hunk'>@@ -0,0 +1,19 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef _MLX5_IB_COUNTERS_H</div><div class='add'>+#define _MLX5_IB_COUNTERS_H</div><div class='add'>+</div><div class='add'>+#include "mlx5_ib.h"</div><div class='add'>+</div><div class='add'>+int mlx5_ib_counters_init(struct mlx5_ib_dev *dev);</div><div class='add'>+void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev);</div><div class='add'>+void mlx5_ib_counters_clear_description(struct ib_counters *counters);</div><div class='add'>+int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters,</div><div class='add'>+				   struct mlx5_ib_create_flow *ucmd);</div><div class='add'>+u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u32 port_num);</div><div class='add'>+bool mlx5r_is_opfc_shared_and_in_use(struct mlx5_ib_op_fc *opfcs, u32 type,</div><div class='add'>+				     struct mlx5_ib_op_fc **opfc);</div><div class='add'>+#endif /* _MLX5_IB_COUNTERS_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c<br/>index 0220736b073e..651d76bca114 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/cq.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/cq.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/cq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/cq.c</a></div><div class='hunk'>@@ -36,8 +36,12 @@</div><div class='ctx'> #include &lt;rdma/ib_cache.h&gt;</div><div class='ctx'> #include "mlx5_ib.h"</div><div class='ctx'> #include "srq.h"</div><div class='add'>+#include "qp.h"</div><div class='ctx'> </div><div class='del'>-static void mlx5_ib_cq_comp(struct mlx5_core_cq *cq)</div><div class='add'>+#define UVERBS_MODULE_NAME mlx5_ib</div><div class='add'>+#include &lt;rdma/uverbs_named_ioctl.h&gt;</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_cq_comp(struct mlx5_core_cq *cq, struct mlx5_eqe *eqe)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_cq *ibcq = &amp;to_mibcq(cq)-&gt;ibcq;</div><div class='ctx'> </div><div class='hunk'>@@ -120,13 +124,13 @@ static void handle_good_req(struct ib_wc *wc, struct mlx5_cqe64 *cqe,</div><div class='ctx'> 	switch (be32_to_cpu(cqe-&gt;sop_drop_qpn) &gt;&gt; 24) {</div><div class='ctx'> 	case MLX5_OPCODE_RDMA_WRITE_IMM:</div><div class='ctx'> 		wc-&gt;wc_flags |= IB_WC_WITH_IMM;</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case MLX5_OPCODE_RDMA_WRITE:</div><div class='ctx'> 		wc-&gt;opcode    = IB_WC_RDMA_WRITE;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_OPCODE_SEND_IMM:</div><div class='ctx'> 		wc-&gt;wc_flags |= IB_WC_WITH_IMM;</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case MLX5_OPCODE_SEND:</div><div class='ctx'> 	case MLX5_OPCODE_SEND_INVAL:</div><div class='ctx'> 		wc-&gt;opcode    = IB_WC_SEND;</div><div class='hunk'>@@ -167,7 +171,7 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,</div><div class='ctx'> {</div><div class='ctx'> 	enum rdma_link_layer ll = rdma_port_get_link_layer(qp-&gt;ibqp.device, 1);</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(qp-&gt;ibqp.device);</div><div class='del'>-	struct mlx5_ib_srq *srq;</div><div class='add'>+	struct mlx5_ib_srq *srq = NULL;</div><div class='ctx'> 	struct mlx5_ib_wq *wq;</div><div class='ctx'> 	u16 wqe_ctr;</div><div class='ctx'> 	u8  roce_packet_type;</div><div class='hunk'>@@ -179,7 +183,8 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,</div><div class='ctx'> </div><div class='ctx'> 		if (qp-&gt;ibqp.xrcd) {</div><div class='ctx'> 			msrq = mlx5_cmd_get_srq(dev, be32_to_cpu(cqe-&gt;srqn));</div><div class='del'>-			srq = to_mibsrq(msrq);</div><div class='add'>+			if (msrq)</div><div class='add'>+				srq = to_mibsrq(msrq);</div><div class='ctx'> 		} else {</div><div class='ctx'> 			srq = to_msrq(qp-&gt;ibqp.srq);</div><div class='ctx'> 		}</div><div class='hunk'>@@ -201,7 +206,7 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,</div><div class='ctx'> 	case MLX5_CQE_RESP_WR_IMM:</div><div class='ctx'> 		wc-&gt;opcode	= IB_WC_RECV_RDMA_WITH_IMM;</div><div class='ctx'> 		wc-&gt;wc_flags	= IB_WC_WITH_IMM;</div><div class='del'>-		wc-&gt;ex.imm_data = cqe-&gt;imm_inval_pkey;</div><div class='add'>+		wc-&gt;ex.imm_data = cqe-&gt;immediate;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CQE_RESP_SEND:</div><div class='ctx'> 		wc-&gt;opcode   = IB_WC_RECV;</div><div class='hunk'>@@ -213,20 +218,20 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,</div><div class='ctx'> 	case MLX5_CQE_RESP_SEND_IMM:</div><div class='ctx'> 		wc-&gt;opcode	= IB_WC_RECV;</div><div class='ctx'> 		wc-&gt;wc_flags	= IB_WC_WITH_IMM;</div><div class='del'>-		wc-&gt;ex.imm_data = cqe-&gt;imm_inval_pkey;</div><div class='add'>+		wc-&gt;ex.imm_data = cqe-&gt;immediate;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CQE_RESP_SEND_INV:</div><div class='ctx'> 		wc-&gt;opcode	= IB_WC_RECV;</div><div class='ctx'> 		wc-&gt;wc_flags	= IB_WC_WITH_INVALIDATE;</div><div class='del'>-		wc-&gt;ex.invalidate_rkey = be32_to_cpu(cqe-&gt;imm_inval_pkey);</div><div class='add'>+		wc-&gt;ex.invalidate_rkey = be32_to_cpu(cqe-&gt;inval_rkey);</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='ctx'> 	wc-&gt;src_qp	   = be32_to_cpu(cqe-&gt;flags_rqpn) &amp; 0xffffff;</div><div class='ctx'> 	wc-&gt;dlid_path_bits = cqe-&gt;ml_path;</div><div class='ctx'> 	g = (be32_to_cpu(cqe-&gt;flags_rqpn) &gt;&gt; 28) &amp; 3;</div><div class='ctx'> 	wc-&gt;wc_flags |= g ? IB_WC_GRH : 0;</div><div class='del'>-	if (unlikely(is_qp1(qp-&gt;ibqp.qp_type))) {</div><div class='del'>-		u16 pkey = be32_to_cpu(cqe-&gt;imm_inval_pkey) &amp; 0xffff;</div><div class='add'>+	if (is_qp1(qp-&gt;type)) {</div><div class='add'>+		u16 pkey = be32_to_cpu(cqe-&gt;pkey) &amp; 0xffff;</div><div class='ctx'> </div><div class='ctx'> 		ib_find_cached_pkey(&amp;dev-&gt;ib_dev, qp-&gt;port, pkey,</div><div class='ctx'> 				    &amp;wc-&gt;pkey_index);</div><div class='hunk'>@@ -253,7 +258,7 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,</div><div class='ctx'> </div><div class='ctx'> 	switch (roce_packet_type) {</div><div class='ctx'> 	case MLX5_CQE_ROCE_L3_HEADER_TYPE_GRH:</div><div class='del'>-		wc-&gt;network_hdr_type = RDMA_NETWORK_IB;</div><div class='add'>+		wc-&gt;network_hdr_type = RDMA_NETWORK_ROCE_V1;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CQE_ROCE_L3_HEADER_TYPE_IPV6:</div><div class='ctx'> 		wc-&gt;network_hdr_type = RDMA_NETWORK_IPV6;</div><div class='hunk'>@@ -265,17 +270,20 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,</div><div class='ctx'> 	wc-&gt;wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void dump_cqe(struct mlx5_ib_dev *dev, struct mlx5_err_cqe *cqe)</div><div class='add'>+static void dump_cqe(struct mlx5_ib_dev *dev, struct mlx5_err_cqe *cqe,</div><div class='add'>+		     struct ib_wc *wc, const char *level)</div><div class='ctx'> {</div><div class='del'>-	mlx5_ib_warn(dev, "dump error cqe\n");</div><div class='del'>-	mlx5_dump_err_cqe(dev-&gt;mdev, cqe);</div><div class='add'>+	mlx5_ib_log(level, dev, "WC error: %d, Message: %s\n", wc-&gt;status,</div><div class='add'>+		    ib_wc_status_msg(wc-&gt;status));</div><div class='add'>+	print_hex_dump(level, "cqe_dump: ", DUMP_PREFIX_OFFSET, 16, 1,</div><div class='add'>+		       cqe, sizeof(*cqe), false);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev,</div><div class='ctx'> 				  struct mlx5_err_cqe *cqe,</div><div class='ctx'> 				  struct ib_wc *wc)</div><div class='ctx'> {</div><div class='del'>-	int dump = 1;</div><div class='add'>+	const char *dump = KERN_WARNING;</div><div class='ctx'> </div><div class='ctx'> 	switch (cqe-&gt;syndrome) {</div><div class='ctx'> 	case MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR:</div><div class='hunk'>@@ -285,10 +293,11 @@ static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev,</div><div class='ctx'> 		wc-&gt;status = IB_WC_LOC_QP_OP_ERR;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CQE_SYNDROME_LOCAL_PROT_ERR:</div><div class='add'>+		dump = KERN_DEBUG;</div><div class='ctx'> 		wc-&gt;status = IB_WC_LOC_PROT_ERR;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CQE_SYNDROME_WR_FLUSH_ERR:</div><div class='del'>-		dump = 0;</div><div class='add'>+		dump = NULL;</div><div class='ctx'> 		wc-&gt;status = IB_WC_WR_FLUSH_ERR;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CQE_SYNDROME_MW_BIND_ERR:</div><div class='hunk'>@@ -304,18 +313,20 @@ static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev,</div><div class='ctx'> 		wc-&gt;status = IB_WC_REM_INV_REQ_ERR;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR:</div><div class='add'>+		dump = KERN_DEBUG;</div><div class='ctx'> 		wc-&gt;status = IB_WC_REM_ACCESS_ERR;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CQE_SYNDROME_REMOTE_OP_ERR:</div><div class='add'>+		dump = KERN_DEBUG;</div><div class='ctx'> 		wc-&gt;status = IB_WC_REM_OP_ERR;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:</div><div class='add'>+		dump = NULL;</div><div class='ctx'> 		wc-&gt;status = IB_WC_RETRY_EXC_ERR;</div><div class='del'>-		dump = 0;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR:</div><div class='add'>+		dump = NULL;</div><div class='ctx'> 		wc-&gt;status = IB_WC_RNR_RETRY_EXC_ERR;</div><div class='del'>-		dump = 0;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR:</div><div class='ctx'> 		wc-&gt;status = IB_WC_REM_ABORT_ERR;</div><div class='hunk'>@@ -327,7 +338,23 @@ static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev,</div><div class='ctx'> </div><div class='ctx'> 	wc-&gt;vendor_err = cqe-&gt;vendor_err_synd;</div><div class='ctx'> 	if (dump)</div><div class='del'>-		dump_cqe(dev, cqe);</div><div class='add'>+		dump_cqe(dev, cqe, wc, dump);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void handle_atomics(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64,</div><div class='add'>+			   u16 tail, u16 head)</div><div class='add'>+{</div><div class='add'>+	u16 idx;</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		idx = tail &amp; (qp-&gt;sq.wqe_cnt - 1);</div><div class='add'>+		if (idx == head)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		tail = qp-&gt;sq.w_list[idx].next;</div><div class='add'>+	} while (1);</div><div class='add'>+	tail = qp-&gt;sq.w_list[idx].next;</div><div class='add'>+	qp-&gt;sq.last_poll = tail;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void free_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf)</div><div class='hunk'>@@ -368,7 +395,7 @@ static void get_sig_err_item(struct mlx5_sig_err_cqe *cqe,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void sw_comp(struct mlx5_ib_qp *qp, int num_entries, struct ib_wc *wc,</div><div class='del'>-		    int *npolled, int is_send)</div><div class='add'>+		    int *npolled, bool is_send)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_wq *wq;</div><div class='ctx'> 	unsigned int cur;</div><div class='hunk'>@@ -383,10 +410,16 @@ static void sw_comp(struct mlx5_ib_qp *qp, int num_entries, struct ib_wc *wc,</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0;  i &lt; cur &amp;&amp; np &lt; num_entries; i++) {</div><div class='del'>-		wc-&gt;wr_id = wq-&gt;wrid[wq-&gt;tail &amp; (wq-&gt;wqe_cnt - 1)];</div><div class='add'>+		unsigned int idx;</div><div class='add'>+</div><div class='add'>+		idx = (is_send) ? wq-&gt;last_poll : wq-&gt;tail;</div><div class='add'>+		idx &amp;= (wq-&gt;wqe_cnt - 1);</div><div class='add'>+		wc-&gt;wr_id = wq-&gt;wrid[idx];</div><div class='ctx'> 		wc-&gt;status = IB_WC_WR_FLUSH_ERR;</div><div class='ctx'> 		wc-&gt;vendor_err = MLX5_CQE_SYNDROME_WR_FLUSH_ERR;</div><div class='ctx'> 		wq-&gt;tail++;</div><div class='add'>+		if (is_send)</div><div class='add'>+			wq-&gt;last_poll = wq-&gt;w_list[idx].next;</div><div class='ctx'> 		np++;</div><div class='ctx'> 		wc-&gt;qp = &amp;qp-&gt;ibqp;</div><div class='ctx'> 		wc++;</div><div class='hunk'>@@ -423,9 +456,6 @@ static int mlx5_poll_one(struct mlx5_ib_cq *cq,</div><div class='ctx'> 	struct mlx5_cqe64 *cqe64;</div><div class='ctx'> 	struct mlx5_core_qp *mqp;</div><div class='ctx'> 	struct mlx5_ib_wq *wq;</div><div class='del'>-	struct mlx5_sig_err_cqe *sig_err_cqe;</div><div class='del'>-	struct mlx5_core_mkey *mmkey;</div><div class='del'>-	struct mlx5_ib_mr *mr;</div><div class='ctx'> 	uint8_t opcode;</div><div class='ctx'> 	uint32_t qpn;</div><div class='ctx'> 	u16 wqe_ctr;</div><div class='hunk'>@@ -460,12 +490,12 @@ repoll:</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	qpn = ntohl(cqe64-&gt;sop_drop_qpn) &amp; 0xffffff;</div><div class='del'>-	if (!*cur_qp || (qpn != (*cur_qp)-&gt;ibqp.qp_num)) {</div><div class='add'>+	if (!*cur_qp || (qpn != (*cur_qp)-&gt;trans_qp.base.mqp.qpn)) {</div><div class='ctx'> 		/* We do not have to take the QP table lock here,</div><div class='ctx'> 		 * because CQs will be locked while QPs are removed</div><div class='ctx'> 		 * from the table.</div><div class='ctx'> 		 */</div><div class='del'>-		mqp = __mlx5_qp_lookup(dev-&gt;mdev, qpn);</div><div class='add'>+		mqp = radix_tree_lookup(&amp;dev-&gt;qp_table.tree, qpn);</div><div class='ctx'> 		*cur_qp = to_mibqp(mqp);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -476,6 +506,7 @@ repoll:</div><div class='ctx'> 		wqe_ctr = be16_to_cpu(cqe64-&gt;wqe_counter);</div><div class='ctx'> 		idx = wqe_ctr &amp; (wq-&gt;wqe_cnt - 1);</div><div class='ctx'> 		handle_good_req(wc, cqe64, wq, idx);</div><div class='add'>+		handle_atomics(*cur_qp, cqe64, wq-&gt;last_poll, idx);</div><div class='ctx'> 		wc-&gt;wr_id = wq-&gt;wrid[idx];</div><div class='ctx'> 		wq-&gt;tail = wq-&gt;wqe_head[idx] + 1;</div><div class='ctx'> 		wc-&gt;status = IB_WC_SUCCESS;</div><div class='hunk'>@@ -498,6 +529,10 @@ repoll:</div><div class='ctx'> 			    "Requestor" : "Responder", cq-&gt;mcq.cqn);</div><div class='ctx'> 		mlx5_ib_dbg(dev, "syndrome 0x%x, vendor syndrome 0x%x\n",</div><div class='ctx'> 			    err_cqe-&gt;syndrome, err_cqe-&gt;vendor_err_synd);</div><div class='add'>+		if (wc-&gt;status != IB_WC_WR_FLUSH_ERR &amp;&amp;</div><div class='add'>+		    (*cur_qp)-&gt;type == MLX5_IB_QPT_REG_UMR)</div><div class='add'>+			dev-&gt;umrc.state = MLX5_UMR_STATE_RECOVER;</div><div class='add'>+</div><div class='ctx'> 		if (opcode == MLX5_CQE_REQ_ERR) {</div><div class='ctx'> 			wq = &amp;(*cur_qp)-&gt;sq;</div><div class='ctx'> 			wqe_ctr = be16_to_cpu(cqe64-&gt;wqe_counter);</div><div class='hunk'>@@ -519,27 +554,29 @@ repoll:</div><div class='ctx'> 			}</div><div class='ctx'> 		}</div><div class='ctx'> 		break;</div><div class='del'>-	case MLX5_CQE_SIG_ERR:</div><div class='del'>-		sig_err_cqe = (struct mlx5_sig_err_cqe *)cqe64;</div><div class='add'>+	case MLX5_CQE_SIG_ERR: {</div><div class='add'>+		struct mlx5_sig_err_cqe *sig_err_cqe =</div><div class='add'>+			(struct mlx5_sig_err_cqe *)cqe64;</div><div class='add'>+		struct mlx5_core_sig_ctx *sig;</div><div class='ctx'> </div><div class='del'>-		xa_lock(&amp;dev-&gt;mdev-&gt;priv.mkey_table);</div><div class='del'>-		mmkey = xa_load(&amp;dev-&gt;mdev-&gt;priv.mkey_table,</div><div class='add'>+		xa_lock(&amp;dev-&gt;sig_mrs);</div><div class='add'>+		sig = xa_load(&amp;dev-&gt;sig_mrs,</div><div class='ctx'> 				mlx5_base_mkey(be32_to_cpu(sig_err_cqe-&gt;mkey)));</div><div class='del'>-		mr = to_mibmr(mmkey);</div><div class='del'>-		get_sig_err_item(sig_err_cqe, &amp;mr-&gt;sig-&gt;err_item);</div><div class='del'>-		mr-&gt;sig-&gt;sig_err_exists = true;</div><div class='del'>-		mr-&gt;sig-&gt;sigerr_count++;</div><div class='add'>+		get_sig_err_item(sig_err_cqe, &amp;sig-&gt;err_item);</div><div class='add'>+		sig-&gt;sig_err_exists = true;</div><div class='add'>+		sig-&gt;sigerr_count++;</div><div class='ctx'> </div><div class='ctx'> 		mlx5_ib_warn(dev, "CQN: 0x%x Got SIGERR on key: 0x%x err_type %x err_offset %llx expected %x actual %x\n",</div><div class='del'>-			     cq-&gt;mcq.cqn, mr-&gt;sig-&gt;err_item.key,</div><div class='del'>-			     mr-&gt;sig-&gt;err_item.err_type,</div><div class='del'>-			     mr-&gt;sig-&gt;err_item.sig_err_offset,</div><div class='del'>-			     mr-&gt;sig-&gt;err_item.expected,</div><div class='del'>-			     mr-&gt;sig-&gt;err_item.actual);</div><div class='add'>+			     cq-&gt;mcq.cqn, sig-&gt;err_item.key,</div><div class='add'>+			     sig-&gt;err_item.err_type,</div><div class='add'>+			     sig-&gt;err_item.sig_err_offset,</div><div class='add'>+			     sig-&gt;err_item.expected,</div><div class='add'>+			     sig-&gt;err_item.actual);</div><div class='ctx'> </div><div class='del'>-		xa_unlock(&amp;dev-&gt;mdev-&gt;priv.mkey_table);</div><div class='add'>+		xa_unlock(&amp;dev-&gt;sig_mrs);</div><div class='ctx'> 		goto repoll;</div><div class='ctx'> 	}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -611,7 +648,7 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_core_dev *mdev = to_mdev(ibcq-&gt;device)-&gt;mdev;</div><div class='ctx'> 	struct mlx5_ib_cq *cq = to_mcq(ibcq);</div><div class='del'>-	void __iomem *uar_page = mdev-&gt;priv.uar-&gt;map;</div><div class='add'>+	void __iomem *uar_page = mdev-&gt;priv.bfreg.up-&gt;map;</div><div class='ctx'> 	unsigned long irq_flags;</div><div class='ctx'> 	int ret = 0;</div><div class='ctx'> </div><div class='hunk'>@@ -680,50 +717,64 @@ static int mini_cqe_res_format_to_hw(struct mlx5_ib_dev *dev, u8 format)</div><div class='ctx'> </div><div class='ctx'> static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,</div><div class='ctx'> 			  struct mlx5_ib_cq *cq, int entries, u32 **cqb,</div><div class='del'>-			  int *cqe_size, int *index, int *inlen)</div><div class='add'>+			  int *cqe_size, int *index, int *inlen,</div><div class='add'>+			  struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_create_cq ucmd = {};</div><div class='add'>+	unsigned long page_size;</div><div class='add'>+	unsigned int page_offset_quantized;</div><div class='ctx'> 	size_t ucmdlen;</div><div class='del'>-	int page_shift;</div><div class='ctx'> 	__be64 *pas;</div><div class='del'>-	int npages;</div><div class='ctx'> 	int ncont;</div><div class='ctx'> 	void *cqc;</div><div class='ctx'> 	int err;</div><div class='ctx'> 	struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(</div><div class='ctx'> 		udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='ctx'> </div><div class='del'>-	ucmdlen = udata-&gt;inlen &lt; sizeof(ucmd) ?</div><div class='del'>-		  (sizeof(ucmd) - sizeof(ucmd.flags)) : sizeof(ucmd);</div><div class='add'>+	ucmdlen = min(udata-&gt;inlen, sizeof(ucmd));</div><div class='add'>+	if (ucmdlen &lt; offsetof(struct mlx5_ib_create_cq, flags))</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	if (ib_copy_from_udata(&amp;ucmd, udata, ucmdlen))</div><div class='ctx'> 		return -EFAULT;</div><div class='ctx'> </div><div class='del'>-	if (ucmdlen == sizeof(ucmd) &amp;&amp;</div><div class='del'>-	    (ucmd.flags &amp; ~(MLX5_IB_CREATE_CQ_FLAGS_CQE_128B_PAD)))</div><div class='add'>+	if ((ucmd.flags &amp; ~(MLX5_IB_CREATE_CQ_FLAGS_CQE_128B_PAD |</div><div class='add'>+			    MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX |</div><div class='add'>+			    MLX5_IB_CREATE_CQ_FLAGS_REAL_TIME_TS)))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (ucmd.cqe_size != 64 &amp;&amp; ucmd.cqe_size != 128)</div><div class='add'>+	if ((ucmd.cqe_size != 64 &amp;&amp; ucmd.cqe_size != 128) ||</div><div class='add'>+	    ucmd.reserved0 || ucmd.reserved1)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	*cqe_size = ucmd.cqe_size;</div><div class='ctx'> </div><div class='ctx'> 	cq-&gt;buf.umem =</div><div class='del'>-		ib_umem_get(udata, ucmd.buf_addr, entries * ucmd.cqe_size,</div><div class='del'>-			    IB_ACCESS_LOCAL_WRITE, 1);</div><div class='add'>+		ib_umem_get(&amp;dev-&gt;ib_dev, ucmd.buf_addr,</div><div class='add'>+			    entries * ucmd.cqe_size, IB_ACCESS_LOCAL_WRITE);</div><div class='ctx'> 	if (IS_ERR(cq-&gt;buf.umem)) {</div><div class='ctx'> 		err = PTR_ERR(cq-&gt;buf.umem);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	err = mlx5_ib_db_map_user(context, udata, ucmd.db_addr, &amp;cq-&gt;db);</div><div class='add'>+	page_size = mlx5_umem_find_best_cq_quantized_pgoff(</div><div class='add'>+		cq-&gt;buf.umem, cqc, log_page_size, MLX5_ADAPTER_PAGE_SHIFT,</div><div class='add'>+		page_offset, 64, &amp;page_offset_quantized);</div><div class='add'>+	if (!page_size) {</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+		goto err_umem;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = mlx5_ib_db_map_user(context, ucmd.db_addr, &amp;cq-&gt;db);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_umem;</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_cont_pages(cq-&gt;buf.umem, ucmd.buf_addr, 0, &amp;npages, &amp;page_shift,</div><div class='del'>-			   &amp;ncont, NULL);</div><div class='del'>-	mlx5_ib_dbg(dev, "addr 0x%llx, size %u, npages %d, page_shift %d, ncont %d\n",</div><div class='del'>-		    ucmd.buf_addr, entries * ucmd.cqe_size, npages, page_shift, ncont);</div><div class='add'>+	ncont = ib_umem_num_dma_blocks(cq-&gt;buf.umem, page_size);</div><div class='add'>+	mlx5_ib_dbg(</div><div class='add'>+		dev,</div><div class='add'>+		"addr 0x%llx, size %u, npages %zu, page_size %lu, ncont %d\n",</div><div class='add'>+		ucmd.buf_addr, entries * ucmd.cqe_size,</div><div class='add'>+		ib_umem_num_pages(cq-&gt;buf.umem), page_size, ncont);</div><div class='ctx'> </div><div class='ctx'> 	*inlen = MLX5_ST_SZ_BYTES(create_cq_in) +</div><div class='ctx'> 		 MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * ncont;</div><div class='hunk'>@@ -734,13 +785,25 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, *cqb, pas);</div><div class='del'>-	mlx5_ib_populate_pas(dev, cq-&gt;buf.umem, page_shift, pas, 0);</div><div class='add'>+	mlx5_ib_populate_pas(cq-&gt;buf.umem, page_size, pas, 0);</div><div class='ctx'> </div><div class='ctx'> 	cqc = MLX5_ADDR_OF(create_cq_in, *cqb, cq_context);</div><div class='ctx'> 	MLX5_SET(cqc, cqc, log_page_size,</div><div class='del'>-		 page_shift - MLX5_ADAPTER_PAGE_SHIFT);</div><div class='add'>+		 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);</div><div class='add'>+	MLX5_SET(cqc, cqc, page_offset, page_offset_quantized);</div><div class='ctx'> </div><div class='del'>-	*index = context-&gt;bfregi.sys_pages[0];</div><div class='add'>+	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_CREATE_CQ_UAR_INDEX)) {</div><div class='add'>+		err = uverbs_copy_from(index, attrs, MLX5_IB_ATTR_CREATE_CQ_UAR_INDEX);</div><div class='add'>+		if (err)</div><div class='add'>+			goto err_cqb;</div><div class='add'>+	} else if (ucmd.flags &amp; MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX) {</div><div class='add'>+		*index = ucmd.uar_page_index;</div><div class='add'>+	} else if (context-&gt;bfregi.lib_uar_dyn) {</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+		goto err_cqb;</div><div class='add'>+	} else {</div><div class='add'>+		*index = context-&gt;bfregi.sys_pages[0];</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	if (ucmd.cqe_comp_en == 1) {</div><div class='ctx'> 		int mini_cqe_format;</div><div class='hunk'>@@ -782,6 +845,9 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,</div><div class='ctx'> 		cq-&gt;private_flags |= MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (ucmd.flags &amp; MLX5_IB_CREATE_CQ_FLAGS_REAL_TIME_TS)</div><div class='add'>+		cq-&gt;private_flags |= MLX5_IB_CQ_PR_FLAGS_REAL_TIME_TS;</div><div class='add'>+</div><div class='ctx'> 	MLX5_SET(create_cq_in, *cqb, uid, context-&gt;devx_uid);</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='hunk'>@@ -805,15 +871,14 @@ static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_udata *udata)</div><div class='ctx'> 	ib_umem_release(cq-&gt;buf.umem);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void init_cq_frag_buf(struct mlx5_ib_cq *cq,</div><div class='del'>-			     struct mlx5_ib_cq_buf *buf)</div><div class='add'>+static void init_cq_frag_buf(struct mlx5_ib_cq_buf *buf)</div><div class='ctx'> {</div><div class='ctx'> 	int i;</div><div class='ctx'> 	void *cqe;</div><div class='ctx'> 	struct mlx5_cqe64 *cqe64;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; buf-&gt;nent; i++) {</div><div class='del'>-		cqe = get_cqe(cq, i);</div><div class='add'>+		cqe = mlx5_frag_buf_get_wqe(&amp;buf-&gt;fbc, i);</div><div class='ctx'> 		cqe64 = buf-&gt;cqe_size == 64 ? cqe : cqe + 64;</div><div class='ctx'> 		cqe64-&gt;op_own = MLX5_CQE_INVALID &lt;&lt; 4;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -839,7 +904,7 @@ static int create_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_db;</div><div class='ctx'> </div><div class='del'>-	init_cq_frag_buf(cq, &amp;cq-&gt;buf);</div><div class='add'>+	init_cq_frag_buf(&amp;cq-&gt;buf);</div><div class='ctx'> </div><div class='ctx'> 	*inlen = MLX5_ST_SZ_BYTES(create_cq_in) +</div><div class='ctx'> 		 MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) *</div><div class='hunk'>@@ -858,7 +923,7 @@ static int create_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,</div><div class='ctx'> 		 cq-&gt;buf.frag_buf.page_shift -</div><div class='ctx'> 		 MLX5_ADAPTER_PAGE_SHIFT);</div><div class='ctx'> </div><div class='del'>-	*index = dev-&gt;mdev-&gt;priv.uar-&gt;index;</div><div class='add'>+	*index = dev-&gt;mdev-&gt;priv.bfreg.up-&gt;index;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='hunk'>@@ -884,37 +949,34 @@ static void notify_soft_wc_handler(struct work_struct *work)</div><div class='ctx'> 	cq-&gt;ibcq.comp_handler(&amp;cq-&gt;ibcq, cq-&gt;ibcq.cq_context);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,</div><div class='del'>-				const struct ib_cq_init_attr *attr,</div><div class='del'>-				struct ib_udata *udata)</div><div class='add'>+int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		      struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='add'>+	struct ib_udata *udata = &amp;attrs-&gt;driver_udata;</div><div class='add'>+	struct ib_device *ibdev = ibcq-&gt;device;</div><div class='ctx'> 	int entries = attr-&gt;cqe;</div><div class='ctx'> 	int vector = attr-&gt;comp_vector;</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='del'>-	struct mlx5_ib_cq *cq;</div><div class='del'>-	int uninitialized_var(index);</div><div class='del'>-	int uninitialized_var(inlen);</div><div class='add'>+	struct mlx5_ib_cq *cq = to_mcq(ibcq);</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(create_cq_out)];</div><div class='add'>+	int index;</div><div class='add'>+	int inlen;</div><div class='ctx'> 	u32 *cqb = NULL;</div><div class='ctx'> 	void *cqc;</div><div class='ctx'> 	int cqe_size;</div><div class='del'>-	unsigned int irqn;</div><div class='ctx'> 	int eqn;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	if (entries &lt; 0 ||</div><div class='ctx'> 	    (entries &gt; (1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev, log_max_cq_sz))))</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	if (check_cq_create_flags(attr-&gt;flags))</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	entries = roundup_pow_of_two(entries + 1);</div><div class='ctx'> 	if (entries &gt; (1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev, log_max_cq_sz)))</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	cq = kzalloc(sizeof(*cq), GFP_KERNEL);</div><div class='del'>-	if (!cq)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	cq-&gt;ibcq.cqe = entries - 1;</div><div class='ctx'> 	mutex_init(&amp;cq-&gt;resize_mutex);</div><div class='hunk'>@@ -927,20 +989,20 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,</div><div class='ctx'> </div><div class='ctx'> 	if (udata) {</div><div class='ctx'> 		err = create_cq_user(dev, udata, cq, entries, &amp;cqb, &amp;cqe_size,</div><div class='del'>-				     &amp;index, &amp;inlen);</div><div class='add'>+				     &amp;index, &amp;inlen, attrs);</div><div class='ctx'> 		if (err)</div><div class='del'>-			goto err_create;</div><div class='add'>+			return err;</div><div class='ctx'> 	} else {</div><div class='ctx'> 		cqe_size = cache_line_size() == 128 ? 128 : 64;</div><div class='ctx'> 		err = create_cq_kernel(dev, cq, entries, cqe_size, &amp;cqb,</div><div class='ctx'> 				       &amp;index, &amp;inlen);</div><div class='ctx'> 		if (err)</div><div class='del'>-			goto err_create;</div><div class='add'>+			return err;</div><div class='ctx'> </div><div class='ctx'> 		INIT_WORK(&amp;cq-&gt;notify_work, notify_soft_wc_handler);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	err = mlx5_vector2eqn(dev-&gt;mdev, vector, &amp;eqn, &amp;irqn);</div><div class='add'>+	err = mlx5_comp_eqn_get(dev-&gt;mdev, vector, &amp;eqn);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_cqb;</div><div class='ctx'> </div><div class='hunk'>@@ -953,21 +1015,23 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,</div><div class='ctx'> 				  MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD));</div><div class='ctx'> 	MLX5_SET(cqc, cqc, log_cq_size, ilog2(entries));</div><div class='ctx'> 	MLX5_SET(cqc, cqc, uar_page, index);</div><div class='del'>-	MLX5_SET(cqc, cqc, c_eqn, eqn);</div><div class='add'>+	MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn);</div><div class='ctx'> 	MLX5_SET64(cqc, cqc, dbr_addr, cq-&gt;db.dma);</div><div class='ctx'> 	if (cq-&gt;create_flags &amp; IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN)</div><div class='ctx'> 		MLX5_SET(cqc, cqc, oi, 1);</div><div class='ctx'> </div><div class='del'>-	err = mlx5_core_create_cq(dev-&gt;mdev, &amp;cq-&gt;mcq, cqb, inlen);</div><div class='add'>+	if (udata) {</div><div class='add'>+		cq-&gt;mcq.comp = mlx5_add_cq_to_tasklet;</div><div class='add'>+		cq-&gt;mcq.tasklet_ctx.comp = mlx5_ib_cq_comp;</div><div class='add'>+	} else {</div><div class='add'>+		cq-&gt;mcq.comp  = mlx5_ib_cq_comp;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = mlx5_core_create_cq(dev-&gt;mdev, &amp;cq-&gt;mcq, cqb, inlen, out, sizeof(out));</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_cqb;</div><div class='ctx'> </div><div class='ctx'> 	mlx5_ib_dbg(dev, "cqn 0x%x\n", cq-&gt;mcq.cqn);</div><div class='del'>-	cq-&gt;mcq.irqn = irqn;</div><div class='del'>-	if (udata)</div><div class='del'>-		cq-&gt;mcq.tasklet_ctx.comp = mlx5_ib_cq_comp;</div><div class='del'>-	else</div><div class='del'>-		cq-&gt;mcq.comp  = mlx5_ib_cq_comp;</div><div class='ctx'> 	cq-&gt;mcq.event = mlx5_ib_cq_event;</div><div class='ctx'> </div><div class='ctx'> 	INIT_LIST_HEAD(&amp;cq-&gt;wc_list);</div><div class='hunk'>@@ -980,7 +1044,7 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,</div><div class='ctx'> </div><div class='ctx'> </div><div class='ctx'> 	kvfree(cqb);</div><div class='del'>-	return &amp;cq-&gt;ibcq;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='ctx'> err_cmd:</div><div class='ctx'> 	mlx5_core_destroy_cq(dev-&gt;mdev, &amp;cq-&gt;mcq);</div><div class='hunk'>@@ -991,26 +1055,34 @@ err_cqb:</div><div class='ctx'> 		destroy_cq_user(cq, udata);</div><div class='ctx'> 	else</div><div class='ctx'> 		destroy_cq_kernel(dev, cq);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-err_create:</div><div class='del'>-	kfree(cq);</div><div class='add'>+int mlx5_ib_pre_destroy_cq(struct ib_cq *cq)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(cq-&gt;device);</div><div class='add'>+	struct mlx5_ib_cq *mcq = to_mcq(cq);</div><div class='ctx'> </div><div class='del'>-	return ERR_PTR(err);</div><div class='add'>+	return mlx5_core_destroy_cq(dev-&gt;mdev, &amp;mcq-&gt;mcq);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_ib_post_destroy_cq(struct ib_cq *cq)</div><div class='add'>+{</div><div class='add'>+	destroy_cq_kernel(to_mdev(cq-&gt;device), to_mcq(cq));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(cq-&gt;device);</div><div class='del'>-	struct mlx5_ib_cq *mcq = to_mcq(cq);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = mlx5_ib_pre_destroy_cq(cq);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='del'>-	mlx5_core_destroy_cq(dev-&gt;mdev, &amp;mcq-&gt;mcq);</div><div class='ctx'> 	if (udata)</div><div class='del'>-		destroy_cq_user(mcq, udata);</div><div class='add'>+		destroy_cq_user(to_mcq(cq), udata);</div><div class='ctx'> 	else</div><div class='del'>-		destroy_cq_kernel(dev, mcq);</div><div class='del'>-</div><div class='del'>-	kfree(mcq);</div><div class='del'>-</div><div class='add'>+		mlx5_ib_post_destroy_cq(cq);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -1101,13 +1173,12 @@ int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,</div><div class='del'>-		       int entries, struct ib_udata *udata, int *npas,</div><div class='del'>-		       int *page_shift, int *cqe_size)</div><div class='add'>+		       int entries, struct ib_udata *udata,</div><div class='add'>+		       int *cqe_size)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_resize_cq ucmd;</div><div class='ctx'> 	struct ib_umem *umem;</div><div class='ctx'> 	int err;</div><div class='del'>-	int npages;</div><div class='ctx'> </div><div class='ctx'> 	err = ib_copy_from_udata(&amp;ucmd, udata, sizeof(ucmd));</div><div class='ctx'> 	if (err)</div><div class='hunk'>@@ -1120,28 +1191,20 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,</div><div class='ctx'> 	if (ucmd.cqe_size &amp;&amp; SIZE_MAX / ucmd.cqe_size &lt;= entries - 1)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	umem = ib_umem_get(udata, ucmd.buf_addr,</div><div class='add'>+	umem = ib_umem_get(&amp;dev-&gt;ib_dev, ucmd.buf_addr,</div><div class='ctx'> 			   (size_t)ucmd.cqe_size * entries,</div><div class='del'>-			   IB_ACCESS_LOCAL_WRITE, 1);</div><div class='add'>+			   IB_ACCESS_LOCAL_WRITE);</div><div class='ctx'> 	if (IS_ERR(umem)) {</div><div class='ctx'> 		err = PTR_ERR(umem);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_cont_pages(umem, ucmd.buf_addr, 0, &amp;npages, page_shift,</div><div class='del'>-			   npas, NULL);</div><div class='del'>-</div><div class='ctx'> 	cq-&gt;resize_umem = umem;</div><div class='ctx'> 	*cqe_size = ucmd.cqe_size;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void un_resize_user(struct mlx5_ib_cq *cq)</div><div class='del'>-{</div><div class='del'>-	ib_umem_release(cq-&gt;resize_umem);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,</div><div class='ctx'> 			 int entries, int cqe_size)</div><div class='ctx'> {</div><div class='hunk'>@@ -1155,7 +1218,7 @@ static int resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto ex;</div><div class='ctx'> </div><div class='del'>-	init_cq_frag_buf(cq, cq-&gt;resize_buf);</div><div class='add'>+	init_cq_frag_buf(cq-&gt;resize_buf);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='hunk'>@@ -1164,12 +1227,6 @@ ex:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void un_resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq)</div><div class='del'>-{</div><div class='del'>-	free_cq_buf(dev, cq-&gt;resize_buf);</div><div class='del'>-	cq-&gt;resize_buf = NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int copy_resize_cqes(struct mlx5_ib_cq *cq)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(cq-&gt;ibcq.device);</div><div class='hunk'>@@ -1234,9 +1291,10 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)</div><div class='ctx'> 	int err;</div><div class='ctx'> 	int npas;</div><div class='ctx'> 	__be64 *pas;</div><div class='del'>-	int page_shift;</div><div class='add'>+	unsigned int page_offset_quantized = 0;</div><div class='add'>+	unsigned int page_shift;</div><div class='ctx'> 	int inlen;</div><div class='del'>-	int uninitialized_var(cqe_size);</div><div class='add'>+	int cqe_size;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> </div><div class='ctx'> 	if (!MLX5_CAP_GEN(dev-&gt;mdev, cq_resize)) {</div><div class='hunk'>@@ -1261,22 +1319,34 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;cq-&gt;resize_mutex);</div><div class='ctx'> 	if (udata) {</div><div class='del'>-		err = resize_user(dev, cq, entries, udata, &amp;npas, &amp;page_shift,</div><div class='del'>-				  &amp;cqe_size);</div><div class='add'>+		unsigned long page_size;</div><div class='add'>+</div><div class='add'>+		err = resize_user(dev, cq, entries, udata, &amp;cqe_size);</div><div class='add'>+		if (err)</div><div class='add'>+			goto ex;</div><div class='add'>+</div><div class='add'>+		page_size = mlx5_umem_find_best_cq_quantized_pgoff(</div><div class='add'>+			cq-&gt;resize_umem, cqc, log_page_size,</div><div class='add'>+			MLX5_ADAPTER_PAGE_SHIFT, page_offset, 64,</div><div class='add'>+			&amp;page_offset_quantized);</div><div class='add'>+		if (!page_size) {</div><div class='add'>+			err = -EINVAL;</div><div class='add'>+			goto ex_resize;</div><div class='add'>+		}</div><div class='add'>+		npas = ib_umem_num_dma_blocks(cq-&gt;resize_umem, page_size);</div><div class='add'>+		page_shift = order_base_2(page_size);</div><div class='ctx'> 	} else {</div><div class='add'>+		struct mlx5_frag_buf *frag_buf;</div><div class='add'>+</div><div class='ctx'> 		cqe_size = 64;</div><div class='ctx'> 		err = resize_kernel(dev, cq, entries, cqe_size);</div><div class='del'>-		if (!err) {</div><div class='del'>-			struct mlx5_frag_buf *frag_buf = &amp;cq-&gt;resize_buf-&gt;frag_buf;</div><div class='del'>-</div><div class='del'>-			npas = frag_buf-&gt;npages;</div><div class='del'>-			page_shift = frag_buf-&gt;page_shift;</div><div class='del'>-		}</div><div class='add'>+		if (err)</div><div class='add'>+			goto ex;</div><div class='add'>+		frag_buf = &amp;cq-&gt;resize_buf-&gt;frag_buf;</div><div class='add'>+		npas = frag_buf-&gt;npages;</div><div class='add'>+		page_shift = frag_buf-&gt;page_shift;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (err)</div><div class='del'>-		goto ex;</div><div class='del'>-</div><div class='ctx'> 	inlen = MLX5_ST_SZ_BYTES(modify_cq_in) +</div><div class='ctx'> 		MLX5_FLD_SZ_BYTES(modify_cq_in, pas[0]) * npas;</div><div class='ctx'> </div><div class='hunk'>@@ -1288,8 +1358,8 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)</div><div class='ctx'> </div><div class='ctx'> 	pas = (__be64 *)MLX5_ADDR_OF(modify_cq_in, in, pas);</div><div class='ctx'> 	if (udata)</div><div class='del'>-		mlx5_ib_populate_pas(dev, cq-&gt;resize_umem, page_shift,</div><div class='del'>-				     pas, 0);</div><div class='add'>+		mlx5_ib_populate_pas(cq-&gt;resize_umem, 1UL &lt;&lt; page_shift, pas,</div><div class='add'>+				     0);</div><div class='ctx'> 	else</div><div class='ctx'> 		mlx5_fill_page_frag_array(&amp;cq-&gt;resize_buf-&gt;frag_buf, pas);</div><div class='ctx'> </div><div class='hunk'>@@ -1303,6 +1373,7 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(cqc, cqc, log_page_size,</div><div class='ctx'> 		 page_shift - MLX5_ADAPTER_PAGE_SHIFT);</div><div class='add'>+	MLX5_SET(cqc, cqc, page_offset, page_offset_quantized);</div><div class='ctx'> 	MLX5_SET(cqc, cqc, cqe_sz,</div><div class='ctx'> 		 cqe_sz_to_mlx_sz(cqe_size,</div><div class='ctx'> 				  cq-&gt;private_flags &amp;</div><div class='hunk'>@@ -1350,10 +1421,11 @@ ex_alloc:</div><div class='ctx'> 	kvfree(in);</div><div class='ctx'> </div><div class='ctx'> ex_resize:</div><div class='del'>-	if (udata)</div><div class='del'>-		un_resize_user(cq);</div><div class='del'>-	else</div><div class='del'>-		un_resize_kernel(dev, cq);</div><div class='add'>+	ib_umem_release(cq-&gt;resize_umem);</div><div class='add'>+	if (!udata) {</div><div class='add'>+		free_cq_buf(dev, cq-&gt;resize_buf);</div><div class='add'>+		cq-&gt;resize_buf = NULL;</div><div class='add'>+	}</div><div class='ctx'> ex:</div><div class='ctx'> 	mutex_unlock(&amp;cq-&gt;resize_mutex);</div><div class='ctx'> 	return err;</div><div class='hunk'>@@ -1393,3 +1465,17 @@ int mlx5_ib_generate_wc(struct ib_cq *ibcq, struct ib_wc *wc)</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='add'>+</div><div class='add'>+ADD_UVERBS_ATTRIBUTES_SIMPLE(</div><div class='add'>+	mlx5_ib_cq_create,</div><div class='add'>+	UVERBS_OBJECT_CQ,</div><div class='add'>+	UVERBS_METHOD_CQ_CREATE,</div><div class='add'>+	UVERBS_ATTR_PTR_IN(</div><div class='add'>+		MLX5_IB_ATTR_CREATE_CQ_UAR_INDEX,</div><div class='add'>+		UVERBS_ATTR_TYPE(u32),</div><div class='add'>+		UA_OPTIONAL));</div><div class='add'>+</div><div class='add'>+const struct uapi_definition mlx5_ib_create_cq_defs[] = {</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_CQ, &amp;mlx5_ib_cq_create),</div><div class='add'>+	{},</div><div class='add'>+};</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c<br/>new file mode 100644<br/>index 000000000000..b81ac5709b56<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/data_direct.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/data_direct.c</a></div><div class='hunk'>@@ -0,0 +1,227 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2024, NVIDIA CORPORATION &amp; AFFILIATES. All rights reserved</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include "mlx5_ib.h"</div><div class='add'>+#include "data_direct.h"</div><div class='add'>+</div><div class='add'>+static LIST_HEAD(mlx5_data_direct_dev_list);</div><div class='add'>+static LIST_HEAD(mlx5_data_direct_reg_list);</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * This mutex should be held when accessing either of the above lists</div><div class='add'>+ */</div><div class='add'>+static DEFINE_MUTEX(mlx5_data_direct_mutex);</div><div class='add'>+</div><div class='add'>+struct mlx5_data_direct_registration {</div><div class='add'>+	struct mlx5_ib_dev *ibdev;</div><div class='add'>+	char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1];</div><div class='add'>+	struct list_head list;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct pci_device_id mlx5_data_direct_pci_table[] = {</div><div class='add'>+	{ PCI_VDEVICE(MELLANOX, 0x2100) }, /* ConnectX-8 Data Direct */</div><div class='add'>+	{ 0, }</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int mlx5_data_direct_vpd_get_vuid(struct mlx5_data_direct_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct pci_dev *pdev = dev-&gt;pdev;</div><div class='add'>+	unsigned int vpd_size, kw_len;</div><div class='add'>+	u8 *vpd_data;</div><div class='add'>+	int start;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	vpd_data = pci_vpd_alloc(pdev, &amp;vpd_size);</div><div class='add'>+	if (IS_ERR(vpd_data)) {</div><div class='add'>+		pci_err(pdev, "Unable to read VPD, err=%pe\n", vpd_data);</div><div class='add'>+		return PTR_ERR(vpd_data);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, "VU", &amp;kw_len);</div><div class='add'>+	if (start &lt; 0) {</div><div class='add'>+		ret = start;</div><div class='add'>+		pci_err(pdev, "VU keyword not found, err=%d\n", ret);</div><div class='add'>+		goto end;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dev-&gt;vuid = kmemdup_nul(vpd_data + start, kw_len, GFP_KERNEL);</div><div class='add'>+	ret = dev-&gt;vuid ? 0 : -ENOMEM;</div><div class='add'>+</div><div class='add'>+end:</div><div class='add'>+	kfree(vpd_data);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_data_direct_shutdown(struct pci_dev *pdev)</div><div class='add'>+{</div><div class='add'>+	pci_disable_device(pdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev)</div><div class='add'>+{</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	err = dma_set_mask_and_coherent(&amp;pdev-&gt;dev, DMA_BIT_MASK(64));</div><div class='add'>+	if (err) {</div><div class='add'>+		dev_warn(&amp;pdev-&gt;dev,</div><div class='add'>+			 "Warning: couldn't set 64-bit PCI DMA mask, err=%d\n", err);</div><div class='add'>+		err = dma_set_mask_and_coherent(&amp;pdev-&gt;dev, DMA_BIT_MASK(32));</div><div class='add'>+		if (err) {</div><div class='add'>+			dev_err(&amp;pdev-&gt;dev, "Can't set PCI DMA mask, err=%d\n", err);</div><div class='add'>+			return err;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dma_set_max_seg_size(&amp;pdev-&gt;dev, SZ_2G);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_data_direct_registration *reg;</div><div class='add'>+	struct mlx5_data_direct_dev *dev;</div><div class='add'>+</div><div class='add'>+	reg = kzalloc(sizeof(*reg), GFP_KERNEL);</div><div class='add'>+	if (!reg)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	reg-&gt;ibdev = ibdev;</div><div class='add'>+	strcpy(reg-&gt;vuid, vuid);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;mlx5_data_direct_mutex);</div><div class='add'>+	list_for_each_entry(dev, &amp;mlx5_data_direct_dev_list, list) {</div><div class='add'>+		if (strcmp(dev-&gt;vuid, vuid) == 0) {</div><div class='add'>+			mlx5_ib_data_direct_bind(ibdev, dev);</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Add the registration to its global list, to be used upon bind/unbind</div><div class='add'>+	 * of its affiliated data direct device</div><div class='add'>+	 */</div><div class='add'>+	list_add_tail(&amp;reg-&gt;list, &amp;mlx5_data_direct_reg_list);</div><div class='add'>+	mutex_unlock(&amp;mlx5_data_direct_mutex);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_data_direct_registration *reg;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;mlx5_data_direct_mutex);</div><div class='add'>+	list_for_each_entry(reg, &amp;mlx5_data_direct_reg_list, list) {</div><div class='add'>+		if (reg-&gt;ibdev == ibdev) {</div><div class='add'>+			list_del(&amp;reg-&gt;list);</div><div class='add'>+			kfree(reg);</div><div class='add'>+			goto end;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	WARN_ON(true);</div><div class='add'>+end:</div><div class='add'>+	mutex_unlock(&amp;mlx5_data_direct_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_data_direct_registration *reg;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;mlx5_data_direct_mutex);</div><div class='add'>+	list_for_each_entry(reg, &amp;mlx5_data_direct_reg_list, list) {</div><div class='add'>+		if (strcmp(dev-&gt;vuid, reg-&gt;vuid) == 0)</div><div class='add'>+			mlx5_ib_data_direct_bind(reg-&gt;ibdev, dev);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Add the data direct device to the global list, further IB devices may</div><div class='add'>+	 * use it later as well</div><div class='add'>+	 */</div><div class='add'>+	list_add_tail(&amp;dev-&gt;list, &amp;mlx5_data_direct_dev_list);</div><div class='add'>+	mutex_unlock(&amp;mlx5_data_direct_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_data_direct_dev_unreg(struct mlx5_data_direct_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_data_direct_registration *reg;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;mlx5_data_direct_mutex);</div><div class='add'>+	/* Prevent any further affiliations */</div><div class='add'>+	list_del(&amp;dev-&gt;list);</div><div class='add'>+	list_for_each_entry(reg, &amp;mlx5_data_direct_reg_list, list) {</div><div class='add'>+		if (strcmp(dev-&gt;vuid, reg-&gt;vuid) == 0)</div><div class='add'>+			mlx5_ib_data_direct_unbind(reg-&gt;ibdev);</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;mlx5_data_direct_mutex);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_data_direct_probe(struct pci_dev *pdev, const struct pci_device_id *id)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_data_direct_dev *dev;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	dev = kzalloc(sizeof(*dev), GFP_KERNEL);</div><div class='add'>+	if (!dev)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	dev-&gt;device = &amp;pdev-&gt;dev;</div><div class='add'>+	dev-&gt;pdev = pdev;</div><div class='add'>+</div><div class='add'>+	pci_set_drvdata(dev-&gt;pdev, dev);</div><div class='add'>+	err = pci_enable_device(pdev);</div><div class='add'>+	if (err) {</div><div class='add'>+		dev_err(dev-&gt;device, "Cannot enable PCI device, err=%d\n", err);</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	pci_set_master(pdev);</div><div class='add'>+	err = mlx5_data_direct_set_dma_caps(pdev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_disable;</div><div class='add'>+</div><div class='add'>+	if (pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) &amp;&amp;</div><div class='add'>+	    pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64) &amp;&amp;</div><div class='add'>+	    pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP128))</div><div class='add'>+		dev_dbg(dev-&gt;device, "Enabling pci atomics failed\n");</div><div class='add'>+</div><div class='add'>+	err = mlx5_data_direct_vpd_get_vuid(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_disable;</div><div class='add'>+</div><div class='add'>+	mlx5_data_direct_dev_reg(dev);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_disable:</div><div class='add'>+	pci_disable_device(pdev);</div><div class='add'>+err:</div><div class='add'>+	kfree(dev);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_data_direct_remove(struct pci_dev *pdev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_data_direct_dev *dev = pci_get_drvdata(pdev);</div><div class='add'>+</div><div class='add'>+	mlx5_data_direct_dev_unreg(dev);</div><div class='add'>+	pci_disable_device(pdev);</div><div class='add'>+	kfree(dev-&gt;vuid);</div><div class='add'>+	kfree(dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct pci_driver mlx5_data_direct_driver = {</div><div class='add'>+	.name = KBUILD_MODNAME,</div><div class='add'>+	.id_table = mlx5_data_direct_pci_table,</div><div class='add'>+	.probe = mlx5_data_direct_probe,</div><div class='add'>+	.remove = mlx5_data_direct_remove,</div><div class='add'>+	.shutdown = mlx5_data_direct_shutdown,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+int mlx5_data_direct_driver_register(void)</div><div class='add'>+{</div><div class='add'>+	return pci_register_driver(&amp;mlx5_data_direct_driver);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_data_direct_driver_unregister(void)</div><div class='add'>+{</div><div class='add'>+	pci_unregister_driver(&amp;mlx5_data_direct_driver);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h<br/>new file mode 100644<br/>index 000000000000..2fd2bdbe8f69<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/data_direct.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/data_direct.h</a></div><div class='hunk'>@@ -0,0 +1,23 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2024, NVIDIA CORPORATION &amp; AFFILIATES. All rights reserved</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef _MLX5_IB_DATA_DIRECT_H</div><div class='add'>+#define _MLX5_IB_DATA_DIRECT_H</div><div class='add'>+</div><div class='add'>+struct mlx5_ib_dev;</div><div class='add'>+</div><div class='add'>+struct mlx5_data_direct_dev {</div><div class='add'>+	struct device *device;</div><div class='add'>+	struct pci_dev *pdev;</div><div class='add'>+	char *vuid;</div><div class='add'>+	struct list_head list;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid);</div><div class='add'>+void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev);</div><div class='add'>+int mlx5_data_direct_driver_register(void);</div><div class='add'>+void mlx5_data_direct_driver_unregister(void);</div><div class='add'>+</div><div class='add'>+#endif</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c<br/>index 931f587dfb8f..d31d7f3005c6 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/devx.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/devx.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/devx.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/devx.c</a></div><div class='hunk'>@@ -13,53 +13,108 @@</div><div class='ctx'> #include &lt;rdma/uverbs_std_types.h&gt;</div><div class='ctx'> #include &lt;linux/mlx5/driver.h&gt;</div><div class='ctx'> #include &lt;linux/mlx5/fs.h&gt;</div><div class='add'>+#include &lt;rdma/ib_ucaps.h&gt;</div><div class='ctx'> #include "mlx5_ib.h"</div><div class='add'>+#include "devx.h"</div><div class='add'>+#include "qp.h"</div><div class='add'>+#include &lt;linux/xarray.h&gt;</div><div class='ctx'> </div><div class='ctx'> #define UVERBS_MODULE_NAME mlx5_ib</div><div class='ctx'> #include &lt;rdma/uverbs_named_ioctl.h&gt;</div><div class='ctx'> </div><div class='add'>+static void dispatch_event_fd(struct list_head *fd_list, const void *data);</div><div class='add'>+</div><div class='ctx'> enum devx_obj_flags {</div><div class='ctx'> 	DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 &lt;&lt; 0,</div><div class='ctx'> 	DEVX_OBJ_FLAGS_DCT = 1 &lt;&lt; 1,</div><div class='add'>+	DEVX_OBJ_FLAGS_CQ = 1 &lt;&lt; 2,</div><div class='add'>+	DEVX_OBJ_FLAGS_HW_FREED = 1 &lt;&lt; 3,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define MAX_ASYNC_CMDS 8</div><div class='add'>+</div><div class='add'>+struct mlx5_async_cmd {</div><div class='add'>+	struct ib_uobject *uobject;</div><div class='add'>+	void *in;</div><div class='add'>+	int in_size;</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];</div><div class='add'>+	int err;</div><div class='add'>+	struct mlx5_async_work cb_work;</div><div class='add'>+	struct completion comp;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct devx_async_data {</div><div class='ctx'> 	struct mlx5_ib_dev *mdev;</div><div class='ctx'> 	struct list_head list;</div><div class='del'>-	struct ib_uobject *fd_uobj;</div><div class='add'>+	struct devx_async_cmd_event_file *ev_file;</div><div class='ctx'> 	struct mlx5_async_work cb_work;</div><div class='ctx'> 	u16 cmd_out_len;</div><div class='ctx'> 	/* must be last field in this structure */</div><div class='ctx'> 	struct mlx5_ib_uapi_devx_async_cmd_hdr hdr;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)</div><div class='del'>-struct devx_obj {</div><div class='del'>-	struct mlx5_core_dev	*mdev;</div><div class='del'>-	u64			obj_id;</div><div class='del'>-	u32			dinlen; /* destroy inbox length */</div><div class='del'>-	u32			dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];</div><div class='del'>-	u32			flags;</div><div class='del'>-	union {</div><div class='del'>-		struct mlx5_ib_devx_mr	devx_mr;</div><div class='del'>-		struct mlx5_core_dct	core_dct;</div><div class='del'>-	};</div><div class='add'>+struct devx_async_event_data {</div><div class='add'>+	struct list_head list; /* headed in ev_file-&gt;event_list */</div><div class='add'>+	struct mlx5_ib_uapi_devx_async_event_hdr hdr;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* first level XA value data structure */</div><div class='add'>+struct devx_event {</div><div class='add'>+	struct xarray object_ids; /* second XA level, Key = object id */</div><div class='add'>+	struct list_head unaffiliated_list;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* second level XA value data structure */</div><div class='add'>+struct devx_obj_event {</div><div class='add'>+	struct rcu_head rcu;</div><div class='add'>+	struct list_head obj_sub_list;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct devx_event_subscription {</div><div class='add'>+	struct list_head file_list; /* headed in ev_file-&gt;</div><div class='add'>+				     * subscribed_events_list</div><div class='add'>+				     */</div><div class='add'>+	struct list_head xa_list; /* headed in devx_event-&gt;unaffiliated_list or</div><div class='add'>+				   * devx_obj_event-&gt;obj_sub_list</div><div class='add'>+				   */</div><div class='add'>+	struct list_head obj_list; /* headed in devx_object */</div><div class='add'>+	struct list_head event_list; /* headed in ev_file-&gt;event_list or in</div><div class='add'>+				      * temp list via subscription</div><div class='add'>+				      */</div><div class='add'>+</div><div class='add'>+	u8 is_cleaned:1;</div><div class='add'>+	u32 xa_key_level1;</div><div class='add'>+	u32 xa_key_level2;</div><div class='add'>+	struct rcu_head	rcu;</div><div class='add'>+	u64 cookie;</div><div class='add'>+	struct devx_async_event_file *ev_file;</div><div class='add'>+	struct eventfd_ctx *eventfd;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct devx_async_event_file {</div><div class='add'>+	struct ib_uobject uobj;</div><div class='add'>+	/* Head of events that are subscribed to this FD */</div><div class='add'>+	struct list_head subscribed_events_list;</div><div class='add'>+	spinlock_t lock;</div><div class='add'>+	wait_queue_head_t poll_wait;</div><div class='add'>+	struct list_head event_list;</div><div class='add'>+	struct mlx5_ib_dev *dev;</div><div class='add'>+	u8 omit_data:1;</div><div class='add'>+	u8 is_overflow_err:1;</div><div class='add'>+	u8 is_destroyed:1;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct devx_umem {</div><div class='ctx'> 	struct mlx5_core_dev		*mdev;</div><div class='ctx'> 	struct ib_umem			*umem;</div><div class='del'>-	u32				page_offset;</div><div class='del'>-	int				page_shift;</div><div class='del'>-	int				ncont;</div><div class='ctx'> 	u32				dinlen;</div><div class='del'>-	u32				dinbox[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)];</div><div class='add'>+	u32				dinbox[MLX5_ST_SZ_DW(destroy_umem_in)];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct devx_umem_reg_cmd {</div><div class='ctx'> 	void				*in;</div><div class='ctx'> 	u32				inlen;</div><div class='del'>-	u32				out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];</div><div class='add'>+	u32				out[MLX5_ST_SZ_DW(create_umem_out)];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static struct mlx5_ib_ucontext *</div><div class='hunk'>@@ -68,10 +123,30 @@ devx_ufile2uctx(const struct uverbs_attr_bundle *attrs)</div><div class='ctx'> 	return to_mucontext(ib_uverbs_get_ucontext(attrs));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user)</div><div class='add'>+static int set_uctx_ucaps(struct mlx5_ib_dev *dev, u64 req_ucaps, u32 *cap)</div><div class='ctx'> {</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {0};</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};</div><div class='add'>+	if (UCAP_ENABLED(req_ucaps, RDMA_UCAP_MLX5_CTRL_LOCAL)) {</div><div class='add'>+		if (MLX5_CAP_GEN(dev-&gt;mdev, uctx_cap) &amp; MLX5_UCTX_CAP_RDMA_CTRL)</div><div class='add'>+			*cap |= MLX5_UCTX_CAP_RDMA_CTRL;</div><div class='add'>+		else</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (UCAP_ENABLED(req_ucaps, RDMA_UCAP_MLX5_CTRL_OTHER_VHCA)) {</div><div class='add'>+		if (MLX5_CAP_GEN(dev-&gt;mdev, uctx_cap) &amp;</div><div class='add'>+		    MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA)</div><div class='add'>+			*cap |= MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA;</div><div class='add'>+		else</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user, u64 req_ucaps)</div><div class='add'>+{</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {};</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(create_uctx_out)] = {};</div><div class='ctx'> 	void *uctx;</div><div class='ctx'> 	int err;</div><div class='ctx'> 	u16 uid;</div><div class='hunk'>@@ -82,14 +157,22 @@ int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx);</div><div class='del'>-	if (is_user &amp;&amp; capable(CAP_NET_RAW) &amp;&amp;</div><div class='del'>-	    (MLX5_CAP_GEN(dev-&gt;mdev, uctx_cap) &amp; MLX5_UCTX_CAP_RAW_TX))</div><div class='add'>+	if (is_user &amp;&amp;</div><div class='add'>+	    (MLX5_CAP_GEN(dev-&gt;mdev, uctx_cap) &amp; MLX5_UCTX_CAP_RAW_TX) &amp;&amp;</div><div class='add'>+	    rdma_dev_has_raw_cap(&amp;dev-&gt;ib_dev))</div><div class='ctx'> 		cap |= MLX5_UCTX_CAP_RAW_TX;</div><div class='del'>-	if (is_user &amp;&amp; capable(CAP_SYS_RAWIO) &amp;&amp;</div><div class='add'>+	if (is_user &amp;&amp;</div><div class='ctx'> 	    (MLX5_CAP_GEN(dev-&gt;mdev, uctx_cap) &amp;</div><div class='del'>-	     MLX5_UCTX_CAP_INTERNAL_DEV_RES))</div><div class='add'>+	     MLX5_UCTX_CAP_INTERNAL_DEV_RES) &amp;&amp;</div><div class='add'>+	    capable(CAP_SYS_RAWIO))</div><div class='ctx'> 		cap |= MLX5_UCTX_CAP_INTERNAL_DEV_RES;</div><div class='ctx'> </div><div class='add'>+	if (req_ucaps) {</div><div class='add'>+		err = set_uctx_ucaps(dev, req_ucaps, &amp;cap);</div><div class='add'>+		if (err)</div><div class='add'>+			return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);</div><div class='ctx'> 	MLX5_SET(uctx, uctx, cap, cap);</div><div class='ctx'> </div><div class='hunk'>@@ -97,14 +180,14 @@ int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user)</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='del'>-	uid = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);</div><div class='add'>+	uid = MLX5_GET(create_uctx_out, out, uid);</div><div class='ctx'> 	return uid;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid)</div><div class='ctx'> {</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {0};</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {};</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(destroy_uctx_out)] = {};</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX);</div><div class='ctx'> 	MLX5_SET(destroy_uctx_in, in, uid, uid);</div><div class='hunk'>@@ -112,41 +195,130 @@ void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid)</div><div class='ctx'> 	mlx5_cmd_exec(dev-&gt;mdev, in, sizeof(in), out, sizeof(out));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type)</div><div class='add'>+static bool is_legacy_unaffiliated_event_num(u16 event_num)</div><div class='ctx'> {</div><div class='del'>-	struct devx_obj *devx_obj = obj;</div><div class='del'>-	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj-&gt;dinbox, opcode);</div><div class='del'>-</div><div class='del'>-	switch (opcode) {</div><div class='del'>-	case MLX5_CMD_OP_DESTROY_TIR:</div><div class='del'>-		*dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;</div><div class='del'>-		*dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj-&gt;dinbox,</div><div class='del'>-				    obj_id);</div><div class='add'>+	switch (event_num) {</div><div class='add'>+	case MLX5_EVENT_TYPE_PORT_CHANGE:</div><div class='ctx'> 		return true;</div><div class='add'>+	default:</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	case MLX5_CMD_OP_DESTROY_FLOW_TABLE:</div><div class='del'>-		*dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;</div><div class='del'>-		*dest_id = MLX5_GET(destroy_flow_table_in, devx_obj-&gt;dinbox,</div><div class='del'>-				    table_id);</div><div class='add'>+static bool is_legacy_obj_event_num(u16 event_num)</div><div class='add'>+{</div><div class='add'>+	switch (event_num) {</div><div class='add'>+	case MLX5_EVENT_TYPE_PATH_MIG:</div><div class='add'>+	case MLX5_EVENT_TYPE_COMM_EST:</div><div class='add'>+	case MLX5_EVENT_TYPE_SQ_DRAINED:</div><div class='add'>+	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:</div><div class='add'>+	case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:</div><div class='add'>+	case MLX5_EVENT_TYPE_CQ_ERROR:</div><div class='add'>+	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:</div><div class='add'>+	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:</div><div class='add'>+	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:</div><div class='add'>+	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:</div><div class='add'>+	case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:</div><div class='add'>+	case MLX5_EVENT_TYPE_DCT_DRAINED:</div><div class='add'>+	case MLX5_EVENT_TYPE_COMP:</div><div class='add'>+	case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:</div><div class='add'>+	case MLX5_EVENT_TYPE_XRQ_ERROR:</div><div class='ctx'> 		return true;</div><div class='ctx'> 	default:</div><div class='ctx'> 		return false;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-bool mlx5_ib_devx_is_flow_counter(void *obj, u32 *counter_id)</div><div class='add'>+static u16 get_legacy_obj_type(u16 opcode)</div><div class='ctx'> {</div><div class='del'>-	struct devx_obj *devx_obj = obj;</div><div class='del'>-	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj-&gt;dinbox, opcode);</div><div class='add'>+	switch (opcode) {</div><div class='add'>+	case MLX5_CMD_OP_CREATE_RQ:</div><div class='add'>+	case MLX5_CMD_OP_CREATE_RMP:</div><div class='add'>+		return MLX5_EVENT_QUEUE_TYPE_RQ;</div><div class='add'>+	case MLX5_CMD_OP_CREATE_QP:</div><div class='add'>+		return MLX5_EVENT_QUEUE_TYPE_QP;</div><div class='add'>+	case MLX5_CMD_OP_CREATE_SQ:</div><div class='add'>+		return MLX5_EVENT_QUEUE_TYPE_SQ;</div><div class='add'>+	case MLX5_CMD_OP_CREATE_DCT:</div><div class='add'>+		return MLX5_EVENT_QUEUE_TYPE_DCT;</div><div class='add'>+	default:</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {</div><div class='del'>-		*counter_id = MLX5_GET(dealloc_flow_counter_in,</div><div class='del'>-				       devx_obj-&gt;dinbox,</div><div class='del'>-				       flow_counter_id);</div><div class='del'>-		return true;</div><div class='add'>+static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num)</div><div class='add'>+{</div><div class='add'>+	u16 opcode;</div><div class='add'>+</div><div class='add'>+	opcode = (obj-&gt;obj_id &gt;&gt; 32) &amp; 0xffff;</div><div class='add'>+</div><div class='add'>+	if (is_legacy_obj_event_num(event_num))</div><div class='add'>+		return get_legacy_obj_type(opcode);</div><div class='add'>+</div><div class='add'>+	switch (opcode) {</div><div class='add'>+	case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:</div><div class='add'>+		return (obj-&gt;obj_id &gt;&gt; 48);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_RQ:</div><div class='add'>+		return MLX5_OBJ_TYPE_RQ;</div><div class='add'>+	case MLX5_CMD_OP_CREATE_QP:</div><div class='add'>+		return MLX5_OBJ_TYPE_QP;</div><div class='add'>+	case MLX5_CMD_OP_CREATE_SQ:</div><div class='add'>+		return MLX5_OBJ_TYPE_SQ;</div><div class='add'>+	case MLX5_CMD_OP_CREATE_DCT:</div><div class='add'>+		return MLX5_OBJ_TYPE_DCT;</div><div class='add'>+	case MLX5_CMD_OP_CREATE_TIR:</div><div class='add'>+		return MLX5_OBJ_TYPE_TIR;</div><div class='add'>+	case MLX5_CMD_OP_CREATE_TIS:</div><div class='add'>+		return MLX5_OBJ_TYPE_TIS;</div><div class='add'>+	case MLX5_CMD_OP_CREATE_PSV:</div><div class='add'>+		return MLX5_OBJ_TYPE_PSV;</div><div class='add'>+	case MLX5_OBJ_TYPE_MKEY:</div><div class='add'>+		return MLX5_OBJ_TYPE_MKEY;</div><div class='add'>+	case MLX5_CMD_OP_CREATE_RMP:</div><div class='add'>+		return MLX5_OBJ_TYPE_RMP;</div><div class='add'>+	case MLX5_CMD_OP_CREATE_XRC_SRQ:</div><div class='add'>+		return MLX5_OBJ_TYPE_XRC_SRQ;</div><div class='add'>+	case MLX5_CMD_OP_CREATE_XRQ:</div><div class='add'>+		return MLX5_OBJ_TYPE_XRQ;</div><div class='add'>+	case MLX5_CMD_OP_CREATE_RQT:</div><div class='add'>+		return MLX5_OBJ_TYPE_RQT;</div><div class='add'>+	case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:</div><div class='add'>+		return MLX5_OBJ_TYPE_FLOW_COUNTER;</div><div class='add'>+	case MLX5_CMD_OP_CREATE_CQ:</div><div class='add'>+		return MLX5_OBJ_TYPE_CQ;</div><div class='add'>+	default:</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u16 get_event_obj_type(unsigned long event_type, struct mlx5_eqe *eqe)</div><div class='add'>+{</div><div class='add'>+	switch (event_type) {</div><div class='add'>+	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:</div><div class='add'>+	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:</div><div class='add'>+	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:</div><div class='add'>+	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:</div><div class='add'>+	case MLX5_EVENT_TYPE_PATH_MIG:</div><div class='add'>+	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:</div><div class='add'>+	case MLX5_EVENT_TYPE_COMM_EST:</div><div class='add'>+	case MLX5_EVENT_TYPE_SQ_DRAINED:</div><div class='add'>+	case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:</div><div class='add'>+	case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:</div><div class='add'>+		return eqe-&gt;data.qp_srq.type;</div><div class='add'>+	case MLX5_EVENT_TYPE_CQ_ERROR:</div><div class='add'>+	case MLX5_EVENT_TYPE_XRQ_ERROR:</div><div class='add'>+		return 0;</div><div class='add'>+	case MLX5_EVENT_TYPE_DCT_DRAINED:</div><div class='add'>+	case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:</div><div class='add'>+		return MLX5_EVENT_QUEUE_TYPE_DCT;</div><div class='add'>+	default:</div><div class='add'>+		return MLX5_GET(affiliated_event_header, &amp;eqe-&gt;data, obj_type);</div><div class='ctx'> 	}</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	return false;</div><div class='add'>+static u32 get_dec_obj_id(u64 obj_id)</div><div class='add'>+{</div><div class='add'>+	return (obj_id &amp; 0xffffffff);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='hunk'>@@ -159,6 +331,80 @@ static u64 get_enc_obj_id(u32 opcode, u32 obj_id)</div><div class='ctx'> 	return ((u64)opcode &lt;&lt; 32) | obj_id;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static u32 devx_get_created_obj_id(const void *in, const void *out, u16 opcode)</div><div class='add'>+{</div><div class='add'>+	switch (opcode) {</div><div class='add'>+	case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:</div><div class='add'>+		return MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_UMEM:</div><div class='add'>+		return MLX5_GET(create_umem_out, out, umem_id);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_MKEY:</div><div class='add'>+		return MLX5_GET(create_mkey_out, out, mkey_index);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_CQ:</div><div class='add'>+		return MLX5_GET(create_cq_out, out, cqn);</div><div class='add'>+	case MLX5_CMD_OP_ALLOC_PD:</div><div class='add'>+		return MLX5_GET(alloc_pd_out, out, pd);</div><div class='add'>+	case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:</div><div class='add'>+		return MLX5_GET(alloc_transport_domain_out, out,</div><div class='add'>+				transport_domain);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_RMP:</div><div class='add'>+		return MLX5_GET(create_rmp_out, out, rmpn);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_SQ:</div><div class='add'>+		return MLX5_GET(create_sq_out, out, sqn);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_RQ:</div><div class='add'>+		return MLX5_GET(create_rq_out, out, rqn);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_RQT:</div><div class='add'>+		return MLX5_GET(create_rqt_out, out, rqtn);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_TIR:</div><div class='add'>+		return MLX5_GET(create_tir_out, out, tirn);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_TIS:</div><div class='add'>+		return MLX5_GET(create_tis_out, out, tisn);</div><div class='add'>+	case MLX5_CMD_OP_ALLOC_Q_COUNTER:</div><div class='add'>+		return MLX5_GET(alloc_q_counter_out, out, counter_set_id);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_FLOW_TABLE:</div><div class='add'>+		return MLX5_GET(create_flow_table_out, out, table_id);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_FLOW_GROUP:</div><div class='add'>+		return MLX5_GET(create_flow_group_out, out, group_id);</div><div class='add'>+	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:</div><div class='add'>+		return MLX5_GET(set_fte_in, in, flow_index);</div><div class='add'>+	case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:</div><div class='add'>+		return MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);</div><div class='add'>+	case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:</div><div class='add'>+		return MLX5_GET(alloc_packet_reformat_context_out, out,</div><div class='add'>+				packet_reformat_id);</div><div class='add'>+	case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:</div><div class='add'>+		return MLX5_GET(alloc_modify_header_context_out, out,</div><div class='add'>+				modify_header_id);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:</div><div class='add'>+		return MLX5_GET(create_scheduling_element_out, out,</div><div class='add'>+				scheduling_element_id);</div><div class='add'>+	case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:</div><div class='add'>+		return MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port);</div><div class='add'>+	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:</div><div class='add'>+		return MLX5_GET(set_l2_table_entry_in, in, table_index);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_QP:</div><div class='add'>+		return MLX5_GET(create_qp_out, out, qpn);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_SRQ:</div><div class='add'>+		return MLX5_GET(create_srq_out, out, srqn);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_XRC_SRQ:</div><div class='add'>+		return MLX5_GET(create_xrc_srq_out, out, xrc_srqn);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_DCT:</div><div class='add'>+		return MLX5_GET(create_dct_out, out, dctn);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_XRQ:</div><div class='add'>+		return MLX5_GET(create_xrq_out, out, xrqn);</div><div class='add'>+	case MLX5_CMD_OP_ATTACH_TO_MCG:</div><div class='add'>+		return MLX5_GET(attach_to_mcg_in, in, qpn);</div><div class='add'>+	case MLX5_CMD_OP_ALLOC_XRCD:</div><div class='add'>+		return MLX5_GET(alloc_xrcd_out, out, xrcd);</div><div class='add'>+	case MLX5_CMD_OP_CREATE_PSV:</div><div class='add'>+		return MLX5_GET(create_psv_out, out, psv0_index);</div><div class='add'>+	default:</div><div class='add'>+		/* The entry must match to one of the devx_is_obj_create_cmd */</div><div class='add'>+		WARN_ON(true);</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static u64 devx_get_obj_id(const void *in)</div><div class='ctx'> {</div><div class='ctx'> 	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);</div><div class='hunk'>@@ -270,8 +516,8 @@ static u64 devx_get_obj_id(const void *in)</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:</div><div class='ctx'> 		obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT,</div><div class='del'>-					MLX5_GET(general_obj_in_cmd_hdr, in,</div><div class='del'>-						 obj_id));</div><div class='add'>+					MLX5_GET(query_modify_header_context_in,</div><div class='add'>+						 in, modify_header_id));</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:</div><div class='ctx'> 		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,</div><div class='hunk'>@@ -306,6 +552,10 @@ static u64 devx_get_obj_id(const void *in)</div><div class='ctx'> 		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,</div><div class='ctx'> 					MLX5_GET(rst2init_qp_in, in, qpn));</div><div class='ctx'> 		break;</div><div class='add'>+	case MLX5_CMD_OP_INIT2INIT_QP:</div><div class='add'>+		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,</div><div class='add'>+					MLX5_GET(init2init_qp_in, in, qpn));</div><div class='add'>+		break;</div><div class='ctx'> 	case MLX5_CMD_OP_INIT2RTR_QP:</div><div class='ctx'> 		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,</div><div class='ctx'> 					MLX5_GET(init2rtr_qp_in, in, qpn));</div><div class='hunk'>@@ -363,6 +613,8 @@ static u64 devx_get_obj_id(const void *in)</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_ARM_XRQ:</div><div class='ctx'> 	case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:</div><div class='add'>+	case MLX5_CMD_OP_RELEASE_XRQ_ERROR:</div><div class='add'>+	case MLX5_CMD_OP_MODIFY_XRQ:</div><div class='ctx'> 		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,</div><div class='ctx'> 					MLX5_GET(arm_xrq_in, in, xrqn));</div><div class='ctx'> 		break;</div><div class='hunk'>@@ -421,10 +673,9 @@ static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 	case UVERBS_OBJECT_QP:</div><div class='ctx'> 	{</div><div class='ctx'> 		struct mlx5_ib_qp *qp = to_mqp(uobj-&gt;object);</div><div class='del'>-		enum ib_qp_type	qp_type = qp-&gt;ibqp.qp_type;</div><div class='ctx'> </div><div class='del'>-		if (qp_type == IB_QPT_RAW_PACKET ||</div><div class='del'>-		    (qp-&gt;flags &amp; MLX5_IB_QP_UNDERLAY)) {</div><div class='add'>+		if (qp-&gt;type == IB_QPT_RAW_PACKET ||</div><div class='add'>+		    (qp-&gt;flags &amp; IB_QP_CREATE_SOURCE_QPN)) {</div><div class='ctx'> 			struct mlx5_ib_raw_packet_qp *raw_packet_qp =</div><div class='ctx'> 							 &amp;qp-&gt;raw_packet_qp;</div><div class='ctx'> 			struct mlx5_ib_rq *rq = &amp;raw_packet_qp-&gt;rq;</div><div class='hunk'>@@ -440,10 +691,9 @@ static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 					       sq-&gt;tisn) == obj_id);</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		if (qp_type == MLX5_IB_QPT_DCT)</div><div class='add'>+		if (qp-&gt;type == MLX5_IB_QPT_DCT)</div><div class='ctx'> 			return get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,</div><div class='ctx'> 					      qp-&gt;dct.mdct.mqp.qpn) == obj_id;</div><div class='del'>-</div><div class='ctx'> 		return get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,</div><div class='ctx'> 				      qp-&gt;ibqp.qp_num) == obj_id;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -459,7 +709,21 @@ static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,</div><div class='ctx'> 				      obj_id;</div><div class='ctx'> </div><div class='ctx'> 	case MLX5_IB_OBJECT_DEVX_OBJ:</div><div class='del'>-		return ((struct devx_obj *)uobj-&gt;object)-&gt;obj_id == obj_id;</div><div class='add'>+	{</div><div class='add'>+		u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);</div><div class='add'>+		struct devx_obj *devx_uobj = uobj-&gt;object;</div><div class='add'>+</div><div class='add'>+		if (opcode == MLX5_CMD_OP_QUERY_FLOW_COUNTER &amp;&amp;</div><div class='add'>+		    devx_uobj-&gt;flow_counter_bulk_size) {</div><div class='add'>+			u64 end;</div><div class='add'>+</div><div class='add'>+			end = devx_uobj-&gt;obj_id +</div><div class='add'>+				devx_uobj-&gt;flow_counter_bulk_size;</div><div class='add'>+			return devx_uobj-&gt;obj_id &lt;= obj_id &amp;&amp; end &gt; obj_id;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		return devx_uobj-&gt;obj_id == obj_id;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	default:</div><div class='ctx'> 		return false;</div><div class='hunk'>@@ -597,6 +861,14 @@ static bool devx_is_obj_create_cmd(const void *in, u16 *opcode)</div><div class='ctx'> 			return true;</div><div class='ctx'> 		return false;</div><div class='ctx'> 	}</div><div class='add'>+	case MLX5_CMD_OP_CREATE_PSV:</div><div class='add'>+	{</div><div class='add'>+		u8 num_psv = MLX5_GET(create_psv_in, in, num_psv);</div><div class='add'>+</div><div class='add'>+		if (num_psv == 1)</div><div class='add'>+			return true;</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='ctx'> 	default:</div><div class='ctx'> 		return false;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -621,6 +893,7 @@ static bool devx_is_obj_modify_cmd(const void *in)</div><div class='ctx'> 	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:</div><div class='ctx'> 	case MLX5_CMD_OP_RST2INIT_QP:</div><div class='ctx'> 	case MLX5_CMD_OP_INIT2RTR_QP:</div><div class='add'>+	case MLX5_CMD_OP_INIT2INIT_QP:</div><div class='ctx'> 	case MLX5_CMD_OP_RTR2RTS_QP:</div><div class='ctx'> 	case MLX5_CMD_OP_RTS2RTS_QP:</div><div class='ctx'> 	case MLX5_CMD_OP_SQERR2RTS_QP:</div><div class='hunk'>@@ -631,6 +904,8 @@ static bool devx_is_obj_modify_cmd(const void *in)</div><div class='ctx'> 	case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:</div><div class='ctx'> 	case MLX5_CMD_OP_ARM_XRQ:</div><div class='ctx'> 	case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:</div><div class='add'>+	case MLX5_CMD_OP_RELEASE_XRQ_ERROR:</div><div class='add'>+	case MLX5_CMD_OP_MODIFY_XRQ:</div><div class='ctx'> 		return true;</div><div class='ctx'> 	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:</div><div class='ctx'> 	{</div><div class='hunk'>@@ -689,6 +964,7 @@ static bool devx_is_whitelist_cmd(void *in)</div><div class='ctx'> 	case MLX5_CMD_OP_QUERY_HCA_CAP:</div><div class='ctx'> 	case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:</div><div class='ctx'> 	case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:</div><div class='add'>+	case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:</div><div class='ctx'> 		return true;</div><div class='ctx'> 	default:</div><div class='ctx'> 		return false;</div><div class='hunk'>@@ -715,12 +991,16 @@ static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in)</div><div class='ctx'> </div><div class='ctx'> 	return c-&gt;devx_uid;</div><div class='ctx'> }</div><div class='del'>-static bool devx_is_general_cmd(void *in)</div><div class='add'>+</div><div class='add'>+static bool devx_is_general_cmd(void *in, struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);</div><div class='ctx'> </div><div class='del'>-	if (opcode &gt;= MLX5_CMD_OP_GENERAL_START &amp;&amp;</div><div class='del'>-	    opcode &lt; MLX5_CMD_OP_GENERAL_END)</div><div class='add'>+	/* Pass all cmds for vhca_tunnel as general, tracking is done in FW */</div><div class='add'>+	if ((MLX5_CAP_GEN_64(dev-&gt;mdev, vhca_tunnel_commands) &amp;&amp;</div><div class='add'>+	     MLX5_GET(general_obj_in_cmd_hdr, in, vhca_tunnel_id)) ||</div><div class='add'>+	    (opcode &gt;= MLX5_CMD_OP_GENERAL_START &amp;&amp;</div><div class='add'>+	     opcode &lt; MLX5_CMD_OP_GENERAL_END))</div><div class='ctx'> 		return true;</div><div class='ctx'> </div><div class='ctx'> 	switch (opcode) {</div><div class='hunk'>@@ -739,6 +1019,8 @@ static bool devx_is_general_cmd(void *in)</div><div class='ctx'> 	case MLX5_CMD_OP_QUERY_CONG_STATUS:</div><div class='ctx'> 	case MLX5_CMD_OP_QUERY_CONG_PARAMS:</div><div class='ctx'> 	case MLX5_CMD_OP_QUERY_CONG_STATISTICS:</div><div class='add'>+	case MLX5_CMD_OP_QUERY_LAG:</div><div class='add'>+	case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:</div><div class='ctx'> 		return true;</div><div class='ctx'> 	default:</div><div class='ctx'> 		return false;</div><div class='hunk'>@@ -752,7 +1034,6 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)(</div><div class='ctx'> 	struct mlx5_ib_dev *dev;</div><div class='ctx'> 	int user_vector;</div><div class='ctx'> 	int dev_eqn;</div><div class='del'>-	unsigned int irqn;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	if (uverbs_copy_from(&amp;user_vector, attrs,</div><div class='hunk'>@@ -764,7 +1045,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)(</div><div class='ctx'> 		return PTR_ERR(c);</div><div class='ctx'> 	dev = to_mdev(c-&gt;ibucontext.device);</div><div class='ctx'> </div><div class='del'>-	err = mlx5_vector2eqn(dev-&gt;mdev, user_vector, &amp;dev_eqn, &amp;irqn);</div><div class='add'>+	err = mlx5_comp_eqn_get(dev-&gt;mdev, user_vector, &amp;dev_eqn);</div><div class='ctx'> 	if (err &lt; 0)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='hunk'>@@ -833,7 +1114,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)(</div><div class='ctx'> 	int cmd_out_len = uverbs_attr_get_len(attrs,</div><div class='ctx'> 					MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT);</div><div class='ctx'> 	void *cmd_out;</div><div class='del'>-	int err;</div><div class='add'>+	int err, err2;</div><div class='ctx'> 	int uid;</div><div class='ctx'> </div><div class='ctx'> 	c = devx_ufile2uctx(attrs);</div><div class='hunk'>@@ -846,7 +1127,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)(</div><div class='ctx'> 		return uid;</div><div class='ctx'> </div><div class='ctx'> 	/* Only white list of some general HCA commands are allowed for this method. */</div><div class='del'>-	if (!devx_is_general_cmd(cmd_in))</div><div class='add'>+	if (!devx_is_general_cmd(cmd_in, dev))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	cmd_out = uverbs_zalloc(attrs, cmd_out_len);</div><div class='hunk'>@@ -854,171 +1135,205 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)(</div><div class='ctx'> 		return PTR_ERR(cmd_out);</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);</div><div class='del'>-	err = mlx5_cmd_exec(dev-&gt;mdev, cmd_in,</div><div class='del'>-			    uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN),</div><div class='del'>-			    cmd_out, cmd_out_len);</div><div class='del'>-	if (err)</div><div class='add'>+	err = mlx5_cmd_do(dev-&gt;mdev, cmd_in,</div><div class='add'>+			  uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN),</div><div class='add'>+			  cmd_out, cmd_out_len);</div><div class='add'>+	if (err &amp;&amp; err != -EREMOTEIO)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='del'>-	return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out,</div><div class='add'>+	err2 = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out,</div><div class='ctx'> 			      cmd_out_len);</div><div class='add'>+</div><div class='add'>+	return err2 ?: err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void devx_obj_build_destroy_cmd(void *in, void *out, void *din,</div><div class='ctx'> 				       u32 *dinlen,</div><div class='ctx'> 				       u32 *obj_id)</div><div class='ctx'> {</div><div class='del'>-	u16 obj_type = MLX5_GET(general_obj_in_cmd_hdr, in, obj_type);</div><div class='add'>+	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);</div><div class='ctx'> 	u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid);</div><div class='ctx'> </div><div class='del'>-	*obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);</div><div class='add'>+	*obj_id = devx_get_created_obj_id(in, out, opcode);</div><div class='ctx'> 	*dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr);</div><div class='del'>-</div><div class='del'>-	MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id);</div><div class='ctx'> 	MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid);</div><div class='ctx'> </div><div class='del'>-	switch (MLX5_GET(general_obj_in_cmd_hdr, in, opcode)) {</div><div class='add'>+	switch (opcode) {</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:</div><div class='ctx'> 		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, obj_type, obj_type);</div><div class='add'>+		MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id);</div><div class='add'>+		MLX5_SET(general_obj_in_cmd_hdr, din, obj_type,</div><div class='add'>+			 MLX5_GET(general_obj_in_cmd_hdr, in, obj_type));</div><div class='ctx'> 		break;</div><div class='ctx'> </div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_UMEM:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode,</div><div class='add'>+		MLX5_SET(destroy_umem_in, din, opcode,</div><div class='ctx'> 			 MLX5_CMD_OP_DESTROY_UMEM);</div><div class='add'>+		MLX5_SET(destroy_umem_in, din, umem_id, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_MKEY:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_MKEY);</div><div class='add'>+		MLX5_SET(destroy_mkey_in, din, opcode,</div><div class='add'>+			 MLX5_CMD_OP_DESTROY_MKEY);</div><div class='add'>+		MLX5_SET(destroy_mkey_in, din, mkey_index, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_CQ:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_CQ);</div><div class='add'>+		MLX5_SET(destroy_cq_in, din, opcode, MLX5_CMD_OP_DESTROY_CQ);</div><div class='add'>+		MLX5_SET(destroy_cq_in, din, cqn, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_ALLOC_PD:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_PD);</div><div class='add'>+		MLX5_SET(dealloc_pd_in, din, opcode, MLX5_CMD_OP_DEALLOC_PD);</div><div class='add'>+		MLX5_SET(dealloc_pd_in, din, pd, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode,</div><div class='add'>+		MLX5_SET(dealloc_transport_domain_in, din, opcode,</div><div class='ctx'> 			 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);</div><div class='add'>+		MLX5_SET(dealloc_transport_domain_in, din, transport_domain,</div><div class='add'>+			 *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_RMP:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RMP);</div><div class='add'>+		MLX5_SET(destroy_rmp_in, din, opcode, MLX5_CMD_OP_DESTROY_RMP);</div><div class='add'>+		MLX5_SET(destroy_rmp_in, din, rmpn, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_SQ:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SQ);</div><div class='add'>+		MLX5_SET(destroy_sq_in, din, opcode, MLX5_CMD_OP_DESTROY_SQ);</div><div class='add'>+		MLX5_SET(destroy_sq_in, din, sqn, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_RQ:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQ);</div><div class='add'>+		MLX5_SET(destroy_rq_in, din, opcode, MLX5_CMD_OP_DESTROY_RQ);</div><div class='add'>+		MLX5_SET(destroy_rq_in, din, rqn, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_RQT:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQT);</div><div class='add'>+		MLX5_SET(destroy_rqt_in, din, opcode, MLX5_CMD_OP_DESTROY_RQT);</div><div class='add'>+		MLX5_SET(destroy_rqt_in, din, rqtn, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_TIR:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIR);</div><div class='add'>+		MLX5_SET(destroy_tir_in, din, opcode, MLX5_CMD_OP_DESTROY_TIR);</div><div class='add'>+		MLX5_SET(destroy_tir_in, din, tirn, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_TIS:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIS);</div><div class='add'>+		MLX5_SET(destroy_tis_in, din, opcode, MLX5_CMD_OP_DESTROY_TIS);</div><div class='add'>+		MLX5_SET(destroy_tis_in, din, tisn, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_ALLOC_Q_COUNTER:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode,</div><div class='add'>+		MLX5_SET(dealloc_q_counter_in, din, opcode,</div><div class='ctx'> 			 MLX5_CMD_OP_DEALLOC_Q_COUNTER);</div><div class='add'>+		MLX5_SET(dealloc_q_counter_in, din, counter_set_id, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_FLOW_TABLE:</div><div class='ctx'> 		*dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in);</div><div class='del'>-		*obj_id = MLX5_GET(create_flow_table_out, out, table_id);</div><div class='ctx'> 		MLX5_SET(destroy_flow_table_in, din, other_vport,</div><div class='ctx'> 			 MLX5_GET(create_flow_table_in,  in, other_vport));</div><div class='ctx'> 		MLX5_SET(destroy_flow_table_in, din, vport_number,</div><div class='ctx'> 			 MLX5_GET(create_flow_table_in,  in, vport_number));</div><div class='add'>+		MLX5_SET(destroy_flow_table_in, din, other_eswitch,</div><div class='add'>+			 MLX5_GET(create_flow_table_in,  in, other_eswitch));</div><div class='add'>+		MLX5_SET(destroy_flow_table_in, din, eswitch_owner_vhca_id,</div><div class='add'>+			 MLX5_GET(create_flow_table_in, in,</div><div class='add'>+				  eswitch_owner_vhca_id));</div><div class='ctx'> 		MLX5_SET(destroy_flow_table_in, din, table_type,</div><div class='ctx'> 			 MLX5_GET(create_flow_table_in,  in, table_type));</div><div class='ctx'> 		MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id);</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode,</div><div class='add'>+		MLX5_SET(destroy_flow_table_in, din, opcode,</div><div class='ctx'> 			 MLX5_CMD_OP_DESTROY_FLOW_TABLE);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_FLOW_GROUP:</div><div class='ctx'> 		*dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in);</div><div class='del'>-		*obj_id = MLX5_GET(create_flow_group_out, out, group_id);</div><div class='ctx'> 		MLX5_SET(destroy_flow_group_in, din, other_vport,</div><div class='ctx'> 			 MLX5_GET(create_flow_group_in, in, other_vport));</div><div class='ctx'> 		MLX5_SET(destroy_flow_group_in, din, vport_number,</div><div class='ctx'> 			 MLX5_GET(create_flow_group_in, in, vport_number));</div><div class='add'>+		MLX5_SET(destroy_flow_group_in, din, other_eswitch,</div><div class='add'>+			 MLX5_GET(create_flow_group_in, in, other_eswitch));</div><div class='add'>+		MLX5_SET(destroy_flow_group_in, din, eswitch_owner_vhca_id,</div><div class='add'>+			 MLX5_GET(create_flow_group_in, in,</div><div class='add'>+				  eswitch_owner_vhca_id));</div><div class='ctx'> 		MLX5_SET(destroy_flow_group_in, din, table_type,</div><div class='ctx'> 			 MLX5_GET(create_flow_group_in, in, table_type));</div><div class='ctx'> 		MLX5_SET(destroy_flow_group_in, din, table_id,</div><div class='ctx'> 			 MLX5_GET(create_flow_group_in, in, table_id));</div><div class='ctx'> 		MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id);</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode,</div><div class='add'>+		MLX5_SET(destroy_flow_group_in, din, opcode,</div><div class='ctx'> 			 MLX5_CMD_OP_DESTROY_FLOW_GROUP);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:</div><div class='ctx'> 		*dinlen = MLX5_ST_SZ_BYTES(delete_fte_in);</div><div class='del'>-		*obj_id = MLX5_GET(set_fte_in, in, flow_index);</div><div class='ctx'> 		MLX5_SET(delete_fte_in, din, other_vport,</div><div class='ctx'> 			 MLX5_GET(set_fte_in,  in, other_vport));</div><div class='ctx'> 		MLX5_SET(delete_fte_in, din, vport_number,</div><div class='ctx'> 			 MLX5_GET(set_fte_in, in, vport_number));</div><div class='add'>+		MLX5_SET(delete_fte_in, din, other_eswitch,</div><div class='add'>+			 MLX5_GET(set_fte_in,  in, other_eswitch));</div><div class='add'>+		MLX5_SET(delete_fte_in, din, eswitch_owner_vhca_id,</div><div class='add'>+			 MLX5_GET(set_fte_in, in, eswitch_owner_vhca_id));</div><div class='ctx'> 		MLX5_SET(delete_fte_in, din, table_type,</div><div class='ctx'> 			 MLX5_GET(set_fte_in, in, table_type));</div><div class='ctx'> 		MLX5_SET(delete_fte_in, din, table_id,</div><div class='ctx'> 			 MLX5_GET(set_fte_in, in, table_id));</div><div class='ctx'> 		MLX5_SET(delete_fte_in, din, flow_index, *obj_id);</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode,</div><div class='add'>+		MLX5_SET(delete_fte_in, din, opcode,</div><div class='ctx'> 			 MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode,</div><div class='add'>+		MLX5_SET(dealloc_flow_counter_in, din, opcode,</div><div class='ctx'> 			 MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);</div><div class='add'>+		MLX5_SET(dealloc_flow_counter_in, din, flow_counter_id,</div><div class='add'>+			 *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode,</div><div class='add'>+		MLX5_SET(dealloc_packet_reformat_context_in, din, opcode,</div><div class='ctx'> 			 MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);</div><div class='add'>+		MLX5_SET(dealloc_packet_reformat_context_in, din,</div><div class='add'>+			 packet_reformat_id, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode,</div><div class='add'>+		MLX5_SET(dealloc_modify_header_context_in, din, opcode,</div><div class='ctx'> 			 MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);</div><div class='add'>+		MLX5_SET(dealloc_modify_header_context_in, din,</div><div class='add'>+			 modify_header_id, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:</div><div class='ctx'> 		*dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in);</div><div class='del'>-		*obj_id = MLX5_GET(create_scheduling_element_out, out,</div><div class='del'>-				   scheduling_element_id);</div><div class='ctx'> 		MLX5_SET(destroy_scheduling_element_in, din,</div><div class='ctx'> 			 scheduling_hierarchy,</div><div class='ctx'> 			 MLX5_GET(create_scheduling_element_in, in,</div><div class='ctx'> 				  scheduling_hierarchy));</div><div class='ctx'> 		MLX5_SET(destroy_scheduling_element_in, din,</div><div class='ctx'> 			 scheduling_element_id, *obj_id);</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode,</div><div class='add'>+		MLX5_SET(destroy_scheduling_element_in, din, opcode,</div><div class='ctx'> 			 MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:</div><div class='ctx'> 		*dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in);</div><div class='del'>-		*obj_id = MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port);</div><div class='ctx'> 		MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id);</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode,</div><div class='add'>+		MLX5_SET(delete_vxlan_udp_dport_in, din, opcode,</div><div class='ctx'> 			 MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:</div><div class='ctx'> 		*dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in);</div><div class='del'>-		*obj_id = MLX5_GET(set_l2_table_entry_in, in, table_index);</div><div class='ctx'> 		MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id);</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode,</div><div class='add'>+		MLX5_SET(delete_l2_table_entry_in, din, opcode,</div><div class='ctx'> 			 MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_QP:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_QP);</div><div class='add'>+		MLX5_SET(destroy_qp_in, din, opcode, MLX5_CMD_OP_DESTROY_QP);</div><div class='add'>+		MLX5_SET(destroy_qp_in, din, qpn, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_SRQ:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SRQ);</div><div class='add'>+		MLX5_SET(destroy_srq_in, din, opcode, MLX5_CMD_OP_DESTROY_SRQ);</div><div class='add'>+		MLX5_SET(destroy_srq_in, din, srqn, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_XRC_SRQ:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode,</div><div class='add'>+		MLX5_SET(destroy_xrc_srq_in, din, opcode,</div><div class='ctx'> 			 MLX5_CMD_OP_DESTROY_XRC_SRQ);</div><div class='add'>+		MLX5_SET(destroy_xrc_srq_in, din, xrc_srqn, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_DCT:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_DCT);</div><div class='add'>+		MLX5_SET(destroy_dct_in, din, opcode, MLX5_CMD_OP_DESTROY_DCT);</div><div class='add'>+		MLX5_SET(destroy_dct_in, din, dctn, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_CREATE_XRQ:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_XRQ);</div><div class='add'>+		MLX5_SET(destroy_xrq_in, din, opcode, MLX5_CMD_OP_DESTROY_XRQ);</div><div class='add'>+		MLX5_SET(destroy_xrq_in, din, xrqn, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_ATTACH_TO_MCG:</div><div class='ctx'> 		*dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in);</div><div class='hunk'>@@ -1027,10 +1342,19 @@ static void devx_obj_build_destroy_cmd(void *in, void *out, void *din,</div><div class='ctx'> 		memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid),</div><div class='ctx'> 		       MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid),</div><div class='ctx'> 		       MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid));</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DETACH_FROM_MCG);</div><div class='add'>+		MLX5_SET(detach_from_mcg_in, din, opcode,</div><div class='add'>+			 MLX5_CMD_OP_DETACH_FROM_MCG);</div><div class='add'>+		MLX5_SET(detach_from_mcg_in, din, qpn, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_ALLOC_XRCD:</div><div class='del'>-		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_XRCD);</div><div class='add'>+		MLX5_SET(dealloc_xrcd_in, din, opcode,</div><div class='add'>+			 MLX5_CMD_OP_DEALLOC_XRCD);</div><div class='add'>+		MLX5_SET(dealloc_xrcd_in, din, xrcd, *obj_id);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_CMD_OP_CREATE_PSV:</div><div class='add'>+		MLX5_SET(destroy_psv_in, din, opcode,</div><div class='add'>+			 MLX5_CMD_OP_DESTROY_PSV);</div><div class='add'>+		MLX5_SET(destroy_psv_in, din, psvn, *obj_id);</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		/* The entry must match to one of the devx_is_obj_create_cmd */</div><div class='hunk'>@@ -1043,24 +1367,19 @@ static int devx_handle_mkey_indirect(struct devx_obj *obj,</div><div class='ctx'> 				     struct mlx5_ib_dev *dev,</div><div class='ctx'> 				     void *in, void *out)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_devx_mr *devx_mr = &amp;obj-&gt;devx_mr;</div><div class='del'>-	struct mlx5_core_mkey *mkey;</div><div class='add'>+	struct mlx5_ib_mkey *mkey = &amp;obj-&gt;mkey;</div><div class='ctx'> 	void *mkc;</div><div class='ctx'> 	u8 key;</div><div class='ctx'> </div><div class='del'>-	mkey = &amp;devx_mr-&gt;mmkey;</div><div class='ctx'> 	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);</div><div class='ctx'> 	key = MLX5_GET(mkc, mkc, mkey_7_0);</div><div class='ctx'> 	mkey-&gt;key = mlx5_idx_to_mkey(</div><div class='ctx'> 			MLX5_GET(create_mkey_out, out, mkey_index)) | key;</div><div class='ctx'> 	mkey-&gt;type = MLX5_MKEY_INDIRECT_DEVX;</div><div class='del'>-	mkey-&gt;iova = MLX5_GET64(mkc, mkc, start_addr);</div><div class='del'>-	mkey-&gt;size = MLX5_GET64(mkc, mkc, len);</div><div class='del'>-	mkey-&gt;pd = MLX5_GET(mkc, mkc, pd);</div><div class='del'>-	devx_mr-&gt;ndescs = MLX5_GET(mkc, mkc, translations_octword_size);</div><div class='add'>+	mkey-&gt;ndescs = MLX5_GET(mkc, mkc, translations_octword_size);</div><div class='add'>+	init_waitqueue_head(&amp;mkey-&gt;wait);</div><div class='ctx'> </div><div class='del'>-	return xa_err(xa_store(&amp;dev-&gt;mdev-&gt;priv.mkey_table,</div><div class='del'>-			       mlx5_base_mkey(mkey-&gt;key), mkey, GFP_KERNEL));</div><div class='add'>+	return mlx5r_store_odp_mkey(dev, mkey);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,</div><div class='hunk'>@@ -1089,30 +1408,40 @@ static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);</div><div class='add'>+	/* TPH is not allowed to bypass the regular kernel's verbs flow */</div><div class='add'>+	MLX5_SET(mkc, mkc, pcie_tph_en, 0);</div><div class='add'>+	MLX5_SET(mkc, mkc, pcie_tph_steering_tag_index,</div><div class='add'>+		 MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void devx_free_indirect_mkey(struct rcu_head *rcu)</div><div class='add'>+static void devx_cleanup_subscription(struct mlx5_ib_dev *dev,</div><div class='add'>+				      struct devx_event_subscription *sub)</div><div class='ctx'> {</div><div class='del'>-	kfree(container_of(rcu, struct devx_obj, devx_mr.rcu));</div><div class='del'>-}</div><div class='add'>+	struct devx_event *event;</div><div class='add'>+	struct devx_obj_event *xa_val_level2;</div><div class='ctx'> </div><div class='del'>-/* This function to delete from the radix tree needs to be called before</div><div class='del'>- * destroying the underlying mkey. Otherwise a race might occur in case that</div><div class='del'>- * other thread will get the same mkey before this one will be deleted,</div><div class='del'>- * in that case it will fail via inserting to the tree its own data.</div><div class='del'>- *</div><div class='del'>- * Note:</div><div class='del'>- * An error in the destroy is not expected unless there is some other indirect</div><div class='del'>- * mkey which points to this one. In a kernel cleanup flow it will be just</div><div class='del'>- * destroyed in the iterative destruction call. In a user flow, in case</div><div class='del'>- * the application didn't close in the expected order it's its own problem,</div><div class='del'>- * the mkey won't be part of the tree, in both cases the kernel is safe.</div><div class='del'>- */</div><div class='del'>-static void devx_cleanup_mkey(struct devx_obj *obj)</div><div class='del'>-{</div><div class='del'>-	xa_erase(&amp;obj-&gt;mdev-&gt;priv.mkey_table,</div><div class='del'>-		 mlx5_base_mkey(obj-&gt;devx_mr.mmkey.key));</div><div class='add'>+	if (sub-&gt;is_cleaned)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	sub-&gt;is_cleaned = 1;</div><div class='add'>+	list_del_rcu(&amp;sub-&gt;xa_list);</div><div class='add'>+</div><div class='add'>+	if (list_empty(&amp;sub-&gt;obj_list))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	list_del_rcu(&amp;sub-&gt;obj_list);</div><div class='add'>+	/* check whether key level 1 for this obj_sub_list is empty */</div><div class='add'>+	event = xa_load(&amp;dev-&gt;devx_event_table.event_xa,</div><div class='add'>+			sub-&gt;xa_key_level1);</div><div class='add'>+	WARN_ON(!event);</div><div class='add'>+</div><div class='add'>+	xa_val_level2 = xa_load(&amp;event-&gt;object_ids, sub-&gt;xa_key_level2);</div><div class='add'>+	if (list_empty(&amp;xa_val_level2-&gt;obj_sub_list)) {</div><div class='add'>+		xa_erase(&amp;event-&gt;object_ids,</div><div class='add'>+			 sub-&gt;xa_key_level2);</div><div class='add'>+		kfree_rcu(xa_val_level2, rcu);</div><div class='add'>+	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int devx_obj_cleanup(struct ib_uobject *uobject,</div><div class='hunk'>@@ -1120,33 +1449,78 @@ static int devx_obj_cleanup(struct ib_uobject *uobject,</div><div class='ctx'> 			    struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];</div><div class='add'>+	struct mlx5_devx_event_table *devx_event_table;</div><div class='ctx'> 	struct devx_obj *obj = uobject-&gt;object;</div><div class='add'>+	struct devx_event_subscription *sub_entry, *tmp;</div><div class='add'>+	struct mlx5_ib_dev *dev;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	if (obj-&gt;flags &amp; DEVX_OBJ_FLAGS_INDIRECT_MKEY)</div><div class='del'>-		devx_cleanup_mkey(obj);</div><div class='del'>-</div><div class='del'>-	if (obj-&gt;flags &amp; DEVX_OBJ_FLAGS_DCT)</div><div class='del'>-		ret = mlx5_core_destroy_dct(obj-&gt;mdev, &amp;obj-&gt;core_dct);</div><div class='add'>+	dev = mlx5_udata_to_mdev(&amp;attrs-&gt;driver_udata);</div><div class='add'>+	if (obj-&gt;flags &amp; DEVX_OBJ_FLAGS_INDIRECT_MKEY &amp;&amp;</div><div class='add'>+	    xa_erase(&amp;obj-&gt;ib_dev-&gt;odp_mkeys,</div><div class='add'>+		     mlx5_base_mkey(obj-&gt;mkey.key)))</div><div class='add'>+		/*</div><div class='add'>+		 * The pagefault_single_data_segment() does commands against</div><div class='add'>+		 * the mmkey, we must wait for that to stop before freeing the</div><div class='add'>+		 * mkey, as another allocation could get the same mkey #.</div><div class='add'>+		 */</div><div class='add'>+		mlx5r_deref_wait_odp_mkey(&amp;obj-&gt;mkey);</div><div class='add'>+</div><div class='add'>+	if (obj-&gt;flags &amp; DEVX_OBJ_FLAGS_HW_FREED)</div><div class='add'>+		ret = 0;</div><div class='add'>+	else if (obj-&gt;flags &amp; DEVX_OBJ_FLAGS_DCT)</div><div class='add'>+		ret = mlx5_core_destroy_dct(obj-&gt;ib_dev, &amp;obj-&gt;core_dct);</div><div class='add'>+	else if (obj-&gt;flags &amp; DEVX_OBJ_FLAGS_CQ)</div><div class='add'>+		ret = mlx5_core_destroy_cq(obj-&gt;ib_dev-&gt;mdev, &amp;obj-&gt;core_cq);</div><div class='ctx'> 	else</div><div class='del'>-		ret = mlx5_cmd_exec(obj-&gt;mdev, obj-&gt;dinbox, obj-&gt;dinlen, out,</div><div class='del'>-				    sizeof(out));</div><div class='del'>-	if (ib_is_destroy_retryable(ret, why, uobject))</div><div class='add'>+		ret = mlx5_cmd_exec(obj-&gt;ib_dev-&gt;mdev, obj-&gt;dinbox,</div><div class='add'>+				    obj-&gt;dinlen, out, sizeof(out));</div><div class='add'>+	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	if (obj-&gt;flags &amp; DEVX_OBJ_FLAGS_INDIRECT_MKEY) {</div><div class='del'>-		struct mlx5_ib_dev *dev =</div><div class='del'>-			mlx5_udata_to_mdev(&amp;attrs-&gt;driver_udata);</div><div class='add'>+	devx_event_table = &amp;dev-&gt;devx_event_table;</div><div class='ctx'> </div><div class='del'>-		call_srcu(&amp;dev-&gt;mr_srcu, &amp;obj-&gt;devx_mr.rcu,</div><div class='del'>-			  devx_free_indirect_mkey);</div><div class='del'>-		return ret;</div><div class='del'>-	}</div><div class='add'>+	mutex_lock(&amp;devx_event_table-&gt;event_xa_lock);</div><div class='add'>+	list_for_each_entry_safe(sub_entry, tmp, &amp;obj-&gt;event_sub, obj_list)</div><div class='add'>+		devx_cleanup_subscription(dev, sub_entry);</div><div class='add'>+	mutex_unlock(&amp;devx_event_table-&gt;event_xa_lock);</div><div class='ctx'> </div><div class='ctx'> 	kfree(obj);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void devx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe)</div><div class='add'>+{</div><div class='add'>+	struct devx_obj *obj = container_of(mcq, struct devx_obj, core_cq);</div><div class='add'>+	struct mlx5_devx_event_table *table;</div><div class='add'>+	struct devx_event *event;</div><div class='add'>+	struct devx_obj_event *obj_event;</div><div class='add'>+	u32 obj_id = mcq-&gt;cqn;</div><div class='add'>+</div><div class='add'>+	table = &amp;obj-&gt;ib_dev-&gt;devx_event_table;</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	event = xa_load(&amp;table-&gt;event_xa, MLX5_EVENT_TYPE_COMP);</div><div class='add'>+	if (!event)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	obj_event = xa_load(&amp;event-&gt;object_ids, obj_id);</div><div class='add'>+	if (!obj_event)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	dispatch_event_fd(&amp;obj_event-&gt;obj_sub_list, eqe);</div><div class='add'>+out:</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool is_apu_cq(struct mlx5_ib_dev *dev, const void *in)</div><div class='add'>+{</div><div class='add'>+	if (!MLX5_CAP_GEN(dev-&gt;mdev, apu) ||</div><div class='add'>+	    !MLX5_GET(cqc, MLX5_ADDR_OF(create_cq_in, in, cq_context), apu_cq))</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(</div><div class='ctx'> 	struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='hunk'>@@ -1164,11 +1538,14 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(</div><div class='ctx'> 	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];</div><div class='ctx'> 	struct devx_obj *obj;</div><div class='ctx'> 	u16 obj_type = 0;</div><div class='del'>-	int err;</div><div class='add'>+	int err, err2 = 0;</div><div class='ctx'> 	int uid;</div><div class='ctx'> 	u32 obj_id;</div><div class='ctx'> 	u16 opcode;</div><div class='ctx'> </div><div class='add'>+	if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='ctx'> 	uid = devx_get_uid(c, cmd_in);</div><div class='ctx'> 	if (uid &lt; 0)</div><div class='ctx'> 		return uid;</div><div class='hunk'>@@ -1195,53 +1572,74 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(</div><div class='ctx'> </div><div class='ctx'> 	if (opcode == MLX5_CMD_OP_CREATE_DCT) {</div><div class='ctx'> 		obj-&gt;flags |= DEVX_OBJ_FLAGS_DCT;</div><div class='del'>-		err = mlx5_core_create_dct(dev-&gt;mdev, &amp;obj-&gt;core_dct,</div><div class='del'>-					   cmd_in, cmd_in_len,</div><div class='del'>-					   cmd_out, cmd_out_len);</div><div class='add'>+		err = mlx5_core_create_dct(dev, &amp;obj-&gt;core_dct, cmd_in,</div><div class='add'>+					   cmd_in_len, cmd_out, cmd_out_len);</div><div class='add'>+	} else if (opcode == MLX5_CMD_OP_CREATE_CQ &amp;&amp;</div><div class='add'>+		   !is_apu_cq(dev, cmd_in)) {</div><div class='add'>+		obj-&gt;flags |= DEVX_OBJ_FLAGS_CQ;</div><div class='add'>+		obj-&gt;core_cq.comp = devx_cq_comp;</div><div class='add'>+		err = mlx5_create_cq(dev-&gt;mdev, &amp;obj-&gt;core_cq,</div><div class='add'>+				     cmd_in, cmd_in_len, cmd_out,</div><div class='add'>+				     cmd_out_len);</div><div class='ctx'> 	} else {</div><div class='del'>-		err = mlx5_cmd_exec(dev-&gt;mdev, cmd_in,</div><div class='del'>-				    cmd_in_len,</div><div class='del'>-				    cmd_out, cmd_out_len);</div><div class='add'>+		err = mlx5_cmd_do(dev-&gt;mdev, cmd_in, cmd_in_len,</div><div class='add'>+				  cmd_out, cmd_out_len);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (err == -EREMOTEIO)</div><div class='add'>+		err2 = uverbs_copy_to(attrs,</div><div class='add'>+				      MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,</div><div class='add'>+				      cmd_out, cmd_out_len);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto obj_free;</div><div class='ctx'> </div><div class='add'>+	if (opcode == MLX5_CMD_OP_ALLOC_FLOW_COUNTER) {</div><div class='add'>+		u32 bulk = MLX5_GET(alloc_flow_counter_in,</div><div class='add'>+				    cmd_in,</div><div class='add'>+				    flow_counter_bulk_log_size);</div><div class='add'>+</div><div class='add'>+		if (bulk)</div><div class='add'>+			bulk = 1 &lt;&lt; bulk;</div><div class='add'>+		else</div><div class='add'>+			bulk = 128UL * MLX5_GET(alloc_flow_counter_in,</div><div class='add'>+						cmd_in,</div><div class='add'>+						flow_counter_bulk);</div><div class='add'>+		obj-&gt;flow_counter_bulk_size = bulk;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	uobj-&gt;object = obj;</div><div class='del'>-	obj-&gt;mdev = dev-&gt;mdev;</div><div class='add'>+	INIT_LIST_HEAD(&amp;obj-&gt;event_sub);</div><div class='add'>+	obj-&gt;ib_dev = dev;</div><div class='ctx'> 	devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj-&gt;dinbox, &amp;obj-&gt;dinlen,</div><div class='ctx'> 				   &amp;obj_id);</div><div class='ctx'> 	WARN_ON(obj-&gt;dinlen &gt; MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32));</div><div class='ctx'> </div><div class='del'>-	if (obj-&gt;flags &amp; DEVX_OBJ_FLAGS_INDIRECT_MKEY) {</div><div class='del'>-		err = devx_handle_mkey_indirect(obj, dev, cmd_in, cmd_out);</div><div class='del'>-		if (err)</div><div class='del'>-			goto obj_destroy;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, cmd_out, cmd_out_len);</div><div class='ctx'> 	if (err)</div><div class='del'>-		goto err_copy;</div><div class='add'>+		goto obj_destroy;</div><div class='ctx'> </div><div class='ctx'> 	if (opcode == MLX5_CMD_OP_CREATE_GENERAL_OBJECT)</div><div class='ctx'> 		obj_type = MLX5_GET(general_obj_in_cmd_hdr, cmd_in, obj_type);</div><div class='del'>-</div><div class='ctx'> 	obj-&gt;obj_id = get_enc_obj_id(opcode | obj_type &lt;&lt; 16, obj_id);</div><div class='ctx'> </div><div class='add'>+	if (obj-&gt;flags &amp; DEVX_OBJ_FLAGS_INDIRECT_MKEY) {</div><div class='add'>+		err = devx_handle_mkey_indirect(obj, dev, cmd_in, cmd_out);</div><div class='add'>+		if (err)</div><div class='add'>+			goto obj_destroy;</div><div class='add'>+	}</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-err_copy:</div><div class='del'>-	if (obj-&gt;flags &amp; DEVX_OBJ_FLAGS_INDIRECT_MKEY)</div><div class='del'>-		devx_cleanup_mkey(obj);</div><div class='ctx'> obj_destroy:</div><div class='ctx'> 	if (obj-&gt;flags &amp; DEVX_OBJ_FLAGS_DCT)</div><div class='del'>-		mlx5_core_destroy_dct(obj-&gt;mdev, &amp;obj-&gt;core_dct);</div><div class='add'>+		mlx5_core_destroy_dct(obj-&gt;ib_dev, &amp;obj-&gt;core_dct);</div><div class='add'>+	else if (obj-&gt;flags &amp; DEVX_OBJ_FLAGS_CQ)</div><div class='add'>+		mlx5_core_destroy_cq(obj-&gt;ib_dev-&gt;mdev, &amp;obj-&gt;core_cq);</div><div class='ctx'> 	else</div><div class='del'>-		mlx5_cmd_exec(obj-&gt;mdev, obj-&gt;dinbox, obj-&gt;dinlen, out,</div><div class='add'>+		mlx5_cmd_exec(obj-&gt;ib_dev-&gt;mdev, obj-&gt;dinbox, obj-&gt;dinlen, out,</div><div class='ctx'> 			      sizeof(out));</div><div class='ctx'> obj_free:</div><div class='ctx'> 	kfree(obj);</div><div class='del'>-	return err;</div><div class='add'>+	return err2 ?: err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(</div><div class='hunk'>@@ -1256,9 +1654,12 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(</div><div class='ctx'> 		&amp;attrs-&gt;driver_udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='ctx'> 	struct mlx5_ib_dev *mdev = to_mdev(c-&gt;ibucontext.device);</div><div class='ctx'> 	void *cmd_out;</div><div class='del'>-	int err;</div><div class='add'>+	int err, err2;</div><div class='ctx'> 	int uid;</div><div class='ctx'> </div><div class='add'>+	if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='ctx'> 	uid = devx_get_uid(c, cmd_in);</div><div class='ctx'> 	if (uid &lt; 0)</div><div class='ctx'> 		return uid;</div><div class='hunk'>@@ -1276,14 +1677,16 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(</div><div class='ctx'> 	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);</div><div class='ctx'> 	devx_set_umem_valid(cmd_in);</div><div class='ctx'> </div><div class='del'>-	err = mlx5_cmd_exec(mdev-&gt;mdev, cmd_in,</div><div class='del'>-			    uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN),</div><div class='del'>-			    cmd_out, cmd_out_len);</div><div class='del'>-	if (err)</div><div class='add'>+	err = mlx5_cmd_do(mdev-&gt;mdev, cmd_in,</div><div class='add'>+			  uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN),</div><div class='add'>+			  cmd_out, cmd_out_len);</div><div class='add'>+	if (err &amp;&amp; err != -EREMOTEIO)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='del'>-	return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,</div><div class='add'>+	err2 = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,</div><div class='ctx'> 			      cmd_out, cmd_out_len);</div><div class='add'>+</div><div class='add'>+	return err2 ?: err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(</div><div class='hunk'>@@ -1297,10 +1700,13 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(</div><div class='ctx'> 	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(</div><div class='ctx'> 		&amp;attrs-&gt;driver_udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='ctx'> 	void *cmd_out;</div><div class='del'>-	int err;</div><div class='add'>+	int err, err2;</div><div class='ctx'> 	int uid;</div><div class='ctx'> 	struct mlx5_ib_dev *mdev = to_mdev(c-&gt;ibucontext.device);</div><div class='ctx'> </div><div class='add'>+	if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='ctx'> 	uid = devx_get_uid(c, cmd_in);</div><div class='ctx'> 	if (uid &lt; 0)</div><div class='ctx'> 		return uid;</div><div class='hunk'>@@ -1316,14 +1722,16 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(</div><div class='ctx'> 		return PTR_ERR(cmd_out);</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);</div><div class='del'>-	err = mlx5_cmd_exec(mdev-&gt;mdev, cmd_in,</div><div class='del'>-			    uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN),</div><div class='del'>-			    cmd_out, cmd_out_len);</div><div class='del'>-	if (err)</div><div class='add'>+	err = mlx5_cmd_do(mdev-&gt;mdev, cmd_in,</div><div class='add'>+			  uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN),</div><div class='add'>+			  cmd_out, cmd_out_len);</div><div class='add'>+	if (err &amp;&amp; err != -EREMOTEIO)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='del'>-	return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,</div><div class='add'>+	err2 = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,</div><div class='ctx'> 			      cmd_out, cmd_out_len);</div><div class='add'>+</div><div class='add'>+	return err2 ?: err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct devx_async_event_queue {</div><div class='hunk'>@@ -1365,25 +1773,56 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)(</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *uobj = uverbs_attr_get_uobject(</div><div class='add'>+		attrs, MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE);</div><div class='add'>+	struct devx_async_event_file *ev_file;</div><div class='add'>+	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(</div><div class='add'>+		&amp;attrs-&gt;driver_udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(c-&gt;ibucontext.device);</div><div class='add'>+	u32 flags;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	err = uverbs_get_flags32(&amp;flags, attrs,</div><div class='add'>+		MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,</div><div class='add'>+		MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA);</div><div class='add'>+</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	ev_file = container_of(uobj, struct devx_async_event_file,</div><div class='add'>+			       uobj);</div><div class='add'>+	spin_lock_init(&amp;ev_file-&gt;lock);</div><div class='add'>+	INIT_LIST_HEAD(&amp;ev_file-&gt;event_list);</div><div class='add'>+	init_waitqueue_head(&amp;ev_file-&gt;poll_wait);</div><div class='add'>+	if (flags &amp; MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA)</div><div class='add'>+		ev_file-&gt;omit_data = 1;</div><div class='add'>+	INIT_LIST_HEAD(&amp;ev_file-&gt;subscribed_events_list);</div><div class='add'>+	ev_file-&gt;dev = dev;</div><div class='add'>+	get_device(&amp;dev-&gt;ib_dev.dev);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static void devx_query_callback(int status, struct mlx5_async_work *context)</div><div class='ctx'> {</div><div class='ctx'> 	struct devx_async_data *async_data =</div><div class='ctx'> 		container_of(context, struct devx_async_data, cb_work);</div><div class='del'>-	struct ib_uobject *fd_uobj = async_data-&gt;fd_uobj;</div><div class='del'>-	struct devx_async_cmd_event_file *ev_file;</div><div class='del'>-	struct devx_async_event_queue *ev_queue;</div><div class='add'>+	struct devx_async_cmd_event_file *ev_file = async_data-&gt;ev_file;</div><div class='add'>+	struct devx_async_event_queue *ev_queue = &amp;ev_file-&gt;ev_queue;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> </div><div class='del'>-	ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file,</div><div class='del'>-			       uobj);</div><div class='del'>-	ev_queue = &amp;ev_file-&gt;ev_queue;</div><div class='del'>-</div><div class='add'>+	/*</div><div class='add'>+	 * Note that if the struct devx_async_cmd_event_file uobj begins to be</div><div class='add'>+	 * destroyed it will block at mlx5_cmd_cleanup_async_ctx() until this</div><div class='add'>+	 * routine returns, ensuring that it always remains valid here.</div><div class='add'>+	 */</div><div class='ctx'> 	spin_lock_irqsave(&amp;ev_queue-&gt;lock, flags);</div><div class='ctx'> 	list_add_tail(&amp;async_data-&gt;list, &amp;ev_queue-&gt;event_list);</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;ev_queue-&gt;lock, flags);</div><div class='ctx'> </div><div class='ctx'> 	wake_up_interruptible(&amp;ev_queue-&gt;poll_wait);</div><div class='del'>-	fput(fd_uobj-&gt;object);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> #define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */</div><div class='hunk'>@@ -1406,6 +1845,9 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(</div><div class='ctx'> 	struct devx_async_cmd_event_file *ev_file;</div><div class='ctx'> 	struct devx_async_data *async_data;</div><div class='ctx'> </div><div class='add'>+	if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='ctx'> 	uid = devx_get_uid(c, cmd_in);</div><div class='ctx'> 	if (uid &lt; 0)</div><div class='ctx'> 		return uid;</div><div class='hunk'>@@ -1449,9 +1891,8 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(</div><div class='ctx'> </div><div class='ctx'> 	async_data-&gt;cmd_out_len = cmd_out_len;</div><div class='ctx'> 	async_data-&gt;mdev = mdev;</div><div class='del'>-	async_data-&gt;fd_uobj = fd_uobj;</div><div class='add'>+	async_data-&gt;ev_file = ev_file;</div><div class='ctx'> </div><div class='del'>-	get_file(fd_uobj-&gt;object);</div><div class='ctx'> 	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);</div><div class='ctx'> 	err = mlx5_cmd_exec_cb(&amp;ev_file-&gt;async_ctx, cmd_in,</div><div class='ctx'> 		    uverbs_attr_get_len(attrs,</div><div class='hunk'>@@ -1461,12 +1902,10 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(</div><div class='ctx'> 		    devx_query_callback, &amp;async_data-&gt;cb_work);</div><div class='ctx'> </div><div class='ctx'> 	if (err)</div><div class='del'>-		goto cb_err;</div><div class='add'>+		goto free_async;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-cb_err:</div><div class='del'>-	fput(fd_uobj-&gt;object);</div><div class='ctx'> free_async:</div><div class='ctx'> 	kvfree(async_data);</div><div class='ctx'> sub_bytes:</div><div class='hunk'>@@ -1474,80 +1913,463 @@ sub_bytes:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void</div><div class='add'>+subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table,</div><div class='add'>+			   u32 key_level1,</div><div class='add'>+			   bool is_level2,</div><div class='add'>+			   u32 key_level2)</div><div class='add'>+{</div><div class='add'>+	struct devx_event *event;</div><div class='add'>+	struct devx_obj_event *xa_val_level2;</div><div class='add'>+</div><div class='add'>+	/* Level 1 is valid for future use, no need to free */</div><div class='add'>+	if (!is_level2)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	event = xa_load(&amp;devx_event_table-&gt;event_xa, key_level1);</div><div class='add'>+	WARN_ON(!event);</div><div class='add'>+</div><div class='add'>+	xa_val_level2 = xa_load(&amp;event-&gt;object_ids,</div><div class='add'>+				key_level2);</div><div class='add'>+	if (list_empty(&amp;xa_val_level2-&gt;obj_sub_list)) {</div><div class='add'>+		xa_erase(&amp;event-&gt;object_ids,</div><div class='add'>+			 key_level2);</div><div class='add'>+		kfree_rcu(xa_val_level2, rcu);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int</div><div class='add'>+subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table,</div><div class='add'>+			 u32 key_level1,</div><div class='add'>+			 bool is_level2,</div><div class='add'>+			 u32 key_level2)</div><div class='add'>+{</div><div class='add'>+	struct devx_obj_event *obj_event;</div><div class='add'>+	struct devx_event *event;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	event = xa_load(&amp;devx_event_table-&gt;event_xa, key_level1);</div><div class='add'>+	if (!event) {</div><div class='add'>+		event = kzalloc(sizeof(*event), GFP_KERNEL);</div><div class='add'>+		if (!event)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+</div><div class='add'>+		INIT_LIST_HEAD(&amp;event-&gt;unaffiliated_list);</div><div class='add'>+		xa_init(&amp;event-&gt;object_ids);</div><div class='add'>+</div><div class='add'>+		err = xa_insert(&amp;devx_event_table-&gt;event_xa,</div><div class='add'>+				key_level1,</div><div class='add'>+				event,</div><div class='add'>+				GFP_KERNEL);</div><div class='add'>+		if (err) {</div><div class='add'>+			kfree(event);</div><div class='add'>+			return err;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!is_level2)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	obj_event = xa_load(&amp;event-&gt;object_ids, key_level2);</div><div class='add'>+	if (!obj_event) {</div><div class='add'>+		obj_event = kzalloc(sizeof(*obj_event), GFP_KERNEL);</div><div class='add'>+		if (!obj_event)</div><div class='add'>+			/* Level1 is valid for future use, no need to free */</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+</div><div class='add'>+		INIT_LIST_HEAD(&amp;obj_event-&gt;obj_sub_list);</div><div class='add'>+		err = xa_insert(&amp;event-&gt;object_ids,</div><div class='add'>+				key_level2,</div><div class='add'>+				obj_event,</div><div class='add'>+				GFP_KERNEL);</div><div class='add'>+		if (err) {</div><div class='add'>+			kfree(obj_event);</div><div class='add'>+			return err;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool is_valid_events_legacy(int num_events, u16 *event_type_num_list,</div><div class='add'>+				   struct devx_obj *obj)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; num_events; i++) {</div><div class='add'>+		if (obj) {</div><div class='add'>+			if (!is_legacy_obj_event_num(event_type_num_list[i]))</div><div class='add'>+				return false;</div><div class='add'>+		} else if (!is_legacy_unaffiliated_event_num(</div><div class='add'>+				event_type_num_list[i])) {</div><div class='add'>+			return false;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define MAX_SUPP_EVENT_NUM 255</div><div class='add'>+static bool is_valid_events(struct mlx5_core_dev *dev,</div><div class='add'>+			    int num_events, u16 *event_type_num_list,</div><div class='add'>+			    struct devx_obj *obj)</div><div class='add'>+{</div><div class='add'>+	__be64 *aff_events;</div><div class='add'>+	__be64 *unaff_events;</div><div class='add'>+	int mask_entry;</div><div class='add'>+	int mask_bit;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_GEN(dev, event_cap)) {</div><div class='add'>+		aff_events = MLX5_CAP_DEV_EVENT(dev,</div><div class='add'>+						user_affiliated_events);</div><div class='add'>+		unaff_events = MLX5_CAP_DEV_EVENT(dev,</div><div class='add'>+						  user_unaffiliated_events);</div><div class='add'>+	} else {</div><div class='add'>+		return is_valid_events_legacy(num_events, event_type_num_list,</div><div class='add'>+					      obj);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; num_events; i++) {</div><div class='add'>+		if (event_type_num_list[i] &gt; MAX_SUPP_EVENT_NUM)</div><div class='add'>+			return false;</div><div class='add'>+</div><div class='add'>+		mask_entry = event_type_num_list[i] / 64;</div><div class='add'>+		mask_bit = event_type_num_list[i] % 64;</div><div class='add'>+</div><div class='add'>+		if (obj) {</div><div class='add'>+			/* CQ completion */</div><div class='add'>+			if (event_type_num_list[i] == 0)</div><div class='add'>+				continue;</div><div class='add'>+</div><div class='add'>+			if (!(be64_to_cpu(aff_events[mask_entry]) &amp;</div><div class='add'>+					(1ull &lt;&lt; mask_bit)))</div><div class='add'>+				return false;</div><div class='add'>+</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (!(be64_to_cpu(unaff_events[mask_entry]) &amp;</div><div class='add'>+				(1ull &lt;&lt; mask_bit)))</div><div class='add'>+			return false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define MAX_NUM_EVENTS 16</div><div class='add'>+static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *devx_uobj = uverbs_attr_get_uobject(</div><div class='add'>+				attrs,</div><div class='add'>+				MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE);</div><div class='add'>+	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(</div><div class='add'>+		&amp;attrs-&gt;driver_udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(c-&gt;ibucontext.device);</div><div class='add'>+	struct ib_uobject *fd_uobj;</div><div class='add'>+	struct devx_obj *obj = NULL;</div><div class='add'>+	struct devx_async_event_file *ev_file;</div><div class='add'>+	struct mlx5_devx_event_table *devx_event_table = &amp;dev-&gt;devx_event_table;</div><div class='add'>+	u16 *event_type_num_list;</div><div class='add'>+	struct devx_event_subscription *event_sub, *tmp_sub;</div><div class='add'>+	struct list_head sub_list;</div><div class='add'>+	int redirect_fd;</div><div class='add'>+	bool use_eventfd = false;</div><div class='add'>+	int num_events;</div><div class='add'>+	u16 obj_type = 0;</div><div class='add'>+	u64 cookie = 0;</div><div class='add'>+	u32 obj_id = 0;</div><div class='add'>+	int err;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (!c-&gt;devx_uid)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!IS_ERR(devx_uobj)) {</div><div class='add'>+		obj = (struct devx_obj *)devx_uobj-&gt;object;</div><div class='add'>+		if (obj)</div><div class='add'>+			obj_id = get_dec_obj_id(obj-&gt;obj_id);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	fd_uobj = uverbs_attr_get_uobject(attrs,</div><div class='add'>+				MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE);</div><div class='add'>+	if (IS_ERR(fd_uobj))</div><div class='add'>+		return PTR_ERR(fd_uobj);</div><div class='add'>+</div><div class='add'>+	ev_file = container_of(fd_uobj, struct devx_async_event_file,</div><div class='add'>+			       uobj);</div><div class='add'>+</div><div class='add'>+	if (uverbs_attr_is_valid(attrs,</div><div class='add'>+				 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM)) {</div><div class='add'>+		err = uverbs_copy_from(&amp;redirect_fd, attrs,</div><div class='add'>+			       MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM);</div><div class='add'>+		if (err)</div><div class='add'>+			return err;</div><div class='add'>+</div><div class='add'>+		use_eventfd = true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (uverbs_attr_is_valid(attrs,</div><div class='add'>+				 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE)) {</div><div class='add'>+		if (use_eventfd)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		err = uverbs_copy_from(&amp;cookie, attrs,</div><div class='add'>+				MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE);</div><div class='add'>+		if (err)</div><div class='add'>+			return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	num_events = uverbs_attr_ptr_get_array_size(</div><div class='add'>+		attrs, MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,</div><div class='add'>+		sizeof(u16));</div><div class='add'>+</div><div class='add'>+	if (num_events &lt; 0)</div><div class='add'>+		return num_events;</div><div class='add'>+</div><div class='add'>+	if (num_events &gt; MAX_NUM_EVENTS)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	event_type_num_list = uverbs_attr_get_alloced_ptr(attrs,</div><div class='add'>+			MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST);</div><div class='add'>+</div><div class='add'>+	if (!is_valid_events(dev-&gt;mdev, num_events, event_type_num_list, obj))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;sub_list);</div><div class='add'>+</div><div class='add'>+	/* Protect from concurrent subscriptions to same XA entries to allow</div><div class='add'>+	 * both to succeed</div><div class='add'>+	 */</div><div class='add'>+	mutex_lock(&amp;devx_event_table-&gt;event_xa_lock);</div><div class='add'>+	for (i = 0; i &lt; num_events; i++) {</div><div class='add'>+		u32 key_level1;</div><div class='add'>+</div><div class='add'>+		if (obj)</div><div class='add'>+			obj_type = get_dec_obj_type(obj,</div><div class='add'>+						    event_type_num_list[i]);</div><div class='add'>+		key_level1 = event_type_num_list[i] | obj_type &lt;&lt; 16;</div><div class='add'>+</div><div class='add'>+		err = subscribe_event_xa_alloc(devx_event_table,</div><div class='add'>+					       key_level1,</div><div class='add'>+					       obj,</div><div class='add'>+					       obj_id);</div><div class='add'>+		if (err)</div><div class='add'>+			goto err;</div><div class='add'>+</div><div class='add'>+		event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL);</div><div class='add'>+		if (!event_sub) {</div><div class='add'>+			err = -ENOMEM;</div><div class='add'>+			goto err;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		list_add_tail(&amp;event_sub-&gt;event_list, &amp;sub_list);</div><div class='add'>+		uverbs_uobject_get(&amp;ev_file-&gt;uobj);</div><div class='add'>+		if (use_eventfd) {</div><div class='add'>+			event_sub-&gt;eventfd =</div><div class='add'>+				eventfd_ctx_fdget(redirect_fd);</div><div class='add'>+</div><div class='add'>+			if (IS_ERR(event_sub-&gt;eventfd)) {</div><div class='add'>+				err = PTR_ERR(event_sub-&gt;eventfd);</div><div class='add'>+				event_sub-&gt;eventfd = NULL;</div><div class='add'>+				goto err;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		event_sub-&gt;cookie = cookie;</div><div class='add'>+		event_sub-&gt;ev_file = ev_file;</div><div class='add'>+		/* May be needed upon cleanup the devx object/subscription */</div><div class='add'>+		event_sub-&gt;xa_key_level1 = key_level1;</div><div class='add'>+		event_sub-&gt;xa_key_level2 = obj_id;</div><div class='add'>+		INIT_LIST_HEAD(&amp;event_sub-&gt;obj_list);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Once all the allocations and the XA data insertions were done we</div><div class='add'>+	 * can go ahead and add all the subscriptions to the relevant lists</div><div class='add'>+	 * without concern of a failure.</div><div class='add'>+	 */</div><div class='add'>+	list_for_each_entry_safe(event_sub, tmp_sub, &amp;sub_list, event_list) {</div><div class='add'>+		struct devx_event *event;</div><div class='add'>+		struct devx_obj_event *obj_event;</div><div class='add'>+</div><div class='add'>+		list_del_init(&amp;event_sub-&gt;event_list);</div><div class='add'>+</div><div class='add'>+		spin_lock_irq(&amp;ev_file-&gt;lock);</div><div class='add'>+		list_add_tail_rcu(&amp;event_sub-&gt;file_list,</div><div class='add'>+				  &amp;ev_file-&gt;subscribed_events_list);</div><div class='add'>+		spin_unlock_irq(&amp;ev_file-&gt;lock);</div><div class='add'>+</div><div class='add'>+		event = xa_load(&amp;devx_event_table-&gt;event_xa,</div><div class='add'>+				event_sub-&gt;xa_key_level1);</div><div class='add'>+		WARN_ON(!event);</div><div class='add'>+</div><div class='add'>+		if (!obj) {</div><div class='add'>+			list_add_tail_rcu(&amp;event_sub-&gt;xa_list,</div><div class='add'>+					  &amp;event-&gt;unaffiliated_list);</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		obj_event = xa_load(&amp;event-&gt;object_ids, obj_id);</div><div class='add'>+		WARN_ON(!obj_event);</div><div class='add'>+		list_add_tail_rcu(&amp;event_sub-&gt;xa_list,</div><div class='add'>+				  &amp;obj_event-&gt;obj_sub_list);</div><div class='add'>+		list_add_tail_rcu(&amp;event_sub-&gt;obj_list,</div><div class='add'>+				  &amp;obj-&gt;event_sub);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_unlock(&amp;devx_event_table-&gt;event_xa_lock);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	list_for_each_entry_safe(event_sub, tmp_sub, &amp;sub_list, event_list) {</div><div class='add'>+		list_del(&amp;event_sub-&gt;event_list);</div><div class='add'>+</div><div class='add'>+		subscribe_event_xa_dealloc(devx_event_table,</div><div class='add'>+					   event_sub-&gt;xa_key_level1,</div><div class='add'>+					   obj,</div><div class='add'>+					   obj_id);</div><div class='add'>+</div><div class='add'>+		if (event_sub-&gt;eventfd)</div><div class='add'>+			eventfd_ctx_put(event_sub-&gt;eventfd);</div><div class='add'>+		uverbs_uobject_put(&amp;event_sub-&gt;ev_file-&gt;uobj);</div><div class='add'>+		kfree(event_sub);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_unlock(&amp;devx_event_table-&gt;event_xa_lock);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,</div><div class='ctx'> 			 struct uverbs_attr_bundle *attrs,</div><div class='del'>-			 struct devx_umem *obj)</div><div class='add'>+			 struct devx_umem *obj, u32 access_flags)</div><div class='ctx'> {</div><div class='ctx'> 	u64 addr;</div><div class='ctx'> 	size_t size;</div><div class='del'>-	u32 access;</div><div class='del'>-	int npages;</div><div class='ctx'> 	int err;</div><div class='del'>-	u32 page_mask;</div><div class='ctx'> </div><div class='ctx'> 	if (uverbs_copy_from(&amp;addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||</div><div class='ctx'> 	    uverbs_copy_from(&amp;size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN))</div><div class='ctx'> 		return -EFAULT;</div><div class='ctx'> </div><div class='del'>-	err = uverbs_get_flags32(&amp;access, attrs,</div><div class='del'>-				 MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,</div><div class='del'>-				 IB_ACCESS_LOCAL_WRITE |</div><div class='del'>-				 IB_ACCESS_REMOTE_WRITE |</div><div class='del'>-				 IB_ACCESS_REMOTE_READ);</div><div class='add'>+	err = ib_check_mr_access(&amp;dev-&gt;ib_dev, access_flags);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='del'>-	err = ib_check_mr_access(access);</div><div class='del'>-	if (err)</div><div class='del'>-		return err;</div><div class='del'>-</div><div class='del'>-	obj-&gt;umem = ib_umem_get(&amp;attrs-&gt;driver_udata, addr, size, access, 0);</div><div class='del'>-	if (IS_ERR(obj-&gt;umem))</div><div class='del'>-		return PTR_ERR(obj-&gt;umem);</div><div class='add'>+	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD)) {</div><div class='add'>+		struct ib_umem_dmabuf *umem_dmabuf;</div><div class='add'>+		int dmabuf_fd;</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_cont_pages(obj-&gt;umem, obj-&gt;umem-&gt;address,</div><div class='del'>-			   MLX5_MKEY_PAGE_SHIFT_MASK, &amp;npages,</div><div class='del'>-			   &amp;obj-&gt;page_shift, &amp;obj-&gt;ncont, NULL);</div><div class='add'>+		err = uverbs_get_raw_fd(&amp;dmabuf_fd, attrs,</div><div class='add'>+					MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD);</div><div class='add'>+		if (err)</div><div class='add'>+			return -EFAULT;</div><div class='ctx'> </div><div class='del'>-	if (!npages) {</div><div class='del'>-		ib_umem_release(obj-&gt;umem);</div><div class='del'>-		return -EINVAL;</div><div class='add'>+		umem_dmabuf = ib_umem_dmabuf_get_pinned(</div><div class='add'>+			&amp;dev-&gt;ib_dev, addr, size, dmabuf_fd, access_flags);</div><div class='add'>+		if (IS_ERR(umem_dmabuf))</div><div class='add'>+			return PTR_ERR(umem_dmabuf);</div><div class='add'>+		obj-&gt;umem = &amp;umem_dmabuf-&gt;umem;</div><div class='add'>+	} else {</div><div class='add'>+		obj-&gt;umem = ib_umem_get(&amp;dev-&gt;ib_dev, addr, size, access_flags);</div><div class='add'>+		if (IS_ERR(obj-&gt;umem))</div><div class='add'>+			return PTR_ERR(obj-&gt;umem);</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='del'>-	page_mask = (1 &lt;&lt; obj-&gt;page_shift) - 1;</div><div class='del'>-	obj-&gt;page_offset = obj-&gt;umem-&gt;address &amp; page_mask;</div><div class='del'>-</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int devx_umem_reg_cmd_alloc(struct uverbs_attr_bundle *attrs,</div><div class='del'>-				   struct devx_umem *obj,</div><div class='del'>-				   struct devx_umem_reg_cmd *cmd)</div><div class='add'>+static unsigned int devx_umem_find_best_pgsize(struct ib_umem *umem,</div><div class='add'>+					       unsigned long pgsz_bitmap)</div><div class='ctx'> {</div><div class='del'>-	cmd-&gt;inlen = MLX5_ST_SZ_BYTES(create_umem_in) +</div><div class='del'>-		    (MLX5_ST_SZ_BYTES(mtt) * obj-&gt;ncont);</div><div class='del'>-	cmd-&gt;in = uverbs_zalloc(attrs, cmd-&gt;inlen);</div><div class='del'>-	return PTR_ERR_OR_ZERO(cmd-&gt;in);</div><div class='add'>+	unsigned long page_size;</div><div class='add'>+</div><div class='add'>+	/* Don't bother checking larger page sizes as offset must be zero and</div><div class='add'>+	 * total DEVX umem length must be equal to total umem length.</div><div class='add'>+	 */</div><div class='add'>+	pgsz_bitmap &amp;= GENMASK_ULL(max_t(u64, order_base_2(umem-&gt;length),</div><div class='add'>+					 PAGE_SHIFT),</div><div class='add'>+				   MLX5_ADAPTER_PAGE_SHIFT);</div><div class='add'>+	if (!pgsz_bitmap)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	page_size = ib_umem_find_best_pgoff(umem, pgsz_bitmap, U64_MAX);</div><div class='add'>+	if (!page_size)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	/* If the page_size is less than the CPU page size then we can use the</div><div class='add'>+	 * offset and create a umem which is a subset of the page list.</div><div class='add'>+	 * For larger page sizes we can't be sure the DMA  list reflects the</div><div class='add'>+	 * VA so we must ensure that the umem extent is exactly equal to the</div><div class='add'>+	 * page list. Reduce the page size until one of these cases is true.</div><div class='add'>+	 */</div><div class='add'>+	while ((ib_umem_dma_offset(umem, page_size) != 0 ||</div><div class='add'>+		(umem-&gt;length % page_size) != 0) &amp;&amp;</div><div class='add'>+		page_size &gt; PAGE_SIZE)</div><div class='add'>+		page_size /= 2;</div><div class='add'>+</div><div class='add'>+	return page_size;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void devx_umem_reg_cmd_build(struct mlx5_ib_dev *dev,</div><div class='del'>-				    struct devx_umem *obj,</div><div class='del'>-				    struct devx_umem_reg_cmd *cmd)</div><div class='add'>+static int devx_umem_reg_cmd_alloc(struct mlx5_ib_dev *dev,</div><div class='add'>+				   struct uverbs_attr_bundle *attrs,</div><div class='add'>+				   struct devx_umem *obj,</div><div class='add'>+				   struct devx_umem_reg_cmd *cmd,</div><div class='add'>+				   int access)</div><div class='ctx'> {</div><div class='del'>-	void *umem;</div><div class='add'>+	unsigned long pgsz_bitmap;</div><div class='add'>+	unsigned int page_size;</div><div class='ctx'> 	__be64 *mtt;</div><div class='add'>+	void *umem;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * If the user does not pass in pgsz_bitmap then the user promises not</div><div class='add'>+	 * to use umem_offset!=0 in any commands that allocate on top of the</div><div class='add'>+	 * umem.</div><div class='add'>+	 *</div><div class='add'>+	 * If the user wants to use a umem_offset then it must pass in</div><div class='add'>+	 * pgsz_bitmap which guides the maximum page size and thus maximum</div><div class='add'>+	 * object alignment inside the umem. See the PRM.</div><div class='add'>+	 *</div><div class='add'>+	 * Users are not allowed to use IOVA here, mkeys are not supported on</div><div class='add'>+	 * umem.</div><div class='add'>+	 */</div><div class='add'>+	ret = uverbs_get_const_default(&amp;pgsz_bitmap, attrs,</div><div class='add'>+			MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP,</div><div class='add'>+			GENMASK_ULL(63,</div><div class='add'>+				    min(PAGE_SHIFT, MLX5_ADAPTER_PAGE_SHIFT)));</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	page_size = devx_umem_find_best_pgsize(obj-&gt;umem, pgsz_bitmap);</div><div class='add'>+	if (!page_size)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	cmd-&gt;inlen = MLX5_ST_SZ_BYTES(create_umem_in) +</div><div class='add'>+		     (MLX5_ST_SZ_BYTES(mtt) *</div><div class='add'>+		      ib_umem_num_dma_blocks(obj-&gt;umem, page_size));</div><div class='add'>+	cmd-&gt;in = uverbs_zalloc(attrs, cmd-&gt;inlen);</div><div class='add'>+	if (IS_ERR(cmd-&gt;in))</div><div class='add'>+		return PTR_ERR(cmd-&gt;in);</div><div class='ctx'> </div><div class='ctx'> 	umem = MLX5_ADDR_OF(create_umem_in, cmd-&gt;in, umem);</div><div class='ctx'> 	mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt);</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(create_umem_in, cmd-&gt;in, opcode, MLX5_CMD_OP_CREATE_UMEM);</div><div class='del'>-	MLX5_SET64(umem, umem, num_of_mtt, obj-&gt;ncont);</div><div class='del'>-	MLX5_SET(umem, umem, log_page_size, obj-&gt;page_shift -</div><div class='del'>-					    MLX5_ADAPTER_PAGE_SHIFT);</div><div class='del'>-	MLX5_SET(umem, umem, page_offset, obj-&gt;page_offset);</div><div class='del'>-	mlx5_ib_populate_pas(dev, obj-&gt;umem, obj-&gt;page_shift, mtt,</div><div class='add'>+	MLX5_SET64(umem, umem, num_of_mtt,</div><div class='add'>+		   ib_umem_num_dma_blocks(obj-&gt;umem, page_size));</div><div class='add'>+	MLX5_SET(umem, umem, log_page_size,</div><div class='add'>+		 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);</div><div class='add'>+	MLX5_SET(umem, umem, page_offset,</div><div class='add'>+		 ib_umem_dma_offset(obj-&gt;umem, page_size));</div><div class='add'>+</div><div class='add'>+	if (mlx5_umem_needs_ats(dev, obj-&gt;umem, access))</div><div class='add'>+		MLX5_SET(umem, umem, ats, 1);</div><div class='add'>+</div><div class='add'>+	mlx5_ib_populate_pas(obj-&gt;umem, page_size, mtt,</div><div class='ctx'> 			     (obj-&gt;umem-&gt;writable ? MLX5_IB_MTT_WRITE : 0) |</div><div class='del'>-			     MLX5_IB_MTT_READ);</div><div class='add'>+				     MLX5_IB_MTT_READ);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(</div><div class='hunk'>@@ -1561,25 +2383,33 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(</div><div class='ctx'> 	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(</div><div class='ctx'> 		&amp;attrs-&gt;driver_udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(c-&gt;ibucontext.device);</div><div class='add'>+	int access_flags;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	if (!c-&gt;devx_uid)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='add'>+	err = uverbs_get_flags32(&amp;access_flags, attrs,</div><div class='add'>+				 MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,</div><div class='add'>+				 IB_ACCESS_LOCAL_WRITE |</div><div class='add'>+				 IB_ACCESS_REMOTE_WRITE |</div><div class='add'>+				 IB_ACCESS_REMOTE_READ |</div><div class='add'>+				 IB_ACCESS_RELAXED_ORDERING);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='ctx'> 	obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL);</div><div class='ctx'> 	if (!obj)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	err = devx_umem_get(dev, &amp;c-&gt;ibucontext, attrs, obj);</div><div class='add'>+	err = devx_umem_get(dev, &amp;c-&gt;ibucontext, attrs, obj, access_flags);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_obj_free;</div><div class='ctx'> </div><div class='del'>-	err = devx_umem_reg_cmd_alloc(attrs, obj, &amp;cmd);</div><div class='add'>+	err = devx_umem_reg_cmd_alloc(dev, attrs, obj, &amp;cmd, access_flags);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_umem_release;</div><div class='ctx'> </div><div class='del'>-	devx_umem_reg_cmd_build(dev, obj, &amp;cmd);</div><div class='del'>-</div><div class='ctx'> 	MLX5_SET(create_umem_in, cmd.in, uid, c-&gt;devx_uid);</div><div class='ctx'> 	err = mlx5_cmd_exec(dev-&gt;mdev, cmd.in, cmd.inlen, cmd.out,</div><div class='ctx'> 			    sizeof(cmd.out));</div><div class='hunk'>@@ -1589,14 +2419,12 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(</div><div class='ctx'> 	obj-&gt;mdev = dev-&gt;mdev;</div><div class='ctx'> 	uobj-&gt;object = obj;</div><div class='ctx'> 	devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj-&gt;dinbox, &amp;obj-&gt;dinlen, &amp;obj_id);</div><div class='del'>-	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &amp;obj_id, sizeof(obj_id));</div><div class='del'>-	if (err)</div><div class='del'>-		goto err_umem_destroy;</div><div class='add'>+	uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &amp;obj_id,</div><div class='add'>+			     sizeof(obj_id));</div><div class='add'>+	return err;</div><div class='ctx'> </div><div class='del'>-err_umem_destroy:</div><div class='del'>-	mlx5_cmd_exec(obj-&gt;mdev, obj-&gt;dinbox, obj-&gt;dinlen, cmd.out, sizeof(cmd.out));</div><div class='ctx'> err_umem_release:</div><div class='ctx'> 	ib_umem_release(obj-&gt;umem);</div><div class='ctx'> err_obj_free:</div><div class='hunk'>@@ -1613,7 +2441,7 @@ static int devx_umem_cleanup(struct ib_uobject *uobject,</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	err = mlx5_cmd_exec(obj-&gt;mdev, obj-&gt;dinbox, obj-&gt;dinlen, out, sizeof(out));</div><div class='del'>-	if (ib_is_destroy_retryable(err, why, uobject))</div><div class='add'>+	if (err)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='ctx'> 	ib_umem_release(obj-&gt;umem);</div><div class='hunk'>@@ -1621,6 +2449,297 @@ static int devx_umem_cleanup(struct ib_uobject *uobject,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static bool is_unaffiliated_event(struct mlx5_core_dev *dev,</div><div class='add'>+				  unsigned long event_type)</div><div class='add'>+{</div><div class='add'>+	__be64 *unaff_events;</div><div class='add'>+	int mask_entry;</div><div class='add'>+	int mask_bit;</div><div class='add'>+</div><div class='add'>+	if (!MLX5_CAP_GEN(dev, event_cap))</div><div class='add'>+		return is_legacy_unaffiliated_event_num(event_type);</div><div class='add'>+</div><div class='add'>+	unaff_events = MLX5_CAP_DEV_EVENT(dev,</div><div class='add'>+					  user_unaffiliated_events);</div><div class='add'>+	WARN_ON(event_type &gt; MAX_SUPP_EVENT_NUM);</div><div class='add'>+</div><div class='add'>+	mask_entry = event_type / 64;</div><div class='add'>+	mask_bit = event_type % 64;</div><div class='add'>+</div><div class='add'>+	if (!(be64_to_cpu(unaff_events[mask_entry]) &amp; (1ull &lt;&lt; mask_bit)))</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u32 devx_get_obj_id_from_event(unsigned long event_type, void *data)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_eqe *eqe = data;</div><div class='add'>+	u32 obj_id = 0;</div><div class='add'>+</div><div class='add'>+	switch (event_type) {</div><div class='add'>+	case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:</div><div class='add'>+	case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:</div><div class='add'>+	case MLX5_EVENT_TYPE_PATH_MIG:</div><div class='add'>+	case MLX5_EVENT_TYPE_COMM_EST:</div><div class='add'>+	case MLX5_EVENT_TYPE_SQ_DRAINED:</div><div class='add'>+	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:</div><div class='add'>+	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:</div><div class='add'>+	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:</div><div class='add'>+	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:</div><div class='add'>+	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:</div><div class='add'>+		obj_id = be32_to_cpu(eqe-&gt;data.qp_srq.qp_srq_n) &amp; 0xffffff;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_EVENT_TYPE_XRQ_ERROR:</div><div class='add'>+		obj_id = be32_to_cpu(eqe-&gt;data.xrq_err.type_xrqn) &amp; 0xffffff;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_EVENT_TYPE_DCT_DRAINED:</div><div class='add'>+	case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:</div><div class='add'>+		obj_id = be32_to_cpu(eqe-&gt;data.dct.dctn) &amp; 0xffffff;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_EVENT_TYPE_CQ_ERROR:</div><div class='add'>+		obj_id = be32_to_cpu(eqe-&gt;data.cq_err.cqn) &amp; 0xffffff;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		obj_id = MLX5_GET(affiliated_event_header, &amp;eqe-&gt;data, obj_id);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return obj_id;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int deliver_event(struct devx_event_subscription *event_sub,</div><div class='add'>+			 const void *data)</div><div class='add'>+{</div><div class='add'>+	struct devx_async_event_file *ev_file;</div><div class='add'>+	struct devx_async_event_data *event_data;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	ev_file = event_sub-&gt;ev_file;</div><div class='add'>+</div><div class='add'>+	if (ev_file-&gt;omit_data) {</div><div class='add'>+		spin_lock_irqsave(&amp;ev_file-&gt;lock, flags);</div><div class='add'>+		if (!list_empty(&amp;event_sub-&gt;event_list) ||</div><div class='add'>+		    ev_file-&gt;is_destroyed) {</div><div class='add'>+			spin_unlock_irqrestore(&amp;ev_file-&gt;lock, flags);</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		list_add_tail(&amp;event_sub-&gt;event_list, &amp;ev_file-&gt;event_list);</div><div class='add'>+		spin_unlock_irqrestore(&amp;ev_file-&gt;lock, flags);</div><div class='add'>+		wake_up_interruptible(&amp;ev_file-&gt;poll_wait);</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	event_data = kzalloc(sizeof(*event_data) + sizeof(struct mlx5_eqe),</div><div class='add'>+			     GFP_ATOMIC);</div><div class='add'>+	if (!event_data) {</div><div class='add'>+		spin_lock_irqsave(&amp;ev_file-&gt;lock, flags);</div><div class='add'>+		ev_file-&gt;is_overflow_err = 1;</div><div class='add'>+		spin_unlock_irqrestore(&amp;ev_file-&gt;lock, flags);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	event_data-&gt;hdr.cookie = event_sub-&gt;cookie;</div><div class='add'>+	memcpy(event_data-&gt;hdr.out_data, data, sizeof(struct mlx5_eqe));</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;ev_file-&gt;lock, flags);</div><div class='add'>+	if (!ev_file-&gt;is_destroyed)</div><div class='add'>+		list_add_tail(&amp;event_data-&gt;list, &amp;ev_file-&gt;event_list);</div><div class='add'>+	else</div><div class='add'>+		kfree(event_data);</div><div class='add'>+	spin_unlock_irqrestore(&amp;ev_file-&gt;lock, flags);</div><div class='add'>+	wake_up_interruptible(&amp;ev_file-&gt;poll_wait);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void dispatch_event_fd(struct list_head *fd_list,</div><div class='add'>+			      const void *data)</div><div class='add'>+{</div><div class='add'>+	struct devx_event_subscription *item;</div><div class='add'>+</div><div class='add'>+	list_for_each_entry_rcu(item, fd_list, xa_list) {</div><div class='add'>+		if (item-&gt;eventfd)</div><div class='add'>+			eventfd_signal(item-&gt;eventfd);</div><div class='add'>+		else</div><div class='add'>+			deliver_event(item, data);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int devx_event_notifier(struct notifier_block *nb,</div><div class='add'>+			       unsigned long event_type, void *data)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_devx_event_table *table;</div><div class='add'>+	struct mlx5_ib_dev *dev;</div><div class='add'>+	struct devx_event *event;</div><div class='add'>+	struct devx_obj_event *obj_event;</div><div class='add'>+	u16 obj_type = 0;</div><div class='add'>+	bool is_unaffiliated;</div><div class='add'>+	u32 obj_id;</div><div class='add'>+</div><div class='add'>+	/* Explicit filtering to kernel events which may occur frequently */</div><div class='add'>+	if (event_type == MLX5_EVENT_TYPE_CMD ||</div><div class='add'>+	    event_type == MLX5_EVENT_TYPE_PAGE_REQUEST)</div><div class='add'>+		return NOTIFY_OK;</div><div class='add'>+</div><div class='add'>+	table = container_of(nb, struct mlx5_devx_event_table, devx_nb.nb);</div><div class='add'>+	dev = container_of(table, struct mlx5_ib_dev, devx_event_table);</div><div class='add'>+	is_unaffiliated = is_unaffiliated_event(dev-&gt;mdev, event_type);</div><div class='add'>+</div><div class='add'>+	if (!is_unaffiliated)</div><div class='add'>+		obj_type = get_event_obj_type(event_type, data);</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	event = xa_load(&amp;table-&gt;event_xa, event_type | (obj_type &lt;&lt; 16));</div><div class='add'>+	if (!event) {</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+		return NOTIFY_DONE;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (is_unaffiliated) {</div><div class='add'>+		dispatch_event_fd(&amp;event-&gt;unaffiliated_list, data);</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+		return NOTIFY_OK;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	obj_id = devx_get_obj_id_from_event(event_type, data);</div><div class='add'>+	obj_event = xa_load(&amp;event-&gt;object_ids, obj_id);</div><div class='add'>+	if (!obj_event) {</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+		return NOTIFY_DONE;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dispatch_event_fd(&amp;obj_event-&gt;obj_sub_list, data);</div><div class='add'>+</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+	return NOTIFY_OK;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_ib_devx_init(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_devx_event_table *table = &amp;dev-&gt;devx_event_table;</div><div class='add'>+	int uid;</div><div class='add'>+</div><div class='add'>+	uid = mlx5_ib_devx_create(dev, false, 0);</div><div class='add'>+	if (uid &gt; 0) {</div><div class='add'>+		dev-&gt;devx_whitelist_uid = uid;</div><div class='add'>+		xa_init(&amp;table-&gt;event_xa);</div><div class='add'>+		mutex_init(&amp;table-&gt;event_xa_lock);</div><div class='add'>+		MLX5_NB_INIT(&amp;table-&gt;devx_nb, devx_event_notifier, NOTIFY_ANY);</div><div class='add'>+		mlx5_eq_notifier_register(dev-&gt;mdev, &amp;table-&gt;devx_nb);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_devx_event_table *table = &amp;dev-&gt;devx_event_table;</div><div class='add'>+	struct devx_event_subscription *sub, *tmp;</div><div class='add'>+	struct devx_event *event;</div><div class='add'>+	void *entry;</div><div class='add'>+	unsigned long id;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;devx_whitelist_uid) {</div><div class='add'>+		mlx5_eq_notifier_unregister(dev-&gt;mdev, &amp;table-&gt;devx_nb);</div><div class='add'>+		mutex_lock(&amp;dev-&gt;devx_event_table.event_xa_lock);</div><div class='add'>+		xa_for_each(&amp;table-&gt;event_xa, id, entry) {</div><div class='add'>+			event = entry;</div><div class='add'>+			list_for_each_entry_safe(</div><div class='add'>+				sub, tmp, &amp;event-&gt;unaffiliated_list, xa_list)</div><div class='add'>+				devx_cleanup_subscription(dev, sub);</div><div class='add'>+			kfree(entry);</div><div class='add'>+		}</div><div class='add'>+		mutex_unlock(&amp;dev-&gt;devx_event_table.event_xa_lock);</div><div class='add'>+		xa_destroy(&amp;table-&gt;event_xa);</div><div class='add'>+</div><div class='add'>+		mlx5_ib_devx_destroy(dev, dev-&gt;devx_whitelist_uid);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void devx_async_destroy_cb(int status, struct mlx5_async_work *context)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_async_cmd *devx_out = container_of(context,</div><div class='add'>+					  struct mlx5_async_cmd, cb_work);</div><div class='add'>+	struct devx_obj *obj = devx_out-&gt;uobject-&gt;object;</div><div class='add'>+</div><div class='add'>+	if (!status)</div><div class='add'>+		obj-&gt;flags |= DEVX_OBJ_FLAGS_HW_FREED;</div><div class='add'>+</div><div class='add'>+	complete(&amp;devx_out-&gt;comp);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void devx_async_destroy(struct mlx5_ib_dev *dev,</div><div class='add'>+			       struct mlx5_async_cmd *cmd)</div><div class='add'>+{</div><div class='add'>+	init_completion(&amp;cmd-&gt;comp);</div><div class='add'>+	cmd-&gt;err = mlx5_cmd_exec_cb(&amp;dev-&gt;async_ctx, cmd-&gt;in, cmd-&gt;in_size,</div><div class='add'>+				    &amp;cmd-&gt;out, sizeof(cmd-&gt;out),</div><div class='add'>+				    devx_async_destroy_cb, &amp;cmd-&gt;cb_work);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void devx_wait_async_destroy(struct mlx5_async_cmd *cmd)</div><div class='add'>+{</div><div class='add'>+	if (!cmd-&gt;err)</div><div class='add'>+		wait_for_completion(&amp;cmd-&gt;comp);</div><div class='add'>+	atomic_set(&amp;cmd-&gt;uobject-&gt;usecnt, 0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_ib_ufile_hw_cleanup(struct ib_uverbs_file *ufile)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_async_cmd *async_cmd;</div><div class='add'>+	struct ib_ucontext *ucontext = ufile-&gt;ucontext;</div><div class='add'>+	struct ib_device *device = ucontext-&gt;device;</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='add'>+	struct ib_uobject *uobject;</div><div class='add'>+	struct devx_obj *obj;</div><div class='add'>+	int head = 0;</div><div class='add'>+	int tail = 0;</div><div class='add'>+</div><div class='add'>+	async_cmd = kcalloc(MAX_ASYNC_CMDS, sizeof(*async_cmd), GFP_KERNEL);</div><div class='add'>+	if (!async_cmd)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	list_for_each_entry(uobject, &amp;ufile-&gt;uobjects, list) {</div><div class='add'>+		WARN_ON(uverbs_try_lock_object(uobject, UVERBS_LOOKUP_WRITE));</div><div class='add'>+</div><div class='add'>+		/*</div><div class='add'>+		 * Currently we only support QP destruction, if other objects</div><div class='add'>+		 * are to be destroyed need to add type synchronization to the</div><div class='add'>+		 * cleanup algorithm and handle pre/post FW cleanup for the</div><div class='add'>+		 * new types if needed.</div><div class='add'>+		 */</div><div class='add'>+		if (uobj_get_object_id(uobject) != MLX5_IB_OBJECT_DEVX_OBJ ||</div><div class='add'>+		    (get_dec_obj_type(uobject-&gt;object, MLX5_EVENT_TYPE_MAX) !=</div><div class='add'>+		     MLX5_OBJ_TYPE_QP)) {</div><div class='add'>+			atomic_set(&amp;uobject-&gt;usecnt, 0);</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		obj = uobject-&gt;object;</div><div class='add'>+</div><div class='add'>+		async_cmd[tail % MAX_ASYNC_CMDS].in = obj-&gt;dinbox;</div><div class='add'>+		async_cmd[tail % MAX_ASYNC_CMDS].in_size = obj-&gt;dinlen;</div><div class='add'>+		async_cmd[tail % MAX_ASYNC_CMDS].uobject = uobject;</div><div class='add'>+</div><div class='add'>+		devx_async_destroy(dev, &amp;async_cmd[tail % MAX_ASYNC_CMDS]);</div><div class='add'>+		tail++;</div><div class='add'>+</div><div class='add'>+		if (tail - head == MAX_ASYNC_CMDS) {</div><div class='add'>+			devx_wait_async_destroy(&amp;async_cmd[head % MAX_ASYNC_CMDS]);</div><div class='add'>+			head++;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	while (head != tail) {</div><div class='add'>+		devx_wait_async_destroy(&amp;async_cmd[head % MAX_ASYNC_CMDS]);</div><div class='add'>+		head++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	kfree(async_cmd);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,</div><div class='ctx'> 					 size_t count, loff_t *pos)</div><div class='ctx'> {</div><div class='hunk'>@@ -1645,11 +2764,11 @@ static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,</div><div class='ctx'> 			return -ERESTARTSYS;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		if (list_empty(&amp;ev_queue-&gt;event_list) &amp;&amp;</div><div class='del'>-		    ev_queue-&gt;is_destroyed)</div><div class='del'>-			return -EIO;</div><div class='del'>-</div><div class='ctx'> 		spin_lock_irq(&amp;ev_queue-&gt;lock);</div><div class='add'>+		if (ev_queue-&gt;is_destroyed) {</div><div class='add'>+			spin_unlock_irq(&amp;ev_queue-&gt;lock);</div><div class='add'>+			return -EIO;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	event = list_entry(ev_queue-&gt;event_list.next,</div><div class='hunk'>@@ -1675,23 +2794,6 @@ static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int devx_async_cmd_event_close(struct inode *inode, struct file *filp)</div><div class='del'>-{</div><div class='del'>-	struct ib_uobject *uobj = filp-&gt;private_data;</div><div class='del'>-	struct devx_async_cmd_event_file *comp_ev_file = container_of(</div><div class='del'>-		uobj, struct devx_async_cmd_event_file, uobj);</div><div class='del'>-	struct devx_async_data *entry, *tmp;</div><div class='del'>-</div><div class='del'>-	spin_lock_irq(&amp;comp_ev_file-&gt;ev_queue.lock);</div><div class='del'>-	list_for_each_entry_safe(entry, tmp,</div><div class='del'>-				 &amp;comp_ev_file-&gt;ev_queue.event_list, list)</div><div class='del'>-		kvfree(entry);</div><div class='del'>-	spin_unlock_irq(&amp;comp_ev_file-&gt;ev_queue.lock);</div><div class='del'>-</div><div class='del'>-	uverbs_close_fd(filp);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static __poll_t devx_async_cmd_event_poll(struct file *filp,</div><div class='ctx'> 					      struct poll_table_struct *wait)</div><div class='ctx'> {</div><div class='hunk'>@@ -1715,27 +2817,193 @@ static const struct file_operations devx_async_cmd_event_fops = {</div><div class='ctx'> 	.owner	 = THIS_MODULE,</div><div class='ctx'> 	.read	 = devx_async_cmd_event_read,</div><div class='ctx'> 	.poll    = devx_async_cmd_event_poll,</div><div class='del'>-	.release = devx_async_cmd_event_close,</div><div class='del'>-	.llseek	 = no_llseek,</div><div class='add'>+	.release = uverbs_uobject_fd_release,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj,</div><div class='del'>-						   enum rdma_remove_reason why)</div><div class='add'>+static ssize_t devx_async_event_read(struct file *filp, char __user *buf,</div><div class='add'>+				     size_t count, loff_t *pos)</div><div class='add'>+{</div><div class='add'>+	struct devx_async_event_file *ev_file = filp-&gt;private_data;</div><div class='add'>+	struct devx_event_subscription *event_sub;</div><div class='add'>+	struct devx_async_event_data *event;</div><div class='add'>+	int ret = 0;</div><div class='add'>+	size_t eventsz;</div><div class='add'>+	bool omit_data;</div><div class='add'>+	void *event_data;</div><div class='add'>+</div><div class='add'>+	omit_data = ev_file-&gt;omit_data;</div><div class='add'>+</div><div class='add'>+	spin_lock_irq(&amp;ev_file-&gt;lock);</div><div class='add'>+</div><div class='add'>+	if (ev_file-&gt;is_overflow_err) {</div><div class='add'>+		ev_file-&gt;is_overflow_err = 0;</div><div class='add'>+		spin_unlock_irq(&amp;ev_file-&gt;lock);</div><div class='add'>+		return -EOVERFLOW;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+</div><div class='add'>+	while (list_empty(&amp;ev_file-&gt;event_list)) {</div><div class='add'>+		spin_unlock_irq(&amp;ev_file-&gt;lock);</div><div class='add'>+</div><div class='add'>+		if (filp-&gt;f_flags &amp; O_NONBLOCK)</div><div class='add'>+			return -EAGAIN;</div><div class='add'>+</div><div class='add'>+		if (wait_event_interruptible(ev_file-&gt;poll_wait,</div><div class='add'>+			    (!list_empty(&amp;ev_file-&gt;event_list) ||</div><div class='add'>+			     ev_file-&gt;is_destroyed))) {</div><div class='add'>+			return -ERESTARTSYS;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		spin_lock_irq(&amp;ev_file-&gt;lock);</div><div class='add'>+		if (ev_file-&gt;is_destroyed) {</div><div class='add'>+			spin_unlock_irq(&amp;ev_file-&gt;lock);</div><div class='add'>+			return -EIO;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (omit_data) {</div><div class='add'>+		event_sub = list_first_entry(&amp;ev_file-&gt;event_list,</div><div class='add'>+					struct devx_event_subscription,</div><div class='add'>+					event_list);</div><div class='add'>+		eventsz = sizeof(event_sub-&gt;cookie);</div><div class='add'>+		event_data = &amp;event_sub-&gt;cookie;</div><div class='add'>+	} else {</div><div class='add'>+		event = list_first_entry(&amp;ev_file-&gt;event_list,</div><div class='add'>+				      struct devx_async_event_data, list);</div><div class='add'>+		eventsz = sizeof(struct mlx5_eqe) +</div><div class='add'>+			sizeof(struct mlx5_ib_uapi_devx_async_event_hdr);</div><div class='add'>+		event_data = &amp;event-&gt;hdr;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (eventsz &gt; count) {</div><div class='add'>+		spin_unlock_irq(&amp;ev_file-&gt;lock);</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (omit_data)</div><div class='add'>+		list_del_init(&amp;event_sub-&gt;event_list);</div><div class='add'>+	else</div><div class='add'>+		list_del(&amp;event-&gt;list);</div><div class='add'>+</div><div class='add'>+	spin_unlock_irq(&amp;ev_file-&gt;lock);</div><div class='add'>+</div><div class='add'>+	if (copy_to_user(buf, event_data, eventsz))</div><div class='add'>+		/* This points to an application issue, not a kernel concern */</div><div class='add'>+		ret = -EFAULT;</div><div class='add'>+	else</div><div class='add'>+		ret = eventsz;</div><div class='add'>+</div><div class='add'>+	if (!omit_data)</div><div class='add'>+		kfree(event);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static __poll_t devx_async_event_poll(struct file *filp,</div><div class='add'>+				      struct poll_table_struct *wait)</div><div class='add'>+{</div><div class='add'>+	struct devx_async_event_file *ev_file = filp-&gt;private_data;</div><div class='add'>+	__poll_t pollflags = 0;</div><div class='add'>+</div><div class='add'>+	poll_wait(filp, &amp;ev_file-&gt;poll_wait, wait);</div><div class='add'>+</div><div class='add'>+	spin_lock_irq(&amp;ev_file-&gt;lock);</div><div class='add'>+	if (ev_file-&gt;is_destroyed)</div><div class='add'>+		pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;</div><div class='add'>+	else if (!list_empty(&amp;ev_file-&gt;event_list))</div><div class='add'>+		pollflags = EPOLLIN | EPOLLRDNORM;</div><div class='add'>+	spin_unlock_irq(&amp;ev_file-&gt;lock);</div><div class='add'>+</div><div class='add'>+	return pollflags;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void devx_free_subscription(struct rcu_head *rcu)</div><div class='add'>+{</div><div class='add'>+	struct devx_event_subscription *event_sub =</div><div class='add'>+		container_of(rcu, struct devx_event_subscription, rcu);</div><div class='add'>+</div><div class='add'>+	if (event_sub-&gt;eventfd)</div><div class='add'>+		eventfd_ctx_put(event_sub-&gt;eventfd);</div><div class='add'>+	uverbs_uobject_put(&amp;event_sub-&gt;ev_file-&gt;uobj);</div><div class='add'>+	kfree(event_sub);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct file_operations devx_async_event_fops = {</div><div class='add'>+	.owner	 = THIS_MODULE,</div><div class='add'>+	.read	 = devx_async_event_read,</div><div class='add'>+	.poll    = devx_async_event_poll,</div><div class='add'>+	.release = uverbs_uobject_fd_release,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static void devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj,</div><div class='add'>+					      enum rdma_remove_reason why)</div><div class='ctx'> {</div><div class='ctx'> 	struct devx_async_cmd_event_file *comp_ev_file =</div><div class='ctx'> 		container_of(uobj, struct devx_async_cmd_event_file,</div><div class='ctx'> 			     uobj);</div><div class='ctx'> 	struct devx_async_event_queue *ev_queue = &amp;comp_ev_file-&gt;ev_queue;</div><div class='add'>+	struct devx_async_data *entry, *tmp;</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irq(&amp;ev_queue-&gt;lock);</div><div class='ctx'> 	ev_queue-&gt;is_destroyed = 1;</div><div class='ctx'> 	spin_unlock_irq(&amp;ev_queue-&gt;lock);</div><div class='del'>-</div><div class='del'>-	if (why == RDMA_REMOVE_DRIVER_REMOVE)</div><div class='del'>-		wake_up_interruptible(&amp;ev_queue-&gt;poll_wait);</div><div class='add'>+	wake_up_interruptible(&amp;ev_queue-&gt;poll_wait);</div><div class='ctx'> </div><div class='ctx'> 	mlx5_cmd_cleanup_async_ctx(&amp;comp_ev_file-&gt;async_ctx);</div><div class='del'>-	return 0;</div><div class='add'>+</div><div class='add'>+	spin_lock_irq(&amp;comp_ev_file-&gt;ev_queue.lock);</div><div class='add'>+	list_for_each_entry_safe(entry, tmp,</div><div class='add'>+				 &amp;comp_ev_file-&gt;ev_queue.event_list, list) {</div><div class='add'>+		list_del(&amp;entry-&gt;list);</div><div class='add'>+		kvfree(entry);</div><div class='add'>+	}</div><div class='add'>+	spin_unlock_irq(&amp;comp_ev_file-&gt;ev_queue.lock);</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static void devx_async_event_destroy_uobj(struct ib_uobject *uobj,</div><div class='add'>+					  enum rdma_remove_reason why)</div><div class='add'>+{</div><div class='add'>+	struct devx_async_event_file *ev_file =</div><div class='add'>+		container_of(uobj, struct devx_async_event_file,</div><div class='add'>+			     uobj);</div><div class='add'>+	struct devx_event_subscription *event_sub, *event_sub_tmp;</div><div class='add'>+	struct mlx5_ib_dev *dev = ev_file-&gt;dev;</div><div class='add'>+</div><div class='add'>+	spin_lock_irq(&amp;ev_file-&gt;lock);</div><div class='add'>+	ev_file-&gt;is_destroyed = 1;</div><div class='add'>+</div><div class='add'>+	/* free the pending events allocation */</div><div class='add'>+	if (ev_file-&gt;omit_data) {</div><div class='add'>+		struct devx_event_subscription *event_sub, *tmp;</div><div class='add'>+</div><div class='add'>+		list_for_each_entry_safe(event_sub, tmp, &amp;ev_file-&gt;event_list,</div><div class='add'>+					 event_list)</div><div class='add'>+			list_del_init(&amp;event_sub-&gt;event_list);</div><div class='add'>+</div><div class='add'>+	} else {</div><div class='add'>+		struct devx_async_event_data *entry, *tmp;</div><div class='add'>+</div><div class='add'>+		list_for_each_entry_safe(entry, tmp, &amp;ev_file-&gt;event_list,</div><div class='add'>+					 list) {</div><div class='add'>+			list_del(&amp;entry-&gt;list);</div><div class='add'>+			kfree(entry);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spin_unlock_irq(&amp;ev_file-&gt;lock);</div><div class='add'>+	wake_up_interruptible(&amp;ev_file-&gt;poll_wait);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;dev-&gt;devx_event_table.event_xa_lock);</div><div class='add'>+	/* delete the subscriptions which are related to this FD */</div><div class='add'>+	list_for_each_entry_safe(event_sub, event_sub_tmp,</div><div class='add'>+				 &amp;ev_file-&gt;subscribed_events_list, file_list) {</div><div class='add'>+		devx_cleanup_subscription(dev, event_sub);</div><div class='add'>+		list_del_rcu(&amp;event_sub-&gt;file_list);</div><div class='add'>+		/* subscription may not be used by the read API any more */</div><div class='add'>+		call_rcu(&amp;event_sub-&gt;rcu, devx_free_subscription);</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;devx_event_table.event_xa_lock);</div><div class='add'>+</div><div class='add'>+	put_device(&amp;dev-&gt;ib_dev.dev);</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> DECLARE_UVERBS_NAMED_METHOD(</div><div class='hunk'>@@ -1750,8 +3018,12 @@ DECLARE_UVERBS_NAMED_METHOD(</div><div class='ctx'> 	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,</div><div class='ctx'> 			   UVERBS_ATTR_TYPE(u64),</div><div class='ctx'> 			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_RAW_FD(MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD,</div><div class='add'>+			   UA_OPTIONAL),</div><div class='ctx'> 	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,</div><div class='ctx'> 			     enum ib_access_flags),</div><div class='add'>+	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP,</div><div class='add'>+			     u64),</div><div class='ctx'> 	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,</div><div class='ctx'> 			    UVERBS_ATTR_TYPE(u32),</div><div class='ctx'> 			    UA_MANDATORY));</div><div class='hunk'>@@ -1820,7 +3092,7 @@ DECLARE_UVERBS_NAMED_METHOD(</div><div class='ctx'> 	MLX5_IB_METHOD_DEVX_OBJ_MODIFY,</div><div class='ctx'> 	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE,</div><div class='ctx'> 			UVERBS_IDR_ANY_OBJECT,</div><div class='del'>-			UVERBS_ACCESS_WRITE,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='ctx'> 			UA_MANDATORY),</div><div class='ctx'> 	UVERBS_ATTR_PTR_IN(</div><div class='ctx'> 		MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN,</div><div class='hunk'>@@ -1869,10 +3141,32 @@ DECLARE_UVERBS_NAMED_METHOD(</div><div class='ctx'> 		UVERBS_ATTR_TYPE(u64),</div><div class='ctx'> 		UA_MANDATORY));</div><div class='ctx'> </div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT,</div><div class='add'>+	UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE,</div><div class='add'>+		MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,</div><div class='add'>+		UVERBS_ACCESS_READ,</div><div class='add'>+		UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE,</div><div class='add'>+		MLX5_IB_OBJECT_DEVX_OBJ,</div><div class='add'>+		UVERBS_ACCESS_READ,</div><div class='add'>+		UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,</div><div class='add'>+		UVERBS_ATTR_MIN_SIZE(sizeof(u16)),</div><div class='add'>+		UA_MANDATORY,</div><div class='add'>+		UA_ALLOC_AND_COPY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE,</div><div class='add'>+		UVERBS_ATTR_TYPE(u64),</div><div class='add'>+		UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM,</div><div class='add'>+		UVERBS_ATTR_TYPE(u32),</div><div class='add'>+		UA_OPTIONAL));</div><div class='add'>+</div><div class='ctx'> DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX,</div><div class='ctx'> 			      &amp;UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER),</div><div class='ctx'> 			      &amp;UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR),</div><div class='del'>-			      &amp;UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN));</div><div class='add'>+			      &amp;UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN),</div><div class='add'>+			      &amp;UVERBS_METHOD(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT));</div><div class='ctx'> </div><div class='ctx'> DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ,</div><div class='ctx'> 			    UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup),</div><div class='hunk'>@@ -1898,11 +3192,29 @@ DECLARE_UVERBS_NAMED_METHOD(</div><div class='ctx'> DECLARE_UVERBS_NAMED_OBJECT(</div><div class='ctx'> 	MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,</div><div class='ctx'> 	UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file),</div><div class='del'>-			     devx_hot_unplug_async_cmd_event_file,</div><div class='add'>+			     devx_async_cmd_event_destroy_uobj,</div><div class='ctx'> 			     &amp;devx_async_cmd_event_fops, "[devx_async_cmd]",</div><div class='ctx'> 			     O_RDONLY),</div><div class='ctx'> 	&amp;UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC));</div><div class='ctx'> </div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC,</div><div class='add'>+	UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE,</div><div class='add'>+			MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,</div><div class='add'>+			UVERBS_ACCESS_NEW,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,</div><div class='add'>+			enum mlx5_ib_uapi_devx_create_event_channel_flags,</div><div class='add'>+			UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_OBJECT(</div><div class='add'>+	MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,</div><div class='add'>+	UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_event_file),</div><div class='add'>+			     devx_async_event_destroy_uobj,</div><div class='add'>+			     &amp;devx_async_event_fops, "[devx_async_event]",</div><div class='add'>+			     O_RDONLY),</div><div class='add'>+	&amp;UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC));</div><div class='add'>+</div><div class='ctx'> static bool devx_is_supported(struct ib_device *device)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='hunk'>@@ -1923,5 +3235,8 @@ const struct uapi_definition mlx5_ib_devx_defs[] = {</div><div class='ctx'> 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(</div><div class='ctx'> 		MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,</div><div class='ctx'> 		UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(</div><div class='add'>+		MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,</div><div class='add'>+		UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),</div><div class='ctx'> 	{},</div><div class='ctx'> };</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/devx.h b/drivers/infiniband/hw/mlx5/devx.h<br/>new file mode 100644<br/>index 000000000000..ee9e7d3af93f<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/devx.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/devx.h</a></div><div class='hunk'>@@ -0,0 +1,50 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2019-2020, Mellanox Technologies inc. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef _MLX5_IB_DEVX_H</div><div class='add'>+#define _MLX5_IB_DEVX_H</div><div class='add'>+</div><div class='add'>+#include "mlx5_ib.h"</div><div class='add'>+</div><div class='add'>+#define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)</div><div class='add'>+struct devx_obj {</div><div class='add'>+	struct mlx5_ib_dev	*ib_dev;</div><div class='add'>+	u64			obj_id;</div><div class='add'>+	u32			dinlen; /* destroy inbox length */</div><div class='add'>+	u32			dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];</div><div class='add'>+	u32			flags;</div><div class='add'>+	union {</div><div class='add'>+		struct mlx5_ib_mkey	mkey;</div><div class='add'>+		struct mlx5_core_dct	core_dct;</div><div class='add'>+		struct mlx5_core_cq	core_cq;</div><div class='add'>+		u32			flow_counter_bulk_size;</div><div class='add'>+	};</div><div class='add'>+	struct list_head event_sub; /* holds devx_event_subscription entries */</div><div class='add'>+};</div><div class='add'>+#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)</div><div class='add'>+int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user, u64 req_ucaps);</div><div class='add'>+void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid);</div><div class='add'>+int mlx5_ib_devx_init(struct mlx5_ib_dev *dev);</div><div class='add'>+void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev);</div><div class='add'>+void mlx5_ib_ufile_hw_cleanup(struct ib_uverbs_file *ufile);</div><div class='add'>+#else</div><div class='add'>+static inline int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user,</div><div class='add'>+				      u64 req_ucaps)</div><div class='add'>+{</div><div class='add'>+	return -EOPNOTSUPP;</div><div class='add'>+}</div><div class='add'>+static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) {}</div><div class='add'>+static inline int mlx5_ib_devx_init(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+static inline void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+}</div><div class='add'>+static inline void mlx5_ib_ufile_hw_cleanup(struct ib_uverbs_file *ufile)</div><div class='add'>+{</div><div class='add'>+}</div><div class='add'>+#endif</div><div class='add'>+#endif /* _MLX5_IB_DEVX_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/dm.c b/drivers/infiniband/hw/mlx5/dm.c<br/>new file mode 100644<br/>index 000000000000..9ded2b7c1e31<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/dm.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/dm.c</a></div><div class='hunk'>@@ -0,0 +1,612 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2021, Mellanox Technologies inc. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/uverbs_std_types.h&gt;</div><div class='add'>+#include "dm.h"</div><div class='add'>+</div><div class='add'>+#define UVERBS_MODULE_NAME mlx5_ib</div><div class='add'>+#include &lt;rdma/uverbs_named_ioctl.h&gt;</div><div class='add'>+</div><div class='add'>+static int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,</div><div class='add'>+				u64 length, u32 alignment)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_core_dev *dev = dm-&gt;dev;</div><div class='add'>+	u64 num_memic_hw_pages = MLX5_CAP_DEV_MEM(dev, memic_bar_size)</div><div class='add'>+					&gt;&gt; PAGE_SHIFT;</div><div class='add'>+	u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);</div><div class='add'>+	u32 max_alignment = MLX5_CAP_DEV_MEM(dev, log_max_memic_addr_alignment);</div><div class='add'>+	u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(alloc_memic_out)] = {};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(alloc_memic_in)] = {};</div><div class='add'>+	u32 mlx5_alignment;</div><div class='add'>+	u64 page_idx = 0;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	if (!length || (length &amp; MLX5_MEMIC_ALLOC_SIZE_MASK))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	/* mlx5 device sets alignment as 64*2^driver_value</div><div class='add'>+	 * so normalizing is needed.</div><div class='add'>+	 */</div><div class='add'>+	mlx5_alignment = (alignment &lt; MLX5_MEMIC_BASE_ALIGN) ? 0 :</div><div class='add'>+			 alignment - MLX5_MEMIC_BASE_ALIGN;</div><div class='add'>+	if (mlx5_alignment &gt; max_alignment)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(alloc_memic_in, in, opcode, MLX5_CMD_OP_ALLOC_MEMIC);</div><div class='add'>+	MLX5_SET(alloc_memic_in, in, range_size, num_pages * PAGE_SIZE);</div><div class='add'>+	MLX5_SET(alloc_memic_in, in, memic_size, length);</div><div class='add'>+	MLX5_SET(alloc_memic_in, in, log_memic_addr_alignment,</div><div class='add'>+		 mlx5_alignment);</div><div class='add'>+</div><div class='add'>+	while (page_idx &lt; num_memic_hw_pages) {</div><div class='add'>+		spin_lock(&amp;dm-&gt;lock);</div><div class='add'>+		page_idx = bitmap_find_next_zero_area(dm-&gt;memic_alloc_pages,</div><div class='add'>+						      num_memic_hw_pages,</div><div class='add'>+						      page_idx,</div><div class='add'>+						      num_pages, 0);</div><div class='add'>+</div><div class='add'>+		if (page_idx &lt; num_memic_hw_pages)</div><div class='add'>+			bitmap_set(dm-&gt;memic_alloc_pages,</div><div class='add'>+				   page_idx, num_pages);</div><div class='add'>+</div><div class='add'>+		spin_unlock(&amp;dm-&gt;lock);</div><div class='add'>+</div><div class='add'>+		if (page_idx &gt;= num_memic_hw_pages)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		MLX5_SET64(alloc_memic_in, in, range_start_addr,</div><div class='add'>+			   hw_start_addr + (page_idx * PAGE_SIZE));</div><div class='add'>+</div><div class='add'>+		ret = mlx5_cmd_exec_inout(dev, alloc_memic, in, out);</div><div class='add'>+		if (ret) {</div><div class='add'>+			spin_lock(&amp;dm-&gt;lock);</div><div class='add'>+			bitmap_clear(dm-&gt;memic_alloc_pages,</div><div class='add'>+				     page_idx, num_pages);</div><div class='add'>+			spin_unlock(&amp;dm-&gt;lock);</div><div class='add'>+</div><div class='add'>+			if (ret == -EAGAIN) {</div><div class='add'>+				page_idx++;</div><div class='add'>+				continue;</div><div class='add'>+			}</div><div class='add'>+</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		*addr = dev-&gt;bar_addr +</div><div class='add'>+			MLX5_GET64(alloc_memic_out, out, memic_start_addr);</div><div class='add'>+</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr,</div><div class='add'>+			    u64 length)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_core_dev *dev = dm-&gt;dev;</div><div class='add'>+	u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);</div><div class='add'>+	u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(dealloc_memic_in)] = {};</div><div class='add'>+	u64 start_page_idx;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	addr -= dev-&gt;bar_addr;</div><div class='add'>+	start_page_idx = (addr - hw_start_addr) &gt;&gt; PAGE_SHIFT;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(dealloc_memic_in, in, opcode, MLX5_CMD_OP_DEALLOC_MEMIC);</div><div class='add'>+	MLX5_SET64(dealloc_memic_in, in, memic_start_addr, addr);</div><div class='add'>+	MLX5_SET(dealloc_memic_in, in, memic_size, length);</div><div class='add'>+</div><div class='add'>+	err =  mlx5_cmd_exec_in(dev, dealloc_memic, in);</div><div class='add'>+	if (err)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	spin_lock(&amp;dm-&gt;lock);</div><div class='add'>+	bitmap_clear(dm-&gt;memic_alloc_pages,</div><div class='add'>+		     start_page_idx, num_pages);</div><div class='add'>+	spin_unlock(&amp;dm-&gt;lock);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_cmd_dealloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr,</div><div class='add'>+			       u8 operation)</div><div class='add'>+{</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(modify_memic_in)] = {};</div><div class='add'>+	struct mlx5_core_dev *dev = dm-&gt;dev;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(modify_memic_in, in, opcode, MLX5_CMD_OP_MODIFY_MEMIC);</div><div class='add'>+	MLX5_SET(modify_memic_in, in, op_mod, MLX5_MODIFY_MEMIC_OP_MOD_DEALLOC);</div><div class='add'>+	MLX5_SET(modify_memic_in, in, memic_operation_type, operation);</div><div class='add'>+	MLX5_SET64(modify_memic_in, in, memic_start_addr, addr - dev-&gt;bar_addr);</div><div class='add'>+</div><div class='add'>+	mlx5_cmd_exec_in(dev, modify_memic, in);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_cmd_alloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr,</div><div class='add'>+				   u8 operation, phys_addr_t *op_addr)</div><div class='add'>+{</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(modify_memic_out)] = {};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(modify_memic_in)] = {};</div><div class='add'>+	struct mlx5_core_dev *dev = dm-&gt;dev;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(modify_memic_in, in, opcode, MLX5_CMD_OP_MODIFY_MEMIC);</div><div class='add'>+	MLX5_SET(modify_memic_in, in, op_mod, MLX5_MODIFY_MEMIC_OP_MOD_ALLOC);</div><div class='add'>+	MLX5_SET(modify_memic_in, in, memic_operation_type, operation);</div><div class='add'>+	MLX5_SET64(modify_memic_in, in, memic_start_addr, addr - dev-&gt;bar_addr);</div><div class='add'>+</div><div class='add'>+	err = mlx5_cmd_exec_inout(dev, modify_memic, in, out);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	*op_addr = dev-&gt;bar_addr +</div><div class='add'>+		   MLX5_GET64(modify_memic_out, out, memic_operation_addr);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int add_dm_mmap_entry(struct ib_ucontext *context,</div><div class='add'>+			     struct mlx5_user_mmap_entry *mentry, u8 mmap_flag,</div><div class='add'>+			     size_t size, u64 address)</div><div class='add'>+{</div><div class='add'>+	mentry-&gt;mmap_flag = mmap_flag;</div><div class='add'>+	mentry-&gt;address = address;</div><div class='add'>+</div><div class='add'>+	return rdma_user_mmap_entry_insert_range(</div><div class='add'>+		context, &amp;mentry-&gt;rdma_entry, size,</div><div class='add'>+		MLX5_IB_MMAP_DEVICE_MEM &lt;&lt; 16,</div><div class='add'>+		(MLX5_IB_MMAP_DEVICE_MEM &lt;&lt; 16) + (1UL &lt;&lt; 16) - 1);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_dm_memic_free(struct kref *kref)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dm_memic *dm =</div><div class='add'>+		container_of(kref, struct mlx5_ib_dm_memic, ref);</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(dm-&gt;base.ibdm.device);</div><div class='add'>+</div><div class='add'>+	mlx5_cmd_dealloc_memic(&amp;dev-&gt;dm, dm-&gt;base.dev_addr, dm-&gt;base.size);</div><div class='add'>+	kfree(dm);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int copy_op_to_user(struct mlx5_ib_dm_op_entry *op_entry,</div><div class='add'>+			   struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	u64 start_offset;</div><div class='add'>+	u16 page_idx;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	page_idx = op_entry-&gt;mentry.rdma_entry.start_pgoff &amp; 0xFFFF;</div><div class='add'>+	start_offset = op_entry-&gt;op_addr &amp; ~PAGE_MASK;</div><div class='add'>+	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_PAGE_INDEX,</div><div class='add'>+			     &amp;page_idx, sizeof(page_idx));</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	return uverbs_copy_to(attrs,</div><div class='add'>+			      MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_START_OFFSET,</div><div class='add'>+			      &amp;start_offset, sizeof(start_offset));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int map_existing_op(struct mlx5_ib_dm_memic *dm, u8 op,</div><div class='add'>+			   struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dm_op_entry *op_entry;</div><div class='add'>+</div><div class='add'>+	op_entry = xa_load(&amp;dm-&gt;ops, op);</div><div class='add'>+	if (!op_entry)</div><div class='add'>+		return -ENOENT;</div><div class='add'>+</div><div class='add'>+	return copy_op_to_user(op_entry, attrs);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(MLX5_IB_METHOD_DM_MAP_OP_ADDR)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *uobj = uverbs_attr_get_uobject(</div><div class='add'>+		attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_HANDLE);</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(uobj-&gt;context-&gt;device);</div><div class='add'>+	struct ib_dm *ibdm = uobj-&gt;object;</div><div class='add'>+	struct mlx5_ib_dm_memic *dm = to_memic(ibdm);</div><div class='add'>+	struct mlx5_ib_dm_op_entry *op_entry;</div><div class='add'>+	int err;</div><div class='add'>+	u8 op;</div><div class='add'>+</div><div class='add'>+	err = uverbs_copy_from(&amp;op, attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_OP);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	if (op &gt;= BITS_PER_TYPE(u32))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (!(MLX5_CAP_DEV_MEM(dev-&gt;mdev, memic_operations) &amp; BIT(op)))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;dm-&gt;ops_xa_lock);</div><div class='add'>+	err = map_existing_op(dm, op, attrs);</div><div class='add'>+	if (!err || err != -ENOENT)</div><div class='add'>+		goto err_unlock;</div><div class='add'>+</div><div class='add'>+	op_entry = kzalloc(sizeof(*op_entry), GFP_KERNEL);</div><div class='add'>+	if (!op_entry)</div><div class='add'>+		goto err_unlock;</div><div class='add'>+</div><div class='add'>+	err = mlx5_cmd_alloc_memic_op(&amp;dev-&gt;dm, dm-&gt;base.dev_addr, op,</div><div class='add'>+				      &amp;op_entry-&gt;op_addr);</div><div class='add'>+	if (err) {</div><div class='add'>+		kfree(op_entry);</div><div class='add'>+		goto err_unlock;</div><div class='add'>+	}</div><div class='add'>+	op_entry-&gt;op = op;</div><div class='add'>+	op_entry-&gt;dm = dm;</div><div class='add'>+</div><div class='add'>+	err = add_dm_mmap_entry(uobj-&gt;context, &amp;op_entry-&gt;mentry,</div><div class='add'>+				MLX5_IB_MMAP_TYPE_MEMIC_OP, dm-&gt;base.size,</div><div class='add'>+				op_entry-&gt;op_addr &amp; PAGE_MASK);</div><div class='add'>+	if (err) {</div><div class='add'>+		mlx5_cmd_dealloc_memic_op(&amp;dev-&gt;dm, dm-&gt;base.dev_addr, op);</div><div class='add'>+		kfree(op_entry);</div><div class='add'>+		goto err_unlock;</div><div class='add'>+	}</div><div class='add'>+	/* From this point, entry will be freed by mmap_free */</div><div class='add'>+	kref_get(&amp;dm-&gt;ref);</div><div class='add'>+</div><div class='add'>+	err = copy_op_to_user(op_entry, attrs);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_remove;</div><div class='add'>+</div><div class='add'>+	err = xa_insert(&amp;dm-&gt;ops, op, op_entry, GFP_KERNEL);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_remove;</div><div class='add'>+	mutex_unlock(&amp;dm-&gt;ops_xa_lock);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_remove:</div><div class='add'>+	rdma_user_mmap_entry_remove(&amp;op_entry-&gt;mentry.rdma_entry);</div><div class='add'>+err_unlock:</div><div class='add'>+	mutex_unlock(&amp;dm-&gt;ops_xa_lock);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct ib_dm *handle_alloc_dm_memic(struct ib_ucontext *ctx,</div><div class='add'>+					   struct ib_dm_alloc_attr *attr,</div><div class='add'>+					   struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_dm *dm_db = &amp;to_mdev(ctx-&gt;device)-&gt;dm;</div><div class='add'>+	struct mlx5_ib_dm_memic *dm;</div><div class='add'>+	u64 start_offset;</div><div class='add'>+	u16 page_idx;</div><div class='add'>+	int err;</div><div class='add'>+	u64 address;</div><div class='add'>+</div><div class='add'>+	if (!dm_db || !MLX5_CAP_DEV_MEM(dm_db-&gt;dev, memic))</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	dm = kzalloc(sizeof(*dm), GFP_KERNEL);</div><div class='add'>+	if (!dm)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	dm-&gt;base.type = MLX5_IB_UAPI_DM_TYPE_MEMIC;</div><div class='add'>+	dm-&gt;base.size = roundup(attr-&gt;length, MLX5_MEMIC_BASE_SIZE);</div><div class='add'>+	dm-&gt;base.ibdm.device = ctx-&gt;device;</div><div class='add'>+</div><div class='add'>+	kref_init(&amp;dm-&gt;ref);</div><div class='add'>+	xa_init(&amp;dm-&gt;ops);</div><div class='add'>+	mutex_init(&amp;dm-&gt;ops_xa_lock);</div><div class='add'>+	dm-&gt;req_length = attr-&gt;length;</div><div class='add'>+</div><div class='add'>+	err = mlx5_cmd_alloc_memic(dm_db, &amp;dm-&gt;base.dev_addr,</div><div class='add'>+				   dm-&gt;base.size, attr-&gt;alignment);</div><div class='add'>+	if (err) {</div><div class='add'>+		kfree(dm);</div><div class='add'>+		return ERR_PTR(err);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	address = dm-&gt;base.dev_addr &amp; PAGE_MASK;</div><div class='add'>+	err = add_dm_mmap_entry(ctx, &amp;dm-&gt;mentry, MLX5_IB_MMAP_TYPE_MEMIC,</div><div class='add'>+				dm-&gt;base.size, address);</div><div class='add'>+	if (err) {</div><div class='add'>+		mlx5_cmd_dealloc_memic(dm_db, dm-&gt;base.dev_addr, dm-&gt;base.size);</div><div class='add'>+		kfree(dm);</div><div class='add'>+		return ERR_PTR(err);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	page_idx = dm-&gt;mentry.rdma_entry.start_pgoff &amp; 0xFFFF;</div><div class='add'>+	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,</div><div class='add'>+			     &amp;page_idx, sizeof(page_idx));</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_copy;</div><div class='add'>+</div><div class='add'>+	start_offset = dm-&gt;base.dev_addr &amp; ~PAGE_MASK;</div><div class='add'>+	err = uverbs_copy_to(attrs,</div><div class='add'>+			     MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,</div><div class='add'>+			     &amp;start_offset, sizeof(start_offset));</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_copy;</div><div class='add'>+</div><div class='add'>+	return &amp;dm-&gt;base.ibdm;</div><div class='add'>+</div><div class='add'>+err_copy:</div><div class='add'>+	rdma_user_mmap_entry_remove(&amp;dm-&gt;mentry.rdma_entry);</div><div class='add'>+	return ERR_PTR(err);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static enum mlx5_sw_icm_type get_icm_type(int uapi_type)</div><div class='add'>+{</div><div class='add'>+	switch (uapi_type) {</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:</div><div class='add'>+		return MLX5_SW_ICM_TYPE_HEADER_MODIFY;</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_PATTERN_SW_ICM:</div><div class='add'>+		return MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN;</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_ENCAP_SW_ICM:</div><div class='add'>+		return MLX5_SW_ICM_TYPE_SW_ENCAP;</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:</div><div class='add'>+	default:</div><div class='add'>+		return MLX5_SW_ICM_TYPE_STEERING;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct ib_dm *handle_alloc_dm_sw_icm(struct ib_ucontext *ctx,</div><div class='add'>+					    struct ib_dm_alloc_attr *attr,</div><div class='add'>+					    struct uverbs_attr_bundle *attrs,</div><div class='add'>+					    int type)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_core_dev *dev = to_mdev(ctx-&gt;device)-&gt;mdev;</div><div class='add'>+	enum mlx5_sw_icm_type icm_type;</div><div class='add'>+	struct mlx5_ib_dm_icm *dm;</div><div class='add'>+	u64 act_size;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (!capable(CAP_SYS_RAWIO) || !capable(CAP_NET_RAW))</div><div class='add'>+		return ERR_PTR(-EPERM);</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_ENCAP_SW_ICM:</div><div class='add'>+		if (!(MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner) ||</div><div class='add'>+		      MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner) ||</div><div class='add'>+		      MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner_v2) ||</div><div class='add'>+		      MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner_v2)))</div><div class='add'>+			return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_PATTERN_SW_ICM:</div><div class='add'>+		if (!MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner_v2) ||</div><div class='add'>+		    !MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner_v2))</div><div class='add'>+			return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dm = kzalloc(sizeof(*dm), GFP_KERNEL);</div><div class='add'>+	if (!dm)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	dm-&gt;base.type = type;</div><div class='add'>+	dm-&gt;base.ibdm.device = ctx-&gt;device;</div><div class='add'>+</div><div class='add'>+	/* Allocation size must a multiple of the basic block size</div><div class='add'>+	 * and a power of 2.</div><div class='add'>+	 */</div><div class='add'>+	act_size = round_up(attr-&gt;length, MLX5_SW_ICM_BLOCK_SIZE(dev));</div><div class='add'>+	act_size = roundup_pow_of_two(act_size);</div><div class='add'>+</div><div class='add'>+	dm-&gt;base.size = act_size;</div><div class='add'>+	icm_type = get_icm_type(type);</div><div class='add'>+</div><div class='add'>+	err = mlx5_dm_sw_icm_alloc(dev, icm_type, act_size, attr-&gt;alignment,</div><div class='add'>+				   to_mucontext(ctx)-&gt;devx_uid,</div><div class='add'>+				   &amp;dm-&gt;base.dev_addr, &amp;dm-&gt;obj_id);</div><div class='add'>+	if (err)</div><div class='add'>+		goto free;</div><div class='add'>+</div><div class='add'>+	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,</div><div class='add'>+			     &amp;dm-&gt;base.dev_addr, sizeof(dm-&gt;base.dev_addr));</div><div class='add'>+	if (err) {</div><div class='add'>+		mlx5_dm_sw_icm_dealloc(dev, icm_type, dm-&gt;base.size,</div><div class='add'>+				       to_mucontext(ctx)-&gt;devx_uid,</div><div class='add'>+				       dm-&gt;base.dev_addr, dm-&gt;obj_id);</div><div class='add'>+		goto free;</div><div class='add'>+	}</div><div class='add'>+	return &amp;dm-&gt;base.ibdm;</div><div class='add'>+free:</div><div class='add'>+	kfree(dm);</div><div class='add'>+	return ERR_PTR(err);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,</div><div class='add'>+			       struct ib_ucontext *context,</div><div class='add'>+			       struct ib_dm_alloc_attr *attr,</div><div class='add'>+			       struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	enum mlx5_ib_uapi_dm_type type;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	err = uverbs_get_const_default(&amp;type, attrs,</div><div class='add'>+				       MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,</div><div class='add'>+				       MLX5_IB_UAPI_DM_TYPE_MEMIC);</div><div class='add'>+	if (err)</div><div class='add'>+		return ERR_PTR(err);</div><div class='add'>+</div><div class='add'>+	mlx5_ib_dbg(to_mdev(ibdev), "alloc_dm req: dm_type=%d user_length=0x%llx log_alignment=%d\n",</div><div class='add'>+		    type, attr-&gt;length, attr-&gt;alignment);</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_MEMIC:</div><div class='add'>+		return handle_alloc_dm_memic(context, attr, attrs);</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_PATTERN_SW_ICM:</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_ENCAP_SW_ICM:</div><div class='add'>+		return handle_alloc_dm_sw_icm(context, attr, attrs, type);</div><div class='add'>+	default:</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void dm_memic_remove_ops(struct mlx5_ib_dm_memic *dm)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dm_op_entry *entry;</div><div class='add'>+	unsigned long idx;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;dm-&gt;ops_xa_lock);</div><div class='add'>+	xa_for_each(&amp;dm-&gt;ops, idx, entry) {</div><div class='add'>+		xa_erase(&amp;dm-&gt;ops, idx);</div><div class='add'>+		rdma_user_mmap_entry_remove(&amp;entry-&gt;mentry.rdma_entry);</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;dm-&gt;ops_xa_lock);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_dm_memic_dealloc(struct mlx5_ib_dm_memic *dm)</div><div class='add'>+{</div><div class='add'>+	dm_memic_remove_ops(dm);</div><div class='add'>+	rdma_user_mmap_entry_remove(&amp;dm-&gt;mentry.rdma_entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_dm_icm_dealloc(struct mlx5_ib_ucontext *ctx,</div><div class='add'>+			       struct mlx5_ib_dm_icm *dm)</div><div class='add'>+{</div><div class='add'>+	enum mlx5_sw_icm_type type = get_icm_type(dm-&gt;base.type);</div><div class='add'>+	struct mlx5_core_dev *dev = to_mdev(dm-&gt;base.ibdm.device)-&gt;mdev;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	err = mlx5_dm_sw_icm_dealloc(dev, type, dm-&gt;base.size, ctx-&gt;devx_uid,</div><div class='add'>+				     dm-&gt;base.dev_addr, dm-&gt;obj_id);</div><div class='add'>+	if (!err)</div><div class='add'>+		kfree(dm);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_dealloc_dm(struct ib_dm *ibdm,</div><div class='add'>+			      struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_ucontext *ctx = rdma_udata_to_drv_context(</div><div class='add'>+		&amp;attrs-&gt;driver_udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='add'>+	struct mlx5_ib_dm *dm = to_mdm(ibdm);</div><div class='add'>+</div><div class='add'>+	switch (dm-&gt;type) {</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_MEMIC:</div><div class='add'>+		mlx5_dm_memic_dealloc(to_memic(ibdm));</div><div class='add'>+		return 0;</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_PATTERN_SW_ICM:</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_ENCAP_SW_ICM:</div><div class='add'>+		return mlx5_dm_icm_dealloc(ctx, to_icm(ibdm));</div><div class='add'>+	default:</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(MLX5_IB_METHOD_DM_QUERY)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_dm *ibdm =</div><div class='add'>+		uverbs_attr_get_obj(attrs, MLX5_IB_ATTR_QUERY_DM_REQ_HANDLE);</div><div class='add'>+	struct mlx5_ib_dm *dm = to_mdm(ibdm);</div><div class='add'>+	struct mlx5_ib_dm_memic *memic;</div><div class='add'>+	u64 start_offset;</div><div class='add'>+	u16 page_idx;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (dm-&gt;type != MLX5_IB_UAPI_DM_TYPE_MEMIC)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	memic = to_memic(ibdm);</div><div class='add'>+	page_idx = memic-&gt;mentry.rdma_entry.start_pgoff &amp; 0xFFFF;</div><div class='add'>+	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_PAGE_INDEX,</div><div class='add'>+			     &amp;page_idx, sizeof(page_idx));</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	start_offset = memic-&gt;base.dev_addr &amp; ~PAGE_MASK;</div><div class='add'>+	err =  uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_START_OFFSET,</div><div class='add'>+			      &amp;start_offset, sizeof(start_offset));</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	return uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_LENGTH,</div><div class='add'>+			      &amp;memic-&gt;req_length,</div><div class='add'>+			      sizeof(memic-&gt;req_length));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_ib_dm_mmap_free(struct mlx5_ib_dev *dev,</div><div class='add'>+			  struct mlx5_user_mmap_entry *mentry)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dm_op_entry *op_entry;</div><div class='add'>+	struct mlx5_ib_dm_memic *mdm;</div><div class='add'>+</div><div class='add'>+	switch (mentry-&gt;mmap_flag) {</div><div class='add'>+	case MLX5_IB_MMAP_TYPE_MEMIC:</div><div class='add'>+		mdm = container_of(mentry, struct mlx5_ib_dm_memic, mentry);</div><div class='add'>+		kref_put(&amp;mdm-&gt;ref, mlx5_ib_dm_memic_free);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_MMAP_TYPE_MEMIC_OP:</div><div class='add'>+		op_entry = container_of(mentry, struct mlx5_ib_dm_op_entry,</div><div class='add'>+					mentry);</div><div class='add'>+		mdm = op_entry-&gt;dm;</div><div class='add'>+		mlx5_cmd_dealloc_memic_op(&amp;dev-&gt;dm, mdm-&gt;base.dev_addr,</div><div class='add'>+					  op_entry-&gt;op);</div><div class='add'>+		kfree(op_entry);</div><div class='add'>+		kref_put(&amp;mdm-&gt;ref, mlx5_ib_dm_memic_free);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		WARN_ON(true);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	MLX5_IB_METHOD_DM_QUERY,</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_QUERY_DM_REQ_HANDLE, UVERBS_OBJECT_DM,</div><div class='add'>+			UVERBS_ACCESS_READ, UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_START_OFFSET,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u64), UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_PAGE_INDEX,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u16), UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_LENGTH,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u64), UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+ADD_UVERBS_ATTRIBUTES_SIMPLE(</div><div class='add'>+	mlx5_ib_dm, UVERBS_OBJECT_DM, UVERBS_METHOD_DM_ALLOC,</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u64), UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,</div><div class='add'>+			     enum mlx5_ib_uapi_dm_type, UA_OPTIONAL));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	MLX5_IB_METHOD_DM_MAP_OP_ADDR,</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_DM,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_OP,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u8),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_START_OFFSET,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			    UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_PAGE_INDEX,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u16),</div><div class='add'>+			    UA_OPTIONAL));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DM,</div><div class='add'>+			      &amp;UVERBS_METHOD(MLX5_IB_METHOD_DM_MAP_OP_ADDR),</div><div class='add'>+			      &amp;UVERBS_METHOD(MLX5_IB_METHOD_DM_QUERY));</div><div class='add'>+</div><div class='add'>+const struct uapi_definition mlx5_ib_dm_defs[] = {</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &amp;mlx5_ib_dm),</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DM),</div><div class='add'>+	{},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+const struct ib_device_ops mlx5_ib_dev_dm_ops = {</div><div class='add'>+	.alloc_dm = mlx5_ib_alloc_dm,</div><div class='add'>+	.dealloc_dm = mlx5_ib_dealloc_dm,</div><div class='add'>+	.reg_dm_mr = mlx5_ib_reg_dm_mr,</div><div class='add'>+};</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/dm.h b/drivers/infiniband/hw/mlx5/dm.h<br/>new file mode 100644<br/>index 000000000000..9674a80d8d70<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/dm.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/dm.h</a></div><div class='hunk'>@@ -0,0 +1,68 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2021, Mellanox Technologies inc. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef _MLX5_IB_DM_H</div><div class='add'>+#define _MLX5_IB_DM_H</div><div class='add'>+</div><div class='add'>+#include "mlx5_ib.h"</div><div class='add'>+</div><div class='add'>+extern const struct ib_device_ops mlx5_ib_dev_dm_ops;</div><div class='add'>+extern const struct uapi_definition mlx5_ib_dm_defs[];</div><div class='add'>+</div><div class='add'>+struct mlx5_ib_dm {</div><div class='add'>+	struct ib_dm		ibdm;</div><div class='add'>+	u32			type;</div><div class='add'>+	phys_addr_t		dev_addr;</div><div class='add'>+	size_t			size;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mlx5_ib_dm_op_entry {</div><div class='add'>+	struct mlx5_user_mmap_entry	mentry;</div><div class='add'>+	phys_addr_t			op_addr;</div><div class='add'>+	struct mlx5_ib_dm_memic		*dm;</div><div class='add'>+	u8				op;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mlx5_ib_dm_memic {</div><div class='add'>+	struct mlx5_ib_dm           base;</div><div class='add'>+	struct mlx5_user_mmap_entry mentry;</div><div class='add'>+	struct xarray               ops;</div><div class='add'>+	struct mutex                ops_xa_lock;</div><div class='add'>+	struct kref                 ref;</div><div class='add'>+	size_t                      req_length;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mlx5_ib_dm_icm {</div><div class='add'>+	struct mlx5_ib_dm      base;</div><div class='add'>+	u32                    obj_id;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline struct mlx5_ib_dm *to_mdm(struct ib_dm *ibdm)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibdm, struct mlx5_ib_dm, ibdm);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct mlx5_ib_dm_memic *to_memic(struct ib_dm *ibdm)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibdm, struct mlx5_ib_dm_memic, base.ibdm);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline struct mlx5_ib_dm_icm *to_icm(struct ib_dm *ibdm)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibdm, struct mlx5_ib_dm_icm, base.ibdm);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,</div><div class='add'>+			       struct ib_ucontext *context,</div><div class='add'>+			       struct ib_dm_alloc_attr *attr,</div><div class='add'>+			       struct uverbs_attr_bundle *attrs);</div><div class='add'>+void mlx5_ib_dm_mmap_free(struct mlx5_ib_dev *dev,</div><div class='add'>+			  struct mlx5_user_mmap_entry *mentry);</div><div class='add'>+void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr,</div><div class='add'>+			    u64 length);</div><div class='add'>+void mlx5_cmd_dealloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr,</div><div class='add'>+			       u8 operation);</div><div class='add'>+</div><div class='add'>+#endif /* _MLX5_IB_DM_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/dmah.c b/drivers/infiniband/hw/mlx5/dmah.c<br/>new file mode 100644<br/>index 000000000000..362a88992ffa<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/dmah.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/dmah.c</a></div><div class='hunk'>@@ -0,0 +1,54 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2025, NVIDIA CORPORATION &amp; AFFILIATES. All rights reserved</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/uverbs_std_types.h&gt;</div><div class='add'>+#include &lt;linux/pci-tph.h&gt;</div><div class='add'>+#include "dmah.h"</div><div class='add'>+</div><div class='add'>+#define UVERBS_MODULE_NAME mlx5_ib</div><div class='add'>+#include &lt;rdma/uverbs_named_ioctl.h&gt;</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_alloc_dmah(struct ib_dmah *ibdmah,</div><div class='add'>+			      struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_core_dev *mdev = to_mdev(ibdmah-&gt;device)-&gt;mdev;</div><div class='add'>+	struct mlx5_ib_dmah *dmah = to_mdmah(ibdmah);</div><div class='add'>+	u16 st_bits = BIT(IB_DMAH_CPU_ID_EXISTS) |</div><div class='add'>+		      BIT(IB_DMAH_MEM_TYPE_EXISTS);</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	/* PH is a must for TPH following PCIe spec 6.2-1.0 */</div><div class='add'>+	if (!(ibdmah-&gt;valid_fields &amp; BIT(IB_DMAH_PH_EXISTS)))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	/* ST is optional; however, partial data for it is not allowed */</div><div class='add'>+	if (ibdmah-&gt;valid_fields &amp; st_bits) {</div><div class='add'>+		if ((ibdmah-&gt;valid_fields &amp; st_bits) != st_bits)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		err = mlx5_st_alloc_index(mdev, ibdmah-&gt;mem_type,</div><div class='add'>+					  ibdmah-&gt;cpu_id, &amp;dmah-&gt;st_index);</div><div class='add'>+		if (err)</div><div class='add'>+			return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_dealloc_dmah(struct ib_dmah *ibdmah,</div><div class='add'>+				struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dmah *dmah = to_mdmah(ibdmah);</div><div class='add'>+	struct mlx5_core_dev *mdev = to_mdev(ibdmah-&gt;device)-&gt;mdev;</div><div class='add'>+</div><div class='add'>+	if (ibdmah-&gt;valid_fields &amp; BIT(IB_DMAH_CPU_ID_EXISTS))</div><div class='add'>+		return mlx5_st_dealloc_index(mdev, dmah-&gt;st_index);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+const struct ib_device_ops mlx5_ib_dev_dmah_ops = {</div><div class='add'>+	.alloc_dmah = mlx5_ib_alloc_dmah,</div><div class='add'>+	.dealloc_dmah = mlx5_ib_dealloc_dmah,</div><div class='add'>+};</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/dmah.h b/drivers/infiniband/hw/mlx5/dmah.h<br/>new file mode 100644<br/>index 000000000000..68de72b4744a<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/dmah.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/dmah.h</a></div><div class='hunk'>@@ -0,0 +1,23 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2025, NVIDIA CORPORATION &amp; AFFILIATES. All rights reserved</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef _MLX5_IB_DMAH_H</div><div class='add'>+#define _MLX5_IB_DMAH_H</div><div class='add'>+</div><div class='add'>+#include "mlx5_ib.h"</div><div class='add'>+</div><div class='add'>+extern const struct ib_device_ops mlx5_ib_dev_dmah_ops;</div><div class='add'>+</div><div class='add'>+struct mlx5_ib_dmah {</div><div class='add'>+	struct ib_dmah ibdmah;</div><div class='add'>+	u16 st_index;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static inline struct mlx5_ib_dmah *to_mdmah(struct ib_dmah *ibdmah)</div><div class='add'>+{</div><div class='add'>+	return container_of(ibdmah, struct mlx5_ib_dmah, ibdmah);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#endif /* _MLX5_IB_DMAH_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/doorbell.c b/drivers/infiniband/hw/mlx5/doorbell.c<br/>index 8f4e5f22b84c..e32111117a5e 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/doorbell.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/doorbell.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/doorbell.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/doorbell.c</a></div><div class='hunk'>@@ -32,6 +32,7 @@</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/kref.h&gt;</div><div class='ctx'> #include &lt;linux/slab.h&gt;</div><div class='add'>+#include &lt;linux/sched/mm.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_umem.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "mlx5_ib.h"</div><div class='hunk'>@@ -41,10 +42,10 @@ struct mlx5_ib_user_db_page {</div><div class='ctx'> 	struct ib_umem	       *umem;</div><div class='ctx'> 	unsigned long		user_virt;</div><div class='ctx'> 	int			refcnt;</div><div class='add'>+	struct mm_struct	*mm;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context,</div><div class='del'>-			struct ib_udata *udata, unsigned long virt,</div><div class='add'>+int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt,</div><div class='ctx'> 			struct mlx5_db *db)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_user_db_page *page;</div><div class='hunk'>@@ -53,7 +54,8 @@ int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context,</div><div class='ctx'> 	mutex_lock(&amp;context-&gt;db_page_mutex);</div><div class='ctx'> </div><div class='ctx'> 	list_for_each_entry(page, &amp;context-&gt;db_page_list, list)</div><div class='del'>-		if (page-&gt;user_virt == (virt &amp; PAGE_MASK))</div><div class='add'>+		if ((current-&gt;mm == page-&gt;mm) &amp;&amp;</div><div class='add'>+		    (page-&gt;user_virt == (virt &amp; PAGE_MASK)))</div><div class='ctx'> 			goto found;</div><div class='ctx'> </div><div class='ctx'> 	page = kmalloc(sizeof(*page), GFP_KERNEL);</div><div class='hunk'>@@ -64,17 +66,21 @@ int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context,</div><div class='ctx'> </div><div class='ctx'> 	page-&gt;user_virt = (virt &amp; PAGE_MASK);</div><div class='ctx'> 	page-&gt;refcnt    = 0;</div><div class='del'>-	page-&gt;umem = ib_umem_get(udata, virt &amp; PAGE_MASK, PAGE_SIZE, 0, 0);</div><div class='add'>+	page-&gt;umem = ib_umem_get(context-&gt;ibucontext.device, virt &amp; PAGE_MASK,</div><div class='add'>+				 PAGE_SIZE, 0);</div><div class='ctx'> 	if (IS_ERR(page-&gt;umem)) {</div><div class='ctx'> 		err = PTR_ERR(page-&gt;umem);</div><div class='ctx'> 		kfree(page);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='add'>+	mmgrab(current-&gt;mm);</div><div class='add'>+	page-&gt;mm = current-&gt;mm;</div><div class='ctx'> </div><div class='ctx'> 	list_add(&amp;page-&gt;list, &amp;context-&gt;db_page_list);</div><div class='ctx'> </div><div class='ctx'> found:</div><div class='del'>-	db-&gt;dma = sg_dma_address(page-&gt;umem-&gt;sg_head.sgl) + (virt &amp; ~PAGE_MASK);</div><div class='add'>+	db-&gt;dma = sg_dma_address(page-&gt;umem-&gt;sgt_append.sgt.sgl) +</div><div class='add'>+		  (virt &amp; ~PAGE_MASK);</div><div class='ctx'> 	db-&gt;u.user_page = page;</div><div class='ctx'> 	++page-&gt;refcnt;</div><div class='ctx'> </div><div class='hunk'>@@ -90,6 +96,7 @@ void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db)</div><div class='ctx'> </div><div class='ctx'> 	if (!--db-&gt;u.user_page-&gt;refcnt) {</div><div class='ctx'> 		list_del(&amp;db-&gt;u.user_page-&gt;list);</div><div class='add'>+		mmdrop(db-&gt;u.user_page-&gt;mm);</div><div class='ctx'> 		ib_umem_release(db-&gt;u.user_page-&gt;umem);</div><div class='ctx'> 		kfree(db-&gt;u.user_page);</div><div class='ctx'> 	}</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/flow.c b/drivers/infiniband/hw/mlx5/flow.c<br/>deleted file mode 100644<br/>index 1fc302d41a53..000000000000<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/flow.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/flow.c</a><br/>+++ /dev/null</div><div class='hunk'>@@ -1,682 +0,0 @@</div><div class='del'>-// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='del'>-/*</div><div class='del'>- * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.</div><div class='del'>- */</div><div class='del'>-</div><div class='del'>-#include &lt;rdma/ib_user_verbs.h&gt;</div><div class='del'>-#include &lt;rdma/ib_verbs.h&gt;</div><div class='del'>-#include &lt;rdma/uverbs_types.h&gt;</div><div class='del'>-#include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='del'>-#include &lt;rdma/uverbs_std_types.h&gt;</div><div class='del'>-#include &lt;rdma/mlx5_user_ioctl_cmds.h&gt;</div><div class='del'>-#include &lt;rdma/mlx5_user_ioctl_verbs.h&gt;</div><div class='del'>-#include &lt;rdma/ib_umem.h&gt;</div><div class='del'>-#include &lt;linux/mlx5/driver.h&gt;</div><div class='del'>-#include &lt;linux/mlx5/fs.h&gt;</div><div class='del'>-#include "mlx5_ib.h"</div><div class='del'>-</div><div class='del'>-#define UVERBS_MODULE_NAME mlx5_ib</div><div class='del'>-#include &lt;rdma/uverbs_named_ioctl.h&gt;</div><div class='del'>-</div><div class='del'>-static int</div><div class='del'>-mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,</div><div class='del'>-			     enum mlx5_flow_namespace_type *namespace)</div><div class='del'>-{</div><div class='del'>-	switch (table_type) {</div><div class='del'>-	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX:</div><div class='del'>-		*namespace = MLX5_FLOW_NAMESPACE_BYPASS;</div><div class='del'>-		break;</div><div class='del'>-	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX:</div><div class='del'>-		*namespace = MLX5_FLOW_NAMESPACE_EGRESS;</div><div class='del'>-		break;</div><div class='del'>-	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB:</div><div class='del'>-		*namespace = MLX5_FLOW_NAMESPACE_FDB;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static const struct uverbs_attr_spec mlx5_ib_flow_type[] = {</div><div class='del'>-	[MLX5_IB_FLOW_TYPE_NORMAL] = {</div><div class='del'>-		.type = UVERBS_ATTR_TYPE_PTR_IN,</div><div class='del'>-		.u.ptr = {</div><div class='del'>-			.len = sizeof(u16), /* data is priority */</div><div class='del'>-			.min_len = sizeof(u16),</div><div class='del'>-		}</div><div class='del'>-	},</div><div class='del'>-	[MLX5_IB_FLOW_TYPE_SNIFFER] = {</div><div class='del'>-		.type = UVERBS_ATTR_TYPE_PTR_IN,</div><div class='del'>-		UVERBS_ATTR_NO_DATA(),</div><div class='del'>-	},</div><div class='del'>-	[MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = {</div><div class='del'>-		.type = UVERBS_ATTR_TYPE_PTR_IN,</div><div class='del'>-		UVERBS_ATTR_NO_DATA(),</div><div class='del'>-	},</div><div class='del'>-	[MLX5_IB_FLOW_TYPE_MC_DEFAULT] = {</div><div class='del'>-		.type = UVERBS_ATTR_TYPE_PTR_IN,</div><div class='del'>-		UVERBS_ATTR_NO_DATA(),</div><div class='del'>-	},</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2</div><div class='del'>-static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(</div><div class='del'>-	struct uverbs_attr_bundle *attrs)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_flow_act flow_act = {.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG};</div><div class='del'>-	struct mlx5_ib_flow_handler *flow_handler;</div><div class='del'>-	struct mlx5_ib_flow_matcher *fs_matcher;</div><div class='del'>-	struct ib_uobject **arr_flow_actions;</div><div class='del'>-	struct ib_uflow_resources *uflow_res;</div><div class='del'>-	void *devx_obj;</div><div class='del'>-	int dest_id, dest_type;</div><div class='del'>-	void *cmd_in;</div><div class='del'>-	int inlen;</div><div class='del'>-	bool dest_devx, dest_qp;</div><div class='del'>-	struct ib_qp *qp = NULL;</div><div class='del'>-	struct ib_uobject *uobj =</div><div class='del'>-		uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE);</div><div class='del'>-	struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&amp;attrs-&gt;driver_udata);</div><div class='del'>-	int len, ret, i;</div><div class='del'>-	u32 counter_id = 0;</div><div class='del'>-</div><div class='del'>-	if (!capable(CAP_NET_RAW))</div><div class='del'>-		return -EPERM;</div><div class='del'>-</div><div class='del'>-	dest_devx =</div><div class='del'>-		uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);</div><div class='del'>-	dest_qp = uverbs_attr_is_valid(attrs,</div><div class='del'>-				       MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);</div><div class='del'>-</div><div class='del'>-	fs_matcher = uverbs_attr_get_obj(attrs,</div><div class='del'>-					 MLX5_IB_ATTR_CREATE_FLOW_MATCHER);</div><div class='del'>-	if (fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_BYPASS &amp;&amp;</div><div class='del'>-	    ((dest_devx &amp;&amp; dest_qp) || (!dest_devx &amp;&amp; !dest_qp)))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	/* Allow only DEVX object as dest when inserting to FDB */</div><div class='del'>-	if (fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_FDB &amp;&amp; !dest_devx)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	if (dest_devx) {</div><div class='del'>-		devx_obj = uverbs_attr_get_obj(</div><div class='del'>-			attrs, MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);</div><div class='del'>-		if (IS_ERR(devx_obj))</div><div class='del'>-			return PTR_ERR(devx_obj);</div><div class='del'>-</div><div class='del'>-		/* Verify that the given DEVX object is a flow</div><div class='del'>-		 * steering destination.</div><div class='del'>-		 */</div><div class='del'>-		if (!mlx5_ib_devx_is_flow_dest(devx_obj, &amp;dest_id, &amp;dest_type))</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		/* Allow only flow table as dest when inserting to FDB */</div><div class='del'>-		if (fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_FDB &amp;&amp;</div><div class='del'>-		    dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)</div><div class='del'>-			return -EINVAL;</div><div class='del'>-	} else if (dest_qp) {</div><div class='del'>-		struct mlx5_ib_qp *mqp;</div><div class='del'>-</div><div class='del'>-		qp = uverbs_attr_get_obj(attrs,</div><div class='del'>-					 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);</div><div class='del'>-		if (IS_ERR(qp))</div><div class='del'>-			return PTR_ERR(qp);</div><div class='del'>-</div><div class='del'>-		if (qp-&gt;qp_type != IB_QPT_RAW_PACKET)</div><div class='del'>-			return -EINVAL;</div><div class='del'>-</div><div class='del'>-		mqp = to_mqp(qp);</div><div class='del'>-		if (mqp-&gt;flags &amp; MLX5_IB_QP_RSS)</div><div class='del'>-			dest_id = mqp-&gt;rss_qp.tirn;</div><div class='del'>-		else</div><div class='del'>-			dest_id = mqp-&gt;raw_packet_qp.rq.tirn;</div><div class='del'>-		dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;</div><div class='del'>-	} else {</div><div class='del'>-		dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	len = uverbs_attr_get_uobjs_arr(attrs,</div><div class='del'>-		MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &amp;arr_flow_actions);</div><div class='del'>-	if (len) {</div><div class='del'>-		devx_obj = arr_flow_actions[0]-&gt;object;</div><div class='del'>-</div><div class='del'>-		if (!mlx5_ib_devx_is_flow_counter(devx_obj, &amp;counter_id))</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR &amp;&amp;</div><div class='del'>-	    fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_EGRESS)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	cmd_in = uverbs_attr_get_alloced_ptr(</div><div class='del'>-		attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);</div><div class='del'>-	inlen = uverbs_attr_get_len(attrs,</div><div class='del'>-				    MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);</div><div class='del'>-</div><div class='del'>-	uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS);</div><div class='del'>-	if (!uflow_res)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	len = uverbs_attr_get_uobjs_arr(attrs,</div><div class='del'>-		MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &amp;arr_flow_actions);</div><div class='del'>-	for (i = 0; i &lt; len; i++) {</div><div class='del'>-		struct mlx5_ib_flow_action *maction =</div><div class='del'>-			to_mflow_act(arr_flow_actions[i]-&gt;object);</div><div class='del'>-</div><div class='del'>-		ret = parse_flow_flow_action(maction, false, &amp;flow_act);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto err_out;</div><div class='del'>-		flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE,</div><div class='del'>-				   arr_flow_actions[i]-&gt;object);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ret = uverbs_copy_from(&amp;flow_act.flow_tag, attrs,</div><div class='del'>-			       MLX5_IB_ATTR_CREATE_FLOW_TAG);</div><div class='del'>-	if (!ret) {</div><div class='del'>-		if (flow_act.flow_tag &gt;= BIT(24)) {</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto err_out;</div><div class='del'>-		}</div><div class='del'>-		flow_act.flags |= FLOW_ACT_HAS_TAG;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	flow_handler = mlx5_ib_raw_fs_rule_add(dev, fs_matcher, &amp;flow_act,</div><div class='del'>-					       counter_id,</div><div class='del'>-					       cmd_in, inlen,</div><div class='del'>-					       dest_id, dest_type);</div><div class='del'>-	if (IS_ERR(flow_handler)) {</div><div class='del'>-		ret = PTR_ERR(flow_handler);</div><div class='del'>-		goto err_out;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ib_set_flow(uobj, &amp;flow_handler-&gt;ibflow, qp, &amp;dev-&gt;ib_dev, uflow_res);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-err_out:</div><div class='del'>-	ib_uverbs_flow_resources_free(uflow_res);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int flow_matcher_cleanup(struct ib_uobject *uobject,</div><div class='del'>-				enum rdma_remove_reason why,</div><div class='del'>-				struct uverbs_attr_bundle *attrs)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_flow_matcher *obj = uobject-&gt;object;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	ret = ib_destroy_usecnt(&amp;obj-&gt;usecnt, why, uobject);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	kfree(obj);</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs,</div><div class='del'>-			      struct mlx5_ib_flow_matcher *obj)</div><div class='del'>-{</div><div class='del'>-	enum mlx5_ib_uapi_flow_table_type ft_type =</div><div class='del'>-		MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX;</div><div class='del'>-	u32 flags;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	/* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older</div><div class='del'>-	 * users should switch to it. We leave this to not break userspace</div><div class='del'>-	 */</div><div class='del'>-	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) &amp;&amp;</div><div class='del'>-	    uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) {</div><div class='del'>-		err = uverbs_get_const(&amp;ft_type, attrs,</div><div class='del'>-				       MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE);</div><div class='del'>-		if (err)</div><div class='del'>-			return err;</div><div class='del'>-</div><div class='del'>-		err = mlx5_ib_ft_type_to_namespace(ft_type, &amp;obj-&gt;ns_type);</div><div class='del'>-		if (err)</div><div class='del'>-			return err;</div><div class='del'>-</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) {</div><div class='del'>-		err = uverbs_get_flags32(&amp;flags, attrs,</div><div class='del'>-					 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,</div><div class='del'>-					 IB_FLOW_ATTR_FLAGS_EGRESS);</div><div class='del'>-		if (err)</div><div class='del'>-			return err;</div><div class='del'>-</div><div class='del'>-		if (flags) {</div><div class='del'>-			mlx5_ib_ft_type_to_namespace(</div><div class='del'>-				MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX,</div><div class='del'>-				&amp;obj-&gt;ns_type);</div><div class='del'>-			return 0;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	obj-&gt;ns_type = MLX5_FLOW_NAMESPACE_BYPASS;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)(</div><div class='del'>-	struct uverbs_attr_bundle *attrs)</div><div class='del'>-{</div><div class='del'>-	struct ib_uobject *uobj = uverbs_attr_get_uobject(</div><div class='del'>-		attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE);</div><div class='del'>-	struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&amp;attrs-&gt;driver_udata);</div><div class='del'>-	struct mlx5_ib_flow_matcher *obj;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL);</div><div class='del'>-	if (!obj)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	obj-&gt;mask_len = uverbs_attr_get_len(</div><div class='del'>-		attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);</div><div class='del'>-	err = uverbs_copy_from(&amp;obj-&gt;matcher_mask,</div><div class='del'>-			       attrs,</div><div class='del'>-			       MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);</div><div class='del'>-	if (err)</div><div class='del'>-		goto end;</div><div class='del'>-</div><div class='del'>-	obj-&gt;flow_type = uverbs_attr_get_enum_id(</div><div class='del'>-		attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);</div><div class='del'>-</div><div class='del'>-	if (obj-&gt;flow_type == MLX5_IB_FLOW_TYPE_NORMAL) {</div><div class='del'>-		err = uverbs_copy_from(&amp;obj-&gt;priority,</div><div class='del'>-				       attrs,</div><div class='del'>-				       MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);</div><div class='del'>-		if (err)</div><div class='del'>-			goto end;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	err = uverbs_copy_from(&amp;obj-&gt;match_criteria_enable,</div><div class='del'>-			       attrs,</div><div class='del'>-			       MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA);</div><div class='del'>-	if (err)</div><div class='del'>-		goto end;</div><div class='del'>-</div><div class='del'>-	err = mlx5_ib_matcher_ns(attrs, obj);</div><div class='del'>-	if (err)</div><div class='del'>-		goto end;</div><div class='del'>-</div><div class='del'>-	uobj-&gt;object = obj;</div><div class='del'>-	obj-&gt;mdev = dev-&gt;mdev;</div><div class='del'>-	atomic_set(&amp;obj-&gt;usecnt, 0);</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-end:</div><div class='del'>-	kfree(obj);</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)</div><div class='del'>-{</div><div class='del'>-	switch (maction-&gt;flow_action_raw.sub_type) {</div><div class='del'>-	case MLX5_IB_FLOW_ACTION_MODIFY_HEADER:</div><div class='del'>-		mlx5_modify_header_dealloc(maction-&gt;flow_action_raw.dev-&gt;mdev,</div><div class='del'>-					   maction-&gt;flow_action_raw.action_id);</div><div class='del'>-		break;</div><div class='del'>-	case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT:</div><div class='del'>-		mlx5_packet_reformat_dealloc(maction-&gt;flow_action_raw.dev-&gt;mdev,</div><div class='del'>-			maction-&gt;flow_action_raw.action_id);</div><div class='del'>-		break;</div><div class='del'>-	case MLX5_IB_FLOW_ACTION_DECAP:</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct ib_flow_action *</div><div class='del'>-mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev,</div><div class='del'>-			     enum mlx5_ib_uapi_flow_table_type ft_type,</div><div class='del'>-			     u8 num_actions, void *in)</div><div class='del'>-{</div><div class='del'>-	enum mlx5_flow_namespace_type namespace;</div><div class='del'>-	struct mlx5_ib_flow_action *maction;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	ret = mlx5_ib_ft_type_to_namespace(ft_type, &amp;namespace);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	maction = kzalloc(sizeof(*maction), GFP_KERNEL);</div><div class='del'>-	if (!maction)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	ret = mlx5_modify_header_alloc(dev-&gt;mdev, namespace, num_actions, in,</div><div class='del'>-				       &amp;maction-&gt;flow_action_raw.action_id);</div><div class='del'>-</div><div class='del'>-	if (ret) {</div><div class='del'>-		kfree(maction);</div><div class='del'>-		return ERR_PTR(ret);</div><div class='del'>-	}</div><div class='del'>-	maction-&gt;flow_action_raw.sub_type =</div><div class='del'>-		MLX5_IB_FLOW_ACTION_MODIFY_HEADER;</div><div class='del'>-	maction-&gt;flow_action_raw.dev = dev;</div><div class='del'>-</div><div class='del'>-	return &amp;maction-&gt;ib_action;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	return MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev,</div><div class='del'>-					 max_modify_header_actions) ||</div><div class='del'>-	       MLX5_CAP_FLOWTABLE_NIC_TX(dev-&gt;mdev, max_modify_header_actions);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)(</div><div class='del'>-	struct uverbs_attr_bundle *attrs)</div><div class='del'>-{</div><div class='del'>-	struct ib_uobject *uobj = uverbs_attr_get_uobject(</div><div class='del'>-		attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE);</div><div class='del'>-	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&amp;attrs-&gt;driver_udata);</div><div class='del'>-	enum mlx5_ib_uapi_flow_table_type ft_type;</div><div class='del'>-	struct ib_flow_action *action;</div><div class='del'>-	int num_actions;</div><div class='del'>-	void *in;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if (!mlx5_ib_modify_header_supported(mdev))</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-	in = uverbs_attr_get_alloced_ptr(attrs,</div><div class='del'>-		MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM);</div><div class='del'>-</div><div class='del'>-	num_actions = uverbs_attr_ptr_get_array_size(</div><div class='del'>-		attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,</div><div class='del'>-		MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto));</div><div class='del'>-	if (num_actions &lt; 0)</div><div class='del'>-		return num_actions;</div><div class='del'>-</div><div class='del'>-	ret = uverbs_get_const(&amp;ft_type, attrs,</div><div class='del'>-			       MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-	action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in);</div><div class='del'>-	if (IS_ERR(action))</div><div class='del'>-		return PTR_ERR(action);</div><div class='del'>-</div><div class='del'>-	uverbs_flow_action_fill_action(action, uobj, &amp;mdev-&gt;ib_dev,</div><div class='del'>-				       IB_FLOW_ACTION_UNSPECIFIED);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev,</div><div class='del'>-						      u8 packet_reformat_type,</div><div class='del'>-						      u8 ft_type)</div><div class='del'>-{</div><div class='del'>-	switch (packet_reformat_type) {</div><div class='del'>-	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:</div><div class='del'>-		if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)</div><div class='del'>-			return MLX5_CAP_FLOWTABLE(ibdev-&gt;mdev,</div><div class='del'>-						  encap_general_header);</div><div class='del'>-		break;</div><div class='del'>-	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:</div><div class='del'>-		if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)</div><div class='del'>-			return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev-&gt;mdev,</div><div class='del'>-				reformat_l2_to_l3_tunnel);</div><div class='del'>-		break;</div><div class='del'>-	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:</div><div class='del'>-		if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)</div><div class='del'>-			return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev-&gt;mdev,</div><div class='del'>-				reformat_l3_tunnel_to_l2);</div><div class='del'>-		break;</div><div class='del'>-	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2:</div><div class='del'>-		if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)</div><div class='del'>-			return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev-&gt;mdev, decap);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return false;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt)</div><div class='del'>-{</div><div class='del'>-	switch (dv_prt) {</div><div class='del'>-	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:</div><div class='del'>-		*prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL;</div><div class='del'>-		break;</div><div class='del'>-	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:</div><div class='del'>-		*prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2;</div><div class='del'>-		break;</div><div class='del'>-	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:</div><div class='del'>-		*prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx5_ib_flow_action_create_packet_reformat_ctx(</div><div class='del'>-	struct mlx5_ib_dev *dev,</div><div class='del'>-	struct mlx5_ib_flow_action *maction,</div><div class='del'>-	u8 ft_type, u8 dv_prt,</div><div class='del'>-	void *in, size_t len)</div><div class='del'>-{</div><div class='del'>-	enum mlx5_flow_namespace_type namespace;</div><div class='del'>-	u8 prm_prt;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	ret = mlx5_ib_ft_type_to_namespace(ft_type, &amp;namespace);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &amp;prm_prt);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	ret = mlx5_packet_reformat_alloc(dev-&gt;mdev, prm_prt, len,</div><div class='del'>-					 in, namespace,</div><div class='del'>-					 &amp;maction-&gt;flow_action_raw.action_id);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	maction-&gt;flow_action_raw.sub_type =</div><div class='del'>-		MLX5_IB_FLOW_ACTION_PACKET_REFORMAT;</div><div class='del'>-	maction-&gt;flow_action_raw.dev = dev;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)(</div><div class='del'>-	struct uverbs_attr_bundle *attrs)</div><div class='del'>-{</div><div class='del'>-	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,</div><div class='del'>-		MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE);</div><div class='del'>-	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&amp;attrs-&gt;driver_udata);</div><div class='del'>-	enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt;</div><div class='del'>-	enum mlx5_ib_uapi_flow_table_type ft_type;</div><div class='del'>-	struct mlx5_ib_flow_action *maction;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	ret = uverbs_get_const(&amp;ft_type, attrs,</div><div class='del'>-			       MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	ret = uverbs_get_const(&amp;dv_prt, attrs,</div><div class='del'>-			       MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type))</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-	maction = kzalloc(sizeof(*maction), GFP_KERNEL);</div><div class='del'>-	if (!maction)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	if (dv_prt ==</div><div class='del'>-	    MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) {</div><div class='del'>-		maction-&gt;flow_action_raw.sub_type =</div><div class='del'>-			MLX5_IB_FLOW_ACTION_DECAP;</div><div class='del'>-		maction-&gt;flow_action_raw.dev = mdev;</div><div class='del'>-	} else {</div><div class='del'>-		void *in;</div><div class='del'>-		int len;</div><div class='del'>-</div><div class='del'>-		in = uverbs_attr_get_alloced_ptr(attrs,</div><div class='del'>-			MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);</div><div class='del'>-		if (IS_ERR(in)) {</div><div class='del'>-			ret = PTR_ERR(in);</div><div class='del'>-			goto free_maction;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		len = uverbs_attr_get_len(attrs,</div><div class='del'>-			MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);</div><div class='del'>-</div><div class='del'>-		ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev,</div><div class='del'>-			maction, ft_type, dv_prt, in, len);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto free_maction;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	uverbs_flow_action_fill_action(&amp;maction-&gt;ib_action, uobj, &amp;mdev-&gt;ib_dev,</div><div class='del'>-				       IB_FLOW_ACTION_UNSPECIFIED);</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-free_maction:</div><div class='del'>-	kfree(maction);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-DECLARE_UVERBS_NAMED_METHOD(</div><div class='del'>-	MLX5_IB_METHOD_CREATE_FLOW,</div><div class='del'>-	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,</div><div class='del'>-			UVERBS_OBJECT_FLOW,</div><div class='del'>-			UVERBS_ACCESS_NEW,</div><div class='del'>-			UA_MANDATORY),</div><div class='del'>-	UVERBS_ATTR_PTR_IN(</div><div class='del'>-		MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE,</div><div class='del'>-		UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),</div><div class='del'>-		UA_MANDATORY,</div><div class='del'>-		UA_ALLOC_AND_COPY),</div><div class='del'>-	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER,</div><div class='del'>-			MLX5_IB_OBJECT_FLOW_MATCHER,</div><div class='del'>-			UVERBS_ACCESS_READ,</div><div class='del'>-			UA_MANDATORY),</div><div class='del'>-	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP,</div><div class='del'>-			UVERBS_OBJECT_QP,</div><div class='del'>-			UVERBS_ACCESS_READ),</div><div class='del'>-	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX,</div><div class='del'>-			MLX5_IB_OBJECT_DEVX_OBJ,</div><div class='del'>-			UVERBS_ACCESS_READ),</div><div class='del'>-	UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS,</div><div class='del'>-			     UVERBS_OBJECT_FLOW_ACTION,</div><div class='del'>-			     UVERBS_ACCESS_READ, 1,</div><div class='del'>-			     MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS,</div><div class='del'>-			     UA_OPTIONAL),</div><div class='del'>-	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG,</div><div class='del'>-			   UVERBS_ATTR_TYPE(u32),</div><div class='del'>-			   UA_OPTIONAL),</div><div class='del'>-	UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX,</div><div class='del'>-			     MLX5_IB_OBJECT_DEVX_OBJ,</div><div class='del'>-			     UVERBS_ACCESS_READ, 1, 1,</div><div class='del'>-			     UA_OPTIONAL));</div><div class='del'>-</div><div class='del'>-DECLARE_UVERBS_NAMED_METHOD_DESTROY(</div><div class='del'>-	MLX5_IB_METHOD_DESTROY_FLOW,</div><div class='del'>-	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,</div><div class='del'>-			UVERBS_OBJECT_FLOW,</div><div class='del'>-			UVERBS_ACCESS_DESTROY,</div><div class='del'>-			UA_MANDATORY));</div><div class='del'>-</div><div class='del'>-ADD_UVERBS_METHODS(mlx5_ib_fs,</div><div class='del'>-		   UVERBS_OBJECT_FLOW,</div><div class='del'>-		   &amp;UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW),</div><div class='del'>-		   &amp;UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW));</div><div class='del'>-</div><div class='del'>-DECLARE_UVERBS_NAMED_METHOD(</div><div class='del'>-	MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER,</div><div class='del'>-	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE,</div><div class='del'>-			UVERBS_OBJECT_FLOW_ACTION,</div><div class='del'>-			UVERBS_ACCESS_NEW,</div><div class='del'>-			UA_MANDATORY),</div><div class='del'>-	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,</div><div class='del'>-			   UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES(</div><div class='del'>-				   set_action_in_add_action_in_auto)),</div><div class='del'>-			   UA_MANDATORY,</div><div class='del'>-			   UA_ALLOC_AND_COPY),</div><div class='del'>-	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE,</div><div class='del'>-			     enum mlx5_ib_uapi_flow_table_type,</div><div class='del'>-			     UA_MANDATORY));</div><div class='del'>-</div><div class='del'>-DECLARE_UVERBS_NAMED_METHOD(</div><div class='del'>-	MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT,</div><div class='del'>-	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE,</div><div class='del'>-			UVERBS_OBJECT_FLOW_ACTION,</div><div class='del'>-			UVERBS_ACCESS_NEW,</div><div class='del'>-			UA_MANDATORY),</div><div class='del'>-	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF,</div><div class='del'>-			   UVERBS_ATTR_MIN_SIZE(1),</div><div class='del'>-			   UA_ALLOC_AND_COPY,</div><div class='del'>-			   UA_OPTIONAL),</div><div class='del'>-	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE,</div><div class='del'>-			     enum mlx5_ib_uapi_flow_action_packet_reformat_type,</div><div class='del'>-			     UA_MANDATORY),</div><div class='del'>-	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE,</div><div class='del'>-			     enum mlx5_ib_uapi_flow_table_type,</div><div class='del'>-			     UA_MANDATORY));</div><div class='del'>-</div><div class='del'>-ADD_UVERBS_METHODS(</div><div class='del'>-	mlx5_ib_flow_actions,</div><div class='del'>-	UVERBS_OBJECT_FLOW_ACTION,</div><div class='del'>-	&amp;UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER),</div><div class='del'>-	&amp;UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT));</div><div class='del'>-</div><div class='del'>-DECLARE_UVERBS_NAMED_METHOD(</div><div class='del'>-	MLX5_IB_METHOD_FLOW_MATCHER_CREATE,</div><div class='del'>-	UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE,</div><div class='del'>-			MLX5_IB_OBJECT_FLOW_MATCHER,</div><div class='del'>-			UVERBS_ACCESS_NEW,</div><div class='del'>-			UA_MANDATORY),</div><div class='del'>-	UVERBS_ATTR_PTR_IN(</div><div class='del'>-		MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK,</div><div class='del'>-		UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),</div><div class='del'>-		UA_MANDATORY),</div><div class='del'>-	UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE,</div><div class='del'>-			    mlx5_ib_flow_type,</div><div class='del'>-			    UA_MANDATORY),</div><div class='del'>-	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA,</div><div class='del'>-			   UVERBS_ATTR_TYPE(u8),</div><div class='del'>-			   UA_MANDATORY),</div><div class='del'>-	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,</div><div class='del'>-			     enum ib_flow_flags,</div><div class='del'>-			     UA_OPTIONAL),</div><div class='del'>-	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE,</div><div class='del'>-			     enum mlx5_ib_uapi_flow_table_type,</div><div class='del'>-			     UA_OPTIONAL));</div><div class='del'>-</div><div class='del'>-DECLARE_UVERBS_NAMED_METHOD_DESTROY(</div><div class='del'>-	MLX5_IB_METHOD_FLOW_MATCHER_DESTROY,</div><div class='del'>-	UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE,</div><div class='del'>-			MLX5_IB_OBJECT_FLOW_MATCHER,</div><div class='del'>-			UVERBS_ACCESS_DESTROY,</div><div class='del'>-			UA_MANDATORY));</div><div class='del'>-</div><div class='del'>-DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER,</div><div class='del'>-			    UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup),</div><div class='del'>-			    &amp;UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE),</div><div class='del'>-			    &amp;UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY));</div><div class='del'>-</div><div class='del'>-const struct uapi_definition mlx5_ib_flow_defs[] = {</div><div class='del'>-	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(</div><div class='del'>-		MLX5_IB_OBJECT_FLOW_MATCHER),</div><div class='del'>-	UAPI_DEF_CHAIN_OBJ_TREE(</div><div class='del'>-		UVERBS_OBJECT_FLOW,</div><div class='del'>-		&amp;mlx5_ib_fs),</div><div class='del'>-	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,</div><div class='del'>-				&amp;mlx5_ib_flow_actions),</div><div class='del'>-	{},</div><div class='del'>-};</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/fs.c b/drivers/infiniband/hw/mlx5/fs.c<br/>new file mode 100644<br/>index 000000000000..d17823ce7f38<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/fs.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/fs.c</a></div><div class='hunk'>@@ -0,0 +1,3516 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/ib_user_verbs.h&gt;</div><div class='add'>+#include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+#include &lt;rdma/uverbs_types.h&gt;</div><div class='add'>+#include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='add'>+#include &lt;rdma/uverbs_std_types.h&gt;</div><div class='add'>+#include &lt;rdma/mlx5_user_ioctl_cmds.h&gt;</div><div class='add'>+#include &lt;rdma/mlx5_user_ioctl_verbs.h&gt;</div><div class='add'>+#include &lt;rdma/ib_hdrs.h&gt;</div><div class='add'>+#include &lt;rdma/ib_umem.h&gt;</div><div class='add'>+#include &lt;rdma/ib_ucaps.h&gt;</div><div class='add'>+#include &lt;linux/mlx5/driver.h&gt;</div><div class='add'>+#include &lt;linux/mlx5/fs.h&gt;</div><div class='add'>+#include &lt;linux/mlx5/fs_helpers.h&gt;</div><div class='add'>+#include &lt;linux/mlx5/eswitch.h&gt;</div><div class='add'>+#include &lt;net/inet_ecn.h&gt;</div><div class='add'>+#include "mlx5_ib.h"</div><div class='add'>+#include "counters.h"</div><div class='add'>+#include "devx.h"</div><div class='add'>+#include "fs.h"</div><div class='add'>+</div><div class='add'>+#define UVERBS_MODULE_NAME mlx5_ib</div><div class='add'>+#include &lt;rdma/uverbs_named_ioctl.h&gt;</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	MATCH_CRITERIA_ENABLE_OUTER_BIT,</div><div class='add'>+	MATCH_CRITERIA_ENABLE_MISC_BIT,</div><div class='add'>+	MATCH_CRITERIA_ENABLE_INNER_BIT,</div><div class='add'>+	MATCH_CRITERIA_ENABLE_MISC2_BIT</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+</div><div class='add'>+struct mlx5_per_qp_opfc {</div><div class='add'>+	struct mlx5_ib_op_fc opfcs[MLX5_IB_OPCOUNTER_MAX];</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define HEADER_IS_ZERO(match_criteria, headers)			           \</div><div class='add'>+	!(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \</div><div class='add'>+		    0, MLX5_FLD_SZ_BYTES(fte_match_param, headers)))       \</div><div class='add'>+</div><div class='add'>+static u8 get_match_criteria_enable(u32 *match_criteria)</div><div class='add'>+{</div><div class='add'>+	u8 match_criteria_enable;</div><div class='add'>+</div><div class='add'>+	match_criteria_enable =</div><div class='add'>+		(!HEADER_IS_ZERO(match_criteria, outer_headers)) &lt;&lt;</div><div class='add'>+		MATCH_CRITERIA_ENABLE_OUTER_BIT;</div><div class='add'>+	match_criteria_enable |=</div><div class='add'>+		(!HEADER_IS_ZERO(match_criteria, misc_parameters)) &lt;&lt;</div><div class='add'>+		MATCH_CRITERIA_ENABLE_MISC_BIT;</div><div class='add'>+	match_criteria_enable |=</div><div class='add'>+		(!HEADER_IS_ZERO(match_criteria, inner_headers)) &lt;&lt;</div><div class='add'>+		MATCH_CRITERIA_ENABLE_INNER_BIT;</div><div class='add'>+	match_criteria_enable |=</div><div class='add'>+		(!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) &lt;&lt;</div><div class='add'>+		MATCH_CRITERIA_ENABLE_MISC2_BIT;</div><div class='add'>+</div><div class='add'>+	return match_criteria_enable;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)</div><div class='add'>+{</div><div class='add'>+	u8 entry_mask;</div><div class='add'>+	u8 entry_val;</div><div class='add'>+	int err = 0;</div><div class='add'>+</div><div class='add'>+	if (!mask)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c,</div><div class='add'>+			      ip_protocol);</div><div class='add'>+	entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v,</div><div class='add'>+			     ip_protocol);</div><div class='add'>+	if (!entry_mask) {</div><div class='add'>+		MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);</div><div class='add'>+		MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+	/* Don't override existing ip protocol */</div><div class='add'>+	if (mask != entry_mask || val != entry_val)</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+out:</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val,</div><div class='add'>+			   bool inner)</div><div class='add'>+{</div><div class='add'>+	if (inner) {</div><div class='add'>+		MLX5_SET(fte_match_set_misc,</div><div class='add'>+			 misc_c, inner_ipv6_flow_label, mask);</div><div class='add'>+		MLX5_SET(fte_match_set_misc,</div><div class='add'>+			 misc_v, inner_ipv6_flow_label, val);</div><div class='add'>+	} else {</div><div class='add'>+		MLX5_SET(fte_match_set_misc,</div><div class='add'>+			 misc_c, outer_ipv6_flow_label, mask);</div><div class='add'>+		MLX5_SET(fte_match_set_misc,</div><div class='add'>+			 misc_v, outer_ipv6_flow_label, val);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)</div><div class='add'>+{</div><div class='add'>+	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);</div><div class='add'>+	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);</div><div class='add'>+	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask &gt;&gt; 2);</div><div class='add'>+	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val &gt;&gt; 2);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask)</div><div class='add'>+{</div><div class='add'>+	if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) &amp;&amp;</div><div class='add'>+	    !(field_support &amp; MLX5_FIELD_SUPPORT_MPLS_LABEL))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) &amp;&amp;</div><div class='add'>+	    !(field_support &amp; MLX5_FIELD_SUPPORT_MPLS_EXP))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) &amp;&amp;</div><div class='add'>+	    !(field_support &amp; MLX5_FIELD_SUPPORT_MPLS_S_BOS))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) &amp;&amp;</div><div class='add'>+	    !(field_support &amp; MLX5_FIELD_SUPPORT_MPLS_TTL))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define LAST_ETH_FIELD vlan_tag</div><div class='add'>+#define LAST_IPV4_FIELD tos</div><div class='add'>+#define LAST_IPV6_FIELD traffic_class</div><div class='add'>+#define LAST_TCP_UDP_FIELD src_port</div><div class='add'>+#define LAST_TUNNEL_FIELD tunnel_id</div><div class='add'>+#define LAST_FLOW_TAG_FIELD tag_id</div><div class='add'>+#define LAST_DROP_FIELD size</div><div class='add'>+#define LAST_COUNTERS_FIELD counters</div><div class='add'>+</div><div class='add'>+/* Field is the last supported field */</div><div class='add'>+#define FIELDS_NOT_SUPPORTED(filter, field)                                    \</div><div class='add'>+	memchr_inv((void *)&amp;filter.field + sizeof(filter.field), 0,            \</div><div class='add'>+		   sizeof(filter) - offsetofend(typeof(filter), field))</div><div class='add'>+</div><div class='add'>+int parse_flow_flow_action(struct mlx5_ib_flow_action *maction,</div><div class='add'>+			   bool is_egress,</div><div class='add'>+			   struct mlx5_flow_act *action)</div><div class='add'>+{</div><div class='add'>+</div><div class='add'>+	switch (maction-&gt;ib_action.type) {</div><div class='add'>+	case IB_FLOW_ACTION_UNSPECIFIED:</div><div class='add'>+		if (maction-&gt;flow_action_raw.sub_type ==</div><div class='add'>+		    MLX5_IB_FLOW_ACTION_MODIFY_HEADER) {</div><div class='add'>+			if (action-&gt;action &amp; MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)</div><div class='add'>+				return -EINVAL;</div><div class='add'>+			action-&gt;action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;</div><div class='add'>+			action-&gt;modify_hdr =</div><div class='add'>+				maction-&gt;flow_action_raw.modify_hdr;</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+		if (maction-&gt;flow_action_raw.sub_type ==</div><div class='add'>+		    MLX5_IB_FLOW_ACTION_DECAP) {</div><div class='add'>+			if (action-&gt;action &amp; MLX5_FLOW_CONTEXT_ACTION_DECAP)</div><div class='add'>+				return -EINVAL;</div><div class='add'>+			action-&gt;action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+		if (maction-&gt;flow_action_raw.sub_type ==</div><div class='add'>+		    MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) {</div><div class='add'>+			if (action-&gt;action &amp;</div><div class='add'>+			    MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT)</div><div class='add'>+				return -EINVAL;</div><div class='add'>+			action-&gt;action |=</div><div class='add'>+				MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;</div><div class='add'>+			action-&gt;pkt_reformat =</div><div class='add'>+				maction-&gt;flow_action_raw.pkt_reformat;</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+		fallthrough;</div><div class='add'>+	default:</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int parse_flow_attr(struct mlx5_core_dev *mdev,</div><div class='add'>+			   struct mlx5_flow_spec *spec,</div><div class='add'>+			   const union ib_flow_spec *ib_spec,</div><div class='add'>+			   const struct ib_flow_attr *flow_attr,</div><div class='add'>+			   struct mlx5_flow_act *action, u32 prev_type)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_flow_context *flow_context = &amp;spec-&gt;flow_context;</div><div class='add'>+	u32 *match_c = spec-&gt;match_criteria;</div><div class='add'>+	u32 *match_v = spec-&gt;match_value;</div><div class='add'>+	void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,</div><div class='add'>+					   misc_parameters);</div><div class='add'>+	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,</div><div class='add'>+					   misc_parameters);</div><div class='add'>+	void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c,</div><div class='add'>+					    misc_parameters_2);</div><div class='add'>+	void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v,</div><div class='add'>+					    misc_parameters_2);</div><div class='add'>+	void *headers_c;</div><div class='add'>+	void *headers_v;</div><div class='add'>+	int match_ipv;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (ib_spec-&gt;type &amp; IB_FLOW_SPEC_INNER) {</div><div class='add'>+		headers_c = MLX5_ADDR_OF(fte_match_param, match_c,</div><div class='add'>+					 inner_headers);</div><div class='add'>+		headers_v = MLX5_ADDR_OF(fte_match_param, match_v,</div><div class='add'>+					 inner_headers);</div><div class='add'>+		match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,</div><div class='add'>+					ft_field_support.inner_ip_version);</div><div class='add'>+	} else {</div><div class='add'>+		headers_c = MLX5_ADDR_OF(fte_match_param, match_c,</div><div class='add'>+					 outer_headers);</div><div class='add'>+		headers_v = MLX5_ADDR_OF(fte_match_param, match_v,</div><div class='add'>+					 outer_headers);</div><div class='add'>+		match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,</div><div class='add'>+					ft_field_support.outer_ip_version);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (ib_spec-&gt;type &amp; ~IB_FLOW_SPEC_INNER) {</div><div class='add'>+	case IB_FLOW_SPEC_ETH:</div><div class='add'>+		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;eth.mask, LAST_ETH_FIELD))</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,</div><div class='add'>+					     dmac_47_16),</div><div class='add'>+				ib_spec-&gt;eth.mask.dst_mac);</div><div class='add'>+		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,</div><div class='add'>+					     dmac_47_16),</div><div class='add'>+				ib_spec-&gt;eth.val.dst_mac);</div><div class='add'>+</div><div class='add'>+		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,</div><div class='add'>+					     smac_47_16),</div><div class='add'>+				ib_spec-&gt;eth.mask.src_mac);</div><div class='add'>+		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,</div><div class='add'>+					     smac_47_16),</div><div class='add'>+				ib_spec-&gt;eth.val.src_mac);</div><div class='add'>+</div><div class='add'>+		if (ib_spec-&gt;eth.mask.vlan_tag) {</div><div class='add'>+			MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='add'>+				 cvlan_tag, 1);</div><div class='add'>+			MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='add'>+				 cvlan_tag, 1);</div><div class='add'>+</div><div class='add'>+			MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='add'>+				 first_vid, ntohs(ib_spec-&gt;eth.mask.vlan_tag));</div><div class='add'>+			MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='add'>+				 first_vid, ntohs(ib_spec-&gt;eth.val.vlan_tag));</div><div class='add'>+</div><div class='add'>+			MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='add'>+				 first_cfi,</div><div class='add'>+				 ntohs(ib_spec-&gt;eth.mask.vlan_tag) &gt;&gt; 12);</div><div class='add'>+			MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='add'>+				 first_cfi,</div><div class='add'>+				 ntohs(ib_spec-&gt;eth.val.vlan_tag) &gt;&gt; 12);</div><div class='add'>+</div><div class='add'>+			MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='add'>+				 first_prio,</div><div class='add'>+				 ntohs(ib_spec-&gt;eth.mask.vlan_tag) &gt;&gt; 13);</div><div class='add'>+			MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='add'>+				 first_prio,</div><div class='add'>+				 ntohs(ib_spec-&gt;eth.val.vlan_tag) &gt;&gt; 13);</div><div class='add'>+		}</div><div class='add'>+		MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='add'>+			 ethertype, ntohs(ib_spec-&gt;eth.mask.ether_type));</div><div class='add'>+		MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='add'>+			 ethertype, ntohs(ib_spec-&gt;eth.val.ether_type));</div><div class='add'>+		break;</div><div class='add'>+	case IB_FLOW_SPEC_IPV4:</div><div class='add'>+		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;ipv4.mask, LAST_IPV4_FIELD))</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+		if (match_ipv) {</div><div class='add'>+			MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='add'>+				 ip_version, 0xf);</div><div class='add'>+			MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='add'>+				 ip_version, MLX5_FS_IPV4_VERSION);</div><div class='add'>+		} else {</div><div class='add'>+			MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='add'>+				 ethertype, 0xffff);</div><div class='add'>+			MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='add'>+				 ethertype, ETH_P_IP);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,</div><div class='add'>+				    src_ipv4_src_ipv6.ipv4_layout.ipv4),</div><div class='add'>+		       &amp;ib_spec-&gt;ipv4.mask.src_ip,</div><div class='add'>+		       sizeof(ib_spec-&gt;ipv4.mask.src_ip));</div><div class='add'>+		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,</div><div class='add'>+				    src_ipv4_src_ipv6.ipv4_layout.ipv4),</div><div class='add'>+		       &amp;ib_spec-&gt;ipv4.val.src_ip,</div><div class='add'>+		       sizeof(ib_spec-&gt;ipv4.val.src_ip));</div><div class='add'>+		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,</div><div class='add'>+				    dst_ipv4_dst_ipv6.ipv4_layout.ipv4),</div><div class='add'>+		       &amp;ib_spec-&gt;ipv4.mask.dst_ip,</div><div class='add'>+		       sizeof(ib_spec-&gt;ipv4.mask.dst_ip));</div><div class='add'>+		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,</div><div class='add'>+				    dst_ipv4_dst_ipv6.ipv4_layout.ipv4),</div><div class='add'>+		       &amp;ib_spec-&gt;ipv4.val.dst_ip,</div><div class='add'>+		       sizeof(ib_spec-&gt;ipv4.val.dst_ip));</div><div class='add'>+</div><div class='add'>+		set_tos(headers_c, headers_v,</div><div class='add'>+			ib_spec-&gt;ipv4.mask.tos, ib_spec-&gt;ipv4.val.tos);</div><div class='add'>+</div><div class='add'>+		if (set_proto(headers_c, headers_v,</div><div class='add'>+			      ib_spec-&gt;ipv4.mask.proto,</div><div class='add'>+			      ib_spec-&gt;ipv4.val.proto))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		break;</div><div class='add'>+	case IB_FLOW_SPEC_IPV6:</div><div class='add'>+		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;ipv6.mask, LAST_IPV6_FIELD))</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+		if (match_ipv) {</div><div class='add'>+			MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='add'>+				 ip_version, 0xf);</div><div class='add'>+			MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='add'>+				 ip_version, MLX5_FS_IPV6_VERSION);</div><div class='add'>+		} else {</div><div class='add'>+			MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='add'>+				 ethertype, 0xffff);</div><div class='add'>+			MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='add'>+				 ethertype, ETH_P_IPV6);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,</div><div class='add'>+				    src_ipv4_src_ipv6.ipv6_layout.ipv6),</div><div class='add'>+		       &amp;ib_spec-&gt;ipv6.mask.src_ip,</div><div class='add'>+		       sizeof(ib_spec-&gt;ipv6.mask.src_ip));</div><div class='add'>+		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,</div><div class='add'>+				    src_ipv4_src_ipv6.ipv6_layout.ipv6),</div><div class='add'>+		       &amp;ib_spec-&gt;ipv6.val.src_ip,</div><div class='add'>+		       sizeof(ib_spec-&gt;ipv6.val.src_ip));</div><div class='add'>+		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,</div><div class='add'>+				    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),</div><div class='add'>+		       &amp;ib_spec-&gt;ipv6.mask.dst_ip,</div><div class='add'>+		       sizeof(ib_spec-&gt;ipv6.mask.dst_ip));</div><div class='add'>+		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,</div><div class='add'>+				    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),</div><div class='add'>+		       &amp;ib_spec-&gt;ipv6.val.dst_ip,</div><div class='add'>+		       sizeof(ib_spec-&gt;ipv6.val.dst_ip));</div><div class='add'>+</div><div class='add'>+		set_tos(headers_c, headers_v,</div><div class='add'>+			ib_spec-&gt;ipv6.mask.traffic_class,</div><div class='add'>+			ib_spec-&gt;ipv6.val.traffic_class);</div><div class='add'>+</div><div class='add'>+		if (set_proto(headers_c, headers_v,</div><div class='add'>+			      ib_spec-&gt;ipv6.mask.next_hdr,</div><div class='add'>+			      ib_spec-&gt;ipv6.val.next_hdr))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		set_flow_label(misc_params_c, misc_params_v,</div><div class='add'>+			       ntohl(ib_spec-&gt;ipv6.mask.flow_label),</div><div class='add'>+			       ntohl(ib_spec-&gt;ipv6.val.flow_label),</div><div class='add'>+			       ib_spec-&gt;type &amp; IB_FLOW_SPEC_INNER);</div><div class='add'>+		break;</div><div class='add'>+	case IB_FLOW_SPEC_ESP:</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	case IB_FLOW_SPEC_TCP:</div><div class='add'>+		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;tcp_udp.mask,</div><div class='add'>+					 LAST_TCP_UDP_FIELD))</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+		if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport,</div><div class='add'>+			 ntohs(ib_spec-&gt;tcp_udp.mask.src_port));</div><div class='add'>+		MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,</div><div class='add'>+			 ntohs(ib_spec-&gt;tcp_udp.val.src_port));</div><div class='add'>+</div><div class='add'>+		MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport,</div><div class='add'>+			 ntohs(ib_spec-&gt;tcp_udp.mask.dst_port));</div><div class='add'>+		MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,</div><div class='add'>+			 ntohs(ib_spec-&gt;tcp_udp.val.dst_port));</div><div class='add'>+		break;</div><div class='add'>+	case IB_FLOW_SPEC_UDP:</div><div class='add'>+		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;tcp_udp.mask,</div><div class='add'>+					 LAST_TCP_UDP_FIELD))</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+		if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,</div><div class='add'>+			 ntohs(ib_spec-&gt;tcp_udp.mask.src_port));</div><div class='add'>+		MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,</div><div class='add'>+			 ntohs(ib_spec-&gt;tcp_udp.val.src_port));</div><div class='add'>+</div><div class='add'>+		MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,</div><div class='add'>+			 ntohs(ib_spec-&gt;tcp_udp.mask.dst_port));</div><div class='add'>+		MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,</div><div class='add'>+			 ntohs(ib_spec-&gt;tcp_udp.val.dst_port));</div><div class='add'>+		break;</div><div class='add'>+	case IB_FLOW_SPEC_GRE:</div><div class='add'>+		if (ib_spec-&gt;gre.mask.c_ks_res0_ver)</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+		if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,</div><div class='add'>+			 0xff);</div><div class='add'>+		MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,</div><div class='add'>+			 IPPROTO_GRE);</div><div class='add'>+</div><div class='add'>+		MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol,</div><div class='add'>+			 ntohs(ib_spec-&gt;gre.mask.protocol));</div><div class='add'>+		MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol,</div><div class='add'>+			 ntohs(ib_spec-&gt;gre.val.protocol));</div><div class='add'>+</div><div class='add'>+		memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c,</div><div class='add'>+				    gre_key.nvgre.hi),</div><div class='add'>+		       &amp;ib_spec-&gt;gre.mask.key,</div><div class='add'>+		       sizeof(ib_spec-&gt;gre.mask.key));</div><div class='add'>+		memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v,</div><div class='add'>+				    gre_key.nvgre.hi),</div><div class='add'>+		       &amp;ib_spec-&gt;gre.val.key,</div><div class='add'>+		       sizeof(ib_spec-&gt;gre.val.key));</div><div class='add'>+		break;</div><div class='add'>+	case IB_FLOW_SPEC_MPLS:</div><div class='add'>+		switch (prev_type) {</div><div class='add'>+		case IB_FLOW_SPEC_UDP:</div><div class='add'>+			if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,</div><div class='add'>+						   ft_field_support.outer_first_mpls_over_udp),</div><div class='add'>+						   &amp;ib_spec-&gt;mpls.mask.tag))</div><div class='add'>+				return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,</div><div class='add'>+					    outer_first_mpls_over_udp),</div><div class='add'>+			       &amp;ib_spec-&gt;mpls.val.tag,</div><div class='add'>+			       sizeof(ib_spec-&gt;mpls.val.tag));</div><div class='add'>+			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,</div><div class='add'>+					    outer_first_mpls_over_udp),</div><div class='add'>+			       &amp;ib_spec-&gt;mpls.mask.tag,</div><div class='add'>+			       sizeof(ib_spec-&gt;mpls.mask.tag));</div><div class='add'>+			break;</div><div class='add'>+		case IB_FLOW_SPEC_GRE:</div><div class='add'>+			if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,</div><div class='add'>+						   ft_field_support.outer_first_mpls_over_gre),</div><div class='add'>+						   &amp;ib_spec-&gt;mpls.mask.tag))</div><div class='add'>+				return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,</div><div class='add'>+					    outer_first_mpls_over_gre),</div><div class='add'>+			       &amp;ib_spec-&gt;mpls.val.tag,</div><div class='add'>+			       sizeof(ib_spec-&gt;mpls.val.tag));</div><div class='add'>+			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,</div><div class='add'>+					    outer_first_mpls_over_gre),</div><div class='add'>+			       &amp;ib_spec-&gt;mpls.mask.tag,</div><div class='add'>+			       sizeof(ib_spec-&gt;mpls.mask.tag));</div><div class='add'>+			break;</div><div class='add'>+		default:</div><div class='add'>+			if (ib_spec-&gt;type &amp; IB_FLOW_SPEC_INNER) {</div><div class='add'>+				if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,</div><div class='add'>+							   ft_field_support.inner_first_mpls),</div><div class='add'>+							   &amp;ib_spec-&gt;mpls.mask.tag))</div><div class='add'>+					return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,</div><div class='add'>+						    inner_first_mpls),</div><div class='add'>+				       &amp;ib_spec-&gt;mpls.val.tag,</div><div class='add'>+				       sizeof(ib_spec-&gt;mpls.val.tag));</div><div class='add'>+				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,</div><div class='add'>+						    inner_first_mpls),</div><div class='add'>+				       &amp;ib_spec-&gt;mpls.mask.tag,</div><div class='add'>+				       sizeof(ib_spec-&gt;mpls.mask.tag));</div><div class='add'>+			} else {</div><div class='add'>+				if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,</div><div class='add'>+							   ft_field_support.outer_first_mpls),</div><div class='add'>+							   &amp;ib_spec-&gt;mpls.mask.tag))</div><div class='add'>+					return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,</div><div class='add'>+						    outer_first_mpls),</div><div class='add'>+				       &amp;ib_spec-&gt;mpls.val.tag,</div><div class='add'>+				       sizeof(ib_spec-&gt;mpls.val.tag));</div><div class='add'>+				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,</div><div class='add'>+						    outer_first_mpls),</div><div class='add'>+				       &amp;ib_spec-&gt;mpls.mask.tag,</div><div class='add'>+				       sizeof(ib_spec-&gt;mpls.mask.tag));</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	case IB_FLOW_SPEC_VXLAN_TUNNEL:</div><div class='add'>+		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;tunnel.mask,</div><div class='add'>+					 LAST_TUNNEL_FIELD))</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+		MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni,</div><div class='add'>+			 ntohl(ib_spec-&gt;tunnel.mask.tunnel_id));</div><div class='add'>+		MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni,</div><div class='add'>+			 ntohl(ib_spec-&gt;tunnel.val.tunnel_id));</div><div class='add'>+		break;</div><div class='add'>+	case IB_FLOW_SPEC_ACTION_TAG:</div><div class='add'>+		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;flow_tag,</div><div class='add'>+					 LAST_FLOW_TAG_FIELD))</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+		if (ib_spec-&gt;flow_tag.tag_id &gt;= BIT(24))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		flow_context-&gt;flow_tag = ib_spec-&gt;flow_tag.tag_id;</div><div class='add'>+		flow_context-&gt;flags |= FLOW_CONTEXT_HAS_TAG;</div><div class='add'>+		break;</div><div class='add'>+	case IB_FLOW_SPEC_ACTION_DROP:</div><div class='add'>+		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;drop,</div><div class='add'>+					 LAST_DROP_FIELD))</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+		action-&gt;action |= MLX5_FLOW_CONTEXT_ACTION_DROP;</div><div class='add'>+		break;</div><div class='add'>+	case IB_FLOW_SPEC_ACTION_HANDLE:</div><div class='add'>+		ret = parse_flow_flow_action(to_mflow_act(ib_spec-&gt;action.act),</div><div class='add'>+			flow_attr-&gt;flags &amp; IB_FLOW_ATTR_FLAGS_EGRESS, action);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+		break;</div><div class='add'>+	case IB_FLOW_SPEC_ACTION_COUNT:</div><div class='add'>+		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;flow_count,</div><div class='add'>+					 LAST_COUNTERS_FIELD))</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+		/* for now support only one counters spec per flow */</div><div class='add'>+		if (action-&gt;action &amp; MLX5_FLOW_CONTEXT_ACTION_COUNT)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		action-&gt;counters = ib_spec-&gt;flow_count.counters;</div><div class='add'>+		action-&gt;action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* If a flow could catch both multicast and unicast packets,</div><div class='add'>+ * it won't fall into the multicast flow steering table and this rule</div><div class='add'>+ * could steal other multicast packets.</div><div class='add'>+ */</div><div class='add'>+static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)</div><div class='add'>+{</div><div class='add'>+	union ib_flow_spec *flow_spec;</div><div class='add'>+</div><div class='add'>+	if (ib_attr-&gt;type != IB_FLOW_ATTR_NORMAL ||</div><div class='add'>+	    ib_attr-&gt;num_of_specs &lt; 1)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	flow_spec = (union ib_flow_spec *)(ib_attr + 1);</div><div class='add'>+	if (flow_spec-&gt;type == IB_FLOW_SPEC_IPV4) {</div><div class='add'>+		struct ib_flow_spec_ipv4 *ipv4_spec;</div><div class='add'>+</div><div class='add'>+		ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec;</div><div class='add'>+		if (ipv4_is_multicast(ipv4_spec-&gt;val.dst_ip))</div><div class='add'>+			return true;</div><div class='add'>+</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (flow_spec-&gt;type == IB_FLOW_SPEC_ETH) {</div><div class='add'>+		struct ib_flow_spec_eth *eth_spec;</div><div class='add'>+</div><div class='add'>+		eth_spec = (struct ib_flow_spec_eth *)flow_spec;</div><div class='add'>+		return is_multicast_ether_addr(eth_spec-&gt;mask.dst_mac) &amp;&amp;</div><div class='add'>+		       is_multicast_ether_addr(eth_spec-&gt;val.dst_mac);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool is_valid_ethertype(struct mlx5_core_dev *mdev,</div><div class='add'>+			       const struct ib_flow_attr *flow_attr,</div><div class='add'>+			       bool check_inner)</div><div class='add'>+{</div><div class='add'>+	union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1);</div><div class='add'>+	int match_ipv = check_inner ?</div><div class='add'>+			MLX5_CAP_FLOWTABLE_NIC_RX(mdev,</div><div class='add'>+					ft_field_support.inner_ip_version) :</div><div class='add'>+			MLX5_CAP_FLOWTABLE_NIC_RX(mdev,</div><div class='add'>+					ft_field_support.outer_ip_version);</div><div class='add'>+	int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0;</div><div class='add'>+	bool ipv4_spec_valid, ipv6_spec_valid;</div><div class='add'>+	unsigned int ip_spec_type = 0;</div><div class='add'>+	bool has_ethertype = false;</div><div class='add'>+	unsigned int spec_index;</div><div class='add'>+	bool mask_valid = true;</div><div class='add'>+	u16 eth_type = 0;</div><div class='add'>+	bool type_valid;</div><div class='add'>+</div><div class='add'>+	/* Validate that ethertype is correct */</div><div class='add'>+	for (spec_index = 0; spec_index &lt; flow_attr-&gt;num_of_specs; spec_index++) {</div><div class='add'>+		if ((ib_spec-&gt;type == (IB_FLOW_SPEC_ETH | inner_bit)) &amp;&amp;</div><div class='add'>+		    ib_spec-&gt;eth.mask.ether_type) {</div><div class='add'>+			mask_valid = (ib_spec-&gt;eth.mask.ether_type ==</div><div class='add'>+				      htons(0xffff));</div><div class='add'>+			has_ethertype = true;</div><div class='add'>+			eth_type = ntohs(ib_spec-&gt;eth.val.ether_type);</div><div class='add'>+		} else if ((ib_spec-&gt;type == (IB_FLOW_SPEC_IPV4 | inner_bit)) ||</div><div class='add'>+			   (ib_spec-&gt;type == (IB_FLOW_SPEC_IPV6 | inner_bit))) {</div><div class='add'>+			ip_spec_type = ib_spec-&gt;type;</div><div class='add'>+		}</div><div class='add'>+		ib_spec = (void *)ib_spec + ib_spec-&gt;size;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	type_valid = (!has_ethertype) || (!ip_spec_type);</div><div class='add'>+	if (!type_valid &amp;&amp; mask_valid) {</div><div class='add'>+		ipv4_spec_valid = (eth_type == ETH_P_IP) &amp;&amp;</div><div class='add'>+			(ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit));</div><div class='add'>+		ipv6_spec_valid = (eth_type == ETH_P_IPV6) &amp;&amp;</div><div class='add'>+			(ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit));</div><div class='add'>+</div><div class='add'>+		type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) ||</div><div class='add'>+			     (((eth_type == ETH_P_MPLS_UC) ||</div><div class='add'>+			       (eth_type == ETH_P_MPLS_MC)) &amp;&amp; match_ipv);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return type_valid;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool is_valid_attr(struct mlx5_core_dev *mdev,</div><div class='add'>+			  const struct ib_flow_attr *flow_attr)</div><div class='add'>+{</div><div class='add'>+	return is_valid_ethertype(mdev, flow_attr, false) &amp;&amp;</div><div class='add'>+	       is_valid_ethertype(mdev, flow_attr, true);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void put_flow_table(struct mlx5_ib_dev *dev,</div><div class='add'>+			   struct mlx5_ib_flow_prio *prio, bool ft_added)</div><div class='add'>+{</div><div class='add'>+	prio-&gt;refcount -= !!ft_added;</div><div class='add'>+	if (!prio-&gt;refcount) {</div><div class='add'>+		mlx5_destroy_flow_table(prio-&gt;flow_table);</div><div class='add'>+		prio-&gt;flow_table = NULL;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_flow_handler *handler = container_of(flow_id,</div><div class='add'>+							  struct mlx5_ib_flow_handler,</div><div class='add'>+							  ibflow);</div><div class='add'>+	struct mlx5_ib_flow_handler *iter, *tmp;</div><div class='add'>+	struct mlx5_ib_dev *dev = handler-&gt;dev;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='add'>+</div><div class='add'>+	list_for_each_entry_safe(iter, tmp, &amp;handler-&gt;list, list) {</div><div class='add'>+		mlx5_del_flow_rules(iter-&gt;rule);</div><div class='add'>+		put_flow_table(dev, iter-&gt;prio, true);</div><div class='add'>+		list_del(&amp;iter-&gt;list);</div><div class='add'>+		kfree(iter);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mlx5_del_flow_rules(handler-&gt;rule);</div><div class='add'>+	put_flow_table(dev, handler-&gt;prio, true);</div><div class='add'>+	mlx5_ib_counters_clear_description(handler-&gt;ibcounters);</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='add'>+	if (handler-&gt;flow_matcher)</div><div class='add'>+		atomic_dec(&amp;handler-&gt;flow_matcher-&gt;usecnt);</div><div class='add'>+	kfree(handler);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)</div><div class='add'>+{</div><div class='add'>+	priority *= 2;</div><div class='add'>+	if (!dont_trap)</div><div class='add'>+		priority++;</div><div class='add'>+	return priority;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+enum flow_table_type {</div><div class='add'>+	MLX5_IB_FT_RX,</div><div class='add'>+	MLX5_IB_FT_TX</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define MLX5_FS_MAX_TYPES	 6</div><div class='add'>+#define MLX5_FS_MAX_ENTRIES	 BIT(16)</div><div class='add'>+</div><div class='add'>+static bool __maybe_unused mlx5_ib_shared_ft_allowed(struct ib_device *device)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='add'>+</div><div class='add'>+	return MLX5_CAP_GEN(dev-&gt;mdev, shared_object_to_user_object_allowed);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns,</div><div class='add'>+					   struct mlx5_ib_flow_prio *prio,</div><div class='add'>+					   struct mlx5_flow_table_attr *ft_attr)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_flow_table *ft;</div><div class='add'>+</div><div class='add'>+	ft = mlx5_create_auto_grouped_flow_table(ns, ft_attr);</div><div class='add'>+	if (IS_ERR(ft))</div><div class='add'>+		return ERR_CAST(ft);</div><div class='add'>+</div><div class='add'>+	prio-&gt;flow_table = ft;</div><div class='add'>+	prio-&gt;refcount = 0;</div><div class='add'>+	return prio;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,</div><div class='add'>+						struct ib_flow_attr *flow_attr,</div><div class='add'>+						enum flow_table_type ft_type)</div><div class='add'>+{</div><div class='add'>+	bool dont_trap = flow_attr-&gt;flags &amp; IB_FLOW_ATTR_FLAGS_DONT_TRAP;</div><div class='add'>+	struct mlx5_flow_table_attr ft_attr = {};</div><div class='add'>+	struct mlx5_flow_namespace *ns = NULL;</div><div class='add'>+	enum mlx5_flow_namespace_type fn_type;</div><div class='add'>+	struct mlx5_ib_flow_prio *prio;</div><div class='add'>+	struct mlx5_flow_table *ft;</div><div class='add'>+	int max_table_size;</div><div class='add'>+	int num_entries;</div><div class='add'>+	int num_groups;</div><div class='add'>+	bool esw_encap;</div><div class='add'>+	u32 flags = 0;</div><div class='add'>+	int priority;</div><div class='add'>+</div><div class='add'>+	max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev,</div><div class='add'>+						       log_max_ft_size));</div><div class='add'>+	esw_encap = mlx5_eswitch_get_encap_mode(dev-&gt;mdev) !=</div><div class='add'>+		DEVLINK_ESWITCH_ENCAP_MODE_NONE;</div><div class='add'>+	switch (flow_attr-&gt;type) {</div><div class='add'>+	case IB_FLOW_ATTR_NORMAL:</div><div class='add'>+		if (flow_is_multicast_only(flow_attr) &amp;&amp; !dont_trap)</div><div class='add'>+			priority = MLX5_IB_FLOW_MCAST_PRIO;</div><div class='add'>+		else</div><div class='add'>+			priority = ib_prio_to_core_prio(flow_attr-&gt;priority,</div><div class='add'>+							dont_trap);</div><div class='add'>+		if (ft_type == MLX5_IB_FT_RX) {</div><div class='add'>+			fn_type = MLX5_FLOW_NAMESPACE_BYPASS;</div><div class='add'>+			prio = &amp;dev-&gt;flow_db-&gt;prios[priority];</div><div class='add'>+			if (!dev-&gt;is_rep &amp;&amp; !esw_encap &amp;&amp;</div><div class='add'>+			    MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev, decap))</div><div class='add'>+				flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;</div><div class='add'>+			if (!dev-&gt;is_rep &amp;&amp; !esw_encap &amp;&amp;</div><div class='add'>+			    MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev,</div><div class='add'>+						      reformat_l3_tunnel_to_l2))</div><div class='add'>+				flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;</div><div class='add'>+		} else {</div><div class='add'>+			max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_TX(</div><div class='add'>+				dev-&gt;mdev, log_max_ft_size));</div><div class='add'>+			fn_type = MLX5_FLOW_NAMESPACE_EGRESS;</div><div class='add'>+			prio = &amp;dev-&gt;flow_db-&gt;egress_prios[priority];</div><div class='add'>+			if (!dev-&gt;is_rep &amp;&amp; !esw_encap &amp;&amp;</div><div class='add'>+			    MLX5_CAP_FLOWTABLE_NIC_TX(dev-&gt;mdev, reformat))</div><div class='add'>+				flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;</div><div class='add'>+		}</div><div class='add'>+		ns = mlx5_get_flow_namespace(dev-&gt;mdev, fn_type);</div><div class='add'>+		num_entries = MLX5_FS_MAX_ENTRIES;</div><div class='add'>+		num_groups = MLX5_FS_MAX_TYPES;</div><div class='add'>+		break;</div><div class='add'>+	case IB_FLOW_ATTR_ALL_DEFAULT:</div><div class='add'>+	case IB_FLOW_ATTR_MC_DEFAULT:</div><div class='add'>+		ns = mlx5_get_flow_namespace(dev-&gt;mdev,</div><div class='add'>+					     MLX5_FLOW_NAMESPACE_LEFTOVERS);</div><div class='add'>+		build_leftovers_ft_param(&amp;priority, &amp;num_entries, &amp;num_groups);</div><div class='add'>+		prio = &amp;dev-&gt;flow_db-&gt;prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];</div><div class='add'>+		break;</div><div class='add'>+	case IB_FLOW_ATTR_SNIFFER:</div><div class='add'>+		if (!MLX5_CAP_FLOWTABLE(dev-&gt;mdev,</div><div class='add'>+					allow_sniffer_and_nic_rx_shared_tir))</div><div class='add'>+			return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+		ns = mlx5_get_flow_namespace(</div><div class='add'>+			dev-&gt;mdev, ft_type == MLX5_IB_FT_RX ?</div><div class='add'>+					   MLX5_FLOW_NAMESPACE_SNIFFER_RX :</div><div class='add'>+					   MLX5_FLOW_NAMESPACE_SNIFFER_TX);</div><div class='add'>+</div><div class='add'>+		prio = &amp;dev-&gt;flow_db-&gt;sniffer[ft_type];</div><div class='add'>+		priority = 0;</div><div class='add'>+		num_entries = 1;</div><div class='add'>+		num_groups = 1;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!ns)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	max_table_size = min_t(int, num_entries, max_table_size);</div><div class='add'>+</div><div class='add'>+	ft = prio-&gt;flow_table;</div><div class='add'>+	if (ft)</div><div class='add'>+		return prio;</div><div class='add'>+</div><div class='add'>+	ft_attr.prio = priority;</div><div class='add'>+	ft_attr.max_fte = max_table_size;</div><div class='add'>+	ft_attr.flags = flags;</div><div class='add'>+	ft_attr.autogroup.max_num_groups = num_groups;</div><div class='add'>+	return _get_prio(ns, prio, &amp;ft_attr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO,</div><div class='add'>+	RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO,</div><div class='add'>+	RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO,</div><div class='add'>+	RDMA_RX_ECN_OPCOUNTER_PRIO,</div><div class='add'>+	RDMA_RX_CNP_OPCOUNTER_PRIO,</div><div class='add'>+	RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO,</div><div class='add'>+	RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO,</div><div class='add'>+	RDMA_TX_CNP_OPCOUNTER_PRIO,</div><div class='add'>+	RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int set_vhca_port_spec(struct mlx5_ib_dev *dev, u32 port_num,</div><div class='add'>+			      struct mlx5_flow_spec *spec)</div><div class='add'>+{</div><div class='add'>+	if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev-&gt;mdev,</div><div class='add'>+					ft_field_support.source_vhca_port) ||</div><div class='add'>+	    !MLX5_CAP_FLOWTABLE_RDMA_TX(dev-&gt;mdev,</div><div class='add'>+					ft_field_support.source_vhca_port))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	MLX5_SET_TO_ONES(fte_match_param, &amp;spec-&gt;match_criteria,</div><div class='add'>+			 misc_parameters.source_vhca_port);</div><div class='add'>+	MLX5_SET(fte_match_param, &amp;spec-&gt;match_value,</div><div class='add'>+		 misc_parameters.source_vhca_port, port_num);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int set_ecn_ce_spec(struct mlx5_ib_dev *dev, u32 port_num,</div><div class='add'>+			   struct mlx5_flow_spec *spec, int ipv)</div><div class='add'>+{</div><div class='add'>+	if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev-&gt;mdev,</div><div class='add'>+					ft_field_support.outer_ip_version))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (mlx5_core_mp_enabled(dev-&gt;mdev) &amp;&amp;</div><div class='add'>+	    set_vhca_port_spec(dev, port_num, spec))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	MLX5_SET_TO_ONES(fte_match_param, spec-&gt;match_criteria,</div><div class='add'>+			 outer_headers.ip_ecn);</div><div class='add'>+	MLX5_SET(fte_match_param, spec-&gt;match_value, outer_headers.ip_ecn,</div><div class='add'>+		 INET_ECN_CE);</div><div class='add'>+	MLX5_SET_TO_ONES(fte_match_param, spec-&gt;match_criteria,</div><div class='add'>+			 outer_headers.ip_version);</div><div class='add'>+	MLX5_SET(fte_match_param, spec-&gt;match_value, outer_headers.ip_version,</div><div class='add'>+		 ipv);</div><div class='add'>+</div><div class='add'>+	spec-&gt;match_criteria_enable =</div><div class='add'>+		get_match_criteria_enable(spec-&gt;match_criteria);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int set_cnp_spec(struct mlx5_ib_dev *dev, u32 port_num,</div><div class='add'>+			struct mlx5_flow_spec *spec)</div><div class='add'>+{</div><div class='add'>+	if (mlx5_core_mp_enabled(dev-&gt;mdev) &amp;&amp;</div><div class='add'>+	    set_vhca_port_spec(dev, port_num, spec))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	MLX5_SET_TO_ONES(fte_match_param, spec-&gt;match_criteria,</div><div class='add'>+			 misc_parameters.bth_opcode);</div><div class='add'>+	MLX5_SET(fte_match_param, spec-&gt;match_value, misc_parameters.bth_opcode,</div><div class='add'>+		 IB_BTH_OPCODE_CNP);</div><div class='add'>+</div><div class='add'>+	spec-&gt;match_criteria_enable =</div><div class='add'>+		get_match_criteria_enable(spec-&gt;match_criteria);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* Returns the prio we should use for the given optional counter type,</div><div class='add'>+ * whereas for bytes type we use the packet type, since they share the same</div><div class='add'>+ * resources.</div><div class='add'>+ */</div><div class='add'>+static struct mlx5_ib_flow_prio *get_opfc_prio(struct mlx5_ib_dev *dev,</div><div class='add'>+					       u32 type)</div><div class='add'>+{</div><div class='add'>+	u32 prio_type;</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:</div><div class='add'>+		prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:</div><div class='add'>+		prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:</div><div class='add'>+		prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:</div><div class='add'>+		prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		prio_type = type;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return &amp;dev-&gt;flow_db-&gt;opfcs[prio_type];</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void put_per_qp_prio(struct mlx5_ib_dev *dev,</div><div class='add'>+			    enum mlx5_ib_optional_counter_type type)</div><div class='add'>+{</div><div class='add'>+	enum mlx5_ib_optional_counter_type per_qp_type;</div><div class='add'>+	struct mlx5_ib_flow_prio *prio;</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:</div><div class='add'>+		per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:</div><div class='add'>+		per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:</div><div class='add'>+		per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:</div><div class='add'>+		per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:</div><div class='add'>+		per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:</div><div class='add'>+		per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:</div><div class='add'>+		per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	prio = get_opfc_prio(dev, per_qp_type);</div><div class='add'>+	put_flow_table(dev, prio, true);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_per_qp_prio(struct mlx5_ib_dev *dev,</div><div class='add'>+			   enum mlx5_ib_optional_counter_type type)</div><div class='add'>+{</div><div class='add'>+	enum mlx5_ib_optional_counter_type per_qp_type;</div><div class='add'>+	struct mlx5_flow_table_attr ft_attr = {};</div><div class='add'>+	enum mlx5_flow_namespace_type fn_type;</div><div class='add'>+	struct mlx5_flow_namespace *ns;</div><div class='add'>+	struct mlx5_ib_flow_prio *prio;</div><div class='add'>+	int priority;</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:</div><div class='add'>+		fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;</div><div class='add'>+		priority = RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO;</div><div class='add'>+		per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:</div><div class='add'>+		fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;</div><div class='add'>+		priority = RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO;</div><div class='add'>+		per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:</div><div class='add'>+		fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;</div><div class='add'>+		priority = RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO;</div><div class='add'>+		per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:</div><div class='add'>+		fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;</div><div class='add'>+		priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;</div><div class='add'>+		per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:</div><div class='add'>+		fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;</div><div class='add'>+		priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;</div><div class='add'>+		per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:</div><div class='add'>+		fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;</div><div class='add'>+		priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;</div><div class='add'>+		per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:</div><div class='add'>+		fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;</div><div class='add'>+		priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;</div><div class='add'>+		per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ns = mlx5_get_flow_namespace(dev-&gt;mdev, fn_type);</div><div class='add'>+	if (!ns)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	prio = get_opfc_prio(dev, per_qp_type);</div><div class='add'>+	if (prio-&gt;flow_table)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	ft_attr.prio = priority;</div><div class='add'>+	ft_attr.max_fte = MLX5_FS_MAX_POOL_SIZE;</div><div class='add'>+	ft_attr.autogroup.max_num_groups = 1;</div><div class='add'>+	prio = _get_prio(ns, prio, &amp;ft_attr);</div><div class='add'>+	if (IS_ERR(prio))</div><div class='add'>+		return PTR_ERR(prio);</div><div class='add'>+</div><div class='add'>+	prio-&gt;refcount = 1;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct mlx5_per_qp_opfc *get_per_qp_opfc(struct xarray *qpn_opfc_xa,</div><div class='add'>+						u32 qp_num, bool *new)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_per_qp_opfc *per_qp_opfc;</div><div class='add'>+</div><div class='add'>+	*new = false;</div><div class='add'>+</div><div class='add'>+	per_qp_opfc = xa_load(qpn_opfc_xa, qp_num);</div><div class='add'>+	if (per_qp_opfc)</div><div class='add'>+		return per_qp_opfc;</div><div class='add'>+	per_qp_opfc = kzalloc(sizeof(*per_qp_opfc), GFP_KERNEL);</div><div class='add'>+</div><div class='add'>+	if (!per_qp_opfc)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	*new = true;</div><div class='add'>+	return per_qp_opfc;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int add_op_fc_rules(struct mlx5_ib_dev *dev,</div><div class='add'>+			   struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX],</div><div class='add'>+			   struct xarray *qpn_opfc_xa,</div><div class='add'>+			   struct mlx5_per_qp_opfc *per_qp_opfc,</div><div class='add'>+			   struct mlx5_ib_flow_prio *prio,</div><div class='add'>+			   enum mlx5_ib_optional_counter_type type,</div><div class='add'>+			   u32 qp_num, u32 port_num)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_op_fc *opfc = &amp;per_qp_opfc-&gt;opfcs[type], *in_use_opfc;</div><div class='add'>+	struct mlx5_flow_act flow_act = {};</div><div class='add'>+	struct mlx5_flow_destination dst;</div><div class='add'>+	struct mlx5_flow_spec *spec;</div><div class='add'>+	int i, err, spec_num;</div><div class='add'>+	bool is_tx;</div><div class='add'>+</div><div class='add'>+	if (opfc-&gt;fc)</div><div class='add'>+		return -EEXIST;</div><div class='add'>+</div><div class='add'>+	if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc-&gt;opfcs, type,</div><div class='add'>+					    &amp;in_use_opfc)) {</div><div class='add'>+		opfc-&gt;fc = in_use_opfc-&gt;fc;</div><div class='add'>+		opfc-&gt;rule[0] = in_use_opfc-&gt;rule[0];</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	opfc-&gt;fc = fc_arr[type];</div><div class='add'>+</div><div class='add'>+	spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL);</div><div class='add'>+	if (!spec) {</div><div class='add'>+		err = -ENOMEM;</div><div class='add'>+		goto null_fc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP:</div><div class='add'>+		if (set_ecn_ce_spec(dev, port_num, &amp;spec[0],</div><div class='add'>+				    MLX5_FS_IPV4_VERSION) ||</div><div class='add'>+		    set_ecn_ce_spec(dev, port_num, &amp;spec[1],</div><div class='add'>+				    MLX5_FS_IPV6_VERSION)) {</div><div class='add'>+			err = -EOPNOTSUPP;</div><div class='add'>+			goto free_spec;</div><div class='add'>+		}</div><div class='add'>+		spec_num = 2;</div><div class='add'>+		is_tx = false;</div><div class='add'>+</div><div class='add'>+		MLX5_SET_TO_ONES(fte_match_param, spec[1].match_criteria,</div><div class='add'>+				 misc_parameters.bth_dst_qp);</div><div class='add'>+		MLX5_SET(fte_match_param, spec[1].match_value,</div><div class='add'>+			 misc_parameters.bth_dst_qp, qp_num);</div><div class='add'>+		spec[1].match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP:</div><div class='add'>+		if (!MLX5_CAP_FLOWTABLE(</div><div class='add'>+			    dev-&gt;mdev,</div><div class='add'>+			    ft_field_support_2_nic_receive_rdma.bth_opcode) ||</div><div class='add'>+		    set_cnp_spec(dev, port_num, &amp;spec[0])) {</div><div class='add'>+			err = -EOPNOTSUPP;</div><div class='add'>+			goto free_spec;</div><div class='add'>+		}</div><div class='add'>+		spec_num = 1;</div><div class='add'>+		is_tx = false;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP:</div><div class='add'>+		if (!MLX5_CAP_FLOWTABLE(</div><div class='add'>+			    dev-&gt;mdev,</div><div class='add'>+			    ft_field_support_2_nic_transmit_rdma.bth_opcode) ||</div><div class='add'>+		    set_cnp_spec(dev, port_num, &amp;spec[0])) {</div><div class='add'>+			err = -EOPNOTSUPP;</div><div class='add'>+			goto free_spec;</div><div class='add'>+		}</div><div class='add'>+		spec_num = 1;</div><div class='add'>+		is_tx = true;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP:</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:</div><div class='add'>+		spec_num = 1;</div><div class='add'>+		is_tx = true;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP:</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:</div><div class='add'>+		spec_num = 1;</div><div class='add'>+		is_tx = false;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+		goto free_spec;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (is_tx) {</div><div class='add'>+		MLX5_SET_TO_ONES(fte_match_param, spec-&gt;match_criteria,</div><div class='add'>+				 misc_parameters.source_sqn);</div><div class='add'>+		MLX5_SET(fte_match_param, spec-&gt;match_value,</div><div class='add'>+			 misc_parameters.source_sqn, qp_num);</div><div class='add'>+	} else {</div><div class='add'>+		MLX5_SET_TO_ONES(fte_match_param, spec-&gt;match_criteria,</div><div class='add'>+				 misc_parameters.bth_dst_qp);</div><div class='add'>+		MLX5_SET(fte_match_param, spec-&gt;match_value,</div><div class='add'>+			 misc_parameters.bth_dst_qp, qp_num);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spec-&gt;match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;</div><div class='add'>+</div><div class='add'>+	dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;</div><div class='add'>+	dst.counter = opfc-&gt;fc;</div><div class='add'>+</div><div class='add'>+	flow_act.action =</div><div class='add'>+		MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; spec_num; i++) {</div><div class='add'>+		opfc-&gt;rule[i] = mlx5_add_flow_rules(prio-&gt;flow_table, &amp;spec[i],</div><div class='add'>+						    &amp;flow_act, &amp;dst, 1);</div><div class='add'>+		if (IS_ERR(opfc-&gt;rule[i])) {</div><div class='add'>+			err = PTR_ERR(opfc-&gt;rule[i]);</div><div class='add'>+			goto del_rules;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	prio-&gt;refcount += spec_num;</div><div class='add'>+</div><div class='add'>+	err = xa_err(xa_store(qpn_opfc_xa, qp_num, per_qp_opfc, GFP_KERNEL));</div><div class='add'>+	if (err)</div><div class='add'>+		goto del_rules;</div><div class='add'>+</div><div class='add'>+	kfree(spec);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+del_rules:</div><div class='add'>+	while (i--)</div><div class='add'>+		mlx5_del_flow_rules(opfc-&gt;rule[i]);</div><div class='add'>+	put_flow_table(dev, prio, false);</div><div class='add'>+free_spec:</div><div class='add'>+	kfree(spec);</div><div class='add'>+null_fc:</div><div class='add'>+	opfc-&gt;fc = NULL;</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool</div><div class='add'>+is_fc_shared_and_in_use(struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX], u32 type,</div><div class='add'>+			struct mlx5_fc **fc)</div><div class='add'>+{</div><div class='add'>+	u32 shared_fc_type;</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP:</div><div class='add'>+		shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:</div><div class='add'>+		shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP:</div><div class='add'>+		shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:</div><div class='add'>+		shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*fc = fc_arr[shared_fc_type];</div><div class='add'>+	if (!(*fc))</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5r_fs_destroy_fcs(struct mlx5_ib_dev *dev,</div><div class='add'>+			  struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX])</div><div class='add'>+{</div><div class='add'>+	struct mlx5_fc *in_use_fc;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;</div><div class='add'>+	     i &lt;= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) {</div><div class='add'>+		if (!fc_arr[i])</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		if (is_fc_shared_and_in_use(fc_arr, i, &amp;in_use_fc)) {</div><div class='add'>+			fc_arr[i] = NULL;</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		mlx5_fc_destroy(dev-&gt;mdev, fc_arr[i]);</div><div class='add'>+		fc_arr[i] = NULL;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev *dev, u32 port_num,</div><div class='add'>+			 struct mlx5_ib_op_fc *opfc,</div><div class='add'>+			 enum mlx5_ib_optional_counter_type type)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_flow_table_attr ft_attr = {};</div><div class='add'>+	enum mlx5_flow_namespace_type fn_type;</div><div class='add'>+	int priority, i, err, spec_num;</div><div class='add'>+	struct mlx5_flow_act flow_act = {};</div><div class='add'>+	struct mlx5_flow_destination dst;</div><div class='add'>+	struct mlx5_flow_namespace *ns;</div><div class='add'>+	struct mlx5_ib_flow_prio *prio;</div><div class='add'>+	struct mlx5_flow_spec *spec;</div><div class='add'>+</div><div class='add'>+	spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL);</div><div class='add'>+	if (!spec)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	switch (type) {</div><div class='add'>+	case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:</div><div class='add'>+		if (set_ecn_ce_spec(dev, port_num, &amp;spec[0],</div><div class='add'>+				    MLX5_FS_IPV4_VERSION) ||</div><div class='add'>+		    set_ecn_ce_spec(dev, port_num, &amp;spec[1],</div><div class='add'>+				    MLX5_FS_IPV6_VERSION)) {</div><div class='add'>+			err = -EOPNOTSUPP;</div><div class='add'>+			goto free;</div><div class='add'>+		}</div><div class='add'>+		spec_num = 2;</div><div class='add'>+		fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;</div><div class='add'>+		priority = RDMA_RX_ECN_OPCOUNTER_PRIO;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:</div><div class='add'>+		if (!MLX5_CAP_FLOWTABLE(dev-&gt;mdev,</div><div class='add'>+					ft_field_support_2_nic_receive_rdma.bth_opcode) ||</div><div class='add'>+		    set_cnp_spec(dev, port_num, &amp;spec[0])) {</div><div class='add'>+			err = -EOPNOTSUPP;</div><div class='add'>+			goto free;</div><div class='add'>+		}</div><div class='add'>+		spec_num = 1;</div><div class='add'>+		fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;</div><div class='add'>+		priority = RDMA_RX_CNP_OPCOUNTER_PRIO;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:</div><div class='add'>+		if (!MLX5_CAP_FLOWTABLE(dev-&gt;mdev,</div><div class='add'>+					ft_field_support_2_nic_transmit_rdma.bth_opcode) ||</div><div class='add'>+		    set_cnp_spec(dev, port_num, &amp;spec[0])) {</div><div class='add'>+			err = -EOPNOTSUPP;</div><div class='add'>+			goto free;</div><div class='add'>+		}</div><div class='add'>+		spec_num = 1;</div><div class='add'>+		fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;</div><div class='add'>+		priority = RDMA_TX_CNP_OPCOUNTER_PRIO;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:</div><div class='add'>+		spec_num = 1;</div><div class='add'>+		fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;</div><div class='add'>+		priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:</div><div class='add'>+	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:</div><div class='add'>+		spec_num = 1;</div><div class='add'>+		fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;</div><div class='add'>+		priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO;</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	default:</div><div class='add'>+		err = -EOPNOTSUPP;</div><div class='add'>+		goto free;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ns = mlx5_get_flow_namespace(dev-&gt;mdev, fn_type);</div><div class='add'>+	if (!ns) {</div><div class='add'>+		err = -EOPNOTSUPP;</div><div class='add'>+		goto free;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	prio = get_opfc_prio(dev, type);</div><div class='add'>+	if (!prio-&gt;flow_table) {</div><div class='add'>+		err = get_per_qp_prio(dev, type);</div><div class='add'>+		if (err)</div><div class='add'>+			goto free;</div><div class='add'>+</div><div class='add'>+		ft_attr.prio = priority;</div><div class='add'>+		ft_attr.max_fte = dev-&gt;num_ports * MAX_OPFC_RULES;</div><div class='add'>+		ft_attr.autogroup.max_num_groups = 1;</div><div class='add'>+		prio = _get_prio(ns, prio, &amp;ft_attr);</div><div class='add'>+		if (IS_ERR(prio)) {</div><div class='add'>+			err = PTR_ERR(prio);</div><div class='add'>+			goto put_prio;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;</div><div class='add'>+	dst.counter = opfc-&gt;fc;</div><div class='add'>+</div><div class='add'>+	flow_act.action =</div><div class='add'>+		MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; spec_num; i++) {</div><div class='add'>+		opfc-&gt;rule[i] = mlx5_add_flow_rules(prio-&gt;flow_table, &amp;spec[i],</div><div class='add'>+						    &amp;flow_act, &amp;dst, 1);</div><div class='add'>+		if (IS_ERR(opfc-&gt;rule[i])) {</div><div class='add'>+			err = PTR_ERR(opfc-&gt;rule[i]);</div><div class='add'>+			goto del_rules;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	prio-&gt;refcount += spec_num;</div><div class='add'>+	kfree(spec);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+del_rules:</div><div class='add'>+	for (i -= 1; i &gt;= 0; i--)</div><div class='add'>+		mlx5_del_flow_rules(opfc-&gt;rule[i]);</div><div class='add'>+	put_flow_table(dev, prio, false);</div><div class='add'>+put_prio:</div><div class='add'>+	put_per_qp_prio(dev, type);</div><div class='add'>+free:</div><div class='add'>+	kfree(spec);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev *dev,</div><div class='add'>+			     struct mlx5_ib_op_fc *opfc,</div><div class='add'>+			     enum mlx5_ib_optional_counter_type type)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_flow_prio *prio;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	prio = get_opfc_prio(dev, type);</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; MAX_OPFC_RULES &amp;&amp; opfc-&gt;rule[i]; i++) {</div><div class='add'>+		mlx5_del_flow_rules(opfc-&gt;rule[i]);</div><div class='add'>+		put_flow_table(dev, prio, true);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	put_per_qp_prio(dev, type);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5r_fs_unbind_op_fc(struct ib_qp *qp, struct xarray *qpn_opfc_xa)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(qp-&gt;device);</div><div class='add'>+	struct mlx5_per_qp_opfc *per_qp_opfc;</div><div class='add'>+	struct mlx5_ib_op_fc *in_use_opfc;</div><div class='add'>+	struct mlx5_ib_flow_prio *prio;</div><div class='add'>+	int i, j;</div><div class='add'>+</div><div class='add'>+	per_qp_opfc = xa_load(qpn_opfc_xa, qp-&gt;qp_num);</div><div class='add'>+	if (!per_qp_opfc)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;</div><div class='add'>+	     i &lt;= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) {</div><div class='add'>+		if (!per_qp_opfc-&gt;opfcs[i].fc)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc-&gt;opfcs, i,</div><div class='add'>+						    &amp;in_use_opfc)) {</div><div class='add'>+			per_qp_opfc-&gt;opfcs[i].fc = NULL;</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		for (j = 0; j &lt; MAX_OPFC_RULES; j++) {</div><div class='add'>+			if (!per_qp_opfc-&gt;opfcs[i].rule[j])</div><div class='add'>+				continue;</div><div class='add'>+			mlx5_del_flow_rules(per_qp_opfc-&gt;opfcs[i].rule[j]);</div><div class='add'>+			prio = get_opfc_prio(dev, i);</div><div class='add'>+			put_flow_table(dev, prio, true);</div><div class='add'>+		}</div><div class='add'>+		per_qp_opfc-&gt;opfcs[i].fc = NULL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	kfree(per_qp_opfc);</div><div class='add'>+	xa_erase(qpn_opfc_xa, qp-&gt;qp_num);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5r_fs_bind_op_fc(struct ib_qp *qp,</div><div class='add'>+			struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX],</div><div class='add'>+			struct xarray *qpn_opfc_xa, u32 port)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(qp-&gt;device);</div><div class='add'>+	struct mlx5_per_qp_opfc *per_qp_opfc;</div><div class='add'>+	struct mlx5_ib_flow_prio *prio;</div><div class='add'>+	struct mlx5_ib_counters *cnts;</div><div class='add'>+	struct mlx5_ib_op_fc *opfc;</div><div class='add'>+	struct mlx5_fc *in_use_fc;</div><div class='add'>+	int i, err, per_qp_type;</div><div class='add'>+	bool new;</div><div class='add'>+</div><div class='add'>+	cnts = &amp;dev-&gt;port[port - 1].cnts;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt;= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES; i++) {</div><div class='add'>+		opfc = &amp;cnts-&gt;opfcs[i];</div><div class='add'>+		if (!opfc-&gt;fc)</div><div class='add'>+			continue;</div><div class='add'>+</div><div class='add'>+		per_qp_type = i + MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;</div><div class='add'>+		prio = get_opfc_prio(dev, per_qp_type);</div><div class='add'>+		WARN_ON(!prio-&gt;flow_table);</div><div class='add'>+</div><div class='add'>+		if (is_fc_shared_and_in_use(fc_arr, per_qp_type, &amp;in_use_fc))</div><div class='add'>+			fc_arr[per_qp_type] = in_use_fc;</div><div class='add'>+</div><div class='add'>+		if (!fc_arr[per_qp_type]) {</div><div class='add'>+			fc_arr[per_qp_type] = mlx5_fc_create(dev-&gt;mdev, false);</div><div class='add'>+			if (IS_ERR(fc_arr[per_qp_type]))</div><div class='add'>+				return PTR_ERR(fc_arr[per_qp_type]);</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		per_qp_opfc = get_per_qp_opfc(qpn_opfc_xa, qp-&gt;qp_num, &amp;new);</div><div class='add'>+		if (!per_qp_opfc) {</div><div class='add'>+			err = -ENOMEM;</div><div class='add'>+			goto free_fc;</div><div class='add'>+		}</div><div class='add'>+		err = add_op_fc_rules(dev, fc_arr, qpn_opfc_xa, per_qp_opfc,</div><div class='add'>+				      prio, per_qp_type, qp-&gt;qp_num, port);</div><div class='add'>+		if (err)</div><div class='add'>+			goto del_rules;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+del_rules:</div><div class='add'>+	mlx5r_fs_unbind_op_fc(qp, qpn_opfc_xa);</div><div class='add'>+	if (new)</div><div class='add'>+		kfree(per_qp_opfc);</div><div class='add'>+free_fc:</div><div class='add'>+	if (xa_empty(qpn_opfc_xa))</div><div class='add'>+		mlx5r_fs_destroy_fcs(dev, fc_arr);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void set_underlay_qp(struct mlx5_ib_dev *dev,</div><div class='add'>+			    struct mlx5_flow_spec *spec,</div><div class='add'>+			    u32 underlay_qpn)</div><div class='add'>+{</div><div class='add'>+	void *misc_params_c = MLX5_ADDR_OF(fte_match_param,</div><div class='add'>+					   spec-&gt;match_criteria,</div><div class='add'>+					   misc_parameters);</div><div class='add'>+	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec-&gt;match_value,</div><div class='add'>+					   misc_parameters);</div><div class='add'>+</div><div class='add'>+	if (underlay_qpn &amp;&amp;</div><div class='add'>+	    MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev,</div><div class='add'>+				      ft_field_support.bth_dst_qp)) {</div><div class='add'>+		MLX5_SET(fte_match_set_misc,</div><div class='add'>+			 misc_params_v, bth_dst_qp, underlay_qpn);</div><div class='add'>+		MLX5_SET(fte_match_set_misc,</div><div class='add'>+			 misc_params_c, bth_dst_qp, 0xffffff);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev,</div><div class='add'>+					 struct mlx5_flow_spec *spec,</div><div class='add'>+					 struct mlx5_eswitch_rep *rep)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_eswitch *esw = dev-&gt;mdev-&gt;priv.eswitch;</div><div class='add'>+	void *misc;</div><div class='add'>+</div><div class='add'>+	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {</div><div class='add'>+		misc = MLX5_ADDR_OF(fte_match_param, spec-&gt;match_value,</div><div class='add'>+				    misc_parameters_2);</div><div class='add'>+</div><div class='add'>+		MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,</div><div class='add'>+			 mlx5_eswitch_get_vport_metadata_for_match(rep-&gt;esw,</div><div class='add'>+								   rep-&gt;vport));</div><div class='add'>+		misc = MLX5_ADDR_OF(fte_match_param, spec-&gt;match_criteria,</div><div class='add'>+				    misc_parameters_2);</div><div class='add'>+</div><div class='add'>+		MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,</div><div class='add'>+			 mlx5_eswitch_get_vport_metadata_mask());</div><div class='add'>+	} else {</div><div class='add'>+		misc = MLX5_ADDR_OF(fte_match_param, spec-&gt;match_value,</div><div class='add'>+				    misc_parameters);</div><div class='add'>+</div><div class='add'>+		MLX5_SET(fte_match_set_misc, misc, source_port, rep-&gt;vport);</div><div class='add'>+</div><div class='add'>+		misc = MLX5_ADDR_OF(fte_match_param, spec-&gt;match_criteria,</div><div class='add'>+				    misc_parameters);</div><div class='add'>+</div><div class='add'>+		MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,</div><div class='add'>+						      struct mlx5_ib_flow_prio *ft_prio,</div><div class='add'>+						      const struct ib_flow_attr *flow_attr,</div><div class='add'>+						      struct mlx5_flow_destination *dst,</div><div class='add'>+						      u32 underlay_qpn,</div><div class='add'>+						      struct mlx5_ib_create_flow *ucmd)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_flow_table	*ft = ft_prio-&gt;flow_table;</div><div class='add'>+	struct mlx5_ib_flow_handler *handler;</div><div class='add'>+	struct mlx5_flow_act flow_act = {};</div><div class='add'>+	struct mlx5_flow_spec *spec;</div><div class='add'>+	struct mlx5_flow_destination dest_arr[2] = {};</div><div class='add'>+	struct mlx5_flow_destination *rule_dst = dest_arr;</div><div class='add'>+	const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);</div><div class='add'>+	unsigned int spec_index;</div><div class='add'>+	u32 prev_type = 0;</div><div class='add'>+	int err = 0;</div><div class='add'>+	int dest_num = 0;</div><div class='add'>+	bool is_egress = flow_attr-&gt;flags &amp; IB_FLOW_ATTR_FLAGS_EGRESS;</div><div class='add'>+</div><div class='add'>+	if (!is_valid_attr(dev-&gt;mdev, flow_attr))</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;is_rep &amp;&amp; is_egress)</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);</div><div class='add'>+	handler = kzalloc(sizeof(*handler), GFP_KERNEL);</div><div class='add'>+	if (!handler || !spec) {</div><div class='add'>+		err = -ENOMEM;</div><div class='add'>+		goto free;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;handler-&gt;list);</div><div class='add'>+</div><div class='add'>+	for (spec_index = 0; spec_index &lt; flow_attr-&gt;num_of_specs; spec_index++) {</div><div class='add'>+		err = parse_flow_attr(dev-&gt;mdev, spec,</div><div class='add'>+				      ib_flow, flow_attr, &amp;flow_act,</div><div class='add'>+				      prev_type);</div><div class='add'>+		if (err &lt; 0)</div><div class='add'>+			goto free;</div><div class='add'>+</div><div class='add'>+		prev_type = ((union ib_flow_spec *)ib_flow)-&gt;type;</div><div class='add'>+		ib_flow += ((union ib_flow_spec *)ib_flow)-&gt;size;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (dst &amp;&amp; !(flow_act.action &amp; MLX5_FLOW_CONTEXT_ACTION_DROP)) {</div><div class='add'>+		memcpy(&amp;dest_arr[0], dst, sizeof(*dst));</div><div class='add'>+		dest_num++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!flow_is_multicast_only(flow_attr))</div><div class='add'>+		set_underlay_qp(dev, spec, underlay_qpn);</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;is_rep &amp;&amp; flow_attr-&gt;type != IB_FLOW_ATTR_SNIFFER) {</div><div class='add'>+		struct mlx5_eswitch_rep *rep;</div><div class='add'>+</div><div class='add'>+		rep = dev-&gt;port[flow_attr-&gt;port - 1].rep;</div><div class='add'>+		if (!rep) {</div><div class='add'>+			err = -EINVAL;</div><div class='add'>+			goto free;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		mlx5_ib_set_rule_source_port(dev, spec, rep);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	spec-&gt;match_criteria_enable = get_match_criteria_enable(spec-&gt;match_criteria);</div><div class='add'>+</div><div class='add'>+	if (flow_act.action &amp; MLX5_FLOW_CONTEXT_ACTION_COUNT) {</div><div class='add'>+		struct mlx5_ib_mcounters *mcounters;</div><div class='add'>+</div><div class='add'>+		err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd);</div><div class='add'>+		if (err)</div><div class='add'>+			goto free;</div><div class='add'>+</div><div class='add'>+		mcounters = to_mcounters(flow_act.counters);</div><div class='add'>+		handler-&gt;ibcounters = flow_act.counters;</div><div class='add'>+		dest_arr[dest_num].type =</div><div class='add'>+			MLX5_FLOW_DESTINATION_TYPE_COUNTER;</div><div class='add'>+		dest_arr[dest_num].counter =</div><div class='add'>+			mcounters-&gt;hw_cntrs_hndl;</div><div class='add'>+		dest_num++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (flow_act.action &amp; MLX5_FLOW_CONTEXT_ACTION_DROP) {</div><div class='add'>+		if (!dest_num)</div><div class='add'>+			rule_dst = NULL;</div><div class='add'>+	} else {</div><div class='add'>+		if (flow_attr-&gt;flags &amp; IB_FLOW_ATTR_FLAGS_DONT_TRAP)</div><div class='add'>+			flow_act.action |=</div><div class='add'>+				MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;</div><div class='add'>+		if (is_egress)</div><div class='add'>+			flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;</div><div class='add'>+		else if (dest_num)</div><div class='add'>+			flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if ((spec-&gt;flow_context.flags &amp; FLOW_CONTEXT_HAS_TAG)  &amp;&amp;</div><div class='add'>+	    (flow_attr-&gt;type == IB_FLOW_ATTR_ALL_DEFAULT ||</div><div class='add'>+	     flow_attr-&gt;type == IB_FLOW_ATTR_MC_DEFAULT)) {</div><div class='add'>+		mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n",</div><div class='add'>+			     spec-&gt;flow_context.flow_tag, flow_attr-&gt;type);</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+		goto free;</div><div class='add'>+	}</div><div class='add'>+	handler-&gt;rule = mlx5_add_flow_rules(ft, spec,</div><div class='add'>+					    &amp;flow_act,</div><div class='add'>+					    rule_dst, dest_num);</div><div class='add'>+</div><div class='add'>+	if (IS_ERR(handler-&gt;rule)) {</div><div class='add'>+		err = PTR_ERR(handler-&gt;rule);</div><div class='add'>+		goto free;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ft_prio-&gt;refcount++;</div><div class='add'>+	handler-&gt;prio = ft_prio;</div><div class='add'>+	handler-&gt;dev = dev;</div><div class='add'>+</div><div class='add'>+	ft_prio-&gt;flow_table = ft;</div><div class='add'>+free:</div><div class='add'>+	if (err &amp;&amp; handler) {</div><div class='add'>+		mlx5_ib_counters_clear_description(handler-&gt;ibcounters);</div><div class='add'>+		kfree(handler);</div><div class='add'>+	}</div><div class='add'>+	kvfree(spec);</div><div class='add'>+	return err ? ERR_PTR(err) : handler;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,</div><div class='add'>+						     struct mlx5_ib_flow_prio *ft_prio,</div><div class='add'>+						     const struct ib_flow_attr *flow_attr,</div><div class='add'>+						     struct mlx5_flow_destination *dst)</div><div class='add'>+{</div><div class='add'>+	return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev,</div><div class='add'>+							  struct mlx5_ib_flow_prio *ft_prio,</div><div class='add'>+							  struct ib_flow_attr *flow_attr,</div><div class='add'>+							  struct mlx5_flow_destination *dst)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_flow_handler *handler_ucast = NULL;</div><div class='add'>+	struct mlx5_ib_flow_handler *handler = NULL;</div><div class='add'>+</div><div class='add'>+	static struct {</div><div class='add'>+		struct ib_flow_spec_eth eth_flow;</div><div class='add'>+		struct ib_flow_attr	flow_attr;</div><div class='add'>+	} leftovers_wc = { .flow_attr = { .num_of_specs = 1,</div><div class='add'>+					  .size = sizeof(leftovers_wc) },</div><div class='add'>+			   .eth_flow = {</div><div class='add'>+				   .type = IB_FLOW_SPEC_ETH,</div><div class='add'>+				   .size = sizeof(struct ib_flow_spec_eth),</div><div class='add'>+				   .mask = { .dst_mac = { 0x1 } },</div><div class='add'>+				   .val = { .dst_mac = { 0x1 } } } };</div><div class='add'>+</div><div class='add'>+	static struct {</div><div class='add'>+		struct ib_flow_spec_eth eth_flow;</div><div class='add'>+		struct ib_flow_attr	flow_attr;</div><div class='add'>+	} leftovers_uc = { .flow_attr = { .num_of_specs = 1,</div><div class='add'>+					  .size = sizeof(leftovers_uc) },</div><div class='add'>+			   .eth_flow = {</div><div class='add'>+				   .type = IB_FLOW_SPEC_ETH,</div><div class='add'>+				   .size = sizeof(struct ib_flow_spec_eth),</div><div class='add'>+				   .mask = { .dst_mac = { 0x1 } },</div><div class='add'>+				   .val = { .dst_mac = {} } } };</div><div class='add'>+</div><div class='add'>+	handler = create_flow_rule(dev, ft_prio, &amp;leftovers_wc.flow_attr, dst);</div><div class='add'>+	if (!IS_ERR(handler) &amp;&amp;</div><div class='add'>+	    flow_attr-&gt;type == IB_FLOW_ATTR_ALL_DEFAULT) {</div><div class='add'>+		handler_ucast = create_flow_rule(dev, ft_prio,</div><div class='add'>+						 &amp;leftovers_uc.flow_attr, dst);</div><div class='add'>+		if (IS_ERR(handler_ucast)) {</div><div class='add'>+			mlx5_del_flow_rules(handler-&gt;rule);</div><div class='add'>+			ft_prio-&gt;refcount--;</div><div class='add'>+			kfree(handler);</div><div class='add'>+			handler = handler_ucast;</div><div class='add'>+		} else {</div><div class='add'>+			list_add(&amp;handler_ucast-&gt;list, &amp;handler-&gt;list);</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return handler;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,</div><div class='add'>+							struct mlx5_ib_flow_prio *ft_rx,</div><div class='add'>+							struct mlx5_ib_flow_prio *ft_tx,</div><div class='add'>+							struct mlx5_flow_destination *dst)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_flow_handler *handler_rx;</div><div class='add'>+	struct mlx5_ib_flow_handler *handler_tx;</div><div class='add'>+	int err;</div><div class='add'>+	static const struct ib_flow_attr flow_attr  = {</div><div class='add'>+		.num_of_specs = 0,</div><div class='add'>+		.type = IB_FLOW_ATTR_SNIFFER,</div><div class='add'>+		.size = sizeof(flow_attr)</div><div class='add'>+	};</div><div class='add'>+</div><div class='add'>+	handler_rx = create_flow_rule(dev, ft_rx, &amp;flow_attr, dst);</div><div class='add'>+	if (IS_ERR(handler_rx)) {</div><div class='add'>+		err = PTR_ERR(handler_rx);</div><div class='add'>+		goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	handler_tx = create_flow_rule(dev, ft_tx, &amp;flow_attr, dst);</div><div class='add'>+	if (IS_ERR(handler_tx)) {</div><div class='add'>+		err = PTR_ERR(handler_tx);</div><div class='add'>+		goto err_tx;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	list_add(&amp;handler_tx-&gt;list, &amp;handler_rx-&gt;list);</div><div class='add'>+</div><div class='add'>+	return handler_rx;</div><div class='add'>+</div><div class='add'>+err_tx:</div><div class='add'>+	mlx5_del_flow_rules(handler_rx-&gt;rule);</div><div class='add'>+	ft_rx-&gt;refcount--;</div><div class='add'>+	kfree(handler_rx);</div><div class='add'>+err:</div><div class='add'>+	return ERR_PTR(err);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,</div><div class='add'>+					   struct ib_flow_attr *flow_attr,</div><div class='add'>+					   struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(qp-&gt;device);</div><div class='add'>+	struct mlx5_ib_qp *mqp = to_mqp(qp);</div><div class='add'>+	struct mlx5_ib_flow_handler *handler = NULL;</div><div class='add'>+	struct mlx5_flow_destination *dst = NULL;</div><div class='add'>+	struct mlx5_ib_flow_prio *ft_prio_tx = NULL;</div><div class='add'>+	struct mlx5_ib_flow_prio *ft_prio;</div><div class='add'>+	bool is_egress = flow_attr-&gt;flags &amp; IB_FLOW_ATTR_FLAGS_EGRESS;</div><div class='add'>+	struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr;</div><div class='add'>+	size_t min_ucmd_sz, required_ucmd_sz;</div><div class='add'>+	int err;</div><div class='add'>+	int underlay_qpn;</div><div class='add'>+</div><div class='add'>+	if (udata &amp;&amp; udata-&gt;inlen) {</div><div class='add'>+		min_ucmd_sz = offsetofend(struct mlx5_ib_create_flow, reserved);</div><div class='add'>+		if (udata-&gt;inlen &lt; min_ucmd_sz)</div><div class='add'>+			return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+		err = ib_copy_from_udata(&amp;ucmd_hdr, udata, min_ucmd_sz);</div><div class='add'>+		if (err)</div><div class='add'>+			return ERR_PTR(err);</div><div class='add'>+</div><div class='add'>+		/* currently supports only one counters data */</div><div class='add'>+		if (ucmd_hdr.ncounters_data &gt; 1)</div><div class='add'>+			return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+		required_ucmd_sz = min_ucmd_sz +</div><div class='add'>+			sizeof(struct mlx5_ib_flow_counters_data) *</div><div class='add'>+			ucmd_hdr.ncounters_data;</div><div class='add'>+		if (udata-&gt;inlen &gt; required_ucmd_sz &amp;&amp;</div><div class='add'>+		    !ib_is_udata_cleared(udata, required_ucmd_sz,</div><div class='add'>+					 udata-&gt;inlen - required_ucmd_sz))</div><div class='add'>+			return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+		ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL);</div><div class='add'>+		if (!ucmd)</div><div class='add'>+			return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+		err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz);</div><div class='add'>+		if (err)</div><div class='add'>+			goto free_ucmd;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (flow_attr-&gt;priority &gt; MLX5_IB_FLOW_LAST_PRIO) {</div><div class='add'>+		err = -ENOMEM;</div><div class='add'>+		goto free_ucmd;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (flow_attr-&gt;flags &amp;</div><div class='add'>+	    ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | IB_FLOW_ATTR_FLAGS_EGRESS)) {</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+		goto free_ucmd;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (is_egress &amp;&amp;</div><div class='add'>+	    (flow_attr-&gt;type == IB_FLOW_ATTR_ALL_DEFAULT ||</div><div class='add'>+	     flow_attr-&gt;type == IB_FLOW_ATTR_MC_DEFAULT)) {</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+		goto free_ucmd;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dst = kzalloc(sizeof(*dst), GFP_KERNEL);</div><div class='add'>+	if (!dst) {</div><div class='add'>+		err = -ENOMEM;</div><div class='add'>+		goto free_ucmd;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='add'>+</div><div class='add'>+	ft_prio = get_flow_table(dev, flow_attr,</div><div class='add'>+				 is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX);</div><div class='add'>+	if (IS_ERR(ft_prio)) {</div><div class='add'>+		err = PTR_ERR(ft_prio);</div><div class='add'>+		goto unlock;</div><div class='add'>+	}</div><div class='add'>+	if (flow_attr-&gt;type == IB_FLOW_ATTR_SNIFFER) {</div><div class='add'>+		ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX);</div><div class='add'>+		if (IS_ERR(ft_prio_tx)) {</div><div class='add'>+			err = PTR_ERR(ft_prio_tx);</div><div class='add'>+			ft_prio_tx = NULL;</div><div class='add'>+			goto destroy_ft;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (is_egress) {</div><div class='add'>+		dst-&gt;type = MLX5_FLOW_DESTINATION_TYPE_PORT;</div><div class='add'>+	} else {</div><div class='add'>+		dst-&gt;type = MLX5_FLOW_DESTINATION_TYPE_TIR;</div><div class='add'>+		if (mqp-&gt;is_rss)</div><div class='add'>+			dst-&gt;tir_num = mqp-&gt;rss_qp.tirn;</div><div class='add'>+		else</div><div class='add'>+			dst-&gt;tir_num = mqp-&gt;raw_packet_qp.rq.tirn;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (flow_attr-&gt;type) {</div><div class='add'>+	case IB_FLOW_ATTR_NORMAL:</div><div class='add'>+		underlay_qpn = (mqp-&gt;flags &amp; IB_QP_CREATE_SOURCE_QPN) ?</div><div class='add'>+				       mqp-&gt;underlay_qpn :</div><div class='add'>+				       0;</div><div class='add'>+		handler = _create_flow_rule(dev, ft_prio, flow_attr, dst,</div><div class='add'>+					    underlay_qpn, ucmd);</div><div class='add'>+		break;</div><div class='add'>+	case IB_FLOW_ATTR_ALL_DEFAULT:</div><div class='add'>+	case IB_FLOW_ATTR_MC_DEFAULT:</div><div class='add'>+		handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst);</div><div class='add'>+		break;</div><div class='add'>+	case IB_FLOW_ATTR_SNIFFER:</div><div class='add'>+		handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+		goto destroy_ft;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (IS_ERR(handler)) {</div><div class='add'>+		err = PTR_ERR(handler);</div><div class='add'>+		handler = NULL;</div><div class='add'>+		goto destroy_ft;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='add'>+	kfree(dst);</div><div class='add'>+	kfree(ucmd);</div><div class='add'>+</div><div class='add'>+	return &amp;handler-&gt;ibflow;</div><div class='add'>+</div><div class='add'>+destroy_ft:</div><div class='add'>+	put_flow_table(dev, ft_prio, false);</div><div class='add'>+	if (ft_prio_tx)</div><div class='add'>+		put_flow_table(dev, ft_prio_tx, false);</div><div class='add'>+unlock:</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='add'>+	kfree(dst);</div><div class='add'>+free_ucmd:</div><div class='add'>+	kfree(ucmd);</div><div class='add'>+	return ERR_PTR(err);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_fill_transport_ns_info(struct mlx5_ib_dev *dev,</div><div class='add'>+					  enum mlx5_flow_namespace_type type,</div><div class='add'>+					  u32 *flags, u16 *vport_idx,</div><div class='add'>+					  u16 *vport,</div><div class='add'>+					  struct mlx5_core_dev **ft_mdev,</div><div class='add'>+					  u32 ib_port, u16 *esw_owner_vhca_id)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_core_dev *esw_mdev;</div><div class='add'>+</div><div class='add'>+	if (!is_mdev_switchdev_mode(dev-&gt;mdev))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (!MLX5_CAP_ADV_RDMA(dev-&gt;mdev, rdma_transport_manager))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;port[ib_port - 1].rep)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	esw_mdev = mlx5_eswitch_get_core_dev(dev-&gt;port[ib_port - 1].rep-&gt;esw);</div><div class='add'>+	if (esw_mdev != dev-&gt;mdev) {</div><div class='add'>+		if (!MLX5_CAP_ADV_RDMA(dev-&gt;mdev,</div><div class='add'>+				       rdma_transport_manager_other_eswitch))</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+		*flags |= MLX5_FLOW_TABLE_OTHER_ESWITCH;</div><div class='add'>+		*esw_owner_vhca_id = MLX5_CAP_GEN(esw_mdev, vhca_id);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	*flags |= MLX5_FLOW_TABLE_OTHER_VPORT;</div><div class='add'>+	*ft_mdev = esw_mdev;</div><div class='add'>+	*vport = dev-&gt;port[ib_port - 1].rep-&gt;vport;</div><div class='add'>+	*vport_idx = dev-&gt;port[ib_port - 1].rep-&gt;vport_index;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct mlx5_ib_flow_prio *</div><div class='add'>+_get_flow_table(struct mlx5_ib_dev *dev, u16 user_priority,</div><div class='add'>+		enum mlx5_flow_namespace_type ns_type,</div><div class='add'>+		bool mcast, u32 ib_port)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_core_dev *ft_mdev = dev-&gt;mdev;</div><div class='add'>+	struct mlx5_flow_table_attr ft_attr = {};</div><div class='add'>+	struct mlx5_flow_namespace *ns = NULL;</div><div class='add'>+	struct mlx5_ib_flow_prio *prio = NULL;</div><div class='add'>+	u16 esw_owner_vhca_id = 0;</div><div class='add'>+	int max_table_size = 0;</div><div class='add'>+	u16 vport_idx = 0;</div><div class='add'>+	bool esw_encap;</div><div class='add'>+	u32 flags = 0;</div><div class='add'>+	u16 vport = 0;</div><div class='add'>+	int priority;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (mcast)</div><div class='add'>+		priority = MLX5_IB_FLOW_MCAST_PRIO;</div><div class='add'>+	else</div><div class='add'>+		priority = ib_prio_to_core_prio(user_priority, false);</div><div class='add'>+</div><div class='add'>+	esw_encap = mlx5_eswitch_get_encap_mode(dev-&gt;mdev) !=</div><div class='add'>+		DEVLINK_ESWITCH_ENCAP_MODE_NONE;</div><div class='add'>+	switch (ns_type) {</div><div class='add'>+	case MLX5_FLOW_NAMESPACE_BYPASS:</div><div class='add'>+		max_table_size = BIT(</div><div class='add'>+			MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev, log_max_ft_size));</div><div class='add'>+		if (MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev, decap) &amp;&amp; !esw_encap)</div><div class='add'>+			flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;</div><div class='add'>+		if (MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev,</div><div class='add'>+					      reformat_l3_tunnel_to_l2) &amp;&amp;</div><div class='add'>+		    !esw_encap)</div><div class='add'>+			flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_FLOW_NAMESPACE_EGRESS:</div><div class='add'>+		max_table_size = BIT(</div><div class='add'>+			MLX5_CAP_FLOWTABLE_NIC_TX(dev-&gt;mdev, log_max_ft_size));</div><div class='add'>+		if (MLX5_CAP_FLOWTABLE_NIC_TX(dev-&gt;mdev, reformat) &amp;&amp;</div><div class='add'>+		    !esw_encap)</div><div class='add'>+			flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_FLOW_NAMESPACE_FDB_BYPASS:</div><div class='add'>+		max_table_size = BIT(</div><div class='add'>+			MLX5_CAP_ESW_FLOWTABLE_FDB(dev-&gt;mdev, log_max_ft_size));</div><div class='add'>+		if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev-&gt;mdev, decap) &amp;&amp; esw_encap)</div><div class='add'>+			flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;</div><div class='add'>+		if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev-&gt;mdev,</div><div class='add'>+					       reformat_l3_tunnel_to_l2) &amp;&amp;</div><div class='add'>+		    esw_encap)</div><div class='add'>+			flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;</div><div class='add'>+		priority = user_priority;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_FLOW_NAMESPACE_RDMA_RX:</div><div class='add'>+		max_table_size = BIT(</div><div class='add'>+			MLX5_CAP_FLOWTABLE_RDMA_RX(dev-&gt;mdev, log_max_ft_size));</div><div class='add'>+		priority = user_priority;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_FLOW_NAMESPACE_RDMA_TX:</div><div class='add'>+		max_table_size = BIT(</div><div class='add'>+			MLX5_CAP_FLOWTABLE_RDMA_TX(dev-&gt;mdev, log_max_ft_size));</div><div class='add'>+		priority = user_priority;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX:</div><div class='add'>+	case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX:</div><div class='add'>+		if (ib_port == 0 ||</div><div class='add'>+		    user_priority &gt;= MLX5_RDMA_TRANSPORT_BYPASS_PRIO)</div><div class='add'>+			return ERR_PTR(-EINVAL);</div><div class='add'>+		ret = mlx5_ib_fill_transport_ns_info(dev, ns_type, &amp;flags,</div><div class='add'>+						     &amp;vport_idx, &amp;vport,</div><div class='add'>+						     &amp;ft_mdev, ib_port,</div><div class='add'>+						     &amp;esw_owner_vhca_id);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ERR_PTR(ret);</div><div class='add'>+</div><div class='add'>+		if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX)</div><div class='add'>+			max_table_size =</div><div class='add'>+				BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_RX(</div><div class='add'>+					ft_mdev, log_max_ft_size));</div><div class='add'>+		else</div><div class='add'>+			max_table_size =</div><div class='add'>+				BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_TX(</div><div class='add'>+					ft_mdev, log_max_ft_size));</div><div class='add'>+		priority = user_priority;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES);</div><div class='add'>+</div><div class='add'>+	if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX ||</div><div class='add'>+	    ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX)</div><div class='add'>+		ns = mlx5_get_flow_vport_namespace(ft_mdev, ns_type, vport_idx);</div><div class='add'>+	else</div><div class='add'>+		ns = mlx5_get_flow_namespace(ft_mdev, ns_type);</div><div class='add'>+</div><div class='add'>+	if (!ns)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	switch (ns_type) {</div><div class='add'>+	case MLX5_FLOW_NAMESPACE_BYPASS:</div><div class='add'>+		prio = &amp;dev-&gt;flow_db-&gt;prios[priority];</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_FLOW_NAMESPACE_EGRESS:</div><div class='add'>+		prio = &amp;dev-&gt;flow_db-&gt;egress_prios[priority];</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_FLOW_NAMESPACE_FDB_BYPASS:</div><div class='add'>+		prio = &amp;dev-&gt;flow_db-&gt;fdb[priority];</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_FLOW_NAMESPACE_RDMA_RX:</div><div class='add'>+		prio = &amp;dev-&gt;flow_db-&gt;rdma_rx[priority];</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_FLOW_NAMESPACE_RDMA_TX:</div><div class='add'>+		prio = &amp;dev-&gt;flow_db-&gt;rdma_tx[priority];</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX:</div><div class='add'>+		prio = &amp;dev-&gt;flow_db-&gt;rdma_transport_rx[priority][ib_port - 1];</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX:</div><div class='add'>+		prio = &amp;dev-&gt;flow_db-&gt;rdma_transport_tx[priority][ib_port - 1];</div><div class='add'>+		break;</div><div class='add'>+	default: return ERR_PTR(-EINVAL);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!prio)</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	if (prio-&gt;flow_table)</div><div class='add'>+		return prio;</div><div class='add'>+</div><div class='add'>+	ft_attr.prio = priority;</div><div class='add'>+	ft_attr.max_fte = max_table_size;</div><div class='add'>+	ft_attr.flags = flags;</div><div class='add'>+	ft_attr.vport = vport;</div><div class='add'>+	ft_attr.esw_owner_vhca_id = esw_owner_vhca_id;</div><div class='add'>+	ft_attr.autogroup.max_num_groups = MLX5_FS_MAX_TYPES;</div><div class='add'>+	return _get_prio(ns, prio, &amp;ft_attr);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct mlx5_ib_flow_handler *</div><div class='add'>+_create_raw_flow_rule(struct mlx5_ib_dev *dev,</div><div class='add'>+		      struct mlx5_ib_flow_prio *ft_prio,</div><div class='add'>+		      struct mlx5_flow_destination *dst,</div><div class='add'>+		      struct mlx5_ib_flow_matcher  *fs_matcher,</div><div class='add'>+		      struct mlx5_flow_context *flow_context,</div><div class='add'>+		      struct mlx5_flow_act *flow_act,</div><div class='add'>+		      void *cmd_in, int inlen,</div><div class='add'>+		      int dst_num)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_flow_handler *handler;</div><div class='add'>+	struct mlx5_flow_spec *spec;</div><div class='add'>+	struct mlx5_flow_table *ft = ft_prio-&gt;flow_table;</div><div class='add'>+	int err = 0;</div><div class='add'>+</div><div class='add'>+	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);</div><div class='add'>+	handler = kzalloc(sizeof(*handler), GFP_KERNEL);</div><div class='add'>+	if (!handler || !spec) {</div><div class='add'>+		err = -ENOMEM;</div><div class='add'>+		goto free;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;handler-&gt;list);</div><div class='add'>+</div><div class='add'>+	memcpy(spec-&gt;match_value, cmd_in, inlen);</div><div class='add'>+	memcpy(spec-&gt;match_criteria, fs_matcher-&gt;matcher_mask.match_params,</div><div class='add'>+	       fs_matcher-&gt;mask_len);</div><div class='add'>+	spec-&gt;match_criteria_enable = fs_matcher-&gt;match_criteria_enable;</div><div class='add'>+	spec-&gt;flow_context = *flow_context;</div><div class='add'>+</div><div class='add'>+	handler-&gt;rule = mlx5_add_flow_rules(ft, spec,</div><div class='add'>+					    flow_act, dst, dst_num);</div><div class='add'>+</div><div class='add'>+	if (IS_ERR(handler-&gt;rule)) {</div><div class='add'>+		err = PTR_ERR(handler-&gt;rule);</div><div class='add'>+		goto free;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ft_prio-&gt;refcount++;</div><div class='add'>+	handler-&gt;prio = ft_prio;</div><div class='add'>+	handler-&gt;dev = dev;</div><div class='add'>+	ft_prio-&gt;flow_table = ft;</div><div class='add'>+</div><div class='add'>+free:</div><div class='add'>+	if (err)</div><div class='add'>+		kfree(handler);</div><div class='add'>+	kvfree(spec);</div><div class='add'>+	return err ? ERR_PTR(err) : handler;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher,</div><div class='add'>+				void *match_v)</div><div class='add'>+{</div><div class='add'>+	void *match_c;</div><div class='add'>+	void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4;</div><div class='add'>+	void *dmac, *dmac_mask;</div><div class='add'>+	void *ipv4, *ipv4_mask;</div><div class='add'>+</div><div class='add'>+	if (!(fs_matcher-&gt;match_criteria_enable &amp;</div><div class='add'>+	      (1 &lt;&lt; MATCH_CRITERIA_ENABLE_OUTER_BIT)))</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	match_c = fs_matcher-&gt;matcher_mask.match_params;</div><div class='add'>+	match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v,</div><div class='add'>+					   outer_headers);</div><div class='add'>+	match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c,</div><div class='add'>+					   outer_headers);</div><div class='add'>+</div><div class='add'>+	dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,</div><div class='add'>+			    dmac_47_16);</div><div class='add'>+	dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,</div><div class='add'>+				 dmac_47_16);</div><div class='add'>+</div><div class='add'>+	if (is_multicast_ether_addr(dmac) &amp;&amp;</div><div class='add'>+	    is_multicast_ether_addr(dmac_mask))</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,</div><div class='add'>+			    dst_ipv4_dst_ipv6.ipv4_layout.ipv4);</div><div class='add'>+</div><div class='add'>+	ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,</div><div class='add'>+				 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);</div><div class='add'>+</div><div class='add'>+	if (ipv4_is_multicast(*(__be32 *)(ipv4)) &amp;&amp;</div><div class='add'>+	    ipv4_is_multicast(*(__be32 *)(ipv4_mask)))</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct mlx5_ib_flow_handler *raw_fs_rule_add(</div><div class='add'>+	struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher,</div><div class='add'>+	struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act,</div><div class='add'>+	struct mlx5_fc *counter, void *cmd_in, int inlen, int dest_id, int dest_type)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_flow_destination *dst;</div><div class='add'>+	struct mlx5_ib_flow_prio *ft_prio;</div><div class='add'>+	struct mlx5_ib_flow_handler *handler;</div><div class='add'>+	int dst_num = 0;</div><div class='add'>+	bool mcast;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (fs_matcher-&gt;flow_type != MLX5_IB_FLOW_TYPE_NORMAL)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	if (fs_matcher-&gt;priority &gt; MLX5_IB_FLOW_LAST_PRIO)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	dst = kcalloc(2, sizeof(*dst), GFP_KERNEL);</div><div class='add'>+	if (!dst)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	mcast = raw_fs_is_multicast(fs_matcher, cmd_in);</div><div class='add'>+	mutex_lock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='add'>+</div><div class='add'>+	ft_prio = _get_flow_table(dev, fs_matcher-&gt;priority,</div><div class='add'>+				  fs_matcher-&gt;ns_type, mcast,</div><div class='add'>+				  fs_matcher-&gt;ib_port);</div><div class='add'>+	if (IS_ERR(ft_prio)) {</div><div class='add'>+		err = PTR_ERR(ft_prio);</div><div class='add'>+		goto unlock;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (dest_type) {</div><div class='add'>+	case MLX5_FLOW_DESTINATION_TYPE_TIR:</div><div class='add'>+		dst[dst_num].type = dest_type;</div><div class='add'>+		dst[dst_num++].tir_num = dest_id;</div><div class='add'>+		flow_act-&gt;action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE:</div><div class='add'>+		dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM;</div><div class='add'>+		dst[dst_num++].ft_num = dest_id;</div><div class='add'>+		flow_act-&gt;action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_FLOW_DESTINATION_TYPE_PORT:</div><div class='add'>+		dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT;</div><div class='add'>+		flow_act-&gt;action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (flow_act-&gt;action &amp; MLX5_FLOW_CONTEXT_ACTION_COUNT) {</div><div class='add'>+		if (WARN_ON(!counter)) {</div><div class='add'>+			err = -EINVAL;</div><div class='add'>+			goto unlock;</div><div class='add'>+		}</div><div class='add'>+		dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;</div><div class='add'>+		dst[dst_num].counter = counter;</div><div class='add'>+		dst_num++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	handler = _create_raw_flow_rule(dev, ft_prio, dst_num ? dst : NULL,</div><div class='add'>+					fs_matcher, flow_context, flow_act,</div><div class='add'>+					cmd_in, inlen, dst_num);</div><div class='add'>+</div><div class='add'>+	if (IS_ERR(handler)) {</div><div class='add'>+		err = PTR_ERR(handler);</div><div class='add'>+		goto destroy_ft;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='add'>+	atomic_inc(&amp;fs_matcher-&gt;usecnt);</div><div class='add'>+	handler-&gt;flow_matcher = fs_matcher;</div><div class='add'>+</div><div class='add'>+	kfree(dst);</div><div class='add'>+</div><div class='add'>+	return handler;</div><div class='add'>+</div><div class='add'>+destroy_ft:</div><div class='add'>+	put_flow_table(dev, ft_prio, false);</div><div class='add'>+unlock:</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='add'>+	kfree(dst);</div><div class='add'>+</div><div class='add'>+	return ERR_PTR(err);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)</div><div class='add'>+{</div><div class='add'>+	switch (maction-&gt;flow_action_raw.sub_type) {</div><div class='add'>+	case MLX5_IB_FLOW_ACTION_MODIFY_HEADER:</div><div class='add'>+		mlx5_modify_header_dealloc(maction-&gt;flow_action_raw.dev-&gt;mdev,</div><div class='add'>+					   maction-&gt;flow_action_raw.modify_hdr);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT:</div><div class='add'>+		mlx5_packet_reformat_dealloc(maction-&gt;flow_action_raw.dev-&gt;mdev,</div><div class='add'>+					     maction-&gt;flow_action_raw.pkt_reformat);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_FLOW_ACTION_DECAP:</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_flow_action *maction = to_mflow_act(action);</div><div class='add'>+</div><div class='add'>+	switch (action-&gt;type) {</div><div class='add'>+	case IB_FLOW_ACTION_UNSPECIFIED:</div><div class='add'>+		destroy_flow_action_raw(maction);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		WARN_ON(true);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	kfree(maction);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int</div><div class='add'>+mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,</div><div class='add'>+			     enum mlx5_flow_namespace_type *namespace)</div><div class='add'>+{</div><div class='add'>+	switch (table_type) {</div><div class='add'>+	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX:</div><div class='add'>+		*namespace = MLX5_FLOW_NAMESPACE_BYPASS;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX:</div><div class='add'>+		*namespace = MLX5_FLOW_NAMESPACE_EGRESS;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB:</div><div class='add'>+		*namespace = MLX5_FLOW_NAMESPACE_FDB_BYPASS;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX:</div><div class='add'>+		*namespace = MLX5_FLOW_NAMESPACE_RDMA_RX;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX:</div><div class='add'>+		*namespace = MLX5_FLOW_NAMESPACE_RDMA_TX;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_RX:</div><div class='add'>+		*namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_TX:</div><div class='add'>+		*namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct uverbs_attr_spec mlx5_ib_flow_type[] = {</div><div class='add'>+	[MLX5_IB_FLOW_TYPE_NORMAL] = {</div><div class='add'>+		.type = UVERBS_ATTR_TYPE_PTR_IN,</div><div class='add'>+		.u.ptr = {</div><div class='add'>+			.len = sizeof(u16), /* data is priority */</div><div class='add'>+			.min_len = sizeof(u16),</div><div class='add'>+		}</div><div class='add'>+	},</div><div class='add'>+	[MLX5_IB_FLOW_TYPE_SNIFFER] = {</div><div class='add'>+		.type = UVERBS_ATTR_TYPE_PTR_IN,</div><div class='add'>+		UVERBS_ATTR_NO_DATA(),</div><div class='add'>+	},</div><div class='add'>+	[MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = {</div><div class='add'>+		.type = UVERBS_ATTR_TYPE_PTR_IN,</div><div class='add'>+		UVERBS_ATTR_NO_DATA(),</div><div class='add'>+	},</div><div class='add'>+	[MLX5_IB_FLOW_TYPE_MC_DEFAULT] = {</div><div class='add'>+		.type = UVERBS_ATTR_TYPE_PTR_IN,</div><div class='add'>+		UVERBS_ATTR_NO_DATA(),</div><div class='add'>+	},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static bool is_flow_dest(void *obj, int *dest_id, int *dest_type)</div><div class='add'>+{</div><div class='add'>+	struct devx_obj *devx_obj = obj;</div><div class='add'>+	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj-&gt;dinbox, opcode);</div><div class='add'>+</div><div class='add'>+	switch (opcode) {</div><div class='add'>+	case MLX5_CMD_OP_DESTROY_TIR:</div><div class='add'>+		*dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;</div><div class='add'>+		*dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj-&gt;dinbox,</div><div class='add'>+				    obj_id);</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	case MLX5_CMD_OP_DESTROY_FLOW_TABLE:</div><div class='add'>+		*dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;</div><div class='add'>+		*dest_id = MLX5_GET(destroy_flow_table_in, devx_obj-&gt;dinbox,</div><div class='add'>+				    table_id);</div><div class='add'>+		return true;</div><div class='add'>+	default:</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_dests(struct uverbs_attr_bundle *attrs,</div><div class='add'>+		     struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id,</div><div class='add'>+		     int *dest_type, struct ib_qp **qp, u32 *flags)</div><div class='add'>+{</div><div class='add'>+	bool dest_devx, dest_qp;</div><div class='add'>+	void *devx_obj;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	dest_devx = uverbs_attr_is_valid(attrs,</div><div class='add'>+					 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);</div><div class='add'>+	dest_qp = uverbs_attr_is_valid(attrs,</div><div class='add'>+				       MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);</div><div class='add'>+</div><div class='add'>+	*flags = 0;</div><div class='add'>+	err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS,</div><div class='add'>+				 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS |</div><div class='add'>+					 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	/* Both flags are not allowed */</div><div class='add'>+	if (*flags &amp; MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS &amp;&amp;</div><div class='add'>+	    *flags &amp; MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {</div><div class='add'>+		if (dest_devx &amp;&amp; (dest_qp || *flags))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		else if (dest_qp &amp;&amp; *flags)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Allow only DEVX object, drop as dest for FDB */</div><div class='add'>+	if (fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS &amp;&amp;</div><div class='add'>+	    !(dest_devx || (*flags &amp; MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	/* Allow only DEVX object or QP as dest when inserting to RDMA_RX */</div><div class='add'>+	if ((fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX ||</div><div class='add'>+	     fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) &amp;&amp;</div><div class='add'>+	    ((!dest_devx &amp;&amp; !dest_qp) || (dest_devx &amp;&amp; dest_qp)))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	*qp = NULL;</div><div class='add'>+	if (dest_devx) {</div><div class='add'>+		devx_obj =</div><div class='add'>+			uverbs_attr_get_obj(attrs,</div><div class='add'>+					    MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);</div><div class='add'>+</div><div class='add'>+		/* Verify that the given DEVX object is a flow</div><div class='add'>+		 * steering destination.</div><div class='add'>+		 */</div><div class='add'>+		if (!is_flow_dest(devx_obj, dest_id, dest_type))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		/* Allow only flow table as dest when inserting to FDB or RDMA_RX */</div><div class='add'>+		if ((fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS ||</div><div class='add'>+		     fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX ||</div><div class='add'>+		     fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) &amp;&amp;</div><div class='add'>+		    *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+	} else if (dest_qp) {</div><div class='add'>+		struct mlx5_ib_qp *mqp;</div><div class='add'>+</div><div class='add'>+		*qp = uverbs_attr_get_obj(attrs,</div><div class='add'>+					  MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);</div><div class='add'>+		if (IS_ERR(*qp))</div><div class='add'>+			return PTR_ERR(*qp);</div><div class='add'>+</div><div class='add'>+		if ((*qp)-&gt;qp_type != IB_QPT_RAW_PACKET)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		mqp = to_mqp(*qp);</div><div class='add'>+		if (mqp-&gt;is_rss)</div><div class='add'>+			*dest_id = mqp-&gt;rss_qp.tirn;</div><div class='add'>+		else</div><div class='add'>+			*dest_id = mqp-&gt;raw_packet_qp.rq.tirn;</div><div class='add'>+		*dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;</div><div class='add'>+	} else if ((fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_EGRESS ||</div><div class='add'>+		    fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX ||</div><div class='add'>+		    fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) &amp;&amp;</div><div class='add'>+		   !(*flags &amp; MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)) {</div><div class='add'>+		*dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR &amp;&amp;</div><div class='add'>+	    (fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_EGRESS ||</div><div class='add'>+	     fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX ||</div><div class='add'>+	     fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool</div><div class='add'>+is_flow_counter(void *obj, u32 offset, u32 *counter_id, u32 *fc_bulk_size)</div><div class='add'>+{</div><div class='add'>+	struct devx_obj *devx_obj = obj;</div><div class='add'>+	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj-&gt;dinbox, opcode);</div><div class='add'>+</div><div class='add'>+	if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {</div><div class='add'>+</div><div class='add'>+		if (offset &amp;&amp; offset &gt;= devx_obj-&gt;flow_counter_bulk_size)</div><div class='add'>+			return false;</div><div class='add'>+</div><div class='add'>+		*fc_bulk_size = devx_obj-&gt;flow_counter_bulk_size;</div><div class='add'>+		*counter_id = MLX5_GET(dealloc_flow_counter_in,</div><div class='add'>+				       devx_obj-&gt;dinbox,</div><div class='add'>+				       flow_counter_id);</div><div class='add'>+		*counter_id += offset;</div><div class='add'>+		return true;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+#define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2</div><div class='add'>+static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_flow_context flow_context = {.flow_tag =</div><div class='add'>+		MLX5_FS_DEFAULT_FLOW_TAG};</div><div class='add'>+	int dest_id, dest_type = -1, inlen, len, ret, i;</div><div class='add'>+	struct mlx5_ib_flow_handler *flow_handler;</div><div class='add'>+	struct mlx5_ib_flow_matcher *fs_matcher;</div><div class='add'>+	struct ib_uobject **arr_flow_actions;</div><div class='add'>+	struct ib_uflow_resources *uflow_res;</div><div class='add'>+	struct mlx5_flow_act flow_act = {};</div><div class='add'>+	struct mlx5_fc *counter = NULL;</div><div class='add'>+	struct ib_qp *qp = NULL;</div><div class='add'>+	void *devx_obj, *cmd_in;</div><div class='add'>+	struct ib_uobject *uobj;</div><div class='add'>+	struct mlx5_ib_dev *dev;</div><div class='add'>+	u32 flags;</div><div class='add'>+</div><div class='add'>+	if (!rdma_uattrs_has_raw_cap(attrs))</div><div class='add'>+		return -EPERM;</div><div class='add'>+</div><div class='add'>+	fs_matcher = uverbs_attr_get_obj(attrs,</div><div class='add'>+					 MLX5_IB_ATTR_CREATE_FLOW_MATCHER);</div><div class='add'>+	uobj =  uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE);</div><div class='add'>+	dev = mlx5_udata_to_mdev(&amp;attrs-&gt;driver_udata);</div><div class='add'>+</div><div class='add'>+	if (get_dests(attrs, fs_matcher, &amp;dest_id, &amp;dest_type, &amp;qp, &amp;flags))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (flags &amp; MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS)</div><div class='add'>+		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS;</div><div class='add'>+</div><div class='add'>+	if (flags &amp; MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)</div><div class='add'>+		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP;</div><div class='add'>+</div><div class='add'>+	len = uverbs_attr_get_uobjs_arr(attrs,</div><div class='add'>+		MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &amp;arr_flow_actions);</div><div class='add'>+	if (len) {</div><div class='add'>+		u32 *offset_attr, fc_bulk_size, offset = 0, counter_id = 0;</div><div class='add'>+		devx_obj = arr_flow_actions[0]-&gt;object;</div><div class='add'>+</div><div class='add'>+		if (uverbs_attr_is_valid(attrs,</div><div class='add'>+					 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) {</div><div class='add'>+</div><div class='add'>+			int num_offsets = uverbs_attr_ptr_get_array_size(</div><div class='add'>+				attrs,</div><div class='add'>+				MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,</div><div class='add'>+				sizeof(u32));</div><div class='add'>+</div><div class='add'>+			if (num_offsets != 1)</div><div class='add'>+				return -EINVAL;</div><div class='add'>+</div><div class='add'>+			offset_attr = uverbs_attr_get_alloced_ptr(</div><div class='add'>+				attrs,</div><div class='add'>+				MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET);</div><div class='add'>+			offset = *offset_attr;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (!is_flow_counter(devx_obj, offset, &amp;counter_id, &amp;fc_bulk_size))</div><div class='add'>+			return -EINVAL;</div><div class='add'>+		counter = mlx5_fc_local_create(counter_id, offset, fc_bulk_size);</div><div class='add'>+		if (IS_ERR(counter))</div><div class='add'>+			return PTR_ERR(counter);</div><div class='add'>+</div><div class='add'>+		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	cmd_in = uverbs_attr_get_alloced_ptr(</div><div class='add'>+		attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);</div><div class='add'>+	inlen = uverbs_attr_get_len(attrs,</div><div class='add'>+				    MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);</div><div class='add'>+</div><div class='add'>+	uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS);</div><div class='add'>+	if (!uflow_res) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto destroy_counter;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	len = uverbs_attr_get_uobjs_arr(attrs,</div><div class='add'>+		MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &amp;arr_flow_actions);</div><div class='add'>+	for (i = 0; i &lt; len; i++) {</div><div class='add'>+		struct mlx5_ib_flow_action *maction =</div><div class='add'>+			to_mflow_act(arr_flow_actions[i]-&gt;object);</div><div class='add'>+</div><div class='add'>+		ret = parse_flow_flow_action(maction, false, &amp;flow_act);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto err_out;</div><div class='add'>+		flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE,</div><div class='add'>+				   arr_flow_actions[i]-&gt;object);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_from(&amp;flow_context.flow_tag, attrs,</div><div class='add'>+			       MLX5_IB_ATTR_CREATE_FLOW_TAG);</div><div class='add'>+	if (!ret) {</div><div class='add'>+		if (flow_context.flow_tag &gt;= BIT(24)) {</div><div class='add'>+			ret = -EINVAL;</div><div class='add'>+			goto err_out;</div><div class='add'>+		}</div><div class='add'>+		flow_context.flags |= FLOW_CONTEXT_HAS_TAG;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	flow_handler =</div><div class='add'>+		raw_fs_rule_add(dev, fs_matcher, &amp;flow_context, &amp;flow_act,</div><div class='add'>+				counter, cmd_in, inlen, dest_id, dest_type);</div><div class='add'>+	if (IS_ERR(flow_handler)) {</div><div class='add'>+		ret = PTR_ERR(flow_handler);</div><div class='add'>+		goto err_out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ib_set_flow(uobj, &amp;flow_handler-&gt;ibflow, qp, &amp;dev-&gt;ib_dev, uflow_res);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+err_out:</div><div class='add'>+	ib_uverbs_flow_resources_free(uflow_res);</div><div class='add'>+destroy_counter:</div><div class='add'>+	if (counter)</div><div class='add'>+		mlx5_fc_local_destroy(counter);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int flow_matcher_cleanup(struct ib_uobject *uobject,</div><div class='add'>+				enum rdma_remove_reason why,</div><div class='add'>+				struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_flow_matcher *obj = uobject-&gt;object;</div><div class='add'>+</div><div class='add'>+	if (atomic_read(&amp;obj-&gt;usecnt))</div><div class='add'>+		return -EBUSY;</div><div class='add'>+</div><div class='add'>+	kfree(obj);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int steering_anchor_create_ft(struct mlx5_ib_dev *dev,</div><div class='add'>+				     struct mlx5_ib_flow_prio *ft_prio,</div><div class='add'>+				     enum mlx5_flow_namespace_type ns_type)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_flow_table_attr ft_attr = {};</div><div class='add'>+	struct mlx5_flow_namespace *ns;</div><div class='add'>+	struct mlx5_flow_table *ft;</div><div class='add'>+</div><div class='add'>+	if (ft_prio-&gt;anchor.ft)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	ns = mlx5_get_flow_namespace(dev-&gt;mdev, ns_type);</div><div class='add'>+	if (!ns)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;</div><div class='add'>+	ft_attr.uid = MLX5_SHARED_RESOURCE_UID;</div><div class='add'>+	ft_attr.prio = 0;</div><div class='add'>+	ft_attr.max_fte = 2;</div><div class='add'>+	ft_attr.level = 1;</div><div class='add'>+</div><div class='add'>+	ft = mlx5_create_flow_table(ns, &amp;ft_attr);</div><div class='add'>+	if (IS_ERR(ft))</div><div class='add'>+		return PTR_ERR(ft);</div><div class='add'>+</div><div class='add'>+	ft_prio-&gt;anchor.ft = ft;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void steering_anchor_destroy_ft(struct mlx5_ib_flow_prio *ft_prio)</div><div class='add'>+{</div><div class='add'>+	if (ft_prio-&gt;anchor.ft) {</div><div class='add'>+		mlx5_destroy_flow_table(ft_prio-&gt;anchor.ft);</div><div class='add'>+		ft_prio-&gt;anchor.ft = NULL;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int</div><div class='add'>+steering_anchor_create_fg_drop(struct mlx5_ib_flow_prio *ft_prio)</div><div class='add'>+{</div><div class='add'>+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);</div><div class='add'>+	struct mlx5_flow_group *fg;</div><div class='add'>+	void *flow_group_in;</div><div class='add'>+	int err = 0;</div><div class='add'>+</div><div class='add'>+	if (ft_prio-&gt;anchor.fg_drop)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	flow_group_in = kvzalloc(inlen, GFP_KERNEL);</div><div class='add'>+	if (!flow_group_in)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 1);</div><div class='add'>+	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1);</div><div class='add'>+</div><div class='add'>+	fg = mlx5_create_flow_group(ft_prio-&gt;anchor.ft, flow_group_in);</div><div class='add'>+	if (IS_ERR(fg)) {</div><div class='add'>+		err = PTR_ERR(fg);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ft_prio-&gt;anchor.fg_drop = fg;</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	kvfree(flow_group_in);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void</div><div class='add'>+steering_anchor_destroy_fg_drop(struct mlx5_ib_flow_prio *ft_prio)</div><div class='add'>+{</div><div class='add'>+	if (ft_prio-&gt;anchor.fg_drop) {</div><div class='add'>+		mlx5_destroy_flow_group(ft_prio-&gt;anchor.fg_drop);</div><div class='add'>+		ft_prio-&gt;anchor.fg_drop = NULL;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int</div><div class='add'>+steering_anchor_create_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio)</div><div class='add'>+{</div><div class='add'>+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);</div><div class='add'>+	struct mlx5_flow_group *fg;</div><div class='add'>+	void *flow_group_in;</div><div class='add'>+	int err = 0;</div><div class='add'>+</div><div class='add'>+	if (ft_prio-&gt;anchor.fg_goto_table)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	flow_group_in = kvzalloc(inlen, GFP_KERNEL);</div><div class='add'>+	if (!flow_group_in)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	fg = mlx5_create_flow_group(ft_prio-&gt;anchor.ft, flow_group_in);</div><div class='add'>+	if (IS_ERR(fg)) {</div><div class='add'>+		err = PTR_ERR(fg);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+	ft_prio-&gt;anchor.fg_goto_table = fg;</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	kvfree(flow_group_in);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void</div><div class='add'>+steering_anchor_destroy_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio)</div><div class='add'>+{</div><div class='add'>+	if (ft_prio-&gt;anchor.fg_goto_table) {</div><div class='add'>+		mlx5_destroy_flow_group(ft_prio-&gt;anchor.fg_goto_table);</div><div class='add'>+		ft_prio-&gt;anchor.fg_goto_table = NULL;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int</div><div class='add'>+steering_anchor_create_rule_drop(struct mlx5_ib_flow_prio *ft_prio)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_flow_act flow_act = {};</div><div class='add'>+	struct mlx5_flow_handle *handle;</div><div class='add'>+</div><div class='add'>+	if (ft_prio-&gt;anchor.rule_drop)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	flow_act.fg = ft_prio-&gt;anchor.fg_drop;</div><div class='add'>+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;</div><div class='add'>+</div><div class='add'>+	handle = mlx5_add_flow_rules(ft_prio-&gt;anchor.ft, NULL, &amp;flow_act,</div><div class='add'>+				     NULL, 0);</div><div class='add'>+	if (IS_ERR(handle))</div><div class='add'>+		return PTR_ERR(handle);</div><div class='add'>+</div><div class='add'>+	ft_prio-&gt;anchor.rule_drop = handle;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void steering_anchor_destroy_rule_drop(struct mlx5_ib_flow_prio *ft_prio)</div><div class='add'>+{</div><div class='add'>+	if (ft_prio-&gt;anchor.rule_drop) {</div><div class='add'>+		mlx5_del_flow_rules(ft_prio-&gt;anchor.rule_drop);</div><div class='add'>+		ft_prio-&gt;anchor.rule_drop = NULL;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int</div><div class='add'>+steering_anchor_create_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_flow_destination dest = {};</div><div class='add'>+	struct mlx5_flow_act flow_act = {};</div><div class='add'>+	struct mlx5_flow_handle *handle;</div><div class='add'>+</div><div class='add'>+	if (ft_prio-&gt;anchor.rule_goto_table)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;</div><div class='add'>+	flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;</div><div class='add'>+	flow_act.fg = ft_prio-&gt;anchor.fg_goto_table;</div><div class='add'>+</div><div class='add'>+	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;</div><div class='add'>+	dest.ft = ft_prio-&gt;flow_table;</div><div class='add'>+</div><div class='add'>+	handle = mlx5_add_flow_rules(ft_prio-&gt;anchor.ft, NULL, &amp;flow_act,</div><div class='add'>+				     &amp;dest, 1);</div><div class='add'>+	if (IS_ERR(handle))</div><div class='add'>+		return PTR_ERR(handle);</div><div class='add'>+</div><div class='add'>+	ft_prio-&gt;anchor.rule_goto_table = handle;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void</div><div class='add'>+steering_anchor_destroy_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio)</div><div class='add'>+{</div><div class='add'>+	if (ft_prio-&gt;anchor.rule_goto_table) {</div><div class='add'>+		mlx5_del_flow_rules(ft_prio-&gt;anchor.rule_goto_table);</div><div class='add'>+		ft_prio-&gt;anchor.rule_goto_table = NULL;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int steering_anchor_create_res(struct mlx5_ib_dev *dev,</div><div class='add'>+				      struct mlx5_ib_flow_prio *ft_prio,</div><div class='add'>+				      enum mlx5_flow_namespace_type ns_type)</div><div class='add'>+{</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	err = steering_anchor_create_ft(dev, ft_prio, ns_type);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = steering_anchor_create_fg_drop(ft_prio);</div><div class='add'>+	if (err)</div><div class='add'>+		goto destroy_ft;</div><div class='add'>+</div><div class='add'>+	err = steering_anchor_create_fg_goto_table(ft_prio);</div><div class='add'>+	if (err)</div><div class='add'>+		goto destroy_fg_drop;</div><div class='add'>+</div><div class='add'>+	err = steering_anchor_create_rule_drop(ft_prio);</div><div class='add'>+	if (err)</div><div class='add'>+		goto destroy_fg_goto_table;</div><div class='add'>+</div><div class='add'>+	err = steering_anchor_create_rule_goto_table(ft_prio);</div><div class='add'>+	if (err)</div><div class='add'>+		goto destroy_rule_drop;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+destroy_rule_drop:</div><div class='add'>+	steering_anchor_destroy_rule_drop(ft_prio);</div><div class='add'>+destroy_fg_goto_table:</div><div class='add'>+	steering_anchor_destroy_fg_goto_table(ft_prio);</div><div class='add'>+destroy_fg_drop:</div><div class='add'>+	steering_anchor_destroy_fg_drop(ft_prio);</div><div class='add'>+destroy_ft:</div><div class='add'>+	steering_anchor_destroy_ft(ft_prio);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_steering_anchor_destroy_res(struct mlx5_ib_flow_prio *ft_prio)</div><div class='add'>+{</div><div class='add'>+	steering_anchor_destroy_rule_goto_table(ft_prio);</div><div class='add'>+	steering_anchor_destroy_rule_drop(ft_prio);</div><div class='add'>+	steering_anchor_destroy_fg_goto_table(ft_prio);</div><div class='add'>+	steering_anchor_destroy_fg_drop(ft_prio);</div><div class='add'>+	steering_anchor_destroy_ft(ft_prio);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int steering_anchor_cleanup(struct ib_uobject *uobject,</div><div class='add'>+				   enum rdma_remove_reason why,</div><div class='add'>+				   struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_steering_anchor *obj = uobject-&gt;object;</div><div class='add'>+</div><div class='add'>+	if (atomic_read(&amp;obj-&gt;usecnt))</div><div class='add'>+		return -EBUSY;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;obj-&gt;dev-&gt;flow_db-&gt;lock);</div><div class='add'>+	if (!--obj-&gt;ft_prio-&gt;anchor.rule_goto_table_ref)</div><div class='add'>+		steering_anchor_destroy_rule_goto_table(obj-&gt;ft_prio);</div><div class='add'>+</div><div class='add'>+	put_flow_table(obj-&gt;dev, obj-&gt;ft_prio, true);</div><div class='add'>+	mutex_unlock(&amp;obj-&gt;dev-&gt;flow_db-&gt;lock);</div><div class='add'>+</div><div class='add'>+	kfree(obj);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void fs_cleanup_anchor(struct mlx5_ib_flow_prio *prio,</div><div class='add'>+			      int count)</div><div class='add'>+{</div><div class='add'>+	while (count--)</div><div class='add'>+		mlx5_steering_anchor_destroy_res(&amp;prio[count]);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	fs_cleanup_anchor(dev-&gt;flow_db-&gt;prios, MLX5_IB_NUM_FLOW_FT);</div><div class='add'>+	fs_cleanup_anchor(dev-&gt;flow_db-&gt;egress_prios, MLX5_IB_NUM_FLOW_FT);</div><div class='add'>+	fs_cleanup_anchor(dev-&gt;flow_db-&gt;sniffer, MLX5_IB_NUM_SNIFFER_FTS);</div><div class='add'>+	fs_cleanup_anchor(dev-&gt;flow_db-&gt;egress, MLX5_IB_NUM_EGRESS_FTS);</div><div class='add'>+	fs_cleanup_anchor(dev-&gt;flow_db-&gt;fdb, MLX5_IB_NUM_FDB_FTS);</div><div class='add'>+	fs_cleanup_anchor(dev-&gt;flow_db-&gt;rdma_rx, MLX5_IB_NUM_FLOW_FT);</div><div class='add'>+	fs_cleanup_anchor(dev-&gt;flow_db-&gt;rdma_tx, MLX5_IB_NUM_FLOW_FT);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs,</div><div class='add'>+			      struct mlx5_ib_flow_matcher *obj)</div><div class='add'>+{</div><div class='add'>+	enum mlx5_ib_uapi_flow_table_type ft_type =</div><div class='add'>+		MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX;</div><div class='add'>+	u32 flags;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	/* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older</div><div class='add'>+	 * users should switch to it. We leave this to not break userspace</div><div class='add'>+	 */</div><div class='add'>+	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) &amp;&amp;</div><div class='add'>+	    uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) {</div><div class='add'>+		err = uverbs_get_const(&amp;ft_type, attrs,</div><div class='add'>+				       MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE);</div><div class='add'>+		if (err)</div><div class='add'>+			return err;</div><div class='add'>+</div><div class='add'>+		err = mlx5_ib_ft_type_to_namespace(ft_type, &amp;obj-&gt;ns_type);</div><div class='add'>+		if (err)</div><div class='add'>+			return err;</div><div class='add'>+</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) {</div><div class='add'>+		err = uverbs_get_flags32(&amp;flags, attrs,</div><div class='add'>+					 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,</div><div class='add'>+					 IB_FLOW_ATTR_FLAGS_EGRESS);</div><div class='add'>+		if (err)</div><div class='add'>+			return err;</div><div class='add'>+</div><div class='add'>+		if (flags)</div><div class='add'>+			return mlx5_ib_ft_type_to_namespace(</div><div class='add'>+				MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX,</div><div class='add'>+				&amp;obj-&gt;ns_type);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	obj-&gt;ns_type = MLX5_FLOW_NAMESPACE_BYPASS;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool verify_context_caps(struct mlx5_ib_dev *dev, u64 enabled_caps)</div><div class='add'>+{</div><div class='add'>+	if (is_mdev_switchdev_mode(dev-&gt;mdev))</div><div class='add'>+		return UCAP_ENABLED(enabled_caps,</div><div class='add'>+				    RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);</div><div class='add'>+</div><div class='add'>+	return UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_LOCAL);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *uobj = uverbs_attr_get_uobject(</div><div class='add'>+		attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE);</div><div class='add'>+	struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&amp;attrs-&gt;driver_udata);</div><div class='add'>+	struct mlx5_ib_flow_matcher *obj;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL);</div><div class='add'>+	if (!obj)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	obj-&gt;mask_len = uverbs_attr_get_len(</div><div class='add'>+		attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);</div><div class='add'>+	err = uverbs_copy_from(&amp;obj-&gt;matcher_mask,</div><div class='add'>+			       attrs,</div><div class='add'>+			       MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);</div><div class='add'>+	if (err)</div><div class='add'>+		goto end;</div><div class='add'>+</div><div class='add'>+	obj-&gt;flow_type = uverbs_attr_get_enum_id(</div><div class='add'>+		attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);</div><div class='add'>+</div><div class='add'>+	if (obj-&gt;flow_type == MLX5_IB_FLOW_TYPE_NORMAL) {</div><div class='add'>+		err = uverbs_copy_from(&amp;obj-&gt;priority,</div><div class='add'>+				       attrs,</div><div class='add'>+				       MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);</div><div class='add'>+		if (err)</div><div class='add'>+			goto end;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = uverbs_copy_from(&amp;obj-&gt;match_criteria_enable,</div><div class='add'>+			       attrs,</div><div class='add'>+			       MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA);</div><div class='add'>+	if (err)</div><div class='add'>+		goto end;</div><div class='add'>+</div><div class='add'>+	err = mlx5_ib_matcher_ns(attrs, obj);</div><div class='add'>+	if (err)</div><div class='add'>+		goto end;</div><div class='add'>+</div><div class='add'>+	if (obj-&gt;ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS &amp;&amp;</div><div class='add'>+	    mlx5_eswitch_mode(dev-&gt;mdev) != MLX5_ESWITCH_OFFLOADS) {</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+		goto end;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT)) {</div><div class='add'>+		err = uverbs_copy_from(&amp;obj-&gt;ib_port, attrs,</div><div class='add'>+				       MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT);</div><div class='add'>+		if (err)</div><div class='add'>+			goto end;</div><div class='add'>+		if (!rdma_is_port_valid(&amp;dev-&gt;ib_dev, obj-&gt;ib_port)) {</div><div class='add'>+			err = -EINVAL;</div><div class='add'>+			goto end;</div><div class='add'>+		}</div><div class='add'>+		if (obj-&gt;ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX &amp;&amp;</div><div class='add'>+		    obj-&gt;ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) {</div><div class='add'>+			err = -EINVAL;</div><div class='add'>+			goto end;</div><div class='add'>+		}</div><div class='add'>+		if (!verify_context_caps(dev, uobj-&gt;context-&gt;enabled_caps)) {</div><div class='add'>+			err = -EOPNOTSUPP;</div><div class='add'>+			goto end;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	uobj-&gt;object = obj;</div><div class='add'>+	obj-&gt;mdev = dev-&gt;mdev;</div><div class='add'>+	atomic_set(&amp;obj-&gt;usecnt, 0);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+end:</div><div class='add'>+	kfree(obj);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *uobj = uverbs_attr_get_uobject(</div><div class='add'>+		attrs, MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE);</div><div class='add'>+	struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&amp;attrs-&gt;driver_udata);</div><div class='add'>+	enum mlx5_ib_uapi_flow_table_type ib_uapi_ft_type;</div><div class='add'>+	enum mlx5_flow_namespace_type ns_type;</div><div class='add'>+	struct mlx5_ib_steering_anchor *obj;</div><div class='add'>+	struct mlx5_ib_flow_prio *ft_prio;</div><div class='add'>+	u16 priority;</div><div class='add'>+	u32 ft_id;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (!rdma_dev_has_raw_cap(&amp;dev-&gt;ib_dev))</div><div class='add'>+		return -EPERM;</div><div class='add'>+</div><div class='add'>+	err = uverbs_get_const(&amp;ib_uapi_ft_type, attrs,</div><div class='add'>+			       MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = mlx5_ib_ft_type_to_namespace(ib_uapi_ft_type, &amp;ns_type);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = uverbs_copy_from(&amp;priority, attrs,</div><div class='add'>+			       MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	obj = kzalloc(sizeof(*obj), GFP_KERNEL);</div><div class='add'>+	if (!obj)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='add'>+</div><div class='add'>+	ft_prio = _get_flow_table(dev, priority, ns_type, 0, 0);</div><div class='add'>+	if (IS_ERR(ft_prio)) {</div><div class='add'>+		err = PTR_ERR(ft_prio);</div><div class='add'>+		goto free_obj;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ft_prio-&gt;refcount++;</div><div class='add'>+</div><div class='add'>+	if (!ft_prio-&gt;anchor.rule_goto_table_ref) {</div><div class='add'>+		err = steering_anchor_create_res(dev, ft_prio, ns_type);</div><div class='add'>+		if (err)</div><div class='add'>+			goto put_flow_table;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ft_prio-&gt;anchor.rule_goto_table_ref++;</div><div class='add'>+</div><div class='add'>+	ft_id = mlx5_flow_table_id(ft_prio-&gt;anchor.ft);</div><div class='add'>+</div><div class='add'>+	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID,</div><div class='add'>+			     &amp;ft_id, sizeof(ft_id));</div><div class='add'>+	if (err)</div><div class='add'>+		goto destroy_res;</div><div class='add'>+</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='add'>+</div><div class='add'>+	uobj-&gt;object = obj;</div><div class='add'>+	obj-&gt;dev = dev;</div><div class='add'>+	obj-&gt;ft_prio = ft_prio;</div><div class='add'>+	atomic_set(&amp;obj-&gt;usecnt, 0);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+destroy_res:</div><div class='add'>+	--ft_prio-&gt;anchor.rule_goto_table_ref;</div><div class='add'>+	mlx5_steering_anchor_destroy_res(ft_prio);</div><div class='add'>+put_flow_table:</div><div class='add'>+	put_flow_table(dev, ft_prio, true);</div><div class='add'>+free_obj:</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='add'>+	kfree(obj);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct ib_flow_action *</div><div class='add'>+mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev,</div><div class='add'>+			     enum mlx5_ib_uapi_flow_table_type ft_type,</div><div class='add'>+			     u8 num_actions, void *in)</div><div class='add'>+{</div><div class='add'>+	enum mlx5_flow_namespace_type namespace;</div><div class='add'>+	struct mlx5_ib_flow_action *maction;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = mlx5_ib_ft_type_to_namespace(ft_type, &amp;namespace);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	maction = kzalloc(sizeof(*maction), GFP_KERNEL);</div><div class='add'>+	if (!maction)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	maction-&gt;flow_action_raw.modify_hdr =</div><div class='add'>+		mlx5_modify_header_alloc(dev-&gt;mdev, namespace, num_actions, in);</div><div class='add'>+</div><div class='add'>+	if (IS_ERR(maction-&gt;flow_action_raw.modify_hdr)) {</div><div class='add'>+		ret = PTR_ERR(maction-&gt;flow_action_raw.modify_hdr);</div><div class='add'>+		kfree(maction);</div><div class='add'>+		return ERR_PTR(ret);</div><div class='add'>+	}</div><div class='add'>+	maction-&gt;flow_action_raw.sub_type =</div><div class='add'>+		MLX5_IB_FLOW_ACTION_MODIFY_HEADER;</div><div class='add'>+	maction-&gt;flow_action_raw.dev = dev;</div><div class='add'>+</div><div class='add'>+	return &amp;maction-&gt;ib_action;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	return MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev,</div><div class='add'>+					 max_modify_header_actions) ||</div><div class='add'>+	       MLX5_CAP_FLOWTABLE_NIC_TX(dev-&gt;mdev,</div><div class='add'>+					 max_modify_header_actions) ||</div><div class='add'>+	       MLX5_CAP_FLOWTABLE_RDMA_TX(dev-&gt;mdev,</div><div class='add'>+					 max_modify_header_actions);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *uobj = uverbs_attr_get_uobject(</div><div class='add'>+		attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE);</div><div class='add'>+	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&amp;attrs-&gt;driver_udata);</div><div class='add'>+	enum mlx5_ib_uapi_flow_table_type ft_type;</div><div class='add'>+	struct ib_flow_action *action;</div><div class='add'>+	int num_actions;</div><div class='add'>+	void *in;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!mlx5_ib_modify_header_supported(mdev))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	in = uverbs_attr_get_alloced_ptr(attrs,</div><div class='add'>+		MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM);</div><div class='add'>+</div><div class='add'>+	num_actions = uverbs_attr_ptr_get_array_size(</div><div class='add'>+		attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,</div><div class='add'>+		MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto));</div><div class='add'>+	if (num_actions &lt; 0)</div><div class='add'>+		return num_actions;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_get_const(&amp;ft_type, attrs,</div><div class='add'>+			       MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+	action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in);</div><div class='add'>+	if (IS_ERR(action))</div><div class='add'>+		return PTR_ERR(action);</div><div class='add'>+</div><div class='add'>+	uverbs_flow_action_fill_action(action, uobj, &amp;mdev-&gt;ib_dev,</div><div class='add'>+				       IB_FLOW_ACTION_UNSPECIFIED);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev,</div><div class='add'>+						      u8 packet_reformat_type,</div><div class='add'>+						      u8 ft_type)</div><div class='add'>+{</div><div class='add'>+	switch (packet_reformat_type) {</div><div class='add'>+	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:</div><div class='add'>+		if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)</div><div class='add'>+			return MLX5_CAP_FLOWTABLE(ibdev-&gt;mdev,</div><div class='add'>+						  encap_general_header);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:</div><div class='add'>+		if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)</div><div class='add'>+			return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev-&gt;mdev,</div><div class='add'>+				reformat_l2_to_l3_tunnel);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:</div><div class='add'>+		if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)</div><div class='add'>+			return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev-&gt;mdev,</div><div class='add'>+				reformat_l3_tunnel_to_l2);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2:</div><div class='add'>+		if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)</div><div class='add'>+			return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev-&gt;mdev, decap);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt)</div><div class='add'>+{</div><div class='add'>+	switch (dv_prt) {</div><div class='add'>+	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:</div><div class='add'>+		*prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:</div><div class='add'>+		*prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:</div><div class='add'>+		*prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_flow_action_create_packet_reformat_ctx(</div><div class='add'>+	struct mlx5_ib_dev *dev,</div><div class='add'>+	struct mlx5_ib_flow_action *maction,</div><div class='add'>+	u8 ft_type, u8 dv_prt,</div><div class='add'>+	void *in, size_t len)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_pkt_reformat_params reformat_params;</div><div class='add'>+	enum mlx5_flow_namespace_type namespace;</div><div class='add'>+	u8 prm_prt;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = mlx5_ib_ft_type_to_namespace(ft_type, &amp;namespace);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &amp;prm_prt);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	memset(&amp;reformat_params, 0, sizeof(reformat_params));</div><div class='add'>+	reformat_params.type = prm_prt;</div><div class='add'>+	reformat_params.size = len;</div><div class='add'>+	reformat_params.data = in;</div><div class='add'>+	maction-&gt;flow_action_raw.pkt_reformat =</div><div class='add'>+		mlx5_packet_reformat_alloc(dev-&gt;mdev, &amp;reformat_params,</div><div class='add'>+					   namespace);</div><div class='add'>+	if (IS_ERR(maction-&gt;flow_action_raw.pkt_reformat)) {</div><div class='add'>+		ret = PTR_ERR(maction-&gt;flow_action_raw.pkt_reformat);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	maction-&gt;flow_action_raw.sub_type =</div><div class='add'>+		MLX5_IB_FLOW_ACTION_PACKET_REFORMAT;</div><div class='add'>+	maction-&gt;flow_action_raw.dev = dev;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,</div><div class='add'>+		MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE);</div><div class='add'>+	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&amp;attrs-&gt;driver_udata);</div><div class='add'>+	enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt;</div><div class='add'>+	enum mlx5_ib_uapi_flow_table_type ft_type;</div><div class='add'>+	struct mlx5_ib_flow_action *maction;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_get_const(&amp;ft_type, attrs,</div><div class='add'>+			       MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = uverbs_get_const(&amp;dv_prt, attrs,</div><div class='add'>+			       MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	maction = kzalloc(sizeof(*maction), GFP_KERNEL);</div><div class='add'>+	if (!maction)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	if (dv_prt ==</div><div class='add'>+	    MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) {</div><div class='add'>+		maction-&gt;flow_action_raw.sub_type =</div><div class='add'>+			MLX5_IB_FLOW_ACTION_DECAP;</div><div class='add'>+		maction-&gt;flow_action_raw.dev = mdev;</div><div class='add'>+	} else {</div><div class='add'>+		void *in;</div><div class='add'>+		int len;</div><div class='add'>+</div><div class='add'>+		in = uverbs_attr_get_alloced_ptr(attrs,</div><div class='add'>+			MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);</div><div class='add'>+		if (IS_ERR(in)) {</div><div class='add'>+			ret = PTR_ERR(in);</div><div class='add'>+			goto free_maction;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		len = uverbs_attr_get_len(attrs,</div><div class='add'>+			MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);</div><div class='add'>+</div><div class='add'>+		ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev,</div><div class='add'>+			maction, ft_type, dv_prt, in, len);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto free_maction;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	uverbs_flow_action_fill_action(&amp;maction-&gt;ib_action, uobj, &amp;mdev-&gt;ib_dev,</div><div class='add'>+				       IB_FLOW_ACTION_UNSPECIFIED);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+free_maction:</div><div class='add'>+	kfree(maction);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	MLX5_IB_METHOD_CREATE_FLOW,</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_FLOW,</div><div class='add'>+			UVERBS_ACCESS_NEW,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(</div><div class='add'>+		MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE,</div><div class='add'>+		UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),</div><div class='add'>+		UA_MANDATORY,</div><div class='add'>+		UA_ALLOC_AND_COPY),</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER,</div><div class='add'>+			MLX5_IB_OBJECT_FLOW_MATCHER,</div><div class='add'>+			UVERBS_ACCESS_READ,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP,</div><div class='add'>+			UVERBS_OBJECT_QP,</div><div class='add'>+			UVERBS_ACCESS_READ),</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX,</div><div class='add'>+			MLX5_IB_OBJECT_DEVX_OBJ,</div><div class='add'>+			UVERBS_ACCESS_READ),</div><div class='add'>+	UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS,</div><div class='add'>+			     UVERBS_OBJECT_FLOW_ACTION,</div><div class='add'>+			     UVERBS_ACCESS_READ, 1,</div><div class='add'>+			     MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS,</div><div class='add'>+			     UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX,</div><div class='add'>+			     MLX5_IB_OBJECT_DEVX_OBJ,</div><div class='add'>+			     UVERBS_ACCESS_READ, 1, 1,</div><div class='add'>+			     UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,</div><div class='add'>+			   UVERBS_ATTR_MIN_SIZE(sizeof(u32)),</div><div class='add'>+			   UA_OPTIONAL,</div><div class='add'>+			   UA_ALLOC_AND_COPY),</div><div class='add'>+	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS,</div><div class='add'>+			     enum mlx5_ib_create_flow_flags,</div><div class='add'>+			     UA_OPTIONAL));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD_DESTROY(</div><div class='add'>+	MLX5_IB_METHOD_DESTROY_FLOW,</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_FLOW,</div><div class='add'>+			UVERBS_ACCESS_DESTROY,</div><div class='add'>+			UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+ADD_UVERBS_METHODS(mlx5_ib_fs,</div><div class='add'>+		   UVERBS_OBJECT_FLOW,</div><div class='add'>+		   &amp;UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW),</div><div class='add'>+		   &amp;UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER,</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_FLOW_ACTION,</div><div class='add'>+			UVERBS_ACCESS_NEW,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,</div><div class='add'>+			   UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES(</div><div class='add'>+				   set_add_copy_action_in_auto)),</div><div class='add'>+			   UA_MANDATORY,</div><div class='add'>+			   UA_ALLOC_AND_COPY),</div><div class='add'>+	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE,</div><div class='add'>+			     enum mlx5_ib_uapi_flow_table_type,</div><div class='add'>+			     UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT,</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE,</div><div class='add'>+			UVERBS_OBJECT_FLOW_ACTION,</div><div class='add'>+			UVERBS_ACCESS_NEW,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF,</div><div class='add'>+			   UVERBS_ATTR_MIN_SIZE(1),</div><div class='add'>+			   UA_ALLOC_AND_COPY,</div><div class='add'>+			   UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE,</div><div class='add'>+			     enum mlx5_ib_uapi_flow_action_packet_reformat_type,</div><div class='add'>+			     UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE,</div><div class='add'>+			     enum mlx5_ib_uapi_flow_table_type,</div><div class='add'>+			     UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+ADD_UVERBS_METHODS(</div><div class='add'>+	mlx5_ib_flow_actions,</div><div class='add'>+	UVERBS_OBJECT_FLOW_ACTION,</div><div class='add'>+	&amp;UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER),</div><div class='add'>+	&amp;UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	MLX5_IB_METHOD_FLOW_MATCHER_CREATE,</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE,</div><div class='add'>+			MLX5_IB_OBJECT_FLOW_MATCHER,</div><div class='add'>+			UVERBS_ACCESS_NEW,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(</div><div class='add'>+		MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK,</div><div class='add'>+		UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),</div><div class='add'>+		UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE,</div><div class='add'>+			    mlx5_ib_flow_type,</div><div class='add'>+			    UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u8),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,</div><div class='add'>+			     enum ib_flow_flags,</div><div class='add'>+			     UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE,</div><div class='add'>+			     enum mlx5_ib_uapi_flow_table_type,</div><div class='add'>+			     UA_OPTIONAL),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_OPTIONAL));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD_DESTROY(</div><div class='add'>+	MLX5_IB_METHOD_FLOW_MATCHER_DESTROY,</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE,</div><div class='add'>+			MLX5_IB_OBJECT_FLOW_MATCHER,</div><div class='add'>+			UVERBS_ACCESS_DESTROY,</div><div class='add'>+			UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER,</div><div class='add'>+			    UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup),</div><div class='add'>+			    &amp;UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE),</div><div class='add'>+			    &amp;UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	MLX5_IB_METHOD_STEERING_ANCHOR_CREATE,</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE,</div><div class='add'>+			MLX5_IB_OBJECT_STEERING_ANCHOR,</div><div class='add'>+			UVERBS_ACCESS_NEW,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE,</div><div class='add'>+			     enum mlx5_ib_uapi_flow_table_type,</div><div class='add'>+			     UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u16),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD_DESTROY(</div><div class='add'>+	MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY,</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_DESTROY_HANDLE,</div><div class='add'>+			MLX5_IB_OBJECT_STEERING_ANCHOR,</div><div class='add'>+			UVERBS_ACCESS_DESTROY,</div><div class='add'>+			UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_OBJECT(</div><div class='add'>+	MLX5_IB_OBJECT_STEERING_ANCHOR,</div><div class='add'>+	UVERBS_TYPE_ALLOC_IDR(steering_anchor_cleanup),</div><div class='add'>+	&amp;UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE),</div><div class='add'>+	&amp;UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY));</div><div class='add'>+</div><div class='add'>+const struct uapi_definition mlx5_ib_flow_defs[] = {</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(</div><div class='add'>+		MLX5_IB_OBJECT_FLOW_MATCHER),</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE(</div><div class='add'>+		UVERBS_OBJECT_FLOW,</div><div class='add'>+		&amp;mlx5_ib_fs),</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,</div><div class='add'>+				&amp;mlx5_ib_flow_actions),</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(</div><div class='add'>+		MLX5_IB_OBJECT_STEERING_ANCHOR,</div><div class='add'>+		UAPI_DEF_IS_OBJ_SUPPORTED(mlx5_ib_shared_ft_allowed)),</div><div class='add'>+	{},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops flow_ops = {</div><div class='add'>+	.create_flow = mlx5_ib_create_flow,</div><div class='add'>+	.destroy_flow = mlx5_ib_destroy_flow,</div><div class='add'>+	.destroy_flow_action = mlx5_ib_destroy_flow_action,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+int mlx5_ib_fs_init(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int i, j;</div><div class='add'>+</div><div class='add'>+	dev-&gt;flow_db = kzalloc(sizeof(*dev-&gt;flow_db), GFP_KERNEL);</div><div class='add'>+</div><div class='add'>+	if (!dev-&gt;flow_db)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; MLX5_RDMA_TRANSPORT_BYPASS_PRIO; i++) {</div><div class='add'>+		dev-&gt;flow_db-&gt;rdma_transport_rx[i] =</div><div class='add'>+			kcalloc(dev-&gt;num_ports,</div><div class='add'>+				sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL);</div><div class='add'>+		if (!dev-&gt;flow_db-&gt;rdma_transport_rx[i])</div><div class='add'>+			goto free_rdma_transport_rx;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	for (j = 0; j &lt; MLX5_RDMA_TRANSPORT_BYPASS_PRIO; j++) {</div><div class='add'>+		dev-&gt;flow_db-&gt;rdma_transport_tx[j] =</div><div class='add'>+			kcalloc(dev-&gt;num_ports,</div><div class='add'>+				sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL);</div><div class='add'>+		if (!dev-&gt;flow_db-&gt;rdma_transport_tx[j])</div><div class='add'>+			goto free_rdma_transport_tx;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_init(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='add'>+</div><div class='add'>+	ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;flow_ops);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+free_rdma_transport_tx:</div><div class='add'>+	while (j--)</div><div class='add'>+		kfree(dev-&gt;flow_db-&gt;rdma_transport_tx[j]);</div><div class='add'>+free_rdma_transport_rx:</div><div class='add'>+	while (i--)</div><div class='add'>+		kfree(dev-&gt;flow_db-&gt;rdma_transport_rx[i]);</div><div class='add'>+	kfree(dev-&gt;flow_db);</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/fs.h b/drivers/infiniband/hw/mlx5/fs.h<br/>new file mode 100644<br/>index 000000000000..7abba0e2837c<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/fs.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/fs.h</a></div><div class='hunk'>@@ -0,0 +1,36 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef _MLX5_IB_FS_H</div><div class='add'>+#define _MLX5_IB_FS_H</div><div class='add'>+</div><div class='add'>+#include "mlx5_ib.h"</div><div class='add'>+</div><div class='add'>+int mlx5_ib_fs_init(struct mlx5_ib_dev *dev);</div><div class='add'>+void mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev *dev);</div><div class='add'>+</div><div class='add'>+static inline void mlx5_ib_fs_cleanup(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	/* When a steering anchor is created, a special flow table is also</div><div class='add'>+	 * created for the user to reference. Since the user can reference it,</div><div class='add'>+	 * the kernel cannot trust that when the user destroys the steering</div><div class='add'>+	 * anchor, they no longer reference the flow table.</div><div class='add'>+	 *</div><div class='add'>+	 * To address this issue, when a user destroys a steering anchor, only</div><div class='add'>+	 * the flow steering rule in the table is destroyed, but the table</div><div class='add'>+	 * itself is kept to deal with the above scenario. The remaining</div><div class='add'>+	 * resources are only removed when the RDMA device is destroyed, which</div><div class='add'>+	 * is a safe assumption that all references are gone.</div><div class='add'>+	 */</div><div class='add'>+	mlx5_ib_fs_cleanup_anchor(dev);</div><div class='add'>+	for (i = 0; i &lt; MLX5_RDMA_TRANSPORT_BYPASS_PRIO; i++)</div><div class='add'>+		kfree(dev-&gt;flow_db-&gt;rdma_transport_tx[i]);</div><div class='add'>+	for (i = 0; i &lt; MLX5_RDMA_TRANSPORT_BYPASS_PRIO; i++)</div><div class='add'>+		kfree(dev-&gt;flow_db-&gt;rdma_transport_rx[i]);</div><div class='add'>+	kfree(dev-&gt;flow_db);</div><div class='add'>+}</div><div class='add'>+#endif /* _MLX5_IB_FS_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/gsi.c b/drivers/infiniband/hw/mlx5/gsi.c<br/>index 4950df3f71b6..d5487834ed25 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/gsi.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/gsi.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/gsi.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/gsi.c</a></div><div class='hunk'>@@ -35,44 +35,19 @@</div><div class='ctx'> struct mlx5_ib_gsi_wr {</div><div class='ctx'> 	struct ib_cqe cqe;</div><div class='ctx'> 	struct ib_wc wc;</div><div class='del'>-	int send_flags;</div><div class='ctx'> 	bool completed:1;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct mlx5_ib_gsi_qp {</div><div class='del'>-	struct ib_qp ibqp;</div><div class='del'>-	struct ib_qp *rx_qp;</div><div class='del'>-	u8 port_num;</div><div class='del'>-	struct ib_qp_cap cap;</div><div class='del'>-	enum ib_sig_type sq_sig_type;</div><div class='del'>-	/* Serialize qp state modifications */</div><div class='del'>-	struct mutex mutex;</div><div class='del'>-	struct ib_cq *cq;</div><div class='del'>-	struct mlx5_ib_gsi_wr *outstanding_wrs;</div><div class='del'>-	u32 outstanding_pi, outstanding_ci;</div><div class='del'>-	int num_qps;</div><div class='del'>-	/* Protects access to the tx_qps. Post send operations synchronize</div><div class='del'>-	 * with tx_qp creation in setup_qp(). Also protects the</div><div class='del'>-	 * outstanding_wrs array and indices.</div><div class='del'>-	 */</div><div class='del'>-	spinlock_t lock;</div><div class='del'>-	struct ib_qp **tx_qps;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct mlx5_ib_gsi_qp *gsi_qp(struct ib_qp *qp)</div><div class='del'>-{</div><div class='del'>-	return container_of(qp, struct mlx5_ib_gsi_qp, ibqp);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static bool mlx5_ib_deth_sqpn_cap(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	return MLX5_CAP_GEN(dev-&gt;mdev, set_deth_sqpn);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* Call with gsi-&gt;lock locked */</div><div class='del'>-static void generate_completions(struct mlx5_ib_gsi_qp *gsi)</div><div class='add'>+static void generate_completions(struct mlx5_ib_qp *mqp)</div><div class='ctx'> {</div><div class='del'>-	struct ib_cq *gsi_cq = gsi-&gt;ibqp.send_cq;</div><div class='add'>+	struct mlx5_ib_gsi_qp *gsi = &amp;mqp-&gt;gsi;</div><div class='add'>+	struct ib_cq *gsi_cq = mqp-&gt;ibqp.send_cq;</div><div class='ctx'> 	struct mlx5_ib_gsi_wr *wr;</div><div class='ctx'> 	u32 index;</div><div class='ctx'> </div><div class='hunk'>@@ -83,10 +58,7 @@ static void generate_completions(struct mlx5_ib_gsi_qp *gsi)</div><div class='ctx'> 		if (!wr-&gt;completed)</div><div class='ctx'> 			break;</div><div class='ctx'> </div><div class='del'>-		if (gsi-&gt;sq_sig_type == IB_SIGNAL_ALL_WR ||</div><div class='del'>-		    wr-&gt;send_flags &amp; IB_SEND_SIGNALED)</div><div class='del'>-			WARN_ON_ONCE(mlx5_ib_generate_wc(gsi_cq, &amp;wr-&gt;wc));</div><div class='del'>-</div><div class='add'>+		WARN_ON_ONCE(mlx5_ib_generate_wc(gsi_cq, &amp;wr-&gt;wc));</div><div class='ctx'> 		wr-&gt;completed = false;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -98,6 +70,7 @@ static void handle_single_completion(struct ib_cq *cq, struct ib_wc *wc)</div><div class='ctx'> 	struct mlx5_ib_gsi_qp *gsi = cq-&gt;cq_context;</div><div class='ctx'> 	struct mlx5_ib_gsi_wr *wr =</div><div class='ctx'> 		container_of(wc-&gt;wr_cqe, struct mlx5_ib_gsi_wr, cqe);</div><div class='add'>+	struct mlx5_ib_qp *mqp = container_of(gsi, struct mlx5_ib_qp, gsi);</div><div class='ctx'> 	u64 wr_id;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> </div><div class='hunk'>@@ -106,54 +79,43 @@ static void handle_single_completion(struct ib_cq *cq, struct ib_wc *wc)</div><div class='ctx'> 	wr_id = wr-&gt;wc.wr_id;</div><div class='ctx'> 	wr-&gt;wc = *wc;</div><div class='ctx'> 	wr-&gt;wc.wr_id = wr_id;</div><div class='del'>-	wr-&gt;wc.qp = &amp;gsi-&gt;ibqp;</div><div class='add'>+	wr-&gt;wc.qp = &amp;mqp-&gt;ibqp;</div><div class='ctx'> </div><div class='del'>-	generate_completions(gsi);</div><div class='add'>+	generate_completions(mqp);</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;gsi-&gt;lock, flags);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd,</div><div class='del'>-				    struct ib_qp_init_attr *init_attr)</div><div class='add'>+int mlx5_ib_create_gsi(struct ib_pd *pd, struct mlx5_ib_qp *mqp,</div><div class='add'>+		       struct ib_qp_init_attr *attr)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='ctx'> 	struct mlx5_ib_gsi_qp *gsi;</div><div class='del'>-	struct ib_qp_init_attr hw_init_attr = *init_attr;</div><div class='del'>-	const u8 port_num = init_attr-&gt;port_num;</div><div class='del'>-	const int num_pkeys = pd-&gt;device-&gt;attrs.max_pkeys;</div><div class='del'>-	const int num_qps = mlx5_ib_deth_sqpn_cap(dev) ? num_pkeys : 0;</div><div class='add'>+	struct ib_qp_init_attr hw_init_attr = *attr;</div><div class='add'>+	const u8 port_num = attr-&gt;port_num;</div><div class='add'>+	int num_qps = 0;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_dbg(dev, "creating GSI QP\n");</div><div class='del'>-</div><div class='del'>-	if (port_num &gt; ARRAY_SIZE(dev-&gt;devr.ports) || port_num &lt; 1) {</div><div class='del'>-		mlx5_ib_warn(dev,</div><div class='del'>-			     "invalid port number %d during GSI QP creation\n",</div><div class='del'>-			     port_num);</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+	if (mlx5_ib_deth_sqpn_cap(dev)) {</div><div class='add'>+		if (MLX5_CAP_GEN(dev-&gt;mdev,</div><div class='add'>+				 port_type) == MLX5_CAP_PORT_TYPE_IB)</div><div class='add'>+			num_qps = pd-&gt;device-&gt;attrs.max_pkeys;</div><div class='add'>+		else if (dev-&gt;lag_active)</div><div class='add'>+			num_qps = dev-&gt;lag_ports;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	gsi = kzalloc(sizeof(*gsi), GFP_KERNEL);</div><div class='del'>-	if (!gsi)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='add'>+	gsi = &amp;mqp-&gt;gsi;</div><div class='ctx'> 	gsi-&gt;tx_qps = kcalloc(num_qps, sizeof(*gsi-&gt;tx_qps), GFP_KERNEL);</div><div class='del'>-	if (!gsi-&gt;tx_qps) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err_free;</div><div class='del'>-	}</div><div class='add'>+	if (!gsi-&gt;tx_qps)</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	gsi-&gt;outstanding_wrs = kcalloc(init_attr-&gt;cap.max_send_wr,</div><div class='del'>-				       sizeof(*gsi-&gt;outstanding_wrs),</div><div class='del'>-				       GFP_KERNEL);</div><div class='add'>+	gsi-&gt;outstanding_wrs =</div><div class='add'>+		kcalloc(attr-&gt;cap.max_send_wr, sizeof(*gsi-&gt;outstanding_wrs),</div><div class='add'>+			GFP_KERNEL);</div><div class='ctx'> 	if (!gsi-&gt;outstanding_wrs) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='ctx'> 		goto err_free_tx;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	mutex_init(&amp;gsi-&gt;mutex);</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;dev-&gt;devr.mutex);</div><div class='del'>-</div><div class='ctx'> 	if (dev-&gt;devr.ports[port_num - 1].gsi) {</div><div class='ctx'> 		mlx5_ib_warn(dev, "GSI QP already exists on port %d\n",</div><div class='ctx'> 			     port_num);</div><div class='hunk'>@@ -163,16 +125,15 @@ struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd,</div><div class='ctx'> 	gsi-&gt;num_qps = num_qps;</div><div class='ctx'> 	spin_lock_init(&amp;gsi-&gt;lock);</div><div class='ctx'> </div><div class='del'>-	gsi-&gt;cap = init_attr-&gt;cap;</div><div class='del'>-	gsi-&gt;sq_sig_type = init_attr-&gt;sq_sig_type;</div><div class='del'>-	gsi-&gt;ibqp.qp_num = 1;</div><div class='add'>+	gsi-&gt;cap = attr-&gt;cap;</div><div class='ctx'> 	gsi-&gt;port_num = port_num;</div><div class='ctx'> </div><div class='del'>-	gsi-&gt;cq = ib_alloc_cq(pd-&gt;device, gsi, init_attr-&gt;cap.max_send_wr, 0,</div><div class='add'>+	gsi-&gt;cq = ib_alloc_cq(pd-&gt;device, gsi, attr-&gt;cap.max_send_wr, 0,</div><div class='ctx'> 			      IB_POLL_SOFTIRQ);</div><div class='ctx'> 	if (IS_ERR(gsi-&gt;cq)) {</div><div class='del'>-		mlx5_ib_warn(dev, "unable to create send CQ for GSI QP. error %ld\n",</div><div class='del'>-			     PTR_ERR(gsi-&gt;cq));</div><div class='add'>+		mlx5_ib_warn(dev,</div><div class='add'>+			     "unable to create send CQ for GSI QP. error %pe\n",</div><div class='add'>+			     gsi-&gt;cq);</div><div class='ctx'> 		ret = PTR_ERR(gsi-&gt;cq);</div><div class='ctx'> 		goto err_free_wrs;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -184,52 +145,43 @@ struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd,</div><div class='ctx'> 		hw_init_attr.cap.max_send_sge = 0;</div><div class='ctx'> 		hw_init_attr.cap.max_inline_data = 0;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='ctx'> 	gsi-&gt;rx_qp = ib_create_qp(pd, &amp;hw_init_attr);</div><div class='ctx'> 	if (IS_ERR(gsi-&gt;rx_qp)) {</div><div class='del'>-		mlx5_ib_warn(dev, "unable to create hardware GSI QP. error %ld\n",</div><div class='del'>-			     PTR_ERR(gsi-&gt;rx_qp));</div><div class='add'>+		mlx5_ib_warn(dev,</div><div class='add'>+			     "unable to create hardware GSI QP. error %pe\n",</div><div class='add'>+			     gsi-&gt;rx_qp);</div><div class='ctx'> 		ret = PTR_ERR(gsi-&gt;rx_qp);</div><div class='ctx'> 		goto err_destroy_cq;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	dev-&gt;devr.ports[init_attr-&gt;port_num - 1].gsi = gsi;</div><div class='del'>-</div><div class='del'>-	mutex_unlock(&amp;dev-&gt;devr.mutex);</div><div class='del'>-</div><div class='del'>-	return &amp;gsi-&gt;ibqp;</div><div class='add'>+	dev-&gt;devr.ports[attr-&gt;port_num - 1].gsi = gsi;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='ctx'> err_destroy_cq:</div><div class='ctx'> 	ib_free_cq(gsi-&gt;cq);</div><div class='ctx'> err_free_wrs:</div><div class='del'>-	mutex_unlock(&amp;dev-&gt;devr.mutex);</div><div class='ctx'> 	kfree(gsi-&gt;outstanding_wrs);</div><div class='ctx'> err_free_tx:</div><div class='ctx'> 	kfree(gsi-&gt;tx_qps);</div><div class='del'>-err_free:</div><div class='del'>-	kfree(gsi);</div><div class='del'>-	return ERR_PTR(ret);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_ib_gsi_destroy_qp(struct ib_qp *qp)</div><div class='add'>+int mlx5_ib_destroy_gsi(struct mlx5_ib_qp *mqp)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(qp-&gt;device);</div><div class='del'>-	struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp);</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(mqp-&gt;ibqp.device);</div><div class='add'>+	struct mlx5_ib_gsi_qp *gsi = &amp;mqp-&gt;gsi;</div><div class='ctx'> 	const int port_num = gsi-&gt;port_num;</div><div class='ctx'> 	int qp_index;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_dbg(dev, "destroying GSI QP\n");</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;dev-&gt;devr.mutex);</div><div class='ctx'> 	ret = ib_destroy_qp(gsi-&gt;rx_qp);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		mlx5_ib_warn(dev, "unable to destroy hardware GSI QP. error %d\n",</div><div class='ctx'> 			     ret);</div><div class='del'>-		mutex_unlock(&amp;dev-&gt;devr.mutex);</div><div class='ctx'> 		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> 	dev-&gt;devr.ports[port_num - 1].gsi = NULL;</div><div class='del'>-	mutex_unlock(&amp;dev-&gt;devr.mutex);</div><div class='ctx'> 	gsi-&gt;rx_qp = NULL;</div><div class='ctx'> </div><div class='ctx'> 	for (qp_index = 0; qp_index &lt; gsi-&gt;num_qps; ++qp_index) {</div><div class='hunk'>@@ -243,8 +195,6 @@ int mlx5_ib_gsi_destroy_qp(struct ib_qp *qp)</div><div class='ctx'> </div><div class='ctx'> 	kfree(gsi-&gt;outstanding_wrs);</div><div class='ctx'> 	kfree(gsi-&gt;tx_qps);</div><div class='del'>-	kfree(gsi);</div><div class='del'>-</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -261,16 +211,15 @@ static struct ib_qp *create_gsi_ud_qp(struct mlx5_ib_gsi_qp *gsi)</div><div class='ctx'> 			.max_send_sge = gsi-&gt;cap.max_send_sge,</div><div class='ctx'> 			.max_inline_data = gsi-&gt;cap.max_inline_data,</div><div class='ctx'> 		},</div><div class='del'>-		.sq_sig_type = gsi-&gt;sq_sig_type,</div><div class='ctx'> 		.qp_type = IB_QPT_UD,</div><div class='del'>-		.create_flags = mlx5_ib_create_qp_sqpn_qp1(),</div><div class='add'>+		.create_flags = MLX5_IB_QP_CREATE_SQPN_QP1,</div><div class='ctx'> 	};</div><div class='ctx'> </div><div class='ctx'> 	return ib_create_qp(pd, &amp;init_attr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int modify_to_rts(struct mlx5_ib_gsi_qp *gsi, struct ib_qp *qp,</div><div class='del'>-			 u16 qp_index)</div><div class='add'>+			 u16 pkey_index)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(qp-&gt;device);</div><div class='ctx'> 	struct ib_qp_attr attr;</div><div class='hunk'>@@ -279,7 +228,7 @@ static int modify_to_rts(struct mlx5_ib_gsi_qp *gsi, struct ib_qp *qp,</div><div class='ctx'> </div><div class='ctx'> 	mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_QKEY | IB_QP_PORT;</div><div class='ctx'> 	attr.qp_state = IB_QPS_INIT;</div><div class='del'>-	attr.pkey_index = qp_index;</div><div class='add'>+	attr.pkey_index = pkey_index;</div><div class='ctx'> 	attr.qkey = IB_QP1_QKEY;</div><div class='ctx'> 	attr.port_num = gsi-&gt;port_num;</div><div class='ctx'> 	ret = ib_modify_qp(qp, &amp;attr, mask);</div><div class='hunk'>@@ -313,12 +262,17 @@ static void setup_qp(struct mlx5_ib_gsi_qp *gsi, u16 qp_index)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_device *device = gsi-&gt;rx_qp-&gt;device;</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='add'>+	int pkey_index = qp_index;</div><div class='add'>+	struct mlx5_ib_qp *mqp;</div><div class='ctx'> 	struct ib_qp *qp;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	u16 pkey;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_query_pkey(device, gsi-&gt;port_num, qp_index, &amp;pkey);</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev,  port_type) != MLX5_CAP_PORT_TYPE_IB)</div><div class='add'>+		pkey_index = 0;</div><div class='add'>+</div><div class='add'>+	ret = ib_query_pkey(device, gsi-&gt;port_num, pkey_index, &amp;pkey);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		mlx5_ib_warn(dev, "unable to read P_Key at port %d, index %d\n",</div><div class='ctx'> 			     gsi-&gt;port_num, qp_index);</div><div class='hunk'>@@ -342,12 +296,16 @@ static void setup_qp(struct mlx5_ib_gsi_qp *gsi, u16 qp_index)</div><div class='ctx'> </div><div class='ctx'> 	qp = create_gsi_ud_qp(gsi);</div><div class='ctx'> 	if (IS_ERR(qp)) {</div><div class='del'>-		mlx5_ib_warn(dev, "unable to create hardware UD QP for GSI: %ld\n",</div><div class='del'>-			     PTR_ERR(qp));</div><div class='add'>+		mlx5_ib_warn(dev,</div><div class='add'>+			     "unable to create hardware UD QP for GSI: %pe\n",</div><div class='add'>+			     qp);</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ret = modify_to_rts(gsi, qp, qp_index);</div><div class='add'>+	mqp = to_mqp(qp);</div><div class='add'>+	if (dev-&gt;lag_active)</div><div class='add'>+		mqp-&gt;gsi_lag_port = qp_index + 1;</div><div class='add'>+	ret = modify_to_rts(gsi, qp, pkey_index);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		goto err_destroy_qp;</div><div class='ctx'> </div><div class='hunk'>@@ -362,58 +320,49 @@ err_destroy_qp:</div><div class='ctx'> 	WARN_ON_ONCE(qp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void setup_qps(struct mlx5_ib_gsi_qp *gsi)</div><div class='del'>-{</div><div class='del'>-	u16 qp_index;</div><div class='del'>-</div><div class='del'>-	for (qp_index = 0; qp_index &lt; gsi-&gt;num_qps; ++qp_index)</div><div class='del'>-		setup_qp(gsi, qp_index);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> int mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,</div><div class='ctx'> 			  int attr_mask)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(qp-&gt;device);</div><div class='del'>-	struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp);</div><div class='add'>+	struct mlx5_ib_qp *mqp = to_mqp(qp);</div><div class='add'>+	struct mlx5_ib_gsi_qp *gsi = &amp;mqp-&gt;gsi;</div><div class='add'>+	u16 qp_index;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	mlx5_ib_dbg(dev, "modifying GSI QP to state %d\n", attr-&gt;qp_state);</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;gsi-&gt;mutex);</div><div class='ctx'> 	ret = ib_modify_qp(gsi-&gt;rx_qp, attr, attr_mask);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		mlx5_ib_warn(dev, "unable to modify GSI rx QP: %d\n", ret);</div><div class='del'>-		goto unlock;</div><div class='add'>+		return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (to_mqp(gsi-&gt;rx_qp)-&gt;state == IB_QPS_RTS)</div><div class='del'>-		setup_qps(gsi);</div><div class='add'>+	if (to_mqp(gsi-&gt;rx_qp)-&gt;state != IB_QPS_RTS)</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-unlock:</div><div class='del'>-	mutex_unlock(&amp;gsi-&gt;mutex);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='add'>+	for (qp_index = 0; qp_index &lt; gsi-&gt;num_qps; ++qp_index)</div><div class='add'>+		setup_qp(gsi, qp_index);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 			 int qp_attr_mask,</div><div class='ctx'> 			 struct ib_qp_init_attr *qp_init_attr)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp);</div><div class='add'>+	struct mlx5_ib_qp *mqp = to_mqp(qp);</div><div class='add'>+	struct mlx5_ib_gsi_qp *gsi = &amp;mqp-&gt;gsi;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;gsi-&gt;mutex);</div><div class='ctx'> 	ret = ib_query_qp(gsi-&gt;rx_qp, qp_attr, qp_attr_mask, qp_init_attr);</div><div class='ctx'> 	qp_init_attr-&gt;cap = gsi-&gt;cap;</div><div class='del'>-	mutex_unlock(&amp;gsi-&gt;mutex);</div><div class='del'>-</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* Call with gsi-&gt;lock locked */</div><div class='del'>-static int mlx5_ib_add_outstanding_wr(struct mlx5_ib_gsi_qp *gsi,</div><div class='add'>+static int mlx5_ib_add_outstanding_wr(struct mlx5_ib_qp *mqp,</div><div class='ctx'> 				      struct ib_ud_wr *wr, struct ib_wc *wc)</div><div class='ctx'> {</div><div class='add'>+	struct mlx5_ib_gsi_qp *gsi = &amp;mqp-&gt;gsi;</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(gsi-&gt;rx_qp-&gt;device);</div><div class='ctx'> 	struct mlx5_ib_gsi_wr *gsi_wr;</div><div class='ctx'> </div><div class='hunk'>@@ -442,22 +391,21 @@ static int mlx5_ib_add_outstanding_wr(struct mlx5_ib_gsi_qp *gsi,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /* Call with gsi-&gt;lock locked */</div><div class='del'>-static int mlx5_ib_gsi_silent_drop(struct mlx5_ib_gsi_qp *gsi,</div><div class='del'>-				    struct ib_ud_wr *wr)</div><div class='add'>+static int mlx5_ib_gsi_silent_drop(struct mlx5_ib_qp *mqp, struct ib_ud_wr *wr)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_wc wc = {</div><div class='ctx'> 		{ .wr_id = wr-&gt;wr.wr_id },</div><div class='ctx'> 		.status = IB_WC_SUCCESS,</div><div class='ctx'> 		.opcode = IB_WC_SEND,</div><div class='del'>-		.qp = &amp;gsi-&gt;ibqp,</div><div class='add'>+		.qp = &amp;mqp-&gt;ibqp,</div><div class='ctx'> 	};</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = mlx5_ib_add_outstanding_wr(gsi, wr, &amp;wc);</div><div class='add'>+	ret = mlx5_ib_add_outstanding_wr(mqp, wr, &amp;wc);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='ctx'> </div><div class='del'>-	generate_completions(gsi);</div><div class='add'>+	generate_completions(mqp);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -466,11 +414,15 @@ static int mlx5_ib_gsi_silent_drop(struct mlx5_ib_gsi_qp *gsi,</div><div class='ctx'> static struct ib_qp *get_tx_qp(struct mlx5_ib_gsi_qp *gsi, struct ib_ud_wr *wr)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(gsi-&gt;rx_qp-&gt;device);</div><div class='add'>+	struct mlx5_ib_ah *ah = to_mah(wr-&gt;ah);</div><div class='ctx'> 	int qp_index = wr-&gt;pkey_index;</div><div class='ctx'> </div><div class='del'>-	if (!mlx5_ib_deth_sqpn_cap(dev))</div><div class='add'>+	if (!gsi-&gt;num_qps)</div><div class='ctx'> 		return gsi-&gt;rx_qp;</div><div class='ctx'> </div><div class='add'>+	if (dev-&gt;lag_active &amp;&amp; ah-&gt;xmit_port)</div><div class='add'>+		qp_index = ah-&gt;xmit_port - 1;</div><div class='add'>+</div><div class='ctx'> 	if (qp_index &gt;= gsi-&gt;num_qps)</div><div class='ctx'> 		return NULL;</div><div class='ctx'> </div><div class='hunk'>@@ -480,7 +432,8 @@ static struct ib_qp *get_tx_qp(struct mlx5_ib_gsi_qp *gsi, struct ib_ud_wr *wr)</div><div class='ctx'> int mlx5_ib_gsi_post_send(struct ib_qp *qp, const struct ib_send_wr *wr,</div><div class='ctx'> 			  const struct ib_send_wr **bad_wr)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp);</div><div class='add'>+	struct mlx5_ib_qp *mqp = to_mqp(qp);</div><div class='add'>+	struct mlx5_ib_gsi_qp *gsi = &amp;mqp-&gt;gsi;</div><div class='ctx'> 	struct ib_qp *tx_qp;</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int ret;</div><div class='hunk'>@@ -493,22 +446,21 @@ int mlx5_ib_gsi_post_send(struct ib_qp *qp, const struct ib_send_wr *wr,</div><div class='ctx'> 		spin_lock_irqsave(&amp;gsi-&gt;lock, flags);</div><div class='ctx'> 		tx_qp = get_tx_qp(gsi, &amp;cur_wr);</div><div class='ctx'> 		if (!tx_qp) {</div><div class='del'>-			ret = mlx5_ib_gsi_silent_drop(gsi, &amp;cur_wr);</div><div class='add'>+			ret = mlx5_ib_gsi_silent_drop(mqp, &amp;cur_wr);</div><div class='ctx'> 			if (ret)</div><div class='ctx'> 				goto err;</div><div class='ctx'> 			spin_unlock_irqrestore(&amp;gsi-&gt;lock, flags);</div><div class='ctx'> 			continue;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		ret = mlx5_ib_add_outstanding_wr(gsi, &amp;cur_wr, NULL);</div><div class='add'>+		ret = mlx5_ib_add_outstanding_wr(mqp, &amp;cur_wr, NULL);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto err;</div><div class='ctx'> </div><div class='ctx'> 		ret = ib_post_send(tx_qp, &amp;cur_wr.wr, bad_wr);</div><div class='ctx'> 		if (ret) {</div><div class='ctx'> 			/* Undo the effect of adding the outstanding wr */</div><div class='del'>-			gsi-&gt;outstanding_pi = (gsi-&gt;outstanding_pi - 1) %</div><div class='del'>-					      gsi-&gt;cap.max_send_wr;</div><div class='add'>+			gsi-&gt;outstanding_pi--;</div><div class='ctx'> 			goto err;</div><div class='ctx'> 		}</div><div class='ctx'> 		spin_unlock_irqrestore(&amp;gsi-&gt;lock, flags);</div><div class='hunk'>@@ -525,17 +477,16 @@ err:</div><div class='ctx'> int mlx5_ib_gsi_post_recv(struct ib_qp *qp, const struct ib_recv_wr *wr,</div><div class='ctx'> 			  const struct ib_recv_wr **bad_wr)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp);</div><div class='add'>+	struct mlx5_ib_qp *mqp = to_mqp(qp);</div><div class='add'>+	struct mlx5_ib_gsi_qp *gsi = &amp;mqp-&gt;gsi;</div><div class='ctx'> </div><div class='ctx'> 	return ib_post_recv(gsi-&gt;rx_qp, wr, bad_wr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi)</div><div class='ctx'> {</div><div class='del'>-	if (!gsi)</div><div class='del'>-		return;</div><div class='add'>+	u16 qp_index;</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;gsi-&gt;mutex);</div><div class='del'>-	setup_qps(gsi);</div><div class='del'>-	mutex_unlock(&amp;gsi-&gt;mutex);</div><div class='add'>+	for (qp_index = 0; qp_index &lt; gsi-&gt;num_qps; ++qp_index)</div><div class='add'>+		setup_qp(gsi, qp_index);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/ib_rep.c b/drivers/infiniband/hw/mlx5/ib_rep.c<br/>index 22e651cb5534..bbecca405171 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/ib_rep.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/ib_rep.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/ib_rep.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/ib_rep.c</a></div><div class='hunk'>@@ -8,19 +8,95 @@</div><div class='ctx'> #include "srq.h"</div><div class='ctx'> </div><div class='ctx'> static int</div><div class='del'>-mlx5_ib_set_vport_rep(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)</div><div class='add'>+mlx5_ib_set_vport_rep(struct mlx5_core_dev *dev,</div><div class='add'>+		      struct mlx5_eswitch_rep *rep,</div><div class='add'>+		      int vport_index)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *ibdev;</div><div class='del'>-	int vport_index;</div><div class='add'>+	struct net_device *ndev;</div><div class='ctx'> </div><div class='del'>-	ibdev = mlx5_ib_get_uplink_ibdev(dev-&gt;priv.eswitch);</div><div class='del'>-	vport_index = ibdev-&gt;free_port++;</div><div class='add'>+	ibdev = mlx5_eswitch_uplink_get_proto_dev(dev-&gt;priv.eswitch, REP_IB);</div><div class='add'>+	if (!ibdev)</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	ibdev-&gt;port[vport_index].rep = rep;</div><div class='del'>-	write_lock(&amp;ibdev-&gt;port[vport_index].roce.netdev_lock);</div><div class='del'>-	ibdev-&gt;port[vport_index].roce.netdev =</div><div class='del'>-		mlx5_ib_get_rep_netdev(dev-&gt;priv.eswitch, rep-&gt;vport);</div><div class='del'>-	write_unlock(&amp;ibdev-&gt;port[vport_index].roce.netdev_lock);</div><div class='add'>+	rep-&gt;rep_data[REP_IB].priv = ibdev;</div><div class='add'>+	ndev = mlx5_ib_get_rep_netdev(rep-&gt;esw, rep-&gt;vport);</div><div class='add'>+</div><div class='add'>+	return ib_device_set_netdev(&amp;ibdev-&gt;ib_dev, ndev, vport_index + 1);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_register_peer_vport_reps(struct mlx5_core_dev *mdev);</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_num_ports_update(struct mlx5_core_dev *dev, u32 *num_ports)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_core_dev *peer_dev;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	mlx5_lag_for_each_peer_mdev(dev, peer_dev, i) {</div><div class='add'>+		u32 peer_num_ports = mlx5_eswitch_get_total_vports(peer_dev);</div><div class='add'>+</div><div class='add'>+		if (mlx5_lag_is_mpesw(peer_dev))</div><div class='add'>+			*num_ports += peer_num_ports;</div><div class='add'>+		else</div><div class='add'>+			/* Only 1 ib port is the representor for all uplinks */</div><div class='add'>+			*num_ports += peer_num_ports - 1;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_set_owner_transport(struct mlx5_core_dev *cur_owner,</div><div class='add'>+					struct mlx5_core_dev *new_owner)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_TX(cur_owner, ft_support) ||</div><div class='add'>+	    !MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_RX(cur_owner, ft_support))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (!MLX5_CAP_ADV_RDMA(new_owner, rdma_transport_manager) ||</div><div class='add'>+	    !MLX5_CAP_ADV_RDMA(new_owner, rdma_transport_manager_other_eswitch))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	ret = mlx5_fs_set_root_dev(cur_owner, new_owner,</div><div class='add'>+				   FS_FT_RDMA_TRANSPORT_TX);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = mlx5_fs_set_root_dev(cur_owner, new_owner,</div><div class='add'>+				   FS_FT_RDMA_TRANSPORT_RX);</div><div class='add'>+	if (ret) {</div><div class='add'>+		mlx5_fs_set_root_dev(cur_owner, cur_owner,</div><div class='add'>+				     FS_FT_RDMA_TRANSPORT_TX);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_release_transport(struct mlx5_core_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_core_dev *peer_dev;</div><div class='add'>+	int i, ret;</div><div class='add'>+</div><div class='add'>+	mlx5_lag_for_each_peer_mdev(dev, peer_dev, i) {</div><div class='add'>+		ret = mlx5_ib_set_owner_transport(peer_dev, peer_dev);</div><div class='add'>+		WARN_ON_ONCE(ret);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_take_transport(struct mlx5_core_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_core_dev *peer_dev;</div><div class='add'>+	int ret;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	mlx5_lag_for_each_peer_mdev(dev, peer_dev, i) {</div><div class='add'>+		ret = mlx5_ib_set_owner_transport(peer_dev, dev);</div><div class='add'>+		if (ret) {</div><div class='add'>+			mlx5_ib_release_transport(dev);</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -28,112 +104,186 @@ mlx5_ib_set_vport_rep(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)</div><div class='ctx'> static int</div><div class='ctx'> mlx5_ib_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)</div><div class='ctx'> {</div><div class='del'>-	int num_ports = MLX5_TOTAL_VPORTS(dev);</div><div class='add'>+	u32 num_ports = mlx5_eswitch_get_total_vports(dev);</div><div class='add'>+	struct mlx5_core_dev *lag_master = dev;</div><div class='ctx'> 	const struct mlx5_ib_profile *profile;</div><div class='add'>+	struct mlx5_core_dev *peer_dev;</div><div class='ctx'> 	struct mlx5_ib_dev *ibdev;</div><div class='add'>+	int new_uplink = false;</div><div class='ctx'> 	int vport_index;</div><div class='add'>+	int ret;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	vport_index = rep-&gt;vport_index;</div><div class='add'>+</div><div class='add'>+	if (mlx5_lag_is_shared_fdb(dev)) {</div><div class='add'>+		if (mlx5_lag_is_master(dev)) {</div><div class='add'>+			mlx5_ib_num_ports_update(dev, &amp;num_ports);</div><div class='add'>+		} else {</div><div class='add'>+			if (rep-&gt;vport == MLX5_VPORT_UPLINK) {</div><div class='add'>+				if (!mlx5_lag_is_mpesw(dev))</div><div class='add'>+					return 0;</div><div class='add'>+				new_uplink = true;</div><div class='add'>+			}</div><div class='add'>+			mlx5_lag_for_each_peer_mdev(dev, peer_dev, i) {</div><div class='add'>+				u32 peer_n_ports = mlx5_eswitch_get_total_vports(peer_dev);</div><div class='add'>+</div><div class='add'>+				if (mlx5_lag_is_master(peer_dev))</div><div class='add'>+					lag_master = peer_dev;</div><div class='add'>+				else if (!mlx5_lag_is_mpesw(dev))</div><div class='add'>+				/* Only 1 ib port is the representor for all uplinks */</div><div class='add'>+					peer_n_ports--;</div><div class='add'>+</div><div class='add'>+				if (mlx5_get_dev_index(peer_dev) &lt; mlx5_get_dev_index(dev))</div><div class='add'>+					vport_index += peer_n_ports;</div><div class='add'>+			}</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (rep-&gt;vport == MLX5_VPORT_UPLINK)</div><div class='del'>-		profile = &amp;uplink_rep_profile;</div><div class='add'>+	if (rep-&gt;vport == MLX5_VPORT_UPLINK &amp;&amp; !new_uplink)</div><div class='add'>+		profile = &amp;raw_eth_profile;</div><div class='ctx'> 	else</div><div class='del'>-		return mlx5_ib_set_vport_rep(dev, rep);</div><div class='add'>+		return mlx5_ib_set_vport_rep(lag_master, rep, vport_index);</div><div class='ctx'> </div><div class='del'>-	ibdev = ib_alloc_device(mlx5_ib_dev, ib_dev);</div><div class='del'>-	if (!ibdev)</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+	if (mlx5_lag_is_shared_fdb(dev)) {</div><div class='add'>+		ret = mlx5_ib_take_transport(lag_master);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ibdev = ib_alloc_device_with_net(mlx5_ib_dev, ib_dev,</div><div class='add'>+					 mlx5_core_net(lag_master));</div><div class='add'>+	if (!ibdev) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto release_transport;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	ibdev-&gt;port = kcalloc(num_ports, sizeof(*ibdev-&gt;port),</div><div class='ctx'> 			      GFP_KERNEL);</div><div class='ctx'> 	if (!ibdev-&gt;port) {</div><div class='del'>-		ib_dealloc_device(&amp;ibdev-&gt;ib_dev);</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto fail_port;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ibdev-&gt;is_rep = true;</div><div class='del'>-	vport_index = ibdev-&gt;free_port++;</div><div class='add'>+	vport_index = rep-&gt;vport_index;</div><div class='ctx'> 	ibdev-&gt;port[vport_index].rep = rep;</div><div class='del'>-	ibdev-&gt;port[vport_index].roce.netdev =</div><div class='del'>-		mlx5_ib_get_rep_netdev(dev-&gt;priv.eswitch, rep-&gt;vport);</div><div class='del'>-	ibdev-&gt;mdev = dev;</div><div class='add'>+	ibdev-&gt;mdev = lag_master;</div><div class='ctx'> 	ibdev-&gt;num_ports = num_ports;</div><div class='del'>-</div><div class='del'>-	if (!__mlx5_ib_add(ibdev, profile))</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	ibdev-&gt;ib_dev.phys_port_cnt = num_ports;</div><div class='add'>+	ret = ib_device_set_netdev(&amp;ibdev-&gt;ib_dev,</div><div class='add'>+			mlx5_ib_get_rep_netdev(lag_master-&gt;priv.eswitch,</div><div class='add'>+					       rep-&gt;vport),</div><div class='add'>+			vport_index + 1);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto fail_add;</div><div class='add'>+</div><div class='add'>+	ret = __mlx5_ib_add(ibdev, profile);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto fail_add;</div><div class='ctx'> </div><div class='ctx'> 	rep-&gt;rep_data[REP_IB].priv = ibdev;</div><div class='add'>+	if (mlx5_lag_is_shared_fdb(lag_master))</div><div class='add'>+		mlx5_ib_register_peer_vport_reps(lag_master);</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='add'>+</div><div class='add'>+fail_add:</div><div class='add'>+	kfree(ibdev-&gt;port);</div><div class='add'>+fail_port:</div><div class='add'>+	ib_dealloc_device(&amp;ibdev-&gt;ib_dev);</div><div class='add'>+release_transport:</div><div class='add'>+	if (mlx5_lag_is_shared_fdb(lag_master))</div><div class='add'>+		mlx5_ib_release_transport(lag_master);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void *mlx5_ib_rep_to_dev(struct mlx5_eswitch_rep *rep)</div><div class='add'>+{</div><div class='add'>+	return rep-&gt;rep_data[REP_IB].priv;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void</div><div class='ctx'> mlx5_ib_vport_rep_unload(struct mlx5_eswitch_rep *rep)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_dev *dev;</div><div class='add'>+	struct mlx5_core_dev *mdev = mlx5_eswitch_get_core_dev(rep-&gt;esw);</div><div class='add'>+	struct mlx5_ib_dev *dev = mlx5_ib_rep_to_dev(rep);</div><div class='add'>+	int vport_index = rep-&gt;vport_index;</div><div class='add'>+	struct mlx5_ib_port *port;</div><div class='add'>+	int i;</div><div class='ctx'> </div><div class='del'>-	if (!rep-&gt;rep_data[REP_IB].priv ||</div><div class='del'>-	    rep-&gt;vport != MLX5_VPORT_UPLINK)</div><div class='add'>+	if (WARN_ON(!mdev))</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='del'>-	dev = mlx5_ib_rep_to_dev(rep);</div><div class='del'>-	__mlx5_ib_remove(dev, dev-&gt;profile, MLX5_IB_STAGE_MAX);</div><div class='del'>-	rep-&gt;rep_data[REP_IB].priv = NULL;</div><div class='del'>-}</div><div class='add'>+	if (!dev)</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-static void *mlx5_ib_vport_get_proto_dev(struct mlx5_eswitch_rep *rep)</div><div class='del'>-{</div><div class='del'>-	return mlx5_ib_rep_to_dev(rep);</div><div class='del'>-}</div><div class='add'>+	if (mlx5_lag_is_shared_fdb(mdev) &amp;&amp;</div><div class='add'>+	    !mlx5_lag_is_master(mdev)) {</div><div class='add'>+		if (rep-&gt;vport == MLX5_VPORT_UPLINK &amp;&amp; !mlx5_lag_is_mpesw(mdev))</div><div class='add'>+			return;</div><div class='add'>+		for (i = 0; i &lt; dev-&gt;num_ports; i++) {</div><div class='add'>+			if (dev-&gt;port[i].rep == rep)</div><div class='add'>+				break;</div><div class='add'>+		}</div><div class='add'>+		if (WARN_ON(i == dev-&gt;num_ports))</div><div class='add'>+			return;</div><div class='add'>+		vport_index = i;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-static const struct mlx5_eswitch_rep_ops rep_ops = {</div><div class='del'>-	.load = mlx5_ib_vport_rep_load,</div><div class='del'>-	.unload = mlx5_ib_vport_rep_unload,</div><div class='del'>-	.get_proto_dev = mlx5_ib_vport_get_proto_dev,</div><div class='del'>-};</div><div class='add'>+	port = &amp;dev-&gt;port[vport_index];</div><div class='ctx'> </div><div class='del'>-void mlx5_ib_register_vport_reps(struct mlx5_core_dev *mdev)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_eswitch *esw = mdev-&gt;priv.eswitch;</div><div class='add'>+	ib_device_set_netdev(&amp;dev-&gt;ib_dev, NULL, vport_index + 1);</div><div class='add'>+	rep-&gt;rep_data[REP_IB].priv = NULL;</div><div class='add'>+	port-&gt;rep = NULL;</div><div class='ctx'> </div><div class='del'>-	mlx5_eswitch_register_vport_reps(esw, &amp;rep_ops, REP_IB);</div><div class='del'>-}</div><div class='add'>+	if (rep-&gt;vport == MLX5_VPORT_UPLINK) {</div><div class='ctx'> </div><div class='del'>-void mlx5_ib_unregister_vport_reps(struct mlx5_core_dev *mdev)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_eswitch *esw = mdev-&gt;priv.eswitch;</div><div class='add'>+		if (mlx5_lag_is_shared_fdb(mdev) &amp;&amp; !mlx5_lag_is_master(mdev))</div><div class='add'>+			return;</div><div class='ctx'> </div><div class='del'>-	mlx5_eswitch_unregister_vport_reps(esw, REP_IB);</div><div class='del'>-}</div><div class='add'>+		if (mlx5_lag_is_shared_fdb(mdev)) {</div><div class='add'>+			struct mlx5_core_dev *peer_mdev;</div><div class='add'>+			struct mlx5_eswitch *esw;</div><div class='ctx'> </div><div class='del'>-u8 mlx5_ib_eswitch_mode(struct mlx5_eswitch *esw)</div><div class='del'>-{</div><div class='del'>-	return mlx5_eswitch_mode(esw);</div><div class='add'>+			mlx5_lag_for_each_peer_mdev(mdev, peer_mdev, i) {</div><div class='add'>+				esw = peer_mdev-&gt;priv.eswitch;</div><div class='add'>+				mlx5_eswitch_unregister_vport_reps(esw, REP_IB);</div><div class='add'>+			}</div><div class='add'>+			mlx5_ib_release_transport(mdev);</div><div class='add'>+		}</div><div class='add'>+		__mlx5_ib_remove(dev, dev-&gt;profile, MLX5_IB_STAGE_MAX);</div><div class='add'>+	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct mlx5_ib_dev *mlx5_ib_get_rep_ibdev(struct mlx5_eswitch *esw,</div><div class='del'>-					  int vport_index)</div><div class='del'>-{</div><div class='del'>-	return mlx5_eswitch_get_proto_dev(esw, vport_index, REP_IB);</div><div class='del'>-}</div><div class='add'>+static const struct mlx5_eswitch_rep_ops rep_ops = {</div><div class='add'>+	.load = mlx5_ib_vport_rep_load,</div><div class='add'>+	.unload = mlx5_ib_vport_rep_unload,</div><div class='add'>+	.get_proto_dev = mlx5_ib_rep_to_dev,</div><div class='add'>+};</div><div class='ctx'> </div><div class='del'>-struct net_device *mlx5_ib_get_rep_netdev(struct mlx5_eswitch *esw,</div><div class='del'>-					  int vport_index)</div><div class='add'>+static void mlx5_ib_register_peer_vport_reps(struct mlx5_core_dev *mdev)</div><div class='ctx'> {</div><div class='del'>-	return mlx5_eswitch_get_proto_dev(esw, vport_index, REP_ETH);</div><div class='del'>-}</div><div class='add'>+	struct mlx5_core_dev *peer_mdev;</div><div class='add'>+	struct mlx5_eswitch *esw;</div><div class='add'>+	int i;</div><div class='ctx'> </div><div class='del'>-struct mlx5_ib_dev *mlx5_ib_get_uplink_ibdev(struct mlx5_eswitch *esw)</div><div class='del'>-{</div><div class='del'>-	return mlx5_eswitch_uplink_get_proto_dev(esw, REP_IB);</div><div class='add'>+	mlx5_lag_for_each_peer_mdev(mdev, peer_mdev, i) {</div><div class='add'>+		esw = peer_mdev-&gt;priv.eswitch;</div><div class='add'>+		mlx5_eswitch_register_vport_reps(esw, &amp;rep_ops, REP_IB);</div><div class='add'>+	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct mlx5_eswitch_rep *mlx5_ib_vport_rep(struct mlx5_eswitch *esw, int vport)</div><div class='add'>+struct net_device *mlx5_ib_get_rep_netdev(struct mlx5_eswitch *esw,</div><div class='add'>+					  u16 vport_num)</div><div class='ctx'> {</div><div class='del'>-	return mlx5_eswitch_vport_rep(esw, vport);</div><div class='add'>+	return mlx5_eswitch_get_proto_dev(esw, vport_num, REP_ETH);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct mlx5_flow_handle *create_flow_rule_vport_sq(struct mlx5_ib_dev *dev,</div><div class='ctx'> 						   struct mlx5_ib_sq *sq,</div><div class='del'>-						   u16 port)</div><div class='add'>+						   u32 port)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_eswitch *esw = dev-&gt;mdev-&gt;priv.eswitch;</div><div class='ctx'> 	struct mlx5_eswitch_rep *rep;</div><div class='hunk'>@@ -146,6 +296,51 @@ struct mlx5_flow_handle *create_flow_rule_vport_sq(struct mlx5_ib_dev *dev,</div><div class='ctx'> </div><div class='ctx'> 	rep = dev-&gt;port[port - 1].rep;</div><div class='ctx'> </div><div class='del'>-	return mlx5_eswitch_add_send_to_vport_rule(esw, rep-&gt;vport,</div><div class='del'>-						   sq-&gt;base.mqp.qpn);</div><div class='add'>+	return mlx5_eswitch_add_send_to_vport_rule(esw, esw, rep, sq-&gt;base.mqp.qpn);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5r_rep_probe(struct auxiliary_device *adev,</div><div class='add'>+			   const struct auxiliary_device_id *id)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_adev *idev = container_of(adev, struct mlx5_adev, adev);</div><div class='add'>+	struct mlx5_core_dev *mdev = idev-&gt;mdev;</div><div class='add'>+	struct mlx5_eswitch *esw;</div><div class='add'>+</div><div class='add'>+	esw = mdev-&gt;priv.eswitch;</div><div class='add'>+	mlx5_eswitch_register_vport_reps(esw, &amp;rep_ops, REP_IB);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5r_rep_remove(struct auxiliary_device *adev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_adev *idev = container_of(adev, struct mlx5_adev, adev);</div><div class='add'>+	struct mlx5_core_dev *mdev = idev-&gt;mdev;</div><div class='add'>+	struct mlx5_eswitch *esw;</div><div class='add'>+</div><div class='add'>+	esw = mdev-&gt;priv.eswitch;</div><div class='add'>+	mlx5_eswitch_unregister_vport_reps(esw, REP_IB);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct auxiliary_device_id mlx5r_rep_id_table[] = {</div><div class='add'>+	{ .name = MLX5_ADEV_NAME ".rdma-rep", },</div><div class='add'>+	{},</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+MODULE_DEVICE_TABLE(auxiliary, mlx5r_rep_id_table);</div><div class='add'>+</div><div class='add'>+static struct auxiliary_driver mlx5r_rep_driver = {</div><div class='add'>+	.name = "rep",</div><div class='add'>+	.probe = mlx5r_rep_probe,</div><div class='add'>+	.remove = mlx5r_rep_remove,</div><div class='add'>+	.id_table = mlx5r_rep_id_table,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+int mlx5r_rep_init(void)</div><div class='add'>+{</div><div class='add'>+	return auxiliary_driver_register(&amp;mlx5r_rep_driver);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5r_rep_cleanup(void)</div><div class='add'>+{</div><div class='add'>+	auxiliary_driver_unregister(&amp;mlx5r_rep_driver);</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/ib_rep.h b/drivers/infiniband/hw/mlx5/ib_rep.h<br/>index 22adce2d6795..9c55e5c528b4 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/ib_rep.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/ib_rep.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/ib_rep.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/ib_rep.h</a></div><div class='hunk'>@@ -9,69 +9,32 @@</div><div class='ctx'> #include &lt;linux/mlx5/eswitch.h&gt;</div><div class='ctx'> #include "mlx5_ib.h"</div><div class='ctx'> </div><div class='del'>-#ifdef CONFIG_MLX5_ESWITCH</div><div class='del'>-extern const struct mlx5_ib_profile uplink_rep_profile;</div><div class='add'>+extern const struct mlx5_ib_profile raw_eth_profile;</div><div class='ctx'> </div><div class='del'>-u8 mlx5_ib_eswitch_mode(struct mlx5_eswitch *esw);</div><div class='del'>-struct mlx5_ib_dev *mlx5_ib_get_rep_ibdev(struct mlx5_eswitch *esw,</div><div class='del'>-					  int vport_index);</div><div class='del'>-struct mlx5_ib_dev *mlx5_ib_get_uplink_ibdev(struct mlx5_eswitch *esw);</div><div class='del'>-struct mlx5_eswitch_rep *mlx5_ib_vport_rep(struct mlx5_eswitch *esw,</div><div class='del'>-					   int vport_index);</div><div class='del'>-void mlx5_ib_register_vport_reps(struct mlx5_core_dev *mdev);</div><div class='del'>-void mlx5_ib_unregister_vport_reps(struct mlx5_core_dev *mdev);</div><div class='add'>+#ifdef CONFIG_MLX5_ESWITCH</div><div class='add'>+int mlx5r_rep_init(void);</div><div class='add'>+void mlx5r_rep_cleanup(void);</div><div class='ctx'> struct mlx5_flow_handle *create_flow_rule_vport_sq(struct mlx5_ib_dev *dev,</div><div class='ctx'> 						   struct mlx5_ib_sq *sq,</div><div class='del'>-						   u16 port);</div><div class='add'>+						   u32 port);</div><div class='ctx'> struct net_device *mlx5_ib_get_rep_netdev(struct mlx5_eswitch *esw,</div><div class='del'>-					  int vport_index);</div><div class='add'>+					  u16 vport_num);</div><div class='ctx'> #else /* CONFIG_MLX5_ESWITCH */</div><div class='del'>-static inline u8 mlx5_ib_eswitch_mode(struct mlx5_eswitch *esw)</div><div class='del'>-{</div><div class='del'>-	return SRIOV_NONE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline</div><div class='del'>-struct mlx5_ib_dev *mlx5_ib_get_rep_ibdev(struct mlx5_eswitch *esw,</div><div class='del'>-					  int vport_index)</div><div class='del'>-{</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline</div><div class='del'>-struct mlx5_ib_dev *mlx5_ib_get_uplink_ibdev(struct mlx5_eswitch *esw)</div><div class='del'>-{</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline</div><div class='del'>-struct mlx5_eswitch_rep *mlx5_ib_vport_rep(struct mlx5_eswitch *esw,</div><div class='del'>-					   int vport_index)</div><div class='del'>-{</div><div class='del'>-	return NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void mlx5_ib_register_vport_reps(struct mlx5_core_dev *mdev) {}</div><div class='del'>-static inline void mlx5_ib_unregister_vport_reps(struct mlx5_core_dev *mdev) {}</div><div class='add'>+static inline int mlx5r_rep_init(void) { return 0; }</div><div class='add'>+static inline void mlx5r_rep_cleanup(void) {}</div><div class='ctx'> static inline</div><div class='ctx'> struct mlx5_flow_handle *create_flow_rule_vport_sq(struct mlx5_ib_dev *dev,</div><div class='ctx'> 						   struct mlx5_ib_sq *sq,</div><div class='del'>-						   u16 port)</div><div class='add'>+						   u32 port)</div><div class='ctx'> {</div><div class='ctx'> 	return NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static inline</div><div class='ctx'> struct net_device *mlx5_ib_get_rep_netdev(struct mlx5_eswitch *esw,</div><div class='del'>-					  int vport_index)</div><div class='add'>+					  u16 vport_num)</div><div class='ctx'> {</div><div class='ctx'> 	return NULL;</div><div class='ctx'> }</div><div class='ctx'> #endif</div><div class='del'>-</div><div class='del'>-static inline</div><div class='del'>-struct mlx5_ib_dev *mlx5_ib_rep_to_dev(struct mlx5_eswitch_rep *rep)</div><div class='del'>-{</div><div class='del'>-	return rep-&gt;rep_data[REP_IB].priv;</div><div class='del'>-}</div><div class='ctx'> #endif /* __MLX5_IB_REP_H__ */</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/ib_virt.c b/drivers/infiniband/hw/mlx5/ib_virt.c<br/>index 649a3364f838..afeb5e53254f 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/ib_virt.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/ib_virt.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/ib_virt.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/ib_virt.c</a></div><div class='hunk'>@@ -30,7 +30,6 @@</div><div class='ctx'>  * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='ctx'> #include &lt;linux/mlx5/vport.h&gt;</div><div class='ctx'> #include "mlx5_ib.h"</div><div class='ctx'> </div><div class='hunk'>@@ -48,7 +47,7 @@ static inline u32 mlx_to_net_policy(enum port_state_policy mlx_policy)</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_ib_get_vf_config(struct ib_device *device, int vf, u8 port,</div><div class='add'>+int mlx5_ib_get_vf_config(struct ib_device *device, int vf, u32 port,</div><div class='ctx'> 			  struct ifla_vf_info *info)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='hunk'>@@ -91,7 +90,7 @@ static inline enum port_state_policy net_to_mlx_policy(int policy)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_set_vf_link_state(struct ib_device *device, int vf,</div><div class='del'>-			      u8 port, int state)</div><div class='add'>+			      u32 port, int state)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='ctx'> 	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='hunk'>@@ -119,7 +118,7 @@ out:</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_get_vf_stats(struct ib_device *device, int vf,</div><div class='del'>-			 u8 port, struct ifla_vf_stats *stats)</div><div class='add'>+			 u32 port, struct ifla_vf_stats *stats)</div><div class='ctx'> {</div><div class='ctx'> 	int out_sz = MLX5_ST_SZ_BYTES(query_vport_counter_out);</div><div class='ctx'> 	struct mlx5_core_dev *mdev;</div><div class='hunk'>@@ -134,7 +133,7 @@ int mlx5_ib_get_vf_stats(struct ib_device *device, int vf,</div><div class='ctx'> 	if (!out)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	err = mlx5_core_query_vport_counter(mdev, true, vf, port, out, out_sz);</div><div class='add'>+	err = mlx5_core_query_vport_counter(mdev, true, vf, port, out);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto ex;</div><div class='ctx'> </div><div class='hunk'>@@ -149,7 +148,8 @@ ex:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int set_vf_node_guid(struct ib_device *device, int vf, u8 port, u64 guid)</div><div class='add'>+static int set_vf_node_guid(struct ib_device *device, int vf, u32 port,</div><div class='add'>+			    u64 guid)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='ctx'> 	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='hunk'>@@ -164,13 +164,16 @@ static int set_vf_node_guid(struct ib_device *device, int vf, u8 port, u64 guid)</div><div class='ctx'> 	in-&gt;field_select = MLX5_HCA_VPORT_SEL_NODE_GUID;</div><div class='ctx'> 	in-&gt;node_guid = guid;</div><div class='ctx'> 	err = mlx5_core_modify_hca_vport_context(mdev, 1, 1, vf + 1, in);</div><div class='del'>-	if (!err)</div><div class='add'>+	if (!err) {</div><div class='ctx'> 		vfs_ctx[vf].node_guid = guid;</div><div class='add'>+		vfs_ctx[vf].node_guid_valid = 1;</div><div class='add'>+	}</div><div class='ctx'> 	kfree(in);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int set_vf_port_guid(struct ib_device *device, int vf, u8 port, u64 guid)</div><div class='add'>+static int set_vf_port_guid(struct ib_device *device, int vf, u32 port,</div><div class='add'>+			    u64 guid)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='ctx'> 	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='hunk'>@@ -185,13 +188,15 @@ static int set_vf_port_guid(struct ib_device *device, int vf, u8 port, u64 guid)</div><div class='ctx'> 	in-&gt;field_select = MLX5_HCA_VPORT_SEL_PORT_GUID;</div><div class='ctx'> 	in-&gt;port_guid = guid;</div><div class='ctx'> 	err = mlx5_core_modify_hca_vport_context(mdev, 1, 1, vf + 1, in);</div><div class='del'>-	if (!err)</div><div class='add'>+	if (!err) {</div><div class='ctx'> 		vfs_ctx[vf].port_guid = guid;</div><div class='add'>+		vfs_ctx[vf].port_guid_valid = 1;</div><div class='add'>+	}</div><div class='ctx'> 	kfree(in);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u8 port,</div><div class='add'>+int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u32 port,</div><div class='ctx'> 			u64 guid, int type)</div><div class='ctx'> {</div><div class='ctx'> 	if (type == IFLA_VF_IB_NODE_GUID)</div><div class='hunk'>@@ -201,3 +206,19 @@ int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u8 port,</div><div class='ctx'> </div><div class='ctx'> 	return -EINVAL;</div><div class='ctx'> }</div><div class='add'>+</div><div class='add'>+int mlx5_ib_get_vf_guid(struct ib_device *device, int vf, u32 port,</div><div class='add'>+			struct ifla_vf_guid *node_guid,</div><div class='add'>+			struct ifla_vf_guid *port_guid)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='add'>+	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='add'>+	struct mlx5_vf_context *vfs_ctx = mdev-&gt;priv.sriov.vfs_ctx;</div><div class='add'>+</div><div class='add'>+	node_guid-&gt;guid =</div><div class='add'>+		vfs_ctx[vf].node_guid_valid ? vfs_ctx[vf].node_guid : 0;</div><div class='add'>+	port_guid-&gt;guid =</div><div class='add'>+		vfs_ctx[vf].port_guid_valid ? vfs_ctx[vf].port_guid : 0;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/macsec.c b/drivers/infiniband/hw/mlx5/macsec.c<br/>new file mode 100644<br/>index 000000000000..3c56eb5eddf3<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/macsec.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/macsec.c</a></div><div class='hunk'>@@ -0,0 +1,364 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/* Copyright (c) 2022, NVIDIA CORPORATION &amp; AFFILIATES. */</div><div class='add'>+</div><div class='add'>+#include "macsec.h"</div><div class='add'>+#include &lt;linux/mlx5/macsec.h&gt;</div><div class='add'>+</div><div class='add'>+struct mlx5_reserved_gids {</div><div class='add'>+	int macsec_index;</div><div class='add'>+	const struct ib_gid_attr *physical_gid;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mlx5_roce_gids {</div><div class='add'>+	struct list_head roce_gid_list_entry;</div><div class='add'>+	u16 gid_idx;</div><div class='add'>+	union {</div><div class='add'>+		struct sockaddr_in  sockaddr_in;</div><div class='add'>+		struct sockaddr_in6 sockaddr_in6;</div><div class='add'>+	} addr;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mlx5_macsec_device {</div><div class='add'>+	struct list_head macsec_devices_list_entry;</div><div class='add'>+	void *macdev;</div><div class='add'>+	struct list_head macsec_roce_gids;</div><div class='add'>+	struct list_head tx_rules_list;</div><div class='add'>+	struct list_head rx_rules_list;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static void cleanup_macsec_device(struct mlx5_macsec_device *macsec_device)</div><div class='add'>+{</div><div class='add'>+	if (!list_empty(&amp;macsec_device-&gt;tx_rules_list) ||</div><div class='add'>+	    !list_empty(&amp;macsec_device-&gt;rx_rules_list) ||</div><div class='add'>+	    !list_empty(&amp;macsec_device-&gt;macsec_roce_gids))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	list_del(&amp;macsec_device-&gt;macsec_devices_list_entry);</div><div class='add'>+	kfree(macsec_device);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct mlx5_macsec_device *get_macsec_device(void *macdev,</div><div class='add'>+						    struct list_head *macsec_devices_list)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_macsec_device *iter, *macsec_device = NULL;</div><div class='add'>+</div><div class='add'>+	list_for_each_entry(iter, macsec_devices_list, macsec_devices_list_entry) {</div><div class='add'>+		if (iter-&gt;macdev == macdev) {</div><div class='add'>+			macsec_device = iter;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (macsec_device)</div><div class='add'>+		return macsec_device;</div><div class='add'>+</div><div class='add'>+	macsec_device = kzalloc(sizeof(*macsec_device), GFP_KERNEL);</div><div class='add'>+	if (!macsec_device)</div><div class='add'>+		return NULL;</div><div class='add'>+</div><div class='add'>+	macsec_device-&gt;macdev = macdev;</div><div class='add'>+	INIT_LIST_HEAD(&amp;macsec_device-&gt;tx_rules_list);</div><div class='add'>+	INIT_LIST_HEAD(&amp;macsec_device-&gt;rx_rules_list);</div><div class='add'>+	INIT_LIST_HEAD(&amp;macsec_device-&gt;macsec_roce_gids);</div><div class='add'>+	list_add(&amp;macsec_device-&gt;macsec_devices_list_entry, macsec_devices_list);</div><div class='add'>+</div><div class='add'>+	return macsec_device;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_macsec_del_roce_gid(struct mlx5_macsec_device *macsec_device, u16 gid_idx)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_roce_gids *current_gid, *next_gid;</div><div class='add'>+</div><div class='add'>+	list_for_each_entry_safe(current_gid, next_gid, &amp;macsec_device-&gt;macsec_roce_gids,</div><div class='add'>+				 roce_gid_list_entry)</div><div class='add'>+		if (current_gid-&gt;gid_idx == gid_idx) {</div><div class='add'>+			list_del(&amp;current_gid-&gt;roce_gid_list_entry);</div><div class='add'>+			kfree(current_gid);</div><div class='add'>+		}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_macsec_save_roce_gid(struct mlx5_macsec_device *macsec_device,</div><div class='add'>+				      const struct sockaddr *addr, u16 gid_idx)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_roce_gids *roce_gids;</div><div class='add'>+</div><div class='add'>+	roce_gids = kzalloc(sizeof(*roce_gids), GFP_KERNEL);</div><div class='add'>+	if (!roce_gids)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	roce_gids-&gt;gid_idx = gid_idx;</div><div class='add'>+	if (addr-&gt;sa_family == AF_INET)</div><div class='add'>+		memcpy(&amp;roce_gids-&gt;addr.sockaddr_in, addr, sizeof(roce_gids-&gt;addr.sockaddr_in));</div><div class='add'>+	else</div><div class='add'>+		memcpy(&amp;roce_gids-&gt;addr.sockaddr_in6, addr, sizeof(roce_gids-&gt;addr.sockaddr_in6));</div><div class='add'>+</div><div class='add'>+	list_add_tail(&amp;roce_gids-&gt;roce_gid_list_entry, &amp;macsec_device-&gt;macsec_roce_gids);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void handle_macsec_gids(struct list_head *macsec_devices_list,</div><div class='add'>+			       struct mlx5_macsec_event_data *data)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_macsec_device *macsec_device;</div><div class='add'>+	struct mlx5_roce_gids *gid;</div><div class='add'>+</div><div class='add'>+	macsec_device = get_macsec_device(data-&gt;macdev, macsec_devices_list);</div><div class='add'>+	if (!macsec_device)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	list_for_each_entry(gid, &amp;macsec_device-&gt;macsec_roce_gids, roce_gid_list_entry) {</div><div class='add'>+		mlx5_macsec_add_roce_sa_rules(data-&gt;fs_id, (struct sockaddr *)&amp;gid-&gt;addr,</div><div class='add'>+					      gid-&gt;gid_idx, &amp;macsec_device-&gt;tx_rules_list,</div><div class='add'>+					      &amp;macsec_device-&gt;rx_rules_list, data-&gt;macsec_fs,</div><div class='add'>+					      data-&gt;is_tx);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void del_sa_roce_rule(struct list_head *macsec_devices_list,</div><div class='add'>+			     struct mlx5_macsec_event_data *data)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_macsec_device *macsec_device;</div><div class='add'>+</div><div class='add'>+	macsec_device = get_macsec_device(data-&gt;macdev, macsec_devices_list);</div><div class='add'>+	WARN_ON(!macsec_device);</div><div class='add'>+</div><div class='add'>+	mlx5_macsec_del_roce_sa_rules(data-&gt;fs_id, data-&gt;macsec_fs,</div><div class='add'>+				      &amp;macsec_device-&gt;tx_rules_list,</div><div class='add'>+				      &amp;macsec_device-&gt;rx_rules_list, data-&gt;is_tx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int macsec_event(struct notifier_block *nb, unsigned long event, void *data)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_macsec *macsec = container_of(nb, struct mlx5_macsec, blocking_events_nb);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;macsec-&gt;lock);</div><div class='add'>+	switch (event) {</div><div class='add'>+	case MLX5_DRIVER_EVENT_MACSEC_SA_ADDED:</div><div class='add'>+		handle_macsec_gids(&amp;macsec-&gt;macsec_devices_list, data);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_DRIVER_EVENT_MACSEC_SA_DELETED:</div><div class='add'>+		del_sa_roce_rule(&amp;macsec-&gt;macsec_devices_list, data);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		mutex_unlock(&amp;macsec-&gt;lock);</div><div class='add'>+		return NOTIFY_DONE;</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;macsec-&gt;lock);</div><div class='add'>+	return NOTIFY_OK;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5r_macsec_event_register(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	if (!mlx5_is_macsec_roce_supported(dev-&gt;mdev)) {</div><div class='add'>+		mlx5_ib_dbg(dev, "RoCE MACsec not supported due to capabilities\n");</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dev-&gt;macsec.blocking_events_nb.notifier_call = macsec_event;</div><div class='add'>+	blocking_notifier_chain_register(&amp;dev-&gt;mdev-&gt;macsec_nh,</div><div class='add'>+					 &amp;dev-&gt;macsec.blocking_events_nb);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5r_macsec_event_unregister(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	if (!mlx5_is_macsec_roce_supported(dev-&gt;mdev)) {</div><div class='add'>+		mlx5_ib_dbg(dev, "RoCE MACsec not supported due to capabilities\n");</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	blocking_notifier_chain_unregister(&amp;dev-&gt;mdev-&gt;macsec_nh,</div><div class='add'>+					   &amp;dev-&gt;macsec.blocking_events_nb);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5r_macsec_init_gids_and_devlist(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int i, j, max_gids;</div><div class='add'>+</div><div class='add'>+	if (!mlx5_is_macsec_roce_supported(dev-&gt;mdev)) {</div><div class='add'>+		mlx5_ib_dbg(dev, "RoCE MACsec not supported due to capabilities\n");</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	max_gids = MLX5_CAP_ROCE(dev-&gt;mdev, roce_address_table_size);</div><div class='add'>+	for (i = 0; i &lt; dev-&gt;num_ports; i++) {</div><div class='add'>+		dev-&gt;port[i].reserved_gids = kcalloc(max_gids,</div><div class='add'>+						     sizeof(*dev-&gt;port[i].reserved_gids),</div><div class='add'>+						     GFP_KERNEL);</div><div class='add'>+		if (!dev-&gt;port[i].reserved_gids)</div><div class='add'>+			goto err;</div><div class='add'>+</div><div class='add'>+		for (j = 0; j &lt; max_gids; j++)</div><div class='add'>+			dev-&gt;port[i].reserved_gids[j].macsec_index = -1;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;dev-&gt;macsec.macsec_devices_list);</div><div class='add'>+	mutex_init(&amp;dev-&gt;macsec.lock);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+err:</div><div class='add'>+	while (i &gt;= 0) {</div><div class='add'>+		kfree(dev-&gt;port[i].reserved_gids);</div><div class='add'>+		i--;</div><div class='add'>+	}</div><div class='add'>+	return -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5r_macsec_dealloc_gids(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	if (!mlx5_is_macsec_roce_supported(dev-&gt;mdev))</div><div class='add'>+		mlx5_ib_dbg(dev, "RoCE MACsec not supported due to capabilities\n");</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; dev-&gt;num_ports; i++)</div><div class='add'>+		kfree(dev-&gt;port[i].reserved_gids);</div><div class='add'>+</div><div class='add'>+	mutex_destroy(&amp;dev-&gt;macsec.lock);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5r_add_gid_macsec_operations(const struct ib_gid_attr *attr)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(attr-&gt;device);</div><div class='add'>+	struct mlx5_macsec_device *macsec_device;</div><div class='add'>+	const struct ib_gid_attr *physical_gid;</div><div class='add'>+	struct mlx5_reserved_gids *mgids;</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+	int ret = 0;</div><div class='add'>+	union {</div><div class='add'>+		struct sockaddr_in  sockaddr_in;</div><div class='add'>+		struct sockaddr_in6 sockaddr_in6;</div><div class='add'>+	} addr;</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (!mlx5_is_macsec_roce_supported(dev-&gt;mdev)) {</div><div class='add'>+		mlx5_ib_dbg(dev, "RoCE MACsec not supported due to capabilities\n");</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	ndev = rcu_dereference(attr-&gt;ndev);</div><div class='add'>+	if (!ndev) {</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+		return -ENODEV;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!netif_is_macsec(ndev) || !macsec_netdev_is_offloaded(ndev)) {</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+	dev_hold(ndev);</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;dev-&gt;macsec.lock);</div><div class='add'>+	macsec_device = get_macsec_device(ndev, &amp;dev-&gt;macsec.macsec_devices_list);</div><div class='add'>+	if (!macsec_device) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto dev_err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	physical_gid = rdma_find_gid(attr-&gt;device, &amp;attr-&gt;gid,</div><div class='add'>+				     attr-&gt;gid_type, NULL);</div><div class='add'>+	if (!IS_ERR(physical_gid)) {</div><div class='add'>+		ret = set_roce_addr(to_mdev(physical_gid-&gt;device),</div><div class='add'>+				    physical_gid-&gt;port_num,</div><div class='add'>+				    physical_gid-&gt;index, NULL,</div><div class='add'>+				    physical_gid);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto gid_err;</div><div class='add'>+</div><div class='add'>+		mgids = &amp;dev-&gt;port[attr-&gt;port_num - 1].reserved_gids[physical_gid-&gt;index];</div><div class='add'>+		mgids-&gt;macsec_index = attr-&gt;index;</div><div class='add'>+		mgids-&gt;physical_gid = physical_gid;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Proceed with adding steering rules, regardless if there was gid ambiguity or not.*/</div><div class='add'>+	rdma_gid2ip((struct sockaddr *)&amp;addr, &amp;attr-&gt;gid);</div><div class='add'>+	ret = mlx5_macsec_add_roce_rule(ndev, (struct sockaddr *)&amp;addr, attr-&gt;index,</div><div class='add'>+					&amp;macsec_device-&gt;tx_rules_list,</div><div class='add'>+					&amp;macsec_device-&gt;rx_rules_list, dev-&gt;mdev-&gt;macsec_fs);</div><div class='add'>+	if (ret &amp;&amp; !IS_ERR(physical_gid))</div><div class='add'>+		goto rule_err;</div><div class='add'>+</div><div class='add'>+	mlx5_macsec_save_roce_gid(macsec_device, (struct sockaddr *)&amp;addr, attr-&gt;index);</div><div class='add'>+</div><div class='add'>+	dev_put(ndev);</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;macsec.lock);</div><div class='add'>+	return ret;</div><div class='add'>+</div><div class='add'>+rule_err:</div><div class='add'>+	set_roce_addr(to_mdev(physical_gid-&gt;device), physical_gid-&gt;port_num,</div><div class='add'>+		      physical_gid-&gt;index, &amp;physical_gid-&gt;gid, physical_gid);</div><div class='add'>+	mgids-&gt;macsec_index = -1;</div><div class='add'>+gid_err:</div><div class='add'>+	rdma_put_gid_attr(physical_gid);</div><div class='add'>+	cleanup_macsec_device(macsec_device);</div><div class='add'>+dev_err:</div><div class='add'>+	dev_put(ndev);</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;macsec.lock);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5r_del_gid_macsec_operations(const struct ib_gid_attr *attr)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(attr-&gt;device);</div><div class='add'>+	struct mlx5_macsec_device *macsec_device;</div><div class='add'>+	struct mlx5_reserved_gids *mgids;</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+	int i, max_gids;</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	if (!mlx5_is_macsec_roce_supported(dev-&gt;mdev)) {</div><div class='add'>+		mlx5_ib_dbg(dev, "RoCE MACsec not supported due to capabilities\n");</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mgids = &amp;dev-&gt;port[attr-&gt;port_num - 1].reserved_gids[attr-&gt;index];</div><div class='add'>+	if (mgids-&gt;macsec_index != -1) { /* Checking if physical gid has ambiguous IP */</div><div class='add'>+		rdma_put_gid_attr(mgids-&gt;physical_gid);</div><div class='add'>+		mgids-&gt;macsec_index = -1;</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	rcu_read_lock();</div><div class='add'>+	ndev = rcu_dereference(attr-&gt;ndev);</div><div class='add'>+	if (!ndev) {</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!netif_is_macsec(ndev) || !macsec_netdev_is_offloaded(ndev)) {</div><div class='add'>+		rcu_read_unlock();</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+	dev_hold(ndev);</div><div class='add'>+	rcu_read_unlock();</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;dev-&gt;macsec.lock);</div><div class='add'>+	max_gids = MLX5_CAP_ROCE(dev-&gt;mdev, roce_address_table_size);</div><div class='add'>+	for (i = 0; i &lt; max_gids; i++) { /* Checking if macsec gid has ambiguous IP */</div><div class='add'>+		mgids = &amp;dev-&gt;port[attr-&gt;port_num - 1].reserved_gids[i];</div><div class='add'>+		if (mgids-&gt;macsec_index == attr-&gt;index) {</div><div class='add'>+			const struct ib_gid_attr *physical_gid = mgids-&gt;physical_gid;</div><div class='add'>+</div><div class='add'>+			set_roce_addr(to_mdev(physical_gid-&gt;device),</div><div class='add'>+				      physical_gid-&gt;port_num,</div><div class='add'>+				      physical_gid-&gt;index,</div><div class='add'>+				      &amp;physical_gid-&gt;gid, physical_gid);</div><div class='add'>+</div><div class='add'>+			rdma_put_gid_attr(physical_gid);</div><div class='add'>+			mgids-&gt;macsec_index = -1;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	macsec_device = get_macsec_device(ndev, &amp;dev-&gt;macsec.macsec_devices_list);</div><div class='add'>+	mlx5_macsec_del_roce_rule(attr-&gt;index, dev-&gt;mdev-&gt;macsec_fs,</div><div class='add'>+				  &amp;macsec_device-&gt;tx_rules_list, &amp;macsec_device-&gt;rx_rules_list);</div><div class='add'>+	mlx5_macsec_del_roce_gid(macsec_device, attr-&gt;index);</div><div class='add'>+	cleanup_macsec_device(macsec_device);</div><div class='add'>+</div><div class='add'>+	dev_put(ndev);</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;macsec.lock);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/macsec.h b/drivers/infiniband/hw/mlx5/macsec.h<br/>new file mode 100644<br/>index 000000000000..9b77ba90f0f4<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/macsec.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/macsec.h</a></div><div class='hunk'>@@ -0,0 +1,29 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/* Copyright (c) 2022, NVIDIA CORPORATION &amp; AFFILIATES. */</div><div class='add'>+</div><div class='add'>+#ifndef __MLX5_MACSEC_H__</div><div class='add'>+#define __MLX5_MACSEC_H__</div><div class='add'>+</div><div class='add'>+#include &lt;net/macsec.h&gt;</div><div class='add'>+#include &lt;rdma/ib_cache.h&gt;</div><div class='add'>+#include &lt;rdma/ib_addr.h&gt;</div><div class='add'>+#include "mlx5_ib.h"</div><div class='add'>+</div><div class='add'>+#ifdef CONFIG_MLX5_MACSEC</div><div class='add'>+struct mlx5_reserved_gids;</div><div class='add'>+</div><div class='add'>+int mlx5r_add_gid_macsec_operations(const struct ib_gid_attr *attr);</div><div class='add'>+void mlx5r_del_gid_macsec_operations(const struct ib_gid_attr *attr);</div><div class='add'>+int mlx5r_macsec_init_gids_and_devlist(struct mlx5_ib_dev *dev);</div><div class='add'>+void mlx5r_macsec_dealloc_gids(struct mlx5_ib_dev *dev);</div><div class='add'>+void mlx5r_macsec_event_register(struct mlx5_ib_dev *dev);</div><div class='add'>+void mlx5r_macsec_event_unregister(struct mlx5_ib_dev *dev);</div><div class='add'>+#else</div><div class='add'>+static inline int mlx5r_add_gid_macsec_operations(const struct ib_gid_attr *attr) { return 0; }</div><div class='add'>+static inline void mlx5r_del_gid_macsec_operations(const struct ib_gid_attr *attr) {}</div><div class='add'>+static inline int mlx5r_macsec_init_gids_and_devlist(struct mlx5_ib_dev *dev) { return 0; }</div><div class='add'>+static inline void mlx5r_macsec_dealloc_gids(struct mlx5_ib_dev *dev) {}</div><div class='add'>+static inline void mlx5r_macsec_event_register(struct mlx5_ib_dev *dev) {}</div><div class='add'>+static inline void mlx5r_macsec_event_unregister(struct mlx5_ib_dev *dev) {}</div><div class='add'>+#endif</div><div class='add'>+#endif /* __MLX5_MACSEC_H__ */</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/mad.c b/drivers/infiniband/hw/mlx5/mad.c<br/>index 6c529e6f3a01..2453ae4384a7 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/mad.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/mad.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/mad.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/mad.c</a></div><div class='hunk'>@@ -30,7 +30,6 @@</div><div class='ctx'>  * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/mlx5/cmd.h&gt;</div><div class='ctx'> #include &lt;linux/mlx5/vport.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_mad.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_smi.h&gt;</div><div class='hunk'>@@ -43,17 +42,17 @@ enum {</div><div class='ctx'> 	MLX5_IB_VENDOR_CLASS2 = 0xa</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static bool can_do_mad_ifc(struct mlx5_ib_dev *dev, u8 port_num,</div><div class='add'>+static bool can_do_mad_ifc(struct mlx5_ib_dev *dev, u32 port_num,</div><div class='ctx'> 			   struct ib_mad *in_mad)</div><div class='ctx'> {</div><div class='ctx'> 	if (in_mad-&gt;mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED &amp;&amp;</div><div class='ctx'> 	    in_mad-&gt;mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)</div><div class='ctx'> 		return true;</div><div class='del'>-	return dev-&gt;mdev-&gt;port_caps[port_num - 1].has_smi;</div><div class='add'>+	return dev-&gt;port_caps[port_num - 1].has_smi;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey,</div><div class='del'>-			int ignore_bkey, u8 port, const struct ib_wc *in_wc,</div><div class='add'>+			int ignore_bkey, u32 port, const struct ib_wc *in_wc,</div><div class='ctx'> 			const struct ib_grh *in_grh, const void *in_mad,</div><div class='ctx'> 			void *response_mad)</div><div class='ctx'> {</div><div class='hunk'>@@ -70,62 +69,10 @@ static int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey,</div><div class='ctx'> 	if (ignore_bkey || !in_wc)</div><div class='ctx'> 		op_modifier |= 0x2;</div><div class='ctx'> </div><div class='del'>-	return mlx5_cmd_mad_ifc(dev-&gt;mdev, in_mad, response_mad, op_modifier,</div><div class='add'>+	return mlx5_cmd_mad_ifc(dev, in_mad, response_mad, op_modifier,</div><div class='ctx'> 				port);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,</div><div class='del'>-		       const struct ib_wc *in_wc, const struct ib_grh *in_grh,</div><div class='del'>-		       const struct ib_mad *in_mad, struct ib_mad *out_mad)</div><div class='del'>-{</div><div class='del'>-	u16 slid;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	slid = in_wc ? ib_lid_cpu16(in_wc-&gt;slid) : be16_to_cpu(IB_LID_PERMISSIVE);</div><div class='del'>-</div><div class='del'>-	if (in_mad-&gt;mad_hdr.method == IB_MGMT_METHOD_TRAP &amp;&amp; slid == 0)</div><div class='del'>-		return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;</div><div class='del'>-</div><div class='del'>-	if (in_mad-&gt;mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED ||</div><div class='del'>-	    in_mad-&gt;mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {</div><div class='del'>-		if (in_mad-&gt;mad_hdr.method   != IB_MGMT_METHOD_GET &amp;&amp;</div><div class='del'>-		    in_mad-&gt;mad_hdr.method   != IB_MGMT_METHOD_SET &amp;&amp;</div><div class='del'>-		    in_mad-&gt;mad_hdr.method   != IB_MGMT_METHOD_TRAP_REPRESS)</div><div class='del'>-			return IB_MAD_RESULT_SUCCESS;</div><div class='del'>-</div><div class='del'>-		/* Don't process SMInfo queries -- the SMA can't handle them.</div><div class='del'>-		 */</div><div class='del'>-		if (in_mad-&gt;mad_hdr.attr_id == IB_SMP_ATTR_SM_INFO)</div><div class='del'>-			return IB_MAD_RESULT_SUCCESS;</div><div class='del'>-	} else if (in_mad-&gt;mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT ||</div><div class='del'>-		   in_mad-&gt;mad_hdr.mgmt_class == MLX5_IB_VENDOR_CLASS1   ||</div><div class='del'>-		   in_mad-&gt;mad_hdr.mgmt_class == MLX5_IB_VENDOR_CLASS2   ||</div><div class='del'>-		   in_mad-&gt;mad_hdr.mgmt_class == IB_MGMT_CLASS_CONG_MGMT) {</div><div class='del'>-		if (in_mad-&gt;mad_hdr.method  != IB_MGMT_METHOD_GET &amp;&amp;</div><div class='del'>-		    in_mad-&gt;mad_hdr.method  != IB_MGMT_METHOD_SET)</div><div class='del'>-			return IB_MAD_RESULT_SUCCESS;</div><div class='del'>-	} else {</div><div class='del'>-		return IB_MAD_RESULT_SUCCESS;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	err = mlx5_MAD_IFC(to_mdev(ibdev),</div><div class='del'>-			   mad_flags &amp; IB_MAD_IGNORE_MKEY,</div><div class='del'>-			   mad_flags &amp; IB_MAD_IGNORE_BKEY,</div><div class='del'>-			   port_num, in_wc, in_grh, in_mad, out_mad);</div><div class='del'>-	if (err)</div><div class='del'>-		return IB_MAD_RESULT_FAILURE;</div><div class='del'>-</div><div class='del'>-	/* set return bit in status of directed route responses */</div><div class='del'>-	if (in_mad-&gt;mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)</div><div class='del'>-		out_mad-&gt;mad_hdr.status |= cpu_to_be16(1 &lt;&lt; 15);</div><div class='del'>-</div><div class='del'>-	if (in_mad-&gt;mad_hdr.method == IB_MGMT_METHOD_TRAP_REPRESS)</div><div class='del'>-		/* no response for trap repress */</div><div class='del'>-		return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;</div><div class='del'>-</div><div class='del'>-	return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static void pma_cnt_ext_assign(struct ib_pma_portcounters_ext *pma_cnt_ext,</div><div class='ctx'> 			       void *out)</div><div class='ctx'> {</div><div class='hunk'>@@ -200,11 +147,91 @@ static void pma_cnt_assign(struct ib_pma_portcounters *pma_cnt,</div><div class='ctx'> 			     vl_15_dropped);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int process_pma_cmd(struct mlx5_core_dev *mdev, u8 port_num,</div><div class='del'>-			   const struct ib_mad *in_mad, struct ib_mad *out_mad)</div><div class='add'>+static void pma_cnt_ext_assign_ppcnt(struct ib_pma_portcounters_ext *cnt_ext,</div><div class='add'>+				     void *out)</div><div class='ctx'> {</div><div class='add'>+	void *out_pma = MLX5_ADDR_OF(ppcnt_reg, out,</div><div class='add'>+				     counter_set);</div><div class='add'>+</div><div class='add'>+#define MLX5_GET_EXT_CNTR(counter_name)			\</div><div class='add'>+	MLX5_GET64(ib_ext_port_cntrs_grp_data_layout,	\</div><div class='add'>+		   out_pma, counter_name##_high)</div><div class='add'>+</div><div class='add'>+	cnt_ext-&gt;port_xmit_data =</div><div class='add'>+		cpu_to_be64(MLX5_GET_EXT_CNTR(port_xmit_data) &gt;&gt; 2);</div><div class='add'>+	cnt_ext-&gt;port_rcv_data =</div><div class='add'>+		cpu_to_be64(MLX5_GET_EXT_CNTR(port_rcv_data) &gt;&gt; 2);</div><div class='add'>+</div><div class='add'>+	cnt_ext-&gt;port_xmit_packets =</div><div class='add'>+		cpu_to_be64(MLX5_GET_EXT_CNTR(port_xmit_pkts));</div><div class='add'>+	cnt_ext-&gt;port_rcv_packets =</div><div class='add'>+		cpu_to_be64(MLX5_GET_EXT_CNTR(port_rcv_pkts));</div><div class='add'>+</div><div class='add'>+	cnt_ext-&gt;port_unicast_xmit_packets =</div><div class='add'>+		cpu_to_be64(MLX5_GET_EXT_CNTR(port_unicast_xmit_pkts));</div><div class='add'>+	cnt_ext-&gt;port_unicast_rcv_packets =</div><div class='add'>+		cpu_to_be64(MLX5_GET_EXT_CNTR(port_unicast_rcv_pkts));</div><div class='add'>+</div><div class='add'>+	cnt_ext-&gt;port_multicast_xmit_packets =</div><div class='add'>+		cpu_to_be64(MLX5_GET_EXT_CNTR(port_multicast_xmit_pkts));</div><div class='add'>+	cnt_ext-&gt;port_multicast_rcv_packets =</div><div class='add'>+		cpu_to_be64(MLX5_GET_EXT_CNTR(port_multicast_rcv_pkts));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int query_ib_ppcnt(struct mlx5_core_dev *dev, u8 port_num, u8 plane_num,</div><div class='add'>+			  void *out, size_t sz, bool ext)</div><div class='add'>+{</div><div class='add'>+	u32 *in;</div><div class='ctx'> 	int err;</div><div class='add'>+</div><div class='add'>+	in  = kvzalloc(sz, GFP_KERNEL);</div><div class='add'>+	if (!in) {</div><div class='add'>+		err = -ENOMEM;</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	MLX5_SET(ppcnt_reg, in, local_port, port_num);</div><div class='add'>+	MLX5_SET(ppcnt_reg, in, plane_ind, plane_num);</div><div class='add'>+</div><div class='add'>+	if (ext)</div><div class='add'>+		MLX5_SET(ppcnt_reg, in, grp,</div><div class='add'>+			 MLX5_INFINIBAND_EXTENDED_PORT_COUNTERS_GROUP);</div><div class='add'>+	else</div><div class='add'>+		MLX5_SET(ppcnt_reg, in, grp,</div><div class='add'>+			 MLX5_INFINIBAND_PORT_COUNTERS_GROUP);</div><div class='add'>+	err = mlx5_core_access_reg(dev, in, sz, out,</div><div class='add'>+				   sz, MLX5_REG_PPCNT, 0, 0);</div><div class='add'>+</div><div class='add'>+	kvfree(in);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int process_pma_cmd(struct mlx5_ib_dev *dev, u32 port_num,</div><div class='add'>+			   const struct ib_mad *in_mad, struct ib_mad *out_mad)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_core_dev *mdev;</div><div class='add'>+	bool native_port = true;</div><div class='add'>+	u32 mdev_port_num;</div><div class='ctx'> 	void *out_cnt;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &amp;mdev_port_num);</div><div class='add'>+	if (!mdev) {</div><div class='add'>+		/* Fail to get the native port, likely due to 2nd port is still</div><div class='add'>+		 * unaffiliated. In such case default to 1st port and attached</div><div class='add'>+		 * PF device.</div><div class='add'>+		 */</div><div class='add'>+		native_port = false;</div><div class='add'>+		mdev = dev-&gt;mdev;</div><div class='add'>+		mdev_port_num = 1;</div><div class='add'>+	}</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, num_ports) == 1 &amp;&amp;</div><div class='add'>+	    !mlx5_core_mp_enabled(mdev) &amp;&amp;</div><div class='add'>+	    dev-&gt;ib_dev.type != RDMA_DEVICE_TYPE_SMI) {</div><div class='add'>+		/* set local port to one for Function-Per-Port HCA. */</div><div class='add'>+		mdev = dev-&gt;mdev;</div><div class='add'>+		mdev_port_num = 1;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	/* Declaring support of extended counters */</div><div class='ctx'> 	if (in_mad-&gt;mad_hdr.attr_id == IB_PMA_CLASS_PORT_INFO) {</div><div class='hunk'>@@ -212,83 +239,131 @@ static int process_pma_cmd(struct mlx5_core_dev *mdev, u8 port_num,</div><div class='ctx'> </div><div class='ctx'> 		cpi.capability_mask = IB_PMA_CLASS_CAP_EXT_WIDTH;</div><div class='ctx'> 		memcpy((out_mad-&gt;data + 40), &amp;cpi, sizeof(cpi));</div><div class='del'>-		return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;</div><div class='add'>+		err = IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;</div><div class='add'>+		goto done;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (in_mad-&gt;mad_hdr.attr_id == IB_PMA_PORT_COUNTERS_EXT) {</div><div class='ctx'> 		struct ib_pma_portcounters_ext *pma_cnt_ext =</div><div class='ctx'> 			(struct ib_pma_portcounters_ext *)(out_mad-&gt;data + 40);</div><div class='del'>-		int sz = MLX5_ST_SZ_BYTES(query_vport_counter_out);</div><div class='add'>+		int sz = max(MLX5_ST_SZ_BYTES(query_vport_counter_out),</div><div class='add'>+			     MLX5_ST_SZ_BYTES(ppcnt_reg));</div><div class='ctx'> </div><div class='ctx'> 		out_cnt = kvzalloc(sz, GFP_KERNEL);</div><div class='del'>-		if (!out_cnt)</div><div class='del'>-			return IB_MAD_RESULT_FAILURE;</div><div class='add'>+		if (!out_cnt) {</div><div class='add'>+			err = IB_MAD_RESULT_FAILURE;</div><div class='add'>+			goto done;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='del'>-		err = mlx5_core_query_vport_counter(mdev, 0, 0,</div><div class='del'>-						    port_num, out_cnt, sz);</div><div class='del'>-		if (!err)</div><div class='del'>-			pma_cnt_ext_assign(pma_cnt_ext, out_cnt);</div><div class='add'>+		if (dev-&gt;ib_dev.type == RDMA_DEVICE_TYPE_SMI) {</div><div class='add'>+			err = query_ib_ppcnt(mdev, mdev_port_num,</div><div class='add'>+					     port_num, out_cnt, sz, 1);</div><div class='add'>+			if (!err)</div><div class='add'>+				pma_cnt_ext_assign_ppcnt(pma_cnt_ext, out_cnt);</div><div class='add'>+		} else {</div><div class='add'>+			err = mlx5_core_query_vport_counter(mdev, 0, 0,</div><div class='add'>+							    mdev_port_num,</div><div class='add'>+							    out_cnt);</div><div class='add'>+			if (!err)</div><div class='add'>+				pma_cnt_ext_assign(pma_cnt_ext, out_cnt);</div><div class='add'>+		}</div><div class='ctx'> 	} else {</div><div class='ctx'> 		struct ib_pma_portcounters *pma_cnt =</div><div class='ctx'> 			(struct ib_pma_portcounters *)(out_mad-&gt;data + 40);</div><div class='ctx'> 		int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);</div><div class='ctx'> </div><div class='ctx'> 		out_cnt = kvzalloc(sz, GFP_KERNEL);</div><div class='del'>-		if (!out_cnt)</div><div class='del'>-			return IB_MAD_RESULT_FAILURE;</div><div class='add'>+		if (!out_cnt) {</div><div class='add'>+			err = IB_MAD_RESULT_FAILURE;</div><div class='add'>+			goto done;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		if (dev-&gt;ib_dev.type == RDMA_DEVICE_TYPE_SMI)</div><div class='add'>+			err = query_ib_ppcnt(mdev, mdev_port_num, port_num,</div><div class='add'>+					     out_cnt, sz, 0);</div><div class='add'>+		else</div><div class='add'>+			err = query_ib_ppcnt(mdev, mdev_port_num, 0,</div><div class='add'>+					     out_cnt, sz, 0);</div><div class='ctx'> </div><div class='del'>-		err = mlx5_core_query_ib_ppcnt(mdev, port_num,</div><div class='del'>-					       out_cnt, sz);</div><div class='ctx'> 		if (!err)</div><div class='ctx'> 			pma_cnt_assign(pma_cnt, out_cnt);</div><div class='del'>-		}</div><div class='del'>-</div><div class='add'>+	}</div><div class='ctx'> 	kvfree(out_cnt);</div><div class='del'>-	if (err)</div><div class='del'>-		return IB_MAD_RESULT_FAILURE;</div><div class='del'>-</div><div class='del'>-	return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;</div><div class='add'>+	err = err ? IB_MAD_RESULT_FAILURE :</div><div class='add'>+		    IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;</div><div class='add'>+done:</div><div class='add'>+	if (native_port)</div><div class='add'>+		mlx5_ib_put_native_port_mdev(dev, port_num);</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,</div><div class='add'>+int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num,</div><div class='ctx'> 			const struct ib_wc *in_wc, const struct ib_grh *in_grh,</div><div class='del'>-			const struct ib_mad_hdr *in, size_t in_mad_size,</div><div class='del'>-			struct ib_mad_hdr *out, size_t *out_mad_size,</div><div class='del'>-			u16 *out_mad_pkey_index)</div><div class='add'>+			const struct ib_mad *in, struct ib_mad *out,</div><div class='add'>+			size_t *out_mad_size, u16 *out_mad_pkey_index)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='del'>-	const struct ib_mad *in_mad = (const struct ib_mad *)in;</div><div class='del'>-	struct ib_mad *out_mad = (struct ib_mad *)out;</div><div class='del'>-	struct mlx5_core_dev *mdev;</div><div class='del'>-	u8 mdev_port_num;</div><div class='del'>-	int ret;</div><div class='add'>+	u8 mgmt_class = in-&gt;mad_hdr.mgmt_class;</div><div class='add'>+	u8 method = in-&gt;mad_hdr.method;</div><div class='add'>+	u16 slid;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-	if (WARN_ON_ONCE(in_mad_size != sizeof(*in_mad) ||</div><div class='del'>-			 *out_mad_size != sizeof(*out_mad)))</div><div class='del'>-		return IB_MAD_RESULT_FAILURE;</div><div class='add'>+	slid = in_wc ? ib_lid_cpu16(in_wc-&gt;slid) :</div><div class='add'>+		       be16_to_cpu(IB_LID_PERMISSIVE);</div><div class='ctx'> </div><div class='del'>-	memset(out_mad-&gt;data, 0, sizeof(out_mad-&gt;data));</div><div class='add'>+	if (method == IB_MGMT_METHOD_TRAP &amp;&amp; !slid)</div><div class='add'>+		return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;</div><div class='ctx'> </div><div class='del'>-	mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &amp;mdev_port_num);</div><div class='del'>-	if (!mdev)</div><div class='del'>-		return IB_MAD_RESULT_FAILURE;</div><div class='add'>+	switch (mgmt_class) {</div><div class='add'>+	case IB_MGMT_CLASS_SUBN_LID_ROUTED:</div><div class='add'>+	case IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE: {</div><div class='add'>+		if (method != IB_MGMT_METHOD_GET &amp;&amp;</div><div class='add'>+		    method != IB_MGMT_METHOD_SET &amp;&amp;</div><div class='add'>+		    method != IB_MGMT_METHOD_TRAP_REPRESS)</div><div class='add'>+			return IB_MAD_RESULT_SUCCESS;</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_GEN(mdev, vport_counters) &amp;&amp;</div><div class='del'>-	    in_mad-&gt;mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT &amp;&amp;</div><div class='del'>-	    in_mad-&gt;mad_hdr.method == IB_MGMT_METHOD_GET) {</div><div class='del'>-		ret = process_pma_cmd(mdev, mdev_port_num, in_mad, out_mad);</div><div class='del'>-	} else {</div><div class='del'>-		ret =  process_mad(ibdev, mad_flags, port_num, in_wc, in_grh,</div><div class='del'>-				   in_mad, out_mad);</div><div class='add'>+		/* Don't process SMInfo queries -- the SMA can't handle them.</div><div class='add'>+		 */</div><div class='add'>+		if (in-&gt;mad_hdr.attr_id == IB_SMP_ATTR_SM_INFO)</div><div class='add'>+			return IB_MAD_RESULT_SUCCESS;</div><div class='add'>+	} break;</div><div class='add'>+	case IB_MGMT_CLASS_PERF_MGMT:</div><div class='add'>+		if (MLX5_CAP_GEN(dev-&gt;mdev, vport_counters) &amp;&amp;</div><div class='add'>+		    method == IB_MGMT_METHOD_GET)</div><div class='add'>+			return process_pma_cmd(dev, port_num, in, out);</div><div class='add'>+		fallthrough;</div><div class='add'>+	case MLX5_IB_VENDOR_CLASS1:</div><div class='add'>+	case MLX5_IB_VENDOR_CLASS2:</div><div class='add'>+	case IB_MGMT_CLASS_CONG_MGMT: {</div><div class='add'>+		if (method != IB_MGMT_METHOD_GET &amp;&amp;</div><div class='add'>+		    method != IB_MGMT_METHOD_SET)</div><div class='add'>+			return IB_MAD_RESULT_SUCCESS;</div><div class='add'>+	} break;</div><div class='add'>+	default:</div><div class='add'>+		return IB_MAD_RESULT_SUCCESS;</div><div class='ctx'> 	}</div><div class='del'>-	mlx5_ib_put_native_port_mdev(dev, port_num);</div><div class='del'>-	return ret;</div><div class='add'>+</div><div class='add'>+	err = mlx5_MAD_IFC(to_mdev(ibdev), mad_flags &amp; IB_MAD_IGNORE_MKEY,</div><div class='add'>+			   mad_flags &amp; IB_MAD_IGNORE_BKEY, port_num, in_wc,</div><div class='add'>+			   in_grh, in, out);</div><div class='add'>+	if (err)</div><div class='add'>+		return IB_MAD_RESULT_FAILURE;</div><div class='add'>+</div><div class='add'>+	/* set return bit in status of directed route responses */</div><div class='add'>+	if (mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)</div><div class='add'>+		out-&gt;mad_hdr.status |= cpu_to_be16(1 &lt;&lt; 15);</div><div class='add'>+</div><div class='add'>+	if (method == IB_MGMT_METHOD_TRAP_REPRESS)</div><div class='add'>+		/* no response for trap repress */</div><div class='add'>+		return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;</div><div class='add'>+</div><div class='add'>+	return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port)</div><div class='add'>+int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, unsigned int port)</div><div class='ctx'> {</div><div class='del'>-	struct ib_smp *in_mad  = NULL;</div><div class='del'>-	struct ib_smp *out_mad = NULL;</div><div class='add'>+	struct ib_smp *in_mad;</div><div class='add'>+	struct ib_smp *out_mad;</div><div class='ctx'> 	int err = -ENOMEM;</div><div class='ctx'> 	u16 packet_error;</div><div class='ctx'> </div><div class='hunk'>@@ -297,7 +372,7 @@ int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port)</div><div class='ctx'> 	if (!in_mad || !out_mad)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	init_query_mad(in_mad);</div><div class='add'>+	ib_init_query_mad(in_mad);</div><div class='ctx'> 	in_mad-&gt;attr_id = MLX5_ATTR_EXTENDED_PORT_INFO;</div><div class='ctx'> 	in_mad-&gt;attr_mod = cpu_to_be32(port);</div><div class='ctx'> </div><div class='hunk'>@@ -305,7 +380,7 @@ int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port)</div><div class='ctx'> </div><div class='ctx'> 	packet_error = be16_to_cpu(out_mad-&gt;status);</div><div class='ctx'> </div><div class='del'>-	dev-&gt;mdev-&gt;port_caps[port - 1].ext_port_cap = (!err &amp;&amp; !packet_error) ?</div><div class='add'>+	dev-&gt;port_caps[port - 1].ext_port_cap = (!err &amp;&amp; !packet_error) ?</div><div class='ctx'> 		MLX_EXT_PORT_CAP_FLAG_EXTENDED_PORT_INFO : 0;</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='hunk'>@@ -314,17 +389,17 @@ out:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev,</div><div class='del'>-					  struct ib_smp *out_mad)</div><div class='add'>+static int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev,</div><div class='add'>+						 struct ib_smp *out_mad)</div><div class='ctx'> {</div><div class='del'>-	struct ib_smp *in_mad = NULL;</div><div class='del'>-	int err = -ENOMEM;</div><div class='add'>+	struct ib_smp *in_mad;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='ctx'> 	in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL);</div><div class='ctx'> 	if (!in_mad)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	init_query_mad(in_mad);</div><div class='add'>+	ib_init_query_mad(in_mad);</div><div class='ctx'> 	in_mad-&gt;attr_id = IB_SMP_ATTR_NODE_INFO;</div><div class='ctx'> </div><div class='ctx'> 	err = mlx5_MAD_IFC(to_mdev(ibdev), 1, 1, 1, NULL, NULL, in_mad,</div><div class='hunk'>@@ -337,8 +412,8 @@ int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev,</div><div class='ctx'> int mlx5_query_mad_ifc_system_image_guid(struct ib_device *ibdev,</div><div class='ctx'> 					 __be64 *sys_image_guid)</div><div class='ctx'> {</div><div class='del'>-	struct ib_smp *out_mad = NULL;</div><div class='del'>-	int err = -ENOMEM;</div><div class='add'>+	struct ib_smp *out_mad;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='ctx'> 	out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL);</div><div class='ctx'> 	if (!out_mad)</div><div class='hunk'>@@ -359,8 +434,8 @@ out:</div><div class='ctx'> int mlx5_query_mad_ifc_max_pkeys(struct ib_device *ibdev,</div><div class='ctx'> 				 u16 *max_pkeys)</div><div class='ctx'> {</div><div class='del'>-	struct ib_smp *out_mad = NULL;</div><div class='del'>-	int err = -ENOMEM;</div><div class='add'>+	struct ib_smp *out_mad;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='ctx'> 	out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL);</div><div class='ctx'> 	if (!out_mad)</div><div class='hunk'>@@ -381,8 +456,8 @@ out:</div><div class='ctx'> int mlx5_query_mad_ifc_vendor_id(struct ib_device *ibdev,</div><div class='ctx'> 				 u32 *vendor_id)</div><div class='ctx'> {</div><div class='del'>-	struct ib_smp *out_mad = NULL;</div><div class='del'>-	int err = -ENOMEM;</div><div class='add'>+	struct ib_smp *out_mad;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='ctx'> 	out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL);</div><div class='ctx'> 	if (!out_mad)</div><div class='hunk'>@@ -402,8 +477,8 @@ out:</div><div class='ctx'> </div><div class='ctx'> int mlx5_query_mad_ifc_node_desc(struct mlx5_ib_dev *dev, char *node_desc)</div><div class='ctx'> {</div><div class='del'>-	struct ib_smp *in_mad  = NULL;</div><div class='del'>-	struct ib_smp *out_mad = NULL;</div><div class='add'>+	struct ib_smp *in_mad;</div><div class='add'>+	struct ib_smp *out_mad;</div><div class='ctx'> 	int err = -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	in_mad  = kzalloc(sizeof(*in_mad), GFP_KERNEL);</div><div class='hunk'>@@ -411,7 +486,7 @@ int mlx5_query_mad_ifc_node_desc(struct mlx5_ib_dev *dev, char *node_desc)</div><div class='ctx'> 	if (!in_mad || !out_mad)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	init_query_mad(in_mad);</div><div class='add'>+	ib_init_query_mad(in_mad);</div><div class='ctx'> 	in_mad-&gt;attr_id = IB_SMP_ATTR_NODE_DESC;</div><div class='ctx'> </div><div class='ctx'> 	err = mlx5_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad);</div><div class='hunk'>@@ -427,8 +502,8 @@ out:</div><div class='ctx'> </div><div class='ctx'> int mlx5_query_mad_ifc_node_guid(struct mlx5_ib_dev *dev, __be64 *node_guid)</div><div class='ctx'> {</div><div class='del'>-	struct ib_smp *in_mad  = NULL;</div><div class='del'>-	struct ib_smp *out_mad = NULL;</div><div class='add'>+	struct ib_smp *in_mad;</div><div class='add'>+	struct ib_smp *out_mad;</div><div class='ctx'> 	int err = -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	in_mad  = kzalloc(sizeof(*in_mad), GFP_KERNEL);</div><div class='hunk'>@@ -436,7 +511,7 @@ int mlx5_query_mad_ifc_node_guid(struct mlx5_ib_dev *dev, __be64 *node_guid)</div><div class='ctx'> 	if (!in_mad || !out_mad)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	init_query_mad(in_mad);</div><div class='add'>+	ib_init_query_mad(in_mad);</div><div class='ctx'> 	in_mad-&gt;attr_id = IB_SMP_ATTR_NODE_INFO;</div><div class='ctx'> </div><div class='ctx'> 	err = mlx5_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad);</div><div class='hunk'>@@ -450,11 +525,11 @@ out:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u8 port, u16 index,</div><div class='add'>+int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u32 port, u16 index,</div><div class='ctx'> 			    u16 *pkey)</div><div class='ctx'> {</div><div class='del'>-	struct ib_smp *in_mad  = NULL;</div><div class='del'>-	struct ib_smp *out_mad = NULL;</div><div class='add'>+	struct ib_smp *in_mad;</div><div class='add'>+	struct ib_smp *out_mad;</div><div class='ctx'> 	int err = -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	in_mad  = kzalloc(sizeof(*in_mad), GFP_KERNEL);</div><div class='hunk'>@@ -462,7 +537,7 @@ int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u8 port, u16 index,</div><div class='ctx'> 	if (!in_mad || !out_mad)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	init_query_mad(in_mad);</div><div class='add'>+	ib_init_query_mad(in_mad);</div><div class='ctx'> 	in_mad-&gt;attr_id  = IB_SMP_ATTR_PKEY_TABLE;</div><div class='ctx'> 	in_mad-&gt;attr_mod = cpu_to_be32(index / 32);</div><div class='ctx'> </div><div class='hunk'>@@ -479,11 +554,11 @@ out:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u8 port, int index,</div><div class='add'>+int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u32 port, int index,</div><div class='ctx'> 			    union ib_gid *gid)</div><div class='ctx'> {</div><div class='del'>-	struct ib_smp *in_mad  = NULL;</div><div class='del'>-	struct ib_smp *out_mad = NULL;</div><div class='add'>+	struct ib_smp *in_mad;</div><div class='add'>+	struct ib_smp *out_mad;</div><div class='ctx'> 	int err = -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	in_mad  = kzalloc(sizeof(*in_mad), GFP_KERNEL);</div><div class='hunk'>@@ -491,7 +566,7 @@ int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u8 port, int index,</div><div class='ctx'> 	if (!in_mad || !out_mad)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	init_query_mad(in_mad);</div><div class='add'>+	ib_init_query_mad(in_mad);</div><div class='ctx'> 	in_mad-&gt;attr_id  = IB_SMP_ATTR_PORT_INFO;</div><div class='ctx'> 	in_mad-&gt;attr_mod = cpu_to_be32(port);</div><div class='ctx'> </div><div class='hunk'>@@ -502,7 +577,7 @@ int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u8 port, int index,</div><div class='ctx'> </div><div class='ctx'> 	memcpy(gid-&gt;raw, out_mad-&gt;data + 8, 8);</div><div class='ctx'> </div><div class='del'>-	init_query_mad(in_mad);</div><div class='add'>+	ib_init_query_mad(in_mad);</div><div class='ctx'> 	in_mad-&gt;attr_id  = IB_SMP_ATTR_GUID_INFO;</div><div class='ctx'> 	in_mad-&gt;attr_mod = cpu_to_be32(index / 8);</div><div class='ctx'> </div><div class='hunk'>@@ -519,13 +594,13 @@ out:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,</div><div class='add'>+int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 			    struct ib_port_attr *props)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='ctx'> 	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='del'>-	struct ib_smp *in_mad  = NULL;</div><div class='del'>-	struct ib_smp *out_mad = NULL;</div><div class='add'>+	struct ib_smp *in_mad;</div><div class='add'>+	struct ib_smp *out_mad;</div><div class='ctx'> 	int ext_active_speed;</div><div class='ctx'> 	int err = -ENOMEM;</div><div class='ctx'> </div><div class='hunk'>@@ -536,7 +611,7 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,</div><div class='ctx'> </div><div class='ctx'> 	/* props being zeroed by the caller, avoid zeroing it here */</div><div class='ctx'> </div><div class='del'>-	init_query_mad(in_mad);</div><div class='add'>+	ib_init_query_mad(in_mad);</div><div class='ctx'> 	in_mad-&gt;attr_id  = IB_SMP_ATTR_PORT_INFO;</div><div class='ctx'> 	in_mad-&gt;attr_mod = cpu_to_be32(port);</div><div class='ctx'> </div><div class='hunk'>@@ -555,7 +630,7 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,</div><div class='ctx'> 	props-&gt;port_cap_flags	= be32_to_cpup((__be32 *)(out_mad-&gt;data + 20));</div><div class='ctx'> 	props-&gt;gid_tbl_len	= out_mad-&gt;data[50];</div><div class='ctx'> 	props-&gt;max_msg_sz	= 1 &lt;&lt; MLX5_CAP_GEN(mdev, log_max_msg);</div><div class='del'>-	props-&gt;pkey_tbl_len	= mdev-&gt;port_caps[port - 1].pkey_table_len;</div><div class='add'>+	props-&gt;pkey_tbl_len	= dev-&gt;pkey_table_len;</div><div class='ctx'> 	props-&gt;bad_pkey_cntr	= be16_to_cpup((__be16 *)(out_mad-&gt;data + 46));</div><div class='ctx'> 	props-&gt;qkey_viol_cntr	= be16_to_cpup((__be16 *)(out_mad-&gt;data + 48));</div><div class='ctx'> 	props-&gt;active_width	= out_mad-&gt;data[31] &amp; 0xf;</div><div class='hunk'>@@ -590,14 +665,32 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,</div><div class='ctx'> 			    props-&gt;port_cap_flags2 &amp; IB_PORT_LINK_SPEED_HDR_SUP)</div><div class='ctx'> 				props-&gt;active_speed = IB_SPEED_HDR;</div><div class='ctx'> 			break;</div><div class='add'>+		case 8:</div><div class='add'>+			if (props-&gt;port_cap_flags &amp; IB_PORT_CAP_MASK2_SUP &amp;&amp;</div><div class='add'>+			    props-&gt;port_cap_flags2 &amp; IB_PORT_LINK_SPEED_NDR_SUP)</div><div class='add'>+				props-&gt;active_speed = IB_SPEED_NDR;</div><div class='add'>+			break;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Check if extended speeds 2 (XDR/...) are supported */</div><div class='add'>+	if (props-&gt;port_cap_flags &amp; IB_PORT_CAP_MASK2_SUP &amp;&amp;</div><div class='add'>+	    props-&gt;port_cap_flags2 &amp; IB_PORT_EXTENDED_SPEEDS2_SUP) {</div><div class='add'>+		ext_active_speed = (out_mad-&gt;data[56] &gt;&gt; 4) &amp; 0x6;</div><div class='add'>+</div><div class='add'>+		switch (ext_active_speed) {</div><div class='add'>+		case 2:</div><div class='add'>+			if (props-&gt;port_cap_flags2 &amp; IB_PORT_LINK_SPEED_XDR_SUP)</div><div class='add'>+				props-&gt;active_speed = IB_SPEED_XDR;</div><div class='add'>+			break;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* If reported active speed is QDR, check if is FDR-10 */</div><div class='ctx'> 	if (props-&gt;active_speed == 4) {</div><div class='del'>-		if (mdev-&gt;port_caps[port - 1].ext_port_cap &amp;</div><div class='add'>+		if (dev-&gt;port_caps[port - 1].ext_port_cap &amp;</div><div class='ctx'> 		    MLX_EXT_PORT_CAP_FLAG_EXTENDED_PORT_INFO) {</div><div class='del'>-			init_query_mad(in_mad);</div><div class='add'>+			ib_init_query_mad(in_mad);</div><div class='ctx'> 			in_mad-&gt;attr_id = MLX5_ATTR_EXTENDED_PORT_INFO;</div><div class='ctx'> 			in_mad-&gt;attr_mod = cpu_to_be32(port);</div><div class='ctx'> </div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c<br/>index abac70ad5c7c..40284bbb45d6 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/main.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/main.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/main.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/main.c</a></div><div class='hunk'>@@ -1,33 +1,7 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='ctx'> /*</div><div class='del'>- * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='add'>+ * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.</div><div class='add'>+ * Copyright (c) 2020, Intel Corporation. All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/debugfs.h&gt;</div><div class='hunk'>@@ -39,9 +13,7 @@</div><div class='ctx'> #include &lt;linux/dma-mapping.h&gt;</div><div class='ctx'> #include &lt;linux/slab.h&gt;</div><div class='ctx'> #include &lt;linux/bitmap.h&gt;</div><div class='del'>-#if defined(CONFIG_X86)</div><div class='del'>-#include &lt;asm/pat.h&gt;</div><div class='del'>-#endif</div><div class='add'>+#include &lt;linux/log2.h&gt;</div><div class='ctx'> #include &lt;linux/sched.h&gt;</div><div class='ctx'> #include &lt;linux/sched/mm.h&gt;</div><div class='ctx'> #include &lt;linux/sched/task.h&gt;</div><div class='hunk'>@@ -52,35 +24,42 @@</div><div class='ctx'> #include &lt;linux/mlx5/port.h&gt;</div><div class='ctx'> #include &lt;linux/mlx5/vport.h&gt;</div><div class='ctx'> #include &lt;linux/mlx5/fs.h&gt;</div><div class='add'>+#include &lt;linux/mlx5/eswitch.h&gt;</div><div class='add'>+#include &lt;linux/mlx5/driver.h&gt;</div><div class='ctx'> #include &lt;linux/list.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_smi.h&gt;</div><div class='del'>-#include &lt;rdma/ib_umem.h&gt;</div><div class='add'>+#include &lt;rdma/ib_umem_odp.h&gt;</div><div class='add'>+#include &lt;rdma/lag.h&gt;</div><div class='ctx'> #include &lt;linux/in.h&gt;</div><div class='ctx'> #include &lt;linux/etherdevice.h&gt;</div><div class='ctx'> #include "mlx5_ib.h"</div><div class='ctx'> #include "ib_rep.h"</div><div class='ctx'> #include "cmd.h"</div><div class='add'>+#include "devx.h"</div><div class='add'>+#include "dm.h"</div><div class='add'>+#include "fs.h"</div><div class='ctx'> #include "srq.h"</div><div class='del'>-#include &lt;linux/mlx5/fs_helpers.h&gt;</div><div class='del'>-#include &lt;linux/mlx5/accel.h&gt;</div><div class='add'>+#include "qp.h"</div><div class='add'>+#include "wr.h"</div><div class='add'>+#include "restrack.h"</div><div class='add'>+#include "counters.h"</div><div class='add'>+#include "umr.h"</div><div class='ctx'> #include &lt;rdma/uverbs_std_types.h&gt;</div><div class='add'>+#include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='ctx'> #include &lt;rdma/mlx5_user_ioctl_verbs.h&gt;</div><div class='ctx'> #include &lt;rdma/mlx5_user_ioctl_cmds.h&gt;</div><div class='add'>+#include &lt;rdma/ib_ucaps.h&gt;</div><div class='add'>+#include "macsec.h"</div><div class='add'>+#include "data_direct.h"</div><div class='add'>+#include "dmah.h"</div><div class='ctx'> </div><div class='ctx'> #define UVERBS_MODULE_NAME mlx5_ib</div><div class='ctx'> #include &lt;rdma/uverbs_named_ioctl.h&gt;</div><div class='ctx'> </div><div class='del'>-#define DRIVER_NAME "mlx5_ib"</div><div class='del'>-#define DRIVER_VERSION "5.0-0"</div><div class='del'>-</div><div class='ctx'> MODULE_AUTHOR("Eli Cohen &lt;eli@mellanox.com&gt;");</div><div class='del'>-MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver");</div><div class='add'>+MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) IB driver");</div><div class='ctx'> MODULE_LICENSE("Dual BSD/GPL");</div><div class='ctx'> </div><div class='del'>-static char mlx5_version[] =</div><div class='del'>-	DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v"</div><div class='del'>-	DRIVER_VERSION "\n";</div><div class='del'>-</div><div class='ctx'> struct mlx5_ib_event_work {</div><div class='ctx'> 	struct work_struct	work;</div><div class='ctx'> 	union {</div><div class='hunk'>@@ -104,12 +83,6 @@ static LIST_HEAD(mlx5_ib_dev_list);</div><div class='ctx'>  */</div><div class='ctx'> static DEFINE_MUTEX(mlx5_ib_multiport_mutex);</div><div class='ctx'> </div><div class='del'>-/* We can't use an array for xlt_emergency_page because dma_map_single</div><div class='del'>- * doesn't work on kernel modules memory</div><div class='del'>- */</div><div class='del'>-static unsigned long xlt_emergency_page;</div><div class='del'>-static struct mutex xlt_emergency_page_mutex;</div><div class='del'>-</div><div class='ctx'> struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev;</div><div class='hunk'>@@ -134,7 +107,7 @@ mlx5_port_type_cap_to_rdma_ll(int port_type_cap)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static enum rdma_link_layer</div><div class='del'>-mlx5_ib_port_link_layer(struct ib_device *device, u8 port_num)</div><div class='add'>+mlx5_ib_port_link_layer(struct ib_device *device, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='ctx'> 	int port_type_cap = MLX5_CAP_GEN(dev-&gt;mdev, port_type);</div><div class='hunk'>@@ -143,7 +116,7 @@ mlx5_ib_port_link_layer(struct ib_device *device, u8 port_num)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int get_port_state(struct ib_device *ibdev,</div><div class='del'>-			  u8 port_num,</div><div class='add'>+			  u32 port_num,</div><div class='ctx'> 			  enum ib_port_state *state)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr attr;</div><div class='hunk'>@@ -158,9 +131,9 @@ static int get_port_state(struct ib_device *ibdev,</div><div class='ctx'> </div><div class='ctx'> static struct mlx5_roce *mlx5_get_rep_roce(struct mlx5_ib_dev *dev,</div><div class='ctx'> 					   struct net_device *ndev,</div><div class='del'>-					   u8 *port_num)</div><div class='add'>+					   struct net_device *upper,</div><div class='add'>+					   u32 *port_num)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_eswitch *esw = dev-&gt;mdev-&gt;priv.eswitch;</div><div class='ctx'> 	struct net_device *rep_ndev;</div><div class='ctx'> 	struct mlx5_ib_port *port;</div><div class='ctx'> 	int i;</div><div class='hunk'>@@ -170,15 +143,59 @@ static struct mlx5_roce *mlx5_get_rep_roce(struct mlx5_ib_dev *dev,</div><div class='ctx'> 		if (!port-&gt;rep)</div><div class='ctx'> 			continue;</div><div class='ctx'> </div><div class='del'>-		read_lock(&amp;port-&gt;roce.netdev_lock);</div><div class='del'>-		rep_ndev = mlx5_ib_get_rep_netdev(esw,</div><div class='del'>-						  port-&gt;rep-&gt;vport);</div><div class='del'>-		if (rep_ndev == ndev) {</div><div class='del'>-			read_unlock(&amp;port-&gt;roce.netdev_lock);</div><div class='add'>+		if (upper == ndev &amp;&amp; port-&gt;rep-&gt;vport == MLX5_VPORT_UPLINK) {</div><div class='ctx'> 			*port_num = i + 1;</div><div class='ctx'> 			return &amp;port-&gt;roce;</div><div class='ctx'> 		}</div><div class='del'>-		read_unlock(&amp;port-&gt;roce.netdev_lock);</div><div class='add'>+</div><div class='add'>+		if (upper &amp;&amp; port-&gt;rep-&gt;vport == MLX5_VPORT_UPLINK)</div><div class='add'>+			continue;</div><div class='add'>+		rep_ndev = ib_device_get_netdev(&amp;dev-&gt;ib_dev, i + 1);</div><div class='add'>+		if (rep_ndev &amp;&amp; rep_ndev == ndev) {</div><div class='add'>+			dev_put(rep_ndev);</div><div class='add'>+			*port_num = i + 1;</div><div class='add'>+			return &amp;port-&gt;roce;</div><div class='add'>+		}</div><div class='add'>+</div><div class='add'>+		dev_put(rep_ndev);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool mlx5_netdev_send_event(struct mlx5_ib_dev *dev,</div><div class='add'>+				   struct net_device *ndev,</div><div class='add'>+				   struct net_device *upper,</div><div class='add'>+				   struct net_device *ib_ndev)</div><div class='add'>+{</div><div class='add'>+	if (!dev-&gt;ib_active)</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	/* Event is about our upper device */</div><div class='add'>+	if (upper == ndev)</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	/* RDMA device is not in lag and not in switchdev */</div><div class='add'>+	if (!dev-&gt;is_rep &amp;&amp; !upper &amp;&amp; ndev == ib_ndev)</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	/* RDMA devie is in switchdev */</div><div class='add'>+	if (dev-&gt;is_rep &amp;&amp; ndev == ib_ndev)</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct net_device *mlx5_ib_get_rep_uplink_netdev(struct mlx5_ib_dev *ibdev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_port *port;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	for (i = 0; i &lt; ibdev-&gt;num_ports; i++) {</div><div class='add'>+		port = &amp;ibdev-&gt;port[i];</div><div class='add'>+		if (port-&gt;rep &amp;&amp; port-&gt;rep-&gt;vport == MLX5_VPORT_UPLINK) {</div><div class='add'>+			return ib_device_get_netdev(&amp;ibdev-&gt;ib_dev, i + 1);</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	return NULL;</div><div class='hunk'>@@ -189,7 +206,8 @@ static int mlx5_netdev_event(struct notifier_block *this,</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_roce *roce = container_of(this, struct mlx5_roce, nb);</div><div class='ctx'> 	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);</div><div class='del'>-	u8 port_num = roce-&gt;native_port_num;</div><div class='add'>+	u32 port_num = roce-&gt;native_port_num;</div><div class='add'>+	struct net_device *ib_ndev = NULL;</div><div class='ctx'> 	struct mlx5_core_dev *mdev;</div><div class='ctx'> 	struct mlx5_ib_dev *ibdev;</div><div class='ctx'> </div><div class='hunk'>@@ -203,46 +221,67 @@ static int mlx5_netdev_event(struct notifier_block *this,</div><div class='ctx'> 		/* Should already be registered during the load */</div><div class='ctx'> 		if (ibdev-&gt;is_rep)</div><div class='ctx'> 			break;</div><div class='del'>-		write_lock(&amp;roce-&gt;netdev_lock);</div><div class='add'>+</div><div class='add'>+		ib_ndev = ib_device_get_netdev(&amp;ibdev-&gt;ib_dev, port_num);</div><div class='add'>+		/* Exit if already registered */</div><div class='add'>+		if (ib_ndev)</div><div class='add'>+			goto put_ndev;</div><div class='add'>+</div><div class='ctx'> 		if (ndev-&gt;dev.parent == mdev-&gt;device)</div><div class='del'>-			roce-&gt;netdev = ndev;</div><div class='del'>-		write_unlock(&amp;roce-&gt;netdev_lock);</div><div class='add'>+			ib_device_set_netdev(&amp;ibdev-&gt;ib_dev, ndev, port_num);</div><div class='ctx'> 		break;</div><div class='ctx'> </div><div class='ctx'> 	case NETDEV_UNREGISTER:</div><div class='ctx'> 		/* In case of reps, ib device goes away before the netdevs */</div><div class='del'>-		write_lock(&amp;roce-&gt;netdev_lock);</div><div class='del'>-		if (roce-&gt;netdev == ndev)</div><div class='del'>-			roce-&gt;netdev = NULL;</div><div class='del'>-		write_unlock(&amp;roce-&gt;netdev_lock);</div><div class='del'>-		break;</div><div class='add'>+		if (ibdev-&gt;is_rep)</div><div class='add'>+			break;</div><div class='add'>+		ib_ndev = ib_device_get_netdev(&amp;ibdev-&gt;ib_dev, port_num);</div><div class='add'>+		if (ib_ndev == ndev)</div><div class='add'>+			ib_device_set_netdev(&amp;ibdev-&gt;ib_dev, NULL, port_num);</div><div class='add'>+		goto put_ndev;</div><div class='ctx'> </div><div class='ctx'> 	case NETDEV_CHANGE:</div><div class='ctx'> 	case NETDEV_UP:</div><div class='ctx'> 	case NETDEV_DOWN: {</div><div class='del'>-		struct net_device *lag_ndev = mlx5_lag_get_roce_netdev(mdev);</div><div class='ctx'> 		struct net_device *upper = NULL;</div><div class='ctx'> </div><div class='del'>-		if (lag_ndev) {</div><div class='del'>-			upper = netdev_master_upper_dev_get(lag_ndev);</div><div class='del'>-			dev_put(lag_ndev);</div><div class='add'>+		if (!netif_is_lag_master(ndev) &amp;&amp; !netif_is_lag_port(ndev) &amp;&amp;</div><div class='add'>+		    !mlx5_core_mp_enabled(mdev))</div><div class='add'>+			return NOTIFY_DONE;</div><div class='add'>+</div><div class='add'>+		if (mlx5_lag_is_roce(mdev) || mlx5_lag_is_sriov(mdev)) {</div><div class='add'>+			struct net_device *lag_ndev;</div><div class='add'>+</div><div class='add'>+			if(mlx5_lag_is_roce(mdev))</div><div class='add'>+				lag_ndev = ib_device_get_netdev(&amp;ibdev-&gt;ib_dev, 1);</div><div class='add'>+			else /* sriov lag */</div><div class='add'>+				lag_ndev = mlx5_ib_get_rep_uplink_netdev(ibdev);</div><div class='add'>+</div><div class='add'>+			if (lag_ndev) {</div><div class='add'>+				upper = netdev_master_upper_dev_get(lag_ndev);</div><div class='add'>+				dev_put(lag_ndev);</div><div class='add'>+			} else {</div><div class='add'>+				goto done;</div><div class='add'>+			}</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='ctx'> 		if (ibdev-&gt;is_rep)</div><div class='del'>-			roce = mlx5_get_rep_roce(ibdev, ndev, &amp;port_num);</div><div class='add'>+			roce = mlx5_get_rep_roce(ibdev, ndev, upper, &amp;port_num);</div><div class='ctx'> 		if (!roce)</div><div class='ctx'> 			return NOTIFY_DONE;</div><div class='del'>-		if ((upper == ndev || (!upper &amp;&amp; ndev == roce-&gt;netdev))</div><div class='del'>-		    &amp;&amp; ibdev-&gt;ib_active) {</div><div class='add'>+</div><div class='add'>+		ib_ndev = ib_device_get_netdev(&amp;ibdev-&gt;ib_dev, port_num);</div><div class='add'>+</div><div class='add'>+		if (mlx5_netdev_send_event(ibdev, ndev, upper, ib_ndev)) {</div><div class='ctx'> 			struct ib_event ibev = { };</div><div class='ctx'> 			enum ib_port_state port_state;</div><div class='ctx'> </div><div class='ctx'> 			if (get_port_state(&amp;ibdev-&gt;ib_dev, port_num,</div><div class='ctx'> 					   &amp;port_state))</div><div class='del'>-				goto done;</div><div class='add'>+				goto put_ndev;</div><div class='ctx'> </div><div class='ctx'> 			if (roce-&gt;last_port_state == port_state)</div><div class='del'>-				goto done;</div><div class='add'>+				goto put_ndev;</div><div class='ctx'> </div><div class='ctx'> 			roce-&gt;last_port_state = port_state;</div><div class='ctx'> 			ibev.device = &amp;ibdev-&gt;ib_dev;</div><div class='hunk'>@@ -251,7 +290,7 @@ static int mlx5_netdev_event(struct notifier_block *this,</div><div class='ctx'> 			else if (port_state == IB_PORT_ACTIVE)</div><div class='ctx'> 				ibev.event = IB_EVENT_PORT_ACTIVE;</div><div class='ctx'> 			else</div><div class='del'>-				goto done;</div><div class='add'>+				goto put_ndev;</div><div class='ctx'> </div><div class='ctx'> 			ibev.element.port_num = port_num;</div><div class='ctx'> 			ib_dispatch_event(&amp;ibev);</div><div class='hunk'>@@ -262,42 +301,16 @@ static int mlx5_netdev_event(struct notifier_block *this,</div><div class='ctx'> 	default:</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='add'>+put_ndev:</div><div class='add'>+	dev_put(ib_ndev);</div><div class='ctx'> done:</div><div class='ctx'> 	mlx5_ib_put_native_port_mdev(ibdev, port_num);</div><div class='ctx'> 	return NOTIFY_DONE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct net_device *mlx5_ib_get_netdev(struct ib_device *device,</div><div class='del'>-					     u8 port_num)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_dev *ibdev = to_mdev(device);</div><div class='del'>-	struct net_device *ndev;</div><div class='del'>-	struct mlx5_core_dev *mdev;</div><div class='del'>-</div><div class='del'>-	mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);</div><div class='del'>-	if (!mdev)</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	ndev = mlx5_lag_get_roce_netdev(mdev);</div><div class='del'>-	if (ndev)</div><div class='del'>-		goto out;</div><div class='del'>-</div><div class='del'>-	/* Ensure ndev does not disappear before we invoke dev_hold()</div><div class='del'>-	 */</div><div class='del'>-	read_lock(&amp;ibdev-&gt;port[port_num - 1].roce.netdev_lock);</div><div class='del'>-	ndev = ibdev-&gt;port[port_num - 1].roce.netdev;</div><div class='del'>-	if (ndev)</div><div class='del'>-		dev_hold(ndev);</div><div class='del'>-	read_unlock(&amp;ibdev-&gt;port[port_num - 1].roce.netdev_lock);</div><div class='del'>-</div><div class='del'>-out:</div><div class='del'>-	mlx5_ib_put_native_port_mdev(ibdev, port_num);</div><div class='del'>-	return ndev;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev,</div><div class='del'>-						   u8 ib_port_num,</div><div class='del'>-						   u8 *native_port_num)</div><div class='add'>+						   u32 ib_port_num,</div><div class='add'>+						   u32 *native_port_num)</div><div class='ctx'> {</div><div class='ctx'> 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&amp;ibdev-&gt;ib_dev,</div><div class='ctx'> 							  ib_port_num);</div><div class='hunk'>@@ -305,6 +318,14 @@ struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev,</div><div class='ctx'> 	struct mlx5_ib_multiport_info *mpi;</div><div class='ctx'> 	struct mlx5_ib_port *port;</div><div class='ctx'> </div><div class='add'>+	if (ibdev-&gt;ib_dev.type == RDMA_DEVICE_TYPE_SMI) {</div><div class='add'>+		if (native_port_num)</div><div class='add'>+			*native_port_num = smi_to_native_portnum(ibdev,</div><div class='add'>+								 ib_port_num);</div><div class='add'>+		return ibdev-&gt;mdev;</div><div class='add'>+</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	if (!mlx5_core_mp_enabled(ibdev-&gt;mdev) ||</div><div class='ctx'> 	    ll != IB_LINK_LAYER_ETHERNET) {</div><div class='ctx'> 		if (native_port_num)</div><div class='hunk'>@@ -316,9 +337,6 @@ struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev,</div><div class='ctx'> 		*native_port_num = 1;</div><div class='ctx'> </div><div class='ctx'> 	port = &amp;ibdev-&gt;port[ib_port_num - 1];</div><div class='del'>-	if (!port)</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='ctx'> 	spin_lock(&amp;port-&gt;mp.mpi_lock);</div><div class='ctx'> 	mpi = ibdev-&gt;port[ib_port_num - 1].mp.mpi;</div><div class='ctx'> 	if (mpi &amp;&amp; !mpi-&gt;unaffiliate) {</div><div class='hunk'>@@ -334,7 +352,7 @@ struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev,</div><div class='ctx'> 	return mdev;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *ibdev, u8 port_num)</div><div class='add'>+void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *ibdev, u32 port_num)</div><div class='ctx'> {</div><div class='ctx'> 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&amp;ibdev-&gt;ib_dev,</div><div class='ctx'> 							  port_num);</div><div class='hunk'>@@ -358,8 +376,8 @@ out:</div><div class='ctx'> 	spin_unlock(&amp;port-&gt;mp.mpi_lock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int translate_eth_legacy_proto_oper(u32 eth_proto_oper, u8 *active_speed,</div><div class='del'>-					   u8 *active_width)</div><div class='add'>+static int translate_eth_legacy_proto_oper(u32 eth_proto_oper,</div><div class='add'>+					   u16 *active_speed, u8 *active_width)</div><div class='ctx'> {</div><div class='ctx'> 	switch (eth_proto_oper) {</div><div class='ctx'> 	case MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII):</div><div class='hunk'>@@ -416,7 +434,7 @@ static int translate_eth_legacy_proto_oper(u32 eth_proto_oper, u8 *active_speed,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u8 *active_speed,</div><div class='add'>+static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u16 *active_speed,</div><div class='ctx'> 					u8 *active_width)</div><div class='ctx'> {</div><div class='ctx'> 	switch (eth_proto_oper) {</div><div class='hunk'>@@ -457,10 +475,46 @@ static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u8 *active_speed,</div><div class='ctx'> 		*active_width = IB_WIDTH_2X;</div><div class='ctx'> 		*active_speed = IB_SPEED_HDR;</div><div class='ctx'> 		break;</div><div class='add'>+	case MLX5E_PROT_MASK(MLX5E_100GAUI_1_100GBASE_CR_KR):</div><div class='add'>+		*active_width = IB_WIDTH_1X;</div><div class='add'>+		*active_speed = IB_SPEED_NDR;</div><div class='add'>+		break;</div><div class='ctx'> 	case MLX5E_PROT_MASK(MLX5E_200GAUI_4_200GBASE_CR4_KR4):</div><div class='ctx'> 		*active_width = IB_WIDTH_4X;</div><div class='ctx'> 		*active_speed = IB_SPEED_HDR;</div><div class='ctx'> 		break;</div><div class='add'>+	case MLX5E_PROT_MASK(MLX5E_200GAUI_2_200GBASE_CR2_KR2):</div><div class='add'>+		*active_width = IB_WIDTH_2X;</div><div class='add'>+		*active_speed = IB_SPEED_NDR;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5E_PROT_MASK(MLX5E_200GAUI_1_200GBASE_CR1_KR1):</div><div class='add'>+		*active_width = IB_WIDTH_1X;</div><div class='add'>+		*active_speed = IB_SPEED_XDR;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5E_PROT_MASK(MLX5E_400GAUI_8_400GBASE_CR8):</div><div class='add'>+		*active_width = IB_WIDTH_8X;</div><div class='add'>+		*active_speed = IB_SPEED_HDR;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5E_PROT_MASK(MLX5E_400GAUI_4_400GBASE_CR4_KR4):</div><div class='add'>+		*active_width = IB_WIDTH_4X;</div><div class='add'>+		*active_speed = IB_SPEED_NDR;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5E_PROT_MASK(MLX5E_400GAUI_2_400GBASE_CR2_KR2):</div><div class='add'>+		*active_width = IB_WIDTH_2X;</div><div class='add'>+		*active_speed = IB_SPEED_XDR;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5E_PROT_MASK(MLX5E_800GAUI_8_800GBASE_CR8_KR8):</div><div class='add'>+		*active_width = IB_WIDTH_8X;</div><div class='add'>+		*active_speed = IB_SPEED_NDR;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5E_PROT_MASK(MLX5E_800GAUI_4_800GBASE_CR4_KR4):</div><div class='add'>+		*active_width = IB_WIDTH_4X;</div><div class='add'>+		*active_speed = IB_SPEED_XDR;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5E_PROT_MASK(MLX5E_1600TAUI_8_1600TBASE_CR8_KR8):</div><div class='add'>+		*active_width = IB_WIDTH_8X;</div><div class='add'>+		*active_speed = IB_SPEED_XDR;</div><div class='add'>+		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -468,7 +522,7 @@ static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u8 *active_speed,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int translate_eth_proto_oper(u32 eth_proto_oper, u8 *active_speed,</div><div class='add'>+static int translate_eth_proto_oper(u32 eth_proto_oper, u16 *active_speed,</div><div class='ctx'> 				    u8 *active_width, bool ext)</div><div class='ctx'> {</div><div class='ctx'> 	return ext ?</div><div class='hunk'>@@ -478,7 +532,7 @@ static int translate_eth_proto_oper(u32 eth_proto_oper, u8 *active_speed,</div><div class='ctx'> 						active_width);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,</div><div class='add'>+static int mlx5_query_port_roce(struct ib_device *device, u32 port_num,</div><div class='ctx'> 				struct ib_port_attr *props)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='hunk'>@@ -487,9 +541,8 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,</div><div class='ctx'> 	struct net_device *ndev, *upper;</div><div class='ctx'> 	enum ib_mtu ndev_ib_mtu;</div><div class='ctx'> 	bool put_mdev = true;</div><div class='del'>-	u16 qkey_viol_cntr;</div><div class='ctx'> 	u32 eth_prot_oper;</div><div class='del'>-	u8 mdev_port_num;</div><div class='add'>+	u32 mdev_port_num;</div><div class='ctx'> 	bool ext;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='hunk'>@@ -510,13 +563,13 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,</div><div class='ctx'> 	 */</div><div class='ctx'> 	if (dev-&gt;is_rep)</div><div class='ctx'> 		err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,</div><div class='del'>-					   1);</div><div class='add'>+					   1, 0);</div><div class='ctx'> 	else</div><div class='ctx'> 		err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,</div><div class='del'>-					   mdev_port_num);</div><div class='add'>+					   mdev_port_num, 0);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto out;</div><div class='del'>-	ext = MLX5_CAP_PCAM_FEATURE(dev-&gt;mdev, ptys_extended_ethernet);</div><div class='add'>+	ext = !!MLX5_GET_ETH_PROTO(ptys_reg, out, true, eth_proto_capability);</div><div class='ctx'> 	eth_prot_oper = MLX5_GET_ETH_PROTO(ptys_reg, out, ext, eth_proto_oper);</div><div class='ctx'> </div><div class='ctx'> 	props-&gt;active_width     = IB_WIDTH_4X;</div><div class='hunk'>@@ -525,29 +578,31 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,</div><div class='ctx'> 	translate_eth_proto_oper(eth_prot_oper, &amp;props-&gt;active_speed,</div><div class='ctx'> 				 &amp;props-&gt;active_width, ext);</div><div class='ctx'> </div><div class='del'>-	props-&gt;port_cap_flags |= IB_PORT_CM_SUP;</div><div class='del'>-	props-&gt;ip_gids = true;</div><div class='add'>+	if (!dev-&gt;is_rep &amp;&amp; dev-&gt;mdev-&gt;roce.roce_en) {</div><div class='add'>+		u16 qkey_viol_cntr;</div><div class='ctx'> </div><div class='del'>-	props-&gt;gid_tbl_len      = MLX5_CAP_ROCE(dev-&gt;mdev,</div><div class='del'>-						roce_address_table_size);</div><div class='add'>+		props-&gt;port_cap_flags |= IB_PORT_CM_SUP;</div><div class='add'>+		props-&gt;ip_gids = true;</div><div class='add'>+		props-&gt;gid_tbl_len = MLX5_CAP_ROCE(dev-&gt;mdev,</div><div class='add'>+						   roce_address_table_size);</div><div class='add'>+		mlx5_query_nic_vport_qkey_viol_cntr(mdev, &amp;qkey_viol_cntr);</div><div class='add'>+		props-&gt;qkey_viol_cntr = qkey_viol_cntr;</div><div class='add'>+	}</div><div class='ctx'> 	props-&gt;max_mtu          = IB_MTU_4096;</div><div class='ctx'> 	props-&gt;max_msg_sz       = 1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev, log_max_msg);</div><div class='ctx'> 	props-&gt;pkey_tbl_len     = 1;</div><div class='ctx'> 	props-&gt;state            = IB_PORT_DOWN;</div><div class='del'>-	props-&gt;phys_state       = 3;</div><div class='del'>-</div><div class='del'>-	mlx5_query_nic_vport_qkey_viol_cntr(mdev, &amp;qkey_viol_cntr);</div><div class='del'>-	props-&gt;qkey_viol_cntr = qkey_viol_cntr;</div><div class='add'>+	props-&gt;phys_state       = IB_PORT_PHYS_STATE_DISABLED;</div><div class='ctx'> </div><div class='ctx'> 	/* If this is a stub query for an unaffiliated port stop here */</div><div class='ctx'> 	if (!put_mdev)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	ndev = mlx5_ib_get_netdev(device, port_num);</div><div class='add'>+	ndev = ib_device_get_netdev(device, port_num);</div><div class='ctx'> 	if (!ndev)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	if (dev-&gt;lag_active) {</div><div class='add'>+	if (mlx5_lag_is_roce(mdev) || mlx5_lag_is_sriov(mdev)) {</div><div class='ctx'> 		rcu_read_lock();</div><div class='ctx'> 		upper = netdev_master_upper_dev_get_rcu(ndev);</div><div class='ctx'> 		if (upper) {</div><div class='hunk'>@@ -560,7 +615,7 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,</div><div class='ctx'> </div><div class='ctx'> 	if (netif_running(ndev) &amp;&amp; netif_carrier_ok(ndev)) {</div><div class='ctx'> 		props-&gt;state      = IB_PORT_ACTIVE;</div><div class='del'>-		props-&gt;phys_state = 5;</div><div class='add'>+		props-&gt;phys_state = IB_PORT_PHYS_STATE_LINK_UP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ndev_ib_mtu = iboe_get_mtu(ndev-&gt;mtu);</div><div class='hunk'>@@ -574,31 +629,31 @@ out:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int set_roce_addr(struct mlx5_ib_dev *dev, u8 port_num,</div><div class='del'>-			 unsigned int index, const union ib_gid *gid,</div><div class='del'>-			 const struct ib_gid_attr *attr)</div><div class='add'>+int set_roce_addr(struct mlx5_ib_dev *dev, u32 port_num,</div><div class='add'>+		  unsigned int index, const union ib_gid *gid,</div><div class='add'>+		  const struct ib_gid_attr *attr)</div><div class='ctx'> {</div><div class='del'>-	enum ib_gid_type gid_type = IB_GID_TYPE_IB;</div><div class='add'>+	enum ib_gid_type gid_type;</div><div class='ctx'> 	u16 vlan_id = 0xffff;</div><div class='ctx'> 	u8 roce_version = 0;</div><div class='ctx'> 	u8 roce_l3_type = 0;</div><div class='ctx'> 	u8 mac[ETH_ALEN];</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='add'>+	gid_type = attr-&gt;gid_type;</div><div class='ctx'> 	if (gid) {</div><div class='del'>-		gid_type = attr-&gt;gid_type;</div><div class='ctx'> 		ret = rdma_read_gid_l2_fields(attr, &amp;vlan_id, &amp;mac[0]);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			return ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	switch (gid_type) {</div><div class='del'>-	case IB_GID_TYPE_IB:</div><div class='add'>+	case IB_GID_TYPE_ROCE:</div><div class='ctx'> 		roce_version = MLX5_ROCE_VERSION_1;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case IB_GID_TYPE_ROCE_UDP_ENCAP:</div><div class='ctx'> 		roce_version = MLX5_ROCE_VERSION_2;</div><div class='del'>-		if (ipv6_addr_v4mapped((void *)gid))</div><div class='add'>+		if (gid &amp;&amp; ipv6_addr_v4mapped((void *)gid))</div><div class='ctx'> 			roce_l3_type = MLX5_ROCE_L3_TYPE_IPV4;</div><div class='ctx'> 		else</div><div class='ctx'> 			roce_l3_type = MLX5_ROCE_L3_TYPE_IPV6;</div><div class='hunk'>@@ -617,6 +672,12 @@ static int set_roce_addr(struct mlx5_ib_dev *dev, u8 port_num,</div><div class='ctx'> static int mlx5_ib_add_gid(const struct ib_gid_attr *attr,</div><div class='ctx'> 			   __always_unused void **context)</div><div class='ctx'> {</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = mlx5r_add_gid_macsec_operations(attr);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='ctx'> 	return set_roce_addr(to_mdev(attr-&gt;device), attr-&gt;port_num,</div><div class='ctx'> 			     attr-&gt;index, &amp;attr-&gt;gid, attr);</div><div class='ctx'> }</div><div class='hunk'>@@ -624,12 +685,19 @@ static int mlx5_ib_add_gid(const struct ib_gid_attr *attr,</div><div class='ctx'> static int mlx5_ib_del_gid(const struct ib_gid_attr *attr,</div><div class='ctx'> 			   __always_unused void **context)</div><div class='ctx'> {</div><div class='del'>-	return set_roce_addr(to_mdev(attr-&gt;device), attr-&gt;port_num,</div><div class='del'>-			     attr-&gt;index, NULL, NULL);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = set_roce_addr(to_mdev(attr-&gt;device), attr-&gt;port_num,</div><div class='add'>+			    attr-&gt;index, NULL, attr);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	mlx5r_del_gid_macsec_operations(attr);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-__be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev,</div><div class='del'>-			       const struct ib_gid_attr *attr)</div><div class='add'>+__be16 mlx5_get_roce_udp_sport_min(const struct mlx5_ib_dev *dev,</div><div class='add'>+				   const struct ib_gid_attr *attr)</div><div class='ctx'> {</div><div class='ctx'> 	if (attr-&gt;gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)</div><div class='ctx'> 		return 0;</div><div class='hunk'>@@ -692,21 +760,6 @@ static void get_atomic_caps_qp(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	get_atomic_caps(dev, atomic_size_qp, props);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void get_atomic_caps_dc(struct mlx5_ib_dev *dev,</div><div class='del'>-			       struct ib_device_attr *props)</div><div class='del'>-{</div><div class='del'>-	u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev-&gt;mdev, atomic_size_dc);</div><div class='del'>-</div><div class='del'>-	get_atomic_caps(dev, atomic_size_qp, props);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-bool mlx5_ib_dc_atomic_is_supported(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	struct ib_device_attr props = {};</div><div class='del'>-</div><div class='del'>-	get_atomic_caps_dc(dev, &amp;props);</div><div class='del'>-	return (props.atomic_cap == IB_ATOMIC_HCA) ? true : false;</div><div class='del'>-}</div><div class='ctx'> static int mlx5_query_system_image_guid(struct ib_device *ibdev,</div><div class='ctx'> 					__be64 *sys_image_guid)</div><div class='ctx'> {</div><div class='hunk'>@@ -793,7 +846,7 @@ static int mlx5_query_node_guid(struct mlx5_ib_dev *dev,</div><div class='ctx'> 		break;</div><div class='ctx'> </div><div class='ctx'> 	case MLX5_VPORT_ACCESS_METHOD_NIC:</div><div class='del'>-		err = mlx5_query_nic_vport_node_guid(dev-&gt;mdev, &amp;tmp);</div><div class='add'>+		err = mlx5_query_nic_vport_node_guid(dev-&gt;mdev, 0, false, &amp;tmp);</div><div class='ctx'> 		break;</div><div class='ctx'> </div><div class='ctx'> 	default:</div><div class='hunk'>@@ -824,10 +877,67 @@ static int mlx5_query_node_desc(struct mlx5_ib_dev *dev, char *node_desc)</div><div class='ctx'> 				    MLX5_REG_NODE_DESC, 0, 0);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void fill_esw_mgr_reg_c0(struct mlx5_core_dev *mdev,</div><div class='add'>+				struct mlx5_ib_query_device_resp *resp)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_eswitch *esw = mdev-&gt;priv.eswitch;</div><div class='add'>+	u16 vport = mlx5_eswitch_manager_vport(mdev);</div><div class='add'>+</div><div class='add'>+	resp-&gt;reg_c0.value = mlx5_eswitch_get_vport_metadata_for_match(esw,</div><div class='add'>+								      vport);</div><div class='add'>+	resp-&gt;reg_c0.mask = mlx5_eswitch_get_vport_metadata_mask();</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * Calculate maximum SQ overhead across all QP types.</div><div class='add'>+ * Other QP types (REG_UMR, UC, RC, UD/SMI/GSI, XRC_TGT)</div><div class='add'>+ * have smaller overhead than the types calculated below,</div><div class='add'>+ * so they are implicitly included.</div><div class='add'>+ */</div><div class='add'>+static u32 mlx5_ib_calc_max_sq_overhead(void)</div><div class='add'>+{</div><div class='add'>+	u32 max_overhead_xrc, overhead_ud_lso, a, b;</div><div class='add'>+</div><div class='add'>+	/* XRC_INI */</div><div class='add'>+	max_overhead_xrc = sizeof(struct mlx5_wqe_xrc_seg);</div><div class='add'>+	max_overhead_xrc += sizeof(struct mlx5_wqe_ctrl_seg);</div><div class='add'>+	a = sizeof(struct mlx5_wqe_atomic_seg) +</div><div class='add'>+	    sizeof(struct mlx5_wqe_raddr_seg);</div><div class='add'>+	b = sizeof(struct mlx5_wqe_umr_ctrl_seg) +</div><div class='add'>+	    sizeof(struct mlx5_mkey_seg) +</div><div class='add'>+	    MLX5_IB_SQ_UMR_INLINE_THRESHOLD / MLX5_IB_UMR_OCTOWORD;</div><div class='add'>+	max_overhead_xrc += max(a, b);</div><div class='add'>+</div><div class='add'>+	/* UD with LSO */</div><div class='add'>+	overhead_ud_lso = sizeof(struct mlx5_wqe_ctrl_seg);</div><div class='add'>+	overhead_ud_lso += sizeof(struct mlx5_wqe_eth_pad);</div><div class='add'>+	overhead_ud_lso += sizeof(struct mlx5_wqe_eth_seg);</div><div class='add'>+	overhead_ud_lso += sizeof(struct mlx5_wqe_datagram_seg);</div><div class='add'>+</div><div class='add'>+	return max(max_overhead_xrc, overhead_ud_lso);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u32 mlx5_ib_calc_max_qp_wr(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='add'>+	u32 max_wqe_bb_units = 1 &lt;&lt; MLX5_CAP_GEN(mdev, log_max_qp_sz);</div><div class='add'>+	u32 max_wqe_size;</div><div class='add'>+	/* max QP overhead + 1 SGE, no inline, no special features */</div><div class='add'>+	max_wqe_size = mlx5_ib_calc_max_sq_overhead() +</div><div class='add'>+		       sizeof(struct mlx5_wqe_data_seg);</div><div class='add'>+</div><div class='add'>+	max_wqe_size = roundup_pow_of_two(max_wqe_size);</div><div class='add'>+</div><div class='add'>+	max_wqe_size = ALIGN(max_wqe_size, MLX5_SEND_WQE_BB);</div><div class='add'>+</div><div class='add'>+	return (max_wqe_bb_units * MLX5_SEND_WQE_BB) / max_wqe_size;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 				struct ib_device_attr *props,</div><div class='ctx'> 				struct ib_udata *uhw)</div><div class='ctx'> {</div><div class='add'>+	size_t uhw_outlen = (uhw) ? uhw-&gt;outlen : 0;</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='ctx'> 	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='ctx'> 	int err = -ENOMEM;</div><div class='hunk'>@@ -841,12 +951,12 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 	u64 max_tso;</div><div class='ctx'> </div><div class='ctx'> 	resp_len = sizeof(resp.comp_mask) + sizeof(resp.response_length);</div><div class='del'>-	if (uhw-&gt;outlen &amp;&amp; uhw-&gt;outlen &lt; resp_len)</div><div class='add'>+	if (uhw_outlen &amp;&amp; uhw_outlen &lt; resp_len)</div><div class='ctx'> 		return -EINVAL;</div><div class='del'>-	else</div><div class='del'>-		resp.response_length = resp_len;</div><div class='ctx'> </div><div class='del'>-	if (uhw-&gt;inlen &amp;&amp; !ib_is_udata_cleared(uhw, 0, uhw-&gt;inlen))</div><div class='add'>+	resp.response_length = resp_len;</div><div class='add'>+</div><div class='add'>+	if (uhw &amp;&amp; uhw-&gt;inlen &amp;&amp; !ib_is_udata_cleared(uhw, 0, uhw-&gt;inlen))</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	memset(props, 0, sizeof(*props));</div><div class='hunk'>@@ -855,9 +965,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='del'>-	err = mlx5_query_max_pkeys(ibdev, &amp;props-&gt;max_pkeys);</div><div class='del'>-	if (err)</div><div class='del'>-		return err;</div><div class='add'>+	props-&gt;max_pkeys = dev-&gt;pkey_table_len;</div><div class='ctx'> </div><div class='ctx'> 	err = mlx5_query_vendor_id(ibdev, &amp;props-&gt;vendor_id);</div><div class='ctx'> 	if (err)</div><div class='hunk'>@@ -884,11 +992,13 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 					   IB_DEVICE_MEM_WINDOW_TYPE_2B;</div><div class='ctx'> 		props-&gt;max_mw = 1 &lt;&lt; MLX5_CAP_GEN(mdev, log_max_mkey);</div><div class='ctx'> 		/* We support 'Gappy' memory registration too */</div><div class='del'>-		props-&gt;device_cap_flags |= IB_DEVICE_SG_GAPS_REG;</div><div class='add'>+		props-&gt;kernel_cap_flags |= IBK_SG_GAPS_REG;</div><div class='ctx'> 	}</div><div class='del'>-	props-&gt;device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;</div><div class='add'>+	/* IB_WR_REG_MR always requires changing the entity size with UMR */</div><div class='add'>+	if (!MLX5_CAP_GEN(dev-&gt;mdev, umr_modify_entity_size_disabled))</div><div class='add'>+		props-&gt;device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;</div><div class='ctx'> 	if (MLX5_CAP_GEN(mdev, sho)) {</div><div class='del'>-		props-&gt;device_cap_flags |= IB_DEVICE_SIGNATURE_HANDOVER;</div><div class='add'>+		props-&gt;kernel_cap_flags |= IBK_INTEGRITY_HANDOVER;</div><div class='ctx'> 		/* At this stage no support for signature handover */</div><div class='ctx'> 		props-&gt;sig_prot_cap = IB_PROT_T10DIF_TYPE_1 |</div><div class='ctx'> 				      IB_PROT_T10DIF_TYPE_2 |</div><div class='hunk'>@@ -897,7 +1007,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 				       IB_GUARD_T10DIF_CSUM;</div><div class='ctx'> 	}</div><div class='ctx'> 	if (MLX5_CAP_GEN(mdev, block_lb_mc))</div><div class='del'>-		props-&gt;device_cap_flags |= IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;</div><div class='add'>+		props-&gt;kernel_cap_flags |= IBK_BLOCK_MULTICAST_LOOPBACK;</div><div class='ctx'> </div><div class='ctx'> 	if (MLX5_CAP_GEN(dev-&gt;mdev, eth_net_offloads) &amp;&amp; raw_support) {</div><div class='ctx'> 		if (MLX5_CAP_ETH(mdev, csum_cap)) {</div><div class='hunk'>@@ -910,7 +1020,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 			props-&gt;raw_packet_caps |=</div><div class='ctx'> 				IB_RAW_PACKET_CAP_CVLAN_STRIPPING;</div><div class='ctx'> </div><div class='del'>-		if (field_avail(typeof(resp), tso_caps, uhw-&gt;outlen)) {</div><div class='add'>+		if (offsetofend(typeof(resp), tso_caps) &lt;= uhw_outlen) {</div><div class='ctx'> 			max_tso = MLX5_CAP_ETH(mdev, max_lso_cap);</div><div class='ctx'> 			if (max_tso) {</div><div class='ctx'> 				resp.tso_caps.max_tso = 1 &lt;&lt; max_tso;</div><div class='hunk'>@@ -920,7 +1030,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 			}</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		if (field_avail(typeof(resp), rss_caps, uhw-&gt;outlen)) {</div><div class='add'>+		if (offsetofend(typeof(resp), rss_caps) &lt;= uhw_outlen) {</div><div class='ctx'> 			resp.rss_caps.rx_hash_function =</div><div class='ctx'> 						MLX5_RX_HASH_FUNC_TOEPLITZ;</div><div class='ctx'> 			resp.rss_caps.rx_hash_fields_mask =</div><div class='hunk'>@@ -933,22 +1043,18 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 						MLX5_RX_HASH_SRC_PORT_UDP |</div><div class='ctx'> 						MLX5_RX_HASH_DST_PORT_UDP |</div><div class='ctx'> 						MLX5_RX_HASH_INNER;</div><div class='del'>-			if (mlx5_accel_ipsec_device_caps(dev-&gt;mdev) &amp;</div><div class='del'>-			    MLX5_ACCEL_IPSEC_CAP_DEVICE)</div><div class='del'>-				resp.rss_caps.rx_hash_fields_mask |=</div><div class='del'>-					MLX5_RX_HASH_IPSEC_SPI;</div><div class='ctx'> 			resp.response_length += sizeof(resp.rss_caps);</div><div class='ctx'> 		}</div><div class='ctx'> 	} else {</div><div class='del'>-		if (field_avail(typeof(resp), tso_caps, uhw-&gt;outlen))</div><div class='add'>+		if (offsetofend(typeof(resp), tso_caps) &lt;= uhw_outlen)</div><div class='ctx'> 			resp.response_length += sizeof(resp.tso_caps);</div><div class='del'>-		if (field_avail(typeof(resp), rss_caps, uhw-&gt;outlen))</div><div class='add'>+		if (offsetofend(typeof(resp), rss_caps) &lt;= uhw_outlen)</div><div class='ctx'> 			resp.response_length += sizeof(resp.rss_caps);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {</div><div class='ctx'> 		props-&gt;device_cap_flags |= IB_DEVICE_UD_IP_CSUM;</div><div class='del'>-		props-&gt;device_cap_flags |= IB_DEVICE_UD_TSO;</div><div class='add'>+		props-&gt;kernel_cap_flags |= IBK_UD_TSO;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (MLX5_CAP_GEN(dev-&gt;mdev, rq_delay_drop) &amp;&amp;</div><div class='hunk'>@@ -985,7 +1091,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 	props-&gt;max_mr_size	   = ~0ull;</div><div class='ctx'> 	props-&gt;page_size_cap	   = ~(min_page_size - 1);</div><div class='ctx'> 	props-&gt;max_qp		   = 1 &lt;&lt; MLX5_CAP_GEN(mdev, log_max_qp);</div><div class='del'>-	props-&gt;max_qp_wr	   = 1 &lt;&lt; MLX5_CAP_GEN(mdev, log_max_qp_sz);</div><div class='add'>+	props-&gt;max_qp_wr = mlx5_ib_calc_max_qp_wr(dev);</div><div class='ctx'> 	max_rq_sg =  MLX5_CAP_GEN(mdev, max_wqe_sz_rq) /</div><div class='ctx'> 		     sizeof(struct mlx5_wqe_data_seg);</div><div class='ctx'> 	max_sq_desc = min_t(int, MLX5_CAP_GEN(mdev, max_wqe_sz_sq), 512);</div><div class='hunk'>@@ -1008,28 +1114,45 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 	props-&gt;max_srq_sge	   = max_rq_sg - 1;</div><div class='ctx'> 	props-&gt;max_fast_reg_page_list_len =</div><div class='ctx'> 		1 &lt;&lt; MLX5_CAP_GEN(mdev, log_max_klm_list_size);</div><div class='add'>+	props-&gt;max_pi_fast_reg_page_list_len =</div><div class='add'>+		props-&gt;max_fast_reg_page_list_len / 2;</div><div class='add'>+	props-&gt;max_sgl_rd =</div><div class='add'>+		MLX5_CAP_GEN(mdev, max_sgl_for_optimized_performance);</div><div class='ctx'> 	get_atomic_caps_qp(dev, props);</div><div class='ctx'> 	props-&gt;masked_atomic_cap   = IB_ATOMIC_NONE;</div><div class='ctx'> 	props-&gt;max_mcast_grp	   = 1 &lt;&lt; MLX5_CAP_GEN(mdev, log_max_mcg);</div><div class='ctx'> 	props-&gt;max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);</div><div class='ctx'> 	props-&gt;max_total_mcast_qp_attach = props-&gt;max_mcast_qp_attach *</div><div class='ctx'> 					   props-&gt;max_mcast_grp;</div><div class='del'>-	props-&gt;max_map_per_fmr = INT_MAX; /* no limit in ConnectIB */</div><div class='ctx'> 	props-&gt;max_ah = INT_MAX;</div><div class='ctx'> 	props-&gt;hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);</div><div class='ctx'> 	props-&gt;timestamp_mask = 0x7FFFFFFFFFFFFFFFULL;</div><div class='ctx'> </div><div class='ctx'> 	if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {</div><div class='del'>-		if (MLX5_CAP_GEN(mdev, pg))</div><div class='del'>-			props-&gt;device_cap_flags |= IB_DEVICE_ON_DEMAND_PAGING;</div><div class='add'>+		if (dev-&gt;odp_caps.general_caps &amp; IB_ODP_SUPPORT)</div><div class='add'>+			props-&gt;kernel_cap_flags |= IBK_ON_DEMAND_PAGING;</div><div class='ctx'> 		props-&gt;odp_caps = dev-&gt;odp_caps;</div><div class='add'>+		if (!uhw) {</div><div class='add'>+			/* ODP for kernel QPs is not implemented for receive</div><div class='add'>+			 * WQEs and SRQ WQEs</div><div class='add'>+			 */</div><div class='add'>+			props-&gt;odp_caps.per_transport_caps.rc_odp_caps &amp;=</div><div class='add'>+				~(IB_ODP_SUPPORT_READ |</div><div class='add'>+				  IB_ODP_SUPPORT_SRQ_RECV);</div><div class='add'>+			props-&gt;odp_caps.per_transport_caps.uc_odp_caps &amp;=</div><div class='add'>+				~(IB_ODP_SUPPORT_READ |</div><div class='add'>+				  IB_ODP_SUPPORT_SRQ_RECV);</div><div class='add'>+			props-&gt;odp_caps.per_transport_caps.ud_odp_caps &amp;=</div><div class='add'>+				~(IB_ODP_SUPPORT_READ |</div><div class='add'>+				  IB_ODP_SUPPORT_SRQ_RECV);</div><div class='add'>+			props-&gt;odp_caps.per_transport_caps.xrc_odp_caps &amp;=</div><div class='add'>+				~(IB_ODP_SUPPORT_READ |</div><div class='add'>+				  IB_ODP_SUPPORT_SRQ_RECV);</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_GEN(mdev, cd))</div><div class='del'>-		props-&gt;device_cap_flags |= IB_DEVICE_CROSS_CHANNEL;</div><div class='del'>-</div><div class='del'>-	if (!mlx5_core_is_pf(mdev))</div><div class='del'>-		props-&gt;device_cap_flags |= IB_DEVICE_VIRTUAL_FUNCTION;</div><div class='add'>+	if (mlx5_core_is_vf(mdev))</div><div class='add'>+		props-&gt;kernel_cap_flags |= IBK_VIRTUAL_FUNCTION;</div><div class='ctx'> </div><div class='ctx'> 	if (mlx5_ib_port_link_layer(ibdev, 1) ==</div><div class='ctx'> 	    IB_LINK_LAYER_ETHERNET &amp;&amp; raw_support) {</div><div class='hunk'>@@ -1043,15 +1166,19 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (MLX5_CAP_GEN(mdev, tag_matching)) {</div><div class='del'>-		props-&gt;tm_caps.max_rndv_hdr_size = MLX5_TM_MAX_RNDV_MSG_SIZE;</div><div class='ctx'> 		props-&gt;tm_caps.max_num_tags =</div><div class='ctx'> 			(1 &lt;&lt; MLX5_CAP_GEN(mdev, log_tag_matching_list_sz)) - 1;</div><div class='del'>-		props-&gt;tm_caps.flags = IB_TM_CAP_RC;</div><div class='ctx'> 		props-&gt;tm_caps.max_ops =</div><div class='ctx'> 			1 &lt;&lt; MLX5_CAP_GEN(mdev, log_max_qp_sz);</div><div class='ctx'> 		props-&gt;tm_caps.max_sge = MLX5_TM_MAX_SGE;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (MLX5_CAP_GEN(mdev, tag_matching) &amp;&amp;</div><div class='add'>+	    MLX5_CAP_GEN(mdev, rndv_offload_rc)) {</div><div class='add'>+		props-&gt;tm_caps.flags = IB_TM_CAP_RNDV_RC;</div><div class='add'>+		props-&gt;tm_caps.max_rndv_hdr_size = MLX5_TM_MAX_RNDV_MSG_SIZE;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	if (MLX5_CAP_GEN(dev-&gt;mdev, cq_moderation)) {</div><div class='ctx'> 		props-&gt;cq_caps.max_cq_moderation_count =</div><div class='ctx'> 						MLX5_MAX_CQ_COUNT;</div><div class='hunk'>@@ -1059,7 +1186,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 						MLX5_MAX_CQ_PERIOD;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (field_avail(typeof(resp), cqe_comp_caps, uhw-&gt;outlen)) {</div><div class='add'>+	if (offsetofend(typeof(resp), cqe_comp_caps) &lt;= uhw_outlen) {</div><div class='ctx'> 		resp.response_length += sizeof(resp.cqe_comp_caps);</div><div class='ctx'> </div><div class='ctx'> 		if (MLX5_CAP_GEN(dev-&gt;mdev, cqe_compression)) {</div><div class='hunk'>@@ -1077,7 +1204,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (field_avail(typeof(resp), packet_pacing_caps, uhw-&gt;outlen) &amp;&amp;</div><div class='add'>+	if (offsetofend(typeof(resp), packet_pacing_caps) &lt;= uhw_outlen &amp;&amp;</div><div class='ctx'> 	    raw_support) {</div><div class='ctx'> 		if (MLX5_CAP_QOS(mdev, packet_pacing) &amp;&amp;</div><div class='ctx'> 		    MLX5_CAP_GEN(mdev, qos)) {</div><div class='hunk'>@@ -1095,8 +1222,8 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 		resp.response_length += sizeof(resp.packet_pacing_caps);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (field_avail(typeof(resp), mlx5_ib_support_multi_pkt_send_wqes,</div><div class='del'>-			uhw-&gt;outlen)) {</div><div class='add'>+	if (offsetofend(typeof(resp), mlx5_ib_support_multi_pkt_send_wqes) &lt;=</div><div class='add'>+	    uhw_outlen) {</div><div class='ctx'> 		if (MLX5_CAP_ETH(mdev, multi_pkt_send_wqe))</div><div class='ctx'> 			resp.mlx5_ib_support_multi_pkt_send_wqes =</div><div class='ctx'> 				MLX5_IB_ALLOW_MPW;</div><div class='hunk'>@@ -1109,7 +1236,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 			sizeof(resp.mlx5_ib_support_multi_pkt_send_wqes);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (field_avail(typeof(resp), flags, uhw-&gt;outlen)) {</div><div class='add'>+	if (offsetofend(typeof(resp), flags) &lt;= uhw_outlen) {</div><div class='ctx'> 		resp.response_length += sizeof(resp.flags);</div><div class='ctx'> </div><div class='ctx'> 		if (MLX5_CAP_GEN(mdev, cqe_compression_128))</div><div class='hunk'>@@ -1123,10 +1250,17 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 				MLX5_IB_QUERY_DEV_RESP_PACKET_BASED_CREDIT_MODE;</div><div class='ctx'> </div><div class='ctx'> 		resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_SCAT2CQE_DCT;</div><div class='add'>+</div><div class='add'>+		if (MLX5_CAP_GEN_2(mdev, dp_ordering_force) &amp;&amp;</div><div class='add'>+		    (MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_xrc) ||</div><div class='add'>+		    MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_dc) ||</div><div class='add'>+		    MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_rc) ||</div><div class='add'>+		    MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_ud) ||</div><div class='add'>+		    MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_uc)))</div><div class='add'>+			resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_OOO_DP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (field_avail(typeof(resp), sw_parsing_caps,</div><div class='del'>-			uhw-&gt;outlen)) {</div><div class='add'>+	if (offsetofend(typeof(resp), sw_parsing_caps) &lt;= uhw_outlen) {</div><div class='ctx'> 		resp.response_length += sizeof(resp.sw_parsing_caps);</div><div class='ctx'> 		if (MLX5_CAP_ETH(mdev, swp)) {</div><div class='ctx'> 			resp.sw_parsing_caps.sw_parsing_offloads |=</div><div class='hunk'>@@ -1146,7 +1280,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (field_avail(typeof(resp), striding_rq_caps, uhw-&gt;outlen) &amp;&amp;</div><div class='add'>+	if (offsetofend(typeof(resp), striding_rq_caps) &lt;= uhw_outlen &amp;&amp;</div><div class='ctx'> 	    raw_support) {</div><div class='ctx'> 		resp.response_length += sizeof(resp.striding_rq_caps);</div><div class='ctx'> 		if (MLX5_CAP_GEN(mdev, striding_rq)) {</div><div class='hunk'>@@ -1154,8 +1288,14 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 				MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES;</div><div class='ctx'> 			resp.striding_rq_caps.max_single_stride_log_num_of_bytes =</div><div class='ctx'> 				MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES;</div><div class='del'>-			resp.striding_rq_caps.min_single_wqe_log_num_of_strides =</div><div class='del'>-				MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES;</div><div class='add'>+			if (MLX5_CAP_GEN(dev-&gt;mdev, ext_stride_num_range))</div><div class='add'>+				resp.striding_rq_caps</div><div class='add'>+					.min_single_wqe_log_num_of_strides =</div><div class='add'>+					MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES;</div><div class='add'>+			else</div><div class='add'>+				resp.striding_rq_caps</div><div class='add'>+					.min_single_wqe_log_num_of_strides =</div><div class='add'>+					MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES;</div><div class='ctx'> 			resp.striding_rq_caps.max_single_wqe_log_num_of_strides =</div><div class='ctx'> 				MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES;</div><div class='ctx'> 			resp.striding_rq_caps.supported_qpts =</div><div class='hunk'>@@ -1163,8 +1303,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (field_avail(typeof(resp), tunnel_offloads_caps,</div><div class='del'>-			uhw-&gt;outlen)) {</div><div class='add'>+	if (offsetofend(typeof(resp), tunnel_offloads_caps) &lt;= uhw_outlen) {</div><div class='ctx'> 		resp.response_length += sizeof(resp.tunnel_offloads_caps);</div><div class='ctx'> 		if (MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan))</div><div class='ctx'> 			resp.tunnel_offloads_caps |=</div><div class='hunk'>@@ -1175,17 +1314,38 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 		if (MLX5_CAP_ETH(mdev, tunnel_stateless_gre))</div><div class='ctx'> 			resp.tunnel_offloads_caps |=</div><div class='ctx'> 				MLX5_IB_TUNNELED_OFFLOADS_GRE;</div><div class='del'>-		if (MLX5_CAP_GEN(mdev, flex_parser_protocols) &amp;</div><div class='del'>-		    MLX5_FLEX_PROTO_CW_MPLS_GRE)</div><div class='add'>+		if (MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre))</div><div class='ctx'> 			resp.tunnel_offloads_caps |=</div><div class='ctx'> 				MLX5_IB_TUNNELED_OFFLOADS_MPLS_GRE;</div><div class='del'>-		if (MLX5_CAP_GEN(mdev, flex_parser_protocols) &amp;</div><div class='del'>-		    MLX5_FLEX_PROTO_CW_MPLS_UDP)</div><div class='add'>+		if (MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_udp))</div><div class='ctx'> 			resp.tunnel_offloads_caps |=</div><div class='ctx'> 				MLX5_IB_TUNNELED_OFFLOADS_MPLS_UDP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (uhw-&gt;outlen) {</div><div class='add'>+	if (offsetofend(typeof(resp), dci_streams_caps) &lt;= uhw_outlen) {</div><div class='add'>+		resp.response_length += sizeof(resp.dci_streams_caps);</div><div class='add'>+</div><div class='add'>+		resp.dci_streams_caps.max_log_num_concurent =</div><div class='add'>+			MLX5_CAP_GEN(mdev, log_max_dci_stream_channels);</div><div class='add'>+</div><div class='add'>+		resp.dci_streams_caps.max_log_num_errored =</div><div class='add'>+			MLX5_CAP_GEN(mdev, log_max_dci_errored_streams);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (offsetofend(typeof(resp), reserved) &lt;= uhw_outlen)</div><div class='add'>+		resp.response_length += sizeof(resp.reserved);</div><div class='add'>+</div><div class='add'>+	if (offsetofend(typeof(resp), reg_c0) &lt;= uhw_outlen) {</div><div class='add'>+		struct mlx5_eswitch *esw = mdev-&gt;priv.eswitch;</div><div class='add'>+</div><div class='add'>+		resp.response_length += sizeof(resp.reg_c0);</div><div class='add'>+</div><div class='add'>+		if (mlx5_eswitch_mode(mdev) == MLX5_ESWITCH_OFFLOADS &amp;&amp;</div><div class='add'>+		    mlx5_eswitch_vport_match_metadata_enabled(esw))</div><div class='add'>+			fill_esw_mgr_reg_c0(mdev, &amp;resp);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (uhw_outlen) {</div><div class='ctx'> 		err = ib_copy_to_udata(uhw, &amp;resp, resp.response_length);</div><div class='ctx'> </div><div class='ctx'> 		if (err)</div><div class='hunk'>@@ -1195,32 +1355,24 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-enum mlx5_ib_width {</div><div class='del'>-	MLX5_IB_WIDTH_1X	= 1 &lt;&lt; 0,</div><div class='del'>-	MLX5_IB_WIDTH_2X	= 1 &lt;&lt; 1,</div><div class='del'>-	MLX5_IB_WIDTH_4X	= 1 &lt;&lt; 2,</div><div class='del'>-	MLX5_IB_WIDTH_8X	= 1 &lt;&lt; 3,</div><div class='del'>-	MLX5_IB_WIDTH_12X	= 1 &lt;&lt; 4</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static void translate_active_width(struct ib_device *ibdev, u8 active_width,</div><div class='del'>-				  u8 *ib_width)</div><div class='add'>+static void translate_active_width(struct ib_device *ibdev, u16 active_width,</div><div class='add'>+				   u8 *ib_width)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='ctx'> </div><div class='del'>-	if (active_width &amp; MLX5_IB_WIDTH_1X)</div><div class='add'>+	if (active_width &amp; MLX5_PTYS_WIDTH_1X)</div><div class='ctx'> 		*ib_width = IB_WIDTH_1X;</div><div class='del'>-	else if (active_width &amp; MLX5_IB_WIDTH_2X)</div><div class='add'>+	else if (active_width &amp; MLX5_PTYS_WIDTH_2X)</div><div class='ctx'> 		*ib_width = IB_WIDTH_2X;</div><div class='del'>-	else if (active_width &amp; MLX5_IB_WIDTH_4X)</div><div class='add'>+	else if (active_width &amp; MLX5_PTYS_WIDTH_4X)</div><div class='ctx'> 		*ib_width = IB_WIDTH_4X;</div><div class='del'>-	else if (active_width &amp; MLX5_IB_WIDTH_8X)</div><div class='add'>+	else if (active_width &amp; MLX5_PTYS_WIDTH_8X)</div><div class='ctx'> 		*ib_width = IB_WIDTH_8X;</div><div class='del'>-	else if (active_width &amp; MLX5_IB_WIDTH_12X)</div><div class='add'>+	else if (active_width &amp; MLX5_PTYS_WIDTH_12X)</div><div class='ctx'> 		*ib_width = IB_WIDTH_12X;</div><div class='ctx'> 	else {</div><div class='ctx'> 		mlx5_ib_dbg(dev, "Invalid active_width %d, setting width to default value: 4x\n",</div><div class='del'>-			    (int)active_width);</div><div class='add'>+			    active_width);</div><div class='ctx'> 		*ib_width = IB_WIDTH_4X;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='hunk'>@@ -1288,17 +1440,17 @@ static int translate_max_vl_num(struct ib_device *ibdev, u8 vl_hw_cap,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_query_hca_port(struct ib_device *ibdev, u8 port,</div><div class='add'>+static int mlx5_query_hca_port(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 			       struct ib_port_attr *props)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='ctx'> 	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='ctx'> 	struct mlx5_hca_vport_context *rep;</div><div class='add'>+	u8 vl_hw_cap, plane_index = 0;</div><div class='ctx'> 	u16 max_mtu;</div><div class='ctx'> 	u16 oper_mtu;</div><div class='ctx'> 	int err;</div><div class='del'>-	u8 ib_link_width_oper;</div><div class='del'>-	u8 vl_hw_cap;</div><div class='add'>+	u16 ib_link_width_oper;</div><div class='ctx'> </div><div class='ctx'> 	rep = kzalloc(sizeof(*rep), GFP_KERNEL);</div><div class='ctx'> 	if (!rep) {</div><div class='hunk'>@@ -1308,6 +1460,11 @@ static int mlx5_query_hca_port(struct ib_device *ibdev, u8 port,</div><div class='ctx'> </div><div class='ctx'> 	/* props being zeroed by the caller, avoid zeroing it here */</div><div class='ctx'> </div><div class='add'>+	if (ibdev-&gt;type == RDMA_DEVICE_TYPE_SMI) {</div><div class='add'>+		plane_index = port;</div><div class='add'>+		port = smi_to_native_portnum(dev, port);</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	err = mlx5_query_hca_vport_context(mdev, 0, port, 0, rep);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto out;</div><div class='hunk'>@@ -1318,7 +1475,14 @@ static int mlx5_query_hca_port(struct ib_device *ibdev, u8 port,</div><div class='ctx'> 	props-&gt;sm_sl		= rep-&gt;sm_sl;</div><div class='ctx'> 	props-&gt;state		= rep-&gt;vport_state;</div><div class='ctx'> 	props-&gt;phys_state	= rep-&gt;port_physical_state;</div><div class='del'>-	props-&gt;port_cap_flags	= rep-&gt;cap_mask1;</div><div class='add'>+</div><div class='add'>+	props-&gt;port_cap_flags = rep-&gt;cap_mask1;</div><div class='add'>+	if (dev-&gt;num_plane) {</div><div class='add'>+		props-&gt;port_cap_flags |= IB_PORT_SM_DISABLED;</div><div class='add'>+		props-&gt;port_cap_flags &amp;= ~IB_PORT_SM;</div><div class='add'>+	} else if (ibdev-&gt;type == RDMA_DEVICE_TYPE_SMI)</div><div class='add'>+		props-&gt;port_cap_flags &amp;= ~IB_PORT_CM_SUP;</div><div class='add'>+</div><div class='ctx'> 	props-&gt;gid_tbl_len	= mlx5_get_gid_table_len(MLX5_CAP_GEN(mdev, gid_table_size));</div><div class='ctx'> 	props-&gt;max_msg_sz	= 1 &lt;&lt; MLX5_CAP_GEN(mdev, log_max_msg);</div><div class='ctx'> 	props-&gt;pkey_tbl_len	= mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev, pkey_table_size));</div><div class='hunk'>@@ -1330,16 +1494,13 @@ static int mlx5_query_hca_port(struct ib_device *ibdev, u8 port,</div><div class='ctx'> 	if (props-&gt;port_cap_flags &amp; IB_PORT_CAP_MASK2_SUP)</div><div class='ctx'> 		props-&gt;port_cap_flags2 = rep-&gt;cap_mask2;</div><div class='ctx'> </div><div class='del'>-	err = mlx5_query_port_link_width_oper(mdev, &amp;ib_link_width_oper, port);</div><div class='add'>+	err = mlx5_query_ib_port_oper(mdev, &amp;ib_link_width_oper,</div><div class='add'>+				      &amp;props-&gt;active_speed, port, plane_index);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='ctx'> 	translate_active_width(ibdev, ib_link_width_oper, &amp;props-&gt;active_width);</div><div class='ctx'> </div><div class='del'>-	err = mlx5_query_port_ib_proto_oper(mdev, &amp;props-&gt;active_speed, port);</div><div class='del'>-	if (err)</div><div class='del'>-		goto out;</div><div class='del'>-</div><div class='ctx'> 	mlx5_query_port_max_mtu(mdev, &amp;max_mtu, port);</div><div class='ctx'> </div><div class='ctx'> 	props-&gt;max_mtu = mlx5_mtu_to_ib_mtu(max_mtu);</div><div class='hunk'>@@ -1359,7 +1520,7 @@ out:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,</div><div class='add'>+int mlx5_ib_query_port(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 		       struct ib_port_attr *props)</div><div class='ctx'> {</div><div class='ctx'> 	unsigned int count;</div><div class='hunk'>@@ -1404,25 +1565,23 @@ int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u8 port,</div><div class='add'>+static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				  struct ib_port_attr *props)</div><div class='ctx'> {</div><div class='del'>-	int ret;</div><div class='add'>+	return mlx5_query_port_roce(ibdev, port, props);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	/* Only link layer == ethernet is valid for representors</div><div class='del'>-	 * and we always use port 1</div><div class='add'>+static int mlx5_ib_rep_query_pkey(struct ib_device *ibdev, u32 port, u16 index,</div><div class='add'>+				  u16 *pkey)</div><div class='add'>+{</div><div class='add'>+	/* Default special Pkey for representor device port as per the</div><div class='add'>+	 * IB specification 1.3 section 10.9.1.2.</div><div class='ctx'> 	 */</div><div class='del'>-	ret = mlx5_query_port_roce(ibdev, port, props);</div><div class='del'>-	if (ret || !props)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	/* We don't support GIDS */</div><div class='del'>-	props-&gt;gid_tbl_len = 0;</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='add'>+	*pkey = 0xffff;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index,</div><div class='add'>+static int mlx5_ib_query_gid(struct ib_device *ibdev, u32 port, int index,</div><div class='ctx'> 			     union ib_gid *gid)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='hunk'>@@ -1441,13 +1600,13 @@ static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index,</div><div class='ctx'> </div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u8 port,</div><div class='add'>+static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 				   u16 index, u16 *pkey)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='ctx'> 	struct mlx5_core_dev *mdev;</div><div class='ctx'> 	bool put_mdev = true;</div><div class='del'>-	u8 mdev_port_num;</div><div class='add'>+	u32 mdev_port_num;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	mdev = mlx5_ib_get_native_port_mdev(dev, port, &amp;mdev_port_num);</div><div class='hunk'>@@ -1468,7 +1627,7 @@ static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u8 port,</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,</div><div class='add'>+static int mlx5_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index,</div><div class='ctx'> 			      u16 *pkey)</div><div class='ctx'> {</div><div class='ctx'> 	switch (mlx5_get_vport_access_method(ibdev)) {</div><div class='hunk'>@@ -1512,12 +1671,12 @@ static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u8 port_num, u32 mask,</div><div class='add'>+static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u32 port_num, u32 mask,</div><div class='ctx'> 				u32 value)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_hca_vport_context ctx = {};</div><div class='ctx'> 	struct mlx5_core_dev *mdev;</div><div class='del'>-	u8 mdev_port_num;</div><div class='add'>+	u32 mdev_port_num;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &amp;mdev_port_num);</div><div class='hunk'>@@ -1546,7 +1705,7 @@ out:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,</div><div class='add'>+static int mlx5_ib_modify_port(struct ib_device *ibdev, u32 port, int mask,</div><div class='ctx'> 			       struct ib_port_modify *props)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='hunk'>@@ -1647,7 +1806,8 @@ static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *conte</div><div class='ctx'> </div><div class='ctx'> 	bfregi = &amp;context-&gt;bfregi;</div><div class='ctx'> 	for (i = 0; i &lt; bfregi-&gt;num_static_sys_pages; i++) {</div><div class='del'>-		err = mlx5_cmd_alloc_uar(dev-&gt;mdev, &amp;bfregi-&gt;sys_pages[i]);</div><div class='add'>+		err = mlx5_cmd_uar_alloc(dev-&gt;mdev, &amp;bfregi-&gt;sys_pages[i],</div><div class='add'>+					 context-&gt;devx_uid);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto error;</div><div class='ctx'> </div><div class='hunk'>@@ -1661,7 +1821,8 @@ static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *conte</div><div class='ctx'> </div><div class='ctx'> error:</div><div class='ctx'> 	for (--i; i &gt;= 0; i--)</div><div class='del'>-		if (mlx5_cmd_free_uar(dev-&gt;mdev, bfregi-&gt;sys_pages[i]))</div><div class='add'>+		if (mlx5_cmd_uar_dealloc(dev-&gt;mdev, bfregi-&gt;sys_pages[i],</div><div class='add'>+					 context-&gt;devx_uid))</div><div class='ctx'> 			mlx5_ib_warn(dev, "failed to free uar %d\n", i);</div><div class='ctx'> </div><div class='ctx'> 	return err;</div><div class='hunk'>@@ -1677,13 +1838,49 @@ static void deallocate_uars(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	for (i = 0; i &lt; bfregi-&gt;num_sys_pages; i++)</div><div class='ctx'> 		if (i &lt; bfregi-&gt;num_static_sys_pages ||</div><div class='ctx'> 		    bfregi-&gt;sys_pages[i] != MLX5_IB_INVALID_UAR_INDEX)</div><div class='del'>-			mlx5_cmd_free_uar(dev-&gt;mdev, bfregi-&gt;sys_pages[i]);</div><div class='add'>+			mlx5_cmd_uar_dealloc(dev-&gt;mdev, bfregi-&gt;sys_pages[i],</div><div class='add'>+					     context-&gt;devx_uid);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_enable_lb_mp(struct mlx5_core_dev *master,</div><div class='add'>+				struct mlx5_core_dev *slave,</div><div class='add'>+				struct mlx5_ib_lb_state *lb_state)</div><div class='add'>+{</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	err = mlx5_nic_vport_update_local_lb(master, true);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = mlx5_nic_vport_update_local_lb(slave, true);</div><div class='add'>+	if (err)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	lb_state-&gt;force_enable = true;</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	mlx5_nic_vport_update_local_lb(master, false);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_disable_lb_mp(struct mlx5_core_dev *master,</div><div class='add'>+				  struct mlx5_core_dev *slave,</div><div class='add'>+				  struct mlx5_ib_lb_state *lb_state)</div><div class='add'>+{</div><div class='add'>+	mlx5_nic_vport_update_local_lb(slave, false);</div><div class='add'>+	mlx5_nic_vport_update_local_lb(master, false);</div><div class='add'>+</div><div class='add'>+	lb_state-&gt;force_enable = false;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)</div><div class='ctx'> {</div><div class='ctx'> 	int err = 0;</div><div class='ctx'> </div><div class='add'>+	if (dev-&gt;lb.force_enable)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='ctx'> 	mutex_lock(&amp;dev-&gt;lb.mutex);</div><div class='ctx'> 	if (td)</div><div class='ctx'> 		dev-&gt;lb.user_td++;</div><div class='hunk'>@@ -1705,6 +1902,9 @@ int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)</div><div class='ctx'> </div><div class='ctx'> void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)</div><div class='ctx'> {</div><div class='add'>+	if (dev-&gt;lb.force_enable)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='ctx'> 	mutex_lock(&amp;dev-&gt;lb.mutex);</div><div class='ctx'> 	if (td)</div><div class='ctx'> 		dev-&gt;lb.user_td--;</div><div class='hunk'>@@ -1758,6 +1958,90 @@ static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn,</div><div class='ctx'> 	mlx5_ib_disable_lb(dev, true, false);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int set_ucontext_resp(struct ib_ucontext *uctx,</div><div class='add'>+			     struct mlx5_ib_alloc_ucontext_resp *resp)</div><div class='add'>+{</div><div class='add'>+	struct ib_device *ibdev = uctx-&gt;device;</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='add'>+	struct mlx5_ib_ucontext *context = to_mucontext(uctx);</div><div class='add'>+	struct mlx5_bfreg_info *bfregi = &amp;context-&gt;bfregi;</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, dump_fill_mkey)) {</div><div class='add'>+		resp-&gt;dump_fill_mkey = dev-&gt;mkeys.dump_fill_mkey;</div><div class='add'>+		resp-&gt;comp_mask |=</div><div class='add'>+			MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	resp-&gt;qp_tab_size = 1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev, log_max_qp);</div><div class='add'>+	if (mlx5_wc_support_get(dev-&gt;mdev))</div><div class='add'>+		resp-&gt;bf_reg_size = 1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev,</div><div class='add'>+						      log_bf_reg_size);</div><div class='add'>+	resp-&gt;cache_line_size = cache_line_size();</div><div class='add'>+	resp-&gt;max_sq_desc_sz = MLX5_CAP_GEN(dev-&gt;mdev, max_wqe_sz_sq);</div><div class='add'>+	resp-&gt;max_rq_desc_sz = MLX5_CAP_GEN(dev-&gt;mdev, max_wqe_sz_rq);</div><div class='add'>+	resp-&gt;max_send_wqebb = 1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev, log_max_qp_sz);</div><div class='add'>+	resp-&gt;max_recv_wr = 1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev, log_max_qp_sz);</div><div class='add'>+	resp-&gt;max_srq_recv_wr = 1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev, log_max_srq_sz);</div><div class='add'>+	resp-&gt;cqe_version = context-&gt;cqe_version;</div><div class='add'>+	resp-&gt;log_uar_size = MLX5_CAP_GEN(dev-&gt;mdev, uar_4k) ?</div><div class='add'>+				MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;</div><div class='add'>+	resp-&gt;num_uars_per_page = MLX5_CAP_GEN(dev-&gt;mdev, uar_4k) ?</div><div class='add'>+					MLX5_CAP_GEN(dev-&gt;mdev,</div><div class='add'>+						     num_of_uars_per_page) : 1;</div><div class='add'>+	resp-&gt;tot_bfregs = bfregi-&gt;lib_uar_dyn ? 0 :</div><div class='add'>+			bfregi-&gt;total_num_bfregs - bfregi-&gt;num_dyn_bfregs;</div><div class='add'>+	resp-&gt;num_ports = dev-&gt;num_ports;</div><div class='add'>+	resp-&gt;cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |</div><div class='add'>+				      MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;</div><div class='add'>+</div><div class='add'>+	if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {</div><div class='add'>+		mlx5_query_min_inline(dev-&gt;mdev, &amp;resp-&gt;eth_min_inline);</div><div class='add'>+		resp-&gt;eth_min_inline++;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;mdev-&gt;clock_info)</div><div class='add'>+		resp-&gt;clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * We don't want to expose information from the PCI bar that is located</div><div class='add'>+	 * after 4096 bytes, so if the arch only supports larger pages, let's</div><div class='add'>+	 * pretend we don't support reading the HCA's core clock. This is also</div><div class='add'>+	 * forced by mmap function.</div><div class='add'>+	 */</div><div class='add'>+	if (PAGE_SIZE &lt;= 4096) {</div><div class='add'>+		resp-&gt;comp_mask |=</div><div class='add'>+			MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;</div><div class='add'>+		resp-&gt;hca_core_clock_offset =</div><div class='add'>+			offsetof(struct mlx5_init_seg,</div><div class='add'>+				 internal_timer_h) % PAGE_SIZE;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, ece_support))</div><div class='add'>+		resp-&gt;comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE;</div><div class='add'>+</div><div class='add'>+	if (rt_supported(MLX5_CAP_GEN(dev-&gt;mdev, sq_ts_format)) &amp;&amp;</div><div class='add'>+	    rt_supported(MLX5_CAP_GEN(dev-&gt;mdev, rq_ts_format)) &amp;&amp;</div><div class='add'>+	    rt_supported(MLX5_CAP_ROCE(dev-&gt;mdev, qp_ts_format)))</div><div class='add'>+		resp-&gt;comp_mask |=</div><div class='add'>+			MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_REAL_TIME_TS;</div><div class='add'>+</div><div class='add'>+	resp-&gt;num_dyn_bfregs = bfregi-&gt;num_dyn_bfregs;</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, drain_sigerr))</div><div class='add'>+		resp-&gt;comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_SQD2RTS;</div><div class='add'>+</div><div class='add'>+	resp-&gt;comp_mask |=</div><div class='add'>+		MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_MKEY_UPDATE_TAG;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool uctx_rdma_ctrl_is_enabled(u64 enabled_caps)</div><div class='add'>+{</div><div class='add'>+	return UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_LOCAL) ||</div><div class='add'>+	       UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,</div><div class='ctx'> 				  struct ib_udata *udata)</div><div class='ctx'> {</div><div class='hunk'>@@ -1765,15 +2049,14 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='ctx'> 	struct mlx5_ib_alloc_ucontext_req_v2 req = {};</div><div class='ctx'> 	struct mlx5_ib_alloc_ucontext_resp resp = {};</div><div class='del'>-	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='ctx'> 	struct mlx5_ib_ucontext *context = to_mucontext(uctx);</div><div class='ctx'> 	struct mlx5_bfreg_info *bfregi;</div><div class='ctx'> 	int ver;</div><div class='ctx'> 	int err;</div><div class='ctx'> 	size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,</div><div class='ctx'> 				     max_cqe_version);</div><div class='del'>-	u32 dump_fill_mkey;</div><div class='ctx'> 	bool lib_uar_4k;</div><div class='add'>+	bool lib_uar_dyn;</div><div class='ctx'> </div><div class='ctx'> 	if (!dev-&gt;ib_active)</div><div class='ctx'> 		return -EAGAIN;</div><div class='hunk'>@@ -1800,44 +2083,33 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,</div><div class='ctx'> 	if (req.num_low_latency_bfregs &gt; req.total_num_bfregs - 1)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	resp.qp_tab_size = 1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev, log_max_qp);</div><div class='del'>-	if (mlx5_core_is_pf(dev-&gt;mdev) &amp;&amp; MLX5_CAP_GEN(dev-&gt;mdev, bf))</div><div class='del'>-		resp.bf_reg_size = 1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev, log_bf_reg_size);</div><div class='del'>-	resp.cache_line_size = cache_line_size();</div><div class='del'>-	resp.max_sq_desc_sz = MLX5_CAP_GEN(dev-&gt;mdev, max_wqe_sz_sq);</div><div class='del'>-	resp.max_rq_desc_sz = MLX5_CAP_GEN(dev-&gt;mdev, max_wqe_sz_rq);</div><div class='del'>-	resp.max_send_wqebb = 1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev, log_max_qp_sz);</div><div class='del'>-	resp.max_recv_wr = 1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev, log_max_qp_sz);</div><div class='del'>-	resp.max_srq_recv_wr = 1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev, log_max_srq_sz);</div><div class='del'>-	resp.cqe_version = min_t(__u8,</div><div class='del'>-				 (__u8)MLX5_CAP_GEN(dev-&gt;mdev, cqe_version),</div><div class='del'>-				 req.max_cqe_version);</div><div class='del'>-	resp.log_uar_size = MLX5_CAP_GEN(dev-&gt;mdev, uar_4k) ?</div><div class='del'>-				MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;</div><div class='del'>-	resp.num_uars_per_page = MLX5_CAP_GEN(dev-&gt;mdev, uar_4k) ?</div><div class='del'>-					MLX5_CAP_GEN(dev-&gt;mdev, num_of_uars_per_page) : 1;</div><div class='del'>-	resp.response_length = min(offsetof(typeof(resp), response_length) +</div><div class='del'>-				   sizeof(resp.response_length), udata-&gt;outlen);</div><div class='del'>-</div><div class='del'>-	if (mlx5_accel_ipsec_device_caps(dev-&gt;mdev) &amp; MLX5_ACCEL_IPSEC_CAP_DEVICE) {</div><div class='del'>-		if (mlx5_get_flow_namespace(dev-&gt;mdev, MLX5_FLOW_NAMESPACE_EGRESS))</div><div class='del'>-			resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM;</div><div class='del'>-		if (mlx5_accel_ipsec_device_caps(dev-&gt;mdev) &amp; MLX5_ACCEL_IPSEC_CAP_REQUIRED_METADATA)</div><div class='del'>-			resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_REQ_METADATA;</div><div class='del'>-		if (MLX5_CAP_FLOWTABLE(dev-&gt;mdev, flow_table_properties_nic_receive.ft_field_support.outer_esp_spi))</div><div class='del'>-			resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_SPI_STEERING;</div><div class='del'>-		if (mlx5_accel_ipsec_device_caps(dev-&gt;mdev) &amp; MLX5_ACCEL_IPSEC_CAP_TX_IV_IS_ESN)</div><div class='del'>-			resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_TX_IV_IS_ESN;</div><div class='del'>-		/* MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_FULL_OFFLOAD is currently always 0 */</div><div class='add'>+	if (req.flags &amp; MLX5_IB_ALLOC_UCTX_DEVX) {</div><div class='add'>+		err = mlx5_ib_devx_create(dev, true, uctx-&gt;enabled_caps);</div><div class='add'>+		if (err &lt; 0)</div><div class='add'>+			goto out_ctx;</div><div class='add'>+		context-&gt;devx_uid = err;</div><div class='add'>+</div><div class='add'>+		if (uctx_rdma_ctrl_is_enabled(uctx-&gt;enabled_caps)) {</div><div class='add'>+			err = mlx5_cmd_add_privileged_uid(dev-&gt;mdev,</div><div class='add'>+							  context-&gt;devx_uid);</div><div class='add'>+			if (err)</div><div class='add'>+				goto out_devx;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	lib_uar_4k = req.lib_caps &amp; MLX5_LIB_CAP_4K_UAR;</div><div class='add'>+	lib_uar_dyn = req.lib_caps &amp; MLX5_LIB_CAP_DYN_UAR;</div><div class='ctx'> 	bfregi = &amp;context-&gt;bfregi;</div><div class='ctx'> </div><div class='add'>+	if (lib_uar_dyn) {</div><div class='add'>+		bfregi-&gt;lib_uar_dyn = lib_uar_dyn;</div><div class='add'>+		goto uar_done;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	/* updates req-&gt;total_num_bfregs */</div><div class='ctx'> 	err = calc_total_bfregs(dev, lib_uar_4k, &amp;req, bfregi);</div><div class='ctx'> 	if (err)</div><div class='del'>-		goto out_ctx;</div><div class='add'>+		goto out_ucap;</div><div class='ctx'> </div><div class='ctx'> 	mutex_init(&amp;bfregi-&gt;lock);</div><div class='ctx'> 	bfregi-&gt;lib_uar_4k = lib_uar_4k;</div><div class='hunk'>@@ -1845,7 +2117,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,</div><div class='ctx'> 				GFP_KERNEL);</div><div class='ctx'> 	if (!bfregi-&gt;count) {</div><div class='ctx'> 		err = -ENOMEM;</div><div class='del'>-		goto out_ctx;</div><div class='add'>+		goto out_ucap;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	bfregi-&gt;sys_pages = kcalloc(bfregi-&gt;num_sys_pages,</div><div class='hunk'>@@ -1860,105 +2132,35 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto out_sys_pages;</div><div class='ctx'> </div><div class='del'>-	if (ibdev-&gt;attrs.device_cap_flags &amp; IB_DEVICE_ON_DEMAND_PAGING)</div><div class='del'>-		context-&gt;ibucontext.invalidate_range =</div><div class='del'>-			&amp;mlx5_ib_invalidate_range;</div><div class='del'>-</div><div class='del'>-	if (req.flags &amp; MLX5_IB_ALLOC_UCTX_DEVX) {</div><div class='del'>-		err = mlx5_ib_devx_create(dev, true);</div><div class='del'>-		if (err &lt; 0)</div><div class='del'>-			goto out_uars;</div><div class='del'>-		context-&gt;devx_uid = err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='add'>+uar_done:</div><div class='ctx'> 	err = mlx5_ib_alloc_transport_domain(dev, &amp;context-&gt;tdn,</div><div class='ctx'> 					     context-&gt;devx_uid);</div><div class='ctx'> 	if (err)</div><div class='del'>-		goto out_devx;</div><div class='del'>-</div><div class='del'>-	if (MLX5_CAP_GEN(dev-&gt;mdev, dump_fill_mkey)) {</div><div class='del'>-		err = mlx5_cmd_dump_fill_mkey(dev-&gt;mdev, &amp;dump_fill_mkey);</div><div class='del'>-		if (err)</div><div class='del'>-			goto out_mdev;</div><div class='del'>-	}</div><div class='add'>+		goto out_uars;</div><div class='ctx'> </div><div class='ctx'> 	INIT_LIST_HEAD(&amp;context-&gt;db_page_list);</div><div class='ctx'> 	mutex_init(&amp;context-&gt;db_page_mutex);</div><div class='ctx'> </div><div class='del'>-	resp.tot_bfregs = req.total_num_bfregs;</div><div class='del'>-	resp.num_ports = dev-&gt;num_ports;</div><div class='del'>-</div><div class='del'>-	if (field_avail(typeof(resp), cqe_version, udata-&gt;outlen))</div><div class='del'>-		resp.response_length += sizeof(resp.cqe_version);</div><div class='del'>-</div><div class='del'>-	if (field_avail(typeof(resp), cmds_supp_uhw, udata-&gt;outlen)) {</div><div class='del'>-		resp.cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |</div><div class='del'>-				      MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;</div><div class='del'>-		resp.response_length += sizeof(resp.cmds_supp_uhw);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (field_avail(typeof(resp), eth_min_inline, udata-&gt;outlen)) {</div><div class='del'>-		if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {</div><div class='del'>-			mlx5_query_min_inline(dev-&gt;mdev, &amp;resp.eth_min_inline);</div><div class='del'>-			resp.eth_min_inline++;</div><div class='del'>-		}</div><div class='del'>-		resp.response_length += sizeof(resp.eth_min_inline);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (field_avail(typeof(resp), clock_info_versions, udata-&gt;outlen)) {</div><div class='del'>-		if (mdev-&gt;clock_info)</div><div class='del'>-			resp.clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);</div><div class='del'>-		resp.response_length += sizeof(resp.clock_info_versions);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * We don't want to expose information from the PCI bar that is located</div><div class='del'>-	 * after 4096 bytes, so if the arch only supports larger pages, let's</div><div class='del'>-	 * pretend we don't support reading the HCA's core clock. This is also</div><div class='del'>-	 * forced by mmap function.</div><div class='del'>-	 */</div><div class='del'>-	if (field_avail(typeof(resp), hca_core_clock_offset, udata-&gt;outlen)) {</div><div class='del'>-		if (PAGE_SIZE &lt;= 4096) {</div><div class='del'>-			resp.comp_mask |=</div><div class='del'>-				MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;</div><div class='del'>-			resp.hca_core_clock_offset =</div><div class='del'>-				offsetof(struct mlx5_init_seg, internal_timer_h) % PAGE_SIZE;</div><div class='del'>-		}</div><div class='del'>-		resp.response_length += sizeof(resp.hca_core_clock_offset);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (field_avail(typeof(resp), log_uar_size, udata-&gt;outlen))</div><div class='del'>-		resp.response_length += sizeof(resp.log_uar_size);</div><div class='del'>-</div><div class='del'>-	if (field_avail(typeof(resp), num_uars_per_page, udata-&gt;outlen))</div><div class='del'>-		resp.response_length += sizeof(resp.num_uars_per_page);</div><div class='del'>-</div><div class='del'>-	if (field_avail(typeof(resp), num_dyn_bfregs, udata-&gt;outlen)) {</div><div class='del'>-		resp.num_dyn_bfregs = bfregi-&gt;num_dyn_bfregs;</div><div class='del'>-		resp.response_length += sizeof(resp.num_dyn_bfregs);</div><div class='del'>-	}</div><div class='add'>+	context-&gt;cqe_version = min_t(__u8,</div><div class='add'>+				 (__u8)MLX5_CAP_GEN(dev-&gt;mdev, cqe_version),</div><div class='add'>+				 req.max_cqe_version);</div><div class='ctx'> </div><div class='del'>-	if (field_avail(typeof(resp), dump_fill_mkey, udata-&gt;outlen)) {</div><div class='del'>-		if (MLX5_CAP_GEN(dev-&gt;mdev, dump_fill_mkey)) {</div><div class='del'>-			resp.dump_fill_mkey = dump_fill_mkey;</div><div class='del'>-			resp.comp_mask |=</div><div class='del'>-				MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY;</div><div class='del'>-		}</div><div class='del'>-		resp.response_length += sizeof(resp.dump_fill_mkey);</div><div class='del'>-	}</div><div class='add'>+	err = set_ucontext_resp(uctx, &amp;resp);</div><div class='add'>+	if (err)</div><div class='add'>+		goto out_mdev;</div><div class='ctx'> </div><div class='add'>+	resp.response_length = min(udata-&gt;outlen, sizeof(resp));</div><div class='ctx'> 	err = ib_copy_to_udata(udata, &amp;resp, resp.response_length);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto out_mdev;</div><div class='ctx'> </div><div class='ctx'> 	bfregi-&gt;ver = ver;</div><div class='ctx'> 	bfregi-&gt;num_low_latency_bfregs = req.num_low_latency_bfregs;</div><div class='del'>-	context-&gt;cqe_version = resp.cqe_version;</div><div class='ctx'> 	context-&gt;lib_caps = req.lib_caps;</div><div class='ctx'> 	print_lib_caps(dev, context-&gt;lib_caps);</div><div class='ctx'> </div><div class='del'>-	if (dev-&gt;lag_active) {</div><div class='del'>-		u8 port = mlx5_core_native_port_num(dev-&gt;mdev) - 1;</div><div class='add'>+	if (mlx5_ib_lag_should_assign_affinity(dev)) {</div><div class='add'>+		u32 port = mlx5_core_native_port_num(dev-&gt;mdev) - 1;</div><div class='ctx'> </div><div class='ctx'> 		atomic_set(&amp;context-&gt;tx_port_affinity,</div><div class='ctx'> 			   atomic_add_return(</div><div class='hunk'>@@ -1969,9 +2171,6 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,</div><div class='ctx'> </div><div class='ctx'> out_mdev:</div><div class='ctx'> 	mlx5_ib_dealloc_transport_domain(dev, context-&gt;tdn, context-&gt;devx_uid);</div><div class='del'>-out_devx:</div><div class='del'>-	if (req.flags &amp; MLX5_IB_ALLOC_UCTX_DEVX)</div><div class='del'>-		mlx5_ib_devx_destroy(dev, context-&gt;devx_uid);</div><div class='ctx'> </div><div class='ctx'> out_uars:</div><div class='ctx'> 	deallocate_uars(dev, context);</div><div class='hunk'>@@ -1982,30 +2181,61 @@ out_sys_pages:</div><div class='ctx'> out_count:</div><div class='ctx'> 	kfree(bfregi-&gt;count);</div><div class='ctx'> </div><div class='add'>+out_ucap:</div><div class='add'>+	if (req.flags &amp; MLX5_IB_ALLOC_UCTX_DEVX &amp;&amp;</div><div class='add'>+	    uctx_rdma_ctrl_is_enabled(uctx-&gt;enabled_caps))</div><div class='add'>+		mlx5_cmd_remove_privileged_uid(dev-&gt;mdev, context-&gt;devx_uid);</div><div class='add'>+</div><div class='add'>+out_devx:</div><div class='add'>+	if (req.flags &amp; MLX5_IB_ALLOC_UCTX_DEVX)</div><div class='add'>+		mlx5_ib_devx_destroy(dev, context-&gt;devx_uid);</div><div class='add'>+</div><div class='ctx'> out_ctx:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int mlx5_ib_query_ucontext(struct ib_ucontext *ibcontext,</div><div class='add'>+				  struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_alloc_ucontext_resp uctx_resp = {};</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = set_ucontext_resp(ibcontext, &amp;uctx_resp);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	uctx_resp.response_length =</div><div class='add'>+		min_t(size_t,</div><div class='add'>+		      uverbs_attr_get_len(attrs,</div><div class='add'>+				MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX),</div><div class='add'>+		      sizeof(uctx_resp));</div><div class='add'>+</div><div class='add'>+	ret = uverbs_copy_to_struct_or_zero(attrs,</div><div class='add'>+					MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,</div><div class='add'>+					&amp;uctx_resp,</div><div class='add'>+					sizeof(uctx_resp));</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static void mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(ibcontext-&gt;device);</div><div class='ctx'> 	struct mlx5_bfreg_info *bfregi;</div><div class='ctx'> </div><div class='del'>-	/* All umem's must be destroyed before destroying the ucontext. */</div><div class='del'>-	mutex_lock(&amp;ibcontext-&gt;per_mm_list_lock);</div><div class='del'>-	WARN_ON(!list_empty(&amp;ibcontext-&gt;per_mm_list));</div><div class='del'>-	mutex_unlock(&amp;ibcontext-&gt;per_mm_list_lock);</div><div class='del'>-</div><div class='ctx'> 	bfregi = &amp;context-&gt;bfregi;</div><div class='ctx'> 	mlx5_ib_dealloc_transport_domain(dev, context-&gt;tdn, context-&gt;devx_uid);</div><div class='ctx'> </div><div class='del'>-	if (context-&gt;devx_uid)</div><div class='del'>-		mlx5_ib_devx_destroy(dev, context-&gt;devx_uid);</div><div class='del'>-</div><div class='ctx'> 	deallocate_uars(dev, context);</div><div class='ctx'> 	kfree(bfregi-&gt;sys_pages);</div><div class='ctx'> 	kfree(bfregi-&gt;count);</div><div class='add'>+</div><div class='add'>+	if (context-&gt;devx_uid) {</div><div class='add'>+		if (uctx_rdma_ctrl_is_enabled(ibcontext-&gt;enabled_caps))</div><div class='add'>+			mlx5_cmd_remove_privileged_uid(dev-&gt;mdev,</div><div class='add'>+						       context-&gt;devx_uid);</div><div class='add'>+		mlx5_ib_devx_destroy(dev, context-&gt;devx_uid);</div><div class='add'>+	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,</div><div class='hunk'>@@ -2018,6 +2248,17 @@ static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	return (dev-&gt;mdev-&gt;bar_addr &gt;&gt; PAGE_SHIFT) + uar_idx / fw_uars_per_page;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static u64 uar_index2paddress(struct mlx5_ib_dev *dev,</div><div class='add'>+				 int uar_idx)</div><div class='add'>+{</div><div class='add'>+	unsigned int fw_uars_per_page;</div><div class='add'>+</div><div class='add'>+	fw_uars_per_page = MLX5_CAP_GEN(dev-&gt;mdev, uar_4k) ?</div><div class='add'>+				MLX5_UARS_IN_PAGE : 1;</div><div class='add'>+</div><div class='add'>+	return (dev-&gt;mdev-&gt;bar_addr + (uar_idx / fw_uars_per_page) * PAGE_SIZE);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int get_command(unsigned long offset)</div><div class='ctx'> {</div><div class='ctx'> 	return (offset &gt;&gt; MLX5_IB_MMAP_CMD_SHIFT) &amp; MLX5_IB_MMAP_CMD_MASK;</div><div class='hunk'>@@ -2056,7 +2297,7 @@ static inline char *mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)</div><div class='ctx'> 	case MLX5_IB_MMAP_DEVICE_MEM:</div><div class='ctx'> 		return "Device Memory";</div><div class='ctx'> 	default:</div><div class='del'>-		return NULL;</div><div class='add'>+		return "Unknown";</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -2073,7 +2314,7 @@ static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev,</div><div class='ctx'> </div><div class='ctx'> 	if (vma-&gt;vm_flags &amp; (VM_WRITE | VM_EXEC))</div><div class='ctx'> 		return -EPERM;</div><div class='del'>-	vma-&gt;vm_flags &amp;= ~VM_MAYWRITE;</div><div class='add'>+	vm_flags_clear(vma, VM_MAYWRITE);</div><div class='ctx'> </div><div class='ctx'> 	if (!dev-&gt;mdev-&gt;clock_info)</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='hunk'>@@ -2082,6 +2323,35 @@ static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev,</div><div class='ctx'> 			      virt_to_page(dev-&gt;mdev-&gt;clock_info));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_user_mmap_entry *mentry = to_mmmap(entry);</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(entry-&gt;ucontext-&gt;device);</div><div class='add'>+	struct mlx5_var_table *var_table = &amp;dev-&gt;var_table;</div><div class='add'>+	struct mlx5_ib_ucontext *context = to_mucontext(entry-&gt;ucontext);</div><div class='add'>+</div><div class='add'>+	switch (mentry-&gt;mmap_flag) {</div><div class='add'>+	case MLX5_IB_MMAP_TYPE_MEMIC:</div><div class='add'>+	case MLX5_IB_MMAP_TYPE_MEMIC_OP:</div><div class='add'>+		mlx5_ib_dm_mmap_free(dev, mentry);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_MMAP_TYPE_VAR:</div><div class='add'>+		mutex_lock(&amp;var_table-&gt;bitmap_lock);</div><div class='add'>+		clear_bit(mentry-&gt;page_idx, var_table-&gt;bitmap);</div><div class='add'>+		mutex_unlock(&amp;var_table-&gt;bitmap_lock);</div><div class='add'>+		kfree(mentry);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_MMAP_TYPE_UAR_WC:</div><div class='add'>+	case MLX5_IB_MMAP_TYPE_UAR_NC:</div><div class='add'>+		mlx5_cmd_uar_dealloc(dev-&gt;mdev, mentry-&gt;page_idx,</div><div class='add'>+				     context-&gt;devx_uid);</div><div class='add'>+		kfree(mentry);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		WARN_ON(true);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,</div><div class='ctx'> 		    struct vm_area_struct *vma,</div><div class='ctx'> 		    struct mlx5_ib_ucontext *context)</div><div class='hunk'>@@ -2097,6 +2367,9 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,</div><div class='ctx'> 	int max_valid_idx = dyn_uar ? bfregi-&gt;num_sys_pages :</div><div class='ctx'> 				bfregi-&gt;num_static_sys_pages;</div><div class='ctx'> </div><div class='add'>+	if (bfregi-&gt;lib_uar_dyn)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='ctx'> 	if (vma-&gt;vm_end - vma-&gt;vm_start != PAGE_SIZE)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='hunk'>@@ -2114,14 +2387,6 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,</div><div class='ctx'> 	switch (cmd) {</div><div class='ctx'> 	case MLX5_IB_MMAP_WC_PAGE:</div><div class='ctx'> 	case MLX5_IB_MMAP_ALLOC_WC:</div><div class='del'>-/* Some architectures don't support WC memory */</div><div class='del'>-#if defined(CONFIG_X86)</div><div class='del'>-		if (!pat_enabled())</div><div class='del'>-			return -EPERM;</div><div class='del'>-#elif !(defined(CONFIG_PPC) || (defined(CONFIG_ARM) &amp;&amp; defined(CONFIG_MMU)))</div><div class='del'>-			return -EPERM;</div><div class='del'>-#endif</div><div class='del'>-	/* fall through */</div><div class='ctx'> 	case MLX5_IB_MMAP_REGULAR_PAGE:</div><div class='ctx'> 		/* For MLX5_IB_MMAP_REGULAR_PAGE do the best effort to get WC */</div><div class='ctx'> 		prot = pgprot_writecombine(vma-&gt;vm_page_prot);</div><div class='hunk'>@@ -2157,7 +2422,8 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,</div><div class='ctx'> 		bfregi-&gt;count[bfreg_dyn_idx]++;</div><div class='ctx'> 		mutex_unlock(&amp;bfregi-&gt;lock);</div><div class='ctx'> </div><div class='del'>-		err = mlx5_cmd_alloc_uar(dev-&gt;mdev, &amp;uar_index);</div><div class='add'>+		err = mlx5_cmd_uar_alloc(dev-&gt;mdev, &amp;uar_index,</div><div class='add'>+					 context-&gt;devx_uid);</div><div class='ctx'> 		if (err) {</div><div class='ctx'> 			mlx5_ib_warn(dev, "UAR alloc failed\n");</div><div class='ctx'> 			goto free_bfreg;</div><div class='hunk'>@@ -2170,7 +2436,7 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,</div><div class='ctx'> 	mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &amp;pfn);</div><div class='ctx'> </div><div class='ctx'> 	err = rdma_user_mmap_io(&amp;context-&gt;ibucontext, vma, pfn, PAGE_SIZE,</div><div class='del'>-				prot);</div><div class='add'>+				prot, NULL);</div><div class='ctx'> 	if (err) {</div><div class='ctx'> 		mlx5_ib_err(dev,</div><div class='ctx'> 			    "rdma_user_mmap_io failed with error=%d, mmap_cmd=%s\n",</div><div class='hunk'>@@ -2186,7 +2452,7 @@ err:</div><div class='ctx'> 	if (!dyn_uar)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='del'>-	mlx5_cmd_free_uar(dev-&gt;mdev, idx);</div><div class='add'>+	mlx5_cmd_uar_dealloc(dev-&gt;mdev, idx, context-&gt;devx_uid);</div><div class='ctx'> </div><div class='ctx'> free_bfreg:</div><div class='ctx'> 	mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx);</div><div class='hunk'>@@ -2194,25 +2460,55 @@ free_bfreg:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int dm_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)</div><div class='add'>+static unsigned long mlx5_vma_to_pgoff(struct vm_area_struct *vma)</div><div class='add'>+{</div><div class='add'>+	unsigned long idx;</div><div class='add'>+	u8 command;</div><div class='add'>+</div><div class='add'>+	command = get_command(vma-&gt;vm_pgoff);</div><div class='add'>+	idx = get_extended_index(vma-&gt;vm_pgoff);</div><div class='add'>+</div><div class='add'>+	return (command &lt;&lt; 16 | idx);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_mmap_offset(struct mlx5_ib_dev *dev,</div><div class='add'>+			       struct vm_area_struct *vma,</div><div class='add'>+			       struct ib_ucontext *ucontext)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_ucontext *mctx = to_mucontext(context);</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(context-&gt;device);</div><div class='del'>-	u16 page_idx = get_extended_index(vma-&gt;vm_pgoff);</div><div class='del'>-	size_t map_size = vma-&gt;vm_end - vma-&gt;vm_start;</div><div class='del'>-	u32 npages = map_size &gt;&gt; PAGE_SHIFT;</div><div class='add'>+	struct mlx5_user_mmap_entry *mentry;</div><div class='add'>+	struct rdma_user_mmap_entry *entry;</div><div class='add'>+	unsigned long pgoff;</div><div class='add'>+	pgprot_t prot;</div><div class='ctx'> 	phys_addr_t pfn;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	if (find_next_zero_bit(mctx-&gt;dm_pages, page_idx + npages, page_idx) !=</div><div class='del'>-	    page_idx + npages)</div><div class='add'>+	pgoff = mlx5_vma_to_pgoff(vma);</div><div class='add'>+	entry = rdma_user_mmap_entry_get_pgoff(ucontext, pgoff);</div><div class='add'>+	if (!entry)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	pfn = ((dev-&gt;mdev-&gt;bar_addr +</div><div class='del'>-	      MLX5_CAP64_DEV_MEM(dev-&gt;mdev, memic_bar_start_addr)) &gt;&gt;</div><div class='del'>-	      PAGE_SHIFT) +</div><div class='del'>-	      page_idx;</div><div class='del'>-	return rdma_user_mmap_io(context, vma, pfn, map_size,</div><div class='del'>-				 pgprot_writecombine(vma-&gt;vm_page_prot));</div><div class='add'>+	mentry = to_mmmap(entry);</div><div class='add'>+	pfn = (mentry-&gt;address &gt;&gt; PAGE_SHIFT);</div><div class='add'>+	if (mentry-&gt;mmap_flag == MLX5_IB_MMAP_TYPE_VAR ||</div><div class='add'>+	    mentry-&gt;mmap_flag == MLX5_IB_MMAP_TYPE_UAR_NC)</div><div class='add'>+		prot = pgprot_noncached(vma-&gt;vm_page_prot);</div><div class='add'>+	else</div><div class='add'>+		prot = pgprot_writecombine(vma-&gt;vm_page_prot);</div><div class='add'>+	ret = rdma_user_mmap_io(ucontext, vma, pfn,</div><div class='add'>+				entry-&gt;npages * PAGE_SIZE,</div><div class='add'>+				prot,</div><div class='add'>+				entry);</div><div class='add'>+	rdma_user_mmap_entry_put(&amp;mentry-&gt;rdma_entry);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u64 mlx5_entry_to_mmap_offset(struct mlx5_user_mmap_entry *entry)</div><div class='add'>+{</div><div class='add'>+	u64 cmd = (entry-&gt;rdma_entry.start_pgoff &gt;&gt; 16) &amp; 0xFFFF;</div><div class='add'>+	u64 index = entry-&gt;rdma_entry.start_pgoff &amp; 0xFFFF;</div><div class='add'>+</div><div class='add'>+	return (((index &gt;&gt; 8) &lt;&lt; 16) | (cmd &lt;&lt; MLX5_IB_MMAP_CMD_SHIFT) |</div><div class='add'>+		(index &amp; 0xFF)) &lt;&lt; PAGE_SHIFT;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)</div><div class='hunk'>@@ -2225,9 +2521,12 @@ static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vm</div><div class='ctx'> 	command = get_command(vma-&gt;vm_pgoff);</div><div class='ctx'> 	switch (command) {</div><div class='ctx'> 	case MLX5_IB_MMAP_WC_PAGE:</div><div class='add'>+	case MLX5_IB_MMAP_ALLOC_WC:</div><div class='add'>+		if (!mlx5_wc_support_get(dev-&gt;mdev))</div><div class='add'>+			return -EPERM;</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case MLX5_IB_MMAP_NC_PAGE:</div><div class='ctx'> 	case MLX5_IB_MMAP_REGULAR_PAGE:</div><div class='del'>-	case MLX5_IB_MMAP_ALLOC_WC:</div><div class='ctx'> 		return uar_mmap(dev, command, vma, context);</div><div class='ctx'> </div><div class='ctx'> 	case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:</div><div class='hunk'>@@ -2239,7 +2538,7 @@ static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vm</div><div class='ctx'> </div><div class='ctx'> 		if (vma-&gt;vm_flags &amp; VM_WRITE)</div><div class='ctx'> 			return -EPERM;</div><div class='del'>-		vma-&gt;vm_flags &amp;= ~VM_MAYWRITE;</div><div class='add'>+		vm_flags_clear(vma, VM_MAYWRITE);</div><div class='ctx'> </div><div class='ctx'> 		/* Don't expose to user-space information it shouldn't have */</div><div class='ctx'> 		if (PAGE_SIZE &gt; 4096)</div><div class='hunk'>@@ -2250,210 +2549,15 @@ static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vm</div><div class='ctx'> 			PAGE_SHIFT;</div><div class='ctx'> 		return rdma_user_mmap_io(&amp;context-&gt;ibucontext, vma, pfn,</div><div class='ctx'> 					 PAGE_SIZE,</div><div class='del'>-					 pgprot_noncached(vma-&gt;vm_page_prot));</div><div class='add'>+					 pgprot_noncached(vma-&gt;vm_page_prot),</div><div class='add'>+					 NULL);</div><div class='ctx'> 	case MLX5_IB_MMAP_CLOCK_INFO:</div><div class='ctx'> 		return mlx5_ib_mmap_clock_info_page(dev, vma, context);</div><div class='ctx'> </div><div class='del'>-	case MLX5_IB_MMAP_DEVICE_MEM:</div><div class='del'>-		return dm_mmap(ibcontext, vma);</div><div class='del'>-</div><div class='del'>-	default:</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline int check_dm_type_support(struct mlx5_ib_dev *dev,</div><div class='del'>-					u32 type)</div><div class='del'>-{</div><div class='del'>-	switch (type) {</div><div class='del'>-	case MLX5_IB_UAPI_DM_TYPE_MEMIC:</div><div class='del'>-		if (!MLX5_CAP_DEV_MEM(dev-&gt;mdev, memic))</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-		break;</div><div class='del'>-	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:</div><div class='del'>-		if (!capable(CAP_SYS_RAWIO) ||</div><div class='del'>-		    !capable(CAP_NET_RAW))</div><div class='del'>-			return -EPERM;</div><div class='del'>-</div><div class='del'>-		if (!(MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev, sw_owner) ||</div><div class='del'>-		      MLX5_CAP_FLOWTABLE_NIC_TX(dev-&gt;mdev, sw_owner)))</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int handle_alloc_dm_memic(struct ib_ucontext *ctx,</div><div class='del'>-				 struct mlx5_ib_dm *dm,</div><div class='del'>-				 struct ib_dm_alloc_attr *attr,</div><div class='del'>-				 struct uverbs_attr_bundle *attrs)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_dm *dm_db = &amp;to_mdev(ctx-&gt;device)-&gt;dm;</div><div class='del'>-	u64 start_offset;</div><div class='del'>-	u32 page_idx;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	dm-&gt;size = roundup(attr-&gt;length, MLX5_MEMIC_BASE_SIZE);</div><div class='del'>-</div><div class='del'>-	err = mlx5_cmd_alloc_memic(dm_db, &amp;dm-&gt;dev_addr,</div><div class='del'>-				   dm-&gt;size, attr-&gt;alignment);</div><div class='del'>-	if (err)</div><div class='del'>-		return err;</div><div class='del'>-</div><div class='del'>-	page_idx = (dm-&gt;dev_addr - pci_resource_start(dm_db-&gt;dev-&gt;pdev, 0) -</div><div class='del'>-		    MLX5_CAP64_DEV_MEM(dm_db-&gt;dev, memic_bar_start_addr)) &gt;&gt;</div><div class='del'>-		    PAGE_SHIFT;</div><div class='del'>-</div><div class='del'>-	err = uverbs_copy_to(attrs,</div><div class='del'>-			     MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,</div><div class='del'>-			     &amp;page_idx, sizeof(page_idx));</div><div class='del'>-	if (err)</div><div class='del'>-		goto err_dealloc;</div><div class='del'>-</div><div class='del'>-	start_offset = dm-&gt;dev_addr &amp; ~PAGE_MASK;</div><div class='del'>-	err = uverbs_copy_to(attrs,</div><div class='del'>-			     MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,</div><div class='del'>-			     &amp;start_offset, sizeof(start_offset));</div><div class='del'>-	if (err)</div><div class='del'>-		goto err_dealloc;</div><div class='del'>-</div><div class='del'>-	bitmap_set(to_mucontext(ctx)-&gt;dm_pages, page_idx,</div><div class='del'>-		   DIV_ROUND_UP(dm-&gt;size, PAGE_SIZE));</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err_dealloc:</div><div class='del'>-	mlx5_cmd_dealloc_memic(dm_db, dm-&gt;dev_addr, dm-&gt;size);</div><div class='del'>-</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int handle_alloc_dm_sw_icm(struct ib_ucontext *ctx,</div><div class='del'>-				  struct mlx5_ib_dm *dm,</div><div class='del'>-				  struct ib_dm_alloc_attr *attr,</div><div class='del'>-				  struct uverbs_attr_bundle *attrs,</div><div class='del'>-				  int type)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_dm *dm_db = &amp;to_mdev(ctx-&gt;device)-&gt;dm;</div><div class='del'>-	u64 act_size;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	/* Allocation size must a multiple of the basic block size</div><div class='del'>-	 * and a power of 2.</div><div class='del'>-	 */</div><div class='del'>-	act_size = roundup(attr-&gt;length, MLX5_SW_ICM_BLOCK_SIZE(dm_db-&gt;dev));</div><div class='del'>-	act_size = roundup_pow_of_two(act_size);</div><div class='del'>-</div><div class='del'>-	dm-&gt;size = act_size;</div><div class='del'>-	err = mlx5_cmd_alloc_sw_icm(dm_db, type, act_size,</div><div class='del'>-				    to_mucontext(ctx)-&gt;devx_uid, &amp;dm-&gt;dev_addr,</div><div class='del'>-				    &amp;dm-&gt;icm_dm.obj_id);</div><div class='del'>-	if (err)</div><div class='del'>-		return err;</div><div class='del'>-</div><div class='del'>-	err = uverbs_copy_to(attrs,</div><div class='del'>-			     MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,</div><div class='del'>-			     &amp;dm-&gt;dev_addr, sizeof(dm-&gt;dev_addr));</div><div class='del'>-	if (err)</div><div class='del'>-		mlx5_cmd_dealloc_sw_icm(dm_db, type, dm-&gt;size,</div><div class='del'>-					to_mucontext(ctx)-&gt;devx_uid,</div><div class='del'>-					dm-&gt;dev_addr, dm-&gt;icm_dm.obj_id);</div><div class='del'>-</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,</div><div class='del'>-			       struct ib_ucontext *context,</div><div class='del'>-			       struct ib_dm_alloc_attr *attr,</div><div class='del'>-			       struct uverbs_attr_bundle *attrs)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_dm *dm;</div><div class='del'>-	enum mlx5_ib_uapi_dm_type type;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	err = uverbs_get_const_default(&amp;type, attrs,</div><div class='del'>-				       MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,</div><div class='del'>-				       MLX5_IB_UAPI_DM_TYPE_MEMIC);</div><div class='del'>-	if (err)</div><div class='del'>-		return ERR_PTR(err);</div><div class='del'>-</div><div class='del'>-	mlx5_ib_dbg(to_mdev(ibdev), "alloc_dm req: dm_type=%d user_length=0x%llx log_alignment=%d\n",</div><div class='del'>-		    type, attr-&gt;length, attr-&gt;alignment);</div><div class='del'>-</div><div class='del'>-	err = check_dm_type_support(to_mdev(ibdev), type);</div><div class='del'>-	if (err)</div><div class='del'>-		return ERR_PTR(err);</div><div class='del'>-</div><div class='del'>-	dm = kzalloc(sizeof(*dm), GFP_KERNEL);</div><div class='del'>-	if (!dm)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	dm-&gt;type = type;</div><div class='del'>-</div><div class='del'>-	switch (type) {</div><div class='del'>-	case MLX5_IB_UAPI_DM_TYPE_MEMIC:</div><div class='del'>-		err = handle_alloc_dm_memic(context, dm,</div><div class='del'>-					    attr,</div><div class='del'>-					    attrs);</div><div class='del'>-		break;</div><div class='del'>-	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:</div><div class='del'>-	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:</div><div class='del'>-		err = handle_alloc_dm_sw_icm(context, dm, attr, attrs, type);</div><div class='del'>-		break;</div><div class='ctx'> 	default:</div><div class='del'>-		err = -EOPNOTSUPP;</div><div class='add'>+		return mlx5_ib_mmap_offset(dev, vma, ibcontext);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (err)</div><div class='del'>-		goto err_free;</div><div class='del'>-</div><div class='del'>-	return &amp;dm-&gt;ibdm;</div><div class='del'>-</div><div class='del'>-err_free:</div><div class='del'>-	kfree(dm);</div><div class='del'>-	return ERR_PTR(err);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int mlx5_ib_dealloc_dm(struct ib_dm *ibdm, struct uverbs_attr_bundle *attrs)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_ucontext *ctx = rdma_udata_to_drv_context(</div><div class='del'>-		&amp;attrs-&gt;driver_udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='del'>-	struct mlx5_dm *dm_db = &amp;to_mdev(ibdm-&gt;device)-&gt;dm;</div><div class='del'>-	struct mlx5_ib_dm *dm = to_mdm(ibdm);</div><div class='del'>-	u32 page_idx;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	switch (dm-&gt;type) {</div><div class='del'>-	case MLX5_IB_UAPI_DM_TYPE_MEMIC:</div><div class='del'>-		ret = mlx5_cmd_dealloc_memic(dm_db, dm-&gt;dev_addr, dm-&gt;size);</div><div class='del'>-		if (ret)</div><div class='del'>-			return ret;</div><div class='del'>-</div><div class='del'>-		page_idx = (dm-&gt;dev_addr -</div><div class='del'>-			    pci_resource_start(dm_db-&gt;dev-&gt;pdev, 0) -</div><div class='del'>-			    MLX5_CAP64_DEV_MEM(dm_db-&gt;dev,</div><div class='del'>-					       memic_bar_start_addr)) &gt;&gt;</div><div class='del'>-			   PAGE_SHIFT;</div><div class='del'>-		bitmap_clear(ctx-&gt;dm_pages, page_idx,</div><div class='del'>-			     DIV_ROUND_UP(dm-&gt;size, PAGE_SIZE));</div><div class='del'>-		break;</div><div class='del'>-	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:</div><div class='del'>-	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:</div><div class='del'>-		ret = mlx5_cmd_dealloc_sw_icm(dm_db, dm-&gt;type, dm-&gt;size,</div><div class='del'>-					      ctx-&gt;devx_uid, dm-&gt;dev_addr,</div><div class='del'>-					      dm-&gt;icm_dm.obj_id);</div><div class='del'>-		if (ret)</div><div class='del'>-			return ret;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	kfree(dm);</div><div class='del'>-</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -2464,7 +2568,7 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='ctx'> 	struct mlx5_ib_alloc_pd_resp resp;</div><div class='ctx'> 	int err;</div><div class='ctx'> 	u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {};</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(alloc_pd_in)]   = {};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {};</div><div class='ctx'> 	u16 uid = 0;</div><div class='ctx'> 	struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(</div><div class='ctx'> 		udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='hunk'>@@ -2472,8 +2576,7 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='ctx'> 	uid = context ? context-&gt;devx_uid : 0;</div><div class='ctx'> 	MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);</div><div class='ctx'> 	MLX5_SET(alloc_pd_in, in, uid, uid);</div><div class='del'>-	err = mlx5_cmd_exec(to_mdev(ibdev)-&gt;mdev, in, sizeof(in),</div><div class='del'>-			    out, sizeof(out));</div><div class='add'>+	err = mlx5_cmd_exec_inout(to_mdev(ibdev)-&gt;mdev, alloc_pd, in, out);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='hunk'>@@ -2490,1790 +2593,12 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void mlx5_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)</div><div class='add'>+static int mlx5_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *mdev = to_mdev(pd-&gt;device);</div><div class='ctx'> 	struct mlx5_ib_pd *mpd = to_mpd(pd);</div><div class='ctx'> </div><div class='del'>-	mlx5_cmd_dealloc_pd(mdev-&gt;mdev, mpd-&gt;pdn, mpd-&gt;uid);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-enum {</div><div class='del'>-	MATCH_CRITERIA_ENABLE_OUTER_BIT,</div><div class='del'>-	MATCH_CRITERIA_ENABLE_MISC_BIT,</div><div class='del'>-	MATCH_CRITERIA_ENABLE_INNER_BIT,</div><div class='del'>-	MATCH_CRITERIA_ENABLE_MISC2_BIT</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define HEADER_IS_ZERO(match_criteria, headers)			           \</div><div class='del'>-	!(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \</div><div class='del'>-		    0, MLX5_FLD_SZ_BYTES(fte_match_param, headers)))       \</div><div class='del'>-</div><div class='del'>-static u8 get_match_criteria_enable(u32 *match_criteria)</div><div class='del'>-{</div><div class='del'>-	u8 match_criteria_enable;</div><div class='del'>-</div><div class='del'>-	match_criteria_enable =</div><div class='del'>-		(!HEADER_IS_ZERO(match_criteria, outer_headers)) &lt;&lt;</div><div class='del'>-		MATCH_CRITERIA_ENABLE_OUTER_BIT;</div><div class='del'>-	match_criteria_enable |=</div><div class='del'>-		(!HEADER_IS_ZERO(match_criteria, misc_parameters)) &lt;&lt;</div><div class='del'>-		MATCH_CRITERIA_ENABLE_MISC_BIT;</div><div class='del'>-	match_criteria_enable |=</div><div class='del'>-		(!HEADER_IS_ZERO(match_criteria, inner_headers)) &lt;&lt;</div><div class='del'>-		MATCH_CRITERIA_ENABLE_INNER_BIT;</div><div class='del'>-	match_criteria_enable |=</div><div class='del'>-		(!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) &lt;&lt;</div><div class='del'>-		MATCH_CRITERIA_ENABLE_MISC2_BIT;</div><div class='del'>-</div><div class='del'>-	return match_criteria_enable;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)</div><div class='del'>-{</div><div class='del'>-	u8 entry_mask;</div><div class='del'>-	u8 entry_val;</div><div class='del'>-	int err = 0;</div><div class='del'>-</div><div class='del'>-	if (!mask)</div><div class='del'>-		goto out;</div><div class='del'>-</div><div class='del'>-	entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c,</div><div class='del'>-			      ip_protocol);</div><div class='del'>-	entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v,</div><div class='del'>-			     ip_protocol);</div><div class='del'>-	if (!entry_mask) {</div><div class='del'>-		MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);</div><div class='del'>-		MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='del'>-	/* Don't override existing ip protocol */</div><div class='del'>-	if (mask != entry_mask || val != entry_val)</div><div class='del'>-		err = -EINVAL;</div><div class='del'>-out:</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val,</div><div class='del'>-			   bool inner)</div><div class='del'>-{</div><div class='del'>-	if (inner) {</div><div class='del'>-		MLX5_SET(fte_match_set_misc,</div><div class='del'>-			 misc_c, inner_ipv6_flow_label, mask);</div><div class='del'>-		MLX5_SET(fte_match_set_misc,</div><div class='del'>-			 misc_v, inner_ipv6_flow_label, val);</div><div class='del'>-	} else {</div><div class='del'>-		MLX5_SET(fte_match_set_misc,</div><div class='del'>-			 misc_c, outer_ipv6_flow_label, mask);</div><div class='del'>-		MLX5_SET(fte_match_set_misc,</div><div class='del'>-			 misc_v, outer_ipv6_flow_label, val);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)</div><div class='del'>-{</div><div class='del'>-	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);</div><div class='del'>-	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);</div><div class='del'>-	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask &gt;&gt; 2);</div><div class='del'>-	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val &gt;&gt; 2);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask)</div><div class='del'>-{</div><div class='del'>-	if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) &amp;&amp;</div><div class='del'>-	    !(field_support &amp; MLX5_FIELD_SUPPORT_MPLS_LABEL))</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-	if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) &amp;&amp;</div><div class='del'>-	    !(field_support &amp; MLX5_FIELD_SUPPORT_MPLS_EXP))</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-	if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) &amp;&amp;</div><div class='del'>-	    !(field_support &amp; MLX5_FIELD_SUPPORT_MPLS_S_BOS))</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-	if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) &amp;&amp;</div><div class='del'>-	    !(field_support &amp; MLX5_FIELD_SUPPORT_MPLS_TTL))</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-#define LAST_ETH_FIELD vlan_tag</div><div class='del'>-#define LAST_IB_FIELD sl</div><div class='del'>-#define LAST_IPV4_FIELD tos</div><div class='del'>-#define LAST_IPV6_FIELD traffic_class</div><div class='del'>-#define LAST_TCP_UDP_FIELD src_port</div><div class='del'>-#define LAST_TUNNEL_FIELD tunnel_id</div><div class='del'>-#define LAST_FLOW_TAG_FIELD tag_id</div><div class='del'>-#define LAST_DROP_FIELD size</div><div class='del'>-#define LAST_COUNTERS_FIELD counters</div><div class='del'>-</div><div class='del'>-/* Field is the last supported field */</div><div class='del'>-#define FIELDS_NOT_SUPPORTED(filter, field)\</div><div class='del'>-	memchr_inv((void *)&amp;filter.field  +\</div><div class='del'>-		   sizeof(filter.field), 0,\</div><div class='del'>-		   sizeof(filter) -\</div><div class='del'>-		   offsetof(typeof(filter), field) -\</div><div class='del'>-		   sizeof(filter.field))</div><div class='del'>-</div><div class='del'>-int parse_flow_flow_action(struct mlx5_ib_flow_action *maction,</div><div class='del'>-			   bool is_egress,</div><div class='del'>-			   struct mlx5_flow_act *action)</div><div class='del'>-{</div><div class='del'>-</div><div class='del'>-	switch (maction-&gt;ib_action.type) {</div><div class='del'>-	case IB_FLOW_ACTION_ESP:</div><div class='del'>-		if (action-&gt;action &amp; (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT |</div><div class='del'>-				      MLX5_FLOW_CONTEXT_ACTION_DECRYPT))</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		/* Currently only AES_GCM keymat is supported by the driver */</div><div class='del'>-		action-&gt;esp_id = (uintptr_t)maction-&gt;esp_aes_gcm.ctx;</div><div class='del'>-		action-&gt;action |= is_egress ?</div><div class='del'>-			MLX5_FLOW_CONTEXT_ACTION_ENCRYPT :</div><div class='del'>-			MLX5_FLOW_CONTEXT_ACTION_DECRYPT;</div><div class='del'>-		return 0;</div><div class='del'>-	case IB_FLOW_ACTION_UNSPECIFIED:</div><div class='del'>-		if (maction-&gt;flow_action_raw.sub_type ==</div><div class='del'>-		    MLX5_IB_FLOW_ACTION_MODIFY_HEADER) {</div><div class='del'>-			if (action-&gt;action &amp; MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)</div><div class='del'>-				return -EINVAL;</div><div class='del'>-			action-&gt;action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;</div><div class='del'>-			action-&gt;modify_id = maction-&gt;flow_action_raw.action_id;</div><div class='del'>-			return 0;</div><div class='del'>-		}</div><div class='del'>-		if (maction-&gt;flow_action_raw.sub_type ==</div><div class='del'>-		    MLX5_IB_FLOW_ACTION_DECAP) {</div><div class='del'>-			if (action-&gt;action &amp; MLX5_FLOW_CONTEXT_ACTION_DECAP)</div><div class='del'>-				return -EINVAL;</div><div class='del'>-			action-&gt;action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;</div><div class='del'>-			return 0;</div><div class='del'>-		}</div><div class='del'>-		if (maction-&gt;flow_action_raw.sub_type ==</div><div class='del'>-		    MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) {</div><div class='del'>-			if (action-&gt;action &amp;</div><div class='del'>-			    MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT)</div><div class='del'>-				return -EINVAL;</div><div class='del'>-			action-&gt;action |=</div><div class='del'>-				MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;</div><div class='del'>-			action-&gt;reformat_id =</div><div class='del'>-				maction-&gt;flow_action_raw.action_id;</div><div class='del'>-			return 0;</div><div class='del'>-		}</div><div class='del'>-		/* fall through */</div><div class='del'>-	default:</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int parse_flow_attr(struct mlx5_core_dev *mdev, u32 *match_c,</div><div class='del'>-			   u32 *match_v, const union ib_flow_spec *ib_spec,</div><div class='del'>-			   const struct ib_flow_attr *flow_attr,</div><div class='del'>-			   struct mlx5_flow_act *action, u32 prev_type)</div><div class='del'>-{</div><div class='del'>-	void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,</div><div class='del'>-					   misc_parameters);</div><div class='del'>-	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,</div><div class='del'>-					   misc_parameters);</div><div class='del'>-	void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c,</div><div class='del'>-					    misc_parameters_2);</div><div class='del'>-	void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v,</div><div class='del'>-					    misc_parameters_2);</div><div class='del'>-	void *headers_c;</div><div class='del'>-	void *headers_v;</div><div class='del'>-	int match_ipv;</div><div class='del'>-	int ret;</div><div class='del'>-</div><div class='del'>-	if (ib_spec-&gt;type &amp; IB_FLOW_SPEC_INNER) {</div><div class='del'>-		headers_c = MLX5_ADDR_OF(fte_match_param, match_c,</div><div class='del'>-					 inner_headers);</div><div class='del'>-		headers_v = MLX5_ADDR_OF(fte_match_param, match_v,</div><div class='del'>-					 inner_headers);</div><div class='del'>-		match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,</div><div class='del'>-					ft_field_support.inner_ip_version);</div><div class='del'>-	} else {</div><div class='del'>-		headers_c = MLX5_ADDR_OF(fte_match_param, match_c,</div><div class='del'>-					 outer_headers);</div><div class='del'>-		headers_v = MLX5_ADDR_OF(fte_match_param, match_v,</div><div class='del'>-					 outer_headers);</div><div class='del'>-		match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,</div><div class='del'>-					ft_field_support.outer_ip_version);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	switch (ib_spec-&gt;type &amp; ~IB_FLOW_SPEC_INNER) {</div><div class='del'>-	case IB_FLOW_SPEC_ETH:</div><div class='del'>-		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;eth.mask, LAST_ETH_FIELD))</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,</div><div class='del'>-					     dmac_47_16),</div><div class='del'>-				ib_spec-&gt;eth.mask.dst_mac);</div><div class='del'>-		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,</div><div class='del'>-					     dmac_47_16),</div><div class='del'>-				ib_spec-&gt;eth.val.dst_mac);</div><div class='del'>-</div><div class='del'>-		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,</div><div class='del'>-					     smac_47_16),</div><div class='del'>-				ib_spec-&gt;eth.mask.src_mac);</div><div class='del'>-		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,</div><div class='del'>-					     smac_47_16),</div><div class='del'>-				ib_spec-&gt;eth.val.src_mac);</div><div class='del'>-</div><div class='del'>-		if (ib_spec-&gt;eth.mask.vlan_tag) {</div><div class='del'>-			MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='del'>-				 cvlan_tag, 1);</div><div class='del'>-			MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='del'>-				 cvlan_tag, 1);</div><div class='del'>-</div><div class='del'>-			MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='del'>-				 first_vid, ntohs(ib_spec-&gt;eth.mask.vlan_tag));</div><div class='del'>-			MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='del'>-				 first_vid, ntohs(ib_spec-&gt;eth.val.vlan_tag));</div><div class='del'>-</div><div class='del'>-			MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='del'>-				 first_cfi,</div><div class='del'>-				 ntohs(ib_spec-&gt;eth.mask.vlan_tag) &gt;&gt; 12);</div><div class='del'>-			MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='del'>-				 first_cfi,</div><div class='del'>-				 ntohs(ib_spec-&gt;eth.val.vlan_tag) &gt;&gt; 12);</div><div class='del'>-</div><div class='del'>-			MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='del'>-				 first_prio,</div><div class='del'>-				 ntohs(ib_spec-&gt;eth.mask.vlan_tag) &gt;&gt; 13);</div><div class='del'>-			MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='del'>-				 first_prio,</div><div class='del'>-				 ntohs(ib_spec-&gt;eth.val.vlan_tag) &gt;&gt; 13);</div><div class='del'>-		}</div><div class='del'>-		MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='del'>-			 ethertype, ntohs(ib_spec-&gt;eth.mask.ether_type));</div><div class='del'>-		MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='del'>-			 ethertype, ntohs(ib_spec-&gt;eth.val.ether_type));</div><div class='del'>-		break;</div><div class='del'>-	case IB_FLOW_SPEC_IPV4:</div><div class='del'>-		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;ipv4.mask, LAST_IPV4_FIELD))</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-		if (match_ipv) {</div><div class='del'>-			MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='del'>-				 ip_version, 0xf);</div><div class='del'>-			MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='del'>-				 ip_version, MLX5_FS_IPV4_VERSION);</div><div class='del'>-		} else {</div><div class='del'>-			MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='del'>-				 ethertype, 0xffff);</div><div class='del'>-			MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='del'>-				 ethertype, ETH_P_IP);</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,</div><div class='del'>-				    src_ipv4_src_ipv6.ipv4_layout.ipv4),</div><div class='del'>-		       &amp;ib_spec-&gt;ipv4.mask.src_ip,</div><div class='del'>-		       sizeof(ib_spec-&gt;ipv4.mask.src_ip));</div><div class='del'>-		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,</div><div class='del'>-				    src_ipv4_src_ipv6.ipv4_layout.ipv4),</div><div class='del'>-		       &amp;ib_spec-&gt;ipv4.val.src_ip,</div><div class='del'>-		       sizeof(ib_spec-&gt;ipv4.val.src_ip));</div><div class='del'>-		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,</div><div class='del'>-				    dst_ipv4_dst_ipv6.ipv4_layout.ipv4),</div><div class='del'>-		       &amp;ib_spec-&gt;ipv4.mask.dst_ip,</div><div class='del'>-		       sizeof(ib_spec-&gt;ipv4.mask.dst_ip));</div><div class='del'>-		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,</div><div class='del'>-				    dst_ipv4_dst_ipv6.ipv4_layout.ipv4),</div><div class='del'>-		       &amp;ib_spec-&gt;ipv4.val.dst_ip,</div><div class='del'>-		       sizeof(ib_spec-&gt;ipv4.val.dst_ip));</div><div class='del'>-</div><div class='del'>-		set_tos(headers_c, headers_v,</div><div class='del'>-			ib_spec-&gt;ipv4.mask.tos, ib_spec-&gt;ipv4.val.tos);</div><div class='del'>-</div><div class='del'>-		if (set_proto(headers_c, headers_v,</div><div class='del'>-			      ib_spec-&gt;ipv4.mask.proto,</div><div class='del'>-			      ib_spec-&gt;ipv4.val.proto))</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		break;</div><div class='del'>-	case IB_FLOW_SPEC_IPV6:</div><div class='del'>-		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;ipv6.mask, LAST_IPV6_FIELD))</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-		if (match_ipv) {</div><div class='del'>-			MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='del'>-				 ip_version, 0xf);</div><div class='del'>-			MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='del'>-				 ip_version, MLX5_FS_IPV6_VERSION);</div><div class='del'>-		} else {</div><div class='del'>-			MLX5_SET(fte_match_set_lyr_2_4, headers_c,</div><div class='del'>-				 ethertype, 0xffff);</div><div class='del'>-			MLX5_SET(fte_match_set_lyr_2_4, headers_v,</div><div class='del'>-				 ethertype, ETH_P_IPV6);</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,</div><div class='del'>-				    src_ipv4_src_ipv6.ipv6_layout.ipv6),</div><div class='del'>-		       &amp;ib_spec-&gt;ipv6.mask.src_ip,</div><div class='del'>-		       sizeof(ib_spec-&gt;ipv6.mask.src_ip));</div><div class='del'>-		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,</div><div class='del'>-				    src_ipv4_src_ipv6.ipv6_layout.ipv6),</div><div class='del'>-		       &amp;ib_spec-&gt;ipv6.val.src_ip,</div><div class='del'>-		       sizeof(ib_spec-&gt;ipv6.val.src_ip));</div><div class='del'>-		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,</div><div class='del'>-				    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),</div><div class='del'>-		       &amp;ib_spec-&gt;ipv6.mask.dst_ip,</div><div class='del'>-		       sizeof(ib_spec-&gt;ipv6.mask.dst_ip));</div><div class='del'>-		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,</div><div class='del'>-				    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),</div><div class='del'>-		       &amp;ib_spec-&gt;ipv6.val.dst_ip,</div><div class='del'>-		       sizeof(ib_spec-&gt;ipv6.val.dst_ip));</div><div class='del'>-</div><div class='del'>-		set_tos(headers_c, headers_v,</div><div class='del'>-			ib_spec-&gt;ipv6.mask.traffic_class,</div><div class='del'>-			ib_spec-&gt;ipv6.val.traffic_class);</div><div class='del'>-</div><div class='del'>-		if (set_proto(headers_c, headers_v,</div><div class='del'>-			      ib_spec-&gt;ipv6.mask.next_hdr,</div><div class='del'>-			      ib_spec-&gt;ipv6.val.next_hdr))</div><div class='del'>-			return -EINVAL;</div><div class='del'>-</div><div class='del'>-		set_flow_label(misc_params_c, misc_params_v,</div><div class='del'>-			       ntohl(ib_spec-&gt;ipv6.mask.flow_label),</div><div class='del'>-			       ntohl(ib_spec-&gt;ipv6.val.flow_label),</div><div class='del'>-			       ib_spec-&gt;type &amp; IB_FLOW_SPEC_INNER);</div><div class='del'>-		break;</div><div class='del'>-	case IB_FLOW_SPEC_ESP:</div><div class='del'>-		if (ib_spec-&gt;esp.mask.seq)</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-		MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi,</div><div class='del'>-			 ntohl(ib_spec-&gt;esp.mask.spi));</div><div class='del'>-		MLX5_SET(fte_match_set_misc, misc_params_v, outer_esp_spi,</div><div class='del'>-			 ntohl(ib_spec-&gt;esp.val.spi));</div><div class='del'>-		break;</div><div class='del'>-	case IB_FLOW_SPEC_TCP:</div><div class='del'>-		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;tcp_udp.mask,</div><div class='del'>-					 LAST_TCP_UDP_FIELD))</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-		if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP))</div><div class='del'>-			return -EINVAL;</div><div class='del'>-</div><div class='del'>-		MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport,</div><div class='del'>-			 ntohs(ib_spec-&gt;tcp_udp.mask.src_port));</div><div class='del'>-		MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,</div><div class='del'>-			 ntohs(ib_spec-&gt;tcp_udp.val.src_port));</div><div class='del'>-</div><div class='del'>-		MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport,</div><div class='del'>-			 ntohs(ib_spec-&gt;tcp_udp.mask.dst_port));</div><div class='del'>-		MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,</div><div class='del'>-			 ntohs(ib_spec-&gt;tcp_udp.val.dst_port));</div><div class='del'>-		break;</div><div class='del'>-	case IB_FLOW_SPEC_UDP:</div><div class='del'>-		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;tcp_udp.mask,</div><div class='del'>-					 LAST_TCP_UDP_FIELD))</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-		if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP))</div><div class='del'>-			return -EINVAL;</div><div class='del'>-</div><div class='del'>-		MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,</div><div class='del'>-			 ntohs(ib_spec-&gt;tcp_udp.mask.src_port));</div><div class='del'>-		MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,</div><div class='del'>-			 ntohs(ib_spec-&gt;tcp_udp.val.src_port));</div><div class='del'>-</div><div class='del'>-		MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,</div><div class='del'>-			 ntohs(ib_spec-&gt;tcp_udp.mask.dst_port));</div><div class='del'>-		MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,</div><div class='del'>-			 ntohs(ib_spec-&gt;tcp_udp.val.dst_port));</div><div class='del'>-		break;</div><div class='del'>-	case IB_FLOW_SPEC_GRE:</div><div class='del'>-		if (ib_spec-&gt;gre.mask.c_ks_res0_ver)</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-		if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE))</div><div class='del'>-			return -EINVAL;</div><div class='del'>-</div><div class='del'>-		MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,</div><div class='del'>-			 0xff);</div><div class='del'>-		MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,</div><div class='del'>-			 IPPROTO_GRE);</div><div class='del'>-</div><div class='del'>-		MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol,</div><div class='del'>-			 ntohs(ib_spec-&gt;gre.mask.protocol));</div><div class='del'>-		MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol,</div><div class='del'>-			 ntohs(ib_spec-&gt;gre.val.protocol));</div><div class='del'>-</div><div class='del'>-		memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c,</div><div class='del'>-				    gre_key.nvgre.hi),</div><div class='del'>-		       &amp;ib_spec-&gt;gre.mask.key,</div><div class='del'>-		       sizeof(ib_spec-&gt;gre.mask.key));</div><div class='del'>-		memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v,</div><div class='del'>-				    gre_key.nvgre.hi),</div><div class='del'>-		       &amp;ib_spec-&gt;gre.val.key,</div><div class='del'>-		       sizeof(ib_spec-&gt;gre.val.key));</div><div class='del'>-		break;</div><div class='del'>-	case IB_FLOW_SPEC_MPLS:</div><div class='del'>-		switch (prev_type) {</div><div class='del'>-		case IB_FLOW_SPEC_UDP:</div><div class='del'>-			if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,</div><div class='del'>-						   ft_field_support.outer_first_mpls_over_udp),</div><div class='del'>-						   &amp;ib_spec-&gt;mpls.mask.tag))</div><div class='del'>-				return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,</div><div class='del'>-					    outer_first_mpls_over_udp),</div><div class='del'>-			       &amp;ib_spec-&gt;mpls.val.tag,</div><div class='del'>-			       sizeof(ib_spec-&gt;mpls.val.tag));</div><div class='del'>-			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,</div><div class='del'>-					    outer_first_mpls_over_udp),</div><div class='del'>-			       &amp;ib_spec-&gt;mpls.mask.tag,</div><div class='del'>-			       sizeof(ib_spec-&gt;mpls.mask.tag));</div><div class='del'>-			break;</div><div class='del'>-		case IB_FLOW_SPEC_GRE:</div><div class='del'>-			if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,</div><div class='del'>-						   ft_field_support.outer_first_mpls_over_gre),</div><div class='del'>-						   &amp;ib_spec-&gt;mpls.mask.tag))</div><div class='del'>-				return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,</div><div class='del'>-					    outer_first_mpls_over_gre),</div><div class='del'>-			       &amp;ib_spec-&gt;mpls.val.tag,</div><div class='del'>-			       sizeof(ib_spec-&gt;mpls.val.tag));</div><div class='del'>-			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,</div><div class='del'>-					    outer_first_mpls_over_gre),</div><div class='del'>-			       &amp;ib_spec-&gt;mpls.mask.tag,</div><div class='del'>-			       sizeof(ib_spec-&gt;mpls.mask.tag));</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			if (ib_spec-&gt;type &amp; IB_FLOW_SPEC_INNER) {</div><div class='del'>-				if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,</div><div class='del'>-							   ft_field_support.inner_first_mpls),</div><div class='del'>-							   &amp;ib_spec-&gt;mpls.mask.tag))</div><div class='del'>-					return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,</div><div class='del'>-						    inner_first_mpls),</div><div class='del'>-				       &amp;ib_spec-&gt;mpls.val.tag,</div><div class='del'>-				       sizeof(ib_spec-&gt;mpls.val.tag));</div><div class='del'>-				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,</div><div class='del'>-						    inner_first_mpls),</div><div class='del'>-				       &amp;ib_spec-&gt;mpls.mask.tag,</div><div class='del'>-				       sizeof(ib_spec-&gt;mpls.mask.tag));</div><div class='del'>-			} else {</div><div class='del'>-				if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,</div><div class='del'>-							   ft_field_support.outer_first_mpls),</div><div class='del'>-							   &amp;ib_spec-&gt;mpls.mask.tag))</div><div class='del'>-					return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,</div><div class='del'>-						    outer_first_mpls),</div><div class='del'>-				       &amp;ib_spec-&gt;mpls.val.tag,</div><div class='del'>-				       sizeof(ib_spec-&gt;mpls.val.tag));</div><div class='del'>-				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,</div><div class='del'>-						    outer_first_mpls),</div><div class='del'>-				       &amp;ib_spec-&gt;mpls.mask.tag,</div><div class='del'>-				       sizeof(ib_spec-&gt;mpls.mask.tag));</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-		break;</div><div class='del'>-	case IB_FLOW_SPEC_VXLAN_TUNNEL:</div><div class='del'>-		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;tunnel.mask,</div><div class='del'>-					 LAST_TUNNEL_FIELD))</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-		MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni,</div><div class='del'>-			 ntohl(ib_spec-&gt;tunnel.mask.tunnel_id));</div><div class='del'>-		MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni,</div><div class='del'>-			 ntohl(ib_spec-&gt;tunnel.val.tunnel_id));</div><div class='del'>-		break;</div><div class='del'>-	case IB_FLOW_SPEC_ACTION_TAG:</div><div class='del'>-		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;flow_tag,</div><div class='del'>-					 LAST_FLOW_TAG_FIELD))</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-		if (ib_spec-&gt;flow_tag.tag_id &gt;= BIT(24))</div><div class='del'>-			return -EINVAL;</div><div class='del'>-</div><div class='del'>-		action-&gt;flow_tag = ib_spec-&gt;flow_tag.tag_id;</div><div class='del'>-		action-&gt;flags |= FLOW_ACT_HAS_TAG;</div><div class='del'>-		break;</div><div class='del'>-	case IB_FLOW_SPEC_ACTION_DROP:</div><div class='del'>-		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;drop,</div><div class='del'>-					 LAST_DROP_FIELD))</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-		action-&gt;action |= MLX5_FLOW_CONTEXT_ACTION_DROP;</div><div class='del'>-		break;</div><div class='del'>-	case IB_FLOW_SPEC_ACTION_HANDLE:</div><div class='del'>-		ret = parse_flow_flow_action(to_mflow_act(ib_spec-&gt;action.act),</div><div class='del'>-			flow_attr-&gt;flags &amp; IB_FLOW_ATTR_FLAGS_EGRESS, action);</div><div class='del'>-		if (ret)</div><div class='del'>-			return ret;</div><div class='del'>-		break;</div><div class='del'>-	case IB_FLOW_SPEC_ACTION_COUNT:</div><div class='del'>-		if (FIELDS_NOT_SUPPORTED(ib_spec-&gt;flow_count,</div><div class='del'>-					 LAST_COUNTERS_FIELD))</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-		/* for now support only one counters spec per flow */</div><div class='del'>-		if (action-&gt;action &amp; MLX5_FLOW_CONTEXT_ACTION_COUNT)</div><div class='del'>-			return -EINVAL;</div><div class='del'>-</div><div class='del'>-		action-&gt;counters = ib_spec-&gt;flow_count.counters;</div><div class='del'>-		action-&gt;action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* If a flow could catch both multicast and unicast packets,</div><div class='del'>- * it won't fall into the multicast flow steering table and this rule</div><div class='del'>- * could steal other multicast packets.</div><div class='del'>- */</div><div class='del'>-static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)</div><div class='del'>-{</div><div class='del'>-	union ib_flow_spec *flow_spec;</div><div class='del'>-</div><div class='del'>-	if (ib_attr-&gt;type != IB_FLOW_ATTR_NORMAL ||</div><div class='del'>-	    ib_attr-&gt;num_of_specs &lt; 1)</div><div class='del'>-		return false;</div><div class='del'>-</div><div class='del'>-	flow_spec = (union ib_flow_spec *)(ib_attr + 1);</div><div class='del'>-	if (flow_spec-&gt;type == IB_FLOW_SPEC_IPV4) {</div><div class='del'>-		struct ib_flow_spec_ipv4 *ipv4_spec;</div><div class='del'>-</div><div class='del'>-		ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec;</div><div class='del'>-		if (ipv4_is_multicast(ipv4_spec-&gt;val.dst_ip))</div><div class='del'>-			return true;</div><div class='del'>-</div><div class='del'>-		return false;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (flow_spec-&gt;type == IB_FLOW_SPEC_ETH) {</div><div class='del'>-		struct ib_flow_spec_eth *eth_spec;</div><div class='del'>-</div><div class='del'>-		eth_spec = (struct ib_flow_spec_eth *)flow_spec;</div><div class='del'>-		return is_multicast_ether_addr(eth_spec-&gt;mask.dst_mac) &amp;&amp;</div><div class='del'>-		       is_multicast_ether_addr(eth_spec-&gt;val.dst_mac);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return false;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-enum valid_spec {</div><div class='del'>-	VALID_SPEC_INVALID,</div><div class='del'>-	VALID_SPEC_VALID,</div><div class='del'>-	VALID_SPEC_NA,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static enum valid_spec</div><div class='del'>-is_valid_esp_aes_gcm(struct mlx5_core_dev *mdev,</div><div class='del'>-		     const struct mlx5_flow_spec *spec,</div><div class='del'>-		     const struct mlx5_flow_act *flow_act,</div><div class='del'>-		     bool egress)</div><div class='del'>-{</div><div class='del'>-	const u32 *match_c = spec-&gt;match_criteria;</div><div class='del'>-	bool is_crypto =</div><div class='del'>-		(flow_act-&gt;action &amp; (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT |</div><div class='del'>-				     MLX5_FLOW_CONTEXT_ACTION_DECRYPT));</div><div class='del'>-	bool is_ipsec = mlx5_fs_is_ipsec_flow(match_c);</div><div class='del'>-	bool is_drop = flow_act-&gt;action &amp; MLX5_FLOW_CONTEXT_ACTION_DROP;</div><div class='del'>-</div><div class='del'>-	/*</div><div class='del'>-	 * Currently only crypto is supported in egress, when regular egress</div><div class='del'>-	 * rules would be supported, always return VALID_SPEC_NA.</div><div class='del'>-	 */</div><div class='del'>-	if (!is_crypto)</div><div class='del'>-		return VALID_SPEC_NA;</div><div class='del'>-</div><div class='del'>-	return is_crypto &amp;&amp; is_ipsec &amp;&amp;</div><div class='del'>-		(!egress || (!is_drop &amp;&amp; !(flow_act-&gt;flags &amp; FLOW_ACT_HAS_TAG))) ?</div><div class='del'>-		VALID_SPEC_VALID : VALID_SPEC_INVALID;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static bool is_valid_spec(struct mlx5_core_dev *mdev,</div><div class='del'>-			  const struct mlx5_flow_spec *spec,</div><div class='del'>-			  const struct mlx5_flow_act *flow_act,</div><div class='del'>-			  bool egress)</div><div class='del'>-{</div><div class='del'>-	/* We curretly only support ipsec egress flow */</div><div class='del'>-	return is_valid_esp_aes_gcm(mdev, spec, flow_act, egress) != VALID_SPEC_INVALID;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static bool is_valid_ethertype(struct mlx5_core_dev *mdev,</div><div class='del'>-			       const struct ib_flow_attr *flow_attr,</div><div class='del'>-			       bool check_inner)</div><div class='del'>-{</div><div class='del'>-	union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1);</div><div class='del'>-	int match_ipv = check_inner ?</div><div class='del'>-			MLX5_CAP_FLOWTABLE_NIC_RX(mdev,</div><div class='del'>-					ft_field_support.inner_ip_version) :</div><div class='del'>-			MLX5_CAP_FLOWTABLE_NIC_RX(mdev,</div><div class='del'>-					ft_field_support.outer_ip_version);</div><div class='del'>-	int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0;</div><div class='del'>-	bool ipv4_spec_valid, ipv6_spec_valid;</div><div class='del'>-	unsigned int ip_spec_type = 0;</div><div class='del'>-	bool has_ethertype = false;</div><div class='del'>-	unsigned int spec_index;</div><div class='del'>-	bool mask_valid = true;</div><div class='del'>-	u16 eth_type = 0;</div><div class='del'>-	bool type_valid;</div><div class='del'>-</div><div class='del'>-	/* Validate that ethertype is correct */</div><div class='del'>-	for (spec_index = 0; spec_index &lt; flow_attr-&gt;num_of_specs; spec_index++) {</div><div class='del'>-		if ((ib_spec-&gt;type == (IB_FLOW_SPEC_ETH | inner_bit)) &amp;&amp;</div><div class='del'>-		    ib_spec-&gt;eth.mask.ether_type) {</div><div class='del'>-			mask_valid = (ib_spec-&gt;eth.mask.ether_type ==</div><div class='del'>-				      htons(0xffff));</div><div class='del'>-			has_ethertype = true;</div><div class='del'>-			eth_type = ntohs(ib_spec-&gt;eth.val.ether_type);</div><div class='del'>-		} else if ((ib_spec-&gt;type == (IB_FLOW_SPEC_IPV4 | inner_bit)) ||</div><div class='del'>-			   (ib_spec-&gt;type == (IB_FLOW_SPEC_IPV6 | inner_bit))) {</div><div class='del'>-			ip_spec_type = ib_spec-&gt;type;</div><div class='del'>-		}</div><div class='del'>-		ib_spec = (void *)ib_spec + ib_spec-&gt;size;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	type_valid = (!has_ethertype) || (!ip_spec_type);</div><div class='del'>-	if (!type_valid &amp;&amp; mask_valid) {</div><div class='del'>-		ipv4_spec_valid = (eth_type == ETH_P_IP) &amp;&amp;</div><div class='del'>-			(ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit));</div><div class='del'>-		ipv6_spec_valid = (eth_type == ETH_P_IPV6) &amp;&amp;</div><div class='del'>-			(ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit));</div><div class='del'>-</div><div class='del'>-		type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) ||</div><div class='del'>-			     (((eth_type == ETH_P_MPLS_UC) ||</div><div class='del'>-			       (eth_type == ETH_P_MPLS_MC)) &amp;&amp; match_ipv);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return type_valid;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static bool is_valid_attr(struct mlx5_core_dev *mdev,</div><div class='del'>-			  const struct ib_flow_attr *flow_attr)</div><div class='del'>-{</div><div class='del'>-	return is_valid_ethertype(mdev, flow_attr, false) &amp;&amp;</div><div class='del'>-	       is_valid_ethertype(mdev, flow_attr, true);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void put_flow_table(struct mlx5_ib_dev *dev,</div><div class='del'>-			   struct mlx5_ib_flow_prio *prio, bool ft_added)</div><div class='del'>-{</div><div class='del'>-	prio-&gt;refcount -= !!ft_added;</div><div class='del'>-	if (!prio-&gt;refcount) {</div><div class='del'>-		mlx5_destroy_flow_table(prio-&gt;flow_table);</div><div class='del'>-		prio-&gt;flow_table = NULL;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void counters_clear_description(struct ib_counters *counters)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;mcounters-&gt;mcntrs_mutex);</div><div class='del'>-	kfree(mcounters-&gt;counters_data);</div><div class='del'>-	mcounters-&gt;counters_data = NULL;</div><div class='del'>-	mcounters-&gt;cntrs_max_index = 0;</div><div class='del'>-	mutex_unlock(&amp;mcounters-&gt;mcntrs_mutex);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_flow_handler *handler = container_of(flow_id,</div><div class='del'>-							  struct mlx5_ib_flow_handler,</div><div class='del'>-							  ibflow);</div><div class='del'>-	struct mlx5_ib_flow_handler *iter, *tmp;</div><div class='del'>-	struct mlx5_ib_dev *dev = handler-&gt;dev;</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='del'>-</div><div class='del'>-	list_for_each_entry_safe(iter, tmp, &amp;handler-&gt;list, list) {</div><div class='del'>-		mlx5_del_flow_rules(iter-&gt;rule);</div><div class='del'>-		put_flow_table(dev, iter-&gt;prio, true);</div><div class='del'>-		list_del(&amp;iter-&gt;list);</div><div class='del'>-		kfree(iter);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mlx5_del_flow_rules(handler-&gt;rule);</div><div class='del'>-	put_flow_table(dev, handler-&gt;prio, true);</div><div class='del'>-	if (handler-&gt;ibcounters &amp;&amp;</div><div class='del'>-	    atomic_read(&amp;handler-&gt;ibcounters-&gt;usecnt) == 1)</div><div class='del'>-		counters_clear_description(handler-&gt;ibcounters);</div><div class='del'>-</div><div class='del'>-	mutex_unlock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='del'>-	if (handler-&gt;flow_matcher)</div><div class='del'>-		atomic_dec(&amp;handler-&gt;flow_matcher-&gt;usecnt);</div><div class='del'>-	kfree(handler);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)</div><div class='del'>-{</div><div class='del'>-	priority *= 2;</div><div class='del'>-	if (!dont_trap)</div><div class='del'>-		priority++;</div><div class='del'>-	return priority;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-enum flow_table_type {</div><div class='del'>-	MLX5_IB_FT_RX,</div><div class='del'>-	MLX5_IB_FT_TX</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define MLX5_FS_MAX_TYPES	 6</div><div class='del'>-#define MLX5_FS_MAX_ENTRIES	 BIT(16)</div><div class='del'>-</div><div class='del'>-static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns,</div><div class='del'>-					   struct mlx5_ib_flow_prio *prio,</div><div class='del'>-					   int priority,</div><div class='del'>-					   int num_entries, int num_groups,</div><div class='del'>-					   u32 flags)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_flow_table *ft;</div><div class='del'>-</div><div class='del'>-	ft = mlx5_create_auto_grouped_flow_table(ns, priority,</div><div class='del'>-						 num_entries,</div><div class='del'>-						 num_groups,</div><div class='del'>-						 0, flags);</div><div class='del'>-	if (IS_ERR(ft))</div><div class='del'>-		return ERR_CAST(ft);</div><div class='del'>-</div><div class='del'>-	prio-&gt;flow_table = ft;</div><div class='del'>-	prio-&gt;refcount = 0;</div><div class='del'>-	return prio;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,</div><div class='del'>-						struct ib_flow_attr *flow_attr,</div><div class='del'>-						enum flow_table_type ft_type)</div><div class='del'>-{</div><div class='del'>-	bool dont_trap = flow_attr-&gt;flags &amp; IB_FLOW_ATTR_FLAGS_DONT_TRAP;</div><div class='del'>-	struct mlx5_flow_namespace *ns = NULL;</div><div class='del'>-	struct mlx5_ib_flow_prio *prio;</div><div class='del'>-	struct mlx5_flow_table *ft;</div><div class='del'>-	int max_table_size;</div><div class='del'>-	int num_entries;</div><div class='del'>-	int num_groups;</div><div class='del'>-	u32 flags = 0;</div><div class='del'>-	int priority;</div><div class='del'>-</div><div class='del'>-	max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev,</div><div class='del'>-						       log_max_ft_size));</div><div class='del'>-	if (flow_attr-&gt;type == IB_FLOW_ATTR_NORMAL) {</div><div class='del'>-		enum mlx5_flow_namespace_type fn_type;</div><div class='del'>-</div><div class='del'>-		if (flow_is_multicast_only(flow_attr) &amp;&amp;</div><div class='del'>-		    !dont_trap)</div><div class='del'>-			priority = MLX5_IB_FLOW_MCAST_PRIO;</div><div class='del'>-		else</div><div class='del'>-			priority = ib_prio_to_core_prio(flow_attr-&gt;priority,</div><div class='del'>-							dont_trap);</div><div class='del'>-		if (ft_type == MLX5_IB_FT_RX) {</div><div class='del'>-			fn_type = MLX5_FLOW_NAMESPACE_BYPASS;</div><div class='del'>-			prio = &amp;dev-&gt;flow_db-&gt;prios[priority];</div><div class='del'>-			if (!dev-&gt;is_rep &amp;&amp;</div><div class='del'>-			    MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev, decap))</div><div class='del'>-				flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;</div><div class='del'>-			if (!dev-&gt;is_rep &amp;&amp;</div><div class='del'>-			    MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev,</div><div class='del'>-					reformat_l3_tunnel_to_l2))</div><div class='del'>-				flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;</div><div class='del'>-		} else {</div><div class='del'>-			max_table_size =</div><div class='del'>-				BIT(MLX5_CAP_FLOWTABLE_NIC_TX(dev-&gt;mdev,</div><div class='del'>-							      log_max_ft_size));</div><div class='del'>-			fn_type = MLX5_FLOW_NAMESPACE_EGRESS;</div><div class='del'>-			prio = &amp;dev-&gt;flow_db-&gt;egress_prios[priority];</div><div class='del'>-			if (!dev-&gt;is_rep &amp;&amp;</div><div class='del'>-			    MLX5_CAP_FLOWTABLE_NIC_TX(dev-&gt;mdev, reformat))</div><div class='del'>-				flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;</div><div class='del'>-		}</div><div class='del'>-		ns = mlx5_get_flow_namespace(dev-&gt;mdev, fn_type);</div><div class='del'>-		num_entries = MLX5_FS_MAX_ENTRIES;</div><div class='del'>-		num_groups = MLX5_FS_MAX_TYPES;</div><div class='del'>-	} else if (flow_attr-&gt;type == IB_FLOW_ATTR_ALL_DEFAULT ||</div><div class='del'>-		   flow_attr-&gt;type == IB_FLOW_ATTR_MC_DEFAULT) {</div><div class='del'>-		ns = mlx5_get_flow_namespace(dev-&gt;mdev,</div><div class='del'>-					     MLX5_FLOW_NAMESPACE_LEFTOVERS);</div><div class='del'>-		build_leftovers_ft_param(&amp;priority,</div><div class='del'>-					 &amp;num_entries,</div><div class='del'>-					 &amp;num_groups);</div><div class='del'>-		prio = &amp;dev-&gt;flow_db-&gt;prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];</div><div class='del'>-	} else if (flow_attr-&gt;type == IB_FLOW_ATTR_SNIFFER) {</div><div class='del'>-		if (!MLX5_CAP_FLOWTABLE(dev-&gt;mdev,</div><div class='del'>-					allow_sniffer_and_nic_rx_shared_tir))</div><div class='del'>-			return ERR_PTR(-ENOTSUPP);</div><div class='del'>-</div><div class='del'>-		ns = mlx5_get_flow_namespace(dev-&gt;mdev, ft_type == MLX5_IB_FT_RX ?</div><div class='del'>-					     MLX5_FLOW_NAMESPACE_SNIFFER_RX :</div><div class='del'>-					     MLX5_FLOW_NAMESPACE_SNIFFER_TX);</div><div class='del'>-</div><div class='del'>-		prio = &amp;dev-&gt;flow_db-&gt;sniffer[ft_type];</div><div class='del'>-		priority = 0;</div><div class='del'>-		num_entries = 1;</div><div class='del'>-		num_groups = 1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!ns)</div><div class='del'>-		return ERR_PTR(-ENOTSUPP);</div><div class='del'>-</div><div class='del'>-	max_table_size = min_t(int, num_entries, max_table_size);</div><div class='del'>-</div><div class='del'>-	ft = prio-&gt;flow_table;</div><div class='del'>-	if (!ft)</div><div class='del'>-		return _get_prio(ns, prio, priority, max_table_size, num_groups,</div><div class='del'>-				 flags);</div><div class='del'>-</div><div class='del'>-	return prio;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_underlay_qp(struct mlx5_ib_dev *dev,</div><div class='del'>-			    struct mlx5_flow_spec *spec,</div><div class='del'>-			    u32 underlay_qpn)</div><div class='del'>-{</div><div class='del'>-	void *misc_params_c = MLX5_ADDR_OF(fte_match_param,</div><div class='del'>-					   spec-&gt;match_criteria,</div><div class='del'>-					   misc_parameters);</div><div class='del'>-	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec-&gt;match_value,</div><div class='del'>-					   misc_parameters);</div><div class='del'>-</div><div class='del'>-	if (underlay_qpn &amp;&amp;</div><div class='del'>-	    MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev,</div><div class='del'>-				      ft_field_support.bth_dst_qp)) {</div><div class='del'>-		MLX5_SET(fte_match_set_misc,</div><div class='del'>-			 misc_params_v, bth_dst_qp, underlay_qpn);</div><div class='del'>-		MLX5_SET(fte_match_set_misc,</div><div class='del'>-			 misc_params_c, bth_dst_qp, 0xffffff);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int read_flow_counters(struct ib_device *ibdev,</div><div class='del'>-			      struct mlx5_read_counters_attr *read_attr)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_fc *fc = read_attr-&gt;hw_cntrs_hndl;</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='del'>-</div><div class='del'>-	return mlx5_fc_query(dev-&gt;mdev, fc,</div><div class='del'>-			     &amp;read_attr-&gt;out[IB_COUNTER_PACKETS],</div><div class='del'>-			     &amp;read_attr-&gt;out[IB_COUNTER_BYTES]);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* flow counters currently expose two counters packets and bytes */</div><div class='del'>-#define FLOW_COUNTERS_NUM 2</div><div class='del'>-static int counters_set_description(struct ib_counters *counters,</div><div class='del'>-				    enum mlx5_ib_counters_type counters_type,</div><div class='del'>-				    struct mlx5_ib_flow_counters_desc *desc_data,</div><div class='del'>-				    u32 ncounters)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);</div><div class='del'>-	u32 cntrs_max_index = 0;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	if (counters_type != MLX5_IB_COUNTERS_FLOW)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	/* init the fields for the object */</div><div class='del'>-	mcounters-&gt;type = counters_type;</div><div class='del'>-	mcounters-&gt;read_counters = read_flow_counters;</div><div class='del'>-	mcounters-&gt;counters_num = FLOW_COUNTERS_NUM;</div><div class='del'>-	mcounters-&gt;ncounters = ncounters;</div><div class='del'>-	/* each counter entry have both description and index pair */</div><div class='del'>-	for (i = 0; i &lt; ncounters; i++) {</div><div class='del'>-		if (desc_data[i].description &gt; IB_COUNTER_BYTES)</div><div class='del'>-			return -EINVAL;</div><div class='del'>-</div><div class='del'>-		if (cntrs_max_index &lt;= desc_data[i].index)</div><div class='del'>-			cntrs_max_index = desc_data[i].index + 1;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;mcounters-&gt;mcntrs_mutex);</div><div class='del'>-	mcounters-&gt;counters_data = desc_data;</div><div class='del'>-	mcounters-&gt;cntrs_max_index = cntrs_max_index;</div><div class='del'>-	mutex_unlock(&amp;mcounters-&gt;mcntrs_mutex);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-#define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2))</div><div class='del'>-static int flow_counters_set_data(struct ib_counters *ibcounters,</div><div class='del'>-				  struct mlx5_ib_create_flow *ucmd)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters);</div><div class='del'>-	struct mlx5_ib_flow_counters_data *cntrs_data = NULL;</div><div class='del'>-	struct mlx5_ib_flow_counters_desc *desc_data = NULL;</div><div class='del'>-	bool hw_hndl = false;</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	if (ucmd &amp;&amp; ucmd-&gt;ncounters_data != 0) {</div><div class='del'>-		cntrs_data = ucmd-&gt;data;</div><div class='del'>-		if (cntrs_data-&gt;ncounters &gt; MAX_COUNTERS_NUM)</div><div class='del'>-			return -EINVAL;</div><div class='del'>-</div><div class='del'>-		desc_data = kcalloc(cntrs_data-&gt;ncounters,</div><div class='del'>-				    sizeof(*desc_data),</div><div class='del'>-				    GFP_KERNEL);</div><div class='del'>-		if (!desc_data)</div><div class='del'>-			return  -ENOMEM;</div><div class='del'>-</div><div class='del'>-		if (copy_from_user(desc_data,</div><div class='del'>-				   u64_to_user_ptr(cntrs_data-&gt;counters_data),</div><div class='del'>-				   sizeof(*desc_data) * cntrs_data-&gt;ncounters)) {</div><div class='del'>-			ret = -EFAULT;</div><div class='del'>-			goto free;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!mcounters-&gt;hw_cntrs_hndl) {</div><div class='del'>-		mcounters-&gt;hw_cntrs_hndl = mlx5_fc_create(</div><div class='del'>-			to_mdev(ibcounters-&gt;device)-&gt;mdev, false);</div><div class='del'>-		if (IS_ERR(mcounters-&gt;hw_cntrs_hndl)) {</div><div class='del'>-			ret = PTR_ERR(mcounters-&gt;hw_cntrs_hndl);</div><div class='del'>-			goto free;</div><div class='del'>-		}</div><div class='del'>-		hw_hndl = true;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (desc_data) {</div><div class='del'>-		/* counters already bound to at least one flow */</div><div class='del'>-		if (mcounters-&gt;cntrs_max_index) {</div><div class='del'>-			ret = -EINVAL;</div><div class='del'>-			goto free_hndl;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		ret = counters_set_description(ibcounters,</div><div class='del'>-					       MLX5_IB_COUNTERS_FLOW,</div><div class='del'>-					       desc_data,</div><div class='del'>-					       cntrs_data-&gt;ncounters);</div><div class='del'>-		if (ret)</div><div class='del'>-			goto free_hndl;</div><div class='del'>-</div><div class='del'>-	} else if (!mcounters-&gt;cntrs_max_index) {</div><div class='del'>-		/* counters not bound yet, must have udata passed */</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-		goto free_hndl;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-free_hndl:</div><div class='del'>-	if (hw_hndl) {</div><div class='del'>-		mlx5_fc_destroy(to_mdev(ibcounters-&gt;device)-&gt;mdev,</div><div class='del'>-				mcounters-&gt;hw_cntrs_hndl);</div><div class='del'>-		mcounters-&gt;hw_cntrs_hndl = NULL;</div><div class='del'>-	}</div><div class='del'>-free:</div><div class='del'>-	kfree(desc_data);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,</div><div class='del'>-						      struct mlx5_ib_flow_prio *ft_prio,</div><div class='del'>-						      const struct ib_flow_attr *flow_attr,</div><div class='del'>-						      struct mlx5_flow_destination *dst,</div><div class='del'>-						      u32 underlay_qpn,</div><div class='del'>-						      struct mlx5_ib_create_flow *ucmd)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_flow_table	*ft = ft_prio-&gt;flow_table;</div><div class='del'>-	struct mlx5_ib_flow_handler *handler;</div><div class='del'>-	struct mlx5_flow_act flow_act = {.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG};</div><div class='del'>-	struct mlx5_flow_spec *spec;</div><div class='del'>-	struct mlx5_flow_destination dest_arr[2] = {};</div><div class='del'>-	struct mlx5_flow_destination *rule_dst = dest_arr;</div><div class='del'>-	const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);</div><div class='del'>-	unsigned int spec_index;</div><div class='del'>-	u32 prev_type = 0;</div><div class='del'>-	int err = 0;</div><div class='del'>-	int dest_num = 0;</div><div class='del'>-	bool is_egress = flow_attr-&gt;flags &amp; IB_FLOW_ATTR_FLAGS_EGRESS;</div><div class='del'>-</div><div class='del'>-	if (!is_valid_attr(dev-&gt;mdev, flow_attr))</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	if (dev-&gt;is_rep &amp;&amp; is_egress)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);</div><div class='del'>-	handler = kzalloc(sizeof(*handler), GFP_KERNEL);</div><div class='del'>-	if (!handler || !spec) {</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto free;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	INIT_LIST_HEAD(&amp;handler-&gt;list);</div><div class='del'>-	if (dst) {</div><div class='del'>-		memcpy(&amp;dest_arr[0], dst, sizeof(*dst));</div><div class='del'>-		dest_num++;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	for (spec_index = 0; spec_index &lt; flow_attr-&gt;num_of_specs; spec_index++) {</div><div class='del'>-		err = parse_flow_attr(dev-&gt;mdev, spec-&gt;match_criteria,</div><div class='del'>-				      spec-&gt;match_value,</div><div class='del'>-				      ib_flow, flow_attr, &amp;flow_act,</div><div class='del'>-				      prev_type);</div><div class='del'>-		if (err &lt; 0)</div><div class='del'>-			goto free;</div><div class='del'>-</div><div class='del'>-		prev_type = ((union ib_flow_spec *)ib_flow)-&gt;type;</div><div class='del'>-		ib_flow += ((union ib_flow_spec *)ib_flow)-&gt;size;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (!flow_is_multicast_only(flow_attr))</div><div class='del'>-		set_underlay_qp(dev, spec, underlay_qpn);</div><div class='del'>-</div><div class='del'>-	if (dev-&gt;is_rep) {</div><div class='del'>-		void *misc;</div><div class='del'>-</div><div class='del'>-		if (!dev-&gt;port[flow_attr-&gt;port - 1].rep) {</div><div class='del'>-			err = -EINVAL;</div><div class='del'>-			goto free;</div><div class='del'>-		}</div><div class='del'>-		misc = MLX5_ADDR_OF(fte_match_param, spec-&gt;match_value,</div><div class='del'>-				    misc_parameters);</div><div class='del'>-		MLX5_SET(fte_match_set_misc, misc, source_port,</div><div class='del'>-			 dev-&gt;port[flow_attr-&gt;port - 1].rep-&gt;vport);</div><div class='del'>-		misc = MLX5_ADDR_OF(fte_match_param, spec-&gt;match_criteria,</div><div class='del'>-				    misc_parameters);</div><div class='del'>-		MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	spec-&gt;match_criteria_enable = get_match_criteria_enable(spec-&gt;match_criteria);</div><div class='del'>-</div><div class='del'>-	if (is_egress &amp;&amp;</div><div class='del'>-	    !is_valid_spec(dev-&gt;mdev, spec, &amp;flow_act, is_egress)) {</div><div class='del'>-		err = -EINVAL;</div><div class='del'>-		goto free;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (flow_act.action &amp; MLX5_FLOW_CONTEXT_ACTION_COUNT) {</div><div class='del'>-		struct mlx5_ib_mcounters *mcounters;</div><div class='del'>-</div><div class='del'>-		err = flow_counters_set_data(flow_act.counters, ucmd);</div><div class='del'>-		if (err)</div><div class='del'>-			goto free;</div><div class='del'>-</div><div class='del'>-		mcounters = to_mcounters(flow_act.counters);</div><div class='del'>-		handler-&gt;ibcounters = flow_act.counters;</div><div class='del'>-		dest_arr[dest_num].type =</div><div class='del'>-			MLX5_FLOW_DESTINATION_TYPE_COUNTER;</div><div class='del'>-		dest_arr[dest_num].counter_id =</div><div class='del'>-			mlx5_fc_id(mcounters-&gt;hw_cntrs_hndl);</div><div class='del'>-		dest_num++;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (flow_act.action &amp; MLX5_FLOW_CONTEXT_ACTION_DROP) {</div><div class='del'>-		if (!(flow_act.action &amp; MLX5_FLOW_CONTEXT_ACTION_COUNT)) {</div><div class='del'>-			rule_dst = NULL;</div><div class='del'>-			dest_num = 0;</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		if (is_egress)</div><div class='del'>-			flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;</div><div class='del'>-		else</div><div class='del'>-			flow_act.action |=</div><div class='del'>-				dest_num ?  MLX5_FLOW_CONTEXT_ACTION_FWD_DEST :</div><div class='del'>-					MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if ((flow_act.flags &amp; FLOW_ACT_HAS_TAG)  &amp;&amp;</div><div class='del'>-	    (flow_attr-&gt;type == IB_FLOW_ATTR_ALL_DEFAULT ||</div><div class='del'>-	     flow_attr-&gt;type == IB_FLOW_ATTR_MC_DEFAULT)) {</div><div class='del'>-		mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n",</div><div class='del'>-			     flow_act.flow_tag, flow_attr-&gt;type);</div><div class='del'>-		err = -EINVAL;</div><div class='del'>-		goto free;</div><div class='del'>-	}</div><div class='del'>-	handler-&gt;rule = mlx5_add_flow_rules(ft, spec,</div><div class='del'>-					    &amp;flow_act,</div><div class='del'>-					    rule_dst, dest_num);</div><div class='del'>-</div><div class='del'>-	if (IS_ERR(handler-&gt;rule)) {</div><div class='del'>-		err = PTR_ERR(handler-&gt;rule);</div><div class='del'>-		goto free;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ft_prio-&gt;refcount++;</div><div class='del'>-	handler-&gt;prio = ft_prio;</div><div class='del'>-	handler-&gt;dev = dev;</div><div class='del'>-</div><div class='del'>-	ft_prio-&gt;flow_table = ft;</div><div class='del'>-free:</div><div class='del'>-	if (err &amp;&amp; handler) {</div><div class='del'>-		if (handler-&gt;ibcounters &amp;&amp;</div><div class='del'>-		    atomic_read(&amp;handler-&gt;ibcounters-&gt;usecnt) == 1)</div><div class='del'>-			counters_clear_description(handler-&gt;ibcounters);</div><div class='del'>-		kfree(handler);</div><div class='del'>-	}</div><div class='del'>-	kvfree(spec);</div><div class='del'>-	return err ? ERR_PTR(err) : handler;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,</div><div class='del'>-						     struct mlx5_ib_flow_prio *ft_prio,</div><div class='del'>-						     const struct ib_flow_attr *flow_attr,</div><div class='del'>-						     struct mlx5_flow_destination *dst)</div><div class='del'>-{</div><div class='del'>-	return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct mlx5_ib_flow_handler *create_dont_trap_rule(struct mlx5_ib_dev *dev,</div><div class='del'>-							  struct mlx5_ib_flow_prio *ft_prio,</div><div class='del'>-							  struct ib_flow_attr *flow_attr,</div><div class='del'>-							  struct mlx5_flow_destination *dst)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_flow_handler *handler_dst = NULL;</div><div class='del'>-	struct mlx5_ib_flow_handler *handler = NULL;</div><div class='del'>-</div><div class='del'>-	handler = create_flow_rule(dev, ft_prio, flow_attr, NULL);</div><div class='del'>-	if (!IS_ERR(handler)) {</div><div class='del'>-		handler_dst = create_flow_rule(dev, ft_prio,</div><div class='del'>-					       flow_attr, dst);</div><div class='del'>-		if (IS_ERR(handler_dst)) {</div><div class='del'>-			mlx5_del_flow_rules(handler-&gt;rule);</div><div class='del'>-			ft_prio-&gt;refcount--;</div><div class='del'>-			kfree(handler);</div><div class='del'>-			handler = handler_dst;</div><div class='del'>-		} else {</div><div class='del'>-			list_add(&amp;handler_dst-&gt;list, &amp;handler-&gt;list);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return handler;</div><div class='del'>-}</div><div class='del'>-enum {</div><div class='del'>-	LEFTOVERS_MC,</div><div class='del'>-	LEFTOVERS_UC,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev,</div><div class='del'>-							  struct mlx5_ib_flow_prio *ft_prio,</div><div class='del'>-							  struct ib_flow_attr *flow_attr,</div><div class='del'>-							  struct mlx5_flow_destination *dst)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_flow_handler *handler_ucast = NULL;</div><div class='del'>-	struct mlx5_ib_flow_handler *handler = NULL;</div><div class='del'>-</div><div class='del'>-	static struct {</div><div class='del'>-		struct ib_flow_attr	flow_attr;</div><div class='del'>-		struct ib_flow_spec_eth eth_flow;</div><div class='del'>-	} leftovers_specs[] = {</div><div class='del'>-		[LEFTOVERS_MC] = {</div><div class='del'>-			.flow_attr = {</div><div class='del'>-				.num_of_specs = 1,</div><div class='del'>-				.size = sizeof(leftovers_specs[0])</div><div class='del'>-			},</div><div class='del'>-			.eth_flow = {</div><div class='del'>-				.type = IB_FLOW_SPEC_ETH,</div><div class='del'>-				.size = sizeof(struct ib_flow_spec_eth),</div><div class='del'>-				.mask = {.dst_mac = {0x1} },</div><div class='del'>-				.val =  {.dst_mac = {0x1} }</div><div class='del'>-			}</div><div class='del'>-		},</div><div class='del'>-		[LEFTOVERS_UC] = {</div><div class='del'>-			.flow_attr = {</div><div class='del'>-				.num_of_specs = 1,</div><div class='del'>-				.size = sizeof(leftovers_specs[0])</div><div class='del'>-			},</div><div class='del'>-			.eth_flow = {</div><div class='del'>-				.type = IB_FLOW_SPEC_ETH,</div><div class='del'>-				.size = sizeof(struct ib_flow_spec_eth),</div><div class='del'>-				.mask = {.dst_mac = {0x1} },</div><div class='del'>-				.val = {.dst_mac = {} }</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	};</div><div class='del'>-</div><div class='del'>-	handler = create_flow_rule(dev, ft_prio,</div><div class='del'>-				   &amp;leftovers_specs[LEFTOVERS_MC].flow_attr,</div><div class='del'>-				   dst);</div><div class='del'>-	if (!IS_ERR(handler) &amp;&amp;</div><div class='del'>-	    flow_attr-&gt;type == IB_FLOW_ATTR_ALL_DEFAULT) {</div><div class='del'>-		handler_ucast = create_flow_rule(dev, ft_prio,</div><div class='del'>-						 &amp;leftovers_specs[LEFTOVERS_UC].flow_attr,</div><div class='del'>-						 dst);</div><div class='del'>-		if (IS_ERR(handler_ucast)) {</div><div class='del'>-			mlx5_del_flow_rules(handler-&gt;rule);</div><div class='del'>-			ft_prio-&gt;refcount--;</div><div class='del'>-			kfree(handler);</div><div class='del'>-			handler = handler_ucast;</div><div class='del'>-		} else {</div><div class='del'>-			list_add(&amp;handler_ucast-&gt;list, &amp;handler-&gt;list);</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return handler;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,</div><div class='del'>-							struct mlx5_ib_flow_prio *ft_rx,</div><div class='del'>-							struct mlx5_ib_flow_prio *ft_tx,</div><div class='del'>-							struct mlx5_flow_destination *dst)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_flow_handler *handler_rx;</div><div class='del'>-	struct mlx5_ib_flow_handler *handler_tx;</div><div class='del'>-	int err;</div><div class='del'>-	static const struct ib_flow_attr flow_attr  = {</div><div class='del'>-		.num_of_specs = 0,</div><div class='del'>-		.size = sizeof(flow_attr)</div><div class='del'>-	};</div><div class='del'>-</div><div class='del'>-	handler_rx = create_flow_rule(dev, ft_rx, &amp;flow_attr, dst);</div><div class='del'>-	if (IS_ERR(handler_rx)) {</div><div class='del'>-		err = PTR_ERR(handler_rx);</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	handler_tx = create_flow_rule(dev, ft_tx, &amp;flow_attr, dst);</div><div class='del'>-	if (IS_ERR(handler_tx)) {</div><div class='del'>-		err = PTR_ERR(handler_tx);</div><div class='del'>-		goto err_tx;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	list_add(&amp;handler_tx-&gt;list, &amp;handler_rx-&gt;list);</div><div class='del'>-</div><div class='del'>-	return handler_rx;</div><div class='del'>-</div><div class='del'>-err_tx:</div><div class='del'>-	mlx5_del_flow_rules(handler_rx-&gt;rule);</div><div class='del'>-	ft_rx-&gt;refcount--;</div><div class='del'>-	kfree(handler_rx);</div><div class='del'>-err:</div><div class='del'>-	return ERR_PTR(err);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,</div><div class='del'>-					   struct ib_flow_attr *flow_attr,</div><div class='del'>-					   int domain,</div><div class='del'>-					   struct ib_udata *udata)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(qp-&gt;device);</div><div class='del'>-	struct mlx5_ib_qp *mqp = to_mqp(qp);</div><div class='del'>-	struct mlx5_ib_flow_handler *handler = NULL;</div><div class='del'>-	struct mlx5_flow_destination *dst = NULL;</div><div class='del'>-	struct mlx5_ib_flow_prio *ft_prio_tx = NULL;</div><div class='del'>-	struct mlx5_ib_flow_prio *ft_prio;</div><div class='del'>-	bool is_egress = flow_attr-&gt;flags &amp; IB_FLOW_ATTR_FLAGS_EGRESS;</div><div class='del'>-	struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr;</div><div class='del'>-	size_t min_ucmd_sz, required_ucmd_sz;</div><div class='del'>-	int err;</div><div class='del'>-	int underlay_qpn;</div><div class='del'>-</div><div class='del'>-	if (udata &amp;&amp; udata-&gt;inlen) {</div><div class='del'>-		min_ucmd_sz = offsetof(typeof(ucmd_hdr), reserved) +</div><div class='del'>-				sizeof(ucmd_hdr.reserved);</div><div class='del'>-		if (udata-&gt;inlen &lt; min_ucmd_sz)</div><div class='del'>-			return ERR_PTR(-EOPNOTSUPP);</div><div class='del'>-</div><div class='del'>-		err = ib_copy_from_udata(&amp;ucmd_hdr, udata, min_ucmd_sz);</div><div class='del'>-		if (err)</div><div class='del'>-			return ERR_PTR(err);</div><div class='del'>-</div><div class='del'>-		/* currently supports only one counters data */</div><div class='del'>-		if (ucmd_hdr.ncounters_data &gt; 1)</div><div class='del'>-			return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-		required_ucmd_sz = min_ucmd_sz +</div><div class='del'>-			sizeof(struct mlx5_ib_flow_counters_data) *</div><div class='del'>-			ucmd_hdr.ncounters_data;</div><div class='del'>-		if (udata-&gt;inlen &gt; required_ucmd_sz &amp;&amp;</div><div class='del'>-		    !ib_is_udata_cleared(udata, required_ucmd_sz,</div><div class='del'>-					 udata-&gt;inlen - required_ucmd_sz))</div><div class='del'>-			return ERR_PTR(-EOPNOTSUPP);</div><div class='del'>-</div><div class='del'>-		ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL);</div><div class='del'>-		if (!ucmd)</div><div class='del'>-			return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-		err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz);</div><div class='del'>-		if (err)</div><div class='del'>-			goto free_ucmd;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (flow_attr-&gt;priority &gt; MLX5_IB_FLOW_LAST_PRIO) {</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto free_ucmd;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (domain != IB_FLOW_DOMAIN_USER ||</div><div class='del'>-	    flow_attr-&gt;port &gt; dev-&gt;num_ports ||</div><div class='del'>-	    (flow_attr-&gt;flags &amp; ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP |</div><div class='del'>-				  IB_FLOW_ATTR_FLAGS_EGRESS))) {</div><div class='del'>-		err = -EINVAL;</div><div class='del'>-		goto free_ucmd;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (is_egress &amp;&amp;</div><div class='del'>-	    (flow_attr-&gt;type == IB_FLOW_ATTR_ALL_DEFAULT ||</div><div class='del'>-	     flow_attr-&gt;type == IB_FLOW_ATTR_MC_DEFAULT)) {</div><div class='del'>-		err = -EINVAL;</div><div class='del'>-		goto free_ucmd;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	dst = kzalloc(sizeof(*dst), GFP_KERNEL);</div><div class='del'>-	if (!dst) {</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto free_ucmd;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mutex_lock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='del'>-</div><div class='del'>-	ft_prio = get_flow_table(dev, flow_attr,</div><div class='del'>-				 is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX);</div><div class='del'>-	if (IS_ERR(ft_prio)) {</div><div class='del'>-		err = PTR_ERR(ft_prio);</div><div class='del'>-		goto unlock;</div><div class='del'>-	}</div><div class='del'>-	if (flow_attr-&gt;type == IB_FLOW_ATTR_SNIFFER) {</div><div class='del'>-		ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX);</div><div class='del'>-		if (IS_ERR(ft_prio_tx)) {</div><div class='del'>-			err = PTR_ERR(ft_prio_tx);</div><div class='del'>-			ft_prio_tx = NULL;</div><div class='del'>-			goto destroy_ft;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (is_egress) {</div><div class='del'>-		dst-&gt;type = MLX5_FLOW_DESTINATION_TYPE_PORT;</div><div class='del'>-	} else {</div><div class='del'>-		dst-&gt;type = MLX5_FLOW_DESTINATION_TYPE_TIR;</div><div class='del'>-		if (mqp-&gt;flags &amp; MLX5_IB_QP_RSS)</div><div class='del'>-			dst-&gt;tir_num = mqp-&gt;rss_qp.tirn;</div><div class='del'>-		else</div><div class='del'>-			dst-&gt;tir_num = mqp-&gt;raw_packet_qp.rq.tirn;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (flow_attr-&gt;type == IB_FLOW_ATTR_NORMAL) {</div><div class='del'>-		if (flow_attr-&gt;flags &amp; IB_FLOW_ATTR_FLAGS_DONT_TRAP)  {</div><div class='del'>-			handler = create_dont_trap_rule(dev, ft_prio,</div><div class='del'>-							flow_attr, dst);</div><div class='del'>-		} else {</div><div class='del'>-			underlay_qpn = (mqp-&gt;flags &amp; MLX5_IB_QP_UNDERLAY) ?</div><div class='del'>-					mqp-&gt;underlay_qpn : 0;</div><div class='del'>-			handler = _create_flow_rule(dev, ft_prio, flow_attr,</div><div class='del'>-						    dst, underlay_qpn, ucmd);</div><div class='del'>-		}</div><div class='del'>-	} else if (flow_attr-&gt;type == IB_FLOW_ATTR_ALL_DEFAULT ||</div><div class='del'>-		   flow_attr-&gt;type == IB_FLOW_ATTR_MC_DEFAULT) {</div><div class='del'>-		handler = create_leftovers_rule(dev, ft_prio, flow_attr,</div><div class='del'>-						dst);</div><div class='del'>-	} else if (flow_attr-&gt;type == IB_FLOW_ATTR_SNIFFER) {</div><div class='del'>-		handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);</div><div class='del'>-	} else {</div><div class='del'>-		err = -EINVAL;</div><div class='del'>-		goto destroy_ft;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (IS_ERR(handler)) {</div><div class='del'>-		err = PTR_ERR(handler);</div><div class='del'>-		handler = NULL;</div><div class='del'>-		goto destroy_ft;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mutex_unlock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='del'>-	kfree(dst);</div><div class='del'>-	kfree(ucmd);</div><div class='del'>-</div><div class='del'>-	return &amp;handler-&gt;ibflow;</div><div class='del'>-</div><div class='del'>-destroy_ft:</div><div class='del'>-	put_flow_table(dev, ft_prio, false);</div><div class='del'>-	if (ft_prio_tx)</div><div class='del'>-		put_flow_table(dev, ft_prio_tx, false);</div><div class='del'>-unlock:</div><div class='del'>-	mutex_unlock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='del'>-	kfree(dst);</div><div class='del'>-free_ucmd:</div><div class='del'>-	kfree(ucmd);</div><div class='del'>-	return ERR_PTR(err);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct mlx5_ib_flow_prio *</div><div class='del'>-_get_flow_table(struct mlx5_ib_dev *dev,</div><div class='del'>-		struct mlx5_ib_flow_matcher *fs_matcher,</div><div class='del'>-		bool mcast)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_flow_namespace *ns = NULL;</div><div class='del'>-	struct mlx5_ib_flow_prio *prio = NULL;</div><div class='del'>-	int max_table_size = 0;</div><div class='del'>-	u32 flags = 0;</div><div class='del'>-	int priority;</div><div class='del'>-</div><div class='del'>-	if (mcast)</div><div class='del'>-		priority = MLX5_IB_FLOW_MCAST_PRIO;</div><div class='del'>-	else</div><div class='del'>-		priority = ib_prio_to_core_prio(fs_matcher-&gt;priority, false);</div><div class='del'>-</div><div class='del'>-	if (fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {</div><div class='del'>-		max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev,</div><div class='del'>-					log_max_ft_size));</div><div class='del'>-		if (MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev, decap))</div><div class='del'>-			flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;</div><div class='del'>-		if (MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev,</div><div class='del'>-					      reformat_l3_tunnel_to_l2))</div><div class='del'>-			flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;</div><div class='del'>-	} else if (fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_EGRESS) {</div><div class='del'>-		max_table_size = BIT(</div><div class='del'>-			MLX5_CAP_FLOWTABLE_NIC_TX(dev-&gt;mdev, log_max_ft_size));</div><div class='del'>-		if (MLX5_CAP_FLOWTABLE_NIC_TX(dev-&gt;mdev, reformat))</div><div class='del'>-			flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;</div><div class='del'>-	} else if (fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_FDB) {</div><div class='del'>-		max_table_size = BIT(</div><div class='del'>-			MLX5_CAP_ESW_FLOWTABLE_FDB(dev-&gt;mdev, log_max_ft_size));</div><div class='del'>-		priority = FDB_BYPASS_PATH;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES);</div><div class='del'>-</div><div class='del'>-	ns = mlx5_get_flow_namespace(dev-&gt;mdev, fs_matcher-&gt;ns_type);</div><div class='del'>-	if (!ns)</div><div class='del'>-		return ERR_PTR(-ENOTSUPP);</div><div class='del'>-</div><div class='del'>-	if (fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_BYPASS)</div><div class='del'>-		prio = &amp;dev-&gt;flow_db-&gt;prios[priority];</div><div class='del'>-	else if (fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_EGRESS)</div><div class='del'>-		prio = &amp;dev-&gt;flow_db-&gt;egress_prios[priority];</div><div class='del'>-	else if (fs_matcher-&gt;ns_type == MLX5_FLOW_NAMESPACE_FDB)</div><div class='del'>-		prio = &amp;dev-&gt;flow_db-&gt;fdb;</div><div class='del'>-</div><div class='del'>-	if (!prio)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	if (prio-&gt;flow_table)</div><div class='del'>-		return prio;</div><div class='del'>-</div><div class='del'>-	return _get_prio(ns, prio, priority, max_table_size,</div><div class='del'>-			 MLX5_FS_MAX_TYPES, flags);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct mlx5_ib_flow_handler *</div><div class='del'>-_create_raw_flow_rule(struct mlx5_ib_dev *dev,</div><div class='del'>-		      struct mlx5_ib_flow_prio *ft_prio,</div><div class='del'>-		      struct mlx5_flow_destination *dst,</div><div class='del'>-		      struct mlx5_ib_flow_matcher  *fs_matcher,</div><div class='del'>-		      struct mlx5_flow_act *flow_act,</div><div class='del'>-		      void *cmd_in, int inlen,</div><div class='del'>-		      int dst_num)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_flow_handler *handler;</div><div class='del'>-	struct mlx5_flow_spec *spec;</div><div class='del'>-	struct mlx5_flow_table *ft = ft_prio-&gt;flow_table;</div><div class='del'>-	int err = 0;</div><div class='del'>-</div><div class='del'>-	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);</div><div class='del'>-	handler = kzalloc(sizeof(*handler), GFP_KERNEL);</div><div class='del'>-	if (!handler || !spec) {</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto free;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	INIT_LIST_HEAD(&amp;handler-&gt;list);</div><div class='del'>-</div><div class='del'>-	memcpy(spec-&gt;match_value, cmd_in, inlen);</div><div class='del'>-	memcpy(spec-&gt;match_criteria, fs_matcher-&gt;matcher_mask.match_params,</div><div class='del'>-	       fs_matcher-&gt;mask_len);</div><div class='del'>-	spec-&gt;match_criteria_enable = fs_matcher-&gt;match_criteria_enable;</div><div class='del'>-</div><div class='del'>-	handler-&gt;rule = mlx5_add_flow_rules(ft, spec,</div><div class='del'>-					    flow_act, dst, dst_num);</div><div class='del'>-</div><div class='del'>-	if (IS_ERR(handler-&gt;rule)) {</div><div class='del'>-		err = PTR_ERR(handler-&gt;rule);</div><div class='del'>-		goto free;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ft_prio-&gt;refcount++;</div><div class='del'>-	handler-&gt;prio = ft_prio;</div><div class='del'>-	handler-&gt;dev = dev;</div><div class='del'>-	ft_prio-&gt;flow_table = ft;</div><div class='del'>-</div><div class='del'>-free:</div><div class='del'>-	if (err)</div><div class='del'>-		kfree(handler);</div><div class='del'>-	kvfree(spec);</div><div class='del'>-	return err ? ERR_PTR(err) : handler;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher,</div><div class='del'>-				void *match_v)</div><div class='del'>-{</div><div class='del'>-	void *match_c;</div><div class='del'>-	void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4;</div><div class='del'>-	void *dmac, *dmac_mask;</div><div class='del'>-	void *ipv4, *ipv4_mask;</div><div class='del'>-</div><div class='del'>-	if (!(fs_matcher-&gt;match_criteria_enable &amp;</div><div class='del'>-	      (1 &lt;&lt; MATCH_CRITERIA_ENABLE_OUTER_BIT)))</div><div class='del'>-		return false;</div><div class='del'>-</div><div class='del'>-	match_c = fs_matcher-&gt;matcher_mask.match_params;</div><div class='del'>-	match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v,</div><div class='del'>-					   outer_headers);</div><div class='del'>-	match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c,</div><div class='del'>-					   outer_headers);</div><div class='del'>-</div><div class='del'>-	dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,</div><div class='del'>-			    dmac_47_16);</div><div class='del'>-	dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,</div><div class='del'>-				 dmac_47_16);</div><div class='del'>-</div><div class='del'>-	if (is_multicast_ether_addr(dmac) &amp;&amp;</div><div class='del'>-	    is_multicast_ether_addr(dmac_mask))</div><div class='del'>-		return true;</div><div class='del'>-</div><div class='del'>-	ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,</div><div class='del'>-			    dst_ipv4_dst_ipv6.ipv4_layout.ipv4);</div><div class='del'>-</div><div class='del'>-	ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,</div><div class='del'>-				 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);</div><div class='del'>-</div><div class='del'>-	if (ipv4_is_multicast(*(__be32 *)(ipv4)) &amp;&amp;</div><div class='del'>-	    ipv4_is_multicast(*(__be32 *)(ipv4_mask)))</div><div class='del'>-		return true;</div><div class='del'>-</div><div class='del'>-	return false;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct mlx5_ib_flow_handler *</div><div class='del'>-mlx5_ib_raw_fs_rule_add(struct mlx5_ib_dev *dev,</div><div class='del'>-			struct mlx5_ib_flow_matcher *fs_matcher,</div><div class='del'>-			struct mlx5_flow_act *flow_act,</div><div class='del'>-			u32 counter_id,</div><div class='del'>-			void *cmd_in, int inlen, int dest_id,</div><div class='del'>-			int dest_type)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_flow_destination *dst;</div><div class='del'>-	struct mlx5_ib_flow_prio *ft_prio;</div><div class='del'>-	struct mlx5_ib_flow_handler *handler;</div><div class='del'>-	int dst_num = 0;</div><div class='del'>-	bool mcast;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	if (fs_matcher-&gt;flow_type != MLX5_IB_FLOW_TYPE_NORMAL)</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='del'>-</div><div class='del'>-	if (fs_matcher-&gt;priority &gt; MLX5_IB_FLOW_LAST_PRIO)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	dst = kcalloc(2, sizeof(*dst), GFP_KERNEL);</div><div class='del'>-	if (!dst)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	mcast = raw_fs_is_multicast(fs_matcher, cmd_in);</div><div class='del'>-	mutex_lock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='del'>-</div><div class='del'>-	ft_prio = _get_flow_table(dev, fs_matcher, mcast);</div><div class='del'>-	if (IS_ERR(ft_prio)) {</div><div class='del'>-		err = PTR_ERR(ft_prio);</div><div class='del'>-		goto unlock;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR) {</div><div class='del'>-		dst[dst_num].type = dest_type;</div><div class='del'>-		dst[dst_num].tir_num = dest_id;</div><div class='del'>-		flow_act-&gt;action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;</div><div class='del'>-	} else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) {</div><div class='del'>-		dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM;</div><div class='del'>-		dst[dst_num].ft_num = dest_id;</div><div class='del'>-		flow_act-&gt;action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;</div><div class='del'>-	} else {</div><div class='del'>-		dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_PORT;</div><div class='del'>-		flow_act-&gt;action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	dst_num++;</div><div class='del'>-</div><div class='del'>-	if (flow_act-&gt;action &amp; MLX5_FLOW_CONTEXT_ACTION_COUNT) {</div><div class='del'>-		dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;</div><div class='del'>-		dst[dst_num].counter_id = counter_id;</div><div class='del'>-		dst_num++;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	handler = _create_raw_flow_rule(dev, ft_prio, dst, fs_matcher, flow_act,</div><div class='del'>-					cmd_in, inlen, dst_num);</div><div class='del'>-</div><div class='del'>-	if (IS_ERR(handler)) {</div><div class='del'>-		err = PTR_ERR(handler);</div><div class='del'>-		goto destroy_ft;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mutex_unlock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='del'>-	atomic_inc(&amp;fs_matcher-&gt;usecnt);</div><div class='del'>-	handler-&gt;flow_matcher = fs_matcher;</div><div class='del'>-</div><div class='del'>-	kfree(dst);</div><div class='del'>-</div><div class='del'>-	return handler;</div><div class='del'>-</div><div class='del'>-destroy_ft:</div><div class='del'>-	put_flow_table(dev, ft_prio, false);</div><div class='del'>-unlock:</div><div class='del'>-	mutex_unlock(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='del'>-	kfree(dst);</div><div class='del'>-</div><div class='del'>-	return ERR_PTR(err);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static u32 mlx5_ib_flow_action_flags_to_accel_xfrm_flags(u32 mlx5_flags)</div><div class='del'>-{</div><div class='del'>-	u32 flags = 0;</div><div class='del'>-</div><div class='del'>-	if (mlx5_flags &amp; MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA)</div><div class='del'>-		flags |= MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA;</div><div class='del'>-</div><div class='del'>-	return flags;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-#define MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED	MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA</div><div class='del'>-static struct ib_flow_action *</div><div class='del'>-mlx5_ib_create_flow_action_esp(struct ib_device *device,</div><div class='del'>-			       const struct ib_flow_action_attrs_esp *attr,</div><div class='del'>-			       struct uverbs_attr_bundle *attrs)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_dev *mdev = to_mdev(device);</div><div class='del'>-	struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm;</div><div class='del'>-	struct mlx5_accel_esp_xfrm_attrs accel_attrs = {};</div><div class='del'>-	struct mlx5_ib_flow_action *action;</div><div class='del'>-	u64 action_flags;</div><div class='del'>-	u64 flags;</div><div class='del'>-	int err = 0;</div><div class='del'>-</div><div class='del'>-	err = uverbs_get_flags64(</div><div class='del'>-		&amp;action_flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS,</div><div class='del'>-		((MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED &lt;&lt; 1) - 1));</div><div class='del'>-	if (err)</div><div class='del'>-		return ERR_PTR(err);</div><div class='del'>-</div><div class='del'>-	flags = mlx5_ib_flow_action_flags_to_accel_xfrm_flags(action_flags);</div><div class='del'>-</div><div class='del'>-	/* We current only support a subset of the standard features. Only a</div><div class='del'>-	 * keymat of type AES_GCM, with icv_len == 16, iv_algo == SEQ and esn</div><div class='del'>-	 * (with overlap). Full offload mode isn't supported.</div><div class='del'>-	 */</div><div class='del'>-	if (!attr-&gt;keymat || attr-&gt;replay || attr-&gt;encap ||</div><div class='del'>-	    attr-&gt;spi || attr-&gt;seq || attr-&gt;tfc_pad ||</div><div class='del'>-	    attr-&gt;hard_limit_pkts ||</div><div class='del'>-	    (attr-&gt;flags &amp; ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |</div><div class='del'>-			     IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT)))</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='del'>-</div><div class='del'>-	if (attr-&gt;keymat-&gt;protocol !=</div><div class='del'>-	    IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM)</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='del'>-</div><div class='del'>-	aes_gcm = &amp;attr-&gt;keymat-&gt;keymat.aes_gcm;</div><div class='del'>-</div><div class='del'>-	if (aes_gcm-&gt;icv_len != 16 ||</div><div class='del'>-	    aes_gcm-&gt;iv_algo != IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ)</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='del'>-</div><div class='del'>-	action = kmalloc(sizeof(*action), GFP_KERNEL);</div><div class='del'>-	if (!action)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	action-&gt;esp_aes_gcm.ib_flags = attr-&gt;flags;</div><div class='del'>-	memcpy(&amp;accel_attrs.keymat.aes_gcm.aes_key, &amp;aes_gcm-&gt;aes_key,</div><div class='del'>-	       sizeof(accel_attrs.keymat.aes_gcm.aes_key));</div><div class='del'>-	accel_attrs.keymat.aes_gcm.key_len = aes_gcm-&gt;key_len * 8;</div><div class='del'>-	memcpy(&amp;accel_attrs.keymat.aes_gcm.salt, &amp;aes_gcm-&gt;salt,</div><div class='del'>-	       sizeof(accel_attrs.keymat.aes_gcm.salt));</div><div class='del'>-	memcpy(&amp;accel_attrs.keymat.aes_gcm.seq_iv, &amp;aes_gcm-&gt;iv,</div><div class='del'>-	       sizeof(accel_attrs.keymat.aes_gcm.seq_iv));</div><div class='del'>-	accel_attrs.keymat.aes_gcm.icv_len = aes_gcm-&gt;icv_len * 8;</div><div class='del'>-	accel_attrs.keymat.aes_gcm.iv_algo = MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ;</div><div class='del'>-	accel_attrs.keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM;</div><div class='del'>-</div><div class='del'>-	accel_attrs.esn = attr-&gt;esn;</div><div class='del'>-	if (attr-&gt;flags &amp; IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED)</div><div class='del'>-		accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED;</div><div class='del'>-	if (attr-&gt;flags &amp; IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)</div><div class='del'>-		accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;</div><div class='del'>-</div><div class='del'>-	if (attr-&gt;flags &amp; IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT)</div><div class='del'>-		accel_attrs.action |= MLX5_ACCEL_ESP_ACTION_ENCRYPT;</div><div class='del'>-</div><div class='del'>-	action-&gt;esp_aes_gcm.ctx =</div><div class='del'>-		mlx5_accel_esp_create_xfrm(mdev-&gt;mdev, &amp;accel_attrs, flags);</div><div class='del'>-	if (IS_ERR(action-&gt;esp_aes_gcm.ctx)) {</div><div class='del'>-		err = PTR_ERR(action-&gt;esp_aes_gcm.ctx);</div><div class='del'>-		goto err_parse;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	action-&gt;esp_aes_gcm.ib_flags = attr-&gt;flags;</div><div class='del'>-</div><div class='del'>-	return &amp;action-&gt;ib_action;</div><div class='del'>-</div><div class='del'>-err_parse:</div><div class='del'>-	kfree(action);</div><div class='del'>-	return ERR_PTR(err);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int</div><div class='del'>-mlx5_ib_modify_flow_action_esp(struct ib_flow_action *action,</div><div class='del'>-			       const struct ib_flow_action_attrs_esp *attr,</div><div class='del'>-			       struct uverbs_attr_bundle *attrs)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_flow_action *maction = to_mflow_act(action);</div><div class='del'>-	struct mlx5_accel_esp_xfrm_attrs accel_attrs;</div><div class='del'>-	int err = 0;</div><div class='del'>-</div><div class='del'>-	if (attr-&gt;keymat || attr-&gt;replay || attr-&gt;encap ||</div><div class='del'>-	    attr-&gt;spi || attr-&gt;seq || attr-&gt;tfc_pad ||</div><div class='del'>-	    attr-&gt;hard_limit_pkts ||</div><div class='del'>-	    (attr-&gt;flags &amp; ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |</div><div class='del'>-			     IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS |</div><div class='del'>-			     IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)))</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-	/* Only the ESN value or the MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP can</div><div class='del'>-	 * be modified.</div><div class='del'>-	 */</div><div class='del'>-	if (!(maction-&gt;esp_aes_gcm.ib_flags &amp;</div><div class='del'>-	      IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) &amp;&amp;</div><div class='del'>-	    attr-&gt;flags &amp; (IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |</div><div class='del'>-			   IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	memcpy(&amp;accel_attrs, &amp;maction-&gt;esp_aes_gcm.ctx-&gt;attrs,</div><div class='del'>-	       sizeof(accel_attrs));</div><div class='del'>-</div><div class='del'>-	accel_attrs.esn = attr-&gt;esn;</div><div class='del'>-	if (attr-&gt;flags &amp; IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)</div><div class='del'>-		accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;</div><div class='del'>-	else</div><div class='del'>-		accel_attrs.flags &amp;= ~MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;</div><div class='del'>-</div><div class='del'>-	err = mlx5_accel_esp_modify_xfrm(maction-&gt;esp_aes_gcm.ctx,</div><div class='del'>-					 &amp;accel_attrs);</div><div class='del'>-	if (err)</div><div class='del'>-		return err;</div><div class='del'>-</div><div class='del'>-	maction-&gt;esp_aes_gcm.ib_flags &amp;=</div><div class='del'>-		~IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW;</div><div class='del'>-	maction-&gt;esp_aes_gcm.ib_flags |=</div><div class='del'>-		attr-&gt;flags &amp; IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_flow_action *maction = to_mflow_act(action);</div><div class='del'>-</div><div class='del'>-	switch (action-&gt;type) {</div><div class='del'>-	case IB_FLOW_ACTION_ESP:</div><div class='del'>-		/*</div><div class='del'>-		 * We only support aes_gcm by now, so we implicitly know this is</div><div class='del'>-		 * the underline crypto.</div><div class='del'>-		 */</div><div class='del'>-		mlx5_accel_esp_destroy_xfrm(maction-&gt;esp_aes_gcm.ctx);</div><div class='del'>-		break;</div><div class='del'>-	case IB_FLOW_ACTION_UNSPECIFIED:</div><div class='del'>-		mlx5_ib_destroy_flow_action_raw(maction);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		WARN_ON(true);</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	kfree(maction);</div><div class='del'>-	return 0;</div><div class='add'>+	return mlx5_cmd_dealloc_pd(mdev-&gt;mdev, mpd-&gt;pdn, mpd-&gt;uid);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)</div><div class='hunk'>@@ -4286,7 +2611,7 @@ static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)</div><div class='ctx'> 	uid = ibqp-&gt;pd ?</div><div class='ctx'> 		to_mpd(ibqp-&gt;pd)-&gt;uid : 0;</div><div class='ctx'> </div><div class='del'>-	if (mqp-&gt;flags &amp; MLX5_IB_QP_UNDERLAY) {</div><div class='add'>+	if (mqp-&gt;flags &amp; IB_QP_CREATE_SOURCE_QPN) {</div><div class='ctx'> 		mlx5_ib_dbg(dev, "Attaching a multi cast group to underlay QP is not supported\n");</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -4334,7 +2659,7 @@ static ssize_t fw_pages_show(struct device *device,</div><div class='ctx'> 	struct mlx5_ib_dev *dev =</div><div class='ctx'> 		rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%d\n", dev-&gt;mdev-&gt;priv.fw_pages);</div><div class='add'>+	return sysfs_emit(buf, "%d\n", dev-&gt;mdev-&gt;priv.fw_pages);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(fw_pages);</div><div class='ctx'> </div><div class='hunk'>@@ -4344,7 +2669,7 @@ static ssize_t reg_pages_show(struct device *device,</div><div class='ctx'> 	struct mlx5_ib_dev *dev =</div><div class='ctx'> 		rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%d\n", atomic_read(&amp;dev-&gt;mdev-&gt;priv.reg_pages));</div><div class='add'>+	return sysfs_emit(buf, "%d\n", atomic_read(&amp;dev-&gt;mdev-&gt;priv.reg_pages));</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(reg_pages);</div><div class='ctx'> </div><div class='hunk'>@@ -4354,7 +2679,7 @@ static ssize_t hca_type_show(struct device *device,</div><div class='ctx'> 	struct mlx5_ib_dev *dev =</div><div class='ctx'> 		rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "MT%d\n", dev-&gt;mdev-&gt;pdev-&gt;device);</div><div class='add'>+	return sysfs_emit(buf, "MT%d\n", dev-&gt;mdev-&gt;pdev-&gt;device);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(hca_type);</div><div class='ctx'> </div><div class='hunk'>@@ -4364,7 +2689,7 @@ static ssize_t hw_rev_show(struct device *device,</div><div class='ctx'> 	struct mlx5_ib_dev *dev =</div><div class='ctx'> 		rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%x\n", dev-&gt;mdev-&gt;rev_id);</div><div class='add'>+	return sysfs_emit(buf, "%x\n", dev-&gt;mdev-&gt;rev_id);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(hw_rev);</div><div class='ctx'> </div><div class='hunk'>@@ -4374,8 +2699,8 @@ static ssize_t board_id_show(struct device *device,</div><div class='ctx'> 	struct mlx5_ib_dev *dev =</div><div class='ctx'> 		rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);</div><div class='ctx'> </div><div class='del'>-	return sprintf(buf, "%.*s\n", MLX5_BOARD_ID_LEN,</div><div class='del'>-		       dev-&gt;mdev-&gt;board_id);</div><div class='add'>+	return sysfs_emit(buf, "%.*s\n", MLX5_BOARD_ID_LEN,</div><div class='add'>+			  dev-&gt;mdev-&gt;board_id);</div><div class='ctx'> }</div><div class='ctx'> static DEVICE_ATTR_RO(board_id);</div><div class='ctx'> </div><div class='hunk'>@@ -4398,9 +2723,14 @@ static void pkey_change_handler(struct work_struct *work)</div><div class='ctx'> 		container_of(work, struct mlx5_ib_port_resources,</div><div class='ctx'> 			     pkey_change_work);</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;ports-&gt;devr-&gt;mutex);</div><div class='add'>+	if (!ports-&gt;gsi)</div><div class='add'>+		/*</div><div class='add'>+		 * We got this event before device was fully configured</div><div class='add'>+		 * and MAD registration code wasn't called/finished yet.</div><div class='add'>+		 */</div><div class='add'>+		return;</div><div class='add'>+</div><div class='ctx'> 	mlx5_ib_gsi_pkey_change(ports-&gt;gsi);</div><div class='del'>-	mutex_unlock(&amp;ports-&gt;devr-&gt;mutex);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)</div><div class='hunk'>@@ -4457,7 +2787,7 @@ static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)</div><div class='ctx'> 	 * lock/unlock above locks Now need to arm all involved CQs.</div><div class='ctx'> 	 */</div><div class='ctx'> 	list_for_each_entry(mcq, &amp;cq_armed_list, reset_notify) {</div><div class='del'>-		mcq-&gt;comp(mcq);</div><div class='add'>+		mcq-&gt;comp(mcq, NULL);</div><div class='ctx'> 	}</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;ibdev-&gt;reset_flow_resource_lock, flags);</div><div class='ctx'> }</div><div class='hunk'>@@ -4472,8 +2802,7 @@ static void delay_drop_handler(struct work_struct *work)</div><div class='ctx'> 	atomic_inc(&amp;delay_drop-&gt;events_cnt);</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;delay_drop-&gt;lock);</div><div class='del'>-	err = mlx5_core_set_delay_drop(delay_drop-&gt;dev-&gt;mdev,</div><div class='del'>-				       delay_drop-&gt;timeout);</div><div class='add'>+	err = mlx5_core_set_delay_drop(delay_drop-&gt;dev, delay_drop-&gt;timeout);</div><div class='ctx'> 	if (err) {</div><div class='ctx'> 		mlx5_ib_warn(delay_drop-&gt;dev, "Failed to set delay drop, timeout=%u\n",</div><div class='ctx'> 			     delay_drop-&gt;timeout);</div><div class='hunk'>@@ -4485,7 +2814,7 @@ static void delay_drop_handler(struct work_struct *work)</div><div class='ctx'> static void handle_general_event(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe,</div><div class='ctx'> 				 struct ib_event *ibev)</div><div class='ctx'> {</div><div class='del'>-	u8 port = (eqe-&gt;data.port.port &gt;&gt; 4) &amp; 0xf;</div><div class='add'>+	u32 port = (eqe-&gt;data.port.port &gt;&gt; 4) &amp; 0xf;</div><div class='ctx'> </div><div class='ctx'> 	switch (eqe-&gt;sub_type) {</div><div class='ctx'> 	case MLX5_GENERAL_SUBTYPE_DELAY_DROP_TIMEOUT:</div><div class='hunk'>@@ -4501,7 +2830,7 @@ static void handle_general_event(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe</div><div class='ctx'> static int handle_port_change(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe,</div><div class='ctx'> 			      struct ib_event *ibev)</div><div class='ctx'> {</div><div class='del'>-	u8 port = (eqe-&gt;data.port.port &gt;&gt; 4) &amp; 0xf;</div><div class='add'>+	u32 port = (eqe-&gt;data.port.port &gt;&gt; 4) &amp; 0xf;</div><div class='ctx'> </div><div class='ctx'> 	ibev-&gt;element.port_num = port;</div><div class='ctx'> </div><div class='hunk'>@@ -4572,7 +2901,7 @@ static void mlx5_ib_handle_event(struct work_struct *_work)</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_EVENT_TYPE_GENERAL_EVENT:</div><div class='ctx'> 		handle_general_event(ibdev, work-&gt;param, &amp;ibev);</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	default:</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -4632,392 +2961,304 @@ static int mlx5_ib_event_slave_port(struct notifier_block *nb,</div><div class='ctx'> 	return NOTIFY_OK;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int set_has_smi_cap(struct mlx5_ib_dev *dev)</div><div class='add'>+static int mlx5_ib_get_plane_num(struct mlx5_core_dev *mdev, u8 *num_plane)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_hca_vport_context vport_ctx;</div><div class='ctx'> 	int err;</div><div class='del'>-	int port;</div><div class='ctx'> </div><div class='del'>-	for (port = 1; port &lt;= ARRAY_SIZE(dev-&gt;mdev-&gt;port_caps); port++) {</div><div class='del'>-		dev-&gt;mdev-&gt;port_caps[port - 1].has_smi = false;</div><div class='del'>-		if (MLX5_CAP_GEN(dev-&gt;mdev, port_type) ==</div><div class='del'>-		    MLX5_CAP_PORT_TYPE_IB) {</div><div class='del'>-			if (MLX5_CAP_GEN(dev-&gt;mdev, ib_virt)) {</div><div class='del'>-				err = mlx5_query_hca_vport_context(dev-&gt;mdev, 0,</div><div class='del'>-								   port, 0,</div><div class='del'>-								   &amp;vport_ctx);</div><div class='del'>-				if (err) {</div><div class='del'>-					mlx5_ib_err(dev, "query_hca_vport_context for port=%d failed %d\n",</div><div class='del'>-						    port, err);</div><div class='del'>-					return err;</div><div class='del'>-				}</div><div class='del'>-				dev-&gt;mdev-&gt;port_caps[port - 1].has_smi =</div><div class='del'>-					vport_ctx.has_smi;</div><div class='del'>-			} else {</div><div class='del'>-				dev-&gt;mdev-&gt;port_caps[port - 1].has_smi = true;</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='add'>+	*num_plane = 0;</div><div class='add'>+	if (!MLX5_CAP_GEN(mdev, ib_virt) || !MLX5_CAP_GEN_2(mdev, multiplane))</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-static void get_ext_port_caps(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	int port;</div><div class='add'>+	err = mlx5_query_hca_vport_context(mdev, 0, 1, 0, &amp;vport_ctx);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='ctx'> </div><div class='del'>-	for (port = 1; port &lt;= dev-&gt;num_ports; port++)</div><div class='del'>-		mlx5_query_ext_port_caps(dev, port);</div><div class='add'>+	*num_plane = vport_ctx.num_plane;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)</div><div class='add'>+static int set_has_smi_cap(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	struct ib_device_attr *dprops = NULL;</div><div class='del'>-	struct ib_port_attr *pprops = NULL;</div><div class='del'>-	int err = -ENOMEM;</div><div class='del'>-	struct ib_udata uhw = {.inlen = 0, .outlen = 0};</div><div class='del'>-</div><div class='del'>-	pprops = kmalloc(sizeof(*pprops), GFP_KERNEL);</div><div class='del'>-	if (!pprops)</div><div class='del'>-		goto out;</div><div class='add'>+	struct mlx5_hca_vport_context vport_ctx;</div><div class='add'>+	int err;</div><div class='add'>+	int port;</div><div class='ctx'> </div><div class='del'>-	dprops = kmalloc(sizeof(*dprops), GFP_KERNEL);</div><div class='del'>-	if (!dprops)</div><div class='del'>-		goto out;</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, port_type) != MLX5_CAP_PORT_TYPE_IB)</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-	err = mlx5_ib_query_device(&amp;dev-&gt;ib_dev, dprops, &amp;uhw);</div><div class='del'>-	if (err) {</div><div class='del'>-		mlx5_ib_warn(dev, "query_device failed %d\n", err);</div><div class='del'>-		goto out;</div><div class='del'>-	}</div><div class='add'>+	for (port = 1; port &lt;= dev-&gt;num_ports; port++) {</div><div class='add'>+		if (dev-&gt;num_plane) {</div><div class='add'>+			dev-&gt;port_caps[port - 1].has_smi = false;</div><div class='add'>+			continue;</div><div class='add'>+		} else if (!MLX5_CAP_GEN(dev-&gt;mdev, ib_virt) ||</div><div class='add'>+			dev-&gt;ib_dev.type == RDMA_DEVICE_TYPE_SMI) {</div><div class='add'>+			dev-&gt;port_caps[port - 1].has_smi = true;</div><div class='add'>+			continue;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='del'>-	memset(pprops, 0, sizeof(*pprops));</div><div class='del'>-	err = mlx5_ib_query_port(&amp;dev-&gt;ib_dev, port, pprops);</div><div class='del'>-	if (err) {</div><div class='del'>-		mlx5_ib_warn(dev, "query_port %d failed %d\n",</div><div class='del'>-			     port, err);</div><div class='del'>-		goto out;</div><div class='add'>+		err = mlx5_query_hca_vport_context(dev-&gt;mdev, 0, port, 0,</div><div class='add'>+						   &amp;vport_ctx);</div><div class='add'>+		if (err) {</div><div class='add'>+			mlx5_ib_err(dev, "query_hca_vport_context for port=%d failed %d\n",</div><div class='add'>+				    port, err);</div><div class='add'>+			return err;</div><div class='add'>+		}</div><div class='add'>+		dev-&gt;port_caps[port - 1].has_smi = vport_ctx.has_smi;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	dev-&gt;mdev-&gt;port_caps[port - 1].pkey_table_len =</div><div class='del'>-					dprops-&gt;max_pkeys;</div><div class='del'>-	dev-&gt;mdev-&gt;port_caps[port - 1].gid_table_len =</div><div class='del'>-					pprops-&gt;gid_tbl_len;</div><div class='del'>-	mlx5_ib_dbg(dev, "port %d: pkey_table_len %d, gid_table_len %d\n",</div><div class='del'>-		    port, dprops-&gt;max_pkeys, pprops-&gt;gid_tbl_len);</div><div class='del'>-</div><div class='del'>-out:</div><div class='del'>-	kfree(pprops);</div><div class='del'>-	kfree(dprops);</div><div class='del'>-</div><div class='del'>-	return err;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int get_port_caps(struct mlx5_ib_dev *dev, u8 port)</div><div class='add'>+static void get_ext_port_caps(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	/* For representors use port 1, is this is the only native</div><div class='del'>-	 * port</div><div class='del'>-	 */</div><div class='del'>-	if (dev-&gt;is_rep)</div><div class='del'>-		return __get_port_caps(dev, 1);</div><div class='del'>-	return __get_port_caps(dev, port);</div><div class='add'>+	unsigned int port;</div><div class='add'>+</div><div class='add'>+	rdma_for_each_port (&amp;dev-&gt;ib_dev, port)</div><div class='add'>+		mlx5_query_ext_port_caps(dev, port);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void destroy_umrc_res(struct mlx5_ib_dev *dev)</div><div class='add'>+static u8 mlx5_get_umr_fence(u8 umr_fence_cap)</div><div class='ctx'> {</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	err = mlx5_mr_cache_cleanup(dev);</div><div class='del'>-	if (err)</div><div class='del'>-		mlx5_ib_warn(dev, "mr cache cleanup failed\n");</div><div class='del'>-</div><div class='del'>-	if (dev-&gt;umrc.qp)</div><div class='del'>-		mlx5_ib_destroy_qp(dev-&gt;umrc.qp, NULL);</div><div class='del'>-	if (dev-&gt;umrc.cq)</div><div class='del'>-		ib_free_cq(dev-&gt;umrc.cq);</div><div class='del'>-	if (dev-&gt;umrc.pd)</div><div class='del'>-		ib_dealloc_pd(dev-&gt;umrc.pd);</div><div class='add'>+	switch (umr_fence_cap) {</div><div class='add'>+	case MLX5_CAP_UMR_FENCE_NONE:</div><div class='add'>+		return MLX5_FENCE_MODE_NONE;</div><div class='add'>+	case MLX5_CAP_UMR_FENCE_SMALL:</div><div class='add'>+		return MLX5_FENCE_MODE_INITIATOR_SMALL;</div><div class='add'>+	default:</div><div class='add'>+		return MLX5_FENCE_MODE_STRONG_ORDERING;</div><div class='add'>+	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-enum {</div><div class='del'>-	MAX_UMR_WR = 128,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static int create_umr_res(struct mlx5_ib_dev *dev)</div><div class='add'>+int mlx5_ib_dev_res_cq_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	struct ib_qp_init_attr *init_attr = NULL;</div><div class='del'>-	struct ib_qp_attr *attr = NULL;</div><div class='add'>+	struct mlx5_ib_resources *devr = &amp;dev-&gt;devr;</div><div class='add'>+	struct ib_cq_init_attr cq_attr = {.cqe = 1};</div><div class='add'>+	struct ib_device *ibdev;</div><div class='ctx'> 	struct ib_pd *pd;</div><div class='ctx'> 	struct ib_cq *cq;</div><div class='del'>-	struct ib_qp *qp;</div><div class='del'>-	int ret;</div><div class='add'>+	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	attr = kzalloc(sizeof(*attr), GFP_KERNEL);</div><div class='del'>-	init_attr = kzalloc(sizeof(*init_attr), GFP_KERNEL);</div><div class='del'>-	if (!attr || !init_attr) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto error_0;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	pd = ib_alloc_pd(&amp;dev-&gt;ib_dev, 0);</div><div class='add'>+	/*</div><div class='add'>+	 * devr-&gt;c0 is set once, never changed until device unload.</div><div class='add'>+	 * Avoid taking the mutex if initialization is already done.</div><div class='add'>+	 */</div><div class='add'>+	if (devr-&gt;c0)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;devr-&gt;cq_lock);</div><div class='add'>+	if (devr-&gt;c0)</div><div class='add'>+		goto unlock;</div><div class='add'>+</div><div class='add'>+	ibdev = &amp;dev-&gt;ib_dev;</div><div class='add'>+	pd = ib_alloc_pd(ibdev, 0);</div><div class='ctx'> 	if (IS_ERR(pd)) {</div><div class='del'>-		mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");</div><div class='ctx'> 		ret = PTR_ERR(pd);</div><div class='del'>-		goto error_0;</div><div class='add'>+		mlx5_ib_err(dev, "Couldn't allocate PD for res init, err=%pe\n",</div><div class='add'>+			    pd);</div><div class='add'>+		goto unlock;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	cq = ib_alloc_cq(&amp;dev-&gt;ib_dev, NULL, 128, 0, IB_POLL_SOFTIRQ);</div><div class='add'>+	cq = ib_create_cq(ibdev, NULL, NULL, NULL, &amp;cq_attr);</div><div class='ctx'> 	if (IS_ERR(cq)) {</div><div class='del'>-		mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n");</div><div class='ctx'> 		ret = PTR_ERR(cq);</div><div class='del'>-		goto error_2;</div><div class='add'>+		mlx5_ib_err(dev, "Couldn't create CQ for res init, err=%pe\n",</div><div class='add'>+			    cq);</div><div class='add'>+		ib_dealloc_pd(pd);</div><div class='add'>+		goto unlock;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	init_attr-&gt;send_cq = cq;</div><div class='del'>-	init_attr-&gt;recv_cq = cq;</div><div class='del'>-	init_attr-&gt;sq_sig_type = IB_SIGNAL_ALL_WR;</div><div class='del'>-	init_attr-&gt;cap.max_send_wr = MAX_UMR_WR;</div><div class='del'>-	init_attr-&gt;cap.max_send_sge = 1;</div><div class='del'>-	init_attr-&gt;qp_type = MLX5_IB_QPT_REG_UMR;</div><div class='del'>-	init_attr-&gt;port_num = 1;</div><div class='del'>-	qp = mlx5_ib_create_qp(pd, init_attr, NULL);</div><div class='del'>-	if (IS_ERR(qp)) {</div><div class='del'>-		mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n");</div><div class='del'>-		ret = PTR_ERR(qp);</div><div class='del'>-		goto error_3;</div><div class='del'>-	}</div><div class='del'>-	qp-&gt;device     = &amp;dev-&gt;ib_dev;</div><div class='del'>-	qp-&gt;real_qp    = qp;</div><div class='del'>-	qp-&gt;uobject    = NULL;</div><div class='del'>-	qp-&gt;qp_type    = MLX5_IB_QPT_REG_UMR;</div><div class='del'>-	qp-&gt;send_cq    = init_attr-&gt;send_cq;</div><div class='del'>-	qp-&gt;recv_cq    = init_attr-&gt;recv_cq;</div><div class='del'>-</div><div class='del'>-	attr-&gt;qp_state = IB_QPS_INIT;</div><div class='del'>-	attr-&gt;port_num = 1;</div><div class='del'>-	ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_PKEY_INDEX |</div><div class='del'>-				IB_QP_PORT, NULL);</div><div class='del'>-	if (ret) {</div><div class='del'>-		mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n");</div><div class='del'>-		goto error_4;</div><div class='del'>-	}</div><div class='add'>+	devr-&gt;p0 = pd;</div><div class='add'>+	devr-&gt;c0 = cq;</div><div class='ctx'> </div><div class='del'>-	memset(attr, 0, sizeof(*attr));</div><div class='del'>-	attr-&gt;qp_state = IB_QPS_RTR;</div><div class='del'>-	attr-&gt;path_mtu = IB_MTU_256;</div><div class='add'>+unlock:</div><div class='add'>+	mutex_unlock(&amp;devr-&gt;cq_lock);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);</div><div class='del'>-	if (ret) {</div><div class='del'>-		mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n");</div><div class='del'>-		goto error_4;</div><div class='del'>-	}</div><div class='add'>+int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_resources *devr = &amp;dev-&gt;devr;</div><div class='add'>+	struct ib_srq_init_attr attr;</div><div class='add'>+	struct ib_srq *s0, *s1;</div><div class='add'>+	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	memset(attr, 0, sizeof(*attr));</div><div class='del'>-	attr-&gt;qp_state = IB_QPS_RTS;</div><div class='del'>-	ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);</div><div class='del'>-	if (ret) {</div><div class='del'>-		mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n");</div><div class='del'>-		goto error_4;</div><div class='del'>-	}</div><div class='add'>+	/*</div><div class='add'>+	 * devr-&gt;s1 is set once, never changed until device unload.</div><div class='add'>+	 * Avoid taking the mutex if initialization is already done.</div><div class='add'>+	 */</div><div class='add'>+	if (devr-&gt;s1)</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-	dev-&gt;umrc.qp = qp;</div><div class='del'>-	dev-&gt;umrc.cq = cq;</div><div class='del'>-	dev-&gt;umrc.pd = pd;</div><div class='add'>+	mutex_lock(&amp;devr-&gt;srq_lock);</div><div class='add'>+	if (devr-&gt;s1)</div><div class='add'>+		goto unlock;</div><div class='ctx'> </div><div class='del'>-	sema_init(&amp;dev-&gt;umrc.sem, MAX_UMR_WR);</div><div class='del'>-	ret = mlx5_mr_cache_init(dev);</div><div class='del'>-	if (ret) {</div><div class='del'>-		mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);</div><div class='del'>-		goto error_4;</div><div class='del'>-	}</div><div class='add'>+	ret = mlx5_ib_dev_res_cq_init(dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto unlock;</div><div class='ctx'> </div><div class='del'>-	kfree(attr);</div><div class='del'>-	kfree(init_attr);</div><div class='add'>+	memset(&amp;attr, 0, sizeof(attr));</div><div class='add'>+	attr.attr.max_sge = 1;</div><div class='add'>+	attr.attr.max_wr = 1;</div><div class='add'>+	attr.srq_type = IB_SRQT_XRC;</div><div class='add'>+	attr.ext.cq = devr-&gt;c0;</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	s0 = ib_create_srq(devr-&gt;p0, &amp;attr);</div><div class='add'>+	if (IS_ERR(s0)) {</div><div class='add'>+		ret = PTR_ERR(s0);</div><div class='add'>+		mlx5_ib_err(dev,</div><div class='add'>+			    "Couldn't create SRQ 0 for res init, err=%pe\n",</div><div class='add'>+			    s0);</div><div class='add'>+		goto unlock;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-error_4:</div><div class='del'>-	mlx5_ib_destroy_qp(qp, NULL);</div><div class='del'>-	dev-&gt;umrc.qp = NULL;</div><div class='add'>+	memset(&amp;attr, 0, sizeof(attr));</div><div class='add'>+	attr.attr.max_sge = 1;</div><div class='add'>+	attr.attr.max_wr = 1;</div><div class='add'>+	attr.srq_type = IB_SRQT_BASIC;</div><div class='ctx'> </div><div class='del'>-error_3:</div><div class='del'>-	ib_free_cq(cq);</div><div class='del'>-	dev-&gt;umrc.cq = NULL;</div><div class='add'>+	s1 = ib_create_srq(devr-&gt;p0, &amp;attr);</div><div class='add'>+	if (IS_ERR(s1)) {</div><div class='add'>+		ret = PTR_ERR(s1);</div><div class='add'>+		mlx5_ib_err(dev,</div><div class='add'>+			    "Couldn't create SRQ 1 for res init, err=%pe\n",</div><div class='add'>+			    s1);</div><div class='add'>+		ib_destroy_srq(s0);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-error_2:</div><div class='del'>-	ib_dealloc_pd(pd);</div><div class='del'>-	dev-&gt;umrc.pd = NULL;</div><div class='add'>+	devr-&gt;s0 = s0;</div><div class='add'>+	devr-&gt;s1 = s1;</div><div class='ctx'> </div><div class='del'>-error_0:</div><div class='del'>-	kfree(attr);</div><div class='del'>-	kfree(init_attr);</div><div class='add'>+unlock:</div><div class='add'>+	mutex_unlock(&amp;devr-&gt;srq_lock);</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static u8 mlx5_get_umr_fence(u8 umr_fence_cap)</div><div class='add'>+static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	switch (umr_fence_cap) {</div><div class='del'>-	case MLX5_CAP_UMR_FENCE_NONE:</div><div class='del'>-		return MLX5_FENCE_MODE_NONE;</div><div class='del'>-	case MLX5_CAP_UMR_FENCE_SMALL:</div><div class='del'>-		return MLX5_FENCE_MODE_INITIATOR_SMALL;</div><div class='del'>-	default:</div><div class='del'>-		return MLX5_FENCE_MODE_STRONG_ORDERING;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='add'>+	struct mlx5_ib_resources *devr = &amp;dev-&gt;devr;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-static int create_dev_resources(struct mlx5_ib_resources *devr)</div><div class='del'>-{</div><div class='del'>-	struct ib_srq_init_attr attr;</div><div class='del'>-	struct mlx5_ib_dev *dev;</div><div class='del'>-	struct ib_device *ibdev;</div><div class='del'>-	struct ib_cq_init_attr cq_attr = {.cqe = 1};</div><div class='del'>-	int port;</div><div class='del'>-	int ret = 0;</div><div class='add'>+	if (!MLX5_CAP_GEN(dev-&gt;mdev, xrc))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='del'>-	dev = container_of(devr, struct mlx5_ib_dev, devr);</div><div class='del'>-	ibdev = &amp;dev-&gt;ib_dev;</div><div class='add'>+	ret = mlx5_cmd_xrcd_alloc(dev-&gt;mdev, &amp;devr-&gt;xrcdn0, 0);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='del'>-	mutex_init(&amp;devr-&gt;mutex);</div><div class='add'>+	ret = mlx5_cmd_xrcd_alloc(dev-&gt;mdev, &amp;devr-&gt;xrcdn1, 0);</div><div class='add'>+	if (ret) {</div><div class='add'>+		mlx5_cmd_xrcd_dealloc(dev-&gt;mdev, devr-&gt;xrcdn0, 0);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	devr-&gt;p0 = rdma_zalloc_drv_obj(ibdev, ib_pd);</div><div class='del'>-	if (!devr-&gt;p0)</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+	mutex_init(&amp;devr-&gt;cq_lock);</div><div class='add'>+	mutex_init(&amp;devr-&gt;srq_lock);</div><div class='ctx'> </div><div class='del'>-	devr-&gt;p0-&gt;device  = ibdev;</div><div class='del'>-	devr-&gt;p0-&gt;uobject = NULL;</div><div class='del'>-	atomic_set(&amp;devr-&gt;p0-&gt;usecnt, 0);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	ret = mlx5_ib_alloc_pd(devr-&gt;p0, NULL);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto error0;</div><div class='add'>+static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_resources *devr = &amp;dev-&gt;devr;</div><div class='ctx'> </div><div class='del'>-	devr-&gt;c0 = mlx5_ib_create_cq(&amp;dev-&gt;ib_dev, &amp;cq_attr, NULL);</div><div class='del'>-	if (IS_ERR(devr-&gt;c0)) {</div><div class='del'>-		ret = PTR_ERR(devr-&gt;c0);</div><div class='del'>-		goto error1;</div><div class='add'>+	/* After s0/s1 init, they are not unset during the device lifetime. */</div><div class='add'>+	if (devr-&gt;s1) {</div><div class='add'>+		ib_destroy_srq(devr-&gt;s1);</div><div class='add'>+		ib_destroy_srq(devr-&gt;s0);</div><div class='ctx'> 	}</div><div class='del'>-	devr-&gt;c0-&gt;device        = &amp;dev-&gt;ib_dev;</div><div class='del'>-	devr-&gt;c0-&gt;uobject       = NULL;</div><div class='del'>-	devr-&gt;c0-&gt;comp_handler  = NULL;</div><div class='del'>-	devr-&gt;c0-&gt;event_handler = NULL;</div><div class='del'>-	devr-&gt;c0-&gt;cq_context    = NULL;</div><div class='del'>-	atomic_set(&amp;devr-&gt;c0-&gt;usecnt, 0);</div><div class='del'>-</div><div class='del'>-	devr-&gt;x0 = mlx5_ib_alloc_xrcd(&amp;dev-&gt;ib_dev, NULL);</div><div class='del'>-	if (IS_ERR(devr-&gt;x0)) {</div><div class='del'>-		ret = PTR_ERR(devr-&gt;x0);</div><div class='del'>-		goto error2;</div><div class='add'>+	mlx5_cmd_xrcd_dealloc(dev-&gt;mdev, devr-&gt;xrcdn1, 0);</div><div class='add'>+	mlx5_cmd_xrcd_dealloc(dev-&gt;mdev, devr-&gt;xrcdn0, 0);</div><div class='add'>+	/* After p0/c0 init, they are not unset during the device lifetime. */</div><div class='add'>+	if (devr-&gt;c0) {</div><div class='add'>+		ib_destroy_cq(devr-&gt;c0);</div><div class='add'>+		ib_dealloc_pd(devr-&gt;p0);</div><div class='ctx'> 	}</div><div class='del'>-	devr-&gt;x0-&gt;device = &amp;dev-&gt;ib_dev;</div><div class='del'>-	devr-&gt;x0-&gt;inode = NULL;</div><div class='del'>-	atomic_set(&amp;devr-&gt;x0-&gt;usecnt, 0);</div><div class='del'>-	mutex_init(&amp;devr-&gt;x0-&gt;tgt_qp_mutex);</div><div class='del'>-	INIT_LIST_HEAD(&amp;devr-&gt;x0-&gt;tgt_qp_list);</div><div class='del'>-</div><div class='del'>-	devr-&gt;x1 = mlx5_ib_alloc_xrcd(&amp;dev-&gt;ib_dev, NULL);</div><div class='del'>-	if (IS_ERR(devr-&gt;x1)) {</div><div class='del'>-		ret = PTR_ERR(devr-&gt;x1);</div><div class='del'>-		goto error3;</div><div class='del'>-	}</div><div class='del'>-	devr-&gt;x1-&gt;device = &amp;dev-&gt;ib_dev;</div><div class='del'>-	devr-&gt;x1-&gt;inode = NULL;</div><div class='del'>-	atomic_set(&amp;devr-&gt;x1-&gt;usecnt, 0);</div><div class='del'>-	mutex_init(&amp;devr-&gt;x1-&gt;tgt_qp_mutex);</div><div class='del'>-	INIT_LIST_HEAD(&amp;devr-&gt;x1-&gt;tgt_qp_list);</div><div class='add'>+	mutex_destroy(&amp;devr-&gt;cq_lock);</div><div class='add'>+	mutex_destroy(&amp;devr-&gt;srq_lock);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	memset(&amp;attr, 0, sizeof(attr));</div><div class='del'>-	attr.attr.max_sge = 1;</div><div class='del'>-	attr.attr.max_wr = 1;</div><div class='del'>-	attr.srq_type = IB_SRQT_XRC;</div><div class='del'>-	attr.ext.cq = devr-&gt;c0;</div><div class='del'>-	attr.ext.xrc.xrcd = devr-&gt;x0;</div><div class='add'>+static int</div><div class='add'>+mlx5_ib_create_data_direct_resources(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);</div><div class='add'>+	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='add'>+	bool ro_supp = false;</div><div class='add'>+	void *mkc;</div><div class='add'>+	u32 mkey;</div><div class='add'>+	u32 pdn;</div><div class='add'>+	u32 *in;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-	devr-&gt;s0 = rdma_zalloc_drv_obj(ibdev, ib_srq);</div><div class='del'>-	if (!devr-&gt;s0) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto error4;</div><div class='add'>+	err = mlx5_core_alloc_pd(mdev, &amp;pdn);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	in = kvzalloc(inlen, GFP_KERNEL);</div><div class='add'>+	if (!in) {</div><div class='add'>+		err = -ENOMEM;</div><div class='add'>+		goto err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	devr-&gt;s0-&gt;device	= &amp;dev-&gt;ib_dev;</div><div class='del'>-	devr-&gt;s0-&gt;pd		= devr-&gt;p0;</div><div class='del'>-	devr-&gt;s0-&gt;srq_type      = IB_SRQT_XRC;</div><div class='del'>-	devr-&gt;s0-&gt;ext.xrc.xrcd	= devr-&gt;x0;</div><div class='del'>-	devr-&gt;s0-&gt;ext.cq	= devr-&gt;c0;</div><div class='del'>-	ret = mlx5_ib_create_srq(devr-&gt;s0, &amp;attr, NULL);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_create;</div><div class='add'>+	MLX5_SET(create_mkey_in, in, data_direct, 1);</div><div class='add'>+	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);</div><div class='add'>+	MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);</div><div class='add'>+	MLX5_SET(mkc, mkc, lw, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, lr, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, rw, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, rr, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, a, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, pd, pdn);</div><div class='add'>+	MLX5_SET(mkc, mkc, length64, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, qpn, 0xffffff);</div><div class='add'>+	err = mlx5_core_create_mkey(mdev, &amp;mkey, in, inlen);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_mkey;</div><div class='ctx'> </div><div class='del'>-	atomic_inc(&amp;devr-&gt;s0-&gt;ext.xrc.xrcd-&gt;usecnt);</div><div class='del'>-	atomic_inc(&amp;devr-&gt;s0-&gt;ext.cq-&gt;usecnt);</div><div class='del'>-	atomic_inc(&amp;devr-&gt;p0-&gt;usecnt);</div><div class='del'>-	atomic_set(&amp;devr-&gt;s0-&gt;usecnt, 0);</div><div class='add'>+	dev-&gt;ddr.mkey = mkey;</div><div class='add'>+	dev-&gt;ddr.pdn = pdn;</div><div class='ctx'> </div><div class='del'>-	memset(&amp;attr, 0, sizeof(attr));</div><div class='del'>-	attr.attr.max_sge = 1;</div><div class='del'>-	attr.attr.max_wr = 1;</div><div class='del'>-	attr.srq_type = IB_SRQT_BASIC;</div><div class='del'>-	devr-&gt;s1 = rdma_zalloc_drv_obj(ibdev, ib_srq);</div><div class='del'>-	if (!devr-&gt;s1) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto error5;</div><div class='add'>+	/* create another mkey with RO support */</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, relaxed_ordering_write)) {</div><div class='add'>+		MLX5_SET(mkc, mkc, relaxed_ordering_write, 1);</div><div class='add'>+		ro_supp = true;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	devr-&gt;s1-&gt;device	= &amp;dev-&gt;ib_dev;</div><div class='del'>-	devr-&gt;s1-&gt;pd		= devr-&gt;p0;</div><div class='del'>-	devr-&gt;s1-&gt;srq_type      = IB_SRQT_BASIC;</div><div class='del'>-	devr-&gt;s1-&gt;ext.cq	= devr-&gt;c0;</div><div class='del'>-</div><div class='del'>-	ret = mlx5_ib_create_srq(devr-&gt;s1, &amp;attr, NULL);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto error6;</div><div class='del'>-</div><div class='del'>-	atomic_inc(&amp;devr-&gt;p0-&gt;usecnt);</div><div class='del'>-	atomic_set(&amp;devr-&gt;s1-&gt;usecnt, 0);</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, relaxed_ordering_read)) {</div><div class='add'>+		MLX5_SET(mkc, mkc, relaxed_ordering_read, 1);</div><div class='add'>+		ro_supp = true;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	for (port = 0; port &lt; ARRAY_SIZE(devr-&gt;ports); ++port) {</div><div class='del'>-		INIT_WORK(&amp;devr-&gt;ports[port].pkey_change_work,</div><div class='del'>-			  pkey_change_handler);</div><div class='del'>-		devr-&gt;ports[port].devr = devr;</div><div class='add'>+	if (ro_supp) {</div><div class='add'>+		err = mlx5_core_create_mkey(mdev, &amp;mkey, in, inlen);</div><div class='add'>+		/* RO is defined as best effort */</div><div class='add'>+		if (!err) {</div><div class='add'>+			dev-&gt;ddr.mkey_ro = mkey;</div><div class='add'>+			dev-&gt;ddr.mkey_ro_valid = true;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	kvfree(in);</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-error6:</div><div class='del'>-	kfree(devr-&gt;s1);</div><div class='del'>-error5:</div><div class='del'>-	mlx5_ib_destroy_srq(devr-&gt;s0, NULL);</div><div class='del'>-err_create:</div><div class='del'>-	kfree(devr-&gt;s0);</div><div class='del'>-error4:</div><div class='del'>-	mlx5_ib_dealloc_xrcd(devr-&gt;x1, NULL);</div><div class='del'>-error3:</div><div class='del'>-	mlx5_ib_dealloc_xrcd(devr-&gt;x0, NULL);</div><div class='del'>-error2:</div><div class='del'>-	mlx5_ib_destroy_cq(devr-&gt;c0, NULL);</div><div class='del'>-error1:</div><div class='del'>-	mlx5_ib_dealloc_pd(devr-&gt;p0, NULL);</div><div class='del'>-error0:</div><div class='del'>-	kfree(devr-&gt;p0);</div><div class='del'>-	return ret;</div><div class='add'>+err_mkey:</div><div class='add'>+	kvfree(in);</div><div class='add'>+err:</div><div class='add'>+	mlx5_core_dealloc_pd(mdev, pdn);</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void destroy_dev_resources(struct mlx5_ib_resources *devr)</div><div class='add'>+static void</div><div class='add'>+mlx5_ib_free_data_direct_resources(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	int port;</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_destroy_srq(devr-&gt;s1, NULL);</div><div class='del'>-	kfree(devr-&gt;s1);</div><div class='del'>-	mlx5_ib_destroy_srq(devr-&gt;s0, NULL);</div><div class='del'>-	kfree(devr-&gt;s0);</div><div class='del'>-	mlx5_ib_dealloc_xrcd(devr-&gt;x0, NULL);</div><div class='del'>-	mlx5_ib_dealloc_xrcd(devr-&gt;x1, NULL);</div><div class='del'>-	mlx5_ib_destroy_cq(devr-&gt;c0, NULL);</div><div class='del'>-	mlx5_ib_dealloc_pd(devr-&gt;p0, NULL);</div><div class='del'>-	kfree(devr-&gt;p0);</div><div class='del'>-</div><div class='del'>-	/* Make sure no change P_Key work items are still executing */</div><div class='del'>-	for (port = 0; port &lt; ARRAY_SIZE(devr-&gt;ports); ++port)</div><div class='del'>-		cancel_work_sync(&amp;devr-&gt;ports[port].pkey_change_work);</div><div class='add'>+	if (dev-&gt;ddr.mkey_ro_valid)</div><div class='add'>+		mlx5_core_destroy_mkey(dev-&gt;mdev, dev-&gt;ddr.mkey_ro);</div><div class='add'>+</div><div class='add'>+	mlx5_core_destroy_mkey(dev-&gt;mdev, dev-&gt;ddr.mkey);</div><div class='add'>+	mlx5_core_dealloc_pd(dev-&gt;mdev, dev-&gt;ddr.pdn);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static u32 get_core_cap_flags(struct ib_device *ibdev,</div><div class='hunk'>@@ -5033,6 +3274,13 @@ static u32 get_core_cap_flags(struct ib_device *ibdev,</div><div class='ctx'> 	if (rep-&gt;grh_required)</div><div class='ctx'> 		ret |= RDMA_CORE_CAP_IB_GRH_REQUIRED;</div><div class='ctx'> </div><div class='add'>+	if (dev-&gt;num_plane)</div><div class='add'>+		return ret | RDMA_CORE_CAP_PROT_IB | RDMA_CORE_CAP_IB_MAD |</div><div class='add'>+			RDMA_CORE_CAP_IB_CM | RDMA_CORE_CAP_IB_SA |</div><div class='add'>+			RDMA_CORE_CAP_AF_IB;</div><div class='add'>+	else if (ibdev-&gt;type == RDMA_DEVICE_TYPE_SMI)</div><div class='add'>+		return ret | RDMA_CORE_CAP_IB_MAD | RDMA_CORE_CAP_IB_SMI;</div><div class='add'>+</div><div class='ctx'> 	if (ll == IB_LINK_LAYER_INFINIBAND)</div><div class='ctx'> 		return ret | RDMA_CORE_PORT_IBA_IB;</div><div class='ctx'> </div><div class='hunk'>@@ -5054,7 +3302,7 @@ static u32 get_core_cap_flags(struct ib_device *ibdev,</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num,</div><div class='add'>+static int mlx5_port_immutable(struct ib_device *ibdev, u32 port_num,</div><div class='ctx'> 			       struct ib_port_immutable *immutable)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr attr;</div><div class='hunk'>@@ -5068,6 +3316,9 @@ static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num,</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='ctx'> 	if (ll == IB_LINK_LAYER_INFINIBAND) {</div><div class='add'>+		if (ibdev-&gt;type == RDMA_DEVICE_TYPE_SMI)</div><div class='add'>+			port_num = smi_to_native_portnum(dev, port_num);</div><div class='add'>+</div><div class='ctx'> 		err = mlx5_query_hca_vport_context(dev-&gt;mdev, 0, port_num, 0,</div><div class='ctx'> 						   &amp;rep);</div><div class='ctx'> 		if (err)</div><div class='hunk'>@@ -5077,13 +3328,12 @@ static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num,</div><div class='ctx'> 	immutable-&gt;pkey_tbl_len = attr.pkey_tbl_len;</div><div class='ctx'> 	immutable-&gt;gid_tbl_len = attr.gid_tbl_len;</div><div class='ctx'> 	immutable-&gt;core_cap_flags = get_core_cap_flags(ibdev, &amp;rep);</div><div class='del'>-	if ((ll == IB_LINK_LAYER_INFINIBAND) || MLX5_CAP_GEN(dev-&gt;mdev, roce))</div><div class='del'>-		immutable-&gt;max_mad_size = IB_MGMT_MAD_SIZE;</div><div class='add'>+	immutable-&gt;max_mad_size = IB_MGMT_MAD_SIZE;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_port_rep_immutable(struct ib_device *ibdev, u8 port_num,</div><div class='add'>+static int mlx5_port_rep_immutable(struct ib_device *ibdev, u32 port_num,</div><div class='ctx'> 				   struct ib_port_immutable *immutable)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_port_attr attr;</div><div class='hunk'>@@ -5111,6 +3361,67 @@ static void get_dev_fw_str(struct ib_device *ibdev, char *str)</div><div class='ctx'> 		 fw_rev_sub(dev-&gt;mdev));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int lag_event(struct notifier_block *nb, unsigned long event, void *data)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = container_of(nb, struct mlx5_ib_dev,</div><div class='add'>+					       lag_events);</div><div class='add'>+	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='add'>+	struct ib_device *ibdev = &amp;dev-&gt;ib_dev;</div><div class='add'>+	struct net_device *old_ndev = NULL;</div><div class='add'>+	struct mlx5_ib_port *port;</div><div class='add'>+	struct net_device *ndev;</div><div class='add'>+	u32 portnum = 0;</div><div class='add'>+	int ret = 0;</div><div class='add'>+	int i;</div><div class='add'>+</div><div class='add'>+	switch (event) {</div><div class='add'>+	case MLX5_DRIVER_EVENT_ACTIVE_BACKUP_LAG_CHANGE_LOWERSTATE:</div><div class='add'>+		ndev = data;</div><div class='add'>+		if (ndev) {</div><div class='add'>+			if (!mlx5_lag_is_roce(mdev)) {</div><div class='add'>+				// sriov lag</div><div class='add'>+				for (i = 0; i &lt; dev-&gt;num_ports; i++) {</div><div class='add'>+					port = &amp;dev-&gt;port[i];</div><div class='add'>+					if (port-&gt;rep &amp;&amp; port-&gt;rep-&gt;vport ==</div><div class='add'>+					    MLX5_VPORT_UPLINK) {</div><div class='add'>+						portnum = i;</div><div class='add'>+						break;</div><div class='add'>+					}</div><div class='add'>+				}</div><div class='add'>+			}</div><div class='add'>+			old_ndev = ib_device_get_netdev(ibdev, portnum + 1);</div><div class='add'>+			ret = ib_device_set_netdev(ibdev, ndev, portnum + 1);</div><div class='add'>+			if (ret)</div><div class='add'>+				goto out;</div><div class='add'>+</div><div class='add'>+			if (old_ndev)</div><div class='add'>+				roce_del_all_netdev_gids(ibdev, portnum + 1,</div><div class='add'>+							 old_ndev);</div><div class='add'>+			rdma_roce_rescan_port(ibdev, portnum + 1);</div><div class='add'>+		}</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return NOTIFY_DONE;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	dev_put(old_ndev);</div><div class='add'>+	return notifier_from_errno(ret);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5e_lag_event_register(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	dev-&gt;lag_events.notifier_call = lag_event;</div><div class='add'>+	blocking_notifier_chain_register(&amp;dev-&gt;mdev-&gt;priv.lag_nh,</div><div class='add'>+					 &amp;dev-&gt;lag_events);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5e_lag_event_unregister(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	blocking_notifier_chain_unregister(&amp;dev-&gt;mdev-&gt;priv.lag_nh,</div><div class='add'>+					   &amp;dev-&gt;lag_events);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='hunk'>@@ -5119,7 +3430,7 @@ static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> 	struct mlx5_flow_table *ft;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	if (!ns || !mlx5_lag_is_roce(mdev))</div><div class='add'>+	if (!ns || !mlx5_lag_is_active(mdev))</div><div class='ctx'> 		return 0;</div><div class='ctx'> </div><div class='ctx'> 	err = mlx5_cmd_create_vport_lag(mdev);</div><div class='hunk'>@@ -5132,7 +3443,9 @@ static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> 		goto err_destroy_vport_lag;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	mlx5e_lag_event_register(dev);</div><div class='ctx'> 	dev-&gt;flow_db-&gt;lag_demux_ft = ft;</div><div class='add'>+	dev-&gt;lag_ports = mlx5_lag_get_num_ports(mdev);</div><div class='ctx'> 	dev-&gt;lag_active = true;</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='hunk'>@@ -5148,6 +3461,7 @@ static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)</div><div class='ctx'> 	if (dev-&gt;lag_active) {</div><div class='ctx'> 		dev-&gt;lag_active = false;</div><div class='ctx'> </div><div class='add'>+		mlx5e_lag_event_unregister(dev);</div><div class='ctx'> 		mlx5_destroy_flow_table(dev-&gt;flow_db-&gt;lag_demux_ft);</div><div class='ctx'> 		dev-&gt;flow_db-&gt;lag_demux_ft = NULL;</div><div class='ctx'> </div><div class='hunk'>@@ -5155,33 +3469,70 @@ static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)</div><div class='add'>+static void mlx5_netdev_notifier_register(struct mlx5_roce *roce,</div><div class='add'>+					  struct net_device *netdev)</div><div class='ctx'> {</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	dev-&gt;port[port_num].roce.nb.notifier_call = mlx5_netdev_event;</div><div class='del'>-	err = register_netdevice_notifier(&amp;dev-&gt;port[port_num].roce.nb);</div><div class='del'>-	if (err) {</div><div class='del'>-		dev-&gt;port[port_num].roce.nb.notifier_call = NULL;</div><div class='del'>-		return err;</div><div class='del'>-	}</div><div class='add'>+	if (roce-&gt;tracking_netdev)</div><div class='add'>+		return;</div><div class='add'>+	roce-&gt;tracking_netdev = netdev;</div><div class='add'>+	roce-&gt;nb.notifier_call = mlx5_netdev_event;</div><div class='add'>+	err = register_netdevice_notifier_dev_net(netdev, &amp;roce-&gt;nb, &amp;roce-&gt;nn);</div><div class='add'>+	WARN_ON(err);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+static void mlx5_netdev_notifier_unregister(struct mlx5_roce *roce)</div><div class='add'>+{</div><div class='add'>+	if (!roce-&gt;tracking_netdev)</div><div class='add'>+		return;</div><div class='add'>+	unregister_netdevice_notifier_dev_net(roce-&gt;tracking_netdev, &amp;roce-&gt;nb,</div><div class='add'>+					      &amp;roce-&gt;nn);</div><div class='add'>+	roce-&gt;tracking_netdev = NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)</div><div class='add'>+static int mlx5e_mdev_notifier_event(struct notifier_block *nb,</div><div class='add'>+				     unsigned long event, void *data)</div><div class='ctx'> {</div><div class='del'>-	if (dev-&gt;port[port_num].roce.nb.notifier_call) {</div><div class='del'>-		unregister_netdevice_notifier(&amp;dev-&gt;port[port_num].roce.nb);</div><div class='del'>-		dev-&gt;port[port_num].roce.nb.notifier_call = NULL;</div><div class='add'>+	struct mlx5_roce *roce = container_of(nb, struct mlx5_roce, mdev_nb);</div><div class='add'>+	struct net_device *netdev = data;</div><div class='add'>+</div><div class='add'>+	switch (event) {</div><div class='add'>+	case MLX5_DRIVER_EVENT_UPLINK_NETDEV:</div><div class='add'>+		if (netdev)</div><div class='add'>+			mlx5_netdev_notifier_register(roce, netdev);</div><div class='add'>+		else</div><div class='add'>+			mlx5_netdev_notifier_unregister(roce);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return NOTIFY_DONE;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	return NOTIFY_OK;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_mdev_netdev_track(struct mlx5_ib_dev *dev, u32 port_num)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_roce *roce = &amp;dev-&gt;port[port_num].roce;</div><div class='add'>+</div><div class='add'>+	roce-&gt;mdev_nb.notifier_call = mlx5e_mdev_notifier_event;</div><div class='add'>+	mlx5_blocking_notifier_register(dev-&gt;mdev, &amp;roce-&gt;mdev_nb);</div><div class='add'>+	mlx5_core_uplink_netdev_event_replay(dev-&gt;mdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_mdev_netdev_untrack(struct mlx5_ib_dev *dev, u32 port_num)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_roce *roce = &amp;dev-&gt;port[port_num].roce;</div><div class='add'>+</div><div class='add'>+	mlx5_blocking_notifier_unregister(dev-&gt;mdev, &amp;roce-&gt;mdev_nb);</div><div class='add'>+	mlx5_netdev_notifier_unregister(roce);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int mlx5_enable_eth(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_GEN(dev-&gt;mdev, roce)) {</div><div class='add'>+	if (!dev-&gt;is_rep &amp;&amp; dev-&gt;profile != &amp;raw_eth_profile) {</div><div class='ctx'> 		err = mlx5_nic_vport_enable_roce(dev-&gt;mdev);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			return err;</div><div class='hunk'>@@ -5194,7 +3545,7 @@ static int mlx5_enable_eth(struct mlx5_ib_dev *dev)</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> err_disable_roce:</div><div class='del'>-	if (MLX5_CAP_GEN(dev-&gt;mdev, roce))</div><div class='add'>+	if (!dev-&gt;is_rep &amp;&amp; dev-&gt;profile != &amp;raw_eth_profile)</div><div class='ctx'> 		mlx5_nic_vport_disable_roce(dev-&gt;mdev);</div><div class='ctx'> </div><div class='ctx'> 	return err;</div><div class='hunk'>@@ -5203,337 +3554,11 @@ err_disable_roce:</div><div class='ctx'> static void mlx5_disable_eth(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	mlx5_eth_lag_cleanup(dev);</div><div class='del'>-	if (MLX5_CAP_GEN(dev-&gt;mdev, roce))</div><div class='add'>+	if (!dev-&gt;is_rep &amp;&amp; dev-&gt;profile != &amp;raw_eth_profile)</div><div class='ctx'> 		mlx5_nic_vport_disable_roce(dev-&gt;mdev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct mlx5_ib_counter {</div><div class='del'>-	const char *name;</div><div class='del'>-	size_t offset;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define INIT_Q_COUNTER(_name)		\</div><div class='del'>-	{ .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}</div><div class='del'>-</div><div class='del'>-static const struct mlx5_ib_counter basic_q_cnts[] = {</div><div class='del'>-	INIT_Q_COUNTER(rx_write_requests),</div><div class='del'>-	INIT_Q_COUNTER(rx_read_requests),</div><div class='del'>-	INIT_Q_COUNTER(rx_atomic_requests),</div><div class='del'>-	INIT_Q_COUNTER(out_of_buffer),</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static const struct mlx5_ib_counter out_of_seq_q_cnts[] = {</div><div class='del'>-	INIT_Q_COUNTER(out_of_sequence),</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static const struct mlx5_ib_counter retrans_q_cnts[] = {</div><div class='del'>-	INIT_Q_COUNTER(duplicate_request),</div><div class='del'>-	INIT_Q_COUNTER(rnr_nak_retry_err),</div><div class='del'>-	INIT_Q_COUNTER(packet_seq_err),</div><div class='del'>-	INIT_Q_COUNTER(implied_nak_seq_err),</div><div class='del'>-	INIT_Q_COUNTER(local_ack_timeout_err),</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define INIT_CONG_COUNTER(_name)		\</div><div class='del'>-	{ .name = #_name, .offset =	\</div><div class='del'>-		MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}</div><div class='del'>-</div><div class='del'>-static const struct mlx5_ib_counter cong_cnts[] = {</div><div class='del'>-	INIT_CONG_COUNTER(rp_cnp_ignored),</div><div class='del'>-	INIT_CONG_COUNTER(rp_cnp_handled),</div><div class='del'>-	INIT_CONG_COUNTER(np_ecn_marked_roce_packets),</div><div class='del'>-	INIT_CONG_COUNTER(np_cnp_sent),</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static const struct mlx5_ib_counter extended_err_cnts[] = {</div><div class='del'>-	INIT_Q_COUNTER(resp_local_length_error),</div><div class='del'>-	INIT_Q_COUNTER(resp_cqe_error),</div><div class='del'>-	INIT_Q_COUNTER(req_cqe_error),</div><div class='del'>-	INIT_Q_COUNTER(req_remote_invalid_request),</div><div class='del'>-	INIT_Q_COUNTER(req_remote_access_errors),</div><div class='del'>-	INIT_Q_COUNTER(resp_remote_access_errors),</div><div class='del'>-	INIT_Q_COUNTER(resp_cqe_flush_error),</div><div class='del'>-	INIT_Q_COUNTER(req_cqe_flush_error),</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-#define INIT_EXT_PPCNT_COUNTER(_name)		\</div><div class='del'>-	{ .name = #_name, .offset =	\</div><div class='del'>-	MLX5_BYTE_OFF(ppcnt_reg, \</div><div class='del'>-		      counter_set.eth_extended_cntrs_grp_data_layout._name##_high)}</div><div class='del'>-</div><div class='del'>-static const struct mlx5_ib_counter ext_ppcnt_cnts[] = {</div><div class='del'>-	INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated),</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; dev-&gt;num_ports; i++) {</div><div class='del'>-		if (dev-&gt;port[i].cnts.set_id_valid)</div><div class='del'>-			mlx5_core_dealloc_q_counter(dev-&gt;mdev,</div><div class='del'>-						    dev-&gt;port[i].cnts.set_id);</div><div class='del'>-		kfree(dev-&gt;port[i].cnts.names);</div><div class='del'>-		kfree(dev-&gt;port[i].cnts.offsets);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,</div><div class='del'>-				    struct mlx5_ib_counters *cnts)</div><div class='del'>-{</div><div class='del'>-	u32 num_counters;</div><div class='del'>-</div><div class='del'>-	num_counters = ARRAY_SIZE(basic_q_cnts);</div><div class='del'>-</div><div class='del'>-	if (MLX5_CAP_GEN(dev-&gt;mdev, out_of_seq_cnt))</div><div class='del'>-		num_counters += ARRAY_SIZE(out_of_seq_q_cnts);</div><div class='del'>-</div><div class='del'>-	if (MLX5_CAP_GEN(dev-&gt;mdev, retransmission_q_counters))</div><div class='del'>-		num_counters += ARRAY_SIZE(retrans_q_cnts);</div><div class='del'>-</div><div class='del'>-	if (MLX5_CAP_GEN(dev-&gt;mdev, enhanced_error_q_counters))</div><div class='del'>-		num_counters += ARRAY_SIZE(extended_err_cnts);</div><div class='del'>-</div><div class='del'>-	cnts-&gt;num_q_counters = num_counters;</div><div class='del'>-</div><div class='del'>-	if (MLX5_CAP_GEN(dev-&gt;mdev, cc_query_allowed)) {</div><div class='del'>-		cnts-&gt;num_cong_counters = ARRAY_SIZE(cong_cnts);</div><div class='del'>-		num_counters += ARRAY_SIZE(cong_cnts);</div><div class='del'>-	}</div><div class='del'>-	if (MLX5_CAP_PCAM_FEATURE(dev-&gt;mdev, rx_icrc_encapsulated_counter)) {</div><div class='del'>-		cnts-&gt;num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts);</div><div class='del'>-		num_counters += ARRAY_SIZE(ext_ppcnt_cnts);</div><div class='del'>-	}</div><div class='del'>-	cnts-&gt;names = kcalloc(num_counters, sizeof(cnts-&gt;names), GFP_KERNEL);</div><div class='del'>-	if (!cnts-&gt;names)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	cnts-&gt;offsets = kcalloc(num_counters,</div><div class='del'>-				sizeof(cnts-&gt;offsets), GFP_KERNEL);</div><div class='del'>-	if (!cnts-&gt;offsets)</div><div class='del'>-		goto err_names;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err_names:</div><div class='del'>-	kfree(cnts-&gt;names);</div><div class='del'>-	cnts-&gt;names = NULL;</div><div class='del'>-	return -ENOMEM;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,</div><div class='del'>-				  const char **names,</div><div class='del'>-				  size_t *offsets)</div><div class='del'>-{</div><div class='del'>-	int i;</div><div class='del'>-	int j = 0;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; ARRAY_SIZE(basic_q_cnts); i++, j++) {</div><div class='del'>-		names[j] = basic_q_cnts[i].name;</div><div class='del'>-		offsets[j] = basic_q_cnts[i].offset;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (MLX5_CAP_GEN(dev-&gt;mdev, out_of_seq_cnt)) {</div><div class='del'>-		for (i = 0; i &lt; ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) {</div><div class='del'>-			names[j] = out_of_seq_q_cnts[i].name;</div><div class='del'>-			offsets[j] = out_of_seq_q_cnts[i].offset;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (MLX5_CAP_GEN(dev-&gt;mdev, retransmission_q_counters)) {</div><div class='del'>-		for (i = 0; i &lt; ARRAY_SIZE(retrans_q_cnts); i++, j++) {</div><div class='del'>-			names[j] = retrans_q_cnts[i].name;</div><div class='del'>-			offsets[j] = retrans_q_cnts[i].offset;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (MLX5_CAP_GEN(dev-&gt;mdev, enhanced_error_q_counters)) {</div><div class='del'>-		for (i = 0; i &lt; ARRAY_SIZE(extended_err_cnts); i++, j++) {</div><div class='del'>-			names[j] = extended_err_cnts[i].name;</div><div class='del'>-			offsets[j] = extended_err_cnts[i].offset;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (MLX5_CAP_GEN(dev-&gt;mdev, cc_query_allowed)) {</div><div class='del'>-		for (i = 0; i &lt; ARRAY_SIZE(cong_cnts); i++, j++) {</div><div class='del'>-			names[j] = cong_cnts[i].name;</div><div class='del'>-			offsets[j] = cong_cnts[i].offset;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (MLX5_CAP_PCAM_FEATURE(dev-&gt;mdev, rx_icrc_encapsulated_counter)) {</div><div class='del'>-		for (i = 0; i &lt; ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) {</div><div class='del'>-			names[j] = ext_ppcnt_cnts[i].name;</div><div class='del'>-			offsets[j] = ext_ppcnt_cnts[i].offset;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	int err = 0;</div><div class='del'>-	int i;</div><div class='del'>-	bool is_shared;</div><div class='del'>-</div><div class='del'>-	is_shared = MLX5_CAP_GEN(dev-&gt;mdev, log_max_uctx) != 0;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; dev-&gt;num_ports; i++) {</div><div class='del'>-		err = __mlx5_ib_alloc_counters(dev, &amp;dev-&gt;port[i].cnts);</div><div class='del'>-		if (err)</div><div class='del'>-			goto err_alloc;</div><div class='del'>-</div><div class='del'>-		mlx5_ib_fill_counters(dev, dev-&gt;port[i].cnts.names,</div><div class='del'>-				      dev-&gt;port[i].cnts.offsets);</div><div class='del'>-</div><div class='del'>-		err = mlx5_cmd_alloc_q_counter(dev-&gt;mdev,</div><div class='del'>-					       &amp;dev-&gt;port[i].cnts.set_id,</div><div class='del'>-					       is_shared ?</div><div class='del'>-					       MLX5_SHARED_RESOURCE_UID : 0);</div><div class='del'>-		if (err) {</div><div class='del'>-			mlx5_ib_warn(dev,</div><div class='del'>-				     "couldn't allocate queue counter for port %d, err %d\n",</div><div class='del'>-				     i + 1, err);</div><div class='del'>-			goto err_alloc;</div><div class='del'>-		}</div><div class='del'>-		dev-&gt;port[i].cnts.set_id_valid = true;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err_alloc:</div><div class='del'>-	mlx5_ib_dealloc_counters(dev);</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev,</div><div class='del'>-						    u8 port_num)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='del'>-	struct mlx5_ib_port *port = &amp;dev-&gt;port[port_num - 1];</div><div class='del'>-</div><div class='del'>-	/* We support only per port stats */</div><div class='del'>-	if (port_num == 0)</div><div class='del'>-		return NULL;</div><div class='del'>-</div><div class='del'>-	return rdma_alloc_hw_stats_struct(port-&gt;cnts.names,</div><div class='del'>-					  port-&gt;cnts.num_q_counters +</div><div class='del'>-					  port-&gt;cnts.num_cong_counters +</div><div class='del'>-					  port-&gt;cnts.num_ext_ppcnt_counters,</div><div class='del'>-					  RDMA_HW_STATS_DEFAULT_LIFESPAN);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,</div><div class='del'>-				    struct mlx5_ib_port *port,</div><div class='del'>-				    struct rdma_hw_stats *stats)</div><div class='del'>-{</div><div class='del'>-	int outlen = MLX5_ST_SZ_BYTES(query_q_counter_out);</div><div class='del'>-	void *out;</div><div class='del'>-	__be32 val;</div><div class='del'>-	int ret, i;</div><div class='del'>-</div><div class='del'>-	out = kvzalloc(outlen, GFP_KERNEL);</div><div class='del'>-	if (!out)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	ret = mlx5_core_query_q_counter(mdev,</div><div class='del'>-					port-&gt;cnts.set_id, 0,</div><div class='del'>-					out, outlen);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto free;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; port-&gt;cnts.num_q_counters; i++) {</div><div class='del'>-		val = *(__be32 *)(out + port-&gt;cnts.offsets[i]);</div><div class='del'>-		stats-&gt;value[i] = (u64)be32_to_cpu(val);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-free:</div><div class='del'>-	kvfree(out);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev,</div><div class='del'>-					  struct mlx5_ib_port *port,</div><div class='del'>-					  struct rdma_hw_stats *stats)</div><div class='del'>-{</div><div class='del'>-	int offset = port-&gt;cnts.num_q_counters + port-&gt;cnts.num_cong_counters;</div><div class='del'>-	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);</div><div class='del'>-	int ret, i;</div><div class='del'>-	void *out;</div><div class='del'>-</div><div class='del'>-	out = kvzalloc(sz, GFP_KERNEL);</div><div class='del'>-	if (!out)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	ret = mlx5_cmd_query_ext_ppcnt_counters(dev-&gt;mdev, out);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto free;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; port-&gt;cnts.num_ext_ppcnt_counters; i++) {</div><div class='del'>-		stats-&gt;value[i + offset] =</div><div class='del'>-			be64_to_cpup((__be64 *)(out +</div><div class='del'>-				    port-&gt;cnts.offsets[i + offset]));</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-free:</div><div class='del'>-	kvfree(out);</div><div class='del'>-	return ret;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,</div><div class='del'>-				struct rdma_hw_stats *stats,</div><div class='del'>-				u8 port_num, int index)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='del'>-	struct mlx5_ib_port *port = &amp;dev-&gt;port[port_num - 1];</div><div class='del'>-	struct mlx5_core_dev *mdev;</div><div class='del'>-	int ret, num_counters;</div><div class='del'>-	u8 mdev_port_num;</div><div class='del'>-</div><div class='del'>-	if (!stats)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	num_counters = port-&gt;cnts.num_q_counters +</div><div class='del'>-		       port-&gt;cnts.num_cong_counters +</div><div class='del'>-		       port-&gt;cnts.num_ext_ppcnt_counters;</div><div class='del'>-</div><div class='del'>-	/* q_counters are per IB device, query the master mdev */</div><div class='del'>-	ret = mlx5_ib_query_q_counters(dev-&gt;mdev, port, stats);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	if (MLX5_CAP_PCAM_FEATURE(dev-&gt;mdev, rx_icrc_encapsulated_counter)) {</div><div class='del'>-		ret =  mlx5_ib_query_ext_ppcnt_counters(dev, port, stats);</div><div class='del'>-		if (ret)</div><div class='del'>-			return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (MLX5_CAP_GEN(dev-&gt;mdev, cc_query_allowed)) {</div><div class='del'>-		mdev = mlx5_ib_get_native_port_mdev(dev, port_num,</div><div class='del'>-						    &amp;mdev_port_num);</div><div class='del'>-		if (!mdev) {</div><div class='del'>-			/* If port is not affiliated yet, its in down state</div><div class='del'>-			 * which doesn't have any counters yet, so it would be</div><div class='del'>-			 * zero. So no need to read from the HCA.</div><div class='del'>-			 */</div><div class='del'>-			goto done;</div><div class='del'>-		}</div><div class='del'>-		ret = mlx5_lag_query_cong_counters(dev-&gt;mdev,</div><div class='del'>-						   stats-&gt;value +</div><div class='del'>-						   port-&gt;cnts.num_q_counters,</div><div class='del'>-						   port-&gt;cnts.num_cong_counters,</div><div class='del'>-						   port-&gt;cnts.offsets +</div><div class='del'>-						   port-&gt;cnts.num_q_counters);</div><div class='del'>-</div><div class='del'>-		mlx5_ib_put_native_port_mdev(dev, port_num);</div><div class='del'>-		if (ret)</div><div class='del'>-			return ret;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-done:</div><div class='del'>-	return num_counters;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx5_ib_rn_get_params(struct ib_device *device, u8 port_num,</div><div class='add'>+static int mlx5_ib_rn_get_params(struct ib_device *device, u32 port_num,</div><div class='ctx'> 				 enum rdma_netdev_t type,</div><div class='ctx'> 				 struct rdma_netdev_alloc_params *params)</div><div class='ctx'> {</div><div class='hunk'>@@ -5543,24 +3568,6 @@ static int mlx5_ib_rn_get_params(struct ib_device *device, u8 port_num,</div><div class='ctx'> 	return mlx5_rdma_rn_get_params(to_mdev(device)-&gt;mdev, device, params);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void delay_drop_debugfs_cleanup(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	if (!dev-&gt;delay_drop.dbg)</div><div class='del'>-		return;</div><div class='del'>-	debugfs_remove_recursive(dev-&gt;delay_drop.dbg-&gt;dir_debugfs);</div><div class='del'>-	kfree(dev-&gt;delay_drop.dbg);</div><div class='del'>-	dev-&gt;delay_drop.dbg = NULL;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void cancel_delay_drop(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	if (!(dev-&gt;ib_dev.attrs.raw_packet_caps &amp; IB_RAW_PACKET_CAP_DELAY_DROP))</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	cancel_work_sync(&amp;dev-&gt;delay_drop.delay_drop_work);</div><div class='del'>-	delay_drop_debugfs_cleanup(dev);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static ssize_t delay_drop_timeout_read(struct file *filp, char __user *buf,</div><div class='ctx'> 				       size_t count, loff_t *pos)</div><div class='ctx'> {</div><div class='hunk'>@@ -5600,81 +3607,26 @@ static const struct file_operations fops_delay_drop_timeout = {</div><div class='ctx'> 	.read	= delay_drop_timeout_read,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static int delay_drop_debugfs_init(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_dbg_delay_drop *dbg;</div><div class='del'>-</div><div class='del'>-	if (!mlx5_debugfs_root)</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	dbg = kzalloc(sizeof(*dbg), GFP_KERNEL);</div><div class='del'>-	if (!dbg)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	dev-&gt;delay_drop.dbg = dbg;</div><div class='del'>-</div><div class='del'>-	dbg-&gt;dir_debugfs =</div><div class='del'>-		debugfs_create_dir("delay_drop",</div><div class='del'>-				   dev-&gt;mdev-&gt;priv.dbg_root);</div><div class='del'>-	if (!dbg-&gt;dir_debugfs)</div><div class='del'>-		goto out_debugfs;</div><div class='del'>-</div><div class='del'>-	dbg-&gt;events_cnt_debugfs =</div><div class='del'>-		debugfs_create_atomic_t("num_timeout_events", 0400,</div><div class='del'>-					dbg-&gt;dir_debugfs,</div><div class='del'>-					&amp;dev-&gt;delay_drop.events_cnt);</div><div class='del'>-	if (!dbg-&gt;events_cnt_debugfs)</div><div class='del'>-		goto out_debugfs;</div><div class='del'>-</div><div class='del'>-	dbg-&gt;rqs_cnt_debugfs =</div><div class='del'>-		debugfs_create_atomic_t("num_rqs", 0400,</div><div class='del'>-					dbg-&gt;dir_debugfs,</div><div class='del'>-					&amp;dev-&gt;delay_drop.rqs_cnt);</div><div class='del'>-	if (!dbg-&gt;rqs_cnt_debugfs)</div><div class='del'>-		goto out_debugfs;</div><div class='del'>-</div><div class='del'>-	dbg-&gt;timeout_debugfs =</div><div class='del'>-		debugfs_create_file("timeout", 0600,</div><div class='del'>-				    dbg-&gt;dir_debugfs,</div><div class='del'>-				    &amp;dev-&gt;delay_drop,</div><div class='del'>-				    &amp;fops_delay_drop_timeout);</div><div class='del'>-	if (!dbg-&gt;timeout_debugfs)</div><div class='del'>-		goto out_debugfs;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-out_debugfs:</div><div class='del'>-	delay_drop_debugfs_cleanup(dev);</div><div class='del'>-	return -ENOMEM;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void init_delay_drop(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	if (!(dev-&gt;ib_dev.attrs.raw_packet_caps &amp; IB_RAW_PACKET_CAP_DELAY_DROP))</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	mutex_init(&amp;dev-&gt;delay_drop.lock);</div><div class='del'>-	dev-&gt;delay_drop.dev = dev;</div><div class='del'>-	dev-&gt;delay_drop.activate = false;</div><div class='del'>-	dev-&gt;delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000;</div><div class='del'>-	INIT_WORK(&amp;dev-&gt;delay_drop.delay_drop_work, delay_drop_handler);</div><div class='del'>-	atomic_set(&amp;dev-&gt;delay_drop.rqs_cnt, 0);</div><div class='del'>-	atomic_set(&amp;dev-&gt;delay_drop.events_cnt, 0);</div><div class='del'>-</div><div class='del'>-	if (delay_drop_debugfs_init(dev))</div><div class='del'>-		mlx5_ib_warn(dev, "Failed to init delay drop debugfs\n");</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* The mlx5_ib_multiport_mutex should be held when calling this function */</div><div class='ctx'> static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,</div><div class='ctx'> 				      struct mlx5_ib_multiport_info *mpi)</div><div class='ctx'> {</div><div class='del'>-	u8 port_num = mlx5_core_native_port_num(mpi-&gt;mdev) - 1;</div><div class='add'>+	u32 port_num = mlx5_core_native_port_num(mpi-&gt;mdev) - 1;</div><div class='ctx'> 	struct mlx5_ib_port *port = &amp;ibdev-&gt;port[port_num];</div><div class='ctx'> 	int comps;</div><div class='ctx'> 	int err;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='add'>+	lockdep_assert_held(&amp;mlx5_ib_multiport_mutex);</div><div class='add'>+</div><div class='add'>+	mlx5_ib_disable_lb_mp(ibdev-&gt;mdev, mpi-&gt;mdev, &amp;ibdev-&gt;lb);</div><div class='add'>+</div><div class='add'>+	mlx5_core_mp_event_replay(ibdev-&gt;mdev,</div><div class='add'>+				  MLX5_DRIVER_EVENT_AFFILIATION_REMOVED,</div><div class='add'>+				  NULL);</div><div class='add'>+	mlx5_core_mp_event_replay(mpi-&gt;mdev,</div><div class='add'>+				  MLX5_DRIVER_EVENT_AFFILIATION_REMOVED,</div><div class='add'>+				  NULL);</div><div class='add'>+</div><div class='ctx'> 	mlx5_ib_cleanup_cong_debugfs(ibdev, port_num);</div><div class='ctx'> </div><div class='ctx'> 	spin_lock(&amp;port-&gt;mp.mpi_lock);</div><div class='hunk'>@@ -5683,14 +3635,13 @@ static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (mpi-&gt;mdev_events.notifier_call)</div><div class='del'>-		mlx5_notifier_unregister(mpi-&gt;mdev, &amp;mpi-&gt;mdev_events);</div><div class='del'>-	mpi-&gt;mdev_events.notifier_call = NULL;</div><div class='del'>-</div><div class='ctx'> 	mpi-&gt;ibdev = NULL;</div><div class='ctx'> </div><div class='ctx'> 	spin_unlock(&amp;port-&gt;mp.mpi_lock);</div><div class='del'>-	mlx5_remove_netdev_notifier(ibdev, port_num);</div><div class='add'>+	if (mpi-&gt;mdev_events.notifier_call)</div><div class='add'>+		mlx5_notifier_unregister(mpi-&gt;mdev, &amp;mpi-&gt;mdev_events);</div><div class='add'>+	mpi-&gt;mdev_events.notifier_call = NULL;</div><div class='add'>+	mlx5_mdev_netdev_untrack(ibdev, port_num);</div><div class='ctx'> 	spin_lock(&amp;port-&gt;mp.mpi_lock);</div><div class='ctx'> </div><div class='ctx'> 	comps = mpi-&gt;mdev_refcnt;</div><div class='hunk'>@@ -5708,13 +3659,11 @@ static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,</div><div class='ctx'> </div><div class='ctx'> 	port-&gt;mp.mpi = NULL;</div><div class='ctx'> </div><div class='del'>-	list_add_tail(&amp;mpi-&gt;list, &amp;mlx5_ib_unaffiliated_port_list);</div><div class='del'>-</div><div class='ctx'> 	spin_unlock(&amp;port-&gt;mp.mpi_lock);</div><div class='ctx'> </div><div class='ctx'> 	err = mlx5_nic_vport_unaffiliate_multiport(mpi-&gt;mdev);</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_dbg(ibdev, "unaffiliated port %d\n", port_num + 1);</div><div class='add'>+	mlx5_ib_dbg(ibdev, "unaffiliated port %u\n", port_num + 1);</div><div class='ctx'> 	/* Log an error, still needed to cleanup the pointers and add</div><div class='ctx'> 	 * it back to the list.</div><div class='ctx'> 	 */</div><div class='hunk'>@@ -5725,16 +3674,18 @@ static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,</div><div class='ctx'> 	ibdev-&gt;port[port_num].roce.last_port_state = IB_PORT_DOWN;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/* The mlx5_ib_multiport_mutex should be held when calling this function */</div><div class='ctx'> static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev,</div><div class='ctx'> 				    struct mlx5_ib_multiport_info *mpi)</div><div class='ctx'> {</div><div class='del'>-	u8 port_num = mlx5_core_native_port_num(mpi-&gt;mdev) - 1;</div><div class='add'>+	u32 port_num = mlx5_core_native_port_num(mpi-&gt;mdev) - 1;</div><div class='add'>+	u64 key;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='add'>+	lockdep_assert_held(&amp;mlx5_ib_multiport_mutex);</div><div class='add'>+</div><div class='ctx'> 	spin_lock(&amp;ibdev-&gt;port[port_num].mp.mpi_lock);</div><div class='ctx'> 	if (ibdev-&gt;port[port_num].mp.mpi) {</div><div class='del'>-		mlx5_ib_dbg(ibdev, "port %d already affiliated.\n",</div><div class='add'>+		mlx5_ib_dbg(ibdev, "port %u already affiliated.\n",</div><div class='ctx'> 			    port_num + 1);</div><div class='ctx'> 		spin_unlock(&amp;ibdev-&gt;port[port_num].mp.mpi_lock);</div><div class='ctx'> 		return false;</div><div class='hunk'>@@ -5749,22 +3700,25 @@ static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev,</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto unbind;</div><div class='ctx'> </div><div class='del'>-	err = get_port_caps(ibdev, mlx5_core_native_port_num(mpi-&gt;mdev));</div><div class='del'>-	if (err)</div><div class='del'>-		goto unbind;</div><div class='del'>-</div><div class='del'>-	err = mlx5_add_netdev_notifier(ibdev, port_num);</div><div class='del'>-	if (err) {</div><div class='del'>-		mlx5_ib_err(ibdev, "failed adding netdev notifier for port %u\n",</div><div class='del'>-			    port_num + 1);</div><div class='del'>-		goto unbind;</div><div class='del'>-	}</div><div class='add'>+	mlx5_mdev_netdev_track(ibdev, port_num);</div><div class='ctx'> </div><div class='ctx'> 	mpi-&gt;mdev_events.notifier_call = mlx5_ib_event_slave_port;</div><div class='ctx'> 	mlx5_notifier_register(mpi-&gt;mdev, &amp;mpi-&gt;mdev_events);</div><div class='ctx'> </div><div class='ctx'> 	mlx5_ib_init_cong_debugfs(ibdev, port_num);</div><div class='ctx'> </div><div class='add'>+	key = mpi-&gt;mdev-&gt;priv.adev_idx;</div><div class='add'>+	mlx5_core_mp_event_replay(mpi-&gt;mdev,</div><div class='add'>+				  MLX5_DRIVER_EVENT_AFFILIATION_DONE,</div><div class='add'>+				  &amp;key);</div><div class='add'>+	mlx5_core_mp_event_replay(ibdev-&gt;mdev,</div><div class='add'>+				  MLX5_DRIVER_EVENT_AFFILIATION_DONE,</div><div class='add'>+				  &amp;key);</div><div class='add'>+</div><div class='add'>+	err = mlx5_ib_enable_lb_mp(ibdev-&gt;mdev, mpi-&gt;mdev, &amp;ibdev-&gt;lb);</div><div class='add'>+	if (err)</div><div class='add'>+		goto unbind;</div><div class='add'>+</div><div class='ctx'> 	return true;</div><div class='ctx'> </div><div class='ctx'> unbind:</div><div class='hunk'>@@ -5772,14 +3726,49 @@ unbind:</div><div class='ctx'> 	return false;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1] = {};</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (!MLX5_CAP_GEN(dev-&gt;mdev, data_direct) ||</div><div class='add'>+	    !MLX5_CAP_GEN_2(dev-&gt;mdev, query_vuid))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	ret = mlx5_cmd_query_vuid(dev-&gt;mdev, true, vuid);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = mlx5_ib_create_data_direct_resources(dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	INIT_LIST_HEAD(&amp;dev-&gt;data_direct_mr_list);</div><div class='add'>+	ret = mlx5_data_direct_ib_reg(dev, vuid);</div><div class='add'>+	if (ret)</div><div class='add'>+		mlx5_ib_free_data_direct_resources(dev);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	if (!MLX5_CAP_GEN(dev-&gt;mdev, data_direct) ||</div><div class='add'>+	    !MLX5_CAP_GEN_2(dev-&gt;mdev, query_vuid))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	mlx5_data_direct_ib_unreg(dev);</div><div class='add'>+	mlx5_ib_free_data_direct_resources(dev);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	int port_num = mlx5_core_native_port_num(dev-&gt;mdev) - 1;</div><div class='add'>+	u32 port_num = mlx5_core_native_port_num(dev-&gt;mdev) - 1;</div><div class='ctx'> 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&amp;dev-&gt;ib_dev,</div><div class='ctx'> 							  port_num + 1);</div><div class='ctx'> 	struct mlx5_ib_multiport_info *mpi;</div><div class='ctx'> 	int err;</div><div class='del'>-	int i;</div><div class='add'>+	u32 i;</div><div class='ctx'> </div><div class='ctx'> 	if (!mlx5_core_is_mp_master(dev-&gt;mdev) || ll != IB_LINK_LAYER_ETHERNET)</div><div class='ctx'> 		return 0;</div><div class='hunk'>@@ -5818,7 +3807,8 @@ static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)</div><div class='ctx'> 		list_for_each_entry(mpi, &amp;mlx5_ib_unaffiliated_port_list,</div><div class='ctx'> 				    list) {</div><div class='ctx'> 			if (dev-&gt;sys_image_guid == mpi-&gt;sys_image_guid &amp;&amp;</div><div class='del'>-			    (mlx5_core_native_port_num(mpi-&gt;mdev) - 1) == i) {</div><div class='add'>+			    (mlx5_core_native_port_num(mpi-&gt;mdev) - 1) == i &amp;&amp;</div><div class='add'>+			    mlx5_core_same_coredev_type(dev-&gt;mdev, mpi-&gt;mdev)) {</div><div class='ctx'> 				bound = mlx5_ib_bind_slave_port(dev, mpi);</div><div class='ctx'> 			}</div><div class='ctx'> </div><div class='hunk'>@@ -5830,11 +3820,9 @@ static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)</div><div class='ctx'> 				break;</div><div class='ctx'> 			}</div><div class='ctx'> 		}</div><div class='del'>-		if (!bound) {</div><div class='del'>-			get_port_caps(dev, i + 1);</div><div class='add'>+		if (!bound)</div><div class='ctx'> 			mlx5_ib_dbg(dev, "no free port found for port %d\n",</div><div class='ctx'> 				    i + 1);</div><div class='del'>-		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	list_add_tail(&amp;dev-&gt;ib_dev_list, &amp;mlx5_ib_dev_list);</div><div class='hunk'>@@ -5844,10 +3832,10 @@ static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)</div><div class='ctx'> </div><div class='ctx'> static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	int port_num = mlx5_core_native_port_num(dev-&gt;mdev) - 1;</div><div class='add'>+	u32 port_num = mlx5_core_native_port_num(dev-&gt;mdev) - 1;</div><div class='ctx'> 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&amp;dev-&gt;ib_dev,</div><div class='ctx'> 							  port_num + 1);</div><div class='del'>-	int i;</div><div class='add'>+	u32 i;</div><div class='ctx'> </div><div class='ctx'> 	if (!mlx5_core_is_mp_master(dev-&gt;mdev) || ll != IB_LINK_LAYER_ETHERNET)</div><div class='ctx'> 		return;</div><div class='hunk'>@@ -5860,8 +3848,12 @@ static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev)</div><div class='ctx'> 				kfree(dev-&gt;port[i].mp.mpi);</div><div class='ctx'> 				dev-&gt;port[i].mp.mpi = NULL;</div><div class='ctx'> 			} else {</div><div class='del'>-				mlx5_ib_dbg(dev, "unbinding port_num: %d\n", i + 1);</div><div class='del'>-				mlx5_ib_unbind_slave_port(dev, dev-&gt;port[i].mp.mpi);</div><div class='add'>+				mlx5_ib_dbg(dev, "unbinding port_num: %u\n",</div><div class='add'>+					    i + 1);</div><div class='add'>+				list_add_tail(&amp;dev-&gt;port[i].mp.mpi-&gt;list,</div><div class='add'>+					      &amp;mlx5_ib_unaffiliated_port_list);</div><div class='add'>+				mlx5_ib_unbind_slave_port(dev,</div><div class='add'>+							  dev-&gt;port[i].mp.mpi);</div><div class='ctx'> 			}</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='hunk'>@@ -5873,297 +3865,432 @@ static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev)</div><div class='ctx'> 	mlx5_nic_vport_disable_roce(dev-&gt;mdev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-ADD_UVERBS_ATTRIBUTES_SIMPLE(</div><div class='del'>-	mlx5_ib_dm,</div><div class='del'>-	UVERBS_OBJECT_DM,</div><div class='del'>-	UVERBS_METHOD_DM_ALLOC,</div><div class='del'>-	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,</div><div class='del'>-			    UVERBS_ATTR_TYPE(u64),</div><div class='del'>-			    UA_MANDATORY),</div><div class='del'>-	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,</div><div class='del'>-			    UVERBS_ATTR_TYPE(u16),</div><div class='del'>-			    UA_OPTIONAL),</div><div class='del'>-	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,</div><div class='del'>-			     enum mlx5_ib_uapi_dm_type,</div><div class='del'>-			     UA_OPTIONAL));</div><div class='del'>-</div><div class='del'>-ADD_UVERBS_ATTRIBUTES_SIMPLE(</div><div class='del'>-	mlx5_ib_flow_action,</div><div class='del'>-	UVERBS_OBJECT_FLOW_ACTION,</div><div class='del'>-	UVERBS_METHOD_FLOW_ACTION_ESP_CREATE,</div><div class='del'>-	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS,</div><div class='del'>-			     enum mlx5_ib_uapi_flow_action_flags));</div><div class='add'>+static int mmap_obj_cleanup(struct ib_uobject *uobject,</div><div class='add'>+			    enum rdma_remove_reason why,</div><div class='add'>+			    struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_user_mmap_entry *obj = uobject-&gt;object;</div><div class='ctx'> </div><div class='del'>-static const struct uapi_definition mlx5_ib_defs[] = {</div><div class='del'>-#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)</div><div class='del'>-	UAPI_DEF_CHAIN(mlx5_ib_devx_defs),</div><div class='del'>-	UAPI_DEF_CHAIN(mlx5_ib_flow_defs),</div><div class='del'>-#endif</div><div class='add'>+	rdma_user_mmap_entry_remove(&amp;obj-&gt;rdma_entry);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,</div><div class='del'>-				&amp;mlx5_ib_flow_action),</div><div class='del'>-	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &amp;mlx5_ib_dm),</div><div class='del'>-	{}</div><div class='del'>-};</div><div class='add'>+static int mlx5_rdma_user_mmap_entry_insert(struct mlx5_ib_ucontext *c,</div><div class='add'>+					    struct mlx5_user_mmap_entry *entry,</div><div class='add'>+					    size_t length)</div><div class='add'>+{</div><div class='add'>+	return rdma_user_mmap_entry_insert_range(</div><div class='add'>+		&amp;c-&gt;ibucontext, &amp;entry-&gt;rdma_entry, length,</div><div class='add'>+		(MLX5_IB_MMAP_OFFSET_START &lt;&lt; 16),</div><div class='add'>+		((MLX5_IB_MMAP_OFFSET_END &lt;&lt; 16) + (1UL &lt;&lt; 16) - 1));</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-static int mlx5_ib_read_counters(struct ib_counters *counters,</div><div class='del'>-				 struct ib_counters_read_attr *read_attr,</div><div class='del'>-				 struct uverbs_attr_bundle *attrs)</div><div class='add'>+static struct mlx5_user_mmap_entry *</div><div class='add'>+alloc_var_entry(struct mlx5_ib_ucontext *c)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);</div><div class='del'>-	struct mlx5_read_counters_attr mread_attr = {};</div><div class='del'>-	struct mlx5_ib_flow_counters_desc *desc;</div><div class='del'>-	int ret, i;</div><div class='add'>+	struct mlx5_user_mmap_entry *entry;</div><div class='add'>+	struct mlx5_var_table *var_table;</div><div class='add'>+	u32 page_idx;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;mcounters-&gt;mcntrs_mutex);</div><div class='del'>-	if (mcounters-&gt;cntrs_max_index &gt; read_attr-&gt;ncounters) {</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-		goto err_bound;</div><div class='del'>-	}</div><div class='add'>+	var_table = &amp;to_mdev(c-&gt;ibucontext.device)-&gt;var_table;</div><div class='add'>+	entry = kzalloc(sizeof(*entry), GFP_KERNEL);</div><div class='add'>+	if (!entry)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='del'>-	mread_attr.out = kcalloc(mcounters-&gt;counters_num, sizeof(u64),</div><div class='del'>-				 GFP_KERNEL);</div><div class='del'>-	if (!mread_attr.out) {</div><div class='del'>-		ret = -ENOMEM;</div><div class='del'>-		goto err_bound;</div><div class='add'>+	mutex_lock(&amp;var_table-&gt;bitmap_lock);</div><div class='add'>+	page_idx = find_first_zero_bit(var_table-&gt;bitmap,</div><div class='add'>+				       var_table-&gt;num_var_hw_entries);</div><div class='add'>+	if (page_idx &gt;= var_table-&gt;num_var_hw_entries) {</div><div class='add'>+		err = -ENOSPC;</div><div class='add'>+		mutex_unlock(&amp;var_table-&gt;bitmap_lock);</div><div class='add'>+		goto end;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	mread_attr.hw_cntrs_hndl = mcounters-&gt;hw_cntrs_hndl;</div><div class='del'>-	mread_attr.flags = read_attr-&gt;flags;</div><div class='del'>-	ret = mcounters-&gt;read_counters(counters-&gt;device, &amp;mread_attr);</div><div class='del'>-	if (ret)</div><div class='del'>-		goto err_read;</div><div class='add'>+	set_bit(page_idx, var_table-&gt;bitmap);</div><div class='add'>+	mutex_unlock(&amp;var_table-&gt;bitmap_lock);</div><div class='ctx'> </div><div class='del'>-	/* do the pass over the counters data array to assign according to the</div><div class='del'>-	 * descriptions and indexing pairs</div><div class='del'>-	 */</div><div class='del'>-	desc = mcounters-&gt;counters_data;</div><div class='del'>-	for (i = 0; i &lt; mcounters-&gt;ncounters; i++)</div><div class='del'>-		read_attr-&gt;counters_buff[desc[i].index] += mread_attr.out[desc[i].description];</div><div class='del'>-</div><div class='del'>-err_read:</div><div class='del'>-	kfree(mread_attr.out);</div><div class='del'>-err_bound:</div><div class='del'>-	mutex_unlock(&amp;mcounters-&gt;mcntrs_mutex);</div><div class='del'>-	return ret;</div><div class='add'>+	entry-&gt;address = var_table-&gt;hw_start_addr +</div><div class='add'>+				(page_idx * var_table-&gt;stride_size);</div><div class='add'>+	entry-&gt;page_idx = page_idx;</div><div class='add'>+	entry-&gt;mmap_flag = MLX5_IB_MMAP_TYPE_VAR;</div><div class='add'>+</div><div class='add'>+	err = mlx5_rdma_user_mmap_entry_insert(c, entry,</div><div class='add'>+					       var_table-&gt;stride_size);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_insert;</div><div class='add'>+</div><div class='add'>+	return entry;</div><div class='add'>+</div><div class='add'>+err_insert:</div><div class='add'>+	mutex_lock(&amp;var_table-&gt;bitmap_lock);</div><div class='add'>+	clear_bit(page_idx, var_table-&gt;bitmap);</div><div class='add'>+	mutex_unlock(&amp;var_table-&gt;bitmap_lock);</div><div class='add'>+end:</div><div class='add'>+	kfree(entry);</div><div class='add'>+	return ERR_PTR(err);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_ib_destroy_counters(struct ib_counters *counters)</div><div class='add'>+static int UVERBS_HANDLER(MLX5_IB_METHOD_VAR_OBJ_ALLOC)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);</div><div class='add'>+	struct ib_uobject *uobj = uverbs_attr_get_uobject(</div><div class='add'>+		attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE);</div><div class='add'>+	struct mlx5_ib_ucontext *c;</div><div class='add'>+	struct mlx5_user_mmap_entry *entry;</div><div class='add'>+	u64 mmap_offset;</div><div class='add'>+	u32 length;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-	counters_clear_description(counters);</div><div class='del'>-	if (mcounters-&gt;hw_cntrs_hndl)</div><div class='del'>-		mlx5_fc_destroy(to_mdev(counters-&gt;device)-&gt;mdev,</div><div class='del'>-				mcounters-&gt;hw_cntrs_hndl);</div><div class='add'>+	c = to_mucontext(ib_uverbs_get_ucontext(attrs));</div><div class='add'>+	if (IS_ERR(c))</div><div class='add'>+		return PTR_ERR(c);</div><div class='ctx'> </div><div class='del'>-	kfree(mcounters);</div><div class='add'>+	entry = alloc_var_entry(c);</div><div class='add'>+	if (IS_ERR(entry))</div><div class='add'>+		return PTR_ERR(entry);</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+	mmap_offset = mlx5_entry_to_mmap_offset(entry);</div><div class='add'>+	length = entry-&gt;rdma_entry.npages * PAGE_SIZE;</div><div class='add'>+	uobj-&gt;object = entry;</div><div class='add'>+	uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE);</div><div class='add'>+</div><div class='add'>+	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,</div><div class='add'>+			     &amp;mmap_offset, sizeof(mmap_offset));</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,</div><div class='add'>+			     &amp;entry-&gt;page_idx, sizeof(entry-&gt;page_idx));</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,</div><div class='add'>+			     &amp;length, sizeof(length));</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct ib_counters *mlx5_ib_create_counters(struct ib_device *device,</div><div class='del'>-						   struct uverbs_attr_bundle *attrs)</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	MLX5_IB_METHOD_VAR_OBJ_ALLOC,</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE,</div><div class='add'>+			MLX5_IB_OBJECT_VAR,</div><div class='add'>+			UVERBS_ACCESS_NEW,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			    UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD_DESTROY(</div><div class='add'>+	MLX5_IB_METHOD_VAR_OBJ_DESTROY,</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_VAR_OBJ_DESTROY_HANDLE,</div><div class='add'>+			MLX5_IB_OBJECT_VAR,</div><div class='add'>+			UVERBS_ACCESS_DESTROY,</div><div class='add'>+			UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_VAR,</div><div class='add'>+			    UVERBS_TYPE_ALLOC_IDR(mmap_obj_cleanup),</div><div class='add'>+			    &amp;UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_ALLOC),</div><div class='add'>+			    &amp;UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_DESTROY));</div><div class='add'>+</div><div class='add'>+static bool var_is_supported(struct ib_device *device)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_mcounters *mcounters;</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='ctx'> </div><div class='del'>-	mcounters = kzalloc(sizeof(*mcounters), GFP_KERNEL);</div><div class='del'>-	if (!mcounters)</div><div class='add'>+	return (MLX5_CAP_GEN_64(dev-&gt;mdev, general_obj_types) &amp;</div><div class='add'>+			MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct mlx5_user_mmap_entry *</div><div class='add'>+alloc_uar_entry(struct mlx5_ib_ucontext *c,</div><div class='add'>+		enum mlx5_ib_uapi_uar_alloc_type alloc_type)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_user_mmap_entry *entry;</div><div class='add'>+	struct mlx5_ib_dev *dev;</div><div class='add'>+	u32 uar_index;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	entry = kzalloc(sizeof(*entry), GFP_KERNEL);</div><div class='add'>+	if (!entry)</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='del'>-	mutex_init(&amp;mcounters-&gt;mcntrs_mutex);</div><div class='add'>+	dev = to_mdev(c-&gt;ibucontext.device);</div><div class='add'>+	err = mlx5_cmd_uar_alloc(dev-&gt;mdev, &amp;uar_index, c-&gt;devx_uid);</div><div class='add'>+	if (err)</div><div class='add'>+		goto end;</div><div class='ctx'> </div><div class='del'>-	return &amp;mcounters-&gt;ibcntrs;</div><div class='add'>+	entry-&gt;page_idx = uar_index;</div><div class='add'>+	entry-&gt;address = uar_index2paddress(dev, uar_index);</div><div class='add'>+	if (alloc_type == MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF)</div><div class='add'>+		entry-&gt;mmap_flag = MLX5_IB_MMAP_TYPE_UAR_WC;</div><div class='add'>+	else</div><div class='add'>+		entry-&gt;mmap_flag = MLX5_IB_MMAP_TYPE_UAR_NC;</div><div class='add'>+</div><div class='add'>+	err = mlx5_rdma_user_mmap_entry_insert(c, entry, PAGE_SIZE);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_insert;</div><div class='add'>+</div><div class='add'>+	return entry;</div><div class='add'>+</div><div class='add'>+err_insert:</div><div class='add'>+	mlx5_cmd_uar_dealloc(dev-&gt;mdev, uar_index, c-&gt;devx_uid);</div><div class='add'>+end:</div><div class='add'>+	kfree(entry);</div><div class='add'>+	return ERR_PTR(err);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)</div><div class='add'>+static int UVERBS_HANDLER(MLX5_IB_METHOD_UAR_OBJ_ALLOC)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='add'>+	struct ib_uobject *uobj = uverbs_attr_get_uobject(</div><div class='add'>+		attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE);</div><div class='add'>+	enum mlx5_ib_uapi_uar_alloc_type alloc_type;</div><div class='add'>+	struct mlx5_ib_ucontext *c;</div><div class='add'>+	struct mlx5_user_mmap_entry *entry;</div><div class='add'>+	u64 mmap_offset;</div><div class='add'>+	u32 length;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_cleanup_multiport_master(dev);</div><div class='del'>-	if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {</div><div class='del'>-		srcu_barrier(&amp;dev-&gt;mr_srcu);</div><div class='del'>-		cleanup_srcu_struct(&amp;dev-&gt;mr_srcu);</div><div class='del'>-	}</div><div class='add'>+	c = to_mucontext(ib_uverbs_get_ucontext(attrs));</div><div class='add'>+	if (IS_ERR(c))</div><div class='add'>+		return PTR_ERR(c);</div><div class='ctx'> </div><div class='del'>-	WARN_ON(!bitmap_empty(dev-&gt;dm.memic_alloc_pages, MLX5_MAX_MEMIC_PAGES));</div><div class='add'>+	err = uverbs_get_const(&amp;alloc_type, attrs,</div><div class='add'>+			       MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='ctx'> </div><div class='del'>-	WARN_ON(dev-&gt;dm.steering_sw_icm_alloc_blocks &amp;&amp;</div><div class='del'>-		!bitmap_empty(</div><div class='del'>-			dev-&gt;dm.steering_sw_icm_alloc_blocks,</div><div class='del'>-			BIT(MLX5_CAP_DEV_MEM(mdev, log_steering_sw_icm_size) -</div><div class='del'>-			    MLX5_LOG_SW_ICM_BLOCK_SIZE(mdev))));</div><div class='add'>+	if (alloc_type != MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF &amp;&amp;</div><div class='add'>+	    alloc_type != MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='del'>-	kfree(dev-&gt;dm.steering_sw_icm_alloc_blocks);</div><div class='add'>+	if (!mlx5_wc_support_get(to_mdev(c-&gt;ibucontext.device)-&gt;mdev) &amp;&amp;</div><div class='add'>+	    alloc_type == MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='del'>-	WARN_ON(dev-&gt;dm.header_modify_sw_icm_alloc_blocks &amp;&amp;</div><div class='del'>-		!bitmap_empty(dev-&gt;dm.header_modify_sw_icm_alloc_blocks,</div><div class='del'>-			      BIT(MLX5_CAP_DEV_MEM(</div><div class='del'>-					  mdev, log_header_modify_sw_icm_size) -</div><div class='del'>-				  MLX5_LOG_SW_ICM_BLOCK_SIZE(mdev))));</div><div class='add'>+	entry = alloc_uar_entry(c, alloc_type);</div><div class='add'>+	if (IS_ERR(entry))</div><div class='add'>+		return PTR_ERR(entry);</div><div class='ctx'> </div><div class='del'>-	kfree(dev-&gt;dm.header_modify_sw_icm_alloc_blocks);</div><div class='add'>+	mmap_offset = mlx5_entry_to_mmap_offset(entry);</div><div class='add'>+	length = entry-&gt;rdma_entry.npages * PAGE_SIZE;</div><div class='add'>+	uobj-&gt;object = entry;</div><div class='add'>+	uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE);</div><div class='add'>+</div><div class='add'>+	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,</div><div class='add'>+			     &amp;mmap_offset, sizeof(mmap_offset));</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,</div><div class='add'>+			     &amp;entry-&gt;page_idx, sizeof(entry-&gt;page_idx));</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,</div><div class='add'>+			     &amp;length, sizeof(length));</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	MLX5_IB_METHOD_UAR_OBJ_ALLOC,</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE,</div><div class='add'>+			MLX5_IB_OBJECT_UAR,</div><div class='add'>+			UVERBS_ACCESS_NEW,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE,</div><div class='add'>+			     enum mlx5_ib_uapi_uar_alloc_type,</div><div class='add'>+			     UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u32),</div><div class='add'>+			   UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,</div><div class='add'>+			    UVERBS_ATTR_TYPE(u64),</div><div class='add'>+			    UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD_DESTROY(</div><div class='add'>+	MLX5_IB_METHOD_UAR_OBJ_DESTROY,</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE,</div><div class='add'>+			MLX5_IB_OBJECT_UAR,</div><div class='add'>+			UVERBS_ACCESS_DESTROY,</div><div class='add'>+			UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_UAR,</div><div class='add'>+			    UVERBS_TYPE_ALLOC_IDR(mmap_obj_cleanup),</div><div class='add'>+			    &amp;UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_ALLOC),</div><div class='add'>+			    &amp;UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_DESTROY));</div><div class='add'>+</div><div class='add'>+ADD_UVERBS_ATTRIBUTES_SIMPLE(</div><div class='add'>+	mlx5_ib_query_context,</div><div class='add'>+	UVERBS_OBJECT_DEVICE,</div><div class='add'>+	UVERBS_METHOD_QUERY_CONTEXT,</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(</div><div class='add'>+		MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,</div><div class='add'>+		UVERBS_ATTR_STRUCT(struct mlx5_ib_alloc_ucontext_resp,</div><div class='add'>+				   dump_fill_mkey),</div><div class='add'>+		UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+ADD_UVERBS_ATTRIBUTES_SIMPLE(</div><div class='add'>+	mlx5_ib_reg_dmabuf_mr,</div><div class='add'>+	UVERBS_OBJECT_MR,</div><div class='add'>+	UVERBS_METHOD_REG_DMABUF_MR,</div><div class='add'>+	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_REG_DMABUF_MR_ACCESS_FLAGS,</div><div class='add'>+			     enum mlx5_ib_uapi_reg_dmabuf_flags,</div><div class='add'>+			     UA_OPTIONAL));</div><div class='add'>+</div><div class='add'>+static const struct uapi_definition mlx5_ib_defs[] = {</div><div class='add'>+	UAPI_DEF_CHAIN(mlx5_ib_devx_defs),</div><div class='add'>+	UAPI_DEF_CHAIN(mlx5_ib_flow_defs),</div><div class='add'>+	UAPI_DEF_CHAIN(mlx5_ib_qos_defs),</div><div class='add'>+	UAPI_DEF_CHAIN(mlx5_ib_std_types_defs),</div><div class='add'>+	UAPI_DEF_CHAIN(mlx5_ib_dm_defs),</div><div class='add'>+	UAPI_DEF_CHAIN(mlx5_ib_create_cq_defs),</div><div class='add'>+</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &amp;mlx5_ib_query_context),</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_MR, &amp;mlx5_ib_reg_dmabuf_mr),</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR,</div><div class='add'>+				UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)),</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_UAR),</div><div class='add'>+	{}</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	mlx5_ib_data_direct_cleanup(dev);</div><div class='add'>+	mlx5_ib_cleanup_multiport_master(dev);</div><div class='add'>+	WARN_ON(!xa_empty(&amp;dev-&gt;odp_mkeys));</div><div class='add'>+	mutex_destroy(&amp;dev-&gt;cap_mask_mutex);</div><div class='add'>+	WARN_ON(!xa_empty(&amp;dev-&gt;sig_mrs));</div><div class='add'>+	WARN_ON(!bitmap_empty(dev-&gt;dm.memic_alloc_pages, MLX5_MAX_MEMIC_PAGES));</div><div class='add'>+	mlx5r_macsec_dealloc_gids(dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='del'>-	u64 header_modify_icm_blocks = 0;</div><div class='del'>-	u64 steering_icm_blocks = 0;</div><div class='del'>-	int err;</div><div class='del'>-	int i;</div><div class='add'>+	int err, i;</div><div class='add'>+</div><div class='add'>+	dev-&gt;ib_dev.node_type = RDMA_NODE_IB_CA;</div><div class='add'>+	dev-&gt;ib_dev.local_dma_lkey = 0 /* not supported for now */;</div><div class='add'>+	dev-&gt;ib_dev.dev.parent = mdev-&gt;device;</div><div class='add'>+	dev-&gt;ib_dev.lag_flags = RDMA_LAG_FLAGS_HASH_ALL_SLAVES;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; dev-&gt;num_ports; i++) {</div><div class='ctx'> 		spin_lock_init(&amp;dev-&gt;port[i].mp.mpi_lock);</div><div class='del'>-		rwlock_init(&amp;dev-&gt;port[i].roce.netdev_lock);</div><div class='ctx'> 		dev-&gt;port[i].roce.dev = dev;</div><div class='ctx'> 		dev-&gt;port[i].roce.native_port_num = i + 1;</div><div class='ctx'> 		dev-&gt;port[i].roce.last_port_state = IB_PORT_DOWN;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	err = mlx5_ib_init_multiport_master(dev);</div><div class='add'>+	err = mlx5r_cmd_query_special_mkeys(dev);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='del'>-	err = set_has_smi_cap(dev);</div><div class='add'>+	err = mlx5r_macsec_init_gids_and_devlist(dev);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='del'>-	if (!mlx5_core_mp_enabled(mdev)) {</div><div class='del'>-		for (i = 1; i &lt;= dev-&gt;num_ports; i++) {</div><div class='del'>-			err = get_port_caps(dev, i);</div><div class='del'>-			if (err)</div><div class='del'>-				break;</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		err = get_port_caps(dev, mlx5_core_native_port_num(mdev));</div><div class='del'>-	}</div><div class='add'>+	err = mlx5_ib_init_multiport_master(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	err = set_has_smi_cap(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_mp;</div><div class='add'>+</div><div class='add'>+	err = mlx5_query_max_pkeys(&amp;dev-&gt;ib_dev, &amp;dev-&gt;pkey_table_len);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_mp;</div><div class='ctx'> </div><div class='ctx'> 	if (mlx5_use_mad_ifc(dev))</div><div class='ctx'> 		get_ext_port_caps(dev);</div><div class='ctx'> </div><div class='del'>-	dev-&gt;ib_dev.owner		= THIS_MODULE;</div><div class='del'>-	dev-&gt;ib_dev.node_type		= RDMA_NODE_IB_CA;</div><div class='del'>-	dev-&gt;ib_dev.local_dma_lkey	= 0 /* not supported for now */;</div><div class='del'>-	dev-&gt;ib_dev.phys_port_cnt	= dev-&gt;num_ports;</div><div class='del'>-	dev-&gt;ib_dev.num_comp_vectors    = mlx5_comp_vectors_count(mdev);</div><div class='del'>-	dev-&gt;ib_dev.dev.parent		= mdev-&gt;device;</div><div class='add'>+	dev-&gt;ib_dev.num_comp_vectors    = mlx5_comp_vectors_max(mdev);</div><div class='ctx'> </div><div class='ctx'> 	mutex_init(&amp;dev-&gt;cap_mask_mutex);</div><div class='add'>+	mutex_init(&amp;dev-&gt;data_direct_lock);</div><div class='ctx'> 	INIT_LIST_HEAD(&amp;dev-&gt;qp_list);</div><div class='ctx'> 	spin_lock_init(&amp;dev-&gt;reset_flow_resource_lock);</div><div class='del'>-</div><div class='del'>-	if (MLX5_CAP_GEN_64(mdev, general_obj_types) &amp;</div><div class='del'>-	    MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM) {</div><div class='del'>-		if (MLX5_CAP64_DEV_MEM(mdev, steering_sw_icm_start_address)) {</div><div class='del'>-			steering_icm_blocks =</div><div class='del'>-				BIT(MLX5_CAP_DEV_MEM(mdev,</div><div class='del'>-						     log_steering_sw_icm_size) -</div><div class='del'>-				    MLX5_LOG_SW_ICM_BLOCK_SIZE(mdev));</div><div class='del'>-</div><div class='del'>-			dev-&gt;dm.steering_sw_icm_alloc_blocks =</div><div class='del'>-				kcalloc(BITS_TO_LONGS(steering_icm_blocks),</div><div class='del'>-					sizeof(unsigned long), GFP_KERNEL);</div><div class='del'>-			if (!dev-&gt;dm.steering_sw_icm_alloc_blocks)</div><div class='del'>-				goto err_mp;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (MLX5_CAP64_DEV_MEM(mdev,</div><div class='del'>-				       header_modify_sw_icm_start_address)) {</div><div class='del'>-			header_modify_icm_blocks = BIT(</div><div class='del'>-				MLX5_CAP_DEV_MEM(</div><div class='del'>-					mdev, log_header_modify_sw_icm_size) -</div><div class='del'>-				MLX5_LOG_SW_ICM_BLOCK_SIZE(mdev));</div><div class='del'>-</div><div class='del'>-			dev-&gt;dm.header_modify_sw_icm_alloc_blocks =</div><div class='del'>-				kcalloc(BITS_TO_LONGS(header_modify_icm_blocks),</div><div class='del'>-					sizeof(unsigned long), GFP_KERNEL);</div><div class='del'>-			if (!dev-&gt;dm.header_modify_sw_icm_alloc_blocks)</div><div class='del'>-				goto err_dm;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='add'>+	xa_init(&amp;dev-&gt;odp_mkeys);</div><div class='add'>+	xa_init(&amp;dev-&gt;sig_mrs);</div><div class='add'>+	atomic_set(&amp;dev-&gt;mkey_var, 0);</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_init(&amp;dev-&gt;dm.lock);</div><div class='ctx'> 	dev-&gt;dm.dev = mdev;</div><div class='del'>-</div><div class='del'>-	if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {</div><div class='del'>-		err = init_srcu_struct(&amp;dev-&gt;mr_srcu);</div><div class='del'>-		if (err)</div><div class='del'>-			goto err_dm;</div><div class='del'>-	}</div><div class='add'>+	err = mlx5_ib_data_direct_init(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_mp;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='del'>-</div><div class='del'>-err_dm:</div><div class='del'>-	kfree(dev-&gt;dm.steering_sw_icm_alloc_blocks);</div><div class='del'>-	kfree(dev-&gt;dm.header_modify_sw_icm_alloc_blocks);</div><div class='del'>-</div><div class='ctx'> err_mp:</div><div class='ctx'> 	mlx5_ib_cleanup_multiport_master(dev);</div><div class='del'>-</div><div class='del'>-	return -ENOMEM;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx5_ib_stage_flow_db_init(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	dev-&gt;flow_db = kzalloc(sizeof(*dev-&gt;flow_db), GFP_KERNEL);</div><div class='del'>-</div><div class='del'>-	if (!dev-&gt;flow_db)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	mutex_init(&amp;dev-&gt;flow_db-&gt;lock);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='add'>+err:</div><div class='add'>+	mlx5r_macsec_dealloc_gids(dev);</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void mlx5_ib_stage_flow_db_cleanup(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	kfree(dev-&gt;flow_db);</div><div class='del'>-}</div><div class='add'>+static struct ib_device *mlx5_ib_add_sub_dev(struct ib_device *parent,</div><div class='add'>+					     enum rdma_nl_dev_type type,</div><div class='add'>+					     const char *name);</div><div class='add'>+static void mlx5_ib_del_sub_dev(struct ib_device *sub_dev);</div><div class='ctx'> </div><div class='ctx'> static const struct ib_device_ops mlx5_ib_dev_ops = {</div><div class='add'>+	.owner = THIS_MODULE,</div><div class='add'>+	.driver_id = RDMA_DRIVER_MLX5,</div><div class='add'>+	.uverbs_abi_ver	= MLX5_IB_UVERBS_ABI_VERSION,</div><div class='add'>+</div><div class='ctx'> 	.add_gid = mlx5_ib_add_gid,</div><div class='add'>+	.add_sub_dev = mlx5_ib_add_sub_dev,</div><div class='ctx'> 	.alloc_mr = mlx5_ib_alloc_mr,</div><div class='add'>+	.alloc_mr_integrity = mlx5_ib_alloc_mr_integrity,</div><div class='ctx'> 	.alloc_pd = mlx5_ib_alloc_pd,</div><div class='ctx'> 	.alloc_ucontext = mlx5_ib_alloc_ucontext,</div><div class='ctx'> 	.attach_mcast = mlx5_ib_mcg_attach,</div><div class='ctx'> 	.check_mr_status = mlx5_ib_check_mr_status,</div><div class='ctx'> 	.create_ah = mlx5_ib_create_ah,</div><div class='del'>-	.create_counters = mlx5_ib_create_counters,</div><div class='ctx'> 	.create_cq = mlx5_ib_create_cq,</div><div class='del'>-	.create_flow = mlx5_ib_create_flow,</div><div class='ctx'> 	.create_qp = mlx5_ib_create_qp,</div><div class='ctx'> 	.create_srq = mlx5_ib_create_srq,</div><div class='add'>+	.create_user_ah = mlx5_ib_create_ah,</div><div class='ctx'> 	.dealloc_pd = mlx5_ib_dealloc_pd,</div><div class='ctx'> 	.dealloc_ucontext = mlx5_ib_dealloc_ucontext,</div><div class='ctx'> 	.del_gid = mlx5_ib_del_gid,</div><div class='add'>+	.del_sub_dev = mlx5_ib_del_sub_dev,</div><div class='ctx'> 	.dereg_mr = mlx5_ib_dereg_mr,</div><div class='ctx'> 	.destroy_ah = mlx5_ib_destroy_ah,</div><div class='del'>-	.destroy_counters = mlx5_ib_destroy_counters,</div><div class='ctx'> 	.destroy_cq = mlx5_ib_destroy_cq,</div><div class='del'>-	.destroy_flow = mlx5_ib_destroy_flow,</div><div class='del'>-	.destroy_flow_action = mlx5_ib_destroy_flow_action,</div><div class='ctx'> 	.destroy_qp = mlx5_ib_destroy_qp,</div><div class='ctx'> 	.destroy_srq = mlx5_ib_destroy_srq,</div><div class='ctx'> 	.detach_mcast = mlx5_ib_mcg_detach,</div><div class='ctx'> 	.disassociate_ucontext = mlx5_ib_disassociate_ucontext,</div><div class='ctx'> 	.drain_rq = mlx5_ib_drain_rq,</div><div class='ctx'> 	.drain_sq = mlx5_ib_drain_sq,</div><div class='add'>+	.device_group = &amp;mlx5_attr_group,</div><div class='ctx'> 	.get_dev_fw_str = get_dev_fw_str,</div><div class='ctx'> 	.get_dma_mr = mlx5_ib_get_dma_mr,</div><div class='ctx'> 	.get_link_layer = mlx5_ib_port_link_layer,</div><div class='ctx'> 	.map_mr_sg = mlx5_ib_map_mr_sg,</div><div class='add'>+	.map_mr_sg_pi = mlx5_ib_map_mr_sg_pi,</div><div class='ctx'> 	.mmap = mlx5_ib_mmap,</div><div class='add'>+	.mmap_free = mlx5_ib_mmap_free,</div><div class='ctx'> 	.modify_cq = mlx5_ib_modify_cq,</div><div class='ctx'> 	.modify_device = mlx5_ib_modify_device,</div><div class='ctx'> 	.modify_port = mlx5_ib_modify_port,</div><div class='ctx'> 	.modify_qp = mlx5_ib_modify_qp,</div><div class='ctx'> 	.modify_srq = mlx5_ib_modify_srq,</div><div class='add'>+	.pre_destroy_cq = mlx5_ib_pre_destroy_cq,</div><div class='ctx'> 	.poll_cq = mlx5_ib_poll_cq,</div><div class='del'>-	.post_recv = mlx5_ib_post_recv,</div><div class='del'>-	.post_send = mlx5_ib_post_send,</div><div class='add'>+	.post_destroy_cq = mlx5_ib_post_destroy_cq,</div><div class='add'>+	.post_recv = mlx5_ib_post_recv_nodrain,</div><div class='add'>+	.post_send = mlx5_ib_post_send_nodrain,</div><div class='ctx'> 	.post_srq_recv = mlx5_ib_post_srq_recv,</div><div class='ctx'> 	.process_mad = mlx5_ib_process_mad,</div><div class='ctx'> 	.query_ah = mlx5_ib_query_ah,</div><div class='hunk'>@@ -6172,29 +4299,31 @@ static const struct ib_device_ops mlx5_ib_dev_ops = {</div><div class='ctx'> 	.query_pkey = mlx5_ib_query_pkey,</div><div class='ctx'> 	.query_qp = mlx5_ib_query_qp,</div><div class='ctx'> 	.query_srq = mlx5_ib_query_srq,</div><div class='del'>-	.read_counters = mlx5_ib_read_counters,</div><div class='add'>+	.query_ucontext = mlx5_ib_query_ucontext,</div><div class='ctx'> 	.reg_user_mr = mlx5_ib_reg_user_mr,</div><div class='add'>+	.reg_user_mr_dmabuf = mlx5_ib_reg_user_mr_dmabuf,</div><div class='ctx'> 	.req_notify_cq = mlx5_ib_arm_cq,</div><div class='ctx'> 	.rereg_user_mr = mlx5_ib_rereg_user_mr,</div><div class='ctx'> 	.resize_cq = mlx5_ib_resize_cq,</div><div class='add'>+	.ufile_hw_cleanup = mlx5_ib_ufile_hw_cleanup,</div><div class='ctx'> </div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_dmah, mlx5_ib_dmah, ibdmah),</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd),</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_qp, mlx5_ib_qp, ibqp),</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq),</div><div class='ctx'> 	INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static const struct ib_device_ops mlx5_ib_dev_flow_ipsec_ops = {</div><div class='del'>-	.create_flow_action_esp = mlx5_ib_create_flow_action_esp,</div><div class='del'>-	.modify_flow_action_esp = mlx5_ib_modify_flow_action_esp,</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> static const struct ib_device_ops mlx5_ib_dev_ipoib_enhanced_ops = {</div><div class='ctx'> 	.rdma_netdev_get_params = mlx5_ib_rn_get_params,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static const struct ib_device_ops mlx5_ib_dev_sriov_ops = {</div><div class='ctx'> 	.get_vf_config = mlx5_ib_get_vf_config,</div><div class='add'>+	.get_vf_guid = mlx5_ib_get_vf_guid,</div><div class='ctx'> 	.get_vf_stats = mlx5_ib_get_vf_stats,</div><div class='ctx'> 	.set_vf_guid = mlx5_ib_set_vf_guid,</div><div class='ctx'> 	.set_vf_link_state = mlx5_ib_set_vf_link_state,</div><div class='hunk'>@@ -6203,61 +4332,91 @@ static const struct ib_device_ops mlx5_ib_dev_sriov_ops = {</div><div class='ctx'> static const struct ib_device_ops mlx5_ib_dev_mw_ops = {</div><div class='ctx'> 	.alloc_mw = mlx5_ib_alloc_mw,</div><div class='ctx'> 	.dealloc_mw = mlx5_ib_dealloc_mw,</div><div class='add'>+</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_mw, mlx5_ib_mw, ibmw),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static const struct ib_device_ops mlx5_ib_dev_xrc_ops = {</div><div class='ctx'> 	.alloc_xrcd = mlx5_ib_alloc_xrcd,</div><div class='ctx'> 	.dealloc_xrcd = mlx5_ib_dealloc_xrcd,</div><div class='del'>-};</div><div class='ctx'> </div><div class='del'>-static const struct ib_device_ops mlx5_ib_dev_dm_ops = {</div><div class='del'>-	.alloc_dm = mlx5_ib_alloc_dm,</div><div class='del'>-	.dealloc_dm = mlx5_ib_dealloc_dm,</div><div class='del'>-	.reg_dm_mr = mlx5_ib_reg_dm_mr,</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx5_ib_xrcd, ibxrcd),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+static int mlx5_ib_init_var_table(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='add'>+	struct mlx5_var_table *var_table = &amp;dev-&gt;var_table;</div><div class='add'>+	u8 log_doorbell_bar_size;</div><div class='add'>+	u8 log_doorbell_stride;</div><div class='add'>+	u64 bar_size;</div><div class='add'>+</div><div class='add'>+	log_doorbell_bar_size = MLX5_CAP_DEV_VDPA_EMULATION(mdev,</div><div class='add'>+					log_doorbell_bar_size);</div><div class='add'>+	log_doorbell_stride = MLX5_CAP_DEV_VDPA_EMULATION(mdev,</div><div class='add'>+					log_doorbell_stride);</div><div class='add'>+	var_table-&gt;hw_start_addr = dev-&gt;mdev-&gt;bar_addr +</div><div class='add'>+				MLX5_CAP64_DEV_VDPA_EMULATION(mdev,</div><div class='add'>+					doorbell_bar_offset);</div><div class='add'>+	bar_size = (1ULL &lt;&lt; log_doorbell_bar_size) * 4096;</div><div class='add'>+	var_table-&gt;stride_size = 1ULL &lt;&lt; log_doorbell_stride;</div><div class='add'>+	var_table-&gt;num_var_hw_entries = div_u64(bar_size,</div><div class='add'>+						var_table-&gt;stride_size);</div><div class='add'>+	mutex_init(&amp;var_table-&gt;bitmap_lock);</div><div class='add'>+	var_table-&gt;bitmap = bitmap_zalloc(var_table-&gt;num_var_hw_entries,</div><div class='add'>+					  GFP_KERNEL);</div><div class='add'>+	return (var_table-&gt;bitmap) ? 0 : -ENOMEM;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_cleanup_ucaps(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, uctx_cap) &amp; MLX5_UCTX_CAP_RDMA_CTRL)</div><div class='add'>+		ib_remove_ucap(RDMA_UCAP_MLX5_CTRL_LOCAL);</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, uctx_cap) &amp;</div><div class='add'>+	    MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA)</div><div class='add'>+		ib_remove_ucap(RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_init_ucaps(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, uctx_cap) &amp; MLX5_UCTX_CAP_RDMA_CTRL) {</div><div class='add'>+		ret = ib_create_ucap(RDMA_UCAP_MLX5_CTRL_LOCAL);</div><div class='add'>+		if (ret)</div><div class='add'>+			return ret;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, uctx_cap) &amp;</div><div class='add'>+	    MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA) {</div><div class='add'>+		ret = ib_create_ucap(RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto remove_local;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+remove_local:</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, uctx_cap) &amp; MLX5_UCTX_CAP_RDMA_CTRL)</div><div class='add'>+		ib_remove_ucap(RDMA_UCAP_MLX5_CTRL_LOCAL);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_stage_caps_cleanup(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	if (MLX5_CAP_GEN_2_64(dev-&gt;mdev, general_obj_types_127_64) &amp;</div><div class='add'>+	    MLX5_HCA_CAP_2_GENERAL_OBJECT_TYPES_RDMA_CTRL)</div><div class='add'>+		mlx5_ib_cleanup_ucaps(dev);</div><div class='add'>+</div><div class='add'>+	bitmap_free(dev-&gt;var_table.bitmap);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	dev-&gt;ib_dev.uverbs_abi_ver	= MLX5_IB_UVERBS_ABI_VERSION;</div><div class='del'>-	dev-&gt;ib_dev.uverbs_cmd_mask	=</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_GET_CONTEXT)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_DEVICE)	|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_PORT)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_ALLOC_PD)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DEALLOC_PD)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_AH)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_AH)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_REG_MR)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_REREG_MR)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DEREG_MR)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL)	|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_CQ)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_RESIZE_CQ)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_CQ)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_QP)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_MODIFY_QP)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_QP)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_QP)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_ATTACH_MCAST)	|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DETACH_MCAST)	|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_SRQ)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_MODIFY_SRQ)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_QUERY_SRQ)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_DESTROY_SRQ)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_CREATE_XSRQ)		|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_CMD_OPEN_QP);</div><div class='del'>-	dev-&gt;ib_dev.uverbs_ex_cmd_mask =</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_QUERY_DEVICE)	|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_CREATE_CQ)	|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_CREATE_QP)	|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_MODIFY_QP)	|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_MODIFY_CQ)	|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_CREATE_FLOW)	|</div><div class='del'>-		(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_DESTROY_FLOW);</div><div class='del'>-</div><div class='ctx'> 	if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &amp;&amp;</div><div class='ctx'> 	    IS_ENABLED(CONFIG_MLX5_CORE_IPOIB))</div><div class='ctx'> 		ib_set_device_ops(&amp;dev-&gt;ib_dev,</div><div class='hunk'>@@ -6268,29 +4427,20 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> </div><div class='ctx'> 	dev-&gt;umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_GEN(mdev, imaicl)) {</div><div class='del'>-		dev-&gt;ib_dev.uverbs_cmd_mask |=</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_ALLOC_MW)	|</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_DEALLOC_MW);</div><div class='add'>+	if (MLX5_CAP_GEN(mdev, imaicl))</div><div class='ctx'> 		ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;mlx5_ib_dev_mw_ops);</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_GEN(mdev, xrc)) {</div><div class='del'>-		dev-&gt;ib_dev.uverbs_cmd_mask |=</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_OPEN_XRCD) |</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_CMD_CLOSE_XRCD);</div><div class='add'>+	if (MLX5_CAP_GEN(mdev, xrc))</div><div class='ctx'> 		ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;mlx5_ib_dev_xrc_ops);</div><div class='del'>-	}</div><div class='ctx'> </div><div class='ctx'> 	if (MLX5_CAP_DEV_MEM(mdev, memic) ||</div><div class='ctx'> 	    MLX5_CAP_GEN_64(dev-&gt;mdev, general_obj_types) &amp;</div><div class='ctx'> 	    MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)</div><div class='ctx'> 		ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;mlx5_ib_dev_dm_ops);</div><div class='ctx'> </div><div class='del'>-	if (mlx5_accel_ipsec_device_caps(dev-&gt;mdev) &amp;</div><div class='del'>-	    MLX5_ACCEL_IPSEC_CAP_DEVICE)</div><div class='del'>-		ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;mlx5_ib_dev_flow_ipsec_ops);</div><div class='del'>-	dev-&gt;ib_dev.driver_id = RDMA_DRIVER_MLX5;</div><div class='add'>+	if (mdev-&gt;st)</div><div class='add'>+		ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;mlx5_ib_dev_dmah_ops);</div><div class='add'>+</div><div class='ctx'> 	ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;mlx5_ib_dev_ops);</div><div class='ctx'> </div><div class='ctx'> 	if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))</div><div class='hunk'>@@ -6305,6 +4455,22 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> 	     MLX5_CAP_GEN(dev-&gt;mdev, disable_local_lb_mc)))</div><div class='ctx'> 		mutex_init(&amp;dev-&gt;lb.mutex);</div><div class='ctx'> </div><div class='add'>+	if (MLX5_CAP_GEN_64(dev-&gt;mdev, general_obj_types) &amp;</div><div class='add'>+			MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) {</div><div class='add'>+		err = mlx5_ib_init_var_table(dev);</div><div class='add'>+		if (err)</div><div class='add'>+			return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_GEN_2_64(dev-&gt;mdev, general_obj_types_127_64) &amp;</div><div class='add'>+	    MLX5_HCA_CAP_2_GENERAL_OBJECT_TYPES_RDMA_CTRL) {</div><div class='add'>+		err = mlx5_ib_init_ucaps(dev);</div><div class='add'>+		if (err)</div><div class='add'>+			return err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	dev-&gt;ib_dev.use_cq_dim = true;</div><div class='add'>+</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -6322,9 +4488,10 @@ static int mlx5_ib_stage_non_default_cb(struct mlx5_ib_dev *dev)</div><div class='ctx'> static const struct ib_device_ops mlx5_ib_dev_port_rep_ops = {</div><div class='ctx'> 	.get_port_immutable = mlx5_port_rep_immutable,</div><div class='ctx'> 	.query_port = mlx5_ib_rep_query_port,</div><div class='add'>+	.query_pkey = mlx5_ib_rep_query_pkey,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static int mlx5_ib_stage_rep_non_default_cb(struct mlx5_ib_dev *dev)</div><div class='add'>+static int mlx5_ib_stage_raw_eth_non_default_cb(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;mlx5_ib_dev_port_rep_ops);</div><div class='ctx'> 	return 0;</div><div class='hunk'>@@ -6335,70 +4502,30 @@ static const struct ib_device_ops mlx5_ib_dev_common_roce_ops = {</div><div class='ctx'> 	.create_wq = mlx5_ib_create_wq,</div><div class='ctx'> 	.destroy_rwq_ind_table = mlx5_ib_destroy_rwq_ind_table,</div><div class='ctx'> 	.destroy_wq = mlx5_ib_destroy_wq,</div><div class='del'>-	.get_netdev = mlx5_ib_get_netdev,</div><div class='ctx'> 	.modify_wq = mlx5_ib_modify_wq,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static int mlx5_ib_stage_common_roce_init(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	u8 port_num;</div><div class='del'>-</div><div class='del'>-	dev-&gt;ib_dev.uverbs_ex_cmd_mask |=</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_CREATE_WQ) |</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_MODIFY_WQ) |</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_DESTROY_WQ) |</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) |</div><div class='del'>-			(1ull &lt;&lt; IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL);</div><div class='del'>-	ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;mlx5_ib_dev_common_roce_ops);</div><div class='del'>-</div><div class='del'>-	port_num = mlx5_core_native_port_num(dev-&gt;mdev) - 1;</div><div class='del'>-</div><div class='del'>-	/* Register only for native ports */</div><div class='del'>-	return mlx5_add_netdev_notifier(dev, port_num);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void mlx5_ib_stage_common_roce_cleanup(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	u8 port_num = mlx5_core_native_port_num(dev-&gt;mdev) - 1;</div><div class='del'>-</div><div class='del'>-	mlx5_remove_netdev_notifier(dev, port_num);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx5_ib_stage_rep_roce_init(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='del'>-	enum rdma_link_layer ll;</div><div class='del'>-	int port_type_cap;</div><div class='del'>-	int err = 0;</div><div class='del'>-</div><div class='del'>-	port_type_cap = MLX5_CAP_GEN(mdev, port_type);</div><div class='del'>-	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);</div><div class='ctx'> </div><div class='del'>-	if (ll == IB_LINK_LAYER_ETHERNET)</div><div class='del'>-		err = mlx5_ib_stage_common_roce_init(dev);</div><div class='del'>-</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void mlx5_ib_stage_rep_roce_cleanup(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	mlx5_ib_stage_common_roce_cleanup(dev);</div><div class='del'>-}</div><div class='add'>+	INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mlx5_ib_rwq_ind_table,</div><div class='add'>+			   ib_rwq_ind_tbl),</div><div class='add'>+};</div><div class='ctx'> </div><div class='del'>-static int mlx5_ib_stage_roce_init(struct mlx5_ib_dev *dev)</div><div class='add'>+static int mlx5_ib_roce_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='ctx'> 	enum rdma_link_layer ll;</div><div class='ctx'> 	int port_type_cap;</div><div class='add'>+	u32 port_num = 0;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	port_type_cap = MLX5_CAP_GEN(mdev, port_type);</div><div class='ctx'> 	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);</div><div class='ctx'> </div><div class='ctx'> 	if (ll == IB_LINK_LAYER_ETHERNET) {</div><div class='del'>-		err = mlx5_ib_stage_common_roce_init(dev);</div><div class='del'>-		if (err)</div><div class='del'>-			return err;</div><div class='add'>+		ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;mlx5_ib_dev_common_roce_ops);</div><div class='add'>+</div><div class='add'>+		port_num = mlx5_core_native_port_num(dev-&gt;mdev) - 1;</div><div class='add'>+</div><div class='add'>+		/* Register only for native ports */</div><div class='add'>+		mlx5_mdev_netdev_track(dev, port_num);</div><div class='ctx'> </div><div class='ctx'> 		err = mlx5_enable_eth(dev);</div><div class='ctx'> 		if (err)</div><div class='hunk'>@@ -6407,68 +4534,26 @@ static int mlx5_ib_stage_roce_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> cleanup:</div><div class='del'>-	mlx5_ib_stage_common_roce_cleanup(dev);</div><div class='del'>-</div><div class='add'>+	mlx5_mdev_netdev_untrack(dev, port_num);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void mlx5_ib_stage_roce_cleanup(struct mlx5_ib_dev *dev)</div><div class='add'>+static void mlx5_ib_roce_cleanup(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='ctx'> 	enum rdma_link_layer ll;</div><div class='ctx'> 	int port_type_cap;</div><div class='add'>+	u32 port_num;</div><div class='ctx'> </div><div class='ctx'> 	port_type_cap = MLX5_CAP_GEN(mdev, port_type);</div><div class='ctx'> 	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);</div><div class='ctx'> </div><div class='ctx'> 	if (ll == IB_LINK_LAYER_ETHERNET) {</div><div class='ctx'> 		mlx5_disable_eth(dev);</div><div class='del'>-		mlx5_ib_stage_common_roce_cleanup(dev);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx5_ib_stage_dev_res_init(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	return create_dev_resources(&amp;dev-&gt;devr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void mlx5_ib_stage_dev_res_cleanup(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	destroy_dev_resources(&amp;dev-&gt;devr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx5_ib_stage_odp_init(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	mlx5_ib_internal_fill_odp_caps(dev);</div><div class='del'>-</div><div class='del'>-	return mlx5_ib_odp_init_one(dev);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void mlx5_ib_stage_odp_cleanup(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	mlx5_ib_odp_cleanup_one(dev);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static const struct ib_device_ops mlx5_ib_dev_hw_stats_ops = {</div><div class='del'>-	.alloc_hw_stats = mlx5_ib_alloc_hw_stats,</div><div class='del'>-	.get_hw_stats = mlx5_ib_get_hw_stats,</div><div class='del'>-};</div><div class='ctx'> </div><div class='del'>-static int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	if (MLX5_CAP_GEN(dev-&gt;mdev, max_qp_cnt)) {</div><div class='del'>-		ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;mlx5_ib_dev_hw_stats_ops);</div><div class='del'>-</div><div class='del'>-		return mlx5_ib_alloc_counters(dev);</div><div class='add'>+		port_num = mlx5_core_native_port_num(dev-&gt;mdev) - 1;</div><div class='add'>+		mlx5_mdev_netdev_untrack(dev, port_num);</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void mlx5_ib_stage_counters_cleanup(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	if (MLX5_CAP_GEN(dev-&gt;mdev, max_qp_cnt))</div><div class='del'>-		mlx5_ib_dealloc_counters(dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev *dev)</div><div class='hunk'>@@ -6484,17 +4569,6 @@ static void mlx5_ib_stage_cong_debugfs_cleanup(struct mlx5_ib_dev *dev)</div><div class='ctx'> 				     mlx5_core_native_port_num(dev-&gt;mdev) - 1);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_ib_stage_uar_init(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	dev-&gt;mdev-&gt;priv.uar = mlx5_get_uars_page(dev-&gt;mdev);</div><div class='del'>-	return PTR_ERR_OR_ZERO(dev-&gt;mdev-&gt;priv.uar);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void mlx5_ib_stage_uar_cleanup(struct mlx5_ib_dev *dev)</div><div class='del'>-{</div><div class='del'>-	mlx5_put_uars_page(dev-&gt;mdev, dev-&gt;mdev-&gt;priv.uar);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	int err;</div><div class='hunk'>@@ -6505,7 +4579,7 @@ static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> </div><div class='ctx'> 	err = mlx5_alloc_bfreg(dev-&gt;mdev, &amp;dev-&gt;fp_bfreg, false, true);</div><div class='ctx'> 	if (err)</div><div class='del'>-		mlx5_free_bfreg(dev-&gt;mdev, &amp;dev-&gt;fp_bfreg);</div><div class='add'>+		mlx5_free_bfreg(dev-&gt;mdev, &amp;dev-&gt;bfreg);</div><div class='ctx'> </div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='hunk'>@@ -6520,17 +4594,21 @@ static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	const char *name;</div><div class='ctx'> </div><div class='del'>-	rdma_set_device_sysfs_group(&amp;dev-&gt;ib_dev, &amp;mlx5_attr_group);</div><div class='del'>-	if (!mlx5_lag_is_roce(dev-&gt;mdev))</div><div class='add'>+	if (dev-&gt;sub_dev_name) {</div><div class='add'>+		name = dev-&gt;sub_dev_name;</div><div class='add'>+		ib_mark_name_assigned_by_user(&amp;dev-&gt;ib_dev);</div><div class='add'>+	} else if (!mlx5_lag_is_active(dev-&gt;mdev))</div><div class='ctx'> 		name = "mlx5_%d";</div><div class='ctx'> 	else</div><div class='ctx'> 		name = "mlx5_bond_%d";</div><div class='del'>-	return ib_register_device(&amp;dev-&gt;ib_dev, name);</div><div class='add'>+	return ib_register_device(&amp;dev-&gt;ib_dev, name, &amp;dev-&gt;mdev-&gt;pdev-&gt;dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	destroy_umrc_res(dev);</div><div class='add'>+	mlx5_mkey_cache_cleanup(dev);</div><div class='add'>+	mlx5r_umr_resource_cleanup(dev);</div><div class='add'>+	mlx5r_umr_cleanup(dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)</div><div class='hunk'>@@ -6540,53 +4618,112 @@ static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)</div><div class='ctx'> </div><div class='ctx'> static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	return create_umr_res(dev);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = mlx5r_umr_init(dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	ret = mlx5_mkey_cache_init(dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	init_delay_drop(dev);</div><div class='add'>+	struct dentry *root;</div><div class='add'>+</div><div class='add'>+	if (!(dev-&gt;ib_dev.attrs.raw_packet_caps &amp; IB_RAW_PACKET_CAP_DELAY_DROP))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	mutex_init(&amp;dev-&gt;delay_drop.lock);</div><div class='add'>+	dev-&gt;delay_drop.dev = dev;</div><div class='add'>+	dev-&gt;delay_drop.activate = false;</div><div class='add'>+	dev-&gt;delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000;</div><div class='add'>+	INIT_WORK(&amp;dev-&gt;delay_drop.delay_drop_work, delay_drop_handler);</div><div class='add'>+	atomic_set(&amp;dev-&gt;delay_drop.rqs_cnt, 0);</div><div class='add'>+	atomic_set(&amp;dev-&gt;delay_drop.events_cnt, 0);</div><div class='add'>+</div><div class='add'>+	if (!mlx5_debugfs_root)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	root = debugfs_create_dir("delay_drop", mlx5_debugfs_get_dev_root(dev-&gt;mdev));</div><div class='add'>+	dev-&gt;delay_drop.dir_debugfs = root;</div><div class='ctx'> </div><div class='add'>+	debugfs_create_atomic_t("num_timeout_events", 0400, root,</div><div class='add'>+				&amp;dev-&gt;delay_drop.events_cnt);</div><div class='add'>+	debugfs_create_atomic_t("num_rqs", 0400, root,</div><div class='add'>+				&amp;dev-&gt;delay_drop.rqs_cnt);</div><div class='add'>+	debugfs_create_file("timeout", 0600, root, &amp;dev-&gt;delay_drop,</div><div class='add'>+			    &amp;fops_delay_drop_timeout);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	cancel_delay_drop(dev);</div><div class='add'>+	if (!(dev-&gt;ib_dev.attrs.raw_packet_caps &amp; IB_RAW_PACKET_CAP_DELAY_DROP))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	cancel_work_sync(&amp;dev-&gt;delay_drop.delay_drop_work);</div><div class='add'>+	if (!dev-&gt;delay_drop.dir_debugfs)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	debugfs_remove_recursive(dev-&gt;delay_drop.dir_debugfs);</div><div class='add'>+	dev-&gt;delay_drop.dir_debugfs = NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int mlx5_ib_stage_dev_notifier_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='add'>+	struct mlx5_ib_resources *devr = &amp;dev-&gt;devr;</div><div class='add'>+	int port;</div><div class='add'>+</div><div class='add'>+	for (port = 0; port &lt; ARRAY_SIZE(devr-&gt;ports); ++port)</div><div class='add'>+		INIT_WORK(&amp;devr-&gt;ports[port].pkey_change_work,</div><div class='add'>+			  pkey_change_handler);</div><div class='add'>+</div><div class='ctx'> 	dev-&gt;mdev_events.notifier_call = mlx5_ib_event;</div><div class='ctx'> 	mlx5_notifier_register(dev-&gt;mdev, &amp;dev-&gt;mdev_events);</div><div class='add'>+</div><div class='add'>+	mlx5r_macsec_event_register(dev);</div><div class='add'>+</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='add'>+	struct mlx5_ib_resources *devr = &amp;dev-&gt;devr;</div><div class='add'>+	int port;</div><div class='add'>+</div><div class='add'>+	mlx5r_macsec_event_unregister(dev);</div><div class='ctx'> 	mlx5_notifier_unregister(dev-&gt;mdev, &amp;dev-&gt;mdev_events);</div><div class='add'>+</div><div class='add'>+	for (port = 0; port &lt; ARRAY_SIZE(devr-&gt;ports); ++port)</div><div class='add'>+		cancel_work_sync(&amp;devr-&gt;ports[port].pkey_change_work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_ib_stage_devx_init(struct mlx5_ib_dev *dev)</div><div class='add'>+void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev,</div><div class='add'>+			      struct mlx5_data_direct_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	int uid;</div><div class='del'>-</div><div class='del'>-	uid = mlx5_ib_devx_create(dev, false);</div><div class='del'>-	if (uid &gt; 0)</div><div class='del'>-		dev-&gt;devx_whitelist_uid = uid;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='add'>+	mutex_lock(&amp;ibdev-&gt;data_direct_lock);</div><div class='add'>+	ibdev-&gt;data_direct_dev = dev;</div><div class='add'>+	mutex_unlock(&amp;ibdev-&gt;data_direct_lock);</div><div class='ctx'> }</div><div class='del'>-static void mlx5_ib_stage_devx_cleanup(struct mlx5_ib_dev *dev)</div><div class='add'>+</div><div class='add'>+void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev)</div><div class='ctx'> {</div><div class='del'>-	if (dev-&gt;devx_whitelist_uid)</div><div class='del'>-		mlx5_ib_devx_destroy(dev, dev-&gt;devx_whitelist_uid);</div><div class='add'>+	mutex_lock(&amp;ibdev-&gt;data_direct_lock);</div><div class='add'>+	mlx5_ib_revoke_data_direct_mrs(ibdev);</div><div class='add'>+	ibdev-&gt;data_direct_dev = NULL;</div><div class='add'>+	mutex_unlock(&amp;ibdev-&gt;data_direct_lock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void __mlx5_ib_remove(struct mlx5_ib_dev *dev,</div><div class='ctx'> 		      const struct mlx5_ib_profile *profile,</div><div class='ctx'> 		      int stage)</div><div class='ctx'> {</div><div class='add'>+	dev-&gt;ib_active = false;</div><div class='add'>+</div><div class='ctx'> 	/* Number of stages to cleanup */</div><div class='ctx'> 	while (stage) {</div><div class='ctx'> 		stage--;</div><div class='hunk'>@@ -6598,12 +4735,14 @@ void __mlx5_ib_remove(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	ib_dealloc_device(&amp;dev-&gt;ib_dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void *__mlx5_ib_add(struct mlx5_ib_dev *dev,</div><div class='del'>-		    const struct mlx5_ib_profile *profile)</div><div class='add'>+int __mlx5_ib_add(struct mlx5_ib_dev *dev,</div><div class='add'>+		  const struct mlx5_ib_profile *profile)</div><div class='ctx'> {</div><div class='ctx'> 	int err;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='add'>+	dev-&gt;profile = profile;</div><div class='add'>+</div><div class='ctx'> 	for (i = 0; i &lt; MLX5_IB_STAGE_MAX; i++) {</div><div class='ctx'> 		if (profile-&gt;stage[i].init) {</div><div class='ctx'> 			err = profile-&gt;stage[i].init(dev);</div><div class='hunk'>@@ -6612,54 +4751,53 @@ void *__mlx5_ib_add(struct mlx5_ib_dev *dev,</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	dev-&gt;profile = profile;</div><div class='ctx'> 	dev-&gt;ib_active = true;</div><div class='del'>-</div><div class='del'>-	return dev;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='ctx'> err_out:</div><div class='del'>-	__mlx5_ib_remove(dev, profile, i);</div><div class='del'>-</div><div class='del'>-	return NULL;</div><div class='add'>+	/* Clean up stages which were initialized */</div><div class='add'>+	while (i) {</div><div class='add'>+		i--;</div><div class='add'>+		if (profile-&gt;stage[i].cleanup)</div><div class='add'>+			profile-&gt;stage[i].cleanup(dev);</div><div class='add'>+	}</div><div class='add'>+	return -ENOMEM;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static const struct mlx5_ib_profile pf_profile = {</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_INIT,</div><div class='ctx'> 		     mlx5_ib_stage_init_init,</div><div class='ctx'> 		     mlx5_ib_stage_init_cleanup),</div><div class='del'>-	STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB,</div><div class='del'>-		     mlx5_ib_stage_flow_db_init,</div><div class='del'>-		     mlx5_ib_stage_flow_db_cleanup),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_FS,</div><div class='add'>+		     mlx5_ib_fs_init,</div><div class='add'>+		     mlx5_ib_fs_cleanup),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_CAPS,</div><div class='ctx'> 		     mlx5_ib_stage_caps_init,</div><div class='del'>-		     NULL),</div><div class='add'>+		     mlx5_ib_stage_caps_cleanup),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,</div><div class='ctx'> 		     mlx5_ib_stage_non_default_cb,</div><div class='ctx'> 		     NULL),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_ROCE,</div><div class='del'>-		     mlx5_ib_stage_roce_init,</div><div class='del'>-		     mlx5_ib_stage_roce_cleanup),</div><div class='add'>+		     mlx5_ib_roce_init,</div><div class='add'>+		     mlx5_ib_roce_cleanup),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_QP,</div><div class='add'>+		     mlx5_init_qp_table,</div><div class='add'>+		     mlx5_cleanup_qp_table),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_SRQ,</div><div class='ctx'> 		     mlx5_init_srq_table,</div><div class='ctx'> 		     mlx5_cleanup_srq_table),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,</div><div class='del'>-		     mlx5_ib_stage_dev_res_init,</div><div class='del'>-		     mlx5_ib_stage_dev_res_cleanup),</div><div class='del'>-	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,</div><div class='del'>-		     mlx5_ib_stage_dev_notifier_init,</div><div class='del'>-		     mlx5_ib_stage_dev_notifier_cleanup),</div><div class='add'>+		     mlx5_ib_dev_res_init,</div><div class='add'>+		     mlx5_ib_dev_res_cleanup),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_ODP,</div><div class='del'>-		     mlx5_ib_stage_odp_init,</div><div class='del'>-		     mlx5_ib_stage_odp_cleanup),</div><div class='add'>+		     mlx5_ib_odp_init_one,</div><div class='add'>+		     mlx5_ib_odp_cleanup_one),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,</div><div class='del'>-		     mlx5_ib_stage_counters_init,</div><div class='del'>-		     mlx5_ib_stage_counters_cleanup),</div><div class='add'>+		     mlx5_ib_counters_init,</div><div class='add'>+		     mlx5_ib_counters_cleanup),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,</div><div class='ctx'> 		     mlx5_ib_stage_cong_debugfs_init,</div><div class='ctx'> 		     mlx5_ib_stage_cong_debugfs_cleanup),</div><div class='del'>-	STAGE_CREATE(MLX5_IB_STAGE_UAR,</div><div class='del'>-		     mlx5_ib_stage_uar_init,</div><div class='del'>-		     mlx5_ib_stage_uar_cleanup),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_BFREG,</div><div class='ctx'> 		     mlx5_ib_stage_bfrag_init,</div><div class='ctx'> 		     mlx5_ib_stage_bfrag_cleanup),</div><div class='hunk'>@@ -6667,50 +4805,56 @@ static const struct mlx5_ib_profile pf_profile = {</div><div class='ctx'> 		     NULL,</div><div class='ctx'> 		     mlx5_ib_stage_pre_ib_reg_umr_cleanup),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,</div><div class='del'>-		     mlx5_ib_stage_devx_init,</div><div class='del'>-		     mlx5_ib_stage_devx_cleanup),</div><div class='add'>+		     mlx5_ib_devx_init,</div><div class='add'>+		     mlx5_ib_devx_cleanup),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,</div><div class='ctx'> 		     mlx5_ib_stage_ib_reg_init,</div><div class='ctx'> 		     mlx5_ib_stage_ib_reg_cleanup),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,</div><div class='add'>+		     mlx5_ib_stage_dev_notifier_init,</div><div class='add'>+		     mlx5_ib_stage_dev_notifier_cleanup),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,</div><div class='ctx'> 		     mlx5_ib_stage_post_ib_reg_umr_init,</div><div class='ctx'> 		     NULL),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,</div><div class='ctx'> 		     mlx5_ib_stage_delay_drop_init,</div><div class='ctx'> 		     mlx5_ib_stage_delay_drop_cleanup),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,</div><div class='add'>+		     mlx5_ib_restrack_init,</div><div class='add'>+		     NULL),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-const struct mlx5_ib_profile uplink_rep_profile = {</div><div class='add'>+const struct mlx5_ib_profile raw_eth_profile = {</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_INIT,</div><div class='ctx'> 		     mlx5_ib_stage_init_init,</div><div class='ctx'> 		     mlx5_ib_stage_init_cleanup),</div><div class='del'>-	STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB,</div><div class='del'>-		     mlx5_ib_stage_flow_db_init,</div><div class='del'>-		     mlx5_ib_stage_flow_db_cleanup),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_FS,</div><div class='add'>+		     mlx5_ib_fs_init,</div><div class='add'>+		     mlx5_ib_fs_cleanup),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_CAPS,</div><div class='ctx'> 		     mlx5_ib_stage_caps_init,</div><div class='del'>-		     NULL),</div><div class='add'>+		     mlx5_ib_stage_caps_cleanup),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,</div><div class='del'>-		     mlx5_ib_stage_rep_non_default_cb,</div><div class='add'>+		     mlx5_ib_stage_raw_eth_non_default_cb,</div><div class='ctx'> 		     NULL),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_ROCE,</div><div class='del'>-		     mlx5_ib_stage_rep_roce_init,</div><div class='del'>-		     mlx5_ib_stage_rep_roce_cleanup),</div><div class='add'>+		     mlx5_ib_roce_init,</div><div class='add'>+		     mlx5_ib_roce_cleanup),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_QP,</div><div class='add'>+		     mlx5_init_qp_table,</div><div class='add'>+		     mlx5_cleanup_qp_table),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_SRQ,</div><div class='ctx'> 		     mlx5_init_srq_table,</div><div class='ctx'> 		     mlx5_cleanup_srq_table),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,</div><div class='del'>-		     mlx5_ib_stage_dev_res_init,</div><div class='del'>-		     mlx5_ib_stage_dev_res_cleanup),</div><div class='del'>-	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,</div><div class='del'>-		     mlx5_ib_stage_dev_notifier_init,</div><div class='del'>-		     mlx5_ib_stage_dev_notifier_cleanup),</div><div class='add'>+		     mlx5_ib_dev_res_init,</div><div class='add'>+		     mlx5_ib_dev_res_cleanup),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,</div><div class='del'>-		     mlx5_ib_stage_counters_init,</div><div class='del'>-		     mlx5_ib_stage_counters_cleanup),</div><div class='del'>-	STAGE_CREATE(MLX5_IB_STAGE_UAR,</div><div class='del'>-		     mlx5_ib_stage_uar_init,</div><div class='del'>-		     mlx5_ib_stage_uar_cleanup),</div><div class='add'>+		     mlx5_ib_counters_init,</div><div class='add'>+		     mlx5_ib_counters_cleanup),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,</div><div class='add'>+		     mlx5_ib_stage_cong_debugfs_init,</div><div class='add'>+		     mlx5_ib_stage_cong_debugfs_cleanup),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_BFREG,</div><div class='ctx'> 		     mlx5_ib_stage_bfrag_init,</div><div class='ctx'> 		     mlx5_ib_stage_bfrag_cleanup),</div><div class='hunk'>@@ -6718,18 +4862,115 @@ const struct mlx5_ib_profile uplink_rep_profile = {</div><div class='ctx'> 		     NULL,</div><div class='ctx'> 		     mlx5_ib_stage_pre_ib_reg_umr_cleanup),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,</div><div class='del'>-		     mlx5_ib_stage_devx_init,</div><div class='del'>-		     mlx5_ib_stage_devx_cleanup),</div><div class='add'>+		     mlx5_ib_devx_init,</div><div class='add'>+		     mlx5_ib_devx_cleanup),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,</div><div class='ctx'> 		     mlx5_ib_stage_ib_reg_init,</div><div class='ctx'> 		     mlx5_ib_stage_ib_reg_cleanup),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,</div><div class='add'>+		     mlx5_ib_stage_dev_notifier_init,</div><div class='add'>+		     mlx5_ib_stage_dev_notifier_cleanup),</div><div class='ctx'> 	STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,</div><div class='ctx'> 		     mlx5_ib_stage_post_ib_reg_umr_init,</div><div class='ctx'> 		     NULL),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,</div><div class='add'>+		     mlx5_ib_stage_delay_drop_init,</div><div class='add'>+		     mlx5_ib_stage_delay_drop_cleanup),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,</div><div class='add'>+		     mlx5_ib_restrack_init,</div><div class='add'>+		     NULL),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev)</div><div class='add'>+static const struct mlx5_ib_profile plane_profile = {</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_INIT,</div><div class='add'>+		     mlx5_ib_stage_init_init,</div><div class='add'>+		     mlx5_ib_stage_init_cleanup),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_CAPS,</div><div class='add'>+		     mlx5_ib_stage_caps_init,</div><div class='add'>+		     mlx5_ib_stage_caps_cleanup),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,</div><div class='add'>+		     mlx5_ib_stage_non_default_cb,</div><div class='add'>+		     NULL),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_QP,</div><div class='add'>+		     mlx5_init_qp_table,</div><div class='add'>+		     mlx5_cleanup_qp_table),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_SRQ,</div><div class='add'>+		     mlx5_init_srq_table,</div><div class='add'>+		     mlx5_cleanup_srq_table),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,</div><div class='add'>+		     mlx5_ib_dev_res_init,</div><div class='add'>+		     mlx5_ib_dev_res_cleanup),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_BFREG,</div><div class='add'>+		     mlx5_ib_stage_bfrag_init,</div><div class='add'>+		     mlx5_ib_stage_bfrag_cleanup),</div><div class='add'>+	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,</div><div class='add'>+		     mlx5_ib_stage_ib_reg_init,</div><div class='add'>+		     mlx5_ib_stage_ib_reg_cleanup),</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static struct ib_device *mlx5_ib_add_sub_dev(struct ib_device *parent,</div><div class='add'>+					     enum rdma_nl_dev_type type,</div><div class='add'>+					     const char *name)</div><div class='ctx'> {</div><div class='add'>+	struct mlx5_ib_dev *mparent = to_mdev(parent), *mplane;</div><div class='add'>+	enum rdma_link_layer ll;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (mparent-&gt;smi_dev)</div><div class='add'>+		return ERR_PTR(-EEXIST);</div><div class='add'>+</div><div class='add'>+	ll = mlx5_port_type_cap_to_rdma_ll(MLX5_CAP_GEN(mparent-&gt;mdev,</div><div class='add'>+							port_type));</div><div class='add'>+	if (type != RDMA_DEVICE_TYPE_SMI || !mparent-&gt;num_plane ||</div><div class='add'>+	    ll != IB_LINK_LAYER_INFINIBAND ||</div><div class='add'>+	    !MLX5_CAP_GEN_2(mparent-&gt;mdev, multiplane_qp_ud))</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	mplane = ib_alloc_device_with_net(mlx5_ib_dev, ib_dev,</div><div class='add'>+					  mlx5_core_net(mparent-&gt;mdev));</div><div class='add'>+	if (!mplane)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	mplane-&gt;port = kcalloc(mparent-&gt;num_plane * mparent-&gt;num_ports,</div><div class='add'>+			       sizeof(*mplane-&gt;port), GFP_KERNEL);</div><div class='add'>+	if (!mplane-&gt;port) {</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto fail_kcalloc;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mplane-&gt;ib_dev.type = type;</div><div class='add'>+	mplane-&gt;mdev = mparent-&gt;mdev;</div><div class='add'>+	mplane-&gt;num_ports = mparent-&gt;num_plane;</div><div class='add'>+	mplane-&gt;sub_dev_name = name;</div><div class='add'>+	mplane-&gt;ib_dev.phys_port_cnt = mplane-&gt;num_ports;</div><div class='add'>+</div><div class='add'>+	ret = __mlx5_ib_add(mplane, &amp;plane_profile);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto fail_ib_add;</div><div class='add'>+</div><div class='add'>+	mparent-&gt;smi_dev = mplane;</div><div class='add'>+	return &amp;mplane-&gt;ib_dev;</div><div class='add'>+</div><div class='add'>+fail_ib_add:</div><div class='add'>+	kfree(mplane-&gt;port);</div><div class='add'>+fail_kcalloc:</div><div class='add'>+	ib_dealloc_device(&amp;mplane-&gt;ib_dev);</div><div class='add'>+	return ERR_PTR(ret);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_del_sub_dev(struct ib_device *sub_dev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *mdev = to_mdev(sub_dev);</div><div class='add'>+</div><div class='add'>+	to_mdev(sub_dev-&gt;parent)-&gt;smi_dev = NULL;</div><div class='add'>+	__mlx5_ib_remove(mdev, mdev-&gt;profile, MLX5_IB_STAGE_MAX);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5r_mp_probe(struct auxiliary_device *adev,</div><div class='add'>+			  const struct auxiliary_device_id *id)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_adev *idev = container_of(adev, struct mlx5_adev, adev);</div><div class='add'>+	struct mlx5_core_dev *mdev = idev-&gt;mdev;</div><div class='ctx'> 	struct mlx5_ib_multiport_info *mpi;</div><div class='ctx'> 	struct mlx5_ib_dev *dev;</div><div class='ctx'> 	bool bound = false;</div><div class='hunk'>@@ -6737,24 +4978,25 @@ static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev)</div><div class='ctx'> </div><div class='ctx'> 	mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);</div><div class='ctx'> 	if (!mpi)</div><div class='del'>-		return NULL;</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	mpi-&gt;mdev = mdev;</div><div class='del'>-</div><div class='ctx'> 	err = mlx5_query_nic_vport_system_image_guid(mdev,</div><div class='ctx'> 						     &amp;mpi-&gt;sys_image_guid);</div><div class='ctx'> 	if (err) {</div><div class='ctx'> 		kfree(mpi);</div><div class='del'>-		return NULL;</div><div class='add'>+		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;mlx5_ib_multiport_mutex);</div><div class='ctx'> 	list_for_each_entry(dev, &amp;mlx5_ib_dev_list, ib_dev_list) {</div><div class='del'>-		if (dev-&gt;sys_image_guid == mpi-&gt;sys_image_guid)</div><div class='add'>+		if (dev-&gt;sys_image_guid == mpi-&gt;sys_image_guid &amp;&amp;</div><div class='add'>+		    mlx5_core_same_coredev_type(dev-&gt;mdev, mpi-&gt;mdev))</div><div class='ctx'> 			bound = mlx5_ib_bind_slave_port(dev, mpi);</div><div class='ctx'> </div><div class='ctx'> 		if (bound) {</div><div class='ctx'> 			rdma_roce_rescan_device(&amp;dev-&gt;ib_dev);</div><div class='add'>+			mpi-&gt;ibdev-&gt;ib_active = true;</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='hunk'>@@ -6766,119 +5008,173 @@ static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev)</div><div class='ctx'> 	}</div><div class='ctx'> 	mutex_unlock(&amp;mlx5_ib_multiport_mutex);</div><div class='ctx'> </div><div class='del'>-	return mpi;</div><div class='add'>+	auxiliary_set_drvdata(adev, mpi);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void *mlx5_ib_add(struct mlx5_core_dev *mdev)</div><div class='add'>+static void mlx5r_mp_remove(struct auxiliary_device *adev)</div><div class='ctx'> {</div><div class='del'>-	enum rdma_link_layer ll;</div><div class='del'>-	struct mlx5_ib_dev *dev;</div><div class='del'>-	int port_type_cap;</div><div class='del'>-	int num_ports;</div><div class='add'>+	struct mlx5_ib_multiport_info *mpi;</div><div class='ctx'> </div><div class='del'>-	printk_once(KERN_INFO "%s", mlx5_version);</div><div class='add'>+	mpi = auxiliary_get_drvdata(adev);</div><div class='add'>+	mutex_lock(&amp;mlx5_ib_multiport_mutex);</div><div class='add'>+	if (mpi-&gt;ibdev)</div><div class='add'>+		mlx5_ib_unbind_slave_port(mpi-&gt;ibdev, mpi);</div><div class='add'>+	else</div><div class='add'>+		list_del(&amp;mpi-&gt;list);</div><div class='add'>+	mutex_unlock(&amp;mlx5_ib_multiport_mutex);</div><div class='add'>+	kfree(mpi);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (MLX5_ESWITCH_MANAGER(mdev) &amp;&amp;</div><div class='del'>-	    mlx5_ib_eswitch_mode(mdev-&gt;priv.eswitch) == SRIOV_OFFLOADS) {</div><div class='del'>-		if (!mlx5_core_mp_enabled(mdev))</div><div class='del'>-			mlx5_ib_register_vport_reps(mdev);</div><div class='del'>-		return mdev;</div><div class='del'>-	}</div><div class='add'>+static int mlx5r_probe(struct auxiliary_device *adev,</div><div class='add'>+		       const struct auxiliary_device_id *id)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_adev *idev = container_of(adev, struct mlx5_adev, adev);</div><div class='add'>+	struct mlx5_core_dev *mdev = idev-&gt;mdev;</div><div class='add'>+	const struct mlx5_ib_profile *profile;</div><div class='add'>+	int port_type_cap, num_ports, ret;</div><div class='add'>+	enum rdma_link_layer ll;</div><div class='add'>+	struct mlx5_ib_dev *dev;</div><div class='ctx'> </div><div class='ctx'> 	port_type_cap = MLX5_CAP_GEN(mdev, port_type);</div><div class='ctx'> 	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);</div><div class='ctx'> </div><div class='del'>-	if (mlx5_core_is_mp_slave(mdev) &amp;&amp; ll == IB_LINK_LAYER_ETHERNET)</div><div class='del'>-		return mlx5_ib_add_slave_port(mdev);</div><div class='del'>-</div><div class='ctx'> 	num_ports = max(MLX5_CAP_GEN(mdev, num_ports),</div><div class='ctx'> 			MLX5_CAP_GEN(mdev, num_vhca_ports));</div><div class='del'>-	dev = ib_alloc_device(mlx5_ib_dev, ib_dev);</div><div class='add'>+	dev = ib_alloc_device_with_net(mlx5_ib_dev, ib_dev,</div><div class='add'>+				       mlx5_core_net(mdev));</div><div class='ctx'> 	if (!dev)</div><div class='del'>-		return NULL;</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	if (ll == IB_LINK_LAYER_INFINIBAND) {</div><div class='add'>+		ret = mlx5_ib_get_plane_num(mdev, &amp;dev-&gt;num_plane);</div><div class='add'>+		if (ret)</div><div class='add'>+			goto fail;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> 	dev-&gt;port = kcalloc(num_ports, sizeof(*dev-&gt;port),</div><div class='ctx'> 			     GFP_KERNEL);</div><div class='ctx'> 	if (!dev-&gt;port) {</div><div class='del'>-		ib_dealloc_device((struct ib_device *)dev);</div><div class='del'>-		return NULL;</div><div class='add'>+		ret = -ENOMEM;</div><div class='add'>+		goto fail;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	dev-&gt;mdev = mdev;</div><div class='ctx'> 	dev-&gt;num_ports = num_ports;</div><div class='add'>+	dev-&gt;ib_dev.phys_port_cnt = num_ports;</div><div class='add'>+</div><div class='add'>+	if (ll == IB_LINK_LAYER_ETHERNET &amp;&amp; !mlx5_get_roce_state(mdev))</div><div class='add'>+		profile = &amp;raw_eth_profile;</div><div class='add'>+	else</div><div class='add'>+		profile = &amp;pf_profile;</div><div class='add'>+</div><div class='add'>+	ret = __mlx5_ib_add(dev, profile);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto fail_ib_add;</div><div class='add'>+</div><div class='add'>+	auxiliary_set_drvdata(adev, dev);</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-	return __mlx5_ib_add(dev, &amp;pf_profile);</div><div class='add'>+fail_ib_add:</div><div class='add'>+	kfree(dev-&gt;port);</div><div class='add'>+fail:</div><div class='add'>+	ib_dealloc_device(&amp;dev-&gt;ib_dev);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)</div><div class='add'>+static void mlx5r_remove(struct auxiliary_device *adev)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_multiport_info *mpi;</div><div class='ctx'> 	struct mlx5_ib_dev *dev;</div><div class='ctx'> </div><div class='del'>-	if (MLX5_ESWITCH_MANAGER(mdev) &amp;&amp; context == mdev) {</div><div class='del'>-		mlx5_ib_unregister_vport_reps(mdev);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (mlx5_core_is_mp_slave(mdev)) {</div><div class='del'>-		mpi = context;</div><div class='del'>-		mutex_lock(&amp;mlx5_ib_multiport_mutex);</div><div class='del'>-		if (mpi-&gt;ibdev)</div><div class='del'>-			mlx5_ib_unbind_slave_port(mpi-&gt;ibdev, mpi);</div><div class='del'>-		list_del(&amp;mpi-&gt;list);</div><div class='del'>-		mutex_unlock(&amp;mlx5_ib_multiport_mutex);</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	dev = context;</div><div class='add'>+	dev = auxiliary_get_drvdata(adev);</div><div class='ctx'> 	__mlx5_ib_remove(dev, dev-&gt;profile, MLX5_IB_STAGE_MAX);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct mlx5_interface mlx5_ib_interface = {</div><div class='del'>-	.add            = mlx5_ib_add,</div><div class='del'>-	.remove         = mlx5_ib_remove,</div><div class='del'>-	.protocol	= MLX5_INTERFACE_PROTOCOL_IB,</div><div class='add'>+static const struct auxiliary_device_id mlx5r_mp_id_table[] = {</div><div class='add'>+	{ .name = MLX5_ADEV_NAME ".multiport", },</div><div class='add'>+	{},</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-unsigned long mlx5_ib_get_xlt_emergency_page(void)</div><div class='del'>-{</div><div class='del'>-	mutex_lock(&amp;xlt_emergency_page_mutex);</div><div class='del'>-	return xlt_emergency_page;</div><div class='del'>-}</div><div class='add'>+static const struct auxiliary_device_id mlx5r_id_table[] = {</div><div class='add'>+	{ .name = MLX5_ADEV_NAME ".rdma", },</div><div class='add'>+	{},</div><div class='add'>+};</div><div class='ctx'> </div><div class='del'>-void mlx5_ib_put_xlt_emergency_page(void)</div><div class='del'>-{</div><div class='del'>-	mutex_unlock(&amp;xlt_emergency_page_mutex);</div><div class='del'>-}</div><div class='add'>+MODULE_DEVICE_TABLE(auxiliary, mlx5r_mp_id_table);</div><div class='add'>+MODULE_DEVICE_TABLE(auxiliary, mlx5r_id_table);</div><div class='add'>+</div><div class='add'>+static struct auxiliary_driver mlx5r_mp_driver = {</div><div class='add'>+	.name = "multiport",</div><div class='add'>+	.probe = mlx5r_mp_probe,</div><div class='add'>+	.remove = mlx5r_mp_remove,</div><div class='add'>+	.id_table = mlx5r_mp_id_table,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static struct auxiliary_driver mlx5r_driver = {</div><div class='add'>+	.name = "rdma",</div><div class='add'>+	.probe = mlx5r_probe,</div><div class='add'>+	.remove = mlx5r_remove,</div><div class='add'>+	.id_table = mlx5r_id_table,</div><div class='add'>+};</div><div class='ctx'> </div><div class='ctx'> static int __init mlx5_ib_init(void)</div><div class='ctx'> {</div><div class='del'>-	int err;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	xlt_emergency_page = __get_free_page(GFP_KERNEL);</div><div class='add'>+	xlt_emergency_page = (void *)__get_free_page(GFP_KERNEL);</div><div class='ctx'> 	if (!xlt_emergency_page)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	mutex_init(&amp;xlt_emergency_page_mutex);</div><div class='del'>-</div><div class='ctx'> 	mlx5_ib_event_wq = alloc_ordered_workqueue("mlx5_ib_event_wq", 0);</div><div class='ctx'> 	if (!mlx5_ib_event_wq) {</div><div class='del'>-		free_page(xlt_emergency_page);</div><div class='add'>+		free_page((unsigned long)xlt_emergency_page);</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	ret = mlx5_ib_qp_event_init();</div><div class='add'>+	if (ret)</div><div class='add'>+		goto qp_event_err;</div><div class='add'>+</div><div class='ctx'> 	mlx5_ib_odp_init();</div><div class='add'>+	ret = mlx5r_rep_init();</div><div class='add'>+	if (ret)</div><div class='add'>+		goto rep_err;</div><div class='add'>+	ret = mlx5_data_direct_driver_register();</div><div class='add'>+	if (ret)</div><div class='add'>+		goto dd_err;</div><div class='add'>+	ret = auxiliary_driver_register(&amp;mlx5r_mp_driver);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto mp_err;</div><div class='add'>+	ret = auxiliary_driver_register(&amp;mlx5r_driver);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto drv_err;</div><div class='ctx'> </div><div class='del'>-	err = mlx5_register_interface(&amp;mlx5_ib_interface);</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-	return err;</div><div class='add'>+drv_err:</div><div class='add'>+	auxiliary_driver_unregister(&amp;mlx5r_mp_driver);</div><div class='add'>+mp_err:</div><div class='add'>+	mlx5_data_direct_driver_unregister();</div><div class='add'>+dd_err:</div><div class='add'>+	mlx5r_rep_cleanup();</div><div class='add'>+rep_err:</div><div class='add'>+	mlx5_ib_qp_event_cleanup();</div><div class='add'>+qp_event_err:</div><div class='add'>+	destroy_workqueue(mlx5_ib_event_wq);</div><div class='add'>+	free_page((unsigned long)xlt_emergency_page);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void __exit mlx5_ib_cleanup(void)</div><div class='ctx'> {</div><div class='del'>-	mlx5_unregister_interface(&amp;mlx5_ib_interface);</div><div class='add'>+	mlx5_data_direct_driver_unregister();</div><div class='add'>+	auxiliary_driver_unregister(&amp;mlx5r_driver);</div><div class='add'>+	auxiliary_driver_unregister(&amp;mlx5r_mp_driver);</div><div class='add'>+	mlx5r_rep_cleanup();</div><div class='add'>+</div><div class='add'>+	mlx5_ib_qp_event_cleanup();</div><div class='ctx'> 	destroy_workqueue(mlx5_ib_event_wq);</div><div class='del'>-	mutex_destroy(&amp;xlt_emergency_page_mutex);</div><div class='del'>-	free_page(xlt_emergency_page);</div><div class='add'>+	free_page((unsigned long)xlt_emergency_page);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> module_init(mlx5_ib_init);</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/mem.c b/drivers/infiniband/hw/mlx5/mem.c<br/>index 9f90be296ee0..af321f6ef7f5 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/mem.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/mem.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/mem.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/mem.c</a></div><div class='hunk'>@@ -30,201 +30,66 @@</div><div class='ctx'>  * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='del'>-#include &lt;rdma/ib_umem.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_umem_odp.h&gt;</div><div class='ctx'> #include "mlx5_ib.h"</div><div class='ctx'> </div><div class='del'>-/* @umem: umem object to scan</div><div class='del'>- * @addr: ib virtual address requested by the user</div><div class='del'>- * @max_page_shift: high limit for page_shift - 0 means no limit</div><div class='del'>- * @count: number of PAGE_SIZE pages covered by umem</div><div class='del'>- * @shift: page shift for the compound pages found in the region</div><div class='del'>- * @ncont: number of compund pages</div><div class='del'>- * @order: log2 of the number of compound pages</div><div class='add'>+/*</div><div class='add'>+ * Fill in a physical address list. ib_umem_num_dma_blocks() entries will be</div><div class='add'>+ * filled in the pas array.</div><div class='ctx'>  */</div><div class='del'>-void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,</div><div class='del'>-			unsigned long max_page_shift,</div><div class='del'>-			int *count, int *shift,</div><div class='del'>-			int *ncont, int *order)</div><div class='add'>+void mlx5_ib_populate_pas(struct ib_umem *umem, size_t page_size, __be64 *pas,</div><div class='add'>+			  u64 access_flags)</div><div class='ctx'> {</div><div class='del'>-	unsigned long tmp;</div><div class='del'>-	unsigned long m;</div><div class='del'>-	u64 base = ~0, p = 0;</div><div class='del'>-	u64 len, pfn;</div><div class='del'>-	int i = 0;</div><div class='del'>-	struct scatterlist *sg;</div><div class='del'>-	int entry;</div><div class='del'>-	unsigned long page_shift = umem-&gt;page_shift;</div><div class='del'>-</div><div class='del'>-	if (umem-&gt;is_odp) {</div><div class='del'>-		*ncont = ib_umem_page_count(umem);</div><div class='del'>-		*count = *ncont &lt;&lt; (page_shift - PAGE_SHIFT);</div><div class='del'>-		*shift = page_shift;</div><div class='del'>-		if (order)</div><div class='del'>-			*order = ilog2(roundup_pow_of_two(*ncont));</div><div class='del'>-</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	addr = addr &gt;&gt; page_shift;</div><div class='del'>-	tmp = (unsigned long)addr;</div><div class='del'>-	m = find_first_bit(&amp;tmp, BITS_PER_LONG);</div><div class='del'>-	if (max_page_shift)</div><div class='del'>-		m = min_t(unsigned long, max_page_shift - page_shift, m);</div><div class='del'>-</div><div class='del'>-	for_each_sg(umem-&gt;sg_head.sgl, sg, umem-&gt;nmap, entry) {</div><div class='del'>-		len = sg_dma_len(sg) &gt;&gt; page_shift;</div><div class='del'>-		pfn = sg_dma_address(sg) &gt;&gt; page_shift;</div><div class='del'>-		if (base + p != pfn) {</div><div class='del'>-			/* If either the offset or the new</div><div class='del'>-			 * base are unaligned update m</div><div class='del'>-			 */</div><div class='del'>-			tmp = (unsigned long)(pfn | p);</div><div class='del'>-			if (!IS_ALIGNED(tmp, 1 &lt;&lt; m))</div><div class='del'>-				m = find_first_bit(&amp;tmp, BITS_PER_LONG);</div><div class='del'>-</div><div class='del'>-			base = pfn;</div><div class='del'>-			p = 0;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		p += len;</div><div class='del'>-		i += len;</div><div class='del'>-	}</div><div class='add'>+	struct ib_block_iter biter;</div><div class='ctx'> </div><div class='del'>-	if (i) {</div><div class='del'>-		m = min_t(unsigned long, ilog2(roundup_pow_of_two(i)), m);</div><div class='del'>-</div><div class='del'>-		if (order)</div><div class='del'>-			*order = ilog2(roundup_pow_of_two(i) &gt;&gt; m);</div><div class='del'>-</div><div class='del'>-		*ncont = DIV_ROUND_UP(i, (1 &lt;&lt; m));</div><div class='del'>-	} else {</div><div class='del'>-		m  = 0;</div><div class='del'>-</div><div class='del'>-		if (order)</div><div class='del'>-			*order = 0;</div><div class='del'>-</div><div class='del'>-		*ncont = 0;</div><div class='add'>+	rdma_umem_for_each_dma_block (umem, &amp;biter, page_size) {</div><div class='add'>+		*pas = cpu_to_be64(rdma_block_iter_dma_address(&amp;biter) |</div><div class='add'>+				   access_flags);</div><div class='add'>+		pas++;</div><div class='ctx'> 	}</div><div class='del'>-	*shift = page_shift + m;</div><div class='del'>-	*count = i;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static u64 umem_dma_to_mtt(dma_addr_t umem_dma)</div><div class='del'>-{</div><div class='del'>-	u64 mtt_entry = umem_dma &amp; ODP_DMA_ADDR_MASK;</div><div class='del'>-</div><div class='del'>-	if (umem_dma &amp; ODP_READ_ALLOWED_BIT)</div><div class='del'>-		mtt_entry |= MLX5_IB_MTT_READ;</div><div class='del'>-	if (umem_dma &amp; ODP_WRITE_ALLOWED_BIT)</div><div class='del'>-		mtt_entry |= MLX5_IB_MTT_WRITE;</div><div class='del'>-</div><div class='del'>-	return mtt_entry;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='del'>- * Populate the given array with bus addresses from the umem.</div><div class='del'>- *</div><div class='del'>- * dev - mlx5_ib device</div><div class='del'>- * umem - umem to use to fill the pages</div><div class='del'>- * page_shift - determines the page size used in the resulting array</div><div class='del'>- * offset - offset into the umem to start from,</div><div class='del'>- *          only implemented for ODP umems</div><div class='del'>- * num_pages - total number of pages to fill</div><div class='del'>- * pas - bus addresses array to fill</div><div class='del'>- * access_flags - access flags to set on all present pages.</div><div class='del'>-		  use enum mlx5_ib_mtt_access_flags for this.</div><div class='add'>+ * Compute the page shift and page_offset for mailboxes that use a quantized</div><div class='add'>+ * page_offset. The granulatity of the page offset scales according to page</div><div class='add'>+ * size.</div><div class='ctx'>  */</div><div class='del'>-void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,</div><div class='del'>-			    int page_shift, size_t offset, size_t num_pages,</div><div class='del'>-			    __be64 *pas, int access_flags)</div><div class='add'>+unsigned long __mlx5_umem_find_best_quantized_pgoff(</div><div class='add'>+	struct ib_umem *umem, unsigned long pgsz_bitmap,</div><div class='add'>+	unsigned int page_offset_bits, u64 pgoff_bitmask, unsigned int scale,</div><div class='add'>+	unsigned int *page_offset_quantized)</div><div class='ctx'> {</div><div class='del'>-	unsigned long umem_page_shift = umem-&gt;page_shift;</div><div class='del'>-	int shift = page_shift - umem_page_shift;</div><div class='del'>-	int mask = (1 &lt;&lt; shift) - 1;</div><div class='del'>-	int i, k, idx;</div><div class='del'>-	u64 cur = 0;</div><div class='del'>-	u64 base;</div><div class='del'>-	int len;</div><div class='del'>-	struct scatterlist *sg;</div><div class='del'>-	int entry;</div><div class='del'>-</div><div class='del'>-	if (umem-&gt;is_odp) {</div><div class='del'>-		WARN_ON(shift != 0);</div><div class='del'>-		WARN_ON(access_flags != (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE));</div><div class='del'>-</div><div class='del'>-		for (i = 0; i &lt; num_pages; ++i) {</div><div class='del'>-			dma_addr_t pa =</div><div class='del'>-				to_ib_umem_odp(umem)-&gt;dma_list[offset + i];</div><div class='del'>-</div><div class='del'>-			pas[i] = cpu_to_be64(umem_dma_to_mtt(pa));</div><div class='del'>-		}</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	i = 0;</div><div class='del'>-	for_each_sg(umem-&gt;sg_head.sgl, sg, umem-&gt;nmap, entry) {</div><div class='del'>-		len = sg_dma_len(sg) &gt;&gt; umem_page_shift;</div><div class='del'>-		base = sg_dma_address(sg);</div><div class='del'>-</div><div class='del'>-		/* Skip elements below offset */</div><div class='del'>-		if (i + len &lt; offset &lt;&lt; shift) {</div><div class='del'>-			i += len;</div><div class='del'>-			continue;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		/* Skip pages below offset */</div><div class='del'>-		if (i &lt; offset &lt;&lt; shift) {</div><div class='del'>-			k = (offset &lt;&lt; shift) - i;</div><div class='del'>-			i = offset &lt;&lt; shift;</div><div class='del'>-		} else {</div><div class='del'>-			k = 0;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		for (; k &lt; len; k++) {</div><div class='del'>-			if (!(i &amp; mask)) {</div><div class='del'>-				cur = base + (k &lt;&lt; umem_page_shift);</div><div class='del'>-				cur |= access_flags;</div><div class='del'>-				idx = (i &gt;&gt; shift) - offset;</div><div class='del'>-</div><div class='del'>-				pas[idx] = cpu_to_be64(cur);</div><div class='del'>-				mlx5_ib_dbg(dev, "pas[%d] 0x%llx\n",</div><div class='del'>-					    i &gt;&gt; shift, be64_to_cpu(pas[idx]));</div><div class='del'>-			}</div><div class='del'>-			i++;</div><div class='del'>-</div><div class='del'>-			/* Stop after num_pages reached */</div><div class='del'>-			if (i &gt;&gt; shift &gt;= offset + num_pages)</div><div class='del'>-				return;</div><div class='del'>-		}</div><div class='add'>+	const u64 page_offset_mask = (1UL &lt;&lt; page_offset_bits) - 1;</div><div class='add'>+	unsigned long page_size;</div><div class='add'>+	u64 page_offset;</div><div class='add'>+</div><div class='add'>+	page_size = ib_umem_find_best_pgoff(umem, pgsz_bitmap, pgoff_bitmask);</div><div class='add'>+	if (!page_size)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * page size is the largest possible page size.</div><div class='add'>+	 *</div><div class='add'>+	 * Reduce the page_size, and thus the page_offset and quanta, until the</div><div class='add'>+	 * page_offset fits into the mailbox field. Once page_size &lt; scale this</div><div class='add'>+	 * loop is guaranteed to terminate.</div><div class='add'>+	 */</div><div class='add'>+	page_offset = ib_umem_dma_offset(umem, page_size);</div><div class='add'>+	while (page_offset &amp; ~(u64)(page_offset_mask * (page_size / scale))) {</div><div class='add'>+		page_size /= 2;</div><div class='add'>+		page_offset = ib_umem_dma_offset(umem, page_size);</div><div class='ctx'> 	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,</div><div class='del'>-			  int page_shift, __be64 *pas, int access_flags)</div><div class='del'>-{</div><div class='del'>-	return __mlx5_ib_populate_pas(dev, umem, page_shift, 0,</div><div class='del'>-				      ib_umem_num_pages(umem), pas,</div><div class='del'>-				      access_flags);</div><div class='del'>-}</div><div class='del'>-int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset)</div><div class='del'>-{</div><div class='del'>-	u64 page_size;</div><div class='del'>-	u64 page_mask;</div><div class='del'>-	u64 off_size;</div><div class='del'>-	u64 off_mask;</div><div class='del'>-	u64 buf_off;</div><div class='del'>-</div><div class='del'>-	page_size = (u64)1 &lt;&lt; page_shift;</div><div class='del'>-	page_mask = page_size - 1;</div><div class='del'>-	buf_off = addr &amp; page_mask;</div><div class='del'>-	off_size = page_size &gt;&gt; 6;</div><div class='del'>-	off_mask = off_size - 1;</div><div class='del'>-</div><div class='del'>-	if (buf_off &amp; off_mask)</div><div class='del'>-		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	*offset = buf_off &gt;&gt; ilog2(off_size);</div><div class='del'>-	return 0;</div><div class='add'>+	/*</div><div class='add'>+	 * The address is not aligned, or otherwise cannot be represented by the</div><div class='add'>+	 * page_offset.</div><div class='add'>+	 */</div><div class='add'>+	if (!(pgsz_bitmap &amp; page_size))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	*page_offset_quantized =</div><div class='add'>+		(unsigned long)page_offset / (page_size / scale);</div><div class='add'>+	if (WARN_ON(*page_offset_quantized &gt; page_offset_mask))</div><div class='add'>+		return 0;</div><div class='add'>+	return page_size;</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h<br/>index a043af7ee366..09d82d5f95e3 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/mlx5_ib.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/mlx5_ib.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/mlx5_ib.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/mlx5_ib.h</a></div><div class='hunk'>@@ -1,33 +1,7 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='ctx'> /*</div><div class='del'>- * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.</div><div class='del'>- *</div><div class='del'>- * This software is available to you under a choice of one of two</div><div class='del'>- * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='del'>- * General Public License (GPL) Version 2, available from the file</div><div class='del'>- * COPYING in the main directory of this source tree, or the</div><div class='del'>- * OpenIB.org BSD license below:</div><div class='del'>- *</div><div class='del'>- *     Redistribution and use in source and binary forms, with or</div><div class='del'>- *     without modification, are permitted provided that the following</div><div class='del'>- *     conditions are met:</div><div class='del'>- *</div><div class='del'>- *      - Redistributions of source code must retain the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer.</div><div class='del'>- *</div><div class='del'>- *      - Redistributions in binary form must reproduce the above</div><div class='del'>- *        copyright notice, this list of conditions and the following</div><div class='del'>- *        disclaimer in the documentation and/or other materials</div><div class='del'>- *        provided with the distribution.</div><div class='del'>- *</div><div class='del'>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,</div><div class='del'>- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF</div><div class='del'>- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND</div><div class='del'>- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS</div><div class='del'>- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN</div><div class='del'>- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN</div><div class='del'>- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE</div><div class='del'>- * SOFTWARE.</div><div class='add'>+ * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.</div><div class='add'>+ * Copyright (c) 2020, Intel Corporation. All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='ctx'> #ifndef MLX5_IB_H</div><div class='hunk'>@@ -51,6 +25,8 @@</div><div class='ctx'> #include &lt;rdma/mlx5_user_ioctl_verbs.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "srq.h"</div><div class='add'>+#include "qp.h"</div><div class='add'>+#include "macsec.h"</div><div class='ctx'> </div><div class='ctx'> #define mlx5_ib_dbg(_dev, format, arg...)                                      \</div><div class='ctx'> 	dev_dbg(&amp;(_dev)-&gt;ib_dev.dev, "%s:%d:(pid %d): " format, __func__,      \</div><div class='hunk'>@@ -64,12 +40,74 @@</div><div class='ctx'> 	dev_warn(&amp;(_dev)-&gt;ib_dev.dev, "%s:%d:(pid %d): " format, __func__,     \</div><div class='ctx'> 		 __LINE__, current-&gt;pid, ##arg)</div><div class='ctx'> </div><div class='del'>-#define field_avail(type, fld, sz) (offsetof(type, fld) +		\</div><div class='del'>-				    sizeof(((type *)0)-&gt;fld) &lt;= (sz))</div><div class='add'>+#define mlx5_ib_log(lvl, _dev, format, arg...)                                 \</div><div class='add'>+	dev_printk(lvl, &amp;(_dev)-&gt;ib_dev.dev,  "%s:%d:(pid %d): " format,       \</div><div class='add'>+		   __func__, __LINE__, current-&gt;pid, ##arg)</div><div class='add'>+</div><div class='ctx'> #define MLX5_IB_DEFAULT_UIDX 0xffffff</div><div class='ctx'> #define MLX5_USER_ASSIGNED_UIDX_MASK __mlx5_mask(qpc, user_index)</div><div class='ctx'> </div><div class='del'>-#define MLX5_MKEY_PAGE_SHIFT_MASK __mlx5_mask(mkc, log_page_size)</div><div class='add'>+static __always_inline unsigned long</div><div class='add'>+__mlx5_log_page_size_to_bitmap(unsigned int log_pgsz_bits,</div><div class='add'>+			       unsigned int pgsz_shift)</div><div class='add'>+{</div><div class='add'>+	unsigned int largest_pg_shift =</div><div class='add'>+		min_t(unsigned long, (1ULL &lt;&lt; log_pgsz_bits) - 1 + pgsz_shift,</div><div class='add'>+		      BITS_PER_LONG - 1);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Despite a command allowing it, the device does not support lower than</div><div class='add'>+	 * 4k page size.</div><div class='add'>+	 */</div><div class='add'>+	pgsz_shift = max_t(unsigned int, MLX5_ADAPTER_PAGE_SHIFT, pgsz_shift);</div><div class='add'>+	return GENMASK(largest_pg_shift, pgsz_shift);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static __always_inline unsigned long</div><div class='add'>+__mlx5_page_offset_to_bitmask(unsigned int page_offset_bits,</div><div class='add'>+			      unsigned int offset_shift)</div><div class='add'>+{</div><div class='add'>+	unsigned int largest_offset_shift =</div><div class='add'>+		min_t(unsigned long, page_offset_bits - 1 + offset_shift,</div><div class='add'>+		      BITS_PER_LONG - 1);</div><div class='add'>+</div><div class='add'>+	return GENMASK(largest_offset_shift, offset_shift);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * QP/CQ/WQ/etc type commands take a page offset that satisifies:</div><div class='add'>+ *   page_offset_quantized * (page_size/scale) = page_offset</div><div class='add'>+ * Which restricts allowed page sizes to ones that satisify the above.</div><div class='add'>+ */</div><div class='add'>+unsigned long __mlx5_umem_find_best_quantized_pgoff(</div><div class='add'>+	struct ib_umem *umem, unsigned long pgsz_bitmap,</div><div class='add'>+	unsigned int page_offset_bits, u64 pgoff_bitmask, unsigned int scale,</div><div class='add'>+	unsigned int *page_offset_quantized);</div><div class='add'>+#define mlx5_umem_find_best_quantized_pgoff(umem, typ, log_pgsz_fld,           \</div><div class='add'>+					    pgsz_shift, page_offset_fld,       \</div><div class='add'>+					    scale, page_offset_quantized)      \</div><div class='add'>+	__mlx5_umem_find_best_quantized_pgoff(                                 \</div><div class='add'>+		umem,                                                          \</div><div class='add'>+		__mlx5_log_page_size_to_bitmap(                                \</div><div class='add'>+			__mlx5_bit_sz(typ, log_pgsz_fld), pgsz_shift),         \</div><div class='add'>+		__mlx5_bit_sz(typ, page_offset_fld),                           \</div><div class='add'>+		GENMASK(31, order_base_2(scale)), scale,                       \</div><div class='add'>+		page_offset_quantized)</div><div class='add'>+</div><div class='add'>+#define mlx5_umem_find_best_cq_quantized_pgoff(umem, typ, log_pgsz_fld,        \</div><div class='add'>+					       pgsz_shift, page_offset_fld,    \</div><div class='add'>+					       scale, page_offset_quantized)   \</div><div class='add'>+	__mlx5_umem_find_best_quantized_pgoff(                                 \</div><div class='add'>+		umem,                                                          \</div><div class='add'>+		__mlx5_log_page_size_to_bitmap(                                \</div><div class='add'>+			__mlx5_bit_sz(typ, log_pgsz_fld), pgsz_shift),         \</div><div class='add'>+		__mlx5_bit_sz(typ, page_offset_fld), 0, scale,                 \</div><div class='add'>+		page_offset_quantized)</div><div class='add'>+</div><div class='add'>+enum {</div><div class='add'>+	MLX5_IB_MMAP_OFFSET_START = 9,</div><div class='add'>+	MLX5_IB_MMAP_OFFSET_END = 255,</div><div class='add'>+};</div><div class='ctx'> </div><div class='ctx'> enum {</div><div class='ctx'> 	MLX5_IB_MMAP_CMD_SHIFT	= 8,</div><div class='hunk'>@@ -118,9 +156,31 @@ enum {</div><div class='ctx'> 	MLX5_MEMIC_BASE_SIZE	= 1 &lt;&lt; MLX5_MEMIC_BASE_ALIGN,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-#define MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)                                        \</div><div class='del'>-	(MLX5_CAP_DEV_MEM(dev, log_sw_icm_alloc_granularity))</div><div class='del'>-#define MLX5_SW_ICM_BLOCK_SIZE(dev) (1 &lt;&lt; MLX5_LOG_SW_ICM_BLOCK_SIZE(dev))</div><div class='add'>+enum mlx5_ib_mmap_type {</div><div class='add'>+	MLX5_IB_MMAP_TYPE_MEMIC = 1,</div><div class='add'>+	MLX5_IB_MMAP_TYPE_VAR = 2,</div><div class='add'>+	MLX5_IB_MMAP_TYPE_UAR_WC = 3,</div><div class='add'>+	MLX5_IB_MMAP_TYPE_UAR_NC = 4,</div><div class='add'>+	MLX5_IB_MMAP_TYPE_MEMIC_OP = 5,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mlx5_bfreg_info {</div><div class='add'>+	u32 *sys_pages;</div><div class='add'>+	int num_low_latency_bfregs;</div><div class='add'>+	unsigned int *count;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * protect bfreg allocation data structs</div><div class='add'>+	 */</div><div class='add'>+	struct mutex lock;</div><div class='add'>+	u32 ver;</div><div class='add'>+	u8 lib_uar_4k : 1;</div><div class='add'>+	u8 lib_uar_dyn : 1;</div><div class='add'>+	u32 num_sys_pages;</div><div class='add'>+	u32 num_static_sys_pages;</div><div class='add'>+	u32 total_num_bfregs;</div><div class='add'>+	u32 num_dyn_bfregs;</div><div class='add'>+};</div><div class='ctx'> </div><div class='ctx'> struct mlx5_ib_ucontext {</div><div class='ctx'> 	struct ib_ucontext	ibucontext;</div><div class='hunk'>@@ -135,7 +195,6 @@ struct mlx5_ib_ucontext {</div><div class='ctx'> 	u32			tdn;</div><div class='ctx'> </div><div class='ctx'> 	u64			lib_caps;</div><div class='del'>-	DECLARE_BITMAP(dm_pages, MLX5_MAX_MEMIC_PAGES);</div><div class='ctx'> 	u16			devx_uid;</div><div class='ctx'> 	/* For RoCE LAG TX affinity */</div><div class='ctx'> 	atomic_t		tx_port_affinity;</div><div class='hunk'>@@ -168,8 +227,20 @@ enum {</div><div class='ctx'> #define MLX5_IB_NUM_FLOW_FT		(MLX5_IB_FLOW_LEFTOVERS_PRIO + 1)</div><div class='ctx'> #define MLX5_IB_NUM_SNIFFER_FTS		2</div><div class='ctx'> #define MLX5_IB_NUM_EGRESS_FTS		1</div><div class='add'>+#define MLX5_IB_NUM_FDB_FTS		MLX5_BY_PASS_NUM_REGULAR_PRIOS</div><div class='add'>+</div><div class='add'>+struct mlx5_ib_anchor {</div><div class='add'>+	struct mlx5_flow_table *ft;</div><div class='add'>+	struct mlx5_flow_group *fg_goto_table;</div><div class='add'>+	struct mlx5_flow_group *fg_drop;</div><div class='add'>+	struct mlx5_flow_handle *rule_goto_table;</div><div class='add'>+	struct mlx5_flow_handle *rule_drop;</div><div class='add'>+	unsigned int rule_goto_table_ref;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct mlx5_ib_flow_prio {</div><div class='ctx'> 	struct mlx5_flow_table		*flow_table;</div><div class='add'>+	struct mlx5_ib_anchor		anchor;</div><div class='ctx'> 	unsigned int			refcount;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -192,6 +263,38 @@ struct mlx5_ib_flow_matcher {</div><div class='ctx'> 	struct mlx5_core_dev	*mdev;</div><div class='ctx'> 	atomic_t		usecnt;</div><div class='ctx'> 	u8			match_criteria_enable;</div><div class='add'>+	u32			ib_port;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mlx5_ib_steering_anchor {</div><div class='add'>+	struct mlx5_ib_flow_prio *ft_prio;</div><div class='add'>+	struct mlx5_ib_dev *dev;</div><div class='add'>+	atomic_t usecnt;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mlx5_ib_pp {</div><div class='add'>+	u16 index;</div><div class='add'>+	struct mlx5_core_dev *mdev;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum mlx5_ib_optional_counter_type {</div><div class='add'>+	MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS,</div><div class='add'>+	MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS,</div><div class='add'>+	MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS,</div><div class='add'>+	MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS,</div><div class='add'>+	MLX5_IB_OPCOUNTER_RDMA_TX_BYTES,</div><div class='add'>+	MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS,</div><div class='add'>+	MLX5_IB_OPCOUNTER_RDMA_RX_BYTES,</div><div class='add'>+</div><div class='add'>+	MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP,</div><div class='add'>+	MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP,</div><div class='add'>+	MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP,</div><div class='add'>+	MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP,</div><div class='add'>+	MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP,</div><div class='add'>+	MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP,</div><div class='add'>+	MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP,</div><div class='add'>+</div><div class='add'>+	MLX5_IB_OPCOUNTER_MAX,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct mlx5_ib_flow_db {</div><div class='hunk'>@@ -199,8 +302,13 @@ struct mlx5_ib_flow_db {</div><div class='ctx'> 	struct mlx5_ib_flow_prio	egress_prios[MLX5_IB_NUM_FLOW_FT];</div><div class='ctx'> 	struct mlx5_ib_flow_prio	sniffer[MLX5_IB_NUM_SNIFFER_FTS];</div><div class='ctx'> 	struct mlx5_ib_flow_prio	egress[MLX5_IB_NUM_EGRESS_FTS];</div><div class='del'>-	struct mlx5_ib_flow_prio	fdb;</div><div class='add'>+	struct mlx5_ib_flow_prio	fdb[MLX5_IB_NUM_FDB_FTS];</div><div class='add'>+	struct mlx5_ib_flow_prio	rdma_rx[MLX5_IB_NUM_FLOW_FT];</div><div class='add'>+	struct mlx5_ib_flow_prio	rdma_tx[MLX5_IB_NUM_FLOW_FT];</div><div class='add'>+	struct mlx5_ib_flow_prio	opfcs[MLX5_IB_OPCOUNTER_MAX];</div><div class='ctx'> 	struct mlx5_flow_table		*lag_demux_ft;</div><div class='add'>+	struct mlx5_ib_flow_prio        *rdma_transport_rx[MLX5_RDMA_TRANSPORT_BYPASS_PRIO];</div><div class='add'>+	struct mlx5_ib_flow_prio        *rdma_transport_tx[MLX5_RDMA_TRANSPORT_BYPASS_PRIO];</div><div class='ctx'> 	/* Protect flow steering bypass flow tables</div><div class='ctx'> 	 * when add/del flow rules.</div><div class='ctx'> 	 * only single add/removal of flow steering rule could be done</div><div class='hunk'>@@ -210,16 +318,9 @@ struct mlx5_ib_flow_db {</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> /* Use macros here so that don't have to duplicate</div><div class='del'>- * enum ib_send_flags and enum ib_qp_type for low-level driver</div><div class='add'>+ * enum ib_qp_type for low-level driver</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#define MLX5_IB_SEND_UMR_ENABLE_MR	       (IB_SEND_RESERVED_START &lt;&lt; 0)</div><div class='del'>-#define MLX5_IB_SEND_UMR_DISABLE_MR	       (IB_SEND_RESERVED_START &lt;&lt; 1)</div><div class='del'>-#define MLX5_IB_SEND_UMR_FAIL_IF_FREE	       (IB_SEND_RESERVED_START &lt;&lt; 2)</div><div class='del'>-#define MLX5_IB_SEND_UMR_UPDATE_XLT	       (IB_SEND_RESERVED_START &lt;&lt; 3)</div><div class='del'>-#define MLX5_IB_SEND_UMR_UPDATE_TRANSLATION    (IB_SEND_RESERVED_START &lt;&lt; 4)</div><div class='del'>-#define MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS       IB_SEND_RESERVED_END</div><div class='del'>-</div><div class='ctx'> #define MLX5_IB_QPT_REG_UMR	IB_QPT_RESERVED1</div><div class='ctx'> /*</div><div class='ctx'>  * IB_QPT_GSI creates the software wrapper around GSI, and MLX5_IB_QPT_HW_GSI</div><div class='hunk'>@@ -230,9 +331,6 @@ struct mlx5_ib_flow_db {</div><div class='ctx'> #define MLX5_IB_QPT_DCT		IB_QPT_RESERVED4</div><div class='ctx'> #define MLX5_IB_WR_UMR		IB_WR_RESERVED1</div><div class='ctx'> </div><div class='del'>-#define MLX5_IB_UMR_OCTOWORD	       16</div><div class='del'>-#define MLX5_IB_UMR_XLT_ALIGNMENT      64</div><div class='del'>-</div><div class='ctx'> #define MLX5_IB_UPD_XLT_ZAP	      BIT(0)</div><div class='ctx'> #define MLX5_IB_UPD_XLT_ENABLE	      BIT(1)</div><div class='ctx'> #define MLX5_IB_UPD_XLT_ATOMIC	      BIT(2)</div><div class='hunk'>@@ -240,18 +338,15 @@ struct mlx5_ib_flow_db {</div><div class='ctx'> #define MLX5_IB_UPD_XLT_PD	      BIT(4)</div><div class='ctx'> #define MLX5_IB_UPD_XLT_ACCESS	      BIT(5)</div><div class='ctx'> #define MLX5_IB_UPD_XLT_INDIRECT      BIT(6)</div><div class='add'>+#define MLX5_IB_UPD_XLT_DOWNGRADE     BIT(7)</div><div class='add'>+#define MLX5_IB_UPD_XLT_KEEP_PGSZ     BIT(8)</div><div class='ctx'> </div><div class='ctx'> /* Private QP creation flags to be passed in ib_qp_init_attr.create_flags.</div><div class='ctx'>  *</div><div class='ctx'>  * These flags are intended for internal use by the mlx5_ib driver, and they</div><div class='ctx'>  * rely on the range reserved for that use in the ib_qp_create_flags enum.</div><div class='ctx'>  */</div><div class='del'>-</div><div class='del'>-/* Create a UD QP whose source QP number is 1 */</div><div class='del'>-static inline enum ib_qp_create_flags mlx5_ib_create_qp_sqpn_qp1(void)</div><div class='del'>-{</div><div class='del'>-	return IB_QP_CREATE_RESERVED_START;</div><div class='del'>-}</div><div class='add'>+#define MLX5_IB_QP_CREATE_SQPN_QP1	IB_QP_CREATE_RESERVED_START</div><div class='ctx'> </div><div class='ctx'> struct wr_list {</div><div class='ctx'> 	u16	opcode;</div><div class='hunk'>@@ -282,6 +377,7 @@ struct mlx5_ib_wq {</div><div class='ctx'> 	unsigned		head;</div><div class='ctx'> 	unsigned		tail;</div><div class='ctx'> 	u16			cur_post;</div><div class='add'>+	u16			last_poll;</div><div class='ctx'> 	void			*cur_edge;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -294,6 +390,7 @@ enum mlx5_ib_wq_flags {</div><div class='ctx'> #define MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES 16</div><div class='ctx'> #define MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES 6</div><div class='ctx'> #define MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES 13</div><div class='add'>+#define MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES 3</div><div class='ctx'> </div><div class='ctx'> struct mlx5_ib_rwq {</div><div class='ctx'> 	struct ib_wq		ibwq;</div><div class='hunk'>@@ -309,7 +406,6 @@ struct mlx5_ib_rwq {</div><div class='ctx'> 	struct ib_umem		*umem;</div><div class='ctx'> 	size_t			buf_size;</div><div class='ctx'> 	unsigned int		page_shift;</div><div class='del'>-	int			create_type;</div><div class='ctx'> 	struct mlx5_db		db;</div><div class='ctx'> 	u32			user_index;</div><div class='ctx'> 	u32			wqe_count;</div><div class='hunk'>@@ -318,17 +414,6 @@ struct mlx5_ib_rwq {</div><div class='ctx'> 	u32			create_flags; /* Use enum mlx5_ib_wq_flags */</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-enum {</div><div class='del'>-	MLX5_QP_USER,</div><div class='del'>-	MLX5_QP_KERNEL,</div><div class='del'>-	MLX5_QP_EMPTY</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-enum {</div><div class='del'>-	MLX5_WQ_USER,</div><div class='del'>-	MLX5_WQ_KERNEL</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> struct mlx5_ib_rwq_ind_table {</div><div class='ctx'> 	struct ib_rwq_ind_table ib_rwq_ind_tbl;</div><div class='ctx'> 	u32			rqtn;</div><div class='hunk'>@@ -350,7 +435,7 @@ struct mlx5_ib_qp_base {</div><div class='ctx'> struct mlx5_ib_qp_trans {</div><div class='ctx'> 	struct mlx5_ib_qp_base	base;</div><div class='ctx'> 	u16			xrcdn;</div><div class='del'>-	u8			alt_port;</div><div class='add'>+	u32			alt_port;</div><div class='ctx'> 	u8			atomic_rd_en;</div><div class='ctx'> 	u8			resp_depth;</div><div class='ctx'> };</div><div class='hunk'>@@ -395,6 +480,22 @@ struct mlx5_ib_dct {</div><div class='ctx'> 	u32                     *in;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct mlx5_ib_gsi_qp {</div><div class='add'>+	struct ib_qp *rx_qp;</div><div class='add'>+	u32 port_num;</div><div class='add'>+	struct ib_qp_cap cap;</div><div class='add'>+	struct ib_cq *cq;</div><div class='add'>+	struct mlx5_ib_gsi_wr *outstanding_wrs;</div><div class='add'>+	u32 outstanding_pi, outstanding_ci;</div><div class='add'>+	int num_qps;</div><div class='add'>+	/* Protects access to the tx_qps. Post send operations synchronize</div><div class='add'>+	 * with tx_qp creation in setup_qp(). Also protects the</div><div class='add'>+	 * outstanding_wrs array and indices.</div><div class='add'>+	 */</div><div class='add'>+	spinlock_t lock;</div><div class='add'>+	struct ib_qp **tx_qps;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct mlx5_ib_qp {</div><div class='ctx'> 	struct ib_qp		ibqp;</div><div class='ctx'> 	union {</div><div class='hunk'>@@ -402,6 +503,7 @@ struct mlx5_ib_qp {</div><div class='ctx'> 		struct mlx5_ib_raw_packet_qp raw_packet_qp;</div><div class='ctx'> 		struct mlx5_ib_rss_qp rss_qp;</div><div class='ctx'> 		struct mlx5_ib_dct dct;</div><div class='add'>+		struct mlx5_ib_gsi_qp gsi;</div><div class='ctx'> 	};</div><div class='ctx'> 	struct mlx5_frag_buf	buf;</div><div class='ctx'> </div><div class='hunk'>@@ -415,33 +517,37 @@ struct mlx5_ib_qp {</div><div class='ctx'> 	/* serialize qp state modifications</div><div class='ctx'> 	 */</div><div class='ctx'> 	struct mutex		mutex;</div><div class='add'>+	/* cached variant of create_flags from struct ib_qp_init_attr */</div><div class='ctx'> 	u32			flags;</div><div class='del'>-	u8			port;</div><div class='add'>+	u32			port;</div><div class='ctx'> 	u8			state;</div><div class='del'>-	int			wq_sig;</div><div class='del'>-	int			scat_cqe;</div><div class='ctx'> 	int			max_inline_data;</div><div class='ctx'> 	struct mlx5_bf	        bf;</div><div class='del'>-	int			has_rq;</div><div class='add'>+	u8			has_rq:1;</div><div class='add'>+	u8			is_rss:1;</div><div class='add'>+	u8			is_ooo_rq:1;</div><div class='ctx'> </div><div class='ctx'> 	/* only for user space QPs. For kernel</div><div class='ctx'> 	 * we have it from the bf object</div><div class='ctx'> 	 */</div><div class='ctx'> 	int			bfregn;</div><div class='ctx'> </div><div class='del'>-	int			create_type;</div><div class='del'>-</div><div class='del'>-	/* Store signature errors */</div><div class='del'>-	bool			signature_en;</div><div class='del'>-</div><div class='ctx'> 	struct list_head	qps_list;</div><div class='ctx'> 	struct list_head	cq_recv_list;</div><div class='ctx'> 	struct list_head	cq_send_list;</div><div class='ctx'> 	struct mlx5_rate_limit	rl;</div><div class='ctx'> 	u32                     underlay_qpn;</div><div class='ctx'> 	u32			flags_en;</div><div class='del'>-	/* storage for qp sub type when core qp type is IB_QPT_DRIVER */</div><div class='del'>-	enum ib_qp_type		qp_sub_type;</div><div class='add'>+	/*</div><div class='add'>+	 * IB/core doesn't store low-level QP types, so</div><div class='add'>+	 * store both MLX and IBTA types in the field below.</div><div class='add'>+	 */</div><div class='add'>+	enum ib_qp_type		type;</div><div class='add'>+	/* A flag to indicate if there's a new counter is configured</div><div class='add'>+	 * but not take effective</div><div class='add'>+	 */</div><div class='add'>+	u32                     counter_pending;</div><div class='add'>+	u16			gsi_lag_port;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct mlx5_ib_cq_buf {</div><div class='hunk'>@@ -452,48 +558,9 @@ struct mlx5_ib_cq_buf {</div><div class='ctx'> 	int			nent;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-enum mlx5_ib_qp_flags {</div><div class='del'>-	MLX5_IB_QP_LSO                          = IB_QP_CREATE_IPOIB_UD_LSO,</div><div class='del'>-	MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK     = IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,</div><div class='del'>-	MLX5_IB_QP_CROSS_CHANNEL            = IB_QP_CREATE_CROSS_CHANNEL,</div><div class='del'>-	MLX5_IB_QP_MANAGED_SEND             = IB_QP_CREATE_MANAGED_SEND,</div><div class='del'>-	MLX5_IB_QP_MANAGED_RECV             = IB_QP_CREATE_MANAGED_RECV,</div><div class='del'>-	MLX5_IB_QP_SIGNATURE_HANDLING           = 1 &lt;&lt; 5,</div><div class='del'>-	/* QP uses 1 as its source QP number */</div><div class='del'>-	MLX5_IB_QP_SQPN_QP1			= 1 &lt;&lt; 6,</div><div class='del'>-	MLX5_IB_QP_CAP_SCATTER_FCS		= 1 &lt;&lt; 7,</div><div class='del'>-	MLX5_IB_QP_RSS				= 1 &lt;&lt; 8,</div><div class='del'>-	MLX5_IB_QP_CVLAN_STRIPPING		= 1 &lt;&lt; 9,</div><div class='del'>-	MLX5_IB_QP_UNDERLAY			= 1 &lt;&lt; 10,</div><div class='del'>-	MLX5_IB_QP_PCI_WRITE_END_PADDING	= 1 &lt;&lt; 11,</div><div class='del'>-	MLX5_IB_QP_TUNNEL_OFFLOAD		= 1 &lt;&lt; 12,</div><div class='del'>-	MLX5_IB_QP_PACKET_BASED_CREDIT		= 1 &lt;&lt; 13,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct mlx5_umr_wr {</div><div class='del'>-	struct ib_send_wr		wr;</div><div class='del'>-	u64				virt_addr;</div><div class='del'>-	u64				offset;</div><div class='del'>-	struct ib_pd		       *pd;</div><div class='del'>-	unsigned int			page_shift;</div><div class='del'>-	unsigned int			xlt_size;</div><div class='del'>-	u64				length;</div><div class='del'>-	int				access_flags;</div><div class='del'>-	u32				mkey;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static inline const struct mlx5_umr_wr *umr_wr(const struct ib_send_wr *wr)</div><div class='del'>-{</div><div class='del'>-	return container_of(wr, struct mlx5_umr_wr, wr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-struct mlx5_shared_mr_info {</div><div class='del'>-	int mr_id;</div><div class='del'>-	struct ib_umem		*umem;</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> enum mlx5_ib_cq_pr_flags {</div><div class='ctx'> 	MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD	= 1 &lt;&lt; 0,</div><div class='add'>+	MLX5_IB_CQ_PR_FLAGS_REAL_TIME_TS = 1 &lt;&lt; 1,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct mlx5_ib_cq {</div><div class='hunk'>@@ -556,17 +623,43 @@ enum mlx5_ib_mtt_access_flags {</div><div class='ctx'> 	MLX5_IB_MTT_WRITE = (1 &lt;&lt; 1),</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct mlx5_ib_dm {</div><div class='del'>-	struct ib_dm		ibdm;</div><div class='del'>-	phys_addr_t		dev_addr;</div><div class='del'>-	u32			type;</div><div class='del'>-	size_t			size;</div><div class='del'>-	union {</div><div class='del'>-		struct {</div><div class='del'>-			u32	obj_id;</div><div class='del'>-		} icm_dm;</div><div class='del'>-		/* other dm types specific params should be added here */</div><div class='del'>-	};</div><div class='add'>+struct mlx5_user_mmap_entry {</div><div class='add'>+	struct rdma_user_mmap_entry rdma_entry;</div><div class='add'>+	u8 mmap_flag;</div><div class='add'>+	u64 address;</div><div class='add'>+	u32 page_idx;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+enum mlx5_mkey_type {</div><div class='add'>+	MLX5_MKEY_MR = 1,</div><div class='add'>+	MLX5_MKEY_MW,</div><div class='add'>+	MLX5_MKEY_INDIRECT_DEVX,</div><div class='add'>+	MLX5_MKEY_NULL,</div><div class='add'>+	MLX5_MKEY_IMPLICIT_CHILD,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+/* Used for non-existent ph value */</div><div class='add'>+#define MLX5_IB_NO_PH 0xff</div><div class='add'>+</div><div class='add'>+struct mlx5r_cache_rb_key {</div><div class='add'>+	u8 ats:1;</div><div class='add'>+	u8 ph;</div><div class='add'>+	u16 st_index;</div><div class='add'>+	unsigned int access_mode;</div><div class='add'>+	unsigned int access_flags;</div><div class='add'>+	unsigned int ndescs;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mlx5_ib_mkey {</div><div class='add'>+	u32 key;</div><div class='add'>+	enum mlx5_mkey_type type;</div><div class='add'>+	unsigned int ndescs;</div><div class='add'>+	struct wait_queue_head wait;</div><div class='add'>+	refcount_t usecount;</div><div class='add'>+	/* Cacheable user Mkey must hold either a rb_key or a cache_ent. */</div><div class='add'>+	struct mlx5r_cache_rb_key rb_key;</div><div class='add'>+	struct mlx5_cache_ent *cache_ent;</div><div class='add'>+	u8 cacheable : 1;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> #define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)</div><div class='hunk'>@@ -582,33 +675,68 @@ struct mlx5_ib_dm {</div><div class='ctx'> 					  IB_ACCESS_REMOTE_READ   |\</div><div class='ctx'> 					  IB_ZERO_BASED)</div><div class='ctx'> </div><div class='add'>+#define mlx5_update_odp_stats(mr, counter_name, value)		\</div><div class='add'>+	atomic64_add(value, &amp;((mr)-&gt;odp_stats.counter_name))</div><div class='add'>+</div><div class='add'>+#define mlx5_update_odp_stats_with_handled(mr, counter_name, value)         \</div><div class='add'>+	do {                                                                \</div><div class='add'>+		mlx5_update_odp_stats(mr, counter_name, value);             \</div><div class='add'>+		atomic64_add(1, &amp;((mr)-&gt;odp_stats.counter_name##_handled)); \</div><div class='add'>+	} while (0)</div><div class='add'>+</div><div class='ctx'> struct mlx5_ib_mr {</div><div class='del'>-	struct ib_mr		ibmr;</div><div class='del'>-	void			*descs;</div><div class='del'>-	dma_addr_t		desc_map;</div><div class='del'>-	int			ndescs;</div><div class='del'>-	int			max_descs;</div><div class='del'>-	int			desc_size;</div><div class='del'>-	int			access_mode;</div><div class='del'>-	struct mlx5_core_mkey	mmkey;</div><div class='del'>-	struct ib_umem	       *umem;</div><div class='del'>-	struct mlx5_shared_mr_info	*smr_info;</div><div class='del'>-	struct list_head	list;</div><div class='del'>-	int			order;</div><div class='del'>-	bool			allocated_from_cache;</div><div class='del'>-	int			npages;</div><div class='del'>-	struct mlx5_ib_dev     *dev;</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(create_mkey_out)];</div><div class='del'>-	struct mlx5_core_sig_ctx    *sig;</div><div class='del'>-	int			live;</div><div class='del'>-	void			*descs_alloc;</div><div class='del'>-	int			access_flags; /* Needed for rereg MR */</div><div class='add'>+	struct ib_mr ibmr;</div><div class='add'>+	struct mlx5_ib_mkey mmkey;</div><div class='add'>+</div><div class='add'>+	struct ib_umem *umem;</div><div class='add'>+	/* The mr is data direct related */</div><div class='add'>+	u8 data_direct :1;</div><div class='ctx'> </div><div class='del'>-	struct mlx5_ib_mr      *parent;</div><div class='del'>-	atomic_t		num_leaf_free;</div><div class='del'>-	wait_queue_head_t       q_leaf_free;</div><div class='del'>-	struct mlx5_async_work  cb_work;</div><div class='del'>-	atomic_t		num_pending_prefetch;</div><div class='add'>+	union {</div><div class='add'>+		/* Used only by kernel MRs (umem == NULL) */</div><div class='add'>+		struct {</div><div class='add'>+			void *descs;</div><div class='add'>+			void *descs_alloc;</div><div class='add'>+			dma_addr_t desc_map;</div><div class='add'>+			int max_descs;</div><div class='add'>+			int desc_size;</div><div class='add'>+			int access_mode;</div><div class='add'>+</div><div class='add'>+			/* For Kernel IB_MR_TYPE_INTEGRITY */</div><div class='add'>+			struct mlx5_core_sig_ctx *sig;</div><div class='add'>+			struct mlx5_ib_mr *pi_mr;</div><div class='add'>+			struct mlx5_ib_mr *klm_mr;</div><div class='add'>+			struct mlx5_ib_mr *mtt_mr;</div><div class='add'>+			u64 data_iova;</div><div class='add'>+			u64 pi_iova;</div><div class='add'>+			int meta_ndescs;</div><div class='add'>+			int meta_length;</div><div class='add'>+			int data_length;</div><div class='add'>+		};</div><div class='add'>+</div><div class='add'>+		/* Used only by User MRs (umem != NULL) */</div><div class='add'>+		struct {</div><div class='add'>+			unsigned int page_shift;</div><div class='add'>+			/* Current access_flags */</div><div class='add'>+			int access_flags;</div><div class='add'>+</div><div class='add'>+			/* For User ODP */</div><div class='add'>+			struct mlx5_ib_mr *parent;</div><div class='add'>+			struct xarray implicit_children;</div><div class='add'>+			union {</div><div class='add'>+				struct work_struct work;</div><div class='add'>+			} odp_destroy;</div><div class='add'>+			struct ib_odp_counters odp_stats;</div><div class='add'>+			bool is_odp_implicit;</div><div class='add'>+			/* The affilated data direct crossed mr */</div><div class='add'>+			struct mlx5_ib_mr *dd_crossed_mr;</div><div class='add'>+			struct list_head dd_node;</div><div class='add'>+			u8 revoked :1;</div><div class='add'>+			/* Indicates previous dmabuf page fault occurred */</div><div class='add'>+			u8 dmabuf_faulted:1;</div><div class='add'>+			struct mlx5_ib_mkey null_mmkey;</div><div class='add'>+		};</div><div class='add'>+	};</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static inline bool is_odp_mr(struct mlx5_ib_mr *mr)</div><div class='hunk'>@@ -617,16 +745,15 @@ static inline bool is_odp_mr(struct mlx5_ib_mr *mr)</div><div class='ctx'> 	       mr-&gt;umem-&gt;is_odp;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static inline bool is_dmabuf_mr(struct mlx5_ib_mr *mr)</div><div class='add'>+{</div><div class='add'>+	return IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) &amp;&amp; mr-&gt;umem &amp;&amp;</div><div class='add'>+	       mr-&gt;umem-&gt;is_dmabuf;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> struct mlx5_ib_mw {</div><div class='ctx'> 	struct ib_mw		ibmw;</div><div class='del'>-	struct mlx5_core_mkey	mmkey;</div><div class='del'>-	int			ndescs;</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct mlx5_ib_devx_mr {</div><div class='del'>-	struct mlx5_core_mkey	mmkey;</div><div class='del'>-	int			ndescs;</div><div class='del'>-	struct rcu_head		rcu;</div><div class='add'>+	struct mlx5_ib_mkey	mmkey;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct mlx5_ib_umr_context {</div><div class='hunk'>@@ -635,84 +762,149 @@ struct mlx5_ib_umr_context {</div><div class='ctx'> 	struct completion	done;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+enum {</div><div class='add'>+	MLX5_UMR_STATE_UNINIT,</div><div class='add'>+	MLX5_UMR_STATE_ACTIVE,</div><div class='add'>+	MLX5_UMR_STATE_RECOVER,</div><div class='add'>+	MLX5_UMR_STATE_ERR,</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct umr_common {</div><div class='ctx'> 	struct ib_pd	*pd;</div><div class='ctx'> 	struct ib_cq	*cq;</div><div class='ctx'> 	struct ib_qp	*qp;</div><div class='del'>-	/* control access to UMR QP</div><div class='add'>+	/* Protects from UMR QP overflow</div><div class='ctx'> 	 */</div><div class='ctx'> 	struct semaphore	sem;</div><div class='add'>+	/* Protects from using UMR while the UMR is not active</div><div class='add'>+	 */</div><div class='add'>+	struct mutex lock;</div><div class='add'>+	unsigned int state;</div><div class='add'>+	/* Protects from repeat UMR QP creation */</div><div class='add'>+	struct mutex init_lock;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-enum {</div><div class='del'>-	MLX5_FMR_INVALID,</div><div class='del'>-	MLX5_FMR_VALID,</div><div class='del'>-	MLX5_FMR_BUSY,</div><div class='add'>+#define NUM_MKEYS_PER_PAGE \</div><div class='add'>+	((PAGE_SIZE - sizeof(struct list_head)) / sizeof(u32))</div><div class='add'>+</div><div class='add'>+struct mlx5_mkeys_page {</div><div class='add'>+	u32 mkeys[NUM_MKEYS_PER_PAGE];</div><div class='add'>+	struct list_head list;</div><div class='ctx'> };</div><div class='add'>+static_assert(sizeof(struct mlx5_mkeys_page) == PAGE_SIZE);</div><div class='ctx'> </div><div class='del'>-struct mlx5_cache_ent {</div><div class='del'>-	struct list_head	head;</div><div class='del'>-	/* sync access to the cahce entry</div><div class='del'>-	 */</div><div class='del'>-	spinlock_t		lock;</div><div class='add'>+struct mlx5_mkeys_queue {</div><div class='add'>+	struct list_head pages_list;</div><div class='add'>+	u32 num_pages;</div><div class='add'>+	unsigned long ci;</div><div class='add'>+	spinlock_t lock; /* sync list ops */</div><div class='add'>+};</div><div class='ctx'> </div><div class='add'>+struct mlx5_cache_ent {</div><div class='add'>+	struct mlx5_mkeys_queue	mkeys_queue;</div><div class='add'>+	u32			pending;</div><div class='ctx'> </div><div class='ctx'> 	char                    name[4];</div><div class='del'>-	u32                     order;</div><div class='del'>-	u32			xlt;</div><div class='del'>-	u32			access_mode;</div><div class='del'>-	u32			page;</div><div class='ctx'> </div><div class='del'>-	u32			size;</div><div class='del'>-	u32                     cur;</div><div class='add'>+	struct rb_node		node;</div><div class='add'>+	struct mlx5r_cache_rb_key rb_key;</div><div class='add'>+</div><div class='add'>+	u8 is_tmp:1;</div><div class='add'>+	u8 disabled:1;</div><div class='add'>+	u8 fill_to_high_water:1;</div><div class='add'>+	u8 tmp_cleanup_scheduled:1;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * - limit is the low water mark for stored mkeys, 2* limit is the</div><div class='add'>+	 *   upper water mark.</div><div class='add'>+	 */</div><div class='add'>+	u32 in_use;</div><div class='add'>+	u32 limit;</div><div class='add'>+</div><div class='add'>+	/* Statistics */</div><div class='ctx'> 	u32                     miss;</div><div class='del'>-	u32			limit;</div><div class='ctx'> </div><div class='ctx'> 	struct mlx5_ib_dev     *dev;</div><div class='del'>-	struct work_struct	work;</div><div class='ctx'> 	struct delayed_work	dwork;</div><div class='del'>-	int			pending;</div><div class='del'>-	struct completion	compl;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct mlx5_mr_cache {</div><div class='add'>+struct mlx5r_async_create_mkey {</div><div class='add'>+	union {</div><div class='add'>+		u32 in[MLX5_ST_SZ_BYTES(create_mkey_in)];</div><div class='add'>+		u32 out[MLX5_ST_SZ_DW(create_mkey_out)];</div><div class='add'>+	};</div><div class='add'>+	struct mlx5_async_work cb_work;</div><div class='add'>+	struct mlx5_cache_ent *ent;</div><div class='add'>+	u32 mkey;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mlx5_mkey_cache {</div><div class='ctx'> 	struct workqueue_struct *wq;</div><div class='del'>-	struct mlx5_cache_ent	ent[MAX_MR_CACHE_ENTRIES];</div><div class='del'>-	int			stopped;</div><div class='del'>-	struct dentry		*root;</div><div class='add'>+	struct rb_root		rb_root;</div><div class='add'>+	struct mutex		rb_lock;</div><div class='add'>+	struct dentry		*fs_root;</div><div class='ctx'> 	unsigned long		last_add;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct mlx5_ib_gsi_qp;</div><div class='del'>-</div><div class='ctx'> struct mlx5_ib_port_resources {</div><div class='del'>-	struct mlx5_ib_resources *devr;</div><div class='ctx'> 	struct mlx5_ib_gsi_qp *gsi;</div><div class='ctx'> 	struct work_struct pkey_change_work;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct mlx5_data_direct_resources {</div><div class='add'>+	u32 pdn;</div><div class='add'>+	u32 mkey;</div><div class='add'>+	u32 mkey_ro;</div><div class='add'>+	u8 mkey_ro_valid :1;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct mlx5_ib_resources {</div><div class='ctx'> 	struct ib_cq	*c0;</div><div class='del'>-	struct ib_xrcd	*x0;</div><div class='del'>-	struct ib_xrcd	*x1;</div><div class='add'>+	struct mutex cq_lock;</div><div class='add'>+	u32 xrcdn0;</div><div class='add'>+	u32 xrcdn1;</div><div class='ctx'> 	struct ib_pd	*p0;</div><div class='ctx'> 	struct ib_srq	*s0;</div><div class='ctx'> 	struct ib_srq	*s1;</div><div class='add'>+	struct mutex srq_lock;</div><div class='ctx'> 	struct mlx5_ib_port_resources ports[2];</div><div class='del'>-	/* Protects changes to the port resources */</div><div class='del'>-	struct mutex	mutex;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+#define MAX_OPFC_RULES 2</div><div class='add'>+</div><div class='add'>+struct mlx5_ib_op_fc {</div><div class='add'>+	struct mlx5_fc *fc;</div><div class='add'>+	struct mlx5_flow_handle *rule[MAX_OPFC_RULES];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct mlx5_ib_counters {</div><div class='del'>-	const char **names;</div><div class='add'>+	struct rdma_stat_desc *descs;</div><div class='ctx'> 	size_t *offsets;</div><div class='ctx'> 	u32 num_q_counters;</div><div class='ctx'> 	u32 num_cong_counters;</div><div class='ctx'> 	u32 num_ext_ppcnt_counters;</div><div class='add'>+	u32 num_op_counters;</div><div class='ctx'> 	u16 set_id;</div><div class='del'>-	bool set_id_valid;</div><div class='add'>+	struct mlx5_ib_op_fc opfcs[MLX5_IB_OPCOUNTER_MAX];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+int mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev *dev, u32 port_num,</div><div class='add'>+			 struct mlx5_ib_op_fc *opfc,</div><div class='add'>+			 enum mlx5_ib_optional_counter_type type);</div><div class='add'>+</div><div class='add'>+void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev *dev,</div><div class='add'>+			     struct mlx5_ib_op_fc *opfc,</div><div class='add'>+			     enum mlx5_ib_optional_counter_type type);</div><div class='add'>+</div><div class='add'>+int mlx5r_fs_bind_op_fc(struct ib_qp *qp,</div><div class='add'>+			struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX],</div><div class='add'>+			struct xarray *qpn_opfc_xa, u32 port);</div><div class='add'>+</div><div class='add'>+void mlx5r_fs_unbind_op_fc(struct ib_qp *qp, struct xarray *qpn_opfc_xa);</div><div class='add'>+</div><div class='add'>+void mlx5r_fs_destroy_fcs(struct mlx5_ib_dev *dev,</div><div class='add'>+			  struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX]);</div><div class='add'>+</div><div class='ctx'> struct mlx5_ib_multiport_info;</div><div class='ctx'> </div><div class='ctx'> struct mlx5_ib_multiport {</div><div class='hunk'>@@ -725,13 +917,14 @@ struct mlx5_roce {</div><div class='ctx'> 	/* Protect mlx5_ib_get_netdev from invoking dev_hold() with a NULL</div><div class='ctx'> 	 * netdev pointer</div><div class='ctx'> 	 */</div><div class='del'>-	rwlock_t		netdev_lock;</div><div class='del'>-	struct net_device	*netdev;</div><div class='ctx'> 	struct notifier_block	nb;</div><div class='add'>+	struct netdev_net_notifier nn;</div><div class='add'>+	struct notifier_block	mdev_nb;</div><div class='add'>+	struct net_device	*tracking_netdev;</div><div class='ctx'> 	atomic_t		tx_port_affinity;</div><div class='ctx'> 	enum ib_port_state last_port_state;</div><div class='ctx'> 	struct mlx5_ib_dev	*dev;</div><div class='del'>-	u8			native_port_num;</div><div class='add'>+	u32			native_port_num;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct mlx5_ib_port {</div><div class='hunk'>@@ -740,13 +933,16 @@ struct mlx5_ib_port {</div><div class='ctx'> 	struct mlx5_ib_dbg_cc_params *dbg_cc_params;</div><div class='ctx'> 	struct mlx5_roce roce;</div><div class='ctx'> 	struct mlx5_eswitch_rep		*rep;</div><div class='add'>+#ifdef CONFIG_MLX5_MACSEC</div><div class='add'>+	struct mlx5_reserved_gids *reserved_gids;</div><div class='add'>+#endif</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct mlx5_ib_dbg_param {</div><div class='ctx'> 	int			offset;</div><div class='ctx'> 	struct mlx5_ib_dev	*dev;</div><div class='ctx'> 	struct dentry		*dentry;</div><div class='del'>-	u8			port_num;</div><div class='add'>+	u32			port_num;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum mlx5_ib_dbg_cc_types {</div><div class='hunk'>@@ -756,6 +952,7 @@ enum mlx5_ib_dbg_cc_types {</div><div class='ctx'> 	MLX5_IB_DBG_CC_RP_BYTE_RESET,</div><div class='ctx'> 	MLX5_IB_DBG_CC_RP_THRESHOLD,</div><div class='ctx'> 	MLX5_IB_DBG_CC_RP_AI_RATE,</div><div class='add'>+	MLX5_IB_DBG_CC_RP_MAX_RATE,</div><div class='ctx'> 	MLX5_IB_DBG_CC_RP_HAI_RATE,</div><div class='ctx'> 	MLX5_IB_DBG_CC_RP_MIN_DEC_FAC,</div><div class='ctx'> 	MLX5_IB_DBG_CC_RP_MIN_RATE,</div><div class='hunk'>@@ -765,9 +962,12 @@ enum mlx5_ib_dbg_cc_types {</div><div class='ctx'> 	MLX5_IB_DBG_CC_RP_RATE_REDUCE_MONITOR_PERIOD,</div><div class='ctx'> 	MLX5_IB_DBG_CC_RP_INITIAL_ALPHA_VALUE,</div><div class='ctx'> 	MLX5_IB_DBG_CC_RP_GD,</div><div class='add'>+	MLX5_IB_DBG_CC_NP_MIN_TIME_BETWEEN_CNPS,</div><div class='ctx'> 	MLX5_IB_DBG_CC_NP_CNP_DSCP,</div><div class='ctx'> 	MLX5_IB_DBG_CC_NP_CNP_PRIO_MODE,</div><div class='ctx'> 	MLX5_IB_DBG_CC_NP_CNP_PRIO,</div><div class='add'>+	MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP_VALID,</div><div class='add'>+	MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP,</div><div class='ctx'> 	MLX5_IB_DBG_CC_MAX,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -780,13 +980,6 @@ enum {</div><div class='ctx'> 	MLX5_MAX_DELAY_DROP_TIMEOUT_MS = 100,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-struct mlx5_ib_dbg_delay_drop {</div><div class='del'>-	struct dentry		*dir_debugfs;</div><div class='del'>-	struct dentry		*rqs_cnt_debugfs;</div><div class='del'>-	struct dentry		*events_cnt_debugfs;</div><div class='del'>-	struct dentry		*timeout_debugfs;</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> struct mlx5_ib_delay_drop {</div><div class='ctx'> 	struct mlx5_ib_dev     *dev;</div><div class='ctx'> 	struct work_struct	delay_drop_work;</div><div class='hunk'>@@ -796,29 +989,29 @@ struct mlx5_ib_delay_drop {</div><div class='ctx'> 	bool			activate;</div><div class='ctx'> 	atomic_t		events_cnt;</div><div class='ctx'> 	atomic_t		rqs_cnt;</div><div class='del'>-	struct mlx5_ib_dbg_delay_drop *dbg;</div><div class='add'>+	struct dentry		*dir_debugfs;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> enum mlx5_ib_stages {</div><div class='ctx'> 	MLX5_IB_STAGE_INIT,</div><div class='del'>-	MLX5_IB_STAGE_FLOW_DB,</div><div class='add'>+	MLX5_IB_STAGE_FS,</div><div class='ctx'> 	MLX5_IB_STAGE_CAPS,</div><div class='ctx'> 	MLX5_IB_STAGE_NON_DEFAULT_CB,</div><div class='ctx'> 	MLX5_IB_STAGE_ROCE,</div><div class='add'>+	MLX5_IB_STAGE_QP,</div><div class='ctx'> 	MLX5_IB_STAGE_SRQ,</div><div class='ctx'> 	MLX5_IB_STAGE_DEVICE_RESOURCES,</div><div class='del'>-	MLX5_IB_STAGE_DEVICE_NOTIFIER,</div><div class='ctx'> 	MLX5_IB_STAGE_ODP,</div><div class='ctx'> 	MLX5_IB_STAGE_COUNTERS,</div><div class='ctx'> 	MLX5_IB_STAGE_CONG_DEBUGFS,</div><div class='del'>-	MLX5_IB_STAGE_UAR,</div><div class='ctx'> 	MLX5_IB_STAGE_BFREG,</div><div class='ctx'> 	MLX5_IB_STAGE_PRE_IB_REG_UMR,</div><div class='ctx'> 	MLX5_IB_STAGE_WHITELIST_UID,</div><div class='ctx'> 	MLX5_IB_STAGE_IB_REG,</div><div class='add'>+	MLX5_IB_STAGE_DEVICE_NOTIFIER,</div><div class='ctx'> 	MLX5_IB_STAGE_POST_IB_REG_UMR,</div><div class='ctx'> 	MLX5_IB_STAGE_DELAY_DROP,</div><div class='del'>-	MLX5_IB_STAGE_CLASS_ATTR,</div><div class='add'>+	MLX5_IB_STAGE_RESTRACK,</div><div class='ctx'> 	MLX5_IB_STAGE_MAX,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='hunk'>@@ -856,7 +1049,10 @@ struct mlx5_ib_flow_action {</div><div class='ctx'> 		struct {</div><div class='ctx'> 			struct mlx5_ib_dev *dev;</div><div class='ctx'> 			u32 sub_type;</div><div class='del'>-			u32 action_id;</div><div class='add'>+			union {</div><div class='add'>+				struct mlx5_modify_hdr *modify_hdr;</div><div class='add'>+				struct mlx5_pkt_reformat *pkt_reformat;</div><div class='add'>+			};</div><div class='ctx'> 		} flow_action_raw;</div><div class='ctx'> 	};</div><div class='ctx'> };</div><div class='hunk'>@@ -869,8 +1065,6 @@ struct mlx5_dm {</div><div class='ctx'> 	 */</div><div class='ctx'> 	spinlock_t lock;</div><div class='ctx'> 	DECLARE_BITMAP(memic_alloc_pages, MLX5_MAX_MEMIC_PAGES);</div><div class='del'>-	unsigned long *steering_sw_icm_alloc_blocks;</div><div class='del'>-	unsigned long *header_modify_sw_icm_alloc_blocks;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct mlx5_read_counters_attr {</div><div class='hunk'>@@ -917,6 +1111,7 @@ struct mlx5_ib_lb_state {</div><div class='ctx'> 	u32			user_td;</div><div class='ctx'> 	int			qps;</div><div class='ctx'> 	bool			enabled;</div><div class='add'>+	bool			force_enable;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> struct mlx5_ib_pf_eq {</div><div class='hunk'>@@ -929,46 +1124,84 @@ struct mlx5_ib_pf_eq {</div><div class='ctx'> 	mempool_t *pool;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct mlx5_devx_event_table {</div><div class='add'>+	struct mlx5_nb devx_nb;</div><div class='add'>+	/* serialize updating the event_xa */</div><div class='add'>+	struct mutex event_xa_lock;</div><div class='add'>+	struct xarray event_xa;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mlx5_var_table {</div><div class='add'>+	/* serialize updating the bitmap */</div><div class='add'>+	struct mutex bitmap_lock;</div><div class='add'>+	unsigned long *bitmap;</div><div class='add'>+	u64 hw_start_addr;</div><div class='add'>+	u32 stride_size;</div><div class='add'>+	u64 num_var_hw_entries;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mlx5_port_caps {</div><div class='add'>+	bool has_smi;</div><div class='add'>+	u8 ext_port_cap;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+</div><div class='add'>+struct mlx5_special_mkeys {</div><div class='add'>+	u32 dump_fill_mkey;</div><div class='add'>+	__be32 null_mkey;</div><div class='add'>+	__be32 terminate_scatter_list_mkey;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+struct mlx5_macsec {</div><div class='add'>+	struct mutex lock; /* Protects mlx5_macsec internal contexts */</div><div class='add'>+	struct list_head macsec_devices_list;</div><div class='add'>+	struct notifier_block blocking_events_nb;</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct mlx5_ib_dev {</div><div class='ctx'> 	struct ib_device		ib_dev;</div><div class='ctx'> 	struct mlx5_core_dev		*mdev;</div><div class='add'>+	struct mlx5_data_direct_dev	*data_direct_dev;</div><div class='add'>+	/* protect accessing data_direct_dev */</div><div class='add'>+	struct mutex			data_direct_lock;</div><div class='ctx'> 	struct notifier_block		mdev_events;</div><div class='add'>+	struct notifier_block           lag_events;</div><div class='ctx'> 	int				num_ports;</div><div class='ctx'> 	/* serialize update of capability mask</div><div class='ctx'> 	 */</div><div class='ctx'> 	struct mutex			cap_mask_mutex;</div><div class='del'>-	bool				ib_active;</div><div class='add'>+	u8				ib_active:1;</div><div class='add'>+	u8				is_rep:1;</div><div class='add'>+	u8				lag_active:1;</div><div class='add'>+	u8				fill_delay;</div><div class='ctx'> 	struct umr_common		umrc;</div><div class='ctx'> 	/* sync used page count stats</div><div class='ctx'> 	 */</div><div class='ctx'> 	struct mlx5_ib_resources	devr;</div><div class='del'>-	struct mlx5_mr_cache		cache;</div><div class='add'>+</div><div class='add'>+	atomic_t			mkey_var;</div><div class='add'>+	struct mlx5_mkey_cache		cache;</div><div class='ctx'> 	struct timer_list		delay_timer;</div><div class='ctx'> 	/* Prevents soft lock on massive reg MRs */</div><div class='ctx'> 	struct mutex			slow_path_mutex;</div><div class='del'>-	int				fill_delay;</div><div class='ctx'> 	struct ib_odp_caps	odp_caps;</div><div class='ctx'> 	u64			odp_max_size;</div><div class='add'>+	struct mutex		odp_eq_mutex;</div><div class='ctx'> 	struct mlx5_ib_pf_eq	odp_pf_eq;</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * Sleepable RCU that prevents destruction of MRs while they are still</div><div class='del'>-	 * being used by a page fault handler.</div><div class='del'>-	 */</div><div class='del'>-	struct srcu_struct      mr_srcu;</div><div class='del'>-	u32			null_mkey;</div><div class='add'>+	struct xarray		odp_mkeys;</div><div class='add'>+</div><div class='ctx'> 	struct mlx5_ib_flow_db	*flow_db;</div><div class='ctx'> 	/* protect resources needed as part of reset flow */</div><div class='ctx'> 	spinlock_t		reset_flow_resource_lock;</div><div class='ctx'> 	struct list_head	qp_list;</div><div class='add'>+	struct list_head data_direct_mr_list;</div><div class='ctx'> 	/* Array with num_ports elements */</div><div class='ctx'> 	struct mlx5_ib_port	*port;</div><div class='ctx'> 	struct mlx5_sq_bfreg	bfreg;</div><div class='ctx'> 	struct mlx5_sq_bfreg	fp_bfreg;</div><div class='ctx'> 	struct mlx5_ib_delay_drop	delay_drop;</div><div class='ctx'> 	const struct mlx5_ib_profile	*profile;</div><div class='del'>-	bool			is_rep;</div><div class='del'>-	int				lag_active;</div><div class='ctx'> </div><div class='ctx'> 	struct mlx5_ib_lb_state		lb;</div><div class='ctx'> 	u8			umr_fence;</div><div class='hunk'>@@ -977,8 +1210,25 @@ struct mlx5_ib_dev {</div><div class='ctx'> 	struct mlx5_dm		dm;</div><div class='ctx'> 	u16			devx_whitelist_uid;</div><div class='ctx'> 	struct mlx5_srq_table   srq_table;</div><div class='add'>+	struct mlx5_qp_table    qp_table;</div><div class='ctx'> 	struct mlx5_async_ctx   async_ctx;</div><div class='del'>-	int			free_port;</div><div class='add'>+	struct mlx5_devx_event_table devx_event_table;</div><div class='add'>+	struct mlx5_var_table var_table;</div><div class='add'>+</div><div class='add'>+	struct xarray sig_mrs;</div><div class='add'>+	struct mlx5_port_caps port_caps[MLX5_MAX_PORTS];</div><div class='add'>+	u16 pkey_table_len;</div><div class='add'>+	u8 lag_ports;</div><div class='add'>+	struct mlx5_special_mkeys mkeys;</div><div class='add'>+	struct mlx5_data_direct_resources ddr;</div><div class='add'>+</div><div class='add'>+#ifdef CONFIG_MLX5_MACSEC</div><div class='add'>+	struct mlx5_macsec macsec;</div><div class='add'>+#endif</div><div class='add'>+</div><div class='add'>+	u8 num_plane;</div><div class='add'>+	struct mlx5_ib_dev *smi_dev;</div><div class='add'>+	const char *sub_dev_name;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)</div><div class='hunk'>@@ -996,6 +1246,11 @@ static inline struct mlx5_ib_dev *to_mdev(struct ib_device *ibdev)</div><div class='ctx'> 	return container_of(ibdev, struct mlx5_ib_dev, ib_dev);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static inline struct mlx5_ib_dev *mr_to_mdev(struct mlx5_ib_mr *mr)</div><div class='add'>+{</div><div class='add'>+	return to_mdev(mr-&gt;ibmr.device);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static inline struct mlx5_ib_dev *mlx5_udata_to_mdev(struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(</div><div class='hunk'>@@ -1019,11 +1274,6 @@ static inline struct mlx5_ib_rwq *to_mibrwq(struct mlx5_core_qp *core_qp)</div><div class='ctx'> 	return container_of(core_qp, struct mlx5_ib_rwq, core_qp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mkey *mmkey)</div><div class='del'>-{</div><div class='del'>-	return container_of(mmkey, struct mlx5_ib_mr, mmkey);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd)</div><div class='ctx'> {</div><div class='ctx'> 	return container_of(ibpd, struct mlx5_ib_pd, ibpd);</div><div class='hunk'>@@ -1054,11 +1304,6 @@ static inline struct mlx5_ib_srq *to_mibsrq(struct mlx5_core_srq *msrq)</div><div class='ctx'> 	return container_of(msrq, struct mlx5_ib_srq, msrq);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline struct mlx5_ib_dm *to_mdm(struct ib_dm *ibdm)</div><div class='del'>-{</div><div class='del'>-	return container_of(ibdm, struct mlx5_ib_dm, ibdm);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static inline struct mlx5_ib_mr *to_mmr(struct ib_mr *ibmr)</div><div class='ctx'> {</div><div class='ctx'> 	return container_of(ibmr, struct mlx5_ib_mr, ibmr);</div><div class='hunk'>@@ -1075,30 +1320,40 @@ to_mflow_act(struct ib_flow_action *ibact)</div><div class='ctx'> 	return container_of(ibact, struct mlx5_ib_flow_action, ib_action);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context,</div><div class='del'>-			struct ib_udata *udata, unsigned long virt,</div><div class='add'>+static inline struct mlx5_user_mmap_entry *</div><div class='add'>+to_mmmap(struct rdma_user_mmap_entry *rdma_entry)</div><div class='add'>+{</div><div class='add'>+	return container_of(rdma_entry,</div><div class='add'>+		struct mlx5_user_mmap_entry, rdma_entry);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_ib_dev_res_cq_init(struct mlx5_ib_dev *dev);</div><div class='add'>+int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev);</div><div class='add'>+int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt,</div><div class='ctx'> 			struct mlx5_db *db);</div><div class='ctx'> void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db);</div><div class='ctx'> void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);</div><div class='ctx'> void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);</div><div class='ctx'> void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index);</div><div class='del'>-int mlx5_ib_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,</div><div class='add'>+int mlx5_ib_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr,</div><div class='ctx'> 		      struct ib_udata *udata);</div><div class='ctx'> int mlx5_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);</div><div class='del'>-void mlx5_ib_destroy_ah(struct ib_ah *ah, u32 flags);</div><div class='add'>+static inline int mlx5_ib_destroy_ah(struct ib_ah *ah, u32 flags)</div><div class='add'>+{</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> int mlx5_ib_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *init_attr,</div><div class='ctx'> 		       struct ib_udata *udata);</div><div class='ctx'> int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,</div><div class='ctx'> 		       enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);</div><div class='ctx'> int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr);</div><div class='del'>-void mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata);</div><div class='add'>+int mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata);</div><div class='ctx'> int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,</div><div class='ctx'> 			  const struct ib_recv_wr **bad_wr);</div><div class='ctx'> int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp);</div><div class='ctx'> void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp);</div><div class='del'>-struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,</div><div class='del'>-				struct ib_qp_init_attr *init_attr,</div><div class='del'>-				struct ib_udata *udata);</div><div class='add'>+int mlx5_ib_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr,</div><div class='add'>+		      struct ib_udata *udata);</div><div class='ctx'> int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> 		      int attr_mask, struct ib_udata *udata);</div><div class='ctx'> int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,</div><div class='hunk'>@@ -1106,63 +1361,64 @@ int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr</div><div class='ctx'> int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata);</div><div class='ctx'> void mlx5_ib_drain_sq(struct ib_qp *qp);</div><div class='ctx'> void mlx5_ib_drain_rq(struct ib_qp *qp);</div><div class='del'>-int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,</div><div class='del'>-		      const struct ib_send_wr **bad_wr);</div><div class='del'>-int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,</div><div class='del'>-		      const struct ib_recv_wr **bad_wr);</div><div class='del'>-int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,</div><div class='del'>-			     int buflen, size_t *bc);</div><div class='del'>-int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,</div><div class='del'>-			     int buflen, size_t *bc);</div><div class='del'>-int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index,</div><div class='del'>-			      void *buffer, int buflen, size_t *bc);</div><div class='del'>-struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,</div><div class='del'>-				const struct ib_cq_init_attr *attr,</div><div class='del'>-				struct ib_udata *udata);</div><div class='add'>+int mlx5_ib_read_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,</div><div class='add'>+			size_t buflen, size_t *bc);</div><div class='add'>+int mlx5_ib_read_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,</div><div class='add'>+			size_t buflen, size_t *bc);</div><div class='add'>+int mlx5_ib_read_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index, void *buffer,</div><div class='add'>+			 size_t buflen, size_t *bc);</div><div class='add'>+int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,</div><div class='add'>+		      struct uverbs_attr_bundle *attrs);</div><div class='ctx'> int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);</div><div class='ctx'> int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);</div><div class='add'>+int mlx5_ib_pre_destroy_cq(struct ib_cq *cq);</div><div class='add'>+void mlx5_ib_post_destroy_cq(struct ib_cq *cq);</div><div class='ctx'> int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);</div><div class='ctx'> int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);</div><div class='ctx'> int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);</div><div class='ctx'> struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc);</div><div class='ctx'> struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='ctx'> 				  u64 virt_addr, int access_flags,</div><div class='add'>+				  struct ib_dmah *dmah,</div><div class='ctx'> 				  struct ib_udata *udata);</div><div class='add'>+struct ib_mr *mlx5_ib_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start,</div><div class='add'>+					 u64 length, u64 virt_addr,</div><div class='add'>+					 int fd, int access_flags,</div><div class='add'>+					 struct ib_dmah *dmah,</div><div class='add'>+					 struct uverbs_attr_bundle *attrs);</div><div class='ctx'> int mlx5_ib_advise_mr(struct ib_pd *pd,</div><div class='ctx'> 		      enum ib_uverbs_advise_mr_advice advice,</div><div class='ctx'> 		      u32 flags,</div><div class='ctx'> 		      struct ib_sge *sg_list,</div><div class='ctx'> 		      u32 num_sge,</div><div class='ctx'> 		      struct uverbs_attr_bundle *attrs);</div><div class='del'>-struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,</div><div class='del'>-			       struct ib_udata *udata);</div><div class='add'>+int mlx5_ib_alloc_mw(struct ib_mw *mw, struct ib_udata *udata);</div><div class='ctx'> int mlx5_ib_dealloc_mw(struct ib_mw *mw);</div><div class='del'>-int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,</div><div class='del'>-		       int page_shift, int flags);</div><div class='ctx'> struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd,</div><div class='del'>-					     struct ib_udata *udata,</div><div class='ctx'> 					     int access_flags);</div><div class='del'>-void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *mr);</div><div class='del'>-int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,</div><div class='del'>-			  u64 length, u64 virt_addr, int access_flags,</div><div class='del'>-			  struct ib_pd *pd, struct ib_udata *udata);</div><div class='add'>+void mlx5_ib_free_odp_mr(struct mlx5_ib_mr *mr);</div><div class='add'>+struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,</div><div class='add'>+				    u64 length, u64 virt_addr, int access_flags,</div><div class='add'>+				    struct ib_pd *pd, struct ib_udata *udata);</div><div class='ctx'> int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);</div><div class='ctx'> struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,</div><div class='del'>-			       u32 max_num_sg, struct ib_udata *udata);</div><div class='add'>+			       u32 max_num_sg);</div><div class='add'>+struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd,</div><div class='add'>+					 u32 max_num_sg,</div><div class='add'>+					 u32 max_num_meta_sg);</div><div class='ctx'> int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,</div><div class='ctx'> 		      unsigned int *sg_offset);</div><div class='del'>-int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,</div><div class='add'>+int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,</div><div class='add'>+			 int data_sg_nents, unsigned int *data_sg_offset,</div><div class='add'>+			 struct scatterlist *meta_sg, int meta_sg_nents,</div><div class='add'>+			 unsigned int *meta_sg_offset);</div><div class='add'>+int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num,</div><div class='ctx'> 			const struct ib_wc *in_wc, const struct ib_grh *in_grh,</div><div class='del'>-			const struct ib_mad_hdr *in, size_t in_mad_size,</div><div class='del'>-			struct ib_mad_hdr *out, size_t *out_mad_size,</div><div class='del'>-			u16 *out_mad_pkey_index);</div><div class='del'>-struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,</div><div class='del'>-				   struct ib_udata *udata);</div><div class='add'>+			const struct ib_mad *in, struct ib_mad *out,</div><div class='add'>+			size_t *out_mad_size, u16 *out_mad_pkey_index);</div><div class='add'>+int mlx5_ib_alloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);</div><div class='ctx'> int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);</div><div class='del'>-int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset);</div><div class='del'>-int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port);</div><div class='del'>-int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev,</div><div class='del'>-					  struct ib_smp *out_mad);</div><div class='add'>+int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, unsigned int port);</div><div class='ctx'> int mlx5_query_mad_ifc_system_image_guid(struct ib_device *ibdev,</div><div class='ctx'> 					 __be64 *sys_image_guid);</div><div class='ctx'> int mlx5_query_mad_ifc_max_pkeys(struct ib_device *ibdev,</div><div class='hunk'>@@ -1171,32 +1427,28 @@ int mlx5_query_mad_ifc_vendor_id(struct ib_device *ibdev,</div><div class='ctx'> 				 u32 *vendor_id);</div><div class='ctx'> int mlx5_query_mad_ifc_node_desc(struct mlx5_ib_dev *dev, char *node_desc);</div><div class='ctx'> int mlx5_query_mad_ifc_node_guid(struct mlx5_ib_dev *dev, __be64 *node_guid);</div><div class='del'>-int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u8 port, u16 index,</div><div class='add'>+int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u32 port, u16 index,</div><div class='ctx'> 			    u16 *pkey);</div><div class='del'>-int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u8 port, int index,</div><div class='add'>+int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u32 port, int index,</div><div class='ctx'> 			    union ib_gid *gid);</div><div class='del'>-int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,</div><div class='add'>+int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 			    struct ib_port_attr *props);</div><div class='del'>-int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,</div><div class='add'>+int mlx5_ib_query_port(struct ib_device *ibdev, u32 port,</div><div class='ctx'> 		       struct ib_port_attr *props);</div><div class='del'>-int mlx5_ib_init_fmr(struct mlx5_ib_dev *dev);</div><div class='del'>-void mlx5_ib_cleanup_fmr(struct mlx5_ib_dev *dev);</div><div class='del'>-void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,</div><div class='del'>-			unsigned long max_page_shift,</div><div class='del'>-			int *count, int *shift,</div><div class='del'>-			int *ncont, int *order);</div><div class='del'>-void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,</div><div class='del'>-			    int page_shift, size_t offset, size_t num_pages,</div><div class='del'>-			    __be64 *pas, int access_flags);</div><div class='del'>-void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,</div><div class='del'>-			  int page_shift, __be64 *pas, int access_flags);</div><div class='del'>-void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num);</div><div class='add'>+void mlx5_ib_populate_pas(struct ib_umem *umem, size_t page_size, __be64 *pas,</div><div class='add'>+			  u64 access_flags);</div><div class='ctx'> int mlx5_ib_get_cqe_size(struct ib_cq *ibcq);</div><div class='del'>-int mlx5_mr_cache_init(struct mlx5_ib_dev *dev);</div><div class='del'>-int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev);</div><div class='add'>+int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev);</div><div class='add'>+void mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev);</div><div class='add'>+struct mlx5_cache_ent *</div><div class='add'>+mlx5r_cache_create_ent_locked(struct mlx5_ib_dev *dev,</div><div class='add'>+			      struct mlx5r_cache_rb_key rb_key,</div><div class='add'>+			      bool persistent_entry);</div><div class='add'>+</div><div class='add'>+struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,</div><div class='add'>+				       int access_flags, int access_mode,</div><div class='add'>+				       int ndescs);</div><div class='ctx'> </div><div class='del'>-struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry);</div><div class='del'>-void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);</div><div class='ctx'> int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,</div><div class='ctx'> 			    struct ib_mr_status *mr_status);</div><div class='ctx'> struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,</div><div class='hunk'>@@ -1205,49 +1457,52 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,</div><div class='ctx'> int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata);</div><div class='ctx'> int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,</div><div class='ctx'> 		      u32 wq_attr_mask, struct ib_udata *udata);</div><div class='del'>-struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,</div><div class='del'>-						      struct ib_rwq_ind_table_init_attr *init_attr,</div><div class='del'>-						      struct ib_udata *udata);</div><div class='add'>+int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,</div><div class='add'>+				 struct ib_rwq_ind_table_init_attr *init_attr,</div><div class='add'>+				 struct ib_udata *udata);</div><div class='ctx'> int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);</div><div class='del'>-bool mlx5_ib_dc_atomic_is_supported(struct mlx5_ib_dev *dev);</div><div class='del'>-struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,</div><div class='del'>-			       struct ib_ucontext *context,</div><div class='del'>-			       struct ib_dm_alloc_attr *attr,</div><div class='del'>-			       struct uverbs_attr_bundle *attrs);</div><div class='del'>-int mlx5_ib_dealloc_dm(struct ib_dm *ibdm, struct uverbs_attr_bundle *attrs);</div><div class='ctx'> struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,</div><div class='ctx'> 				struct ib_dm_mr_attr *attr,</div><div class='ctx'> 				struct uverbs_attr_bundle *attrs);</div><div class='add'>+void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev,</div><div class='add'>+			      struct mlx5_data_direct_dev *dev);</div><div class='add'>+void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev);</div><div class='add'>+void mlx5_ib_revoke_data_direct_mrs(struct mlx5_ib_dev *dev);</div><div class='ctx'> </div><div class='ctx'> #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING</div><div class='del'>-void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev);</div><div class='ctx'> int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev);</div><div class='add'>+int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq);</div><div class='ctx'> void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *ibdev);</div><div class='ctx'> int __init mlx5_ib_odp_init(void);</div><div class='ctx'> void mlx5_ib_odp_cleanup(void);</div><div class='del'>-void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start,</div><div class='del'>-			      unsigned long end);</div><div class='del'>-void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent);</div><div class='del'>-void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset,</div><div class='del'>-			   size_t nentries, struct mlx5_ib_mr *mr, int flags);</div><div class='add'>+int mlx5_odp_init_mkey_cache(struct mlx5_ib_dev *dev);</div><div class='add'>+int mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries,</div><div class='add'>+			  struct mlx5_ib_mr *mr, int flags);</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,</div><div class='ctx'> 			       enum ib_uverbs_advise_mr_advice advice,</div><div class='ctx'> 			       u32 flags, struct ib_sge *sg_list, u32 num_sge);</div><div class='add'>+int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr);</div><div class='add'>+int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr);</div><div class='ctx'> #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */</div><div class='del'>-static inline void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev)</div><div class='add'>+static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; }</div><div class='add'>+static inline int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev,</div><div class='add'>+				      struct mlx5_ib_pf_eq *eq)</div><div class='ctx'> {</div><div class='del'>-	return;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='del'>-</div><div class='del'>-static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; }</div><div class='ctx'> static inline void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *ibdev) {}</div><div class='ctx'> static inline int mlx5_ib_odp_init(void) { return 0; }</div><div class='ctx'> static inline void mlx5_ib_odp_cleanup(void)				    {}</div><div class='del'>-static inline void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent) {}</div><div class='del'>-static inline void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset,</div><div class='del'>-					 size_t nentries, struct mlx5_ib_mr *mr,</div><div class='del'>-					 int flags) {}</div><div class='add'>+static inline int mlx5_odp_init_mkey_cache(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+static inline int mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries,</div><div class='add'>+					struct mlx5_ib_mr *mr, int flags)</div><div class='add'>+{</div><div class='add'>+	return -EOPNOTSUPP;</div><div class='add'>+}</div><div class='ctx'> </div><div class='ctx'> static inline int</div><div class='ctx'> mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,</div><div class='hunk'>@@ -1256,37 +1511,47 @@ mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,</div><div class='ctx'> {</div><div class='ctx'> 	return -EOPNOTSUPP;</div><div class='ctx'> }</div><div class='del'>-static inline void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp,</div><div class='del'>-					    unsigned long start,</div><div class='del'>-					    unsigned long end){};</div><div class='add'>+static inline int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr)</div><div class='add'>+{</div><div class='add'>+	return -EOPNOTSUPP;</div><div class='add'>+}</div><div class='add'>+static inline int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr)</div><div class='add'>+{</div><div class='add'>+	return -EOPNOTSUPP;</div><div class='add'>+}</div><div class='ctx'> #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */</div><div class='ctx'> </div><div class='add'>+extern const struct mmu_interval_notifier_ops mlx5_mn_ops;</div><div class='add'>+</div><div class='ctx'> /* Needed for rep profile */</div><div class='ctx'> void __mlx5_ib_remove(struct mlx5_ib_dev *dev,</div><div class='ctx'> 		      const struct mlx5_ib_profile *profile,</div><div class='ctx'> 		      int stage);</div><div class='del'>-void *__mlx5_ib_add(struct mlx5_ib_dev *dev,</div><div class='del'>-		    const struct mlx5_ib_profile *profile);</div><div class='add'>+int __mlx5_ib_add(struct mlx5_ib_dev *dev,</div><div class='add'>+		  const struct mlx5_ib_profile *profile);</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_get_vf_config(struct ib_device *device, int vf,</div><div class='del'>-			  u8 port, struct ifla_vf_info *info);</div><div class='add'>+			  u32 port, struct ifla_vf_info *info);</div><div class='ctx'> int mlx5_ib_set_vf_link_state(struct ib_device *device, int vf,</div><div class='del'>-			      u8 port, int state);</div><div class='add'>+			      u32 port, int state);</div><div class='ctx'> int mlx5_ib_get_vf_stats(struct ib_device *device, int vf,</div><div class='del'>-			 u8 port, struct ifla_vf_stats *stats);</div><div class='del'>-int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u8 port,</div><div class='add'>+			 u32 port, struct ifla_vf_stats *stats);</div><div class='add'>+int mlx5_ib_get_vf_guid(struct ib_device *device, int vf, u32 port,</div><div class='add'>+			struct ifla_vf_guid *node_guid,</div><div class='add'>+			struct ifla_vf_guid *port_guid);</div><div class='add'>+int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u32 port,</div><div class='ctx'> 			u64 guid, int type);</div><div class='ctx'> </div><div class='del'>-__be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev,</div><div class='del'>-			       const struct ib_gid_attr *attr);</div><div class='add'>+__be16 mlx5_get_roce_udp_sport_min(const struct mlx5_ib_dev *dev,</div><div class='add'>+				   const struct ib_gid_attr *attr);</div><div class='ctx'> </div><div class='del'>-void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num);</div><div class='del'>-void mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num);</div><div class='add'>+void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev, u32 port_num);</div><div class='add'>+void mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev, u32 port_num);</div><div class='ctx'> </div><div class='ctx'> /* GSI QP helper functions */</div><div class='del'>-struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd,</div><div class='del'>-				    struct ib_qp_init_attr *init_attr);</div><div class='del'>-int mlx5_ib_gsi_destroy_qp(struct ib_qp *qp);</div><div class='add'>+int mlx5_ib_create_gsi(struct ib_pd *pd, struct mlx5_ib_qp *mqp,</div><div class='add'>+		       struct ib_qp_init_attr *attr);</div><div class='add'>+int mlx5_ib_destroy_gsi(struct mlx5_ib_qp *mqp);</div><div class='ctx'> int mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,</div><div class='ctx'> 			  int attr_mask);</div><div class='ctx'> int mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,</div><div class='hunk'>@@ -1304,66 +1569,22 @@ void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi,</div><div class='ctx'> 			int bfregn);</div><div class='ctx'> struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi);</div><div class='ctx'> struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *dev,</div><div class='del'>-						   u8 ib_port_num,</div><div class='del'>-						   u8 *native_port_num);</div><div class='add'>+						   u32 ib_port_num,</div><div class='add'>+						   u32 *native_port_num);</div><div class='ctx'> void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *dev,</div><div class='del'>-				  u8 port_num);</div><div class='add'>+				  u32 port_num);</div><div class='ctx'> </div><div class='del'>-#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)</div><div class='del'>-int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user);</div><div class='del'>-void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid);</div><div class='del'>-const struct uverbs_object_tree_def *mlx5_ib_get_devx_tree(void);</div><div class='ctx'> extern const struct uapi_definition mlx5_ib_devx_defs[];</div><div class='ctx'> extern const struct uapi_definition mlx5_ib_flow_defs[];</div><div class='del'>-struct mlx5_ib_flow_handler *mlx5_ib_raw_fs_rule_add(</div><div class='del'>-	struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher,</div><div class='del'>-	struct mlx5_flow_act *flow_act, u32 counter_id,</div><div class='del'>-	void *cmd_in, int inlen, int dest_id, int dest_type);</div><div class='del'>-bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type);</div><div class='del'>-bool mlx5_ib_devx_is_flow_counter(void *obj, u32 *counter_id);</div><div class='del'>-int mlx5_ib_get_flow_trees(const struct uverbs_object_tree_def **root);</div><div class='del'>-void mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction);</div><div class='del'>-#else</div><div class='del'>-static inline int</div><div class='del'>-mlx5_ib_devx_create(struct mlx5_ib_dev *dev,</div><div class='del'>-			   bool is_user) { return -EOPNOTSUPP; }</div><div class='del'>-static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) {}</div><div class='del'>-static inline bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id,</div><div class='del'>-					     int *dest_type)</div><div class='del'>-{</div><div class='del'>-	return false;</div><div class='del'>-}</div><div class='del'>-static inline void</div><div class='del'>-mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)</div><div class='del'>-{</div><div class='del'>-	return;</div><div class='del'>-};</div><div class='del'>-#endif</div><div class='del'>-static inline void init_query_mad(struct ib_smp *mad)</div><div class='del'>-{</div><div class='del'>-	mad-&gt;base_version  = 1;</div><div class='del'>-	mad-&gt;mgmt_class    = IB_MGMT_CLASS_SUBN_LID_ROUTED;</div><div class='del'>-	mad-&gt;class_version = 1;</div><div class='del'>-	mad-&gt;method	   = IB_MGMT_METHOD_GET;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline u8 convert_access(int acc)</div><div class='del'>-{</div><div class='del'>-	return (acc &amp; IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC       : 0) |</div><div class='del'>-	       (acc &amp; IB_ACCESS_REMOTE_WRITE  ? MLX5_PERM_REMOTE_WRITE : 0) |</div><div class='del'>-	       (acc &amp; IB_ACCESS_REMOTE_READ   ? MLX5_PERM_REMOTE_READ  : 0) |</div><div class='del'>-	       (acc &amp; IB_ACCESS_LOCAL_WRITE   ? MLX5_PERM_LOCAL_WRITE  : 0) |</div><div class='del'>-	       MLX5_PERM_LOCAL_READ;</div><div class='del'>-}</div><div class='add'>+extern const struct uapi_definition mlx5_ib_qos_defs[];</div><div class='add'>+extern const struct uapi_definition mlx5_ib_std_types_defs[];</div><div class='add'>+extern const struct uapi_definition mlx5_ib_create_cq_defs[];</div><div class='ctx'> </div><div class='ctx'> static inline int is_qp1(enum ib_qp_type qp_type)</div><div class='ctx'> {</div><div class='del'>-	return qp_type == MLX5_IB_QPT_HW_GSI;</div><div class='add'>+	return qp_type == MLX5_IB_QPT_HW_GSI || qp_type == IB_QPT_GSI;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-#define MLX5_MAX_UMR_SHIFT 16</div><div class='del'>-#define MLX5_MAX_UMR_PAGES (1 &lt;&lt; MLX5_MAX_UMR_SHIFT)</div><div class='del'>-</div><div class='ctx'> static inline u32 check_cq_create_flags(u32 flags)</div><div class='ctx'> {</div><div class='ctx'> 	/*</div><div class='hunk'>@@ -1396,12 +1617,11 @@ static inline int get_qp_user_index(struct mlx5_ib_ucontext *ucontext,</div><div class='ctx'> {</div><div class='ctx'> 	u8 cqe_version = ucontext-&gt;cqe_version;</div><div class='ctx'> </div><div class='del'>-	if (field_avail(struct mlx5_ib_create_qp, uidx, inlen) &amp;&amp;</div><div class='del'>-	    !cqe_version &amp;&amp; (ucmd-&gt;uidx == MLX5_IB_DEFAULT_UIDX))</div><div class='add'>+	if ((offsetofend(typeof(*ucmd), uidx) &lt;= inlen) &amp;&amp; !cqe_version &amp;&amp;</div><div class='add'>+	    (ucmd-&gt;uidx == MLX5_IB_DEFAULT_UIDX))</div><div class='ctx'> 		return 0;</div><div class='ctx'> </div><div class='del'>-	if (!!(field_avail(struct mlx5_ib_create_qp, uidx, inlen) !=</div><div class='del'>-	       !!cqe_version))</div><div class='add'>+	if ((offsetofend(typeof(*ucmd), uidx) &lt;= inlen) != !!cqe_version)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	return verify_assign_uidx(cqe_version, ucmd-&gt;uidx, user_index);</div><div class='hunk'>@@ -1414,12 +1634,11 @@ static inline int get_srq_user_index(struct mlx5_ib_ucontext *ucontext,</div><div class='ctx'> {</div><div class='ctx'> 	u8 cqe_version = ucontext-&gt;cqe_version;</div><div class='ctx'> </div><div class='del'>-	if (field_avail(struct mlx5_ib_create_srq, uidx, inlen) &amp;&amp;</div><div class='del'>-	    !cqe_version &amp;&amp; (ucmd-&gt;uidx == MLX5_IB_DEFAULT_UIDX))</div><div class='add'>+	if ((offsetofend(typeof(*ucmd), uidx) &lt;= inlen) &amp;&amp; !cqe_version &amp;&amp;</div><div class='add'>+	    (ucmd-&gt;uidx == MLX5_IB_DEFAULT_UIDX))</div><div class='ctx'> 		return 0;</div><div class='ctx'> </div><div class='del'>-	if (!!(field_avail(struct mlx5_ib_create_srq, uidx, inlen) !=</div><div class='del'>-	       !!cqe_version))</div><div class='add'>+	if ((offsetofend(typeof(*ucmd), uidx) &lt;= inlen) != !!cqe_version)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	return verify_assign_uidx(cqe_version, ucmd-&gt;uidx, user_index);</div><div class='hunk'>@@ -1431,16 +1650,176 @@ static inline int get_uars_per_sys_page(struct mlx5_ib_dev *dev, bool lib_suppor</div><div class='ctx'> 				MLX5_UARS_IN_PAGE : 1;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline int get_num_static_uars(struct mlx5_ib_dev *dev,</div><div class='del'>-				      struct mlx5_bfreg_info *bfregi)</div><div class='del'>-{</div><div class='del'>-	return get_uars_per_sys_page(dev, bfregi-&gt;lib_uar_4k) * bfregi-&gt;num_static_sys_pages;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-unsigned long mlx5_ib_get_xlt_emergency_page(void);</div><div class='del'>-void mlx5_ib_put_xlt_emergency_page(void);</div><div class='add'>+extern void *xlt_emergency_page;</div><div class='ctx'> </div><div class='ctx'> int bfregn_to_uar_index(struct mlx5_ib_dev *dev,</div><div class='ctx'> 			struct mlx5_bfreg_info *bfregi, u32 bfregn,</div><div class='ctx'> 			bool dyn_bfreg);</div><div class='add'>+</div><div class='add'>+static inline int mlx5r_store_odp_mkey(struct mlx5_ib_dev *dev,</div><div class='add'>+				       struct mlx5_ib_mkey *mmkey)</div><div class='add'>+{</div><div class='add'>+	refcount_set(&amp;mmkey-&gt;usecount, 1);</div><div class='add'>+</div><div class='add'>+	return xa_err(xa_store(&amp;dev-&gt;odp_mkeys, mlx5_base_mkey(mmkey-&gt;key),</div><div class='add'>+			       mmkey, GFP_KERNEL));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* deref an mkey that can participate in ODP flow */</div><div class='add'>+static inline void mlx5r_deref_odp_mkey(struct mlx5_ib_mkey *mmkey)</div><div class='add'>+{</div><div class='add'>+	if (refcount_dec_and_test(&amp;mmkey-&gt;usecount))</div><div class='add'>+		wake_up(&amp;mmkey-&gt;wait);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* deref an mkey that can participate in ODP flow and wait for relese */</div><div class='add'>+static inline void mlx5r_deref_wait_odp_mkey(struct mlx5_ib_mkey *mmkey)</div><div class='add'>+{</div><div class='add'>+	mlx5r_deref_odp_mkey(mmkey);</div><div class='add'>+	wait_event(mmkey-&gt;wait, refcount_read(&amp;mmkey-&gt;usecount) == 0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool mlx5_ib_lag_should_assign_affinity(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	/*</div><div class='add'>+	 * If the driver is in hash mode and the port_select_flow_table_bypass cap</div><div class='add'>+	 * is supported, it means that the driver no longer needs to assign the port</div><div class='add'>+	 * affinity by default. If a user wants to set the port affinity explicitly,</div><div class='add'>+	 * the user has a dedicated API to do that, so there is no need to assign</div><div class='add'>+	 * the port affinity by default.</div><div class='add'>+	 */</div><div class='add'>+	if (dev-&gt;lag_active &amp;&amp;</div><div class='add'>+	    mlx5_lag_mode_is_hash(dev-&gt;mdev) &amp;&amp;</div><div class='add'>+	    MLX5_CAP_PORT_SELECTION(dev-&gt;mdev, port_select_flow_table_bypass))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (mlx5_lag_is_lacp_owner(dev-&gt;mdev) &amp;&amp; !dev-&gt;lag_active)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	return dev-&gt;lag_active ||</div><div class='add'>+		(MLX5_CAP_GEN(dev-&gt;mdev, num_lag_ports) &gt; 1 &amp;&amp;</div><div class='add'>+		 MLX5_CAP_GEN(dev-&gt;mdev, lag_tx_port_affinity));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline bool rt_supported(int ts_cap)</div><div class='add'>+{</div><div class='add'>+	return ts_cap == MLX5_TIMESTAMP_FORMAT_CAP_REAL_TIME ||</div><div class='add'>+	       ts_cap == MLX5_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * PCI Peer to Peer is a trainwreck. If no switch is present then things</div><div class='add'>+ * sometimes work, depending on the pci_distance_p2p logic for excluding broken</div><div class='add'>+ * root complexes. However if a switch is present in the path, then things get</div><div class='add'>+ * really ugly depending on how the switch is setup. This table assumes that the</div><div class='add'>+ * root complex is strict and is validating that all req/reps are matches</div><div class='add'>+ * perfectly - so any scenario where it sees only half the transaction is a</div><div class='add'>+ * failure.</div><div class='add'>+ *</div><div class='add'>+ * CR/RR/DT  ATS RO P2P</div><div class='add'>+ * 00X       X   X  OK</div><div class='add'>+ * 010       X   X  fails (request is routed to root but root never sees comp)</div><div class='add'>+ * 011       0   X  fails (request is routed to root but root never sees comp)</div><div class='add'>+ * 011       1   X  OK</div><div class='add'>+ * 10X       X   1  OK</div><div class='add'>+ * 101       X   0  fails (completion is routed to root but root didn't see req)</div><div class='add'>+ * 110       X   0  SLOW</div><div class='add'>+ * 111       0   0  SLOW</div><div class='add'>+ * 111       1   0  fails (completion is routed to root but root didn't see req)</div><div class='add'>+ * 111       1   1  OK</div><div class='add'>+ *</div><div class='add'>+ * Unfortunately we cannot reliably know if a switch is present or what the</div><div class='add'>+ * CR/RR/DT ACS settings are, as in a VM that is all hidden. Assume that</div><div class='add'>+ * CR/RR/DT is 111 if the ATS cap is enabled and follow the last three rows.</div><div class='add'>+ *</div><div class='add'>+ * For now assume if the umem is a dma_buf then it is P2P.</div><div class='add'>+ */</div><div class='add'>+static inline bool mlx5_umem_needs_ats(struct mlx5_ib_dev *dev,</div><div class='add'>+				       struct ib_umem *umem, int access_flags)</div><div class='add'>+{</div><div class='add'>+	if (!MLX5_CAP_GEN(dev-&gt;mdev, ats) || !umem-&gt;is_dmabuf)</div><div class='add'>+		return false;</div><div class='add'>+	return access_flags &amp; IB_ACCESS_RELAXED_ORDERING;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int set_roce_addr(struct mlx5_ib_dev *dev, u32 port_num,</div><div class='add'>+		  unsigned int index, const union ib_gid *gid,</div><div class='add'>+		  const struct ib_gid_attr *attr);</div><div class='add'>+</div><div class='add'>+static inline u32 smi_to_native_portnum(struct mlx5_ib_dev *dev, u32 port)</div><div class='add'>+{</div><div class='add'>+	return (port - 1) / dev-&gt;num_ports + 1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline unsigned int get_max_log_entity_size_cap(struct mlx5_ib_dev *dev,</div><div class='add'>+						       int access_mode)</div><div class='add'>+{</div><div class='add'>+	int max_log_size = 0;</div><div class='add'>+</div><div class='add'>+	if (access_mode == MLX5_MKC_ACCESS_MODE_MTT)</div><div class='add'>+		max_log_size =</div><div class='add'>+			MLX5_CAP_GEN_2(dev-&gt;mdev, max_mkey_log_entity_size_mtt);</div><div class='add'>+	else if (access_mode == MLX5_MKC_ACCESS_MODE_KSM)</div><div class='add'>+		max_log_size = MLX5_CAP_GEN_2(</div><div class='add'>+			dev-&gt;mdev, max_mkey_log_entity_size_fixed_buffer);</div><div class='add'>+</div><div class='add'>+	if (!max_log_size ||</div><div class='add'>+	    (max_log_size &gt; 31 &amp;&amp;</div><div class='add'>+	     !MLX5_CAP_GEN_2(dev-&gt;mdev, umr_log_entity_size_5)))</div><div class='add'>+		max_log_size = 31;</div><div class='add'>+</div><div class='add'>+	return max_log_size;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline unsigned int get_min_log_entity_size_cap(struct mlx5_ib_dev *dev,</div><div class='add'>+						       int access_mode)</div><div class='add'>+{</div><div class='add'>+	int min_log_size = 0;</div><div class='add'>+</div><div class='add'>+	if (access_mode == MLX5_MKC_ACCESS_MODE_KSM &amp;&amp;</div><div class='add'>+	    MLX5_CAP_GEN_2(dev-&gt;mdev,</div><div class='add'>+			   min_mkey_log_entity_size_fixed_buffer_valid))</div><div class='add'>+		min_log_size = MLX5_CAP_GEN_2(</div><div class='add'>+			dev-&gt;mdev, min_mkey_log_entity_size_fixed_buffer);</div><div class='add'>+	else</div><div class='add'>+		min_log_size =</div><div class='add'>+			MLX5_CAP_GEN_2(dev-&gt;mdev, log_min_mkey_entity_size);</div><div class='add'>+</div><div class='add'>+	min_log_size = max(min_log_size, MLX5_ADAPTER_PAGE_SHIFT);</div><div class='add'>+	return min_log_size;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * For mkc users, instead of a page_offset the command has a start_iova which</div><div class='add'>+ * specifies both the page_offset and the on-the-wire IOVA</div><div class='add'>+ */</div><div class='add'>+static __always_inline unsigned long</div><div class='add'>+mlx5_umem_mkc_find_best_pgsz(struct mlx5_ib_dev *dev, struct ib_umem *umem,</div><div class='add'>+			     u64 iova, int access_mode)</div><div class='add'>+{</div><div class='add'>+	unsigned int max_log_entity_size_cap, min_log_entity_size_cap;</div><div class='add'>+	unsigned long bitmap;</div><div class='add'>+</div><div class='add'>+	max_log_entity_size_cap = get_max_log_entity_size_cap(dev, access_mode);</div><div class='add'>+	min_log_entity_size_cap = get_min_log_entity_size_cap(dev, access_mode);</div><div class='add'>+</div><div class='add'>+	bitmap = GENMASK_ULL(max_log_entity_size_cap, min_log_entity_size_cap);</div><div class='add'>+</div><div class='add'>+	/* In KSM mode HW requires IOVA and mkey's page size to be aligned */</div><div class='add'>+	if (access_mode == MLX5_MKC_ACCESS_MODE_KSM &amp;&amp; iova)</div><div class='add'>+		bitmap &amp;= GENMASK_ULL(__ffs64(iova), 0);</div><div class='add'>+</div><div class='add'>+	return ib_umem_find_best_pgsz(umem, bitmap, iova);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static inline unsigned long</div><div class='add'>+mlx5_umem_dmabuf_find_best_pgsz(struct ib_umem_dmabuf *umem_dmabuf,</div><div class='add'>+				int access_mode)</div><div class='add'>+{</div><div class='add'>+	return mlx5_umem_mkc_find_best_pgsz(to_mdev(umem_dmabuf-&gt;umem.ibdev),</div><div class='add'>+					    &amp;umem_dmabuf-&gt;umem,</div><div class='add'>+					    umem_dmabuf-&gt;umem.iova,</div><div class='add'>+					    access_mode);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> #endif /* MLX5_IB_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c<br/>index 83b452d977d4..325fa04cbe8a 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/mr.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/mr.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/mr.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/mr.c</a></div><div class='hunk'>@@ -1,5 +1,6 @@</div><div class='ctx'> /*</div><div class='ctx'>  * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.</div><div class='add'>+ * Copyright (c) 2020, Intel Corporation. All rights reserved.</div><div class='ctx'>  *</div><div class='ctx'>  * This software is available to you under a choice of one of two</div><div class='ctx'>  * licenses.  You may choose to be licensed under the terms of the GNU</div><div class='hunk'>@@ -36,230 +37,346 @@</div><div class='ctx'> #include &lt;linux/debugfs.h&gt;</div><div class='ctx'> #include &lt;linux/export.h&gt;</div><div class='ctx'> #include &lt;linux/delay.h&gt;</div><div class='del'>-#include &lt;rdma/ib_umem.h&gt;</div><div class='add'>+#include &lt;linux/dma-buf.h&gt;</div><div class='add'>+#include &lt;linux/dma-resv.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_umem_odp.h&gt;</div><div class='del'>-#include &lt;rdma/ib_verbs.h&gt;</div><div class='add'>+#include "dm.h"</div><div class='ctx'> #include "mlx5_ib.h"</div><div class='add'>+#include "umr.h"</div><div class='add'>+#include "data_direct.h"</div><div class='add'>+#include "dmah.h"</div><div class='ctx'> </div><div class='ctx'> enum {</div><div class='ctx'> 	MAX_PENDING_REG_MR = 8,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+#define MLX5_MR_CACHE_PERSISTENT_ENTRY_MIN_DESCS 4</div><div class='ctx'> #define MLX5_UMR_ALIGN 2048</div><div class='ctx'> </div><div class='del'>-static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);</div><div class='del'>-static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);</div><div class='del'>-static int mr_cache_max_order(struct mlx5_ib_dev *dev);</div><div class='del'>-static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);</div><div class='del'>-static bool umr_can_modify_entity_size(struct mlx5_ib_dev *dev)</div><div class='add'>+static void</div><div class='add'>+create_mkey_callback(int status, struct mlx5_async_work *context);</div><div class='add'>+static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,</div><div class='add'>+				     u64 iova, int access_flags,</div><div class='add'>+				     unsigned long page_size, bool populate,</div><div class='add'>+				     int access_mode, u16 st_index, u8 ph);</div><div class='add'>+static int __mlx5_ib_dereg_mr(struct ib_mr *ibmr);</div><div class='add'>+</div><div class='add'>+static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr,</div><div class='add'>+					  struct ib_pd *pd)</div><div class='ctx'> {</div><div class='del'>-	return !MLX5_CAP_GEN(dev-&gt;mdev, umr_modify_entity_size_disabled);</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='add'>+</div><div class='add'>+	MLX5_SET(mkc, mkc, a, !!(acc &amp; IB_ACCESS_REMOTE_ATOMIC));</div><div class='add'>+	MLX5_SET(mkc, mkc, rw, !!(acc &amp; IB_ACCESS_REMOTE_WRITE));</div><div class='add'>+	MLX5_SET(mkc, mkc, rr, !!(acc &amp; IB_ACCESS_REMOTE_READ));</div><div class='add'>+	MLX5_SET(mkc, mkc, lw, !!(acc &amp; IB_ACCESS_LOCAL_WRITE));</div><div class='add'>+	MLX5_SET(mkc, mkc, lr, 1);</div><div class='add'>+</div><div class='add'>+	if (acc &amp; IB_ACCESS_RELAXED_ORDERING) {</div><div class='add'>+		if (MLX5_CAP_GEN(dev-&gt;mdev, relaxed_ordering_write))</div><div class='add'>+			MLX5_SET(mkc, mkc, relaxed_ordering_write, 1);</div><div class='add'>+</div><div class='add'>+		if (MLX5_CAP_GEN(dev-&gt;mdev, relaxed_ordering_read) ||</div><div class='add'>+		    (MLX5_CAP_GEN(dev-&gt;mdev,</div><div class='add'>+				  relaxed_ordering_read_pci_enabled) &amp;&amp;</div><div class='add'>+		     pcie_relaxed_ordering_enabled(dev-&gt;mdev-&gt;pdev)))</div><div class='add'>+			MLX5_SET(mkc, mkc, relaxed_ordering_read, 1);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	MLX5_SET(mkc, mkc, pd, to_mpd(pd)-&gt;pdn);</div><div class='add'>+	MLX5_SET(mkc, mkc, qpn, 0xffffff);</div><div class='add'>+	MLX5_SET64(mkc, mkc, start_addr, start_addr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev)</div><div class='add'>+static void assign_mkey_variant(struct mlx5_ib_dev *dev, u32 *mkey, u32 *in)</div><div class='ctx'> {</div><div class='del'>-	return !MLX5_CAP_GEN(dev-&gt;mdev, umr_indirect_mkey_disabled);</div><div class='add'>+	u8 key = atomic_inc_return(&amp;dev-&gt;mkey_var);</div><div class='add'>+	void *mkc;</div><div class='add'>+</div><div class='add'>+	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);</div><div class='add'>+	MLX5_SET(mkc, mkc, mkey_7_0, key);</div><div class='add'>+	*mkey = key;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static bool use_umr(struct mlx5_ib_dev *dev, int order)</div><div class='add'>+static int mlx5_ib_create_mkey(struct mlx5_ib_dev *dev,</div><div class='add'>+			       struct mlx5_ib_mkey *mkey, u32 *in, int inlen)</div><div class='ctx'> {</div><div class='del'>-	return order &lt;= mr_cache_max_order(dev) &amp;&amp;</div><div class='del'>-		umr_can_modify_entity_size(dev);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	assign_mkey_variant(dev, &amp;mkey-&gt;key, in);</div><div class='add'>+	ret = mlx5_core_create_mkey(dev-&gt;mdev, &amp;mkey-&gt;key, in, inlen);</div><div class='add'>+	if (!ret)</div><div class='add'>+		init_waitqueue_head(&amp;mkey-&gt;wait);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)</div><div class='add'>+static int mlx5_ib_create_mkey_cb(struct mlx5r_async_create_mkey *async_create)</div><div class='ctx'> {</div><div class='del'>-	int err = mlx5_core_destroy_mkey(dev-&gt;mdev, &amp;mr-&gt;mmkey);</div><div class='add'>+	struct mlx5_ib_dev *dev = async_create-&gt;ent-&gt;dev;</div><div class='add'>+	size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);</div><div class='add'>+	size_t outlen = MLX5_ST_SZ_BYTES(create_mkey_out);</div><div class='add'>+</div><div class='add'>+	MLX5_SET(create_mkey_in, async_create-&gt;in, opcode,</div><div class='add'>+		 MLX5_CMD_OP_CREATE_MKEY);</div><div class='add'>+	assign_mkey_variant(dev, &amp;async_create-&gt;mkey, async_create-&gt;in);</div><div class='add'>+	return mlx5_cmd_exec_cb(&amp;dev-&gt;async_ctx, async_create-&gt;in, inlen,</div><div class='add'>+				async_create-&gt;out, outlen, create_mkey_callback,</div><div class='add'>+				&amp;async_create-&gt;cb_work);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))</div><div class='del'>-		/* Wait until all page fault handlers using the mr complete. */</div><div class='del'>-		synchronize_srcu(&amp;dev-&gt;mr_srcu);</div><div class='add'>+static int mkey_cache_max_order(struct mlx5_ib_dev *dev);</div><div class='add'>+static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent);</div><div class='ctx'> </div><div class='del'>-	return err;</div><div class='add'>+static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)</div><div class='add'>+{</div><div class='add'>+	WARN_ON(xa_load(&amp;dev-&gt;odp_mkeys, mlx5_base_mkey(mr-&gt;mmkey.key)));</div><div class='add'>+</div><div class='add'>+	return mlx5_core_destroy_mkey(dev-&gt;mdev, mr-&gt;mmkey.key);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int order2idx(struct mlx5_ib_dev *dev, int order)</div><div class='add'>+static void create_mkey_warn(struct mlx5_ib_dev *dev, int status, void *out)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_mr_cache *cache = &amp;dev-&gt;cache;</div><div class='add'>+	if (status == -ENXIO) /* core driver is not available */</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-	if (order &lt; cache-&gt;ent[0].order)</div><div class='del'>-		return 0;</div><div class='del'>-	else</div><div class='del'>-		return order - cache-&gt;ent[0].order;</div><div class='add'>+	mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status);</div><div class='add'>+	if (status != -EREMOTEIO) /* driver specific failure */</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	/* Failed in FW, print cmd out failure details */</div><div class='add'>+	mlx5_cmd_out_err(dev-&gt;mdev, MLX5_CMD_OP_CREATE_MKEY, 0, out);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static bool use_umr_mtt_update(struct mlx5_ib_mr *mr, u64 start, u64 length)</div><div class='add'>+static int push_mkey_locked(struct mlx5_cache_ent *ent, u32 mkey)</div><div class='ctx'> {</div><div class='del'>-	return ((u64)1 &lt;&lt; mr-&gt;order) * MLX5_ADAPTER_PAGE_SIZE &gt;=</div><div class='del'>-		length + (start &amp; (MLX5_ADAPTER_PAGE_SIZE - 1));</div><div class='add'>+	unsigned long tmp = ent-&gt;mkeys_queue.ci % NUM_MKEYS_PER_PAGE;</div><div class='add'>+	struct mlx5_mkeys_page *page;</div><div class='add'>+</div><div class='add'>+	lockdep_assert_held(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+	if (ent-&gt;mkeys_queue.ci &gt;=</div><div class='add'>+	    ent-&gt;mkeys_queue.num_pages * NUM_MKEYS_PER_PAGE) {</div><div class='add'>+		page = kzalloc(sizeof(*page), GFP_ATOMIC);</div><div class='add'>+		if (!page)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		ent-&gt;mkeys_queue.num_pages++;</div><div class='add'>+		list_add_tail(&amp;page-&gt;list, &amp;ent-&gt;mkeys_queue.pages_list);</div><div class='add'>+	} else {</div><div class='add'>+		page = list_last_entry(&amp;ent-&gt;mkeys_queue.pages_list,</div><div class='add'>+				       struct mlx5_mkeys_page, list);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	page-&gt;mkeys[tmp] = mkey;</div><div class='add'>+	ent-&gt;mkeys_queue.ci++;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void update_odp_mr(struct mlx5_ib_mr *mr)</div><div class='add'>+static int pop_mkey_locked(struct mlx5_cache_ent *ent)</div><div class='ctx'> {</div><div class='del'>-	if (is_odp_mr(mr)) {</div><div class='del'>-		/*</div><div class='del'>-		 * This barrier prevents the compiler from moving the</div><div class='del'>-		 * setting of umem-&gt;odp_data-&gt;private to point to our</div><div class='del'>-		 * MR, before reg_umr finished, to ensure that the MR</div><div class='del'>-		 * initialization have finished before starting to</div><div class='del'>-		 * handle invalidations.</div><div class='del'>-		 */</div><div class='del'>-		smp_wmb();</div><div class='del'>-		to_ib_umem_odp(mr-&gt;umem)-&gt;private = mr;</div><div class='del'>-		/*</div><div class='del'>-		 * Make sure we will see the new</div><div class='del'>-		 * umem-&gt;odp_data-&gt;private value in the invalidation</div><div class='del'>-		 * routines, before we can get page faults on the</div><div class='del'>-		 * MR. Page faults can happen once we put the MR in</div><div class='del'>-		 * the tree, below this line. Without the barrier,</div><div class='del'>-		 * there can be a fault handling and an invalidation</div><div class='del'>-		 * before umem-&gt;odp_data-&gt;private == mr is visible to</div><div class='del'>-		 * the invalidation handler.</div><div class='del'>-		 */</div><div class='del'>-		smp_wmb();</div><div class='add'>+	unsigned long tmp = (ent-&gt;mkeys_queue.ci - 1) % NUM_MKEYS_PER_PAGE;</div><div class='add'>+	struct mlx5_mkeys_page *last_page;</div><div class='add'>+	u32 mkey;</div><div class='add'>+</div><div class='add'>+	lockdep_assert_held(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+	last_page = list_last_entry(&amp;ent-&gt;mkeys_queue.pages_list,</div><div class='add'>+				    struct mlx5_mkeys_page, list);</div><div class='add'>+	mkey = last_page-&gt;mkeys[tmp];</div><div class='add'>+	last_page-&gt;mkeys[tmp] = 0;</div><div class='add'>+	ent-&gt;mkeys_queue.ci--;</div><div class='add'>+	if (ent-&gt;mkeys_queue.num_pages &gt; 1 &amp;&amp; !tmp) {</div><div class='add'>+		list_del(&amp;last_page-&gt;list);</div><div class='add'>+		ent-&gt;mkeys_queue.num_pages--;</div><div class='add'>+		kfree(last_page);</div><div class='ctx'> 	}</div><div class='add'>+	return mkey;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void reg_mr_callback(int status, struct mlx5_async_work *context)</div><div class='add'>+static void create_mkey_callback(int status, struct mlx5_async_work *context)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_mr *mr =</div><div class='del'>-		container_of(context, struct mlx5_ib_mr, cb_work);</div><div class='del'>-	struct mlx5_ib_dev *dev = mr-&gt;dev;</div><div class='del'>-	struct mlx5_mr_cache *cache = &amp;dev-&gt;cache;</div><div class='del'>-	int c = order2idx(dev, mr-&gt;order);</div><div class='del'>-	struct mlx5_cache_ent *ent = &amp;cache-&gt;ent[c];</div><div class='del'>-	u8 key;</div><div class='add'>+	struct mlx5r_async_create_mkey *mkey_out =</div><div class='add'>+		container_of(context, struct mlx5r_async_create_mkey, cb_work);</div><div class='add'>+	struct mlx5_cache_ent *ent = mkey_out-&gt;ent;</div><div class='add'>+	struct mlx5_ib_dev *dev = ent-&gt;dev;</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	struct xarray *mkeys = &amp;dev-&gt;mdev-&gt;priv.mkey_table;</div><div class='del'>-	int err;</div><div class='ctx'> </div><div class='del'>-	spin_lock_irqsave(&amp;ent-&gt;lock, flags);</div><div class='del'>-	ent-&gt;pending--;</div><div class='del'>-	spin_unlock_irqrestore(&amp;ent-&gt;lock, flags);</div><div class='ctx'> 	if (status) {</div><div class='del'>-		mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status);</div><div class='del'>-		kfree(mr);</div><div class='del'>-		dev-&gt;fill_delay = 1;</div><div class='add'>+		create_mkey_warn(dev, status, mkey_out-&gt;out);</div><div class='add'>+		kfree(mkey_out);</div><div class='add'>+		spin_lock_irqsave(&amp;ent-&gt;mkeys_queue.lock, flags);</div><div class='add'>+		ent-&gt;pending--;</div><div class='add'>+		WRITE_ONCE(dev-&gt;fill_delay, 1);</div><div class='add'>+		spin_unlock_irqrestore(&amp;ent-&gt;mkeys_queue.lock, flags);</div><div class='ctx'> 		mod_timer(&amp;dev-&gt;delay_timer, jiffies + HZ);</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	mr-&gt;mmkey.type = MLX5_MKEY_MR;</div><div class='del'>-	spin_lock_irqsave(&amp;dev-&gt;mdev-&gt;priv.mkey_lock, flags);</div><div class='del'>-	key = dev-&gt;mdev-&gt;priv.mkey_key++;</div><div class='del'>-	spin_unlock_irqrestore(&amp;dev-&gt;mdev-&gt;priv.mkey_lock, flags);</div><div class='del'>-	mr-&gt;mmkey.key = mlx5_idx_to_mkey(MLX5_GET(create_mkey_out, mr-&gt;out, mkey_index)) | key;</div><div class='del'>-</div><div class='del'>-	cache-&gt;last_add = jiffies;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;ent-&gt;lock, flags);</div><div class='del'>-	list_add_tail(&amp;mr-&gt;list, &amp;ent-&gt;head);</div><div class='del'>-	ent-&gt;cur++;</div><div class='del'>-	ent-&gt;size++;</div><div class='del'>-	spin_unlock_irqrestore(&amp;ent-&gt;lock, flags);</div><div class='del'>-</div><div class='del'>-	xa_lock_irqsave(mkeys, flags);</div><div class='del'>-	err = xa_err(__xa_store(mkeys, mlx5_base_mkey(mr-&gt;mmkey.key),</div><div class='del'>-				&amp;mr-&gt;mmkey, GFP_ATOMIC));</div><div class='del'>-	xa_unlock_irqrestore(mkeys, flags);</div><div class='del'>-	if (err)</div><div class='del'>-		pr_err("Error inserting to mkey tree. 0x%x\n", -err);</div><div class='add'>+	mkey_out-&gt;mkey |= mlx5_idx_to_mkey(</div><div class='add'>+		MLX5_GET(create_mkey_out, mkey_out-&gt;out, mkey_index));</div><div class='add'>+	WRITE_ONCE(dev-&gt;cache.last_add, jiffies);</div><div class='ctx'> </div><div class='del'>-	if (!completion_done(&amp;ent-&gt;compl))</div><div class='del'>-		complete(&amp;ent-&gt;compl);</div><div class='add'>+	spin_lock_irqsave(&amp;ent-&gt;mkeys_queue.lock, flags);</div><div class='add'>+	push_mkey_locked(ent, mkey_out-&gt;mkey);</div><div class='add'>+	ent-&gt;pending--;</div><div class='add'>+	/* If we are doing fill_to_high_water then keep going. */</div><div class='add'>+	queue_adjust_cache_locked(ent);</div><div class='add'>+	spin_unlock_irqrestore(&amp;ent-&gt;mkeys_queue.lock, flags);</div><div class='add'>+	kfree(mkey_out);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int add_keys(struct mlx5_ib_dev *dev, int c, int num)</div><div class='add'>+static int get_mkc_octo_size(unsigned int access_mode, unsigned int ndescs)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_mr_cache *cache = &amp;dev-&gt;cache;</div><div class='del'>-	struct mlx5_cache_ent *ent = &amp;cache-&gt;ent[c];</div><div class='del'>-	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);</div><div class='del'>-	struct mlx5_ib_mr *mr;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	switch (access_mode) {</div><div class='add'>+	case MLX5_MKC_ACCESS_MODE_MTT:</div><div class='add'>+		ret = DIV_ROUND_UP(ndescs, MLX5_IB_UMR_OCTOWORD /</div><div class='add'>+						   sizeof(struct mlx5_mtt));</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_MKC_ACCESS_MODE_KSM:</div><div class='add'>+		ret = DIV_ROUND_UP(ndescs, MLX5_IB_UMR_OCTOWORD /</div><div class='add'>+						   sizeof(struct mlx5_klm));</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		WARN_ON(1);</div><div class='add'>+	}</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void set_cache_mkc(struct mlx5_cache_ent *ent, void *mkc)</div><div class='add'>+{</div><div class='add'>+	set_mkc_access_pd_addr_fields(mkc, ent-&gt;rb_key.access_flags, 0,</div><div class='add'>+				      ent-&gt;dev-&gt;umrc.pd);</div><div class='add'>+	MLX5_SET(mkc, mkc, free, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, umr_en, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, access_mode_1_0, ent-&gt;rb_key.access_mode &amp; 0x3);</div><div class='add'>+	MLX5_SET(mkc, mkc, access_mode_4_2,</div><div class='add'>+		(ent-&gt;rb_key.access_mode &gt;&gt; 2) &amp; 0x7);</div><div class='add'>+	MLX5_SET(mkc, mkc, ma_translation_mode, !!ent-&gt;rb_key.ats);</div><div class='add'>+</div><div class='add'>+	MLX5_SET(mkc, mkc, translations_octword_size,</div><div class='add'>+		 get_mkc_octo_size(ent-&gt;rb_key.access_mode,</div><div class='add'>+				   ent-&gt;rb_key.ndescs));</div><div class='add'>+	MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);</div><div class='add'>+</div><div class='add'>+	if (ent-&gt;rb_key.ph != MLX5_IB_NO_PH) {</div><div class='add'>+		MLX5_SET(mkc, mkc, pcie_tph_en, 1);</div><div class='add'>+		MLX5_SET(mkc, mkc, pcie_tph_ph, ent-&gt;rb_key.ph);</div><div class='add'>+		if (ent-&gt;rb_key.st_index != MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX)</div><div class='add'>+			MLX5_SET(mkc, mkc, pcie_tph_steering_tag_index,</div><div class='add'>+				 ent-&gt;rb_key.st_index);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+/* Asynchronously schedule new MRs to be populated in the cache. */</div><div class='add'>+static int add_keys(struct mlx5_cache_ent *ent, unsigned int num)</div><div class='add'>+{</div><div class='add'>+	struct mlx5r_async_create_mkey *async_create;</div><div class='ctx'> 	void *mkc;</div><div class='del'>-	u32 *in;</div><div class='ctx'> 	int err = 0;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='del'>-	in = kzalloc(inlen, GFP_KERNEL);</div><div class='del'>-	if (!in)</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);</div><div class='ctx'> 	for (i = 0; i &lt; num; i++) {</div><div class='add'>+		async_create = kzalloc(sizeof(struct mlx5r_async_create_mkey),</div><div class='add'>+				       GFP_KERNEL);</div><div class='add'>+		if (!async_create)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		mkc = MLX5_ADDR_OF(create_mkey_in, async_create-&gt;in,</div><div class='add'>+				   memory_key_mkey_entry);</div><div class='add'>+		set_cache_mkc(ent, mkc);</div><div class='add'>+		async_create-&gt;ent = ent;</div><div class='add'>+</div><div class='add'>+		spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='ctx'> 		if (ent-&gt;pending &gt;= MAX_PENDING_REG_MR) {</div><div class='ctx'> 			err = -EAGAIN;</div><div class='del'>-			break;</div><div class='add'>+			goto free_async_create;</div><div class='ctx'> 		}</div><div class='add'>+		ent-&gt;pending++;</div><div class='add'>+		spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='ctx'> </div><div class='del'>-		mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='del'>-		if (!mr) {</div><div class='del'>-			err = -ENOMEM;</div><div class='del'>-			break;</div><div class='add'>+		err = mlx5_ib_create_mkey_cb(async_create);</div><div class='add'>+		if (err) {</div><div class='add'>+			mlx5_ib_warn(ent-&gt;dev, "create mkey failed %d\n", err);</div><div class='add'>+			goto err_create_mkey;</div><div class='ctx'> 		}</div><div class='del'>-		mr-&gt;order = ent-&gt;order;</div><div class='del'>-		mr-&gt;allocated_from_cache = 1;</div><div class='del'>-		mr-&gt;dev = dev;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		MLX5_SET(mkc, mkc, free, 1);</div><div class='del'>-		MLX5_SET(mkc, mkc, umr_en, 1);</div><div class='del'>-		MLX5_SET(mkc, mkc, access_mode_1_0, ent-&gt;access_mode &amp; 0x3);</div><div class='del'>-		MLX5_SET(mkc, mkc, access_mode_4_2,</div><div class='del'>-			 (ent-&gt;access_mode &gt;&gt; 2) &amp; 0x7);</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-		MLX5_SET(mkc, mkc, qpn, 0xffffff);</div><div class='del'>-		MLX5_SET(mkc, mkc, translations_octword_size, ent-&gt;xlt);</div><div class='del'>-		MLX5_SET(mkc, mkc, log_page_size, ent-&gt;page);</div><div class='add'>+err_create_mkey:</div><div class='add'>+	spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+	ent-&gt;pending--;</div><div class='add'>+free_async_create:</div><div class='add'>+	spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+	kfree(async_create);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		spin_lock_irq(&amp;ent-&gt;lock);</div><div class='del'>-		ent-&gt;pending++;</div><div class='del'>-		spin_unlock_irq(&amp;ent-&gt;lock);</div><div class='del'>-		err = mlx5_core_create_mkey_cb(dev-&gt;mdev, &amp;mr-&gt;mmkey,</div><div class='del'>-					       &amp;dev-&gt;async_ctx, in, inlen,</div><div class='del'>-					       mr-&gt;out, sizeof(mr-&gt;out),</div><div class='del'>-					       reg_mr_callback, &amp;mr-&gt;cb_work);</div><div class='del'>-		if (err) {</div><div class='del'>-			spin_lock_irq(&amp;ent-&gt;lock);</div><div class='del'>-			ent-&gt;pending--;</div><div class='del'>-			spin_unlock_irq(&amp;ent-&gt;lock);</div><div class='del'>-			mlx5_ib_warn(dev, "create mkey failed %d\n", err);</div><div class='del'>-			kfree(mr);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='add'>+/* Synchronously create a MR in the cache */</div><div class='add'>+static int create_cache_mkey(struct mlx5_cache_ent *ent, u32 *mkey)</div><div class='add'>+{</div><div class='add'>+	size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);</div><div class='add'>+	void *mkc;</div><div class='add'>+	u32 *in;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='add'>+	in = kzalloc(inlen, GFP_KERNEL);</div><div class='add'>+	if (!in)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);</div><div class='add'>+	set_cache_mkc(ent, mkc);</div><div class='add'>+</div><div class='add'>+	err = mlx5_core_create_mkey(ent-&gt;dev-&gt;mdev, mkey, in, inlen);</div><div class='add'>+	if (err)</div><div class='add'>+		goto free_in;</div><div class='add'>+</div><div class='add'>+	WRITE_ONCE(ent-&gt;dev-&gt;cache.last_add, jiffies);</div><div class='add'>+free_in:</div><div class='ctx'> 	kfree(in);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void remove_keys(struct mlx5_ib_dev *dev, int c, int num)</div><div class='add'>+static void remove_cache_mr_locked(struct mlx5_cache_ent *ent)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_mr_cache *cache = &amp;dev-&gt;cache;</div><div class='del'>-	struct mlx5_cache_ent *ent = &amp;cache-&gt;ent[c];</div><div class='del'>-	struct mlx5_ib_mr *tmp_mr;</div><div class='del'>-	struct mlx5_ib_mr *mr;</div><div class='del'>-	LIST_HEAD(del_list);</div><div class='del'>-	int i;</div><div class='add'>+	u32 mkey;</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; num; i++) {</div><div class='del'>-		spin_lock_irq(&amp;ent-&gt;lock);</div><div class='del'>-		if (list_empty(&amp;ent-&gt;head)) {</div><div class='del'>-			spin_unlock_irq(&amp;ent-&gt;lock);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		mr = list_first_entry(&amp;ent-&gt;head, struct mlx5_ib_mr, list);</div><div class='del'>-		list_move(&amp;mr-&gt;list, &amp;del_list);</div><div class='del'>-		ent-&gt;cur--;</div><div class='del'>-		ent-&gt;size--;</div><div class='del'>-		spin_unlock_irq(&amp;ent-&gt;lock);</div><div class='del'>-		mlx5_core_destroy_mkey(dev-&gt;mdev, &amp;mr-&gt;mmkey);</div><div class='del'>-	}</div><div class='add'>+	lockdep_assert_held(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+	if (!ent-&gt;mkeys_queue.ci)</div><div class='add'>+		return;</div><div class='add'>+	mkey = pop_mkey_locked(ent);</div><div class='add'>+	spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+	mlx5_core_destroy_mkey(ent-&gt;dev-&gt;mdev, mkey);</div><div class='add'>+	spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))</div><div class='del'>-		synchronize_srcu(&amp;dev-&gt;mr_srcu);</div><div class='add'>+static int resize_available_mrs(struct mlx5_cache_ent *ent, unsigned int target,</div><div class='add'>+				bool limit_fill)</div><div class='add'>+	__acquires(&amp;ent-&gt;mkeys_queue.lock) __releases(&amp;ent-&gt;mkeys_queue.lock)</div><div class='add'>+{</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-	list_for_each_entry_safe(mr, tmp_mr, &amp;del_list, list) {</div><div class='del'>-		list_del(&amp;mr-&gt;list);</div><div class='del'>-		kfree(mr);</div><div class='add'>+	lockdep_assert_held(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+</div><div class='add'>+	while (true) {</div><div class='add'>+		if (limit_fill)</div><div class='add'>+			target = ent-&gt;limit * 2;</div><div class='add'>+		if (target == ent-&gt;pending + ent-&gt;mkeys_queue.ci)</div><div class='add'>+			return 0;</div><div class='add'>+		if (target &gt; ent-&gt;pending + ent-&gt;mkeys_queue.ci) {</div><div class='add'>+			u32 todo = target - (ent-&gt;pending + ent-&gt;mkeys_queue.ci);</div><div class='add'>+</div><div class='add'>+			spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+			err = add_keys(ent, todo);</div><div class='add'>+			if (err == -EAGAIN)</div><div class='add'>+				usleep_range(3000, 5000);</div><div class='add'>+			spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+			if (err) {</div><div class='add'>+				if (err != -EAGAIN)</div><div class='add'>+					return err;</div><div class='add'>+			} else</div><div class='add'>+				return 0;</div><div class='add'>+		} else {</div><div class='add'>+			remove_cache_mr_locked(ent);</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -267,37 +384,38 @@ static ssize_t size_write(struct file *filp, const char __user *buf,</div><div class='ctx'> 			  size_t count, loff_t *pos)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_cache_ent *ent = filp-&gt;private_data;</div><div class='del'>-	struct mlx5_ib_dev *dev = ent-&gt;dev;</div><div class='del'>-	char lbuf[20] = {0};</div><div class='del'>-	u32 var;</div><div class='add'>+	u32 target;</div><div class='ctx'> 	int err;</div><div class='del'>-	int c;</div><div class='del'>-</div><div class='del'>-	count = min(count, sizeof(lbuf) - 1);</div><div class='del'>-	if (copy_from_user(lbuf, buf, count))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	c = order2idx(dev, ent-&gt;order);</div><div class='ctx'> </div><div class='del'>-	if (sscanf(lbuf, "%u", &amp;var) != 1)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	if (var &lt; ent-&gt;limit)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	if (var &gt; ent-&gt;size) {</div><div class='del'>-		do {</div><div class='del'>-			err = add_keys(dev, c, var - ent-&gt;size);</div><div class='del'>-			if (err &amp;&amp; err != -EAGAIN)</div><div class='del'>-				return err;</div><div class='add'>+	err = kstrtou32_from_user(buf, count, 0, &amp;target);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='ctx'> </div><div class='del'>-			usleep_range(3000, 5000);</div><div class='del'>-		} while (err);</div><div class='del'>-	} else if (var &lt; ent-&gt;size) {</div><div class='del'>-		remove_keys(dev, c, ent-&gt;size - var);</div><div class='add'>+	/*</div><div class='add'>+	 * Target is the new value of total_mrs the user requests, however we</div><div class='add'>+	 * cannot free MRs that are in use. Compute the target value for stored</div><div class='add'>+	 * mkeys.</div><div class='add'>+	 */</div><div class='add'>+	spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+	if (target &lt; ent-&gt;in_use) {</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+		goto err_unlock;</div><div class='ctx'> 	}</div><div class='add'>+	target = target - ent-&gt;in_use;</div><div class='add'>+	if (target &lt; ent-&gt;limit || target &gt; ent-&gt;limit*2) {</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+		goto err_unlock;</div><div class='add'>+	}</div><div class='add'>+	err = resize_available_mrs(ent, target, false);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_unlock;</div><div class='add'>+	spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='ctx'> </div><div class='ctx'> 	return count;</div><div class='add'>+</div><div class='add'>+err_unlock:</div><div class='add'>+	spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static ssize_t size_read(struct file *filp, char __user *buf, size_t count,</div><div class='hunk'>@@ -307,7 +425,8 @@ static ssize_t size_read(struct file *filp, char __user *buf, size_t count,</div><div class='ctx'> 	char lbuf[20];</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent-&gt;size);</div><div class='add'>+	err = snprintf(lbuf, sizeof(lbuf), "%ld\n",</div><div class='add'>+		       ent-&gt;mkeys_queue.ci + ent-&gt;in_use);</div><div class='ctx'> 	if (err &lt; 0)</div><div class='ctx'> 		return err;</div><div class='ctx'> </div><div class='hunk'>@@ -325,32 +444,23 @@ static ssize_t limit_write(struct file *filp, const char __user *buf,</div><div class='ctx'> 			   size_t count, loff_t *pos)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_cache_ent *ent = filp-&gt;private_data;</div><div class='del'>-	struct mlx5_ib_dev *dev = ent-&gt;dev;</div><div class='del'>-	char lbuf[20] = {0};</div><div class='ctx'> 	u32 var;</div><div class='ctx'> 	int err;</div><div class='del'>-	int c;</div><div class='del'>-</div><div class='del'>-	count = min(count, sizeof(lbuf) - 1);</div><div class='del'>-	if (copy_from_user(lbuf, buf, count))</div><div class='del'>-		return -EFAULT;</div><div class='del'>-</div><div class='del'>-	c = order2idx(dev, ent-&gt;order);</div><div class='ctx'> </div><div class='del'>-	if (sscanf(lbuf, "%u", &amp;var) != 1)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	if (var &gt; ent-&gt;size)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	err = kstrtou32_from_user(buf, count, 0, &amp;var);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='ctx'> </div><div class='add'>+	/*</div><div class='add'>+	 * Upon set we immediately fill the cache to high water mark implied by</div><div class='add'>+	 * the limit.</div><div class='add'>+	 */</div><div class='add'>+	spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='ctx'> 	ent-&gt;limit = var;</div><div class='del'>-</div><div class='del'>-	if (ent-&gt;cur &lt; ent-&gt;limit) {</div><div class='del'>-		err = add_keys(dev, c, 2 * ent-&gt;limit - ent-&gt;cur);</div><div class='del'>-		if (err)</div><div class='del'>-			return err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='add'>+	err = resize_available_mrs(ent, 0, true);</div><div class='add'>+	spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='ctx'> 	return count;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -375,68 +485,140 @@ static const struct file_operations limit_fops = {</div><div class='ctx'> 	.read	= limit_read,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static int someone_adding(struct mlx5_mr_cache *cache)</div><div class='add'>+static bool someone_adding(struct mlx5_mkey_cache *cache)</div><div class='ctx'> {</div><div class='del'>-	int i;</div><div class='add'>+	struct mlx5_cache_ent *ent;</div><div class='add'>+	struct rb_node *node;</div><div class='add'>+	bool ret;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;cache-&gt;rb_lock);</div><div class='add'>+	for (node = rb_first(&amp;cache-&gt;rb_root); node; node = rb_next(node)) {</div><div class='add'>+		ent = rb_entry(node, struct mlx5_cache_ent, node);</div><div class='add'>+		spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+		ret = ent-&gt;mkeys_queue.ci &lt; ent-&gt;limit;</div><div class='add'>+		spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+		if (ret) {</div><div class='add'>+			mutex_unlock(&amp;cache-&gt;rb_lock);</div><div class='add'>+			return true;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;cache-&gt;rb_lock);</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; MAX_MR_CACHE_ENTRIES; i++) {</div><div class='del'>-		if (cache-&gt;ent[i].cur &lt; cache-&gt;ent[i].limit)</div><div class='del'>-			return 1;</div><div class='add'>+/*</div><div class='add'>+ * Check if the bucket is outside the high/low water mark and schedule an async</div><div class='add'>+ * update. The cache refill has hysteresis, once the low water mark is hit it is</div><div class='add'>+ * refilled up to the high mark.</div><div class='add'>+ */</div><div class='add'>+static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent)</div><div class='add'>+{</div><div class='add'>+	lockdep_assert_held(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+</div><div class='add'>+	if (ent-&gt;disabled || READ_ONCE(ent-&gt;dev-&gt;fill_delay) || ent-&gt;is_tmp)</div><div class='add'>+		return;</div><div class='add'>+	if (ent-&gt;mkeys_queue.ci &lt; ent-&gt;limit) {</div><div class='add'>+		ent-&gt;fill_to_high_water = true;</div><div class='add'>+		mod_delayed_work(ent-&gt;dev-&gt;cache.wq, &amp;ent-&gt;dwork, 0);</div><div class='add'>+	} else if (ent-&gt;fill_to_high_water &amp;&amp;</div><div class='add'>+		   ent-&gt;mkeys_queue.ci + ent-&gt;pending &lt; 2 * ent-&gt;limit) {</div><div class='add'>+		/*</div><div class='add'>+		 * Once we start populating due to hitting a low water mark</div><div class='add'>+		 * continue until we pass the high water mark.</div><div class='add'>+		 */</div><div class='add'>+		mod_delayed_work(ent-&gt;dev-&gt;cache.wq, &amp;ent-&gt;dwork, 0);</div><div class='add'>+	} else if (ent-&gt;mkeys_queue.ci == 2 * ent-&gt;limit) {</div><div class='add'>+		ent-&gt;fill_to_high_water = false;</div><div class='add'>+	} else if (ent-&gt;mkeys_queue.ci &gt; 2 * ent-&gt;limit) {</div><div class='add'>+		/* Queue deletion of excess entries */</div><div class='add'>+		ent-&gt;fill_to_high_water = false;</div><div class='add'>+		if (ent-&gt;pending)</div><div class='add'>+			queue_delayed_work(ent-&gt;dev-&gt;cache.wq, &amp;ent-&gt;dwork,</div><div class='add'>+					   secs_to_jiffies(1));</div><div class='add'>+		else</div><div class='add'>+			mod_delayed_work(ent-&gt;dev-&gt;cache.wq, &amp;ent-&gt;dwork, 0);</div><div class='ctx'> 	}</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='add'>+static void clean_keys(struct mlx5_ib_dev *dev, struct mlx5_cache_ent *ent)</div><div class='add'>+{</div><div class='add'>+	u32 mkey;</div><div class='add'>+</div><div class='add'>+	spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+	while (ent-&gt;mkeys_queue.ci) {</div><div class='add'>+		mkey = pop_mkey_locked(ent);</div><div class='add'>+		spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+		mlx5_core_destroy_mkey(dev-&gt;mdev, mkey);</div><div class='add'>+		spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+	}</div><div class='add'>+	ent-&gt;tmp_cleanup_scheduled = false;</div><div class='add'>+	spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void __cache_work_func(struct mlx5_cache_ent *ent)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = ent-&gt;dev;</div><div class='del'>-	struct mlx5_mr_cache *cache = &amp;dev-&gt;cache;</div><div class='del'>-	int i = order2idx(dev, ent-&gt;order);</div><div class='add'>+	struct mlx5_mkey_cache *cache = &amp;dev-&gt;cache;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	if (cache-&gt;stopped)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	ent = &amp;dev-&gt;cache.ent[i];</div><div class='del'>-	if (ent-&gt;cur &lt; 2 * ent-&gt;limit &amp;&amp; !dev-&gt;fill_delay) {</div><div class='del'>-		err = add_keys(dev, i, 1);</div><div class='del'>-		if (ent-&gt;cur &lt; 2 * ent-&gt;limit) {</div><div class='del'>-			if (err == -EAGAIN) {</div><div class='del'>-				mlx5_ib_dbg(dev, "returned eagain, order %d\n",</div><div class='del'>-					    i + 2);</div><div class='del'>-				queue_delayed_work(cache-&gt;wq, &amp;ent-&gt;dwork,</div><div class='del'>-						   msecs_to_jiffies(3));</div><div class='del'>-			} else if (err) {</div><div class='del'>-				mlx5_ib_warn(dev, "command failed order %d, err %d\n",</div><div class='del'>-					     i + 2, err);</div><div class='add'>+	spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+	if (ent-&gt;disabled)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	if (ent-&gt;fill_to_high_water &amp;&amp;</div><div class='add'>+	    ent-&gt;mkeys_queue.ci + ent-&gt;pending &lt; 2 * ent-&gt;limit &amp;&amp;</div><div class='add'>+	    !READ_ONCE(dev-&gt;fill_delay)) {</div><div class='add'>+		spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+		err = add_keys(ent, 1);</div><div class='add'>+		spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+		if (ent-&gt;disabled)</div><div class='add'>+			goto out;</div><div class='add'>+		if (err) {</div><div class='add'>+			/*</div><div class='add'>+			 * EAGAIN only happens if there are pending MRs, so we</div><div class='add'>+			 * will be rescheduled when storing them. The only</div><div class='add'>+			 * failure path here is ENOMEM.</div><div class='add'>+			 */</div><div class='add'>+			if (err != -EAGAIN) {</div><div class='add'>+				mlx5_ib_warn(</div><div class='add'>+					dev,</div><div class='add'>+					"add keys command failed, err %d\n",</div><div class='add'>+					err);</div><div class='ctx'> 				queue_delayed_work(cache-&gt;wq, &amp;ent-&gt;dwork,</div><div class='del'>-						   msecs_to_jiffies(1000));</div><div class='del'>-			} else {</div><div class='del'>-				queue_work(cache-&gt;wq, &amp;ent-&gt;work);</div><div class='add'>+						   secs_to_jiffies(1));</div><div class='ctx'> 			}</div><div class='ctx'> 		}</div><div class='del'>-	} else if (ent-&gt;cur &gt; 2 * ent-&gt;limit) {</div><div class='add'>+	} else if (ent-&gt;mkeys_queue.ci &gt; 2 * ent-&gt;limit) {</div><div class='add'>+		bool need_delay;</div><div class='add'>+</div><div class='ctx'> 		/*</div><div class='del'>-		 * The remove_keys() logic is performed as garbage collection</div><div class='del'>-		 * task. Such task is intended to be run when no other active</div><div class='del'>-		 * processes are running.</div><div class='add'>+		 * The remove_cache_mr() logic is performed as garbage</div><div class='add'>+		 * collection task. Such task is intended to be run when no</div><div class='add'>+		 * other active processes are running.</div><div class='ctx'> 		 *</div><div class='ctx'> 		 * The need_resched() will return TRUE if there are user tasks</div><div class='ctx'> 		 * to be activated in near future.</div><div class='ctx'> 		 *</div><div class='del'>-		 * In such case, we don't execute remove_keys() and postpone</div><div class='del'>-		 * the garbage collection work to try to run in next cycle,</div><div class='del'>-		 * in order to free CPU resources to other tasks.</div><div class='add'>+		 * In such case, we don't execute remove_cache_mr() and postpone</div><div class='add'>+		 * the garbage collection work to try to run in next cycle, in</div><div class='add'>+		 * order to free CPU resources to other tasks.</div><div class='ctx'> 		 */</div><div class='del'>-		if (!need_resched() &amp;&amp; !someone_adding(cache) &amp;&amp;</div><div class='del'>-		    time_after(jiffies, cache-&gt;last_add + 300 * HZ)) {</div><div class='del'>-			remove_keys(dev, i, 1);</div><div class='del'>-			if (ent-&gt;cur &gt; ent-&gt;limit)</div><div class='del'>-				queue_work(cache-&gt;wq, &amp;ent-&gt;work);</div><div class='del'>-		} else {</div><div class='add'>+		spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+		need_delay = need_resched() || someone_adding(cache) ||</div><div class='add'>+			     !time_after(jiffies,</div><div class='add'>+					 READ_ONCE(cache-&gt;last_add) + 300 * HZ);</div><div class='add'>+		spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+		if (ent-&gt;disabled)</div><div class='add'>+			goto out;</div><div class='add'>+		if (need_delay) {</div><div class='ctx'> 			queue_delayed_work(cache-&gt;wq, &amp;ent-&gt;dwork, 300 * HZ);</div><div class='add'>+			goto out;</div><div class='ctx'> 		}</div><div class='add'>+		remove_cache_mr_locked(ent);</div><div class='add'>+		queue_adjust_cache_locked(ent);</div><div class='ctx'> 	}</div><div class='add'>+out:</div><div class='add'>+	spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void delayed_cache_work_func(struct work_struct *work)</div><div class='hunk'>@@ -444,206 +626,353 @@ static void delayed_cache_work_func(struct work_struct *work)</div><div class='ctx'> 	struct mlx5_cache_ent *ent;</div><div class='ctx'> </div><div class='ctx'> 	ent = container_of(work, struct mlx5_cache_ent, dwork.work);</div><div class='del'>-	__cache_work_func(ent);</div><div class='add'>+	/* temp entries are never filled, only cleaned */</div><div class='add'>+	if (ent-&gt;is_tmp)</div><div class='add'>+		clean_keys(ent-&gt;dev, ent);</div><div class='add'>+	else</div><div class='add'>+		__cache_work_func(ent);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void cache_work_func(struct work_struct *work)</div><div class='add'>+static int cache_ent_key_cmp(struct mlx5r_cache_rb_key key1,</div><div class='add'>+			     struct mlx5r_cache_rb_key key2)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_cache_ent *ent;</div><div class='add'>+	int res;</div><div class='add'>+</div><div class='add'>+	res = key1.ats - key2.ats;</div><div class='add'>+	if (res)</div><div class='add'>+		return res;</div><div class='add'>+</div><div class='add'>+	res = key1.access_mode - key2.access_mode;</div><div class='add'>+	if (res)</div><div class='add'>+		return res;</div><div class='ctx'> </div><div class='del'>-	ent = container_of(work, struct mlx5_cache_ent, work);</div><div class='del'>-	__cache_work_func(ent);</div><div class='add'>+	res = key1.access_flags - key2.access_flags;</div><div class='add'>+	if (res)</div><div class='add'>+		return res;</div><div class='add'>+</div><div class='add'>+	res = key1.st_index - key2.st_index;</div><div class='add'>+	if (res)</div><div class='add'>+		return res;</div><div class='add'>+</div><div class='add'>+	res = key1.ph - key2.ph;</div><div class='add'>+	if (res)</div><div class='add'>+		return res;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * keep ndescs the last in the compare table since the find function</div><div class='add'>+	 * searches for an exact match on all properties and only closest</div><div class='add'>+	 * match in size.</div><div class='add'>+	 */</div><div class='add'>+	return key1.ndescs - key2.ndescs;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry)</div><div class='add'>+static int mlx5_cache_ent_insert(struct mlx5_mkey_cache *cache,</div><div class='add'>+				 struct mlx5_cache_ent *ent)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_mr_cache *cache = &amp;dev-&gt;cache;</div><div class='del'>-	struct mlx5_cache_ent *ent;</div><div class='del'>-	struct mlx5_ib_mr *mr;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	if (entry &lt; 0 || entry &gt;= MAX_MR_CACHE_ENTRIES) {</div><div class='del'>-		mlx5_ib_err(dev, "cache entry %d is out of range\n", entry);</div><div class='del'>-		return NULL;</div><div class='add'>+	struct rb_node **new = &amp;cache-&gt;rb_root.rb_node, *parent = NULL;</div><div class='add'>+	struct mlx5_cache_ent *cur;</div><div class='add'>+	int cmp;</div><div class='add'>+</div><div class='add'>+	/* Figure out where to put new node */</div><div class='add'>+	while (*new) {</div><div class='add'>+		cur = rb_entry(*new, struct mlx5_cache_ent, node);</div><div class='add'>+		parent = *new;</div><div class='add'>+		cmp = cache_ent_key_cmp(cur-&gt;rb_key, ent-&gt;rb_key);</div><div class='add'>+		if (cmp &gt; 0)</div><div class='add'>+			new = &amp;((*new)-&gt;rb_left);</div><div class='add'>+		if (cmp &lt; 0)</div><div class='add'>+			new = &amp;((*new)-&gt;rb_right);</div><div class='add'>+		if (cmp == 0)</div><div class='add'>+			return -EEXIST;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	ent = &amp;cache-&gt;ent[entry];</div><div class='del'>-	while (1) {</div><div class='del'>-		spin_lock_irq(&amp;ent-&gt;lock);</div><div class='del'>-		if (list_empty(&amp;ent-&gt;head)) {</div><div class='del'>-			spin_unlock_irq(&amp;ent-&gt;lock);</div><div class='add'>+	/* Add new node and rebalance tree. */</div><div class='add'>+	rb_link_node(&amp;ent-&gt;node, parent, new);</div><div class='add'>+	rb_insert_color(&amp;ent-&gt;node, &amp;cache-&gt;rb_root);</div><div class='ctx'> </div><div class='del'>-			err = add_keys(dev, entry, 1);</div><div class='del'>-			if (err &amp;&amp; err != -EAGAIN)</div><div class='del'>-				return ERR_PTR(err);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-			wait_for_completion(&amp;ent-&gt;compl);</div><div class='del'>-		} else {</div><div class='del'>-			mr = list_first_entry(&amp;ent-&gt;head, struct mlx5_ib_mr,</div><div class='del'>-					      list);</div><div class='del'>-			list_del(&amp;mr-&gt;list);</div><div class='del'>-			ent-&gt;cur--;</div><div class='del'>-			spin_unlock_irq(&amp;ent-&gt;lock);</div><div class='del'>-			if (ent-&gt;cur &lt; ent-&gt;limit)</div><div class='del'>-				queue_work(cache-&gt;wq, &amp;ent-&gt;work);</div><div class='del'>-			return mr;</div><div class='add'>+static struct mlx5_cache_ent *</div><div class='add'>+mkey_cache_ent_from_rb_key(struct mlx5_ib_dev *dev,</div><div class='add'>+			   struct mlx5r_cache_rb_key rb_key)</div><div class='add'>+{</div><div class='add'>+	struct rb_node *node = dev-&gt;cache.rb_root.rb_node;</div><div class='add'>+	struct mlx5_cache_ent *cur, *smallest = NULL;</div><div class='add'>+	u64 ndescs_limit;</div><div class='add'>+	int cmp;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Find the smallest ent with order &gt;= requested_order.</div><div class='add'>+	 */</div><div class='add'>+	while (node) {</div><div class='add'>+		cur = rb_entry(node, struct mlx5_cache_ent, node);</div><div class='add'>+		cmp = cache_ent_key_cmp(cur-&gt;rb_key, rb_key);</div><div class='add'>+		if (cmp &gt; 0) {</div><div class='add'>+			smallest = cur;</div><div class='add'>+			node = node-&gt;rb_left;</div><div class='ctx'> 		}</div><div class='add'>+		if (cmp &lt; 0)</div><div class='add'>+			node = node-&gt;rb_right;</div><div class='add'>+		if (cmp == 0)</div><div class='add'>+			return cur;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Limit the usage of mkeys larger than twice the required size while</div><div class='add'>+	 * also allowing the usage of smallest cache entry for small MRs.</div><div class='add'>+	 */</div><div class='add'>+	ndescs_limit = max_t(u64, rb_key.ndescs * 2,</div><div class='add'>+			     MLX5_MR_CACHE_PERSISTENT_ENTRY_MIN_DESCS);</div><div class='add'>+</div><div class='add'>+	return (smallest &amp;&amp;</div><div class='add'>+		smallest-&gt;rb_key.access_mode == rb_key.access_mode &amp;&amp;</div><div class='add'>+		smallest-&gt;rb_key.access_flags == rb_key.access_flags &amp;&amp;</div><div class='add'>+		smallest-&gt;rb_key.ats == rb_key.ats &amp;&amp;</div><div class='add'>+		smallest-&gt;rb_key.st_index == rb_key.st_index &amp;&amp;</div><div class='add'>+		smallest-&gt;rb_key.ph == rb_key.ph &amp;&amp;</div><div class='add'>+		smallest-&gt;rb_key.ndescs &lt;= ndescs_limit) ?</div><div class='add'>+		       smallest :</div><div class='add'>+		       NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order)</div><div class='add'>+static struct mlx5_ib_mr *_mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,</div><div class='add'>+					       struct mlx5_cache_ent *ent)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_mr_cache *cache = &amp;dev-&gt;cache;</div><div class='del'>-	struct mlx5_ib_mr *mr = NULL;</div><div class='del'>-	struct mlx5_cache_ent *ent;</div><div class='del'>-	int last_umr_cache_entry;</div><div class='del'>-	int c;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	c = order2idx(dev, order);</div><div class='del'>-	last_umr_cache_entry = order2idx(dev, mr_cache_max_order(dev));</div><div class='del'>-	if (c &lt; 0 || c &gt; last_umr_cache_entry) {</div><div class='del'>-		mlx5_ib_warn(dev, "order %d, cache index %d\n", order, c);</div><div class='del'>-		return NULL;</div><div class='del'>-	}</div><div class='add'>+	struct mlx5_ib_mr *mr;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-	for (i = c; i &lt;= last_umr_cache_entry; i++) {</div><div class='del'>-		ent = &amp;cache-&gt;ent[i];</div><div class='add'>+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='add'>+	if (!mr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='del'>-		mlx5_ib_dbg(dev, "order %d, cache index %d\n", ent-&gt;order, i);</div><div class='add'>+	spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+	ent-&gt;in_use++;</div><div class='ctx'> </div><div class='del'>-		spin_lock_irq(&amp;ent-&gt;lock);</div><div class='del'>-		if (!list_empty(&amp;ent-&gt;head)) {</div><div class='del'>-			mr = list_first_entry(&amp;ent-&gt;head, struct mlx5_ib_mr,</div><div class='del'>-					      list);</div><div class='del'>-			list_del(&amp;mr-&gt;list);</div><div class='del'>-			ent-&gt;cur--;</div><div class='del'>-			spin_unlock_irq(&amp;ent-&gt;lock);</div><div class='del'>-			if (ent-&gt;cur &lt; ent-&gt;limit)</div><div class='del'>-				queue_work(cache-&gt;wq, &amp;ent-&gt;work);</div><div class='del'>-			break;</div><div class='add'>+	if (!ent-&gt;mkeys_queue.ci) {</div><div class='add'>+		queue_adjust_cache_locked(ent);</div><div class='add'>+		ent-&gt;miss++;</div><div class='add'>+		spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+		err = create_cache_mkey(ent, &amp;mr-&gt;mmkey.key);</div><div class='add'>+		if (err) {</div><div class='add'>+			spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+			ent-&gt;in_use--;</div><div class='add'>+			spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+			kfree(mr);</div><div class='add'>+			return ERR_PTR(err);</div><div class='ctx'> 		}</div><div class='del'>-		spin_unlock_irq(&amp;ent-&gt;lock);</div><div class='del'>-</div><div class='del'>-		queue_work(cache-&gt;wq, &amp;ent-&gt;work);</div><div class='add'>+	} else {</div><div class='add'>+		mr-&gt;mmkey.key = pop_mkey_locked(ent);</div><div class='add'>+		queue_adjust_cache_locked(ent);</div><div class='add'>+		spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='ctx'> 	}</div><div class='add'>+	mr-&gt;mmkey.cache_ent = ent;</div><div class='add'>+	mr-&gt;mmkey.type = MLX5_MKEY_MR;</div><div class='add'>+	mr-&gt;mmkey.rb_key = ent-&gt;rb_key;</div><div class='add'>+	mr-&gt;mmkey.cacheable = true;</div><div class='add'>+	init_waitqueue_head(&amp;mr-&gt;mmkey.wait);</div><div class='add'>+	return mr;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (!mr)</div><div class='del'>-		cache-&gt;ent[c].miss++;</div><div class='add'>+static int get_unchangeable_access_flags(struct mlx5_ib_dev *dev,</div><div class='add'>+					 int access_flags)</div><div class='add'>+{</div><div class='add'>+	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	return mr;</div><div class='add'>+	if ((access_flags &amp; IB_ACCESS_REMOTE_ATOMIC) &amp;&amp;</div><div class='add'>+	    MLX5_CAP_GEN(dev-&gt;mdev, atomic) &amp;&amp;</div><div class='add'>+	    MLX5_CAP_GEN(dev-&gt;mdev, umr_modify_atomic_disabled))</div><div class='add'>+		ret |= IB_ACCESS_REMOTE_ATOMIC;</div><div class='add'>+</div><div class='add'>+	if ((access_flags &amp; IB_ACCESS_RELAXED_ORDERING) &amp;&amp;</div><div class='add'>+	    MLX5_CAP_GEN(dev-&gt;mdev, relaxed_ordering_write) &amp;&amp;</div><div class='add'>+	    !MLX5_CAP_GEN(dev-&gt;mdev, relaxed_ordering_write_umr))</div><div class='add'>+		ret |= IB_ACCESS_RELAXED_ORDERING;</div><div class='add'>+</div><div class='add'>+	if ((access_flags &amp; IB_ACCESS_RELAXED_ORDERING) &amp;&amp;</div><div class='add'>+	    (MLX5_CAP_GEN(dev-&gt;mdev, relaxed_ordering_read) ||</div><div class='add'>+	     MLX5_CAP_GEN(dev-&gt;mdev, relaxed_ordering_read_pci_enabled)) &amp;&amp;</div><div class='add'>+	    !MLX5_CAP_GEN(dev-&gt;mdev, relaxed_ordering_read_umr))</div><div class='add'>+		ret |= IB_ACCESS_RELAXED_ORDERING;</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)</div><div class='add'>+struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,</div><div class='add'>+				       int access_flags, int access_mode,</div><div class='add'>+				       int ndescs)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_mr_cache *cache = &amp;dev-&gt;cache;</div><div class='del'>-	struct mlx5_cache_ent *ent;</div><div class='del'>-	int shrink = 0;</div><div class='del'>-	int c;</div><div class='add'>+	struct mlx5r_cache_rb_key rb_key = {</div><div class='add'>+		.ndescs = ndescs,</div><div class='add'>+		.access_mode = access_mode,</div><div class='add'>+		.access_flags = get_unchangeable_access_flags(dev, access_flags),</div><div class='add'>+		.ph = MLX5_IB_NO_PH,</div><div class='add'>+	};</div><div class='add'>+	struct mlx5_cache_ent *ent = mkey_cache_ent_from_rb_key(dev, rb_key);</div><div class='add'>+</div><div class='add'>+	if (!ent)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='ctx'> </div><div class='del'>-	if (!mr-&gt;allocated_from_cache)</div><div class='del'>-		return;</div><div class='add'>+	return _mlx5_mr_cache_alloc(dev, ent);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	c = order2idx(dev, mr-&gt;order);</div><div class='del'>-	if (c &lt; 0 || c &gt;= MAX_MR_CACHE_ENTRIES) {</div><div class='del'>-		mlx5_ib_warn(dev, "order %d, cache index %d\n", mr-&gt;order, c);</div><div class='add'>+static void mlx5_mkey_cache_debugfs_cleanup(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	if (!mlx5_debugfs_root || dev-&gt;is_rep)</div><div class='ctx'> 		return;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	if (unreg_umr(dev, mr))</div><div class='add'>+	debugfs_remove_recursive(dev-&gt;cache.fs_root);</div><div class='add'>+	dev-&gt;cache.fs_root = NULL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_mkey_cache_debugfs_add_ent(struct mlx5_ib_dev *dev,</div><div class='add'>+					    struct mlx5_cache_ent *ent)</div><div class='add'>+{</div><div class='add'>+	int order = order_base_2(ent-&gt;rb_key.ndescs);</div><div class='add'>+	struct dentry *dir;</div><div class='add'>+</div><div class='add'>+	if (!mlx5_debugfs_root || dev-&gt;is_rep)</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='del'>-	ent = &amp;cache-&gt;ent[c];</div><div class='del'>-	spin_lock_irq(&amp;ent-&gt;lock);</div><div class='del'>-	list_add_tail(&amp;mr-&gt;list, &amp;ent-&gt;head);</div><div class='del'>-	ent-&gt;cur++;</div><div class='del'>-	if (ent-&gt;cur &gt; 2 * ent-&gt;limit)</div><div class='del'>-		shrink = 1;</div><div class='del'>-	spin_unlock_irq(&amp;ent-&gt;lock);</div><div class='add'>+	if (ent-&gt;rb_key.access_mode == MLX5_MKC_ACCESS_MODE_KSM)</div><div class='add'>+		order = MLX5_IMR_KSM_CACHE_ENTRY + 2;</div><div class='ctx'> </div><div class='del'>-	if (shrink)</div><div class='del'>-		queue_work(cache-&gt;wq, &amp;ent-&gt;work);</div><div class='add'>+	sprintf(ent-&gt;name, "%d", order);</div><div class='add'>+	dir = debugfs_create_dir(ent-&gt;name, dev-&gt;cache.fs_root);</div><div class='add'>+	debugfs_create_file("size", 0600, dir, ent, &amp;size_fops);</div><div class='add'>+	debugfs_create_file("limit", 0600, dir, ent, &amp;limit_fops);</div><div class='add'>+	debugfs_create_ulong("cur", 0400, dir, &amp;ent-&gt;mkeys_queue.ci);</div><div class='add'>+	debugfs_create_u32("miss", 0600, dir, &amp;ent-&gt;miss);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void clean_keys(struct mlx5_ib_dev *dev, int c)</div><div class='add'>+static void mlx5_mkey_cache_debugfs_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_mr_cache *cache = &amp;dev-&gt;cache;</div><div class='del'>-	struct mlx5_cache_ent *ent = &amp;cache-&gt;ent[c];</div><div class='del'>-	struct mlx5_ib_mr *tmp_mr;</div><div class='del'>-	struct mlx5_ib_mr *mr;</div><div class='del'>-	LIST_HEAD(del_list);</div><div class='add'>+	struct dentry *dbg_root = mlx5_debugfs_get_dev_root(dev-&gt;mdev);</div><div class='add'>+	struct mlx5_mkey_cache *cache = &amp;dev-&gt;cache;</div><div class='ctx'> </div><div class='del'>-	cancel_delayed_work(&amp;ent-&gt;dwork);</div><div class='del'>-	while (1) {</div><div class='del'>-		spin_lock_irq(&amp;ent-&gt;lock);</div><div class='del'>-		if (list_empty(&amp;ent-&gt;head)) {</div><div class='del'>-			spin_unlock_irq(&amp;ent-&gt;lock);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-		mr = list_first_entry(&amp;ent-&gt;head, struct mlx5_ib_mr, list);</div><div class='del'>-		list_move(&amp;mr-&gt;list, &amp;del_list);</div><div class='del'>-		ent-&gt;cur--;</div><div class='del'>-		ent-&gt;size--;</div><div class='del'>-		spin_unlock_irq(&amp;ent-&gt;lock);</div><div class='del'>-		mlx5_core_destroy_mkey(dev-&gt;mdev, &amp;mr-&gt;mmkey);</div><div class='del'>-	}</div><div class='add'>+	if (!mlx5_debugfs_root || dev-&gt;is_rep)</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING</div><div class='del'>-	synchronize_srcu(&amp;dev-&gt;mr_srcu);</div><div class='del'>-#endif</div><div class='add'>+	cache-&gt;fs_root = debugfs_create_dir("mr_cache", dbg_root);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	list_for_each_entry_safe(mr, tmp_mr, &amp;del_list, list) {</div><div class='del'>-		list_del(&amp;mr-&gt;list);</div><div class='del'>-		kfree(mr);</div><div class='del'>-	}</div><div class='add'>+static void delay_time_func(struct timer_list *t)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = timer_container_of(dev, t, delay_timer);</div><div class='add'>+</div><div class='add'>+	WRITE_ONCE(dev-&gt;fill_delay, 0);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev)</div><div class='add'>+static int mlx5r_mkeys_init(struct mlx5_cache_ent *ent)</div><div class='ctx'> {</div><div class='del'>-	if (!mlx5_debugfs_root || dev-&gt;is_rep)</div><div class='del'>-		return;</div><div class='add'>+	struct mlx5_mkeys_page *page;</div><div class='ctx'> </div><div class='del'>-	debugfs_remove_recursive(dev-&gt;cache.root);</div><div class='del'>-	dev-&gt;cache.root = NULL;</div><div class='add'>+	page = kzalloc(sizeof(*page), GFP_KERNEL);</div><div class='add'>+	if (!page)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	INIT_LIST_HEAD(&amp;ent-&gt;mkeys_queue.pages_list);</div><div class='add'>+	spin_lock_init(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+	list_add_tail(&amp;page-&gt;list, &amp;ent-&gt;mkeys_queue.pages_list);</div><div class='add'>+	ent-&gt;mkeys_queue.num_pages++;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5r_mkeys_uninit(struct mlx5_cache_ent *ent)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_mkeys_page *page;</div><div class='add'>+</div><div class='add'>+	WARN_ON(ent-&gt;mkeys_queue.ci || ent-&gt;mkeys_queue.num_pages &gt; 1);</div><div class='add'>+	page = list_last_entry(&amp;ent-&gt;mkeys_queue.pages_list,</div><div class='add'>+			       struct mlx5_mkeys_page, list);</div><div class='add'>+	list_del(&amp;page-&gt;list);</div><div class='add'>+	kfree(page);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev)</div><div class='add'>+struct mlx5_cache_ent *</div><div class='add'>+mlx5r_cache_create_ent_locked(struct mlx5_ib_dev *dev,</div><div class='add'>+			      struct mlx5r_cache_rb_key rb_key,</div><div class='add'>+			      bool persistent_entry)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_mr_cache *cache = &amp;dev-&gt;cache;</div><div class='ctx'> 	struct mlx5_cache_ent *ent;</div><div class='del'>-	struct dentry *dir;</div><div class='del'>-	int i;</div><div class='add'>+	int order;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	if (!mlx5_debugfs_root || dev-&gt;is_rep)</div><div class='del'>-		return;</div><div class='add'>+	ent = kzalloc(sizeof(*ent), GFP_KERNEL);</div><div class='add'>+	if (!ent)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='del'>-	cache-&gt;root = debugfs_create_dir("mr_cache", dev-&gt;mdev-&gt;priv.dbg_root);</div><div class='add'>+	ret = mlx5r_mkeys_init(ent);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto mkeys_err;</div><div class='add'>+	ent-&gt;rb_key = rb_key;</div><div class='add'>+	ent-&gt;dev = dev;</div><div class='add'>+	ent-&gt;is_tmp = !persistent_entry;</div><div class='add'>+</div><div class='add'>+	INIT_DELAYED_WORK(&amp;ent-&gt;dwork, delayed_cache_work_func);</div><div class='add'>+</div><div class='add'>+	ret = mlx5_cache_ent_insert(&amp;dev-&gt;cache, ent);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto ent_insert_err;</div><div class='add'>+</div><div class='add'>+	if (persistent_entry) {</div><div class='add'>+		if (rb_key.access_mode == MLX5_MKC_ACCESS_MODE_KSM)</div><div class='add'>+			order = MLX5_IMR_KSM_CACHE_ENTRY;</div><div class='add'>+		else</div><div class='add'>+			order = order_base_2(rb_key.ndescs) - 2;</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; MAX_MR_CACHE_ENTRIES; i++) {</div><div class='del'>-		ent = &amp;cache-&gt;ent[i];</div><div class='del'>-		sprintf(ent-&gt;name, "%d", ent-&gt;order);</div><div class='del'>-		dir = debugfs_create_dir(ent-&gt;name, cache-&gt;root);</div><div class='del'>-		debugfs_create_file("size", 0600, dir, ent, &amp;size_fops);</div><div class='del'>-		debugfs_create_file("limit", 0600, dir, ent, &amp;limit_fops);</div><div class='del'>-		debugfs_create_u32("cur", 0400, dir, &amp;ent-&gt;cur);</div><div class='del'>-		debugfs_create_u32("miss", 0600, dir, &amp;ent-&gt;miss);</div><div class='add'>+		if ((dev-&gt;mdev-&gt;profile.mask &amp; MLX5_PROF_MASK_MR_CACHE) &amp;&amp;</div><div class='add'>+		    !dev-&gt;is_rep &amp;&amp; mlx5_core_is_pf(dev-&gt;mdev) &amp;&amp;</div><div class='add'>+		    mlx5r_umr_can_load_pas(dev, 0))</div><div class='add'>+			ent-&gt;limit = dev-&gt;mdev-&gt;profile.mr_cache[order].limit;</div><div class='add'>+		else</div><div class='add'>+			ent-&gt;limit = 0;</div><div class='add'>+</div><div class='add'>+		mlx5_mkey_cache_debugfs_add_ent(dev, ent);</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	return ent;</div><div class='add'>+ent_insert_err:</div><div class='add'>+	mlx5r_mkeys_uninit(ent);</div><div class='add'>+mkeys_err:</div><div class='add'>+	kfree(ent);</div><div class='add'>+	return ERR_PTR(ret);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void delay_time_func(struct timer_list *t)</div><div class='add'>+static void mlx5r_destroy_cache_entries(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_dev *dev = from_timer(dev, t, delay_timer);</div><div class='del'>-</div><div class='del'>-	dev-&gt;fill_delay = 0;</div><div class='add'>+	struct rb_root *root = &amp;dev-&gt;cache.rb_root;</div><div class='add'>+	struct mlx5_cache_ent *ent;</div><div class='add'>+	struct rb_node *node;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;dev-&gt;cache.rb_lock);</div><div class='add'>+	node = rb_first(root);</div><div class='add'>+	while (node) {</div><div class='add'>+		ent = rb_entry(node, struct mlx5_cache_ent, node);</div><div class='add'>+		node = rb_next(node);</div><div class='add'>+		clean_keys(dev, ent);</div><div class='add'>+		rb_erase(&amp;ent-&gt;node, root);</div><div class='add'>+		mlx5r_mkeys_uninit(ent);</div><div class='add'>+		kfree(ent);</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;cache.rb_lock);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)</div><div class='add'>+int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_mr_cache *cache = &amp;dev-&gt;cache;</div><div class='add'>+	struct mlx5_mkey_cache *cache = &amp;dev-&gt;cache;</div><div class='add'>+	struct rb_root *root = &amp;dev-&gt;cache.rb_root;</div><div class='add'>+	struct mlx5r_cache_rb_key rb_key = {</div><div class='add'>+		.access_mode = MLX5_MKC_ACCESS_MODE_MTT,</div><div class='add'>+		.ph = MLX5_IB_NO_PH,</div><div class='add'>+	};</div><div class='ctx'> 	struct mlx5_cache_ent *ent;</div><div class='add'>+	struct rb_node *node;</div><div class='add'>+	int ret;</div><div class='ctx'> 	int i;</div><div class='ctx'> </div><div class='ctx'> 	mutex_init(&amp;dev-&gt;slow_path_mutex);</div><div class='add'>+	mutex_init(&amp;dev-&gt;cache.rb_lock);</div><div class='add'>+	dev-&gt;cache.rb_root = RB_ROOT;</div><div class='ctx'> 	cache-&gt;wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM);</div><div class='ctx'> 	if (!cache-&gt;wq) {</div><div class='ctx'> 		mlx5_ib_warn(dev, "failed to create work queue\n");</div><div class='hunk'>@@ -652,71 +981,79 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)</div><div class='ctx'> </div><div class='ctx'> 	mlx5_cmd_init_async_ctx(dev-&gt;mdev, &amp;dev-&gt;async_ctx);</div><div class='ctx'> 	timer_setup(&amp;dev-&gt;delay_timer, delay_time_func, 0);</div><div class='del'>-	for (i = 0; i &lt; MAX_MR_CACHE_ENTRIES; i++) {</div><div class='del'>-		ent = &amp;cache-&gt;ent[i];</div><div class='del'>-		INIT_LIST_HEAD(&amp;ent-&gt;head);</div><div class='del'>-		spin_lock_init(&amp;ent-&gt;lock);</div><div class='del'>-		ent-&gt;order = i + 2;</div><div class='del'>-		ent-&gt;dev = dev;</div><div class='del'>-		ent-&gt;limit = 0;</div><div class='del'>-</div><div class='del'>-		init_completion(&amp;ent-&gt;compl);</div><div class='del'>-		INIT_WORK(&amp;ent-&gt;work, cache_work_func);</div><div class='del'>-		INIT_DELAYED_WORK(&amp;ent-&gt;dwork, delayed_cache_work_func);</div><div class='del'>-</div><div class='del'>-		if (i &gt; MR_CACHE_LAST_STD_ENTRY) {</div><div class='del'>-			mlx5_odp_init_mr_cache_entry(ent);</div><div class='del'>-			continue;</div><div class='add'>+	mlx5_mkey_cache_debugfs_init(dev);</div><div class='add'>+	mutex_lock(&amp;cache-&gt;rb_lock);</div><div class='add'>+	for (i = 0; i &lt;= mkey_cache_max_order(dev); i++) {</div><div class='add'>+		rb_key.ndescs = MLX5_MR_CACHE_PERSISTENT_ENTRY_MIN_DESCS &lt;&lt; i;</div><div class='add'>+		ent = mlx5r_cache_create_ent_locked(dev, rb_key, true);</div><div class='add'>+		if (IS_ERR(ent)) {</div><div class='add'>+			ret = PTR_ERR(ent);</div><div class='add'>+			goto err;</div><div class='ctx'> 		}</div><div class='del'>-</div><div class='del'>-		if (ent-&gt;order &gt; mr_cache_max_order(dev))</div><div class='del'>-			continue;</div><div class='del'>-</div><div class='del'>-		ent-&gt;page = PAGE_SHIFT;</div><div class='del'>-		ent-&gt;xlt = (1 &lt;&lt; ent-&gt;order) * sizeof(struct mlx5_mtt) /</div><div class='del'>-			   MLX5_IB_UMR_OCTOWORD;</div><div class='del'>-		ent-&gt;access_mode = MLX5_MKC_ACCESS_MODE_MTT;</div><div class='del'>-		if ((dev-&gt;mdev-&gt;profile-&gt;mask &amp; MLX5_PROF_MASK_MR_CACHE) &amp;&amp;</div><div class='del'>-		    !dev-&gt;is_rep &amp;&amp;</div><div class='del'>-		    mlx5_core_is_pf(dev-&gt;mdev))</div><div class='del'>-			ent-&gt;limit = dev-&gt;mdev-&gt;profile-&gt;mr_cache[i].limit;</div><div class='del'>-		else</div><div class='del'>-			ent-&gt;limit = 0;</div><div class='del'>-		queue_work(cache-&gt;wq, &amp;ent-&gt;work);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	mlx5_mr_cache_debugfs_init(dev);</div><div class='add'>+	ret = mlx5_odp_init_mkey_cache(dev);</div><div class='add'>+	if (ret)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	mutex_unlock(&amp;cache-&gt;rb_lock);</div><div class='add'>+	for (node = rb_first(root); node; node = rb_next(node)) {</div><div class='add'>+		ent = rb_entry(node, struct mlx5_cache_ent, node);</div><div class='add'>+		spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+		queue_adjust_cache_locked(ent);</div><div class='add'>+		spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	mutex_unlock(&amp;cache-&gt;rb_lock);</div><div class='add'>+	mlx5_mkey_cache_debugfs_cleanup(dev);</div><div class='add'>+	mlx5r_destroy_cache_entries(dev);</div><div class='add'>+	destroy_workqueue(cache-&gt;wq);</div><div class='add'>+	mlx5_ib_warn(dev, "failed to create mkey cache entry\n");</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)</div><div class='add'>+void mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	int i;</div><div class='add'>+	struct rb_root *root = &amp;dev-&gt;cache.rb_root;</div><div class='add'>+	struct mlx5_cache_ent *ent;</div><div class='add'>+	struct rb_node *node;</div><div class='ctx'> </div><div class='ctx'> 	if (!dev-&gt;cache.wq)</div><div class='del'>-		return 0;</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-	dev-&gt;cache.stopped = 1;</div><div class='add'>+	mutex_lock(&amp;dev-&gt;cache.rb_lock);</div><div class='add'>+	for (node = rb_first(root); node; node = rb_next(node)) {</div><div class='add'>+		ent = rb_entry(node, struct mlx5_cache_ent, node);</div><div class='add'>+		spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+		ent-&gt;disabled = true;</div><div class='add'>+		spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+		cancel_delayed_work(&amp;ent-&gt;dwork);</div><div class='add'>+	}</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;cache.rb_lock);</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * After all entries are disabled and will not reschedule on WQ,</div><div class='add'>+	 * flush it and all async commands.</div><div class='add'>+	 */</div><div class='ctx'> 	flush_workqueue(dev-&gt;cache.wq);</div><div class='ctx'> </div><div class='del'>-	mlx5_mr_cache_debugfs_cleanup(dev);</div><div class='add'>+	mlx5_mkey_cache_debugfs_cleanup(dev);</div><div class='ctx'> 	mlx5_cmd_cleanup_async_ctx(&amp;dev-&gt;async_ctx);</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; MAX_MR_CACHE_ENTRIES; i++)</div><div class='del'>-		clean_keys(dev, i);</div><div class='add'>+	/* At this point all entries are disabled and have no concurrent work. */</div><div class='add'>+	mlx5r_destroy_cache_entries(dev);</div><div class='ctx'> </div><div class='ctx'> 	destroy_workqueue(dev-&gt;cache.wq);</div><div class='del'>-	del_timer_sync(&amp;dev-&gt;delay_timer);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='add'>+	timer_delete_sync(&amp;dev-&gt;delay_timer);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='ctx'> 	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);</div><div class='del'>-	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='ctx'> 	struct mlx5_ib_mr *mr;</div><div class='ctx'> 	void *mkc;</div><div class='ctx'> 	u32 *in;</div><div class='hunk'>@@ -735,18 +1072,12 @@ struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc)</div><div class='ctx'> 	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);</div><div class='del'>-	MLX5_SET(mkc, mkc, a, !!(acc &amp; IB_ACCESS_REMOTE_ATOMIC));</div><div class='del'>-	MLX5_SET(mkc, mkc, rw, !!(acc &amp; IB_ACCESS_REMOTE_WRITE));</div><div class='del'>-	MLX5_SET(mkc, mkc, rr, !!(acc &amp; IB_ACCESS_REMOTE_READ));</div><div class='del'>-	MLX5_SET(mkc, mkc, lw, !!(acc &amp; IB_ACCESS_LOCAL_WRITE));</div><div class='del'>-	MLX5_SET(mkc, mkc, lr, 1);</div><div class='del'>-</div><div class='ctx'> 	MLX5_SET(mkc, mkc, length64, 1);</div><div class='del'>-	MLX5_SET(mkc, mkc, pd, to_mpd(pd)-&gt;pdn);</div><div class='del'>-	MLX5_SET(mkc, mkc, qpn, 0xffffff);</div><div class='del'>-	MLX5_SET64(mkc, mkc, start_addr, 0);</div><div class='add'>+	set_mkc_access_pd_addr_fields(mkc, acc | IB_ACCESS_RELAXED_ORDERING, 0,</div><div class='add'>+				      pd);</div><div class='add'>+	MLX5_SET(mkc, mkc, ma_translation_mode, MLX5_CAP_GEN(dev-&gt;mdev, ats));</div><div class='ctx'> </div><div class='del'>-	err = mlx5_core_create_mkey(mdev, &amp;mr-&gt;mmkey, in, inlen);</div><div class='add'>+	err = mlx5_ib_create_mkey(dev, &amp;mr-&gt;mmkey, in, inlen);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_in;</div><div class='ctx'> </div><div class='hunk'>@@ -778,295 +1109,154 @@ static int get_octo_len(u64 addr, u64 len, int page_shift)</div><div class='ctx'> 	return (npages + 1) / 2;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mr_cache_max_order(struct mlx5_ib_dev *dev)</div><div class='add'>+static int mkey_cache_max_order(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	if (MLX5_CAP_GEN(dev-&gt;mdev, umr_extended_translation_offset))</div><div class='del'>-		return MR_CACHE_LAST_STD_ENTRY + 2;</div><div class='add'>+		return MKEY_CACHE_LAST_STD_ENTRY;</div><div class='ctx'> 	return MLX5_MAX_UMR_SHIFT;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mr_umem_get(struct mlx5_ib_dev *dev, struct ib_udata *udata,</div><div class='del'>-		       u64 start, u64 length, int access_flags,</div><div class='del'>-		       struct ib_umem **umem, int *npages, int *page_shift,</div><div class='del'>-		       int *ncont, int *order)</div><div class='del'>-{</div><div class='del'>-	struct ib_umem *u;</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	*umem = NULL;</div><div class='del'>-</div><div class='del'>-	u = ib_umem_get(udata, start, length, access_flags, 0);</div><div class='del'>-	err = PTR_ERR_OR_ZERO(u);</div><div class='del'>-	if (err) {</div><div class='del'>-		mlx5_ib_dbg(dev, "umem get failed (%d)\n", err);</div><div class='del'>-		return err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mlx5_ib_cont_pages(u, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages,</div><div class='del'>-			   page_shift, ncont, order);</div><div class='del'>-	if (!*npages) {</div><div class='del'>-		mlx5_ib_warn(dev, "avoid zero region\n");</div><div class='del'>-		ib_umem_release(u);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	*umem = u;</div><div class='del'>-</div><div class='del'>-	mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",</div><div class='del'>-		    *npages, *ncont, *order, *page_shift);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_umr_context *context =</div><div class='del'>-		container_of(wc-&gt;wr_cqe, struct mlx5_ib_umr_context, cqe);</div><div class='del'>-</div><div class='del'>-	context-&gt;status = wc-&gt;status;</div><div class='del'>-	complete(&amp;context-&gt;done);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context)</div><div class='add'>+static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,</div><div class='add'>+			  u64 length, int access_flags, u64 iova)</div><div class='ctx'> {</div><div class='del'>-	context-&gt;cqe.done = mlx5_ib_umr_done;</div><div class='del'>-	context-&gt;status = -1;</div><div class='del'>-	init_completion(&amp;context-&gt;done);</div><div class='add'>+	mr-&gt;ibmr.lkey = mr-&gt;mmkey.key;</div><div class='add'>+	mr-&gt;ibmr.rkey = mr-&gt;mmkey.key;</div><div class='add'>+	mr-&gt;ibmr.length = length;</div><div class='add'>+	mr-&gt;ibmr.device = &amp;dev-&gt;ib_dev;</div><div class='add'>+	mr-&gt;ibmr.iova = iova;</div><div class='add'>+	mr-&gt;access_flags = access_flags;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_ib_post_send_wait(struct mlx5_ib_dev *dev,</div><div class='del'>-				  struct mlx5_umr_wr *umrwr)</div><div class='add'>+static unsigned int mlx5_umem_dmabuf_default_pgsz(struct ib_umem *umem,</div><div class='add'>+						  u64 iova)</div><div class='ctx'> {</div><div class='del'>-	struct umr_common *umrc = &amp;dev-&gt;umrc;</div><div class='del'>-	const struct ib_send_wr *bad;</div><div class='del'>-	int err;</div><div class='del'>-	struct mlx5_ib_umr_context umr_context;</div><div class='del'>-</div><div class='del'>-	mlx5_ib_init_umr_context(&amp;umr_context);</div><div class='del'>-	umrwr-&gt;wr.wr_cqe = &amp;umr_context.cqe;</div><div class='del'>-</div><div class='del'>-	down(&amp;umrc-&gt;sem);</div><div class='del'>-	err = ib_post_send(umrc-&gt;qp, &amp;umrwr-&gt;wr, &amp;bad);</div><div class='del'>-	if (err) {</div><div class='del'>-		mlx5_ib_warn(dev, "UMR post send failed, err %d\n", err);</div><div class='del'>-	} else {</div><div class='del'>-		wait_for_completion(&amp;umr_context.done);</div><div class='del'>-		if (umr_context.status != IB_WC_SUCCESS) {</div><div class='del'>-			mlx5_ib_warn(dev, "reg umr failed (%u)\n",</div><div class='del'>-				     umr_context.status);</div><div class='del'>-			err = -EFAULT;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-	up(&amp;umrc-&gt;sem);</div><div class='del'>-	return err;</div><div class='add'>+	/*</div><div class='add'>+	 * The alignment of iova has already been checked upon entering</div><div class='add'>+	 * UVERBS_METHOD_REG_DMABUF_MR</div><div class='add'>+	 */</div><div class='add'>+	umem-&gt;iova = iova;</div><div class='add'>+	return PAGE_SIZE;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct mlx5_ib_mr *alloc_mr_from_cache(</div><div class='del'>-				  struct ib_pd *pd, struct ib_umem *umem,</div><div class='del'>-				  u64 virt_addr, u64 len, int npages,</div><div class='del'>-				  int page_shift, int order, int access_flags)</div><div class='add'>+static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd,</div><div class='add'>+					     struct ib_umem *umem, u64 iova,</div><div class='add'>+					     int access_flags, int access_mode,</div><div class='add'>+					     u16 st_index, u8 ph)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='add'>+	struct mlx5r_cache_rb_key rb_key = {};</div><div class='add'>+	struct mlx5_cache_ent *ent;</div><div class='ctx'> 	struct mlx5_ib_mr *mr;</div><div class='del'>-	int err = 0;</div><div class='del'>-	int i;</div><div class='add'>+	unsigned long page_size;</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; 1; i++) {</div><div class='del'>-		mr = alloc_cached_mr(dev, order);</div><div class='del'>-		if (mr)</div><div class='del'>-			break;</div><div class='add'>+	if (umem-&gt;is_dmabuf)</div><div class='add'>+		page_size = mlx5_umem_dmabuf_default_pgsz(umem, iova);</div><div class='add'>+	else</div><div class='add'>+		page_size = mlx5_umem_mkc_find_best_pgsz(dev, umem, iova,</div><div class='add'>+							 access_mode);</div><div class='add'>+	if (WARN_ON(!page_size))</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='ctx'> </div><div class='del'>-		err = add_keys(dev, order2idx(dev, order), 1);</div><div class='del'>-		if (err &amp;&amp; err != -EAGAIN) {</div><div class='del'>-			mlx5_ib_warn(dev, "add_keys failed, err %d\n", err);</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='add'>+	rb_key.access_mode = access_mode;</div><div class='add'>+	rb_key.ndescs = ib_umem_num_dma_blocks(umem, page_size);</div><div class='add'>+	rb_key.ats = mlx5_umem_needs_ats(dev, umem, access_flags);</div><div class='add'>+	rb_key.access_flags = get_unchangeable_access_flags(dev, access_flags);</div><div class='add'>+	rb_key.st_index = st_index;</div><div class='add'>+	rb_key.ph = ph;</div><div class='add'>+	ent = mkey_cache_ent_from_rb_key(dev, rb_key);</div><div class='add'>+	/*</div><div class='add'>+	 * If the MR can't come from the cache then synchronously create an uncached</div><div class='add'>+	 * one.</div><div class='add'>+	 */</div><div class='add'>+	if (!ent) {</div><div class='add'>+		mutex_lock(&amp;dev-&gt;slow_path_mutex);</div><div class='add'>+		mr = reg_create(pd, umem, iova, access_flags, page_size, false, access_mode,</div><div class='add'>+				st_index, ph);</div><div class='add'>+		mutex_unlock(&amp;dev-&gt;slow_path_mutex);</div><div class='add'>+		if (IS_ERR(mr))</div><div class='add'>+			return mr;</div><div class='add'>+		mr-&gt;mmkey.rb_key = rb_key;</div><div class='add'>+		mr-&gt;mmkey.cacheable = true;</div><div class='add'>+		return mr;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!mr)</div><div class='del'>-		return ERR_PTR(-EAGAIN);</div><div class='add'>+	mr = _mlx5_mr_cache_alloc(dev, ent);</div><div class='add'>+	if (IS_ERR(mr))</div><div class='add'>+		return mr;</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;ibmr.pd = pd;</div><div class='ctx'> 	mr-&gt;umem = umem;</div><div class='del'>-	mr-&gt;access_flags = access_flags;</div><div class='del'>-	mr-&gt;desc_size = sizeof(struct mlx5_mtt);</div><div class='del'>-	mr-&gt;mmkey.iova = virt_addr;</div><div class='del'>-	mr-&gt;mmkey.size = len;</div><div class='del'>-	mr-&gt;mmkey.pd = to_mpd(pd)-&gt;pdn;</div><div class='add'>+	mr-&gt;page_shift = order_base_2(page_size);</div><div class='add'>+	set_mr_fields(dev, mr, umem-&gt;length, access_flags, iova);</div><div class='ctx'> </div><div class='ctx'> 	return mr;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline int populate_xlt(struct mlx5_ib_mr *mr, int idx, int npages,</div><div class='del'>-			       void *xlt, int page_shift, size_t size,</div><div class='del'>-			       int flags)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_dev *dev = mr-&gt;dev;</div><div class='del'>-	struct ib_umem *umem = mr-&gt;umem;</div><div class='del'>-</div><div class='del'>-	if (flags &amp; MLX5_IB_UPD_XLT_INDIRECT) {</div><div class='del'>-		if (!umr_can_use_indirect_mkey(dev))</div><div class='del'>-			return -EPERM;</div><div class='del'>-		mlx5_odp_populate_klm(xlt, idx, npages, mr, flags);</div><div class='del'>-		return npages;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	npages = min_t(size_t, npages, ib_umem_num_pages(umem) - idx);</div><div class='del'>-</div><div class='del'>-	if (!(flags &amp; MLX5_IB_UPD_XLT_ZAP)) {</div><div class='del'>-		__mlx5_ib_populate_pas(dev, umem, page_shift,</div><div class='del'>-				       idx, npages, xlt,</div><div class='del'>-				       MLX5_IB_MTT_PRESENT);</div><div class='del'>-		/* Clear padding after the pages</div><div class='del'>-		 * brought from the umem.</div><div class='del'>-		 */</div><div class='del'>-		memset(xlt + (npages * sizeof(struct mlx5_mtt)), 0,</div><div class='del'>-		       size - npages * sizeof(struct mlx5_mtt));</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return npages;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-#define MLX5_MAX_UMR_CHUNK ((1 &lt;&lt; (MLX5_MAX_UMR_SHIFT + 4)) - \</div><div class='del'>-			    MLX5_UMR_MTT_ALIGNMENT)</div><div class='del'>-#define MLX5_SPARE_UMR_CHUNK 0x10000</div><div class='del'>-</div><div class='del'>-int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,</div><div class='del'>-		       int page_shift, int flags)</div><div class='add'>+static struct ib_mr *</div><div class='add'>+reg_create_crossing_vhca_mr(struct ib_pd *pd, u64 iova, u64 length, int access_flags,</div><div class='add'>+			    u32 crossed_lkey)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_dev *dev = mr-&gt;dev;</div><div class='del'>-	struct device *ddev = dev-&gt;ib_dev.dev.parent;</div><div class='del'>-	int size;</div><div class='del'>-	void *xlt;</div><div class='del'>-	dma_addr_t dma;</div><div class='del'>-	struct mlx5_umr_wr wr;</div><div class='del'>-	struct ib_sge sg;</div><div class='del'>-	int err = 0;</div><div class='del'>-	int desc_size = (flags &amp; MLX5_IB_UPD_XLT_INDIRECT)</div><div class='del'>-			       ? sizeof(struct mlx5_klm)</div><div class='del'>-			       : sizeof(struct mlx5_mtt);</div><div class='del'>-	const int page_align = MLX5_UMR_MTT_ALIGNMENT / desc_size;</div><div class='del'>-	const int page_mask = page_align - 1;</div><div class='del'>-	size_t pages_mapped = 0;</div><div class='del'>-	size_t pages_to_map = 0;</div><div class='del'>-	size_t pages_iter = 0;</div><div class='del'>-	gfp_t gfp;</div><div class='del'>-	bool use_emergency_page = false;</div><div class='del'>-</div><div class='del'>-	if ((flags &amp; MLX5_IB_UPD_XLT_INDIRECT) &amp;&amp;</div><div class='del'>-	    !umr_can_use_indirect_mkey(dev))</div><div class='del'>-		return -EPERM;</div><div class='del'>-</div><div class='del'>-	/* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes,</div><div class='del'>-	 * so we need to align the offset and length accordingly</div><div class='del'>-	 */</div><div class='del'>-	if (idx &amp; page_mask) {</div><div class='del'>-		npages += idx &amp; page_mask;</div><div class='del'>-		idx &amp;= ~page_mask;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	gfp = flags &amp; MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC : GFP_KERNEL;</div><div class='del'>-	gfp |= __GFP_ZERO | __GFP_NOWARN;</div><div class='del'>-</div><div class='del'>-	pages_to_map = ALIGN(npages, page_align);</div><div class='del'>-	size = desc_size * pages_to_map;</div><div class='del'>-	size = min_t(int, size, MLX5_MAX_UMR_CHUNK);</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='add'>+	int access_mode = MLX5_MKC_ACCESS_MODE_CROSSING;</div><div class='add'>+	struct mlx5_ib_mr *mr;</div><div class='add'>+	void *mkc;</div><div class='add'>+	int inlen;</div><div class='add'>+	u32 *in;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-	xlt = (void *)__get_free_pages(gfp, get_order(size));</div><div class='del'>-	if (!xlt &amp;&amp; size &gt; MLX5_SPARE_UMR_CHUNK) {</div><div class='del'>-		mlx5_ib_dbg(dev, "Failed to allocate %d bytes of order %d. fallback to spare UMR allocation od %d bytes\n",</div><div class='del'>-			    size, get_order(size), MLX5_SPARE_UMR_CHUNK);</div><div class='add'>+	if (!MLX5_CAP_GEN(dev-&gt;mdev, crossing_vhca_mkey))</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='ctx'> </div><div class='del'>-		size = MLX5_SPARE_UMR_CHUNK;</div><div class='del'>-		xlt = (void *)__get_free_pages(gfp, get_order(size));</div><div class='del'>-	}</div><div class='add'>+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='add'>+	if (!mr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='del'>-	if (!xlt) {</div><div class='del'>-		mlx5_ib_warn(dev, "Using XLT emergency buffer\n");</div><div class='del'>-		xlt = (void *)mlx5_ib_get_xlt_emergency_page();</div><div class='del'>-		size = PAGE_SIZE;</div><div class='del'>-		memset(xlt, 0, size);</div><div class='del'>-		use_emergency_page = true;</div><div class='del'>-	}</div><div class='del'>-	pages_iter = size / desc_size;</div><div class='del'>-	dma = dma_map_single(ddev, xlt, size, DMA_TO_DEVICE);</div><div class='del'>-	if (dma_mapping_error(ddev, dma)) {</div><div class='del'>-		mlx5_ib_err(dev, "unable to map DMA during XLT update.\n");</div><div class='add'>+	inlen = MLX5_ST_SZ_BYTES(create_mkey_in);</div><div class='add'>+	in = kvzalloc(inlen, GFP_KERNEL);</div><div class='add'>+	if (!in) {</div><div class='ctx'> 		err = -ENOMEM;</div><div class='del'>-		goto free_xlt;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	sg.addr = dma;</div><div class='del'>-	sg.lkey = dev-&gt;umrc.pd-&gt;local_dma_lkey;</div><div class='del'>-</div><div class='del'>-	memset(&amp;wr, 0, sizeof(wr));</div><div class='del'>-	wr.wr.send_flags = MLX5_IB_SEND_UMR_UPDATE_XLT;</div><div class='del'>-	if (!(flags &amp; MLX5_IB_UPD_XLT_ENABLE))</div><div class='del'>-		wr.wr.send_flags |= MLX5_IB_SEND_UMR_FAIL_IF_FREE;</div><div class='del'>-	wr.wr.sg_list = &amp;sg;</div><div class='del'>-	wr.wr.num_sge = 1;</div><div class='del'>-	wr.wr.opcode = MLX5_IB_WR_UMR;</div><div class='del'>-</div><div class='del'>-	wr.pd = mr-&gt;ibmr.pd;</div><div class='del'>-	wr.mkey = mr-&gt;mmkey.key;</div><div class='del'>-	wr.length = mr-&gt;mmkey.size;</div><div class='del'>-	wr.virt_addr = mr-&gt;mmkey.iova;</div><div class='del'>-	wr.access_flags = mr-&gt;access_flags;</div><div class='del'>-	wr.page_shift = page_shift;</div><div class='del'>-</div><div class='del'>-	for (pages_mapped = 0;</div><div class='del'>-	     pages_mapped &lt; pages_to_map &amp;&amp; !err;</div><div class='del'>-	     pages_mapped += pages_iter, idx += pages_iter) {</div><div class='del'>-		npages = min_t(int, pages_iter, pages_to_map - pages_mapped);</div><div class='del'>-		dma_sync_single_for_cpu(ddev, dma, size, DMA_TO_DEVICE);</div><div class='del'>-		npages = populate_xlt(mr, idx, npages, xlt,</div><div class='del'>-				      page_shift, size, flags);</div><div class='del'>-</div><div class='del'>-		dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE);</div><div class='del'>-</div><div class='del'>-		sg.length = ALIGN(npages * desc_size,</div><div class='del'>-				  MLX5_UMR_MTT_ALIGNMENT);</div><div class='del'>-</div><div class='del'>-		if (pages_mapped + pages_iter &gt;= pages_to_map) {</div><div class='del'>-			if (flags &amp; MLX5_IB_UPD_XLT_ENABLE)</div><div class='del'>-				wr.wr.send_flags |=</div><div class='del'>-					MLX5_IB_SEND_UMR_ENABLE_MR |</div><div class='del'>-					MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS |</div><div class='del'>-					MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;</div><div class='del'>-			if (flags &amp; MLX5_IB_UPD_XLT_PD ||</div><div class='del'>-			    flags &amp; MLX5_IB_UPD_XLT_ACCESS)</div><div class='del'>-				wr.wr.send_flags |=</div><div class='del'>-					MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;</div><div class='del'>-			if (flags &amp; MLX5_IB_UPD_XLT_ADDR)</div><div class='del'>-				wr.wr.send_flags |=</div><div class='del'>-					MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		wr.offset = idx * desc_size;</div><div class='del'>-		wr.xlt_size = sg.length;</div><div class='del'>-</div><div class='del'>-		err = mlx5_ib_post_send_wait(dev, &amp;wr);</div><div class='add'>+		goto err_1;</div><div class='ctx'> 	}</div><div class='del'>-	dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE);</div><div class='ctx'> </div><div class='del'>-free_xlt:</div><div class='del'>-	if (use_emergency_page)</div><div class='del'>-		mlx5_ib_put_xlt_emergency_page();</div><div class='del'>-	else</div><div class='del'>-		free_pages((unsigned long)xlt, get_order(size));</div><div class='add'>+	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);</div><div class='add'>+	MLX5_SET(mkc, mkc, crossing_target_vhca_id,</div><div class='add'>+		 MLX5_CAP_GEN(dev-&gt;mdev, vhca_id));</div><div class='add'>+	MLX5_SET(mkc, mkc, translations_octword_size, crossed_lkey);</div><div class='add'>+	MLX5_SET(mkc, mkc, access_mode_1_0, access_mode &amp; 0x3);</div><div class='add'>+	MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode &gt;&gt; 2) &amp; 0x7);</div><div class='add'>+</div><div class='add'>+	/* for this crossing mkey IOVA should be 0 and len should be IOVA + len */</div><div class='add'>+	set_mkc_access_pd_addr_fields(mkc, access_flags, 0, pd);</div><div class='add'>+	MLX5_SET64(mkc, mkc, len, iova + length);</div><div class='add'>+</div><div class='add'>+	MLX5_SET(mkc, mkc, free, 0);</div><div class='add'>+	MLX5_SET(mkc, mkc, umr_en, 0);</div><div class='add'>+	err = mlx5_ib_create_mkey(dev, &amp;mr-&gt;mmkey, in, inlen);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_2;</div><div class='ctx'> </div><div class='del'>-	return err;</div><div class='add'>+	mr-&gt;mmkey.type = MLX5_MKEY_MR;</div><div class='add'>+	set_mr_fields(dev, mr, length, access_flags, iova);</div><div class='add'>+	mr-&gt;ibmr.pd = pd;</div><div class='add'>+	kvfree(in);</div><div class='add'>+	mlx5_ib_dbg(dev, "crossing mkey = 0x%x\n", mr-&gt;mmkey.key);</div><div class='add'>+</div><div class='add'>+	return &amp;mr-&gt;ibmr;</div><div class='add'>+err_2:</div><div class='add'>+	kvfree(in);</div><div class='add'>+err_1:</div><div class='add'>+	kfree(mr);</div><div class='add'>+	return ERR_PTR(err);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * If ibmr is NULL it will be allocated by reg_create.</div><div class='ctx'>  * Else, the given ibmr will be used.</div><div class='ctx'>  */</div><div class='del'>-static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,</div><div class='del'>-				     u64 virt_addr, u64 length,</div><div class='del'>-				     struct ib_umem *umem, int npages,</div><div class='del'>-				     int page_shift, int access_flags,</div><div class='del'>-				     bool populate)</div><div class='add'>+static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,</div><div class='add'>+				     u64 iova, int access_flags,</div><div class='add'>+				     unsigned long page_size, bool populate,</div><div class='add'>+				     int access_mode, u16 st_index, u8 ph)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='ctx'> 	struct mlx5_ib_mr *mr;</div><div class='hunk'>@@ -1075,63 +1265,88 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,</div><div class='ctx'> 	int inlen;</div><div class='ctx'> 	u32 *in;</div><div class='ctx'> 	int err;</div><div class='del'>-	bool pg_cap = !!(MLX5_CAP_GEN(dev-&gt;mdev, pg));</div><div class='add'>+	bool pg_cap = !!(MLX5_CAP_GEN(dev-&gt;mdev, pg)) &amp;&amp;</div><div class='add'>+		(access_mode == MLX5_MKC_ACCESS_MODE_MTT) &amp;&amp;</div><div class='add'>+		(ph == MLX5_IB_NO_PH);</div><div class='add'>+	bool ksm_mode = (access_mode == MLX5_MKC_ACCESS_MODE_KSM);</div><div class='ctx'> </div><div class='del'>-	mr = ibmr ? to_mmr(ibmr) : kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='add'>+	if (!page_size)</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='ctx'> 	if (!mr)</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;ibmr.pd = pd;</div><div class='ctx'> 	mr-&gt;access_flags = access_flags;</div><div class='add'>+	mr-&gt;page_shift = order_base_2(page_size);</div><div class='ctx'> </div><div class='ctx'> 	inlen = MLX5_ST_SZ_BYTES(create_mkey_in);</div><div class='ctx'> 	if (populate)</div><div class='del'>-		inlen += sizeof(*pas) * roundup(npages, 2);</div><div class='add'>+		inlen += sizeof(*pas) *</div><div class='add'>+			 roundup(ib_umem_num_dma_blocks(umem, page_size), 2);</div><div class='ctx'> 	in = kvzalloc(inlen, GFP_KERNEL);</div><div class='ctx'> 	if (!in) {</div><div class='ctx'> 		err = -ENOMEM;</div><div class='ctx'> 		goto err_1;</div><div class='ctx'> 	}</div><div class='ctx'> 	pas = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);</div><div class='del'>-	if (populate &amp;&amp; !(access_flags &amp; IB_ACCESS_ON_DEMAND))</div><div class='del'>-		mlx5_ib_populate_pas(dev, umem, page_shift, pas,</div><div class='add'>+	if (populate) {</div><div class='add'>+		if (WARN_ON(access_flags &amp; IB_ACCESS_ON_DEMAND || ksm_mode)) {</div><div class='add'>+			err = -EINVAL;</div><div class='add'>+			goto err_2;</div><div class='add'>+		}</div><div class='add'>+		mlx5_ib_populate_pas(umem, 1UL &lt;&lt; mr-&gt;page_shift, pas,</div><div class='ctx'> 				     pg_cap ? MLX5_IB_MTT_PRESENT : 0);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	/* The pg_access bit allows setting the access flags</div><div class='del'>-	 * in the page list submitted with the command. */</div><div class='add'>+	 * in the page list submitted with the command.</div><div class='add'>+	 */</div><div class='ctx'> 	MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap));</div><div class='ctx'> </div><div class='ctx'> 	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);</div><div class='add'>+	set_mkc_access_pd_addr_fields(mkc, access_flags, iova,</div><div class='add'>+				      populate ? pd : dev-&gt;umrc.pd);</div><div class='add'>+	/* In case a data direct flow, overwrite the pdn field by its internal kernel PD */</div><div class='add'>+	if (umem-&gt;is_dmabuf &amp;&amp; ksm_mode)</div><div class='add'>+		MLX5_SET(mkc, mkc, pd, dev-&gt;ddr.pdn);</div><div class='add'>+</div><div class='ctx'> 	MLX5_SET(mkc, mkc, free, !populate);</div><div class='del'>-	MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);</div><div class='del'>-	MLX5_SET(mkc, mkc, a, !!(access_flags &amp; IB_ACCESS_REMOTE_ATOMIC));</div><div class='del'>-	MLX5_SET(mkc, mkc, rw, !!(access_flags &amp; IB_ACCESS_REMOTE_WRITE));</div><div class='del'>-	MLX5_SET(mkc, mkc, rr, !!(access_flags &amp; IB_ACCESS_REMOTE_READ));</div><div class='del'>-	MLX5_SET(mkc, mkc, lw, !!(access_flags &amp; IB_ACCESS_LOCAL_WRITE));</div><div class='del'>-	MLX5_SET(mkc, mkc, lr, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, access_mode_1_0, access_mode);</div><div class='ctx'> 	MLX5_SET(mkc, mkc, umr_en, 1);</div><div class='ctx'> </div><div class='del'>-	MLX5_SET64(mkc, mkc, start_addr, virt_addr);</div><div class='del'>-	MLX5_SET64(mkc, mkc, len, length);</div><div class='del'>-	MLX5_SET(mkc, mkc, pd, to_mpd(pd)-&gt;pdn);</div><div class='add'>+	MLX5_SET64(mkc, mkc, len, umem-&gt;length);</div><div class='ctx'> 	MLX5_SET(mkc, mkc, bsf_octword_size, 0);</div><div class='del'>-	MLX5_SET(mkc, mkc, translations_octword_size,</div><div class='del'>-		 get_octo_len(virt_addr, length, page_shift));</div><div class='del'>-	MLX5_SET(mkc, mkc, log_page_size, page_shift);</div><div class='del'>-	MLX5_SET(mkc, mkc, qpn, 0xffffff);</div><div class='add'>+	if (ksm_mode)</div><div class='add'>+		MLX5_SET(mkc, mkc, translations_octword_size,</div><div class='add'>+			 get_octo_len(iova, umem-&gt;length, mr-&gt;page_shift) * 2);</div><div class='add'>+	else</div><div class='add'>+		MLX5_SET(mkc, mkc, translations_octword_size,</div><div class='add'>+			 get_octo_len(iova, umem-&gt;length, mr-&gt;page_shift));</div><div class='add'>+	MLX5_SET(mkc, mkc, log_page_size, mr-&gt;page_shift);</div><div class='add'>+	if (mlx5_umem_needs_ats(dev, umem, access_flags))</div><div class='add'>+		MLX5_SET(mkc, mkc, ma_translation_mode, 1);</div><div class='ctx'> 	if (populate) {</div><div class='ctx'> 		MLX5_SET(create_mkey_in, in, translations_octword_actual_size,</div><div class='del'>-			 get_octo_len(virt_addr, length, page_shift));</div><div class='add'>+			 get_octo_len(iova, umem-&gt;length, mr-&gt;page_shift));</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ph != MLX5_IB_NO_PH) {</div><div class='add'>+		MLX5_SET(mkc, mkc, pcie_tph_en, 1);</div><div class='add'>+		MLX5_SET(mkc, mkc, pcie_tph_ph, ph);</div><div class='add'>+		if (st_index != MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX)</div><div class='add'>+			MLX5_SET(mkc, mkc, pcie_tph_steering_tag_index, st_index);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	err = mlx5_core_create_mkey(dev-&gt;mdev, &amp;mr-&gt;mmkey, in, inlen);</div><div class='add'>+	err = mlx5_ib_create_mkey(dev, &amp;mr-&gt;mmkey, in, inlen);</div><div class='ctx'> 	if (err) {</div><div class='ctx'> 		mlx5_ib_warn(dev, "create mkey failed\n");</div><div class='ctx'> 		goto err_2;</div><div class='ctx'> 	}</div><div class='ctx'> 	mr-&gt;mmkey.type = MLX5_MKEY_MR;</div><div class='del'>-	mr-&gt;desc_size = sizeof(struct mlx5_mtt);</div><div class='del'>-	mr-&gt;dev = dev;</div><div class='add'>+	mr-&gt;mmkey.ndescs = get_octo_len(iova, umem-&gt;length, mr-&gt;page_shift);</div><div class='add'>+	mr-&gt;umem = umem;</div><div class='add'>+	set_mr_fields(dev, mr, umem-&gt;length, access_flags, iova);</div><div class='ctx'> 	kvfree(in);</div><div class='ctx'> </div><div class='ctx'> 	mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr-&gt;mmkey.key);</div><div class='hunk'>@@ -1140,31 +1355,16 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,</div><div class='ctx'> </div><div class='ctx'> err_2:</div><div class='ctx'> 	kvfree(in);</div><div class='del'>-</div><div class='ctx'> err_1:</div><div class='del'>-	if (!ibmr)</div><div class='del'>-		kfree(mr);</div><div class='del'>-</div><div class='add'>+	kfree(mr);</div><div class='ctx'> 	return ERR_PTR(err);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,</div><div class='del'>-			  int npages, u64 length, int access_flags)</div><div class='del'>-{</div><div class='del'>-	mr-&gt;npages = npages;</div><div class='del'>-	atomic_add(npages, &amp;dev-&gt;mdev-&gt;priv.reg_pages);</div><div class='del'>-	mr-&gt;ibmr.lkey = mr-&gt;mmkey.key;</div><div class='del'>-	mr-&gt;ibmr.rkey = mr-&gt;mmkey.key;</div><div class='del'>-	mr-&gt;ibmr.length = length;</div><div class='del'>-	mr-&gt;access_flags = access_flags;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static struct ib_mr *mlx5_ib_get_dm_mr(struct ib_pd *pd, u64 start_addr,</div><div class='ctx'> 				       u64 length, int acc, int mode)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='ctx'> 	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);</div><div class='del'>-	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='ctx'> 	struct mlx5_ib_mr *mr;</div><div class='ctx'> 	void *mkc;</div><div class='ctx'> 	u32 *in;</div><div class='hunk'>@@ -1184,25 +1384,16 @@ static struct ib_mr *mlx5_ib_get_dm_mr(struct ib_pd *pd, u64 start_addr,</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(mkc, mkc, access_mode_1_0, mode &amp; 0x3);</div><div class='ctx'> 	MLX5_SET(mkc, mkc, access_mode_4_2, (mode &gt;&gt; 2) &amp; 0x7);</div><div class='del'>-	MLX5_SET(mkc, mkc, a, !!(acc &amp; IB_ACCESS_REMOTE_ATOMIC));</div><div class='del'>-	MLX5_SET(mkc, mkc, rw, !!(acc &amp; IB_ACCESS_REMOTE_WRITE));</div><div class='del'>-	MLX5_SET(mkc, mkc, rr, !!(acc &amp; IB_ACCESS_REMOTE_READ));</div><div class='del'>-	MLX5_SET(mkc, mkc, lw, !!(acc &amp; IB_ACCESS_LOCAL_WRITE));</div><div class='del'>-	MLX5_SET(mkc, mkc, lr, 1);</div><div class='del'>-</div><div class='ctx'> 	MLX5_SET64(mkc, mkc, len, length);</div><div class='del'>-	MLX5_SET(mkc, mkc, pd, to_mpd(pd)-&gt;pdn);</div><div class='del'>-	MLX5_SET(mkc, mkc, qpn, 0xffffff);</div><div class='del'>-	MLX5_SET64(mkc, mkc, start_addr, start_addr);</div><div class='add'>+	set_mkc_access_pd_addr_fields(mkc, acc, start_addr, pd);</div><div class='ctx'> </div><div class='del'>-	err = mlx5_core_create_mkey(mdev, &amp;mr-&gt;mmkey, in, inlen);</div><div class='add'>+	err = mlx5_ib_create_mkey(dev, &amp;mr-&gt;mmkey, in, inlen);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_in;</div><div class='ctx'> </div><div class='ctx'> 	kfree(in);</div><div class='ctx'> </div><div class='del'>-	mr-&gt;umem = NULL;</div><div class='del'>-	set_mr_fields(dev, mr, 0, length, acc);</div><div class='add'>+	set_mr_fields(dev, mr, length, acc, start_addr);</div><div class='ctx'> </div><div class='ctx'> 	return &amp;mr-&gt;ibmr;</div><div class='ctx'> </div><div class='hunk'>@@ -1223,7 +1414,8 @@ int mlx5_ib_advise_mr(struct ib_pd *pd,</div><div class='ctx'> 		      struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	if (advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH &amp;&amp;</div><div class='del'>-	    advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE)</div><div class='add'>+	    advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE &amp;&amp;</div><div class='add'>+	    advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT)</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	return mlx5_ib_advise_mr_prefetch(pd, advice, flags,</div><div class='hunk'>@@ -1249,6 +1441,8 @@ struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:</div><div class='ctx'> 	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_PATTERN_SW_ICM:</div><div class='add'>+	case MLX5_IB_UAPI_DM_TYPE_ENCAP_SW_ICM:</div><div class='ctx'> 		if (attr-&gt;access_flags &amp; ~MLX5_IB_DM_SW_ICM_ALLOWED_ACCESS)</div><div class='ctx'> 			return ERR_PTR(-EINVAL);</div><div class='ctx'> </div><div class='hunk'>@@ -1262,257 +1456,519 @@ struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,</div><div class='ctx'> 				 attr-&gt;access_flags, mode);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='del'>-				  u64 virt_addr, int access_flags,</div><div class='del'>-				  struct ib_udata *udata)</div><div class='add'>+static struct ib_mr *create_real_mr(struct ib_pd *pd, struct ib_umem *umem,</div><div class='add'>+				    u64 iova, int access_flags,</div><div class='add'>+				    struct ib_dmah *dmah)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='ctx'> 	struct mlx5_ib_mr *mr = NULL;</div><div class='del'>-	bool populate_mtts = false;</div><div class='del'>-	struct ib_umem *umem;</div><div class='del'>-	int page_shift;</div><div class='del'>-	int npages;</div><div class='del'>-	int ncont;</div><div class='del'>-	int order;</div><div class='add'>+	bool xlt_with_umr;</div><div class='add'>+	u16 st_index = MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX;</div><div class='add'>+	u8 ph = MLX5_IB_NO_PH;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM))</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+	if (dmah) {</div><div class='add'>+		struct mlx5_ib_dmah *mdmah = to_mdmah(dmah);</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",</div><div class='del'>-		    start, virt_addr, length, access_flags);</div><div class='add'>+		ph = dmah-&gt;ph;</div><div class='add'>+		if (dmah-&gt;valid_fields &amp; BIT(IB_DMAH_CPU_ID_EXISTS))</div><div class='add'>+			st_index = mdmah-&gt;st_index;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) &amp;&amp; !start &amp;&amp;</div><div class='del'>-	    length == U64_MAX) {</div><div class='del'>-		if (!(access_flags &amp; IB_ACCESS_ON_DEMAND) ||</div><div class='del'>-		    !(dev-&gt;odp_caps.general_caps &amp; IB_ODP_SUPPORT_IMPLICIT))</div><div class='del'>-			return ERR_PTR(-EINVAL);</div><div class='add'>+	xlt_with_umr = mlx5r_umr_can_load_pas(dev, umem-&gt;length);</div><div class='add'>+	if (xlt_with_umr) {</div><div class='add'>+		mr = alloc_cacheable_mr(pd, umem, iova, access_flags,</div><div class='add'>+					MLX5_MKC_ACCESS_MODE_MTT,</div><div class='add'>+					st_index, ph);</div><div class='add'>+	} else {</div><div class='add'>+		unsigned long page_size = mlx5_umem_mkc_find_best_pgsz(</div><div class='add'>+				dev, umem, iova, MLX5_MKC_ACCESS_MODE_MTT);</div><div class='ctx'> </div><div class='del'>-		mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), udata, access_flags);</div><div class='del'>-		if (IS_ERR(mr))</div><div class='del'>-			return ERR_CAST(mr);</div><div class='del'>-		return &amp;mr-&gt;ibmr;</div><div class='add'>+		mutex_lock(&amp;dev-&gt;slow_path_mutex);</div><div class='add'>+		mr = reg_create(pd, umem, iova, access_flags, page_size,</div><div class='add'>+				true, MLX5_MKC_ACCESS_MODE_MTT,</div><div class='add'>+				st_index, ph);</div><div class='add'>+		mutex_unlock(&amp;dev-&gt;slow_path_mutex);</div><div class='add'>+	}</div><div class='add'>+	if (IS_ERR(mr)) {</div><div class='add'>+		ib_umem_release(umem);</div><div class='add'>+		return ERR_CAST(mr);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	err = mr_umem_get(dev, udata, start, length, access_flags, &amp;umem,</div><div class='del'>-			  &amp;npages, &amp;page_shift, &amp;ncont, &amp;order);</div><div class='add'>+	mlx5_ib_dbg(dev, "mkey 0x%x\n", mr-&gt;mmkey.key);</div><div class='ctx'> </div><div class='del'>-	if (err &lt; 0)</div><div class='del'>-		return ERR_PTR(err);</div><div class='add'>+	atomic_add(ib_umem_num_pages(umem), &amp;dev-&gt;mdev-&gt;priv.reg_pages);</div><div class='ctx'> </div><div class='del'>-	if (use_umr(dev, order)) {</div><div class='del'>-		mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont,</div><div class='del'>-					 page_shift, order, access_flags);</div><div class='del'>-		if (PTR_ERR(mr) == -EAGAIN) {</div><div class='del'>-			mlx5_ib_dbg(dev, "cache empty for order %d\n", order);</div><div class='del'>-			mr = NULL;</div><div class='del'>-		}</div><div class='del'>-		populate_mtts = false;</div><div class='del'>-	} else if (!MLX5_CAP_GEN(dev-&gt;mdev, umr_extended_translation_offset)) {</div><div class='del'>-		if (access_flags &amp; IB_ACCESS_ON_DEMAND) {</div><div class='del'>-			err = -EINVAL;</div><div class='del'>-			pr_err("Got MR registration for ODP MR &gt; 512MB, not supported for Connect-IB\n");</div><div class='del'>-			goto error;</div><div class='add'>+	if (xlt_with_umr) {</div><div class='add'>+		/*</div><div class='add'>+		 * If the MR was created with reg_create then it will be</div><div class='add'>+		 * configured properly but left disabled. It is safe to go ahead</div><div class='add'>+		 * and configure it again via UMR while enabling it.</div><div class='add'>+		 */</div><div class='add'>+		err = mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ENABLE);</div><div class='add'>+		if (err) {</div><div class='add'>+			mlx5_ib_dereg_mr(&amp;mr-&gt;ibmr, NULL);</div><div class='add'>+			return ERR_PTR(err);</div><div class='ctx'> 		}</div><div class='del'>-		populate_mtts = true;</div><div class='ctx'> 	}</div><div class='add'>+	return &amp;mr-&gt;ibmr;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (!mr) {</div><div class='del'>-		if (!umr_can_modify_entity_size(dev))</div><div class='del'>-			populate_mtts = true;</div><div class='del'>-		mutex_lock(&amp;dev-&gt;slow_path_mutex);</div><div class='del'>-		mr = reg_create(NULL, pd, virt_addr, length, umem, ncont,</div><div class='del'>-				page_shift, access_flags, populate_mtts);</div><div class='del'>-		mutex_unlock(&amp;dev-&gt;slow_path_mutex);</div><div class='add'>+static struct ib_mr *create_user_odp_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='add'>+					u64 iova, int access_flags,</div><div class='add'>+					struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='add'>+	struct ib_umem_odp *odp;</div><div class='add'>+	struct mlx5_ib_mr *mr;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (!IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	err = mlx5r_odp_create_eq(dev, &amp;dev-&gt;odp_pf_eq);</div><div class='add'>+	if (err)</div><div class='add'>+		return ERR_PTR(err);</div><div class='add'>+	if (!start &amp;&amp; length == U64_MAX) {</div><div class='add'>+		if (iova != 0)</div><div class='add'>+			return ERR_PTR(-EINVAL);</div><div class='add'>+		if (!(dev-&gt;odp_caps.general_caps &amp; IB_ODP_SUPPORT_IMPLICIT))</div><div class='add'>+			return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+		mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), access_flags);</div><div class='add'>+		if (IS_ERR(mr))</div><div class='add'>+			return ERR_CAST(mr);</div><div class='add'>+		return &amp;mr-&gt;ibmr;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	/* ODP requires xlt update via umr to work. */</div><div class='add'>+	if (!mlx5r_umr_can_load_pas(dev, length))</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	odp = ib_umem_odp_get(&amp;dev-&gt;ib_dev, start, length, access_flags,</div><div class='add'>+			      &amp;mlx5_mn_ops);</div><div class='add'>+	if (IS_ERR(odp))</div><div class='add'>+		return ERR_CAST(odp);</div><div class='add'>+</div><div class='add'>+	mr = alloc_cacheable_mr(pd, &amp;odp-&gt;umem, iova, access_flags,</div><div class='add'>+				MLX5_MKC_ACCESS_MODE_MTT,</div><div class='add'>+				MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX,</div><div class='add'>+				MLX5_IB_NO_PH);</div><div class='ctx'> 	if (IS_ERR(mr)) {</div><div class='del'>-		err = PTR_ERR(mr);</div><div class='del'>-		goto error;</div><div class='add'>+		ib_umem_release(&amp;odp-&gt;umem);</div><div class='add'>+		return ERR_CAST(mr);</div><div class='ctx'> 	}</div><div class='add'>+	xa_init(&amp;mr-&gt;implicit_children);</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_dbg(dev, "mkey 0x%x\n", mr-&gt;mmkey.key);</div><div class='add'>+	odp-&gt;private = mr;</div><div class='add'>+	err = mlx5r_store_odp_mkey(dev, &amp;mr-&gt;mmkey);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_dereg_mr;</div><div class='ctx'> </div><div class='del'>-	mr-&gt;umem = umem;</div><div class='del'>-	set_mr_fields(dev, mr, npages, length, access_flags);</div><div class='add'>+	err = mlx5_ib_init_odp_mr(mr);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_dereg_mr;</div><div class='add'>+	return &amp;mr-&gt;ibmr;</div><div class='ctx'> </div><div class='del'>-	update_odp_mr(mr);</div><div class='add'>+err_dereg_mr:</div><div class='add'>+	mlx5_ib_dereg_mr(&amp;mr-&gt;ibmr, NULL);</div><div class='add'>+	return ERR_PTR(err);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (!populate_mtts) {</div><div class='del'>-		int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;</div><div class='add'>+struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,</div><div class='add'>+				  u64 iova, int access_flags,</div><div class='add'>+				  struct ib_dmah *dmah,</div><div class='add'>+				  struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='add'>+	struct ib_umem *umem;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-		if (access_flags &amp; IB_ACCESS_ON_DEMAND)</div><div class='del'>-			update_xlt_flags |= MLX5_IB_UPD_XLT_ZAP;</div><div class='add'>+	if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM) ||</div><div class='add'>+	    ((access_flags &amp; IB_ACCESS_ON_DEMAND) &amp;&amp; dmah))</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='ctx'> </div><div class='del'>-		err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,</div><div class='del'>-					 update_xlt_flags);</div><div class='add'>+	mlx5_ib_dbg(dev, "start 0x%llx, iova 0x%llx, length 0x%llx, access_flags 0x%x\n",</div><div class='add'>+		    start, iova, length, access_flags);</div><div class='ctx'> </div><div class='del'>-		if (err) {</div><div class='del'>-			dereg_mr(dev, mr);</div><div class='del'>-			return ERR_PTR(err);</div><div class='del'>-		}</div><div class='add'>+	err = mlx5r_umr_resource_init(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		return ERR_PTR(err);</div><div class='add'>+</div><div class='add'>+	if (access_flags &amp; IB_ACCESS_ON_DEMAND)</div><div class='add'>+		return create_user_odp_mr(pd, start, length, iova, access_flags,</div><div class='add'>+					  udata);</div><div class='add'>+	umem = ib_umem_get(&amp;dev-&gt;ib_dev, start, length, access_flags);</div><div class='add'>+	if (IS_ERR(umem))</div><div class='add'>+		return ERR_CAST(umem);</div><div class='add'>+	return create_real_mr(pd, umem, iova, access_flags, dmah);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_dmabuf_invalidate_cb(struct dma_buf_attachment *attach)</div><div class='add'>+{</div><div class='add'>+	struct ib_umem_dmabuf *umem_dmabuf = attach-&gt;importer_priv;</div><div class='add'>+	struct mlx5_ib_mr *mr = umem_dmabuf-&gt;private;</div><div class='add'>+</div><div class='add'>+	dma_resv_assert_held(umem_dmabuf-&gt;attach-&gt;dmabuf-&gt;resv);</div><div class='add'>+</div><div class='add'>+	if (!umem_dmabuf-&gt;sgt || !mr)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ZAP);</div><div class='add'>+	ib_umem_dmabuf_unmap_pages(umem_dmabuf);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct dma_buf_attach_ops mlx5_ib_dmabuf_attach_ops = {</div><div class='add'>+	.allow_peer2peer = 1,</div><div class='add'>+	.move_notify = mlx5_ib_dmabuf_invalidate_cb,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static struct ib_mr *</div><div class='add'>+reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device,</div><div class='add'>+		   u64 offset, u64 length, u64 virt_addr,</div><div class='add'>+		   int fd, int access_flags, int access_mode,</div><div class='add'>+		   struct ib_dmah *dmah)</div><div class='add'>+{</div><div class='add'>+	bool pinned_mode = (access_mode == MLX5_MKC_ACCESS_MODE_KSM);</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='add'>+	struct mlx5_ib_mr *mr = NULL;</div><div class='add'>+	struct ib_umem_dmabuf *umem_dmabuf;</div><div class='add'>+	u16 st_index = MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX;</div><div class='add'>+	u8 ph = MLX5_IB_NO_PH;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	err = mlx5r_umr_resource_init(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		return ERR_PTR(err);</div><div class='add'>+</div><div class='add'>+	if (!pinned_mode)</div><div class='add'>+		umem_dmabuf = ib_umem_dmabuf_get(&amp;dev-&gt;ib_dev,</div><div class='add'>+						 offset, length, fd,</div><div class='add'>+						 access_flags,</div><div class='add'>+						 &amp;mlx5_ib_dmabuf_attach_ops);</div><div class='add'>+	else</div><div class='add'>+		umem_dmabuf = ib_umem_dmabuf_get_pinned_with_dma_device(&amp;dev-&gt;ib_dev,</div><div class='add'>+				dma_device, offset, length,</div><div class='add'>+				fd, access_flags);</div><div class='add'>+</div><div class='add'>+	if (IS_ERR(umem_dmabuf)) {</div><div class='add'>+		mlx5_ib_dbg(dev, "umem_dmabuf get failed (%pe)\n", umem_dmabuf);</div><div class='add'>+		return ERR_CAST(umem_dmabuf);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {</div><div class='del'>-		mr-&gt;live = 1;</div><div class='del'>-		atomic_set(&amp;mr-&gt;num_pending_prefetch, 0);</div><div class='add'>+	if (dmah) {</div><div class='add'>+		struct mlx5_ib_dmah *mdmah = to_mdmah(dmah);</div><div class='add'>+</div><div class='add'>+		ph = dmah-&gt;ph;</div><div class='add'>+		if (dmah-&gt;valid_fields &amp; BIT(IB_DMAH_CPU_ID_EXISTS))</div><div class='add'>+			st_index = mdmah-&gt;st_index;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mr = alloc_cacheable_mr(pd, &amp;umem_dmabuf-&gt;umem, virt_addr,</div><div class='add'>+				access_flags, access_mode,</div><div class='add'>+				st_index, ph);</div><div class='add'>+	if (IS_ERR(mr)) {</div><div class='add'>+		ib_umem_release(&amp;umem_dmabuf-&gt;umem);</div><div class='add'>+		return ERR_CAST(mr);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mlx5_ib_dbg(dev, "mkey 0x%x\n", mr-&gt;mmkey.key);</div><div class='add'>+</div><div class='add'>+	atomic_add(ib_umem_num_pages(mr-&gt;umem), &amp;dev-&gt;mdev-&gt;priv.reg_pages);</div><div class='add'>+	umem_dmabuf-&gt;private = mr;</div><div class='add'>+	if (!pinned_mode) {</div><div class='add'>+		err = mlx5r_store_odp_mkey(dev, &amp;mr-&gt;mmkey);</div><div class='add'>+		if (err)</div><div class='add'>+			goto err_dereg_mr;</div><div class='add'>+	} else {</div><div class='add'>+		mr-&gt;data_direct = true;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	err = mlx5_ib_init_dmabuf_mr(mr);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_dereg_mr;</div><div class='ctx'> 	return &amp;mr-&gt;ibmr;</div><div class='del'>-error:</div><div class='del'>-	ib_umem_release(umem);</div><div class='add'>+</div><div class='add'>+err_dereg_mr:</div><div class='add'>+	__mlx5_ib_dereg_mr(&amp;mr-&gt;ibmr);</div><div class='ctx'> 	return ERR_PTR(err);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)</div><div class='add'>+static struct ib_mr *</div><div class='add'>+reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset,</div><div class='add'>+				  u64 length, u64 virt_addr,</div><div class='add'>+				  int fd, int access_flags)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='del'>-	struct mlx5_umr_wr umrwr = {};</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='add'>+	struct mlx5_data_direct_dev *data_direct_dev;</div><div class='add'>+	struct ib_mr *crossing_mr;</div><div class='add'>+	struct ib_mr *crossed_mr;</div><div class='add'>+	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	if (mdev-&gt;state == MLX5_DEVICE_STATE_INTERNAL_ERROR)</div><div class='del'>-		return 0;</div><div class='add'>+	/* As of HW behaviour the IOVA must be page aligned in KSM mode */</div><div class='add'>+	if (!PAGE_ALIGNED(virt_addr) || (access_flags &amp; IB_ACCESS_ON_DEMAND))</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='ctx'> </div><div class='del'>-	umrwr.wr.send_flags = MLX5_IB_SEND_UMR_DISABLE_MR |</div><div class='del'>-			      MLX5_IB_SEND_UMR_FAIL_IF_FREE;</div><div class='del'>-	umrwr.wr.opcode = MLX5_IB_WR_UMR;</div><div class='del'>-	umrwr.mkey = mr-&gt;mmkey.key;</div><div class='add'>+	mutex_lock(&amp;dev-&gt;data_direct_lock);</div><div class='add'>+	data_direct_dev = dev-&gt;data_direct_dev;</div><div class='add'>+	if (!data_direct_dev) {</div><div class='add'>+		ret = -EINVAL;</div><div class='add'>+		goto end;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* If no device's 'data direct mkey' with RO flags exists</div><div class='add'>+	 * mask it out accordingly.</div><div class='add'>+	 */</div><div class='add'>+	if (!dev-&gt;ddr.mkey_ro_valid)</div><div class='add'>+		access_flags &amp;= ~IB_ACCESS_RELAXED_ORDERING;</div><div class='add'>+	crossed_mr = reg_user_mr_dmabuf(pd, &amp;data_direct_dev-&gt;pdev-&gt;dev,</div><div class='add'>+					offset, length, virt_addr, fd,</div><div class='add'>+					access_flags, MLX5_MKC_ACCESS_MODE_KSM,</div><div class='add'>+					NULL);</div><div class='add'>+	if (IS_ERR(crossed_mr)) {</div><div class='add'>+		ret = PTR_ERR(crossed_mr);</div><div class='add'>+		goto end;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	return mlx5_ib_post_send_wait(dev, &amp;umrwr);</div><div class='add'>+	mutex_lock(&amp;dev-&gt;slow_path_mutex);</div><div class='add'>+	crossing_mr = reg_create_crossing_vhca_mr(pd, virt_addr, length, access_flags,</div><div class='add'>+						  crossed_mr-&gt;lkey);</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;slow_path_mutex);</div><div class='add'>+	if (IS_ERR(crossing_mr)) {</div><div class='add'>+		__mlx5_ib_dereg_mr(crossed_mr);</div><div class='add'>+		ret = PTR_ERR(crossing_mr);</div><div class='add'>+		goto end;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	list_add_tail(&amp;to_mmr(crossed_mr)-&gt;dd_node, &amp;dev-&gt;data_direct_mr_list);</div><div class='add'>+	to_mmr(crossing_mr)-&gt;dd_crossed_mr = to_mmr(crossed_mr);</div><div class='add'>+	to_mmr(crossing_mr)-&gt;data_direct = true;</div><div class='add'>+end:</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;data_direct_lock);</div><div class='add'>+	return ret ? ERR_PTR(ret) : crossing_mr;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int rereg_umr(struct ib_pd *pd, struct mlx5_ib_mr *mr,</div><div class='del'>-		     int access_flags, int flags)</div><div class='add'>+struct ib_mr *mlx5_ib_reg_user_mr_dmabuf(struct ib_pd *pd, u64 offset,</div><div class='add'>+					 u64 length, u64 virt_addr,</div><div class='add'>+					 int fd, int access_flags,</div><div class='add'>+					 struct ib_dmah *dmah,</div><div class='add'>+					 struct uverbs_attr_bundle *attrs)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='del'>-	struct mlx5_umr_wr umrwr = {};</div><div class='add'>+	int mlx5_access_flags = 0;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	umrwr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE;</div><div class='del'>-</div><div class='del'>-	umrwr.wr.opcode = MLX5_IB_WR_UMR;</div><div class='del'>-	umrwr.mkey = mr-&gt;mmkey.key;</div><div class='add'>+	if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM) ||</div><div class='add'>+	    !IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='ctx'> </div><div class='del'>-	if (flags &amp; IB_MR_REREG_PD || flags &amp; IB_MR_REREG_ACCESS) {</div><div class='del'>-		umrwr.pd = pd;</div><div class='del'>-		umrwr.access_flags = access_flags;</div><div class='del'>-		umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;</div><div class='add'>+	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_REG_DMABUF_MR_ACCESS_FLAGS)) {</div><div class='add'>+		err = uverbs_get_flags32(&amp;mlx5_access_flags, attrs,</div><div class='add'>+					 MLX5_IB_ATTR_REG_DMABUF_MR_ACCESS_FLAGS,</div><div class='add'>+					 MLX5_IB_UAPI_REG_DMABUF_ACCESS_DATA_DIRECT);</div><div class='add'>+		if (err)</div><div class='add'>+			return ERR_PTR(err);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	err = mlx5_ib_post_send_wait(dev, &amp;umrwr);</div><div class='add'>+	mlx5_ib_dbg(dev,</div><div class='add'>+		    "offset 0x%llx, virt_addr 0x%llx, length 0x%llx, fd %d, access_flags 0x%x, mlx5_access_flags 0x%x\n",</div><div class='add'>+		    offset, virt_addr, length, fd, access_flags, mlx5_access_flags);</div><div class='ctx'> </div><div class='del'>-	return err;</div><div class='add'>+	/* dmabuf requires xlt update via umr to work. */</div><div class='add'>+	if (!mlx5r_umr_can_load_pas(dev, length))</div><div class='add'>+		return ERR_PTR(-EINVAL);</div><div class='add'>+</div><div class='add'>+	if (mlx5_access_flags &amp; MLX5_IB_UAPI_REG_DMABUF_ACCESS_DATA_DIRECT)</div><div class='add'>+		return reg_user_mr_dmabuf_by_data_direct(pd, offset, length, virt_addr,</div><div class='add'>+							 fd, access_flags);</div><div class='add'>+</div><div class='add'>+	return reg_user_mr_dmabuf(pd, pd-&gt;device-&gt;dma_device,</div><div class='add'>+				  offset, length, virt_addr,</div><div class='add'>+				  fd, access_flags, MLX5_MKC_ACCESS_MODE_MTT,</div><div class='add'>+				  dmah);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,</div><div class='del'>-			  u64 length, u64 virt_addr, int new_access_flags,</div><div class='del'>-			  struct ib_pd *new_pd, struct ib_udata *udata)</div><div class='add'>+/*</div><div class='add'>+ * True if the change in access flags can be done via UMR, only some access</div><div class='add'>+ * flags can be updated.</div><div class='add'>+ */</div><div class='add'>+static bool can_use_umr_rereg_access(struct mlx5_ib_dev *dev,</div><div class='add'>+				     unsigned int current_access_flags,</div><div class='add'>+				     unsigned int target_access_flags)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(ib_mr-&gt;device);</div><div class='del'>-	struct mlx5_ib_mr *mr = to_mmr(ib_mr);</div><div class='del'>-	struct ib_pd *pd = (flags &amp; IB_MR_REREG_PD) ? new_pd : ib_mr-&gt;pd;</div><div class='del'>-	int access_flags = flags &amp; IB_MR_REREG_ACCESS ?</div><div class='del'>-			    new_access_flags :</div><div class='del'>-			    mr-&gt;access_flags;</div><div class='del'>-	int page_shift = 0;</div><div class='del'>-	int upd_flags = 0;</div><div class='del'>-	int npages = 0;</div><div class='del'>-	int ncont = 0;</div><div class='del'>-	int order = 0;</div><div class='del'>-	u64 addr, len;</div><div class='del'>-	int err;</div><div class='add'>+	unsigned int diffs = current_access_flags ^ target_access_flags;</div><div class='add'>+</div><div class='add'>+	if (diffs &amp; ~(IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE |</div><div class='add'>+		      IB_ACCESS_REMOTE_READ | IB_ACCESS_RELAXED_ORDERING |</div><div class='add'>+		      IB_ACCESS_REMOTE_ATOMIC))</div><div class='add'>+		return false;</div><div class='add'>+	return mlx5r_umr_can_reconfig(dev, current_access_flags,</div><div class='add'>+				      target_access_flags);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",</div><div class='del'>-		    start, virt_addr, length, access_flags);</div><div class='add'>+static bool can_use_umr_rereg_pas(struct mlx5_ib_mr *mr,</div><div class='add'>+				  struct ib_umem *new_umem,</div><div class='add'>+				  int new_access_flags, u64 iova,</div><div class='add'>+				  unsigned long *page_size)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(mr-&gt;ibmr.device);</div><div class='add'>+</div><div class='add'>+	/* We only track the allocated sizes of MRs from the cache */</div><div class='add'>+	if (!mr-&gt;mmkey.cache_ent)</div><div class='add'>+		return false;</div><div class='add'>+	if (!mlx5r_umr_can_load_pas(dev, new_umem-&gt;length))</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	*page_size = mlx5_umem_mkc_find_best_pgsz(</div><div class='add'>+		dev, new_umem, iova, mr-&gt;mmkey.cache_ent-&gt;rb_key.access_mode);</div><div class='add'>+	if (WARN_ON(!*page_size))</div><div class='add'>+		return false;</div><div class='add'>+	return (mr-&gt;mmkey.cache_ent-&gt;rb_key.ndescs) &gt;=</div><div class='add'>+	       ib_umem_num_dma_blocks(new_umem, *page_size);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	atomic_sub(mr-&gt;npages, &amp;dev-&gt;mdev-&gt;priv.reg_pages);</div><div class='add'>+static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd,</div><div class='add'>+			 int access_flags, int flags, struct ib_umem *new_umem,</div><div class='add'>+			 u64 iova, unsigned long page_size)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(mr-&gt;ibmr.device);</div><div class='add'>+	int upd_flags = MLX5_IB_UPD_XLT_ADDR | MLX5_IB_UPD_XLT_ENABLE;</div><div class='add'>+	struct ib_umem *old_umem = mr-&gt;umem;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-	if (!mr-&gt;umem)</div><div class='del'>-		return -EINVAL;</div><div class='add'>+	/*</div><div class='add'>+	 * To keep everything simple the MR is revoked before we start to mess</div><div class='add'>+	 * with it. This ensure the change is atomic relative to any use of the</div><div class='add'>+	 * MR.</div><div class='add'>+	 */</div><div class='add'>+	err = mlx5r_umr_revoke_mr(mr);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='ctx'> </div><div class='del'>-	if (flags &amp; IB_MR_REREG_TRANS) {</div><div class='del'>-		addr = virt_addr;</div><div class='del'>-		len = length;</div><div class='del'>-	} else {</div><div class='del'>-		addr = mr-&gt;umem-&gt;address;</div><div class='del'>-		len = mr-&gt;umem-&gt;length;</div><div class='add'>+	if (flags &amp; IB_MR_REREG_PD) {</div><div class='add'>+		mr-&gt;ibmr.pd = pd;</div><div class='add'>+		upd_flags |= MLX5_IB_UPD_XLT_PD;</div><div class='add'>+	}</div><div class='add'>+	if (flags &amp; IB_MR_REREG_ACCESS) {</div><div class='add'>+		mr-&gt;access_flags = access_flags;</div><div class='add'>+		upd_flags |= MLX5_IB_UPD_XLT_ACCESS;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (flags != IB_MR_REREG_PD) {</div><div class='add'>+	mr-&gt;ibmr.iova = iova;</div><div class='add'>+	mr-&gt;ibmr.length = new_umem-&gt;length;</div><div class='add'>+	mr-&gt;page_shift = order_base_2(page_size);</div><div class='add'>+	mr-&gt;umem = new_umem;</div><div class='add'>+	err = mlx5r_umr_update_mr_pas(mr, upd_flags);</div><div class='add'>+	if (err) {</div><div class='ctx'> 		/*</div><div class='del'>-		 * Replace umem. This needs to be done whether or not UMR is</div><div class='del'>-		 * used.</div><div class='add'>+		 * The MR is revoked at this point so there is no issue to free</div><div class='add'>+		 * new_umem.</div><div class='ctx'> 		 */</div><div class='del'>-		flags |= IB_MR_REREG_TRANS;</div><div class='del'>-		ib_umem_release(mr-&gt;umem);</div><div class='del'>-		mr-&gt;umem = NULL;</div><div class='del'>-		err = mr_umem_get(dev, udata, addr, len, access_flags,</div><div class='del'>-				  &amp;mr-&gt;umem, &amp;npages, &amp;page_shift, &amp;ncont,</div><div class='del'>-				  &amp;order);</div><div class='del'>-		if (err)</div><div class='del'>-			goto err;</div><div class='add'>+		mr-&gt;umem = old_umem;</div><div class='add'>+		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (flags &amp; IB_MR_REREG_TRANS &amp;&amp; !use_umr_mtt_update(mr, addr, len)) {</div><div class='del'>-		/*</div><div class='del'>-		 * UMR can't be used - MKey needs to be replaced.</div><div class='del'>-		 */</div><div class='del'>-		if (mr-&gt;allocated_from_cache)</div><div class='del'>-			err = unreg_umr(dev, mr);</div><div class='del'>-		else</div><div class='del'>-			err = destroy_mkey(dev, mr);</div><div class='del'>-		if (err)</div><div class='del'>-			goto err;</div><div class='add'>+	atomic_sub(ib_umem_num_pages(old_umem), &amp;dev-&gt;mdev-&gt;priv.reg_pages);</div><div class='add'>+	ib_umem_release(old_umem);</div><div class='add'>+	atomic_add(ib_umem_num_pages(new_umem), &amp;dev-&gt;mdev-&gt;priv.reg_pages);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,</div><div class='add'>+				    u64 length, u64 iova, int new_access_flags,</div><div class='add'>+				    struct ib_pd *new_pd,</div><div class='add'>+				    struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ib_mr-&gt;device);</div><div class='add'>+	struct mlx5_ib_mr *mr = to_mmr(ib_mr);</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-		mr = reg_create(ib_mr, pd, addr, len, mr-&gt;umem, ncont,</div><div class='del'>-				page_shift, access_flags, true);</div><div class='add'>+	if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM) || mr-&gt;data_direct ||</div><div class='add'>+	    mr-&gt;mmkey.rb_key.ph != MLX5_IB_NO_PH)</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='ctx'> </div><div class='del'>-		if (IS_ERR(mr)) {</div><div class='del'>-			err = PTR_ERR(mr);</div><div class='del'>-			mr = to_mmr(ib_mr);</div><div class='del'>-			goto err;</div><div class='add'>+	mlx5_ib_dbg(</div><div class='add'>+		dev,</div><div class='add'>+		"start 0x%llx, iova 0x%llx, length 0x%llx, access_flags 0x%x\n",</div><div class='add'>+		start, iova, length, new_access_flags);</div><div class='add'>+</div><div class='add'>+	if (flags &amp; ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS))</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+</div><div class='add'>+	if (!(flags &amp; IB_MR_REREG_ACCESS))</div><div class='add'>+		new_access_flags = mr-&gt;access_flags;</div><div class='add'>+	if (!(flags &amp; IB_MR_REREG_PD))</div><div class='add'>+		new_pd = ib_mr-&gt;pd;</div><div class='add'>+</div><div class='add'>+	if (!(flags &amp; IB_MR_REREG_TRANS)) {</div><div class='add'>+		struct ib_umem *umem;</div><div class='add'>+</div><div class='add'>+		/* Fast path for PD/access change */</div><div class='add'>+		if (can_use_umr_rereg_access(dev, mr-&gt;access_flags,</div><div class='add'>+					     new_access_flags)) {</div><div class='add'>+			err = mlx5r_umr_rereg_pd_access(mr, new_pd,</div><div class='add'>+							new_access_flags);</div><div class='add'>+			if (err)</div><div class='add'>+				return ERR_PTR(err);</div><div class='add'>+			return NULL;</div><div class='ctx'> 		}</div><div class='add'>+		/* DM or ODP MR's don't have a normal umem so we can't re-use it */</div><div class='add'>+		if (!mr-&gt;umem || is_odp_mr(mr) || is_dmabuf_mr(mr))</div><div class='add'>+			goto recreate;</div><div class='ctx'> </div><div class='del'>-		mr-&gt;allocated_from_cache = 0;</div><div class='del'>-		if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))</div><div class='del'>-			mr-&gt;live = 1;</div><div class='del'>-	} else {</div><div class='ctx'> 		/*</div><div class='del'>-		 * Send a UMR WQE</div><div class='add'>+		 * Only one active MR can refer to a umem at one time, revoke</div><div class='add'>+		 * the old MR before assigning the umem to the new one.</div><div class='ctx'> 		 */</div><div class='del'>-		mr-&gt;ibmr.pd = pd;</div><div class='del'>-		mr-&gt;access_flags = access_flags;</div><div class='del'>-		mr-&gt;mmkey.iova = addr;</div><div class='del'>-		mr-&gt;mmkey.size = len;</div><div class='del'>-		mr-&gt;mmkey.pd = to_mpd(pd)-&gt;pdn;</div><div class='del'>-</div><div class='del'>-		if (flags &amp; IB_MR_REREG_TRANS) {</div><div class='del'>-			upd_flags = MLX5_IB_UPD_XLT_ADDR;</div><div class='del'>-			if (flags &amp; IB_MR_REREG_PD)</div><div class='del'>-				upd_flags |= MLX5_IB_UPD_XLT_PD;</div><div class='del'>-			if (flags &amp; IB_MR_REREG_ACCESS)</div><div class='del'>-				upd_flags |= MLX5_IB_UPD_XLT_ACCESS;</div><div class='del'>-			err = mlx5_ib_update_xlt(mr, 0, npages, page_shift,</div><div class='del'>-						 upd_flags);</div><div class='del'>-		} else {</div><div class='del'>-			err = rereg_umr(pd, mr, access_flags, flags);</div><div class='del'>-		}</div><div class='del'>-</div><div class='add'>+		err = mlx5r_umr_revoke_mr(mr);</div><div class='ctx'> 		if (err)</div><div class='del'>-			goto err;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	set_mr_fields(dev, mr, npages, len, access_flags);</div><div class='add'>+			return ERR_PTR(err);</div><div class='add'>+		umem = mr-&gt;umem;</div><div class='add'>+		mr-&gt;umem = NULL;</div><div class='add'>+		atomic_sub(ib_umem_num_pages(umem), &amp;dev-&gt;mdev-&gt;priv.reg_pages);</div><div class='ctx'> </div><div class='del'>-	update_odp_mr(mr);</div><div class='del'>-	return 0;</div><div class='add'>+		return create_real_mr(new_pd, umem, mr-&gt;ibmr.iova,</div><div class='add'>+				      new_access_flags, NULL);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-err:</div><div class='del'>-	if (mr-&gt;umem) {</div><div class='del'>-		ib_umem_release(mr-&gt;umem);</div><div class='del'>-		mr-&gt;umem = NULL;</div><div class='add'>+	/*</div><div class='add'>+	 * DM doesn't have a PAS list so we can't re-use it, odp/dmabuf does</div><div class='add'>+	 * but the logic around releasing the umem is different</div><div class='add'>+	 */</div><div class='add'>+	if (!mr-&gt;umem || is_odp_mr(mr) || is_dmabuf_mr(mr))</div><div class='add'>+		goto recreate;</div><div class='add'>+</div><div class='add'>+	if (!(new_access_flags &amp; IB_ACCESS_ON_DEMAND) &amp;&amp;</div><div class='add'>+	    can_use_umr_rereg_access(dev, mr-&gt;access_flags, new_access_flags)) {</div><div class='add'>+		struct ib_umem *new_umem;</div><div class='add'>+		unsigned long page_size;</div><div class='add'>+</div><div class='add'>+		new_umem = ib_umem_get(&amp;dev-&gt;ib_dev, start, length,</div><div class='add'>+				       new_access_flags);</div><div class='add'>+		if (IS_ERR(new_umem))</div><div class='add'>+			return ERR_CAST(new_umem);</div><div class='add'>+</div><div class='add'>+		/* Fast path for PAS change */</div><div class='add'>+		if (can_use_umr_rereg_pas(mr, new_umem, new_access_flags, iova,</div><div class='add'>+					  &amp;page_size)) {</div><div class='add'>+			err = umr_rereg_pas(mr, new_pd, new_access_flags, flags,</div><div class='add'>+					    new_umem, iova, page_size);</div><div class='add'>+			if (err) {</div><div class='add'>+				ib_umem_release(new_umem);</div><div class='add'>+				return ERR_PTR(err);</div><div class='add'>+			}</div><div class='add'>+			return NULL;</div><div class='add'>+		}</div><div class='add'>+		return create_real_mr(new_pd, new_umem, iova, new_access_flags, NULL);</div><div class='ctx'> 	}</div><div class='del'>-	clean_mr(dev, mr);</div><div class='del'>-	return err;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Everything else has no state we can preserve, just create a new MR</div><div class='add'>+	 * from scratch</div><div class='add'>+	 */</div><div class='add'>+recreate:</div><div class='add'>+	return mlx5_ib_reg_user_mr(new_pd, start, length, iova,</div><div class='add'>+				   new_access_flags, NULL, udata);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int</div><div class='hunk'>@@ -1521,11 +1977,18 @@ mlx5_alloc_priv_descs(struct ib_device *device,</div><div class='ctx'> 		      int ndescs,</div><div class='ctx'> 		      int desc_size)</div><div class='ctx'> {</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='add'>+	struct device *ddev = &amp;dev-&gt;mdev-&gt;pdev-&gt;dev;</div><div class='ctx'> 	int size = ndescs * desc_size;</div><div class='ctx'> 	int add_size;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='ctx'> 	add_size = max_t(int, MLX5_UMR_ALIGN - ARCH_KMALLOC_MINALIGN, 0);</div><div class='add'>+	if (is_power_of_2(MLX5_UMR_ALIGN) &amp;&amp; add_size) {</div><div class='add'>+		int end = max_t(int, MLX5_UMR_ALIGN, roundup_pow_of_two(size));</div><div class='add'>+</div><div class='add'>+		add_size = min_t(int, end - size, add_size);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;descs_alloc = kzalloc(size + add_size, GFP_KERNEL);</div><div class='ctx'> 	if (!mr-&gt;descs_alloc)</div><div class='hunk'>@@ -1533,9 +1996,8 @@ mlx5_alloc_priv_descs(struct ib_device *device,</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;descs = PTR_ALIGN(mr-&gt;descs_alloc, MLX5_UMR_ALIGN);</div><div class='ctx'> </div><div class='del'>-	mr-&gt;desc_map = dma_map_single(device-&gt;dev.parent, mr-&gt;descs,</div><div class='del'>-				      size, DMA_TO_DEVICE);</div><div class='del'>-	if (dma_mapping_error(device-&gt;dev.parent, mr-&gt;desc_map)) {</div><div class='add'>+	mr-&gt;desc_map = dma_map_single(ddev, mr-&gt;descs, size, DMA_TO_DEVICE);</div><div class='add'>+	if (dma_mapping_error(ddev, mr-&gt;desc_map)) {</div><div class='ctx'> 		ret = -ENOMEM;</div><div class='ctx'> 		goto err;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -1550,107 +2012,333 @@ err:</div><div class='ctx'> static void</div><div class='ctx'> mlx5_free_priv_descs(struct mlx5_ib_mr *mr)</div><div class='ctx'> {</div><div class='del'>-	if (mr-&gt;descs) {</div><div class='add'>+	if (!mr-&gt;umem &amp;&amp; !mr-&gt;data_direct &amp;&amp;</div><div class='add'>+	    mr-&gt;ibmr.type != IB_MR_TYPE_DM &amp;&amp; mr-&gt;descs) {</div><div class='ctx'> 		struct ib_device *device = mr-&gt;ibmr.device;</div><div class='ctx'> 		int size = mr-&gt;max_descs * mr-&gt;desc_size;</div><div class='add'>+		struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='ctx'> </div><div class='del'>-		dma_unmap_single(device-&gt;dev.parent, mr-&gt;desc_map,</div><div class='del'>-				 size, DMA_TO_DEVICE);</div><div class='add'>+		dma_unmap_single(&amp;dev-&gt;mdev-&gt;pdev-&gt;dev, mr-&gt;desc_map, size,</div><div class='add'>+				 DMA_TO_DEVICE);</div><div class='ctx'> 		kfree(mr-&gt;descs_alloc);</div><div class='ctx'> 		mr-&gt;descs = NULL;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)</div><div class='add'>+static int cache_ent_find_and_store(struct mlx5_ib_dev *dev,</div><div class='add'>+				    struct mlx5_ib_mr *mr)</div><div class='ctx'> {</div><div class='del'>-	int allocated_from_cache = mr-&gt;allocated_from_cache;</div><div class='add'>+	struct mlx5_mkey_cache *cache = &amp;dev-&gt;cache;</div><div class='add'>+	struct mlx5_cache_ent *ent;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (mr-&gt;mmkey.cache_ent) {</div><div class='add'>+		spin_lock_irq(&amp;mr-&gt;mmkey.cache_ent-&gt;mkeys_queue.lock);</div><div class='add'>+		goto end;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;cache-&gt;rb_lock);</div><div class='add'>+	ent = mkey_cache_ent_from_rb_key(dev, mr-&gt;mmkey.rb_key);</div><div class='add'>+	if (ent) {</div><div class='add'>+		if (ent-&gt;rb_key.ndescs == mr-&gt;mmkey.rb_key.ndescs) {</div><div class='add'>+			if (ent-&gt;disabled) {</div><div class='add'>+				mutex_unlock(&amp;cache-&gt;rb_lock);</div><div class='add'>+				return -EOPNOTSUPP;</div><div class='add'>+			}</div><div class='add'>+			mr-&gt;mmkey.cache_ent = ent;</div><div class='add'>+			spin_lock_irq(&amp;mr-&gt;mmkey.cache_ent-&gt;mkeys_queue.lock);</div><div class='add'>+			mutex_unlock(&amp;cache-&gt;rb_lock);</div><div class='add'>+			goto end;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ent = mlx5r_cache_create_ent_locked(dev, mr-&gt;mmkey.rb_key, false);</div><div class='add'>+	mutex_unlock(&amp;cache-&gt;rb_lock);</div><div class='add'>+	if (IS_ERR(ent))</div><div class='add'>+		return PTR_ERR(ent);</div><div class='add'>+</div><div class='add'>+	mr-&gt;mmkey.cache_ent = ent;</div><div class='add'>+	spin_lock_irq(&amp;mr-&gt;mmkey.cache_ent-&gt;mkeys_queue.lock);</div><div class='add'>+</div><div class='add'>+end:</div><div class='add'>+	ret = push_mkey_locked(mr-&gt;mmkey.cache_ent, mr-&gt;mmkey.key);</div><div class='add'>+	spin_unlock_irq(&amp;mr-&gt;mmkey.cache_ent-&gt;mkeys_queue.lock);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_revoke_data_direct_mr(struct mlx5_ib_mr *mr)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(mr-&gt;ibmr.device);</div><div class='add'>+	struct ib_umem_dmabuf *umem_dmabuf = to_ib_umem_dmabuf(mr-&gt;umem);</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	lockdep_assert_held(&amp;dev-&gt;data_direct_lock);</div><div class='add'>+	mr-&gt;revoked = true;</div><div class='add'>+	err = mlx5r_umr_revoke_mr(mr);</div><div class='add'>+	if (WARN_ON(err))</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	ib_umem_dmabuf_revoke(umem_dmabuf);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_ib_revoke_data_direct_mrs(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mr *mr, *next;</div><div class='add'>+</div><div class='add'>+	lockdep_assert_held(&amp;dev-&gt;data_direct_lock);</div><div class='add'>+</div><div class='add'>+	list_for_each_entry_safe(mr, next, &amp;dev-&gt;data_direct_mr_list, dd_node) {</div><div class='add'>+		list_del(&amp;mr-&gt;dd_node);</div><div class='add'>+		mlx5_ib_revoke_data_direct_mr(mr);</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_umr_revoke_mr_with_lock(struct mlx5_ib_mr *mr)</div><div class='add'>+{</div><div class='add'>+	bool is_odp_dma_buf = is_dmabuf_mr(mr) &amp;&amp;</div><div class='add'>+			      !to_ib_umem_dmabuf(mr-&gt;umem)-&gt;pinned;</div><div class='add'>+	bool is_odp = is_odp_mr(mr);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (is_odp)</div><div class='add'>+		mutex_lock(&amp;to_ib_umem_odp(mr-&gt;umem)-&gt;umem_mutex);</div><div class='add'>+</div><div class='add'>+	if (is_odp_dma_buf)</div><div class='add'>+		dma_resv_lock(to_ib_umem_dmabuf(mr-&gt;umem)-&gt;attach-&gt;dmabuf-&gt;resv,</div><div class='add'>+			      NULL);</div><div class='add'>+</div><div class='add'>+	ret = mlx5r_umr_revoke_mr(mr);</div><div class='add'>+</div><div class='add'>+	if (is_odp) {</div><div class='add'>+		if (!ret)</div><div class='add'>+			to_ib_umem_odp(mr-&gt;umem)-&gt;private = NULL;</div><div class='add'>+		mutex_unlock(&amp;to_ib_umem_odp(mr-&gt;umem)-&gt;umem_mutex);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (is_odp_dma_buf) {</div><div class='add'>+		if (!ret)</div><div class='add'>+			to_ib_umem_dmabuf(mr-&gt;umem)-&gt;private = NULL;</div><div class='add'>+		dma_resv_unlock(</div><div class='add'>+			to_ib_umem_dmabuf(mr-&gt;umem)-&gt;attach-&gt;dmabuf-&gt;resv);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr)</div><div class='add'>+{</div><div class='add'>+	bool is_odp_dma_buf = is_dmabuf_mr(mr) &amp;&amp;</div><div class='add'>+			      !to_ib_umem_dmabuf(mr-&gt;umem)-&gt;pinned;</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(mr-&gt;ibmr.device);</div><div class='add'>+	struct mlx5_cache_ent *ent = mr-&gt;mmkey.cache_ent;</div><div class='add'>+	bool is_odp = is_odp_mr(mr);</div><div class='add'>+	bool from_cache = !!ent;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (mr-&gt;mmkey.cacheable &amp;&amp; !mlx5_umr_revoke_mr_with_lock(mr) &amp;&amp;</div><div class='add'>+	    !cache_ent_find_and_store(dev, mr)) {</div><div class='add'>+		ent = mr-&gt;mmkey.cache_ent;</div><div class='add'>+		/* upon storing to a clean temp entry - schedule its cleanup */</div><div class='add'>+		spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+		if (from_cache)</div><div class='add'>+			ent-&gt;in_use--;</div><div class='add'>+		if (ent-&gt;is_tmp &amp;&amp; !ent-&gt;tmp_cleanup_scheduled) {</div><div class='add'>+			mod_delayed_work(ent-&gt;dev-&gt;cache.wq, &amp;ent-&gt;dwork,</div><div class='add'>+					 secs_to_jiffies(30));</div><div class='add'>+			ent-&gt;tmp_cleanup_scheduled = true;</div><div class='add'>+		}</div><div class='add'>+		spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (ent) {</div><div class='add'>+		spin_lock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+		ent-&gt;in_use--;</div><div class='add'>+		mr-&gt;mmkey.cache_ent = NULL;</div><div class='add'>+		spin_unlock_irq(&amp;ent-&gt;mkeys_queue.lock);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (is_odp)</div><div class='add'>+		mutex_lock(&amp;to_ib_umem_odp(mr-&gt;umem)-&gt;umem_mutex);</div><div class='add'>+</div><div class='add'>+	if (is_odp_dma_buf)</div><div class='add'>+		dma_resv_lock(to_ib_umem_dmabuf(mr-&gt;umem)-&gt;attach-&gt;dmabuf-&gt;resv,</div><div class='add'>+			      NULL);</div><div class='add'>+	ret = destroy_mkey(dev, mr);</div><div class='add'>+	if (is_odp) {</div><div class='add'>+		if (!ret)</div><div class='add'>+			to_ib_umem_odp(mr-&gt;umem)-&gt;private = NULL;</div><div class='add'>+		mutex_unlock(&amp;to_ib_umem_odp(mr-&gt;umem)-&gt;umem_mutex);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (is_odp_dma_buf) {</div><div class='add'>+		if (!ret)</div><div class='add'>+			to_ib_umem_dmabuf(mr-&gt;umem)-&gt;private = NULL;</div><div class='add'>+		dma_resv_unlock(</div><div class='add'>+			to_ib_umem_dmabuf(mr-&gt;umem)-&gt;attach-&gt;dmabuf-&gt;resv);</div><div class='add'>+	}</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int __mlx5_ib_dereg_mr(struct ib_mr *ibmr)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mr *mr = to_mmr(ibmr);</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ibmr-&gt;device);</div><div class='add'>+	int rc;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * Any async use of the mr must hold the refcount, once the refcount</div><div class='add'>+	 * goes to zero no other thread, such as ODP page faults, prefetch, any</div><div class='add'>+	 * UMR activity, etc can touch the mkey. Thus it is safe to destroy it.</div><div class='add'>+	 */</div><div class='add'>+	if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) &amp;&amp;</div><div class='add'>+	    refcount_read(&amp;mr-&gt;mmkey.usecount) != 0 &amp;&amp;</div><div class='add'>+	    xa_erase(&amp;mr_to_mdev(mr)-&gt;odp_mkeys, mlx5_base_mkey(mr-&gt;mmkey.key)))</div><div class='add'>+		mlx5r_deref_wait_odp_mkey(&amp;mr-&gt;mmkey);</div><div class='add'>+</div><div class='add'>+	if (ibmr-&gt;type == IB_MR_TYPE_INTEGRITY) {</div><div class='add'>+		xa_cmpxchg(&amp;dev-&gt;sig_mrs, mlx5_base_mkey(mr-&gt;mmkey.key),</div><div class='add'>+			   mr-&gt;sig, NULL, GFP_KERNEL);</div><div class='add'>+</div><div class='add'>+		if (mr-&gt;mtt_mr) {</div><div class='add'>+			rc = mlx5_ib_dereg_mr(&amp;mr-&gt;mtt_mr-&gt;ibmr, NULL);</div><div class='add'>+			if (rc)</div><div class='add'>+				return rc;</div><div class='add'>+			mr-&gt;mtt_mr = NULL;</div><div class='add'>+		}</div><div class='add'>+		if (mr-&gt;klm_mr) {</div><div class='add'>+			rc = mlx5_ib_dereg_mr(&amp;mr-&gt;klm_mr-&gt;ibmr, NULL);</div><div class='add'>+			if (rc)</div><div class='add'>+				return rc;</div><div class='add'>+			mr-&gt;klm_mr = NULL;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='del'>-	if (mr-&gt;sig) {</div><div class='ctx'> 		if (mlx5_core_destroy_psv(dev-&gt;mdev,</div><div class='ctx'> 					  mr-&gt;sig-&gt;psv_memory.psv_idx))</div><div class='ctx'> 			mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",</div><div class='ctx'> 				     mr-&gt;sig-&gt;psv_memory.psv_idx);</div><div class='del'>-		if (mlx5_core_destroy_psv(dev-&gt;mdev,</div><div class='del'>-					  mr-&gt;sig-&gt;psv_wire.psv_idx))</div><div class='add'>+		if (mlx5_core_destroy_psv(dev-&gt;mdev, mr-&gt;sig-&gt;psv_wire.psv_idx))</div><div class='ctx'> 			mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",</div><div class='ctx'> 				     mr-&gt;sig-&gt;psv_wire.psv_idx);</div><div class='ctx'> 		kfree(mr-&gt;sig);</div><div class='ctx'> 		mr-&gt;sig = NULL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	mlx5_free_priv_descs(mr);</div><div class='add'>+	/* Stop DMA */</div><div class='add'>+	rc = mlx5r_handle_mkey_cleanup(mr);</div><div class='add'>+	if (rc)</div><div class='add'>+		return rc;</div><div class='add'>+</div><div class='add'>+	if (mr-&gt;umem) {</div><div class='add'>+		bool is_odp = is_odp_mr(mr);</div><div class='add'>+</div><div class='add'>+		if (!is_odp)</div><div class='add'>+			atomic_sub(ib_umem_num_pages(mr-&gt;umem),</div><div class='add'>+				   &amp;dev-&gt;mdev-&gt;priv.reg_pages);</div><div class='add'>+		ib_umem_release(mr-&gt;umem);</div><div class='add'>+		if (is_odp)</div><div class='add'>+			mlx5_ib_free_odp_mr(mr);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (!allocated_from_cache)</div><div class='del'>-		destroy_mkey(dev, mr);</div><div class='add'>+	if (!mr-&gt;mmkey.cache_ent)</div><div class='add'>+		mlx5_free_priv_descs(mr);</div><div class='add'>+</div><div class='add'>+	kfree(mr);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)</div><div class='add'>+static int dereg_crossing_data_direct_mr(struct mlx5_ib_dev *dev,</div><div class='add'>+					struct mlx5_ib_mr *mr)</div><div class='ctx'> {</div><div class='del'>-	int npages = mr-&gt;npages;</div><div class='del'>-	struct ib_umem *umem = mr-&gt;umem;</div><div class='add'>+	struct mlx5_ib_mr *dd_crossed_mr = mr-&gt;dd_crossed_mr;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	if (is_odp_mr(mr)) {</div><div class='del'>-		struct ib_umem_odp *umem_odp = to_ib_umem_odp(umem);</div><div class='add'>+	ret = __mlx5_ib_dereg_mr(&amp;mr-&gt;ibmr);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='del'>-		/* Prevent new page faults and</div><div class='del'>-		 * prefetch requests from succeeding</div><div class='del'>-		 */</div><div class='del'>-		mr-&gt;live = 0;</div><div class='del'>-</div><div class='del'>-		/* dequeue pending prefetch requests for the mr */</div><div class='del'>-		if (atomic_read(&amp;mr-&gt;num_pending_prefetch))</div><div class='del'>-			flush_workqueue(system_unbound_wq);</div><div class='del'>-		WARN_ON(atomic_read(&amp;mr-&gt;num_pending_prefetch));</div><div class='del'>-</div><div class='del'>-		/* Wait for all running page-fault handlers to finish. */</div><div class='del'>-		synchronize_srcu(&amp;dev-&gt;mr_srcu);</div><div class='del'>-		/* Destroy all page mappings */</div><div class='del'>-		if (umem_odp-&gt;page_list)</div><div class='del'>-			mlx5_ib_invalidate_range(umem_odp, ib_umem_start(umem),</div><div class='del'>-						 ib_umem_end(umem));</div><div class='del'>-		else</div><div class='del'>-			mlx5_ib_free_implicit_mr(mr);</div><div class='del'>-		/*</div><div class='del'>-		 * We kill the umem before the MR for ODP,</div><div class='del'>-		 * so that there will not be any invalidations in</div><div class='del'>-		 * flight, looking at the *mr struct.</div><div class='del'>-		 */</div><div class='del'>-		ib_umem_release(umem);</div><div class='del'>-		atomic_sub(npages, &amp;dev-&gt;mdev-&gt;priv.reg_pages);</div><div class='add'>+	mutex_lock(&amp;dev-&gt;data_direct_lock);</div><div class='add'>+	if (!dd_crossed_mr-&gt;revoked)</div><div class='add'>+		list_del(&amp;dd_crossed_mr-&gt;dd_node);</div><div class='ctx'> </div><div class='del'>-		/* Avoid double-freeing the umem. */</div><div class='del'>-		umem = NULL;</div><div class='del'>-	}</div><div class='add'>+	ret = __mlx5_ib_dereg_mr(&amp;dd_crossed_mr-&gt;ibmr);</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;data_direct_lock);</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mr *mr = to_mmr(ibmr);</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ibmr-&gt;device);</div><div class='ctx'> </div><div class='del'>-	clean_mr(dev, mr);</div><div class='add'>+	if (mr-&gt;data_direct)</div><div class='add'>+		return dereg_crossing_data_direct_mr(dev, mr);</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * We should unregister the DMA address from the HCA before</div><div class='del'>-	 * remove the DMA mapping.</div><div class='del'>-	 */</div><div class='del'>-	mlx5_mr_cache_free(dev, mr);</div><div class='del'>-	if (umem) {</div><div class='del'>-		ib_umem_release(umem);</div><div class='del'>-		atomic_sub(npages, &amp;dev-&gt;mdev-&gt;priv.reg_pages);</div><div class='del'>-	}</div><div class='del'>-	if (!mr-&gt;allocated_from_cache)</div><div class='del'>-		kfree(mr);</div><div class='add'>+	return __mlx5_ib_dereg_mr(ibmr);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)</div><div class='add'>+static void mlx5_set_umr_free_mkey(struct ib_pd *pd, u32 *in, int ndescs,</div><div class='add'>+				   int access_mode, int page_shift)</div><div class='ctx'> {</div><div class='del'>-	dereg_mr(to_mdev(ibmr-&gt;device), to_mmr(ibmr));</div><div class='del'>-	return 0;</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='add'>+	void *mkc;</div><div class='add'>+</div><div class='add'>+	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);</div><div class='add'>+</div><div class='add'>+	/* This is only used from the kernel, so setting the PD is OK. */</div><div class='add'>+	set_mkc_access_pd_addr_fields(mkc, IB_ACCESS_RELAXED_ORDERING, 0, pd);</div><div class='add'>+	MLX5_SET(mkc, mkc, free, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, translations_octword_size, ndescs);</div><div class='add'>+	MLX5_SET(mkc, mkc, access_mode_1_0, access_mode &amp; 0x3);</div><div class='add'>+	MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode &gt;&gt; 2) &amp; 0x7);</div><div class='add'>+	MLX5_SET(mkc, mkc, umr_en, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, log_page_size, page_shift);</div><div class='add'>+	if (access_mode == MLX5_MKC_ACCESS_MODE_PA ||</div><div class='add'>+	    access_mode == MLX5_MKC_ACCESS_MODE_MTT)</div><div class='add'>+		MLX5_SET(mkc, mkc, ma_translation_mode, MLX5_CAP_GEN(dev-&gt;mdev, ats));</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,</div><div class='del'>-			       u32 max_num_sg, struct ib_udata *udata)</div><div class='add'>+static int _mlx5_alloc_mkey_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,</div><div class='add'>+				  int ndescs, int desc_size, int page_shift,</div><div class='add'>+				  int access_mode, u32 *in, int inlen)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mr-&gt;access_mode = access_mode;</div><div class='add'>+	mr-&gt;desc_size = desc_size;</div><div class='add'>+	mr-&gt;max_descs = ndescs;</div><div class='add'>+</div><div class='add'>+	err = mlx5_alloc_priv_descs(pd-&gt;device, mr, ndescs, desc_size);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	mlx5_set_umr_free_mkey(pd, in, ndescs, access_mode, page_shift);</div><div class='add'>+</div><div class='add'>+	err = mlx5_ib_create_mkey(dev, &amp;mr-&gt;mmkey, in, inlen);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_free_descs;</div><div class='add'>+</div><div class='add'>+	mr-&gt;mmkey.type = MLX5_MKEY_MR;</div><div class='add'>+	mr-&gt;ibmr.lkey = mr-&gt;mmkey.key;</div><div class='add'>+	mr-&gt;ibmr.rkey = mr-&gt;mmkey.key;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_free_descs:</div><div class='add'>+	mlx5_free_priv_descs(mr);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd,</div><div class='add'>+				u32 max_num_sg, u32 max_num_meta_sg,</div><div class='add'>+				int desc_size, int access_mode)</div><div class='add'>+{</div><div class='ctx'> 	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);</div><div class='del'>-	int ndescs = ALIGN(max_num_sg, 4);</div><div class='add'>+	int ndescs = ALIGN(max_num_sg + max_num_meta_sg, 4);</div><div class='add'>+	int page_shift = 0;</div><div class='ctx'> 	struct mlx5_ib_mr *mr;</div><div class='del'>-	void *mkc;</div><div class='ctx'> 	u32 *in;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='hunk'>@@ -1658,99 +2346,175 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,</div><div class='ctx'> 	if (!mr)</div><div class='ctx'> 		return ERR_PTR(-ENOMEM);</div><div class='ctx'> </div><div class='add'>+	mr-&gt;ibmr.pd = pd;</div><div class='add'>+	mr-&gt;ibmr.device = pd-&gt;device;</div><div class='add'>+</div><div class='ctx'> 	in = kzalloc(inlen, GFP_KERNEL);</div><div class='ctx'> 	if (!in) {</div><div class='ctx'> 		err = -ENOMEM;</div><div class='ctx'> 		goto err_free;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (access_mode == MLX5_MKC_ACCESS_MODE_MTT)</div><div class='add'>+		page_shift = PAGE_SHIFT;</div><div class='add'>+</div><div class='add'>+	err = _mlx5_alloc_mkey_descs(pd, mr, ndescs, desc_size, page_shift,</div><div class='add'>+				     access_mode, in, inlen);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_free_in;</div><div class='add'>+</div><div class='add'>+	mr-&gt;umem = NULL;</div><div class='add'>+	kfree(in);</div><div class='add'>+</div><div class='add'>+	return mr;</div><div class='add'>+</div><div class='add'>+err_free_in:</div><div class='add'>+	kfree(in);</div><div class='add'>+err_free:</div><div class='add'>+	kfree(mr);</div><div class='add'>+	return ERR_PTR(err);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_alloc_mem_reg_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,</div><div class='add'>+				    int ndescs, u32 *in, int inlen)</div><div class='add'>+{</div><div class='add'>+	return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_mtt),</div><div class='add'>+				      PAGE_SHIFT, MLX5_MKC_ACCESS_MODE_MTT, in,</div><div class='add'>+				      inlen);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_alloc_sg_gaps_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,</div><div class='add'>+				    int ndescs, u32 *in, int inlen)</div><div class='add'>+{</div><div class='add'>+	return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_klm),</div><div class='add'>+				      0, MLX5_MKC_ACCESS_MODE_KLMS, in, inlen);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_alloc_integrity_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,</div><div class='add'>+				      int max_num_sg, int max_num_meta_sg,</div><div class='add'>+				      u32 *in, int inlen)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='add'>+	u32 psv_index[2];</div><div class='add'>+	void *mkc;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	mr-&gt;sig = kzalloc(sizeof(*mr-&gt;sig), GFP_KERNEL);</div><div class='add'>+	if (!mr-&gt;sig)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	/* create mem &amp; wire PSVs */</div><div class='add'>+	err = mlx5_core_create_psv(dev-&gt;mdev, to_mpd(pd)-&gt;pdn, 2, psv_index);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_free_sig;</div><div class='add'>+</div><div class='add'>+	mr-&gt;sig-&gt;psv_memory.psv_idx = psv_index[0];</div><div class='add'>+	mr-&gt;sig-&gt;psv_wire.psv_idx = psv_index[1];</div><div class='add'>+</div><div class='add'>+	mr-&gt;sig-&gt;sig_status_checked = true;</div><div class='add'>+	mr-&gt;sig-&gt;sig_err_exists = false;</div><div class='add'>+	/* Next UMR, Arm SIGERR */</div><div class='add'>+	++mr-&gt;sig-&gt;sigerr_count;</div><div class='add'>+	mr-&gt;klm_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg,</div><div class='add'>+					 sizeof(struct mlx5_klm),</div><div class='add'>+					 MLX5_MKC_ACCESS_MODE_KLMS);</div><div class='add'>+	if (IS_ERR(mr-&gt;klm_mr)) {</div><div class='add'>+		err = PTR_ERR(mr-&gt;klm_mr);</div><div class='add'>+		goto err_destroy_psv;</div><div class='add'>+	}</div><div class='add'>+	mr-&gt;mtt_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg,</div><div class='add'>+					 sizeof(struct mlx5_mtt),</div><div class='add'>+					 MLX5_MKC_ACCESS_MODE_MTT);</div><div class='add'>+	if (IS_ERR(mr-&gt;mtt_mr)) {</div><div class='add'>+		err = PTR_ERR(mr-&gt;mtt_mr);</div><div class='add'>+		goto err_free_klm_mr;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* Set bsf descriptors for mkey */</div><div class='ctx'> 	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);</div><div class='del'>-	MLX5_SET(mkc, mkc, free, 1);</div><div class='del'>-	MLX5_SET(mkc, mkc, translations_octword_size, ndescs);</div><div class='del'>-	MLX5_SET(mkc, mkc, qpn, 0xffffff);</div><div class='del'>-	MLX5_SET(mkc, mkc, pd, to_mpd(pd)-&gt;pdn);</div><div class='add'>+	MLX5_SET(mkc, mkc, bsf_en, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE);</div><div class='ctx'> </div><div class='del'>-	if (mr_type == IB_MR_TYPE_MEM_REG) {</div><div class='del'>-		mr-&gt;access_mode = MLX5_MKC_ACCESS_MODE_MTT;</div><div class='del'>-		MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);</div><div class='del'>-		err = mlx5_alloc_priv_descs(pd-&gt;device, mr,</div><div class='del'>-					    ndescs, sizeof(struct mlx5_mtt));</div><div class='del'>-		if (err)</div><div class='del'>-			goto err_free_in;</div><div class='add'>+	err = _mlx5_alloc_mkey_descs(pd, mr, 4, sizeof(struct mlx5_klm), 0,</div><div class='add'>+				     MLX5_MKC_ACCESS_MODE_KLMS, in, inlen);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_free_mtt_mr;</div><div class='ctx'> </div><div class='del'>-		mr-&gt;desc_size = sizeof(struct mlx5_mtt);</div><div class='del'>-		mr-&gt;max_descs = ndescs;</div><div class='del'>-	} else if (mr_type == IB_MR_TYPE_SG_GAPS) {</div><div class='del'>-		mr-&gt;access_mode = MLX5_MKC_ACCESS_MODE_KLMS;</div><div class='add'>+	err = xa_err(xa_store(&amp;dev-&gt;sig_mrs, mlx5_base_mkey(mr-&gt;mmkey.key),</div><div class='add'>+			      mr-&gt;sig, GFP_KERNEL));</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_free_descs;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-		err = mlx5_alloc_priv_descs(pd-&gt;device, mr,</div><div class='del'>-					    ndescs, sizeof(struct mlx5_klm));</div><div class='del'>-		if (err)</div><div class='del'>-			goto err_free_in;</div><div class='del'>-		mr-&gt;desc_size = sizeof(struct mlx5_klm);</div><div class='del'>-		mr-&gt;max_descs = ndescs;</div><div class='del'>-	} else if (mr_type == IB_MR_TYPE_SIGNATURE) {</div><div class='del'>-		u32 psv_index[2];</div><div class='del'>-</div><div class='del'>-		MLX5_SET(mkc, mkc, bsf_en, 1);</div><div class='del'>-		MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE);</div><div class='del'>-		mr-&gt;sig = kzalloc(sizeof(*mr-&gt;sig), GFP_KERNEL);</div><div class='del'>-		if (!mr-&gt;sig) {</div><div class='del'>-			err = -ENOMEM;</div><div class='del'>-			goto err_free_in;</div><div class='del'>-		}</div><div class='add'>+err_free_descs:</div><div class='add'>+	destroy_mkey(dev, mr);</div><div class='add'>+	mlx5_free_priv_descs(mr);</div><div class='add'>+err_free_mtt_mr:</div><div class='add'>+	mlx5_ib_dereg_mr(&amp;mr-&gt;mtt_mr-&gt;ibmr, NULL);</div><div class='add'>+	mr-&gt;mtt_mr = NULL;</div><div class='add'>+err_free_klm_mr:</div><div class='add'>+	mlx5_ib_dereg_mr(&amp;mr-&gt;klm_mr-&gt;ibmr, NULL);</div><div class='add'>+	mr-&gt;klm_mr = NULL;</div><div class='add'>+err_destroy_psv:</div><div class='add'>+	if (mlx5_core_destroy_psv(dev-&gt;mdev, mr-&gt;sig-&gt;psv_memory.psv_idx))</div><div class='add'>+		mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",</div><div class='add'>+			     mr-&gt;sig-&gt;psv_memory.psv_idx);</div><div class='add'>+	if (mlx5_core_destroy_psv(dev-&gt;mdev, mr-&gt;sig-&gt;psv_wire.psv_idx))</div><div class='add'>+		mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",</div><div class='add'>+			     mr-&gt;sig-&gt;psv_wire.psv_idx);</div><div class='add'>+err_free_sig:</div><div class='add'>+	kfree(mr-&gt;sig);</div><div class='ctx'> </div><div class='del'>-		/* create mem &amp; wire PSVs */</div><div class='del'>-		err = mlx5_core_create_psv(dev-&gt;mdev, to_mpd(pd)-&gt;pdn,</div><div class='del'>-					   2, psv_index);</div><div class='del'>-		if (err)</div><div class='del'>-			goto err_free_sig;</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		mr-&gt;access_mode = MLX5_MKC_ACCESS_MODE_KLMS;</div><div class='del'>-		mr-&gt;sig-&gt;psv_memory.psv_idx = psv_index[0];</div><div class='del'>-		mr-&gt;sig-&gt;psv_wire.psv_idx = psv_index[1];</div><div class='add'>+static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd,</div><div class='add'>+					enum ib_mr_type mr_type, u32 max_num_sg,</div><div class='add'>+					u32 max_num_meta_sg)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='add'>+	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);</div><div class='add'>+	int ndescs = ALIGN(max_num_sg, 4);</div><div class='add'>+	struct mlx5_ib_mr *mr;</div><div class='add'>+	u32 *in;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-		mr-&gt;sig-&gt;sig_status_checked = true;</div><div class='del'>-		mr-&gt;sig-&gt;sig_err_exists = false;</div><div class='del'>-		/* Next UMR, Arm SIGERR */</div><div class='del'>-		++mr-&gt;sig-&gt;sigerr_count;</div><div class='del'>-	} else {</div><div class='add'>+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);</div><div class='add'>+	if (!mr)</div><div class='add'>+		return ERR_PTR(-ENOMEM);</div><div class='add'>+</div><div class='add'>+	in = kzalloc(inlen, GFP_KERNEL);</div><div class='add'>+	if (!in) {</div><div class='add'>+		err = -ENOMEM;</div><div class='add'>+		goto err_free;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mr-&gt;ibmr.device = pd-&gt;device;</div><div class='add'>+	mr-&gt;umem = NULL;</div><div class='add'>+</div><div class='add'>+	switch (mr_type) {</div><div class='add'>+	case IB_MR_TYPE_MEM_REG:</div><div class='add'>+		err = mlx5_alloc_mem_reg_descs(pd, mr, ndescs, in, inlen);</div><div class='add'>+		break;</div><div class='add'>+	case IB_MR_TYPE_SG_GAPS:</div><div class='add'>+		err = mlx5_alloc_sg_gaps_descs(pd, mr, ndescs, in, inlen);</div><div class='add'>+		break;</div><div class='add'>+	case IB_MR_TYPE_INTEGRITY:</div><div class='add'>+		err = mlx5_alloc_integrity_descs(pd, mr, max_num_sg,</div><div class='add'>+						 max_num_meta_sg, in, inlen);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='ctx'> 		mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type);</div><div class='ctx'> 		err = -EINVAL;</div><div class='del'>-		goto err_free_in;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	MLX5_SET(mkc, mkc, access_mode_1_0, mr-&gt;access_mode &amp; 0x3);</div><div class='del'>-	MLX5_SET(mkc, mkc, access_mode_4_2, (mr-&gt;access_mode &gt;&gt; 2) &amp; 0x7);</div><div class='del'>-	MLX5_SET(mkc, mkc, umr_en, 1);</div><div class='del'>-</div><div class='del'>-	mr-&gt;ibmr.device = pd-&gt;device;</div><div class='del'>-	err = mlx5_core_create_mkey(dev-&gt;mdev, &amp;mr-&gt;mmkey, in, inlen);</div><div class='ctx'> 	if (err)</div><div class='del'>-		goto err_destroy_psv;</div><div class='add'>+		goto err_free_in;</div><div class='ctx'> </div><div class='del'>-	mr-&gt;mmkey.type = MLX5_MKEY_MR;</div><div class='del'>-	mr-&gt;ibmr.lkey = mr-&gt;mmkey.key;</div><div class='del'>-	mr-&gt;ibmr.rkey = mr-&gt;mmkey.key;</div><div class='del'>-	mr-&gt;umem = NULL;</div><div class='ctx'> 	kfree(in);</div><div class='ctx'> </div><div class='ctx'> 	return &amp;mr-&gt;ibmr;</div><div class='ctx'> </div><div class='del'>-err_destroy_psv:</div><div class='del'>-	if (mr-&gt;sig) {</div><div class='del'>-		if (mlx5_core_destroy_psv(dev-&gt;mdev,</div><div class='del'>-					  mr-&gt;sig-&gt;psv_memory.psv_idx))</div><div class='del'>-			mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",</div><div class='del'>-				     mr-&gt;sig-&gt;psv_memory.psv_idx);</div><div class='del'>-		if (mlx5_core_destroy_psv(dev-&gt;mdev,</div><div class='del'>-					  mr-&gt;sig-&gt;psv_wire.psv_idx))</div><div class='del'>-			mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",</div><div class='del'>-				     mr-&gt;sig-&gt;psv_wire.psv_idx);</div><div class='del'>-	}</div><div class='del'>-	mlx5_free_priv_descs(mr);</div><div class='del'>-err_free_sig:</div><div class='del'>-	kfree(mr-&gt;sig);</div><div class='ctx'> err_free_in:</div><div class='ctx'> 	kfree(in);</div><div class='ctx'> err_free:</div><div class='hunk'>@@ -1758,15 +2522,27 @@ err_free:</div><div class='ctx'> 	return ERR_PTR(err);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,</div><div class='del'>-			       struct ib_udata *udata)</div><div class='add'>+struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,</div><div class='add'>+			       u32 max_num_sg)</div><div class='add'>+{</div><div class='add'>+	return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd,</div><div class='add'>+					 u32 max_num_sg, u32 max_num_meta_sg)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='add'>+	return __mlx5_ib_alloc_mr(pd, IB_MR_TYPE_INTEGRITY, max_num_sg,</div><div class='add'>+				  max_num_meta_sg);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ibmw-&gt;device);</div><div class='ctx'> 	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);</div><div class='del'>-	struct mlx5_ib_mw *mw = NULL;</div><div class='add'>+	struct mlx5_ib_mw *mw = to_mmw(ibmw);</div><div class='add'>+	unsigned int ndescs;</div><div class='ctx'> 	u32 *in = NULL;</div><div class='ctx'> 	void *mkc;</div><div class='del'>-	int ndescs;</div><div class='ctx'> 	int err;</div><div class='ctx'> 	struct mlx5_ib_alloc_mw req = {};</div><div class='ctx'> 	struct {</div><div class='hunk'>@@ -1776,73 +2552,79 @@ struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,</div><div class='ctx'> </div><div class='ctx'> 	err = ib_copy_from_udata(&amp;req, udata, min(udata-&gt;inlen, sizeof(req)));</div><div class='ctx'> 	if (err)</div><div class='del'>-		return ERR_PTR(err);</div><div class='add'>+		return err;</div><div class='ctx'> </div><div class='ctx'> 	if (req.comp_mask || req.reserved1 || req.reserved2)</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	if (udata-&gt;inlen &gt; sizeof(req) &amp;&amp;</div><div class='ctx'> 	    !ib_is_udata_cleared(udata, sizeof(req),</div><div class='ctx'> 				 udata-&gt;inlen - sizeof(req)))</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4);</div><div class='ctx'> </div><div class='del'>-	mw = kzalloc(sizeof(*mw), GFP_KERNEL);</div><div class='ctx'> 	in = kzalloc(inlen, GFP_KERNEL);</div><div class='del'>-	if (!mw || !in) {</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto free;</div><div class='del'>-	}</div><div class='add'>+	if (!in)</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(mkc, mkc, free, 1);</div><div class='ctx'> 	MLX5_SET(mkc, mkc, translations_octword_size, ndescs);</div><div class='del'>-	MLX5_SET(mkc, mkc, pd, to_mpd(pd)-&gt;pdn);</div><div class='add'>+	MLX5_SET(mkc, mkc, pd, to_mpd(ibmw-&gt;pd)-&gt;pdn);</div><div class='ctx'> 	MLX5_SET(mkc, mkc, umr_en, 1);</div><div class='ctx'> 	MLX5_SET(mkc, mkc, lr, 1);</div><div class='ctx'> 	MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS);</div><div class='del'>-	MLX5_SET(mkc, mkc, en_rinval, !!((type == IB_MW_TYPE_2)));</div><div class='add'>+	MLX5_SET(mkc, mkc, en_rinval, !!((ibmw-&gt;type == IB_MW_TYPE_2)));</div><div class='ctx'> 	MLX5_SET(mkc, mkc, qpn, 0xffffff);</div><div class='ctx'> </div><div class='del'>-	err = mlx5_core_create_mkey(dev-&gt;mdev, &amp;mw-&gt;mmkey, in, inlen);</div><div class='add'>+	err = mlx5_ib_create_mkey(dev, &amp;mw-&gt;mmkey, in, inlen);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto free;</div><div class='ctx'> </div><div class='ctx'> 	mw-&gt;mmkey.type = MLX5_MKEY_MW;</div><div class='del'>-	mw-&gt;ibmw.rkey = mw-&gt;mmkey.key;</div><div class='del'>-	mw-&gt;ndescs = ndescs;</div><div class='add'>+	ibmw-&gt;rkey = mw-&gt;mmkey.key;</div><div class='add'>+	mw-&gt;mmkey.ndescs = ndescs;</div><div class='ctx'> </div><div class='del'>-	resp.response_length = min(offsetof(typeof(resp), response_length) +</div><div class='del'>-				   sizeof(resp.response_length), udata-&gt;outlen);</div><div class='add'>+	resp.response_length =</div><div class='add'>+		min(offsetofend(typeof(resp), response_length), udata-&gt;outlen);</div><div class='ctx'> 	if (resp.response_length) {</div><div class='ctx'> 		err = ib_copy_to_udata(udata, &amp;resp, resp.response_length);</div><div class='del'>-		if (err) {</div><div class='del'>-			mlx5_core_destroy_mkey(dev-&gt;mdev, &amp;mw-&gt;mmkey);</div><div class='del'>-			goto free;</div><div class='del'>-		}</div><div class='add'>+		if (err)</div><div class='add'>+			goto free_mkey;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {</div><div class='add'>+		err = mlx5r_store_odp_mkey(dev, &amp;mw-&gt;mmkey);</div><div class='add'>+		if (err)</div><div class='add'>+			goto free_mkey;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	kfree(in);</div><div class='del'>-	return &amp;mw-&gt;ibmw;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='add'>+free_mkey:</div><div class='add'>+	mlx5_core_destroy_mkey(dev-&gt;mdev, mw-&gt;mmkey.key);</div><div class='ctx'> free:</div><div class='del'>-	kfree(mw);</div><div class='ctx'> 	kfree(in);</div><div class='del'>-	return ERR_PTR(err);</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_dealloc_mw(struct ib_mw *mw)</div><div class='ctx'> {</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(mw-&gt;device);</div><div class='ctx'> 	struct mlx5_ib_mw *mmw = to_mmw(mw);</div><div class='del'>-	int err;</div><div class='ctx'> </div><div class='del'>-	err =  mlx5_core_destroy_mkey((to_mdev(mw-&gt;device))-&gt;mdev,</div><div class='del'>-				      &amp;mmw-&gt;mmkey);</div><div class='del'>-	if (!err)</div><div class='del'>-		kfree(mmw);</div><div class='del'>-	return err;</div><div class='add'>+	if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) &amp;&amp;</div><div class='add'>+	    xa_erase(&amp;dev-&gt;odp_mkeys, mlx5_base_mkey(mmw-&gt;mmkey.key)))</div><div class='add'>+		/*</div><div class='add'>+		 * pagefault_single_data_segment() may be accessing mmw</div><div class='add'>+		 * if the user bound an ODP MR to this MW.</div><div class='add'>+		 */</div><div class='add'>+		mlx5r_deref_wait_odp_mkey(&amp;mmw-&gt;mmkey);</div><div class='add'>+</div><div class='add'>+	return mlx5_core_destroy_mkey(dev-&gt;mdev, mmw-&gt;mmkey.key);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,</div><div class='hunk'>@@ -1887,16 +2669,53 @@ done:</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int</div><div class='add'>+mlx5_ib_map_pa_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,</div><div class='add'>+			int data_sg_nents, unsigned int *data_sg_offset,</div><div class='add'>+			struct scatterlist *meta_sg, int meta_sg_nents,</div><div class='add'>+			unsigned int *meta_sg_offset)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mr *mr = to_mmr(ibmr);</div><div class='add'>+	unsigned int sg_offset = 0;</div><div class='add'>+	int n = 0;</div><div class='add'>+</div><div class='add'>+	mr-&gt;meta_length = 0;</div><div class='add'>+	if (data_sg_nents == 1) {</div><div class='add'>+		n++;</div><div class='add'>+		mr-&gt;mmkey.ndescs = 1;</div><div class='add'>+		if (data_sg_offset)</div><div class='add'>+			sg_offset = *data_sg_offset;</div><div class='add'>+		mr-&gt;data_length = sg_dma_len(data_sg) - sg_offset;</div><div class='add'>+		mr-&gt;data_iova = sg_dma_address(data_sg) + sg_offset;</div><div class='add'>+		if (meta_sg_nents == 1) {</div><div class='add'>+			n++;</div><div class='add'>+			mr-&gt;meta_ndescs = 1;</div><div class='add'>+			if (meta_sg_offset)</div><div class='add'>+				sg_offset = *meta_sg_offset;</div><div class='add'>+			else</div><div class='add'>+				sg_offset = 0;</div><div class='add'>+			mr-&gt;meta_length = sg_dma_len(meta_sg) - sg_offset;</div><div class='add'>+			mr-&gt;pi_iova = sg_dma_address(meta_sg) + sg_offset;</div><div class='add'>+		}</div><div class='add'>+		ibmr-&gt;length = mr-&gt;data_length + mr-&gt;meta_length;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return n;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int</div><div class='ctx'> mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,</div><div class='ctx'> 		   struct scatterlist *sgl,</div><div class='ctx'> 		   unsigned short sg_nents,</div><div class='del'>-		   unsigned int *sg_offset_p)</div><div class='add'>+		   unsigned int *sg_offset_p,</div><div class='add'>+		   struct scatterlist *meta_sgl,</div><div class='add'>+		   unsigned short meta_sg_nents,</div><div class='add'>+		   unsigned int *meta_sg_offset_p)</div><div class='ctx'> {</div><div class='ctx'> 	struct scatterlist *sg = sgl;</div><div class='ctx'> 	struct mlx5_klm *klms = mr-&gt;descs;</div><div class='ctx'> 	unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;</div><div class='ctx'> 	u32 lkey = mr-&gt;ibmr.pd-&gt;local_dma_lkey;</div><div class='del'>-	int i;</div><div class='add'>+	int i, j = 0;</div><div class='ctx'> </div><div class='ctx'> 	mr-&gt;ibmr.iova = sg_dma_address(sg) + sg_offset;</div><div class='ctx'> 	mr-&gt;ibmr.length = 0;</div><div class='hunk'>@@ -1911,12 +2730,36 @@ mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,</div><div class='ctx'> </div><div class='ctx'> 		sg_offset = 0;</div><div class='ctx'> 	}</div><div class='del'>-	mr-&gt;ndescs = i;</div><div class='ctx'> </div><div class='ctx'> 	if (sg_offset_p)</div><div class='ctx'> 		*sg_offset_p = sg_offset;</div><div class='ctx'> </div><div class='del'>-	return i;</div><div class='add'>+	mr-&gt;mmkey.ndescs = i;</div><div class='add'>+	mr-&gt;data_length = mr-&gt;ibmr.length;</div><div class='add'>+</div><div class='add'>+	if (meta_sg_nents) {</div><div class='add'>+		sg = meta_sgl;</div><div class='add'>+		sg_offset = meta_sg_offset_p ? *meta_sg_offset_p : 0;</div><div class='add'>+		for_each_sg(meta_sgl, sg, meta_sg_nents, j) {</div><div class='add'>+			if (unlikely(i + j &gt;= mr-&gt;max_descs))</div><div class='add'>+				break;</div><div class='add'>+			klms[i + j].va = cpu_to_be64(sg_dma_address(sg) +</div><div class='add'>+						     sg_offset);</div><div class='add'>+			klms[i + j].bcount = cpu_to_be32(sg_dma_len(sg) -</div><div class='add'>+							 sg_offset);</div><div class='add'>+			klms[i + j].key = cpu_to_be32(lkey);</div><div class='add'>+			mr-&gt;ibmr.length += sg_dma_len(sg) - sg_offset;</div><div class='add'>+</div><div class='add'>+			sg_offset = 0;</div><div class='add'>+		}</div><div class='add'>+		if (meta_sg_offset_p)</div><div class='add'>+			*meta_sg_offset_p = sg_offset;</div><div class='add'>+</div><div class='add'>+		mr-&gt;meta_ndescs = j;</div><div class='add'>+		mr-&gt;meta_length = mr-&gt;ibmr.length - mr-&gt;data_length;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return i + j;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int mlx5_set_page(struct ib_mr *ibmr, u64 addr)</div><div class='hunk'>@@ -1924,11 +2767,186 @@ static int mlx5_set_page(struct ib_mr *ibmr, u64 addr)</div><div class='ctx'> 	struct mlx5_ib_mr *mr = to_mmr(ibmr);</div><div class='ctx'> 	__be64 *descs;</div><div class='ctx'> </div><div class='del'>-	if (unlikely(mr-&gt;ndescs == mr-&gt;max_descs))</div><div class='add'>+	if (unlikely(mr-&gt;mmkey.ndescs == mr-&gt;max_descs))</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	descs = mr-&gt;descs;</div><div class='del'>-	descs[mr-&gt;ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);</div><div class='add'>+	descs[mr-&gt;mmkey.ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_set_page_pi(struct ib_mr *ibmr, u64 addr)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mr *mr = to_mmr(ibmr);</div><div class='add'>+	__be64 *descs;</div><div class='add'>+</div><div class='add'>+	if (unlikely(mr-&gt;mmkey.ndescs + mr-&gt;meta_ndescs == mr-&gt;max_descs))</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	descs = mr-&gt;descs;</div><div class='add'>+	descs[mr-&gt;mmkey.ndescs + mr-&gt;meta_ndescs++] =</div><div class='add'>+		cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int</div><div class='add'>+mlx5_ib_map_mtt_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,</div><div class='add'>+			 int data_sg_nents, unsigned int *data_sg_offset,</div><div class='add'>+			 struct scatterlist *meta_sg, int meta_sg_nents,</div><div class='add'>+			 unsigned int *meta_sg_offset)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mr *mr = to_mmr(ibmr);</div><div class='add'>+	struct mlx5_ib_mr *pi_mr = mr-&gt;mtt_mr;</div><div class='add'>+	int n;</div><div class='add'>+</div><div class='add'>+	pi_mr-&gt;mmkey.ndescs = 0;</div><div class='add'>+	pi_mr-&gt;meta_ndescs = 0;</div><div class='add'>+	pi_mr-&gt;meta_length = 0;</div><div class='add'>+</div><div class='add'>+	ib_dma_sync_single_for_cpu(ibmr-&gt;device, pi_mr-&gt;desc_map,</div><div class='add'>+				   pi_mr-&gt;desc_size * pi_mr-&gt;max_descs,</div><div class='add'>+				   DMA_TO_DEVICE);</div><div class='add'>+</div><div class='add'>+	pi_mr-&gt;ibmr.page_size = ibmr-&gt;page_size;</div><div class='add'>+	n = ib_sg_to_pages(&amp;pi_mr-&gt;ibmr, data_sg, data_sg_nents, data_sg_offset,</div><div class='add'>+			   mlx5_set_page);</div><div class='add'>+	if (n != data_sg_nents)</div><div class='add'>+		return n;</div><div class='add'>+</div><div class='add'>+	pi_mr-&gt;data_iova = pi_mr-&gt;ibmr.iova;</div><div class='add'>+	pi_mr-&gt;data_length = pi_mr-&gt;ibmr.length;</div><div class='add'>+	pi_mr-&gt;ibmr.length = pi_mr-&gt;data_length;</div><div class='add'>+	ibmr-&gt;length = pi_mr-&gt;data_length;</div><div class='add'>+</div><div class='add'>+	if (meta_sg_nents) {</div><div class='add'>+		u64 page_mask = ~((u64)ibmr-&gt;page_size - 1);</div><div class='add'>+		u64 iova = pi_mr-&gt;data_iova;</div><div class='add'>+</div><div class='add'>+		n += ib_sg_to_pages(&amp;pi_mr-&gt;ibmr, meta_sg, meta_sg_nents,</div><div class='add'>+				    meta_sg_offset, mlx5_set_page_pi);</div><div class='add'>+</div><div class='add'>+		pi_mr-&gt;meta_length = pi_mr-&gt;ibmr.length;</div><div class='add'>+		/*</div><div class='add'>+		 * PI address for the HW is the offset of the metadata address</div><div class='add'>+		 * relative to the first data page address.</div><div class='add'>+		 * It equals to first data page address + size of data pages +</div><div class='add'>+		 * metadata offset at the first metadata page</div><div class='add'>+		 */</div><div class='add'>+		pi_mr-&gt;pi_iova = (iova &amp; page_mask) +</div><div class='add'>+				 pi_mr-&gt;mmkey.ndescs * ibmr-&gt;page_size +</div><div class='add'>+				 (pi_mr-&gt;ibmr.iova &amp; ~page_mask);</div><div class='add'>+		/*</div><div class='add'>+		 * In order to use one MTT MR for data and metadata, we register</div><div class='add'>+		 * also the gaps between the end of the data and the start of</div><div class='add'>+		 * the metadata (the sig MR will verify that the HW will access</div><div class='add'>+		 * to right addresses). This mapping is safe because we use</div><div class='add'>+		 * internal mkey for the registration.</div><div class='add'>+		 */</div><div class='add'>+		pi_mr-&gt;ibmr.length = pi_mr-&gt;pi_iova + pi_mr-&gt;meta_length - iova;</div><div class='add'>+		pi_mr-&gt;ibmr.iova = iova;</div><div class='add'>+		ibmr-&gt;length += pi_mr-&gt;meta_length;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	ib_dma_sync_single_for_device(ibmr-&gt;device, pi_mr-&gt;desc_map,</div><div class='add'>+				      pi_mr-&gt;desc_size * pi_mr-&gt;max_descs,</div><div class='add'>+				      DMA_TO_DEVICE);</div><div class='add'>+</div><div class='add'>+	return n;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int</div><div class='add'>+mlx5_ib_map_klm_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,</div><div class='add'>+			 int data_sg_nents, unsigned int *data_sg_offset,</div><div class='add'>+			 struct scatterlist *meta_sg, int meta_sg_nents,</div><div class='add'>+			 unsigned int *meta_sg_offset)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mr *mr = to_mmr(ibmr);</div><div class='add'>+	struct mlx5_ib_mr *pi_mr = mr-&gt;klm_mr;</div><div class='add'>+	int n;</div><div class='add'>+</div><div class='add'>+	pi_mr-&gt;mmkey.ndescs = 0;</div><div class='add'>+	pi_mr-&gt;meta_ndescs = 0;</div><div class='add'>+	pi_mr-&gt;meta_length = 0;</div><div class='add'>+</div><div class='add'>+	ib_dma_sync_single_for_cpu(ibmr-&gt;device, pi_mr-&gt;desc_map,</div><div class='add'>+				   pi_mr-&gt;desc_size * pi_mr-&gt;max_descs,</div><div class='add'>+				   DMA_TO_DEVICE);</div><div class='add'>+</div><div class='add'>+	n = mlx5_ib_sg_to_klms(pi_mr, data_sg, data_sg_nents, data_sg_offset,</div><div class='add'>+			       meta_sg, meta_sg_nents, meta_sg_offset);</div><div class='add'>+</div><div class='add'>+	ib_dma_sync_single_for_device(ibmr-&gt;device, pi_mr-&gt;desc_map,</div><div class='add'>+				      pi_mr-&gt;desc_size * pi_mr-&gt;max_descs,</div><div class='add'>+				      DMA_TO_DEVICE);</div><div class='add'>+</div><div class='add'>+	/* This is zero-based memory region */</div><div class='add'>+	pi_mr-&gt;data_iova = 0;</div><div class='add'>+	pi_mr-&gt;ibmr.iova = 0;</div><div class='add'>+	pi_mr-&gt;pi_iova = pi_mr-&gt;data_length;</div><div class='add'>+	ibmr-&gt;length = pi_mr-&gt;ibmr.length;</div><div class='add'>+</div><div class='add'>+	return n;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,</div><div class='add'>+			 int data_sg_nents, unsigned int *data_sg_offset,</div><div class='add'>+			 struct scatterlist *meta_sg, int meta_sg_nents,</div><div class='add'>+			 unsigned int *meta_sg_offset)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mr *mr = to_mmr(ibmr);</div><div class='add'>+	struct mlx5_ib_mr *pi_mr = NULL;</div><div class='add'>+	int n;</div><div class='add'>+</div><div class='add'>+	WARN_ON(ibmr-&gt;type != IB_MR_TYPE_INTEGRITY);</div><div class='add'>+</div><div class='add'>+	mr-&gt;mmkey.ndescs = 0;</div><div class='add'>+	mr-&gt;data_length = 0;</div><div class='add'>+	mr-&gt;data_iova = 0;</div><div class='add'>+	mr-&gt;meta_ndescs = 0;</div><div class='add'>+	mr-&gt;pi_iova = 0;</div><div class='add'>+	/*</div><div class='add'>+	 * As a performance optimization, if possible, there is no need to</div><div class='add'>+	 * perform UMR operation to register the data/metadata buffers.</div><div class='add'>+	 * First try to map the sg lists to PA descriptors with local_dma_lkey.</div><div class='add'>+	 * Fallback to UMR only in case of a failure.</div><div class='add'>+	 */</div><div class='add'>+	n = mlx5_ib_map_pa_mr_sg_pi(ibmr, data_sg, data_sg_nents,</div><div class='add'>+				    data_sg_offset, meta_sg, meta_sg_nents,</div><div class='add'>+				    meta_sg_offset);</div><div class='add'>+	if (n == data_sg_nents + meta_sg_nents)</div><div class='add'>+		goto out;</div><div class='add'>+	/*</div><div class='add'>+	 * As a performance optimization, if possible, there is no need to map</div><div class='add'>+	 * the sg lists to KLM descriptors. First try to map the sg lists to MTT</div><div class='add'>+	 * descriptors and fallback to KLM only in case of a failure.</div><div class='add'>+	 * It's more efficient for the HW to work with MTT descriptors</div><div class='add'>+	 * (especially in high load).</div><div class='add'>+	 * Use KLM (indirect access) only if it's mandatory.</div><div class='add'>+	 */</div><div class='add'>+	pi_mr = mr-&gt;mtt_mr;</div><div class='add'>+	n = mlx5_ib_map_mtt_mr_sg_pi(ibmr, data_sg, data_sg_nents,</div><div class='add'>+				     data_sg_offset, meta_sg, meta_sg_nents,</div><div class='add'>+				     meta_sg_offset);</div><div class='add'>+	if (n == data_sg_nents + meta_sg_nents)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	pi_mr = mr-&gt;klm_mr;</div><div class='add'>+	n = mlx5_ib_map_klm_mr_sg_pi(ibmr, data_sg, data_sg_nents,</div><div class='add'>+				     data_sg_offset, meta_sg, meta_sg_nents,</div><div class='add'>+				     meta_sg_offset);</div><div class='add'>+	if (unlikely(n != data_sg_nents + meta_sg_nents))</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	/* This is zero-based memory region */</div><div class='add'>+	ibmr-&gt;iova = 0;</div><div class='add'>+	mr-&gt;pi_mr = pi_mr;</div><div class='add'>+	if (pi_mr)</div><div class='add'>+		ibmr-&gt;sig_attrs-&gt;meta_length = pi_mr-&gt;meta_length;</div><div class='add'>+	else</div><div class='add'>+		ibmr-&gt;sig_attrs-&gt;meta_length = mr-&gt;meta_length;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='hunk'>@@ -1939,14 +2957,15 @@ int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,</div><div class='ctx'> 	struct mlx5_ib_mr *mr = to_mmr(ibmr);</div><div class='ctx'> 	int n;</div><div class='ctx'> </div><div class='del'>-	mr-&gt;ndescs = 0;</div><div class='add'>+	mr-&gt;mmkey.ndescs = 0;</div><div class='ctx'> </div><div class='ctx'> 	ib_dma_sync_single_for_cpu(ibmr-&gt;device, mr-&gt;desc_map,</div><div class='ctx'> 				   mr-&gt;desc_size * mr-&gt;max_descs,</div><div class='ctx'> 				   DMA_TO_DEVICE);</div><div class='ctx'> </div><div class='ctx'> 	if (mr-&gt;access_mode == MLX5_MKC_ACCESS_MODE_KLMS)</div><div class='del'>-		n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset);</div><div class='add'>+		n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset, NULL, 0,</div><div class='add'>+				       NULL);</div><div class='ctx'> 	else</div><div class='ctx'> 		n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset,</div><div class='ctx'> 				mlx5_set_page);</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c<br/>index c594489eb2d7..e71ee3d52eb0 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/odp.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/odp.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/odp.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/odp.c</a></div><div class='hunk'>@@ -30,19 +30,25 @@</div><div class='ctx'>  * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;rdma/ib_umem.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_umem_odp.h&gt;</div><div class='ctx'> #include &lt;linux/kernel.h&gt;</div><div class='add'>+#include &lt;linux/dma-buf.h&gt;</div><div class='add'>+#include &lt;linux/dma-resv.h&gt;</div><div class='add'>+#include &lt;linux/hmm.h&gt;</div><div class='add'>+#include &lt;linux/hmm-dma.h&gt;</div><div class='add'>+#include &lt;linux/pci-p2pdma.h&gt;</div><div class='ctx'> </div><div class='ctx'> #include "mlx5_ib.h"</div><div class='ctx'> #include "cmd.h"</div><div class='add'>+#include "umr.h"</div><div class='add'>+#include "qp.h"</div><div class='ctx'> </div><div class='ctx'> #include &lt;linux/mlx5/eq.h&gt;</div><div class='ctx'> </div><div class='ctx'> /* Contains the details of a pagefault. */</div><div class='ctx'> struct mlx5_pagefault {</div><div class='ctx'> 	u32			bytes_committed;</div><div class='del'>-	u32			token;</div><div class='add'>+	u64			token;</div><div class='ctx'> 	u8			event_subtype;</div><div class='ctx'> 	u8			type;</div><div class='ctx'> 	union {</div><div class='hunk'>@@ -71,6 +77,14 @@ struct mlx5_pagefault {</div><div class='ctx'> 			u32	rdma_op_len;</div><div class='ctx'> 			u64	rdma_va;</div><div class='ctx'> 		} rdma;</div><div class='add'>+		struct {</div><div class='add'>+			u64	va;</div><div class='add'>+			u32	mkey;</div><div class='add'>+			u32	fault_byte_count;</div><div class='add'>+			u32     prefetch_before_byte_count;</div><div class='add'>+			u32     prefetch_after_byte_count;</div><div class='add'>+			u8	flags;</div><div class='add'>+		} memory;</div><div class='ctx'> 	};</div><div class='ctx'> </div><div class='ctx'> 	struct mlx5_ib_pf_eq	*eq;</div><div class='hunk'>@@ -83,164 +97,203 @@ struct mlx5_pagefault {</div><div class='ctx'>  * a pagefault. */</div><div class='ctx'> #define MMU_NOTIFIER_TIMEOUT 1000</div><div class='ctx'> </div><div class='del'>-#define MLX5_IMR_MTT_BITS (30 - PAGE_SHIFT)</div><div class='del'>-#define MLX5_IMR_MTT_SHIFT (MLX5_IMR_MTT_BITS + PAGE_SHIFT)</div><div class='del'>-#define MLX5_IMR_MTT_ENTRIES BIT_ULL(MLX5_IMR_MTT_BITS)</div><div class='del'>-#define MLX5_IMR_MTT_SIZE BIT_ULL(MLX5_IMR_MTT_SHIFT)</div><div class='del'>-#define MLX5_IMR_MTT_MASK (~(MLX5_IMR_MTT_SIZE - 1))</div><div class='del'>-</div><div class='del'>-#define MLX5_KSM_PAGE_SHIFT MLX5_IMR_MTT_SHIFT</div><div class='del'>-</div><div class='ctx'> static u64 mlx5_imr_ksm_entries;</div><div class='add'>+static u64 mlx5_imr_mtt_entries;</div><div class='add'>+static u64 mlx5_imr_mtt_size;</div><div class='add'>+static u8 mlx5_imr_mtt_shift;</div><div class='add'>+static u8 mlx5_imr_ksm_page_shift;</div><div class='ctx'> </div><div class='del'>-static int check_parent(struct ib_umem_odp *odp,</div><div class='del'>-			       struct mlx5_ib_mr *parent)</div><div class='add'>+static void populate_ksm(struct mlx5_ksm *pksm, size_t idx, size_t nentries,</div><div class='add'>+			struct mlx5_ib_mr *imr, int flags)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_mr *mr = odp-&gt;private;</div><div class='add'>+	struct mlx5_core_dev *dev = mr_to_mdev(imr)-&gt;mdev;</div><div class='add'>+	struct mlx5_ksm *end = pksm + nentries;</div><div class='add'>+	u64 step = MLX5_CAP_ODP(dev, mem_page_fault) ? mlx5_imr_mtt_size : 0;</div><div class='add'>+	__be32 key = MLX5_CAP_ODP(dev, mem_page_fault) ?</div><div class='add'>+			     cpu_to_be32(imr-&gt;null_mmkey.key) :</div><div class='add'>+			     mr_to_mdev(imr)-&gt;mkeys.null_mkey;</div><div class='add'>+	u64 va =</div><div class='add'>+		MLX5_CAP_ODP(dev, mem_page_fault) ? idx * mlx5_imr_mtt_size : 0;</div><div class='ctx'> </div><div class='del'>-	return mr &amp;&amp; mr-&gt;parent == parent &amp;&amp; !odp-&gt;dying;</div><div class='del'>-}</div><div class='add'>+	if (flags &amp; MLX5_IB_UPD_XLT_ZAP) {</div><div class='add'>+		for (; pksm != end; pksm++, idx++, va += step) {</div><div class='add'>+			pksm-&gt;key = key;</div><div class='add'>+			pksm-&gt;va = cpu_to_be64(va);</div><div class='add'>+		}</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-static struct ib_ucontext_per_mm *mr_to_per_mm(struct mlx5_ib_mr *mr)</div><div class='del'>-{</div><div class='del'>-	if (WARN_ON(!mr || !is_odp_mr(mr)))</div><div class='del'>-		return NULL;</div><div class='add'>+	/*</div><div class='add'>+	 * The locking here is pretty subtle. Ideally the implicit_children</div><div class='add'>+	 * xarray would be protected by the umem_mutex, however that is not</div><div class='add'>+	 * possible. Instead this uses a weaker update-then-lock pattern:</div><div class='add'>+	 *</div><div class='add'>+	 *    xa_store()</div><div class='add'>+	 *    mutex_lock(umem_mutex)</div><div class='add'>+	 *     mlx5r_umr_update_xlt()</div><div class='add'>+	 *    mutex_unlock(umem_mutex)</div><div class='add'>+	 *    destroy lkey</div><div class='add'>+	 *</div><div class='add'>+	 * ie any change the xarray must be followed by the locked update_xlt</div><div class='add'>+	 * before destroying.</div><div class='add'>+	 *</div><div class='add'>+	 * The umem_mutex provides the acquire/release semantic needed to make</div><div class='add'>+	 * the xa_store() visible to a racing thread.</div><div class='add'>+	 */</div><div class='add'>+	lockdep_assert_held(&amp;to_ib_umem_odp(imr-&gt;umem)-&gt;umem_mutex);</div><div class='ctx'> </div><div class='del'>-	return to_ib_umem_odp(mr-&gt;umem)-&gt;per_mm;</div><div class='del'>-}</div><div class='add'>+	for (; pksm != end; pksm++, idx++, va += step) {</div><div class='add'>+		struct mlx5_ib_mr *mtt = xa_load(&amp;imr-&gt;implicit_children, idx);</div><div class='ctx'> </div><div class='del'>-static struct ib_umem_odp *odp_next(struct ib_umem_odp *odp)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_mr *mr = odp-&gt;private, *parent = mr-&gt;parent;</div><div class='del'>-	struct ib_ucontext_per_mm *per_mm = odp-&gt;per_mm;</div><div class='del'>-	struct rb_node *rb;</div><div class='del'>-</div><div class='del'>-	down_read(&amp;per_mm-&gt;umem_rwsem);</div><div class='del'>-	while (1) {</div><div class='del'>-		rb = rb_next(&amp;odp-&gt;interval_tree.rb);</div><div class='del'>-		if (!rb)</div><div class='del'>-			goto not_found;</div><div class='del'>-		odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb);</div><div class='del'>-		if (check_parent(odp, parent))</div><div class='del'>-			goto end;</div><div class='add'>+		if (mtt) {</div><div class='add'>+			pksm-&gt;key = cpu_to_be32(mtt-&gt;ibmr.lkey);</div><div class='add'>+			pksm-&gt;va = cpu_to_be64(idx * mlx5_imr_mtt_size);</div><div class='add'>+		} else {</div><div class='add'>+			pksm-&gt;key = key;</div><div class='add'>+			pksm-&gt;va = cpu_to_be64(va);</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='del'>-not_found:</div><div class='del'>-	odp = NULL;</div><div class='del'>-end:</div><div class='del'>-	up_read(&amp;per_mm-&gt;umem_rwsem);</div><div class='del'>-	return odp;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct ib_umem_odp *odp_lookup(u64 start, u64 length,</div><div class='del'>-				      struct mlx5_ib_mr *parent)</div><div class='add'>+static int populate_mtt(__be64 *pas, size_t start, size_t nentries,</div><div class='add'>+			struct mlx5_ib_mr *mr, int flags)</div><div class='ctx'> {</div><div class='del'>-	struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(parent);</div><div class='del'>-	struct ib_umem_odp *odp;</div><div class='del'>-	struct rb_node *rb;</div><div class='add'>+	struct ib_umem_odp *odp = to_ib_umem_odp(mr-&gt;umem);</div><div class='add'>+	bool downgrade = flags &amp; MLX5_IB_UPD_XLT_DOWNGRADE;</div><div class='add'>+	struct pci_p2pdma_map_state p2pdma_state = {};</div><div class='add'>+	struct ib_device *dev = odp-&gt;umem.ibdev;</div><div class='add'>+	size_t i;</div><div class='ctx'> </div><div class='del'>-	down_read(&amp;per_mm-&gt;umem_rwsem);</div><div class='del'>-	odp = rbt_ib_umem_lookup(&amp;per_mm-&gt;umem_tree, start, length);</div><div class='del'>-	if (!odp)</div><div class='del'>-		goto end;</div><div class='add'>+	if (flags &amp; MLX5_IB_UPD_XLT_ZAP)</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-	while (1) {</div><div class='del'>-		if (check_parent(odp, parent))</div><div class='del'>-			goto end;</div><div class='del'>-		rb = rb_next(&amp;odp-&gt;interval_tree.rb);</div><div class='del'>-		if (!rb)</div><div class='del'>-			goto not_found;</div><div class='del'>-		odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb);</div><div class='del'>-		if (ib_umem_start(&amp;odp-&gt;umem) &gt; start + length)</div><div class='del'>-			goto not_found;</div><div class='del'>-	}</div><div class='del'>-not_found:</div><div class='del'>-	odp = NULL;</div><div class='del'>-end:</div><div class='del'>-	up_read(&amp;per_mm-&gt;umem_rwsem);</div><div class='del'>-	return odp;</div><div class='del'>-}</div><div class='add'>+	for (i = 0; i &lt; nentries; i++) {</div><div class='add'>+		unsigned long pfn = odp-&gt;map.pfn_list[start + i];</div><div class='add'>+		dma_addr_t dma_addr;</div><div class='ctx'> </div><div class='del'>-void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset,</div><div class='del'>-			   size_t nentries, struct mlx5_ib_mr *mr, int flags)</div><div class='del'>-{</div><div class='del'>-	struct ib_pd *pd = mr-&gt;ibmr.pd;</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='del'>-	struct ib_umem_odp *odp;</div><div class='del'>-	unsigned long va;</div><div class='del'>-	int i;</div><div class='add'>+		pfn = odp-&gt;map.pfn_list[start + i];</div><div class='add'>+		if (!(pfn &amp; HMM_PFN_VALID))</div><div class='add'>+			/* ODP initialization */</div><div class='add'>+			continue;</div><div class='ctx'> </div><div class='del'>-	if (flags &amp; MLX5_IB_UPD_XLT_ZAP) {</div><div class='del'>-		for (i = 0; i &lt; nentries; i++, pklm++) {</div><div class='del'>-			pklm-&gt;bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE);</div><div class='del'>-			pklm-&gt;key = cpu_to_be32(dev-&gt;null_mkey);</div><div class='del'>-			pklm-&gt;va = 0;</div><div class='del'>-		}</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='add'>+		dma_addr = hmm_dma_map_pfn(dev-&gt;dma_device, &amp;odp-&gt;map,</div><div class='add'>+					   start + i, &amp;p2pdma_state);</div><div class='add'>+		if (ib_dma_mapping_error(dev, dma_addr))</div><div class='add'>+			return -EFAULT;</div><div class='ctx'> </div><div class='del'>-	odp = odp_lookup(offset * MLX5_IMR_MTT_SIZE,</div><div class='del'>-			 nentries * MLX5_IMR_MTT_SIZE, mr);</div><div class='add'>+		dma_addr |= MLX5_IB_MTT_READ;</div><div class='add'>+		if ((pfn &amp; HMM_PFN_WRITE) &amp;&amp; !downgrade)</div><div class='add'>+			dma_addr |= MLX5_IB_MTT_WRITE;</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; nentries; i++, pklm++) {</div><div class='del'>-		pklm-&gt;bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE);</div><div class='del'>-		va = (offset + i) * MLX5_IMR_MTT_SIZE;</div><div class='del'>-		if (odp &amp;&amp; odp-&gt;umem.address == va) {</div><div class='del'>-			struct mlx5_ib_mr *mtt = odp-&gt;private;</div><div class='add'>+		pas[i] = cpu_to_be64(dma_addr);</div><div class='add'>+		odp-&gt;npages++;</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-			pklm-&gt;key = cpu_to_be32(mtt-&gt;ibmr.lkey);</div><div class='del'>-			odp = odp_next(odp);</div><div class='del'>-		} else {</div><div class='del'>-			pklm-&gt;key = cpu_to_be32(dev-&gt;null_mkey);</div><div class='del'>-		}</div><div class='del'>-		mlx5_ib_dbg(dev, "[%d] va %lx key %x\n",</div><div class='del'>-			    i, va, be32_to_cpu(pklm-&gt;key));</div><div class='add'>+int mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries,</div><div class='add'>+			  struct mlx5_ib_mr *mr, int flags)</div><div class='add'>+{</div><div class='add'>+	if (flags &amp; MLX5_IB_UPD_XLT_INDIRECT) {</div><div class='add'>+		populate_ksm(xlt, idx, nentries, mr, flags);</div><div class='add'>+		return 0;</div><div class='add'>+	} else {</div><div class='add'>+		return populate_mtt(xlt, idx, nentries, mr, flags);</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void mr_leaf_free_action(struct work_struct *work)</div><div class='add'>+/*</div><div class='add'>+ * This must be called after the mr has been removed from implicit_children.</div><div class='add'>+ * NOTE: The MR does not necessarily have to be</div><div class='add'>+ * empty here, parallel page faults could have raced with the free process and</div><div class='add'>+ * added pages to it.</div><div class='add'>+ */</div><div class='add'>+static void free_implicit_child_mr_work(struct work_struct *work)</div><div class='ctx'> {</div><div class='del'>-	struct ib_umem_odp *odp = container_of(work, struct ib_umem_odp, work);</div><div class='del'>-	int idx = ib_umem_start(&amp;odp-&gt;umem) &gt;&gt; MLX5_IMR_MTT_SHIFT;</div><div class='del'>-	struct mlx5_ib_mr *mr = odp-&gt;private, *imr = mr-&gt;parent;</div><div class='add'>+	struct mlx5_ib_mr *mr =</div><div class='add'>+		container_of(work, struct mlx5_ib_mr, odp_destroy.work);</div><div class='add'>+	struct mlx5_ib_mr *imr = mr-&gt;parent;</div><div class='add'>+	struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr-&gt;umem);</div><div class='add'>+	struct ib_umem_odp *odp = to_ib_umem_odp(mr-&gt;umem);</div><div class='add'>+</div><div class='add'>+	mlx5r_deref_wait_odp_mkey(&amp;mr-&gt;mmkey);</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;odp_imr-&gt;umem_mutex);</div><div class='add'>+	mlx5r_umr_update_xlt(mr-&gt;parent,</div><div class='add'>+			     ib_umem_start(odp) &gt;&gt; mlx5_imr_mtt_shift, 1, 0,</div><div class='add'>+			     MLX5_IB_UPD_XLT_INDIRECT | MLX5_IB_UPD_XLT_ATOMIC);</div><div class='add'>+	mutex_unlock(&amp;odp_imr-&gt;umem_mutex);</div><div class='add'>+	mlx5_ib_dereg_mr(&amp;mr-&gt;ibmr, NULL);</div><div class='add'>+</div><div class='add'>+	mlx5r_deref_odp_mkey(&amp;imr-&gt;mmkey);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	mr-&gt;parent = NULL;</div><div class='del'>-	synchronize_srcu(&amp;mr-&gt;dev-&gt;mr_srcu);</div><div class='add'>+static void destroy_unused_implicit_child_mr(struct mlx5_ib_mr *mr)</div><div class='add'>+{</div><div class='add'>+	struct ib_umem_odp *odp = to_ib_umem_odp(mr-&gt;umem);</div><div class='add'>+	unsigned long idx = ib_umem_start(odp) &gt;&gt; mlx5_imr_mtt_shift;</div><div class='add'>+	struct mlx5_ib_mr *imr = mr-&gt;parent;</div><div class='ctx'> </div><div class='del'>-	ib_umem_release(&amp;odp-&gt;umem);</div><div class='del'>-	if (imr-&gt;live)</div><div class='del'>-		mlx5_ib_update_xlt(imr, idx, 1, 0,</div><div class='del'>-				   MLX5_IB_UPD_XLT_INDIRECT |</div><div class='del'>-				   MLX5_IB_UPD_XLT_ATOMIC);</div><div class='del'>-	mlx5_mr_cache_free(mr-&gt;dev, mr);</div><div class='add'>+	/*</div><div class='add'>+	 * If userspace is racing freeing the parent implicit ODP MR then we can</div><div class='add'>+	 * loose the race with parent destruction. In this case</div><div class='add'>+	 * mlx5_ib_free_odp_mr() will free everything in the implicit_children</div><div class='add'>+	 * xarray so NOP is fine. This child MR cannot be destroyed here because</div><div class='add'>+	 * we are under its umem_mutex.</div><div class='add'>+	 */</div><div class='add'>+	if (!refcount_inc_not_zero(&amp;imr-&gt;mmkey.usecount))</div><div class='add'>+		return;</div><div class='ctx'> </div><div class='del'>-	if (atomic_dec_and_test(&amp;imr-&gt;num_leaf_free))</div><div class='del'>-		wake_up(&amp;imr-&gt;q_leaf_free);</div><div class='add'>+	xa_lock(&amp;imr-&gt;implicit_children);</div><div class='add'>+	if (__xa_cmpxchg(&amp;imr-&gt;implicit_children, idx, mr, NULL, GFP_KERNEL) !=</div><div class='add'>+	    mr) {</div><div class='add'>+		xa_unlock(&amp;imr-&gt;implicit_children);</div><div class='add'>+		mlx5r_deref_odp_mkey(&amp;imr-&gt;mmkey);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_ODP(mr_to_mdev(mr)-&gt;mdev, mem_page_fault))</div><div class='add'>+		xa_erase(&amp;mr_to_mdev(mr)-&gt;odp_mkeys,</div><div class='add'>+			 mlx5_base_mkey(mr-&gt;mmkey.key));</div><div class='add'>+	xa_unlock(&amp;imr-&gt;implicit_children);</div><div class='add'>+</div><div class='add'>+	/* Freeing a MR is a sleeping operation, so bounce to a work queue */</div><div class='add'>+	INIT_WORK(&amp;mr-&gt;odp_destroy.work, free_implicit_child_mr_work);</div><div class='add'>+	queue_work(system_dfl_wq, &amp;mr-&gt;odp_destroy.work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start,</div><div class='del'>-			      unsigned long end)</div><div class='add'>+static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni,</div><div class='add'>+				     const struct mmu_notifier_range *range,</div><div class='add'>+				     unsigned long cur_seq)</div><div class='ctx'> {</div><div class='add'>+	struct ib_umem_odp *umem_odp =</div><div class='add'>+		container_of(mni, struct ib_umem_odp, notifier);</div><div class='ctx'> 	struct mlx5_ib_mr *mr;</div><div class='del'>-	const u64 umr_block_mask = (MLX5_UMR_MTT_ALIGNMENT /</div><div class='del'>-				    sizeof(struct mlx5_mtt)) - 1;</div><div class='add'>+	const u64 umr_block_mask = MLX5_UMR_MTT_NUM_ENTRIES_ALIGNMENT - 1;</div><div class='ctx'> 	u64 idx = 0, blk_start_idx = 0;</div><div class='del'>-	struct ib_umem *umem;</div><div class='add'>+	u64 invalidations = 0;</div><div class='add'>+	unsigned long start;</div><div class='add'>+	unsigned long end;</div><div class='ctx'> 	int in_block = 0;</div><div class='ctx'> 	u64 addr;</div><div class='ctx'> </div><div class='del'>-	if (!umem_odp) {</div><div class='del'>-		pr_err("invalidation called on NULL umem or non-ODP umem\n");</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-	umem = &amp;umem_odp-&gt;umem;</div><div class='add'>+	if (!mmu_notifier_range_blockable(range))</div><div class='add'>+		return false;</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;umem_odp-&gt;umem_mutex);</div><div class='add'>+	mmu_interval_set_seq(mni, cur_seq);</div><div class='add'>+	/*</div><div class='add'>+	 * If npages is zero then umem_odp-&gt;private may not be setup yet. This</div><div class='add'>+	 * does not complete until after the first page is mapped for DMA.</div><div class='add'>+	 */</div><div class='add'>+	if (!umem_odp-&gt;npages)</div><div class='add'>+		goto out;</div><div class='ctx'> 	mr = umem_odp-&gt;private;</div><div class='add'>+	if (!mr)</div><div class='add'>+		goto out;</div><div class='ctx'> </div><div class='del'>-	if (!mr || !mr-&gt;ibmr.pd)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	start = max_t(u64, ib_umem_start(umem), start);</div><div class='del'>-	end = min_t(u64, ib_umem_end(umem), end);</div><div class='add'>+	start = max_t(u64, ib_umem_start(umem_odp), range-&gt;start);</div><div class='add'>+	end = min_t(u64, ib_umem_end(umem_odp), range-&gt;end);</div><div class='ctx'> </div><div class='ctx'> 	/*</div><div class='ctx'> 	 * Iteration one - zap the HW's MTTs. The notifiers_count ensures that</div><div class='hunk'>@@ -248,17 +301,15 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start,</div><div class='ctx'> 	 * overwrite the same MTTs.  Concurent invalidations might race us,</div><div class='ctx'> 	 * but they will write 0s as well, so no difference in the end result.</div><div class='ctx'> 	 */</div><div class='del'>-</div><div class='del'>-	for (addr = start; addr &lt; end; addr += BIT(umem-&gt;page_shift)) {</div><div class='del'>-		idx = (addr - ib_umem_start(umem)) &gt;&gt; umem-&gt;page_shift;</div><div class='add'>+	for (addr = start; addr &lt; end; addr += BIT(umem_odp-&gt;page_shift)) {</div><div class='add'>+		idx = (addr - ib_umem_start(umem_odp)) &gt;&gt; umem_odp-&gt;page_shift;</div><div class='ctx'> 		/*</div><div class='ctx'> 		 * Strive to write the MTTs in chunks, but avoid overwriting</div><div class='ctx'> 		 * non-existing MTTs. The huristic here can be improved to</div><div class='ctx'> 		 * estimate the cost of another UMR vs. the cost of bigger</div><div class='ctx'> 		 * UMR.</div><div class='ctx'> 		 */</div><div class='del'>-		if (umem_odp-&gt;dma_list[idx] &amp;</div><div class='del'>-		    (ODP_READ_ALLOWED_BIT | ODP_WRITE_ALLOWED_BIT)) {</div><div class='add'>+		if (umem_odp-&gt;map.pfn_list[idx] &amp; HMM_PFN_VALID) {</div><div class='ctx'> 			if (!in_block) {</div><div class='ctx'> 				blk_start_idx = idx;</div><div class='ctx'> 				in_block = 1;</div><div class='hunk'>@@ -267,19 +318,27 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start,</div><div class='ctx'> 			u64 umr_offset = idx &amp; umr_block_mask;</div><div class='ctx'> </div><div class='ctx'> 			if (in_block &amp;&amp; umr_offset == 0) {</div><div class='del'>-				mlx5_ib_update_xlt(mr, blk_start_idx,</div><div class='del'>-						   idx - blk_start_idx, 0,</div><div class='del'>-						   MLX5_IB_UPD_XLT_ZAP |</div><div class='del'>-						   MLX5_IB_UPD_XLT_ATOMIC);</div><div class='add'>+				mlx5r_umr_update_xlt(mr, blk_start_idx,</div><div class='add'>+						     idx - blk_start_idx, 0,</div><div class='add'>+						     MLX5_IB_UPD_XLT_ZAP |</div><div class='add'>+						     MLX5_IB_UPD_XLT_ATOMIC);</div><div class='ctx'> 				in_block = 0;</div><div class='add'>+				/* Count page invalidations */</div><div class='add'>+				invalidations += idx - blk_start_idx + 1;</div><div class='ctx'> 			}</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='del'>-	if (in_block)</div><div class='del'>-		mlx5_ib_update_xlt(mr, blk_start_idx,</div><div class='del'>-				   idx - blk_start_idx + 1, 0,</div><div class='del'>-				   MLX5_IB_UPD_XLT_ZAP |</div><div class='del'>-				   MLX5_IB_UPD_XLT_ATOMIC);</div><div class='add'>+	if (in_block) {</div><div class='add'>+		mlx5r_umr_update_xlt(mr, blk_start_idx,</div><div class='add'>+				     idx - blk_start_idx + 1, 0,</div><div class='add'>+				     MLX5_IB_UPD_XLT_ZAP |</div><div class='add'>+				     MLX5_IB_UPD_XLT_ATOMIC);</div><div class='add'>+		/* Count page invalidations */</div><div class='add'>+		invalidations += idx - blk_start_idx + 1;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mlx5_update_odp_stats_with_handled(mr, invalidations, invalidations);</div><div class='add'>+</div><div class='ctx'> 	/*</div><div class='ctx'> 	 * We are now sure that the device will not access the</div><div class='ctx'> 	 * memory. We can safely unmap it, and mark it as dirty if</div><div class='hunk'>@@ -288,21 +347,24 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start,</div><div class='ctx'> </div><div class='ctx'> 	ib_umem_odp_unmap_dma_pages(umem_odp, start, end);</div><div class='ctx'> </div><div class='del'>-	if (unlikely(!umem_odp-&gt;npages &amp;&amp; mr-&gt;parent &amp;&amp;</div><div class='del'>-		     !umem_odp-&gt;dying)) {</div><div class='del'>-		WRITE_ONCE(umem_odp-&gt;dying, 1);</div><div class='del'>-		atomic_inc(&amp;mr-&gt;parent-&gt;num_leaf_free);</div><div class='del'>-		schedule_work(&amp;umem_odp-&gt;work);</div><div class='del'>-	}</div><div class='add'>+	if (unlikely(!umem_odp-&gt;npages &amp;&amp; mr-&gt;parent))</div><div class='add'>+		destroy_unused_implicit_child_mr(mr);</div><div class='add'>+out:</div><div class='add'>+	mutex_unlock(&amp;umem_odp-&gt;umem_mutex);</div><div class='add'>+	return true;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev)</div><div class='add'>+const struct mmu_interval_notifier_ops mlx5_mn_ops = {</div><div class='add'>+	.invalidate = mlx5_ib_invalidate_range,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static void internal_fill_odp_caps(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_odp_caps *caps = &amp;dev-&gt;odp_caps;</div><div class='ctx'> </div><div class='ctx'> 	memset(caps, 0, sizeof(*caps));</div><div class='ctx'> </div><div class='del'>-	if (!MLX5_CAP_GEN(dev-&gt;mdev, pg))</div><div class='add'>+	if (!MLX5_CAP_GEN(dev-&gt;mdev, pg) || !mlx5r_umr_can_load_pas(dev, 0))</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='ctx'> 	caps-&gt;general_caps = IB_ODP_SUPPORT;</div><div class='hunk'>@@ -312,54 +374,56 @@ void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev)</div><div class='ctx'> 	else</div><div class='ctx'> 		dev-&gt;odp_max_size = BIT_ULL(MLX5_MAX_UMR_SHIFT + PAGE_SHIFT);</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_ODP(dev-&gt;mdev, ud_odp_caps.send))</div><div class='add'>+	if (MLX5_CAP_ODP_SCHEME(dev-&gt;mdev, ud_odp_caps.send))</div><div class='ctx'> 		caps-&gt;per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SEND;</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_ODP(dev-&gt;mdev, ud_odp_caps.srq_receive))</div><div class='add'>+	if (MLX5_CAP_ODP_SCHEME(dev-&gt;mdev, ud_odp_caps.srq_receive))</div><div class='ctx'> 		caps-&gt;per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_ODP(dev-&gt;mdev, rc_odp_caps.send))</div><div class='add'>+	if (MLX5_CAP_ODP_SCHEME(dev-&gt;mdev, rc_odp_caps.send))</div><div class='ctx'> 		caps-&gt;per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SEND;</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_ODP(dev-&gt;mdev, rc_odp_caps.receive))</div><div class='add'>+	if (MLX5_CAP_ODP_SCHEME(dev-&gt;mdev, rc_odp_caps.receive))</div><div class='ctx'> 		caps-&gt;per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_RECV;</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_ODP(dev-&gt;mdev, rc_odp_caps.write))</div><div class='add'>+	if (MLX5_CAP_ODP_SCHEME(dev-&gt;mdev, rc_odp_caps.write))</div><div class='ctx'> 		caps-&gt;per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_WRITE;</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_ODP(dev-&gt;mdev, rc_odp_caps.read))</div><div class='add'>+	if (MLX5_CAP_ODP_SCHEME(dev-&gt;mdev, rc_odp_caps.read))</div><div class='ctx'> 		caps-&gt;per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ;</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_ODP(dev-&gt;mdev, rc_odp_caps.atomic))</div><div class='add'>+	if (MLX5_CAP_ODP_SCHEME(dev-&gt;mdev, rc_odp_caps.atomic))</div><div class='ctx'> 		caps-&gt;per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC;</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_ODP(dev-&gt;mdev, rc_odp_caps.srq_receive))</div><div class='add'>+	if (MLX5_CAP_ODP_SCHEME(dev-&gt;mdev, rc_odp_caps.srq_receive))</div><div class='ctx'> 		caps-&gt;per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_ODP(dev-&gt;mdev, xrc_odp_caps.send))</div><div class='add'>+	if (MLX5_CAP_ODP_SCHEME(dev-&gt;mdev, xrc_odp_caps.send))</div><div class='ctx'> 		caps-&gt;per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SEND;</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_ODP(dev-&gt;mdev, xrc_odp_caps.receive))</div><div class='add'>+	if (MLX5_CAP_ODP_SCHEME(dev-&gt;mdev, xrc_odp_caps.receive))</div><div class='ctx'> 		caps-&gt;per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_RECV;</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_ODP(dev-&gt;mdev, xrc_odp_caps.write))</div><div class='add'>+	if (MLX5_CAP_ODP_SCHEME(dev-&gt;mdev, xrc_odp_caps.write))</div><div class='ctx'> 		caps-&gt;per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_WRITE;</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_ODP(dev-&gt;mdev, xrc_odp_caps.read))</div><div class='add'>+	if (MLX5_CAP_ODP_SCHEME(dev-&gt;mdev, xrc_odp_caps.read))</div><div class='ctx'> 		caps-&gt;per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_READ;</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_ODP(dev-&gt;mdev, xrc_odp_caps.atomic))</div><div class='add'>+	if (MLX5_CAP_ODP_SCHEME(dev-&gt;mdev, xrc_odp_caps.atomic))</div><div class='ctx'> 		caps-&gt;per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_ATOMIC;</div><div class='ctx'> </div><div class='del'>-	if (MLX5_CAP_ODP(dev-&gt;mdev, xrc_odp_caps.srq_receive))</div><div class='add'>+	if (MLX5_CAP_ODP_SCHEME(dev-&gt;mdev, xrc_odp_caps.srq_receive))</div><div class='ctx'> 		caps-&gt;per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;</div><div class='ctx'> </div><div class='ctx'> 	if (MLX5_CAP_GEN(dev-&gt;mdev, fixed_buffer_size) &amp;&amp;</div><div class='ctx'> 	    MLX5_CAP_GEN(dev-&gt;mdev, null_mkey) &amp;&amp;</div><div class='del'>-	    MLX5_CAP_GEN(dev-&gt;mdev, umr_extended_translation_offset))</div><div class='add'>+	    MLX5_CAP_GEN(dev-&gt;mdev, umr_extended_translation_offset) &amp;&amp;</div><div class='add'>+	    !MLX5_CAP_GEN(dev-&gt;mdev, umr_indirect_mkey_disabled) &amp;&amp;</div><div class='add'>+	    mlx5_imr_ksm_entries != 0 &amp;&amp;</div><div class='add'>+	    !(mlx5_imr_ksm_page_shift &gt;</div><div class='add'>+	      get_max_log_entity_size_cap(dev, MLX5_MKC_ACCESS_MODE_KSM)))</div><div class='ctx'> 		caps-&gt;general_caps |= IB_ODP_SUPPORT_IMPLICIT;</div><div class='del'>-</div><div class='del'>-	return;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev,</div><div class='hunk'>@@ -368,339 +432,520 @@ static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev,</div><div class='ctx'> {</div><div class='ctx'> 	int wq_num = pfault-&gt;event_subtype == MLX5_PFAULT_SUBTYPE_WQE ?</div><div class='ctx'> 		     pfault-&gt;wqe.wq_num : pfault-&gt;token;</div><div class='del'>-	u32 out[MLX5_ST_SZ_DW(page_fault_resume_out)] = { };</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(page_fault_resume_in)]   = { };</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(page_fault_resume_in)] = {};</div><div class='add'>+	void *info;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(page_fault_resume_in, in, opcode, MLX5_CMD_OP_PAGE_FAULT_RESUME);</div><div class='del'>-	MLX5_SET(page_fault_resume_in, in, page_fault_type, pfault-&gt;type);</div><div class='del'>-	MLX5_SET(page_fault_resume_in, in, token, pfault-&gt;token);</div><div class='del'>-	MLX5_SET(page_fault_resume_in, in, wq_number, wq_num);</div><div class='del'>-	MLX5_SET(page_fault_resume_in, in, error, !!error);</div><div class='ctx'> </div><div class='del'>-	err = mlx5_cmd_exec(dev-&gt;mdev, in, sizeof(in), out, sizeof(out));</div><div class='add'>+	if (pfault-&gt;event_subtype == MLX5_PFAULT_SUBTYPE_MEMORY) {</div><div class='add'>+		info = MLX5_ADDR_OF(page_fault_resume_in, in,</div><div class='add'>+				    page_fault_info.mem_page_fault_info);</div><div class='add'>+		MLX5_SET(mem_page_fault_info, info, fault_token_31_0,</div><div class='add'>+			 pfault-&gt;token &amp; 0xffffffff);</div><div class='add'>+		MLX5_SET(mem_page_fault_info, info, fault_token_47_32,</div><div class='add'>+			 (pfault-&gt;token &gt;&gt; 32) &amp; 0xffff);</div><div class='add'>+		MLX5_SET(mem_page_fault_info, info, error, !!error);</div><div class='add'>+	} else {</div><div class='add'>+		info = MLX5_ADDR_OF(page_fault_resume_in, in,</div><div class='add'>+				    page_fault_info.trans_page_fault_info);</div><div class='add'>+		MLX5_SET(trans_page_fault_info, info, page_fault_type,</div><div class='add'>+			 pfault-&gt;type);</div><div class='add'>+		MLX5_SET(trans_page_fault_info, info, fault_token,</div><div class='add'>+			 pfault-&gt;token);</div><div class='add'>+		MLX5_SET(trans_page_fault_info, info, wq_number, wq_num);</div><div class='add'>+		MLX5_SET(trans_page_fault_info, info, error, !!error);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	err = mlx5_cmd_exec_in(dev-&gt;mdev, page_fault_resume, in);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		mlx5_ib_err(dev, "Failed to resolve the page fault on WQ 0x%x err %d\n",</div><div class='ctx'> 			    wq_num, err);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct mlx5_ib_mr *implicit_mr_alloc(struct ib_pd *pd,</div><div class='del'>-					    struct ib_umem *umem,</div><div class='del'>-					    bool ksm, int access_flags)</div><div class='add'>+static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr,</div><div class='add'>+						unsigned long idx)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='add'>+	struct mlx5_ib_dev *dev = mr_to_mdev(imr);</div><div class='add'>+	struct ib_umem_odp *odp;</div><div class='ctx'> 	struct mlx5_ib_mr *mr;</div><div class='add'>+	struct mlx5_ib_mr *ret;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	mr = mlx5_mr_cache_alloc(dev, ksm ? MLX5_IMR_KSM_CACHE_ENTRY :</div><div class='del'>-					    MLX5_IMR_MTT_CACHE_ENTRY);</div><div class='del'>-</div><div class='del'>-	if (IS_ERR(mr))</div><div class='add'>+	odp = ib_umem_odp_alloc_child(to_ib_umem_odp(imr-&gt;umem),</div><div class='add'>+				      idx * mlx5_imr_mtt_size,</div><div class='add'>+				      mlx5_imr_mtt_size, &amp;mlx5_mn_ops);</div><div class='add'>+	if (IS_ERR(odp))</div><div class='add'>+		return ERR_CAST(odp);</div><div class='add'>+</div><div class='add'>+	mr = mlx5_mr_cache_alloc(dev, imr-&gt;access_flags,</div><div class='add'>+				 MLX5_MKC_ACCESS_MODE_MTT,</div><div class='add'>+				 mlx5_imr_mtt_entries);</div><div class='add'>+	if (IS_ERR(mr)) {</div><div class='add'>+		ib_umem_odp_release(odp);</div><div class='ctx'> 		return mr;</div><div class='del'>-</div><div class='del'>-	mr-&gt;ibmr.pd = pd;</div><div class='del'>-</div><div class='del'>-	mr-&gt;dev = dev;</div><div class='del'>-	mr-&gt;access_flags = access_flags;</div><div class='del'>-	mr-&gt;mmkey.iova = 0;</div><div class='del'>-	mr-&gt;umem = umem;</div><div class='del'>-</div><div class='del'>-	if (ksm) {</div><div class='del'>-		err = mlx5_ib_update_xlt(mr, 0,</div><div class='del'>-					 mlx5_imr_ksm_entries,</div><div class='del'>-					 MLX5_KSM_PAGE_SHIFT,</div><div class='del'>-					 MLX5_IB_UPD_XLT_INDIRECT |</div><div class='del'>-					 MLX5_IB_UPD_XLT_ZAP |</div><div class='del'>-					 MLX5_IB_UPD_XLT_ENABLE);</div><div class='del'>-</div><div class='del'>-	} else {</div><div class='del'>-		err = mlx5_ib_update_xlt(mr, 0,</div><div class='del'>-					 MLX5_IMR_MTT_ENTRIES,</div><div class='del'>-					 PAGE_SHIFT,</div><div class='del'>-					 MLX5_IB_UPD_XLT_ZAP |</div><div class='del'>-					 MLX5_IB_UPD_XLT_ENABLE |</div><div class='del'>-					 MLX5_IB_UPD_XLT_ATOMIC);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (err)</div><div class='del'>-		goto fail;</div><div class='del'>-</div><div class='add'>+	mr-&gt;access_flags = imr-&gt;access_flags;</div><div class='add'>+	mr-&gt;ibmr.pd = imr-&gt;ibmr.pd;</div><div class='add'>+	mr-&gt;ibmr.device = &amp;mr_to_mdev(imr)-&gt;ib_dev;</div><div class='add'>+	mr-&gt;umem = &amp;odp-&gt;umem;</div><div class='ctx'> 	mr-&gt;ibmr.lkey = mr-&gt;mmkey.key;</div><div class='ctx'> 	mr-&gt;ibmr.rkey = mr-&gt;mmkey.key;</div><div class='add'>+	mr-&gt;ibmr.iova = idx * mlx5_imr_mtt_size;</div><div class='add'>+	mr-&gt;parent = imr;</div><div class='add'>+	odp-&gt;private = mr;</div><div class='ctx'> </div><div class='del'>-	mr-&gt;live = 1;</div><div class='add'>+	/*</div><div class='add'>+	 * First refcount is owned by the xarray and second refconut</div><div class='add'>+	 * is returned to the caller.</div><div class='add'>+	 */</div><div class='add'>+	refcount_set(&amp;mr-&gt;mmkey.usecount, 2);</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_dbg(dev, "key %x dev %p mr %p\n",</div><div class='del'>-		    mr-&gt;mmkey.key, dev-&gt;mdev, mr);</div><div class='add'>+	err = mlx5r_umr_update_xlt(mr, 0,</div><div class='add'>+				   mlx5_imr_mtt_entries,</div><div class='add'>+				   PAGE_SHIFT,</div><div class='add'>+				   MLX5_IB_UPD_XLT_ZAP |</div><div class='add'>+				   MLX5_IB_UPD_XLT_ENABLE);</div><div class='add'>+	if (err) {</div><div class='add'>+		ret = ERR_PTR(err);</div><div class='add'>+		goto out_mr;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	return mr;</div><div class='add'>+	xa_lock(&amp;imr-&gt;implicit_children);</div><div class='add'>+	ret = __xa_cmpxchg(&amp;imr-&gt;implicit_children, idx, NULL, mr,</div><div class='add'>+			   GFP_KERNEL);</div><div class='add'>+	if (unlikely(ret)) {</div><div class='add'>+		if (xa_is_err(ret)) {</div><div class='add'>+			ret = ERR_PTR(xa_err(ret));</div><div class='add'>+			goto out_lock;</div><div class='add'>+		}</div><div class='add'>+		/*</div><div class='add'>+		 * Another thread beat us to creating the child mr, use</div><div class='add'>+		 * theirs.</div><div class='add'>+		 */</div><div class='add'>+		refcount_inc(&amp;ret-&gt;mmkey.usecount);</div><div class='add'>+		goto out_lock;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-fail:</div><div class='del'>-	mlx5_ib_err(dev, "Failed to register MKEY %d\n", err);</div><div class='del'>-	mlx5_mr_cache_free(dev, mr);</div><div class='add'>+	if (MLX5_CAP_ODP(dev-&gt;mdev, mem_page_fault)) {</div><div class='add'>+		ret = xa_store(&amp;dev-&gt;odp_mkeys, mlx5_base_mkey(mr-&gt;mmkey.key),</div><div class='add'>+			       &amp;mr-&gt;mmkey, GFP_KERNEL);</div><div class='add'>+		if (xa_is_err(ret)) {</div><div class='add'>+			ret = ERR_PTR(xa_err(ret));</div><div class='add'>+			__xa_erase(&amp;imr-&gt;implicit_children, idx);</div><div class='add'>+			goto out_lock;</div><div class='add'>+		}</div><div class='add'>+		mr-&gt;mmkey.type = MLX5_MKEY_IMPLICIT_CHILD;</div><div class='add'>+	}</div><div class='add'>+	xa_unlock(&amp;imr-&gt;implicit_children);</div><div class='add'>+	mlx5_ib_dbg(mr_to_mdev(imr), "key %x mr %p\n", mr-&gt;mmkey.key, mr);</div><div class='add'>+	return mr;</div><div class='ctx'> </div><div class='del'>-	return ERR_PTR(err);</div><div class='add'>+out_lock:</div><div class='add'>+	xa_unlock(&amp;imr-&gt;implicit_children);</div><div class='add'>+out_mr:</div><div class='add'>+	mlx5_ib_dereg_mr(&amp;mr-&gt;ibmr, NULL);</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct ib_umem_odp *implicit_mr_get_data(struct mlx5_ib_mr *mr,</div><div class='del'>-						u64 io_virt, size_t bcnt)</div><div class='add'>+/*</div><div class='add'>+ * When using memory scheme ODP, implicit MRs can't use the reserved null mkey</div><div class='add'>+ * and each implicit MR needs to assign a private null mkey to get the page</div><div class='add'>+ * faults on.</div><div class='add'>+ * The null mkey is created with the properties to enable getting the page</div><div class='add'>+ * fault for every time it is accessed and having all relevant access flags.</div><div class='add'>+ */</div><div class='add'>+static int alloc_implicit_mr_null_mkey(struct mlx5_ib_dev *dev,</div><div class='add'>+				       struct mlx5_ib_mr *imr,</div><div class='add'>+				       struct mlx5_ib_pd *pd)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(mr-&gt;ibmr.pd-&gt;device);</div><div class='del'>-	struct ib_umem_odp *odp, *result = NULL;</div><div class='del'>-	struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr-&gt;umem);</div><div class='del'>-	u64 addr = io_virt &amp; MLX5_IMR_MTT_MASK;</div><div class='del'>-	int nentries = 0, start_idx = 0, ret;</div><div class='del'>-	struct mlx5_ib_mr *mtt;</div><div class='add'>+	size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + 64;</div><div class='add'>+	void *mkc;</div><div class='add'>+	u32 *in;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;odp_mr-&gt;umem_mutex);</div><div class='del'>-	odp = odp_lookup(addr, 1, mr);</div><div class='add'>+	in = kzalloc(inlen, GFP_KERNEL);</div><div class='add'>+	if (!in)</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_dbg(dev, "io_virt:%llx bcnt:%zx addr:%llx odp:%p\n",</div><div class='del'>-		    io_virt, bcnt, addr, odp);</div><div class='add'>+	MLX5_SET(create_mkey_in, in, translations_octword_actual_size, 4);</div><div class='add'>+	MLX5_SET(create_mkey_in, in, pg_access, 1);</div><div class='add'>+</div><div class='add'>+	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);</div><div class='add'>+	MLX5_SET(mkc, mkc, a, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, rw, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, rr, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, lw, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, lr, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, free, 0);</div><div class='add'>+	MLX5_SET(mkc, mkc, umr_en, 0);</div><div class='add'>+	MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);</div><div class='add'>+</div><div class='add'>+	MLX5_SET(mkc, mkc, translations_octword_size, 4);</div><div class='add'>+	MLX5_SET(mkc, mkc, log_page_size, 61);</div><div class='add'>+	MLX5_SET(mkc, mkc, length64, 1);</div><div class='add'>+	MLX5_SET(mkc, mkc, pd, pd-&gt;pdn);</div><div class='add'>+	MLX5_SET64(mkc, mkc, start_addr, 0);</div><div class='add'>+	MLX5_SET(mkc, mkc, qpn, 0xffffff);</div><div class='add'>+</div><div class='add'>+	err = mlx5_core_create_mkey(dev-&gt;mdev, &amp;imr-&gt;null_mmkey.key, in, inlen);</div><div class='add'>+	if (err)</div><div class='add'>+		goto free_in;</div><div class='ctx'> </div><div class='del'>-next_mr:</div><div class='del'>-	if (likely(odp)) {</div><div class='del'>-		if (nentries)</div><div class='del'>-			nentries++;</div><div class='del'>-	} else {</div><div class='del'>-		odp = ib_alloc_odp_umem(odp_mr, addr,</div><div class='del'>-					MLX5_IMR_MTT_SIZE);</div><div class='del'>-		if (IS_ERR(odp)) {</div><div class='del'>-			mutex_unlock(&amp;odp_mr-&gt;umem_mutex);</div><div class='del'>-			return ERR_CAST(odp);</div><div class='del'>-		}</div><div class='add'>+	imr-&gt;null_mmkey.type = MLX5_MKEY_NULL;</div><div class='ctx'> </div><div class='del'>-		mtt = implicit_mr_alloc(mr-&gt;ibmr.pd, &amp;odp-&gt;umem, 0,</div><div class='del'>-					mr-&gt;access_flags);</div><div class='del'>-		if (IS_ERR(mtt)) {</div><div class='del'>-			mutex_unlock(&amp;odp_mr-&gt;umem_mutex);</div><div class='del'>-			ib_umem_release(&amp;odp-&gt;umem);</div><div class='del'>-			return ERR_CAST(mtt);</div><div class='del'>-		}</div><div class='add'>+free_in:</div><div class='add'>+	kfree(in);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		odp-&gt;private = mtt;</div><div class='del'>-		mtt-&gt;umem = &amp;odp-&gt;umem;</div><div class='del'>-		mtt-&gt;mmkey.iova = addr;</div><div class='del'>-		mtt-&gt;parent = mr;</div><div class='del'>-		INIT_WORK(&amp;odp-&gt;work, mr_leaf_free_action);</div><div class='add'>+struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd,</div><div class='add'>+					     int access_flags)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;ibpd.device);</div><div class='add'>+	struct ib_umem_odp *umem_odp;</div><div class='add'>+	struct mlx5_ib_mr *imr;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-		if (!nentries)</div><div class='del'>-			start_idx = addr &gt;&gt; MLX5_IMR_MTT_SHIFT;</div><div class='del'>-		nentries++;</div><div class='del'>-	}</div><div class='add'>+	if (!mlx5r_umr_can_load_pas(dev, mlx5_imr_mtt_entries * PAGE_SIZE))</div><div class='add'>+		return ERR_PTR(-EOPNOTSUPP);</div><div class='ctx'> </div><div class='del'>-	/* Return first odp if region not covered by single one */</div><div class='del'>-	if (likely(!result))</div><div class='del'>-		result = odp;</div><div class='add'>+	umem_odp = ib_umem_odp_alloc_implicit(&amp;dev-&gt;ib_dev, access_flags);</div><div class='add'>+	if (IS_ERR(umem_odp))</div><div class='add'>+		return ERR_CAST(umem_odp);</div><div class='ctx'> </div><div class='del'>-	addr += MLX5_IMR_MTT_SIZE;</div><div class='del'>-	if (unlikely(addr &lt; io_virt + bcnt)) {</div><div class='del'>-		odp = odp_next(odp);</div><div class='del'>-		if (odp &amp;&amp; odp-&gt;umem.address != addr)</div><div class='del'>-			odp = NULL;</div><div class='del'>-		goto next_mr;</div><div class='add'>+	imr = mlx5_mr_cache_alloc(dev, access_flags, MLX5_MKC_ACCESS_MODE_KSM,</div><div class='add'>+				  mlx5_imr_ksm_entries);</div><div class='add'>+	if (IS_ERR(imr)) {</div><div class='add'>+		ib_umem_odp_release(umem_odp);</div><div class='add'>+		return imr;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (unlikely(nentries)) {</div><div class='del'>-		ret = mlx5_ib_update_xlt(mr, start_idx, nentries, 0,</div><div class='del'>-					 MLX5_IB_UPD_XLT_INDIRECT |</div><div class='del'>-					 MLX5_IB_UPD_XLT_ATOMIC);</div><div class='del'>-		if (ret) {</div><div class='del'>-			mlx5_ib_err(dev, "Failed to update PAS\n");</div><div class='del'>-			result = ERR_PTR(ret);</div><div class='del'>-		}</div><div class='add'>+	imr-&gt;access_flags = access_flags;</div><div class='add'>+	imr-&gt;ibmr.pd = &amp;pd-&gt;ibpd;</div><div class='add'>+	imr-&gt;ibmr.iova = 0;</div><div class='add'>+	imr-&gt;umem = &amp;umem_odp-&gt;umem;</div><div class='add'>+	imr-&gt;ibmr.lkey = imr-&gt;mmkey.key;</div><div class='add'>+	imr-&gt;ibmr.rkey = imr-&gt;mmkey.key;</div><div class='add'>+	imr-&gt;ibmr.device = &amp;dev-&gt;ib_dev;</div><div class='add'>+	imr-&gt;is_odp_implicit = true;</div><div class='add'>+	xa_init(&amp;imr-&gt;implicit_children);</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_ODP(dev-&gt;mdev, mem_page_fault)) {</div><div class='add'>+		err = alloc_implicit_mr_null_mkey(dev, imr, pd);</div><div class='add'>+		if (err)</div><div class='add'>+			goto out_mr;</div><div class='add'>+</div><div class='add'>+		err = mlx5r_store_odp_mkey(dev, &amp;imr-&gt;null_mmkey);</div><div class='add'>+		if (err)</div><div class='add'>+			goto out_mr;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	mutex_unlock(&amp;odp_mr-&gt;umem_mutex);</div><div class='del'>-	return result;</div><div class='add'>+	err = mlx5r_umr_update_xlt(imr, 0,</div><div class='add'>+				   mlx5_imr_ksm_entries,</div><div class='add'>+				   mlx5_imr_ksm_page_shift,</div><div class='add'>+				   MLX5_IB_UPD_XLT_INDIRECT |</div><div class='add'>+				   MLX5_IB_UPD_XLT_ZAP |</div><div class='add'>+				   MLX5_IB_UPD_XLT_ENABLE);</div><div class='add'>+	if (err)</div><div class='add'>+		goto out_mr;</div><div class='add'>+</div><div class='add'>+	err = mlx5r_store_odp_mkey(dev, &amp;imr-&gt;mmkey);</div><div class='add'>+	if (err)</div><div class='add'>+		goto out_mr;</div><div class='add'>+</div><div class='add'>+	mlx5_ib_dbg(dev, "key %x mr %p\n", imr-&gt;mmkey.key, imr);</div><div class='add'>+	return imr;</div><div class='add'>+out_mr:</div><div class='add'>+	mlx5_ib_err(dev, "Failed to register MKEY %d\n", err);</div><div class='add'>+	mlx5_ib_dereg_mr(&amp;imr-&gt;ibmr, NULL);</div><div class='add'>+	return ERR_PTR(err);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd,</div><div class='del'>-					     struct ib_udata *udata,</div><div class='del'>-					     int access_flags)</div><div class='add'>+void mlx5_ib_free_odp_mr(struct mlx5_ib_mr *mr)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_mr *imr;</div><div class='del'>-	struct ib_umem *umem;</div><div class='del'>-</div><div class='del'>-	umem = ib_umem_get(udata, 0, 0, access_flags, 0);</div><div class='del'>-	if (IS_ERR(umem))</div><div class='del'>-		return ERR_CAST(umem);</div><div class='add'>+	struct mlx5_ib_mr *mtt;</div><div class='add'>+	unsigned long idx;</div><div class='ctx'> </div><div class='del'>-	imr = implicit_mr_alloc(&amp;pd-&gt;ibpd, umem, 1, access_flags);</div><div class='del'>-	if (IS_ERR(imr)) {</div><div class='del'>-		ib_umem_release(umem);</div><div class='del'>-		return ERR_CAST(imr);</div><div class='add'>+	/*</div><div class='add'>+	 * If this is an implicit MR it is already invalidated so we can just</div><div class='add'>+	 * delete the children mkeys.</div><div class='add'>+	 */</div><div class='add'>+	xa_for_each(&amp;mr-&gt;implicit_children, idx, mtt) {</div><div class='add'>+		xa_erase(&amp;mr-&gt;implicit_children, idx);</div><div class='add'>+		mlx5_ib_dereg_mr(&amp;mtt-&gt;ibmr, NULL);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	imr-&gt;umem = umem;</div><div class='del'>-	init_waitqueue_head(&amp;imr-&gt;q_leaf_free);</div><div class='del'>-	atomic_set(&amp;imr-&gt;num_leaf_free, 0);</div><div class='del'>-	atomic_set(&amp;imr-&gt;num_pending_prefetch, 0);</div><div class='add'>+	if (mr-&gt;null_mmkey.key) {</div><div class='add'>+		xa_erase(&amp;mr_to_mdev(mr)-&gt;odp_mkeys,</div><div class='add'>+			 mlx5_base_mkey(mr-&gt;null_mmkey.key));</div><div class='ctx'> </div><div class='del'>-	return imr;</div><div class='add'>+		mlx5_core_destroy_mkey(mr_to_mdev(mr)-&gt;mdev,</div><div class='add'>+				       mr-&gt;null_mmkey.key);</div><div class='add'>+	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mr_leaf_free(struct ib_umem_odp *umem_odp, u64 start, u64 end,</div><div class='del'>-			void *cookie)</div><div class='add'>+#define MLX5_PF_FLAGS_DOWNGRADE BIT(1)</div><div class='add'>+#define MLX5_PF_FLAGS_SNAPSHOT BIT(2)</div><div class='add'>+#define MLX5_PF_FLAGS_ENABLE BIT(3)</div><div class='add'>+static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,</div><div class='add'>+			     u64 user_va, size_t bcnt, u32 *bytes_mapped,</div><div class='add'>+			     u32 flags)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_mr *mr = umem_odp-&gt;private, *imr = cookie;</div><div class='del'>-	struct ib_umem *umem = &amp;umem_odp-&gt;umem;</div><div class='add'>+	int page_shift, ret, np;</div><div class='add'>+	bool downgrade = flags &amp; MLX5_PF_FLAGS_DOWNGRADE;</div><div class='add'>+	u64 access_mask = 0;</div><div class='add'>+	u64 start_idx;</div><div class='add'>+	bool fault = !(flags &amp; MLX5_PF_FLAGS_SNAPSHOT);</div><div class='add'>+	u32 xlt_flags = MLX5_IB_UPD_XLT_ATOMIC;</div><div class='ctx'> </div><div class='del'>-	if (mr-&gt;parent != imr)</div><div class='del'>-		return 0;</div><div class='add'>+	if (flags &amp; MLX5_PF_FLAGS_ENABLE)</div><div class='add'>+		xlt_flags |= MLX5_IB_UPD_XLT_ENABLE;</div><div class='ctx'> </div><div class='del'>-	ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem),</div><div class='del'>-				    ib_umem_end(umem));</div><div class='add'>+	if (flags &amp; MLX5_PF_FLAGS_DOWNGRADE)</div><div class='add'>+		xlt_flags |= MLX5_IB_UPD_XLT_DOWNGRADE;</div><div class='ctx'> </div><div class='del'>-	if (umem_odp-&gt;dying)</div><div class='del'>-		return 0;</div><div class='add'>+	page_shift = odp-&gt;page_shift;</div><div class='add'>+	start_idx = (user_va - ib_umem_start(odp)) &gt;&gt; page_shift;</div><div class='ctx'> </div><div class='del'>-	WRITE_ONCE(umem_odp-&gt;dying, 1);</div><div class='del'>-	atomic_inc(&amp;imr-&gt;num_leaf_free);</div><div class='del'>-	schedule_work(&amp;umem_odp-&gt;work);</div><div class='add'>+	if (odp-&gt;umem.writable &amp;&amp; !downgrade)</div><div class='add'>+		access_mask |= HMM_PFN_WRITE;</div><div class='ctx'> </div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='add'>+	np = ib_umem_odp_map_dma_and_lock(odp, user_va, bcnt, access_mask, fault);</div><div class='add'>+	if (np &lt; 0)</div><div class='add'>+		return np;</div><div class='ctx'> </div><div class='del'>-void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr)</div><div class='del'>-{</div><div class='del'>-	struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(imr);</div><div class='add'>+	/*</div><div class='add'>+	 * No need to check whether the MTTs really belong to this MR, since</div><div class='add'>+	 * ib_umem_odp_map_dma_and_lock already checks this.</div><div class='add'>+	 */</div><div class='add'>+	ret = mlx5r_umr_update_xlt(mr, start_idx, np, page_shift, xlt_flags);</div><div class='add'>+	mutex_unlock(&amp;odp-&gt;umem_mutex);</div><div class='ctx'> </div><div class='del'>-	down_read(&amp;per_mm-&gt;umem_rwsem);</div><div class='del'>-	rbt_ib_umem_for_each_in_range(&amp;per_mm-&gt;umem_tree, 0, ULLONG_MAX,</div><div class='del'>-				      mr_leaf_free, true, imr);</div><div class='del'>-	up_read(&amp;per_mm-&gt;umem_rwsem);</div><div class='add'>+	if (ret &lt; 0) {</div><div class='add'>+		if (ret != -EAGAIN)</div><div class='add'>+			mlx5_ib_err(mr_to_mdev(mr),</div><div class='add'>+				    "Failed to update mkey page tables\n");</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	wait_event(imr-&gt;q_leaf_free, !atomic_read(&amp;imr-&gt;num_leaf_free));</div><div class='add'>+	if (bytes_mapped) {</div><div class='add'>+		u32 new_mappings = (np &lt;&lt; page_shift) -</div><div class='add'>+			(user_va - round_down(user_va, 1 &lt;&lt; page_shift));</div><div class='add'>+</div><div class='add'>+		*bytes_mapped += min_t(u32, new_mappings, bcnt);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return np &lt;&lt; (page_shift - PAGE_SHIFT);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-#define MLX5_PF_FLAGS_PREFETCH  BIT(0)</div><div class='del'>-#define MLX5_PF_FLAGS_DOWNGRADE BIT(1)</div><div class='del'>-static int pagefault_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,</div><div class='del'>-			u64 io_virt, size_t bcnt, u32 *bytes_mapped,</div><div class='del'>-			u32 flags)</div><div class='add'>+static int pagefault_implicit_mr(struct mlx5_ib_mr *imr,</div><div class='add'>+				 struct ib_umem_odp *odp_imr, u64 user_va,</div><div class='add'>+				 size_t bcnt, u32 *bytes_mapped, u32 flags)</div><div class='ctx'> {</div><div class='del'>-	int npages = 0, current_seq, page_shift, ret, np;</div><div class='del'>-	bool implicit = false;</div><div class='del'>-	struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr-&gt;umem);</div><div class='del'>-	bool downgrade = flags &amp; MLX5_PF_FLAGS_DOWNGRADE;</div><div class='del'>-	bool prefetch = flags &amp; MLX5_PF_FLAGS_PREFETCH;</div><div class='del'>-	u64 access_mask;</div><div class='del'>-	u64 start_idx, page_mask;</div><div class='del'>-	struct ib_umem_odp *odp;</div><div class='del'>-	size_t size;</div><div class='add'>+	unsigned long end_idx = (user_va + bcnt - 1) &gt;&gt; mlx5_imr_mtt_shift;</div><div class='add'>+	unsigned long upd_start_idx = end_idx + 1;</div><div class='add'>+	unsigned long upd_len = 0;</div><div class='add'>+	unsigned long npages = 0;</div><div class='add'>+	int err;</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	if (!odp_mr-&gt;page_list) {</div><div class='del'>-		odp = implicit_mr_get_data(mr, io_virt, bcnt);</div><div class='add'>+	if (unlikely(user_va &gt;= mlx5_imr_ksm_entries * mlx5_imr_mtt_size ||</div><div class='add'>+		     mlx5_imr_ksm_entries * mlx5_imr_mtt_size - user_va &lt; bcnt))</div><div class='add'>+		return -EFAULT;</div><div class='ctx'> </div><div class='del'>-		if (IS_ERR(odp))</div><div class='del'>-			return PTR_ERR(odp);</div><div class='del'>-		mr = odp-&gt;private;</div><div class='del'>-		implicit = true;</div><div class='del'>-	} else {</div><div class='del'>-		odp = odp_mr;</div><div class='del'>-	}</div><div class='add'>+	/* Fault each child mr that intersects with our interval. */</div><div class='add'>+	while (bcnt) {</div><div class='add'>+		unsigned long idx = user_va &gt;&gt; mlx5_imr_mtt_shift;</div><div class='add'>+		struct ib_umem_odp *umem_odp;</div><div class='add'>+		struct mlx5_ib_mr *mtt;</div><div class='add'>+		u64 len;</div><div class='add'>+</div><div class='add'>+		xa_lock(&amp;imr-&gt;implicit_children);</div><div class='add'>+		mtt = xa_load(&amp;imr-&gt;implicit_children, idx);</div><div class='add'>+		if (unlikely(!mtt)) {</div><div class='add'>+			xa_unlock(&amp;imr-&gt;implicit_children);</div><div class='add'>+			mtt = implicit_get_child_mr(imr, idx);</div><div class='add'>+			if (IS_ERR(mtt)) {</div><div class='add'>+				ret = PTR_ERR(mtt);</div><div class='add'>+				goto out;</div><div class='add'>+			}</div><div class='add'>+			upd_start_idx = min(upd_start_idx, idx);</div><div class='add'>+			upd_len = idx - upd_start_idx + 1;</div><div class='add'>+		} else {</div><div class='add'>+			refcount_inc(&amp;mtt-&gt;mmkey.usecount);</div><div class='add'>+			xa_unlock(&amp;imr-&gt;implicit_children);</div><div class='add'>+		}</div><div class='ctx'> </div><div class='del'>-next_mr:</div><div class='del'>-	size = min_t(size_t, bcnt, ib_umem_end(&amp;odp-&gt;umem) - io_virt);</div><div class='add'>+		umem_odp = to_ib_umem_odp(mtt-&gt;umem);</div><div class='add'>+		len = min_t(u64, user_va + bcnt, ib_umem_end(umem_odp)) -</div><div class='add'>+		      user_va;</div><div class='ctx'> </div><div class='del'>-	page_shift = mr-&gt;umem-&gt;page_shift;</div><div class='del'>-	page_mask = ~(BIT(page_shift) - 1);</div><div class='del'>-	start_idx = (io_virt - (mr-&gt;mmkey.iova &amp; page_mask)) &gt;&gt; page_shift;</div><div class='del'>-	access_mask = ODP_READ_ALLOWED_BIT;</div><div class='add'>+		ret = pagefault_real_mr(mtt, umem_odp, user_va, len,</div><div class='add'>+					bytes_mapped, flags);</div><div class='ctx'> </div><div class='del'>-	if (prefetch &amp;&amp; !downgrade &amp;&amp; !mr-&gt;umem-&gt;writable) {</div><div class='del'>-		/* prefetch with write-access must</div><div class='del'>-		 * be supported by the MR</div><div class='del'>-		 */</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-		goto out;</div><div class='add'>+		mlx5r_deref_odp_mkey(&amp;mtt-&gt;mmkey);</div><div class='add'>+</div><div class='add'>+		if (ret &lt; 0)</div><div class='add'>+			goto out;</div><div class='add'>+		user_va += len;</div><div class='add'>+		bcnt -= len;</div><div class='add'>+		npages += ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (mr-&gt;umem-&gt;writable &amp;&amp; !downgrade)</div><div class='del'>-		access_mask |= ODP_WRITE_ALLOWED_BIT;</div><div class='add'>+	ret = npages;</div><div class='ctx'> </div><div class='del'>-	current_seq = READ_ONCE(odp-&gt;notifiers_seq);</div><div class='ctx'> 	/*</div><div class='del'>-	 * Ensure the sequence number is valid for some time before we call</div><div class='del'>-	 * gup.</div><div class='add'>+	 * Any time the implicit_children are changed we must perform an</div><div class='add'>+	 * update of the xlt before exiting to ensure the HW and the</div><div class='add'>+	 * implicit_children remains synchronized.</div><div class='ctx'> 	 */</div><div class='del'>-	smp_rmb();</div><div class='add'>+out:</div><div class='add'>+	if (likely(!upd_len))</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='del'>-	ret = ib_umem_odp_map_dma_pages(to_ib_umem_odp(mr-&gt;umem), io_virt, size,</div><div class='del'>-					access_mask, current_seq);</div><div class='add'>+	/*</div><div class='add'>+	 * Notice this is not strictly ordered right, the KSM is updated after</div><div class='add'>+	 * the implicit_children is updated, so a parallel page fault could</div><div class='add'>+	 * see a MR that is not yet visible in the KSM.  This is similar to a</div><div class='add'>+	 * parallel page fault seeing a MR that is being concurrently removed</div><div class='add'>+	 * from the KSM. Both of these improbable situations are resolved</div><div class='add'>+	 * safely by resuming the HW and then taking another page fault. The</div><div class='add'>+	 * next pagefault handler will see the new information.</div><div class='add'>+	 */</div><div class='add'>+	mutex_lock(&amp;odp_imr-&gt;umem_mutex);</div><div class='add'>+	err = mlx5r_umr_update_xlt(imr, upd_start_idx, upd_len, 0,</div><div class='add'>+				   MLX5_IB_UPD_XLT_INDIRECT |</div><div class='add'>+					  MLX5_IB_UPD_XLT_ATOMIC);</div><div class='add'>+	mutex_unlock(&amp;odp_imr-&gt;umem_mutex);</div><div class='add'>+	if (err) {</div><div class='add'>+		mlx5_ib_err(mr_to_mdev(imr), "Failed to update PAS\n");</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (ret &lt; 0)</div><div class='del'>-		goto out;</div><div class='add'>+static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt,</div><div class='add'>+			       u32 *bytes_mapped, u32 flags)</div><div class='add'>+{</div><div class='add'>+	struct ib_umem_dmabuf *umem_dmabuf = to_ib_umem_dmabuf(mr-&gt;umem);</div><div class='add'>+	int access_mode = mr-&gt;data_direct ? MLX5_MKC_ACCESS_MODE_KSM :</div><div class='add'>+					    MLX5_MKC_ACCESS_MODE_MTT;</div><div class='add'>+	unsigned int old_page_shift = mr-&gt;page_shift;</div><div class='add'>+	unsigned int page_shift;</div><div class='add'>+	unsigned long page_size;</div><div class='add'>+	u32 xlt_flags = 0;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-	np = ret;</div><div class='add'>+	if (flags &amp; MLX5_PF_FLAGS_ENABLE)</div><div class='add'>+		xlt_flags |= MLX5_IB_UPD_XLT_ENABLE;</div><div class='ctx'> </div><div class='del'>-	mutex_lock(&amp;odp-&gt;umem_mutex);</div><div class='del'>-	if (!ib_umem_mmu_notifier_retry(to_ib_umem_odp(mr-&gt;umem),</div><div class='del'>-					current_seq)) {</div><div class='del'>-		/*</div><div class='del'>-		 * No need to check whether the MTTs really belong to</div><div class='del'>-		 * this MR, since ib_umem_odp_map_dma_pages already</div><div class='del'>-		 * checks this.</div><div class='del'>-		 */</div><div class='del'>-		ret = mlx5_ib_update_xlt(mr, start_idx, np,</div><div class='del'>-					 page_shift, MLX5_IB_UPD_XLT_ATOMIC);</div><div class='del'>-	} else {</div><div class='del'>-		ret = -EAGAIN;</div><div class='add'>+	dma_resv_lock(umem_dmabuf-&gt;attach-&gt;dmabuf-&gt;resv, NULL);</div><div class='add'>+	err = ib_umem_dmabuf_map_pages(umem_dmabuf);</div><div class='add'>+	if (err) {</div><div class='add'>+		dma_resv_unlock(umem_dmabuf-&gt;attach-&gt;dmabuf-&gt;resv);</div><div class='add'>+		return err;</div><div class='ctx'> 	}</div><div class='del'>-	mutex_unlock(&amp;odp-&gt;umem_mutex);</div><div class='ctx'> </div><div class='del'>-	if (ret &lt; 0) {</div><div class='del'>-		if (ret != -EAGAIN)</div><div class='del'>-			mlx5_ib_err(dev, "Failed to update mkey page tables\n");</div><div class='del'>-		goto out;</div><div class='add'>+	page_size = mlx5_umem_dmabuf_find_best_pgsz(umem_dmabuf, access_mode);</div><div class='add'>+	if (!page_size) {</div><div class='add'>+		ib_umem_dmabuf_unmap_pages(umem_dmabuf);</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+	} else {</div><div class='add'>+		page_shift = order_base_2(page_size);</div><div class='add'>+		if (page_shift != mr-&gt;page_shift &amp;&amp; mr-&gt;dmabuf_faulted) {</div><div class='add'>+			err = mlx5r_umr_dmabuf_update_pgsz(mr, xlt_flags,</div><div class='add'>+							   page_shift);</div><div class='add'>+		} else {</div><div class='add'>+			mr-&gt;page_shift = page_shift;</div><div class='add'>+			if (mr-&gt;data_direct)</div><div class='add'>+				err = mlx5r_umr_update_data_direct_ksm_pas(</div><div class='add'>+					mr, xlt_flags);</div><div class='add'>+			else</div><div class='add'>+				err = mlx5r_umr_update_mr_pas(mr,</div><div class='add'>+							      xlt_flags);</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='add'>+	dma_resv_unlock(umem_dmabuf-&gt;attach-&gt;dmabuf-&gt;resv);</div><div class='ctx'> </div><div class='del'>-	if (bytes_mapped) {</div><div class='del'>-		u32 new_mappings = (np &lt;&lt; page_shift) -</div><div class='del'>-			(io_virt - round_down(io_virt, 1 &lt;&lt; page_shift));</div><div class='del'>-		*bytes_mapped += min_t(u32, new_mappings, size);</div><div class='add'>+	if (err) {</div><div class='add'>+		mr-&gt;page_shift = old_page_shift;</div><div class='add'>+		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	npages += np &lt;&lt; (page_shift - PAGE_SHIFT);</div><div class='del'>-	bcnt -= size;</div><div class='add'>+	mr-&gt;dmabuf_faulted = 1;</div><div class='ctx'> </div><div class='del'>-	if (unlikely(bcnt)) {</div><div class='del'>-		struct ib_umem_odp *next;</div><div class='add'>+	if (bytes_mapped)</div><div class='add'>+		*bytes_mapped += bcnt;</div><div class='ctx'> </div><div class='del'>-		io_virt += size;</div><div class='del'>-		next = odp_next(odp);</div><div class='del'>-		if (unlikely(!next || next-&gt;umem.address != io_virt)) {</div><div class='del'>-			mlx5_ib_dbg(dev, "next implicit leaf removed at 0x%llx. got %p\n",</div><div class='del'>-				    io_virt, next);</div><div class='del'>-			return -EAGAIN;</div><div class='del'>-		}</div><div class='del'>-		odp = next;</div><div class='del'>-		mr = odp-&gt;private;</div><div class='del'>-		goto next_mr;</div><div class='del'>-	}</div><div class='add'>+	return ib_umem_num_pages(mr-&gt;umem);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	return npages;</div><div class='add'>+/*</div><div class='add'>+ * Returns:</div><div class='add'>+ *  -EFAULT: The io_virt-&gt;bcnt is not within the MR, it covers pages that are</div><div class='add'>+ *           not accessible, or the MR is no longer valid.</div><div class='add'>+ *  -EAGAIN/-ENOMEM: The operation should be retried</div><div class='add'>+ *</div><div class='add'>+ *  -EINVAL/others: General internal malfunction</div><div class='add'>+ *  &gt;0: Number of pages mapped</div><div class='add'>+ */</div><div class='add'>+static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt,</div><div class='add'>+			u32 *bytes_mapped, u32 flags, bool permissive_fault)</div><div class='add'>+{</div><div class='add'>+	struct ib_umem_odp *odp = to_ib_umem_odp(mr-&gt;umem);</div><div class='ctx'> </div><div class='del'>-out:</div><div class='del'>-	if (ret == -EAGAIN) {</div><div class='del'>-		if (implicit || !odp-&gt;dying) {</div><div class='del'>-			unsigned long timeout =</div><div class='del'>-				msecs_to_jiffies(MMU_NOTIFIER_TIMEOUT);</div><div class='del'>-</div><div class='del'>-			if (!wait_for_completion_timeout(</div><div class='del'>-					&amp;odp-&gt;notifier_completion,</div><div class='del'>-					timeout)) {</div><div class='del'>-				mlx5_ib_warn(dev, "timeout waiting for mmu notifier. seq %d against %d. notifiers_count=%d\n",</div><div class='del'>-					     current_seq, odp-&gt;notifiers_seq, odp-&gt;notifiers_count);</div><div class='del'>-			}</div><div class='del'>-		} else {</div><div class='del'>-			/* The MR is being killed, kill the QP as well. */</div><div class='del'>-			ret = -EFAULT;</div><div class='del'>-		}</div><div class='add'>+	if (unlikely(io_virt &lt; mr-&gt;ibmr.iova) &amp;&amp; !permissive_fault)</div><div class='add'>+		return -EFAULT;</div><div class='add'>+</div><div class='add'>+	if (mr-&gt;umem-&gt;is_dmabuf)</div><div class='add'>+		return pagefault_dmabuf_mr(mr, bcnt, bytes_mapped, flags);</div><div class='add'>+</div><div class='add'>+	if (!odp-&gt;is_implicit_odp) {</div><div class='add'>+		u64 offset = io_virt &lt; mr-&gt;ibmr.iova ? 0 : io_virt - mr-&gt;ibmr.iova;</div><div class='add'>+		u64 user_va;</div><div class='add'>+</div><div class='add'>+		if (check_add_overflow(offset, (u64)odp-&gt;umem.address,</div><div class='add'>+				       &amp;user_va))</div><div class='add'>+			return -EFAULT;</div><div class='add'>+</div><div class='add'>+		if (permissive_fault) {</div><div class='add'>+			if (user_va &lt; ib_umem_start(odp))</div><div class='add'>+				user_va = ib_umem_start(odp);</div><div class='add'>+			if ((user_va + bcnt) &gt; ib_umem_end(odp))</div><div class='add'>+				bcnt = ib_umem_end(odp) - user_va;</div><div class='add'>+		} else if (unlikely(user_va &gt;= ib_umem_end(odp) ||</div><div class='add'>+				    ib_umem_end(odp) - user_va &lt; bcnt))</div><div class='add'>+			return -EFAULT;</div><div class='add'>+		return pagefault_real_mr(mr, odp, user_va, bcnt, bytes_mapped,</div><div class='add'>+					 flags);</div><div class='ctx'> 	}</div><div class='add'>+	return pagefault_implicit_mr(mr, odp, io_virt, bcnt, bytes_mapped,</div><div class='add'>+				     flags);</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	return ret;</div><div class='add'>+int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = pagefault_real_mr(mr, to_ib_umem_odp(mr-&gt;umem), mr-&gt;umem-&gt;address,</div><div class='add'>+				mr-&gt;umem-&gt;length, NULL,</div><div class='add'>+				MLX5_PF_FLAGS_SNAPSHOT | MLX5_PF_FLAGS_ENABLE);</div><div class='add'>+	return ret &gt;= 0 ? 0 : ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = pagefault_dmabuf_mr(mr, mr-&gt;umem-&gt;length, NULL,</div><div class='add'>+				  MLX5_PF_FLAGS_ENABLE);</div><div class='add'>+</div><div class='add'>+	return ret &gt;= 0 ? 0 : ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct pf_frame {</div><div class='hunk'>@@ -711,35 +956,41 @@ struct pf_frame {</div><div class='ctx'> 	int depth;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static bool mkey_is_eq(struct mlx5_core_mkey *mmkey, u32 key)</div><div class='add'>+static bool mkey_is_eq(struct mlx5_ib_mkey *mmkey, u32 key)</div><div class='ctx'> {</div><div class='ctx'> 	if (!mmkey)</div><div class='ctx'> 		return false;</div><div class='del'>-	if (mmkey-&gt;type == MLX5_MKEY_MW)</div><div class='add'>+	if (mmkey-&gt;type == MLX5_MKEY_MW ||</div><div class='add'>+	    mmkey-&gt;type == MLX5_MKEY_INDIRECT_DEVX)</div><div class='ctx'> 		return mlx5_base_mkey(mmkey-&gt;key) == mlx5_base_mkey(key);</div><div class='ctx'> 	return mmkey-&gt;key == key;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int get_indirect_num_descs(struct mlx5_core_mkey *mmkey)</div><div class='add'>+static struct mlx5_ib_mkey *find_odp_mkey(struct mlx5_ib_dev *dev, u32 key)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_mw *mw;</div><div class='del'>-	struct mlx5_ib_devx_mr *devx_mr;</div><div class='add'>+	struct mlx5_ib_mkey *mmkey;</div><div class='ctx'> </div><div class='del'>-	if (mmkey-&gt;type == MLX5_MKEY_MW) {</div><div class='del'>-		mw = container_of(mmkey, struct mlx5_ib_mw, mmkey);</div><div class='del'>-		return mw-&gt;ndescs;</div><div class='add'>+	xa_lock(&amp;dev-&gt;odp_mkeys);</div><div class='add'>+	mmkey = xa_load(&amp;dev-&gt;odp_mkeys, mlx5_base_mkey(key));</div><div class='add'>+	if (!mmkey) {</div><div class='add'>+		mmkey = ERR_PTR(-ENOENT);</div><div class='add'>+		goto out;</div><div class='ctx'> 	}</div><div class='add'>+	if (!mkey_is_eq(mmkey, key)) {</div><div class='add'>+		mmkey = ERR_PTR(-EFAULT);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+	refcount_inc(&amp;mmkey-&gt;usecount);</div><div class='add'>+out:</div><div class='add'>+	xa_unlock(&amp;dev-&gt;odp_mkeys);</div><div class='ctx'> </div><div class='del'>-	devx_mr = container_of(mmkey, struct mlx5_ib_devx_mr,</div><div class='del'>-			       mmkey);</div><div class='del'>-	return devx_mr-&gt;ndescs;</div><div class='add'>+	return mmkey;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> /*</div><div class='ctx'>  * Handle a single data segment in a page-fault WQE or RDMA region.</div><div class='ctx'>  *</div><div class='del'>- * Returns number of OS pages retrieved on success. The caller may continue to</div><div class='del'>- * the next data segment.</div><div class='add'>+ * Returns zero on success. The caller may continue to the next data segment.</div><div class='ctx'>  * Can return the following error codes:</div><div class='ctx'>  * -EAGAIN to designate a temporary error. The caller will abort handling the</div><div class='ctx'>  *  page fault and resolve it.</div><div class='hunk'>@@ -750,93 +1001,77 @@ static int pagefault_single_data_segment(struct mlx5_ib_dev *dev,</div><div class='ctx'> 					 struct ib_pd *pd, u32 key,</div><div class='ctx'> 					 u64 io_virt, size_t bcnt,</div><div class='ctx'> 					 u32 *bytes_committed,</div><div class='del'>-					 u32 *bytes_mapped, u32 flags)</div><div class='add'>+					 u32 *bytes_mapped)</div><div class='ctx'> {</div><div class='del'>-	int npages = 0, srcu_key, ret, i, outlen, cur_outlen = 0, depth = 0;</div><div class='del'>-	bool prefetch = flags &amp; MLX5_PF_FLAGS_PREFETCH;</div><div class='add'>+	int ret, i, outlen, cur_outlen = 0, depth = 0, pages_in_range;</div><div class='ctx'> 	struct pf_frame *head = NULL, *frame;</div><div class='del'>-	struct mlx5_core_mkey *mmkey;</div><div class='add'>+	struct mlx5_ib_mkey *mmkey;</div><div class='ctx'> 	struct mlx5_ib_mr *mr;</div><div class='ctx'> 	struct mlx5_klm *pklm;</div><div class='ctx'> 	u32 *out = NULL;</div><div class='ctx'> 	size_t offset;</div><div class='del'>-	int ndescs;</div><div class='del'>-</div><div class='del'>-	srcu_key = srcu_read_lock(&amp;dev-&gt;mr_srcu);</div><div class='ctx'> </div><div class='ctx'> 	io_virt += *bytes_committed;</div><div class='ctx'> 	bcnt -= *bytes_committed;</div><div class='del'>-</div><div class='ctx'> next_mr:</div><div class='del'>-	mmkey = xa_load(&amp;dev-&gt;mdev-&gt;priv.mkey_table, mlx5_base_mkey(key));</div><div class='del'>-	if (!mkey_is_eq(mmkey, key)) {</div><div class='del'>-		mlx5_ib_dbg(dev, "failed to find mkey %x\n", key);</div><div class='del'>-		ret = -EFAULT;</div><div class='del'>-		goto srcu_unlock;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (prefetch &amp;&amp; mmkey-&gt;type != MLX5_MKEY_MR) {</div><div class='del'>-		mlx5_ib_dbg(dev, "prefetch is allowed only for MR\n");</div><div class='del'>-		ret = -EINVAL;</div><div class='del'>-		goto srcu_unlock;</div><div class='add'>+	mmkey = find_odp_mkey(dev, key);</div><div class='add'>+	if (IS_ERR(mmkey)) {</div><div class='add'>+		ret = PTR_ERR(mmkey);</div><div class='add'>+		if (ret == -ENOENT) {</div><div class='add'>+			mlx5_ib_dbg(</div><div class='add'>+				dev,</div><div class='add'>+				"skipping non ODP MR (lkey=0x%06x) in page fault handler.\n",</div><div class='add'>+				key);</div><div class='add'>+			if (bytes_mapped)</div><div class='add'>+				*bytes_mapped += bcnt;</div><div class='add'>+			/*</div><div class='add'>+			 * The user could specify a SGL with multiple lkeys and</div><div class='add'>+			 * only some of them are ODP. Treat the non-ODP ones as</div><div class='add'>+			 * fully faulted.</div><div class='add'>+			 */</div><div class='add'>+			ret = 0;</div><div class='add'>+		}</div><div class='add'>+		goto end;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	switch (mmkey-&gt;type) {</div><div class='ctx'> 	case MLX5_MKEY_MR:</div><div class='ctx'> 		mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);</div><div class='del'>-		if (!mr-&gt;live || !mr-&gt;ibmr.pd) {</div><div class='del'>-			mlx5_ib_dbg(dev, "got dead MR\n");</div><div class='del'>-			ret = -EFAULT;</div><div class='del'>-			goto srcu_unlock;</div><div class='del'>-		}</div><div class='ctx'> </div><div class='del'>-		if (prefetch) {</div><div class='del'>-			if (!is_odp_mr(mr) ||</div><div class='del'>-			    mr-&gt;ibmr.pd != pd) {</div><div class='del'>-				mlx5_ib_dbg(dev, "Invalid prefetch request: %s\n",</div><div class='del'>-					    is_odp_mr(mr) ?  "MR is not ODP" :</div><div class='del'>-					    "PD is not of the MR");</div><div class='del'>-				ret = -EINVAL;</div><div class='del'>-				goto srcu_unlock;</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='add'>+		pages_in_range = (ALIGN(io_virt + bcnt, PAGE_SIZE) -</div><div class='add'>+				  (io_virt &amp; PAGE_MASK)) &gt;&gt;</div><div class='add'>+				 PAGE_SHIFT;</div><div class='add'>+		ret = pagefault_mr(mr, io_virt, bcnt, bytes_mapped, 0, false);</div><div class='add'>+		if (ret &lt; 0)</div><div class='add'>+			goto end;</div><div class='ctx'> </div><div class='del'>-		if (!is_odp_mr(mr)) {</div><div class='del'>-			mlx5_ib_dbg(dev, "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n",</div><div class='del'>-				    key);</div><div class='del'>-			if (bytes_mapped)</div><div class='del'>-				*bytes_mapped += bcnt;</div><div class='del'>-			ret = 0;</div><div class='del'>-			goto srcu_unlock;</div><div class='del'>-		}</div><div class='add'>+		mlx5_update_odp_stats_with_handled(mr, faults, ret);</div><div class='ctx'> </div><div class='del'>-		ret = pagefault_mr(dev, mr, io_virt, bcnt, bytes_mapped, flags);</div><div class='del'>-		if (ret &lt; 0)</div><div class='del'>-			goto srcu_unlock;</div><div class='add'>+		if (ret &lt; pages_in_range) {</div><div class='add'>+			ret = -EFAULT;</div><div class='add'>+			goto end;</div><div class='add'>+		}</div><div class='ctx'> </div><div class='del'>-		npages += ret;</div><div class='ctx'> 		ret = 0;</div><div class='ctx'> 		break;</div><div class='ctx'> </div><div class='ctx'> 	case MLX5_MKEY_MW:</div><div class='ctx'> 	case MLX5_MKEY_INDIRECT_DEVX:</div><div class='del'>-		ndescs = get_indirect_num_descs(mmkey);</div><div class='del'>-</div><div class='ctx'> 		if (depth &gt;= MLX5_CAP_GEN(dev-&gt;mdev, max_indirection)) {</div><div class='ctx'> 			mlx5_ib_dbg(dev, "indirection level exceeded\n");</div><div class='ctx'> 			ret = -EFAULT;</div><div class='del'>-			goto srcu_unlock;</div><div class='add'>+			goto end;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='ctx'> 		outlen = MLX5_ST_SZ_BYTES(query_mkey_out) +</div><div class='del'>-			sizeof(*pklm) * (ndescs - 2);</div><div class='add'>+			sizeof(*pklm) * (mmkey-&gt;ndescs - 2);</div><div class='ctx'> </div><div class='ctx'> 		if (outlen &gt; cur_outlen) {</div><div class='ctx'> 			kfree(out);</div><div class='ctx'> 			out = kzalloc(outlen, GFP_KERNEL);</div><div class='ctx'> 			if (!out) {</div><div class='ctx'> 				ret = -ENOMEM;</div><div class='del'>-				goto srcu_unlock;</div><div class='add'>+				goto end;</div><div class='ctx'> 			}</div><div class='ctx'> 			cur_outlen = outlen;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -844,14 +1079,14 @@ next_mr:</div><div class='ctx'> 		pklm = (struct mlx5_klm *)MLX5_ADDR_OF(query_mkey_out, out,</div><div class='ctx'> 						       bsf0_klm0_pas_mtt0_1);</div><div class='ctx'> </div><div class='del'>-		ret = mlx5_core_query_mkey(dev-&gt;mdev, mmkey, out, outlen);</div><div class='add'>+		ret = mlx5_core_query_mkey(dev-&gt;mdev, mmkey-&gt;key, out, outlen);</div><div class='ctx'> 		if (ret)</div><div class='del'>-			goto srcu_unlock;</div><div class='add'>+			goto end;</div><div class='ctx'> </div><div class='ctx'> 		offset = io_virt - MLX5_GET64(query_mkey_out, out,</div><div class='ctx'> 					      memory_key_mkey_entry.start_addr);</div><div class='ctx'> </div><div class='del'>-		for (i = 0; bcnt &amp;&amp; i &lt; ndescs; i++, pklm++) {</div><div class='add'>+		for (i = 0; bcnt &amp;&amp; i &lt; mmkey-&gt;ndescs; i++, pklm++) {</div><div class='ctx'> 			if (offset &gt;= be32_to_cpu(pklm-&gt;bcount)) {</div><div class='ctx'> 				offset -= be32_to_cpu(pklm-&gt;bcount);</div><div class='ctx'> 				continue;</div><div class='hunk'>@@ -860,7 +1095,7 @@ next_mr:</div><div class='ctx'> 			frame = kzalloc(sizeof(*frame), GFP_KERNEL);</div><div class='ctx'> 			if (!frame) {</div><div class='ctx'> 				ret = -ENOMEM;</div><div class='del'>-				goto srcu_unlock;</div><div class='add'>+				goto end;</div><div class='ctx'> 			}</div><div class='ctx'> </div><div class='ctx'> 			frame-&gt;key = be32_to_cpu(pklm-&gt;key);</div><div class='hunk'>@@ -879,7 +1114,7 @@ next_mr:</div><div class='ctx'> 	default:</div><div class='ctx'> 		mlx5_ib_dbg(dev, "wrong mkey type %d\n", mmkey-&gt;type);</div><div class='ctx'> 		ret = -EFAULT;</div><div class='del'>-		goto srcu_unlock;</div><div class='add'>+		goto end;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (head) {</div><div class='hunk'>@@ -892,10 +1127,13 @@ next_mr:</div><div class='ctx'> 		depth = frame-&gt;depth;</div><div class='ctx'> 		kfree(frame);</div><div class='ctx'> </div><div class='add'>+		mlx5r_deref_odp_mkey(mmkey);</div><div class='ctx'> 		goto next_mr;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-srcu_unlock:</div><div class='add'>+end:</div><div class='add'>+	if (!IS_ERR(mmkey))</div><div class='add'>+		mlx5r_deref_odp_mkey(mmkey);</div><div class='ctx'> 	while (head) {</div><div class='ctx'> 		frame = head;</div><div class='ctx'> 		head = frame-&gt;next;</div><div class='hunk'>@@ -903,27 +1141,27 @@ srcu_unlock:</div><div class='ctx'> 	}</div><div class='ctx'> 	kfree(out);</div><div class='ctx'> </div><div class='del'>-	srcu_read_unlock(&amp;dev-&gt;mr_srcu, srcu_key);</div><div class='ctx'> 	*bytes_committed = 0;</div><div class='del'>-	return ret ? ret : npages;</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-/**</div><div class='add'>+/*</div><div class='ctx'>  * Parse a series of data segments for page fault handling.</div><div class='ctx'>  *</div><div class='del'>- * @pfault contains page fault information.</div><div class='del'>- * @wqe points at the first data segment in the WQE.</div><div class='del'>- * @wqe_end points after the end of the WQE.</div><div class='del'>- * @bytes_mapped receives the number of bytes that the function was able to</div><div class='del'>- *               map. This allows the caller to decide intelligently whether</div><div class='del'>- *               enough memory was mapped to resolve the page fault</div><div class='del'>- *               successfully (e.g. enough for the next MTU, or the entire</div><div class='del'>- *               WQE).</div><div class='del'>- * @total_wqe_bytes receives the total data size of this WQE in bytes (minus</div><div class='del'>- *                  the committed bytes).</div><div class='add'>+ * @dev:  Pointer to mlx5 IB device</div><div class='add'>+ * @pfault: contains page fault information.</div><div class='add'>+ * @wqe: points at the first data segment in the WQE.</div><div class='add'>+ * @wqe_end: points after the end of the WQE.</div><div class='add'>+ * @bytes_mapped: receives the number of bytes that the function was able to</div><div class='add'>+ *                map. This allows the caller to decide intelligently whether</div><div class='add'>+ *                enough memory was mapped to resolve the page fault</div><div class='add'>+ *                successfully (e.g. enough for the next MTU, or the entire</div><div class='add'>+ *                WQE).</div><div class='add'>+ * @total_wqe_bytes: receives the total data size of this WQE in bytes (minus</div><div class='add'>+ *                   the committed bytes).</div><div class='add'>+ * @receive_queue: receive WQE end of sg list</div><div class='ctx'>  *</div><div class='del'>- * Returns the number of pages loaded if positive, zero for an empty WQE, or a</div><div class='del'>- * negative error code.</div><div class='add'>+ * Returns zero for success or a negative error code.</div><div class='ctx'>  */</div><div class='ctx'> static int pagefault_data_segments(struct mlx5_ib_dev *dev,</div><div class='ctx'> 				   struct mlx5_pagefault *pfault,</div><div class='hunk'>@@ -931,9 +1169,9 @@ static int pagefault_data_segments(struct mlx5_ib_dev *dev,</div><div class='ctx'> 				   void *wqe_end, u32 *bytes_mapped,</div><div class='ctx'> 				   u32 *total_wqe_bytes, bool receive_queue)</div><div class='ctx'> {</div><div class='del'>-	int ret = 0, npages = 0;</div><div class='add'>+	int ret = 0;</div><div class='ctx'> 	u64 io_virt;</div><div class='del'>-	u32 key;</div><div class='add'>+	__be32 key;</div><div class='ctx'> 	u32 byte_count;</div><div class='ctx'> 	size_t bcnt;</div><div class='ctx'> 	int inline_segment;</div><div class='hunk'>@@ -947,7 +1185,7 @@ static int pagefault_data_segments(struct mlx5_ib_dev *dev,</div><div class='ctx'> 		struct mlx5_wqe_data_seg *dseg = wqe;</div><div class='ctx'> </div><div class='ctx'> 		io_virt = be64_to_cpu(dseg-&gt;addr);</div><div class='del'>-		key = be32_to_cpu(dseg-&gt;lkey);</div><div class='add'>+		key = dseg-&gt;lkey;</div><div class='ctx'> 		byte_count = be32_to_cpu(dseg-&gt;byte_count);</div><div class='ctx'> 		inline_segment = !!(byte_count &amp;  MLX5_INLINE_SEG);</div><div class='ctx'> 		bcnt	       = byte_count &amp; ~MLX5_INLINE_SEG;</div><div class='hunk'>@@ -961,7 +1199,8 @@ static int pagefault_data_segments(struct mlx5_ib_dev *dev,</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='ctx'> 		/* receive WQE end of sg list. */</div><div class='del'>-		if (receive_queue &amp;&amp; bcnt == 0 &amp;&amp; key == MLX5_INVALID_LKEY &amp;&amp;</div><div class='add'>+		if (receive_queue &amp;&amp; bcnt == 0 &amp;&amp;</div><div class='add'>+		    key == dev-&gt;mkeys.terminate_scatter_list_mkey &amp;&amp;</div><div class='ctx'> 		    io_virt == 0)</div><div class='ctx'> 			break;</div><div class='ctx'> </div><div class='hunk'>@@ -981,29 +1220,17 @@ static int pagefault_data_segments(struct mlx5_ib_dev *dev,</div><div class='ctx'> 			continue;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='del'>-		ret = pagefault_single_data_segment(dev, NULL, key,</div><div class='add'>+		ret = pagefault_single_data_segment(dev, NULL, be32_to_cpu(key),</div><div class='ctx'> 						    io_virt, bcnt,</div><div class='ctx'> 						    &amp;pfault-&gt;bytes_committed,</div><div class='del'>-						    bytes_mapped, 0);</div><div class='add'>+						    bytes_mapped);</div><div class='ctx'> 		if (ret &lt; 0)</div><div class='ctx'> 			break;</div><div class='del'>-		npages += ret;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return ret &lt; 0 ? ret : npages;</div><div class='add'>+	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static const u32 mlx5_ib_odp_opcode_cap[] = {</div><div class='del'>-	[MLX5_OPCODE_SEND]	       = IB_ODP_SUPPORT_SEND,</div><div class='del'>-	[MLX5_OPCODE_SEND_IMM]	       = IB_ODP_SUPPORT_SEND,</div><div class='del'>-	[MLX5_OPCODE_SEND_INVAL]       = IB_ODP_SUPPORT_SEND,</div><div class='del'>-	[MLX5_OPCODE_RDMA_WRITE]       = IB_ODP_SUPPORT_WRITE,</div><div class='del'>-	[MLX5_OPCODE_RDMA_WRITE_IMM]   = IB_ODP_SUPPORT_WRITE,</div><div class='del'>-	[MLX5_OPCODE_RDMA_READ]	       = IB_ODP_SUPPORT_READ,</div><div class='del'>-	[MLX5_OPCODE_ATOMIC_CS]	       = IB_ODP_SUPPORT_ATOMIC,</div><div class='del'>-	[MLX5_OPCODE_ATOMIC_FA]	       = IB_ODP_SUPPORT_ATOMIC,</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> /*</div><div class='ctx'>  * Parse initiator WQE. Advances the wqe pointer to point at the</div><div class='ctx'>  * scatter-gather list, and set wqe_end to the end of the WQE.</div><div class='hunk'>@@ -1014,12 +1241,8 @@ static int mlx5_ib_mr_initiator_pfault_handler(</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_wqe_ctrl_seg *ctrl = *wqe;</div><div class='ctx'> 	u16 wqe_index = pfault-&gt;wqe.wqe_index;</div><div class='del'>-	u32 transport_caps;</div><div class='ctx'> 	struct mlx5_base_av *av;</div><div class='ctx'> 	unsigned ds, opcode;</div><div class='del'>-#if defined(DEBUG)</div><div class='del'>-	u32 ctrl_wqe_index, ctrl_qpn;</div><div class='del'>-#endif</div><div class='ctx'> 	u32 qpn = qp-&gt;trans_qp.base.mqp.qpn;</div><div class='ctx'> </div><div class='ctx'> 	ds = be32_to_cpu(ctrl-&gt;qpn_ds) &amp; MLX5_WQE_CTRL_DS_MASK;</div><div class='hunk'>@@ -1035,58 +1258,16 @@ static int mlx5_ib_mr_initiator_pfault_handler(</div><div class='ctx'> 		return -EFAULT;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-#if defined(DEBUG)</div><div class='del'>-	ctrl_wqe_index = (be32_to_cpu(ctrl-&gt;opmod_idx_opcode) &amp;</div><div class='del'>-			MLX5_WQE_CTRL_WQE_INDEX_MASK) &gt;&gt;</div><div class='del'>-			MLX5_WQE_CTRL_WQE_INDEX_SHIFT;</div><div class='del'>-	if (wqe_index != ctrl_wqe_index) {</div><div class='del'>-		mlx5_ib_err(dev, "Got WQE with invalid wqe_index. wqe_index=0x%x, qpn=0x%x ctrl-&gt;wqe_index=0x%x\n",</div><div class='del'>-			    wqe_index, qpn,</div><div class='del'>-			    ctrl_wqe_index);</div><div class='del'>-		return -EFAULT;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	ctrl_qpn = (be32_to_cpu(ctrl-&gt;qpn_ds) &amp; MLX5_WQE_CTRL_QPN_MASK) &gt;&gt;</div><div class='del'>-		MLX5_WQE_CTRL_QPN_SHIFT;</div><div class='del'>-	if (qpn != ctrl_qpn) {</div><div class='del'>-		mlx5_ib_err(dev, "Got WQE with incorrect QP number. wqe_index=0x%x, qpn=0x%x ctrl-&gt;qpn=0x%x\n",</div><div class='del'>-			    wqe_index, qpn,</div><div class='del'>-			    ctrl_qpn);</div><div class='del'>-		return -EFAULT;</div><div class='del'>-	}</div><div class='del'>-#endif /* DEBUG */</div><div class='del'>-</div><div class='ctx'> 	*wqe_end = *wqe + ds * MLX5_WQE_DS_UNITS;</div><div class='ctx'> 	*wqe += sizeof(*ctrl);</div><div class='ctx'> </div><div class='ctx'> 	opcode = be32_to_cpu(ctrl-&gt;opmod_idx_opcode) &amp;</div><div class='ctx'> 		 MLX5_WQE_CTRL_OPCODE_MASK;</div><div class='ctx'> </div><div class='del'>-	switch (qp-&gt;ibqp.qp_type) {</div><div class='del'>-	case IB_QPT_XRC_INI:</div><div class='add'>+	if (qp-&gt;type == IB_QPT_XRC_INI)</div><div class='ctx'> 		*wqe += sizeof(struct mlx5_wqe_xrc_seg);</div><div class='del'>-		transport_caps = dev-&gt;odp_caps.per_transport_caps.xrc_odp_caps;</div><div class='del'>-		break;</div><div class='del'>-	case IB_QPT_RC:</div><div class='del'>-		transport_caps = dev-&gt;odp_caps.per_transport_caps.rc_odp_caps;</div><div class='del'>-		break;</div><div class='del'>-	case IB_QPT_UD:</div><div class='del'>-		transport_caps = dev-&gt;odp_caps.per_transport_caps.ud_odp_caps;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		mlx5_ib_err(dev, "ODP fault on QP of an unsupported transport 0x%x\n",</div><div class='del'>-			    qp-&gt;ibqp.qp_type);</div><div class='del'>-		return -EFAULT;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (unlikely(opcode &gt;= ARRAY_SIZE(mlx5_ib_odp_opcode_cap) ||</div><div class='del'>-		     !(transport_caps &amp; mlx5_ib_odp_opcode_cap[opcode]))) {</div><div class='del'>-		mlx5_ib_err(dev, "ODP fault on QP of an unsupported opcode 0x%x\n",</div><div class='del'>-			    opcode);</div><div class='del'>-		return -EFAULT;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;ibqp.qp_type == IB_QPT_UD) {</div><div class='add'>+	if (qp-&gt;type == IB_QPT_UD || qp-&gt;type == MLX5_IB_QPT_DCI) {</div><div class='ctx'> 		av = *wqe;</div><div class='ctx'> 		if (av-&gt;dqp_dct &amp; cpu_to_be32(MLX5_EXTENDED_UD_AV))</div><div class='ctx'> 			*wqe += sizeof(struct mlx5_av);</div><div class='hunk'>@@ -1139,7 +1320,7 @@ static int mlx5_ib_mr_responder_pfault_handler_rq(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	struct mlx5_ib_wq *wq = &amp;qp-&gt;rq;</div><div class='ctx'> 	int wqe_size = 1 &lt;&lt; wq-&gt;wqe_shift;</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;wq_sig) {</div><div class='add'>+	if (qp-&gt;flags_en &amp; MLX5_QP_FLAG_SIGNATURE) {</div><div class='ctx'> 		mlx5_ib_err(dev, "ODP fault with WQE signatures is not supported\n");</div><div class='ctx'> 		return -EFAULT;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -1149,19 +1330,6 @@ static int mlx5_ib_mr_responder_pfault_handler_rq(struct mlx5_ib_dev *dev,</div><div class='ctx'> 		return -EFAULT;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	switch (qp-&gt;ibqp.qp_type) {</div><div class='del'>-	case IB_QPT_RC:</div><div class='del'>-		if (!(dev-&gt;odp_caps.per_transport_caps.rc_odp_caps &amp;</div><div class='del'>-		      IB_ODP_SUPPORT_RECV))</div><div class='del'>-			goto invalid_transport_or_opcode;</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-invalid_transport_or_opcode:</div><div class='del'>-		mlx5_ib_err(dev, "ODP fault on QP of an unsupported transport. transport: 0x%x\n",</div><div class='del'>-			    qp-&gt;ibqp.qp_type);</div><div class='del'>-		return -EFAULT;</div><div class='del'>-	}</div><div class='del'>-</div><div class='ctx'> 	*wqe_end = wqe + wqe_size;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='hunk'>@@ -1182,7 +1350,7 @@ static inline struct mlx5_core_rsc_common *odp_get_rsc(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	case MLX5_WQE_PF_TYPE_REQ_SEND_OR_WRITE:</div><div class='ctx'> 	case MLX5_WQE_PF_TYPE_RESP:</div><div class='ctx'> 	case MLX5_WQE_PF_TYPE_REQ_READ_OR_ATOMIC:</div><div class='del'>-		common = mlx5_core_res_hold(dev-&gt;mdev, wq_num, MLX5_RES_QP);</div><div class='add'>+		common = mlx5_core_res_hold(dev, wq_num, MLX5_RES_QP);</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		break;</div><div class='hunk'>@@ -1211,7 +1379,7 @@ static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev,</div><div class='ctx'> {</div><div class='ctx'> 	bool sq = pfault-&gt;type &amp; MLX5_PFAULT_REQUESTOR;</div><div class='ctx'> 	u16 wqe_index = pfault-&gt;wqe.wqe_index;</div><div class='del'>-	void *wqe = NULL, *wqe_end = NULL;</div><div class='add'>+	void *wqe, *wqe_start = NULL, *wqe_end = NULL;</div><div class='ctx'> 	u32 bytes_mapped, total_wqe_bytes;</div><div class='ctx'> 	struct mlx5_core_rsc_common *res;</div><div class='ctx'> 	int resume_with_error = 1;</div><div class='hunk'>@@ -1232,23 +1400,24 @@ static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev,</div><div class='ctx'> 		goto resolve_page_fault;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	wqe = (void *)__get_free_page(GFP_KERNEL);</div><div class='del'>-	if (!wqe) {</div><div class='add'>+	wqe_start = (void *)__get_free_page(GFP_KERNEL);</div><div class='add'>+	if (!wqe_start) {</div><div class='ctx'> 		mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n");</div><div class='ctx'> 		goto resolve_page_fault;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	wqe = wqe_start;</div><div class='ctx'> 	qp = (res-&gt;res == MLX5_RES_QP) ? res_to_qp(res) : NULL;</div><div class='ctx'> 	if (qp &amp;&amp; sq) {</div><div class='del'>-		ret = mlx5_ib_read_user_wqe_sq(qp, wqe_index, wqe, PAGE_SIZE,</div><div class='del'>-					       &amp;bytes_copied);</div><div class='add'>+		ret = mlx5_ib_read_wqe_sq(qp, wqe_index, wqe, PAGE_SIZE,</div><div class='add'>+					  &amp;bytes_copied);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto read_user;</div><div class='ctx'> 		ret = mlx5_ib_mr_initiator_pfault_handler(</div><div class='ctx'> 			dev, pfault, qp, &amp;wqe, &amp;wqe_end, bytes_copied);</div><div class='ctx'> 	} else if (qp &amp;&amp; !sq) {</div><div class='del'>-		ret = mlx5_ib_read_user_wqe_rq(qp, wqe_index, wqe, PAGE_SIZE,</div><div class='del'>-					       &amp;bytes_copied);</div><div class='add'>+		ret = mlx5_ib_read_wqe_rq(qp, wqe_index, wqe, PAGE_SIZE,</div><div class='add'>+					  &amp;bytes_copied);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto read_user;</div><div class='ctx'> 		ret = mlx5_ib_mr_responder_pfault_handler_rq(</div><div class='hunk'>@@ -1256,8 +1425,8 @@ static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	} else if (!qp) {</div><div class='ctx'> 		struct mlx5_ib_srq *srq = res_to_srq(res);</div><div class='ctx'> </div><div class='del'>-		ret = mlx5_ib_read_user_wqe_srq(srq, wqe_index, wqe, PAGE_SIZE,</div><div class='del'>-						&amp;bytes_copied);</div><div class='add'>+		ret = mlx5_ib_read_wqe_srq(srq, wqe_index, wqe, PAGE_SIZE,</div><div class='add'>+					   &amp;bytes_copied);</div><div class='ctx'> 		if (ret)</div><div class='ctx'> 			goto read_user;</div><div class='ctx'> 		ret = mlx5_ib_mr_responder_pfault_handler_srq(</div><div class='hunk'>@@ -1283,7 +1452,7 @@ read_user:</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		mlx5_ib_err(</div><div class='ctx'> 			dev,</div><div class='del'>-			"Failed reading a WQE following page fault, error %d, wqe_index %x, qpn %x\n",</div><div class='add'>+			"Failed reading a WQE following page fault, error %d, wqe_index %x, qpn %llx\n",</div><div class='ctx'> 			ret, wqe_index, pfault-&gt;token);</div><div class='ctx'> </div><div class='ctx'> resolve_page_fault:</div><div class='hunk'>@@ -1292,13 +1461,7 @@ resolve_page_fault:</div><div class='ctx'> 		    pfault-&gt;wqe.wq_num, resume_with_error,</div><div class='ctx'> 		    pfault-&gt;type);</div><div class='ctx'> 	mlx5_core_res_put(res);</div><div class='del'>-	free_page((unsigned long)wqe);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int pages_in_range(u64 address, u32 length)</div><div class='del'>-{</div><div class='del'>-	return (ALIGN(address + length, PAGE_SIZE) -</div><div class='del'>-		(address &amp; PAGE_MASK)) &gt;&gt; PAGE_SHIFT;</div><div class='add'>+	free_page((unsigned long)wqe_start);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev,</div><div class='hunk'>@@ -1335,21 +1498,20 @@ static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	ret = pagefault_single_data_segment(dev, NULL, rkey, address, length,</div><div class='del'>-					    &amp;pfault-&gt;bytes_committed, NULL,</div><div class='del'>-					    0);</div><div class='add'>+					    &amp;pfault-&gt;bytes_committed, NULL);</div><div class='ctx'> 	if (ret == -EAGAIN) {</div><div class='ctx'> 		/* We're racing with an invalidation, don't prefetch */</div><div class='ctx'> 		prefetch_activated = 0;</div><div class='del'>-	} else if (ret &lt; 0 || pages_in_range(address, length) &gt; ret) {</div><div class='add'>+	} else if (ret &lt; 0) {</div><div class='ctx'> 		mlx5_ib_page_fault_resume(dev, pfault, 1);</div><div class='ctx'> 		if (ret != -ENOENT)</div><div class='del'>-			mlx5_ib_dbg(dev, "PAGE FAULT error %d. QP 0x%x, type: 0x%x\n",</div><div class='add'>+			mlx5_ib_dbg(dev, "PAGE FAULT error %d. QP 0x%llx, type: 0x%x\n",</div><div class='ctx'> 				    ret, pfault-&gt;token, pfault-&gt;type);</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	mlx5_ib_page_fault_resume(dev, pfault, 0);</div><div class='del'>-	mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x, type: 0x%x, prefetch_activated: %d\n",</div><div class='add'>+	mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%llx, type: 0x%x, prefetch_activated: %d\n",</div><div class='ctx'> 		    pfault-&gt;token, pfault-&gt;type,</div><div class='ctx'> 		    prefetch_activated);</div><div class='ctx'> </div><div class='hunk'>@@ -1363,15 +1525,82 @@ static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev,</div><div class='ctx'> </div><div class='ctx'> 		ret = pagefault_single_data_segment(dev, NULL, rkey, address,</div><div class='ctx'> 						    prefetch_len,</div><div class='del'>-						    &amp;bytes_committed, NULL,</div><div class='del'>-						    0);</div><div class='add'>+						    &amp;bytes_committed, NULL);</div><div class='ctx'> 		if (ret &lt; 0 &amp;&amp; ret != -EAGAIN) {</div><div class='del'>-			mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n",</div><div class='add'>+			mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%llx, address: 0x%.16llx, length = 0x%.16x\n",</div><div class='ctx'> 				    ret, pfault-&gt;token, address, prefetch_len);</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+#define MLX5_MEMORY_PAGE_FAULT_FLAGS_LAST BIT(7)</div><div class='add'>+static void mlx5_ib_mr_memory_pfault_handler(struct mlx5_ib_dev *dev,</div><div class='add'>+					     struct mlx5_pagefault *pfault)</div><div class='add'>+{</div><div class='add'>+	u64 prefetch_va =</div><div class='add'>+		pfault-&gt;memory.va - pfault-&gt;memory.prefetch_before_byte_count;</div><div class='add'>+	size_t prefetch_size = pfault-&gt;memory.prefetch_before_byte_count +</div><div class='add'>+			       pfault-&gt;memory.fault_byte_count +</div><div class='add'>+			       pfault-&gt;memory.prefetch_after_byte_count;</div><div class='add'>+	struct mlx5_ib_mkey *mmkey;</div><div class='add'>+	struct mlx5_ib_mr *mr, *child_mr;</div><div class='add'>+	int ret = 0;</div><div class='add'>+</div><div class='add'>+	mmkey = find_odp_mkey(dev, pfault-&gt;memory.mkey);</div><div class='add'>+	if (IS_ERR(mmkey))</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	switch (mmkey-&gt;type) {</div><div class='add'>+	case MLX5_MKEY_IMPLICIT_CHILD:</div><div class='add'>+		child_mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);</div><div class='add'>+		mr = child_mr-&gt;parent;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_MKEY_NULL:</div><div class='add'>+		mr = container_of(mmkey, struct mlx5_ib_mr, null_mmkey);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	/* If prefetch fails, handle only demanded page fault */</div><div class='add'>+	ret = pagefault_mr(mr, prefetch_va, prefetch_size, NULL, 0, true);</div><div class='add'>+	if (ret &lt; 0) {</div><div class='add'>+		ret = pagefault_mr(mr, pfault-&gt;memory.va,</div><div class='add'>+				   pfault-&gt;memory.fault_byte_count, NULL, 0,</div><div class='add'>+				   true);</div><div class='add'>+		if (ret &lt; 0)</div><div class='add'>+			goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mlx5_update_odp_stats_with_handled(mr, faults, ret);</div><div class='add'>+	mlx5r_deref_odp_mkey(mmkey);</div><div class='add'>+</div><div class='add'>+	if (pfault-&gt;memory.flags &amp; MLX5_MEMORY_PAGE_FAULT_FLAGS_LAST)</div><div class='add'>+		mlx5_ib_page_fault_resume(dev, pfault, 0);</div><div class='add'>+</div><div class='add'>+	mlx5_ib_dbg(</div><div class='add'>+		dev,</div><div class='add'>+		"PAGE FAULT completed %s. token 0x%llx, mkey: 0x%x, va: 0x%llx, byte_count: 0x%x\n",</div><div class='add'>+		pfault-&gt;memory.flags &amp; MLX5_MEMORY_PAGE_FAULT_FLAGS_LAST ?</div><div class='add'>+			"" :</div><div class='add'>+			"without resume cmd",</div><div class='add'>+		pfault-&gt;token, pfault-&gt;memory.mkey, pfault-&gt;memory.va,</div><div class='add'>+		pfault-&gt;memory.fault_byte_count);</div><div class='add'>+</div><div class='add'>+	return;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	if (!IS_ERR(mmkey))</div><div class='add'>+		mlx5r_deref_odp_mkey(mmkey);</div><div class='add'>+	mlx5_ib_page_fault_resume(dev, pfault, 1);</div><div class='add'>+	mlx5_ib_dbg(</div><div class='add'>+		dev,</div><div class='add'>+		"PAGE FAULT error. token 0x%llx, mkey: 0x%x, va: 0x%llx, byte_count: 0x%x, err: %d\n",</div><div class='add'>+		pfault-&gt;token, pfault-&gt;memory.mkey, pfault-&gt;memory.va,</div><div class='add'>+		pfault-&gt;memory.fault_byte_count, ret);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault)</div><div class='ctx'> {</div><div class='ctx'> 	u8 event_subtype = pfault-&gt;event_subtype;</div><div class='hunk'>@@ -1383,6 +1612,9 @@ static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfaul</div><div class='ctx'> 	case MLX5_PFAULT_SUBTYPE_RDMA:</div><div class='ctx'> 		mlx5_ib_mr_rdma_pfault_handler(dev, pfault);</div><div class='ctx'> 		break;</div><div class='add'>+	case MLX5_PFAULT_SUBTYPE_MEMORY:</div><div class='add'>+		mlx5_ib_mr_memory_pfault_handler(dev, pfault);</div><div class='add'>+		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		mlx5_ib_err(dev, "Invalid page fault event subtype: 0x%x\n",</div><div class='ctx'> 			    event_subtype);</div><div class='hunk'>@@ -1401,6 +1633,7 @@ static void mlx5_ib_eqe_pf_action(struct work_struct *work)</div><div class='ctx'> 	mempool_free(pfault, eq-&gt;pool);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+#define MEMORY_SCHEME_PAGE_FAULT_GRANULARITY 4096</div><div class='ctx'> static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_eqe_page_fault *pf_eqe;</div><div class='hunk'>@@ -1417,15 +1650,12 @@ static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq)</div><div class='ctx'> </div><div class='ctx'> 		pf_eqe = &amp;eqe-&gt;data.page_fault;</div><div class='ctx'> 		pfault-&gt;event_subtype = eqe-&gt;sub_type;</div><div class='del'>-		pfault-&gt;bytes_committed = be32_to_cpu(pf_eqe-&gt;bytes_committed);</div><div class='del'>-</div><div class='del'>-		mlx5_ib_dbg(eq-&gt;dev,</div><div class='del'>-			    "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x\n",</div><div class='del'>-			    eqe-&gt;sub_type, pfault-&gt;bytes_committed);</div><div class='ctx'> </div><div class='ctx'> 		switch (eqe-&gt;sub_type) {</div><div class='ctx'> 		case MLX5_PFAULT_SUBTYPE_RDMA:</div><div class='ctx'> 			/* RDMA based event */</div><div class='add'>+			pfault-&gt;bytes_committed =</div><div class='add'>+				be32_to_cpu(pf_eqe-&gt;rdma.bytes_committed);</div><div class='ctx'> 			pfault-&gt;type =</div><div class='ctx'> 				be32_to_cpu(pf_eqe-&gt;rdma.pftype_token) &gt;&gt; 24;</div><div class='ctx'> 			pfault-&gt;token =</div><div class='hunk'>@@ -1439,10 +1669,12 @@ static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq)</div><div class='ctx'> 				be32_to_cpu(pf_eqe-&gt;rdma.rdma_op_len);</div><div class='ctx'> 			pfault-&gt;rdma.rdma_va =</div><div class='ctx'> 				be64_to_cpu(pf_eqe-&gt;rdma.rdma_va);</div><div class='del'>-			mlx5_ib_dbg(eq-&gt;dev,</div><div class='del'>-				    "PAGE_FAULT: type:0x%x, token: 0x%06x, r_key: 0x%08x\n",</div><div class='del'>-				    pfault-&gt;type, pfault-&gt;token,</div><div class='del'>-				    pfault-&gt;rdma.r_key);</div><div class='add'>+			mlx5_ib_dbg(</div><div class='add'>+				eq-&gt;dev,</div><div class='add'>+				"PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x, type:0x%x, token: 0x%06llx, r_key: 0x%08x\n",</div><div class='add'>+				eqe-&gt;sub_type, pfault-&gt;bytes_committed,</div><div class='add'>+				pfault-&gt;type, pfault-&gt;token,</div><div class='add'>+				pfault-&gt;rdma.r_key);</div><div class='ctx'> 			mlx5_ib_dbg(eq-&gt;dev,</div><div class='ctx'> 				    "PAGE_FAULT: rdma_op_len: 0x%08x, rdma_va: 0x%016llx\n",</div><div class='ctx'> 				    pfault-&gt;rdma.rdma_op_len,</div><div class='hunk'>@@ -1451,6 +1683,8 @@ static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq)</div><div class='ctx'> </div><div class='ctx'> 		case MLX5_PFAULT_SUBTYPE_WQE:</div><div class='ctx'> 			/* WQE based event */</div><div class='add'>+			pfault-&gt;bytes_committed =</div><div class='add'>+				be32_to_cpu(pf_eqe-&gt;wqe.bytes_committed);</div><div class='ctx'> 			pfault-&gt;type =</div><div class='ctx'> 				(be32_to_cpu(pf_eqe-&gt;wqe.pftype_wq) &gt;&gt; 24) &amp; 0x7;</div><div class='ctx'> 			pfault-&gt;token =</div><div class='hunk'>@@ -1462,11 +1696,47 @@ static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq)</div><div class='ctx'> 				be16_to_cpu(pf_eqe-&gt;wqe.wqe_index);</div><div class='ctx'> 			pfault-&gt;wqe.packet_size =</div><div class='ctx'> 				be16_to_cpu(pf_eqe-&gt;wqe.packet_length);</div><div class='del'>-			mlx5_ib_dbg(eq-&gt;dev,</div><div class='del'>-				    "PAGE_FAULT: type:0x%x, token: 0x%06x, wq_num: 0x%06x, wqe_index: 0x%04x\n",</div><div class='del'>-				    pfault-&gt;type, pfault-&gt;token,</div><div class='del'>-				    pfault-&gt;wqe.wq_num,</div><div class='del'>-				    pfault-&gt;wqe.wqe_index);</div><div class='add'>+			mlx5_ib_dbg(</div><div class='add'>+				eq-&gt;dev,</div><div class='add'>+				"PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x, type:0x%x, token: 0x%06llx, wq_num: 0x%06x, wqe_index: 0x%04x\n",</div><div class='add'>+				eqe-&gt;sub_type, pfault-&gt;bytes_committed,</div><div class='add'>+				pfault-&gt;type, pfault-&gt;token, pfault-&gt;wqe.wq_num,</div><div class='add'>+				pfault-&gt;wqe.wqe_index);</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		case MLX5_PFAULT_SUBTYPE_MEMORY:</div><div class='add'>+			/* Memory based event */</div><div class='add'>+			pfault-&gt;bytes_committed = 0;</div><div class='add'>+			pfault-&gt;token =</div><div class='add'>+				be32_to_cpu(pf_eqe-&gt;memory.token31_0) |</div><div class='add'>+				((u64)be16_to_cpu(pf_eqe-&gt;memory.token47_32)</div><div class='add'>+				 &lt;&lt; 32);</div><div class='add'>+			pfault-&gt;memory.va = be64_to_cpu(pf_eqe-&gt;memory.va);</div><div class='add'>+			pfault-&gt;memory.mkey = be32_to_cpu(pf_eqe-&gt;memory.mkey);</div><div class='add'>+			pfault-&gt;memory.fault_byte_count = (be32_to_cpu(</div><div class='add'>+				pf_eqe-&gt;memory.demand_fault_pages) &gt;&gt; 12) *</div><div class='add'>+				MEMORY_SCHEME_PAGE_FAULT_GRANULARITY;</div><div class='add'>+			pfault-&gt;memory.prefetch_before_byte_count =</div><div class='add'>+				be16_to_cpu(</div><div class='add'>+					pf_eqe-&gt;memory.pre_demand_fault_pages) *</div><div class='add'>+				MEMORY_SCHEME_PAGE_FAULT_GRANULARITY;</div><div class='add'>+			pfault-&gt;memory.prefetch_after_byte_count =</div><div class='add'>+				be16_to_cpu(</div><div class='add'>+					pf_eqe-&gt;memory.post_demand_fault_pages) *</div><div class='add'>+				MEMORY_SCHEME_PAGE_FAULT_GRANULARITY;</div><div class='add'>+			pfault-&gt;memory.flags = pf_eqe-&gt;memory.flags;</div><div class='add'>+			mlx5_ib_dbg(</div><div class='add'>+				eq-&gt;dev,</div><div class='add'>+				"PAGE_FAULT: subtype: 0x%02x, token: 0x%06llx, mkey: 0x%06x, fault_byte_count: 0x%06x, va: 0x%016llx, flags: 0x%02x\n",</div><div class='add'>+				eqe-&gt;sub_type, pfault-&gt;token,</div><div class='add'>+				pfault-&gt;memory.mkey,</div><div class='add'>+				pfault-&gt;memory.fault_byte_count,</div><div class='add'>+				pfault-&gt;memory.va, pfault-&gt;memory.flags);</div><div class='add'>+			mlx5_ib_dbg(</div><div class='add'>+				eq-&gt;dev,</div><div class='add'>+				"PAGE_FAULT: prefetch size: before: 0x%06x, after 0x%06x\n",</div><div class='add'>+				pfault-&gt;memory.prefetch_before_byte_count,</div><div class='add'>+				pfault-&gt;memory.prefetch_after_byte_count);</div><div class='ctx'> 			break;</div><div class='ctx'> </div><div class='ctx'> 		default:</div><div class='hunk'>@@ -1532,20 +1802,24 @@ enum {</div><div class='ctx'> 	MLX5_IB_NUM_PF_DRAIN	= 64,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static int</div><div class='del'>-mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)</div><div class='add'>+int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_eq_param param = {};</div><div class='del'>-	int err;</div><div class='add'>+	int err = 0;</div><div class='ctx'> </div><div class='add'>+	mutex_lock(&amp;dev-&gt;odp_eq_mutex);</div><div class='add'>+	if (eq-&gt;core)</div><div class='add'>+		goto unlock;</div><div class='ctx'> 	INIT_WORK(&amp;eq-&gt;work, mlx5_ib_eq_pf_action);</div><div class='ctx'> 	spin_lock_init(&amp;eq-&gt;lock);</div><div class='ctx'> 	eq-&gt;dev = dev;</div><div class='ctx'> </div><div class='ctx'> 	eq-&gt;pool = mempool_create_kmalloc_pool(MLX5_IB_NUM_PF_DRAIN,</div><div class='ctx'> 					       sizeof(struct mlx5_pagefault));</div><div class='del'>-	if (!eq-&gt;pool)</div><div class='del'>-		return -ENOMEM;</div><div class='add'>+	if (!eq-&gt;pool) {</div><div class='add'>+		err = -ENOMEM;</div><div class='add'>+		goto unlock;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	eq-&gt;wq = alloc_workqueue("mlx5_ib_page_fault",</div><div class='ctx'> 				 WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM,</div><div class='hunk'>@@ -1557,10 +1831,9 @@ mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)</div><div class='ctx'> </div><div class='ctx'> 	eq-&gt;irq_nb.notifier_call = mlx5_ib_eq_pf_int;</div><div class='ctx'> 	param = (struct mlx5_eq_param) {</div><div class='del'>-		.irq_index = 0,</div><div class='del'>-		.mask = 1 &lt;&lt; MLX5_EVENT_TYPE_PAGE_FAULT,</div><div class='ctx'> 		.nent = MLX5_IB_NUM_PF_EQE,</div><div class='ctx'> 	};</div><div class='add'>+	param.mask[0] = 1ull &lt;&lt; MLX5_EVENT_TYPE_PAGE_FAULT;</div><div class='ctx'> 	eq-&gt;core = mlx5_eq_create_generic(dev-&gt;mdev, &amp;param);</div><div class='ctx'> 	if (IS_ERR(eq-&gt;core)) {</div><div class='ctx'> 		err = PTR_ERR(eq-&gt;core);</div><div class='hunk'>@@ -1572,21 +1845,27 @@ mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)</div><div class='ctx'> 		goto err_eq;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	mutex_unlock(&amp;dev-&gt;odp_eq_mutex);</div><div class='ctx'> 	return 0;</div><div class='ctx'> err_eq:</div><div class='ctx'> 	mlx5_eq_destroy_generic(dev-&gt;mdev, eq-&gt;core);</div><div class='ctx'> err_wq:</div><div class='add'>+	eq-&gt;core = NULL;</div><div class='ctx'> 	destroy_workqueue(eq-&gt;wq);</div><div class='ctx'> err_mempool:</div><div class='ctx'> 	mempool_destroy(eq-&gt;pool);</div><div class='add'>+unlock:</div><div class='add'>+	mutex_unlock(&amp;dev-&gt;odp_eq_mutex);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int</div><div class='del'>-mlx5_ib_destroy_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)</div><div class='add'>+mlx5_ib_odp_destroy_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)</div><div class='ctx'> {</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='add'>+	if (!eq-&gt;core)</div><div class='add'>+		return 0;</div><div class='ctx'> 	mlx5_eq_disable(dev-&gt;mdev, eq-&gt;core, &amp;eq-&gt;irq_nb);</div><div class='ctx'> 	err = mlx5_eq_destroy_generic(dev-&gt;mdev, eq-&gt;core);</div><div class='ctx'> 	cancel_work_sync(&amp;eq-&gt;work);</div><div class='hunk'>@@ -1596,30 +1875,23 @@ mlx5_ib_destroy_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent)</div><div class='add'>+int mlx5_odp_init_mkey_cache(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	if (!(ent-&gt;dev-&gt;odp_caps.general_caps &amp; IB_ODP_SUPPORT_IMPLICIT))</div><div class='del'>-		return;</div><div class='add'>+	struct mlx5r_cache_rb_key rb_key = {</div><div class='add'>+		.access_mode = MLX5_MKC_ACCESS_MODE_KSM,</div><div class='add'>+		.ndescs = mlx5_imr_ksm_entries,</div><div class='add'>+		.ph = MLX5_IB_NO_PH,</div><div class='add'>+	};</div><div class='add'>+	struct mlx5_cache_ent *ent;</div><div class='ctx'> </div><div class='del'>-	switch (ent-&gt;order - 2) {</div><div class='del'>-	case MLX5_IMR_MTT_CACHE_ENTRY:</div><div class='del'>-		ent-&gt;page = PAGE_SHIFT;</div><div class='del'>-		ent-&gt;xlt = MLX5_IMR_MTT_ENTRIES *</div><div class='del'>-			   sizeof(struct mlx5_mtt) /</div><div class='del'>-			   MLX5_IB_UMR_OCTOWORD;</div><div class='del'>-		ent-&gt;access_mode = MLX5_MKC_ACCESS_MODE_MTT;</div><div class='del'>-		ent-&gt;limit = 0;</div><div class='del'>-		break;</div><div class='add'>+	if (!(dev-&gt;odp_caps.general_caps &amp; IB_ODP_SUPPORT_IMPLICIT))</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-	case MLX5_IMR_KSM_CACHE_ENTRY:</div><div class='del'>-		ent-&gt;page = MLX5_KSM_PAGE_SHIFT;</div><div class='del'>-		ent-&gt;xlt = mlx5_imr_ksm_entries *</div><div class='del'>-			   sizeof(struct mlx5_klm) /</div><div class='del'>-			   MLX5_IB_UMR_OCTOWORD;</div><div class='del'>-		ent-&gt;access_mode = MLX5_MKC_ACCESS_MODE_KSM;</div><div class='del'>-		ent-&gt;limit = 0;</div><div class='del'>-		break;</div><div class='del'>-	}</div><div class='add'>+	ent = mlx5r_cache_create_ent_locked(dev, rb_key, true);</div><div class='add'>+	if (IS_ERR(ent))</div><div class='add'>+		return PTR_ERR(ent);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static const struct ib_device_ops mlx5_ib_dev_odp_ops = {</div><div class='hunk'>@@ -1628,197 +1900,212 @@ static const struct ib_device_ops mlx5_ib_dev_odp_ops = {</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	int ret = 0;</div><div class='del'>-</div><div class='del'>-	if (dev-&gt;odp_caps.general_caps &amp; IB_ODP_SUPPORT)</div><div class='del'>-		ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;mlx5_ib_dev_odp_ops);</div><div class='del'>-</div><div class='del'>-	if (dev-&gt;odp_caps.general_caps &amp; IB_ODP_SUPPORT_IMPLICIT) {</div><div class='del'>-		ret = mlx5_cmd_null_mkey(dev-&gt;mdev, &amp;dev-&gt;null_mkey);</div><div class='del'>-		if (ret) {</div><div class='del'>-			mlx5_ib_err(dev, "Error getting null_mkey %d\n", ret);</div><div class='del'>-			return ret;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='add'>+	internal_fill_odp_caps(dev);</div><div class='ctx'> </div><div class='del'>-	if (!MLX5_CAP_GEN(dev-&gt;mdev, pg))</div><div class='del'>-		return ret;</div><div class='add'>+	if (!(dev-&gt;odp_caps.general_caps &amp; IB_ODP_SUPPORT))</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-	ret = mlx5_ib_create_pf_eq(dev, &amp;dev-&gt;odp_pf_eq);</div><div class='add'>+	ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;mlx5_ib_dev_odp_ops);</div><div class='ctx'> </div><div class='del'>-	return ret;</div><div class='add'>+	mutex_init(&amp;dev-&gt;odp_eq_mutex);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev)</div><div class='ctx'> {</div><div class='del'>-	if (!MLX5_CAP_GEN(dev-&gt;mdev, pg))</div><div class='add'>+	if (!(dev-&gt;odp_caps.general_caps &amp; IB_ODP_SUPPORT))</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_destroy_pf_eq(dev, &amp;dev-&gt;odp_pf_eq);</div><div class='add'>+	mlx5_ib_odp_destroy_eq(dev, &amp;dev-&gt;odp_pf_eq);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_odp_init(void)</div><div class='ctx'> {</div><div class='add'>+	u32 log_va_pages = ilog2(TASK_SIZE) - PAGE_SHIFT;</div><div class='add'>+	u8 mlx5_imr_mtt_bits;</div><div class='add'>+</div><div class='add'>+	/* 48 is default ARM64 VA space and covers X86 4-level paging which is 47 */</div><div class='add'>+	if (log_va_pages &lt;= 48 - PAGE_SHIFT)</div><div class='add'>+		mlx5_imr_mtt_shift = 30;</div><div class='add'>+	/* 56 is x86-64, 5-level paging */</div><div class='add'>+	else if (log_va_pages &lt;= 56 - PAGE_SHIFT)</div><div class='add'>+		mlx5_imr_mtt_shift = 34;</div><div class='add'>+	else</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	mlx5_imr_mtt_size = BIT_ULL(mlx5_imr_mtt_shift);</div><div class='add'>+	mlx5_imr_mtt_bits = mlx5_imr_mtt_shift - PAGE_SHIFT;</div><div class='add'>+	mlx5_imr_mtt_entries = BIT_ULL(mlx5_imr_mtt_bits);</div><div class='ctx'> 	mlx5_imr_ksm_entries = BIT_ULL(get_order(TASK_SIZE) -</div><div class='del'>-				       MLX5_IMR_MTT_BITS);</div><div class='add'>+				       mlx5_imr_mtt_bits);</div><div class='ctx'> </div><div class='add'>+	mlx5_imr_ksm_page_shift = mlx5_imr_mtt_shift;</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> struct prefetch_mr_work {</div><div class='ctx'> 	struct work_struct work;</div><div class='del'>-	struct ib_pd *pd;</div><div class='ctx'> 	u32 pf_flags;</div><div class='ctx'> 	u32 num_sge;</div><div class='del'>-	struct ib_sge sg_list[0];</div><div class='add'>+	struct {</div><div class='add'>+		u64 io_virt;</div><div class='add'>+		struct mlx5_ib_mr *mr;</div><div class='add'>+		size_t length;</div><div class='add'>+	} frags[];</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static void num_pending_prefetch_dec(struct mlx5_ib_dev *dev,</div><div class='del'>-				     struct ib_sge *sg_list, u32 num_sge,</div><div class='del'>-				     u32 from)</div><div class='add'>+static void destroy_prefetch_work(struct prefetch_mr_work *work)</div><div class='ctx'> {</div><div class='ctx'> 	u32 i;</div><div class='del'>-	int srcu_key;</div><div class='ctx'> </div><div class='del'>-	srcu_key = srcu_read_lock(&amp;dev-&gt;mr_srcu);</div><div class='add'>+	for (i = 0; i &lt; work-&gt;num_sge; ++i)</div><div class='add'>+		mlx5r_deref_odp_mkey(&amp;work-&gt;frags[i].mr-&gt;mmkey);</div><div class='ctx'> </div><div class='del'>-	for (i = from; i &lt; num_sge; ++i) {</div><div class='del'>-		struct mlx5_core_mkey *mmkey;</div><div class='del'>-		struct mlx5_ib_mr *mr;</div><div class='del'>-</div><div class='del'>-		mmkey = xa_load(&amp;dev-&gt;mdev-&gt;priv.mkey_table,</div><div class='del'>-				mlx5_base_mkey(sg_list[i].lkey));</div><div class='del'>-		mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);</div><div class='del'>-		atomic_dec(&amp;mr-&gt;num_pending_prefetch);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	srcu_read_unlock(&amp;dev-&gt;mr_srcu, srcu_key);</div><div class='add'>+	kvfree(work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static bool num_pending_prefetch_inc(struct ib_pd *pd,</div><div class='del'>-				     struct ib_sge *sg_list, u32 num_sge)</div><div class='add'>+static struct mlx5_ib_mr *</div><div class='add'>+get_prefetchable_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice,</div><div class='add'>+		    u32 lkey)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='del'>-	bool ret = true;</div><div class='del'>-	u32 i;</div><div class='add'>+	struct mlx5_ib_mr *mr = NULL;</div><div class='add'>+	struct mlx5_ib_mkey *mmkey;</div><div class='ctx'> </div><div class='del'>-	for (i = 0; i &lt; num_sge; ++i) {</div><div class='del'>-		struct mlx5_core_mkey *mmkey;</div><div class='del'>-		struct mlx5_ib_mr *mr;</div><div class='add'>+	xa_lock(&amp;dev-&gt;odp_mkeys);</div><div class='add'>+	mmkey = xa_load(&amp;dev-&gt;odp_mkeys, mlx5_base_mkey(lkey));</div><div class='add'>+	if (!mmkey || mmkey-&gt;key != lkey) {</div><div class='add'>+		mr = ERR_PTR(-ENOENT);</div><div class='add'>+		goto end;</div><div class='add'>+	}</div><div class='add'>+	if (mmkey-&gt;type != MLX5_MKEY_MR) {</div><div class='add'>+		mr = ERR_PTR(-EINVAL);</div><div class='add'>+		goto end;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		mmkey = xa_load(&amp;dev-&gt;mdev-&gt;priv.mkey_table,</div><div class='del'>-				mlx5_base_mkey(sg_list[i].lkey));</div><div class='del'>-		if (!mmkey || mmkey-&gt;key != sg_list[i].lkey) {</div><div class='del'>-			ret = false;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='add'>+	mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);</div><div class='ctx'> </div><div class='del'>-		if (mmkey-&gt;type != MLX5_MKEY_MR) {</div><div class='del'>-			ret = false;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='add'>+	if (mr-&gt;ibmr.pd != pd) {</div><div class='add'>+		mr = ERR_PTR(-EPERM);</div><div class='add'>+		goto end;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);</div><div class='add'>+	/* prefetch with write-access must be supported by the MR */</div><div class='add'>+	if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE &amp;&amp;</div><div class='add'>+	    !mr-&gt;umem-&gt;writable) {</div><div class='add'>+		mr = ERR_PTR(-EPERM);</div><div class='add'>+		goto end;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		if (mr-&gt;ibmr.pd != pd) {</div><div class='del'>-			ret = false;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='add'>+	refcount_inc(&amp;mmkey-&gt;usecount);</div><div class='add'>+end:</div><div class='add'>+	xa_unlock(&amp;dev-&gt;odp_mkeys);</div><div class='add'>+	return mr;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		if (!mr-&gt;live) {</div><div class='del'>-			ret = false;</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='add'>+static void mlx5_ib_prefetch_mr_work(struct work_struct *w)</div><div class='add'>+{</div><div class='add'>+	struct prefetch_mr_work *work =</div><div class='add'>+		container_of(w, struct prefetch_mr_work, work);</div><div class='add'>+	u32 bytes_mapped = 0;</div><div class='add'>+	int ret;</div><div class='add'>+	u32 i;</div><div class='ctx'> </div><div class='del'>-		atomic_inc(&amp;mr-&gt;num_pending_prefetch);</div><div class='add'>+	/* We rely on IB/core that work is executed if we have num_sge != 0 only. */</div><div class='add'>+	WARN_ON(!work-&gt;num_sge);</div><div class='add'>+	for (i = 0; i &lt; work-&gt;num_sge; ++i) {</div><div class='add'>+		ret = pagefault_mr(work-&gt;frags[i].mr, work-&gt;frags[i].io_virt,</div><div class='add'>+				   work-&gt;frags[i].length, &amp;bytes_mapped,</div><div class='add'>+				   work-&gt;pf_flags, false);</div><div class='add'>+		if (ret &lt;= 0)</div><div class='add'>+			continue;</div><div class='add'>+		mlx5_update_odp_stats(work-&gt;frags[i].mr, prefetch, ret);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!ret)</div><div class='del'>-		num_pending_prefetch_dec(dev, sg_list, i, 0);</div><div class='del'>-</div><div class='del'>-	return ret;</div><div class='add'>+	destroy_prefetch_work(work);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd, u32 pf_flags,</div><div class='del'>-				    struct ib_sge *sg_list, u32 num_sge)</div><div class='add'>+static int init_prefetch_work(struct ib_pd *pd,</div><div class='add'>+			       enum ib_uverbs_advise_mr_advice advice,</div><div class='add'>+			       u32 pf_flags, struct prefetch_mr_work *work,</div><div class='add'>+			       struct ib_sge *sg_list, u32 num_sge)</div><div class='ctx'> {</div><div class='ctx'> 	u32 i;</div><div class='del'>-	int ret = 0;</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='add'>+</div><div class='add'>+	INIT_WORK(&amp;work-&gt;work, mlx5_ib_prefetch_mr_work);</div><div class='add'>+	work-&gt;pf_flags = pf_flags;</div><div class='ctx'> </div><div class='ctx'> 	for (i = 0; i &lt; num_sge; ++i) {</div><div class='del'>-		struct ib_sge *sg = &amp;sg_list[i];</div><div class='del'>-		int bytes_committed = 0;</div><div class='add'>+		struct mlx5_ib_mr *mr;</div><div class='ctx'> </div><div class='del'>-		ret = pagefault_single_data_segment(dev, pd, sg-&gt;lkey, sg-&gt;addr,</div><div class='del'>-						    sg-&gt;length,</div><div class='del'>-						    &amp;bytes_committed, NULL,</div><div class='del'>-						    pf_flags);</div><div class='del'>-		if (ret &lt; 0)</div><div class='del'>-			break;</div><div class='add'>+		mr = get_prefetchable_mr(pd, advice, sg_list[i].lkey);</div><div class='add'>+		if (IS_ERR(mr)) {</div><div class='add'>+			work-&gt;num_sge = i;</div><div class='add'>+			return PTR_ERR(mr);</div><div class='add'>+		}</div><div class='add'>+		work-&gt;frags[i].io_virt = sg_list[i].addr;</div><div class='add'>+		work-&gt;frags[i].length = sg_list[i].length;</div><div class='add'>+		work-&gt;frags[i].mr = mr;</div><div class='ctx'> 	}</div><div class='del'>-</div><div class='del'>-	return ret &lt; 0 ? ret : 0;</div><div class='add'>+	work-&gt;num_sge = num_sge;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void mlx5_ib_prefetch_mr_work(struct work_struct *work)</div><div class='add'>+static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd,</div><div class='add'>+				    enum ib_uverbs_advise_mr_advice advice,</div><div class='add'>+				    u32 pf_flags, struct ib_sge *sg_list,</div><div class='add'>+				    u32 num_sge)</div><div class='ctx'> {</div><div class='del'>-	struct prefetch_mr_work *w =</div><div class='del'>-		container_of(work, struct prefetch_mr_work, work);</div><div class='add'>+	u32 bytes_mapped = 0;</div><div class='add'>+	int ret = 0;</div><div class='add'>+	u32 i;</div><div class='ctx'> </div><div class='del'>-	if (ib_device_try_get(w-&gt;pd-&gt;device)) {</div><div class='del'>-		mlx5_ib_prefetch_sg_list(w-&gt;pd, w-&gt;pf_flags, w-&gt;sg_list,</div><div class='del'>-					 w-&gt;num_sge);</div><div class='del'>-		ib_device_put(w-&gt;pd-&gt;device);</div><div class='add'>+	for (i = 0; i &lt; num_sge; ++i) {</div><div class='add'>+		struct mlx5_ib_mr *mr;</div><div class='add'>+</div><div class='add'>+		mr = get_prefetchable_mr(pd, advice, sg_list[i].lkey);</div><div class='add'>+		if (IS_ERR(mr))</div><div class='add'>+			return PTR_ERR(mr);</div><div class='add'>+		ret = pagefault_mr(mr, sg_list[i].addr, sg_list[i].length,</div><div class='add'>+				   &amp;bytes_mapped, pf_flags, false);</div><div class='add'>+		if (ret &lt; 0) {</div><div class='add'>+			mlx5r_deref_odp_mkey(&amp;mr-&gt;mmkey);</div><div class='add'>+			return ret;</div><div class='add'>+		}</div><div class='add'>+		mlx5_update_odp_stats(mr, prefetch, ret);</div><div class='add'>+		mlx5r_deref_odp_mkey(&amp;mr-&gt;mmkey);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	num_pending_prefetch_dec(to_mdev(w-&gt;pd-&gt;device), w-&gt;sg_list,</div><div class='del'>-				 w-&gt;num_sge, 0);</div><div class='del'>-	kfree(w);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,</div><div class='ctx'> 			       enum ib_uverbs_advise_mr_advice advice,</div><div class='ctx'> 			       u32 flags, struct ib_sge *sg_list, u32 num_sge)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(pd-&gt;device);</div><div class='del'>-	u32 pf_flags = MLX5_PF_FLAGS_PREFETCH;</div><div class='add'>+	u32 pf_flags = 0;</div><div class='ctx'> 	struct prefetch_mr_work *work;</div><div class='del'>-	bool valid_req;</div><div class='del'>-	int srcu_key;</div><div class='add'>+	int rc;</div><div class='ctx'> </div><div class='ctx'> 	if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH)</div><div class='ctx'> 		pf_flags |= MLX5_PF_FLAGS_DOWNGRADE;</div><div class='ctx'> </div><div class='add'>+	if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT)</div><div class='add'>+		pf_flags |= MLX5_PF_FLAGS_SNAPSHOT;</div><div class='add'>+</div><div class='ctx'> 	if (flags &amp; IB_UVERBS_ADVISE_MR_FLAG_FLUSH)</div><div class='del'>-		return mlx5_ib_prefetch_sg_list(pd, pf_flags, sg_list,</div><div class='add'>+		return mlx5_ib_prefetch_sg_list(pd, advice, pf_flags, sg_list,</div><div class='ctx'> 						num_sge);</div><div class='ctx'> </div><div class='del'>-	work = kvzalloc(struct_size(work, sg_list, num_sge), GFP_KERNEL);</div><div class='add'>+	work = kvzalloc(struct_size(work, frags, num_sge), GFP_KERNEL);</div><div class='ctx'> 	if (!work)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	memcpy(work-&gt;sg_list, sg_list, num_sge * sizeof(struct ib_sge));</div><div class='del'>-</div><div class='del'>-	/* It is guaranteed that the pd when work is executed is the pd when</div><div class='del'>-	 * work was queued since pd can't be destroyed while it holds MRs and</div><div class='del'>-	 * destroying a MR leads to flushing the workquque</div><div class='del'>-	 */</div><div class='del'>-	work-&gt;pd = pd;</div><div class='del'>-	work-&gt;pf_flags = pf_flags;</div><div class='del'>-	work-&gt;num_sge = num_sge;</div><div class='del'>-</div><div class='del'>-	INIT_WORK(&amp;work-&gt;work, mlx5_ib_prefetch_mr_work);</div><div class='del'>-</div><div class='del'>-	srcu_key = srcu_read_lock(&amp;dev-&gt;mr_srcu);</div><div class='del'>-</div><div class='del'>-	valid_req = num_pending_prefetch_inc(pd, sg_list, num_sge);</div><div class='del'>-	if (valid_req)</div><div class='del'>-		queue_work(system_unbound_wq, &amp;work-&gt;work);</div><div class='del'>-	else</div><div class='del'>-		kfree(work);</div><div class='del'>-</div><div class='del'>-	srcu_read_unlock(&amp;dev-&gt;mr_srcu, srcu_key);</div><div class='del'>-</div><div class='del'>-	return valid_req ? 0 : -EINVAL;</div><div class='add'>+	rc = init_prefetch_work(pd, advice, pf_flags, work, sg_list, num_sge);</div><div class='add'>+	if (rc) {</div><div class='add'>+		destroy_prefetch_work(work);</div><div class='add'>+		return rc;</div><div class='add'>+	}</div><div class='add'>+	queue_work(system_dfl_wq, &amp;work-&gt;work);</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/qos.c b/drivers/infiniband/hw/mlx5/qos.c<br/>new file mode 100644<br/>index 000000000000..dce92554142a<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/qos.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/qos.c</a></div><div class='hunk'>@@ -0,0 +1,133 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2020, Mellanox Technologies inc.  All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;rdma/uverbs_ioctl.h&gt;</div><div class='add'>+#include &lt;rdma/mlx5_user_ioctl_cmds.h&gt;</div><div class='add'>+#include &lt;rdma/mlx5_user_ioctl_verbs.h&gt;</div><div class='add'>+#include &lt;linux/mlx5/driver.h&gt;</div><div class='add'>+#include "mlx5_ib.h"</div><div class='add'>+</div><div class='add'>+#define UVERBS_MODULE_NAME mlx5_ib</div><div class='add'>+#include &lt;rdma/uverbs_named_ioctl.h&gt;</div><div class='add'>+</div><div class='add'>+static bool pp_is_supported(struct ib_device *device)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='add'>+</div><div class='add'>+	return (MLX5_CAP_GEN(dev-&gt;mdev, qos) &amp;&amp;</div><div class='add'>+		MLX5_CAP_QOS(dev-&gt;mdev, packet_pacing) &amp;&amp;</div><div class='add'>+		MLX5_CAP_QOS(dev-&gt;mdev, packet_pacing_uid));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int UVERBS_HANDLER(MLX5_IB_METHOD_PP_OBJ_ALLOC)(</div><div class='add'>+	struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	u8 rl_raw[MLX5_ST_SZ_BYTES(set_pp_rate_limit_context)] = {};</div><div class='add'>+	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,</div><div class='add'>+		MLX5_IB_ATTR_PP_OBJ_ALLOC_HANDLE);</div><div class='add'>+	struct mlx5_ib_dev *dev;</div><div class='add'>+	struct mlx5_ib_ucontext *c;</div><div class='add'>+	struct mlx5_ib_pp *pp_entry;</div><div class='add'>+	void *in_ctx;</div><div class='add'>+	u16 uid;</div><div class='add'>+	int inlen;</div><div class='add'>+	u32 flags;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	c = to_mucontext(ib_uverbs_get_ucontext(attrs));</div><div class='add'>+	if (IS_ERR(c))</div><div class='add'>+		return PTR_ERR(c);</div><div class='add'>+</div><div class='add'>+	/* The allocated entry can be used only by a DEVX context */</div><div class='add'>+	if (!c-&gt;devx_uid)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	dev = to_mdev(c-&gt;ibucontext.device);</div><div class='add'>+	pp_entry = kzalloc(sizeof(*pp_entry), GFP_KERNEL);</div><div class='add'>+	if (!pp_entry)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	in_ctx = uverbs_attr_get_alloced_ptr(attrs,</div><div class='add'>+					     MLX5_IB_ATTR_PP_OBJ_ALLOC_CTX);</div><div class='add'>+	inlen = uverbs_attr_get_len(attrs,</div><div class='add'>+				    MLX5_IB_ATTR_PP_OBJ_ALLOC_CTX);</div><div class='add'>+	memcpy(rl_raw, in_ctx, inlen);</div><div class='add'>+	err = uverbs_get_flags32(&amp;flags, attrs,</div><div class='add'>+		MLX5_IB_ATTR_PP_OBJ_ALLOC_FLAGS,</div><div class='add'>+		MLX5_IB_UAPI_PP_ALLOC_FLAGS_DEDICATED_INDEX);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	uid = (flags &amp; MLX5_IB_UAPI_PP_ALLOC_FLAGS_DEDICATED_INDEX) ?</div><div class='add'>+		c-&gt;devx_uid : MLX5_SHARED_RESOURCE_UID;</div><div class='add'>+</div><div class='add'>+	err = mlx5_rl_add_rate_raw(dev-&gt;mdev, rl_raw, uid,</div><div class='add'>+			(flags &amp; MLX5_IB_UAPI_PP_ALLOC_FLAGS_DEDICATED_INDEX),</div><div class='add'>+			&amp;pp_entry-&gt;index);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	pp_entry-&gt;mdev = dev-&gt;mdev;</div><div class='add'>+	uobj-&gt;object = pp_entry;</div><div class='add'>+	uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_PP_OBJ_ALLOC_HANDLE);</div><div class='add'>+</div><div class='add'>+	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_PP_OBJ_ALLOC_INDEX,</div><div class='add'>+			     &amp;pp_entry-&gt;index, sizeof(pp_entry-&gt;index));</div><div class='add'>+	return err;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	kfree(pp_entry);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int pp_obj_cleanup(struct ib_uobject *uobject,</div><div class='add'>+			  enum rdma_remove_reason why,</div><div class='add'>+			  struct uverbs_attr_bundle *attrs)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_pp *pp_entry = uobject-&gt;object;</div><div class='add'>+</div><div class='add'>+	mlx5_rl_remove_rate_raw(pp_entry-&gt;mdev, pp_entry-&gt;index);</div><div class='add'>+	kfree(pp_entry);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD(</div><div class='add'>+	MLX5_IB_METHOD_PP_OBJ_ALLOC,</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_PP_OBJ_ALLOC_HANDLE,</div><div class='add'>+			MLX5_IB_OBJECT_PP,</div><div class='add'>+			UVERBS_ACCESS_NEW,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_IN(</div><div class='add'>+		MLX5_IB_ATTR_PP_OBJ_ALLOC_CTX,</div><div class='add'>+		UVERBS_ATTR_SIZE(1,</div><div class='add'>+			MLX5_ST_SZ_BYTES(set_pp_rate_limit_context)),</div><div class='add'>+		UA_MANDATORY,</div><div class='add'>+		UA_ALLOC_AND_COPY),</div><div class='add'>+	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_PP_OBJ_ALLOC_FLAGS,</div><div class='add'>+			enum mlx5_ib_uapi_pp_alloc_flags,</div><div class='add'>+			UA_MANDATORY),</div><div class='add'>+	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_PP_OBJ_ALLOC_INDEX,</div><div class='add'>+			   UVERBS_ATTR_TYPE(u16),</div><div class='add'>+			   UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_METHOD_DESTROY(</div><div class='add'>+	MLX5_IB_METHOD_PP_OBJ_DESTROY,</div><div class='add'>+	UVERBS_ATTR_IDR(MLX5_IB_ATTR_PP_OBJ_DESTROY_HANDLE,</div><div class='add'>+			MLX5_IB_OBJECT_PP,</div><div class='add'>+			UVERBS_ACCESS_DESTROY,</div><div class='add'>+			UA_MANDATORY));</div><div class='add'>+</div><div class='add'>+DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_PP,</div><div class='add'>+			    UVERBS_TYPE_ALLOC_IDR(pp_obj_cleanup),</div><div class='add'>+			    &amp;UVERBS_METHOD(MLX5_IB_METHOD_PP_OBJ_ALLOC),</div><div class='add'>+			    &amp;UVERBS_METHOD(MLX5_IB_METHOD_PP_OBJ_DESTROY));</div><div class='add'>+</div><div class='add'>+</div><div class='add'>+const struct uapi_definition mlx5_ib_qos_defs[] = {</div><div class='add'>+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(</div><div class='add'>+		MLX5_IB_OBJECT_PP,</div><div class='add'>+		UAPI_DEF_IS_OBJ_SUPPORTED(pp_is_supported)),</div><div class='add'>+	{},</div><div class='add'>+};</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c<br/>index f6623c77443a..69af20790481 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/qp.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/qp.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/qp.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/qp.c</a></div><div class='hunk'>@@ -30,17 +30,19 @@</div><div class='ctx'>  * SOFTWARE.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='add'>+#include &lt;linux/etherdevice.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_umem.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_cache.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_user_verbs.h&gt;</div><div class='add'>+#include &lt;rdma/rdma_counter.h&gt;</div><div class='ctx'> #include &lt;linux/mlx5/fs.h&gt;</div><div class='ctx'> #include "mlx5_ib.h"</div><div class='ctx'> #include "ib_rep.h"</div><div class='add'>+#include "counters.h"</div><div class='ctx'> #include "cmd.h"</div><div class='del'>-</div><div class='del'>-/* not supported currently */</div><div class='del'>-static int wq_signature;</div><div class='add'>+#include "umr.h"</div><div class='add'>+#include "qp.h"</div><div class='add'>+#include "wr.h"</div><div class='ctx'> </div><div class='ctx'> enum {</div><div class='ctx'> 	MLX5_IB_ACK_REQ_FREQ	= 8,</div><div class='hunk'>@@ -53,37 +55,15 @@ enum {</div><div class='ctx'> 	MLX5_IB_LINK_TYPE_ETH		= 1</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-enum {</div><div class='del'>-	MLX5_IB_SQ_STRIDE	= 6,</div><div class='del'>-	MLX5_IB_SQ_UMR_INLINE_THRESHOLD = 64,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-static const u32 mlx5_ib_opcode[] = {</div><div class='del'>-	[IB_WR_SEND]				= MLX5_OPCODE_SEND,</div><div class='del'>-	[IB_WR_LSO]				= MLX5_OPCODE_LSO,</div><div class='del'>-	[IB_WR_SEND_WITH_IMM]			= MLX5_OPCODE_SEND_IMM,</div><div class='del'>-	[IB_WR_RDMA_WRITE]			= MLX5_OPCODE_RDMA_WRITE,</div><div class='del'>-	[IB_WR_RDMA_WRITE_WITH_IMM]		= MLX5_OPCODE_RDMA_WRITE_IMM,</div><div class='del'>-	[IB_WR_RDMA_READ]			= MLX5_OPCODE_RDMA_READ,</div><div class='del'>-	[IB_WR_ATOMIC_CMP_AND_SWP]		= MLX5_OPCODE_ATOMIC_CS,</div><div class='del'>-	[IB_WR_ATOMIC_FETCH_AND_ADD]		= MLX5_OPCODE_ATOMIC_FA,</div><div class='del'>-	[IB_WR_SEND_WITH_INV]			= MLX5_OPCODE_SEND_INVAL,</div><div class='del'>-	[IB_WR_LOCAL_INV]			= MLX5_OPCODE_UMR,</div><div class='del'>-	[IB_WR_REG_MR]				= MLX5_OPCODE_UMR,</div><div class='del'>-	[IB_WR_MASKED_ATOMIC_CMP_AND_SWP]	= MLX5_OPCODE_ATOMIC_MASKED_CS,</div><div class='del'>-	[IB_WR_MASKED_ATOMIC_FETCH_AND_ADD]	= MLX5_OPCODE_ATOMIC_MASKED_FA,</div><div class='del'>-	[MLX5_IB_WR_UMR]			= MLX5_OPCODE_UMR,</div><div class='del'>-};</div><div class='del'>-</div><div class='del'>-struct mlx5_wqe_eth_pad {</div><div class='del'>-	u8 rsvd0[16];</div><div class='del'>-};</div><div class='del'>-</div><div class='ctx'> enum raw_qp_set_mask_map {</div><div class='ctx'> 	MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID		= 1UL &lt;&lt; 0,</div><div class='ctx'> 	MLX5_RAW_QP_RATE_LIMIT			= 1UL &lt;&lt; 1,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+enum {</div><div class='add'>+	MLX5_QP_RM_GO_BACK_N			= 0x1,</div><div class='add'>+};</div><div class='add'>+</div><div class='ctx'> struct mlx5_modify_raw_qp_param {</div><div class='ctx'> 	u16 operation;</div><div class='ctx'> </div><div class='hunk'>@@ -92,9 +72,17 @@ struct mlx5_modify_raw_qp_param {</div><div class='ctx'> 	struct mlx5_rate_limit rl;</div><div class='ctx'> </div><div class='ctx'> 	u8 rq_q_ctr_id;</div><div class='del'>-	u16 port;</div><div class='add'>+	u32 port;</div><div class='ctx'> };</div><div class='ctx'> </div><div class='add'>+struct mlx5_ib_qp_event_work {</div><div class='add'>+	struct work_struct work;</div><div class='add'>+	struct mlx5_core_qp *qp;</div><div class='add'>+	int type;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static struct workqueue_struct *mlx5_ib_qp_event_wq;</div><div class='add'>+</div><div class='ctx'> static void get_cqs(enum ib_qp_type qp_type,</div><div class='ctx'> 		    struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,</div><div class='ctx'> 		    struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq);</div><div class='hunk'>@@ -128,14 +116,10 @@ static int is_sqp(enum ib_qp_type qp_type)</div><div class='ctx'>  *</div><div class='ctx'>  * Return: zero on success, or an error code.</div><div class='ctx'>  */</div><div class='del'>-static int mlx5_ib_read_user_wqe_common(struct ib_umem *umem,</div><div class='del'>-					void *buffer,</div><div class='del'>-					u32 buflen,</div><div class='del'>-					int wqe_index,</div><div class='del'>-					int wq_offset,</div><div class='del'>-					int wq_wqe_cnt,</div><div class='del'>-					int wq_wqe_shift,</div><div class='del'>-					int bcnt,</div><div class='add'>+static int mlx5_ib_read_user_wqe_common(struct ib_umem *umem, void *buffer,</div><div class='add'>+					size_t buflen, int wqe_index,</div><div class='add'>+					int wq_offset, int wq_wqe_cnt,</div><div class='add'>+					int wq_wqe_shift, int bcnt,</div><div class='ctx'> 					size_t *bytes_copied)</div><div class='ctx'> {</div><div class='ctx'> 	size_t offset = wq_offset + ((wqe_index % wq_wqe_cnt) &lt;&lt; wq_wqe_shift);</div><div class='hunk'>@@ -159,11 +143,43 @@ static int mlx5_ib_read_user_wqe_common(struct ib_umem *umem,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp *qp,</div><div class='del'>-			     int wqe_index,</div><div class='del'>-			     void *buffer,</div><div class='del'>-			     int buflen,</div><div class='del'>-			     size_t *bc)</div><div class='add'>+static int mlx5_ib_read_kernel_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index,</div><div class='add'>+				      void *buffer, size_t buflen, size_t *bc)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_wqe_ctrl_seg *ctrl;</div><div class='add'>+	size_t bytes_copied = 0;</div><div class='add'>+	size_t wqe_length;</div><div class='add'>+	void *p;</div><div class='add'>+	int ds;</div><div class='add'>+</div><div class='add'>+	wqe_index = wqe_index &amp; qp-&gt;sq.fbc.sz_m1;</div><div class='add'>+</div><div class='add'>+	/* read the control segment first */</div><div class='add'>+	p = mlx5_frag_buf_get_wqe(&amp;qp-&gt;sq.fbc, wqe_index);</div><div class='add'>+	ctrl = p;</div><div class='add'>+	ds = be32_to_cpu(ctrl-&gt;qpn_ds) &amp; MLX5_WQE_CTRL_DS_MASK;</div><div class='add'>+	wqe_length = ds * MLX5_WQE_DS_UNITS;</div><div class='add'>+</div><div class='add'>+	/* read rest of WQE if it spreads over more than one stride */</div><div class='add'>+	while (bytes_copied &lt; wqe_length) {</div><div class='add'>+		size_t copy_length =</div><div class='add'>+			min_t(size_t, buflen - bytes_copied, MLX5_SEND_WQE_BB);</div><div class='add'>+</div><div class='add'>+		if (!copy_length)</div><div class='add'>+			break;</div><div class='add'>+</div><div class='add'>+		memcpy(buffer + bytes_copied, p, copy_length);</div><div class='add'>+		bytes_copied += copy_length;</div><div class='add'>+</div><div class='add'>+		wqe_index = (wqe_index + 1) &amp; qp-&gt;sq.fbc.sz_m1;</div><div class='add'>+		p = mlx5_frag_buf_get_wqe(&amp;qp-&gt;sq.fbc, wqe_index);</div><div class='add'>+	}</div><div class='add'>+	*bc = bytes_copied;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index,</div><div class='add'>+				    void *buffer, size_t buflen, size_t *bc)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_qp_base *base = &amp;qp-&gt;trans_qp.base;</div><div class='ctx'> 	struct ib_umem *umem = base-&gt;ubuffer.umem;</div><div class='hunk'>@@ -175,18 +191,10 @@ int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp *qp,</div><div class='ctx'> 	int ret;</div><div class='ctx'> 	int ds;</div><div class='ctx'> </div><div class='del'>-	if (buflen &lt; sizeof(*ctrl))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='ctx'> 	/* at first read as much as possible */</div><div class='del'>-	ret = mlx5_ib_read_user_wqe_common(umem,</div><div class='del'>-					   buffer,</div><div class='del'>-					   buflen,</div><div class='del'>-					   wqe_index,</div><div class='del'>-					   wq-&gt;offset,</div><div class='del'>-					   wq-&gt;wqe_cnt,</div><div class='del'>-					   wq-&gt;wqe_shift,</div><div class='del'>-					   buflen,</div><div class='add'>+	ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index,</div><div class='add'>+					   wq-&gt;offset, wq-&gt;wqe_cnt,</div><div class='add'>+					   wq-&gt;wqe_shift, buflen,</div><div class='ctx'> 					   &amp;bytes_copied);</div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='hunk'>@@ -209,13 +217,9 @@ int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp *qp,</div><div class='ctx'> 	 * so read the remaining bytes starting</div><div class='ctx'> 	 * from  wqe_index 0</div><div class='ctx'> 	 */</div><div class='del'>-	ret = mlx5_ib_read_user_wqe_common(umem,</div><div class='del'>-					   buffer + bytes_copied,</div><div class='del'>-					   buflen - bytes_copied,</div><div class='del'>-					   0,</div><div class='del'>-					   wq-&gt;offset,</div><div class='del'>-					   wq-&gt;wqe_cnt,</div><div class='del'>-					   wq-&gt;wqe_shift,</div><div class='add'>+	ret = mlx5_ib_read_user_wqe_common(umem, buffer + bytes_copied,</div><div class='add'>+					   buflen - bytes_copied, 0, wq-&gt;offset,</div><div class='add'>+					   wq-&gt;wqe_cnt, wq-&gt;wqe_shift,</div><div class='ctx'> 					   wqe_length - bytes_copied,</div><div class='ctx'> 					   &amp;bytes_copied2);</div><div class='ctx'> </div><div class='hunk'>@@ -225,11 +229,24 @@ int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp *qp,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp,</div><div class='del'>-			     int wqe_index,</div><div class='del'>-			     void *buffer,</div><div class='del'>-			     int buflen,</div><div class='del'>-			     size_t *bc)</div><div class='add'>+int mlx5_ib_read_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,</div><div class='add'>+			size_t buflen, size_t *bc)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_qp_base *base = &amp;qp-&gt;trans_qp.base;</div><div class='add'>+	struct ib_umem *umem = base-&gt;ubuffer.umem;</div><div class='add'>+</div><div class='add'>+	if (buflen &lt; sizeof(struct mlx5_wqe_ctrl_seg))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!umem)</div><div class='add'>+		return mlx5_ib_read_kernel_wqe_sq(qp, wqe_index, buffer,</div><div class='add'>+						  buflen, bc);</div><div class='add'>+</div><div class='add'>+	return mlx5_ib_read_user_wqe_sq(qp, wqe_index, buffer, buflen, bc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index,</div><div class='add'>+				    void *buffer, size_t buflen, size_t *bc)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_qp_base *base = &amp;qp-&gt;trans_qp.base;</div><div class='ctx'> 	struct ib_umem *umem = base-&gt;ubuffer.umem;</div><div class='hunk'>@@ -237,14 +254,9 @@ int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp,</div><div class='ctx'> 	size_t bytes_copied;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = mlx5_ib_read_user_wqe_common(umem,</div><div class='del'>-					   buffer,</div><div class='del'>-					   buflen,</div><div class='del'>-					   wqe_index,</div><div class='del'>-					   wq-&gt;offset,</div><div class='del'>-					   wq-&gt;wqe_cnt,</div><div class='del'>-					   wq-&gt;wqe_shift,</div><div class='del'>-					   buflen,</div><div class='add'>+	ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index,</div><div class='add'>+					   wq-&gt;offset, wq-&gt;wqe_cnt,</div><div class='add'>+					   wq-&gt;wqe_shift, buflen,</div><div class='ctx'> 					   &amp;bytes_copied);</div><div class='ctx'> </div><div class='ctx'> 	if (ret)</div><div class='hunk'>@@ -253,25 +265,33 @@ int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq,</div><div class='del'>-			      int wqe_index,</div><div class='del'>-			      void *buffer,</div><div class='del'>-			      int buflen,</div><div class='del'>-			      size_t *bc)</div><div class='add'>+int mlx5_ib_read_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,</div><div class='add'>+			size_t buflen, size_t *bc)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_qp_base *base = &amp;qp-&gt;trans_qp.base;</div><div class='add'>+	struct ib_umem *umem = base-&gt;ubuffer.umem;</div><div class='add'>+	struct mlx5_ib_wq *wq = &amp;qp-&gt;rq;</div><div class='add'>+	size_t wqe_size = 1 &lt;&lt; wq-&gt;wqe_shift;</div><div class='add'>+</div><div class='add'>+	if (buflen &lt; wqe_size)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!umem)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	return mlx5_ib_read_user_wqe_rq(qp, wqe_index, buffer, buflen, bc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index,</div><div class='add'>+				     void *buffer, size_t buflen, size_t *bc)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_umem *umem = srq-&gt;umem;</div><div class='ctx'> 	size_t bytes_copied;</div><div class='ctx'> 	int ret;</div><div class='ctx'> </div><div class='del'>-	ret = mlx5_ib_read_user_wqe_common(umem,</div><div class='del'>-					   buffer,</div><div class='del'>-					   buflen,</div><div class='del'>-					   wqe_index,</div><div class='del'>-					   0,</div><div class='del'>-					   srq-&gt;msrq.max,</div><div class='del'>-					   srq-&gt;msrq.wqe_shift,</div><div class='del'>-					   buflen,</div><div class='del'>-					   &amp;bytes_copied);</div><div class='add'>+	ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index, 0,</div><div class='add'>+					   srq-&gt;msrq.max, srq-&gt;msrq.wqe_shift,</div><div class='add'>+					   buflen, &amp;bytes_copied);</div><div class='ctx'> </div><div class='ctx'> 	if (ret)</div><div class='ctx'> 		return ret;</div><div class='hunk'>@@ -279,51 +299,135 @@ int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+int mlx5_ib_read_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index, void *buffer,</div><div class='add'>+			 size_t buflen, size_t *bc)</div><div class='add'>+{</div><div class='add'>+	struct ib_umem *umem = srq-&gt;umem;</div><div class='add'>+	size_t wqe_size = 1 &lt;&lt; srq-&gt;msrq.wqe_shift;</div><div class='add'>+</div><div class='add'>+	if (buflen &lt; wqe_size)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (!umem)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	return mlx5_ib_read_user_wqe_srq(srq, wqe_index, buffer, buflen, bc);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_qp_err_syndrome(struct ib_qp *ibqp)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ibqp-&gt;device);</div><div class='add'>+	int outlen = MLX5_ST_SZ_BYTES(query_qp_out);</div><div class='add'>+	struct mlx5_ib_qp *qp = to_mqp(ibqp);</div><div class='add'>+	void *pas_ext_union, *err_syn;</div><div class='add'>+	u32 *outb;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (!MLX5_CAP_GEN(dev-&gt;mdev, qpc_extension) ||</div><div class='add'>+	    !MLX5_CAP_GEN(dev-&gt;mdev, qp_error_syndrome))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	outb = kzalloc(outlen, GFP_KERNEL);</div><div class='add'>+	if (!outb)</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	err = mlx5_core_qp_query(dev, &amp;qp-&gt;trans_qp.base.mqp, outb, outlen,</div><div class='add'>+				 true);</div><div class='add'>+	if (err)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	pas_ext_union =</div><div class='add'>+		MLX5_ADDR_OF(query_qp_out, outb, qp_pas_or_qpc_ext_and_pas);</div><div class='add'>+	err_syn = MLX5_ADDR_OF(qpc_extension_and_pas_list_in, pas_ext_union,</div><div class='add'>+			       qpc_data_extension.error_syndrome);</div><div class='add'>+</div><div class='add'>+	pr_err("%s/%d: QP %d error: %s (0x%x 0x%x 0x%x)\n",</div><div class='add'>+	       ibqp-&gt;device-&gt;name, ibqp-&gt;port, ibqp-&gt;qp_num,</div><div class='add'>+	       ib_wc_status_msg(</div><div class='add'>+		       MLX5_GET(cqe_error_syndrome, err_syn, syndrome)),</div><div class='add'>+	       MLX5_GET(cqe_error_syndrome, err_syn, vendor_error_syndrome),</div><div class='add'>+	       MLX5_GET(cqe_error_syndrome, err_syn, hw_syndrome_type),</div><div class='add'>+	       MLX5_GET(cqe_error_syndrome, err_syn, hw_error_syndrome));</div><div class='add'>+out:</div><div class='add'>+	kfree(outb);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mlx5_ib_handle_qp_event(struct work_struct *_work)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_qp_event_work *qpe_work =</div><div class='add'>+		container_of(_work, struct mlx5_ib_qp_event_work, work);</div><div class='add'>+	struct ib_qp *ibqp = &amp;to_mibqp(qpe_work-&gt;qp)-&gt;ibqp;</div><div class='add'>+	struct ib_event event = {};</div><div class='add'>+</div><div class='add'>+	event.device = ibqp-&gt;device;</div><div class='add'>+	event.element.qp = ibqp;</div><div class='add'>+	switch (qpe_work-&gt;type) {</div><div class='add'>+	case MLX5_EVENT_TYPE_PATH_MIG:</div><div class='add'>+		event.event = IB_EVENT_PATH_MIG;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_EVENT_TYPE_COMM_EST:</div><div class='add'>+		event.event = IB_EVENT_COMM_EST;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_EVENT_TYPE_SQ_DRAINED:</div><div class='add'>+		event.event = IB_EVENT_SQ_DRAINED;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:</div><div class='add'>+		event.event = IB_EVENT_QP_LAST_WQE_REACHED;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:</div><div class='add'>+		event.event = IB_EVENT_QP_FATAL;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:</div><div class='add'>+		event.event = IB_EVENT_PATH_MIG_ERR;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:</div><div class='add'>+		event.event = IB_EVENT_QP_REQ_ERR;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:</div><div class='add'>+		event.event = IB_EVENT_QP_ACCESS_ERR;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n",</div><div class='add'>+			qpe_work-&gt;type, qpe_work-&gt;qp-&gt;qpn);</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if ((event.event == IB_EVENT_QP_FATAL) ||</div><div class='add'>+	    (event.event == IB_EVENT_QP_ACCESS_ERR))</div><div class='add'>+		mlx5_ib_qp_err_syndrome(ibqp);</div><div class='add'>+</div><div class='add'>+	ibqp-&gt;event_handler(&amp;event, ibqp-&gt;qp_context);</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	mlx5_core_res_put(&amp;qpe_work-&gt;qp-&gt;common);</div><div class='add'>+	kfree(qpe_work);</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type)</div><div class='ctx'> {</div><div class='ctx'> 	struct ib_qp *ibqp = &amp;to_mibqp(qp)-&gt;ibqp;</div><div class='del'>-	struct ib_event event;</div><div class='add'>+	struct mlx5_ib_qp_event_work *qpe_work;</div><div class='ctx'> </div><div class='ctx'> 	if (type == MLX5_EVENT_TYPE_PATH_MIG) {</div><div class='ctx'> 		/* This event is only valid for trans_qps */</div><div class='ctx'> 		to_mibqp(qp)-&gt;port = to_mibqp(qp)-&gt;trans_qp.alt_port;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (ibqp-&gt;event_handler) {</div><div class='del'>-		event.device     = ibqp-&gt;device;</div><div class='del'>-		event.element.qp = ibqp;</div><div class='del'>-		switch (type) {</div><div class='del'>-		case MLX5_EVENT_TYPE_PATH_MIG:</div><div class='del'>-			event.event = IB_EVENT_PATH_MIG;</div><div class='del'>-			break;</div><div class='del'>-		case MLX5_EVENT_TYPE_COMM_EST:</div><div class='del'>-			event.event = IB_EVENT_COMM_EST;</div><div class='del'>-			break;</div><div class='del'>-		case MLX5_EVENT_TYPE_SQ_DRAINED:</div><div class='del'>-			event.event = IB_EVENT_SQ_DRAINED;</div><div class='del'>-			break;</div><div class='del'>-		case MLX5_EVENT_TYPE_SRQ_LAST_WQE:</div><div class='del'>-			event.event = IB_EVENT_QP_LAST_WQE_REACHED;</div><div class='del'>-			break;</div><div class='del'>-		case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:</div><div class='del'>-			event.event = IB_EVENT_QP_FATAL;</div><div class='del'>-			break;</div><div class='del'>-		case MLX5_EVENT_TYPE_PATH_MIG_FAILED:</div><div class='del'>-			event.event = IB_EVENT_PATH_MIG_ERR;</div><div class='del'>-			break;</div><div class='del'>-		case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:</div><div class='del'>-			event.event = IB_EVENT_QP_REQ_ERR;</div><div class='del'>-			break;</div><div class='del'>-		case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:</div><div class='del'>-			event.event = IB_EVENT_QP_ACCESS_ERR;</div><div class='del'>-			break;</div><div class='del'>-		default:</div><div class='del'>-			pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n", type, qp-&gt;qpn);</div><div class='del'>-			return;</div><div class='del'>-		}</div><div class='add'>+	if (!ibqp-&gt;event_handler)</div><div class='add'>+		goto out_no_handler;</div><div class='ctx'> </div><div class='del'>-		ibqp-&gt;event_handler(&amp;event, ibqp-&gt;qp_context);</div><div class='del'>-	}</div><div class='add'>+	qpe_work = kzalloc(sizeof(*qpe_work), GFP_ATOMIC);</div><div class='add'>+	if (!qpe_work)</div><div class='add'>+		goto out_no_handler;</div><div class='add'>+</div><div class='add'>+	qpe_work-&gt;qp = qp;</div><div class='add'>+	qpe_work-&gt;type = type;</div><div class='add'>+	INIT_WORK(&amp;qpe_work-&gt;work, mlx5_ib_handle_qp_event);</div><div class='add'>+	queue_work(mlx5_ib_qp_event_wq, &amp;qpe_work-&gt;work);</div><div class='add'>+	return;</div><div class='add'>+</div><div class='add'>+out_no_handler:</div><div class='add'>+	mlx5_core_res_put(&amp;qp-&gt;common);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap,</div><div class='hunk'>@@ -343,17 +447,26 @@ static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap,</div><div class='ctx'> 		cap-&gt;max_recv_wr = 0;</div><div class='ctx'> 		cap-&gt;max_recv_sge = 0;</div><div class='ctx'> 	} else {</div><div class='add'>+		int wq_sig = !!(qp-&gt;flags_en &amp; MLX5_QP_FLAG_SIGNATURE);</div><div class='add'>+</div><div class='ctx'> 		if (ucmd) {</div><div class='ctx'> 			qp-&gt;rq.wqe_cnt = ucmd-&gt;rq_wqe_count;</div><div class='ctx'> 			if (ucmd-&gt;rq_wqe_shift &gt; BITS_PER_BYTE * sizeof(ucmd-&gt;rq_wqe_shift))</div><div class='ctx'> 				return -EINVAL;</div><div class='ctx'> 			qp-&gt;rq.wqe_shift = ucmd-&gt;rq_wqe_shift;</div><div class='del'>-			if ((1 &lt;&lt; qp-&gt;rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) &lt; qp-&gt;wq_sig)</div><div class='add'>+			if ((1 &lt;&lt; qp-&gt;rq.wqe_shift) /</div><div class='add'>+				    sizeof(struct mlx5_wqe_data_seg) &lt;</div><div class='add'>+			    wq_sig)</div><div class='ctx'> 				return -EINVAL;</div><div class='del'>-			qp-&gt;rq.max_gs = (1 &lt;&lt; qp-&gt;rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp-&gt;wq_sig;</div><div class='add'>+			qp-&gt;rq.max_gs =</div><div class='add'>+				(1 &lt;&lt; qp-&gt;rq.wqe_shift) /</div><div class='add'>+					sizeof(struct mlx5_wqe_data_seg) -</div><div class='add'>+				wq_sig;</div><div class='ctx'> 			qp-&gt;rq.max_post = qp-&gt;rq.wqe_cnt;</div><div class='ctx'> 		} else {</div><div class='del'>-			wqe_size = qp-&gt;wq_sig ? sizeof(struct mlx5_wqe_signature_seg) : 0;</div><div class='add'>+			wqe_size =</div><div class='add'>+				wq_sig ? sizeof(struct mlx5_wqe_signature_seg) :</div><div class='add'>+					 0;</div><div class='ctx'> 			wqe_size += cap-&gt;max_recv_sge * sizeof(struct mlx5_wqe_data_seg);</div><div class='ctx'> 			wqe_size = roundup_pow_of_two(wqe_size);</div><div class='ctx'> 			wq_size = roundup_pow_of_two(cap-&gt;max_recv_wr) * wqe_size;</div><div class='hunk'>@@ -367,7 +480,10 @@ static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap,</div><div class='ctx'> 				return -EINVAL;</div><div class='ctx'> 			}</div><div class='ctx'> 			qp-&gt;rq.wqe_shift = ilog2(wqe_size);</div><div class='del'>-			qp-&gt;rq.max_gs = (1 &lt;&lt; qp-&gt;rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp-&gt;wq_sig;</div><div class='add'>+			qp-&gt;rq.max_gs =</div><div class='add'>+				(1 &lt;&lt; qp-&gt;rq.wqe_shift) /</div><div class='add'>+					sizeof(struct mlx5_wqe_data_seg) -</div><div class='add'>+				wq_sig;</div><div class='ctx'> 			qp-&gt;rq.max_post = qp-&gt;rq.wqe_cnt;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='hunk'>@@ -382,7 +498,7 @@ static int sq_overhead(struct ib_qp_init_attr *attr)</div><div class='ctx'> 	switch (attr-&gt;qp_type) {</div><div class='ctx'> 	case IB_QPT_XRC_INI:</div><div class='ctx'> 		size += sizeof(struct mlx5_wqe_xrc_seg);</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case IB_QPT_RC:</div><div class='ctx'> 		size += sizeof(struct mlx5_wqe_ctrl_seg) +</div><div class='ctx'> 			max(sizeof(struct mlx5_wqe_atomic_seg) +</div><div class='hunk'>@@ -407,7 +523,7 @@ static int sq_overhead(struct ib_qp_init_attr *attr)</div><div class='ctx'> 		if (attr-&gt;create_flags &amp; IB_QP_CREATE_IPOIB_UD_LSO)</div><div class='ctx'> 			size += sizeof(struct mlx5_wqe_eth_pad) +</div><div class='ctx'> 				sizeof(struct mlx5_wqe_eth_seg);</div><div class='del'>-		/* fall through */</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case IB_QPT_SMI:</div><div class='ctx'> 	case MLX5_IB_QPT_HW_GSI:</div><div class='ctx'> 		size += sizeof(struct mlx5_wqe_ctrl_seg) +</div><div class='hunk'>@@ -442,9 +558,9 @@ static int calc_send_wqe(struct ib_qp_init_attr *attr)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	size += attr-&gt;cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg);</div><div class='del'>-	if (attr-&gt;create_flags &amp; IB_QP_CREATE_SIGNATURE_EN &amp;&amp;</div><div class='add'>+	if (attr-&gt;create_flags &amp; IB_QP_CREATE_INTEGRITY_EN &amp;&amp;</div><div class='ctx'> 	    ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) &lt; MLX5_SIG_WQE_SIZE)</div><div class='del'>-			return MLX5_SIG_WQE_SIZE;</div><div class='add'>+		return MLX5_SIG_WQE_SIZE;</div><div class='ctx'> 	else</div><div class='ctx'> 		return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB);</div><div class='ctx'> }</div><div class='hunk'>@@ -496,9 +612,6 @@ static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,</div><div class='ctx'> 			      sizeof(struct mlx5_wqe_inline_seg);</div><div class='ctx'> 	attr-&gt;cap.max_inline_data = qp-&gt;max_inline_data;</div><div class='ctx'> </div><div class='del'>-	if (attr-&gt;create_flags &amp; IB_QP_CREATE_SIGNATURE_EN)</div><div class='del'>-		qp-&gt;signature_en = true;</div><div class='del'>-</div><div class='ctx'> 	wq_size = roundup_pow_of_two(attr-&gt;cap.max_send_wr * wqe_size);</div><div class='ctx'> 	qp-&gt;sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB;</div><div class='ctx'> 	if (qp-&gt;sq.wqe_cnt &gt; (1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev, log_max_qp_sz))) {</div><div class='hunk'>@@ -550,7 +663,7 @@ static int set_user_buf_size(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (attr-&gt;qp_type == IB_QPT_RAW_PACKET ||</div><div class='del'>-	    qp-&gt;flags &amp; MLX5_IB_QP_UNDERLAY) {</div><div class='add'>+	    qp-&gt;flags &amp; IB_QP_CREATE_SOURCE_QPN) {</div><div class='ctx'> 		base-&gt;ubuffer.buf_size = qp-&gt;rq.wqe_cnt &lt;&lt; qp-&gt;rq.wqe_shift;</div><div class='ctx'> 		qp-&gt;raw_packet_qp.sq.ubuffer.buf_size = qp-&gt;sq.wqe_cnt &lt;&lt; 6;</div><div class='ctx'> 	} else {</div><div class='hunk'>@@ -583,7 +696,8 @@ enum {</div><div class='ctx'> </div><div class='ctx'> static int max_bfregs(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi)</div><div class='ctx'> {</div><div class='del'>-	return get_num_static_uars(dev, bfregi) * MLX5_NON_FP_BFREGS_PER_UAR;</div><div class='add'>+	return get_uars_per_sys_page(dev, bfregi-&gt;lib_uar_4k) *</div><div class='add'>+	       bfregi-&gt;num_static_sys_pages * MLX5_NON_FP_BFREGS_PER_UAR;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int num_med_bfreg(struct mlx5_ib_dev *dev,</div><div class='hunk'>@@ -652,6 +766,9 @@ static int alloc_bfreg(struct mlx5_ib_dev *dev,</div><div class='ctx'> {</div><div class='ctx'> 	int bfregn = -ENOMEM;</div><div class='ctx'> </div><div class='add'>+	if (bfregi-&gt;lib_uar_dyn)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='ctx'> 	mutex_lock(&amp;bfregi-&gt;lock);</div><div class='ctx'> 	if (bfregi-&gt;ver &gt;= 2) {</div><div class='ctx'> 		bfregn = alloc_high_class_bfreg(dev, bfregi);</div><div class='hunk'>@@ -702,10 +819,7 @@ static int to_mlx5_st(enum ib_qp_type type)</div><div class='ctx'> 	case IB_QPT_SMI:		return MLX5_QP_ST_QP0;</div><div class='ctx'> 	case MLX5_IB_QPT_HW_GSI:	return MLX5_QP_ST_QP1;</div><div class='ctx'> 	case MLX5_IB_QPT_DCI:		return MLX5_QP_ST_DCI;</div><div class='del'>-	case IB_QPT_RAW_IPV6:		return MLX5_QP_ST_RAW_IPV6;</div><div class='del'>-	case IB_QPT_RAW_PACKET:</div><div class='del'>-	case IB_QPT_RAW_ETHERTYPE:	return MLX5_QP_ST_RAW_ETHERTYPE;</div><div class='del'>-	case IB_QPT_MAX:</div><div class='add'>+	case IB_QPT_RAW_PACKET:		return MLX5_QP_ST_RAW_ETHERTYPE;</div><div class='ctx'> 	default:		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='hunk'>@@ -723,6 +837,9 @@ int bfregn_to_uar_index(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	u32 index_of_sys_page;</div><div class='ctx'> 	u32 offset;</div><div class='ctx'> </div><div class='add'>+	if (bfregi-&gt;lib_uar_dyn)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='ctx'> 	bfregs_per_sys_page = get_uars_per_sys_page(dev, bfregi-&gt;lib_uar_4k) *</div><div class='ctx'> 				MLX5_NON_FP_BFREGS_PER_UAR;</div><div class='ctx'> 	index_of_sys_page = bfregn / bfregs_per_sys_page;</div><div class='hunk'>@@ -744,39 +861,6 @@ int bfregn_to_uar_index(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	return bfregi-&gt;sys_pages[index_of_sys_page] + offset;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int mlx5_ib_umem_get(struct mlx5_ib_dev *dev, struct ib_udata *udata,</div><div class='del'>-			    unsigned long addr, size_t size,</div><div class='del'>-			    struct ib_umem **umem, int *npages, int *page_shift,</div><div class='del'>-			    int *ncont, u32 *offset)</div><div class='del'>-{</div><div class='del'>-	int err;</div><div class='del'>-</div><div class='del'>-	*umem = ib_umem_get(udata, addr, size, 0, 0);</div><div class='del'>-	if (IS_ERR(*umem)) {</div><div class='del'>-		mlx5_ib_dbg(dev, "umem_get failed\n");</div><div class='del'>-		return PTR_ERR(*umem);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mlx5_ib_cont_pages(*umem, addr, 0, npages, page_shift, ncont, NULL);</div><div class='del'>-</div><div class='del'>-	err = mlx5_ib_get_buf_offset(addr, *page_shift, offset);</div><div class='del'>-	if (err) {</div><div class='del'>-		mlx5_ib_warn(dev, "bad offset\n");</div><div class='del'>-		goto err_umem;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mlx5_ib_dbg(dev, "addr 0x%lx, size %zu, npages %d, page_shift %d, ncont %d, offset %d\n",</div><div class='del'>-		    addr, size, *npages, *page_shift, *ncont, *offset);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-</div><div class='del'>-err_umem:</div><div class='del'>-	ib_umem_release(*umem);</div><div class='del'>-	*umem = NULL;</div><div class='del'>-</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static void destroy_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='ctx'> 			    struct mlx5_ib_rwq *rwq, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='hunk'>@@ -790,8 +874,7 @@ static void destroy_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='ctx'> 		atomic_dec(&amp;dev-&gt;delay_drop.rqs_cnt);</div><div class='ctx'> </div><div class='ctx'> 	mlx5_ib_db_unmap_user(context, &amp;rwq-&gt;db);</div><div class='del'>-	if (rwq-&gt;umem)</div><div class='del'>-		ib_umem_release(rwq-&gt;umem);</div><div class='add'>+	ib_umem_release(rwq-&gt;umem);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='hunk'>@@ -800,47 +883,47 @@ static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(</div><div class='ctx'> 		udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='del'>-	int page_shift = 0;</div><div class='del'>-	int npages;</div><div class='add'>+	unsigned long page_size = 0;</div><div class='ctx'> 	u32 offset = 0;</div><div class='del'>-	int ncont = 0;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='ctx'> 	if (!ucmd-&gt;buf_addr)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	rwq-&gt;umem = ib_umem_get(udata, ucmd-&gt;buf_addr, rwq-&gt;buf_size, 0, 0);</div><div class='add'>+	rwq-&gt;umem = ib_umem_get(&amp;dev-&gt;ib_dev, ucmd-&gt;buf_addr, rwq-&gt;buf_size, 0);</div><div class='ctx'> 	if (IS_ERR(rwq-&gt;umem)) {</div><div class='ctx'> 		mlx5_ib_dbg(dev, "umem_get failed\n");</div><div class='ctx'> 		err = PTR_ERR(rwq-&gt;umem);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_cont_pages(rwq-&gt;umem, ucmd-&gt;buf_addr, 0, &amp;npages, &amp;page_shift,</div><div class='del'>-			   &amp;ncont, NULL);</div><div class='del'>-	err = mlx5_ib_get_buf_offset(ucmd-&gt;buf_addr, page_shift,</div><div class='del'>-				     &amp;rwq-&gt;rq_page_offset);</div><div class='del'>-	if (err) {</div><div class='add'>+	page_size = mlx5_umem_find_best_quantized_pgoff(</div><div class='add'>+		rwq-&gt;umem, wq, log_wq_pg_sz, MLX5_ADAPTER_PAGE_SHIFT,</div><div class='add'>+		page_offset, 64, &amp;rwq-&gt;rq_page_offset);</div><div class='add'>+	if (!page_size) {</div><div class='ctx'> 		mlx5_ib_warn(dev, "bad offset\n");</div><div class='add'>+		err = -EINVAL;</div><div class='ctx'> 		goto err_umem;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	rwq-&gt;rq_num_pas = ncont;</div><div class='del'>-	rwq-&gt;page_shift = page_shift;</div><div class='del'>-	rwq-&gt;log_page_size =  page_shift - MLX5_ADAPTER_PAGE_SHIFT;</div><div class='add'>+	rwq-&gt;rq_num_pas = ib_umem_num_dma_blocks(rwq-&gt;umem, page_size);</div><div class='add'>+	rwq-&gt;page_shift = order_base_2(page_size);</div><div class='add'>+	rwq-&gt;log_page_size =  rwq-&gt;page_shift - MLX5_ADAPTER_PAGE_SHIFT;</div><div class='ctx'> 	rwq-&gt;wq_sig = !!(ucmd-&gt;flags &amp; MLX5_WQ_FLAG_SIGNATURE);</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_dbg(dev, "addr 0x%llx, size %zd, npages %d, page_shift %d, ncont %d, offset %d\n",</div><div class='del'>-		    (unsigned long long)ucmd-&gt;buf_addr, rwq-&gt;buf_size,</div><div class='del'>-		    npages, page_shift, ncont, offset);</div><div class='add'>+	mlx5_ib_dbg(</div><div class='add'>+		dev,</div><div class='add'>+		"addr 0x%llx, size %zd, npages %zu, page_size %ld, ncont %d, offset %d\n",</div><div class='add'>+		(unsigned long long)ucmd-&gt;buf_addr, rwq-&gt;buf_size,</div><div class='add'>+		ib_umem_num_pages(rwq-&gt;umem), page_size, rwq-&gt;rq_num_pas,</div><div class='add'>+		offset);</div><div class='ctx'> </div><div class='del'>-	err = mlx5_ib_db_map_user(ucontext, udata, ucmd-&gt;db_addr, &amp;rwq-&gt;db);</div><div class='add'>+	err = mlx5_ib_db_map_user(ucontext, ucmd-&gt;db_addr, &amp;rwq-&gt;db);</div><div class='ctx'> 	if (err) {</div><div class='ctx'> 		mlx5_ib_dbg(dev, "map failed\n");</div><div class='ctx'> 		goto err_umem;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	rwq-&gt;create_type = MLX5_WQ_USER;</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> err_umem:</div><div class='hunk'>@@ -855,53 +938,51 @@ static int adjust_bfregn(struct mlx5_ib_dev *dev,</div><div class='ctx'> 				bfregn % MLX5_NON_FP_BFREGS_PER_UAR;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='del'>-			  struct mlx5_ib_qp *qp, struct ib_udata *udata,</div><div class='del'>-			  struct ib_qp_init_attr *attr,</div><div class='del'>-			  u32 **in,</div><div class='del'>-			  struct mlx5_ib_create_qp_resp *resp, int *inlen,</div><div class='del'>-			  struct mlx5_ib_qp_base *base)</div><div class='add'>+static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='add'>+			   struct mlx5_ib_qp *qp, struct ib_udata *udata,</div><div class='add'>+			   struct ib_qp_init_attr *attr, u32 **in,</div><div class='add'>+			   struct mlx5_ib_create_qp_resp *resp, int *inlen,</div><div class='add'>+			   struct mlx5_ib_qp_base *base,</div><div class='add'>+			   struct mlx5_ib_create_qp *ucmd)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_ucontext *context;</div><div class='del'>-	struct mlx5_ib_create_qp ucmd;</div><div class='ctx'> 	struct mlx5_ib_ubuffer *ubuffer = &amp;base-&gt;ubuffer;</div><div class='del'>-	int page_shift = 0;</div><div class='add'>+	unsigned int page_offset_quantized = 0;</div><div class='add'>+	unsigned long page_size = 0;</div><div class='ctx'> 	int uar_index = 0;</div><div class='del'>-	int npages;</div><div class='del'>-	u32 offset = 0;</div><div class='ctx'> 	int bfregn;</div><div class='ctx'> 	int ncont = 0;</div><div class='ctx'> 	__be64 *pas;</div><div class='ctx'> 	void *qpc;</div><div class='ctx'> 	int err;</div><div class='ctx'> 	u16 uid;</div><div class='del'>-</div><div class='del'>-	err = ib_copy_from_udata(&amp;ucmd, udata, sizeof(ucmd));</div><div class='del'>-	if (err) {</div><div class='del'>-		mlx5_ib_dbg(dev, "copy failed\n");</div><div class='del'>-		return err;</div><div class='del'>-	}</div><div class='add'>+	u32 uar_flags;</div><div class='ctx'> </div><div class='ctx'> 	context = rdma_udata_to_drv_context(udata, struct mlx5_ib_ucontext,</div><div class='ctx'> 					    ibucontext);</div><div class='del'>-	if (ucmd.flags &amp; MLX5_QP_FLAG_BFREG_INDEX) {</div><div class='add'>+	uar_flags = qp-&gt;flags_en &amp;</div><div class='add'>+		    (MLX5_QP_FLAG_UAR_PAGE_INDEX | MLX5_QP_FLAG_BFREG_INDEX);</div><div class='add'>+	switch (uar_flags) {</div><div class='add'>+	case MLX5_QP_FLAG_UAR_PAGE_INDEX:</div><div class='add'>+		uar_index = ucmd-&gt;bfreg_index;</div><div class='add'>+		bfregn = MLX5_IB_INVALID_BFREG;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_QP_FLAG_BFREG_INDEX:</div><div class='ctx'> 		uar_index = bfregn_to_uar_index(dev, &amp;context-&gt;bfregi,</div><div class='del'>-						ucmd.bfreg_index, true);</div><div class='add'>+						ucmd-&gt;bfreg_index, true);</div><div class='ctx'> 		if (uar_index &lt; 0)</div><div class='ctx'> 			return uar_index;</div><div class='del'>-</div><div class='ctx'> 		bfregn = MLX5_IB_INVALID_BFREG;</div><div class='del'>-	} else if (qp-&gt;flags &amp; MLX5_IB_QP_CROSS_CHANNEL) {</div><div class='del'>-		/*</div><div class='del'>-		 * TBD: should come from the verbs when we have the API</div><div class='del'>-		 */</div><div class='del'>-		/* In CROSS_CHANNEL CQ and QP must use the same UAR */</div><div class='del'>-		bfregn = MLX5_CROSS_CHANNEL_BFREG;</div><div class='del'>-	}</div><div class='del'>-	else {</div><div class='add'>+		break;</div><div class='add'>+	case 0:</div><div class='add'>+		if (qp-&gt;flags &amp; IB_QP_CREATE_CROSS_CHANNEL)</div><div class='add'>+			return -EINVAL;</div><div class='ctx'> 		bfregn = alloc_bfreg(dev, &amp;context-&gt;bfregi);</div><div class='ctx'> 		if (bfregn &lt; 0)</div><div class='ctx'> 			return bfregn;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	mlx5_ib_dbg(dev, "bfregn 0x%x, uar_index 0x%x\n", bfregn, uar_index);</div><div class='hunk'>@@ -913,17 +994,27 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='ctx'> 	qp-&gt;sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);</div><div class='ctx'> 	qp-&gt;sq.offset = qp-&gt;rq.wqe_cnt &lt;&lt; qp-&gt;rq.wqe_shift;</div><div class='ctx'> </div><div class='del'>-	err = set_user_buf_size(dev, qp, &amp;ucmd, base, attr);</div><div class='add'>+	err = set_user_buf_size(dev, qp, ucmd, base, attr);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto err_bfreg;</div><div class='ctx'> </div><div class='del'>-	if (ucmd.buf_addr &amp;&amp; ubuffer-&gt;buf_size) {</div><div class='del'>-		ubuffer-&gt;buf_addr = ucmd.buf_addr;</div><div class='del'>-		err = mlx5_ib_umem_get(dev, udata, ubuffer-&gt;buf_addr,</div><div class='del'>-				       ubuffer-&gt;buf_size, &amp;ubuffer-&gt;umem,</div><div class='del'>-				       &amp;npages, &amp;page_shift, &amp;ncont, &amp;offset);</div><div class='del'>-		if (err)</div><div class='add'>+	if (ucmd-&gt;buf_addr &amp;&amp; ubuffer-&gt;buf_size) {</div><div class='add'>+		ubuffer-&gt;buf_addr = ucmd-&gt;buf_addr;</div><div class='add'>+		ubuffer-&gt;umem = ib_umem_get(&amp;dev-&gt;ib_dev, ubuffer-&gt;buf_addr,</div><div class='add'>+					    ubuffer-&gt;buf_size, 0);</div><div class='add'>+		if (IS_ERR(ubuffer-&gt;umem)) {</div><div class='add'>+			err = PTR_ERR(ubuffer-&gt;umem);</div><div class='ctx'> 			goto err_bfreg;</div><div class='add'>+		}</div><div class='add'>+		page_size = mlx5_umem_find_best_quantized_pgoff(</div><div class='add'>+			ubuffer-&gt;umem, qpc, log_page_size,</div><div class='add'>+			MLX5_ADAPTER_PAGE_SHIFT, page_offset, 64,</div><div class='add'>+			&amp;page_offset_quantized);</div><div class='add'>+		if (!page_size) {</div><div class='add'>+			err = -EINVAL;</div><div class='add'>+			goto err_umem;</div><div class='add'>+		}</div><div class='add'>+		ncont = ib_umem_num_dma_blocks(ubuffer-&gt;umem, page_size);</div><div class='ctx'> 	} else {</div><div class='ctx'> 		ubuffer-&gt;umem = NULL;</div><div class='ctx'> 	}</div><div class='hunk'>@@ -936,18 +1027,16 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='ctx'> 		goto err_umem;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	uid = (attr-&gt;qp_type != IB_QPT_XRC_TGT &amp;&amp;</div><div class='del'>-	       attr-&gt;qp_type != IB_QPT_XRC_INI) ? to_mpd(pd)-&gt;uid : 0;</div><div class='add'>+	uid = (attr-&gt;qp_type != IB_QPT_XRC_INI) ? to_mpd(pd)-&gt;uid : 0;</div><div class='ctx'> 	MLX5_SET(create_qp_in, *in, uid, uid);</div><div class='del'>-	pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas);</div><div class='del'>-	if (ubuffer-&gt;umem)</div><div class='del'>-		mlx5_ib_populate_pas(dev, ubuffer-&gt;umem, page_shift, pas, 0);</div><div class='del'>-</div><div class='ctx'> 	qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);</div><div class='del'>-</div><div class='del'>-	MLX5_SET(qpc, qpc, log_page_size, page_shift - MLX5_ADAPTER_PAGE_SHIFT);</div><div class='del'>-	MLX5_SET(qpc, qpc, page_offset, offset);</div><div class='del'>-</div><div class='add'>+	pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas);</div><div class='add'>+	if (ubuffer-&gt;umem) {</div><div class='add'>+		mlx5_ib_populate_pas(ubuffer-&gt;umem, page_size, pas, 0);</div><div class='add'>+		MLX5_SET(qpc, qpc, log_page_size,</div><div class='add'>+			 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);</div><div class='add'>+		MLX5_SET(qpc, qpc, page_offset, page_offset_quantized);</div><div class='add'>+	}</div><div class='ctx'> 	MLX5_SET(qpc, qpc, uar_page, uar_index);</div><div class='ctx'> 	if (bfregn != MLX5_IB_INVALID_BFREG)</div><div class='ctx'> 		resp-&gt;bfreg_index = adjust_bfregn(dev, &amp;context-&gt;bfregi, bfregn);</div><div class='hunk'>@@ -955,30 +1044,19 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='ctx'> 		resp-&gt;bfreg_index = MLX5_IB_INVALID_BFREG;</div><div class='ctx'> 	qp-&gt;bfregn = bfregn;</div><div class='ctx'> </div><div class='del'>-	err = mlx5_ib_db_map_user(context, udata, ucmd.db_addr, &amp;qp-&gt;db);</div><div class='add'>+	err = mlx5_ib_db_map_user(context, ucmd-&gt;db_addr, &amp;qp-&gt;db);</div><div class='ctx'> 	if (err) {</div><div class='ctx'> 		mlx5_ib_dbg(dev, "map failed\n");</div><div class='ctx'> 		goto err_free;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	err = ib_copy_to_udata(udata, resp, min(udata-&gt;outlen, sizeof(*resp)));</div><div class='del'>-	if (err) {</div><div class='del'>-		mlx5_ib_dbg(dev, "copy failed\n");</div><div class='del'>-		goto err_unmap;</div><div class='del'>-	}</div><div class='del'>-	qp-&gt;create_type = MLX5_QP_USER;</div><div class='del'>-</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-err_unmap:</div><div class='del'>-	mlx5_ib_db_unmap_user(context, &amp;qp-&gt;db);</div><div class='del'>-</div><div class='ctx'> err_free:</div><div class='ctx'> 	kvfree(*in);</div><div class='ctx'> </div><div class='ctx'> err_umem:</div><div class='del'>-	if (ubuffer-&gt;umem)</div><div class='del'>-		ib_umem_release(ubuffer-&gt;umem);</div><div class='add'>+	ib_umem_release(ubuffer-&gt;umem);</div><div class='ctx'> </div><div class='ctx'> err_bfreg:</div><div class='ctx'> 	if (bfregn != MLX5_IB_INVALID_BFREG)</div><div class='hunk'>@@ -986,69 +1064,47 @@ err_bfreg:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void destroy_qp_user(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='del'>-			    struct mlx5_ib_qp *qp, struct mlx5_ib_qp_base *base,</div><div class='del'>-			    struct ib_udata *udata)</div><div class='add'>+static void destroy_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='add'>+		       struct mlx5_ib_qp_base *base, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_ucontext *context =</div><div class='del'>-		rdma_udata_to_drv_context(</div><div class='del'>-			udata,</div><div class='del'>-			struct mlx5_ib_ucontext,</div><div class='del'>-			ibucontext);</div><div class='add'>+	struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(</div><div class='add'>+		udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_db_unmap_user(context, &amp;qp-&gt;db);</div><div class='del'>-	if (base-&gt;ubuffer.umem)</div><div class='add'>+	if (udata) {</div><div class='add'>+		/* User QP */</div><div class='add'>+		mlx5_ib_db_unmap_user(context, &amp;qp-&gt;db);</div><div class='ctx'> 		ib_umem_release(base-&gt;ubuffer.umem);</div><div class='ctx'> </div><div class='del'>-	/*</div><div class='del'>-	 * Free only the BFREGs which are handled by the kernel.</div><div class='del'>-	 * BFREGs of UARs allocated dynamically are handled by user.</div><div class='del'>-	 */</div><div class='del'>-	if (qp-&gt;bfregn != MLX5_IB_INVALID_BFREG)</div><div class='del'>-		mlx5_ib_free_bfreg(dev, &amp;context-&gt;bfregi, qp-&gt;bfregn);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* get_sq_edge - Get the next nearby edge.</div><div class='del'>- *</div><div class='del'>- * An 'edge' is defined as the first following address after the end</div><div class='del'>- * of the fragment or the SQ. Accordingly, during the WQE construction</div><div class='del'>- * which repetitively increases the pointer to write the next data, it</div><div class='del'>- * simply should check if it gets to an edge.</div><div class='del'>- *</div><div class='del'>- * @sq - SQ buffer.</div><div class='del'>- * @idx - Stride index in the SQ buffer.</div><div class='del'>- *</div><div class='del'>- * Return:</div><div class='del'>- *	The new edge.</div><div class='del'>- */</div><div class='del'>-static void *get_sq_edge(struct mlx5_ib_wq *sq, u32 idx)</div><div class='del'>-{</div><div class='del'>-	void *fragment_end;</div><div class='del'>-</div><div class='del'>-	fragment_end = mlx5_frag_buf_get_wqe</div><div class='del'>-		(&amp;sq-&gt;fbc,</div><div class='del'>-		 mlx5_frag_buf_get_idx_last_contig_stride(&amp;sq-&gt;fbc, idx));</div><div class='add'>+		/*</div><div class='add'>+		 * Free only the BFREGs which are handled by the kernel.</div><div class='add'>+		 * BFREGs of UARs allocated dynamically are handled by user.</div><div class='add'>+		 */</div><div class='add'>+		if (qp-&gt;bfregn != MLX5_IB_INVALID_BFREG)</div><div class='add'>+			mlx5_ib_free_bfreg(dev, &amp;context-&gt;bfregi, qp-&gt;bfregn);</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	return fragment_end + MLX5_SEND_WQE_BB;</div><div class='add'>+	/* Kernel QP */</div><div class='add'>+	kvfree(qp-&gt;sq.wqe_head);</div><div class='add'>+	kvfree(qp-&gt;sq.w_list);</div><div class='add'>+	kvfree(qp-&gt;sq.wrid);</div><div class='add'>+	kvfree(qp-&gt;sq.wr_data);</div><div class='add'>+	kvfree(qp-&gt;rq.wrid);</div><div class='add'>+	if (qp-&gt;db.db)</div><div class='add'>+		mlx5_db_free(dev-&gt;mdev, &amp;qp-&gt;db);</div><div class='add'>+	if (qp-&gt;buf.frags)</div><div class='add'>+		mlx5_frag_buf_free(dev-&gt;mdev, &amp;qp-&gt;buf);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int create_kernel_qp(struct mlx5_ib_dev *dev,</div><div class='del'>-			    struct ib_qp_init_attr *init_attr,</div><div class='del'>-			    struct mlx5_ib_qp *qp,</div><div class='del'>-			    u32 **in, int *inlen,</div><div class='del'>-			    struct mlx5_ib_qp_base *base)</div><div class='add'>+static int _create_kernel_qp(struct mlx5_ib_dev *dev,</div><div class='add'>+			     struct ib_qp_init_attr *init_attr,</div><div class='add'>+			     struct mlx5_ib_qp *qp, u32 **in, int *inlen,</div><div class='add'>+			     struct mlx5_ib_qp_base *base)</div><div class='ctx'> {</div><div class='ctx'> 	int uar_index;</div><div class='ctx'> 	void *qpc;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;create_flags &amp; ~(IB_QP_CREATE_SIGNATURE_EN |</div><div class='del'>-					IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |</div><div class='del'>-					IB_QP_CREATE_IPOIB_UD_LSO |</div><div class='del'>-					IB_QP_CREATE_NETIF_QP |</div><div class='del'>-					mlx5_ib_create_qp_sqpn_qp1()))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='ctx'> 	if (init_attr-&gt;qp_type == MLX5_IB_QPT_REG_UMR)</div><div class='ctx'> 		qp-&gt;bf.bfreg = &amp;dev-&gt;fp_bfreg;</div><div class='ctx'> 	else</div><div class='hunk'>@@ -1103,16 +1159,15 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,</div><div class='ctx'> </div><div class='ctx'> 	qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);</div><div class='ctx'> 	MLX5_SET(qpc, qpc, uar_page, uar_index);</div><div class='add'>+	MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(dev-&gt;mdev));</div><div class='ctx'> 	MLX5_SET(qpc, qpc, log_page_size, qp-&gt;buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);</div><div class='ctx'> </div><div class='ctx'> 	/* Set "fast registration enabled" for all kernel QPs */</div><div class='ctx'> 	MLX5_SET(qpc, qpc, fre, 1);</div><div class='ctx'> 	MLX5_SET(qpc, qpc, rlky, 1);</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;create_flags &amp; mlx5_ib_create_qp_sqpn_qp1()) {</div><div class='add'>+	if (qp-&gt;flags &amp; MLX5_IB_QP_CREATE_SQPN_QP1)</div><div class='ctx'> 		MLX5_SET(qpc, qpc, deth_sqpn, 1);</div><div class='del'>-		qp-&gt;flags |= MLX5_IB_QP_SQPN_QP1;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='ctx'> 	mlx5_fill_page_frag_array(&amp;qp-&gt;buf,</div><div class='ctx'> 				  (__be64 *)MLX5_ADDR_OF(create_qp_in,</div><div class='hunk'>@@ -1140,7 +1195,6 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,</div><div class='ctx'> 		err = -ENOMEM;</div><div class='ctx'> 		goto err_wrid;</div><div class='ctx'> 	}</div><div class='del'>-	qp-&gt;create_type = MLX5_QP_KERNEL;</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='hunk'>@@ -1160,36 +1214,15 @@ err_buf:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void destroy_qp_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)</div><div class='del'>-{</div><div class='del'>-	kvfree(qp-&gt;sq.wqe_head);</div><div class='del'>-	kvfree(qp-&gt;sq.w_list);</div><div class='del'>-	kvfree(qp-&gt;sq.wrid);</div><div class='del'>-	kvfree(qp-&gt;sq.wr_data);</div><div class='del'>-	kvfree(qp-&gt;rq.wrid);</div><div class='del'>-	mlx5_db_free(dev-&gt;mdev, &amp;qp-&gt;db);</div><div class='del'>-	mlx5_frag_buf_free(dev-&gt;mdev, &amp;qp-&gt;buf);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr)</div><div class='ctx'> {</div><div class='del'>-	if (attr-&gt;srq || (attr-&gt;qp_type == IB_QPT_XRC_TGT) ||</div><div class='del'>-	    (attr-&gt;qp_type == MLX5_IB_QPT_DCI) ||</div><div class='del'>-	    (attr-&gt;qp_type == IB_QPT_XRC_INI))</div><div class='add'>+	if (attr-&gt;srq || (qp-&gt;type == IB_QPT_XRC_TGT) ||</div><div class='add'>+	    (qp-&gt;type == MLX5_IB_QPT_DCI) || (qp-&gt;type == IB_QPT_XRC_INI))</div><div class='ctx'> 		return MLX5_SRQ_RQ;</div><div class='ctx'> 	else if (!qp-&gt;has_rq)</div><div class='ctx'> 		return MLX5_ZERO_LEN_RQ;</div><div class='del'>-	else</div><div class='del'>-		return MLX5_NON_ZERO_RQ;</div><div class='del'>-}</div><div class='ctx'> </div><div class='del'>-static int is_connected(enum ib_qp_type qp_type)</div><div class='del'>-{</div><div class='del'>-	if (qp_type == IB_QPT_RC || qp_type == IB_QPT_UC ||</div><div class='del'>-	    qp_type == MLX5_IB_QPT_DCI)</div><div class='del'>-		return 1;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='add'>+	return MLX5_NON_ZERO_RQ;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,</div><div class='hunk'>@@ -1197,15 +1230,18 @@ static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,</div><div class='ctx'> 				    struct mlx5_ib_sq *sq, u32 tdn,</div><div class='ctx'> 				    struct ib_pd *pd)</div><div class='ctx'> {</div><div class='del'>-	u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};</div><div class='ctx'> 	void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(create_tis_in, in, uid, to_mpd(pd)-&gt;uid);</div><div class='ctx'> 	MLX5_SET(tisc, tisc, transport_domain, tdn);</div><div class='del'>-	if (qp-&gt;flags &amp; MLX5_IB_QP_UNDERLAY)</div><div class='add'>+	if (!mlx5_ib_lag_should_assign_affinity(dev) &amp;&amp;</div><div class='add'>+	    mlx5_lag_is_lacp_owner(dev-&gt;mdev))</div><div class='add'>+		MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_SOURCE_QPN)</div><div class='ctx'> 		MLX5_SET(tisc, tisc, underlay_qpn, qp-&gt;underlay_qpn);</div><div class='ctx'> </div><div class='del'>-	return mlx5_core_create_tis(dev-&gt;mdev, in, sizeof(in), &amp;sq-&gt;tisn);</div><div class='add'>+	return mlx5_core_create_tis(dev-&gt;mdev, in, &amp;sq-&gt;tisn);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev,</div><div class='hunk'>@@ -1221,10 +1257,85 @@ static void destroy_flow_rule_vport_sq(struct mlx5_ib_sq *sq)</div><div class='ctx'> 	sq-&gt;flow_rule = NULL;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static bool fr_supported(int ts_cap)</div><div class='add'>+{</div><div class='add'>+	return ts_cap == MLX5_TIMESTAMP_FORMAT_CAP_FREE_RUNNING ||</div><div class='add'>+	       ts_cap == MLX5_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,</div><div class='add'>+			 bool fr_sup, bool rt_sup)</div><div class='add'>+{</div><div class='add'>+	if (cq-&gt;private_flags &amp; MLX5_IB_CQ_PR_FLAGS_REAL_TIME_TS) {</div><div class='add'>+		if (!rt_sup) {</div><div class='add'>+			mlx5_ib_dbg(dev,</div><div class='add'>+				    "Real time TS format is not supported\n");</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+		}</div><div class='add'>+		return MLX5_TIMESTAMP_FORMAT_REAL_TIME;</div><div class='add'>+	}</div><div class='add'>+	if (cq-&gt;create_flags &amp; IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION) {</div><div class='add'>+		if (!fr_sup) {</div><div class='add'>+			mlx5_ib_dbg(dev,</div><div class='add'>+				    "Free running TS format is not supported\n");</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+		}</div><div class='add'>+		return MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;</div><div class='add'>+	}</div><div class='add'>+	return fr_sup ? MLX5_TIMESTAMP_FORMAT_FREE_RUNNING :</div><div class='add'>+			MLX5_TIMESTAMP_FORMAT_DEFAULT;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_rq_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *recv_cq)</div><div class='add'>+{</div><div class='add'>+	u8 ts_cap = MLX5_CAP_GEN(dev-&gt;mdev, rq_ts_format);</div><div class='add'>+</div><div class='add'>+	return get_ts_format(dev, recv_cq, fr_supported(ts_cap),</div><div class='add'>+			     rt_supported(ts_cap));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_sq_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq)</div><div class='add'>+{</div><div class='add'>+	u8 ts_cap = MLX5_CAP_GEN(dev-&gt;mdev, sq_ts_format);</div><div class='add'>+</div><div class='add'>+	return get_ts_format(dev, send_cq, fr_supported(ts_cap),</div><div class='add'>+			     rt_supported(ts_cap));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_qp_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq,</div><div class='add'>+			    struct mlx5_ib_cq *recv_cq)</div><div class='add'>+{</div><div class='add'>+	u8 ts_cap = MLX5_CAP_ROCE(dev-&gt;mdev, qp_ts_format);</div><div class='add'>+	bool fr_sup = fr_supported(ts_cap);</div><div class='add'>+	bool rt_sup = rt_supported(ts_cap);</div><div class='add'>+	u8 default_ts = fr_sup ? MLX5_TIMESTAMP_FORMAT_FREE_RUNNING :</div><div class='add'>+				 MLX5_TIMESTAMP_FORMAT_DEFAULT;</div><div class='add'>+	int send_ts_format =</div><div class='add'>+		send_cq ? get_ts_format(dev, send_cq, fr_sup, rt_sup) :</div><div class='add'>+			  default_ts;</div><div class='add'>+	int recv_ts_format =</div><div class='add'>+		recv_cq ? get_ts_format(dev, recv_cq, fr_sup, rt_sup) :</div><div class='add'>+			  default_ts;</div><div class='add'>+</div><div class='add'>+	if (send_ts_format &lt; 0 || recv_ts_format &lt; 0)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (send_ts_format != MLX5_TIMESTAMP_FORMAT_DEFAULT &amp;&amp;</div><div class='add'>+	    recv_ts_format != MLX5_TIMESTAMP_FORMAT_DEFAULT &amp;&amp;</div><div class='add'>+	    send_ts_format != recv_ts_format) {</div><div class='add'>+		mlx5_ib_dbg(</div><div class='add'>+			dev,</div><div class='add'>+			"The send ts_format does not match the receive ts_format\n");</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return send_ts_format == default_ts ? recv_ts_format : send_ts_format;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,</div><div class='ctx'> 				   struct ib_udata *udata,</div><div class='ctx'> 				   struct mlx5_ib_sq *sq, void *qpin,</div><div class='del'>-				   struct ib_pd *pd)</div><div class='add'>+				   struct ib_pd *pd, struct mlx5_ib_cq *cq)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_ubuffer *ubuffer = &amp;sq-&gt;ubuffer;</div><div class='ctx'> 	__be64 *pas;</div><div class='hunk'>@@ -1234,18 +1345,29 @@ static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	void *wq;</div><div class='ctx'> 	int inlen;</div><div class='ctx'> 	int err;</div><div class='del'>-	int page_shift = 0;</div><div class='del'>-	int npages;</div><div class='del'>-	int ncont = 0;</div><div class='del'>-	u32 offset = 0;</div><div class='del'>-</div><div class='del'>-	err = mlx5_ib_umem_get(dev, udata, ubuffer-&gt;buf_addr, ubuffer-&gt;buf_size,</div><div class='del'>-			       &amp;sq-&gt;ubuffer.umem, &amp;npages, &amp;page_shift, &amp;ncont,</div><div class='del'>-			       &amp;offset);</div><div class='del'>-	if (err)</div><div class='del'>-		return err;</div><div class='add'>+	unsigned int page_offset_quantized;</div><div class='add'>+	unsigned long page_size;</div><div class='add'>+	int ts_format;</div><div class='add'>+</div><div class='add'>+	ts_format = get_sq_ts_format(dev, cq);</div><div class='add'>+	if (ts_format &lt; 0)</div><div class='add'>+		return ts_format;</div><div class='add'>+</div><div class='add'>+	sq-&gt;ubuffer.umem = ib_umem_get(&amp;dev-&gt;ib_dev, ubuffer-&gt;buf_addr,</div><div class='add'>+				       ubuffer-&gt;buf_size, 0);</div><div class='add'>+	if (IS_ERR(sq-&gt;ubuffer.umem))</div><div class='add'>+		return PTR_ERR(sq-&gt;ubuffer.umem);</div><div class='add'>+	page_size = mlx5_umem_find_best_quantized_pgoff(</div><div class='add'>+		ubuffer-&gt;umem, wq, log_wq_pg_sz, MLX5_ADAPTER_PAGE_SHIFT,</div><div class='add'>+		page_offset, 64, &amp;page_offset_quantized);</div><div class='add'>+	if (!page_size) {</div><div class='add'>+		err = -EINVAL;</div><div class='add'>+		goto err_umem;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	inlen = MLX5_ST_SZ_BYTES(create_sq_in) + sizeof(u64) * ncont;</div><div class='add'>+	inlen = MLX5_ST_SZ_BYTES(create_sq_in) +</div><div class='add'>+		sizeof(u64) *</div><div class='add'>+			ib_umem_num_dma_blocks(sq-&gt;ubuffer.umem, page_size);</div><div class='ctx'> 	in = kvzalloc(inlen, GFP_KERNEL);</div><div class='ctx'> 	if (!in) {</div><div class='ctx'> 		err = -ENOMEM;</div><div class='hunk'>@@ -1258,6 +1380,7 @@ static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	if (MLX5_CAP_ETH(dev-&gt;mdev, multi_pkt_send_wqe))</div><div class='ctx'> 		MLX5_SET(sqc, sqc, allow_multi_pkt_send_wqe, 1);</div><div class='ctx'> 	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);</div><div class='add'>+	MLX5_SET(sqc, sqc, ts_format, ts_format);</div><div class='ctx'> 	MLX5_SET(sqc, sqc, user_index, MLX5_GET(qpc, qpc, user_index));</div><div class='ctx'> 	MLX5_SET(sqc, sqc, cqn, MLX5_GET(qpc, qpc, cqn_snd));</div><div class='ctx'> 	MLX5_SET(sqc, sqc, tis_lst_sz, 1);</div><div class='hunk'>@@ -1273,13 +1396,14 @@ static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));</div><div class='ctx'> 	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));</div><div class='ctx'> 	MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_sq_size));</div><div class='del'>-	MLX5_SET(wq, wq, log_wq_pg_sz,  page_shift - MLX5_ADAPTER_PAGE_SHIFT);</div><div class='del'>-	MLX5_SET(wq, wq, page_offset, offset);</div><div class='add'>+	MLX5_SET(wq, wq, log_wq_pg_sz,</div><div class='add'>+		 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);</div><div class='add'>+	MLX5_SET(wq, wq, page_offset, page_offset_quantized);</div><div class='ctx'> </div><div class='ctx'> 	pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);</div><div class='del'>-	mlx5_ib_populate_pas(dev, sq-&gt;ubuffer.umem, page_shift, pas, 0);</div><div class='add'>+	mlx5_ib_populate_pas(sq-&gt;ubuffer.umem, page_size, pas, 0);</div><div class='ctx'> </div><div class='del'>-	err = mlx5_core_create_sq_tracked(dev-&gt;mdev, in, inlen, &amp;sq-&gt;base.mqp);</div><div class='add'>+	err = mlx5_core_create_sq_tracked(dev, in, inlen, &amp;sq-&gt;base.mqp);</div><div class='ctx'> </div><div class='ctx'> 	kvfree(in);</div><div class='ctx'> </div><div class='hunk'>@@ -1299,44 +1423,40 @@ static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev,</div><div class='ctx'> 				     struct mlx5_ib_sq *sq)</div><div class='ctx'> {</div><div class='ctx'> 	destroy_flow_rule_vport_sq(sq);</div><div class='del'>-	mlx5_core_destroy_sq_tracked(dev-&gt;mdev, &amp;sq-&gt;base.mqp);</div><div class='add'>+	mlx5_core_destroy_sq_tracked(dev, &amp;sq-&gt;base.mqp);</div><div class='ctx'> 	ib_umem_release(sq-&gt;ubuffer.umem);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static size_t get_rq_pas_size(void *qpc)</div><div class='del'>-{</div><div class='del'>-	u32 log_page_size = MLX5_GET(qpc, qpc, log_page_size) + 12;</div><div class='del'>-	u32 log_rq_stride = MLX5_GET(qpc, qpc, log_rq_stride);</div><div class='del'>-	u32 log_rq_size   = MLX5_GET(qpc, qpc, log_rq_size);</div><div class='del'>-	u32 page_offset   = MLX5_GET(qpc, qpc, page_offset);</div><div class='del'>-	u32 po_quanta	  = 1 &lt;&lt; (log_page_size - 6);</div><div class='del'>-	u32 rq_sz	  = 1 &lt;&lt; (log_rq_size + 4 + log_rq_stride);</div><div class='del'>-	u32 page_size	  = 1 &lt;&lt; log_page_size;</div><div class='del'>-	u32 rq_sz_po      = rq_sz + (page_offset * po_quanta);</div><div class='del'>-	u32 rq_num_pas	  = (rq_sz_po + page_size - 1) / page_size;</div><div class='del'>-</div><div class='del'>-	return rq_num_pas * sizeof(u64);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,</div><div class='ctx'> 				   struct mlx5_ib_rq *rq, void *qpin,</div><div class='del'>-				   size_t qpinlen, struct ib_pd *pd)</div><div class='add'>+				   struct ib_pd *pd, struct mlx5_ib_cq *cq)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_qp *mqp = rq-&gt;base.container_mibqp;</div><div class='ctx'> 	__be64 *pas;</div><div class='del'>-	__be64 *qp_pas;</div><div class='ctx'> 	void *in;</div><div class='ctx'> 	void *rqc;</div><div class='ctx'> 	void *wq;</div><div class='ctx'> 	void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);</div><div class='del'>-	size_t rq_pas_size = get_rq_pas_size(qpc);</div><div class='add'>+	struct ib_umem *umem = rq-&gt;base.ubuffer.umem;</div><div class='add'>+	unsigned int page_offset_quantized;</div><div class='add'>+	unsigned long page_size = 0;</div><div class='add'>+	int ts_format;</div><div class='ctx'> 	size_t inlen;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	if (qpinlen &lt; rq_pas_size + MLX5_BYTE_OFF(create_qp_in, pas))</div><div class='add'>+	ts_format = get_rq_ts_format(dev, cq);</div><div class='add'>+	if (ts_format &lt; 0)</div><div class='add'>+		return ts_format;</div><div class='add'>+</div><div class='add'>+	page_size = mlx5_umem_find_best_quantized_pgoff(umem, wq, log_wq_pg_sz,</div><div class='add'>+							MLX5_ADAPTER_PAGE_SHIFT,</div><div class='add'>+							page_offset, 64,</div><div class='add'>+							&amp;page_offset_quantized);</div><div class='add'>+	if (!page_size)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	inlen = MLX5_ST_SZ_BYTES(create_rq_in) + rq_pas_size;</div><div class='add'>+	inlen = MLX5_ST_SZ_BYTES(create_rq_in) +</div><div class='add'>+		sizeof(u64) * ib_umem_num_dma_blocks(umem, page_size);</div><div class='ctx'> 	in = kvzalloc(inlen, GFP_KERNEL);</div><div class='ctx'> 	if (!in)</div><div class='ctx'> 		return -ENOMEM;</div><div class='hunk'>@@ -1347,29 +1467,30 @@ static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,</div><div class='ctx'> 		MLX5_SET(rqc, rqc, vsd, 1);</div><div class='ctx'> 	MLX5_SET(rqc, rqc, mem_rq_type, MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);</div><div class='ctx'> 	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);</div><div class='add'>+	MLX5_SET(rqc, rqc, ts_format, ts_format);</div><div class='ctx'> 	MLX5_SET(rqc, rqc, flush_in_error_en, 1);</div><div class='ctx'> 	MLX5_SET(rqc, rqc, user_index, MLX5_GET(qpc, qpc, user_index));</div><div class='ctx'> 	MLX5_SET(rqc, rqc, cqn, MLX5_GET(qpc, qpc, cqn_rcv));</div><div class='ctx'> </div><div class='del'>-	if (mqp-&gt;flags &amp; MLX5_IB_QP_CAP_SCATTER_FCS)</div><div class='add'>+	if (mqp-&gt;flags &amp; IB_QP_CREATE_SCATTER_FCS)</div><div class='ctx'> 		MLX5_SET(rqc, rqc, scatter_fcs, 1);</div><div class='ctx'> </div><div class='ctx'> 	wq = MLX5_ADDR_OF(rqc, rqc, wq);</div><div class='ctx'> 	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);</div><div class='ctx'> 	if (rq-&gt;flags &amp; MLX5_IB_RQ_PCI_WRITE_END_PADDING)</div><div class='ctx'> 		MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);</div><div class='del'>-	MLX5_SET(wq, wq, page_offset, MLX5_GET(qpc, qpc, page_offset));</div><div class='add'>+	MLX5_SET(wq, wq, page_offset, page_offset_quantized);</div><div class='ctx'> 	MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));</div><div class='ctx'> 	MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));</div><div class='ctx'> 	MLX5_SET(wq, wq, log_wq_stride, MLX5_GET(qpc, qpc, log_rq_stride) + 4);</div><div class='del'>-	MLX5_SET(wq, wq, log_wq_pg_sz, MLX5_GET(qpc, qpc, log_page_size));</div><div class='add'>+	MLX5_SET(wq, wq, log_wq_pg_sz,</div><div class='add'>+		 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);</div><div class='ctx'> 	MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_rq_size));</div><div class='ctx'> </div><div class='ctx'> 	pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);</div><div class='del'>-	qp_pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, qpin, pas);</div><div class='del'>-	memcpy(pas, qp_pas, rq_pas_size);</div><div class='add'>+	mlx5_ib_populate_pas(umem, page_size, pas, 0);</div><div class='ctx'> </div><div class='del'>-	err = mlx5_core_create_rq_tracked(dev-&gt;mdev, in, inlen, &amp;rq-&gt;base.mqp);</div><div class='add'>+	err = mlx5_core_create_rq_tracked(dev, in, inlen, &amp;rq-&gt;base.mqp);</div><div class='ctx'> </div><div class='ctx'> 	kvfree(in);</div><div class='ctx'> </div><div class='hunk'>@@ -1379,14 +1500,7 @@ static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,</div><div class='ctx'> static void destroy_raw_packet_qp_rq(struct mlx5_ib_dev *dev,</div><div class='ctx'> 				     struct mlx5_ib_rq *rq)</div><div class='ctx'> {</div><div class='del'>-	mlx5_core_destroy_rq_tracked(dev-&gt;mdev, &amp;rq-&gt;base.mqp);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static bool tunnel_offload_supported(struct mlx5_core_dev *dev)</div><div class='del'>-{</div><div class='del'>-	return  (MLX5_CAP_ETH(dev, tunnel_stateless_vxlan) ||</div><div class='del'>-		 MLX5_CAP_ETH(dev, tunnel_stateless_gre) ||</div><div class='del'>-		 MLX5_CAP_ETH(dev, tunnel_stateless_geneve_rx));</div><div class='add'>+	mlx5_core_destroy_rq_tracked(dev, &amp;rq-&gt;base.mqp);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,</div><div class='hunk'>@@ -1402,9 +1516,8 @@ static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,</div><div class='ctx'> </div><div class='ctx'> static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,</div><div class='ctx'> 				    struct mlx5_ib_rq *rq, u32 tdn,</div><div class='del'>-				    u32 *qp_flags_en,</div><div class='del'>-				    struct ib_pd *pd,</div><div class='del'>-				    u32 *out, int outlen)</div><div class='add'>+				    u32 *qp_flags_en, struct ib_pd *pd,</div><div class='add'>+				    u32 *out)</div><div class='ctx'> {</div><div class='ctx'> 	u8 lb_flag = 0;</div><div class='ctx'> 	u32 *in;</div><div class='hunk'>@@ -1437,9 +1550,8 @@ static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(tirc, tirc, self_lb_block, lb_flag);</div><div class='del'>-</div><div class='del'>-	err = mlx5_core_create_tir_out(dev-&gt;mdev, in, inlen, out, outlen);</div><div class='del'>-</div><div class='add'>+	MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);</div><div class='add'>+	err = mlx5_cmd_exec_inout(dev-&gt;mdev, create_tir, in, out);</div><div class='ctx'> 	rq-&gt;tirn = MLX5_GET(create_tir_out, out, tirn);</div><div class='ctx'> 	if (!err &amp;&amp; MLX5_GET(tirc, tirc, self_lb_block)) {</div><div class='ctx'> 		err = mlx5_ib_enable_lb(dev, false, true);</div><div class='hunk'>@@ -1453,10 +1565,10 @@ static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='del'>-				u32 *in, size_t inlen,</div><div class='del'>-				struct ib_pd *pd,</div><div class='add'>+				u32 *in, size_t inlen, struct ib_pd *pd,</div><div class='ctx'> 				struct ib_udata *udata,</div><div class='del'>-				struct mlx5_ib_create_qp_resp *resp)</div><div class='add'>+				struct mlx5_ib_create_qp_resp *resp,</div><div class='add'>+				struct ib_qp_init_attr *init_attr)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &amp;qp-&gt;raw_packet_qp;</div><div class='ctx'> 	struct mlx5_ib_sq *sq = &amp;raw_packet_qp-&gt;sq;</div><div class='hunk'>@@ -1468,12 +1580,15 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 	u16 uid = to_mpd(pd)-&gt;uid;</div><div class='ctx'> 	u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};</div><div class='ctx'> </div><div class='add'>+	if (!qp-&gt;sq.wqe_cnt &amp;&amp; !qp-&gt;rq.wqe_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	if (qp-&gt;sq.wqe_cnt) {</div><div class='ctx'> 		err = create_raw_packet_qp_tis(dev, qp, sq, tdn, pd);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			return err;</div><div class='ctx'> </div><div class='del'>-		err = create_raw_packet_qp_sq(dev, udata, sq, in, pd);</div><div class='add'>+		err = create_raw_packet_qp_sq(dev, udata, sq, in, pd,</div><div class='add'>+					      to_mcq(init_attr-&gt;send_cq));</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto err_destroy_tis;</div><div class='ctx'> </div><div class='hunk'>@@ -1491,17 +1606,17 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 	if (qp-&gt;rq.wqe_cnt) {</div><div class='ctx'> 		rq-&gt;base.container_mibqp = qp;</div><div class='ctx'> </div><div class='del'>-		if (qp-&gt;flags &amp; MLX5_IB_QP_CVLAN_STRIPPING)</div><div class='add'>+		if (qp-&gt;flags &amp; IB_QP_CREATE_CVLAN_STRIPPING)</div><div class='ctx'> 			rq-&gt;flags |= MLX5_IB_RQ_CVLAN_STRIPPING;</div><div class='del'>-		if (qp-&gt;flags &amp; MLX5_IB_QP_PCI_WRITE_END_PADDING)</div><div class='add'>+		if (qp-&gt;flags &amp; IB_QP_CREATE_PCI_WRITE_END_PADDING)</div><div class='ctx'> 			rq-&gt;flags |= MLX5_IB_RQ_PCI_WRITE_END_PADDING;</div><div class='del'>-		err = create_raw_packet_qp_rq(dev, rq, in, inlen, pd);</div><div class='add'>+		err = create_raw_packet_qp_rq(dev, rq, in, pd,</div><div class='add'>+					      to_mcq(init_attr-&gt;recv_cq));</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto err_destroy_sq;</div><div class='ctx'> </div><div class='del'>-		err = create_raw_packet_qp_tir(</div><div class='del'>-			dev, rq, tdn, &amp;qp-&gt;flags_en, pd, out,</div><div class='del'>-			MLX5_ST_SZ_BYTES(create_tir_out));</div><div class='add'>+		err = create_raw_packet_qp_tir(dev, rq, tdn, &amp;qp-&gt;flags_en, pd,</div><div class='add'>+					       out);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto err_destroy_rq;</div><div class='ctx'> </div><div class='hunk'>@@ -1510,7 +1625,8 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 			resp-&gt;comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_RQN;</div><div class='ctx'> 			resp-&gt;tirn = rq-&gt;tirn;</div><div class='ctx'> 			resp-&gt;comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;</div><div class='del'>-			if (MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev, sw_owner)) {</div><div class='add'>+			if (MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev, sw_owner) ||</div><div class='add'>+			    MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev, sw_owner_v2)) {</div><div class='ctx'> 				resp-&gt;tir_icm_addr = MLX5_GET(</div><div class='ctx'> 					create_tir_out, out, icm_address_31_0);</div><div class='ctx'> 				resp-&gt;tir_icm_addr |=</div><div class='hunk'>@@ -1529,14 +1645,8 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> </div><div class='ctx'> 	qp-&gt;trans_qp.base.mqp.qpn = qp-&gt;sq.wqe_cnt ? sq-&gt;base.mqp.qpn :</div><div class='ctx'> 						     rq-&gt;base.mqp.qpn;</div><div class='del'>-	err = ib_copy_to_udata(udata, resp, min(udata-&gt;outlen, sizeof(*resp)));</div><div class='del'>-	if (err)</div><div class='del'>-		goto err_destroy_tir;</div><div class='del'>-</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-err_destroy_tir:</div><div class='del'>-	destroy_raw_packet_qp_tir(dev, rq, qp-&gt;flags_en, pd);</div><div class='ctx'> err_destroy_rq:</div><div class='ctx'> 	destroy_raw_packet_qp_rq(dev, rq);</div><div class='ctx'> err_destroy_sq:</div><div class='hunk'>@@ -1588,14 +1698,27 @@ static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *q</div><div class='ctx'> 			     to_mpd(qp-&gt;ibqp.pd)-&gt;uid);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='del'>-				 struct ib_pd *pd,</div><div class='del'>-				 struct ib_qp_init_attr *init_attr,</div><div class='del'>-				 struct ib_udata *udata)</div><div class='add'>+struct mlx5_create_qp_params {</div><div class='add'>+	struct ib_udata *udata;</div><div class='add'>+	size_t inlen;</div><div class='add'>+	size_t outlen;</div><div class='add'>+	size_t ucmd_size;</div><div class='add'>+	void *ucmd;</div><div class='add'>+	u8 is_rss_raw : 1;</div><div class='add'>+	struct ib_qp_init_attr *attr;</div><div class='add'>+	u32 uidx;</div><div class='add'>+	struct mlx5_ib_create_qp_resp resp;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='add'>+				 struct mlx5_ib_qp *qp,</div><div class='add'>+				 struct mlx5_create_qp_params *params)</div><div class='ctx'> {</div><div class='add'>+	struct ib_qp_init_attr *init_attr = params-&gt;attr;</div><div class='add'>+	struct mlx5_ib_create_qp_rss *ucmd = params-&gt;ucmd;</div><div class='add'>+	struct ib_udata *udata = params-&gt;udata;</div><div class='ctx'> 	struct mlx5_ib_ucontext *mucontext = rdma_udata_to_drv_context(</div><div class='ctx'> 		udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='del'>-	struct mlx5_ib_create_qp_resp resp = {};</div><div class='ctx'> 	int inlen;</div><div class='ctx'> 	int outlen;</div><div class='ctx'> 	int err;</div><div class='hunk'>@@ -1605,79 +1728,28 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 	void *hfso;</div><div class='ctx'> 	u32 selected_fields = 0;</div><div class='ctx'> 	u32 outer_l4;</div><div class='del'>-	size_t min_resp_len;</div><div class='ctx'> 	u32 tdn = mucontext-&gt;tdn;</div><div class='del'>-	struct mlx5_ib_create_qp_rss ucmd = {};</div><div class='del'>-	size_t required_cmd_sz;</div><div class='ctx'> 	u8 lb_flag = 0;</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;qp_type != IB_QPT_RAW_PACKET)</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-</div><div class='del'>-	if (init_attr-&gt;create_flags || init_attr-&gt;send_cq)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	min_resp_len = offsetof(typeof(resp), bfreg_index) + sizeof(resp.bfreg_index);</div><div class='del'>-	if (udata-&gt;outlen &lt; min_resp_len)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	required_cmd_sz = offsetof(typeof(ucmd), flags) + sizeof(ucmd.flags);</div><div class='del'>-	if (udata-&gt;inlen &lt; required_cmd_sz) {</div><div class='del'>-		mlx5_ib_dbg(dev, "invalid inlen\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (udata-&gt;inlen &gt; sizeof(ucmd) &amp;&amp;</div><div class='del'>-	    !ib_is_udata_cleared(udata, sizeof(ucmd),</div><div class='del'>-				 udata-&gt;inlen - sizeof(ucmd))) {</div><div class='del'>-		mlx5_ib_dbg(dev, "inlen is not supported\n");</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (ib_copy_from_udata(&amp;ucmd, udata, min(sizeof(ucmd), udata-&gt;inlen))) {</div><div class='del'>-		mlx5_ib_dbg(dev, "copy failed\n");</div><div class='del'>-		return -EFAULT;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (ucmd.comp_mask) {</div><div class='add'>+	if (ucmd-&gt;comp_mask) {</div><div class='ctx'> 		mlx5_ib_dbg(dev, "invalid comp mask\n");</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (ucmd.flags &amp; ~(MLX5_QP_FLAG_TUNNEL_OFFLOADS |</div><div class='del'>-			   MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |</div><div class='del'>-			   MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)) {</div><div class='del'>-		mlx5_ib_dbg(dev, "invalid flags\n");</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (ucmd.flags &amp; MLX5_QP_FLAG_TUNNEL_OFFLOADS &amp;&amp;</div><div class='del'>-	    !tunnel_offload_supported(dev-&gt;mdev)) {</div><div class='del'>-		mlx5_ib_dbg(dev, "tunnel offloads isn't supported\n");</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_INNER &amp;&amp;</div><div class='del'>-	    !(ucmd.flags &amp; MLX5_QP_FLAG_TUNNEL_OFFLOADS)) {</div><div class='add'>+	if (ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_INNER &amp;&amp;</div><div class='add'>+	    !(ucmd-&gt;flags &amp; MLX5_QP_FLAG_TUNNEL_OFFLOADS)) {</div><div class='ctx'> 		mlx5_ib_dbg(dev, "Tunnel offloads must be set for inner RSS\n");</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (ucmd.flags &amp; MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC || dev-&gt;is_rep) {</div><div class='del'>-		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;</div><div class='add'>+	if (dev-&gt;is_rep)</div><div class='ctx'> 		qp-&gt;flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='del'>-	if (ucmd.flags &amp; MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC) {</div><div class='del'>-		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;</div><div class='del'>-		qp-&gt;flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC;</div><div class='del'>-	}</div><div class='add'>+	if (qp-&gt;flags_en &amp; MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC)</div><div class='add'>+		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;</div><div class='ctx'> </div><div class='del'>-	err = ib_copy_to_udata(udata, &amp;resp, min(udata-&gt;outlen, sizeof(resp)));</div><div class='del'>-	if (err) {</div><div class='del'>-		mlx5_ib_dbg(dev, "copy failed\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='add'>+	if (qp-&gt;flags_en &amp; MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)</div><div class='add'>+		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;</div><div class='ctx'> </div><div class='ctx'> 	inlen = MLX5_ST_SZ_BYTES(create_tir_in);</div><div class='ctx'> 	outlen = MLX5_ST_SZ_BYTES(create_tir_out);</div><div class='hunk'>@@ -1696,30 +1768,29 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> </div><div class='ctx'> 	hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);</div><div class='ctx'> </div><div class='del'>-	if (ucmd.flags &amp; MLX5_QP_FLAG_TUNNEL_OFFLOADS)</div><div class='add'>+	if (ucmd-&gt;flags &amp; MLX5_QP_FLAG_TUNNEL_OFFLOADS)</div><div class='ctx'> 		MLX5_SET(tirc, tirc, tunneled_offload_en, 1);</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(tirc, tirc, self_lb_block, lb_flag);</div><div class='ctx'> </div><div class='del'>-	if (ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_INNER)</div><div class='add'>+	if (ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_INNER)</div><div class='ctx'> 		hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner);</div><div class='ctx'> 	else</div><div class='ctx'> 		hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);</div><div class='ctx'> </div><div class='del'>-	switch (ucmd.rx_hash_function) {</div><div class='add'>+	switch (ucmd-&gt;rx_hash_function) {</div><div class='ctx'> 	case MLX5_RX_HASH_FUNC_TOEPLITZ:</div><div class='ctx'> 	{</div><div class='ctx'> 		void *rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);</div><div class='ctx'> 		size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key);</div><div class='ctx'> </div><div class='del'>-		if (len != ucmd.rx_key_len) {</div><div class='add'>+		if (len != ucmd-&gt;rx_key_len) {</div><div class='ctx'> 			err = -EINVAL;</div><div class='ctx'> 			goto err;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='ctx'> 		MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ);</div><div class='del'>-		MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);</div><div class='del'>-		memcpy(rss_key, ucmd.rx_hash_key, len);</div><div class='add'>+		memcpy(rss_key, ucmd-&gt;rx_hash_key, len);</div><div class='ctx'> 		break;</div><div class='ctx'> 	}</div><div class='ctx'> 	default:</div><div class='hunk'>@@ -1727,7 +1798,7 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 		goto err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!ucmd.rx_hash_fields_mask) {</div><div class='add'>+	if (!ucmd-&gt;rx_hash_fields_mask) {</div><div class='ctx'> 		/* special case when this TIR serves as steering entry without hashing */</div><div class='ctx'> 		if (!init_attr-&gt;rwq_ind_tbl-&gt;log_ind_tbl_size)</div><div class='ctx'> 			goto create_tir;</div><div class='hunk'>@@ -1735,29 +1806,31 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 		goto err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (((ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_IPV4) ||</div><div class='del'>-	     (ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_IPV4)) &amp;&amp;</div><div class='del'>-	     ((ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_IPV6) ||</div><div class='del'>-	     (ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_IPV6))) {</div><div class='add'>+	if (((ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_IPV4) ||</div><div class='add'>+	     (ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_IPV4)) &amp;&amp;</div><div class='add'>+	     ((ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_IPV6) ||</div><div class='add'>+	     (ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_IPV6))) {</div><div class='ctx'> 		err = -EINVAL;</div><div class='ctx'> 		goto err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* If none of IPV4 &amp; IPV6 SRC/DST was set - this bit field is ignored */</div><div class='del'>-	if ((ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_IPV4) ||</div><div class='del'>-	    (ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_IPV4))</div><div class='add'>+	if ((ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_IPV4) ||</div><div class='add'>+	    (ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_IPV4))</div><div class='ctx'> 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,</div><div class='ctx'> 			 MLX5_L3_PROT_TYPE_IPV4);</div><div class='del'>-	else if ((ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_IPV6) ||</div><div class='del'>-		 (ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_IPV6))</div><div class='add'>+	else if ((ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_IPV6) ||</div><div class='add'>+		 (ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_IPV6))</div><div class='ctx'> 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,</div><div class='ctx'> 			 MLX5_L3_PROT_TYPE_IPV6);</div><div class='ctx'> </div><div class='del'>-	outer_l4 = ((ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_PORT_TCP) ||</div><div class='del'>-		    (ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_PORT_TCP)) &lt;&lt; 0 |</div><div class='del'>-		   ((ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_PORT_UDP) ||</div><div class='del'>-		    (ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_PORT_UDP)) &lt;&lt; 1 |</div><div class='del'>-		   (ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_IPSEC_SPI) &lt;&lt; 2;</div><div class='add'>+	outer_l4 = ((ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_PORT_TCP) ||</div><div class='add'>+		    (ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_PORT_TCP))</div><div class='add'>+			   &lt;&lt; 0 |</div><div class='add'>+		   ((ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_PORT_UDP) ||</div><div class='add'>+		    (ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_PORT_UDP))</div><div class='add'>+			   &lt;&lt; 1 |</div><div class='add'>+		   (ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_IPSEC_SPI) &lt;&lt; 2;</div><div class='ctx'> </div><div class='ctx'> 	/* Check that only one l4 protocol is set */</div><div class='ctx'> 	if (outer_l4 &amp; (outer_l4 - 1)) {</div><div class='hunk'>@@ -1766,38 +1839,39 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* If none of TCP &amp; UDP SRC/DST was set - this bit field is ignored */</div><div class='del'>-	if ((ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_PORT_TCP) ||</div><div class='del'>-	    (ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_PORT_TCP))</div><div class='add'>+	if ((ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_PORT_TCP) ||</div><div class='add'>+	    (ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_PORT_TCP))</div><div class='ctx'> 		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,</div><div class='ctx'> 			 MLX5_L4_PROT_TYPE_TCP);</div><div class='del'>-	else if ((ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_PORT_UDP) ||</div><div class='del'>-		 (ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_PORT_UDP))</div><div class='add'>+	else if ((ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_PORT_UDP) ||</div><div class='add'>+		 (ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_PORT_UDP))</div><div class='ctx'> 		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,</div><div class='ctx'> 			 MLX5_L4_PROT_TYPE_UDP);</div><div class='ctx'> </div><div class='del'>-	if ((ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_IPV4) ||</div><div class='del'>-	    (ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_IPV6))</div><div class='add'>+	if ((ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_IPV4) ||</div><div class='add'>+	    (ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_IPV6))</div><div class='ctx'> 		selected_fields |= MLX5_HASH_FIELD_SEL_SRC_IP;</div><div class='ctx'> </div><div class='del'>-	if ((ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_IPV4) ||</div><div class='del'>-	    (ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_IPV6))</div><div class='add'>+	if ((ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_IPV4) ||</div><div class='add'>+	    (ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_IPV6))</div><div class='ctx'> 		selected_fields |= MLX5_HASH_FIELD_SEL_DST_IP;</div><div class='ctx'> </div><div class='del'>-	if ((ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_PORT_TCP) ||</div><div class='del'>-	    (ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_PORT_UDP))</div><div class='add'>+	if ((ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_PORT_TCP) ||</div><div class='add'>+	    (ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_SRC_PORT_UDP))</div><div class='ctx'> 		selected_fields |= MLX5_HASH_FIELD_SEL_L4_SPORT;</div><div class='ctx'> </div><div class='del'>-	if ((ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_PORT_TCP) ||</div><div class='del'>-	    (ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_PORT_UDP))</div><div class='add'>+	if ((ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_PORT_TCP) ||</div><div class='add'>+	    (ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_DST_PORT_UDP))</div><div class='ctx'> 		selected_fields |= MLX5_HASH_FIELD_SEL_L4_DPORT;</div><div class='ctx'> </div><div class='del'>-	if (ucmd.rx_hash_fields_mask &amp; MLX5_RX_HASH_IPSEC_SPI)</div><div class='add'>+	if (ucmd-&gt;rx_hash_fields_mask &amp; MLX5_RX_HASH_IPSEC_SPI)</div><div class='ctx'> 		selected_fields |= MLX5_HASH_FIELD_SEL_IPSEC_SPI;</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(rx_hash_field_select, hfso, selected_fields, selected_fields);</div><div class='ctx'> </div><div class='ctx'> create_tir:</div><div class='del'>-	err = mlx5_core_create_tir_out(dev-&gt;mdev, in, inlen, out, outlen);</div><div class='add'>+	MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);</div><div class='add'>+	err = mlx5_cmd_exec_inout(dev-&gt;mdev, create_tir, in, out);</div><div class='ctx'> </div><div class='ctx'> 	qp-&gt;rss_qp.tirn = MLX5_GET(create_tir_out, out, tirn);</div><div class='ctx'> 	if (!err &amp;&amp; MLX5_GET(tirc, tirc, self_lb_block)) {</div><div class='hunk'>@@ -1812,75 +1886,45 @@ create_tir:</div><div class='ctx'> 		goto err;</div><div class='ctx'> </div><div class='ctx'> 	if (mucontext-&gt;devx_uid) {</div><div class='del'>-		resp.comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;</div><div class='del'>-		resp.tirn = qp-&gt;rss_qp.tirn;</div><div class='del'>-		if (MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev, sw_owner)) {</div><div class='del'>-			resp.tir_icm_addr =</div><div class='add'>+		params-&gt;resp.comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;</div><div class='add'>+		params-&gt;resp.tirn = qp-&gt;rss_qp.tirn;</div><div class='add'>+		if (MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev, sw_owner) ||</div><div class='add'>+		    MLX5_CAP_FLOWTABLE_NIC_RX(dev-&gt;mdev, sw_owner_v2)) {</div><div class='add'>+			params-&gt;resp.tir_icm_addr =</div><div class='ctx'> 				MLX5_GET(create_tir_out, out, icm_address_31_0);</div><div class='del'>-			resp.tir_icm_addr |= (u64)MLX5_GET(create_tir_out, out,</div><div class='del'>-							   icm_address_39_32)</div><div class='del'>-					     &lt;&lt; 32;</div><div class='del'>-			resp.tir_icm_addr |= (u64)MLX5_GET(create_tir_out, out,</div><div class='del'>-							   icm_address_63_40)</div><div class='del'>-					     &lt;&lt; 40;</div><div class='del'>-			resp.comp_mask |=</div><div class='add'>+			params-&gt;resp.tir_icm_addr |=</div><div class='add'>+				(u64)MLX5_GET(create_tir_out, out,</div><div class='add'>+					      icm_address_39_32)</div><div class='add'>+				&lt;&lt; 32;</div><div class='add'>+			params-&gt;resp.tir_icm_addr |=</div><div class='add'>+				(u64)MLX5_GET(create_tir_out, out,</div><div class='add'>+					      icm_address_63_40)</div><div class='add'>+				&lt;&lt; 40;</div><div class='add'>+			params-&gt;resp.comp_mask |=</div><div class='ctx'> 				MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	err = ib_copy_to_udata(udata, &amp;resp, min(udata-&gt;outlen, sizeof(resp)));</div><div class='del'>-	if (err)</div><div class='del'>-		goto err_copy;</div><div class='del'>-</div><div class='ctx'> 	kvfree(in);</div><div class='ctx'> 	/* qpn is reserved for that QP */</div><div class='ctx'> 	qp-&gt;trans_qp.base.mqp.qpn = 0;</div><div class='del'>-	qp-&gt;flags |= MLX5_IB_QP_RSS;</div><div class='add'>+	qp-&gt;is_rss = true;</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-err_copy:</div><div class='del'>-	mlx5_cmd_destroy_tir(dev-&gt;mdev, qp-&gt;rss_qp.tirn, mucontext-&gt;devx_uid);</div><div class='ctx'> err:</div><div class='ctx'> 	kvfree(in);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static void configure_responder_scat_cqe(struct ib_qp_init_attr *init_attr,</div><div class='del'>-					 void *qpc)</div><div class='del'>-{</div><div class='del'>-	int rcqe_sz;</div><div class='del'>-</div><div class='del'>-	if (init_attr-&gt;qp_type == MLX5_IB_QPT_DCI)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	rcqe_sz = mlx5_ib_get_cqe_size(init_attr-&gt;recv_cq);</div><div class='del'>-</div><div class='del'>-	if (init_attr-&gt;qp_type == MLX5_IB_QPT_DCT) {</div><div class='del'>-		if (rcqe_sz == 128)</div><div class='del'>-			MLX5_SET(dctc, qpc, cs_res, MLX5_RES_SCAT_DATA64_CQE);</div><div class='del'>-</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	MLX5_SET(qpc, qpc, cs_res,</div><div class='del'>-		 rcqe_sz == 128 ? MLX5_RES_SCAT_DATA64_CQE :</div><div class='del'>-				  MLX5_RES_SCAT_DATA32_CQE);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static void configure_requester_scat_cqe(struct mlx5_ib_dev *dev,</div><div class='add'>+					 struct mlx5_ib_qp *qp,</div><div class='ctx'> 					 struct ib_qp_init_attr *init_attr,</div><div class='del'>-					 struct mlx5_ib_create_qp *ucmd,</div><div class='ctx'> 					 void *qpc)</div><div class='ctx'> {</div><div class='del'>-	enum ib_qp_type qpt = init_attr-&gt;qp_type;</div><div class='ctx'> 	int scqe_sz;</div><div class='del'>-	bool allow_scat_cqe = 0;</div><div class='add'>+	bool allow_scat_cqe = false;</div><div class='ctx'> </div><div class='del'>-	if (qpt == IB_QPT_UC || qpt == IB_QPT_UD)</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	if (ucmd)</div><div class='del'>-		allow_scat_cqe = ucmd-&gt;flags &amp; MLX5_QP_FLAG_ALLOW_SCATTER_CQE;</div><div class='add'>+	allow_scat_cqe = qp-&gt;flags_en &amp; MLX5_QP_FLAG_ALLOW_SCATTER_CQE;</div><div class='ctx'> </div><div class='ctx'> 	if (!allow_scat_cqe &amp;&amp; init_attr-&gt;sq_sig_type != IB_SIGNAL_ALL_WR)</div><div class='ctx'> 		return;</div><div class='hunk'>@@ -1916,7 +1960,7 @@ static int atomic_size_to_mode(int size_mask)</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int get_atomic_mode(struct mlx5_ib_dev *dev,</div><div class='del'>-			   enum ib_qp_type qp_type)</div><div class='add'>+			   struct mlx5_ib_qp *qp)</div><div class='ctx'> {</div><div class='ctx'> 	u8 atomic_operations = MLX5_CAP_ATOMIC(dev-&gt;mdev, atomic_operations);</div><div class='ctx'> 	u8 atomic = MLX5_CAP_GEN(dev-&gt;mdev, atomic);</div><div class='hunk'>@@ -1926,7 +1970,7 @@ static int get_atomic_mode(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	if (!atomic)</div><div class='ctx'> 		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='del'>-	if (qp_type == MLX5_IB_QPT_DCT)</div><div class='add'>+	if (qp-&gt;type == MLX5_IB_QPT_DCT)</div><div class='ctx'> 		atomic_size_mask = MLX5_CAP_ATOMIC(dev-&gt;mdev, atomic_size_dc);</div><div class='ctx'> 	else</div><div class='ctx'> 		atomic_size_mask = MLX5_CAP_ATOMIC(dev-&gt;mdev, atomic_size_qp);</div><div class='hunk'>@@ -1940,277 +1984,362 @@ static int get_atomic_mode(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	     atomic_operations &amp; MLX5_ATOMIC_OPS_FETCH_ADD))</div><div class='ctx'> 		atomic_mode = MLX5_ATOMIC_MODE_IB_COMP;</div><div class='ctx'> </div><div class='add'>+	/* OOO DP QPs do not support larger than 8-Bytes atomic operations */</div><div class='add'>+	if (atomic_mode &gt; MLX5_ATOMIC_MODE_8B &amp;&amp; qp-&gt;is_ooo_rq)</div><div class='add'>+		atomic_mode = MLX5_ATOMIC_MODE_8B;</div><div class='add'>+</div><div class='ctx'> 	return atomic_mode;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static inline bool check_flags_mask(uint64_t input, uint64_t supported)</div><div class='add'>+static int create_xrc_tgt_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='add'>+			     struct mlx5_create_qp_params *params)</div><div class='ctx'> {</div><div class='del'>-	return (input &amp; ~supported) == 0;</div><div class='add'>+	struct ib_qp_init_attr *attr = params-&gt;attr;</div><div class='add'>+	u32 uidx = params-&gt;uidx;</div><div class='add'>+	struct mlx5_ib_resources *devr = &amp;dev-&gt;devr;</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};</div><div class='add'>+	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);</div><div class='add'>+	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='add'>+	struct mlx5_ib_qp_base *base;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	void *qpc;</div><div class='add'>+	u32 *in;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;sq_sig_type == IB_SIGNAL_ALL_WR)</div><div class='add'>+		qp-&gt;sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;</div><div class='add'>+</div><div class='add'>+	in = kvzalloc(inlen, GFP_KERNEL);</div><div class='add'>+	if (!in)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);</div><div class='add'>+</div><div class='add'>+	MLX5_SET(qpc, qpc, st, MLX5_QP_ST_XRC);</div><div class='add'>+	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);</div><div class='add'>+	MLX5_SET(qpc, qpc, pd, to_mpd(devr-&gt;p0)-&gt;pdn);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)</div><div class='add'>+		MLX5_SET(qpc, qpc, block_lb_mc, 1);</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_CROSS_CHANNEL)</div><div class='add'>+		MLX5_SET(qpc, qpc, cd_master, 1);</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_MANAGED_SEND)</div><div class='add'>+		MLX5_SET(qpc, qpc, cd_slave_send, 1);</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_MANAGED_RECV)</div><div class='add'>+		MLX5_SET(qpc, qpc, cd_slave_receive, 1);</div><div class='add'>+</div><div class='add'>+	MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(dev-&gt;mdev));</div><div class='add'>+	MLX5_SET(qpc, qpc, rq_type, MLX5_SRQ_RQ);</div><div class='add'>+	MLX5_SET(qpc, qpc, no_sq, 1);</div><div class='add'>+	MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr-&gt;c0)-&gt;mcq.cqn);</div><div class='add'>+	MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr-&gt;c0)-&gt;mcq.cqn);</div><div class='add'>+	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr-&gt;s0)-&gt;msrq.srqn);</div><div class='add'>+	MLX5_SET(qpc, qpc, xrcd, to_mxrcd(attr-&gt;xrcd)-&gt;xrcdn);</div><div class='add'>+	MLX5_SET64(qpc, qpc, dbr_addr, qp-&gt;db.dma);</div><div class='add'>+</div><div class='add'>+	/* 0xffffff means we ask to work with cqe version 0 */</div><div class='add'>+	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)</div><div class='add'>+		MLX5_SET(qpc, qpc, user_index, uidx);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_PCI_WRITE_END_PADDING) {</div><div class='add'>+		MLX5_SET(qpc, qpc, end_padding_mode,</div><div class='add'>+			 MLX5_WQ_END_PAD_MODE_ALIGN);</div><div class='add'>+		/* Special case to clean flag */</div><div class='add'>+		qp-&gt;flags &amp;= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	base = &amp;qp-&gt;trans_qp.base;</div><div class='add'>+	err = mlx5_qpc_create_qp(dev, &amp;base-&gt;mqp, in, inlen, out);</div><div class='add'>+	kvfree(in);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	base-&gt;container_mibqp = qp;</div><div class='add'>+	base-&gt;mqp.event = mlx5_ib_qp_event;</div><div class='add'>+	if (MLX5_CAP_GEN(mdev, ece_support))</div><div class='add'>+		params-&gt;resp.ece_options = MLX5_GET(create_qp_out, out, ece);</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;dev-&gt;reset_flow_resource_lock, flags);</div><div class='add'>+	list_add_tail(&amp;qp-&gt;qps_list, &amp;dev-&gt;qp_list);</div><div class='add'>+	spin_unlock_irqrestore(&amp;dev-&gt;reset_flow_resource_lock, flags);</div><div class='add'>+</div><div class='add'>+	qp-&gt;trans_qp.xrcdn = to_mxrcd(attr-&gt;xrcd)-&gt;xrcdn;</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='del'>-			    struct ib_qp_init_attr *init_attr,</div><div class='del'>-			    struct ib_udata *udata, struct mlx5_ib_qp *qp)</div><div class='add'>+static int create_dci(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='add'>+		      struct mlx5_ib_qp *qp,</div><div class='add'>+		      struct mlx5_create_qp_params *params)</div><div class='ctx'> {</div><div class='add'>+	struct ib_qp_init_attr *init_attr = params-&gt;attr;</div><div class='add'>+	struct mlx5_ib_create_qp *ucmd = params-&gt;ucmd;</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};</div><div class='add'>+	struct ib_udata *udata = params-&gt;udata;</div><div class='add'>+	u32 uidx = params-&gt;uidx;</div><div class='ctx'> 	struct mlx5_ib_resources *devr = &amp;dev-&gt;devr;</div><div class='ctx'> 	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);</div><div class='ctx'> 	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='del'>-	struct mlx5_ib_create_qp_resp resp = {};</div><div class='del'>-	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(</div><div class='del'>-		udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='ctx'> 	struct mlx5_ib_cq *send_cq;</div><div class='ctx'> 	struct mlx5_ib_cq *recv_cq;</div><div class='ctx'> 	unsigned long flags;</div><div class='del'>-	u32 uidx = MLX5_IB_DEFAULT_UIDX;</div><div class='del'>-	struct mlx5_ib_create_qp ucmd;</div><div class='ctx'> 	struct mlx5_ib_qp_base *base;</div><div class='add'>+	int ts_format;</div><div class='ctx'> 	int mlx5_st;</div><div class='ctx'> 	void *qpc;</div><div class='ctx'> 	u32 *in;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	mutex_init(&amp;qp-&gt;mutex);</div><div class='ctx'> 	spin_lock_init(&amp;qp-&gt;sq.lock);</div><div class='ctx'> 	spin_lock_init(&amp;qp-&gt;rq.lock);</div><div class='ctx'> </div><div class='del'>-	mlx5_st = to_mlx5_st(init_attr-&gt;qp_type);</div><div class='add'>+	mlx5_st = to_mlx5_st(qp-&gt;type);</div><div class='ctx'> 	if (mlx5_st &lt; 0)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;rwq_ind_tbl) {</div><div class='del'>-		if (!udata)</div><div class='del'>-			return -ENOSYS;</div><div class='add'>+	if (init_attr-&gt;sq_sig_type == IB_SIGNAL_ALL_WR)</div><div class='add'>+		qp-&gt;sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;</div><div class='add'>+</div><div class='add'>+	base = &amp;qp-&gt;trans_qp.base;</div><div class='ctx'> </div><div class='del'>-		err = create_rss_raw_qp_tir(dev, qp, pd, init_attr, udata);</div><div class='add'>+	qp-&gt;has_rq = qp_has_rq(init_attr);</div><div class='add'>+	err = set_rq_size(dev, &amp;init_attr-&gt;cap, qp-&gt;has_rq, qp, ucmd);</div><div class='add'>+	if (err) {</div><div class='add'>+		mlx5_ib_dbg(dev, "err %d\n", err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;create_flags &amp; IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {</div><div class='del'>-		if (!MLX5_CAP_GEN(mdev, block_lb_mc)) {</div><div class='del'>-			mlx5_ib_dbg(dev, "block multicast loopback isn't supported\n");</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		} else {</div><div class='del'>-			qp-&gt;flags |= MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK;</div><div class='del'>-		}</div><div class='add'>+	if (ucmd-&gt;rq_wqe_shift != qp-&gt;rq.wqe_shift ||</div><div class='add'>+	    ucmd-&gt;rq_wqe_count != qp-&gt;rq.wqe_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (ucmd-&gt;sq_wqe_count &gt; (1 &lt;&lt; MLX5_CAP_GEN(mdev, log_max_qp_sz)))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	ts_format = get_qp_ts_format(dev, to_mcq(init_attr-&gt;send_cq),</div><div class='add'>+				     to_mcq(init_attr-&gt;recv_cq));</div><div class='add'>+</div><div class='add'>+	if (ts_format &lt; 0)</div><div class='add'>+		return ts_format;</div><div class='add'>+</div><div class='add'>+	err = _create_user_qp(dev, pd, qp, udata, init_attr, &amp;in, &amp;params-&gt;resp,</div><div class='add'>+			      &amp;inlen, base, ucmd);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	if (MLX5_CAP_GEN(mdev, ece_support))</div><div class='add'>+		MLX5_SET(create_qp_in, in, ece, ucmd-&gt;ece_options);</div><div class='add'>+	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);</div><div class='add'>+</div><div class='add'>+	MLX5_SET(qpc, qpc, st, mlx5_st);</div><div class='add'>+	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);</div><div class='add'>+	MLX5_SET(qpc, qpc, pd, to_mpd(pd)-&gt;pdn);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;flags_en &amp; MLX5_QP_FLAG_SIGNATURE)</div><div class='add'>+		MLX5_SET(qpc, qpc, wq_signature, 1);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_CROSS_CHANNEL)</div><div class='add'>+		MLX5_SET(qpc, qpc, cd_master, 1);</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_MANAGED_SEND)</div><div class='add'>+		MLX5_SET(qpc, qpc, cd_slave_send, 1);</div><div class='add'>+	if (qp-&gt;flags_en &amp; MLX5_QP_FLAG_SCATTER_CQE)</div><div class='add'>+		configure_requester_scat_cqe(dev, qp, init_attr, qpc);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;rq.wqe_cnt) {</div><div class='add'>+		MLX5_SET(qpc, qpc, log_rq_stride, qp-&gt;rq.wqe_shift - 4);</div><div class='add'>+		MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp-&gt;rq.wqe_cnt));</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;create_flags &amp;</div><div class='del'>-			(IB_QP_CREATE_CROSS_CHANNEL |</div><div class='del'>-			 IB_QP_CREATE_MANAGED_SEND |</div><div class='del'>-			 IB_QP_CREATE_MANAGED_RECV)) {</div><div class='del'>-		if (!MLX5_CAP_GEN(mdev, cd)) {</div><div class='del'>-			mlx5_ib_dbg(dev, "cross-channel isn't supported\n");</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-		if (init_attr-&gt;create_flags &amp; IB_QP_CREATE_CROSS_CHANNEL)</div><div class='del'>-			qp-&gt;flags |= MLX5_IB_QP_CROSS_CHANNEL;</div><div class='del'>-		if (init_attr-&gt;create_flags &amp; IB_QP_CREATE_MANAGED_SEND)</div><div class='del'>-			qp-&gt;flags |= MLX5_IB_QP_MANAGED_SEND;</div><div class='del'>-		if (init_attr-&gt;create_flags &amp; IB_QP_CREATE_MANAGED_RECV)</div><div class='del'>-			qp-&gt;flags |= MLX5_IB_QP_MANAGED_RECV;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (init_attr-&gt;qp_type == IB_QPT_UD &amp;&amp;</div><div class='del'>-	    (init_attr-&gt;create_flags &amp; IB_QP_CREATE_IPOIB_UD_LSO))</div><div class='del'>-		if (!MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {</div><div class='del'>-			mlx5_ib_dbg(dev, "ipoib UD lso qp isn't supported\n");</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-		}</div><div class='add'>+	if (qp-&gt;flags_en &amp; MLX5_QP_FLAG_DCI_STREAM) {</div><div class='add'>+		MLX5_SET(qpc, qpc, log_num_dci_stream_channels,</div><div class='add'>+			 ucmd-&gt;dci_streams.log_num_concurent);</div><div class='add'>+		MLX5_SET(qpc, qpc, log_num_dci_errored_streams,</div><div class='add'>+			 ucmd-&gt;dci_streams.log_num_errored);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;create_flags &amp; IB_QP_CREATE_SCATTER_FCS) {</div><div class='del'>-		if (init_attr-&gt;qp_type != IB_QPT_RAW_PACKET) {</div><div class='del'>-			mlx5_ib_dbg(dev, "Scatter FCS is supported only for Raw Packet QPs");</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-		}</div><div class='del'>-		if (!MLX5_CAP_GEN(dev-&gt;mdev, eth_net_offloads) ||</div><div class='del'>-		    !MLX5_CAP_ETH(dev-&gt;mdev, scatter_fcs)) {</div><div class='del'>-			mlx5_ib_dbg(dev, "Scatter FCS isn't supported\n");</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-		}</div><div class='del'>-		qp-&gt;flags |= MLX5_IB_QP_CAP_SCATTER_FCS;</div><div class='add'>+	MLX5_SET(qpc, qpc, ts_format, ts_format);</div><div class='add'>+	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));</div><div class='add'>+</div><div class='add'>+	MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp-&gt;sq.wqe_cnt));</div><div class='add'>+</div><div class='add'>+	/* Set default resources */</div><div class='add'>+	if (init_attr-&gt;srq) {</div><div class='add'>+		MLX5_SET(qpc, qpc, xrcd, devr-&gt;xrcdn0);</div><div class='add'>+		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,</div><div class='add'>+			 to_msrq(init_attr-&gt;srq)-&gt;msrq.srqn);</div><div class='add'>+	} else {</div><div class='add'>+		MLX5_SET(qpc, qpc, xrcd, devr-&gt;xrcdn1);</div><div class='add'>+		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,</div><div class='add'>+			 to_msrq(devr-&gt;s1)-&gt;msrq.srqn);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;sq_sig_type == IB_SIGNAL_ALL_WR)</div><div class='del'>-		qp-&gt;sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;</div><div class='add'>+	if (init_attr-&gt;send_cq)</div><div class='add'>+		MLX5_SET(qpc, qpc, cqn_snd,</div><div class='add'>+			 to_mcq(init_attr-&gt;send_cq)-&gt;mcq.cqn);</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;create_flags &amp; IB_QP_CREATE_CVLAN_STRIPPING) {</div><div class='del'>-		if (!(MLX5_CAP_GEN(dev-&gt;mdev, eth_net_offloads) &amp;&amp;</div><div class='del'>-		      MLX5_CAP_ETH(dev-&gt;mdev, vlan_cap)) ||</div><div class='del'>-		    (init_attr-&gt;qp_type != IB_QPT_RAW_PACKET))</div><div class='del'>-			return -EOPNOTSUPP;</div><div class='del'>-		qp-&gt;flags |= MLX5_IB_QP_CVLAN_STRIPPING;</div><div class='add'>+	if (init_attr-&gt;recv_cq)</div><div class='add'>+		MLX5_SET(qpc, qpc, cqn_rcv,</div><div class='add'>+			 to_mcq(init_attr-&gt;recv_cq)-&gt;mcq.cqn);</div><div class='add'>+</div><div class='add'>+	MLX5_SET64(qpc, qpc, dbr_addr, qp-&gt;db.dma);</div><div class='add'>+</div><div class='add'>+	/* 0xffffff means we ask to work with cqe version 0 */</div><div class='add'>+	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)</div><div class='add'>+		MLX5_SET(qpc, qpc, user_index, uidx);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_PCI_WRITE_END_PADDING) {</div><div class='add'>+		MLX5_SET(qpc, qpc, end_padding_mode,</div><div class='add'>+			 MLX5_WQ_END_PAD_MODE_ALIGN);</div><div class='add'>+		/* Special case to clean flag */</div><div class='add'>+		qp-&gt;flags &amp;= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (udata) {</div><div class='del'>-		if (ib_copy_from_udata(&amp;ucmd, udata, sizeof(ucmd))) {</div><div class='del'>-			mlx5_ib_dbg(dev, "copy failed\n");</div><div class='del'>-			return -EFAULT;</div><div class='del'>-		}</div><div class='add'>+	err = mlx5_qpc_create_qp(dev, &amp;base-&gt;mqp, in, inlen, out);</div><div class='ctx'> </div><div class='del'>-		if (!check_flags_mask(ucmd.flags,</div><div class='del'>-				      MLX5_QP_FLAG_ALLOW_SCATTER_CQE |</div><div class='del'>-				      MLX5_QP_FLAG_BFREG_INDEX |</div><div class='del'>-				      MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE |</div><div class='del'>-				      MLX5_QP_FLAG_SCATTER_CQE |</div><div class='del'>-				      MLX5_QP_FLAG_SIGNATURE |</div><div class='del'>-				      MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC |</div><div class='del'>-				      MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |</div><div class='del'>-				      MLX5_QP_FLAG_TUNNEL_OFFLOADS |</div><div class='del'>-				      MLX5_QP_FLAG_TYPE_DCI |</div><div class='del'>-				      MLX5_QP_FLAG_TYPE_DCT))</div><div class='del'>-			return -EINVAL;</div><div class='add'>+	kvfree(in);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_create;</div><div class='ctx'> </div><div class='del'>-		err = get_qp_user_index(ucontext, &amp;ucmd, udata-&gt;inlen, &amp;uidx);</div><div class='del'>-		if (err)</div><div class='del'>-			return err;</div><div class='add'>+	base-&gt;container_mibqp = qp;</div><div class='add'>+	base-&gt;mqp.event = mlx5_ib_qp_event;</div><div class='add'>+	if (MLX5_CAP_GEN(mdev, ece_support))</div><div class='add'>+		params-&gt;resp.ece_options = MLX5_GET(create_qp_out, out, ece);</div><div class='ctx'> </div><div class='del'>-		qp-&gt;wq_sig = !!(ucmd.flags &amp; MLX5_QP_FLAG_SIGNATURE);</div><div class='del'>-		if (MLX5_CAP_GEN(dev-&gt;mdev, sctr_data_cqe))</div><div class='del'>-			qp-&gt;scat_cqe = !!(ucmd.flags &amp; MLX5_QP_FLAG_SCATTER_CQE);</div><div class='del'>-		if (ucmd.flags &amp; MLX5_QP_FLAG_TUNNEL_OFFLOADS) {</div><div class='del'>-			if (init_attr-&gt;qp_type != IB_QPT_RAW_PACKET ||</div><div class='del'>-			    !tunnel_offload_supported(mdev)) {</div><div class='del'>-				mlx5_ib_dbg(dev, "Tunnel offload isn't supported\n");</div><div class='del'>-				return -EOPNOTSUPP;</div><div class='del'>-			}</div><div class='del'>-			qp-&gt;flags_en |= MLX5_QP_FLAG_TUNNEL_OFFLOADS;</div><div class='del'>-		}</div><div class='add'>+	get_cqs(qp-&gt;type, init_attr-&gt;send_cq, init_attr-&gt;recv_cq,</div><div class='add'>+		&amp;send_cq, &amp;recv_cq);</div><div class='add'>+	spin_lock_irqsave(&amp;dev-&gt;reset_flow_resource_lock, flags);</div><div class='add'>+	mlx5_ib_lock_cqs(send_cq, recv_cq);</div><div class='add'>+	/* Maintain device to QPs access, needed for further handling via reset</div><div class='add'>+	 * flow</div><div class='add'>+	 */</div><div class='add'>+	list_add_tail(&amp;qp-&gt;qps_list, &amp;dev-&gt;qp_list);</div><div class='add'>+	/* Maintain CQ to QPs access, needed for further handling via reset flow</div><div class='add'>+	 */</div><div class='add'>+	if (send_cq)</div><div class='add'>+		list_add_tail(&amp;qp-&gt;cq_send_list, &amp;send_cq-&gt;list_send_qp);</div><div class='add'>+	if (recv_cq)</div><div class='add'>+		list_add_tail(&amp;qp-&gt;cq_recv_list, &amp;recv_cq-&gt;list_recv_qp);</div><div class='add'>+	mlx5_ib_unlock_cqs(send_cq, recv_cq);</div><div class='add'>+	spin_unlock_irqrestore(&amp;dev-&gt;reset_flow_resource_lock, flags);</div><div class='ctx'> </div><div class='del'>-		if (ucmd.flags &amp; MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC) {</div><div class='del'>-			if (init_attr-&gt;qp_type != IB_QPT_RAW_PACKET) {</div><div class='del'>-				mlx5_ib_dbg(dev, "Self-LB UC isn't supported\n");</div><div class='del'>-				return -EOPNOTSUPP;</div><div class='del'>-			}</div><div class='del'>-			qp-&gt;flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;</div><div class='del'>-		}</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='del'>-		if (ucmd.flags &amp; MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC) {</div><div class='del'>-			if (init_attr-&gt;qp_type != IB_QPT_RAW_PACKET) {</div><div class='del'>-				mlx5_ib_dbg(dev, "Self-LB UM isn't supported\n");</div><div class='del'>-				return -EOPNOTSUPP;</div><div class='del'>-			}</div><div class='del'>-			qp-&gt;flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC;</div><div class='del'>-		}</div><div class='add'>+err_create:</div><div class='add'>+	destroy_qp(dev, qp, base, udata);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-		if (ucmd.flags &amp; MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE) {</div><div class='del'>-			if (init_attr-&gt;qp_type != IB_QPT_RC ||</div><div class='del'>-				!MLX5_CAP_GEN(dev-&gt;mdev, qp_packet_based)) {</div><div class='del'>-				mlx5_ib_dbg(dev, "packet based credit mode isn't supported\n");</div><div class='del'>-				return -EOPNOTSUPP;</div><div class='del'>-			}</div><div class='del'>-			qp-&gt;flags |= MLX5_IB_QP_PACKET_BASED_CREDIT;</div><div class='del'>-		}</div><div class='add'>+static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='add'>+			  struct mlx5_ib_qp *qp,</div><div class='add'>+			  struct mlx5_create_qp_params *params)</div><div class='add'>+{</div><div class='add'>+	struct ib_qp_init_attr *init_attr = params-&gt;attr;</div><div class='add'>+	struct mlx5_ib_create_qp *ucmd = params-&gt;ucmd;</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};</div><div class='add'>+	struct ib_udata *udata = params-&gt;udata;</div><div class='add'>+	u32 uidx = params-&gt;uidx;</div><div class='add'>+	struct mlx5_ib_resources *devr = &amp;dev-&gt;devr;</div><div class='add'>+	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);</div><div class='add'>+	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='add'>+	struct mlx5_ib_cq *send_cq;</div><div class='add'>+	struct mlx5_ib_cq *recv_cq;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	struct mlx5_ib_qp_base *base;</div><div class='add'>+	int ts_format;</div><div class='add'>+	int mlx5_st;</div><div class='add'>+	void *qpc;</div><div class='add'>+	u32 *in;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='del'>-		if (init_attr-&gt;create_flags &amp; IB_QP_CREATE_SOURCE_QPN) {</div><div class='del'>-			if (init_attr-&gt;qp_type != IB_QPT_UD ||</div><div class='del'>-			    (MLX5_CAP_GEN(dev-&gt;mdev, port_type) !=</div><div class='del'>-			     MLX5_CAP_PORT_TYPE_IB) ||</div><div class='del'>-			    !mlx5_get_flow_namespace(dev-&gt;mdev, MLX5_FLOW_NAMESPACE_BYPASS)) {</div><div class='del'>-				mlx5_ib_dbg(dev, "Source QP option isn't supported\n");</div><div class='del'>-				return -EOPNOTSUPP;</div><div class='del'>-			}</div><div class='add'>+	spin_lock_init(&amp;qp-&gt;sq.lock);</div><div class='add'>+	spin_lock_init(&amp;qp-&gt;rq.lock);</div><div class='ctx'> </div><div class='del'>-			qp-&gt;flags |= MLX5_IB_QP_UNDERLAY;</div><div class='del'>-			qp-&gt;underlay_qpn = init_attr-&gt;source_qpn;</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		qp-&gt;wq_sig = !!wq_signature;</div><div class='del'>-	}</div><div class='add'>+	mlx5_st = to_mlx5_st(qp-&gt;type);</div><div class='add'>+	if (mlx5_st &lt; 0)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (init_attr-&gt;sq_sig_type == IB_SIGNAL_ALL_WR)</div><div class='add'>+		qp-&gt;sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_SOURCE_QPN)</div><div class='add'>+		qp-&gt;underlay_qpn = init_attr-&gt;source_qpn;</div><div class='ctx'> </div><div class='ctx'> 	base = (init_attr-&gt;qp_type == IB_QPT_RAW_PACKET ||</div><div class='del'>-		qp-&gt;flags &amp; MLX5_IB_QP_UNDERLAY) ?</div><div class='add'>+		qp-&gt;flags &amp; IB_QP_CREATE_SOURCE_QPN) ?</div><div class='ctx'> 	       &amp;qp-&gt;raw_packet_qp.rq.base :</div><div class='ctx'> 	       &amp;qp-&gt;trans_qp.base;</div><div class='ctx'> </div><div class='ctx'> 	qp-&gt;has_rq = qp_has_rq(init_attr);</div><div class='del'>-	err = set_rq_size(dev, &amp;init_attr-&gt;cap, qp-&gt;has_rq,</div><div class='del'>-			  qp, udata ? &amp;ucmd : NULL);</div><div class='add'>+	err = set_rq_size(dev, &amp;init_attr-&gt;cap, qp-&gt;has_rq, qp, ucmd);</div><div class='ctx'> 	if (err) {</div><div class='ctx'> 		mlx5_ib_dbg(dev, "err %d\n", err);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (pd) {</div><div class='del'>-		if (udata) {</div><div class='del'>-			__u32 max_wqes =</div><div class='del'>-				1 &lt;&lt; MLX5_CAP_GEN(mdev, log_max_qp_sz);</div><div class='del'>-			mlx5_ib_dbg(dev, "requested sq_wqe_count (%d)\n", ucmd.sq_wqe_count);</div><div class='del'>-			if (ucmd.rq_wqe_shift != qp-&gt;rq.wqe_shift ||</div><div class='del'>-			    ucmd.rq_wqe_count != qp-&gt;rq.wqe_cnt) {</div><div class='del'>-				mlx5_ib_dbg(dev, "invalid rq params\n");</div><div class='del'>-				return -EINVAL;</div><div class='del'>-			}</div><div class='del'>-			if (ucmd.sq_wqe_count &gt; max_wqes) {</div><div class='del'>-				mlx5_ib_dbg(dev, "requested sq_wqe_count (%d) &gt; max allowed (%d)\n",</div><div class='del'>-					    ucmd.sq_wqe_count, max_wqes);</div><div class='del'>-				return -EINVAL;</div><div class='del'>-			}</div><div class='del'>-			if (init_attr-&gt;create_flags &amp;</div><div class='del'>-			    mlx5_ib_create_qp_sqpn_qp1()) {</div><div class='del'>-				mlx5_ib_dbg(dev, "user-space is not allowed to create UD QPs spoofing as QP1\n");</div><div class='del'>-				return -EINVAL;</div><div class='del'>-			}</div><div class='del'>-			err = create_user_qp(dev, pd, qp, udata, init_attr, &amp;in,</div><div class='del'>-					     &amp;resp, &amp;inlen, base);</div><div class='del'>-			if (err)</div><div class='del'>-				mlx5_ib_dbg(dev, "err %d\n", err);</div><div class='del'>-		} else {</div><div class='del'>-			err = create_kernel_qp(dev, init_attr, qp, &amp;in, &amp;inlen,</div><div class='del'>-					       base);</div><div class='del'>-			if (err)</div><div class='del'>-				mlx5_ib_dbg(dev, "err %d\n", err);</div><div class='del'>-		}</div><div class='add'>+	if (ucmd-&gt;rq_wqe_shift != qp-&gt;rq.wqe_shift ||</div><div class='add'>+	    ucmd-&gt;rq_wqe_count != qp-&gt;rq.wqe_cnt)</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-		if (err)</div><div class='del'>-			return err;</div><div class='del'>-	} else {</div><div class='del'>-		in = kvzalloc(inlen, GFP_KERNEL);</div><div class='del'>-		if (!in)</div><div class='del'>-			return -ENOMEM;</div><div class='add'>+	if (ucmd-&gt;sq_wqe_count &gt; (1 &lt;&lt; MLX5_CAP_GEN(mdev, log_max_qp_sz)))</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-		qp-&gt;create_type = MLX5_QP_EMPTY;</div><div class='add'>+	if (init_attr-&gt;qp_type != IB_QPT_RAW_PACKET) {</div><div class='add'>+		ts_format = get_qp_ts_format(dev, to_mcq(init_attr-&gt;send_cq),</div><div class='add'>+					     to_mcq(init_attr-&gt;recv_cq));</div><div class='add'>+		if (ts_format &lt; 0)</div><div class='add'>+			return ts_format;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	err = _create_user_qp(dev, pd, qp, udata, init_attr, &amp;in, &amp;params-&gt;resp,</div><div class='add'>+			      &amp;inlen, base, ucmd);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='ctx'> 	if (is_sqp(init_attr-&gt;qp_type))</div><div class='ctx'> 		qp-&gt;port = init_attr-&gt;port_num;</div><div class='ctx'> </div><div class='add'>+	if (MLX5_CAP_GEN(mdev, ece_support))</div><div class='add'>+		MLX5_SET(create_qp_in, in, ece, ucmd-&gt;ece_options);</div><div class='ctx'> 	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);</div><div class='ctx'> </div><div class='ctx'> 	MLX5_SET(qpc, qpc, st, mlx5_st);</div><div class='ctx'> 	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);</div><div class='add'>+	MLX5_SET(qpc, qpc, pd, to_mpd(pd)-&gt;pdn);</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;qp_type != MLX5_IB_QPT_REG_UMR)</div><div class='del'>-		MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr-&gt;p0)-&gt;pdn);</div><div class='del'>-	else</div><div class='del'>-		MLX5_SET(qpc, qpc, latency_sensitive, 1);</div><div class='del'>-</div><div class='del'>-</div><div class='del'>-	if (qp-&gt;wq_sig)</div><div class='add'>+	if (qp-&gt;flags_en &amp; MLX5_QP_FLAG_SIGNATURE)</div><div class='ctx'> 		MLX5_SET(qpc, qpc, wq_signature, 1);</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;flags &amp; MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)</div><div class='ctx'> 		MLX5_SET(qpc, qpc, block_lb_mc, 1);</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;flags &amp; MLX5_IB_QP_CROSS_CHANNEL)</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_CROSS_CHANNEL)</div><div class='ctx'> 		MLX5_SET(qpc, qpc, cd_master, 1);</div><div class='del'>-	if (qp-&gt;flags &amp; MLX5_IB_QP_MANAGED_SEND)</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_MANAGED_SEND)</div><div class='ctx'> 		MLX5_SET(qpc, qpc, cd_slave_send, 1);</div><div class='del'>-	if (qp-&gt;flags &amp; MLX5_IB_QP_MANAGED_RECV)</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_MANAGED_RECV)</div><div class='ctx'> 		MLX5_SET(qpc, qpc, cd_slave_receive, 1);</div><div class='del'>-	if (qp-&gt;flags &amp; MLX5_IB_QP_PACKET_BASED_CREDIT)</div><div class='add'>+	if (qp-&gt;flags_en &amp; MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE)</div><div class='ctx'> 		MLX5_SET(qpc, qpc, req_e2e_credit_mode, 1);</div><div class='del'>-	if (qp-&gt;scat_cqe &amp;&amp; is_connected(init_attr-&gt;qp_type)) {</div><div class='del'>-		configure_responder_scat_cqe(init_attr, qpc);</div><div class='del'>-		configure_requester_scat_cqe(dev, init_attr,</div><div class='del'>-					     udata ? &amp;ucmd : NULL,</div><div class='del'>-					     qpc);</div><div class='add'>+	if ((qp-&gt;flags_en &amp; MLX5_QP_FLAG_SCATTER_CQE) &amp;&amp;</div><div class='add'>+	    (init_attr-&gt;qp_type == IB_QPT_RC ||</div><div class='add'>+	     init_attr-&gt;qp_type == IB_QPT_UC)) {</div><div class='add'>+		int rcqe_sz = mlx5_ib_get_cqe_size(init_attr-&gt;recv_cq);</div><div class='add'>+</div><div class='add'>+		MLX5_SET(qpc, qpc, cs_res,</div><div class='add'>+			 rcqe_sz == 128 ? MLX5_RES_SCAT_DATA64_CQE :</div><div class='add'>+					  MLX5_RES_SCAT_DATA32_CQE);</div><div class='ctx'> 	}</div><div class='add'>+	if ((qp-&gt;flags_en &amp; MLX5_QP_FLAG_SCATTER_CQE) &amp;&amp;</div><div class='add'>+	    (qp-&gt;type == MLX5_IB_QPT_DCI || qp-&gt;type == IB_QPT_RC))</div><div class='add'>+		configure_requester_scat_cqe(dev, qp, init_attr, qpc);</div><div class='ctx'> </div><div class='ctx'> 	if (qp-&gt;rq.wqe_cnt) {</div><div class='ctx'> 		MLX5_SET(qpc, qpc, log_rq_stride, qp-&gt;rq.wqe_shift - 4);</div><div class='ctx'> 		MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp-&gt;rq.wqe_cnt));</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if (init_attr-&gt;qp_type != IB_QPT_RAW_PACKET)</div><div class='add'>+		MLX5_SET(qpc, qpc, ts_format, ts_format);</div><div class='add'>+</div><div class='ctx'> 	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));</div><div class='ctx'> </div><div class='ctx'> 	if (qp-&gt;sq.wqe_cnt) {</div><div class='hunk'>@@ -2225,23 +2354,17 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='ctx'> </div><div class='ctx'> 	/* Set default resources */</div><div class='ctx'> 	switch (init_attr-&gt;qp_type) {</div><div class='del'>-	case IB_QPT_XRC_TGT:</div><div class='del'>-		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr-&gt;c0)-&gt;mcq.cqn);</div><div class='del'>-		MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr-&gt;c0)-&gt;mcq.cqn);</div><div class='del'>-		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr-&gt;s0)-&gt;msrq.srqn);</div><div class='del'>-		MLX5_SET(qpc, qpc, xrcd, to_mxrcd(init_attr-&gt;xrcd)-&gt;xrcdn);</div><div class='del'>-		break;</div><div class='ctx'> 	case IB_QPT_XRC_INI:</div><div class='ctx'> 		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr-&gt;c0)-&gt;mcq.cqn);</div><div class='del'>-		MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr-&gt;x1)-&gt;xrcdn);</div><div class='add'>+		MLX5_SET(qpc, qpc, xrcd, devr-&gt;xrcdn1);</div><div class='ctx'> 		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr-&gt;s0)-&gt;msrq.srqn);</div><div class='ctx'> 		break;</div><div class='ctx'> 	default:</div><div class='ctx'> 		if (init_attr-&gt;srq) {</div><div class='del'>-			MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr-&gt;x0)-&gt;xrcdn);</div><div class='add'>+			MLX5_SET(qpc, qpc, xrcd, devr-&gt;xrcdn0);</div><div class='ctx'> 			MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr-&gt;srq)-&gt;msrq.srqn);</div><div class='ctx'> 		} else {</div><div class='del'>-			MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr-&gt;x1)-&gt;xrcdn);</div><div class='add'>+			MLX5_SET(qpc, qpc, xrcd, devr-&gt;xrcdn1);</div><div class='ctx'> 			MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr-&gt;s1)-&gt;msrq.srqn);</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='hunk'>@@ -2258,52 +2381,166 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='ctx'> 	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)</div><div class='ctx'> 		MLX5_SET(qpc, qpc, user_index, uidx);</div><div class='ctx'> </div><div class='del'>-	/* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */</div><div class='del'>-	if (init_attr-&gt;qp_type == IB_QPT_UD &amp;&amp;</div><div class='del'>-	    (init_attr-&gt;create_flags &amp; IB_QP_CREATE_IPOIB_UD_LSO)) {</div><div class='del'>-		MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);</div><div class='del'>-		qp-&gt;flags |= MLX5_IB_QP_LSO;</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_PCI_WRITE_END_PADDING &amp;&amp;</div><div class='add'>+	    init_attr-&gt;qp_type != IB_QPT_RAW_PACKET) {</div><div class='add'>+		MLX5_SET(qpc, qpc, end_padding_mode,</div><div class='add'>+			 MLX5_WQ_END_PAD_MODE_ALIGN);</div><div class='add'>+		/* Special case to clean flag */</div><div class='add'>+		qp-&gt;flags &amp;= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;create_flags &amp; IB_QP_CREATE_PCI_WRITE_END_PADDING) {</div><div class='del'>-		if (!MLX5_CAP_GEN(dev-&gt;mdev, end_pad)) {</div><div class='del'>-			mlx5_ib_dbg(dev, "scatter end padding is not supported\n");</div><div class='del'>-			err = -EOPNOTSUPP;</div><div class='del'>-			goto err;</div><div class='del'>-		} else if (init_attr-&gt;qp_type != IB_QPT_RAW_PACKET) {</div><div class='del'>-			MLX5_SET(qpc, qpc, end_padding_mode,</div><div class='del'>-				 MLX5_WQ_END_PAD_MODE_ALIGN);</div><div class='del'>-		} else {</div><div class='del'>-			qp-&gt;flags |= MLX5_IB_QP_PCI_WRITE_END_PADDING;</div><div class='del'>-		}</div><div class='add'>+	if (init_attr-&gt;qp_type == IB_QPT_RAW_PACKET ||</div><div class='add'>+	    qp-&gt;flags &amp; IB_QP_CREATE_SOURCE_QPN) {</div><div class='add'>+		qp-&gt;raw_packet_qp.sq.ubuffer.buf_addr = ucmd-&gt;sq_buf_addr;</div><div class='add'>+		raw_packet_qp_copy_info(qp, &amp;qp-&gt;raw_packet_qp);</div><div class='add'>+		err = create_raw_packet_qp(dev, qp, in, inlen, pd, udata,</div><div class='add'>+					   &amp;params-&gt;resp, init_attr);</div><div class='add'>+	} else</div><div class='add'>+		err = mlx5_qpc_create_qp(dev, &amp;base-&gt;mqp, in, inlen, out);</div><div class='add'>+</div><div class='add'>+	kvfree(in);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_create;</div><div class='add'>+</div><div class='add'>+	base-&gt;container_mibqp = qp;</div><div class='add'>+	base-&gt;mqp.event = mlx5_ib_qp_event;</div><div class='add'>+	if (MLX5_CAP_GEN(mdev, ece_support))</div><div class='add'>+		params-&gt;resp.ece_options = MLX5_GET(create_qp_out, out, ece);</div><div class='add'>+</div><div class='add'>+	get_cqs(qp-&gt;type, init_attr-&gt;send_cq, init_attr-&gt;recv_cq,</div><div class='add'>+		&amp;send_cq, &amp;recv_cq);</div><div class='add'>+	spin_lock_irqsave(&amp;dev-&gt;reset_flow_resource_lock, flags);</div><div class='add'>+	mlx5_ib_lock_cqs(send_cq, recv_cq);</div><div class='add'>+	/* Maintain device to QPs access, needed for further handling via reset</div><div class='add'>+	 * flow</div><div class='add'>+	 */</div><div class='add'>+	list_add_tail(&amp;qp-&gt;qps_list, &amp;dev-&gt;qp_list);</div><div class='add'>+	/* Maintain CQ to QPs access, needed for further handling via reset flow</div><div class='add'>+	 */</div><div class='add'>+	if (send_cq)</div><div class='add'>+		list_add_tail(&amp;qp-&gt;cq_send_list, &amp;send_cq-&gt;list_send_qp);</div><div class='add'>+	if (recv_cq)</div><div class='add'>+		list_add_tail(&amp;qp-&gt;cq_recv_list, &amp;recv_cq-&gt;list_recv_qp);</div><div class='add'>+	mlx5_ib_unlock_cqs(send_cq, recv_cq);</div><div class='add'>+	spin_unlock_irqrestore(&amp;dev-&gt;reset_flow_resource_lock, flags);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_create:</div><div class='add'>+	destroy_qp(dev, qp, base, udata);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int create_kernel_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='add'>+			    struct mlx5_ib_qp *qp,</div><div class='add'>+			    struct mlx5_create_qp_params *params)</div><div class='add'>+{</div><div class='add'>+	struct ib_qp_init_attr *attr = params-&gt;attr;</div><div class='add'>+	u32 uidx = params-&gt;uidx;</div><div class='add'>+	struct mlx5_ib_resources *devr = &amp;dev-&gt;devr;</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};</div><div class='add'>+	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);</div><div class='add'>+	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='add'>+	struct mlx5_ib_cq *send_cq;</div><div class='add'>+	struct mlx5_ib_cq *recv_cq;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	struct mlx5_ib_qp_base *base;</div><div class='add'>+	int mlx5_st;</div><div class='add'>+	void *qpc;</div><div class='add'>+	u32 *in;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;qp-&gt;sq.lock);</div><div class='add'>+	spin_lock_init(&amp;qp-&gt;rq.lock);</div><div class='add'>+</div><div class='add'>+	mlx5_st = to_mlx5_st(qp-&gt;type);</div><div class='add'>+	if (mlx5_st &lt; 0)</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;sq_sig_type == IB_SIGNAL_ALL_WR)</div><div class='add'>+		qp-&gt;sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;</div><div class='add'>+</div><div class='add'>+	base = &amp;qp-&gt;trans_qp.base;</div><div class='add'>+</div><div class='add'>+	qp-&gt;has_rq = qp_has_rq(attr);</div><div class='add'>+	err = set_rq_size(dev, &amp;attr-&gt;cap, qp-&gt;has_rq, qp, NULL);</div><div class='add'>+	if (err) {</div><div class='add'>+		mlx5_ib_dbg(dev, "err %d\n", err);</div><div class='add'>+		return err;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (inlen &lt; 0) {</div><div class='del'>-		err = -EINVAL;</div><div class='del'>-		goto err;</div><div class='add'>+	err = _create_kernel_qp(dev, attr, qp, &amp;in, &amp;inlen, base);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	if (is_sqp(attr-&gt;qp_type))</div><div class='add'>+		qp-&gt;port = attr-&gt;port_num;</div><div class='add'>+</div><div class='add'>+	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);</div><div class='add'>+</div><div class='add'>+	MLX5_SET(qpc, qpc, st, mlx5_st);</div><div class='add'>+	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;qp_type != MLX5_IB_QPT_REG_UMR)</div><div class='add'>+		MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr-&gt;p0)-&gt;pdn);</div><div class='add'>+	else</div><div class='add'>+		MLX5_SET(qpc, qpc, latency_sensitive, 1);</div><div class='add'>+</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)</div><div class='add'>+		MLX5_SET(qpc, qpc, block_lb_mc, 1);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;rq.wqe_cnt) {</div><div class='add'>+		MLX5_SET(qpc, qpc, log_rq_stride, qp-&gt;rq.wqe_shift - 4);</div><div class='add'>+		MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp-&gt;rq.wqe_cnt));</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;qp_type == IB_QPT_RAW_PACKET ||</div><div class='del'>-	    qp-&gt;flags &amp; MLX5_IB_QP_UNDERLAY) {</div><div class='del'>-		qp-&gt;raw_packet_qp.sq.ubuffer.buf_addr = ucmd.sq_buf_addr;</div><div class='del'>-		raw_packet_qp_copy_info(qp, &amp;qp-&gt;raw_packet_qp);</div><div class='del'>-		err = create_raw_packet_qp(dev, qp, in, inlen, pd, udata,</div><div class='del'>-					   &amp;resp);</div><div class='add'>+	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, attr));</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;sq.wqe_cnt)</div><div class='add'>+		MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp-&gt;sq.wqe_cnt));</div><div class='add'>+	else</div><div class='add'>+		MLX5_SET(qpc, qpc, no_sq, 1);</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;srq) {</div><div class='add'>+		MLX5_SET(qpc, qpc, xrcd, devr-&gt;xrcdn0);</div><div class='add'>+		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,</div><div class='add'>+			 to_msrq(attr-&gt;srq)-&gt;msrq.srqn);</div><div class='ctx'> 	} else {</div><div class='del'>-		err = mlx5_core_create_qp(dev-&gt;mdev, &amp;base-&gt;mqp, in, inlen);</div><div class='add'>+		MLX5_SET(qpc, qpc, xrcd, devr-&gt;xrcdn1);</div><div class='add'>+		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,</div><div class='add'>+			 to_msrq(devr-&gt;s1)-&gt;msrq.srqn);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (err) {</div><div class='del'>-		mlx5_ib_dbg(dev, "create qp failed\n");</div><div class='del'>-		goto err_create;</div><div class='del'>-	}</div><div class='add'>+	if (attr-&gt;send_cq)</div><div class='add'>+		MLX5_SET(qpc, qpc, cqn_snd, to_mcq(attr-&gt;send_cq)-&gt;mcq.cqn);</div><div class='add'>+</div><div class='add'>+	if (attr-&gt;recv_cq)</div><div class='add'>+		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(attr-&gt;recv_cq)-&gt;mcq.cqn);</div><div class='add'>+</div><div class='add'>+	MLX5_SET64(qpc, qpc, dbr_addr, qp-&gt;db.dma);</div><div class='add'>+</div><div class='add'>+	/* 0xffffff means we ask to work with cqe version 0 */</div><div class='add'>+	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)</div><div class='add'>+		MLX5_SET(qpc, qpc, user_index, uidx);</div><div class='add'>+</div><div class='add'>+	/* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_IPOIB_UD_LSO)</div><div class='add'>+		MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_INTEGRITY_EN &amp;&amp;</div><div class='add'>+	    MLX5_CAP_GEN(mdev, go_back_n))</div><div class='add'>+		MLX5_SET(qpc, qpc, retry_mode, MLX5_QP_RM_GO_BACK_N);</div><div class='ctx'> </div><div class='add'>+	err = mlx5_qpc_create_qp(dev, &amp;base-&gt;mqp, in, inlen, out);</div><div class='ctx'> 	kvfree(in);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_create;</div><div class='ctx'> </div><div class='ctx'> 	base-&gt;container_mibqp = qp;</div><div class='ctx'> 	base-&gt;mqp.event = mlx5_ib_qp_event;</div><div class='ctx'> </div><div class='del'>-	get_cqs(init_attr-&gt;qp_type, init_attr-&gt;send_cq, init_attr-&gt;recv_cq,</div><div class='add'>+	get_cqs(qp-&gt;type, attr-&gt;send_cq, attr-&gt;recv_cq,</div><div class='ctx'> 		&amp;send_cq, &amp;recv_cq);</div><div class='ctx'> 	spin_lock_irqsave(&amp;dev-&gt;reset_flow_resource_lock, flags);</div><div class='ctx'> 	mlx5_ib_lock_cqs(send_cq, recv_cq);</div><div class='hunk'>@@ -2323,13 +2560,7 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='ctx'> err_create:</div><div class='del'>-	if (qp-&gt;create_type == MLX5_QP_USER)</div><div class='del'>-		destroy_qp_user(dev, pd, qp, base, udata);</div><div class='del'>-	else if (qp-&gt;create_type == MLX5_QP_KERNEL)</div><div class='del'>-		destroy_qp_kernel(dev, qp);</div><div class='del'>-</div><div class='del'>-err:</div><div class='del'>-	kvfree(in);</div><div class='add'>+	destroy_qp(dev, qp, base, NULL);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -2391,11 +2622,6 @@ static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *re</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct mlx5_ib_pd *get_pd(struct mlx5_ib_qp *qp)</div><div class='del'>-{</div><div class='del'>-	return to_mpd(qp-&gt;ibqp.pd);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static void get_cqs(enum ib_qp_type qp_type,</div><div class='ctx'> 		    struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,</div><div class='ctx'> 		    struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq)</div><div class='hunk'>@@ -2416,14 +2642,10 @@ static void get_cqs(enum ib_qp_type qp_type,</div><div class='ctx'> 	case IB_QPT_RC:</div><div class='ctx'> 	case IB_QPT_UC:</div><div class='ctx'> 	case IB_QPT_UD:</div><div class='del'>-	case IB_QPT_RAW_IPV6:</div><div class='del'>-	case IB_QPT_RAW_ETHERTYPE:</div><div class='ctx'> 	case IB_QPT_RAW_PACKET:</div><div class='ctx'> 		*send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;</div><div class='ctx'> 		*recv_cq = ib_recv_cq ? to_mcq(ib_recv_cq) : NULL;</div><div class='ctx'> 		break;</div><div class='del'>-</div><div class='del'>-	case IB_QPT_MAX:</div><div class='ctx'> 	default:</div><div class='ctx'> 		*send_cq = NULL;</div><div class='ctx'> 		*recv_cq = NULL;</div><div class='hunk'>@@ -2443,22 +2665,21 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 	unsigned long flags;</div><div class='ctx'> 	int err;</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;ibqp.rwq_ind_tbl) {</div><div class='add'>+	if (qp-&gt;is_rss) {</div><div class='ctx'> 		destroy_rss_raw_qp_tir(dev, qp);</div><div class='ctx'> 		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	base = (qp-&gt;ibqp.qp_type == IB_QPT_RAW_PACKET ||</div><div class='del'>-		qp-&gt;flags &amp; MLX5_IB_QP_UNDERLAY) ?</div><div class='del'>-	       &amp;qp-&gt;raw_packet_qp.rq.base :</div><div class='del'>-	       &amp;qp-&gt;trans_qp.base;</div><div class='add'>+	base = (qp-&gt;type == IB_QPT_RAW_PACKET ||</div><div class='add'>+		qp-&gt;flags &amp; IB_QP_CREATE_SOURCE_QPN) ?</div><div class='add'>+		       &amp;qp-&gt;raw_packet_qp.rq.base :</div><div class='add'>+		       &amp;qp-&gt;trans_qp.base;</div><div class='ctx'> </div><div class='ctx'> 	if (qp-&gt;state != IB_QPS_RESET) {</div><div class='del'>-		if (qp-&gt;ibqp.qp_type != IB_QPT_RAW_PACKET &amp;&amp;</div><div class='del'>-		    !(qp-&gt;flags &amp; MLX5_IB_QP_UNDERLAY)) {</div><div class='del'>-			err = mlx5_core_qp_modify(dev-&gt;mdev,</div><div class='del'>-						  MLX5_CMD_OP_2RST_QP, 0,</div><div class='del'>-						  NULL, &amp;base-&gt;mqp);</div><div class='add'>+		if (qp-&gt;type != IB_QPT_RAW_PACKET &amp;&amp;</div><div class='add'>+		    !(qp-&gt;flags &amp; IB_QP_CREATE_SOURCE_QPN)) {</div><div class='add'>+			err = mlx5_core_qp_modify(dev, MLX5_CMD_OP_2RST_QP, 0,</div><div class='add'>+						  NULL, &amp;base-&gt;mqp, NULL);</div><div class='ctx'> 		} else {</div><div class='ctx'> 			struct mlx5_modify_raw_qp_param raw_qp_param = {</div><div class='ctx'> 				.operation = MLX5_CMD_OP_2RST_QP</div><div class='hunk'>@@ -2471,8 +2692,8 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 				     base-&gt;mqp.qpn);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	get_cqs(qp-&gt;ibqp.qp_type, qp-&gt;ibqp.send_cq, qp-&gt;ibqp.recv_cq,</div><div class='del'>-		&amp;send_cq, &amp;recv_cq);</div><div class='add'>+	get_cqs(qp-&gt;type, qp-&gt;ibqp.send_cq, qp-&gt;ibqp.recv_cq, &amp;send_cq,</div><div class='add'>+		&amp;recv_cq);</div><div class='ctx'> </div><div class='ctx'> 	spin_lock_irqsave(&amp;dev-&gt;reset_flow_resource_lock, flags);</div><div class='ctx'> 	mlx5_ib_lock_cqs(send_cq, recv_cq);</div><div class='hunk'>@@ -2484,7 +2705,7 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 	if (recv_cq)</div><div class='ctx'> 		list_del(&amp;qp-&gt;cq_recv_list);</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;create_type == MLX5_QP_KERNEL) {</div><div class='add'>+	if (!udata) {</div><div class='ctx'> 		__mlx5_ib_cq_clean(recv_cq, base-&gt;mqp.qpn,</div><div class='ctx'> 				   qp-&gt;ibqp.srq ? to_msrq(qp-&gt;ibqp.srq) : NULL);</div><div class='ctx'> 		if (send_cq != recv_cq)</div><div class='hunk'>@@ -2494,264 +2715,488 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 	mlx5_ib_unlock_cqs(send_cq, recv_cq);</div><div class='ctx'> 	spin_unlock_irqrestore(&amp;dev-&gt;reset_flow_resource_lock, flags);</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;ibqp.qp_type == IB_QPT_RAW_PACKET ||</div><div class='del'>-	    qp-&gt;flags &amp; MLX5_IB_QP_UNDERLAY) {</div><div class='add'>+	if (qp-&gt;type == IB_QPT_RAW_PACKET ||</div><div class='add'>+	    qp-&gt;flags &amp; IB_QP_CREATE_SOURCE_QPN) {</div><div class='ctx'> 		destroy_raw_packet_qp(dev, qp);</div><div class='ctx'> 	} else {</div><div class='del'>-		err = mlx5_core_destroy_qp(dev-&gt;mdev, &amp;base-&gt;mqp);</div><div class='add'>+		err = mlx5_core_destroy_qp(dev, &amp;base-&gt;mqp);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n",</div><div class='ctx'> 				     base-&gt;mqp.qpn);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;create_type == MLX5_QP_KERNEL)</div><div class='del'>-		destroy_qp_kernel(dev, qp);</div><div class='del'>-	else if (qp-&gt;create_type == MLX5_QP_USER)</div><div class='del'>-		destroy_qp_user(dev, &amp;get_pd(qp)-&gt;ibpd, qp, base, udata);</div><div class='add'>+	destroy_qp(dev, qp, base, udata);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static const char *ib_qp_type_str(enum ib_qp_type type)</div><div class='add'>+static int create_dct(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='add'>+		      struct mlx5_ib_qp *qp,</div><div class='add'>+		      struct mlx5_create_qp_params *params)</div><div class='ctx'> {</div><div class='del'>-	switch (type) {</div><div class='del'>-	case IB_QPT_SMI:</div><div class='del'>-		return "IB_QPT_SMI";</div><div class='del'>-	case IB_QPT_GSI:</div><div class='del'>-		return "IB_QPT_GSI";</div><div class='add'>+	struct ib_qp_init_attr *attr = params-&gt;attr;</div><div class='add'>+	struct mlx5_ib_create_qp *ucmd = params-&gt;ucmd;</div><div class='add'>+	u32 uidx = params-&gt;uidx;</div><div class='add'>+	void *dctc;</div><div class='add'>+</div><div class='add'>+	if (mlx5_lag_is_active(dev-&gt;mdev) &amp;&amp; !MLX5_CAP_GEN(dev-&gt;mdev, lag_dct))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	qp-&gt;dct.in = kzalloc(MLX5_ST_SZ_BYTES(create_dct_in), GFP_KERNEL);</div><div class='add'>+	if (!qp-&gt;dct.in)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(create_dct_in, qp-&gt;dct.in, uid, to_mpd(pd)-&gt;uid);</div><div class='add'>+	dctc = MLX5_ADDR_OF(create_dct_in, qp-&gt;dct.in, dct_context_entry);</div><div class='add'>+	MLX5_SET(dctc, dctc, pd, to_mpd(pd)-&gt;pdn);</div><div class='add'>+	MLX5_SET(dctc, dctc, srqn_xrqn, to_msrq(attr-&gt;srq)-&gt;msrq.srqn);</div><div class='add'>+	MLX5_SET(dctc, dctc, cqn, to_mcq(attr-&gt;recv_cq)-&gt;mcq.cqn);</div><div class='add'>+	MLX5_SET64(dctc, dctc, dc_access_key, ucmd-&gt;access_key);</div><div class='add'>+	MLX5_SET(dctc, dctc, user_index, uidx);</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, ece_support))</div><div class='add'>+		MLX5_SET(dctc, dctc, ece, ucmd-&gt;ece_options);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;flags_en &amp; MLX5_QP_FLAG_SCATTER_CQE) {</div><div class='add'>+		int rcqe_sz = mlx5_ib_get_cqe_size(attr-&gt;recv_cq);</div><div class='add'>+</div><div class='add'>+		if (rcqe_sz == 128)</div><div class='add'>+			MLX5_SET(dctc, dctc, cs_res, MLX5_RES_SCAT_DATA64_CQE);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	qp-&gt;state = IB_QPS_RESET;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int check_qp_type(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,</div><div class='add'>+			 enum ib_qp_type *type)</div><div class='add'>+{</div><div class='add'>+	if (attr-&gt;qp_type == IB_QPT_DRIVER &amp;&amp; !MLX5_CAP_GEN(dev-&gt;mdev, dct))</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	switch (attr-&gt;qp_type) {</div><div class='add'>+	case IB_QPT_XRC_TGT:</div><div class='add'>+	case IB_QPT_XRC_INI:</div><div class='add'>+		if (!MLX5_CAP_GEN(dev-&gt;mdev, xrc))</div><div class='add'>+			goto out;</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case IB_QPT_RC:</div><div class='del'>-		return "IB_QPT_RC";</div><div class='ctx'> 	case IB_QPT_UC:</div><div class='del'>-		return "IB_QPT_UC";</div><div class='del'>-	case IB_QPT_UD:</div><div class='del'>-		return "IB_QPT_UD";</div><div class='del'>-	case IB_QPT_RAW_IPV6:</div><div class='del'>-		return "IB_QPT_RAW_IPV6";</div><div class='del'>-	case IB_QPT_RAW_ETHERTYPE:</div><div class='del'>-		return "IB_QPT_RAW_ETHERTYPE";</div><div class='del'>-	case IB_QPT_XRC_INI:</div><div class='del'>-		return "IB_QPT_XRC_INI";</div><div class='del'>-	case IB_QPT_XRC_TGT:</div><div class='del'>-		return "IB_QPT_XRC_TGT";</div><div class='add'>+	case IB_QPT_SMI:</div><div class='add'>+	case MLX5_IB_QPT_HW_GSI:</div><div class='add'>+	case IB_QPT_DRIVER:</div><div class='add'>+	case IB_QPT_GSI:</div><div class='ctx'> 	case IB_QPT_RAW_PACKET:</div><div class='del'>-		return "IB_QPT_RAW_PACKET";</div><div class='add'>+	case IB_QPT_UD:</div><div class='ctx'> 	case MLX5_IB_QPT_REG_UMR:</div><div class='del'>-		return "MLX5_IB_QPT_REG_UMR";</div><div class='del'>-	case IB_QPT_DRIVER:</div><div class='del'>-		return "IB_QPT_DRIVER";</div><div class='del'>-	case IB_QPT_MAX:</div><div class='add'>+		break;</div><div class='ctx'> 	default:</div><div class='del'>-		return "Invalid QP type";</div><div class='add'>+		goto out;</div><div class='ctx'> 	}</div><div class='add'>+</div><div class='add'>+	*type = attr-&gt;qp_type;</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	mlx5_ib_dbg(dev, "Unsupported QP type %d\n", attr-&gt;qp_type);</div><div class='add'>+	return -EOPNOTSUPP;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static struct ib_qp *mlx5_ib_create_dct(struct ib_pd *pd,</div><div class='del'>-					struct ib_qp_init_attr *attr,</div><div class='del'>-					struct mlx5_ib_create_qp *ucmd,</div><div class='del'>-					struct ib_udata *udata)</div><div class='add'>+static int check_valid_flow(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='add'>+			    struct ib_qp_init_attr *attr,</div><div class='add'>+			    struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(</div><div class='ctx'> 		udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='del'>-	struct mlx5_ib_qp *qp;</div><div class='del'>-	int err = 0;</div><div class='del'>-	u32 uidx = MLX5_IB_DEFAULT_UIDX;</div><div class='del'>-	void *dctc;</div><div class='ctx'> </div><div class='del'>-	if (!attr-&gt;srq || !attr-&gt;recv_cq)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+	if (!udata) {</div><div class='add'>+		/* Kernel create_qp callers */</div><div class='add'>+		if (attr-&gt;rwq_ind_tbl)</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='del'>-	err = get_qp_user_index(ucontext, ucmd, sizeof(*ucmd), &amp;uidx);</div><div class='del'>-	if (err)</div><div class='del'>-		return ERR_PTR(err);</div><div class='add'>+		switch (attr-&gt;qp_type) {</div><div class='add'>+		case IB_QPT_RAW_PACKET:</div><div class='add'>+		case IB_QPT_DRIVER:</div><div class='add'>+			return -EOPNOTSUPP;</div><div class='add'>+		default:</div><div class='add'>+			return 0;</div><div class='add'>+		}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	qp = kzalloc(sizeof(*qp), GFP_KERNEL);</div><div class='del'>-	if (!qp)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='add'>+	/* Userspace create_qp callers */</div><div class='add'>+	if (attr-&gt;qp_type == IB_QPT_RAW_PACKET &amp;&amp; !ucontext-&gt;cqe_version) {</div><div class='add'>+		mlx5_ib_dbg(dev,</div><div class='add'>+			"Raw Packet QP is only supported for CQE version &gt; 0\n");</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	qp-&gt;dct.in = kzalloc(MLX5_ST_SZ_BYTES(create_dct_in), GFP_KERNEL);</div><div class='del'>-	if (!qp-&gt;dct.in) {</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto err_free;</div><div class='add'>+	if (attr-&gt;qp_type != IB_QPT_RAW_PACKET &amp;&amp; attr-&gt;rwq_ind_tbl) {</div><div class='add'>+		mlx5_ib_dbg(dev,</div><div class='add'>+			    "Wrong QP type %d for the RWQ indirect table\n",</div><div class='add'>+			    attr-&gt;qp_type);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	MLX5_SET(create_dct_in, qp-&gt;dct.in, uid, to_mpd(pd)-&gt;uid);</div><div class='del'>-	dctc = MLX5_ADDR_OF(create_dct_in, qp-&gt;dct.in, dct_context_entry);</div><div class='del'>-	qp-&gt;qp_sub_type = MLX5_IB_QPT_DCT;</div><div class='del'>-	MLX5_SET(dctc, dctc, pd, to_mpd(pd)-&gt;pdn);</div><div class='del'>-	MLX5_SET(dctc, dctc, srqn_xrqn, to_msrq(attr-&gt;srq)-&gt;msrq.srqn);</div><div class='del'>-	MLX5_SET(dctc, dctc, cqn, to_mcq(attr-&gt;recv_cq)-&gt;mcq.cqn);</div><div class='del'>-	MLX5_SET64(dctc, dctc, dc_access_key, ucmd-&gt;access_key);</div><div class='del'>-	MLX5_SET(dctc, dctc, user_index, uidx);</div><div class='add'>+	/*</div><div class='add'>+	 * We don't need to see this warning, it means that kernel code</div><div class='add'>+	 * missing ib_pd. Placed here to catch developer's mistakes.</div><div class='add'>+	 */</div><div class='add'>+	WARN_ONCE(!pd &amp;&amp; attr-&gt;qp_type != IB_QPT_XRC_TGT,</div><div class='add'>+		  "There is a missing PD pointer assignment\n");</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	if (ucmd-&gt;flags &amp; MLX5_QP_FLAG_SCATTER_CQE)</div><div class='del'>-		configure_responder_scat_cqe(attr, dctc);</div><div class='add'>+static bool get_dp_ooo_cap(struct mlx5_core_dev *mdev, enum ib_qp_type qp_type)</div><div class='add'>+{</div><div class='add'>+	if (!MLX5_CAP_GEN_2(mdev, dp_ordering_force))</div><div class='add'>+		return false;</div><div class='ctx'> </div><div class='del'>-	qp-&gt;state = IB_QPS_RESET;</div><div class='add'>+	switch (qp_type) {</div><div class='add'>+	case IB_QPT_RC:</div><div class='add'>+		return MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_rc);</div><div class='add'>+	case IB_QPT_XRC_INI:</div><div class='add'>+	case IB_QPT_XRC_TGT:</div><div class='add'>+		return MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_xrc);</div><div class='add'>+	case IB_QPT_UC:</div><div class='add'>+		return MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_uc);</div><div class='add'>+	case IB_QPT_UD:</div><div class='add'>+		return MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_ud);</div><div class='add'>+	case MLX5_IB_QPT_DCI:</div><div class='add'>+	case MLX5_IB_QPT_DCT:</div><div class='add'>+		return MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_dc);</div><div class='add'>+	default:</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='ctx'> </div><div class='del'>-	return &amp;qp-&gt;ibqp;</div><div class='del'>-err_free:</div><div class='del'>-	kfree(qp);</div><div class='del'>-	return ERR_PTR(err);</div><div class='add'>+static void process_vendor_flag(struct mlx5_ib_dev *dev, int *flags, int flag,</div><div class='add'>+				bool cond, struct mlx5_ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	if (!(*flags &amp; flag))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	if (cond) {</div><div class='add'>+		qp-&gt;flags_en |= flag;</div><div class='add'>+		*flags &amp;= ~flag;</div><div class='add'>+		return;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	switch (flag) {</div><div class='add'>+	case MLX5_QP_FLAG_SCATTER_CQE:</div><div class='add'>+	case MLX5_QP_FLAG_ALLOW_SCATTER_CQE:</div><div class='add'>+		/*</div><div class='add'>+		 * We don't return error if these flags were provided,</div><div class='add'>+		 * and mlx5 doesn't have right capability.</div><div class='add'>+		 */</div><div class='add'>+		*flags &amp;= ~(MLX5_QP_FLAG_SCATTER_CQE |</div><div class='add'>+			    MLX5_QP_FLAG_ALLOW_SCATTER_CQE);</div><div class='add'>+		return;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+	mlx5_ib_dbg(dev, "Vendor create QP flag 0x%X is not supported\n", flag);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int set_mlx_qp_type(struct mlx5_ib_dev *dev,</div><div class='del'>-			   struct ib_qp_init_attr *init_attr,</div><div class='del'>-			   struct mlx5_ib_create_qp *ucmd,</div><div class='del'>-			   struct ib_udata *udata)</div><div class='add'>+static int process_vendor_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='add'>+				void *ucmd, struct ib_qp_init_attr *attr)</div><div class='ctx'> {</div><div class='del'>-	enum { MLX_QP_FLAGS = MLX5_QP_FLAG_TYPE_DCT | MLX5_QP_FLAG_TYPE_DCI };</div><div class='del'>-	int err;</div><div class='add'>+	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='add'>+	bool cond;</div><div class='add'>+	int flags;</div><div class='ctx'> </div><div class='del'>-	if (!udata)</div><div class='add'>+	if (attr-&gt;rwq_ind_tbl)</div><div class='add'>+		flags = ((struct mlx5_ib_create_qp_rss *)ucmd)-&gt;flags;</div><div class='add'>+	else</div><div class='add'>+		flags = ((struct mlx5_ib_create_qp *)ucmd)-&gt;flags;</div><div class='add'>+</div><div class='add'>+	switch (flags &amp; (MLX5_QP_FLAG_TYPE_DCT | MLX5_QP_FLAG_TYPE_DCI)) {</div><div class='add'>+	case MLX5_QP_FLAG_TYPE_DCI:</div><div class='add'>+		qp-&gt;type = MLX5_IB_QPT_DCI;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_QP_FLAG_TYPE_DCT:</div><div class='add'>+		qp-&gt;type = MLX5_IB_QPT_DCT;</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		if (qp-&gt;type != IB_QPT_DRIVER)</div><div class='add'>+			break;</div><div class='add'>+		/*</div><div class='add'>+		 * It is IB_QPT_DRIVER and or no subtype or</div><div class='add'>+		 * wrong subtype were provided.</div><div class='add'>+		 */</div><div class='ctx'> 		return -EINVAL;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (udata-&gt;inlen &lt; sizeof(*ucmd)) {</div><div class='del'>-		mlx5_ib_dbg(dev, "create_qp user command is smaller than expected\n");</div><div class='add'>+	process_vendor_flag(dev, &amp;flags, MLX5_QP_FLAG_TYPE_DCI, true, qp);</div><div class='add'>+	process_vendor_flag(dev, &amp;flags, MLX5_QP_FLAG_TYPE_DCT, true, qp);</div><div class='add'>+	process_vendor_flag(dev, &amp;flags, MLX5_QP_FLAG_DCI_STREAM,</div><div class='add'>+			    MLX5_CAP_GEN(mdev, log_max_dci_stream_channels),</div><div class='add'>+			    qp);</div><div class='add'>+</div><div class='add'>+	process_vendor_flag(dev, &amp;flags, MLX5_QP_FLAG_SIGNATURE, true, qp);</div><div class='add'>+	process_vendor_flag(dev, &amp;flags, MLX5_QP_FLAG_SCATTER_CQE,</div><div class='add'>+			    MLX5_CAP_GEN(mdev, sctr_data_cqe), qp);</div><div class='add'>+	process_vendor_flag(dev, &amp;flags, MLX5_QP_FLAG_ALLOW_SCATTER_CQE,</div><div class='add'>+			    MLX5_CAP_GEN(mdev, sctr_data_cqe), qp);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;type == IB_QPT_RAW_PACKET) {</div><div class='add'>+		cond = MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan) ||</div><div class='add'>+		       MLX5_CAP_ETH(mdev, tunnel_stateless_gre) ||</div><div class='add'>+		       MLX5_CAP_ETH(mdev, tunnel_stateless_geneve_rx);</div><div class='add'>+		process_vendor_flag(dev, &amp;flags, MLX5_QP_FLAG_TUNNEL_OFFLOADS,</div><div class='add'>+				    cond, qp);</div><div class='add'>+		process_vendor_flag(dev, &amp;flags,</div><div class='add'>+				    MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC, true,</div><div class='add'>+				    qp);</div><div class='add'>+		process_vendor_flag(dev, &amp;flags,</div><div class='add'>+				    MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC, true,</div><div class='add'>+				    qp);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;type == IB_QPT_RC)</div><div class='add'>+		process_vendor_flag(dev, &amp;flags,</div><div class='add'>+				    MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE,</div><div class='add'>+				    MLX5_CAP_GEN(mdev, qp_packet_based), qp);</div><div class='add'>+</div><div class='add'>+	process_vendor_flag(dev, &amp;flags, MLX5_QP_FLAG_BFREG_INDEX, true, qp);</div><div class='add'>+	process_vendor_flag(dev, &amp;flags, MLX5_QP_FLAG_UAR_PAGE_INDEX, true, qp);</div><div class='add'>+</div><div class='add'>+	cond = qp-&gt;flags_en &amp; ~(MLX5_QP_FLAG_TUNNEL_OFFLOADS |</div><div class='add'>+				MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |</div><div class='add'>+				MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC);</div><div class='add'>+	if (attr-&gt;rwq_ind_tbl &amp;&amp; cond) {</div><div class='add'>+		mlx5_ib_dbg(dev, "RSS RAW QP has unsupported flags 0x%X\n",</div><div class='add'>+			    cond);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='del'>-	err = ib_copy_from_udata(ucmd, udata, sizeof(*ucmd));</div><div class='del'>-	if (err)</div><div class='del'>-		return err;</div><div class='ctx'> </div><div class='del'>-	if ((ucmd-&gt;flags &amp; MLX_QP_FLAGS) == MLX5_QP_FLAG_TYPE_DCI) {</div><div class='del'>-		init_attr-&gt;qp_type = MLX5_IB_QPT_DCI;</div><div class='del'>-	} else {</div><div class='del'>-		if ((ucmd-&gt;flags &amp; MLX_QP_FLAGS) == MLX5_QP_FLAG_TYPE_DCT) {</div><div class='del'>-			init_attr-&gt;qp_type = MLX5_IB_QPT_DCT;</div><div class='del'>-		} else {</div><div class='del'>-			mlx5_ib_dbg(dev, "Invalid QP flags\n");</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='add'>+	if (flags)</div><div class='add'>+		mlx5_ib_dbg(dev, "udata has unsupported flags 0x%X\n", flags);</div><div class='add'>+</div><div class='add'>+	return (flags) ? -EINVAL : 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (!MLX5_CAP_GEN(dev-&gt;mdev, dct)) {</div><div class='del'>-		mlx5_ib_dbg(dev, "DC transport is not supported\n");</div><div class='del'>-		return -EOPNOTSUPP;</div><div class='add'>+static void process_create_flag(struct mlx5_ib_dev *dev, int *flags, int flag,</div><div class='add'>+				bool cond, struct mlx5_ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	if (!(*flags &amp; flag))</div><div class='add'>+		return;</div><div class='add'>+</div><div class='add'>+	if (cond) {</div><div class='add'>+		qp-&gt;flags |= flag;</div><div class='add'>+		*flags &amp;= ~flag;</div><div class='add'>+		return;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	mlx5_ib_dbg(dev, "Verbs create QP flag 0x%X is not supported\n", flag);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int process_create_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='add'>+				struct ib_qp_init_attr *attr)</div><div class='add'>+{</div><div class='add'>+	enum ib_qp_type qp_type = qp-&gt;type;</div><div class='add'>+	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='add'>+	int create_flags = attr-&gt;create_flags;</div><div class='add'>+	bool cond;</div><div class='add'>+</div><div class='add'>+	if (qp_type == MLX5_IB_QPT_DCT)</div><div class='add'>+		return (create_flags) ? -EINVAL : 0;</div><div class='add'>+</div><div class='add'>+	if (qp_type == IB_QPT_RAW_PACKET &amp;&amp; attr-&gt;rwq_ind_tbl)</div><div class='add'>+		return (create_flags) ? -EINVAL : 0;</div><div class='add'>+</div><div class='add'>+	process_create_flag(dev, &amp;create_flags, IB_QP_CREATE_NETIF_QP,</div><div class='add'>+			    mlx5_get_flow_namespace(dev-&gt;mdev,</div><div class='add'>+						    MLX5_FLOW_NAMESPACE_BYPASS),</div><div class='add'>+			    qp);</div><div class='add'>+	process_create_flag(dev, &amp;create_flags,</div><div class='add'>+			    IB_QP_CREATE_INTEGRITY_EN,</div><div class='add'>+			    MLX5_CAP_GEN(mdev, sho), qp);</div><div class='add'>+	process_create_flag(dev, &amp;create_flags,</div><div class='add'>+			    IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,</div><div class='add'>+			    MLX5_CAP_GEN(mdev, block_lb_mc), qp);</div><div class='add'>+	process_create_flag(dev, &amp;create_flags, IB_QP_CREATE_CROSS_CHANNEL,</div><div class='add'>+			    MLX5_CAP_GEN(mdev, cd), qp);</div><div class='add'>+	process_create_flag(dev, &amp;create_flags, IB_QP_CREATE_MANAGED_SEND,</div><div class='add'>+			    MLX5_CAP_GEN(mdev, cd), qp);</div><div class='add'>+	process_create_flag(dev, &amp;create_flags, IB_QP_CREATE_MANAGED_RECV,</div><div class='add'>+			    MLX5_CAP_GEN(mdev, cd), qp);</div><div class='add'>+</div><div class='add'>+	if (qp_type == IB_QPT_UD) {</div><div class='add'>+		process_create_flag(dev, &amp;create_flags,</div><div class='add'>+				    IB_QP_CREATE_IPOIB_UD_LSO,</div><div class='add'>+				    MLX5_CAP_GEN(mdev, ipoib_basic_offloads),</div><div class='add'>+				    qp);</div><div class='add'>+		cond = MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_IB;</div><div class='add'>+		process_create_flag(dev, &amp;create_flags, IB_QP_CREATE_SOURCE_QPN,</div><div class='add'>+				    cond, qp);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (qp_type == IB_QPT_RAW_PACKET) {</div><div class='add'>+		cond = MLX5_CAP_GEN(mdev, eth_net_offloads) &amp;&amp;</div><div class='add'>+		       MLX5_CAP_ETH(mdev, scatter_fcs);</div><div class='add'>+		process_create_flag(dev, &amp;create_flags,</div><div class='add'>+				    IB_QP_CREATE_SCATTER_FCS, cond, qp);</div><div class='add'>+</div><div class='add'>+		cond = MLX5_CAP_GEN(mdev, eth_net_offloads) &amp;&amp;</div><div class='add'>+		       MLX5_CAP_ETH(mdev, vlan_cap);</div><div class='add'>+		process_create_flag(dev, &amp;create_flags,</div><div class='add'>+				    IB_QP_CREATE_CVLAN_STRIPPING, cond, qp);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	process_create_flag(dev, &amp;create_flags,</div><div class='add'>+			    IB_QP_CREATE_PCI_WRITE_END_PADDING,</div><div class='add'>+			    MLX5_CAP_GEN(mdev, end_pad), qp);</div><div class='add'>+</div><div class='add'>+	process_create_flag(dev, &amp;create_flags, MLX5_IB_QP_CREATE_SQPN_QP1,</div><div class='add'>+			    true, qp);</div><div class='add'>+</div><div class='add'>+	if (create_flags) {</div><div class='add'>+		mlx5_ib_dbg(dev, "Create QP has unsupported flags 0x%X\n",</div><div class='add'>+			    create_flags);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+	}</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,</div><div class='del'>-				struct ib_qp_init_attr *verbs_init_attr,</div><div class='del'>-				struct ib_udata *udata)</div><div class='add'>+static int process_udata_size(struct mlx5_ib_dev *dev,</div><div class='add'>+			      struct mlx5_create_qp_params *params)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_dev *dev;</div><div class='del'>-	struct mlx5_ib_qp *qp;</div><div class='del'>-	u16 xrcdn = 0;</div><div class='del'>-	int err;</div><div class='del'>-	struct ib_qp_init_attr mlx_init_attr;</div><div class='del'>-	struct ib_qp_init_attr *init_attr = verbs_init_attr;</div><div class='del'>-	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(</div><div class='del'>-		udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='add'>+	size_t ucmd = sizeof(struct mlx5_ib_create_qp);</div><div class='add'>+	struct ib_udata *udata = params-&gt;udata;</div><div class='add'>+	size_t outlen = udata-&gt;outlen;</div><div class='add'>+	size_t inlen = udata-&gt;inlen;</div><div class='ctx'> </div><div class='del'>-	if (pd) {</div><div class='del'>-		dev = to_mdev(pd-&gt;device);</div><div class='add'>+	params-&gt;outlen = min(outlen, sizeof(struct mlx5_ib_create_qp_resp));</div><div class='add'>+	params-&gt;ucmd_size = ucmd;</div><div class='add'>+	if (!params-&gt;is_rss_raw) {</div><div class='add'>+		/* User has old rdma-core, which doesn't support ECE */</div><div class='add'>+		size_t min_inlen =</div><div class='add'>+			offsetof(struct mlx5_ib_create_qp, ece_options);</div><div class='ctx'> </div><div class='del'>-		if (init_attr-&gt;qp_type == IB_QPT_RAW_PACKET) {</div><div class='del'>-			if (!ucontext) {</div><div class='del'>-				mlx5_ib_dbg(dev, "Raw Packet QP is not supported for kernel consumers\n");</div><div class='del'>-				return ERR_PTR(-EINVAL);</div><div class='del'>-			} else if (!ucontext-&gt;cqe_version) {</div><div class='del'>-				mlx5_ib_dbg(dev, "Raw Packet QP is only supported for CQE version &gt; 0\n");</div><div class='del'>-				return ERR_PTR(-EINVAL);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	} else {</div><div class='del'>-		/* being cautious here */</div><div class='del'>-		if (init_attr-&gt;qp_type != IB_QPT_XRC_TGT &amp;&amp;</div><div class='del'>-		    init_attr-&gt;qp_type != MLX5_IB_QPT_REG_UMR) {</div><div class='del'>-			pr_warn("%s: no PD for transport %s\n", __func__,</div><div class='del'>-				ib_qp_type_str(init_attr-&gt;qp_type));</div><div class='del'>-			return ERR_PTR(-EINVAL);</div><div class='del'>-		}</div><div class='del'>-		dev = to_mdev(to_mxrcd(init_attr-&gt;xrcd)-&gt;ibxrcd.device);</div><div class='add'>+		/*</div><div class='add'>+		 * We will check in check_ucmd_data() that user</div><div class='add'>+		 * cleared everything after inlen.</div><div class='add'>+		 */</div><div class='add'>+		params-&gt;inlen = (inlen &lt; min_inlen) ? 0 : min(inlen, ucmd);</div><div class='add'>+		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;qp_type == IB_QPT_DRIVER) {</div><div class='del'>-		struct mlx5_ib_create_qp ucmd;</div><div class='add'>+	/* RSS RAW QP */</div><div class='add'>+	if (inlen &lt; offsetofend(struct mlx5_ib_create_qp_rss, flags))</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-		init_attr = &amp;mlx_init_attr;</div><div class='del'>-		memcpy(init_attr, verbs_init_attr, sizeof(*verbs_init_attr));</div><div class='del'>-		err = set_mlx_qp_type(dev, init_attr, &amp;ucmd, udata);</div><div class='del'>-		if (err)</div><div class='del'>-			return ERR_PTR(err);</div><div class='add'>+	if (outlen &lt; offsetofend(struct mlx5_ib_create_qp_resp, bfreg_index))</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-		if (init_attr-&gt;qp_type == MLX5_IB_QPT_DCI) {</div><div class='del'>-			if (init_attr-&gt;cap.max_recv_wr ||</div><div class='del'>-			    init_attr-&gt;cap.max_recv_sge) {</div><div class='del'>-				mlx5_ib_dbg(dev, "DCI QP requires zero size receive queue\n");</div><div class='del'>-				return ERR_PTR(-EINVAL);</div><div class='del'>-			}</div><div class='del'>-		} else {</div><div class='del'>-			return mlx5_ib_create_dct(pd, init_attr, &amp;ucmd, udata);</div><div class='del'>-		}</div><div class='add'>+	ucmd = sizeof(struct mlx5_ib_create_qp_rss);</div><div class='add'>+	params-&gt;ucmd_size = ucmd;</div><div class='add'>+	if (inlen &gt; ucmd &amp;&amp; !ib_is_udata_cleared(udata, ucmd, inlen - ucmd))</div><div class='add'>+		return -EINVAL;</div><div class='add'>+</div><div class='add'>+	params-&gt;inlen = min(ucmd, inlen);</div><div class='add'>+out:</div><div class='add'>+	if (!params-&gt;inlen)</div><div class='add'>+		mlx5_ib_dbg(dev, "udata is too small\n");</div><div class='add'>+</div><div class='add'>+	return (params-&gt;inlen) ? 0 : -EINVAL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,</div><div class='add'>+		     struct mlx5_ib_qp *qp,</div><div class='add'>+		     struct mlx5_create_qp_params *params)</div><div class='add'>+{</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	if (params-&gt;is_rss_raw) {</div><div class='add'>+		err = create_rss_raw_qp_tir(dev, pd, qp, params);</div><div class='add'>+		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	switch (init_attr-&gt;qp_type) {</div><div class='add'>+	switch (qp-&gt;type) {</div><div class='add'>+	case MLX5_IB_QPT_DCT:</div><div class='add'>+		err = create_dct(dev, pd, qp, params);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_QPT_DCI:</div><div class='add'>+		err = create_dci(dev, pd, qp, params);</div><div class='add'>+		break;</div><div class='ctx'> 	case IB_QPT_XRC_TGT:</div><div class='del'>-	case IB_QPT_XRC_INI:</div><div class='del'>-		if (!MLX5_CAP_GEN(dev-&gt;mdev, xrc)) {</div><div class='del'>-			mlx5_ib_dbg(dev, "XRC not supported\n");</div><div class='del'>-			return ERR_PTR(-ENOSYS);</div><div class='del'>-		}</div><div class='del'>-		init_attr-&gt;recv_cq = NULL;</div><div class='del'>-		if (init_attr-&gt;qp_type == IB_QPT_XRC_TGT) {</div><div class='del'>-			xrcdn = to_mxrcd(init_attr-&gt;xrcd)-&gt;xrcdn;</div><div class='del'>-			init_attr-&gt;send_cq = NULL;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		/* fall through */</div><div class='del'>-	case IB_QPT_RAW_PACKET:</div><div class='del'>-	case IB_QPT_RC:</div><div class='del'>-	case IB_QPT_UC:</div><div class='del'>-	case IB_QPT_UD:</div><div class='del'>-	case IB_QPT_SMI:</div><div class='add'>+		err = create_xrc_tgt_qp(dev, qp, params);</div><div class='add'>+		break;</div><div class='add'>+	case IB_QPT_GSI:</div><div class='add'>+		err = mlx5_ib_create_gsi(pd, qp, params-&gt;attr);</div><div class='add'>+		break;</div><div class='ctx'> 	case MLX5_IB_QPT_HW_GSI:</div><div class='add'>+		rdma_restrack_no_track(&amp;qp-&gt;ibqp.res);</div><div class='add'>+		fallthrough;</div><div class='ctx'> 	case MLX5_IB_QPT_REG_UMR:</div><div class='del'>-	case MLX5_IB_QPT_DCI:</div><div class='del'>-		qp = kzalloc(sizeof(*qp), GFP_KERNEL);</div><div class='del'>-		if (!qp)</div><div class='del'>-			return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-		err = create_qp_common(dev, pd, init_attr, udata, qp);</div><div class='del'>-		if (err) {</div><div class='del'>-			mlx5_ib_dbg(dev, "create_qp_common failed\n");</div><div class='del'>-			kfree(qp);</div><div class='del'>-			return ERR_PTR(err);</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (is_qp0(init_attr-&gt;qp_type))</div><div class='del'>-			qp-&gt;ibqp.qp_num = 0;</div><div class='del'>-		else if (is_qp1(init_attr-&gt;qp_type))</div><div class='del'>-			qp-&gt;ibqp.qp_num = 1;</div><div class='add'>+	default:</div><div class='add'>+		if (params-&gt;udata)</div><div class='add'>+			err = create_user_qp(dev, pd, qp, params);</div><div class='ctx'> 		else</div><div class='del'>-			qp-&gt;ibqp.qp_num = qp-&gt;trans_qp.base.mqp.qpn;</div><div class='add'>+			err = create_kernel_qp(dev, pd, qp, params);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		mlx5_ib_dbg(dev, "ib qpnum 0x%x, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x\n",</div><div class='del'>-			    qp-&gt;ibqp.qp_num, qp-&gt;trans_qp.base.mqp.qpn,</div><div class='del'>-			    init_attr-&gt;recv_cq ? to_mcq(init_attr-&gt;recv_cq)-&gt;mcq.cqn : -1,</div><div class='del'>-			    init_attr-&gt;send_cq ? to_mcq(init_attr-&gt;send_cq)-&gt;mcq.cqn : -1);</div><div class='add'>+out:</div><div class='add'>+	if (err) {</div><div class='add'>+		mlx5_ib_err(dev, "Create QP type %d failed\n", qp-&gt;type);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-		qp-&gt;trans_qp.xrcdn = xrcdn;</div><div class='add'>+	if (is_qp0(qp-&gt;type))</div><div class='add'>+		qp-&gt;ibqp.qp_num = 0;</div><div class='add'>+	else if (is_qp1(qp-&gt;type))</div><div class='add'>+		qp-&gt;ibqp.qp_num = 1;</div><div class='add'>+	else</div><div class='add'>+		qp-&gt;ibqp.qp_num = qp-&gt;trans_qp.base.mqp.qpn;</div><div class='ctx'> </div><div class='del'>-		break;</div><div class='add'>+	mlx5_ib_dbg(dev,</div><div class='add'>+		"QP type %d, ib qpn 0x%X, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x, ece 0x%x\n",</div><div class='add'>+		qp-&gt;type, qp-&gt;ibqp.qp_num, qp-&gt;trans_qp.base.mqp.qpn,</div><div class='add'>+		params-&gt;attr-&gt;recv_cq ? to_mcq(params-&gt;attr-&gt;recv_cq)-&gt;mcq.cqn :</div><div class='add'>+					-1,</div><div class='add'>+		params-&gt;attr-&gt;send_cq ? to_mcq(params-&gt;attr-&gt;send_cq)-&gt;mcq.cqn :</div><div class='add'>+					-1,</div><div class='add'>+		params-&gt;resp.ece_options);</div><div class='ctx'> </div><div class='del'>-	case IB_QPT_GSI:</div><div class='del'>-		return mlx5_ib_gsi_create_qp(pd, init_attr);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int check_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='add'>+			 struct ib_qp_init_attr *attr)</div><div class='add'>+{</div><div class='add'>+	int ret = 0;</div><div class='ctx'> </div><div class='del'>-	case IB_QPT_RAW_IPV6:</div><div class='del'>-	case IB_QPT_RAW_ETHERTYPE:</div><div class='del'>-	case IB_QPT_MAX:</div><div class='add'>+	switch (qp-&gt;type) {</div><div class='add'>+	case MLX5_IB_QPT_DCT:</div><div class='add'>+		ret = (!attr-&gt;srq || !attr-&gt;recv_cq) ? -EINVAL : 0;</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_QPT_DCI:</div><div class='add'>+		ret = (attr-&gt;cap.max_recv_wr || attr-&gt;cap.max_recv_sge) ?</div><div class='add'>+			      -EINVAL :</div><div class='add'>+			      0;</div><div class='add'>+		break;</div><div class='add'>+	case IB_QPT_RAW_PACKET:</div><div class='add'>+		ret = (attr-&gt;rwq_ind_tbl &amp;&amp; attr-&gt;send_cq) ? -EINVAL : 0;</div><div class='add'>+		break;</div><div class='ctx'> 	default:</div><div class='del'>-		mlx5_ib_dbg(dev, "unsupported qp type %d\n",</div><div class='del'>-			    init_attr-&gt;qp_type);</div><div class='del'>-		/* Don't support raw QPs */</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		break;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (verbs_init_attr-&gt;qp_type == IB_QPT_DRIVER)</div><div class='del'>-		qp-&gt;qp_sub_type = init_attr-&gt;qp_type;</div><div class='add'>+	if (ret)</div><div class='add'>+		mlx5_ib_dbg(dev, "QP type %d has wrong attributes\n", qp-&gt;type);</div><div class='add'>+</div><div class='add'>+	return ret;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_qp_uidx(struct mlx5_ib_qp *qp,</div><div class='add'>+		       struct mlx5_create_qp_params *params)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_create_qp *ucmd = params-&gt;ucmd;</div><div class='add'>+	struct ib_udata *udata = params-&gt;udata;</div><div class='add'>+	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(</div><div class='add'>+		udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='add'>+</div><div class='add'>+	if (params-&gt;is_rss_raw)</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-	return &amp;qp-&gt;ibqp;</div><div class='add'>+	return get_qp_user_index(ucontext, ucmd, sizeof(*ucmd), &amp;params-&gt;uidx);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp)</div><div class='hunk'>@@ -2761,7 +3206,7 @@ static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp)</div><div class='ctx'> 	if (mqp-&gt;state == IB_QPS_RTR) {</div><div class='ctx'> 		int err;</div><div class='ctx'> </div><div class='del'>-		err = mlx5_core_destroy_dct(dev-&gt;mdev, &amp;mqp-&gt;dct.mdct);</div><div class='add'>+		err = mlx5_core_destroy_dct(dev, &amp;mqp-&gt;dct.mdct);</div><div class='ctx'> 		if (err) {</div><div class='ctx'> 			mlx5_ib_warn(dev, "failed to destroy DCT %d\n", err);</div><div class='ctx'> 			return err;</div><div class='hunk'>@@ -2769,36 +3214,165 @@ static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp)</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	kfree(mqp-&gt;dct.in);</div><div class='del'>-	kfree(mqp);</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static int check_ucmd_data(struct mlx5_ib_dev *dev,</div><div class='add'>+			   struct mlx5_create_qp_params *params)</div><div class='add'>+{</div><div class='add'>+	struct ib_udata *udata = params-&gt;udata;</div><div class='add'>+	size_t size, last;</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (params-&gt;is_rss_raw)</div><div class='add'>+		/*</div><div class='add'>+		 * These QPs don't have "reserved" field in their</div><div class='add'>+		 * create_qp input struct, so their data is always valid.</div><div class='add'>+		 */</div><div class='add'>+		last = sizeof(struct mlx5_ib_create_qp_rss);</div><div class='add'>+	else</div><div class='add'>+		last = offsetof(struct mlx5_ib_create_qp, reserved);</div><div class='add'>+</div><div class='add'>+	if (udata-&gt;inlen &lt;= last)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	/*</div><div class='add'>+	 * User provides different create_qp structures based on the</div><div class='add'>+	 * flow and we need to know if he cleared memory after our</div><div class='add'>+	 * struct create_qp ends.</div><div class='add'>+	 */</div><div class='add'>+	size = udata-&gt;inlen - last;</div><div class='add'>+	ret = ib_is_udata_cleared(params-&gt;udata, last, size);</div><div class='add'>+	if (!ret)</div><div class='add'>+		mlx5_ib_dbg(</div><div class='add'>+			dev,</div><div class='add'>+			"udata is not cleared, inlen = %zu, ucmd = %zu, last = %zu, size = %zu\n",</div><div class='add'>+			udata-&gt;inlen, params-&gt;ucmd_size, last, size);</div><div class='add'>+	return ret ? 0 : -EINVAL;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,</div><div class='add'>+		      struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_create_qp_params params = {};</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ibqp-&gt;device);</div><div class='add'>+	struct mlx5_ib_qp *qp = to_mqp(ibqp);</div><div class='add'>+	struct ib_pd *pd = ibqp-&gt;pd;</div><div class='add'>+	enum ib_qp_type type;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	err = mlx5_ib_dev_res_srq_init(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = check_qp_type(dev, attr, &amp;type);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = check_valid_flow(dev, pd, attr, udata);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	params.udata = udata;</div><div class='add'>+	params.uidx = MLX5_IB_DEFAULT_UIDX;</div><div class='add'>+	params.attr = attr;</div><div class='add'>+	params.is_rss_raw = !!attr-&gt;rwq_ind_tbl;</div><div class='add'>+</div><div class='add'>+	if (udata) {</div><div class='add'>+		err = process_udata_size(dev, &amp;params);</div><div class='add'>+		if (err)</div><div class='add'>+			return err;</div><div class='add'>+</div><div class='add'>+		err = check_ucmd_data(dev, &amp;params);</div><div class='add'>+		if (err)</div><div class='add'>+			return err;</div><div class='add'>+</div><div class='add'>+		params.ucmd = kzalloc(params.ucmd_size, GFP_KERNEL);</div><div class='add'>+		if (!params.ucmd)</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+</div><div class='add'>+		err = ib_copy_from_udata(params.ucmd, udata, params.inlen);</div><div class='add'>+		if (err)</div><div class='add'>+			goto free_ucmd;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mutex_init(&amp;qp-&gt;mutex);</div><div class='add'>+	qp-&gt;type = type;</div><div class='add'>+	if (udata) {</div><div class='add'>+		err = process_vendor_flags(dev, qp, params.ucmd, attr);</div><div class='add'>+		if (err)</div><div class='add'>+			goto free_ucmd;</div><div class='add'>+</div><div class='add'>+		err = get_qp_uidx(qp, &amp;params);</div><div class='add'>+		if (err)</div><div class='add'>+			goto free_ucmd;</div><div class='add'>+	}</div><div class='add'>+	err = process_create_flags(dev, qp, attr);</div><div class='add'>+	if (err)</div><div class='add'>+		goto free_ucmd;</div><div class='add'>+</div><div class='add'>+	err = check_qp_attr(dev, qp, attr);</div><div class='add'>+	if (err)</div><div class='add'>+		goto free_ucmd;</div><div class='add'>+</div><div class='add'>+	err = create_qp(dev, pd, qp, &amp;params);</div><div class='add'>+	if (err)</div><div class='add'>+		goto free_ucmd;</div><div class='add'>+</div><div class='add'>+	kfree(params.ucmd);</div><div class='add'>+	params.ucmd = NULL;</div><div class='add'>+</div><div class='add'>+	if (udata)</div><div class='add'>+		/*</div><div class='add'>+		 * It is safe to copy response for all user create QP flows,</div><div class='add'>+		 * including MLX5_IB_QPT_DCT, which doesn't need it.</div><div class='add'>+		 * In that case, resp will be filled with zeros.</div><div class='add'>+		 */</div><div class='add'>+		err = ib_copy_to_udata(udata, &amp;params.resp, params.outlen);</div><div class='add'>+	if (err)</div><div class='add'>+		goto destroy_qp;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+destroy_qp:</div><div class='add'>+	switch (qp-&gt;type) {</div><div class='add'>+	case MLX5_IB_QPT_DCT:</div><div class='add'>+		mlx5_ib_destroy_dct(qp);</div><div class='add'>+		break;</div><div class='add'>+	case IB_QPT_GSI:</div><div class='add'>+		mlx5_ib_destroy_gsi(qp);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		destroy_qp_common(dev, qp, udata);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+free_ucmd:</div><div class='add'>+	kfree(params.ucmd);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(qp-&gt;device);</div><div class='ctx'> 	struct mlx5_ib_qp *mqp = to_mqp(qp);</div><div class='ctx'> </div><div class='del'>-	if (unlikely(qp-&gt;qp_type == IB_QPT_GSI))</div><div class='del'>-		return mlx5_ib_gsi_destroy_qp(qp);</div><div class='add'>+	if (mqp-&gt;type == IB_QPT_GSI)</div><div class='add'>+		return mlx5_ib_destroy_gsi(mqp);</div><div class='ctx'> </div><div class='del'>-	if (mqp-&gt;qp_sub_type == MLX5_IB_QPT_DCT)</div><div class='add'>+	if (mqp-&gt;type == MLX5_IB_QPT_DCT)</div><div class='ctx'> 		return mlx5_ib_destroy_dct(mqp);</div><div class='ctx'> </div><div class='ctx'> 	destroy_qp_common(dev, mqp, udata);</div><div class='del'>-</div><div class='del'>-	kfree(mqp);</div><div class='del'>-</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int to_mlx5_access_flags(struct mlx5_ib_qp *qp,</div><div class='del'>-				const struct ib_qp_attr *attr,</div><div class='del'>-				int attr_mask, __be32 *hw_access_flags_be)</div><div class='add'>+static int set_qpc_atomic_flags(struct mlx5_ib_qp *qp,</div><div class='add'>+				const struct ib_qp_attr *attr, int attr_mask,</div><div class='add'>+				void *qpc)</div><div class='ctx'> {</div><div class='del'>-	u8 dest_rd_atomic;</div><div class='del'>-	u32 access_flags, hw_access_flags = 0;</div><div class='del'>-</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(qp-&gt;ibqp.device);</div><div class='add'>+	u8 dest_rd_atomic;</div><div class='add'>+	u32 access_flags;</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_MAX_DEST_RD_ATOMIC)</div><div class='ctx'> 		dest_rd_atomic = attr-&gt;max_dest_rd_atomic;</div><div class='hunk'>@@ -2813,24 +3387,20 @@ static int to_mlx5_access_flags(struct mlx5_ib_qp *qp,</div><div class='ctx'> 	if (!dest_rd_atomic)</div><div class='ctx'> 		access_flags &amp;= IB_ACCESS_REMOTE_WRITE;</div><div class='ctx'> </div><div class='del'>-	if (access_flags &amp; IB_ACCESS_REMOTE_READ)</div><div class='del'>-		hw_access_flags |= MLX5_QP_BIT_RRE;</div><div class='add'>+	MLX5_SET(qpc, qpc, rre, !!(access_flags &amp; IB_ACCESS_REMOTE_READ));</div><div class='add'>+</div><div class='ctx'> 	if (access_flags &amp; IB_ACCESS_REMOTE_ATOMIC) {</div><div class='ctx'> 		int atomic_mode;</div><div class='ctx'> </div><div class='del'>-		atomic_mode = get_atomic_mode(dev, qp-&gt;ibqp.qp_type);</div><div class='add'>+		atomic_mode = get_atomic_mode(dev, qp);</div><div class='ctx'> 		if (atomic_mode &lt; 0)</div><div class='ctx'> 			return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='del'>-		hw_access_flags |= MLX5_QP_BIT_RAE;</div><div class='del'>-		hw_access_flags |= atomic_mode &lt;&lt; MLX5_ATOMIC_MODE_OFFSET;</div><div class='add'>+		MLX5_SET(qpc, qpc, rae, 1);</div><div class='add'>+		MLX5_SET(qpc, qpc, atomic_mode, atomic_mode);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (access_flags &amp; IB_ACCESS_REMOTE_WRITE)</div><div class='del'>-		hw_access_flags |= MLX5_QP_BIT_RWE;</div><div class='del'>-</div><div class='del'>-	*hw_access_flags_be = cpu_to_be32(hw_access_flags);</div><div class='del'>-</div><div class='add'>+	MLX5_SET(qpc, qpc, rwe, !!(access_flags &amp; IB_ACCESS_REMOTE_WRITE));</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -2840,20 +3410,60 @@ enum {</div><div class='ctx'> 	MLX5_PATH_FLAG_COUNTER	= 1 &lt;&lt; 2,</div><div class='ctx'> };</div><div class='ctx'> </div><div class='del'>-static int ib_rate_to_mlx5(struct mlx5_ib_dev *dev, u8 rate)</div><div class='add'>+static int mlx5_to_ib_rate_map(u8 rate)</div><div class='ctx'> {</div><div class='del'>-	if (rate == IB_RATE_PORT_CURRENT)</div><div class='add'>+	static const int rates[] = { IB_RATE_PORT_CURRENT, IB_RATE_56_GBPS,</div><div class='add'>+				     IB_RATE_25_GBPS,	   IB_RATE_100_GBPS,</div><div class='add'>+				     IB_RATE_200_GBPS,	   IB_RATE_50_GBPS,</div><div class='add'>+				     IB_RATE_400_GBPS };</div><div class='add'>+</div><div class='add'>+	if (rate &lt; ARRAY_SIZE(rates))</div><div class='add'>+		return rates[rate];</div><div class='add'>+</div><div class='add'>+	return rate - MLX5_STAT_RATE_OFFSET;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int ib_to_mlx5_rate_map(u8 rate)</div><div class='add'>+{</div><div class='add'>+	switch (rate) {</div><div class='add'>+	case IB_RATE_PORT_CURRENT:</div><div class='ctx'> 		return 0;</div><div class='add'>+	case IB_RATE_56_GBPS:</div><div class='add'>+		return 1;</div><div class='add'>+	case IB_RATE_25_GBPS:</div><div class='add'>+		return 2;</div><div class='add'>+	case IB_RATE_100_GBPS:</div><div class='add'>+		return 3;</div><div class='add'>+	case IB_RATE_200_GBPS:</div><div class='add'>+		return 4;</div><div class='add'>+	case IB_RATE_50_GBPS:</div><div class='add'>+		return 5;</div><div class='add'>+	case IB_RATE_400_GBPS:</div><div class='add'>+		return 6;</div><div class='add'>+	default:</div><div class='add'>+		return rate + MLX5_STAT_RATE_OFFSET;</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	if (rate &lt; IB_RATE_2_5_GBPS || rate &gt; IB_RATE_600_GBPS)</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5r_ib_rate(struct mlx5_ib_dev *dev, u8 rate)</div><div class='add'>+{</div><div class='add'>+	u32 stat_rate_support;</div><div class='add'>+</div><div class='add'>+	if (rate == IB_RATE_PORT_CURRENT || rate == IB_RATE_800_GBPS ||</div><div class='add'>+	    rate == IB_RATE_1600_GBPS)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	if (rate &lt; IB_RATE_2_5_GBPS || rate &gt; IB_RATE_1600_GBPS)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='add'>+	stat_rate_support = MLX5_CAP_GEN(dev-&gt;mdev, stat_rate_support);</div><div class='ctx'> 	while (rate != IB_RATE_PORT_CURRENT &amp;&amp;</div><div class='del'>-	       !(1 &lt;&lt; (rate + MLX5_STAT_RATE_OFFSET) &amp;</div><div class='del'>-		 MLX5_CAP_GEN(dev-&gt;mdev, stat_rate_support)))</div><div class='add'>+	       !(1 &lt;&lt; ib_to_mlx5_rate_map(rate) &amp; stat_rate_support))</div><div class='ctx'> 		--rate;</div><div class='ctx'> </div><div class='del'>-	return rate ? rate + MLX5_STAT_RATE_OFFSET : rate;</div><div class='add'>+	return ib_to_mlx5_rate_map(rate);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,</div><div class='hunk'>@@ -2876,7 +3486,7 @@ static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,</div><div class='ctx'> 	tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);</div><div class='ctx'> 	MLX5_SET(tisc, tisc, prio, ((sl &amp; 0x7) &lt;&lt; 1));</div><div class='ctx'> </div><div class='del'>-	err = mlx5_core_modify_tis(dev, sq-&gt;tisn, in, inlen);</div><div class='add'>+	err = mlx5_core_modify_tis(dev, sq-&gt;tisn, in);</div><div class='ctx'> </div><div class='ctx'> 	kvfree(in);</div><div class='ctx'> </div><div class='hunk'>@@ -2903,18 +3513,29 @@ static int modify_raw_packet_tx_affinity(struct mlx5_core_dev *dev,</div><div class='ctx'> 	tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);</div><div class='ctx'> 	MLX5_SET(tisc, tisc, lag_tx_port_affinity, tx_affinity);</div><div class='ctx'> </div><div class='del'>-	err = mlx5_core_modify_tis(dev, sq-&gt;tisn, in, inlen);</div><div class='add'>+	err = mlx5_core_modify_tis(dev, sq-&gt;tisn, in);</div><div class='ctx'> </div><div class='ctx'> 	kvfree(in);</div><div class='ctx'> </div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static void mlx5_set_path_udp_sport(void *path, const struct rdma_ah_attr *ah,</div><div class='add'>+				    u32 lqpn, u32 rqpn)</div><div class='add'>+</div><div class='add'>+{</div><div class='add'>+	u32 fl = ah-&gt;grh.flow_label;</div><div class='add'>+</div><div class='add'>+	if (!fl)</div><div class='add'>+		fl = rdma_calc_flow_label(lqpn, rqpn);</div><div class='add'>+</div><div class='add'>+	MLX5_SET(ads, path, udp_sport, rdma_flow_label_to_udp_sport(fl));</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='del'>-			 const struct rdma_ah_attr *ah,</div><div class='del'>-			 struct mlx5_qp_path *path, u8 port, int attr_mask,</div><div class='del'>-			 u32 path_flags, const struct ib_qp_attr *attr,</div><div class='del'>-			 bool alt)</div><div class='add'>+			 const struct rdma_ah_attr *ah, void *path, u8 port,</div><div class='add'>+			 int attr_mask, u32 path_flags,</div><div class='add'>+			 const struct ib_qp_attr *attr, bool alt)</div><div class='ctx'> {</div><div class='ctx'> 	const struct ib_global_route *grh = rdma_ah_read_grh(ah);</div><div class='ctx'> 	int err;</div><div class='hunk'>@@ -2923,15 +3544,17 @@ static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 	u8 sl = rdma_ah_get_sl(ah);</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_PKEY_INDEX)</div><div class='del'>-		path-&gt;pkey_index = cpu_to_be16(alt ? attr-&gt;alt_pkey_index :</div><div class='del'>-						     attr-&gt;pkey_index);</div><div class='add'>+		MLX5_SET(ads, path, pkey_index,</div><div class='add'>+			 alt ? attr-&gt;alt_pkey_index : attr-&gt;pkey_index);</div><div class='ctx'> </div><div class='ctx'> 	if (ah_flags &amp; IB_AH_GRH) {</div><div class='del'>-		if (grh-&gt;sgid_index &gt;=</div><div class='del'>-		    dev-&gt;mdev-&gt;port_caps[port - 1].gid_table_len) {</div><div class='add'>+		const struct ib_port_immutable *immutable;</div><div class='add'>+</div><div class='add'>+		immutable = ib_port_immutable_read(&amp;dev-&gt;ib_dev, port);</div><div class='add'>+		if (grh-&gt;sgid_index &gt;= immutable-&gt;gid_tbl_len) {</div><div class='ctx'> 			pr_err("sgid_index (%u) too large. max is %d\n",</div><div class='ctx'> 			       grh-&gt;sgid_index,</div><div class='del'>-			       dev-&gt;mdev-&gt;port_caps[port - 1].gid_table_len);</div><div class='add'>+			       immutable-&gt;gid_tbl_len);</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='hunk'>@@ -2940,47 +3563,51 @@ static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 		if (!(ah_flags &amp; IB_AH_GRH))</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> </div><div class='del'>-		memcpy(path-&gt;rmac, ah-&gt;roce.dmac, sizeof(ah-&gt;roce.dmac));</div><div class='del'>-		if (qp-&gt;ibqp.qp_type == IB_QPT_RC ||</div><div class='del'>-		    qp-&gt;ibqp.qp_type == IB_QPT_UC ||</div><div class='del'>-		    qp-&gt;ibqp.qp_type == IB_QPT_XRC_INI ||</div><div class='del'>-		    qp-&gt;ibqp.qp_type == IB_QPT_XRC_TGT)</div><div class='del'>-			path-&gt;udp_sport =</div><div class='del'>-				mlx5_get_roce_udp_sport(dev, ah-&gt;grh.sgid_attr);</div><div class='del'>-		path-&gt;dci_cfi_prio_sl = (sl &amp; 0x7) &lt;&lt; 4;</div><div class='add'>+		ether_addr_copy(MLX5_ADDR_OF(ads, path, rmac_47_32),</div><div class='add'>+				ah-&gt;roce.dmac);</div><div class='add'>+		if ((qp-&gt;type == IB_QPT_RC ||</div><div class='add'>+		     qp-&gt;type == IB_QPT_UC ||</div><div class='add'>+		     qp-&gt;type == IB_QPT_XRC_INI ||</div><div class='add'>+		     qp-&gt;type == IB_QPT_XRC_TGT) &amp;&amp;</div><div class='add'>+		    (grh-&gt;sgid_attr-&gt;gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) &amp;&amp;</div><div class='add'>+		    (attr_mask &amp; IB_QP_DEST_QPN))</div><div class='add'>+			mlx5_set_path_udp_sport(path, ah,</div><div class='add'>+						qp-&gt;ibqp.qp_num,</div><div class='add'>+						attr-&gt;dest_qp_num);</div><div class='add'>+		MLX5_SET(ads, path, eth_prio, sl &amp; 0x7);</div><div class='ctx'> 		gid_type = ah-&gt;grh.sgid_attr-&gt;gid_type;</div><div class='ctx'> 		if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)</div><div class='del'>-			path-&gt;ecn_dscp = (grh-&gt;traffic_class &gt;&gt; 2) &amp; 0x3f;</div><div class='add'>+			MLX5_SET(ads, path, dscp, grh-&gt;traffic_class &gt;&gt; 2);</div><div class='ctx'> 	} else {</div><div class='del'>-		path-&gt;fl_free_ar = (path_flags &amp; MLX5_PATH_FLAG_FL) ? 0x80 : 0;</div><div class='del'>-		path-&gt;fl_free_ar |=</div><div class='del'>-			(path_flags &amp; MLX5_PATH_FLAG_FREE_AR) ? 0x40 : 0;</div><div class='del'>-		path-&gt;rlid = cpu_to_be16(rdma_ah_get_dlid(ah));</div><div class='del'>-		path-&gt;grh_mlid = rdma_ah_get_path_bits(ah) &amp; 0x7f;</div><div class='del'>-		if (ah_flags &amp; IB_AH_GRH)</div><div class='del'>-			path-&gt;grh_mlid	|= 1 &lt;&lt; 7;</div><div class='del'>-		path-&gt;dci_cfi_prio_sl = sl &amp; 0xf;</div><div class='add'>+		MLX5_SET(ads, path, fl, !!(path_flags &amp; MLX5_PATH_FLAG_FL));</div><div class='add'>+		MLX5_SET(ads, path, free_ar,</div><div class='add'>+			 !!(path_flags &amp; MLX5_PATH_FLAG_FREE_AR));</div><div class='add'>+		MLX5_SET(ads, path, rlid, rdma_ah_get_dlid(ah));</div><div class='add'>+		MLX5_SET(ads, path, mlid, rdma_ah_get_path_bits(ah));</div><div class='add'>+		MLX5_SET(ads, path, grh, !!(ah_flags &amp; IB_AH_GRH));</div><div class='add'>+		MLX5_SET(ads, path, sl, sl);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (ah_flags &amp; IB_AH_GRH) {</div><div class='del'>-		path-&gt;mgid_index = grh-&gt;sgid_index;</div><div class='del'>-		path-&gt;hop_limit  = grh-&gt;hop_limit;</div><div class='del'>-		path-&gt;tclass_flowlabel =</div><div class='del'>-			cpu_to_be32((grh-&gt;traffic_class &lt;&lt; 20) |</div><div class='del'>-				    (grh-&gt;flow_label));</div><div class='del'>-		memcpy(path-&gt;rgid, grh-&gt;dgid.raw, 16);</div><div class='add'>+		MLX5_SET(ads, path, src_addr_index, grh-&gt;sgid_index);</div><div class='add'>+		MLX5_SET(ads, path, hop_limit, grh-&gt;hop_limit);</div><div class='add'>+		MLX5_SET(ads, path, tclass, grh-&gt;traffic_class);</div><div class='add'>+		MLX5_SET(ads, path, flow_label, grh-&gt;flow_label);</div><div class='add'>+		memcpy(MLX5_ADDR_OF(ads, path, rgid_rip), grh-&gt;dgid.raw,</div><div class='add'>+		       sizeof(grh-&gt;dgid.raw));</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	err = ib_rate_to_mlx5(dev, rdma_ah_get_static_rate(ah));</div><div class='add'>+	err = mlx5r_ib_rate(dev, rdma_ah_get_static_rate(ah));</div><div class='ctx'> 	if (err &lt; 0)</div><div class='ctx'> 		return err;</div><div class='del'>-	path-&gt;static_rate = err;</div><div class='del'>-	path-&gt;port = port;</div><div class='add'>+	MLX5_SET(ads, path, stat_rate, err);</div><div class='add'>+	MLX5_SET(ads, path, vhca_port_num, port);</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_TIMEOUT)</div><div class='del'>-		path-&gt;ackto_lt = (alt ? attr-&gt;alt_timeout : attr-&gt;timeout) &lt;&lt; 3;</div><div class='add'>+		MLX5_SET(ads, path, ack_timeout,</div><div class='add'>+			 alt ? attr-&gt;alt_timeout : attr-&gt;timeout);</div><div class='ctx'> </div><div class='del'>-	if ((qp-&gt;ibqp.qp_type == IB_QPT_RAW_PACKET) &amp;&amp; qp-&gt;sq.wqe_cnt)</div><div class='add'>+	if ((qp-&gt;type == IB_QPT_RAW_PACKET) &amp;&amp; qp-&gt;sq.wqe_cnt)</div><div class='ctx'> 		return modify_raw_packet_eth_prio(dev-&gt;mdev,</div><div class='ctx'> 						  &amp;qp-&gt;raw_packet_qp.sq,</div><div class='ctx'> 						  sl &amp; 0xf, qp-&gt;ibqp.pd);</div><div class='hunk'>@@ -2995,10 +3622,12 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q</div><div class='ctx'> 					  MLX5_QP_OPTPAR_RAE		|</div><div class='ctx'> 					  MLX5_QP_OPTPAR_RWE		|</div><div class='ctx'> 					  MLX5_QP_OPTPAR_PKEY_INDEX	|</div><div class='del'>-					  MLX5_QP_OPTPAR_PRI_PORT,</div><div class='add'>+					  MLX5_QP_OPTPAR_PRI_PORT	|</div><div class='add'>+					  MLX5_QP_OPTPAR_LAG_TX_AFF,</div><div class='ctx'> 			[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE		|</div><div class='ctx'> 					  MLX5_QP_OPTPAR_PKEY_INDEX	|</div><div class='del'>-					  MLX5_QP_OPTPAR_PRI_PORT,</div><div class='add'>+					  MLX5_QP_OPTPAR_PRI_PORT	|</div><div class='add'>+					  MLX5_QP_OPTPAR_LAG_TX_AFF,</div><div class='ctx'> 			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX	|</div><div class='ctx'> 					  MLX5_QP_OPTPAR_Q_KEY		|</div><div class='ctx'> 					  MLX5_QP_OPTPAR_PRI_PORT,</div><div class='hunk'>@@ -3006,17 +3635,20 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q</div><div class='ctx'> 					  MLX5_QP_OPTPAR_RAE		|</div><div class='ctx'> 					  MLX5_QP_OPTPAR_RWE		|</div><div class='ctx'> 					  MLX5_QP_OPTPAR_PKEY_INDEX	|</div><div class='del'>-					  MLX5_QP_OPTPAR_PRI_PORT,</div><div class='add'>+					  MLX5_QP_OPTPAR_PRI_PORT	|</div><div class='add'>+					  MLX5_QP_OPTPAR_LAG_TX_AFF,</div><div class='ctx'> 		},</div><div class='ctx'> 		[MLX5_QP_STATE_RTR] = {</div><div class='ctx'> 			[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH  |</div><div class='ctx'> 					  MLX5_QP_OPTPAR_RRE            |</div><div class='ctx'> 					  MLX5_QP_OPTPAR_RAE            |</div><div class='ctx'> 					  MLX5_QP_OPTPAR_RWE            |</div><div class='del'>-					  MLX5_QP_OPTPAR_PKEY_INDEX,</div><div class='add'>+					  MLX5_QP_OPTPAR_PKEY_INDEX	|</div><div class='add'>+					  MLX5_QP_OPTPAR_LAG_TX_AFF,</div><div class='ctx'> 			[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH  |</div><div class='ctx'> 					  MLX5_QP_OPTPAR_RWE            |</div><div class='del'>-					  MLX5_QP_OPTPAR_PKEY_INDEX,</div><div class='add'>+					  MLX5_QP_OPTPAR_PKEY_INDEX	|</div><div class='add'>+					  MLX5_QP_OPTPAR_LAG_TX_AFF,</div><div class='ctx'> 			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX     |</div><div class='ctx'> 					  MLX5_QP_OPTPAR_Q_KEY,</div><div class='ctx'> 			[MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_PKEY_INDEX	|</div><div class='hunk'>@@ -3025,7 +3657,8 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q</div><div class='ctx'> 					  MLX5_QP_OPTPAR_RRE            |</div><div class='ctx'> 					  MLX5_QP_OPTPAR_RAE            |</div><div class='ctx'> 					  MLX5_QP_OPTPAR_RWE            |</div><div class='del'>-					  MLX5_QP_OPTPAR_PKEY_INDEX,</div><div class='add'>+					  MLX5_QP_OPTPAR_PKEY_INDEX	|</div><div class='add'>+					  MLX5_QP_OPTPAR_LAG_TX_AFF,</div><div class='ctx'> 		},</div><div class='ctx'> 	},</div><div class='ctx'> 	[MLX5_QP_STATE_RTR] = {</div><div class='hunk'>@@ -3085,6 +3718,17 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q</div><div class='ctx'> 					   MLX5_QP_OPTPAR_RRE,</div><div class='ctx'> 		},</div><div class='ctx'> 	},</div><div class='add'>+	[MLX5_QP_STATE_SQD] = {</div><div class='add'>+		[MLX5_QP_STATE_RTS] = {</div><div class='add'>+			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,</div><div class='add'>+			[MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_Q_KEY,</div><div class='add'>+			[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE,</div><div class='add'>+			[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RNR_TIMEOUT	|</div><div class='add'>+					  MLX5_QP_OPTPAR_RWE		|</div><div class='add'>+					  MLX5_QP_OPTPAR_RAE		|</div><div class='add'>+					  MLX5_QP_OPTPAR_RRE,</div><div class='add'>+		},</div><div class='add'>+	},</div><div class='ctx'> };</div><div class='ctx'> </div><div class='ctx'> static int ib_nr_to_mlx5_nr(int ib_mask)</div><div class='hunk'>@@ -3183,7 +3827,7 @@ static int modify_raw_packet_qp_rq(</div><div class='ctx'> 				"RAW PACKET QP counters are not supported on current FW\n");</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	err = mlx5_core_modify_rq(dev-&gt;mdev, rq-&gt;base.mqp.qpn, in, inlen);</div><div class='add'>+	err = mlx5_core_modify_rq(dev-&gt;mdev, rq-&gt;base.mqp.qpn, in);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='hunk'>@@ -3246,7 +3890,7 @@ static int modify_raw_packet_qp_sq(</div><div class='ctx'> 		MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, rl_index);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	err = mlx5_core_modify_sq(dev, sq-&gt;base.mqp.qpn, in, inlen);</div><div class='add'>+	err = mlx5_core_modify_sq(dev, sq-&gt;base.mqp.qpn, in);</div><div class='ctx'> 	if (err) {</div><div class='ctx'> 		/* Remove new rate from table if failed */</div><div class='ctx'> 		if (new_rate_added)</div><div class='hunk'>@@ -3255,10 +3899,12 @@ static int modify_raw_packet_qp_sq(</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	/* Only remove the old rate after new rate was set */</div><div class='del'>-	if ((old_rl.rate &amp;&amp;</div><div class='del'>-	     !mlx5_rl_are_equal(&amp;old_rl, &amp;new_rl)) ||</div><div class='del'>-	    (new_state != MLX5_SQC_STATE_RDY))</div><div class='add'>+	if ((old_rl.rate &amp;&amp; !mlx5_rl_are_equal(&amp;old_rl, &amp;new_rl)) ||</div><div class='add'>+	    (new_state != MLX5_SQC_STATE_RDY)) {</div><div class='ctx'> 		mlx5_rl_remove_rate(dev, &amp;old_rl);</div><div class='add'>+		if (new_state != MLX5_SQC_STATE_RDY)</div><div class='add'>+			memset(&amp;new_rl, 0, sizeof(new_rl));</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	ibqp-&gt;rl = new_rl;</div><div class='ctx'> 	sq-&gt;state = new_state;</div><div class='hunk'>@@ -3284,7 +3930,7 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 	switch (raw_qp_param-&gt;operation) {</div><div class='ctx'> 	case MLX5_CMD_OP_RST2INIT_QP:</div><div class='ctx'> 		rq_state = MLX5_RQC_STATE_RDY;</div><div class='del'>-		sq_state = MLX5_SQC_STATE_RDY;</div><div class='add'>+		sq_state = MLX5_SQC_STATE_RST;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_2ERR_QP:</div><div class='ctx'> 		rq_state = MLX5_RQC_STATE_ERR;</div><div class='hunk'>@@ -3296,13 +3942,11 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_RTR2RTS_QP:</div><div class='ctx'> 	case MLX5_CMD_OP_RTS2RTS_QP:</div><div class='del'>-		if (raw_qp_param-&gt;set_mask ==</div><div class='del'>-		    MLX5_RAW_QP_RATE_LIMIT) {</div><div class='del'>-			modify_rq = 0;</div><div class='del'>-			sq_state = sq-&gt;state;</div><div class='del'>-		} else {</div><div class='del'>-			return raw_qp_param-&gt;set_mask ? -EINVAL : 0;</div><div class='del'>-		}</div><div class='add'>+		if (raw_qp_param-&gt;set_mask &amp; ~MLX5_RAW_QP_RATE_LIMIT)</div><div class='add'>+			return -EINVAL;</div><div class='add'>+</div><div class='add'>+		modify_rq = 0;</div><div class='add'>+		sq_state = MLX5_SQC_STATE_RDY;</div><div class='ctx'> 		break;</div><div class='ctx'> 	case MLX5_CMD_OP_INIT2INIT_QP:</div><div class='ctx'> 	case MLX5_CMD_OP_INIT2RTR_QP:</div><div class='hunk'>@@ -3357,33 +4001,121 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static unsigned int get_tx_affinity(struct mlx5_ib_dev *dev,</div><div class='del'>-				    struct mlx5_ib_pd *pd,</div><div class='del'>-				    struct mlx5_ib_qp_base *qp_base,</div><div class='del'>-				    u8 port_num, struct ib_udata *udata)</div><div class='add'>+static unsigned int get_tx_affinity_rr(struct mlx5_ib_dev *dev,</div><div class='add'>+				       struct ib_udata *udata)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(</div><div class='add'>+		udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='add'>+	u8 port_num = mlx5_core_native_port_num(dev-&gt;mdev) - 1;</div><div class='add'>+	atomic_t *tx_port_affinity;</div><div class='add'>+</div><div class='add'>+	if (ucontext)</div><div class='add'>+		tx_port_affinity = &amp;ucontext-&gt;tx_port_affinity;</div><div class='add'>+	else</div><div class='add'>+		tx_port_affinity = &amp;dev-&gt;port[port_num].roce.tx_port_affinity;</div><div class='add'>+</div><div class='add'>+	return (unsigned int)atomic_add_return(1, tx_port_affinity) %</div><div class='add'>+		(dev-&gt;lag_active ? dev-&gt;lag_ports : MLX5_CAP_GEN(dev-&gt;mdev, num_lag_ports)) + 1;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool qp_supports_affinity(struct mlx5_ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	if ((qp-&gt;type == IB_QPT_RC) || (qp-&gt;type == IB_QPT_UD) ||</div><div class='add'>+	    (qp-&gt;type == IB_QPT_UC) || (qp-&gt;type == IB_QPT_RAW_PACKET) ||</div><div class='add'>+	    (qp-&gt;type == IB_QPT_XRC_INI) || (qp-&gt;type == IB_QPT_XRC_TGT) ||</div><div class='add'>+	    (qp-&gt;type == MLX5_IB_QPT_DCI))</div><div class='add'>+		return true;</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static unsigned int get_tx_affinity(struct ib_qp *qp,</div><div class='add'>+				    const struct ib_qp_attr *attr,</div><div class='add'>+				    int attr_mask, u8 init,</div><div class='add'>+				    struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(</div><div class='ctx'> 		udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='del'>-	unsigned int tx_port_affinity;</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(qp-&gt;device);</div><div class='add'>+	struct mlx5_ib_qp *mqp = to_mqp(qp);</div><div class='add'>+	struct mlx5_ib_qp_base *qp_base;</div><div class='add'>+	unsigned int tx_affinity;</div><div class='add'>+</div><div class='add'>+	if (!(mlx5_ib_lag_should_assign_affinity(dev) &amp;&amp;</div><div class='add'>+	      qp_supports_affinity(mqp)))</div><div class='add'>+		return 0;</div><div class='ctx'> </div><div class='del'>-	if (ucontext) {</div><div class='del'>-		tx_port_affinity = (unsigned int)atomic_add_return(</div><div class='del'>-					   1, &amp;ucontext-&gt;tx_port_affinity) %</div><div class='del'>-					   MLX5_MAX_PORTS +</div><div class='del'>-				   1;</div><div class='add'>+	if (mqp-&gt;flags &amp; MLX5_IB_QP_CREATE_SQPN_QP1)</div><div class='add'>+		tx_affinity = mqp-&gt;gsi_lag_port;</div><div class='add'>+	else if (init)</div><div class='add'>+		tx_affinity = get_tx_affinity_rr(dev, udata);</div><div class='add'>+	else if ((attr_mask &amp; IB_QP_AV) &amp;&amp; attr-&gt;xmit_slave)</div><div class='add'>+		tx_affinity =</div><div class='add'>+			mlx5_lag_get_slave_port(dev-&gt;mdev, attr-&gt;xmit_slave);</div><div class='add'>+	else</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	qp_base = &amp;mqp-&gt;trans_qp.base;</div><div class='add'>+	if (ucontext)</div><div class='ctx'> 		mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x ucontext %p\n",</div><div class='del'>-				tx_port_affinity, qp_base-&gt;mqp.qpn, ucontext);</div><div class='del'>-	} else {</div><div class='del'>-		tx_port_affinity =</div><div class='del'>-			(unsigned int)atomic_add_return(</div><div class='del'>-				1, &amp;dev-&gt;port[port_num].roce.tx_port_affinity) %</div><div class='del'>-				MLX5_MAX_PORTS +</div><div class='del'>-			1;</div><div class='add'>+			    tx_affinity, qp_base-&gt;mqp.qpn, ucontext);</div><div class='add'>+	else</div><div class='ctx'> 		mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x\n",</div><div class='del'>-				tx_port_affinity, qp_base-&gt;mqp.qpn);</div><div class='del'>-	}</div><div class='add'>+			    tx_affinity, qp_base-&gt;mqp.qpn);</div><div class='add'>+	return tx_affinity;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int __mlx5_ib_qp_set_raw_qp_counter(struct mlx5_ib_qp *qp, u32 set_id,</div><div class='add'>+					   struct mlx5_core_dev *mdev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &amp;qp-&gt;raw_packet_qp;</div><div class='add'>+	struct mlx5_ib_rq *rq = &amp;raw_packet_qp-&gt;rq;</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {};</div><div class='add'>+	void *rqc;</div><div class='add'>+</div><div class='add'>+	if (!qp-&gt;rq.wqe_cnt)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(modify_rq_in, in, rq_state, rq-&gt;state);</div><div class='add'>+	MLX5_SET(modify_rq_in, in, uid, to_mpd(qp-&gt;ibqp.pd)-&gt;uid);</div><div class='add'>+</div><div class='add'>+	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);</div><div class='add'>+	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);</div><div class='add'>+</div><div class='add'>+	MLX5_SET64(modify_rq_in, in, modify_bitmask,</div><div class='add'>+		   MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);</div><div class='add'>+	MLX5_SET(rqc, rqc, counter_set_id, set_id);</div><div class='ctx'> </div><div class='del'>-	return tx_port_affinity;</div><div class='add'>+	return mlx5_core_modify_rq(mdev, rq-&gt;base.mqp.qpn, in);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int __mlx5_ib_qp_set_counter(struct ib_qp *qp,</div><div class='add'>+				    struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(qp-&gt;device);</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(rts2rts_qp_in)] = {};</div><div class='add'>+	struct mlx5_ib_qp *mqp = to_mqp(qp);</div><div class='add'>+	struct mlx5_ib_qp_base *base;</div><div class='add'>+	u32 set_id;</div><div class='add'>+	u32 *qpc;</div><div class='add'>+</div><div class='add'>+	if (counter)</div><div class='add'>+		set_id = counter-&gt;id;</div><div class='add'>+	else</div><div class='add'>+		set_id = mlx5_ib_get_counters_id(dev, mqp-&gt;port - 1);</div><div class='add'>+</div><div class='add'>+	if (mqp-&gt;type == IB_QPT_RAW_PACKET)</div><div class='add'>+		return __mlx5_ib_qp_set_raw_qp_counter(mqp, set_id, dev-&gt;mdev);</div><div class='add'>+</div><div class='add'>+	base = &amp;mqp-&gt;trans_qp.base;</div><div class='add'>+	MLX5_SET(rts2rts_qp_in, in, opcode, MLX5_CMD_OP_RTS2RTS_QP);</div><div class='add'>+	MLX5_SET(rts2rts_qp_in, in, qpn, base-&gt;mqp.qpn);</div><div class='add'>+	MLX5_SET(rts2rts_qp_in, in, uid, base-&gt;mqp.uid);</div><div class='add'>+	MLX5_SET(rts2rts_qp_in, in, opt_param_mask,</div><div class='add'>+		 MLX5_QP_OPTPAR_COUNTER_SET_ID);</div><div class='add'>+</div><div class='add'>+	qpc = MLX5_ADDR_OF(rts2rts_qp_in, in, qpc);</div><div class='add'>+	MLX5_SET(qpc, qpc, counter_set_id, set_id);</div><div class='add'>+	return mlx5_cmd_exec_in(dev-&gt;mdev, rts2rts_qp, in);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,</div><div class='hunk'>@@ -3391,6 +4123,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,</div><div class='ctx'> 			       enum ib_qp_state cur_state,</div><div class='ctx'> 			       enum ib_qp_state new_state,</div><div class='ctx'> 			       const struct mlx5_ib_modify_qp *ucmd,</div><div class='add'>+			       struct mlx5_ib_modify_qp_resp *resp,</div><div class='ctx'> 			       struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	static const u16 optab[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE] = {</div><div class='hunk'>@@ -3418,6 +4151,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,</div><div class='ctx'> 		[MLX5_QP_STATE_SQD] = {</div><div class='ctx'> 			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,</div><div class='ctx'> 			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,</div><div class='add'>+			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_SQD_RTS_QP,</div><div class='ctx'> 		},</div><div class='ctx'> 		[MLX5_QP_STATE_SQER] = {</div><div class='ctx'> 			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,</div><div class='hunk'>@@ -3434,67 +4168,60 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,</div><div class='ctx'> 	struct mlx5_ib_qp *qp = to_mqp(ibqp);</div><div class='ctx'> 	struct mlx5_ib_qp_base *base = &amp;qp-&gt;trans_qp.base;</div><div class='ctx'> 	struct mlx5_ib_cq *send_cq, *recv_cq;</div><div class='del'>-	struct mlx5_qp_context *context;</div><div class='ctx'> 	struct mlx5_ib_pd *pd;</div><div class='del'>-	struct mlx5_ib_port *mibport = NULL;</div><div class='ctx'> 	enum mlx5_qp_state mlx5_cur, mlx5_new;</div><div class='del'>-	enum mlx5_qp_optpar optpar;</div><div class='add'>+	void *qpc, *pri_path, *alt_path;</div><div class='add'>+	enum mlx5_qp_optpar optpar = 0;</div><div class='add'>+	u32 set_id = 0;</div><div class='ctx'> 	int mlx5_st;</div><div class='ctx'> 	int err;</div><div class='ctx'> 	u16 op;</div><div class='ctx'> 	u8 tx_affinity = 0;</div><div class='ctx'> </div><div class='del'>-	mlx5_st = to_mlx5_st(ibqp-&gt;qp_type == IB_QPT_DRIVER ?</div><div class='del'>-			     qp-&gt;qp_sub_type : ibqp-&gt;qp_type);</div><div class='add'>+	mlx5_st = to_mlx5_st(qp-&gt;type);</div><div class='ctx'> 	if (mlx5_st &lt; 0)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='del'>-	context = kzalloc(sizeof(*context), GFP_KERNEL);</div><div class='del'>-	if (!context)</div><div class='add'>+	qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL);</div><div class='add'>+	if (!qpc)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	pd = get_pd(qp);</div><div class='del'>-	context-&gt;flags = cpu_to_be32(mlx5_st &lt;&lt; 16);</div><div class='add'>+	pd = to_mpd(qp-&gt;ibqp.pd);</div><div class='add'>+	MLX5_SET(qpc, qpc, st, mlx5_st);</div><div class='ctx'> </div><div class='ctx'> 	if (!(attr_mask &amp; IB_QP_PATH_MIG_STATE)) {</div><div class='del'>-		context-&gt;flags |= cpu_to_be32(MLX5_QP_PM_MIGRATED &lt;&lt; 11);</div><div class='add'>+		MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);</div><div class='ctx'> 	} else {</div><div class='ctx'> 		switch (attr-&gt;path_mig_state) {</div><div class='ctx'> 		case IB_MIG_MIGRATED:</div><div class='del'>-			context-&gt;flags |= cpu_to_be32(MLX5_QP_PM_MIGRATED &lt;&lt; 11);</div><div class='add'>+			MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);</div><div class='ctx'> 			break;</div><div class='ctx'> 		case IB_MIG_REARM:</div><div class='del'>-			context-&gt;flags |= cpu_to_be32(MLX5_QP_PM_REARM &lt;&lt; 11);</div><div class='add'>+			MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_REARM);</div><div class='ctx'> 			break;</div><div class='ctx'> 		case IB_MIG_ARMED:</div><div class='del'>-			context-&gt;flags |= cpu_to_be32(MLX5_QP_PM_ARMED &lt;&lt; 11);</div><div class='add'>+			MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_ARMED);</div><div class='ctx'> 			break;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if ((cur_state == IB_QPS_RESET) &amp;&amp; (new_state == IB_QPS_INIT)) {</div><div class='del'>-		if ((ibqp-&gt;qp_type == IB_QPT_RC) ||</div><div class='del'>-		    (ibqp-&gt;qp_type == IB_QPT_UD &amp;&amp;</div><div class='del'>-		     !(qp-&gt;flags &amp; MLX5_IB_QP_SQPN_QP1)) ||</div><div class='del'>-		    (ibqp-&gt;qp_type == IB_QPT_UC) ||</div><div class='del'>-		    (ibqp-&gt;qp_type == IB_QPT_RAW_PACKET) ||</div><div class='del'>-		    (ibqp-&gt;qp_type == IB_QPT_XRC_INI) ||</div><div class='del'>-		    (ibqp-&gt;qp_type == IB_QPT_XRC_TGT)) {</div><div class='del'>-			if (dev-&gt;lag_active) {</div><div class='del'>-				u8 p = mlx5_core_native_port_num(dev-&gt;mdev) - 1;</div><div class='del'>-				tx_affinity = get_tx_affinity(dev, pd, base, p,</div><div class='del'>-							      udata);</div><div class='del'>-				context-&gt;flags |= cpu_to_be32(tx_affinity &lt;&lt; 24);</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='add'>+	tx_affinity = get_tx_affinity(ibqp, attr, attr_mask,</div><div class='add'>+				      cur_state == IB_QPS_RESET &amp;&amp;</div><div class='add'>+				      new_state == IB_QPS_INIT, udata);</div><div class='ctx'> </div><div class='del'>-	if (is_sqp(ibqp-&gt;qp_type)) {</div><div class='del'>-		context-&gt;mtu_msgmax = (IB_MTU_256 &lt;&lt; 5) | 8;</div><div class='del'>-	} else if ((ibqp-&gt;qp_type == IB_QPT_UD &amp;&amp;</div><div class='del'>-		    !(qp-&gt;flags &amp; MLX5_IB_QP_UNDERLAY)) ||</div><div class='del'>-		   ibqp-&gt;qp_type == MLX5_IB_QPT_REG_UMR) {</div><div class='del'>-		context-&gt;mtu_msgmax = (IB_MTU_4096 &lt;&lt; 5) | 12;</div><div class='add'>+	MLX5_SET(qpc, qpc, lag_tx_port_affinity, tx_affinity);</div><div class='add'>+	if (tx_affinity &amp;&amp; new_state == IB_QPS_RTR &amp;&amp;</div><div class='add'>+	    MLX5_CAP_GEN(dev-&gt;mdev, init2_lag_tx_port_affinity))</div><div class='add'>+		optpar |= MLX5_QP_OPTPAR_LAG_TX_AFF;</div><div class='add'>+</div><div class='add'>+	if (is_sqp(qp-&gt;type)) {</div><div class='add'>+		MLX5_SET(qpc, qpc, mtu, IB_MTU_256);</div><div class='add'>+		MLX5_SET(qpc, qpc, log_msg_max, 8);</div><div class='add'>+	} else if ((qp-&gt;type == IB_QPT_UD &amp;&amp;</div><div class='add'>+		    !(qp-&gt;flags &amp; IB_QP_CREATE_SOURCE_QPN)) ||</div><div class='add'>+		   qp-&gt;type == MLX5_IB_QPT_REG_UMR) {</div><div class='add'>+		MLX5_SET(qpc, qpc, mtu, IB_MTU_4096);</div><div class='add'>+		MLX5_SET(qpc, qpc, log_msg_max, 12);</div><div class='ctx'> 	} else if (attr_mask &amp; IB_QP_PATH_MTU) {</div><div class='ctx'> 		if (attr-&gt;path_mtu &lt; IB_MTU_256 ||</div><div class='ctx'> 		    attr-&gt;path_mtu &gt; IB_MTU_4096) {</div><div class='hunk'>@@ -3502,114 +4229,125 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,</div><div class='ctx'> 			err = -EINVAL;</div><div class='ctx'> 			goto out;</div><div class='ctx'> 		}</div><div class='del'>-		context-&gt;mtu_msgmax = (attr-&gt;path_mtu &lt;&lt; 5) |</div><div class='del'>-				      (u8)MLX5_CAP_GEN(dev-&gt;mdev, log_max_msg);</div><div class='add'>+		MLX5_SET(qpc, qpc, mtu, attr-&gt;path_mtu);</div><div class='add'>+		MLX5_SET(qpc, qpc, log_msg_max,</div><div class='add'>+			 MLX5_CAP_GEN(dev-&gt;mdev, log_max_msg));</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_DEST_QPN)</div><div class='del'>-		context-&gt;log_pg_sz_remote_qpn = cpu_to_be32(attr-&gt;dest_qp_num);</div><div class='add'>+		MLX5_SET(qpc, qpc, remote_qpn, attr-&gt;dest_qp_num);</div><div class='add'>+</div><div class='add'>+	pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);</div><div class='add'>+	alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_PKEY_INDEX)</div><div class='del'>-		context-&gt;pri_path.pkey_index = cpu_to_be16(attr-&gt;pkey_index);</div><div class='add'>+		MLX5_SET(ads, pri_path, pkey_index, attr-&gt;pkey_index);</div><div class='ctx'> </div><div class='ctx'> 	/* todo implement counter_index functionality */</div><div class='ctx'> </div><div class='del'>-	if (is_sqp(ibqp-&gt;qp_type))</div><div class='del'>-		context-&gt;pri_path.port = qp-&gt;port;</div><div class='add'>+	if (dev-&gt;ib_dev.type == RDMA_DEVICE_TYPE_SMI &amp;&amp; is_qp0(qp-&gt;type)) {</div><div class='add'>+		MLX5_SET(ads, pri_path, vhca_port_num,</div><div class='add'>+			 smi_to_native_portnum(dev, qp-&gt;port));</div><div class='add'>+		if (cur_state == IB_QPS_INIT &amp;&amp; new_state == IB_QPS_RTR)</div><div class='add'>+			MLX5_SET(ads, pri_path, plane_index, qp-&gt;port);</div><div class='add'>+	} else if (is_sqp(qp-&gt;type))</div><div class='add'>+		MLX5_SET(ads, pri_path, vhca_port_num, qp-&gt;port);</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_PORT)</div><div class='del'>-		context-&gt;pri_path.port = attr-&gt;port_num;</div><div class='add'>+		MLX5_SET(ads, pri_path, vhca_port_num, attr-&gt;port_num);</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_AV) {</div><div class='del'>-		err = mlx5_set_path(dev, qp, &amp;attr-&gt;ah_attr, &amp;context-&gt;pri_path,</div><div class='del'>-				    attr_mask &amp; IB_QP_PORT ? attr-&gt;port_num : qp-&gt;port,</div><div class='add'>+		err = mlx5_set_path(dev, qp, &amp;attr-&gt;ah_attr, pri_path,</div><div class='add'>+				    attr_mask &amp; IB_QP_PORT ? attr-&gt;port_num :</div><div class='add'>+							     qp-&gt;port,</div><div class='ctx'> 				    attr_mask, 0, attr, false);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_TIMEOUT)</div><div class='del'>-		context-&gt;pri_path.ackto_lt |= attr-&gt;timeout &lt;&lt; 3;</div><div class='add'>+		MLX5_SET(ads, pri_path, ack_timeout, attr-&gt;timeout);</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_ALT_PATH) {</div><div class='del'>-		err = mlx5_set_path(dev, qp, &amp;attr-&gt;alt_ah_attr,</div><div class='del'>-				    &amp;context-&gt;alt_path,</div><div class='add'>+		err = mlx5_set_path(dev, qp, &amp;attr-&gt;alt_ah_attr, alt_path,</div><div class='ctx'> 				    attr-&gt;alt_port_num,</div><div class='del'>-				    attr_mask | IB_QP_PKEY_INDEX | IB_QP_TIMEOUT,</div><div class='add'>+				    attr_mask | IB_QP_PKEY_INDEX |</div><div class='add'>+					    IB_QP_TIMEOUT,</div><div class='ctx'> 				    0, attr, true);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	get_cqs(qp-&gt;ibqp.qp_type, qp-&gt;ibqp.send_cq, qp-&gt;ibqp.recv_cq,</div><div class='add'>+	get_cqs(qp-&gt;type, qp-&gt;ibqp.send_cq, qp-&gt;ibqp.recv_cq,</div><div class='ctx'> 		&amp;send_cq, &amp;recv_cq);</div><div class='ctx'> </div><div class='del'>-	context-&gt;flags_pd = cpu_to_be32(pd ? pd-&gt;pdn : to_mpd(dev-&gt;devr.p0)-&gt;pdn);</div><div class='del'>-	context-&gt;cqn_send = send_cq ? cpu_to_be32(send_cq-&gt;mcq.cqn) : 0;</div><div class='del'>-	context-&gt;cqn_recv = recv_cq ? cpu_to_be32(recv_cq-&gt;mcq.cqn) : 0;</div><div class='del'>-	context-&gt;params1  = cpu_to_be32(MLX5_IB_ACK_REQ_FREQ &lt;&lt; 28);</div><div class='add'>+	MLX5_SET(qpc, qpc, pd, pd ? pd-&gt;pdn : to_mpd(dev-&gt;devr.p0)-&gt;pdn);</div><div class='add'>+	if (send_cq)</div><div class='add'>+		MLX5_SET(qpc, qpc, cqn_snd, send_cq-&gt;mcq.cqn);</div><div class='add'>+	if (recv_cq)</div><div class='add'>+		MLX5_SET(qpc, qpc, cqn_rcv, recv_cq-&gt;mcq.cqn);</div><div class='add'>+</div><div class='add'>+	MLX5_SET(qpc, qpc, log_ack_req_freq, MLX5_IB_ACK_REQ_FREQ);</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_RNR_RETRY)</div><div class='del'>-		context-&gt;params1 |= cpu_to_be32(attr-&gt;rnr_retry &lt;&lt; 13);</div><div class='add'>+		MLX5_SET(qpc, qpc, rnr_retry, attr-&gt;rnr_retry);</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_RETRY_CNT)</div><div class='del'>-		context-&gt;params1 |= cpu_to_be32(attr-&gt;retry_cnt &lt;&lt; 16);</div><div class='add'>+		MLX5_SET(qpc, qpc, retry_count, attr-&gt;retry_cnt);</div><div class='ctx'> </div><div class='del'>-	if (attr_mask &amp; IB_QP_MAX_QP_RD_ATOMIC) {</div><div class='del'>-		if (attr-&gt;max_rd_atomic)</div><div class='del'>-			context-&gt;params1 |=</div><div class='del'>-				cpu_to_be32(fls(attr-&gt;max_rd_atomic - 1) &lt;&lt; 21);</div><div class='del'>-	}</div><div class='add'>+	if (attr_mask &amp; IB_QP_MAX_QP_RD_ATOMIC &amp;&amp; attr-&gt;max_rd_atomic)</div><div class='add'>+		MLX5_SET(qpc, qpc, log_sra_max, fls(attr-&gt;max_rd_atomic - 1));</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_SQ_PSN)</div><div class='del'>-		context-&gt;next_send_psn = cpu_to_be32(attr-&gt;sq_psn);</div><div class='add'>+		MLX5_SET(qpc, qpc, next_send_psn, attr-&gt;sq_psn);</div><div class='ctx'> </div><div class='del'>-	if (attr_mask &amp; IB_QP_MAX_DEST_RD_ATOMIC) {</div><div class='del'>-		if (attr-&gt;max_dest_rd_atomic)</div><div class='del'>-			context-&gt;params2 |=</div><div class='del'>-				cpu_to_be32(fls(attr-&gt;max_dest_rd_atomic - 1) &lt;&lt; 21);</div><div class='del'>-	}</div><div class='add'>+	if (attr_mask &amp; IB_QP_MAX_DEST_RD_ATOMIC &amp;&amp; attr-&gt;max_dest_rd_atomic)</div><div class='add'>+		MLX5_SET(qpc, qpc, log_rra_max,</div><div class='add'>+			 fls(attr-&gt;max_dest_rd_atomic - 1));</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {</div><div class='del'>-		__be32 access_flags;</div><div class='del'>-</div><div class='del'>-		err = to_mlx5_access_flags(qp, attr, attr_mask, &amp;access_flags);</div><div class='add'>+		err = set_qpc_atomic_flags(qp, attr, attr_mask, qpc);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto out;</div><div class='del'>-</div><div class='del'>-		context-&gt;params2 |= access_flags;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_MIN_RNR_TIMER)</div><div class='del'>-		context-&gt;rnr_nextrecvpsn |= cpu_to_be32(attr-&gt;min_rnr_timer &lt;&lt; 24);</div><div class='add'>+		MLX5_SET(qpc, qpc, min_rnr_nak, attr-&gt;min_rnr_timer);</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_RQ_PSN)</div><div class='del'>-		context-&gt;rnr_nextrecvpsn |= cpu_to_be32(attr-&gt;rq_psn);</div><div class='add'>+		MLX5_SET(qpc, qpc, next_rcv_psn, attr-&gt;rq_psn);</div><div class='ctx'> </div><div class='ctx'> 	if (attr_mask &amp; IB_QP_QKEY)</div><div class='del'>-		context-&gt;qkey = cpu_to_be32(attr-&gt;qkey);</div><div class='add'>+		MLX5_SET(qpc, qpc, q_key, attr-&gt;qkey);</div><div class='ctx'> </div><div class='ctx'> 	if (qp-&gt;rq.wqe_cnt &amp;&amp; cur_state == IB_QPS_RESET &amp;&amp; new_state == IB_QPS_INIT)</div><div class='del'>-		context-&gt;db_rec_addr = cpu_to_be64(qp-&gt;db.dma);</div><div class='add'>+		MLX5_SET64(qpc, qpc, dbr_addr, qp-&gt;db.dma);</div><div class='ctx'> </div><div class='ctx'> 	if (cur_state == IB_QPS_RESET &amp;&amp; new_state == IB_QPS_INIT) {</div><div class='ctx'> 		u8 port_num = (attr_mask &amp; IB_QP_PORT ? attr-&gt;port_num :</div><div class='ctx'> 			       qp-&gt;port) - 1;</div><div class='ctx'> </div><div class='ctx'> 		/* Underlay port should be used - index 0 function per port */</div><div class='del'>-		if (qp-&gt;flags &amp; MLX5_IB_QP_UNDERLAY)</div><div class='add'>+		if (qp-&gt;flags &amp; IB_QP_CREATE_SOURCE_QPN)</div><div class='ctx'> 			port_num = 0;</div><div class='ctx'> </div><div class='del'>-		mibport = &amp;dev-&gt;port[port_num];</div><div class='del'>-		context-&gt;qp_counter_set_usr_page |=</div><div class='del'>-			cpu_to_be32((u32)(mibport-&gt;cnts.set_id) &lt;&lt; 24);</div><div class='add'>+		if (ibqp-&gt;counter)</div><div class='add'>+			set_id = ibqp-&gt;counter-&gt;id;</div><div class='add'>+		else</div><div class='add'>+			set_id = mlx5_ib_get_counters_id(dev, port_num);</div><div class='add'>+		MLX5_SET(qpc, qpc, counter_set_id, set_id);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (!ibqp-&gt;uobject &amp;&amp; cur_state == IB_QPS_RESET &amp;&amp; new_state == IB_QPS_INIT)</div><div class='del'>-		context-&gt;sq_crq_size |= cpu_to_be16(1 &lt;&lt; 4);</div><div class='add'>+		MLX5_SET(qpc, qpc, rlky, 1);</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;flags &amp; MLX5_IB_QP_SQPN_QP1)</div><div class='del'>-		context-&gt;deth_sqpn = cpu_to_be32(1);</div><div class='add'>+	if (qp-&gt;flags &amp; MLX5_IB_QP_CREATE_SQPN_QP1)</div><div class='add'>+		MLX5_SET(qpc, qpc, deth_sqpn, 1);</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;is_ooo_rq &amp;&amp; cur_state == IB_QPS_INIT &amp;&amp; new_state == IB_QPS_RTR) {</div><div class='add'>+		MLX5_SET(qpc, qpc, dp_ordering_1, 1);</div><div class='add'>+		MLX5_SET(qpc, qpc, dp_ordering_force, 1);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	mlx5_cur = to_mlx5_state(cur_state);</div><div class='ctx'> 	mlx5_new = to_mlx5_state(new_state);</div><div class='hunk'>@@ -3621,16 +4359,16 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	op = optab[mlx5_cur][mlx5_new];</div><div class='del'>-	optpar = ib_mask_to_mlx5_opt(attr_mask);</div><div class='add'>+	optpar |= ib_mask_to_mlx5_opt(attr_mask);</div><div class='ctx'> 	optpar &amp;= opt_mask[mlx5_cur][mlx5_new][mlx5_st];</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;ibqp.qp_type == IB_QPT_RAW_PACKET ||</div><div class='del'>-	    qp-&gt;flags &amp; MLX5_IB_QP_UNDERLAY) {</div><div class='add'>+	if (qp-&gt;type == IB_QPT_RAW_PACKET ||</div><div class='add'>+	    qp-&gt;flags &amp; IB_QP_CREATE_SOURCE_QPN) {</div><div class='ctx'> 		struct mlx5_modify_raw_qp_param raw_qp_param = {};</div><div class='ctx'> </div><div class='ctx'> 		raw_qp_param.operation = op;</div><div class='ctx'> 		if (cur_state == IB_QPS_RESET &amp;&amp; new_state == IB_QPS_INIT) {</div><div class='del'>-			raw_qp_param.rq_q_ctr_id = mibport-&gt;cnts.set_id;</div><div class='add'>+			raw_qp_param.rq_q_ctr_id = set_id;</div><div class='ctx'> 			raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID;</div><div class='ctx'> 		}</div><div class='ctx'> </div><div class='hunk'>@@ -3667,8 +4405,15 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,</div><div class='ctx'> </div><div class='ctx'> 		err = modify_raw_packet_qp(dev, qp, &amp;raw_qp_param, tx_affinity);</div><div class='ctx'> 	} else {</div><div class='del'>-		err = mlx5_core_qp_modify(dev-&gt;mdev, op, optpar, context,</div><div class='del'>-					  &amp;base-&gt;mqp);</div><div class='add'>+		if (udata) {</div><div class='add'>+			/* For the kernel flows, the resp will stay zero */</div><div class='add'>+			resp-&gt;ece_options =</div><div class='add'>+				MLX5_CAP_GEN(dev-&gt;mdev, ece_support) ?</div><div class='add'>+					ucmd-&gt;ece_options : 0;</div><div class='add'>+			resp-&gt;response_length = sizeof(*resp);</div><div class='add'>+		}</div><div class='add'>+		err = mlx5_core_qp_modify(dev, op, optpar, qpc, &amp;base-&gt;mqp,</div><div class='add'>+					  &amp;resp-&gt;ece_options);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (err)</div><div class='hunk'>@@ -3690,7 +4435,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,</div><div class='ctx'> 	 * entries and reinitialize the QP.</div><div class='ctx'> 	 */</div><div class='ctx'> 	if (new_state == IB_QPS_RESET &amp;&amp;</div><div class='del'>-	    !ibqp-&gt;uobject &amp;&amp; ibqp-&gt;qp_type != IB_QPT_XRC_TGT) {</div><div class='add'>+	    !ibqp-&gt;uobject &amp;&amp; qp-&gt;type != IB_QPT_XRC_TGT) {</div><div class='ctx'> 		mlx5_ib_cq_clean(recv_cq, base-&gt;mqp.qpn,</div><div class='ctx'> 				 ibqp-&gt;srq ? to_msrq(ibqp-&gt;srq) : NULL);</div><div class='ctx'> 		if (send_cq != recv_cq)</div><div class='hunk'>@@ -3703,12 +4448,19 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,</div><div class='ctx'> 		qp-&gt;sq.cur_post = 0;</div><div class='ctx'> 		if (qp-&gt;sq.wqe_cnt)</div><div class='ctx'> 			qp-&gt;sq.cur_edge = get_sq_edge(&amp;qp-&gt;sq, 0);</div><div class='add'>+		qp-&gt;sq.last_poll = 0;</div><div class='ctx'> 		qp-&gt;db.db[MLX5_RCV_DBR] = 0;</div><div class='ctx'> 		qp-&gt;db.db[MLX5_SND_DBR] = 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	if ((new_state == IB_QPS_RTS) &amp;&amp; qp-&gt;counter_pending) {</div><div class='add'>+		err = __mlx5_ib_qp_set_counter(ibqp, ibqp-&gt;counter);</div><div class='add'>+		if (!err)</div><div class='add'>+			qp-&gt;counter_pending = 0;</div><div class='add'>+	}</div><div class='add'>+</div><div class='ctx'> out:</div><div class='del'>-	kfree(context);</div><div class='add'>+	kfree(qpc);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -3766,14 +4518,15 @@ static bool modify_dci_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state new</div><div class='ctx'>  * Other transitions and attributes are illegal</div><div class='ctx'>  */</div><div class='ctx'> static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='del'>-			      int attr_mask, struct ib_udata *udata)</div><div class='add'>+			      int attr_mask, struct mlx5_ib_modify_qp *ucmd,</div><div class='add'>+			      struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_qp *qp = to_mqp(ibqp);</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(ibqp-&gt;device);</div><div class='ctx'> 	enum ib_qp_state cur_state, new_state;</div><div class='del'>-	int err = 0;</div><div class='ctx'> 	int required = IB_QP_STATE;</div><div class='ctx'> 	void *dctc;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='ctx'> 	if (!(attr_mask &amp; IB_QP_STATE))</div><div class='ctx'> 		return -EINVAL;</div><div class='hunk'>@@ -3782,13 +4535,24 @@ static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> 	new_state = attr-&gt;qp_state;</div><div class='ctx'> </div><div class='ctx'> 	dctc = MLX5_ADDR_OF(create_dct_in, qp-&gt;dct.in, dct_context_entry);</div><div class='add'>+	if (MLX5_CAP_GEN(dev-&gt;mdev, ece_support) &amp;&amp; ucmd-&gt;ece_options)</div><div class='add'>+		/*</div><div class='add'>+		 * DCT doesn't initialize QP till modify command is executed,</div><div class='add'>+		 * so we need to overwrite previously set ECE field if user</div><div class='add'>+		 * provided any value except zero, which means not set/not</div><div class='add'>+		 * valid.</div><div class='add'>+		 */</div><div class='add'>+		MLX5_SET(dctc, dctc, ece, ucmd-&gt;ece_options);</div><div class='add'>+</div><div class='ctx'> 	if (cur_state == IB_QPS_RESET &amp;&amp; new_state == IB_QPS_INIT) {</div><div class='add'>+		u16 set_id;</div><div class='add'>+</div><div class='ctx'> 		required |= IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX | IB_QP_PORT;</div><div class='ctx'> 		if (!is_valid_mask(attr_mask, required, 0))</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 		if (attr-&gt;port_num == 0 ||</div><div class='del'>-		    attr-&gt;port_num &gt; MLX5_CAP_GEN(dev-&gt;mdev, num_ports)) {</div><div class='add'>+		    attr-&gt;port_num &gt; dev-&gt;num_ports) {</div><div class='ctx'> 			mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",</div><div class='ctx'> 				    attr-&gt;port_num, dev-&gt;num_ports);</div><div class='ctx'> 			return -EINVAL;</div><div class='hunk'>@@ -3800,7 +4564,7 @@ static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> 		if (attr-&gt;qp_access_flags &amp; IB_ACCESS_REMOTE_ATOMIC) {</div><div class='ctx'> 			int atomic_mode;</div><div class='ctx'> </div><div class='del'>-			atomic_mode = get_atomic_mode(dev, MLX5_IB_QPT_DCT);</div><div class='add'>+			atomic_mode = get_atomic_mode(dev, qp);</div><div class='ctx'> 			if (atomic_mode &lt; 0)</div><div class='ctx'> 				return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='hunk'>@@ -3808,18 +4572,30 @@ static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> 			MLX5_SET(dctc, dctc, rae, 1);</div><div class='ctx'> 		}</div><div class='ctx'> 		MLX5_SET(dctc, dctc, pkey_index, attr-&gt;pkey_index);</div><div class='del'>-		MLX5_SET(dctc, dctc, port, attr-&gt;port_num);</div><div class='del'>-		MLX5_SET(dctc, dctc, counter_set_id, dev-&gt;port[attr-&gt;port_num - 1].cnts.set_id);</div><div class='add'>+		if (mlx5_lag_is_active(dev-&gt;mdev))</div><div class='add'>+			MLX5_SET(dctc, dctc, port,</div><div class='add'>+				 get_tx_affinity_rr(dev, udata));</div><div class='add'>+		else</div><div class='add'>+			MLX5_SET(dctc, dctc, port, attr-&gt;port_num);</div><div class='ctx'> </div><div class='add'>+		set_id = mlx5_ib_get_counters_id(dev, attr-&gt;port_num - 1);</div><div class='add'>+		MLX5_SET(dctc, dctc, counter_set_id, set_id);</div><div class='add'>+</div><div class='add'>+		qp-&gt;port = attr-&gt;port_num;</div><div class='ctx'> 	} else if (cur_state == IB_QPS_INIT &amp;&amp; new_state == IB_QPS_RTR) {</div><div class='ctx'> 		struct mlx5_ib_modify_qp_resp resp = {};</div><div class='del'>-		u32 out[MLX5_ST_SZ_DW(create_dct_out)] = {0};</div><div class='del'>-		u32 min_resp_len = offsetof(typeof(resp), dctn) +</div><div class='del'>-				   sizeof(resp.dctn);</div><div class='add'>+		u32 out[MLX5_ST_SZ_DW(create_dct_out)] = {};</div><div class='add'>+		u32 min_resp_len = offsetofend(typeof(resp), dctn);</div><div class='ctx'> </div><div class='ctx'> 		if (udata-&gt;outlen &lt; min_resp_len)</div><div class='ctx'> 			return -EINVAL;</div><div class='del'>-		resp.response_length = min_resp_len;</div><div class='add'>+		/*</div><div class='add'>+		 * If we don't have enough space for the ECE options,</div><div class='add'>+		 * simply indicate it with resp.response_length.</div><div class='add'>+		 */</div><div class='add'>+		resp.response_length = (udata-&gt;outlen &lt; sizeof(resp)) ?</div><div class='add'>+					       min_resp_len :</div><div class='add'>+					       sizeof(resp);</div><div class='ctx'> </div><div class='ctx'> 		required |= IB_QP_MIN_RNR_TIMER | IB_QP_AV | IB_QP_PATH_MTU;</div><div class='ctx'> 		if (!is_valid_mask(attr_mask, required, 0))</div><div class='hunk'>@@ -3830,48 +4606,104 @@ static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> 		MLX5_SET(dctc, dctc, mtu, attr-&gt;path_mtu);</div><div class='ctx'> 		MLX5_SET(dctc, dctc, my_addr_index, attr-&gt;ah_attr.grh.sgid_index);</div><div class='ctx'> 		MLX5_SET(dctc, dctc, hop_limit, attr-&gt;ah_attr.grh.hop_limit);</div><div class='add'>+		if (attr-&gt;ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)</div><div class='add'>+			MLX5_SET(dctc, dctc, eth_prio, attr-&gt;ah_attr.sl &amp; 0x7);</div><div class='add'>+		if (qp-&gt;is_ooo_rq) {</div><div class='add'>+			MLX5_SET(dctc, dctc, dp_ordering_1, 1);</div><div class='add'>+			MLX5_SET(dctc, dctc, dp_ordering_force, 1);</div><div class='add'>+		}</div><div class='ctx'> </div><div class='del'>-		err = mlx5_core_create_dct(dev-&gt;mdev, &amp;qp-&gt;dct.mdct, qp-&gt;dct.in,</div><div class='add'>+		err = mlx5_core_create_dct(dev, &amp;qp-&gt;dct.mdct, qp-&gt;dct.in,</div><div class='ctx'> 					   MLX5_ST_SZ_BYTES(create_dct_in), out,</div><div class='ctx'> 					   sizeof(out));</div><div class='add'>+		err = mlx5_cmd_check(dev-&gt;mdev, err, qp-&gt;dct.in, out);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			return err;</div><div class='ctx'> 		resp.dctn = qp-&gt;dct.mdct.mqp.qpn;</div><div class='add'>+		if (MLX5_CAP_GEN(dev-&gt;mdev, ece_support))</div><div class='add'>+			resp.ece_options = MLX5_GET(create_dct_out, out, ece);</div><div class='ctx'> 		err = ib_copy_to_udata(udata, &amp;resp, resp.response_length);</div><div class='ctx'> 		if (err) {</div><div class='del'>-			mlx5_core_destroy_dct(dev-&gt;mdev, &amp;qp-&gt;dct.mdct);</div><div class='add'>+			mlx5_core_destroy_dct(dev, &amp;qp-&gt;dct.mdct);</div><div class='ctx'> 			return err;</div><div class='ctx'> 		}</div><div class='ctx'> 	} else {</div><div class='ctx'> 		mlx5_ib_warn(dev, "Modify DCT: Invalid transition from %d to %d\n", cur_state, new_state);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='del'>-	if (err)</div><div class='del'>-		qp-&gt;state = IB_QPS_ERR;</div><div class='del'>-	else</div><div class='del'>-		qp-&gt;state = new_state;</div><div class='del'>-	return err;</div><div class='add'>+</div><div class='add'>+	qp-&gt;state = new_state;</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool mlx5_ib_modify_qp_allowed(struct mlx5_ib_dev *dev,</div><div class='add'>+				      struct mlx5_ib_qp *qp)</div><div class='add'>+{</div><div class='add'>+	if (dev-&gt;profile != &amp;raw_eth_profile)</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;type == IB_QPT_RAW_PACKET || qp-&gt;type == MLX5_IB_QPT_REG_UMR)</div><div class='add'>+		return true;</div><div class='add'>+</div><div class='add'>+	return false;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int validate_rd_atomic(struct mlx5_ib_dev *dev, struct ib_qp_attr *attr,</div><div class='add'>+			      int attr_mask, enum ib_qp_type qp_type)</div><div class='add'>+{</div><div class='add'>+	int log_max_ra_res;</div><div class='add'>+	int log_max_ra_req;</div><div class='add'>+</div><div class='add'>+	if (qp_type == MLX5_IB_QPT_DCI) {</div><div class='add'>+		log_max_ra_res = 1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev,</div><div class='add'>+						   log_max_ra_res_dc);</div><div class='add'>+		log_max_ra_req = 1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev,</div><div class='add'>+						   log_max_ra_req_dc);</div><div class='add'>+	} else {</div><div class='add'>+		log_max_ra_res = 1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev,</div><div class='add'>+						   log_max_ra_res_qp);</div><div class='add'>+		log_max_ra_req = 1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev,</div><div class='add'>+						   log_max_ra_req_qp);</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_MAX_QP_RD_ATOMIC &amp;&amp;</div><div class='add'>+	    attr-&gt;max_rd_atomic &gt; log_max_ra_res) {</div><div class='add'>+		mlx5_ib_dbg(dev, "invalid max_rd_atomic value %d\n",</div><div class='add'>+			    attr-&gt;max_rd_atomic);</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; IB_QP_MAX_DEST_RD_ATOMIC &amp;&amp;</div><div class='add'>+	    attr-&gt;max_dest_rd_atomic &gt; log_max_ra_req) {</div><div class='add'>+		mlx5_ib_dbg(dev, "invalid max_dest_rd_atomic value %d\n",</div><div class='add'>+			    attr-&gt;max_dest_rd_atomic);</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='add'>+	return true;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> 		      int attr_mask, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(ibqp-&gt;device);</div><div class='add'>+	struct mlx5_ib_modify_qp_resp resp = {};</div><div class='ctx'> 	struct mlx5_ib_qp *qp = to_mqp(ibqp);</div><div class='ctx'> 	struct mlx5_ib_modify_qp ucmd = {};</div><div class='ctx'> 	enum ib_qp_type qp_type;</div><div class='ctx'> 	enum ib_qp_state cur_state, new_state;</div><div class='del'>-	size_t required_cmd_sz;</div><div class='ctx'> 	int err = -EINVAL;</div><div class='del'>-	int port;</div><div class='add'>+</div><div class='add'>+	if (!mlx5_ib_modify_qp_allowed(dev, qp))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	if (attr_mask &amp; ~(IB_QP_ATTR_STANDARD_BITS | IB_QP_RATE_LIMIT))</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	if (ibqp-&gt;rwq_ind_tbl)</div><div class='ctx'> 		return -ENOSYS;</div><div class='ctx'> </div><div class='ctx'> 	if (udata &amp;&amp; udata-&gt;inlen) {</div><div class='del'>-		required_cmd_sz = offsetof(typeof(ucmd), reserved) +</div><div class='del'>-			sizeof(ucmd.reserved);</div><div class='del'>-		if (udata-&gt;inlen &lt; required_cmd_sz)</div><div class='add'>+		if (udata-&gt;inlen &lt; offsetofend(typeof(ucmd), ece_options))</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 		if (udata-&gt;inlen &gt; sizeof(ucmd) &amp;&amp;</div><div class='hunk'>@@ -3883,35 +4715,32 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> 				       min(udata-&gt;inlen, sizeof(ucmd))))</div><div class='ctx'> 			return -EFAULT;</div><div class='ctx'> </div><div class='del'>-		if (ucmd.comp_mask ||</div><div class='del'>-		    memchr_inv(&amp;ucmd.reserved, 0, sizeof(ucmd.reserved)) ||</div><div class='add'>+		if (ucmd.comp_mask &amp; ~MLX5_IB_MODIFY_QP_OOO_DP ||</div><div class='ctx'> 		    memchr_inv(&amp;ucmd.burst_info.reserved, 0,</div><div class='ctx'> 			       sizeof(ucmd.burst_info.reserved)))</div><div class='ctx'> 			return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+		if (ucmd.comp_mask &amp; MLX5_IB_MODIFY_QP_OOO_DP) {</div><div class='add'>+			if (!get_dp_ooo_cap(dev-&gt;mdev, qp-&gt;type))</div><div class='add'>+				return -EOPNOTSUPP;</div><div class='add'>+			qp-&gt;is_ooo_rq = 1;</div><div class='add'>+		}</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (unlikely(ibqp-&gt;qp_type == IB_QPT_GSI))</div><div class='add'>+	if (qp-&gt;type == IB_QPT_GSI)</div><div class='ctx'> 		return mlx5_ib_gsi_modify_qp(ibqp, attr, attr_mask);</div><div class='ctx'> </div><div class='del'>-	if (ibqp-&gt;qp_type == IB_QPT_DRIVER)</div><div class='del'>-		qp_type = qp-&gt;qp_sub_type;</div><div class='del'>-	else</div><div class='del'>-		qp_type = (unlikely(ibqp-&gt;qp_type == MLX5_IB_QPT_HW_GSI)) ?</div><div class='del'>-			IB_QPT_GSI : ibqp-&gt;qp_type;</div><div class='add'>+	qp_type = (qp-&gt;type == MLX5_IB_QPT_HW_GSI) ? IB_QPT_GSI : qp-&gt;type;</div><div class='ctx'> </div><div class='ctx'> 	if (qp_type == MLX5_IB_QPT_DCT)</div><div class='del'>-		return mlx5_ib_modify_dct(ibqp, attr, attr_mask, udata);</div><div class='add'>+		return mlx5_ib_modify_dct(ibqp, attr, attr_mask, &amp;ucmd, udata);</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;qp-&gt;mutex);</div><div class='ctx'> </div><div class='ctx'> 	cur_state = attr_mask &amp; IB_QP_CUR_STATE ? attr-&gt;cur_qp_state : qp-&gt;state;</div><div class='ctx'> 	new_state = attr_mask &amp; IB_QP_STATE ? attr-&gt;qp_state : cur_state;</div><div class='ctx'> </div><div class='del'>-	if (!(cur_state == new_state &amp;&amp; cur_state == IB_QPS_RESET)) {</div><div class='del'>-		port = attr_mask &amp; IB_QP_PORT ? attr-&gt;port_num : qp-&gt;port;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (qp-&gt;flags &amp; MLX5_IB_QP_UNDERLAY) {</div><div class='add'>+	if (qp-&gt;flags &amp; IB_QP_CREATE_SOURCE_QPN) {</div><div class='ctx'> 		if (attr_mask &amp; ~(IB_QP_STATE | IB_QP_CUR_STATE)) {</div><div class='ctx'> 			mlx5_ib_dbg(dev, "invalid attr_mask 0x%x when underlay QP is used\n",</div><div class='ctx'> 				    attr_mask);</div><div class='hunk'>@@ -3922,7 +4751,7 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> 		   !ib_modify_qp_is_ok(cur_state, new_state, qp_type,</div><div class='ctx'> 				       attr_mask)) {</div><div class='ctx'> 		mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",</div><div class='del'>-			    cur_state, new_state, ibqp-&gt;qp_type, attr_mask);</div><div class='add'>+			    cur_state, new_state, qp-&gt;type, attr_mask);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	} else if (qp_type == MLX5_IB_QPT_DCI &amp;&amp;</div><div class='ctx'> 		   !modify_dci_qp_is_ok(cur_state, new_state, attr_mask)) {</div><div class='hunk'>@@ -3939,31 +4768,14 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (attr_mask &amp; IB_QP_PKEY_INDEX) {</div><div class='del'>-		port = attr_mask &amp; IB_QP_PORT ? attr-&gt;port_num : qp-&gt;port;</div><div class='del'>-		if (attr-&gt;pkey_index &gt;=</div><div class='del'>-		    dev-&gt;mdev-&gt;port_caps[port - 1].pkey_table_len) {</div><div class='del'>-			mlx5_ib_dbg(dev, "invalid pkey index %d\n",</div><div class='del'>-				    attr-&gt;pkey_index);</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (attr_mask &amp; IB_QP_MAX_QP_RD_ATOMIC &amp;&amp;</div><div class='del'>-	    attr-&gt;max_rd_atomic &gt;</div><div class='del'>-	    (1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev, log_max_ra_res_qp))) {</div><div class='del'>-		mlx5_ib_dbg(dev, "invalid max_rd_atomic value %d\n",</div><div class='del'>-			    attr-&gt;max_rd_atomic);</div><div class='add'>+	if ((attr_mask &amp; IB_QP_PKEY_INDEX) &amp;&amp;</div><div class='add'>+	    attr-&gt;pkey_index &gt;= dev-&gt;pkey_table_len) {</div><div class='add'>+		mlx5_ib_dbg(dev, "invalid pkey index %d\n", attr-&gt;pkey_index);</div><div class='ctx'> 		goto out;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	if (attr_mask &amp; IB_QP_MAX_DEST_RD_ATOMIC &amp;&amp;</div><div class='del'>-	    attr-&gt;max_dest_rd_atomic &gt;</div><div class='del'>-	    (1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev, log_max_ra_req_qp))) {</div><div class='del'>-		mlx5_ib_dbg(dev, "invalid max_dest_rd_atomic value %d\n",</div><div class='del'>-			    attr-&gt;max_dest_rd_atomic);</div><div class='add'>+	if (!validate_rd_atomic(dev, attr, attr_mask, qp_type))</div><div class='ctx'> 		goto out;</div><div class='del'>-	}</div><div class='ctx'> </div><div class='ctx'> 	if (cur_state == new_state &amp;&amp; cur_state == IB_QPS_RESET) {</div><div class='ctx'> 		err = 0;</div><div class='hunk'>@@ -3971,1346 +4783,19 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	err = __mlx5_ib_modify_qp(ibqp, attr, attr_mask, cur_state,</div><div class='del'>-				  new_state, &amp;ucmd, udata);</div><div class='del'>-</div><div class='del'>-out:</div><div class='del'>-	mutex_unlock(&amp;qp-&gt;mutex);</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void _handle_post_send_edge(struct mlx5_ib_wq *sq, void **seg,</div><div class='del'>-				   u32 wqe_sz, void **cur_edge)</div><div class='del'>-{</div><div class='del'>-	u32 idx;</div><div class='del'>-</div><div class='del'>-	idx = (sq-&gt;cur_post + (wqe_sz &gt;&gt; 2)) &amp; (sq-&gt;wqe_cnt - 1);</div><div class='del'>-	*cur_edge = get_sq_edge(sq, idx);</div><div class='add'>+				  new_state, &amp;ucmd, &amp;resp, udata);</div><div class='ctx'> </div><div class='del'>-	*seg = mlx5_frag_buf_get_wqe(&amp;sq-&gt;fbc, idx);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* handle_post_send_edge - Check if we get to SQ edge. If yes, update to the</div><div class='del'>- * next nearby edge and get new address translation for current WQE position.</div><div class='del'>- * @sq - SQ buffer.</div><div class='del'>- * @seg: Current WQE position (16B aligned).</div><div class='del'>- * @wqe_sz: Total current WQE size [16B].</div><div class='del'>- * @cur_edge: Updated current edge.</div><div class='del'>- */</div><div class='del'>-static inline void handle_post_send_edge(struct mlx5_ib_wq *sq, void **seg,</div><div class='del'>-					 u32 wqe_sz, void **cur_edge)</div><div class='del'>-{</div><div class='del'>-	if (likely(*seg != *cur_edge))</div><div class='del'>-		return;</div><div class='del'>-</div><div class='del'>-	_handle_post_send_edge(sq, seg, wqe_sz, cur_edge);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-/* memcpy_send_wqe - copy data from src to WQE and update the relevant WQ's</div><div class='del'>- * pointers. At the end @seg is aligned to 16B regardless the copied size.</div><div class='del'>- * @sq - SQ buffer.</div><div class='del'>- * @cur_edge: Updated current edge.</div><div class='del'>- * @seg: Current WQE position (16B aligned).</div><div class='del'>- * @wqe_sz: Total current WQE size [16B].</div><div class='del'>- * @src: Pointer to copy from.</div><div class='del'>- * @n: Number of bytes to copy.</div><div class='del'>- */</div><div class='del'>-static inline void memcpy_send_wqe(struct mlx5_ib_wq *sq, void **cur_edge,</div><div class='del'>-				   void **seg, u32 *wqe_sz, const void *src,</div><div class='del'>-				   size_t n)</div><div class='del'>-{</div><div class='del'>-	while (likely(n)) {</div><div class='del'>-		size_t leftlen = *cur_edge - *seg;</div><div class='del'>-		size_t copysz = min_t(size_t, leftlen, n);</div><div class='del'>-		size_t stride;</div><div class='del'>-</div><div class='del'>-		memcpy(*seg, src, copysz);</div><div class='del'>-</div><div class='del'>-		n -= copysz;</div><div class='del'>-		src += copysz;</div><div class='del'>-		stride = !n ? ALIGN(copysz, 16) : copysz;</div><div class='del'>-		*seg += stride;</div><div class='del'>-		*wqe_sz += stride &gt;&gt; 4;</div><div class='del'>-		handle_post_send_edge(sq, seg, *wqe_sz, cur_edge);</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx5_wq_overflow(struct mlx5_ib_wq *wq, int nreq, struct ib_cq *ib_cq)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_cq *cq;</div><div class='del'>-	unsigned cur;</div><div class='del'>-</div><div class='del'>-	cur = wq-&gt;head - wq-&gt;tail;</div><div class='del'>-	if (likely(cur + nreq &lt; wq-&gt;max_post))</div><div class='del'>-		return 0;</div><div class='del'>-</div><div class='del'>-	cq = to_mcq(ib_cq);</div><div class='del'>-	spin_lock(&amp;cq-&gt;lock);</div><div class='del'>-	cur = wq-&gt;head - wq-&gt;tail;</div><div class='del'>-	spin_unlock(&amp;cq-&gt;lock);</div><div class='del'>-</div><div class='del'>-	return cur + nreq &gt;= wq-&gt;max_post;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg,</div><div class='del'>-					  u64 remote_addr, u32 rkey)</div><div class='del'>-{</div><div class='del'>-	rseg-&gt;raddr    = cpu_to_be64(remote_addr);</div><div class='del'>-	rseg-&gt;rkey     = cpu_to_be32(rkey);</div><div class='del'>-	rseg-&gt;reserved = 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_eth_seg(const struct ib_send_wr *wr, struct mlx5_ib_qp *qp,</div><div class='del'>-			void **seg, int *size, void **cur_edge)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_wqe_eth_seg *eseg = *seg;</div><div class='del'>-</div><div class='del'>-	memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg));</div><div class='del'>-</div><div class='del'>-	if (wr-&gt;send_flags &amp; IB_SEND_IP_CSUM)</div><div class='del'>-		eseg-&gt;cs_flags = MLX5_ETH_WQE_L3_CSUM |</div><div class='del'>-				 MLX5_ETH_WQE_L4_CSUM;</div><div class='del'>-</div><div class='del'>-	if (wr-&gt;opcode == IB_WR_LSO) {</div><div class='del'>-		struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr);</div><div class='del'>-		size_t left, copysz;</div><div class='del'>-		void *pdata = ud_wr-&gt;header;</div><div class='del'>-		size_t stride;</div><div class='del'>-</div><div class='del'>-		left = ud_wr-&gt;hlen;</div><div class='del'>-		eseg-&gt;mss = cpu_to_be16(ud_wr-&gt;mss);</div><div class='del'>-		eseg-&gt;inline_hdr.sz = cpu_to_be16(left);</div><div class='del'>-</div><div class='del'>-		/* memcpy_send_wqe should get a 16B align address. Hence, we</div><div class='del'>-		 * first copy up to the current edge and then, if needed,</div><div class='del'>-		 * fall-through to memcpy_send_wqe.</div><div class='del'>-		 */</div><div class='del'>-		copysz = min_t(u64, *cur_edge - (void *)eseg-&gt;inline_hdr.start,</div><div class='del'>-			       left);</div><div class='del'>-		memcpy(eseg-&gt;inline_hdr.start, pdata, copysz);</div><div class='del'>-		stride = ALIGN(sizeof(struct mlx5_wqe_eth_seg) -</div><div class='del'>-			       sizeof(eseg-&gt;inline_hdr.start) + copysz, 16);</div><div class='del'>-		*size += stride / 16;</div><div class='del'>-		*seg += stride;</div><div class='del'>-</div><div class='del'>-		if (copysz &lt; left) {</div><div class='del'>-			handle_post_send_edge(&amp;qp-&gt;sq, seg, *size, cur_edge);</div><div class='del'>-			left -= copysz;</div><div class='del'>-			pdata += copysz;</div><div class='del'>-			memcpy_send_wqe(&amp;qp-&gt;sq, cur_edge, seg, size, pdata,</div><div class='del'>-					left);</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		return;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	*seg += sizeof(struct mlx5_wqe_eth_seg);</div><div class='del'>-	*size += sizeof(struct mlx5_wqe_eth_seg) / 16;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,</div><div class='del'>-			     const struct ib_send_wr *wr)</div><div class='del'>-{</div><div class='del'>-	memcpy(&amp;dseg-&gt;av, &amp;to_mah(ud_wr(wr)-&gt;ah)-&gt;av, sizeof(struct mlx5_av));</div><div class='del'>-	dseg-&gt;av.dqp_dct = cpu_to_be32(ud_wr(wr)-&gt;remote_qpn | MLX5_EXTENDED_UD_AV);</div><div class='del'>-	dseg-&gt;av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)-&gt;remote_qkey);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg)</div><div class='del'>-{</div><div class='del'>-	dseg-&gt;byte_count = cpu_to_be32(sg-&gt;length);</div><div class='del'>-	dseg-&gt;lkey       = cpu_to_be32(sg-&gt;lkey);</div><div class='del'>-	dseg-&gt;addr       = cpu_to_be64(sg-&gt;addr);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static u64 get_xlt_octo(u64 bytes)</div><div class='del'>-{</div><div class='del'>-	return ALIGN(bytes, MLX5_IB_UMR_XLT_ALIGNMENT) /</div><div class='del'>-	       MLX5_IB_UMR_OCTOWORD;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static __be64 frwr_mkey_mask(void)</div><div class='del'>-{</div><div class='del'>-	u64 result;</div><div class='del'>-</div><div class='del'>-	result = MLX5_MKEY_MASK_LEN		|</div><div class='del'>-		MLX5_MKEY_MASK_PAGE_SIZE	|</div><div class='del'>-		MLX5_MKEY_MASK_START_ADDR	|</div><div class='del'>-		MLX5_MKEY_MASK_EN_RINVAL	|</div><div class='del'>-		MLX5_MKEY_MASK_KEY		|</div><div class='del'>-		MLX5_MKEY_MASK_LR		|</div><div class='del'>-		MLX5_MKEY_MASK_LW		|</div><div class='del'>-		MLX5_MKEY_MASK_RR		|</div><div class='del'>-		MLX5_MKEY_MASK_RW		|</div><div class='del'>-		MLX5_MKEY_MASK_A		|</div><div class='del'>-		MLX5_MKEY_MASK_SMALL_FENCE	|</div><div class='del'>-		MLX5_MKEY_MASK_FREE;</div><div class='del'>-</div><div class='del'>-	return cpu_to_be64(result);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static __be64 sig_mkey_mask(void)</div><div class='del'>-{</div><div class='del'>-	u64 result;</div><div class='del'>-</div><div class='del'>-	result = MLX5_MKEY_MASK_LEN		|</div><div class='del'>-		MLX5_MKEY_MASK_PAGE_SIZE	|</div><div class='del'>-		MLX5_MKEY_MASK_START_ADDR	|</div><div class='del'>-		MLX5_MKEY_MASK_EN_SIGERR	|</div><div class='del'>-		MLX5_MKEY_MASK_EN_RINVAL	|</div><div class='del'>-		MLX5_MKEY_MASK_KEY		|</div><div class='del'>-		MLX5_MKEY_MASK_LR		|</div><div class='del'>-		MLX5_MKEY_MASK_LW		|</div><div class='del'>-		MLX5_MKEY_MASK_RR		|</div><div class='del'>-		MLX5_MKEY_MASK_RW		|</div><div class='del'>-		MLX5_MKEY_MASK_SMALL_FENCE	|</div><div class='del'>-		MLX5_MKEY_MASK_FREE		|</div><div class='del'>-		MLX5_MKEY_MASK_BSF_EN;</div><div class='del'>-</div><div class='del'>-	return cpu_to_be64(result);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,</div><div class='del'>-			    struct mlx5_ib_mr *mr, bool umr_inline)</div><div class='del'>-{</div><div class='del'>-	int size = mr-&gt;ndescs * mr-&gt;desc_size;</div><div class='del'>-</div><div class='del'>-	memset(umr, 0, sizeof(*umr));</div><div class='del'>-</div><div class='del'>-	umr-&gt;flags = MLX5_UMR_CHECK_NOT_FREE;</div><div class='del'>-	if (umr_inline)</div><div class='del'>-		umr-&gt;flags |= MLX5_UMR_INLINE;</div><div class='del'>-	umr-&gt;xlt_octowords = cpu_to_be16(get_xlt_octo(size));</div><div class='del'>-	umr-&gt;mkey_mask = frwr_mkey_mask();</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr)</div><div class='del'>-{</div><div class='del'>-	memset(umr, 0, sizeof(*umr));</div><div class='del'>-	umr-&gt;mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);</div><div class='del'>-	umr-&gt;flags = MLX5_UMR_INLINE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static __be64 get_umr_enable_mr_mask(void)</div><div class='del'>-{</div><div class='del'>-	u64 result;</div><div class='del'>-</div><div class='del'>-	result = MLX5_MKEY_MASK_KEY |</div><div class='del'>-		 MLX5_MKEY_MASK_FREE;</div><div class='del'>-</div><div class='del'>-	return cpu_to_be64(result);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static __be64 get_umr_disable_mr_mask(void)</div><div class='del'>-{</div><div class='del'>-	u64 result;</div><div class='del'>-</div><div class='del'>-	result = MLX5_MKEY_MASK_FREE;</div><div class='del'>-</div><div class='del'>-	return cpu_to_be64(result);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static __be64 get_umr_update_translation_mask(void)</div><div class='del'>-{</div><div class='del'>-	u64 result;</div><div class='del'>-</div><div class='del'>-	result = MLX5_MKEY_MASK_LEN |</div><div class='del'>-		 MLX5_MKEY_MASK_PAGE_SIZE |</div><div class='del'>-		 MLX5_MKEY_MASK_START_ADDR;</div><div class='del'>-</div><div class='del'>-	return cpu_to_be64(result);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static __be64 get_umr_update_access_mask(int atomic)</div><div class='del'>-{</div><div class='del'>-	u64 result;</div><div class='del'>-</div><div class='del'>-	result = MLX5_MKEY_MASK_LR |</div><div class='del'>-		 MLX5_MKEY_MASK_LW |</div><div class='del'>-		 MLX5_MKEY_MASK_RR |</div><div class='del'>-		 MLX5_MKEY_MASK_RW;</div><div class='del'>-</div><div class='del'>-	if (atomic)</div><div class='del'>-		result |= MLX5_MKEY_MASK_A;</div><div class='del'>-</div><div class='del'>-	return cpu_to_be64(result);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static __be64 get_umr_update_pd_mask(void)</div><div class='del'>-{</div><div class='del'>-	u64 result;</div><div class='del'>-</div><div class='del'>-	result = MLX5_MKEY_MASK_PD;</div><div class='del'>-</div><div class='del'>-	return cpu_to_be64(result);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask)</div><div class='del'>-{</div><div class='del'>-	if ((mask &amp; MLX5_MKEY_MASK_PAGE_SIZE &amp;&amp;</div><div class='del'>-	     MLX5_CAP_GEN(dev-&gt;mdev, umr_modify_entity_size_disabled)) ||</div><div class='del'>-	    (mask &amp; MLX5_MKEY_MASK_A &amp;&amp;</div><div class='del'>-	     MLX5_CAP_GEN(dev-&gt;mdev, umr_modify_atomic_disabled)))</div><div class='del'>-		return -EPERM;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int set_reg_umr_segment(struct mlx5_ib_dev *dev,</div><div class='del'>-			       struct mlx5_wqe_umr_ctrl_seg *umr,</div><div class='del'>-			       const struct ib_send_wr *wr, int atomic)</div><div class='del'>-{</div><div class='del'>-	const struct mlx5_umr_wr *umrwr = umr_wr(wr);</div><div class='del'>-</div><div class='del'>-	memset(umr, 0, sizeof(*umr));</div><div class='del'>-</div><div class='del'>-	if (wr-&gt;send_flags &amp; MLX5_IB_SEND_UMR_FAIL_IF_FREE)</div><div class='del'>-		umr-&gt;flags = MLX5_UMR_CHECK_FREE; /* fail if free */</div><div class='del'>-	else</div><div class='del'>-		umr-&gt;flags = MLX5_UMR_CHECK_NOT_FREE; /* fail if not free */</div><div class='del'>-</div><div class='del'>-	umr-&gt;xlt_octowords = cpu_to_be16(get_xlt_octo(umrwr-&gt;xlt_size));</div><div class='del'>-	if (wr-&gt;send_flags &amp; MLX5_IB_SEND_UMR_UPDATE_XLT) {</div><div class='del'>-		u64 offset = get_xlt_octo(umrwr-&gt;offset);</div><div class='del'>-</div><div class='del'>-		umr-&gt;xlt_offset = cpu_to_be16(offset &amp; 0xffff);</div><div class='del'>-		umr-&gt;xlt_offset_47_16 = cpu_to_be32(offset &gt;&gt; 16);</div><div class='del'>-		umr-&gt;flags |= MLX5_UMR_TRANSLATION_OFFSET_EN;</div><div class='del'>-	}</div><div class='del'>-	if (wr-&gt;send_flags &amp; MLX5_IB_SEND_UMR_UPDATE_TRANSLATION)</div><div class='del'>-		umr-&gt;mkey_mask |= get_umr_update_translation_mask();</div><div class='del'>-	if (wr-&gt;send_flags &amp; MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) {</div><div class='del'>-		umr-&gt;mkey_mask |= get_umr_update_access_mask(atomic);</div><div class='del'>-		umr-&gt;mkey_mask |= get_umr_update_pd_mask();</div><div class='del'>-	}</div><div class='del'>-	if (wr-&gt;send_flags &amp; MLX5_IB_SEND_UMR_ENABLE_MR)</div><div class='del'>-		umr-&gt;mkey_mask |= get_umr_enable_mr_mask();</div><div class='del'>-	if (wr-&gt;send_flags &amp; MLX5_IB_SEND_UMR_DISABLE_MR)</div><div class='del'>-		umr-&gt;mkey_mask |= get_umr_disable_mr_mask();</div><div class='del'>-</div><div class='del'>-	if (!wr-&gt;num_sge)</div><div class='del'>-		umr-&gt;flags |= MLX5_UMR_INLINE;</div><div class='del'>-</div><div class='del'>-	return umr_check_mkey_mask(dev, be64_to_cpu(umr-&gt;mkey_mask));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static u8 get_umr_flags(int acc)</div><div class='del'>-{</div><div class='del'>-	return (acc &amp; IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC       : 0) |</div><div class='del'>-	       (acc &amp; IB_ACCESS_REMOTE_WRITE  ? MLX5_PERM_REMOTE_WRITE : 0) |</div><div class='del'>-	       (acc &amp; IB_ACCESS_REMOTE_READ   ? MLX5_PERM_REMOTE_READ  : 0) |</div><div class='del'>-	       (acc &amp; IB_ACCESS_LOCAL_WRITE   ? MLX5_PERM_LOCAL_WRITE  : 0) |</div><div class='del'>-		MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,</div><div class='del'>-			     struct mlx5_ib_mr *mr,</div><div class='del'>-			     u32 key, int access)</div><div class='del'>-{</div><div class='del'>-	int ndescs = ALIGN(mr-&gt;ndescs, 8) &gt;&gt; 1;</div><div class='del'>-</div><div class='del'>-	memset(seg, 0, sizeof(*seg));</div><div class='del'>-</div><div class='del'>-	if (mr-&gt;access_mode == MLX5_MKC_ACCESS_MODE_MTT)</div><div class='del'>-		seg-&gt;log2_page_size = ilog2(mr-&gt;ibmr.page_size);</div><div class='del'>-	else if (mr-&gt;access_mode == MLX5_MKC_ACCESS_MODE_KLMS)</div><div class='del'>-		/* KLMs take twice the size of MTTs */</div><div class='del'>-		ndescs *= 2;</div><div class='del'>-</div><div class='del'>-	seg-&gt;flags = get_umr_flags(access) | mr-&gt;access_mode;</div><div class='del'>-	seg-&gt;qpn_mkey7_0 = cpu_to_be32((key &amp; 0xff) | 0xffffff00);</div><div class='del'>-	seg-&gt;flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);</div><div class='del'>-	seg-&gt;start_addr = cpu_to_be64(mr-&gt;ibmr.iova);</div><div class='del'>-	seg-&gt;len = cpu_to_be64(mr-&gt;ibmr.length);</div><div class='del'>-	seg-&gt;xlt_oct_size = cpu_to_be32(ndescs);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)</div><div class='del'>-{</div><div class='del'>-	memset(seg, 0, sizeof(*seg));</div><div class='del'>-	seg-&gt;status = MLX5_MKEY_STATUS_FREE;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg,</div><div class='del'>-				 const struct ib_send_wr *wr)</div><div class='del'>-{</div><div class='del'>-	const struct mlx5_umr_wr *umrwr = umr_wr(wr);</div><div class='del'>-</div><div class='del'>-	memset(seg, 0, sizeof(*seg));</div><div class='del'>-	if (wr-&gt;send_flags &amp; MLX5_IB_SEND_UMR_DISABLE_MR)</div><div class='del'>-		seg-&gt;status = MLX5_MKEY_STATUS_FREE;</div><div class='del'>-</div><div class='del'>-	seg-&gt;flags = convert_access(umrwr-&gt;access_flags);</div><div class='del'>-	if (umrwr-&gt;pd)</div><div class='del'>-		seg-&gt;flags_pd = cpu_to_be32(to_mpd(umrwr-&gt;pd)-&gt;pdn);</div><div class='del'>-	if (wr-&gt;send_flags &amp; MLX5_IB_SEND_UMR_UPDATE_TRANSLATION &amp;&amp;</div><div class='del'>-	    !umrwr-&gt;length)</div><div class='del'>-		seg-&gt;flags_pd |= cpu_to_be32(MLX5_MKEY_LEN64);</div><div class='del'>-</div><div class='del'>-	seg-&gt;start_addr = cpu_to_be64(umrwr-&gt;virt_addr);</div><div class='del'>-	seg-&gt;len = cpu_to_be64(umrwr-&gt;length);</div><div class='del'>-	seg-&gt;log2_page_size = umrwr-&gt;page_shift;</div><div class='del'>-	seg-&gt;qpn_mkey7_0 = cpu_to_be32(0xffffff00 |</div><div class='del'>-				       mlx5_mkey_variant(umrwr-&gt;mkey));</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,</div><div class='del'>-			     struct mlx5_ib_mr *mr,</div><div class='del'>-			     struct mlx5_ib_pd *pd)</div><div class='del'>-{</div><div class='del'>-	int bcount = mr-&gt;desc_size * mr-&gt;ndescs;</div><div class='del'>-</div><div class='del'>-	dseg-&gt;addr = cpu_to_be64(mr-&gt;desc_map);</div><div class='del'>-	dseg-&gt;byte_count = cpu_to_be32(ALIGN(bcount, 64));</div><div class='del'>-	dseg-&gt;lkey = cpu_to_be32(pd-&gt;ibpd.local_dma_lkey);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static __be32 send_ieth(const struct ib_send_wr *wr)</div><div class='del'>-{</div><div class='del'>-	switch (wr-&gt;opcode) {</div><div class='del'>-	case IB_WR_SEND_WITH_IMM:</div><div class='del'>-	case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='del'>-		return wr-&gt;ex.imm_data;</div><div class='del'>-</div><div class='del'>-	case IB_WR_SEND_WITH_INV:</div><div class='del'>-		return cpu_to_be32(wr-&gt;ex.invalidate_rkey);</div><div class='del'>-</div><div class='del'>-	default:</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static u8 calc_sig(void *wqe, int size)</div><div class='del'>-{</div><div class='del'>-	u8 *p = wqe;</div><div class='del'>-	u8 res = 0;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; size; i++)</div><div class='del'>-		res ^= p[i];</div><div class='del'>-</div><div class='del'>-	return ~res;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static u8 wq_sig(void *wqe)</div><div class='del'>-{</div><div class='del'>-	return calc_sig(wqe, (*((u8 *)wqe + 8) &amp; 0x3f) &lt;&lt; 4);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int set_data_inl_seg(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,</div><div class='del'>-			    void **wqe, int *wqe_sz, void **cur_edge)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_wqe_inline_seg *seg;</div><div class='del'>-	size_t offset;</div><div class='del'>-	int inl = 0;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	seg = *wqe;</div><div class='del'>-	*wqe += sizeof(*seg);</div><div class='del'>-	offset = sizeof(*seg);</div><div class='del'>-</div><div class='del'>-	for (i = 0; i &lt; wr-&gt;num_sge; i++) {</div><div class='del'>-		size_t len  = wr-&gt;sg_list[i].length;</div><div class='del'>-		void *addr = (void *)(unsigned long)(wr-&gt;sg_list[i].addr);</div><div class='del'>-</div><div class='del'>-		inl += len;</div><div class='del'>-</div><div class='del'>-		if (unlikely(inl &gt; qp-&gt;max_inline_data))</div><div class='del'>-			return -ENOMEM;</div><div class='del'>-</div><div class='del'>-		while (likely(len)) {</div><div class='del'>-			size_t leftlen;</div><div class='del'>-			size_t copysz;</div><div class='del'>-</div><div class='del'>-			handle_post_send_edge(&amp;qp-&gt;sq, wqe,</div><div class='del'>-					      *wqe_sz + (offset &gt;&gt; 4),</div><div class='del'>-					      cur_edge);</div><div class='del'>-</div><div class='del'>-			leftlen = *cur_edge - *wqe;</div><div class='del'>-			copysz = min_t(size_t, leftlen, len);</div><div class='del'>-</div><div class='del'>-			memcpy(*wqe, addr, copysz);</div><div class='del'>-			len -= copysz;</div><div class='del'>-			addr += copysz;</div><div class='del'>-			*wqe += copysz;</div><div class='del'>-			offset += copysz;</div><div class='del'>-		}</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	seg-&gt;byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG);</div><div class='del'>-</div><div class='del'>-	*wqe_sz +=  ALIGN(inl + sizeof(seg-&gt;byte_count), 16) / 16;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static u16 prot_field_size(enum ib_signature_type type)</div><div class='del'>-{</div><div class='del'>-	switch (type) {</div><div class='del'>-	case IB_SIG_TYPE_T10_DIF:</div><div class='del'>-		return MLX5_DIF_SIZE;</div><div class='del'>-	default:</div><div class='del'>-		return 0;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static u8 bs_selector(int block_size)</div><div class='del'>-{</div><div class='del'>-	switch (block_size) {</div><div class='del'>-	case 512:	    return 0x1;</div><div class='del'>-	case 520:	    return 0x2;</div><div class='del'>-	case 4096:	    return 0x3;</div><div class='del'>-	case 4160:	    return 0x4;</div><div class='del'>-	case 1073741824:    return 0x5;</div><div class='del'>-	default:	    return 0;</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain,</div><div class='del'>-			      struct mlx5_bsf_inl *inl)</div><div class='del'>-{</div><div class='del'>-	/* Valid inline section and allow BSF refresh */</div><div class='del'>-	inl-&gt;vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID |</div><div class='del'>-				       MLX5_BSF_REFRESH_DIF);</div><div class='del'>-	inl-&gt;dif_apptag = cpu_to_be16(domain-&gt;sig.dif.app_tag);</div><div class='del'>-	inl-&gt;dif_reftag = cpu_to_be32(domain-&gt;sig.dif.ref_tag);</div><div class='del'>-	/* repeating block */</div><div class='del'>-	inl-&gt;rp_inv_seed = MLX5_BSF_REPEAT_BLOCK;</div><div class='del'>-	inl-&gt;sig_type = domain-&gt;sig.dif.bg_type == IB_T10DIF_CRC ?</div><div class='del'>-			MLX5_DIF_CRC : MLX5_DIF_IPCS;</div><div class='del'>-</div><div class='del'>-	if (domain-&gt;sig.dif.ref_remap)</div><div class='del'>-		inl-&gt;dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG;</div><div class='del'>-</div><div class='del'>-	if (domain-&gt;sig.dif.app_escape) {</div><div class='del'>-		if (domain-&gt;sig.dif.ref_escape)</div><div class='del'>-			inl-&gt;dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE;</div><div class='del'>-		else</div><div class='del'>-			inl-&gt;dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	inl-&gt;dif_app_bitmask_check =</div><div class='del'>-		cpu_to_be16(domain-&gt;sig.dif.apptag_check_mask);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int mlx5_set_bsf(struct ib_mr *sig_mr,</div><div class='del'>-			struct ib_sig_attrs *sig_attrs,</div><div class='del'>-			struct mlx5_bsf *bsf, u32 data_size)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)-&gt;sig;</div><div class='del'>-	struct mlx5_bsf_basic *basic = &amp;bsf-&gt;basic;</div><div class='del'>-	struct ib_sig_domain *mem = &amp;sig_attrs-&gt;mem;</div><div class='del'>-	struct ib_sig_domain *wire = &amp;sig_attrs-&gt;wire;</div><div class='del'>-</div><div class='del'>-	memset(bsf, 0, sizeof(*bsf));</div><div class='del'>-</div><div class='del'>-	/* Basic + Extended + Inline */</div><div class='del'>-	basic-&gt;bsf_size_sbs = 1 &lt;&lt; 7;</div><div class='del'>-	/* Input domain check byte mask */</div><div class='del'>-	basic-&gt;check_byte_mask = sig_attrs-&gt;check_mask;</div><div class='del'>-	basic-&gt;raw_data_size = cpu_to_be32(data_size);</div><div class='del'>-</div><div class='del'>-	/* Memory domain */</div><div class='del'>-	switch (sig_attrs-&gt;mem.sig_type) {</div><div class='del'>-	case IB_SIG_TYPE_NONE:</div><div class='del'>-		break;</div><div class='del'>-	case IB_SIG_TYPE_T10_DIF:</div><div class='del'>-		basic-&gt;mem.bs_selector = bs_selector(mem-&gt;sig.dif.pi_interval);</div><div class='del'>-		basic-&gt;m_bfs_psv = cpu_to_be32(msig-&gt;psv_memory.psv_idx);</div><div class='del'>-		mlx5_fill_inl_bsf(mem, &amp;bsf-&gt;m_inl);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	/* Wire domain */</div><div class='del'>-	switch (sig_attrs-&gt;wire.sig_type) {</div><div class='del'>-	case IB_SIG_TYPE_NONE:</div><div class='del'>-		break;</div><div class='del'>-	case IB_SIG_TYPE_T10_DIF:</div><div class='del'>-		if (mem-&gt;sig.dif.pi_interval == wire-&gt;sig.dif.pi_interval &amp;&amp;</div><div class='del'>-		    mem-&gt;sig_type == wire-&gt;sig_type) {</div><div class='del'>-			/* Same block structure */</div><div class='del'>-			basic-&gt;bsf_size_sbs |= 1 &lt;&lt; 4;</div><div class='del'>-			if (mem-&gt;sig.dif.bg_type == wire-&gt;sig.dif.bg_type)</div><div class='del'>-				basic-&gt;wire.copy_byte_mask |= MLX5_CPY_GRD_MASK;</div><div class='del'>-			if (mem-&gt;sig.dif.app_tag == wire-&gt;sig.dif.app_tag)</div><div class='del'>-				basic-&gt;wire.copy_byte_mask |= MLX5_CPY_APP_MASK;</div><div class='del'>-			if (mem-&gt;sig.dif.ref_tag == wire-&gt;sig.dif.ref_tag)</div><div class='del'>-				basic-&gt;wire.copy_byte_mask |= MLX5_CPY_REF_MASK;</div><div class='del'>-		} else</div><div class='del'>-			basic-&gt;wire.bs_selector = bs_selector(wire-&gt;sig.dif.pi_interval);</div><div class='del'>-</div><div class='del'>-		basic-&gt;w_bfs_psv = cpu_to_be32(msig-&gt;psv_wire.psv_idx);</div><div class='del'>-		mlx5_fill_inl_bsf(wire, &amp;bsf-&gt;w_inl);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int set_sig_data_segment(const struct ib_sig_handover_wr *wr,</div><div class='del'>-				struct mlx5_ib_qp *qp, void **seg,</div><div class='del'>-				int *size, void **cur_edge)</div><div class='del'>-{</div><div class='del'>-	struct ib_sig_attrs *sig_attrs = wr-&gt;sig_attrs;</div><div class='del'>-	struct ib_mr *sig_mr = wr-&gt;sig_mr;</div><div class='del'>-	struct mlx5_bsf *bsf;</div><div class='del'>-	u32 data_len = wr-&gt;wr.sg_list-&gt;length;</div><div class='del'>-	u32 data_key = wr-&gt;wr.sg_list-&gt;lkey;</div><div class='del'>-	u64 data_va = wr-&gt;wr.sg_list-&gt;addr;</div><div class='del'>-	int ret;</div><div class='del'>-	int wqe_size;</div><div class='del'>-</div><div class='del'>-	if (!wr-&gt;prot ||</div><div class='del'>-	    (data_key == wr-&gt;prot-&gt;lkey &amp;&amp;</div><div class='del'>-	     data_va == wr-&gt;prot-&gt;addr &amp;&amp;</div><div class='del'>-	     data_len == wr-&gt;prot-&gt;length)) {</div><div class='del'>-		/**</div><div class='del'>-		 * Source domain doesn't contain signature information</div><div class='del'>-		 * or data and protection are interleaved in memory.</div><div class='del'>-		 * So need construct:</div><div class='del'>-		 *                  ------------------</div><div class='del'>-		 *                 |     data_klm     |</div><div class='del'>-		 *                  ------------------</div><div class='del'>-		 *                 |       BSF        |</div><div class='del'>-		 *                  ------------------</div><div class='del'>-		 **/</div><div class='del'>-		struct mlx5_klm *data_klm = *seg;</div><div class='del'>-</div><div class='del'>-		data_klm-&gt;bcount = cpu_to_be32(data_len);</div><div class='del'>-		data_klm-&gt;key = cpu_to_be32(data_key);</div><div class='del'>-		data_klm-&gt;va = cpu_to_be64(data_va);</div><div class='del'>-		wqe_size = ALIGN(sizeof(*data_klm), 64);</div><div class='del'>-	} else {</div><div class='del'>-		/**</div><div class='del'>-		 * Source domain contains signature information</div><div class='del'>-		 * So need construct a strided block format:</div><div class='del'>-		 *               ---------------------------</div><div class='del'>-		 *              |     stride_block_ctrl     |</div><div class='del'>-		 *               ---------------------------</div><div class='del'>-		 *              |          data_klm         |</div><div class='del'>-		 *               ---------------------------</div><div class='del'>-		 *              |          prot_klm         |</div><div class='del'>-		 *               ---------------------------</div><div class='del'>-		 *              |             BSF           |</div><div class='del'>-		 *               ---------------------------</div><div class='del'>-		 **/</div><div class='del'>-		struct mlx5_stride_block_ctrl_seg *sblock_ctrl;</div><div class='del'>-		struct mlx5_stride_block_entry *data_sentry;</div><div class='del'>-		struct mlx5_stride_block_entry *prot_sentry;</div><div class='del'>-		u32 prot_key = wr-&gt;prot-&gt;lkey;</div><div class='del'>-		u64 prot_va = wr-&gt;prot-&gt;addr;</div><div class='del'>-		u16 block_size = sig_attrs-&gt;mem.sig.dif.pi_interval;</div><div class='del'>-		int prot_size;</div><div class='del'>-</div><div class='del'>-		sblock_ctrl = *seg;</div><div class='del'>-		data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl);</div><div class='del'>-		prot_sentry = (void *)data_sentry + sizeof(*data_sentry);</div><div class='del'>-</div><div class='del'>-		prot_size = prot_field_size(sig_attrs-&gt;mem.sig_type);</div><div class='del'>-		if (!prot_size) {</div><div class='del'>-			pr_err("Bad block size given: %u\n", block_size);</div><div class='del'>-			return -EINVAL;</div><div class='del'>-		}</div><div class='del'>-		sblock_ctrl-&gt;bcount_per_cycle = cpu_to_be32(block_size +</div><div class='del'>-							    prot_size);</div><div class='del'>-		sblock_ctrl-&gt;op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP);</div><div class='del'>-		sblock_ctrl-&gt;repeat_count = cpu_to_be32(data_len / block_size);</div><div class='del'>-		sblock_ctrl-&gt;num_entries = cpu_to_be16(2);</div><div class='del'>-</div><div class='del'>-		data_sentry-&gt;bcount = cpu_to_be16(block_size);</div><div class='del'>-		data_sentry-&gt;key = cpu_to_be32(data_key);</div><div class='del'>-		data_sentry-&gt;va = cpu_to_be64(data_va);</div><div class='del'>-		data_sentry-&gt;stride = cpu_to_be16(block_size);</div><div class='del'>-</div><div class='del'>-		prot_sentry-&gt;bcount = cpu_to_be16(prot_size);</div><div class='del'>-		prot_sentry-&gt;key = cpu_to_be32(prot_key);</div><div class='del'>-		prot_sentry-&gt;va = cpu_to_be64(prot_va);</div><div class='del'>-		prot_sentry-&gt;stride = cpu_to_be16(prot_size);</div><div class='del'>-</div><div class='del'>-		wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) +</div><div class='del'>-				 sizeof(*prot_sentry), 64);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	*seg += wqe_size;</div><div class='del'>-	*size += wqe_size / 16;</div><div class='del'>-	handle_post_send_edge(&amp;qp-&gt;sq, seg, *size, cur_edge);</div><div class='del'>-</div><div class='del'>-	bsf = *seg;</div><div class='del'>-	ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len);</div><div class='del'>-	if (ret)</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	*seg += sizeof(*bsf);</div><div class='del'>-	*size += sizeof(*bsf) / 16;</div><div class='del'>-	handle_post_send_edge(&amp;qp-&gt;sq, seg, *size, cur_edge);</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,</div><div class='del'>-				 const struct ib_sig_handover_wr *wr, u32 size,</div><div class='del'>-				 u32 length, u32 pdn)</div><div class='del'>-{</div><div class='del'>-	struct ib_mr *sig_mr = wr-&gt;sig_mr;</div><div class='del'>-	u32 sig_key = sig_mr-&gt;rkey;</div><div class='del'>-	u8 sigerr = to_mmr(sig_mr)-&gt;sig-&gt;sigerr_count &amp; 1;</div><div class='del'>-</div><div class='del'>-	memset(seg, 0, sizeof(*seg));</div><div class='del'>-</div><div class='del'>-	seg-&gt;flags = get_umr_flags(wr-&gt;access_flags) |</div><div class='del'>-				   MLX5_MKC_ACCESS_MODE_KLMS;</div><div class='del'>-	seg-&gt;qpn_mkey7_0 = cpu_to_be32((sig_key &amp; 0xff) | 0xffffff00);</div><div class='del'>-	seg-&gt;flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr &lt;&lt; 26 |</div><div class='del'>-				    MLX5_MKEY_BSF_EN | pdn);</div><div class='del'>-	seg-&gt;len = cpu_to_be64(length);</div><div class='del'>-	seg-&gt;xlt_oct_size = cpu_to_be32(get_xlt_octo(size));</div><div class='del'>-	seg-&gt;bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,</div><div class='del'>-				u32 size)</div><div class='del'>-{</div><div class='del'>-	memset(umr, 0, sizeof(*umr));</div><div class='del'>-</div><div class='del'>-	umr-&gt;flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;</div><div class='del'>-	umr-&gt;xlt_octowords = cpu_to_be16(get_xlt_octo(size));</div><div class='del'>-	umr-&gt;bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);</div><div class='del'>-	umr-&gt;mkey_mask = sig_mkey_mask();</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-</div><div class='del'>-static int set_sig_umr_wr(const struct ib_send_wr *send_wr,</div><div class='del'>-			  struct mlx5_ib_qp *qp, void **seg, int *size,</div><div class='del'>-			  void **cur_edge)</div><div class='del'>-{</div><div class='del'>-	const struct ib_sig_handover_wr *wr = sig_handover_wr(send_wr);</div><div class='del'>-	struct mlx5_ib_mr *sig_mr = to_mmr(wr-&gt;sig_mr);</div><div class='del'>-	u32 pdn = get_pd(qp)-&gt;pdn;</div><div class='del'>-	u32 xlt_size;</div><div class='del'>-	int region_len, ret;</div><div class='del'>-</div><div class='del'>-	if (unlikely(wr-&gt;wr.num_sge != 1) ||</div><div class='del'>-	    unlikely(wr-&gt;access_flags &amp; IB_ACCESS_REMOTE_ATOMIC) ||</div><div class='del'>-	    unlikely(!sig_mr-&gt;sig) || unlikely(!qp-&gt;signature_en) ||</div><div class='del'>-	    unlikely(!sig_mr-&gt;sig-&gt;sig_status_checked))</div><div class='del'>-		return -EINVAL;</div><div class='del'>-</div><div class='del'>-	/* length of the protected region, data + protection */</div><div class='del'>-	region_len = wr-&gt;wr.sg_list-&gt;length;</div><div class='del'>-	if (wr-&gt;prot &amp;&amp;</div><div class='del'>-	    (wr-&gt;prot-&gt;lkey != wr-&gt;wr.sg_list-&gt;lkey  ||</div><div class='del'>-	     wr-&gt;prot-&gt;addr != wr-&gt;wr.sg_list-&gt;addr  ||</div><div class='del'>-	     wr-&gt;prot-&gt;length != wr-&gt;wr.sg_list-&gt;length))</div><div class='del'>-		region_len += wr-&gt;prot-&gt;length;</div><div class='del'>-</div><div class='del'>-	/**</div><div class='del'>-	 * KLM octoword size - if protection was provided</div><div class='del'>-	 * then we use strided block format (3 octowords),</div><div class='del'>-	 * else we use single KLM (1 octoword)</div><div class='del'>-	 **/</div><div class='del'>-	xlt_size = wr-&gt;prot ? 0x30 : sizeof(struct mlx5_klm);</div><div class='del'>-</div><div class='del'>-	set_sig_umr_segment(*seg, xlt_size);</div><div class='del'>-	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);</div><div class='del'>-	*size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;</div><div class='del'>-	handle_post_send_edge(&amp;qp-&gt;sq, seg, *size, cur_edge);</div><div class='del'>-</div><div class='del'>-	set_sig_mkey_segment(*seg, wr, xlt_size, region_len, pdn);</div><div class='del'>-	*seg += sizeof(struct mlx5_mkey_seg);</div><div class='del'>-	*size += sizeof(struct mlx5_mkey_seg) / 16;</div><div class='del'>-	handle_post_send_edge(&amp;qp-&gt;sq, seg, *size, cur_edge);</div><div class='del'>-</div><div class='del'>-	ret = set_sig_data_segment(wr, qp, seg, size, cur_edge);</div><div class='del'>-	if (ret)</div><div class='del'>-		return ret;</div><div class='del'>-</div><div class='del'>-	sig_mr-&gt;sig-&gt;sig_status_checked = false;</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int set_psv_wr(struct ib_sig_domain *domain,</div><div class='del'>-		      u32 psv_idx, void **seg, int *size)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_seg_set_psv *psv_seg = *seg;</div><div class='del'>-</div><div class='del'>-	memset(psv_seg, 0, sizeof(*psv_seg));</div><div class='del'>-	psv_seg-&gt;psv_num = cpu_to_be32(psv_idx);</div><div class='del'>-	switch (domain-&gt;sig_type) {</div><div class='del'>-	case IB_SIG_TYPE_NONE:</div><div class='del'>-		break;</div><div class='del'>-	case IB_SIG_TYPE_T10_DIF:</div><div class='del'>-		psv_seg-&gt;transient_sig = cpu_to_be32(domain-&gt;sig.dif.bg &lt;&lt; 16 |</div><div class='del'>-						     domain-&gt;sig.dif.app_tag);</div><div class='del'>-		psv_seg-&gt;ref_tag = cpu_to_be32(domain-&gt;sig.dif.ref_tag);</div><div class='del'>-		break;</div><div class='del'>-	default:</div><div class='del'>-		pr_err("Bad signature type (%d) is given.\n",</div><div class='del'>-		       domain-&gt;sig_type);</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	*seg += sizeof(*psv_seg);</div><div class='del'>-	*size += sizeof(*psv_seg) / 16;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int set_reg_wr(struct mlx5_ib_qp *qp,</div><div class='del'>-		      const struct ib_reg_wr *wr,</div><div class='del'>-		      void **seg, int *size, void **cur_edge)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_mr *mr = to_mmr(wr-&gt;mr);</div><div class='del'>-	struct mlx5_ib_pd *pd = to_mpd(qp-&gt;ibqp.pd);</div><div class='del'>-	size_t mr_list_size = mr-&gt;ndescs * mr-&gt;desc_size;</div><div class='del'>-	bool umr_inline = mr_list_size &lt;= MLX5_IB_SQ_UMR_INLINE_THRESHOLD;</div><div class='del'>-</div><div class='del'>-	if (unlikely(wr-&gt;wr.send_flags &amp; IB_SEND_INLINE)) {</div><div class='del'>-		mlx5_ib_warn(to_mdev(qp-&gt;ibqp.device),</div><div class='del'>-			     "Invalid IB_SEND_INLINE send flag\n");</div><div class='del'>-		return -EINVAL;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	set_reg_umr_seg(*seg, mr, umr_inline);</div><div class='del'>-	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);</div><div class='del'>-	*size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;</div><div class='del'>-	handle_post_send_edge(&amp;qp-&gt;sq, seg, *size, cur_edge);</div><div class='del'>-</div><div class='del'>-	set_reg_mkey_seg(*seg, mr, wr-&gt;key, wr-&gt;access);</div><div class='del'>-	*seg += sizeof(struct mlx5_mkey_seg);</div><div class='del'>-	*size += sizeof(struct mlx5_mkey_seg) / 16;</div><div class='del'>-	handle_post_send_edge(&amp;qp-&gt;sq, seg, *size, cur_edge);</div><div class='del'>-</div><div class='del'>-	if (umr_inline) {</div><div class='del'>-		memcpy_send_wqe(&amp;qp-&gt;sq, cur_edge, seg, size, mr-&gt;descs,</div><div class='del'>-				mr_list_size);</div><div class='del'>-		*size = ALIGN(*size, MLX5_SEND_WQE_BB &gt;&gt; 4);</div><div class='del'>-	} else {</div><div class='del'>-		set_reg_data_seg(*seg, mr, pd);</div><div class='del'>-		*seg += sizeof(struct mlx5_wqe_data_seg);</div><div class='del'>-		*size += (sizeof(struct mlx5_wqe_data_seg) / 16);</div><div class='del'>-	}</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size,</div><div class='del'>-			void **cur_edge)</div><div class='del'>-{</div><div class='del'>-	set_linv_umr_seg(*seg);</div><div class='del'>-	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);</div><div class='del'>-	*size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;</div><div class='del'>-	handle_post_send_edge(&amp;qp-&gt;sq, seg, *size, cur_edge);</div><div class='del'>-	set_linv_mkey_seg(*seg);</div><div class='del'>-	*seg += sizeof(struct mlx5_mkey_seg);</div><div class='del'>-	*size += sizeof(struct mlx5_mkey_seg) / 16;</div><div class='del'>-	handle_post_send_edge(&amp;qp-&gt;sq, seg, *size, cur_edge);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void dump_wqe(struct mlx5_ib_qp *qp, u32 idx, int size_16)</div><div class='del'>-{</div><div class='del'>-	__be32 *p = NULL;</div><div class='del'>-	int i, j;</div><div class='del'>-</div><div class='del'>-	pr_debug("dump WQE index %u:\n", idx);</div><div class='del'>-	for (i = 0, j = 0; i &lt; size_16 * 4; i += 4, j += 4) {</div><div class='del'>-		if ((i &amp; 0xf) == 0) {</div><div class='del'>-			p = mlx5_frag_buf_get_wqe(&amp;qp-&gt;sq.fbc, idx);</div><div class='del'>-			pr_debug("WQBB at %p:\n", (void *)p);</div><div class='del'>-			j = 0;</div><div class='del'>-			idx = (idx + 1) &amp; (qp-&gt;sq.wqe_cnt - 1);</div><div class='del'>-		}</div><div class='del'>-		pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]),</div><div class='del'>-			 be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]),</div><div class='del'>-			 be32_to_cpu(p[j + 3]));</div><div class='del'>-	}</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int __begin_wqe(struct mlx5_ib_qp *qp, void **seg,</div><div class='del'>-		       struct mlx5_wqe_ctrl_seg **ctrl,</div><div class='del'>-		       const struct ib_send_wr *wr, unsigned int *idx,</div><div class='del'>-		       int *size, void **cur_edge, int nreq,</div><div class='del'>-		       bool send_signaled, bool solicited)</div><div class='del'>-{</div><div class='del'>-	if (unlikely(mlx5_wq_overflow(&amp;qp-&gt;sq, nreq, qp-&gt;ibqp.send_cq)))</div><div class='del'>-		return -ENOMEM;</div><div class='del'>-</div><div class='del'>-	*idx = qp-&gt;sq.cur_post &amp; (qp-&gt;sq.wqe_cnt - 1);</div><div class='del'>-	*seg = mlx5_frag_buf_get_wqe(&amp;qp-&gt;sq.fbc, *idx);</div><div class='del'>-	*ctrl = *seg;</div><div class='del'>-	*(uint32_t *)(*seg + 8) = 0;</div><div class='del'>-	(*ctrl)-&gt;imm = send_ieth(wr);</div><div class='del'>-	(*ctrl)-&gt;fm_ce_se = qp-&gt;sq_signal_bits |</div><div class='del'>-		(send_signaled ? MLX5_WQE_CTRL_CQ_UPDATE : 0) |</div><div class='del'>-		(solicited ? MLX5_WQE_CTRL_SOLICITED : 0);</div><div class='del'>-</div><div class='del'>-	*seg += sizeof(**ctrl);</div><div class='del'>-	*size = sizeof(**ctrl) / 16;</div><div class='del'>-	*cur_edge = qp-&gt;sq.cur_edge;</div><div class='del'>-</div><div class='del'>-	return 0;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,</div><div class='del'>-		     struct mlx5_wqe_ctrl_seg **ctrl,</div><div class='del'>-		     const struct ib_send_wr *wr, unsigned *idx,</div><div class='del'>-		     int *size, void **cur_edge, int nreq)</div><div class='del'>-{</div><div class='del'>-	return __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq,</div><div class='del'>-			   wr-&gt;send_flags &amp; IB_SEND_SIGNALED,</div><div class='del'>-			   wr-&gt;send_flags &amp; IB_SEND_SOLICITED);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void finish_wqe(struct mlx5_ib_qp *qp,</div><div class='del'>-		       struct mlx5_wqe_ctrl_seg *ctrl,</div><div class='del'>-		       void *seg, u8 size, void *cur_edge,</div><div class='del'>-		       unsigned int idx, u64 wr_id, int nreq, u8 fence,</div><div class='del'>-		       u32 mlx5_opcode)</div><div class='del'>-{</div><div class='del'>-	u8 opmod = 0;</div><div class='del'>-</div><div class='del'>-	ctrl-&gt;opmod_idx_opcode = cpu_to_be32(((u32)(qp-&gt;sq.cur_post) &lt;&lt; 8) |</div><div class='del'>-					     mlx5_opcode | ((u32)opmod &lt;&lt; 24));</div><div class='del'>-	ctrl-&gt;qpn_ds = cpu_to_be32(size | (qp-&gt;trans_qp.base.mqp.qpn &lt;&lt; 8));</div><div class='del'>-	ctrl-&gt;fm_ce_se |= fence;</div><div class='del'>-	if (unlikely(qp-&gt;wq_sig))</div><div class='del'>-		ctrl-&gt;signature = wq_sig(ctrl);</div><div class='del'>-</div><div class='del'>-	qp-&gt;sq.wrid[idx] = wr_id;</div><div class='del'>-	qp-&gt;sq.w_list[idx].opcode = mlx5_opcode;</div><div class='del'>-	qp-&gt;sq.wqe_head[idx] = qp-&gt;sq.head + nreq;</div><div class='del'>-	qp-&gt;sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);</div><div class='del'>-	qp-&gt;sq.w_list[idx].next = qp-&gt;sq.cur_post;</div><div class='del'>-</div><div class='del'>-	/* We save the edge which was possibly updated during the WQE</div><div class='del'>-	 * construction, into SQ's cache.</div><div class='del'>-	 */</div><div class='del'>-	seg = PTR_ALIGN(seg, MLX5_SEND_WQE_BB);</div><div class='del'>-	qp-&gt;sq.cur_edge = (unlikely(seg == cur_edge)) ?</div><div class='del'>-			  get_sq_edge(&amp;qp-&gt;sq, qp-&gt;sq.cur_post &amp;</div><div class='del'>-				      (qp-&gt;sq.wqe_cnt - 1)) :</div><div class='del'>-			  cur_edge;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,</div><div class='del'>-			      const struct ib_send_wr **bad_wr, bool drain)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_wqe_ctrl_seg *ctrl = NULL;  /* compiler warning */</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(ibqp-&gt;device);</div><div class='del'>-	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='del'>-	struct mlx5_ib_qp *qp;</div><div class='del'>-	struct mlx5_ib_mr *mr;</div><div class='del'>-	struct mlx5_wqe_xrc_seg *xrc;</div><div class='del'>-	struct mlx5_bf *bf;</div><div class='del'>-	void *cur_edge;</div><div class='del'>-	int uninitialized_var(size);</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	unsigned idx;</div><div class='del'>-	int err = 0;</div><div class='del'>-	int num_sge;</div><div class='del'>-	void *seg;</div><div class='del'>-	int nreq;</div><div class='del'>-	int i;</div><div class='del'>-	u8 next_fence = 0;</div><div class='del'>-	u8 fence;</div><div class='del'>-</div><div class='del'>-	if (unlikely(mdev-&gt;state == MLX5_DEVICE_STATE_INTERNAL_ERROR &amp;&amp;</div><div class='del'>-		     !drain)) {</div><div class='del'>-		*bad_wr = wr;</div><div class='del'>-		return -EIO;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (unlikely(ibqp-&gt;qp_type == IB_QPT_GSI))</div><div class='del'>-		return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);</div><div class='del'>-</div><div class='del'>-	qp = to_mqp(ibqp);</div><div class='del'>-	bf = &amp;qp-&gt;bf;</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;qp-&gt;sq.lock, flags);</div><div class='del'>-</div><div class='del'>-	for (nreq = 0; wr; nreq++, wr = wr-&gt;next) {</div><div class='del'>-		if (unlikely(wr-&gt;opcode &gt;= ARRAY_SIZE(mlx5_ib_opcode))) {</div><div class='del'>-			mlx5_ib_warn(dev, "\n");</div><div class='del'>-			err = -EINVAL;</div><div class='del'>-			*bad_wr = wr;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		num_sge = wr-&gt;num_sge;</div><div class='del'>-		if (unlikely(num_sge &gt; qp-&gt;sq.max_gs)) {</div><div class='del'>-			mlx5_ib_warn(dev, "\n");</div><div class='del'>-			err = -EINVAL;</div><div class='del'>-			*bad_wr = wr;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		err = begin_wqe(qp, &amp;seg, &amp;ctrl, wr, &amp;idx, &amp;size, &amp;cur_edge,</div><div class='del'>-				nreq);</div><div class='del'>-		if (err) {</div><div class='del'>-			mlx5_ib_warn(dev, "\n");</div><div class='del'>-			err = -ENOMEM;</div><div class='del'>-			*bad_wr = wr;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (wr-&gt;opcode == IB_WR_REG_MR) {</div><div class='del'>-			fence = dev-&gt;umr_fence;</div><div class='del'>-			next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;</div><div class='del'>-		} else  {</div><div class='del'>-			if (wr-&gt;send_flags &amp; IB_SEND_FENCE) {</div><div class='del'>-				if (qp-&gt;next_fence)</div><div class='del'>-					fence = MLX5_FENCE_MODE_SMALL_AND_FENCE;</div><div class='del'>-				else</div><div class='del'>-					fence = MLX5_FENCE_MODE_FENCE;</div><div class='del'>-			} else {</div><div class='del'>-				fence = qp-&gt;next_fence;</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		switch (ibqp-&gt;qp_type) {</div><div class='del'>-		case IB_QPT_XRC_INI:</div><div class='del'>-			xrc = seg;</div><div class='del'>-			seg += sizeof(*xrc);</div><div class='del'>-			size += sizeof(*xrc) / 16;</div><div class='del'>-			/* fall through */</div><div class='del'>-		case IB_QPT_RC:</div><div class='del'>-			switch (wr-&gt;opcode) {</div><div class='del'>-			case IB_WR_RDMA_READ:</div><div class='del'>-			case IB_WR_RDMA_WRITE:</div><div class='del'>-			case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='del'>-				set_raddr_seg(seg, rdma_wr(wr)-&gt;remote_addr,</div><div class='del'>-					      rdma_wr(wr)-&gt;rkey);</div><div class='del'>-				seg += sizeof(struct mlx5_wqe_raddr_seg);</div><div class='del'>-				size += sizeof(struct mlx5_wqe_raddr_seg) / 16;</div><div class='del'>-				break;</div><div class='del'>-</div><div class='del'>-			case IB_WR_ATOMIC_CMP_AND_SWP:</div><div class='del'>-			case IB_WR_ATOMIC_FETCH_AND_ADD:</div><div class='del'>-			case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:</div><div class='del'>-				mlx5_ib_warn(dev, "Atomic operations are not supported yet\n");</div><div class='del'>-				err = -ENOSYS;</div><div class='del'>-				*bad_wr = wr;</div><div class='del'>-				goto out;</div><div class='del'>-</div><div class='del'>-			case IB_WR_LOCAL_INV:</div><div class='del'>-				qp-&gt;sq.wr_data[idx] = IB_WR_LOCAL_INV;</div><div class='del'>-				ctrl-&gt;imm = cpu_to_be32(wr-&gt;ex.invalidate_rkey);</div><div class='del'>-				set_linv_wr(qp, &amp;seg, &amp;size, &amp;cur_edge);</div><div class='del'>-				num_sge = 0;</div><div class='del'>-				break;</div><div class='del'>-</div><div class='del'>-			case IB_WR_REG_MR:</div><div class='del'>-				qp-&gt;sq.wr_data[idx] = IB_WR_REG_MR;</div><div class='del'>-				ctrl-&gt;imm = cpu_to_be32(reg_wr(wr)-&gt;key);</div><div class='del'>-				err = set_reg_wr(qp, reg_wr(wr), &amp;seg, &amp;size,</div><div class='del'>-						 &amp;cur_edge);</div><div class='del'>-				if (err) {</div><div class='del'>-					*bad_wr = wr;</div><div class='del'>-					goto out;</div><div class='del'>-				}</div><div class='del'>-				num_sge = 0;</div><div class='del'>-				break;</div><div class='del'>-</div><div class='del'>-			case IB_WR_REG_SIG_MR:</div><div class='del'>-				qp-&gt;sq.wr_data[idx] = IB_WR_REG_SIG_MR;</div><div class='del'>-				mr = to_mmr(sig_handover_wr(wr)-&gt;sig_mr);</div><div class='del'>-</div><div class='del'>-				ctrl-&gt;imm = cpu_to_be32(mr-&gt;ibmr.rkey);</div><div class='del'>-				err = set_sig_umr_wr(wr, qp, &amp;seg, &amp;size,</div><div class='del'>-						     &amp;cur_edge);</div><div class='del'>-				if (err) {</div><div class='del'>-					mlx5_ib_warn(dev, "\n");</div><div class='del'>-					*bad_wr = wr;</div><div class='del'>-					goto out;</div><div class='del'>-				}</div><div class='del'>-</div><div class='del'>-				finish_wqe(qp, ctrl, seg, size, cur_edge, idx,</div><div class='del'>-					   wr-&gt;wr_id, nreq, fence,</div><div class='del'>-					   MLX5_OPCODE_UMR);</div><div class='del'>-				/*</div><div class='del'>-				 * SET_PSV WQEs are not signaled and solicited</div><div class='del'>-				 * on error</div><div class='del'>-				 */</div><div class='del'>-				err = __begin_wqe(qp, &amp;seg, &amp;ctrl, wr, &amp;idx,</div><div class='del'>-						  &amp;size, &amp;cur_edge, nreq, false,</div><div class='del'>-						  true);</div><div class='del'>-				if (err) {</div><div class='del'>-					mlx5_ib_warn(dev, "\n");</div><div class='del'>-					err = -ENOMEM;</div><div class='del'>-					*bad_wr = wr;</div><div class='del'>-					goto out;</div><div class='del'>-				}</div><div class='del'>-</div><div class='del'>-				err = set_psv_wr(&amp;sig_handover_wr(wr)-&gt;sig_attrs-&gt;mem,</div><div class='del'>-						 mr-&gt;sig-&gt;psv_memory.psv_idx, &amp;seg,</div><div class='del'>-						 &amp;size);</div><div class='del'>-				if (err) {</div><div class='del'>-					mlx5_ib_warn(dev, "\n");</div><div class='del'>-					*bad_wr = wr;</div><div class='del'>-					goto out;</div><div class='del'>-				}</div><div class='del'>-</div><div class='del'>-				finish_wqe(qp, ctrl, seg, size, cur_edge, idx,</div><div class='del'>-					   wr-&gt;wr_id, nreq, fence,</div><div class='del'>-					   MLX5_OPCODE_SET_PSV);</div><div class='del'>-				err = __begin_wqe(qp, &amp;seg, &amp;ctrl, wr, &amp;idx,</div><div class='del'>-						  &amp;size, &amp;cur_edge, nreq, false,</div><div class='del'>-						  true);</div><div class='del'>-				if (err) {</div><div class='del'>-					mlx5_ib_warn(dev, "\n");</div><div class='del'>-					err = -ENOMEM;</div><div class='del'>-					*bad_wr = wr;</div><div class='del'>-					goto out;</div><div class='del'>-				}</div><div class='del'>-</div><div class='del'>-				err = set_psv_wr(&amp;sig_handover_wr(wr)-&gt;sig_attrs-&gt;wire,</div><div class='del'>-						 mr-&gt;sig-&gt;psv_wire.psv_idx, &amp;seg,</div><div class='del'>-						 &amp;size);</div><div class='del'>-				if (err) {</div><div class='del'>-					mlx5_ib_warn(dev, "\n");</div><div class='del'>-					*bad_wr = wr;</div><div class='del'>-					goto out;</div><div class='del'>-				}</div><div class='del'>-</div><div class='del'>-				finish_wqe(qp, ctrl, seg, size, cur_edge, idx,</div><div class='del'>-					   wr-&gt;wr_id, nreq, fence,</div><div class='del'>-					   MLX5_OPCODE_SET_PSV);</div><div class='del'>-				qp-&gt;next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;</div><div class='del'>-				num_sge = 0;</div><div class='del'>-				goto skip_psv;</div><div class='del'>-</div><div class='del'>-			default:</div><div class='del'>-				break;</div><div class='del'>-			}</div><div class='del'>-			break;</div><div class='del'>-</div><div class='del'>-		case IB_QPT_UC:</div><div class='del'>-			switch (wr-&gt;opcode) {</div><div class='del'>-			case IB_WR_RDMA_WRITE:</div><div class='del'>-			case IB_WR_RDMA_WRITE_WITH_IMM:</div><div class='del'>-				set_raddr_seg(seg, rdma_wr(wr)-&gt;remote_addr,</div><div class='del'>-					      rdma_wr(wr)-&gt;rkey);</div><div class='del'>-				seg  += sizeof(struct mlx5_wqe_raddr_seg);</div><div class='del'>-				size += sizeof(struct mlx5_wqe_raddr_seg) / 16;</div><div class='del'>-				break;</div><div class='del'>-</div><div class='del'>-			default:</div><div class='del'>-				break;</div><div class='del'>-			}</div><div class='del'>-			break;</div><div class='del'>-</div><div class='del'>-		case IB_QPT_SMI:</div><div class='del'>-			if (unlikely(!mdev-&gt;port_caps[qp-&gt;port - 1].has_smi)) {</div><div class='del'>-				mlx5_ib_warn(dev, "Send SMP MADs is not allowed\n");</div><div class='del'>-				err = -EPERM;</div><div class='del'>-				*bad_wr = wr;</div><div class='del'>-				goto out;</div><div class='del'>-			}</div><div class='del'>-			/* fall through */</div><div class='del'>-		case MLX5_IB_QPT_HW_GSI:</div><div class='del'>-			set_datagram_seg(seg, wr);</div><div class='del'>-			seg += sizeof(struct mlx5_wqe_datagram_seg);</div><div class='del'>-			size += sizeof(struct mlx5_wqe_datagram_seg) / 16;</div><div class='del'>-			handle_post_send_edge(&amp;qp-&gt;sq, &amp;seg, size, &amp;cur_edge);</div><div class='del'>-</div><div class='del'>-			break;</div><div class='del'>-		case IB_QPT_UD:</div><div class='del'>-			set_datagram_seg(seg, wr);</div><div class='del'>-			seg += sizeof(struct mlx5_wqe_datagram_seg);</div><div class='del'>-			size += sizeof(struct mlx5_wqe_datagram_seg) / 16;</div><div class='del'>-			handle_post_send_edge(&amp;qp-&gt;sq, &amp;seg, size, &amp;cur_edge);</div><div class='del'>-</div><div class='del'>-			/* handle qp that supports ud offload */</div><div class='del'>-			if (qp-&gt;flags &amp; IB_QP_CREATE_IPOIB_UD_LSO) {</div><div class='del'>-				struct mlx5_wqe_eth_pad *pad;</div><div class='del'>-</div><div class='del'>-				pad = seg;</div><div class='del'>-				memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad));</div><div class='del'>-				seg += sizeof(struct mlx5_wqe_eth_pad);</div><div class='del'>-				size += sizeof(struct mlx5_wqe_eth_pad) / 16;</div><div class='del'>-				set_eth_seg(wr, qp, &amp;seg, &amp;size, &amp;cur_edge);</div><div class='del'>-				handle_post_send_edge(&amp;qp-&gt;sq, &amp;seg, size,</div><div class='del'>-						      &amp;cur_edge);</div><div class='del'>-			}</div><div class='del'>-			break;</div><div class='del'>-		case MLX5_IB_QPT_REG_UMR:</div><div class='del'>-			if (wr-&gt;opcode != MLX5_IB_WR_UMR) {</div><div class='del'>-				err = -EINVAL;</div><div class='del'>-				mlx5_ib_warn(dev, "bad opcode\n");</div><div class='del'>-				goto out;</div><div class='del'>-			}</div><div class='del'>-			qp-&gt;sq.wr_data[idx] = MLX5_IB_WR_UMR;</div><div class='del'>-			ctrl-&gt;imm = cpu_to_be32(umr_wr(wr)-&gt;mkey);</div><div class='del'>-			err = set_reg_umr_segment(dev, seg, wr, !!(MLX5_CAP_GEN(mdev, atomic)));</div><div class='del'>-			if (unlikely(err))</div><div class='del'>-				goto out;</div><div class='del'>-			seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);</div><div class='del'>-			size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;</div><div class='del'>-			handle_post_send_edge(&amp;qp-&gt;sq, &amp;seg, size, &amp;cur_edge);</div><div class='del'>-			set_reg_mkey_segment(seg, wr);</div><div class='del'>-			seg += sizeof(struct mlx5_mkey_seg);</div><div class='del'>-			size += sizeof(struct mlx5_mkey_seg) / 16;</div><div class='del'>-			handle_post_send_edge(&amp;qp-&gt;sq, &amp;seg, size, &amp;cur_edge);</div><div class='del'>-			break;</div><div class='del'>-</div><div class='del'>-		default:</div><div class='del'>-			break;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (wr-&gt;send_flags &amp; IB_SEND_INLINE &amp;&amp; num_sge) {</div><div class='del'>-			err = set_data_inl_seg(qp, wr, &amp;seg, &amp;size, &amp;cur_edge);</div><div class='del'>-			if (unlikely(err)) {</div><div class='del'>-				mlx5_ib_warn(dev, "\n");</div><div class='del'>-				*bad_wr = wr;</div><div class='del'>-				goto out;</div><div class='del'>-			}</div><div class='del'>-		} else {</div><div class='del'>-			for (i = 0; i &lt; num_sge; i++) {</div><div class='del'>-				handle_post_send_edge(&amp;qp-&gt;sq, &amp;seg, size,</div><div class='del'>-						      &amp;cur_edge);</div><div class='del'>-				if (likely(wr-&gt;sg_list[i].length)) {</div><div class='del'>-					set_data_ptr_seg</div><div class='del'>-					((struct mlx5_wqe_data_seg *)seg,</div><div class='del'>-					 wr-&gt;sg_list + i);</div><div class='del'>-					size += sizeof(struct mlx5_wqe_data_seg) / 16;</div><div class='del'>-					seg += sizeof(struct mlx5_wqe_data_seg);</div><div class='del'>-				}</div><div class='del'>-			}</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		qp-&gt;next_fence = next_fence;</div><div class='del'>-		finish_wqe(qp, ctrl, seg, size, cur_edge, idx, wr-&gt;wr_id, nreq,</div><div class='del'>-			   fence, mlx5_ib_opcode[wr-&gt;opcode]);</div><div class='del'>-skip_psv:</div><div class='del'>-		if (0)</div><div class='del'>-			dump_wqe(qp, idx, size);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-out:</div><div class='del'>-	if (likely(nreq)) {</div><div class='del'>-		qp-&gt;sq.head += nreq;</div><div class='del'>-</div><div class='del'>-		/* Make sure that descriptors are written before</div><div class='del'>-		 * updating doorbell record and ringing the doorbell</div><div class='del'>-		 */</div><div class='del'>-		wmb();</div><div class='del'>-</div><div class='del'>-		qp-&gt;db.db[MLX5_SND_DBR] = cpu_to_be32(qp-&gt;sq.cur_post);</div><div class='del'>-</div><div class='del'>-		/* Make sure doorbell record is visible to the HCA before</div><div class='del'>-		 * we hit doorbell */</div><div class='del'>-		wmb();</div><div class='del'>-</div><div class='del'>-		/* currently we support only regular doorbells */</div><div class='del'>-		mlx5_write64((__be32 *)ctrl, bf-&gt;bfreg-&gt;map + bf-&gt;offset);</div><div class='del'>-		/* Make sure doorbells don't leak out of SQ spinlock</div><div class='del'>-		 * and reach the HCA out of order.</div><div class='del'>-		 */</div><div class='del'>-		bf-&gt;offset ^= bf-&gt;buf_size;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	spin_unlock_irqrestore(&amp;qp-&gt;sq.lock, flags);</div><div class='del'>-</div><div class='del'>-	return err;</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,</div><div class='del'>-		      const struct ib_send_wr **bad_wr)</div><div class='del'>-{</div><div class='del'>-	return _mlx5_ib_post_send(ibqp, wr, bad_wr, false);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static void set_sig_seg(struct mlx5_rwqe_sig *sig, int size)</div><div class='del'>-{</div><div class='del'>-	sig-&gt;signature = calc_sig(sig, size);</div><div class='del'>-}</div><div class='del'>-</div><div class='del'>-static int _mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,</div><div class='del'>-		      const struct ib_recv_wr **bad_wr, bool drain)</div><div class='del'>-{</div><div class='del'>-	struct mlx5_ib_qp *qp = to_mqp(ibqp);</div><div class='del'>-	struct mlx5_wqe_data_seg *scat;</div><div class='del'>-	struct mlx5_rwqe_sig *sig;</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(ibqp-&gt;device);</div><div class='del'>-	struct mlx5_core_dev *mdev = dev-&gt;mdev;</div><div class='del'>-	unsigned long flags;</div><div class='del'>-	int err = 0;</div><div class='del'>-	int nreq;</div><div class='del'>-	int ind;</div><div class='del'>-	int i;</div><div class='del'>-</div><div class='del'>-	if (unlikely(mdev-&gt;state == MLX5_DEVICE_STATE_INTERNAL_ERROR &amp;&amp;</div><div class='del'>-		     !drain)) {</div><div class='del'>-		*bad_wr = wr;</div><div class='del'>-		return -EIO;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	if (unlikely(ibqp-&gt;qp_type == IB_QPT_GSI))</div><div class='del'>-		return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr);</div><div class='del'>-</div><div class='del'>-	spin_lock_irqsave(&amp;qp-&gt;rq.lock, flags);</div><div class='del'>-</div><div class='del'>-	ind = qp-&gt;rq.head &amp; (qp-&gt;rq.wqe_cnt - 1);</div><div class='del'>-</div><div class='del'>-	for (nreq = 0; wr; nreq++, wr = wr-&gt;next) {</div><div class='del'>-		if (mlx5_wq_overflow(&amp;qp-&gt;rq, nreq, qp-&gt;ibqp.recv_cq)) {</div><div class='del'>-			err = -ENOMEM;</div><div class='del'>-			*bad_wr = wr;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (unlikely(wr-&gt;num_sge &gt; qp-&gt;rq.max_gs)) {</div><div class='del'>-			err = -EINVAL;</div><div class='del'>-			*bad_wr = wr;</div><div class='del'>-			goto out;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		scat = mlx5_frag_buf_get_wqe(&amp;qp-&gt;rq.fbc, ind);</div><div class='del'>-		if (qp-&gt;wq_sig)</div><div class='del'>-			scat++;</div><div class='del'>-</div><div class='del'>-		for (i = 0; i &lt; wr-&gt;num_sge; i++)</div><div class='del'>-			set_data_ptr_seg(scat + i, wr-&gt;sg_list + i);</div><div class='del'>-</div><div class='del'>-		if (i &lt; qp-&gt;rq.max_gs) {</div><div class='del'>-			scat[i].byte_count = 0;</div><div class='del'>-			scat[i].lkey       = cpu_to_be32(MLX5_INVALID_LKEY);</div><div class='del'>-			scat[i].addr       = 0;</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		if (qp-&gt;wq_sig) {</div><div class='del'>-			sig = (struct mlx5_rwqe_sig *)scat;</div><div class='del'>-			set_sig_seg(sig, (qp-&gt;rq.max_gs + 1) &lt;&lt; 2);</div><div class='del'>-		}</div><div class='del'>-</div><div class='del'>-		qp-&gt;rq.wrid[ind] = wr-&gt;wr_id;</div><div class='del'>-</div><div class='del'>-		ind = (ind + 1) &amp; (qp-&gt;rq.wqe_cnt - 1);</div><div class='del'>-	}</div><div class='add'>+	/* resp.response_length is set in ECE supported flows only */</div><div class='add'>+	if (!err &amp;&amp; resp.response_length &amp;&amp;</div><div class='add'>+	    udata-&gt;outlen &gt;= resp.response_length)</div><div class='add'>+		/* Return -EFAULT to the user and expect him to destroy QP. */</div><div class='add'>+		err = ib_copy_to_udata(udata, &amp;resp, resp.response_length);</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='del'>-	if (likely(nreq)) {</div><div class='del'>-		qp-&gt;rq.head += nreq;</div><div class='del'>-</div><div class='del'>-		/* Make sure that descriptors are written before</div><div class='del'>-		 * doorbell record.</div><div class='del'>-		 */</div><div class='del'>-		wmb();</div><div class='del'>-</div><div class='del'>-		*qp-&gt;db.db = cpu_to_be32(qp-&gt;rq.head &amp; 0xffff);</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	spin_unlock_irqrestore(&amp;qp-&gt;rq.lock, flags);</div><div class='del'>-</div><div class='add'>+	mutex_unlock(&amp;qp-&gt;mutex);</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,</div><div class='del'>-		      const struct ib_recv_wr **bad_wr)</div><div class='del'>-{</div><div class='del'>-	return _mlx5_ib_post_recv(ibqp, wr, bad_wr, false);</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static inline enum ib_qp_state to_ib_qp_state(enum mlx5_qp_state mlx5_state)</div><div class='ctx'> {</div><div class='ctx'> 	switch (mlx5_state) {</div><div class='hunk'>@@ -5336,48 +4821,34 @@ static inline enum ib_mig_state to_ib_mig_state(int mlx5_mig_state)</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-static int to_ib_qp_access_flags(int mlx5_flags)</div><div class='del'>-{</div><div class='del'>-	int ib_flags = 0;</div><div class='del'>-</div><div class='del'>-	if (mlx5_flags &amp; MLX5_QP_BIT_RRE)</div><div class='del'>-		ib_flags |= IB_ACCESS_REMOTE_READ;</div><div class='del'>-	if (mlx5_flags &amp; MLX5_QP_BIT_RWE)</div><div class='del'>-		ib_flags |= IB_ACCESS_REMOTE_WRITE;</div><div class='del'>-	if (mlx5_flags &amp; MLX5_QP_BIT_RAE)</div><div class='del'>-		ib_flags |= IB_ACCESS_REMOTE_ATOMIC;</div><div class='del'>-</div><div class='del'>-	return ib_flags;</div><div class='del'>-}</div><div class='del'>-</div><div class='ctx'> static void to_rdma_ah_attr(struct mlx5_ib_dev *ibdev,</div><div class='del'>-			    struct rdma_ah_attr *ah_attr,</div><div class='del'>-			    struct mlx5_qp_path *path)</div><div class='add'>+			    struct rdma_ah_attr *ah_attr, void *path)</div><div class='ctx'> {</div><div class='add'>+	int port = MLX5_GET(ads, path, vhca_port_num);</div><div class='add'>+	int static_rate;</div><div class='ctx'> </div><div class='ctx'> 	memset(ah_attr, 0, sizeof(*ah_attr));</div><div class='ctx'> </div><div class='del'>-	if (!path-&gt;port || path-&gt;port &gt; ibdev-&gt;num_ports)</div><div class='add'>+	if (!port || port &gt; ibdev-&gt;num_ports)</div><div class='ctx'> 		return;</div><div class='ctx'> </div><div class='del'>-	ah_attr-&gt;type = rdma_ah_find_type(&amp;ibdev-&gt;ib_dev, path-&gt;port);</div><div class='add'>+	ah_attr-&gt;type = rdma_ah_find_type(&amp;ibdev-&gt;ib_dev, port);</div><div class='ctx'> </div><div class='del'>-	rdma_ah_set_port_num(ah_attr, path-&gt;port);</div><div class='del'>-	rdma_ah_set_sl(ah_attr, path-&gt;dci_cfi_prio_sl &amp; 0xf);</div><div class='add'>+	rdma_ah_set_port_num(ah_attr, port);</div><div class='add'>+	rdma_ah_set_sl(ah_attr, MLX5_GET(ads, path, sl));</div><div class='ctx'> </div><div class='del'>-	rdma_ah_set_dlid(ah_attr, be16_to_cpu(path-&gt;rlid));</div><div class='del'>-	rdma_ah_set_path_bits(ah_attr, path-&gt;grh_mlid &amp; 0x7f);</div><div class='del'>-	rdma_ah_set_static_rate(ah_attr,</div><div class='del'>-				path-&gt;static_rate ? path-&gt;static_rate - 5 : 0);</div><div class='del'>-	if (path-&gt;grh_mlid &amp; (1 &lt;&lt; 7)) {</div><div class='del'>-		u32 tc_fl = be32_to_cpu(path-&gt;tclass_flowlabel);</div><div class='add'>+	rdma_ah_set_dlid(ah_attr, MLX5_GET(ads, path, rlid));</div><div class='add'>+	rdma_ah_set_path_bits(ah_attr, MLX5_GET(ads, path, mlid));</div><div class='ctx'> </div><div class='del'>-		rdma_ah_set_grh(ah_attr, NULL,</div><div class='del'>-				tc_fl &amp; 0xfffff,</div><div class='del'>-				path-&gt;mgid_index,</div><div class='del'>-				path-&gt;hop_limit,</div><div class='del'>-				(tc_fl &gt;&gt; 20) &amp; 0xff);</div><div class='del'>-		rdma_ah_set_dgid_raw(ah_attr, path-&gt;rgid);</div><div class='add'>+	static_rate = MLX5_GET(ads, path, stat_rate);</div><div class='add'>+	rdma_ah_set_static_rate(ah_attr, mlx5_to_ib_rate_map(static_rate));</div><div class='add'>+	if (MLX5_GET(ads, path, grh) ||</div><div class='add'>+	    ah_attr-&gt;type == RDMA_AH_ATTR_TYPE_ROCE) {</div><div class='add'>+		rdma_ah_set_grh(ah_attr, NULL, MLX5_GET(ads, path, flow_label),</div><div class='add'>+				MLX5_GET(ads, path, src_addr_index),</div><div class='add'>+				MLX5_GET(ads, path, hop_limit),</div><div class='add'>+				MLX5_GET(ads, path, tclass));</div><div class='add'>+		rdma_ah_set_dgid_raw(ah_attr, MLX5_ADDR_OF(ads, path, rgid_rip));</div><div class='ctx'> 	}</div><div class='ctx'> }</div><div class='ctx'> </div><div class='hunk'>@@ -5434,7 +4905,7 @@ static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state,</div><div class='ctx'> 			[MLX5_SQ_STATE_NA]	= IB_QPS_RESET,</div><div class='ctx'> 		},</div><div class='ctx'> 		[MLX5_RQC_STATE_RDY] = {</div><div class='del'>-			[MLX5_SQC_STATE_RST]	= MLX5_QP_STATE_BAD,</div><div class='add'>+			[MLX5_SQC_STATE_RST]	= MLX5_QP_STATE,</div><div class='ctx'> 			[MLX5_SQC_STATE_RDY]	= MLX5_QP_STATE,</div><div class='ctx'> 			[MLX5_SQC_STATE_ERR]	= IB_QPS_SQE,</div><div class='ctx'> 			[MLX5_SQ_STATE_NA]	= MLX5_QP_STATE,</div><div class='hunk'>@@ -5446,7 +4917,7 @@ static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state,</div><div class='ctx'> 			[MLX5_SQ_STATE_NA]	= IB_QPS_ERR,</div><div class='ctx'> 		},</div><div class='ctx'> 		[MLX5_RQ_STATE_NA] = {</div><div class='del'>-			[MLX5_SQC_STATE_RST]    = IB_QPS_RESET,</div><div class='add'>+			[MLX5_SQC_STATE_RST]    = MLX5_QP_STATE,</div><div class='ctx'> 			[MLX5_SQC_STATE_RDY]	= MLX5_QP_STATE,</div><div class='ctx'> 			[MLX5_SQC_STATE_ERR]	= MLX5_QP_STATE,</div><div class='ctx'> 			[MLX5_SQ_STATE_NA]	= MLX5_QP_STATE_BAD,</div><div class='hunk'>@@ -5499,61 +4970,60 @@ static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,</div><div class='ctx'> 			 struct ib_qp_attr *qp_attr)</div><div class='ctx'> {</div><div class='ctx'> 	int outlen = MLX5_ST_SZ_BYTES(query_qp_out);</div><div class='del'>-	struct mlx5_qp_context *context;</div><div class='del'>-	int mlx5_state;</div><div class='add'>+	void *qpc, *pri_path, *alt_path;</div><div class='ctx'> 	u32 *outb;</div><div class='del'>-	int err = 0;</div><div class='add'>+	int err;</div><div class='ctx'> </div><div class='ctx'> 	outb = kzalloc(outlen, GFP_KERNEL);</div><div class='ctx'> 	if (!outb)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	err = mlx5_core_qp_query(dev-&gt;mdev, &amp;qp-&gt;trans_qp.base.mqp, outb,</div><div class='del'>-				 outlen);</div><div class='add'>+	err = mlx5_core_qp_query(dev, &amp;qp-&gt;trans_qp.base.mqp, outb, outlen,</div><div class='add'>+				 false);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	/* FIXME: use MLX5_GET rather than mlx5_qp_context manual struct */</div><div class='del'>-	context = (struct mlx5_qp_context *)MLX5_ADDR_OF(query_qp_out, outb, qpc);</div><div class='add'>+	qpc = MLX5_ADDR_OF(query_qp_out, outb, qpc);</div><div class='ctx'> </div><div class='del'>-	mlx5_state = be32_to_cpu(context-&gt;flags) &gt;&gt; 28;</div><div class='add'>+	qp-&gt;state = to_ib_qp_state(MLX5_GET(qpc, qpc, state));</div><div class='add'>+	if (MLX5_GET(qpc, qpc, state) == MLX5_QP_STATE_SQ_DRAINING)</div><div class='add'>+		qp_attr-&gt;sq_draining = 1;</div><div class='ctx'> </div><div class='del'>-	qp-&gt;state		     = to_ib_qp_state(mlx5_state);</div><div class='del'>-	qp_attr-&gt;path_mtu	     = context-&gt;mtu_msgmax &gt;&gt; 5;</div><div class='del'>-	qp_attr-&gt;path_mig_state	     =</div><div class='del'>-		to_ib_mig_state((be32_to_cpu(context-&gt;flags) &gt;&gt; 11) &amp; 0x3);</div><div class='del'>-	qp_attr-&gt;qkey		     = be32_to_cpu(context-&gt;qkey);</div><div class='del'>-	qp_attr-&gt;rq_psn		     = be32_to_cpu(context-&gt;rnr_nextrecvpsn) &amp; 0xffffff;</div><div class='del'>-	qp_attr-&gt;sq_psn		     = be32_to_cpu(context-&gt;next_send_psn) &amp; 0xffffff;</div><div class='del'>-	qp_attr-&gt;dest_qp_num	     = be32_to_cpu(context-&gt;log_pg_sz_remote_qpn) &amp; 0xffffff;</div><div class='del'>-	qp_attr-&gt;qp_access_flags     =</div><div class='del'>-		to_ib_qp_access_flags(be32_to_cpu(context-&gt;params2));</div><div class='add'>+	qp_attr-&gt;path_mtu = MLX5_GET(qpc, qpc, mtu);</div><div class='add'>+	qp_attr-&gt;path_mig_state = to_ib_mig_state(MLX5_GET(qpc, qpc, pm_state));</div><div class='add'>+	qp_attr-&gt;qkey = MLX5_GET(qpc, qpc, q_key);</div><div class='add'>+	qp_attr-&gt;rq_psn = MLX5_GET(qpc, qpc, next_rcv_psn);</div><div class='add'>+	qp_attr-&gt;sq_psn = MLX5_GET(qpc, qpc, next_send_psn);</div><div class='add'>+	qp_attr-&gt;dest_qp_num = MLX5_GET(qpc, qpc, remote_qpn);</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;ibqp.qp_type == IB_QPT_RC || qp-&gt;ibqp.qp_type == IB_QPT_UC) {</div><div class='del'>-		to_rdma_ah_attr(dev, &amp;qp_attr-&gt;ah_attr, &amp;context-&gt;pri_path);</div><div class='del'>-		to_rdma_ah_attr(dev, &amp;qp_attr-&gt;alt_ah_attr, &amp;context-&gt;alt_path);</div><div class='del'>-		qp_attr-&gt;alt_pkey_index =</div><div class='del'>-			be16_to_cpu(context-&gt;alt_path.pkey_index);</div><div class='del'>-		qp_attr-&gt;alt_port_num	=</div><div class='del'>-			rdma_ah_get_port_num(&amp;qp_attr-&gt;alt_ah_attr);</div><div class='del'>-	}</div><div class='add'>+	if (MLX5_GET(qpc, qpc, rre))</div><div class='add'>+		qp_attr-&gt;qp_access_flags |= IB_ACCESS_REMOTE_READ;</div><div class='add'>+	if (MLX5_GET(qpc, qpc, rwe))</div><div class='add'>+		qp_attr-&gt;qp_access_flags |= IB_ACCESS_REMOTE_WRITE;</div><div class='add'>+	if (MLX5_GET(qpc, qpc, rae))</div><div class='add'>+		qp_attr-&gt;qp_access_flags |= IB_ACCESS_REMOTE_ATOMIC;</div><div class='ctx'> </div><div class='del'>-	qp_attr-&gt;pkey_index = be16_to_cpu(context-&gt;pri_path.pkey_index);</div><div class='del'>-	qp_attr-&gt;port_num = context-&gt;pri_path.port;</div><div class='add'>+	qp_attr-&gt;max_rd_atomic = 1 &lt;&lt; MLX5_GET(qpc, qpc, log_sra_max);</div><div class='add'>+	qp_attr-&gt;max_dest_rd_atomic = 1 &lt;&lt; MLX5_GET(qpc, qpc, log_rra_max);</div><div class='add'>+	qp_attr-&gt;min_rnr_timer = MLX5_GET(qpc, qpc, min_rnr_nak);</div><div class='add'>+	qp_attr-&gt;retry_cnt = MLX5_GET(qpc, qpc, retry_count);</div><div class='add'>+	qp_attr-&gt;rnr_retry = MLX5_GET(qpc, qpc, rnr_retry);</div><div class='ctx'> </div><div class='del'>-	/* qp_attr-&gt;en_sqd_async_notify is only applicable in modify qp */</div><div class='del'>-	qp_attr-&gt;sq_draining = mlx5_state == MLX5_QP_STATE_SQ_DRAINING;</div><div class='add'>+	pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);</div><div class='add'>+	alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);</div><div class='ctx'> </div><div class='del'>-	qp_attr-&gt;max_rd_atomic = 1 &lt;&lt; ((be32_to_cpu(context-&gt;params1) &gt;&gt; 21) &amp; 0x7);</div><div class='add'>+	if (qp-&gt;type == IB_QPT_RC || qp-&gt;type == IB_QPT_UC ||</div><div class='add'>+	    qp-&gt;type == IB_QPT_XRC_INI || qp-&gt;type == IB_QPT_XRC_TGT) {</div><div class='add'>+		to_rdma_ah_attr(dev, &amp;qp_attr-&gt;ah_attr, pri_path);</div><div class='add'>+		to_rdma_ah_attr(dev, &amp;qp_attr-&gt;alt_ah_attr, alt_path);</div><div class='add'>+		qp_attr-&gt;alt_pkey_index = MLX5_GET(ads, alt_path, pkey_index);</div><div class='add'>+		qp_attr-&gt;alt_port_num = MLX5_GET(ads, alt_path, vhca_port_num);</div><div class='add'>+	}</div><div class='ctx'> </div><div class='del'>-	qp_attr-&gt;max_dest_rd_atomic =</div><div class='del'>-		1 &lt;&lt; ((be32_to_cpu(context-&gt;params2) &gt;&gt; 21) &amp; 0x7);</div><div class='del'>-	qp_attr-&gt;min_rnr_timer	    =</div><div class='del'>-		(be32_to_cpu(context-&gt;rnr_nextrecvpsn) &gt;&gt; 24) &amp; 0x1f;</div><div class='del'>-	qp_attr-&gt;timeout	    = context-&gt;pri_path.ackto_lt &gt;&gt; 3;</div><div class='del'>-	qp_attr-&gt;retry_cnt	    = (be32_to_cpu(context-&gt;params1) &gt;&gt; 16) &amp; 0x7;</div><div class='del'>-	qp_attr-&gt;rnr_retry	    = (be32_to_cpu(context-&gt;params1) &gt;&gt; 13) &amp; 0x7;</div><div class='del'>-	qp_attr-&gt;alt_timeout	    = context-&gt;alt_path.ackto_lt &gt;&gt; 3;</div><div class='add'>+	qp_attr-&gt;pkey_index = MLX5_GET(ads, pri_path, pkey_index);</div><div class='add'>+	qp_attr-&gt;port_num = MLX5_GET(ads, pri_path, vhca_port_num);</div><div class='add'>+	qp_attr-&gt;timeout = MLX5_GET(ads, pri_path, ack_timeout);</div><div class='add'>+	qp_attr-&gt;alt_timeout = MLX5_GET(ads, alt_path, ack_timeout);</div><div class='ctx'> </div><div class='ctx'> out:</div><div class='ctx'> 	kfree(outb);</div><div class='hunk'>@@ -5587,7 +5057,7 @@ static int mlx5_ib_dct_query_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *mqp,</div><div class='ctx'> 	if (!out)</div><div class='ctx'> 		return -ENOMEM;</div><div class='ctx'> </div><div class='del'>-	err = mlx5_core_dct_query(dev-&gt;mdev, dct, out, outlen);</div><div class='add'>+	err = mlx5_core_dct_query(dev, dct, out, outlen);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='hunk'>@@ -5607,7 +5077,7 @@ static int mlx5_ib_dct_query_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *mqp,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (qp_attr_mask &amp; IB_QP_PORT)</div><div class='del'>-		qp_attr-&gt;port_num = MLX5_GET(dctc, dctc, port);</div><div class='add'>+		qp_attr-&gt;port_num = mqp-&gt;port;</div><div class='ctx'> 	if (qp_attr_mask &amp; IB_QP_MIN_RNR_TIMER)</div><div class='ctx'> 		qp_attr-&gt;min_rnr_timer = MLX5_GET(dctc, dctc, min_rnr_nak);</div><div class='ctx'> 	if (qp_attr_mask &amp; IB_QP_AV) {</div><div class='hunk'>@@ -5636,7 +5106,7 @@ int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 	if (ibqp-&gt;rwq_ind_tbl)</div><div class='ctx'> 		return -ENOSYS;</div><div class='ctx'> </div><div class='del'>-	if (unlikely(ibqp-&gt;qp_type == IB_QPT_GSI))</div><div class='add'>+	if (qp-&gt;type == IB_QPT_GSI)</div><div class='ctx'> 		return mlx5_ib_gsi_query_qp(ibqp, qp_attr, qp_attr_mask,</div><div class='ctx'> 					    qp_init_attr);</div><div class='ctx'> </div><div class='hunk'>@@ -5644,14 +5114,14 @@ int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 	memset(qp_init_attr, 0, sizeof(*qp_init_attr));</div><div class='ctx'> 	memset(qp_attr, 0, sizeof(*qp_attr));</div><div class='ctx'> </div><div class='del'>-	if (unlikely(qp-&gt;qp_sub_type == MLX5_IB_QPT_DCT))</div><div class='add'>+	if (unlikely(qp-&gt;type == MLX5_IB_QPT_DCT))</div><div class='ctx'> 		return mlx5_ib_dct_query_qp(dev, qp, qp_attr,</div><div class='ctx'> 					    qp_attr_mask, qp_init_attr);</div><div class='ctx'> </div><div class='ctx'> 	mutex_lock(&amp;qp-&gt;mutex);</div><div class='ctx'> </div><div class='del'>-	if (qp-&gt;ibqp.qp_type == IB_QPT_RAW_PACKET ||</div><div class='del'>-	    qp-&gt;flags &amp; MLX5_IB_QP_UNDERLAY) {</div><div class='add'>+	if (qp-&gt;type == IB_QPT_RAW_PACKET ||</div><div class='add'>+	    qp-&gt;flags &amp; IB_QP_CREATE_SOURCE_QPN) {</div><div class='ctx'> 		err = query_raw_packet_qp_state(dev, qp, &amp;raw_packet_qp_state);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto out;</div><div class='hunk'>@@ -5677,7 +5147,7 @@ int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> 		qp_attr-&gt;cap.max_send_sge = 0;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	qp_init_attr-&gt;qp_type = ibqp-&gt;qp_type;</div><div class='add'>+	qp_init_attr-&gt;qp_type = qp-&gt;type;</div><div class='ctx'> 	qp_init_attr-&gt;recv_cq = ibqp-&gt;recv_cq;</div><div class='ctx'> 	qp_init_attr-&gt;send_cq = ibqp-&gt;send_cq;</div><div class='ctx'> 	qp_init_attr-&gt;srq = ibqp-&gt;srq;</div><div class='hunk'>@@ -5685,18 +5155,7 @@ int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,</div><div class='ctx'> </div><div class='ctx'> 	qp_init_attr-&gt;cap	     = qp_attr-&gt;cap;</div><div class='ctx'> </div><div class='del'>-	qp_init_attr-&gt;create_flags = 0;</div><div class='del'>-	if (qp-&gt;flags &amp; MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)</div><div class='del'>-		qp_init_attr-&gt;create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;</div><div class='del'>-</div><div class='del'>-	if (qp-&gt;flags &amp; MLX5_IB_QP_CROSS_CHANNEL)</div><div class='del'>-		qp_init_attr-&gt;create_flags |= IB_QP_CREATE_CROSS_CHANNEL;</div><div class='del'>-	if (qp-&gt;flags &amp; MLX5_IB_QP_MANAGED_SEND)</div><div class='del'>-		qp_init_attr-&gt;create_flags |= IB_QP_CREATE_MANAGED_SEND;</div><div class='del'>-	if (qp-&gt;flags &amp; MLX5_IB_QP_MANAGED_RECV)</div><div class='del'>-		qp_init_attr-&gt;create_flags |= IB_QP_CREATE_MANAGED_RECV;</div><div class='del'>-	if (qp-&gt;flags &amp; MLX5_IB_QP_SQPN_QP1)</div><div class='del'>-		qp_init_attr-&gt;create_flags |= mlx5_ib_create_qp_sqpn_qp1();</div><div class='add'>+	qp_init_attr-&gt;create_flags = qp-&gt;flags;</div><div class='ctx'> </div><div class='ctx'> 	qp_init_attr-&gt;sq_sig_type = qp-&gt;sq_signal_bits &amp; MLX5_WQE_CTRL_CQ_UPDATE ?</div><div class='ctx'> 		IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;</div><div class='hunk'>@@ -5706,41 +5165,23 @@ out:</div><div class='ctx'> 	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,</div><div class='del'>-				   struct ib_udata *udata)</div><div class='add'>+int mlx5_ib_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(ibdev);</div><div class='del'>-	struct mlx5_ib_xrcd *xrcd;</div><div class='del'>-	int err;</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ibxrcd-&gt;device);</div><div class='add'>+	struct mlx5_ib_xrcd *xrcd = to_mxrcd(ibxrcd);</div><div class='ctx'> </div><div class='ctx'> 	if (!MLX5_CAP_GEN(dev-&gt;mdev, xrc))</div><div class='del'>-		return ERR_PTR(-ENOSYS);</div><div class='del'>-</div><div class='del'>-	xrcd = kmalloc(sizeof(*xrcd), GFP_KERNEL);</div><div class='del'>-	if (!xrcd)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-</div><div class='del'>-	err = mlx5_cmd_xrcd_alloc(dev-&gt;mdev, &amp;xrcd-&gt;xrcdn, 0);</div><div class='del'>-	if (err) {</div><div class='del'>-		kfree(xrcd);</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='del'>-	}</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='del'>-	return &amp;xrcd-&gt;ibxrcd;</div><div class='add'>+	return mlx5_cmd_xrcd_alloc(dev-&gt;mdev, &amp;xrcd-&gt;xrcdn, 0);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(xrcd-&gt;device);</div><div class='ctx'> 	u32 xrcdn = to_mxrcd(xrcd)-&gt;xrcdn;</div><div class='del'>-	int err;</div><div class='ctx'> </div><div class='del'>-	err = mlx5_cmd_xrcd_dealloc(dev-&gt;mdev, xrcdn, 0);</div><div class='del'>-	if (err)</div><div class='del'>-		mlx5_ib_warn(dev, "failed to dealloc xrcdn 0x%x\n", xrcdn);</div><div class='del'>-</div><div class='del'>-	kfree(xrcd);</div><div class='del'>-	return 0;</div><div class='add'>+	return mlx5_cmd_xrcd_dealloc(dev-&gt;mdev, xrcdn, 0);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)</div><div class='hunk'>@@ -5773,7 +5214,7 @@ static int set_delay_drop(struct mlx5_ib_dev *dev)</div><div class='ctx'> 	if (dev-&gt;delay_drop.activate)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='del'>-	err = mlx5_core_set_delay_drop(dev-&gt;mdev, dev-&gt;delay_drop.timeout);</div><div class='add'>+	err = mlx5_core_set_delay_drop(dev, dev-&gt;delay_drop.timeout);</div><div class='ctx'> 	if (err)</div><div class='ctx'> 		goto out;</div><div class='ctx'> </div><div class='hunk'>@@ -5792,6 +5233,7 @@ static int  create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,</div><div class='ctx'> 	struct mlx5_ib_dev *dev;</div><div class='ctx'> 	int has_net_offloads;</div><div class='ctx'> 	__be64 *rq_pas0;</div><div class='add'>+	int ts_format;</div><div class='ctx'> 	void *in;</div><div class='ctx'> 	void *rqc;</div><div class='ctx'> 	void *wq;</div><div class='hunk'>@@ -5800,6 +5242,10 @@ static int  create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,</div><div class='ctx'> </div><div class='ctx'> 	dev = to_mdev(pd-&gt;device);</div><div class='ctx'> </div><div class='add'>+	ts_format = get_rq_ts_format(dev, to_mcq(init_attr-&gt;cq));</div><div class='add'>+	if (ts_format &lt; 0)</div><div class='add'>+		return ts_format;</div><div class='add'>+</div><div class='ctx'> 	inlen = MLX5_ST_SZ_BYTES(create_rq_in) + sizeof(u64) * rwq-&gt;rq_num_pas;</div><div class='ctx'> 	in = kvzalloc(inlen, GFP_KERNEL);</div><div class='ctx'> 	if (!in)</div><div class='hunk'>@@ -5809,6 +5255,7 @@ static int  create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,</div><div class='ctx'> 	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);</div><div class='ctx'> 	MLX5_SET(rqc,  rqc, mem_rq_type,</div><div class='ctx'> 		 MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);</div><div class='add'>+	MLX5_SET(rqc, rqc, ts_format, ts_format);</div><div class='ctx'> 	MLX5_SET(rqc, rqc, user_index, rwq-&gt;user_index);</div><div class='ctx'> 	MLX5_SET(rqc,  rqc, cqn, to_mcq(init_attr-&gt;cq)-&gt;mcq.cqn);</div><div class='ctx'> 	MLX5_SET(rqc,  rqc, state, MLX5_RQC_STATE_RST);</div><div class='hunk'>@@ -5828,12 +5275,21 @@ static int  create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,</div><div class='ctx'> 	}</div><div class='ctx'> 	MLX5_SET(wq, wq, log_wq_stride, rwq-&gt;log_rq_stride);</div><div class='ctx'> 	if (rwq-&gt;create_flags &amp; MLX5_IB_WQ_FLAGS_STRIDING_RQ) {</div><div class='add'>+		/*</div><div class='add'>+		 * In Firmware number of strides in each WQE is:</div><div class='add'>+		 *   "512 * 2^single_wqe_log_num_of_strides"</div><div class='add'>+		 * Values 3 to 8 are accepted as 10 to 15, 9 to 18 are</div><div class='add'>+		 * accepted as 0 to 9</div><div class='add'>+		 */</div><div class='add'>+		static const u8 fw_map[] = { 10, 11, 12, 13, 14, 15, 0, 1,</div><div class='add'>+					     2,  3,  4,  5,  6,  7,  8, 9 };</div><div class='ctx'> 		MLX5_SET(wq, wq, two_byte_shift_en, rwq-&gt;two_byte_shift_en);</div><div class='ctx'> 		MLX5_SET(wq, wq, log_wqe_stride_size,</div><div class='ctx'> 			 rwq-&gt;single_stride_log_num_of_bytes -</div><div class='ctx'> 			 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES);</div><div class='del'>-		MLX5_SET(wq, wq, log_wqe_num_of_strides, rwq-&gt;log_num_strides -</div><div class='del'>-			 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES);</div><div class='add'>+		MLX5_SET(wq, wq, log_wqe_num_of_strides,</div><div class='add'>+			 fw_map[rwq-&gt;log_num_strides -</div><div class='add'>+				MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES]);</div><div class='ctx'> 	}</div><div class='ctx'> 	MLX5_SET(wq, wq, log_wq_sz, rwq-&gt;log_rq_size);</div><div class='ctx'> 	MLX5_SET(wq, wq, pd, to_mpd(pd)-&gt;pdn);</div><div class='hunk'>@@ -5869,14 +5325,14 @@ static int  create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,</div><div class='ctx'> 		MLX5_SET(rqc, rqc, delay_drop_en, 1);</div><div class='ctx'> 	}</div><div class='ctx'> 	rq_pas0 = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);</div><div class='del'>-	mlx5_ib_populate_pas(dev, rwq-&gt;umem, rwq-&gt;page_shift, rq_pas0, 0);</div><div class='del'>-	err = mlx5_core_create_rq_tracked(dev-&gt;mdev, in, inlen, &amp;rwq-&gt;core_qp);</div><div class='add'>+	mlx5_ib_populate_pas(rwq-&gt;umem, 1UL &lt;&lt; rwq-&gt;page_shift, rq_pas0, 0);</div><div class='add'>+	err = mlx5_core_create_rq_tracked(dev, in, inlen, &amp;rwq-&gt;core_qp);</div><div class='ctx'> 	if (!err &amp;&amp; init_attr-&gt;create_flags &amp; IB_WQ_FLAGS_DELAY_DROP) {</div><div class='ctx'> 		err = set_delay_drop(dev);</div><div class='ctx'> 		if (err) {</div><div class='ctx'> 			mlx5_ib_warn(dev, "Failed to enable delay drop err=%d\n",</div><div class='ctx'> 				     err);</div><div class='del'>-			mlx5_core_destroy_rq_tracked(dev-&gt;mdev, &amp;rwq-&gt;core_qp);</div><div class='add'>+			mlx5_core_destroy_rq_tracked(dev, &amp;rwq-&gt;core_qp);</div><div class='ctx'> 		} else {</div><div class='ctx'> 			rwq-&gt;create_flags |= MLX5_IB_WQ_FLAGS_DELAY_DROP;</div><div class='ctx'> 		}</div><div class='hunk'>@@ -5908,6 +5364,19 @@ static int set_user_rq_size(struct mlx5_ib_dev *dev,</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='add'>+static bool log_of_strides_valid(struct mlx5_ib_dev *dev, u32 log_num_strides)</div><div class='add'>+{</div><div class='add'>+	if ((log_num_strides &gt; MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES) ||</div><div class='add'>+	    (log_num_strides &lt; MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES))</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	if (!MLX5_CAP_GEN(dev-&gt;mdev, ext_stride_num_range) &amp;&amp;</div><div class='add'>+	    (log_num_strides &lt; MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES))</div><div class='add'>+		return false;</div><div class='add'>+</div><div class='add'>+	return true;</div><div class='add'>+}</div><div class='add'>+</div><div class='ctx'> static int prepare_user_rq(struct ib_pd *pd,</div><div class='ctx'> 			   struct ib_wq_init_attr *init_attr,</div><div class='ctx'> 			   struct ib_udata *udata,</div><div class='hunk'>@@ -5918,8 +5387,8 @@ static int prepare_user_rq(struct ib_pd *pd,</div><div class='ctx'> 	int err;</div><div class='ctx'> 	size_t required_cmd_sz;</div><div class='ctx'> </div><div class='del'>-	required_cmd_sz = offsetof(typeof(ucmd), single_stride_log_num_of_bytes)</div><div class='del'>-		+ sizeof(ucmd.single_stride_log_num_of_bytes);</div><div class='add'>+	required_cmd_sz = offsetofend(struct mlx5_ib_create_wq,</div><div class='add'>+				      single_stride_log_num_of_bytes);</div><div class='ctx'> 	if (udata-&gt;inlen &lt; required_cmd_sz) {</div><div class='ctx'> 		mlx5_ib_dbg(dev, "invalid inlen\n");</div><div class='ctx'> 		return -EINVAL;</div><div class='hunk'>@@ -5955,14 +5424,16 @@ static int prepare_user_rq(struct ib_pd *pd,</div><div class='ctx'> 				    MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES);</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 		}</div><div class='del'>-		if ((ucmd.single_wqe_log_num_of_strides &gt;</div><div class='del'>-		    MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES) ||</div><div class='del'>-		     (ucmd.single_wqe_log_num_of_strides &lt;</div><div class='del'>-			MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES)) {</div><div class='del'>-			mlx5_ib_dbg(dev, "Invalid log num strides (%u. Range is %u - %u)\n",</div><div class='del'>-				    ucmd.single_wqe_log_num_of_strides,</div><div class='del'>-				    MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES,</div><div class='del'>-				    MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES);</div><div class='add'>+		if (!log_of_strides_valid(dev,</div><div class='add'>+					  ucmd.single_wqe_log_num_of_strides)) {</div><div class='add'>+			mlx5_ib_dbg(</div><div class='add'>+				dev,</div><div class='add'>+				"Invalid log num strides (%u. Range is %u - %u)\n",</div><div class='add'>+				ucmd.single_wqe_log_num_of_strides,</div><div class='add'>+				MLX5_CAP_GEN(dev-&gt;mdev, ext_stride_num_range) ?</div><div class='add'>+					MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES :</div><div class='add'>+					MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES,</div><div class='add'>+				MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES);</div><div class='ctx'> 			return -EINVAL;</div><div class='ctx'> 		}</div><div class='ctx'> 		rwq-&gt;single_stride_log_num_of_bytes =</div><div class='hunk'>@@ -6001,10 +5472,14 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,</div><div class='ctx'> 	if (!udata)</div><div class='ctx'> 		return ERR_PTR(-ENOSYS);</div><div class='ctx'> </div><div class='del'>-	min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);</div><div class='add'>+	min_resp_len = offsetofend(struct mlx5_ib_create_wq_resp, reserved);</div><div class='ctx'> 	if (udata-&gt;outlen &amp;&amp; udata-&gt;outlen &lt; min_resp_len)</div><div class='ctx'> 		return ERR_PTR(-EINVAL);</div><div class='ctx'> </div><div class='add'>+	if (!capable(CAP_SYS_RAWIO) &amp;&amp;</div><div class='add'>+	    init_attr-&gt;create_flags &amp; IB_WQ_FLAGS_DELAY_DROP)</div><div class='add'>+		return ERR_PTR(-EPERM);</div><div class='add'>+</div><div class='ctx'> 	dev = to_mdev(pd-&gt;device);</div><div class='ctx'> 	switch (init_attr-&gt;wq_type) {</div><div class='ctx'> 	case IB_WQT_RQ:</div><div class='hunk'>@@ -6027,8 +5502,8 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,</div><div class='ctx'> 	rwq-&gt;ibwq.wq_num = rwq-&gt;core_qp.qpn;</div><div class='ctx'> 	rwq-&gt;ibwq.state = IB_WQS_RESET;</div><div class='ctx'> 	if (udata-&gt;outlen) {</div><div class='del'>-		resp.response_length = offsetof(typeof(resp), response_length) +</div><div class='del'>-				sizeof(resp.response_length);</div><div class='add'>+		resp.response_length = offsetofend(</div><div class='add'>+			struct mlx5_ib_create_wq_resp, response_length);</div><div class='ctx'> 		err = ib_copy_to_udata(udata, &amp;resp, resp.response_length);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto err_copy;</div><div class='hunk'>@@ -6039,7 +5514,7 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,</div><div class='ctx'> 	return &amp;rwq-&gt;ibwq;</div><div class='ctx'> </div><div class='ctx'> err_copy:</div><div class='del'>-	mlx5_core_destroy_rq_tracked(dev-&gt;mdev, &amp;rwq-&gt;core_qp);</div><div class='add'>+	mlx5_core_destroy_rq_tracked(dev, &amp;rwq-&gt;core_qp);</div><div class='ctx'> err_user_rq:</div><div class='ctx'> 	destroy_user_rq(dev, pd, rwq, udata);</div><div class='ctx'> err:</div><div class='hunk'>@@ -6051,20 +5526,23 @@ int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(wq-&gt;device);</div><div class='ctx'> 	struct mlx5_ib_rwq *rwq = to_mrwq(wq);</div><div class='add'>+	int ret;</div><div class='ctx'> </div><div class='del'>-	mlx5_core_destroy_rq_tracked(dev-&gt;mdev, &amp;rwq-&gt;core_qp);</div><div class='add'>+	ret = mlx5_core_destroy_rq_tracked(dev, &amp;rwq-&gt;core_qp);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> 	destroy_user_rq(dev, wq-&gt;pd, rwq, udata);</div><div class='ctx'> 	kfree(rwq);</div><div class='del'>-</div><div class='ctx'> 	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,</div><div class='del'>-						      struct ib_rwq_ind_table_init_attr *init_attr,</div><div class='del'>-						      struct ib_udata *udata)</div><div class='add'>+int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,</div><div class='add'>+				 struct ib_rwq_ind_table_init_attr *init_attr,</div><div class='add'>+				 struct ib_udata *udata)</div><div class='ctx'> {</div><div class='del'>-	struct mlx5_ib_dev *dev = to_mdev(device);</div><div class='del'>-	struct mlx5_ib_rwq_ind_table *rwq_ind_tbl;</div><div class='add'>+	struct mlx5_ib_rwq_ind_table *rwq_ind_tbl =</div><div class='add'>+		to_mrwq_ind_table(ib_rwq_ind_table);</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_table-&gt;device);</div><div class='ctx'> 	int sz = 1 &lt;&lt; init_attr-&gt;log_ind_tbl_size;</div><div class='ctx'> 	struct mlx5_ib_create_rwq_ind_tbl_resp resp = {};</div><div class='ctx'> 	size_t min_resp_len;</div><div class='hunk'>@@ -6077,30 +5555,25 @@ struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,</div><div class='ctx'> 	if (udata-&gt;inlen &gt; 0 &amp;&amp;</div><div class='ctx'> 	    !ib_is_udata_cleared(udata, 0,</div><div class='ctx'> 				 udata-&gt;inlen))</div><div class='del'>-		return ERR_PTR(-EOPNOTSUPP);</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='ctx'> </div><div class='ctx'> 	if (init_attr-&gt;log_ind_tbl_size &gt;</div><div class='ctx'> 	    MLX5_CAP_GEN(dev-&gt;mdev, log_max_rqt_size)) {</div><div class='ctx'> 		mlx5_ib_dbg(dev, "log_ind_tbl_size = %d is bigger than supported = %d\n",</div><div class='ctx'> 			    init_attr-&gt;log_ind_tbl_size,</div><div class='ctx'> 			    MLX5_CAP_GEN(dev-&gt;mdev, log_max_rqt_size));</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);</div><div class='add'>+	min_resp_len =</div><div class='add'>+		offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp, reserved);</div><div class='ctx'> 	if (udata-&gt;outlen &amp;&amp; udata-&gt;outlen &lt; min_resp_len)</div><div class='del'>-		return ERR_PTR(-EINVAL);</div><div class='del'>-</div><div class='del'>-	rwq_ind_tbl = kzalloc(sizeof(*rwq_ind_tbl), GFP_KERNEL);</div><div class='del'>-	if (!rwq_ind_tbl)</div><div class='del'>-		return ERR_PTR(-ENOMEM);</div><div class='add'>+		return -EINVAL;</div><div class='ctx'> </div><div class='ctx'> 	inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;</div><div class='ctx'> 	in = kvzalloc(inlen, GFP_KERNEL);</div><div class='del'>-	if (!in) {</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto err;</div><div class='del'>-	}</div><div class='add'>+	if (!in)</div><div class='add'>+		return -ENOMEM;</div><div class='ctx'> </div><div class='ctx'> 	rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);</div><div class='ctx'> </div><div class='hunk'>@@ -6115,26 +5588,24 @@ struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,</div><div class='ctx'> </div><div class='ctx'> 	err = mlx5_core_create_rqt(dev-&gt;mdev, in, inlen, &amp;rwq_ind_tbl-&gt;rqtn);</div><div class='ctx'> 	kvfree(in);</div><div class='del'>-</div><div class='ctx'> 	if (err)</div><div class='del'>-		goto err;</div><div class='add'>+		return err;</div><div class='ctx'> </div><div class='ctx'> 	rwq_ind_tbl-&gt;ib_rwq_ind_tbl.ind_tbl_num = rwq_ind_tbl-&gt;rqtn;</div><div class='ctx'> 	if (udata-&gt;outlen) {</div><div class='del'>-		resp.response_length = offsetof(typeof(resp), response_length) +</div><div class='del'>-					sizeof(resp.response_length);</div><div class='add'>+		resp.response_length =</div><div class='add'>+			offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp,</div><div class='add'>+				    response_length);</div><div class='ctx'> 		err = ib_copy_to_udata(udata, &amp;resp, resp.response_length);</div><div class='ctx'> 		if (err)</div><div class='ctx'> 			goto err_copy;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	return &amp;rwq_ind_tbl-&gt;ib_rwq_ind_tbl;</div><div class='add'>+	return 0;</div><div class='ctx'> </div><div class='ctx'> err_copy:</div><div class='ctx'> 	mlx5_cmd_destroy_rqt(dev-&gt;mdev, rwq_ind_tbl-&gt;rqtn, rwq_ind_tbl-&gt;uid);</div><div class='del'>-err:</div><div class='del'>-	kfree(rwq_ind_tbl);</div><div class='del'>-	return ERR_PTR(err);</div><div class='add'>+	return err;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)</div><div class='hunk'>@@ -6142,10 +5613,7 @@ int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)</div><div class='ctx'> 	struct mlx5_ib_rwq_ind_table *rwq_ind_tbl = to_mrwq_ind_table(ib_rwq_ind_tbl);</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_tbl-&gt;device);</div><div class='ctx'> </div><div class='del'>-	mlx5_cmd_destroy_rqt(dev-&gt;mdev, rwq_ind_tbl-&gt;rqtn, rwq_ind_tbl-&gt;uid);</div><div class='del'>-</div><div class='del'>-	kfree(rwq_ind_tbl);</div><div class='del'>-	return 0;</div><div class='add'>+	return mlx5_cmd_destroy_rqt(dev-&gt;mdev, rwq_ind_tbl-&gt;rqtn, rwq_ind_tbl-&gt;uid);</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,</div><div class='hunk'>@@ -6162,7 +5630,7 @@ int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,</div><div class='ctx'> 	void *rqc;</div><div class='ctx'> 	void *in;</div><div class='ctx'> </div><div class='del'>-	required_cmd_sz = offsetof(typeof(ucmd), reserved) + sizeof(ucmd.reserved);</div><div class='add'>+	required_cmd_sz = offsetofend(struct mlx5_ib_modify_wq, reserved);</div><div class='ctx'> 	if (udata-&gt;inlen &lt; required_cmd_sz)</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> </div><div class='hunk'>@@ -6184,10 +5652,8 @@ int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,</div><div class='ctx'> </div><div class='ctx'> 	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);</div><div class='ctx'> </div><div class='del'>-	curr_wq_state = (wq_attr_mask &amp; IB_WQ_CUR_STATE) ?</div><div class='del'>-		wq_attr-&gt;curr_wq_state : wq-&gt;state;</div><div class='del'>-	wq_state = (wq_attr_mask &amp; IB_WQ_STATE) ?</div><div class='del'>-		wq_attr-&gt;wq_state : curr_wq_state;</div><div class='add'>+	curr_wq_state = wq_attr-&gt;curr_wq_state;</div><div class='add'>+	wq_state = wq_attr-&gt;wq_state;</div><div class='ctx'> 	if (curr_wq_state == IB_WQS_ERR)</div><div class='ctx'> 		curr_wq_state = MLX5_RQC_STATE_ERR;</div><div class='ctx'> 	if (wq_state == IB_WQS_ERR)</div><div class='hunk'>@@ -6200,8 +5666,7 @@ int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,</div><div class='ctx'> 		if (wq_attr-&gt;flags_mask &amp; IB_WQ_FLAGS_CVLAN_STRIPPING) {</div><div class='ctx'> 			if (!(MLX5_CAP_GEN(dev-&gt;mdev, eth_net_offloads) &amp;&amp;</div><div class='ctx'> 			      MLX5_CAP_ETH(dev-&gt;mdev, vlan_cap))) {</div><div class='del'>-				mlx5_ib_dbg(dev, "VLAN offloads are not "</div><div class='del'>-					    "supported\n");</div><div class='add'>+				mlx5_ib_dbg(dev, "VLAN offloads are not supported\n");</div><div class='ctx'> 				err = -EOPNOTSUPP;</div><div class='ctx'> 				goto out;</div><div class='ctx'> 			}</div><div class='hunk'>@@ -6219,18 +5684,20 @@ int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	if (curr_wq_state == IB_WQS_RESET &amp;&amp; wq_state == IB_WQS_RDY) {</div><div class='add'>+		u16 set_id;</div><div class='add'>+</div><div class='add'>+		set_id = mlx5_ib_get_counters_id(dev, 0);</div><div class='ctx'> 		if (MLX5_CAP_GEN(dev-&gt;mdev, modify_rq_counter_set_id)) {</div><div class='ctx'> 			MLX5_SET64(modify_rq_in, in, modify_bitmask,</div><div class='ctx'> 				   MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);</div><div class='del'>-			MLX5_SET(rqc, rqc, counter_set_id,</div><div class='del'>-				 dev-&gt;port-&gt;cnts.set_id);</div><div class='add'>+			MLX5_SET(rqc, rqc, counter_set_id, set_id);</div><div class='ctx'> 		} else</div><div class='ctx'> 			dev_info_once(</div><div class='ctx'> 				&amp;dev-&gt;ib_dev.dev,</div><div class='ctx'> 				"Receive WQ counters are not supported on current FW\n");</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	err = mlx5_core_modify_rq(dev-&gt;mdev, rwq-&gt;core_qp.qpn, in, inlen);</div><div class='add'>+	err = mlx5_core_modify_rq(dev-&gt;mdev, rwq-&gt;core_qp.qpn, in);</div><div class='ctx'> 	if (!err)</div><div class='ctx'> 		rwq-&gt;ibwq.state = (wq_state == MLX5_RQC_STATE_ERR) ? IB_WQS_ERR : wq_state;</div><div class='ctx'> </div><div class='hunk'>@@ -6297,7 +5764,7 @@ static void handle_drain_completion(struct ib_cq *cq,</div><div class='ctx'> 		/* Run the CQ handler - this makes sure that the drain WR will</div><div class='ctx'> 		 * be processed if wasn't processed yet.</div><div class='ctx'> 		 */</div><div class='del'>-		mcq-&gt;mcq.comp(&amp;mcq-&gt;mcq);</div><div class='add'>+		mcq-&gt;mcq.comp(&amp;mcq-&gt;mcq, NULL);</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='ctx'> 	wait_for_completion(&amp;sdrain-&gt;done);</div><div class='hunk'>@@ -6329,7 +5796,7 @@ void mlx5_ib_drain_sq(struct ib_qp *qp)</div><div class='ctx'> 	sdrain.cqe.done = mlx5_ib_drain_qp_done;</div><div class='ctx'> 	init_completion(&amp;sdrain.done);</div><div class='ctx'> </div><div class='del'>-	ret = _mlx5_ib_post_send(qp, &amp;swr.wr, &amp;bad_swr, true);</div><div class='add'>+	ret = mlx5_ib_post_send_drain(qp, &amp;swr.wr, &amp;bad_swr);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);</div><div class='ctx'> 		return;</div><div class='hunk'>@@ -6359,7 +5826,7 @@ void mlx5_ib_drain_rq(struct ib_qp *qp)</div><div class='ctx'> 	rdrain.cqe.done = mlx5_ib_drain_qp_done;</div><div class='ctx'> 	init_completion(&amp;rdrain.done);</div><div class='ctx'> </div><div class='del'>-	ret = _mlx5_ib_post_recv(qp, &amp;rwr, &amp;bad_rwr, true);</div><div class='add'>+	ret = mlx5_ib_post_recv_drain(qp, &amp;rwr, &amp;bad_rwr);</div><div class='ctx'> 	if (ret) {</div><div class='ctx'> 		WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);</div><div class='ctx'> 		return;</div><div class='hunk'>@@ -6367,3 +5834,54 @@ void mlx5_ib_drain_rq(struct ib_qp *qp)</div><div class='ctx'> </div><div class='ctx'> 	handle_drain_completion(cq, &amp;rdrain, dev);</div><div class='ctx'> }</div><div class='add'>+</div><div class='add'>+/*</div><div class='add'>+ * Bind a qp to a counter. If @counter is NULL then bind the qp to</div><div class='add'>+ * the default counter</div><div class='add'>+ */</div><div class='add'>+int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(qp-&gt;device);</div><div class='add'>+	struct mlx5_ib_qp *mqp = to_mqp(qp);</div><div class='add'>+	int err = 0;</div><div class='add'>+</div><div class='add'>+	mutex_lock(&amp;mqp-&gt;mutex);</div><div class='add'>+	if (mqp-&gt;state == IB_QPS_RESET) {</div><div class='add'>+		qp-&gt;counter = counter;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (!MLX5_CAP_GEN(dev-&gt;mdev, rts2rts_qp_counters_set_id)) {</div><div class='add'>+		err = -EOPNOTSUPP;</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	if (mqp-&gt;state == IB_QPS_RTS) {</div><div class='add'>+		err = __mlx5_ib_qp_set_counter(qp, counter);</div><div class='add'>+		if (!err)</div><div class='add'>+			qp-&gt;counter = counter;</div><div class='add'>+</div><div class='add'>+		goto out;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	mqp-&gt;counter_pending = 1;</div><div class='add'>+	qp-&gt;counter = counter;</div><div class='add'>+</div><div class='add'>+out:</div><div class='add'>+	mutex_unlock(&amp;mqp-&gt;mutex);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_ib_qp_event_init(void)</div><div class='add'>+{</div><div class='add'>+	mlx5_ib_qp_event_wq = alloc_ordered_workqueue("mlx5_ib_qp_event_wq", 0);</div><div class='add'>+	if (!mlx5_ib_qp_event_wq)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_ib_qp_event_cleanup(void)</div><div class='add'>+{</div><div class='add'>+	destroy_workqueue(mlx5_ib_qp_event_wq);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/qp.h b/drivers/infiniband/hw/mlx5/qp.h<br/>new file mode 100644<br/>index 000000000000..2530e7730635<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/qp.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/qp.h</a></div><div class='hunk'>@@ -0,0 +1,60 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef _MLX5_IB_QP_H</div><div class='add'>+#define _MLX5_IB_QP_H</div><div class='add'>+</div><div class='add'>+struct mlx5_ib_dev;</div><div class='add'>+</div><div class='add'>+struct mlx5_qp_table {</div><div class='add'>+	struct notifier_block nb;</div><div class='add'>+	struct xarray dct_xa;</div><div class='add'>+</div><div class='add'>+	/* protect radix tree</div><div class='add'>+	 */</div><div class='add'>+	spinlock_t lock;</div><div class='add'>+	struct radix_tree_root tree;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+int mlx5_init_qp_table(struct mlx5_ib_dev *dev);</div><div class='add'>+void mlx5_cleanup_qp_table(struct mlx5_ib_dev *dev);</div><div class='add'>+</div><div class='add'>+int mlx5_core_create_dct(struct mlx5_ib_dev *dev, struct mlx5_core_dct *qp,</div><div class='add'>+			 u32 *in, int inlen, u32 *out, int outlen);</div><div class='add'>+int mlx5_qpc_create_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,</div><div class='add'>+		       u32 *in, int inlen, u32 *out);</div><div class='add'>+int mlx5_core_qp_modify(struct mlx5_ib_dev *dev, u16 opcode, u32 opt_param_mask,</div><div class='add'>+			void *qpc, struct mlx5_core_qp *qp, u32 *ece);</div><div class='add'>+int mlx5_core_destroy_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp);</div><div class='add'>+int mlx5_core_destroy_dct(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct);</div><div class='add'>+int mlx5_core_qp_query(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,</div><div class='add'>+		       u32 *out, int outlen, bool qpc_ext);</div><div class='add'>+int mlx5_core_dct_query(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct,</div><div class='add'>+			u32 *out, int outlen);</div><div class='add'>+</div><div class='add'>+int mlx5_core_set_delay_drop(struct mlx5_ib_dev *dev, u32 timeout_usec);</div><div class='add'>+</div><div class='add'>+int mlx5_core_destroy_rq_tracked(struct mlx5_ib_dev *dev,</div><div class='add'>+				 struct mlx5_core_qp *rq);</div><div class='add'>+int mlx5_core_create_sq_tracked(struct mlx5_ib_dev *dev, u32 *in, int inlen,</div><div class='add'>+				struct mlx5_core_qp *sq);</div><div class='add'>+void mlx5_core_destroy_sq_tracked(struct mlx5_ib_dev *dev,</div><div class='add'>+				  struct mlx5_core_qp *sq);</div><div class='add'>+</div><div class='add'>+int mlx5_core_create_rq_tracked(struct mlx5_ib_dev *dev, u32 *in, int inlen,</div><div class='add'>+				struct mlx5_core_qp *rq);</div><div class='add'>+</div><div class='add'>+struct mlx5_core_rsc_common *mlx5_core_res_hold(struct mlx5_ib_dev *dev,</div><div class='add'>+						int res_num,</div><div class='add'>+						enum mlx5_res_type res_type);</div><div class='add'>+void mlx5_core_res_put(struct mlx5_core_rsc_common *res);</div><div class='add'>+</div><div class='add'>+int mlx5_core_xrcd_alloc(struct mlx5_ib_dev *dev, u32 *xrcdn);</div><div class='add'>+int mlx5_core_xrcd_dealloc(struct mlx5_ib_dev *dev, u32 xrcdn);</div><div class='add'>+int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter);</div><div class='add'>+int mlx5_ib_qp_event_init(void);</div><div class='add'>+void mlx5_ib_qp_event_cleanup(void);</div><div class='add'>+int mlx5r_ib_rate(struct mlx5_ib_dev *dev, u8 rate);</div><div class='add'>+#endif /* _MLX5_IB_QP_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/qpc.c b/drivers/infiniband/hw/mlx5/qpc.c<br/>new file mode 100644<br/>index 000000000000..146d03ae40bd<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/qpc.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/qpc.c</a></div><div class='hunk'>@@ -0,0 +1,697 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;linux/gfp.h&gt;</div><div class='add'>+#include &lt;linux/mlx5/qp.h&gt;</div><div class='add'>+#include &lt;linux/mlx5/driver.h&gt;</div><div class='add'>+#include "mlx5_ib.h"</div><div class='add'>+#include "qp.h"</div><div class='add'>+</div><div class='add'>+static int mlx5_core_drain_dct(struct mlx5_ib_dev *dev,</div><div class='add'>+			       struct mlx5_core_dct *dct);</div><div class='add'>+</div><div class='add'>+static struct mlx5_core_rsc_common *</div><div class='add'>+mlx5_get_rsc(struct mlx5_qp_table *table, u32 rsn)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_core_rsc_common *common;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;table-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	common = radix_tree_lookup(&amp;table-&gt;tree, rsn);</div><div class='add'>+	if (common &amp;&amp; !common-&gt;invalid)</div><div class='add'>+		refcount_inc(&amp;common-&gt;refcount);</div><div class='add'>+	else</div><div class='add'>+		common = NULL;</div><div class='add'>+</div><div class='add'>+	spin_unlock_irqrestore(&amp;table-&gt;lock, flags);</div><div class='add'>+</div><div class='add'>+	return common;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_core_put_rsc(struct mlx5_core_rsc_common *common)</div><div class='add'>+{</div><div class='add'>+	if (refcount_dec_and_test(&amp;common-&gt;refcount))</div><div class='add'>+		complete(&amp;common-&gt;free);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u64 qp_allowed_event_types(void)</div><div class='add'>+{</div><div class='add'>+	u64 mask;</div><div class='add'>+</div><div class='add'>+	mask = BIT(MLX5_EVENT_TYPE_PATH_MIG) |</div><div class='add'>+	       BIT(MLX5_EVENT_TYPE_COMM_EST) |</div><div class='add'>+	       BIT(MLX5_EVENT_TYPE_SQ_DRAINED) |</div><div class='add'>+	       BIT(MLX5_EVENT_TYPE_SRQ_LAST_WQE) |</div><div class='add'>+	       BIT(MLX5_EVENT_TYPE_WQ_CATAS_ERROR) |</div><div class='add'>+	       BIT(MLX5_EVENT_TYPE_PATH_MIG_FAILED) |</div><div class='add'>+	       BIT(MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR) |</div><div class='add'>+	       BIT(MLX5_EVENT_TYPE_WQ_ACCESS_ERROR);</div><div class='add'>+</div><div class='add'>+	return mask;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u64 rq_allowed_event_types(void)</div><div class='add'>+{</div><div class='add'>+	u64 mask;</div><div class='add'>+</div><div class='add'>+	mask = BIT(MLX5_EVENT_TYPE_SRQ_LAST_WQE) |</div><div class='add'>+	       BIT(MLX5_EVENT_TYPE_WQ_CATAS_ERROR);</div><div class='add'>+</div><div class='add'>+	return mask;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u64 sq_allowed_event_types(void)</div><div class='add'>+{</div><div class='add'>+	return BIT(MLX5_EVENT_TYPE_WQ_CATAS_ERROR);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static u64 dct_allowed_event_types(void)</div><div class='add'>+{</div><div class='add'>+	return BIT(MLX5_EVENT_TYPE_DCT_DRAINED);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static bool is_event_type_allowed(int rsc_type, int event_type)</div><div class='add'>+{</div><div class='add'>+	switch (rsc_type) {</div><div class='add'>+	case MLX5_EVENT_QUEUE_TYPE_QP:</div><div class='add'>+		return BIT(event_type) &amp; qp_allowed_event_types();</div><div class='add'>+	case MLX5_EVENT_QUEUE_TYPE_RQ:</div><div class='add'>+		return BIT(event_type) &amp; rq_allowed_event_types();</div><div class='add'>+	case MLX5_EVENT_QUEUE_TYPE_SQ:</div><div class='add'>+		return BIT(event_type) &amp; sq_allowed_event_types();</div><div class='add'>+	case MLX5_EVENT_QUEUE_TYPE_DCT:</div><div class='add'>+		return BIT(event_type) &amp; dct_allowed_event_types();</div><div class='add'>+	default:</div><div class='add'>+		WARN(1, "Event arrived for unknown resource type");</div><div class='add'>+		return false;</div><div class='add'>+	}</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int dct_event_notifier(struct mlx5_ib_dev *dev, struct mlx5_eqe *eqe)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_core_dct *dct;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+	u32 qpn;</div><div class='add'>+</div><div class='add'>+	qpn = be32_to_cpu(eqe-&gt;data.dct.dctn) &amp; 0xFFFFFF;</div><div class='add'>+	xa_lock_irqsave(&amp;dev-&gt;qp_table.dct_xa, flags);</div><div class='add'>+	dct = xa_load(&amp;dev-&gt;qp_table.dct_xa, qpn);</div><div class='add'>+	if (dct)</div><div class='add'>+		complete(&amp;dct-&gt;drained);</div><div class='add'>+	xa_unlock_irqrestore(&amp;dev-&gt;qp_table.dct_xa, flags);</div><div class='add'>+	return NOTIFY_OK;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int rsc_event_notifier(struct notifier_block *nb,</div><div class='add'>+			      unsigned long type, void *data)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev =</div><div class='add'>+		container_of(nb, struct mlx5_ib_dev, qp_table.nb);</div><div class='add'>+	struct mlx5_core_rsc_common *common;</div><div class='add'>+	struct mlx5_eqe *eqe = data;</div><div class='add'>+	u8 event_type = (u8)type;</div><div class='add'>+	struct mlx5_core_qp *qp;</div><div class='add'>+	u32 rsn;</div><div class='add'>+</div><div class='add'>+	switch (event_type) {</div><div class='add'>+	case MLX5_EVENT_TYPE_DCT_DRAINED:</div><div class='add'>+		return dct_event_notifier(dev, eqe);</div><div class='add'>+	case MLX5_EVENT_TYPE_PATH_MIG:</div><div class='add'>+	case MLX5_EVENT_TYPE_COMM_EST:</div><div class='add'>+	case MLX5_EVENT_TYPE_SQ_DRAINED:</div><div class='add'>+	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:</div><div class='add'>+	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:</div><div class='add'>+	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:</div><div class='add'>+	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:</div><div class='add'>+	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:</div><div class='add'>+		rsn = be32_to_cpu(eqe-&gt;data.qp_srq.qp_srq_n) &amp; 0xffffff;</div><div class='add'>+		rsn |= (eqe-&gt;data.qp_srq.type &lt;&lt; MLX5_USER_INDEX_LEN);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return NOTIFY_DONE;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	common = mlx5_get_rsc(&amp;dev-&gt;qp_table, rsn);</div><div class='add'>+	if (!common)</div><div class='add'>+		return NOTIFY_OK;</div><div class='add'>+</div><div class='add'>+	if (!is_event_type_allowed((rsn &gt;&gt; MLX5_USER_INDEX_LEN), event_type))</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	switch (common-&gt;res) {</div><div class='add'>+	case MLX5_RES_QP:</div><div class='add'>+	case MLX5_RES_RQ:</div><div class='add'>+	case MLX5_RES_SQ:</div><div class='add'>+		qp = (struct mlx5_core_qp *)common;</div><div class='add'>+		qp-&gt;event(qp, event_type);</div><div class='add'>+		/* Need to put resource in event handler */</div><div class='add'>+		return NOTIFY_OK;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+out:</div><div class='add'>+	mlx5_core_put_rsc(common);</div><div class='add'>+</div><div class='add'>+	return NOTIFY_OK;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int create_resource_common(struct mlx5_ib_dev *dev,</div><div class='add'>+				  struct mlx5_core_qp *qp, int rsc_type)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_qp_table *table = &amp;dev-&gt;qp_table;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	qp-&gt;common.res = rsc_type;</div><div class='add'>+	spin_lock_irq(&amp;table-&gt;lock);</div><div class='add'>+	err = radix_tree_insert(&amp;table-&gt;tree,</div><div class='add'>+				qp-&gt;qpn | (rsc_type &lt;&lt; MLX5_USER_INDEX_LEN),</div><div class='add'>+				qp);</div><div class='add'>+	spin_unlock_irq(&amp;table-&gt;lock);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	refcount_set(&amp;qp-&gt;common.refcount, 1);</div><div class='add'>+	init_completion(&amp;qp-&gt;common.free);</div><div class='add'>+	qp-&gt;pid = current-&gt;pid;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void modify_resource_common_state(struct mlx5_ib_dev *dev,</div><div class='add'>+					 struct mlx5_core_qp *qp,</div><div class='add'>+					 bool invalid)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_qp_table *table = &amp;dev-&gt;qp_table;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;table-&gt;lock, flags);</div><div class='add'>+	qp-&gt;common.invalid = invalid;</div><div class='add'>+	spin_unlock_irqrestore(&amp;table-&gt;lock, flags);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void destroy_resource_common(struct mlx5_ib_dev *dev,</div><div class='add'>+				    struct mlx5_core_qp *qp)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_qp_table *table = &amp;dev-&gt;qp_table;</div><div class='add'>+	unsigned long flags;</div><div class='add'>+</div><div class='add'>+	spin_lock_irqsave(&amp;table-&gt;lock, flags);</div><div class='add'>+	radix_tree_delete(&amp;table-&gt;tree,</div><div class='add'>+			  qp-&gt;qpn | (qp-&gt;common.res &lt;&lt; MLX5_USER_INDEX_LEN));</div><div class='add'>+	spin_unlock_irqrestore(&amp;table-&gt;lock, flags);</div><div class='add'>+	mlx5_core_put_rsc((struct mlx5_core_rsc_common *)qp);</div><div class='add'>+	wait_for_completion(&amp;qp-&gt;common.free);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int _mlx5_core_destroy_dct(struct mlx5_ib_dev *dev,</div><div class='add'>+				  struct mlx5_core_dct *dct)</div><div class='add'>+{</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(destroy_dct_in)] = {};</div><div class='add'>+	struct mlx5_core_qp *qp = &amp;dct-&gt;mqp;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(destroy_dct_in, in, opcode, MLX5_CMD_OP_DESTROY_DCT);</div><div class='add'>+	MLX5_SET(destroy_dct_in, in, dctn, qp-&gt;qpn);</div><div class='add'>+	MLX5_SET(destroy_dct_in, in, uid, qp-&gt;uid);</div><div class='add'>+	return mlx5_cmd_exec_in(dev-&gt;mdev, destroy_dct, in);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_core_create_dct(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct,</div><div class='add'>+			 u32 *in, int inlen, u32 *out, int outlen)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_core_qp *qp = &amp;dct-&gt;mqp;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	init_completion(&amp;dct-&gt;drained);</div><div class='add'>+	MLX5_SET(create_dct_in, in, opcode, MLX5_CMD_OP_CREATE_DCT);</div><div class='add'>+</div><div class='add'>+	err = mlx5_cmd_do(dev-&gt;mdev, in, inlen, out, outlen);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	qp-&gt;qpn = MLX5_GET(create_dct_out, out, dctn);</div><div class='add'>+	qp-&gt;uid = MLX5_GET(create_dct_in, in, uid);</div><div class='add'>+	err = xa_err(xa_store_irq(&amp;dev-&gt;qp_table.dct_xa, qp-&gt;qpn, dct, GFP_KERNEL));</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_cmd;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+err_cmd:</div><div class='add'>+	_mlx5_core_destroy_dct(dev, dct);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_qpc_create_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,</div><div class='add'>+		       u32 *in, int inlen, u32 *out)</div><div class='add'>+{</div><div class='add'>+	u32 din[MLX5_ST_SZ_DW(destroy_qp_in)] = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);</div><div class='add'>+</div><div class='add'>+	err = mlx5_cmd_exec(dev-&gt;mdev, in, inlen, out,</div><div class='add'>+			    MLX5_ST_SZ_BYTES(create_qp_out));</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	qp-&gt;uid = MLX5_GET(create_qp_in, in, uid);</div><div class='add'>+	qp-&gt;qpn = MLX5_GET(create_qp_out, out, qpn);</div><div class='add'>+</div><div class='add'>+	err = create_resource_common(dev, qp, MLX5_RES_QP);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_cmd;</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;ib_dev.type != RDMA_DEVICE_TYPE_SMI)</div><div class='add'>+		mlx5_debug_qp_add(dev-&gt;mdev, qp);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_cmd:</div><div class='add'>+	MLX5_SET(destroy_qp_in, din, opcode, MLX5_CMD_OP_DESTROY_QP);</div><div class='add'>+	MLX5_SET(destroy_qp_in, din, qpn, qp-&gt;qpn);</div><div class='add'>+	MLX5_SET(destroy_qp_in, din, uid, qp-&gt;uid);</div><div class='add'>+	mlx5_cmd_exec_in(dev-&gt;mdev, destroy_qp, din);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int mlx5_core_drain_dct(struct mlx5_ib_dev *dev,</div><div class='add'>+			       struct mlx5_core_dct *dct)</div><div class='add'>+{</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(drain_dct_in)] = {};</div><div class='add'>+	struct mlx5_core_qp *qp = &amp;dct-&gt;mqp;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(drain_dct_in, in, opcode, MLX5_CMD_OP_DRAIN_DCT);</div><div class='add'>+	MLX5_SET(drain_dct_in, in, dctn, qp-&gt;qpn);</div><div class='add'>+	MLX5_SET(drain_dct_in, in, uid, qp-&gt;uid);</div><div class='add'>+	return mlx5_cmd_exec_in(dev-&gt;mdev, drain_dct, in);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_core_destroy_dct(struct mlx5_ib_dev *dev,</div><div class='add'>+			  struct mlx5_core_dct *dct)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_qp_table *table = &amp;dev-&gt;qp_table;</div><div class='add'>+	struct mlx5_core_dct *tmp;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	err = mlx5_core_drain_dct(dev, dct);</div><div class='add'>+	if (err) {</div><div class='add'>+		if (dev-&gt;mdev-&gt;state == MLX5_DEVICE_STATE_INTERNAL_ERROR)</div><div class='add'>+			goto destroy;</div><div class='add'>+</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+	wait_for_completion(&amp;dct-&gt;drained);</div><div class='add'>+</div><div class='add'>+destroy:</div><div class='add'>+	tmp = xa_cmpxchg_irq(&amp;table-&gt;dct_xa, dct-&gt;mqp.qpn, dct, XA_ZERO_ENTRY, GFP_KERNEL);</div><div class='add'>+	if (WARN_ON(tmp != dct))</div><div class='add'>+		return xa_err(tmp) ?: -EINVAL;</div><div class='add'>+</div><div class='add'>+	err = _mlx5_core_destroy_dct(dev, dct);</div><div class='add'>+	if (err) {</div><div class='add'>+		xa_cmpxchg_irq(&amp;table-&gt;dct_xa, dct-&gt;mqp.qpn, XA_ZERO_ENTRY, dct, 0);</div><div class='add'>+		return err;</div><div class='add'>+	}</div><div class='add'>+	xa_erase_irq(&amp;table-&gt;dct_xa, dct-&gt;mqp.qpn);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_core_destroy_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp)</div><div class='add'>+{</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {};</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;ib_dev.type != RDMA_DEVICE_TYPE_SMI)</div><div class='add'>+		mlx5_debug_qp_remove(dev-&gt;mdev, qp);</div><div class='add'>+</div><div class='add'>+	destroy_resource_common(dev, qp);</div><div class='add'>+</div><div class='add'>+	MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP);</div><div class='add'>+	MLX5_SET(destroy_qp_in, in, qpn, qp-&gt;qpn);</div><div class='add'>+	MLX5_SET(destroy_qp_in, in, uid, qp-&gt;uid);</div><div class='add'>+	return mlx5_cmd_exec_in(dev-&gt;mdev, destroy_qp, in);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_core_set_delay_drop(struct mlx5_ib_dev *dev,</div><div class='add'>+			     u32 timeout_usec)</div><div class='add'>+{</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(set_delay_drop_params_in)] = {};</div><div class='add'>+</div><div class='add'>+	MLX5_SET(set_delay_drop_params_in, in, opcode,</div><div class='add'>+		 MLX5_CMD_OP_SET_DELAY_DROP_PARAMS);</div><div class='add'>+	MLX5_SET(set_delay_drop_params_in, in, delay_drop_timeout,</div><div class='add'>+		 timeout_usec / 100);</div><div class='add'>+	return mlx5_cmd_exec_in(dev-&gt;mdev, set_delay_drop_params, in);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct mbox_info {</div><div class='add'>+	u32 *in;</div><div class='add'>+	u32 *out;</div><div class='add'>+	int inlen;</div><div class='add'>+	int outlen;</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+static int mbox_alloc(struct mbox_info *mbox, int inlen, int outlen)</div><div class='add'>+{</div><div class='add'>+	mbox-&gt;inlen  = inlen;</div><div class='add'>+	mbox-&gt;outlen = outlen;</div><div class='add'>+	mbox-&gt;in = kzalloc(mbox-&gt;inlen, GFP_KERNEL);</div><div class='add'>+	mbox-&gt;out = kzalloc(mbox-&gt;outlen, GFP_KERNEL);</div><div class='add'>+	if (!mbox-&gt;in || !mbox-&gt;out) {</div><div class='add'>+		kfree(mbox-&gt;in);</div><div class='add'>+		kfree(mbox-&gt;out);</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void mbox_free(struct mbox_info *mbox)</div><div class='add'>+{</div><div class='add'>+	kfree(mbox-&gt;in);</div><div class='add'>+	kfree(mbox-&gt;out);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int get_ece_from_mbox(void *out, u16 opcode)</div><div class='add'>+{</div><div class='add'>+	int ece = 0;</div><div class='add'>+</div><div class='add'>+	switch (opcode) {</div><div class='add'>+	case MLX5_CMD_OP_INIT2INIT_QP:</div><div class='add'>+		ece = MLX5_GET(init2init_qp_out, out, ece);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_CMD_OP_INIT2RTR_QP:</div><div class='add'>+		ece = MLX5_GET(init2rtr_qp_out, out, ece);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_CMD_OP_RTR2RTS_QP:</div><div class='add'>+		ece = MLX5_GET(rtr2rts_qp_out, out, ece);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_CMD_OP_RTS2RTS_QP:</div><div class='add'>+		ece = MLX5_GET(rts2rts_qp_out, out, ece);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_CMD_OP_RST2INIT_QP:</div><div class='add'>+		ece = MLX5_GET(rst2init_qp_out, out, ece);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		break;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	return ece;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int modify_qp_mbox_alloc(struct mlx5_core_dev *dev, u16 opcode, int qpn,</div><div class='add'>+				u32 opt_param_mask, void *qpc,</div><div class='add'>+				struct mbox_info *mbox, u16 uid, u32 ece)</div><div class='add'>+{</div><div class='add'>+	mbox-&gt;out = NULL;</div><div class='add'>+	mbox-&gt;in = NULL;</div><div class='add'>+</div><div class='add'>+#define MBOX_ALLOC(mbox, typ)  \</div><div class='add'>+	mbox_alloc(mbox, MLX5_ST_SZ_BYTES(typ##_in), MLX5_ST_SZ_BYTES(typ##_out))</div><div class='add'>+</div><div class='add'>+#define MOD_QP_IN_SET(typ, in, _opcode, _qpn, _uid)                            \</div><div class='add'>+	do {                                                                   \</div><div class='add'>+		MLX5_SET(typ##_in, in, opcode, _opcode);                       \</div><div class='add'>+		MLX5_SET(typ##_in, in, qpn, _qpn);                             \</div><div class='add'>+		MLX5_SET(typ##_in, in, uid, _uid);                             \</div><div class='add'>+	} while (0)</div><div class='add'>+</div><div class='add'>+#define MOD_QP_IN_SET_QPC(typ, in, _opcode, _qpn, _opt_p, _qpc, _uid)          \</div><div class='add'>+	do {                                                                   \</div><div class='add'>+		MOD_QP_IN_SET(typ, in, _opcode, _qpn, _uid);                   \</div><div class='add'>+		MLX5_SET(typ##_in, in, opt_param_mask, _opt_p);                \</div><div class='add'>+		memcpy(MLX5_ADDR_OF(typ##_in, in, qpc), _qpc,                  \</div><div class='add'>+		       MLX5_ST_SZ_BYTES(qpc));                                 \</div><div class='add'>+	} while (0)</div><div class='add'>+</div><div class='add'>+	switch (opcode) {</div><div class='add'>+	/* 2RST &amp; 2ERR */</div><div class='add'>+	case MLX5_CMD_OP_2RST_QP:</div><div class='add'>+		if (MBOX_ALLOC(mbox, qp_2rst))</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		MOD_QP_IN_SET(qp_2rst, mbox-&gt;in, opcode, qpn, uid);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_CMD_OP_2ERR_QP:</div><div class='add'>+		if (MBOX_ALLOC(mbox, qp_2err))</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		MOD_QP_IN_SET(qp_2err, mbox-&gt;in, opcode, qpn, uid);</div><div class='add'>+		break;</div><div class='add'>+</div><div class='add'>+	/* MODIFY with QPC */</div><div class='add'>+	case MLX5_CMD_OP_RST2INIT_QP:</div><div class='add'>+		if (MBOX_ALLOC(mbox, rst2init_qp))</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		MOD_QP_IN_SET_QPC(rst2init_qp, mbox-&gt;in, opcode, qpn,</div><div class='add'>+				  opt_param_mask, qpc, uid);</div><div class='add'>+		MLX5_SET(rst2init_qp_in, mbox-&gt;in, ece, ece);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_CMD_OP_INIT2RTR_QP:</div><div class='add'>+		if (MBOX_ALLOC(mbox, init2rtr_qp))</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		MOD_QP_IN_SET_QPC(init2rtr_qp, mbox-&gt;in, opcode, qpn,</div><div class='add'>+				  opt_param_mask, qpc, uid);</div><div class='add'>+		MLX5_SET(init2rtr_qp_in, mbox-&gt;in, ece, ece);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_CMD_OP_RTR2RTS_QP:</div><div class='add'>+		if (MBOX_ALLOC(mbox, rtr2rts_qp))</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		MOD_QP_IN_SET_QPC(rtr2rts_qp, mbox-&gt;in, opcode, qpn,</div><div class='add'>+				  opt_param_mask, qpc, uid);</div><div class='add'>+		MLX5_SET(rtr2rts_qp_in, mbox-&gt;in, ece, ece);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_CMD_OP_RTS2RTS_QP:</div><div class='add'>+		if (MBOX_ALLOC(mbox, rts2rts_qp))</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		MOD_QP_IN_SET_QPC(rts2rts_qp, mbox-&gt;in, opcode, qpn,</div><div class='add'>+				  opt_param_mask, qpc, uid);</div><div class='add'>+		MLX5_SET(rts2rts_qp_in, mbox-&gt;in, ece, ece);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_CMD_OP_SQERR2RTS_QP:</div><div class='add'>+		if (MBOX_ALLOC(mbox, sqerr2rts_qp))</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		MOD_QP_IN_SET_QPC(sqerr2rts_qp, mbox-&gt;in, opcode, qpn,</div><div class='add'>+				  opt_param_mask, qpc, uid);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_CMD_OP_SQD_RTS_QP:</div><div class='add'>+		if (MBOX_ALLOC(mbox, sqd2rts_qp))</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		MOD_QP_IN_SET_QPC(sqd2rts_qp, mbox-&gt;in, opcode, qpn,</div><div class='add'>+				  opt_param_mask, qpc, uid);</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_CMD_OP_INIT2INIT_QP:</div><div class='add'>+		if (MBOX_ALLOC(mbox, init2init_qp))</div><div class='add'>+			return -ENOMEM;</div><div class='add'>+		MOD_QP_IN_SET_QPC(init2init_qp, mbox-&gt;in, opcode, qpn,</div><div class='add'>+				  opt_param_mask, qpc, uid);</div><div class='add'>+		MLX5_SET(init2init_qp_in, mbox-&gt;in, ece, ece);</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return -EINVAL;</div><div class='add'>+	}</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_core_qp_modify(struct mlx5_ib_dev *dev, u16 opcode, u32 opt_param_mask,</div><div class='add'>+			void *qpc, struct mlx5_core_qp *qp, u32 *ece)</div><div class='add'>+{</div><div class='add'>+	struct mbox_info mbox;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	err = modify_qp_mbox_alloc(dev-&gt;mdev, opcode, qp-&gt;qpn, opt_param_mask,</div><div class='add'>+				   qpc, &amp;mbox, qp-&gt;uid, (ece) ? *ece : 0);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	err = mlx5_cmd_exec(dev-&gt;mdev, mbox.in, mbox.inlen, mbox.out,</div><div class='add'>+			    mbox.outlen);</div><div class='add'>+</div><div class='add'>+	if (ece)</div><div class='add'>+		*ece = get_ece_from_mbox(mbox.out, opcode);</div><div class='add'>+</div><div class='add'>+	mbox_free(&amp;mbox);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_init_qp_table(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_qp_table *table = &amp;dev-&gt;qp_table;</div><div class='add'>+</div><div class='add'>+	spin_lock_init(&amp;table-&gt;lock);</div><div class='add'>+	INIT_RADIX_TREE(&amp;table-&gt;tree, GFP_ATOMIC);</div><div class='add'>+	xa_init(&amp;table-&gt;dct_xa);</div><div class='add'>+</div><div class='add'>+	if (dev-&gt;ib_dev.type != RDMA_DEVICE_TYPE_SMI)</div><div class='add'>+		mlx5_qp_debugfs_init(dev-&gt;mdev);</div><div class='add'>+</div><div class='add'>+	table-&gt;nb.notifier_call = rsc_event_notifier;</div><div class='add'>+	mlx5_notifier_register(dev-&gt;mdev, &amp;table-&gt;nb);</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_cleanup_qp_table(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_qp_table *table = &amp;dev-&gt;qp_table;</div><div class='add'>+</div><div class='add'>+	mlx5_notifier_unregister(dev-&gt;mdev, &amp;table-&gt;nb);</div><div class='add'>+	if (dev-&gt;ib_dev.type != RDMA_DEVICE_TYPE_SMI)</div><div class='add'>+		mlx5_qp_debugfs_cleanup(dev-&gt;mdev);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_core_qp_query(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,</div><div class='add'>+		       u32 *out, int outlen, bool qpc_ext)</div><div class='add'>+{</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(query_qp_in)] = {};</div><div class='add'>+</div><div class='add'>+	MLX5_SET(query_qp_in, in, opcode, MLX5_CMD_OP_QUERY_QP);</div><div class='add'>+	MLX5_SET(query_qp_in, in, qpn, qp-&gt;qpn);</div><div class='add'>+	MLX5_SET(query_qp_in, in, qpc_ext, qpc_ext);</div><div class='add'>+</div><div class='add'>+	return mlx5_cmd_exec(dev-&gt;mdev, in, sizeof(in), out, outlen);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_core_dct_query(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct,</div><div class='add'>+			u32 *out, int outlen)</div><div class='add'>+{</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(query_dct_in)] = {};</div><div class='add'>+	struct mlx5_core_qp *qp = &amp;dct-&gt;mqp;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(query_dct_in, in, opcode, MLX5_CMD_OP_QUERY_DCT);</div><div class='add'>+	MLX5_SET(query_dct_in, in, dctn, qp-&gt;qpn);</div><div class='add'>+</div><div class='add'>+	return mlx5_cmd_exec(dev-&gt;mdev, (void *)&amp;in, sizeof(in), (void *)out,</div><div class='add'>+			     outlen);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_core_xrcd_alloc(struct mlx5_ib_dev *dev, u32 *xrcdn)</div><div class='add'>+{</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(alloc_xrcd_out)] = {};</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(alloc_xrcd_in)] = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(alloc_xrcd_in, in, opcode, MLX5_CMD_OP_ALLOC_XRCD);</div><div class='add'>+	err = mlx5_cmd_exec_inout(dev-&gt;mdev, alloc_xrcd, in, out);</div><div class='add'>+	if (!err)</div><div class='add'>+		*xrcdn = MLX5_GET(alloc_xrcd_out, out, xrcd);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_core_xrcd_dealloc(struct mlx5_ib_dev *dev, u32 xrcdn)</div><div class='add'>+{</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(dealloc_xrcd_in)] = {};</div><div class='add'>+</div><div class='add'>+	MLX5_SET(dealloc_xrcd_in, in, opcode, MLX5_CMD_OP_DEALLOC_XRCD);</div><div class='add'>+	MLX5_SET(dealloc_xrcd_in, in, xrcd, xrcdn);</div><div class='add'>+	return mlx5_cmd_exec_in(dev-&gt;mdev, dealloc_xrcd, in);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int destroy_rq_tracked(struct mlx5_ib_dev *dev, u32 rqn, u16 uid)</div><div class='add'>+{</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(destroy_rq_in)] = {};</div><div class='add'>+</div><div class='add'>+	MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ);</div><div class='add'>+	MLX5_SET(destroy_rq_in, in, rqn, rqn);</div><div class='add'>+	MLX5_SET(destroy_rq_in, in, uid, uid);</div><div class='add'>+	return mlx5_cmd_exec_in(dev-&gt;mdev, destroy_rq, in);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_core_create_rq_tracked(struct mlx5_ib_dev *dev, u32 *in, int inlen,</div><div class='add'>+				struct mlx5_core_qp *rq)</div><div class='add'>+{</div><div class='add'>+	int err;</div><div class='add'>+	u32 rqn;</div><div class='add'>+</div><div class='add'>+	err = mlx5_core_create_rq(dev-&gt;mdev, in, inlen, &amp;rqn);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	rq-&gt;uid = MLX5_GET(create_rq_in, in, uid);</div><div class='add'>+	rq-&gt;qpn = rqn;</div><div class='add'>+	err = create_resource_common(dev, rq, MLX5_RES_RQ);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_destroy_rq;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_destroy_rq:</div><div class='add'>+	destroy_rq_tracked(dev, rq-&gt;qpn, rq-&gt;uid);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_core_destroy_rq_tracked(struct mlx5_ib_dev *dev,</div><div class='add'>+				 struct mlx5_core_qp *rq)</div><div class='add'>+{</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	/* The rq destruction can be called again in case it fails, hence we</div><div class='add'>+	 * mark the common resource as invalid and only once FW destruction</div><div class='add'>+	 * is completed successfully we actually destroy the resources.</div><div class='add'>+	 */</div><div class='add'>+	modify_resource_common_state(dev, rq, true);</div><div class='add'>+	ret = destroy_rq_tracked(dev, rq-&gt;qpn, rq-&gt;uid);</div><div class='add'>+	if (ret) {</div><div class='add'>+		modify_resource_common_state(dev, rq, false);</div><div class='add'>+		return ret;</div><div class='add'>+	}</div><div class='add'>+	destroy_resource_common(dev, rq);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static void destroy_sq_tracked(struct mlx5_ib_dev *dev, u32 sqn, u16 uid)</div><div class='add'>+{</div><div class='add'>+	u32 in[MLX5_ST_SZ_DW(destroy_sq_in)] = {};</div><div class='add'>+</div><div class='add'>+	MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ);</div><div class='add'>+	MLX5_SET(destroy_sq_in, in, sqn, sqn);</div><div class='add'>+	MLX5_SET(destroy_sq_in, in, uid, uid);</div><div class='add'>+	mlx5_cmd_exec_in(dev-&gt;mdev, destroy_sq, in);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+int mlx5_core_create_sq_tracked(struct mlx5_ib_dev *dev, u32 *in, int inlen,</div><div class='add'>+				struct mlx5_core_qp *sq)</div><div class='add'>+{</div><div class='add'>+	u32 out[MLX5_ST_SZ_DW(create_sq_out)] = {};</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ);</div><div class='add'>+	err = mlx5_cmd_exec(dev-&gt;mdev, in, inlen, out, sizeof(out));</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='add'>+	sq-&gt;qpn = MLX5_GET(create_sq_out, out, sqn);</div><div class='add'>+	sq-&gt;uid = MLX5_GET(create_sq_in, in, uid);</div><div class='add'>+	err = create_resource_common(dev, sq, MLX5_RES_SQ);</div><div class='add'>+	if (err)</div><div class='add'>+		goto err_destroy_sq;</div><div class='add'>+</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_destroy_sq:</div><div class='add'>+	destroy_sq_tracked(dev, sq-&gt;qpn, sq-&gt;uid);</div><div class='add'>+</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_core_destroy_sq_tracked(struct mlx5_ib_dev *dev,</div><div class='add'>+				  struct mlx5_core_qp *sq)</div><div class='add'>+{</div><div class='add'>+	destroy_resource_common(dev, sq);</div><div class='add'>+	destroy_sq_tracked(dev, sq-&gt;qpn, sq-&gt;uid);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+struct mlx5_core_rsc_common *mlx5_core_res_hold(struct mlx5_ib_dev *dev,</div><div class='add'>+						int res_num,</div><div class='add'>+						enum mlx5_res_type res_type)</div><div class='add'>+{</div><div class='add'>+	u32 rsn = res_num | (res_type &lt;&lt; MLX5_USER_INDEX_LEN);</div><div class='add'>+	struct mlx5_qp_table *table = &amp;dev-&gt;qp_table;</div><div class='add'>+</div><div class='add'>+	return mlx5_get_rsc(table, rsn);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+void mlx5_core_res_put(struct mlx5_core_rsc_common *res)</div><div class='add'>+{</div><div class='add'>+	mlx5_core_put_rsc(res);</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/restrack.c b/drivers/infiniband/hw/mlx5/restrack.c<br/>new file mode 100644<br/>index 000000000000..67841922c7b8<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/restrack.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/restrack.c</a></div><div class='hunk'>@@ -0,0 +1,217 @@</div><div class='add'>+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2019-2020, Mellanox Technologies Ltd. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#include &lt;uapi/rdma/rdma_netlink.h&gt;</div><div class='add'>+#include &lt;linux/mlx5/rsc_dump.h&gt;</div><div class='add'>+#include &lt;rdma/ib_umem_odp.h&gt;</div><div class='add'>+#include &lt;rdma/restrack.h&gt;</div><div class='add'>+#include "mlx5_ib.h"</div><div class='add'>+#include "restrack.h"</div><div class='add'>+</div><div class='add'>+#define MAX_DUMP_SIZE 1024</div><div class='add'>+</div><div class='add'>+static int dump_rsc(struct mlx5_core_dev *dev, enum mlx5_sgmt_type type,</div><div class='add'>+		    int index, void *data, int *data_len)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_core_dev *mdev = dev;</div><div class='add'>+	struct mlx5_rsc_dump_cmd *cmd;</div><div class='add'>+	struct mlx5_rsc_key key = {};</div><div class='add'>+	struct page *page;</div><div class='add'>+	int offset = 0;</div><div class='add'>+	int err = 0;</div><div class='add'>+	int cmd_err;</div><div class='add'>+	int size;</div><div class='add'>+</div><div class='add'>+	page = alloc_page(GFP_KERNEL);</div><div class='add'>+	if (!page)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	key.size = PAGE_SIZE;</div><div class='add'>+	key.rsc = type;</div><div class='add'>+	key.index1 = index;</div><div class='add'>+	key.num_of_obj1 = 1;</div><div class='add'>+</div><div class='add'>+	cmd = mlx5_rsc_dump_cmd_create(mdev, &amp;key);</div><div class='add'>+	if (IS_ERR(cmd)) {</div><div class='add'>+		err = PTR_ERR(cmd);</div><div class='add'>+		goto free_page;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	do {</div><div class='add'>+		cmd_err = mlx5_rsc_dump_next(mdev, cmd, page, &amp;size);</div><div class='add'>+		if (cmd_err &lt; 0 || size + offset &gt; MAX_DUMP_SIZE) {</div><div class='add'>+			err = cmd_err;</div><div class='add'>+			goto destroy_cmd;</div><div class='add'>+		}</div><div class='add'>+		memcpy(data + offset, page_address(page), size);</div><div class='add'>+		offset += size;</div><div class='add'>+	} while (cmd_err &gt; 0);</div><div class='add'>+	*data_len = offset;</div><div class='add'>+</div><div class='add'>+destroy_cmd:</div><div class='add'>+	mlx5_rsc_dump_cmd_destroy(cmd);</div><div class='add'>+free_page:</div><div class='add'>+	__free_page(page);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_res_raw(struct sk_buff *msg, struct mlx5_ib_dev *dev,</div><div class='add'>+			enum mlx5_sgmt_type type, u32 key)</div><div class='add'>+{</div><div class='add'>+	int len = 0;</div><div class='add'>+	void *data;</div><div class='add'>+	int err;</div><div class='add'>+</div><div class='add'>+	data = kzalloc(MAX_DUMP_SIZE, GFP_KERNEL);</div><div class='add'>+	if (!data)</div><div class='add'>+		return -ENOMEM;</div><div class='add'>+</div><div class='add'>+	err = dump_rsc(dev-&gt;mdev, type, key, data, &amp;len);</div><div class='add'>+	if (err)</div><div class='add'>+		goto out;</div><div class='add'>+</div><div class='add'>+	err = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, len, data);</div><div class='add'>+out:</div><div class='add'>+	kfree(data);</div><div class='add'>+	return err;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_stat_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mr *mr = to_mmr(ibmr);</div><div class='add'>+	struct nlattr *table_attr;</div><div class='add'>+</div><div class='add'>+	if (!(mr-&gt;access_flags &amp; IB_ACCESS_ON_DEMAND))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	table_attr = nla_nest_start(msg,</div><div class='add'>+				    RDMA_NLDEV_ATTR_STAT_HWCOUNTERS);</div><div class='add'>+</div><div class='add'>+	if (!table_attr)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	if (rdma_nl_stat_hwcounter_entry(msg, "page_faults",</div><div class='add'>+					 atomic64_read(&amp;mr-&gt;odp_stats.faults)))</div><div class='add'>+		goto err_table;</div><div class='add'>+	if (rdma_nl_stat_hwcounter_entry(</div><div class='add'>+		    msg, "page_faults_handled",</div><div class='add'>+		    atomic64_read(&amp;mr-&gt;odp_stats.faults_handled)))</div><div class='add'>+		goto err_table;</div><div class='add'>+	if (rdma_nl_stat_hwcounter_entry(</div><div class='add'>+		    msg, "page_invalidations",</div><div class='add'>+		    atomic64_read(&amp;mr-&gt;odp_stats.invalidations)))</div><div class='add'>+		goto err_table;</div><div class='add'>+	if (rdma_nl_stat_hwcounter_entry(</div><div class='add'>+		    msg, "page_invalidations_handled",</div><div class='add'>+		    atomic64_read(&amp;mr-&gt;odp_stats.invalidations_handled)))</div><div class='add'>+		goto err_table;</div><div class='add'>+</div><div class='add'>+	if (rdma_nl_stat_hwcounter_entry(msg, "page_prefetch",</div><div class='add'>+					 atomic64_read(&amp;mr-&gt;odp_stats.prefetch)))</div><div class='add'>+		goto err_table;</div><div class='add'>+</div><div class='add'>+	nla_nest_end(msg, table_attr);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err_table:</div><div class='add'>+	nla_nest_cancel(msg, table_attr);</div><div class='add'>+err:</div><div class='add'>+	return -EMSGSIZE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ibmr)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mr *mr = to_mmr(ibmr);</div><div class='add'>+</div><div class='add'>+	return fill_res_raw(msg, mr_to_mdev(mr), MLX5_SGMT_TYPE_PRM_QUERY_MKEY,</div><div class='add'>+			    mlx5_mkey_to_idx(mr-&gt;mmkey.key));</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_mr *mr = to_mmr(ibmr);</div><div class='add'>+	struct nlattr *table_attr;</div><div class='add'>+</div><div class='add'>+	if (!(mr-&gt;access_flags &amp; IB_ACCESS_ON_DEMAND))</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);</div><div class='add'>+	if (!table_attr)</div><div class='add'>+		goto err;</div><div class='add'>+</div><div class='add'>+	if (mr-&gt;is_odp_implicit) {</div><div class='add'>+		if (rdma_nl_put_driver_string(msg, "odp", "implicit"))</div><div class='add'>+			goto err;</div><div class='add'>+	} else {</div><div class='add'>+		if (rdma_nl_put_driver_string(msg, "odp", "explicit"))</div><div class='add'>+			goto err;</div><div class='add'>+	}</div><div class='add'>+</div><div class='add'>+	nla_nest_end(msg, table_attr);</div><div class='add'>+	return 0;</div><div class='add'>+</div><div class='add'>+err:</div><div class='add'>+	nla_nest_cancel(msg, table_attr);</div><div class='add'>+	return -EMSGSIZE;</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ibcq)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ibcq-&gt;device);</div><div class='add'>+	struct mlx5_ib_cq *cq = to_mcq(ibcq);</div><div class='add'>+</div><div class='add'>+	return fill_res_raw(msg, dev, MLX5_SGMT_TYPE_PRM_QUERY_CQ, cq-&gt;mcq.cqn);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_qp *qp = to_mqp(ibqp);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	if (qp-&gt;type &lt; IB_QPT_DRIVER)</div><div class='add'>+		return 0;</div><div class='add'>+</div><div class='add'>+	switch (qp-&gt;type) {</div><div class='add'>+	case MLX5_IB_QPT_REG_UMR:</div><div class='add'>+		ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE,</div><div class='add'>+				     "REG_UMR");</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_QPT_DCT:</div><div class='add'>+		ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "DCT");</div><div class='add'>+		break;</div><div class='add'>+	case MLX5_IB_QPT_DCI:</div><div class='add'>+		ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "DCI");</div><div class='add'>+		break;</div><div class='add'>+	default:</div><div class='add'>+		return 0;</div><div class='add'>+	}</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='add'>+</div><div class='add'>+	return nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, IB_QPT_DRIVER);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static int fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ibqp)</div><div class='add'>+{</div><div class='add'>+	struct mlx5_ib_dev *dev = to_mdev(ibqp-&gt;device);</div><div class='add'>+</div><div class='add'>+	return fill_res_raw(msg, dev, MLX5_SGMT_TYPE_PRM_QUERY_QP,</div><div class='add'>+			    ibqp-&gt;qp_num);</div><div class='add'>+}</div><div class='add'>+</div><div class='add'>+static const struct ib_device_ops restrack_ops = {</div><div class='add'>+	.fill_res_cq_entry_raw = fill_res_cq_entry_raw,</div><div class='add'>+	.fill_res_mr_entry = fill_res_mr_entry,</div><div class='add'>+	.fill_res_mr_entry_raw = fill_res_mr_entry_raw,</div><div class='add'>+	.fill_res_qp_entry = fill_res_qp_entry,</div><div class='add'>+	.fill_res_qp_entry_raw = fill_res_qp_entry_raw,</div><div class='add'>+	.fill_stat_mr_entry = fill_stat_mr_entry,</div><div class='add'>+};</div><div class='add'>+</div><div class='add'>+int mlx5_ib_restrack_init(struct mlx5_ib_dev *dev)</div><div class='add'>+{</div><div class='add'>+	ib_set_device_ops(&amp;dev-&gt;ib_dev, &amp;restrack_ops);</div><div class='add'>+	return 0;</div><div class='add'>+}</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/restrack.h b/drivers/infiniband/hw/mlx5/restrack.h<br/>new file mode 100644<br/>index 000000000000..e8d81270f1b6<br/>--- /dev/null<br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/restrack.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/restrack.h</a></div><div class='hunk'>@@ -0,0 +1,13 @@</div><div class='add'>+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */</div><div class='add'>+/*</div><div class='add'>+ * Copyright (c) 2013-2020, Mellanox Technologies Ltd. All rights reserved.</div><div class='add'>+ */</div><div class='add'>+</div><div class='add'>+#ifndef _MLX5_IB_RESTRACK_H</div><div class='add'>+#define _MLX5_IB_RESTRACK_H</div><div class='add'>+</div><div class='add'>+#include "mlx5_ib.h"</div><div class='add'>+</div><div class='add'>+int mlx5_ib_restrack_init(struct mlx5_ib_dev *dev);</div><div class='add'>+</div><div class='add'>+#endif /* _MLX5_IB_RESTRACK_H */</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c<br/>index 4e7fde86c96b..bcb6b324af50 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/srq.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/srq.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/srq.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/srq.c</a></div><div class='hunk'>@@ -3,7 +3,6 @@</div><div class='ctx'>  * Copyright (c) 2013-2018, Mellanox Technologies inc.  All rights reserved.</div><div class='ctx'>  */</div><div class='ctx'> </div><div class='del'>-#include &lt;linux/module.h&gt;</div><div class='ctx'> #include &lt;linux/mlx5/qp.h&gt;</div><div class='ctx'> #include &lt;linux/slab.h&gt;</div><div class='ctx'> #include &lt;rdma/ib_umem.h&gt;</div><div class='hunk'>@@ -51,10 +50,6 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,</div><div class='ctx'> 		udata, struct mlx5_ib_ucontext, ibucontext);</div><div class='ctx'> 	size_t ucmdlen;</div><div class='ctx'> 	int err;</div><div class='del'>-	int npages;</div><div class='del'>-	int page_shift;</div><div class='del'>-	int ncont;</div><div class='del'>-	u32 offset;</div><div class='ctx'> 	u32 uidx = MLX5_IB_DEFAULT_UIDX;</div><div class='ctx'> </div><div class='ctx'> 	ucmdlen = min(udata-&gt;inlen, sizeof(ucmd));</div><div class='hunk'>@@ -80,38 +75,20 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,</div><div class='ctx'> </div><div class='ctx'> 	srq-&gt;wq_sig = !!(ucmd.flags &amp; MLX5_SRQ_FLAG_SIGNATURE);</div><div class='ctx'> </div><div class='del'>-	srq-&gt;umem = ib_umem_get(udata, ucmd.buf_addr, buf_size, 0, 0);</div><div class='add'>+	srq-&gt;umem = ib_umem_get(pd-&gt;device, ucmd.buf_addr, buf_size, 0);</div><div class='ctx'> 	if (IS_ERR(srq-&gt;umem)) {</div><div class='ctx'> 		mlx5_ib_dbg(dev, "failed umem get, size %d\n", buf_size);</div><div class='ctx'> 		err = PTR_ERR(srq-&gt;umem);</div><div class='ctx'> 		return err;</div><div class='ctx'> 	}</div><div class='add'>+	in-&gt;umem = srq-&gt;umem;</div><div class='ctx'> </div><div class='del'>-	mlx5_ib_cont_pages(srq-&gt;umem, ucmd.buf_addr, 0, &amp;npages,</div><div class='del'>-			   &amp;page_shift, &amp;ncont, NULL);</div><div class='del'>-	err = mlx5_ib_get_buf_offset(ucmd.buf_addr, page_shift,</div><div class='del'>-				     &amp;offset);</div><div class='del'>-	if (err) {</div><div class='del'>-		mlx5_ib_warn(dev, "bad offset\n");</div><div class='del'>-		goto err_umem;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	in-&gt;pas = kvcalloc(ncont, sizeof(*in-&gt;pas), GFP_KERNEL);</div><div class='del'>-	if (!in-&gt;pas) {</div><div class='del'>-		err = -ENOMEM;</div><div class='del'>-		goto err_umem;</div><div class='del'>-	}</div><div class='del'>-</div><div class='del'>-	mlx5_ib_populate_pas(dev, srq-&gt;umem, page_shift, in-&gt;pas, 0);</div><div class='del'>-</div><div class='del'>-	err = mlx5_ib_db_map_user(ucontext, udata, ucmd.db_addr, &amp;srq-&gt;db);</div><div class='add'>+	err = mlx5_ib_db_map_user(ucontext, ucmd.db_addr, &amp;srq-&gt;db);</div><div class='ctx'> 	if (err) {</div><div class='ctx'> 		mlx5_ib_dbg(dev, "map doorbell failed\n");</div><div class='del'>-		goto err_in;</div><div class='add'>+		goto err_umem;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='del'>-	in-&gt;log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT;</div><div class='del'>-	in-&gt;page_offset = offset;</div><div class='ctx'> 	in-&gt;uid = (in-&gt;type != IB_SRQT_XRC) ?  to_mpd(pd)-&gt;uid : 0;</div><div class='ctx'> 	if (MLX5_CAP_GEN(dev-&gt;mdev, cqe_version) == MLX5_CQE_VERSION_V1 &amp;&amp;</div><div class='ctx'> 	    in-&gt;type != IB_SRQT_BASIC)</div><div class='hunk'>@@ -119,9 +96,6 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,</div><div class='ctx'> </div><div class='ctx'> 	return 0;</div><div class='ctx'> </div><div class='del'>-err_in:</div><div class='del'>-	kvfree(in-&gt;pas);</div><div class='del'>-</div><div class='ctx'> err_umem:</div><div class='ctx'> 	ib_umem_release(srq-&gt;umem);</div><div class='ctx'> </div><div class='hunk'>@@ -225,15 +199,27 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq,</div><div class='ctx'> 	int err;</div><div class='ctx'> 	struct mlx5_srq_attr in = {};</div><div class='ctx'> 	__u32 max_srq_wqes = 1 &lt;&lt; MLX5_CAP_GEN(dev-&gt;mdev, log_max_srq_sz);</div><div class='del'>-</div><div class='del'>-	/* Sanity check SRQ size before proceeding */</div><div class='del'>-	if (init_attr-&gt;attr.max_wr &gt;= max_srq_wqes) {</div><div class='del'>-		mlx5_ib_dbg(dev, "max_wr %d, cap %d\n",</div><div class='del'>-			    init_attr-&gt;attr.max_wr,</div><div class='del'>-			    max_srq_wqes);</div><div class='add'>+	__u32 max_sge_sz =  MLX5_CAP_GEN(dev-&gt;mdev, max_wqe_sz_rq) /</div><div class='add'>+			    sizeof(struct mlx5_wqe_data_seg);</div><div class='add'>+</div><div class='add'>+	if (init_attr-&gt;srq_type != IB_SRQT_BASIC &amp;&amp;</div><div class='add'>+	    init_attr-&gt;srq_type != IB_SRQT_XRC &amp;&amp;</div><div class='add'>+	    init_attr-&gt;srq_type != IB_SRQT_TM)</div><div class='add'>+		return -EOPNOTSUPP;</div><div class='add'>+</div><div class='add'>+	/* Sanity check SRQ and sge size before proceeding */</div><div class='add'>+	if (init_attr-&gt;attr.max_wr &gt;= max_srq_wqes ||</div><div class='add'>+	    init_attr-&gt;attr.max_sge &gt; max_sge_sz) {</div><div class='add'>+		mlx5_ib_dbg(dev, "max_wr %d,wr_cap %d,max_sge %d, sge_cap:%d\n",</div><div class='add'>+			    init_attr-&gt;attr.max_wr, max_srq_wqes,</div><div class='add'>+			    init_attr-&gt;attr.max_sge, max_sge_sz);</div><div class='ctx'> 		return -EINVAL;</div><div class='ctx'> 	}</div><div class='ctx'> </div><div class='add'>+	err = mlx5_ib_dev_res_cq_init(dev);</div><div class='add'>+	if (err)</div><div class='add'>+		return err;</div><div class='add'>+</div><div class='ctx'> 	mutex_init(&amp;srq-&gt;mutex);</div><div class='ctx'> 	spin_lock_init(&amp;srq-&gt;lock);</div><div class='ctx'> 	srq-&gt;msrq.max    = roundup_pow_of_two(init_attr-&gt;attr.max_wr + 1);</div><div class='hunk'>@@ -274,10 +260,10 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq,</div><div class='ctx'> 	if (srq-&gt;wq_sig)</div><div class='ctx'> 		in.flags |= MLX5_SRQ_FLAG_WQ_SIG;</div><div class='ctx'> </div><div class='del'>-	if (init_attr-&gt;srq_type == IB_SRQT_XRC)</div><div class='add'>+	if (init_attr-&gt;srq_type == IB_SRQT_XRC &amp;&amp; init_attr-&gt;ext.xrc.xrcd)</div><div class='ctx'> 		in.xrcd = to_mxrcd(init_attr-&gt;ext.xrc.xrcd)-&gt;xrcdn;</div><div class='ctx'> 	else</div><div class='del'>-		in.xrcd = to_mxrcd(dev-&gt;devr.x0)-&gt;xrcdn;</div><div class='add'>+		in.xrcd = dev-&gt;devr.xrcdn0;</div><div class='ctx'> </div><div class='ctx'> 	if (init_attr-&gt;srq_type == IB_SRQT_TM) {</div><div class='ctx'> 		in.tm_log_list_size =</div><div class='hunk'>@@ -310,12 +296,18 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq,</div><div class='ctx'> 	srq-&gt;msrq.event = mlx5_ib_srq_event;</div><div class='ctx'> 	srq-&gt;ibsrq.ext.xrc.srq_num = srq-&gt;msrq.srqn;</div><div class='ctx'> </div><div class='del'>-	if (udata)</div><div class='del'>-		if (ib_copy_to_udata(udata, &amp;srq-&gt;msrq.srqn, sizeof(__u32))) {</div><div class='add'>+	if (udata) {</div><div class='add'>+		struct mlx5_ib_create_srq_resp resp = {</div><div class='add'>+			.srqn = srq-&gt;msrq.srqn,</div><div class='add'>+		};</div><div class='add'>+</div><div class='add'>+		if (ib_copy_to_udata(udata, &amp;resp, min(udata-&gt;outlen,</div><div class='add'>+				     sizeof(resp)))) {</div><div class='ctx'> 			mlx5_ib_dbg(dev, "copy to user failed\n");</div><div class='ctx'> 			err = -EFAULT;</div><div class='ctx'> 			goto err_core;</div><div class='ctx'> 		}</div><div class='add'>+	}</div><div class='ctx'> </div><div class='ctx'> 	init_attr-&gt;attr.max_wr = srq-&gt;msrq.max - 1;</div><div class='ctx'> </div><div class='hunk'>@@ -383,24 +375,21 @@ out_box:</div><div class='ctx'> 	return ret;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='del'>-void mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)</div><div class='add'>+int mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)</div><div class='ctx'> {</div><div class='ctx'> 	struct mlx5_ib_dev *dev = to_mdev(srq-&gt;device);</div><div class='ctx'> 	struct mlx5_ib_srq *msrq = to_msrq(srq);</div><div class='add'>+	int ret;</div><div class='add'>+</div><div class='add'>+	ret = mlx5_cmd_destroy_srq(dev, &amp;msrq-&gt;msrq);</div><div class='add'>+	if (ret)</div><div class='add'>+		return ret;</div><div class='ctx'> </div><div class='del'>-	mlx5_cmd_destroy_srq(dev, &amp;msrq-&gt;msrq);</div><div class='del'>-</div><div class='del'>-	if (srq-&gt;uobject) {</div><div class='del'>-		mlx5_ib_db_unmap_user(</div><div class='del'>-			rdma_udata_to_drv_context(</div><div class='del'>-				udata,</div><div class='del'>-				struct mlx5_ib_ucontext,</div><div class='del'>-				ibucontext),</div><div class='del'>-			&amp;msrq-&gt;db);</div><div class='del'>-		ib_umem_release(msrq-&gt;umem);</div><div class='del'>-	} else {</div><div class='add'>+	if (udata)</div><div class='add'>+		destroy_srq_user(srq-&gt;pd, msrq, udata);</div><div class='add'>+	else</div><div class='ctx'> 		destroy_srq_kernel(dev, msrq);</div><div class='del'>-	}</div><div class='add'>+	return 0;</div><div class='ctx'> }</div><div class='ctx'> </div><div class='ctx'> void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index)</div><div class='hunk'>@@ -465,7 +454,7 @@ int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,</div><div class='ctx'> </div><div class='ctx'> 		if (i &lt; srq-&gt;msrq.max_avail_gather) {</div><div class='ctx'> 			scat[i].byte_count = 0;</div><div class='del'>-			scat[i].lkey       = cpu_to_be32(MLX5_INVALID_LKEY);</div><div class='add'>+			scat[i].lkey = dev-&gt;mkeys.terminate_scatter_list_mkey;</div><div class='ctx'> 			scat[i].addr       = 0;</div><div class='ctx'> 		}</div><div class='ctx'> 	}</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/srq.h b/drivers/infiniband/hw/mlx5/srq.h<br/>index af197c36d757..a7e3dc5564ac 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/srq.h?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/srq.h</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/srq.h?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/srq.h</a></div><div class='hunk'>@@ -28,6 +28,7 @@ struct mlx5_srq_attr {</div><div class='ctx'> 	u32 user_index;</div><div class='ctx'> 	u64 db_record;</div><div class='ctx'> 	__be64 *pas;</div><div class='add'>+	struct ib_umem *umem;</div><div class='ctx'> 	u32 tm_log_list_size;</div><div class='ctx'> 	u32 tm_next_tag;</div><div class='ctx'> 	u32 tm_hw_phase_cnt;</div><div class='hunk'>@@ -56,7 +57,7 @@ struct mlx5_srq_table {</div><div class='ctx'> </div><div class='ctx'> int mlx5_cmd_create_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,</div><div class='ctx'> 			struct mlx5_srq_attr *in);</div><div class='del'>-void mlx5_cmd_destroy_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq);</div><div class='add'>+int mlx5_cmd_destroy_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq);</div><div class='ctx'> int mlx5_cmd_query_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,</div><div class='ctx'> 		       struct mlx5_srq_attr *out);</div><div class='ctx'> int mlx5_cmd_arm_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,</div><div class='head'>diff --git a/drivers/infiniband/hw/mlx5/srq_cmd.c b/drivers/infiniband/hw/mlx5/srq_cmd.c<br/>index b0d0687c7a68..8b3385396599 100644<br/>--- a/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/srq_cmd.c?id=84b0d6a7a11ec976da347d50de9d5a556743de16'>drivers/infiniband/hw/mlx5/srq_cmd.c</a><br/>+++ b/<a href='/cgit/linux-arm.git/tree/drivers/infiniband/hw/mlx5/srq_cmd.c?id=c9b47175e9131118e6f221cc8fb81397d62e7c91'>drivers/infiniband/hw/mlx5/srq_cmd.c</a></div>